From 1cea80d29f4f1c61ed56ad1261b74ed42611bf64 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Fri, 6 Apr 2018 13:49:10 +0300
Subject: [PATCH 001/543] IGNITE-8018 Optimized GridCacheMapEntry
 initialValue() - Fixes #3686.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../processors/cache/GridCacheMapEntry.java   | 158 ++++++++++++++----
 .../colocated/GridDhtDetachedCacheEntry.java  |   3 +-
 .../distributed/near/GridNearCacheEntry.java  |   3 +-
 3 files changed, 131 insertions(+), 33 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 74dabe9a1f7dd..a6ef0d284d6b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -77,6 +77,7 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_EXPIRED;
@@ -2699,40 +2700,80 @@ protected final boolean hasValueUnlocked() {
     ) throws IgniteCheckedException, GridCacheEntryRemovedException {
         ensureFreeSpace();
 
+        boolean deferred = false;
+        boolean obsolete = false;
+
+        GridCacheVersion oldVer = null;
+
         lockEntry();
 
         try {
             checkObsolete();
 
+            boolean walEnabled = !cctx.isNear() && cctx.group().persistenceEnabled() && cctx.group().walEnabled();
+
+            long expTime = expireTime < 0 ? CU.toExpireTime(ttl) : expireTime;
+
+            val = cctx.kernalContext().cacheObjects().prepareForCache(val, cctx);
+
+            final boolean unswapped = ((flags & IS_UNSWAPPED_MASK) != 0);
+
             boolean update;
 
-            boolean walEnabled = !cctx.isNear() && cctx.group().persistenceEnabled() && cctx.group().walEnabled();
+            IgnitePredicate<CacheDataRow> p = new IgnitePredicate<CacheDataRow>() {
+                @Override public boolean apply(@Nullable CacheDataRow row) {
+                    boolean update0;
+
+                    GridCacheVersion currentVer = row != null ? row.version() : GridCacheMapEntry.this.ver;
 
-            if (cctx.group().persistenceEnabled()) {
-                unswap(false);
+                    boolean isStartVer = currentVer.nodeOrder() == cctx.localNode().order()
+                        && currentVer.order() == startVer;
 
-                if (!isNew()) {
-                    if (cctx.atomic())
-                        update = ATOMIC_VER_COMPARATOR.compare(this.ver, ver) < 0;
+                    if (cctx.group().persistenceEnabled()) {
+                        if (!isStartVer) {
+                            if (cctx.atomic())
+                                update0 = ATOMIC_VER_COMPARATOR.compare(currentVer, ver) < 0;
+                            else
+                                update0 = currentVer.compareTo(ver) < 0;
+                        }
+                        else
+                            update0 = true;
+                    }
                     else
-                        update = this.ver.compareTo(ver) < 0;
+                        update0 = isStartVer;
+
+                    update0 |= (!preload && deletedUnlocked());
+
+                    return update0;
                 }
-                else
-                    update = true;
-            }
-            else
-                update = isNew() && !cctx.offheap().containsKey(this);
+            };
 
-            update |= !preload && deletedUnlocked();
+            if (unswapped) {
+                update = p.apply(null);
 
-            if (update) {
-                long expTime = expireTime < 0 ? CU.toExpireTime(ttl) : expireTime;
+                if (update) {
+                    // If entry is already unswapped and we are modifying it, we must run deletion callbacks for old value.
+                    long oldExpTime = expireTimeUnlocked();
+                    long delta = (oldExpTime == 0 ? 0 : oldExpTime - U.currentTimeMillis());
 
-                val = cctx.kernalContext().cacheObjects().prepareForCache(val, cctx);
+                    if (delta < 0) {
+                        if (onExpired(this.val, null)) {
+                            if (cctx.deferredDelete()) {
+                                deferred = true;
+                                oldVer = this.ver;
+                            }
+                            else if (val == null)
+                                obsolete = true;
+                        }
+                    }
 
-                if (val != null)
                     storeValue(val, expTime, ver, null);
+                }
+            }
+            else // Optimization to access storage only once.
+                update = storeValue(val, expTime, ver, null, p);
 
+            if (update) {
                 update(val, expTime, ttl, ver, true);
 
                 boolean skipQryNtf = false;
@@ -2797,6 +2838,20 @@ else if (deletedUnlocked())
         }
         finally {
             unlockEntry();
+
+            // It is necessary to execute these callbacks outside of lock to avoid deadlocks.
+
+            if (obsolete) {
+                onMarkedObsolete();
+
+                cctx.cache().removeEntry(this);
+            }
+
+            if (deferred) {
+                assert oldVer != null;
+
+                cctx.onDeferredDelete(this, oldVer);
+            }
         }
     }
 
@@ -3516,14 +3571,39 @@ private IgniteTxLocalAdapter currentTx() {
      * @param oldRow Old row if available.
      * @throws IgniteCheckedException If update failed.
      */
-    protected void storeValue(CacheObject val,
+    protected boolean storeValue(CacheObject val,
         long expireTime,
         GridCacheVersion ver,
         @Nullable CacheDataRow oldRow) throws IgniteCheckedException {
-        assert lock.isHeldByCurrentThread();
         assert val != null : "null values in update for key: " + key;
 
-        cctx.offheap().invoke(cctx, key,  localPartition(), new UpdateClosure(this, val, ver, expireTime));
+        return storeValue(val, expireTime, ver, oldRow, null);
+    }
+
+    /**
+     * Stores value in offheap.
+     *
+     * @param val Value.
+     * @param expireTime Expire time.
+     * @param ver New entry version.
+     * @param oldRow Old row if available.
+     * @param predicate Optional predicate.
+     * @throws IgniteCheckedException If update failed.
+     * @return {@code True} if storage was modified.
+     */
+    protected boolean storeValue(
+        @Nullable CacheObject val,
+        long expireTime,
+        GridCacheVersion ver,
+        @Nullable CacheDataRow oldRow,
+        @Nullable IgnitePredicate<CacheDataRow> predicate) throws IgniteCheckedException {
+        assert lock.isHeldByCurrentThread();
+
+        UpdateClosure closure = new UpdateClosure(this, val, ver, expireTime, predicate);
+
+        cctx.offheap().invoke(cctx, key,  localPartition(), closure);
+
+        return closure.treeOp != IgniteTree.OperationType.NOOP;
     }
 
     /**
@@ -4295,7 +4375,7 @@ private static class UpdateClosure implements IgniteCacheOffheapManager.OffheapI
         private final GridCacheMapEntry entry;
 
         /** */
-        private final CacheObject val;
+        @Nullable private final CacheObject val;
 
         /** */
         private final GridCacheVersion ver;
@@ -4303,6 +4383,9 @@ private static class UpdateClosure implements IgniteCacheOffheapManager.OffheapI
         /** */
         private final long expireTime;
 
+        /** */
+        @Nullable private final IgnitePredicate<CacheDataRow> predicate;
+
         /** */
         private CacheDataRow newRow;
 
@@ -4317,31 +4400,44 @@ private static class UpdateClosure implements IgniteCacheOffheapManager.OffheapI
          * @param val New value.
          * @param ver New version.
          * @param expireTime New expire time.
+         * @param predicate Optional predicate.
          */
-        UpdateClosure(GridCacheMapEntry entry, CacheObject val, GridCacheVersion ver, long expireTime) {
+        UpdateClosure(GridCacheMapEntry entry, @Nullable CacheObject val, GridCacheVersion ver, long expireTime,
+            @Nullable IgnitePredicate<CacheDataRow> predicate) {
             this.entry = entry;
             this.val = val;
             this.ver = ver;
             this.expireTime = expireTime;
+            this.predicate = predicate;
         }
 
         /** {@inheritDoc} */
         @Override public void call(@Nullable CacheDataRow oldRow) throws IgniteCheckedException {
             this.oldRow = oldRow;
 
+            if (predicate != null && !predicate.apply(oldRow)) {
+                treeOp = IgniteTree.OperationType.NOOP;
+
+                return;
+            }
+
             if (oldRow != null)
                 oldRow.key(entry.key);
 
-            newRow = entry.cctx.offheap().dataStore(entry.localPartition()).createRow(
-                entry.cctx,
-                entry.key,
-                val,
-                ver,
-                expireTime,
-                oldRow);
+            if (val != null) {
+                newRow = entry.cctx.offheap().dataStore(entry.localPartition()).createRow(
+                    entry.cctx,
+                    entry.key,
+                    val,
+                    ver,
+                    expireTime,
+                    oldRow);
 
-            treeOp = oldRow != null && oldRow.link() == newRow.link() ?
-                IgniteTree.OperationType.NOOP : IgniteTree.OperationType.PUT;
+                treeOp = oldRow != null && oldRow.link() == newRow.link() ?
+                    IgniteTree.OperationType.NOOP : IgniteTree.OperationType.PUT;
+            }
+            else
+                treeOp = oldRow != null ? IgniteTree.OperationType.REMOVE : IgniteTree.OperationType.NOOP;
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
index 3536908823d2d..d02015b665489 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
@@ -65,10 +65,11 @@ public void resetFromPrimary(CacheObject val, GridCacheVersion ver) {
     }
 
     /** {@inheritDoc} */
-    @Override protected void storeValue(CacheObject val,
+    @Override protected boolean storeValue(CacheObject val,
         long expireTime,
         GridCacheVersion ver,
         CacheDataRow oldRow) throws IgniteCheckedException {
+        return false;
         // No-op for detached entries, index is updated on primary nodes.
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index 322e63ce5f7c8..fb41f5c2e306f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -458,7 +458,8 @@ public boolean loadedValue(@Nullable IgniteInternalTx tx,
     }
 
     /** {@inheritDoc} */
-    @Override protected void storeValue(CacheObject val, long expireTime, GridCacheVersion ver, CacheDataRow oldRow) {
+    @Override protected boolean storeValue(CacheObject val, long expireTime, GridCacheVersion ver, CacheDataRow oldRow) {
+        return false;
         // No-op: queries are disabled for near cache.
     }
 

From 37fc72542eb6baa8be8b41aecd08a194102d13c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9=20=D0=A1=D1=82?=
 =?UTF-8?q?=D0=B5=D0=BB=D1=8C=D0=BC=D0=B0=D0=BA?=
 <astelmak@macbook-pro-aleksej.local>
Date: Fri, 6 Apr 2018 18:28:22 +0300
Subject: [PATCH 002/543] IGNITE-8049 Limit the number of operation cycles in
 B+Tree - Fixes #3769.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit e491f10)
---
 .../apache/ignite/IgniteSystemProperties.java |  5 +++
 .../cache/persistence/tree/BPlusTree.java     | 41 ++++++++++++++++++-
 .../database/BPlusTreeSelfTest.java           | 29 +++++++++++++
 3 files changed, 74 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index f7128c0ed1fc9..152d845afbac5 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -844,6 +844,11 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_READ_LOAD_BALANCING = "IGNITE_READ_LOAD_BALANCING";
 
+    /**
+     * Number of repetitions to capture a lock in the B+Tree.
+     */
+    public static final String IGNITE_BPLUS_TREE_LOCK_RETRIES = "IGNITE_BPLUS_TREE_LOCK_RETRIES";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
index a520dce829c04..4d050952b4be6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -27,6 +27,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
@@ -86,6 +87,13 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
     /** */
     private static volatile boolean interrupted;
 
+    /** */
+    private static final int IGNITE_BPLUS_TREE_LOCK_RETRIES_DEFAULT = 1000;
+
+    /** */
+    private static final int LOCK_RETRIES = IgniteSystemProperties.getInteger(
+        IgniteSystemProperties.IGNITE_BPLUS_TREE_LOCK_RETRIES, IGNITE_BPLUS_TREE_LOCK_RETRIES_DEFAULT);
+
     /** */
     private final AtomicBoolean destroyed = new AtomicBoolean(false);
 
@@ -1123,6 +1131,8 @@ private Result findDown(final Get g, final long pageId, final long fwdId, final
 
         try {
             for (;;) {
+                g.checkLockRetry();
+
                 // Init args.
                 g.pageId = pageId;
                 g.fwdId = fwdId;
@@ -1657,13 +1667,18 @@ private Result invokeDown(final Invoke x, final long pageId, final long backId,
         long page = acquirePage(pageId);
 
         try {
+            Result res = RETRY;
+
             for (;;) {
+                if (res == RETRY)
+                    x.checkLockRetry();
+
                 // Init args.
                 x.pageId(pageId);
                 x.fwdId(fwdId);
                 x.backId(backId);
 
-                Result res = read(pageId, page, search, x, lvl, RETRY);
+                res = read(pageId, page, search, x, lvl, RETRY);
 
                 switch (res) {
                     case GO_DOWN_X:
@@ -1813,6 +1828,8 @@ private Result removeDown(final Remove r, final long pageId, final long backId,
 
         try {
             for (;;) {
+                r.checkLockRetry();
+
                 // Init args.
                 r.pageId = pageId;
                 r.fwdId = fwdId;
@@ -2310,6 +2327,8 @@ private Result putDown(final Put p, final long pageId, final long fwdId, final i
 
         try {
             for (;;) {
+                p.checkLockRetry();
+
                 // Init args.
                 p.pageId = pageId;
                 p.fwdId = fwdId;
@@ -2422,6 +2441,9 @@ private abstract class Get {
         /** Ignore row passed, find last row */
         boolean findLast;
 
+        /** Number of repetitions to capture a lock in the B+Tree (countdown). */
+        int lockRetriesCnt = getLockRetries();
+
         /**
          * @param row Row.
          * @param findLast find last row.
@@ -2534,6 +2556,16 @@ void fwdId(long fwdId) {
         boolean isFinished() {
             throw new IllegalStateException();
         }
+
+        /**
+         * @throws IgniteCheckedException If the operation can not be retried.
+         */
+        final void checkLockRetry() throws IgniteCheckedException {
+            if (lockRetriesCnt == 0)
+                throw new IgniteCheckedException("Maximum of retries " + getLockRetries() + " reached.");
+
+            lockRetriesCnt--;
+        }
     }
 
     /**
@@ -4911,4 +4943,11 @@ public interface TreeRowClosure<L, T extends L> {
         public boolean apply(BPlusTree<L, T> tree, BPlusIO<L> io, long pageAddr, int idx)
             throws IgniteCheckedException;
     }
+
+    /**
+     * @return Return number of retries.
+     */
+    protected int getLockRetries() {
+        return LOCK_RETRIES;
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index 72d2e0ff92d6c..83d0ddd278a79 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -227,6 +227,26 @@ public void testFind() throws IgniteCheckedException {
         checkCursor(tree.find(10L, 70L), map.subMap(10L, true, 70L, true).values().iterator());
     }
 
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    public void testRetries() throws IgniteCheckedException {
+        TestTree tree = createTestTree(true);
+
+        tree.numRetries = 1;
+
+        long size = CNT * CNT;
+
+        try {
+            for (long i = 1; i <= size; i++)
+                tree.put(i);
+
+            fail();
+        }
+        catch (IgniteCheckedException ignored) {
+        }
+    }
+
     /**
      * @throws IgniteCheckedException If failed.
      */
@@ -2349,6 +2369,9 @@ protected static class TestTree extends BPlusTree<Long, Long> {
         /** */
         private static ConcurrentMap<Object, Map<Long, Long>> writeLocks = new ConcurrentHashMap<>();
 
+        /** Number of retries. */
+        private int numRetries = super.getLockRetries();
+
         /**
          * @param reuseList Reuse list.
          * @param canGetRow Can get row from inner page.
@@ -2536,6 +2559,12 @@ static String printLocks() {
 
             return b.toString();
         }
+
+        /** {@inheritDoc} */
+        @Override
+        protected int getLockRetries() {
+            return numRetries;
+        }
     }
 
     /**

From 76e293654e34c927d6c9efc85a12e736b58a21f2 Mon Sep 17 00:00:00 2001
From: Eduard Shangareev <eshangareev@gridgain.com>
Date: Fri, 6 Apr 2018 19:22:07 +0300
Subject: [PATCH 003/543] IGNITE-8114 Add fail recovery mechanism to tracking
 pages - Fixes #3734.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit 0829397)
---
 .../TrackingPageIsCorruptedException.java     |  60 +++++++
 .../cache/persistence/tree/io/PageMetaIO.java |   6 +-
 .../persistence/tree/io/TrackingPageIO.java   | 156 +++++++++++++++---
 .../tree/io/TrackingPageIOTest.java           | 116 ++++++++++---
 4 files changed, 289 insertions(+), 49 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/TrackingPageIsCorruptedException.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/TrackingPageIsCorruptedException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/TrackingPageIsCorruptedException.java
new file mode 100644
index 0000000000000..2b82bff27f95e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/TrackingPageIsCorruptedException.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.snapshot;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Thrown when corrupted tracking page was queried.
+ */
+public class TrackingPageIsCorruptedException extends IgniteCheckedException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Instance. */
+    public static final TrackingPageIsCorruptedException INSTANCE = new TrackingPageIsCorruptedException(-1, -1);
+
+    /** Last tag. */
+    private final long lastTag;
+
+    /** Passed tag. */
+    private final long passedTag;
+
+    /**
+     * @param lastTag Last tag.
+     * @param passedTag Passed tag.
+     */
+    public TrackingPageIsCorruptedException(long lastTag, long passedTag) {
+        this.lastTag = lastTag;
+        this.passedTag = passedTag;
+    }
+
+    /**
+     * @return Last tag.
+     */
+    public long lastTag() {
+        return lastTag;
+    }
+
+    /**
+     * @return Passed tag.
+     */
+    public long passedTag() {
+        return passedTag;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
index d2921eec3f776..d676cfd3f6805 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
@@ -152,10 +152,10 @@ public long getLastSuccessfulFullSnapshotId(long pageAddr) {
 
     /**
      * @param pageAddr Page address.
-     * @param nextSnapshotId Next snapshot id.
+     * @param nextSnapshotTag Next snapshot tag.
      */
-    public void setNextSnapshotTag(long pageAddr, long nextSnapshotId) {
-        PageUtils.putLong(pageAddr, NEXT_SNAPSHOT_TAG_OFF, nextSnapshotId);
+    public void setNextSnapshotTag(long pageAddr, long nextSnapshotTag) {
+        PageUtils.putLong(pageAddr, NEXT_SNAPSHOT_TAG_OFF, nextSnapshotTag);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
index 1bd70f8f5ec66..94885e4889fc8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
@@ -20,6 +20,7 @@
 import java.nio.ByteBuffer;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.TrackingPageIsCorruptedException;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
 import org.apache.ignite.internal.util.GridStringBuilder;
 import org.apache.ignite.internal.util.GridUnsafe;
@@ -50,6 +51,12 @@ public class TrackingPageIO extends PageIO {
         new TrackingPageIO(1)
     );
 
+    /** Corrupt flag mask. */
+    public static final long CORRUPT_FLAG_MASK = 1L << 63;
+
+    /** Corrupt flag mask. */
+    public static final long CORRUPT_FLAG_FILTER_MASK = ~CORRUPT_FLAG_MASK;
+
     /** Last snapshot offset. */
     public static final int LAST_SNAPSHOT_TAG_OFFSET = COMMON_HEADER_END;
 
@@ -77,11 +84,11 @@ protected TrackingPageIO(int ver) {
      *
      * @param buf Buffer.
      * @param pageId Page id.
-     * @param nextSnapshotTag tag of next snapshot.
+     * @param nextSnapshotTag Tag of next snapshot.
      * @param pageSize Page size.
      */
-    public boolean markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
-        validateSnapshotId(buf, nextSnapshotTag, lastSuccessfulSnapshotTag, pageSize);
+    public void markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
+        validateSnapshotTag(buf, nextSnapshotTag, lastSuccessfulSnapshotTag, pageSize);
 
         int cntOfPage = countOfPageToTrack(pageSize);
 
@@ -98,7 +105,7 @@ public boolean markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, lo
         byte newVal =  (byte) (byteToUpdate | updateTemplate);
 
         if (byteToUpdate == newVal)
-            return false;
+            return;
 
         buf.put(idx, newVal);
 
@@ -107,8 +114,6 @@ public boolean markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, lo
         buf.putShort(sizeOff, newSize);
 
         assert newSize == countOfChangedPage(buf, nextSnapshotTag, pageSize);
-
-        return true;
     }
 
     /**
@@ -116,22 +121,30 @@ public boolean markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, lo
      * @param nextSnapshotTag Next snapshot id.
      * @param lastSuccessfulSnapshotTag Last successful snapshot id.
      * @param pageSize Page size.
+     *
+     * @return <code>-1</code> if everything is ok, otherwise last saved tag.
      */
-    private void validateSnapshotId(ByteBuffer buf, long nextSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
+    private long validateSnapshotTag(ByteBuffer buf, long nextSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
         assert nextSnapshotTag != lastSuccessfulSnapshotTag : "nextSnapshotTag = " + nextSnapshotTag +
             ", lastSuccessfulSnapshotId = " + lastSuccessfulSnapshotTag;
 
         long last = getLastSnapshotTag(buf);
 
-        assert last <= nextSnapshotTag : "last = " + last + ", nextSnapshotTag = " + nextSnapshotTag;
+        if(last > nextSnapshotTag) { //we have lost snapshot tag therefore should mark this tracking as corrupted
+            PageHandler.zeroMemory(buf, LAST_SNAPSHOT_TAG_OFFSET, buf.capacity() - LAST_SNAPSHOT_TAG_OFFSET);
+
+            setLastSnasphotTag(buf, nextSnapshotTag | CORRUPT_FLAG_MASK);
+
+            return last;
+        }
 
         if (nextSnapshotTag == last) //everything is ok
-            return;
+            return -1;
 
         int cntOfPage = countOfPageToTrack(pageSize);
 
         if (last <= lastSuccessfulSnapshotTag) { //we can drop our data
-            buf.putLong(LAST_SNAPSHOT_TAG_OFFSET, nextSnapshotTag);
+            setLastSnasphotTag(buf, nextSnapshotTag);
 
             PageHandler.zeroMemory(buf, SIZE_FIELD_OFFSET, buf.capacity() - SIZE_FIELD_OFFSET);
         } else { //we can't drop data, it is still necessary for incremental snapshots
@@ -167,37 +180,119 @@ private void validateSnapshotId(ByteBuffer buf, long nextSnapshotTag, long lastS
                 buf.putShort(sizeOff2, (short)newSize);
             }
 
-            buf.putLong(LAST_SNAPSHOT_TAG_OFFSET, nextSnapshotTag);
+            setLastSnasphotTag(buf, nextSnapshotTag);
 
             PageHandler.zeroMemory(buf, sizeOff, len + SIZE_FIELD_SIZE);
         }
+
+        return -1;
+    }
+
+    /**
+     * @param buf Buffer.
+     * @param nextSnapshotTag Next snapshot tag.
+     */
+    private void setLastSnasphotTag(ByteBuffer buf, long nextSnapshotTag) {
+        if (isCorrupted(buf))
+            nextSnapshotTag = nextSnapshotTag | CORRUPT_FLAG_MASK;
+
+        buf.putLong(LAST_SNAPSHOT_TAG_OFFSET, nextSnapshotTag);
     }
 
     /**
      * @param buf Buffer.
+     * @param nextSnapshotTag Next snapshot tag.
      */
-    long getLastSnapshotTag(ByteBuffer buf) {
-        return buf.getLong(LAST_SNAPSHOT_TAG_OFFSET);
+    private void setLastSnasphotTag0(ByteBuffer buf, long nextSnapshotTag) {
+        buf.putLong(LAST_SNAPSHOT_TAG_OFFSET, nextSnapshotTag);
     }
 
     /**
-     * @param addr Address.
+     * @param addr address.
+     * @param nextSnapshotTag Next snapshot tag.
      */
-    long getLastSnapshotTag(long addr) {
+    private void setLastSnasphotTag0(long addr, long nextSnapshotTag) {
+        GridUnsafe.putLong(addr + LAST_SNAPSHOT_TAG_OFFSET, nextSnapshotTag);
+    }
+
+    /**
+     * @param buf Buffer.
+     * @return Saved max seen snapshot tag.
+     */
+    private long getLastSnapshotTag(ByteBuffer buf) {
+        return getLastSnapshotTag0(buf) & CORRUPT_FLAG_FILTER_MASK;
+    }
+
+    /**
+     * @param buf Buffer.
+     * @return Saved value in {@link TrackingPageIO#LAST_SNAPSHOT_TAG_OFFSET}.
+     */
+    private long getLastSnapshotTag0(ByteBuffer buf) {
+        return buf.getLong(LAST_SNAPSHOT_TAG_OFFSET) ;
+    }
+
+    /**
+     * @param addr Address of buffer start.
+     * @return Saved max seen snapshot tag.
+     */
+    private long getLastSnapshotTag(long addr) {
+        return getLastSnapshotTag0(addr) & CORRUPT_FLAG_FILTER_MASK;
+    }
+
+    /**
+     * @param addr Address of buffer start.
+     * @return Saved value in {@link TrackingPageIO#LAST_SNAPSHOT_TAG_OFFSET}.
+     */
+    private long getLastSnapshotTag0(long addr) {
         return GridUnsafe.getLong(addr + LAST_SNAPSHOT_TAG_OFFSET);
     }
 
     /**
-     * Check that pageId was marked as changed between previous snapshot finish and current snapshot start.
+     * @param buf Buffer.
+     * @return Was tracking page marked as corrupted or not.
+     */
+    public boolean isCorrupted(ByteBuffer buf) {
+        return getLastSnapshotTag0(buf) < 0; //if true it means that first bit set to 1
+    }
+
+    /**
+     * Reset corrupted flag to false.
+     *
+     * @param buf Buffer.
+     */
+    public void resetCorruptFlag(ByteBuffer buf) {
+        setLastSnasphotTag0(buf, getLastSnapshotTag(buf));
+    }
+
+    /**
+     * Reset corrupted flag to false.
+     *
+     * @param addr Buffer.
+     */
+    public void resetCorruptFlag(long addr) {
+        setLastSnasphotTag0(addr, getLastSnapshotTag(addr));
+    }
+
+    /**
      *
      * @param buf Buffer.
      * @param pageId Page id.
      * @param curSnapshotTag Snapshot tag.
      * @param lastSuccessfulSnapshotTag Last successful snapshot id.
      * @param pageSize Page size.
+     *
+     * @return Was that pageId marked as changed between previous snapshot finish and current snapshot start or not.
+     * @throws TrackingPageIsCorruptedException if this tracking page was marked as corrupted.
      */
-    public boolean wasChanged(ByteBuffer buf, long pageId, long curSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
-        validateSnapshotId(buf, curSnapshotTag + 1, lastSuccessfulSnapshotTag, pageSize);
+    public boolean wasChanged(ByteBuffer buf, long pageId, long curSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize)
+        throws TrackingPageIsCorruptedException {
+        if (isCorrupted(buf))
+            throw TrackingPageIsCorruptedException.INSTANCE;
+
+        long lastTag = validateSnapshotTag(buf, curSnapshotTag + 1, lastSuccessfulSnapshotTag, pageSize);
+
+        if (lastTag >= 0)
+            throw new TrackingPageIsCorruptedException(lastTag, curSnapshotTag);
 
         if (countOfChangedPage(buf, curSnapshotTag, pageSize) < 1)
             return false;
@@ -223,7 +318,7 @@ public boolean wasChanged(ByteBuffer buf, long pageId, long curSnapshotTag, long
      * @param snapshotTag Snapshot tag.
      * @param pageSize Page size.
      *
-     * @return count of pages which were marked as change for given snapshotTag
+     * @return Count of pages which were marked as change for given snapshotTag.
      */
     public short countOfChangedPage(ByteBuffer buf, long snapshotTag, int pageSize) {
         long dif = getLastSnapshotTag(buf) - snapshotTag;
@@ -240,16 +335,16 @@ public short countOfChangedPage(ByteBuffer buf, long snapshotTag, int pageSize)
     /**
      * @param snapshotTag Snapshot id.
      *
-     * @return true if snapshotTag is odd, otherwise - false
+     * @return true if snapshotTag is odd, otherwise - false.
      */
-    boolean useLeftHalf(long snapshotTag) {
+    private boolean useLeftHalf(long snapshotTag) {
         return (snapshotTag & 0b1) == 0;
     }
 
     /**
      * @param pageId Page id.
      * @param pageSize Page size.
-     * @return pageId of tracking page which set pageId belongs to
+     * @return Page id of tracking page which set pageId belongs to.
      */
     public long trackingPageFor(long pageId, int pageSize) {
         assert PageIdUtils.pageIndex(pageId) > 0;
@@ -267,7 +362,7 @@ public long trackingPageFor(long pageId, int pageSize) {
     /**
      * @param pageSize Page size.
      *
-     * @return how many page we can track with 1 page
+     * @return How many page we can track with 1 page.
      */
     public int countOfPageToTrack(int pageSize) {
         return ((pageSize - SIZE_FIELD_OFFSET) / 2 - SIZE_FIELD_SIZE)  << 3;
@@ -279,11 +374,18 @@ public int countOfPageToTrack(int pageSize) {
      * @param curSnapshotTag Snapshot id.
      * @param lastSuccessfulSnapshotTag  Last successful snapshot id.
      * @param pageSize Page size.
-     * @return set pageId if it was changed or next closest one, if there is no changed page {@code null} will be returned
+     * @return Passed pageId if it was changed or next closest one, if there is no changed page {@code null} will be returned.
+     * @throws TrackingPageIsCorruptedException if this tracking page was marked as corrupted.
      */
     @Nullable public Long findNextChangedPage(ByteBuffer buf, long start, long curSnapshotTag,
-        long lastSuccessfulSnapshotTag, int pageSize) {
-        validateSnapshotId(buf, curSnapshotTag + 1, lastSuccessfulSnapshotTag, pageSize);
+        long lastSuccessfulSnapshotTag, int pageSize) throws TrackingPageIsCorruptedException {
+        if (isCorrupted(buf))
+            throw TrackingPageIsCorruptedException.INSTANCE;
+
+        long lastTag = validateSnapshotTag(buf, curSnapshotTag + 1, lastSuccessfulSnapshotTag, pageSize);
+
+        if (lastTag >= 0)
+            throw new TrackingPageIsCorruptedException(lastTag, curSnapshotTag);
 
         int cntOfPage = countOfPageToTrack(pageSize);
 
@@ -332,6 +434,8 @@ public int countOfPageToTrack(int pageSize) {
     /**
      * @param byteToTest Byte to test.
      * @param firstBitToTest First bit to test.
+     *
+     * @return Index of bit which is set to 1, if there is no such index then <code>-1</code>.
      */
     private static int foundSetBit(byte byteToTest, int firstBitToTest) {
         assert firstBitToTest < 8;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIOTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIOTest.java
index b50f0262fdab1..cacea48c5289c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIOTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIOTest.java
@@ -26,14 +26,19 @@
 import java.util.TreeSet;
 import java.util.concurrent.ThreadLocalRandom;
 import junit.framework.TestCase;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.processors.cache.persistence.snapshot.TrackingPageIsCorruptedException;
 import org.apache.ignite.internal.util.GridUnsafe;
+import org.jetbrains.annotations.NotNull;
 
 /**
  *
  */
 public class TrackingPageIOTest extends TestCase {
     /** Page size. */
-    public static final int PAGE_SIZE = 2048;
+    public static final int PAGE_SIZE = 4096;
+
 
     /** */
     private final TrackingPageIO io = TrackingPageIO.VERSIONS.latest();
@@ -41,9 +46,8 @@ public class TrackingPageIOTest extends TestCase {
     /**
      *
      */
-    public void testBasics() {
-        ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
-        buf.order(ByteOrder.nativeOrder());
+    public void testBasics() throws Exception {
+        ByteBuffer buf = createBuffer();
 
         io.markChanged(buf, 2, 0, -1, PAGE_SIZE);
 
@@ -55,13 +59,21 @@ public void testBasics() {
     }
 
     /**
-     *
+     * @return byte buffer with right order.
      */
-    public void testMarkingRandomly() {
+    @NotNull private ByteBuffer createBuffer() {
         ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
+
         buf.order(ByteOrder.nativeOrder());
 
-        int cntOfPageToTrack = io.countOfPageToTrack(PAGE_SIZE);
+        return buf;
+    }
+
+    /**
+     *
+     */
+    public void testMarkingRandomly() throws Exception {
+        ByteBuffer buf = createBuffer();
 
         for (int i = 0; i < 1001; i++)
             checkMarkingRandomly(buf, i, false);
@@ -70,9 +82,8 @@ public void testMarkingRandomly() {
     /**
      *
      */
-    public void testZeroingRandomly() {
-        ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
-        buf.order(ByteOrder.nativeOrder());
+    public void testZeroingRandomly() throws Exception {
+        ByteBuffer buf = createBuffer();
 
         for (int i = 0; i < 1001; i++)
             checkMarkingRandomly(buf, i, true);
@@ -82,7 +93,7 @@ public void testZeroingRandomly() {
      * @param buf Buffer.
      * @param backupId Backup id.
      */
-    private void checkMarkingRandomly(ByteBuffer buf, int backupId, boolean testZeroing) {
+    private void checkMarkingRandomly(ByteBuffer buf, int backupId, boolean testZeroing) throws Exception {
         ThreadLocalRandom rand = ThreadLocalRandom.current();
 
         int track = io.countOfPageToTrack(PAGE_SIZE);
@@ -132,8 +143,7 @@ private void checkMarkingRandomly(ByteBuffer buf, int backupId, boolean testZero
      * @throws Exception If failed.
      */
     public void testFindNextChangedPage() throws Exception {
-        ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
-        buf.order(ByteOrder.nativeOrder());
+        ByteBuffer buf = createBuffer();
 
         for (int i = 0; i < 101; i++)
             checkFindingRandomly(buf, i);
@@ -143,7 +153,7 @@ public void testFindNextChangedPage() throws Exception {
      * @param buf Buffer.
      * @param backupId Backup id.
      */
-    private void checkFindingRandomly(ByteBuffer buf, int backupId) {
+    private void checkFindingRandomly(ByteBuffer buf, int backupId) throws Exception {
         ThreadLocalRandom rand = ThreadLocalRandom.current();
 
         int track = io.countOfPageToTrack(PAGE_SIZE);
@@ -187,9 +197,8 @@ else if (setIdx.contains(pageId))
     /**
      *
      */
-    public void testMerging() {
-        ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
-        buf.order(ByteOrder.nativeOrder());
+    public void testMerging() throws Exception {
+        ByteBuffer buf = createBuffer();
 
         ThreadLocalRandom rand = ThreadLocalRandom.current();
 
@@ -226,9 +235,8 @@ public void testMerging() {
     /**
      *
      */
-    public void testMerging_MarksShouldBeDropForSuccessfulBackup() {
-        ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
-        buf.order(ByteOrder.nativeOrder());
+    public void testMerging_MarksShouldBeDropForSuccessfulBackup() throws Exception {
+        ByteBuffer buf = createBuffer();
 
         ThreadLocalRandom rand = ThreadLocalRandom.current();
 
@@ -260,6 +268,15 @@ public void testMerging_MarksShouldBeDropForSuccessfulBackup() {
             assertEquals("pageId = " + i, setIdx2.contains(i), io.wasChanged(buf, i, 5, 4, PAGE_SIZE));
     }
 
+    /**
+     * @param buf Buffer.
+     * @param track Track.
+     * @param basePageId Base page id.
+     * @param maxPageId Max page id.
+     * @param setIdx Set index.
+     * @param backupId Backup id.
+     * @param successfulBackupId Successful backup id.
+     */
     private void generateMarking(
         ByteBuffer buf,
         int track,
@@ -280,4 +297,63 @@ private void generateMarking(
             }
         }
     }
+
+    /**
+     * We should handle case when we lost snapshot tag and now it's lower than saved.
+     *
+     * @throws Exception if failed.
+     */
+    public void testThatWeDontFailIfSnapshotTagWasLost() throws Exception {
+        ByteBuffer buf = createBuffer();
+
+        long basePageId = PageIdUtils.pageId(0, PageIdAllocator.FLAG_IDX, 1);
+
+        assert basePageId >= 0;
+
+        PageIO.setPageId(GridUnsafe.bufferAddress(buf), basePageId);
+
+        int oldTag = 10;
+
+        io.markChanged(buf, basePageId + 1, oldTag, oldTag - 1, PAGE_SIZE);
+
+        for (int i = 1; i < 100; i++)
+            io.markChanged(buf, basePageId + i, oldTag - 1, oldTag - 2, PAGE_SIZE);
+
+        assertTrue(io.isCorrupted(buf));
+
+        for (int i = 1; i < 100; i++) {
+            try {
+                long id = basePageId + i;
+
+                io.wasChanged(buf, id, oldTag - 1, oldTag - 2, PAGE_SIZE);
+
+                fail();
+            }
+            catch (TrackingPageIsCorruptedException e) {
+                //ignore
+            }
+        }
+
+        for (int i = 1; i < 100; i++) {
+            long id = basePageId + i + 1000;
+
+            io.markChanged(buf, id, oldTag, oldTag - 2, PAGE_SIZE);
+        }
+
+        io.resetCorruptFlag(buf);
+
+        assertFalse(io.isCorrupted(buf));
+
+        for (int i = 1; i < 100; i++) {
+            long id = basePageId + i + 1000;
+
+            assertTrue(io.wasChanged(buf, id, oldTag, oldTag - 1, PAGE_SIZE));
+        }
+
+        for (int i = 1; i < 100; i++) {
+            long id = basePageId + i;
+
+            assertFalse(io.wasChanged(buf, id, oldTag, oldTag - 1, PAGE_SIZE));
+        }
+    }
 }
\ No newline at end of file

From 49f11db727febc83297c7f0f5de9e6f98f0197fa Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Mon, 9 Apr 2018 09:25:50 +0700
Subject: [PATCH 004/543] IGNITE-8159 control.sh: Fixed NPE on adding nodes on
 empty baseline and not active cluster.

(cherry picked from commit 834869c)
---
 .../visor/baseline/VisorBaselineTask.java     | 24 ++++++-----
 .../ignite/util/GridCommandHandlerTest.java   | 43 +++++++++++++++++++
 2 files changed, 56 insertions(+), 11 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
index 56c2dd94c53d3..721b4b3d7519d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
@@ -71,10 +71,10 @@ private VisorBaselineTaskResult collect() {
 
             Collection<? extends BaselineNode> baseline = cluster.currentBaselineTopology();
 
-            Collection<? extends BaselineNode> servers = cluster.forServers().nodes();
+            Collection<? extends BaselineNode> srvrs = cluster.forServers().nodes();
 
-            return new VisorBaselineTaskResult(ignite.active(), cluster.topologyVersion(),
-                F.isEmpty(baseline) ? null : baseline, servers);
+            return new VisorBaselineTaskResult(ignite.cluster().active(), cluster.topologyVersion(),
+                F.isEmpty(baseline) ? null : baseline, srvrs);
         }
 
         /**
@@ -93,12 +93,14 @@ private VisorBaselineTaskResult set0(Collection<BaselineNode> baselineTop) {
          * @return Current baseline.
          */
         private Map<String, BaselineNode> currentBaseLine() {
-            Collection<BaselineNode> baseline = ignite.cluster().currentBaselineTopology();
-
             Map<String, BaselineNode> nodes = new HashMap<>();
 
-            for (BaselineNode node : baseline)
-                nodes.put(node.consistentId().toString(), node);
+            Collection<BaselineNode> baseline = ignite.cluster().currentBaselineTopology();
+
+            if (!F.isEmpty(baseline)) {
+                for (BaselineNode node : baseline)
+                    nodes.put(node.consistentId().toString(), node);
+            }
 
             return nodes;
         }
@@ -122,12 +124,12 @@ private Map<String, BaselineNode> currentServers() {
          * @return New baseline.
          */
         private VisorBaselineTaskResult set(List<String> consistentIds) {
-            Map<String, BaselineNode> servers = currentServers();
+            Map<String, BaselineNode> srvrs = currentServers();
 
             Collection<BaselineNode> baselineTop = new ArrayList<>();
 
             for (String consistentId : consistentIds) {
-                BaselineNode node = servers.get(consistentId);
+                BaselineNode node = srvrs.get(consistentId);
 
                 if (node == null)
                     throw new IllegalStateException("Node not found for consistent ID: " + consistentId);
@@ -146,10 +148,10 @@ private VisorBaselineTaskResult set(List<String> consistentIds) {
          */
         private VisorBaselineTaskResult add(List<String> consistentIds) {
             Map<String, BaselineNode> baseline = currentBaseLine();
-            Map<String, BaselineNode> servers = currentServers();
+            Map<String, BaselineNode> srvrs = currentServers();
 
             for (String consistentId : consistentIds) {
-                BaselineNode node = servers.get(consistentId);
+                BaselineNode node = srvrs.get(consistentId);
 
                 if (node == null)
                     throw new IllegalStateException("Node not found for consistent ID: " + consistentId);
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index d5051d7ba0bc7..eb18e29680082 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -17,7 +17,9 @@
 
 package org.apache.ignite.util;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import org.apache.ignite.Ignite;
@@ -61,6 +63,11 @@ protected File folder(String folder) throws IgniteCheckedException {
         cleanPersistenceDir();
     }
 
+    /** {@inheritDoc} */
+    @Override public String getTestIgniteInstanceName() {
+        return "bltTest";
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -76,6 +83,8 @@ protected File folder(String folder) throws IgniteCheckedException {
         dsCfg.setWalMode(WALMode.LOG_ONLY);
         dsCfg.getDefaultDataRegionConfiguration().setPersistenceEnabled(true);
 
+        cfg.setConsistentId(igniteInstanceName);
+
         return cfg;
     }
 
@@ -205,6 +214,40 @@ public void testBaselineAdd() throws Exception {
         assertEquals(2, ignite.cluster().currentBaselineTopology().size());
     }
 
+    /**
+     * Test baseline add items works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testBaselineAddOnNotActiveCluster() throws Exception {
+        try {
+            Ignite ignite = startGrid(1);
+
+            assertFalse(ignite.cluster().active());
+
+            String consistentIDs = getTestIgniteInstanceName(1);
+
+            ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
+            System.setOut(new PrintStream(out));
+
+            assertEquals(EXIT_CODE_UNEXPECTED_ERROR, execute("--baseline", "add", consistentIDs));
+
+            assertTrue(out.toString().contains("Changing BaselineTopology on inactive cluster is not allowed."));
+
+            consistentIDs =
+                getTestIgniteInstanceName(1) + ", " +
+                    getTestIgniteInstanceName(2) + "," +
+                    getTestIgniteInstanceName(3);
+
+            assertEquals(EXIT_CODE_UNEXPECTED_ERROR, execute("--baseline", "add", consistentIDs));
+
+            assertTrue(out.toString().contains("Node not found for consistent ID: bltTest2"));
+        }
+        finally {
+            System.setOut(System.out);
+        }
+    }
+
     /**
      * Test baseline remove works via control.sh
      *

From 9ad7be2f51b6dcdcdf43fedb298cd4e240f0adab Mon Sep 17 00:00:00 2001
From: Ilya Borisov <klaster1@gmail.com>
Date: Mon, 9 Apr 2018 20:59:32 +0700
Subject: [PATCH 005/543] IGNITE-8155 Web Console: Fixed number pattern warning
 in browser console.

(cherry picked from commit 5d8f570)
---
 .../app/modules/states/configuration/clusters/communication.pug | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug b/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
index bd8971ad9f193..8b43521ff5a0a 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
+++ b/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
@@ -94,6 +94,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     tip: 'Message queue limit for incoming and outgoing messages'
                 })
             .pc-form-grid-col-30
+                //- allowInvalid: true prevents from infinite digest loop when old value was 0 and becomes less than allowed minimum
                 +sane-ignite-form-field-number({
                     label: 'Unacknowledged messages:',
                     model: `${communication}.unacknowledgedMessagesBufferSize`,
@@ -111,7 +112,6 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                         <li>At least message queue limit * {{ ::$ctrl.Clusters.unacknowledgedMessagesBufferSize.validRatio }}</li>
                     </ul>`
                 })(
-                    //- allowInvalid: true prevents from infinite digest loop when old value was 0 and becomes less than allowed minimum
                     ng-model-options=`{
                         allowInvalid: true
                     }`

From 4aa56751906e5db7aad025a7193933fa929aae26 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Mon, 9 Apr 2018 22:13:21 +0700
Subject: [PATCH 006/543] IGNITE-7940 Visor CMD: Added "cache -slp" and "cache
 -rlp" commands to show and reset lost partitions for specified cache.

(cherry picked from commit abfa0f5)
---
 .../cache/VisorCacheLostPartitionsTask.java   |  85 +++++++++
 .../VisorCacheLostPartitionsTaskArg.java      |  73 ++++++++
 .../VisorCacheLostPartitionsTaskResult.java   |  74 ++++++++
 .../VisorCacheResetLostPartitionsTask.java    |  65 +++++++
 .../VisorCacheResetLostPartitionsTaskArg.java |  73 ++++++++
 .../commands/cache/VisorCacheCommand.scala    |  34 +++-
 .../VisorCacheLostPartitionsCommand.scala     | 170 ++++++++++++++++++
 ...VisorCacheResetLostPartitionsCommand.scala | 132 ++++++++++++++
 8 files changed, 702 insertions(+), 4 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTaskArg.java
 create mode 100644 modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheLostPartitionsCommand.scala
 create mode 100644 modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheResetLostPartitionsCommand.scala

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java
new file mode 100644
index 0000000000000..24b406915a02b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.cache;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+
+/**
+ * Collect list of lost partitions.
+ */
+@GridInternal
+public class VisorCacheLostPartitionsTask
+    extends VisorOneNodeTask<VisorCacheLostPartitionsTaskArg, VisorCacheLostPartitionsTaskResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCacheLostPartitionsJob job(VisorCacheLostPartitionsTaskArg arg) {
+        return new VisorCacheLostPartitionsJob(arg, debug);
+    }
+
+    /**
+     * Job that collect list of lost partitions.
+     */
+    private static class VisorCacheLostPartitionsJob
+        extends VisorJob<VisorCacheLostPartitionsTaskArg, VisorCacheLostPartitionsTaskResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param arg Object with list of cache names to collect lost partitions.
+         * @param debug Debug flag.
+         */
+        private VisorCacheLostPartitionsJob(VisorCacheLostPartitionsTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorCacheLostPartitionsTaskResult run(VisorCacheLostPartitionsTaskArg arg) {
+            Map<String, List<Integer>> res = new HashMap<>();
+
+            for (String cacheName: arg.getCacheNames()) {
+                IgniteInternalCache cache = ignite.cachex(cacheName);
+
+                if (cache != null) {
+                    GridDhtPartitionTopology topology = cache.context().topology();
+                    List<Integer> lostPartitions = new ArrayList<>(topology.lostPartitions());
+
+                    if (!lostPartitions.isEmpty())
+                        res.put(cacheName, lostPartitions);
+                }
+            }
+
+            return new VisorCacheLostPartitionsTaskResult(res);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorCacheLostPartitionsJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskArg.java
new file mode 100644
index 0000000000000..d6404bfdeb5ec
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskArg.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.cache;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Argument for {@link VisorCacheLostPartitionsTask}.
+ */
+public class VisorCacheLostPartitionsTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** List of cache names. */
+    private List<String> cacheNames;
+
+    /**
+     * Default constructor.
+     */
+    public VisorCacheLostPartitionsTaskArg() {
+        // No-op.
+    }
+
+    /**
+     * @param cacheNames List of cache names.
+     */
+    public VisorCacheLostPartitionsTaskArg(List<String> cacheNames) {
+        this.cacheNames = cacheNames;
+    }
+
+    /**
+     * @return List of cache names.
+     */
+    public List<String> getCacheNames() {
+        return cacheNames;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeCollection(out, cacheNames);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        cacheNames = U.readList(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorCacheLostPartitionsTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskResult.java
new file mode 100644
index 0000000000000..b9a0e6b13ad81
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTaskResult.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.cache;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Result for {@link VisorCacheLostPartitionsTask}.
+ */
+public class VisorCacheLostPartitionsTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** List of lost partitions by caches. */
+    private Map<String, List<Integer>> lostPartitions;
+
+    /**
+     * Default constructor.
+     */
+    public VisorCacheLostPartitionsTaskResult() {
+        // No-op.
+    }
+
+    /**
+     * @param lostPartitions List of lost partitions by caches.
+     */
+    public VisorCacheLostPartitionsTaskResult(Map<String, List<Integer>> lostPartitions) {
+        this.lostPartitions = lostPartitions;
+    }
+
+    /**
+     * @return List of lost partitions by caches.
+     */
+    public Map<String, List<Integer>> getLostPartitions() {
+        return lostPartitions;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, lostPartitions);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        lostPartitions = U.readMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorCacheLostPartitionsTaskResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java
new file mode 100644
index 0000000000000..eb48cd2efd458
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.cache;
+
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+
+/**
+ * Reset lost partitions for caches.
+ */
+@GridInternal
+public class VisorCacheResetLostPartitionsTask extends VisorOneNodeTask<VisorCacheResetLostPartitionsTaskArg, Void> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCacheResetLostPartitionsJob job(VisorCacheResetLostPartitionsTaskArg arg) {
+        return new VisorCacheResetLostPartitionsJob(arg, debug);
+    }
+
+    /**
+     * Job that reset lost partitions for caches.
+     */
+    private static class VisorCacheResetLostPartitionsJob extends VisorJob<VisorCacheResetLostPartitionsTaskArg, Void> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param arg Object with list cache names to reset lost partitons.
+         * @param debug Debug flag.
+         */
+        private VisorCacheResetLostPartitionsJob(VisorCacheResetLostPartitionsTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected Void run(VisorCacheResetLostPartitionsTaskArg arg) {
+            ignite.resetLostPartitions(arg.getCacheNames());
+
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorCacheResetLostPartitionsJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTaskArg.java
new file mode 100644
index 0000000000000..2f365c82d6fba
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTaskArg.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.cache;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Argument for {@link VisorCacheResetLostPartitionsTask}.
+ */
+public class VisorCacheResetLostPartitionsTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** List of cache names. */
+    private List<String> cacheNames;
+
+    /**
+     * Default constructor.
+     */
+    public VisorCacheResetLostPartitionsTaskArg() {
+        // No-op.
+    }
+
+    /**
+     * @param cacheNames List of cache names.
+     */
+    public VisorCacheResetLostPartitionsTaskArg(List<String> cacheNames) {
+        this.cacheNames = cacheNames;
+    }
+
+    /**
+     * @return List of cache names.
+     */
+    public List<String> getCacheNames() {
+        return cacheNames;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeCollection(out, cacheNames);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        cacheNames = U.readList(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorCacheResetLostPartitionsTaskArg.class, this);
+    }
+}
diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheCommand.scala
index d55fed1e72402..e3e200148a252 100755
--- a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheCommand.scala
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheCommand.scala
@@ -62,6 +62,10 @@ import scala.language.{implicitConversions, reflectiveCalls}
  * +-------------------------------------------------------------------------------------------+
  * | cache -rebalance | Re-balance partitions for cache with specified name.                   |
  * +-------------------------------------------------------------------------------------------+
+ * | cache -slp       | Show list of lost partitions for specified cache.                      |
+ * +-------------------------------------------------------------------------------------------+
+ * | cache -rlp       | Reset lost partitions for specified cache.                             |
+ * +-------------------------------------------------------------------------------------------+
  *
  * }}}
  *
@@ -75,6 +79,8 @@ import scala.language.{implicitConversions, reflectiveCalls}
  *     cache -stop -c=<cache-name>
  *     cache -reset -c=<cache-name>
  *     cache -rebalance -c=<cache-name>
+ *     cache -slp -c=<cache-name>
+ *     cache -rlp -c=<cache-name>
  * }}}
  *
  * ====Arguments====
@@ -123,6 +129,10 @@ import scala.language.{implicitConversions, reflectiveCalls}
  *          Reset metrics for cache with specified name.
  *     -rebalance
  *          Re-balance partitions for cache with specified name.
+ *     -slp
+ *          Show list of lost partitions for specified cache.
+ *     -rlp
+ *          Reset lost partitions for specified cache.
  *     -p=<page size>
  *         Number of object to fetch from cache at once.
  *         Valid range from 1 to 100.
@@ -163,6 +173,10 @@ import scala.language.{implicitConversions, reflectiveCalls}
  *         Reset metrics for cache with name 'cache'.
  *     cache -rebalance -c=cache
  *         Re-balance partitions for cache with name 'cache'.
+ *     cache -slp -c=cache
+ *         Show list of lost partitions for cache with name 'cache'.
+ *     cache -rlp -c=cache
+ *         Reset lost partitions for cache with name 'cache'.
  *
  * }}}
  */
@@ -265,7 +279,7 @@ class VisorCacheCommand extends VisorConsoleCommand {
             // Get cache stats data from all nodes.
             val aggrData = cacheData(node, cacheName, showSystem)
 
-            if (hasArgFlagIn("clear", "scan", "stop", "reset", "rebalance")) {
+            if (hasArgFlagIn("clear", "scan", "stop", "reset", "rebalance", "slp", "rlp")) {
                 if (cacheName.isEmpty)
                     askForCache("Select cache from:", node, showSystem
                         && !hasArgFlagIn("clear", "stop", "reset", "rebalance"), aggrData) match {
@@ -291,6 +305,10 @@ class VisorCacheCommand extends VisorConsoleCommand {
                                     VisorCacheResetCommand().reset(argLst, node)
                                 else if (hasArgFlag("rebalance", argLst))
                                     VisorCacheRebalanceCommand().rebalance(argLst, node)
+                                else if (hasArgFlag("slp", argLst))
+                                    VisorCacheLostPartitionsCommand().showLostPartitions(argLst, node)
+                                else if (hasArgFlag("rlp", argLst))
+                                    VisorCacheResetLostPartitionsCommand().resetLostPartitions(argLst, node)
                             }
                             else {
                                 if (hasArgFlag("scan", argLst))
@@ -716,7 +734,9 @@ object VisorCacheCommand {
             " ",
             "Clears cache.",
             " ",
-            "Prints list of all entries from cache."
+            "Prints list of all entries from cache.",
+            " ",
+            "Prints or clear list lost partitions from cache."
         ),
         spec = Seq(
             "cache",
@@ -726,7 +746,9 @@ object VisorCacheCommand {
             "cache -scan -c=<cache-name> {-near} {-id=<node-id>|id8=<node-id8>} {-p=<page size>}",
             "cache -stop -c=<cache-name>",
             "cache -reset -c=<cache-name>",
-            "cache -rebalance -c=<cache-name>"
+            "cache -rebalance -c=<cache-name>",
+            "cache -slp -c=<cache-name>",
+            "cache -rlp -c=<cache-name>"
   ),
         args = Seq(
             "-id8=<node-id>" -> Seq(
@@ -752,6 +774,8 @@ object VisorCacheCommand {
             "-near" -> "Prints list of all entries from near cache of cache.",
             "-stop" -> "Stop cache with specified name.",
             "-reset" -> "Reset metrics of cache with specified name.",
+            "-slp" -> "Show list of lost partitions for specified cache.",
+            "-rlp" -> "Reset lost partitions for specified cache.",
             "-rebalance" -> "Re-balance partitions for cache with specified name.",
             "-s=hi|mi|rd|wr|cn" -> Seq(
                 "Defines sorting type. Sorted by:",
@@ -812,7 +836,9 @@ object VisorCacheCommand {
                 "Prints list entries from near cache of cache with name 'cache' and node '12345678' ID8.",
             "cache -stop -c=@c0" -> "Stop cache with name taken from 'c0' memory variable.",
             "cache -reset -c=@c0" -> "Reset metrics for cache with name taken from 'c0' memory variable.",
-            "cache -rebalance -c=cache" -> "Re-balance partitions for cache with name 'cache'."
+            "cache -rebalance -c=cache" -> "Re-balance partitions for cache with name 'cache'.",
+            "cache -slp -c=@c0" -> "Show list of lost partitions for cache with name taken from 'c0' memory variable.",
+            "cache -rlp -c=@c0" -> "Reset lost partitions for cache with name taken from 'c0' memory variable."
         ),
         emptyArgs = cmd.cache,
         withArgs = cmd.cache
diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheLostPartitionsCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheLostPartitionsCommand.scala
new file mode 100644
index 0000000000000..d6830310d11d1
--- /dev/null
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheLostPartitionsCommand.scala
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.visor.commands.cache
+
+import java.util.Collections
+
+import org.apache.ignite.cluster.{ClusterGroupEmptyException, ClusterNode}
+import org.apache.ignite.internal.visor.cache.{VisorCacheLostPartitionsTask, VisorCacheLostPartitionsTaskArg}
+import org.apache.ignite.visor.commands.common.VisorTextTable
+import org.apache.ignite.visor.visor._
+import org.apache.ignite.internal.visor.util.VisorTaskUtils._
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable
+
+/**
+ * ==Overview==
+ * Visor 'lost partitions' command implementation.
+ *
+ * ====Specification====
+ * {{{
+ *     cache -slp -c=<cache name>
+ * }}}
+ *
+ * ====Arguments====
+ * {{{
+ *     <cache-name>
+ *         Name of the cache.
+ * }}}
+ *
+ * ====Examples====
+ * {{{
+ *    cache -slp -c=cache
+ *        Show list of lost partitions for cache with name 'cache'.
+ * }}}
+ */
+class VisorCacheLostPartitionsCommand {
+    /**
+     * Prints error message and advise.
+     *
+     * @param errMsgs Error messages.
+     */
+    private def scold(errMsgs: Any*) {
+        assert(errMsgs != null)
+
+        warn(errMsgs: _*)
+        warn("Type 'help cache' to see how to use this command.")
+    }
+
+    private def error(e: Throwable) {
+        var cause: Throwable = e
+
+        while (cause.getCause != null)
+            cause = cause.getCause
+
+        scold(cause.getMessage)
+    }
+
+    /**
+     * ===Command===
+     * Show list of lost partitions in cache with specified name.
+     *
+     * ===Examples===
+     * <ex>cache -slp -c=cache</ex>
+     *     Show list of lost partitions from cache with name 'cache'.
+     *
+     * @param argLst Command arguments.
+     */
+    def showLostPartitions(argLst: ArgList, node: Option[ClusterNode]) {
+        val cacheArg = argValue("c", argLst)
+        val cacheName = cacheArg match {
+            case None => null // default cache.
+
+            case Some(s) if s.startsWith("@") =>
+                warn("Can't find cache variable with specified name: " + s,
+                    "Type 'cache' to see available cache variables."
+                )
+
+                return
+
+            case Some(name) => name
+        }
+
+        val lostPartitions =
+            try
+                executeRandom(groupForDataNode(node, cacheName), classOf[VisorCacheLostPartitionsTask],
+                    new VisorCacheLostPartitionsTaskArg(Collections.singletonList(cacheName)))
+            catch {
+                case _: ClusterGroupEmptyException =>
+                    scold(messageNodeNotFound(node, cacheName))
+
+                    return
+                case e: Throwable =>
+                    error(e)
+
+                    return
+            }
+
+        if (lostPartitions.getLostPartitions.isEmpty) {
+            println(s"""Lost partitions for cache: "${escapeName(cacheName)}" is not found""")
+
+            return
+        }
+
+        lostPartitions.getLostPartitions.foreach(cacheLostPartitions => {
+            val t = VisorTextTable()
+
+            t #= ("Interval", "Partitions")
+
+            val partitions = cacheLostPartitions._2.toIndexedSeq
+            val partitionCnt = partitions.size
+
+            val indexes = mutable.ArrayBuffer.empty[String]
+            val partitionRows = mutable.ArrayBuffer.empty[String]
+            var startIdx = 0
+            var idx = 0
+            val b = new StringBuilder
+
+            partitions.foreach((part) => {
+                if (idx % 10 == 0)
+                    startIdx = part
+
+                b.append(part)
+                idx += 1
+
+                if (idx != partitionCnt)
+                    b.append(", ")
+
+
+                if (idx % 10 == 0 || idx == partitionCnt) {
+                    indexes += startIdx + "-" + part
+                    partitionRows += b.toString().trim
+                    b.clear()
+                }
+            })
+
+            t += (indexes, partitionRows)
+            println(s"Lost partitions for cache: ${escapeName(cacheLostPartitions._1)} ($partitionCnt)")
+            t.render()
+        })
+    }
+}
+
+/**
+ * Companion object that does initialization of the command.
+ */
+object VisorCacheLostPartitionsCommand {
+    /** Singleton command. */
+    private val cmd = new VisorCacheLostPartitionsCommand
+
+    /**
+     * Singleton.
+     */
+    def apply(): VisorCacheLostPartitionsCommand = cmd
+}
diff --git a/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheResetLostPartitionsCommand.scala b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheResetLostPartitionsCommand.scala
new file mode 100644
index 0000000000000..a72a80f46ff4d
--- /dev/null
+++ b/modules/visor-console/src/main/scala/org/apache/ignite/visor/commands/cache/VisorCacheResetLostPartitionsCommand.scala
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.visor.commands.cache
+
+import java.util.Collections
+
+import org.apache.ignite.cluster.{ClusterGroupEmptyException, ClusterNode}
+import org.apache.ignite.internal.visor.cache.{VisorCacheResetLostPartitionsTask, VisorCacheResetLostPartitionsTaskArg}
+import org.apache.ignite.internal.visor.util.VisorTaskUtils._
+import org.apache.ignite.visor.visor._
+
+import scala.language.reflectiveCalls
+
+/**
+ * ==Overview==
+ * Visor 'lost partition reset' command implementation.
+ *
+ * ====Specification====
+ * {{{
+ *     cache -rlp -c=<cache name>
+ * }}}
+ *
+ * ====Arguments====
+ * {{{
+ *     <cache-name>
+ *         Name of the cache.
+ * }}}
+ *
+ * ====Examples====
+ * {{{
+ *    cache -rlp -c=@c0
+ *        Reset lost partition for cache with name taken from 'c0' memory variable.
+ * }}}
+ */
+class VisorCacheResetLostPartitionsCommand {
+    /**
+     * Prints error message and advise.
+     *
+     * @param errMsgs Error messages.
+     */
+    private def scold(errMsgs: Any*) {
+        assert(errMsgs != null)
+
+        warn(errMsgs: _*)
+        warn("Type 'help cache' to see how to use this command.")
+    }
+
+    private def error(e: Exception) {
+        var cause: Throwable = e
+
+        while (cause.getCause != null)
+            cause = cause.getCause
+
+        scold(cause.getMessage)
+    }
+
+    /**
+     * ===Command===
+     * Reset lost partitions for cache with specified name.
+     *
+     * ===Examples===
+     * <ex>cache -c=cache -rlp</ex>
+     * Reset lost partitions for cache with name 'cache'.
+     *
+     * @param argLst Command arguments.
+     */
+    def resetLostPartitions(argLst: ArgList, node: Option[ClusterNode]) {
+        val cacheArg = argValue("c", argLst)
+
+        val cacheName = cacheArg match {
+            case None => null // default cache.
+
+            case Some(s) if s.startsWith("@") =>
+                warn("Can't find cache variable with specified name: " + s,
+                    "Type 'cache' to see available cache variables."
+                )
+
+                return
+
+            case Some(name) => name
+        }
+
+        val grp = try {
+            groupForDataNode(node, cacheName)
+        }
+        catch {
+            case _: ClusterGroupEmptyException =>
+                scold(messageNodeNotFound(node, cacheName))
+
+                return
+        }
+
+        try {
+            executeRandom(grp, classOf[VisorCacheResetLostPartitionsTask],
+                new VisorCacheResetLostPartitionsTaskArg(Collections.singletonList(cacheName)))
+
+            println("Visor successfully reset lost partitions for cache: " + escapeName(cacheName))
+        }
+        catch {
+            case _: ClusterGroupEmptyException => scold(messageNodeNotFound(node, cacheName))
+            case e: Exception => error(e)
+        }
+    }
+}
+
+/**
+ * Companion object that does initialization of the command.
+ */
+object VisorCacheResetLostPartitionsCommand {
+    /** Singleton command. */
+    private val cmd = new VisorCacheResetLostPartitionsCommand
+
+    /**
+      * Singleton.
+      */
+    def apply() = cmd
+}

From cc04c5c70af1bdbba834f73330e73277b60e23fc Mon Sep 17 00:00:00 2001
From: Eduard Shangareev <eshangareev@gridgain.com>
Date: Mon, 9 Apr 2018 19:15:50 +0300
Subject: [PATCH 007/543] IGNITE-8114 Additional fix for Add fail recovery
 mechanism to tracking pages

(cherry picked from commit 961fc35)
---
 .../cache/persistence/tree/io/TrackingPageIO.java      | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
index 94885e4889fc8..80dbf253e84b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
@@ -86,9 +86,11 @@ protected TrackingPageIO(int ver) {
      * @param pageId Page id.
      * @param nextSnapshotTag Tag of next snapshot.
      * @param pageSize Page size.
+     *
+     * @return <code>-1</code> if everything is ok, otherwise last saved tag.
      */
-    public void markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
-        validateSnapshotTag(buf, nextSnapshotTag, lastSuccessfulSnapshotTag, pageSize);
+    public long markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
+        long tag = validateSnapshotTag(buf, nextSnapshotTag, lastSuccessfulSnapshotTag, pageSize);
 
         int cntOfPage = countOfPageToTrack(pageSize);
 
@@ -105,7 +107,7 @@ public void markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, long
         byte newVal =  (byte) (byteToUpdate | updateTemplate);
 
         if (byteToUpdate == newVal)
-            return;
+            return tag;
 
         buf.put(idx, newVal);
 
@@ -114,6 +116,8 @@ public void markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, long
         buf.putShort(sizeOff, newSize);
 
         assert newSize == countOfChangedPage(buf, nextSnapshotTag, pageSize);
+
+        return tag;
     }
 
     /**

From c70d85aa36c702ea0f29bd8668e9bf0790f9ba11 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Tue, 10 Apr 2018 15:42:24 +0700
Subject: [PATCH 008/543] IGNITE-8126 Web Console: Fixed code generation for
 cache load.

(cherry picked from commit a0a187b)
---
 .../configuration/summary/summary.worker.js   |   2 +-
 .../web-console/frontend/package-lock.json    | 929 ++++++++----------
 2 files changed, 409 insertions(+), 522 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
index c80d698dd588c..b3b0bce4bf3f8 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
+++ b/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
@@ -109,7 +109,7 @@ onmessage = function(e) {
     }
 
     // Generate loader for caches with configured store.
-    const cachesToLoad = filter(cluster.caches, (cache) => nonNil(cache.cacheStoreFactory));
+    const cachesToLoad = filter(cluster.caches, (cache) => nonNil(_.get(cache, 'cacheStoreFactory.kind')));
 
     if (nonEmpty(cachesToLoad))
         zip.file(`${srcPath}/load/LoadCaches.java`, java.loadCaches(cachesToLoad, 'load', 'LoadCaches', `"${clientXml}"`));
diff --git a/modules/web-console/frontend/package-lock.json b/modules/web-console/frontend/package-lock.json
index 071dec058b2c3..1834621f6b316 100644
--- a/modules/web-console/frontend/package-lock.json
+++ b/modules/web-console/frontend/package-lock.json
@@ -95,9 +95,9 @@
       "dev": true
     },
     "@types/node": {
-      "version": "9.6.2",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.2.tgz",
-      "integrity": "sha512-UWkRY9X7RQHp5OhhRIIka58/gVVycL1zHZu0OTsT5LI86ABaMOSbUjAl+b0FeDhQcxclrkyft3kW5QWdMRs8wQ==",
+      "version": "7.0.60",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-7.0.60.tgz",
+      "integrity": "sha512-ZfCUDgCOPBDn0aAsyBOcNh1nLksuGp3LAL+8GULccZN2IkMBG2KfiwFIRrIuQkLKg1W1dIB9kQZ9MIF3IgAqlw==",
       "dev": true
     },
     "@types/sinon": {
@@ -113,9 +113,9 @@
       "dev": true
     },
     "@types/uglify-js": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/@types/uglify-js/-/uglify-js-3.0.1.tgz",
-      "integrity": "sha512-eWwNO88HxJonNKyxZ3dR62yle3N+aBPIsjTrPtoMcldLXGeIKAIlewNIWT4cxjZ4gy3YdBobkaKSv74HJXSzRg==",
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/@types/uglify-js/-/uglify-js-3.0.2.tgz",
+      "integrity": "sha512-o8hU2+4xsyGC27Vujoklvxl88Ew5zmJuTBYMX1Uro2rYUt4HEFJKL6fuq8aGykvS+ssIsIzerWWP2DRxonownQ==",
       "dev": true,
       "requires": {
         "source-map": "0.6.1"
@@ -145,9 +145,9 @@
       "integrity": "sha512-NoGVTCumOsyFfuy3934f3ktiJi+wcXHJFxT47tby3iCpuo6M/WjFA9VqT5bYO+FE46i3R0N00RpJX75HxHKDaQ==",
       "dev": true,
       "requires": {
-        "@types/node": "9.6.2",
+        "@types/node": "7.0.60",
         "@types/tapable": "1.0.1",
-        "@types/uglify-js": "3.0.1",
+        "@types/uglify-js": "3.0.2",
         "source-map": "0.6.1"
       },
       "dependencies": {
@@ -205,6 +205,16 @@
         "preact": "7.2.1"
       }
     },
+    "JSONStream": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.2.tgz",
+      "integrity": "sha1-wQI3G27Dp887hHygDCC7D85Mbeo=",
+      "dev": true,
+      "requires": {
+        "jsonparse": "1.3.1",
+        "through": "2.3.8"
+      }
+    },
     "abbrev": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -455,9 +465,9 @@
       "integrity": "sha1-ClsQZgGLQOcAuMbuNLDJf8MhlSs="
     },
     "angular-ui-grid": {
-      "version": "4.4.5",
-      "resolved": "https://registry.npmjs.org/angular-ui-grid/-/angular-ui-grid-4.4.5.tgz",
-      "integrity": "sha512-ZwnDi4+6oh1A089nKJao8Q9XQseDAsHFYYwT/felyUeCpW+5izq8pJqQszM0HLqNeQj08+IJVQFbWto+UlH07A==",
+      "version": "4.4.6",
+      "resolved": "https://registry.npmjs.org/angular-ui-grid/-/angular-ui-grid-4.4.6.tgz",
+      "integrity": "sha512-0d14HDY4XeqFHI508thxeufiR0AlFoZQ8ihk0x8TRCQc+b9CCk1/F63W2zihirxF0cdOAqBCY2pVSM7vfZvXBQ==",
       "requires": {
         "angular": "1.6.6"
       }
@@ -517,17 +527,15 @@
           "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg="
         },
         "braces": {
-          "version": "2.3.1",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.1.tgz",
-          "integrity": "sha512-SO5lYHA3vO6gz66erVvedSCkp7AKWdv6VcQ2N4ysXfPxdAlxAMMAdwegGGcv1Bqwm7naF1hNdk5d6AAIEHV2nQ==",
+          "version": "2.3.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
+          "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
           "requires": {
             "arr-flatten": "1.1.0",
             "array-unique": "0.3.2",
-            "define-property": "1.0.0",
             "extend-shallow": "2.0.1",
             "fill-range": "4.0.0",
             "isobject": "3.0.1",
-            "kind-of": "6.0.2",
             "repeat-element": "1.1.2",
             "snapdragon": "0.8.2",
             "snapdragon-node": "2.1.1",
@@ -535,14 +543,6 @@
             "to-regex": "3.0.2"
           },
           "dependencies": {
-            "define-property": {
-              "version": "1.0.0",
-              "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-              "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
-              "requires": {
-                "is-descriptor": "1.0.2"
-              }
-            },
             "extend-shallow": {
               "version": "2.0.1",
               "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
@@ -592,6 +592,42 @@
                 "is-extendable": "0.1.1"
               }
             },
+            "is-accessor-descriptor": {
+              "version": "0.1.6",
+              "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
+              "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
+              "requires": {
+                "kind-of": "3.2.2"
+              },
+              "dependencies": {
+                "kind-of": {
+                  "version": "3.2.2",
+                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+                  "requires": {
+                    "is-buffer": "1.1.6"
+                  }
+                }
+              }
+            },
+            "is-data-descriptor": {
+              "version": "0.1.4",
+              "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
+              "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
+              "requires": {
+                "kind-of": "3.2.2"
+              },
+              "dependencies": {
+                "kind-of": {
+                  "version": "3.2.2",
+                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+                  "requires": {
+                    "is-buffer": "1.1.6"
+                  }
+                }
+              }
+            },
             "is-descriptor": {
               "version": "0.1.6",
               "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
@@ -683,39 +719,29 @@
           }
         },
         "is-accessor-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-          "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
           "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
+            "kind-of": "6.0.2"
           }
         },
         "is-data-descriptor": {
-          "version": "0.1.4",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-          "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
           "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
+            "kind-of": "6.0.2"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "requires": {
+            "is-accessor-descriptor": "1.0.0",
+            "is-data-descriptor": "1.0.0",
+            "kind-of": "6.0.2"
           }
         },
         "is-number": {
@@ -753,7 +779,7 @@
           "requires": {
             "arr-diff": "4.0.0",
             "array-unique": "0.3.2",
-            "braces": "2.3.1",
+            "braces": "2.3.2",
             "define-property": "2.0.2",
             "extend-shallow": "3.0.2",
             "extglob": "2.0.4",
@@ -996,7 +1022,7 @@
       "integrity": "sha1-Hb0cg1ZY41zj+ZhAmdsAWFx4IBQ=",
       "requires": {
         "browserslist": "1.7.7",
-        "caniuse-db": "1.0.30000823",
+        "caniuse-db": "1.0.30000827",
         "normalize-range": "0.1.2",
         "num2fraction": "1.2.2",
         "postcss": "5.2.18",
@@ -1009,9 +1035,9 @@
       "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8="
     },
     "aws4": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz",
-      "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4="
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz",
+      "integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w=="
     },
     "babel-code-frame": {
       "version": "6.26.0",
@@ -1654,7 +1680,7 @@
       "integrity": "sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=",
       "requires": {
         "babel-runtime": "6.26.0",
-        "core-js": "2.5.4",
+        "core-js": "2.5.5",
         "regenerator-runtime": "0.10.5"
       },
       "dependencies": {
@@ -1736,7 +1762,7 @@
       "requires": {
         "babel-core": "6.26.0",
         "babel-runtime": "6.26.0",
-        "core-js": "2.5.4",
+        "core-js": "2.5.5",
         "home-or-tmp": "2.0.0",
         "lodash": "4.17.5",
         "mkdirp": "0.5.1",
@@ -1748,7 +1774,7 @@
       "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
       "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=",
       "requires": {
-        "core-js": "2.5.4",
+        "core-js": "2.5.5",
         "regenerator-runtime": "0.11.1"
       }
     },
@@ -1916,6 +1942,11 @@
         "type-is": "1.6.16"
       },
       "dependencies": {
+        "iconv-lite": {
+          "version": "0.4.19",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
+          "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
+        },
         "qs": {
           "version": "6.5.1",
           "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz",
@@ -1992,9 +2023,9 @@
       "integrity": "sha512-erYug8XoqzU3IfcU8fUgyHqyOXqIE4tUTTQ+7mqUjQlvnXkOO6OlT9c/ZoJVHYoAaqGxr09CN53G7XIsO4KtWA==",
       "dev": true,
       "requires": {
+        "JSONStream": "1.3.2",
         "combine-source-map": "0.8.0",
         "defined": "1.0.0",
-        "JSONStream": "1.3.2",
         "safe-buffer": "5.1.1",
         "through2": "2.0.3",
         "umd": "3.0.3"
@@ -2034,6 +2065,7 @@
       "integrity": "sha512-gKfOsNQv/toWz+60nSPfYzuwSEdzvV2WdxrVPUbPD/qui44rAkB3t3muNtmmGYHqrG56FGwX9SUEQmzNLAeS7g==",
       "dev": true,
       "requires": {
+        "JSONStream": "1.3.2",
         "assert": "1.4.1",
         "browser-pack": "6.1.0",
         "browser-resolve": "1.11.2",
@@ -2054,8 +2086,7 @@
         "htmlescape": "1.1.1",
         "https-browserify": "1.0.0",
         "inherits": "2.0.3",
-        "insert-module-globals": "7.0.5",
-        "JSONStream": "1.3.2",
+        "insert-module-globals": "7.0.6",
         "labeled-stream-splicer": "2.0.1",
         "module-deps": "4.1.1",
         "os-browserify": "0.3.0",
@@ -2066,7 +2097,7 @@
         "querystring-es3": "0.2.1",
         "read-only-stream": "2.0.0",
         "readable-stream": "2.3.6",
-        "resolve": "1.6.0",
+        "resolve": "1.7.0",
         "shasum": "1.0.2",
         "shell-quote": "1.6.1",
         "stream-browserify": "2.0.1",
@@ -2233,7 +2264,7 @@
       "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-1.7.7.tgz",
       "integrity": "sha1-C9dnBCWL6CmyOYu1Dkti0aFmsLk=",
       "requires": {
-        "caniuse-db": "1.0.30000823",
+        "caniuse-db": "1.0.30000827",
         "electron-to-chromium": "1.3.42"
       }
     },
@@ -2445,15 +2476,15 @@
       "integrity": "sha1-tTTnxzTE+B7F++isoq0kNUuWLGw=",
       "requires": {
         "browserslist": "1.7.7",
-        "caniuse-db": "1.0.30000823",
+        "caniuse-db": "1.0.30000827",
         "lodash.memoize": "4.1.2",
         "lodash.uniq": "4.5.0"
       }
     },
     "caniuse-db": {
-      "version": "1.0.30000823",
-      "resolved": "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30000823.tgz",
-      "integrity": "sha1-5o5fjHB4PvQFnS6g3oH1UWUdpvw="
+      "version": "1.0.30000827",
+      "resolved": "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30000827.tgz",
+      "integrity": "sha1-vSg53Rlgk7RMKMF/k1ExQMnZJYg="
     },
     "caseless": {
       "version": "0.11.0",
@@ -2546,7 +2577,7 @@
       "requires": {
         "anymatch": "2.0.0",
         "async-each": "1.0.1",
-        "braces": "2.3.1",
+        "braces": "2.3.2",
         "glob-parent": "3.1.0",
         "inherits": "2.0.3",
         "is-binary-path": "1.0.1",
@@ -2563,17 +2594,15 @@
           "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg="
         },
         "braces": {
-          "version": "2.3.1",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.1.tgz",
-          "integrity": "sha512-SO5lYHA3vO6gz66erVvedSCkp7AKWdv6VcQ2N4ysXfPxdAlxAMMAdwegGGcv1Bqwm7naF1hNdk5d6AAIEHV2nQ==",
+          "version": "2.3.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
+          "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
           "requires": {
             "arr-flatten": "1.1.0",
             "array-unique": "0.3.2",
-            "define-property": "1.0.0",
             "extend-shallow": "2.0.1",
             "fill-range": "4.0.0",
             "isobject": "3.0.1",
-            "kind-of": "6.0.2",
             "repeat-element": "1.1.2",
             "snapdragon": "0.8.2",
             "snapdragon-node": "2.1.1",
@@ -2630,27 +2659,12 @@
           "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
           "requires": {
             "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
           }
         },
         "isobject": {
           "version": "3.0.1",
           "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
           "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        },
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
         }
       }
     },
@@ -2719,61 +2733,10 @@
             "is-descriptor": "0.1.6"
           }
         },
-        "is-accessor-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-          "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "is-data-descriptor": {
-          "version": "0.1.4",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-          "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "is-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-          "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-          "requires": {
-            "is-accessor-descriptor": "0.1.6",
-            "is-data-descriptor": "0.1.4",
-            "kind-of": "5.1.0"
-          }
-        },
         "isobject": {
           "version": "3.0.1",
           "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
           "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        },
-        "kind-of": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
         }
       }
     },
@@ -3261,9 +3224,9 @@
       }
     },
     "core-js": {
-      "version": "2.5.4",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.5.4.tgz",
-      "integrity": "sha1-8si/GB8qgLkvNgEhQpzmOi8K6uA="
+      "version": "2.5.5",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.5.5.tgz",
+      "integrity": "sha1-sU3ek2xkDAV5prUMq8wTLdYSfjs="
     },
     "core-util-is": {
       "version": "1.0.2",
@@ -3689,6 +3652,39 @@
       "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
       "requires": {
         "is-descriptor": "1.0.2"
+      },
+      "dependencies": {
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "requires": {
+            "kind-of": "6.0.2"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "requires": {
+            "kind-of": "6.0.2"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "requires": {
+            "is-accessor-descriptor": "1.0.0",
+            "is-data-descriptor": "1.0.0",
+            "kind-of": "6.0.2"
+          }
+        },
+        "kind-of": {
+          "version": "6.0.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
+          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
+        }
       }
     },
     "defined": {
@@ -4019,7 +4015,7 @@
       "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=",
       "dev": true,
       "requires": {
-        "iconv-lite": "0.4.19"
+        "iconv-lite": "0.4.21"
       }
     },
     "end-of-stream": {
@@ -4591,7 +4587,7 @@
       "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==",
       "requires": {
         "chardet": "0.4.2",
-        "iconv-lite": "0.4.19",
+        "iconv-lite": "0.4.21",
         "tmp": "0.0.33"
       }
     },
@@ -4650,18 +4646,16 @@
           "dev": true
         },
         "braces": {
-          "version": "2.3.1",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.1.tgz",
-          "integrity": "sha512-SO5lYHA3vO6gz66erVvedSCkp7AKWdv6VcQ2N4ysXfPxdAlxAMMAdwegGGcv1Bqwm7naF1hNdk5d6AAIEHV2nQ==",
+          "version": "2.3.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
+          "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
           "dev": true,
           "requires": {
             "arr-flatten": "1.1.0",
             "array-unique": "0.3.2",
-            "define-property": "1.0.0",
             "extend-shallow": "2.0.1",
             "fill-range": "4.0.0",
             "isobject": "3.0.1",
-            "kind-of": "6.0.2",
             "repeat-element": "1.1.2",
             "snapdragon": "0.8.2",
             "snapdragon-node": "2.1.1",
@@ -4669,15 +4663,6 @@
             "to-regex": "3.0.2"
           },
           "dependencies": {
-            "define-property": {
-              "version": "1.0.0",
-              "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-              "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
-              "dev": true,
-              "requires": {
-                "is-descriptor": "1.0.2"
-              }
-            },
             "extend-shallow": {
               "version": "2.0.1",
               "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
@@ -4732,6 +4717,46 @@
                 "is-extendable": "0.1.1"
               }
             },
+            "is-accessor-descriptor": {
+              "version": "0.1.6",
+              "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
+              "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
+              "dev": true,
+              "requires": {
+                "kind-of": "3.2.2"
+              },
+              "dependencies": {
+                "kind-of": {
+                  "version": "3.2.2",
+                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+                  "dev": true,
+                  "requires": {
+                    "is-buffer": "1.1.6"
+                  }
+                }
+              }
+            },
+            "is-data-descriptor": {
+              "version": "0.1.4",
+              "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
+              "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
+              "dev": true,
+              "requires": {
+                "kind-of": "3.2.2"
+              },
+              "dependencies": {
+                "kind-of": {
+                  "version": "3.2.2",
+                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+                  "dev": true,
+                  "requires": {
+                    "is-buffer": "1.1.6"
+                  }
+                }
+              }
+            },
             "is-descriptor": {
               "version": "0.1.6",
               "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
@@ -4853,43 +4878,32 @@
           }
         },
         "is-accessor-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-          "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
           "dev": true,
           "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "dev": true,
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
+            "kind-of": "6.0.2"
           }
         },
         "is-data-descriptor": {
-          "version": "0.1.4",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-          "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
           "dev": true,
           "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "dev": true,
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
+            "kind-of": "6.0.2"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "dev": true,
+          "requires": {
+            "is-accessor-descriptor": "1.0.0",
+            "is-data-descriptor": "1.0.0",
+            "kind-of": "6.0.2"
           }
         },
         "is-extglob": {
@@ -4947,7 +4961,7 @@
           "requires": {
             "arr-diff": "4.0.0",
             "array-unique": "0.3.2",
-            "braces": "2.3.1",
+            "braces": "2.3.2",
             "define-property": "2.0.2",
             "extend-shallow": "3.0.2",
             "extglob": "2.0.4",
@@ -5816,9 +5830,9 @@
       }
     },
     "html-minifier": {
-      "version": "3.5.13",
-      "resolved": "https://registry.npmjs.org/html-minifier/-/html-minifier-3.5.13.tgz",
-      "integrity": "sha512-B7P99uf0LPQ5lslyhrAZAXE7Lk1tpiv52KVapKbeFhgqNMUI7JBd/fYLX55imu3Rz7sCTzZM6r/IBe4oT7qCjg==",
+      "version": "3.5.14",
+      "resolved": "https://registry.npmjs.org/html-minifier/-/html-minifier-3.5.14.tgz",
+      "integrity": "sha512-sZjw6zhQgyUnIlIPU+W80XpRjWjdxHtNcxjfyOskOsCTDKytcfLY04wsQY/83Yqb4ndoiD2FtauiL7Yg6uUQFQ==",
       "requires": {
         "camel-case": "3.0.0",
         "clean-css": "4.1.11",
@@ -5826,7 +5840,7 @@
         "he": "1.1.1",
         "param-case": "2.1.1",
         "relateurl": "0.2.7",
-        "uglify-js": "3.3.18"
+        "uglify-js": "3.3.20"
       },
       "dependencies": {
         "source-map": {
@@ -5835,9 +5849,9 @@
           "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
         },
         "uglify-js": {
-          "version": "3.3.18",
-          "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.18.tgz",
-          "integrity": "sha512-VhjIFv93KnTx/ntNi9yTBbfrsWnQnqUy02MT32uqU/5i2oEJ8GAEJ0AwYV206JeOmIzSjm41Ba0iXVKv6j7y9g==",
+          "version": "3.3.20",
+          "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.20.tgz",
+          "integrity": "sha512-WpLkWCf9sGvGZnIvBV0PNID9BATQNT/IXKAmqegfKzIPcTmTV3FP8NQpoogQkt/Y402x2sOFdaHUmqFY9IZp+g==",
           "requires": {
             "commander": "2.15.1",
             "source-map": "0.6.1"
@@ -5850,7 +5864,7 @@
       "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz",
       "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=",
       "requires": {
-        "html-minifier": "3.5.13",
+        "html-minifier": "3.5.14",
         "loader-utils": "0.2.17",
         "lodash": "4.17.5",
         "pretty-error": "2.1.1",
@@ -5888,7 +5902,7 @@
         "posthtml": "0.11.3",
         "posthtml-render": "1.1.3",
         "svgo": "1.0.5",
-        "uglify-js": "3.3.18"
+        "uglify-js": "3.3.20"
       },
       "dependencies": {
         "coa": {
@@ -5954,9 +5968,9 @@
           }
         },
         "uglify-js": {
-          "version": "3.3.18",
-          "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.18.tgz",
-          "integrity": "sha512-VhjIFv93KnTx/ntNi9yTBbfrsWnQnqUy02MT32uqU/5i2oEJ8GAEJ0AwYV206JeOmIzSjm41Ba0iXVKv6j7y9g==",
+          "version": "3.3.20",
+          "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.20.tgz",
+          "integrity": "sha512-WpLkWCf9sGvGZnIvBV0PNID9BATQNT/IXKAmqegfKzIPcTmTV3FP8NQpoogQkt/Y402x2sOFdaHUmqFY9IZp+g==",
           "requires": {
             "commander": "2.15.1",
             "source-map": "0.6.1"
@@ -6061,9 +6075,12 @@
       "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM="
     },
     "iconv-lite": {
-      "version": "0.4.19",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
-      "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
+      "version": "0.4.21",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.21.tgz",
+      "integrity": "sha512-En5V9za5mBt2oUA03WGD3TwDv0MKAruqsuxstbMUZaj9W9k/m1CV/9py3l0L5kw9Bln8fdHQmzHSYtvpvTLpKw==",
+      "requires": {
+        "safer-buffer": "2.1.2"
+      }
     },
     "icss-replace-symbols": {
       "version": "1.1.0",
@@ -6290,16 +6307,17 @@
       }
     },
     "insert-module-globals": {
-      "version": "7.0.5",
-      "resolved": "https://registry.npmjs.org/insert-module-globals/-/insert-module-globals-7.0.5.tgz",
-      "integrity": "sha512-wgRtrCpMm0ruH2hgLUIx+9YfJsgJQmU1KkPUzTuatW9dbH19yPRqAQhFX1HJU6zbmg2IMmt80BgSE5MWuksw3Q==",
+      "version": "7.0.6",
+      "resolved": "https://registry.npmjs.org/insert-module-globals/-/insert-module-globals-7.0.6.tgz",
+      "integrity": "sha512-R3sidKJr3SsggqQQ5cEwQb3pWG8RNx0UnpyeiOSR6jorRIeAOzH2gkTWnNdMnyRiVbjrG047K7UCtlMkQ1Mo9w==",
       "dev": true,
       "requires": {
+        "JSONStream": "1.3.2",
         "combine-source-map": "0.8.0",
         "concat-stream": "1.6.2",
         "is-buffer": "1.1.6",
-        "JSONStream": "1.3.2",
         "lexical-scope": "1.2.0",
+        "path-is-absolute": "1.0.1",
         "process": "0.11.10",
         "through2": "2.0.3",
         "xtend": "4.0.1"
@@ -6356,18 +6374,11 @@
       "integrity": "sha1-UFMN+4T8yap9vnhS6Do3uTufKqY="
     },
     "is-accessor-descriptor": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-      "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
+      "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
       "requires": {
-        "kind-of": "6.0.2"
-      },
-      "dependencies": {
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
-        }
+        "kind-of": "3.2.2"
       }
     },
     "is-arrayish": {
@@ -6409,18 +6420,11 @@
       "integrity": "sha1-hut1OSgF3cM69xySoO7fdO52BLI="
     },
     "is-data-descriptor": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-      "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
+      "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
       "requires": {
-        "kind-of": "6.0.2"
-      },
-      "dependencies": {
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
-        }
+        "kind-of": "3.2.2"
       }
     },
     "is-date-object": {
@@ -6429,19 +6433,19 @@
       "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY="
     },
     "is-descriptor": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-      "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+      "version": "0.1.6",
+      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
+      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
       "requires": {
-        "is-accessor-descriptor": "1.0.0",
-        "is-data-descriptor": "1.0.0",
-        "kind-of": "6.0.2"
+        "is-accessor-descriptor": "0.1.6",
+        "is-data-descriptor": "0.1.4",
+        "kind-of": "5.1.0"
       },
       "dependencies": {
         "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
+          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
         }
       }
     },
@@ -6937,16 +6941,6 @@
       "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.0.1.tgz",
       "integrity": "sha1-T9kss04OnbPInIYi7PUfm5eMbLk="
     },
-    "JSONStream": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.2.tgz",
-      "integrity": "sha1-wQI3G27Dp887hHygDCC7D85Mbeo=",
-      "dev": true,
-      "requires": {
-        "jsonparse": "1.3.1",
-        "through": "2.3.8"
-      }
-    },
     "jsprim": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
@@ -7029,7 +7023,7 @@
         "colors": "1.1.2",
         "combine-lists": "1.0.1",
         "connect": "3.6.6",
-        "core-js": "2.5.4",
+        "core-js": "2.5.5",
         "di": "0.0.1",
         "dom-serialize": "2.2.1",
         "expand-braces": "0.1.2",
@@ -7742,9 +7736,9 @@
       "integrity": "sha1-4PyVEztu8nbNyIh82vJKpvFW+Po="
     },
     "loglevelnext": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/loglevelnext/-/loglevelnext-1.0.3.tgz",
-      "integrity": "sha512-OCxd/b78TijTB4b6zVqLbMrxhebyvdZKwqpL0VHUZ0pYhavXaPD4l6Xrr4n5xqTYWiqtb0i7ikSoJY/myQ/Org=="
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/loglevelnext/-/loglevelnext-1.0.4.tgz",
+      "integrity": "sha512-V3N6LAJAiGwa/zjtvmgs2tyeiCJ23bGNhxXN8R+v7k6TNlSlTz40mIyZYdmO762eBnEFymn0Mhha+WuAhnwMBg=="
     },
     "lolex": {
       "version": "1.6.0",
@@ -8240,6 +8234,7 @@
       "integrity": "sha1-IyFYM/HaE/1gbMuAh7RIUty4If0=",
       "dev": true,
       "requires": {
+        "JSONStream": "1.3.2",
         "browser-resolve": "1.11.2",
         "cached-path-relative": "1.0.1",
         "concat-stream": "1.5.2",
@@ -8247,10 +8242,9 @@
         "detective": "4.7.1",
         "duplexer2": "0.1.4",
         "inherits": "2.0.3",
-        "JSONStream": "1.3.2",
         "parents": "1.0.1",
         "readable-stream": "2.3.6",
-        "resolve": "1.6.0",
+        "resolve": "1.7.0",
         "stream-combiner2": "1.1.1",
         "subarg": "1.0.0",
         "through2": "2.0.3",
@@ -8398,6 +8392,32 @@
             "is-extendable": "1.0.1"
           }
         },
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "requires": {
+            "kind-of": "6.0.2"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "requires": {
+            "kind-of": "6.0.2"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "requires": {
+            "is-accessor-descriptor": "1.0.0",
+            "is-data-descriptor": "1.0.0",
+            "kind-of": "6.0.2"
+          }
+        },
         "is-extendable": {
           "version": "1.0.1",
           "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
@@ -8744,39 +8764,6 @@
           "requires": {
             "is-descriptor": "0.1.6"
           }
-        },
-        "is-accessor-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-          "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
-          "requires": {
-            "kind-of": "3.2.2"
-          }
-        },
-        "is-data-descriptor": {
-          "version": "0.1.4",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-          "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
-          "requires": {
-            "kind-of": "3.2.2"
-          }
-        },
-        "is-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-          "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-          "requires": {
-            "is-accessor-descriptor": "0.1.6",
-            "is-data-descriptor": "0.1.4",
-            "kind-of": "5.1.0"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "5.1.0",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-              "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
-            }
-          }
         }
       }
     },
@@ -10116,7 +10103,7 @@
         "jstransformer": "1.0.0",
         "pug-error": "1.3.2",
         "pug-walk": "1.1.7",
-        "resolve": "1.6.0",
+        "resolve": "1.7.0",
         "uglify-js": "2.8.29"
       }
     },
@@ -10177,7 +10164,7 @@
       "requires": {
         "loader-utils": "1.1.0",
         "pug-walk": "1.1.7",
-        "resolve": "1.6.0"
+        "resolve": "1.7.0"
       }
     },
     "pug-parser": {
@@ -10357,6 +10344,11 @@
             "statuses": "1.4.0"
           }
         },
+        "iconv-lite": {
+          "version": "0.4.19",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
+          "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
+        },
         "setprototypeof": {
           "version": "1.0.3",
           "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz",
@@ -10490,7 +10482,7 @@
       "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
       "integrity": "sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q=",
       "requires": {
-        "resolve": "1.6.0"
+        "resolve": "1.7.0"
       }
     },
     "redent": {
@@ -10753,7 +10745,7 @@
       "integrity": "sha1-Tf5b9r6LjNw3/Pk+BLZVd3InEN4=",
       "requires": {
         "aws-sign2": "0.6.0",
-        "aws4": "1.6.0",
+        "aws4": "1.7.0",
         "caseless": "0.11.0",
         "combined-stream": "1.0.6",
         "extend": "3.0.1",
@@ -10799,9 +10791,9 @@
       "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
     },
     "resolve": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.6.0.tgz",
-      "integrity": "sha512-mw7JQNu5ExIkcw4LPih0owX/TZXjD/ZUF/ZQ/pDnkw3ZKhDcZZw5klmBlj6gVMwjQ3Pz5Jgu7F3d0jcDVuEWdw==",
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.7.0.tgz",
+      "integrity": "sha512-QdgZ5bjR1WAlpLaO5yHepFvC+o3rCr6wpfE2tpJNMkXdulf2jKomQBdNRQITF3ZKHNlT71syG98yQP03gasgnA==",
       "requires": {
         "path-parse": "1.0.5"
       }
@@ -10952,7 +10944,7 @@
         "acorn": "5.5.3",
         "estree-walker": "0.5.1",
         "magic-string": "0.22.5",
-        "resolve": "1.6.0",
+        "resolve": "1.7.0",
         "rollup-pluginutils": "2.0.1"
       }
     },
@@ -10963,7 +10955,7 @@
       "requires": {
         "builtin-modules": "2.0.0",
         "is-module": "1.0.0",
-        "resolve": "1.6.0"
+        "resolve": "1.7.0"
       }
     },
     "rollup-plugin-progress": {
@@ -11091,6 +11083,11 @@
         "ret": "0.1.15"
       }
     },
+    "safer-buffer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+    },
     "samsam": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.3.0.tgz",
@@ -11490,57 +11487,6 @@
           "requires": {
             "is-descriptor": "0.1.6"
           }
-        },
-        "is-accessor-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-          "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "is-data-descriptor": {
-          "version": "0.1.4",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-          "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "is-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-          "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-          "requires": {
-            "is-accessor-descriptor": "0.1.6",
-            "is-data-descriptor": "0.1.4",
-            "kind-of": "5.1.0"
-          }
-        },
-        "kind-of": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
         }
       }
     },
@@ -11808,7 +11754,7 @@
         "faye-websocket": "0.11.1",
         "inherits": "2.0.3",
         "json3": "3.3.2",
-        "url-parse": "1.2.0"
+        "url-parse": "1.3.0"
       },
       "dependencies": {
         "faye-websocket": {
@@ -11998,57 +11944,6 @@
           "requires": {
             "is-descriptor": "0.1.6"
           }
-        },
-        "is-accessor-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-          "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "is-data-descriptor": {
-          "version": "0.1.4",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-          "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "is-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-          "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-          "requires": {
-            "is-accessor-descriptor": "0.1.6",
-            "is-data-descriptor": "0.1.4",
-            "kind-of": "5.1.0"
-          }
-        },
-        "kind-of": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
         }
       }
     },
@@ -12156,14 +12051,6 @@
       "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz",
       "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM="
     },
-    "string_decoder": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-      "requires": {
-        "safe-buffer": "5.1.1"
-      }
-    },
     "string-template": {
       "version": "0.2.1",
       "resolved": "https://registry.npmjs.org/string-template/-/string-template-0.2.1.tgz",
@@ -12193,6 +12080,14 @@
         }
       }
     },
+    "string_decoder": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
+      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
+      "requires": {
+        "safe-buffer": "5.1.1"
+      }
+    },
     "stringstream": {
       "version": "0.0.5",
       "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz",
@@ -12313,29 +12208,20 @@
           "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg="
         },
         "braces": {
-          "version": "2.3.1",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.1.tgz",
-          "integrity": "sha512-SO5lYHA3vO6gz66erVvedSCkp7AKWdv6VcQ2N4ysXfPxdAlxAMMAdwegGGcv1Bqwm7naF1hNdk5d6AAIEHV2nQ==",
+          "version": "2.3.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
+          "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
           "requires": {
             "arr-flatten": "1.1.0",
             "array-unique": "0.3.2",
-            "define-property": "1.0.0",
             "extend-shallow": "2.0.1",
             "fill-range": "4.0.0",
             "isobject": "3.0.1",
-            "kind-of": "6.0.2",
             "repeat-element": "1.1.2",
             "snapdragon": "0.8.2",
             "snapdragon-node": "2.1.1",
             "split-string": "3.1.0",
             "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "6.0.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-              "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
-            }
           }
         },
         "clone": {
@@ -12393,52 +12279,6 @@
             "to-regex-range": "2.1.1"
           }
         },
-        "is-accessor-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-          "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "is-data-descriptor": {
-          "version": "0.1.4",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-          "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "is-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-          "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-          "requires": {
-            "is-accessor-descriptor": "0.1.6",
-            "is-data-descriptor": "0.1.4",
-            "kind-of": "5.1.0"
-          }
-        },
         "is-number": {
           "version": "3.0.0",
           "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
@@ -12474,7 +12314,7 @@
           "requires": {
             "arr-diff": "4.0.0",
             "array-unique": "0.3.2",
-            "braces": "2.3.1",
+            "braces": "2.3.2",
             "define-property": "1.0.0",
             "extend-shallow": "2.0.1",
             "extglob": "2.0.4",
@@ -12749,6 +12589,32 @@
             "is-extendable": "1.0.1"
           }
         },
+        "is-accessor-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
+          "requires": {
+            "kind-of": "6.0.2"
+          }
+        },
+        "is-data-descriptor": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
+          "requires": {
+            "kind-of": "6.0.2"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "requires": {
+            "is-accessor-descriptor": "1.0.0",
+            "is-data-descriptor": "1.0.0",
+            "kind-of": "6.0.2"
+          }
+        },
         "is-extendable": {
           "version": "1.0.1",
           "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
@@ -12761,6 +12627,11 @@
           "version": "3.0.1",
           "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
           "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
+        },
+        "kind-of": {
+          "version": "6.0.2",
+          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
+          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
         }
       }
     },
@@ -13121,9 +12992,9 @@
       "integrity": "sha1-TTNA6AfTdzvamZH4MFrNzCpmXSo="
     },
     "url-parse": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.2.0.tgz",
-      "integrity": "sha512-DT1XbYAfmQP65M/mE6OALxmXzZ/z1+e5zk2TcSKe/KiYbNGZxgtttzC0mR/sjopbpOXcbniq7eIKmocJnUWlEw==",
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.3.0.tgz",
+      "integrity": "sha512-zPvPA3T7P6M+0iNsgX+iAcAz4GshKrowtQBHHc/28tVsBc8jK7VRCNX+2GEcoE6zDB6XqXhcyiUWPVZY6C70Cg==",
       "requires": {
         "querystringify": "1.0.0",
         "requires-port": "1.0.0"
@@ -13393,17 +13264,15 @@
           "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg="
         },
         "braces": {
-          "version": "2.3.1",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.1.tgz",
-          "integrity": "sha512-SO5lYHA3vO6gz66erVvedSCkp7AKWdv6VcQ2N4ysXfPxdAlxAMMAdwegGGcv1Bqwm7naF1hNdk5d6AAIEHV2nQ==",
+          "version": "2.3.2",
+          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
+          "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
           "requires": {
             "arr-flatten": "1.1.0",
             "array-unique": "0.3.2",
-            "define-property": "1.0.0",
             "extend-shallow": "2.0.1",
             "fill-range": "4.0.0",
             "isobject": "3.0.1",
-            "kind-of": "6.0.2",
             "repeat-element": "1.1.2",
             "snapdragon": "0.8.2",
             "snapdragon-node": "2.1.1",
@@ -13411,14 +13280,6 @@
             "to-regex": "3.0.2"
           },
           "dependencies": {
-            "define-property": {
-              "version": "1.0.0",
-              "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-              "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
-              "requires": {
-                "is-descriptor": "1.0.2"
-              }
-            },
             "extend-shallow": {
               "version": "2.0.1",
               "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
@@ -13468,6 +13329,42 @@
                 "is-extendable": "0.1.1"
               }
             },
+            "is-accessor-descriptor": {
+              "version": "0.1.6",
+              "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
+              "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
+              "requires": {
+                "kind-of": "3.2.2"
+              },
+              "dependencies": {
+                "kind-of": {
+                  "version": "3.2.2",
+                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+                  "requires": {
+                    "is-buffer": "1.1.6"
+                  }
+                }
+              }
+            },
+            "is-data-descriptor": {
+              "version": "0.1.4",
+              "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
+              "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
+              "requires": {
+                "kind-of": "3.2.2"
+              },
+              "dependencies": {
+                "kind-of": {
+                  "version": "3.2.2",
+                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
+                  "requires": {
+                    "is-buffer": "1.1.6"
+                  }
+                }
+              }
+            },
             "is-descriptor": {
               "version": "0.1.6",
               "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
@@ -13559,39 +13456,29 @@
           }
         },
         "is-accessor-descriptor": {
-          "version": "0.1.6",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-          "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
+          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
           "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
+            "kind-of": "6.0.2"
           }
         },
         "is-data-descriptor": {
-          "version": "0.1.4",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-          "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
+          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
           "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
+            "kind-of": "6.0.2"
+          }
+        },
+        "is-descriptor": {
+          "version": "1.0.2",
+          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
+          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
+          "requires": {
+            "is-accessor-descriptor": "1.0.0",
+            "is-data-descriptor": "1.0.0",
+            "kind-of": "6.0.2"
           }
         },
         "is-number": {
@@ -13629,7 +13516,7 @@
           "requires": {
             "arr-diff": "4.0.0",
             "array-unique": "0.3.2",
-            "braces": "2.3.1",
+            "braces": "2.3.2",
             "define-property": "2.0.2",
             "extend-shallow": "3.0.2",
             "extglob": "2.0.4",
@@ -13695,7 +13582,7 @@
           "integrity": "sha512-y7ANxCWmMW8xLOaiopiRDlyjQ9ajKRENBH+2wjntIbk3A6ZR1+BLQttkmSHMY7Arl+AAZFwJ10grg2T6f1WI8A==",
           "requires": {
             "ast-types": "0.10.1",
-            "core-js": "2.5.4",
+            "core-js": "2.5.5",
             "esprima": "4.0.0",
             "private": "0.1.8",
             "source-map": "0.6.1"
@@ -13905,7 +13792,7 @@
         "path-is-absolute": "1.0.1",
         "range-parser": "1.2.0",
         "url-join": "4.0.0",
-        "webpack-log": "1.1.2"
+        "webpack-log": "1.2.0"
       },
       "dependencies": {
         "mime": {
@@ -13946,7 +13833,7 @@
         "strip-ansi": "3.0.1",
         "supports-color": "5.3.0",
         "webpack-dev-middleware": "3.0.1",
-        "webpack-log": "1.1.2",
+        "webpack-log": "1.2.0",
         "yargs": "9.0.1"
       },
       "dependencies": {
@@ -14006,13 +13893,13 @@
       }
     },
     "webpack-log": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/webpack-log/-/webpack-log-1.1.2.tgz",
-      "integrity": "sha512-B53SD4N4BHpZdUwZcj4st2QT7gVfqZtqHDruC1N+K2sciq0Rt/3F1Dx6RlylVkcrToMLTaiaeT48k9Lq4iDVDA==",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/webpack-log/-/webpack-log-1.2.0.tgz",
+      "integrity": "sha512-U9AnICnu50HXtiqiDxuli5gLB5PGBo7VvcHx36jRZHwK4vzOYLbImqT4lwWwoMHdQWwEKw736fCHEekokTEKHA==",
       "requires": {
         "chalk": "2.3.2",
         "log-symbols": "2.2.0",
-        "loglevelnext": "1.0.3",
+        "loglevelnext": "1.0.4",
         "uuid": "3.2.1"
       },
       "dependencies": {

From 8d3755b9c58eef12c5fc9cabfc0b1c05f6db716e Mon Sep 17 00:00:00 2001
From: Semyon Boikov <sboikov@apache.org>
Date: Tue, 10 Apr 2018 11:37:39 +0300
Subject: [PATCH 009/543] IGNITE-7222 Added ZooKeeper discovery SPI

---
 .../JdbcAbstractDmlStatementSelfTest.java     |    6 +-
 .../CommunicationFailureContext.java          |   62 +
 .../CommunicationFailureResolver.java         |   28 +
 .../DefaultCommunicationFailureResolver.java  |  305 ++
 .../configuration/IgniteConfiguration.java    |   22 +
 .../org/apache/ignite/internal/GridTopic.java |    3 +
 .../apache/ignite/internal/IgniteKernal.java  |    5 +-
 .../apache/ignite/internal/IgnitionEx.java    |    3 +
 .../internal/managers/GridManagerAdapter.java |    8 +
 .../managers/communication/GridIoManager.java |    4 +-
 .../communication/GridIoMessageFactory.java   |   12 +
 .../discovery/CustomMessageWrapper.java       |    5 +
 .../managers/discovery/DiscoCache.java        |    8 +
 .../discovery/DiscoveryCustomMessage.java     |   10 +-
 .../DiscoveryMessageResultsCollector.java     |  222 +
 .../discovery/GridDiscoveryManager.java       |  128 +-
 .../managers/discovery/IgniteClusterNode.java |   69 +
 .../discovery/IgniteDiscoverySpi.java         |   67 +
 .../IgniteDiscoverySpiInternalListener.java   |   42 +
 .../authentication/UserAcceptedMessage.java   |    5 +
 .../authentication/UserProposedMessage.java   |    5 +
 .../cache/CacheAffinityChangeMessage.java     |    5 +
 .../cache/CacheAffinitySharedManager.java     |   26 +-
 .../CacheStatisticsModeChangeMessage.java     |    5 +
 .../ClientCacheChangeDiscoveryMessage.java    |    5 +
 ...lientCacheChangeDummyDiscoveryMessage.java |    5 +
 .../cache/DynamicCacheChangeBatch.java        |    5 +
 .../processors/cache/GridCacheAdapter.java    |    3 +-
 .../GridCachePartitionExchangeManager.java    |   16 +-
 .../processors/cache/GridCacheProcessor.java  |    4 +-
 .../processors/cache/GridCacheUtils.java      |    6 +-
 .../cache/WalStateFinishMessage.java          |    5 +
 .../cache/WalStateProposeMessage.java         |    5 +
 .../cache/binary/BinaryMetadataTransport.java |   24 +-
 .../binary/MetadataUpdateAcceptedMessage.java |    5 +
 .../binary/MetadataUpdateProposedMessage.java |    5 +
 .../dht/GridClientPartitionTopology.java      |   39 +
 .../distributed/dht/GridDhtCacheAdapter.java  |    4 +
 .../dht/GridDhtPartitionTopology.java         |    6 +
 .../dht/GridDhtPartitionTopologyImpl.java     |   39 +
 .../GridDhtPartitionsExchangeFuture.java      |   26 +-
 .../ChangeGlobalStateFinishMessage.java       |    5 +
 .../cluster/ChangeGlobalStateMessage.java     |    5 +
 .../cluster/ClusterMetricsUpdateMessage.java  |  158 +
 .../cluster/ClusterNodeMetrics.java           |   62 +
 .../processors/cluster/ClusterProcessor.java  |  249 +-
 .../continuous/AbstractContinuousMessage.java |    5 +
 .../continuous/ContinuousRoutineInfo.java     |  100 +
 .../ContinuousRoutineStartResultMessage.java  |  206 +
 ...ContinuousRoutinesCommonDiscoveryData.java |   45 +
 .../continuous/ContinuousRoutinesInfo.java    |  132 +
 ...nuousRoutinesJoiningNodeDiscoveryData.java |   45 +
 .../continuous/GridContinuousProcessor.java   |  862 ++-
 .../continuous/StartRequestDataV2.java        |  164 +
 .../StartRoutineDiscoveryMessageV2.java       |   77 +
 .../StopRoutineAckDiscoveryMessage.java       |    5 +
 .../datastreamer/DataStreamerImpl.java        |   27 +-
 .../marshaller/MappingAcceptedMessage.java    |    5 +
 .../marshaller/MappingProposedMessage.java    |    5 +
 .../message/SchemaFinishDiscoveryMessage.java |    5 +
 .../SchemaProposeDiscoveryMessage.java        |    5 +
 .../internal/util/nio/GridNioServer.java      |   18 +-
 .../apache/ignite/spi/IgniteSpiAdapter.java   |   10 +
 .../apache/ignite/spi/IgniteSpiContext.java   |   11 +
 .../tcp/TcpCommunicationSpi.java              |  190 +-
 .../tcp/internal/ConnectionKey.java           |  117 +
 ...TcpCommunicationConnectionCheckFuture.java |  519 ++
 ...ommunicationNodeConnectionCheckFuture.java |   30 +
 .../discovery/DiscoverySpiCustomMessage.java  |   15 +-
 ...scoverySpiMutableCustomMessageSupport.java |   40 +
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   55 +-
 .../tcp/internal/TcpDiscoveryNode.java        |   32 +-
 .../resources/META-INF/classnames.properties  |    2 +
 ...ctionExcludeNeighborsAbstractSelfTest.java |    8 +-
 .../failure/FailureHandlerTriggeredTest.java  |    4 +
 .../internal/ClusterGroupHostsSelfTest.java   |    3 +
 .../ignite/internal/ClusterGroupSelfTest.java |    2 +
 .../ClusterNodeMetricsUpdateTest.java         |  173 +
 .../internal/DiscoverySpiTestListener.java    |  162 +
 .../internal/GridDiscoverySelfTest.java       |   14 +-
 .../GridJobMasterLeaveAwareSelfTest.java      |    2 +
 .../internal/GridJobStealingSelfTest.java     |    2 +
 .../internal/GridSameVmStartupSelfTest.java   |   19 +-
 .../apache/ignite/internal/GridSelfTest.java  |    2 +
 .../IgniteClientReconnectAbstractTest.java    |   53 +-
 ...IgniteClientReconnectApiExceptionTest.java |   21 +-
 .../IgniteClientReconnectAtomicsTest.java     |   30 +-
 .../IgniteClientReconnectCacheTest.java       |   49 +-
 .../IgniteClientReconnectCollectionsTest.java |   14 +-
 .../IgniteClientReconnectComputeTest.java     |    6 +-
 ...lientReconnectContinuousProcessorTest.java |   13 +-
 ...niteClientReconnectDiscoveryStateTest.java |   22 +-
 ...teClientReconnectFailoverAbstractTest.java |   12 +-
 .../IgniteClientReconnectServicesTest.java    |    8 +-
 .../IgniteClientReconnectStopTest.java        |   12 +-
 .../IgniteClientReconnectStreamerTest.java    |    4 +-
 .../internal/IgniteClientRejoinTest.java      |    3 +
 ...ridDiscoveryManagerAliveCacheSelfTest.java |   16 +-
 ...GridAffinityProcessorAbstractSelfTest.java |    4 +-
 .../CacheMetricsForClusterGroupSelfTest.java  |   12 +-
 .../cache/GridCacheAbstractSelfTest.java      |    2 +
 .../IgniteCacheNearLockValueSelfTest.java     |    4 +-
 .../IgniteCacheP2pUnmarshallingErrorTest.java |   11 +
 .../IgniteClusterActivateDeactivateTest.java  |   65 +
 .../IgniteDaemonNodeMarshallerCacheTest.java  |    3 +-
 .../binary/BinaryMetadataUpdatesFlowTest.java |   12 +-
 ...NodeBinaryObjectMetadataMultinodeTest.java |    2 +-
 .../GridCacheQueueClientDisconnectTest.java   |   10 +
 ...gniteClientDataStructuresAbstractTest.java |    3 +-
 .../CacheLateAffinityAssignmentTest.java      |  127 +-
 .../GridCacheNodeFailureAbstractTest.java     |    5 +-
 .../IgniteCache150ClientsTest.java            |    2 +
 .../IgniteCacheManyClientsTest.java           |   44 +-
 .../IgniteOptimisticTxSuspendResumeTest.java  |    2 +
 ...dCacheDhtPreloadMultiThreadedSelfTest.java |    4 +
 .../dht/GridCacheDhtPreloadSelfTest.java      |    2 +
 .../dht/TxRecoveryStoreEnabledTest.java       |   15 +-
 ...tionedExplicitLockNodeFailureSelfTest.java |    3 +-
 .../ClientReconnectContinuousQueryTest.java   |   19 +-
 ...emoteFilterMissingInClassPathSelfTest.java |   23 +-
 ...cheContinuousQueryClientReconnectTest.java |    3 +
 .../CacheVersionedEntryAbstractTest.java      |   33 +-
 .../continuous/GridEventConsumeSelfTest.java  |   34 +-
 .../ClosureServiceClientsNodesTest.java       |   19 +-
 .../internal/util/GridTestClockTimer.java     |    9 +
 .../GridMarshallerMappingConsistencyTest.java |    4 +
 .../messaging/GridMessagingSelfTest.java      |  126 +-
 .../GridTcpCommunicationSpiAbstractTest.java  |   71 +
 .../FilterDataForClientNodeDiscoveryTest.java |    5 +
 .../testframework/GridSpiTestContext.java     |   10 +
 .../config/GridTestProperties.java            |    9 +
 .../junits/GridAbstractTest.java              |  129 +-
 .../junits/multijvm/IgniteNodeRunner.java     |    2 +
 .../IgniteComputeGridTestSuite.java           |    2 +
 ...teCacheDistributedQueryCancelSelfTest.java |    2 +-
 .../DynamicIndexAbstractBasicSelfTest.java    |    5 +-
 .../GridJtaTransactionManagerSelfTest.java    |   21 +-
 ...ridPartitionedCacheJtaFactorySelfTest.java |   19 +-
 .../org/apache/ignite/spark/IgniteRDD.scala   |    9 +-
 .../ignite/internal/GridFactorySelfTest.java  |    3 +-
 .../p2p/GridP2PUserVersionChangeSelfTest.java |    5 +-
 modules/yardstick/pom-standalone.xml          |    6 +
 modules/yardstick/pom.xml                     |    6 +
 modules/zookeeper/pom.xml                     |   40 +
 .../discovery/zk/ZookeeperDiscoverySpi.java   |  557 ++
 .../zk/internal/ZkAbstractCallabck.java       |   83 +
 .../internal/ZkAbstractChildrenCallback.java  |   61 +
 .../zk/internal/ZkAbstractWatcher.java        |   55 +
 .../zk/internal/ZkAliveNodeData.java          |   40 +
 .../zk/internal/ZkBulkJoinContext.java        |   50 +
 .../discovery/zk/internal/ZkClusterNodes.java |  103 +
 .../ZkCommunicationErrorNodeState.java        |   46 +
 .../ZkCommunicationErrorProcessFuture.java    |  411 ++
 ...ommunicationErrorResolveFinishMessage.java |   69 +
 .../ZkCommunicationErrorResolveResult.java    |   45 +
 ...CommunicationErrorResolveStartMessage.java |   61 +
 .../ZkCommunicationFailureContext.java        |  188 +
 .../internal/ZkDiscoveryCustomEventData.java  |   89 +
 .../zk/internal/ZkDiscoveryEventData.java     |  165 +
 .../zk/internal/ZkDiscoveryEventsData.java    |  121 +
 .../ZkDiscoveryNodeFailEventData.java         |   55 +
 .../ZkDiscoveryNodeJoinEventData.java         |   60 +
 .../ZkDistributedCollectDataFuture.java       |  250 +
 .../zk/internal/ZkForceNodeFailMessage.java   |   65 +
 .../discovery/zk/internal/ZkIgnitePaths.java  |  307 ++
 .../internal/ZkInternalJoinErrorMessage.java  |   44 +
 .../zk/internal/ZkInternalMessage.java        |   27 +
 .../zk/internal/ZkJoinEventDataForJoined.java |   83 +
 .../zk/internal/ZkJoinedNodeEvtData.java      |   79 +
 .../zk/internal/ZkJoiningNodeData.java        |   87 +
 .../zk/internal/ZkNoServersMessage.java       |   50 +
 .../zk/internal/ZkNodeValidateResult.java     |   43 +
 .../spi/discovery/zk/internal/ZkRunnable.java |   51 +
 .../discovery/zk/internal/ZkRuntimeState.java |  135 +
 .../zk/internal/ZkTimeoutObject.java          |   54 +
 .../zk/internal/ZookeeperClient.java          | 1219 +++++
 .../ZookeeperClientFailedException.java       |   40 +
 .../zk/internal/ZookeeperClusterNode.java     |  362 ++
 .../zk/internal/ZookeeperDiscoveryImpl.java   | 4464 +++++++++++++++
 .../java/org/apache/ZookeeperNodeStart.java   |   46 +
 ...cheEntryListenerWithZkDiscoAtomicTest.java |   32 +
 ...ookeeperDiscoverySpiAbstractTestSuite.java |  118 +
 .../zk/ZookeeperDiscoverySpiTestSuite1.java   |   44 +
 .../zk/ZookeeperDiscoverySpiTestSuite2.java   |   94 +
 ...okeeperDiscoverySuitePreprocessorTest.java |  101 +
 .../zk/internal/ZookeeperClientTest.java      |  495 ++
 ...eeperDiscoverySpiSaslAuthAbstractTest.java |  247 +
 ...okeeperDiscoverySpiSaslFailedAuthTest.java |   44 +
 ...perDiscoverySpiSaslSuccessfulAuthTest.java |   48 +
 .../internal/ZookeeperDiscoverySpiTest.java   | 4847 +++++++++++++++++
 .../zookeeper/ZkTestClientCnxnSocketNIO.java  |  137 +
 191 files changed, 21158 insertions(+), 777 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureContext.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureResolver.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryMessageResultsCollector.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpiInternalListener.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterNodeMetrics.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineInfo.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesCommonDiscoveryData.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesInfo.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesJoiningNodeDiscoveryData.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRequestDataV2.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRoutineDiscoveryMessageV2.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMutableCustomMessageSupport.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsUpdateTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/DiscoverySpiTestListener.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAliveNodeData.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkBulkJoinContext.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkClusterNodes.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventData.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeFailEventData.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeJoinEventData.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalJoinErrorMessage.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalMessage.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinEventDataForJoined.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinedNodeEvtData.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoiningNodeData.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNoServersMessage.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNodeValidateResult.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkTimeoutObject.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientFailedException.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ZookeeperNodeStart.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/IgniteCacheEntryListenerWithZkDiscoAtomicTest.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySuitePreprocessorTest.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslAuthAbstractTest.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslFailedAuthTest.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslSuccessfulAuthTest.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/zookeeper/ZkTestClientCnxnSocketNIO.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
index f4c0ca3464ed6..0a055a9c8679f 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcAbstractDmlStatementSelfTest.java
@@ -138,8 +138,10 @@ protected String getCfgUrl() {
     @Override protected void afterTest() throws Exception {
         ((IgniteEx)ignite(0)).context().cache().dynamicDestroyCache(DEFAULT_CACHE_NAME, true, true, false);
 
-        conn.close();
-        assertTrue(conn.isClosed());
+        if (conn != null) {
+            conn.close();
+            assertTrue(conn.isClosed());
+        }
 
         cleanUpWorkingDir();
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureContext.java b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureContext.java
new file mode 100644
index 0000000000000..a32d38c65be60
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureContext.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.spi.communication.CommunicationSpi;
+
+/**
+ * Communication Failure Context.
+ */
+public interface CommunicationFailureContext {
+    /**
+     * @return Current topology snapshot.
+     */
+    public List<ClusterNode> topologySnapshot();
+
+    /**
+     * @param node1 First node.
+     * @param node2 Second node.
+     * @return {@code True} if {@link CommunicationSpi} is able to establish connection from first node to second node.
+     */
+    public boolean connectionAvailable(ClusterNode node1, ClusterNode node2);
+
+    /**
+     * @return Currently started caches.
+     */
+    public Map<String, CacheConfiguration<?, ?>> startedCaches();
+
+    /**
+     * @param cacheName Cache name.
+     * @return Cache partitions affinity assignment.
+     */
+    public List<List<ClusterNode>> cacheAffinity(String cacheName);
+
+    /**
+     * @param cacheName Cache name.
+     * @return Cache partitions owners.
+     */
+    public List<List<ClusterNode>> cachePartitionOwners(String cacheName);
+
+    /**
+     * @param node Node to kill.
+     */
+    public void killNode(ClusterNode node);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureResolver.java b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureResolver.java
new file mode 100644
index 0000000000000..a4d92f33c73f0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/CommunicationFailureResolver.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+/**
+ * Communication Failure Resolver.
+ */
+public interface CommunicationFailureResolver {
+    /**
+     * @param ctx Context.
+     */
+    public void resolve(CommunicationFailureContext ctx);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
new file mode 100644
index 0000000000000..a4c6da9e9986b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.configuration;
+
+import java.util.BitSet;
+import java.util.List;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.resources.LoggerResource;
+
+/**
+ * Default Communication Failure Resolver.
+ */
+public class DefaultCommunicationFailureResolver implements CommunicationFailureResolver {
+    /** */
+    @LoggerResource
+    private IgniteLogger log;
+
+    /** {@inheritDoc} */
+    @Override public void resolve(CommunicationFailureContext ctx) {
+        ClusterGraph graph = new ClusterGraph(log, ctx);
+
+        ClusterSearch cluster = graph.findLargestIndependentCluster();
+
+        List<ClusterNode> nodes = ctx.topologySnapshot();
+
+        assert nodes.size() > 0;
+        assert cluster != null;
+
+        if (graph.checkFullyConnected(cluster.nodesBitSet)) {
+            assert cluster.nodeCnt <= nodes.size();
+
+            if (cluster.nodeCnt < nodes.size()) {
+                if (log.isInfoEnabled()) {
+                    log.info("Communication problem resolver found fully connected independent cluster [" +
+                        "clusterSrvCnt=" + cluster.srvCnt +
+                        ", clusterTotalNodes=" + cluster.nodeCnt +
+                        ", totalAliveNodes=" + nodes.size() + "]");
+                }
+
+                for (int i = 0; i < nodes.size(); i++) {
+                    if (!cluster.nodesBitSet.get(i))
+                        ctx.killNode(nodes.get(i));
+                }
+            }
+            else
+                U.warn(log, "All alive nodes are fully connected, this should be resolved automatically.");
+        }
+        else {
+            if (log.isInfoEnabled()) {
+                log.info("Communication problem resolver failed to find fully connected independent cluster.");
+            }
+        }
+    }
+
+    /**
+     * @param cluster Cluster nodes mask.
+     * @param nodes Nodes.
+     * @param limit IDs limit.
+     * @return Cluster node IDs string.
+     */
+    private static String clusterNodeIds(BitSet cluster, List<ClusterNode> nodes, int limit) {
+        int startIdx = 0;
+
+        StringBuilder builder = new StringBuilder();
+
+        int cnt = 0;
+
+        for (;;) {
+            int idx = cluster.nextSetBit(startIdx);
+
+            if (idx == -1)
+                break;
+
+            startIdx = idx + 1;
+
+            if (builder.length() == 0) {
+                builder.append('[');
+            }
+            else
+                builder.append(", ");
+
+            builder.append(nodes.get(idx).id());
+
+            if (cnt++ > limit)
+                builder.append(", ...");
+        }
+
+        builder.append(']');
+
+        return builder.toString();
+    }
+
+    /**
+     *
+     */
+    private static class ClusterSearch {
+        /** */
+        int srvCnt;
+
+        /** */
+        int nodeCnt;
+
+        /** */
+        final BitSet nodesBitSet;
+
+        /**
+         * @param nodes Total nodes.
+         */
+        ClusterSearch(int nodes) {
+            nodesBitSet = new BitSet(nodes);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class ClusterGraph {
+        /** */
+        private final static int WORD_IDX_SHIFT = 6;
+
+        /** */
+        private final IgniteLogger log;
+
+        /** */
+        private final int nodeCnt;
+
+        /** */
+        private final long[] visitBitSet;
+
+        /** */
+        private final CommunicationFailureContext ctx;
+
+        /** */
+        private final List<ClusterNode> nodes;
+
+        /**
+         * @param log Logger.
+         * @param ctx Context.
+         */
+        ClusterGraph(IgniteLogger log, CommunicationFailureContext ctx) {
+            this.log = log;
+            this.ctx = ctx;
+
+            nodes = ctx.topologySnapshot();
+
+            nodeCnt = nodes.size();
+
+            assert nodeCnt > 0;
+
+            visitBitSet = initBitSet(nodeCnt);
+        }
+
+        /**
+         * @param bitIndex Bit index.
+         * @return Word index containing bit with given index.
+         */
+        private static int wordIndex(int bitIndex) {
+            return bitIndex >> WORD_IDX_SHIFT;
+        }
+
+        /**
+         * @param bitCnt Number of bits.
+         * @return Bit set words.
+         */
+        static long[] initBitSet(int bitCnt) {
+            return new long[wordIndex(bitCnt - 1) + 1];
+        }
+
+        /**
+         * @return Cluster nodes bit set.
+         */
+        ClusterSearch findLargestIndependentCluster() {
+            ClusterSearch maxCluster = null;
+
+            for (int i = 0; i < nodeCnt; i++) {
+                if (getBit(visitBitSet, i))
+                    continue;
+
+                ClusterSearch cluster = new ClusterSearch(nodeCnt);
+
+                search(cluster, i);
+
+                if (log.isInfoEnabled()) {
+                    log.info("Communication problem resolver found cluster [srvCnt=" + cluster.srvCnt +
+                        ", totalNodeCnt=" + cluster.nodeCnt +
+                        ", nodeIds=" + clusterNodeIds(cluster.nodesBitSet, nodes, 1000) + "]");
+                }
+
+                if (maxCluster == null || cluster.srvCnt > maxCluster.srvCnt)
+                    maxCluster = cluster;
+            }
+
+            return maxCluster;
+        }
+
+        /**
+         * @param cluster Cluster nodes bit set.
+         * @return {@code True} if all cluster nodes are able to connect to each other.
+         */
+        boolean checkFullyConnected(BitSet cluster) {
+            int startIdx = 0;
+
+            int clusterNodes = cluster.cardinality();
+
+            for (;;) {
+                int idx = cluster.nextSetBit(startIdx);
+
+                if (idx == -1)
+                    break;
+
+                ClusterNode node1 = nodes.get(idx);
+
+                for (int i = 0; i < clusterNodes; i++) {
+                    if (!cluster.get(i) || i == idx)
+                        continue;
+
+                    ClusterNode node2 = nodes.get(i);
+
+                    if (cluster.get(i) && !ctx.connectionAvailable(node1, node2))
+                        return false;
+                }
+
+                startIdx = idx + 1;
+            }
+
+            return true;
+        }
+
+        /**
+         * @param cluster Current cluster bit set.
+         * @param idx Node index.
+         */
+        void search(ClusterSearch cluster, int idx) {
+            assert !getBit(visitBitSet, idx);
+
+            setBit(visitBitSet, idx);
+
+            cluster.nodesBitSet.set(idx);
+            cluster.nodeCnt++;
+
+            ClusterNode node1 = nodes.get(idx);
+
+            if (!CU.clientNode(node1))
+                cluster.srvCnt++;
+
+            for (int i = 0; i < nodeCnt; i++) {
+                if (i == idx || getBit(visitBitSet, i))
+                    continue;
+
+                ClusterNode node2 = nodes.get(i);
+
+                boolean connected = ctx.connectionAvailable(node1, node2) ||
+                    ctx.connectionAvailable(node2, node1);
+
+                if (connected)
+                    search(cluster, i);
+            }
+        }
+
+        /**
+         * @param words Bit set words.
+         * @param bitIndex Bit index.
+         */
+        static void setBit(long words[], int bitIndex) {
+            int wordIndex = wordIndex(bitIndex);
+
+            words[wordIndex] |= (1L << bitIndex);
+        }
+
+        /**
+         * @param words Bit set words.
+         * @param bitIndex Bit index.
+         * @return Bit value.
+         */
+        static boolean getBit(long[] words, int bitIndex) {
+            int wordIndex = wordIndex(bitIndex);
+
+            return (words[wordIndex] & (1L << bitIndex)) != 0;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DefaultCommunicationFailureResolver.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index add388045d1e4..cc3ea1006bef1 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -493,6 +493,9 @@ public class IgniteConfiguration {
     /** Failure handler. */
     private FailureHandler failureHnd;
 
+    /** Communication failure resolver */
+    private CommunicationFailureResolver commFailureRslvr;
+
     /**
      * Creates valid grid configuration with all default values.
      */
@@ -520,6 +523,8 @@ public IgniteConfiguration(IgniteConfiguration cfg) {
         loadBalancingSpi = cfg.getLoadBalancingSpi();
         indexingSpi = cfg.getIndexingSpi();
 
+        commFailureRslvr = cfg.getCommunicationFailureResolver();
+
         /*
          * Order alphabetically for maintenance purposes.
          */
@@ -606,6 +611,23 @@ public IgniteConfiguration(IgniteConfiguration cfg) {
         authEnabled = cfg.isAuthenticationEnabled();
     }
 
+    /**
+     * @return Communication failure resovler.
+     */
+    public CommunicationFailureResolver getCommunicationFailureResolver() {
+        return commFailureRslvr;
+    }
+
+    /**
+     * @param commFailureRslvr Communication failure resovler.
+     * @return {@code this} instance.
+     */
+    public IgniteConfiguration setCommunicationFailureResolver(CommunicationFailureResolver commFailureRslvr) {
+        this.commFailureRslvr = commFailureRslvr;
+
+        return this;
+    }
+
     /**
      * Gets optional grid name. Returns {@code null} if non-default grid name was not
      * provided.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 4932e671376dc..1227e8cf4abd4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -120,6 +120,9 @@ public enum GridTopic {
     /** */
     TOPIC_WAL,
 
+    /** */
+    TOPIC_METRICS,
+
     /** */
     TOPIC_AUTH;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 8bc46fd1db2f6..0b102e5b665f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1298,7 +1298,7 @@ private long checkPoolStarvation(
         ackStart(rtBean);
 
         if (!isDaemon())
-            ctx.discovery().ackTopology(localNode().order());
+            ctx.discovery().ackTopology(ctx.discovery().localJoin().joinTopologyVersion().topologyVersion());
     }
 
     /**
@@ -2623,6 +2623,9 @@ private Iterable<Object> lifecycleAwares(IgniteConfiguration cfg) {
         objs.add(cfg.getGridLogger());
         objs.add(cfg.getMBeanServer());
 
+        if (cfg.getCommunicationFailureResolver() != null)
+            objs.add(cfg.getCommunicationFailureResolver());
+
         return objs;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 8073faab8c8fa..3abc7111c43a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -75,6 +75,7 @@
 import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
 import org.apache.ignite.internal.processors.igfs.IgfsThreadFactory;
 import org.apache.ignite.internal.processors.igfs.IgfsUtils;
@@ -2243,6 +2244,8 @@ private IgniteConfiguration initializeConfiguration(IgniteConfiguration cfg)
 
             initializeDefaultSpi(myCfg);
 
+            GridDiscoveryManager.initCommunicationErrorResolveConfiguration(myCfg);
+
             initializeDefaultCacheConfiguration(myCfg);
 
             ExecutorConfiguration[] execCfgs = myCfg.getExecutorConfiguration();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
index 74f5a102d2f19..b0756cfe80801 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/GridManagerAdapter.java
@@ -618,6 +618,14 @@ protected final String stopInfo() {
                         return ctx.nodeAttributes();
                     }
 
+                    @Override public boolean communicationFailureResolveSupported() {
+                        return ctx.discovery().communicationErrorResolveSupported();
+                    }
+
+                    @Override public void resolveCommunicationFailure(ClusterNode node, Exception err) {
+                        ctx.discovery().resolveCommunicationError(node, err);
+                    }
+
                     /**
                      * @param e Exception to handle.
                      * @return GridSpiException Converted exception.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index d5cdd2dd7fc53..8d9a70034aeb3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -298,9 +298,9 @@ public void resetMetrics() {
 
                 @Override public MessageReader reader(UUID rmtNodeId, MessageFactory msgFactory)
                     throws IgniteCheckedException {
-                    assert rmtNodeId != null;
 
-                    return new DirectMessageReader(msgFactory, U.directProtocolVersion(ctx, rmtNodeId));
+                    return new DirectMessageReader(msgFactory,
+                        rmtNodeId != null ? U.directProtocolVersion(ctx, rmtNodeId) : GridIoManager.DIRECT_PROTO_VER);
                 }
             };
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index a0fc2f8088287..5616fd035d445 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -123,6 +123,8 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheRawVersionedEntry;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx;
+import org.apache.ignite.internal.processors.cluster.ClusterMetricsUpdateMessage;
+import org.apache.ignite.internal.processors.continuous.ContinuousRoutineStartResultMessage;
 import org.apache.ignite.internal.processors.continuous.GridContinuousMessage;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamerEntry;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamerRequest;
@@ -909,6 +911,16 @@ public GridIoMessageFactory(MessageFactory[] ext) {
 
                 break;
 
+            case 133:
+                msg = new ClusterMetricsUpdateMessage();
+
+                break;
+
+            case 134:
+                msg = new ContinuousRoutineStartResultMessage();
+
+                break;
+
             // [-3..119] [124..129] [-23..-27] [-36..-55]- this
             // [120..123] - DR
             // [-4..-22, -30..-35] - SQL
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
index 426888631729d..4b6b7a2cfabf7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
@@ -49,6 +49,11 @@ public class CustomMessageWrapper implements DiscoverySpiCustomMessage {
         return delegate.isMutable();
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return delegate.stopProcess();
+    }
+
     /**
      * @return Delegate.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index c21698f4b71ed..fef44fad30bb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -311,6 +311,14 @@ public boolean baselineNode(ClusterNode node) {
         return null;
     }
 
+    /**
+     * @param nodeId Node ID.
+     * @return {@code True} if node is in alives list.
+     */
+    public boolean alive(UUID nodeId) {
+        return alives.contains(nodeId);
+    }
+
     /**
      * Gets all nodes that have cache with given name.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryCustomMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryCustomMessage.java
index c708c6247b2c2..6ed2096faf386 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryCustomMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryCustomMessage.java
@@ -20,6 +20,7 @@
 import java.io.Serializable;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddedMessage;
 import org.jetbrains.annotations.Nullable;
@@ -87,10 +88,17 @@ public interface DiscoveryCustomMessage extends Serializable {
     @Nullable public DiscoveryCustomMessage ackMessage();
 
     /**
-     * @return {@code true} if message can be modified during listener notification. Changes will be send to next nodes.
+     * @return {@code True} if message can be modified during listener notification. Changes will be sent to next nodes.
      */
     public boolean isMutable();
 
+    /**
+     * See {@link DiscoverySpiCustomMessage#stopProcess()}.
+     *
+     * @return {@code True} if message should not be sent to others nodes after it was processed on coordinator.
+     */
+    public boolean stopProcess();
+
     /**
      * Creates new discovery cache if message caused topology version change.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryMessageResultsCollector.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryMessageResultsCollector.java
new file mode 100644
index 0000000000000..43be952ea9137
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoveryMessageResultsCollector.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public abstract class DiscoveryMessageResultsCollector<M, R>  {
+    /** */
+    private final Map<UUID, NodeMessage<M>> rcvd = new HashMap<>();
+
+    /** */
+    private int leftMsgs;
+
+    /** */
+    protected DiscoCache discoCache;
+
+    /** */
+    protected final GridKernalContext ctx;
+
+    /**
+     * @param ctx Context.
+     */
+    protected DiscoveryMessageResultsCollector(GridKernalContext ctx) {
+        this.ctx = ctx;
+    }
+
+    /**
+     * @param rcvd Received messages.
+     * @return Result.
+     */
+    protected abstract R createResult(Map<UUID, NodeMessage<M>> rcvd);
+
+    /**
+     * @param r Result.
+     */
+    protected abstract void onResultsCollected(R r);
+
+    /**
+     * @param discoCache Discovery state when discovery message was received.
+     * @param node Node.
+     * @return {@code True} if need wait for result from given node.
+     */
+    protected abstract boolean waitForNode(DiscoCache discoCache, ClusterNode node);
+
+    /**
+     * @param discoCache Discovery state.
+     */
+    public final void init(DiscoCache discoCache) {
+        assert discoCache != null;
+
+        R res = null;
+
+        synchronized (this) {
+            assert this.discoCache == null;
+            assert leftMsgs == 0 : leftMsgs;
+
+            this.discoCache = discoCache;
+
+            for (ClusterNode node : discoCache.allNodes()) {
+                if (ctx.discovery().alive(node) && waitForNode(discoCache, node) && !rcvd.containsKey(node.id())) {
+                    rcvd.put(node.id(), new NodeMessage<>((M)null));
+
+                    leftMsgs++;
+                }
+            }
+
+            if (leftMsgs == 0)
+                res = createResult(rcvd);
+        }
+
+        if (res != null)
+            onResultsCollected(res);
+    }
+
+    /**
+     * @param nodeId Node ID.
+     * @param msg Message.
+     */
+    public final void onMessage(UUID nodeId, M msg) {
+        R res = null;
+
+        synchronized (this) {
+            if (allReceived())
+                return;
+
+            NodeMessage<M> expMsg = rcvd.get(nodeId);
+
+            if (expMsg == null)
+                rcvd.put(nodeId, new NodeMessage<>(msg));
+            else if (expMsg.set(msg)) {
+                assert leftMsgs > 0;
+
+                leftMsgs--;
+
+                if (allReceived())
+                    res = createResult(rcvd);
+            }
+        }
+
+        if (res != null)
+            onResultsCollected(res);
+    }
+
+    /**
+     * @param nodeId Failed node ID.
+     */
+    public final void onNodeFail(UUID nodeId) {
+        R res = null;
+
+        synchronized (this) {
+            if (allReceived())
+                return;
+
+            NodeMessage expMsg = rcvd.get(nodeId);
+
+            if (expMsg != null && expMsg.onNodeFailed()) {
+                assert leftMsgs > 0 : leftMsgs;
+
+                leftMsgs--;
+
+                if (allReceived())
+                    res = createResult(rcvd);
+            }
+        }
+
+        if (res != null)
+            onResultsCollected(res);
+    }
+
+    /**
+     * @return {@code True} if expected messages are initialized and all message are received.
+     */
+    private boolean allReceived() {
+        return discoCache != null && leftMsgs == 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DiscoveryMessageResultsCollector.class, this);
+    }
+
+    /**
+     *
+     */
+    protected static class NodeMessage<M> {
+        /** */
+        boolean nodeFailed;
+
+        /** */
+        M msg;
+
+        /**
+         * @param msg Message.
+         */
+        NodeMessage(M msg) {
+            this.msg = msg;
+        }
+
+        /**
+         * @return Message or {@code null} if node failed.
+         */
+        @Nullable public M message() {
+            return msg;
+        }
+
+        /**
+         * @return {@code True} if node result was not set before.
+         */
+        boolean onNodeFailed() {
+            if (nodeFailed || msg != null)
+                return false;
+
+            nodeFailed = true;
+
+            return true;
+        }
+
+        /**
+         * @param msg Received message.
+         * @return {@code True} if node result was not set before.
+         */
+        boolean set(M msg) {
+            assert msg != null;
+
+            if (this.msg != null)
+                return false;
+
+            this.msg = msg;
+
+            return !nodeFailed;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(NodeMessage.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 2e814d45aeeae..4c5690e919d34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -54,8 +54,11 @@
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.CommunicationFailureResolver;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.DefaultCommunicationFailureResolver;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.failure.FailureContext;
@@ -112,6 +115,8 @@
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.segmentation.SegmentationPolicy;
 import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.discovery.DiscoveryDataBag;
 import org.apache.ignite.spi.discovery.DiscoveryDataBag.JoiningNodeDiscoveryData;
 import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
@@ -120,10 +125,10 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
 import org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport;
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.ignite.spi.discovery.DiscoverySpiMutableCustomMessageSupport;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -478,7 +483,7 @@ private void updateClientNodes(UUID leftNodeId) {
 
     /** {@inheritDoc} */
     @Override protected void onKernalStart0() throws IgniteCheckedException {
-        if (Boolean.TRUE.equals(ctx.config().isClientMode()) && !getSpi().isClientMode())
+        if ((getSpi() instanceof TcpDiscoverySpi) && Boolean.TRUE.equals(ctx.config().isClientMode()) && !getSpi().isClientMode())
             ctx.performance().add("Enable client mode for TcpDiscoverySpi " +
                 "(set TcpDiscoverySpi.forceServerMode to false)");
     }
@@ -551,6 +556,9 @@ private void updateClientNodes(UUID leftNodeId) {
             });
         }
 
+        if (ctx.config().getCommunicationFailureResolver() != null)
+            ctx.resource().injectGeneric(ctx.config().getCommunicationFailureResolver());
+
         spi.setListener(new DiscoverySpiListener() {
             private long gridStartTime;
 
@@ -559,8 +567,8 @@ private void updateClientNodes(UUID leftNodeId) {
                 for (IgniteInClosure<ClusterNode> lsnr : locNodeInitLsnrs)
                     lsnr.apply(locNode);
 
-                if (locNode instanceof TcpDiscoveryNode) {
-                    final TcpDiscoveryNode node = (TcpDiscoveryNode)locNode;
+                if (locNode instanceof IgniteClusterNode) {
+                    final IgniteClusterNode node = (IgniteClusterNode)locNode;
 
                     if (consistentId != null)
                         node.setConsistentId(consistentId);
@@ -1052,7 +1060,7 @@ private GridLocalMetrics createMetrics() {
     /**
      * @return Metrics provider.
      */
-    private DiscoveryMetricsProvider createMetricsProvider() {
+    public DiscoveryMetricsProvider createMetricsProvider() {
         return new DiscoveryMetricsProvider() {
             /** */
             private final long startTime = U.currentTimeMillis();
@@ -1679,13 +1687,15 @@ public boolean pingNode(UUID nodeId) throws IgniteClientDisconnectedCheckedExcep
             return getSpi().pingNode(nodeId);
         }
         catch (IgniteException e) {
-            if (e.hasCause(IgniteClientDisconnectedCheckedException.class)) {
+            if (e.hasCause(IgniteClientDisconnectedCheckedException.class, IgniteClientDisconnectedException.class)) {
                 IgniteFuture<?> reconnectFut = ctx.cluster().clientReconnectFuture();
 
                 throw new IgniteClientDisconnectedCheckedException(reconnectFut, e.getMessage());
             }
 
-            throw e;
+            LT.warn(log, "Ping failed with error [node=" + nodeId + ", err=" + e + ']');
+
+            return true;
         }
         finally {
             busyLock.leaveBusy();
@@ -2025,7 +2035,16 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer)
 
         Map<Long, Collection<ClusterNode>> snapshots = topHist;
 
-        return snapshots.get(topVer);
+        Collection<ClusterNode> nodes = snapshots.get(topVer);
+
+        if (nodes == null) {
+            DiscoCache cache = discoCacheHist.get(new AffinityTopologyVersion(topVer, 0));
+
+            if (cache != null)
+                nodes = cache.allNodes();
+        }
+
+        return nodes;
     }
 
     /**
@@ -2157,6 +2176,19 @@ public void clientCacheStartEvent(UUID reqId,
         }
     }
 
+    /**
+     * @param discoCache
+     * @param node
+     */
+    public void metricsUpdateEvent(DiscoCache discoCache, ClusterNode node) {
+        discoWrk.addEvent(EVT_NODE_METRICS_UPDATED,
+            discoCache.version(),
+            node,
+            discoCache,
+            discoCache.nodeMap.values(),
+            null);
+    }
+
     /**
      * Gets first grid node start time, see {@link DiscoverySpi#getGridStartTime()}.
      *
@@ -2211,8 +2243,9 @@ public void failNode(UUID nodeId, @Nullable String warning) {
     public boolean reconnectSupported() {
         DiscoverySpi spi = getSpi();
 
-        return ctx.discovery().localNode().isClient() && (spi instanceof TcpDiscoverySpi) &&
-            !(((TcpDiscoverySpi) spi).isClientReconnectDisabled());
+        return ctx.discovery().localNode().isClient() &&
+            (spi instanceof IgniteDiscoverySpi) &&
+            ((IgniteDiscoverySpi)spi).clientReconnectSupported();
     }
 
     /**
@@ -2225,7 +2258,7 @@ public void reconnect() {
 
         DiscoverySpi discoverySpi = getSpi();
 
-        ((TcpDiscoverySpi)discoverySpi).reconnect();
+        ((IgniteDiscoverySpi)discoverySpi).clientReconnect();
     }
 
     /**
@@ -2379,6 +2412,76 @@ private void addToMap(Map<Integer, List<ClusterNode>> cacheMap, String cacheName
         cacheNodes.add(rich);
     }
 
+    /**
+     * @param cfg Configuration.
+     * @throws IgniteCheckedException If configuration is not valid.
+     */
+    public static void initCommunicationErrorResolveConfiguration(IgniteConfiguration cfg) throws IgniteCheckedException {
+        CommunicationFailureResolver rslvr = cfg.getCommunicationFailureResolver();
+        CommunicationSpi commSpi = cfg.getCommunicationSpi();
+        DiscoverySpi discoverySpi = cfg.getDiscoverySpi();
+
+        if (rslvr != null) {
+            if (!supportsCommunicationErrorResolve(commSpi))
+                throw new IgniteCheckedException("CommunicationFailureResolver is configured, but CommunicationSpi does not support communication" +
+                    "problem resolve: " + commSpi.getClass().getName());
+
+            if (!supportsCommunicationErrorResolve(discoverySpi))
+                throw new IgniteCheckedException("CommunicationFailureResolver is configured, but DiscoverySpi does not support communication" +
+                    "problem resolve: " + discoverySpi.getClass().getName());
+        }
+        else {
+            if (supportsCommunicationErrorResolve(commSpi) && supportsCommunicationErrorResolve(discoverySpi))
+                cfg.setCommunicationFailureResolver(new DefaultCommunicationFailureResolver());
+        }
+    }
+
+    /**
+     * @param spi Discovery SPI.
+     * @return {@code True} if SPI supports communication error resolve.
+     */
+    private static boolean supportsCommunicationErrorResolve(DiscoverySpi spi) {
+        return spi instanceof IgniteDiscoverySpi && ((IgniteDiscoverySpi)spi).supportsCommunicationFailureResolve();
+    }
+
+    /**
+     * @param spi Discovery SPI.
+     * @return {@code True} if SPI supports communication error resolve.
+     */
+    private static boolean supportsCommunicationErrorResolve(CommunicationSpi spi) {
+        return spi instanceof TcpCommunicationSpi;
+    }
+
+    /**
+     * @return {@code True} if communication error resolve is supported.
+     */
+    public boolean communicationErrorResolveSupported() {
+        return ctx.config().getCommunicationFailureResolver() != null;
+    }
+
+    /**
+     * @return {@code True} if configured {@link DiscoverySpi} supports mutable custom messages.
+     */
+    public boolean mutableCustomMessages() {
+        DiscoverySpiMutableCustomMessageSupport ann = U.getAnnotation(ctx.config().getDiscoverySpi().getClass(),
+            DiscoverySpiMutableCustomMessageSupport.class);
+
+        return ann != null && ann.value();
+    }
+
+    /**
+     * @param node Problem node.
+     * @param err Error.
+     */
+    public void resolveCommunicationError(ClusterNode node, Exception err) {
+        DiscoverySpi spi = getSpi();
+
+        if (!supportsCommunicationErrorResolve(spi) || !supportsCommunicationErrorResolve(ctx.config().getCommunicationSpi()))
+            throw new UnsupportedOperationException();
+
+        ((IgniteDiscoverySpi)spi).resolveCommunicationFailure(node, err);
+    }
+
     /** Worker for network segment checks. */
     private class SegmentCheckWorker extends GridWorker {
         /** */
@@ -2587,6 +2690,9 @@ private void body0() throws InterruptedException {
 
             AffinityTopologyVersion topVer = evt.get2();
 
+            if (type == EVT_NODE_METRICS_UPDATED && topVer.compareTo(discoCache.version()) < 0)
+                return;
+
             ClusterNode node = evt.get3();
 
             boolean isDaemon = node.isDaemon();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java
new file mode 100644
index 0000000000000..cbc706afbe3b3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public interface IgniteClusterNode extends ClusterNode {
+    /**
+     * Sets consistent globally unique node ID which survives node restarts.
+     *
+     * @param consistentId Consistent globally unique node ID.
+     */
+    public void setConsistentId(Serializable consistentId);
+
+    /**
+     * Sets node metrics.
+     *
+     * @param metrics Node metrics.
+     */
+    public void setMetrics(ClusterMetrics metrics);
+
+    /**
+     * Gets collections of cache metrics for this node. Note that node cache metrics are constantly updated
+     * and provide up to date information about caches.
+     * <p>
+     * Cache metrics are updated with some delay which is directly related to metrics update
+     * frequency. For example, by default the update will happen every {@code 2} seconds.
+     *
+     * @return Runtime metrics snapshots for this node.
+     */
+    public Map<Integer, CacheMetrics> cacheMetrics();
+
+    /**
+     * Sets node cache metrics.
+     *
+     * @param cacheMetrics Cache metrics.
+     */
+    public void setCacheMetrics(Map<Integer, CacheMetrics> cacheMetrics);
+
+    /**
+     * Whether this node is cache client (see {@link IgniteConfiguration#isClientMode()}).
+     *
+     * @return {@code True if client}.
+     */
+    public boolean isCacheClient();
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java
new file mode 100644
index 0000000000000..9aa5d140bbab5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpi.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+
+/**
+ *
+ */
+public interface IgniteDiscoverySpi extends DiscoverySpi {
+    /**
+     * @param nodeId Node ID.
+     * @return {@code True} if node joining or already joined topology.
+     */
+    public boolean knownNode(UUID nodeId);
+
+    /**
+     *
+     * @return {@code True} if SPI supports client reconnect.
+     */
+    public boolean clientReconnectSupported();
+
+    /**
+     *
+     */
+    public void clientReconnect();
+
+    /**
+     * For TESTING only.
+     */
+    public void simulateNodeFailure();
+
+    /**
+     * For TESTING only.
+     *
+     * @param lsnr Listener.
+     */
+    public void setInternalListener(IgniteDiscoverySpiInternalListener lsnr);
+
+    /**
+     * @return {@code True} if supports communication error resolve.
+     */
+    public boolean supportsCommunicationFailureResolve();
+
+    /**
+     * @param node Problem node.
+     * @param err Connection error.
+     */
+    public void resolveCommunicationFailure(ClusterNode node, Exception err);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpiInternalListener.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpiInternalListener.java
new file mode 100644
index 0000000000000..24405f8101f0a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteDiscoverySpiInternalListener.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+
+/**
+ * For TESTING only.
+ */
+public interface IgniteDiscoverySpiInternalListener {
+    /**
+     * @param locNode Local node.
+     * @param log Log.
+     */
+    public void beforeJoin(ClusterNode locNode, IgniteLogger log);
+
+    /**
+     * @param spi SPI instance.
+     * @param log Logger.
+     * @param msg Custom message.
+     * @return {@code False} to cancel event send.
+     */
+    public boolean beforeSendCustomEvent(DiscoverySpi spi, IgniteLogger log, DiscoverySpiCustomMessage msg);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAcceptedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAcceptedMessage.java
index ef87a444cb374..2e2aed9dcf0b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAcceptedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAcceptedMessage.java
@@ -71,6 +71,11 @@ public class UserAcceptedMessage implements DiscoveryCustomMessage {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserProposedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserProposedMessage.java
index 1a0be8ecaa2c6..19f9e82cf18cb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserProposedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserProposedMessage.java
@@ -71,6 +71,11 @@ public class UserProposedMessage implements DiscoveryServerOnlyCustomMessage {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinityChangeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinityChangeMessage.java
index fe1014cf6e010..937a8892bdfae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinityChangeMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinityChangeMessage.java
@@ -155,6 +155,11 @@ public AffinityTopologyVersion topologyVersion() {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 6691b132f876f..92b8d3ea87019 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -1310,20 +1310,17 @@ public void applyAffinityFromFullMessage(final GridDhtPartitionsExchangeFuture f
      * @param fut Current exchange future.
      * @param msg Message finish message.
      * @param resTopVer Result topology version.
-     * @throws IgniteCheckedException If failed.
      */
     public void onLocalJoin(final GridDhtPartitionsExchangeFuture fut,
         GridDhtPartitionsFullMessage msg,
-        final AffinityTopologyVersion resTopVer)
-        throws IgniteCheckedException {
+        final AffinityTopologyVersion resTopVer) {
         final Set<Integer> affReq = fut.context().groupsAffinityRequestOnJoin();
 
         final Map<Long, ClusterNode> nodesByOrder = new HashMap<>();
 
         final Map<Integer, CacheGroupAffinityMessage> joinedNodeAff = msg.joinedNodeAffinity();
 
-        assert !F.isEmpty(joinedNodeAff) : msg;
-        assert joinedNodeAff.size() >= affReq.size();
+        assert F.isEmpty(affReq) || (!F.isEmpty(joinedNodeAff) && joinedNodeAff.size() >= affReq.size()) : msg;
 
         forAllCacheGroups(false, new IgniteInClosureX<GridAffinityAssignmentCache>() {
             @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException {
@@ -1333,7 +1330,7 @@ public void onLocalJoin(final GridDhtPartitionsExchangeFuture fut,
 
                 assert grp != null;
 
-                if (affReq.contains(aff.groupId())) {
+                if (affReq != null && affReq.contains(aff.groupId())) {
                     assert AffinityTopologyVersion.NONE.equals(aff.lastVersion());
 
                     CacheGroupAffinityMessage affMsg = joinedNodeAff.get(aff.groupId());
@@ -2281,6 +2278,23 @@ public Map<Integer, CacheGroupDescriptor> cacheGroups() {
         return caches.registeredGrps;
     }
 
+    /**
+     * @return All registered cache groups.
+     */
+    public Map<Integer, DynamicCacheDescriptor> caches() {
+        return caches.registeredCaches;
+    }
+
+    /**
+     * @param grpId Cache group ID
+     * @return Cache affinity cache.
+     */
+    @Nullable public GridAffinityAssignmentCache groupAffinity(int grpId) {
+        CacheGroupHolder grpHolder = grpHolders.get(grpId);
+
+        return grpHolder != null ? grpHolder.affinity() : null;
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStatisticsModeChangeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStatisticsModeChangeMessage.java
index 40bcfaf12de16..e33256fbed341 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStatisticsModeChangeMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheStatisticsModeChangeMessage.java
@@ -100,6 +100,11 @@ public CacheStatisticsModeChangeMessage(UUID reqId, Collection<String> caches, b
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
         DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDiscoveryMessage.java
index e35d80e5c1c6f..ae76c950421db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDiscoveryMessage.java
@@ -172,6 +172,11 @@ public void updateTimeoutObject(ClientCacheUpdateTimeout updateTimeoutObj) {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDummyDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDummyDiscoveryMessage.java
index 6ed3ecc505f25..4ce0c87bf9898 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDummyDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClientCacheChangeDummyDiscoveryMessage.java
@@ -104,6 +104,11 @@ Set<String> cachesToClose() {
         throw new UnsupportedOperationException();
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
index 83459a5c03589..d85e29b673fe2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeBatch.java
@@ -76,6 +76,11 @@ public DynamicCacheChangeBatch(Collection<DynamicCacheChangeRequest> reqs) {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
         DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 55357ffb1834f..c2d0f427fc917 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -82,6 +82,7 @@
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
 import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityImpl;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteExternalizableExpiryPolicy;
@@ -3234,7 +3235,7 @@ protected IgniteInternalFuture<Boolean> removeAsync0(final K key, @Nullable fina
         List<CacheMetrics> metrics = new ArrayList<>(grp.nodes().size());
 
         for (ClusterNode node : grp.nodes()) {
-            Map<Integer, CacheMetrics> nodeCacheMetrics = ((TcpDiscoveryNode)node).cacheMetrics();
+            Map<Integer, CacheMetrics> nodeCacheMetrics = ((IgniteClusterNode)node).cacheMetrics();
 
             if (nodeCacheMetrics != null) {
                 CacheMetrics e = nodeCacheMetrics.get(context().cacheId());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index a30a24ad687ea..77ffce3bdaf06 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -427,7 +427,7 @@ private void onDiscoveryEvent(DiscoveryEvent evt, DiscoCache cache) {
         if (evt.type() != EVT_DISCOVERY_CUSTOM_EVT) {
             assert evt.type() != EVT_NODE_JOINED || n.isLocal() || n.order() > loc.order() :
                 "Node joined with smaller-than-local " +
-                    "order [newOrder=" + n.order() + ", locOrder=" + loc.order() + ']';
+                    "order [newOrder=" + n.order() + ", locOrder=" + loc.order() + ", evt=" + evt + ']';
 
             exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
 
@@ -570,12 +570,6 @@ public void onKernalStart(boolean active, boolean reconnect) throws IgniteChecke
         for (ClusterNode n : cctx.discovery().remoteNodes())
             cctx.versions().onReceived(n.id(), n.metrics().getLastDataVersion());
 
-        ClusterNode loc = cctx.localNode();
-
-        long startTime = loc.metrics().getStartTime();
-
-        assert startTime > 0;
-
         DiscoveryLocalJoinData locJoin = cctx.discovery().localJoin();
 
         GridDhtPartitionsExchangeFuture fut = null;
@@ -756,6 +750,14 @@ public Object interruptLock() {
         return interruptLock;
     }
 
+    /**
+     * @param grpId Cache group ID.
+     * @return Topology.
+     */
+    @Nullable public GridDhtPartitionTopology clientTopologyIfExists(int grpId) {
+        return clientTops.get(grpId);
+    }
+
     /**
      * @param grpId Cache group ID.
      * @param discoCache Discovery data cache.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index a3f7c9430ecd2..7edac7374fbe9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3363,7 +3363,7 @@ private IgniteNodeValidationResult validateRestartingCaches(ClusterNode node) {
      * @return Validation result or {@code null} in case of success.
      */
     @Nullable private IgniteNodeValidationResult validateHashIdResolvers(ClusterNode node) {
-        if (!node.isClient()) {
+        if (!CU.clientNode(node)) {
             for (DynamicCacheDescriptor desc : cacheDescriptors().values()) {
                 CacheConfiguration cfg = desc.cacheConfiguration();
 
@@ -3372,7 +3372,7 @@ private IgniteNodeValidationResult validateRestartingCaches(ClusterNode node) {
 
                     Object nodeHashObj = aff.resolveNodeHash(node);
 
-                    for (ClusterNode topNode : ctx.discovery().allNodes()) {
+                    for (ClusterNode topNode : ctx.discovery().aliveServerNodes()) {
                         Object topNodeHashObj = aff.resolveNodeHash(topNode);
 
                         if (nodeHashObj.hashCode() == topNodeHashObj.hashCode()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 09a96d368528c..6d4c1f21cb273 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -65,6 +65,7 @@
 import org.apache.ignite.internal.cluster.ClusterGroupEmptyCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
@@ -97,7 +98,6 @@
 import org.apache.ignite.lang.IgniteReducer;
 import org.apache.ignite.lifecycle.LifecycleAware;
 import org.apache.ignite.plugin.CachePluginConfiguration;
-import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
@@ -1347,8 +1347,8 @@ public static <T extends CachePluginConfiguration> List<T> cachePluginConfigurat
      * @return {@code True} if given node is client node (has flag {@link IgniteConfiguration#isClientMode()} set).
      */
     public static boolean clientNode(ClusterNode node) {
-        if (node instanceof TcpDiscoveryNode)
-            return ((TcpDiscoveryNode)node).isCacheClient();
+        if (node instanceof IgniteClusterNode)
+            return ((IgniteClusterNode)node).isCacheClient();
         else
             return clientNodeDirect(node);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
index 57f25d0e80ca1..4afa403bfbe73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateFinishMessage.java
@@ -66,6 +66,11 @@ public boolean changed() {
         return errMsg;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(WalStateFinishMessage.class, this, "super", super.toString());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
index 747fd6af0a3af..b9d96fc35a039 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateProposeMessage.java
@@ -97,6 +97,11 @@ public void affinityNode(boolean affNode) {
         this.affNode = affNode;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(WalStateProposeMessage.class, this, "super", super.toString());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
index 9402a3270180c..38450dfec5c78 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
@@ -155,22 +155,30 @@ void addBinaryMetadataUpdateListener(BinaryMetadataUpdatedListener lsnr) {
      * @param metadata Metadata proposed for update.
      * @return Future to wait for update result on.
      */
-    GridFutureAdapter<MetadataUpdateResult> requestMetadataUpdate(BinaryMetadata metadata) throws IgniteCheckedException {
+    GridFutureAdapter<MetadataUpdateResult> requestMetadataUpdate(BinaryMetadata metadata) {
         MetadataUpdateResultFuture resFut = new MetadataUpdateResultFuture();
 
         if (log.isDebugEnabled())
             log.debug("Requesting metadata update for " + metadata.typeId() + "; caller thread is blocked on future "
                 + resFut);
 
-        synchronized (this) {
-            unlabeledFutures.add(resFut);
+        try {
+            synchronized (this) {
+                unlabeledFutures.add(resFut);
 
-            if (!stopping)
-                discoMgr.sendCustomEvent(new MetadataUpdateProposedMessage(metadata, ctx.localNodeId()));
-            else
-                resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult());
+                if (!stopping)
+                    discoMgr.sendCustomEvent(new MetadataUpdateProposedMessage(metadata, ctx.localNodeId()));
+                else
+                    resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult());
+            }
+        }
+        catch (Exception e) {
+            resFut.onDone(MetadataUpdateResult.createUpdateDisabledResult(), e);
         }
 
+        if (ctx.clientDisconnected())
+            onDisconnected();
+
         return resFut;
     }
 
@@ -237,6 +245,8 @@ private void cancelFutures(MetadataUpdateResult res) {
         for (MetadataUpdateResultFuture fut : unlabeledFutures)
             fut.onDone(res);
 
+        unlabeledFutures.clear();
+
         for (MetadataUpdateResultFuture fut : syncMap.values())
             fut.onDone(res);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateAcceptedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateAcceptedMessage.java
index 0416746a04b68..df64613b71063 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateAcceptedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateAcceptedMessage.java
@@ -70,6 +70,11 @@ public class MetadataUpdateAcceptedMessage implements DiscoveryCustomMessage {
         return true;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateProposedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateProposedMessage.java
index f9bd66078cba2..84e32e1b3d7cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateProposedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/MetadataUpdateProposedMessage.java
@@ -133,6 +133,11 @@ public MetadataUpdateProposedMessage(BinaryMetadata metadata, UUID origNodeId) {
         return true;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 9b3c1ec9bafff..5bbbb3102c663 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -597,6 +597,45 @@ private List<ClusterNode> nodes(int p, AffinityTopologyVersion topVer, GridDhtPa
         return owners(p, AffinityTopologyVersion.NONE);
     }
 
+    /** {@inheritDoc} */
+    @Override public List<List<ClusterNode>> allOwners() {
+        lock.readLock().lock();
+
+        try {
+            int parts = partitions();
+
+            List<List<ClusterNode>> res = new ArrayList<>(parts);
+
+            for (int i = 0; i < parts; i++)
+                res.add(new ArrayList<>());
+
+            List<ClusterNode> allNodes = discoCache.cacheGroupAffinityNodes(grpId);
+
+            for (int i = 0; i < allNodes.size(); i++) {
+                ClusterNode node = allNodes.get(i);
+
+                GridDhtPartitionMap nodeParts = node2part.get(node.id());
+
+                if (nodeParts != null) {
+                    for (Map.Entry<Integer, GridDhtPartitionState> e : nodeParts.map().entrySet()) {
+                        if (e.getValue() == OWNING) {
+                            int part = e.getKey();
+
+                            List<ClusterNode> owners = res.get(part);
+
+                            owners.add(node);
+                        }
+                    }
+                }
+            }
+
+            return res;
+        }
+        finally {
+            lock.readLock().unlock();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public List<ClusterNode> moving(int p) {
         return nodes(p, AffinityTopologyVersion.NONE, MOVING, null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index ba55543233210..ea99f5d0d8263 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -970,6 +970,10 @@ else if (req.needVersion())
                 try {
                     ctx.io().send(nodeId, res, ctx.ioPolicy());
                 }
+                catch (ClusterTopologyCheckedException e) {
+                    if (log.isDebugEnabled())
+                        log.debug("Failed to send get response to node, node failed: " + nodeId);
+                }
                 catch (IgniteCheckedException e) {
                     U.error(log, "Failed to send get response to node (is node still alive?) [nodeId=" + nodeId +
                         ",req=" + req + ", res=" + res + ']', e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 13564c2af2666..7f900cb67f533 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -236,6 +236,12 @@ public boolean initPartitionsWhenAffinityReady(AffinityTopologyVersion affVer, G
      */
     public List<ClusterNode> owners(int p);
 
+    /**
+     * @return List indexed by partition number, each list element is collection of all nodes who
+     *      owns corresponding partition.
+     */
+    public List<List<ClusterNode>> allOwners();
+
     /**
      * @param p Partition ID.
      * @param topVer Topology version.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 528f0a6e51c7a..538c57ec9ba97 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -1216,6 +1216,45 @@ private List<ClusterNode> nodes(
         return owners(p, AffinityTopologyVersion.NONE);
     }
 
+    /** {@inheritDoc} */
+    @Override public List<List<ClusterNode>> allOwners() {
+        lock.readLock().lock();
+
+        try {
+            int parts = partitions();
+
+            List<List<ClusterNode>> res = new ArrayList<>(parts);
+
+            for (int i = 0; i < parts; i++)
+                res.add(new ArrayList<>());
+
+            List<ClusterNode> allNodes = discoCache.cacheGroupAffinityNodes(grp.groupId());
+
+            for (int i = 0; i < allNodes.size(); i++) {
+                ClusterNode node = allNodes.get(i);
+
+                GridDhtPartitionMap nodeParts = node2part.get(node.id());
+
+                if (nodeParts != null) {
+                    for (Map.Entry<Integer, GridDhtPartitionState> e : nodeParts.map().entrySet()) {
+                        if (e.getValue() == OWNING) {
+                            int part = e.getKey();
+
+                            List<ClusterNode> owners = res.get(part);
+
+                            owners.add(node);
+                        }
+                    }
+                }
+            }
+
+            return res;
+        }
+        finally {
+            lock.readLock().unlock();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public List<ClusterNode> moving(int p) {
         if (!grp.rebalanceEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 8da91a8915ae3..cbb49851e1f81 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1506,12 +1506,16 @@ private void sendPartitions(ClusterNode oldestNode) {
         }
         catch (ClusterTopologyCheckedException ignore) {
             if (log.isDebugEnabled())
-                log.debug("Oldest node left during partition exchange [nodeId=" + oldestNode.id() +
+                log.debug("Coordinator left during partition exchange [nodeId=" + oldestNode.id() +
                     ", exchId=" + exchId + ']');
         }
         catch (IgniteCheckedException e) {
-            U.error(log, "Failed to send local partitions to oldest node (will retry after timeout) [oldestNodeId=" +
-                oldestNode.id() + ", exchId=" + exchId + ']', e);
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else {
+                U.error(log, "Failed to send local partitions to coordinator [crd=" + oldestNode.id() +
+                    ", exchId=" + exchId + ']', e);
+            }
         }
     }
 
@@ -3369,9 +3373,13 @@ public void onNodeLeft(final ClusterNode node) {
                             }
 
                             if (allReceived) {
-                                awaitSingleMapUpdates();
+                                cctx.kernalContext().getSystemExecutorService().submit(new Runnable() {
+                                    @Override public void run() {
+                                        awaitSingleMapUpdates();
 
-                                onAllReceived(null);
+                                        onAllReceived(null);
+                                    }
+                                });
                             }
                         }
                         else {
@@ -3399,7 +3407,13 @@ public void onNodeLeft(final ClusterNode node) {
                                         ", newCrd=" + crd0.id() + ']');
                                 }
 
-                                sendPartitions(crd0);
+                                final ClusterNode newCrd = crd0;
+
+                                cctx.kernalContext().getSystemExecutorService().submit(new Runnable() {
+                                    @Override public void run() {
+                                        sendPartitions(newCrd);
+                                    }
+                                });
                             }
                         }
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
index d7dfa166b5d14..bbbd9996f358b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateFinishMessage.java
@@ -93,6 +93,11 @@ public boolean success() {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
index 50fc0223101e8..81855fcbf44a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ChangeGlobalStateMessage.java
@@ -130,6 +130,11 @@ void exchangeActions(ExchangeActions exchangeActions) {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
         DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
new file mode 100644
index 0000000000000..5c3044b6808cc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterMetricsUpdateMessage.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cluster;
+
+import java.nio.ByteBuffer;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.GridDirectMap;
+import org.apache.ignite.internal.managers.communication.GridIoMessageFactory;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class ClusterMetricsUpdateMessage implements Message {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private byte[] nodeMetrics;
+
+    /** */
+    @GridDirectMap(keyType = UUID.class, valueType = byte[].class)
+    private Map<UUID, byte[]> allNodesMetrics;
+
+    /**
+     * Required by {@link GridIoMessageFactory}.
+     */
+    public ClusterMetricsUpdateMessage() {
+        // No-op.
+    }
+
+    /**
+     * @param nodeMetrics Node metrics.
+     */
+    ClusterMetricsUpdateMessage(byte[] nodeMetrics) {
+        this.nodeMetrics = nodeMetrics;
+    }
+
+    /**
+     * @param allNodesMetrics All nodes metrcis.
+     */
+    ClusterMetricsUpdateMessage(Map<UUID, byte[]> allNodesMetrics) {
+        this.allNodesMetrics = allNodesMetrics;
+    }
+
+    /**
+     * @return Node metrics.
+     */
+    @Nullable byte[] nodeMetrics() {
+        return nodeMetrics;
+    }
+
+    /**
+     * @return All nodes metrics.
+     */
+    @Nullable Map<UUID, byte[]> allNodesMetrics() {
+        return allNodesMetrics;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeMap("allNodesMetrics", allNodesMetrics, MessageCollectionItemType.UUID, MessageCollectionItemType.BYTE_ARR))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeByteArray("nodeMetrics", nodeMetrics))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                allNodesMetrics = reader.readMap("allNodesMetrics", MessageCollectionItemType.UUID, MessageCollectionItemType.BYTE_ARR, false);
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                nodeMetrics = reader.readByteArray("nodeMetrics");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        return reader.afterMessageRead(ClusterMetricsUpdateMessage.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public short directType() {
+        return 133;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 2;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ClusterMetricsUpdateMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterNodeMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterNodeMetrics.java
new file mode 100644
index 0000000000000..22a385ff03a49
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterNodeMetrics.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cluster;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.Map;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.internal.ClusterMetricsSnapshot;
+
+/**
+ *
+ */
+class ClusterNodeMetrics implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final byte[] metrics;
+
+    /** */
+    private final Map<Integer, CacheMetrics> cacheMetrics;
+
+    /**
+     * @param metrics Metrics.
+     * @param cacheMetrics Cache metrics.
+     */
+    ClusterNodeMetrics(ClusterMetrics metrics, Map<Integer, CacheMetrics> cacheMetrics) {
+        this.metrics = ClusterMetricsSnapshot.serialize(metrics);
+        this.cacheMetrics = cacheMetrics;
+    }
+
+    /**
+     * @return Metrics.
+     */
+    byte[] metrics() {
+        return metrics;
+    }
+
+    /**
+     * @return Cache metrics.
+     */
+    Map<Integer, CacheMetrics> cacheMetrics() {
+        return cacheMetrics != null ? cacheMetrics : Collections.<Integer, CacheMetrics>emptyMap();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
index 5f2c66ce7fe59..8796302a0b466 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/ClusterProcessor.java
@@ -33,6 +33,8 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.ClusterMetricsSnapshot;
+import org.apache.ignite.internal.GridDirectMap;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteDiagnosticInfo;
 import org.apache.ignite.internal.IgniteDiagnosticMessage;
@@ -42,21 +44,29 @@
 import org.apache.ignite.internal.cluster.IgniteClusterImpl;
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridTimerTask;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.CI1;
+import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.spi.discovery.DiscoveryDataBag;
 import org.apache.ignite.spi.discovery.DiscoveryDataBag.GridDiscoveryData;
+import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DIAGNOSTIC_ENABLED;
@@ -66,6 +76,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.CLUSTER_PROC;
 import static org.apache.ignite.internal.GridTopic.TOPIC_INTERNAL_DIAGNOSTIC;
+import static org.apache.ignite.internal.GridTopic.TOPIC_METRICS;
 import static org.apache.ignite.internal.IgniteVersionUtils.VER_STR;
 
 /**
@@ -102,6 +113,18 @@ public class ClusterProcessor extends GridProcessorAdapter {
     /** */
     private final AtomicLong diagFutId = new AtomicLong();
 
+    /** */
+    private final Map<UUID, byte[]> allNodesMetrics = new ConcurrentHashMap<>();
+
+    /** */
+    private final JdkMarshaller marsh = new JdkMarshaller();
+
+    /** */
+    private DiscoveryMetricsProvider metricsProvider;
+
+    /** */
+    private boolean sndMetrics;
+
     /**
      * @param ctx Kernal context.
      */
@@ -111,6 +134,8 @@ public ClusterProcessor(GridKernalContext ctx) {
         notifyEnabled.set(IgniteSystemProperties.getBoolean(IGNITE_UPDATE_NOTIFIER, true));
 
         cluster = new IgniteClusterImpl(ctx);
+
+        sndMetrics = !(ctx.config().getDiscoverySpi() instanceof TcpDiscoverySpi);
     }
 
     /**
@@ -120,33 +145,31 @@ public boolean diagnosticEnabled() {
         return getBoolean(IGNITE_DIAGNOSTIC_ENABLED, true);
     }
 
-    /** */
-    private final JdkMarshaller marsh = new JdkMarshaller();
-
     /**
      * @throws IgniteCheckedException If failed.
      */
     public void initDiagnosticListeners() throws IgniteCheckedException {
         ctx.event().addLocalEventListener(new GridLocalEventListener() {
-                @Override public void onEvent(Event evt) {
-                    assert evt instanceof DiscoveryEvent;
-                    assert evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT;
+            @Override public void onEvent(Event evt) {
+                assert evt instanceof DiscoveryEvent;
+                assert evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT;
 
-                    DiscoveryEvent discoEvt = (DiscoveryEvent)evt;
+                DiscoveryEvent discoEvt = (DiscoveryEvent)evt;
 
-                    UUID nodeId = discoEvt.eventNode().id();
+                UUID nodeId = discoEvt.eventNode().id();
 
-                    ConcurrentHashMap<Long, InternalDiagnosticFuture> futs = diagnosticFutMap.get();
+                ConcurrentHashMap<Long, InternalDiagnosticFuture> futs = diagnosticFutMap.get();
 
-                    if (futs != null) {
-                        for (InternalDiagnosticFuture fut : futs.values()) {
-                            if (fut.nodeId.equals(nodeId))
-                                fut.onDone(new IgniteDiagnosticInfo("Target node failed: " + nodeId));
-                        }
+                if (futs != null) {
+                    for (InternalDiagnosticFuture fut : futs.values()) {
+                        if (fut.nodeId.equals(nodeId))
+                            fut.onDone(new IgniteDiagnosticInfo("Target node failed: " + nodeId));
                     }
                 }
-            },
-            EVT_NODE_FAILED, EVT_NODE_LEFT);
+
+                allNodesMetrics.remove(nodeId);
+            }
+        }, EVT_NODE_FAILED, EVT_NODE_LEFT);
 
         ctx.io().addMessageListener(TOPIC_INTERNAL_DIAGNOSTIC, new GridMessageListener() {
             @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
@@ -233,6 +256,17 @@ public void initDiagnosticListeners() throws IgniteCheckedException {
                     U.warn(diagnosticLog, "Received unexpected message: " + msg);
             }
         });
+
+        if (sndMetrics) {
+            ctx.io().addMessageListener(TOPIC_METRICS, new GridMessageListener() {
+                @Override public void onMessage(UUID nodeId, Object msg, byte plc) {
+                    if (msg instanceof ClusterMetricsUpdateMessage)
+                        processMetricsUpdateMessage(nodeId, (ClusterMetricsUpdateMessage)msg);
+                    else
+                        U.warn(log, "Received unexpected message for TOPIC_METRICS: " + msg);
+                }
+            });
+        }
     }
 
     /**
@@ -296,7 +330,6 @@ private Serializable getDiscoveryData() {
         }
     }
 
-
     /**
      * @param vals collection to seek through.
      */
@@ -334,6 +367,14 @@ private Boolean findLastFlag(Collection<Serializable> vals) {
                     log.debug("Failed to create GridUpdateNotifier: " + e);
             }
         }
+
+        if (sndMetrics) {
+            metricsProvider = ctx.discovery().createMetricsProvider();
+
+            long updateFreq = ctx.config().getMetricsUpdateFrequency();
+
+            ctx.timeout().addTimeoutObject(new MetricsUpdateTimeoutObject(updateFreq));
+        }
     }
 
     /** {@inheritDoc} */
@@ -351,6 +392,133 @@ private Boolean findLastFlag(Collection<Serializable> vals) {
             ctx.io().removeMessageListener(TOPIC_INTERNAL_DIAGNOSTIC);
     }
 
+    /**
+     * @param sndNodeId Sender node ID.
+     * @param msg Message.
+     */
+    private void processMetricsUpdateMessage(UUID sndNodeId, ClusterMetricsUpdateMessage msg) {
+        byte[] nodeMetrics = msg.nodeMetrics();
+
+        if (nodeMetrics != null) {
+            assert msg.allNodesMetrics() == null;
+
+            allNodesMetrics.put(sndNodeId, nodeMetrics);
+
+            updateNodeMetrics(ctx.discovery().discoCache(), sndNodeId, nodeMetrics);
+        }
+        else {
+            Map<UUID, byte[]> allNodesMetrics = msg.allNodesMetrics();
+
+            assert allNodesMetrics != null;
+
+            DiscoCache discoCache = ctx.discovery().discoCache();
+
+            for (Map.Entry<UUID, byte[]> e : allNodesMetrics.entrySet()) {
+                if (!ctx.localNodeId().equals(e.getKey()))
+                    updateNodeMetrics(discoCache, e.getKey(), e.getValue());
+            }
+        }
+    }
+
+    /**
+     * @param discoCache Discovery data cache.
+     * @param nodeId Node ID.
+     * @param metricsBytes Marshalled metrics.
+     */
+    private void updateNodeMetrics(DiscoCache discoCache, UUID nodeId, byte[] metricsBytes) {
+        ClusterNode node = discoCache.node(nodeId);
+
+        if (node == null || !discoCache.alive(nodeId))
+            return;
+
+        try {
+            ClusterNodeMetrics metrics = U.unmarshalZip(ctx.config().getMarshaller(), metricsBytes, null);
+
+            assert node instanceof IgniteClusterNode : node;
+
+            IgniteClusterNode node0 = (IgniteClusterNode)node;
+
+            node0.setMetrics(ClusterMetricsSnapshot.deserialize(metrics.metrics(), 0));
+            node0.setCacheMetrics(metrics.cacheMetrics());
+
+            ctx.discovery().metricsUpdateEvent(discoCache, node0);
+        }
+        catch (IgniteCheckedException e) {
+            U.warn(log, "Failed to unmarshal node metrics: " + e);
+        }
+    }
+
+    /**
+     *
+     */
+    private void updateMetrics() {
+        if (ctx.isStopping() || ctx.clientDisconnected())
+            return;
+
+        ClusterNode oldest = ctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE);
+
+        if (oldest == null)
+            return;
+
+        if (ctx.localNodeId().equals(oldest.id())) {
+            IgniteClusterNode locNode = (IgniteClusterNode)ctx.discovery().localNode();
+
+            locNode.setMetrics(metricsProvider.metrics());
+            locNode.setCacheMetrics(metricsProvider.cacheMetrics());
+
+            ClusterNodeMetrics metrics = new ClusterNodeMetrics(locNode.metrics(), locNode.cacheMetrics());
+
+            try {
+                byte[] metricsBytes = U.zip(U.marshal(ctx.config().getMarshaller(), metrics));
+
+                allNodesMetrics.put(ctx.localNodeId(), metricsBytes);
+            }
+            catch (IgniteCheckedException e) {
+                U.warn(log, "Failed to marshal local node metrics: " + e, e);
+            }
+
+            ctx.discovery().metricsUpdateEvent(ctx.discovery().discoCache(), locNode);
+
+            Collection<ClusterNode> allNodes = ctx.discovery().allNodes();
+
+            ClusterMetricsUpdateMessage msg = new ClusterMetricsUpdateMessage(new HashMap<>(allNodesMetrics));
+
+            for (ClusterNode node : allNodes) {
+                if (ctx.localNodeId().equals(node.id()) || !ctx.discovery().alive(node.id()))
+                    continue;
+
+                try {
+                    ctx.io().sendToGridTopic(node, TOPIC_METRICS, msg, GridIoPolicy.SYSTEM_POOL);
+                }
+                catch (ClusterTopologyCheckedException e) {
+                    if (log.isDebugEnabled())
+                        log.debug("Failed to send metrics update, node failed: " + e);
+                }
+                catch (IgniteCheckedException e) {
+                    U.warn(log, "Failed to send metrics update: " + e, e);
+                }
+            }
+        }
+        else {
+            ClusterNodeMetrics metrics = new ClusterNodeMetrics(metricsProvider.metrics(), metricsProvider.cacheMetrics());
+
+            try {
+                byte[] metricsBytes = U.zip(U.marshal(ctx.config().getMarshaller(), metrics));
+
+                ClusterMetricsUpdateMessage msg = new ClusterMetricsUpdateMessage(metricsBytes);
+
+                ctx.io().sendToGridTopic(oldest, TOPIC_METRICS, msg, GridIoPolicy.SYSTEM_POOL);
+            }
+            catch (ClusterTopologyCheckedException e) {
+                if (log.isDebugEnabled())
+                    log.debug("Failed to send metrics update to oldest, node failed: " + e);
+            }
+            catch (IgniteCheckedException e) {
+                LT.warn(log, e, "Failed to send metrics update to oldest: " + e, false, false);
+            }
+        }
+    }
+
     /**
      * Disables update notifier.
      */
@@ -571,4 +739,51 @@ public void onResponse(IgniteDiagnosticInfo res) {
             return S.toString(InternalDiagnosticFuture.class, this);
         }
     }
+
+    /**
+     *
+     */
+    private class MetricsUpdateTimeoutObject implements GridTimeoutObject, Runnable {
+        /** */
+        private final IgniteUuid id = IgniteUuid.randomUuid();
+
+        /** */
+        private long endTime;
+
+        /** */
+        private final long timeout;
+
+        /**
+         * @param timeout Timeout.
+         */
+        MetricsUpdateTimeoutObject(long timeout) {
+            this.timeout = timeout;
+
+            endTime = U.currentTimeMillis() + timeout;
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteUuid timeoutId() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long endTime() {
+            return endTime;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            updateMetrics();
+
+            endTime = U.currentTimeMillis() + timeout;
+
+            ctx.timeout().addTimeoutObject(this);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onTimeout() {
+            ctx.getSystemExecutorService().execute(this);
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/AbstractContinuousMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/AbstractContinuousMessage.java
index e9754d12cd966..928c619e128a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/AbstractContinuousMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/AbstractContinuousMessage.java
@@ -62,6 +62,11 @@ public UUID routineId() {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineInfo.java
new file mode 100644
index 0000000000000..fc0f1810e481d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineInfo.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.io.Serializable;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+class ContinuousRoutineInfo implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    UUID srcNodeId;
+
+    /** */
+    final UUID routineId;
+
+    /** */
+    final byte[] hnd;
+
+    /** */
+    final byte[] nodeFilter;
+
+    /** */
+    final int bufSize;
+
+    /** */
+    final long interval;
+
+    /** */
+    final boolean autoUnsubscribe;
+
+    /** */
+    transient boolean disconnected;
+
+    /**
+     * @param srcNodeId Source node ID.
+     * @param routineId Routine ID.
+     * @param hnd Marshalled handler.
+     * @param nodeFilter Marshalled node filter.
+     * @param bufSize Handler buffer size.
+     * @param interval Time interval.
+     * @param autoUnsubscribe Auto unsubscribe flag.
+     */
+    ContinuousRoutineInfo(
+        UUID srcNodeId,
+        UUID routineId,
+        byte[] hnd,
+        byte[] nodeFilter,
+        int bufSize,
+        long interval,
+        boolean autoUnsubscribe)
+    {
+        this.srcNodeId = srcNodeId;
+        this.routineId = routineId;
+        this.hnd = hnd;
+        this.nodeFilter = nodeFilter;
+        this.bufSize = bufSize;
+        this.interval = interval;
+        this.autoUnsubscribe = autoUnsubscribe;
+    }
+
+    /**
+     * @param srcNodeId Source node ID.
+     */
+    void sourceNodeId(UUID srcNodeId) {
+        this.srcNodeId = srcNodeId;
+    }
+
+    /**
+     *
+     */
+    void onDisconnected() {
+        disconnected = true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ContinuousRoutineInfo.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
new file mode 100644
index 0000000000000..581ac603eea4c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutineStartResultMessage.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.nio.ByteBuffer;
+import java.util.UUID;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class ContinuousRoutineStartResultMessage implements Message {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private static final int ERROR_FLAG = 0x01;
+
+    /** */
+    private UUID routineId;
+
+    /** */
+    private byte[] errBytes;
+
+    /** */
+    private byte[] cntrsMapBytes;
+
+    /** */
+    private int flags;
+
+    /**
+     *
+     */
+    public ContinuousRoutineStartResultMessage() {
+        // No-op.
+    }
+
+    /**
+     * @param routineId Routine ID.
+     * @param cntrsMapBytes Marshalled {@link CachePartitionPartialCountersMap}.
+     * @param errBytes Error bytes.
+     * @param err {@code True} if failed to start routine.
+     */
+    ContinuousRoutineStartResultMessage(UUID routineId, byte[] cntrsMapBytes, byte[] errBytes, boolean err) {
+        this.routineId = routineId;
+        this.cntrsMapBytes = cntrsMapBytes;
+        this.errBytes = errBytes;
+
+        if (err)
+            flags |= ERROR_FLAG;
+    }
+
+    /**
+     * @return Marshalled {@link CachePartitionPartialCountersMap}.
+     */
+    @Nullable byte[] countersMapBytes() {
+        return cntrsMapBytes;
+    }
+
+    /**
+     * @return {@code True} if failed to start routine.
+     */
+    boolean error() {
+        return (flags & ERROR_FLAG) != 0;
+    }
+
+    /**
+     * @return Routine ID.
+     */
+    UUID routineId() {
+        return routineId;
+    }
+
+    /**
+     * @return Error bytes.
+     */
+    @Nullable byte[] errorBytes() {
+        return errBytes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeByteArray("cntrsMapBytes", cntrsMapBytes))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeByteArray("errBytes", errBytes))
+                    return false;
+
+                writer.incrementState();
+
+            case 2:
+                if (!writer.writeInt("flags", flags))
+                    return false;
+
+                writer.incrementState();
+
+            case 3:
+                if (!writer.writeUuid("routineId", routineId))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                cntrsMapBytes = reader.readByteArray("cntrsMapBytes");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                errBytes = reader.readByteArray("errBytes");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 2:
+                flags = reader.readInt("flags");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 3:
+                routineId = reader.readUuid("routineId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        return reader.afterMessageRead(ContinuousRoutineStartResultMessage.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public short directType() {
+        return 134;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 4;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ContinuousRoutineStartResultMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesCommonDiscoveryData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesCommonDiscoveryData.java
new file mode 100644
index 0000000000000..d29de89b9b111
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesCommonDiscoveryData.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class ContinuousRoutinesCommonDiscoveryData implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    final List<ContinuousRoutineInfo> startedRoutines;
+
+    /**
+     * @param startedRoutines Routines started in cluster.
+     */
+    ContinuousRoutinesCommonDiscoveryData(List<ContinuousRoutineInfo> startedRoutines) {
+        this.startedRoutines = startedRoutines;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ContinuousRoutinesCommonDiscoveryData.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesInfo.java
new file mode 100644
index 0000000000000..ad24ff1805387
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesInfo.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
+
+import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.CONTINUOUS_PROC;
+
+/**
+ *
+ */
+class ContinuousRoutinesInfo {
+    /** */
+    private final Map<UUID, ContinuousRoutineInfo> startedRoutines = new HashMap<>();
+
+    /**
+     * @param dataBag Discovery data bag.
+     */
+    void collectGridNodeData(DiscoveryDataBag dataBag) {
+        synchronized (startedRoutines) {
+            if (!dataBag.commonDataCollectedFor(CONTINUOUS_PROC.ordinal()))
+                dataBag.addGridCommonData(CONTINUOUS_PROC.ordinal(),
+                    new ContinuousRoutinesCommonDiscoveryData(new ArrayList<>(startedRoutines.values())));
+        }
+    }
+
+    /**
+     * @param dataBag Discovery data bag.
+     */
+    void collectJoiningNodeData(DiscoveryDataBag dataBag) {
+        synchronized (startedRoutines) {
+            for (ContinuousRoutineInfo info : startedRoutines.values()) {
+                if (info.disconnected)
+                    info.sourceNodeId(dataBag.joiningNodeId());
+            }
+
+            dataBag.addJoiningNodeData(CONTINUOUS_PROC.ordinal(),
+                new ContinuousRoutinesJoiningNodeDiscoveryData(new ArrayList<>(startedRoutines.values())));
+        }
+    }
+
+    /**
+     * @param info Routine info.
+     */
+    void addRoutineInfo(ContinuousRoutineInfo info) {
+        synchronized (startedRoutines) {
+            startedRoutines.put(info.routineId, info);
+        }
+    }
+
+    /**
+     * @param routineId Routine ID.
+     * @return {@code True} if routine exists.
+     */
+    boolean routineExists(UUID routineId) {
+        synchronized (startedRoutines) {
+            return startedRoutines.containsKey(routineId);
+        }
+    }
+
+    /**
+     * @param routineId Routine ID.
+     */
+    void removeRoutine(UUID routineId) {
+        synchronized (startedRoutines) {
+            startedRoutines.remove(routineId);
+        }
+    }
+
+    /**
+     * @param locRoutines Routines IDs which can survive reconnect.
+     */
+    void onClientDisconnected(Collection<UUID> locRoutines) {
+        synchronized (startedRoutines) {
+            for (Iterator<Map.Entry<UUID, ContinuousRoutineInfo>> it = startedRoutines.entrySet().iterator(); it.hasNext();) {
+                Map.Entry<UUID, ContinuousRoutineInfo> e = it.next();
+
+                ContinuousRoutineInfo info = e.getValue();
+
+                if (!locRoutines.contains(info.routineId))
+                    it.remove();
+                else
+                    info.onDisconnected();
+            }
+        }
+    }
+
+    /**
+     * Removes all routines with autoUnsubscribe=false started by given node.
+     *
+     * @param nodeId Node ID.
+     */
+    void onNodeFail(UUID nodeId) {
+        synchronized (startedRoutines) {
+            for (Iterator<Map.Entry<UUID, ContinuousRoutineInfo>> it = startedRoutines.entrySet().iterator(); it.hasNext();) {
+                Map.Entry<UUID, ContinuousRoutineInfo> e = it.next();
+
+                ContinuousRoutineInfo info = e.getValue();
+
+                if (info.autoUnsubscribe && info.srcNodeId.equals(nodeId))
+                    it.remove();
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ContinuousRoutinesInfo.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesJoiningNodeDiscoveryData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesJoiningNodeDiscoveryData.java
new file mode 100644
index 0000000000000..9be6ef8e07eab
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/ContinuousRoutinesJoiningNodeDiscoveryData.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class ContinuousRoutinesJoiningNodeDiscoveryData implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    final List<ContinuousRoutineInfo> startedRoutines;
+
+    /**
+     * @param startedRoutines Routines registered on nodes, to be started in cluster.
+     */
+    ContinuousRoutinesJoiningNodeDiscoveryData(List<ContinuousRoutineInfo> startedRoutines) {
+        this.startedRoutines = startedRoutines;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ContinuousRoutinesJoiningNodeDiscoveryData.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 01a5a71a75f9c..cebe4b177e300 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -56,6 +56,8 @@
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfoBean;
 import org.apache.ignite.internal.managers.discovery.CustomEventListener;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.DiscoveryMessageResultsCollector;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.managers.eventstorage.HighPriorityListener;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
@@ -63,6 +65,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryHandler;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -147,6 +150,12 @@ public class GridContinuousProcessor extends GridProcessorAdapter {
     /** Query sequence number for message topic. */
     private final AtomicLong seq = new AtomicLong();
 
+    /** */
+    private ContinuousRoutinesInfo routinesInfo;
+
+    /** */
+    private int discoProtoVer;
+
     /**
      * @param ctx Kernal context.
      */
@@ -156,6 +165,11 @@ public GridContinuousProcessor(GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public void start() throws IgniteCheckedException {
+        discoProtoVer = ctx.discovery().mutableCustomMessages() ? 1 : 2;
+
+        if (discoProtoVer == 2)
+            routinesInfo = new ContinuousRoutinesInfo();
+
         if (ctx.config().isDaemon())
             return;
 
@@ -177,6 +191,8 @@ public GridContinuousProcessor(GridKernalContext ctx) {
                 @Override public void onCustomEvent(AffinityTopologyVersion topVer,
                     ClusterNode snd,
                     StartRoutineDiscoveryMessage msg) {
+                    assert discoProtoVer == 1 : discoProtoVer;
+
                     if (ctx.isStopping())
                         return;
 
@@ -184,6 +200,20 @@ public GridContinuousProcessor(GridKernalContext ctx) {
                 }
             });
 
+        ctx.discovery().setCustomEventListener(StartRoutineDiscoveryMessageV2.class,
+            new CustomEventListener<StartRoutineDiscoveryMessageV2>() {
+                @Override public void onCustomEvent(AffinityTopologyVersion topVer,
+                    ClusterNode snd,
+                    StartRoutineDiscoveryMessageV2 msg) {
+                    assert discoProtoVer == 2 : discoProtoVer;
+
+                    if (ctx.isStopping())
+                        return;
+
+                    processStartRequestV2(topVer, snd, msg);
+                }
+            });
+
         ctx.discovery().setCustomEventListener(StartRoutineAckDiscoveryMessage.class,
             new CustomEventListener<StartRoutineAckDiscoveryMessage>() {
                 @Override public void onCustomEvent(AffinityTopologyVersion topVer,
@@ -201,6 +231,9 @@ public GridContinuousProcessor(GridKernalContext ctx) {
                 @Override public void onCustomEvent(AffinityTopologyVersion topVer,
                     ClusterNode snd,
                     StopRoutineDiscoveryMessage msg) {
+                    if (discoProtoVer == 2)
+                        routinesInfo.removeRoutine(msg.routineId);
+
                     if (ctx.isStopping())
                         return;
 
@@ -222,32 +255,36 @@ public GridContinuousProcessor(GridKernalContext ctx) {
 
         ctx.io().addMessageListener(TOPIC_CONTINUOUS, new GridMessageListener() {
             @Override public void onMessage(UUID nodeId, Object obj, byte plc) {
-                GridContinuousMessage msg = (GridContinuousMessage)obj;
+                if (obj instanceof ContinuousRoutineStartResultMessage)
+                    processRoutineStartResultMessage(nodeId, (ContinuousRoutineStartResultMessage)obj);
+                else {
+                    GridContinuousMessage msg = (GridContinuousMessage)obj;
 
-                if (msg.data() == null && msg.dataBytes() != null) {
-                    try {
-                        msg.data(U.unmarshal(marsh, msg.dataBytes(), U.resolveClassLoader(ctx.config())));
-                    }
-                    catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to process message (ignoring): " + msg, e);
+                    if (msg.data() == null && msg.dataBytes() != null) {
+                        try {
+                            msg.data(U.unmarshal(marsh, msg.dataBytes(), U.resolveClassLoader(ctx.config())));
+                        }
+                        catch (IgniteCheckedException e) {
+                            U.error(log, "Failed to process message (ignoring): " + msg, e);
 
-                        return;
+                            return;
+                        }
                     }
-                }
 
-                switch (msg.type()) {
-                    case MSG_EVT_NOTIFICATION:
-                        processNotification(nodeId, msg);
+                    switch (msg.type()) {
+                        case MSG_EVT_NOTIFICATION:
+                            processNotification(nodeId, msg);
 
-                        break;
+                            break;
 
-                    case MSG_EVT_ACK:
-                        processMessageAck(msg);
+                        case MSG_EVT_ACK:
+                            processMessageAck(msg);
 
-                        break;
+                            break;
 
-                    default:
-                        assert false : "Unexpected message received: " + msg.type();
+                        default:
+                            assert false : "Unexpected message received: " + msg.type();
+                    }
                 }
             }
         });
@@ -341,6 +378,15 @@ public void unlockStopping() {
 
     /** {@inheritDoc} */
     @Override public void collectJoiningNodeData(DiscoveryDataBag dataBag) {
+        if (ctx.isDaemon())
+            return;
+
+        if (discoProtoVer == 2) {
+            routinesInfo.collectJoiningNodeData(dataBag);
+
+            return;
+        }
+
         Serializable data = getDiscoveryData(dataBag.joiningNodeId());
 
         if (data != null)
@@ -349,6 +395,15 @@ public void unlockStopping() {
 
     /** {@inheritDoc} */
     @Override public void collectGridNodeData(DiscoveryDataBag dataBag) {
+        if (ctx.isDaemon())
+            return;
+
+        if (discoProtoVer == 2) {
+            routinesInfo.collectGridNodeData(dataBag);
+
+            return;
+        }
+
         Serializable data = getDiscoveryData(dataBag.joiningNodeId());
 
         if (data != null)
@@ -393,6 +448,7 @@ private Serializable getDiscoveryData(UUID joiningNodeId) {
 
             return data;
         }
+
         return null;
     }
 
@@ -430,22 +486,118 @@ private Map<UUID, LocalRoutineInfo> copyLocalInfos(Map<UUID, LocalRoutineInfo> l
     @Override public void onJoiningNodeDataReceived(JoiningNodeDiscoveryData data) {
         if (log.isDebugEnabled()) {
             log.debug("onJoiningNodeDataReceived [joining=" + data.joiningNodeId() +
-                    ", loc=" + ctx.localNodeId() +
-                    ", data=" + data.joiningNodeData() +
-                    ']');
+                ", loc=" + ctx.localNodeId() +
+                ", data=" + data.joiningNodeData() +
+                ']');
         }
 
-        if (data.hasJoiningNodeData())
-            onDiscoDataReceived((DiscoveryData) data.joiningNodeData());
+        if (discoProtoVer == 2) {
+            if (data.hasJoiningNodeData())    {
+                ContinuousRoutinesJoiningNodeDiscoveryData nodeData = (ContinuousRoutinesJoiningNodeDiscoveryData)
+                    data.joiningNodeData();
+
+                for (ContinuousRoutineInfo routineInfo : nodeData.startedRoutines) {
+                    routinesInfo.addRoutineInfo(routineInfo);
+
+                    startDiscoveryDataRoutine(routineInfo);
+                }
+            }
+        }
+        else {
+            if (data.hasJoiningNodeData())
+                onDiscoDataReceived((DiscoveryData) data.joiningNodeData());
+        }
     }
 
     /** {@inheritDoc} */
     @Override public void onGridDataReceived(GridDiscoveryData data) {
-        Map<UUID, Serializable> nodeSpecData = data.nodeSpecificData();
+        if (discoProtoVer == 2) {
+            if (ctx.isDaemon())
+                return;
+
+            if (data.commonData() != null) {
+                ContinuousRoutinesCommonDiscoveryData commonData =
+                    (ContinuousRoutinesCommonDiscoveryData)data.commonData();
+
+                for (ContinuousRoutineInfo routineInfo : commonData.startedRoutines) {
+                    if (routinesInfo.routineExists(routineInfo.routineId))
+                        continue;
+
+                    routinesInfo.addRoutineInfo(routineInfo);
+
+                    startDiscoveryDataRoutine(routineInfo);
+                }
+            }
+        }
+        else {
+            Map<UUID, Serializable> nodeSpecData = data.nodeSpecificData();
+
+            if (nodeSpecData != null) {
+                for (Map.Entry<UUID, Serializable> e : nodeSpecData.entrySet())
+                    onDiscoDataReceived((DiscoveryData) e.getValue());
+            }
+        }
+    }
+
+    /**
+     * @param routineInfo Routine info.
+     */
+    private void startDiscoveryDataRoutine(ContinuousRoutineInfo routineInfo) {
+        IgnitePredicate<ClusterNode> nodeFilter = null;
+
+        try {
+            if (routineInfo.nodeFilter != null) {
+                nodeFilter = U.unmarshal(marsh, routineInfo.nodeFilter, U.resolveClassLoader(ctx.config()));
+
+                ctx.resource().injectGeneric(nodeFilter);
+            }
+        }
+        catch (IgniteCheckedException e) {
+            U.error(log, "Failed to unmarshal continuous routine filter, ignore routine [" +
+                "routineId=" + routineInfo.routineId +
+                ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+
+            return;
+        }
+
+        if (nodeFilter == null || nodeFilter.apply(ctx.discovery().localNode())) {
+            GridContinuousHandler hnd;
 
-        if (nodeSpecData != null) {
-            for (Map.Entry<UUID, Serializable> e : nodeSpecData.entrySet())
-                onDiscoDataReceived((DiscoveryData) e.getValue());
+            try {
+                hnd = U.unmarshal(marsh, routineInfo.hnd, U.resolveClassLoader(ctx.config()));
+
+                if (ctx.config().isPeerClassLoadingEnabled())
+                    hnd.p2pUnmarshal(routineInfo.srcNodeId, ctx);
+            }
+            catch (IgniteCheckedException e) {
+                U.error(log, "Failed to unmarshal continuous routine handler, ignore routine [" +
+                    "routineId=" + routineInfo.routineId +
+                    ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+
+                return;
+            }
+
+            try {
+                registerHandler(routineInfo.srcNodeId,
+                    routineInfo.routineId,
+                    hnd,
+                    routineInfo.bufSize,
+                    routineInfo.interval,
+                    routineInfo.autoUnsubscribe,
+                    false);
+            }
+            catch (IgniteCheckedException e) {
+                U.error(log, "Failed to register continuous routine handler, ignore routine [" +
+                    "routineId=" + routineInfo.routineId +
+                    ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+            }
+        }
+        else {
+            if (log.isDebugEnabled()) {
+                log.debug("Do not register continuous routine, rejected by node filter [" +
+                    "routineId=" + routineInfo.routineId +
+                    ", srcNodeId=" + routineInfo.srcNodeId + ']');
+            }
         }
     }
 
@@ -564,13 +716,14 @@ public void onCacheStop(GridCacheContext ctx) {
      * @param rmtFilter Remote filter.
      * @param prjPred Projection predicate.
      * @return Routine ID.
+     * @throws IgniteCheckedException If failed.
      */
     @SuppressWarnings("unchecked")
     public UUID registerStaticRoutine(
         String cacheName,
         CacheEntryUpdatedListener<?, ?> locLsnr,
         CacheEntryEventSerializableFilter rmtFilter,
-        @Nullable IgnitePredicate<ClusterNode> prjPred) {
+        @Nullable IgnitePredicate<ClusterNode> prjPred) throws IgniteCheckedException {
         String topicPrefix = "CONTINUOUS_QUERY_STATIC" + "_" + cacheName;
 
         CacheContinuousQueryHandler hnd = new CacheContinuousQueryHandler(
@@ -589,6 +742,17 @@ public UUID registerStaticRoutine(
 
         LocalRoutineInfo routineInfo = new LocalRoutineInfo(prjPred, hnd, 1, 0, true);
 
+        if (discoProtoVer == 2) {
+            routinesInfo.addRoutineInfo(createRoutineInfo(
+                ctx.localNodeId(),
+                routineId,
+                hnd,
+                prjPred,
+                routineInfo.bufSize,
+                routineInfo.interval,
+                routineInfo.autoUnsubscribe));
+        }
+
         locInfos.put(routineId, routineInfo);
 
         registerMessageListener(hnd);
@@ -596,6 +760,40 @@ public UUID registerStaticRoutine(
         return routineId;
     }
 
+    /**
+     * @param srcNodeId Source node ID.
+     * @param routineId Routine ID.
+     * @param hnd Handler.
+     * @param nodeFilter Node filter.
+     * @param bufSize Handler buffer size.
+     * @param interval Time interval.
+     * @param autoUnsubscribe Auto unsubscribe flag.
+     * @return Routine info instance.
+     * @throws IgniteCheckedException If failed.
+     */
+    private ContinuousRoutineInfo createRoutineInfo(
+        UUID srcNodeId,
+        UUID routineId,
+        GridContinuousHandler hnd,
+        @Nullable IgnitePredicate<ClusterNode> nodeFilter,
+        int bufSize,
+        long interval,
+        boolean autoUnsubscribe)
+        throws IgniteCheckedException {
+        byte[] hndBytes = marsh.marshal(hnd);
+
+        byte[] filterBytes = nodeFilter != null ? marsh.marshal(nodeFilter) : null;
+
+        return new ContinuousRoutineInfo(
+            srcNodeId,
+            routineId,
+            hndBytes,
+            filterBytes,
+            bufSize,
+            interval,
+            autoUnsubscribe);
+    }
+
     /**
      * @param hnd Handler.
      * @param bufSize Buffer size.
@@ -638,30 +836,10 @@ public IgniteInternalFuture<UUID> startRoutine(GridContinuousHandler hnd,
         // Whether local node is included in routine.
         boolean locIncluded = prjPred == null || prjPred.apply(ctx.discovery().localNode());
 
-        StartRequestData reqData = new StartRequestData(prjPred, hnd.clone(), bufSize, interval, autoUnsubscribe);
+        AbstractContinuousMessage msg;
 
         try {
-            if (ctx.config().isPeerClassLoadingEnabled()) {
-                // Handle peer deployment for projection predicate.
-                if (prjPred != null && !U.isGrid(prjPred.getClass())) {
-                    Class cls = U.detectClass(prjPred);
-
-                    String clsName = cls.getName();
-
-                    GridDeployment dep = ctx.deploy().deploy(cls, U.detectClassLoader(cls));
-
-                    if (dep == null)
-                        throw new IgniteDeploymentCheckedException("Failed to deploy projection predicate: " + prjPred);
-
-                    reqData.className(clsName);
-                    reqData.deploymentInfo(new GridDeploymentInfoBean(dep));
-
-                    reqData.p2pMarshal(marsh);
-                }
-
-                // Handle peer deployment for other handler-specific objects.
-                reqData.handler().p2pMarshal(ctx);
-            }
+            msg = createStartMessage(routineId, hnd, bufSize, interval, autoUnsubscribe, prjPred);
         }
         catch (IgniteCheckedException e) {
             return new GridFinishedFuture<>(e);
@@ -674,20 +852,26 @@ public IgniteInternalFuture<UUID> startRoutine(GridContinuousHandler hnd,
             return new GridFinishedFuture<>(new NodeStoppingException("Failed to start continuous query (node is stopping)"));
 
         try {
-            StartFuture fut = new StartFuture(ctx, routineId);
+            StartFuture fut = new StartFuture(routineId);
 
             startFuts.put(routineId, fut);
 
             try {
-                if (locIncluded || hnd.isQuery())
-                    registerHandler(ctx.localNodeId(), routineId, hnd, bufSize, interval, autoUnsubscribe, true);
+                if (locIncluded || hnd.isQuery()) {
+                    registerHandler(ctx.localNodeId(),
+                        routineId,
+                        hnd,
+                        bufSize,
+                        interval,
+                        autoUnsubscribe,
+                        true);
+                }
 
-                ctx.discovery().sendCustomEvent(new StartRoutineDiscoveryMessage(routineId, reqData,
-                    reqData.handler().keepBinary()));
-            }
-            catch (IgniteCheckedException e) {
-                startFuts.remove(routineId);
-                locInfos.remove(routineId);
+            ctx.discovery().sendCustomEvent(msg);
+        }
+        catch (IgniteCheckedException e) {
+            startFuts.remove(routineId);
+            locInfos.remove(routineId);
 
                 unregisterHandler(routineId, hnd, true);
 
@@ -706,6 +890,92 @@ public IgniteInternalFuture<UUID> startRoutine(GridContinuousHandler hnd,
         }
     }
 
+    /**
+     * @param routineId Routine ID.
+     * @param hnd Handler.
+     * @param bufSize Buffer size.
+     * @param interval Interval.
+     * @param autoUnsubscribe Auto unsubscribe flag.
+     * @param nodeFilter Node filter.
+     * @return Routine start message.
+     * @throws IgniteCheckedException If failed.
+     */
+    private AbstractContinuousMessage createStartMessage(UUID routineId,
+        GridContinuousHandler hnd,
+        int bufSize,
+        long interval,
+        boolean autoUnsubscribe,
+        @Nullable IgnitePredicate<ClusterNode> nodeFilter)
+        throws IgniteCheckedException
+    {
+        hnd = hnd.clone();
+
+        String clsName = null;
+        GridDeploymentInfoBean dep = null;
+
+        if (ctx.config().isPeerClassLoadingEnabled()) {
+            // Handle peer deployment for projection predicate.
+            if (nodeFilter != null && !U.isGrid(nodeFilter.getClass())) {
+                Class cls = U.detectClass(nodeFilter);
+
+                clsName = cls.getName();
+
+                GridDeployment dep0 = ctx.deploy().deploy(cls, U.detectClassLoader(cls));
+
+                if (dep0 == null)
+                    throw new IgniteDeploymentCheckedException("Failed to deploy projection predicate: " + nodeFilter);
+
+                dep = new GridDeploymentInfoBean(dep0);
+            }
+
+            // Handle peer deployment for other handler-specific objects.
+            hnd.p2pMarshal(ctx);
+        }
+
+        if (discoProtoVer == 1) {
+            StartRequestData reqData = new StartRequestData(
+                nodeFilter,
+                hnd,
+                bufSize,
+                interval,
+                autoUnsubscribe);
+
+            if (clsName != null) {
+                reqData.className(clsName);
+                reqData.deploymentInfo(dep);
+
+                reqData.p2pMarshal(marsh);
+            }
+
+            return new StartRoutineDiscoveryMessage(
+                routineId,
+                reqData,
+                reqData.handler().keepBinary());
+        }
+        else {
+            assert discoProtoVer == 2 : discoProtoVer;
+
+            byte[] nodeFilterBytes = nodeFilter != null ? U.marshal(marsh, nodeFilter) : null;
+            byte[] hndBytes =  U.marshal(marsh, hnd);
+
+            StartRequestDataV2 reqData = new StartRequestDataV2(nodeFilterBytes,
+                hndBytes,
+                bufSize,
+                interval,
+                autoUnsubscribe);
+
+            if (clsName != null) {
+                reqData.className(clsName);
+                reqData.deploymentInfo(dep);
+            }
+
+            return new StartRoutineDiscoveryMessageV2(
+                routineId,
+                reqData,
+                hnd.keepBinary());
+        }
+    }
+
     /**
      * @param hnd Handler.
      */
@@ -760,29 +1030,38 @@ public IgniteInternalFuture<?> stopRoutine(UUID routineId) {
                     doStop = true;
             }
 
-            if (doStop) {
-                // Unregister routine locally.
-                LocalRoutineInfo routine = locInfos.remove(routineId);
-
-                // Finish if routine is not found (wrong ID is provided).
-                if (routine == null) {
-                    stopFuts.remove(routineId);
+        if (doStop) {
+            boolean stop = false;
 
-                    fut.onDone();
+            // Unregister routine locally.
+            LocalRoutineInfo routine = locInfos.remove(routineId);
 
-                    return fut;
-                }
+            if (routine != null) {
+                stop = true;
 
                 // Unregister handler locally.
                 unregisterHandler(routineId, routine.hnd, true);
+            }
 
-                try {
-                    ctx.discovery().sendCustomEvent(new StopRoutineDiscoveryMessage(routineId));
-                }
-                catch (IgniteCheckedException e) {
-                    fut.onDone(e);
+            if (!stop && discoProtoVer == 2)
+                stop = routinesInfo.routineExists(routineId);
+
+            // Finish if routine is not found (wrong ID is provided).
+            if (!stop) {
+                stopFuts.remove(routineId);
+
+                    fut.onDone();
+
+                    return fut;
                 }
 
+            try {
+                ctx.discovery().sendCustomEvent(new StopRoutineDiscoveryMessage(routineId));
+            }
+            catch (IgniteCheckedException e) {
+                fut.onDone(e);
+            }
+
                 if (ctx.isStopping())
                     fut.onDone();
             }
@@ -924,6 +1203,9 @@ public void addNotification(UUID nodeId,
 
         clientInfos.clear();
 
+        if (discoProtoVer == 2)
+            routinesInfo.onClientDisconnected(locInfos.keySet());
+
         if (log.isDebugEnabled()) {
             log.debug("after onDisconnected [rmtInfos=" + rmtInfos +
                 ", locInfos=" + locInfos +
@@ -996,35 +1278,11 @@ private void processStartAckRequest(AffinityTopologyVersion topVer,
         StartFuture fut = startFuts.remove(msg.routineId());
 
         if (fut != null) {
-            if (msg.errs().isEmpty()) {
-                LocalRoutineInfo routine = locInfos.get(msg.routineId());
-
-                // Update partition counters.
-                if (routine != null && routine.handler().isQuery()) {
-                    Map<UUID, Map<Integer, T2<Long, Long>>> cntrsPerNode = msg.updateCountersPerNode();
-                    Map<Integer, T2<Long, Long>> cntrs = msg.updateCounters();
-
-                    GridCacheAdapter<Object, Object> interCache =
-                        ctx.cache().internalCache(routine.handler().cacheName());
-
-                    GridCacheContext cctx = interCache != null ? interCache.context() : null;
-
-                    if (cctx != null && cntrsPerNode != null && !cctx.isLocal() && cctx.affinityNode())
-                        cntrsPerNode.put(ctx.localNodeId(),
-                            toCountersMap(cctx.topology().localUpdateCounters(false)));
-
-                    routine.handler().updateCounters(topVer, cntrsPerNode, cntrs);
-                }
-
-                fut.onRemoteRegistered();
-            }
-            else {
-                IgniteCheckedException firstEx = F.first(msg.errs().values());
-
-                fut.onDone(firstEx);
-
-                stopRoutine(msg.routineId());
-            }
+            fut.onAllRemoteRegistered(
+                topVer,
+                msg.errs(),
+                msg.updateCountersPerNode(),
+                msg.updateCounters());
         }
     }
 
@@ -1137,6 +1395,199 @@ private void processStartRequest(ClusterNode node, StartRoutineDiscoveryMessage
             req.addError(ctx.localNodeId(), err);
     }
 
+    /**
+     * @param sndId Sender node ID.
+     * @param msg Message.
+     */
+    private void processRoutineStartResultMessage(UUID sndId, ContinuousRoutineStartResultMessage msg) {
+        StartFuture fut = startFuts.get(msg.routineId());
+
+        if (fut != null)
+            fut.onResult(sndId, msg);
+    }
+
+    /**
+     * @param topVer Current topology version.
+     * @param snd Sender.
+     * @param msg Start request.
+     */
+    private void processStartRequestV2(final AffinityTopologyVersion topVer,
+        final ClusterNode snd,
+        final StartRoutineDiscoveryMessageV2 msg) {
+        StartRequestDataV2 reqData = msg.startRequestData();
+
+        ContinuousRoutineInfo routineInfo = new ContinuousRoutineInfo(snd.id(),
+            msg.routineId(),
+            reqData.handlerBytes(),
+            reqData.nodeFilterBytes(),
+            reqData.bufferSize(),
+            reqData.interval(),
+            reqData.autoUnsubscribe());
+
+        routinesInfo.addRoutineInfo(routineInfo);
+
+        final DiscoCache discoCache = ctx.discovery().discoCache(topVer);
+
+        // Should not use marshaller and send messages from discovery thread.
+        ctx.getSystemExecutorService().execute(new Runnable() {
+            @Override public void run() {
+                if (snd.id().equals(ctx.localNodeId())) {
+                    StartFuture fut = startFuts.get(msg.routineId());
+
+                    if (fut != null)
+                        fut.initRemoteNodes(discoCache);
+
+                    return;
+                }
+
+                StartRequestDataV2 reqData = msg.startRequestData();
+
+                Exception err = null;
+
+                IgnitePredicate<ClusterNode> nodeFilter = null;
+
+                byte[] cntrs = null;
+
+                if (reqData.nodeFilterBytes() != null) {
+                    try {
+                        if (ctx.config().isPeerClassLoadingEnabled() && reqData.className() != null) {
+                            String clsName = reqData.className();
+                            GridDeploymentInfo depInfo = reqData.deploymentInfo();
+
+                            GridDeployment dep = ctx.deploy().getGlobalDeployment(depInfo.deployMode(),
+                                clsName,
+                                clsName,
+                                depInfo.userVersion(),
+                                snd.id(),
+                                depInfo.classLoaderId(),
+                                depInfo.participants(),
+                                null);
+
+                            if (dep == null) {
+                                throw new IgniteDeploymentCheckedException("Failed to obtain deployment " +
+                                    "for class: " + clsName);
+                            }
+
+                            nodeFilter = U.unmarshal(marsh,
+                                reqData.nodeFilterBytes(),
+                                U.resolveClassLoader(dep.classLoader(), ctx.config()));
+                        }
+                        else {
+                            nodeFilter = U.unmarshal(marsh,
+                                reqData.nodeFilterBytes(),
+                                U.resolveClassLoader(ctx.config()));
+                        }
+
+                        if (nodeFilter != null)
+                            ctx.resource().injectGeneric(nodeFilter);
+                    }
+                    catch (Exception e) {
+                        err = e;
+
+                        U.error(log, "Failed to unmarshal continuous routine filter [" +
+                            "routineId=" + msg.routineId +
+                            ", srcNodeId=" + snd.id() + ']', e);
+                    }
+                }
+
+                boolean register = err == null &&
+                    (nodeFilter == null || nodeFilter.apply(ctx.discovery().localNode()));
+
+                if (register) {
+                    try {
+                        GridContinuousHandler hnd = U.unmarshal(marsh,
+                            reqData.handlerBytes(),
+                            U.resolveClassLoader(ctx.config()));
+
+                        if (ctx.config().isPeerClassLoadingEnabled())
+                            hnd.p2pUnmarshal(snd.id(), ctx);
+
+                        if (msg.keepBinary()) {
+                            assert hnd instanceof CacheContinuousQueryHandler : hnd;
+
+                            ((CacheContinuousQueryHandler)hnd).keepBinary(true);
+                        }
+
+                        GridContinuousHandler hnd0 = hnd instanceof GridMessageListenHandler ?
+                            new GridMessageListenHandler((GridMessageListenHandler)hnd) :
+                            hnd;
+
+                        registerHandler(snd.id(),
+                            msg.routineId,
+                            hnd0,
+                            reqData.bufferSize(),
+                            reqData.interval(),
+                            reqData.autoUnsubscribe(),
+                            false);
+
+                        if (hnd0.isQuery()) {
+                            GridCacheProcessor proc = ctx.cache();
+
+                            if (proc != null) {
+                                GridCacheAdapter cache = ctx.cache().internalCache(hnd0.cacheName());
+
+                                if (cache != null && !cache.isLocal() && cache.context().userCache()) {
+                                    CachePartitionPartialCountersMap cntrsMap =
+                                        cache.context().topology().localUpdateCounters(false);
+
+                                    cntrs = U.marshal(marsh, cntrsMap);
+                                }
+                            }
+                        }
+                    }
+                    catch (Exception e) {
+                        err = e;
+
+                        U.error(log, "Failed to register continuous routine handler [" +
+                            "routineId=" + msg.routineId +
+                            ", srcNodeId=" + snd.id() + ']', e);
+                    }
+                }
+
+                sendMessageStartResult(snd, msg.routineId(), cntrs, err);
+            }
+        });
+    }
+
+    /**
+     * @param node Target node.
+     * @param routineId Routine ID.
+     * @param cntrsMapBytes Marshalled {@link CachePartitionPartialCountersMap}.
+     * @param err Start error if any.
+     */
+    private void sendMessageStartResult(final ClusterNode node,
+        final UUID routineId,
+        byte[] cntrsMapBytes,
+        final @Nullable Exception err)
+    {
+        byte[] errBytes = null;
+
+        if (err != null) {
+            try {
+                errBytes = U.marshal(marsh, err);
+            }
+            catch (Exception e) {
+                U.error(log, "Failed to marshal routine start error: " + e, e);
+            }
+        }
+
+        ContinuousRoutineStartResultMessage msg = new ContinuousRoutineStartResultMessage(routineId,
+            cntrsMapBytes,
+            errBytes,
+            err != null);
+
+        try {
+            ctx.io().sendToGridTopic(node, TOPIC_CONTINUOUS, msg, SYSTEM_POOL, null);
+        }
+        catch (ClusterTopologyCheckedException e) {
+            if (log.isDebugEnabled())
+                log.debug("Failed to send routine start result, node failed: " + e);
+        }
+        catch (IgniteCheckedException e) {
+            U.error(log, "Failed to send routine start result: " + e, e);
+        }
+    }
+
     /**
      * @param msg Message.
      */
@@ -1455,6 +1906,13 @@ private class DiscoveryListener implements GridLocalEventListener, HighPriorityL
 
             UUID nodeId = ((DiscoveryEvent)evt).eventNode().id();
 
+            if (discoProtoVer == 2) {
+                routinesInfo.onNodeFail(nodeId);
+
+                for (StartFuture fut : startFuts.values())
+                    fut.onNodeFail(nodeId);
+            }
+
             clientInfos.remove(nodeId);
 
             // Unregister handlers created by left node.
@@ -1894,10 +2352,7 @@ public DiscoveryDataItem() {
     /**
      * Future for start routine.
      */
-    private static class StartFuture extends GridFutureAdapter<UUID> {
-        /** */
-        private GridKernalContext ctx;
-
+    private class StartFuture extends GridFutureAdapter<UUID> {
         /** Consume ID. */
         private UUID routineId;
 
@@ -1907,56 +2362,170 @@ private static class StartFuture extends GridFutureAdapter<UUID> {
         /** All remote listeners are registered. */
         private volatile boolean rmt;
 
-        /** Timeout object. */
-        private volatile GridTimeoutObject timeoutObj;
+        /** */
+        private final DiscoveryMessageResultsCollector<ContinuousRoutineStartResultMessage, RoutineRegisterResults>
+            resCollect;
 
         /**
-         * @param ctx Kernal context.
          * @param routineId Consume ID.
          */
-        StartFuture(GridKernalContext ctx, UUID routineId) {
-            this.ctx = ctx;
-
+        StartFuture(UUID routineId) {
             this.routineId = routineId;
+
+            resCollect = new DiscoveryMessageResultsCollector<ContinuousRoutineStartResultMessage, RoutineRegisterResults>(ctx) {
+                @Override protected RoutineRegisterResults createResult(Map<UUID, NodeMessage<ContinuousRoutineStartResultMessage>> rcvd) {
+                    Map<UUID, Exception> errs = null;
+                    Map<UUID, Map<Integer, T2<Long, Long>>> cntrsPerNode = null;
+
+                    for (Map.Entry<UUID, NodeMessage<ContinuousRoutineStartResultMessage>> entry : rcvd.entrySet()) {
+                        ContinuousRoutineStartResultMessage msg = entry.getValue().message();
+
+                        if (msg == null)
+                            continue;
+
+                        if (msg.error()) {
+                            byte[] errBytes = msg.errorBytes();
+
+                            Exception err = null;
+
+                            if (errBytes != null) {
+                                try {
+                                    err = U.unmarshal(marsh, errBytes, U.resolveClassLoader(ctx.config()));
+                                }
+                                catch (Exception e) {
+                                    U.warn(log, "Failed to unmarhal continuous routine start error: " + e);
+                                }
+                            }
+
+                            if (err == null) {
+                                err = new IgniteCheckedException("Failed to start continuous " +
+                                    "routine on node: " + entry.getKey());
+                            }
+
+                            if (errs == null)
+                                errs = new HashMap<>();
+
+                            errs.put(entry.getKey(), err);
+                        }
+                        else {
+                            byte[] cntrsMapBytes = msg.countersMapBytes();
+
+                            if (cntrsMapBytes != null) {
+                                try {
+                                    CachePartitionPartialCountersMap cntrsMap = U.unmarshal(
+                                        marsh,
+                                        cntrsMapBytes,
+                                        U.resolveClassLoader(ctx.config()));
+
+                                    if (cntrsPerNode == null)
+                                        cntrsPerNode = new HashMap<>();
+
+                                    cntrsPerNode.put(entry.getKey(), CachePartitionPartialCountersMap.toCountersMap(cntrsMap));
+                                }
+                                catch (Exception e) {
+                                    U.warn(log, "Failed to unmarhal continuous query update counters: " + e);
+                                }
+                            }
+                        }
+                    }
+
+                    return new RoutineRegisterResults(discoCache.version(), errs, cntrsPerNode);
+                }
+
+                @Override protected void onResultsCollected(RoutineRegisterResults res0) {
+                    onAllRemoteRegistered(res0.topVer, res0.errs, res0.cntrsPerNode, null);
+                }
+
+                @Override protected boolean waitForNode(DiscoCache discoCache, ClusterNode node) {
+                    return !ctx.localNodeId().equals(node.id());
+                }
+            };
         }
 
         /**
-         * Called when local listener is registered.
+         * @param topVer Topology version.
+         * @param errs Errors.
+         * @param cntrsPerNode Update counters.
+         * @param cntrs Update counters.
          */
-        public void onLocalRegistered() {
-            loc = true;
+        private void onAllRemoteRegistered(
+            AffinityTopologyVersion topVer,
+            @Nullable Map<UUID, ? extends Exception> errs,
+            Map<UUID, Map<Integer, T2<Long, Long>>> cntrsPerNode,
+            Map<Integer, T2<Long, Long>> cntrs) {
+            try {
+                if (errs == null || errs.isEmpty()) {
+                    LocalRoutineInfo routine = locInfos.get(routineId);
 
-            if (rmt && !isDone())
-                onDone(routineId);
+                    // Update partition counters.
+                    if (routine != null && routine.handler().isQuery()) {
+                        GridCacheAdapter<Object, Object> interCache =
+                            ctx.cache().internalCache(routine.handler().cacheName());
+
+                        GridCacheContext cctx = interCache != null ? interCache.context() : null;
+
+                        if (cctx != null && cntrsPerNode != null && !cctx.isLocal() && cctx.affinityNode())
+                            cntrsPerNode.put(ctx.localNodeId(),
+                                toCountersMap(cctx.topology().localUpdateCounters(false)));
+
+                        routine.handler().updateCounters(topVer, cntrsPerNode, cntrs);
+                    }
+
+                    onRemoteRegistered();
+                }
+                else {
+                    Exception firstEx = F.first(errs.values());
+
+                    onDone(firstEx);
+
+                    stopRoutine(routineId);
+                }
+            }
+            finally {
+                startFuts.remove(routineId, this);
+            }
         }
 
         /**
-         * Called when all remote listeners are registered.
+         * @param discoCache Discovery state.
          */
-        public void onRemoteRegistered() {
-            rmt = true;
+        void initRemoteNodes(DiscoCache discoCache) {
+            resCollect.init(discoCache);
+        }
 
-            if (loc && !isDone())
-                onDone(routineId);
+        /**
+         * @param nodeId Node ID.
+         * @param msg Message.
+         */
+        void onResult(UUID nodeId, ContinuousRoutineStartResultMessage msg) {
+            resCollect.onMessage(nodeId, msg);
         }
 
         /**
-         * @param timeoutObj Timeout object.
+         * @param nodeId Failed node ID.
          */
-        public void addTimeoutObject(GridTimeoutObject timeoutObj) {
-            assert timeoutObj != null;
+        void onNodeFail(UUID nodeId) {
+            resCollect.onNodeFail(nodeId);
+        }
 
-            this.timeoutObj = timeoutObj;
+        /**
+         * Called when local listener is registered.
+         */
+        void onLocalRegistered() {
+            loc = true;
 
-            ctx.timeout().addTimeoutObject(timeoutObj);
+            if (rmt && !isDone())
+                onDone(routineId);
         }
 
-        /** {@inheritDoc} */
-        @Override public boolean onDone(@Nullable UUID res, @Nullable Throwable err) {
-            if (timeoutObj != null)
-                ctx.timeout().removeTimeoutObject(timeoutObj);
+        /**
+         * Called when all remote listeners are registered.
+         */
+        void onRemoteRegistered() {
+            rmt = true;
 
-            return super.onDone(res, err);
+            if (loc && !isDone())
+                onDone(routineId);
         }
 
         /** {@inheritDoc} */
@@ -1965,6 +2534,33 @@ public void addTimeoutObject(GridTimeoutObject timeoutObj) {
         }
     }
 
+    /**
+     *
+     */
+    private static class RoutineRegisterResults {
+        /** */
+        private final AffinityTopologyVersion topVer;
+
+        /** */
+        private final Map<UUID, ? extends Exception> errs;
+
+        /** */
+        private final Map<UUID, Map<Integer, T2<Long, Long>>> cntrsPerNode;
+
+        /**
+         * @param topVer Topology version.
+         * @param errs Errors.
+         * @param cntrsPerNode Update counters.
+         */
+        RoutineRegisterResults(AffinityTopologyVersion topVer,
+            Map<UUID, ? extends Exception> errs,
+            Map<UUID, Map<Integer, T2<Long, Long>>> cntrsPerNode) {
+            this.topVer = topVer;
+            this.errs = errs;
+            this.cntrsPerNode = cntrsPerNode;
+        }
+    }
+
     /**
      * Future for stop routine.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRequestDataV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRequestDataV2.java
new file mode 100644
index 0000000000000..c001616eab3c1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRequestDataV2.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Start request data.
+ */
+class StartRequestDataV2 implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Serialized node filter. */
+    private byte[] nodeFilterBytes;
+
+    /** Deployment class name. */
+    private String clsName;
+
+    /** Deployment info. */
+    private GridDeploymentInfo depInfo;
+
+    /** Serialized handler. */
+    private byte[] hndBytes;
+
+    /** Buffer size. */
+    private int bufSize;
+
+    /** Time interval. */
+    private long interval;
+
+    /** Automatic unsubscribe flag. */
+    private boolean autoUnsubscribe;
+
+    /**
+     * @param nodeFilterBytes Serialized node filter.
+     * @param hndBytes Serialized handler.
+     * @param bufSize Buffer size.
+     * @param interval Time interval.
+     * @param autoUnsubscribe Automatic unsubscribe flag.
+     */
+    StartRequestDataV2(
+        byte[] nodeFilterBytes,
+        byte[] hndBytes,
+        int bufSize,
+        long interval,
+        boolean autoUnsubscribe) {
+        assert hndBytes != null;
+        assert bufSize > 0;
+        assert interval >= 0;
+
+        this.nodeFilterBytes = nodeFilterBytes;
+        this.hndBytes = hndBytes;
+        this.bufSize = bufSize;
+        this.interval = interval;
+        this.autoUnsubscribe = autoUnsubscribe;
+    }
+
+    /**
+     * @return Serialized node filter.
+     */
+    public byte[] nodeFilterBytes() {
+        return nodeFilterBytes;
+    }
+
+    /**
+     * @return Deployment class name.
+     */
+    public String className() {
+        return clsName;
+    }
+
+    /**
+     * @param clsName New deployment class name.
+     */
+    public void className(String clsName) {
+        this.clsName = clsName;
+    }
+
+    /**
+     * @return Deployment info.
+     */
+    public GridDeploymentInfo deploymentInfo() {
+        return depInfo;
+    }
+
+    /**
+     * @param depInfo New deployment info.
+     */
+    public void deploymentInfo(GridDeploymentInfo depInfo) {
+        this.depInfo = depInfo;
+    }
+
+    /**
+     * @return Handler.
+     */
+    public byte[] handlerBytes() {
+        return hndBytes;
+    }
+
+    /**
+     * @return Buffer size.
+     */
+    public int bufferSize() {
+        return bufSize;
+    }
+
+    /**
+     * @param bufSize New buffer size.
+     */
+    public void bufferSize(int bufSize) {
+        this.bufSize = bufSize;
+    }
+
+    /**
+     * @return Time interval.
+     */
+    public long interval() {
+        return interval;
+    }
+
+    /**
+     * @param interval New time interval.
+     */
+    public void interval(long interval) {
+        this.interval = interval;
+    }
+
+    /**
+     * @return Automatic unsubscribe flag.
+     */
+    public boolean autoUnsubscribe() {
+        return autoUnsubscribe;
+    }
+
+    /**
+     * @param autoUnsubscribe New automatic unsubscribe flag.
+     */
+    public void autoUnsubscribe(boolean autoUnsubscribe) {
+        this.autoUnsubscribe = autoUnsubscribe;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(StartRequestDataV2.class, this);
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRoutineDiscoveryMessageV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRoutineDiscoveryMessageV2.java
new file mode 100644
index 0000000000000..275765da73050
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StartRoutineDiscoveryMessageV2.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.continuous;
+
+import java.util.UUID;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class StartRoutineDiscoveryMessageV2 extends AbstractContinuousMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private static final int KEEP_BINARY_FLAG = 0x01;
+
+    /** */
+    private final StartRequestDataV2 startReqData;
+
+    /** Flags. */
+    private int flags;
+
+    /**
+     * @param routineId Routine id.
+     * @param startReqData Start request data.
+     * @param keepBinary Keep binary flag.
+     */
+    StartRoutineDiscoveryMessageV2(UUID routineId, StartRequestDataV2 startReqData, boolean keepBinary) {
+        super(routineId);
+
+        this.startReqData = startReqData;
+
+        if (keepBinary)
+            flags |= KEEP_BINARY_FLAG;
+    }
+
+    /**
+     * @return Start request data.
+     */
+    public StartRequestDataV2 startRequestData() {
+        return startReqData;
+    }
+
+    /**
+     * @return {@code True} if keep binary flag was set on continuous handler.
+     */
+    public boolean keepBinary() {
+        return (flags & KEEP_BINARY_FLAG) != 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public DiscoveryCustomMessage ackMessage() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(StartRoutineDiscoveryMessageV2.class, this, "routineId", routineId());
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StopRoutineAckDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StopRoutineAckDiscoveryMessage.java
index 79d8b29a24081..dfba0e759f9f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StopRoutineAckDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/StopRoutineAckDiscoveryMessage.java
@@ -41,6 +41,11 @@ public StopRoutineAckDiscoveryMessage(UUID routineId) {
         return null;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(StopRoutineAckDiscoveryMessage.class, this, "routineId", routineId());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index 4a893f4096877..8cad342a7e696 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -47,6 +47,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.IgniteDataStreamerTimeoutException;
 import org.apache.ignite.IgniteException;
@@ -100,6 +101,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.GPC;
@@ -1059,6 +1061,9 @@ private void doFlush() throws IgniteCheckedException {
             return;
 
         while (true) {
+            if (disconnectErr != null)
+                throw disconnectErr;
+
             Queue<IgniteInternalFuture<?>> q = null;
 
             for (Buffer buf : bufMappings.values()) {
@@ -1826,15 +1831,19 @@ private void submit(
                 catch (IgniteCheckedException e) {
                     GridFutureAdapter<Object> fut0 = ((GridFutureAdapter<Object>)fut);
 
-                    try {
-                        if (ctx.discovery().alive(node) && ctx.discovery().pingNode(node.id()))
-                            fut0.onDone(e);
-                        else
-                            fut0.onDone(new ClusterTopologyCheckedException("Failed to send request (node has left): "
-                                + node.id()));
-                    }
-                    catch (IgniteClientDisconnectedCheckedException e0) {
-                        fut0.onDone(e0);
+                    if (X.hasCause(e, IgniteClientDisconnectedCheckedException.class, IgniteClientDisconnectedException.class))
+                        fut0.onDone(e);
+                    else {
+                        try {
+                            if (ctx.discovery().alive(node) && ctx.discovery().pingNode(node.id()))
+                                fut0.onDone(e);
+                            else
+                                fut0.onDone(new ClusterTopologyCheckedException("Failed to send request (node has left): "
+                                    + node.id()));
+                        }
+                        catch (IgniteClientDisconnectedCheckedException e0) {
+                            fut0.onDone(e0);
+                        }
                     }
                 }
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingAcceptedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingAcceptedMessage.java
index 7af0559752b3c..80e3f7dafd6ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingAcceptedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingAcceptedMessage.java
@@ -62,6 +62,11 @@ public class MappingAcceptedMessage implements DiscoveryCustomMessage {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingProposedMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingProposedMessage.java
index b4e13fba14bf0..93585858668d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingProposedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/marshaller/MappingProposedMessage.java
@@ -97,6 +97,11 @@ private enum ProposalStatus {
         return true;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
         AffinityTopologyVersion topVer, DiscoCache discoCache) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaFinishDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaFinishDiscoveryMessage.java
index 2245b24667ac7..f802e090dfa39 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaFinishDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaFinishDiscoveryMessage.java
@@ -58,6 +58,11 @@ public SchemaFinishDiscoveryMessage(SchemaAbstractOperation op, SchemaOperationE
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean exchange() {
         return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaProposeDiscoveryMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaProposeDiscoveryMessage.java
index 0e1270b17b623..62b6d6abb9f0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaProposeDiscoveryMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaProposeDiscoveryMessage.java
@@ -59,6 +59,11 @@ public SchemaProposeDiscoveryMessage(SchemaAbstractOperation op) {
         return true;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean exchange() {
         return exchange;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index e0ec8d1abf103..0fcde0e136644 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -489,6 +489,17 @@ public GridNioFuture<Boolean> close(GridNioSession ses) {
         return fut;
     }
 
+    /**
+     * @param ses Session.
+     */
+    public void closeFromWorkerThread(GridNioSession ses) {
+        assert ses instanceof GridSelectorNioSessionImpl : ses;
+
+        GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl)ses;
+
+        ((AbstractNioClientWorker)ses0.worker()).close((GridSelectorNioSessionImpl)ses, null);
+    }
+
     /**
      * @param ses Session.
      * @param msg Message.
@@ -2170,7 +2181,12 @@ private void dumpStats(StringBuilder sb,
                 dumpSelectorInfo(sb, keys);
 
             for (SelectionKey key : keys) {
-                GridSelectorNioSessionImpl ses = (GridSelectorNioSessionImpl)key.attachment();
+                GridNioKeyAttachment attach = (GridNioKeyAttachment)key.attachment();
+
+                if (!attach.hasSession())
+                    continue;
+
+                GridSelectorNioSessionImpl ses = attach.session();
 
                 boolean sesInfo = p == null || p.apply(ses);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index 1754cc8cfcb66..e8c27d2f90626 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -963,5 +963,15 @@ private class GridDummySpiContext implements IgniteSpiContext {
         @Override public Map<String, Object> nodeAttributes() {
             return Collections.emptyMap();
         }
+
+        /** {@inheritDoc} */
+        @Override public boolean communicationFailureResolveSupported() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void resolveCommunicationFailure(ClusterNode node, Exception err) {
+            throw new UnsupportedOperationException();
+        }
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
index 108c4d443c6e0..d4402f41e1928 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiContext.java
@@ -365,4 +365,15 @@ public interface IgniteSpiContext {
      * @return Current node attributes.
      */
     public Map<String, Object> nodeAttributes();
+
+    /**
+     * @return {@code True} if cluster supports communication error resolving.
+     */
+    public boolean communicationFailureResolveSupported();
+
+    /**
+     * @param node Problem node.
+     * @param err Error.
+     */
+    public void resolveCommunicationFailure(ClusterNode node, Exception err);
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index c5f366bfb8d26..a3fccbcad5da2 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -32,6 +32,7 @@
 import java.nio.channels.spi.AbstractInterruptibleChannel;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.BitSet;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -65,12 +66,14 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.managers.eventstorage.HighPriorityListener;
 import org.apache.ignite.internal.util.GridConcurrentFactory;
 import org.apache.ignite.internal.util.GridSpinReadWriteLock;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
 import org.apache.ignite.internal.util.ipc.IpcEndpoint;
 import org.apache.ignite.internal.util.ipc.IpcToNioAdapter;
 import org.apache.ignite.internal.util.ipc.shmem.IpcOutOfSystemResourcesException;
@@ -133,6 +136,9 @@
 import org.apache.ignite.spi.IgniteSpiTimeoutObject;
 import org.apache.ignite.spi.communication.CommunicationListener;
 import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.internal.ConnectionKey;
+import org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture;
+import org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationNodeConnectionCheckFuture;
 import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage;
 import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2;
 import org.apache.ignite.spi.communication.tcp.messages.NodeIdMessage;
@@ -145,6 +151,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
+import static org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture.SES_FUT_META;
 import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.ALREADY_CONNECTED;
 import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.NEED_WAIT;
 import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.NODE_STOPPING;
@@ -309,7 +316,7 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
     private static final IgniteProductVersion VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT = IgniteProductVersion.fromString("2.1.4");
 
     /** Connection index meta for session. */
-    private static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
+    public static final int CONN_IDX_META = GridNioSessionMetaKey.nextUniqueKey();
 
     /** Message tracker meta for session. */
     private static final int TRACKER_META = GridNioSessionMetaKey.nextUniqueKey();
@@ -407,6 +414,9 @@ public class TcpCommunicationSpi extends IgniteSpiAdapter implements Communicati
                 ConnectionKey connId = ses.meta(CONN_IDX_META);
 
                 if (connId != null) {
+                    if (connId.dummy())
+                        return;
+
                     UUID id = connId.nodeId();
 
                     GridCommunicationClient[] nodeClients = clients.get(id);
@@ -480,20 +490,22 @@ private void onFirstMessage(final GridNioSession ses, Message msg) {
                 if (rmtNode == null) {
                     DiscoverySpi discoverySpi = ignite().configuration().getDiscoverySpi();
 
-                    assert discoverySpi instanceof TcpDiscoverySpi;
-
-                    TcpDiscoverySpi tcpDiscoverySpi = (TcpDiscoverySpi) discoverySpi;
+                    boolean unknownNode = true;
 
-                    ClusterNode node0 = tcpDiscoverySpi.getNode0(sndId);
+                    if (discoverySpi instanceof TcpDiscoverySpi) {
+                        TcpDiscoverySpi tcpDiscoverySpi = (TcpDiscoverySpi) discoverySpi;
 
-                    boolean unknownNode = true;
+                        ClusterNode node0 = tcpDiscoverySpi.getNode0(sndId);
 
-                    if (node0 != null) {
-                        assert node0.isClient() : node0;
+                        if (node0 != null) {
+                            assert node0.isClient() : node0;
 
-                        if (node0.version().compareTo(VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT) >= 0)
-                            unknownNode = false;
+                            if (node0.version().compareTo(VERSION_SINCE_CLIENT_COULD_WAIT_TO_CONNECT) >= 0)
+                                unknownNode = false;
+                        }
                     }
+                    else if (discoverySpi instanceof IgniteDiscoverySpi)
+                        unknownNode = !((IgniteDiscoverySpi) discoverySpi).knownNode(sndId);
 
                     if (unknownNode) {
                         U.warn(log, "Close incoming connection, unknown node [nodeId=" + sndId + ", ses=" + ses + ']');
@@ -708,9 +720,9 @@ private void closeStaleConnections(ConnectionKey connKey) {
                     }
                 }
                 else {
-                    metricsLsnr.onMessageReceived(msg, connKey.nodeId());
-
                     if (msg instanceof RecoveryLastReceivedMessage) {
+                        metricsLsnr.onMessageReceived(msg, connKey.nodeId());
+
                         GridNioRecoveryDescriptor recovery = ses.outRecoveryDescriptor();
 
                         if (recovery != null) {
@@ -723,9 +735,9 @@ private void closeStaleConnections(ConnectionKey connKey) {
                             }
 
                             recovery.ackReceived(msg0.received());
-
-                            return;
                         }
+
+                        return;
                     }
                     else {
                         GridNioRecoveryDescriptor recovery = ses.inRecoveryDescriptor();
@@ -745,8 +757,23 @@ private void closeStaleConnections(ConnectionKey connKey) {
                                 recovery.lastAcknowledged(rcvCnt);
                             }
                         }
+                        else if (connKey.dummy()) {
+                            assert msg instanceof NodeIdMessage : msg;
+
+                            TcpCommunicationNodeConnectionCheckFuture fut = ses.meta(SES_FUT_META);
+
+                            assert fut != null : msg;
+
+                            fut.onConnected(U.bytesToUuid(((NodeIdMessage)msg).nodeIdBytes(), 0));
+
+                            nioSrvr.closeFromWorkerThread(ses);
+
+                            return;
+                        }
                     }
 
+                    metricsLsnr.onMessageReceived(msg, connKey.nodeId());
+
                     IgniteRunnable c;
 
                     if (msgQueueLimit > 0) {
@@ -2111,6 +2138,13 @@ private void dumpInfo(StringBuilder sb, UUID dstNodeId) {
         }
     }
 
+    /**
+     * @return Bound TCP server port.
+     */
+    public int boundPort() {
+        return boundTcpPort;
+    }
+
     /** {@inheritDoc} */
     @Override public void spiStart(String igniteInstanceName) throws IgniteSpiException {
         assert locHost != null;
@@ -2568,6 +2602,27 @@ private void checkAttributePresence(ClusterNode node, String attrName) {
         sendMessage0(node, msg, null);
     }
 
+    /**
+     * @param nodes Nodes to check connection with.
+     * @return Result future (each bit in result BitSet contains connection status to corresponding node).
+     */
+    public IgniteFuture<BitSet> checkConnection(List<ClusterNode> nodes) {
+        TcpCommunicationConnectionCheckFuture fut = new TcpCommunicationConnectionCheckFuture(
+            this,
+            log.getLogger(TcpCommunicationConnectionCheckFuture.class),
+            nioSrvr,
+            nodes);
+
+        long timeout = failureDetectionTimeoutEnabled() ? failureDetectionTimeout() : connTimeout;
+
+        if (log.isInfoEnabled())
+            log.info("Start check connection process [nodeCnt=" + nodes.size() + ", timeout=" + timeout + ']');
+
+        fut.init(timeout);
+
+        return new IgniteFutureImpl<>(fut);
+    }
+
     /**
      * Sends given message to destination node. Note that characteristics of the
      * exchange such as durability, guaranteed delivery or error notification is
@@ -2988,7 +3043,7 @@ private void checkClientQueueSize(GridNioSession ses, int msgQueueSize) {
             ConnectionKey id = ses.meta(CONN_IDX_META);
 
             if (id != null) {
-                ClusterNode node = getSpiContext().node(id.nodeId);
+                ClusterNode node = getSpiContext().node(id.nodeId());
 
                 if (node != null && node.isClient()) {
                     String msg = "Client node outbound message queue size exceeded slowClientQueueLimit, " +
@@ -3009,9 +3064,20 @@ private void checkClientQueueSize(GridNioSession ses, int msgQueueSize) {
     /**
      * @param node Node.
      * @return Node addresses.
+     * @throws IgniteCheckedException If failed.
+     */
+    private Collection<InetSocketAddress> nodeAddresses(ClusterNode node) throws IgniteCheckedException {
+        return nodeAddresses(node, filterReachableAddresses);
+    }
+
+    /**
+     * @param node Node.
+     * @param filterReachableAddresses Filter addresses flag.
+     * @return Node addresses.
      * @throws IgniteCheckedException If node does not have addresses.
      */
-    private LinkedHashSet<InetSocketAddress> nodeAddresses(ClusterNode node) throws IgniteCheckedException {
+    public Collection<InetSocketAddress> nodeAddresses(ClusterNode node, boolean filterReachableAddresses)
+        throws IgniteCheckedException {
         Collection<String> rmtAddrs0 = node.attribute(createSpiAttributeName(ATTR_ADDRS));
         Collection<String> rmtHostNames0 = node.attribute(createSpiAttributeName(ATTR_HOST_NAMES));
         Integer boundPort = node.attribute(createSpiAttributeName(ATTR_PORT));
@@ -3092,7 +3158,7 @@ private LinkedHashSet<InetSocketAddress> nodeAddresses(ClusterNode node) throws
      * @throws IgniteCheckedException If failed.
      */
     protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
-        LinkedHashSet<InetSocketAddress> addrs = nodeAddresses(node);
+        Collection<InetSocketAddress> addrs = nodeAddresses(node);
 
         GridCommunicationClient client = null;
         IgniteCheckedException errs = null;
@@ -3110,6 +3176,9 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
             int lastWaitingTimeout = 1;
 
             while (client == null) { // Reconnection on handshake timeout.
+                if (stopping)
+                    throw new IgniteSpiException("Node is stopping.");
+
                 if (addr.getAddress().isLoopbackAddress() && addr.getPort() == boundTcpPort) {
                     if (log.isDebugEnabled())
                         log.debug("Skipping local address [addr=" + addr +
@@ -3350,8 +3419,18 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     "operating system firewall is disabled on local and remote hosts) " +
                     "[addrs=" + addrs + ']');
 
-            if (enableForcibleNodeKill) {
-                if (getSpiContext().node(node.id()) != null
+            boolean commErrResolve = false;
+
+            IgniteSpiContext ctx = getSpiContext();
+
+            if (connectionError(errs) && ctx.communicationFailureResolveSupported()) {
+                commErrResolve = true;
+
+                ctx.resolveCommunicationFailure(node, errs);
+            }
+
+            if (!commErrResolve && enableForcibleNodeKill) {
+                if (ctx.node(node.id()) != null
                     && (CU.clientNode(node) ||  !CU.clientNode(getLocalNode())) &&
                     connectionError(errs)) {
                     String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
@@ -3362,7 +3441,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     else
                         U.warn(log, msg);
 
-                    getSpiContext().failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
+                    ctx.failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
                         "rmtNode=" + node +
                         ", errs=" + errs +
                         ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
@@ -4565,77 +4644,6 @@ private static class DisconnectedSessionInfo {
         }
     }
 
-    /**
-     *
-     */
-    private static class ConnectionKey {
-        /** */
-        private final UUID nodeId;
-
-        /** */
-        private final int idx;
-
-        /** */
-        private final long connCnt;
-
-        /**
-         * @param nodeId Node ID.
-         * @param idx Connection index.
-         * @param connCnt Connection counter (set only for incoming connections).
-         */
-        ConnectionKey(UUID nodeId, int idx, long connCnt) {
-            this.nodeId = nodeId;
-            this.idx = idx;
-            this.connCnt = connCnt;
-        }
-
-        /**
-         * @return Connection counter.
-         */
-        long connectCount() {
-            return connCnt;
-        }
-
-        /**
-         * @return Node ID.
-         */
-        UUID nodeId() {
-            return nodeId;
-        }
-
-        /**
-         * @return Connection index.
-         */
-        int connectionIndex() {
-            return idx;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean equals(Object o) {
-            if (this == o)
-                return true;
-
-            if (o == null || getClass() != o.getClass())
-                return false;
-
-            ConnectionKey key = (ConnectionKey) o;
-
-            return idx == key.idx && nodeId.equals(key.nodeId);
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hashCode() {
-            int res = nodeId.hashCode();
-            res = 31 * res + idx;
-            return res;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return S.toString(ConnectionKey.class, this);
-        }
-    }
-
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java
new file mode 100644
index 0000000000000..0559df7892459
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/ConnectionKey.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.internal;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Connection Key.
+ */
+public class ConnectionKey {
+    /** */
+    private final UUID nodeId;
+
+    /** */
+    private final int idx;
+
+    /** */
+    private final long connCnt;
+
+    /** */
+    private final boolean dummy;
+
+    /**
+     * Creates ConnectionKey with false value of dummy flag.
+     *
+     * @param nodeId Node ID. Should be not null.
+     * @param idx Connection index.
+     * @param connCnt Connection counter (set only for incoming connections).
+     */
+    public ConnectionKey(@NotNull UUID nodeId, int idx, long connCnt) {
+        this(nodeId, idx, connCnt, false);
+    }
+
+    /**
+     * @param nodeId Node ID. Should be not null.
+     * @param idx Connection index.
+     * @param connCnt Connection counter (set only for incoming connections).
+     * @param dummy Indicates that session with this ConnectionKey is temporary
+     *              (for now dummy sessions are used only for Communication Failure Resolving process).
+     */
+    public ConnectionKey(@NotNull UUID nodeId, int idx, long connCnt, boolean dummy) {
+        this.nodeId = nodeId;
+        this.idx = idx;
+        this.connCnt = connCnt;
+        this.dummy = dummy;
+    }
+
+    /**
+     * @return Connection counter.
+     */
+    public long connectCount() {
+        return connCnt;
+    }
+
+    /**
+     * @return Node ID.
+     */
+    public UUID nodeId() {
+        return nodeId;
+    }
+
+    /**
+     * @return Connection index.
+     */
+    public int connectionIndex() {
+        return idx;
+    }
+
+    /**
+     * @return {@code True} if this ConnectionKey is dummy and serves temporary session.
+     */
+    public boolean dummy() {
+        return dummy;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        ConnectionKey key = (ConnectionKey) o;
+
+        return idx == key.idx && nodeId.equals(key.nodeId);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = nodeId.hashCode();
+        res = 31 * res + idx;
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ConnectionKey.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
new file mode 100644
index 0000000000000..c42fa576c768a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationConnectionCheckFuture.java
@@ -0,0 +1,519 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.internal;
+
+import java.net.InetSocketAddress;
+import java.nio.channels.SocketChannel;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.util.GridLeanMap;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.nio.GridNioServer;
+import org.apache.ignite.internal.util.nio.GridNioSession;
+import org.apache.ignite.internal.util.nio.GridNioSessionMetaKey;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+
+/**
+ * Tcp Communication Connection Check Future.
+ */
+public class TcpCommunicationConnectionCheckFuture extends GridFutureAdapter<BitSet> implements IgniteSpiTimeoutObject, GridLocalEventListener {
+    /** Session future. */
+    public static final int SES_FUT_META = GridNioSessionMetaKey.nextUniqueKey();
+
+    /** */
+    private static final AtomicIntegerFieldUpdater<SingleAddressConnectFuture> connFutDoneUpdater =
+        AtomicIntegerFieldUpdater.newUpdater(SingleAddressConnectFuture.class, "done");
+
+    /** */
+    private static final AtomicIntegerFieldUpdater<MultipleAddressesConnectFuture> connResCntUpdater =
+        AtomicIntegerFieldUpdater.newUpdater(MultipleAddressesConnectFuture.class, "resCnt");
+
+    /** */
+    private final AtomicInteger resCntr = new AtomicInteger();
+
+    /** */
+    private final List<ClusterNode> nodes;
+
+    /** */
+    private volatile ConnectFuture[] futs;
+
+    /** */
+    private final GridNioServer nioSrvr;
+
+    /** */
+    private final TcpCommunicationSpi spi;
+
+    /** */
+    private final IgniteUuid timeoutObjId = IgniteUuid.randomUuid();
+
+    /** */
+    private final BitSet resBitSet;
+
+    /** */
+    private long endTime;
+
+    /** */
+    private final IgniteLogger log;
+
+    /**
+     * @param spi SPI instance.
+     * @param log Logger.
+     * @param nioSrvr NIO server.
+     * @param nodes Nodes to check.
+     */
+    public TcpCommunicationConnectionCheckFuture(TcpCommunicationSpi spi,
+        IgniteLogger log,
+        GridNioServer nioSrvr,
+        List<ClusterNode> nodes)
+    {
+        this.spi = spi;
+        this.log = log;
+        this.nioSrvr = nioSrvr;
+        this.nodes = nodes;
+
+        resBitSet = new BitSet(nodes.size());
+    }
+
+    /**
+     * @param timeout Connect timeout.
+     */
+    public void init(long timeout) {
+        ConnectFuture[] futs = new ConnectFuture[nodes.size()];
+
+        UUID locId = spi.getSpiContext().localNode().id();
+
+        for (int i = 0; i < nodes.size(); i++) {
+            ClusterNode node = nodes.get(i);
+
+            if (!node.id().equals(locId)) {
+                if (spi.getSpiContext().node(node.id()) == null) {
+                    receivedConnectionStatus(i, false);
+
+                    continue;
+                }
+
+                Collection<InetSocketAddress> addrs;
+
+                try {
+                    addrs = spi.nodeAddresses(node, false);
+                }
+                catch (Exception e) {
+                    U.error(log, "Failed to get node addresses: " + node, e);
+
+                    receivedConnectionStatus(i, false);
+
+                    continue;
+                }
+
+                if (addrs.size() == 1) {
+                    SingleAddressConnectFuture fut = new SingleAddressConnectFuture(i);
+
+                    fut.init(addrs.iterator().next(), node.id());
+
+                    futs[i] = fut;
+                }
+                else {
+                    MultipleAddressesConnectFuture fut = new MultipleAddressesConnectFuture(i);
+
+                    fut.init(addrs, node.id());
+
+                    futs[i] = fut;
+                }
+            }
+            else
+                receivedConnectionStatus(i, true);
+        }
+
+        this.futs = futs;
+
+        spi.getSpiContext().addLocalEventListener(this, EVT_NODE_LEFT, EVT_NODE_FAILED);
+
+        if (!isDone()) {
+            endTime = System.currentTimeMillis() + timeout;
+
+            spi.getSpiContext().addTimeoutObject(this);
+        }
+    }
+
+    /**
+     * @param idx Node index.
+     * @param res Success flag.
+     */
+    private void receivedConnectionStatus(int idx, boolean res) {
+        assert resCntr.get() < nodes.size();
+
+        synchronized (resBitSet) {
+            resBitSet.set(idx, res);
+        }
+
+        if (resCntr.incrementAndGet() == nodes.size())
+            onDone(resBitSet);
+    }
+
+    /**
+     * @param nodeIdx Node index.
+     * @return Node ID.
+     */
+    private UUID nodeId(int nodeIdx) {
+        return nodes.get(nodeIdx).id();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteUuid id() {
+        return timeoutObjId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long endTime() {
+        return endTime;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onEvent(Event evt) {
+        if (isDone())
+            return;
+
+        assert evt instanceof DiscoveryEvent : evt;
+        assert evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED ;
+
+        UUID nodeId = ((DiscoveryEvent)evt).eventNode().id();
+
+        for (int i = 0; i < nodes.size(); i++) {
+            if (nodes.get(i).id().equals(nodeId)) {
+                ConnectFuture fut = futs[i];
+
+                if (fut != null)
+                    fut.onNodeFailed();
+
+                return;
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onTimeout() {
+        if (isDone())
+            return;
+
+        ConnectFuture[] futs = this.futs;
+
+        for (int i = 0; i < futs.length; i++) {
+            ConnectFuture fut = futs[i];
+
+            if (fut != null)
+                fut.onTimeout();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean onDone(@Nullable BitSet res, @Nullable Throwable err) {
+        if (super.onDone(res, err)) {
+            spi.getSpiContext().removeTimeoutObject(this);
+
+            spi.getSpiContext().removeLocalEventListener(this);
+
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     *
+     */
+    private interface ConnectFuture {
+        /**
+         *
+         */
+        void onTimeout();
+
+        /**
+         *
+         */
+        void onNodeFailed();
+    }
+
+    /**
+     *
+     */
+    private class SingleAddressConnectFuture implements TcpCommunicationNodeConnectionCheckFuture, ConnectFuture {
+        /** */
+        final int nodeIdx;
+
+        /** */
+        volatile int done;
+
+        /** */
+        Map<Integer, Object> sesMeta;
+
+        /** */
+        private SocketChannel ch;
+
+        /**
+         * @param nodeIdx Node index.
+         */
+        SingleAddressConnectFuture(int nodeIdx) {
+            this.nodeIdx = nodeIdx;
+        }
+
+        /**
+         * @param addr Node address.
+         * @param rmtNodeId Id of node to open connection check session with.
+         */
+        public void init(InetSocketAddress addr, UUID rmtNodeId) {
+            boolean connect;
+
+            try {
+                ch = SocketChannel.open();
+
+                ch.configureBlocking(false);
+
+                ch.socket().setTcpNoDelay(true);
+                ch.socket().setKeepAlive(false);
+
+                connect = ch.connect(addr);
+            }
+            catch (Exception e) {
+                finish(false);
+
+                return;
+            }
+
+            if (!connect) {
+                sesMeta = new GridLeanMap<>(3);
+
+                // Set dummy key to identify connection-check outgoing connection.
+                sesMeta.put(TcpCommunicationSpi.CONN_IDX_META, new ConnectionKey(rmtNodeId, -1, -1, true));
+                sesMeta.put(SES_FUT_META, this);
+
+                nioSrvr.createSession(ch, sesMeta, true, new IgniteInClosure<IgniteInternalFuture<GridNioSession>>() {
+                    @Override public void apply(IgniteInternalFuture<GridNioSession> fut) {
+                        if (fut.error() != null)
+                            finish(false);
+                    }
+                });
+            }
+        }
+
+        /**
+         *
+         */
+        @SuppressWarnings("unchecked")
+        void cancel() {
+            if (finish(false))
+                nioSrvr.cancelConnect(ch, sesMeta);
+        }
+
+        /** {@inheritDoc} */
+        public void onTimeout() {
+            cancel();
+        }
+
+        /** {@inheritDoc} */
+        public void onConnected(UUID rmtNodeId) {
+            finish(nodeId(nodeIdx).equals(rmtNodeId));
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onNodeFailed() {
+            cancel();
+        }
+
+        /**
+         * @param res Result.
+         * @return {@code True} if result was set by this call.
+         */
+        public boolean finish(boolean res) {
+            if (connFutDoneUpdater.compareAndSet(this, 0, 1)) {
+                onStatusReceived(res);
+
+                return true;
+            }
+
+            return false;
+        }
+
+        /**
+         * @param res Result.
+         */
+        void onStatusReceived(boolean res) {
+            receivedConnectionStatus(nodeIdx, res);
+        }
+    }
+
+    /**
+     *
+     */
+    private class MultipleAddressesConnectFuture implements ConnectFuture {
+        /** */
+        volatile int resCnt;
+
+        /** */
+        volatile SingleAddressConnectFuture[] futs;
+
+        /** */
+        final int nodeIdx;
+
+        /**
+         * @param nodeIdx Node index.
+         */
+        MultipleAddressesConnectFuture(int nodeIdx) {
+            this.nodeIdx = nodeIdx;
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onNodeFailed() {
+            SingleAddressConnectFuture[] futs = this.futs;
+
+            for (int i = 0; i < futs.length; i++) {
+                ConnectFuture fut = futs[i];
+
+                if (fut != null)
+                    fut.onNodeFailed();
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onTimeout() {
+            SingleAddressConnectFuture[] futs = this.futs;
+
+            for (int i = 0; i < futs.length; i++) {
+                ConnectFuture fut = futs[i];
+
+                if (fut != null)
+                    fut.onTimeout();
+            }
+        }
+
+        /**
+         * @param addrs Node addresses.
+         * @param rmtNodeId Id of node to open connection check session with.
+         */
+        void init(Collection<InetSocketAddress> addrs, UUID rmtNodeId) {
+            SingleAddressConnectFuture[] futs = new SingleAddressConnectFuture[addrs.size()];
+
+            for (int i = 0; i < addrs.size(); i++) {
+                SingleAddressConnectFuture fut = new SingleAddressConnectFuture(nodeIdx) {
+                    @Override void onStatusReceived(boolean res) {
+                        receivedAddressStatus(res);
+                    }
+                };
+
+                futs[i] = fut;
+            }
+
+            this.futs = futs;
+
+            int idx = 0;
+
+            for (InetSocketAddress addr : addrs) {
+                futs[idx++].init(addr, rmtNodeId);
+
+                if (resCnt == Integer.MAX_VALUE)
+                    return;
+            }
+
+            // Close race.
+            if (done())
+                cancelFutures();
+        }
+
+        /**
+         * @return {@code True}
+         */
+        private boolean done() {
+            int resCnt0 = resCnt;
+
+            return resCnt0 == Integer.MAX_VALUE || resCnt0 == futs.length;
+        }
+
+        /**
+         *
+         */
+        private void cancelFutures() {
+            SingleAddressConnectFuture[] futs = this.futs;
+
+            if (futs != null) {
+                for (int i = 0; i < futs.length; i++) {
+                    SingleAddressConnectFuture fut = futs[i];
+
+                    fut.cancel();
+                }
+            }
+        }
+
+        /**
+         * @param res Result.
+         */
+        void receivedAddressStatus(boolean res) {
+            if (res) {
+                for (;;) {
+                    int resCnt0 = resCnt;
+
+                    if (resCnt0 == Integer.MAX_VALUE)
+                        return;
+
+                    if (connResCntUpdater.compareAndSet(this, resCnt0, Integer.MAX_VALUE)) {
+                        receivedConnectionStatus(nodeIdx, true);
+
+                        cancelFutures(); // Cancel others connects if they are still in progress.
+
+                        return;
+                    }
+                }
+            }
+            else {
+                for (;;) {
+                    int resCnt0 = resCnt;
+
+                    if (resCnt0 == Integer.MAX_VALUE)
+                        return;
+
+                    int resCnt1 = resCnt0 + 1;
+
+                    if (connResCntUpdater.compareAndSet(this, resCnt0, resCnt1)) {
+                        if (resCnt1 == futs.length)
+                            receivedConnectionStatus(nodeIdx, false);
+
+                        return;
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java
new file mode 100644
index 0000000000000..cbf27b5c2af4b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/internal/TcpCommunicationNodeConnectionCheckFuture.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp.internal;
+
+import java.util.UUID;
+
+/**
+ * Tcp Communication Node Connection Check Future.
+ */
+public interface TcpCommunicationNodeConnectionCheckFuture {
+    /**
+     * @param nodeId Remote node ID.
+     */
+    public void onConnected(UUID nodeId);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiCustomMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiCustomMessage.java
index a0f9b7563d1d8..f26ad3353b417 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiCustomMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiCustomMessage.java
@@ -30,12 +30,23 @@
  */
 public interface DiscoverySpiCustomMessage extends Serializable {
     /**
-     * Called when message passed the ring.
+     * Called when custom message has been handled by all nodes.
+     *
+     * @return Ack message or {@code null} if ack is not required.
      */
     @Nullable public DiscoverySpiCustomMessage ackMessage();
 
     /**
-     * @return {@code true} if message can be modified during listener notification. Changes will be send to next nodes.
+     * @return {@code True} if message can be modified during listener notification. Changes will be send to next nodes.
      */
     public boolean isMutable();
+
+    /**
+     * Called on discovery coordinator node after listener is notified. If returns {@code true}
+     * then message is not passed to others nodes, if after this method {@link #ackMessage()} returns non-null ack
+     * message, it is sent to all nodes.
+     *
+     * @return {@code True} if message should not be sent to all nodes.
+     */
+    public boolean stopProcess();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMutableCustomMessageSupport.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMutableCustomMessageSupport.java
new file mode 100644
index 0000000000000..37aa3235c9909
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMutableCustomMessageSupport.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * This annotation is for all implementations of {@link DiscoverySpi} that support
+ * topology mutable {@link DiscoverySpiCustomMessage}s.
+ */
+@Documented
+@Inherited
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE})
+public @interface DiscoverySpiMutableCustomMessageSupport {
+    /**
+     * @return Whether or not target SPI supports mutable {@link DiscoverySpiCustomMessage}s.
+     */
+    public boolean value();
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 2d9a31407c21f..f0a5186f0ae08 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -55,6 +55,8 @@
 import org.apache.ignite.configuration.AddressResolver;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
@@ -88,6 +90,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
 import org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport;
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.ignite.spi.discovery.DiscoverySpiMutableCustomMessageSupport;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
 import org.apache.ignite.spi.discovery.tcp.internal.DiscoveryDataPacket;
@@ -103,6 +106,7 @@
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryCheckFailedMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryDuplicateIdMessage;
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryEnsureDelivery;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryJoinRequestMessage;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT;
@@ -223,7 +227,8 @@
 @IgniteSpiMultipleInstancesSupport(true)
 @DiscoverySpiOrderSupport(true)
 @DiscoverySpiHistorySupport(true)
-public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
+@DiscoverySpiMutableCustomMessageSupport(true)
+public class TcpDiscoverySpi extends IgniteSpiAdapter implements IgniteDiscoverySpi {
     /** Node attribute that is mapped to node's external addresses (value is <tt>disc.tcp.ext-addrs</tt>). */
     public static final String ATTR_EXT_ADDRS = "disc.tcp.ext-addrs";
 
@@ -409,6 +414,9 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi {
     /** */
     protected IgniteSpiContext spiCtx;
 
+    /** */
+    private IgniteDiscoverySpiInternalListener internalLsnr;
+
     /**
      * Gets current SPI state.
      *
@@ -473,6 +481,13 @@ public ClusterNode getNode0(UUID id) {
 
     /** {@inheritDoc} */
     @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException {
+        IgniteDiscoverySpiInternalListener internalLsnr = this.internalLsnr;
+
+        if (internalLsnr != null) {
+            if (!internalLsnr.beforeSendCustomEvent(this, log, msg))
+                return;
+        }
+
         impl.sendCustomEvent(msg);
     }
 
@@ -1559,6 +1574,9 @@ protected void writeToSocket(Socket sock,
         OutputStream out,
         TcpDiscoveryAbstractMessage msg,
         long timeout) throws IOException, IgniteCheckedException {
+        if (internalLsnr != null && msg instanceof TcpDiscoveryJoinRequestMessage)
+            internalLsnr.beforeJoin(locNode, log);
+
         assert sock != null;
         assert msg != null;
         assert out != null;
@@ -2118,15 +2136,31 @@ boolean isSslEnabled() {
         return ignite().configuration().getSslContextFactory() != null;
     }
 
-    /**
-     * Force reconnect to cluster.
-     *
-     * @throws IgniteSpiException If failed.
-     */
-    public void reconnect() throws IgniteSpiException {
+    /** {@inheritDoc} */
+    public void clientReconnect() throws IgniteSpiException {
         impl.reconnect();
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean knownNode(UUID nodeId) {
+        return getNode0(nodeId) != null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean clientReconnectSupported() {
+        return !clientReconnectDisabled;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean supportsCommunicationFailureResolve() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void resolveCommunicationFailure(ClusterNode node, Exception err) {
+        throw new UnsupportedOperationException();
+    }
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      */
@@ -2148,6 +2182,11 @@ public void addSendMessageListener(IgniteInClosure<TcpDiscoveryAbstractMessage>
         sndMsgLsnrs.add(lsnr);
     }
 
+    /** {@inheritDoc} */
+    @Override public void setInternalListener(IgniteDiscoverySpiInternalListener lsnr) {
+        this.internalLsnr = lsnr;
+    }
+
     /**
      * <strong>FOR TEST ONLY!!!</strong>
      */
@@ -2185,7 +2224,7 @@ public void waitForClientMessagePrecessed() {
      * <p>
      * This method is intended for test purposes only.
      */
-    protected void simulateNodeFailure() {
+    public void simulateNodeFailure() {
         impl.simulateNodeFailure();
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
index 01534f7e43c6e..55fe4e65dc5cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
@@ -33,9 +33,9 @@
 import org.apache.ignite.cache.CacheMetrics;
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.ClusterMetricsSnapshot;
 import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
 import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -58,7 +58,7 @@
  * <strong>This class is not intended for public use</strong> and has been made
  * <tt>public</tt> due to certain limitations of Java technology.
  */
-public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements ClusterNode,
+public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements IgniteClusterNode,
     Comparable<TcpDiscoveryNode>, Externalizable {
     /** */
     private static final long serialVersionUID = 0L;
@@ -291,26 +291,14 @@ public Map<String, Object> getAttributes() {
         return metrics;
     }
 
-    /**
-     * Sets node metrics.
-     *
-     * @param metrics Node metrics.
-     */
+    /** {@inheritDoc} */
     public void setMetrics(ClusterMetrics metrics) {
         assert metrics != null;
 
         this.metrics = metrics;
     }
 
-    /**
-     * Gets collections of cache metrics for this node. Note that node cache metrics are constantly updated
-     * and provide up to date information about caches.
-     * <p>
-     * Cache metrics are updated with some delay which is directly related to metrics update
-     * frequency. For example, by default the update will happen every {@code 2} seconds.
-     *
-     * @return Runtime metrics snapshots for this node.
-     */
+    /** {@inheritDoc} */
     public Map<Integer, CacheMetrics> cacheMetrics() {
         if (metricsProvider != null) {
             Map<Integer, CacheMetrics> cacheMetrics0 = metricsProvider.cacheMetrics();
@@ -323,11 +311,7 @@ public Map<Integer, CacheMetrics> cacheMetrics() {
         return cacheMetrics;
     }
 
-    /**
-     * Sets node cache metrics.
-     *
-     * @param cacheMetrics Cache metrics.
-     */
+    /** {@inheritDoc} */
     public void setCacheMetrics(Map<Integer, CacheMetrics> cacheMetrics) {
         this.cacheMetrics = cacheMetrics != null ? cacheMetrics : Collections.<Integer, CacheMetrics>emptyMap();
     }
@@ -544,11 +528,7 @@ public TcpDiscoveryNode clientReconnectNode(Map<String, Object> nodeAttrs) {
         return node;
     }
 
-    /**
-     * Whether this node is cache client (see {@link IgniteConfiguration#isClientMode()}).
-     *
-     * @return {@code True if client}.
-     */
+    /** {@inheritDoc} */
     public boolean isCacheClient() {
         if (!cacheCliInit) {
             cacheCli = CU.clientNodeDirect(this);
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index f0f143ddc66df..6dc3d85d592b1 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -832,6 +832,7 @@ org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPar
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader$2
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloaderAssignments
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionCountersMap
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionCountersMap2
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionHistorySuppliersMap
@@ -1129,6 +1130,7 @@ org.apache.ignite.internal.processors.closure.GridClosureProcessor$T8
 org.apache.ignite.internal.processors.closure.GridClosureProcessor$T9
 org.apache.ignite.internal.processors.closure.GridClosureProcessor$TaskNoReduceAdapter
 org.apache.ignite.internal.processors.closure.GridPeerDeployAwareTaskAdapter
+org.apache.ignite.internal.processors.cluster.ClusterNodeMetrics
 org.apache.ignite.internal.processors.cluster.BaselineTopology
 org.apache.ignite.internal.processors.cluster.BaselineTopologyHistory
 org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionExcludeNeighborsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionExcludeNeighborsAbstractSelfTest.java
index 900d4f54bd237..eee47c717286b 100644
--- a/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionExcludeNeighborsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/cache/affinity/AffinityFunctionExcludeNeighborsAbstractSelfTest.java
@@ -32,7 +32,6 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -124,12 +123,9 @@ public void testAffinityMultiNode() throws Exception {
 
                 Affinity<Object> aff = g.affinity(DEFAULT_CACHE_NAME);
 
-                List<TcpDiscoveryNode> top = new ArrayList<>();
+                List<ClusterNode> top = new ArrayList<>(g.cluster().nodes());
 
-                for (ClusterNode node : g.cluster().nodes())
-                    top.add((TcpDiscoveryNode) node);
-
-                Collections.sort(top);
+                Collections.sort((List)top);
 
                 assertEquals(grids, top.size());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java b/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java
index 4e4d75a662144..5eca7d698af7c 100644
--- a/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java
@@ -120,6 +120,10 @@ static class ExchangeWorkerFailureTask extends SchemaExchangeWorkerTask implemen
                 @Override public boolean isMutable() {
                     return false;
                 }
+
+                @Override public boolean stopProcess() {
+                    return false;
+                }
             });
         }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java
index 2328c84df6b7d..141f4af7f9194 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupHostsSelfTest.java
@@ -61,6 +61,9 @@ public class ClusterGroupHostsSelfTest extends GridCommonAbstractTest {
      * @throws Exception If failed.
      */
     public void testForHosts() throws Exception {
+        if (!tcpDiscovery())
+            return;
+
         Ignite ignite = grid();
 
         assertEquals(1, ignite.cluster().forHost("h_1").nodes().size());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupSelfTest.java
index 9df561ac2f099..99006d1f99f91 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterGroupSelfTest.java
@@ -68,6 +68,8 @@ public class ClusterGroupSelfTest extends ClusterGroupAbstractTest {
                 if (i == 0)
                     ignite = g;
             }
+
+            waitForTopology(NODES_CNT);
         }
         finally {
             Ignition.setClientMode(false);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsUpdateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsUpdateTest.java
new file mode 100644
index 0000000000000..6e6b4a4fd8344
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsUpdateTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import junit.framework.AssertionFailedError;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class ClusterNodeMetricsUpdateTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setMetricsUpdateFrequency(500);
+
+        cfg.setClientMode(client);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMetrics() throws Exception {
+        int NODES = 6;
+
+        Ignite srv0 = startGridsMultiThreaded(NODES / 2);
+
+        client = true;
+
+        startGridsMultiThreaded(NODES / 2, NODES / 2);
+
+        Map<UUID, Integer> expJobs = new HashMap<>();
+
+        for (int i = 0; i < NODES; i++)
+            expJobs.put(nodeId(i), 0);
+
+        checkMetrics(NODES, expJobs);
+
+        for (int i = 0; i < NODES; i++) {
+            UUID nodeId = nodeId(i);
+
+            IgniteCompute c = srv0.compute(srv0.cluster().forNodeId(nodeId(i)));
+
+            c.call(new DummyCallable(null));
+
+            expJobs.put(nodeId, 1);
+        }
+    }
+
+    /**
+     * @param expNodes Expected nodes.
+     * @param expJobs Expected jobs number per node.
+     */
+    private void checkMetrics0(int expNodes, Map<UUID, Integer> expJobs) {
+        List<Ignite> nodes = Ignition.allGrids();
+
+        assertEquals(expNodes, nodes.size());
+        assertEquals(expNodes, expJobs.size());
+
+        int totalJobs = 0;
+
+        for (Integer c : expJobs.values())
+            totalJobs += c;
+
+        for (final Ignite ignite : nodes) {
+            ClusterMetrics m = ignite.cluster().metrics();
+
+            assertEquals(expNodes, m.getTotalNodes());
+            assertEquals(totalJobs, m.getTotalExecutedJobs());
+
+            for (Map.Entry<UUID, Integer> e : expJobs.entrySet()) {
+                UUID nodeId = e.getKey();
+
+                ClusterGroup g = ignite.cluster().forNodeId(nodeId);
+
+                ClusterMetrics nodeM = g.metrics();
+
+                assertEquals(e.getValue(), (Integer)nodeM.getTotalExecutedJobs());
+            }
+        }
+    }
+
+    /**
+     * @param expNodes Expected nodes.
+     * @param expJobs Expected jobs number per node.
+     * @throws Exception If failed.
+     */
+    private void checkMetrics(final int expNodes, final Map<UUID, Integer> expJobs) throws Exception {
+        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                try {
+                    checkMetrics0(expNodes, expJobs);
+                }
+                catch (AssertionFailedError e) {
+                    return false;
+                }
+
+                return true;
+            }
+        }, 5000);
+
+        checkMetrics0(expNodes, expJobs);
+    }
+
+    /**
+     *
+     */
+    private static class DummyCallable implements IgniteCallable<Object> {
+        /** */
+        private byte[] data;
+
+        /**
+         * @param data Data.
+         */
+        DummyCallable(byte[] data) {
+            this.data = data;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object call() throws Exception {
+            return data;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/DiscoverySpiTestListener.java b/modules/core/src/test/java/org/apache/ignite/internal/DiscoverySpiTestListener.java
new file mode 100644
index 0000000000000..46d9edc6854b7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/DiscoverySpiTestListener.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Test callback for discovery SPI.
+ * <p>
+ * Allows block/delay node join and custom event sending.
+ */
+public class DiscoverySpiTestListener implements IgniteDiscoverySpiInternalListener {
+    /** */
+    private volatile CountDownLatch joinLatch;
+
+    /** */
+    private Set<Class<?>> blockCustomEvtCls;
+
+    /** */
+    private final Object mux = new Object();
+
+    /** */
+    private List<DiscoverySpiCustomMessage> blockedMsgs = new ArrayList<>();
+
+    /** */
+    private volatile DiscoverySpi spi;
+
+    /** */
+    private volatile IgniteLogger log;
+
+    /**
+     *
+     */
+    public void startBlockJoin() {
+        joinLatch = new CountDownLatch(1);
+    }
+
+    /**
+     *
+     */
+    public void stopBlockJoin() {
+        joinLatch.countDown();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void beforeJoin(ClusterNode locNode, IgniteLogger log) {
+        try {
+            CountDownLatch writeLatch0 = joinLatch;
+
+            if (writeLatch0 != null) {
+                log.info("Block join");
+
+                U.await(writeLatch0);
+            }
+        }
+        catch (Exception e) {
+            throw new IgniteException(e);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean beforeSendCustomEvent(DiscoverySpi spi, IgniteLogger log, DiscoverySpiCustomMessage msg) {
+        this.spi = spi;
+        this.log = log;
+
+        synchronized (mux) {
+            if (blockCustomEvtCls != null) {
+                DiscoveryCustomMessage msg0 = GridTestUtils.getFieldValue(msg, "delegate");
+
+                if (blockCustomEvtCls.contains(msg0.getClass())) {
+                    log.info("Block custom message: " + msg0);
+
+                    blockedMsgs.add(msg);
+
+                    mux.notifyAll();
+
+                    return false;
+                }
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * @param blockCustomEvtCls Event class to block.
+     */
+    public void blockCustomEvent(Class<?> cls0, Class<?> ... blockCustomEvtCls) {
+        synchronized (mux) {
+            assert blockedMsgs.isEmpty() : blockedMsgs;
+
+            this.blockCustomEvtCls = new HashSet<>();
+
+            this.blockCustomEvtCls.add(cls0);
+
+            Collections.addAll(this.blockCustomEvtCls, blockCustomEvtCls);
+        }
+    }
+
+    /**
+     * @throws InterruptedException If interrupted.
+     */
+    public void waitCustomEvent() throws InterruptedException {
+        synchronized (mux) {
+            while (blockedMsgs.isEmpty())
+                mux.wait();
+        }
+    }
+
+    /**
+     *
+     */
+    public void stopBlockCustomEvents() {
+        if (spi == null)
+            return;
+
+        List<DiscoverySpiCustomMessage> msgs;
+
+        synchronized (this) {
+            msgs = new ArrayList<>(blockedMsgs);
+
+            blockCustomEvtCls = null;
+
+            blockedMsgs.clear();
+        }
+
+        for (DiscoverySpiCustomMessage msg : msgs) {
+            log.info("Resend blocked message: " + msg);
+
+            spi.sendCustomEvent(msg);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridDiscoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridDiscoverySelfTest.java
index e6b678b313a7c..883d677d645af 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridDiscoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridDiscoverySelfTest.java
@@ -49,6 +49,7 @@
 import org.jetbrains.annotations.Nullable;
 
 import static java.util.concurrent.TimeUnit.MINUTES;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.lang.IgniteProductVersion.fromString;
@@ -158,10 +159,10 @@ public void testDiscoveryListener() throws Exception {
 
         final AtomicInteger cnt = new AtomicInteger();
 
-        /** Joined nodes counter. */
+        // Joined nodes counter.
         final CountDownLatch joinedCnt = new CountDownLatch(NODES_CNT);
 
-        /** Left nodes counter. */
+        // Left nodes counter.
         final CountDownLatch leftCnt = new CountDownLatch(NODES_CNT);
 
         IgnitePredicate<Event> lsnr = new IgnitePredicate<Event>() {
@@ -171,7 +172,7 @@ public void testDiscoveryListener() throws Exception {
 
                     joinedCnt.countDown();
                 }
-                else if (EVT_NODE_LEFT == evt.type()) {
+                else if (EVT_NODE_LEFT == evt.type() || EVT_NODE_FAILED == evt.type()) {
                     int i = cnt.decrementAndGet();
 
                     assert i >= 0;
@@ -185,7 +186,10 @@ else if (EVT_NODE_LEFT == evt.type()) {
             }
         };
 
-        ignite.events().localListen(lsnr, EVT_NODE_LEFT, EVT_NODE_JOINED);
+        int[] evts = tcpDiscovery() ? new int[]{EVT_NODE_LEFT, EVT_NODE_JOINED} :
+            new int[]{EVT_NODE_LEFT, EVT_NODE_FAILED, EVT_NODE_JOINED};
+
+        ignite.events().localListen(lsnr, evts);
 
         try {
             for (int i = 0; i < NODES_CNT; i++)
@@ -242,6 +246,8 @@ public void testCacheNodes() throws Exception {
             for (int i = 0; i < NODES_CNT; i++)
                 stopGrid(i);
 
+            waitForTopology(1);
+
             final long topVer = discoMgr.topologyVersion();
 
             assert topVer == topVer0 + NODES_CNT * 2 : "Unexpected topology version: " + topVer;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridJobMasterLeaveAwareSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridJobMasterLeaveAwareSelfTest.java
index cd6b2c081d2d4..a8be5419314df 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridJobMasterLeaveAwareSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridJobMasterLeaveAwareSelfTest.java
@@ -259,6 +259,8 @@ public void testCannotSendJobExecuteResponse() throws Exception {
         // Now we stop master grid.
         stopGrid(lastGridIdx, true);
 
+        waitForTopology(GRID_CNT - 1);
+
         // Release communication SPI wait latches. As master node is stopped, job worker will receive and exception.
         for (int i = 0; i < lastGridIdx; i++)
             ((CommunicationSpi)grid(i).configuration().getCommunicationSpi()).releaseWaitLatch();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java
index f3a19aaeccb4f..6824d51bcb17e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridJobStealingSelfTest.java
@@ -187,6 +187,8 @@ public void testProjectionPredicate() throws Exception {
     public void testProjectionPredicateInternalStealing() throws Exception {
         final Ignite ignite3 = startGrid(3);
 
+        waitForTopology(3);
+
         final UUID node1 = ignite1.cluster().localNode().id();
         final UUID node3 = ignite3.cluster().localNode().id();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridSameVmStartupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridSameVmStartupSelfTest.java
index 66e9cf43b3297..a04c38e7a7590 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridSameVmStartupSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridSameVmStartupSelfTest.java
@@ -20,6 +20,7 @@
 import java.util.Collection;
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.DiscoveryEvent;
@@ -75,8 +76,10 @@ public void testSameVmStartup() throws Exception {
 
             ignite2.events().localListen(new IgnitePredicate<Event>() {
                 @Override public boolean apply(Event evt) {
-                    assert evt.type() != EVT_NODE_FAILED :
-                        "Node1 did not exit gracefully.";
+                    boolean tcpDiscovery = tcpDiscovery();
+
+                    if (tcpDiscovery)
+                        assert evt.type() != EVT_NODE_FAILED : "Node1 did not exit gracefully.";
 
                     if (evt instanceof DiscoveryEvent) {
                         // Local node can send METRICS_UPDATED event.
@@ -86,8 +89,14 @@ public void testSameVmStartup() throws Exception {
                                 ((DiscoveryEvent) evt).eventNode().id() + ", expected=" + grid1LocNodeId +
                                 ", type=" + evt.type() + ']';
 
-                        if (evt.type() == EVT_NODE_LEFT)
-                            latch.countDown();
+                        if (tcpDiscovery) {
+                            if (evt.type() == EVT_NODE_LEFT)
+                                latch.countDown();
+                        }
+                        else {
+                            if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
+                                latch.countDown();
+                        }
                     }
 
                     return true;
@@ -96,7 +105,7 @@ public void testSameVmStartup() throws Exception {
 
             stopGrid(1);
 
-            latch.await();
+            assertTrue(latch.await(10, TimeUnit.SECONDS));
 
             Collection<ClusterNode> top2 = ignite2.cluster().forRemotes().nodes();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridSelfTest.java
index 7e368cb51717c..f71ffb0ea97d2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/GridSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/GridSelfTest.java
@@ -45,6 +45,8 @@ public class GridSelfTest extends ClusterGroupAbstractTest {
 
         for (int i = 0; i < NODES_CNT; i++)
             startGrid(i);
+
+        waitForTopology(NODES_CNT);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
index fa9cc35313b9a..e68ea1306d0bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAbstractTest.java
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.Socket;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -38,6 +39,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteFuture;
@@ -141,6 +143,14 @@ protected static TestTcpDiscoverySpi spi(Ignite ignite) {
         return ((TestTcpDiscoverySpi)ignite.configuration().getDiscoverySpi());
     }
 
+    /**
+     * @param ignite Node.
+     * @return Discovery SPI.
+     */
+    protected static IgniteDiscoverySpi spi0(Ignite ignite) {
+        return ((IgniteDiscoverySpi)ignite.configuration().getDiscoverySpi());
+    }
+
     /**
      * @param ignite Node.
      * @return Communication SPI.
@@ -185,16 +195,28 @@ protected BlockTcpCommunicationSpi commSpi(Ignite ignite) {
      * @return Server node client connected to.
      */
     protected Ignite clientRouter(Ignite client) {
-        TcpDiscoveryNode node = (TcpDiscoveryNode)client.cluster().localNode();
+        if (tcpDiscovery()) {
+            TcpDiscoveryNode node = (TcpDiscoveryNode)client.cluster().localNode();
+
+            assertTrue(node.isClient());
+            assertNotNull(node.clientRouterNodeId());
 
-        assertTrue(node.isClient());
-        assertNotNull(node.clientRouterNodeId());
+            Ignite srv = G.ignite(node.clientRouterNodeId());
 
-        Ignite srv = G.ignite(node.clientRouterNodeId());
+            assertNotNull(srv);
+
+            return srv;
+        }
+        else {
+            for (Ignite node : G.allGrids()) {
+                if (!node.cluster().localNode().isClient())
+                    return node;
+            }
 
-        assertNotNull(srv);
+            fail();
 
-        return srv;
+            return null;
+        }
     }
 
     /**
@@ -251,15 +273,24 @@ protected static void reconnectClientNodes(final IgniteLogger log,
         List<Ignite> clients, Ignite srv,
         @Nullable Runnable disconnectedC)
         throws Exception {
-        final TestTcpDiscoverySpi srvSpi = spi(srv);
+        final IgniteDiscoverySpi srvSpi = spi0(srv);
 
         final CountDownLatch disconnectLatch = new CountDownLatch(clients.size());
         final CountDownLatch reconnectLatch = new CountDownLatch(clients.size());
 
         log.info("Block reconnect.");
 
-        for (Ignite client : clients)
-            spi(client).writeLatch = new CountDownLatch(1);
+        List<DiscoverySpiTestListener> blockLsnrs = new ArrayList<>();
+
+        for (Ignite client : clients) {
+            DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+            lsnr.startBlockJoin();
+
+            blockLsnrs.add(lsnr);
+
+            spi0(client).setInternalListener(lsnr);
+        }
 
         IgnitePredicate<Event> p = new IgnitePredicate<Event>() {
             @Override public boolean apply(Event evt) {
@@ -291,8 +322,8 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
 
         log.info("Allow reconnect.");
 
-        for (Ignite client : clients)
-            spi(client).writeLatch.countDown();
+        for (DiscoverySpiTestListener blockLsnr : blockLsnrs)
+            blockLsnr.stopBlockJoin();
 
         waitReconnectEvent(log, reconnectLatch);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java
index 06bde99de673d..43da2d15a7d0e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectApiExceptionTest.java
@@ -44,6 +44,7 @@
 import org.apache.ignite.configuration.CollectionConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -51,6 +52,7 @@
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
 import org.apache.ignite.testframework.GridTestUtils;
 
 import static java.util.concurrent.TimeUnit.SECONDS;
@@ -99,7 +101,7 @@ public void testErrorOnDisconnect() throws Exception {
      * @throws Exception If failed.
      */
     @SuppressWarnings("unchecked")
-    public void dataStructureOperationsTest() throws Exception {
+    private void dataStructureOperationsTest() throws Exception {
         clientMode = true;
 
         final Ignite client = startGrid(serverCount());
@@ -219,7 +221,7 @@ public void dataStructureOperationsTest() throws Exception {
      * @throws Exception If failed.
      */
     @SuppressWarnings("unchecked")
-    public void cacheOperationsTest() throws Exception {
+    private void cacheOperationsTest() throws Exception {
         clientMode = true;
 
         final Ignite client = startGrid(serverCount());
@@ -537,7 +539,7 @@ public void cacheOperationsTest() throws Exception {
      * @throws Exception If failed.
      */
     @SuppressWarnings("unchecked")
-    public void igniteOperationsTest() throws Exception {
+    private void igniteOperationsTest() throws Exception {
         clientMode = true;
 
         final Ignite client = startGrid(serverCount());
@@ -775,11 +777,11 @@ private void doTestIgniteOperationOnDisconnect(Ignite client, final List<T2<Call
         throws Exception {
         assertNotNull(client.cache(DEFAULT_CACHE_NAME));
 
-        final TestTcpDiscoverySpi clientSpi = spi(client);
+        final IgniteDiscoverySpi clientSpi = spi0(client);
 
         Ignite srv = clientRouter(client);
 
-        TestTcpDiscoverySpi srvSpi = spi(srv);
+        DiscoverySpi srvSpi = spi0(srv);
 
         final CountDownLatch disconnectLatch = new CountDownLatch(1);
 
@@ -787,7 +789,10 @@ private void doTestIgniteOperationOnDisconnect(Ignite client, final List<T2<Call
 
         log.info("Block reconnect.");
 
-        clientSpi.writeLatch = new CountDownLatch(1);
+        DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+        clientSpi.setInternalListener(lsnr);
+        lsnr.startBlockJoin();
 
         final List<IgniteInternalFuture> futs = new ArrayList<>();
 
@@ -832,7 +837,7 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
 
             log.info("Allow reconnect.");
 
-            clientSpi.writeLatch.countDown();
+            lsnr.stopBlockJoin();
 
             waitReconnectEvent(reconnectLatch);
 
@@ -857,7 +862,7 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
             }
         }
         finally {
-            clientSpi.writeLatch.countDown();
+            lsnr.stopBlockJoin();
 
             for (IgniteInternalFuture fut : futs)
                 fut.cancel();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAtomicsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAtomicsTest.java
index 00daf5feb90a4..d1e3ade29b70a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAtomicsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectAtomicsTest.java
@@ -111,7 +111,7 @@ public void testAtomicSeqReconnect() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         IgniteAtomicSequence clientAtomicSeq = client.atomicSequence("atomicSeq", 0, true);
 
@@ -144,7 +144,7 @@ public void testAtomicSeqReconnectRemoved() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final IgniteAtomicSequence clientAtomicSeq = client.atomicSequence("atomicSeqRmv", 0, true);
 
@@ -192,7 +192,7 @@ public void testAtomicSeqReconnectInProgress() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         BlockTcpCommunicationSpi commSpi = commSpi(srv);
 
@@ -253,7 +253,7 @@ public void testAtomicReferenceReconnect() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         IgniteAtomicReference<String> clientAtomicRef = client.atomicReference("atomicRef", "1st value", true);
 
@@ -294,7 +294,7 @@ public void testAtomicReferenceReconnectRemoved() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final IgniteAtomicReference<String> clientAtomicRef =
             client.atomicReference("atomicRefRemoved", "1st value", true);
@@ -347,7 +347,7 @@ public void testAtomicReferenceReconnectInProgress() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final IgniteAtomicReference<String> clientAtomicRef =
             client.atomicReference("atomicRefInProg", "1st value", true);
@@ -414,7 +414,7 @@ public void testAtomicStampedReconnect() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         IgniteAtomicStamped clientAtomicStamped = client.atomicStamped("atomicStamped", 0, 0, true);
 
@@ -455,7 +455,7 @@ public void testAtomicStampedReconnectRemoved() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final IgniteAtomicStamped clientAtomicStamped = client.atomicStamped("atomicStampedRemoved", 0, 0, true);
 
@@ -506,7 +506,7 @@ public void testAtomicStampedReconnectInProgress() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final IgniteAtomicStamped clientAtomicStamped = client.atomicStamped("atomicStampedInProgress", 0, 0, true);
 
@@ -574,7 +574,7 @@ public void testAtomicLongReconnect() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         IgniteAtomicLong clientAtomicLong = client.atomicLong("atomicLong", 0, true);
 
@@ -605,7 +605,7 @@ public void testAtomicLongReconnectRemoved() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final IgniteAtomicLong clientAtomicLong = client.atomicLong("atomicLongRmv", 0, true);
 
@@ -646,7 +646,7 @@ public void testAtomicLongReconnectInProgress() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         BlockTcpCommunicationSpi commSpi = commSpi(srv);
 
@@ -701,7 +701,7 @@ public void testLatchReconnect() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         IgniteCountDownLatch clientLatch = client.countDownLatch("latch1", 3, false, true);
 
@@ -742,7 +742,7 @@ public void testSemaphoreReconnect() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         IgniteSemaphore clientSemaphore = client.semaphore("semaphore1", 3, false, true);
 
@@ -789,7 +789,7 @@ private void testReentrantLockReconnect(final boolean fair) throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         IgniteLock clientLock = client.reentrantLock("lock1", true, fair, true);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 518e674d6c052..3cb82e07cab14 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -49,6 +49,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateResponse;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetResponse;
@@ -67,6 +68,7 @@
 import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.transactions.Transaction;
@@ -155,11 +157,11 @@ public void testReconnect() throws Exception {
 
         IgniteEx client = startGrid(SRV_CNT);
 
-        final TestTcpDiscoverySpi clientSpi = spi(client);
+        final IgniteDiscoverySpi clientSpi = spi0(client);
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
-        TestTcpDiscoverySpi srvSpi = spi(srv);
+        DiscoverySpi srvSpi = ignite(0).configuration().getDiscoverySpi();
 
         final IgniteCache<Object, Object> cache = client.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME));
 
@@ -188,7 +190,11 @@ public void testReconnect() throws Exception {
 
         log.info("Block reconnect.");
 
-        clientSpi.writeLatch = new CountDownLatch(1);
+        DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+        clientSpi.setInternalListener(lsnr);
+
+        lsnr.startBlockJoin();
 
         final AtomicReference<IgniteInternalFuture> blockPutRef = new AtomicReference<>();
 
@@ -254,7 +260,7 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
 
         log.info("Allow reconnect.");
 
-        clientSpi.writeLatch.countDown();
+        lsnr.stopBlockJoin();
 
         assertTrue(reconnectLatch.await(5000, MILLISECONDS));
 
@@ -319,7 +325,7 @@ public void testReconnectTransactions() throws Exception {
 
         IgniteEx client = startGrid(SRV_CNT);
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
 
@@ -412,17 +418,21 @@ private void reconnectTransactionInProgress1(IgniteEx client,
         final TransactionConcurrency txConcurrency,
         final IgniteCache<Object, Object> cache)
         throws Exception {
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
-        final TestTcpDiscoverySpi clientSpi = spi(client);
-        final TestTcpDiscoverySpi srvSpi = spi(srv);
+        final IgniteDiscoverySpi clientSpi = spi0(client);
+        final DiscoverySpi srvSpi = spi0(srv);
 
         final CountDownLatch disconnectLatch = new CountDownLatch(1);
         final CountDownLatch reconnectLatch = new CountDownLatch(1);
 
         log.info("Block reconnect.");
 
-        clientSpi.writeLatch = new CountDownLatch(1);
+        DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+        clientSpi.setInternalListener(lsnr);
+
+        lsnr.startBlockJoin();
 
         client.events().localListen(new IgnitePredicate<Event>() {
             @Override public boolean apply(Event evt) {
@@ -530,7 +540,7 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
 
         assertTrue(putFailed.await(5000, MILLISECONDS));
 
-        clientSpi.writeLatch.countDown();
+        lsnr.stopBlockJoin();
 
         waitReconnectEvent(reconnectLatch);
 
@@ -604,9 +614,9 @@ public void testReconnectExchangeInProgress() throws Exception {
 
         IgniteEx client = startGrid(SRV_CNT);
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
-        TestTcpDiscoverySpi srvSpi = spi(srv);
+        DiscoverySpi srvSpi = spi0(srv);
 
         TestCommunicationSpi coordCommSpi = (TestCommunicationSpi)grid(0).configuration().getCommunicationSpi();
 
@@ -691,7 +701,7 @@ public void testReconnectInitialExchangeInProgress() throws Exception {
         IgniteInternalFuture<Boolean> fut = GridTestUtils.runAsync(new Callable<Boolean>() {
             @Override public Boolean call() throws Exception {
                 try {
-                    Ignition.start(optimize(getConfiguration(getTestIgniteInstanceName(SRV_CNT))));
+                    startGrid(optimize(getConfiguration(getTestIgniteInstanceName(SRV_CNT))));
 
                     // Commented due to IGNITE-4473, because
                     // IgniteClientDisconnectedException won't
@@ -722,7 +732,7 @@ public void testReconnectInitialExchangeInProgress() throws Exception {
             }
         });
 
-        TestTcpDiscoverySpi srvSpi = spi(srv);
+        DiscoverySpi srvSpi = spi0(srv);
 
         try {
             if (!joinLatch.await(10_000, MILLISECONDS)) {
@@ -1256,30 +1266,35 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
      *
      */
     static class TestClass1 implements Serializable {
+        // No-op.
     }
 
     /**
      *
      */
     static class TestClass2 implements Serializable {
+        // No-op.
     }
 
     /**
      *
      */
     static class TestClass3 implements Serializable {
+        // No-op.
     }
 
     /**
      *
      */
     static class TestClass4 implements Serializable {
+        // No-op.
     }
 
     /**
      *
      */
     static class TestClass5 implements Serializable {
+        // No-op.
     }
 
     /**
@@ -1294,11 +1309,11 @@ private void checkOperationInProgressFails(final IgniteEx client,
         Class<?> msgToBlock,
         final IgniteInClosure<IgniteCache<Object, Object>> c)
         throws Exception {
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final UUID id = client.localNode().id();
 
-        TestTcpDiscoverySpi srvSpi = spi(srv);
+        DiscoverySpi srvSpi = spi0(srv);
 
         final IgniteCache<Object, Object> cache = client.getOrCreateCache(ccfg);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCollectionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCollectionsTest.java
index 3f0e33d3046ab..5be59b0537ea2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCollectionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCollectionsTest.java
@@ -180,7 +180,7 @@ public void testServerReconnect() throws Exception {
     private void serverNodeReconnect(CollectionConfiguration colCfg) throws Exception {
         final Ignite client = grid(serverCount());
 
-        final Ignite srv = clientRouter(client);
+        final Ignite srv = ignite(0);
 
         assertNotNull(srv.queue("q", 0, colCfg));
         assertNotNull(srv.set("s", colCfg));
@@ -201,7 +201,7 @@ private void setReconnect(CollectionConfiguration colCfg) throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final String setName = "set-" + colCfg.getAtomicityMode();
 
@@ -235,7 +235,7 @@ private void setReconnectRemove(CollectionConfiguration colCfg) throws Exception
 
         assertTrue(client.cluster().localNode().isClient());
 
-        final Ignite srv = clientRouter(client);
+        final Ignite srv = ignite(0);
 
         final String setName = "set-rm-" + colCfg.getAtomicityMode();
 
@@ -281,7 +281,7 @@ private void setReconnectInProgress(final CollectionConfiguration colCfg) throws
 
         assertTrue(client.cluster().localNode().isClient());
 
-        final Ignite srv = clientRouter(client);
+        final Ignite srv = ignite(0);
 
         final String setName = "set-in-progress-" + colCfg.getAtomicityMode();
 
@@ -347,7 +347,7 @@ private void queueReconnect(CollectionConfiguration colCfg) throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final String setName = "queue-" + colCfg.getAtomicityMode();
 
@@ -379,7 +379,7 @@ private void queueReconnectRemoved(CollectionConfiguration colCfg) throws Except
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final String setName = "queue-rmv" + colCfg.getAtomicityMode();
 
@@ -423,7 +423,7 @@ private void queueReconnectInProgress(final CollectionConfiguration colCfg) thro
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final String setName = "queue-rmv" + colCfg.getAtomicityMode();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectComputeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectComputeTest.java
index cce0c7e3489f9..57d31882db6d7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectComputeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectComputeTest.java
@@ -49,7 +49,7 @@ public void testReconnectAffinityCallInProgress() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         IgniteCache<Integer, Integer> cache = client.getOrCreateCache("test-cache");
 
@@ -103,7 +103,7 @@ public void testReconnectBroadcastInProgress() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         BlockTcpCommunicationSpi commSpi = commSpi(srv);
 
@@ -152,7 +152,7 @@ public void testReconnectApplyInProgress() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         BlockTcpCommunicationSpi commSpi = commSpi(srv);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
index ca0d88974111e..d68fc1cd3cceb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectContinuousProcessorTest.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.util.typedef.P2;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteRunnable;
@@ -61,9 +62,9 @@ public void testEventListenerReconnect() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
-        TestTcpDiscoverySpi srvSpi = spi(srv);
+        IgniteDiscoverySpi srvSpi = spi0(srv);
 
         EventListener lsnr = new EventListener();
 
@@ -133,9 +134,9 @@ private void testMessageListenerReconnect(boolean stopFromClient) throws Excepti
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
-        TestTcpDiscoverySpi srvSpi = spi(srv);
+        IgniteDiscoverySpi srvSpi = spi0(srv);
 
         final String topic = "testTopic";
 
@@ -309,9 +310,9 @@ private void continuousQueryReconnect(Ignite client,
         CacheEventListener lsnr)
         throws Exception
     {
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
-        TestTcpDiscoverySpi srvSpi = spi(srv);
+        IgniteDiscoverySpi srvSpi = spi0(srv);
 
         final CountDownLatch reconnectLatch = new CountDownLatch(1);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDiscoveryStateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDiscoveryStateTest.java
index c071ee2ae68e5..6e77742b758f6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDiscoveryStateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectDiscoveryStateTest.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
 
 import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
 import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
@@ -64,20 +65,23 @@ public void testReconnect() throws Exception {
         nodeCnt.put(1, 1);
         nodeCnt.put(2, 2);
         nodeCnt.put(3, 3);
-        nodeCnt.put(4, 4);
 
-        for (Map.Entry<Integer, Integer> e : nodeCnt.entrySet()) {
-            Collection<ClusterNode> nodes = cluster.topology(e.getKey());
+        if (tcpDiscovery()) {
+            nodeCnt.put(4, 4);
 
-            assertNotNull("No nodes for topology: " + e.getKey(), nodes);
-            assertEquals((int)e.getValue(), nodes.size());
+            for (Map.Entry<Integer, Integer> e : nodeCnt.entrySet()) {
+                Collection<ClusterNode> nodes = cluster.topology(e.getKey());
+
+                assertNotNull("No nodes for topology: " + e.getKey(), nodes);
+                assertEquals((int)e.getValue(), nodes.size());
+            }
         }
 
         ClusterNode locNode = cluster.localNode();
 
         assertEquals(topVer, locNode.order());
 
-        TestTcpDiscoverySpi srvSpi = spi(clientRouter(client));
+        DiscoverySpi srvSpi = ignite(0).configuration().getDiscoverySpi();
 
         final CountDownLatch reconnectLatch = new CountDownLatch(1);
 
@@ -112,7 +116,11 @@ else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
         assertEquals(topVer, locNode.order());
         assertEquals(topVer, cluster.topologyVersion());
 
-        nodeCnt.put(5, 3);
+        if (tcpDiscovery())
+            nodeCnt.put(5, 3);
+        else
+            nodeCnt.clear();
+
         nodeCnt.put(6, 4);
 
         for (Map.Entry<Integer, Integer> e : nodeCnt.entrySet()) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectFailoverAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectFailoverAbstractTest.java
index 3e98051efb0ee..37292ff12b342 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectFailoverAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectFailoverAbstractTest.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -87,9 +88,9 @@ protected final void reconnectFailover(final Callable<Void> c) throws Exception
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
-        TestTcpDiscoverySpi srvSpi = spi(srv);
+        IgniteDiscoverySpi srvSpi = spi0(srv);
 
         final AtomicBoolean stop = new AtomicBoolean(false);
 
@@ -209,14 +210,17 @@ else if (evt.type() == EVT_CLIENT_NODE_DISCONNECTED) {
             }
 
             if (err != null) {
-                log.error(err);
+                log.error("Test error: " + err);
 
                 U.dumpThreads(log);
 
                 CyclicBarrier barrier0 = barrier;
 
-                if (barrier0 != null)
+                if (barrier0 != null) {
+                    barrier = null;
+
                     barrier0.reset();
+                }
 
                 stop.set(true);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectServicesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectServicesTest.java
index 3e961e537eccb..1e6dd64f970b2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectServicesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectServicesTest.java
@@ -65,7 +65,7 @@ public void testReconnect() throws Exception {
 
         assertEquals((Object)topVer, srvc.test());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         reconnectClientNode(client, srv, null);
 
@@ -88,7 +88,7 @@ public void testServiceRemove() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         IgniteServices clnServices = client.services();
 
@@ -132,7 +132,7 @@ public void testReconnectInDeploying() throws Exception {
 
         final IgniteServices services = client.services();
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         BlockTcpCommunicationSpi commSpi = commSpi(srv);
 
@@ -179,7 +179,7 @@ public void testReconnectInProgress() throws Exception {
 
         final IgniteServices services = client.services();
 
-        final Ignite srv = clientRouter(client);
+        final Ignite srv = ignite(0);
 
         services.deployClusterSingleton("testReconnectInProgress", new TestServiceImpl());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStopTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStopTest.java
index e863cdfcb8655..b5c3ee86d0d4a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStopTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStopTest.java
@@ -23,8 +23,10 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
 
 import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
 import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
@@ -50,15 +52,19 @@ public void testStopWhenDisconnected() throws Exception {
 
         Ignite srv = clientRouter(client);
 
-        TestTcpDiscoverySpi srvSpi = spi(srv);
+        DiscoverySpi srvSpi = spi0(srv);
         final CountDownLatch disconnectLatch = new CountDownLatch(1);
         final CountDownLatch reconnectLatch = new CountDownLatch(1);
 
-        final TestTcpDiscoverySpi clientSpi = spi(client);
+        final IgniteDiscoverySpi clientSpi = spi0(client);
+
+        DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+        clientSpi.setInternalListener(lsnr);
 
         log.info("Block reconnect.");
 
-        clientSpi.writeLatch = new CountDownLatch(1);
+        lsnr.startBlockJoin();
 
         client.events().localListen(new IgnitePredicate<Event>() {
             @Override public boolean apply(Event evt) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStreamerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStreamerTest.java
index 3959feb80804d..36b989093440e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStreamerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectStreamerTest.java
@@ -71,7 +71,7 @@ public void testStreamerReconnect() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final IgniteCache<Object, Object> srvCache = srv.cache(CACHE_NAME);
 
@@ -135,7 +135,7 @@ public void testStreamerReconnectInProgress() throws Exception {
 
         assertTrue(client.cluster().localNode().isClient());
 
-        Ignite srv = clientRouter(client);
+        Ignite srv = ignite(0);
 
         final IgniteCache<Object, Object> srvCache = srv.cache(CACHE_NAME);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
index a5d42e9a12cba..8edbb52babae1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -256,6 +256,9 @@ public void testClientsReconnectDisabled() throws Exception {
 
         Ignite srv1 = startGrid("server1");
 
+        if (!tcpDiscovery())
+            return;
+
         crd = ((IgniteKernal)srv1).localNode();
 
         Ignite srv2 = startGrid("server2");
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
index a8afa8b928a4f..8fad6404f1936 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAliveCacheSelfTest.java
@@ -178,16 +178,20 @@ public void testAlivesClient() throws Exception {
      * Waits while topology on all nodes became equals to the expected size.
      *
      * @param nodesCnt Expected nodes count.
-     * @throws InterruptedException If interrupted.
+     * @throws Exception If interrupted.
      */
     @SuppressWarnings("BusyWait")
-    private void awaitDiscovery(long nodesCnt) throws InterruptedException {
-        for (Ignite g : alive) {
-            ((TcpDiscoverySpi)g.configuration().getDiscoverySpi()).waitForClientMessagePrecessed();
+    private void awaitDiscovery(int nodesCnt) throws Exception {
+        if (tcpDiscovery()) {
+            for (Ignite g : alive) {
+                ((TcpDiscoverySpi)g.configuration().getDiscoverySpi()).waitForClientMessagePrecessed();
 
-            while (g.cluster().nodes().size() != nodesCnt)
-                Thread.sleep(10);
+                while (g.cluster().nodes().size() != nodesCnt)
+                    Thread.sleep(10);
+            }
         }
+        else
+            waitForTopology(nodesCnt);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessorAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessorAbstractSelfTest.java
index 1d70246993d36..aa2abaed7cd44 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessorAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/affinity/GridAffinityProcessorAbstractSelfTest.java
@@ -91,12 +91,12 @@ public abstract class GridAffinityProcessorAbstractSelfTest extends GridCommonAb
     @Override protected void beforeTestsStarted() throws Exception {
         assert NODES_CNT >= 1;
 
-        withCache = false;
+        withCache = true;
 
         for (int i = 0; i < NODES_CNT; i++)
             startGrid(i);
 
-        withCache = true;
+        withCache = false;
 
         for (int i = NODES_CNT; i < 2 * NODES_CNT; i++)
             startGrid(i);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java
index aefbc23d28f28..b8f9d70dc5dde 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java
@@ -27,9 +27,9 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
@@ -103,7 +103,7 @@ public void testMetricsStatisticsEnabled() throws Exception {
             Collection<ClusterNode> nodes = grid(0).cluster().forRemotes().nodes();
 
             for (ClusterNode node : nodes) {
-                Map<Integer, CacheMetrics> metrics = ((TcpDiscoveryNode)node).cacheMetrics();
+                Map<Integer, CacheMetrics> metrics = ((IgniteClusterNode)node).cacheMetrics();
                 assertNotNull(metrics);
                 assertFalse(metrics.isEmpty());
             }
@@ -118,6 +118,8 @@ public void testMetricsStatisticsEnabled() throws Exception {
 
     /**
      * Test cluster group metrics in case of statistics disabled.
+     *
+     * @throws Exception If failed.
      */
     public void testMetricsStatisticsDisabled() throws Exception {
         createCaches(false);
@@ -134,7 +136,7 @@ public void testMetricsStatisticsDisabled() throws Exception {
             Collection<ClusterNode> nodes = grid(0).cluster().forRemotes().nodes();
 
             for (ClusterNode node : nodes) {
-                Map<Integer, CacheMetrics> metrics = ((TcpDiscoveryNode) node).cacheMetrics();
+                Map<Integer, CacheMetrics> metrics = ((IgniteClusterNode)node).cacheMetrics();
                 assertNotNull(metrics);
                 assertTrue(metrics.isEmpty());
             }
@@ -172,7 +174,9 @@ private void destroyCaches() {
     }
 
     /**
-     * Wait for {@link EventType#EVT_NODE_METRICS_UPDATED} event will be receieved.
+     * Wait for {@link EventType#EVT_NODE_METRICS_UPDATED} event will be received.
+     *
+     * @throws InterruptedException If interrupted.
      */
     private void awaitMetricsUpdate() throws InterruptedException {
         final CountDownLatch latch = new CountDownLatch((GRID_CNT + 1) * 2);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
index df93ae4aab2c3..299dbf4a6eb69 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
@@ -115,6 +115,8 @@ public abstract class GridCacheAbstractSelfTest extends GridCommonAbstractTest {
     protected void initStoreStrategy() throws IgniteCheckedException {
         if (storeStgy == null)
             storeStgy = isMultiJvm() ? new H2CacheStoreStrategy() : new MapCacheStoreStrategy();
+        else if (isMultiJvm() && !(storeStgy instanceof H2CacheStoreStrategy))
+            storeStgy = new H2CacheStoreStrategy();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java
index 0069110662ac4..c135f2df9557a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheNearLockValueSelfTest.java
@@ -45,7 +45,9 @@ public class IgniteCacheNearLockValueSelfTest extends GridCommonAbstractTest {
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
-        startGridsMultiThreaded(2);
+        startGrid(1);
+
+        startGrid(0);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingErrorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingErrorTest.java
index f3214532410df..55ff31a62ed17 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingErrorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheP2pUnmarshallingErrorTest.java
@@ -66,6 +66,17 @@ public class IgniteCacheP2pUnmarshallingErrorTest extends IgniteCacheAbstractTes
         return null;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void startGrids() throws Exception {
+        int cnt = gridCount();
+
+        assert cnt >= 1 : "At least one grid must be started";
+
+        startGridsMultiThreaded(1, cnt - 1);
+
+        startGrid(0);
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 2337329c5ce83..838e56deebdd3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
@@ -1108,6 +1109,70 @@ private void stateChangeFailover2(boolean activate) throws Exception {
         checkCaches1(10);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testActivateFailover3() throws Exception {
+        stateChangeFailover3(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeactivateFailover3() throws Exception {
+        stateChangeFailover3(false);
+    }
+
+    /**
+     * @param activate If {@code true} tests activation, otherwise deactivation.
+     * @throws Exception If failed.
+     */
+    private void stateChangeFailover3(boolean activate) throws Exception {
+        testReconnectSpi = true;
+
+        startNodesAndBlockStatusChange(4, 0, 0, !activate);
+
+        client = false;
+
+        IgniteInternalFuture startFut1 = GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                startGrid(4);
+
+                return null;
+            }
+        }, "start-node1");
+
+        IgniteInternalFuture startFut2 = GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                startGrid(5);
+
+                return null;
+            }
+        }, "start-node2");
+
+        U.sleep(1000);
+
+        // Stop all nodes participating in state change and not allow last node to finish exchange.
+        for (int i = 0; i < 4; i++)
+            ((IgniteDiscoverySpi)ignite(i).configuration().getDiscoverySpi()).simulateNodeFailure();
+
+        for (int i = 0; i < 4; i++)
+            stopGrid(getTestIgniteInstanceName(i), true, false);
+
+        startFut1.get();
+        startFut2.get();
+
+        assertFalse(ignite(4).active());
+        assertFalse(ignite(5).active());
+
+        ignite(4).active(true);
+
+        for (int i = 0; i < 4; i++)
+            startGrid(i);
+
+        checkCaches1(6);
+    }
+
     /**
      * @param exp If {@code true} there should be recorded messages.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java
index 566860de75e01..2f9bd533ae916 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDaemonNodeMarshallerCacheTest.java
@@ -79,7 +79,7 @@ public void testMarshalOnDaemonNode2() throws Exception {
      * @param startFirst If {@code true} daemon node is started first.
      * @throws Exception If failed.
      */
-    public void marshalOnDaemonNode(boolean startFirst) throws Exception {
+    private void marshalOnDaemonNode(boolean startFirst) throws Exception {
         int nodeIdx = 0;
 
         if (!startFirst) {
@@ -92,6 +92,7 @@ public void marshalOnDaemonNode(boolean startFirst) throws Exception {
 
         Ignite daemonNode = startGrid(nodeIdx++);
 
+        assertTrue(daemonNode.cluster().localNode().isDaemon());
         assertEquals("true", daemonNode.cluster().localNode().attribute(ATTR_DAEMON));
 
         daemon = false;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataUpdatesFlowTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataUpdatesFlowTest.java
index 3ee51c8e8ddfb..7e8c086c9c2b1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataUpdatesFlowTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataUpdatesFlowTest.java
@@ -184,6 +184,7 @@ public class BinaryMetadataUpdatesFlowTest extends GridCommonAbstractTest {
      * Starts new ignite node and submits computation job to it.
      * @param idx Index.
      * @param stopFlag Stop flag.
+     * @throws Exception If failed.
      */
     private void startComputation(int idx, AtomicBoolean stopFlag) throws Exception {
         clientMode = false;
@@ -199,6 +200,7 @@ private void startComputation(int idx, AtomicBoolean stopFlag) throws Exception
      * @param idx Index.
      * @param deafClient Deaf client.
      * @param observedIds Observed ids.
+     * @throws Exception If failed.
      */
     private void startListening(int idx, boolean deafClient, Set<Integer> observedIds) throws Exception {
         clientMode = true;
@@ -269,7 +271,7 @@ private static class CQListener implements CacheEntryUpdatedListener {
     }
 
     /**
-     *
+     * @throws Exception If failed.
      */
     public void testFlowNoConflicts() throws Exception {
         startComputation(0, stopFlag0);
@@ -311,11 +313,14 @@ public void testFlowNoConflicts() throws Exception {
     }
 
     /**
-     *
+     * @throws Exception If failed.
      */
     public void testFlowNoConflictsWithClients() throws Exception {
         startComputation(0, stopFlag0);
 
+        if (!tcpDiscovery())
+            return;
+
         startComputation(1, stopFlag1);
 
         startComputation(2, stopFlag2);
@@ -617,6 +622,9 @@ private static final class BinaryObjectAdder implements IgniteCallable<Object> {
             while (!updatesQueue.isEmpty()) {
                 BinaryUpdateDescription desc = updatesQueue.poll();
 
+                if (desc == null)
+                    break;
+
                 BinaryObjectBuilder builder = ignite.binary().builder(BINARY_TYPE_NAME);
 
                 BinaryObject bo = newBinaryObject(builder, desc);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheClientNodeBinaryObjectMetadataMultinodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheClientNodeBinaryObjectMetadataMultinodeTest.java
index 313aaf910308d..81614cb9ff7a5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheClientNodeBinaryObjectMetadataMultinodeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/binary/GridCacheClientNodeBinaryObjectMetadataMultinodeTest.java
@@ -242,7 +242,7 @@ public void testFailoverOnStart() throws Exception {
                 @Override public boolean apply() {
                     Collection<BinaryType> metaCol = p0.types();
 
-                    return metaCol.size() == 1000;
+                    return metaCol.size() >= 1000;
                 }
             }, getTestTimeout());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java
index ed54377327e2e..dac3ff6730e41 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/GridCacheQueueClientDisconnectTest.java
@@ -32,6 +32,9 @@
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+/**
+ *
+ */
 public class GridCacheQueueClientDisconnectTest extends GridCommonAbstractTest {
     /** */
     private static final String IGNITE_QUEUE_NAME = "ignite-queue-client-reconnect-test";
@@ -66,6 +69,10 @@ public class GridCacheQueueClientDisconnectTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /**
+     * @param cacheAtomicityMode Atomicity mode.
+     * @return Configuration.
+     */
     private static CollectionConfiguration collectionConfiguration(CacheAtomicityMode cacheAtomicityMode) {
         CollectionConfiguration colCfg = new CollectionConfiguration();
 
@@ -74,6 +81,9 @@ private static CollectionConfiguration collectionConfiguration(CacheAtomicityMod
         return colCfg;
     }
 
+    /**
+     * @throws Exception If failed.
+     */
     public void testClientDisconnect() throws Exception {
         try {
             Ignite server = startGrid(0);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java
index 51764b5be8fe1..d85201a4b2a8b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteClientDataStructuresAbstractTest.java
@@ -494,7 +494,8 @@ private Ignite clientIgnite() {
 
         assertTrue(ignite.configuration().isClientMode());
 
-        assertEquals(clientDiscovery(), ignite.configuration().getDiscoverySpi().isClientMode());
+        if (tcpDiscovery())
+            assertEquals(clientDiscovery(), ignite.configuration().getDiscoverySpi().isClientMode());
 
         return ignite;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 0704dbdfe845d..e4560478daae2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -52,6 +52,7 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.internal.DiscoverySpiTestListener;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.cluster.NodeOrderComparator;
 import org.apache.ignite.internal.cluster.NodeOrderLegacyComparator;
@@ -60,7 +61,7 @@
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
-import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityFunctionContextImpl;
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -88,7 +89,6 @@
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceContext;
-import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -158,7 +158,7 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
 
         cfg.setCommunicationSpi(commSpi);
 
-        TestTcpDiscoverySpi discoSpi = new TestTcpDiscoverySpi();
+        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
 
         discoSpi.setForceServerMode(forceSrvMode);
         discoSpi.setIpFinder(ipFinder);
@@ -674,9 +674,11 @@ public void testNodeLeaveExchangeWaitAffinityMessage() throws Exception {
 
             checkAffinity(4, topVer(4, 0), true);
 
-            TestTcpDiscoverySpi discoSpi = (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+            DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
 
-            discoSpi.blockCustomEvent();
+            ((IgniteDiscoverySpi)ignite0.configuration().getDiscoverySpi()).setInternalListener(lsnr);
+
+            lsnr.blockCustomEvent(CacheAffinityChangeMessage.class);
 
             stopGrid(1);
 
@@ -687,7 +689,7 @@ public void testNodeLeaveExchangeWaitAffinityMessage() throws Exception {
             for (IgniteInternalFuture<?> fut : futs)
                 assertFalse(fut.isDone());
 
-            discoSpi.stopBlock();
+            lsnr.stopBlockCustomEvents();
 
             checkAffinity(3, topVer(5, 0), false);
 
@@ -1409,8 +1411,10 @@ public void testJoinExchangeBecomeCoordinator() throws Exception {
     public void testDelayAssignmentAffinityChanged() throws Exception {
         Ignite ignite0 = startServer(0, 1);
 
-        TestTcpDiscoverySpi discoSpi0 =
-            (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+        DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+        ((IgniteDiscoverySpi)ignite0.configuration().getDiscoverySpi()).setInternalListener(lsnr);
+
         TestRecordingCommunicationSpi commSpi0 =
             (TestRecordingCommunicationSpi)ignite0.configuration().getCommunicationSpi();
 
@@ -1418,19 +1422,19 @@ public void testDelayAssignmentAffinityChanged() throws Exception {
 
         checkAffinity(2, topVer(2, 0), true);
 
-        discoSpi0.blockCustomEvent();
+        lsnr.blockCustomEvent(CacheAffinityChangeMessage.class);
 
         startServer(2, 3);
 
         checkAffinity(3, topVer(3, 0), false);
 
-        discoSpi0.waitCustomEvent();
+        lsnr.waitCustomEvent();
 
         blockSupplySend(commSpi0, CACHE_NAME1);
 
         startServer(3, 4);
 
-        discoSpi0.stopBlock();
+        lsnr.stopBlockCustomEvents();
 
         checkAffinity(4, topVer(4, 0), false);
 
@@ -1452,8 +1456,10 @@ public void testDelayAssignmentAffinityChanged2() throws Exception {
         try {
             Ignite ignite0 = startServer(0, 1);
 
-            TestTcpDiscoverySpi discoSpi0 =
-                (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+            DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+            ((IgniteDiscoverySpi)ignite0.configuration().getDiscoverySpi()).setInternalListener(lsnr);
+
             TestRecordingCommunicationSpi commSpi0 =
                 (TestRecordingCommunicationSpi)ignite0.configuration().getCommunicationSpi();
 
@@ -1465,11 +1471,11 @@ public void testDelayAssignmentAffinityChanged2() throws Exception {
 
             checkAffinity(3, topVer(3, 1), false);
 
-            discoSpi0.blockCustomEvent();
+            lsnr.blockCustomEvent(CacheAffinityChangeMessage.class);
 
             stopNode(2, 4);
 
-            discoSpi0.waitCustomEvent();
+            lsnr.waitCustomEvent();
 
             blockSupplySend(commSpi0, CACHE_NAME1);
 
@@ -1483,7 +1489,7 @@ public void testDelayAssignmentAffinityChanged2() throws Exception {
 
             Thread.sleep(2_000);
 
-            discoSpi0.stopBlock();
+            lsnr.stopBlockCustomEvents();
 
             boolean started = GridTestUtils.waitForCondition(new GridAbsPredicate() {
                 @Override public boolean apply() {
@@ -1534,14 +1540,16 @@ public void testDelayAssignmentCacheDestroyCreate() throws Exception {
 
         ignite0.createCache(ccfg);
 
-        TestTcpDiscoverySpi discoSpi0 =
-            (TestTcpDiscoverySpi)ignite0.configuration().getDiscoverySpi();
+        DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+        ((IgniteDiscoverySpi)ignite0.configuration().getDiscoverySpi()).setInternalListener(lsnr);
+
         TestRecordingCommunicationSpi spi =
             (TestRecordingCommunicationSpi)ignite0.configuration().getCommunicationSpi();
 
         blockSupplySend(spi, CACHE_NAME2);
 
-        discoSpi0.blockCustomEvent();
+        lsnr.blockCustomEvent(CacheAffinityChangeMessage.class);
 
         startServer(1, 2);
 
@@ -1551,7 +1559,7 @@ public void testDelayAssignmentCacheDestroyCreate() throws Exception {
 
         spi.stopBlock();
 
-        discoSpi0.waitCustomEvent();
+        lsnr.waitCustomEvent();
 
         ignite0.destroyCache(CACHE_NAME2);
 
@@ -1561,7 +1569,7 @@ public void testDelayAssignmentCacheDestroyCreate() throws Exception {
 
         ignite0.createCache(ccfg);
 
-        discoSpi0.stopBlock();
+        lsnr.stopBlockCustomEvents();
 
         checkAffinity(3, topVer(3, 1), false);
         checkAffinity(3, topVer(3, 2), false);
@@ -2964,83 +2972,6 @@ public TestServiceImpl(int key) {
         }
     }
 
-    /**
-     *
-     */
-    static class TestTcpDiscoverySpi extends TcpDiscoverySpi {
-        /** */
-        private boolean blockCustomEvt;
-
-        /** */
-        private final Object mux = new Object();
-
-        /** */
-        private List<DiscoverySpiCustomMessage> blockedMsgs = new ArrayList<>();
-
-        /** {@inheritDoc} */
-        @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException {
-            synchronized (mux) {
-                if (blockCustomEvt) {
-                    DiscoveryCustomMessage msg0 = GridTestUtils.getFieldValue(msg, "delegate");
-
-                    if (msg0 instanceof CacheAffinityChangeMessage) {
-                        log.info("Block custom message: " + msg0);
-
-                        blockedMsgs.add(msg);
-
-                        mux.notifyAll();
-
-                        return;
-                    }
-                }
-            }
-
-            super.sendCustomEvent(msg);
-        }
-
-        /**
-         *
-         */
-        public void blockCustomEvent() {
-            synchronized (mux) {
-                assert blockedMsgs.isEmpty() : blockedMsgs;
-
-                blockCustomEvt = true;
-            }
-        }
-
-        /**
-         * @throws InterruptedException If interrupted.
-         */
-        public void waitCustomEvent() throws InterruptedException {
-            synchronized (mux) {
-                while (blockedMsgs.isEmpty())
-                    mux.wait();
-            }
-        }
-
-        /**
-         *
-         */
-        public void stopBlock() {
-            List<DiscoverySpiCustomMessage> msgs;
-
-            synchronized (this) {
-                msgs = new ArrayList<>(blockedMsgs);
-
-                blockCustomEvt = false;
-
-                blockedMsgs.clear();
-            }
-
-            for (DiscoverySpiCustomMessage msg : msgs) {
-                log.info("Resend blocked message: " + msg);
-
-                super.sendCustomEvent(msg);
-            }
-        }
-    }
-
     /**
      *
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java
index 3834df9b5dcc7..5dea5d9238e2d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheNodeFailureAbstractTest.java
@@ -45,6 +45,7 @@
 import static org.apache.ignite.IgniteState.STOPPED;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_TX_SALVAGE_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.getInteger;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
@@ -188,7 +189,7 @@ private void checkTransaction(TransactionConcurrency concurrency, TransactionIso
 
                     return true;
                 }
-            }, EVT_NODE_LEFT);
+            }, EVT_NODE_LEFT, EVT_NODE_FAILED);
 
             stopGrid(idx);
 
@@ -268,7 +269,7 @@ public void testLock() throws Exception {
 
                 return true;
             }
-        }, EVT_NODE_LEFT);
+        }, EVT_NODE_LEFT, EVT_NODE_FAILED);
 
         stopGrid(idx);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java
index e71d3ee3ca290..b7ae84400c2fe 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCache150ClientsTest.java
@@ -168,6 +168,8 @@ public void test150Clients() throws Exception {
 
         log.info("Started all clients.");
 
+        waitForTopology(CLIENTS + 1);
+
         checkNodes(CLIENTS + 1);
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheManyClientsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheManyClientsTest.java
index a0be40ec79237..7785a3ce13941 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheManyClientsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheManyClientsTest.java
@@ -178,7 +178,7 @@ private void manyClientsSequentially() throws Exception {
         log.info("All clients started.");
 
         try {
-            checkNodes(SRVS + CLIENTS);
+            checkNodes0(SRVS + CLIENTS);
         }
         finally {
             for (Ignite client : clients)
@@ -186,6 +186,30 @@ private void manyClientsSequentially() throws Exception {
         }
     }
 
+    /**
+     * @param expCnt Expected number of nodes.
+     * @throws Exception If failed.
+     */
+    private void checkNodes0(final int expCnt) throws Exception {
+        boolean wait = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                try {
+                    checkNodes(expCnt);
+
+                    return true;
+                }
+                catch (AssertionFailedError e) {
+                    log.info("Check failed, will retry: " + e);
+                }
+
+                return false;
+            }
+        }, 10_000);
+
+        if (!wait)
+            checkNodes(expCnt);
+    }
+
     /**
      * @param expCnt Expected number of nodes.
      */
@@ -297,23 +321,7 @@ private void manyClientsPutGet() throws Throwable {
             if (err0 != null)
                 throw err0;
 
-            boolean wait = GridTestUtils.waitForCondition(new GridAbsPredicate() {
-                @Override public boolean apply() {
-                    try {
-                        checkNodes(SRVS + THREADS);
-
-                        return true;
-                    }
-                    catch (AssertionFailedError e) {
-                        log.info("Check failed, will retry: " + e);
-                    }
-
-                    return false;
-                }
-            }, 10_000);
-
-            if (!wait)
-                checkNodes(SRVS + THREADS);
+            checkNodes0(SRVS + THREADS);
 
             log.info("Stop clients.");
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
index a55f21de6f68b..486fd6075d2ca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
@@ -713,6 +713,8 @@ private void executeTestForAllCaches(CI2<Ignite, IgniteCache<Integer, Integer>>
                 ", backups=" + ccfg.getBackups() +
                 ", near=" + (ccfg.getNearConfiguration() != null) + "]");
 
+            awaitPartitionMapExchange();
+
             int srvNum = serversNumber();
             if (serversNumber() > 1) {
                 ignite(serversNumber() + 1).createNearCache(ccfg.getName(), new NearCacheConfiguration<>());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java
index e3fa1165cffbd..60f2f0a80bd3c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadMultiThreadedSelfTest.java
@@ -116,6 +116,8 @@ public void testConcurrentNodesStart() throws Exception {
                     @Nullable @Override public Object call() throws Exception {
                         IgniteConfiguration cfg = loadConfiguration("modules/core/src/test/config/spring-multicache.xml");
 
+                        cfg.setGridLogger(getTestResources().getLogger());
+
                         startGrid(Thread.currentThread().getName(), cfg);
 
                         return null;
@@ -161,6 +163,8 @@ public void testConcurrentNodesStartStop() throws Exception {
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = loadConfiguration("modules/core/src/test/config/spring-multicache.xml");
 
+        cfg.setGridLogger(getTestResources().getLogger());
+
         cfg.setIgniteInstanceName(igniteInstanceName);
 
         for (CacheConfiguration cCfg : cfg.getCacheConfiguration()) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
index 05a9759cd559c..83eff893b8894 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
@@ -308,6 +308,8 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
 
                 // Check all left nodes.
                 checkActiveState(ignites);
+
+                awaitPartitionMapExchange(); // Need wait, otherwise test logic is broken if EVT_NODE_FAILED exchanges are merged.
             }
 
             info("Waiting for preload futures: " + F.view(futs, new IgnitePredicate<IgniteFuture<?>>() {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
index 7b350c8a41bb8..060af21f8d363 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
@@ -34,11 +34,11 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
@@ -65,7 +65,6 @@ public class TxRecoveryStoreEnabledTest extends GridCommonAbstractTest {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
         cfg.setCommunicationSpi(new TestCommunicationSpi());
-        cfg.setDiscoverySpi(new TestDiscoverySpi());
 
         CacheConfiguration ccfg = defaultCacheConfiguration();
 
@@ -126,7 +125,7 @@ private void checkTxRecovery(TransactionConcurrency concurrency) throws Exceptio
                     IgniteConfiguration cfg = node0.configuration();
 
                     ((TestCommunicationSpi)cfg.getCommunicationSpi()).block();
-                    ((TestDiscoverySpi)cfg.getDiscoverySpi()).simulateNodeFailure();
+                    ((IgniteDiscoverySpi)cfg.getDiscoverySpi()).simulateNodeFailure();
                 }
                 catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
@@ -198,16 +197,6 @@ private static class TestCacheStore extends CacheStoreAdapter<Integer, Integer>
         }
     }
 
-    /**
-     *
-     */
-    private static class TestDiscoverySpi extends TcpDiscoverySpi {
-        /** {@inheritDoc} */
-        @Override protected void simulateNodeFailure() {
-            super.simulateNodeFailure();
-        }
-    }
-
     /**
      *
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java
index 3c57957179e43..96fb8f65bb56c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedExplicitLockNodeFailureSelfTest.java
@@ -37,6 +37,7 @@
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 
 /**
@@ -125,7 +126,7 @@ public void testLockFromNearOrBackup() throws Exception {
 
                         return true;
                     }
-                }, EVT_NODE_LEFT));
+                }, EVT_NODE_LEFT, EVT_NODE_FAILED));
         }
 
         stopGrid(idx);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ClientReconnectContinuousQueryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ClientReconnectContinuousQueryTest.java
index c8b3bb6afc95f..9b531c68277a4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ClientReconnectContinuousQueryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ClientReconnectContinuousQueryTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.query.continuous;
 
+import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import javax.cache.event.CacheEntryListenerException;
@@ -28,6 +29,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.managers.communication.GridIoManager;
 import org.apache.ignite.internal.util.nio.GridNioServer;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -90,7 +92,7 @@ public void testClientReconnect() throws Exception {
         try {
             startGrids(2);
 
-            IgniteEx client = grid(CLIENT_IDX);
+            final IgniteEx client = grid(CLIENT_IDX);
 
             client.events().localListen(new DisconnectListener(), EventType.EVT_CLIENT_NODE_DISCONNECTED);
 
@@ -112,11 +114,19 @@ public void testClientReconnect() throws Exception {
 
             skipRead(client, true);
 
-            putSomeKeys(1_000);
+            IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    assertTrue(disconLatch.await(10_000, TimeUnit.MILLISECONDS));
+
+                    skipRead(client, false);
 
-            assertTrue(disconLatch.await(10_000, TimeUnit.MILLISECONDS));
+                    return null;
+                }
+            });
 
-            skipRead(client, false);
+            putSomeKeys(1_000);
+
+            fut.get();
 
             assertTrue(reconLatch.await(10_000, TimeUnit.MILLISECONDS));
 
@@ -129,7 +139,6 @@ public void testClientReconnect() throws Exception {
         finally {
             stopAllGrids();
         }
-
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryRemoteFilterMissingInClassPathSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryRemoteFilterMissingInClassPathSelfTest.java
index 92c1760832274..226302ff9d616 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryRemoteFilterMissingInClassPathSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/ContinuousQueryRemoteFilterMissingInClassPathSelfTest.java
@@ -104,7 +104,7 @@ public void testWarningMessageOnClientNode() throws Exception {
         setExternalLoader = true;
         final Ignite ignite0 = startGrid(1);
 
-        executeContiniouseQuery(ignite0.cache("simple"));
+        executeContinuousQuery(ignite0.cache("simple"));
 
         log = new GridStringLogger();
         clientMode = true;
@@ -112,8 +112,10 @@ public void testWarningMessageOnClientNode() throws Exception {
 
         startGrid(2);
 
-        assertTrue(log.toString().contains("Failed to unmarshal continuous query remote filter on client node. " +
-            "Can be ignored."));
+        String logStr = log.toString();
+
+        assertTrue(logStr.contains("Failed to unmarshal continuous query remote filter on client node. " +
+            "Can be ignored.") || logStr.contains("Failed to unmarshal continuous routine handler"));
     }
 
     /**
@@ -127,7 +129,7 @@ public void testNoWarningMessageOnClientNode() throws Exception {
         clientMode = false;
         final Ignite ignite0 = startGrid(1);
 
-        executeContiniouseQuery(ignite0.cache("simple"));
+        executeContinuousQuery(ignite0.cache("simple"));
 
         log = new GridStringLogger();
         clientMode = true;
@@ -149,15 +151,18 @@ public void testExceptionOnServerNode() throws Exception {
         setExternalLoader = true;
         final Ignite ignite0 = startGrid(1);
 
-        executeContiniouseQuery(ignite0.cache("simple"));
+        executeContinuousQuery(ignite0.cache("simple"));
 
         log = new GridStringLogger();
         setExternalLoader = false;
 
         startGrid(2);
 
-        assertTrue(log.toString().contains("class org.apache.ignite.IgniteCheckedException: " +
-            "Failed to find class with given class loader for unmarshalling"));
+        String logStr = log.toString();
+
+        assertTrue(logStr.contains("class org.apache.ignite.IgniteCheckedException: " +
+            "Failed to find class with given class loader for unmarshalling")
+            || logStr.contains("Failed to unmarshal continuous routine handler"));
     }
 
     /**
@@ -171,7 +176,7 @@ public void testNoExceptionOnServerNode() throws Exception {
         setExternalLoader = true;
         final Ignite ignite0 = startGrid(1);
 
-        executeContiniouseQuery(ignite0.cache("simple"));
+        executeContinuousQuery(ignite0.cache("simple"));
 
         log = new GridStringLogger();
 
@@ -185,7 +190,7 @@ public void testNoExceptionOnServerNode() throws Exception {
      * @param cache Ignite cache.
      * @throws Exception If fail.
      */
-    private void executeContiniouseQuery(IgniteCache cache) throws Exception {
+    private void executeContinuousQuery(IgniteCache cache) throws Exception {
         ContinuousQuery<Integer, String> qry = new ContinuousQuery<>();
 
         qry.setLocalListener(
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java
index 9ad6d4ec6c0a4..906cc7d8d190c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/IgniteCacheContinuousQueryClientReconnectTest.java
@@ -119,6 +119,9 @@ public void testReconnectClient() throws Exception {
      * @throws Exception If failed.
      */
     public void testReconnectClientAndLeftRouter() throws Exception {
+        if (!tcpDiscovery())
+            return;
+
         Ignite client = grid(serverCount());
 
         final Ignite srv = clientRouter(client);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryAbstractTest.java
index 61ceef7128dbb..16ea84857840a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/version/CacheVersionedEntryAbstractTest.java
@@ -18,11 +18,12 @@
 package org.apache.ignite.internal.processors.cache.version;
 
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
 import javax.cache.Cache;
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
 import javax.cache.processor.MutableEntry;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheEntry;
@@ -56,23 +57,15 @@ public abstract class CacheVersionedEntryAbstractTest extends GridCacheAbstractS
     public void testInvoke() throws Exception {
         Cache<Integer, String> cache = grid(0).cache(DEFAULT_CACHE_NAME);
 
-        final AtomicInteger invoked = new AtomicInteger();
-
-        cache.invoke(100, new EntryProcessor<Integer, String, Object>() {
-            @Override public Object process(MutableEntry<Integer, String> entry, Object... arguments)
-                throws EntryProcessorException {
-
-                invoked.incrementAndGet();
-
+        assertNotNull(cache.invoke(100, new EntryProcessor<Integer, String, Object>() {
+            @Override public Object process(MutableEntry<Integer, String> entry, Object... args) {
                 CacheEntry<Integer, String> verEntry = entry.unwrap(CacheEntry.class);
 
                 checkVersionedEntry(verEntry);
 
-                return entry;
+                return verEntry.version();
             }
-        });
-
-        assert invoked.get() > 0;
+        }));
     }
 
     /**
@@ -86,23 +79,17 @@ public void testInvokeAll() throws Exception {
         for (int i = 0; i < ENTRIES_NUM; i++)
             keys.add(i);
 
-        final AtomicInteger invoked = new AtomicInteger();
-
-        cache.invokeAll(keys, new EntryProcessor<Integer, String, Object>() {
-            @Override public Object process(MutableEntry<Integer, String> entry, Object... arguments)
-                throws EntryProcessorException {
-
-                invoked.incrementAndGet();
-
+        Map<Integer, EntryProcessorResult<Object>> res = cache.invokeAll(keys, new EntryProcessor<Integer, String, Object>() {
+            @Override public Object process(MutableEntry<Integer, String> entry, Object... args) {
                 CacheEntry<Integer, String> verEntry = entry.unwrap(CacheEntry.class);
 
                 checkVersionedEntry(verEntry);
 
-                return null;
+                return verEntry.version();
             }
         });
 
-        assert invoked.get() > 0;
+        assertEquals(ENTRIES_NUM, res.size());
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/continuous/GridEventConsumeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/continuous/GridEventConsumeSelfTest.java
index f07b1a3c02d70..1a7abd45fa921 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/continuous/GridEventConsumeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/continuous/GridEventConsumeSelfTest.java
@@ -447,7 +447,7 @@ public void testAllEvents() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(GRID_CNT, nodeIds.size());
             assertEquals(GRID_CNT, cnt.get());
@@ -488,7 +488,7 @@ public void testEventsByType() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(GRID_CNT, nodeIds.size());
             assertEquals(GRID_CNT, cnt.get());
@@ -532,7 +532,7 @@ public void testEventsByFilter() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(GRID_CNT, nodeIds.size());
             assertEquals(GRID_CNT, cnt.get());
@@ -578,7 +578,7 @@ public void testEventsByTypeAndFilter() throws Exception {
             grid(0).compute().broadcast(F.noop());
             grid(0).compute().withName("exclude").run(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(GRID_CNT, nodeIds.size());
             assertEquals(GRID_CNT, cnt.get());
@@ -619,7 +619,7 @@ public void testRemoteProjection() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(GRID_CNT - 1, nodeIds.size());
             assertEquals(GRID_CNT - 1, cnt.get());
@@ -660,7 +660,7 @@ public void testProjectionWithLocalNode() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(GRID_CNT - 1, nodeIds.size());
             assertEquals(GRID_CNT - 1, cnt.get());
@@ -701,7 +701,7 @@ public void testLocalNodeOnly() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(1, nodeIds.size());
             assertEquals(1, cnt.get());
@@ -744,7 +744,7 @@ public void testStopByCallback() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(1, nodeIds.size());
             assertEquals(1, cnt.get());
@@ -785,7 +785,7 @@ public void testStopRemoteListen() throws Exception {
 
             grid(0).compute().run(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(1, nodeIds.size());
             assertEquals(1, cnt.get());
@@ -828,7 +828,7 @@ public void testStopLocalListenByCallback() throws Exception {
 
         compute(grid(0).cluster().forLocal()).run(F.noop());
 
-        assert latch.await(2, SECONDS);
+        assert latch.await(10, SECONDS) : latch;
 
         assertEquals(1, cnt.get());
 
@@ -878,7 +878,7 @@ public void testNodeJoin() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(GRID_CNT + 1, nodeIds.size());
             assertEquals(GRID_CNT + 1, cnt.get());
@@ -929,7 +929,7 @@ public void testNodeJoinWithProjection() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(GRID_CNT, nodeIds.size());
             assertEquals(GRID_CNT, cnt.get());
@@ -980,7 +980,7 @@ public void testNodeJoinWithP2P() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(GRID_CNT + 1, nodeIds.size());
             assertEquals(GRID_CNT + 1, cnt.get());
@@ -1036,7 +1036,7 @@ public void testResources() throws Exception {
 
             grid(0).compute().broadcast(F.noop());
 
-            assert latch.await(2, SECONDS);
+            assert latch.await(10, SECONDS) : latch;
 
             assertEquals(GRID_CNT, nodeIds.size());
             assertEquals(GRID_CNT, cnt.get());
@@ -1145,7 +1145,7 @@ public void testMasterNodeLeaveNoAutoUnsubscribe() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testMultithreadedWithNodeRestart() throws Exception {
+    public void _testMultithreadedWithNodeRestart() throws Exception {
         final AtomicBoolean stop = new AtomicBoolean();
         final BlockingQueue<IgniteBiTuple<Integer, UUID>> queue = new LinkedBlockingQueue<>();
         final Collection<UUID> started = new GridConcurrentHashSet<>();
@@ -1153,9 +1153,11 @@ public void testMultithreadedWithNodeRestart() throws Exception {
 
         final Random rnd = new Random();
 
+        final int consumeCnt = tcpDiscovery() ? CONSUME_CNT : CONSUME_CNT / 2;
+
         IgniteInternalFuture<?> starterFut = multithreadedAsync(new Callable<Object>() {
             @Override public Object call() throws Exception {
-                for (int i = 0; i < CONSUME_CNT; i++) {
+                for (int i = 0; i < consumeCnt; i++) {
                     int idx = rnd.nextInt(GRID_CNT);
 
                     try {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ClosureServiceClientsNodesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ClosureServiceClientsNodesTest.java
index c1af32382a4cd..8f03c4cefaf0a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ClosureServiceClientsNodesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ClosureServiceClientsNodesTest.java
@@ -49,6 +49,9 @@ public class ClosureServiceClientsNodesTest extends GridCommonAbstractTest {
     /** Number of grids started for tests. */
     private static final int NODES_CNT = 4;
 
+    /** */
+    private static final int CLIENT_IDX = 1;
+
     /** Test singleton service name. */
     private static final String SINGLETON_NAME = "testSingleton";
 
@@ -61,11 +64,11 @@ public class ClosureServiceClientsNodesTest extends GridCommonAbstractTest {
 
         cfg.setMarshaller(new BinaryMarshaller());
 
-        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(ipFinder).setForceServerMode(true));
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
 
         cfg.setCacheConfiguration();
 
-        if (igniteInstanceName.equals(getTestIgniteInstanceName(0)))
+        if (igniteInstanceName.equals(getTestIgniteInstanceName(CLIENT_IDX)))
             cfg.setClientMode(true);
 
         return cfg;
@@ -88,8 +91,10 @@ public class ClosureServiceClientsNodesTest extends GridCommonAbstractTest {
     public void testDefaultClosure() throws Exception {
         Set<String> srvNames = new HashSet<>(NODES_CNT - 1);
 
-        for (int i = 1; i < NODES_CNT; ++i)
-            srvNames.add(getTestIgniteInstanceName(i));
+        for (int i = 0; i < NODES_CNT; ++i) {
+            if (i != CLIENT_IDX)
+                srvNames.add(getTestIgniteInstanceName(i));
+        }
 
         for (int i = 0 ; i < NODES_CNT; i++) {
             log.info("Iteration: " + i);
@@ -137,7 +142,7 @@ public void testClientClosure() throws Exception {
 
             assertEquals(1, res.size());
 
-            assertEquals(getTestIgniteInstanceName(0), F.first(res));
+            assertEquals(getTestIgniteInstanceName(CLIENT_IDX), F.first(res));
         }
     }
 
@@ -168,7 +173,7 @@ public void testCustomClosure() throws Exception {
      * @throws Exception If failed.
      */
     public void testDefaultService() throws Exception {
-        UUID clientNodeId = grid(0).cluster().localNode().id();
+        UUID clientNodeId = grid(CLIENT_IDX).cluster().localNode().id();
 
         for (int i = 0 ; i < NODES_CNT; i++) {
             log.info("Iteration: " + i);
@@ -209,7 +214,7 @@ public void testDefaultService() throws Exception {
      * @throws Exception If failed.
      */
     public void testClientService() throws Exception {
-        UUID clientNodeId = grid(0).cluster().localNode().id();
+        UUID clientNodeId = grid(CLIENT_IDX).cluster().localNode().id();
 
         for (int i = 0 ; i < NODES_CNT; i++) {
             log.info("Iteration: " + i);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java b/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java
index 5da9042705d3a..2d26b72bf9063 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/GridTestClockTimer.java
@@ -32,6 +32,15 @@ public GridTestClockTimer() {
         }
     }
 
+    /**
+     * @return {@code True} if need start test time.
+     */
+    public static boolean startTestTimer() {
+        synchronized (IgniteUtils.mux) {
+            return IgniteUtils.gridCnt == 0;
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void run() {
         while (true) {
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index cd8e757f945c7..78f3c03eda818 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -157,6 +158,9 @@ public void testPersistedMappingsSharedOnJoin() throws Exception {
         Ignite g2 = startGrid(2);
         startGrid(1);
 
+        assertTrue("Failed to wait for automatic grid activation",
+            GridTestUtils.waitForCondition(() -> g2.cluster().active(), getTestTimeout()));
+
         IgniteCache<Integer, DummyObject> c2 = g2.cache(CACHE_NAME);
 
         assertEquals(k, c2.get(k).val);
diff --git a/modules/core/src/test/java/org/apache/ignite/messaging/GridMessagingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/messaging/GridMessagingSelfTest.java
index 7541cec455c63..a7c452122d566 100644
--- a/modules/core/src/test/java/org/apache/ignite/messaging/GridMessagingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/messaging/GridMessagingSelfTest.java
@@ -24,7 +24,6 @@
 import java.io.Serializable;
 import java.net.URL;
 import java.net.URLClassLoader;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -37,22 +36,20 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.DiscoverySpiTestListener;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessage;
+import org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessageV2;
 import org.apache.ignite.internal.processors.continuous.StopRoutineDiscoveryMessage;
-import org.apache.ignite.internal.processors.marshaller.MappingAcceptedMessage;
-import org.apache.ignite.internal.processors.marshaller.MappingProposedMessage;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.typedef.P2;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -206,11 +203,7 @@ public TestMessage() {
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
-        TestTcpDiscoverySpi discoSpi = new TestTcpDiscoverySpi();
-
-        discoSpi.setIpFinder(ipFinder);
-
-        cfg.setDiscoverySpi(discoSpi);
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
 
         return cfg;
     }
@@ -1036,7 +1029,11 @@ public void testNullMessages() throws Exception {
     public void testAsyncOld() throws Exception {
         final AtomicInteger msgCnt = new AtomicInteger();
 
-        TestTcpDiscoverySpi discoSpi = (TestTcpDiscoverySpi)ignite2.configuration().getDiscoverySpi();
+        IgniteDiscoverySpi discoSpi = (IgniteDiscoverySpi)ignite2.configuration().getDiscoverySpi();
+
+        DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+        discoSpi.setInternalListener(lsnr);
 
         assertFalse(ignite2.message().isAsync());
 
@@ -1054,7 +1051,7 @@ public void testAsyncOld() throws Exception {
             }
         }, IllegalStateException.class, null);
 
-        discoSpi.blockCustomEvent();
+        lsnr.blockCustomEvent(StartRoutineDiscoveryMessage.class, StartRoutineDiscoveryMessageV2.class);
 
         final String topic = "topic";
 
@@ -1079,7 +1076,7 @@ public void testAsyncOld() throws Exception {
 
         Assert.assertFalse(starFut.isDone());
 
-        discoSpi.stopBlock();
+        lsnr.stopBlockCustomEvents();
 
         GridTestUtils.assertThrows(log, new Callable<Void>() {
             @Override public Void call() throws Exception {
@@ -1095,7 +1092,7 @@ public void testAsyncOld() throws Exception {
 
         Assert.assertTrue(starFut.isDone());
 
-        discoSpi.blockCustomEvent();
+        lsnr.blockCustomEvent(StopRoutineDiscoveryMessage.class);
 
         message(ignite1.cluster().forRemotes()).send(topic, "msg1");
 
@@ -1125,7 +1122,7 @@ public void testAsyncOld() throws Exception {
 
         Assert.assertFalse(stopFut.isDone());
 
-        discoSpi.stopBlock();
+        lsnr.stopBlockCustomEvents();
 
         stopFut.get();
 
@@ -1144,9 +1141,13 @@ public void testAsyncOld() throws Exception {
     public void testAsync() throws Exception {
         final AtomicInteger msgCnt = new AtomicInteger();
 
-        TestTcpDiscoverySpi discoSpi = (TestTcpDiscoverySpi)ignite2.configuration().getDiscoverySpi();
+        IgniteDiscoverySpi discoSpi = (IgniteDiscoverySpi)ignite2.configuration().getDiscoverySpi();
 
-        discoSpi.blockCustomEvent();
+        DiscoverySpiTestListener lsnr = new DiscoverySpiTestListener();
+
+        discoSpi.setInternalListener(lsnr);
+
+        lsnr.blockCustomEvent(StartRoutineDiscoveryMessage.class, StartRoutineDiscoveryMessageV2.class);
 
         final String topic = "topic";
 
@@ -1167,7 +1168,7 @@ public void testAsync() throws Exception {
 
         Assert.assertFalse(starFut.isDone());
 
-        discoSpi.stopBlock();
+        lsnr.stopBlockCustomEvents();
 
         UUID id = starFut.get();
 
@@ -1175,7 +1176,7 @@ public void testAsync() throws Exception {
 
         Assert.assertTrue(starFut.isDone());
 
-        discoSpi.blockCustomEvent();
+        lsnr.blockCustomEvent(StopRoutineDiscoveryMessage.class);
 
         message(ignite1.cluster().forRemotes()).send(topic, "msg1");
 
@@ -1195,7 +1196,7 @@ public void testAsync() throws Exception {
 
         Assert.assertFalse(stopFut.isDone());
 
-        discoSpi.stopBlock();
+        lsnr.stopBlockCustomEvents();
 
         stopFut.get();
 
@@ -1208,89 +1209,6 @@ public void testAsync() throws Exception {
         assertEquals(1, msgCnt.get());
     }
 
-    /**
-     *
-     */
-    static class TestTcpDiscoverySpi extends TcpDiscoverySpi {
-        /** */
-        private boolean blockCustomEvt;
-
-        /** */
-        private final Object mux = new Object();
-
-        /** */
-        private List<DiscoverySpiCustomMessage> blockedMsgs = new ArrayList<>();
-
-        /** {@inheritDoc} */
-        @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException {
-            synchronized (mux) {
-                if (blockCustomEvt) {
-                    DiscoveryCustomMessage msg0 = GridTestUtils.getFieldValue(msg, "delegate");
-
-                    if (msg0 instanceof MappingProposedMessage || msg0 instanceof MappingAcceptedMessage){
-                        super.sendCustomEvent(msg);
-
-                        return;
-                    }
-
-                    if (msg0 instanceof StopRoutineDiscoveryMessage || msg0 instanceof StartRoutineDiscoveryMessage) {
-                        log.info("Block custom message: " + msg0);
-
-                        blockedMsgs.add(msg);
-
-                        mux.notifyAll();
-
-                        return;
-                    }
-                }
-            }
-
-            super.sendCustomEvent(msg);
-        }
-
-        /**
-         *
-         */
-        public void blockCustomEvent() {
-            synchronized (mux) {
-                assert blockedMsgs.isEmpty() : blockedMsgs;
-
-                blockCustomEvt = true;
-            }
-        }
-
-        /**
-         * @throws InterruptedException If interrupted.
-         */
-        public void waitCustomEvent() throws InterruptedException {
-            synchronized (mux) {
-                while (blockedMsgs.isEmpty())
-                    mux.wait();
-            }
-        }
-
-        /**
-         *
-         */
-        public void stopBlock() {
-            List<DiscoverySpiCustomMessage> msgs;
-
-            synchronized (this) {
-                msgs = new ArrayList<>(blockedMsgs);
-
-                blockCustomEvt = false;
-
-                blockedMsgs.clear();
-            }
-
-            for (DiscoverySpiCustomMessage msg : msgs) {
-                log.info("Resend blocked message: " + msg);
-
-                super.sendCustomEvent(msg);
-            }
-        }
-    }
-
     /**
      * Tests that message listener registers only for one oldest node.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
index 54b3a785dc250..e89a4c828fb75 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/GridTcpCommunicationSpiAbstractTest.java
@@ -17,11 +17,21 @@
 
 package org.apache.ignite.spi.communication.tcp;
 
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CyclicBarrier;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.nio.GridCommunicationClient;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiAdapter;
 import org.apache.ignite.spi.communication.CommunicationSpi;
 import org.apache.ignite.spi.communication.GridAbstractCommunicationSelfTest;
@@ -85,6 +95,67 @@ protected GridTcpCommunicationSpiAbstractTest(boolean useShmem) {
         }
     }
 
+    /**
+     *
+     */
+    public void testCheckConnection1() {
+        for (int i = 0; i < 100; i++) {
+            for (Map.Entry<UUID, CommunicationSpi<Message>> entry : spis.entrySet()) {
+                TcpCommunicationSpi spi = (TcpCommunicationSpi)entry.getValue();
+
+                List<ClusterNode> checkNodes = new ArrayList<>(nodes);
+
+                assert checkNodes.size() > 1;
+
+                IgniteFuture<BitSet> fut = spi.checkConnection(checkNodes);
+
+                BitSet res = fut.get();
+
+                for (int n = 0; n < checkNodes.size(); n++)
+                    assertTrue(res.get(n));
+            }
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCheckConnection2() throws Exception {
+        final int THREADS = spis.size();
+
+        final CyclicBarrier b = new CyclicBarrier(THREADS);
+
+        List<IgniteInternalFuture> futs = new ArrayList<>();
+
+        for (Map.Entry<UUID, CommunicationSpi<Message>> entry : spis.entrySet()) {
+            final TcpCommunicationSpi spi = (TcpCommunicationSpi)entry.getValue();
+
+            futs.add(GridTestUtils.runAsync(new Callable() {
+                @Override public Object call() throws Exception {
+                    List<ClusterNode> checkNodes = new ArrayList<>(nodes);
+
+                    assert checkNodes.size() > 1;
+
+                    b.await();
+
+                    for (int i = 0; i < 100; i++) {
+                        IgniteFuture<BitSet> fut = spi.checkConnection(checkNodes);
+
+                        BitSet res = fut.get();
+
+                        for (int n = 0; n < checkNodes.size(); n++)
+                            assertTrue(res.get(n));
+                    }
+
+                    return null;
+                }
+            }));
+        }
+
+        for (IgniteInternalFuture f : futs)
+            f.get();
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         super.afterTest();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/FilterDataForClientNodeDiscoveryTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/FilterDataForClientNodeDiscoveryTest.java
index 54b48e5a9df61..9a45d2d68d5be 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/FilterDataForClientNodeDiscoveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/FilterDataForClientNodeDiscoveryTest.java
@@ -205,6 +205,11 @@ private static class MessageForServer implements DiscoveryServerOnlyCustomMessag
             return false;
         }
 
+        /** {@inheritDoc} */
+        @Override public boolean stopProcess() {
+            return false;
+        }
+
         /** {@inheritDoc} */
         @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
             DiscoCache discoCache) {
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
index ca052883e35e2..51dcb236dfcf9 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridSpiTestContext.java
@@ -613,6 +613,16 @@ public void triggerEvent(Event evt) {
         return Collections.emptyMap();
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean communicationFailureResolveSupported() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void resolveCommunicationFailure(ClusterNode node, Exception err) {
+        throw new UnsupportedOperationException();
+    }
+
     /**
      * @param cacheName Cache name.
      * @return Map representing cache.
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/config/GridTestProperties.java b/modules/core/src/test/java/org/apache/ignite/testframework/config/GridTestProperties.java
index 4507572f92a7a..e2594ca04b29a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/config/GridTestProperties.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/config/GridTestProperties.java
@@ -83,6 +83,15 @@ public final class GridTestProperties {
     /** "True value" enables {@link BinaryBasicNameMapper} in {@link BinaryTypeConfiguration#getNameMapper()}  */
     public static final String BINARY_MARSHALLER_USE_SIMPLE_NAME_MAPPER = "binary.marshaller.use.simple.name.mapper";
 
+    /**
+     * Name of class which provides static method preprocessConfiguration(IgniteConfiguration cfg) to
+     * alter {@link org.apache.ignite.configuration.IgniteConfiguration} before node is started.
+     * <p>
+     * Note: this pre-preprocessor is started only if test starts node using one of GridAbstractTest's startGrid
+     * method.
+     */
+    public static final String IGNITE_CFG_PREPROCESSOR_CLS = "ignite.cfg.preprocessor.class";
+
     /** */
     static {
         // Initialize IGNITE_HOME system property.
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index c3b262c3667da..f5784ebcdccd2 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -46,6 +46,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
@@ -84,6 +85,7 @@
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.logger.NullLogger;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.marshaller.MarshallerContextTestImpl;
@@ -92,6 +94,7 @@
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointSpi;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
@@ -124,6 +127,7 @@
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
 import static org.apache.ignite.internal.GridKernalState.DISCONNECTED;
 import static org.apache.ignite.testframework.config.GridTestProperties.BINARY_MARSHALLER_USE_SIMPLE_NAME_MAPPER;
+import static org.apache.ignite.testframework.config.GridTestProperties.IGNITE_CFG_PREPROCESSOR_CLS;
 
 /**
  * Common abstract test for Ignite tests.
@@ -203,13 +207,15 @@ public abstract class GridAbstractTest extends TestCase {
         if (BINARY_MARSHALLER)
             GridTestProperties.setProperty(GridTestProperties.MARSH_CLASS_NAME, BinaryMarshaller.class.getName());
 
-        Thread timer = new Thread(new GridTestClockTimer(), "ignite-clock-for-tests");
+        if (GridTestClockTimer.startTestTimer()) {
+            Thread timer = new Thread(new GridTestClockTimer(), "ignite-clock-for-tests");
 
-        timer.setDaemon(true);
+            timer.setDaemon(true);
 
-        timer.setPriority(10);
+            timer.setPriority(10);
 
-        timer.start();
+            timer.start();
+        }
     }
 
     /** */
@@ -838,6 +844,7 @@ protected Ignite startGrid(String igniteInstanceName) throws Exception {
     protected Ignite startGrid(String igniteInstanceName, GridSpringResourceContext ctx) throws Exception {
         return startGrid(igniteInstanceName, optimize(getConfiguration(igniteInstanceName)), ctx);
     }
+
     /**
      * Starts new grid with given name.
      *
@@ -852,12 +859,33 @@ protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, G
             startingIgniteInstanceName.set(igniteInstanceName);
 
             try {
+                String cfgProcClsName = System.getProperty(IGNITE_CFG_PREPROCESSOR_CLS);
+
+                if (cfgProcClsName != null) {
+                    try {
+                        Class<?> cfgProc = Class.forName(cfgProcClsName);
+
+                        Method method = cfgProc.getMethod("preprocessConfiguration", IgniteConfiguration.class);
+
+                        if (!Modifier.isStatic(method.getModifiers()))
+                            throw new Exception("Non-static pre-processor method in pre-processor class: " + cfgProcClsName);
+
+                        method.invoke(null, cfg);
+                    }
+                    catch (Exception e) {
+                        log.error("Failed to pre-process IgniteConfiguration using pre-processor class: " + cfgProcClsName);
+
+                        throw new IgniteException(e);
+                    }
+                }
+
                 Ignite node = IgnitionEx.start(cfg, ctx);
 
                 IgniteConfiguration nodeCfg = node.configuration();
 
                 log.info("Node started with the following configuration [id=" + node.cluster().localNode().id()
                     + ", marshaller=" + nodeCfg.getMarshaller()
+                    + ", discovery=" + nodeCfg.getDiscoverySpi()
                     + ", binaryCfg=" + nodeCfg.getBinaryConfiguration()
                     + ", lateAff=" + nodeCfg.isLateAffinityAssignment() + "]");
 
@@ -967,6 +995,26 @@ protected Ignite startRemoteGrid(String igniteInstanceName, IgniteConfiguration
         if (cfg == null)
             cfg = optimize(getConfiguration(igniteInstanceName));
 
+        if (locNode != null) {
+            DiscoverySpi discoverySpi = locNode.configuration().getDiscoverySpi();
+
+            if (discoverySpi != null && !(discoverySpi instanceof TcpDiscoverySpi)) {
+                try {
+                    // Clone added to support ZookeeperDiscoverySpi.
+                    Method m = discoverySpi.getClass().getDeclaredMethod("cloneSpiConfiguration");
+
+                    m.setAccessible(true);
+
+                    cfg.setDiscoverySpi((DiscoverySpi) m.invoke(discoverySpi));
+
+                    resetDiscovery = false;
+                }
+                catch (NoSuchMethodException e) {
+                    // Ignore.
+                }
+            }
+        }
+
         return new IgniteProcessProxy(cfg, log, locNode, resetDiscovery);
     }
 
@@ -1075,7 +1123,9 @@ protected void stopAllGrids(boolean cancel) {
             for (Ignite g : srvs)
                 stopGrid(g.name(), cancel, false);
 
-            assert G.allGrids().isEmpty();
+            List<Ignite> nodes = G.allGrids();
+
+            assert nodes.isEmpty() : nodes;
         }
         finally {
             IgniteProcessProxy.killAll(); // In multi-JVM case.
@@ -1176,6 +1226,14 @@ protected Ignite ignite(int idx) {
         return grid(idx);
     }
 
+    /**
+     * @param nodeIdx Node index.
+     * @return Node ID.
+     */
+    protected final UUID nodeId(int nodeIdx) {
+        return ignite(nodeIdx).cluster().localNode().id();
+    }
+
     /**
      * Gets grid for given test.
      *
@@ -1217,7 +1275,11 @@ protected final Ignite grid(ClusterNode node) {
      * @throws Exception If failed.
      */
     protected Ignite startGrid(String igniteInstanceName, String springCfgPath) throws Exception {
-        return startGrid(igniteInstanceName, loadConfiguration(springCfgPath));
+        IgniteConfiguration cfg = loadConfiguration(springCfgPath);
+
+        cfg.setGridLogger(getTestResources().getLogger());
+
+        return startGrid(igniteInstanceName, cfg);
     }
 
     /**
@@ -2142,6 +2204,50 @@ private void awaitTopologyChange() throws IgniteInterruptedCheckedException {
             }
         }
     }
+    /**
+     * @param expSize Expected nodes number.
+     * @throws Exception If failed.
+     */
+    protected void waitForTopology(final int expSize) throws Exception {
+        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                List<Ignite> nodes = G.allGrids();
+
+                if (nodes.size() != expSize) {
+                    info("Wait all nodes [size=" + nodes.size() + ", exp=" + expSize + ']');
+
+                    return false;
+                }
+
+                for (Ignite node: nodes) {
+                    try {
+                        IgniteFuture<?> reconnectFut = node.cluster().clientReconnectFuture();
+
+                        if (reconnectFut != null && !reconnectFut.isDone()) {
+                            info("Wait for size on node, reconnect is in progress [node=" + node.name() + ']');
+
+                            return false;
+                        }
+
+                        int sizeOnNode = node.cluster().nodes().size();
+
+                        if (sizeOnNode != expSize) {
+                            info("Wait for size on node [node=" + node.name() + ", size=" + sizeOnNode + ", exp=" + expSize + ']');
+
+                            return false;
+                        }
+                    }
+                    catch (IgniteClientDisconnectedException e) {
+                        info("Wait for size on node, node disconnected [node=" + node.name() + ']');
+
+                        return false;
+                    }
+                }
+
+                return true;
+            }
+        }, 30_000));
+    }
 
     /**
      * @param millis Time to sleep.
@@ -2171,6 +2277,17 @@ protected final int groupIdForCache(Ignite node, String cacheName) {
         return 0;
     }
 
+    /**
+     * @return {@code True} if nodes use {@link TcpDiscoverySpi}.
+     */
+    protected static boolean tcpDiscovery() {
+        List<Ignite> nodes = G.allGrids();
+
+        assertFalse("There are no nodes", nodes.isEmpty());
+
+        return nodes.get(0).configuration().getDiscoverySpi() instanceof TcpDiscoverySpi;
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteNodeRunner.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteNodeRunner.java
index d7be576d77c1e..2b3a19cdba956 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteNodeRunner.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteNodeRunner.java
@@ -162,6 +162,8 @@ private static IgniteConfiguration readCfgFromFileAndDeleteFile(String fileName)
                 cfg.setDiscoverySpi(disco);
             }
 
+            X.println("Configured discovery: " + cfg.getDiscoverySpi().getClass().getName());
+
             return cfg;
         }
         finally {
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 2ffa11ebf2849..be99adf3b169d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.ClusterNodeMetricsSelfTest;
+import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest;
 import org.apache.ignite.internal.GridAffinityNoCacheSelfTest;
 import org.apache.ignite.internal.GridAffinitySelfTest;
 import org.apache.ignite.internal.GridAlwaysFailoverSpiFailSelfTest;
@@ -121,6 +122,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridAlwaysFailoverSpiFailSelfTest.class);
         suite.addTestSuite(GridTaskInstanceExecutionSelfTest.class);
         suite.addTestSuite(ClusterNodeMetricsSelfTest.class);
+        suite.addTestSuite(ClusterNodeMetricsUpdateTest.class);
         suite.addTestSuite(GridNonHistoryMetricsSelfTest.class);
         suite.addTestSuite(GridCancelledJobsMetricsSelfTest.class);
         suite.addTestSuite(GridCollisionJobsContextSelfTest.class);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
index b9ef1e44bb351..e26b2111f90c5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
@@ -117,7 +117,7 @@ public void testQueryCancelsOnGridShutdown() throws Exception {
                     }
 
                     for (Ignite g : G.allGrids())
-                        if (!g.configuration().getDiscoverySpi().isClientMode())
+                        if (!g.configuration().isClientMode())
                             stopGrid(g.name(), true);
                 }
             }, 1);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
index 97720d5666057..bd3b0939e92bd 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
@@ -89,11 +89,14 @@ public abstract class DynamicIndexAbstractBasicSelfTest extends DynamicIndexAbst
      * @param mode Mode.
      * @param atomicityMode Atomicity mode.
      * @param near Near flag.
+     * @throws Exception If failed.
      */
     private void initialize(CacheMode mode, CacheAtomicityMode atomicityMode, boolean near)
-        throws IgniteCheckedException {
+        throws Exception {
         createSqlCache(node(), cacheConfiguration(mode, atomicityMode, near));
 
+        awaitPartitionMapExchange();
+
         grid(IDX_CLI_NEAR_ONLY).getOrCreateNearCache(CACHE_NAME, new NearCacheConfiguration<>());
 
         assertNoIndex(CACHE_NAME, TBL_NAME, IDX_NAME_1);
diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
index a181068e8ae1e..5cad1673c1908 100644
--- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
+++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
@@ -44,13 +44,7 @@ public class GridJtaTransactionManagerSelfTest extends GridCommonAbstractTest {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName).
             setCacheConfiguration(defaultCacheConfiguration().setCacheMode(PARTITIONED));
 
-        cfg.getTransactionConfiguration().setTxManagerFactory(new Factory<TransactionManager>() {
-            private static final long serialVersionUID = 0L;
-
-            @Override public TransactionManager create() {
-                return jotm.getTransactionManager();
-            }
-        });
+        cfg.getTransactionConfiguration().setTxManagerFactory(new TestTxManagerFactory());
 
         return cfg;
     }
@@ -205,4 +199,17 @@ public void testJtaTxContextSwitchWithExistingTx() throws Exception {
             cache.removeAll();
         }
     }
+
+    /**
+     *
+     */
+    static class TestTxManagerFactory implements Factory<TransactionManager> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public TransactionManager create() {
+            return jotm.getTransactionManager();
+        }
+    }
 }
diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
index f6fd5c7675540..14b7fae291c27 100644
--- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
+++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
@@ -30,12 +30,19 @@ public class GridPartitionedCacheJtaFactorySelfTest extends AbstractCacheJtaSelf
     @Override protected void configureJta(IgniteConfiguration cfg) {
         TransactionConfiguration txCfg = cfg.getTransactionConfiguration();
 
-        txCfg.setTxManagerFactory(new Factory<TransactionManager>() {
-            private static final long serialVersionUID = 0L;
+        txCfg.setTxManagerFactory(new TestTxManagerFactory());
+    }
+
+    /**
+     *
+     */
+    static class TestTxManagerFactory implements Factory<TransactionManager> {
+        /** */
+        private static final long serialVersionUID = 0L;
 
-            @Override public TransactionManager create() {
-                return jotm.getTransactionManager();
-            }
-        });
+        /** {@inheritDoc} */
+        @Override public TransactionManager create() {
+            return jotm.getTransactionManager();
+        }
     }
 }
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
index fce47a6e621ff..d87ea0a206d88 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
@@ -25,6 +25,7 @@ import org.apache.ignite.internal.processors.cache.query.QueryCursorEx
 import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata
 import org.apache.ignite.lang.IgniteUuid
 import org.apache.ignite.spark.impl._
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode
 import org.apache.spark._
 import org.apache.spark.rdd.RDD
@@ -91,8 +92,14 @@ class IgniteRDD[K, V] (
     override protected[spark] def getPreferredLocations(split: Partition): Seq[String] = {
         ensureCache()
 
-        ic.ignite().affinity(cacheName).mapPartitionToPrimaryAndBackups(split.index)
+        if (ic.ignite().configuration().getDiscoverySpi().isInstanceOf[TcpDiscoverySpi]) {
+          ic.ignite().affinity(cacheName).mapPartitionToPrimaryAndBackups(split.index)
             .map(_.asInstanceOf[TcpDiscoveryNode].socketAddresses()).flatten.map(_.getHostName).toList
+        }
+        else {
+          ic.ignite().affinity(cacheName).mapPartitionToPrimaryAndBackups(split.index)
+            .flatten(_.hostNames).toSeq
+        }
     }
 
     /**
diff --git a/modules/spring/src/test/java/org/apache/ignite/internal/GridFactorySelfTest.java b/modules/spring/src/test/java/org/apache/ignite/internal/GridFactorySelfTest.java
index 75128fc374c45..b45385893c1b5 100644
--- a/modules/spring/src/test/java/org/apache/ignite/internal/GridFactorySelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/internal/GridFactorySelfTest.java
@@ -998,7 +998,8 @@ public void testRepeatingStart() throws Exception {
 
             startGrid("1", c);
 
-            assert ((TcpDiscoverySpi)c.getDiscoverySpi()).started();
+            if (tcpDiscovery())
+                assert ((TcpDiscoverySpi)c.getDiscoverySpi()).started();
 
             try {
                 startGrid("2", c);
diff --git a/modules/spring/src/test/java/org/apache/ignite/p2p/GridP2PUserVersionChangeSelfTest.java b/modules/spring/src/test/java/org/apache/ignite/p2p/GridP2PUserVersionChangeSelfTest.java
index b861e192a9978..46da3cc18dec0 100644
--- a/modules/spring/src/test/java/org/apache/ignite/p2p/GridP2PUserVersionChangeSelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/p2p/GridP2PUserVersionChangeSelfTest.java
@@ -44,6 +44,7 @@
 import org.apache.ignite.testsuites.IgniteIgnore;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.events.EventType.EVT_TASK_UNDEPLOYED;
 
@@ -255,12 +256,12 @@ public void testRedeployOnNodeRestartSharedMode() throws Exception {
 
             ignite2.events().localListen(new IgnitePredicate<Event>() {
                 @Override public boolean apply(Event evt) {
-                    if (evt.type() == EVT_NODE_LEFT)
+                    if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
                         discoLatch.countDown();
 
                     return true;
                 }
-            }, EVT_NODE_LEFT);
+            }, EVT_NODE_LEFT, EVT_NODE_FAILED);
 
             Integer res1 = (Integer)ignite1.compute().execute(task1, ignite2.cluster().localNode().id());
 
diff --git a/modules/yardstick/pom-standalone.xml b/modules/yardstick/pom-standalone.xml
index 577a95ede2ab2..6905d94811c22 100644
--- a/modules/yardstick/pom-standalone.xml
+++ b/modules/yardstick/pom-standalone.xml
@@ -52,6 +52,12 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-zookeeper</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.ignite</groupId>
             <artifactId>ignite-log4j</artifactId>
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index 8cad24b75b002..9923bb7c7ddf9 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -53,6 +53,12 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-zookeeper</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.ignite</groupId>
             <artifactId>ignite-log4j</artifactId>
diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index c3c36797e3d54..2d47ece408719 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -47,6 +47,12 @@
             <version>${curator.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.curator</groupId>
+            <artifactId>curator-recipes</artifactId>
+            <version>${curator.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-x-discovery</artifactId>
@@ -107,6 +113,13 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-indexing</artifactId>
+            <version>${project.version}</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.ignite</groupId>
             <artifactId>ignite-log4j</artifactId>
@@ -121,6 +134,13 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.thoughtworks.xstream</groupId>
+            <artifactId>xstream</artifactId>
+            <version>1.4.8</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.apache.ignite</groupId>
             <artifactId>ignite-core</artifactId>
@@ -128,10 +148,30 @@
             <type>test-jar</type>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.ignite</groupId>
+            <artifactId>ignite-indexing</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
             <!-- Generate the OSGi MANIFEST.MF for this bundle. -->
             <plugin>
                 <groupId>org.apache.felix</groupId>
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
new file mode 100644
index 0000000000000..860c71c16e972
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
@@ -0,0 +1,557 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.curator.utils.PathUtils;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.IgniteSpiAdapter;
+import org.apache.ignite.spi.IgniteSpiConfiguration;
+import org.apache.ignite.spi.IgniteSpiContext;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiMultipleInstancesSupport;
+import org.apache.ignite.spi.communication.CommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
+import org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport;
+import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.ignite.spi.discovery.DiscoverySpiMutableCustomMessageSupport;
+import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
+import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClusterNode;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT;
+import static org.apache.ignite.IgniteSystemProperties.getBoolean;
+
+/**
+ * Zookeeper Discovery Spi.
+ */
+@IgniteSpiMultipleInstancesSupport(true)
+@DiscoverySpiOrderSupport(true)
+@DiscoverySpiHistorySupport(true)
+@DiscoverySpiMutableCustomMessageSupport(false)
+public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, IgniteDiscoverySpi {
+    /** */
+    public static final String DFLT_ROOT_PATH = "/apacheIgnite";
+
+    /** */
+    public static final long DFLT_JOIN_TIMEOUT = 0;
+
+    /** */
+    @GridToStringInclude
+    private String zkRootPath = DFLT_ROOT_PATH;
+
+    /** */
+    @GridToStringInclude
+    private String zkConnectionString;
+
+    /** */
+    private long joinTimeout = DFLT_JOIN_TIMEOUT;
+
+    /** */
+    @GridToStringInclude
+    private long sesTimeout;
+
+    /** */
+    private boolean clientReconnectDisabled;
+
+    /** */
+    @GridToStringExclude
+    private DiscoverySpiListener lsnr;
+
+    /** */
+    @GridToStringExclude
+    private DiscoverySpiDataExchange exchange;
+
+    /** */
+    @GridToStringExclude
+    private DiscoverySpiNodeAuthenticator nodeAuth;
+
+    /** */
+    @GridToStringExclude
+    private DiscoveryMetricsProvider metricsProvider;
+
+    /** */
+    @GridToStringExclude
+    private ZookeeperDiscoveryImpl impl;
+
+    /** */
+    @GridToStringExclude
+    private Map<String, Object> locNodeAttrs;
+
+    /** */
+    @GridToStringExclude
+    private IgniteProductVersion locNodeVer;
+
+    /** */
+    @GridToStringExclude
+    private Serializable consistentId;
+
+    /** Local node addresses. */
+    private IgniteBiTuple<Collection<String>, Collection<String>> addrs;
+
+    /** */
+    @LoggerResource
+    @GridToStringExclude
+    private IgniteLogger log;
+
+    /** */
+    private IgniteDiscoverySpiInternalListener internalLsnr;
+
+    /**
+     * @return Base path in ZK for znodes created by SPI.
+     */
+    public String getZkRootPath() {
+        return zkRootPath;
+    }
+
+    /**
+     * @param zkRootPath Base path in ZooKeeper for znodes created by SPI.
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public ZookeeperDiscoverySpi setZkRootPath(String zkRootPath) {
+        this.zkRootPath = zkRootPath;
+
+        return this;
+    }
+
+    /**
+     * @return ZooKeeper session timeout.
+     */
+    public long getSessionTimeout() {
+        return sesTimeout;
+    }
+
+    /**
+     * @param sesTimeout ZooKeeper session timeout.
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public ZookeeperDiscoverySpi setSessionTimeout(long sesTimeout) {
+        this.sesTimeout = sesTimeout;
+
+        return this;
+    }
+
+    /**
+     * @return Cluster join timeout.
+     */
+    public long getJoinTimeout() {
+        return joinTimeout;
+    }
+
+    /**
+     * @param joinTimeout Cluster join timeout ({@code 0} means wait forever).
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public ZookeeperDiscoverySpi setJoinTimeout(long joinTimeout) {
+        this.joinTimeout = joinTimeout;
+
+        return this;
+    }
+
+    /**
+     * @return ZooKeeper connection string
+     */
+    public String getZkConnectionString() {
+        return zkConnectionString;
+    }
+
+    /**
+     * @param zkConnectionString ZooKeeper connection string
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = false)
+    public ZookeeperDiscoverySpi setZkConnectionString(String zkConnectionString) {
+        this.zkConnectionString = zkConnectionString;
+
+        return this;
+    }
+
+    /**
+     * If {@code true} client does not try to reconnect.
+     *
+     * @return Client reconnect disabled flag.
+     */
+    public boolean isClientReconnectDisabled() {
+        return clientReconnectDisabled;
+    }
+
+    /**
+     * Sets client reconnect disabled flag.
+     *
+     * @param clientReconnectDisabled Client reconnect disabled flag.
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public ZookeeperDiscoverySpi setClientReconnectDisabled(boolean clientReconnectDisabled) {
+        this.clientReconnectDisabled = clientReconnectDisabled;
+
+        return this;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean clientReconnectSupported() {
+        return !clientReconnectDisabled;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void clientReconnect() {
+        impl.reconnect();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean knownNode(UUID nodeId) {
+        return impl.knownNode(nodeId);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean supportsCommunicationFailureResolve() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void resolveCommunicationFailure(ClusterNode node, Exception err) {
+        impl.resolveCommunicationError(node, err);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Serializable consistentId() throws IgniteSpiException {
+        if (consistentId == null) {
+            consistentId = ignite.configuration().getConsistentId();
+
+            if (consistentId == null) {
+                initAddresses();
+
+                final List<String> sortedAddrs = new ArrayList<>(addrs.get1());
+
+                Collections.sort(sortedAddrs);
+
+                if (getBoolean(IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT))
+                    consistentId = U.consistentId(sortedAddrs);
+                else {
+                    Integer commPort = null;
+
+                    if (locNodeAttrs != null) {
+                        commPort = (Integer)locNodeAttrs.get(
+                            TcpCommunicationSpi.class.getSimpleName() + "." + TcpCommunicationSpi.ATTR_PORT);
+                    }
+                    else {
+                        CommunicationSpi commSpi = ignite.configuration().getCommunicationSpi();
+
+                        if (commSpi instanceof TcpCommunicationSpi) {
+                            commPort = ((TcpCommunicationSpi)commSpi).boundPort();
+
+                            if (commPort == -1)
+                                commPort = null;
+                        }
+                    }
+
+                    if (commPort == null) {
+                        U.warn(log, "Can not initialize default consistentId, TcpCommunicationSpi port is not initialized.");
+
+                        consistentId = ignite.configuration().getNodeId();
+                    }
+                    else
+                        consistentId = U.consistentId(sortedAddrs, commPort);
+                }
+            }
+        }
+
+        return consistentId;
+    }
+
+    /**
+     *
+     */
+    private void initAddresses() {
+        if (addrs == null) {
+            String locHost = ignite != null ? ignite.configuration().getLocalHost() : null;
+
+            InetAddress locAddr;
+
+            try {
+                locAddr = U.resolveLocalHost(locHost);
+            }
+            catch (IOException e) {
+                throw new IgniteSpiException("Unknown local address: " + locHost, e);
+            }
+
+            try {
+                addrs = U.resolveLocalAddresses(locAddr);
+            }
+            catch (Exception e) {
+                throw new IgniteSpiException("Failed to resolve local host to set of external addresses: " + locHost,
+                    e);
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<ClusterNode> getRemoteNodes() {
+        return impl.remoteNodes();
+    }
+
+    /** {@inheritDoc} */
+    @Override public ClusterNode getLocalNode() {
+        return impl != null ? impl.localNode() : null;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public ClusterNode getNode(UUID nodeId) {
+        return impl.node(nodeId);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean pingNode(UUID nodeId) {
+        return impl.pingNode(nodeId);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setNodeAttributes(Map<String, Object> attrs, IgniteProductVersion ver) {
+        assert locNodeAttrs == null;
+        assert locNodeVer == null;
+
+        if (log.isDebugEnabled()) {
+            log.debug("Node attributes to set: " + attrs);
+            log.debug("Node version to set: " + ver);
+        }
+
+        locNodeAttrs = attrs;
+        locNodeVer = ver;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setListener(@Nullable DiscoverySpiListener lsnr) {
+        this.lsnr = lsnr;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setDataExchange(DiscoverySpiDataExchange exchange) {
+        this.exchange = exchange;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setMetricsProvider(DiscoveryMetricsProvider metricsProvider) {
+        this.metricsProvider = metricsProvider;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void disconnect() throws IgniteSpiException {
+        impl.stop();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setAuthenticator(DiscoverySpiNodeAuthenticator auth) {
+        this.nodeAuth = auth;
+    }
+
+    /**
+     * @return Authenticator.
+     */
+    public DiscoverySpiNodeAuthenticator getAuthenticator() {
+        return nodeAuth;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getGridStartTime() {
+        return impl.gridStartTime();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) {
+        IgniteDiscoverySpiInternalListener internalLsnr = impl.internalLsnr;
+
+        if (internalLsnr != null) {
+            if (!internalLsnr.beforeSendCustomEvent(this, log, msg))
+                return;
+        }
+
+        impl.sendCustomMessage(msg);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void failNode(UUID nodeId, @Nullable String warning) {
+        impl.failNode(nodeId, warning);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isClientMode() throws IllegalStateException {
+        return impl.localNode().isClient();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void onContextInitialized0(IgniteSpiContext spiCtx) throws IgniteSpiException {
+        super.onContextInitialized0(spiCtx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void spiStart(@Nullable String igniteInstanceName) throws IgniteSpiException {
+        if (sesTimeout == 0)
+            sesTimeout = ignite.configuration().getFailureDetectionTimeout().intValue();
+
+        assertParameter(sesTimeout > 0, "sessionTimeout > 0");
+
+        A.notNullOrEmpty(zkConnectionString, "zkConnectionString can not be empty");
+
+        A.notNullOrEmpty(zkRootPath, "zkRootPath can not be empty");
+
+        zkRootPath = zkRootPath.trim();
+
+        if (zkRootPath.endsWith("/"))
+            zkRootPath = zkRootPath.substring(0, zkRootPath.length() - 1);
+
+        try {
+            PathUtils.validatePath(zkRootPath);
+        }
+        catch (IllegalArgumentException e) {
+            throw new IgniteSpiException("zkRootPath is invalid: " + zkRootPath, e);
+        }
+
+        ZookeeperClusterNode locNode = initLocalNode();
+
+        if (log.isInfoEnabled()) {
+            log.info("Start Zookeeper discovery [zkConnectionString=" + zkConnectionString +
+                ", sessionTimeout=" + sesTimeout +
+                ", zkRootPath=" + zkRootPath + ']');
+        }
+
+        impl = new ZookeeperDiscoveryImpl(
+            this,
+            igniteInstanceName,
+            log,
+            zkRootPath,
+            locNode,
+            lsnr,
+            exchange,
+            internalLsnr);
+
+        try {
+            impl.startJoinAndWait();
+        }
+        catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+
+            throw new IgniteSpiException("Failed to join cluster, thread was interrupted", e);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setInternalListener(IgniteDiscoverySpiInternalListener lsnr) {
+        if (impl != null)
+            impl.internalLsnr = lsnr;
+        else
+            internalLsnr = lsnr;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void simulateNodeFailure() {
+        impl.simulateNodeFailure();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void spiStop() throws IgniteSpiException {
+        if (impl != null)
+            impl.stop();
+    }
+
+    /**
+     * @return Local node instance.
+     */
+    private ZookeeperClusterNode initLocalNode() {
+        assert ignite != null;
+
+        initAddresses();
+
+        ZookeeperClusterNode locNode = new ZookeeperClusterNode(
+            ignite.configuration().getNodeId(),
+            addrs.get1(),
+            addrs.get2(),
+            locNodeVer,
+            locNodeAttrs,
+            consistentId(),
+            sesTimeout,
+            ignite.configuration().isClientMode(),
+            metricsProvider);
+
+        locNode.local(true);
+
+        DiscoverySpiListener lsnr = this.lsnr;
+
+        if (lsnr != null)
+            lsnr.onLocalNodeInitialized(locNode);
+
+        if (log.isDebugEnabled())
+            log.debug("Local node initialized: " + locNode);
+
+        if (metricsProvider != null) {
+            locNode.setMetrics(metricsProvider.metrics());
+            locNode.setCacheMetrics(metricsProvider.cacheMetrics());
+        }
+
+        return locNode;
+    }
+
+    /**
+     * Used in tests (called via reflection).
+     *
+     * @return Copy of SPI.
+     */
+    private ZookeeperDiscoverySpi cloneSpiConfiguration() {
+        ZookeeperDiscoverySpi spi = new ZookeeperDiscoverySpi();
+
+        spi.setZkRootPath(zkRootPath);
+        spi.setZkConnectionString(zkConnectionString);
+        spi.setSessionTimeout(sesTimeout);
+        spi.setJoinTimeout(joinTimeout);
+        spi.setClientReconnectDisabled(clientReconnectDisabled);
+
+        return spi;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ZookeeperDiscoverySpi.class, this);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java
new file mode 100644
index 0000000000000..b80a9ddbf129a
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractCallabck.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+
+/**
+ *
+ */
+abstract class ZkAbstractCallabck {
+    /** */
+    final ZkRuntimeState rtState;
+
+    /** */
+    private final ZookeeperDiscoveryImpl impl;
+
+    /** */
+    private final GridSpinBusyLock busyLock;
+
+    /**
+     * @param rtState Runtime state.
+     * @param impl Discovery impl.
+     */
+    ZkAbstractCallabck(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+        this.rtState = rtState;
+        this.impl = impl;
+
+        busyLock = impl.busyLock;
+    }
+
+    /**
+     * @return {@code True} if is able to start processing.
+     */
+    final boolean onProcessStart() {
+        boolean start = rtState.errForClose == null && busyLock.enterBusy();
+
+        if (!start) {
+            assert rtState.errForClose != null;
+
+            onStartFailed();
+
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     *
+     */
+    void onStartFailed() {
+        // No-op.
+    }
+
+    /**
+     *
+     */
+    final void onProcessEnd() {
+        busyLock.leaveBusy();
+    }
+
+    /**
+     * @param e Error.
+     */
+    final void onProcessError(Throwable e) {
+        impl.onFatalError(busyLock, e);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
new file mode 100644
index 0000000000000..2292e35056041
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractChildrenCallback.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.List;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ *
+ */
+abstract class ZkAbstractChildrenCallback extends ZkAbstractCallabck implements AsyncCallback.Children2Callback {
+    /**
+     * @param rtState Runtime state.
+     * @param impl Discovery impl.
+     */
+    ZkAbstractChildrenCallback(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+        super(rtState, impl);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+        if (!onProcessStart())
+            return;
+
+        try {
+            processResult0(rc, path, ctx, children, stat);
+
+            onProcessEnd();
+        }
+        catch (Throwable e) {
+            onProcessError(e);
+        }
+    }
+
+    /**
+     * @param rc
+     * @param path
+     * @param ctx
+     * @param children
+     * @param stat
+     * @throws Exception If failed.
+     */
+    abstract void processResult0(int rc, String path, Object ctx, List<String> children, Stat stat)
+        throws Exception;
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java
new file mode 100644
index 0000000000000..9098d0520a52a
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAbstractWatcher.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+
+/**
+ *
+ */
+abstract class ZkAbstractWatcher extends ZkAbstractCallabck implements Watcher {
+    /**
+     * @param rtState Runtime state.
+     * @param impl Discovery impl.
+     */
+    ZkAbstractWatcher(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+        super(rtState, impl);
+    }
+
+    /** {@inheritDoc} */
+    @Override public final void process(WatchedEvent evt) {
+        if (!onProcessStart())
+            return;
+
+        try {
+            process0(evt);
+
+            onProcessEnd();
+        }
+        catch (Throwable e) {
+            onProcessError(e);
+        }
+    }
+
+    /**
+     * @param evt Event.
+     * @throws Exception If failed.
+     */
+    protected abstract void process0(WatchedEvent evt) throws Exception;
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAliveNodeData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAliveNodeData.java
new file mode 100644
index 0000000000000..d82437758bf83
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkAliveNodeData.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Zk Alive Node Data.
+ */
+public class ZkAliveNodeData implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    long lastProcEvt = -1;
+
+    /** */
+    transient boolean needUpdate;
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ZkAliveNodeData.class, this);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkBulkJoinContext.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkBulkJoinContext.java
new file mode 100644
index 0000000000000..a186aed526567
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkBulkJoinContext.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.T2;
+
+/**
+ *
+ */
+class ZkBulkJoinContext {
+    /** */
+    List<T2<ZkJoinedNodeEvtData, Map<Integer, Serializable>>> nodes;
+
+    /**
+     * @param nodeEvtData Node event data.
+     * @param discoData Discovery data for node.
+     */
+    void addJoinedNode(ZkJoinedNodeEvtData nodeEvtData, Map<Integer, Serializable> discoData) {
+        if (nodes == null)
+            nodes = new ArrayList<>();
+
+        nodes.add(new T2<>(nodeEvtData, discoData));
+    }
+
+    /**
+     * @return Number of joined nodes.
+     */
+    int nodes() {
+        return nodes != null ? nodes.size() : 0;
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkClusterNodes.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkClusterNodes.java
new file mode 100644
index 0000000000000..7e2ea7b6e83b3
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkClusterNodes.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ * Zk Cluster Nodes.
+ */
+public class ZkClusterNodes {
+    /** */
+    final ConcurrentSkipListMap<Long, ZookeeperClusterNode> nodesByOrder = new ConcurrentSkipListMap<>();
+
+    /** */
+    final ConcurrentSkipListMap<Long, ZookeeperClusterNode> nodesByInternalId = new ConcurrentSkipListMap<>();
+
+    /** */
+    final ConcurrentHashMap<UUID, ZookeeperClusterNode> nodesById = new ConcurrentHashMap<>();
+
+    /**
+     * @return Remote nodes.
+     */
+    public Collection<ClusterNode> remoteNodes() {
+        List<ClusterNode> nodes = new ArrayList<>();
+
+        for (ClusterNode node : nodesById.values()) {
+            if (!node.isLocal())
+                nodes.add(node);
+        }
+
+        return nodes;
+    }
+
+    /**
+     * @return Current nodes in topology.
+     */
+    @SuppressWarnings("unchecked")
+    List<ClusterNode> topologySnapshot() {
+        return new ArrayList<>((Collection)nodesByOrder.values());
+    }
+
+    /**
+     * @param node New node.
+     */
+    void addNode(ZookeeperClusterNode node) {
+        assert node.id() != null : node;
+        assert node.order() > 0 : node;
+
+        ZookeeperClusterNode old = nodesById.put(node.id(), node);
+
+        assert old == null : old;
+
+        old = nodesByOrder.put(node.order(), node);
+
+        assert old == null : old;
+
+        old = nodesByInternalId.put(node.internalId(), node);
+
+        assert old == null : old;
+    }
+
+    /**
+     * @param internalId Node internal ID.
+     * @return Removed node.
+     */
+    ZookeeperClusterNode removeNode(long internalId) {
+        ZookeeperClusterNode node = nodesByInternalId.remove(internalId);
+
+        assert node != null : internalId;
+        assert node.order() > 0 : node;
+
+        Object rvmd = nodesByOrder.remove(node.order());
+
+        assert rvmd != null;
+
+        rvmd = nodesById.remove(node.id());
+
+        assert rvmd != null;
+
+        return node;
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
new file mode 100644
index 0000000000000..9c21f13889cfd
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorNodeState.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.BitSet;
+
+/**
+ *
+ */
+class ZkCommunicationErrorNodeState implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    final BitSet commState;
+
+    /** */
+    final Exception err;
+
+    /**
+     * @param commState Communication state.
+     * @param err Error if failed get communication state..
+     */
+    ZkCommunicationErrorNodeState(BitSet commState, Exception err) {
+        assert commState != null || err != null;
+
+        this.commState = commState;
+        this.err = err;
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
new file mode 100644
index 0000000000000..accda6e5fb6c1
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.jboss.netty.util.internal.ConcurrentHashMap;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Future is created on each node when either connection error occurs or resolve communication error request
+ * received.
+ */
+class ZkCommunicationErrorProcessFuture extends GridFutureAdapter<Void> implements IgniteSpiTimeoutObject, Runnable {
+    /** */
+    private final ZookeeperDiscoveryImpl impl;
+
+    /** */
+    private final IgniteLogger log;
+
+    /** */
+    private final Map<Long, GridFutureAdapter<Boolean>> nodeFuts = new ConcurrentHashMap<>();
+
+    /** */
+    private final long endTime;
+
+    /** */
+    private final IgniteUuid id;
+
+    /** */
+    private State state;
+
+    /** */
+    private long resolveTopVer;
+
+    /** */
+    private Set<Long> resFailedNodes;
+
+    /** */
+    private Exception resErr;
+
+    /** */
+    private ZkDistributedCollectDataFuture collectResFut;
+
+    /**
+     * @param impl Discovery impl.
+     * @param timeout Timeout to wait before initiating resolve process.
+     * @return Future.
+     */
+    static ZkCommunicationErrorProcessFuture createOnCommunicationError(ZookeeperDiscoveryImpl impl, long timeout) {
+        return new ZkCommunicationErrorProcessFuture(impl, State.WAIT_TIMEOUT, timeout);
+    }
+
+    /**
+     * @param impl Discovery impl.
+     * @return Future.
+     */
+    static ZkCommunicationErrorProcessFuture createOnStartResolveRequest(ZookeeperDiscoveryImpl impl) {
+        return new ZkCommunicationErrorProcessFuture(impl, State.RESOLVE_STARTED, 0);
+    }
+
+    /**
+     * @param impl Discovery implementation.
+     * @param state Initial state.
+     * @param timeout Wait timeout before initiating communication errors resolve.
+     */
+    private ZkCommunicationErrorProcessFuture(ZookeeperDiscoveryImpl impl, State state, long timeout) {
+        assert state != State.DONE;
+
+        this.impl = impl;
+        this.log = impl.log();
+
+        if (state == State.WAIT_TIMEOUT) {
+            assert timeout > 0 : timeout;
+
+            id = IgniteUuid.fromUuid(impl.localNode().id());
+            endTime = System.currentTimeMillis() + timeout;
+        }
+        else {
+            id = null;
+            endTime = 0;
+        }
+
+        this.state = state;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public IgniteLogger logger() {
+        return log;
+    }
+
+    /**
+     * @param collectResFut Collect nodes' communication status future.
+     */
+    void nodeResultCollectFuture(ZkDistributedCollectDataFuture collectResFut) {
+        assert this.collectResFut == null : collectResFut;
+
+        this.collectResFut = collectResFut;
+    }
+
+    /**
+     * @param top Topology.
+     * @throws Exception If failed.
+     */
+    void onTopologyChange(ZkClusterNodes top) throws Exception {
+        for (Map.Entry<Long, GridFutureAdapter<Boolean>> e : nodeFuts.entrySet()) {
+            if (!top.nodesByOrder.containsKey(e.getKey()))
+                e.getValue().onDone(false);
+        }
+
+        if (collectResFut != null)
+            collectResFut.onTopologyChange(top);
+    }
+
+    /**
+     * @param rtState Runtime state.
+     * @param futPath Future path.
+     * @param nodes Nodes to ping.
+     */
+    void checkConnection(final ZkRuntimeState rtState, final String futPath, List<ClusterNode> nodes) {
+        final TcpCommunicationSpi spi = (TcpCommunicationSpi)impl.spi.ignite().configuration().getCommunicationSpi();
+
+        IgniteFuture<BitSet> fut = spi.checkConnection(nodes);
+
+        fut.listen(new IgniteInClosure<IgniteFuture<BitSet>>() {
+            @Override public void apply(final IgniteFuture<BitSet> fut) {
+                // Future completed either from NIO thread or timeout worker, save result from another thread.
+                impl.runInWorkerThread(new ZkRunnable(rtState, impl) {
+                    @Override public void run0() throws Exception {
+                        BitSet commState = null;
+                        Exception err = null;
+
+                        try {
+                            commState = fut.get();
+                        }
+                        catch (Exception e) {
+                            err = e;
+                        }
+
+                        ZkCommunicationErrorNodeState state = new ZkCommunicationErrorNodeState(commState, err);
+
+                        ZkDistributedCollectDataFuture.saveNodeResult(futPath,
+                            rtState.zkClient,
+                            impl.localNode().order(),
+                            impl.marshalZip(state));
+                    }
+
+                    @Override void onStartFailed() {
+                        onError(rtState.errForClose);
+                    }
+                });
+
+            }
+        });
+    }
+
+    /**
+     *
+     */
+    void scheduleCheckOnTimeout() {
+        synchronized (this) {
+            if (state == State.WAIT_TIMEOUT)
+                impl.spi.getSpiContext().addTimeoutObject(this);
+        }
+    }
+
+    /**
+     * @param topVer Topology version.
+     * @return {@code False} if future was already completed and need create another future instance.
+     */
+    boolean onStartResolveRequest(long topVer) {
+        synchronized (this) {
+            if (state == State.DONE)
+                return false;
+
+            if (state == State.WAIT_TIMEOUT)
+                impl.spi.getSpiContext().removeTimeoutObject(this);
+
+            assert resolveTopVer == 0 : resolveTopVer;
+
+            resolveTopVer = topVer;
+
+            state = State.RESOLVE_STARTED;
+        }
+
+        return true;
+    }
+
+    /**
+     * @param err Error.
+     */
+    void onError(Exception err) {
+        assert err != null;
+
+        Map<Long, GridFutureAdapter<Boolean>> futs;
+
+        synchronized (this) {
+            if (state == State.DONE) {
+                assert resErr != null;
+
+                return;
+            }
+
+            state = State.DONE;
+
+            resErr = err;
+
+            futs = nodeFuts; // nodeFuts should not be modified after state changed to DONE.
+        }
+
+        for (Map.Entry<Long, GridFutureAdapter<Boolean>> e : futs.entrySet())
+            e.getValue().onDone(err);
+
+        onDone(err);
+    }
+
+    /**
+     * @param failedNodes Node failed as result of resolve process.
+     */
+    void onFinishResolve(Set<Long> failedNodes) {
+        Map<Long, GridFutureAdapter<Boolean>> futs;
+
+        synchronized (this) {
+            if (state == State.DONE) {
+                assert resErr != null;
+
+                return;
+            }
+
+            assert state == State.RESOLVE_STARTED : state;
+
+            state = State.DONE;
+
+            resFailedNodes = failedNodes;
+
+            futs = nodeFuts; // nodeFuts should not be modified after state changed to DONE.
+        }
+
+        for (Map.Entry<Long, GridFutureAdapter<Boolean>> e : futs.entrySet()) {
+            Boolean res = !F.contains(resFailedNodes, e.getKey());
+
+            e.getValue().onDone(res);
+        }
+
+        onDone();
+    }
+
+    /**
+     * @param node Node.
+     * @return Future finished when communication error resolve is done or {@code null} if another
+     *      resolve process should be started.
+     */
+    @Nullable IgniteInternalFuture<Boolean> nodeStatusFuture(ClusterNode node) {
+        GridFutureAdapter<Boolean> fut;
+
+        synchronized (this) {
+            if (state == State.DONE) {
+                if (resolveTopVer != 0 && node.order() <= resolveTopVer) {
+                    Boolean res = !F.contains(resFailedNodes, node.order());
+
+                    return new GridFinishedFuture<>(res);
+                }
+                else
+                    return null;
+            }
+
+            fut = nodeFuts.get(node.order());
+
+            if (fut == null)
+                nodeFuts.put(node.order(), fut = new GridFutureAdapter<>());
+        }
+
+        if (impl.node(node.order()) == null)
+            fut.onDone(false);
+
+        return fut;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void run() {
+        // Run from zk discovery worker pool after timeout.
+        if (needProcessTimeout()) {
+            try {
+                UUID reqId = UUID.randomUUID();
+
+                if (log.isInfoEnabled()) {
+                    log.info("Initiate cluster-wide communication error resolve process [reqId=" + reqId +
+                        ", errNodes=" + nodeFuts.size() + ']');
+                }
+
+                impl.sendCustomMessage(new ZkCommunicationErrorResolveStartMessage(reqId));
+            }
+            catch (Exception e) {
+                Collection<GridFutureAdapter<Boolean>> futs;
+
+                synchronized (this) {
+                    if (state != State.WAIT_TIMEOUT)
+                        return;
+
+                    state = State.DONE;
+                    resErr = e;
+
+                    futs = nodeFuts.values(); // nodeFuts should not be modified after state changed to DONE.
+                }
+
+                for (GridFutureAdapter<Boolean> fut : futs)
+                    fut.onDone(e);
+
+                onDone(e);
+            }
+        }
+    }
+
+    /**
+     * @return {@code True} if need initiate resolve process after timeout expired.
+     */
+    private boolean needProcessTimeout() {
+        synchronized (this) {
+            if (state != State.WAIT_TIMEOUT)
+                return false;
+
+            for (GridFutureAdapter<Boolean> fut : nodeFuts.values()) {
+                if (!fut.isDone())
+                    return true;
+            }
+
+            state = State.DONE;
+        }
+
+        onDone(null, null);
+
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteUuid id() {
+        return id;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long endTime() {
+        return endTime;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onTimeout() {
+        if (needProcessTimeout())
+            impl.runInWorkerThread(this);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean onDone(@Nullable Void res, @Nullable Throwable err) {
+        if (super.onDone(res, err)) {
+            impl.clearCommunicationErrorProcessFuture(this);
+
+            return true;
+        }
+
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ZkCommunicationErrorProcessFuture.class, this);
+    }
+
+    /**
+     *
+     */
+    enum State {
+        /** */
+        DONE,
+
+        /** */
+        WAIT_TIMEOUT,
+
+        /** */
+        RESOLVE_STARTED
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
new file mode 100644
index 0000000000000..9b7476c5355f5
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveFinishMessage.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkCommunicationErrorResolveFinishMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    final UUID futId;
+
+    /** */
+    final long topVer;
+
+    /** */
+    transient ZkCommunicationErrorResolveResult res;
+
+    /**
+     * @param futId Future ID.
+     * @param topVer Topology version when resolve process finished.
+     */
+    ZkCommunicationErrorResolveFinishMessage(UUID futId, long topVer) {
+        this.futId = futId;
+        this.topVer = topVer;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMutable() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ZkCommunicationErrorResolveFinishMessage.class, this);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
new file mode 100644
index 0000000000000..23495aae72df1
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveResult.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import org.apache.ignite.internal.util.GridLongList;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkCommunicationErrorResolveResult implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    final GridLongList killedNodes;
+
+    /** */
+    final Exception err;
+
+    /**
+     * @param killedNodes Killed nodes.
+     * @param err Error.
+     */
+    ZkCommunicationErrorResolveResult(@Nullable GridLongList killedNodes, Exception err) {
+        this.killedNodes = killedNodes;
+        this.err = err;
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
new file mode 100644
index 0000000000000..0c79c36aee0a9
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorResolveStartMessage.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Zk Communication Error Resolve Start Message.
+ */
+public class ZkCommunicationErrorResolveStartMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    final UUID id;
+
+    /**
+     * @param id Unique ID.
+     */
+    ZkCommunicationErrorResolveStartMessage(UUID id) {
+        this.id = id;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMutable() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ZkCommunicationErrorResolveStartMessage.class, this);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java
new file mode 100644
index 0000000000000..d27b717485e2c
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.CommunicationFailureContext;
+import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+class ZkCommunicationFailureContext implements CommunicationFailureContext {
+    /** */
+    private static final Comparator<ClusterNode> NODE_ORDER_CMP = new Comparator<ClusterNode>() {
+        @Override public int compare(ClusterNode node1, ClusterNode node2) {
+            return Long.compare(node1.order(), node2.order());
+        }
+    };
+
+    /** */
+    private Set<ClusterNode> killedNodes = new HashSet<>();
+
+    /** */
+    private final Map<UUID, BitSet> nodesState;
+
+    /** */
+    private final List<ClusterNode> initialNodes;
+
+    /** */
+    private final List<ClusterNode> curNodes;
+
+    /** */
+    private final GridCacheSharedContext<?, ?> ctx;
+
+    /**
+     * @param ctx Context.
+     * @param curNodes Current topology snapshot.
+     * @param initialNodes Topology snapshot when communication error resolve started.
+     * @param nodesState Nodes communication state.
+     */
+    ZkCommunicationFailureContext(
+        GridCacheSharedContext<?, ?> ctx,
+        List<ClusterNode> curNodes,
+        List<ClusterNode> initialNodes,
+        Map<UUID, BitSet> nodesState)
+    {
+        this.ctx = ctx;
+        this.curNodes = Collections.unmodifiableList(curNodes);
+        this.initialNodes = initialNodes;
+        this.nodesState = nodesState;
+    }
+
+    /** {@inheritDoc} */
+    @Override public List<ClusterNode> topologySnapshot() {
+        return curNodes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean connectionAvailable(ClusterNode node1, ClusterNode node2) {
+        BitSet nodeState = nodesState.get(node1.id());
+
+        if (nodeState == null)
+            throw new IllegalArgumentException("Invalid node: " + node1);
+
+        int nodeIdx = Collections.binarySearch(initialNodes, node2, NODE_ORDER_CMP);
+
+        if (nodeIdx < 0)
+            throw new IllegalArgumentException("Invalid node: " + node2);
+
+        assert nodeIdx < nodeState.size() : nodeIdx;
+
+        return nodeState.get(nodeIdx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map<String, CacheConfiguration<?, ?>> startedCaches() {
+        Map<Integer, DynamicCacheDescriptor> cachesMap = ctx.affinity().caches();
+
+        Map<String, CacheConfiguration<?, ?>> res = U.newHashMap(cachesMap.size());
+
+        for (DynamicCacheDescriptor desc : cachesMap.values()) {
+            if (desc.cacheType().userCache())
+                res.put(desc.cacheName(), desc.cacheConfiguration());
+        }
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public List<List<ClusterNode>> cacheAffinity(String cacheName) {
+        if (cacheName == null)
+            throw new NullPointerException("Null cache name.");
+
+        DynamicCacheDescriptor cacheDesc = ctx.affinity().caches().get(CU.cacheId(cacheName));
+
+        if (cacheDesc == null)
+            throw new IllegalArgumentException("Invalid cache name: " + cacheName);
+
+        GridAffinityAssignmentCache aff = ctx.affinity().groupAffinity(cacheDesc.groupId());
+
+        assert aff != null : cacheName;
+
+        return aff.readyAssignments(aff.lastVersion());
+    }
+
+    /** {@inheritDoc} */
+    @Override public List<List<ClusterNode>> cachePartitionOwners(String cacheName) {
+        if (cacheName == null)
+            throw new NullPointerException("Null cache name.");
+
+        DynamicCacheDescriptor cacheDesc = ctx.affinity().caches().get(CU.cacheId(cacheName));
+
+        if (cacheDesc == null)
+            throw new IllegalArgumentException("Invalid cache name: " + cacheName);
+
+        if (cacheDesc.cacheConfiguration().getCacheMode() == CacheMode.LOCAL)
+            return Collections.emptyList();
+
+        CacheGroupContext grp = ctx.cache().cacheGroup(cacheDesc.groupId());
+
+        GridDhtPartitionTopology top;
+
+        if (grp == null) {
+            top = ctx.exchange().clientTopologyIfExists(cacheDesc.groupId());
+
+            assert top != null : cacheName;
+        }
+        else
+            top = grp.topology();
+
+        return top.allOwners();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void killNode(ClusterNode node) {
+        if (node == null)
+            throw new NullPointerException();
+
+        if (Collections.binarySearch(curNodes, node, NODE_ORDER_CMP) < 0)
+            throw new IllegalArgumentException("Invalid node: " + node);
+
+        killedNodes.add(node);
+    }
+
+    /**
+     * @return Nodes to fail.
+     */
+    Set<ClusterNode> killedNodes() {
+        return killedNodes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "ZkCommunicationFailureContext []";
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java
new file mode 100644
index 0000000000000..21dfe628e4ff9
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryCustomEventData.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+
+/**
+ *
+ */
+class ZkDiscoveryCustomEventData extends ZkDiscoveryEventData {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    final long origEvtId;
+
+    /** */
+    final UUID sndNodeId;
+
+    /** */
+    final String evtPath;
+
+    /** Message instance (can be marshalled as part of ZkDiscoveryCustomEventData or stored in separate znode. */
+    DiscoverySpiCustomMessage msg;
+
+    /** Unmarshalled message. */
+    transient DiscoverySpiCustomMessage resolvedMsg;
+
+    /**
+     * @param evtId Event ID.
+     * @param origEvtId For acknowledge events ID of original event.
+     * @param topVer Topology version.
+     * @param sndNodeId Sender node ID.
+     * @param msg Message instance.
+     * @param evtPath Event path.
+     */
+    ZkDiscoveryCustomEventData(
+        long evtId,
+        long origEvtId,
+        long topVer,
+        UUID sndNodeId,
+        DiscoverySpiCustomMessage msg,
+        String evtPath)
+    {
+        super(evtId, ZK_EVT_CUSTOM_EVT, topVer);
+
+        assert sndNodeId != null;
+        assert msg != null || origEvtId != 0 || !F.isEmpty(evtPath);
+
+        this.origEvtId = origEvtId;
+        this.msg = msg;
+        this.sndNodeId = sndNodeId;
+        this.evtPath = evtPath;
+    }
+
+    /**
+     * @return {@code True} for custom event ack message.
+     */
+    boolean ackEvent() {
+        return origEvtId != 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "ZkDiscoveryCustomEventData [" +
+            "evtId=" + eventId() +
+            ", topVer=" + topologyVersion() +
+            ", sndNode=" + sndNodeId +
+            ", ack=" + ackEvent() +
+            ']';
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventData.java
new file mode 100644
index 0000000000000..d667a17f6643c
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventData.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Set;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+abstract class ZkDiscoveryEventData implements Serializable {
+    /** */
+    static final byte ZK_EVT_NODE_JOIN = 1;
+
+    /** */
+    static final byte ZK_EVT_NODE_FAILED = 2;
+
+    /** */
+    static final byte ZK_EVT_CUSTOM_EVT = 3;
+
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final long evtId;
+
+    /** */
+    private final byte evtType;
+
+    /** */
+    private final long topVer;
+
+    /** */
+    private transient Set<Long> remainingAcks;
+
+    /** */
+    int flags;
+
+    /**
+     * @param evtId Event ID.
+     * @param evtType Event type.
+     * @param topVer Topology version.
+     */
+    ZkDiscoveryEventData(long evtId, byte evtType, long topVer) {
+        assert evtType == ZK_EVT_NODE_JOIN || evtType == ZK_EVT_NODE_FAILED || evtType == ZK_EVT_CUSTOM_EVT : evtType;
+
+        this.evtId = evtId;
+        this.evtType = evtType;
+        this.topVer = topVer;
+    }
+
+    /**
+     * @param nodes Current nodes in topology.
+     */
+    void initRemainingAcks(Collection<ZookeeperClusterNode> nodes) {
+        assert remainingAcks == null : this;
+
+        remainingAcks = U.newHashSet(nodes.size());
+
+        for (ZookeeperClusterNode node : nodes) {
+            if (!node.isLocal() && node.order() <= topVer) {
+                boolean add = remainingAcks.add(node.internalId());
+
+                assert add : node;
+            }
+        }
+    }
+
+    /**
+     * @param node Node.
+     */
+    void addRemainingAck(ZookeeperClusterNode node) {
+        assert node.order() <= topVer : node;
+
+        boolean add = remainingAcks.add(node.internalId());
+
+        assert add : node;
+    }
+
+    /**
+     * @return {@code True} if all nodes processed event.
+     */
+    boolean allAcksReceived() {
+        return remainingAcks.isEmpty();
+    }
+
+    /**
+     * @return Remaining acks.
+     */
+    Set<Long> remainingAcks() {
+        return remainingAcks;
+    }
+
+    /**
+     * @param nodeInternalId Node ID.
+     * @param ackEvtId Last event ID processed on node.
+     * @return {@code True} if all nodes processed event.
+     */
+    boolean onAckReceived(Long nodeInternalId, long ackEvtId) {
+        assert remainingAcks != null;
+
+        if (ackEvtId >= evtId)
+            remainingAcks.remove(nodeInternalId);
+
+        return remainingAcks.isEmpty();
+    }
+
+    /**
+     * @param node Failed node.
+     * @return {@code True} if all nodes processed event.
+     */
+    boolean onNodeFail(ZookeeperClusterNode node) {
+        assert remainingAcks != null : this;
+
+        remainingAcks.remove(node.internalId());
+
+        return remainingAcks.isEmpty();
+    }
+
+    /**
+     * @param flag Flag mask.
+     * @return {@code True} if flag set.
+     */
+    boolean flagSet(int flag) {
+        return (flags & flag) == flag;
+    }
+
+    /**
+     * @return Event ID.
+     */
+    long eventId() {
+        return evtId;
+    }
+
+    /**
+     * @return Event type.
+     */
+    byte eventType() {
+        return evtType;
+    }
+
+    /**
+     * @return Event topology version.
+     */
+    long topologyVersion() {
+        return topVer;
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
new file mode 100644
index 0000000000000..dce861b523e4c
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.TreeMap;
+import java.util.UUID;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkDiscoveryEventsData implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Unique cluster ID (generated when first node in cluster starts). */
+    final UUID clusterId;
+
+    /** Internal order of last processed custom event. */
+    long procCustEvt = -1;
+
+    /** Event ID counter. */
+    long evtIdGen;
+
+    /** Current topology version. */
+    long topVer;
+
+    /** Max node internal order in cluster. */
+    long maxInternalOrder;
+
+    /** Cluster start time (recorded when first node in cluster starts). */
+    final long clusterStartTime;
+
+    /** Events to process. */
+    final TreeMap<Long, ZkDiscoveryEventData> evts;
+
+    /** ID of current active communication error resolve process. */
+    private UUID commErrFutId;
+
+    /**
+     * @param clusterStartTime Start time of first node in cluster.
+     * @return Events.
+     */
+    static ZkDiscoveryEventsData createForNewCluster(long clusterStartTime) {
+        return new ZkDiscoveryEventsData(
+            UUID.randomUUID(),
+            clusterStartTime,
+            1L,
+            new TreeMap<Long, ZkDiscoveryEventData>()
+        );
+    }
+
+    /**
+     * @param clusterId Cluster ID.
+     * @param topVer Current topology version.
+     * @param clusterStartTime Cluster start time.
+     * @param evts Events history.
+     */
+    private ZkDiscoveryEventsData(
+        UUID clusterId,
+        long clusterStartTime,
+        long topVer,
+        TreeMap<Long, ZkDiscoveryEventData> evts)
+    {
+        this.clusterId = clusterId;
+        this.clusterStartTime = clusterStartTime;
+        this.topVer = topVer;
+        this.evts = evts;
+    }
+
+    /**
+     * @param node Joined node.
+     */
+    void onNodeJoin(ZookeeperClusterNode node) {
+        if (node.internalId() > maxInternalOrder)
+            maxInternalOrder = node.internalId();
+    }
+
+    /**
+     * @return Future ID.
+     */
+    @Nullable UUID communicationErrorResolveFutureId() {
+        return commErrFutId;
+    }
+
+    /**
+     * @param id Future ID.
+     */
+     void communicationErrorResolveFutureId(@Nullable UUID id) {
+        commErrFutId = id;
+    }
+
+    /**
+     * @param nodes Current nodes in topology (these nodes should ack that event processed).
+     * @param evt Event.
+     */
+    void addEvent(Collection<ZookeeperClusterNode> nodes, ZkDiscoveryEventData evt) {
+        Object old = evts.put(evt.eventId(), evt);
+
+        assert old == null : old;
+
+        evt.initRemainingAcks(nodes);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeFailEventData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeFailEventData.java
new file mode 100644
index 0000000000000..c76158ff090a7
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeFailEventData.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+class ZkDiscoveryNodeFailEventData extends ZkDiscoveryEventData {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private long failedNodeInternalId;
+
+    /**
+     * @param evtId Event ID.
+     * @param topVer Topology version.
+     * @param failedNodeInternalId Failed node ID.
+     */
+    ZkDiscoveryNodeFailEventData(long evtId, long topVer, long failedNodeInternalId) {
+        super(evtId, ZK_EVT_NODE_FAILED, topVer);
+
+        this.failedNodeInternalId = failedNodeInternalId;
+    }
+
+    /**
+     * @return Failed node ID.
+     */
+    long failedNodeInternalId() {
+        return failedNodeInternalId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "ZkDiscoveryNodeFailEventData [" +
+            "evtId=" + eventId() +
+            ", topVer=" + topologyVersion() +
+            ", nodeId=" + failedNodeInternalId + ']';
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeJoinEventData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeJoinEventData.java
new file mode 100644
index 0000000000000..e46d52dcd019e
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryNodeJoinEventData.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.List;
+
+/**
+ *
+ */
+class ZkDiscoveryNodeJoinEventData extends ZkDiscoveryEventData {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    final List<ZkJoinedNodeEvtData> joinedNodes;
+
+    /** */
+    final int dataForJoinedPartCnt;
+
+    /**
+     * @param evtId Event ID.
+     * @param topVer Topology version.
+     * @param joinedNodes Joined nodes data.
+     * @param dataForJoinedPartCnt Data for joined part count.
+     */
+    ZkDiscoveryNodeJoinEventData(
+        long evtId,
+        long topVer,
+        List<ZkJoinedNodeEvtData> joinedNodes,
+        int dataForJoinedPartCnt)
+    {
+        super(evtId, ZK_EVT_NODE_JOIN, topVer);
+
+        this.joinedNodes = joinedNodes;
+        this.dataForJoinedPartCnt = dataForJoinedPartCnt;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "ZkDiscoveryNodeJoinEventData [" +
+            "evtId=" + eventId() +
+            ", topVer=" + topologyVersion() +
+            ", nodes=" + joinedNodes + ']';
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
new file mode 100644
index 0000000000000..174d698fe27e4
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ *
+ */
+class ZkDistributedCollectDataFuture extends GridFutureAdapter<Void> {
+    /** */
+    private final IgniteLogger log;
+
+    /** */
+    private final String futPath;
+
+    /** */
+    private final Set<Long> remainingNodes;
+
+    /** */
+    private final Callable<Void> lsnr;
+
+    /**
+     * @param impl Disovery impl
+     * @param rtState Runtime state.
+     * @param futPath Future path.
+     * @param lsnr Future listener.
+     * @throws Exception If listener call failed.
+     */
+    ZkDistributedCollectDataFuture(
+        ZookeeperDiscoveryImpl impl,
+        ZkRuntimeState rtState,
+        String futPath,
+        Callable<Void> lsnr)
+        throws Exception
+    {
+        this.log = impl.log();
+        this.futPath = futPath;
+        this.lsnr = lsnr;
+
+        ZkClusterNodes top = rtState.top;
+
+        // Assume new nodes can not join while future is in progress.
+
+        remainingNodes = U.newHashSet(top.nodesByOrder.size());
+
+        for (ZookeeperClusterNode node : top.nodesByInternalId.values())
+            remainingNodes.add(node.order());
+
+        NodeResultsWatcher watcher = new NodeResultsWatcher(rtState, impl);
+
+        if (remainingNodes.isEmpty())
+            completeAndNotifyListener();
+        else {
+            if (log.isInfoEnabled()) {
+                log.info("Initialize data collect future [futPath=" + futPath + ", " +
+                    "remainingNodes=" + remainingNodes.size() + ']');
+            }
+
+            rtState.zkClient.getChildrenAsync(futPath, watcher, watcher);
+        }
+    }
+
+    /**
+     * @throws Exception If listener call failed.
+     */
+    private void completeAndNotifyListener() throws Exception {
+        if (super.onDone())
+            lsnr.call();
+    }
+
+    /**
+     * @param futPath
+     * @param client
+     * @param nodeOrder
+     * @param data
+     * @throws Exception If failed.
+     */
+    static void saveNodeResult(String futPath, ZookeeperClient client, long nodeOrder, byte[] data) throws Exception {
+        client.createIfNeeded(futPath + "/" + nodeOrder, data, CreateMode.PERSISTENT);
+    }
+
+    /**
+     * @param futPath
+     * @param client
+     * @param nodeOrder
+     * @return Node result data.
+     * @throws Exception If fai.ed
+     */
+    static byte[] readNodeResult(String futPath, ZookeeperClient client, long nodeOrder) throws Exception {
+        return client.getData(futPath + "/" + nodeOrder);
+    }
+
+    /**
+     * @param futResPath Result path.
+     * @param client Client.
+     * @param data Result data.
+     * @throws Exception If failed.
+     */
+    static void saveResult(String futResPath, ZookeeperClient client, byte[] data) throws Exception {
+        client.createIfNeeded(futResPath, data, CreateMode.PERSISTENT);
+    }
+
+    static byte[] readResult(ZookeeperClient client, ZkIgnitePaths paths, UUID futId) throws Exception {
+        return client.getData(paths.distributedFutureResultPath(futId));
+    }
+
+    /**
+     * @param client Client.
+     * @param paths Paths utils.
+     * @param futId Future ID.
+     * @param log Ignite Logger.
+     * @throws Exception If failed.
+     */
+    static void deleteFutureData(ZookeeperClient client,
+        ZkIgnitePaths paths,
+        UUID futId,
+        IgniteLogger log
+    ) throws Exception {
+        // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8189
+        String evtDir = paths.distributedFutureBasePath(futId);
+
+        try {
+            client.deleteAll(evtDir,
+                client.getChildren(evtDir),
+                -1);
+        }
+        catch (KeeperException.NoNodeException e) {
+            U.log(log, "Node for deletion was not found: " + e.getPath());
+
+            // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8189
+        }
+
+        client.deleteIfExists(evtDir, -1);
+
+        client.deleteIfExists(paths.distributedFutureResultPath(futId), -1);
+    }
+
+    /**
+     * @param top Current topology.
+     * @throws Exception If listener call failed.
+     */
+    void onTopologyChange(ZkClusterNodes top) throws Exception {
+        if (remainingNodes.isEmpty())
+            return;
+
+        for (Iterator<Long> it = remainingNodes.iterator(); it.hasNext();) {
+            Long nodeOrder = it.next();
+
+            if (!top.nodesByOrder.containsKey(nodeOrder)) {
+                it.remove();
+
+                int remaining = remainingNodes.size();
+
+                if (log.isInfoEnabled()) {
+                    log.info("ZkDistributedCollectDataFuture removed remaining failed node [node=" + nodeOrder +
+                        ", remaining=" + remaining +
+                        ", futPath=" + futPath + ']');
+                }
+
+                if (remaining == 0) {
+                    completeAndNotifyListener();
+
+                    break;
+                }
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    class NodeResultsWatcher extends ZkAbstractWatcher implements AsyncCallback.Children2Callback {
+        /**
+         * @param rtState Runtime state.
+         * @param impl Discovery impl.
+         */
+        NodeResultsWatcher(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+            super(rtState, impl);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void process0(WatchedEvent evt) {
+            if (evt.getType() == Watcher.Event.EventType.NodeChildrenChanged)
+                rtState.zkClient.getChildrenAsync(evt.getPath(), this, this);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+            if (!onProcessStart())
+                return;
+
+            try {
+                if (!isDone()) {
+                    assert rc == 0 : KeeperException.Code.get(rc);
+
+                    for (int i = 0; i < children.size(); i++) {
+                        Long nodeOrder = Long.parseLong(children.get(i));
+
+                        if (remainingNodes.remove(nodeOrder)) {
+                            int remaining = remainingNodes.size();
+
+                            if (log.isInfoEnabled()) {
+                                log.info("ZkDistributedCollectDataFuture added new result [node=" + nodeOrder +
+                                    ", remaining=" + remaining +
+                                    ", futPath=" + path + ']');
+                            }
+
+                            if (remaining == 0)
+                                completeAndNotifyListener();
+                        }
+                    }
+                }
+
+                onProcessEnd();
+            }
+            catch (Throwable e) {
+                onProcessError(e);
+            }
+        }
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java
new file mode 100644
index 0000000000000..de7291c0d453c
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkForceNodeFailMessage.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Zk Force Node Fail Message.
+ */
+public class ZkForceNodeFailMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    final long nodeInternalId;
+
+    /** */
+    final String warning;
+
+    /**
+     * @param nodeInternalId Node ID.
+     * @param warning Warning to be displayed on all nodes.
+     */
+    ZkForceNodeFailMessage(long nodeInternalId, String warning) {
+        this.nodeInternalId = nodeInternalId;
+        this.warning = warning;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMutable() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ZkForceNodeFailMessage.class, this);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
new file mode 100644
index 0000000000000..9caf00fb64a93
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkIgnitePaths.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.UUID;
+
+/**
+ *
+ */
+class ZkIgnitePaths {
+    /** */
+    static final String PATH_SEPARATOR = "/";
+
+    /** */
+    private static final byte CLIENT_NODE_FLAG_MASK = 0x01;
+
+    /** */
+    private static final int UUID_LEN = 36;
+
+    /** Directory to store joined node data. */
+    private static final String JOIN_DATA_DIR = "jd";
+
+    /** Directory to store new custom events. */
+    private static final String CUSTOM_EVTS_DIR = "ce";
+
+    /** Directory to store parts of multi-parts custom events. */
+    private static final String CUSTOM_EVTS_PARTS_DIR = "cp";
+
+    /** Directory to store acknowledge messages for custom events. */
+    private static final String CUSTOM_EVTS_ACKS_DIR = "ca";
+
+    /** Directory to store EPHEMERAL znodes for alive cluster nodes. */
+    static final String ALIVE_NODES_DIR = "n";
+
+    /** Path to store discovery events {@link ZkDiscoveryEventsData}. */
+    private static final String DISCO_EVENTS_PATH = "e";
+
+    /** */
+    final String clusterDir;
+
+    /** */
+    final String aliveNodesDir;
+
+    /** */
+    final String joinDataDir;
+
+    /** */
+    final String evtsPath;
+
+    /** */
+    final String customEvtsDir;
+
+    /** */
+    final String customEvtsPartsDir;
+
+    /** */
+    final String customEvtsAcksDir;
+
+    /**
+     * @param zkRootPath Base Zookeeper directory for all Ignite nodes.
+     */
+    ZkIgnitePaths(String zkRootPath) {
+        clusterDir = zkRootPath;
+
+        aliveNodesDir = zkPath(ALIVE_NODES_DIR);
+        joinDataDir = zkPath(JOIN_DATA_DIR);
+        evtsPath = zkPath(DISCO_EVENTS_PATH);
+        customEvtsDir = zkPath(CUSTOM_EVTS_DIR);
+        customEvtsPartsDir = zkPath(CUSTOM_EVTS_PARTS_DIR);
+        customEvtsAcksDir = zkPath(CUSTOM_EVTS_ACKS_DIR);
+    }
+
+    /**
+     * @param path Relative path.
+     * @return Full path.
+     */
+    private String zkPath(String path) {
+        return clusterDir + "/" + path;
+    }
+
+    /**
+     * @param nodeId Node ID.
+     * @param prefixId Unique prefix ID.
+     * @return Path.
+     */
+    String joiningNodeDataPath(UUID nodeId, UUID prefixId) {
+        return joinDataDir + '/' + prefixId + ":" + nodeId.toString();
+    }
+
+    /**
+     * @param path Alive node zk path.
+     * @return Node internal ID.
+     */
+    static long aliveInternalId(String path) {
+        int idx = path.lastIndexOf('|');
+
+        return Integer.parseInt(path.substring(idx + 1));
+    }
+
+    /**
+     * @param prefix Node unique path prefix.
+     * @param node Node.
+     * @return Path.
+     */
+    String aliveNodePathForCreate(String prefix, ZookeeperClusterNode node) {
+        byte flags = 0;
+
+        if (node.isClient())
+            flags |= CLIENT_NODE_FLAG_MASK;
+
+        return aliveNodesDir + "/" + prefix + ":" + node.id() + ":" + encodeFlags(flags) + "|";
+    }
+
+    /**
+     * @param path Alive node zk path.
+     * @return {@code True} if node is client.
+     */
+    static boolean aliveNodeClientFlag(String path) {
+        return (aliveFlags(path) & CLIENT_NODE_FLAG_MASK) != 0;
+    }
+
+    /**
+     * @param path Alive node zk path.
+     * @return Node ID.
+     */
+    static UUID aliveNodePrefixId(String path) {
+        return UUID.fromString(path.substring(0, ZkIgnitePaths.UUID_LEN));
+    }
+
+    /**
+     * @param path Alive node zk path.
+     * @return Node ID.
+     */
+    static UUID aliveNodeId(String path) {
+        // <uuid prefix>:<node id>:<flags>|<alive seq>
+        int startIdx = ZkIgnitePaths.UUID_LEN + 1;
+
+        String idStr = path.substring(startIdx, startIdx + ZkIgnitePaths.UUID_LEN);
+
+        return UUID.fromString(idStr);
+    }
+
+    /**
+     * @param path Event zk path.
+     * @return Event sequence number.
+     */
+    static int customEventSequence(String path) {
+        int idx = path.lastIndexOf('|');
+
+        return Integer.parseInt(path.substring(idx + 1));
+    }
+
+    /**
+     * @param path Custom event zl path.
+     * @return Event node ID.
+     */
+    static UUID customEventSendNodeId(String path) {
+        // <uuid prefix>:<node id>:<partCnt>|<seq>
+        int startIdx = ZkIgnitePaths.UUID_LEN + 1;
+
+        String idStr = path.substring(startIdx, startIdx + ZkIgnitePaths.UUID_LEN);
+
+        return UUID.fromString(idStr);
+    }
+
+    /**
+     * @param path Event path.
+     * @return Event unique prefix.
+     */
+    static String customEventPrefix(String path) {
+        // <uuid prefix>:<node id>:<partCnt>|<seq>
+
+        return path.substring(0, ZkIgnitePaths.UUID_LEN);
+    }
+
+    /**
+     * @param path Custom event zl path.
+     * @return Event node ID.
+     */
+    static int customEventPartsCount(String path) {
+        // <uuid prefix>:<node id>:<partCnt>|<seq>
+        int startIdx = 2 * ZkIgnitePaths.UUID_LEN + 2;
+
+        String cntStr = path.substring(startIdx, startIdx + 4);
+
+        int partCnt = Integer.parseInt(cntStr);
+
+        assert partCnt >= 1 : partCnt;
+
+        return partCnt;
+    }
+
+    /**
+     * @param prefix Prefix.
+     * @param nodeId Node ID.
+     * @param partCnt Parts count.
+     * @return Path.
+     */
+    String createCustomEventPath(String prefix, UUID nodeId, int partCnt) {
+        return customEvtsDir + "/" + prefix + ":" + nodeId + ":" + String.format("%04d", partCnt) + '|';
+    }
+
+    /**
+     * @param prefix Prefix.
+     * @param nodeId Node ID.
+     * @return Path.
+     */
+    String customEventPartsBasePath(String prefix, UUID nodeId) {
+        return customEvtsPartsDir + "/" + prefix + ":" + nodeId + ":";
+    }
+
+    /**
+     * @param prefix Prefix.
+     * @param nodeId Node ID.
+     * @param part Part number.
+     * @return Path.
+     */
+    String customEventPartPath(String prefix, UUID nodeId, int part) {
+        return customEventPartsBasePath(prefix, nodeId) + String.format("%04d", part);
+    }
+
+    /**
+     * @param evtId Event ID.
+     * @return Event zk path.
+     */
+    String joinEventDataPathForJoined(long evtId) {
+        return evtsPath + "/fj-" + evtId;
+    }
+
+    /**
+     * @param topVer Event topology version.
+     * @return Event zk path.
+     */
+    String joinEventSecuritySubjectPath(long topVer) {
+        return evtsPath + "/s-" + topVer;
+    }
+
+    /**
+     * @param origEvtId ID of original custom event.
+     * @return Path for custom event ack.
+     */
+    String ackEventDataPath(long origEvtId) {
+        assert origEvtId != 0;
+
+        return customEvtsAcksDir + "/" + String.valueOf(origEvtId);
+    }
+
+    /**
+     * @param id Future ID.
+     * @return Future path.
+     */
+    String distributedFutureBasePath(UUID id) {
+        return evtsPath + "/f-" + id;
+    }
+
+    /**
+     * @param id Future ID.
+     * @return Future path.
+     */
+    String distributedFutureResultPath(UUID id) {
+        return evtsPath + "/fr-" + id;
+    }
+
+    /**
+     * @param flags Flags.
+     * @return Flags string.
+     */
+    private static String encodeFlags(byte flags) {
+        int intVal = flags + 128;
+
+        String str = Integer.toString(intVal, 16);
+
+        if (str.length() == 1)
+            str = '0' + str;
+
+        assert str.length() == 2  : str;
+
+        return str;
+    }
+
+    /**
+     * @param path Alive node zk path.
+     * @return Flags.
+     */
+    private static byte aliveFlags(String path) {
+        int startIdx = path.lastIndexOf(':') + 1;
+
+        String flagsStr = path.substring(startIdx, startIdx + 2);
+
+        return (byte)(Integer.parseInt(flagsStr, 16) - 128);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalJoinErrorMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalJoinErrorMessage.java
new file mode 100644
index 0000000000000..a73312cfecf05
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalJoinErrorMessage.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+class ZkInternalJoinErrorMessage implements ZkInternalMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    transient boolean notifyNode = true;
+
+    /** */
+    final long nodeInternalId;
+
+    /** */
+    final String err;
+
+    /**
+     * @param nodeInternalId Joining node internal ID.
+     * @param err Error message.
+     */
+    ZkInternalJoinErrorMessage(long nodeInternalId, String err) {
+        this.nodeInternalId = nodeInternalId;
+        this.err = err;
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalMessage.java
new file mode 100644
index 0000000000000..c1d56f0eedabd
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkInternalMessage.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+
+/**
+ *
+ */
+interface ZkInternalMessage extends Serializable {
+    // No-op.
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinEventDataForJoined.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinEventDataForJoined.java
new file mode 100644
index 0000000000000..e4ae4ba0aeb95
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinEventDataForJoined.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkJoinEventDataForJoined implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final List<ZookeeperClusterNode> top;
+
+    /** */
+    private final Map<Long, byte[]> discoData;
+
+    /** */
+    private final Map<Long, Long> dupDiscoData;
+
+    /**
+     * @param top Topology.
+     * @param discoData Discovery data.
+     */
+    ZkJoinEventDataForJoined(List<ZookeeperClusterNode> top, Map<Long, byte[]> discoData, @Nullable Map<Long, Long> dupDiscoData) {
+        assert top != null;
+        assert discoData != null && !discoData.isEmpty();
+
+        this.top = top;
+        this.discoData = discoData;
+        this.dupDiscoData = dupDiscoData;
+    }
+
+    byte[] discoveryDataForNode(long nodeOrder) {
+        assert discoData != null;
+
+        byte[] dataBytes = discoData.get(nodeOrder);
+
+        if (dataBytes != null)
+            return dataBytes;
+
+        assert dupDiscoData != null;
+
+        Long dupDataNode = dupDiscoData.get(nodeOrder);
+
+        assert dupDataNode != null;
+
+        dataBytes = discoData.get(dupDataNode);
+
+        assert dataBytes != null;
+
+        return dataBytes;
+    }
+
+    /**
+     * @return Current topology.
+     */
+    List<ZookeeperClusterNode> topology() {
+        assert top != null;
+
+        return top;
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinedNodeEvtData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinedNodeEvtData.java
new file mode 100644
index 0000000000000..3c367cf754762
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoinedNodeEvtData.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.UUID;
+
+/**
+ * Zk Joined Node Evt Data.
+ */
+public class ZkJoinedNodeEvtData implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    final long topVer;
+
+    /** */
+    final long joinedInternalId;
+
+    /** */
+    final UUID nodeId;
+
+    /** */
+    final int joinDataPartCnt;
+
+    /** */
+    final int secSubjPartCnt;
+
+    /** */
+    final UUID joinDataPrefixId;
+
+    /** */
+    transient ZkJoiningNodeData joiningNodeData;
+
+    /**
+     * @param topVer Topology version for node join event.
+     * @param nodeId Joined node ID.
+     * @param joinedInternalId Joined node internal ID.
+     * @param joinDataPrefixId Join data unique prefix.
+     * @param joinDataPartCnt Join data part count.
+     * @param secSubjPartCnt Security subject part count.
+     */
+    ZkJoinedNodeEvtData(
+        long topVer,
+        UUID nodeId,
+        long joinedInternalId,
+        UUID joinDataPrefixId,
+        int joinDataPartCnt,
+        int secSubjPartCnt)
+    {
+        this.topVer = topVer;
+        this.nodeId = nodeId;
+        this.joinedInternalId = joinedInternalId;
+        this.joinDataPrefixId = joinDataPrefixId;
+        this.joinDataPartCnt = joinDataPartCnt;
+        this.secSubjPartCnt = secSubjPartCnt;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "ZkJoinedNodeData [id=" + nodeId + ", order=" + topVer + ']';
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoiningNodeData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoiningNodeData.java
new file mode 100644
index 0000000000000..ff8311d071ba8
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkJoiningNodeData.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+class ZkJoiningNodeData implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private int partCnt;
+
+    /** */
+    @GridToStringInclude
+    private ZookeeperClusterNode node;
+
+    /** */
+    @GridToStringInclude
+    private Map<Integer, Serializable> discoData;
+
+    /**
+     * @param partCnt Number of parts in multi-parts message.
+     */
+    ZkJoiningNodeData(int partCnt) {
+        this.partCnt = partCnt;
+    }
+
+    /**
+     * @param node Node.
+     * @param discoData Discovery data.
+     */
+    ZkJoiningNodeData(ZookeeperClusterNode node, Map<Integer, Serializable> discoData) {
+        assert node != null && node.id() != null : node;
+        assert discoData != null;
+
+        this.node = node;
+        this.discoData = discoData;
+    }
+
+    /**
+     * @return Number of parts in multi-parts message.
+     */
+    int partCount() {
+        return partCnt;
+    }
+
+    /**
+     * @return Node.
+     */
+    ZookeeperClusterNode node() {
+        return node;
+    }
+
+    /**
+     * @return Discovery data.
+     */
+    Map<Integer, Serializable> discoveryData() {
+        return discoData;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ZkJoiningNodeData.class, this);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNoServersMessage.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNoServersMessage.java
new file mode 100644
index 0000000000000..626fe742d1c1b
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNoServersMessage.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+class ZkNoServersMessage implements DiscoverySpiCustomMessage, ZkInternalMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Nullable @Override public DiscoverySpiCustomMessage ackMessage() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMutable() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ZkNoServersMessage.class, this);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNodeValidateResult.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNodeValidateResult.java
new file mode 100644
index 0000000000000..2abfee3d61600
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkNodeValidateResult.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+class ZkNodeValidateResult {
+    /** */
+    String err;
+
+    /** */
+    byte[] secSubjZipBytes;
+
+    /**
+     * @param err Error.
+     */
+    ZkNodeValidateResult(String err) {
+        this.err = err;
+    }
+
+    /**
+     * @param secSubjZipBytes Marshalled security subject.
+     */
+    ZkNodeValidateResult(byte[] secSubjZipBytes) {
+        this.secSubjZipBytes = secSubjZipBytes;
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java
new file mode 100644
index 0000000000000..965bdc0f45851
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRunnable.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ * Zk Runnable.
+ */
+public abstract class ZkRunnable extends ZkAbstractCallabck implements Runnable {
+    /**
+     * @param rtState Runtime state.
+     * @param impl Discovery impl.
+     */
+    ZkRunnable(ZkRuntimeState rtState, ZookeeperDiscoveryImpl impl) {
+        super(rtState, impl);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void run() {
+        if (!onProcessStart())
+            return;
+
+        try {
+            run0();
+
+            onProcessEnd();
+        }
+        catch (Throwable e) {
+            onProcessError(e);
+        }
+    }
+
+    /**
+     *
+     */
+    protected abstract void run0() throws Exception;
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java
new file mode 100644
index 0000000000000..cb04ac3de01e8
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkRuntimeState.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.List;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.Watcher;
+
+/**
+ *
+ */
+class ZkRuntimeState {
+    /** */
+    ZkWatcher watcher;
+
+    /** */
+    ZkAliveNodeDataWatcher aliveNodeDataWatcher;
+
+    /** */
+    volatile Exception errForClose;
+
+    /** */
+    final boolean prevJoined;
+
+    /** */
+    ZookeeperClient zkClient;
+
+    /** */
+    long internalOrder;
+
+    /** */
+    int joinDataPartCnt;
+
+    /** */
+    long gridStartTime;
+
+    /** */
+    volatile boolean joined;
+
+    /** */
+    ZkDiscoveryEventsData evtsData;
+
+    /** */
+    boolean crd;
+
+    /** */
+    String locNodeZkPath;
+
+    /** */
+    final ZkAliveNodeData locNodeInfo = new ZkAliveNodeData();
+
+    /** */
+    int procEvtCnt;
+
+    /** */
+    final ZkClusterNodes top = new ZkClusterNodes();
+
+    /** */
+    List<ClusterNode> commErrProcNodes;
+
+    /** Timeout callback registering watcher for join error
+     * (set this watcher after timeout as a minor optimization).
+     */
+    ZkTimeoutObject joinErrTo;
+
+    /** Timeout callback set to wait for join timeout. */
+    ZkTimeoutObject joinTo;
+
+    /** Timeout callback to update processed events counter. */
+    ZkTimeoutObject procEvtsUpdateTo;
+
+    /** */
+    boolean updateAlives;
+
+    /**
+     * @param prevJoined {@code True} if joined topology before reconnect attempt.
+     */
+    ZkRuntimeState(boolean prevJoined) {
+        this.prevJoined = prevJoined;
+    }
+
+    /**
+     * @param watcher Watcher.
+     * @param aliveNodeDataWatcher Alive nodes data watcher.
+     */
+    void init(ZkWatcher watcher, ZkAliveNodeDataWatcher aliveNodeDataWatcher) {
+        this.watcher = watcher;
+        this.aliveNodeDataWatcher = aliveNodeDataWatcher;
+    }
+
+    /**
+     * @param err Error.
+     */
+    void onCloseStart(Exception err) {
+        assert err != null;
+
+        errForClose = err;
+
+        ZookeeperClient zkClient = this.zkClient;
+
+        if (zkClient != null)
+            zkClient.onCloseStart();
+    }
+
+    /**
+     *
+     */
+    interface ZkWatcher extends Watcher, AsyncCallback.Children2Callback, AsyncCallback.DataCallback {
+        // No-op.
+    }
+
+    /**
+     *
+     */
+    interface ZkAliveNodeDataWatcher extends Watcher, AsyncCallback.DataCallback {
+        // No-op.
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkTimeoutObject.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkTimeoutObject.java
new file mode 100644
index 0000000000000..4d3d5b4885cc1
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkTimeoutObject.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+
+/**
+ *
+ */
+abstract class ZkTimeoutObject implements IgniteSpiTimeoutObject {
+    /** */
+    private final IgniteUuid id = IgniteUuid.randomUuid();
+
+    /** */
+    private final long endTime;
+
+    /** */
+    volatile boolean cancelled;
+
+    /**
+     * @param timeout Timeout.
+     */
+    ZkTimeoutObject(long timeout) {
+        long endTime = timeout >= 0 ? System.currentTimeMillis() + timeout : Long.MAX_VALUE;
+
+        this.endTime = endTime >= 0 ? endTime : Long.MAX_VALUE;
+    }
+
+    /** {@inheritDoc} */
+    @Override public final IgniteUuid id() {
+        return id;
+    }
+
+    /** {@inheritDoc} */
+    @Override public final long endTime() {
+        return endTime;
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
new file mode 100644
index 0000000000000..21703c66a5518
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -0,0 +1,1219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.Op;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Zookeeper Client.
+ */
+public class ZookeeperClient implements Watcher {
+    /** */
+    private static final long RETRY_TIMEOUT =
+        IgniteSystemProperties.getLong("IGNITE_ZOOKEEPER_DISCOVERY_RETRY_TIMEOUT", 2000);
+
+    /** */
+    private static final int MAX_RETRY_COUNT =
+        IgniteSystemProperties.getInteger("IGNITE_ZOOKEEPER_DISCOVERY_MAX_RETRY_COUNT", 10);
+
+    /** */
+    private final AtomicInteger retryCount = new AtomicInteger();
+
+    /** */
+    private static final int MAX_REQ_SIZE = 1048528;
+
+    /** */
+    private static final List<ACL> ZK_ACL = ZooDefs.Ids.OPEN_ACL_UNSAFE;
+
+    /** */
+    private static final byte[] EMPTY_BYTES = {};
+
+    /** */
+    private final ZooKeeper zk;
+
+    /** */
+    private final IgniteLogger log;
+
+    /** */
+    private ConnectionState state = ConnectionState.Disconnected;
+
+    /** */
+    private long connLossTimeout;
+
+    /** */
+    private volatile long connStartTime;
+
+    /** */
+    private final Object stateMux = new Object();
+
+    /** */
+    private final IgniteRunnable connLostC;
+
+    /** */
+    private final Timer connTimer;
+
+    /** */
+    private final ArrayDeque<ZkAsyncOperation> retryQ = new ArrayDeque<>();
+
+    /** */
+    private volatile boolean closing;
+
+    /**
+     * @param log Logger.
+     * @param connectString ZK connection string.
+     * @param sesTimeout ZK session timeout.
+     * @param connLostC Lost connection callback.
+     * @throws Exception If failed.
+     */
+    ZookeeperClient(IgniteLogger log, String connectString, int sesTimeout, IgniteRunnable connLostC) throws Exception {
+        this(null, log, connectString, sesTimeout, connLostC);
+    }
+
+    /**
+     * @param igniteInstanceName Ignite instance name.
+     * @param log Logger.
+     * @param connectString ZK connection string.
+     * @param sesTimeout ZK session timeout.
+     * @param connLostC Lost connection callback.
+     * @throws Exception If failed.
+     */
+    ZookeeperClient(String igniteInstanceName,
+        IgniteLogger log,
+        String connectString,
+        int sesTimeout,
+        IgniteRunnable connLostC)
+        throws Exception
+    {
+        this.log = log.getLogger(getClass());
+        this.connLostC = connLostC;
+
+        connLossTimeout = sesTimeout;
+
+        long connStartTime = this.connStartTime = System.currentTimeMillis();
+
+        connTimer = new Timer("zk-client-timer-" + igniteInstanceName);
+
+        String threadName = Thread.currentThread().getName();
+
+        // ZK generates internal threads' names using current thread name.
+        Thread.currentThread().setName("zk-" + igniteInstanceName);
+
+        try {
+            zk = new ZooKeeper(connectString, sesTimeout, this);
+        }
+        finally {
+            Thread.currentThread().setName(threadName);
+        }
+
+        synchronized (stateMux) {
+            if (connStartTime == this.connStartTime && state == ConnectionState.Disconnected)
+                scheduleConnectionCheck();
+        }
+    }
+
+    /**
+     * @return Zookeeper client.
+     */
+    ZooKeeper zk() {
+        return zk;
+    }
+
+    /**
+     * @return {@code True} if connected to ZooKeeper.
+     */
+    boolean connected() {
+        synchronized (stateMux) {
+            return state == ConnectionState.Connected;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void process(WatchedEvent evt) {
+        if (closing)
+            return;
+
+        if (evt.getType() == Event.EventType.None) {
+            ConnectionState newState;
+
+            synchronized (stateMux) {
+                if (state == ConnectionState.Lost) {
+                    U.warn(log, "Received event after connection was lost [evtState=" + evt.getState() + "]");
+
+                    return;
+                }
+
+                if (!zk.getState().isAlive())
+                    return;
+
+                Event.KeeperState zkState = evt.getState();
+
+                switch (zkState) {
+                    case SaslAuthenticated:
+                        return; // No-op.
+
+                    case AuthFailed:
+                        newState = state;
+
+                        break;
+
+                    case Disconnected:
+                        newState = ConnectionState.Disconnected;
+
+                        break;
+
+                    case SyncConnected:
+                        newState = ConnectionState.Connected;
+
+                        break;
+
+                    case Expired:
+                        U.warn(log, "Session expired, changing state to Lost");
+
+                        newState = ConnectionState.Lost;
+
+                        break;
+
+                    default:
+                        U.error(log, "Unexpected state for ZooKeeper client, close connection: " + zkState);
+
+                        newState = ConnectionState.Lost;
+                }
+
+                if (newState != state) {
+                    if (log.isInfoEnabled())
+                        log.info("ZooKeeper client state changed [prevState=" + state + ", newState=" + newState + ']');
+
+                    state = newState;
+
+                    if (newState == ConnectionState.Disconnected) {
+                        connStartTime = System.currentTimeMillis();
+
+                        scheduleConnectionCheck();
+                    }
+                    else if (newState == ConnectionState.Connected) {
+                        retryCount.set(0);
+
+                        stateMux.notifyAll();
+                    }
+                    else
+                        assert state == ConnectionState.Lost : state;
+                }
+                else
+                    return;
+            }
+
+            if (newState == ConnectionState.Lost) {
+                closeClient();
+
+                notifyConnectionLost();
+            }
+            else if (newState == ConnectionState.Connected) {
+                for (ZkAsyncOperation op : retryQ)
+                    op.execute();
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private void notifyConnectionLost() {
+        if (!closing && state == ConnectionState.Lost && connLostC != null)
+            connLostC.run();
+
+        connTimer.cancel();
+    }
+
+    /**
+     * @param path Path.
+     * @return {@code True} if node exists.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    boolean exists(String path) throws ZookeeperClientFailedException, InterruptedException {
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                return zk.exists(path, false) != null;
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+        }
+    }
+
+    /**
+     *
+     * @param paths Paths to create.
+     * @param createMode Create mode.
+     * @throws KeeperException.NodeExistsException If at least one of target node already exists.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    void createAll(List<String> paths, CreateMode createMode)
+        throws ZookeeperClientFailedException, InterruptedException, KeeperException.NodeExistsException
+    {
+        // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8188
+        List<Op> ops = new ArrayList<>(paths.size());
+
+        for (String path : paths)
+            ops.add(Op.create(path, EMPTY_BYTES, ZK_ACL, createMode));
+
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                zk.multi(ops);
+
+                return;
+            }
+            catch (KeeperException.NodeExistsException e) {
+                throw e;
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+        }
+    }
+
+    /**
+     * @param path Path.
+     * @param data Data.
+     * @param overhead Extra overhead.
+     * @return {@code True} If data size exceeds max request size and should be splitted into multiple parts.
+     */
+    boolean needSplitNodeData(String path, byte[] data, int overhead) {
+        return requestOverhead(path) + data.length + overhead > MAX_REQ_SIZE;
+    }
+
+    /**
+     * @param path Path.
+     * @param data Data.
+     * @param overhead Extra overhead.
+     * @return Splitted data.
+     */
+    List<byte[]> splitNodeData(String path, byte[] data, int overhead) {
+        int partSize = MAX_REQ_SIZE - requestOverhead(path) - overhead;
+
+        int partCnt = data.length / partSize;
+
+        if (data.length % partSize != 0)
+            partCnt++;
+
+        assert partCnt > 1 : "Do not need split";
+
+        List<byte[]> parts = new ArrayList<>(partCnt);
+
+        int remaining = data.length;
+
+        for (int i = 0; i < partCnt; i++) {
+            int partSize0 = Math.min(remaining, partSize);
+
+            byte[] part = new byte[partSize0];
+
+            System.arraycopy(data, i * partSize, part, 0, part.length);
+
+            remaining -= partSize0;
+
+            parts.add(part);
+        }
+
+        assert remaining == 0 : remaining;
+
+        return parts;
+    }
+
+    /**
+     * TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8187
+     * @param path Request path.
+     * @return Marshalled request overhead.
+     */
+    private int requestOverhead(String path) {
+        return path.length();
+    }
+
+    /**
+     * @param path Path.
+     * @param data Data.
+     * @param createMode Create mode.
+     * @return Created path.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    String createIfNeeded(String path, byte[] data, CreateMode createMode)
+        throws ZookeeperClientFailedException, InterruptedException
+    {
+        assert !createMode.isSequential() : createMode;
+
+        if (data == null)
+            data = EMPTY_BYTES;
+
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                return zk.create(path, data, ZK_ACL, createMode);
+            }
+            catch (KeeperException.NodeExistsException e) {
+                if (log.isDebugEnabled())
+                    log.debug("Node already exists: " + path);
+
+                return path;
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+        }
+    }
+
+    /**
+     * @param checkPrefix Unique prefix to check in case of retry.
+     * @param parentPath Parent node path.
+     * @param path Node to create.
+     * @param data Node data.
+     * @param createMode Create mode.
+     * @return Create path.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    String createSequential(String checkPrefix, String parentPath, String path, byte[] data, CreateMode createMode)
+        throws ZookeeperClientFailedException, InterruptedException
+    {
+        assert createMode.isSequential() : createMode;
+
+        if (data == null)
+            data = EMPTY_BYTES;
+
+        boolean first = true;
+
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                if (!first) {
+                    List<String> children = zk.getChildren(parentPath, false);
+
+                    for (int i = 0; i < children.size(); i++) {
+                        String child = children.get(i);
+
+                        if (children.get(i).startsWith(checkPrefix)) {
+                            String resPath = parentPath + "/" + child;
+
+                            if (log.isDebugEnabled())
+                                log.debug("Check before retry, node already created: " + resPath);
+
+                            return resPath;
+                        }
+                    }
+                }
+
+                return zk.create(path, data, ZK_ACL, createMode);
+            }
+            catch (KeeperException.NodeExistsException e) {
+                assert !createMode.isSequential() : createMode;
+
+                if (log.isDebugEnabled())
+                    log.debug("Node already exists: " + path);
+
+                return path;
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+
+            first = false;
+        }
+    }
+
+    /**
+     * @param path Path.
+     * @return Children nodes.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    List<String> getChildren(String path)
+        throws ZookeeperClientFailedException, InterruptedException
+    {
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                return zk.getChildren(path, false);
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+        }
+    }
+
+    /**
+     * @param path Path.
+     * @throws InterruptedException If interrupted.
+     * @throws KeeperException In case of error.
+     * @return {@code True} if given path exists.
+     */
+    boolean existsNoRetry(String path) throws InterruptedException, KeeperException {
+        return zk.exists(path, false) != null;
+    }
+
+    /**
+     * @param path Path.
+     * @param ver Expected version.
+     * @throws InterruptedException If interrupted.
+     * @throws KeeperException In case of error.
+     */
+    void deleteIfExistsNoRetry(String path, int ver) throws InterruptedException, KeeperException {
+        try {
+            zk.delete(path, ver);
+        }
+        catch (KeeperException.NoNodeException e) {
+            // No-op if znode does not exist.
+        }
+    }
+
+    /**
+     * @param path Path.
+     * @param ver Version.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    void deleteIfExists(String path, int ver)
+        throws ZookeeperClientFailedException, InterruptedException
+    {
+        try {
+            delete(path, ver);
+        }
+        catch (KeeperException.NoNodeException e) {
+            // No-op if znode does not exist.
+        }
+    }
+
+    /**
+     * @param parent Parent path.
+     * @param paths Children paths.
+     * @param ver Version.
+     * @throws KeeperException.NoNodeException If at least one of nodes does not exist.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    void deleteAll(@Nullable String parent, List<String> paths, int ver)
+        throws KeeperException.NoNodeException, ZookeeperClientFailedException, InterruptedException
+    {
+        if (paths.isEmpty())
+            return;
+
+        // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8188
+        List<Op> ops = new ArrayList<>(paths.size());
+
+        for (String path : paths) {
+            String path0 = parent != null ? parent + "/" + path : path;
+
+            ops.add(Op.delete(path0, ver));
+        }
+
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                zk.multi(ops);
+
+                return;
+            }
+            catch (KeeperException.NoNodeException e) {
+                throw e;
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+        }
+    }
+
+    /**
+     * @param path Path.
+     * @param ver Version.
+     * @throws KeeperException.NoNodeException If target node does not exist.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    private void delete(String path, int ver)
+        throws KeeperException.NoNodeException, ZookeeperClientFailedException, InterruptedException
+    {
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                zk.delete(path, ver);
+
+                return;
+            }
+            catch (KeeperException.NoNodeException e) {
+                throw e;
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+        }
+    }
+
+    /**
+     * @param path Path.
+     * @param data Data.
+     * @param ver Version.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     * @throws KeeperException.NoNodeException If node does not exist.
+     * @throws KeeperException.BadVersionException If version does not match.
+     */
+    void setData(String path, byte[] data, int ver)
+        throws ZookeeperClientFailedException, InterruptedException, KeeperException.NoNodeException,
+        KeeperException.BadVersionException
+    {
+        if (data == null)
+            data = EMPTY_BYTES;
+
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                zk.setData(path, data, ver);
+
+                return;
+            }
+            catch (KeeperException.BadVersionException | KeeperException.NoNodeException e) {
+                throw e;
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+        }
+    }
+
+    /**
+     * @param path Path.
+     * @param stat Optional {@link Stat} instance to return znode state.
+     * @return Data.
+     * @throws KeeperException.NoNodeException If target node does not exist.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    byte[] getData(String path, @Nullable Stat stat)
+        throws KeeperException.NoNodeException, ZookeeperClientFailedException, InterruptedException {
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                return zk.getData(path, false, stat);
+            }
+            catch (KeeperException.NoNodeException e) {
+                throw e;
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+        }
+    }
+
+    /**
+     * @param path Path.
+     * @return Data.
+     * @throws KeeperException.NoNodeException If target node does not exist.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    byte[] getData(String path)
+        throws KeeperException.NoNodeException, ZookeeperClientFailedException, InterruptedException
+    {
+        return getData(path, null);
+    }
+
+    /**
+     * @param path Path.
+     */
+    void deleteIfExistsAsync(String path) {
+        new DeleteIfExistsOperation(path).execute();
+    }
+
+    /**
+     * @param path Path.
+     * @param watcher Watcher.
+     * @param cb Callback.
+     */
+    void existsAsync(String path, Watcher watcher, AsyncCallback.StatCallback cb) {
+        ExistsOperation op = new ExistsOperation(path, watcher, cb);
+
+        zk.exists(path, watcher, new StatCallbackWrapper(op), null);
+    }
+
+    /**
+     * @param path Path.
+     * @param watcher Watcher.
+     * @param cb Callback.
+     */
+    void getChildrenAsync(String path, Watcher watcher, AsyncCallback.Children2Callback cb) {
+        GetChildrenOperation op = new GetChildrenOperation(path, watcher, cb);
+
+        zk.getChildren(path, watcher, new ChildrenCallbackWrapper(op), null);
+    }
+
+    /**
+     * @param path Path.
+     * @param watcher Watcher.
+     * @param cb Callback.
+     */
+    void getDataAsync(String path, Watcher watcher, AsyncCallback.DataCallback cb) {
+        GetDataOperation op = new GetDataOperation(path, watcher, cb);
+
+        zk.getData(path, watcher, new DataCallbackWrapper(op), null);
+    }
+
+    /**
+     * @param path Path.
+     * @param data Data.
+     * @param createMode Create mode.
+     * @param cb Callback.
+     */
+    private void createAsync(String path, byte[] data, CreateMode createMode, AsyncCallback.StringCallback cb) {
+        if (data == null)
+            data = EMPTY_BYTES;
+
+        CreateOperation op = new CreateOperation(path, data, createMode, cb);
+
+        zk.create(path, data, ZK_ACL, createMode, new CreateCallbackWrapper(op), null);
+    }
+
+    /**
+     *
+     */
+    void onCloseStart() {
+        closing = true;
+
+        synchronized (stateMux) {
+            stateMux.notifyAll();
+        }
+    }
+
+    /**
+     *
+     */
+    public void close() {
+        closeClient();
+    }
+
+    /**
+     * @param prevConnStartTime Time when connection was established.
+     * @param e Error.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    private void onZookeeperError(long prevConnStartTime, Exception e)
+        throws ZookeeperClientFailedException, InterruptedException
+    {
+        ZookeeperClientFailedException err = null;
+
+        synchronized (stateMux) {
+            if (closing)
+                throw new ZookeeperClientFailedException("ZooKeeper client is closed.");
+
+            U.warn(log, "Failed to execute ZooKeeper operation [err=" + e + ", state=" + state + ']');
+
+            if (state == ConnectionState.Lost) {
+                U.error(log, "Operation failed with unexpected error, connection lost: " + e, e);
+
+                throw new ZookeeperClientFailedException(e);
+            }
+
+            boolean retry = (e instanceof KeeperException) && needRetry(((KeeperException)e).code().intValue());
+
+            if (retry) {
+                long remainingTime;
+
+                if (state == ConnectionState.Connected && connStartTime == prevConnStartTime) {
+                    state = ConnectionState.Disconnected;
+
+                    connStartTime = System.currentTimeMillis();
+
+                    remainingTime = connLossTimeout;
+                }
+                else {
+                    assert connStartTime != 0;
+
+                    assert state == ConnectionState.Disconnected : state;
+
+                    remainingTime = connLossTimeout - (System.currentTimeMillis() - connStartTime);
+
+                    if (remainingTime <= 0) {
+                        state = ConnectionState.Lost;
+
+                        U.warn(log, "Failed to establish ZooKeeper connection, close client " +
+                            "[timeout=" + connLossTimeout + ']');
+
+                        err = new ZookeeperClientFailedException(e);
+                    }
+                }
+
+                if (err == null) {
+                    U.warn(log, "ZooKeeper operation failed, will retry [err=" + e +
+                        ", retryTimeout=" + RETRY_TIMEOUT +
+                        ", connLossTimeout=" + connLossTimeout +
+                        ", path=" + ((KeeperException)e).getPath() +
+                        ", remainingWaitTime=" + remainingTime + ']');
+
+                    stateMux.wait(RETRY_TIMEOUT);
+
+                    if (closing)
+                        throw new ZookeeperClientFailedException("ZooKeeper client is closed.");
+                }
+            }
+            else {
+                U.error(log, "Operation failed with unexpected error, close ZooKeeper client: " + e, e);
+
+                state = ConnectionState.Lost;
+
+                err = new ZookeeperClientFailedException(e);
+            }
+        }
+
+        if (err != null) {
+            closeClient();
+
+            notifyConnectionLost();
+
+            throw err;
+        }
+    }
+
+    /**
+     * @param code Zookeeper error code.
+     * @return {@code True} if can retry operation.
+     */
+    private boolean needRetry(int code) {
+        boolean retryByErrorCode = code == KeeperException.Code.CONNECTIONLOSS.intValue() ||
+            code == KeeperException.Code.SESSIONMOVED.intValue() ||
+            code == KeeperException.Code.OPERATIONTIMEOUT.intValue();
+
+        if (retryByErrorCode) {
+            if (MAX_RETRY_COUNT <= 0 || retryCount.incrementAndGet() < MAX_RETRY_COUNT)
+                return true;
+            else
+                return false;
+        }
+        else
+            return false;
+    }
+
+    /**
+     *
+     */
+    private void closeClient() {
+        try {
+            zk.close();
+        }
+        catch (Exception closeErr) {
+            U.warn(log, "Failed to close ZooKeeper client: " + closeErr, closeErr);
+        }
+
+        connTimer.cancel();
+    }
+
+    /**
+     *
+     */
+    private void scheduleConnectionCheck() {
+        assert state == ConnectionState.Disconnected : state;
+
+        connTimer.schedule(new ConnectionTimeoutTask(connStartTime), connLossTimeout);
+    }
+
+    /**
+     *
+     */
+    interface ZkAsyncOperation {
+        /**
+         *
+         */
+        void execute();
+    }
+
+    /**
+     *
+     */
+    class GetChildrenOperation implements ZkAsyncOperation {
+        /** */
+        private final String path;
+
+        /** */
+        private final Watcher watcher;
+
+        /** */
+        private final AsyncCallback.Children2Callback cb;
+
+        /**
+         * @param path Path.
+         * @param watcher Watcher.
+         * @param cb Callback.
+         */
+        GetChildrenOperation(String path, Watcher watcher, AsyncCallback.Children2Callback cb) {
+            this.path = path;
+            this.watcher = watcher;
+            this.cb = cb;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute() {
+            getChildrenAsync(path, watcher, cb);
+        }
+    }
+
+    /**
+     *
+     */
+    class GetDataOperation implements ZkAsyncOperation {
+        /** */
+        private final String path;
+
+        /** */
+        private final Watcher watcher;
+
+        /** */
+        private final AsyncCallback.DataCallback cb;
+
+        /**
+         * @param path Path.
+         * @param watcher Watcher.
+         * @param cb Callback.
+         */
+        GetDataOperation(String path, Watcher watcher, AsyncCallback.DataCallback cb) {
+            this.path = path;
+            this.watcher = watcher;
+            this.cb = cb;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute() {
+            getDataAsync(path, watcher, cb);
+        }
+    }
+
+    /**
+     *
+     */
+    class ExistsOperation implements ZkAsyncOperation {
+        /** */
+        private final String path;
+
+        /** */
+        private final Watcher watcher;
+
+        /** */
+        private final AsyncCallback.StatCallback cb;
+
+        /**
+         * @param path Path.
+         * @param watcher Watcher.
+         * @param cb Callback.
+         */
+        ExistsOperation(String path, Watcher watcher, AsyncCallback.StatCallback cb) {
+            this.path = path;
+            this.watcher = watcher;
+            this.cb = cb;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute() {
+            existsAsync(path, watcher, cb);
+        }
+    }
+
+    /**
+     *
+     */
+    class CreateOperation implements ZkAsyncOperation {
+        /** */
+        private final String path;
+
+        /** */
+        private final byte[] data;
+
+        /** */
+        private final CreateMode createMode;
+
+        /** */
+        private final AsyncCallback.StringCallback cb;
+
+        /**
+         * @param path path.
+         * @param data Data.
+         * @param createMode Create mode.
+         * @param cb Callback.
+         */
+        CreateOperation(String path, byte[] data, CreateMode createMode, AsyncCallback.StringCallback cb) {
+            this.path = path;
+            this.data = data;
+            this.createMode = createMode;
+            this.cb = cb;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute() {
+            createAsync(path, data, createMode, cb);
+        }
+    }
+
+    /**
+     *
+     */
+    class DeleteIfExistsOperation implements AsyncCallback.VoidCallback, ZkAsyncOperation {
+        /** */
+        private final String path;
+
+        /**
+         * @param path Path.
+         */
+        DeleteIfExistsOperation(String path) {
+            this.path = path;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute() {
+            zk.delete(path, -1, this, null);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx) {
+            if (closing)
+                return;
+
+            if (rc == KeeperException.Code.NONODE.intValue())
+                return;
+
+            if (needRetry(rc)) {
+                U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [" +
+                    "path=" + path + ']');
+
+                retryQ.add(this);
+            }
+            else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+                U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
+            else
+                assert rc == 0 : KeeperException.Code.get(rc);
+        }
+    }
+
+    /**
+     *
+     */
+    class CreateCallbackWrapper implements AsyncCallback.StringCallback {
+        /** */
+        final CreateOperation op;
+
+        /**
+         * @param op Operation.
+         */
+        CreateCallbackWrapper(CreateOperation op) {
+            this.op = op;
+        }
+
+        @Override public void processResult(int rc, String path, Object ctx, String name) {
+            if (closing)
+                return;
+
+            if (rc == KeeperException.Code.NODEEXISTS.intValue())
+                return;
+
+            if (needRetry(rc)) {
+                U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [path=" + path + ']');
+
+                retryQ.add(op);
+            }
+            else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+                U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
+            else {
+                if (op.cb != null)
+                    op.cb.processResult(rc, path, ctx, name);
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    class ChildrenCallbackWrapper implements AsyncCallback.Children2Callback {
+        /** */
+        private final GetChildrenOperation op;
+
+        /**
+         * @param op Operation.
+         */
+        private ChildrenCallbackWrapper(GetChildrenOperation op) {
+            this.op = op;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+            if (closing)
+                return;
+
+            if (needRetry(rc)) {
+                U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [path=" + path + ']');
+
+                retryQ.add(op);
+            }
+            else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+                U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
+            else
+                op.cb.processResult(rc, path, ctx, children, stat);
+        }
+    }
+
+    /**
+     *
+     */
+    class DataCallbackWrapper implements AsyncCallback.DataCallback {
+        /** */
+        private final GetDataOperation op;
+
+        /**
+         * @param op Operation.
+         */
+        private DataCallbackWrapper(GetDataOperation op) {
+            this.op = op;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) {
+            if (closing)
+                return;
+
+            if (needRetry(rc)) {
+                U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [path=" + path + ']');
+
+                retryQ.add(op);
+            }
+            else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+                U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
+            else
+                op.cb.processResult(rc, path, ctx, data, stat);
+        }
+    }
+
+    /**
+     *
+     */
+    class StatCallbackWrapper implements AsyncCallback.StatCallback {
+        /** */
+        private final ExistsOperation op;
+
+        /**
+         * @param op Operation.
+         */
+        private StatCallbackWrapper(ExistsOperation op) {
+            this.op = op;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx, Stat stat) {
+            if (closing)
+                return;
+
+            if (needRetry(rc)) {
+                U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [path=" + path + ']');
+
+                retryQ.add(op);
+            }
+            else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+                U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']');
+            else
+                op.cb.processResult(rc, path, ctx, stat);
+        }
+    }
+
+    /**
+     *
+     */
+    private class ConnectionTimeoutTask extends TimerTask {
+        /** */
+        private final long connectStartTime;
+
+        /**
+         * @param connectStartTime Time was connection started.
+         */
+        ConnectionTimeoutTask(long connectStartTime) {
+            this.connectStartTime = connectStartTime;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            boolean connLoss = false;
+
+            synchronized (stateMux) {
+                if (closing)
+                    return;
+
+                if (state == ConnectionState.Disconnected &&
+                    ZookeeperClient.this.connStartTime == connectStartTime) {
+
+                    state = ConnectionState.Lost;
+
+                    U.warn(log, "Failed to establish ZooKeeper connection, close client " +
+                        "[timeout=" + connLossTimeout + ']');
+
+                    connLoss = true;
+                }
+            }
+
+            if (connLoss) {
+                closeClient();
+
+                notifyConnectionLost();
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private enum ConnectionState {
+        /** */
+        Connected,
+        /** */
+        Disconnected,
+        /** */
+        Lost
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientFailedException.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientFailedException.java
new file mode 100644
index 0000000000000..01d011b0cff5d
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientFailedException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+class ZookeeperClientFailedException extends Exception {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * @param msg Message.
+     */
+    ZookeeperClientFailedException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * @param cause Cause.
+     */
+    ZookeeperClientFailedException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
new file mode 100644
index 0000000000000..3cb5fad3129cf
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
@@ -0,0 +1,362 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DAEMON;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID;
+
+/**
+ * Zookeeper Cluster Node.
+ */
+public class ZookeeperClusterNode implements IgniteClusterNode, Serializable, Comparable<ZookeeperClusterNode> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private static final byte CLIENT_NODE_MASK = 0x01;
+
+    /** */
+    private UUID id;
+
+    /** */
+    private Serializable consistentId;
+
+    /** */
+    private long internalId;
+
+    /** */
+    private long order;
+
+    /** */
+    private IgniteProductVersion ver;
+
+    /** Node attributes. */
+    private Map<String, Object> attrs;
+
+    /** Internal discovery addresses as strings. */
+    private Collection<String> addrs;
+
+    /** Internal discovery host names as strings. */
+    private Collection<String> hostNames;
+
+    /** */
+    private long sesTimeout;
+
+    /** Metrics provider. */
+    private transient DiscoveryMetricsProvider metricsProvider;
+
+    /** */
+    private transient boolean loc;
+
+    /** */
+    private transient volatile ClusterMetrics metrics;
+
+    /** Node cache metrics. */
+    @GridToStringExclude
+    private transient volatile Map<Integer, CacheMetrics> cacheMetrics;
+
+    /** */
+    private byte flags;
+
+    /** Daemon node flag. */
+    @GridToStringExclude
+    private transient boolean daemon;
+
+    /** Daemon node initialization flag. */
+    @GridToStringExclude
+    private transient volatile boolean daemonInit;
+
+    /**
+     * @param id Node ID.
+     * @param addrs Node addresses.
+     * @param hostNames Node host names.
+     * @param ver Node version.
+     * @param attrs Node attributes.
+     * @param consistentId Consistent ID.
+     * @param sesTimeout Zookeeper session timeout.
+     * @param client Client node flag.
+     * @param metricsProvider Metrics provider.
+     */
+    public ZookeeperClusterNode(
+        UUID id,
+        Collection<String> addrs,
+        Collection<String> hostNames,
+        IgniteProductVersion ver,
+        Map<String, Object> attrs,
+        Serializable consistentId,
+        long sesTimeout,
+        boolean client,
+        DiscoveryMetricsProvider metricsProvider
+    ) {
+        assert id != null;
+        assert consistentId != null;
+
+        this.id = id;
+        this.ver = ver;
+        this.attrs = Collections.unmodifiableMap(attrs);
+        this.addrs = addrs;
+        this.hostNames = hostNames;
+        this.consistentId = consistentId;
+        this.sesTimeout = sesTimeout;
+        this.metricsProvider = metricsProvider;
+
+        if (client)
+            flags |= CLIENT_NODE_MASK;
+    }
+
+    /** {@inheritDoc} */
+    @Override public UUID id() {
+        return id;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Object consistentId() {
+        return consistentId;
+    }
+
+    /** {@inheritDoc} */
+    public void setConsistentId(Serializable consistentId) {
+        this.consistentId = consistentId;
+
+        final Map<String, Object> map = new HashMap<>(attrs);
+
+        map.put(ATTR_NODE_CONSISTENT_ID, consistentId);
+
+        attrs = Collections.unmodifiableMap(map);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isCacheClient() {
+        return isClient();
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public <T> T attribute(String name) {
+        // Even though discovery SPI removes this attribute after authentication, keep this check for safety.
+        if (IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS.equals(name))
+            return null;
+
+        return (T)attrs.get(name);
+    }
+
+    /**
+     * Sets node attributes.
+     *
+     * @param attrs Node attributes.
+     */
+    void setAttributes(Map<String, Object> attrs) {
+        this.attrs = U.sealMap(attrs);
+    }
+
+    /**
+     * Gets node attributes without filtering.
+     *
+     * @return Node attributes without filtering.
+     */
+    Map<String, Object> getAttributes() {
+        return attrs;
+    }
+
+    /** {@inheritDoc} */
+    @Override public ClusterMetrics metrics() {
+        if (metricsProvider != null) {
+            ClusterMetrics metrics0 = metricsProvider.metrics();
+
+            assert metrics0 != null;
+
+            metrics = metrics0;
+
+            return metrics0;
+        }
+
+        return metrics;
+    }
+
+    /** {@inheritDoc} */
+    public void setMetrics(ClusterMetrics metrics) {
+        assert metrics != null;
+
+        this.metrics = metrics;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map<Integer, CacheMetrics> cacheMetrics() {
+        if (metricsProvider != null) {
+            Map<Integer, CacheMetrics> cacheMetrics0 = metricsProvider.cacheMetrics();
+
+            cacheMetrics = cacheMetrics0;
+
+            return cacheMetrics0;
+        }
+
+        return cacheMetrics;
+    }
+
+    /** {@inheritDoc} */
+    public void setCacheMetrics(Map<Integer, CacheMetrics> cacheMetrics) {
+        this.cacheMetrics = cacheMetrics != null ? cacheMetrics : Collections.<Integer, CacheMetrics>emptyMap();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map<String, Object> attributes() {
+        // Even though discovery SPI removes this attribute after authentication, keep this check for safety.
+        return F.view(attrs, new IgnitePredicate<String>() {
+            @Override public boolean apply(String s) {
+                return !IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS.equals(s);
+            }
+        });
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<String> addresses() {
+        return addrs;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<String> hostNames() {
+        return hostNames;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long order() {
+        return order;
+    }
+
+    /**
+     * @return Internal ID corresponds to Zookeeper sequential node.
+     */
+    long internalId() {
+        return internalId;
+    }
+
+    /**
+     * @param internalId Internal ID corresponds to Zookeeper sequential node.
+     */
+    void internalId(long internalId) {
+        this.internalId = internalId;
+    }
+
+    /**
+     * @param order Node order.
+     */
+    void order(long order) {
+        assert order > 0 : order;
+
+        this.order = order;
+    }
+
+    /**
+     * @param newId New node ID.
+     */
+    public void onClientDisconnected(UUID newId) {
+        id = newId;
+    }
+
+    /**
+     * @return Session timeout.
+     */
+    long sessionTimeout() {
+        return sesTimeout;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteProductVersion version() {
+        return ver;
+    }
+
+    /**
+     * @param loc Local node flag.
+     */
+    public void local(boolean loc) {
+        this.loc = loc;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLocal() {
+        return loc;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isDaemon() {
+        if (!daemonInit) {
+            daemon = "true".equalsIgnoreCase((String)attribute(ATTR_DAEMON));
+
+            daemonInit = true;
+        }
+
+        return daemon;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isClient() {
+        return (CLIENT_NODE_MASK & flags) != 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int compareTo(@Nullable ZookeeperClusterNode node) {
+        if (node == null)
+            return 1;
+
+        int res = Long.compare(order, node.order);
+
+        if (res == 0) {
+            assert id().equals(node.id()) : "Duplicate order [this=" + this + ", other=" + node + ']';
+
+            res = id().compareTo(node.id());
+        }
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        return id.hashCode();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object obj) {
+        return F.eqNodes(this, obj);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "ZookeeperClusterNode [id=" + id +
+            ", addrs=" + addrs +
+            ", order=" + order +
+            ", loc=" + loc +
+            ", client=" + isClient() + ']';
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
new file mode 100644
index 0000000000000..7708358f687dc
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -0,0 +1,4464 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.ByteArrayInputStream;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.zip.DataFormatException;
+import java.util.zip.Deflater;
+import java.util.zip.Inflater;
+import java.util.zip.InflaterInputStream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteClientDisconnectedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteInterruptedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CommunicationFailureResolver;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.ClusterMetricsSnapshot;
+import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.events.DiscoveryCustomEvent;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.internal.util.GridLongList;
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.io.GridByteArrayOutputStream;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.LT;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.marshaller.MarshallerUtils;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.apache.ignite.plugin.security.SecurityCredentials;
+import org.apache.ignite.spi.IgniteNodeValidationResult;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiTimeoutObject;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
+import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+import org.apache.ignite.thread.IgniteThreadPoolExecutor;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.data.Stat;
+import org.jboss.netty.util.internal.ConcurrentHashMap;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
+import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
+import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2;
+import static org.apache.zookeeper.CreateMode.EPHEMERAL_SEQUENTIAL;
+import static org.apache.zookeeper.CreateMode.PERSISTENT;
+
+/**
+ * Zookeeper Discovery Impl.
+ */
+public class ZookeeperDiscoveryImpl {
+    /** */
+    static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD";
+
+    /** */
+    static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT";
+
+    /** */
+    static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS";
+
+    /** */
+    private static final String IGNITE_ZOOKEEPER_DISCOVERY_SPI_EVTS_THROTTLE = "IGNITE_ZOOKEEPER_DISCOVERY_SPI_EVTS_THROTTLE";
+
+    /** */
+    final ZookeeperDiscoverySpi spi;
+
+    /** */
+    private final String igniteInstanceName;
+
+    /** */
+    private final String connectString;
+
+    /** */
+    private final int sesTimeout;
+
+    /** */
+    private final JdkMarshaller marsh = new JdkMarshaller();
+
+    /** */
+    private final ZkIgnitePaths zkPaths;
+
+    /** */
+    private final IgniteLogger log;
+
+    /** */
+    final GridSpinBusyLock busyLock = new GridSpinBusyLock();
+
+    /** */
+    private final ZookeeperClusterNode locNode;
+
+    /** */
+    private final DiscoverySpiListener lsnr;
+
+    /** */
+    private final DiscoverySpiDataExchange exchange;
+
+    /** */
+    private final boolean clientReconnectEnabled;
+
+    /** */
+    private final GridFutureAdapter<Void> joinFut = new GridFutureAdapter<>();
+
+    /** */
+    private final int evtsAckThreshold;
+
+    /** */
+    private IgniteThreadPoolExecutor utilityPool;
+
+    /** */
+    private ZkRuntimeState rtState;
+
+    /** */
+    private volatile ConnectionState connState = ConnectionState.STARTED;
+
+    /** */
+    private final AtomicBoolean stop = new AtomicBoolean();
+
+    /** */
+    private final Object stateMux = new Object();
+
+    /** */
+    public volatile IgniteDiscoverySpiInternalListener internalLsnr;
+
+    /** */
+    private final ConcurrentHashMap<Long, PingFuture> pingFuts = new ConcurrentHashMap<>();
+
+    /** */
+    private final AtomicReference<ZkCommunicationErrorProcessFuture> commErrProcFut = new AtomicReference<>();
+
+    /** */
+    private long prevSavedEvtsTopVer;
+
+    /**
+     * @param spi Discovery SPI.
+     * @param igniteInstanceName Instance name.
+     * @param log Logger.
+     * @param zkRootPath Zookeeper base path node all nodes.
+     * @param locNode Local node instance.
+     * @param lsnr Discovery events listener.
+     * @param exchange Discovery data exchange.
+     * @param internalLsnr Internal listener (used for testing only).
+     */
+    public ZookeeperDiscoveryImpl(
+        ZookeeperDiscoverySpi spi,
+        String igniteInstanceName,
+        IgniteLogger log,
+        String zkRootPath,
+        ZookeeperClusterNode locNode,
+        DiscoverySpiListener lsnr,
+        DiscoverySpiDataExchange exchange,
+        IgniteDiscoverySpiInternalListener internalLsnr) {
+        assert locNode.id() != null && locNode.isLocal() : locNode;
+
+        MarshallerUtils.setNodeName(marsh, igniteInstanceName);
+
+        zkPaths = new ZkIgnitePaths(zkRootPath);
+
+        this.spi = spi;
+        this.igniteInstanceName = igniteInstanceName;
+        this.connectString = spi.getZkConnectionString();
+        this.sesTimeout = (int)spi.getSessionTimeout();
+        this.log = log.getLogger(getClass());
+        this.locNode = locNode;
+        this.lsnr = lsnr;
+        this.exchange = exchange;
+        this.clientReconnectEnabled = locNode.isClient() && !spi.isClientReconnectDisabled();
+
+        int evtsAckThreshold = IgniteSystemProperties.getInteger(IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD, 5);
+
+        if (evtsAckThreshold <= 0)
+            evtsAckThreshold = 1;
+
+        this.evtsAckThreshold = evtsAckThreshold;
+
+        if (internalLsnr != null)
+            this.internalLsnr = internalLsnr;
+    }
+
+    /**
+     * @return Exception.
+     */
+    private static IgniteClientDisconnectedCheckedException disconnectError() {
+        return new IgniteClientDisconnectedCheckedException(null, "Client node disconnected.");
+    }
+
+    /**
+     * @return Logger.
+     */
+    IgniteLogger log() {
+        return log;
+    }
+
+    /**
+     * @return Local node instance.
+     */
+    public ClusterNode localNode() {
+        return locNode;
+    }
+
+    /**
+     * @param nodeId Node ID.
+     * @return Node instance.
+     */
+    @Nullable public ZookeeperClusterNode node(UUID nodeId) {
+        assert nodeId != null;
+
+        return rtState.top.nodesById.get(nodeId);
+    }
+
+    /**
+     * @param nodeOrder Node order.
+     * @return Node instance.
+     */
+    @Nullable public ZookeeperClusterNode node(long nodeOrder) {
+        assert nodeOrder > 0 : nodeOrder;
+
+        return rtState.top.nodesByOrder.get(nodeOrder);
+    }
+
+    /**
+     * @param fut Future to remove.
+     */
+    void clearCommunicationErrorProcessFuture(ZkCommunicationErrorProcessFuture fut) {
+        assert fut.isDone() : fut;
+
+        commErrProcFut.compareAndSet(fut, null);
+    }
+
+    /**
+     * @param node0 Problem node ID
+     * @param err Connect error.
+     */
+    public void resolveCommunicationError(ClusterNode node0, Exception err) {
+        ZookeeperClusterNode node = node(node0.id());
+
+        if (node == null)
+            throw new IgniteSpiException(new ClusterTopologyCheckedException("Node failed: " + node0.id()));
+
+        IgniteInternalFuture<Boolean> nodeStatusFut;
+
+        for (;;) {
+            checkState();
+
+            ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
+
+            if (fut == null || fut.isDone()) {
+                ZkCommunicationErrorProcessFuture newFut = ZkCommunicationErrorProcessFuture.createOnCommunicationError(
+                    this,
+                    node.sessionTimeout() + 1000);
+
+                if (commErrProcFut.compareAndSet(fut, newFut)) {
+                    fut = newFut;
+
+                    if (log.isInfoEnabled()) {
+                        log.info("Created new communication error process future [errNode=" + node0.id() +
+                            ", err=" + err + ']');
+                    }
+
+                    try {
+                        checkState();
+                    }
+                    catch (Exception e) {
+                        fut.onError(e);
+
+                        throw e;
+                    }
+
+                    fut.scheduleCheckOnTimeout();
+                }
+                else {
+                    fut = commErrProcFut.get();
+
+                    if (fut == null)
+                        continue;
+                }
+            }
+
+            nodeStatusFut = fut.nodeStatusFuture(node);
+
+            if (nodeStatusFut != null)
+                break;
+            else {
+                try {
+                    fut.get();
+                }
+                catch (IgniteCheckedException e) {
+                    U.warn(log, "Previous communication error process future failed: " + e);
+                }
+            }
+        }
+
+        try {
+            if (!nodeStatusFut.get())
+                throw new IgniteSpiException(new ClusterTopologyCheckedException("Node failed: " + node0.id()));
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteSpiException(e);
+        }
+    }
+
+    /**
+     * @param nodeId Node ID.
+     * @return Ping result.
+     */
+    public boolean pingNode(UUID nodeId) {
+        checkState();
+
+        ZkRuntimeState rtState = this.rtState;
+
+        ZookeeperClusterNode node = rtState.top.nodesById.get(nodeId);
+
+        if (node == null)
+            return false;
+
+        if (node.isLocal())
+            return true;
+
+        PingFuture fut = pingFuts.get(node.order());
+
+        if (fut == null) {
+            fut = new PingFuture(rtState, node);
+
+            PingFuture old = pingFuts.putIfAbsent(node.order(), fut);
+
+            if (old == null) {
+                if (fut.checkNodeAndState())
+                    spi.getSpiContext().addTimeoutObject(fut);
+                else
+                    assert fut.isDone();
+            }
+            else
+                fut = old;
+        }
+
+        try {
+            return fut.get();
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteSpiException(e);
+        }
+    }
+
+    /**
+     * @param nodeId Node ID.
+     * @param warning Warning.
+     */
+    public void failNode(UUID nodeId, @Nullable String warning) {
+        ZookeeperClusterNode node = rtState.top.nodesById.get(nodeId);
+
+        if (node == null) {
+            if (log.isDebugEnabled())
+                log.debug("Ignore forcible node fail request, node does not exist: " + nodeId);
+
+            return;
+        }
+
+        if (!node.isClient()) {
+            U.warn(log, "Ignore forcible node fail request for non-client node: " + node);
+
+            return;
+        }
+
+        sendCustomMessage(new ZkForceNodeFailMessage(node.internalId(), warning));
+    }
+
+    /**
+     *
+     */
+    public void reconnect() {
+        assert clientReconnectEnabled;
+
+        synchronized (stateMux) {
+            if (connState == ConnectionState.STARTED) {
+                connState = ConnectionState.DISCONNECTED;
+
+                rtState.onCloseStart(disconnectError());
+            }
+            else
+                return;
+        }
+
+        busyLock.block();
+
+        busyLock.unblock();
+
+        rtState.zkClient.close();
+
+        UUID newId = UUID.randomUUID();
+
+        U.quietAndWarn(log, "Local node will try to reconnect to cluster with new id due to network problems [" +
+            "newId=" + newId +
+            ", prevId=" + locNode.id() +
+            ", locNode=" + locNode + ']');
+
+        runInWorkerThread(new ReconnectClosure(newId));
+    }
+
+    /**
+     * @param newId New ID.
+     */
+    private void doReconnect(UUID newId) {
+        if (rtState.joined) {
+            assert rtState.evtsData != null;
+
+            lsnr.onDiscovery(EVT_CLIENT_NODE_DISCONNECTED,
+                rtState.evtsData.topVer,
+                locNode,
+                rtState.top.topologySnapshot(),
+                Collections.<Long, Collection<ClusterNode>>emptyMap(),
+                null);
+        }
+
+        try {
+            locNode.onClientDisconnected(newId);
+
+            joinTopology(rtState);
+        }
+        catch (Exception e) {
+            if (stopping()) {
+                if (log.isDebugEnabled())
+                    log.debug("Reconnect failed, node is stopping [err=" + e + ']');
+
+                return;
+            }
+
+            U.error(log, "Failed to reconnect: " + e, e);
+
+            onSegmented(e);
+        }
+    }
+
+    /**
+     * @return {@code True} if started to stop.
+     */
+    private boolean stopping() {
+        if (stop.get())
+            return true;
+
+        synchronized (stateMux) {
+            if (connState == ConnectionState.STOPPED)
+                return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * @param e Error.
+     */
+    private void onSegmented(Exception e) {
+        rtState.errForClose = e;
+
+        if (rtState.joined || joinFut.isDone()) {
+            synchronized (stateMux) {
+                connState = ConnectionState.STOPPED;
+            }
+
+            notifySegmented();
+        }
+        else
+            joinFut.onDone(e);
+    }
+
+    /**
+     *
+     */
+    private void notifySegmented() {
+        List<ClusterNode> nodes = rtState.top.topologySnapshot();
+
+        if (nodes.isEmpty())
+            nodes = Collections.singletonList((ClusterNode)locNode);
+
+        lsnr.onDiscovery(EVT_NODE_SEGMENTED,
+            rtState.evtsData != null ? rtState.evtsData.topVer : 1L,
+            locNode,
+            nodes,
+            Collections.<Long, Collection<ClusterNode>>emptyMap(),
+            null);
+    }
+
+    /**
+     * @return Remote nodes.
+     */
+    public Collection<ClusterNode> remoteNodes() {
+        checkState();
+
+        return rtState.top.remoteNodes();
+    }
+
+    /**
+     *
+     */
+    private void checkState() {
+        switch (connState) {
+            case STARTED:
+                break;
+
+            case STOPPED:
+                throw new IgniteSpiException("Node stopped.");
+
+            case DISCONNECTED:
+                throw new IgniteClientDisconnectedException(null, "Client is disconnected.");
+        }
+    }
+
+    /**
+     * @param nodeId Node ID.
+     * @return {@code True} if node joined or joining topology.
+     */
+    public boolean knownNode(UUID nodeId) {
+        while (!busyLock.enterBusy())
+            checkState();
+
+        try {
+            List<String> children = rtState.zkClient.getChildren(zkPaths.aliveNodesDir);
+
+            for (int i = 0; i < children.size(); i++) {
+                UUID id = ZkIgnitePaths.aliveNodeId(children.get(i));
+
+                if (nodeId.equals(id))
+                    return true;
+            }
+
+            return false;
+        }
+        catch (ZookeeperClientFailedException e) {
+            if (clientReconnectEnabled)
+                throw new IgniteClientDisconnectedException(null, "Client is disconnected.");
+
+            throw new IgniteException(e);
+        }
+        catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+
+            throw new IgniteInterruptedException(e);
+        }
+        finally {
+            busyLock.leaveBusy();
+        }
+    }
+
+    /**
+     * @param msg Message.
+     */
+    public void sendCustomMessage(DiscoverySpiCustomMessage msg) {
+        assert msg != null;
+
+        byte[] msgBytes;
+
+        try {
+            msgBytes = marshalZip(msg);
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteSpiException("Failed to marshal custom message: " + msg, e);
+        }
+
+        while (!busyLock.enterBusy())
+            checkState();
+
+        try {
+            ZookeeperClient zkClient = rtState.zkClient;
+
+            saveCustomMessage(zkClient, msgBytes);
+        }
+        catch (ZookeeperClientFailedException e) {
+            if (clientReconnectEnabled)
+                throw new IgniteClientDisconnectedException(null, "Client is disconnected.");
+
+            throw new IgniteException(e);
+        }
+        catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+
+            throw new IgniteInterruptedException(e);
+        }
+        finally {
+            busyLock.leaveBusy();
+        }
+    }
+
+    /**
+     * @param zkClient Client.
+     * @param msgBytes Marshalled message.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    private void saveCustomMessage(ZookeeperClient zkClient, byte[] msgBytes)
+        throws ZookeeperClientFailedException, InterruptedException
+    {
+        String prefix = UUID.randomUUID().toString();
+
+        int partCnt = 1;
+
+        int overhead = 10;
+
+        UUID locId = locNode.id();
+
+        String path = zkPaths.createCustomEventPath(prefix, locId, partCnt);
+
+        if (zkClient.needSplitNodeData(path, msgBytes, overhead)) {
+            List<byte[]> parts = zkClient.splitNodeData(path, msgBytes, overhead);
+
+            String partsBasePath = zkPaths.customEventPartsBasePath(prefix, locId);
+
+            saveMultipleParts(zkClient, partsBasePath, parts);
+
+            msgBytes = null;
+
+            partCnt = parts.size();
+        }
+
+        zkClient.createSequential(prefix,
+            zkPaths.customEvtsDir,
+            zkPaths.createCustomEventPath(prefix, locId, partCnt),
+            msgBytes,
+            CreateMode.PERSISTENT_SEQUENTIAL);
+    }
+
+    /**
+     * @return Cluster start time.
+     */
+    public long gridStartTime() {
+        return rtState.gridStartTime;
+    }
+
+    /**
+     * Starts join procedure and waits for {@link EventType#EVT_NODE_JOINED} event for local node.
+     *
+     * @throws InterruptedException If interrupted.
+     */
+    public void startJoinAndWait() throws InterruptedException {
+        joinTopology(null);
+
+        for (;;) {
+            try {
+                joinFut.get(10_000);
+
+                break;
+            }
+            catch (IgniteFutureTimeoutCheckedException e) {
+                U.warn(log, "Waiting for local join event [nodeId=" + locNode.id() + ", name=" + igniteInstanceName + ']');
+            }
+            catch (Exception e) {
+                IgniteSpiException spiErr = X.cause(e, IgniteSpiException.class);
+
+                if (spiErr != null)
+                    throw spiErr;
+
+                throw new IgniteSpiException("Failed to join cluster", e);
+            }
+        }
+    }
+
+    /**
+     * @param prevState Previous state in case of connect retry.
+     * @throws InterruptedException If interrupted.
+     */
+    private void joinTopology(@Nullable ZkRuntimeState prevState) throws InterruptedException {
+        if (!busyLock.enterBusy())
+            return;
+
+        try {
+            boolean reconnect = prevState != null;
+
+            // Need fire EVT_CLIENT_NODE_RECONNECTED event if reconnect after already joined.
+            boolean prevJoined = prevState != null && prevState.joined;
+
+            IgniteDiscoverySpiInternalListener internalLsnr = this.internalLsnr;
+
+            if (internalLsnr != null)
+                internalLsnr.beforeJoin(locNode, log);
+
+            if (locNode.isClient() && reconnect)
+                locNode.setAttributes(spi.getSpiContext().nodeAttributes());
+
+            marshalCredentialsOnJoin(locNode);
+
+            synchronized (stateMux) {
+                if (connState == ConnectionState.STOPPED)
+                    return;
+
+                connState = ConnectionState.STARTED;
+            }
+
+            ZkRuntimeState rtState = this.rtState = new ZkRuntimeState(prevJoined);
+
+            DiscoveryDataBag discoDataBag = new DiscoveryDataBag(locNode.id(), locNode.isClient());
+
+            exchange.collect(discoDataBag);
+
+            ZkJoiningNodeData joinData = new ZkJoiningNodeData(locNode, discoDataBag.joiningNodeData());
+
+            byte[] joinDataBytes;
+
+            try {
+                joinDataBytes = marshalZip(joinData);
+            }
+            catch (Exception e) {
+                throw new IgniteSpiException("Failed to marshal joining node data", e);
+            }
+
+            try {
+                rtState.zkClient = new ZookeeperClient(
+                    igniteInstanceName,
+                    log,
+                    connectString,
+                    sesTimeout,
+                    new ConnectionLossListener());
+            }
+            catch (Exception e) {
+                throw new IgniteSpiException("Failed to create Zookeeper client", e);
+            }
+
+            startJoin(rtState, prevState, joinDataBytes);
+        }
+        finally {
+            busyLock.leaveBusy();
+        }
+    }
+
+    /**
+     * @throws InterruptedException If interrupted.
+     */
+    private void initZkNodes() throws InterruptedException {
+        try {
+            ZookeeperClient client = rtState.zkClient;
+
+            if (!client.exists(zkPaths.clusterDir)) {
+                createRootPathParents(zkPaths.clusterDir, client);
+
+                client.createIfNeeded(zkPaths.clusterDir, null, PERSISTENT);
+            }
+
+            List<String> createdDirs = client.getChildren(zkPaths.clusterDir);
+
+            String[] requiredDirs = {
+                zkPaths.evtsPath,
+                zkPaths.joinDataDir,
+                zkPaths.customEvtsDir,
+                zkPaths.customEvtsPartsDir,
+                zkPaths.customEvtsAcksDir,
+                zkPaths.aliveNodesDir};
+
+            List<String> dirs = new ArrayList<>();
+
+            for (String dir : requiredDirs) {
+                String dir0 = dir.substring(zkPaths.clusterDir.length() + 1);
+
+                if (!createdDirs.contains(dir0))
+                    dirs.add(dir);
+            }
+
+            try {
+                if (!dirs.isEmpty())
+                    client.createAll(dirs, PERSISTENT);
+            }
+            catch (KeeperException.NodeExistsException e) {
+                if (log.isDebugEnabled())
+                    log.debug("Failed to create nodes using bulk operation: " + e);
+
+                for (String dir : dirs)
+                    client.createIfNeeded(dir, null, PERSISTENT);
+            }
+        }
+        catch (ZookeeperClientFailedException e) {
+            throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e);
+        }
+    }
+
+    /**
+     * @param rootDir Root directory.
+     * @param client Client.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    private void createRootPathParents(String rootDir, ZookeeperClient client)
+        throws ZookeeperClientFailedException, InterruptedException {
+        int startIdx = 0;
+
+        for (;;) {
+            int separatorIdx = rootDir.indexOf(ZkIgnitePaths.PATH_SEPARATOR, startIdx);
+
+            if (separatorIdx == -1)
+                break;
+
+            if (separatorIdx > 0) {
+                String path = rootDir.substring(0, separatorIdx);
+
+                client.createIfNeeded(path, null, CreateMode.PERSISTENT);
+            }
+
+            startIdx = separatorIdx + 1;
+        }
+    }
+
+    /**
+     * @param zkClient Client.
+     * @param basePath Base path.
+     * @param partCnt Parts count.
+     */
+    private void deleteMultiplePartsAsync(ZookeeperClient zkClient, String basePath, int partCnt) {
+        for (int i = 0; i < partCnt; i++) {
+            String path = multipartPathName(basePath, i);
+
+            zkClient.deleteIfExistsAsync(path);
+        }
+    }
+
+    /**
+     * @param zkClient Client.
+     * @param basePath Base path.
+     * @param partCnt Parts count.
+     * @return Read parts.
+     * @throws Exception If failed.
+     */
+    private byte[] readMultipleParts(ZookeeperClient zkClient, String basePath, int partCnt)
+        throws Exception {
+        assert partCnt >= 1;
+
+        if (partCnt > 1) {
+            List<byte[]> parts = new ArrayList<>(partCnt);
+
+            int totSize = 0;
+
+            for (int i = 0; i < partCnt; i++) {
+                byte[] part = zkClient.getData(multipartPathName(basePath, i));
+
+                parts.add(part);
+
+                totSize += part.length;
+            }
+
+            byte[] res = new byte[totSize];
+
+            int pos = 0;
+
+            for (int i = 0; i < partCnt; i++) {
+                byte[] part = parts.get(i);
+
+                System.arraycopy(part, 0, res, pos, part.length);
+
+                pos += part.length;
+            }
+
+            return res;
+        }
+        else
+            return zkClient.getData(multipartPathName(basePath, 0));
+    }
+
+    /**
+     * @param zkClient Client.
+     * @param basePath Base path.
+     * @param parts Data parts.
+     * @return Number of parts.
+     * @throws ZookeeperClientFailedException If client failed.
+     * @throws InterruptedException If interrupted.
+     */
+    private int saveMultipleParts(ZookeeperClient zkClient, String basePath, List<byte[]> parts)
+        throws ZookeeperClientFailedException, InterruptedException
+    {
+        assert parts.size() > 1;
+
+        for (int i = 0; i < parts.size(); i++) {
+            byte[] part = parts.get(i);
+
+            String path = multipartPathName(basePath, i);
+
+            zkClient.createIfNeeded(path, part, PERSISTENT);
+        }
+
+        return parts.size();
+    }
+
+    /**
+     * @param basePath Base path.
+     * @param part Part number.
+     * @return Path.
+     */
+    private static String multipartPathName(String basePath, int part) {
+        return basePath + String.format("%04d", part);
+    }
+
+    /**
+     * @param rtState Runtime state.
+     * @param joinDataBytes Joining node data.
+     * @param prevState Previous state in case of connect retry.
+     * @throws InterruptedException If interrupted.
+     */
+    private void startJoin(ZkRuntimeState rtState, @Nullable ZkRuntimeState prevState, final byte[] joinDataBytes)
+        throws InterruptedException
+    {
+        try {
+            long startTime = System.currentTimeMillis();
+
+            initZkNodes();
+
+            String prefix = UUID.randomUUID().toString();
+
+            rtState.init(new ZkWatcher(rtState), new AliveNodeDataWatcher(rtState));
+
+            ZookeeperClient zkClient = rtState.zkClient;
+
+            final int OVERHEAD = 5;
+
+            // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8193
+            String joinDataPath = zkPaths.joinDataDir + "/" + prefix + ":" + locNode.id();
+
+            if (zkClient.needSplitNodeData(joinDataPath, joinDataBytes, OVERHEAD)) {
+                List<byte[]> parts = zkClient.splitNodeData(joinDataPath, joinDataBytes, OVERHEAD);
+
+                rtState.joinDataPartCnt = parts.size();
+
+                saveMultipleParts(zkClient, joinDataPath + ":", parts);
+
+                joinDataPath = zkClient.createIfNeeded(
+                    joinDataPath,
+                    marshalZip(new ZkJoiningNodeData(parts.size())),
+                    PERSISTENT);
+            }
+            else {
+                joinDataPath = zkClient.createIfNeeded(
+                    joinDataPath,
+                    joinDataBytes,
+                    PERSISTENT);
+            }
+
+            rtState.locNodeZkPath = zkClient.createSequential(
+                prefix,
+                zkPaths.aliveNodesDir,
+                zkPaths.aliveNodePathForCreate(prefix, locNode),
+                null,
+                EPHEMERAL_SEQUENTIAL);
+
+            rtState.internalOrder = ZkIgnitePaths.aliveInternalId(rtState.locNodeZkPath);
+
+            if (log.isInfoEnabled()) {
+                log.info("Node started join [nodeId=" + locNode.id() +
+                    ", instanceName=" + locNode.attribute(ATTR_IGNITE_INSTANCE_NAME) +
+                    ", zkSessionId=0x" + Long.toHexString(rtState.zkClient.zk().getSessionId()) +
+                    ", joinDataSize=" + joinDataBytes.length +
+                    (rtState.joinDataPartCnt > 1 ? (", joinDataPartCnt=" + rtState.joinDataPartCnt) : "") +
+                    ", consistentId=" + locNode.consistentId() +
+                    ", initTime=" + (System.currentTimeMillis() - startTime) +
+                    ", nodePath=" + rtState.locNodeZkPath + ']');
+            }
+
+            /*
+            If node can not join due to validation error this error is reported in join data,
+            As a minor optimization do not start watch join data immediately, but only if do not receive
+            join event after some timeout.
+             */
+            CheckJoinErrorWatcher joinErrorWatcher = new CheckJoinErrorWatcher(5000, joinDataPath, rtState);
+
+            rtState.joinErrTo = joinErrorWatcher.timeoutObj;
+
+            if (locNode.isClient() && spi.getJoinTimeout() > 0) {
+                ZkTimeoutObject joinTimeoutObj = prevState != null ? prevState.joinTo : null;
+
+                if (joinTimeoutObj == null) {
+                    joinTimeoutObj = new JoinTimeoutObject(spi.getJoinTimeout());
+
+                    spi.getSpiContext().addTimeoutObject(joinTimeoutObj);
+                }
+
+                rtState.joinTo = joinTimeoutObj;
+            }
+
+            if (!locNode.isClient())
+                zkClient.getChildrenAsync(zkPaths.aliveNodesDir, null, new CheckCoordinatorCallback(rtState));
+
+            zkClient.getDataAsync(zkPaths.evtsPath, rtState.watcher, rtState.watcher);
+
+            spi.getSpiContext().addTimeoutObject(rtState.joinErrTo);
+        }
+        catch (IgniteCheckedException | ZookeeperClientFailedException e) {
+            throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e);
+        }
+    }
+
+    /**
+     * Authenticate local node.
+     *
+     * @param nodeAuth Authenticator.
+     * @param locCred Local security credentials for authentication.
+     * @throws IgniteSpiException If any error occurs.
+     */
+    private void localAuthentication(DiscoverySpiNodeAuthenticator nodeAuth, SecurityCredentials locCred){
+        assert nodeAuth != null;
+        assert locCred != null;
+
+        try {
+            SecurityContext subj = nodeAuth.authenticateNode(locNode, locCred);
+
+            // Note: exception message is checked in tests.
+            if (subj == null)
+                throw new IgniteSpiException("Authentication failed for local node.");
+
+            if (!(subj instanceof Serializable))
+                throw new IgniteSpiException("Authentication subject is not Serializable.");
+
+            Map<String, Object> attrs = new HashMap<>(locNode.attributes());
+
+            attrs.put(ATTR_SECURITY_SUBJECT_V2, U.marshal(marsh, subj));
+
+            locNode.setAttributes(attrs);
+        }
+        catch (Exception e) {
+            throw new IgniteSpiException("Failed to authenticate local node (will shutdown local node).", e);
+        }
+    }
+
+    /**
+     * @param node Node.
+     * @param zipBytes Zip-compressed marshalled security subject.
+     * @throws Exception If failed.
+     */
+    private void setNodeSecuritySubject(ZookeeperClusterNode node, byte[] zipBytes) throws Exception {
+        assert zipBytes != null;
+
+        Map<String, Object> attrs = new HashMap<>(node.getAttributes());
+
+        attrs.put(ATTR_SECURITY_SUBJECT_V2, unzip(zipBytes));
+
+        node.setAttributes(attrs);
+    }
+
+    /**
+     * @param node Node.
+     * @return Credentials.
+     * @throws IgniteCheckedException If failed to unmarshal.
+     */
+    private SecurityCredentials unmarshalCredentials(ZookeeperClusterNode node) throws Exception {
+        byte[] credBytes = (byte[])node.getAttributes().get(ATTR_SECURITY_CREDENTIALS);
+
+        if (credBytes == null)
+            return null;
+
+        return unmarshalZip(credBytes);
+    }
+
+    /**
+     * Marshalls credentials with discovery SPI marshaller (will replace attribute value).
+     *
+     * @param node Node to marshall credentials for.
+     * @throws IgniteSpiException If marshalling failed.
+     */
+    private void marshalCredentialsOnJoin(ZookeeperClusterNode node) throws IgniteSpiException {
+        try {
+            // Use security-unsafe getter.
+            Map<String, Object> attrs0 = node.getAttributes();
+
+            Object creds = attrs0.get(ATTR_SECURITY_CREDENTIALS);
+
+            if (creds != null) {
+                Map<String, Object> attrs = new HashMap<>(attrs0);
+
+                assert !(creds instanceof byte[]);
+
+                attrs.put(ATTR_SECURITY_CREDENTIALS, marshalZip(creds));
+
+                node.setAttributes(attrs);
+            }
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteSpiException("Failed to marshal node security credentials: " + node.id(), e);
+        }
+    }
+
+    /**
+     *
+     */
+    private class UpdateProcessedEventsTimeoutObject extends ZkTimeoutObject {
+        /** */
+        private final ZkRuntimeState rtState;
+
+        /**
+         * @param rtState Runtime state.
+         * @param timeout Timeout.
+         */
+        UpdateProcessedEventsTimeoutObject(ZkRuntimeState rtState, long timeout) {
+            super(timeout);
+
+            this.rtState = rtState;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onTimeout() {
+            runInWorkerThread(new ZkRunnable(rtState, ZookeeperDiscoveryImpl.this) {
+                @Override protected void run0() throws Exception {
+                    updateProcessedEventsOnTimeout(rtState, UpdateProcessedEventsTimeoutObject.this);
+                }
+            });
+        }
+    }
+
+    /**
+     *
+     */
+    private class JoinTimeoutObject extends ZkTimeoutObject {
+        /**
+         * @param timeout Timeout.
+         */
+        JoinTimeoutObject(long timeout) {
+            super(timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onTimeout() {
+            if (cancelled || rtState.joined)
+                return;
+
+            runInWorkerThread(new Runnable() {
+                @Override public void run() {
+                    synchronized (stateMux) {
+                        if (cancelled || rtState.joined)
+                            return;
+
+                        if (connState == ConnectionState.STOPPED)
+                            return;
+
+                        connState = ConnectionState.STOPPED;
+                    }
+
+                    U.warn(log, "Failed to connect to cluster, either connection to ZooKeeper can not be established or there " +
+                        "are no alive server nodes (consider increasing 'joinTimeout' configuration  property) [" +
+                        "joinTimeout=" + spi.getJoinTimeout() + ']');
+
+                    // Note: exception message is checked in tests.
+                    onSegmented(new IgniteSpiException("Failed to connect to cluster within configured timeout"));
+                }
+            });
+        }
+    }
+
+    /**
+     *
+     */
+    private class CheckJoinErrorWatcher extends ZkAbstractWatcher implements AsyncCallback.DataCallback {
+        /** */
+        private final String joinDataPath;
+
+        /** */
+        private ZkTimeoutObject timeoutObj;
+
+        /**
+         * @param timeout Timeout.
+         * @param joinDataPath0 Node joined data path.
+         * @param rtState0 State.
+         */
+        CheckJoinErrorWatcher(long timeout, String joinDataPath0, ZkRuntimeState rtState0) {
+            super(rtState0, ZookeeperDiscoveryImpl.this);
+
+            this.joinDataPath = joinDataPath0;
+
+            timeoutObj = new ZkTimeoutObject(timeout) {
+                @Override public void onTimeout() {
+                    if (rtState.errForClose != null || rtState.joined)
+                        return;
+
+                    synchronized (stateMux) {
+                        if (connState != ConnectionState.STARTED)
+                            return;
+                    }
+
+                    rtState.zkClient.getDataAsync(joinDataPath,
+                        CheckJoinErrorWatcher.this,
+                        CheckJoinErrorWatcher.this);
+                }
+            };
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) {
+            if (rc != 0)
+                return;
+
+            if (!onProcessStart())
+                return;
+
+            try {
+                Object obj = unmarshalZip(data);
+
+                if (obj instanceof ZkInternalJoinErrorMessage) {
+                    ZkInternalJoinErrorMessage joinErr = (ZkInternalJoinErrorMessage)obj;
+
+                    onSegmented(new IgniteSpiException(joinErr.err));
+                }
+
+                onProcessEnd();
+            }
+            catch (Throwable e) {
+                onProcessError(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void process0(WatchedEvent evt) {
+            if (rtState.errForClose != null || rtState.joined)
+                return;
+
+            if (evt.getType() == Event.EventType.NodeDataChanged)
+                rtState.zkClient.getDataAsync(evt.getPath(), this, this);
+        }
+    }
+
+    /**
+     * @param aliveNodes Alive nodes.
+     * @throws Exception If failed.
+     */
+    private void checkIsCoordinator(final List<String> aliveNodes) throws Exception {
+        assert !locNode.isClient();
+
+        TreeMap<Long, String> aliveSrvs = new TreeMap<>();
+
+        long locInternalOrder = rtState.internalOrder;
+
+        for (String aliveNodePath : aliveNodes) {
+            if (ZkIgnitePaths.aliveNodeClientFlag(aliveNodePath))
+                continue;
+
+            Long internalId = ZkIgnitePaths.aliveInternalId(aliveNodePath);
+
+            aliveSrvs.put(internalId, aliveNodePath);
+        }
+
+        assert !aliveSrvs.isEmpty();
+
+        Map.Entry<Long, String> crdE = aliveSrvs.firstEntry();
+
+        if (locInternalOrder == crdE.getKey())
+            onBecomeCoordinator(aliveNodes);
+        else {
+            assert aliveSrvs.size() > 1 : aliveSrvs;
+
+            Map.Entry<Long, String> prevE = aliveSrvs.floorEntry(locInternalOrder - 1);
+
+            assert prevE != null;
+
+            if (log.isInfoEnabled()) {
+                log.info("Discovery coordinator already exists, watch for previous server node [" +
+                    "locId=" + locNode.id() +
+                    ", watchPath=" + prevE.getValue() + ']');
+             }
+
+            PreviousNodeWatcher watcher = new ServerPreviousNodeWatcher(rtState);
+
+            rtState.zkClient.existsAsync(zkPaths.aliveNodesDir + "/" + prevE.getValue(), watcher, watcher);
+        }
+    }
+
+    /**
+     * @param aliveNodes Alive nodes.
+     * @throws Exception If failed.
+     */
+    private void checkClientsStatus(final List<String> aliveNodes) throws Exception {
+        assert locNode.isClient() : locNode;
+        assert rtState.joined;
+        assert rtState.evtsData != null;
+
+        TreeMap<Long, String> aliveClients = new TreeMap<>();
+
+        String srvPath = null;
+        Long srvInternalOrder = null;
+
+        long locInternalOrder = rtState.internalOrder;
+
+        for (String aliveNodePath : aliveNodes) {
+            Long internalId = ZkIgnitePaths.aliveInternalId(aliveNodePath);
+
+            if (ZkIgnitePaths.aliveNodeClientFlag(aliveNodePath))
+                aliveClients.put(internalId, aliveNodePath);
+            else {
+                if (srvInternalOrder == null || internalId < srvInternalOrder) {
+                    srvPath = aliveNodePath;
+                    srvInternalOrder = internalId;
+                }
+            }
+        }
+
+        assert !aliveClients.isEmpty();
+
+        Map.Entry<Long, String> oldest = aliveClients.firstEntry();
+
+        boolean oldestClient = locInternalOrder == oldest.getKey();
+
+        if (srvPath == null) {
+            if (oldestClient) {
+                Stat stat = new Stat();
+
+                ZkDiscoveryEventsData prevEvts = rtState.evtsData;
+
+                byte[] evtsBytes = rtState.zkClient.getData(zkPaths.evtsPath, stat);
+
+                assert evtsBytes.length > 0;
+
+                ZkDiscoveryEventsData newEvts = unmarshalZip(evtsBytes);
+
+                if (prevEvts.clusterId.equals(newEvts.clusterId)) {
+                    U.warn(log, "All server nodes failed, notify all clients [locId=" + locNode.id() + ']');
+
+                    generateNoServersEvent(newEvts, stat);
+                }
+                else
+                    U.warn(log, "All server nodes failed (received events from new cluster).");
+            }
+        }
+        else {
+            String watchPath;
+
+            if (oldestClient) {
+                watchPath = srvPath;
+
+                if (log.isInfoEnabled()) {
+                    log.info("Servers exists, watch for server node [locId=" + locNode.id() +
+                        ", watchPath=" + watchPath + ']');
+                }
+            }
+            else {
+                assert aliveClients.size() > 1 : aliveClients;
+
+                Map.Entry<Long, String> prevE = aliveClients.floorEntry(locInternalOrder - 1);
+
+                assert prevE != null;
+
+                watchPath = prevE.getValue();
+
+                if (log.isInfoEnabled()) {
+                    log.info("Servers exists, watch for previous node [locId=" + locNode.id() +
+                        ", watchPath=" + watchPath + ']');
+                }
+            }
+
+            PreviousNodeWatcher watcher = new ClientPreviousNodeWatcher(rtState);
+
+            rtState.zkClient.existsAsync(zkPaths.aliveNodesDir + "/" + watchPath, watcher, watcher);
+        }
+    }
+
+    /**
+     * @param evtsData Events data.
+     * @param evtsStat Events zookeeper state.
+     * @throws Exception If failed.
+     */
+    private void generateNoServersEvent(ZkDiscoveryEventsData evtsData, Stat evtsStat) throws Exception {
+        evtsData.evtIdGen++;
+
+        ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
+            evtsData.evtIdGen,
+            0L,
+            evtsData.topVer,
+            locNode.id(),
+            new ZkNoServersMessage(),
+            null);
+
+        Collection<ZookeeperClusterNode> nodesToAck = Collections.emptyList();
+
+        evtsData.addEvent(nodesToAck, evtData);
+
+        byte[] newEvtsBytes = marshalZip(evtsData);
+
+        try {
+            rtState.zkClient.setData(zkPaths.evtsPath, newEvtsBytes, evtsStat.getVersion());
+        }
+        catch (KeeperException.BadVersionException e) {
+            // Version can change if new cluster started and saved new events.
+            if (log.isDebugEnabled())
+                log.debug("Failed to save no servers message");
+        }
+    }
+
+    /**
+     * @param lastEvts Last events from previous coordinator.
+     * @throws Exception If failed.
+     */
+    private void previousCoordinatorCleanup(ZkDiscoveryEventsData lastEvts) throws Exception {
+        for (ZkDiscoveryEventData evtData : lastEvts.evts.values()) {
+            if (evtData instanceof ZkDiscoveryCustomEventData) {
+                ZkDiscoveryCustomEventData evtData0 = (ZkDiscoveryCustomEventData)evtData;
+
+                // It is possible previous coordinator failed before finished cleanup.
+                if (evtData0.msg instanceof ZkCommunicationErrorResolveFinishMessage) {
+                    try {
+                        ZkCommunicationErrorResolveFinishMessage msg =
+                            (ZkCommunicationErrorResolveFinishMessage)evtData0.msg;
+
+                        ZkCommunicationErrorResolveResult res = unmarshalZip(
+                            ZkDistributedCollectDataFuture.readResult(rtState.zkClient, zkPaths, msg.futId));
+
+                        deleteAliveNodes(res.killedNodes);
+                    }
+                    catch (KeeperException.NoNodeException ignore) {
+                        // No-op.
+                    }
+                }
+                else if (evtData0.resolvedMsg instanceof ZkForceNodeFailMessage)
+                    deleteAliveNode(((ZkForceNodeFailMessage)evtData0.resolvedMsg).nodeInternalId);
+            }
+        }
+    }
+
+    /**
+     * @param aliveNodes Alive nodes paths.
+     * @throws Exception If failed.
+     */
+    private void onBecomeCoordinator(List<String> aliveNodes) throws Exception {
+        ZkDiscoveryEventsData prevEvts = processNewEvents(rtState.zkClient.getData(zkPaths.evtsPath));
+
+        rtState.crd = true;
+
+        if (rtState.joined) {
+            if (log.isInfoEnabled())
+                log.info("Node is new discovery coordinator [locId=" + locNode.id() + ']');
+
+            assert locNode.order() > 0 : locNode;
+            assert rtState.evtsData != null;
+
+            previousCoordinatorCleanup(rtState.evtsData);
+
+            UUID futId = rtState.evtsData.communicationErrorResolveFutureId();
+
+            if (futId != null) {
+                if (log.isInfoEnabled()) {
+                    log.info("New discovery coordinator will handle already started cluster-wide communication " +
+                        "error resolve [reqId=" + futId + ']');
+                }
+
+                ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
+
+                ZkDistributedCollectDataFuture collectResFut = collectCommunicationStatusFuture(futId);
+
+                if (fut != null)
+                    fut.nodeResultCollectFuture(collectResFut);
+            }
+
+            for (ZkDiscoveryEventData evtData : rtState.evtsData.evts.values())
+                evtData.initRemainingAcks(rtState.top.nodesByOrder.values());
+
+            handleProcessedEvents("crd");
+        }
+        else {
+            String locAlivePath = rtState.locNodeZkPath.substring(rtState.locNodeZkPath.lastIndexOf('/') + 1);
+
+            deleteJoiningNodeData(locNode.id(),
+                ZkIgnitePaths.aliveNodePrefixId(locAlivePath),
+                rtState.joinDataPartCnt);
+
+            DiscoverySpiNodeAuthenticator nodeAuth = spi.getAuthenticator();
+
+            if (nodeAuth != null) {
+                try {
+                    if (log.isInfoEnabled()) {
+                        log.info("Node is first server node in cluster, try authenticate local node " +
+                            "[locId=" + locNode.id() + ']');
+                    }
+
+                    localAuthentication(nodeAuth, unmarshalCredentials(locNode));
+                }
+                catch (Exception e) {
+                    U.warn(log, "Local node authentication failed: " + e, e);
+
+                    onSegmented(e);
+
+                    // Stop any further processing.
+                    throw new ZookeeperClientFailedException("Local node authentication failed: " + e);
+                }
+            }
+
+            newClusterStarted(prevEvts);
+        }
+
+        rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+        rtState.zkClient.getChildrenAsync(zkPaths.customEvtsDir, rtState.watcher, rtState.watcher);
+
+        for (String alivePath : aliveNodes)
+            watchAliveNodeData(alivePath);
+    }
+
+    /**
+     * @param alivePath Node path.
+     */
+    private void watchAliveNodeData(String alivePath) {
+        assert rtState.locNodeZkPath != null;
+
+        String path = zkPaths.aliveNodesDir + "/" + alivePath;
+
+        if (!path.equals(rtState.locNodeZkPath))
+            rtState.zkClient.getDataAsync(path, rtState.aliveNodeDataWatcher, rtState.aliveNodeDataWatcher);
+    }
+
+    /**
+     * @param aliveNodes ZK nodes representing alive cluster nodes.
+     * @throws Exception If failed.
+     */
+    private void generateTopologyEvents(List<String> aliveNodes) throws Exception {
+        assert rtState.crd;
+
+        if (log.isInfoEnabled())
+            log.info("Process alive nodes change [alives=" + aliveNodes.size() + "]");
+
+        if (rtState.updateAlives) {
+            aliveNodes = rtState.zkClient.getChildren(zkPaths.aliveNodesDir);
+
+            rtState.updateAlives = false;
+        }
+
+        TreeMap<Long, String> alives = new TreeMap<>();
+
+        for (String child : aliveNodes) {
+            Long internalId = ZkIgnitePaths.aliveInternalId(child);
+
+            Object old = alives.put(internalId, child);
+
+            assert old == null;
+        }
+
+        TreeMap<Long, ZookeeperClusterNode> curTop = new TreeMap<>(rtState.top.nodesByOrder);
+
+        int newEvts = 0;
+
+        final int MAX_NEW_EVTS = IgniteSystemProperties.getInteger(IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS, 100);
+
+        List<ZookeeperClusterNode> failedNodes = null;
+
+        for (Map.Entry<Long, ZookeeperClusterNode> e : rtState.top.nodesByInternalId.entrySet()) {
+            if (!alives.containsKey(e.getKey())) {
+                ZookeeperClusterNode failedNode = e.getValue();
+
+                if (failedNodes == null)
+                    failedNodes = new ArrayList<>();
+
+                failedNodes.add(failedNode);
+
+                generateNodeFail(curTop, failedNode);
+
+                newEvts++;
+
+                if (newEvts == MAX_NEW_EVTS) {
+                    saveAndProcessNewEvents();
+
+                    if (log.isInfoEnabled()) {
+                        log.info("Delay alive nodes change process, max event threshold reached [newEvts=" + newEvts +
+                            ", totalEvts=" + rtState.evtsData.evts.size() + ']');
+                    }
+
+                    handleProcessedEventsOnNodesFail(failedNodes);
+
+                    throttleNewEventsGeneration();
+
+                    rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+
+                    return;
+                }
+            }
+        }
+
+        // Process failures before processing join, otherwise conflicts are possible in case of fast node stop/re-start.
+        if (newEvts > 0) {
+            saveAndProcessNewEvents();
+
+            handleProcessedEventsOnNodesFail(failedNodes);
+
+            rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+
+            return;
+        }
+
+        generateJoinEvents(curTop, alives, MAX_NEW_EVTS);
+
+        if (failedNodes != null)
+            handleProcessedEventsOnNodesFail(failedNodes);
+    }
+
+    /**
+     * @param curTop Current topology.
+     * @param alives Alive znodes.
+     * @param MAX_NEW_EVTS Max event to process.
+     * @throws Exception If failed.
+     */
+    private void generateJoinEvents(TreeMap<Long, ZookeeperClusterNode> curTop,
+        TreeMap<Long, String> alives,
+        final int MAX_NEW_EVTS) throws Exception
+    {
+       ZkBulkJoinContext joinCtx = new ZkBulkJoinContext();
+
+       for (Map.Entry<Long, String> e : alives.entrySet()) {
+           Long internalId = e.getKey();
+
+           if (!rtState.top.nodesByInternalId.containsKey(internalId)) {
+               UUID rslvFutId = rtState.evtsData.communicationErrorResolveFutureId();
+
+               if (rslvFutId != null) {
+                   if (log.isInfoEnabled()) {
+                       log.info("Delay alive nodes change process while communication error resolve " +
+                           "is in progress [reqId=" + rslvFutId + ']');
+                   }
+
+                   break;
+               }
+
+               processJoinOnCoordinator(joinCtx, curTop, internalId, e.getValue());
+
+               if (joinCtx.nodes() == MAX_NEW_EVTS) {
+                   generateBulkJoinEvent(curTop, joinCtx);
+
+                   if (log.isInfoEnabled()) {
+                       log.info("Delay alive nodes change process, max event threshold reached [" +
+                           "newEvts=" + joinCtx.nodes() +
+                           ", totalEvts=" + rtState.evtsData.evts.size() + ']');
+                   }
+
+                   throttleNewEventsGeneration();
+
+                   rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+
+                   return;
+               }
+           }
+       }
+
+       if (joinCtx.nodes() > 0)
+           generateBulkJoinEvent(curTop, joinCtx);
+    }
+
+    /**
+     * @param curTop Current topology.
+     * @param joinCtx Joined nodes context.
+     * @throws Exception If failed.
+     */
+    private void generateBulkJoinEvent(TreeMap<Long, ZookeeperClusterNode> curTop, ZkBulkJoinContext joinCtx)
+        throws Exception
+    {
+        rtState.evtsData.evtIdGen++;
+
+        long evtId = rtState.evtsData.evtIdGen;
+
+        List<T2<ZkJoinedNodeEvtData, Map<Integer, Serializable>>> nodes = joinCtx.nodes;
+
+        assert nodes != null && nodes.size() > 0;
+
+        int nodeCnt = nodes.size();
+
+        List<ZkJoinedNodeEvtData> joinedNodes = new ArrayList<>(nodeCnt);
+
+        Map<Long, byte[]> discoDataMap = U.newHashMap(nodeCnt);
+        Map<Long, Long> dupDiscoData = null;
+
+        for (int i = 0; i < nodeCnt; i++) {
+            T2<ZkJoinedNodeEvtData, Map<Integer, Serializable>> nodeEvtData = nodes.get(i);
+
+            Map<Integer, Serializable> discoData = nodeEvtData.get2();
+
+            byte[] discoDataBytes = U.marshal(marsh, discoData);
+
+            Long dupDataNode = null;
+
+            for (Map.Entry<Long, byte[]> e : discoDataMap.entrySet()) {
+                if (Arrays.equals(discoDataBytes, e.getValue())) {
+                    dupDataNode = e.getKey();
+
+                    break;
+                }
+            }
+
+            long nodeTopVer = nodeEvtData.get1().topVer;
+
+            if (dupDataNode != null) {
+                if (dupDiscoData == null)
+                    dupDiscoData = new HashMap<>();
+
+                Long old = dupDiscoData.put(nodeTopVer, dupDataNode);
+
+                assert old == null : old;
+            }
+            else
+                discoDataMap.put(nodeTopVer, discoDataBytes);
+
+            joinedNodes.add(nodeEvtData.get1());
+        }
+
+        int overhead = 5;
+
+        ZkJoinEventDataForJoined dataForJoined = new ZkJoinEventDataForJoined(
+            new ArrayList<>(curTop.values()),
+            discoDataMap,
+            dupDiscoData);
+
+        byte[] dataForJoinedBytes = marshalZip(dataForJoined);
+
+        long addDataStart = System.currentTimeMillis();
+
+        int dataForJoinedPartCnt = saveData(zkPaths.joinEventDataPathForJoined(evtId),
+            dataForJoinedBytes,
+            overhead);
+
+        long addDataTime = System.currentTimeMillis() - addDataStart;
+
+        ZkDiscoveryNodeJoinEventData evtData = new ZkDiscoveryNodeJoinEventData(
+            evtId,
+            rtState.evtsData.topVer,
+            joinedNodes,
+            dataForJoinedPartCnt);
+
+        rtState.evtsData.addEvent(curTop.values(), evtData);
+
+        if (log.isInfoEnabled()) {
+            if (nodeCnt > 1) {
+                log.info("Generated NODE_JOINED bulk event [" +
+                    "nodeCnt=" + nodeCnt +
+                    ", dataForJoinedSize=" + dataForJoinedBytes.length +
+                    ", dataForJoinedPartCnt=" + dataForJoinedPartCnt +
+                    ", addDataTime=" + addDataTime +
+                    ", evt=" + evtData + ']');
+            }
+            else {
+                log.info("Generated NODE_JOINED event [" +
+                    "dataForJoinedSize=" + dataForJoinedBytes.length +
+                    ", dataForJoinedPartCnt=" + dataForJoinedPartCnt +
+                    ", addDataTime=" + addDataTime +
+                    ", evt=" + evtData + ']');
+            }
+        }
+
+        saveAndProcessNewEvents();
+    }
+
+    /**
+     *
+     */
+    private void throttleNewEventsGeneration() {
+        long delay = IgniteSystemProperties.getLong(IGNITE_ZOOKEEPER_DISCOVERY_SPI_EVTS_THROTTLE, 0);
+
+        if (delay > 0) {
+            if (log.isInfoEnabled())
+                log.info("Sleep delay before generate new events [delay=" + delay + ']');
+
+            try {
+                Thread.sleep(delay);
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+            }
+        }
+    }
+
+    /**
+     * @param nodeId Node ID.
+     * @param prefixId Path prefix.
+     * @return Join data.
+     * @throws Exception If failed.
+     */
+    private ZkJoiningNodeData unmarshalJoinData(UUID nodeId, UUID prefixId) throws Exception {
+        String joinDataPath = zkPaths.joiningNodeDataPath(nodeId, prefixId);
+
+        byte[] joinData = rtState.zkClient.getData(joinDataPath);
+
+        Object dataObj = unmarshalZip(joinData);
+
+        if (!(dataObj instanceof ZkJoiningNodeData))
+            throw new Exception("Invalid joined node data: " + dataObj);
+
+        ZkJoiningNodeData joiningNodeData = (ZkJoiningNodeData)dataObj;
+
+        if (joiningNodeData.partCount() > 1) {
+            joinData = readMultipleParts(rtState.zkClient, joinDataPath + ":", joiningNodeData.partCount());
+
+            joiningNodeData = unmarshalZip(joinData);
+        }
+
+        return joiningNodeData;
+    }
+
+    /**
+     * @param nodeId Node ID.
+     * @param prefixId Path prefix.
+     * @param aliveNodePath Node path.
+     * @return Join data.
+     * @throws Exception If failed.
+     */
+    private Object unmarshalJoinDataOnCoordinator(UUID nodeId, UUID prefixId, String aliveNodePath) throws Exception {
+        String joinDataPath = zkPaths.joiningNodeDataPath(nodeId, prefixId);
+
+        byte[] joinData = rtState.zkClient.getData(joinDataPath);
+
+        Object dataObj;
+
+        try {
+            dataObj = unmarshalZip(joinData);
+
+            if (dataObj instanceof ZkInternalJoinErrorMessage)
+                return dataObj;
+        }
+        catch (Exception e) {
+            U.error(log, "Failed to unmarshal joining node data [nodePath=" + aliveNodePath + "']", e);
+
+            return new ZkInternalJoinErrorMessage(ZkIgnitePaths.aliveInternalId(aliveNodePath),
+                "Failed to unmarshal join data: " + e);
+        }
+
+        assert dataObj instanceof ZkJoiningNodeData : dataObj;
+
+        ZkJoiningNodeData joiningNodeData = (ZkJoiningNodeData)dataObj;
+
+        if (joiningNodeData.partCount() > 1) {
+            joinData = readMultipleParts(rtState.zkClient, joinDataPath + ":", joiningNodeData.partCount());
+
+            try {
+                joiningNodeData = unmarshalZip(joinData);
+            }
+            catch (Exception e) {
+                U.error(log, "Failed to unmarshal joining node data [nodePath=" + aliveNodePath + "']", e);
+
+                return new ZkInternalJoinErrorMessage(ZkIgnitePaths.aliveInternalId(aliveNodePath),
+                    "Failed to unmarshal join data: " + e);
+            }
+        }
+
+        assert joiningNodeData.node() != null : joiningNodeData;
+
+        return joiningNodeData;
+    }
+
+    /**
+     * @param joinCtx Joined nodes context.
+     * @param curTop Current nodes.
+     * @param internalId Joined node internal ID.
+     * @param aliveNodePath Joined node path.
+     * @throws Exception If failed.
+     */
+    private void processJoinOnCoordinator(
+        ZkBulkJoinContext joinCtx,
+        TreeMap<Long, ZookeeperClusterNode> curTop,
+        long internalId,
+        String aliveNodePath)
+        throws Exception
+    {
+        UUID nodeId = ZkIgnitePaths.aliveNodeId(aliveNodePath);
+        UUID prefixId = ZkIgnitePaths.aliveNodePrefixId(aliveNodePath);
+
+        Object data = unmarshalJoinDataOnCoordinator(nodeId, prefixId, aliveNodePath);
+
+        if (data instanceof ZkJoiningNodeData) {
+            ZkJoiningNodeData joiningNodeData = (ZkJoiningNodeData)data;
+
+            ZkNodeValidateResult validateRes = validateJoiningNode(joiningNodeData.node());
+
+            if (validateRes.err == null) {
+                ZookeeperClusterNode joinedNode = joiningNodeData.node();
+
+                assert nodeId.equals(joinedNode.id()) : joiningNodeData.node();
+
+                addJoinedNode(
+                    joinCtx,
+                    curTop,
+                    joiningNodeData,
+                    internalId,
+                    prefixId,
+                    validateRes.secSubjZipBytes);
+
+                watchAliveNodeData(aliveNodePath);
+            }
+            else {
+                ZkInternalJoinErrorMessage joinErr = new ZkInternalJoinErrorMessage(
+                    ZkIgnitePaths.aliveInternalId(aliveNodePath),
+                    validateRes.err);
+
+                processJoinError(aliveNodePath, nodeId, prefixId, joinErr);
+            }
+        }
+        else {
+            assert data instanceof ZkInternalJoinErrorMessage : data;
+
+            processJoinError(aliveNodePath, nodeId, prefixId, (ZkInternalJoinErrorMessage)data);
+        }
+    }
+
+    /**
+     * @param aliveNodePath Joined node path.
+     * @param nodeId Node ID.
+     * @param prefixId Path prefix ID.
+     * @param joinErr Join error message.
+     * @throws Exception If failed.
+     */
+    private void processJoinError(String aliveNodePath,
+        UUID nodeId,
+        UUID prefixId,
+        ZkInternalJoinErrorMessage joinErr) throws Exception {
+        ZookeeperClient client = rtState.zkClient;
+
+        if (joinErr.notifyNode) {
+            String joinDataPath = zkPaths.joiningNodeDataPath(nodeId, prefixId);
+
+            client.setData(joinDataPath, marshalZip(joinErr), -1);
+
+            client.deleteIfExists(zkPaths.aliveNodesDir + "/" + aliveNodePath, -1);
+        }
+        else {
+            if (log.isInfoEnabled())
+                log.info("Ignore join data, node was failed by previous coordinator: " + aliveNodePath);
+
+            client.deleteIfExists(zkPaths.aliveNodesDir + "/" + aliveNodePath, -1);
+        }
+    }
+
+    /**
+     * @param node Joining node.
+     * @return Validation result.
+     */
+    private ZkNodeValidateResult validateJoiningNode(ZookeeperClusterNode node) {
+        ZookeeperClusterNode node0 = rtState.top.nodesById.get(node.id());
+
+        if (node0 != null) {
+            U.error(log, "Failed to include node in cluster, node with the same ID already exists [joiningNode=" + node +
+                ", existingNode=" + node0 + ']');
+
+            // Note: exception message is checked in tests.
+            return new ZkNodeValidateResult("Node with the same ID already exists: " + node0);
+        }
+
+        ZkNodeValidateResult res = authenticateNode(node);
+
+        if (res.err != null)
+            return res;
+
+        IgniteNodeValidationResult err = spi.getSpiContext().validateNode(node);
+
+        if (err != null) {
+            LT.warn(log, err.message());
+
+            res.err = err.sendMessage();
+        }
+
+        return res;
+    }
+
+    /**
+     * @param node Node.
+     * @return Validation result.
+     */
+    private ZkNodeValidateResult authenticateNode(ZookeeperClusterNode node) {
+        DiscoverySpiNodeAuthenticator nodeAuth = spi.getAuthenticator();
+
+        if (nodeAuth == null)
+            return new ZkNodeValidateResult((byte[])null);
+
+        SecurityCredentials cred;
+
+        try {
+            cred = unmarshalCredentials(node);
+        }
+        catch (Exception e) {
+            U.error(log, "Failed to unmarshal node credentials: " + e, e);
+
+            return new ZkNodeValidateResult("Failed to unmarshal node credentials");
+        }
+
+        SecurityContext subj = nodeAuth.authenticateNode(node, cred);
+
+        if (subj == null) {
+            U.warn(log, "Authentication failed [nodeId=" + node.id() +
+                    ", addrs=" + U.addressesAsString(node) + ']',
+                "Authentication failed [nodeId=" + U.id8(node.id()) + ", addrs=" +
+                    U.addressesAsString(node) + ']');
+
+            // Note: exception message test is checked in tests.
+            return new ZkNodeValidateResult("Authentication failed");
+        }
+
+        if (!(subj instanceof Serializable)) {
+            U.warn(log, "Authentication subject is not Serializable [nodeId=" + node.id() +
+                    ", addrs=" + U.addressesAsString(node) + ']',
+                "Authentication subject is not Serializable [nodeId=" + U.id8(node.id()) +
+                    ", addrs=" +
+                    U.addressesAsString(node) + ']');
+
+            return new ZkNodeValidateResult("Authentication subject is not serializable");
+        }
+
+        byte[] secSubjZipBytes;
+
+        try {
+            secSubjZipBytes = marshalZip(subj);
+        }
+        catch (Exception e) {
+            U.error(log, "Failed to marshal node security subject: " + e, e);
+
+            return new ZkNodeValidateResult("Failed to marshal node security subject");
+        }
+
+        return new ZkNodeValidateResult(secSubjZipBytes);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void saveAndProcessNewEvents() throws Exception {
+        if (stopping())
+            return;
+
+        long start = System.currentTimeMillis();
+
+        byte[] evtsBytes = marshalZip(rtState.evtsData);
+
+        rtState.zkClient.setData(zkPaths.evtsPath, evtsBytes, -1);
+
+        long time = System.currentTimeMillis() - start;
+
+        if (prevSavedEvtsTopVer != rtState.evtsData.topVer) {
+            if (log.isInfoEnabled()) {
+                log.info("Discovery coordinator saved new topology events [topVer=" + rtState.evtsData.topVer +
+                    ", size=" + evtsBytes.length +
+                    ", evts=" + rtState.evtsData.evts.size() +
+                    ", lastEvt=" + rtState.evtsData.evtIdGen +
+                    ", saveTime=" + time + ']');
+            }
+
+            prevSavedEvtsTopVer = rtState.evtsData.topVer;
+        }
+        else if (log.isDebugEnabled()) {
+            log.debug("Discovery coordinator saved new topology events [topVer=" + rtState.evtsData.topVer +
+                ", size=" + evtsBytes.length +
+                ", evts=" + rtState.evtsData.evts.size() +
+                ", lastEvt=" + rtState.evtsData.evtIdGen +
+                ", saveTime=" + time + ']');
+        }
+
+        processNewEvents(rtState.evtsData);
+    }
+
+    /**
+     * @param curTop Current topology.
+     * @param failedNode Failed node.
+     */
+    private void generateNodeFail(TreeMap<Long, ZookeeperClusterNode> curTop, ZookeeperClusterNode failedNode) {
+        Object rmvd = curTop.remove(failedNode.order());
+
+        assert rmvd != null;
+
+        rtState.evtsData.topVer++;
+        rtState.evtsData.evtIdGen++;
+
+        ZkDiscoveryNodeFailEventData evtData = new ZkDiscoveryNodeFailEventData(
+            rtState.evtsData.evtIdGen,
+            rtState.evtsData.topVer,
+            failedNode.internalId());
+
+        rtState.evtsData.addEvent(curTop.values(), evtData);
+
+        if (log.isInfoEnabled())
+            log.info("Generated NODE_FAILED event [evt=" + evtData + ']');
+    }
+
+    /**
+     * @param curTop Current nodes.
+     * @param joiningNodeData Join data.
+     * @param internalId Joined node internal ID.
+     * @param prefixId Unique path prefix.
+     * @param secSubjZipBytes Marshalled security subject.
+     * @throws Exception If failed.
+     */
+    private void addJoinedNode(
+        ZkBulkJoinContext joinCtx,
+        TreeMap<Long, ZookeeperClusterNode> curTop,
+        ZkJoiningNodeData joiningNodeData,
+        long internalId,
+        UUID prefixId,
+        @Nullable byte[] secSubjZipBytes)
+        throws Exception
+    {
+        ZookeeperClusterNode joinedNode = joiningNodeData.node();
+
+        UUID nodeId = joinedNode.id();
+
+        rtState.evtsData.topVer++;
+
+        joinedNode.order(rtState.evtsData.topVer);
+        joinedNode.internalId(internalId);
+
+        DiscoveryDataBag joiningNodeBag = new DiscoveryDataBag(nodeId, joiningNodeData.node().isClient());
+
+        joiningNodeBag.joiningNodeData(joiningNodeData.discoveryData());
+
+        exchange.onExchange(joiningNodeBag);
+
+        DiscoveryDataBag collectBag = new DiscoveryDataBag(nodeId,
+            new HashSet<Integer>(),
+            joiningNodeData.node().isClient());
+
+        collectBag.joiningNodeData(joiningNodeBag.joiningNodeData());
+
+        exchange.collect(collectBag);
+
+        Map<Integer, Serializable> commonData = collectBag.commonData();
+
+        Object old = curTop.put(joinedNode.order(), joinedNode);
+
+        assert old == null;
+
+        int overhead = 5;
+
+        int secSubjPartCnt = 0;
+
+        if (secSubjZipBytes != null) {
+            secSubjPartCnt = saveData(zkPaths.joinEventSecuritySubjectPath(joinedNode.order()),
+                secSubjZipBytes,
+                overhead);
+
+            assert secSubjPartCnt > 0 : secSubjPartCnt;
+
+            setNodeSecuritySubject(joinedNode, secSubjZipBytes);
+        }
+
+        ZkJoinedNodeEvtData nodeEvtData = new ZkJoinedNodeEvtData(
+            rtState.evtsData.topVer,
+            joinedNode.id(),
+            joinedNode.internalId(),
+            prefixId,
+            joiningNodeData.partCount(),
+            secSubjPartCnt);
+
+        nodeEvtData.joiningNodeData = joiningNodeData;
+
+        joinCtx.addJoinedNode(nodeEvtData, commonData);
+
+        rtState.evtsData.onNodeJoin(joinedNode);
+    }
+
+    /**
+     * @param path Path to save.
+     * @param bytes Bytes to save.
+     * @param overhead Extra overhead.
+     * @return Parts count.
+     * @throws Exception If failed.
+     */
+    private int saveData(String path, byte[] bytes, int overhead) throws Exception {
+        int dataForJoinedPartCnt = 1;
+
+        if (rtState.zkClient.needSplitNodeData(path, bytes, overhead)) {
+            dataForJoinedPartCnt = saveMultipleParts(rtState.zkClient,
+                path,
+                rtState.zkClient.splitNodeData(path, bytes, overhead));
+        }
+        else {
+            rtState.zkClient.createIfNeeded(multipartPathName(path, 0),
+                bytes,
+                PERSISTENT);
+        }
+
+        return dataForJoinedPartCnt;
+    }
+
+    /**
+     * @param prevEvts Events from previous cluster.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    private void newClusterStarted(@Nullable ZkDiscoveryEventsData prevEvts) throws Exception {
+        assert !locNode.isClient() : locNode;
+
+        long locInternalId = rtState.internalOrder;
+
+        assert prevEvts == null || prevEvts.maxInternalOrder < locInternalId;
+
+        spi.getSpiContext().removeTimeoutObject(rtState.joinErrTo);
+
+        cleanupPreviousClusterData(prevEvts != null ? prevEvts.maxInternalOrder + 1 : -1L);
+
+        rtState.joined = true;
+        rtState.gridStartTime = System.currentTimeMillis();
+
+        rtState.evtsData = ZkDiscoveryEventsData.createForNewCluster(rtState.gridStartTime);
+
+        if (log.isInfoEnabled()) {
+            log.info("New cluster started [locId=" + locNode.id() +
+                ", clusterId=" + rtState.evtsData.clusterId +
+                ", startTime=" + rtState.evtsData.clusterStartTime + ']');
+        }
+
+        locNode.internalId(locInternalId);
+        locNode.order(1);
+
+        rtState.evtsData.onNodeJoin(locNode);
+
+        rtState.top.addNode(locNode);
+
+        final List<ClusterNode> topSnapshot = Collections.singletonList((ClusterNode)locNode);
+
+        lsnr.onDiscovery(EVT_NODE_JOINED,
+            1L,
+            locNode,
+            topSnapshot,
+            Collections.<Long, Collection<ClusterNode>>emptyMap(),
+            null);
+
+        // Reset events (this is also notification for clients left from previous cluster).
+        rtState.zkClient.setData(zkPaths.evtsPath, marshalZip(rtState.evtsData), -1);
+
+        joinFut.onDone();
+    }
+
+    /**
+     * @param startInternalOrder Starting internal order for cluster (znodes having lower order belong
+     *      to clients from previous cluster and should be removed).
+
+     * @throws Exception If failed.
+     */
+    private void cleanupPreviousClusterData(long startInternalOrder) throws Exception {
+        long start = System.currentTimeMillis();
+
+        ZookeeperClient client = rtState.zkClient;
+
+        // TODO ZK: use multi, better batching + max-size safe + NoNodeException safe.
+        List<String> evtChildren = rtState.zkClient.getChildren(zkPaths.evtsPath);
+
+        for (String evtPath : evtChildren) {
+            String evtDir = zkPaths.evtsPath + "/" + evtPath;
+
+            removeChildren(evtDir);
+        }
+
+        client.deleteAll(zkPaths.evtsPath, evtChildren, -1);
+
+        client.deleteAll(zkPaths.customEvtsDir,
+            client.getChildren(zkPaths.customEvtsDir),
+            -1);
+
+        rtState.zkClient.deleteAll(zkPaths.customEvtsPartsDir,
+            rtState.zkClient.getChildren(zkPaths.customEvtsPartsDir),
+            -1);
+
+        rtState.zkClient.deleteAll(zkPaths.customEvtsAcksDir,
+            rtState.zkClient.getChildren(zkPaths.customEvtsAcksDir),
+            -1);
+
+        if (startInternalOrder > 0) {
+            for (String alive : rtState.zkClient.getChildren(zkPaths.aliveNodesDir)) {
+                if (ZkIgnitePaths.aliveInternalId(alive) < startInternalOrder)
+                    rtState.zkClient.deleteIfExists(zkPaths.aliveNodesDir + "/" + alive, -1);
+            }
+        }
+
+        long time = System.currentTimeMillis() - start;
+
+        if (time > 0) {
+            if (log.isInfoEnabled())
+                log.info("Previous cluster data cleanup time: " + time);
+        }
+    }
+
+    /**
+     * @param path Path.
+     * @throws Exception If failed.
+     */
+    private void removeChildren(String path) throws Exception {
+        rtState.zkClient.deleteAll(path, rtState.zkClient.getChildren(path), -1);
+    }
+
+    /**
+     * @param zkClient Client.
+     * @param evtPath Event path.
+     * @param sndNodeId Sender node ID.
+     * @return Event data.
+     * @throws Exception If failed.
+     */
+    private byte[] readCustomEventData(ZookeeperClient zkClient, String evtPath, UUID sndNodeId) throws Exception {
+        int partCnt = ZkIgnitePaths.customEventPartsCount(evtPath);
+
+        if (partCnt > 1) {
+            String partsBasePath = zkPaths.customEventPartsBasePath(
+                ZkIgnitePaths.customEventPrefix(evtPath), sndNodeId);
+
+            return readMultipleParts(zkClient, partsBasePath, partCnt);
+        }
+        else
+            return zkClient.getData(zkPaths.customEvtsDir + "/" + evtPath);
+    }
+
+    /**
+     * @param customEvtNodes ZK nodes representing custom events to process.
+     * @throws Exception If failed.
+     */
+    private void generateCustomEvents(List<String> customEvtNodes) throws Exception {
+        assert rtState.crd;
+
+        ZookeeperClient zkClient = rtState.zkClient;
+        ZkDiscoveryEventsData evtsData = rtState.evtsData;
+
+        TreeMap<Integer, String> unprocessedEvts = null;
+
+        for (int i = 0; i < customEvtNodes.size(); i++) {
+            String evtPath = customEvtNodes.get(i);
+
+            int evtSeq = ZkIgnitePaths.customEventSequence(evtPath);
+
+            if (evtSeq > evtsData.procCustEvt) {
+                if (unprocessedEvts == null)
+                    unprocessedEvts = new TreeMap<>();
+
+                unprocessedEvts.put(evtSeq, evtPath);
+            }
+        }
+
+        if (unprocessedEvts == null)
+            return;
+
+        for (Map.Entry<Integer, String> evtE : unprocessedEvts.entrySet()) {
+            evtsData.procCustEvt = evtE.getKey();
+
+            String evtPath = evtE.getValue();
+
+            UUID sndNodeId = ZkIgnitePaths.customEventSendNodeId(evtPath);
+
+            ZookeeperClusterNode sndNode = rtState.top.nodesById.get(sndNodeId);
+
+            if (sndNode != null) {
+                byte[] evtBytes = readCustomEventData(zkClient, evtPath, sndNodeId);
+
+                DiscoverySpiCustomMessage msg;
+
+                try {
+                    msg = unmarshalZip(evtBytes);
+                }
+                catch (Exception e) {
+                    U.error(log, "Failed to unmarshal custom discovery message: " + e, e);
+
+                    deleteCustomEventDataAsync(rtState.zkClient, evtPath);
+
+                    continue;
+                }
+
+                generateAndProcessCustomEventOnCoordinator(evtPath, sndNode, msg);
+            }
+            else {
+                U.warn(log, "Ignore custom event from unknown node: " + sndNodeId);
+
+                deleteCustomEventDataAsync(rtState.zkClient, evtPath);
+            }
+        }
+    }
+
+    /**
+     * @param evtPath Event data path.
+     * @param sndNode Sender node.
+     * @param msg Message instance.
+     * @throws Exception If failed.
+     */
+    private void generateAndProcessCustomEventOnCoordinator(String evtPath,
+        ZookeeperClusterNode sndNode,
+        DiscoverySpiCustomMessage msg) throws Exception
+    {
+        ZookeeperClient zkClient = rtState.zkClient;
+        ZkDiscoveryEventsData evtsData = rtState.evtsData;
+
+        ZookeeperClusterNode failedNode = null;
+
+        if (msg instanceof ZkForceNodeFailMessage) {
+            ZkForceNodeFailMessage msg0 = (ZkForceNodeFailMessage)msg;
+
+            failedNode = rtState.top.nodesByInternalId.get(msg0.nodeInternalId);
+
+            if (failedNode != null)
+                evtsData.topVer++;
+            else {
+                if (log.isDebugEnabled())
+                    log.debug("Ignore forcible node fail request for unknown node: " + msg0.nodeInternalId);
+
+                deleteCustomEventDataAsync(zkClient, evtPath);
+
+                return;
+            }
+        }
+        else if (msg instanceof ZkCommunicationErrorResolveStartMessage) {
+            ZkCommunicationErrorResolveStartMessage msg0 =
+                (ZkCommunicationErrorResolveStartMessage)msg;
+
+            if (evtsData.communicationErrorResolveFutureId() != null) {
+                if (log.isInfoEnabled()) {
+                    log.info("Ignore communication error resolve message, resolve process " +
+                        "already started [sndNode=" + sndNode + ']');
+                }
+
+                deleteCustomEventDataAsync(zkClient, evtPath);
+
+                return;
+            }
+            else {
+                if (log.isInfoEnabled()) {
+                    log.info("Start cluster-wide communication error resolve [sndNode=" + sndNode +
+                        ", reqId=" + msg0.id +
+                        ", topVer=" + evtsData.topVer + ']');
+                }
+
+                zkClient.createIfNeeded(zkPaths.distributedFutureBasePath(msg0.id),
+                    null,
+                    PERSISTENT);
+
+                evtsData.communicationErrorResolveFutureId(msg0.id);
+            }
+        }
+
+        evtsData.evtIdGen++;
+
+        ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
+            evtsData.evtIdGen,
+            0L,
+            evtsData.topVer,
+            sndNode.id(),
+            null,
+            evtPath);
+
+        evtData.resolvedMsg = msg;
+
+        if (log.isDebugEnabled())
+            log.debug("Generated CUSTOM event [evt=" + evtData + ", msg=" + msg + ']');
+
+        boolean fastStopProcess = false;
+
+        if (msg instanceof ZkInternalMessage)
+            processInternalMessage(evtData, (ZkInternalMessage)msg);
+        else {
+            notifyCustomEvent(evtData, msg);
+
+            if (msg.stopProcess()) {
+                if (log.isDebugEnabled())
+                    log.debug("Fast stop process custom event [evt=" + evtData + ", msg=" + msg + ']');
+
+                fastStopProcess = true;
+
+                // No need to process this event on others nodes, skip this event.
+                evtsData.evts.remove(evtData.eventId());
+
+                evtsData.evtIdGen--;
+
+                DiscoverySpiCustomMessage ack = msg.ackMessage();
+
+                if (ack != null) {
+                    evtData = createAckEvent(ack, evtData);
+
+                    if (log.isDebugEnabled())
+                        log.debug("Generated CUSTOM event (ack for fast stop process) [evt=" + evtData + ", msg=" + msg + ']');
+
+                    notifyCustomEvent(evtData, ack);
+                }
+                else
+                    evtData = null;
+            }
+        }
+
+        if (evtData != null) {
+            evtsData.addEvent(rtState.top.nodesByOrder.values(), evtData);
+
+            rtState.locNodeInfo.lastProcEvt = evtData.eventId();
+
+            saveAndProcessNewEvents();
+
+            if (fastStopProcess)
+                deleteCustomEventDataAsync(zkClient, evtPath);
+
+            if (failedNode != null) {
+                deleteAliveNode(failedNode.internalId());
+
+                handleProcessedEventsOnNodesFail(Collections.singletonList(failedNode));
+
+                rtState.updateAlives = true;
+            }
+        }
+    }
+
+    /**
+     * @param internalId Node internal ID.
+     * @throws Exception If failed.
+     */
+    private void deleteAliveNode(long internalId) throws Exception {
+        for (String child : rtState.zkClient.getChildren(zkPaths.aliveNodesDir)) {
+            if (ZkIgnitePaths.aliveInternalId(child) == internalId) {
+                // Need use sync delete to do not process again join of this node again.
+                rtState.zkClient.deleteIfExists(zkPaths.aliveNodesDir + "/" + child, -1);
+
+                return;
+            }
+        }
+    }
+
+    /**
+     * @param zkClient Client.
+     * @param evtPath Event path.
+     */
+    private void deleteCustomEventDataAsync(ZookeeperClient zkClient, String evtPath) {
+        if (log.isDebugEnabled())
+            log.debug("Delete custom event data: " + evtPath);
+
+        String prefix = ZkIgnitePaths.customEventPrefix(evtPath);
+        UUID sndNodeId = ZkIgnitePaths.customEventSendNodeId(evtPath);
+        int partCnt = ZkIgnitePaths.customEventPartsCount(evtPath);
+
+        assert partCnt >= 1 : partCnt;
+
+        if (partCnt > 1) {
+            for (int i = 0; i < partCnt; i++) {
+                String path0 = zkPaths.customEventPartPath(prefix, sndNodeId, i);
+
+                zkClient.deleteIfExistsAsync(path0);
+            }
+        }
+
+        zkClient.deleteIfExistsAsync(zkPaths.customEvtsDir + "/" + evtPath);
+    }
+
+    /**
+     * @param data Marshalled events.
+     * @throws Exception If failed.
+     * @return Events.
+     */
+    @Nullable private ZkDiscoveryEventsData processNewEvents(byte[] data) throws Exception {
+        ZkDiscoveryEventsData newEvts = data.length > 0 ? (ZkDiscoveryEventsData)unmarshalZip(data) : null;
+
+        if (rtState.joined && (newEvts == null || !rtState.evtsData.clusterId.equals(newEvts.clusterId))) {
+            assert locNode.isClient() : locNode;
+
+            throw localNodeFail("All server nodes failed, client node disconnected (received events from new custer) " +
+                "[locId=" + locNode.id() + ']', true);
+        }
+
+        if (newEvts == null)
+            return null;
+
+        assert !rtState.crd;
+
+        // Need keep processed custom events since they contain message object which is needed to create ack.
+        if (!locNode.isClient() && rtState.evtsData != null) {
+            for (Map.Entry<Long, ZkDiscoveryEventData> e : rtState.evtsData.evts.entrySet()) {
+                ZkDiscoveryEventData evtData = e.getValue();
+
+                if (evtData.eventType() == ZkDiscoveryEventData.ZK_EVT_CUSTOM_EVT) {
+                    ZkDiscoveryCustomEventData evtData0 =
+                        (ZkDiscoveryCustomEventData)newEvts.evts.get(evtData.eventId());
+
+                    if (evtData0 != null)
+                        evtData0.resolvedMsg = ((ZkDiscoveryCustomEventData)evtData).resolvedMsg;
+                }
+            }
+        }
+
+        processNewEvents(newEvts);
+
+        if (rtState.joined)
+            rtState.evtsData = newEvts;
+
+        return newEvts;
+    }
+
+    /**
+     * @param evtsData Events.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    private void processNewEvents(final ZkDiscoveryEventsData evtsData) throws Exception {
+        TreeMap<Long, ZkDiscoveryEventData> evts = evtsData.evts;
+
+        ZookeeperClient zkClient = rtState.zkClient;
+
+        boolean evtProcessed = false;
+        boolean updateNodeInfo = false;
+
+        try {
+            for (ZkDiscoveryEventData evtData : evts.tailMap(rtState.locNodeInfo.lastProcEvt, false).values()) {
+                if (log.isDebugEnabled())
+                    log.debug("New discovery event data [evt=" + evtData + ", evtsHist=" + evts.size() + ']');
+
+                switch (evtData.eventType()) {
+                    case ZkDiscoveryEventData.ZK_EVT_NODE_JOIN: {
+                        evtProcessed = processBulkJoin(evtsData, (ZkDiscoveryNodeJoinEventData)evtData);
+
+                        break;
+                    }
+
+                    case ZkDiscoveryEventData.ZK_EVT_NODE_FAILED: {
+                        if (!rtState.joined)
+                            break;
+
+                        evtProcessed = true;
+
+                        notifyNodeFail((ZkDiscoveryNodeFailEventData)evtData);
+
+                        break;
+                    }
+
+                    case ZkDiscoveryEventData.ZK_EVT_CUSTOM_EVT: {
+                        if (!rtState.joined)
+                            break;
+
+                        evtProcessed = true;
+
+                        ZkDiscoveryCustomEventData evtData0 = (ZkDiscoveryCustomEventData)evtData;
+
+                        if (evtData0.ackEvent() && evtData0.topologyVersion() < locNode.order())
+                            break;
+
+                        DiscoverySpiCustomMessage msg;
+
+                        if (rtState.crd) {
+                            assert evtData0.resolvedMsg != null : evtData0;
+
+                            msg = evtData0.resolvedMsg;
+                        }
+                        else {
+                            if (evtData0.msg == null) {
+                                if (evtData0.ackEvent()) {
+                                    String path = zkPaths.ackEventDataPath(evtData0.origEvtId);
+
+                                    msg = unmarshalZip(zkClient.getData(path));
+                                }
+                                else {
+                                    assert evtData0.evtPath != null : evtData0;
+
+                                    byte[] msgBytes = readCustomEventData(zkClient,
+                                        evtData0.evtPath,
+                                        evtData0.sndNodeId);
+
+                                    msg = unmarshalZip(msgBytes);
+                                }
+                            }
+                            else
+                                msg = evtData0.msg;
+
+                            evtData0.resolvedMsg = msg;
+                        }
+
+                        if (msg instanceof ZkInternalMessage)
+                            processInternalMessage(evtData0, (ZkInternalMessage)msg);
+                        else {
+                            notifyCustomEvent(evtData0, msg);
+
+                            if (!evtData0.ackEvent())
+                                updateNodeInfo = true;
+                        }
+
+                        break;
+                    }
+
+                    default:
+                        assert false : "Invalid event: " + evtData;
+                }
+
+                if (rtState.joined) {
+                    rtState.locNodeInfo.lastProcEvt = evtData.eventId();
+
+                    rtState.procEvtCnt++;
+
+                    if (rtState.procEvtCnt % evtsAckThreshold == 0)
+                        updateNodeInfo = true;
+                }
+            }
+        }
+        catch (KeeperException.NoNodeException e) {
+            // Can get NoNodeException if local node was forcible failed,
+            // in this case coordinator does not wait when this node process all events.
+            boolean exists;
+
+            try {
+                exists = rtState.zkClient.exists(rtState.locNodeZkPath);
+            }
+            catch (Exception e0) {
+                if (log.isDebugEnabled())
+                    log.debug("Failed to check is local node is alive:" + e0);
+
+                exists = true;
+            }
+
+            if (!exists) {
+                U.warn(log, "Failed to process discovery event, local node was forced to stop.", e);
+
+                throw localNodeFail("Local node was forced to stop.", true);
+            }
+
+            throw e;
+        }
+
+        if (rtState.crd)
+            handleProcessedEvents("procEvt");
+        else
+            onEventProcessed(rtState, updateNodeInfo, evtProcessed);
+
+        ZkCommunicationErrorProcessFuture commErrFut = commErrProcFut.get();
+
+        if (commErrFut != null)
+            commErrFut.onTopologyChange(rtState.top); // This can add new event, notify out of event process loop.
+    }
+
+    private boolean processBulkJoin(ZkDiscoveryEventsData evtsData, ZkDiscoveryNodeJoinEventData evtData)
+        throws Exception
+    {
+        boolean evtProcessed = false;
+
+        for (int i = 0; i < evtData.joinedNodes.size(); i++) {
+            ZkJoinedNodeEvtData joinedEvtData = evtData.joinedNodes.get(i);
+
+            if (!rtState.joined) {
+                UUID joinedId = joinedEvtData.nodeId;
+
+                boolean locJoin = joinedEvtData.joinedInternalId == rtState.internalOrder;
+
+                if (locJoin) {
+                    assert locNode.id().equals(joinedId);
+
+                    processLocalJoin(evtsData, joinedEvtData, evtData);
+
+                    evtProcessed = true;
+                }
+            }
+            else {
+                ZkJoiningNodeData joiningData;
+
+                if (rtState.crd) {
+                    assert joinedEvtData.joiningNodeData != null;
+
+                    joiningData = joinedEvtData.joiningNodeData;
+                }
+                else {
+                    joiningData = unmarshalJoinData(joinedEvtData.nodeId, joinedEvtData.joinDataPrefixId);
+
+                    DiscoveryDataBag dataBag = new DiscoveryDataBag(joinedEvtData.nodeId, joiningData.node().isClient());
+
+                    dataBag.joiningNodeData(joiningData.discoveryData());
+
+                    exchange.onExchange(dataBag);
+                }
+
+                if (joinedEvtData.secSubjPartCnt > 0 && joiningData.node().attribute(ATTR_SECURITY_SUBJECT_V2) == null)
+                    readAndInitSecuritySubject(joiningData.node(), joinedEvtData);
+
+                notifyNodeJoin(joinedEvtData, joiningData);
+            }
+        }
+
+        return evtProcessed;
+    }
+
+    /**
+     * @param rtState Runtime state.
+     * @param updateNodeInfo {@code True} if need update processed events without delay.
+     * @param evtProcessed {@code True} if new event was processed.
+     * @throws Exception If failed.
+     */
+    private void onEventProcessed(ZkRuntimeState rtState,
+        boolean updateNodeInfo,
+        boolean evtProcessed) throws Exception
+    {
+        synchronized (stateMux) {
+            if (updateNodeInfo) {
+                assert rtState.locNodeZkPath != null;
+
+                if (log.isDebugEnabled())
+                    log.debug("Update processed events: " + rtState.locNodeInfo.lastProcEvt);
+
+                updateProcessedEvents(rtState);
+
+                if (rtState.procEvtsUpdateTo != null) {
+                    spi.getSpiContext().removeTimeoutObject(rtState.procEvtsUpdateTo);
+
+                    rtState.procEvtsUpdateTo = null;
+                }
+            }
+            else if (evtProcessed) {
+                rtState.locNodeInfo.needUpdate = true;
+
+                if (rtState.procEvtsUpdateTo == null) {
+                    long updateTimeout = IgniteSystemProperties.getLong(IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT,
+                        60_000);
+
+                    if (updateTimeout > 0) {
+                        rtState.procEvtsUpdateTo = new UpdateProcessedEventsTimeoutObject(rtState, updateTimeout);
+
+                        spi.getSpiContext().addTimeoutObject(rtState.procEvtsUpdateTo);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * @param rtState Runtime state.
+     * @param procEvtsUpdateTo Timeout object.
+     * @throws Exception If failed.
+     */
+    private void updateProcessedEventsOnTimeout(ZkRuntimeState rtState, ZkTimeoutObject procEvtsUpdateTo)
+        throws Exception
+    {
+        synchronized (stateMux) {
+            if (rtState.procEvtsUpdateTo == procEvtsUpdateTo && rtState.locNodeInfo.needUpdate) {
+                if (log.isDebugEnabled())
+                    log.debug("Update processed events on timeout: " + rtState.locNodeInfo.lastProcEvt);
+
+                updateProcessedEvents(rtState);
+            }
+        }
+    }
+
+    /**
+     * @param rtState Runtime state.
+     * @throws Exception If failed.
+     */
+    private void updateProcessedEvents(ZkRuntimeState rtState) throws Exception {
+        try {
+            rtState.zkClient.setData(rtState.locNodeZkPath, marshalZip(rtState.locNodeInfo), -1);
+
+            rtState.locNodeInfo.needUpdate = false;
+        }
+        catch (KeeperException.NoNodeException e) {
+            // Possible if node is forcible failed.
+            if (log.isDebugEnabled())
+                log.debug("Failed to update processed events, no node: " + rtState.locNodeInfo.lastProcEvt);
+        }
+    }
+
+    /**
+     * @param node Node.
+     * @param joinedEvtData Joined node information.
+     * @throws Exception If failed.
+     */
+    private void readAndInitSecuritySubject(ZookeeperClusterNode node, ZkJoinedNodeEvtData joinedEvtData) throws Exception {
+        if (joinedEvtData.secSubjPartCnt > 0) {
+            byte[] zipBytes = readMultipleParts(rtState.zkClient,
+                zkPaths.joinEventSecuritySubjectPath(joinedEvtData.topVer),
+                joinedEvtData.secSubjPartCnt);
+
+            setNodeSecuritySubject(node, zipBytes);
+        }
+    }
+
+    /**
+     * @param evtsData Events data.
+     * @param evtData Local join event data.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    private void processLocalJoin(ZkDiscoveryEventsData evtsData,
+        ZkJoinedNodeEvtData joinedEvtData,
+        ZkDiscoveryNodeJoinEventData evtData)
+        throws Exception
+    {
+        synchronized (stateMux) {
+            if (connState == ConnectionState.STOPPED)
+                return;
+
+            if (rtState.joinTo != null) {
+                spi.getSpiContext().removeTimeoutObject(rtState.joinTo);
+
+                rtState.joinTo.cancelled = true;
+                rtState.joinTo = null;
+            }
+
+            spi.getSpiContext().removeTimeoutObject(rtState.joinErrTo);
+
+            if (log.isInfoEnabled())
+                log.info("Local join event data: " + joinedEvtData + ']');
+
+            String path = zkPaths.joinEventDataPathForJoined(evtData.eventId());
+
+            byte[] dataForJoinedBytes = readMultipleParts(rtState.zkClient, path, evtData.dataForJoinedPartCnt);
+
+            ZkJoinEventDataForJoined dataForJoined = unmarshalZip(dataForJoinedBytes);
+
+            rtState.gridStartTime = evtsData.clusterStartTime;
+
+            locNode.internalId(joinedEvtData.joinedInternalId);
+            locNode.order(joinedEvtData.topVer);
+
+            readAndInitSecuritySubject(locNode, joinedEvtData);
+
+            byte[] discoDataBytes = dataForJoined.discoveryDataForNode(locNode.order());
+
+            Map<Integer, Serializable> commonDiscoData =
+                marsh.unmarshal(discoDataBytes, U.resolveClassLoader(spi.ignite().configuration()));
+
+            DiscoveryDataBag dataBag = new DiscoveryDataBag(locNode.id(), locNode.isClient());
+
+            dataBag.commonData(commonDiscoData);
+
+            exchange.onExchange(dataBag);
+
+            List<ZookeeperClusterNode> allNodes = dataForJoined.topology();
+
+            for (int i = 0; i < allNodes.size(); i++) {
+                ZookeeperClusterNode node = allNodes.get(i);
+
+                // Need filter since ZkJoinEventDataForJoined contains single topology snapshot for all joined nodes.
+                if (node.order() >= locNode.order())
+                    break;
+
+                node.setMetrics(new ClusterMetricsSnapshot());
+
+                rtState.top.addNode(node);
+            }
+
+            rtState.top.addNode(locNode);
+
+            final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
+
+            lsnr.onDiscovery(EVT_NODE_JOINED,
+                joinedEvtData.topVer,
+                locNode,
+                topSnapshot,
+                Collections.<Long, Collection<ClusterNode>>emptyMap(),
+                null);
+
+            if (rtState.prevJoined) {
+                lsnr.onDiscovery(EVT_CLIENT_NODE_RECONNECTED,
+                    joinedEvtData.topVer,
+                    locNode,
+                    topSnapshot,
+                    Collections.<Long, Collection<ClusterNode>>emptyMap(),
+                    null);
+
+                U.quietAndWarn(log, "Client node was reconnected after it was already considered failed [locId=" + locNode.id() + ']');
+            }
+
+            rtState.joined = true;
+        }
+
+        joinFut.onDone();
+
+        if (locNode.isClient())
+            rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, null, new CheckClientsStatusCallback(rtState));
+    }
+
+    /**
+     * @param evtData Event daa.
+     * @param msg Message.
+     * @throws Exception If failed.
+     */
+    private void processInternalMessage(ZkDiscoveryCustomEventData evtData, ZkInternalMessage msg) throws Exception {
+        if (msg instanceof ZkForceNodeFailMessage)
+            processForceNodeFailMessage((ZkForceNodeFailMessage)msg, evtData);
+        else if (msg instanceof ZkCommunicationErrorResolveStartMessage) {
+            processCommunicationErrorResolveStartMessage(
+                (ZkCommunicationErrorResolveStartMessage)msg,
+                evtData);
+        }
+        else if (msg instanceof ZkCommunicationErrorResolveFinishMessage) {
+            processCommunicationErrorResolveFinishMessage(
+                (ZkCommunicationErrorResolveFinishMessage)msg);
+        }
+        else if (msg instanceof ZkNoServersMessage)
+            processNoServersMessage((ZkNoServersMessage)msg);
+    }
+
+    /**
+     * @param msg Message.
+     * @throws Exception If failed.
+     */
+    private void processNoServersMessage(ZkNoServersMessage msg) throws Exception {
+        assert locNode.isClient() : locNode;
+
+        throw localNodeFail("All server nodes failed, client node disconnected " +
+            "(received 'no-servers' message) [locId=" + locNode.id() + ']', true);
+    }
+
+    /**
+     * @param msg Message.
+     * @param evtData Event data.
+     * @throws Exception If failed.
+     */
+    private void processForceNodeFailMessage(ZkForceNodeFailMessage msg, ZkDiscoveryCustomEventData evtData)
+        throws Exception {
+        ClusterNode creatorNode = rtState.top.nodesById.get(evtData.sndNodeId);
+
+        ZookeeperClusterNode node = rtState.top.nodesByInternalId.get(msg.nodeInternalId);
+
+        assert node != null : msg.nodeInternalId;
+
+        if (msg.warning != null) {
+            U.warn(log, "Received force EVT_NODE_FAILED event with warning [" +
+                "nodeId=" + node.id() +
+                ", msg=" + msg.warning +
+                ", nodeInitiatedEvt=" + (creatorNode != null ? creatorNode : evtData.sndNodeId) + ']');
+        }
+        else {
+            U.warn(log, "Received force EVT_NODE_FAILED event [" +
+                "nodeId=" + node.id() +
+                ", nodeInitiatedEvt=" + (creatorNode != null ? creatorNode : evtData.sndNodeId) + ']');
+        }
+
+        if (node.isLocal())
+            throw localNodeFail("Received force EVT_NODE_FAILED event for local node.", true);
+        else
+            notifyNodeFail(node.internalId(), evtData.topologyVersion());
+    }
+
+    /**
+     * @param msg Message.
+     * @throws Exception If failed.
+     */
+    private void processCommunicationErrorResolveFinishMessage(ZkCommunicationErrorResolveFinishMessage msg)
+        throws Exception
+    {
+        UUID futId = msg.futId;
+
+        assert futId != null;
+
+        if (log.isInfoEnabled())
+            log.info("Received communication error resolve finish message [reqId=" + futId + ']');
+
+        rtState.commErrProcNodes = null;
+
+        ZkCommunicationErrorResolveResult res = msg.res;
+
+        if (res == null)
+            res = unmarshalZip(ZkDistributedCollectDataFuture.readResult(rtState.zkClient, zkPaths, futId));
+
+        ZkCommunicationErrorProcessFuture fut = commErrProcFut.get();
+
+        assert fut != null;
+
+        Set<Long> failedNodes = null;
+
+        if (res.err != null)
+            U.error(log, "Communication error resolve failed: " + res.err, res.err);
+        else {
+            if (res.killedNodes != null) {
+                failedNodes = U.newHashSet(res.killedNodes.size());
+
+                for (int i = 0; i < res.killedNodes.size(); i++) {
+                    long internalId = res.killedNodes.get(i);
+
+                    if (internalId == locNode.internalId()) {
+                        fut.onError(new IgniteCheckedException("Local node is forced to stop " +
+                            "by communication error resolver"));
+
+                        if (rtState.crd)
+                            deleteAliveNodes(res.killedNodes);
+
+                        throw localNodeFail("Local node is forced to stop by communication error resolver " +
+                            "[nodeId=" + locNode.id() + ']', false);
+                    }
+
+                    ZookeeperClusterNode node = rtState.top.nodesByInternalId.get(internalId);
+
+                    assert node != null : internalId;
+
+                    failedNodes.add(node.order());
+                }
+
+                long topVer = msg.topVer;
+
+                for (int i = 0; i < res.killedNodes.size(); i++) {
+                    long nodeInternalId = res.killedNodes.get(i);
+
+                    ClusterNode node = rtState.top.nodesByInternalId.get(nodeInternalId);
+
+                    assert node != null : nodeInternalId;
+
+                    if (log.isInfoEnabled())
+                        log.info("Node stop is forced by communication error resolver [nodeId=" + node.id() + ']');
+
+                    notifyNodeFail(nodeInternalId, ++topVer);
+                }
+            }
+        }
+
+        fut.onFinishResolve(failedNodes);
+
+        if (rtState.crd)
+            deleteAliveNodes(res.killedNodes);
+    }
+
+    /**
+     * @param internalIds Nodes internal IDs.
+     * @throws Exception If failed.
+     */
+    private void deleteAliveNodes(@Nullable GridLongList internalIds) throws Exception {
+        if (internalIds == null)
+            return;
+
+        List<String> alives = rtState.zkClient.getChildren(zkPaths.aliveNodesDir);
+
+        for (int i = 0; i < alives.size(); i++) {
+            String alive = alives.get(i);
+
+            if (internalIds.contains(ZkIgnitePaths.aliveInternalId(alive)))
+                rtState.zkClient.deleteIfExistsAsync(zkPaths.aliveNodesDir + "/" + alive);
+        }
+    }
+
+    /**
+     * @param msg Message.
+     * @param evtData Event data.
+     * @throws Exception If failed.
+     */
+    private void processCommunicationErrorResolveStartMessage(ZkCommunicationErrorResolveStartMessage msg,
+        ZkDiscoveryCustomEventData evtData) throws Exception {
+        ZkCommunicationErrorProcessFuture fut;
+
+        for (;;) {
+            fut = commErrProcFut.get();
+
+            if (fut == null || fut.isDone()) {
+                ZkCommunicationErrorProcessFuture newFut =
+                    ZkCommunicationErrorProcessFuture.createOnStartResolveRequest(this);
+
+                if (commErrProcFut.compareAndSet(fut, newFut))
+                    fut = newFut;
+                else
+                    fut = commErrProcFut.get();
+            }
+
+            if (fut.onStartResolveRequest(evtData.topologyVersion()))
+                break;
+            else {
+                try {
+                    fut.get();
+                }
+                catch (Exception e) {
+                    U.warn(log, "Previous communication error process future failed: " + e);
+                }
+            }
+        }
+
+        if (log.isInfoEnabled()) {
+            log.info("Received communication error resolve request [reqId=" + msg.id +
+                ", topVer=" + rtState.top.topologySnapshot() + ']');
+        }
+
+        assert !fut.isDone() : fut;
+
+        final String futPath = zkPaths.distributedFutureBasePath(msg.id);
+        final ZkCommunicationErrorProcessFuture fut0 = fut;
+
+        rtState.commErrProcNodes = rtState.top.topologySnapshot();
+
+        if (rtState.crd) {
+            ZkDistributedCollectDataFuture nodeResFut = collectCommunicationStatusFuture(msg.id);
+
+            fut.nodeResultCollectFuture(nodeResFut);
+        }
+
+        runInWorkerThread(new ZkRunnable(rtState, this) {
+            @Override protected void run0() throws Exception {
+                fut0.checkConnection(rtState, futPath, rtState.commErrProcNodes);
+            }
+        });
+    }
+
+    /**
+     * @param futId Future ID.
+     * @return Future.
+     * @throws Exception If failed.
+     */
+    private ZkDistributedCollectDataFuture collectCommunicationStatusFuture(UUID futId) throws Exception {
+        return new ZkDistributedCollectDataFuture(this, rtState, zkPaths.distributedFutureBasePath(futId),
+            new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    // Future is completed from ZK event thread.
+                    onCommunicationErrorResolveStatusReceived(rtState);
+
+                    return null;
+                }
+            }
+        );
+    }
+
+    /**
+     * @param rtState Runtime state.
+     * @throws Exception If failed.
+     */
+    private void onCommunicationErrorResolveStatusReceived(final ZkRuntimeState rtState) throws Exception {
+        ZkDiscoveryEventsData evtsData = rtState.evtsData;
+
+        UUID futId = evtsData.communicationErrorResolveFutureId();
+
+        if (log.isInfoEnabled())
+            log.info("Received communication status from all nodes [reqId=" + futId + ']');
+
+        assert futId != null;
+
+        String futPath = zkPaths.distributedFutureBasePath(futId);
+
+        List<ClusterNode> initialNodes = rtState.commErrProcNodes;
+
+        assert initialNodes != null;
+
+        rtState.commErrProcNodes = null;
+
+        List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
+
+        Map<UUID, BitSet> nodesRes = U.newHashMap(topSnapshot.size());
+
+        Exception err = null;
+
+        for (ClusterNode node : topSnapshot) {
+            byte[] stateBytes = ZkDistributedCollectDataFuture.readNodeResult(futPath,
+                rtState.zkClient,
+                node.order());
+
+            ZkCommunicationErrorNodeState nodeState = unmarshalZip(stateBytes);
+
+            if (nodeState.err != null) {
+                if (err == null)
+                    err = new Exception("Failed to resolve communication error.");
+
+                err.addSuppressed(nodeState.err);
+            }
+            else {
+                assert nodeState.commState != null;
+
+                nodesRes.put(node.id(), nodeState.commState);
+            }
+        }
+
+        long topVer = evtsData.topVer;
+
+        GridLongList killedNodesList = null;
+
+        if (err == null) {
+            boolean fullyConnected = true;
+
+            for (Map.Entry<UUID, BitSet> e : nodesRes.entrySet()) {
+                if (!checkFullyConnected(e.getValue(), initialNodes, rtState.top)) {
+                    fullyConnected = false;
+
+                    break;
+                }
+            }
+
+            if (fullyConnected) {
+                if (log.isInfoEnabled()) {
+                    log.info("Finish communication error resolve process automatically, there are no " +
+                        "communication errors [reqId=" + futId + ']');
+                }
+            }
+            else {
+                CommunicationFailureResolver rslvr = spi.ignite().configuration().getCommunicationFailureResolver();
+
+                if (rslvr != null) {
+                    if (log.isInfoEnabled()) {
+                        log.info("Call communication error resolver [reqId=" + futId +
+                            ", rslvr=" + rslvr.getClass().getSimpleName() + ']');
+                    }
+
+                    ZkCommunicationFailureContext ctx = new ZkCommunicationFailureContext(
+                        ((IgniteKernal)spi.ignite()).context().cache().context(),
+                        topSnapshot,
+                        initialNodes,
+                        nodesRes);
+
+                    try {
+                        rslvr.resolve(ctx);
+
+                        Set<ClusterNode> killedNodes = ctx.killedNodes();
+
+                        if (killedNodes != null) {
+                            if (log.isInfoEnabled()) {
+                                log.info("Communication error resolver forced nodes stop [reqId=" + futId +
+                                    ", killNodeCnt=" + killedNodes.size() +
+                                    ", nodeIds=" + U.nodeIds(killedNodes) + ']');
+                            }
+
+                            killedNodesList = new GridLongList(killedNodes.size());
+
+                            for (ClusterNode killedNode : killedNodes) {
+                                killedNodesList.add(((ZookeeperClusterNode)killedNode).internalId());
+
+                                evtsData.topVer++;
+                            }
+                        }
+                    }
+                    catch (Exception e) {
+                        err = e;
+
+                        U.error(log, "Failed to resolve communication error with configured resolver [reqId=" + futId + ']', e);
+                    }
+                }
+            }
+        }
+
+        evtsData.communicationErrorResolveFutureId(null);
+
+        ZkCommunicationErrorResolveResult res = new ZkCommunicationErrorResolveResult(killedNodesList, err);
+
+        ZkCommunicationErrorResolveFinishMessage msg = new ZkCommunicationErrorResolveFinishMessage(futId, topVer);
+
+        msg.res = res;
+
+        ZkDistributedCollectDataFuture.saveResult(zkPaths.distributedFutureResultPath(futId),
+            rtState.zkClient,
+            marshalZip(res));
+
+        evtsData.evtIdGen++;
+
+        ZkDiscoveryCustomEventData evtData = new ZkDiscoveryCustomEventData(
+            evtsData.evtIdGen,
+            0L,
+            topVer,
+            locNode.id(),
+            msg,
+            null);
+
+        evtData.resolvedMsg = msg;
+
+        evtsData.addEvent(rtState.top.nodesByOrder.values(), evtData);
+
+        saveAndProcessNewEvents();
+
+        // Need re-check alive nodes in case join was delayed.
+        rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, rtState.watcher, rtState.watcher);
+    }
+
+    /**
+     * @param commState Node communication state.
+     * @param initialNodes Topology snapshot when communication error resolve started.
+     * @param top Current topology.
+     * @return {@code True} if node has connection to all alive nodes.
+     */
+    private boolean checkFullyConnected(BitSet commState, List<ClusterNode> initialNodes, ZkClusterNodes top) {
+        int startIdx = 0;
+
+        for (;;) {
+            int idx = commState.nextClearBit(startIdx);
+
+            if (idx >= initialNodes.size())
+                return true;
+
+            ClusterNode node = initialNodes.get(idx);
+
+            if (top.nodesById.containsKey(node.id()))
+                return false;
+
+            startIdx = idx + 1;
+        }
+    }
+
+    /**
+     *
+     */
+    public void simulateNodeFailure() {
+        ZkRuntimeState rtState = this.rtState;
+
+        ZookeeperClient client = rtState.zkClient;
+
+        client.deleteIfExistsAsync(zkPaths.aliveNodesDir);
+
+        rtState.onCloseStart(new IgniteCheckedException("Simulate node failure error."));
+
+        rtState.zkClient.close();
+    }
+
+    /**
+     * @param evtData Event data.
+     * @param msg Custom message.
+     */
+    @SuppressWarnings("unchecked")
+    private void notifyCustomEvent(final ZkDiscoveryCustomEventData evtData, final DiscoverySpiCustomMessage msg) {
+        assert !(msg instanceof ZkInternalMessage) : msg;
+
+        if (log.isDebugEnabled())
+            log.debug(" [topVer=" + evtData.topologyVersion() + ", msg=" + msg + ']');
+
+        final ZookeeperClusterNode sndNode = rtState.top.nodesById.get(evtData.sndNodeId);
+
+        assert sndNode != null : evtData;
+
+        final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
+
+        lsnr.onDiscovery(
+            DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT,
+            evtData.topologyVersion(),
+            sndNode,
+            topSnapshot,
+            Collections.<Long, Collection<ClusterNode>>emptyMap(),
+            msg);
+    }
+
+    /**
+     * @param joinedEvtData Event data.
+     * @param joiningData Joining node data.
+     */
+    @SuppressWarnings("unchecked")
+    private void notifyNodeJoin(ZkJoinedNodeEvtData joinedEvtData, ZkJoiningNodeData joiningData) {
+        final ZookeeperClusterNode joinedNode = joiningData.node();
+
+        joinedNode.order(joinedEvtData.topVer);
+        joinedNode.internalId(joinedEvtData.joinedInternalId);
+
+        joinedNode.setMetrics(new ClusterMetricsSnapshot());
+
+        rtState.top.addNode(joinedNode);
+
+        final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
+
+        lsnr.onDiscovery(EVT_NODE_JOINED,
+            joinedEvtData.topVer,
+            joinedNode,
+            topSnapshot,
+            Collections.<Long, Collection<ClusterNode>>emptyMap(),
+            null);
+    }
+
+    /**
+     * @param evtData Event data.
+     */
+    private void notifyNodeFail(final ZkDiscoveryNodeFailEventData evtData) {
+        notifyNodeFail(evtData.failedNodeInternalId(), evtData.topologyVersion());
+    }
+
+    /**
+     * @param nodeInternalOrder Node order.
+     * @param topVer Topology version.
+     */
+    private void notifyNodeFail(long nodeInternalOrder, long topVer) {
+        final ZookeeperClusterNode failedNode = rtState.top.removeNode(nodeInternalOrder);
+
+        assert failedNode != null && !failedNode.isLocal() : failedNode;
+
+        PingFuture pingFut = pingFuts.get(failedNode.order());
+
+        if (pingFut != null)
+            pingFut.onDone(false);
+
+        final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
+
+        lsnr.onDiscovery(EVT_NODE_FAILED,
+            topVer,
+            failedNode,
+            topSnapshot,
+            Collections.<Long, Collection<ClusterNode>>emptyMap(),
+            null);
+    }
+
+    /**
+     * @param msg Message to log.
+     * @param clientReconnect {@code True} if allow client reconnect.
+     * @return Exception to be thrown.
+     */
+    private ZookeeperClientFailedException localNodeFail(String msg, boolean clientReconnect) {
+        U.warn(log, msg);
+
+//        if (locNode.isClient() && rtState.zkClient.connected()) {
+//            String path = rtState.locNodeZkPath.substring(rtState.locNodeZkPath.lastIndexOf('/') + 1);
+//
+//            String joinDataPath = zkPaths.joiningNodeDataPath(locNode.id(), ZkIgnitePaths.aliveNodePrefixId(path));
+//
+//            try {
+//                if (rtState.zkClient.existsNoRetry(joinDataPath))
+//                    rtState.zkClient.deleteIfExistsNoRetry(joinDataPath, -1);
+//            }
+//            catch (Exception e) {
+//                if (log.isDebugEnabled())
+//                    log.debug("Failed to clean local node's join data on stop: " + e);
+//            }
+//        }
+
+        if (rtState.zkClient.connected())
+            rtState.zkClient.close();
+
+        if (clientReconnect && clientReconnectEnabled) {
+            assert locNode.isClient() : locNode;
+
+            boolean reconnect = false;
+
+            synchronized (stateMux) {
+                if (connState == ConnectionState.STARTED) {
+                    reconnect = true;
+
+                    connState = ConnectionState.DISCONNECTED;
+
+                    rtState.onCloseStart(disconnectError());
+                }
+            }
+
+            if (reconnect) {
+                UUID newId = UUID.randomUUID();
+
+                U.quietAndWarn(log, "Client node will try to reconnect with new id [" +
+                    "newId=" + newId +
+                    ", prevId=" + locNode.id() +
+                    ", locNode=" + locNode + ']');
+
+                runInWorkerThread(new ReconnectClosure(newId));
+            }
+        }
+        else {
+            rtState.errForClose = new IgniteCheckedException(msg);
+
+            notifySegmented();
+        }
+
+        // Stop any further processing.
+        return new ZookeeperClientFailedException(msg);
+    }
+
+    /**
+     * @param ctx Context for logging.
+     * @throws Exception If failed.
+     */
+    private void handleProcessedEvents(String ctx) throws Exception {
+        Iterator<ZkDiscoveryEventData> it = rtState.evtsData.evts.values().iterator();
+
+        List<ZkDiscoveryCustomEventData> newEvts = null;
+
+        ZkDiscoveryEventData prevEvtData = null;
+
+        while (it.hasNext()) {
+            ZkDiscoveryEventData evtData = it.next();
+
+            if (evtData.allAcksReceived()) {
+                if (prevEvtData != null) {
+                    if (log.isInfoEnabled()) {
+                        log.info("Previous event is not acked [" +
+                            "evtId=" + evtData.eventId() +
+                            ", prevEvtData=" + prevEvtData +
+                            ", remaining=" + prevEvtData.remainingAcks() + ']');
+                    }
+                }
+
+                prevEvtData = null;
+
+                switch (evtData.eventType()) {
+                    case ZkDiscoveryEventData.ZK_EVT_NODE_JOIN: {
+                        handleProcessedJoinEventAsync((ZkDiscoveryNodeJoinEventData)evtData);
+
+                        break;
+                    }
+
+                    case ZkDiscoveryEventData.ZK_EVT_CUSTOM_EVT: {
+                        DiscoverySpiCustomMessage ack = handleProcessedCustomEvent(ctx,
+                            (ZkDiscoveryCustomEventData)evtData);
+
+                        if (ack != null) {
+                            ZkDiscoveryCustomEventData ackEvtData = createAckEvent(
+                                ack,
+                                (ZkDiscoveryCustomEventData)evtData);
+
+                            if (newEvts == null)
+                                newEvts = new ArrayList<>();
+
+                            newEvts.add(ackEvtData);
+                        }
+
+                        break;
+                    }
+
+                    case ZkDiscoveryEventData.ZK_EVT_NODE_FAILED: {
+                        if (log.isDebugEnabled())
+                            log.debug("All nodes processed node fail [evtData=" + evtData + ']');
+
+                        break; // Do not need addition cleanup.
+                    }
+                }
+
+                it.remove();
+            }
+            else
+                prevEvtData = evtData;
+        }
+
+        if (newEvts != null) {
+            Collection<ZookeeperClusterNode> nodes = rtState.top.nodesByOrder.values();
+
+            for (int i = 0; i < newEvts.size(); i++)
+                rtState.evtsData.addEvent(nodes, newEvts.get(i));
+
+            saveAndProcessNewEvents();
+        }
+    }
+
+    /**
+     * @param ack Ack message.
+     * @param origEvt Original custom event.
+     * @return Event data.
+     * @throws Exception If failed.
+     */
+    private ZkDiscoveryCustomEventData createAckEvent(
+        DiscoverySpiCustomMessage ack,
+        ZkDiscoveryCustomEventData origEvt) throws Exception {
+        assert ack != null;
+
+        rtState.evtsData.evtIdGen++;
+
+        long evtId = rtState.evtsData.evtIdGen;
+
+        byte[] ackBytes = marshalZip(ack);
+
+        String path = zkPaths.ackEventDataPath(origEvt.eventId());
+
+        if (log.isDebugEnabled())
+            log.debug("Create ack event: " + path);
+
+        // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8194
+        rtState.zkClient.createIfNeeded(
+            path,
+            ackBytes,
+            CreateMode.PERSISTENT);
+
+        ZkDiscoveryCustomEventData ackEvtData = new ZkDiscoveryCustomEventData(
+            evtId,
+            origEvt.eventId(),
+            origEvt.topologyVersion(), // Use topology version from original event.
+            locNode.id(),
+            null,
+            null);
+
+        ackEvtData.resolvedMsg = ack;
+
+        if (log.isDebugEnabled()) {
+            log.debug("Generated CUSTOM event ack [origEvtId=" + origEvt.eventId() +
+                ", evt=" + ackEvtData +
+                ", evtSize=" + ackBytes.length +
+                ", msg=" + ack + ']');
+        }
+
+        return ackEvtData;
+    }
+
+    /**
+     * @param failedNodes Failed nodes.
+     * @throws Exception If failed.
+     */
+    private void handleProcessedEventsOnNodesFail(List<ZookeeperClusterNode> failedNodes) throws Exception {
+        boolean processed = false;
+
+        for (Iterator<Map.Entry<Long, ZkDiscoveryEventData>> it = rtState.evtsData.evts.entrySet().iterator(); it.hasNext();) {
+            Map.Entry<Long, ZkDiscoveryEventData> e = it.next();
+
+            ZkDiscoveryEventData evtData = e.getValue();
+
+            for (int i = 0; i < failedNodes.size(); i++) {
+                ZookeeperClusterNode failedNode = failedNodes.get(i);
+
+                if (evtData.onNodeFail(failedNode))
+                    processed = true;
+            }
+        }
+
+        if (processed)
+            handleProcessedEvents("fail-" + U.nodeIds(failedNodes));
+    }
+
+    /**
+     * @param evtData Event data.
+     * @throws Exception If failed.
+     */
+    private void handleProcessedJoinEventAsync(ZkDiscoveryNodeJoinEventData evtData) throws Exception {
+        if (log.isDebugEnabled())
+            log.debug("All nodes processed node join [evtData=" + evtData + ']');
+
+        for (int i = 0; i < evtData.joinedNodes.size(); i++) {
+            ZkJoinedNodeEvtData joinedEvtData = evtData.joinedNodes.get(i);
+
+            deleteJoiningNodeData(joinedEvtData.nodeId, joinedEvtData.joinDataPrefixId, joinedEvtData.joinDataPartCnt);
+
+            if (joinedEvtData.secSubjPartCnt > 0) {
+                deleteMultiplePartsAsync(rtState.zkClient,
+                    zkPaths.joinEventSecuritySubjectPath(evtData.eventId()),
+                    joinedEvtData.secSubjPartCnt);
+            }
+        }
+
+        deleteDataForJoinedAsync(evtData);
+    }
+
+    /**
+     * @param nodeId Node ID.
+     * @param joinDataPrefixId Path prefix.
+     * @param partCnt Parts count.
+     */
+    private void deleteJoiningNodeData(UUID nodeId, UUID joinDataPrefixId, int partCnt) {
+        String evtDataPath = zkPaths.joiningNodeDataPath(nodeId, joinDataPrefixId);
+
+        if (log.isDebugEnabled())
+            log.debug("Delete joining node data [path=" + evtDataPath + ']');
+
+        rtState.zkClient.deleteIfExistsAsync(evtDataPath);
+
+        if (partCnt > 1)
+            deleteMultiplePartsAsync(rtState.zkClient, evtDataPath + ":", partCnt);
+    }
+
+    /**
+     * @param evtData Event data.
+     */
+    private void deleteDataForJoinedAsync(ZkDiscoveryNodeJoinEventData evtData) {
+        String dataForJoinedPath = zkPaths.joinEventDataPathForJoined(evtData.eventId());
+
+        if (log.isDebugEnabled())
+            log.debug("Delete data for joined node [path=" + dataForJoinedPath + ']');
+
+        deleteMultiplePartsAsync(rtState.zkClient, dataForJoinedPath, evtData.dataForJoinedPartCnt);
+    }
+
+    /**
+     * @param ctx Context for log.
+     * @param evtData Event data.
+     * @return Ack message.
+     * @throws Exception If failed.
+     */
+    @Nullable private DiscoverySpiCustomMessage handleProcessedCustomEvent(String ctx, ZkDiscoveryCustomEventData evtData)
+        throws Exception {
+        if (log.isDebugEnabled())
+            log.debug("All nodes processed custom event [ctx=" + ctx + ", evtData=" + evtData + ']');
+
+        if (!evtData.ackEvent()) {
+            if (evtData.evtPath != null)
+                deleteCustomEventDataAsync(rtState.zkClient, evtData.evtPath);
+            else {
+                if (evtData.resolvedMsg instanceof ZkCommunicationErrorResolveFinishMessage) {
+                    UUID futId = ((ZkCommunicationErrorResolveFinishMessage)evtData.resolvedMsg).futId;
+
+                    ZkDistributedCollectDataFuture.deleteFutureData(rtState.zkClient, zkPaths, futId, log);
+                }
+            }
+
+            assert evtData.resolvedMsg != null || locNode.order() > evtData.topologyVersion() : evtData;
+
+            if (evtData.resolvedMsg != null)
+                return evtData.resolvedMsg.ackMessage();
+        }
+        else {
+            String path = zkPaths.ackEventDataPath(evtData.origEvtId);
+
+            if (log.isDebugEnabled())
+                log.debug("Delete path: " + path);
+
+            rtState.zkClient.deleteIfExistsAsync(path);
+        }
+
+        return null;
+    }
+
+    /**
+     * @param c Closure to run.
+     */
+    void runInWorkerThread(Runnable c) {
+        IgniteThreadPoolExecutor pool;
+
+        synchronized (stateMux) {
+            if (connState == ConnectionState.STOPPED) {
+                LT.warn(log, "Do not run closure, node is stopped.");
+
+                return;
+            }
+
+            if (utilityPool == null) {
+                utilityPool = new IgniteThreadPoolExecutor("zk-discovery-pool",
+                    igniteInstanceName,
+                    0,
+                    1,
+                    2000,
+                    new LinkedBlockingQueue<Runnable>());
+            }
+
+            pool = utilityPool;
+        }
+
+        pool.submit(c);
+    }
+
+    /**
+     *
+     */
+    public void stop() {
+        stop0(new IgniteSpiException("Node stopped"));
+    }
+
+    /**
+     * @param e Error.
+     */
+    private void stop0(Throwable e) {
+        if (!stop.compareAndSet(false, true))
+            return;
+
+        ZkRuntimeState rtState = this.rtState;
+
+        if (rtState.zkClient != null && rtState.locNodeZkPath != null && rtState.zkClient.connected()) {
+            try {
+                rtState.zkClient.deleteIfExistsNoRetry(rtState.locNodeZkPath, -1);
+            }
+            catch (Exception err) {
+                if (log.isDebugEnabled())
+                    log.debug("Failed to delete local node's znode on stop: " + err);
+            }
+        }
+
+        IgniteCheckedException err = new IgniteCheckedException("Node stopped.");
+
+        synchronized (stateMux) {
+            connState = ConnectionState.STOPPED;
+
+            rtState.onCloseStart(err);
+        }
+
+        IgniteUtils.shutdownNow(ZookeeperDiscoveryImpl.class, utilityPool, log);
+
+        busyLock.block();
+
+        busyLock.unblock();
+
+        joinFut.onDone(e);
+
+        ZookeeperClient zkClient = rtState.zkClient;
+
+        if (zkClient != null)
+            zkClient.close();
+
+        finishFutures(err);
+    }
+
+    /**
+     * @param err Error.
+     */
+    private void finishFutures(IgniteCheckedException err) {
+        ZkCommunicationErrorProcessFuture commErrFut = commErrProcFut.get();
+
+        if (commErrFut != null)
+            commErrFut.onError(err);
+
+        for (PingFuture fut : pingFuts.values())
+            fut.onDone(err);
+    }
+
+    /**
+     * @param busyLock Busy lock.
+     * @param err Error.
+     */
+    void onFatalError(GridSpinBusyLock busyLock, Throwable err) {
+        busyLock.leaveBusy();
+
+        if (err instanceof ZookeeperClientFailedException)
+            return; // Processed by ZookeeperClient listener.
+
+        Ignite ignite = spi.ignite();
+
+        if (stopping() || ignite == null)
+            return;
+
+        U.error(log, "Fatal error in ZookeeperDiscovery. " +
+            "Stopping the node in order to prevent cluster wide instability.", err);
+
+        stop0(err);
+
+        new Thread(new Runnable() {
+            @Override public void run() {
+                try {
+                    IgnitionEx.stop(igniteInstanceName, true, true);
+
+                    U.log(log, "Stopped the node successfully in response to fatal error in ZookeeperDiscoverySpi.");
+                }
+                catch (Throwable e) {
+                    U.error(log, "Failed to stop the node successfully in response to fatal error in " +
+                        "ZookeeperDiscoverySpi.", e);
+                }
+            }
+        }, "node-stop-thread").start();
+
+        if (err instanceof Error)
+            throw (Error)err;
+    }
+
+    /**
+     * @param zipBytes Zip-compressed bytes.
+     * @return Unmarshalled object.
+     * @throws IgniteCheckedException If failed.
+     */
+    private <T> T unmarshalZip(byte[] zipBytes) throws Exception {
+        assert zipBytes != null && zipBytes.length > 0;
+
+        InflaterInputStream in = new InflaterInputStream(new ByteArrayInputStream(zipBytes));
+
+        return marsh.unmarshal(in, U.resolveClassLoader(spi.ignite().configuration()));
+    }
+
+    /**
+     * @param obj Object.
+     * @return Bytes.
+     * @throws IgniteCheckedException If failed.
+     */
+    byte[] marshalZip(Object obj) throws IgniteCheckedException {
+        assert obj != null;
+
+        return zip(U.marshal(marsh, obj));
+    }
+
+    /**
+     * @param bytes Bytes to compress.
+     * @return Zip-compressed bytes.
+     */
+    private static byte[] zip(byte[] bytes) {
+        Deflater deflater = new Deflater();
+
+        deflater.setInput(bytes);
+        deflater.finish();
+
+        GridByteArrayOutputStream out = new GridByteArrayOutputStream(bytes.length);
+
+        final byte[] buf = new byte[bytes.length];
+
+        while (!deflater.finished()) {
+            int cnt = deflater.deflate(buf);
+
+            out.write(buf, 0, cnt);
+        }
+
+        return out.toByteArray();
+    }
+
+    /**
+     * @param zipBytes Zip-compressed bytes.
+     * @return Uncompressed bytes.
+     * @throws DataFormatException If compressed data format is invalid.
+     */
+    public static byte[] unzip(byte[] zipBytes) throws DataFormatException {
+        Inflater inflater = new Inflater();
+
+        inflater.setInput(zipBytes);
+
+        GridByteArrayOutputStream out = new GridByteArrayOutputStream(zipBytes.length * 2);
+
+        final byte[] buf = new byte[zipBytes.length];
+
+        while (!inflater.finished()) {
+            int cnt = inflater.inflate(buf);
+
+            out.write(buf, 0, cnt);
+        }
+
+        return out.toByteArray();
+    }
+
+    /**
+     *
+     */
+    private class ReconnectClosure implements Runnable {
+        /** */
+        private final UUID newId;
+
+        /**
+         * @param newId New ID.
+         */
+        ReconnectClosure(UUID newId) {
+            assert newId != null;
+
+            this.newId = newId;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            finishFutures(disconnectError());
+
+            busyLock.block();
+
+            busyLock.unblock();
+
+            doReconnect(newId);
+        }
+    }
+
+    /**
+     *
+     */
+    private class ConnectionLossListener implements IgniteRunnable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            if (clientReconnectEnabled) {
+                synchronized (stateMux) {
+                    if (connState == ConnectionState.STARTED) {
+                        connState = ConnectionState.DISCONNECTED;
+
+                        rtState.onCloseStart(disconnectError());
+                    }
+                    else
+                        return;
+                }
+
+                UUID newId = UUID.randomUUID();
+
+                U.quietAndWarn(log, "Connection to Zookeeper server is lost, local node will try to reconnect with new id [" +
+                    "newId=" + newId +
+                    ", prevId=" + locNode.id() +
+                    ", locNode=" + locNode + ']');
+
+                runInWorkerThread(new ReconnectClosure(newId));
+            }
+            else {
+                U.warn(log, "Connection to Zookeeper server is lost, local node SEGMENTED.");
+
+                onSegmented(new IgniteSpiException("Zookeeper connection loss."));
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private class ZkWatcher extends ZkAbstractWatcher implements ZkRuntimeState.ZkWatcher {
+        /**
+         * @param rtState Runtime state.
+         */
+        ZkWatcher(ZkRuntimeState rtState) {
+            super(rtState, ZookeeperDiscoveryImpl.this);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void process0(WatchedEvent evt) {
+            if (evt.getType() == Event.EventType.NodeDataChanged) {
+                if (evt.getPath().equals(zkPaths.evtsPath)) {
+                    if (!rtState.crd)
+                        rtState.zkClient.getDataAsync(evt.getPath(), this, this);
+                }
+                else
+                    U.warn(log, "Received NodeDataChanged for unexpected path: " + evt.getPath());
+            }
+            else if (evt.getType() == Event.EventType.NodeChildrenChanged) {
+                if (evt.getPath().equals(zkPaths.aliveNodesDir))
+                    rtState.zkClient.getChildrenAsync(evt.getPath(), this, this);
+                else if (evt.getPath().equals(zkPaths.customEvtsDir))
+                    rtState.zkClient.getChildrenAsync(evt.getPath(), this, this);
+                else
+                    U.warn(log, "Received NodeChildrenChanged for unexpected path: " + evt.getPath());
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+            if (!onProcessStart())
+                return;
+
+            try {
+                assert rc == 0 : KeeperException.Code.get(rc);
+
+                if (path.equals(zkPaths.aliveNodesDir))
+                    generateTopologyEvents(children);
+                else if (path.equals(zkPaths.customEvtsDir))
+                    generateCustomEvents(children);
+                else
+                    U.warn(log, "Children callback for unexpected path: " + path);
+
+                onProcessEnd();
+            }
+            catch (Throwable e) {
+                onProcessError(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) {
+            if (!onProcessStart())
+                return;
+
+            try {
+                assert rc == 0 : KeeperException.Code.get(rc);
+
+                if (path.equals(zkPaths.evtsPath)) {
+                    if (!rtState.crd)
+                        processNewEvents(data);
+                }
+                else
+                    U.warn(log, "Data callback for unknown path: " + path);
+
+                onProcessEnd();
+            }
+            catch (Throwable e) {
+                onProcessError(e);
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private class AliveNodeDataWatcher extends ZkAbstractWatcher implements ZkRuntimeState.ZkAliveNodeDataWatcher {
+        /**
+         * @param rtState Runtime state.
+         */
+        AliveNodeDataWatcher(ZkRuntimeState rtState) {
+            super(rtState, ZookeeperDiscoveryImpl.this);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void process0(WatchedEvent evt) {
+            if (evt.getType() == Event.EventType.NodeDataChanged)
+                rtState.zkClient.getDataAsync(evt.getPath(), this, this);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) {
+            if (!onProcessStart())
+                return;
+
+            try {
+                assert rtState.crd;
+
+                processResult0(rc, path, data);
+
+                onProcessEnd();
+            }
+            catch (Throwable e) {
+                onProcessError(e);
+            }
+        }
+
+        /**
+         * @param rc Result code.
+         * @param path Path.
+         * @param data Data.
+         * @throws Exception If failed.
+         */
+        private void processResult0(int rc, String path, byte[] data) throws Exception {
+            if (rc == KeeperException.Code.NONODE.intValue()) {
+                if (log.isDebugEnabled())
+                    log.debug("Alive node callaback, no node: " + path);
+
+                return;
+            }
+
+            assert rc == 0 : KeeperException.Code.get(rc);
+
+            if (data.length > 0) {
+                ZkAliveNodeData nodeData = unmarshalZip(data);
+
+                Long nodeInternalId = ZkIgnitePaths.aliveInternalId(path);
+
+                Iterator<ZkDiscoveryEventData> it = rtState.evtsData.evts.values().iterator();
+
+                boolean processed = false;
+
+                while (it.hasNext()) {
+                    ZkDiscoveryEventData evtData = it.next();
+
+                    if (evtData.onAckReceived(nodeInternalId, nodeData.lastProcEvt))
+                        processed = true;
+                }
+
+                if (processed)
+                    handleProcessedEvents("ack-" + nodeInternalId);
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private abstract class PreviousNodeWatcher extends ZkAbstractWatcher implements AsyncCallback.StatCallback {
+        /**
+         * @param rtState Runtime state.
+         */
+        PreviousNodeWatcher(ZkRuntimeState rtState) {
+            super(rtState, ZookeeperDiscoveryImpl.this);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void process0(WatchedEvent evt) {
+            if (evt.getType() == Event.EventType.NodeDeleted)
+                onPreviousNodeFail();
+            else {
+                if (evt.getType() != Event.EventType.None)
+                    rtState.zkClient.existsAsync(evt.getPath(), this, this);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx, Stat stat) {
+            if (!onProcessStart())
+                return;
+
+            try {
+                assert rc == 0 || rc == KeeperException.Code.NONODE.intValue() : KeeperException.Code.get(rc);
+
+                if (rc == KeeperException.Code.NONODE.intValue() || stat == null)
+                    onPreviousNodeFail();
+
+                onProcessEnd();
+            }
+            catch (Throwable e) {
+                onProcessError(e);
+            }
+        }
+
+        /**
+         *
+         */
+        abstract void onPreviousNodeFail();
+    }
+
+    /**
+     *
+     */
+    private class ServerPreviousNodeWatcher extends PreviousNodeWatcher {
+        /**
+         * @param rtState Runtime state.
+         */
+        ServerPreviousNodeWatcher(ZkRuntimeState rtState) {
+            super(rtState);
+
+            assert !locNode.isClient() : locNode;
+        }
+
+        /** {@inheritDoc} */
+        @Override void onPreviousNodeFail() {
+            if (log.isInfoEnabled())
+                log.info("Previous server node failed, check is node new coordinator [locId=" + locNode.id() + ']');
+
+            rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, null, new CheckCoordinatorCallback(rtState));
+        }
+    }
+
+    /**
+     *
+     */
+    private class ClientPreviousNodeWatcher extends PreviousNodeWatcher {
+        /**
+         * @param rtState Runtime state.
+         */
+        ClientPreviousNodeWatcher(ZkRuntimeState rtState) {
+            super(rtState);
+
+            assert locNode.isClient() : locNode;
+        }
+
+        /** {@inheritDoc} */
+        @Override void onPreviousNodeFail() {
+            if (log.isInfoEnabled())
+                log.info("Watched node failed, check if there are alive servers [locId=" + locNode.id() + ']');
+
+            rtState.zkClient.getChildrenAsync(zkPaths.aliveNodesDir, null, new CheckClientsStatusCallback(rtState));
+        }
+    }
+
+    /**
+     *
+     */
+    class CheckCoordinatorCallback extends ZkAbstractChildrenCallback {
+        /**
+         * @param rtState Runtime state.
+         */
+        CheckCoordinatorCallback(ZkRuntimeState rtState) {
+            super(rtState, ZookeeperDiscoveryImpl.this);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult0(int rc, String path, Object ctx, List<String> children, Stat stat)
+            throws Exception
+        {
+            assert rc == 0 : KeeperException.Code.get(rc);
+
+            checkIsCoordinator(children);
+        }
+    }
+
+    /**
+     *
+     */
+    class CheckClientsStatusCallback extends ZkAbstractChildrenCallback {
+        /**
+         * @param rtState Runtime state.
+         */
+        CheckClientsStatusCallback(ZkRuntimeState rtState) {
+            super(rtState, ZookeeperDiscoveryImpl.this);
+        }
+
+        /** {@inheritDoc} */
+        @Override void processResult0(int rc, String path, Object ctx, List<String> children, Stat stat)
+            throws Exception
+        {
+            assert rc == 0 : KeeperException.Code.get(rc);
+
+            checkClientsStatus(children);
+        }
+    }
+
+    /**
+     *
+     */
+    private class PingFuture extends GridFutureAdapter<Boolean> implements IgniteSpiTimeoutObject {
+        /** */
+        private final ZookeeperClusterNode node;
+
+        /** */
+        private final long endTime;
+
+        /** */
+        private final IgniteUuid id;
+
+        /** */
+        private final ZkRuntimeState rtState;
+
+        /**
+         * @param rtState Runtime state.
+         * @param node Node.
+         */
+        PingFuture(ZkRuntimeState rtState, ZookeeperClusterNode node) {
+            this.rtState = rtState;
+            this.node = node;
+
+            id = IgniteUuid.fromUuid(node.id());
+
+            endTime = System.currentTimeMillis() + node.sessionTimeout() + 1000;
+        };
+
+        /** {@inheritDoc} */
+        @Override public IgniteUuid id() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long endTime() {
+            return endTime;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onTimeout() {
+            if (checkNodeAndState()) {
+                runInWorkerThread(new ZkRunnable(rtState, ZookeeperDiscoveryImpl.this) {
+                    @Override protected void run0() throws Exception {
+                        if (checkNodeAndState()) {
+                            try {
+                                for (String path : rtState.zkClient.getChildren(zkPaths.aliveNodesDir)) {
+                                    if (node.internalId() == ZkIgnitePaths.aliveInternalId(path)) {
+                                        onDone(true);
+
+                                        return;
+                                    }
+                                }
+
+                                onDone(false);
+                            }
+                            catch (Exception e) {
+                                onDone(e);
+
+                                throw e;
+                            }
+                        }
+                    }
+
+                    @Override void onStartFailed() {
+                        onDone(rtState.errForClose);
+                    }
+                });
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean onDone(@Nullable Boolean res, @Nullable Throwable err) {
+            if (super.onDone(res, err)) {
+                pingFuts.remove(node.order(), this);
+
+                return true;
+            }
+
+            return false;
+        }
+
+        /**
+         * @return {@code False} if future was completed.
+         */
+        boolean checkNodeAndState() {
+            if (isDone())
+                return false;
+
+            Exception err = rtState.errForClose;
+
+            if (err != null) {
+                onDone(err);
+
+                return false;
+            }
+
+            ConnectionState connState = ZookeeperDiscoveryImpl.this.connState;
+
+            if (connState == ConnectionState.DISCONNECTED) {
+                onDone(new IgniteClientDisconnectedException(null, "Client is disconnected."));
+
+                return false;
+            }
+            else if (connState == ConnectionState.STOPPED) {
+                onDone(new IgniteException("Node stopped."));
+
+                return false;
+            }
+
+            if (node(node.id()) == null) {
+                onDone(false);
+
+                return false;
+            }
+
+            return true;
+        }
+    }
+
+    /**
+     *
+     */
+    enum ConnectionState {
+        /** */
+        STARTED,
+        /** */
+        DISCONNECTED,
+        /** */
+        STOPPED
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ZookeeperNodeStart.java b/modules/zookeeper/src/test/java/org/apache/ZookeeperNodeStart.java
new file mode 100644
index 0000000000000..ef4d5f47f51fa
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ZookeeperNodeStart.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache;
+
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+
+/**
+ *
+ */
+public class ZookeeperNodeStart {
+    public static void main(String[] args) throws Exception {
+        try {
+            IgniteConfiguration cfg = new IgniteConfiguration();
+
+            ZookeeperDiscoverySpi spi = new ZookeeperDiscoverySpi();
+
+            spi.setZkConnectionString("localhost:2181");
+
+            cfg.setDiscoverySpi(spi);
+
+            Ignition.start(cfg);
+        }
+        catch (Throwable e) {
+            e.printStackTrace(System.out);
+
+            System.exit(1);
+        }
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/IgniteCacheEntryListenerWithZkDiscoAtomicTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/IgniteCacheEntryListenerWithZkDiscoAtomicTest.java
new file mode 100644
index 0000000000000..754a6bf05b10a
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/IgniteCacheEntryListenerWithZkDiscoAtomicTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk;
+
+import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicTest;
+
+/**
+ * Class is added to mute {@link #testConcurrentRegisterDeregister} test in ZooKeeper suite
+ * (see related ticket).
+ *
+ * When slow down is tracked down and fixed this class can be replaced back with its parent.
+ */
+public class IgniteCacheEntryListenerWithZkDiscoAtomicTest extends IgniteCacheEntryListenerAtomicTest {
+    /** {@inheritDoc} */
+    @Override public void testConcurrentRegisterDeregister() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8109");
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
new file mode 100644
index 0000000000000..766635cb5580c
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import junit.framework.TestSuite;
+import org.apache.curator.test.InstanceSpec;
+import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.config.GridTestProperties;
+
+/**
+ * Allows to run regular Ignite tests with {@link org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi}.
+ */
+public abstract class ZookeeperDiscoverySpiAbstractTestSuite extends TestSuite {
+    /** */
+    private static TestingCluster testingCluster;
+
+    /**
+     * @throws Exception If failed.
+     */
+    public static void initSuite() throws Exception {
+        System.setProperty("zookeeper.forceSync", "false");
+
+        testingCluster = createTestingCluster(3);
+
+        testingCluster.start();
+
+        System.setProperty(GridTestProperties.IGNITE_CFG_PREPROCESSOR_CLS, ZookeeperDiscoverySpiAbstractTestSuite.class.getName());
+    }
+
+    /**
+     * Called via reflection by {@link org.apache.ignite.testframework.junits.GridAbstractTest}.
+     *
+     * @param cfg Configuration to change.
+     */
+    public synchronized static void preprocessConfiguration(IgniteConfiguration cfg) {
+        if (testingCluster == null)
+            throw new IllegalStateException("Test Zookeeper cluster is not started.");
+
+        ZookeeperDiscoverySpi zkSpi = new ZookeeperDiscoverySpi();
+
+        DiscoverySpi spi = cfg.getDiscoverySpi();
+
+        if (spi instanceof TcpDiscoverySpi)
+            zkSpi.setClientReconnectDisabled(((TcpDiscoverySpi)spi).isClientReconnectDisabled());
+
+        zkSpi.setSessionTimeout(30_000);
+        zkSpi.setZkConnectionString(testingCluster.getConnectString());
+
+        cfg.setDiscoverySpi(zkSpi);
+    }
+
+    /**
+     * @param instances Number of instances in
+     * @return Test cluster.
+     */
+    public static TestingCluster createTestingCluster(int instances) {
+        String tmpDir = System.getProperty("java.io.tmpdir");
+
+        List<InstanceSpec> specs = new ArrayList<>();
+
+        for (int i = 0; i < instances; i++) {
+            File file = new File(tmpDir, "apacheIgniteTestZk-" + i);
+
+            if (file.isDirectory())
+                deleteRecursively0(file);
+            else {
+                if (!file.mkdirs())
+                    throw new IgniteException("Failed to create directory for test Zookeeper server: " + file.getAbsolutePath());
+            }
+
+            specs.add(new InstanceSpec(file, -1, -1, -1, true, -1, -1, 500));
+        }
+
+        return new TestingCluster(specs);
+    }
+
+    /**
+     * @param file File or directory to delete.
+     */
+    private static void deleteRecursively0(File file) {
+        File[] files = file.listFiles();
+
+        if (files == null)
+            return;
+
+        for (File f : files) {
+            if (f.isDirectory())
+                deleteRecursively0(f);
+            else {
+                if (!f.delete())
+                    throw new IgniteException("Failed to delete file: " + f.getAbsolutePath());
+            }
+        }
+    }
+
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
new file mode 100644
index 0000000000000..860488b1a926f
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClientTest;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoverySpiSaslSuccessfulAuthTest;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoverySpiTest;
+
+/**
+ *
+ */
+public class ZookeeperDiscoverySpiTestSuite1 extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        System.setProperty("zookeeper.forceSync", "false");
+
+        TestSuite suite = new TestSuite("ZookeeperDiscoverySpi Test Suite");
+
+        suite.addTestSuite(ZookeeperClientTest.class);
+        suite.addTestSuite(ZookeeperDiscoverySpiTest.class);
+        suite.addTestSuite(ZookeeperDiscoverySpiSaslSuccessfulAuthTest.class);
+
+        return suite;
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
new file mode 100644
index 0000000000000..3775aa1df9b3d
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk;
+
+import junit.framework.TestSuite;
+import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest;
+import org.apache.ignite.internal.IgniteClientReconnectCacheTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicTest;
+import org.apache.ignite.internal.processors.cache.datastructures.IgniteClientDataStructuresTest;
+import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedNodeRestartTxSelfTest;
+import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedSequenceApiSelfTest;
+import org.apache.ignite.internal.processors.cache.datastructures.replicated.GridCacheReplicatedSequenceApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePutRetryAtomicSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCachePutRetryTransactionalSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNodeRestartTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedNodeRestartSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheReplicatedQuerySelfTest;
+import org.apache.ignite.internal.processors.cache.multijvm.GridCacheAtomicMultiJvmFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.multijvm.GridCachePartitionedMultiJvmFullApiSelfTest;
+import org.apache.ignite.internal.processors.continuous.GridEventConsumeSelfTest;
+
+/**
+ * Regular Ignite tests executed with {@link org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi}.
+ */
+public class ZookeeperDiscoverySpiTestSuite2 extends ZookeeperDiscoverySpiAbstractTestSuite {
+    /** */
+    private static TestingCluster testingCluster;
+
+    /**
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        System.setProperty("H2_JDBC_CONNECTIONS", "500"); // For multi-jvm tests.
+
+        initSuite();
+
+        TestSuite suite = new TestSuite("ZookeeperDiscoverySpi Test Suite");
+
+        suite.addTestSuite(ZookeeperDiscoverySuitePreprocessorTest.class);
+
+        suite.addTestSuite(GridCacheReplicatedNodeRestartSelfTest.class);
+        suite.addTestSuite(GridCachePartitionedNodeRestartTest.class);
+
+        suite.addTestSuite(IgniteCacheEntryListenerWithZkDiscoAtomicTest.class);
+
+        suite.addTestSuite(GridEventConsumeSelfTest.class);
+
+        suite.addTestSuite(IgniteClientReconnectCacheTest.class);
+
+        suite.addTestSuite(IgniteCachePutRetryAtomicSelfTest.class);
+        suite.addTestSuite(IgniteCachePutRetryTransactionalSelfTest.class);
+
+        suite.addTestSuite(ClusterNodeMetricsUpdateTest.class);
+
+        suite.addTestSuite(GridCachePartitionedMultiNodeFullApiSelfTest.class);
+        suite.addTestSuite(GridCacheReplicatedMultiNodeFullApiSelfTest.class);
+
+        suite.addTestSuite(GridCacheAtomicMultiNodeFullApiSelfTest.class);
+        suite.addTestSuite(GridCacheReplicatedAtomicMultiNodeFullApiSelfTest.class);
+
+        suite.addTestSuite(GridCachePartitionedNodeRestartTxSelfTest.class);
+        suite.addTestSuite(IgniteClientDataStructuresTest.class);
+        suite.addTestSuite(GridCacheReplicatedSequenceApiSelfTest.class);
+        suite.addTestSuite(GridCachePartitionedSequenceApiSelfTest.class);
+
+        suite.addTestSuite(IgniteCacheReplicatedQuerySelfTest.class);
+
+        suite.addTestSuite(GridCacheAtomicMultiJvmFullApiSelfTest.class);
+        suite.addTestSuite(GridCachePartitionedMultiJvmFullApiSelfTest.class);
+
+        return suite;
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySuitePreprocessorTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySuitePreprocessorTest.java
new file mode 100644
index 0000000000000..28cf17f017c42
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySuitePreprocessorTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk;
+
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.config.GridTestProperties;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Sanity test verifying that configuration callback specified via
+ * {@link GridTestProperties#IGNITE_CFG_PREPROCESSOR_CLS} really works.
+ * <p>
+ * This test should be run as part of {@link ZookeeperDiscoverySpiTestSuite2}.
+ */
+public class ZookeeperDiscoverySuitePreprocessorTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        // Test sets TcpDiscoverySpi, but it should be automatically changed to ZookeeperDiscoverySpi.
+        TcpDiscoverySpi spi = new TcpDiscoverySpi();
+
+        spi.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(spi);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSpiConfigurationIsChanged() throws Exception {
+        startGrid(0);
+
+        checkDiscoverySpi(1);
+
+        startGrid(1);
+
+        checkDiscoverySpi(2);
+
+        startGridsMultiThreaded(2, 2);
+
+        checkDiscoverySpi(4);
+
+        startGrid();
+
+        checkDiscoverySpi(5);
+    }
+
+    /**
+     * @param expNodes Expected nodes number.
+     * @throws Exception If failed.
+     */
+    private void checkDiscoverySpi(int expNodes) throws Exception {
+        List<Ignite> nodes = G.allGrids();
+
+        assertEquals(expNodes, nodes.size());
+
+        for (Ignite node : nodes) {
+            DiscoverySpi spi = node.configuration().getDiscoverySpi();
+
+            assertTrue("Node should be started with " + ZookeeperDiscoverySpi.class.getName(),
+                    spi instanceof ZookeeperDiscoverySpi);
+        }
+
+        waitForTopology(expNodes);
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
new file mode 100644
index 0000000000000..e7cb97a39434f
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ *
+ */
+public class ZookeeperClientTest extends GridCommonAbstractTest {
+    /** */
+    private static final int SES_TIMEOUT = 60_000;
+
+    /** */
+    private TestingCluster zkCluster;
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        closeZK();
+
+        super.afterTest();
+    }
+
+    /**
+     * @param sesTimeout Session timeout.
+     * @return Client.
+     * @throws Exception If failed.
+     */
+    private ZookeeperClient createClient(int sesTimeout) throws Exception {
+        return new ZookeeperClient(log, zkCluster.getConnectString(), sesTimeout, null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSaveLargeValue() throws Exception {
+        startZK(1);
+
+        final ZookeeperClient client = createClient(SES_TIMEOUT);
+
+        byte[] data = new byte[1024 * 1024];
+
+        String basePath = "/ignite";
+
+        assertTrue(client.needSplitNodeData(basePath, data, 2));
+
+        List<byte[]> parts = client.splitNodeData(basePath, data, 2);
+
+        assertTrue(parts.size() > 1);
+
+        ZooKeeper zk = client.zk();
+
+        for (int i = 0; i < parts.size(); i++) {
+            byte[] part = parts.get(i);
+
+            assertTrue(part.length > 0);
+
+            String path0 = basePath + ":" + i;
+
+            zk.create(path0, part, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClose() throws Exception {
+        startZK(1);
+
+        final ZookeeperClient client = createClient(SES_TIMEOUT);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        client.zk().close();
+
+        GridTestUtils.assertThrows(log, new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+
+                return null;
+            }
+        }, ZookeeperClientFailedException.class, null);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCreateAll() throws Exception {
+        startZK(1);
+
+        ZookeeperClient client = createClient(SES_TIMEOUT);
+
+        client.createIfNeeded("/apacheIgnite", null, CreateMode.PERSISTENT);
+
+        List<String> paths = new ArrayList<>();
+
+        paths.add("/apacheIgnite/1");
+        paths.add("/apacheIgnite/2");
+        paths.add("/apacheIgnite/3");
+
+        client.createAll(paths, CreateMode.PERSISTENT);
+
+        assertEquals(3, client.getChildren("/apacheIgnite").size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeleteAll() throws Exception {
+        startZK(1);
+
+        ZookeeperClient client = createClient(SES_TIMEOUT);
+
+        client.createIfNeeded("/apacheIgnite", null, CreateMode.PERSISTENT);
+        client.createIfNeeded("/apacheIgnite/1", null, CreateMode.PERSISTENT);
+        client.createIfNeeded("/apacheIgnite/2", null, CreateMode.PERSISTENT);
+
+        client.deleteAll("/apacheIgnite", Arrays.asList("1", "2"), -1);
+
+        assertTrue(client.getChildren("/apacheIgnite").isEmpty());
+
+        client.createIfNeeded("/apacheIgnite/1", null, CreateMode.PERSISTENT);
+        client.deleteAll("/apacheIgnite", Collections.singletonList("1"), -1);
+
+        assertTrue(client.getChildren("/apacheIgnite").isEmpty());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLoss1() throws Exception {
+        ZookeeperClient client = new ZookeeperClient(log, "localhost:2200", 3000, null);
+
+        try {
+            client.createIfNeeded("/apacheIgnite", null, CreateMode.PERSISTENT);
+
+            fail();
+        }
+        catch (ZookeeperClientFailedException e) {
+            info("Expected error: " + e);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLoss2() throws Exception {
+        startZK(1);
+
+        ZookeeperClient client = createClient(3000);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        closeZK();
+
+        try {
+            client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+
+            fail();
+        }
+        catch (ZookeeperClientFailedException e) {
+            info("Expected error: " + e);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLoss3() throws Exception {
+        startZK(1);
+
+        CallbackFuture cb = new CallbackFuture();
+
+        ZookeeperClient client = new ZookeeperClient(log, zkCluster.getConnectString(), 3000, cb);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        closeZK();
+
+        final AtomicBoolean res = new AtomicBoolean();
+
+        client.getChildrenAsync("/apacheIgnite1", null, new AsyncCallback.Children2Callback() {
+            @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+                if (rc == 0)
+                    res.set(true);
+            }
+        });
+
+        cb.get(60_000);
+
+        assertFalse(res.get());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionLoss4() throws Exception {
+        startZK(1);
+
+        CallbackFuture cb = new CallbackFuture();
+
+        final ZookeeperClient client = new ZookeeperClient(log, zkCluster.getConnectString(), 3000, cb);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        final CountDownLatch l = new CountDownLatch(1);
+
+        client.getChildrenAsync("/apacheIgnite1", null, new AsyncCallback.Children2Callback() {
+            @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+                closeZK();
+
+                try {
+                    client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+                }
+                catch (ZookeeperClientFailedException e) {
+                    info("Expected error: " + e);
+
+                    l.countDown();
+                }
+                catch (Exception e) {
+                    fail("Unexpected error: " + e);
+                }
+            }
+        });
+
+        assertTrue(l.await(10, TimeUnit.SECONDS));
+
+        cb.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnect1() throws Exception {
+        startZK(1);
+
+        ZookeeperClient client = createClient(SES_TIMEOUT);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        zkCluster.getServers().get(0).stop();
+
+        IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                U.sleep(2000);
+
+                info("Restart zookeeper server");
+
+                zkCluster.getServers().get(0).restart();
+
+                info("Zookeeper server restarted");
+
+                return null;
+            }
+        }, "start-zk");
+
+        client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+
+        fut.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnect1_Callback() throws Exception {
+        startZK(1);
+
+        ZookeeperClient client = createClient(SES_TIMEOUT);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        zkCluster.getServers().get(0).stop();
+
+        final CountDownLatch l = new CountDownLatch(1);
+
+        client.getChildrenAsync("/apacheIgnite1", null, new AsyncCallback.Children2Callback() {
+            @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+                info("Callback: " + rc);
+
+                if (rc == 0)
+                    l.countDown();
+            }
+        });
+
+        IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                U.sleep(2000);
+
+                info("Restart zookeeper server");
+
+                zkCluster.getServers().get(0).restart();
+
+                info("Zookeeper server restarted");
+
+                return null;
+            }
+        }, "start-zk");
+
+        assertTrue(l.await(10, TimeUnit.SECONDS));
+
+        fut.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnect1_InCallback() throws Exception {
+        startZK(1);
+
+        final ZookeeperClient client = createClient(SES_TIMEOUT);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        final CountDownLatch l = new CountDownLatch(1);
+
+        client.getChildrenAsync("/apacheIgnite1", null, new AsyncCallback.Children2Callback() {
+            @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+                try {
+                    zkCluster.getServers().get(0).stop();
+
+                    IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            U.sleep(2000);
+
+                            info("Restart zookeeper server");
+
+                            zkCluster.getServers().get(0).restart();
+
+                            info("Zookeeper server restarted");
+
+                            return null;
+                        }
+                    }, "start-zk");
+
+                    client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+
+                    l.countDown();
+
+                    fut.get();
+                }
+                catch (Exception e) {
+                    fail("Unexpected error: " + e);
+                }
+            }
+        });
+
+        assertTrue(l.await(10, TimeUnit.SECONDS));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnect2() throws Exception {
+        startZK(1);
+
+        ZookeeperClient client = createClient(SES_TIMEOUT);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        zkCluster.getServers().get(0).restart();
+
+        client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnect3() throws Exception {
+        startZK(3);
+
+        ZookeeperClient client = createClient(SES_TIMEOUT);
+
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        for (int i = 0; i < 30; i++) {
+            info("Iteration: " + i);
+
+            int idx = rnd.nextInt(3);
+
+            zkCluster.getServers().get(idx).restart();
+
+            doSleep(rnd.nextLong(100) + 1);
+
+            client.createIfNeeded("/apacheIgnite" + i, null, CreateMode.PERSISTENT);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnect4() throws Exception {
+        startZK(3);
+
+        ZookeeperClient client = new ZookeeperClient(log,
+            zkCluster.getServers().get(2).getInstanceSpec().getConnectString(),
+            60_000,
+            null);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        zkCluster.getServers().get(0).stop();
+        zkCluster.getServers().get(1).stop();
+
+        IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                U.sleep(2000);
+
+                info("Restart zookeeper server");
+
+                zkCluster.getServers().get(0).restart();
+
+                info("Zookeeper server restarted");
+
+                return null;
+            }
+        }, "start-zk");
+
+        client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT);
+
+        fut.get();
+    }
+
+    /**
+     * @param instances Number of servers in ZK ensemble.
+     * @throws Exception If failed.
+     */
+    private void startZK(int instances) throws Exception {
+        assert zkCluster == null;
+
+        zkCluster = new TestingCluster(instances);
+
+        zkCluster.start();
+    }
+
+    /**
+     *
+     */
+    private void closeZK() {
+        if (zkCluster != null) {
+            try {
+                zkCluster.close();
+            }
+            catch (Exception e) {
+                U.error(log, "Failed to stop Zookeeper client: " + e, e);
+            }
+
+            zkCluster = null;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class CallbackFuture extends GridFutureAdapter<Void> implements IgniteRunnable {
+        /** {@inheritDoc} */
+        @Override public void run() {
+            onDone();
+        }
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslAuthAbstractTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslAuthAbstractTest.java
new file mode 100644
index 0000000000000..ac94bf2dd88a8
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslAuthAbstractTest.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Paths;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.apache.zookeeper.server.ZKDatabase;
+import org.apache.zookeeper.server.ZooKeeperServer;
+
+import static org.apache.curator.test.DirectoryUtils.deleteRecursively;
+
+/**
+ * Implements methods to prepare SASL tests infrastructure: jaas.conf files, starting up ZooKeeper server,
+ * clean up procedures when the test has finished etc.
+ */
+public abstract class ZookeeperDiscoverySpiSaslAuthAbstractTest extends GridCommonAbstractTest {
+    /** */
+    private File tmpDir = createTmpDir();
+
+    /** */
+    private static final String JAAS_CONF_FILE = "jaas.conf";
+
+    /** */
+    private static final String AUTH_PROVIDER = "zookeeper.authProvider.1";
+
+    /** */
+    private static final String SASL_CONFIG = "java.security.auth.login.config";
+
+    /** */
+    private long joinTimeout = 2_000;
+
+    /** */
+    private long sesTimeout = 10_000;
+
+    /** */
+    private ServerCnxnFactory serverFactory;
+
+    /** */
+    private String hostPort = "localhost:2181";
+
+    /** */
+    private int maxCnxns;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String instanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(instanceName);
+
+        ZookeeperDiscoverySpi zkSpi = new ZookeeperDiscoverySpi();
+
+        if (joinTimeout != 0)
+            zkSpi.setJoinTimeout(joinTimeout);
+
+        zkSpi.setSessionTimeout(sesTimeout > 0 ? sesTimeout : 10_000);
+
+        zkSpi.setZkConnectionString(hostPort);
+
+        cfg.setDiscoverySpi(zkSpi);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        prepareJaasConfigFile();
+
+        prepareSaslSystemProperties();
+
+        startZooKeeperServer();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopZooKeeperServer();
+
+        stopAllGrids();
+
+        clearSaslSystemProperties();
+
+        clearTmpDir();
+    }
+
+    /** */
+    private void clearTmpDir() throws Exception {
+        deleteRecursively(tmpDir);
+    }
+
+    /** */
+    protected void clearSaslSystemProperties() {
+        System.clearProperty(AUTH_PROVIDER);
+
+        System.clearProperty(SASL_CONFIG);
+
+        System.clearProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void prepareJaasConfigFile() throws Exception {
+        U.ensureDirectory(tmpDir, "Temp directory for JAAS configuration file wasn't created", null);
+
+        File saslConfFile = new File(tmpDir, JAAS_CONF_FILE);
+
+        FileWriter fwriter = new FileWriter(saslConfFile);
+
+        writeServerConfigSection(fwriter, "validPassword");
+
+        writeClientConfigSection(fwriter, "ValidZookeeperClient", "validPassword");
+
+        writeClientConfigSection(fwriter, "InvalidZookeeperClient", "invalidPassword");
+
+        fwriter.close();
+    }
+
+    /** */
+    private void prepareSaslSystemProperties() {
+        System.setProperty(SASL_CONFIG, Paths.get(tmpDir.getPath().toString(), JAAS_CONF_FILE).toString());
+
+        System.setProperty(AUTH_PROVIDER, "org.apache.zookeeper.server.auth.SASLAuthenticationProvider");
+    }
+
+    /** */
+    private void writeClientConfigSection(FileWriter fwriter, String clientName, String pass) throws IOException {
+        fwriter.write(clientName + "{\n" +
+            "       org.apache.zookeeper.server.auth.DigestLoginModule required\n" +
+            "       username=\"zkUser\"\n" +
+            "       password=\"" + pass + "\";\n" +
+            "};" + "\n");
+    }
+
+    /** */
+    private void writeServerConfigSection(FileWriter fwriter, String pass) throws IOException {
+        fwriter.write("Server {\n" +
+            "          org.apache.zookeeper.server.auth.DigestLoginModule required\n" +
+            "          user_zkUser=\"" + pass + "\";\n" +
+            "};\n");
+    }
+
+    /** */
+    private File createTmpDir() {
+        File jaasConfDir = Paths.get(System.getProperty("java.io.tmpdir"), "zk_disco_spi_test").toFile();
+
+        try {
+            U.ensureDirectory(jaasConfDir, "", null);
+        }
+        catch (IgniteCheckedException e) {
+            // ignored
+        }
+
+        return jaasConfDir;
+    }
+
+    /** */
+    private void stopZooKeeperServer() throws Exception {
+        shutdownServerInstance(serverFactory);
+        serverFactory = null;
+    }
+
+    /** */
+    private void shutdownServerInstance(ServerCnxnFactory factory)
+    {
+        if (factory != null) {
+            ZKDatabase zkDb = null;
+            {
+                ZooKeeperServer zs = getServer(factory);
+                if (zs != null)
+                    zkDb = zs.getZKDatabase();
+            }
+            factory.shutdown();
+            try {
+                if (zkDb != null)
+                    zkDb.close();
+            } catch (IOException ie) {
+                // ignore
+            }
+        }
+    }
+
+    /** */
+    private ZooKeeperServer getServer(ServerCnxnFactory fac) {
+        ZooKeeperServer zs = U.field(fac, "zkServer");
+
+        return zs;
+    }
+
+    /** */
+    private void startZooKeeperServer() throws Exception {
+        serverFactory = createNewServerInstance(serverFactory, hostPort,
+            maxCnxns);
+        startServerInstance(tmpDir, serverFactory);
+    }
+
+    /** */
+    private ServerCnxnFactory createNewServerInstance(
+        ServerCnxnFactory factory, String hostPort, int maxCnxns)
+        throws IOException {
+        final int port = getPort(hostPort);
+
+        if (factory == null)
+            factory = ServerCnxnFactory.createFactory(port, maxCnxns);
+
+        return factory;
+    }
+
+    /** */
+    private void startServerInstance(File dataDir,
+        ServerCnxnFactory factory) throws IOException,
+        InterruptedException {
+        ZooKeeperServer zks = new ZooKeeperServer(dataDir, dataDir, 3000);
+        factory.startup(zks);
+    }
+
+    /** */
+    private int getPort(String hostPort) {
+        String[] split = hostPort.split(":");
+        String portstr = split[split.length-1];
+        String[] pc = portstr.split("/");
+
+        if (pc.length > 1)
+            portstr = pc[0];
+
+        return Integer.parseInt(portstr);
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslFailedAuthTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslFailedAuthTest.java
new file mode 100644
index 0000000000000..864ac96d72a2c
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslFailedAuthTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class ZookeeperDiscoverySpiSaslFailedAuthTest extends ZookeeperDiscoverySpiSaslAuthAbstractTest {
+    /**
+     * @throws Exception If failed.
+     */
+    public void testIgniteNodeWithInvalidPasswordFailsToJoin() throws Exception {
+        System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
+            "InvalidZookeeperClient");
+
+        System.setProperty("IGNITE_ZOOKEEPER_DISCOVERY_MAX_RETRY_COUNT", Integer.toString(1));
+
+        try {
+            startGrid(0);
+
+            Assert.fail("Ignite node with invalid password should fail on join.");
+        }
+        catch (Exception e) {
+            //ignored
+        }
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslSuccessfulAuthTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslSuccessfulAuthTest.java
new file mode 100644
index 0000000000000..5ee0a4300af7a
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiSaslSuccessfulAuthTest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import org.apache.zookeeper.client.ZooKeeperSaslClient;
+
+/**
+ *
+ */
+public class ZookeeperDiscoverySpiSaslSuccessfulAuthTest extends ZookeeperDiscoverySpiSaslAuthAbstractTest {
+    /**
+     * @throws Exception If failed.
+     */
+    public void testIgniteNodesWithValidPasswordSuccessfullyJoins() throws Exception {
+        System.setProperty(ZooKeeperSaslClient.LOGIN_CONTEXT_NAME_KEY,
+            "ValidZookeeperClient");
+
+        startGrids(3);
+
+        waitForTopology(3);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testIgniteNodeWithoutSaslConfigurationSuccessfullyJoins() throws Exception {
+        //clearing SASL-related system properties that were set in beforeTest
+        clearSaslSystemProperties();
+
+        startGrid(0);
+
+        waitForTopology(1);
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
new file mode 100644
index 0000000000000..fb12c3a5e9801
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -0,0 +1,4847 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.curator.test.TestingCluster;
+import org.apache.curator.test.TestingZooKeeperServer;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.CommunicationFailureContext;
+import org.apache.ignite.configuration.CommunicationFailureResolver;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.DiscoverySpiTestListener;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.managers.discovery.CustomEventListener;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.TestCacheNodeExcludingFilter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
+import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.T3;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteOutClosure;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.security.SecurityCredentials;
+import org.apache.ignite.plugin.security.SecurityPermission;
+import org.apache.ignite.plugin.security.SecuritySubject;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiTestSuite2;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZKUtil;
+import org.apache.zookeeper.ZkTestClientCnxnSocketNIO;
+import org.apache.zookeeper.ZooKeeper;
+import org.jetbrains.annotations.Nullable;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
+import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_RECONNECTED;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
+import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2;
+import static org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD;
+import static org.apache.zookeeper.ZooKeeper.ZOOKEEPER_CLIENT_CNXN_SOCKET;
+
+/**
+ *
+ */
+@SuppressWarnings("deprecation")
+public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
+    /** */
+    private static final String IGNITE_ZK_ROOT = ZookeeperDiscoverySpi.DFLT_ROOT_PATH;
+
+    /** */
+    private static final int ZK_SRVS = 3;
+
+    /** */
+    private static TestingCluster zkCluster;
+
+    /** To run test with real local ZK. */
+    private static final boolean USE_TEST_CLUSTER = true;
+
+    /** */
+    private boolean client;
+
+    /** */
+    private static ThreadLocal<Boolean> clientThreadLoc = new ThreadLocal<>();
+
+    /** */
+    private static ConcurrentHashMap<UUID, Map<Long, DiscoveryEvent>> evts = new ConcurrentHashMap<>();
+
+    /** */
+    private static volatile boolean err;
+
+    /** */
+    private boolean testSockNio;
+
+    /** */
+    private boolean testCommSpi;
+
+    /** */
+    private long sesTimeout;
+
+    /** */
+    private long joinTimeout;
+
+    /** */
+    private boolean clientReconnectDisabled;
+
+    /** */
+    private ConcurrentHashMap<String, ZookeeperDiscoverySpi> spis = new ConcurrentHashMap<>();
+
+    /** */
+    private Map<String, Object> userAttrs;
+
+    /** */
+    private boolean dfltConsistenId;
+
+    /** */
+    private UUID nodeId;
+
+    /** */
+    private boolean persistence;
+
+    /** */
+    private IgniteOutClosure<CommunicationFailureResolver> commFailureRslvr;
+
+    /** */
+    private IgniteOutClosure<DiscoverySpiNodeAuthenticator> auth;
+
+    /** */
+    private String zkRootPath;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String igniteInstanceName) throws Exception {
+        if (testSockNio)
+            System.setProperty(ZOOKEEPER_CLIENT_CNXN_SOCKET, ZkTestClientCnxnSocketNIO.class.getName());
+
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (nodeId != null)
+            cfg.setNodeId(nodeId);
+
+        if (!dfltConsistenId)
+            cfg.setConsistentId(igniteInstanceName);
+
+        ZookeeperDiscoverySpi zkSpi = new ZookeeperDiscoverySpi();
+
+        if (joinTimeout != 0)
+            zkSpi.setJoinTimeout(joinTimeout);
+
+        zkSpi.setSessionTimeout(sesTimeout > 0 ? sesTimeout : 10_000);
+
+        zkSpi.setClientReconnectDisabled(clientReconnectDisabled);
+
+        // Set authenticator for basic sanity tests.
+        if (auth != null) {
+            zkSpi.setAuthenticator(auth.apply());
+
+            zkSpi.setInternalListener(new IgniteDiscoverySpiInternalListener() {
+                @Override public void beforeJoin(ClusterNode locNode, IgniteLogger log) {
+                    ZookeeperClusterNode locNode0 = (ZookeeperClusterNode)locNode;
+
+                    Map<String, Object> attrs = new HashMap<>(locNode0.getAttributes());
+
+                    attrs.put(ATTR_SECURITY_CREDENTIALS, new SecurityCredentials(null, null, igniteInstanceName));
+
+                    locNode0.setAttributes(attrs);
+                }
+
+                @Override public boolean beforeSendCustomEvent(DiscoverySpi spi, IgniteLogger log, DiscoverySpiCustomMessage msg) {
+                    return false;
+                }
+            });
+        }
+
+        spis.put(igniteInstanceName, zkSpi);
+
+        if (USE_TEST_CLUSTER) {
+            assert zkCluster != null;
+
+            zkSpi.setZkConnectionString(zkCluster.getConnectString());
+
+            if (zkRootPath != null)
+                zkSpi.setZkRootPath(zkRootPath);
+        }
+        else
+            zkSpi.setZkConnectionString("localhost:2181");
+
+        cfg.setDiscoverySpi(zkSpi);
+
+        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        Boolean clientMode = clientThreadLoc.get();
+
+        if (clientMode != null)
+            cfg.setClientMode(clientMode);
+        else
+            cfg.setClientMode(client);
+
+        if (userAttrs != null)
+            cfg.setUserAttributes(userAttrs);
+
+        Map<IgnitePredicate<? extends Event>, int[]> lsnrs = new HashMap<>();
+
+        lsnrs.put(new IgnitePredicate<Event>() {
+            /** */
+            @IgniteInstanceResource
+            private Ignite ignite;
+
+            @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+            @Override public boolean apply(Event evt) {
+                try {
+                    DiscoveryEvent discoveryEvt = (DiscoveryEvent)evt;
+
+                    UUID locId = ((IgniteKernal)ignite).context().localNodeId();
+
+                    Map<Long, DiscoveryEvent> nodeEvts = evts.get(locId);
+
+                    if (nodeEvts == null) {
+                        Object old = evts.put(locId, nodeEvts = new TreeMap<>());
+
+                        assertNull(old);
+
+                        synchronized (nodeEvts) {
+                            DiscoveryLocalJoinData locJoin = ((IgniteKernal)ignite).context().discovery().localJoin();
+
+                            nodeEvts.put(locJoin.event().topologyVersion(), locJoin.event());
+                        }
+                    }
+
+                    synchronized (nodeEvts) {
+                        DiscoveryEvent old = nodeEvts.put(discoveryEvt.topologyVersion(), discoveryEvt);
+
+                        assertNull(old);
+                    }
+                }
+                catch (Throwable e) {
+                    error("Unexpected error [evt=" + evt + ", err=" + e + ']', e);
+
+                    err = true;
+                }
+
+                return true;
+            }
+        }, new int[]{EVT_NODE_JOINED, EVT_NODE_FAILED, EVT_NODE_LEFT});
+
+        cfg.setLocalEventListeners(lsnrs);
+
+        if (persistence) {
+            DataStorageConfiguration memCfg = new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setMaxSize(100 * 1024 * 1024).
+                    setPersistenceEnabled(true))
+                .setPageSize(1024)
+                .setWalMode(WALMode.LOG_ONLY);
+
+            cfg.setDataStorageConfiguration(memCfg);
+        }
+
+        if (testCommSpi)
+            cfg.setCommunicationSpi(new ZkTestCommunicationSpi());
+
+        if (commFailureRslvr != null)
+            cfg.setCommunicationFailureResolver(commFailureRslvr.apply());
+
+        return cfg;
+    }
+
+    /**
+     * @param clientMode Client mode flag for started nodes.
+     */
+    private void clientMode(boolean clientMode) {
+        client = clientMode;
+    }
+
+    /**
+     * @param clientMode Client mode flag for nodes started from current thread.
+     */
+    private void clientModeThreadLocal(boolean clientMode) {
+        clientThreadLoc.set(clientMode);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT, "1000");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopZkCluster();
+
+        System.clearProperty(ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_TIMEOUT);
+
+        super.afterTestsStopped();
+    }
+
+    /**
+     *
+     */
+    private void stopZkCluster() {
+        if (zkCluster != null) {
+            try {
+                zkCluster.close();
+            }
+            catch (Exception e) {
+                U.error(log, "Failed to stop Zookeeper client: " + e, e);
+            }
+
+            zkCluster = null;
+        }
+    }
+
+    /**
+     *
+     */
+    private static void ackEveryEventSystemProperty() {
+        System.setProperty(IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD, "1");
+    }
+
+    /**
+     *
+     */
+    private void clearAckEveryEventSystemProperty() {
+        System.setProperty(IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD, "1");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        if (USE_TEST_CLUSTER && zkCluster == null) {
+            zkCluster = ZookeeperDiscoverySpiTestSuite2.createTestingCluster(ZK_SRVS);
+
+            zkCluster.start();
+        }
+
+        reset();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        clearAckEveryEventSystemProperty();
+
+        try {
+            assertFalse("Unexpected error, see log for details", err);
+
+            checkEventsConsistency();
+
+            checkInternalStructuresCleanup();
+
+            //TODO uncomment when https://issues.apache.org/jira/browse/IGNITE-8193 is fixed
+//            checkZkNodesCleanup();
+        }
+        finally {
+            reset();
+
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkInternalStructuresCleanup() throws Exception {
+        for (Ignite node : G.allGrids()) {
+            final AtomicReference<?> res = GridTestUtils.getFieldValue(spi(node), "impl", "commErrProcFut");
+
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return res.get() == null;
+                }
+            }, 30_000);
+
+            assertNull(res.get());
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testZkRootNotExists() throws Exception {
+        zkRootPath = "/a/b/c";
+
+        for (int i = 0; i < 3; i++) {
+            reset();
+
+            startGridsMultiThreaded(5);
+
+            waitForTopology(5);
+
+            stopAllGrids();
+
+            checkEventsConsistency();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMetadataUpdate() throws Exception {
+        startGrid(0);
+
+        GridTestUtils.runMultiThreaded(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                ignite(0).configuration().getMarshaller().marshal(new C1());
+                ignite(0).configuration().getMarshaller().marshal(new C2());
+
+                return null;
+            }
+        }, 64, "marshal");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNodeAddresses() throws Exception {
+        startGridsMultiThreaded(3);
+
+        clientMode(true);
+
+        startGridsMultiThreaded(3, 3);
+
+        waitForTopology(6);
+
+        for (Ignite node : G.allGrids()) {
+            ClusterNode locNode0 = node.cluster().localNode();
+
+            assertTrue(locNode0.addresses().size() > 0);
+            assertTrue(locNode0.hostNames().size() > 0);
+
+            for (ClusterNode node0 : node.cluster().nodes()) {
+                assertTrue(node0.addresses().size() > 0);
+                assertTrue(node0.hostNames().size() > 0);
+            }
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSetConsistentId() throws Exception {
+        startGridsMultiThreaded(3);
+
+        clientMode(true);
+
+        startGridsMultiThreaded(3, 3);
+
+        waitForTopology(6);
+
+        for (Ignite node : G.allGrids()) {
+            ClusterNode locNode0 = node.cluster().localNode();
+
+            assertEquals(locNode0.attribute(ATTR_IGNITE_INSTANCE_NAME),
+                locNode0.consistentId());
+
+            for (ClusterNode node0 : node.cluster().nodes()) {
+                assertEquals(node0.attribute(ATTR_IGNITE_INSTANCE_NAME),
+                    node0.consistentId());
+            }
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDefaultConsistentId() throws Exception {
+        dfltConsistenId = true;
+
+        startGridsMultiThreaded(3);
+
+        clientMode(true);
+
+        startGridsMultiThreaded(3, 3);
+
+        waitForTopology(6);
+
+        for (Ignite node : G.allGrids()) {
+            ClusterNode locNode0 = node.cluster().localNode();
+
+            assertNotNull(locNode0.consistentId());
+
+            for (ClusterNode node0 : node.cluster().nodes())
+                assertNotNull(node0.consistentId());
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientNodesStatus() throws Exception {
+        startGrid(0);
+
+        for (Ignite node : G.allGrids()) {
+            assertEquals(0, node.cluster().forClients().nodes().size());
+            assertEquals(1, node.cluster().forServers().nodes().size());
+        }
+
+        clientMode(true);
+
+        startGrid(1);
+
+        for (Ignite node : G.allGrids()) {
+            assertEquals(1, node.cluster().forClients().nodes().size());
+            assertEquals(1, node.cluster().forServers().nodes().size());
+        }
+
+        clientMode(false);
+
+        startGrid(2);
+
+        clientMode(true);
+
+        startGrid(3);
+
+        for (Ignite node : G.allGrids()) {
+            assertEquals(2, node.cluster().forClients().nodes().size());
+            assertEquals(2, node.cluster().forServers().nodes().size());
+        }
+
+        stopGrid(1);
+
+        waitForTopology(3);
+
+        for (Ignite node : G.allGrids()) {
+            assertEquals(1, node.cluster().forClients().nodes().size());
+            assertEquals(2, node.cluster().forServers().nodes().size());
+        }
+
+        stopGrid(2);
+
+        waitForTopology(2);
+
+        for (Ignite node : G.allGrids()) {
+            assertEquals(1, node.cluster().forClients().nodes().size());
+            assertEquals(1, node.cluster().forServers().nodes().size());
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void _testLocalAuthenticationFails() throws Exception {
+        auth = ZkTestNodeAuthenticator.factory(getTestIgniteInstanceName(0));
+
+        Throwable err = GridTestUtils.assertThrows(log, new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                startGrid(0);
+
+                return null;
+            }
+        }, IgniteCheckedException.class, null);
+
+        IgniteSpiException spiErr = X.cause(err, IgniteSpiException.class);
+
+        assertNotNull(spiErr);
+        assertTrue(spiErr.getMessage().contains("Authentication failed for local node"));
+
+        startGrid(1);
+        startGrid(2);
+
+        checkTestSecuritySubject(2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAuthentication() throws Exception {
+        auth = ZkTestNodeAuthenticator.factory(getTestIgniteInstanceName(1),
+            getTestIgniteInstanceName(5));
+
+        startGrid(0);
+
+        checkTestSecuritySubject(1);
+
+        {
+            clientMode(false);
+            checkStartFail(1);
+
+            clientMode(true);
+            checkStartFail(1);
+
+            clientMode(false);
+        }
+
+        startGrid(2);
+
+        checkTestSecuritySubject(2);
+
+        stopGrid(2);
+
+        checkTestSecuritySubject(1);
+
+        startGrid(2);
+
+        checkTestSecuritySubject(2);
+
+        stopGrid(0);
+
+        checkTestSecuritySubject(1);
+
+        checkStartFail(1);
+
+        clientMode(false);
+
+        startGrid(3);
+
+        clientMode(true);
+
+        startGrid(4);
+
+        clientMode(false);
+
+        startGrid(0);
+
+        checkTestSecuritySubject(4);
+
+        checkStartFail(1);
+        checkStartFail(5);
+
+        clientMode(true);
+
+        checkStartFail(1);
+        checkStartFail(5);
+    }
+
+    /**
+     * @param nodeIdx Node index.
+     */
+    private void checkStartFail(final int nodeIdx) {
+        Throwable err = GridTestUtils.assertThrows(log, new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                startGrid(nodeIdx);
+
+                return null;
+            }
+        }, IgniteCheckedException.class, null);
+
+        IgniteSpiException spiErr = X.cause(err, IgniteSpiException.class);
+
+        assertNotNull(spiErr);
+        assertTrue(spiErr.getMessage().contains("Authentication failed"));
+    }
+
+    /**
+     * @param expNodes Expected nodes number.
+     * @throws Exception If failed.
+     */
+    private void checkTestSecuritySubject(int expNodes) throws Exception {
+        waitForTopology(expNodes);
+
+        List<Ignite> nodes = G.allGrids();
+
+        JdkMarshaller marsh = new JdkMarshaller();
+
+        for (Ignite ignite : nodes) {
+            Collection<ClusterNode> nodes0 = ignite.cluster().nodes();
+
+            assertEquals(nodes.size(), nodes0.size());
+
+            for (ClusterNode node : nodes0) {
+                byte[] secSubj = node.attribute(ATTR_SECURITY_SUBJECT_V2);
+
+                assertNotNull(secSubj);
+
+                ZkTestNodeAuthenticator.TestSecurityContext secCtx = marsh.unmarshal(secSubj, null);
+
+                assertEquals(node.attribute(ATTR_IGNITE_INSTANCE_NAME), secCtx.nodeName);
+            }
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStopNode_1() throws Exception {
+        startGrids(5);
+
+        waitForTopology(5);
+
+        stopGrid(3);
+
+        waitForTopology(4);
+
+        startGrid(3);
+
+        waitForTopology(5);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCustomEventsSimple1_SingleNode() throws Exception {
+        ackEveryEventSystemProperty();
+
+        Ignite srv0 = startGrid(0);
+
+        srv0.createCache(new CacheConfiguration<>("c1"));
+
+        waitForEventsAcks(srv0);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCustomEventsSimple1_5_Nodes() throws Exception {
+        ackEveryEventSystemProperty();
+
+        Ignite srv0 = startGrids(5);
+
+        srv0.createCache(new CacheConfiguration<>("c1"));
+
+        awaitPartitionMapExchange();
+
+        waitForEventsAcks(srv0);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCustomEvents_FastStopProcess_1() throws Exception {
+        customEvents_FastStopProcess(1, 0);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCustomEvents_FastStopProcess_2() throws Exception {
+        customEvents_FastStopProcess(5, 5);
+    }
+
+    /**
+     * @param srvs Servers number.
+     * @param clients Clients number.
+     * @throws Exception If failed.
+     */
+    private void customEvents_FastStopProcess(int srvs, int clients) throws Exception {
+        ackEveryEventSystemProperty();
+
+        Map<UUID, List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>>> rcvdMsgs =
+            new ConcurrentHashMap<>();
+
+        Ignite crd = startGrid(0);
+
+        UUID crdId = crd.cluster().localNode().id();
+
+        if (srvs > 1)
+            startGridsMultiThreaded(1, srvs - 1);
+
+        if (clients > 0) {
+            client = true;
+
+            startGridsMultiThreaded(srvs, clients);
+        }
+
+        awaitPartitionMapExchange();
+
+        List<Ignite> nodes = G.allGrids();
+
+        assertEquals(srvs + clients, nodes.size());
+
+        for (Ignite node : nodes)
+            registerTestEventListeners(node, rcvdMsgs);
+
+        int payload = 0;
+
+        AffinityTopologyVersion topVer = ((IgniteKernal)crd).context().discovery().topologyVersionEx();
+
+        for (Ignite node : nodes) {
+            UUID sndId = node.cluster().localNode().id();
+
+            info("Send from node: " + sndId);
+
+            GridDiscoveryManager discoveryMgr = ((IgniteKernal)node).context().discovery();
+
+            {
+                List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> expCrdMsgs = new ArrayList<>();
+                List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> expNodesMsgs = Collections.emptyList();
+
+                TestFastStopProcessCustomMessage msg = new TestFastStopProcessCustomMessage(false, payload++);
+
+                expCrdMsgs.add(new T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>(topVer, sndId, msg));
+
+                discoveryMgr.sendCustomEvent(msg);
+
+                doSleep(200); // Wait some time to check extra messages are not received.
+
+                checkEvents(crd, rcvdMsgs, expCrdMsgs);
+
+                for (Ignite node0 : nodes) {
+                    if (node0 != crd)
+                        checkEvents(node0, rcvdMsgs, expNodesMsgs);
+                }
+
+                rcvdMsgs.clear();
+            }
+            {
+                List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> expCrdMsgs = new ArrayList<>();
+                List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> expNodesMsgs = new ArrayList<>();
+
+                TestFastStopProcessCustomMessage msg = new TestFastStopProcessCustomMessage(true, payload++);
+
+                expCrdMsgs.add(new T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>(topVer, sndId, msg));
+
+                discoveryMgr.sendCustomEvent(msg);
+
+                TestFastStopProcessCustomMessageAck ackMsg = new TestFastStopProcessCustomMessageAck(msg.payload);
+
+                expCrdMsgs.add(new T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>(topVer, crdId, ackMsg));
+                expNodesMsgs.add(new T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>(topVer, crdId, ackMsg));
+
+                doSleep(200); // Wait some time to check extra messages are not received.
+
+                checkEvents(crd, rcvdMsgs, expCrdMsgs);
+
+                for (Ignite node0 : nodes) {
+                    if (node0 != crd)
+                        checkEvents(node0, rcvdMsgs, expNodesMsgs);
+                }
+
+                rcvdMsgs.clear();
+            }
+
+            waitForEventsAcks(crd);
+        }
+    }
+
+    /**
+     * @param node Node to check.
+     * @param rcvdMsgs Received messages.
+     * @param expMsgs Expected messages.
+     * @throws Exception If failed.
+     */
+    private void checkEvents(
+        Ignite node,
+        final Map<UUID, List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>>> rcvdMsgs,
+        final List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> expMsgs) throws Exception {
+        final UUID nodeId = node.cluster().localNode().id();
+
+        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> msgs = rcvdMsgs.get(nodeId);
+
+                int size = msgs == null ? 0 : msgs.size();
+
+                return size >= expMsgs.size();
+            }
+        }, 5000));
+
+        List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> msgs = rcvdMsgs.get(nodeId);
+
+        if (msgs == null)
+            msgs = Collections.emptyList();
+
+        assertEqualsCollections(expMsgs, msgs);
+    }
+
+    /**
+     * @param node Node.
+     * @param rcvdMsgs Map to store received events.
+     */
+    private void registerTestEventListeners(Ignite node,
+        final Map<UUID, List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>>> rcvdMsgs) {
+        GridDiscoveryManager discoveryMgr = ((IgniteKernal)node).context().discovery();
+
+        final UUID nodeId = node.cluster().localNode().id();
+
+        discoveryMgr.setCustomEventListener(TestFastStopProcessCustomMessage.class,
+            new CustomEventListener<TestFastStopProcessCustomMessage>() {
+                @Override public void onCustomEvent(AffinityTopologyVersion topVer, ClusterNode snd, TestFastStopProcessCustomMessage msg) {
+                    List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> list = rcvdMsgs.get(nodeId);
+
+                    if (list == null)
+                        rcvdMsgs.put(nodeId, list = new ArrayList<>());
+
+                    list.add(new T3<>(topVer, snd.id(), (DiscoveryCustomMessage)msg));
+                }
+            }
+        );
+        discoveryMgr.setCustomEventListener(TestFastStopProcessCustomMessageAck.class,
+            new CustomEventListener<TestFastStopProcessCustomMessageAck>() {
+                @Override public void onCustomEvent(AffinityTopologyVersion topVer, ClusterNode snd, TestFastStopProcessCustomMessageAck msg) {
+                    List<T3<AffinityTopologyVersion, UUID, DiscoveryCustomMessage>> list = rcvdMsgs.get(nodeId);
+
+                    if (list == null)
+                        rcvdMsgs.put(nodeId, list = new ArrayList<>());
+
+                    list.add(new T3<>(topVer, snd.id(), (DiscoveryCustomMessage)msg));
+                }
+            }
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSegmentation1() throws Exception {
+        sesTimeout = 2000;
+        testSockNio = true;
+
+        Ignite node0 = startGrid(0);
+
+        final CountDownLatch l = new CountDownLatch(1);
+
+        node0.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                l.countDown();
+
+                return false;
+            }
+        }, EventType.EVT_NODE_SEGMENTED);
+
+        ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+        c0.closeSocket(true);
+
+        for (int i = 0; i < 10; i++) {
+            Thread.sleep(1_000);
+
+            if (l.getCount() == 0)
+                break;
+        }
+
+        info("Allow connect");
+
+        c0.allowConnect();
+
+        assertTrue(l.await(10, TimeUnit.SECONDS));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSegmentation2() throws Exception {
+        sesTimeout = 2000;
+
+        Ignite node0 = startGrid(0);
+
+        final CountDownLatch l = new CountDownLatch(1);
+
+        node0.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                l.countDown();
+
+                return false;
+            }
+        }, EventType.EVT_NODE_SEGMENTED);
+
+        try {
+            zkCluster.close();
+
+            assertTrue(l.await(10, TimeUnit.SECONDS));
+        }
+        finally {
+            zkCluster = ZookeeperDiscoverySpiTestSuite2.createTestingCluster(ZK_SRVS);
+
+            zkCluster.start();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSegmentation3() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8183");
+
+        sesTimeout = 5000;
+
+        Ignite node0 = startGrid(0);
+
+        final CountDownLatch l = new CountDownLatch(1);
+
+        node0.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                l.countDown();
+
+                return false;
+            }
+        }, EventType.EVT_NODE_SEGMENTED);
+
+        List<TestingZooKeeperServer> srvs = zkCluster.getServers();
+
+        assertEquals(3, srvs.size());
+
+        try {
+            srvs.get(0).stop();
+            srvs.get(1).stop();
+
+            assertTrue(l.await(20, TimeUnit.SECONDS));
+        }
+        finally {
+            zkCluster.close();
+
+            zkCluster = ZookeeperDiscoverySpiTestSuite2.createTestingCluster(ZK_SRVS);
+
+            zkCluster.start();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testQuorumRestore() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8180");
+
+        sesTimeout = 15_000;
+
+        startGrids(3);
+
+        waitForTopology(3);
+
+        List<TestingZooKeeperServer> srvs = zkCluster.getServers();
+
+        assertEquals(3, srvs.size());
+
+        try {
+            srvs.get(0).stop();
+            srvs.get(1).stop();
+
+            U.sleep(2000);
+
+            srvs.get(1).restart();
+
+            U.sleep(4000);
+
+            startGrid(4);
+
+            waitForTopology(4);
+        }
+        finally {
+            zkCluster.close();
+
+            zkCluster = ZookeeperDiscoverySpiTestSuite2.createTestingCluster(ZK_SRVS);
+
+            zkCluster.start();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionRestore1() throws Exception {
+        testSockNio = true;
+
+        Ignite node0 = startGrid(0);
+
+        ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+        c0.closeSocket(false);
+
+        startGrid(1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionRestore2() throws Exception {
+        testSockNio = true;
+
+        Ignite node0 = startGrid(0);
+
+        ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+        c0.closeSocket(false);
+
+        startGridsMultiThreaded(1, 5);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionRestore_NonCoordinator1() throws Exception {
+        connectionRestore_NonCoordinator(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionRestore_NonCoordinator2() throws Exception {
+        connectionRestore_NonCoordinator(true);
+    }
+
+    /**
+     * @param failWhenDisconnected {@code True} if fail node while another node is disconnected.
+     * @throws Exception If failed.
+     */
+    private void connectionRestore_NonCoordinator(boolean failWhenDisconnected) throws Exception {
+        testSockNio = true;
+
+        Ignite node0 = startGrid(0);
+        Ignite node1 = startGrid(1);
+
+        ZkTestClientCnxnSocketNIO c1 = ZkTestClientCnxnSocketNIO.forNode(node1);
+
+        c1.closeSocket(true);
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() {
+                try {
+                    startGrid(2);
+                }
+                catch (Exception e) {
+                    info("Start error: " + e);
+                }
+
+                return null;
+            }
+        }, "start-node");
+
+        checkEvents(node0, joinEvent(3));
+
+        if (failWhenDisconnected) {
+            ZookeeperDiscoverySpi spi = spis.get(getTestIgniteInstanceName(2));
+
+            closeZkClient(spi);
+
+            checkEvents(node0, failEvent(4));
+        }
+
+        c1.allowConnect();
+
+        checkEvents(ignite(1), joinEvent(3));
+
+        if (failWhenDisconnected) {
+            checkEvents(ignite(1), failEvent(4));
+
+            IgnitionEx.stop(getTestIgniteInstanceName(2), true, true);
+        }
+
+        fut.get();
+
+        waitForTopology(failWhenDisconnected ? 2 : 3);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionRestore_Coordinator1() throws Exception {
+        connectionRestore_Coordinator(1, 1, 0);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionRestore_Coordinator1_1() throws Exception {
+        connectionRestore_Coordinator(1, 1, 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionRestore_Coordinator2() throws Exception {
+        connectionRestore_Coordinator(1, 3, 0);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionRestore_Coordinator3() throws Exception {
+        connectionRestore_Coordinator(3, 3, 0);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionRestore_Coordinator4() throws Exception {
+        connectionRestore_Coordinator(3, 3, 1);
+    }
+
+    /**
+     * @param initNodes Number of initially started nodes.
+     * @param startNodes Number of nodes to start after coordinator loose connection.
+     * @param failCnt Number of nodes to stop after coordinator loose connection.
+     * @throws Exception If failed.
+     */
+    private void connectionRestore_Coordinator(final int initNodes, int startNodes, int failCnt) throws Exception {
+        sesTimeout = 30_000;
+        testSockNio = true;
+
+        Ignite node0 = startGrids(initNodes);
+
+        ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+        c0.closeSocket(true);
+
+        final AtomicInteger nodeIdx = new AtomicInteger(initNodes);
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+            @Override public Void call() {
+                try {
+                    startGrid(nodeIdx.getAndIncrement());
+                }
+                catch (Exception e) {
+                    error("Start failed: " + e);
+                }
+
+                return null;
+            }
+        }, startNodes, "start-node");
+
+        int cnt = 0;
+
+        DiscoveryEvent[] expEvts = new DiscoveryEvent[startNodes - failCnt];
+
+        int expEvtCnt = 0;
+
+        sesTimeout = 1000;
+
+        List<ZkTestClientCnxnSocketNIO> blockedC = new ArrayList<>();
+
+        final List<String> failedZkNodes = new ArrayList<>(failCnt);
+
+        for (int i = initNodes; i < initNodes + startNodes; i++) {
+            final ZookeeperDiscoverySpi spi = waitSpi(getTestIgniteInstanceName(i));
+
+            assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    Object spiImpl = GridTestUtils.getFieldValue(spi, "impl");
+
+                    if (spiImpl == null)
+                        return false;
+
+                    long internalOrder = GridTestUtils.getFieldValue(spiImpl, "rtState", "internalOrder");
+
+                    return internalOrder > 0;
+                }
+            }, 10_000));
+
+            if (cnt++ < failCnt) {
+                ZkTestClientCnxnSocketNIO c = ZkTestClientCnxnSocketNIO.forNode(getTestIgniteInstanceName(i));
+
+                c.closeSocket(true);
+
+                blockedC.add(c);
+
+                failedZkNodes.add(aliveZkNodePath(spi));
+            }
+            else {
+                expEvts[expEvtCnt] = joinEvent(initNodes + expEvtCnt + 1);
+
+                expEvtCnt++;
+            }
+        }
+
+        waitNoAliveZkNodes(log, zkCluster.getConnectString(), failedZkNodes, 30_000);
+
+        c0.allowConnect();
+
+        for (ZkTestClientCnxnSocketNIO c : blockedC)
+            c.allowConnect();
+
+        if (expEvts.length > 0) {
+            for (int i = 0; i < initNodes; i++)
+                checkEvents(ignite(i), expEvts);
+        }
+
+        fut.get();
+
+        waitForTopology(initNodes + startNodes - failCnt);
+    }
+
+    /**
+     * @param node Node.
+     * @return Corresponding znode.
+     */
+    private static String aliveZkNodePath(Ignite node) {
+        return aliveZkNodePath(node.configuration().getDiscoverySpi());
+    }
+
+    /**
+     * @param spi SPI.
+     * @return Znode related to given SPI.
+     */
+    private static String aliveZkNodePath(DiscoverySpi spi) {
+        String path = GridTestUtils.getFieldValue(spi, "impl", "rtState", "locNodeZkPath");
+
+        return path.substring(path.lastIndexOf('/') + 1);
+    }
+
+    /**
+     * @param log Logger.
+     * @param connectString Zookeeper connect string.
+     * @param failedZkNodes Znodes which should be removed.
+     * @param timeout Timeout.
+     * @throws Exception If failed.
+     */
+    private static void waitNoAliveZkNodes(final IgniteLogger log,
+        String connectString,
+        final List<String> failedZkNodes,
+        long timeout)
+        throws Exception
+    {
+        final ZookeeperClient zkClient = new ZookeeperClient(log, connectString, 10_000, null);
+
+        try {
+            assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    try {
+                        List<String> c = zkClient.getChildren(IGNITE_ZK_ROOT + "/" + ZkIgnitePaths.ALIVE_NODES_DIR);
+
+                        for (String failedZkNode : failedZkNodes) {
+                            if (c.contains(failedZkNode)) {
+                                log.info("Alive node is not removed [node=" + failedZkNode + ", all=" + c + ']');
+
+                                return false;
+                            }
+                        }
+
+                        return true;
+                    }
+                    catch (Exception e) {
+                        e.printStackTrace();
+
+                        fail();
+
+                        return true;
+                    }
+                }
+            }, timeout));
+        }
+        finally {
+            zkClient.close();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentStartWithClient() throws Exception {
+        final int NODES = 20;
+
+        for (int i = 0; i < 3; i++) {
+            info("Iteration: " + i);
+
+            final int srvIdx = ThreadLocalRandom.current().nextInt(NODES);
+
+            final AtomicInteger idx = new AtomicInteger();
+
+            GridTestUtils.runMultiThreaded(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    int threadIdx = idx.getAndIncrement();
+
+                    clientModeThreadLocal(threadIdx == srvIdx || ThreadLocalRandom.current().nextBoolean());
+
+                    startGrid(threadIdx);
+
+                    return null;
+                }
+            }, NODES, "start-node");
+
+            waitForTopology(NODES);
+
+            stopAllGrids();
+
+            checkEventsConsistency();
+
+            evts.clear();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentStart() throws Exception {
+        final int NODES = 20;
+
+        for (int i = 0; i < 3; i++) {
+            info("Iteration: " + i);
+
+            final AtomicInteger idx = new AtomicInteger();
+
+            final CyclicBarrier b = new CyclicBarrier(NODES);
+
+            GridTestUtils.runMultiThreaded(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    b.await();
+
+                    int threadIdx = idx.getAndIncrement();
+
+                    startGrid(threadIdx);
+
+                    return null;
+                }
+            }, NODES, "start-node");
+
+            waitForTopology(NODES);
+
+            stopAllGrids();
+
+            checkEventsConsistency();
+
+            evts.clear();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentStartStop1() throws Exception {
+       concurrentStartStop(1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentStartStop2() throws Exception {
+        concurrentStartStop(5);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConcurrentStartStop2_EventsThrottle() throws Exception {
+        System.setProperty(ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS, "1");
+
+        try {
+            concurrentStartStop(5);
+        }
+        finally {
+            System.clearProperty(ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_MAX_EVTS);
+        }
+    }
+
+    /**
+     * @param initNodes Number of initially started nnodes.
+     * @throws Exception If failed.
+     */
+    private void concurrentStartStop(final int initNodes) throws Exception {
+        startGrids(initNodes);
+
+        final int NODES = 5;
+
+        long topVer = initNodes;
+
+        for (int i = 0; i < 10; i++) {
+            info("Iteration: " + i);
+
+            DiscoveryEvent[] expEvts = new DiscoveryEvent[NODES];
+
+            startGridsMultiThreaded(initNodes, NODES);
+
+            for (int j = 0; j < NODES; j++)
+                expEvts[j] = joinEvent(++topVer);
+
+            checkEvents(ignite(0), expEvts);
+
+            checkEventsConsistency();
+
+            final CyclicBarrier b = new CyclicBarrier(NODES);
+
+            GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+                @Override public void apply(Integer idx) {
+                    try {
+                        b.await();
+
+                        stopGrid(initNodes + idx);
+                    }
+                    catch (Exception e) {
+                        e.printStackTrace();
+
+                        fail();
+                    }
+                }
+            }, NODES, "stop-node");
+
+            for (int j = 0; j < NODES; j++)
+                expEvts[j] = failEvent(++topVer);
+
+            checkEventsConsistency();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClusterRestart() throws Exception {
+        startGridsMultiThreaded(3, false);
+
+        stopAllGrids();
+
+        evts.clear();
+
+        startGridsMultiThreaded(3, false);
+
+        waitForTopology(3);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionRestore4() throws Exception {
+        testSockNio = true;
+
+        Ignite node0 = startGrid(0);
+
+        ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+        c0.closeSocket(false);
+
+        startGrid(1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartStop_1_Node() throws Exception {
+        startGrid(0);
+
+        waitForTopology(1);
+
+        stopGrid(0);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRestarts_2_Nodes() throws Exception {
+        startGrid(0);
+
+        for (int i = 0; i < 10; i++) {
+            info("Iteration: " + i);
+
+            startGrid(1);
+
+            waitForTopology(2);
+
+            stopGrid(1);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartStop_2_Nodes_WithCache() throws Exception {
+        startGrids(2);
+
+        for (Ignite node : G.allGrids()) {
+            IgniteCache<Object, Object> cache = node.cache(DEFAULT_CACHE_NAME);
+
+            assertNotNull(cache);
+
+            for (int i = 0; i < 100; i++) {
+                cache.put(i, node.name());
+
+                assertEquals(node.name(), cache.get(i));
+            }
+        }
+
+        awaitPartitionMapExchange();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartStop_2_Nodes() throws Exception {
+        ackEveryEventSystemProperty();
+
+        startGrid(0);
+
+        waitForTopology(1);
+
+        startGrid(1);
+
+        waitForTopology(2);
+
+        for (Ignite node : G.allGrids())
+            node.compute().broadcast(new DummyCallable(null));
+
+        awaitPartitionMapExchange();
+
+        waitForEventsAcks(ignite(0));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testMultipleClusters() throws Exception {
+        Ignite c0 = startGrid(0);
+
+        zkRootPath = "/cluster2";
+
+        Ignite c1 = startGridsMultiThreaded(1, 5);
+
+        zkRootPath = "/cluster3";
+
+        Ignite c2 = startGridsMultiThreaded(6, 3);
+
+        checkNodesNumber(c0, 1);
+        checkNodesNumber(c1, 5);
+        checkNodesNumber(c2, 3);
+
+        stopGrid(2);
+
+        checkNodesNumber(c0, 1);
+        checkNodesNumber(c1, 4);
+        checkNodesNumber(c2, 3);
+
+        for (int i = 0; i < 3; i++)
+            stopGrid(i + 6);
+
+        checkNodesNumber(c0, 1);
+        checkNodesNumber(c1, 4);
+
+        c2 = startGridsMultiThreaded(6, 2);
+
+        checkNodesNumber(c0, 1);
+        checkNodesNumber(c1, 4);
+        checkNodesNumber(c2, 2);
+
+        evts.clear();
+    }
+
+    /**
+     * @param node Node.
+     * @param expNodes Expected node in cluster.
+     * @throws Exception If failed.
+     */
+    private void checkNodesNumber(final Ignite node, final int expNodes) throws Exception {
+        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return node.cluster().nodes().size() == expNodes;
+            }
+        }, 5000);
+
+        assertEquals(expNodes, node.cluster().nodes().size());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartStop1() throws Exception {
+        ackEveryEventSystemProperty();
+
+        startGridsMultiThreaded(5, false);
+
+        waitForTopology(5);
+
+        awaitPartitionMapExchange();
+
+        waitForEventsAcks(ignite(0));
+
+        stopGrid(0);
+
+        waitForTopology(4);
+
+        for (Ignite node : G.allGrids())
+            node.compute().broadcast(new DummyCallable(null));
+
+        startGrid(0);
+
+        waitForTopology(5);
+
+        awaitPartitionMapExchange();
+
+        waitForEventsAcks(grid(CU.oldest(ignite(1).cluster().nodes())));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartStop3() throws Exception {
+        startGrids(4);
+
+        awaitPartitionMapExchange();
+
+        stopGrid(0);
+
+        startGrid(5);
+
+        awaitPartitionMapExchange();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartStop4() throws Exception {
+        startGrids(6);
+
+        awaitPartitionMapExchange();
+
+        stopGrid(2);
+
+        if (ThreadLocalRandom.current().nextBoolean())
+            awaitPartitionMapExchange();
+
+        stopGrid(1);
+
+        if (ThreadLocalRandom.current().nextBoolean())
+            awaitPartitionMapExchange();
+
+        stopGrid(0);
+
+        if (ThreadLocalRandom.current().nextBoolean())
+            awaitPartitionMapExchange();
+
+        startGrid(7);
+
+        awaitPartitionMapExchange();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartStop2() throws Exception {
+        startGridsMultiThreaded(10, false);
+
+        GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+            @Override public void apply(Integer idx) {
+                stopGrid(idx);
+            }
+        }, 3, "stop-node-thread");
+
+        waitForTopology(7);
+
+        startGridsMultiThreaded(0, 3);
+
+        waitForTopology(10);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartStopWithClients() throws Exception {
+        final int SRVS = 3;
+
+        startGrids(SRVS);
+
+        clientMode(true);
+
+        final int THREADS = 30;
+
+        for (int i = 0; i < 5; i++) {
+            info("Iteration: " + i);
+
+            startGridsMultiThreaded(SRVS, THREADS);
+
+            waitForTopology(SRVS + THREADS);
+
+            GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+                @Override public void apply(Integer idx) {
+                    stopGrid(idx + SRVS);
+                }
+            }, THREADS, "stop-node");
+
+            waitForTopology(SRVS);
+
+            checkEventsConsistency();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTopologyChangeMultithreaded() throws Exception {
+        topologyChangeWithRestarts(false, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTopologyChangeMultithreaded_RestartZk() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8184");
+
+        try {
+            topologyChangeWithRestarts(true, false);
+        }
+        finally {
+            zkCluster.stop();
+
+            zkCluster = null;
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTopologyChangeMultithreaded_RestartZk_CloseClients() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8184");
+
+        try {
+            topologyChangeWithRestarts(true, true);
+        }
+        finally {
+            zkCluster.stop();
+
+            zkCluster = null;
+        }
+    }
+
+    /**
+     * @param restartZk If {@code true} in background restarts on of ZK servers.
+     * @param closeClientSock If {@code true} in background closes zk clients' sockets.
+     * @throws Exception If failed.
+     */
+    private void topologyChangeWithRestarts(boolean restartZk, boolean closeClientSock) throws Exception {
+        sesTimeout = 30_000;
+
+        if (closeClientSock)
+            testSockNio = true;
+
+        long stopTime = System.currentTimeMillis() + 60_000;
+
+        AtomicBoolean stop = new AtomicBoolean();
+
+        IgniteInternalFuture<?> fut1 = null;
+
+        IgniteInternalFuture<?> fut2 = null;
+
+        try {
+            fut1 = restartZk ? startRestartZkServers(stopTime, stop) : null;
+            fut2 = closeClientSock ? startCloseZkClientSocket(stopTime, stop) : null;
+
+            int INIT_NODES = 10;
+
+            startGridsMultiThreaded(INIT_NODES);
+
+            final int MAX_NODES = 20;
+
+            final List<Integer> startedNodes = new ArrayList<>();
+
+            for (int i = 0; i < INIT_NODES; i++)
+                startedNodes.add(i);
+
+            ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+            final AtomicInteger startIdx = new AtomicInteger(INIT_NODES);
+
+            while (System.currentTimeMillis() < stopTime) {
+                if (startedNodes.size() >= MAX_NODES) {
+                    int stopNodes = rnd.nextInt(5) + 1;
+
+                    log.info("Next, stop nodes: " + stopNodes);
+
+                    final List<Integer> idxs = new ArrayList<>();
+
+                    while (idxs.size() < stopNodes) {
+                        Integer stopIdx = rnd.nextInt(startedNodes.size());
+
+                        if (!idxs.contains(stopIdx))
+                            idxs.add(startedNodes.get(stopIdx));
+                    }
+
+                    GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+                        @Override public void apply(Integer threadIdx) {
+                            int stopNodeIdx = idxs.get(threadIdx);
+
+                            info("Stop node: " + stopNodeIdx);
+
+                            stopGrid(stopNodeIdx);
+                        }
+                    }, stopNodes, "stop-node");
+
+                    startedNodes.removeAll(idxs);
+                }
+                else {
+                    int startNodes = rnd.nextInt(5) + 1;
+
+                    log.info("Next, start nodes: " + startNodes);
+
+                    GridTestUtils.runMultiThreaded(new Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            int idx = startIdx.incrementAndGet();
+
+                            log.info("Start node: " + idx);
+
+                            startGrid(idx);
+
+                            synchronized (startedNodes) {
+                                startedNodes.add(idx);
+                            }
+
+                            return null;
+                        }
+                    }, startNodes, "start-node");
+                }
+
+                U.sleep(rnd.nextInt(100) + 1);
+            }
+        }
+        finally {
+            stop.set(true);
+        }
+
+        if (fut1 != null)
+            fut1.get();
+
+        if (fut2 != null)
+            fut2.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRandomTopologyChanges() throws Exception {
+        randomTopologyChanges(false, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void checkZkNodesCleanup() throws Exception {
+        final ZookeeperClient zkClient = new ZookeeperClient(getTestResources().getLogger(),
+            zkCluster.getConnectString(),
+            30_000,
+            null);
+
+        final String basePath = IGNITE_ZK_ROOT + "/";
+
+        final String aliveDir = basePath + ZkIgnitePaths.ALIVE_NODES_DIR + "/";
+
+        try {
+            List<String> znodes = listSubTree(zkClient.zk(), IGNITE_ZK_ROOT);
+
+            boolean foundAlive = false;
+
+            for (String znode : znodes) {
+                if (znode.startsWith(aliveDir)) {
+                    foundAlive = true;
+
+                    break;
+                }
+            }
+
+            assertTrue(foundAlive); // Sanity check to make sure we check correct directory.
+
+            assertTrue("Failed to wait for unused znodes cleanup", GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    try {
+                        List<String> znodes = listSubTree(zkClient.zk(), IGNITE_ZK_ROOT);
+
+                        for (String znode : znodes) {
+                            if (znode.startsWith(aliveDir) || znode.length() < basePath.length())
+                                continue;
+
+                            znode = znode.substring(basePath.length());
+
+                            if (!znode.contains("/")) // Ignore roots.
+                                continue;
+
+                            // TODO ZK: https://issues.apache.org/jira/browse/IGNITE-8193
+                            if (znode.startsWith("jd/"))
+                                continue;
+
+                            log.info("Found unexpected znode: " + znode);
+
+                            return false;
+                        }
+
+                        return true;
+                    }
+                    catch (Exception e) {
+                        error("Unexpected error: " + e, e);
+
+                        fail("Unexpected error: " + e);
+                    }
+
+                    return false;
+                }
+            }, 10_000));
+        }
+        finally {
+            zkClient.close();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRandomTopologyChanges_RestartZk() throws Exception {
+        randomTopologyChanges(true, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testRandomTopologyChanges_CloseClients() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8182");
+
+        randomTopologyChanges(false, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployService1() throws Exception {
+        startGridsMultiThreaded(3);
+
+        grid(0).services(grid(0).cluster()).deployNodeSingleton("test", new GridCacheAbstractFullApiSelfTest.DummyServiceImpl());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployService2() throws Exception {
+        clientMode(false);
+
+        startGrid(0);
+
+        clientMode(true);
+
+        startGrid(1);
+
+        grid(0).services(grid(0).cluster()).deployNodeSingleton("test", new GridCacheAbstractFullApiSelfTest.DummyServiceImpl());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployService3() throws Exception {
+        IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() {
+            @Override public Object call() throws Exception {
+                clientModeThreadLocal(true);
+
+                startGrid(0);
+
+                return null;
+            }
+        }, "start-node");
+
+        clientModeThreadLocal(false);
+
+        startGrid(1);
+
+        fut.get();
+
+        grid(0).services(grid(0).cluster()).deployNodeSingleton("test", new GridCacheAbstractFullApiSelfTest.DummyServiceImpl());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLargeUserAttribute1() throws Exception {
+        initLargeAttribute();
+
+        startGrid(0);
+
+        checkZkNodesCleanup();
+
+        userAttrs = null;
+
+        startGrid(1);
+
+        waitForEventsAcks(ignite(0));
+
+        waitForTopology(2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLargeUserAttribute2() throws Exception {
+        startGrid(0);
+
+        initLargeAttribute();
+
+        startGrid(1);
+
+        waitForEventsAcks(ignite(0));
+
+        checkZkNodesCleanup();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLargeUserAttribute3() throws Exception {
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        long stopTime = System.currentTimeMillis() + 60_000;
+
+        int nodes = 0;
+
+        for (int i = 0; i < 25; i++) {
+            info("Iteration: " + i);
+
+            if (rnd.nextBoolean())
+                initLargeAttribute();
+            else
+                userAttrs = null;
+
+            clientMode(i > 5);
+
+            startGrid(i);
+
+            nodes++;
+
+            if (System.currentTimeMillis() >= stopTime)
+                break;
+        }
+
+        waitForTopology(nodes);
+    }
+
+    /**
+     *
+     */
+    private void initLargeAttribute() {
+        userAttrs = new HashMap<>();
+
+        int[] attr = new int[1024 * 1024 + ThreadLocalRandom.current().nextInt(1024)];
+
+        for (int i = 0; i < attr.length; i++)
+            attr[i] = i;
+
+        userAttrs.put("testAttr", attr);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLargeCustomEvent() throws Exception {
+        Ignite srv0 = startGrid(0);
+
+        // Send large message, single node in topology.
+        IgniteCache<Object, Object> cache = srv0.createCache(largeCacheConfiguration("c1"));
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, i);
+
+        assertEquals(1, cache.get(1));
+
+        waitForEventsAcks(ignite(0));
+
+        startGridsMultiThreaded(1, 3);
+
+        srv0.destroyCache("c1");
+
+        // Send large message, multiple nodes in topology.
+        cache = srv0.createCache(largeCacheConfiguration("c1"));
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, i);
+
+        waitForTopology(4);
+
+        ignite(3).createCache(largeCacheConfiguration("c2"));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientReconnectSessionExpire1_1() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8131");
+
+        clientReconnectSessionExpire(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientReconnectSessionExpire1_2() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8131");
+
+        clientReconnectSessionExpire(true);
+    }
+
+    /**
+     * @param closeSock Test mode flag.
+     * @throws Exception If failed.
+     */
+    private void clientReconnectSessionExpire(boolean closeSock) throws Exception {
+        startGrid(0);
+
+        sesTimeout = 2000;
+        clientMode(true);
+        testSockNio = true;
+
+        Ignite client = startGrid(1);
+
+        client.cache(DEFAULT_CACHE_NAME).put(1, 1);
+
+        reconnectClientNodes(log, Collections.singletonList(client), closeSock);
+
+        assertEquals(1, client.cache(DEFAULT_CACHE_NAME).get(1));
+
+        client.compute().broadcast(new DummyCallable(null));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testForceClientReconnect() throws Exception {
+        final int SRVS = 3;
+
+        startGrids(SRVS);
+
+        clientMode(true);
+
+        startGrid(SRVS);
+
+        reconnectClientNodes(Collections.singletonList(ignite(SRVS)), new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                ZookeeperDiscoverySpi spi = waitSpi(getTestIgniteInstanceName(SRVS));
+
+                spi.clientReconnect();
+
+                return null;
+            }
+        });
+
+        waitForTopology(SRVS + 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testForcibleClientFail() throws Exception {
+        final int SRVS = 3;
+
+        startGrids(SRVS);
+
+        clientMode(true);
+
+        startGrid(SRVS);
+
+        reconnectClientNodes(Collections.singletonList(ignite(SRVS)), new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                ZookeeperDiscoverySpi spi = waitSpi(getTestIgniteInstanceName(0));
+
+                spi.failNode(ignite(SRVS).cluster().localNode().id(), "Test forcible node fail");
+
+                return null;
+            }
+        });
+
+        waitForTopology(SRVS + 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDuplicatedNodeId() throws Exception {
+        UUID nodeId0 = nodeId = UUID.randomUUID();
+
+        startGrid(0);
+
+        int failingNodeIdx = 100;
+
+        for (int i = 0; i < 5; i++) {
+            final int idx = failingNodeIdx++;
+
+            nodeId = nodeId0;
+
+            info("Start node with duplicated ID [iter=" + i + ", nodeId=" + nodeId + ']');
+
+            Throwable err = GridTestUtils.assertThrows(log, new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    startGrid(idx);
+
+                    return null;
+                }
+            }, IgniteCheckedException.class, null);
+
+            IgniteSpiException spiErr = X.cause(err, IgniteSpiException.class);
+
+            assertNotNull(spiErr);
+            assertTrue(spiErr.getMessage().contains("Node with the same ID already exists"));
+
+            nodeId = null;
+
+            info("Start node with unique ID [iter=" + i + ']');
+
+            Ignite ignite = startGrid(idx);
+
+            nodeId0 = ignite.cluster().localNode().id();
+
+            waitForTopology(i + 2);
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPing() throws Exception {
+        sesTimeout = 5000;
+
+        startGrids(3);
+
+        final ZookeeperDiscoverySpi spi = waitSpi(getTestIgniteInstanceName(1));
+
+        final UUID nodeId = ignite(2).cluster().localNode().id();
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+            @Override public void run() {
+                assertTrue(spi.pingNode(nodeId));
+            }
+        }, 32, "ping");
+
+        fut.get();
+
+        fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+            @Override public void run() {
+                spi.pingNode(nodeId);
+            }
+        }, 32, "ping");
+
+        U.sleep(100);
+
+        stopGrid(2);
+
+        fut.get();
+
+        fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+            @Override public void run() {
+                assertFalse(spi.pingNode(nodeId));
+            }
+        }, 32, "ping");
+
+        fut.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWithPersistence1() throws Exception {
+        startWithPersistence(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWithPersistence2() throws Exception {
+        startWithPersistence(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoOpCommunicationFailureResolve_1() throws Exception {
+        communicationFailureResolve_Simple(2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoOpCommunicationErrorResolve_2() throws Exception {
+        communicationFailureResolve_Simple(10);
+    }
+
+    /**
+     * @param nodes Nodes number.
+     * @throws Exception If failed.
+     */
+    private void communicationFailureResolve_Simple(int nodes) throws Exception {
+        assert nodes > 1;
+
+        sesTimeout = 2000;
+        commFailureRslvr = NoOpCommunicationFailureResolver.FACTORY;
+
+        startGridsMultiThreaded(nodes);
+
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        for (int i = 0; i < 3; i++) {
+            info("Iteration: " + i);
+
+            int idx1 = rnd.nextInt(nodes);
+
+            int idx2;
+
+            do {
+                idx2 = rnd.nextInt(nodes);
+            }
+            while (idx1 == idx2);
+
+            ZookeeperDiscoverySpi spi = spi(ignite(idx1));
+
+            spi.resolveCommunicationFailure(ignite(idx2).cluster().localNode(), new Exception("test"));
+
+            checkInternalStructuresCleanup();
+        }
+    }
+
+    /**
+     * Tests case when one node fails before sending communication status.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNoOpCommunicationErrorResolve_3() throws Exception {
+        sesTimeout = 2000;
+        commFailureRslvr = NoOpCommunicationFailureResolver.FACTORY;
+
+        startGridsMultiThreaded(3);
+
+        sesTimeout = 10_000;
+
+        testSockNio = true;
+        sesTimeout = 5000;
+
+        startGrid(3);
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() {
+                ZookeeperDiscoverySpi spi = spi(ignite(0));
+
+                spi.resolveCommunicationFailure(ignite(1).cluster().localNode(), new Exception("test"));
+
+                return null;
+            }
+        });
+
+        U.sleep(1000);
+
+        ZkTestClientCnxnSocketNIO nio = ZkTestClientCnxnSocketNIO.forNode(ignite(3));
+
+        nio.closeSocket(true);
+
+        try {
+            stopGrid(3);
+
+            fut.get();
+        }
+        finally {
+            nio.allowConnect();
+        }
+
+        waitForTopology(3);
+    }
+
+    /**
+     * Tests case when Coordinator fails while resolve process is in progress.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNoOpCommunicationErrorResolve_4() throws Exception {
+        testCommSpi = true;
+
+        sesTimeout = 2000;
+        commFailureRslvr = NoOpCommunicationFailureResolver.FACTORY;
+
+        startGrid(0);
+
+        startGridsMultiThreaded(1, 3);
+
+        ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.testSpi(ignite(3));
+
+        commSpi.pingLatch = new CountDownLatch(1);
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() {
+                ZookeeperDiscoverySpi spi = spi(ignite(1));
+
+                spi.resolveCommunicationFailure(ignite(2).cluster().localNode(), new Exception("test"));
+
+                return null;
+            }
+        });
+
+        U.sleep(1000);
+
+        assertFalse(fut.isDone());
+
+        stopGrid(0);
+
+        commSpi.pingLatch.countDown();
+
+        fut.get();
+
+        waitForTopology(3);
+    }
+
+    /**
+     * Tests that nodes join is delayed while resolve is in progress.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNoOpCommunicationErrorResolve_5() throws Exception {
+        testCommSpi = true;
+
+        sesTimeout = 2000;
+        commFailureRslvr = NoOpCommunicationFailureResolver.FACTORY;
+
+        startGrid(0);
+
+        startGridsMultiThreaded(1, 3);
+
+        ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.testSpi(ignite(3));
+
+        commSpi.pingStartLatch = new CountDownLatch(1);
+        commSpi.pingLatch = new CountDownLatch(1);
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() {
+                ZookeeperDiscoverySpi spi = spi(ignite(1));
+
+                spi.resolveCommunicationFailure(ignite(2).cluster().localNode(), new Exception("test"));
+
+                return null;
+            }
+        });
+
+        assertTrue(commSpi.pingStartLatch.await(10, SECONDS));
+
+        try {
+            assertFalse(fut.isDone());
+
+            final AtomicInteger nodeIdx = new AtomicInteger(3);
+
+            IgniteInternalFuture<?> startFut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    startGrid(nodeIdx.incrementAndGet());
+
+                    return null;
+                }
+            }, 3, "start-node");
+
+            U.sleep(1000);
+
+            assertFalse(startFut.isDone());
+
+            assertEquals(4, ignite(0).cluster().nodes().size());
+
+            commSpi.pingLatch.countDown();
+
+            startFut.get();
+            fut.get();
+
+            waitForTopology(7);
+        }
+        finally {
+            commSpi.pingLatch.countDown();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationErrorResolve_KillNode_1() throws Exception {
+        communicationFailureResolve_KillNodes(2, Collections.singleton(2L));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationErrorResolve_KillNode_2() throws Exception {
+        communicationFailureResolve_KillNodes(3, Collections.singleton(2L));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationErrorResolve_KillNode_3() throws Exception {
+        communicationFailureResolve_KillNodes(10, Arrays.asList(2L, 4L, 6L));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationErrorResolve_KillCoordinator_1() throws Exception {
+        communicationFailureResolve_KillNodes(2, Collections.singleton(1L));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationErrorResolve_KillCoordinator_2() throws Exception {
+        communicationFailureResolve_KillNodes(3, Collections.singleton(1L));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationErrorResolve_KillCoordinator_3() throws Exception {
+        communicationFailureResolve_KillNodes(10, Arrays.asList(1L, 4L, 6L));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationErrorResolve_KillCoordinator_4() throws Exception {
+        communicationFailureResolve_KillNodes(10, Arrays.asList(1L, 2L, 3L));
+    }
+
+    /**
+     * @param startNodes Number of nodes to start.
+     * @param killNodes Nodes to kill by resolve process.
+     * @throws Exception If failed.
+     */
+    private void communicationFailureResolve_KillNodes(int startNodes, Collection<Long> killNodes) throws Exception {
+        testCommSpi = true;
+
+        commFailureRslvr = TestNodeKillCommunicationFailureResolver.factory(killNodes);
+
+        startGrids(startNodes);
+
+        ZkTestCommunicationSpi commSpi = ZkTestCommunicationSpi.testSpi(ignite(0));
+
+        commSpi.checkRes = new BitSet(startNodes);
+
+        ZookeeperDiscoverySpi spi = null;
+        UUID killNodeId = null;
+
+        for (Ignite node : G.allGrids()) {
+            ZookeeperDiscoverySpi spi0 = spi(node);
+
+            if (!killNodes.contains(node.cluster().localNode().order()))
+                spi = spi0;
+            else
+                killNodeId = node.cluster().localNode().id();
+        }
+
+        assertNotNull(spi);
+        assertNotNull(killNodeId);
+
+        try {
+            spi.resolveCommunicationFailure(spi.getNode(killNodeId), new Exception("test"));
+
+            fail("Exception is not thrown");
+        }
+        catch (IgniteSpiException e) {
+            assertTrue("Unexpected exception: " + e, e.getCause() instanceof ClusterTopologyCheckedException);
+        }
+
+        int expNodes = startNodes - killNodes.size();
+
+        waitForTopology(expNodes);
+
+        for (Ignite node : G.allGrids())
+            assertFalse(killNodes.contains(node.cluster().localNode().order()));
+
+        startGrid(startNodes);
+
+        waitForTopology(expNodes + 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationFailureResolve_KillCoordinator_5() throws Exception {
+        sesTimeout = 2000;
+
+        testCommSpi = true;
+        commFailureRslvr = KillCoordinatorCommunicationFailureResolver.FACTORY;
+
+        startGrids(10);
+
+        int crd = 0;
+
+        int nodeIdx = 10;
+
+        for (int i = 0; i < 10; i++) {
+            info("Iteration: " + i);
+
+            for (Ignite node : G.allGrids())
+                ZkTestCommunicationSpi.testSpi(node).initCheckResult(10);
+
+            UUID crdId = ignite(crd).cluster().localNode().id();
+
+            ZookeeperDiscoverySpi spi = spi(ignite(crd + 1));
+
+            try {
+                spi.resolveCommunicationFailure(spi.getNode(crdId), new Exception("test"));
+
+                fail("Exception is not thrown");
+            }
+            catch (IgniteSpiException e) {
+                assertTrue("Unexpected exception: " + e, e.getCause() instanceof ClusterTopologyCheckedException);
+            }
+
+            waitForTopology(9);
+
+            startGrid(nodeIdx++);
+
+            waitForTopology(10);
+
+            crd++;
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationFailureResolve_KillRandom() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8179");
+
+        sesTimeout = 2000;
+
+        testCommSpi = true;
+        commFailureRslvr = KillRandomCommunicationFailureResolver.FACTORY;
+
+        startGridsMultiThreaded(10);
+
+        clientMode(true);
+
+        startGridsMultiThreaded(10, 5);
+
+        int nodeIdx = 15;
+
+        for (int i = 0; i < 10; i++) {
+            info("Iteration: " + i);
+
+            ZookeeperDiscoverySpi spi = null;
+
+            for (Ignite node : G.allGrids()) {
+                ZkTestCommunicationSpi.testSpi(node).initCheckResult(100);
+
+                spi = spi(node);
+            }
+
+            assert spi != null;
+
+            try {
+                spi.resolveCommunicationFailure(spi.getRemoteNodes().iterator().next(), new Exception("test"));
+            }
+            catch (IgniteSpiException ignore) {
+                // No-op.
+            }
+
+            clientMode(ThreadLocalRandom.current().nextBoolean());
+
+            startGrid(nodeIdx++);
+
+            awaitPartitionMapExchange();
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDefaultCommunicationFailureResolver1() throws Exception {
+        testCommSpi = true;
+        sesTimeout = 5000;
+
+        startGrids(3);
+
+        ZkTestCommunicationSpi.testSpi(ignite(0)).initCheckResult(3, 0, 1);
+        ZkTestCommunicationSpi.testSpi(ignite(1)).initCheckResult(3, 0, 1);
+        ZkTestCommunicationSpi.testSpi(ignite(2)).initCheckResult(3, 2);
+
+        UUID killedId = nodeId(2);
+
+        assertNotNull(ignite(0).cluster().node(killedId));
+
+        ZookeeperDiscoverySpi spi = spi(ignite(0));
+
+        spi.resolveCommunicationFailure(spi.getNode(ignite(1).cluster().localNode().id()), new Exception("test"));
+
+        waitForTopology(2);
+
+        assertNull(ignite(0).cluster().node(killedId));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDefaultCommunicationFailureResolver2() throws Exception {
+        testCommSpi = true;
+        sesTimeout = 5000;
+
+        startGrids(3);
+
+        clientMode(true);
+
+        startGridsMultiThreaded(3, 2);
+
+        ZkTestCommunicationSpi.testSpi(ignite(0)).initCheckResult(5, 0, 1);
+        ZkTestCommunicationSpi.testSpi(ignite(1)).initCheckResult(5, 0, 1);
+        ZkTestCommunicationSpi.testSpi(ignite(2)).initCheckResult(5, 2, 3, 4);
+        ZkTestCommunicationSpi.testSpi(ignite(3)).initCheckResult(5, 2, 3, 4);
+        ZkTestCommunicationSpi.testSpi(ignite(4)).initCheckResult(5, 2, 3, 4);
+
+        ZookeeperDiscoverySpi spi = spi(ignite(0));
+
+        spi.resolveCommunicationFailure(spi.getNode(ignite(1).cluster().localNode().id()), new Exception("test"));
+
+        waitForTopology(2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDefaultCommunicationFailureResolver3() throws Exception {
+        defaultCommunicationFailureResolver_BreakCommunication(3, 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDefaultCommunicationFailureResolver4() throws Exception {
+        defaultCommunicationFailureResolver_BreakCommunication(3, 0);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDefaultCommunicationFailureResolver5() throws Exception {
+        defaultCommunicationFailureResolver_BreakCommunication(10, 1, 3, 6);
+    }
+
+    /**
+     * @param startNodes Initial nodes number.
+     * @param breakNodes Node indices where communication server is closed.
+     * @throws Exception If failed.
+     */
+    private void defaultCommunicationFailureResolver_BreakCommunication(int startNodes, final int...breakNodes) throws Exception {
+        sesTimeout = 5000;
+
+        startGridsMultiThreaded(startNodes);
+
+        final CyclicBarrier b = new CyclicBarrier(breakNodes.length);
+
+        GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+            @Override public void apply(Integer threadIdx) {
+                try {
+                    b.await();
+
+                    int nodeIdx = breakNodes[threadIdx];
+
+                    info("Close communication: " + nodeIdx);
+
+                    ((TcpCommunicationSpi)ignite(nodeIdx).configuration().getCommunicationSpi()).simulateNodeFailure();
+                }
+                catch (Exception e) {
+                    fail("Unexpected error: " + e);
+                }
+            }
+        }, breakNodes.length, "break-communication");
+
+        waitForTopology(startNodes - breakNodes.length);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationFailureResolve_CachesInfo1() throws Exception {
+        testCommSpi = true;
+        sesTimeout = 5000;
+
+        final CacheInfoCommunicationFailureResolver rslvr = new CacheInfoCommunicationFailureResolver();
+
+        commFailureRslvr = new IgniteOutClosure<CommunicationFailureResolver>() {
+            @Override public CommunicationFailureResolver apply() {
+                return rslvr;
+            }
+        };
+
+        startGrids(2);
+
+        awaitPartitionMapExchange();
+
+        Map<String, T3<Integer, Integer, Integer>> expCaches = new HashMap<>();
+
+        expCaches.put(DEFAULT_CACHE_NAME, new T3<>(RendezvousAffinityFunction.DFLT_PARTITION_COUNT, 0, 1));
+
+        checkResolverCachesInfo(ignite(0), expCaches);
+
+        List<CacheConfiguration> caches = new ArrayList<>();
+
+        CacheConfiguration c1 = new CacheConfiguration("c1");
+        c1.setBackups(1);
+        c1.setAffinity(new RendezvousAffinityFunction(false, 64));
+        caches.add(c1);
+
+        CacheConfiguration c2 = new CacheConfiguration("c2");
+        c2.setBackups(2);
+        c2.setAffinity(new RendezvousAffinityFunction(false, 128));
+        caches.add(c2);
+
+        CacheConfiguration c3 = new CacheConfiguration("c3");
+        c3.setCacheMode(CacheMode.REPLICATED);
+        c3.setAffinity(new RendezvousAffinityFunction(false, 256));
+        caches.add(c3);
+
+        ignite(0).createCaches(caches);
+
+        expCaches.put("c1", new T3<>(64, 1, 2));
+        expCaches.put("c2", new T3<>(128, 2, 2));
+        expCaches.put("c3", new T3<>(256, 1, 2));
+
+        checkResolverCachesInfo(ignite(0), expCaches);
+
+        startGrid(2);
+        startGrid(3);
+
+        awaitPartitionMapExchange();
+
+        expCaches.put("c2", new T3<>(128, 2, 3));
+        expCaches.put("c3", new T3<>(256, 1, 4));
+
+        checkResolverCachesInfo(ignite(0), expCaches);
+
+        CacheConfiguration c4 = new CacheConfiguration("c4");
+        c4.setCacheMode(CacheMode.PARTITIONED);
+        c4.setBackups(0);
+        c4.setAffinity(new RendezvousAffinityFunction(false, 256));
+        c4.setNodeFilter(new TestCacheNodeExcludingFilter(getTestIgniteInstanceName(0), getTestIgniteInstanceName(1)));
+
+        ignite(2).createCache(c4);
+
+        expCaches.put("c4", new T3<>(256, 0, 1));
+
+        checkResolverCachesInfo(ignite(0), expCaches);
+
+        stopGrid(0); // Stop current coordinator, check new coordinator will initialize required caches information.
+
+        awaitPartitionMapExchange();
+
+        expCaches.put("c3", new T3<>(256, 1, 3));
+
+        checkResolverCachesInfo(ignite(1), expCaches);
+
+        startGrid(0);
+
+        expCaches.put("c3", new T3<>(256, 1, 4));
+
+        checkResolverCachesInfo(ignite(1), expCaches);
+
+        stopGrid(1);
+
+        expCaches.put("c3", new T3<>(256, 1, 3));
+
+        checkResolverCachesInfo(ignite(3), expCaches);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationFailureResolve_CachesInfo2() throws Exception {
+        testCommSpi = true;
+        sesTimeout = 5000;
+
+        final CacheInfoCommunicationFailureResolver rslvr = new CacheInfoCommunicationFailureResolver();
+
+        commFailureRslvr = new IgniteOutClosure<CommunicationFailureResolver>() {
+            @Override public CommunicationFailureResolver apply() {
+                return rslvr;
+            }
+        };
+
+        Ignite srv0 = startGrid(0);
+
+        CacheConfiguration ccfg = new CacheConfiguration("c1");
+        ccfg.setBackups(1);
+
+        srv0.createCache(ccfg);
+
+        // Block rebalance to make sure node0 will be the only owner.
+        TestRecordingCommunicationSpi.spi(srv0).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+            @Override public boolean apply(ClusterNode node, Message msg) {
+                return msg instanceof GridDhtPartitionSupplyMessage &&
+                    ((GridDhtPartitionSupplyMessage) msg).groupId() == CU.cacheId("c1");
+            }
+        });
+
+        startGrid(1);
+
+        U.sleep(1000);
+
+        ZookeeperDiscoverySpi spi = spi(srv0);
+
+        rslvr.latch = new CountDownLatch(1);
+
+        ZkTestCommunicationSpi.testSpi(srv0).initCheckResult(2, 0);
+
+        spi.resolveCommunicationFailure(spi.getRemoteNodes().iterator().next(), new Exception("test"));
+
+        assertTrue(rslvr.latch.await(10, SECONDS));
+
+        List<List<ClusterNode>> cacheOwners = rslvr.ownersMap.get("c1");
+
+        ClusterNode node0 = srv0.cluster().localNode();
+
+        for (int p = 0; p < RendezvousAffinityFunction.DFLT_PARTITION_COUNT; p++) {
+            List<ClusterNode> owners = cacheOwners.get(p);
+
+            assertEquals(1, owners.size());
+            assertEquals(node0, owners.get(0));
+        }
+
+        TestRecordingCommunicationSpi.spi(srv0).stopBlock();
+
+        awaitPartitionMapExchange();
+
+        Map<String, T3<Integer, Integer, Integer>> expCaches = new HashMap<>();
+
+        expCaches.put(DEFAULT_CACHE_NAME, new T3<>(RendezvousAffinityFunction.DFLT_PARTITION_COUNT, 0, 1));
+        expCaches.put("c1", new T3<>(RendezvousAffinityFunction.DFLT_PARTITION_COUNT, 1, 2));
+
+        checkResolverCachesInfo(srv0, expCaches);
+    }
+
+    /**
+     * @param crd Coordinator node.
+     * @param expCaches Expected caches info.
+     * @throws Exception If failed.
+     */
+    private void checkResolverCachesInfo(Ignite crd, Map<String, T3<Integer, Integer, Integer>> expCaches)
+        throws Exception
+    {
+        CacheInfoCommunicationFailureResolver rslvr =
+            (CacheInfoCommunicationFailureResolver)crd.configuration().getCommunicationFailureResolver();
+
+        assertNotNull(rslvr);
+
+        ZookeeperDiscoverySpi spi = spi(crd);
+
+        rslvr.latch = new CountDownLatch(1);
+
+        ZkTestCommunicationSpi.testSpi(crd).initCheckResult(crd.cluster().nodes().size(), 0);
+
+        spi.resolveCommunicationFailure(spi.getRemoteNodes().iterator().next(), new Exception("test"));
+
+        assertTrue(rslvr.latch.await(10, SECONDS));
+
+        rslvr.checkCachesInfo(expCaches);
+
+        rslvr.reset();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    public void testCommunicationFailureResolve_ConcurrentDiscoveyEvents() throws Exception {
+        sesTimeout = 5000;
+
+        commFailureRslvr = NoOpCommunicationFailureResolver.FACTORY;
+
+        final int INITIAL_NODES = 5;
+
+        startGridsMultiThreaded(INITIAL_NODES);
+
+        final CyclicBarrier b = new CyclicBarrier(4);
+
+        GridCompoundFuture<?, ?> fut = new GridCompoundFuture<>();
+
+        final AtomicBoolean stop = new AtomicBoolean();
+
+        fut.add((IgniteInternalFuture)GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                b.await();
+
+                ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                for (int i = 0; i < 10; i++) {
+                    startGrid(i + INITIAL_NODES);
+
+                    Thread.sleep(rnd.nextLong(1000) + 10);
+
+                    if (stop.get())
+                        break;
+                }
+
+                return null;
+            }
+        }, "test-node-start"));
+
+        fut.add((IgniteInternalFuture)GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                b.await();
+
+                ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                while (!stop.get()) {
+                    startGrid(100);
+
+                    Thread.sleep(rnd.nextLong(1000) + 10);
+
+                    stopGrid(100);
+
+                    Thread.sleep(rnd.nextLong(1000) + 10);
+                }
+
+                return null;
+            }
+        }, "test-node-restart"));
+
+        fut.add((IgniteInternalFuture)GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                b.await();
+
+                ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                int idx = 0;
+
+                while (!stop.get()) {
+                    CacheConfiguration ccfg = new CacheConfiguration("c-" + idx++);
+                    ccfg.setBackups(rnd.nextInt(5));
+
+                    ignite(rnd.nextInt(INITIAL_NODES)).createCache(ccfg);
+
+                    Thread.sleep(rnd.nextLong(1000) + 10);
+
+                    ignite(rnd.nextInt(INITIAL_NODES)).destroyCache(ccfg.getName());
+
+                    Thread.sleep(rnd.nextLong(1000) + 10);
+                }
+
+                return null;
+            }
+        }, "test-create-cache"));
+
+        fut.add((IgniteInternalFuture)GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                try {
+                    b.await();
+
+                    ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                    for (int i = 0; i < 5; i++) {
+                        info("resolveCommunicationFailure: " + i);
+
+                        ZookeeperDiscoverySpi spi = spi(ignite(rnd.nextInt(INITIAL_NODES)));
+
+                        spi.resolveCommunicationFailure(ignite(rnd.nextInt(INITIAL_NODES)).cluster().localNode(),
+                            new Exception("test"));
+                    }
+
+                    return null;
+                }
+                finally {
+                    stop.set(true);
+                }
+            }
+        }, 5, "test-resolve-failure"));
+
+        fut.markInitialized();
+
+        fut.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCommunicationFailureResolve_ConcurrentMultinode() throws Exception {
+        sesTimeout = 5000;
+
+        commFailureRslvr = NoOpCommunicationFailureResolver.FACTORY;
+
+        startGridsMultiThreaded(5);
+
+        client = true;
+
+        startGridsMultiThreaded(5, 5);
+
+        final int NODES = 10;
+
+        GridTestUtils.runMultiThreaded(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                for (int i = 0; i < 5; i++) {
+                    info("resolveCommunicationFailure: " + i);
+
+                    ZookeeperDiscoverySpi spi = spi(ignite(rnd.nextInt(NODES)));
+
+                    spi.resolveCommunicationFailure(spi.getRemoteNodes().iterator().next(), new Exception("test"));
+                }
+
+                return null;
+            }
+        }, 30, "test-resolve-failure");
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConnectionCheck() throws Exception {
+       final int NODES = 5;
+
+        startGridsMultiThreaded(NODES);
+
+       for (int i = 0; i < NODES; i++) {
+           Ignite node = ignite(i);
+
+           TcpCommunicationSpi spi = (TcpCommunicationSpi)node.configuration().getCommunicationSpi();
+
+           List<ClusterNode> nodes = new ArrayList<>(node.cluster().nodes());
+
+           BitSet res = spi.checkConnection(nodes).get();
+
+           for (int j = 0; j < NODES; j++)
+               assertTrue(res.get(j));
+       }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectDisabled_ConnectionLost() throws Exception {
+        clientReconnectDisabled = true;
+
+        startGrid(0);
+
+        sesTimeout = 3000;
+        testSockNio = true;
+        client = true;
+
+        Ignite client = startGrid(1);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        client.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                latch.countDown();
+
+                return false;
+            }
+        }, EventType.EVT_NODE_SEGMENTED);
+
+        ZkTestClientCnxnSocketNIO nio = ZkTestClientCnxnSocketNIO.forNode(client);
+
+        nio.closeSocket(true);
+
+        try {
+            waitNoAliveZkNodes(log,
+                zkCluster.getConnectString(),
+                Collections.singletonList(aliveZkNodePath(client)),
+                10_000);
+        }
+        finally {
+            nio.allowConnect();
+        }
+
+        assertTrue(latch.await(10, SECONDS));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testServersLeft_FailOnTimeout() throws Exception {
+        startGrid(0);
+
+        final int CLIENTS = 5;
+
+        joinTimeout = 3000;
+
+        clientMode(true);
+
+        startGridsMultiThreaded(1, CLIENTS);
+
+        waitForTopology(CLIENTS + 1);
+
+        final CountDownLatch latch = new CountDownLatch(CLIENTS);
+
+        for (int i = 0; i < CLIENTS; i++) {
+            Ignite node = ignite(i + 1);
+
+            node.events().localListen(new IgnitePredicate<Event>() {
+                @Override public boolean apply(Event evt) {
+                    latch.countDown();
+
+                    return false;
+                }
+            }, EventType.EVT_NODE_SEGMENTED);
+        }
+
+        stopGrid(getTestIgniteInstanceName(0), true, false);
+
+        assertTrue(latch.await(10, SECONDS));
+
+        evts.clear();
+    }
+
+    /**
+     *
+     */
+    public void testStartNoServers_FailOnTimeout() {
+        joinTimeout = 3000;
+
+        clientMode(true);
+
+        long start = System.currentTimeMillis();
+
+        Throwable err = GridTestUtils.assertThrows(log, new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                startGrid(0);
+
+                return null;
+            }
+        }, IgniteCheckedException.class, null);
+
+        assertTrue(System.currentTimeMillis() >= start + joinTimeout);
+
+        IgniteSpiException spiErr = X.cause(err, IgniteSpiException.class);
+
+        assertNotNull(spiErr);
+        assertTrue(spiErr.getMessage().contains("Failed to connect to cluster within configured timeout"));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartNoServer_WaitForServers1() throws Exception {
+        startNoServer_WaitForServers(0);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartNoServer_WaitForServers2() throws Exception {
+        startNoServer_WaitForServers(10_000);
+    }
+
+    /**
+     * @param joinTimeout Join timeout.
+     * @throws Exception If failed.
+     */
+    private void startNoServer_WaitForServers(long joinTimeout) throws Exception {
+        this.joinTimeout = joinTimeout;
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                clientModeThreadLocal(true);
+
+                startGrid(0);
+
+                return null;
+            }
+        });
+
+        U.sleep(3000);
+
+        waitSpi(getTestIgniteInstanceName(0));
+
+        clientModeThreadLocal(false);
+
+        startGrid(1);
+
+        fut.get();
+
+        waitForTopology(2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDisconnectOnServersLeft_1() throws Exception {
+        disconnectOnServersLeft(1, 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDisconnectOnServersLeft_2() throws Exception {
+        disconnectOnServersLeft(5, 1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDisconnectOnServersLeft_3() throws Exception {
+        disconnectOnServersLeft(1, 10);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDisconnectOnServersLeft_4() throws Exception {
+        disconnectOnServersLeft(5, 10);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDisconnectOnServersLeft_5() throws Exception {
+        joinTimeout = 10_000;
+
+        disconnectOnServersLeft(5, 10);
+    }
+
+    /**
+     * @param srvs Number of servers.
+     * @param clients Number of clients.
+     * @throws Exception If failed.
+     */
+    private void disconnectOnServersLeft(int srvs, int clients) throws Exception {
+        startGridsMultiThreaded(srvs);
+
+        clientMode(true);
+
+        startGridsMultiThreaded(srvs, clients);
+
+        for (int i = 0; i < 5; i++) {
+            info("Iteration: " + i);
+
+            final CountDownLatch disconnectLatch = new CountDownLatch(clients);
+            final CountDownLatch reconnectLatch = new CountDownLatch(clients);
+
+            IgnitePredicate<Event> p = new IgnitePredicate<Event>() {
+                @Override public boolean apply(Event evt) {
+                    if (evt.type() == EVT_CLIENT_NODE_DISCONNECTED) {
+                        log.info("Disconnected: " + evt);
+
+                        disconnectLatch.countDown();
+                    }
+                    else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
+                        log.info("Reconnected: " + evt);
+
+                        reconnectLatch.countDown();
+
+                        return false;
+                    }
+
+                    return true;
+                }
+            };
+
+            for (int c = 0; c < clients; c++) {
+                Ignite client = ignite(srvs + c);
+
+                assertTrue(client.configuration().isClientMode());
+
+                client.events().localListen(p, EVT_CLIENT_NODE_DISCONNECTED, EVT_CLIENT_NODE_RECONNECTED);
+            }
+
+            log.info("Stop all servers.");
+
+            GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+                @Override public void apply(Integer threadIdx) {
+                    stopGrid(getTestIgniteInstanceName(threadIdx), true, false);
+                }
+            }, srvs, "stop-server");
+
+            waitReconnectEvent(log, disconnectLatch);
+
+            evts.clear();
+
+            clientMode(false);
+
+            log.info("Restart servers.");
+
+            startGridsMultiThreaded(0, srvs);
+
+            waitReconnectEvent(log, reconnectLatch);
+
+            waitForTopology(srvs + clients);
+
+            log.info("Reconnect finished.");
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectServersRestart_1() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8178");
+
+        reconnectServersRestart(1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectServersRestart_2() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8178");
+
+        reconnectServersRestart(3);
+    }
+
+    /**
+     * @param srvs Number of server nodes in test.
+     * @throws Exception If failed.
+     */
+    private void reconnectServersRestart(int srvs) throws Exception {
+        startGridsMultiThreaded(srvs);
+
+        clientMode(true);
+
+        final int CLIENTS = 10;
+
+        startGridsMultiThreaded(srvs, CLIENTS);
+
+        clientMode(false);
+
+        long stopTime = System.currentTimeMillis() + 30_000;
+
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        final int NODES = srvs + CLIENTS;
+
+        int iter = 0;
+
+        while (System.currentTimeMillis() < stopTime) {
+            int restarts = rnd.nextInt(10) + 1;
+
+            info("Test iteration [iter=" + iter++ + ", restarts=" + restarts + ']');
+
+            for (int i = 0; i < restarts; i++) {
+                GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+                    @Override public void apply(Integer threadIdx) {
+                        stopGrid(getTestIgniteInstanceName(threadIdx), true, false);
+                    }
+                }, srvs, "stop-server");
+
+                startGridsMultiThreaded(0, srvs);
+            }
+
+            final Ignite srv = ignite(0);
+
+            assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return srv.cluster().nodes().size() == NODES;
+                }
+            }, 30_000));
+
+            waitForTopology(NODES);
+
+            awaitPartitionMapExchange();
+        }
+
+        evts.clear();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnectServersRestart_3() throws Exception {
+        startGrid(0);
+
+        clientMode(true);
+
+        startGridsMultiThreaded(10, 10);
+
+        stopGrid(getTestIgniteInstanceName(0), true, false);
+
+        final int srvIdx = ThreadLocalRandom.current().nextInt(10);
+
+        final AtomicInteger idx = new AtomicInteger();
+
+        info("Restart nodes.");
+
+        // Test concurrent start when there are disconnected nodes from previous cluster.
+        GridTestUtils.runMultiThreaded(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                int threadIdx = idx.getAndIncrement();
+
+                clientModeThreadLocal(threadIdx == srvIdx || ThreadLocalRandom.current().nextBoolean());
+
+                startGrid(threadIdx);
+
+                return null;
+            }
+        }, 10, "start-node");
+
+        waitForTopology(20);
+
+        evts.clear();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStartNoZk() throws Exception {
+        stopZkCluster();
+
+        sesTimeout = 30_000;
+
+        zkCluster = ZookeeperDiscoverySpiTestSuite2.createTestingCluster(3);
+
+        try {
+            final AtomicInteger idx = new AtomicInteger();
+
+            IgniteInternalFuture fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+                @Override public Void call() throws Exception {
+                    startGrid(idx.getAndIncrement());
+
+                    return null;
+                }
+            }, 5, "start-node");
+
+            U.sleep(5000);
+
+            assertFalse(fut.isDone());
+
+            zkCluster.start();
+
+            fut.get();
+
+            waitForTopology(5);
+        }
+        finally {
+            zkCluster.start();
+        }
+    }
+
+    /**
+     * @param dfltConsistenId Default consistent ID flag.
+     * @throws Exception If failed.
+     */
+    private void startWithPersistence(boolean dfltConsistenId) throws Exception {
+        this.dfltConsistenId = dfltConsistenId;
+
+        persistence = true;
+
+        for (int i = 0; i < 3; i++) {
+            info("Iteration: " + i);
+
+            clientMode(false);
+
+            startGridsMultiThreaded(4, i == 0);
+
+            clientMode(true);
+
+            startGridsMultiThreaded(4, 3);
+
+            waitForTopology(7);
+
+            stopGrid(1);
+
+            waitForTopology(6);
+
+            stopGrid(4);
+
+            waitForTopology(5);
+
+            stopGrid(0);
+
+            waitForTopology(4);
+
+            checkEventsConsistency();
+
+            stopAllGrids();
+
+            evts.clear();
+        }
+    }
+
+    /**
+     * @param clients Clients.
+     * @param c Closure to run.
+     * @throws Exception If failed.
+     */
+    private void reconnectClientNodes(List<Ignite> clients, Callable<Void> c)
+        throws Exception {
+        final CountDownLatch disconnectLatch = new CountDownLatch(clients.size());
+        final CountDownLatch reconnectLatch = new CountDownLatch(clients.size());
+
+        IgnitePredicate<Event> p = new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                if (evt.type() == EVT_CLIENT_NODE_DISCONNECTED) {
+                    log.info("Disconnected: " + evt);
+
+                    disconnectLatch.countDown();
+                }
+                else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
+                    log.info("Reconnected: " + evt);
+
+                    reconnectLatch.countDown();
+                }
+
+                return true;
+            }
+        };
+
+        for (Ignite client : clients)
+            client.events().localListen(p, EVT_CLIENT_NODE_DISCONNECTED, EVT_CLIENT_NODE_RECONNECTED);
+
+        c.call();
+
+        waitReconnectEvent(log, disconnectLatch);
+
+        waitReconnectEvent(log, reconnectLatch);
+
+        for (Ignite client : clients)
+            client.events().stopLocalListen(p);
+    }
+
+    /**
+     * @param restartZk If {@code true} in background restarts on of ZK servers.
+     * @param closeClientSock If {@code true} in background closes zk clients' sockets.
+     * @throws Exception If failed.
+     */
+    private void randomTopologyChanges(boolean restartZk, boolean closeClientSock) throws Exception {
+        sesTimeout = 30_000;
+
+        if (closeClientSock)
+            testSockNio = true;
+
+        List<Integer> startedNodes = new ArrayList<>();
+        List<String> startedCaches = new ArrayList<>();
+
+        int nextNodeIdx = 0;
+        int nextCacheIdx = 0;
+
+        long stopTime = System.currentTimeMillis() + 60_000;
+
+        int MAX_NODES = 20;
+        int MAX_CACHES = 10;
+
+        AtomicBoolean stop = new AtomicBoolean();
+
+        IgniteInternalFuture<?> fut1 = restartZk ? startRestartZkServers(stopTime, stop) : null;
+        IgniteInternalFuture<?> fut2 = closeClientSock ? startCloseZkClientSocket(stopTime, stop) : null;
+
+        try {
+            ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+            while (System.currentTimeMillis() < stopTime) {
+                if (startedNodes.size() > 0 && rnd.nextInt(10) == 0) {
+                    boolean startCache = startedCaches.size() < 2 ||
+                        (startedCaches.size() < MAX_CACHES && rnd.nextInt(5) != 0);
+
+                    int nodeIdx = startedNodes.get(rnd.nextInt(startedNodes.size()));
+
+                    if (startCache) {
+                        String cacheName = "cache-" + nextCacheIdx++;
+
+                        log.info("Next, start new cache [cacheName=" + cacheName +
+                            ", node=" + nodeIdx +
+                            ", crd=" + (startedNodes.isEmpty() ? null : Collections.min(startedNodes)) +
+                            ", curCaches=" + startedCaches.size() + ']');
+
+                        ignite(nodeIdx).createCache(new CacheConfiguration<>(cacheName));
+
+                        startedCaches.add(cacheName);
+                    }
+                    else {
+                        if (startedCaches.size() > 1) {
+                            String cacheName = startedCaches.get(rnd.nextInt(startedCaches.size()));
+
+                            log.info("Next, stop cache [nodeIdx=" + nodeIdx +
+                                ", node=" + nodeIdx +
+                                ", crd=" + (startedNodes.isEmpty() ? null : Collections.min(startedNodes)) +
+                                ", cacheName=" + startedCaches.size() + ']');
+
+                            ignite(nodeIdx).destroyCache(cacheName);
+
+                            assertTrue(startedCaches.remove(cacheName));
+                        }
+                    }
+                }
+                else {
+                    boolean startNode = startedNodes.size() < 2 ||
+                        (startedNodes.size() < MAX_NODES && rnd.nextInt(5) != 0);
+
+                    if (startNode) {
+                        int nodeIdx = nextNodeIdx++;
+
+                        log.info("Next, start new node [nodeIdx=" + nodeIdx +
+                            ", crd=" + (startedNodes.isEmpty() ? null : Collections.min(startedNodes)) +
+                            ", curNodes=" + startedNodes.size() + ']');
+
+                        startGrid(nodeIdx);
+
+                        assertTrue(startedNodes.add(nodeIdx));
+                    }
+                    else {
+                        if (startedNodes.size() > 1) {
+                            int nodeIdx = startedNodes.get(rnd.nextInt(startedNodes.size()));
+
+                            log.info("Next, stop [nodeIdx=" + nodeIdx +
+                                ", crd=" + (startedNodes.isEmpty() ? null : Collections.min(startedNodes)) +
+                                ", curNodes=" + startedNodes.size() + ']');
+
+                            stopGrid(nodeIdx);
+
+                            assertTrue(startedNodes.remove((Integer)nodeIdx));
+                        }
+                    }
+                }
+
+                U.sleep(rnd.nextInt(100) + 1);
+            }
+        }
+        finally {
+            stop.set(true);
+        }
+
+        if (fut1 != null)
+            fut1.get();
+
+        if (fut2 != null)
+            fut2.get();
+    }
+
+    /**
+     *
+     */
+    private void reset() {
+        System.clearProperty(ZOOKEEPER_CLIENT_CNXN_SOCKET);
+
+        ZkTestClientCnxnSocketNIO.reset();
+
+        System.clearProperty(ZOOKEEPER_CLIENT_CNXN_SOCKET);
+
+        err = false;
+
+        evts.clear();
+
+        try {
+            cleanPersistenceDir();
+        }
+        catch (Exception e) {
+            error("Failed to delete DB files: " + e, e);
+        }
+
+        clientThreadLoc.set(null);
+    }
+
+    /**
+     * @param stopTime Stop time.
+     * @param stop Stop flag.
+     * @return Future.
+     */
+    private IgniteInternalFuture<?> startRestartZkServers(final long stopTime, final AtomicBoolean stop) {
+        return GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                while (!stop.get() && System.currentTimeMillis() < stopTime) {
+                    U.sleep(rnd.nextLong(500) + 500);
+
+                    int idx = rnd.nextInt(ZK_SRVS);
+
+                    log.info("Restart ZK server: " + idx);
+
+                    zkCluster.getServers().get(idx).restart();
+
+                }
+
+                return null;
+            }
+        }, "zk-restart-thread");
+    }
+
+    /**
+     * @param stopTime Stop time.
+     * @param stop Stop flag.
+     * @return Future.
+     */
+    private IgniteInternalFuture<?> startCloseZkClientSocket(final long stopTime, final AtomicBoolean stop) {
+        assert testSockNio;
+
+        return GridTestUtils.runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                while (!stop.get() && System.currentTimeMillis() < stopTime) {
+                    U.sleep(rnd.nextLong(100) + 50);
+
+                    List<Ignite> nodes = G.allGrids();
+
+                    if (nodes.size() > 0) {
+                        Ignite node = nodes.get(rnd.nextInt(nodes.size()));
+
+                        ZkTestClientCnxnSocketNIO nio = ZkTestClientCnxnSocketNIO.forNode(node);
+
+                        if (nio != null) {
+                            info("Close zk client socket for node: " + node.name());
+
+                            try {
+                                nio.closeSocket(false);
+                            }
+                            catch (Exception e) {
+                                info("Failed to close zk client socket for node: " + node.name());
+                            }
+                        }
+                    }
+                }
+
+                return null;
+            }
+        }, "zk-restart-thread");
+    }
+
+    /**
+     * @param node Node.
+     * @throws Exception If failed.
+     */
+    private void waitForEventsAcks(final Ignite node) throws Exception {
+        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                Map<Object, Object> evts = GridTestUtils.getFieldValue(node.configuration().getDiscoverySpi(),
+                    "impl", "rtState", "evtsData", "evts");
+
+                if (!evts.isEmpty()) {
+                    info("Unacked events: " + evts);
+
+                    return false;
+                }
+
+                return true;
+            }
+        }, 10_000));
+    }
+
+    /**
+     *
+     */
+    @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+    private void checkEventsConsistency() {
+        for (Map.Entry<UUID, Map<Long, DiscoveryEvent>> nodeEvtEntry : evts.entrySet()) {
+            UUID nodeId = nodeEvtEntry.getKey();
+            Map<Long, DiscoveryEvent> nodeEvts = nodeEvtEntry.getValue();
+
+            for (Map.Entry<UUID, Map<Long, DiscoveryEvent>> nodeEvtEntry0 : evts.entrySet()) {
+                if (!nodeId.equals(nodeEvtEntry0.getKey())) {
+                    Map<Long, DiscoveryEvent> nodeEvts0 = nodeEvtEntry0.getValue();
+
+                    synchronized (nodeEvts) {
+                        synchronized (nodeEvts0) {
+                            checkEventsConsistency(nodeEvts, nodeEvts0);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * @param evts1 Received events.
+     * @param evts2 Received events.
+     */
+    private void checkEventsConsistency(Map<Long, DiscoveryEvent> evts1, Map<Long, DiscoveryEvent> evts2) {
+        for (Map.Entry<Long, DiscoveryEvent> e1 : evts1.entrySet()) {
+            DiscoveryEvent evt1 = e1.getValue();
+            DiscoveryEvent evt2 = evts2.get(e1.getKey());
+
+            if (evt2 != null) {
+                assertEquals(evt1.topologyVersion(), evt2.topologyVersion());
+                assertEquals(evt1.eventNode(), evt2.eventNode());
+                assertEquals(evt1.topologyNodes(), evt2.topologyNodes());
+            }
+        }
+    }
+
+    /**
+     * @param node Node.
+     * @return Node's discovery SPI.
+     */
+    private static ZookeeperDiscoverySpi spi(Ignite node) {
+        return (ZookeeperDiscoverySpi)node.configuration().getDiscoverySpi();
+    }
+
+    /**
+     * @param nodeName Node name.
+     * @return Node's discovery SPI.
+     * @throws Exception If failed.
+     */
+    private ZookeeperDiscoverySpi waitSpi(final String nodeName) throws Exception {
+        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                ZookeeperDiscoverySpi spi = spis.get(nodeName);
+
+                return spi != null && GridTestUtils.getFieldValue(spi, "impl") != null;
+
+            }
+        }, 5000);
+
+        ZookeeperDiscoverySpi spi = spis.get(nodeName);
+
+        assertNotNull("Failed to get SPI for node: " + nodeName, spi);
+
+        return spi;
+    }
+
+    /**
+     * @param topVer Topology version.
+     * @return Expected event instance.
+     */
+    private static DiscoveryEvent joinEvent(long topVer) {
+        DiscoveryEvent expEvt = new DiscoveryEvent(null, null, EventType.EVT_NODE_JOINED, null);
+
+        expEvt.topologySnapshot(topVer, null);
+
+        return expEvt;
+    }
+
+    /**
+     * @param topVer Topology version.
+     * @return Expected event instance.
+     */
+    private static DiscoveryEvent failEvent(long topVer) {
+        DiscoveryEvent expEvt = new DiscoveryEvent(null, null, EventType.EVT_NODE_FAILED, null);
+
+        expEvt.topologySnapshot(topVer, null);
+
+        return expEvt;
+    }
+
+    /**
+     * @param node Node.
+     * @param expEvts Expected events.
+     * @throws Exception If fialed.
+     */
+    private void checkEvents(final Ignite node, final DiscoveryEvent...expEvts) throws Exception {
+        checkEvents(node.cluster().localNode().id(), expEvts);
+    }
+
+    /**
+     * @param nodeId Node ID.
+     * @param expEvts Expected events.
+     * @throws Exception If failed.
+     */
+    private void checkEvents(final UUID nodeId, final DiscoveryEvent...expEvts) throws Exception {
+        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+            @Override public boolean apply() {
+                Map<Long, DiscoveryEvent> nodeEvts = evts.get(nodeId);
+
+                if (nodeEvts == null) {
+                    info("No events for node: " + nodeId);
+
+                    return false;
+                }
+
+                synchronized (nodeEvts) {
+                    for (DiscoveryEvent expEvt : expEvts) {
+                        DiscoveryEvent evt0 = nodeEvts.get(expEvt.topologyVersion());
+
+                        if (evt0 == null) {
+                            info("No event for version: " + expEvt.topologyVersion());
+
+                            return false;
+                        }
+
+                        assertEquals("Unexpected event [topVer=" + expEvt.topologyVersion() +
+                            ", exp=" + U.gridEventName(expEvt.type()) +
+                            ", evt=" + evt0 + ']', expEvt.type(), evt0.type());
+                    }
+                }
+
+                return true;
+            }
+        }, 30000));
+    }
+
+    /**
+     * @param spi Spi instance.
+     */
+    private static void closeZkClient(ZookeeperDiscoverySpi spi) {
+        ZooKeeper zk = zkClient(spi);
+
+        try {
+            zk.close();
+        }
+        catch (Exception e) {
+            fail("Unexpected error: " + e);
+        }
+    }
+
+    /**
+     * @param spi Spi instance.
+     * @return Zookeeper client.
+     */
+    private static ZooKeeper zkClient(ZookeeperDiscoverySpi spi) {
+        return GridTestUtils.getFieldValue(spi, "impl", "rtState", "zkClient", "zk");
+    }
+
+    /**
+     * Reconnect client node.
+     *
+     * @param log  Logger.
+     * @param clients Clients.
+     * @param closeSock {@code True} to simulate reconnect by closing zk client's socket.
+     * @throws Exception If failed.
+     */
+    private static void reconnectClientNodes(final IgniteLogger log,
+        List<Ignite> clients,
+        boolean closeSock)
+        throws Exception {
+        final CountDownLatch disconnectLatch = new CountDownLatch(clients.size());
+        final CountDownLatch reconnectLatch = new CountDownLatch(clients.size());
+
+        IgnitePredicate<Event> p = new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                if (evt.type() == EVT_CLIENT_NODE_DISCONNECTED) {
+                    log.info("Disconnected: " + evt);
+
+                    disconnectLatch.countDown();
+                }
+                else if (evt.type() == EVT_CLIENT_NODE_RECONNECTED) {
+                    log.info("Reconnected: " + evt);
+
+                    reconnectLatch.countDown();
+                }
+
+                return true;
+            }
+        };
+
+        List<String> zkNodes = new ArrayList<>();
+
+        for (Ignite client : clients) {
+            client.events().localListen(p, EVT_CLIENT_NODE_DISCONNECTED, EVT_CLIENT_NODE_RECONNECTED);
+
+            zkNodes.add(aliveZkNodePath(client));
+        }
+
+        long timeout = 15_000;
+
+        if (closeSock) {
+            for (Ignite client : clients) {
+                ZookeeperDiscoverySpi spi = (ZookeeperDiscoverySpi)client.configuration().getDiscoverySpi();
+
+                ZkTestClientCnxnSocketNIO.forNode(client.name()).closeSocket(true);
+
+                timeout = Math.max(timeout, (long)(spi.getSessionTimeout() * 1.5f));
+            }
+        }
+        else {
+            /*
+             * Use hack to simulate session expire without waiting session timeout:
+             * create and close ZooKeeper with the same session ID as ignite node's ZooKeeper.
+             */
+            List<ZooKeeper> dummyClients = new ArrayList<>();
+
+            for (Ignite client : clients) {
+                ZookeeperDiscoverySpi spi = (ZookeeperDiscoverySpi)client.configuration().getDiscoverySpi();
+
+                ZooKeeper zk = zkClient(spi);
+
+                ZooKeeper dummyZk = new ZooKeeper(
+                    spi.getZkConnectionString(),
+                    10_000,
+                    null,
+                    zk.getSessionId(),
+                    zk.getSessionPasswd());
+
+                dummyZk.exists("/a", false);
+
+                dummyClients.add(dummyZk);
+            }
+
+            for (ZooKeeper zk : dummyClients)
+                zk.close();
+        }
+
+        waitNoAliveZkNodes(log,
+            ((ZookeeperDiscoverySpi)clients.get(0).configuration().getDiscoverySpi()).getZkConnectionString(),
+            zkNodes,
+            timeout);
+
+        if (closeSock) {
+            for (Ignite client : clients)
+                ZkTestClientCnxnSocketNIO.forNode(client.name()).allowConnect();
+        }
+
+        waitReconnectEvent(log, disconnectLatch);
+
+        waitReconnectEvent(log, reconnectLatch);
+
+        for (Ignite client : clients)
+            client.events().stopLocalListen(p);
+    }
+
+    /**
+     * @param zk ZooKeeper client.
+     * @param root Root path.
+     * @return All children znodes for given path.
+     * @throws Exception If failed/
+     */
+    private List<String> listSubTree(ZooKeeper zk, String root) throws Exception {
+        for (int i = 0; i < 30; i++) {
+            try {
+                return ZKUtil.listSubTreeBFS(zk, root);
+            }
+            catch (KeeperException.NoNodeException e) {
+                info("NoNodeException when get znodes, will retry: " + e);
+            }
+        }
+
+        throw new Exception("Failed to get znodes: " + root);
+    }
+
+    /**
+     * @param log Logger.
+     * @param latch Latch.
+     * @throws Exception If failed.
+     */
+    private static void waitReconnectEvent(IgniteLogger log, CountDownLatch latch) throws Exception {
+        if (!latch.await(30_000, MILLISECONDS)) {
+            log.error("Failed to wait for reconnect event, will dump threads, latch count: " + latch.getCount());
+
+            U.dumpThreads(log);
+
+            fail("Failed to wait for disconnect/reconnect event.");
+        }
+    }
+
+    /**
+     * @param cacheName Cache name.
+     * @return Configuration.
+     */
+    private CacheConfiguration<Object, Object> largeCacheConfiguration(String cacheName) {
+        CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(cacheName);
+
+        ccfg.setAffinity(new TestAffinityFunction(1024 * 1024));
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+        return ccfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void waitForTopology(int expSize) throws Exception {
+        super.waitForTopology(expSize);
+
+        // checkZkNodesCleanup();
+    }
+
+    /**
+     *
+     */
+    @SuppressWarnings("MismatchedReadAndWriteOfArray")
+    static class TestAffinityFunction extends RendezvousAffinityFunction {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private int[] dummyData;
+
+        /**
+         * @param dataSize Dummy data size.
+         */
+        TestAffinityFunction(int dataSize) {
+            dummyData = new int[dataSize];
+
+            for (int i = 0; i < dataSize; i++)
+                dummyData[i] = i;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class DummyCallable implements IgniteCallable<Object> {
+        /** */
+        private byte[] data;
+
+        /**
+         * @param data Data.
+         */
+        DummyCallable(byte[] data) {
+            this.data = data;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Object call() throws Exception {
+            return data;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class C1 implements Serializable {
+        // No-op.
+    }
+
+    /**
+     *
+     */
+    private static class C2 implements Serializable {
+        // No-op.
+    }
+
+    /**
+     *
+     */
+    static class ZkTestNodeAuthenticator implements DiscoverySpiNodeAuthenticator {
+        /**
+         * @param failAuthNodes Node names which should not pass authentication.
+         * @return Factory.
+         */
+        static IgniteOutClosure<DiscoverySpiNodeAuthenticator> factory(final String...failAuthNodes) {
+            return new IgniteOutClosure<DiscoverySpiNodeAuthenticator>() {
+                @Override public DiscoverySpiNodeAuthenticator apply() {
+                    return new ZkTestNodeAuthenticator(Arrays.asList(failAuthNodes));
+                }
+            };
+        }
+
+        /** */
+        private final Collection<String> failAuthNodes;
+
+        /**
+         * @param failAuthNodes Node names which should not pass authentication.
+         */
+        ZkTestNodeAuthenticator(Collection<String> failAuthNodes) {
+            this.failAuthNodes = failAuthNodes;
+        }
+
+        /** {@inheritDoc} */
+        @Override public SecurityContext authenticateNode(ClusterNode node, SecurityCredentials cred) {
+            assertNotNull(cred);
+
+            String nodeName = node.attribute(ATTR_IGNITE_INSTANCE_NAME);
+
+            assertEquals(nodeName, cred.getUserObject());
+
+            boolean auth = !failAuthNodes.contains(nodeName);
+
+            System.out.println(Thread.currentThread().getName() + " authenticateNode [node=" + node.id() + ", res=" + auth + ']');
+
+            return auth ? new TestSecurityContext(nodeName) : null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isGlobalNodeAuthentication() {
+            return false;
+        }
+
+        /**
+         *
+         */
+        private static class TestSecurityContext implements SecurityContext, Serializable {
+            /** Serial version uid. */
+            private static final long serialVersionUID = 0L;
+
+            /** */
+            final String nodeName;
+
+            /**
+             * @param nodeName Authenticated node name.
+             */
+            TestSecurityContext(String nodeName) {
+                this.nodeName = nodeName;
+            }
+
+            /** {@inheritDoc} */
+            @Override public SecuritySubject subject() {
+                return null;
+            }
+
+            /** {@inheritDoc} */
+            @Override public boolean taskOperationAllowed(String taskClsName, SecurityPermission perm) {
+                return true;
+            }
+
+            /** {@inheritDoc} */
+            @Override public boolean cacheOperationAllowed(String cacheName, SecurityPermission perm) {
+                return true;
+            }
+
+            /** {@inheritDoc} */
+            @Override public boolean serviceOperationAllowed(String srvcName, SecurityPermission perm) {
+                return true;
+            }
+
+            /** {@inheritDoc} */
+            @Override public boolean systemOperationAllowed(SecurityPermission perm) {
+                return true;
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    static class CacheInfoCommunicationFailureResolver implements CommunicationFailureResolver {
+        /** */
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** */
+        Map<String, CacheConfiguration<?, ?>>  caches;
+
+        /** */
+        Map<String, List<List<ClusterNode>>> affMap;
+
+        /** */
+        Map<String, List<List<ClusterNode>>> ownersMap;
+
+        /** */
+        volatile CountDownLatch latch;
+
+        /** {@inheritDoc} */
+        @Override public void resolve(CommunicationFailureContext ctx) {
+            assert latch != null;
+            assert latch.getCount() == 1L : latch.getCount();
+
+            caches = ctx.startedCaches();
+
+            log.info("Resolver called, started caches: " + caches.keySet());
+
+            assertNotNull(caches);
+
+            affMap = new HashMap<>();
+            ownersMap = new HashMap<>();
+
+            for (String cache : caches.keySet()) {
+                affMap.put(cache, ctx.cacheAffinity(cache));
+                ownersMap.put(cache, ctx.cachePartitionOwners(cache));
+            }
+
+            latch.countDown();
+        }
+
+        /**
+         * @param expCaches Expected caches information (when late assignment doen and rebalance finished).
+         */
+        void checkCachesInfo(Map<String, T3<Integer, Integer, Integer>> expCaches) {
+            assertNotNull(caches);
+            assertNotNull(affMap);
+            assertNotNull(ownersMap);
+
+            for (Map.Entry<String, T3<Integer, Integer, Integer>> e : expCaches.entrySet()) {
+                String cacheName = e.getKey();
+
+                int parts = e.getValue().get1();
+                int backups = e.getValue().get2();
+                int expNodes = e.getValue().get3();
+
+                assertTrue(cacheName, caches.containsKey(cacheName));
+
+                CacheConfiguration ccfg = caches.get(cacheName);
+
+                assertEquals(cacheName, ccfg.getName());
+
+                if (ccfg.getCacheMode() == CacheMode.REPLICATED)
+                    assertEquals(Integer.MAX_VALUE, ccfg.getBackups());
+                else
+                    assertEquals(backups, ccfg.getBackups());
+
+                assertEquals(parts, ccfg.getAffinity().partitions());
+
+                List<List<ClusterNode>> aff = affMap.get(cacheName);
+
+                assertNotNull(cacheName, aff);
+                assertEquals(parts, aff.size());
+
+                List<List<ClusterNode>> owners = ownersMap.get(cacheName);
+
+                assertNotNull(cacheName, owners);
+                assertEquals(parts, owners.size());
+
+                for (int i = 0; i < parts; i++) {
+                    List<ClusterNode> partAff = aff.get(i);
+
+                    assertEquals(cacheName, expNodes, partAff.size());
+
+                    List<ClusterNode> partOwners = owners.get(i);
+
+                    assertEquals(cacheName, expNodes, partOwners.size());
+
+                    assertTrue(cacheName, partAff.containsAll(partOwners));
+                    assertTrue(cacheName, partOwners.containsAll(partAff));
+                }
+            }
+        }
+
+        /**
+         *
+         */
+        void reset() {
+            caches = null;
+            affMap = null;
+            ownersMap = null;
+        }
+    }
+
+    /**
+     *
+     */
+    static class NoOpCommunicationFailureResolver implements CommunicationFailureResolver {
+        /** */
+        static final IgniteOutClosure<CommunicationFailureResolver> FACTORY = new IgniteOutClosure<CommunicationFailureResolver>() {
+            @Override public CommunicationFailureResolver apply() {
+                return new NoOpCommunicationFailureResolver();
+            }
+        };
+
+        /** {@inheritDoc} */
+        @Override public void resolve(CommunicationFailureContext ctx) {
+            // No-op.
+        }
+    }
+
+    /**
+     *
+     */
+    static class KillCoordinatorCommunicationFailureResolver implements CommunicationFailureResolver {
+        /** */
+        static final IgniteOutClosure<CommunicationFailureResolver> FACTORY = new IgniteOutClosure<CommunicationFailureResolver>() {
+            @Override public CommunicationFailureResolver apply() {
+                return new KillCoordinatorCommunicationFailureResolver();
+            }
+        };
+
+        /** */
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** {@inheritDoc} */
+        @Override public void resolve(CommunicationFailureContext ctx) {
+            List<ClusterNode> nodes = ctx.topologySnapshot();
+
+            ClusterNode node = nodes.get(0);
+
+            log.info("Resolver kills node: " + node.id());
+
+            ctx.killNode(node);
+        }
+    }
+
+    /**
+     *
+     */
+    static class KillRandomCommunicationFailureResolver implements CommunicationFailureResolver {
+        /** */
+        static final IgniteOutClosure<CommunicationFailureResolver> FACTORY = new IgniteOutClosure<CommunicationFailureResolver>() {
+            @Override public CommunicationFailureResolver apply() {
+                return new KillRandomCommunicationFailureResolver();
+            }
+        };
+
+        /** */
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** {@inheritDoc} */
+        @Override public void resolve(CommunicationFailureContext ctx) {
+            List<ClusterNode> nodes = ctx.topologySnapshot();
+
+            ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+            int killNodes = rnd.nextInt(nodes.size() / 2);
+
+            log.info("Resolver kills nodes [total=" + nodes.size() + ", kill=" + killNodes + ']');
+
+            Set<Integer> idxs = new HashSet<>();
+
+            while (idxs.size() < killNodes)
+                idxs.add(rnd.nextInt(nodes.size()));
+
+            for (int idx : idxs) {
+                ClusterNode node = nodes.get(idx);
+
+                log.info("Resolver kills node: " + node.id());
+
+                ctx.killNode(node);
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    static class TestNodeKillCommunicationFailureResolver implements CommunicationFailureResolver {
+        /**
+         * @param killOrders Killed nodes order.
+         * @return Factory.
+         */
+        static IgniteOutClosure<CommunicationFailureResolver> factory(final Collection<Long> killOrders)  {
+            return new IgniteOutClosure<CommunicationFailureResolver>() {
+                @Override public CommunicationFailureResolver apply() {
+                    return new TestNodeKillCommunicationFailureResolver(killOrders);
+                }
+            };
+        }
+
+        /** */
+        final Collection<Long> killNodeOrders;
+
+        /**
+         * @param killOrders Killed nodes order.
+         */
+        TestNodeKillCommunicationFailureResolver(Collection<Long> killOrders) {
+            this.killNodeOrders = killOrders;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void resolve(CommunicationFailureContext ctx) {
+            List<ClusterNode> nodes = ctx.topologySnapshot();
+
+            assertTrue(nodes.size() > 0);
+
+            for (ClusterNode node : nodes) {
+                if (killNodeOrders.contains(node.order()))
+                    ctx.killNode(node);
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    static class ZkTestCommunicationSpi extends TestRecordingCommunicationSpi {
+        /** */
+        private volatile CountDownLatch pingStartLatch;
+
+        /** */
+        private volatile CountDownLatch pingLatch;
+
+        /** */
+        private volatile BitSet checkRes;
+
+        /**
+         * @param ignite Node.
+         * @return Node's communication SPI.
+         */
+        static ZkTestCommunicationSpi testSpi(Ignite ignite) {
+            return (ZkTestCommunicationSpi)ignite.configuration().getCommunicationSpi();
+        }
+
+        /**
+         * @param nodes Number of nodes.
+         * @param setBitIdxs Bits indexes to set in check result.
+         */
+        void initCheckResult(int nodes, Integer... setBitIdxs) {
+            checkRes = new BitSet(nodes);
+
+            for (Integer bitIdx : setBitIdxs)
+                checkRes.set(bitIdx);
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteFuture<BitSet> checkConnection(List<ClusterNode> nodes) {
+            CountDownLatch pingStartLatch = this.pingStartLatch;
+
+            if (pingStartLatch != null)
+                pingStartLatch.countDown();
+
+            CountDownLatch pingLatch = this.pingLatch;
+
+            try {
+                if (pingLatch != null)
+                    pingLatch.await();
+            }
+            catch (InterruptedException e) {
+                throw new IgniteException(e);
+            }
+
+            BitSet checkRes = this.checkRes;
+
+            if (checkRes != null) {
+                this.checkRes = null;
+
+                return new IgniteFinishedFutureImpl<>(checkRes);
+            }
+
+            return super.checkConnection(nodes);
+        }
+    }
+
+    /**
+     *
+     */
+    static class TestFastStopProcessCustomMessage implements DiscoveryCustomMessage {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteUuid id = IgniteUuid.randomUuid();
+
+        /** */
+        private final boolean createAck;
+
+        /** */
+        private final int payload;
+
+        /**
+         * @param createAck Create ack message flag.
+         * @param payload Payload.
+         */
+        TestFastStopProcessCustomMessage(boolean createAck, int payload) {
+            this.createAck = createAck;
+            this.payload = payload;
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteUuid id() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public DiscoveryCustomMessage ackMessage() {
+            return createAck ? new TestFastStopProcessCustomMessageAck(payload) : null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isMutable() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean stopProcess() {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
+            AffinityTopologyVersion topVer,
+            DiscoCache discoCache) {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            TestFastStopProcessCustomMessage that = (TestFastStopProcessCustomMessage)o;
+
+            return createAck == that.createAck && payload == that.payload;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return Objects.hash(createAck, payload);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(TestFastStopProcessCustomMessage.class, this);
+        }
+    }
+
+    /**
+     *
+     */
+    static class TestFastStopProcessCustomMessageAck implements DiscoveryCustomMessage {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteUuid id = IgniteUuid.randomUuid();
+
+        /** */
+        private final int payload;
+
+        /**
+         * @param payload Payload.
+         */
+        TestFastStopProcessCustomMessageAck(int payload) {
+            this.payload = payload;
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteUuid id() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public DiscoveryCustomMessage ackMessage() {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isMutable() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean stopProcess() {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr,
+            AffinityTopologyVersion topVer,
+            DiscoCache discoCache) {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            TestFastStopProcessCustomMessageAck that = (TestFastStopProcessCustomMessageAck)o;
+            return payload == that.payload;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return Objects.hash(payload);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(TestFastStopProcessCustomMessageAck.class, this);
+        }
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/zookeeper/ZkTestClientCnxnSocketNIO.java b/modules/zookeeper/src/test/java/org/apache/zookeeper/ZkTestClientCnxnSocketNIO.java
new file mode 100644
index 0000000000000..7892b5e7f2bd9
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/zookeeper/ZkTestClientCnxnSocketNIO.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.SelectionKey;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.logger.java.JavaLogger;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class ZkTestClientCnxnSocketNIO extends ClientCnxnSocketNIO {
+    /** */
+    public static final IgniteLogger log = new JavaLogger().getLogger(ZkTestClientCnxnSocketNIO.class);
+
+    /** */
+    public static volatile boolean DEBUG = false;
+
+    /** */
+    public volatile CountDownLatch blockConnectLatch;
+
+    /** */
+    public static ConcurrentHashMap<String, ZkTestClientCnxnSocketNIO> clients = new ConcurrentHashMap<>();
+
+    /** */
+    private final String nodeName;
+
+    /**
+     *
+     */
+    public static void reset() {
+        clients.clear();
+    }
+
+    /**
+     * @param node Node.
+     * @return ZK client.
+     */
+    public static ZkTestClientCnxnSocketNIO forNode(Ignite node) {
+        return clients.get(node.name());
+    }
+
+    /**
+     * @param instanceName Ignite instance name.
+     * @return ZK client.
+     */
+    public static ZkTestClientCnxnSocketNIO forNode(String instanceName) {
+        return clients.get(instanceName);
+    }
+
+    /**
+     * @throws IOException If failed.
+     */
+    public ZkTestClientCnxnSocketNIO() throws IOException {
+        super();
+
+        String threadName = Thread.currentThread().getName();
+
+        nodeName = threadName.substring(threadName.indexOf('-') + 1);
+
+        if (DEBUG)
+            log.info("ZkTestClientCnxnSocketNIO created for node: " + nodeName);
+    }
+
+    /** {@inheritDoc} */
+    @Override void connect(InetSocketAddress addr) throws IOException {
+        CountDownLatch blockConnect = this.blockConnectLatch;
+
+        if (DEBUG)
+            log.info("ZkTestClientCnxnSocketNIO connect [node=" + nodeName + ", addr=" + addr + ']');
+
+        if (blockConnect != null && blockConnect.getCount() > 0) {
+            try {
+                log.info("ZkTestClientCnxnSocketNIO block connect");
+
+                blockConnect.await(60, TimeUnit.SECONDS);
+
+                log.info("ZkTestClientCnxnSocketNIO finish block connect");
+            }
+            catch (Exception e) {
+                log.error("Error in ZkTestClientCnxnSocketNIO: " + e, e);
+            }
+        }
+
+        super.connect(addr);
+
+        clients.put(nodeName, this);
+    }
+
+    /**
+     *
+     */
+    public void allowConnect() {
+        assert blockConnectLatch != null && blockConnectLatch.getCount() == 1 : blockConnectLatch;
+
+        log.info("ZkTestClientCnxnSocketNIO allowConnect [node=" + nodeName + ']');
+
+        blockConnectLatch.countDown();
+    }
+
+    /**
+     * @param blockConnect {@code True} to block client reconnect.
+     * @throws Exception If failed.
+     */
+    public void closeSocket(boolean blockConnect) throws Exception {
+        if (blockConnect)
+            blockConnectLatch = new CountDownLatch(1);
+
+        log.info("ZkTestClientCnxnSocketNIO closeSocket [node=" + nodeName + ", block=" + blockConnect + ']');
+
+        SelectionKey k = GridTestUtils.getFieldValue(this, ClientCnxnSocketNIO.class, "sockKey");
+
+        k.channel().close();
+    }
+}

From b096a463c338565a7661f8a853a257518d872997 Mon Sep 17 00:00:00 2001
From: Stanislav Lukyanov <lukyanov.lsa@gmail.com>
Date: Mon, 9 Apr 2018 14:33:13 +0300
Subject: [PATCH 010/543] IGNITE-7904: Changed IgniteUtils::cast not to trim
 exception chains. This closes #3683.

---
 .../ignite/compute/ComputeTaskAdapter.java    |   2 +-
 .../processors/cache/GridCacheUtils.java      |   5 +-
 .../cache/IgniteCacheProxyImpl.java           |   3 +
 .../processors/igfs/IgfsMetaManager.java      |  30 ++-
 .../processors/job/GridJobWorker.java         |   2 +-
 .../platform/services/PlatformServices.java   |   8 +-
 .../processors/service/GridServiceProxy.java  |  27 ++-
 .../ignite/internal/util/IgniteUtils.java     |  32 +--
 .../IgniteComputeResultExceptionTest.java     | 186 ++++++++++++++++++
 .../GridCacheAbstractFullApiSelfTest.java     |   9 +-
 .../closure/GridClosureSerializationTest.java |   2 +-
 .../GridServiceProcessorProxySelfTest.java    |  12 +-
 .../IgniteComputeGridTestSuite.java           |   2 +
 ...teCacheLockPartitionOnAffinityRunTest.java |  46 ++---
 .../CacheJdbcPojoStoreFactorySelfTest.java    |  11 +-
 .../GridServiceInjectionSelfTest.java         |  64 +++---
 .../GridSpringResourceInjectionSelfTest.java  |  58 +++---
 17 files changed, 339 insertions(+), 160 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/compute/ComputeTaskAdapter.java b/modules/core/src/main/java/org/apache/ignite/compute/ComputeTaskAdapter.java
index c5352aacb41ff..fc55ad9f30f8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/compute/ComputeTaskAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/compute/ComputeTaskAdapter.java
@@ -99,7 +99,7 @@ public abstract class ComputeTaskAdapter<T, R> implements ComputeTask<T, R> {
                 return ComputeJobResultPolicy.FAILOVER;
 
             throw new IgniteException("Remote job threw user exception (override or implement ComputeTask.result(..) " +
-                "method if you would like to have automatic failover for this exception).", e);
+                "method if you would like to have automatic failover for this exception): " + e.getMessage(), e);
         }
 
         // Wait for all job responses.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 6d4c1f21cb273..a5169d26dcc13 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1283,8 +1283,9 @@ else if (e instanceof ClusterTopologyServerNotFoundException)
         else if (e instanceof SchemaOperationException)
             return new CacheException(e.getMessage(), e);
 
-        if (e.getCause() instanceof CacheException)
-            return (CacheException)e.getCause();
+        CacheException ce = X.cause(e, CacheException.class);
+        if (ce != null)
+            return ce;
 
         if (e.getCause() instanceof NullPointerException)
             return (NullPointerException)e.getCause();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index c5d68b58f1efc..be4b0dbffaf5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -1727,6 +1727,9 @@ private RuntimeException cacheException(Exception e) {
                         ctx.name(), e);
         }
 
+        if (e instanceof IgniteException && X.hasCause(e, CacheException.class))
+            e = X.cause(e, CacheException.class);
+
         if (e instanceof IgniteCheckedException)
             return CU.convertToCacheException((IgniteCheckedException) e);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index a26239caa40b5..e821b80641b4d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -89,6 +89,7 @@
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T1;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.transactions.TransactionConcurrency;
@@ -253,29 +254,20 @@ boolean isClient() {
      * @return Result.
      */
     <T> T runClientTask(IgfsClientAbstractCallable<T> task) {
-        try {
-            return runClientTask(IgfsUtils.ROOT_ID, task);
-        }
-        catch (ClusterTopologyException e) {
-            throw new IgfsException("Failed to execute operation because there are no IGFS metadata nodes." , e);
-        }
-    }
-
-    /**
-     * Run client task.
-     *
-     * @param affinityFileId Affinity fileId.
-     * @param task Task.
-     * @return Result.
-     */
-    <T> T runClientTask(IgniteUuid affinityFileId, IgfsClientAbstractCallable<T> task) {
         try {
             return (cfg.isColocateMetadata()) ?
-                clientCompute().affinityCall(metaCacheName, affinityFileId, task) :
+                clientCompute().affinityCall(metaCacheName, IgfsUtils.ROOT_ID, task) :
                 clientCompute().call(task);
         }
-        catch (ClusterTopologyException e) {
-            throw new IgfsException("Failed to execute operation because there are no IGFS metadata nodes." , e);
+        catch (Exception e) {
+            if (X.hasCause(e, ClusterTopologyException.class))
+                throw new IgfsException("Failed to execute operation because there are no IGFS metadata nodes." , e);
+
+            IgfsException igfsEx = X.cause(e, IgfsException.class);
+            if (igfsEx != null)
+                throw igfsEx;
+
+            throw e;
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index cce1077d42425..6d2e621c3b9af 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -678,7 +678,7 @@ else if (sysStopping && X.hasCause(e, InterruptedException.class, IgniteInterrup
 
         if (msg == null) {
             msg = "Failed to execute job due to unexpected runtime exception [jobId=" + ses.getJobId() +
-                ", ses=" + ses + ']';
+                ", ses=" + ses + ", err=" + e.getMessage() + ']';
 
             ex = new ComputeUserUndeclaredException(msg, e);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
index ccb04d4d6c0d6..4ae59b99929e3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/services/PlatformServices.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.processors.platform.utils.PlatformWriterBiClosure;
 import org.apache.ignite.internal.processors.platform.utils.PlatformWriterClosure;
 import org.apache.ignite.internal.processors.service.GridServiceProxy;
+import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.T3;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -581,7 +582,12 @@ public Object invoke(String mthdName, boolean srvKeepBinary, Object[] args)
 
                 Method mtd = getMethod(serviceClass, mthdName, args);
 
-                return ((GridServiceProxy)proxy).invokeMethod(mtd, args);
+                try {
+                    return ((GridServiceProxy)proxy).invokeMethod(mtd, args);
+                }
+                catch (Throwable t) {
+                    throw IgniteUtils.cast(t);
+                }
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
index e55c2e5d4099b..c5a2ceea19f54 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProxy.java
@@ -47,6 +47,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
@@ -150,7 +151,7 @@ private boolean hasLocalNode(ClusterGroup prj) {
      * @return Result.
      */
     @SuppressWarnings("BusyWait")
-    public Object invokeMethod(final Method mtd, final Object[] args) {
+    public Object invokeMethod(final Method mtd, final Object[] args) throws Throwable {
         if (U.isHashCodeMethod(mtd))
             return System.identityHashCode(proxy);
         else if (U.isEqualsMethod(mtd))
@@ -205,6 +206,12 @@ else if (U.isToStringMethod(mtd))
                     throw e;
                 }
                 catch (IgniteCheckedException e) {
+                    // Rethrow original service method exception so that calling user code can handle it correctly.
+                    ServiceProxyException svcProxyE = X.cause(e, ServiceProxyException.class);
+
+                    if (svcProxyE != null)
+                        throw svcProxyE.getCause();
+
                     throw U.convertException(e);
                 }
                 catch (Exception e) {
@@ -352,7 +359,7 @@ private class ProxyInvocationHandler implements InvocationHandler {
 
         /** {@inheritDoc} */
         @SuppressWarnings("BusyWait")
-        @Override public Object invoke(Object proxy, final Method mtd, final Object[] args) {
+        @Override public Object invoke(Object proxy, final Method mtd, final Object[] args) throws Throwable {
             return invokeMethod(mtd, args);
         }
     }
@@ -418,8 +425,7 @@ private ServiceProxyCallable(String mtdName, String svcName, Class[] argTypes, O
                 return mtd.invoke(svcCtx.service(), args);
             }
             catch (InvocationTargetException e) {
-                // Get error message.
-                throw new IgniteCheckedException(e.getCause().getMessage(), e);
+                throw new ServiceProxyException(e.getCause());
             }
         }
 
@@ -444,4 +450,17 @@ private ServiceProxyCallable(String mtdName, String svcName, Class[] argTypes, O
             return S.toString(ServiceProxyCallable.class, this);
         }
     }
+
+    /**
+     * Exception class that wraps an exception thrown by the service implementation.
+     */
+    private static class ServiceProxyException extends RuntimeException {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        ServiceProxyException(Throwable cause) {
+            super(cause);
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 18e182dde22c8..93f4fb4a39f2e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -7285,35 +7285,23 @@ public static Exception unwrap(Throwable t) {
     }
 
     /**
-     * Casts this throwable as {@link IgniteCheckedException}. Creates wrapping
-     * {@link IgniteCheckedException}, if needed.
+     * Casts the passed {@code Throwable t} to {@link IgniteCheckedException}.<br>
+     * If {@code t} is a {@link GridClosureException}, it is unwrapped and then cast to {@link IgniteCheckedException}.
+     * If {@code t} is an {@link IgniteCheckedException}, it is returned.
+     * If {@code t} is not a {@link IgniteCheckedException}, a new {@link IgniteCheckedException} caused by {@code t}
+     * is returned.
      *
      * @param t Throwable to cast.
-     * @return Grid exception.
+     * @return {@code t} cast to {@link IgniteCheckedException}.
      */
     public static IgniteCheckedException cast(Throwable t) {
         assert t != null;
 
-        while (true) {
-            if (t instanceof Error)
-                throw (Error)t;
+        t = unwrap(t);
 
-            if (t instanceof GridClosureException) {
-                t = ((GridClosureException)t).unwrap();
-
-                continue;
-            }
-
-            if (t instanceof IgniteCheckedException)
-                return (IgniteCheckedException)t;
-
-            if (!(t instanceof IgniteException) || t.getCause() == null)
-                return new IgniteCheckedException(t);
-
-            assert t.getCause() != null; // ...and it is IgniteException.
-
-            t = t.getCause();
-        }
+        return t instanceof IgniteCheckedException
+            ? (IgniteCheckedException)t
+            : new IgniteCheckedException(t);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java
new file mode 100644
index 0000000000000..fab5de6d25229
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteComputeResultExceptionTest.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
+import org.apache.ignite.compute.ComputeTask;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Testing that if {@link ComputeTask#result(ComputeJobResult, List)} throws an {@link IgniteException}
+ * then that exception is thrown as the execution result.
+ */
+public class IgniteComputeResultExceptionTest extends GridCommonAbstractTest {
+    /** */
+    public void testIgniteExceptionExecute() throws Exception {
+        checkExecuteException(new IgniteException());
+    }
+
+    /** */
+    public void testIgniteExceptionWithCauseExecute() throws Exception {
+        checkExecuteException(new IgniteException(new Exception()));
+    }
+
+    /** */
+    public void testIgniteExceptionWithCauseChainExecute() throws Exception {
+        checkExecuteException(new IgniteException(new Exception(new Throwable())));
+    }
+
+    /** */
+    public void testCustomExceptionExecute() throws Exception {
+        checkExecuteException(new TaskException());
+    }
+
+    /** */
+    public void testCustomExceptionWithCauseExecute() throws Exception {
+        checkExecuteException(new TaskException(new Exception()));
+    }
+
+    /** */
+    public void testCustomExceptionWithCauseChainExecute() throws Exception {
+        checkExecuteException(new TaskException(new Exception(new Throwable())));
+    }
+
+    /** */
+    private void checkExecuteException(IgniteException resE) throws Exception {
+        try (Ignite ignite = startGrid()) {
+            IgniteCompute compute = ignite.compute();
+            try {
+                compute.execute(new ResultExceptionTask(resE), null);
+            } catch (IgniteException e) {
+                assertSame(resE, e);
+            }
+        }
+    }
+
+    /** */
+    public void testIgniteExceptionExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new IgniteException());
+    }
+
+    /** */
+    public void testIgniteExceptionWithCauseExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new IgniteException(new Exception()));
+    }
+
+    /** */
+    public void testIgniteExceptionWithCauseChainExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new IgniteException(new Exception(new Throwable())));
+    }
+
+
+    /** */
+    public void testCustomExceptionExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new TaskException());
+    }
+
+    /** */
+    public void testCustomExceptionWithCauseExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new TaskException(new Exception()));
+    }
+
+    /** */
+    public void testCustomExceptionWithCauseChainExecuteAsync() throws Exception {
+        checkExecuteAsyncException(new TaskException(new Exception(new Throwable())));
+    }
+
+    /** */
+    private void checkExecuteAsyncException(IgniteException resE) throws Exception {
+        try (Ignite ignite = startGrid()) {
+            IgniteCompute compute = ignite.compute();
+            ComputeTaskFuture<Object> fut = compute.executeAsync(new ResultExceptionTask(resE), null);
+            try {
+                fut.get();
+            } catch (IgniteException e) {
+                assertSame(resE, e);
+            }
+        }
+    }
+
+    /** */
+    private static class TaskException extends IgniteException {
+        /** */
+        public TaskException() {
+            // No-op.
+        }
+
+        /** */
+        public TaskException(Throwable cause) {
+            super(cause);
+        }
+    }
+
+    /** */
+    private static class NoopJob implements ComputeJob {
+        /** */
+        @Override public void cancel() {
+            // No-op.
+        }
+
+        /** */
+        @Override public Object execute() throws IgniteException {
+            return null;
+        }
+    }
+
+    /** */
+    private static class ResultExceptionTask implements ComputeTask<Object, Object> {
+        /** */
+        private final IgniteException resE;
+
+        /**
+         * @param resE Exception to be rethrown by the
+         */
+        ResultExceptionTask(IgniteException resE) {
+            this.resE = resE;
+        }
+
+        /** */
+        @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
+            @Nullable Object arg) throws IgniteException {
+            Map<ComputeJob, ClusterNode> jobs = new HashMap<>();
+
+            for (ClusterNode node : subgrid)
+                jobs.put(new NoopJob(), node);
+
+            return jobs;
+        }
+
+        /** */
+        @Override
+        public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) throws IgniteException {
+            throw resE;
+        }
+
+        /** */
+        @Nullable @Override public Object reduce(List<ComputeJobResult> results) throws IgniteException {
+            return null;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
index 2e6a19cb5903a..e5df2c83d6af8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java
@@ -82,6 +82,7 @@
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -5041,12 +5042,16 @@ private void waitForIteratorsCleared(IgniteCache<String, Integer> cache, int sec
 
                 checkIteratorsCleared();
             }
-            catch (AssertionFailedError e) {
+            catch (Throwable t) {
+                // If AssertionFailedError is in the chain, assume we need to wait and retry.
+                if (!X.hasCause(t, AssertionFailedError.class))
+                    throw t;
+                
                 if (i == 9) {
                     for (int j = 0; j < gridCount(); j++)
                         executeOnLocalOrRemoteJvm(j, new PrintIteratorStateTask());
 
-                    throw e;
+                    throw t;
                 }
 
                 log.info("Iterators not cleared, will wait");
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
index 8bcbd817574e3..324a9e9c99252 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/closure/GridClosureSerializationTest.java
@@ -68,7 +68,7 @@ public void testSerializationFailure() throws Exception {
         final IgniteEx ignite0 = grid(0);
         final IgniteEx ignite1 = grid(1);
 
-        GridTestUtils.assertThrows(null, new Callable<Object>() {
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
                 ignite1.compute(ignite1.cluster().forNode(ignite0.localNode())).call(new IgniteCallable<Object>() {
                     @Override public Object call() throws Exception {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
index d1c5294d48604..97d5f05e9e52b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceProcessorProxySelfTest.java
@@ -89,7 +89,7 @@ public void testException() throws Exception {
 
                 return null;
             }
-        }, IgniteException.class, "Test exception");
+        }, ErrorServiceException.class, "Test exception");
 
     }
 
@@ -450,9 +450,15 @@ protected class ErrorServiceImpl implements ErrorService {
 
         /** {@inheritDoc} */
         @Override public void go() throws Exception {
-            throw new Exception("Test exception");
+            throw new ErrorServiceException("Test exception");
         }
     }
 
-
+    /** */
+    private static class ErrorServiceException extends Exception {
+        /** */
+        ErrorServiceException(String msg) {
+            super(msg);
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index be99adf3b169d..14eb296424a47 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -68,6 +68,7 @@
 import org.apache.ignite.internal.GridTaskTimeoutSelfTest;
 import org.apache.ignite.internal.IgniteComputeEmptyClusterGroupTest;
 import org.apache.ignite.internal.IgniteComputeJobOneThreadTest;
+import org.apache.ignite.internal.IgniteComputeResultExceptionTest;
 import org.apache.ignite.internal.IgniteComputeTopologyExceptionTest;
 import org.apache.ignite.internal.IgniteExecutorServiceTest;
 import org.apache.ignite.internal.IgniteExplicitImplicitDeploymentSelfTest;
@@ -158,6 +159,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridMultinodeRedeployIsolatedModeSelfTest.class);
         suite.addTestSuite(IgniteComputeEmptyClusterGroupTest.class);
         suite.addTestSuite(IgniteComputeTopologyExceptionTest.class);
+        suite.addTestSuite(IgniteComputeResultExceptionTest.class);
         suite.addTestSuite(GridTaskFailoverAffinityRunTest.class);
         suite.addTestSuite(TaskNodeRestartTest.class);
         suite.addTestSuite(IgniteRoundRobinErrorAfterClientReconnectTest.class);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
index 598212bd04e3a..7bddafc88231a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
@@ -23,6 +23,7 @@
 import java.io.ObjectOutput;
 import java.util.Arrays;
 import java.util.List;
+import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
@@ -404,39 +405,20 @@ public void testMultipleCaches() throws Exception {
     public void testCheckReservePartitionException() throws Exception {
         int orgId = primaryKey(grid(1).cache(Organization.class.getSimpleName()));
 
-        try {
-            grid(0).compute().affinityRun(
-                Arrays.asList(Organization.class.getSimpleName(), OTHER_CACHE_NAME),
-                new Integer(orgId),
-                new IgniteRunnable() {
-                    @Override public void run() {
-                        // No-op.
-                    }
-                });
-
-            fail("Exception is expected");
-        }
-        catch (Exception e) {
-            assertTrue(e.getMessage()
-                .startsWith("Failed partition reservation. Partition is not primary on the node."));
-        }
-
-        try {
-            grid(0).compute().affinityCall(
-                Arrays.asList(Organization.class.getSimpleName(), OTHER_CACHE_NAME),
-                new Integer(orgId),
-                new IgniteCallable<Object>() {
-                    @Override public Object call() throws Exception {
-                        return null;
-                    }
-                });
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                grid(0).compute().affinityRun(
+                    Arrays.asList(Organization.class.getSimpleName(), OTHER_CACHE_NAME),
+                    new Integer(orgId),
+                    new IgniteRunnable() {
+                        @Override public void run() {
+                            // No-op.
+                        }
+                    });
 
-            fail("Exception is expected");
-        }
-        catch (Exception e) {
-            assertTrue(e.getMessage()
-                .startsWith("Failed partition reservation. Partition is not primary on the node."));
-        }
+                return null;
+            }
+        }, IgniteException.class, "Failed partition reservation. Partition is not primary on the node.");
     }
 
     /**
diff --git a/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java b/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java
index 0fd16f0cd94e7..0d6d6a28279fe 100644
--- a/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java
@@ -21,7 +21,7 @@
 import java.util.concurrent.Callable;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.store.jdbc.dialect.H2Dialect;
 import org.apache.ignite.cache.store.jdbc.dialect.JdbcDialect;
@@ -69,15 +69,14 @@ public void testSerializable() throws Exception {
      * @throws Exception If failed.
      */
     public void testIncorrectBeanConfiguration() throws Exception {
-        GridTestUtils.assertThrows(log, new Callable<Object>() {
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
-                try(Ignite ignite = Ignition.start("modules/spring/src/test/config/pojo-incorrect-store-cache.xml")) {
-                    ignite.cache(CACHE_NAME).getConfiguration(CacheConfiguration.class).
-                        getCacheStoreFactory().create();
+                try (Ignite ignored = Ignition.start("modules/spring/src/test/config/pojo-incorrect-store-cache.xml")) {
+                    // No-op.
                 }
                 return null;
             }
-        }, IgniteException.class, "Spring bean with provided name doesn't exist");
+        }, IgniteCheckedException.class, "Spring bean with provided name doesn't exist");
     }
 
     /**
diff --git a/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridServiceInjectionSelfTest.java b/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridServiceInjectionSelfTest.java
index b59bf2457f4de..52581b6d699ac 100644
--- a/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridServiceInjectionSelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridServiceInjectionSelfTest.java
@@ -18,11 +18,13 @@
 package org.apache.ignite.internal.processors.resource;
 
 import java.io.Serializable;
-import org.apache.ignite.IgniteException;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.resources.ServiceResource;
 import org.apache.ignite.services.Service;
 import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -124,23 +126,22 @@ public void testClosureFieldLocalProxy() throws Exception {
      * @throws Exception If failed.
      */
     public void testClosureFieldWithIncorrectType() throws Exception {
-        try {
-            grid(0).compute().call(new IgniteCallable<Object>() {
-                @ServiceResource(serviceName = SERVICE_NAME1)
-                private String svcName;
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Void>() {
+            @Override public Void call() {
+                grid(0).compute().call(new IgniteCallable<Object>() {
+                    @ServiceResource(serviceName = SERVICE_NAME1)
+                    private String svcName;
 
-                @Override public Object call() throws Exception {
-                    fail();
+                    @Override public Object call() throws Exception {
+                        fail();
 
-                    return null;
-                }
-            });
+                        return null;
+                    }
+                });
 
-            fail();
-        }
-        catch (IgniteException e) {
-            assertTrue(e.getMessage().startsWith("Resource field is not assignable from the resource"));
-        }
+                return null;
+            }
+        }, IgniteCheckedException.class, "Resource field is not assignable from the resource");
     }
 
     /**
@@ -221,23 +222,22 @@ private void service(DummyService svc) {
      * @throws Exception If failed.
      */
     public void testClosureMethodWithIncorrectType() throws Exception {
-        try {
-            grid(0).compute().call(new IgniteCallable<Object>() {
-                @ServiceResource(serviceName = SERVICE_NAME1)
-                private void service(String svcs) {
-                    fail();
-                }
-
-                @Override public Object call() throws Exception {
-                    return null;
-                }
-            });
-
-            fail();
-        }
-        catch (IgniteException e) {
-            assertTrue(e.getMessage().startsWith("Setter does not have single parameter of required type"));
-        }
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Void>() {
+            @Override public Void call() {
+                grid(0).compute().call(new IgniteCallable<Object>() {
+                    @ServiceResource(serviceName = SERVICE_NAME1)
+                    private void service(String svcs) {
+                        fail();
+                    }
+
+                    @Override public Object call() throws Exception {
+                        return null;
+                    }
+                });
+
+                return null;
+            }
+        }, IgniteCheckedException.class, "Setter does not have single parameter of required type");
     }
 
     /**
diff --git a/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridSpringResourceInjectionSelfTest.java b/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridSpringResourceInjectionSelfTest.java
index 827dd72efc2dd..08fe69da55852 100644
--- a/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridSpringResourceInjectionSelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/internal/processors/resource/GridSpringResourceInjectionSelfTest.java
@@ -27,6 +27,8 @@
 import org.apache.ignite.resources.SpringResource;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.springframework.beans.factory.NoSuchBeanDefinitionException;
+import org.springframework.beans.factory.NoUniqueBeanDefinitionException;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 
 /**
@@ -93,7 +95,7 @@ public void testClosureFieldByResourceClassWithMultipleBeans() throws Exception
         Ignite anotherGrid = IgniteSpring.start(anotherCfg, new ClassPathXmlApplicationContext(
             "/org/apache/ignite/internal/processors/resource/spring-resource-with-duplicate-beans.xml"));
 
-        Throwable err = assertError(new IgniteCallable<Object>() {
+        assertError(new IgniteCallable<Object>() {
             @SpringResource(resourceClass = DummyResourceBean.class)
             private transient DummyResourceBean dummyRsrcBean;
 
@@ -102,11 +104,9 @@ public void testClosureFieldByResourceClassWithMultipleBeans() throws Exception
 
                 return null;
             }
-        }, anotherGrid, null);
-
-        assertTrue("Unexpected message: " + err.getMessage(), err.getMessage().startsWith("No qualifying bean of type " +
+        }, anotherGrid, NoUniqueBeanDefinitionException.class, "No qualifying bean of type " +
             "'org.apache.ignite.internal.processors.resource.GridSpringResourceInjectionSelfTest$DummyResourceBean'" +
-            " available: expected single matching bean but found 2:"));
+            " available: expected single matching bean but found 2:");
 
         G.stop("anotherGrid", false);
     }
@@ -124,7 +124,7 @@ public void testClosureFieldWithWrongResourceName() {
 
                 return null;
             }
-        }, "No bean named 'nonExistentResource' available");
+        }, grid, NoSuchBeanDefinitionException.class, "No bean named 'nonExistentResource' available");
     }
 
     /**
@@ -140,7 +140,7 @@ public void testClosureFieldWithWrongResourceClass() {
 
                 return null;
             }
-        }, "No qualifying bean of type 'org.apache.ignite.internal.processors.resource." +
+        }, grid, NoSuchBeanDefinitionException.class, "No qualifying bean of type 'org.apache.ignite.internal.processors.resource." +
             "GridSpringResourceInjectionSelfTest$AnotherDummyResourceBean' available");
     }
 
@@ -157,7 +157,7 @@ public void testClosureFieldByResourceClassAndName() {
 
                 return null;
             }
-        }, "Either bean name or its class must be specified in @SpringResource, but not both");
+        }, grid, IgniteException.class, "Either bean name or its class must be specified in @SpringResource, but not both");
     }
 
     /**
@@ -173,7 +173,7 @@ public void testClosureFieldWithNoParams() {
 
                 return null;
             }
-        }, "Either bean name or its class must be specified in @SpringResource, but not both");
+        }, grid, IgniteException.class, "Either bean name or its class must be specified in @SpringResource, but not both");
     }
 
     /**
@@ -232,7 +232,7 @@ public void testClosureMethodWithResourceClassWithMultipleBeans() throws Excepti
             "/org/apache/ignite/internal/processors/resource/spring-resource-with-duplicate-beans.xml"));
 
         try {
-            Throwable err = assertError(new IgniteCallable<Object>() {
+            assertError(new IgniteCallable<Object>() {
                 private DummyResourceBean dummyRsrcBean;
 
                 @SpringResource(resourceClass = DummyResourceBean.class)
@@ -247,11 +247,9 @@ private void setDummyResourceBean(DummyResourceBean dummyRsrcBean) {
 
                     return null;
                 }
-            }, anotherGrid, null);
-
-            assertTrue("Unexpected message: " + err.getMessage(), err.getMessage().startsWith("No qualifying bean of type " +
+            }, anotherGrid, NoUniqueBeanDefinitionException.class, "No qualifying bean of type " +
                 "'org.apache.ignite.internal.processors.resource.GridSpringResourceInjectionSelfTest$DummyResourceBean'" +
-                " available: expected single matching bean but found 2:"));
+                " available: expected single matching bean but found 2:");
         }
         finally {
             G.stop("anotherGrid", false);
@@ -275,7 +273,7 @@ private void setDummyResourceBean(DummyResourceBean dummyRsrcBean) {
 
                 return null;
             }
-        }, "No bean named 'nonExistentResource' available");
+        }, grid, NoSuchBeanDefinitionException.class, "No bean named 'nonExistentResource' available");
     }
 
     /**
@@ -295,7 +293,7 @@ private void setDummyResourceBean(AnotherDummyResourceBean dummyRsrcBean) {
 
                 return null;
             }
-        }, "No qualifying bean of type 'org.apache.ignite.internal.processors.resource" +
+        }, grid, NoSuchBeanDefinitionException.class,"No qualifying bean of type 'org.apache.ignite.internal.processors.resource" +
             ".GridSpringResourceInjectionSelfTest$AnotherDummyResourceBean' available");
     }
 
@@ -312,7 +310,7 @@ public void testClosureMethodByResourceClassAndName() {
 
                 return null;
             }
-        }, "Either bean name or its class must be specified in @SpringResource, but not both");
+        }, grid, IgniteException.class, "Either bean name or its class must be specified in @SpringResource, but not both");
     }
 
     /**
@@ -328,32 +326,24 @@ public void testClosureMethodWithNoParams() {
 
                 return null;
             }
-        }, "Either bean name or its class must be specified in @SpringResource, but not both");
+        }, grid, IgniteException.class, "Either bean name or its class must be specified in @SpringResource, but not both");
     }
 
     /**
-     * @param job {@link IgniteCallable} to be run
-     * @param grid Node.
-     * @param expEMsg Message that {@link IgniteException} thrown from <tt>job</tt> should bear
-     * @return Thrown error.
+     * @param job {@link IgniteCallable} to be run.
+     * @param grid Node to run the job on.
+     * @param expE Expected exception type.
+     * @param expEMsg Expected exception message.
      */
     @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    private Throwable assertError(final IgniteCallable<?> job, final Ignite grid, String expEMsg) {
-        return GridTestUtils.assertThrows(log, new Callable<Object>() {
+    private void assertError(final IgniteCallable<?> job, final Ignite grid, Class<? extends Throwable> expE,
+        String expEMsg) {
+        GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
                 grid.compute(grid.cluster().forLocal()).call(job);
                 return null;
             }
-        }, IgniteException.class, expEMsg);
-    }
-
-    /**
-     * @param job {@link IgniteCallable} to be run
-     * @param expEMsg Message that {@link IgniteException} thrown from <tt>job</tt> should bear
-     */
-    @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    private void assertError(final IgniteCallable<?> job, String expEMsg) {
-        assertError(job, grid, expEMsg);
+        }, expE, expEMsg);
     }
 
     /**

From 82a4c024fe06ef8c8deeaf762f0cc20a8e481252 Mon Sep 17 00:00:00 2001
From: Roman Guseinov <gromcase@gmail.com>
Date: Mon, 9 Apr 2018 14:45:44 +0300
Subject: [PATCH 011/543] IGNITE-7944: Disconnected client node tries to send
 JOB_CANCEL message. Applied fix: - Skip sending message if client
 disconnected; - Throw IgniteCheckedException if a client node is disconnected
 and communication client is null. This closes #3737.

---
 .../processors/task/GridTaskProcessor.java    |   2 +-
 .../tcp/TcpCommunicationSpi.java              |  26 +-
 ...cpCommunicationSpiSkipMessageSendTest.java | 414 ++++++++++++++++++
 .../IgniteSpiCommunicationSelfTestSuite.java  |   3 +
 4 files changed, 442 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index d27e11680e538..2f0aa7b858940 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -171,7 +171,7 @@ public GridTaskProcessor(GridKernalContext ctx) {
         IgniteClientDisconnectedCheckedException err = disconnectedError(reconnectFut);
 
         for (GridTaskWorker<?, ?> worker : tasks.values())
-            worker.finishTask(null, err);
+            worker.finishTask(null, err, false);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index a3fccbcad5da2..4a0710e790f48 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -65,6 +65,7 @@
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
@@ -2656,6 +2657,11 @@ private void sendMessage0(ClusterNode node, Message msg, IgniteInClosure<IgniteE
         if (log.isTraceEnabled())
             log.trace("Sending message with ack to node [node=" + node + ", msg=" + msg + ']');
 
+        if (isLocalNodeDisconnected()) {
+            throw new IgniteSpiException("Failed to send a message to remote node because local node has " +
+                "been disconnected [rmtNodeId=" + node.id() + ']');
+        }
+
         ClusterNode locNode = getLocalNode();
 
         if (locNode == null)
@@ -2708,6 +2714,18 @@ private void sendMessage0(ClusterNode node, Message msg, IgniteInClosure<IgniteE
         }
     }
 
+    /**
+     * @return {@code True} if local node in disconnected state.
+     */
+    private boolean isLocalNodeDisconnected() {
+        boolean disconnected = false;
+
+        if (ignite instanceof IgniteKernal)
+            disconnected = ((IgniteKernal)ignite).context().clientDisconnected();
+
+        return disconnected;
+    }
+
     /**
      * @param nodeId Node ID.
      * @param rmvClient Client to remove.
@@ -2853,8 +2871,12 @@ private GridCommunicationClient reserveClient(ClusterNode node, int connIdx) thr
 
                 client = fut.get();
 
-                if (client == null)
-                    continue;
+                if (client == null) {
+                    if (isLocalNodeDisconnected())
+                        throw new IgniteCheckedException("Unable to create TCP client due to local node disconnecting.");
+                    else
+                        continue;
+                }
 
                 if (getSpiContext().node(nodeId) == null) {
                     if (removeNodeClient(nodeId, client))
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java
new file mode 100644
index 0000000000000..c4bc8f2922767
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpiSkipMessageSendTest.java
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.communication.tcp;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.Socket;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
+import org.apache.ignite.spi.collision.fifoqueue.FifoQueueCollisionSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Tests that the client will be segmented in time and won't hang due to canceling compute jobs.
+ */
+public class TcpCommunicationSpiSkipMessageSendTest extends GridCommonAbstractTest {
+    /** */
+    private static final CountDownLatch COMPUTE_JOB_STARTED = new CountDownLatch(1);
+
+    /** */
+    private static final long FAILURE_DETECTION_TIMEOUT = 10000;
+
+    /** */
+    private static final long JOIN_TIMEOUT = 10000;
+
+    /** */
+    private static final long START_JOB_TIMEOUT = 10000;
+
+    /** */
+    private static final long DISABLE_NETWORK_DELAY = 2000;
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 2 * 60 * 1000;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientSegmented() throws Exception {
+        Ignite server = null;
+        Ignite client = null;
+
+        try {
+            server = Ignition.start(getConfig(false));
+
+            final CountDownLatch clientDisconnected = new CountDownLatch(1);
+            final CountDownLatch clientSegmented = new CountDownLatch(1);
+
+            client = startClient(clientDisconnected, clientSegmented);
+
+            final IgniteCompute compute = client.compute();
+
+            runJobAsync(compute);
+
+            if (!COMPUTE_JOB_STARTED.await(START_JOB_TIMEOUT, TimeUnit.MILLISECONDS))
+                fail("Compute job wasn't started.");
+
+            disableNetwork(client);
+
+            if (!clientDisconnected.await(FAILURE_DETECTION_TIMEOUT * 3, TimeUnit.MILLISECONDS))
+                fail("Client wasn't disconnected.");
+
+            if (!clientSegmented.await(JOIN_TIMEOUT * 2, TimeUnit.MILLISECONDS))
+                fail("Client wasn't segmented.");
+        }
+        finally {
+            if (client != null)
+                client.close();
+
+            if (server != null)
+                server.close();
+        }
+    }
+
+    /**
+     * Simulate network disabling.
+     *
+     * @param ignite Ignite instance.
+     * @throws IgniteInterruptedCheckedException If thread sleep interrupted.
+     * @throws InterruptedException If waiting for network disabled failed (interrupted).
+     */
+    private void disableNetwork(Ignite ignite) throws IgniteInterruptedCheckedException, InterruptedException {
+        U.sleep(DISABLE_NETWORK_DELAY);
+
+        CustomCommunicationSpi communicationSpi = (CustomCommunicationSpi)ignite.configuration().getCommunicationSpi();
+
+        CustomDiscoverySpi discoverySpi = (CustomDiscoverySpi)ignite.configuration().getDiscoverySpi();
+
+        discoverySpi.disableNetwork();
+
+        communicationSpi.disableNetwork();
+
+        if (!discoverySpi.awaitNetworkDisabled(FAILURE_DETECTION_TIMEOUT * 2))
+            fail("Network wasn't disabled.");
+    }
+
+    /**
+     * Start compute jobs in the separate thread.
+     *
+     * @param compute Ignite compute instance.
+     */
+    private void runJobAsync(final IgniteCompute compute) {
+        new Thread(new Runnable() {
+            @Override public void run() {
+                try {
+                    compute.call(new IgniteCallable<Integer>() {
+                        @Override public Integer call() throws Exception {
+                            COMPUTE_JOB_STARTED.countDown();
+
+                            // Simulate long-running job.
+                            new CountDownLatch(1).await();
+
+                            return null;
+                        }
+                    });
+                }
+                catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        }).start();
+    }
+
+    /**
+     * Create Communication Spi instance.
+     *
+     * @param client Is a client node.
+     * @return Communication Spi.
+     */
+    private TcpCommunicationSpi getCommunicationSpi(boolean client) {
+        TcpCommunicationSpi spi = new CustomCommunicationSpi(client);
+
+        spi.setName("CustomCommunicationSpi");
+
+        return spi;
+    }
+
+    /**
+     * Create Discovery Spi instance.
+     *
+     * @return Discovery Spi.
+     */
+    private TcpDiscoverySpi getDiscoverySpi() {
+        TcpDiscoverySpi spi = new CustomDiscoverySpi();
+
+        spi.setName("CustomDiscoverySpi");
+
+        spi.setIpFinder(LOCAL_IP_FINDER);
+
+        return spi;
+    }
+
+    /**
+     * Create Ignite configuration.
+     *
+     * @param clientMode Client mode.
+     * @return Ignite configuration.
+     */
+    private IgniteConfiguration getConfig(boolean clientMode) {
+        IgniteConfiguration cfg = new IgniteConfiguration();
+
+        cfg.setIgniteInstanceName(clientMode ? "client-node" : "server-node");
+
+        cfg.setClientMode(clientMode);
+
+        cfg.setCommunicationSpi(getCommunicationSpi(clientMode));
+
+        if (!clientMode) {
+            cfg.setDiscoverySpi(getDiscoverySpi());
+
+            FifoQueueCollisionSpi collisionSpi = new FifoQueueCollisionSpi();
+
+            collisionSpi.setParallelJobsNumber(1);
+
+            cfg.setCollisionSpi(collisionSpi);
+        }
+        else {
+            cfg.setFailureDetectionTimeout(FAILURE_DETECTION_TIMEOUT);
+
+            cfg.setDiscoverySpi(getDiscoverySpi().setJoinTimeout(JOIN_TIMEOUT));
+        }
+
+        return cfg;
+    }
+
+    /**
+     * Start client node.
+     *
+     * @param clientDisconnected Client is disconnected.
+     * @param clientSegmented Client is segmented.
+     * @return Ignite instance.
+     */
+    private Ignite startClient(final CountDownLatch clientDisconnected, final CountDownLatch clientSegmented) {
+        Ignite ignite = Ignition.start(getConfig(true));
+
+        IgnitePredicate<Event> locLsnr = new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event event) {
+                log.info("Client node received event: " + event.name());
+
+                if (event.type() == EventType.EVT_CLIENT_NODE_DISCONNECTED)
+                    clientDisconnected.countDown();
+
+                if (event.type() == EventType.EVT_NODE_SEGMENTED)
+                    clientSegmented.countDown();
+
+                return true;
+            }
+        };
+
+        ignite.events().localListen(locLsnr,
+            EventType.EVT_NODE_SEGMENTED,
+            EventType.EVT_CLIENT_NODE_DISCONNECTED);
+
+        return ignite;
+    }
+
+    /**
+     * Communication Spi that emulates connection troubles.
+     */
+    class CustomCommunicationSpi extends TcpCommunicationSpi {
+        /** Network is disabled. */
+        private volatile boolean networkDisabled = false;
+
+        /** Additional logging is enabled. */
+        private final boolean logEnabled;
+
+        /**
+         * @param enableLogs Enable additional logging.
+         */
+        CustomCommunicationSpi(boolean enableLogs) {
+            super();
+            this.logEnabled = enableLogs;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            String message = msg.toString();
+
+            if (logEnabled)
+                log.info("CustomCommunicationSpi.sendMessage: " + message);
+
+            if (message.contains("TOPIC_JOB_CANCEL"))
+                closeTcpConnections();
+
+            super.sendMessage(node, msg, ackC);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
+            if (logEnabled)
+                log.info(String.format("CustomCommunicationSpi.createTcpClient [networkDisabled=%s, node=%s]", networkDisabled, node));
+
+            if (networkDisabled) {
+                IgniteSpiOperationTimeoutHelper timeoutHelper = new IgniteSpiOperationTimeoutHelper(this, !node.isClient());
+
+                long timeout = timeoutHelper.nextTimeoutChunk(getConnectTimeout());
+
+                if (logEnabled)
+                    log.info("CustomCommunicationSpi.createTcpClient [timeoutHelper.nextTimeoutChunk=" + timeout + "]");
+
+                sleep(timeout);
+
+                return null;
+            }
+            else
+                return super.createTcpClient(node, connIdx);
+        }
+
+        /**
+         * Simulate network disabling.
+         */
+        void disableNetwork() {
+            networkDisabled = true;
+        }
+
+        /**
+         * Close communication clients. It will lead that sendMessage method will be trying to create new ones.
+         */
+        private void closeTcpConnections() {
+            final ConcurrentMap<UUID, GridCommunicationClient[]> clients = U.field(this, "clients");
+
+            Set<UUID> ids = clients.keySet();
+
+            if (ids.size() > 0) {
+                log.info("Close TCP clients: " + ids);
+
+                for (UUID nodeId : ids) {
+                    GridCommunicationClient[] clients0 = clients.remove(nodeId);
+
+                    if (clients0 != null) {
+                        for (GridCommunicationClient client : clients0) {
+                            if (client != null)
+                                client.forceClose();
+                        }
+                    }
+                }
+
+                log.info("TCP clients are closed.");
+            }
+        }
+    }
+
+    /**
+     * Discovery Spi that emulates connection troubles.
+     */
+    class CustomDiscoverySpi extends TcpDiscoverySpi {
+        /** Network is disabled. */
+        private volatile boolean networkDisabled = false;
+
+        /** */
+        private final CountDownLatch networkDisabledLatch = new CountDownLatch(1);
+
+        /** */
+        CustomDiscoverySpi() {
+            super();
+
+            setName("CustomDiscoverySpi");
+        }
+
+        /** {@inheritDoc} */
+        @Override protected <T> T readMessage(Socket sock, @Nullable InputStream in,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (networkDisabled) {
+                sleep(timeout);
+
+                return null;
+            }
+            else
+                return super.readMessage(sock, in, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (networkDisabled) {
+                sleep(timeout);
+
+                networkDisabledLatch.countDown();
+
+                throw new IgniteCheckedException("CustomDiscoverySpi: network is disabled.");
+            }
+            else
+                super.writeToSocket(sock, msg, timeout);
+        }
+
+        /**
+         * Simulate network disabling.
+         */
+        void disableNetwork() {
+            networkDisabled = true;
+        }
+
+        /**
+         * Wait until the network is disabled.
+         */
+        boolean awaitNetworkDisabled(long timeout) throws InterruptedException {
+            return networkDisabledLatch.await(timeout, TimeUnit.MILLISECONDS);
+        }
+    }
+
+    /**
+     * Sleeps for given number of milliseconds.
+     *
+     * @param timeout Time to sleep (2000 ms by default).
+     * @throws IgniteInterruptedCheckedException If current thread interrupted.
+     */
+    static void sleep(long timeout) throws IgniteInterruptedCheckedException {
+        if (timeout > 0)
+            U.sleep(timeout);
+        else
+            U.sleep(2000);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
index 7a4de1b3feeec..1b962bc21e0c6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiCommunicationSelfTestSuite.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiDropNodesTest;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiFaultyClientTest;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiHalfOpenedConnectionTest;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpiSkipMessageSendTest;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationStatisticsTest;
 
 /**
@@ -78,6 +79,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTest(new TestSuite(GridTcpCommunicationSpiConfigSelfTest.class));
 
+        suite.addTest(new TestSuite(TcpCommunicationSpiSkipMessageSendTest.class));
+
         suite.addTest(new TestSuite(TcpCommunicationSpiFaultyClientTest.class));
         suite.addTest(new TestSuite(TcpCommunicationSpiDropNodesTest.class));
         suite.addTest(new TestSuite(TcpCommunicationSpiHalfOpenedConnectionTest.class));

From c1745de37891026e0a719f0c1d1afe768dfccbf3 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Tue, 10 Apr 2018 17:48:52 +0700
Subject: [PATCH 012/543] IGNITE-7927 Web Console: Fixed demo for
 non-collocated joins.

(cherry picked from commit 647620b)
---
 modules/web-console/backend/routes/demo.js    |  2 ++
 .../demo/service/DemoCachesLoadService.java   | 22 +++++++++++++++++--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/backend/routes/demo.js b/modules/web-console/backend/routes/demo.js
index a18fa7ac543c3..b081d0c667215 100644
--- a/modules/web-console/backend/routes/demo.js
+++ b/modules/web-console/backend/routes/demo.js
@@ -95,10 +95,12 @@ module.exports.factory = (errors, settings, mongo, spacesService) => {
 
                                             domain.space = cacheDoc.space;
                                             domain.caches.push(cacheDoc._id);
+                                            domain.clusters.push(cluster._id);
 
                                             return domain.save()
                                                 .then((domainDoc) => {
                                                     cacheDoc.domains.push(domainDoc._id);
+                                                    cluster.models.push(domainDoc._id);
 
                                                     return cacheDoc.save();
                                                 });
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
index 6691d1d3ec321..2aace06b1749f 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoCachesLoadService.java
@@ -269,6 +269,14 @@ private static CacheConfiguration cacheDepartment() {
 
         type.setFields(qryFlds);
 
+        // Indexes for DEPARTMENT.
+
+        ArrayList<QueryIndex> indexes = new ArrayList<>();
+
+        indexes.add(new QueryIndex("countryId", QueryIndexType.SORTED, false, "DEP_COUNTRY"));
+
+        type.setIndexes(indexes);
+
         ccfg.setQueryEntities(qryEntities);
 
         return ccfg;
@@ -312,6 +320,11 @@ private static CacheConfiguration cacheEmployee() {
 
         // Indexes for EMPLOYEE.
 
+        Collection<QueryIndex> indexes = new ArrayList<>();
+
+        indexes.add(new QueryIndex("departmentId", QueryIndexType.SORTED, false, "EMP_DEPARTMENT"));
+        indexes.add(new QueryIndex("managerId", QueryIndexType.SORTED, false, "EMP_MANAGER"));
+
         QueryIndex idx = new QueryIndex();
 
         idx.setName("EMP_NAMES");
@@ -323,8 +336,6 @@ private static CacheConfiguration cacheEmployee() {
 
         idx.setFields(indFlds);
 
-        Collection<QueryIndex> indexes = new ArrayList<>();
-
         indexes.add(idx);
         indexes.add(new QueryIndex("salary", QueryIndexType.SORTED, false, "EMP_SALARY"));
 
@@ -392,6 +403,13 @@ private static CacheConfiguration cacheCar() {
 
         type.setFields(qryFlds);
 
+        // Indexes for CAR.
+
+        ArrayList<QueryIndex> indexes = new ArrayList<>();
+
+        indexes.add(new QueryIndex("parkingId", QueryIndexType.SORTED, false, "CAR_PARKING"));
+        type.setIndexes(indexes);
+
         ccfg.setQueryEntities(qryEntities);
 
         return ccfg;

From b28287d1861fd841a18d0eef95eff309d21a55ef Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 10 Apr 2018 16:22:28 +0300
Subject: [PATCH 013/543] IGNITE-8025 Future must fail if assertion error has
 been thrown in the worker thread

---
 .../java/org/apache/ignite/testframework/GridTestUtils.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 4e9a7c210fe07..e6c6657578c4d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -814,7 +814,7 @@ public static IgniteInternalFuture<Long> runMultiThreadedAsync(Callable<?> call,
             catch (IgniteFutureCancelledCheckedException e) {
                 resFut.onCancelled();
             }
-            catch (IgniteCheckedException e) {
+            catch (Throwable e) {
                 resFut.onDone(e);
             }
         });

From a832f2b2e5788c45114c3cb5529d7cf53d08f9a6 Mon Sep 17 00:00:00 2001
From: Andrey Kuznetsov <stkuzma@gmail.com>
Date: Tue, 10 Apr 2018 17:30:12 +0300
Subject: [PATCH 014/543] ignite-7772 System workers critical failures handling

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../apache/ignite/internal/IgnitionEx.java    |  15 +-
 .../GridClientConnectionManagerAdapter.java   |   6 +
 .../impl/GridTcpRouterNioListenerAdapter.java |   6 +
 .../discovery/GridDiscoveryManager.java       |  16 +-
 .../GridCachePartitionExchangeManager.java    |  12 +-
 .../GridCacheSharedTtlCleanupManager.java     |  41 ++++-
 .../GridCacheDatabaseSharedManager.java       |  60 +++++--
 .../wal/FileWriteAheadLogManager.java         | 157 +++++++++++-------
 .../FsyncModeFileWriteAheadLogManager.java    |  34 +++-
 .../timeout/GridTimeoutProcessor.java         | 102 +++++++-----
 .../ignite/internal/util/StripedExecutor.java |  69 +++++---
 .../internal/util/nio/GridNioServer.java      |  43 ++++-
 .../util/nio/GridNioServerListener.java       |   6 +
 .../nio/GridNioServerListenerAdapter.java     |   6 +
 .../tcp/TcpCommunicationSpi.java              |  41 ++++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  51 +++++-
 .../internal/util/StripedExecutorTest.java    |   2 +-
 17 files changed, 501 insertions(+), 166 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 3abc7111c43a9..c0de08050fe60 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -137,6 +137,7 @@
 import static org.apache.ignite.configuration.IgniteConfiguration.DFLT_THREAD_KEEP_ALIVE_TIME;
 import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEMORY_POLICY_MAX_SIZE;
 import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.IgniteComponentType.SPRING;
 import static org.apache.ignite.plugin.segmentation.SegmentationPolicy.RESTART_JVM;
 
@@ -1806,7 +1807,13 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 cfg.getStripedPoolSize(),
                 cfg.getIgniteInstanceName(),
                 "sys",
-                log);
+                log,
+                new Thread.UncaughtExceptionHandler() {
+                    @Override public void uncaughtException(Thread thread, Throwable t) {
+                        if (grid != null)
+                            grid.context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, t));
+                    }
+                });
 
             // Note that since we use 'LinkedBlockingQueue', number of
             // maximum threads has no effect.
@@ -1846,6 +1853,12 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 cfg.getIgniteInstanceName(),
                 "data-streamer",
                 log,
+                new Thread.UncaughtExceptionHandler() {
+                    @Override public void uncaughtException(Thread thread, Throwable t) {
+                        if (grid != null)
+                            grid.context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, t));
+                    }
+                },
                 true);
 
             // Note that we do not pre-start threads here as igfs pool may not be needed.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
index 829b188a823e3..fe0453f0076e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientConnectionManagerAdapter.java
@@ -38,6 +38,7 @@
 import javax.net.ssl.SSLContext;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.client.GridClientClosedException;
 import org.apache.ignite.internal.client.GridClientConfiguration;
 import org.apache.ignite.internal.client.GridClientException;
@@ -656,6 +657,11 @@ private NioListener(Logger log) {
             }
         }
 
+        /** {@inheritDoc} */
+        @Override public void onFailure(FailureType failureType, Throwable failure) {
+            // No-op.
+        }
+
         /**
          * Handles client handshake response.
          *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
index 22f5152a2ea0c..75aa6f29ccc3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
@@ -24,6 +24,7 @@
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.client.GridClientException;
 import org.apache.ignite.internal.client.GridClientFuture;
 import org.apache.ignite.internal.client.GridClientFutureListener;
@@ -190,6 +191,11 @@ else if (msg instanceof GridClientPingPacket)
             throw new IllegalArgumentException("Unsupported input message: " + msg);
     }
 
+    /** {@inheritDoc} */
+    @Override public void onFailure(FailureType failureType, Throwable failure) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void onSessionWriteTimeout(GridNioSession ses) {
         U.warn(log, "Closing NIO session because of write timeout.");
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 4c5690e919d34..b0d32564aebf0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -147,6 +147,8 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
 import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DATA_REGIONS_OFFHEAP_SIZE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
@@ -2669,13 +2671,21 @@ void addEvent(
                     body0();
                 }
                 catch (InterruptedException e) {
+                    if (!isCancelled)
+                        ctx.failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, e));
+
                     throw e;
                 }
                 catch (Throwable t) {
-                    U.error(log, "Unexpected exception in discovery worker thread (ignored).", t);
+                    U.error(log, "Exception in discovery worker thread.", t);
+
+                    if (t instanceof Error) {
+                        FailureType type = t instanceof OutOfMemoryError ? CRITICAL_ERROR : SYSTEM_WORKER_TERMINATION;
 
-                    if (t instanceof Error)
-                        throw (Error)t;
+                        ctx.failure().process(new FailureContext(type, t));
+
+                        throw t;
+                    }
                 }
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 77ffce3bdaf06..e40493fbc589a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -123,6 +123,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
 import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE;
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
@@ -2274,11 +2275,20 @@ void dumpExchangeDebugInfo() {
             try {
                 body0();
             }
+            catch (InterruptedException | IgniteInterruptedCheckedException e) {
+                if (!stop)
+                    err = e;
+            }
             catch (Throwable e) {
                 err = e;
             }
             finally {
-                if (!stop)
+                if (err == null && !stop)
+                    err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+                if (err instanceof OutOfMemoryError)
+                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
                     cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
index 8f3d738c8eccb..613e93bc88c23 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
@@ -20,11 +20,15 @@
 import java.util.List;
 import java.util.concurrent.CopyOnWriteArrayList;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.thread.IgniteThread;
 
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
+
 /**
  * Periodically removes expired entities from caches with {@link CacheConfiguration#isEagerTtl()} flag set.
  */
@@ -122,19 +126,38 @@ private class CleanupWorker extends GridWorker {
 
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
-            while (!isCancelled()) {
-                boolean expiredRemains = false;
+            Throwable err = null;
+
+            try {
+                while (!isCancelled()) {
+                    boolean expiredRemains = false;
+
+                    for (GridCacheTtlManager mgr : mgrs) {
+                        if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT))
+                            expiredRemains = true;
 
-                for (GridCacheTtlManager mgr : mgrs) {
-                    if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT))
-                        expiredRemains = true;
+                        if (isCancelled())
+                            return;
+                    }
 
-                    if (isCancelled())
-                        return;
+                    if (!expiredRemains)
+                        U.sleep(CLEANUP_WORKER_SLEEP_INTERVAL);
                 }
+            }
+            catch (Throwable t) {
+                if (!(t instanceof IgniteInterruptedCheckedException))
+                    err = t;
 
-                if (!expiredRemains)
-                    U.sleep(CLEANUP_WORKER_SLEEP_INTERVAL);
+                throw t;
+            }
+            finally {
+                if (err == null && !isCancelled)
+                    err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+                if (err instanceof OutOfMemoryError)
+                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
             }
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 71f3baa7917bc..5beaafc585c19 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -162,6 +162,8 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID;
 
 /**
@@ -2739,32 +2741,58 @@ protected Checkpointer(@Nullable String gridName, String name, IgniteLogger log)
         }
 
         /** {@inheritDoc} */
-        @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
-            while (!isCancelled()) {
-                waitCheckpointEvent();
+        @Override protected void body() {
+            Throwable err = null;
 
-                GridFutureAdapter<Void> enableChangeApplied = GridCacheDatabaseSharedManager.this.enableChangeApplied;
+            try {
+                while (!isCancelled()) {
+                    waitCheckpointEvent();
 
-                if (enableChangeApplied != null) {
-                    enableChangeApplied.onDone();
+                    GridFutureAdapter<Void> enableChangeApplied = GridCacheDatabaseSharedManager.this.enableChangeApplied;
 
-                    GridCacheDatabaseSharedManager.this.enableChangeApplied = null;
-                }
+                    if (enableChangeApplied != null) {
+                        enableChangeApplied.onDone();
 
-                if (checkpointsEnabled)
-                    doCheckpoint();
-                else {
-                    synchronized (this) {
-                        scheduledCp.nextCpTs = U.currentTimeMillis() + checkpointFreq;
+                        GridCacheDatabaseSharedManager.this.enableChangeApplied = null;
+                    }
+
+                    if (checkpointsEnabled)
+                        doCheckpoint();
+                    else {
+                        synchronized (this) {
+                            scheduledCp.nextCpTs = U.currentTimeMillis() + checkpointFreq;
+                        }
                     }
                 }
             }
+            catch (Throwable t) {
+                err = t;
+
+                scheduledCp.cpFinishFut.onDone(t);
+
+                throw t;
+            }
+            finally {
+                if (err == null && !(stopping && isCancelled))
+                    err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+                if (err instanceof OutOfMemoryError)
+                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+            }
 
             // Final run after the cancellation.
-            if (checkpointsEnabled && !shutdownNow)
-                doCheckpoint();
+            if (checkpointsEnabled && !shutdownNow) {
+                try {
+                    doCheckpoint();
 
-            scheduledCp.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
+                    scheduledCp.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
+                }
+                catch (Throwable e) {
+                    scheduledCp.cpFinishFut.onDone(e);
+                }
+            }
         }
 
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 2fff481380c08..a40811bfb6de4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -117,6 +117,8 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
 import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.SWITCH_SEGMENT_RECORD;
 import static org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBuffer.BufferMode.DIRECT;
 import static org.apache.ignite.internal.util.IgniteUtils.findField;
@@ -682,7 +684,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
         catch (IgniteCheckedException e) {
             U.error(log, "Unable to perform segment rollover: " + e.getMessage(), e);
 
-            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+            cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
         }
     }
 
@@ -1234,7 +1236,7 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageE
         catch (IOException e) {
             StorageException se = new StorageException("Unable to initialize WAL segment", e);
 
-            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, se));
+            cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
 
             throw se;
         }
@@ -1499,6 +1501,8 @@ private synchronized boolean locked(long absIdx) {
                 }
             }
 
+            Throwable err = null;
+
             try {
                 synchronized (this) {
                     while (curAbsWalIdx == -1 && !stopped)
@@ -1560,6 +1564,18 @@ private synchronized boolean locked(long absIdx) {
             catch (InterruptedException ignore) {
                 Thread.currentThread().interrupt();
             }
+            catch (Throwable t) {
+                err = t;
+            }
+            finally {
+                if (err == null && !stopped)
+                    err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+
+                if (err instanceof OutOfMemoryError)
+                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+            }
         }
 
         /**
@@ -1884,8 +1900,6 @@ private void deleteObsoleteRawSegments() {
                 }
                 catch (IgniteCheckedException | IOException e) {
                     U.error(log, "Unexpected error during WAL compression", e);
-
-                    cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
                 }
                 catch (InterruptedException ignore) {
                     Thread.currentThread().interrupt();
@@ -2005,6 +2019,8 @@ private class FileDecompressor extends Thread {
 
         /** {@inheritDoc} */
         @Override public void run() {
+            Throwable err = null;
+
             while (!Thread.currentThread().isInterrupted() && !stopped) {
                 try {
                     long segmentToDecompress = segmentsQueue.take();
@@ -2034,10 +2050,17 @@ private class FileDecompressor extends Thread {
                 catch (InterruptedException ignore) {
                     Thread.currentThread().interrupt();
                 }
-                catch (IOException e) {
-                    U.error(log, "Unexpected error during WAL decompression", e);
+                catch (Throwable t) {
+                    err = t;
+                }
+                finally {
+                    if (err == null && !stopped)
+                        err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
 
-                    cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+                    if (err instanceof OutOfMemoryError)
+                        cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                    else if (err != null)
+                        cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
                 }
             }
         }
@@ -3146,78 +3169,94 @@ private class WALWriter extends Thread {
 
         /** {@inheritDoc} */
         @Override public void run() {
-            while (!shutdown && !Thread.currentThread().isInterrupted()) {
-                while (waiters.isEmpty()) {
-                    if (!shutdown)
-                        LockSupport.park();
-                    else {
-                        unparkWaiters(Long.MAX_VALUE);
-
-                        return;
-                    }
-                }
+            Throwable err = null;
 
-                Long pos = null;
+            try {
+                while (!shutdown && !Thread.currentThread().isInterrupted()) {
+                    while (waiters.isEmpty()) {
+                        if (!shutdown)
+                            LockSupport.park();
+                        else {
+                            unparkWaiters(Long.MAX_VALUE);
 
-                for (Long val : waiters.values()) {
-                    if (val > Long.MIN_VALUE)
-                        pos = val;
-                }
+                            return;
+                        }
+                    }
 
-                if (pos == null)
-                    continue;
-                else if (pos < UNCONDITIONAL_FLUSH) {
-                    try {
-                        assert pos == FILE_CLOSE || pos == FILE_FORCE : pos;
+                    Long pos = null;
 
-                        if (pos == FILE_CLOSE)
-                            currHnd.fileIO.close();
-                        else if (pos == FILE_FORCE)
-                            currHnd.fileIO.force();
+                    for (Long val : waiters.values()) {
+                        if (val > Long.MIN_VALUE)
+                            pos = val;
                     }
-                    catch (IOException e) {
-                        log.error("Exception in WAL writer thread: ", e);
 
-                        err = e;
+                    if (pos == null)
+                        continue;
+                    else if (pos < UNCONDITIONAL_FLUSH) {
+                        try {
+                            assert pos == FILE_CLOSE || pos == FILE_FORCE : pos;
 
-                        unparkWaiters(Long.MAX_VALUE);
+                            if (pos == FILE_CLOSE)
+                                currHnd.fileIO.close();
+                            else if (pos == FILE_FORCE)
+                                currHnd.fileIO.force();
+                        }
+                        catch (IOException e) {
+                            log.error("Exception in WAL writer thread: ", e);
 
-                        return;
-                    }
+                            err = e;
 
-                    unparkWaiters(pos);
-                }
+                            unparkWaiters(Long.MAX_VALUE);
 
-                List<SegmentedRingByteBuffer.ReadSegment> segs = currentHandle().buf.poll(pos);
+                            return;
+                        }
 
-                if (segs == null) {
-                    unparkWaiters(pos);
+                        unparkWaiters(pos);
+                    }
 
-                    continue;
-                }
+                    List<SegmentedRingByteBuffer.ReadSegment> segs = currentHandle().buf.poll(pos);
 
-                for (int i = 0; i < segs.size(); i++) {
-                    SegmentedRingByteBuffer.ReadSegment seg = segs.get(i);
+                    if (segs == null) {
+                        unparkWaiters(pos);
 
-                    try {
-                        writeBuffer(seg.position(), seg.buffer());
+                        continue;
                     }
-                    catch (Throwable e) {
-                        log.error("Exception in WAL writer thread: ", e);
 
-                        err = e;
-                    }
-                    finally {
-                        seg.release();
+                    for (int i = 0; i < segs.size(); i++) {
+                        SegmentedRingByteBuffer.ReadSegment seg = segs.get(i);
+
+                        try {
+                            writeBuffer(seg.position(), seg.buffer());
+                        }
+                        catch (Throwable e) {
+                            log.error("Exception in WAL writer thread: ", e);
+
+                            err = e;
+                        }
+                        finally {
+                            seg.release();
 
-                        long p = pos <= UNCONDITIONAL_FLUSH || err != null ? Long.MAX_VALUE : currentHandle().written;
+                            long p = pos <= UNCONDITIONAL_FLUSH || err != null ? Long.MAX_VALUE : currentHandle().written;
 
-                        unparkWaiters(p);
+                            unparkWaiters(p);
+                        }
                     }
                 }
+
+                unparkWaiters(Long.MAX_VALUE);
             }
+            catch (Throwable t) {
+                err = t;
+            }
+            finally {
+                if (err == null && !shutdown)
+                    err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
 
-            unparkWaiters(Long.MAX_VALUE);
+                if (err instanceof OutOfMemoryError)
+                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+            }
         }
 
         /**
@@ -3283,7 +3322,7 @@ void flushBuffer(long expPos) throws StorageException, IgniteCheckedException {
             Throwable err = walWriter.err;
 
             if (err != null)
-                cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, err));
+                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
 
             if (expPos == UNCONDITIONAL_FLUSH)
                 expPos = (currentHandle().buf.tail());
@@ -3372,7 +3411,7 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, Igni
             catch (IOException e) {
                 StorageException se = new StorageException("Unable to write", e);
 
-                cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, se));
+                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
 
                 throw se;
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 59196bbaf2c8e..c7d2c115ef7e4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -110,6 +110,7 @@
 import static java.nio.file.StandardOpenOption.WRITE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
 import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 
 /**
  * File WAL manager.
@@ -1338,6 +1339,8 @@ private synchronized void release(long absIdx) {
                 }
             }
 
+            Throwable err = null;
+
             try {
                 synchronized (this) {
                     while (curAbsWalIdx == -1 && !stopped)
@@ -1399,6 +1402,18 @@ private synchronized void release(long absIdx) {
             catch (InterruptedException ignore) {
                 Thread.currentThread().interrupt();
             }
+            catch (Throwable t) {
+                err = t;
+            }
+            finally {
+                if (err == null && !stopped)
+                    err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+
+                if (err instanceof OutOfMemoryError)
+                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+            }
         }
 
         /**
@@ -1721,8 +1736,6 @@ private void deleteObsoleteRawSegments() {
                 }
                 catch (IgniteCheckedException | IOException e) {
                     U.error(log, "Unexpected error during WAL compression", e);
-
-                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
                 }
                 catch (InterruptedException e) {
                     Thread.currentThread().interrupt();
@@ -1814,6 +1827,8 @@ private class FileDecompressor extends Thread {
 
         /** {@inheritDoc} */
         @Override public void run() {
+            Throwable err = null;
+
             while (!Thread.currentThread().isInterrupted() && !stopped) {
                 try {
                     long segmentToDecompress = segmentsQueue.take();
@@ -1840,13 +1855,20 @@ private class FileDecompressor extends Thread {
                         decompressionFutures.remove(segmentToDecompress).onDone();
                     }
                 }
-                catch (InterruptedException e){
+                catch (InterruptedException ignore) {
                     Thread.currentThread().interrupt();
                 }
-                catch (IOException e) {
-                    U.error(log, "Unexpected error during WAL decompression", e);
+                catch (Throwable t) {
+                    err = t;
+                }
+                finally {
+                    if (err == null && !stopped)
+                        err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
 
-                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
+                    if (err instanceof OutOfMemoryError)
+                        cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                    else if (err != null)
+                        cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
                 }
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index ff6beb417991e..a09d6faad85e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -21,6 +21,7 @@
 import java.util.Comparator;
 import java.util.Iterator;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
 import org.apache.ignite.internal.util.GridConcurrentSkipListSet;
@@ -32,6 +33,9 @@
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.thread.IgniteThread;
 
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
+
 /**
  * Detects timeout events and processes them.
  */
@@ -146,61 +150,81 @@ private class TimeoutWorker extends GridWorker {
 
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException {
-            while (!isCancelled()) {
-                long now = U.currentTimeMillis();
+            Throwable err = null;
 
-                for (Iterator<GridTimeoutObject> iter = timeoutObjs.iterator(); iter.hasNext();) {
-                    GridTimeoutObject timeoutObj = iter.next();
+            try {
+                while (!isCancelled()) {
+                    long now = U.currentTimeMillis();
 
-                    if (timeoutObj.endTime() <= now) {
-                        try {
-                            boolean rmvd = timeoutObjs.remove(timeoutObj);
+                    for (Iterator<GridTimeoutObject> iter = timeoutObjs.iterator(); iter.hasNext(); ) {
+                        GridTimeoutObject timeoutObj = iter.next();
 
-                            if (log.isDebugEnabled())
-                                log.debug("Timeout has occurred [obj=" + timeoutObj + ", process=" + rmvd + ']');
+                        if (timeoutObj.endTime() <= now) {
+                            try {
+                                boolean rmvd = timeoutObjs.remove(timeoutObj);
 
-                            if (rmvd)
-                                timeoutObj.onTimeout();
-                        }
-                        catch (Throwable e) {
-                            if (isCancelled() && !(e instanceof Error)){
                                 if (log.isDebugEnabled())
-                                    log.debug("Error when executing timeout callback: " + timeoutObj);
+                                    log.debug("Timeout has occurred [obj=" + timeoutObj + ", process=" + rmvd + ']');
 
-                                return;
+                                if (rmvd)
+                                    timeoutObj.onTimeout();
                             }
+                            catch (Throwable e) {
+                                if (isCancelled() && !(e instanceof Error)) {
+                                    if (log.isDebugEnabled())
+                                        log.debug("Error when executing timeout callback: " + timeoutObj);
 
-                            U.error(log, "Error when executing timeout callback: " + timeoutObj, e);
+                                    return;
+                                }
 
-                            if (e instanceof Error)
-                                throw e;
+                                U.error(log, "Error when executing timeout callback: " + timeoutObj, e);
+
+                                if (e instanceof Error)
+                                    throw e;
+                            }
                         }
+                        else
+                            break;
                     }
-                    else
-                        break;
-                }
-
-                synchronized (mux) {
-                    while (!isCancelled()) {
-                        // Access of the first element must be inside of
-                        // synchronization block, so we don't miss out
-                        // on thread notification events sent from
-                        // 'addTimeoutObject(..)' method.
-                        GridTimeoutObject first = timeoutObjs.firstx();
-
-                        if (first != null) {
-                            long waitTime = first.endTime() - U.currentTimeMillis();
 
-                            if (waitTime > 0)
-                                mux.wait(waitTime);
+                    synchronized (mux) {
+                        while (!isCancelled()) {
+                            // Access of the first element must be inside of
+                            // synchronization block, so we don't miss out
+                            // on thread notification events sent from
+                            // 'addTimeoutObject(..)' method.
+                            GridTimeoutObject first = timeoutObjs.firstx();
+
+                            if (first != null) {
+                                long waitTime = first.endTime() - U.currentTimeMillis();
+
+                                if (waitTime > 0)
+                                    mux.wait(waitTime);
+                                else
+                                    break;
+                            }
                             else
-                                break;
+                                mux.wait(5000);
                         }
-                        else
-                            mux.wait(5000);
                     }
                 }
             }
+            catch (Throwable t) {
+                if (!(t instanceof InterruptedException))
+                    err = t;
+
+                throw t;
+            }
+            finally {
+                if (err == null && !isCancelled)
+                    err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly.");
+
+                if (err instanceof OutOfMemoryError)
+                    ctx.failure().process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    ctx.failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+            }
+
         }
     }
 
@@ -284,4 +308,4 @@ public class CancelableTask implements GridTimeoutObject, Closeable {
             return S.toString(CancelableTask.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
index 630d34c9414c8..c6383ee41319e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
@@ -64,9 +64,11 @@ public class StripedExecutor implements ExecutorService {
      * @param igniteInstanceName Node name.
      * @param poolName Pool name.
      * @param log Logger.
+     * @param errHnd Exception handler.
      */
-    public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log) {
-        this(cnt, igniteInstanceName, poolName, log, false);
+    public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log,
+        Thread.UncaughtExceptionHandler errHnd) {
+        this(cnt, igniteInstanceName, poolName, log, errHnd, false);
     }
 
     /**
@@ -74,9 +76,11 @@ public StripedExecutor(int cnt, String igniteInstanceName, String poolName, fina
      * @param igniteInstanceName Node name.
      * @param poolName Pool name.
      * @param log Logger.
+     * @param errHnd Exception handler.
      * @param stealTasks {@code True} to steal tasks.
      */
-    public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log, boolean stealTasks) {
+    public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log,
+        Thread.UncaughtExceptionHandler errHnd, boolean stealTasks) {
         A.ensure(cnt > 0, "cnt > 0");
 
         boolean success = false;
@@ -91,15 +95,9 @@ public StripedExecutor(int cnt, String igniteInstanceName, String poolName, fina
 
         try {
             for (int i = 0; i < cnt; i++) {
-                stripes[i] = stealTasks ? new StripeConcurrentQueue(
-                    igniteInstanceName,
-                    poolName,
-                    i,
-                    log, stripes) : new StripeConcurrentQueue(
-                        igniteInstanceName,
-                        poolName,
-                        i,
-                        log);
+                stripes[i] = stealTasks
+                    ? new StripeConcurrentQueue(igniteInstanceName, poolName, i, log, stripes, errHnd)
+                    : new StripeConcurrentQueue(igniteInstanceName, poolName, i, log, errHnd);
             }
 
             for (int i = 0; i < cnt; i++)
@@ -434,22 +432,28 @@ private static abstract class Stripe implements Runnable {
         /** Thread executing the loop. */
         protected Thread thread;
 
+        /** Exception handler. */
+        private Thread.UncaughtExceptionHandler errHnd;
+
         /**
          * @param igniteInstanceName Ignite instance name.
          * @param poolName Pool name.
          * @param idx Stripe index.
          * @param log Logger.
+         * @param errHnd Exception handler.
          */
         public Stripe(
             String igniteInstanceName,
             String poolName,
             int idx,
-            IgniteLogger log
+            IgniteLogger log,
+            Thread.UncaughtExceptionHandler errHnd
         ) {
             this.igniteInstanceName = igniteInstanceName;
             this.poolName = poolName;
             this.idx = idx;
             this.log = log;
+            this.errHnd = errHnd;
         }
 
         /**
@@ -463,6 +467,8 @@ void start() {
                 idx,
                 GridIoPolicy.UNDEFINED);
 
+            thread.setUncaughtExceptionHandler(errHnd);
+
             thread.start();
         }
 
@@ -518,9 +524,19 @@ void awaitStop() {
                     return;
                 }
                 catch (Throwable e) {
+                    if (e instanceof OutOfMemoryError) {
+                        // Re-throwing to exploit uncaught exception handler.
+                        throw e;
+                    }
+
                     U.error(log, "Failed to execute runnable.", e);
                 }
             }
+
+            if (!stopping) {
+                throw new IllegalStateException("Thread " + Thread.currentThread().getName() +
+                    " is terminated unexpectedly");
+            }
         }
 
         /**
@@ -576,14 +592,16 @@ private static class StripeConcurrentQueue extends Stripe {
          * @param poolName Pool name.
          * @param idx Stripe index.
          * @param log Logger.
+         * @param errHnd Exception handler.
          */
         StripeConcurrentQueue(
             String igniteInstanceName,
             String poolName,
             int idx,
-            IgniteLogger log
+            IgniteLogger log,
+            Thread.UncaughtExceptionHandler errHnd
         ) {
-            this(igniteInstanceName, poolName, idx, log, null);
+            this(igniteInstanceName, poolName, idx, log, null, errHnd);
         }
 
         /**
@@ -591,19 +609,22 @@ private static class StripeConcurrentQueue extends Stripe {
          * @param poolName Pool name.
          * @param idx Stripe index.
          * @param log Logger.
+         * @param errHnd Exception handler.
          */
         StripeConcurrentQueue(
             String igniteInstanceName,
             String poolName,
             int idx,
             IgniteLogger log,
-            Stripe[] others
+            Stripe[] others,
+            Thread.UncaughtExceptionHandler errHnd
         ) {
             super(
                 igniteInstanceName,
                 poolName,
                 idx,
-                log);
+                log,
+                errHnd);
 
             this.others = others;
 
@@ -702,17 +723,20 @@ private static class StripeConcurrentQueueNoPark extends Stripe {
          * @param poolName Pool name.
          * @param idx Stripe index.
          * @param log Logger.
+         * @param errHnd Exception handler.
          */
         public StripeConcurrentQueueNoPark(
             String igniteInstanceName,
             String poolName,
             int idx,
-            IgniteLogger log
+            IgniteLogger log,
+            Thread.UncaughtExceptionHandler errHnd
         ) {
             super(igniteInstanceName,
                 poolName,
                 idx,
-                log);
+                log,
+                errHnd);
         }
 
         /** {@inheritDoc} */
@@ -758,17 +782,20 @@ private static class StripeConcurrentBlockingQueue extends Stripe {
          * @param poolName Pool name.
          * @param idx Stripe index.
          * @param log Logger.
+         * @param errHnd Exception handler.
          */
         public StripeConcurrentBlockingQueue(
             String igniteInstanceName,
             String poolName,
             int idx,
-            IgniteLogger log
+            IgniteLogger log,
+            Thread.UncaughtExceptionHandler errHnd
         ) {
             super(igniteInstanceName,
                 poolName,
                 idx,
-                log);
+                log,
+                errHnd);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 0fcde0e136644..3597a050051f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -77,6 +77,8 @@
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.MSG_WRITER;
 import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.NIO_OPERATION;
 
@@ -1749,6 +1751,8 @@ protected AbstractNioClientWorker(int idx, @Nullable String igniteInstanceName,
 
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+            Throwable err = null;
+
             try {
                 boolean reset = false;
 
@@ -1774,9 +1778,24 @@ protected AbstractNioClientWorker(int idx, @Nullable String igniteInstanceName,
             catch (Throwable e) {
                 U.error(log, "Caught unhandled exception in NIO worker thread (restart the node).", e);
 
+                err = e;
+
                 if (e instanceof Error)
                     throw e;
             }
+            finally {
+                if (err instanceof OutOfMemoryError)
+                    lsnr.onFailure(CRITICAL_ERROR, err);
+                else if (!closed) {
+                    if (err == null)
+                        lsnr.onFailure(SYSTEM_WORKER_TERMINATION,
+                            new IllegalStateException("Thread " + name() + " is terminated unexpectedly"));
+                    else if (err instanceof InterruptedException)
+                        lsnr.onFailure(SYSTEM_WORKER_TERMINATION, err);
+                }
+                else if (err != null)
+                    lsnr.onFailure(SYSTEM_WORKER_TERMINATION, err);
+            }
         }
 
         /**
@@ -2790,6 +2809,8 @@ protected GridNioAcceptWorker(
 
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+            Throwable err = null;
+
             try {
                 boolean reset = false;
 
@@ -2812,8 +2833,28 @@ protected GridNioAcceptWorker(
                     }
                 }
             }
+            catch (Throwable t) {
+                if (!(t instanceof IgniteInterruptedCheckedException))
+                    err = t;
+
+                throw t;
+            }
             finally {
-                closeSelector(); // Safety.
+                try {
+                    closeSelector(); // Safety.
+                }
+                catch (RuntimeException ignore) {
+                    // No-op.
+                }
+
+                if (err == null && !closed)
+                    err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
+
+                if (err instanceof OutOfMemoryError)
+                    lsnr.onFailure(CRITICAL_ERROR, err);
+                else if (err != null)
+                    lsnr.onFailure(SYSTEM_WORKER_TERMINATION, err);
+
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
index db28792347235..14c5a748a8a81 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListener.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.util.nio;
 
+import org.apache.ignite.failure.FailureType;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -69,4 +70,9 @@ public interface GridNioServerListener<T> {
      * @param ses Session that is idle.
      */
     public void onSessionIdleTimeout(GridNioSession ses);
+
+    /**
+     * Called when critical failure occurs in server implementation.
+     */
+    public void onFailure(FailureType failureType, Throwable failure);
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
index 5d222c1915886..b6b20b2854da2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServerListenerAdapter.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.util.nio;
 
+import org.apache.ignite.failure.FailureType;
+
 /**
  * Server listener adapter providing empty methods implementation for rarely used methods.
  */
@@ -35,4 +37,8 @@ public abstract class GridNioServerListenerAdapter<T> implements GridNioServerLi
     @Override public void onMessageSent(GridNioSession ses, T msg) {
         // No-op.
     }
+
+    @Override public void onFailure(FailureType failureType, Throwable failure) {
+        // No-op.
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 4a0710e790f48..9e7b59235db80 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -62,7 +62,10 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
@@ -151,6 +154,8 @@
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.util.nio.GridNioSessionMetaKey.SSL_META;
 import static org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture.SES_FUT_META;
 import static org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage.ALREADY_CONNECTED;
@@ -798,6 +803,11 @@ else if (connKey.dummy()) {
                 }
             }
 
+            /** {@inheritDoc} */
+            @Override public void onFailure(FailureType failureType, Throwable failure) {
+                ((IgniteEx)ignite).context().failure().process(new FailureContext(failureType, failure));
+            }
+
             /**
              * @param recovery Recovery descriptor.
              * @param ses Session.
@@ -4190,13 +4200,32 @@ private CommunicationWorker(String igniteInstanceName) {
             if (log.isDebugEnabled())
                 log.debug("Tcp communication worker has been started.");
 
-            while (!isInterrupted()) {
-                DisconnectedSessionInfo disconnectData = q.poll(idleConnTimeout, TimeUnit.MILLISECONDS);
+            Throwable err = null;
 
-                if (disconnectData != null)
-                    processDisconnect(disconnectData);
-                else
-                    processIdle();
+            try {
+                while (!isInterrupted()) {
+                    DisconnectedSessionInfo disconnectData = q.poll(idleConnTimeout, TimeUnit.MILLISECONDS);
+
+                    if (disconnectData != null)
+                        processDisconnect(disconnectData);
+                    else
+                        processIdle();
+                }
+            }
+            catch (Throwable t) {
+                if (!(t instanceof InterruptedException))
+                    err = t;
+
+                throw t;
+            }
+            finally {
+                if (err == null && !stopping)
+                    err = new IllegalStateException("Thread  " + getName() + " is terminated unexpectedly.");
+
+                if (err instanceof OutOfMemoryError)
+                    ((IgniteEx)ignite).context().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    ((IgniteEx)ignite).context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 4aa13163e1bf7..7bf37e1b0ac43 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -50,6 +50,7 @@
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.LinkedBlockingDeque;
@@ -66,6 +67,8 @@
 import org.apache.ignite.cache.CacheMetrics;
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
@@ -73,6 +76,7 @@
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
 import org.apache.ignite.internal.managers.discovery.DiscoveryServerOnlyCustomMessage;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.processors.security.SecurityContext;
 import org.apache.ignite.internal.processors.security.SecurityUtils;
 import org.apache.ignite.internal.util.GridBoundedLinkedHashSet;
@@ -137,7 +141,6 @@
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryStatusCheckMessage;
 import org.apache.ignite.thread.IgniteThreadPoolExecutor;
 import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCOVERY_CLIENT_RECONNECT_HISTORY_SIZE;
@@ -149,6 +152,8 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
 import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
+import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_LATE_AFFINITY_ASSIGNMENT;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_COMPACT_FOOTER;
@@ -2609,12 +2614,20 @@ void addMessage(TcpDiscoveryAbstractMessage msg) {
 
         /** {@inheritDoc} */
         @Override protected void body() throws InterruptedException {
+            Throwable err = null;
+
             try {
                 super.body();
             }
+            catch (InterruptedException e) {
+                if (!spi.isNodeStopping0())
+                    err = e;
+
+                throw e;
+            }
             catch (Throwable e) {
                 if (!spi.isNodeStopping0() && spiStateCopy() != DISCONNECTING) {
-                        final Ignite ignite = spi.ignite();
+                    final Ignite ignite = spi.ignite();
 
                     if (ignite != null) {
                         U.error(log, "TcpDiscoverSpi's message worker thread failed abnormally. " +
@@ -2637,9 +2650,22 @@ void addMessage(TcpDiscoveryAbstractMessage msg) {
                     }
                 }
 
+                err = e;
+
                 // Must be processed by IgniteSpiThread as well.
                 throw e;
             }
+            finally {
+                if (err == null && !spi.isNodeStopping0())
+                    err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
+
+                FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
+
+                if (err instanceof OutOfMemoryError)
+                    failure.process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    failure.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+            }
         }
 
         /**
@@ -5597,7 +5623,9 @@ private class TcpServer extends IgniteSpiThread {
         }
 
         /** {@inheritDoc} */
-        @Override protected void body() throws InterruptedException {
+        @Override protected void body() {
+            Throwable err = null;
+
             try {
                 while (!isInterrupted()) {
                     Socket sock = srvrSock.accept();
@@ -5630,13 +5658,30 @@ private class TcpServer extends IgniteSpiThread {
                 onException("Failed to accept TCP connection.", e);
 
                 if (!isInterrupted()) {
+                    err = e;
+
                     if (U.isMacInvalidArgumentError(e))
                         U.error(log, "Failed to accept TCP connection\n\t" + U.MAC_INVALID_ARG_MSG, e);
                     else
                         U.error(log, "Failed to accept TCP connection.", e);
                 }
             }
+            catch (Throwable t) {
+                err = t;
+
+                throw t;
+            }
             finally {
+                if (err == null && !spi.isNodeStopping0())
+                    err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
+
+                FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
+
+                if (err instanceof OutOfMemoryError)
+                    failure.process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    failure.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+
                 U.closeQuiet(srvrSock);
             }
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
index 543907fcde60a..3fca7afed7ff5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
@@ -29,7 +29,7 @@ public class StripedExecutorTest extends GridCommonAbstractTest {
 
     /** {@inheritDoc} */
     @Override public void beforeTest() {
-        stripedExecSvc = new StripedExecutor(3, "foo name", "pool name", new JavaLogger());
+        stripedExecSvc = new StripedExecutor(3, "foo name", "pool name", new JavaLogger(), (thread, t) -> {});
     }
 
     /** {@inheritDoc} */

From 912433ba9aa113508d05930691b251eccd8f5870 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Tue, 10 Apr 2018 18:54:03 +0300
Subject: [PATCH 015/543] IGNITE-8069 IgniteOutOfMemoryException should be
 handled accordingly to provided failure handler

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../pagemem/impl/PageMemoryNoStoreImpl.java   |  17 ++-
 .../persistence/pagemem/PageMemoryImpl.java   |  17 ++-
 .../failure/AbstractFailureHandlerTest.java   |  74 +++++++++
 .../failure/IoomFailureHandlerTest.java       | 144 ++++++++++++++++++
 .../pagemem/PageMemoryImplTest.java           |   9 ++
 .../testsuites/IgniteBasicTestSuite.java      |   2 +
 6 files changed, 259 insertions(+), 4 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index 7424af664a313..d4b22a6a2b1cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -28,6 +28,8 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.DirectMemoryRegion;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
@@ -158,6 +160,9 @@ public class PageMemoryNoStoreImpl implements PageMemory {
     /** */
     private final boolean trackAcquiredPages;
 
+    /** Shared context. */
+    private final GridCacheSharedContext<?, ?> ctx;
+
     /**
      * @param log Logger.
      * @param directMemoryProvider Memory allocator to use.
@@ -184,6 +189,7 @@ public PageMemoryNoStoreImpl(
         this.trackAcquiredPages = trackAcquiredPages;
         this.memMetrics = memMetrics;
         this.dataRegionCfg = dataRegionCfg;
+        this.ctx = sharedCtx;
 
         sysPageSize = pageSize + PAGE_OVERHEAD;
 
@@ -288,8 +294,8 @@ public PageMemoryNoStoreImpl(
             }
         }
 
-        if (relPtr == INVALID_REL_PTR)
-            throw new IgniteOutOfMemoryException("Out of memory in data region [" +
+        if (relPtr == INVALID_REL_PTR) {
+            IgniteOutOfMemoryException oom = new IgniteOutOfMemoryException("Out of memory in data region [" +
                 "name=" + dataRegionCfg.getName() +
                 ", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), false) +
                 ", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), false) +
@@ -299,6 +305,13 @@ public PageMemoryNoStoreImpl(
                 "  ^-- Enable eviction or expiration policies"
             );
 
+            if (ctx != null)
+                ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom));
+
+            throw oom;
+        }
+
+
         assert (relPtr & ~PageIdUtils.PAGE_IDX_MASK) == 0 : U.hexLong(relPtr & ~PageIdUtils.PAGE_IDX_MASK);
 
         // Assign page ID according to flags and partition ID.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 46fb7ddbb37c7..4463224b64686 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -40,6 +40,8 @@
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.DirectMemoryRegion;
@@ -543,7 +545,7 @@ else if (throttlingPlc == ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY)
         catch (IgniteOutOfMemoryException oom) {
             DataRegionConfiguration dataRegionCfg = getDataRegionConfiguration();
 
-            throw (IgniteOutOfMemoryException) new IgniteOutOfMemoryException("Out of memory in data region [" +
+            IgniteOutOfMemoryException e = new IgniteOutOfMemoryException("Out of memory in data region [" +
                 "name=" + dataRegionCfg.getName() +
                 ", initSize=" + U.readableSize(dataRegionCfg.getInitialSize(), false) +
                 ", maxSize=" + U.readableSize(dataRegionCfg.getMaxSize(), false) +
@@ -551,7 +553,13 @@ else if (throttlingPlc == ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY)
                 "  ^-- Increase maximum off-heap memory size (DataRegionConfiguration.maxSize)" + U.nl() +
                 "  ^-- Enable Ignite persistence (DataRegionConfiguration.persistenceEnabled)" + U.nl() +
                 "  ^-- Enable eviction or expiration policies"
-            ).initCause(oom);
+            );
+
+            e.initCause(oom);
+
+            ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+            throw e;
         }
         finally {
             seg.writeLock().unlock();
@@ -746,6 +754,11 @@ else if (relPtr == OUTDATED_REL_PTR) {
 
             return absPtr;
         }
+        catch (IgniteOutOfMemoryException oom) {
+            ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, oom));
+
+            throw oom;
+        }
         finally {
             seg.writeLock().unlock();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
new file mode 100644
index 0000000000000..dc5f1f5e9caeb
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/AbstractFailureHandlerTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Abstract failure handler test.
+ */
+public class AbstractFailureHandlerTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected FailureHandler getFailureHandler(String igniteInstanceName) {
+        return new DummyFailureHandler();
+    }
+
+    /**
+     * Gets dummy failure handler for ignite instance.
+     *
+     * @param ignite Ignite.
+     */
+    protected static DummyFailureHandler dummyFailureHandler(Ignite ignite) {
+        return (DummyFailureHandler)ignite.configuration().getFailureHandler();
+    }
+
+    /**
+     *
+     */
+    protected static class DummyFailureHandler implements FailureHandler {
+        /** Failure. */
+        private volatile boolean failure;
+
+        /** Failure context. */
+        private volatile FailureContext ctx;
+
+        /** {@inheritDoc} */
+        @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+            failure = true;
+
+            ctx = failureCtx;
+
+            return true;
+        }
+
+        /**
+         * @return Failure.
+         */
+        public boolean failure() {
+            return failure;
+        }
+
+        /**
+         * @return Failure context.
+         */
+        public FailureContext failureContext() {
+            return ctx;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
new file mode 100644
index 0000000000000..a777f815a89e2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/IoomFailureHandlerTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.transactions.Transaction;
+
+/**
+ * IgniteOutOfMemoryError failure handler test.
+ */
+public class IoomFailureHandlerTest extends AbstractFailureHandlerTest {
+    /** Offheap size for memory policy. */
+    private static final int SIZE = 10 * 1024 * 1024;
+
+    /** Page size. */
+    static final int PAGE_SIZE = 2048;
+
+    /** Number of entries. */
+    static final int ENTRIES = 10_000;
+
+    /** PDS enabled. */
+    private boolean pds;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+
+        DataRegionConfiguration dfltPlcCfg = new DataRegionConfiguration();
+
+        dfltPlcCfg.setName("dfltPlc");
+        dfltPlcCfg.setInitialSize(SIZE);
+        dfltPlcCfg.setMaxSize(SIZE);
+
+        if (pds)
+            dfltPlcCfg.setPersistenceEnabled(true);
+
+        dsCfg.setDefaultDataRegionConfiguration(dfltPlcCfg);
+        dsCfg.setPageSize(PAGE_SIZE);
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        CacheConfiguration<?, ?> ccfg = new CacheConfiguration<>()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setBackups(0)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Test IgniteOutOfMemoryException handling with no store.
+     */
+    public void testIoomErrorNoStoreHandling() throws Exception {
+        testIoomErrorHandling(false);
+    }
+
+    /**
+     * Test IgniteOutOfMemoryException handling with PDS.
+     */
+    public void testIoomErrorPdsHandling() throws Exception {
+        testIoomErrorHandling(true);
+    }
+
+    /**
+     * Test IOOME handling.
+     */
+    public void testIoomErrorHandling(boolean pds) throws Exception {
+        this.pds = pds;
+
+        IgniteEx ignite0 = startGrid(0);
+        IgniteEx ignite1 = startGrid(1);
+
+        try {
+            if (pds)
+                ignite0.cluster().active(true);
+
+            IgniteCache<Integer, Object> cache0 = ignite0.getOrCreateCache(DEFAULT_CACHE_NAME);
+            IgniteCache<Integer, Object> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+            awaitPartitionMapExchange();
+
+            try (Transaction tx = ignite0.transactions().txStart()) {
+                for (Integer i : primaryKeys(cache1, ENTRIES))
+                    cache0.put(i, new byte[PAGE_SIZE / 3 * 2]);
+
+                tx.commit();
+            }
+            catch (Throwable ignore) {
+                // Expected.
+            }
+
+            assertFalse(dummyFailureHandler(ignite0).failure());
+            assertTrue(dummyFailureHandler(ignite1).failure());
+            assertTrue(X.hasCause(dummyFailureHandler(ignite1).failureContext().error(), IgniteOutOfMemoryException.class));
+        }
+        finally {
+            stopGrid(1);
+            stopGrid(0);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
index 31af1181cdfe7..3697c4cc7b499 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.NoOpFailureHandler;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
@@ -41,6 +42,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor;
 import org.apache.ignite.internal.util.lang.GridInClosure3X;
 import org.apache.ignite.plugin.PluginProvider;
@@ -268,10 +270,17 @@ private PageMemoryImpl createPageMemory(PageMemoryImpl.ThrottlingPolicy throttli
 
         IgniteConfiguration igniteCfg = new IgniteConfiguration();
         igniteCfg.setDataStorageConfiguration(new DataStorageConfiguration());
+        igniteCfg.setFailureHandler(new NoOpFailureHandler());
 
         GridTestKernalContext kernalCtx = new GridTestKernalContext(new GridTestLog4jLogger(), igniteCfg);
         kernalCtx.add(new IgnitePluginProcessor(kernalCtx, igniteCfg, Collections.<PluginProvider>emptyList()));
 
+        FailureProcessor failureProc = new FailureProcessor(kernalCtx);
+
+        failureProc.start();
+
+        kernalCtx.add(failureProc);
+
         GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
             kernalCtx,
             null,
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index dd9cdfdbcb835..c4b7d9227f4ee 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -21,6 +21,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.GridSuppressedExceptionSelfTest;
 import org.apache.ignite.failure.FailureHandlerTriggeredTest;
+import org.apache.ignite.failure.IoomFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest;
 import org.apache.ignite.internal.ClassSetTest;
@@ -197,6 +198,7 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(FailureHandlerTriggeredTest.class);
         suite.addTestSuite(StopNodeFailureHandlerTest.class);
         suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
+        suite.addTestSuite(IoomFailureHandlerTest.class);
 
         return suite;
     }

From 99feab6ace66d011b677fd4d57b44fc54da8fd4f Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 10 Apr 2018 20:33:47 +0300
Subject: [PATCH 016/543] IGNITE-6430 Complete failing test early

---
 .../CacheGroupsMetricsRebalanceTest.java      | 31 +++++++++++--------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
index 89c82365c1905..ceb98522d5a25 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
@@ -227,29 +227,34 @@ public void testRebalanceEstimateFinishTime() throws Exception {
 
                 System.out.println("Wait until keys left will be less " + keysLine);
 
-                while (finishRebalanceLatch.getCount() != 0) {
-                    CacheMetrics m = ig2.cache(CACHE1).localMetrics();
+                try {
+                    while (finishRebalanceLatch.getCount() != 0) {
+                        CacheMetrics m = ig2.cache(CACHE1).localMetrics();
 
-                    long keyLeft = m.getKeysToRebalanceLeft();
+                        long keyLeft = m.getKeysToRebalanceLeft();
 
-                    if (keyLeft > 0 && keyLeft < keysLine)
-                        latch.countDown();
+                        if (keyLeft > 0 && keyLeft < keysLine)
+                            latch.countDown();
 
-                    System.out.println("Keys left: " + m.getKeysToRebalanceLeft());
+                        System.out.println("Keys left: " + m.getKeysToRebalanceLeft());
 
-                    try {
-                        Thread.sleep(1_000);
-                    }
-                    catch (InterruptedException e) {
-                        System.out.println("Interrupt thread: " + e.getMessage());
+                        try {
+                            Thread.sleep(1_000);
+                        }
+                        catch (InterruptedException e) {
+                            System.out.println("Interrupt thread: " + e.getMessage());
 
-                        Thread.currentThread().interrupt();
+                            Thread.currentThread().interrupt();
+                        }
                     }
                 }
+                finally {
+                    latch.countDown();
+                }
             }
         });
 
-        latch.await();
+        assertTrue(latch.await(getTestTimeout(), TimeUnit.MILLISECONDS));
 
         long finishTime = ig2.cache(CACHE1).localMetrics().getEstimatedRebalancingFinishTime();
 

From 526fb0ee612ef71fde58a1274db35e8205304a63 Mon Sep 17 00:00:00 2001
From: Dmitriy Sorokin <d.w.sorokin@gmail.com>
Date: Tue, 10 Apr 2018 22:20:41 +0300
Subject: [PATCH 017/543] IGNITE-8101 Ability to terminate system workers by
 JMX for test purposes.

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../apache/ignite/IgniteSystemProperties.java |  7 ++
 .../failure/StopNodeOrHaltFailureHandler.java |  2 +-
 .../ignite/internal/GridKernalContext.java    |  8 ++
 .../internal/GridKernalContextImpl.java       | 10 +++
 .../apache/ignite/internal/IgniteKernal.java  | 16 +++-
 .../discovery/GridDiscoveryManager.java       |  2 +-
 .../GridCachePartitionExchangeManager.java    |  3 +-
 .../GridCacheSharedTtlCleanupManager.java     |  3 +-
 .../reader/StandaloneGridKernalContext.java   |  6 ++
 .../timeout/GridTimeoutProcessor.java         |  3 +-
 .../ignite/internal/util/IgniteUtils.java     |  7 +-
 .../worker/WorkersControlMXBeanImpl.java      | 62 ++++++++++++++
 .../internal/worker/WorkersRegistry.java      | 80 +++++++++++++++++++
 .../ignite/internal/worker/package-info.java  | 22 +++++
 .../ignite/mxbean/WorkersControlMXBean.java   | 49 ++++++++++++
 15 files changed, 271 insertions(+), 9 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 152d845afbac5..9da123e01fb35 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -422,6 +422,13 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_MBEANS_DISABLED = "IGNITE_MBEANS_DISABLED";
 
+    /**
+     * If property is set to {@code true}, then test features will be enabled.
+     *
+     * Default is {@code false}.
+     */
+    public static final String IGNITE_TEST_FEATURES_ENABLED = "IGNITE_TEST_FEATURES_ENABLED";
+
     /**
      * Property controlling size of buffer holding last exception. Default value of {@code 1000}.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
index 4f7440616c097..3ce4ff6f27ef6 100644
--- a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
@@ -92,7 +92,7 @@ public StopNodeOrHaltFailureHandler(boolean tryStop, long timeout) {
             ).start();
         }
         else {
-            U.error(log, "JVM will be halted immediately on ignite failure: [failureCtx=" + failureCtx + ']');
+            U.error(log, "JVM will be halted immediately due to the failure: [failureCtx=" + failureCtx + ']');
 
             Runtime.getRuntime().halt(Ignition.KILL_EXIT_CODE);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 0b40054ec2606..505c3d6a4b0b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.internal.managers.failover.GridFailoverManager;
 import org.apache.ignite.internal.managers.indexing.GridIndexingManager;
 import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
+import org.apache.ignite.internal.worker.WorkersRegistry;
 import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
 import org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor;
 import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
@@ -422,6 +423,13 @@ public interface GridKernalContext extends Iterable<GridComponent> {
      */
     public GridIndexingManager indexing();
 
+    /**
+     * Gets workers registry.
+     *
+     * @return Workers registry.
+     */
+    public WorkersRegistry workersRegistry();
+
     /**
      * Gets data structures processor.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 34083340bfefe..ac4970859d2b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -46,6 +46,7 @@
 import org.apache.ignite.internal.managers.failover.GridFailoverManager;
 import org.apache.ignite.internal.managers.indexing.GridIndexingManager;
 import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
+import org.apache.ignite.internal.worker.WorkersRegistry;
 import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
 import org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor;
 import org.apache.ignite.internal.processors.cache.CacheConflictResolutionManager;
@@ -360,6 +361,10 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     @GridToStringExclude
     private Map<String, Object> attrs = new HashMap<>();
 
+    /** */
+    @GridToStringExclude
+    private final WorkersRegistry workersRegistry = new WorkersRegistry();
+
     /** */
     private IgniteEx grid;
 
@@ -779,6 +784,11 @@ else if (helper instanceof HadoopHelper)
         return indexingMgr;
     }
 
+    /** {@inheritDoc} */
+    @Override public WorkersRegistry workersRegistry() {
+        return workersRegistry;
+    }
+
     /** {@inheritDoc} */
     @Override public GridAffinityProcessor affinity() {
         return affProc;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 0b102e5b665f7..1cb07b91e82b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -125,7 +125,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
 import org.apache.ignite.internal.processors.cache.GridCacheUtilityKey;
-import org.apache.ignite.internal.processors.cache.GridCacheUtils;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
@@ -185,6 +184,8 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.worker.WorkersControlMXBeanImpl;
+import org.apache.ignite.internal.worker.WorkersRegistry;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -197,6 +198,7 @@
 import org.apache.ignite.mxbean.ClusterMetricsMXBean;
 import org.apache.ignite.mxbean.IgniteMXBean;
 import org.apache.ignite.mxbean.StripedExecutorMXBean;
+import org.apache.ignite.mxbean.WorkersControlMXBean;
 import org.apache.ignite.mxbean.ThreadPoolMXBean;
 import org.apache.ignite.plugin.IgnitePlugin;
 import org.apache.ignite.plugin.PluginNotFoundException;
@@ -1085,7 +1087,7 @@ public void start(
             // Register MBeans.
             mBeansMgr.registerAllMBeans(utilityCachePool, execSvc, svcExecSvc, sysExecSvc, stripedExecSvc, p2pExecSvc,
                 mgmtExecSvc, igfsExecSvc, dataStreamExecSvc, restExecSvc, affExecSvc, idxExecSvc, callbackExecSvc,
-                qryExecSvc, schemaExecSvc, customExecSvcs);
+                qryExecSvc, schemaExecSvc, customExecSvcs, ctx.workersRegistry());
 
             // Lifecycle bean notifications.
             notifyLifecycleBeans(AFTER_NODE_START);
@@ -4175,7 +4177,8 @@ private void registerAllMBeans(
             IgniteStripedThreadPoolExecutor callbackExecSvc,
             ExecutorService qryExecSvc,
             ExecutorService schemaExecSvc,
-            @Nullable final Map<String, ? extends ExecutorService> customExecSvcs
+            @Nullable final Map<String, ? extends ExecutorService> customExecSvcs,
+            WorkersRegistry workersRegistry
         ) throws IgniteCheckedException {
             if (U.IGNITE_MBEANS_DISABLED)
                 return;
@@ -4221,6 +4224,13 @@ private void registerAllMBeans(
                 for (Map.Entry<String, ? extends ExecutorService> entry : customExecSvcs.entrySet())
                     registerExecutorMBean(entry.getKey(), entry.getValue());
             }
+
+            if (U.IGNITE_TEST_FEATURES_ENABLED) {
+                WorkersControlMXBean workerCtrlMXBean = new WorkersControlMXBeanImpl(workersRegistry);
+
+                registerMBean("Kernal", workerCtrlMXBean.getClass().getSimpleName(),
+                    workerCtrlMXBean, WorkersControlMXBean.class);
+            }
         }
 
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index b0d32564aebf0..a1d84e56bb414 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2591,7 +2591,7 @@ private class DiscoveryWorker extends GridWorker {
          *
          */
         private DiscoveryWorker() {
-            super(ctx.igniteInstanceName(), "disco-event-worker", GridDiscoveryManager.this.log);
+            super(ctx.igniteInstanceName(), "disco-event-worker", GridDiscoveryManager.this.log, ctx.workersRegistry());
         }
 
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index e40493fbc589a..1a0e65f5b8a9f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -2077,7 +2077,8 @@ private class ExchangeWorker extends GridWorker {
          * Constructor.
          */
         private ExchangeWorker() {
-            super(cctx.igniteInstanceName(), "partition-exchanger", GridCachePartitionExchangeManager.this.log);
+            super(cctx.igniteInstanceName(), "partition-exchanger", GridCachePartitionExchangeManager.this.log,
+                cctx.kernalContext().workersRegistry());
         }
 
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
index 613e93bc88c23..7adabc3694e9f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
@@ -121,7 +121,8 @@ private class CleanupWorker extends GridWorker {
          * Creates cleanup worker.
          */
         CleanupWorker() {
-            super(cctx.igniteInstanceName(), "ttl-cleanup-worker", cctx.logger(GridCacheSharedTtlCleanupManager.class));
+            super(cctx.igniteInstanceName(), "ttl-cleanup-worker", cctx.logger(GridCacheSharedTtlCleanupManager.class),
+                cctx.kernalContext().workersRegistry());
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index 429a5ce82bd52..cb04575c8810a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -86,6 +86,7 @@
 import org.apache.ignite.internal.util.IgniteExceptionRegistry;
 import org.apache.ignite.internal.util.StripedExecutor;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.worker.WorkersRegistry;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.plugin.PluginNotFoundException;
 import org.apache.ignite.plugin.PluginProvider;
@@ -453,6 +454,11 @@ private IgniteConfiguration prepareIgniteConfiguration() {
         return null;
     }
 
+    /** {@inheritDoc} */
+    @Override public WorkersRegistry workersRegistry() {
+        return null;
+    }
+
     /** {@inheritDoc} */
     @Override public DataStructuresProcessor dataStructures() {
         return null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index a09d6faad85e9..25151cf7221bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -145,7 +145,8 @@ private class TimeoutWorker extends GridWorker {
          *
          */
         TimeoutWorker() {
-            super(ctx.config().getIgniteInstanceName(), "grid-timeout-worker", GridTimeoutProcessor.this.log);
+            super(ctx.config().getIgniteInstanceName(), "grid-timeout-worker",
+                GridTimeoutProcessor.this.log, ctx.workersRegistry());
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 93f4fb4a39f2e..42e96fb6067f0 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -522,7 +522,12 @@ public abstract class IgniteUtils {
     };
 
     /** Ignite MBeans disabled flag. */
-    public static boolean IGNITE_MBEANS_DISABLED = IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_MBEANS_DISABLED);
+    public static boolean IGNITE_MBEANS_DISABLED =
+        IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_MBEANS_DISABLED);
+
+    /** Ignite test features enabled flag. */
+    public static boolean IGNITE_TEST_FEATURES_ENABLED =
+        IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_TEST_FEATURES_ENABLED);
 
     /** */
     private static final boolean assertionsEnabled;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
new file mode 100644
index 0000000000000..9e427e8f2ab1c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.worker;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.mxbean.WorkersControlMXBean;
+
+/**
+ * MBean that provides control of system workersRegistry.
+ */
+public class WorkersControlMXBeanImpl implements WorkersControlMXBean {
+    /** System worker registry. */
+    private final WorkersRegistry workerRegistry;
+
+    /**
+     * Constructor.
+     *
+     * @param registry System worker registry.
+     */
+    public WorkersControlMXBeanImpl(WorkersRegistry registry) {
+        workerRegistry = registry;
+    }
+
+    /** {@inheritDoc} */
+    @Override public List<String> getWorkerNames() {
+        return new ArrayList<>(workerRegistry.names());
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean terminateWorker(String name) {
+        GridWorker w = workerRegistry.worker(name);
+
+        if (w == null || w.isCancelled())
+            return false;
+
+        Thread t = w.runner();
+
+        if (t == null)
+            return false;
+
+        t.interrupt();
+
+        return true;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
new file mode 100644
index 0000000000000..e8d46fb2ebddb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.worker;
+
+import java.util.Collection;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.internal.util.worker.GridWorkerListener;
+
+/**
+ * Workers registry.
+ */
+public class WorkersRegistry implements GridWorkerListener {
+    /** Registered workers. */
+    private final ConcurrentMap<String, GridWorker> registeredWorkers = new ConcurrentHashMap<>();
+
+    /**
+     * Adds worker to the registry.
+     *
+     * @param w Worker.
+     */
+    public void register(GridWorker w) {
+        if (registeredWorkers.putIfAbsent(w.name(), w) != null)
+            throw new IllegalStateException("Worker is already registered [worker=" + w + ']');
+    }
+
+    /**
+     * Removes worker from the registry.
+     *
+     * @param name Worker name.
+     */
+    public void unregister(String name) {
+        registeredWorkers.remove(name);
+    }
+
+    /**
+     * Returns names of all registered workers.
+     *
+     * @return Registered worker names.
+     */
+    public Collection<String> names() {
+        return registeredWorkers.keySet();
+    }
+
+    /**
+     * Returns worker with given name.
+     *
+     * @param name Name.
+     * @return Registered {@link GridWorker} with name {@code name} or {@code null} if not found.
+     */
+    public GridWorker worker(String name) {
+        return registeredWorkers.get(name);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onStarted(GridWorker w) {
+        register(w);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onStopped(GridWorker w) {
+        unregister(w.name());
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
new file mode 100644
index 0000000000000..03ca62116b7f7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * System worker registry and control MBean implementation.
+ */
+package org.apache.ignite.internal.worker;
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
new file mode 100644
index 0000000000000..0f5419b3b4aac
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mxbean;
+
+import java.util.List;
+
+/**
+ * MBean that provides ability to terminate worker that registered in the workers registry.
+ */
+@MXBeanDescription("MBean that provides ability to terminate worker that registered in the workers registry.")
+public interface WorkersControlMXBean {
+    /**
+     * Returns names of all registered workers.
+     *
+     * @return Worker names.
+     */
+    @MXBeanDescription("Names of registered workers.")
+    public List<String> getWorkerNames();
+
+    /**
+     * Terminates worker.
+     *
+     * @param name Worker name.
+     * @return {@code True} if worker has been terminated successfully, {@code false} otherwise.
+     */
+    @MXBeanDescription("Terminates worker.")
+    @MXBeanParametersNames(
+        "name"
+    )
+    @MXBeanParametersDescriptions(
+        "Name of worker to terminate."
+    )
+    public boolean terminateWorker(String name);
+}

From b4cb2f0df944534743a9d73811e047eda572258c Mon Sep 17 00:00:00 2001
From: mcherkasov <mcherkasov@gridgain.com>
Date: Tue, 10 Apr 2018 17:27:20 -0700
Subject: [PATCH 018/543] IGNITE-8153 Nodes fail to connect each other when SSL
 is enabled - Fixes #3773.

Signed-off-by: Valentin Kulichenko <valentin.kulichenko@gmail.com>
---
 .../internal/util/nio/ssl/BlockingSslHandler.java      | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
index 638106f667a3c..0099c4675a381 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/ssl/BlockingSslHandler.java
@@ -373,9 +373,10 @@ private HandshakeStatus runTasks() {
      * @throws GridNioException If failed to pass event to the next filter.
      */
     private Status unwrapHandshake() throws SSLException, IgniteCheckedException {
-        // Flip input buffer so we can read the collected data.
-        readFromNet();
+        if(!inNetBuf.hasRemaining())
+            readFromNet();
 
+        // Flip input buffer so we can read the collected data.
         inNetBuf.flip();
 
         SSLEngineResult res = unwrap0();
@@ -399,7 +400,10 @@ private Status unwrapHandshake() throws SSLException, IgniteCheckedException {
         else if (res.getStatus() == BUFFER_UNDERFLOW) {
             inNetBuf.compact();
 
-            inNetBuf = expandBuffer(inNetBuf, inNetBuf.capacity() * 2);
+            if(inNetBuf.capacity() == inNetBuf.limit())
+                inNetBuf = expandBuffer(inNetBuf, inNetBuf.capacity() * 2);
+
+            readFromNet();
         }
         else
             // prepare to be written again

From b4cc9f2d45d78c360abe224165e707c23533469e Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 11 Apr 2018 11:23:46 +0300
Subject: [PATCH 019/543] IGNITE-7871 Implemented additional synchronization
 phase for correct partition counters update

---
 .../org/apache/ignite/internal/GridTopic.java |   5 +-
 .../communication/GridIoMessageFactory.java   |   6 +
 .../discovery/GridDiscoveryManager.java       |  10 +
 .../MetaPageUpdatePartitionDataRecord.java    |   2 +-
 .../processors/cache/CacheMetricsImpl.java    |   2 +-
 .../cache/GridCacheMvccManager.java           |  38 +
 .../GridCachePartitionExchangeManager.java    |  17 +
 .../cache/GridCacheSharedContext.java         |   9 +-
 .../processors/cache/GridCacheUtils.java      |   2 +-
 .../cache/IgniteCacheOffheapManager.java      |   8 +-
 .../cache/IgniteCacheOffheapManagerImpl.java  |  10 +-
 .../dht/GridClientPartitionTopology.java      |   5 +
 .../dht/GridDhtLocalPartition.java            |   9 +-
 .../dht/GridDhtPartitionTopology.java         |   6 +
 .../dht/GridDhtPartitionTopologyImpl.java     |  26 +-
 .../dht/GridDhtPartitionsStateValidator.java  | 255 +++++++
 .../cache/distributed/dht/GridDhtTxLocal.java |   5 +
 .../GridDhtPartitionsExchangeFuture.java      |  96 ++-
 .../GridDhtPartitionsSingleMessage.java       |  68 +-
 .../preloader/InitNewCoordinatorFuture.java   |   2 +-
 .../preloader/latch/ExchangeLatchManager.java | 695 ++++++++++++++++++
 .../dht/preloader/latch/Latch.java            |  52 ++
 .../dht/preloader/latch/LatchAckMessage.java  | 165 +++++
 .../distributed/near/GridNearTxLocal.java     |  10 +
 .../persistence/GridCacheOffheapManager.java  |  10 +-
 .../cache/transactions/IgniteTxAdapter.java   |   2 +-
 .../cache/transactions/IgniteTxManager.java   |  36 +-
 ...eDhtLocalPartitionAfterRemoveSelfTest.java |   2 +-
 .../cache/IgniteCacheGroupsTest.java          |   1 +
 ...changeLatchManagerCoordinatorFailTest.java | 244 ++++++
 ...ridCachePartitionsStateValidationTest.java | 316 ++++++++
 ...CachePartitionsStateValidatorSelfTest.java | 158 ++++
 .../TxOptimisticOnPartitionExchangeTest.java  | 322 ++++++++
 .../testsuites/IgniteCacheTestSuite.java      |   4 +
 .../testsuites/IgniteCacheTestSuite6.java     |   6 +
 35 files changed, 2568 insertions(+), 36 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/Latch.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticOnPartitionExchangeTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
index 1227e8cf4abd4..0b2d41a39ec89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTopic.java
@@ -124,7 +124,10 @@ public enum GridTopic {
     TOPIC_METRICS,
 
     /** */
-    TOPIC_AUTH;
+    TOPIC_AUTH,
+
+    /** */
+    TOPIC_EXCHANGE;
 
     /** Enum values. */
     private static final GridTopic[] VALS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 5616fd035d445..581c32e4b1fb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -53,6 +53,7 @@
 import org.apache.ignite.internal.processors.cache.WalStateAckMessage;
 import org.apache.ignite.internal.processors.cache.binary.MetadataRequestMessage;
 import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.LatchAckMessage;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTtlUpdateRequest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryRequest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryResponse;
@@ -921,6 +922,11 @@ public GridIoMessageFactory(MessageFactory[] ext) {
 
                 break;
 
+            case 135:
+                msg = new LatchAckMessage();
+
+                break;
+
             // [-3..119] [124..129] [-23..-27] [-36..-55]- this
             // [120..123] - DR
             // [-4..-22, -30..-35] - SQL
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index a1d84e56bb414..400bb5fd28742 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -793,6 +793,9 @@ else if (type == EVT_CLIENT_NODE_DISCONNECTED) {
 
                     ((IgniteKernal)ctx.grid()).onDisconnected();
 
+                    if (!locJoin.isDone())
+                        locJoin.onDone(new IgniteCheckedException("Node disconnected"));
+
                     locJoin = new GridFutureAdapter<>();
 
                     registeredCaches.clear();
@@ -2141,6 +2144,13 @@ public DiscoveryLocalJoinData localJoin() {
         }
     }
 
+    /**
+     * @return Local join future.
+     */
+    public GridFutureAdapter<DiscoveryLocalJoinData> localJoinFuture() {
+        return locJoin;
+    }
+
     /**
      * @param msg Custom message.
      * @throws IgniteCheckedException If failed.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
index bafbf475abe9f..e5bd343bcb5ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
@@ -32,7 +32,7 @@ public class MetaPageUpdatePartitionDataRecord extends PageDeltaRecord {
     /** */
     private long globalRmvId;
 
-    /** */
+    /** TODO: Partition size may be long */
     private int partSize;
 
     /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
index 6fae8feff3c6c..b402ff2bb9170 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
@@ -792,7 +792,7 @@ public EntriesStatMetrics getEntriesStat() {
                     if (cctx.cache() == null)
                         continue;
 
-                    int cacheSize = part.dataStore().cacheSize(cctx.cacheId());
+                    long cacheSize = part.dataStore().cacheSize(cctx.cacheId());
 
                     offHeapEntriesCnt += cacheSize;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
index a9fa3c777a26b..fade83377a8e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
@@ -44,6 +44,8 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheMappedVersion;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishFuture;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -313,6 +315,42 @@ public Collection<GridCacheFuture<?>> activeFutures() {
         return col;
     }
 
+    /**
+     * Creates a future that will wait for finishing all remote transactions (primary -> backup)
+     * with topology version less or equal to {@code topVer}.
+     *
+     * @param topVer Topology version.
+     * @return Compound future of all {@link GridDhtTxFinishFuture} futures.
+     */
+    public IgniteInternalFuture<?> finishRemoteTxs(AffinityTopologyVersion topVer) {
+        GridCompoundFuture<?, ?> res = new CacheObjectsReleaseFuture<>("RemoteTx", topVer);
+
+        for (GridCacheFuture<?> fut : futs.values()) {
+            if (fut instanceof GridDhtTxFinishFuture) {
+                GridDhtTxFinishFuture finishTxFuture = (GridDhtTxFinishFuture) fut;
+
+                if (cctx.tm().needWaitTransaction(finishTxFuture.tx(), topVer))
+                    res.add(ignoreErrors(finishTxFuture));
+            }
+        }
+
+        res.markInitialized();
+
+        return res;
+    }
+
+    /**
+     * Future wrapper which ignores any underlying future errors.
+     *
+     * @param f Underlying future.
+     * @return Future wrapper which ignore any underlying future errors.
+     */
+    private IgniteInternalFuture ignoreErrors(IgniteInternalFuture<?> f) {
+        GridFutureAdapter<?> wrapper = new GridFutureAdapter();
+        f.listen(future -> wrapper.onDone());
+        return wrapper;
+    }
+
     /**
      * @param leftNodeId Left node ID.
      * @param topVer Topology version.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 1a0e65f5b8a9f..20a3ccbf4b2f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -64,6 +64,7 @@
 import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
@@ -216,6 +217,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     /** For tests only. */
     private volatile AffinityTopologyVersion exchMergeTestWaitVer;
 
+    /** Distributed latch manager. */
+    private ExchangeLatchManager latchMgr;
+
     /** Discovery listener. */
     private final DiscoveryEventListener discoLsnr = new DiscoveryEventListener() {
         @Override public void onEvent(DiscoveryEvent evt, DiscoCache cache) {
@@ -309,6 +313,8 @@ private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) {
 
         exchWorker = new ExchangeWorker();
 
+        latchMgr = new ExchangeLatchManager(cctx.kernalContext());
+
         cctx.gridEvents().addDiscoveryEventListener(discoLsnr, EVT_NODE_JOINED, EVT_NODE_LEFT, EVT_NODE_FAILED,
             EVT_DISCOVERY_CUSTOM_EVT);
 
@@ -1255,6 +1261,8 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(
 
                     m.addPartitionUpdateCounters(grp.groupId(),
                         newCntrMap ? cntrsMap : CachePartitionPartialCountersMap.toCountersMap(cntrsMap));
+
+                    m.addPartitionSizes(grp.groupId(), grp.topology().partitionSizes());
                 }
             }
         }
@@ -1277,6 +1285,8 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(
 
                 m.addPartitionUpdateCounters(top.groupId(),
                     newCntrMap ? cntrsMap : CachePartitionPartialCountersMap.toCountersMap(cntrsMap));
+
+                m.addPartitionSizes(top.groupId(), top.partitionSizes());
             }
         }
 
@@ -1569,6 +1579,13 @@ private void processSinglePartitionRequest(ClusterNode node, GridDhtPartitionsSi
         }
     }
 
+    /**
+     * @return Latch manager instance.
+     */
+    public ExchangeLatchManager latch() {
+        return latchMgr;
+    }
+
     /**
      * @param exchFut Optional current exchange future.
      * @throws Exception If failed.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index c2f9229b8fe5c..b3b4f0dc7b432 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -26,7 +26,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicIntegerArray;
-import java.util.function.BiFunction;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
@@ -711,7 +710,7 @@ public GridCacheIoManager io() {
 
     /**
      * @return Ttl cleanup manager.
-     * */
+     */
     public GridCacheSharedTtlCleanupManager ttl() {
         return ttlMgr;
     }
@@ -854,10 +853,14 @@ public IgniteInternalFuture<?> partitionReleaseFuture(AffinityTopologyVersion to
         GridCompoundFuture f = new CacheObjectsReleaseFuture("Partition", topVer);
 
         f.add(mvcc().finishExplicitLocks(topVer));
-        f.add(tm().finishTxs(topVer));
         f.add(mvcc().finishAtomicUpdates(topVer));
         f.add(mvcc().finishDataStreamerUpdates(topVer));
 
+        IgniteInternalFuture<?> finishLocalTxsFuture = tm().finishLocalTxs(topVer);
+        // To properly track progress of finishing local tx updates we explicitly add this future to compound set.
+        f.add(finishLocalTxsFuture);
+        f.add(tm().finishAllTxs(finishLocalTxsFuture, topVer));
+
         f.markInitialized();
 
         return f;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index a5169d26dcc13..d672420fafdc8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1732,7 +1732,7 @@ private void process(KeyCacheObject key, CacheObject val, GridCacheVersion ver,
                             ver,
                             expiryPlc == null ? 0 : expiryPlc.forCreate(),
                             expiryPlc == null ? 0 : toExpireTime(expiryPlc.forCreate()),
-                            false,
+                            true,
                             topVer,
                             GridDrType.DR_BACKUP,
                             true);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
index 3d83f87105db2..a12c0334912a5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
@@ -22,11 +22,11 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.RootPage;
 import org.apache.ignite.internal.processors.cache.persistence.RowStore;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.query.GridQueryRowCacheCleaner;
 import org.apache.ignite.internal.util.GridAtomicLong;
@@ -344,7 +344,7 @@ public long cacheEntriesCount(int cacheId, boolean primary, boolean backup, Affi
      * @param part Partition.
      * @return Number of entries.
      */
-    public int totalPartitionEntriesCount(int part);
+    public long totalPartitionEntriesCount(int part);
 
     /**
      *
@@ -381,7 +381,7 @@ interface CacheDataStore {
          * @param cacheId Cache ID.
          * @return Size.
          */
-        int cacheSize(int cacheId);
+        long cacheSize(int cacheId);
 
         /**
          * @return Cache sizes if store belongs to group containing multiple caches.
@@ -391,7 +391,7 @@ interface CacheDataStore {
         /**
          * @return Total size.
          */
-        int fullSize();
+        long fullSize();
 
         /**
          * @return Update counter.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index b2019352c2f1a..f8cc86f8cecdb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -252,7 +252,7 @@ public CacheDataStore dataStore(GridDhtLocalPartition part) {
     }
 
     /** {@inheritDoc} */
-    @Override public int totalPartitionEntriesCount(int p) {
+    @Override public long totalPartitionEntriesCount(int p) {
         if (grp.isLocal())
             return locCacheDataStore.fullSize();
         else {
@@ -1152,14 +1152,14 @@ void decrementSize(int cacheId) {
         }
 
         /** {@inheritDoc} */
-        @Override public int cacheSize(int cacheId) {
+        @Override public long cacheSize(int cacheId) {
             if (grp.sharedGroup()) {
                 AtomicLong size = cacheSizes.get(cacheId);
 
                 return size != null ? (int)size.get() : 0;
             }
 
-            return (int)storageSize.get();
+            return storageSize.get();
         }
 
         /** {@inheritDoc} */
@@ -1176,8 +1176,8 @@ void decrementSize(int cacheId) {
         }
 
         /** {@inheritDoc} */
-        @Override public int fullSize() {
-            return (int)storageSize.get();
+        @Override public long fullSize() {
+            return storageSize.get();
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 5bbbb3102c663..3e3bb0db9b9bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -1195,6 +1195,11 @@ private void removeNode(UUID nodeId) {
         return CachePartitionPartialCountersMap.EMPTY;
     }
 
+    /** {@inheritDoc} */
+    @Override public Map<Integer, Long> partitionSizes() {
+        return Collections.emptyMap();
+    }
+
     /** {@inheritDoc} */
     @Override public boolean rebalanceFinished(AffinityTopologyVersion topVer) {
         assert false : "Should not be called on non-affinity node";
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 7a47f31c99055..ea20dbf4b1184 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -929,7 +929,7 @@ public long updateCounter() {
     /**
      * @return Initial update counter.
      */
-    public Long initialUpdateCounter() {
+    public long initialUpdateCounter() {
         return store.initialUpdateCounter();
     }
 
@@ -947,6 +947,13 @@ public void initialUpdateCounter(long val) {
         store.updateInitialCounter(val);
     }
 
+    /**
+     * @return Total size of all caches.
+     */
+    public long fullSize() {
+        return store.fullSize();
+    }
+
     /**
      * Removes all entries and rows from this partition.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 7f900cb67f533..6f68dbbaeb511 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -19,6 +19,7 @@
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
@@ -344,6 +345,11 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
      */
     public CachePartitionPartialCountersMap localUpdateCounters(boolean skipZeros);
 
+    /**
+     * @return Partition cache sizes.
+     */
+    public Map<Integer, Long> partitionSizes();
+
     /**
      * @param part Partition to own.
      * @return {@code True} if owned.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 538c57ec9ba97..740903e415baf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -31,6 +31,8 @@
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceArray;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
@@ -2525,6 +2527,28 @@ private void removeNode(UUID nodeId) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public Map<Integer, Long> partitionSizes() {
+        lock.readLock().lock();
+
+        try {
+            Map<Integer, Long> partitionSizes = new HashMap<>();
+
+            for (int p = 0; p < locParts.length(); p++) {
+                GridDhtLocalPartition part = locParts.get(p);
+                if (part == null || part.fullSize() == 0)
+                    continue;
+
+                partitionSizes.put(part.id(), part.fullSize());
+            }
+
+            return partitionSizes;
+        }
+        finally {
+            lock.readLock().unlock();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public boolean rebalanceFinished(AffinityTopologyVersion topVer) {
         AffinityTopologyVersion curTopVer = this.readyTopVer;
@@ -2587,7 +2611,7 @@ public void onCacheStopped(int cacheId) {
                 if (part == null)
                     continue;
 
-                int size = part.dataStore().fullSize();
+                long size = part.dataStore().fullSize();
 
                 if (size >= threshold)
                     X.println(">>>   Local partition [part=" + part.id() + ", size=" + size + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
new file mode 100644
index 0000000000000..92a05848e3d00
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.lang.IgniteProductVersion;
+
+import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
+
+/**
+ * Class to validate partitions update counters and cache sizes during exchange process.
+ */
+public class GridDhtPartitionsStateValidator {
+    /** Version since node is able to send cache sizes in {@link GridDhtPartitionsSingleMessage}. */
+    private static final IgniteProductVersion SIZES_VALIDATION_AVAILABLE_SINCE = IgniteProductVersion.fromString("2.5.0");
+
+    /** Cache shared context. */
+    private final GridCacheSharedContext<?, ?> cctx;
+
+    /**
+     * Constructor.
+     *
+     * @param cctx Cache shared context.
+     */
+    public GridDhtPartitionsStateValidator(GridCacheSharedContext<?, ?> cctx) {
+        this.cctx = cctx;
+    }
+
+    /**
+     * Validates partition states - update counters and cache sizes for all nodes.
+     * If update counter value or cache size for the same partitions are different on some nodes
+     * method throws exception with full information about inconsistent partitions.
+     *
+     * @param fut Current exchange future.
+     * @param top Topology to validate.
+     * @param messages Single messages received from all nodes.
+     * @throws IgniteCheckedException If validation failed. Exception message contains
+     * full information about all partitions which update counters or cache sizes are not consistent.
+     */
+    public void validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture fut,
+                                                  GridDhtPartitionTopology top,
+                                                  Map<UUID, GridDhtPartitionsSingleMessage> messages) throws IgniteCheckedException {
+        // Ignore just joined nodes.
+        final Set<UUID> ignoringNodes = new HashSet<>();
+
+        for (DiscoveryEvent evt : fut.events().events())
+            if (evt.type() == EVT_NODE_JOINED)
+                ignoringNodes.add(evt.eventNode().id());
+
+        AffinityTopologyVersion topVer = fut.context().events().topologyVersion();
+
+        // Validate update counters.
+        Map<Integer, Map<UUID, Long>> result = validatePartitionsUpdateCounters(top, messages, ignoringNodes);
+        if (!result.isEmpty())
+            throw new IgniteCheckedException("Partitions update counters are inconsistent for " + fold(topVer, result));
+
+        // For sizes validation ignore also nodes which are not able to send cache sizes.
+        for (UUID id : messages.keySet()) {
+            ClusterNode node = cctx.discovery().node(id);
+            if (node != null && node.version().compareTo(SIZES_VALIDATION_AVAILABLE_SINCE) < 0)
+                ignoringNodes.add(id);
+        }
+
+        // Validate cache sizes.
+        result = validatePartitionsSizes(top, messages, ignoringNodes);
+        if (!result.isEmpty())
+            throw new IgniteCheckedException("Partitions cache sizes are inconsistent for " + fold(topVer, result));
+    }
+
+    /**
+     * Validate partitions update counters for given {@code top}.
+     *
+     * @param top Topology to validate.
+     * @param messages Single messages received from all nodes.
+     * @param ignoringNodes Nodes for what we ignore validation.
+     * @return Invalid partitions map with following structure: (partId, (nodeId, updateCounter)).
+     * If map is empty validation is successful.
+     */
+     Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
+            GridDhtPartitionTopology top,
+            Map<UUID, GridDhtPartitionsSingleMessage> messages,
+            Set<UUID> ignoringNodes) {
+        Map<Integer, Map<UUID, Long>> invalidPartitions = new HashMap<>();
+
+        Map<Integer, T2<UUID, Long>> updateCountersAndNodesByPartitions = new HashMap<>();
+
+        // Populate counters statistics from local node partitions.
+        for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
+            if (top.partitionState(cctx.localNodeId(), part.id()) != GridDhtPartitionState.OWNING)
+                continue;
+
+            updateCountersAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.updateCounter()));
+        }
+
+        int partitions = top.partitions();
+
+        // Then process and validate counters from other nodes.
+        for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> e : messages.entrySet()) {
+            UUID nodeId = e.getKey();
+            if (ignoringNodes.contains(nodeId))
+                continue;
+
+            CachePartitionPartialCountersMap countersMap = e.getValue().partitionUpdateCounters(top.groupId(), partitions);
+
+            for (int part = 0; part < partitions; part++) {
+                if (top.partitionState(nodeId, part) != GridDhtPartitionState.OWNING)
+                    continue;
+
+                int partIdx = countersMap.partitionIndex(part);
+                long currentCounter = partIdx >= 0 ? countersMap.updateCounterAt(partIdx) : 0;
+
+                process(invalidPartitions, updateCountersAndNodesByPartitions, part, nodeId, currentCounter);
+            }
+        }
+
+        return invalidPartitions;
+    }
+
+    /**
+     * Validate partitions cache sizes for given {@code top}.
+     *
+     * @param top Topology to validate.
+     * @param messages Single messages received from all nodes.
+     * @param ignoringNodes Nodes for what we ignore validation.
+     * @return Invalid partitions map with following structure: (partId, (nodeId, cacheSize)).
+     * If map is empty validation is successful.
+     */
+     Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
+            GridDhtPartitionTopology top,
+            Map<UUID, GridDhtPartitionsSingleMessage> messages,
+            Set<UUID> ignoringNodes) {
+        Map<Integer, Map<UUID, Long>> invalidPartitions = new HashMap<>();
+
+        Map<Integer, T2<UUID, Long>> sizesAndNodesByPartitions = new HashMap<>();
+
+        // Populate sizes statistics from local node partitions.
+        for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
+            if (top.partitionState(cctx.localNodeId(), part.id()) != GridDhtPartitionState.OWNING)
+                continue;
+
+            sizesAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.fullSize()));
+        }
+
+        int partitions = top.partitions();
+
+        // Then process and validate sizes from other nodes.
+        for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> e : messages.entrySet()) {
+            UUID nodeId = e.getKey();
+            if (ignoringNodes.contains(nodeId))
+                continue;
+
+            Map<Integer, Long> sizesMap = e.getValue().partitionSizes(top.groupId());
+
+            for (int part = 0; part < partitions; part++) {
+                if (top.partitionState(nodeId, part) != GridDhtPartitionState.OWNING)
+                    continue;
+
+                long currentSize = sizesMap.containsKey(part) ? sizesMap.get(part) : 0L;
+
+                process(invalidPartitions, sizesAndNodesByPartitions, part, nodeId, currentSize);
+            }
+        }
+
+        return invalidPartitions;
+    }
+
+    /**
+     * Processes given {@code counter} for partition {@code part} reported by {@code node}.
+     * Populates {@code invalidPartitions} map if existing counter and current {@code counter} are different.
+     *
+     * @param invalidPartitions Invalid partitions map.
+     * @param countersAndNodes Current map of counters and nodes by partitions.
+     * @param part Processing partition.
+     * @param node Node id.
+     * @param counter Counter value reported by {@code node}.
+     */
+    private void process(Map<Integer, Map<UUID, Long>> invalidPartitions,
+                         Map<Integer, T2<UUID, Long>> countersAndNodes,
+                         int part,
+                         UUID node,
+                         long counter) {
+        T2<UUID, Long> existingData = countersAndNodes.get(part);
+
+        if (existingData == null)
+            countersAndNodes.put(part, new T2<>(node, counter));
+
+        if (existingData != null && counter != existingData.get2()) {
+            if (!invalidPartitions.containsKey(part)) {
+                Map<UUID, Long> map = new HashMap<>();
+                map.put(existingData.get1(), existingData.get2());
+                invalidPartitions.put(part, map);
+            }
+
+            invalidPartitions.get(part).put(node, counter);
+        }
+    }
+
+    /**
+     * Folds given map of invalid partition states to string representation in the following format:
+     * Part [id]: [consistentId=value*]
+     *
+     * Value can be both update counter or cache size.
+     *
+     * @param topVer Last topology version.
+     * @param invalidPartitions Invalid partitions map.
+     * @return String representation of invalid partitions.
+     */
+    private String fold(AffinityTopologyVersion topVer, Map<Integer, Map<UUID, Long>> invalidPartitions) {
+        SB sb = new SB();
+
+        NavigableMap<Integer, Map<UUID, Long>> sortedPartitions = new TreeMap<>(invalidPartitions);
+
+        for (Map.Entry<Integer, Map<UUID, Long>> p : sortedPartitions.entrySet()) {
+            sb.a("Part ").a(p.getKey()).a(": [");
+            for (Map.Entry<UUID, Long> e : p.getValue().entrySet()) {
+                Object consistentId = cctx.discovery().node(topVer, e.getKey()).consistentId();
+                sb.a(consistentId).a("=").a(e.getValue()).a(" ");
+            }
+            sb.a("] ");
+        }
+
+        return sb.toString();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
index 28cc018657a6c..0609f04017643 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
@@ -447,6 +447,11 @@ private void finishTx(boolean commit, @Nullable IgniteInternalFuture prepFut, Gr
 
             err = e;
         }
+        catch (Throwable t) {
+            fut.onDone(t);
+
+            throw t;
+        }
 
         if (primarySync)
             sendFinishReply(err);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index cbb49851e1f81..dd4a57157a1e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -41,6 +41,7 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
@@ -75,10 +76,12 @@
 import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
 import org.apache.ignite.internal.processors.cache.StateChangeRequest;
 import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsStateValidator;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
@@ -290,6 +293,10 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     @GridToStringExclude
     private GridDhtPartitionsExchangeFuture mergedWith;
 
+    /** Validator for partition states. */
+    @GridToStringExclude
+    private final GridDhtPartitionsStateValidator validator;
+
     /**
      * @param cctx Cache context.
      * @param busyLock Busy lock.
@@ -314,6 +321,7 @@ public GridDhtPartitionsExchangeFuture(
         this.exchId = exchId;
         this.exchActions = exchActions;
         this.affChangeMsg = affChangeMsg;
+        this.validator = new GridDhtPartitionsStateValidator(cctx);
 
         log = cctx.logger(getClass());
         exchLog = cctx.logger(EXCHANGE_LOG);
@@ -1099,7 +1107,11 @@ private void distributedExchange() throws IgniteCheckedException {
         // To correctly rebalance when persistence is enabled, it is necessary to reserve history within exchange.
         partHistReserved = cctx.database().reserveHistoryForExchange();
 
-        waitPartitionRelease();
+        // On first phase we wait for finishing all local tx updates, atomic updates and lock releases.
+        waitPartitionRelease(1);
+
+        // Second phase is needed to wait for finishing all tx updates from primary to backup nodes remaining after first phase.
+        waitPartitionRelease(2);
 
         boolean topChanged = firstDiscoEvt.type() != EVT_DISCOVERY_CUSTOM_EVT || affChangeMsg != null;
 
@@ -1202,9 +1214,17 @@ private void changeWalModeIfNeeded() {
      * For the exact list of the objects being awaited for see
      * {@link GridCacheSharedContext#partitionReleaseFuture(AffinityTopologyVersion)} javadoc.
      *
+     * @param phase Phase of partition release.
+     *
      * @throws IgniteCheckedException If failed.
      */
-    private void waitPartitionRelease() throws IgniteCheckedException {
+    private void waitPartitionRelease(int phase) throws IgniteCheckedException {
+        Latch releaseLatch = null;
+
+        // Wait for other nodes only on first phase.
+        if (phase == 1)
+            releaseLatch = cctx.exchange().latch().getOrCreate("exchange", initialVersion());
+
         IgniteInternalFuture<?> partReleaseFut = cctx.partitionReleaseFuture(initialVersion());
 
         // Assign to class variable so it will be included into toString() method.
@@ -1238,6 +1258,11 @@ private void waitPartitionRelease() throws IgniteCheckedException {
                     nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, futTimeout);
                 }
             }
+            catch (IgniteCheckedException e) {
+                U.warn(log,"Unable to await partitions release future", e);
+
+                throw e;
+            }
         }
 
         long waitEnd = U.currentTimeMillis();
@@ -1290,6 +1315,35 @@ private void waitPartitionRelease() throws IgniteCheckedException {
                 }
             }
         }
+
+        if (releaseLatch == null)
+            return;
+
+        releaseLatch.countDown();
+
+        if (!localJoinExchange()) {
+            try {
+                while (true) {
+                    try {
+                        releaseLatch.await(futTimeout, TimeUnit.MILLISECONDS);
+
+                        if (log.isInfoEnabled())
+                            log.info("Finished waiting for partitions release latch: " + releaseLatch);
+
+                        break;
+                    }
+                    catch (IgniteFutureTimeoutCheckedException ignored) {
+                        U.warn(log, "Unable to await partitions release latch within timeout: " + releaseLatch);
+
+                        // Try to resend ack.
+                        releaseLatch.countDown();
+                    }
+                }
+            }
+            catch (IgniteCheckedException e) {
+                U.warn(log, "Stop waiting for partitions release latch: " + e.getMessage());
+            }
+        }
     }
 
     /**
@@ -2499,6 +2553,8 @@ private void finishExchangeOnCoordinator(@Nullable Collection<ClusterNode> sndRe
                 }
             }
 
+            validatePartitionsState();
+
             if (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
                 assert firstDiscoEvt instanceof DiscoveryCustomEvent;
 
@@ -2682,6 +2738,42 @@ else if (forceAffReassignment)
         }
     }
 
+    /**
+     * Validates that partition update counters and cache sizes for all caches are consistent.
+     */
+    private void validatePartitionsState() {
+        for (Map.Entry<Integer, CacheGroupDescriptor> e : cctx.affinity().cacheGroups().entrySet()) {
+            CacheGroupDescriptor grpDesc = e.getValue();
+            if (grpDesc.config().getCacheMode() == CacheMode.LOCAL)
+                continue;
+
+            int grpId = e.getKey();
+
+            CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpId);
+
+            GridDhtPartitionTopology top = grpCtx != null ?
+                    grpCtx.topology() :
+                    cctx.exchange().clientTopology(grpId, events().discoveryCache());
+
+            // Do not validate read or write through caches or caches with disabled rebalance.
+            if (grpCtx == null
+                    || grpCtx.config().isReadThrough()
+                    || grpCtx.config().isWriteThrough()
+                    || grpCtx.config().getCacheStoreFactory() != null
+                    || grpCtx.config().getRebalanceDelay() != -1
+                    || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE)
+                continue;
+
+            try {
+                validator.validatePartitionCountersAndSizes(this, top, msgs);
+            }
+            catch (IgniteCheckedException ex) {
+                log.warning("Partition states validation was failed for cache " + grpDesc.cacheOrGroupName(), ex);
+                // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833
+            }
+        }
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 215152d771149..6ebafac22611e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -17,9 +17,9 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
-import java.util.Collection;
 import java.io.Externalizable;
 import java.nio.ByteBuffer;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -67,6 +67,14 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
     /** Serialized partitions counters. */
     private byte[] partCntrsBytes;
 
+    /** Partitions sizes. */
+    @GridToStringInclude
+    @GridDirectTransient
+    private Map<Integer, Map<Integer, Long>> partSizes;
+
+    /** Serialized partitions counters. */
+    private byte[] partSizesBytes;
+
     /** Partitions history reservation counters. */
     @GridToStringInclude
     @GridDirectTransient
@@ -219,6 +227,35 @@ public CachePartitionPartialCountersMap partitionUpdateCounters(int grpId, int p
         return CachePartitionPartialCountersMap.fromCountersMap(map, partsCnt);
     }
 
+    /**
+     * Adds partition sizes map for specified {@code grpId} to the current message.
+     *
+     * @param grpId Group id.
+     * @param partSizesMap Partition sizes map.
+     */
+    public void addPartitionSizes(int grpId, Map<Integer, Long> partSizesMap) {
+        if (partSizesMap.isEmpty())
+            return;
+
+        if (partSizes == null)
+            partSizes = new HashMap<>();
+
+        partSizes.put(grpId, partSizesMap);
+    }
+
+    /**
+     * Returns partition sizes map for specified {@code grpId}.
+     *
+     * @param grpId Group id.
+     * @return Partition sizes map (partId, partSize).
+     */
+    public Map<Integer, Long> partitionSizes(int grpId) {
+        if (partSizes == null)
+            return Collections.emptyMap();
+
+        return partSizes.getOrDefault(grpId, Collections.emptyMap());
+    }
+
     /**
      * @param grpId Cache group ID.
      * @param cntrMap Partition history counters.
@@ -287,12 +324,14 @@ public void setError(Exception ex) {
         boolean marshal = (parts != null && partsBytes == null) ||
             (partCntrs != null && partCntrsBytes == null) ||
             (partHistCntrs != null && partHistCntrsBytes == null) ||
+            (partSizes != null && partSizesBytes == null) ||
             (err != null && errBytes == null);
 
         if (marshal) {
             byte[] partsBytes0 = null;
             byte[] partCntrsBytes0 = null;
             byte[] partHistCntrsBytes0 = null;
+            byte[] partSizesBytes0 = null;
             byte[] errBytes0 = null;
 
             if (parts != null && partsBytes == null)
@@ -304,6 +343,9 @@ public void setError(Exception ex) {
             if (partHistCntrs != null && partHistCntrsBytes == null)
                 partHistCntrsBytes0 = U.marshal(ctx, partHistCntrs);
 
+            if (partSizes != null && partSizesBytes == null)
+                partSizesBytes0 = U.marshal(ctx, partSizes);
+
             if (err != null && errBytes == null)
                 errBytes0 = U.marshal(ctx, err);
 
@@ -314,11 +356,13 @@ public void setError(Exception ex) {
                     byte[] partsBytesZip = U.zip(partsBytes0);
                     byte[] partCntrsBytesZip = U.zip(partCntrsBytes0);
                     byte[] partHistCntrsBytesZip = U.zip(partHistCntrsBytes0);
+                    byte[] partSizesBytesZip = U.zip(partSizesBytes0);
                     byte[] exBytesZip = U.zip(errBytes0);
 
                     partsBytes0 = partsBytesZip;
                     partCntrsBytes0 = partCntrsBytesZip;
                     partHistCntrsBytes0 = partHistCntrsBytesZip;
+                    partSizesBytes0 = partSizesBytesZip;
                     errBytes0 = exBytesZip;
 
                     compressed(true);
@@ -331,6 +375,7 @@ public void setError(Exception ex) {
             partsBytes = partsBytes0;
             partCntrsBytes = partCntrsBytes0;
             partHistCntrsBytes = partHistCntrsBytes0;
+            partSizesBytes = partSizesBytes0;
             errBytes = errBytes0;
         }
     }
@@ -360,6 +405,13 @@ public void setError(Exception ex) {
                 partHistCntrs = U.unmarshal(ctx, partHistCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
         }
 
+        if (partSizesBytes != null && partSizes == null) {
+            if (compressed())
+                partSizes = U.unmarshalZip(ctx.marshaller(), partSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+            else
+                partSizes = U.unmarshal(ctx, partSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+        }
+
         if (errBytes != null && err == null) {
             if (compressed())
                 err = U.unmarshalZip(ctx.marshaller(), errBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
@@ -451,6 +503,11 @@ public void setError(Exception ex) {
 
                 writer.incrementState();
 
+            case 13:
+                if (!writer.writeByteArray("partsSizesBytes", partSizesBytes))
+                    return false;
+
+                writer.incrementState();
         }
 
         return true;
@@ -531,6 +588,13 @@ public void setError(Exception ex) {
 
                 reader.incrementState();
 
+            case 13:
+                partSizesBytes = reader.readByteArray("partsSizesBytes");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
         }
 
         return reader.afterMessageRead(GridDhtPartitionsSingleMessage.class);
@@ -543,7 +607,7 @@ public void setError(Exception ex) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 13;
+        return 14;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
index 596fa8c4f8976..42a9ba6891955 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
@@ -235,7 +235,7 @@ public void onMessage(ClusterNode node, GridDhtPartitionsSingleMessage msg) {
             if (awaited.remove(node.id())) {
                 GridDhtPartitionsFullMessage fullMsg0 = msg.finishMessage();
 
-                if (fullMsg0 != null) {
+                if (fullMsg0 != null && fullMsg0.resultTopologyVersion() != null) {
                     assert fullMsg == null || fullMsg.resultTopologyVersion().equals(fullMsg0.resultTopologyVersion());
 
                     fullMsg  = fullMsg0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
new file mode 100644
index 0000000000000..c205cb14e40d9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -0,0 +1,695 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.GridTopic;
+import org.apache.ignite.internal.managers.communication.GridIoManager;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+
+/**
+ * Class is responsible to create and manage instances of distributed latches {@link Latch}.
+ */
+public class ExchangeLatchManager {
+    /** Version since latch management is available. */
+    private static final IgniteProductVersion VERSION_SINCE = IgniteProductVersion.fromString("2.5.0");
+
+    /** Logger. */
+    private final IgniteLogger log;
+
+    /** Context. */
+    private final GridKernalContext ctx;
+
+    /** Discovery manager. */
+    private final GridDiscoveryManager discovery;
+
+    /** IO manager. */
+    private final GridIoManager io;
+
+    /** Current coordinator. */
+    private volatile ClusterNode coordinator;
+
+    /** Pending acks collection. */
+    private final ConcurrentMap<T2<String, AffinityTopologyVersion>, Set<UUID>> pendingAcks = new ConcurrentHashMap<>();
+
+    /** Server latches collection. */
+    private final ConcurrentMap<T2<String, AffinityTopologyVersion>, ServerLatch> serverLatches = new ConcurrentHashMap<>();
+
+    /** Client latches collection. */
+    private final ConcurrentMap<T2<String, AffinityTopologyVersion>, ClientLatch> clientLatches = new ConcurrentHashMap<>();
+
+    /** Lock. */
+    private final ReentrantLock lock = new ReentrantLock();
+
+    /**
+     * Constructor.
+     *
+     * @param ctx Kernal context.
+     */
+    public ExchangeLatchManager(GridKernalContext ctx) {
+        this.ctx = ctx;
+        this.log = ctx.log(getClass());
+        this.discovery = ctx.discovery();
+        this.io = ctx.io();
+
+        if (!ctx.clientNode()) {
+            ctx.io().addMessageListener(GridTopic.TOPIC_EXCHANGE, (nodeId, msg, plc) -> {
+                if (msg instanceof LatchAckMessage) {
+                    processAck(nodeId, (LatchAckMessage) msg);
+                }
+            });
+
+            // First coordinator initialization.
+            ctx.discovery().localJoinFuture().listen(f -> {
+                this.coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+            });
+
+            ctx.event().addDiscoveryEventListener((e, cache) -> {
+                assert e != null;
+                assert e.type() == EVT_NODE_LEFT || e.type() == EVT_NODE_FAILED : this;
+
+                // Do not process from discovery thread.
+                ctx.closure().runLocalSafe(() -> processNodeLeft(e.eventNode()));
+            }, EVT_NODE_LEFT, EVT_NODE_FAILED);
+        }
+    }
+
+    /**
+     * Creates server latch with given {@code id} and {@code topVer}.
+     * Adds corresponding pending acks to it.
+     *
+     * @param id Latch id.
+     * @param topVer Latch topology version.
+     * @param participants Participant nodes.
+     * @return Server latch instance.
+     */
+    private Latch createServerLatch(String id, AffinityTopologyVersion topVer, Collection<ClusterNode> participants) {
+        final T2<String, AffinityTopologyVersion> latchId = new T2<>(id, topVer);
+
+        if (serverLatches.containsKey(latchId))
+            return serverLatches.get(latchId);
+
+        ServerLatch latch = new ServerLatch(id, topVer, participants);
+
+        serverLatches.put(latchId, latch);
+
+        if (log.isDebugEnabled())
+            log.debug("Server latch is created [latch=" + latchId + ", participantsSize=" + participants.size() + "]");
+
+        if (pendingAcks.containsKey(latchId)) {
+            Set<UUID> acks = pendingAcks.get(latchId);
+
+            for (UUID node : acks)
+                if (latch.hasParticipant(node) && !latch.hasAck(node))
+                    latch.ack(node);
+
+            pendingAcks.remove(latchId);
+        }
+
+        if (latch.isCompleted())
+            serverLatches.remove(latchId);
+
+        return latch;
+    }
+
+    /**
+     * Creates client latch.
+     * If there is final ack corresponds to given {@code id} and {@code topVer}, latch will be completed immediately.
+     *
+     * @param id Latch id.
+     * @param topVer Latch topology version.
+     * @param coordinator Coordinator node.
+     * @param participants Participant nodes.
+     * @return Client latch instance.
+     */
+    private Latch createClientLatch(String id, AffinityTopologyVersion topVer, ClusterNode coordinator, Collection<ClusterNode> participants) {
+        final T2<String, AffinityTopologyVersion> latchId = new T2<>(id, topVer);
+
+        if (clientLatches.containsKey(latchId))
+            return clientLatches.get(latchId);
+
+        ClientLatch latch = new ClientLatch(id, topVer, coordinator, participants);
+
+        if (log.isDebugEnabled())
+            log.debug("Client latch is created [latch=" + latchId
+                    + ", crd=" + coordinator
+                    + ", participantsSize=" + participants.size() + "]");
+
+        // There is final ack for created latch.
+        if (pendingAcks.containsKey(latchId)) {
+            latch.complete();
+            pendingAcks.remove(latchId);
+        }
+        else
+            clientLatches.put(latchId, latch);
+
+        return latch;
+    }
+
+    /**
+     * Creates new latch with specified {@code id} and {@code topVer} or returns existing latch.
+     *
+     * Participants of latch are calculated from given {@code topVer} as alive server nodes.
+     * If local node is coordinator {@code ServerLatch} instance will be created, otherwise {@code ClientLatch} instance.
+     *
+     * @param id Latch id.
+     * @param topVer Latch topology version.
+     * @return Latch instance.
+     */
+    public Latch getOrCreate(String id, AffinityTopologyVersion topVer) {
+        lock.lock();
+
+        try {
+            ClusterNode coordinator = getLatchCoordinator(topVer);
+
+            if (coordinator == null) {
+                ClientLatch latch = new ClientLatch(id, AffinityTopologyVersion.NONE, null, Collections.emptyList());
+                latch.complete();
+
+                return latch;
+            }
+
+            Collection<ClusterNode> participants = getLatchParticipants(topVer);
+
+            return coordinator.isLocal()
+                ? createServerLatch(id, topVer, participants)
+                : createClientLatch(id, topVer, coordinator, participants);
+        }
+        finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * @param topVer Latch topology version.
+     * @return Collection of alive server nodes with latch functionality.
+     */
+    private Collection<ClusterNode> getLatchParticipants(AffinityTopologyVersion topVer) {
+        Collection<ClusterNode> aliveNodes = topVer == AffinityTopologyVersion.NONE
+                ? discovery.aliveServerNodes()
+                : discovery.discoCache(topVer).aliveServerNodes();
+
+        return aliveNodes
+                .stream()
+                .filter(node -> node.version().compareTo(VERSION_SINCE) >= 0)
+                .collect(Collectors.toList());
+    }
+
+    /**
+     * @param topVer Latch topology version.
+     * @return Oldest alive server node with latch functionality.
+     */
+    @Nullable private ClusterNode getLatchCoordinator(AffinityTopologyVersion topVer) {
+        Collection<ClusterNode> aliveNodes = topVer == AffinityTopologyVersion.NONE
+                ? discovery.aliveServerNodes()
+                : discovery.discoCache(topVer).aliveServerNodes();
+
+        return aliveNodes
+                .stream()
+                .filter(node -> node.version().compareTo(VERSION_SINCE) >= 0)
+                .findFirst()
+                .orElse(null);
+    }
+
+    /**
+     * Processes ack message from given {@code from} node.
+     *
+     * Completes client latch in case of final ack message.
+     *
+     * If no latch is associated with message, ack is placed to {@link #pendingAcks} set.
+     *
+     * @param from Node sent ack.
+     * @param message Ack message.
+     */
+    private void processAck(UUID from, LatchAckMessage message) {
+        lock.lock();
+
+        try {
+            ClusterNode coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+
+            if (coordinator == null)
+                return;
+
+            T2<String, AffinityTopologyVersion> latchId = new T2<>(message.latchId(), message.topVer());
+
+            if (message.isFinal()) {
+                if (log.isDebugEnabled())
+                    log.debug("Process final ack [latch=" + latchId + ", from=" + from + "]");
+
+                if (clientLatches.containsKey(latchId)) {
+                    ClientLatch latch = clientLatches.remove(latchId);
+                    latch.complete();
+                }
+                else if (!coordinator.isLocal()) {
+                    pendingAcks.computeIfAbsent(latchId, (id) -> new GridConcurrentHashSet<>());
+                    pendingAcks.get(latchId).add(from);
+                }
+            } else {
+                if (log.isDebugEnabled())
+                    log.debug("Process ack [latch=" + latchId + ", from=" + from + "]");
+
+                if (serverLatches.containsKey(latchId)) {
+                    ServerLatch latch = serverLatches.get(latchId);
+
+                    if (latch.hasParticipant(from) && !latch.hasAck(from)) {
+                        latch.ack(from);
+
+                        if (latch.isCompleted())
+                            serverLatches.remove(latchId);
+                    }
+                }
+                else {
+                    pendingAcks.computeIfAbsent(latchId, (id) -> new GridConcurrentHashSet<>());
+                    pendingAcks.get(latchId).add(from);
+                }
+            }
+        }
+        finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * Changes coordinator to current local node.
+     * Restores all server latches from pending acks and own client latches.
+     */
+    private void becomeNewCoordinator() {
+        if (log.isInfoEnabled())
+            log.info("Become new coordinator " + coordinator.id());
+
+        List<T2<String, AffinityTopologyVersion>> latchesToRestore = new ArrayList<>();
+        latchesToRestore.addAll(pendingAcks.keySet());
+        latchesToRestore.addAll(clientLatches.keySet());
+
+        for (T2<String, AffinityTopologyVersion> latchId : latchesToRestore) {
+            String id = latchId.get1();
+            AffinityTopologyVersion topVer = latchId.get2();
+            Collection<ClusterNode> participants = getLatchParticipants(topVer);
+
+            if (!participants.isEmpty())
+                createServerLatch(id, topVer, participants);
+        }
+    }
+
+    /**
+     * Handles node left discovery event.
+     *
+     * Summary:
+     * Removes pending acks corresponds to the left node.
+     * Adds fake acknowledgements to server latches where such node was participant.
+     * Changes client latches coordinator to oldest available server node where such node was coordinator.
+     * Detects coordinator change.
+     *
+     * @param left Left node.
+     */
+    private void processNodeLeft(ClusterNode left) {
+        assert this.coordinator != null : "Coordinator is not initialized";
+
+        lock.lock();
+
+        try {
+            if (log.isDebugEnabled())
+                log.debug("Process node left " + left.id());
+
+            ClusterNode coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+
+            if (coordinator == null)
+                return;
+
+            // Clear pending acks.
+            for (Map.Entry<T2<String, AffinityTopologyVersion>, Set<UUID>> ackEntry : pendingAcks.entrySet())
+                if (ackEntry.getValue().contains(left.id()))
+                    pendingAcks.get(ackEntry.getKey()).remove(left.id());
+
+            // Change coordinator for client latches.
+            for (Map.Entry<T2<String, AffinityTopologyVersion>, ClientLatch> latchEntry : clientLatches.entrySet()) {
+                ClientLatch latch = latchEntry.getValue();
+                if (latch.hasCoordinator(left.id())) {
+                    // Change coordinator for latch and re-send ack if necessary.
+                    if (latch.hasParticipant(coordinator.id()))
+                        latch.newCoordinator(coordinator);
+                    else {
+                        /* If new coordinator is not able to take control on the latch,
+                           it means that all other latch participants are left from topology
+                           and there is no reason to track such latch. */
+                        AffinityTopologyVersion topVer = latchEntry.getKey().get2();
+
+                        assert getLatchParticipants(topVer).isEmpty();
+
+                        latch.complete(new IgniteCheckedException("All latch participants are left from topology."));
+                        clientLatches.remove(latchEntry.getKey());
+                    }
+                }
+            }
+
+            // Add acknowledgements from left node.
+            for (Map.Entry<T2<String, AffinityTopologyVersion>, ServerLatch> latchEntry : serverLatches.entrySet()) {
+                ServerLatch latch = latchEntry.getValue();
+
+                if (latch.hasParticipant(left.id()) && !latch.hasAck(left.id())) {
+                    if (log.isDebugEnabled())
+                        log.debug("Process node left [latch=" + latchEntry.getKey() + ", left=" + left.id() + "]");
+
+                    latch.ack(left.id());
+
+                    if (latch.isCompleted())
+                        serverLatches.remove(latchEntry.getKey());
+                }
+            }
+
+            // Coordinator is changed.
+            if (coordinator.isLocal() && this.coordinator.id() != coordinator.id()) {
+                this.coordinator = coordinator;
+
+                becomeNewCoordinator();
+            }
+        }
+        finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * Latch creating on coordinator node.
+     * Latch collects acks from participants: non-coordinator nodes and current local node.
+     * Latch completes when all acks from all participants are received.
+     *
+     * After latch completion final ack is sent to all participants.
+     */
+    class ServerLatch extends CompletableLatch {
+        /** Number of latch permits. This is needed to track number of countDown invocations. */
+        private final AtomicInteger permits;
+
+        /** Set of received acks. */
+        private final Set<UUID> acks = new GridConcurrentHashSet<>();
+
+        /**
+         * Constructor.
+         *
+         * @param id Latch id.
+         * @param topVer Latch topology version.
+         * @param participants Participant nodes.
+         */
+        ServerLatch(String id, AffinityTopologyVersion topVer, Collection<ClusterNode> participants) {
+            super(id, topVer, participants);
+            this.permits = new AtomicInteger(participants.size());
+
+            // Send final acks when latch is completed.
+            this.complete.listen(f -> {
+                for (ClusterNode node : participants) {
+                    try {
+                        if (discovery.alive(node)) {
+                            io.sendToGridTopic(node, GridTopic.TOPIC_EXCHANGE, new LatchAckMessage(id, topVer, true), GridIoPolicy.SYSTEM_POOL);
+
+                            if (log.isDebugEnabled())
+                                log.debug("Final ack is ackSent [latch=" + latchId() + ", to=" + node.id() + "]");
+                        }
+                    } catch (IgniteCheckedException e) {
+                        if (log.isDebugEnabled())
+                            log.debug("Unable to send final ack [latch=" + latchId() + ", to=" + node.id() + "]");
+                    }
+                }
+            });
+        }
+
+        /**
+         * Checks if latch has ack from given node.
+         *
+         * @param from Node.
+         * @return {@code true} if latch has ack from given node.
+         */
+        private boolean hasAck(UUID from) {
+            return acks.contains(from);
+        }
+
+        /**
+         * Receives ack from given node.
+         * Count downs latch if ack was not already processed.
+         *
+         * @param from Node.
+         */
+        private void ack(UUID from) {
+            if (log.isDebugEnabled())
+                log.debug("Ack is accepted [latch=" + latchId() + ", from=" + from + "]");
+
+            countDown0(from);
+        }
+
+        /**
+         * Count down latch from ack of given node.
+         * Completes latch if all acks are received.
+         *
+         * @param node Node.
+         */
+        private void countDown0(UUID node) {
+            if (isCompleted() || acks.contains(node))
+                return;
+
+            acks.add(node);
+
+            int remaining = permits.decrementAndGet();
+
+            if (log.isDebugEnabled())
+                log.debug("Count down + [latch=" + latchId() + ", remaining=" + remaining + "]");
+
+            if (remaining == 0)
+                complete();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void countDown() {
+            countDown0(ctx.localNodeId());
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            Set<UUID> pendingAcks = participants.stream().filter(ack -> !acks.contains(ack)).collect(Collectors.toSet());
+
+            return S.toString(ServerLatch.class, this,
+                    "pendingAcks", pendingAcks,
+                    "super", super.toString());
+        }
+    }
+
+    /**
+     * Latch creating on non-coordinator node.
+     * Latch completes when final ack from coordinator is received.
+     */
+    class ClientLatch extends CompletableLatch {
+        /** Latch coordinator node. Can be changed if coordinator is left from topology. */
+        private volatile ClusterNode coordinator;
+
+        /** Flag indicates that ack is sent to coordinator. */
+        private boolean ackSent;
+
+        /**
+         * Constructor.
+         *
+         * @param id Latch id.
+         * @param topVer Latch topology version.
+         * @param coordinator Coordinator node.
+         * @param participants Participant nodes.
+         */
+        ClientLatch(String id, AffinityTopologyVersion topVer, ClusterNode coordinator, Collection<ClusterNode> participants) {
+            super(id, topVer, participants);
+
+            this.coordinator = coordinator;
+        }
+
+        /**
+         * Checks if latch coordinator is given {@code node}.
+         *
+         * @param node Node.
+         * @return {@code true} if latch coordinator is given node.
+         */
+        private boolean hasCoordinator(UUID node) {
+            return coordinator.id().equals(node);
+        }
+
+        /**
+         * Changes coordinator of latch and resends ack to new coordinator if needed.
+         *
+         * @param coordinator New coordinator.
+         */
+        private void newCoordinator(ClusterNode coordinator) {
+            if (log.isDebugEnabled())
+                log.debug("Coordinator is changed [latch=" + latchId() + ", crd=" + coordinator.id() + "]");
+
+            synchronized (this) {
+                this.coordinator = coordinator;
+
+                // Resend ack to new coordinator.
+                if (ackSent)
+                    sendAck();
+            }
+        }
+
+        /**
+         * Sends ack to coordinator node.
+         * There is ack deduplication on coordinator. So it's fine to send same ack twice.
+         */
+        private void sendAck() {
+            try {
+                ackSent = true;
+
+                io.sendToGridTopic(coordinator, GridTopic.TOPIC_EXCHANGE, new LatchAckMessage(id, topVer, false), GridIoPolicy.SYSTEM_POOL);
+
+                if (log.isDebugEnabled())
+                    log.debug("Ack is ackSent + [latch=" + latchId() + ", to=" + coordinator.id() + "]");
+            } catch (IgniteCheckedException e) {
+                // Coordinator is unreachable. On coodinator node left discovery event ack will be resent.
+                if (log.isDebugEnabled())
+                    log.debug("Unable to send ack [latch=" + latchId() + ", to=" + coordinator.id() + "]: " + e.getMessage());
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void countDown() {
+            if (isCompleted())
+                return;
+
+            // Synchronize in case of changed coordinator.
+            synchronized (this) {
+                sendAck();
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(ClientLatch.class, this,
+                    "super", super.toString());
+        }
+    }
+
+    /**
+     * Base latch functionality with implemented complete / await logic.
+     */
+    private abstract static class CompletableLatch implements Latch {
+        /** Latch id. */
+        @GridToStringInclude
+        protected final String id;
+
+        /** Latch topology version. */
+        @GridToStringInclude
+        protected final AffinityTopologyVersion topVer;
+
+        /** Latch node participants. Only participant nodes are able to change state of latch. */
+        @GridToStringExclude
+        protected final Set<UUID> participants;
+
+        /** Future indicates that latch is completed. */
+        @GridToStringExclude
+        protected final GridFutureAdapter<?> complete = new GridFutureAdapter<>();
+
+        /**
+         * Constructor.
+         *
+         * @param id Latch id.
+         * @param topVer Latch topology version.
+         * @param participants Participant nodes.
+         */
+        CompletableLatch(String id, AffinityTopologyVersion topVer, Collection<ClusterNode> participants) {
+            this.id = id;
+            this.topVer = topVer;
+            this.participants = participants.stream().map(ClusterNode::id).collect(Collectors.toSet());
+        }
+
+        /** {@inheritDoc} */
+        @Override public void await() throws IgniteCheckedException {
+            complete.get();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void await(long timeout, TimeUnit timeUnit) throws IgniteCheckedException {
+            complete.get(timeout, timeUnit);
+        }
+
+        /**
+         * Checks if latch participants contain given {@code node}.
+         *
+         * @param node Node.
+         * @return {@code true} if latch participants contain given node.
+         */
+        boolean hasParticipant(UUID node) {
+            return participants.contains(node);
+        }
+
+        /**
+         * @return {@code true} if latch is completed.
+         */
+        boolean isCompleted() {
+            return complete.isDone();
+        }
+
+        /**
+         * Completes current latch.
+         */
+        void complete() {
+            complete.onDone();
+        }
+
+        /**
+         * Completes current latch with given {@code error}.
+         *
+         * @param error Error.
+         */
+        void complete(Throwable error) {
+            complete.onDone(error);
+        }
+
+        /**
+         * @return Full latch id.
+         */
+        String latchId() {
+            return id + "-" + topVer;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(CompletableLatch.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/Latch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/Latch.java
new file mode 100644
index 0000000000000..9704c2eacad7e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/Latch.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Simple distributed count down latch interface.
+ * Latch supports count down and await logic.
+ * Latch functionality is not relied on caches and has own state management {@link ExchangeLatchManager}.
+ */
+public interface Latch {
+    /**
+     * Decrements count on current latch.
+     * Release all latch waiters on all nodes if count reaches zero.
+     *
+     * This is idempotent operation. Invoking this method twice or more on the same node doesn't have any effect.
+     */
+    void countDown();
+
+    /**
+     * Awaits current latch completion.
+     *
+     * @throws IgniteCheckedException If await is failed.
+     */
+    void await() throws IgniteCheckedException;
+
+    /**
+     * Awaits current latch completion with specified timeout.
+     *
+     * @param timeout Timeout value.
+     * @param timeUnit Timeout time unit.
+     * @throws IgniteCheckedException If await is failed.
+     */
+    void await(long timeout, TimeUnit timeUnit) throws IgniteCheckedException;
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java
new file mode 100644
index 0000000000000..bad1b6137bac5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ * Message is used to send acks for {@link Latch} instances management.
+ */
+public class LatchAckMessage implements Message {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Latch id. */
+    private String latchId;
+
+    /** Latch topology version. */
+    private AffinityTopologyVersion topVer;
+
+    /** Flag indicates that ack is final. */
+    private boolean isFinal;
+
+    /**
+     * Constructor.
+     *
+     * @param latchId Latch id.
+     * @param topVer Latch topology version.
+     * @param isFinal Final acknowledgement flag.
+     */
+    public LatchAckMessage(String latchId, AffinityTopologyVersion topVer, boolean isFinal) {
+        this.latchId = latchId;
+        this.topVer = topVer;
+        this.isFinal = isFinal;
+    }
+
+    /**
+     * Empty constructor for marshalling purposes.
+     */
+    public LatchAckMessage() {
+    }
+
+    /**
+     * @return Latch id.
+     */
+    public String latchId() {
+        return latchId;
+    }
+
+    /**
+     * @return Latch topology version.
+     */
+    public AffinityTopologyVersion topVer() {
+        return topVer;
+    }
+
+    /**
+     * @return {@code} if ack is final.
+     */
+    public boolean isFinal() {
+        return isFinal;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 0:
+                if (!writer.writeBoolean("isFinal", isFinal))
+                    return false;
+
+                writer.incrementState();
+
+            case 1:
+                if (!writer.writeString("latchId", latchId))
+                    return false;
+
+                writer.incrementState();
+
+            case 2:
+                if (!writer.writeMessage("topVer", topVer))
+                    return false;
+
+                writer.incrementState();
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        switch (reader.state()) {
+            case 0:
+                isFinal = reader.readBoolean("isFinal");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 1:
+                latchId = reader.readString("latchId");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 2:
+                topVer = reader.readMessage("topVer");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+        }
+
+        return reader.afterMessageRead(LatchAckMessage.class);
+    }
+
+    /** {@inheritDoc} */
+    @Override public short directType() {
+        return 135;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 3;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void onAckReceived() {
+        // No-op.
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 77856055a0ca0..33f84f030db71 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -3525,6 +3525,11 @@ public IgniteInternalFuture<IgniteInternalTx> commitAsyncLocal() {
 
                 U.error(log, "Failed to prepare transaction: " + this, e);
             }
+            catch (Throwable t) {
+                fut.onDone(t);
+
+                throw t;
+            }
 
             if (err != null)
                 fut.rollbackOnError(err);
@@ -3544,6 +3549,11 @@ public IgniteInternalFuture<IgniteInternalTx> commitAsyncLocal() {
 
                         U.error(log, "Failed to prepare transaction: " + this, e);
                     }
+                    catch (Throwable t) {
+                        fut.onDone(t);
+
+                        throw t;
+                    }
 
                     if (err != null)
                         fut.rollbackOnError(err);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 5cfd92d269051..68ec83db1b4b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -189,7 +189,7 @@ private boolean saveStoreMetadata(
             freeList.saveMetadata();
 
             long updCntr = store.updateCounter();
-            int size = store.fullSize();
+            long size = store.fullSize();
             long rmvId = globalRemoveId().get();
 
             PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
@@ -318,7 +318,7 @@ else if (state == MOVING || state == RENTING) {
                                 partMetaId,
                                 updCntr,
                                 rmvId,
-                                size,
+                                (int)size, // TODO: Partition size may be long
                                 cntrsPageId,
                                 state == null ? -1 : (byte)state.ordinal(),
                                 pageCnt
@@ -549,7 +549,7 @@ private static boolean addPartition(
             final int grpId,
             final int partId,
             final int currAllocatedPageCnt,
-            final int partSize
+            final long partSize
     ) {
         if (part != null) {
             boolean reserved = part.reserve();
@@ -1301,7 +1301,7 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException {
         }
 
         /** {@inheritDoc} */
-        @Override public int fullSize() {
+        @Override public long fullSize() {
             try {
                 CacheDataStore delegate0 = init0(true);
 
@@ -1313,7 +1313,7 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException {
         }
 
         /** {@inheritDoc} */
-        @Override public int cacheSize(int cacheId) {
+        @Override public long cacheSize(int cacheId) {
             try {
                 CacheDataStore delegate0 = init0(true);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 9bfaaf3b59f96..945ef486fbe5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -490,7 +490,7 @@ protected void uncommit(boolean nodeStopping) {
     @Override public AffinityTopologyVersion topologyVersion() {
         AffinityTopologyVersion res = topVer;
 
-        if (res.equals(AffinityTopologyVersion.NONE)) {
+        if (res == null || res.equals(AffinityTopologyVersion.NONE)) {
             if (system()) {
                 AffinityTopologyVersion topVer = cctx.tm().lockedTopologyVersion(Thread.currentThread().getId(), this);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index fbdeca10eb46f..9fb87770df64d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -545,10 +545,10 @@ public GridNearTxLocal newTx(
      * @param topVer Topology version.
      * @return Future that will be completed when all ongoing transactions are finished.
      */
-    public IgniteInternalFuture<Boolean> finishTxs(AffinityTopologyVersion topVer) {
+    public IgniteInternalFuture<Boolean> finishLocalTxs(AffinityTopologyVersion topVer) {
         GridCompoundFuture<IgniteInternalTx, Boolean> res =
             new CacheObjectsReleaseFuture<>(
-                "Tx",
+                "LocalTx",
                 topVer,
                 new IgniteReducer<IgniteInternalTx, Boolean>() {
                     @Override public boolean collect(IgniteInternalTx e) {
@@ -561,8 +561,9 @@ public IgniteInternalFuture<Boolean> finishTxs(AffinityTopologyVersion topVer) {
                 });
 
         for (IgniteInternalTx tx : txs()) {
-            if (needWaitTransaction(tx, topVer))
+            if (needWaitTransaction(tx, topVer)) {
                 res.add(tx.finishFuture());
+            }
         }
 
         res.markInitialized();
@@ -570,6 +571,29 @@ public IgniteInternalFuture<Boolean> finishTxs(AffinityTopologyVersion topVer) {
         return res;
     }
 
+    /**
+     * Creates a future that will wait for finishing all tx updates on backups after all local transactions are finished.
+     *
+     * NOTE:
+     * As we send finish request to backup nodes after transaction successfully completed on primary node
+     * it's important to ensure that all updates from primary to backup are finished or at least remote transaction has created on backup node.
+     *
+     * @param finishLocalTxsFuture Local transactions finish future.
+     * @param topVer Topology version.
+     * @return Future that will be completed when all ongoing transactions are finished.
+     */
+    public IgniteInternalFuture<?> finishAllTxs(IgniteInternalFuture<?> finishLocalTxsFuture, AffinityTopologyVersion topVer) {
+        final GridCompoundFuture finishAllTxsFuture = new CacheObjectsReleaseFuture("AllTx", topVer);
+
+        // After finishing all local updates, wait for finishing all tx updates on backups.
+        finishLocalTxsFuture.listen(future -> {
+            finishAllTxsFuture.add(cctx.mvcc().finishRemoteTxs(topVer));
+            finishAllTxsFuture.markInitialized();
+        });
+
+        return finishAllTxsFuture;
+    }
+
     /**
      * @param tx Transaction.
      * @param topVer Exchange version.
@@ -1834,12 +1858,12 @@ public IgniteInternalFuture<Boolean> txCommitted(GridCacheVersion xidVer) {
      * @return Finish future for related remote transactions.
      */
     @SuppressWarnings("unchecked")
-    public IgniteInternalFuture<?> remoteTxFinishFuture(GridCacheVersion nearVer) {
-        GridCompoundFuture<Void, Void> fut = new GridCompoundFuture<>();
+    public IgniteInternalFuture<IgniteInternalTx> remoteTxFinishFuture(GridCacheVersion nearVer) {
+        GridCompoundFuture<IgniteInternalTx, IgniteInternalTx> fut = new GridCompoundFuture<>();
 
         for (final IgniteInternalTx tx : txs()) {
             if (!tx.local() && nearVer.equals(tx.nearXidVersion()))
-                fut.add((IgniteInternalFuture) tx.finishFuture());
+                fut.add(tx.finishFuture());
         }
 
         fut.markInitialized();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java
index 726365624570d..702b188bd4005 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java
@@ -76,7 +76,7 @@ public void testMemoryUsage() throws Exception {
             cache = grid(g).cache(DEFAULT_CACHE_NAME);
 
             for (GridDhtLocalPartition p : dht(cache).topology().localPartitions()) {
-                int size = p.dataStore().fullSize();
+                long size = p.dataStore().fullSize();
 
                 assertTrue("Unexpected size: " + size, size <= 32);
             }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
index 468bbc8bbc009..6c570d744ff0e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
@@ -86,6 +86,7 @@
 import org.apache.ignite.internal.util.lang.GridPlainCallable;
 import org.apache.ignite.internal.util.lang.gridfunc.ContainsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java
new file mode 100644
index 0000000000000..52cd0338a87d0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.datastructures;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.collect.Lists;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
+import org.apache.ignite.lang.IgniteBiClosure;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ * Tests for {@link ExchangeLatchManager} functionality when latch coordinator is failed.
+ */
+public class IgniteExchangeLatchManagerCoordinatorFailTest extends GridCommonAbstractTest {
+    /** */
+    private static final String LATCH_NAME = "test";
+
+    /** 5 nodes. */
+    private final AffinityTopologyVersion latchTopVer = new AffinityTopologyVersion(5, 0);
+
+    /** Wait before latch creation. */
+    private final IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean> beforeCreate = (mgr, syncLatch) -> {
+        try {
+            syncLatch.countDown();
+            syncLatch.await();
+
+            Latch distributedLatch = mgr.getOrCreate(LATCH_NAME, latchTopVer);
+
+            distributedLatch.countDown();
+
+            distributedLatch.await();
+        } catch (Exception e) {
+            log.error("Unexpected exception", e);
+
+            return false;
+        }
+
+        return true;
+    };
+
+    /** Wait before latch count down. */
+    private final IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean> beforeCountDown = (mgr, syncLatch) -> {
+        try {
+            Latch distributedLatch = mgr.getOrCreate(LATCH_NAME, latchTopVer);
+
+            syncLatch.countDown();
+            syncLatch.await();
+
+            distributedLatch.countDown();
+
+            distributedLatch.await();
+        } catch (Exception e) {
+            log.error("Unexpected exception ", e);
+
+            return false;
+        }
+
+        return true;
+    };
+
+    /** Wait after all operations are successful. */
+    private final IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean> all = (mgr, syncLatch) -> {
+        try {
+            Latch distributedLatch = mgr.getOrCreate(LATCH_NAME, latchTopVer);
+
+            distributedLatch.countDown();
+
+            syncLatch.countDown();
+
+            distributedLatch.await();
+
+            syncLatch.await();
+        } catch (Exception e) {
+            log.error("Unexpected exception ", e);
+
+            return false;
+        }
+
+        return true;
+    };
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * Test scenarios description:
+     *
+     * We have existing coordinator and 4 other nodes.
+     * Each node do following operations:
+     * 1) Create latch
+     * 2) Countdown latch
+     * 3) Await latch
+     *
+     * While nodes do the operations we shutdown coordinator and next oldest node become new coordinator.
+     * We should check that new coordinator properly restored latch and all nodes finished latch completion successfully after that.
+     *
+     * Each node before coordinator shutdown can be in 3 different states:
+     *
+     * State {@link #beforeCreate} - Node didn't create latch yet.
+     * State {@link #beforeCountDown} - Node created latch but didn't count down it yet.
+     * State {@link #all} - Node created latch and count downed it.
+     *
+     * We should check important cases when future coordinator is in one of these states, and other 3 nodes have 3 different states.
+     */
+
+    /**
+     * Scenario 1:
+     *
+     * Node 1 state -> {@link #beforeCreate}
+     * Node 2 state -> {@link #beforeCountDown}
+     * Node 3 state -> {@link #all}
+     * Node 4 state -> {@link #beforeCreate}
+     */
+    public void testCoordinatorFail1() throws Exception {
+        List<IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean>> nodeStates = Lists.newArrayList(
+            beforeCreate,
+            beforeCountDown,
+            all,
+            beforeCreate
+        );
+
+        doTestCoordinatorFail(nodeStates);
+    }
+
+    /**
+     * Scenario 2:
+     *
+     * Node 1 state -> {@link #beforeCountDown}
+     * Node 2 state -> {@link #beforeCountDown}
+     * Node 3 state -> {@link #all}
+     * Node 4 state -> {@link #beforeCreate}
+     */
+    public void testCoordinatorFail2() throws Exception {
+        List<IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean>> nodeStates = Lists.newArrayList(
+            beforeCountDown,
+            beforeCountDown,
+            all,
+            beforeCreate
+        );
+
+        doTestCoordinatorFail(nodeStates);
+    }
+
+    /**
+     * Scenario 3:
+     *
+     * Node 1 state -> {@link #all}
+     * Node 2 state -> {@link #beforeCountDown}
+     * Node 3 state -> {@link #all}
+     * Node 4 state -> {@link #beforeCreate}
+     */
+    public void testCoordinatorFail3() throws Exception {
+        List<IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean>> nodeStates = Lists.newArrayList(
+            all,
+            beforeCountDown,
+            all,
+            beforeCreate
+        );
+
+        doTestCoordinatorFail(nodeStates);
+    }
+
+    /**
+     * Test latch coordinator fail with specified scenarios.
+     *
+     * @param nodeScenarios Node scenarios.
+     * @throws Exception If failed.
+     */
+    private void doTestCoordinatorFail(List<IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean>> nodeScenarios) throws Exception {
+        IgniteEx crd = (IgniteEx) startGridsMultiThreaded(5);
+        crd.cluster().active(true);
+
+        // Latch to synchronize node states.
+        CountDownLatch syncLatch = new CountDownLatch(5);
+
+        GridCompoundFuture finishAllLatches = new GridCompoundFuture();
+
+        AtomicBoolean hasErrors = new AtomicBoolean();
+
+        for (int node = 1; node < 5; node++) {
+            IgniteEx grid = grid(node);
+            ExchangeLatchManager latchMgr = grid.context().cache().context().exchange().latch();
+            final int stateIdx = node - 1;
+
+            IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(() -> {
+                boolean success = nodeScenarios.get(stateIdx).apply(latchMgr, syncLatch);
+                if (!success)
+                    hasErrors.set(true);
+            }, 1, "latch-runner-" + node);
+
+            finishAllLatches.add(fut);
+        }
+
+        finishAllLatches.markInitialized();
+
+        // Wait while all nodes reaches their states.
+        while (syncLatch.getCount() != 1) {
+            Thread.sleep(10);
+
+            if (hasErrors.get())
+                throw new Exception("All nodes should complete latches without errors");
+        }
+
+        crd.close();
+
+        // Resume progress for all nodes.
+        syncLatch.countDown();
+
+        // Wait for distributed latch completion.
+        finishAllLatches.get(5000);
+
+        Assert.assertFalse("All nodes should complete latches without errors", hasErrors.get());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
new file mode 100644
index 0000000000000..63d772a0077c0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
@@ -0,0 +1,316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+/**
+ *
+ */
+public class GridCachePartitionsStateValidationTest extends GridCommonAbstractTest {
+    /** Cache name. */
+    private static final String CACHE_NAME = "cache";
+
+    /** */
+    private boolean clientMode;
+
+    /** {@inheritDoc */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        cfg.setCacheConfiguration(new CacheConfiguration(CACHE_NAME)
+                .setBackups(1)
+                .setAffinity(new RendezvousAffinityFunction(false, 32))
+        );
+
+        cfg.setCommunicationSpi(new SingleMessageInterceptorCommunicationSpi(2));
+
+        if (clientMode)
+            cfg.setClientMode(true);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        clientMode = false;
+    }
+
+    /**
+     * Test that partitions state validation works correctly.
+     *
+     * @throws Exception If failed.
+     */
+    public void testValidationIfPartitionCountersAreInconsistent() throws Exception {
+        IgniteEx ignite = (IgniteEx) startGrids(2);
+        ignite.cluster().active(true);
+
+        awaitPartitionMapExchange();
+
+        // Modify update counter for some partition.
+        for (GridDhtLocalPartition partition : ignite.cachex(CACHE_NAME).context().topology().localPartitions()) {
+            partition.updateCounter(100500L);
+            break;
+        }
+
+        // Trigger exchange.
+        startGrid(2);
+
+        awaitPartitionMapExchange();
+
+        // Nothing should happen (just log error message) and we're still able to put data to corrupted cache.
+        ignite.cache(CACHE_NAME).put(0, 0);
+
+        stopAllGrids();
+    }
+
+    /**
+     * Test that all nodes send correct {@link GridDhtPartitionsSingleMessage} with consistent update counters.
+     *
+     * @throws Exception If failed.
+     */
+    public void testPartitionCountersConsistencyOnExchange() throws Exception {
+        IgniteEx ignite = (IgniteEx) startGrids(4);
+        ignite.cluster().active(true);
+
+        awaitPartitionMapExchange();
+
+        final String atomicCacheName = "atomic-cache";
+        final String txCacheName = "tx-cache";
+
+        clientMode = true;
+
+        Ignite client = startGrid(4);
+
+        clientMode = false;
+
+        IgniteCache atomicCache = client.getOrCreateCache(new CacheConfiguration<>(atomicCacheName)
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setBackups(2)
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+        );
+
+        IgniteCache txCache = client.getOrCreateCache(new CacheConfiguration<>(txCacheName)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setBackups(2)
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+        );
+
+        for (int it = 0; it < 10; it++) {
+            SingleMessageInterceptorCommunicationSpi spi = (SingleMessageInterceptorCommunicationSpi) ignite.configuration().getCommunicationSpi();
+            spi.clear();
+
+            // Stop load future.
+            final AtomicBoolean stop = new AtomicBoolean();
+
+            // Run atomic load.
+            IgniteInternalFuture atomicLoadFuture = GridTestUtils.runMultiThreadedAsync(() -> {
+                int k = 0;
+
+                while (!stop.get()) {
+                    k++;
+                    try {
+                        atomicCache.put(k, k);
+                    } catch (Exception ignored) {}
+                }
+            }, 1, "atomic-load");
+
+            // Run tx load.
+            IgniteInternalFuture txLoadFuture = GridTestUtils.runMultiThreadedAsync(() -> {
+                final int txOps = 5;
+
+                while (!stop.get()) {
+                    List<Integer> randomKeys = Stream.generate(() -> ThreadLocalRandom.current().nextInt(5))
+                        .limit(txOps)
+                        .sorted()
+                        .collect(Collectors.toList());
+
+                    try (Transaction tx = ignite.transactions().txStart(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED)) {
+                        for (Integer key : randomKeys)
+                            txCache.put(key, key);
+
+                        tx.commit();
+                    }
+                    catch (Exception ignored) { }
+                }
+            }, 4, "tx-load");
+
+            // Wait for some data.
+            Thread.sleep(1000);
+
+            // Prevent sending full message.
+            spi.blockFullMessage();
+
+            // Trigger exchange.
+            IgniteInternalFuture nodeStopFuture = GridTestUtils.runAsync(() -> stopGrid(3));
+
+            try {
+                spi.waitUntilAllSingleMessagesAreSent();
+
+                List<GridDhtPartitionsSingleMessage> interceptedMessages = spi.getMessages();
+
+                // Associate each message with existing node UUID.
+                Map<UUID, GridDhtPartitionsSingleMessage> messagesMap = new HashMap<>();
+                for (int i = 0; i < interceptedMessages.size(); i++)
+                    messagesMap.put(grid(i + 1).context().localNodeId(), interceptedMessages.get(i));
+
+                GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(ignite.context().cache().context());
+
+                // Validate partition update counters. If counters are not consistent, exception will be thrown.
+                validator.validatePartitionsUpdateCounters(ignite.cachex(atomicCacheName).context().topology(), messagesMap, Collections.emptySet());
+                validator.validatePartitionsUpdateCounters(ignite.cachex(txCacheName).context().topology(), messagesMap, Collections.emptySet());
+
+            } finally {
+                // Stop load and resume exchange.
+                spi.unblockFullMessage();
+
+                stop.set(true);
+
+                atomicLoadFuture.get();
+                txLoadFuture.get();
+                nodeStopFuture.get();
+            }
+
+            // Return grid to initial state.
+            startGrid(3);
+
+            awaitPartitionMapExchange();
+        }
+    }
+
+    /**
+     * SPI which intercepts single messages during exchange.
+     */
+    private static class SingleMessageInterceptorCommunicationSpi extends TcpCommunicationSpi {
+        /** */
+        private static final List<GridDhtPartitionsSingleMessage> messages = new CopyOnWriteArrayList<>();
+
+        /** Future completes when {@link #singleMessagesThreshold} messages are sent to coordinator. */
+        private static final GridFutureAdapter allSingleMessagesSent = new GridFutureAdapter();
+
+        /** A number of single messages we're waiting for send. */
+        private final int singleMessagesThreshold;
+
+        /** Latch which blocks full message sending. */
+        private volatile CountDownLatch blockFullMsgLatch;
+
+        /**
+         * Constructor.
+         */
+        private SingleMessageInterceptorCommunicationSpi(int singleMessagesThreshold) {
+            this.singleMessagesThreshold = singleMessagesThreshold;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (((GridIoMessage) msg).message() instanceof GridDhtPartitionsSingleMessage) {
+                GridDhtPartitionsSingleMessage singleMsg = (GridDhtPartitionsSingleMessage) ((GridIoMessage) msg).message();
+
+                // We're interesting for only exchange messages and when node is stopped.
+                if (singleMsg.exchangeId() != null && singleMsg.exchangeId().isLeft() && !singleMsg.client()) {
+                    messages.add(singleMsg);
+
+                    if (messages.size() == singleMessagesThreshold)
+                        allSingleMessagesSent.onDone();
+                }
+            }
+
+            try {
+                if (((GridIoMessage) msg).message() instanceof GridDhtPartitionsFullMessage) {
+                    if (blockFullMsgLatch != null)
+                        blockFullMsgLatch.await();
+                }
+            }
+            catch (Exception ignored) { }
+
+            super.sendMessage(node, msg, ackC);
+        }
+
+        /** */
+        public void clear() {
+            messages.clear();
+            allSingleMessagesSent.reset();
+        }
+
+        /** */
+        public List<GridDhtPartitionsSingleMessage> getMessages() {
+            return Collections.unmodifiableList(messages);
+        }
+
+        /** */
+        public void blockFullMessage() {
+            blockFullMsgLatch = new CountDownLatch(1);
+        }
+
+        /** */
+        public void unblockFullMessage() {
+            blockFullMsgLatch.countDown();
+        }
+
+        /** */
+        public void waitUntilAllSingleMessagesAreSent() throws IgniteCheckedException {
+            allSingleMessagesSent.get();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
new file mode 100644
index 0000000000000..9ed8d54080a37
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+
+/**
+ * Test correct behaviour of {@link GridDhtPartitionsStateValidator} class.
+ */
+public class GridCachePartitionsStateValidatorSelfTest extends GridCommonAbstractTest {
+    /** Mocks and stubs. */
+    private final UUID localNodeId = UUID.randomUUID();
+    /** */
+    private GridCacheSharedContext cctxMock;
+    /** */
+    private GridDhtPartitionTopology topologyMock;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        // Prepare mocks.
+        cctxMock = Mockito.mock(GridCacheSharedContext.class);
+        Mockito.when(cctxMock.localNodeId()).thenReturn(localNodeId);
+
+        topologyMock = Mockito.mock(GridDhtPartitionTopology.class);
+        Mockito.when(topologyMock.partitionState(Matchers.any(), Matchers.anyInt())).thenReturn(GridDhtPartitionState.OWNING);
+        Mockito.when(topologyMock.groupId()).thenReturn(0);
+        Mockito.when(topologyMock.partitions()).thenReturn(3);
+
+        List<GridDhtLocalPartition> localPartitions = Lists.newArrayList(
+                partitionMock(0, 1, 1),
+                partitionMock(1, 2, 2),
+                partitionMock(2, 3, 3)
+        );
+        Mockito.when(topologyMock.localPartitions()).thenReturn(localPartitions);
+        Mockito.when(topologyMock.currentLocalPartitions()).thenReturn(localPartitions);
+    }
+
+    /**
+     * @return Partition mock with specified {@code id}, {@code updateCounter} and {@code size}.
+     */
+    private GridDhtLocalPartition partitionMock(int id, long updateCounter, long size) {
+        GridDhtLocalPartition partitionMock = Mockito.mock(GridDhtLocalPartition.class);
+        Mockito.when(partitionMock.id()).thenReturn(id);
+        Mockito.when(partitionMock.updateCounter()).thenReturn(updateCounter);
+        Mockito.when(partitionMock.fullSize()).thenReturn(size);
+        return partitionMock;
+    }
+
+    /**
+     * @return Message containing specified {@code countersMap}.
+     */
+    private GridDhtPartitionsSingleMessage fromUpdateCounters(Map<Integer, T2<Long, Long>> countersMap) {
+        GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage();
+        msg.addPartitionUpdateCounters(0, countersMap);
+        return msg;
+    }
+
+    /**
+     * @return Message containing specified {@code sizesMap}.
+     */
+    private GridDhtPartitionsSingleMessage fromCacheSizes(Map<Integer, Long> sizesMap) {
+        GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage();
+        msg.addPartitionSizes(0, sizesMap);
+        return msg;
+    }
+
+    /**
+     * Test partition update counters validation.
+     */
+    public void testPartitionCountersValidation() {
+        UUID remoteNode = UUID.randomUUID();
+        UUID ignoreNode = UUID.randomUUID();
+
+        // For partitions 0 and 2 (zero counter) we have inconsistent update counters.
+        Map<Integer, T2<Long, Long>> updateCountersMap = new HashMap<>();
+        updateCountersMap.put(0, new T2<>(2L, 2L));
+        updateCountersMap.put(1, new T2<>(2L, 2L));
+
+        // Form single messages map.
+        Map<UUID, GridDhtPartitionsSingleMessage> messages = new HashMap<>();
+        messages.put(remoteNode, fromUpdateCounters(updateCountersMap));
+        messages.put(ignoreNode, fromUpdateCounters(updateCountersMap));
+
+        GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(cctxMock);
+
+        // (partId, (nodeId, updateCounter))
+        Map<Integer, Map<UUID, Long>> result = validator.validatePartitionsUpdateCounters(topologyMock, messages, Sets.newHashSet(ignoreNode));
+
+        // Check that validation result contains all necessary information.
+        Assert.assertEquals(2, result.size());
+        Assert.assertTrue(result.containsKey(0));
+        Assert.assertTrue(result.containsKey(2));
+        Assert.assertTrue(result.get(0).get(localNodeId) == 1L);
+        Assert.assertTrue(result.get(0).get(remoteNode) == 2L);
+        Assert.assertTrue(result.get(2).get(localNodeId) == 3L);
+        Assert.assertTrue(result.get(2).get(remoteNode) == 0L);
+    }
+
+    /**
+     * Test partition cache sizes validation.
+     */
+    public void testPartitionCacheSizesValidation() {
+        UUID remoteNode = UUID.randomUUID();
+        UUID ignoreNode = UUID.randomUUID();
+
+        // For partitions 0 and 2 we have inconsistent cache sizes.
+        Map<Integer, Long> cacheSizesMap = new HashMap<>();
+        cacheSizesMap.put(0, 2L);
+        cacheSizesMap.put(1, 2L);
+        cacheSizesMap.put(2, 2L);
+
+        // Form single messages map.
+        Map<UUID, GridDhtPartitionsSingleMessage> messages = new HashMap<>();
+        messages.put(remoteNode, fromCacheSizes(cacheSizesMap));
+        messages.put(ignoreNode, fromCacheSizes(cacheSizesMap));
+
+        GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(cctxMock);
+
+        // (partId, (nodeId, cacheSize))
+        Map<Integer, Map<UUID, Long>> result = validator.validatePartitionsSizes(topologyMock, messages, Sets.newHashSet(ignoreNode));
+
+        // Check that validation result contains all necessary information.
+        Assert.assertEquals(2, result.size());
+        Assert.assertTrue(result.containsKey(0));
+        Assert.assertTrue(result.containsKey(2));
+        Assert.assertTrue(result.get(0).get(localNodeId) == 1L);
+        Assert.assertTrue(result.get(0).get(remoteNode) == 2L);
+        Assert.assertTrue(result.get(2).get(localNodeId) == 3L);
+        Assert.assertTrue(result.get(2).get(remoteNode) == 2L);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticOnPartitionExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticOnPartitionExchangeTest.java
new file mode 100644
index 0000000000000..03ea0f7f97d49
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticOnPartitionExchangeTest.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager;
+import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.T1;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
+
+/**
+ *
+ */
+public class TxOptimisticOnPartitionExchangeTest extends GridCommonAbstractTest {
+    /** Nodes count. */
+    private static final int NODES_CNT = 3;
+
+    /** Tx size. */
+    private static final int TX_SIZE = 20 * NODES_CNT;
+
+    /** Cache name. */
+    private static final String CACHE_NAME = "cache";
+
+    /** Logger started. */
+    private static volatile boolean msgInterception;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGrids(NODES_CNT);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setCommunicationSpi(new TestCommunicationSpi(log()));
+
+        cfg.setCacheConfiguration(defaultCacheConfiguration()
+            .setName(CACHE_NAME)
+            .setAtomicityMode(TRANSACTIONAL)
+            .setWriteSynchronizationMode(FULL_SYNC)
+            .setCacheMode(PARTITIONED)
+            .setBackups(1));
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testConsistencyOnPartitionExchange() throws Exception {
+        doTest(SERIALIZABLE, true);
+        doTest(READ_COMMITTED, true);
+        doTest(SERIALIZABLE, false);
+        doTest(READ_COMMITTED, false);
+    }
+
+    /**
+     * @param isolation {@link TransactionIsolation}.
+     * @param txInitiatorPrimary False If the transaction does not use the keys of the node that initiated it.
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    public void doTest(final TransactionIsolation isolation, boolean txInitiatorPrimary) throws Exception {
+        final CountDownLatch txStarted = new CountDownLatch(1);
+
+        final IgniteCache cache = ignite(0).cache(CACHE_NAME);
+
+        final Map<Integer, Integer> txValues = new TreeMap<>();
+
+        ClusterNode node = ignite(0).cluster().node();
+
+        GridCacheAffinityManager affinity = ((IgniteCacheProxy)cache).context().affinity();
+
+        for (int i = 0; txValues.size() < TX_SIZE; i++) {
+            if (!txInitiatorPrimary && node.equals(affinity.primaryByKey(i, NONE)))
+                continue;
+
+            txValues.put(i, i);
+        }
+
+        TestCommunicationSpi.init();
+
+        msgInterception = true;
+
+        IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() {
+                try (Transaction tx = ignite(0).transactions().txStart(OPTIMISTIC, isolation)) {
+                    info(">>> TX started.");
+
+                    txStarted.countDown();
+
+                    cache.putAll(txValues);
+
+                    tx.commit();
+
+                    info(">>> TX committed.");
+                }
+
+                return null;
+            }
+        });
+
+        txStarted.await();
+
+        try {
+            info(">>> Grid starting.");
+
+            IgniteEx ignite = startGrid(NODES_CNT);
+
+            info(">>> Grid started.");
+
+            fut.get();
+
+            awaitPartitionMapExchange();
+
+            msgInterception = false;
+
+            IgniteCache<Object, Object> cacheStartedNode = ignite.cache(CACHE_NAME);
+
+            try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                Set<Object> keys = cacheStartedNode.getAll(txValues.keySet()).keySet();
+
+                assertEquals(txValues.keySet(), new TreeSet<>(keys));
+
+                tx.commit();
+            }
+        }
+        finally {
+            msgInterception = false;
+
+            stopGrid(NODES_CNT);
+        }
+    }
+
+    /**
+     *
+     */
+    @SuppressWarnings("ConstantConditions")
+    private static class TestCommunicationSpi extends TcpCommunicationSpi {
+        /** Partition single message sent from added node. */
+        private static volatile CountDownLatch partSingleMsgSentFromAddedNode;
+
+        /** Partition supply message sent count. */
+        private static final AtomicInteger partSupplyMsgSentCnt = new AtomicInteger();
+
+        /** Logger. */
+        private IgniteLogger log;
+
+        /**
+         * @param log Logger.
+         */
+        public TestCommunicationSpi(IgniteLogger log) {
+            this.log = log;
+        }
+
+        /**
+         *
+         */
+        public static void init() {
+            partSingleMsgSentFromAddedNode = new CountDownLatch(1);
+
+            partSupplyMsgSentCnt.set(0);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(
+            final ClusterNode node,
+            final Message msg,
+            final IgniteInClosure<IgniteException> ackC
+        ) throws IgniteSpiException {
+            if (msgInterception) {
+                if (msg instanceof GridIoMessage) {
+                    final Message msg0 = ((GridIoMessage)msg).message();
+
+                    String locNodeId = ((IgniteEx)ignite).context().localNodeId().toString();
+
+                    int nodeIdx = Integer.parseInt(locNodeId.substring(locNodeId.length() - 3));
+
+                    if (nodeIdx == 0) {
+                        if (msg0 instanceof GridNearTxPrepareRequest || msg0 instanceof GridDhtTxPrepareRequest) {
+                            GridTestUtils.runAsync(new Callable<Void>() {
+                                @Override public Void call() throws Exception {
+                                    partSingleMsgSentFromAddedNode.await();
+
+                                    sendMessage(node, msg, ackC, true);
+
+                                    return null;
+                                }
+                            });
+
+                            return;
+
+                        }
+                        else if (msg0 instanceof GridNearTxFinishRequest || msg0 instanceof GridDhtTxFinishRequest) {
+                            GridTestUtils.runAsync(new Callable<Void>() {
+                                @Override public Void call() throws Exception {
+                                    final T1<Integer> i = new T1<>(0);
+
+                                    while (waitForCondition(new GridAbsPredicate() {
+                                        @Override public boolean apply() {
+                                            return partSupplyMsgSentCnt.get() > i.get();
+                                        }
+                                    }, i.get() == 0 ? 5_000 : 500))
+                                        i.set(partSupplyMsgSentCnt.get());
+
+                                    sendMessage(node, msg, ackC, true);
+
+                                    return null;
+                                }
+                            });
+
+                            return;
+                        }
+                    }
+                    else if (nodeIdx == NODES_CNT && msg0 instanceof GridDhtPartitionsSingleMessage)
+                        partSingleMsgSentFromAddedNode.countDown();
+
+                    if (msg0 instanceof GridDhtPartitionSupplyMessage)
+                        partSupplyMsgSentCnt.incrementAndGet();
+                }
+            }
+
+            sendMessage(node, msg, ackC, msgInterception);
+        }
+
+        /**
+         * @param node Node.
+         * @param msg Message.
+         * @param ackC Ack closure.
+         * @param logMsg Log Messages.
+         */
+        private void sendMessage(
+            final ClusterNode node,
+            final Message msg,
+            final IgniteInClosure<IgniteException> ackC,
+            boolean logMsg
+        ) throws IgniteSpiException {
+            if (logMsg) {
+                String id = node.id().toString();
+                String locNodeId = ((IgniteEx)ignite).context().localNodeId().toString();
+
+                Message msg0 = ((GridIoMessage)msg).message();
+
+                log.info(
+                    String.format(">>> Output msg[type=%s, fromNode= %s, toNode=%s]",
+                        msg0.getClass().getSimpleName(),
+                        locNodeId.charAt(locNodeId.length() - 1),
+                        id.charAt(id.length() - 1)
+                    )
+                );
+            }
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index bb397f7214355..06126151271e9 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -133,6 +133,8 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheAtomicNearCacheSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheColocatedTxExceptionSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheGlobalLoadTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionsStateValidationTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionsStateValidatorSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheGetStoreErrorSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearTxExceptionSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedTxExceptionSelfTest;
@@ -292,6 +294,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(IgniteCacheSystemTransactionsSelfTest.class);
         suite.addTestSuite(CacheDeferredDeleteSanitySelfTest.class);
         suite.addTestSuite(CacheDeferredDeleteQueueTest.class);
+        suite.addTestSuite(GridCachePartitionsStateValidatorSelfTest.class);
+        suite.addTestSuite(GridCachePartitionsStateValidationTest.class);
 
         suite.addTest(IgniteCacheTcpClientDiscoveryTestSuite.suite());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index f8add30ef5535..415479dae4138 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.internal.processors.cache.WalModeChangeAdvancedSelfTest;
 import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
 import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
+import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchangeLatchManagerCoordinatorFailTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest;
 import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest;
@@ -40,6 +41,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgnitePessimisticTxSuspendResumeTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticOnPartitionExchangeTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNearCacheTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest;
@@ -93,6 +95,10 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(PartitionedTransactionalOptimisticCacheGetsDistributionTest.class);
         suite.addTestSuite(PartitionedTransactionalPessimisticCacheGetsDistributionTest.class);
 
+        suite.addTestSuite(TxOptimisticOnPartitionExchangeTest.class);
+
+        suite.addTestSuite(IgniteExchangeLatchManagerCoordinatorFailTest.class);
+
         return suite;
     }
 }

From 9abfee69aa153888456f9e8574ece1f2d0cbe4d9 Mon Sep 17 00:00:00 2001
From: dmitrievanthony <dmitrievanthony@gmail.com>
Date: Tue, 10 Apr 2018 12:46:43 +0300
Subject: [PATCH 020/543] IGNITE-8059: Integrate decision tree with partition
 based dataset.

this closes #3760

(cherry picked from commit 139c2af)
---
 ...isionTreeClassificationTrainerExample.java | 147 +++++
 .../DecisionTreeRegressionTrainerExample.java | 124 ++++
 .../ml/{trees => tree}/package-info.java      |   2 +-
 .../ml/trees/DecisionTreesExample.java        | 354 -----------
 .../java/org/apache/ignite/ml/Trainer.java    |   3 -
 .../apache/ignite/ml/tree/DecisionTree.java   | 252 ++++++++
 .../DecisionTreeClassificationTrainer.java    |  93 +++
 .../ml/tree/DecisionTreeConditionalNode.java  |  78 +++
 .../DecisionTreeLeafNode.java}                |  35 +-
 .../ignite/ml/tree/DecisionTreeNode.java      |  26 +
 .../tree/DecisionTreeRegressionTrainer.java   |  60 ++
 .../TreeFilter.java}                          |  21 +-
 .../ignite/ml/tree/data/DecisionTreeData.java | 128 ++++
 .../ml/tree/data/DecisionTreeDataBuilder.java |  73 +++
 .../ignite/ml/tree/data/package-info.java     |  22 +
 .../ml/tree/impurity/ImpurityMeasure.java     |  55 ++
 .../impurity/ImpurityMeasureCalculator.java   |  38 ++
 .../impurity/gini/GiniImpurityMeasure.java    | 115 ++++
 .../gini/GiniImpurityMeasureCalculator.java   | 110 ++++
 .../impurity/gini}/package-info.java          |   4 +-
 .../tree/impurity/mse/MSEImpurityMeasure.java | 133 ++++
 .../mse/MSEImpurityMeasureCalculator.java     |  80 +++
 .../impurity/mse}/package-info.java           |   4 +-
 .../ignite/ml/tree/impurity/package-info.java |  22 +
 .../util/SimpleStepFunctionCompressor.java    | 149 +++++
 .../ml/tree/impurity/util/StepFunction.java   | 162 +++++
 .../impurity/util/StepFunctionCompressor.java |  55 ++
 .../ml/tree/impurity/util/package-info.java   |  22 +
 .../leaf/DecisionTreeLeafBuilder.java}        |  32 +-
 .../leaf/MeanDecisionTreeLeafBuilder.java     |  73 +++
 .../MostCommonDecisionTreeLeafBuilder.java    |  86 +++
 .../regcalcs => tree/leaf}/package-info.java  |   4 +-
 .../ml/{trees => tree}/package-info.java      |   4 +-
 .../ml/trees/CategoricalRegionInfo.java       |  72 ---
 .../ignite/ml/trees/CategoricalSplitInfo.java |  68 ---
 .../ignite/ml/trees/ContinuousRegionInfo.java |  74 ---
 .../ml/trees/ContinuousSplitCalculator.java   |  51 --
 .../apache/ignite/ml/trees/RegionInfo.java    |  62 --
 .../ml/trees/nodes/CategoricalSplitNode.java  |  50 --
 .../ml/trees/nodes/ContinuousSplitNode.java   |  56 --
 .../ignite/ml/trees/nodes/SplitNode.java      | 100 ---
 .../trees/trainers/columnbased/BiIndex.java   | 113 ----
 ...edCacheColumnDecisionTreeTrainerInput.java |  57 --
 .../CacheColumnDecisionTreeTrainerInput.java  | 141 -----
 .../ColumnDecisionTreeTrainer.java            | 568 ------------------
 .../ColumnDecisionTreeTrainerInput.java       |  55 --
 .../MatrixColumnDecisionTreeTrainerInput.java |  83 ---
 .../columnbased/RegionProjection.java         | 109 ----
 .../trainers/columnbased/TrainingContext.java | 166 -----
 .../columnbased/caches/ContextCache.java      |  68 ---
 .../columnbased/caches/FeaturesCache.java     | 151 -----
 .../columnbased/caches/ProjectionsCache.java  | 286 ---------
 .../columnbased/caches/SplitCache.java        | 206 -------
 .../columnbased/caches/package-info.java      |  22 -
 .../ContinuousSplitCalculators.java           |  34 --
 .../contsplitcalcs/GiniSplitCalculator.java   | 234 --------
 .../VarianceSplitCalculator.java              | 179 ------
 .../contsplitcalcs/package-info.java          |  22 -
 .../trainers/columnbased/package-info.java    |  22 -
 .../regcalcs/RegionCalculators.java           |  85 ---
 .../vectors/CategoricalFeatureProcessor.java  | 212 -------
 .../vectors/ContinuousFeatureProcessor.java   | 111 ----
 .../vectors/ContinuousSplitInfo.java          |  71 ---
 .../columnbased/vectors/FeatureProcessor.java |  82 ---
 .../vectors/FeatureVectorProcessorUtils.java  |  57 --
 .../columnbased/vectors/SampleInfo.java       |  80 ---
 .../columnbased/vectors/SplitInfo.java        | 106 ----
 .../columnbased/vectors/package-info.java     |  22 -
 .../apache/ignite/ml/IgniteMLTestSuite.java   |   4 +-
 .../ml/nn/performance/MnistMLPTestUtil.java   |   9 +-
 ...eClassificationTrainerIntegrationTest.java | 100 +++
 ...DecisionTreeClassificationTrainerTest.java |  91 +++
 ...nTreeRegressionTrainerIntegrationTest.java | 100 +++
 .../DecisionTreeRegressionTrainerTest.java    |  91 +++
 .../ignite/ml/tree/DecisionTreeTestSuite.java |  48 ++
 .../ml/tree/data/DecisionTreeDataTest.java    |  59 ++
 .../GiniImpurityMeasureCalculatorTest.java    | 103 ++++
 .../gini/GiniImpurityMeasureTest.java         | 131 ++++
 .../mse/MSEImpurityMeasureCalculatorTest.java |  59 ++
 .../impurity/mse/MSEImpurityMeasureTest.java  | 109 ++++
 .../SimpleStepFunctionCompressorTest.java     |  75 +++
 .../tree/impurity/util/StepFunctionTest.java  |  71 +++
 .../impurity/util/TestImpurityMeasure.java    |  88 +++
 .../DecisionTreeMNISTIntegrationTest.java     | 105 ++++
 .../performance/DecisionTreeMNISTTest.java    |  74 +++
 .../ignite/ml/trees/BaseDecisionTreeTest.java |  70 ---
 .../trees/ColumnDecisionTreeTrainerTest.java  | 191 ------
 .../ml/trees/DecisionTreesTestSuite.java      |  33 -
 .../ml/trees/GiniSplitCalculatorTest.java     | 141 -----
 .../ignite/ml/trees/SplitDataGenerator.java   | 390 ------------
 .../ml/trees/VarianceSplitCalculatorTest.java |  84 ---
 .../ColumnDecisionTreeTrainerBenchmark.java   | 456 --------------
 ...IgniteColumnDecisionTreeGiniBenchmark.java |  70 ---
 ...teColumnDecisionTreeVarianceBenchmark.java |  71 ---
 .../ml/trees/SplitDataGenerator.java          | 426 -------------
 .../yardstick/ml/trees/package-info.java      |  22 -
 96 files changed, 3465 insertions(+), 6247 deletions(-)
 create mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
 create mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
 rename examples/src/main/java/org/apache/ignite/examples/ml/{trees => tree}/package-info.java (95%)
 delete mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
 rename modules/ml/src/main/java/org/apache/ignite/ml/{trees/nodes/Leaf.java => tree/DecisionTreeLeafNode.java} (61%)
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
 rename modules/ml/src/main/java/org/apache/ignite/ml/{trees/nodes/DecisionTreeNode.java => tree/TreeFilter.java} (59%)
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java
 rename modules/ml/src/main/java/org/apache/ignite/ml/{trees/nodes => tree/impurity/gini}/package-info.java (89%)
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java
 rename modules/ml/src/main/java/org/apache/ignite/ml/{trees/models => tree/impurity/mse}/package-info.java (88%)
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java
 rename modules/ml/src/main/java/org/apache/ignite/ml/{trees/models/DecisionTreeModel.java => tree/leaf/DecisionTreeLeafBuilder.java} (55%)
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java
 rename modules/ml/src/main/java/org/apache/ignite/ml/{trees/trainers/columnbased/regcalcs => tree/leaf}/package-info.java (90%)
 rename modules/ml/src/main/java/org/apache/ignite/ml/{trees => tree}/package-info.java (92%)
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java

diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
new file mode 100644
index 0000000000000..cef63683cb4ec
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.tree;
+
+import java.util.Random;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Example of using distributed {@link DecisionTreeClassificationTrainer}.
+ */
+public class DecisionTreeClassificationTrainerExample {
+    /**
+     * Executes example.
+     *
+     * @param args Command line arguments, none required.
+     */
+    public static void main(String... args) throws InterruptedException {
+        System.out.println(">>> Decision tree classification trainer example started.");
+
+        // Start ignite grid.
+        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+            System.out.println(">>> Ignite grid started.");
+
+            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+                DecisionTreeClassificationTrainerExample.class.getSimpleName(), () -> {
+
+                // Create cache with training data.
+                CacheConfiguration<Integer, LabeledPoint> trainingSetCfg = new CacheConfiguration<>();
+                trainingSetCfg.setName("TRAINING_SET");
+                trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+                IgniteCache<Integer, LabeledPoint> trainingSet = ignite.createCache(trainingSetCfg);
+
+                Random rnd = new Random(0);
+
+                // Fill training data.
+                for (int i = 0; i < 1000; i++)
+                    trainingSet.put(i, generatePoint(rnd));
+
+                // Create classification trainer.
+                DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(4, 0);
+
+                // Train decision tree model.
+                DecisionTreeNode mdl = trainer.fit(
+                    new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+                    (k, v) -> new double[]{v.x, v.y},
+                    (k, v) -> v.lb
+                );
+
+                // Calculate score.
+                int correctPredictions = 0;
+                for (int i = 0; i < 1000; i++) {
+                    LabeledPoint pnt = generatePoint(rnd);
+
+                    double prediction = mdl.apply(new double[]{pnt.x, pnt.y});
+
+                    if (prediction == pnt.lb)
+                        correctPredictions++;
+                }
+
+                System.out.println(">>> Accuracy: " + correctPredictions / 10.0 + "%");
+
+                System.out.println(">>> Decision tree classification trainer example completed.");
+            });
+
+            igniteThread.start();
+
+            igniteThread.join();
+        }
+    }
+
+    /**
+     * Generate point with {@code x} in (-0.5, 0.5) and {@code y} in the same interval. If {@code x * y > 0} then label
+     * is 1, otherwise 0.
+     *
+     * @param rnd Random.
+     * @return Point with label.
+     */
+    private static LabeledPoint generatePoint(Random rnd) {
+
+        double x = rnd.nextDouble() - 0.5;
+        double y = rnd.nextDouble() - 0.5;
+
+        return new LabeledPoint(x, y, x * y > 0 ? 1 : 0);
+    }
+
+    /** Point data class. */
+    private static class Point {
+        /** X coordinate. */
+        final double x;
+
+        /** Y coordinate. */
+        final double y;
+
+        /**
+         * Constructs a new instance of point.
+         *
+         * @param x X coordinate.
+         * @param y Y coordinate.
+         */
+        Point(double x, double y) {
+            this.x = x;
+            this.y = y;
+        }
+    }
+
+    /** Labeled point data class. */
+    private static class LabeledPoint extends Point {
+        /** Point label. */
+        final double lb;
+
+        /**
+         * Constructs a new instance of labeled point data.
+         *
+         * @param x X coordinate.
+         * @param y Y coordinate.
+         * @param lb Point label.
+         */
+        LabeledPoint(double x, double y, double lb) {
+            super(x, y);
+            this.lb = lb;
+        }
+    }
+}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
new file mode 100644
index 0000000000000..61ba5f9dca471
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.tree;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.ml.tree.DecisionTreeRegressionTrainer;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Example of using distributed {@link DecisionTreeRegressionTrainer}.
+ */
+public class DecisionTreeRegressionTrainerExample {
+    /**
+     * Executes example.
+     *
+     * @param args Command line arguments, none required.
+     */
+    public static void main(String... args) throws InterruptedException {
+        System.out.println(">>> Decision tree regression trainer example started.");
+
+        // Start ignite grid.
+        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+            System.out.println(">>> Ignite grid started.");
+
+            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+                DecisionTreeRegressionTrainerExample.class.getSimpleName(), () -> {
+
+                // Create cache with training data.
+                CacheConfiguration<Integer, Point> trainingSetCfg = new CacheConfiguration<>();
+                trainingSetCfg.setName("TRAINING_SET");
+                trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+                IgniteCache<Integer, Point> trainingSet = ignite.createCache(trainingSetCfg);
+
+                // Fill training data.
+                generatePoints(trainingSet);
+
+                // Create regression trainer.
+                DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(10, 0);
+
+                // Train decision tree model.
+                DecisionTreeNode mdl = trainer.fit(
+                    new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+                    (k, v) -> new double[] {v.x},
+                    (k, v) -> v.y
+                );
+
+                System.out.println(">>> Linear regression model: " + mdl);
+
+                System.out.println(">>> ---------------------------------");
+                System.out.println(">>> | Prediction\t| Ground Truth\t|");
+                System.out.println(">>> ---------------------------------");
+
+                // Calculate score.
+                for (int x = 0; x < 10; x++) {
+                    double predicted = mdl.apply(new double[] {x});
+
+                    System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", predicted, Math.sin(x));
+                }
+
+                System.out.println(">>> ---------------------------------");
+
+                System.out.println(">>> Decision tree regression trainer example completed.");
+            });
+
+            igniteThread.start();
+
+            igniteThread.join();
+        }
+    }
+
+    /**
+     * Generates {@code sin(x)} on interval [0, 10) and loads into the specified cache.
+     */
+    private static void generatePoints(IgniteCache<Integer, Point> trainingSet) {
+        for (int i = 0; i < 1000; i++) {
+            double x = i / 100.0;
+            double y = Math.sin(x);
+
+            trainingSet.put(i, new Point(x, y));
+        }
+    }
+
+    /** Point data class. */
+    private static class Point {
+        /** X coordinate. */
+        final double x;
+
+        /** Y coordinate. */
+        final double y;
+
+        /**
+         * Constructs a new instance of point.
+         *
+         * @param x X coordinate.
+         * @param y Y coordinate.
+         */
+        Point(double x, double y) {
+            this.x = x;
+            this.y = y;
+        }
+    }
+}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java
similarity index 95%
rename from examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java
rename to examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java
index d944f60570b0b..d8d9de60a2169 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/trees/package-info.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/package-info.java
@@ -19,4 +19,4 @@
  * <!-- Package description. -->
  * Decision trees examples.
  */
-package org.apache.ignite.examples.ml.trees;
+package org.apache.ignite.examples.ml.tree;
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java
deleted file mode 100644
index b1b2c421a9f5f..0000000000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/trees/DecisionTreesExample.java
+++ /dev/null
@@ -1,354 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.trees;
-
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.channels.Channels;
-import java.nio.channels.ReadableByteChannel;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Random;
-import java.util.Scanner;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import java.util.zip.GZIPInputStream;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.examples.ml.MLExamplesCommonArgs;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.estimators.Estimators;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndex;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndexedCacheColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.GiniSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.ml.util.MnistUtils;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * <p>
- * Example of usage of decision trees algorithm for MNIST dataset
- * (it can be found here: http://yann.lecun.com/exdb/mnist/). </p>
- * <p>
- * Remote nodes should always be started with special configuration file which
- * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
- * with {@code examples/config/example-ignite.xml} configuration.</p>
- * <p>
- * It is recommended to start at least one node prior to launching this example if you intend
- * to run it with default memory settings.</p>
- * <p>
- * This example should be run with program arguments, for example
- * -cfg examples/config/example-ignite.xml.</p>
- * <p>
- * -cfg specifies path to a config path.</p>
- */
-public class DecisionTreesExample {
-    /** Name of parameter specifying path of Ignite config. */
-    private static final String CONFIG = "cfg";
-
-    /** Default config path. */
-    private static final String DEFAULT_CONFIG = "examples/config/example-ignite.xml";
-
-    /**
-     * Folder in which MNIST dataset is expected.
-     */
-    private static String MNIST_DIR = "examples/src/main/resources/";
-
-    /**
-     * Key for MNIST training images.
-     */
-    private static String MNIST_TRAIN_IMAGES = "train_images";
-
-    /**
-     * Key for MNIST training labels.
-     */
-    private static String MNIST_TRAIN_LABELS = "train_labels";
-
-    /**
-     * Key for MNIST test images.
-     */
-    private static String MNIST_TEST_IMAGES = "test_images";
-
-    /**
-     * Key for MNIST test labels.
-     */
-    private static String MNIST_TEST_LABELS = "test_labels";
-
-    /**
-     * Launches example.
-     *
-     * @param args Program arguments.
-     */
-    public static void main(String[] args) throws IOException {
-        System.out.println(">>> Decision trees example started.");
-
-        String igniteCfgPath;
-
-        CommandLineParser parser = new BasicParser();
-
-        String trainingImagesPath;
-        String trainingLabelsPath;
-
-        String testImagesPath;
-        String testLabelsPath;
-
-        Map<String, String> mnistPaths = new HashMap<>();
-
-        mnistPaths.put(MNIST_TRAIN_IMAGES, "train-images-idx3-ubyte");
-        mnistPaths.put(MNIST_TRAIN_LABELS, "train-labels-idx1-ubyte");
-        mnistPaths.put(MNIST_TEST_IMAGES, "t10k-images-idx3-ubyte");
-        mnistPaths.put(MNIST_TEST_LABELS, "t10k-labels-idx1-ubyte");
-
-        try {
-            // Parse the command line arguments.
-            CommandLine line = parser.parse(buildOptions(), args);
-
-            if (line.hasOption(MLExamplesCommonArgs.UNATTENDED)) {
-                System.out.println(">>> Skipped example execution because 'unattended' mode is used.");
-                System.out.println(">>> Decision trees example finished.");
-                return;
-            }
-
-            igniteCfgPath = line.getOptionValue(CONFIG, DEFAULT_CONFIG);
-        }
-        catch (ParseException e) {
-            e.printStackTrace();
-            return;
-        }
-
-        if (!getMNIST(mnistPaths.values())) {
-            System.out.println(">>> You should have MNIST dataset in " + MNIST_DIR + " to run this example.");
-            return;
-        }
-
-        trainingImagesPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
-            mnistPaths.get(MNIST_TRAIN_IMAGES))).getPath();
-        trainingLabelsPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
-            mnistPaths.get(MNIST_TRAIN_LABELS))).getPath();
-        testImagesPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
-            mnistPaths.get(MNIST_TEST_IMAGES))).getPath();
-        testLabelsPath = Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" +
-            mnistPaths.get(MNIST_TEST_LABELS))).getPath();
-
-        try (Ignite ignite = Ignition.start(igniteCfgPath)) {
-            IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-            int ptsCnt = 60000;
-            int featCnt = 28 * 28;
-
-            Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnistAsStream(trainingImagesPath, trainingLabelsPath,
-                new Random(123L), ptsCnt);
-
-            Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnistAsStream(testImagesPath, testLabelsPath,
-                new Random(123L), 10_000);
-
-            IgniteCache<BiIndex, Double> cache = createBiIndexedCache(ignite);
-
-            loadVectorsIntoBiIndexedCache(cache.getName(), trainingMnistStream.iterator(), featCnt + 1, ignite);
-
-            ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer = new ColumnDecisionTreeTrainer<>(10,
-                ContinuousSplitCalculators.GINI.apply(ignite),
-                RegionCalculators.GINI,
-                RegionCalculators.MOST_COMMON,
-                ignite);
-
-            System.out.println(">>> Training started");
-            long before = System.currentTimeMillis();
-            DecisionTreeModel mdl = trainer.train(new BiIndexedCacheColumnDecisionTreeTrainerInput(cache, new HashMap<>(), ptsCnt, featCnt));
-            System.out.println(">>> Training finished in " + (System.currentTimeMillis() - before));
-
-            IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse =
-                Estimators.errorsPercentage();
-
-            Double accuracy = mse.apply(mdl, testMnistStream.map(v ->
-                new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
-
-            System.out.println(">>> Errs percentage: " + accuracy);
-        }
-        catch (IOException e) {
-            e.printStackTrace();
-        }
-
-        System.out.println(">>> Decision trees example finished.");
-    }
-
-    /**
-     * Get MNIST dataset. Value of predicate 'MNIST dataset is present in expected folder' is returned.
-     *
-     * @param mnistFileNames File names of MNIST dataset.
-     * @return Value of predicate 'MNIST dataset is present in expected folder'.
-     * @throws IOException In case of file system errors.
-     */
-    private static boolean getMNIST(Collection<String> mnistFileNames) throws IOException {
-        List<String> missing = mnistFileNames.stream().
-            filter(f -> IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" + f) == null).
-            collect(Collectors.toList());
-
-        if (!missing.isEmpty()) {
-            System.out.println(">>> You have not fully downloaded MNIST dataset in directory " + MNIST_DIR +
-                ", do you want it to be downloaded? [y]/n");
-            Scanner s = new Scanner(System.in);
-            String str = s.nextLine();
-
-            if (!str.isEmpty() && !str.toLowerCase().equals("y"))
-                return false;
-        }
-
-        for (String s : missing) {
-            String f = s + ".gz";
-            System.out.println(">>> Downloading " + f + "...");
-            URL website = new URL("http://yann.lecun.com/exdb/mnistAsStream/" + f);
-            ReadableByteChannel rbc = Channels.newChannel(website.openStream());
-            FileOutputStream fos = new FileOutputStream(MNIST_DIR + "/" + f);
-            fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
-            System.out.println(">>> Done.");
-
-            System.out.println(">>> Unzipping " + f + "...");
-            unzip(MNIST_DIR + "/" + f, MNIST_DIR + "/" + s);
-
-            System.out.println(">>> Deleting gzip " + f + ", status: " +
-                Objects.requireNonNull(IgniteUtils.resolveIgnitePath(MNIST_DIR + "/" + f)).delete());
-
-            System.out.println(">>> Done.");
-        }
-
-        return true;
-    }
-
-    /**
-     * Unzip file located in {@code input} to {@code output}.
-     *
-     * @param input Input file path.
-     * @param output Output file path.
-     * @throws IOException In case of file system errors.
-     */
-    private static void unzip(String input, String output) throws IOException {
-        byte[] buf = new byte[1024];
-
-        try (GZIPInputStream gis = new GZIPInputStream(new FileInputStream(input));
-             FileOutputStream out = new FileOutputStream(output)) {
-            int sz;
-            while ((sz = gis.read(buf)) > 0)
-                out.write(buf, 0, sz);
-        }
-    }
-
-    /**
-     * Build cli options.
-     */
-    @NotNull private static Options buildOptions() {
-        Options options = new Options();
-
-        Option cfgOpt = OptionBuilder
-            .withArgName(CONFIG)
-            .withLongOpt(CONFIG)
-            .hasArg()
-            .withDescription("Path to the config.")
-            .isRequired(false).create();
-
-        Option unattended = OptionBuilder
-            .withArgName(MLExamplesCommonArgs.UNATTENDED)
-            .withLongOpt(MLExamplesCommonArgs.UNATTENDED)
-            .withDescription("Is example run unattended.")
-            .isRequired(false).create();
-
-        options.addOption(cfgOpt);
-        options.addOption(unattended);
-
-        return options;
-    }
-
-    /**
-     * Creates cache where data for training is stored.
-     *
-     * @param ignite Ignite instance.
-     * @return cache where data for training is stored.
-     */
-    private static IgniteCache<BiIndex, Double> createBiIndexedCache(Ignite ignite) {
-        CacheConfiguration<BiIndex, Double> cfg = new CacheConfiguration<>();
-
-        // Write to primary.
-        cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
-        // No copying of values.
-        cfg.setCopyOnRead(false);
-
-        cfg.setName("TMP_BI_INDEXED_CACHE");
-
-        return ignite.getOrCreateCache(cfg);
-    }
-
-    /**
-     * Loads vectors into cache.
-     *
-     * @param cacheName Name of cache.
-     * @param vectorsIter Iterator over vectors to load.
-     * @param vectorSize Size of vector.
-     * @param ignite Ignite instance.
-     */
-    private static void loadVectorsIntoBiIndexedCache(String cacheName, Iterator<? extends Vector> vectorsIter,
-        int vectorSize, Ignite ignite) {
-        try (IgniteDataStreamer<BiIndex, Double> streamer =
-                 ignite.dataStreamer(cacheName)) {
-            int sampleIdx = 0;
-
-            streamer.perNodeBufferSize(10000);
-
-            while (vectorsIter.hasNext()) {
-                org.apache.ignite.ml.math.Vector next = vectorsIter.next();
-
-                for (int i = 0; i < vectorSize; i++)
-                    streamer.addData(new BiIndex(sampleIdx, i), next.getX(i));
-
-                sampleIdx++;
-
-                if (sampleIdx % 1000 == 0)
-                    System.out.println(">>> Loaded " + sampleIdx + " vectors.");
-            }
-        }
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
index 4e0a5704473ef..f53b80110cb4c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
@@ -17,11 +17,8 @@
 
 package org.apache.ignite.ml;
 
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-
 /**
  * Interface for Trainers. Trainer is just a function which produces model from the data.
- * See for example {@link ColumnDecisionTreeTrainer}.
  *
  * @param <M> Type of produced model.
  * @param <T> Type of data needed for model producing.
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
new file mode 100644
index 0000000000000..c0b88fc05a4a9
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTree.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.trainers.DatasetTrainer;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.data.DecisionTreeDataBuilder;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionCompressor;
+import org.apache.ignite.ml.tree.leaf.DecisionTreeLeafBuilder;
+
+/**
+ * Distributed decision tree trainer that allows to fit trees using row-partitioned dataset.
+ *
+ * @param <T> Type of impurity measure.
+ */
+abstract class DecisionTree<T extends ImpurityMeasure<T>> implements DatasetTrainer<DecisionTreeNode, Double> {
+    /** Max tree deep. */
+    private final int maxDeep;
+
+    /** Min impurity decrease. */
+    private final double minImpurityDecrease;
+
+    /** Step function compressor. */
+    private final StepFunctionCompressor<T> compressor;
+
+    /** Decision tree leaf builder. */
+    private final DecisionTreeLeafBuilder decisionTreeLeafBuilder;
+
+    /**
+     * Constructs a new distributed decision tree trainer.
+     *
+     * @param maxDeep Max tree deep.
+     * @param minImpurityDecrease Min impurity decrease.
+     * @param compressor Impurity function compressor.
+     * @param decisionTreeLeafBuilder Decision tree leaf builder.
+     */
+    DecisionTree(int maxDeep, double minImpurityDecrease, StepFunctionCompressor<T> compressor, DecisionTreeLeafBuilder decisionTreeLeafBuilder) {
+        this.maxDeep = maxDeep;
+        this.minImpurityDecrease = minImpurityDecrease;
+        this.compressor = compressor;
+        this.decisionTreeLeafBuilder = decisionTreeLeafBuilder;
+    }
+
+    /** {@inheritDoc} */
+    @Override public <K, V> DecisionTreeNode fit(DatasetBuilder<K, V> datasetBuilder,
+        IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+        try (Dataset<EmptyContext, DecisionTreeData> dataset = datasetBuilder.build(
+            new EmptyContextBuilder<>(),
+            new DecisionTreeDataBuilder<>(featureExtractor, lbExtractor)
+        )) {
+            return split(dataset, e -> true, 0, getImpurityMeasureCalculator(dataset));
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Returns impurity measure calculator.
+     *
+     * @param dataset Dataset.
+     * @return Impurity measure calculator.
+     */
+    abstract ImpurityMeasureCalculator<T> getImpurityMeasureCalculator(Dataset<EmptyContext, DecisionTreeData> dataset);
+
+    /**
+     * Splits the node specified by the given dataset and predicate and returns decision tree node.
+     *
+     * @param dataset Dataset.
+     * @param filter Decision tree node predicate.
+     * @param deep Current tree deep.
+     * @param impurityCalc Impurity measure calculator.
+     * @return Decision tree node.
+     */
+    private DecisionTreeNode split(Dataset<EmptyContext, DecisionTreeData> dataset, TreeFilter filter, int deep,
+        ImpurityMeasureCalculator<T> impurityCalc) {
+        if (deep >= maxDeep)
+            return decisionTreeLeafBuilder.createLeafNode(dataset, filter);
+
+        StepFunction<T>[] criterionFunctions = calculateImpurityForAllColumns(dataset, filter, impurityCalc);
+
+        if (criterionFunctions == null)
+            return decisionTreeLeafBuilder.createLeafNode(dataset, filter);
+
+        SplitPoint splitPnt = calculateBestSplitPoint(criterionFunctions);
+
+        if (splitPnt == null)
+            return decisionTreeLeafBuilder.createLeafNode(dataset, filter);
+
+        return new DecisionTreeConditionalNode(
+            splitPnt.col,
+            splitPnt.threshold,
+            split(dataset, updatePredicateForThenNode(filter, splitPnt), deep + 1, impurityCalc),
+            split(dataset, updatePredicateForElseNode(filter, splitPnt), deep + 1, impurityCalc)
+        );
+    }
+
+    /**
+     * Calculates impurity measure functions for all columns for the node specified by the given dataset and predicate.
+     *
+     * @param dataset Dataset.
+     * @param filter Decision tree node predicate.
+     * @param impurityCalc Impurity measure calculator.
+     * @return Array of impurity measure functions for all columns.
+     */
+    private StepFunction<T>[] calculateImpurityForAllColumns(Dataset<EmptyContext, DecisionTreeData> dataset,
+        TreeFilter filter, ImpurityMeasureCalculator<T> impurityCalc) {
+        return dataset.compute(
+            part -> {
+                if (compressor != null)
+                    return compressor.compress(impurityCalc.calculate(part.filter(filter)));
+                else
+                    return impurityCalc.calculate(part.filter(filter));
+            }, this::reduce
+        );
+    }
+
+    /**
+     * Calculates best split point.
+     *
+     * @param criterionFunctions Array of impurity measure functions for all columns.
+     * @return Best split point.
+     */
+    private SplitPoint calculateBestSplitPoint(StepFunction<T>[] criterionFunctions) {
+        SplitPoint<T> res = null;
+
+        for (int col = 0; col < criterionFunctions.length; col++) {
+            StepFunction<T> criterionFunctionForCol = criterionFunctions[col];
+
+            double[] arguments = criterionFunctionForCol.getX();
+            T[] values = criterionFunctionForCol.getY();
+
+            for (int leftSize = 1; leftSize < values.length - 1; leftSize++) {
+                if ((values[0].impurity() - values[leftSize].impurity()) > minImpurityDecrease
+                    && (res == null || values[leftSize].compareTo(res.val) < 0))
+                    res = new SplitPoint<>(values[leftSize], col, calculateThreshold(arguments, leftSize));
+            }
+        }
+
+        return res;
+    }
+
+    /**
+     * Merges two arrays gotten from two partitions.
+     *
+     * @param a First step function.
+     * @param b Second step function.
+     * @return Merged step function.
+     */
+    private StepFunction<T>[] reduce(StepFunction<T>[] a, StepFunction<T>[] b) {
+        if (a == null)
+            return b;
+        if (b == null)
+            return a;
+        else {
+            StepFunction<T>[] res = Arrays.copyOf(a, a.length);
+
+            for (int i = 0; i < res.length; i++)
+                res[i] = res[i].add(b[i]);
+
+            return res;
+        }
+    }
+
+    /**
+     * Calculates threshold based on the given step function arguments and split point (specified left size).
+     *
+     * @param arguments Step function arguments.
+     * @param leftSize Split point (left size).
+     * @return Threshold.
+     */
+    private double calculateThreshold(double[] arguments, int leftSize) {
+        return (arguments[leftSize] + arguments[leftSize + 1]) / 2.0;
+    }
+
+    /**
+     * Constructs a new predicate for "then" node based on the parent node predicate and split point.
+     *
+     * @param filter Parent node predicate.
+     * @param splitPnt Split point.
+     * @return Predicate for "then" node.
+     */
+    private TreeFilter updatePredicateForThenNode(TreeFilter filter, SplitPoint splitPnt) {
+        return filter.and(f -> f[splitPnt.col] > splitPnt.threshold);
+    }
+
+    /**
+     * Constructs a new predicate for "else" node based on the parent node predicate and split point.
+     *
+     * @param filter Parent node predicate.
+     * @param splitPnt Split point.
+     * @return Predicate for "else" node.
+     */
+    private TreeFilter updatePredicateForElseNode(TreeFilter filter, SplitPoint splitPnt) {
+        return filter.and(f -> f[splitPnt.col] <= splitPnt.threshold);
+    }
+
+    /**
+     * Util class that represents split point.
+     */
+    private static class SplitPoint<T extends ImpurityMeasure<T>> implements Serializable {
+        /** */
+        private static final long serialVersionUID = -1758525953544425043L;
+
+        /** Split point impurity measure value. */
+        private final T val;
+
+        /** Column. */
+        private final int col;
+
+        /** Threshold. */
+        private final double threshold;
+
+        /**
+         * Constructs a new instance of split point.
+         *
+         * @param val Split point impurity measure value.
+         * @param col Column.
+         * @param threshold Threshold.
+         */
+        SplitPoint(T val, int col, double threshold) {
+            this.val = val;
+            this.col = col;
+            this.threshold = threshold;
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
new file mode 100644
index 0000000000000..ce75190583e90
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainer.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasure;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionCompressor;
+import org.apache.ignite.ml.tree.leaf.MostCommonDecisionTreeLeafBuilder;
+
+/**
+ * Decision tree classifier based on distributed decision tree trainer that allows to fit trees using row-partitioned
+ * dataset.
+ */
+public class DecisionTreeClassificationTrainer extends DecisionTree<GiniImpurityMeasure> {
+    /**
+     * Constructs a new decision tree classifier with default impurity function compressor.
+     *
+     * @param maxDeep Max tree deep.
+     * @param minImpurityDecrease Min impurity decrease.
+     */
+    public DecisionTreeClassificationTrainer(int maxDeep, double minImpurityDecrease) {
+        this(maxDeep, minImpurityDecrease, null);
+    }
+
+    /**
+     * Constructs a new instance of decision tree classifier.
+     *
+     * @param maxDeep Max tree deep.
+     * @param minImpurityDecrease Min impurity decrease.
+     */
+    public DecisionTreeClassificationTrainer(int maxDeep, double minImpurityDecrease,
+        StepFunctionCompressor<GiniImpurityMeasure> compressor) {
+        super(maxDeep, minImpurityDecrease, compressor, new MostCommonDecisionTreeLeafBuilder());
+    }
+
+    /** {@inheritDoc} */
+    @Override ImpurityMeasureCalculator<GiniImpurityMeasure> getImpurityMeasureCalculator(
+        Dataset<EmptyContext, DecisionTreeData> dataset) {
+        Set<Double> labels = dataset.compute(part -> {
+
+            if (part.getLabels() != null) {
+                Set<Double> list = new HashSet<>();
+
+                for (double lb : part.getLabels())
+                    list.add(lb);
+
+                return list;
+            }
+
+            return null;
+        }, (a, b) -> {
+            if (a == null)
+                return b;
+            else if (b == null)
+                return a;
+            else {
+                a.addAll(b);
+                return a;
+            }
+        });
+
+        Map<Double, Integer> encoder = new HashMap<>();
+
+        int idx = 0;
+        for (Double lb : labels)
+            encoder.put(lb, idx++);
+
+        return new GiniImpurityMeasureCalculator(encoder);
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
new file mode 100644
index 0000000000000..98182399a5603
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeConditionalNode.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+/**
+ * Decision tree conditional (non-leaf) node.
+ */
+public class DecisionTreeConditionalNode implements DecisionTreeNode {
+    /** */
+    private static final long serialVersionUID = 981630737007982172L;
+
+    /** Column of the value to be tested. */
+    private final int col;
+
+    /** Threshold. */
+    private final double threshold;
+
+    /** Node that will be used in case tested value is greater then threshold. */
+    private final DecisionTreeNode thenNode;
+
+    /** Node that will be used in case tested value is not greater then threshold. */
+    private final DecisionTreeNode elseNode;
+
+    /**
+     * Constructs a new instance of decision tree conditional node.
+     *
+     * @param col Column of the value to be tested.
+     * @param threshold Threshold.
+     * @param thenNode Node that will be used in case tested value is greater then threshold.
+     * @param elseNode Node that will be used in case tested value is not greater then threshold.
+     */
+    DecisionTreeConditionalNode(int col, double threshold, DecisionTreeNode thenNode, DecisionTreeNode elseNode) {
+        this.col = col;
+        this.threshold = threshold;
+        this.thenNode = thenNode;
+        this.elseNode = elseNode;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Double apply(double[] features) {
+        return features[col] > threshold ? thenNode.apply(features) : elseNode.apply(features);
+    }
+
+    /** */
+    public int getCol() {
+        return col;
+    }
+
+    /** */
+    public double getThreshold() {
+        return threshold;
+    }
+
+    /** */
+    public DecisionTreeNode getThenNode() {
+        return thenNode;
+    }
+
+    /** */
+    public DecisionTreeNode getElseNode() {
+        return elseNode;
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
similarity index 61%
rename from modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
index 79b441f3ac77b..4c6369ddd569f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/Leaf.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeLeafNode.java
@@ -15,35 +15,34 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
+package org.apache.ignite.ml.tree;
 
 /**
- * Terminal node of the decision tree.
+ * Decision tree leaf node which contains value.
  */
-public class Leaf implements DecisionTreeNode {
-    /**
-     * Value in subregion represented by this node.
-     */
+public class DecisionTreeLeafNode implements DecisionTreeNode {
+    /** */
+    private static final long serialVersionUID = -472145568088482206L;
+
+    /** Value of the node. */
     private final double val;
 
     /**
-     * Construct the leaf of decision tree.
+     * Constructs a new decision tree leaf node.
      *
-     * @param val Value in subregion represented by this node.
+     * @param val Value of the node.
      */
-    public Leaf(double val) {
+    public DecisionTreeLeafNode(double val) {
         this.val = val;
     }
 
-    /**
-     * Return value in subregion represented by this node.
-     *
-     * @param v Vector.
-     * @return Value in subregion represented by this node.
-     */
-    @Override public double process(Vector v) {
+    /** {@inheritDoc} */
+    @Override public Double apply(double[] doubles) {
+        return val;
+    }
+
+    /** */
+    public double getVal() {
         return val;
     }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java
new file mode 100644
index 0000000000000..94878eb4610e6
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeNode.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import org.apache.ignite.ml.Model;
+
+/**
+ * Base interface for decision tree nodes.
+ */
+public interface DecisionTreeNode extends Model<double[], Double> {
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
new file mode 100644
index 0000000000000..2bf09d32be39c
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainer.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasure;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionCompressor;
+import org.apache.ignite.ml.tree.leaf.MeanDecisionTreeLeafBuilder;
+
+/**
+ * Decision tree regressor based on distributed decision tree trainer that allows to fit trees using row-partitioned
+ * dataset.
+ */
+public class DecisionTreeRegressionTrainer extends DecisionTree<MSEImpurityMeasure> {
+    /**
+     * Constructs a new decision tree regressor with default impurity function compressor.
+     *
+     * @param maxDeep Max tree deep.
+     * @param minImpurityDecrease Min impurity decrease.
+     */
+    public DecisionTreeRegressionTrainer(int maxDeep, double minImpurityDecrease) {
+        this(maxDeep, minImpurityDecrease, null);
+    }
+
+    /**
+     * Constructs a new decision tree regressor.
+     *
+     * @param maxDeep Max tree deep.
+     * @param minImpurityDecrease Min impurity decrease.
+     */
+    public DecisionTreeRegressionTrainer(int maxDeep, double minImpurityDecrease,
+        StepFunctionCompressor<MSEImpurityMeasure> compressor) {
+        super(maxDeep, minImpurityDecrease, compressor, new MeanDecisionTreeLeafBuilder());
+    }
+
+    /** {@inheritDoc} */
+    @Override ImpurityMeasureCalculator<MSEImpurityMeasure> getImpurityMeasureCalculator(
+        Dataset<EmptyContext, DecisionTreeData> dataset) {
+        return new MSEImpurityMeasureCalculator();
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java
similarity index 59%
rename from modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java
index d31623d42a002..3e4dc00da7270 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/DecisionTreeNode.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/TreeFilter.java
@@ -15,19 +15,24 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml.trees.nodes;
+package org.apache.ignite.ml.tree;
 
-import org.apache.ignite.ml.math.Vector;
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.function.Predicate;
 
 /**
- * Node of decision tree.
+ * Predicate used to define objects that placed in decision tree node.
  */
-public interface DecisionTreeNode {
+public interface TreeFilter extends Predicate<double[]>, Serializable {
     /**
-     * Assign the double value to the given vector.
+     * Returns a composed predicate.
      *
-     * @param v Vector.
-     * @return Value assigned to the given vector.
+     * @param other Predicate that will be logically-ANDed with this predicate.
+     * @return Returns a composed predicate
      */
-    double process(Vector v);
+    default TreeFilter and(TreeFilter other) {
+        Objects.requireNonNull(other);
+        return (t) -> test(t) && other.test(t);
+    }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
new file mode 100644
index 0000000000000..34deb46cc9e91
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeData.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.data;
+
+import org.apache.ignite.ml.tree.TreeFilter;
+
+/**
+ * A partition {@code data} of the containing matrix of features and vector of labels stored in heap.
+ */
+public class DecisionTreeData implements AutoCloseable {
+    /** Matrix with features. */
+    private final double[][] features;
+
+    /** Vector with labels. */
+    private final double[] labels;
+
+    /**
+     * Constructs a new instance of decision tree data.
+     *
+     * @param features Matrix with features.
+     * @param labels Vector with labels.
+     */
+    public DecisionTreeData(double[][] features, double[] labels) {
+        assert features.length == labels.length : "Features and labels have to be the same length";
+
+        this.features = features;
+        this.labels = labels;
+    }
+
+    /**
+     * Filters objects and returns only data that passed filter.
+     *
+     * @param filter Filter.
+     * @return Data passed filter.
+     */
+    public DecisionTreeData filter(TreeFilter filter) {
+        int size = 0;
+
+        for (int i = 0; i < features.length; i++)
+            if (filter.test(features[i]))
+                size++;
+
+        double[][] newFeatures = new double[size][];
+        double[] newLabels = new double[size];
+
+        int ptr = 0;
+
+        for (int i = 0; i < features.length; i++) {
+            if (filter.test(features[i])) {
+                newFeatures[ptr] = features[i];
+                newLabels[ptr] = labels[i];
+
+                ptr++;
+            }
+        }
+
+        return new DecisionTreeData(newFeatures, newLabels);
+    }
+
+    /**
+     * Sorts data by specified column in ascending order.
+     *
+     * @param col Column.
+     */
+    public void sort(int col) {
+        sort(col, 0, features.length - 1);
+    }
+
+    /** */
+    private void sort(int col, int from, int to) {
+        if (from < to) {
+            double pivot = features[(from + to) / 2][col];
+
+            int i = from, j = to;
+
+            while (i <= j) {
+                while (features[i][col] < pivot) i++;
+                while (features[j][col] > pivot) j--;
+
+                if (i <= j) {
+                    double[] tmpFeature = features[i];
+                    features[i] = features[j];
+                    features[j] = tmpFeature;
+
+                    double tmpLb = labels[i];
+                    labels[i] = labels[j];
+                    labels[j] = tmpLb;
+
+                    i++;
+                    j--;
+                }
+            }
+
+            sort(col, from, j);
+            sort(col, i, to);
+        }
+    }
+
+    /** */
+    public double[][] getFeatures() {
+        return features;
+    }
+
+    /** */
+    public double[] getLabels() {
+        return labels;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close() {
+        // Do nothing, GC will clean up.
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java
new file mode 100644
index 0000000000000..67109ae9bc81e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/DecisionTreeDataBuilder.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.data;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
+import org.apache.ignite.ml.dataset.UpstreamEntry;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+
+/**
+ * A partition {@code data} builder that makes {@link DecisionTreeData}.
+ *
+ * @param <K> Type of a key in <tt>upstream</tt> data.
+ * @param <V> Type of a value in <tt>upstream</tt> data.
+ * @param <C> Type of a partition <tt>context</tt>.
+ */
+public class DecisionTreeDataBuilder<K, V, C extends Serializable>
+    implements PartitionDataBuilder<K, V, C, DecisionTreeData> {
+    /** */
+    private static final long serialVersionUID = 3678784980215216039L;
+
+    /** Function that extracts features from an {@code upstream} data. */
+    private final IgniteBiFunction<K, V, double[]> featureExtractor;
+
+    /** Function that extracts labels from an {@code upstream} data. */
+    private final IgniteBiFunction<K, V, Double> lbExtractor;
+
+    /**
+     * Constructs a new instance of decision tree data builder.
+     *
+     * @param featureExtractor Function that extracts features from an {@code upstream} data.
+     * @param lbExtractor Function that extracts labels from an {@code upstream} data.
+     */
+    public DecisionTreeDataBuilder(IgniteBiFunction<K, V, double[]> featureExtractor,
+        IgniteBiFunction<K, V, Double> lbExtractor) {
+        this.featureExtractor = featureExtractor;
+        this.lbExtractor = lbExtractor;
+    }
+
+    /** {@inheritDoc} */
+    @Override public DecisionTreeData build(Iterator<UpstreamEntry<K, V>> upstreamData, long upstreamDataSize, C ctx) {
+        double[][] features = new double[Math.toIntExact(upstreamDataSize)][];
+        double[] labels = new double[Math.toIntExact(upstreamDataSize)];
+
+        int ptr = 0;
+        while (upstreamData.hasNext()) {
+            UpstreamEntry<K, V> entry = upstreamData.next();
+
+            features[ptr] = featureExtractor.apply(entry.getKey(), entry.getValue());
+            labels[ptr] = lbExtractor.apply(entry.getKey(), entry.getValue());
+
+            ptr++;
+        }
+
+        return new DecisionTreeData(features, labels);
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java
new file mode 100644
index 0000000000000..192b07f9e0cbe
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/data/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains data and data builder required for decision tree trainers built on top of partition based dataset.
+ */
+package org.apache.ignite.ml.tree.data;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java
new file mode 100644
index 0000000000000..7ad2b80deaf43
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasure.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity;
+
+import java.io.Serializable;
+
+/**
+ * Base interface for impurity measures that can be used in distributed decision tree algorithm.
+ *
+ * @param <T> Type of this impurity measure.
+ */
+public interface ImpurityMeasure<T extends ImpurityMeasure<T>> extends Comparable<T>, Serializable {
+    /**
+     * Calculates impurity measure as a single double value.
+     *
+     * @return Impurity measure value.
+     */
+    public double impurity();
+
+    /**
+     * Adds the given impurity to this.
+     *
+     * @param measure Another impurity.
+     * @return Sum of this and the given impurity.
+     */
+    public T add(T measure);
+
+    /**
+     * Subtracts the given impurity for this.
+     *
+     * @param measure Another impurity.
+     * @return Difference of this and the given impurity.
+     */
+    public T subtract(T measure);
+
+    /** {@inheritDoc} */
+    default public int compareTo(T o) {
+        return Double.compare(impurity(), o.impurity());
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java
new file mode 100644
index 0000000000000..2b69356841572
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/ImpurityMeasureCalculator.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity;
+
+import java.io.Serializable;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+
+/**
+ * Base interface for impurity measure calculators that calculates all impurity measures required to find a best split.
+ *
+ * @param <T> Type of impurity measure.
+ */
+public interface ImpurityMeasureCalculator<T extends ImpurityMeasure<T>> extends Serializable {
+    /**
+     * Calculates all impurity measures required required to find a best split and returns them as an array of
+     * {@link StepFunction} (for every column).
+     *
+     * @param data Features and labels.
+     * @return Impurity measures as an array of {@link StepFunction} (for every column).
+     */
+    public StepFunction<T>[] calculate(DecisionTreeData data);
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java
new file mode 100644
index 0000000000000..817baf5da1713
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasure.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Gini impurity measure which is calculated the following way:
+ * {@code \-frac{1}{L}\sum_{i=1}^{s}l_i^2 - \frac{1}{R}\sum_{i=s+1}^{n}r_i^2}.
+ */
+public class GiniImpurityMeasure implements ImpurityMeasure<GiniImpurityMeasure> {
+    /** */
+    private static final long serialVersionUID = 5338129703395229970L;
+
+    /** Number of elements of each type in the left part. */
+    private final long[] left;
+
+    /** Number of elements of each type in the right part. */
+    private final long[] right;
+
+    /**
+     * Constructs a new instance of Gini impurity measure.
+     *
+     * @param left Number of elements of each type in the left part.
+     * @param right Number of elements of each type in the right part.
+     */
+    GiniImpurityMeasure(long[] left, long[] right) {
+        assert left.length == right.length : "Left and right parts have to be the same length";
+
+        this.left = left;
+        this.right = right;
+    }
+
+    /** {@inheritDoc} */
+    @Override public double impurity() {
+        long leftCnt = 0;
+        long rightCnt = 0;
+
+        double leftImpurity = 0;
+        double rightImpurity = 0;
+
+        for (long e : left)
+            leftCnt += e;
+
+        for (long e : right)
+            rightCnt += e;
+
+        if (leftCnt > 0)
+            for (long e : left)
+                leftImpurity += Math.pow(e, 2) / leftCnt;
+
+        if (rightCnt > 0)
+            for (long e : right)
+                rightImpurity += Math.pow(e, 2) / rightCnt;
+
+        return -(leftImpurity + rightImpurity);
+    }
+
+    /** {@inheritDoc} */
+    @Override public GiniImpurityMeasure add(GiniImpurityMeasure b) {
+        assert left.length == b.left.length : "Subtracted measure has to have length " + left.length;
+        assert left.length == b.right.length : "Subtracted measure has to have length " + left.length;
+
+        long[] leftRes = new long[left.length];
+        long[] rightRes = new long[left.length];
+
+        for (int i = 0; i < left.length; i++) {
+            leftRes[i] = left[i] + b.left[i];
+            rightRes[i] = right[i] + b.right[i];
+        }
+
+        return new GiniImpurityMeasure(leftRes, rightRes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public GiniImpurityMeasure subtract(GiniImpurityMeasure b) {
+        assert left.length == b.left.length : "Subtracted measure has to have length " + left.length;
+        assert left.length == b.right.length : "Subtracted measure has to have length " + left.length;
+
+        long[] leftRes = new long[left.length];
+        long[] rightRes = new long[left.length];
+
+        for (int i = 0; i < left.length; i++) {
+            leftRes[i] = left[i] - b.left[i];
+            rightRes[i] = right[i] - b.right[i];
+        }
+
+        return new GiniImpurityMeasure(leftRes, rightRes);
+    }
+
+    /** */
+    public long[] getLeft() {
+        return left;
+    }
+
+    /** */
+    public long[] getRight() {
+        return right;
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java
new file mode 100644
index 0000000000000..0dd0a1049e80e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculator.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import java.util.Arrays;
+import java.util.Map;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+
+/**
+ * Gini impurity measure calculator.
+ */
+public class GiniImpurityMeasureCalculator implements ImpurityMeasureCalculator<GiniImpurityMeasure> {
+    /** */
+    private static final long serialVersionUID = -522995134128519679L;
+
+    /** Label encoder which defines integer value for every label class. */
+    private final Map<Double, Integer> lbEncoder;
+
+    /**
+     * Constructs a new instance of Gini impurity measure calculator.
+     *
+     * @param lbEncoder Label encoder which defines integer value for every label class.
+     */
+    public GiniImpurityMeasureCalculator(Map<Double, Integer> lbEncoder) {
+        this.lbEncoder = lbEncoder;
+    }
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override public StepFunction<GiniImpurityMeasure>[] calculate(DecisionTreeData data) {
+        double[][] features = data.getFeatures();
+        double[] labels = data.getLabels();
+
+        if (features.length > 0) {
+            StepFunction<GiniImpurityMeasure>[] res = new StepFunction[features[0].length];
+
+            for (int col = 0; col < res.length; col++) {
+                data.sort(col);
+
+                double[] x = new double[features.length + 1];
+                GiniImpurityMeasure[] y = new GiniImpurityMeasure[features.length + 1];
+
+                int xPtr = 0, yPtr = 0;
+
+                long[] left = new long[lbEncoder.size()];
+                long[] right = new long[lbEncoder.size()];
+
+                for (int i = 0; i < labels.length; i++)
+                    right[getLabelCode(labels[i])]++;
+
+                x[xPtr++] = Double.NEGATIVE_INFINITY;
+                y[yPtr++] = new GiniImpurityMeasure(
+                    Arrays.copyOf(left, left.length),
+                    Arrays.copyOf(right, right.length)
+                );
+
+                for (int i = 0; i < features.length; i++) {
+                    left[getLabelCode(labels[i])]++;
+                    right[getLabelCode(labels[i])]--;
+
+                    if (i < (features.length - 1) && features[i + 1][col] == features[i][col])
+                        continue;
+
+                    x[xPtr++] = features[i][col];
+                    y[yPtr++] = new GiniImpurityMeasure(
+                        Arrays.copyOf(left, left.length),
+                        Arrays.copyOf(right, right.length)
+                    );
+                }
+
+                res[col] = new StepFunction<>(Arrays.copyOf(x, xPtr), Arrays.copyOf(y, yPtr));
+            }
+
+            return res;
+        }
+
+        return null;
+    }
+
+    /**
+     * Returns label code.
+     *
+     * @param lb Label.
+     * @return Label code.
+     */
+    int getLabelCode(double lb) {
+        Integer code = lbEncoder.get(lb);
+
+        assert code != null : "Can't find code for label " + lb;
+
+        return code;
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java
similarity index 89%
rename from modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java
index d6deb9d6e4fbc..d14cd9241a2cd 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/package-info.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/gini/package-info.java
@@ -17,6 +17,6 @@
 
 /**
  * <!-- Package description. -->
- * Contains classes representing decision tree nodes.
+ * Contains Gini impurity measure and calculator.
  */
-package org.apache.ignite.ml.trees.nodes;
\ No newline at end of file
+package org.apache.ignite.ml.tree.impurity.gini;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java
new file mode 100644
index 0000000000000..3fc85153c34f7
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasure.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Mean squared error (variance) impurity measure which is calculated the following way:
+ * {@code \frac{1}{L}\sum_{i=0}^{n}(y_i - \mu)^2}.
+ */
+public class MSEImpurityMeasure implements ImpurityMeasure<MSEImpurityMeasure> {
+    /** */
+    private static final long serialVersionUID = 4536394578628409689L;
+
+    /** Sum of all elements in the left part. */
+    private final double leftY;
+
+    /** Sum of all squared elements in the left part. */
+    private final double leftY2;
+
+    /** Number of elements in the left part. */
+    private final long leftCnt;
+
+    /** Sum of all elements in the right part. */
+    private final double rightY;
+
+    /** Sum of all squared elements in the right part. */
+    private final double rightY2;
+
+    /** Number of elements in the right part. */
+    private final long rightCnt;
+
+    /**
+     * Constructs a new instance of mean squared error (variance) impurity measure.
+     *
+     * @param leftY Sum of all elements in the left part.
+     * @param leftY2 Sum of all squared elements in the left part.
+     * @param leftCnt Number of elements in the left part.
+     * @param rightY Sum of all elements in the right part.
+     * @param rightY2 Sum of all squared elements in the right part.
+     * @param rightCnt Number of elements in the right part.
+     */
+    public MSEImpurityMeasure(double leftY, double leftY2, long leftCnt, double rightY, double rightY2, long rightCnt) {
+        this.leftY = leftY;
+        this.leftY2 = leftY2;
+        this.leftCnt = leftCnt;
+        this.rightY = rightY;
+        this.rightY2 = rightY2;
+        this.rightCnt = rightCnt;
+    }
+
+    /** {@inheritDoc} */
+    @Override public double impurity() {
+        double impurity = 0;
+
+        if (leftCnt > 0)
+            impurity += leftY2 - 2.0 * leftY / leftCnt * leftY + Math.pow(leftY / leftCnt, 2) * leftCnt;
+
+        if (rightCnt > 0)
+            impurity += rightY2 - 2.0 * rightY / rightCnt * rightY + Math.pow(rightY / rightCnt, 2) * rightCnt;
+
+        return impurity;
+    }
+
+    /** {@inheritDoc} */
+    @Override public MSEImpurityMeasure add(MSEImpurityMeasure b) {
+        return new MSEImpurityMeasure(
+            leftY + b.leftY,
+            leftY2 + b.leftY2,
+            leftCnt + b.leftCnt,
+            rightY + b.rightY,
+            rightY2 + b.rightY2,
+            rightCnt + b.rightCnt
+        );
+    }
+
+    /** {@inheritDoc} */
+    @Override public MSEImpurityMeasure subtract(MSEImpurityMeasure b) {
+        return new MSEImpurityMeasure(
+            leftY - b.leftY,
+            leftY2 - b.leftY2,
+            leftCnt - b.leftCnt,
+            rightY - b.rightY,
+            rightY2 - b.rightY2,
+            rightCnt - b.rightCnt
+        );
+    }
+
+    /** */
+    public double getLeftY() {
+        return leftY;
+    }
+
+    /** */
+    public double getLeftY2() {
+        return leftY2;
+    }
+
+    /** */
+    public long getLeftCnt() {
+        return leftCnt;
+    }
+
+    /** */
+    public double getRightY() {
+        return rightY;
+    }
+
+    /** */
+    public double getRightY2() {
+        return rightY2;
+    }
+
+    /** */
+    public long getRightCnt() {
+        return rightCnt;
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java
new file mode 100644
index 0000000000000..cb5019c4e179d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculator.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasureCalculator;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+
+/**
+ * Meas squared error (variance) impurity measure calculator.
+ */
+public class MSEImpurityMeasureCalculator implements ImpurityMeasureCalculator<MSEImpurityMeasure> {
+    /** */
+    private static final long serialVersionUID = 288747414953756824L;
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override public StepFunction<MSEImpurityMeasure>[] calculate(DecisionTreeData data) {
+        double[][] features = data.getFeatures();
+        double[] labels = data.getLabels();
+
+        if (features.length > 0) {
+            StepFunction<MSEImpurityMeasure>[] res = new StepFunction[features[0].length];
+
+            for (int col = 0; col < res.length; col++) {
+                data.sort(col);
+
+                double[] x = new double[features.length + 1];
+                MSEImpurityMeasure[] y = new MSEImpurityMeasure[features.length + 1];
+
+                x[0] = Double.NEGATIVE_INFINITY;
+
+                for (int leftSize = 0; leftSize <= features.length; leftSize++) {
+                    double leftY = 0;
+                    double leftY2 = 0;
+                    double rightY = 0;
+                    double rightY2 = 0;
+
+                    for (int i = 0; i < leftSize; i++) {
+                        leftY += labels[i];
+                        leftY2 += Math.pow(labels[i], 2);
+                    }
+
+                    for (int i = leftSize; i < features.length; i++) {
+                        rightY += labels[i];
+                        rightY2 += Math.pow(labels[i], 2);
+                    }
+
+                    if (leftSize < features.length)
+                        x[leftSize + 1] = features[leftSize][col];
+
+                    y[leftSize] = new MSEImpurityMeasure(
+                        leftY, leftY2, leftSize, rightY, rightY2, features.length - leftSize
+                    );
+                }
+
+                res[col] = new StepFunction<>(x, y);
+            }
+
+            return res;
+        }
+
+        return null;
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java
similarity index 88%
rename from modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java
index ce8418e87cf39..23ec4e03f09b7 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/package-info.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/mse/package-info.java
@@ -17,6 +17,6 @@
 
 /**
  * <!-- Package description. -->
- * Contains decision tree models.
+ * Contains mean squared error impurity measure and calculator.
  */
-package org.apache.ignite.ml.trees.models;
\ No newline at end of file
+package org.apache.ignite.ml.tree.impurity.mse;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java
new file mode 100644
index 0000000000000..4155593aae3d7
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Root package for decision tree impurity measures and calculators.
+ */
+package org.apache.ignite.ml.tree.impurity;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java
new file mode 100644
index 0000000000000..2418571089ba6
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressor.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Simple step function compressor.
+ *
+ * @param <T> Type of step function values.
+ */
+public class SimpleStepFunctionCompressor<T extends ImpurityMeasure<T>> implements StepFunctionCompressor<T> {
+    /** */
+    private static final long serialVersionUID = -3231787633598409157L;
+
+    /** Min size of step function to be compressed. */
+    private final int minSizeToBeCompressed;
+
+    /** In case of compression min impurity increase that will be recorded. */
+    private final double minImpurityIncreaseForRecord;
+
+    /** In case of compression min impurity decrease that will be recorded. */
+    private final double minImpurityDecreaseForRecord;
+
+    /**
+     * Constructs a new instance of simple step function compressor with default parameters.
+     */
+    public SimpleStepFunctionCompressor() {
+        this(10, 0.1, 0.05);
+    }
+
+    /**
+     * Constructs a new instance of simple step function compressor.
+     *
+     * @param minSizeToBeCompressed Min size of step function to be compressed.
+     * @param minImpurityIncreaseForRecord In case of compression min impurity increase that will be recorded.
+     * @param minImpurityDecreaseForRecord In case of compression min impurity decrease that will be recorded.
+     */
+    public SimpleStepFunctionCompressor(int minSizeToBeCompressed, double minImpurityIncreaseForRecord,
+        double minImpurityDecreaseForRecord) {
+        this.minSizeToBeCompressed = minSizeToBeCompressed;
+        this.minImpurityIncreaseForRecord = minImpurityIncreaseForRecord;
+        this.minImpurityDecreaseForRecord = minImpurityDecreaseForRecord;
+    }
+
+    /** {@inheritDoc} */
+    @Override public StepFunction<T> compress(StepFunction<T> function) {
+        double[] arguments = function.getX();
+        T[] values = function.getY();
+
+        if (arguments.length >= minSizeToBeCompressed) {
+            List<StepFunctionPoint> points = new ArrayList<>();
+
+            for (int i = 0; i < arguments.length; i++)
+                points.add(new StepFunctionPoint(arguments[i], values[i]));
+
+            points = compress(points);
+
+            double[] resX = new double[points.size()];
+            T[] resY = Arrays.copyOf(values, points.size());
+
+            for (int i = 0; i < points.size(); i++) {
+                StepFunctionPoint pnt = points.get(i);
+                resX[i] = pnt.x;
+                resY[i] = pnt.y;
+            }
+
+            return new StepFunction<>(resX, resY);
+        }
+
+        return function;
+    }
+
+    /**
+     * Compresses list of step function points.
+     *
+     * @param points Step function points.
+     * @return Compressed step function points.
+     */
+    private List<StepFunctionPoint> compress(List<StepFunctionPoint> points) {
+        List<StepFunctionPoint> res = new ArrayList<>();
+
+        double minImpurity = Double.MAX_VALUE, maxImpurity = Double.MIN_VALUE;
+        for (int i = 0; i < points.size(); i++) {
+            StepFunctionPoint pnt = points.get(i);
+
+            double impurity = pnt.y.impurity();
+
+            if (impurity > maxImpurity)
+                maxImpurity = impurity;
+
+            if (impurity < minImpurity)
+                minImpurity = impurity;
+        }
+
+        Double prev = null;
+        for (StepFunctionPoint pnt : points) {
+            double impurity = (pnt.y.impurity() - minImpurity) / (maxImpurity - minImpurity);
+            if (prev == null ||
+                prev - impurity >= minImpurityDecreaseForRecord ||
+                impurity - prev >= minImpurityIncreaseForRecord) {
+                prev = impurity;
+                res.add(pnt);
+            }
+        }
+
+        return res;
+    }
+
+    /**
+     * Util class that represents step function point.
+     */
+    private class StepFunctionPoint {
+        /** Argument of the step start. */
+        private final double x;
+
+        /** Value of the step. */
+        private final T y;
+
+        /**
+         * Constructs a new instance of util class that represents step function point.
+         *
+         * @param x Argument of the step start.
+         * @param y Value of the step.
+         */
+        StepFunctionPoint(double x, T y) {
+            this.x = x;
+            this.y = y;
+        }
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java
new file mode 100644
index 0000000000000..431503d3d11de
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunction.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.util.Arrays;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Step function described by {@code x} and {@code y} points.
+ *
+ * @param <T> Type of function values.
+ */
+public class StepFunction<T extends ImpurityMeasure<T>> {
+    /** Argument of every steps start. Should be ascendingly sorted all the time. */
+    private final double[] x;
+
+    /** Value of every step. */
+    private final T[] y;
+
+    /**
+     * Constructs a new instance of step function.
+     *
+     * @param x Argument of every steps start.
+     * @param y Value of every step.
+     */
+    public StepFunction(double[] x, T[] y) {
+        assert x.length == y.length : "Argument and value arrays have to be the same length";
+
+        this.x = x;
+        this.y = y;
+
+        sort(x, y, 0, x.length - 1);
+    }
+
+    /**
+     * Adds the given step function to this.
+     *
+     * @param b Another step function.
+     * @return Sum of this and the given function.
+     */
+    public StepFunction<T> add(StepFunction<T> b) {
+        int resSize = 0, leftPtr = 0, rightPtr = 0;
+        double previousPnt = 0;
+
+        while (leftPtr < x.length || rightPtr < b.x.length) {
+            if (rightPtr >= b.x.length || (leftPtr < x.length && x[leftPtr] < b.x[rightPtr])) {
+                if (resSize == 0 || x[leftPtr] != previousPnt) {
+                    previousPnt = x[leftPtr];
+                    resSize++;
+                }
+
+                leftPtr++;
+            }
+            else {
+                if (resSize == 0 || b.x[rightPtr] != previousPnt) {
+                    previousPnt = b.x[rightPtr];
+                    resSize++;
+                }
+
+                rightPtr++;
+            }
+        }
+
+        double[] resX = new double[resSize];
+        T[] resY = Arrays.copyOf(y, resSize);
+
+        leftPtr = 0;
+        rightPtr = 0;
+
+        for (int i = 0; leftPtr < x.length || rightPtr < b.x.length; i++) {
+            if (rightPtr >= b.x.length || (leftPtr < x.length && x[leftPtr] < b.x[rightPtr])) {
+                boolean override = i > 0 && x[leftPtr] == resX[i - 1];
+                int target = override ? i - 1 : i;
+
+                resY[target] = override ? resY[target] : null;
+                resY[target] = i > 0 ? resY[i - 1] : null;
+                resY[target] = resY[target] == null ? y[leftPtr] : resY[target].add(y[leftPtr]);
+
+                if (leftPtr > 0)
+                    resY[target] = resY[target].subtract(y[leftPtr - 1]);
+
+                resX[target] = x[leftPtr];
+                i = target;
+
+                leftPtr++;
+            }
+            else {
+                boolean override = i > 0 && b.x[rightPtr] == resX[i - 1];
+                int target = override ? i - 1 : i;
+
+                resY[target] = override ? resY[target] : null;
+                resY[target] = i > 0 ? resY[i - 1] : null;
+
+                resY[target] = resY[target] == null ? b.y[rightPtr] : resY[target].add(b.y[rightPtr]);
+
+                if (rightPtr > 0)
+                    resY[target] = resY[target].subtract(b.y[rightPtr - 1]);
+
+                resX[target] = b.x[rightPtr];
+                i = target;
+
+                rightPtr++;
+            }
+        }
+
+        return new StepFunction<>(resX, resY);
+    }
+
+    /** */
+    private void sort(double[] x, T[] y, int from, int to) {
+        if (from < to) {
+            double pivot = x[(from + to) / 2];
+
+            int i = from, j = to;
+            while (i <= j) {
+                while (x[i] < pivot) i++;
+                while (x[j] > pivot) j--;
+
+                if (i <= j) {
+                    double tmpX = x[i];
+                    x[i] = x[j];
+                    x[j] = tmpX;
+
+                    T tmpY = y[i];
+                    y[i] = y[j];
+                    y[j] = tmpY;
+
+                    i++;
+                    j--;
+                }
+            }
+
+            sort(x, y, from, j);
+            sort(x, y, i, to);
+        }
+    }
+
+    /** */
+    public double[] getX() {
+        return x;
+    }
+
+    /** */
+    public T[] getY() {
+        return y;
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java
new file mode 100644
index 0000000000000..41baa292d4a33
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionCompressor.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Base interface for step function compressors which reduces step function size.
+ *
+ * @param <T> Type of step function value.
+ */
+public interface StepFunctionCompressor<T extends ImpurityMeasure<T>> extends Serializable {
+    /**
+     * Compresses the given step function.
+     *
+     * @param function Step function.
+     * @return Compressed step function.
+     */
+    public StepFunction<T> compress(StepFunction<T> function);
+
+    /**
+     * Compresses every step function in the given array.
+     *
+     * @param functions Array of step functions.
+     * @return Arrays of compressed step function.
+     */
+    default public StepFunction<T>[] compress(StepFunction<T>[] functions) {
+        if (functions == null)
+            return null;
+
+        StepFunction<T>[] res = Arrays.copyOf(functions, functions.length);
+
+        for (int i = 0; i < res.length; i++)
+            res[i] = compress(res[i]);
+
+        return res;
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java
new file mode 100644
index 0000000000000..99df6180e5ddb
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/impurity/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains util classes used in decision tree impurity calculators.
+ */
+package org.apache.ignite.ml.tree.impurity.util;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java
similarity index 55%
rename from modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java
index 572e64a332184..976e30d34be4c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/models/DecisionTreeModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/DecisionTreeLeafBuilder.java
@@ -15,30 +15,24 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml.trees.models;
+package org.apache.ignite.ml.tree.leaf;
 
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.trees.nodes.DecisionTreeNode;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.DecisionTreeLeafNode;
+import org.apache.ignite.ml.tree.TreeFilter;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
 
 /**
- * Model for decision tree.
+ * Base interface for decision tree leaf builders.
  */
-public class DecisionTreeModel implements Model<Vector, Double> {
-    /** Root node of the decision tree. */
-    private final DecisionTreeNode root;
-
+public interface DecisionTreeLeafBuilder {
     /**
-     * Construct decision tree model.
+     * Creates new leaf node for given dataset and node predicate.
      *
-     * @param root Root of decision tree.
+     * @param dataset Dataset.
+     * @param pred Node predicate.
+     * @return Leaf node.
      */
-    public DecisionTreeModel(DecisionTreeNode root) {
-        this.root = root;
-    }
-
-    /** {@inheritDoc} */
-    @Override public Double apply(Vector val) {
-        return root.process(val);
-    }
+    public DecisionTreeLeafNode createLeafNode(Dataset<EmptyContext, DecisionTreeData> dataset, TreeFilter pred);
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java
new file mode 100644
index 0000000000000..2e05215b9dd7c
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MeanDecisionTreeLeafBuilder.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.leaf;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.DecisionTreeLeafNode;
+import org.apache.ignite.ml.tree.TreeFilter;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+
+/**
+ * Decision tree leaf node builder that chooses mean value as a leaf value.
+ */
+public class MeanDecisionTreeLeafBuilder implements DecisionTreeLeafBuilder {
+    /** {@inheritDoc} */
+    @Override public DecisionTreeLeafNode createLeafNode(Dataset<EmptyContext, DecisionTreeData> dataset,
+        TreeFilter pred) {
+        double[] aa = dataset.compute(part -> {
+            double mean = 0;
+            int cnt = 0;
+
+            for (int i = 0; i < part.getFeatures().length; i++) {
+                if (pred.test(part.getFeatures()[i])) {
+                    mean += part.getLabels()[i];
+                    cnt++;
+                }
+            }
+
+            if (cnt != 0) {
+                mean = mean / cnt;
+
+                return new double[] {mean, cnt};
+            }
+
+            return null;
+        }, this::reduce);
+
+        return aa != null ? new DecisionTreeLeafNode(aa[0]) : null;
+    }
+
+    /** */
+    private double[] reduce(double[] a, double[] b) {
+        if (a == null)
+            return b;
+        else if (b == null)
+            return a;
+        else {
+            double aMean = a[0];
+            double aCnt = a[1];
+            double bMean = b[0];
+            double bCnt = b[1];
+
+            double mean = (aMean * aCnt + bMean * bCnt) / (aCnt + bCnt);
+
+            return new double[] {mean, aCnt + bCnt};
+        }
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java
new file mode 100644
index 0000000000000..1e8b9414d740e
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/MostCommonDecisionTreeLeafBuilder.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.leaf;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.tree.DecisionTreeLeafNode;
+import org.apache.ignite.ml.tree.TreeFilter;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+
+/**
+ * Decision tree leaf node builder that chooses most common value as a leaf node value.
+ */
+public class MostCommonDecisionTreeLeafBuilder implements DecisionTreeLeafBuilder {
+    /** {@inheritDoc} */
+    @Override public DecisionTreeLeafNode createLeafNode(Dataset<EmptyContext, DecisionTreeData> dataset,
+        TreeFilter pred) {
+        Map<Double, Integer> cnt = dataset.compute(part -> {
+
+            if (part.getFeatures() != null) {
+                Map<Double, Integer> map = new HashMap<>();
+
+                for (int i = 0; i < part.getFeatures().length; i++) {
+                    if (pred.test(part.getFeatures()[i])) {
+                        double lb = part.getLabels()[i];
+
+                        if (map.containsKey(lb))
+                            map.put(lb, map.get(lb) + 1);
+                        else
+                            map.put(lb, 1);
+                    }
+                }
+
+                return map;
+            }
+
+            return null;
+        }, this::reduce);
+
+        double bestVal = 0;
+        int bestCnt = -1;
+
+        for (Map.Entry<Double, Integer> e : cnt.entrySet()) {
+            if (e.getValue() > bestCnt) {
+                bestCnt = e.getValue();
+                bestVal = e.getKey();
+            }
+        }
+
+        return new DecisionTreeLeafNode(bestVal);
+    }
+
+    /** */
+    private Map<Double, Integer> reduce(Map<Double, Integer> a, Map<Double, Integer> b) {
+        if (a == null)
+            return b;
+        else if (b == null)
+            return a;
+        else {
+            for (Map.Entry<Double, Integer> e : b.entrySet()) {
+                if (a.containsKey(e.getKey()))
+                    a.put(e.getKey(), a.get(e.getKey()) + e.getValue());
+                else
+                    a.put(e.getKey(), e.getValue());
+            }
+            return a;
+        }
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java
similarity index 90%
rename from modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java
index e8edd8f16f0bb..26ec67dd97af2 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/package-info.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/leaf/package-info.java
@@ -17,6 +17,6 @@
 
 /**
  * <!-- Package description. -->
- * Region calculators.
+ * Root package for decision trees leaf builders.
  */
-package org.apache.ignite.ml.trees.trainers.columnbased.regcalcs;
\ No newline at end of file
+package org.apache.ignite.ml.tree.leaf;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java
similarity index 92%
rename from modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java
index b07ba4acb360d..660f3f3c0197e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/package-info.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/tree/package-info.java
@@ -17,6 +17,6 @@
 
 /**
  * <!-- Package description. -->
- * Contains decision tree algorithms.
+ * Root package for decision trees.
  */
-package org.apache.ignite.ml.trees;
\ No newline at end of file
+package org.apache.ignite.ml.tree;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java
deleted file mode 100644
index 3ae474e12e66e..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalRegionInfo.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.BitSet;
-
-/**
- * Information about categorical region.
- */
-public class CategoricalRegionInfo extends RegionInfo implements Externalizable {
-    /**
-     * Bitset representing categories of this region.
-     */
-    private BitSet cats;
-
-    /**
-     * @param impurity Impurity of region.
-     * @param cats Bitset representing categories of this region.
-     */
-    public CategoricalRegionInfo(double impurity, BitSet cats) {
-        super(impurity);
-
-        this.cats = cats;
-    }
-
-    /**
-     * No-op constructor for serialization/deserialization.
-     */
-    public CategoricalRegionInfo() {
-        // No-op
-    }
-
-    /**
-     * Get bitset representing categories of this region.
-     *
-     * @return Bitset representing categories of this region.
-     */
-    public BitSet cats() {
-        return cats;
-    }
-
-    /** {@inheritDoc} */
-    @Override public void writeExternal(ObjectOutput out) throws IOException {
-        super.writeExternal(out);
-        out.writeObject(cats);
-    }
-
-    /** {@inheritDoc} */
-    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-        super.readExternal(in);
-        cats = (BitSet)in.readObject();
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java
deleted file mode 100644
index 94cb1e8be9df4..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/CategoricalSplitInfo.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.BitSet;
-import org.apache.ignite.ml.trees.nodes.CategoricalSplitNode;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-
-/**
- * Information about split of categorical feature.
- *
- * @param <D> Class representing information of left and right subregions.
- */
-public class CategoricalSplitInfo<D extends RegionInfo> extends SplitInfo<D> {
-    /** Bitset indicating which vectors are assigned to left subregion. */
-    private final BitSet bs;
-
-    /**
-     * @param regionIdx Index of region which is split.
-     * @param leftData Data of left subregion.
-     * @param rightData Data of right subregion.
-     * @param bs Bitset indicating which vectors are assigned to left subregion.
-     */
-    public CategoricalSplitInfo(int regionIdx, D leftData, D rightData,
-        BitSet bs) {
-        super(regionIdx, leftData, rightData);
-        this.bs = bs;
-    }
-
-    /** {@inheritDoc} */
-    @Override public SplitNode createSplitNode(int featureIdx) {
-        return new CategoricalSplitNode(featureIdx, bs);
-    }
-
-    /**
-     * Get bitset indicating which vectors are assigned to left subregion.
-     */
-    public BitSet bitSet() {
-        return bs;
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return "CategoricalSplitInfo [" +
-            "infoGain=" + infoGain +
-            ", regionIdx=" + regionIdx +
-            ", leftData=" + leftData +
-            ", bs=" + bs +
-            ", rightData=" + rightData +
-            ']';
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java
deleted file mode 100644
index e98bb728c6aaa..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousRegionInfo.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-
-/**
- * Information about region used by continuous features.
- */
-public class ContinuousRegionInfo extends RegionInfo {
-    /**
-     * Count of samples in this region.
-     */
-    private int size;
-
-    /**
-     * @param impurity Impurity of the region.
-     * @param size Size of this region
-     */
-    public ContinuousRegionInfo(double impurity, int size) {
-        super(impurity);
-        this.size = size;
-    }
-
-    /**
-     * No-op constructor for serialization/deserialization.
-     */
-    public ContinuousRegionInfo() {
-        // No-op
-    }
-
-    /**
-     * Get the size of region.
-     */
-    public int getSize() {
-        return size;
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return "ContinuousRegionInfo [" +
-            "size=" + size +
-            ']';
-    }
-
-    /** {@inheritDoc} */
-    @Override public void writeExternal(ObjectOutput out) throws IOException {
-        super.writeExternal(out);
-        out.writeInt(size);
-    }
-
-    /** {@inheritDoc} */
-    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-        super.readExternal(in);
-        size = in.readInt();
-    }
-}
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java
deleted file mode 100644
index 3a0e9da3982ce..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/ContinuousSplitCalculator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousFeatureProcessor;
-
-/**
- * This class is used for calculation of best split by continuous feature.
- *
- * @param <C> Class in which information about region will be stored.
- */
-public interface ContinuousSplitCalculator<C extends ContinuousRegionInfo> {
-    /**
-     * Calculate region info 'from scratch'.
-     *
-     * @param s Stream of labels in this region.
-     * @param l Index of sample projection on this feature in array sorted by this projection value and intervals
-     * bitsets. ({@link ContinuousFeatureProcessor}).
-     * @return Region info.
-     */
-    C calculateRegionInfo(DoubleStream s, int l);
-
-    /**
-     * Calculate split info of best split of region given information about this region.
-     *
-     * @param sampleIndexes Indexes of samples of this region.
-     * @param values All values of this feature.
-     * @param labels All labels of this feature.
-     * @param regionIdx Index of region being split.
-     * @param data Information about region being split which can be used for computations.
-     * @return Information about best split of region with index given by regionIdx.
-     */
-    SplitInfo<C> splitRegion(Integer[] sampleIndexes, double[] values, double[] labels, int regionIdx, C data);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java
deleted file mode 100644
index 8ec7db3713f46..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/RegionInfo.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-
-/** Class containing information about region. */
-public class RegionInfo implements Externalizable {
-    /** Impurity in this region. */
-    private double impurity;
-
-    /**
-     * @param impurity Impurity of this region.
-     */
-    public RegionInfo(double impurity) {
-        this.impurity = impurity;
-    }
-
-    /**
-     * No-op constructor for serialization/deserialization.
-     */
-    public RegionInfo() {
-        // No-op
-    }
-
-    /**
-     * Get impurity in this region.
-     *
-     * @return Impurity of this region.
-     */
-    public double impurity() {
-        return impurity;
-    }
-
-    /** {@inheritDoc} */
-    @Override public void writeExternal(ObjectOutput out) throws IOException {
-        out.writeDouble(impurity);
-    }
-
-    /** {@inheritDoc} */
-    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-        impurity = in.readDouble();
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java
deleted file mode 100644
index cae6d4a6d2da5..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/CategoricalSplitNode.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import java.util.BitSet;
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Split node by categorical feature.
- */
-public class CategoricalSplitNode extends SplitNode {
-    /** Bitset specifying which categories belong to left subregion. */
-    private final BitSet bs;
-
-    /**
-     * Construct categorical split node.
-     *
-     * @param featureIdx Index of feature by which split is done.
-     * @param bs Bitset specifying which categories go to the left subtree.
-     */
-    public CategoricalSplitNode(int featureIdx, BitSet bs) {
-        super(featureIdx);
-        this.bs = bs;
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean goLeft(Vector v) {
-        return bs.get((int)v.getX(featureIdx));
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return "CategoricalSplitNode [bs=" + bs + ']';
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java
deleted file mode 100644
index 285cfcd3ef735..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/ContinuousSplitNode.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Split node representing split of continuous feature.
- */
-public class ContinuousSplitNode extends SplitNode {
-    /** Threshold. Values which are less or equal then threshold are assigned to the left subregion. */
-    private final double threshold;
-
-    /**
-     * Construct ContinuousSplitNode by threshold and feature index.
-     *
-     * @param threshold Threshold.
-     * @param featureIdx Feature index.
-     */
-    public ContinuousSplitNode(double threshold, int featureIdx) {
-        super(featureIdx);
-        this.threshold = threshold;
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean goLeft(Vector v) {
-        return v.getX(featureIdx) <= threshold;
-    }
-
-    /** Threshold. Values which are less or equal then threshold are assigned to the left subregion. */
-    public double threshold() {
-        return threshold;
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return "ContinuousSplitNode [" +
-            "threshold=" + threshold +
-            ']';
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java
deleted file mode 100644
index 4c258d175b189..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/nodes/SplitNode.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.nodes;
-
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Node in decision tree representing a split.
- */
-public abstract class SplitNode implements DecisionTreeNode {
-    /** Left subtree. */
-    protected DecisionTreeNode l;
-
-    /** Right subtree. */
-    protected DecisionTreeNode r;
-
-    /** Feature index. */
-    protected final int featureIdx;
-
-    /**
-     * Constructs SplitNode with a given feature index.
-     *
-     * @param featureIdx Feature index.
-     */
-    public SplitNode(int featureIdx) {
-        this.featureIdx = featureIdx;
-    }
-
-    /**
-     * Indicates if the given vector is in left subtree.
-     *
-     * @param v Vector
-     * @return Status of given vector being left subtree.
-     */
-    abstract boolean goLeft(Vector v);
-
-    /**
-     * Left subtree.
-     *
-     * @return Left subtree.
-     */
-    public DecisionTreeNode left() {
-        return l;
-    }
-
-    /**
-     * Right subtree.
-     *
-     * @return Right subtree.
-     */
-    public DecisionTreeNode right() {
-        return r;
-    }
-
-    /**
-     * Set the left subtree.
-     *
-     * @param n left subtree.
-     */
-    public void setLeft(DecisionTreeNode n) {
-        l = n;
-    }
-
-    /**
-     * Set the right subtree.
-     *
-     * @param n right subtree.
-     */
-    public void setRight(DecisionTreeNode n) {
-        r = n;
-    }
-
-    /**
-     * Delegates processing to subtrees.
-     *
-     * @param v Vector.
-     * @return Value assigned to the given vector.
-     */
-    @Override public double process(Vector v) {
-        if (left() != null && goLeft(v))
-            return left().process(v);
-        else
-            return right().process(v);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java
deleted file mode 100644
index 0d27c8a6c06f8..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndex.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-
-/**
- * Class representing a simple index in 2d matrix in the form (row, col).
- */
-public class BiIndex implements Externalizable {
-    /** Row. */
-    private int row;
-
-    /** Column. */
-    @AffinityKeyMapped
-    private int col;
-
-    /**
-     * No-op constructor for serialization/deserialization.
-     */
-    public BiIndex() {
-        // No-op.
-    }
-
-    /**
-     * Construct BiIndex from row and column.
-     *
-     * @param row Row.
-     * @param col Column.
-     */
-    public BiIndex(int row, int col) {
-        this.row = row;
-        this.col = col;
-    }
-
-    /**
-     * Returns row.
-     *
-     * @return Row.
-     */
-    public int row() {
-        return row;
-    }
-
-    /**
-     * Returns column.
-     *
-     * @return Column.
-     */
-    public int col() {
-        return col;
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean equals(Object o) {
-        if (this == o)
-            return true;
-        if (o == null || getClass() != o.getClass())
-            return false;
-
-        BiIndex idx = (BiIndex)o;
-
-        if (row != idx.row)
-            return false;
-        return col == idx.col;
-    }
-
-    /** {@inheritDoc} */
-    @Override public int hashCode() {
-        int res = row;
-        res = 31 * res + col;
-        return res;
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return "BiIndex [" +
-            "row=" + row +
-            ", col=" + col +
-            ']';
-    }
-
-    /** {@inheritDoc} */
-    @Override public void writeExternal(ObjectOutput out) throws IOException {
-        out.writeInt(row);
-        out.writeInt(col);
-    }
-
-    /** {@inheritDoc} */
-    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-        row = in.readInt();
-        col = in.readInt();
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index 04281fba8de45..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/BiIndexedCacheColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.Map;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.lang.IgniteBiTuple;
-
-/**
- * Adapter for column decision tree trainer for bi-indexed cache.
- */
-public class BiIndexedCacheColumnDecisionTreeTrainerInput extends CacheColumnDecisionTreeTrainerInput<BiIndex, Double> {
-    /**
-     * Construct an input for {@link ColumnDecisionTreeTrainer}.
-     *
-     * @param cache Bi-indexed cache.
-     * @param catFeaturesInfo Information about categorical feature in the form (feature index -> number of
-     * categories).
-     * @param samplesCnt Count of samples.
-     * @param featuresCnt Count of features.
-     */
-    public BiIndexedCacheColumnDecisionTreeTrainerInput(IgniteCache<BiIndex, Double> cache,
-        Map<Integer, Integer> catFeaturesInfo, int samplesCnt, int featuresCnt) {
-        super(cache,
-            () -> IntStream.range(0, samplesCnt).mapToObj(s -> new BiIndex(s, featuresCnt)),
-            e -> Stream.of(new IgniteBiTuple<>(e.getKey().row(), e.getValue())),
-            DoubleStream::of,
-            fIdx -> IntStream.range(0, samplesCnt).mapToObj(s -> new BiIndex(s, fIdx)),
-            catFeaturesInfo,
-            featuresCnt,
-            samplesCnt);
-    }
-
-    /** {@inheritDoc} */
-    @Override public Object affinityKey(int idx, Ignite ignite) {
-        return idx;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index 40927b77d7fa6..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/CacheColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.Stream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-
-/**
- * Adapter of a given cache to {@link CacheColumnDecisionTreeTrainerInput}
- *
- * @param <K> Class of keys of the cache.
- * @param <V> Class of values of the cache.
- */
-public abstract class CacheColumnDecisionTreeTrainerInput<K, V> implements ColumnDecisionTreeTrainerInput {
-    /** Supplier of labels key. */
-    private final IgniteSupplier<Stream<K>> labelsKeys;
-
-    /** Count of features. */
-    private final int featuresCnt;
-
-    /** Function which maps feature index to Stream of keys corresponding to this feature index. */
-    private final IgniteFunction<Integer, Stream<K>> keyMapper;
-
-    /** Information about which features are categorical in form of feature index -> number of categories. */
-    private final Map<Integer, Integer> catFeaturesInfo;
-
-    /** Cache name. */
-    private final String cacheName;
-
-    /** Count of samples. */
-    private final int samplesCnt;
-
-    /** Function used for mapping cache values to stream of tuples. */
-    private final IgniteFunction<Cache.Entry<K, V>, Stream<IgniteBiTuple<Integer, Double>>> valuesMapper;
-
-    /**
-     * Function which map value of entry with label key to DoubleStream.
-     * Look at {@code CacheColumnDecisionTreeTrainerInput::labels} for understanding how {@code labelsKeys} and
-     * {@code labelsMapper} interact.
-     */
-    private final IgniteFunction<V, DoubleStream> labelsMapper;
-
-    /**
-     * Constructs input for {@link ColumnDecisionTreeTrainer}.
-     *
-     * @param c Cache.
-     * @param valuesMapper Function for mapping cache entry to stream used by {@link ColumnDecisionTreeTrainer}.
-     * @param labelsMapper Function used for mapping cache value to labels array.
-     * @param keyMapper Function used for mapping feature index to the cache key.
-     * @param catFeaturesInfo Information about which features are categorical in form of feature index -> number of
-     * categories.
-     * @param featuresCnt Count of features.
-     * @param samplesCnt Count of samples.
-     */
-    // TODO: IGNITE-5724 think about boxing/unboxing
-    public CacheColumnDecisionTreeTrainerInput(IgniteCache<K, V> c,
-        IgniteSupplier<Stream<K>> labelsKeys,
-        IgniteFunction<Cache.Entry<K, V>, Stream<IgniteBiTuple<Integer, Double>>> valuesMapper,
-        IgniteFunction<V, DoubleStream> labelsMapper,
-        IgniteFunction<Integer, Stream<K>> keyMapper,
-        Map<Integer, Integer> catFeaturesInfo,
-        int featuresCnt, int samplesCnt) {
-
-        cacheName = c.getName();
-        this.labelsKeys = labelsKeys;
-        this.valuesMapper = valuesMapper;
-        this.labelsMapper = labelsMapper;
-        this.keyMapper = keyMapper;
-        this.catFeaturesInfo = catFeaturesInfo;
-        this.samplesCnt = samplesCnt;
-        this.featuresCnt = featuresCnt;
-    }
-
-    /** {@inheritDoc} */
-    @Override public Stream<IgniteBiTuple<Integer, Double>> values(int idx) {
-        return cache(Ignition.localIgnite()).getAll(keyMapper.apply(idx).collect(Collectors.toSet())).
-            entrySet().
-            stream().
-            flatMap(ent -> valuesMapper.apply(new CacheEntryImpl<>(ent.getKey(), ent.getValue())));
-    }
-
-    /** {@inheritDoc} */
-    @Override public double[] labels(Ignite ignite) {
-        return labelsKeys.get().map(k -> get(k, ignite)).flatMapToDouble(labelsMapper).toArray();
-    }
-
-    /** {@inheritDoc} */
-    @Override public Map<Integer, Integer> catFeaturesInfo() {
-        return catFeaturesInfo;
-    }
-
-    /** {@inheritDoc} */
-    @Override public int featuresCount() {
-        return featuresCnt;
-    }
-
-    /** {@inheritDoc} */
-    @Override public Object affinityKey(int idx, Ignite ignite) {
-        return ignite.affinity(cacheName).affinityKey(keyMapper.apply(idx));
-    }
-
-    /** */
-    private V get(K k, Ignite ignite) {
-        V res = cache(ignite).localPeek(k);
-
-        if (res == null)
-            res = cache(ignite).get(k);
-
-        return res;
-    }
-
-    /** */
-    private IgniteCache<K, V> cache(Ignite ignite) {
-        return ignite.getOrCreateCache(cacheName);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java
deleted file mode 100644
index fec0a83fd503c..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainer.java
+++ /dev/null
@@ -1,568 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.function.Consumer;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CachePeekMode;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distributed.CacheUtils;
-import org.apache.ignite.ml.math.functions.Functions;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteCurriedBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.nodes.DecisionTreeNode;
-import org.apache.ignite.ml.trees.nodes.Leaf;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ContextCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache.FeatureKey;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ProjectionsCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ProjectionsCache.RegionKey;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.SplitCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.SplitCache.SplitKey;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureProcessor;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.jetbrains.annotations.NotNull;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache.getFeatureCacheKey;
-
-/**
- * This trainer stores observations as columns and features as rows.
- * Ideas from https://github.com/fabuzaid21/yggdrasil are used here.
- */
-public class ColumnDecisionTreeTrainer<D extends ContinuousRegionInfo> implements
-    Trainer<DecisionTreeModel, ColumnDecisionTreeTrainerInput> {
-    /**
-     * Function used to assign a value to a region.
-     */
-    private final IgniteFunction<DoubleStream, Double> regCalc;
-
-    /**
-     * Function used to calculate impurity in regions used by categorical features.
-     */
-    private final IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> continuousCalculatorProvider;
-
-    /**
-     * Categorical calculator provider.
-     **/
-    private final IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> categoricalCalculatorProvider;
-
-    /**
-     * Cache used for storing data for training.
-     */
-    private IgniteCache<RegionKey, List<RegionProjection>> prjsCache;
-
-    /**
-     * Minimal information gain.
-     */
-    private static final double MIN_INFO_GAIN = 1E-10;
-
-    /**
-     * Maximal depth of the decision tree.
-     */
-    private final int maxDepth;
-
-    /**
-     * Size of block which is used for storing regions in cache.
-     */
-    private static final int BLOCK_SIZE = 1 << 4;
-
-    /** Ignite instance. */
-    private final Ignite ignite;
-
-    /** Logger */
-    private final IgniteLogger log;
-
-    /**
-     * Construct {@link ColumnDecisionTreeTrainer}.
-     *
-     * @param maxDepth Maximal depth of the decision tree.
-     * @param continuousCalculatorProvider Provider of calculator of splits for region projection on continuous
-     * features.
-     * @param categoricalCalculatorProvider Provider of calculator of splits for region projection on categorical
-     * features.
-     * @param regCalc Function used to assign a value to a region.
-     */
-    public ColumnDecisionTreeTrainer(int maxDepth,
-        IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> continuousCalculatorProvider,
-        IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> categoricalCalculatorProvider,
-        IgniteFunction<DoubleStream, Double> regCalc,
-        Ignite ignite) {
-        this.maxDepth = maxDepth;
-        this.continuousCalculatorProvider = continuousCalculatorProvider;
-        this.categoricalCalculatorProvider = categoricalCalculatorProvider;
-        this.regCalc = regCalc;
-        this.ignite = ignite;
-        this.log = ignite.log();
-    }
-
-    /**
-     * Utility class used to get index of feature by which split is done and split info.
-     */
-    private static class IndexAndSplitInfo {
-        /**
-         * Index of feature by which split is done.
-         */
-        private final int featureIdx;
-
-        /**
-         * Split information.
-         */
-        private final SplitInfo info;
-
-        /**
-         * @param featureIdx Index of feature by which split is done.
-         * @param info Split information.
-         */
-        IndexAndSplitInfo(int featureIdx, SplitInfo info) {
-            this.featureIdx = featureIdx;
-            this.info = info;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return "IndexAndSplitInfo [featureIdx=" + featureIdx + ", info=" + info + ']';
-        }
-    }
-
-    /**
-     * Utility class used to build decision tree. Basically it is pointer to leaf node.
-     */
-    private static class TreeTip {
-        /** */
-        private Consumer<DecisionTreeNode> leafSetter;
-
-        /** */
-        private int depth;
-
-        /** */
-        TreeTip(Consumer<DecisionTreeNode> leafSetter, int depth) {
-            this.leafSetter = leafSetter;
-            this.depth = depth;
-        }
-    }
-
-    /**
-     * Utility class used as decision tree root node.
-     */
-    private static class RootNode implements DecisionTreeNode {
-        /** */
-        private DecisionTreeNode s;
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public double process(Vector v) {
-            return s.process(v);
-        }
-
-        /** */
-        void setSplit(DecisionTreeNode s) {
-            this.s = s;
-        }
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override public DecisionTreeModel train(ColumnDecisionTreeTrainerInput i) {
-        prjsCache = ProjectionsCache.getOrCreate(ignite);
-        IgniteCache<UUID, TrainingContext<D>> ctxtCache = ContextCache.getOrCreate(ignite);
-        SplitCache.getOrCreate(ignite);
-
-        UUID trainingUUID = UUID.randomUUID();
-
-        TrainingContext<D> ct = new TrainingContext<>(i, continuousCalculatorProvider.apply(i), categoricalCalculatorProvider.apply(i), trainingUUID, ignite);
-        ctxtCache.put(trainingUUID, ct);
-
-        CacheUtils.bcast(prjsCache.getName(), ignite, () -> {
-            Ignite ignite = Ignition.localIgnite();
-            IgniteCache<RegionKey, List<RegionProjection>> projCache = ProjectionsCache.getOrCreate(ignite);
-            IgniteCache<FeatureKey, double[]> featuresCache = FeaturesCache.getOrCreate(ignite);
-
-            Affinity<RegionKey> targetAffinity = ignite.affinity(ProjectionsCache.CACHE_NAME);
-
-            ClusterNode locNode = ignite.cluster().localNode();
-
-            Map<FeatureKey, double[]> fm = new ConcurrentHashMap<>();
-            Map<RegionKey, List<RegionProjection>> pm = new ConcurrentHashMap<>();
-
-            targetAffinity.
-                mapKeysToNodes(IntStream.range(0, i.featuresCount()).
-                    mapToObj(idx -> ProjectionsCache.key(idx, 0, i.affinityKey(idx, ignite), trainingUUID)).
-                    collect(Collectors.toSet())).getOrDefault(locNode, Collections.emptyList()).
-                forEach(k -> {
-                    FeatureProcessor vec;
-
-                    int featureIdx = k.featureIdx();
-
-                    IgniteCache<UUID, TrainingContext<D>> ctxCache = ContextCache.getOrCreate(ignite);
-                    TrainingContext ctx = ctxCache.get(trainingUUID);
-                    double[] vals = new double[ctx.labels().length];
-
-                    vec = ctx.featureProcessor(featureIdx);
-                    i.values(featureIdx).forEach(t -> vals[t.get1()] = t.get2());
-
-                    fm.put(getFeatureCacheKey(featureIdx, trainingUUID, i.affinityKey(featureIdx, ignite)), vals);
-
-                    List<RegionProjection> newReg = new ArrayList<>(BLOCK_SIZE);
-                    newReg.add(vec.createInitialRegion(getSamples(i.values(featureIdx), ctx.labels().length), vals, ctx.labels()));
-                    pm.put(k, newReg);
-                });
-
-            featuresCache.putAll(fm);
-            projCache.putAll(pm);
-
-            return null;
-        });
-
-        return doTrain(i, trainingUUID);
-    }
-
-    /**
-     * Get samples array.
-     *
-     * @param values Stream of tuples in the form of (index, value).
-     * @param size size of stream.
-     * @return Samples array.
-     */
-    private Integer[] getSamples(Stream<IgniteBiTuple<Integer, Double>> values, int size) {
-        Integer[] res = new Integer[size];
-
-        values.forEach(v -> res[v.get1()] = v.get1());
-
-        return res;
-    }
-
-    /** */
-    @NotNull
-    private DecisionTreeModel doTrain(ColumnDecisionTreeTrainerInput input, UUID uuid) {
-        RootNode root = new RootNode();
-
-        // List containing setters of leaves of the tree.
-        List<TreeTip> tips = new LinkedList<>();
-        tips.add(new TreeTip(root::setSplit, 0));
-
-        int curDepth = 0;
-        int regsCnt = 1;
-
-        int featuresCnt = input.featuresCount();
-        IntStream.range(0, featuresCnt).mapToObj(fIdx -> SplitCache.key(fIdx, input.affinityKey(fIdx, ignite), uuid)).
-            forEach(k -> SplitCache.getOrCreate(ignite).put(k, new IgniteBiTuple<>(0, 0.0)));
-        updateSplitCache(0, regsCnt, featuresCnt, ig -> i -> input.affinityKey(i, ig), uuid);
-
-        // TODO: IGNITE-5893 Currently if the best split makes tree deeper than max depth process will be terminated, but actually we should
-        // only stop when *any* improving split makes tree deeper than max depth. Can be fixed if we will store which
-        // regions cannot be split more and split only those that can.
-        while (true) {
-            long before = System.currentTimeMillis();
-
-            IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>> b = findBestSplitIndexForFeatures(featuresCnt, input::affinityKey, uuid);
-
-            long findBestRegIdx = System.currentTimeMillis() - before;
-
-            Integer bestFeatureIdx = b.get1();
-
-            Integer regIdx = b.get2().get1();
-            Double bestInfoGain = b.get2().get2();
-
-            if (regIdx >= 0 && bestInfoGain > MIN_INFO_GAIN) {
-                before = System.currentTimeMillis();
-
-                SplitInfo bi = ignite.compute().affinityCall(ProjectionsCache.CACHE_NAME,
-                    input.affinityKey(bestFeatureIdx, ignite),
-                    () -> {
-                        TrainingContext<ContinuousRegionInfo> ctx = ContextCache.getOrCreate(ignite).get(uuid);
-                        Ignite ignite = Ignition.localIgnite();
-                        RegionKey key = ProjectionsCache.key(bestFeatureIdx,
-                            regIdx / BLOCK_SIZE,
-                            input.affinityKey(bestFeatureIdx, Ignition.localIgnite()),
-                            uuid);
-                        RegionProjection reg = ProjectionsCache.getOrCreate(ignite).localPeek(key).get(regIdx % BLOCK_SIZE);
-                        return ctx.featureProcessor(bestFeatureIdx).findBestSplit(reg, ctx.values(bestFeatureIdx, ignite), ctx.labels(), regIdx);
-                    });
-
-                long findBestSplit = System.currentTimeMillis() - before;
-
-                IndexAndSplitInfo best = new IndexAndSplitInfo(bestFeatureIdx, bi);
-
-                regsCnt++;
-
-                if (log.isDebugEnabled())
-                    log.debug("Globally best: " + best.info + " idx time: " + findBestRegIdx + ", calculate best: " + findBestSplit + " fi: " + best.featureIdx + ", regs: " + regsCnt);
-                // Request bitset for split region.
-                int ind = best.info.regionIndex();
-
-                SparseBitSet bs = ignite.compute().affinityCall(ProjectionsCache.CACHE_NAME,
-                    input.affinityKey(bestFeatureIdx, ignite),
-                    () -> {
-                        Ignite ignite = Ignition.localIgnite();
-                        IgniteCache<FeatureKey, double[]> featuresCache = FeaturesCache.getOrCreate(ignite);
-                        IgniteCache<UUID, TrainingContext<D>> ctxCache = ContextCache.getOrCreate(ignite);
-                        TrainingContext ctx = ctxCache.localPeek(uuid);
-
-                        double[] values = featuresCache.localPeek(getFeatureCacheKey(bestFeatureIdx, uuid, input.affinityKey(bestFeatureIdx, Ignition.localIgnite())));
-                        RegionKey key = ProjectionsCache.key(bestFeatureIdx,
-                            regIdx / BLOCK_SIZE,
-                            input.affinityKey(bestFeatureIdx, Ignition.localIgnite()),
-                            uuid);
-                        RegionProjection reg = ProjectionsCache.getOrCreate(ignite).localPeek(key).get(regIdx % BLOCK_SIZE);
-                        return ctx.featureProcessor(bestFeatureIdx).calculateOwnershipBitSet(reg, values, best.info);
-
-                    });
-
-                SplitNode sn = best.info.createSplitNode(best.featureIdx);
-
-                TreeTip tipToSplit = tips.get(ind);
-                tipToSplit.leafSetter.accept(sn);
-                tipToSplit.leafSetter = sn::setLeft;
-                int d = tipToSplit.depth++;
-                tips.add(new TreeTip(sn::setRight, d));
-
-                if (d > curDepth) {
-                    curDepth = d;
-                    if (log.isDebugEnabled()) {
-                        log.debug("Depth: " + curDepth);
-                        log.debug("Cache size: " + prjsCache.size(CachePeekMode.PRIMARY));
-                    }
-                }
-
-                before = System.currentTimeMillis();
-                // Perform split on all feature vectors.
-                IgniteSupplier<Set<RegionKey>> bestRegsKeys = () -> IntStream.range(0, featuresCnt).
-                    mapToObj(fIdx -> ProjectionsCache.key(fIdx, ind / BLOCK_SIZE, input.affinityKey(fIdx, Ignition.localIgnite()), uuid)).
-                    collect(Collectors.toSet());
-
-                int rc = regsCnt;
-
-                // Perform split.
-                CacheUtils.update(prjsCache.getName(), ignite,
-                    (Ignite ign, Cache.Entry<RegionKey, List<RegionProjection>> e) -> {
-                        RegionKey k = e.getKey();
-
-                        List<RegionProjection> leftBlock = e.getValue();
-
-                        int fIdx = k.featureIdx();
-                        int idxInBlock = ind % BLOCK_SIZE;
-
-                        IgniteCache<UUID, TrainingContext<D>> ctxCache = ContextCache.getOrCreate(ign);
-                        TrainingContext<D> ctx = ctxCache.get(uuid);
-
-                        RegionProjection targetRegProj = leftBlock.get(idxInBlock);
-
-                        IgniteBiTuple<RegionProjection, RegionProjection> regs = ctx.
-                            performSplit(input, bs, fIdx, best.featureIdx, targetRegProj, best.info.leftData(), best.info.rightData(), ign);
-
-                        RegionProjection left = regs.get1();
-                        RegionProjection right = regs.get2();
-
-                        leftBlock.set(idxInBlock, left);
-                        RegionKey rightKey = ProjectionsCache.key(fIdx, (rc - 1) / BLOCK_SIZE, input.affinityKey(fIdx, ign), uuid);
-
-                        IgniteCache<RegionKey, List<RegionProjection>> c = ProjectionsCache.getOrCreate(ign);
-
-                        List<RegionProjection> rightBlock = rightKey.equals(k) ? leftBlock : c.localPeek(rightKey);
-
-                        if (rightBlock == null) {
-                            List<RegionProjection> newBlock = new ArrayList<>(BLOCK_SIZE);
-                            newBlock.add(right);
-                            return Stream.of(new CacheEntryImpl<>(k, leftBlock), new CacheEntryImpl<>(rightKey, newBlock));
-                        }
-                        else {
-                            rightBlock.add(right);
-                            return rightBlock.equals(k) ?
-                                Stream.of(new CacheEntryImpl<>(k, leftBlock)) :
-                                Stream.of(new CacheEntryImpl<>(k, leftBlock), new CacheEntryImpl<>(rightKey, rightBlock));
-                        }
-                    },
-                    bestRegsKeys);
-
-                if (log.isDebugEnabled())
-                    log.debug("Update of projections cache time: " + (System.currentTimeMillis() - before));
-
-                before = System.currentTimeMillis();
-
-                updateSplitCache(ind, rc, featuresCnt, ig -> i -> input.affinityKey(i, ig), uuid);
-
-                if (log.isDebugEnabled())
-                    log.debug("Update of split cache time: " + (System.currentTimeMillis() - before));
-            }
-            else {
-                if (log.isDebugEnabled())
-                    log.debug("Best split [bestFeatureIdx=" + bestFeatureIdx + ", bestInfoGain=" + bestInfoGain + "]");
-                break;
-            }
-        }
-
-        int rc = regsCnt;
-
-        IgniteSupplier<Iterable<Cache.Entry<RegionKey, List<RegionProjection>>>> featZeroRegs = () -> {
-            IgniteCache<RegionKey, List<RegionProjection>> projsCache = ProjectionsCache.getOrCreate(Ignition.localIgnite());
-
-            return () -> IntStream.range(0, (rc - 1) / BLOCK_SIZE + 1).
-                mapToObj(rBIdx -> ProjectionsCache.key(0, rBIdx, input.affinityKey(0, Ignition.localIgnite()), uuid)).
-                map(k -> (Cache.Entry<RegionKey, List<RegionProjection>>)new CacheEntryImpl<>(k, projsCache.localPeek(k))).iterator();
-        };
-
-        Map<Integer, Double> vals = CacheUtils.reduce(prjsCache.getName(), ignite,
-            (TrainingContext ctx, Cache.Entry<RegionKey, List<RegionProjection>> e, Map<Integer, Double> m) -> {
-                int regBlockIdx = e.getKey().regionBlockIndex();
-
-                if (e.getValue() != null) {
-                    for (int i = 0; i < e.getValue().size(); i++) {
-                        int regIdx = regBlockIdx * BLOCK_SIZE + i;
-                        RegionProjection reg = e.getValue().get(i);
-
-                        Double res = regCalc.apply(Arrays.stream(reg.sampleIndexes()).mapToDouble(s -> ctx.labels()[s]));
-                        m.put(regIdx, res);
-                    }
-                }
-
-                return m;
-            },
-            () -> ContextCache.getOrCreate(Ignition.localIgnite()).get(uuid),
-            featZeroRegs,
-            (infos, infos2) -> {
-                Map<Integer, Double> res = new HashMap<>();
-                res.putAll(infos);
-                res.putAll(infos2);
-                return res;
-            },
-            HashMap::new
-        );
-
-        int i = 0;
-        for (TreeTip tip : tips) {
-            tip.leafSetter.accept(new Leaf(vals.get(i)));
-            i++;
-        }
-
-        ProjectionsCache.clear(featuresCnt, rc, input::affinityKey, uuid, ignite);
-        ContextCache.getOrCreate(ignite).remove(uuid);
-        FeaturesCache.clear(featuresCnt, input::affinityKey, uuid, ignite);
-        SplitCache.clear(featuresCnt, input::affinityKey, uuid, ignite);
-
-        return new DecisionTreeModel(root.s);
-    }
-
-    /**
-     * Find the best split in the form (feature index, (index of region with the best split, impurity of region with the
-     * best split)).
-     *
-     * @param featuresCnt Count of features.
-     * @param affinity Affinity function.
-     * @param trainingUUID UUID of training.
-     * @return Best split in the form (feature index, (index of region with the best split, impurity of region with the
-     * best split)).
-     */
-    private IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>> findBestSplitIndexForFeatures(int featuresCnt,
-        IgniteBiFunction<Integer, Ignite, Object> affinity,
-        UUID trainingUUID) {
-        Set<Integer> featureIndexes = IntStream.range(0, featuresCnt).boxed().collect(Collectors.toSet());
-
-        return CacheUtils.reduce(SplitCache.CACHE_NAME, ignite,
-            (Object ctx, Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>> e, IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>> r) ->
-                Functions.MAX_GENERIC(new IgniteBiTuple<>(e.getKey().featureIdx(), e.getValue()), r, comparator()),
-            () -> null,
-            () -> SplitCache.localEntries(featureIndexes, affinity, trainingUUID),
-            (i1, i2) -> Functions.MAX_GENERIC(i1, i2, Comparator.comparingDouble(bt -> bt.get2().get2())),
-            () -> new IgniteBiTuple<>(-1, new IgniteBiTuple<>(-1, Double.NEGATIVE_INFINITY))
-        );
-    }
-
-    /** */
-    private static Comparator<IgniteBiTuple<Integer, IgniteBiTuple<Integer, Double>>> comparator() {
-        return Comparator.comparingDouble(bt -> bt != null && bt.get2() != null ? bt.get2().get2() : Double.NEGATIVE_INFINITY);
-    }
-
-    /**
-     * Update split cache.
-     *
-     * @param lastSplitRegionIdx Index of region which had last best split.
-     * @param regsCnt Count of regions.
-     * @param featuresCnt Count of features.
-     * @param affinity Affinity function.
-     * @param trainingUUID UUID of current training.
-     */
-    private void updateSplitCache(int lastSplitRegionIdx, int regsCnt, int featuresCnt,
-        IgniteCurriedBiFunction<Ignite, Integer, Object> affinity,
-        UUID trainingUUID) {
-        CacheUtils.update(SplitCache.CACHE_NAME, ignite,
-            (Ignite ign, Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>> e) -> {
-                Integer bestRegIdx = e.getValue().get1();
-                int fIdx = e.getKey().featureIdx();
-                TrainingContext ctx = ContextCache.getOrCreate(ign).get(trainingUUID);
-
-                Map<Integer, RegionProjection> toCompare;
-
-                // Fully recalculate best.
-                if (bestRegIdx == lastSplitRegionIdx)
-                    toCompare = ProjectionsCache.projectionsOfFeature(fIdx, maxDepth, regsCnt, BLOCK_SIZE, affinity.apply(ign), trainingUUID, ign);
-                    // Just compare previous best and two regions which are produced by split.
-                else
-                    toCompare = ProjectionsCache.projectionsOfRegions(fIdx, maxDepth,
-                        IntStream.of(bestRegIdx, lastSplitRegionIdx, regsCnt - 1), BLOCK_SIZE, affinity.apply(ign), trainingUUID, ign);
-
-                double[] values = ctx.values(fIdx, ign);
-                double[] labels = ctx.labels();
-
-                Optional<IgniteBiTuple<Integer, Double>> max = toCompare.entrySet().stream().
-                    map(ent -> {
-                        SplitInfo bestSplit = ctx.featureProcessor(fIdx).findBestSplit(ent.getValue(), values, labels, ent.getKey());
-                        return new IgniteBiTuple<>(ent.getKey(), bestSplit != null ? bestSplit.infoGain() : Double.NEGATIVE_INFINITY);
-                    }).
-                    max(Comparator.comparingDouble(IgniteBiTuple::get2));
-
-                return max.<Stream<Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>>>>
-                    map(objects -> Stream.of(new CacheEntryImpl<>(e.getKey(), objects))).orElseGet(Stream::empty);
-            },
-            () -> IntStream.range(0, featuresCnt).mapToObj(fIdx -> SplitCache.key(fIdx, affinity.apply(ignite).apply(fIdx), trainingUUID)).collect(Collectors.toSet())
-        );
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index bf8790b705de8..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/ColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.Map;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.lang.IgniteBiTuple;
-
-/**
- * Input for {@link ColumnDecisionTreeTrainer}.
- */
-public interface ColumnDecisionTreeTrainerInput {
-    /**
-     * Projection of data on feature with the given index.
-     *
-     * @param idx Feature index.
-     * @return Projection of data on feature with the given index.
-     */
-    Stream<IgniteBiTuple<Integer, Double>> values(int idx);
-
-    /**
-     * Labels.
-     *
-     * @param ignite Ignite instance.
-     */
-    double[] labels(Ignite ignite);
-
-    /** Information about which features are categorical in the form of feature index -> number of categories. */
-    Map<Integer, Integer> catFeaturesInfo();
-
-    /** Number of features. */
-    int featuresCount();
-
-    /**
-     * Get affinity key for the given column index.
-     * Affinity key should be pure-functionally dependent from idx.
-     */
-    Object affinityKey(int idx, Ignite ignite);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java
deleted file mode 100644
index 3da6bad218f11..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/MatrixColumnDecisionTreeTrainerInput.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey;
-import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Adapter of SparseDistributedMatrix to ColumnDecisionTreeTrainerInput.
- * Sparse SparseDistributedMatrix should be in {@link StorageConstants#COLUMN_STORAGE_MODE} and
- * should contain samples in rows last position in row being label of this sample.
- */
-public class MatrixColumnDecisionTreeTrainerInput extends CacheColumnDecisionTreeTrainerInput<RowColMatrixKey, Map<Integer, Double>> {
-    /**
-     * @param m Sparse SparseDistributedMatrix should be in {@link StorageConstants#COLUMN_STORAGE_MODE}
-     * containing samples in rows last position in row being label of this sample.
-     * @param catFeaturesInfo Information about which features are categorical in form of feature index -> number of
-     * categories.
-     */
-    public MatrixColumnDecisionTreeTrainerInput(SparseDistributedMatrix m, Map<Integer, Integer> catFeaturesInfo) {
-        super(((SparseDistributedMatrixStorage)m.getStorage()).cache(),
-            () -> Stream.of(new SparseMatrixKey(m.columnSize() - 1, m.getUUID(), m.columnSize() - 1)),
-            valuesMapper(m),
-            labels(m),
-            keyMapper(m),
-            catFeaturesInfo,
-            m.columnSize() - 1,
-            m.rowSize());
-    }
-
-    /** Values mapper. See {@link CacheColumnDecisionTreeTrainerInput#valuesMapper} */
-    @NotNull
-    private static IgniteFunction<Cache.Entry<RowColMatrixKey, Map<Integer, Double>>, Stream<IgniteBiTuple<Integer, Double>>> valuesMapper(
-        SparseDistributedMatrix m) {
-        return ent -> {
-            Map<Integer, Double> map = ent.getValue() != null ? ent.getValue() : new HashMap<>();
-            return IntStream.range(0, m.rowSize()).mapToObj(k -> new IgniteBiTuple<>(k, map.getOrDefault(k, 0.0)));
-        };
-    }
-
-    /** Key mapper. See {@link CacheColumnDecisionTreeTrainerInput#keyMapper} */
-    @NotNull private static IgniteFunction<Integer, Stream<RowColMatrixKey>> keyMapper(SparseDistributedMatrix m) {
-        return i -> Stream.of(new SparseMatrixKey(i, ((SparseDistributedMatrixStorage)m.getStorage()).getUUID(), i));
-    }
-
-    /** Labels mapper. See {@link CacheColumnDecisionTreeTrainerInput#labelsMapper} */
-    @NotNull private static IgniteFunction<Map<Integer, Double>, DoubleStream> labels(SparseDistributedMatrix m) {
-        return mp -> IntStream.range(0, m.rowSize()).mapToDouble(k -> mp.getOrDefault(k, 0.0));
-    }
-
-    /** {@inheritDoc} */
-    @Override public Object affinityKey(int idx, Ignite ignite) {
-        return idx;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java
deleted file mode 100644
index e95f57b6a79a0..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/RegionProjection.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import org.apache.ignite.ml.trees.RegionInfo;
-
-/**
- * Projection of region on given feature.
- *
- * @param <D> Data of region.
- */
-public class RegionProjection<D extends RegionInfo> implements Externalizable {
-    /** Samples projections. */
-    protected Integer[] sampleIndexes;
-
-    /** Region data */
-    protected D data;
-
-    /** Depth of this region. */
-    protected int depth;
-
-    /**
-     * @param sampleIndexes Samples indexes.
-     * @param data Region data.
-     * @param depth Depth of this region.
-     */
-    public RegionProjection(Integer[] sampleIndexes, D data, int depth) {
-        this.data = data;
-        this.depth = depth;
-        this.sampleIndexes = sampleIndexes;
-    }
-
-    /**
-     * No-op constructor used for serialization/deserialization.
-     */
-    public RegionProjection() {
-        // No-op.
-    }
-
-    /**
-     * Get samples indexes.
-     *
-     * @return Samples indexes.
-     */
-    public Integer[] sampleIndexes() {
-        return sampleIndexes;
-    }
-
-    /**
-     * Get region data.
-     *
-     * @return Region data.
-     */
-    public D data() {
-        return data;
-    }
-
-    /**
-     * Get region depth.
-     *
-     * @return Region depth.
-     */
-    public int depth() {
-        return depth;
-    }
-
-    /** {@inheritDoc} */
-    @Override public void writeExternal(ObjectOutput out) throws IOException {
-        out.writeInt(sampleIndexes.length);
-
-        for (Integer sampleIndex : sampleIndexes)
-            out.writeInt(sampleIndex);
-
-        out.writeObject(data);
-        out.writeInt(depth);
-    }
-
-    /** {@inheritDoc} */
-    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-        int size = in.readInt();
-
-        sampleIndexes = new Integer[size];
-
-        for (int i = 0; i < size; i++)
-            sampleIndexes[i] = in.readInt();
-
-        data = (D)in.readObject();
-        depth = in.readInt();
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java
deleted file mode 100644
index 6415dab4bbfce..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/TrainingContext.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.Map;
-import java.util.UUID;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.CategoricalFeatureProcessor;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousFeatureProcessor;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureProcessor;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache.COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME;
-
-/**
- * Context of training with {@link ColumnDecisionTreeTrainer}.
- *
- * @param <D> Class for storing of information used in calculation of impurity of continuous feature region.
- */
-public class TrainingContext<D extends ContinuousRegionInfo> {
-    /** Input for training with {@link ColumnDecisionTreeTrainer}. */
-    private final ColumnDecisionTreeTrainerInput input;
-
-    /** Labels. */
-    private final double[] labels;
-
-    /** Calculator used for finding splits of region of continuous features. */
-    private final ContinuousSplitCalculator<D> continuousSplitCalculator;
-
-    /** Calculator used for finding splits of region of categorical feature. */
-    private final IgniteFunction<DoubleStream, Double> categoricalSplitCalculator;
-
-    /** UUID of current training. */
-    private final UUID trainingUUID;
-
-    /**
-     * Construct context for training with {@link ColumnDecisionTreeTrainer}.
-     *
-     * @param input Input for training.
-     * @param continuousSplitCalculator Calculator used for calculations of splits of continuous features regions.
-     * @param categoricalSplitCalculator Calculator used for calculations of splits of categorical features regions.
-     * @param trainingUUID UUID of the current training.
-     * @param ignite Ignite instance.
-     */
-    public TrainingContext(ColumnDecisionTreeTrainerInput input,
-        ContinuousSplitCalculator<D> continuousSplitCalculator,
-        IgniteFunction<DoubleStream, Double> categoricalSplitCalculator,
-        UUID trainingUUID,
-        Ignite ignite) {
-        this.input = input;
-        this.labels = input.labels(ignite);
-        this.continuousSplitCalculator = continuousSplitCalculator;
-        this.categoricalSplitCalculator = categoricalSplitCalculator;
-        this.trainingUUID = trainingUUID;
-    }
-
-    /**
-     * Get processor used for calculating splits of categorical features.
-     *
-     * @param catsCnt Count of categories.
-     * @return Processor used for calculating splits of categorical features.
-     */
-    public CategoricalFeatureProcessor categoricalFeatureProcessor(int catsCnt) {
-        return new CategoricalFeatureProcessor(categoricalSplitCalculator, catsCnt);
-    }
-
-    /**
-     * Get processor used for calculating splits of continuous features.
-     *
-     * @return Processor used for calculating splits of continuous features.
-     */
-    public ContinuousFeatureProcessor<D> continuousFeatureProcessor() {
-        return new ContinuousFeatureProcessor<>(continuousSplitCalculator);
-    }
-
-    /**
-     * Get labels.
-     *
-     * @return Labels.
-     */
-    public double[] labels() {
-        return labels;
-    }
-
-    /**
-     * Get values of feature with given index.
-     *
-     * @param featIdx Feature index.
-     * @param ignite Ignite instance.
-     * @return Values of feature with given index.
-     */
-    public double[] values(int featIdx, Ignite ignite) {
-        IgniteCache<FeaturesCache.FeatureKey, double[]> featuresCache = ignite.getOrCreateCache(COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME);
-        return featuresCache.localPeek(FeaturesCache.getFeatureCacheKey(featIdx, trainingUUID, input.affinityKey(featIdx, ignite)));
-    }
-
-    /**
-     * Perform best split on the given region projection.
-     *
-     * @param input Input of {@link ColumnDecisionTreeTrainer} performing split.
-     * @param bitSet Bit set specifying split.
-     * @param targetFeatIdx Index of feature for performing split.
-     * @param bestFeatIdx Index of feature with best split.
-     * @param targetRegionPrj Projection of region to split on feature with index {@code featureIdx}.
-     * @param leftData Data of left region of split.
-     * @param rightData Data of right region of split.
-     * @param ignite Ignite instance.
-     * @return Perform best split on the given region projection.
-     */
-    public IgniteBiTuple<RegionProjection, RegionProjection> performSplit(ColumnDecisionTreeTrainerInput input,
-        SparseBitSet bitSet, int targetFeatIdx, int bestFeatIdx, RegionProjection targetRegionPrj, RegionInfo leftData,
-        RegionInfo rightData, Ignite ignite) {
-
-        Map<Integer, Integer> catFeaturesInfo = input.catFeaturesInfo();
-
-        if (!catFeaturesInfo.containsKey(targetFeatIdx) && !catFeaturesInfo.containsKey(bestFeatIdx))
-            return continuousFeatureProcessor().performSplit(bitSet, targetRegionPrj, (D)leftData, (D)rightData);
-        else if (catFeaturesInfo.containsKey(targetFeatIdx))
-            return categoricalFeatureProcessor(catFeaturesInfo.get(targetFeatIdx)).performSplitGeneric(bitSet, values(targetFeatIdx, ignite), targetRegionPrj, leftData, rightData);
-        return continuousFeatureProcessor().performSplitGeneric(bitSet, labels, targetRegionPrj, leftData, rightData);
-    }
-
-    /**
-     * Processor used for calculating splits for feature with the given index.
-     *
-     * @param featureIdx Index of feature to process.
-     * @return Processor used for calculating splits for feature with the given index.
-     */
-    public FeatureProcessor featureProcessor(int featureIdx) {
-        return input.catFeaturesInfo().containsKey(featureIdx) ? categoricalFeatureProcessor(input.catFeaturesInfo().get(featureIdx)) : continuousFeatureProcessor();
-    }
-
-    /**
-     * Shortcut for affinity key.
-     *
-     * @param idx Feature index.
-     * @return Affinity key.
-     */
-    public Object affinityKey(int idx) {
-        return input.affinityKey(idx, Ignition.localIgnite());
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java
deleted file mode 100644
index 51ea359a3e00f..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ContextCache.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.UUID;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.TrainingContext;
-
-/**
- * Class for operations related to cache containing training context for {@link ColumnDecisionTreeTrainer}.
- */
-public class ContextCache {
-    /**
-     * Name of cache containing training context for {@link ColumnDecisionTreeTrainer}.
-     */
-    public static final String COLUMN_DECISION_TREE_TRAINER_CONTEXT_CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_CONTEXT_CACHE_NAME";
-
-    /**
-     * Get or create cache for training context.
-     *
-     * @param ignite Ignite instance.
-     * @param <D> Class storing information about continuous regions.
-     * @return Cache for training context.
-     */
-    public static <D extends ContinuousRegionInfo> IgniteCache<UUID, TrainingContext<D>> getOrCreate(Ignite ignite) {
-        CacheConfiguration<UUID, TrainingContext<D>> cfg = new CacheConfiguration<>();
-
-        cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-
-        cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
-        cfg.setEvictionPolicy(null);
-
-        cfg.setCopyOnRead(false);
-
-        cfg.setCacheMode(CacheMode.REPLICATED);
-
-        cfg.setOnheapCacheEnabled(true);
-
-        cfg.setReadFromBackup(true);
-
-        cfg.setName(COLUMN_DECISION_TREE_TRAINER_CONTEXT_CACHE_NAME);
-
-        return ignite.getOrCreateCache(cfg);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java
deleted file mode 100644
index fcc1f16489317..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/FeaturesCache.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-
-/**
- * Cache storing features for {@link ColumnDecisionTreeTrainer}.
- */
-public class FeaturesCache {
-    /**
-     * Name of cache which is used for storing features for {@link ColumnDecisionTreeTrainer}.
-     */
-    public static final String COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME";
-
-    /**
-     * Key of features cache.
-     */
-    public static class FeatureKey {
-        /** Column key of cache used as input for {@link ColumnDecisionTreeTrainer}. */
-        @AffinityKeyMapped
-        private Object parentColKey;
-
-        /** Index of feature. */
-        private final int featureIdx;
-
-        /** UUID of training. */
-        private final UUID trainingUUID;
-
-        /**
-         * Construct FeatureKey.
-         *
-         * @param featureIdx Feature index.
-         * @param trainingUUID UUID of training.
-         * @param parentColKey Column key of cache used as input.
-         */
-        public FeatureKey(int featureIdx, UUID trainingUUID, Object parentColKey) {
-            this.parentColKey = parentColKey;
-            this.featureIdx = featureIdx;
-            this.trainingUUID = trainingUUID;
-            this.parentColKey = parentColKey;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean equals(Object o) {
-            if (this == o)
-                return true;
-            if (o == null || getClass() != o.getClass())
-                return false;
-
-            FeatureKey key = (FeatureKey)o;
-
-            if (featureIdx != key.featureIdx)
-                return false;
-            return trainingUUID != null ? trainingUUID.equals(key.trainingUUID) : key.trainingUUID == null;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hashCode() {
-            int res = trainingUUID != null ? trainingUUID.hashCode() : 0;
-            res = 31 * res + featureIdx;
-            return res;
-        }
-    }
-
-    /**
-     * Create new projections cache for ColumnDecisionTreeTrainer if needed.
-     *
-     * @param ignite Ignite instance.
-     */
-    public static IgniteCache<FeatureKey, double[]> getOrCreate(Ignite ignite) {
-        CacheConfiguration<FeatureKey, double[]> cfg = new CacheConfiguration<>();
-
-        // Write to primary.
-        cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
-        // Atomic transactions only.
-        cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
-        // No eviction.
-        cfg.setEvictionPolicy(null);
-
-        // No copying of values.
-        cfg.setCopyOnRead(false);
-
-        // Cache is partitioned.
-        cfg.setCacheMode(CacheMode.PARTITIONED);
-
-        cfg.setOnheapCacheEnabled(true);
-
-        cfg.setBackups(0);
-
-        cfg.setName(COLUMN_DECISION_TREE_TRAINER_FEATURES_CACHE_NAME);
-
-        return ignite.getOrCreateCache(cfg);
-    }
-
-    /**
-     * Construct FeatureKey from index, uuid and affinity key.
-     *
-     * @param idx Feature index.
-     * @param uuid UUID of training.
-     * @param aff Affinity key.
-     * @return FeatureKey.
-     */
-    public static FeatureKey getFeatureCacheKey(int idx, UUID uuid, Object aff) {
-        return new FeatureKey(idx, uuid, aff);
-    }
-
-    /**
-     * Clear all data from features cache related to given training.
-     *
-     * @param featuresCnt Count of features.
-     * @param affinity Affinity function.
-     * @param uuid Training uuid.
-     * @param ignite Ignite instance.
-     */
-    public static void clear(int featuresCnt, IgniteBiFunction<Integer, Ignite, Object> affinity, UUID uuid,
-        Ignite ignite) {
-        Set<FeatureKey> toRmv = IntStream.range(0, featuresCnt).boxed().map(fIdx -> getFeatureCacheKey(fIdx, uuid, affinity.apply(fIdx, ignite))).collect(Collectors.toSet());
-
-        getOrCreate(ignite).removeAll(toRmv);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java
deleted file mode 100644
index 080cb6636ff7a..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/ProjectionsCache.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.PrimitiveIterator;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-/**
- * Cache used for storing data of region projections on features.
- */
-public class ProjectionsCache {
-    /**
-     * Name of cache which is used for storing data of region projections on features of {@link
-     * ColumnDecisionTreeTrainer}.
-     */
-    public static final String CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_PROJECTIONS_CACHE_NAME";
-
-    /**
-     * Key of region projections cache.
-     */
-    public static class RegionKey {
-        /** Column key of cache used as input for {@link ColumnDecisionTreeTrainer}. */
-        @AffinityKeyMapped
-        private final Object parentColKey;
-
-        /** Feature index. */
-        private final int featureIdx;
-
-        /** Region index. */
-        private final int regBlockIdx;
-
-        /** Training UUID. */
-        private final UUID trainingUUID;
-
-        /**
-         * Construct a RegionKey from feature index, index of block, key of column in input cache and UUID of training.
-         *
-         * @param featureIdx Feature index.
-         * @param regBlockIdx Index of block.
-         * @param parentColKey Key of column in input cache.
-         * @param trainingUUID UUID of training.
-         */
-        public RegionKey(int featureIdx, int regBlockIdx, Object parentColKey, UUID trainingUUID) {
-            this.featureIdx = featureIdx;
-            this.regBlockIdx = regBlockIdx;
-            this.trainingUUID = trainingUUID;
-            this.parentColKey = parentColKey;
-        }
-
-        /**
-         * Feature index.
-         *
-         * @return Feature index.
-         */
-        public int featureIdx() {
-            return featureIdx;
-        }
-
-        /**
-         * Region block index.
-         *
-         * @return Region block index.
-         */
-        public int regionBlockIndex() {
-            return regBlockIdx;
-        }
-
-        /**
-         * UUID of training.
-         *
-         * @return UUID of training.
-         */
-        public UUID trainingUUID() {
-            return trainingUUID;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean equals(Object o) {
-            if (this == o)
-                return true;
-            if (o == null || getClass() != o.getClass())
-                return false;
-
-            RegionKey key = (RegionKey)o;
-
-            if (featureIdx != key.featureIdx)
-                return false;
-            if (regBlockIdx != key.regBlockIdx)
-                return false;
-            return trainingUUID != null ? trainingUUID.equals(key.trainingUUID) : key.trainingUUID == null;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hashCode() {
-            int res = trainingUUID != null ? trainingUUID.hashCode() : 0;
-            res = 31 * res + featureIdx;
-            res = 31 * res + regBlockIdx;
-            return res;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return "RegionKey [" +
-                "parentColKey=" + parentColKey +
-                ", featureIdx=" + featureIdx +
-                ", regBlockIdx=" + regBlockIdx +
-                ", trainingUUID=" + trainingUUID +
-                ']';
-        }
-    }
-
-    /**
-     * Affinity service for region projections cache.
-     *
-     * @return Affinity service for region projections cache.
-     */
-    public static Affinity<RegionKey> affinity() {
-        return Ignition.localIgnite().affinity(CACHE_NAME);
-    }
-
-    /**
-     * Get or create region projections cache.
-     *
-     * @param ignite Ignite instance.
-     * @return Region projections cache.
-     */
-    public static IgniteCache<RegionKey, List<RegionProjection>> getOrCreate(Ignite ignite) {
-        CacheConfiguration<RegionKey, List<RegionProjection>> cfg = new CacheConfiguration<>();
-
-        // Write to primary.
-        cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
-        // Atomic transactions only.
-        cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
-        // No eviction.
-        cfg.setEvictionPolicy(null);
-
-        // No copying of values.
-        cfg.setCopyOnRead(false);
-
-        // Cache is partitioned.
-        cfg.setCacheMode(CacheMode.PARTITIONED);
-
-        cfg.setBackups(0);
-
-        cfg.setOnheapCacheEnabled(true);
-
-        cfg.setName(CACHE_NAME);
-
-        return ignite.getOrCreateCache(cfg);
-    }
-
-    /**
-     * Get region projections in the form of map (regionIndex -> regionProjections).
-     *
-     * @param featureIdx Feature index.
-     * @param maxDepth Max depth of decision tree.
-     * @param regionIndexes Indexes of regions for which we want get projections.
-     * @param blockSize Size of regions block.
-     * @param affinity Affinity function.
-     * @param trainingUUID UUID of training.
-     * @param ignite Ignite instance.
-     * @return Region projections in the form of map (regionIndex -> regionProjections).
-     */
-    public static Map<Integer, RegionProjection> projectionsOfRegions(int featureIdx, int maxDepth,
-        IntStream regionIndexes, int blockSize, IgniteFunction<Integer, Object> affinity, UUID trainingUUID,
-        Ignite ignite) {
-        HashMap<Integer, RegionProjection> regsForSearch = new HashMap<>();
-        IgniteCache<RegionKey, List<RegionProjection>> cache = getOrCreate(ignite);
-
-        PrimitiveIterator.OfInt itr = regionIndexes.iterator();
-
-        int curBlockIdx = -1;
-        List<RegionProjection> block = null;
-
-        Object affinityKey = affinity.apply(featureIdx);
-
-        while (itr.hasNext()) {
-            int i = itr.nextInt();
-
-            int blockIdx = i / blockSize;
-
-            if (blockIdx != curBlockIdx) {
-                block = cache.localPeek(key(featureIdx, blockIdx, affinityKey, trainingUUID));
-                curBlockIdx = blockIdx;
-            }
-
-            if (block == null)
-                throw new IllegalStateException("Unexpected null block at index " + i);
-
-            RegionProjection reg = block.get(i % blockSize);
-
-            if (reg.depth() < maxDepth)
-                regsForSearch.put(i, reg);
-        }
-
-        return regsForSearch;
-    }
-
-    /**
-     * Returns projections of regions on given feature filtered by maximal depth in the form of (region index -> region
-     * projection).
-     *
-     * @param featureIdx Feature index.
-     * @param maxDepth Maximal depth of the tree.
-     * @param regsCnt Count of regions.
-     * @param blockSize Size of regions blocks.
-     * @param affinity Affinity function.
-     * @param trainingUUID UUID of training.
-     * @param ignite Ignite instance.
-     * @return Projections of regions on given feature filtered by maximal depth in the form of (region index -> region
-     *      projection).
-     */
-    public static Map<Integer, RegionProjection> projectionsOfFeature(int featureIdx, int maxDepth, int regsCnt,
-        int blockSize, IgniteFunction<Integer, Object> affinity, UUID trainingUUID, Ignite ignite) {
-        return projectionsOfRegions(featureIdx, maxDepth, IntStream.range(0, regsCnt), blockSize, affinity, trainingUUID, ignite);
-    }
-
-    /**
-     * Construct key for projections cache.
-     *
-     * @param featureIdx Feature index.
-     * @param regBlockIdx Region block index.
-     * @param parentColKey Column key of cache used as input for {@link ColumnDecisionTreeTrainer}.
-     * @param uuid UUID of training.
-     * @return Key for projections cache.
-     */
-    public static RegionKey key(int featureIdx, int regBlockIdx, Object parentColKey, UUID uuid) {
-        return new RegionKey(featureIdx, regBlockIdx, parentColKey, uuid);
-    }
-
-    /**
-     * Clear data from projections cache related to given training.
-     *
-     * @param featuresCnt Features count.
-     * @param regs Regions count.
-     * @param aff Affinity function.
-     * @param uuid UUID of training.
-     * @param ignite Ignite instance.
-     */
-    public static void clear(int featuresCnt, int regs, IgniteBiFunction<Integer, Ignite, Object> aff, UUID uuid,
-        Ignite ignite) {
-        Set<RegionKey> toRmv = IntStream.range(0, featuresCnt).boxed().
-            flatMap(fIdx -> IntStream.range(0, regs).boxed().map(reg -> new IgniteBiTuple<>(fIdx, reg))).
-            map(t -> key(t.get1(), t.get2(), aff.apply(t.get1(), ignite), uuid)).
-            collect(Collectors.toSet());
-
-        getOrCreate(ignite).removeAll(toRmv);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java
deleted file mode 100644
index ecbc86198ece6..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/SplitCache.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
-
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Set;
-import java.util.UUID;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import javax.cache.Cache;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.processors.cache.CacheEntryImpl;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-
-/**
- * Class for working with cache used for storing of best splits during training with {@link ColumnDecisionTreeTrainer}.
- */
-public class SplitCache {
-    /** Name of splits cache. */
-    public static final String CACHE_NAME = "COLUMN_DECISION_TREE_TRAINER_SPLIT_CACHE_NAME";
-
-    /**
-     * Class used for keys in the splits cache.
-     */
-    public static class SplitKey {
-        /** UUID of current training. */
-        private final UUID trainingUUID;
-
-        /** Affinity key of input data. */
-        @AffinityKeyMapped
-        private final Object parentColKey;
-
-        /** Index of feature by which the split is made. */
-        private final int featureIdx;
-
-        /**
-         * Construct SplitKey.
-         *
-         * @param trainingUUID UUID of the training.
-         * @param parentColKey Affinity key used to ensure that cache entry for given feature will be on the same node
-         * as column with that feature in input.
-         * @param featureIdx Feature index.
-         */
-        public SplitKey(UUID trainingUUID, Object parentColKey, int featureIdx) {
-            this.trainingUUID = trainingUUID;
-            this.featureIdx = featureIdx;
-            this.parentColKey = parentColKey;
-        }
-
-        /** Get UUID of current training. */
-        public UUID trainingUUID() {
-            return trainingUUID;
-        }
-
-        /**
-         * Get feature index.
-         *
-         * @return Feature index.
-         */
-        public int featureIdx() {
-            return featureIdx;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean equals(Object o) {
-            if (this == o)
-                return true;
-            if (o == null || getClass() != o.getClass())
-                return false;
-
-            SplitKey splitKey = (SplitKey)o;
-
-            if (featureIdx != splitKey.featureIdx)
-                return false;
-            return trainingUUID != null ? trainingUUID.equals(splitKey.trainingUUID) : splitKey.trainingUUID == null;
-
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hashCode() {
-            int res = trainingUUID != null ? trainingUUID.hashCode() : 0;
-            res = 31 * res + featureIdx;
-            return res;
-        }
-    }
-
-    /**
-     * Construct the key for splits cache.
-     *
-     * @param featureIdx Feature index.
-     * @param parentColKey Affinity key used to ensure that cache entry for given feature will be on the same node as
-     * column with that feature in input.
-     * @param uuid UUID of current training.
-     * @return Key for splits cache.
-     */
-    public static SplitKey key(int featureIdx, Object parentColKey, UUID uuid) {
-        return new SplitKey(uuid, parentColKey, featureIdx);
-    }
-
-    /**
-     * Get or create splits cache.
-     *
-     * @param ignite Ignite instance.
-     * @return Splits cache.
-     */
-    public static IgniteCache<SplitKey, IgniteBiTuple<Integer, Double>> getOrCreate(Ignite ignite) {
-        CacheConfiguration<SplitKey, IgniteBiTuple<Integer, Double>> cfg = new CacheConfiguration<>();
-
-        // Write to primary.
-        cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
-        // Atomic transactions only.
-        cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
-        // No eviction.
-        cfg.setEvictionPolicy(null);
-
-        // No copying of values.
-        cfg.setCopyOnRead(false);
-
-        // Cache is partitioned.
-        cfg.setCacheMode(CacheMode.PARTITIONED);
-
-        cfg.setBackups(0);
-
-        cfg.setOnheapCacheEnabled(true);
-
-        cfg.setName(CACHE_NAME);
-
-        return ignite.getOrCreateCache(cfg);
-    }
-
-    /**
-     * Affinity function used in splits cache.
-     *
-     * @return Affinity function used in splits cache.
-     */
-    public static Affinity<SplitKey> affinity() {
-        return Ignition.localIgnite().affinity(CACHE_NAME);
-    }
-
-    /**
-     * Returns local entries for keys corresponding to {@code featureIndexes}.
-     *
-     * @param featureIndexes Index of features.
-     * @param affinity Affinity function.
-     * @param trainingUUID UUID of training.
-     * @return local entries for keys corresponding to {@code featureIndexes}.
-     */
-    public static Iterable<Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>>> localEntries(
-        Set<Integer> featureIndexes,
-        IgniteBiFunction<Integer, Ignite, Object> affinity,
-        UUID trainingUUID) {
-        Ignite ignite = Ignition.localIgnite();
-        Set<SplitKey> keys = featureIndexes.stream().map(fIdx -> new SplitKey(trainingUUID, affinity.apply(fIdx, ignite), fIdx)).collect(Collectors.toSet());
-
-        Collection<SplitKey> locKeys = affinity().mapKeysToNodes(keys).getOrDefault(ignite.cluster().localNode(), Collections.emptyList());
-
-        return () -> {
-            Function<SplitKey, Cache.Entry<SplitKey, IgniteBiTuple<Integer, Double>>> f = k -> (new CacheEntryImpl<>(k, getOrCreate(ignite).localPeek(k)));
-            return locKeys.stream().map(f).iterator();
-        };
-    }
-
-    /**
-     * Clears data related to current training from splits cache related to given training.
-     *
-     * @param featuresCnt Count of features.
-     * @param affinity Affinity function.
-     * @param uuid UUID of the given training.
-     * @param ignite Ignite instance.
-     */
-    public static void clear(int featuresCnt, IgniteBiFunction<Integer, Ignite, Object> affinity, UUID uuid,
-        Ignite ignite) {
-        Set<SplitKey> toRmv = IntStream.range(0, featuresCnt).boxed().map(fIdx -> new SplitKey(uuid, affinity.apply(fIdx, ignite), fIdx)).collect(Collectors.toSet());
-
-        getOrCreate(ignite).removeAll(toRmv);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java
deleted file mode 100644
index 0a488ab1f0289..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/caches/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains cache configurations for columnbased decision tree trainer with some related logic.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.caches;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java
deleted file mode 100644
index 9fd4c6682b0d8..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/ContinuousSplitCalculators.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.functions.IgniteCurriedBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-
-/** Continuous Split Calculators. */
-public class ContinuousSplitCalculators {
-    /** Variance split calculator. */
-    public static IgniteFunction<ColumnDecisionTreeTrainerInput, VarianceSplitCalculator> VARIANCE = input ->
-        new VarianceSplitCalculator();
-
-    /** Gini split calculator. */
-    public static IgniteCurriedBiFunction<Ignite, ColumnDecisionTreeTrainerInput, GiniSplitCalculator> GINI = ignite ->
-        input -> new GiniSplitCalculator(input.labels(ignite));
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java
deleted file mode 100644
index 259c84c76fddf..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/GiniSplitCalculator.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
-
-import it.unimi.dsi.fastutil.doubles.Double2IntArrayMap;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.PrimitiveIterator;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousSplitInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-
-/**
- * Calculator for Gini impurity.
- */
-public class GiniSplitCalculator implements ContinuousSplitCalculator<GiniSplitCalculator.GiniData> {
-    /** Mapping assigning index to each member value */
-    private final Map<Double, Integer> mapping = new Double2IntArrayMap();
-
-    /**
-     * Create Gini split calculator from labels.
-     *
-     * @param labels Labels.
-     */
-    public GiniSplitCalculator(double[] labels) {
-        int i = 0;
-
-        for (double label : labels) {
-            if (!mapping.containsKey(label)) {
-                mapping.put(label, i);
-                i++;
-            }
-        }
-    }
-
-    /** {@inheritDoc} */
-    @Override public GiniData calculateRegionInfo(DoubleStream s, int l) {
-        PrimitiveIterator.OfDouble itr = s.iterator();
-
-        Map<Double, Integer> m = new HashMap<>();
-
-        int size = 0;
-
-        while (itr.hasNext()) {
-            size++;
-            m.compute(itr.next(), (a, i) -> i != null ? i + 1 : 1);
-        }
-
-        double c2 = m.values().stream().mapToDouble(v -> v * v).sum();
-
-        int[] cnts = new int[mapping.size()];
-
-        m.forEach((key, value) -> cnts[mapping.get(key)] = value);
-
-        return new GiniData(size != 0 ? 1 - c2 / (size * size) : 0.0, size, cnts, c2);
-    }
-
-    /** {@inheritDoc} */
-    @Override public SplitInfo<GiniData> splitRegion(Integer[] s, double[] values, double[] labels, int regionIdx,
-        GiniData d) {
-        int size = d.getSize();
-
-        double lg = 0.0;
-        double rg = d.impurity();
-
-        double lc2 = 0.0;
-        double rc2 = d.c2;
-        int lSize = 0;
-
-        double minImpurity = d.impurity() * size;
-        double curThreshold;
-        double curImpurity;
-        double threshold = Double.NEGATIVE_INFINITY;
-
-        int i = 0;
-        int nextIdx = s[0];
-        i++;
-        double[] lrImps = new double[] {0.0, d.impurity(), lc2, rc2};
-
-        int[] lMapCur = new int[d.counts().length];
-        int[] rMapCur = new int[d.counts().length];
-
-        System.arraycopy(d.counts(), 0, rMapCur, 0, d.counts().length);
-
-        int[] lMap = new int[d.counts().length];
-        int[] rMap = new int[d.counts().length];
-
-        System.arraycopy(d.counts(), 0, rMap, 0, d.counts().length);
-
-        do {
-            // Process all values equal to prev.
-            while (i < s.length) {
-                moveLeft(labels[nextIdx], i, size - i, lMapCur, rMapCur, lrImps);
-                curImpurity = (i * lrImps[0] + (size - i) * lrImps[1]);
-                curThreshold = values[nextIdx];
-
-                if (values[nextIdx] != values[(nextIdx = s[i++])]) {
-                    if (curImpurity < minImpurity) {
-                        lSize = i - 1;
-
-                        lg = lrImps[0];
-                        rg = lrImps[1];
-
-                        lc2 = lrImps[2];
-                        rc2 = lrImps[3];
-
-                        System.arraycopy(lMapCur, 0, lMap, 0, lMapCur.length);
-                        System.arraycopy(rMapCur, 0, rMap, 0, rMapCur.length);
-
-                        minImpurity = curImpurity;
-                        threshold = curThreshold;
-                    }
-
-                    break;
-                }
-            }
-        }
-        while (i < s.length - 1);
-
-        if (lSize == size || lSize == 0)
-            return null;
-
-        GiniData lData = new GiniData(lg, lSize, lMap, lc2);
-        int rSize = size - lSize;
-        GiniData rData = new GiniData(rg, rSize, rMap, rc2);
-
-        return new ContinuousSplitInfo<>(regionIdx, threshold, lData, rData);
-    }
-
-    /**
-     * Add point to the left interval and remove it from the right interval and calculate necessary statistics on
-     * intervals with new bounds.
-     */
-    private void moveLeft(double x, int lSize, int rSize, int[] lMap, int[] rMap, double[] data) {
-        double lc2 = data[2];
-        double rc2 = data[3];
-
-        Integer idx = mapping.get(x);
-
-        int cxl = lMap[idx];
-        int cxr = rMap[idx];
-
-        lc2 += 2 * cxl + 1;
-        rc2 -= 2 * cxr - 1;
-
-        lMap[idx] += 1;
-        rMap[idx] -= 1;
-
-        data[0] = 1 - lc2 / (lSize * lSize);
-        data[1] = 1 - rc2 / (rSize * rSize);
-
-        data[2] = lc2;
-        data[3] = rc2;
-    }
-
-    /**
-     * Data used for gini impurity calculations.
-     */
-    public static class GiniData extends ContinuousRegionInfo {
-        /** Sum of squares of counts of each label. */
-        private double c2;
-
-        /** Counts of each label. On i-th position there is count of label which is mapped to index i. */
-        private int[] m;
-
-        /**
-         * Create Gini data.
-         *
-         * @param impurity Impurity (i.e. Gini impurity).
-         * @param size Count of samples.
-         * @param m Counts of each label.
-         * @param c2 Sum of squares of counts of each label.
-         */
-        public GiniData(double impurity, int size, int[] m, double c2) {
-            super(impurity, size);
-            this.m = m;
-            this.c2 = c2;
-        }
-
-        /**
-         * No-op constructor for serialization/deserialization..
-         */
-        public GiniData() {
-            // No-op.
-        }
-
-        /** Get counts of each label. */
-        public int[] counts() {
-            return m;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void writeExternal(ObjectOutput out) throws IOException {
-            super.writeExternal(out);
-            out.writeDouble(c2);
-            out.writeInt(m.length);
-            for (int i : m)
-                out.writeInt(i);
-
-        }
-
-        /** {@inheritDoc} */
-        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-            super.readExternal(in);
-
-            c2 = in.readDouble();
-            int size = in.readInt();
-            m = new int[size];
-
-            for (int i = 0; i < size; i++)
-                m[i] = in.readInt();
-        }
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java
deleted file mode 100644
index 66c54f299f805..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/VarianceSplitCalculator.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
-
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.PrimitiveIterator;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.ContinuousSplitInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-
-/**
- * Calculator of variance in a given region.
- */
-public class VarianceSplitCalculator implements ContinuousSplitCalculator<VarianceSplitCalculator.VarianceData> {
-    /**
-     * Data used in variance calculations.
-     */
-    public static class VarianceData extends ContinuousRegionInfo {
-        /** Mean value in a given region. */
-        double mean;
-
-        /**
-         * @param var Variance in this region.
-         * @param size Size of data for which variance is calculated.
-         * @param mean Mean value in this region.
-         */
-        public VarianceData(double var, int size, double mean) {
-            super(var, size);
-            this.mean = mean;
-        }
-
-        /**
-         * No-op constructor. For serialization/deserialization.
-         */
-        public VarianceData() {
-            // No-op.
-        }
-
-        /** {@inheritDoc} */
-        @Override public void writeExternal(ObjectOutput out) throws IOException {
-            super.writeExternal(out);
-            out.writeDouble(mean);
-        }
-
-        /** {@inheritDoc} */
-        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-            super.readExternal(in);
-            mean = in.readDouble();
-        }
-
-        /**
-         * Returns mean.
-         */
-        public double mean() {
-            return mean;
-        }
-    }
-
-    /** {@inheritDoc} */
-    @Override public VarianceData calculateRegionInfo(DoubleStream s, int size) {
-        PrimitiveIterator.OfDouble itr = s.iterator();
-        int i = 0;
-
-        double mean = 0.0;
-        double m2 = 0.0;
-
-        // Here we calculate variance and mean by incremental computation.
-        while (itr.hasNext()) {
-            i++;
-            double x = itr.next();
-            double delta = x - mean;
-            mean += delta / i;
-            double delta2 = x - mean;
-            m2 += delta * delta2;
-        }
-
-        return new VarianceData(m2 / i, size, mean);
-    }
-
-    /** {@inheritDoc} */
-    @Override public SplitInfo<VarianceData> splitRegion(Integer[] s, double[] values, double[] labels, int regionIdx,
-        VarianceData d) {
-        int size = d.getSize();
-
-        double lm2 = 0.0;
-        double rm2 = d.impurity() * size;
-        int lSize = size;
-
-        double lMean = 0.0;
-        double rMean = d.mean;
-
-        double minImpurity = d.impurity() * size;
-        double curThreshold;
-        double curImpurity;
-        double threshold = Double.NEGATIVE_INFINITY;
-
-        int i = 0;
-        int nextIdx = s[0];
-        i++;
-        double[] lrImps = new double[] {lm2, rm2, lMean, rMean};
-
-        do {
-            // Process all values equal to prev.
-            while (i < s.length) {
-                moveLeft(labels[nextIdx], lrImps[2], i, lrImps[0], lrImps[3], size - i, lrImps[1], lrImps);
-                curImpurity = (lrImps[0] + lrImps[1]);
-                curThreshold = values[nextIdx];
-
-                if (values[nextIdx] != values[(nextIdx = s[i++])]) {
-                    if (curImpurity < minImpurity) {
-                        lSize = i - 1;
-
-                        lm2 = lrImps[0];
-                        rm2 = lrImps[1];
-
-                        lMean = lrImps[2];
-                        rMean = lrImps[3];
-
-                        minImpurity = curImpurity;
-                        threshold = curThreshold;
-                    }
-
-                    break;
-                }
-            }
-        }
-        while (i < s.length - 1);
-
-        if (lSize == size)
-            return null;
-
-        VarianceData lData = new VarianceData(lm2 / (lSize != 0 ? lSize : 1), lSize, lMean);
-        int rSize = size - lSize;
-        VarianceData rData = new VarianceData(rm2 / (rSize != 0 ? rSize : 1), rSize, rMean);
-
-        return new ContinuousSplitInfo<>(regionIdx, threshold, lData, rData);
-    }
-
-    /**
-     * Add point to the left interval and remove it from the right interval and calculate necessary statistics on
-     * intervals with new bounds.
-     */
-    private void moveLeft(double x, double lMean, int lSize, double lm2, double rMean, int rSize, double rm2,
-        double[] data) {
-        // We add point to the left interval.
-        double lDelta = x - lMean;
-        double lMeanNew = lMean + lDelta / lSize;
-        double lm2New = lm2 + lDelta * (x - lMeanNew);
-
-        // We remove point from the right interval. lSize + 1 is the size of right interval before removal.
-        double rMeanNew = (rMean * (rSize + 1) - x) / rSize;
-        double rm2New = rm2 - (x - rMean) * (x - rMeanNew);
-
-        data[0] = lm2New;
-        data[1] = rm2New;
-
-        data[2] = lMeanNew;
-        data[3] = rMeanNew;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java
deleted file mode 100644
index 08c8a75bd7d8e..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/contsplitcalcs/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Calculators of splits by continuous features.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java
deleted file mode 100644
index 85239141eb0b4..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains column based decision tree algorithms.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java
deleted file mode 100644
index 5c4b354854eeb..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/regcalcs/RegionCalculators.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.regcalcs;
-
-import it.unimi.dsi.fastutil.doubles.Double2IntOpenHashMap;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.PrimitiveIterator;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-
-/** Some commonly used functions for calculations of regions of space which correspond to decision tree leaf nodes. */
-public class RegionCalculators {
-    /** Mean value in the region. */
-    public static final IgniteFunction<DoubleStream, Double> MEAN = s -> s.average().orElse(0.0);
-
-    /** Most common value in the region. */
-    public static final IgniteFunction<DoubleStream, Double> MOST_COMMON =
-        s -> {
-            PrimitiveIterator.OfDouble itr = s.iterator();
-            Map<Double, Integer> voc = new HashMap<>();
-
-            while (itr.hasNext())
-                voc.compute(itr.next(), (d, i) -> i != null ? i + 1 : 0);
-
-            return voc.entrySet().stream().max(Comparator.comparing(Map.Entry::getValue)).map(Map.Entry::getKey).orElse(0.0);
-        };
-
-    /** Variance of a region. */
-    public static final IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> VARIANCE = input ->
-        s -> {
-            PrimitiveIterator.OfDouble itr = s.iterator();
-            int i = 0;
-
-            double mean = 0.0;
-            double m2 = 0.0;
-
-            while (itr.hasNext()) {
-                i++;
-                double x = itr.next();
-                double delta = x - mean;
-                mean += delta / i;
-                double delta2 = x - mean;
-                m2 += delta * delta2;
-            }
-
-            return i > 0 ? m2 / i : 0.0;
-        };
-
-    /** Gini impurity of a region. */
-    public static final IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> GINI = input ->
-        s -> {
-            PrimitiveIterator.OfDouble itr = s.iterator();
-
-            Double2IntOpenHashMap m = new Double2IntOpenHashMap();
-
-            int size = 0;
-
-            while (itr.hasNext()) {
-                size++;
-                m.compute(itr.next(), (a, i) -> i != null ? i + 1 : 1);
-            }
-
-            double c2 = m.values().stream().mapToDouble(v -> v * v).sum();
-
-            return size != 0 ? 1 - c2 / (size * size) : 0.0;
-        };
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java
deleted file mode 100644
index 3232ac209c7c5..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/CategoricalFeatureProcessor.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.Stream;
-import java.util.stream.StreamSupport;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.trees.CategoricalRegionInfo;
-import org.apache.ignite.ml.trees.CategoricalSplitInfo;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureVectorProcessorUtils.splitByBitSet;
-
-/**
- * Categorical feature vector processor implementation used by {@link ColumnDecisionTreeTrainer}.
- */
-public class CategoricalFeatureProcessor
-    implements FeatureProcessor<CategoricalRegionInfo, CategoricalSplitInfo<CategoricalRegionInfo>> {
-    /** Count of categories for this feature. */
-    private final int catsCnt;
-
-    /** Function for calculating impurity of a given region of points. */
-    private final IgniteFunction<DoubleStream, Double> calc;
-
-    /**
-     * @param calc Function for calculating impurity of a given region of points.
-     * @param catsCnt Number of categories.
-     */
-    public CategoricalFeatureProcessor(IgniteFunction<DoubleStream, Double> calc, int catsCnt) {
-        this.calc = calc;
-        this.catsCnt = catsCnt;
-    }
-
-    /** */
-    private SplitInfo<CategoricalRegionInfo> split(BitSet leftCats, int intervalIdx, Map<Integer, Integer> mapping,
-        Integer[] sampleIndexes, double[] values, double[] labels, double impurity) {
-        Map<Boolean, List<Integer>> leftRight = Arrays.stream(sampleIndexes).
-            collect(Collectors.partitioningBy((smpl) -> leftCats.get(mapping.get((int)values[smpl]))));
-
-        List<Integer> left = leftRight.get(true);
-        int leftSize = left.size();
-        double leftImpurity = calc.apply(left.stream().mapToDouble(s -> labels[s]));
-
-        List<Integer> right = leftRight.get(false);
-        int rightSize = right.size();
-        double rightImpurity = calc.apply(right.stream().mapToDouble(s -> labels[s]));
-
-        int totalSize = leftSize + rightSize;
-
-        // Result of this call will be sent back to trainer node, we do not need vectors inside of sent data.
-        CategoricalSplitInfo<CategoricalRegionInfo> res = new CategoricalSplitInfo<>(intervalIdx,
-            new CategoricalRegionInfo(leftImpurity, null), // cats can be computed on the last step.
-            new CategoricalRegionInfo(rightImpurity, null),
-            leftCats);
-
-        res.setInfoGain(impurity - (double)leftSize / totalSize * leftImpurity - (double)rightSize / totalSize * rightImpurity);
-        return res;
-    }
-
-    /**
-     * Get a stream of subsets given categories count.
-     *
-     * @param catsCnt categories count.
-     * @return Stream of subsets given categories count.
-     */
-    private Stream<BitSet> powerSet(int catsCnt) {
-        Iterable<BitSet> iterable = () -> new PSI(catsCnt);
-        return StreamSupport.stream(iterable.spliterator(), false);
-    }
-
-    /** {@inheritDoc} */
-    @Override public SplitInfo findBestSplit(RegionProjection<CategoricalRegionInfo> regionPrj, double[] values,
-        double[] labels, int regIdx) {
-        Map<Integer, Integer> mapping = mapping(regionPrj.data().cats());
-
-        return powerSet(regionPrj.data().cats().length()).
-            map(s -> split(s, regIdx, mapping, regionPrj.sampleIndexes(), values, labels, regionPrj.data().impurity())).
-            max(Comparator.comparingDouble(SplitInfo::infoGain)).
-            orElse(null);
-    }
-
-    /** {@inheritDoc} */
-    @Override public RegionProjection<CategoricalRegionInfo> createInitialRegion(Integer[] sampleIndexes,
-        double[] values, double[] labels) {
-        BitSet set = new BitSet();
-        set.set(0, catsCnt);
-
-        Double impurity = calc.apply(Arrays.stream(labels));
-
-        return new RegionProjection<>(sampleIndexes, new CategoricalRegionInfo(impurity, set), 0);
-    }
-
-    /** {@inheritDoc} */
-    @Override public SparseBitSet calculateOwnershipBitSet(RegionProjection<CategoricalRegionInfo> regionPrj,
-        double[] values,
-        CategoricalSplitInfo<CategoricalRegionInfo> s) {
-        SparseBitSet res = new SparseBitSet();
-        Arrays.stream(regionPrj.sampleIndexes()).forEach(smpl -> res.set(smpl, s.bitSet().get((int)values[smpl])));
-        return res;
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplit(SparseBitSet bs,
-        RegionProjection<CategoricalRegionInfo> reg, CategoricalRegionInfo leftData, CategoricalRegionInfo rightData) {
-        return performSplitGeneric(bs, null, reg, leftData, rightData);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplitGeneric(
-        SparseBitSet bs, double[] values, RegionProjection<CategoricalRegionInfo> reg, RegionInfo leftData,
-        RegionInfo rightData) {
-        int depth = reg.depth();
-
-        int lSize = bs.cardinality();
-        int rSize = reg.sampleIndexes().length - lSize;
-        IgniteBiTuple<Integer[], Integer[]> lrSamples = splitByBitSet(lSize, rSize, reg.sampleIndexes(), bs);
-        BitSet leftCats = calculateCats(lrSamples.get1(), values);
-        CategoricalRegionInfo lInfo = new CategoricalRegionInfo(leftData.impurity(), leftCats);
-
-        // TODO: IGNITE-5892 Check how it will work with sparse data.
-        BitSet rightCats = calculateCats(lrSamples.get2(), values);
-        CategoricalRegionInfo rInfo = new CategoricalRegionInfo(rightData.impurity(), rightCats);
-
-        RegionProjection<CategoricalRegionInfo> rPrj = new RegionProjection<>(lrSamples.get2(), rInfo, depth + 1);
-        RegionProjection<CategoricalRegionInfo> lPrj = new RegionProjection<>(lrSamples.get1(), lInfo, depth + 1);
-        return new IgniteBiTuple<>(lPrj, rPrj);
-    }
-
-    /**
-     * Powerset iterator. Iterates not over the whole powerset, but on half of it.
-     */
-    private static class PSI implements Iterator<BitSet> {
-
-        /** Current subset number. */
-        private int i = 1; // We are not interested in {emptyset, set} split and therefore start from 1.
-
-        /** Size of set, subsets of which we iterate over. */
-        final int size;
-
-        /**
-         * @param bitCnt Size of set, subsets of which we iterate over.
-         */
-        PSI(int bitCnt) {
-            this.size = 1 << (bitCnt - 1);
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean hasNext() {
-            return i < size;
-        }
-
-        /** {@inheritDoc} */
-        @Override public BitSet next() {
-            BitSet res = BitSet.valueOf(new long[] {i});
-            i++;
-            return res;
-        }
-    }
-
-    /** */
-    private Map<Integer, Integer> mapping(BitSet bs) {
-        int bn = 0;
-        Map<Integer, Integer> res = new HashMap<>();
-
-        int i = 0;
-        while ((bn = bs.nextSetBit(bn)) != -1) {
-            res.put(bn, i);
-            i++;
-            bn++;
-        }
-
-        return res;
-    }
-
-    /** Get set of categories of given samples */
-    private BitSet calculateCats(Integer[] sampleIndexes, double[] values) {
-        BitSet res = new BitSet();
-
-        for (int smpl : sampleIndexes)
-            res.set((int)values[smpl]);
-
-        return res;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java
deleted file mode 100644
index 4117993aa68a2..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousFeatureProcessor.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import java.util.Arrays;
-import java.util.Comparator;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-import static org.apache.ignite.ml.trees.trainers.columnbased.vectors.FeatureVectorProcessorUtils.splitByBitSet;
-
-/**
- * Container of projection of samples on continuous feature.
- *
- * @param <D> Information about regions. Designed to contain information which will make computations of impurity
- * optimal.
- */
-public class ContinuousFeatureProcessor<D extends ContinuousRegionInfo> implements
-    FeatureProcessor<D, ContinuousSplitInfo<D>> {
-    /** ContinuousSplitCalculator used for calculating of best split of each region. */
-    private final ContinuousSplitCalculator<D> calc;
-
-    /**
-     * @param splitCalc Calculator used for calculating splits.
-     */
-    public ContinuousFeatureProcessor(ContinuousSplitCalculator<D> splitCalc) {
-        this.calc = splitCalc;
-    }
-
-    /** {@inheritDoc} */
-    @Override public SplitInfo<D> findBestSplit(RegionProjection<D> ri, double[] values, double[] labels, int regIdx) {
-        SplitInfo<D> res = calc.splitRegion(ri.sampleIndexes(), values, labels, regIdx, ri.data());
-
-        if (res == null)
-            return null;
-
-        double lWeight = (double)res.leftData.getSize() / ri.sampleIndexes().length;
-        double rWeight = (double)res.rightData.getSize() / ri.sampleIndexes().length;
-
-        double infoGain = ri.data().impurity() - lWeight * res.leftData().impurity() - rWeight * res.rightData().impurity();
-        res.setInfoGain(infoGain);
-
-        return res;
-    }
-
-    /** {@inheritDoc} */
-    @Override public RegionProjection<D> createInitialRegion(Integer[] samples, double[] values, double[] labels) {
-        Arrays.sort(samples, Comparator.comparingDouble(s -> values[s]));
-        return new RegionProjection<>(samples, calc.calculateRegionInfo(Arrays.stream(labels), samples.length), 0);
-    }
-
-    /** {@inheritDoc} */
-    @Override public SparseBitSet calculateOwnershipBitSet(RegionProjection<D> reg, double[] values,
-        ContinuousSplitInfo<D> s) {
-        SparseBitSet res = new SparseBitSet();
-
-        for (int i = 0; i < s.leftData().getSize(); i++)
-            res.set(reg.sampleIndexes()[i]);
-
-        return res;
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplit(SparseBitSet bs,
-        RegionProjection<D> reg, D leftData, D rightData) {
-        int lSize = leftData.getSize();
-        int rSize = rightData.getSize();
-        int depth = reg.depth();
-
-        IgniteBiTuple<Integer[], Integer[]> lrSamples = splitByBitSet(lSize, rSize, reg.sampleIndexes(), bs);
-
-        RegionProjection<D> left = new RegionProjection<>(lrSamples.get1(), leftData, depth + 1);
-        RegionProjection<D> right = new RegionProjection<>(lrSamples.get2(), rightData, depth + 1);
-
-        return new IgniteBiTuple<>(left, right);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteBiTuple<RegionProjection, RegionProjection> performSplitGeneric(SparseBitSet bs,
-        double[] labels, RegionProjection<D> reg, RegionInfo leftData, RegionInfo rightData) {
-        int lSize = bs.cardinality();
-        int rSize = reg.sampleIndexes().length - lSize;
-        int depth = reg.depth();
-
-        IgniteBiTuple<Integer[], Integer[]> lrSamples = splitByBitSet(lSize, rSize, reg.sampleIndexes(), bs);
-
-        D ld = calc.calculateRegionInfo(Arrays.stream(lrSamples.get1()).mapToDouble(s -> labels[s]), lSize);
-        D rd = calc.calculateRegionInfo(Arrays.stream(lrSamples.get2()).mapToDouble(s -> labels[s]), rSize);
-
-        return new IgniteBiTuple<>(new RegionProjection<>(lrSamples.get1(), ld, depth + 1), new RegionProjection<>(lrSamples.get2(), rd, depth + 1));
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java
deleted file mode 100644
index 8b45cb580ce36..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/ContinuousSplitInfo.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.nodes.ContinuousSplitNode;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-
-/**
- * Information about split of continuous region.
- *
- * @param <D> Class encapsulating information about the region.
- */
-public class ContinuousSplitInfo<D extends RegionInfo> extends SplitInfo<D> {
-    /**
-     * Threshold used for split.
-     * Samples with values less or equal than this go to left region, others go to the right region.
-     */
-    private final double threshold;
-
-    /**
-     * @param regionIdx Index of region being split.
-     * @param threshold Threshold used for split. Samples with values less or equal than this go to left region, others
-     * go to the right region.
-     * @param leftData Information about left subregion.
-     * @param rightData Information about right subregion.
-     */
-    public ContinuousSplitInfo(int regionIdx, double threshold, D leftData, D rightData) {
-        super(regionIdx, leftData, rightData);
-        this.threshold = threshold;
-    }
-
-    /** {@inheritDoc} */
-    @Override public SplitNode createSplitNode(int featureIdx) {
-        return new ContinuousSplitNode(threshold, featureIdx);
-    }
-
-    /**
-     * Threshold used for splits.
-     * Samples with values less or equal than this go to left region, others go to the right region.
-     */
-    public double threshold() {
-        return threshold;
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return "ContinuousSplitInfo [" +
-            "threshold=" + threshold +
-            ", infoGain=" + infoGain +
-            ", regionIdx=" + regionIdx +
-            ", leftData=" + leftData +
-            ", rightData=" + rightData +
-            ']';
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java
deleted file mode 100644
index 56508e5e37f19..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureProcessor.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.RegionProjection;
-
-/**
- * Base interface for feature processors used in {@link ColumnDecisionTreeTrainer}
- *
- * @param <D> Class representing data of regions resulted from split.
- * @param <S> Class representing data of split.
- */
-public interface FeatureProcessor<D extends RegionInfo, S extends SplitInfo<D>> {
-    /**
-     * Finds best split by this feature among all splits of all regions.
-     *
-     * @return best split by this feature among all splits of all regions.
-     */
-    SplitInfo findBestSplit(RegionProjection<D> regionPrj, double[] values, double[] labels, int regIdx);
-
-    /**
-     * Creates initial region from samples.
-     *
-     * @param samples samples.
-     * @return region.
-     */
-    RegionProjection<D> createInitialRegion(Integer[] samples, double[] values, double[] labels);
-
-    /**
-     * Calculates the bitset mapping each data point to left (corresponding bit is set) or right subregion.
-     *
-     * @param s data used for calculating the split.
-     * @return Bitset mapping each data point to left (corresponding bit is set) or right subregion.
-     */
-    SparseBitSet calculateOwnershipBitSet(RegionProjection<D> regionPrj, double[] values, S s);
-
-    /**
-     * Splits given region using bitset which maps data point to left or right subregion.
-     * This method is present for the vectors of the same type to be able to pass between them information about regions
-     * and therefore used iff the optimal split is received on feature of the same type.
-     *
-     * @param bs Bitset which maps data point to left or right subregion.
-     * @param leftData Data of the left subregion.
-     * @param rightData Data of the right subregion.
-     * @return This feature vector.
-     */
-    IgniteBiTuple<RegionProjection, RegionProjection> performSplit(SparseBitSet bs, RegionProjection<D> reg, D leftData,
-        D rightData);
-
-    /**
-     * Splits given region using bitset which maps data point to left or right subregion. This method is used iff the
-     * optimal split is received on feature of different type, therefore information about regions is limited to the
-     * {@link RegionInfo} class which is base for all classes used to represent region data.
-     *
-     * @param bs Bitset which maps data point to left or right subregion.
-     * @param leftData Data of the left subregion.
-     * @param rightData Data of the right subregion.
-     * @return This feature vector.
-     */
-    IgniteBiTuple<RegionProjection, RegionProjection> performSplitGeneric(SparseBitSet bs, double[] values,
-        RegionProjection<D> reg, RegionInfo leftData,
-        RegionInfo rightData);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java
deleted file mode 100644
index 69ff019917501..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/FeatureVectorProcessorUtils.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import com.zaxxer.sparsebits.SparseBitSet;
-import org.apache.ignite.lang.IgniteBiTuple;
-
-/** Utility class for feature vector processors. */
-public class FeatureVectorProcessorUtils {
-    /**
-     * Split target array into two (left and right) arrays by bitset.
-     *
-     * @param lSize Left array size;
-     * @param rSize Right array size.
-     * @param samples Arrays to split size.
-     * @param bs Bitset specifying split.
-     * @return BiTuple containing result of split.
-     */
-    public static IgniteBiTuple<Integer[], Integer[]> splitByBitSet(int lSize, int rSize, Integer[] samples,
-        SparseBitSet bs) {
-        Integer[] lArr = new Integer[lSize];
-        Integer[] rArr = new Integer[rSize];
-
-        int lc = 0;
-        int rc = 0;
-
-        for (int i = 0; i < lSize + rSize; i++) {
-            int si = samples[i];
-
-            if (bs.get(si)) {
-                lArr[lc] = si;
-                lc++;
-            }
-            else {
-                rArr[rc] = si;
-                rc++;
-            }
-        }
-
-        return new IgniteBiTuple<>(lArr, rArr);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java
deleted file mode 100644
index 8aa4f79c0b18c..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SampleInfo.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-
-/**
- * Information about given sample within given fixed feature.
- */
-public class SampleInfo implements Externalizable {
-    /** Value of projection of this sample on given fixed feature. */
-    private double val;
-
-    /** Sample index. */
-    private int sampleIdx;
-
-    /**
-     * @param val Value of projection of this sample on given fixed feature.
-     * @param sampleIdx Sample index.
-     */
-    public SampleInfo(double val, int sampleIdx) {
-        this.val = val;
-        this.sampleIdx = sampleIdx;
-    }
-
-    /**
-     * No-op constructor used for serialization/deserialization.
-     */
-    public SampleInfo() {
-        // No-op.
-    }
-
-    /**
-     * Get the value of projection of this sample on given fixed feature.
-     *
-     * @return Value of projection of this sample on given fixed feature.
-     */
-    public double val() {
-        return val;
-    }
-
-    /**
-     * Get the sample index.
-     *
-     * @return Sample index.
-     */
-    public int sampleInd() {
-        return sampleIdx;
-    }
-
-    /** {@inheritDoc} */
-    @Override public void writeExternal(ObjectOutput out) throws IOException {
-        out.writeDouble(val);
-        out.writeInt(sampleIdx);
-    }
-
-    /** {@inheritDoc} */
-    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-        val = in.readDouble();
-        sampleIdx = in.readInt();
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java
deleted file mode 100644
index 124e82f938316..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/SplitInfo.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
-
-import org.apache.ignite.ml.trees.RegionInfo;
-import org.apache.ignite.ml.trees.nodes.SplitNode;
-
-/**
- * Class encapsulating information about the split.
- *
- * @param <D> Class representing information of left and right subregions.
- */
-public abstract class SplitInfo<D extends RegionInfo> {
-    /** Information gain of this split. */
-    protected double infoGain;
-
-    /** Index of the region to split. */
-    protected final int regionIdx;
-
-    /** Data of left subregion. */
-    protected final D leftData;
-
-    /** Data of right subregion. */
-    protected final D rightData;
-
-    /**
-     * Construct the split info.
-     *
-     * @param regionIdx Index of the region to split.
-     * @param leftData Data of left subregion.
-     * @param rightData Data of right subregion.
-     */
-    public SplitInfo(int regionIdx, D leftData, D rightData) {
-        this.regionIdx = regionIdx;
-        this.leftData = leftData;
-        this.rightData = rightData;
-    }
-
-    /**
-     * Index of region to split.
-     *
-     * @return Index of region to split.
-     */
-    public int regionIndex() {
-        return regionIdx;
-    }
-
-    /**
-     * Information gain of the split.
-     *
-     * @return Information gain of the split.
-     */
-    public double infoGain() {
-        return infoGain;
-    }
-
-    /**
-     * Data of right subregion.
-     *
-     * @return Data of right subregion.
-     */
-    public D rightData() {
-        return rightData;
-    }
-
-    /**
-     * Data of left subregion.
-     *
-     * @return Data of left subregion.
-     */
-    public D leftData() {
-        return leftData;
-    }
-
-    /**
-     * Create SplitNode from this split info.
-     *
-     * @param featureIdx Index of feature by which goes split.
-     * @return SplitNode from this split info.
-     */
-    public abstract SplitNode createSplitNode(int featureIdx);
-
-    /**
-     * Set information gain.
-     *
-     * @param infoGain Information gain.
-     */
-    public void setInfoGain(double infoGain) {
-        this.infoGain = infoGain;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java
deleted file mode 100644
index 0dea204b530aa..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trees/trainers/columnbased/vectors/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains feature containers needed by column based decision tree trainers.
- */
-package org.apache.ignite.ml.trees.trainers.columnbased.vectors;
\ No newline at end of file
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
index e22a3a5fb4c38..9900f854be9f4 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
@@ -28,7 +28,7 @@
 import org.apache.ignite.ml.regressions.RegressionsTestSuite;
 import org.apache.ignite.ml.svm.SVMTestSuite;
 import org.apache.ignite.ml.trainers.group.TrainersGroupTestSuite;
-import org.apache.ignite.ml.trees.DecisionTreesTestSuite;
+import org.apache.ignite.ml.tree.DecisionTreeTestSuite;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 
@@ -41,7 +41,7 @@
     RegressionsTestSuite.class,
     SVMTestSuite.class,
     ClusteringTestSuite.class,
-    DecisionTreesTestSuite.class,
+    DecisionTreeTestSuite.class,
     KNNTestSuite.class,
     LocalModelsTest.class,
     MLPTestSuite.class,
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
index e624004081a0c..d68b3553c2b2b 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MnistMLPTestUtil.java
@@ -25,11 +25,10 @@
 import java.util.stream.Stream;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.performance.ColumnDecisionTreeTrainerBenchmark;
 import org.apache.ignite.ml.util.MnistUtils;
 
 /** */
-class MnistMLPTestUtil {
+public class MnistMLPTestUtil {
     /** Name of the property specifying path to training set images. */
     private static final String PROP_TRAINING_IMAGES = "mnist.training.images";
 
@@ -62,7 +61,7 @@ static IgniteBiTuple<Stream<DenseLocalOnHeapVector>, Stream<DenseLocalOnHeapVect
      * @return List of MNIST images.
      * @throws IOException In case of exception.
      */
-    static List<MnistUtils.MnistLabeledImage> loadTrainingSet(int cnt) throws IOException {
+    public static List<MnistUtils.MnistLabeledImage> loadTrainingSet(int cnt) throws IOException {
         Properties props = loadMNISTProperties();
         return MnistUtils.mnistAsList(props.getProperty(PROP_TRAINING_IMAGES), props.getProperty(PROP_TRAINING_LABELS), new Random(123L), cnt);
     }
@@ -74,7 +73,7 @@ static List<MnistUtils.MnistLabeledImage> loadTrainingSet(int cnt) throws IOExce
      * @return List of MNIST images.
      * @throws IOException In case of exception.
      */
-    static List<MnistUtils.MnistLabeledImage> loadTestSet(int cnt) throws IOException {
+    public static List<MnistUtils.MnistLabeledImage> loadTestSet(int cnt) throws IOException {
         Properties props = loadMNISTProperties();
         return MnistUtils.mnistAsList(props.getProperty(PROP_TEST_IMAGES), props.getProperty(PROP_TEST_LABELS), new Random(123L), cnt);
     }
@@ -83,7 +82,7 @@ static List<MnistUtils.MnistLabeledImage> loadTestSet(int cnt) throws IOExceptio
     private static Properties loadMNISTProperties() throws IOException {
         Properties res = new Properties();
 
-        InputStream is = ColumnDecisionTreeTrainerBenchmark.class.getClassLoader().getResourceAsStream("manualrun/trees/columntrees.manualrun.properties");
+        InputStream is = MnistMLPTestUtil.class.getClassLoader().getResourceAsStream("manualrun/trees/columntrees.manualrun.properties");
 
         res.load(is);
 
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
new file mode 100644
index 0000000000000..94bca3f83e2f6
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.Arrays;
+import java.util.Random;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for {@link DecisionTreeClassificationTrainer} that require to start the whole Ignite infrastructure.
+ */
+public class DecisionTreeClassificationTrainerIntegrationTest extends GridCommonAbstractTest {
+    /** Number of nodes in grid */
+    private static final int NODE_COUNT = 3;
+
+    /** Ignite instance. */
+    private Ignite ignite;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        for (int i = 1; i <= NODE_COUNT; i++)
+            startGrid(i);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() {
+        stopAllGrids();
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected void beforeTest() throws Exception {
+        /* Grid instance. */
+        ignite = grid(NODE_COUNT);
+        ignite.configuration().setPeerClassLoadingEnabled(true);
+        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+    }
+
+    /** */
+    public void testFit() {
+        int size = 100;
+
+        CacheConfiguration<Integer, double[]> trainingSetCacheCfg = new CacheConfiguration<>();
+        trainingSetCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+        trainingSetCacheCfg.setName("TRAINING_SET");
+
+        IgniteCache<Integer, double[]> data = ignite.createCache(trainingSetCacheCfg);
+
+        Random rnd = new Random(0);
+        for (int i = 0; i < size; i++) {
+            double x = rnd.nextDouble() - 0.5;
+            data.put(i, new double[]{x, x > 0 ? 1 : 0});
+        }
+
+        DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(1, 0);
+
+        DecisionTreeNode tree = trainer.fit(
+            new CacheBasedDatasetBuilder<>(ignite, data),
+            (k, v) -> Arrays.copyOf(v, v.length - 1),
+            (k, v) -> v[v.length - 1]
+        );
+
+        assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+        DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+        assertEquals(0, node.getThreshold(), 1e-3);
+
+        assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+        assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+        DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+        DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+        assertEquals(1, thenNode.getVal(), 1e-10);
+        assertEquals(0, elseNode.getVal(), 1e-10);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
new file mode 100644
index 0000000000000..2599bfe2b17e1
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Tests for {@link DecisionTreeClassificationTrainer}.
+ */
+@RunWith(Parameterized.class)
+public class DecisionTreeClassificationTrainerTest {
+    /** Number of parts to be tested. */
+    private static final int[] partsToBeTested = new int[] {1, 2, 3, 4, 5, 7};
+
+    /** Number of partitions. */
+    @Parameterized.Parameter
+    public int parts;
+
+    @Parameterized.Parameters(name = "Data divided on {0} partitions")
+    public static Iterable<Integer[]> data() {
+        List<Integer[]> res = new ArrayList<>();
+        for (int part : partsToBeTested)
+            res.add(new Integer[] {part});
+
+        return res;
+    }
+
+    /** */
+    @Test
+    public void testFit() {
+        int size = 100;
+
+        Map<Integer, double[]> data = new HashMap<>();
+
+        Random rnd = new Random(0);
+        for (int i = 0; i < size; i++) {
+            double x = rnd.nextDouble() - 0.5;
+            data.put(i, new double[]{x, x > 0 ? 1 : 0});
+        }
+
+        DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(1, 0);
+
+        DecisionTreeNode tree = trainer.fit(
+            new LocalDatasetBuilder<>(data, parts),
+            (k, v) -> Arrays.copyOf(v, v.length - 1),
+            (k, v) -> v[v.length - 1]
+        );
+
+        assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+        DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+        assertEquals(0, node.getThreshold(), 1e-3);
+
+        assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+        assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+        DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+        DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+        assertEquals(1, thenNode.getVal(), 1e-10);
+        assertEquals(0, elseNode.getVal(), 1e-10);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
new file mode 100644
index 0000000000000..754ff20f8046e
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.Arrays;
+import java.util.Random;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for {@link DecisionTreeRegressionTrainer} that require to start the whole Ignite infrastructure.
+ */
+public class DecisionTreeRegressionTrainerIntegrationTest extends GridCommonAbstractTest {
+    /** Number of nodes in grid */
+    private static final int NODE_COUNT = 3;
+
+    /** Ignite instance. */
+    private Ignite ignite;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        for (int i = 1; i <= NODE_COUNT; i++)
+            startGrid(i);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() {
+        stopAllGrids();
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected void beforeTest() throws Exception {
+        /* Grid instance. */
+        ignite = grid(NODE_COUNT);
+        ignite.configuration().setPeerClassLoadingEnabled(true);
+        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+    }
+
+    /** */
+    public void testFit() {
+        int size = 100;
+
+        CacheConfiguration<Integer, double[]> trainingSetCacheCfg = new CacheConfiguration<>();
+        trainingSetCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+        trainingSetCacheCfg.setName("TRAINING_SET");
+
+        IgniteCache<Integer, double[]> data = ignite.createCache(trainingSetCacheCfg);
+
+        Random rnd = new Random(0);
+        for (int i = 0; i < size; i++) {
+            double x = rnd.nextDouble() - 0.5;
+            data.put(i, new double[]{x, x > 0 ? 1 : 0});
+        }
+
+        DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0);
+
+        DecisionTreeNode tree = trainer.fit(
+            new CacheBasedDatasetBuilder<>(ignite, data),
+            (k, v) -> Arrays.copyOf(v, v.length - 1),
+            (k, v) -> v[v.length - 1]
+        );
+
+        assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+        DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+        assertEquals(0, node.getThreshold(), 1e-3);
+
+        assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+        assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+        DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+        DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+        assertEquals(1, thenNode.getVal(), 1e-10);
+        assertEquals(0, elseNode.getVal(), 1e-10);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
new file mode 100644
index 0000000000000..3bdbf60675a3f
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Tests for {@link DecisionTreeRegressionTrainer}.
+ */
+@RunWith(Parameterized.class)
+public class DecisionTreeRegressionTrainerTest {
+    /** Number of parts to be tested. */
+    private static final int[] partsToBeTested = new int[] {1, 2, 3, 4, 5, 7};
+
+    /** Number of partitions. */
+    @Parameterized.Parameter
+    public int parts;
+
+    @Parameterized.Parameters(name = "Data divided on {0} partitions")
+    public static Iterable<Integer[]> data() {
+        List<Integer[]> res = new ArrayList<>();
+        for (int part : partsToBeTested)
+            res.add(new Integer[] {part});
+
+        return res;
+    }
+
+    /** */
+    @Test
+    public void testFit() {
+        int size = 100;
+
+        Map<Integer, double[]> data = new HashMap<>();
+
+        Random rnd = new Random(0);
+        for (int i = 0; i < size; i++) {
+            double x = rnd.nextDouble() - 0.5;
+            data.put(i, new double[]{x, x > 0 ? 1 : 0});
+        }
+
+        DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0);
+
+        DecisionTreeNode tree = trainer.fit(
+            new LocalDatasetBuilder<>(data, parts),
+            (k, v) -> Arrays.copyOf(v, v.length - 1),
+            (k, v) -> v[v.length - 1]
+        );
+
+        assertTrue(tree instanceof DecisionTreeConditionalNode);
+
+        DecisionTreeConditionalNode node = (DecisionTreeConditionalNode) tree;
+
+        assertEquals(0, node.getThreshold(), 1e-3);
+
+        assertTrue(node.getThenNode() instanceof DecisionTreeLeafNode);
+        assertTrue(node.getElseNode() instanceof DecisionTreeLeafNode);
+
+        DecisionTreeLeafNode thenNode = (DecisionTreeLeafNode) node.getThenNode();
+        DecisionTreeLeafNode elseNode = (DecisionTreeLeafNode) node.getElseNode();
+
+        assertEquals(1, thenNode.getVal(), 1e-10);
+        assertEquals(0, elseNode.getVal(), 1e-10);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java
new file mode 100644
index 0000000000000..2cbb486c3df5f
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeTestSuite.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree;
+
+import org.apache.ignite.ml.tree.data.DecisionTreeDataTest;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasureCalculatorTest;
+import org.apache.ignite.ml.tree.impurity.gini.GiniImpurityMeasureTest;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasureCalculatorTest;
+import org.apache.ignite.ml.tree.impurity.mse.MSEImpurityMeasureTest;
+import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressorTest;
+import org.apache.ignite.ml.tree.impurity.util.StepFunctionTest;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Test suite for all tests located in {@link org.apache.ignite.ml.tree} package.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+    DecisionTreeClassificationTrainerTest.class,
+    DecisionTreeRegressionTrainerTest.class,
+    DecisionTreeClassificationTrainerIntegrationTest.class,
+    DecisionTreeRegressionTrainerIntegrationTest.class,
+    DecisionTreeDataTest.class,
+    GiniImpurityMeasureCalculatorTest.class,
+    GiniImpurityMeasureTest.class,
+    MSEImpurityMeasureCalculatorTest.class,
+    MSEImpurityMeasureTest.class,
+    StepFunctionTest.class,
+    SimpleStepFunctionCompressorTest.class
+})
+public class DecisionTreeTestSuite {
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java
new file mode 100644
index 0000000000000..0c89d4e11d05d
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/data/DecisionTreeDataTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.data;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link DecisionTreeData}.
+ */
+public class DecisionTreeDataTest {
+    /** */
+    @Test
+    public void testFilter() {
+        double[][] features = new double[][]{{0}, {1}, {2}, {3}, {4}, {5}};
+        double[] labels = new double[]{0, 1, 2, 3, 4, 5};
+
+        DecisionTreeData data = new DecisionTreeData(features, labels);
+        DecisionTreeData filteredData = data.filter(obj -> obj[0] > 2);
+
+        assertArrayEquals(new double[][]{{3}, {4}, {5}}, filteredData.getFeatures());
+        assertArrayEquals(new double[]{3, 4, 5}, filteredData.getLabels(), 1e-10);
+    }
+
+    /** */
+    @Test
+    public void testSort() {
+        double[][] features = new double[][]{{4, 1}, {3, 3}, {2, 0}, {1, 4}, {0, 2}};
+        double[] labels = new double[]{0, 1, 2, 3, 4};
+
+        DecisionTreeData data = new DecisionTreeData(features, labels);
+
+        data.sort(0);
+
+        assertArrayEquals(new double[][]{{0, 2}, {1, 4}, {2, 0}, {3, 3}, {4, 1}}, features);
+        assertArrayEquals(new double[]{4, 3, 2, 1, 0}, labels, 1e-10);
+
+        data.sort(1);
+
+        assertArrayEquals(new double[][]{{2, 0}, {4, 1}, {0, 2}, {3, 3}, {1, 4}}, features);
+        assertArrayEquals(new double[]{2, 0, 4, 1, 3}, labels, 1e-10);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java
new file mode 100644
index 0000000000000..afd81e882491d
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureCalculatorTest.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link GiniImpurityMeasureCalculator}.
+ */
+public class GiniImpurityMeasureCalculatorTest {
+    /** */
+    @Test
+    public void testCalculate() {
+        double[][] data = new double[][]{{0, 1}, {1, 0}, {2, 2}, {3, 3}};
+        double[] labels = new double[]{0, 1, 1, 1};
+
+        Map<Double, Integer> encoder = new HashMap<>();
+        encoder.put(0.0, 0);
+        encoder.put(1.0, 1);
+        GiniImpurityMeasureCalculator calculator = new GiniImpurityMeasureCalculator(encoder);
+
+        StepFunction<GiniImpurityMeasure>[] impurity = calculator.calculate(new DecisionTreeData(data, labels));
+
+        assertEquals(2, impurity.length);
+
+        // Check Gini calculated for the first column.
+        assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[0].getX(), 1e-10);
+        assertEquals(-2.500, impurity[0].getY()[0].impurity(), 1e-3);
+        assertEquals(-4.000, impurity[0].getY()[1].impurity(),1e-3);
+        assertEquals(-3.000, impurity[0].getY()[2].impurity(),1e-3);
+        assertEquals(-2.666, impurity[0].getY()[3].impurity(),1e-3);
+        assertEquals(-2.500, impurity[0].getY()[4].impurity(),1e-3);
+
+        // Check Gini calculated for the second column.
+        assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[1].getX(), 1e-10);
+        assertEquals(-2.500, impurity[1].getY()[0].impurity(),1e-3);
+        assertEquals(-2.666, impurity[1].getY()[1].impurity(),1e-3);
+        assertEquals(-3.000, impurity[1].getY()[2].impurity(),1e-3);
+        assertEquals(-2.666, impurity[1].getY()[3].impurity(),1e-3);
+        assertEquals(-2.500, impurity[1].getY()[4].impurity(),1e-3);
+    }
+
+    /** */
+    @Test
+    public void testCalculateWithRepeatedData() {
+        double[][] data = new double[][]{{0}, {1}, {2}, {2}, {3}};
+        double[] labels = new double[]{0, 1, 1, 1, 1};
+
+        Map<Double, Integer> encoder = new HashMap<>();
+        encoder.put(0.0, 0);
+        encoder.put(1.0, 1);
+        GiniImpurityMeasureCalculator calculator = new GiniImpurityMeasureCalculator(encoder);
+
+        StepFunction<GiniImpurityMeasure>[] impurity = calculator.calculate(new DecisionTreeData(data, labels));
+
+        assertEquals(1, impurity.length);
+
+        // Check Gini calculated for the first column.
+        assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[0].getX(), 1e-10);
+        assertEquals(-3.400, impurity[0].getY()[0].impurity(), 1e-3);
+        assertEquals(-5.000, impurity[0].getY()[1].impurity(),1e-3);
+        assertEquals(-4.000, impurity[0].getY()[2].impurity(),1e-3);
+        assertEquals(-3.500, impurity[0].getY()[3].impurity(),1e-3);
+        assertEquals(-3.400, impurity[0].getY()[4].impurity(),1e-3);
+    }
+
+    /** */
+    @Test
+    public void testGetLabelCode() {
+        Map<Double, Integer> encoder = new HashMap<>();
+        encoder.put(0.0, 0);
+        encoder.put(1.0, 1);
+        encoder.put(2.0, 2);
+
+        GiniImpurityMeasureCalculator calculator = new GiniImpurityMeasureCalculator(encoder);
+
+        assertEquals(0, calculator.getLabelCode(0.0));
+        assertEquals(1, calculator.getLabelCode(1.0));
+        assertEquals(2, calculator.getLabelCode(2.0));
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java
new file mode 100644
index 0000000000000..35c456aa51adf
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/gini/GiniImpurityMeasureTest.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.gini;
+
+import java.util.Random;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+
+/**
+ * Tests for {@link GiniImpurityMeasure}.
+ */
+public class GiniImpurityMeasureTest {
+    /** */
+    @Test
+    public void testImpurityOnEmptyData() {
+        long[] left = new long[]{0, 0, 0};
+        long[] right = new long[]{0, 0, 0};
+
+        GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+        assertEquals(0.0, impurity.impurity(), 1e-10);
+    }
+
+    /** */
+    @Test
+    public void testImpurityLeftPart() {
+        long[] left = new long[]{3, 0, 0};
+        long[] right = new long[]{0, 0, 0};
+
+        GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+        assertEquals(-3, impurity.impurity(), 1e-10);
+    }
+
+    /** */
+    @Test
+    public void testImpurityRightPart() {
+        long[] left = new long[]{0, 0, 0};
+        long[] right = new long[]{3, 0, 0};
+
+        GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+        assertEquals(-3, impurity.impurity(), 1e-10);
+    }
+
+    /** */
+    @Test
+    public void testImpurityLeftAndRightPart() {
+        long[] left = new long[]{3, 0, 0};
+        long[] right = new long[]{0, 3, 0};
+
+        GiniImpurityMeasure impurity = new GiniImpurityMeasure(left, right);
+
+        assertEquals(-6, impurity.impurity(), 1e-10);
+    }
+
+    /** */
+    @Test
+    public void testAdd() {
+        Random rnd = new Random(0);
+
+        GiniImpurityMeasure a = new GiniImpurityMeasure(
+            new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+            new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+        );
+
+
+        GiniImpurityMeasure b = new GiniImpurityMeasure(
+            new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+            new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+        );
+
+        GiniImpurityMeasure c = a.add(b);
+
+        assertEquals(a.getLeft()[0] + b.getLeft()[0], c.getLeft()[0]);
+        assertEquals(a.getLeft()[1] + b.getLeft()[1], c.getLeft()[1]);
+        assertEquals(a.getLeft()[2] + b.getLeft()[2], c.getLeft()[2]);
+
+        assertEquals(a.getRight()[0] + b.getRight()[0], c.getRight()[0]);
+        assertEquals(a.getRight()[1] + b.getRight()[1], c.getRight()[1]);
+        assertEquals(a.getRight()[2] + b.getRight()[2], c.getRight()[2]);
+    }
+
+    /** */
+    @Test
+    public void testSubtract() {
+        Random rnd = new Random(0);
+
+        GiniImpurityMeasure a = new GiniImpurityMeasure(
+            new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+            new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+        );
+
+
+        GiniImpurityMeasure b = new GiniImpurityMeasure(
+            new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)},
+            new long[]{randCnt(rnd), randCnt(rnd), randCnt(rnd)}
+        );
+
+        GiniImpurityMeasure c = a.subtract(b);
+
+        assertEquals(a.getLeft()[0] - b.getLeft()[0], c.getLeft()[0]);
+        assertEquals(a.getLeft()[1] - b.getLeft()[1], c.getLeft()[1]);
+        assertEquals(a.getLeft()[2] - b.getLeft()[2], c.getLeft()[2]);
+
+        assertEquals(a.getRight()[0] - b.getRight()[0], c.getRight()[0]);
+        assertEquals(a.getRight()[1] - b.getRight()[1], c.getRight()[1]);
+        assertEquals(a.getRight()[2] - b.getRight()[2], c.getRight()[2]);
+    }
+
+    /** Generates random count. */
+    private long randCnt(Random rnd) {
+        return Math.abs(rnd.nextInt());
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java
new file mode 100644
index 0000000000000..510c18fe3750e
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureCalculatorTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import org.apache.ignite.ml.tree.data.DecisionTreeData;
+import org.apache.ignite.ml.tree.impurity.util.StepFunction;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link MSEImpurityMeasureCalculator}.
+ */
+public class MSEImpurityMeasureCalculatorTest {
+    /** */
+    @Test
+    public void testCalculate() {
+        double[][] data = new double[][]{{0, 2}, {1, 1}, {2, 0}, {3, 3}};
+        double[] labels = new double[]{1, 2, 2, 1};
+
+        MSEImpurityMeasureCalculator calculator = new MSEImpurityMeasureCalculator();
+
+        StepFunction<MSEImpurityMeasure>[] impurity = calculator.calculate(new DecisionTreeData(data, labels));
+
+        assertEquals(2, impurity.length);
+
+        // Test MSE calculated for the first column.
+        assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[0].getX(), 1e-10);
+        assertEquals(1.000, impurity[0].getY()[0].impurity(), 1e-3);
+        assertEquals(0.666, impurity[0].getY()[1].impurity(),1e-3);
+        assertEquals(1.000, impurity[0].getY()[2].impurity(),1e-3);
+        assertEquals(0.666, impurity[0].getY()[3].impurity(),1e-3);
+        assertEquals(1.000, impurity[0].getY()[4].impurity(),1e-3);
+
+        // Test MSE calculated for the second column.
+        assertArrayEquals(new double[]{Double.NEGATIVE_INFINITY, 0, 1, 2, 3}, impurity[1].getX(), 1e-10);
+        assertEquals(1.000, impurity[1].getY()[0].impurity(),1e-3);
+        assertEquals(0.666, impurity[1].getY()[1].impurity(),1e-3);
+        assertEquals(0.000, impurity[1].getY()[2].impurity(),1e-3);
+        assertEquals(0.666, impurity[1].getY()[3].impurity(),1e-3);
+        assertEquals(1.000, impurity[1].getY()[4].impurity(),1e-3);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java
new file mode 100644
index 0000000000000..3d11d9d7d95f5
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/mse/MSEImpurityMeasureTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.mse;
+
+import java.util.Random;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+
+/**
+ * Tests for {@link MSEImpurityMeasure}.
+ */
+public class MSEImpurityMeasureTest {
+    /** */
+    @Test
+    public void testImpurityOnEmptyData() {
+        MSEImpurityMeasure impurity = new MSEImpurityMeasure(0, 0, 0, 0, 0, 0);
+
+        assertEquals(0.0, impurity.impurity(), 1e-10);
+    }
+
+    /** */
+    @Test
+    public void testImpurityLeftPart() {
+        // Test on left part [1, 2, 2, 1, 1, 1].
+        MSEImpurityMeasure impurity = new MSEImpurityMeasure(8, 12, 6, 0, 0, 0);
+
+        assertEquals(1.333, impurity.impurity(), 1e-3);
+    }
+
+    /** */
+    @Test
+    public void testImpurityRightPart() {
+        // Test on right part [1, 2, 2, 1, 1, 1].
+        MSEImpurityMeasure impurity = new MSEImpurityMeasure(0, 0, 0, 8, 12, 6);
+
+        assertEquals(1.333, impurity.impurity(), 1e-3);
+    }
+
+    /** */
+    @Test
+    public void testImpurityLeftAndRightPart() {
+        // Test on left part [1, 2, 2] and right part [1, 1, 1].
+        MSEImpurityMeasure impurity = new MSEImpurityMeasure(5, 9, 3, 3, 3, 3);
+
+        assertEquals(0.666, impurity.impurity(), 1e-3);
+    }
+
+    /** */
+    @Test
+    public void testAdd() {
+        Random rnd = new Random(0);
+
+        MSEImpurityMeasure a = new MSEImpurityMeasure(
+            rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+        );
+
+        MSEImpurityMeasure b = new MSEImpurityMeasure(
+            rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+        );
+
+        MSEImpurityMeasure c = a.add(b);
+
+        assertEquals(a.getLeftY() + b.getLeftY(), c.getLeftY(), 1e-10);
+        assertEquals(a.getLeftY2() + b.getLeftY2(), c.getLeftY2(), 1e-10);
+        assertEquals(a.getLeftCnt() + b.getLeftCnt(), c.getLeftCnt());
+        assertEquals(a.getRightY() + b.getRightY(), c.getRightY(), 1e-10);
+        assertEquals(a.getRightY2() + b.getRightY2(), c.getRightY2(), 1e-10);
+        assertEquals(a.getRightCnt() + b.getRightCnt(), c.getRightCnt());
+    }
+
+    /** */
+    @Test
+    public void testSubtract() {
+        Random rnd = new Random(0);
+
+        MSEImpurityMeasure a = new MSEImpurityMeasure(
+            rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+        );
+
+        MSEImpurityMeasure b = new MSEImpurityMeasure(
+            rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt(), rnd.nextDouble(), rnd.nextDouble(), rnd.nextInt()
+        );
+
+        MSEImpurityMeasure c = a.subtract(b);
+
+        assertEquals(a.getLeftY() - b.getLeftY(), c.getLeftY(), 1e-10);
+        assertEquals(a.getLeftY2() - b.getLeftY2(), c.getLeftY2(), 1e-10);
+        assertEquals(a.getLeftCnt() - b.getLeftCnt(), c.getLeftCnt());
+        assertEquals(a.getRightY() - b.getRightY(), c.getRightY(), 1e-10);
+        assertEquals(a.getRightY2() - b.getRightY2(), c.getRightY2(), 1e-10);
+        assertEquals(a.getRightCnt() - b.getRightCnt(), c.getRightCnt());
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java
new file mode 100644
index 0000000000000..001404fe52126
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/SimpleStepFunctionCompressorTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link SimpleStepFunctionCompressor}.
+ */
+public class SimpleStepFunctionCompressorTest {
+    /** */
+    @Test
+    public void testCompressSmallFunction() {
+        StepFunction<TestImpurityMeasure> function = new StepFunction<>(
+            new double[]{1, 2, 3, 4},
+            TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4)
+        );
+
+        SimpleStepFunctionCompressor<TestImpurityMeasure> compressor = new SimpleStepFunctionCompressor<>(5, 0, 0);
+
+        StepFunction<TestImpurityMeasure> resFunction = compressor.compress(function);
+
+        assertArrayEquals(new double[]{1, 2, 3, 4}, resFunction.getX(), 1e-10);
+        assertArrayEquals(TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4), resFunction.getY());
+    }
+
+    /** */
+    @Test
+    public void testCompressIncreasingFunction() {
+        StepFunction<TestImpurityMeasure> function = new StepFunction<>(
+            new double[]{1, 2, 3, 4, 5},
+            TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4, 5)
+        );
+
+        SimpleStepFunctionCompressor<TestImpurityMeasure> compressor = new SimpleStepFunctionCompressor<>(1, 0.4, 0);
+
+        StepFunction<TestImpurityMeasure> resFunction = compressor.compress(function);
+
+        assertArrayEquals(new double[]{1, 3, 5}, resFunction.getX(), 1e-10);
+        assertArrayEquals(TestImpurityMeasure.asTestImpurityMeasures(1, 3, 5), resFunction.getY());
+    }
+
+    /** */
+    @Test
+    public void testCompressDecreasingFunction() {
+        StepFunction<TestImpurityMeasure> function = new StepFunction<>(
+            new double[]{1, 2, 3, 4, 5},
+            TestImpurityMeasure.asTestImpurityMeasures(5, 4, 3, 2, 1)
+        );
+
+        SimpleStepFunctionCompressor<TestImpurityMeasure> compressor = new SimpleStepFunctionCompressor<>(1, 0, 0.4);
+
+        StepFunction<TestImpurityMeasure> resFunction = compressor.compress(function);
+
+        assertArrayEquals(new double[]{1, 3, 5}, resFunction.getX(), 1e-10);
+        assertArrayEquals(TestImpurityMeasure.asTestImpurityMeasures(5, 3, 1), resFunction.getY());
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java
new file mode 100644
index 0000000000000..2a0279cec0eef
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/StepFunctionTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+
+/**
+ * Tests for {@link StepFunction}.
+ */
+public class StepFunctionTest {
+    /** */
+    @Test
+    public void testAddIncreasingFunctions() {
+        StepFunction<TestImpurityMeasure> a = new StepFunction<>(
+            new double[]{1, 3, 5},
+            TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3)
+        );
+
+        StepFunction<TestImpurityMeasure> b = new StepFunction<>(
+            new double[]{0, 2, 4},
+            TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3)
+        );
+
+        StepFunction<TestImpurityMeasure> c = a.add(b);
+
+        assertArrayEquals(new double[]{0, 1, 2, 3, 4, 5}, c.getX(), 1e-10);
+        assertArrayEquals(
+            TestImpurityMeasure.asTestImpurityMeasures(1, 2, 3, 4, 5, 6),
+            c.getY()
+        );
+    }
+
+    /** */
+    @Test
+    public void testAddDecreasingFunctions() {
+        StepFunction<TestImpurityMeasure> a = new StepFunction<>(
+            new double[]{1, 3, 5},
+            TestImpurityMeasure.asTestImpurityMeasures(3, 2, 1)
+        );
+
+        StepFunction<TestImpurityMeasure> b = new StepFunction<>(
+            new double[]{0, 2, 4},
+            TestImpurityMeasure.asTestImpurityMeasures(3, 2, 1)
+        );
+
+        StepFunction<TestImpurityMeasure> c = a.add(b);
+
+        assertArrayEquals(new double[]{0, 1, 2, 3, 4, 5}, c.getX(), 1e-10);
+        assertArrayEquals(
+            TestImpurityMeasure.asTestImpurityMeasures(3, 6, 5, 4, 3, 2),
+            c.getY()
+        );
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java
new file mode 100644
index 0000000000000..c0d1911a586e8
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/impurity/util/TestImpurityMeasure.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.impurity.util;
+
+import java.util.Objects;
+import org.apache.ignite.ml.tree.impurity.ImpurityMeasure;
+
+/**
+ * Utils class used as impurity measure in tests.
+ */
+class TestImpurityMeasure implements ImpurityMeasure<TestImpurityMeasure> {
+    /** */
+    private static final long serialVersionUID = 2414020770162797847L;
+
+    /** Impurity. */
+    private final double impurity;
+
+    /**
+     * Constructs a new instance of test impurity measure.
+     *
+     * @param impurity Impurity.
+     */
+    private TestImpurityMeasure(double impurity) {
+        this.impurity = impurity;
+    }
+
+    /**
+     * Convert doubles to array of test impurity measures.
+     *
+     * @param impurity Impurity as array of doubles.
+     * @return Test impurity measure objects as array.
+     */
+    static TestImpurityMeasure[] asTestImpurityMeasures(double... impurity) {
+        TestImpurityMeasure[] res = new TestImpurityMeasure[impurity.length];
+
+        for (int i = 0; i < impurity.length; i++)
+            res[i] = new TestImpurityMeasure(impurity[i]);
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public double impurity() {
+        return impurity;
+    }
+
+    /** {@inheritDoc} */
+    @Override public TestImpurityMeasure add(TestImpurityMeasure measure) {
+        return new TestImpurityMeasure(impurity + measure.impurity);
+    }
+
+    /** {@inheritDoc} */
+    @Override public TestImpurityMeasure subtract(TestImpurityMeasure measure) {
+        return new TestImpurityMeasure(impurity - measure.impurity);
+    }
+
+    /** */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        TestImpurityMeasure measure = (TestImpurityMeasure)o;
+
+        return Double.compare(measure.impurity, impurity) == 0;
+    }
+
+    /** */
+    @Override public int hashCode() {
+
+        return Objects.hash(impurity);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
new file mode 100644
index 0000000000000..b259ec9700de5
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.performance;
+
+import java.io.IOException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.nn.performance.MnistMLPTestUtil;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressor;
+import org.apache.ignite.ml.util.MnistUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests {@link DecisionTreeClassificationTrainer} on the MNIST dataset that require to start the whole Ignite
+ * infrastructure. For manual run.
+ */
+public class DecisionTreeMNISTIntegrationTest extends GridCommonAbstractTest {
+    /** Number of nodes in grid */
+    private static final int NODE_COUNT = 3;
+
+    /** Ignite instance. */
+    private Ignite ignite;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        for (int i = 1; i <= NODE_COUNT; i++)
+            startGrid(i);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() {
+        stopAllGrids();
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected void beforeTest() throws Exception {
+        /* Grid instance. */
+        ignite = grid(NODE_COUNT);
+        ignite.configuration().setPeerClassLoadingEnabled(true);
+        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+    }
+
+    /** Tests on the MNIST dataset. For manual run. */
+    public void testMNIST() throws IOException {
+        CacheConfiguration<Integer, MnistUtils.MnistLabeledImage> trainingSetCacheCfg = new CacheConfiguration<>();
+        trainingSetCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+        trainingSetCacheCfg.setName("MNIST_TRAINING_SET");
+
+        IgniteCache<Integer, MnistUtils.MnistLabeledImage> trainingSet = ignite.createCache(trainingSetCacheCfg);
+
+        int i = 0;
+        for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTrainingSet(60_000))
+            trainingSet.put(i++, e);
+
+        DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(
+            8,
+            0,
+            new SimpleStepFunctionCompressor<>());
+
+        DecisionTreeNode mdl = trainer.fit(
+            new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+            (k, v) -> v.getPixels(),
+            (k, v) -> (double) v.getLabel()
+        );
+
+        int correctAnswers = 0;
+        int incorrectAnswers = 0;
+
+        for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTestSet(10_000)) {
+            double res = mdl.apply(e.getPixels());
+
+            if (res == e.getLabel())
+                correctAnswers++;
+            else
+                incorrectAnswers++;
+        }
+
+        double accuracy = 1.0 * correctAnswers / (correctAnswers + incorrectAnswers);
+
+        assertTrue(accuracy > 0.8);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
new file mode 100644
index 0000000000000..6dbd44c5919b3
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.tree.performance;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.nn.performance.MnistMLPTestUtil;
+import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
+import org.apache.ignite.ml.tree.DecisionTreeNode;
+import org.apache.ignite.ml.tree.impurity.util.SimpleStepFunctionCompressor;
+import org.apache.ignite.ml.util.MnistUtils;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertTrue;
+
+/**
+ * Tests {@link DecisionTreeClassificationTrainer} on the MNIST dataset using locally stored data. For manual run.
+ */
+public class DecisionTreeMNISTTest {
+    /** Tests on the MNIST dataset. For manual run. */
+    @Test
+    public void testMNIST() throws IOException {
+        Map<Integer, MnistUtils.MnistLabeledImage> trainingSet = new HashMap<>();
+
+        int i = 0;
+        for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTrainingSet(60_000))
+            trainingSet.put(i++, e);
+
+
+        DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(
+            8,
+            0,
+            new SimpleStepFunctionCompressor<>());
+
+        DecisionTreeNode mdl = trainer.fit(
+            new LocalDatasetBuilder<>(trainingSet, 10),
+            (k, v) -> v.getPixels(),
+            (k, v) -> (double) v.getLabel()
+        );
+
+        int correctAnswers = 0;
+        int incorrectAnswers = 0;
+
+        for (MnistUtils.MnistLabeledImage e : MnistMLPTestUtil.loadTestSet(10_000)) {
+            double res = mdl.apply(e.getPixels());
+
+            if (res == e.getLabel())
+                correctAnswers++;
+            else
+                incorrectAnswers++;
+        }
+
+        double accuracy = 1.0 * correctAnswers / (correctAnswers + incorrectAnswers);
+
+        assertTrue(accuracy > 0.8);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java
deleted file mode 100644
index 65f0ae49b919f..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/BaseDecisionTreeTest.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.Arrays;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Base class for decision trees test.
- */
-public class BaseDecisionTreeTest extends GridCommonAbstractTest {
-    /** Count of nodes. */
-    private static final int NODE_COUNT = 4;
-
-    /** Grid instance. */
-    protected Ignite ignite;
-
-    /**
-     * Default constructor.
-     */
-    public BaseDecisionTreeTest() {
-        super(false);
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override protected void beforeTest() throws Exception {
-        ignite = grid(NODE_COUNT);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        for (int i = 1; i <= NODE_COUNT; i++)
-            startGrid(i);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        stopAllGrids();
-    }
-
-    /**
-     * Convert double array to  {@link LabeledVectorDouble}
-     *
-     * @param arr Array for conversion.
-     * @return LabeledVectorDouble.
-     */
-    protected static LabeledVectorDouble<DenseLocalOnHeapVector> asLabeledVector(double arr[]) {
-        return new LabeledVectorDouble<>(new DenseLocalOnHeapVector(Arrays.copyOf(arr, arr.length - 1)), arr[arr.length - 1]);
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java
deleted file mode 100644
index b090f43ace881..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/ColumnDecisionTreeTrainerTest.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Tracer;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-
-/** Tests behaviour of ColumnDecisionTreeTrainer. */
-public class ColumnDecisionTreeTrainerTest extends BaseDecisionTreeTest {
-    /**
-     * Test {@link ColumnDecisionTreeTrainerTest} for mixed (continuous and categorical) data with Gini impurity.
-     */
-    public void testCacheMixedGini() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        int totalPts = 1 << 10;
-        int featCnt = 2;
-
-        HashMap<Integer, Integer> catsInfo = new HashMap<>();
-        catsInfo.put(1, 3);
-
-        Random rnd = new Random(12349L);
-
-        SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
-            featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
-            split(0, 1, new int[] {0, 2}).
-            split(1, 0, -10.0);
-
-        testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MEAN, rnd);
-    }
-
-    /**
-     * Test {@link ColumnDecisionTreeTrainerTest} for mixed (continuous and categorical) data with Variance impurity.
-     */
-    public void testCacheMixed() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        int totalPts = 1 << 10;
-        int featCnt = 2;
-
-        HashMap<Integer, Integer> catsInfo = new HashMap<>();
-        catsInfo.put(1, 3);
-
-        Random rnd = new Random(12349L);
-
-        SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
-            featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
-            split(0, 1, new int[] {0, 2}).
-            split(1, 0, -10.0);
-
-        testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, rnd);
-    }
-
-    /**
-     * Test {@link ColumnDecisionTreeTrainerTest} for continuous data with Variance impurity.
-     */
-    public void testCacheCont() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        int totalPts = 1 << 10;
-        int featCnt = 12;
-
-        HashMap<Integer, Integer> catsInfo = new HashMap<>();
-
-        Random rnd = new Random(12349L);
-
-        SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
-            featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
-            split(0, 0, -10.0).
-            split(1, 0, 0.0).
-            split(1, 1, 2.0).
-            split(3, 7, 50.0);
-
-        testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, rnd);
-    }
-
-    /**
-     * Test {@link ColumnDecisionTreeTrainerTest} for continuous data with Gini impurity.
-     */
-    public void testCacheContGini() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        int totalPts = 1 << 10;
-        int featCnt = 12;
-
-        HashMap<Integer, Integer> catsInfo = new HashMap<>();
-
-        Random rnd = new Random(12349L);
-
-        SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
-            featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
-            split(0, 0, -10.0).
-            split(1, 0, 0.0).
-            split(1, 1, 2.0).
-            split(3, 7, 50.0);
-
-        testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MEAN, rnd);
-    }
-
-    /**
-     * Test {@link ColumnDecisionTreeTrainerTest} for categorical data with Variance impurity.
-     */
-    public void testCacheCat() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        int totalPts = 1 << 10;
-        int featCnt = 12;
-
-        HashMap<Integer, Integer> catsInfo = new HashMap<>();
-        catsInfo.put(5, 7);
-
-        Random rnd = new Random(12349L);
-
-        SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
-            featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
-            split(0, 5, new int[] {0, 2, 5});
-
-        testByGen(totalPts, catsInfo, gen, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, rnd);
-    }
-
-    /** */
-    private <D extends ContinuousRegionInfo> void testByGen(int totalPts, HashMap<Integer, Integer> catsInfo,
-        SplitDataGenerator<DenseLocalOnHeapVector> gen,
-        IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> calc,
-        IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> catImpCalc,
-        IgniteFunction<DoubleStream, Double> regCalc, Random rnd) {
-
-        List<IgniteBiTuple<Integer, DenseLocalOnHeapVector>> lst = gen.
-            points(totalPts, (i, rn) -> i).
-            collect(Collectors.toList());
-
-        int featCnt = gen.featuresCnt();
-
-        Collections.shuffle(lst, rnd);
-
-        SparseDistributedMatrix m = new SparseDistributedMatrix(totalPts, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
-        Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
-
-        int i = 0;
-        for (IgniteBiTuple<Integer, DenseLocalOnHeapVector> bt : lst) {
-            byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
-            byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
-            m.setRow(i, bt.get2().getStorage().data());
-            i++;
-        }
-
-        ColumnDecisionTreeTrainer<D> trainer =
-            new ColumnDecisionTreeTrainer<>(3, calc, catImpCalc, regCalc, ignite);
-
-        DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catsInfo));
-
-        byRegion.keySet().forEach(k -> {
-            LabeledVectorDouble sp = byRegion.get(k).get(0);
-            Tracer.showAscii(sp.features());
-            X.println("Actual and predicted vectors [act=" + sp.label() + " " + ", pred=" + mdl.apply(sp.features()) + "]");
-            assert mdl.apply(sp.features()) == sp.doubleLabel();
-        });
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java
deleted file mode 100644
index 3343503d97b73..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/DecisionTreesTestSuite.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/**
- * Test suite for all tests located in org.apache.ignite.ml.trees package
- */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
-    ColumnDecisionTreeTrainerTest.class,
-    GiniSplitCalculatorTest.class,
-    VarianceSplitCalculatorTest.class
-})
-public class DecisionTreesTestSuite {
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java
deleted file mode 100644
index c92b4f54f33a3..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/GiniSplitCalculatorTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.GiniSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.junit.Test;
-
-/**
- * Test of {@link GiniSplitCalculator}.
- */
-public class GiniSplitCalculatorTest {
-    /** Test calculation of region info consisting from one point. */
-    @Test
-    public void testCalculateRegionInfoSimple() {
-        double labels[] = new double[] {0.0};
-
-        assert new GiniSplitCalculator(labels).calculateRegionInfo(DoubleStream.of(labels), 0).impurity() == 0.0;
-    }
-
-    /** Test calculation of region info consisting from two distinct classes. */
-    @Test
-    public void testCalculateRegionInfoTwoClasses() {
-        double labels[] = new double[] {0.0, 1.0};
-
-        assert new GiniSplitCalculator(labels).calculateRegionInfo(DoubleStream.of(labels), 0).impurity() == 0.5;
-    }
-
-    /** Test calculation of region info consisting from three distinct classes. */
-    @Test
-    public void testCalculateRegionInfoThreeClasses() {
-        double labels[] = new double[] {0.0, 1.0, 2.0};
-
-        assert Math.abs(new GiniSplitCalculator(labels).calculateRegionInfo(DoubleStream.of(labels), 0).impurity() - 2.0 / 3) < 1E-5;
-    }
-
-    /** Test calculation of split of region consisting from one point. */
-    @Test
-    public void testSplitSimple() {
-        double labels[] = new double[] {0.0};
-        double values[] = new double[] {0.0};
-        Integer[] samples = new Integer[] {0};
-
-        int cnts[] = new int[] {1};
-
-        GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(0.0, 1, cnts, 1);
-
-        assert new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data) == null;
-    }
-
-    /** Test calculation of split of region consisting from two points. */
-    @Test
-    public void testSplitTwoClassesTwoPoints() {
-        double labels[] = new double[] {0.0, 1.0};
-        double values[] = new double[] {0.0, 1.0};
-        Integer[] samples = new Integer[] {0, 1};
-
-        int cnts[] = new int[] {1, 1};
-
-        GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(0.5, 2, cnts, 1.0 * 1.0 + 1.0 * 1.0);
-
-        SplitInfo<GiniSplitCalculator.GiniData> split = new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data);
-
-        assert split.leftData().impurity() == 0;
-        assert split.leftData().counts()[0] == 1;
-        assert split.leftData().counts()[1] == 0;
-        assert split.leftData().getSize() == 1;
-
-        assert split.rightData().impurity() == 0;
-        assert split.rightData().counts()[0] == 0;
-        assert split.rightData().counts()[1] == 1;
-        assert split.rightData().getSize() == 1;
-    }
-
-    /** Test calculation of split of region consisting from four distinct values. */
-    @Test
-    public void testSplitTwoClassesFourPoints() {
-        double labels[] = new double[] {0.0, 0.0, 1.0, 1.0};
-        double values[] = new double[] {0.0, 1.0, 2.0, 3.0};
-
-        Integer[] samples = new Integer[] {0, 1, 2, 3};
-
-        int[] cnts = new int[] {2, 2};
-
-        GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(0.5, 4, cnts, 2.0 * 2.0 + 2.0 * 2.0);
-
-        SplitInfo<GiniSplitCalculator.GiniData> split = new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data);
-
-        assert split.leftData().impurity() == 0;
-        assert split.leftData().counts()[0] == 2;
-        assert split.leftData().counts()[1] == 0;
-        assert split.leftData().getSize() == 2;
-
-        assert split.rightData().impurity() == 0;
-        assert split.rightData().counts()[0] == 0;
-        assert split.rightData().counts()[1] == 2;
-        assert split.rightData().getSize() == 2;
-    }
-
-    /** Test calculation of split of region consisting from three distinct values. */
-    @Test
-    public void testSplitThreePoints() {
-        double labels[] = new double[] {0.0, 1.0, 2.0};
-        double values[] = new double[] {0.0, 1.0, 2.0};
-        Integer[] samples = new Integer[] {0, 1, 2};
-
-        int[] cnts = new int[] {1, 1, 1};
-
-        GiniSplitCalculator.GiniData data = new GiniSplitCalculator.GiniData(2.0 / 3, 3, cnts, 1.0 * 1.0 + 1.0 * 1.0 + 1.0 * 1.0);
-
-        SplitInfo<GiniSplitCalculator.GiniData> split = new GiniSplitCalculator(labels).splitRegion(samples, values, labels, 0, data);
-
-        assert split.leftData().impurity() == 0.0;
-        assert split.leftData().counts()[0] == 1;
-        assert split.leftData().counts()[1] == 0;
-        assert split.leftData().counts()[2] == 0;
-        assert split.leftData().getSize() == 1;
-
-        assert split.rightData().impurity() == 0.5;
-        assert split.rightData().counts()[0] == 0;
-        assert split.rightData().counts()[1] == 1;
-        assert split.rightData().counts()[2] == 1;
-        assert split.rightData().getSize() == 2;
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java
deleted file mode 100644
index 279e6851c52b5..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/SplitDataGenerator.java
+++ /dev/null
@@ -1,390 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.ml.trees;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.util.Utils;
-
-/**
- * Utility class for generating data which has binary tree split structure.
- *
- * @param <V>
- */
-public class SplitDataGenerator<V extends Vector> {
-    /** */
-    private static final double DELTA = 100.0;
-
-    /** Map of the form of (is categorical -> list of region indexes). */
-    private final Map<Boolean, List<Integer>> di;
-
-    /** List of regions. */
-    private final List<Region> regs;
-
-    /** Data of bounds of regions. */
-    private final Map<Integer, IgniteBiTuple<Double, Double>> boundsData;
-
-    /** Random numbers generator. */
-    private final Random rnd;
-
-    /** Supplier of vectors. */
-    private final Supplier<V> supplier;
-
-    /** Features count. */
-    private final int featCnt;
-
-    /**
-     * Create SplitDataGenerator.
-     *
-     * @param featCnt Features count.
-     * @param catFeaturesInfo Information about categorical features in form of map (feature index -> categories
-     * count).
-     * @param supplier Supplier of vectors.
-     * @param rnd Random numbers generator.
-     */
-    public SplitDataGenerator(int featCnt, Map<Integer, Integer> catFeaturesInfo, Supplier<V> supplier, Random rnd) {
-        regs = new LinkedList<>();
-        boundsData = new HashMap<>();
-        this.rnd = rnd;
-        this.supplier = supplier;
-        this.featCnt = featCnt;
-
-        // Divide indexes into indexes of categorical coordinates and indexes of continuous coordinates.
-        di = IntStream.range(0, featCnt).
-            boxed().
-            collect(Collectors.partitioningBy(catFeaturesInfo::containsKey));
-
-        // Categorical coordinates info.
-        Map<Integer, CatCoordInfo> catCoords = new HashMap<>();
-        di.get(true).forEach(i -> {
-            BitSet bs = new BitSet();
-            bs.set(0, catFeaturesInfo.get(i));
-            catCoords.put(i, new CatCoordInfo(bs));
-        });
-
-        // Continuous coordinates info.
-        Map<Integer, ContCoordInfo> contCoords = new HashMap<>();
-        di.get(false).forEach(i -> {
-            contCoords.put(i, new ContCoordInfo());
-            boundsData.put(i, new IgniteBiTuple<>(-1.0, 1.0));
-        });
-
-        Region firstReg = new Region(catCoords, contCoords, 0);
-        regs.add(firstReg);
-    }
-
-    /**
-     * Categorical coordinate info.
-     */
-    private static class CatCoordInfo implements Serializable {
-        /**
-         * Defines categories which are included in this region
-         */
-        private final BitSet bs;
-
-        /**
-         * Construct CatCoordInfo.
-         *
-         * @param bs Bitset.
-         */
-        CatCoordInfo(BitSet bs) {
-            this.bs = bs;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return "CatCoordInfo [" +
-                "bs=" + bs +
-                ']';
-        }
-    }
-
-    /**
-     * Continuous coordinate info.
-     */
-    private static class ContCoordInfo implements Serializable {
-        /**
-         * Left (min) bound of region.
-         */
-        private double left;
-
-        /**
-         * Right (max) bound of region.
-         */
-        private double right;
-
-        /**
-         * Construct ContCoordInfo.
-         */
-        ContCoordInfo() {
-            left = Double.NEGATIVE_INFINITY;
-            right = Double.POSITIVE_INFINITY;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return "ContCoordInfo [" +
-                "left=" + left +
-                ", right=" + right +
-                ']';
-        }
-    }
-
-    /**
-     * Class representing information about region.
-     */
-    private static class Region implements Serializable {
-        /**
-         * Information about categorical coordinates restrictions of this region in form of
-         * (coordinate index -> restriction)
-         */
-        private final Map<Integer, CatCoordInfo> catCoords;
-
-        /**
-         * Information about continuous coordinates restrictions of this region in form of
-         * (coordinate index -> restriction)
-         */
-        private final Map<Integer, ContCoordInfo> contCoords;
-
-        /**
-         * Region should contain {@code 1/2^twoPow * totalPoints} points.
-         */
-        private int twoPow;
-
-        /**
-         * Construct region by information about restrictions on coordinates (features) values.
-         *
-         * @param catCoords Restrictions on categorical coordinates.
-         * @param contCoords Restrictions on continuous coordinates
-         * @param twoPow Region should contain {@code 1/2^twoPow * totalPoints} points.
-         */
-        Region(Map<Integer, CatCoordInfo> catCoords, Map<Integer, ContCoordInfo> contCoords, int twoPow) {
-            this.catCoords = catCoords;
-            this.contCoords = contCoords;
-            this.twoPow = twoPow;
-        }
-
-        /** */
-        int divideBy() {
-            return 1 << twoPow;
-        }
-
-        /** */
-        void incTwoPow() {
-            twoPow++;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return "Region [" +
-                "catCoords=" + catCoords +
-                ", contCoords=" + contCoords +
-                ", twoPow=" + twoPow +
-                ']';
-        }
-
-        /**
-         * Generate continuous coordinate for this region.
-         *
-         * @param coordIdx Coordinate index.
-         * @param boundsData Data with bounds
-         * @param rnd Random numbers generator.
-         * @return Categorical coordinate value.
-         */
-        double generateContCoord(int coordIdx, Map<Integer, IgniteBiTuple<Double, Double>> boundsData,
-            Random rnd) {
-            ContCoordInfo cci = contCoords.get(coordIdx);
-            double left = cci.left;
-            double right = cci.right;
-
-            if (left == Double.NEGATIVE_INFINITY)
-                left = boundsData.get(coordIdx).get1() - DELTA;
-
-            if (right == Double.POSITIVE_INFINITY)
-                right = boundsData.get(coordIdx).get2() + DELTA;
-
-            double size = right - left;
-
-            return left + rnd.nextDouble() * size;
-        }
-
-        /**
-         * Generate categorical coordinate value for this region.
-         *
-         * @param coordIdx Coordinate index.
-         * @param rnd Random numbers generator.
-         * @return Categorical coordinate value.
-         */
-        double generateCatCoord(int coordIdx, Random rnd) {
-            // Pick random bit.
-            BitSet bs = catCoords.get(coordIdx).bs;
-            int j = rnd.nextInt(bs.length());
-
-            int i = 0;
-            int bn = 0;
-            int bnp = 0;
-
-            while ((bn = bs.nextSetBit(bn)) != -1 && i <= j) {
-                i++;
-                bnp = bn;
-                bn++;
-            }
-
-            return bnp;
-        }
-
-        /**
-         * Generate points for this region.
-         *
-         * @param ptsCnt Count of points to generate.
-         * @param val Label for all points in this region.
-         * @param boundsData Data about bounds of continuous coordinates.
-         * @param catCont Data about which categories can be in this region in the form (coordinate index -> list of
-         * categories indexes).
-         * @param s Vectors supplier.
-         * @param rnd Random numbers generator.
-         * @param <V> Type of vectors.
-         * @return Stream of generated points for this region.
-         */
-        <V extends Vector> Stream<V> generatePoints(int ptsCnt, double val,
-            Map<Integer, IgniteBiTuple<Double, Double>> boundsData, Map<Boolean, List<Integer>> catCont,
-            Supplier<V> s,
-            Random rnd) {
-            return IntStream.range(0, ptsCnt / divideBy()).mapToObj(i -> {
-                V v = s.get();
-                int coordsCnt = v.size();
-                catCont.get(false).forEach(ci -> v.setX(ci, generateContCoord(ci, boundsData, rnd)));
-                catCont.get(true).forEach(ci -> v.setX(ci, generateCatCoord(ci, rnd)));
-
-                v.setX(coordsCnt - 1, val);
-                return v;
-            });
-        }
-    }
-
-    /**
-     * Split region by continuous coordinate.using given threshold.
-     *
-     * @param regIdx Region index.
-     * @param coordIdx Coordinate index.
-     * @param threshold Threshold.
-     * @return {@code this}.
-     */
-    public SplitDataGenerator<V> split(int regIdx, int coordIdx, double threshold) {
-        Region regToSplit = regs.get(regIdx);
-        ContCoordInfo cci = regToSplit.contCoords.get(coordIdx);
-
-        double left = cci.left;
-        double right = cci.right;
-
-        if (threshold < left || threshold > right)
-            throw new MathIllegalArgumentException("Threshold is out of region bounds.");
-
-        regToSplit.incTwoPow();
-
-        Region newReg = Utils.copy(regToSplit);
-        newReg.contCoords.get(coordIdx).left = threshold;
-
-        regs.add(regIdx + 1, newReg);
-        cci.right = threshold;
-
-        IgniteBiTuple<Double, Double> bounds = boundsData.get(coordIdx);
-        double min = bounds.get1();
-        double max = bounds.get2();
-        boundsData.put(coordIdx, new IgniteBiTuple<>(Math.min(threshold, min), Math.max(max, threshold)));
-
-        return this;
-    }
-
-    /**
-     * Split region by categorical coordinate.
-     *
-     * @param regIdx Region index.
-     * @param coordIdx Coordinate index.
-     * @param cats Categories allowed for the left sub region.
-     * @return {@code this}.
-     */
-    public SplitDataGenerator<V> split(int regIdx, int coordIdx, int[] cats) {
-        BitSet subset = new BitSet();
-        Arrays.stream(cats).forEach(subset::set);
-        Region regToSplit = regs.get(regIdx);
-        CatCoordInfo cci = regToSplit.catCoords.get(coordIdx);
-
-        BitSet ssc = (BitSet)subset.clone();
-        BitSet set = cci.bs;
-        ssc.and(set);
-        if (ssc.length() != subset.length())
-            throw new MathIllegalArgumentException("Splitter set is not a subset of a parent subset.");
-
-        ssc.xor(set);
-        set.and(subset);
-
-        regToSplit.incTwoPow();
-        Region newReg = Utils.copy(regToSplit);
-        newReg.catCoords.put(coordIdx, new CatCoordInfo(ssc));
-
-        regs.add(regIdx + 1, newReg);
-
-        return this;
-    }
-
-    /**
-     * Get stream of points generated by this generator.
-     *
-     * @param ptsCnt Points count.
-     */
-    public Stream<IgniteBiTuple<Integer, V>> points(int ptsCnt, BiFunction<Double, Random, Double> f) {
-        regs.forEach(System.out::println);
-
-        return IntStream.range(0, regs.size()).
-            boxed().
-            map(i -> regs.get(i).generatePoints(ptsCnt, f.apply((double)i, rnd), boundsData, di, supplier, rnd).map(v -> new IgniteBiTuple<>(i, v))).flatMap(Function.identity());
-    }
-
-    /**
-     * Count of regions.
-     *
-     * @return Count of regions.
-     */
-    public int regsCount() {
-        return regs.size();
-    }
-
-    /**
-     * Get features count.
-     *
-     * @return Features count.
-     */
-    public int featuresCnt() {
-        return featCnt;
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java
deleted file mode 100644
index d67cbc6005e2f..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/VarianceSplitCalculatorTest.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees;
-
-import java.util.stream.DoubleStream;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.VarianceSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.vectors.SplitInfo;
-import org.junit.Test;
-
-/**
- * Test for {@link VarianceSplitCalculator}.
- */
-public class VarianceSplitCalculatorTest {
-    /** Test calculation of region info consisting from one point. */
-    @Test
-    public void testCalculateRegionInfoSimple() {
-        double labels[] = new double[] {0.0};
-
-        assert new VarianceSplitCalculator().calculateRegionInfo(DoubleStream.of(labels), 1).impurity() == 0.0;
-    }
-
-    /** Test calculation of region info consisting from two classes. */
-    @Test
-    public void testCalculateRegionInfoTwoClasses() {
-        double labels[] = new double[] {0.0, 1.0};
-
-        assert new VarianceSplitCalculator().calculateRegionInfo(DoubleStream.of(labels), 2).impurity() == 0.25;
-    }
-
-    /** Test calculation of region info consisting from three classes. */
-    @Test
-    public void testCalculateRegionInfoThreeClasses() {
-        double labels[] = new double[] {1.0, 2.0, 3.0};
-
-        assert Math.abs(new VarianceSplitCalculator().calculateRegionInfo(DoubleStream.of(labels), 3).impurity() - 2.0 / 3) < 1E-10;
-    }
-
-    /** Test calculation of split of region consisting from one point. */
-    @Test
-    public void testSplitSimple() {
-        double labels[] = new double[] {0.0};
-        double values[] = new double[] {0.0};
-        Integer[] samples = new Integer[] {0};
-
-        VarianceSplitCalculator.VarianceData data = new VarianceSplitCalculator.VarianceData(0.0, 1, 0.0);
-
-        assert new VarianceSplitCalculator().splitRegion(samples, values, labels, 0, data) == null;
-    }
-
-    /** Test calculation of split of region consisting from two classes. */
-    @Test
-    public void testSplitTwoClassesTwoPoints() {
-        double labels[] = new double[] {0.0, 1.0};
-        double values[] = new double[] {0.0, 1.0};
-        Integer[] samples = new Integer[] {0, 1};
-
-        VarianceSplitCalculator.VarianceData data = new VarianceSplitCalculator.VarianceData(0.25, 2, 0.5);
-
-        SplitInfo<VarianceSplitCalculator.VarianceData> split = new VarianceSplitCalculator().splitRegion(samples, values, labels, 0, data);
-
-        assert split.leftData().impurity() == 0;
-        assert split.leftData().mean() == 0;
-        assert split.leftData().getSize() == 1;
-
-        assert split.rightData().impurity() == 0;
-        assert split.rightData().mean() == 1;
-        assert split.rightData().getSize() == 1;
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java b/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java
deleted file mode 100644
index 21fd692366a43..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trees/performance/ColumnDecisionTreeTrainerBenchmark.java
+++ /dev/null
@@ -1,456 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trees.performance;
-
-import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Random;
-import java.util.UUID;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.estimators.Estimators;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Tracer;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.ml.trees.BaseDecisionTreeTest;
-import org.apache.ignite.ml.trees.SplitDataGenerator;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndex;
-import org.apache.ignite.ml.trees.trainers.columnbased.BiIndexedCacheColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ContextCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.FeaturesCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.ProjectionsCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.caches.SplitCache;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.GiniSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.VarianceSplitCalculator;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.ml.util.MnistUtils;
-import org.apache.ignite.stream.StreamTransformer;
-import org.apache.ignite.testframework.junits.IgniteTestResources;
-import org.apache.log4j.Level;
-import org.junit.Assert;
-
-/**
- * Various benchmarks for hand runs.
- */
-public class ColumnDecisionTreeTrainerBenchmark extends BaseDecisionTreeTest {
-    /** Name of the property specifying path to training set images. */
-    private static final String PROP_TRAINING_IMAGES = "mnist.training.images";
-
-    /** Name of property specifying path to training set labels. */
-    private static final String PROP_TRAINING_LABELS = "mnist.training.labels";
-
-    /** Name of property specifying path to test set images. */
-    private static final String PROP_TEST_IMAGES = "mnist.test.images";
-
-    /** Name of property specifying path to test set labels. */
-    private static final String PROP_TEST_LABELS = "mnist.test.labels";
-
-    /** Function to approximate. */
-    private static final Function<Vector, Double> f1 = v -> v.get(0) * v.get(0) + 2 * Math.sin(v.get(1)) + v.get(2);
-
-    /** {@inheritDoc} */
-    @Override protected long getTestTimeout() {
-        return 6000000;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName,
-        IgniteTestResources rsrcs) throws Exception {
-        IgniteConfiguration configuration = super.getConfiguration(igniteInstanceName, rsrcs);
-        // We do not need any extra event types.
-        configuration.setIncludeEventTypes();
-        configuration.setPeerClassLoadingEnabled(false);
-
-        resetLog4j(Level.INFO, false, GridCacheProcessor.class.getPackage().getName());
-
-        return configuration;
-    }
-
-    /**
-     * This test is for manual run only.
-     * To run this test rename this method so it starts from 'test'.
-     */
-    public void tstCacheMixed() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        int ptsPerReg = 150;
-        int featCnt = 10;
-
-        HashMap<Integer, Integer> catsInfo = new HashMap<>();
-        catsInfo.put(1, 3);
-
-        Random rnd = new Random(12349L);
-
-        SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
-            featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1), rnd).
-            split(0, 1, new int[] {0, 2}).
-            split(1, 0, -10.0).
-            split(0, 0, 0.0);
-
-        testByGenStreamerLoad(ptsPerReg, catsInfo, gen, rnd);
-    }
-
-    /**
-     * Run decision tree classifier on MNIST using bi-indexed cache as a storage for dataset.
-     * To run this test rename this method so it starts from 'test'.
-     *
-     * @throws IOException In case of loading MNIST dataset errors.
-     */
-    public void tstMNISTBiIndexedCache() throws IOException {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        int ptsCnt = 40_000;
-        int featCnt = 28 * 28;
-
-        Properties props = loadMNISTProperties();
-
-        Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TRAINING_IMAGES), props.getProperty(PROP_TRAINING_LABELS), new Random(123L), ptsCnt);
-        Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TEST_IMAGES), props.getProperty(PROP_TEST_LABELS), new Random(123L), 10_000);
-
-        IgniteCache<BiIndex, Double> cache = createBiIndexedCache();
-
-        loadVectorsIntoBiIndexedCache(cache.getName(), trainingMnistStream.iterator(), featCnt + 1);
-
-        ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer =
-            new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MOST_COMMON, ignite);
-
-        X.println("Training started.");
-        long before = System.currentTimeMillis();
-        DecisionTreeModel mdl = trainer.train(new BiIndexedCacheColumnDecisionTreeTrainerInput(cache, new HashMap<>(), ptsCnt, featCnt));
-        X.println("Training finished in " + (System.currentTimeMillis() - before));
-
-        IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.errorsPercentage();
-        Double accuracy = mse.apply(mdl, testMnistStream.map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
-        X.println("Errors percentage: " + accuracy);
-
-        Assert.assertEquals(0, SplitCache.getOrCreate(ignite).size());
-        Assert.assertEquals(0, FeaturesCache.getOrCreate(ignite).size());
-        Assert.assertEquals(0, ContextCache.getOrCreate(ignite).size());
-        Assert.assertEquals(0, ProjectionsCache.getOrCreate(ignite).size());
-    }
-
-    /**
-     * Run decision tree classifier on MNIST using sparse distributed matrix as a storage for dataset.
-     * To run this test rename this method so it starts from 'test'.
-     *
-     * @throws IOException In case of loading MNIST dataset errors.
-     */
-    public void tstMNISTSparseDistributedMatrix() throws IOException {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        int ptsCnt = 30_000;
-        int featCnt = 28 * 28;
-
-        Properties props = loadMNISTProperties();
-
-        Stream<DenseLocalOnHeapVector> trainingMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TRAINING_IMAGES), props.getProperty(PROP_TRAINING_LABELS), new Random(123L), ptsCnt);
-        Stream<DenseLocalOnHeapVector> testMnistStream = MnistUtils.mnistAsStream(props.getProperty(PROP_TEST_IMAGES), props.getProperty(PROP_TEST_LABELS), new Random(123L), 10_000);
-
-        SparseDistributedMatrix m = new SparseDistributedMatrix(ptsCnt, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
-        SparseDistributedMatrixStorage sto = (SparseDistributedMatrixStorage)m.getStorage();
-
-        loadVectorsIntoSparseDistributedMatrixCache(sto.cache().getName(), sto.getUUID(), trainingMnistStream.iterator(), featCnt + 1);
-
-        ColumnDecisionTreeTrainer<GiniSplitCalculator.GiniData> trainer =
-            new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.GINI.apply(ignite), RegionCalculators.GINI, RegionCalculators.MOST_COMMON, ignite);
-
-        X.println("Training started");
-        long before = System.currentTimeMillis();
-        DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, new HashMap<>()));
-        X.println("Training finished in " + (System.currentTimeMillis() - before));
-
-        IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.errorsPercentage();
-        Double accuracy = mse.apply(mdl, testMnistStream.map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
-        X.println("Errors percentage: " + accuracy);
-
-        Assert.assertEquals(0, SplitCache.getOrCreate(ignite).size());
-        Assert.assertEquals(0, FeaturesCache.getOrCreate(ignite).size());
-        Assert.assertEquals(0, ContextCache.getOrCreate(ignite).size());
-        Assert.assertEquals(0, ProjectionsCache.getOrCreate(ignite).size());
-    }
-
-    /** Load properties for MNIST tests. */
-    private static Properties loadMNISTProperties() throws IOException {
-        Properties res = new Properties();
-
-        InputStream is = ColumnDecisionTreeTrainerBenchmark.class.getClassLoader().getResourceAsStream("manualrun/trees/columntrees.manualrun.properties");
-
-        res.load(is);
-
-        return res;
-    }
-
-    /** */
-    private void testByGenStreamerLoad(int ptsPerReg, HashMap<Integer, Integer> catsInfo,
-        SplitDataGenerator<DenseLocalOnHeapVector> gen, Random rnd) {
-
-        List<IgniteBiTuple<Integer, DenseLocalOnHeapVector>> lst = gen.
-            points(ptsPerReg, (i, rn) -> i).
-            collect(Collectors.toList());
-
-        int featCnt = gen.featuresCnt();
-
-        Collections.shuffle(lst, rnd);
-
-        int numRegs = gen.regsCount();
-
-        SparseDistributedMatrix m = new SparseDistributedMatrix(numRegs * ptsPerReg, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
-        IgniteFunction<DoubleStream, Double> regCalc = s -> s.average().orElse(0.0);
-
-        Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
-
-        SparseDistributedMatrixStorage sto = (SparseDistributedMatrixStorage)m.getStorage();
-        long before = System.currentTimeMillis();
-        X.println("Batch loading started...");
-        loadVectorsIntoSparseDistributedMatrixCache(sto.cache().getName(), sto.getUUID(), gen.
-            points(ptsPerReg, (i, rn) -> i).map(IgniteBiTuple::get2).iterator(), featCnt + 1);
-        X.println("Batch loading took " + (System.currentTimeMillis() - before) + " ms.");
-
-        for (IgniteBiTuple<Integer, DenseLocalOnHeapVector> bt : lst) {
-            byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
-            byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
-        }
-
-        ColumnDecisionTreeTrainer<VarianceSplitCalculator.VarianceData> trainer =
-            new ColumnDecisionTreeTrainer<>(2, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, regCalc, ignite);
-
-        before = System.currentTimeMillis();
-        DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catsInfo));
-
-        X.println("Training took: " + (System.currentTimeMillis() - before) + " ms.");
-
-        byRegion.keySet().forEach(k -> {
-            LabeledVectorDouble sp = byRegion.get(k).get(0);
-            Tracer.showAscii(sp.features());
-            X.println("Predicted value and label [pred=" + mdl.apply(sp.features()) + ", label=" + sp.doubleLabel() + "]");
-            assert mdl.apply(sp.features()) == sp.doubleLabel();
-        });
-    }
-
-    /**
-     * Test decision tree regression.
-     * To run this test rename this method so it starts from 'test'.
-     */
-    public void tstF1() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        int ptsCnt = 10000;
-        Map<Integer, double[]> ranges = new HashMap<>();
-
-        ranges.put(0, new double[] {-100.0, 100.0});
-        ranges.put(1, new double[] {-100.0, 100.0});
-        ranges.put(2, new double[] {-100.0, 100.0});
-
-        int featCnt = 100;
-        double[] defRng = {-1.0, 1.0};
-
-        Vector[] trainVectors = vecsFromRanges(ranges, featCnt, defRng, new Random(123L), ptsCnt, f1);
-
-        SparseDistributedMatrix m = new SparseDistributedMatrix(ptsCnt, featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
-        SparseDistributedMatrixStorage sto = (SparseDistributedMatrixStorage)m.getStorage();
-
-        loadVectorsIntoSparseDistributedMatrixCache(sto.cache().getName(), sto.getUUID(), Arrays.stream(trainVectors).iterator(), featCnt + 1);
-
-        IgniteFunction<DoubleStream, Double> regCalc = s -> s.average().orElse(0.0);
-
-        ColumnDecisionTreeTrainer<VarianceSplitCalculator.VarianceData> trainer =
-            new ColumnDecisionTreeTrainer<>(10, ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, regCalc, ignite);
-
-        X.println("Training started.");
-        long before = System.currentTimeMillis();
-        DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, new HashMap<>()));
-        X.println("Training finished in: " + (System.currentTimeMillis() - before) + " ms.");
-
-        Vector[] testVectors = vecsFromRanges(ranges, featCnt, defRng, new Random(123L), 20, f1);
-
-        IgniteTriFunction<Model<Vector, Double>, Stream<IgniteBiTuple<Vector, Double>>, Function<Double, Double>, Double> mse = Estimators.MSE();
-        Double accuracy = mse.apply(mdl, Arrays.stream(testVectors).map(v -> new IgniteBiTuple<>(v.viewPart(0, featCnt), v.getX(featCnt))), Function.identity());
-        X.println("MSE: " + accuracy);
-    }
-
-    /**
-     * Load vectors into sparse distributed matrix.
-     *
-     * @param cacheName Name of cache where matrix is stored.
-     * @param uuid UUID of matrix.
-     * @param iter Iterator over vectors.
-     * @param vectorSize size of vectors.
-     */
-    private void loadVectorsIntoSparseDistributedMatrixCache(String cacheName, UUID uuid,
-        Iterator<? extends org.apache.ignite.ml.math.Vector> iter, int vectorSize) {
-        try (IgniteDataStreamer<SparseMatrixKey, Map<Integer, Double>> streamer =
-                 Ignition.localIgnite().dataStreamer(cacheName)) {
-            int sampleIdx = 0;
-            streamer.allowOverwrite(true);
-
-            streamer.receiver(StreamTransformer.from((e, arg) -> {
-                Map<Integer, Double> val = e.getValue();
-
-                if (val == null)
-                    val = new Int2DoubleOpenHashMap();
-
-                val.putAll((Map<Integer, Double>)arg[0]);
-
-                e.setValue(val);
-
-                return null;
-            }));
-
-            // Feature index -> (sample index -> value)
-            Map<Integer, Map<Integer, Double>> batch = new HashMap<>();
-            IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
-            int batchSize = 1000;
-
-            while (iter.hasNext()) {
-                org.apache.ignite.ml.math.Vector next = iter.next();
-
-                for (int i = 0; i < vectorSize; i++)
-                    batch.get(i).put(sampleIdx, next.getX(i));
-
-                X.println("Sample index: " + sampleIdx);
-                if (sampleIdx % batchSize == 0) {
-                    batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
-                    IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
-                }
-                sampleIdx++;
-            }
-            if (sampleIdx % batchSize != 0) {
-                batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
-                IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
-            }
-        }
-    }
-
-    /**
-     * Load vectors into bi-indexed cache.
-     *
-     * @param cacheName Name of cache.
-     * @param iter Iterator over vectors.
-     * @param vectorSize size of vectors.
-     */
-    private void loadVectorsIntoBiIndexedCache(String cacheName,
-        Iterator<? extends org.apache.ignite.ml.math.Vector> iter, int vectorSize) {
-        try (IgniteDataStreamer<BiIndex, Double> streamer =
-                 Ignition.localIgnite().dataStreamer(cacheName)) {
-            int sampleIdx = 0;
-
-            streamer.perNodeBufferSize(10000);
-
-            while (iter.hasNext()) {
-                org.apache.ignite.ml.math.Vector next = iter.next();
-
-                for (int i = 0; i < vectorSize; i++)
-                    streamer.addData(new BiIndex(sampleIdx, i), next.getX(i));
-
-                sampleIdx++;
-
-                if (sampleIdx % 1000 == 0)
-                    System.out.println("Loaded: " + sampleIdx + " vectors.");
-            }
-        }
-    }
-
-    /**
-     * Create bi-indexed cache for tests.
-     *
-     * @return Bi-indexed cache.
-     */
-    private IgniteCache<BiIndex, Double> createBiIndexedCache() {
-        CacheConfiguration<BiIndex, Double> cfg = new CacheConfiguration<>();
-
-        // Write to primary.
-        cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
-        // Atomic transactions only.
-        cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
-        // No eviction.
-        cfg.setEvictionPolicy(null);
-
-        // No copying of values.
-        cfg.setCopyOnRead(false);
-
-        // Cache is partitioned.
-        cfg.setCacheMode(CacheMode.PARTITIONED);
-
-        cfg.setBackups(0);
-
-        cfg.setName("TMP_BI_INDEXED_CACHE");
-
-        return Ignition.localIgnite().getOrCreateCache(cfg);
-    }
-
-    /** */
-    private Vector[] vecsFromRanges(Map<Integer, double[]> ranges, int featCnt, double[] defRng, Random rnd, int ptsCnt,
-        Function<Vector, Double> f) {
-        int vs = featCnt + 1;
-        DenseLocalOnHeapVector[] res = new DenseLocalOnHeapVector[ptsCnt];
-        for (int pt = 0; pt < ptsCnt; pt++) {
-            DenseLocalOnHeapVector v = new DenseLocalOnHeapVector(vs);
-            for (int i = 0; i < featCnt; i++) {
-                double[] range = ranges.getOrDefault(i, defRng);
-                double from = range[0];
-                double to = range[1];
-                double rng = to - from;
-
-                v.setX(i, rnd.nextDouble() * rng);
-            }
-            v.setX(featCnt, f.apply(v));
-            res[pt] = v;
-        }
-
-        return res;
-    }
-}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
deleted file mode 100644
index f8a7c0834d825..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeGiniBenchmark.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteColumnDecisionTreeGiniBenchmark extends IgniteAbstractBenchmark {
-    /** */
-    @IgniteInstanceResource
-    private Ignite ignite;
-
-    /** {@inheritDoc} */
-    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
-        // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
-        // because we create ignite cache internally.
-        IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-            this.getClass().getSimpleName(), new Runnable() {
-            /** {@inheritDoc} */
-            @Override public void run() {
-                // IMPL NOTE originally taken from ColumnDecisionTreeTrainerTest#testCacheMixedGini
-                int totalPts = 1 << 10;
-                int featCnt = 2;
-
-                HashMap<Integer, Integer> catsInfo = new HashMap<>();
-                catsInfo.put(1, 3);
-
-                SplitDataGenerator<DenseLocalOnHeapVector> gen = new SplitDataGenerator<>(
-                    featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1)).
-                    split(0, 1, new int[] {0, 2}).
-                    split(1, 0, -10.0);
-
-                gen.testByGen(totalPts, ContinuousSplitCalculators.GINI.apply(ignite),
-                    RegionCalculators.GINI, RegionCalculators.MEAN, ignite);
-            }
-        });
-
-        igniteThread.start();
-
-        igniteThread.join();
-
-        return true;
-    }
-}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
deleted file mode 100644
index f9d417f47051e..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/IgniteColumnDecisionTreeVarianceBenchmark.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.trees.trainers.columnbased.contsplitcalcs.ContinuousSplitCalculators;
-import org.apache.ignite.ml.trees.trainers.columnbased.regcalcs.RegionCalculators;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteColumnDecisionTreeVarianceBenchmark extends IgniteAbstractBenchmark {
-    /** */
-    @IgniteInstanceResource
-    private Ignite ignite;
-
-    /** {@inheritDoc} */
-    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
-        // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
-        // because we create ignite cache internally.
-        IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-            this.getClass().getSimpleName(), new Runnable() {
-            /** {@inheritDoc} */
-            @Override public void run() {
-                // IMPL NOTE originally taken from ColumnDecisionTreeTrainerTest#testCacheMixed
-                int totalPts = 1 << 10;
-                int featCnt = 2;
-
-                HashMap<Integer, Integer> catsInfo = new HashMap<>();
-                catsInfo.put(1, 3);
-
-                SplitDataGenerator<DenseLocalOnHeapVector> gen
-                    = new SplitDataGenerator<>(
-                    featCnt, catsInfo, () -> new DenseLocalOnHeapVector(featCnt + 1)).
-                    split(0, 1, new int[] {0, 2}).
-                    split(1, 0, -10.0);
-
-                gen.testByGen(totalPts,
-                    ContinuousSplitCalculators.VARIANCE, RegionCalculators.VARIANCE, RegionCalculators.MEAN, ignite);
-            }
-        });
-
-        igniteThread.start();
-
-        igniteThread.join();
-
-        return true;
-    }
-}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
deleted file mode 100644
index f9117f4bb38be..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/SplitDataGenerator.java
+++ /dev/null
@@ -1,426 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.trees;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.function.BiFunction;
-import java.util.function.Function;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.DoubleStream;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.structures.LabeledVectorDouble;
-import org.apache.ignite.ml.trees.ContinuousRegionInfo;
-import org.apache.ignite.ml.trees.ContinuousSplitCalculator;
-import org.apache.ignite.ml.trees.models.DecisionTreeModel;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainer;
-import org.apache.ignite.ml.trees.trainers.columnbased.ColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.trees.trainers.columnbased.MatrixColumnDecisionTreeTrainerInput;
-import org.apache.ignite.ml.util.Utils;
-
-/** */
-class SplitDataGenerator<V extends Vector> {
-    /** */
-    private static final Random rnd = new Random(12349L);
-
-    /** */
-    private static final double DELTA = 100.0;
-
-    /** Map of the form of (is categorical -> list of region indexes). */
-    private final Map<Boolean, List<Integer>> di;
-
-    /** List of regions. */
-    private final List<Region> regs;
-
-    /** Data of bounds of regions. */
-    private final Map<Integer, IgniteBiTuple<Double, Double>> boundsData;
-
-    /** */
-    private final Map<Integer, Integer> catFeaturesInfo;
-
-    /** Supplier of vectors. */
-    private final Supplier<V> supplier;
-
-    /** Features count. */
-    private final int featCnt;
-
-    /**
-     * Create SplitDataGenerator.
-     *
-     * @param featCnt Features count.
-     * @param catFeaturesInfo Information about categorical features in form of map (feature index -> categories
-     * count).
-     * @param supplier Supplier of vectors.
-     */
-    SplitDataGenerator(int featCnt, Map<Integer, Integer> catFeaturesInfo, Supplier<V> supplier) {
-        regs = new LinkedList<>();
-        boundsData = new HashMap<>();
-        this.supplier = supplier;
-        this.featCnt = featCnt;
-        this.catFeaturesInfo = catFeaturesInfo;
-
-        // Divide indexes into indexes of categorical coordinates and indexes of continuous coordinates.
-        di = IntStream.range(0, featCnt).
-            boxed().
-            collect(Collectors.partitioningBy(catFeaturesInfo::containsKey));
-
-        // Categorical coordinates info.
-        Map<Integer, CatCoordInfo> catCoords = new HashMap<>();
-        di.get(true).forEach(i -> {
-            BitSet bs = new BitSet();
-            bs.set(0, catFeaturesInfo.get(i));
-            catCoords.put(i, new CatCoordInfo(bs));
-        });
-
-        // Continuous coordinates info.
-        Map<Integer, ContCoordInfo> contCoords = new HashMap<>();
-        di.get(false).forEach(i -> {
-            contCoords.put(i, new ContCoordInfo());
-            boundsData.put(i, new IgniteBiTuple<>(-1.0, 1.0));
-        });
-
-        Region firstReg = new Region(catCoords, contCoords, 0);
-        regs.add(firstReg);
-    }
-
-    /** */
-    <D extends ContinuousRegionInfo> void testByGen(int totalPts,
-        IgniteFunction<ColumnDecisionTreeTrainerInput, ? extends ContinuousSplitCalculator<D>> calc,
-        IgniteFunction<ColumnDecisionTreeTrainerInput, IgniteFunction<DoubleStream, Double>> catImpCalc,
-        IgniteFunction<DoubleStream, Double> regCalc, Ignite ignite) {
-
-        List<IgniteBiTuple<Integer, V>> lst = points(totalPts, (i, rn) -> i).collect(Collectors.toList());
-
-        Collections.shuffle(lst, rnd);
-
-        SparseDistributedMatrix m = new SparseDistributedMatrix(totalPts,
-            featCnt + 1, StorageConstants.COLUMN_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
-        Map<Integer, List<LabeledVectorDouble>> byRegion = new HashMap<>();
-
-        int i = 0;
-        for (IgniteBiTuple<Integer, V> bt : lst) {
-            byRegion.putIfAbsent(bt.get1(), new LinkedList<>());
-            byRegion.get(bt.get1()).add(asLabeledVector(bt.get2().getStorage().data()));
-            m.setRow(i, bt.get2().getStorage().data());
-            i++;
-        }
-
-        ColumnDecisionTreeTrainer<D> trainer =
-            new ColumnDecisionTreeTrainer<>(3, calc, catImpCalc, regCalc, ignite);
-
-        DecisionTreeModel mdl = trainer.train(new MatrixColumnDecisionTreeTrainerInput(m, catFeaturesInfo));
-
-        byRegion.keySet().forEach(k -> mdl.apply(byRegion.get(k).get(0).features()));
-    }
-
-    /**
-     * Split region by continuous coordinate using given threshold.
-     *
-     * @param regIdx Region index.
-     * @param coordIdx Coordinate index.
-     * @param threshold Threshold.
-     * @return {@code this}.
-     */
-    SplitDataGenerator<V> split(int regIdx, int coordIdx, double threshold) {
-        Region regToSplit = regs.get(regIdx);
-        ContCoordInfo cci = regToSplit.contCoords.get(coordIdx);
-
-        double left = cci.left;
-        double right = cci.right;
-
-        if (threshold < left || threshold > right)
-            throw new MathIllegalArgumentException("Threshold is out of region bounds.");
-
-        regToSplit.incTwoPow();
-
-        Region newReg = Utils.copy(regToSplit);
-        newReg.contCoords.get(coordIdx).left = threshold;
-
-        regs.add(regIdx + 1, newReg);
-        cci.right = threshold;
-
-        IgniteBiTuple<Double, Double> bounds = boundsData.get(coordIdx);
-        double min = bounds.get1();
-        double max = bounds.get2();
-        boundsData.put(coordIdx, new IgniteBiTuple<>(Math.min(threshold, min), Math.max(max, threshold)));
-
-        return this;
-    }
-
-    /**
-     * Split region by categorical coordinate.
-     *
-     * @param regIdx Region index.
-     * @param coordIdx Coordinate index.
-     * @param cats Categories allowed for the left sub region.
-     * @return {@code this}.
-     */
-    SplitDataGenerator<V> split(int regIdx, int coordIdx, int[] cats) {
-        BitSet subset = new BitSet();
-        Arrays.stream(cats).forEach(subset::set);
-        Region regToSplit = regs.get(regIdx);
-        CatCoordInfo cci = regToSplit.catCoords.get(coordIdx);
-
-        BitSet ssc = (BitSet)subset.clone();
-        BitSet set = cci.bs;
-        ssc.and(set);
-        if (ssc.length() != subset.length())
-            throw new MathIllegalArgumentException("Splitter set is not a subset of a parent subset.");
-
-        ssc.xor(set);
-        set.and(subset);
-
-        regToSplit.incTwoPow();
-        Region newReg = Utils.copy(regToSplit);
-        newReg.catCoords.put(coordIdx, new CatCoordInfo(ssc));
-
-        regs.add(regIdx + 1, newReg);
-
-        return this;
-    }
-
-    /**
-     * Get stream of points generated by this generator.
-     *
-     * @param ptsCnt Points count.
-     */
-    private Stream<IgniteBiTuple<Integer, V>> points(int ptsCnt, BiFunction<Double, Random, Double> f) {
-        return IntStream.range(0, regs.size()).
-            boxed().
-            map(i -> regs.get(i).generatePoints(ptsCnt, f.apply((double)i, rnd), boundsData, di, supplier, rnd)
-                .map(v -> new IgniteBiTuple<>(i, v))).flatMap(Function.identity());
-    }
-
-    /**
-     * Convert double array to  {@link LabeledVectorDouble}
-     *
-     * @param arr Array for conversion.
-     * @return LabeledVectorDouble.
-     */
-    private static LabeledVectorDouble<DenseLocalOnHeapVector> asLabeledVector(double arr[]) {
-        return new LabeledVectorDouble<>(new DenseLocalOnHeapVector(
-            Arrays.copyOf(arr, arr.length - 1)), arr[arr.length - 1]);
-    }
-
-    /**
-     * Categorical coordinate info.
-     */
-    private static class CatCoordInfo implements Serializable {
-        /**
-         * Defines categories which are included in this region
-         */
-        private final BitSet bs;
-
-        /**
-         * Construct CatCoordInfo.
-         *
-         * @param bs Bitset.
-         */
-        CatCoordInfo(BitSet bs) {
-            this.bs = bs;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return "CatCoordInfo [" +
-                "bs=" + bs +
-                ']';
-        }
-    }
-
-    /**
-     * Continuous coordinate info.
-     */
-    private static class ContCoordInfo implements Serializable {
-        /**
-         * Left (min) bound of region.
-         */
-        private double left;
-
-        /**
-         * Right (max) bound of region.
-         */
-        private double right;
-
-        /**
-         * Construct ContCoordInfo.
-         */
-        ContCoordInfo() {
-            left = Double.NEGATIVE_INFINITY;
-            right = Double.POSITIVE_INFINITY;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return "ContCoordInfo [" +
-                "left=" + left +
-                ", right=" + right +
-                ']';
-        }
-    }
-
-    /**
-     * Class representing information about region.
-     */
-    private static class Region implements Serializable {
-        /**
-         * Information about categorical coordinates restrictions of this region in form of
-         * (coordinate index -> restriction)
-         */
-        private final Map<Integer, CatCoordInfo> catCoords;
-
-        /**
-         * Information about continuous coordinates restrictions of this region in form of
-         * (coordinate index -> restriction)
-         */
-        private final Map<Integer, ContCoordInfo> contCoords;
-
-        /**
-         * Region should contain {@code 1/2^twoPow * totalPoints} points.
-         */
-        private int twoPow;
-
-        /**
-         * Construct region by information about restrictions on coordinates (features) values.
-         *
-         * @param catCoords Restrictions on categorical coordinates.
-         * @param contCoords Restrictions on continuous coordinates
-         * @param twoPow Region should contain {@code 1/2^twoPow * totalPoints} points.
-         */
-        Region(Map<Integer, CatCoordInfo> catCoords, Map<Integer, ContCoordInfo> contCoords, int twoPow) {
-            this.catCoords = catCoords;
-            this.contCoords = contCoords;
-            this.twoPow = twoPow;
-        }
-
-        /** */
-        int divideBy() {
-            return 1 << twoPow;
-        }
-
-        /** */
-        void incTwoPow() {
-            twoPow++;
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return "Region [" +
-                "catCoords=" + catCoords +
-                ", contCoords=" + contCoords +
-                ", twoPow=" + twoPow +
-                ']';
-        }
-
-        /**
-         * Generate continuous coordinate for this region.
-         *
-         * @param coordIdx Coordinate index.
-         * @param boundsData Data with bounds
-         * @param rnd Random numbers generator.
-         * @return Categorical coordinate value.
-         */
-        double generateContCoord(int coordIdx, Map<Integer, IgniteBiTuple<Double, Double>> boundsData,
-            Random rnd) {
-            ContCoordInfo cci = contCoords.get(coordIdx);
-            double left = cci.left;
-            double right = cci.right;
-
-            if (left == Double.NEGATIVE_INFINITY)
-                left = boundsData.get(coordIdx).get1() - DELTA;
-
-            if (right == Double.POSITIVE_INFINITY)
-                right = boundsData.get(coordIdx).get2() + DELTA;
-
-            double size = right - left;
-
-            return left + rnd.nextDouble() * size;
-        }
-
-        /**
-         * Generate categorical coordinate value for this region.
-         *
-         * @param coordIdx Coordinate index.
-         * @param rnd Random numbers generator.
-         * @return Categorical coordinate value.
-         */
-        double generateCatCoord(int coordIdx, Random rnd) {
-            // Pick random bit.
-            BitSet bs = catCoords.get(coordIdx).bs;
-            int j = rnd.nextInt(bs.length());
-
-            int i = 0;
-            int bn = 0;
-            int bnp = 0;
-
-            while ((bn = bs.nextSetBit(bn)) != -1 && i <= j) {
-                i++;
-                bnp = bn;
-                bn++;
-            }
-
-            return bnp;
-        }
-
-        /**
-         * Generate points for this region.
-         *
-         * @param ptsCnt Count of points to generate.
-         * @param val Label for all points in this region.
-         * @param boundsData Data about bounds of continuous coordinates.
-         * @param catCont Data about which categories can be in this region in the form (coordinate index -> list of
-         * categories indexes).
-         * @param s Vectors supplier.
-         * @param rnd Random numbers generator.
-         * @param <V> Type of vectors.
-         * @return Stream of generated points for this region.
-         */
-        <V extends Vector> Stream<V> generatePoints(int ptsCnt, double val,
-            Map<Integer, IgniteBiTuple<Double, Double>> boundsData, Map<Boolean, List<Integer>> catCont,
-            Supplier<V> s,
-            Random rnd) {
-            return IntStream.range(0, ptsCnt / divideBy()).mapToObj(i -> {
-                V v = s.get();
-                int coordsCnt = v.size();
-                catCont.get(false).forEach(ci -> v.setX(ci, generateContCoord(ci, boundsData, rnd)));
-                catCont.get(true).forEach(ci -> v.setX(ci, generateCatCoord(ci, rnd)));
-
-                v.setX(coordsCnt - 1, val);
-                return v;
-            });
-        }
-    }
-}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java
deleted file mode 100644
index fc379a607bcf9..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/trees/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * ML Grid decision tree benchmarks.
- */
-package org.apache.ignite.yardstick.ml.trees;
\ No newline at end of file

From 1ea17c821f51d8725290021599a60cbbdfd2ee25 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Wed, 11 Apr 2018 12:28:40 +0300
Subject: [PATCH 021/543] IGNITE-8216 Fixed javadoc for release build

---
 parent/pom.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/parent/pom.xml b/parent/pom.xml
index 16a9395c874d8..3decc16612374 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -403,6 +403,10 @@
                                 <title>SPI: Event Storage</title>
                                 <packages>org.apache.ignite.spi.eventstorage*</packages>
                             </group>
+                            <group>
+                                <title>Communication Failure Detection</title>
+                                <packages>org.apache.ignite.failure</packages>
+                            </group>
                             <group>
                                 <title>Segmentation Detection</title>
                                 <packages>org.apache.ignite.plugin.segmentation</packages>

From 6bc937575d5dfbe1d2c05865c8df0cabc6ace90c Mon Sep 17 00:00:00 2001
From: zaleslaw <zaleslaw.sin@gmail.com>
Date: Wed, 11 Apr 2018 12:31:48 +0300
Subject: [PATCH 022/543] IGNITE-7830: Knn Lin Reg with new datasets

this closes #3583

(cherry picked from commit a4653b7)
---
 .../org/apache/ignite/ml/knn/KNNUtils.java    |  59 ++++++++
 .../KNNClassificationTrainer.java             |  23 +--
 .../ml/knn/regression/KNNRegressionModel.java |  87 +++++++++++
 .../knn/regression/KNNRegressionTrainer.java  |  40 +++++
 .../ml/knn/regression/package-info.java       |  22 +++
 .../ignite/ml/knn/KNNRegressionTest.java      | 143 ++++++++++++++++++
 .../apache/ignite/ml/knn/KNNTestSuite.java    |   1 +
 7 files changed, 354 insertions(+), 21 deletions(-)
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/package-info.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java

diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java
new file mode 100644
index 0000000000000..88fa70f8c9ea2
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.knn;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
+import org.apache.ignite.ml.knn.partitions.KNNPartitionContext;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.structures.LabeledDataset;
+import org.apache.ignite.ml.structures.LabeledVector;
+import org.apache.ignite.ml.structures.partition.LabeledDatasetPartitionDataBuilderOnHeap;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Helper class for KNNRegression.
+ */
+public class KNNUtils {
+    /**
+     * Builds dataset.
+     *
+     * @param datasetBuilder Dataset builder.
+     * @param featureExtractor Feature extractor.
+     * @param lbExtractor Label extractor.
+     * @return Dataset.
+     */
+    @Nullable public static <K, V> Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> buildDataset(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+        PartitionDataBuilder<K, V, KNNPartitionContext, LabeledDataset<Double, LabeledVector>> partDataBuilder
+            = new LabeledDatasetPartitionDataBuilderOnHeap<>(
+            featureExtractor,
+            lbExtractor
+        );
+
+        Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> dataset = null;
+
+        if (datasetBuilder != null) {
+            dataset = datasetBuilder.build(
+                (upstream, upstreamSize) -> new KNNPartitionContext(),
+                partDataBuilder
+            );
+        }
+        return dataset;
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
index 357047f81313f..c0c8e6593d3e4 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationTrainer.java
@@ -17,14 +17,9 @@
 
 package org.apache.ignite.ml.knn.classification;
 
-import org.apache.ignite.ml.dataset.Dataset;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
-import org.apache.ignite.ml.dataset.PartitionDataBuilder;
-import org.apache.ignite.ml.knn.partitions.KNNPartitionContext;
+import org.apache.ignite.ml.knn.KNNUtils;
 import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.structures.LabeledDataset;
-import org.apache.ignite.ml.structures.partition.LabeledDatasetPartitionDataBuilderOnHeap;
-import org.apache.ignite.ml.structures.LabeledVector;
 import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
 
 /**
@@ -41,20 +36,6 @@ public class KNNClassificationTrainer implements SingleLabelDatasetTrainer<KNNCl
      */
     @Override public <K, V> KNNClassificationModel fit(DatasetBuilder<K, V> datasetBuilder,
         IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
-        PartitionDataBuilder<K, V, KNNPartitionContext, LabeledDataset<Double, LabeledVector>> partDataBuilder
-            = new LabeledDatasetPartitionDataBuilderOnHeap<>(
-            featureExtractor,
-            lbExtractor
-        );
-
-        Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> dataset = null;
-
-        if (datasetBuilder != null) {
-            dataset = datasetBuilder.build(
-                (upstream, upstreamSize) -> new KNNPartitionContext(),
-                partDataBuilder
-            );
-        }
-        return new KNNClassificationModel<>(dataset);
+        return new KNNClassificationModel<>(KNNUtils.buildDataset(datasetBuilder, featureExtractor, lbExtractor));
     }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java
new file mode 100644
index 0000000000000..cabc1438e1d77
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.ml.knn.regression;
+
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
+import org.apache.ignite.ml.knn.partitions.KNNPartitionContext;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
+import org.apache.ignite.ml.structures.LabeledDataset;
+import org.apache.ignite.ml.structures.LabeledVector;
+
+import java.util.List;
+
+/**
+ * This class provides kNN Multiple Linear Regression or Locally [weighted] regression (Simple and Weighted versions).
+ *
+ * <p> This is an instance-based learning method. </p>
+ *
+ * <ul>
+ *     <li>Local means using nearby points (i.e. a nearest neighbors approach).</li>
+ *     <li>Weighted means we value points based upon how far away they are.</li>
+ *     <li>Regression means approximating a function.</li>
+ * </ul>
+ */
+public class KNNRegressionModel<K,V> extends KNNClassificationModel<K,V> {
+    /**
+     * Builds the model via prepared dataset.
+     * @param dataset Specially prepared object to run algorithm over it.
+     */
+    public KNNRegressionModel(Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> dataset) {
+        super(dataset);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Double apply(Vector v) {
+        List<LabeledVector> neighbors = findKNearestNeighbors(v);
+
+        return predictYBasedOn(neighbors, v);
+    }
+
+    /** */
+    private double predictYBasedOn(List<LabeledVector> neighbors, Vector v) {
+        switch (stgy) {
+            case SIMPLE:
+                return simpleRegression(neighbors);
+            case WEIGHTED:
+                return weightedRegression(neighbors, v);
+            default:
+                throw new UnsupportedOperationException("Strategy " + stgy.name() + " is not supported");
+        }
+    }
+
+    /** */
+    private double weightedRegression(List<LabeledVector> neighbors, Vector v) {
+        double sum = 0.0;
+        double div = 0.0;
+        for (LabeledVector<Vector, Double> neighbor : neighbors) {
+            double distance = distanceMeasure.compute(v, neighbor.features());
+            sum += neighbor.label() * distance;
+            div += distance;
+        }
+        return sum / div;
+    }
+
+    /** */
+    private double simpleRegression(List<LabeledVector> neighbors) {
+        double sum = 0.0;
+        for (LabeledVector<Vector, Double> neighbor : neighbors)
+            sum += neighbor.label();
+        return sum / (double)k;
+    }
+}
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java
new file mode 100644
index 0000000000000..2d13cd50417df
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionTrainer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.knn.regression;
+
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.knn.KNNUtils;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+
+/**
+ * kNN algorithm trainer to solve regression task.
+ */
+public class KNNRegressionTrainer{
+    /**
+     * Trains model based on the specified data.
+     *
+     * @param datasetBuilder Dataset builder.
+     * @param featureExtractor Feature extractor.
+     * @param lbExtractor Label extractor.
+     * @return Model.
+     */
+    public <K, V> KNNRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
+        IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+        return new KNNRegressionModel<>(KNNUtils.buildDataset(datasetBuilder, featureExtractor, lbExtractor));
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/package-info.java
new file mode 100644
index 0000000000000..82e71929e1c9f
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * Contains helper classes for kNN regression algorithms.
+ */
+package org.apache.ignite.ml.knn.regression;
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
new file mode 100644
index 0000000000000..66dbca9ff038e
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.knn;
+
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.knn.classification.KNNStrategy;
+import org.apache.ignite.ml.knn.regression.KNNRegressionModel;
+import org.apache.ignite.ml.knn.regression.KNNRegressionTrainer;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.distances.EuclideanDistance;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.junit.Assert;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Tests for {@link KNNRegressionTrainer}.
+ */
+public class KNNRegressionTest extends BaseKNNTest {
+    /** */
+    private double[] y;
+
+    /** */
+    private double[][] x;
+
+    /** */
+    public void testSimpleRegressionWithOneNeighbour() {
+        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+        Map<Integer, double[]> data = new HashMap<>();
+        data.put(0, new double[] {11.0, 0, 0, 0, 0, 0});
+        data.put(1, new double[] {12.0, 2.0, 0, 0, 0, 0});
+        data.put(2, new double[] {13.0, 0, 3.0, 0, 0, 0});
+        data.put(3, new double[] {14.0, 0, 0, 4.0, 0, 0});
+        data.put(4, new double[] {15.0, 0, 0, 0, 5.0, 0});
+        data.put(5, new double[] {16.0, 0, 0, 0, 0, 6.0});
+
+        KNNRegressionTrainer trainer = new KNNRegressionTrainer();
+
+        KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
+            new LocalDatasetBuilder<>(data, 2),
+            (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+            (k, v) -> v[0]
+        ).withK(1)
+            .withDistanceMeasure(new EuclideanDistance())
+            .withStrategy(KNNStrategy.SIMPLE);
+
+        Vector vector = new DenseLocalOnHeapVector(new double[] {0, 0, 0, 5.0, 0.0});
+        System.out.println(knnMdl.apply(vector));
+        Assert.assertEquals(15, knnMdl.apply(vector), 1E-12);
+    }
+
+    /** */
+    public void testLongly() {
+
+        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+        Map<Integer, double[]> data = new HashMap<>();
+        data.put(0, new double[] {60323, 83.0, 234289, 2356, 1590, 107608, 1947});
+        data.put(1, new double[] {61122, 88.5, 259426, 2325, 1456, 108632, 1948});
+        data.put(2, new double[] {60171, 88.2, 258054, 3682, 1616, 109773, 1949});
+        data.put(3, new double[] {61187, 89.5, 284599, 3351, 1650, 110929, 1950});
+        data.put(4, new double[] {63221, 96.2, 328975, 2099, 3099, 112075, 1951});
+        data.put(5, new double[] {63639, 98.1, 346999, 1932, 3594, 113270, 1952});
+        data.put(6, new double[] {64989, 99.0, 365385, 1870, 3547, 115094, 1953});
+        data.put(7, new double[] {63761, 100.0, 363112, 3578, 3350, 116219, 1954});
+        data.put(8, new double[] {66019, 101.2, 397469, 2904, 3048, 117388, 1955});
+        data.put(9, new double[] {68169, 108.4, 442769, 2936, 2798, 120445, 1957});
+        data.put(10, new double[] {66513, 110.8, 444546, 4681, 2637, 121950, 1958});
+        data.put(11, new double[] {68655, 112.6, 482704, 3813, 2552, 123366, 1959});
+        data.put(12, new double[] {69564, 114.2, 502601, 3931, 2514, 125368, 1960});
+        data.put(13, new double[] {69331, 115.7, 518173, 4806, 2572, 127852, 1961});
+        data.put(14, new double[] {70551, 116.9, 554894, 4007, 2827, 130081, 1962});
+
+        KNNRegressionTrainer trainer = new KNNRegressionTrainer();
+
+        KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
+            new LocalDatasetBuilder<>(data, 2),
+            (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+            (k, v) -> v[0]
+        ).withK(3)
+            .withDistanceMeasure(new EuclideanDistance())
+            .withStrategy(KNNStrategy.SIMPLE);
+
+        Vector vector = new DenseLocalOnHeapVector(new double[] {104.6, 419180, 2822, 2857, 118734, 1956});
+        System.out.println(knnMdl.apply(vector));
+        Assert.assertEquals(67857, knnMdl.apply(vector), 2000);
+    }
+
+    /** */
+    public void testLonglyWithWeightedStrategy() {
+        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+
+        Map<Integer, double[]> data = new HashMap<>();
+        data.put(0, new double[] {60323, 83.0, 234289, 2356, 1590, 107608, 1947});
+        data.put(1, new double[] {61122, 88.5, 259426, 2325, 1456, 108632, 1948});
+        data.put(2, new double[] {60171, 88.2, 258054, 3682, 1616, 109773, 1949});
+        data.put(3, new double[] {61187, 89.5, 284599, 3351, 1650, 110929, 1950});
+        data.put(4, new double[] {63221, 96.2, 328975, 2099, 3099, 112075, 1951});
+        data.put(5, new double[] {63639, 98.1, 346999, 1932, 3594, 113270, 1952});
+        data.put(6, new double[] {64989, 99.0, 365385, 1870, 3547, 115094, 1953});
+        data.put(7, new double[] {63761, 100.0, 363112, 3578, 3350, 116219, 1954});
+        data.put(8, new double[] {66019, 101.2, 397469, 2904, 3048, 117388, 1955});
+        data.put(9, new double[] {68169, 108.4, 442769, 2936, 2798, 120445, 1957});
+        data.put(10, new double[] {66513, 110.8, 444546, 4681, 2637, 121950, 1958});
+        data.put(11, new double[] {68655, 112.6, 482704, 3813, 2552, 123366, 1959});
+        data.put(12, new double[] {69564, 114.2, 502601, 3931, 2514, 125368, 1960});
+        data.put(13, new double[] {69331, 115.7, 518173, 4806, 2572, 127852, 1961});
+        data.put(14, new double[] {70551, 116.9, 554894, 4007, 2827, 130081, 1962});
+
+        KNNRegressionTrainer trainer = new KNNRegressionTrainer();
+
+        KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
+            new LocalDatasetBuilder<>(data, 2),
+            (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+            (k, v) -> v[0]
+        ).withK(3)
+            .withDistanceMeasure(new EuclideanDistance())
+            .withStrategy(KNNStrategy.SIMPLE);
+
+        Vector vector = new DenseLocalOnHeapVector(new double[] {104.6, 419180, 2822, 2857, 118734, 1956});
+        System.out.println(knnMdl.apply(vector));
+        Assert.assertEquals(67857, knnMdl.apply(vector), 2000);
+    }
+}
\ No newline at end of file
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNTestSuite.java
index 95ebec5a2859b..55ef24e700005 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNTestSuite.java
@@ -26,6 +26,7 @@
 @RunWith(Suite.class)
 @Suite.SuiteClasses({
     KNNClassificationTest.class,
+    KNNRegressionTest.class,
     LabeledDatasetTest.class
 })
 public class KNNTestSuite {

From b96271866aeb09b434e2b95a457655a44d913254 Mon Sep 17 00:00:00 2001
From: Alexander Kalinin <verbalab@yandex.ru>
Date: Wed, 11 Apr 2018 17:09:41 +0700
Subject: [PATCH 023/543] IGNITE-4091 Web Console: Refactored using of internal
 Angular API.

(cherry picked from commit 74d2545)
---
 .../web-console/frontend/app/app.config.js    | 14 +++---
 .../modal-import-models/component.js          |  4 +-
 .../app/components/page-profile/controller.js |  4 +-
 .../frontend/app/modules/ace.module.js        | 47 ++++++++++---------
 .../services/AngularStrapSelect.decorator.js  |  5 +-
 .../services/AngularStrapTooltip.decorator.js |  8 ++--
 .../app/services/FormUtils.service.js         |  3 +-
 7 files changed, 45 insertions(+), 40 deletions(-)

diff --git a/modules/web-console/frontend/app/app.config.js b/modules/web-console/frontend/app/app.config.js
index 9d8dc99cec52a..e2bc057fc4ece 100644
--- a/modules/web-console/frontend/app/app.config.js
+++ b/modules/web-console/frontend/app/app.config.js
@@ -43,7 +43,7 @@ igniteConsoleCfg.config(['$animateProvider', ($animateProvider) => {
 
 // AngularStrap modal popup configuration.
 igniteConsoleCfg.config(['$modalProvider', ($modalProvider) => {
-    angular.extend($modalProvider.defaults, {
+    Object.assign($modalProvider.defaults, {
         animation: 'am-fade-and-scale',
         placement: 'center',
         html: true
@@ -52,7 +52,7 @@ igniteConsoleCfg.config(['$modalProvider', ($modalProvider) => {
 
 // AngularStrap popover configuration.
 igniteConsoleCfg.config(['$popoverProvider', ($popoverProvider) => {
-    angular.extend($popoverProvider.defaults, {
+    Object.assign($popoverProvider.defaults, {
         trigger: 'manual',
         placement: 'right',
         container: 'body',
@@ -62,7 +62,7 @@ igniteConsoleCfg.config(['$popoverProvider', ($popoverProvider) => {
 
 // AngularStrap tooltips configuration.
 igniteConsoleCfg.config(['$tooltipProvider', ($tooltipProvider) => {
-    angular.extend($tooltipProvider.defaults, {
+    Object.assign($tooltipProvider.defaults, {
         container: 'body',
         delay: {show: 150, hide: 150},
         placement: 'right',
@@ -73,7 +73,7 @@ igniteConsoleCfg.config(['$tooltipProvider', ($tooltipProvider) => {
 
 // AngularStrap select (combobox) configuration.
 igniteConsoleCfg.config(['$selectProvider', ($selectProvider) => {
-    angular.extend($selectProvider.defaults, {
+    Object.assign($selectProvider.defaults, {
         container: 'body',
         maxLength: '5',
         allText: 'Select All',
@@ -87,7 +87,7 @@ igniteConsoleCfg.config(['$selectProvider', ($selectProvider) => {
 
 // AngularStrap alerts configuration.
 igniteConsoleCfg.config(['$alertProvider', ($alertProvider) => {
-    angular.extend($alertProvider.defaults, {
+    Object.assign($alertProvider.defaults, {
         container: 'body',
         placement: 'top-right',
         duration: '5',
@@ -99,7 +99,7 @@ igniteConsoleCfg.config(['$alertProvider', ($alertProvider) => {
 
 // AngularStrap dropdowns () configuration.
 igniteConsoleCfg.config(['$dropdownProvider', ($dropdownProvider) => {
-    angular.extend($dropdownProvider.defaults, {
+    Object.assign($dropdownProvider.defaults, {
         templateUrl: dropdownTemplateUrl,
         animation: ''
     });
@@ -107,7 +107,7 @@ igniteConsoleCfg.config(['$dropdownProvider', ($dropdownProvider) => {
 
 // AngularStrap dropdowns () configuration.
 igniteConsoleCfg.config(['$datepickerProvider', ($datepickerProvider) => {
-    angular.extend($datepickerProvider.defaults, {
+    Object.assign($datepickerProvider.defaults, {
         autoclose: true,
         iconLeft: 'icon-datepicker-left',
         iconRight: 'icon-datepicker-right'
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
index 7f852b02cd3ec..813c998f0e00e 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
@@ -84,7 +84,7 @@ const DFLT_REPLICATED_CACHE = {
 const CACHE_TEMPLATES = [DFLT_PARTITIONED_CACHE, DFLT_REPLICATED_CACHE];
 
 export class ModalImportModels {
-    /** 
+    /**
      * Cluster ID to import models into
      * @type {string}
      */
@@ -771,7 +771,7 @@ export class ModalImportModels {
 
                 // Prepare caches for generation.
                 if (table.action === IMPORT_DM_NEW_CACHE) {
-                    const newCache = angular.copy(this.loadedCaches[table.cacheOrTemplate]);
+                    const newCache = _.cloneDeep(this.loadedCaches[table.cacheOrTemplate]);
 
                     batchAction.newCache = newCache;
 
diff --git a/modules/web-console/frontend/app/components/page-profile/controller.js b/modules/web-console/frontend/app/components/page-profile/controller.js
index 05fe1183001c9..c67a603d6a72b 100644
--- a/modules/web-console/frontend/app/components/page-profile/controller.js
+++ b/modules/web-console/frontend/app/components/page-profile/controller.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
 export default class PageProfileController {
     static $inject = [
         '$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteCountries', 'User'
@@ -28,7 +30,7 @@ export default class PageProfileController {
         this.ui = {};
 
         this.User.read()
-            .then((user) => this.ui.user = angular.copy(user));
+            .then((user) => this.ui.user = _.cloneDeep(user));
 
         this.ui.countries = this.Countries.getAll();
     }
diff --git a/modules/web-console/frontend/app/modules/ace.module.js b/modules/web-console/frontend/app/modules/ace.module.js
index a28536a0233fa..6a6e70a45f83e 100644
--- a/modules/web-console/frontend/app/modules/ace.module.js
+++ b/modules/web-console/frontend/app/modules/ace.module.js
@@ -16,12 +16,13 @@
  */
 
 import angular from 'angular';
+import _ from 'lodash';
 
 angular
     .module('ignite-console.ace', [])
     .constant('igniteAceConfig', {})
     .directive('igniteAce', ['igniteAceConfig', (aceConfig) => {
-        if (angular.isUndefined(window.ace))
+        if (_.isUndefined(window.ace))
             throw new Error('ignite-ace need ace to work... (o rly?)');
 
         /**
@@ -43,7 +44,7 @@ angular
          */
         const setOptions = (acee, session, opts) => {
             // Sets the ace worker path, if running from concatenated or minified source.
-            if (angular.isDefined(opts.workerPath)) {
+            if (!_.isUndefined(opts.workerPath)) {
                 const config = window.ace.acequire('ace/config');
 
                 config.set('workerPath', opts.workerPath);
@@ -53,26 +54,26 @@ angular
             _.forEach(opts.require, (n) => window.ace.acequire(n));
 
             // Boolean options.
-            if (angular.isDefined(opts.showGutter))
+            if (!_.isUndefined(opts.showGutter))
                 acee.renderer.setShowGutter(opts.showGutter);
 
-            if (angular.isDefined(opts.useWrapMode))
+            if (!_.isUndefined(opts.useWrapMode))
                 session.setUseWrapMode(opts.useWrapMode);
 
-            if (angular.isDefined(opts.showInvisibles))
+            if (!_.isUndefined(opts.showInvisibles))
                 acee.renderer.setShowInvisibles(opts.showInvisibles);
 
-            if (angular.isDefined(opts.showIndentGuides))
+            if (!_.isUndefined(opts.showIndentGuides))
                 acee.renderer.setDisplayIndentGuides(opts.showIndentGuides);
 
-            if (angular.isDefined(opts.useSoftTabs))
+            if (!_.isUndefined(opts.useSoftTabs))
                 session.setUseSoftTabs(opts.useSoftTabs);
 
-            if (angular.isDefined(opts.showPrintMargin))
+            if (!_.isUndefined(opts.showPrintMargin))
                 acee.setShowPrintMargin(opts.showPrintMargin);
 
             // Commands.
-            if (angular.isDefined(opts.disableSearch) && opts.disableSearch) {
+            if (!_.isUndefined(opts.disableSearch) && opts.disableSearch) {
                 acee.commands.addCommands([{
                     name: 'unfind',
                     bindKey: {
@@ -85,21 +86,21 @@ angular
             }
 
             // Base options.
-            if (angular.isString(opts.theme))
+            if (_.isString(opts.theme))
                 acee.setTheme('ace/theme/' + opts.theme);
 
-            if (angular.isString(opts.mode))
+            if (_.isString(opts.mode))
                 session.setMode('ace/mode/' + opts.mode);
 
-            if (angular.isDefined(opts.firstLineNumber)) {
-                if (angular.isNumber(opts.firstLineNumber))
+            if (!_.isUndefined(opts.firstLineNumber)) {
+                if (_.isNumber(opts.firstLineNumber))
                     session.setOption('firstLineNumber', opts.firstLineNumber);
-                else if (angular.isFunction(opts.firstLineNumber))
+                else if (_.isFunction(opts.firstLineNumber))
                     session.setOption('firstLineNumber', opts.firstLineNumber());
             }
 
             // Advanced options.
-            if (angular.isDefined(opts.advanced)) {
+            if (!_.isUndefined(opts.advanced)) {
                 for (const key in opts.advanced) {
                     if (opts.advanced.hasOwnProperty(key)) {
                         // Create a javascript object with the key and value.
@@ -112,7 +113,7 @@ angular
             }
 
             // Advanced options for the renderer.
-            if (angular.isDefined(opts.rendererOptions)) {
+            if (!_.isUndefined(opts.rendererOptions)) {
                 for (const key in opts.rendererOptions) {
                     if (opts.rendererOptions.hasOwnProperty(key)) {
                         // Create a javascript object with the key and value.
@@ -126,7 +127,7 @@ angular
 
             // onLoad callbacks.
             _.forEach(opts.callbacks, (cb) => {
-                if (angular.isFunction(cb))
+                if (_.isFunction(cb))
                     cb(acee);
             });
         };
@@ -147,7 +148,7 @@ angular
                  *
                  * @type object
                  */
-                let opts = angular.extend({}, options, scope.$eval(attrs.igniteAce));
+                let opts = Object.assign({}, options, scope.$eval(attrs.igniteAce));
 
                 /**
                  * ACE editor.
@@ -191,9 +192,9 @@ angular
                             !scope.$$phase && !scope.$root.$$phase)
                             scope.$eval(() => ngModel.$setViewValue(newValue));
 
-                        if (angular.isDefined(callback)) {
+                        if (!_.isUndefined(callback)) {
                             scope.$evalAsync(() => {
-                                if (angular.isFunction(callback))
+                                if (_.isFunction(callback))
                                     callback([e, acee]);
                                 else
                                     throw new Error('ignite-ace use a function as callback');
@@ -210,10 +211,10 @@ angular
                     form && form.$removeControl(ngModel);
 
                     ngModel.$formatters.push((value) => {
-                        if (angular.isUndefined(value) || value === null)
+                        if (_.isUndefined(value) || value === null)
                             return '';
 
-                        if (angular.isObject(value) || angular.isArray(value))
+                        if (_.isObject(value) || _.isArray(value))
                             throw new Error('ignite-ace cannot use an object or an array as a model');
 
                         return value;
@@ -229,7 +230,7 @@ angular
                     if (current === previous)
                         return;
 
-                    opts = angular.extend({}, options, scope.$eval(attrs.igniteAce));
+                    opts = Object.assign({}, options, scope.$eval(attrs.igniteAce));
 
                     opts.callbacks = [opts.onLoad];
 
diff --git a/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js b/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js
index 39f7ccdb6bec2..32fa167f2d1af 100644
--- a/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js
+++ b/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js
@@ -16,6 +16,7 @@
  */
 
 import angular from 'angular';
+import _ from 'lodash';
 
 /**
  * Special decorator that fix problem in AngularStrap selectAll / deselectAll methods.
@@ -27,12 +28,12 @@ export default angular.module('mgcrea.ngStrap.select')
             const delegate = $delegate(element, controller, config);
 
             // Common vars.
-            const options = angular.extend({}, $delegate.defaults, config);
+            const options = Object.assign({}, $delegate.defaults, config);
 
             const scope = delegate.$scope;
 
             const valueByIndex = (index) => {
-                if (angular.isUndefined(scope.$matches[index]))
+                if (_.isUndefined(scope.$matches[index]))
                     return null;
 
                 return scope.$matches[index].value;
diff --git a/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js b/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js
index d01a45075b794..fa59f32765111 100644
--- a/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js
+++ b/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js
@@ -16,7 +16,7 @@
  */
 
 import angular from 'angular';
-import flow from 'lodash/flow';
+import _ from 'lodash';
 
 /**
  * Decorator that fix problem in AngularStrap $tooltip.
@@ -62,7 +62,7 @@ export default angular
 
                 scope.$emit(options.prefixEvent + '.hide.before', $tooltip);
 
-                if (angular.isDefined(options.onBeforeHide) && angular.isFunction(options.onBeforeHide))
+                if (!_.isUndefined(options.onBeforeHide) && _.isFunction(options.onBeforeHide))
                     options.onBeforeHide($tooltip);
 
                 $tooltip.$isShown = scope.$isShown = false;
@@ -82,8 +82,8 @@ export default angular
             const $tooltip = $delegate(el, config);
 
             $tooltip.$referenceElement = el;
-            $tooltip.destroy = flow($tooltip.destroy, () => $tooltip.$referenceElement = null);
-            $tooltip.$applyPlacement = flow($tooltip.$applyPlacement, () => {
+            $tooltip.destroy = _.flow($tooltip.destroy, () => $tooltip.$referenceElement = null);
+            $tooltip.$applyPlacement = _.flow($tooltip.$applyPlacement, () => {
                 if (!$tooltip.$element)
                     return;
 
diff --git a/modules/web-console/frontend/app/services/FormUtils.service.js b/modules/web-console/frontend/app/services/FormUtils.service.js
index f22d4bc6c0af8..da1d73700d59b 100644
--- a/modules/web-console/frontend/app/services/FormUtils.service.js
+++ b/modules/web-console/frontend/app/services/FormUtils.service.js
@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import _ from 'lodash';
 
 export default ['IgniteFormUtils', ['$window', 'IgniteFocus', ($window, Focus) => {
     function ensureActivePanel(ui, pnl, focusId) {
@@ -41,7 +42,7 @@ export default ['IgniteFormUtils', ['$window', 'IgniteFocus', ($window, Focus) =
                 if (!activePanels || activePanels.length < 1)
                     ui.activePanels = [idx];
                 else if (!_.includes(activePanels, idx)) {
-                    const newActivePanels = angular.copy(activePanels);
+                    const newActivePanels = _.cloneDeep(activePanels);
 
                     newActivePanels.push(idx);
 

From 2be10fdcde3c529b866f85a5b74ff3d471c5c9c7 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Wed, 11 Apr 2018 19:32:52 +0700
Subject: [PATCH 024/543] IGNITE-8106 Collect suppressed exceptions from
 causes. - Fixes #3735.

Signed-off-by: Alexey Kuznetsov <akuznetsov@apache.org>

(cherry picked from commit 98ef925)
---
 .../GridChangeStateCommandHandler.java        |  3 +-
 .../ignite/internal/util/typedef/X.java       | 37 +++++++++++++++----
 .../visor/util/VisorExceptionWrapper.java     | 11 +++---
 .../tcp/TcpCommunicationSpi.java              |  2 +-
 .../GridSuppressedExceptionSelfTest.java      | 23 +++++++++++-
 5 files changed, 59 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
index 7bb13d9b41e84..619be34546cd7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.processors.rest.request.GridRestChangeStateRequest;
 import org.apache.ignite.internal.processors.rest.request.GridRestRequest;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -78,7 +79,7 @@ public GridChangeStateCommandHandler(GridKernalContext ctx) {
 
             sb.a(e.getMessage()).a("\n").a("suppressed: \n");
 
-            for (Throwable t:e.getSuppressed())
+            for (Throwable t : X.getSuppressedList(e))
                 sb.a(t.getMessage()).a("\n");
 
             res.setError(sb.toString());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java b/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
index 395de237386ef..1a43daa09c918 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
@@ -469,14 +469,12 @@ public static boolean hasSuppressed(@Nullable Throwable t, @Nullable Class<? ext
         if (t == null || cls == null)
             return false;
 
-        if (t.getSuppressed() != null) {
-            for (Throwable th : t.getSuppressed()) {
-                if (cls.isAssignableFrom(th.getClass()))
-                    return true;
+        for (Throwable th : t.getSuppressed()) {
+            if (cls.isAssignableFrom(th.getClass()))
+                return true;
 
-                if (hasSuppressed(th, cls))
-                    return true;
-            }
+            if (hasSuppressed(th, cls))
+                return true;
         }
 
         return false;
@@ -748,6 +746,29 @@ public static Throwable[] getThrowables(Throwable throwable) {
         return list.toArray(new Throwable[list.size()]);
     }
 
+    /**
+     * Collects suppressed exceptions from throwable and all it causes.
+     *
+     * @param t Throwable.
+     * @return List of suppressed throwables.
+     */
+    public static List<Throwable> getSuppressedList(@Nullable Throwable t) {
+        List<Throwable> result = new ArrayList<>();
+
+        if (t == null)
+            return result;
+
+        do {
+            for (Throwable suppressed : t.getSuppressed()) {
+                result.add(suppressed);
+
+                result.addAll(getSuppressedList(suppressed));
+            }
+        } while ((t = t.getCause()) != null);
+
+        return result;
+    }
+
     /**
      * A way to get the entire nested stack-trace of an throwable.
      *
@@ -889,4 +910,4 @@ public static double parseDouble(@Nullable String s, double dflt) {
             return dflt;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorExceptionWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorExceptionWrapper.java
index 15e9557004bb2..ba52c5fd8b190 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorExceptionWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorExceptionWrapper.java
@@ -17,7 +17,8 @@
 
 package org.apache.ignite.internal.visor.util;
 
-import org.apache.ignite.internal.util.typedef.F;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.X;
 
 /**
  * Exception wrapper for safe for transferring to Visor.
@@ -56,12 +57,10 @@ public VisorExceptionWrapper(Throwable cause) {
         if (cause.getCause() != null)
             initCause(new VisorExceptionWrapper(cause.getCause()));
 
-        Throwable[] suppressed = cause.getSuppressed();
+        List<Throwable> suppressed = X.getSuppressedList(cause);
 
-        if (!F.isEmpty(suppressed)) {
-            for (Throwable sup : suppressed)
-                addSuppressed(new VisorExceptionWrapper(sup));
-        }
+        for (Throwable sup : suppressed)
+            addSuppressed(new VisorExceptionWrapper(sup));
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 9e7b59235db80..df37dff1f6f10 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3476,7 +3476,7 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                     ctx.failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
                         "rmtNode=" + node +
                         ", errs=" + errs +
-                        ", connectErrs=" + Arrays.toString(errs.getSuppressed()) + ']');
+                        ", connectErrs=" + X.getSuppressedList(errs) + ']');
                 }
             }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/GridSuppressedExceptionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/GridSuppressedExceptionSelfTest.java
index 6e32249b54f4c..55e54fb884fac 100644
--- a/modules/core/src/test/java/org/apache/ignite/GridSuppressedExceptionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/GridSuppressedExceptionSelfTest.java
@@ -18,6 +18,7 @@
 package org.apache.ignite;
 
 import java.io.IOException;
+import java.util.List;
 import junit.framework.TestCase;
 import org.apache.ignite.internal.util.typedef.X;
 
@@ -67,6 +68,26 @@ public void testXHasCause() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testXGetSuppressedList() throws Exception {
+        IgniteCheckedException me = prepareMultiException();
+
+        assertEquals(3, X.getSuppressedList(me).size());
+
+        RuntimeException e = new RuntimeException();
+        e.addSuppressed(me);
+
+        List<Throwable> suppresseds = X.getSuppressedList(e);
+
+        assertEquals(4, suppresseds.size());
+
+        assertEquals("Test message.", suppresseds.get(0).getMessage());
+        for (int i = 1; i <= 3; i++)
+            assertEquals("Demo exception.", suppresseds.get(1).getMessage());
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -116,4 +137,4 @@ private void generateException(int calls, Throwable cause) throws IgniteCheckedE
         else
             generateException(calls - 1, cause);
     }
-}
\ No newline at end of file
+}

From 7f463be9ff6c5d2d84b0aacc888159ffe68bf269 Mon Sep 17 00:00:00 2001
From: Alexander Paschenko <alexander.a.paschenko@gmail.com>
Date: Wed, 11 Apr 2018 16:20:16 +0300
Subject: [PATCH 025/543] IGNITE-8204: SQL: fixed hangs when lazy flag is
 enabled. This closes #3785.

---
 .../query/h2/twostep/GridMapQueryExecutor.java      |  7 +++++++
 .../query/h2/twostep/MapQueryLazyWorker.java        | 13 +++++++++++--
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 9b1e4faa4346c..930ada22f523e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -233,6 +233,13 @@ else if (msg instanceof GridH2DmlRequest)
         }
     }
 
+    /**
+     * @return Busy lock for lazy workers to guard their operations with.
+     */
+    GridSpinBusyLock busyLock() {
+        return busyLock;
+    }
+
     /**
      * @param node Node.
      * @param msg Message.
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryLazyWorker.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryLazyWorker.java
index 59c050ffc2bbf..98f3df98260fc 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryLazyWorker.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryLazyWorker.java
@@ -80,8 +80,17 @@ public MapQueryLazyWorker(@Nullable String instanceName, MapQueryLazyWorkerKey k
             while (!isCancelled()) {
                 Runnable task = tasks.take();
 
-                if (task != null)
-                    task.run();
+                if (task != null) {
+                    if (!exec.busyLock().enterBusy())
+                        return;
+
+                    try {
+                        task.run();
+                    }
+                    finally {
+                        exec.busyLock().leaveBusy();
+                    }
+                }
             }
         }
         finally {

From 7eee6e24736d8c0958e0107471c47ef4035037d8 Mon Sep 17 00:00:00 2001
From: Alexey Kukushkin <alexeykukushkin@yahoo.com>
Date: Wed, 11 Apr 2018 16:29:07 +0300
Subject: [PATCH 026/543] IGNITE-8221: Security for thin clients.

---
 .../apache/ignite/IgniteSystemProperties.java |  6 +++
 .../client/ClientAuthenticationException.java |  2 +-
 .../client/ClientAuthorizationException.java  | 46 +++++++++++++++++++
 .../internal/client/thin/ClientChannel.java   |  3 +-
 .../client/thin/TcpClientChannel.java         | 39 ++++++++--------
 .../IgniteAuthenticationProcessor.java        |  5 +-
 .../processors/cache/GridCacheProcessor.java  | 32 +++++++++++++
 .../processors/cache/GridCacheUtils.java      |  5 ++
 .../client/ClientConnectionContext.java       | 45 +++++++++++++++++-
 .../platform/client/ClientRequest.java        | 29 ++++++++++++
 .../platform/client/ClientStatus.java         |  3 ++
 .../cache/ClientCacheClearKeyRequest.java     |  3 ++
 .../cache/ClientCacheClearKeysRequest.java    |  3 ++
 .../client/cache/ClientCacheClearRequest.java |  3 ++
 .../cache/ClientCacheContainsKeyRequest.java  |  3 ++
 .../cache/ClientCacheContainsKeysRequest.java |  3 ++
 ...ntCacheCreateWithConfigurationRequest.java |  6 ++-
 .../ClientCacheCreateWithNameRequest.java     |  3 ++
 .../cache/ClientCacheDestroyRequest.java      |  3 ++
 .../cache/ClientCacheGetAllRequest.java       |  3 ++
 .../ClientCacheGetAndPutIfAbsentRequest.java  |  3 ++
 .../cache/ClientCacheGetAndPutRequest.java    |  3 ++
 .../cache/ClientCacheGetAndRemoveRequest.java |  3 ++
 .../ClientCacheGetAndReplaceRequest.java      |  3 ++
 ...heGetOrCreateWithConfigurationRequest.java |  6 ++-
 ...ClientCacheGetOrCreateWithNameRequest.java |  3 ++
 .../client/cache/ClientCacheGetRequest.java   |  3 ++
 .../cache/ClientCacheGetSizeRequest.java      |  3 ++
 .../cache/ClientCachePutAllRequest.java       |  3 ++
 .../cache/ClientCachePutIfAbsentRequest.java  |  3 ++
 .../client/cache/ClientCachePutRequest.java   |  3 ++
 .../cache/ClientCacheRemoveAllRequest.java    |  3 ++
 .../ClientCacheRemoveIfEqualsRequest.java     |  3 ++
 .../cache/ClientCacheRemoveKeyRequest.java    |  3 ++
 .../cache/ClientCacheRemoveKeysRequest.java   |  3 ++
 .../ClientCacheReplaceIfEqualsRequest.java    |  3 ++
 .../cache/ClientCacheReplaceRequest.java      |  3 ++
 .../client/cache/ClientCacheRequest.java      | 32 +++++++++++++
 .../cache/ClientCacheScanQueryRequest.java    |  3 ++
 .../ClientCacheSqlFieldsQueryRequest.java     |  1 +
 .../cache/ClientCacheSqlQueryRequest.java     |  1 +
 .../security/AuthenticationContext.java       | 40 ++++++++++++++++
 .../plugin/security/SecurityPermission.java   | 11 ++++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 12 ++++-
 44 files changed, 371 insertions(+), 28 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/client/ClientAuthorizationException.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 9da123e01fb35..ac7947b7e45aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -834,6 +834,12 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER = "IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER";
 
+    /**
+     * When set to {@code true}, on-heap cache cannot be enabled - see
+     * {@link CacheConfiguration#setOnheapCacheEnabled(boolean)}.
+     * Default is {@code false}.
+     */
+    public static final String IGNITE_DISABLE_ONHEAP_CACHE = "IGNITE_DISABLE_ONHEAP_CACHE";
     /**
      * When set to {@code false}, loaded pages implementation is switched to previous version of implementation,
      * FullPageIdTable. {@code True} value enables 'Robin Hood hashing: backward shift deletion'.
diff --git a/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
index dc39c7a0ab780..0c24db8880a4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
+++ b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
@@ -18,7 +18,7 @@
 package org.apache.ignite.client;
 
 /**
- * Indicates Ignite server the client is connected to closed the connection and no longer available.
+ * Indicates user name or password is invalid.
  */
 public class ClientAuthenticationException extends ClientException {
     /** Serial version uid. */
diff --git a/modules/core/src/main/java/org/apache/ignite/client/ClientAuthorizationException.java b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthorizationException.java
new file mode 100644
index 0000000000000..cacede67bb356
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthorizationException.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.client;
+
+/**
+ * Indicates user has no permission to perform operation.
+ */
+public class ClientAuthorizationException extends ClientException {
+    /** Serial version uid. */
+    private static final long serialVersionUID = 0L;
+
+    /** Message. */
+    private static final String MSG = "User is not authorized to perform this operation";
+
+    /**
+     * Default constructor.
+     */
+    public ClientAuthorizationException() {
+        super(MSG);
+    }
+
+    /**
+     * Constructs a new exception with the specified cause and a detail
+     * message of <tt>(cause==null ? null : cause.toString())</tt>.
+     *
+     * @param cause the cause.
+     */
+    public ClientAuthorizationException(Throwable cause) {
+        super(MSG, cause);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientChannel.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientChannel.java
index 71502a4760483..eb62c808225f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientChannel.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientChannel.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.internal.binary.streams.BinaryInputStream;
 import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
 import org.apache.ignite.client.ClientConnectionException;
+import org.apache.ignite.client.ClientAuthorizationException;
 
 /**
  * Processing thin client requests and responses.
@@ -41,5 +42,5 @@ interface ClientChannel extends AutoCloseable {
      * @return Received operation payload or {@code null} if response has no payload.
      */
     public <T> T receive(ClientOperation op, long reqId, Function<BinaryInputStream, T> payloadReader)
-        throws ClientConnectionException;
+        throws ClientConnectionException, ClientAuthorizationException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
index 404793a83f338..8e8294f1ef6f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
@@ -50,6 +50,7 @@
 import javax.net.ssl.TrustManagerFactory;
 import javax.net.ssl.X509TrustManager;
 import org.apache.ignite.client.ClientAuthenticationException;
+import org.apache.ignite.client.ClientAuthorizationException;
 import org.apache.ignite.client.ClientConnectionException;
 import org.apache.ignite.client.SslMode;
 import org.apache.ignite.client.SslProtocol;
@@ -62,6 +63,7 @@
 import org.apache.ignite.internal.binary.streams.BinaryInputStream;
 import org.apache.ignite.internal.binary.streams.BinaryOffheapOutputStream;
 import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
+import org.apache.ignite.internal.processors.platform.client.ClientStatus;
 
 /**
  * Implements {@link ClientChannel} over TCP.
@@ -138,7 +140,8 @@ class TcpClientChannel implements ClientChannel {
 
     /** {@inheritDoc} */
     public <T> T receive(ClientOperation op, long reqId, Function<BinaryInputStream, T> payloadReader)
-        throws ClientConnectionException {
+        throws ClientConnectionException, ClientAuthorizationException {
+
         final int MIN_RES_SIZE = 8 + 4; // minimal response size: long (8 bytes) ID + int (4 bytes) status
 
         int resSize = new BinaryHeapInputStream(read(4)).readInt();
@@ -163,7 +166,12 @@ public <T> T receive(ClientOperation op, long reqId, Function<BinaryInputStream,
 
             String err = new BinaryReaderExImpl(null, resIn, null, true).readString();
 
-            throw new ClientServerError(err, status, reqId);
+            switch (status) {
+                case ClientStatus.SECURITY_VIOLATION:
+                    throw new ClientAuthorizationException();
+                default:
+                    throw new ClientServerError(err, status, reqId);
+            }
         }
 
         if (resSize <= MIN_RES_SIZE || payloadReader == null)
@@ -539,16 +547,10 @@ private static TrustManager[] getTrustManagers(
 
         /** */
         private static KeyStore loadKeyStore(String lb, String path, String type, char[] pwd) {
-            InputStream in = null;
+            KeyStore store;
 
             try {
-                KeyStore store = KeyStore.getInstance(type);
-
-                in = new FileInputStream(new File(path));
-
-                store.load(in, pwd);
-
-                return store;
+                store = KeyStore.getInstance(type);
             }
             catch (KeyStoreException e) {
                 throw new ClientError(
@@ -556,6 +558,13 @@ private static KeyStore loadKeyStore(String lb, String path, String type, char[]
                     e
                 );
             }
+
+            try (InputStream in = new FileInputStream(new File(path))) {
+
+                store.load(in, pwd);
+
+                return store;
+            }
             catch (FileNotFoundException e) {
                 throw new ClientError(String.format("%s key store file [%s] does not exist", lb, path), e);
             }
@@ -571,16 +580,6 @@ private static KeyStore loadKeyStore(String lb, String path, String type, char[]
             catch (IOException e) {
                 throw new ClientError(String.format("Could not read %s key store", lb), e);
             }
-            finally {
-                if (in != null) {
-                    try {
-                        in.close();
-                    }
-                    catch (IOException ignored) {
-                        // Fail silently
-                    }
-                }
-            }
         }
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
index c025e8c0b5f14..93b3a5bc2cce4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
@@ -897,7 +897,10 @@ public void onLocalJoin() {
 
             // Can be empty on initial start of PDS cluster (default user will be created and stored after activate)
             if (!F.isEmpty(initUsrs.usrs)) {
-                users.clear();
+                if (users == null)
+                    users = new ConcurrentHashMap<>();
+                else
+                    users.clear();
 
                 for (User u : initUsrs.usrs)
                     users.put(u.name(), u);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 7edac7374fbe9..3aa6603a47d96 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -149,6 +149,8 @@
 import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.mxbean.CacheGroupMetricsMXBean;
 import org.apache.ignite.mxbean.IgniteMBeanAware;
+import org.apache.ignite.plugin.security.SecurityException;
+import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.spi.IgniteNodeValidationResult;
 import org.apache.ignite.spi.discovery.DiscoveryDataBag;
 import org.apache.ignite.spi.discovery.DiscoveryDataBag.GridDiscoveryData;
@@ -1126,6 +1128,9 @@ private void startCache(GridCacheAdapter<?, ?> cache, QuerySchema schema) throws
 
         CacheConfiguration cfg = cacheCtx.config();
 
+        if (cacheCtx.userCache())
+            authorizeCacheCreate(cacheCtx.name(), cfg);
+
         // Intentionally compare Boolean references using '!=' below to check if the flag has been explicitly set.
         if (cfg.isStoreKeepBinary() && cfg.isStoreKeepBinary() != CacheConfiguration.DFLT_STORE_KEEP_BINARY
             && !(ctx.config().getMarshaller() instanceof BinaryMarshaller))
@@ -3151,6 +3156,8 @@ private Collection<DynamicCacheStartFuture> initiateCacheChanges(
         Collection<DynamicCacheChangeRequest> sndReqs = new ArrayList<>(reqs.size());
 
         for (DynamicCacheChangeRequest req : reqs) {
+            authorizeCacheChange(req);
+
             DynamicCacheStartFuture fut = new DynamicCacheStartFuture(req.requestId());
 
             try {
@@ -3215,6 +3222,31 @@ private Collection<DynamicCacheStartFuture> initiateCacheChanges(
         return res;
     }
 
+    /**
+     * Authorize dynamic cache management.
+     */
+    private void authorizeCacheChange(DynamicCacheChangeRequest req) {
+        if (req.cacheType() == null || req.cacheType() == CacheType.USER) {
+            if (req.stop())
+                ctx.security().authorize(req.cacheName(), SecurityPermission.CACHE_DESTROY, null);
+            else
+                authorizeCacheCreate(req.cacheName(), req.startCacheConfiguration());
+        }
+    }
+
+    /**
+     * Authorize start/create cache operation.
+     */
+    private void authorizeCacheCreate(String cacheName, CacheConfiguration cacheCfg) {
+        ctx.security().authorize(cacheName, SecurityPermission.CACHE_CREATE, null);
+
+        if (cacheCfg != null && cacheCfg.isOnheapCacheEnabled() &&
+            System.getProperty(IgniteSystemProperties.IGNITE_DISABLE_ONHEAP_CACHE, "false")
+                .toUpperCase().equals("TRUE")
+            )
+            throw new SecurityException("Authorization failed for enabling on-heap cache.");
+    }
+
     /**
      * @return Non null exception if node is stopping or disconnected.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index d672420fafdc8..e244c75ad4a0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -98,6 +98,8 @@
 import org.apache.ignite.lang.IgniteReducer;
 import org.apache.ignite.lifecycle.LifecycleAware;
 import org.apache.ignite.plugin.CachePluginConfiguration;
+import org.apache.ignite.plugin.security.SecurityException;
+import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
@@ -1290,6 +1292,9 @@ else if (e instanceof SchemaOperationException)
         if (e.getCause() instanceof NullPointerException)
             return (NullPointerException)e.getCause();
 
+        if (e.getCause() instanceof SecurityException)
+            return (SecurityException)e.getCause();
+
         C1<IgniteCheckedException, IgniteException> converter = U.getExceptionConverter(e.getClass());
 
         return converter != null ? new CacheException(converter.apply(e)) : new CacheException(e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
index 7ab2d33e5dbc0..061aab32c0459 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
@@ -20,16 +20,24 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.binary.BinaryReaderExImpl;
 import org.apache.ignite.internal.processors.authentication.AuthorizationContext;
+import org.apache.ignite.internal.processors.authentication.IgniteAccessControlException;
 import org.apache.ignite.internal.processors.odbc.ClientListenerConnectionContext;
 import org.apache.ignite.internal.processors.odbc.ClientListenerMessageParser;
 import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
 import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler;
 
 import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.plugin.security.AuthenticationContext;
+import org.apache.ignite.plugin.security.SecurityCredentials;
+
+import static org.apache.ignite.plugin.security.SecuritySubjectType.REMOTE_CLIENT;
 
 /**
  * Thin Client connection context.
@@ -62,6 +70,9 @@ public class ClientConnectionContext implements ClientListenerConnectionContext
     /** Cursor counter. */
     private final AtomicLong curCnt = new AtomicLong();
 
+    /** Security context or {@code null} if security is disabled. */
+    private SecurityContext secCtx = null;
+
     /**
      * Ctor.
      *
@@ -129,7 +140,9 @@ public GridKernalContext kernalContext() {
             }
         }
 
-        if (kernalCtx.authentication().enabled()) {
+        if (kernalCtx.security().enabled())
+            authCtx = thirdPartyAuthentication(user, pwd).authorizationContext();
+        else if (kernalCtx.authentication().enabled()) {
             if (user == null || user.length() == 0)
                 throw new IgniteCheckedException("Unauthenticated sessions are prohibited.");
 
@@ -179,4 +192,34 @@ public void incrementCursors() {
     public void decrementCursors() {
         curCnt.decrementAndGet();
     }
+
+    /**
+     * @return Security context or {@code null} if security is disabled.
+     */
+    public SecurityContext securityContext() {
+        return secCtx;
+    }
+
+    /**
+     * Do 3-rd party authentication.
+     */
+    private AuthenticationContext thirdPartyAuthentication(String user, String pwd) throws IgniteCheckedException {
+        SecurityCredentials cred = new SecurityCredentials(user, pwd);
+
+        AuthenticationContext authCtx = new AuthenticationContext();
+
+        authCtx.subjectType(REMOTE_CLIENT);
+        authCtx.subjectId(UUID.randomUUID());
+        authCtx.nodeAttributes(Collections.emptyMap());
+        authCtx.credentials(cred);
+
+        secCtx = kernalCtx.security().authenticate(authCtx);
+
+        if (secCtx == null)
+            throw new IgniteAccessControlException(
+                String.format("The user name or password is incorrect [userName=%s]", user)
+            );
+
+        return authCtx;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java
index 76823b592ce32..799b3e733f410 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequest.java
@@ -19,6 +19,9 @@
 
 import org.apache.ignite.binary.BinaryRawReader;
 import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.plugin.security.SecurityException;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Thin client request.
@@ -58,4 +61,30 @@ public ClientRequest(long reqId) {
     public ClientResponse process(ClientConnectionContext ctx) {
         return new ClientResponse(reqId);
     }
+
+    /**
+     * Run the code with converting {@link SecurityException} to {@link IgniteClientException}.
+     */
+    protected static void runWithSecurityExceptionHandler(Runnable runnable) {
+        try {
+            runnable.run();
+        }
+        catch (SecurityException ex) {
+            throw new IgniteClientException(
+                ClientStatus.SECURITY_VIOLATION,
+                "Client is not authorized to perform this operation",
+                ex
+            );
+        }
+    }
+
+    /**
+     * Authorize for specified permission.
+     */
+    protected void authorize(ClientConnectionContext ctx, SecurityPermission perm) {
+        SecurityContext secCtx = ctx.securityContext();
+
+        if (secCtx != null)
+            runWithSecurityExceptionHandler(() -> ctx.kernalContext().security().authorize(null, perm, secCtx));
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
index e0049b472a18f..b8dfb1fa64b9e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
@@ -48,4 +48,7 @@ private ClientStatus (){
 
     /** Resource does not exist. */
     public static final int RESOURCE_DOES_NOT_EXIST = 1011;
+
+    /** Resource does not exist. */
+    public static final int SECURITY_VIOLATION = 1012;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java
index 6bcbbe89b2636..5f8e952234bf8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java
@@ -20,6 +20,7 @@
 import org.apache.ignite.internal.binary.BinaryRawReaderEx;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Clear key request.
@@ -37,6 +38,8 @@ public ClientCacheClearKeyRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
         cache(ctx).clear(key());
 
         return super.process(ctx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java
index 04eb7f60c9502..d803f697420f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeysRequest.java
@@ -20,6 +20,7 @@
 import org.apache.ignite.internal.binary.BinaryRawReaderEx;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Clear keys request.
@@ -37,6 +38,8 @@ public ClientCacheClearKeysRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
         cache(ctx).clearAll(keys());
 
         return super.process(ctx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java
index 0e5f20de1eb1b..7b84522921c7c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearRequest.java
@@ -20,6 +20,7 @@
 import org.apache.ignite.binary.BinaryRawReader;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache clear request.
@@ -37,6 +38,8 @@ public ClientCacheClearRequest(BinaryRawReader reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
         cache(ctx).clear();
 
         return super.process(ctx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java
index 8470828e424a1..386f448bb4b53 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * ContainsKey request.
@@ -38,6 +39,8 @@ public ClientCacheContainsKeyRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ);
+
         boolean val = cache(ctx).containsKey(key());
 
         return new ClientBooleanResponse(requestId(), val);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java
index 41e13068db1f5..b5184bfc1afc9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeysRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * ContainsKeys request.
@@ -38,6 +39,8 @@ public ClientCacheContainsKeysRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ);
+
         boolean val = cache(ctx).containsKeys(keys());
 
         return new ClientBooleanResponse(requestId(), val);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java
index 4b4dcece7c2a6..65f97841b9ad3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithConfigurationRequest.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientStatus;
 import org.apache.ignite.internal.processors.platform.client.IgniteClientException;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache create with configuration request.
@@ -47,8 +48,11 @@ public ClientCacheCreateWithConfigurationRequest(BinaryRawReader reader) {
 
     /** {@inheritDoc} */
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_CREATE);
+
         try {
-            ctx.kernalContext().grid().createCache(cacheCfg);
+            // Use security exception handler since the code authorizes "enable on-heap cache" permission
+            runWithSecurityExceptionHandler(() -> ctx.kernalContext().grid().createCache(cacheCfg));
         } catch (CacheExistsException e) {
             throw new IgniteClientException(ClientStatus.CACHE_EXISTS, e.getMessage());
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java
index 9155d76bfee00..cacf099b4f81f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheCreateWithNameRequest.java
@@ -24,6 +24,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientStatus;
 import org.apache.ignite.internal.processors.platform.client.IgniteClientException;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache create with name request.
@@ -45,6 +46,8 @@ public ClientCacheCreateWithNameRequest(BinaryRawReader reader) {
 
     /** {@inheritDoc} */
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_CREATE);
+
         try {
             ctx.kernalContext().grid().createCache(cacheName);
         } catch (CacheExistsException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java
index 6645a03a06b4f..b6f85eec3d9fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheDestroyRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientRequest;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache destroy request.
@@ -42,6 +43,8 @@ public ClientCacheDestroyRequest(BinaryRawReader reader) {
 
     /** {@inheritDoc} */
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_DESTROY);
+
         String cacheName = ClientCacheRequest.cacheDescriptor(ctx, cacheId).cacheName();
 
         ctx.kernalContext().grid().destroyCache(cacheName);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java
index 2b33af1cb69f8..a07305c4ce14b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAllRequest.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
 
 import java.util.Map;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * GetAll request.
@@ -39,6 +40,8 @@ public ClientCacheGetAllRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ);
+
         Map val = cache(ctx).getAll(keys());
 
         return new ClientCacheGetAllResponse(requestId(), val);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java
index 836021313c5fc..8713a211bb4eb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache get and put if absent request.
@@ -38,6 +39,8 @@ public ClientCacheGetAndPutIfAbsentRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
         Object res = cache(ctx).getAndPutIfAbsent(key(), val());
 
         return new ClientObjectResponse(requestId(), res);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java
index 7a540e8473ac9..dde5181303cef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache get and put request.
@@ -38,6 +39,8 @@ public ClientCacheGetAndPutRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
         Object res = cache(ctx).getAndPut(key(), val());
 
         return new ClientObjectResponse(requestId(), res);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java
index e4fd735b186ac..3b9dd4bab88c3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache get and remove request.
@@ -38,6 +39,8 @@ public ClientCacheGetAndRemoveRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_REMOVE);
+
         Object val = cache(ctx).getAndRemove(key());
 
         return new ClientObjectResponse(requestId(), val);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java
index dba8639e4c07a..8ba157a762c9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache get and replace request.
@@ -38,6 +39,8 @@ public ClientCacheGetAndReplaceRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
         Object res = cache(ctx).getAndReplace(key(), val());
 
         return new ClientObjectResponse(requestId(), res);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java
index 267318a3e3349..48569b447e156 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithConfigurationRequest.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientStatus;
 import org.apache.ignite.internal.processors.platform.client.IgniteClientException;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache get or create with configuration request.
@@ -47,8 +48,11 @@ public ClientCacheGetOrCreateWithConfigurationRequest(BinaryRawReader reader) {
 
     /** {@inheritDoc} */
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_CREATE);
+
         try {
-            ctx.kernalContext().grid().getOrCreateCache(cacheCfg);
+            // Use security exception handler since the code authorizes "enable on-heap cache" permission
+            runWithSecurityExceptionHandler(() -> ctx.kernalContext().grid().getOrCreateCache(cacheCfg));
         } catch (CacheExistsException e) {
             throw new IgniteClientException(ClientStatus.CACHE_EXISTS, e.getMessage());
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java
index 94dd115d6075f..3c4ce7b06a694 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetOrCreateWithNameRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientRequest;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache create with name request.
@@ -42,6 +43,8 @@ public ClientCacheGetOrCreateWithNameRequest(BinaryRawReader reader) {
 
     /** {@inheritDoc} */
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_CREATE);
+
         ctx.kernalContext().grid().getOrCreateCache(cacheName);
 
         return super.process(ctx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java
index 41558c2863d03..dc17cbfbce548 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientObjectResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache get request.
@@ -38,6 +39,8 @@ public ClientCacheGetRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ);
+
         Object val = cache(ctx).get(key());
 
         return new ClientObjectResponse(requestId(), val);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java
index ba185bf7415d8..474c206b8cae3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetSizeRequest.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientLongResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache size request.
@@ -50,6 +51,8 @@ public ClientCacheGetSizeRequest(BinaryRawReader reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ);
+
         long res = cache(ctx).sizeLong(modes);
 
         return new ClientLongResponse(requestId(), res);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java
index 28a7fa57e3ee5..57e31443b474a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutAllRequest.java
@@ -23,6 +23,7 @@
 
 import java.util.LinkedHashMap;
 import java.util.Map;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * PutAll request.
@@ -50,6 +51,8 @@ public ClientCachePutAllRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_PUT);
+
         cache(ctx).putAll(map);
 
         return super.process(ctx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java
index 4dd2cde58ce06..ec81bc0c0fe48 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache put if absent request.
@@ -38,6 +39,8 @@ public ClientCachePutIfAbsentRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
         boolean res = cache(ctx).putIfAbsent(key(), val());
 
         return new ClientBooleanResponse(requestId(), res);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java
index 2c396b7ede87a..116460eece965 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java
@@ -20,6 +20,7 @@
 import org.apache.ignite.internal.binary.BinaryRawReaderEx;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache put request.
@@ -37,6 +38,8 @@ public ClientCachePutRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_PUT);
+
         cache(ctx).put(key(), val());
 
         return super.process(ctx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java
index f5adc6378912e..d90d873968105 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveAllRequest.java
@@ -20,6 +20,7 @@
 import org.apache.ignite.binary.BinaryRawReader;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache removeAll request.
@@ -37,6 +38,8 @@ public ClientCacheRemoveAllRequest(BinaryRawReader reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
         cache(ctx).removeAll();
 
         return super.process(ctx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java
index b86f2f8895d64..26c191f5b5553 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache remove request with value.
@@ -38,6 +39,8 @@ public ClientCacheRemoveIfEqualsRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_REMOVE);
+
         boolean res = cache(ctx).remove(key(), val());
 
         return new ClientBooleanResponse(requestId(), res);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java
index a68c32730f4fe..5af9743b3cac3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Remove request.
@@ -38,6 +39,8 @@ public ClientCacheRemoveKeyRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
         boolean val = cache(ctx).remove(key());
 
         return new ClientBooleanResponse(requestId(), val);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java
index 043b5688a3f43..62dea00201af8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeysRequest.java
@@ -20,6 +20,7 @@
 import org.apache.ignite.internal.binary.BinaryRawReaderEx;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Remove keys request.
@@ -37,6 +38,8 @@ public ClientCacheRemoveKeysRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_REMOVE);
+
         cache(ctx).removeAll(keys());
 
         return super.process(ctx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java
index 8645fbb817322..056367d71d2a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache replace request.
@@ -43,6 +44,8 @@ public ClientCacheReplaceIfEqualsRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
         boolean res = cache(ctx).replace(key(), val(), newVal);
 
         return new ClientBooleanResponse(requestId(), res);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java
index bd7a642bb39e0..ea04593e7cf15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientBooleanResponse;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache replace request.
@@ -38,6 +39,8 @@ public ClientCacheReplaceRequest(BinaryRawReaderEx reader) {
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ, SecurityPermission.CACHE_PUT);
+
         boolean res = cache(ctx).replace(key(), val());
 
         return new ClientBooleanResponse(requestId(), res);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java
index 52b799f345120..9e2d1f1f29298 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRequest.java
@@ -24,6 +24,8 @@
 import org.apache.ignite.internal.processors.platform.client.ClientRequest;
 import org.apache.ignite.internal.processors.platform.client.ClientStatus;
 import org.apache.ignite.internal.processors.platform.client.IgniteClientException;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Cache get request.
@@ -119,4 +121,34 @@ public static DynamicCacheDescriptor cacheDescriptor(ClientConnectionContext ctx
     protected int cacheId() {
         return cacheId;
     }
+
+    /** {@inheritDoc} */
+    protected void authorize(ClientConnectionContext ctx, SecurityPermission perm) {
+        SecurityContext secCtx = ctx.securityContext();
+
+        if (secCtx != null) {
+            DynamicCacheDescriptor cacheDesc = cacheDescriptor(ctx, cacheId);
+
+            runWithSecurityExceptionHandler(() -> {
+                ctx.kernalContext().security().authorize(cacheDesc.cacheName(), perm, secCtx);
+            });
+        }
+    }
+
+    /**
+     * Authorize for multiple permissions.
+     */
+    protected void authorize(ClientConnectionContext ctx, SecurityPermission... perm)
+        throws IgniteClientException {
+        SecurityContext secCtx = ctx.securityContext();
+
+        if (secCtx != null) {
+            DynamicCacheDescriptor cacheDesc = cacheDescriptor(ctx, cacheId);
+
+            runWithSecurityExceptionHandler(() -> {
+                for (SecurityPermission p : perm)
+                    ctx.kernalContext().security().authorize(cacheDesc.cacheName(), p, secCtx);
+            });
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java
index 26ab236e8be1e..70b6966e999c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
 import org.apache.ignite.internal.processors.platform.utils.PlatformUtils;
 import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Scan query request.
@@ -80,6 +81,8 @@ public ClientCacheScanQueryRequest(BinaryRawReaderEx reader) {
 
     /** {@inheritDoc} */
     @Override public ClientResponse process(ClientConnectionContext ctx) {
+        authorize(ctx, SecurityPermission.CACHE_READ);
+
         IgniteCache cache = filterPlatform == FILTER_PLATFORM_JAVA && !isKeepBinary() ? rawCache(ctx) : cache(ctx);
 
         ScanQuery qry = new ScanQuery()
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
index cfd4498cb4852..3aa95bf2a1a37 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
 import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Sql query request.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java
index 8c21be14a800a..40693e74bd5d6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
 
 import java.util.concurrent.TimeUnit;
+import org.apache.ignite.plugin.security.SecurityPermission;
 
 /**
  * Sql query request.
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java
index 91f337929af50..9f2cfe2ddd4ad 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/AuthenticationContext.java
@@ -21,6 +21,7 @@
 import java.util.Collections;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.ignite.internal.processors.authentication.AuthorizationContext;
 
 /**
  * Authentication context.
@@ -41,6 +42,12 @@ public class AuthenticationContext {
     /** */
     private Map<String, Object> nodeAttrs;
 
+    /** Authorization context. */
+    private AuthorizationContext athrCtx;
+
+    /** True if this is a client node context. */
+    private boolean client;
+
     /**
      * Gets subject type.
      *
@@ -130,4 +137,37 @@ public Map<String, Object> nodeAttributes() {
     public void nodeAttributes(Map<String, Object> nodeAttrs) {
         this.nodeAttrs = nodeAttrs;
     }
+
+    /**
+     * @return Native Apache Ignite authorization context acquired after authentication or {@code null} if native
+     * Ignite authentication is not used.
+     */
+    public AuthorizationContext authorizationContext(){
+        return athrCtx;
+    }
+
+    /**
+     * Set authorization context acquired after native Apache Ignite authentication.
+     */
+    public AuthenticationContext authorizationContext(AuthorizationContext newVal) {
+        athrCtx = newVal;
+
+        return this;
+    }
+
+    /**
+     * @return {@code true} if this is a client node context.
+     */
+    public boolean isClient() {
+        return client;
+    }
+
+    /**
+     * Sets flag indicating if this is client node context.
+     */
+    public AuthenticationContext setClient(boolean newVal) {
+        client = newVal;
+
+        return this;
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
index 54361614e62a4..bca667ddb3d7a 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java
@@ -64,7 +64,16 @@ public enum SecurityPermission {
     SERVICE_CANCEL,
 
     /** Service invoke permission. */
-    SERVICE_INVOKE;
+    SERVICE_INVOKE,
+
+    /** Cache create permission. */
+    CACHE_CREATE,
+
+    /** Cache create permission. */
+    CACHE_DESTROY,
+
+    /** Join as server node permission. */
+    JOIN_AS_SERVER;
 
     /** Enumerated values. */
     private static final SecurityPermission[] VALS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 7bf37e1b0ac43..6d3864e704215 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -99,6 +99,7 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.security.SecurityCredentials;
+import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.plugin.security.SecurityPermissionSet;
 import org.apache.ignite.spi.IgniteNodeValidationResult;
 import org.apache.ignite.spi.IgniteSpiContext;
@@ -3559,6 +3560,8 @@ else if (log.isDebugEnabled())
                             return;
                         }
                         else {
+                            String authFailedMsg = null;
+
                             if (!(subj instanceof Serializable)) {
                                 // Node has not pass authentication.
                                 LT.warn(log, "Authentication subject is not Serializable [nodeId=" + node.id() +
@@ -3567,9 +3570,16 @@ else if (log.isDebugEnabled())
                                         ", addrs=" +
                                         U.addressesAsString(node) + ']');
 
+                                authFailedMsg = "Authentication subject is not serializable";
+                            }
+                            else if (!node.isClient() &&
+                                !subj.systemOperationAllowed(SecurityPermission.JOIN_AS_SERVER))
+                                authFailedMsg = "Node is not authorised to join as a server node";
+
+                            if (authFailedMsg != null) {
                                 // Always output in debug.
                                 if (log.isDebugEnabled())
-                                    log.debug("Authentication subject is not serializable [nodeId=" + node.id() +
+                                    log.debug(authFailedMsg + " [nodeId=" + node.id() +
                                         ", addrs=" + U.addressesAsString(node));
 
                                 try {

From b6ad3705c1e68683b72d2237037af66fea23a7ae Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Wed, 11 Apr 2018 16:44:33 +0300
Subject: [PATCH 027/543] IGNITE-8148: JDBC thin: semicolon as delimiter for
 properties. This closes #3794.

---
 .../jdbc/thin/JdbcThinConnectionSelfTest.java | 233 ++++++++++++++----
 .../jdbc/thin/ConnectionPropertiesImpl.java   | 161 ++++++++----
 2 files changed, 300 insertions(+), 94 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
index 14b91b260c9fe..ed0b32403db3a 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
@@ -184,6 +184,38 @@ public void testSocketBuffers() throws Exception {
         }
     }
 
+    /**
+     * Test invalid socket buffer sizes with semicolon.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSocketBuffersSemicolon() throws Exception {
+        final int dfltDufSize = 64 * 1024;
+
+        assertInvalid("jdbc:ignite:thin://127.0.0.1;socketSendBuffer=-1",
+            "Property cannot be lower than 0 [name=socketSendBuffer, value=-1]");
+
+        assertInvalid("jdbc:ignite:thin://127.0.0.1;socketReceiveBuffer=-1",
+            "Property cannot be lower than 0 [name=socketReceiveBuffer, value=-1]");
+
+        // Note that SO_* options are hints, so we check that value is equals to either what we set or to default.
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;socketSendBuffer=1024")) {
+            assertEquals(1024, io(conn).connectionProperties().getSocketSendBuffer());
+            assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketReceiveBuffer());
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;socketReceiveBuffer=1024")) {
+            assertEquals(dfltDufSize, io(conn).connectionProperties().getSocketSendBuffer());
+            assertEquals(1024, io(conn).connectionProperties().getSocketReceiveBuffer());
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;" +
+            "socketSendBuffer=1024;socketReceiveBuffer=2048")) {
+            assertEquals(1024, io(conn).connectionProperties().getSocketSendBuffer());
+            assertEquals(2048, io(conn).connectionProperties().getSocketReceiveBuffer());
+        }
+    }
+
     /**
      * Test SQL hints.
      *
@@ -191,79 +223,97 @@ public void testSocketBuffers() throws Exception {
      */
     public void testSqlHints() throws Exception {
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1")) {
-            assertFalse(io(conn).connectionProperties().isDistributedJoins());
-            assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
-            assertFalse(io(conn).connectionProperties().isCollocated());
-            assertFalse(io(conn).connectionProperties().isReplicatedOnly());
-            assertFalse(io(conn).connectionProperties().isLazy());
-            assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+            assertHints(conn, false, false, false, false, false, false);
         }
 
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?distributedJoins=true")) {
-            assertTrue(io(conn).connectionProperties().isDistributedJoins());
-            assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
-            assertFalse(io(conn).connectionProperties().isCollocated());
-            assertFalse(io(conn).connectionProperties().isReplicatedOnly());
-            assertFalse(io(conn).connectionProperties().isLazy());
-            assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+            assertHints(conn, true, false, false, false, false, false);
         }
 
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?enforceJoinOrder=true")) {
-            assertFalse(io(conn).connectionProperties().isDistributedJoins());
-            assertTrue(io(conn).connectionProperties().isEnforceJoinOrder());
-            assertFalse(io(conn).connectionProperties().isCollocated());
-            assertFalse(io(conn).connectionProperties().isReplicatedOnly());
-            assertFalse(io(conn).connectionProperties().isLazy());
-            assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+            assertHints(conn, false, true, false, false, false, false);
         }
 
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?collocated=true")) {
-            assertFalse(io(conn).connectionProperties().isDistributedJoins());
-            assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
-            assertTrue(io(conn).connectionProperties().isCollocated());
-            assertFalse(io(conn).connectionProperties().isReplicatedOnly());
-            assertFalse(io(conn).connectionProperties().isLazy());
-            assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+            assertHints(conn, false, false, true, false, false, false);
         }
 
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?replicatedOnly=true")) {
-            assertFalse(io(conn).connectionProperties().isDistributedJoins());
-            assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
-            assertFalse(io(conn).connectionProperties().isCollocated());
-            assertTrue(io(conn).connectionProperties().isReplicatedOnly());
-            assertFalse(io(conn).connectionProperties().isLazy());
-            assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+            assertHints(conn, false, false, false, true, false, false);
         }
 
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?lazy=true")) {
-            assertFalse(io(conn).connectionProperties().isDistributedJoins());
-            assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
-            assertFalse(io(conn).connectionProperties().isCollocated());
-            assertFalse(io(conn).connectionProperties().isReplicatedOnly());
-            assertTrue(io(conn).connectionProperties().isLazy());
-            assertFalse(io(conn).connectionProperties().isSkipReducerOnUpdate());
+            assertHints(conn, false, false, false, false, true, false);
         }
 
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?skipReducerOnUpdate=true")) {
-            assertFalse(io(conn).connectionProperties().isDistributedJoins());
-            assertFalse(io(conn).connectionProperties().isEnforceJoinOrder());
-            assertFalse(io(conn).connectionProperties().isCollocated());
-            assertFalse(io(conn).connectionProperties().isReplicatedOnly());
-            assertFalse(io(conn).connectionProperties().isLazy());
-            assertTrue(io(conn).connectionProperties().isSkipReducerOnUpdate());
+            assertHints(conn, false, false, false, false, false, true);
         }
 
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1?distributedJoins=true&" +
             "enforceJoinOrder=true&collocated=true&replicatedOnly=true&lazy=true&skipReducerOnUpdate=true")) {
-            assertTrue(io(conn).connectionProperties().isDistributedJoins());
-            assertTrue(io(conn).connectionProperties().isEnforceJoinOrder());
-            assertTrue(io(conn).connectionProperties().isCollocated());
-            assertTrue(io(conn).connectionProperties().isReplicatedOnly());
-            assertTrue(io(conn).connectionProperties().isLazy());
-            assertTrue(io(conn).connectionProperties().isSkipReducerOnUpdate());
+            assertHints(conn, true, true, true, true, true, true);
         }
     }
 
+    /**
+     * Test SQL hints with semicolon.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSqlHintsSemicolon() throws Exception {
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;distributedJoins=true")) {
+            assertHints(conn, true, false, false, false, false, false);
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;enforceJoinOrder=true")) {
+            assertHints(conn, false, true, false, false, false, false);
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;collocated=true")) {
+            assertHints(conn, false, false, true, false, false, false);
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;replicatedOnly=true")) {
+            assertHints(conn, false, false, false, true, false, false);
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;lazy=true")) {
+            assertHints(conn, false, false, false, false, true, false);
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;skipReducerOnUpdate=true")) {
+            assertHints(conn, false, false, false, false, false, true);
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;distributedJoins=true;" +
+            "enforceJoinOrder=true;collocated=true;replicatedOnly=true;lazy=true;skipReducerOnUpdate=true")) {
+            assertHints(conn, true, true, true, true, true, true);
+        }
+    }
+
+    /**
+     * Assert hints.
+     *
+     * @param conn Connection.
+     * @param distributedJoins Distributed joins.
+     * @param enforceJoinOrder Enforce join order.
+     * @param collocated Co-located.
+     * @param replicatedOnly Replicated only.
+     * @param lazy Lazy.
+     * @param skipReducerOnUpdate Skip reducer on update.
+     * @throws Exception If failed.
+     */
+    private void assertHints(Connection conn, boolean distributedJoins, boolean enforceJoinOrder, boolean collocated,
+        boolean replicatedOnly, boolean lazy, boolean skipReducerOnUpdate)throws Exception {
+        assertEquals(distributedJoins, io(conn).connectionProperties().isDistributedJoins());
+        assertEquals(enforceJoinOrder, io(conn).connectionProperties().isEnforceJoinOrder());
+        assertEquals(collocated, io(conn).connectionProperties().isCollocated());
+        assertEquals(replicatedOnly, io(conn).connectionProperties().isReplicatedOnly());
+        assertEquals(lazy, io(conn).connectionProperties().isLazy());
+        assertEquals(skipReducerOnUpdate, io(conn).connectionProperties().isSkipReducerOnUpdate());
+    }
+
     /**
      * Test TCP no delay property handling.
      *
@@ -303,6 +353,41 @@ public void testTcpNoDelay() throws Exception {
         }
     }
 
+    /**
+     * Test TCP no delay property handling with semicolon.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTcpNoDelaySemicolon() throws Exception {
+        assertInvalid("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=0",
+            "Invalid property value. [name=tcpNoDelay, val=0, choices=[true, false]]");
+
+        assertInvalid("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=1",
+            "Invalid property value. [name=tcpNoDelay, val=1, choices=[true, false]]");
+
+        assertInvalid("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=false1",
+            "Invalid property value. [name=tcpNoDelay, val=false1, choices=[true, false]]");
+
+        assertInvalid("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=true1",
+            "Invalid property value. [name=tcpNoDelay, val=true1, choices=[true, false]]");
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=true")) {
+            assertTrue(io(conn).connectionProperties().isTcpNoDelay());
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=True")) {
+            assertTrue(io(conn).connectionProperties().isTcpNoDelay());
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=false")) {
+            assertFalse(io(conn).connectionProperties().isTcpNoDelay());
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;tcpNoDelay=False")) {
+            assertFalse(io(conn).connectionProperties().isTcpNoDelay());
+        }
+    }
+
     /**
      * Test autoCloseServerCursor property handling.
      *
@@ -339,6 +424,38 @@ public void testAutoCloseServerCursorProperty() throws Exception {
         }
     }
 
+    /**
+     * Test autoCloseServerCursor property handling with semicolon.
+     *
+     * @throws Exception If failed.
+     */
+    public void testAutoCloseServerCursorPropertySemicolon() throws Exception {
+        String url = "jdbc:ignite:thin://127.0.0.1;autoCloseServerCursor";
+
+        String err = "Invalid property value. [name=autoCloseServerCursor";
+
+        assertInvalid(url + "=0", err);
+        assertInvalid(url + "=1", err);
+        assertInvalid(url + "=false1", err);
+        assertInvalid(url + "=true1", err);
+
+        try (Connection conn = DriverManager.getConnection(url + "=true")) {
+            assertTrue(io(conn).connectionProperties().isAutoCloseServerCursor());
+        }
+
+        try (Connection conn = DriverManager.getConnection(url + "=True")) {
+            assertTrue(io(conn).connectionProperties().isAutoCloseServerCursor());
+        }
+
+        try (Connection conn = DriverManager.getConnection(url + "=false")) {
+            assertFalse(io(conn).connectionProperties().isAutoCloseServerCursor());
+        }
+
+        try (Connection conn = DriverManager.getConnection(url + "=False")) {
+            assertFalse(io(conn).connectionProperties().isAutoCloseServerCursor());
+        }
+    }
+
     /**
      * Test schema property in URL.
      *
@@ -361,6 +478,25 @@ public void testSchema() throws Exception {
         }
     }
 
+    /**
+     * Test schema property in URL with semicolon.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSchemaSemicolon() throws Exception {
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;schema=public")) {
+            assertEquals("Invalid schema", "PUBLIC", conn.getSchema());
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;schema=\"" + DEFAULT_CACHE_NAME + '"')) {
+            assertEquals("Invalid schema", DEFAULT_CACHE_NAME, conn.getSchema());
+        }
+
+        try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1;schema=_not_exist_schema_")) {
+            assertEquals("Invalid schema", "_NOT_EXIST_SCHEMA_", conn.getSchema());
+        }
+    }
+
     /**
      * Get client socket for connection.
      *
@@ -1010,6 +1146,7 @@ public void testGetSetTransactionIsolation() throws Exception {
             // Invalid parameter value
             GridTestUtils.assertThrows(log,
                 new Callable<Object>() {
+                    @SuppressWarnings("MagicConstant")
                     @Override public Object call() throws Exception {
                         conn.setTransactionIsolation(-1);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
index 5d770054f46d4..86dc2980ecd9a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/ConnectionPropertiesImpl.java
@@ -23,8 +23,6 @@
 import java.util.Arrays;
 import java.util.Properties;
 import java.util.StringTokenizer;
-import javax.naming.RefAddr;
-import javax.naming.Reference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.ClientConnectorConfiguration;
 import org.apache.ignite.internal.processors.odbc.SqlStateCode;
@@ -44,6 +42,9 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
     /** Default socket buffer size. */
     private static final int DFLT_SOCK_BUFFER_SIZE = 64 * 1024;
 
+    /** Property: schema. */
+    private static final String PROP_SCHEMA = "schema";
+
     /** Connection URL. */
     private String url;
 
@@ -51,7 +52,7 @@ public class ConnectionPropertiesImpl implements ConnectionProperties, Serializa
     private HostAndPortRange [] addrs;
 
     /** Schema name. Hidden property. Is used to set default schema name part of the URL. */
-    private StringProperty schema = new StringProperty("schema",
+    private StringProperty schema = new StringProperty(PROP_SCHEMA,
         "Schema name of the connection", "PUBLIC", null, false, null);
 
     /** Distributed joins property. */
@@ -487,21 +488,113 @@ private void parseUrl(String url, Properties props) throws SQLException {
 
         String nakedUrl = url.substring(JdbcThinUtils.URL_PREFIX.length()).trim();
 
-        int pathPartEndPos = nakedUrl.indexOf('?');
+        parseUrl0(nakedUrl, props);
+    }
+
+    /**
+     * Parse naked URL (i.e. without {@link JdbcThinUtils#URL_PREFIX}).
+     *
+     * @param url Naked URL.
+     * @param props Properties.
+     * @throws SQLException If failed.
+     */
+    private void parseUrl0(String url, Properties props) throws SQLException {
+        // Determine mode - semicolon or ampersand.
+        int semicolonPos = url.indexOf(";");
+        int slashPos = url.indexOf("/");
+        int queryPos = url.indexOf("?");
+
+        boolean semicolonMode;
+
+        if (semicolonPos == -1 && slashPos == -1 && queryPos == -1)
+            // No special char -> any mode could be used, choose semicolon for simplicity.
+            semicolonMode = true;
+        else {
+            if (semicolonPos != -1) {
+                // Use semicolon mode if it appears earlier than slash or query.
+                semicolonMode =
+                    (slashPos == -1 || semicolonPos < slashPos) && (queryPos == -1 || semicolonPos < queryPos);
+            }
+            else
+                // Semicolon is not found.
+                semicolonMode = false;
+        }
+
+        if (semicolonMode)
+            parseUrlWithSemicolon(url, props);
+        else
+            parseUrlWithQuery(url, props);
+    }
+
+    /**
+     * Parse URL in semicolon mode.
+     *
+     * @param url Naked URL
+     * @param props Properties.
+     * @throws SQLException If failed.
+     */
+    private void parseUrlWithSemicolon(String url, Properties props) throws SQLException {
+        int pathPartEndPos = url.indexOf(';');
 
         if (pathPartEndPos == -1)
-            pathPartEndPos = nakedUrl.length();
+            pathPartEndPos = url.length();
 
-        String pathPart = nakedUrl.substring(0, pathPartEndPos);
+        String pathPart = url.substring(0, pathPartEndPos);
 
         String paramPart = null;
 
-        if (pathPartEndPos > 0 && pathPartEndPos < nakedUrl.length())
-            paramPart = nakedUrl.substring(pathPartEndPos + 1, nakedUrl.length());
+        if (pathPartEndPos > 0 && pathPartEndPos < url.length())
+            paramPart = url.substring(pathPartEndPos + 1, url.length());
+
+        parseEndpoints(pathPart);
+
+        if (!F.isEmpty(paramPart))
+            parseParameters(paramPart, props, ";");
+    }
+
+    /**
+     * Parse URL in query mode.
+     *
+     * @param url Naked URL
+     * @param props Properties.
+     * @throws SQLException If failed.
+     */
+    private void parseUrlWithQuery(String url, Properties props) throws SQLException {
+        int pathPartEndPos = url.indexOf('?');
+
+        if (pathPartEndPos == -1)
+            pathPartEndPos = url.length();
+
+        String pathPart = url.substring(0, pathPartEndPos);
+
+        String paramPart = null;
+
+        if (pathPartEndPos > 0 && pathPartEndPos < url.length())
+            paramPart = url.substring(pathPartEndPos + 1, url.length());
 
         String[] pathParts = pathPart.split("/");
 
-        String [] endpoints = pathParts[0].split(",");
+        parseEndpoints(pathParts[0]);
+
+        if (pathParts.length > 2) {
+            throw new SQLException("Invalid URL format (only schema name is allowed in URL path parameter " +
+                "'host:port[/schemaName]'): " + this.url, SqlStateCode.CLIENT_CONNECTION_FAILED);
+        }
+
+        setSchema(pathParts.length == 2 ? pathParts[1] : null);
+
+        if (!F.isEmpty(paramPart))
+            parseParameters(paramPart, props, "&");
+    }
+
+    /**
+     * Parse endpoints.
+     *
+     * @param endpointStr Endpoint string.
+     * @throws SQLException If failed.
+     */
+    private void parseEndpoints(String endpointStr) throws SQLException {
+        String [] endpoints = endpointStr.split(",");
 
         if (endpoints.length > 0)
             addrs = new HostAndPortRange[endpoints.length];
@@ -519,16 +612,6 @@ private void parseUrl(String url, Properties props) throws SQLException {
 
         if (F.isEmpty(addrs) || F.isEmpty(addrs[0].host()))
             throw new SQLException("Host name is empty", SqlStateCode.CLIENT_CONNECTION_FAILED);
-
-        if (pathParts.length > 2) {
-            throw new SQLException("Invalid URL format (only schema name is allowed in URL path parameter " +
-                "'host:port[/schemaName]'): " + url, SqlStateCode.CLIENT_CONNECTION_FAILED);
-        }
-
-        setSchema(pathParts.length == 2 ? pathParts[1] : null);
-
-        if (!F.isEmpty(paramPart))
-            parseParameters(paramPart, props);
     }
 
     /**
@@ -536,10 +619,11 @@ private void parseUrl(String url, Properties props) throws SQLException {
      *
      * @param paramStr Parameters string.
      * @param props Properties.
+     * @param delimChar Delimiter character.
      * @throws SQLException If failed.
      */
-    private void parseParameters(String paramStr, Properties props) throws SQLException {
-        StringTokenizer st = new StringTokenizer(paramStr, "&");
+    private void parseParameters(String paramStr, Properties props, String delimChar) throws SQLException {
+        StringTokenizer st = new StringTokenizer(paramStr, delimChar);
 
         boolean insideBrace = false;
 
@@ -553,8 +637,8 @@ private void parseParameters(String paramStr, Properties props) throws SQLExcept
                 int eqSymPos = token.indexOf('=');
 
                 if (eqSymPos < 0) {
-                    throw new SQLException("Invalid parameter format " +
-                        "(URL properties format: key0=value0&key1=value1&... etc. pair: " + token);
+                    throw new SQLException("Invalid parameter format (should be \"key1=val1" + delimChar +
+                        "key2=val2" + delimChar + "...\"): " + token);
                 }
 
                 if (eqSymPos == token.length())
@@ -570,7 +654,7 @@ private void parseParameters(String paramStr, Properties props) throws SQLExcept
                 }
             }
             else
-                val += "&" + token;
+                val += delimChar + token;
 
             if (val.endsWith("}")) {
                 insideBrace = false;
@@ -587,22 +671,24 @@ private void parseParameters(String paramStr, Properties props) throws SQLExcept
                 if (key.isEmpty() || val.isEmpty())
                     throw new SQLException("Invalid parameter format (key and value cannot be empty): " + token);
 
-                props.setProperty(PROP_PREFIX + key, val);
+                if (PROP_SCHEMA.equalsIgnoreCase(key))
+                    setSchema(val);
+                else
+                    props.setProperty(PROP_PREFIX + key, val);
             }
         }
     }
 
-
     /**
      * @return Driver's properties info array.
      */
     public DriverPropertyInfo[] getDriverPropertyInfo() {
-        DriverPropertyInfo[] dpis = new DriverPropertyInfo[propsArray.length];
+        DriverPropertyInfo[] infos = new DriverPropertyInfo[propsArray.length];
 
         for (int i = 0; i < propsArray.length; ++i)
-            dpis[i] = propsArray[i].getDriverPropertyInfo();
+            infos[i] = propsArray[i].getDriverPropertyInfo();
 
-        return dpis;
+        return infos;
     }
 
     /**
@@ -740,23 +826,6 @@ protected void checkChoices(String strVal) throws SQLException {
             }
         }
 
-        /**
-         * @param ref Reference object.
-         * @throws SQLException On error.
-         */
-        void init(Reference ref) throws SQLException {
-            RefAddr refAddr = ref.get(name);
-
-            if (refAddr != null) {
-                String str = (String) refAddr.getContent();
-
-                if (validator != null)
-                    validator.validate(str);
-
-                init(str);
-            }
-        }
-
         /**
          * @param str String representation of the
          * @throws SQLException on error.

From e6c30e17c6f0f1852fa781078ee54ccf8c654846 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 11 Apr 2018 14:12:50 +0300
Subject: [PATCH 028/543] IGNITE-7871 Check local join future on error. - Fixes
 #3793.

Signed-off-by: dpavlov <dpavlov@apache.org>
---
 .../distributed/dht/preloader/latch/ExchangeLatchManager.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index c205cb14e40d9..404f88f9f0695 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -104,7 +104,8 @@ public ExchangeLatchManager(GridKernalContext ctx) {
 
             // First coordinator initialization.
             ctx.discovery().localJoinFuture().listen(f -> {
-                this.coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+                if (f.error() == null)
+                    this.coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
             });
 
             ctx.event().addDiscoveryEventListener((e, cache) -> {

From 2769981a5df64f3cd0c38b7599c49580c66192fa Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Wed, 11 Apr 2018 18:24:51 +0300
Subject: [PATCH 029/543] IGNITE-6892 OOM should be covered by failure handling

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../apache/ignite/IgniteSystemProperties.java |  15 ++
 .../apache/ignite/internal/IgnitionEx.java    |  50 +++-
 .../discovery/GridDiscoveryManager.java       |   3 +
 .../processors/cache/WalStateManager.java     |   8 +-
 .../continuous/GridContinuousProcessor.java   |   3 +
 .../datastreamer/DataStreamProcessor.java     |   3 +
 .../processors/failure/FailureProcessor.java  |  11 +
 .../processors/job/GridJobWorker.java         |   8 +-
 .../service/GridServiceProcessor.java         |  15 +-
 .../IgniteStripedThreadPoolExecutor.java      |   8 +-
 .../ignite/thread/IgniteThreadFactory.java    |  30 ++-
 .../thread/IgniteThreadPoolExecutor.java      |  12 +-
 .../ignite/thread/OomExceptionHandler.java    |  44 +++
 .../ignite/failure/OomFailureHandlerTest.java | 255 ++++++++++++++++++
 .../testsuites/IgniteBasicTestSuite.java      |   2 +
 15 files changed, 437 insertions(+), 30 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/thread/OomExceptionHandler.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index ac7947b7e45aa..437f49f6a0a9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -862,6 +862,21 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_BPLUS_TREE_LOCK_RETRIES = "IGNITE_BPLUS_TREE_LOCK_RETRIES";
 
+    /**
+     * Amount of memory reserved in the heap at node start, which can be dropped to increase the chances of success when
+     * handling OutOfMemoryError.
+     *
+     * Default is {@code 64kb}.
+     */
+    public static final String IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE = "IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE";
+
+    /**
+     * The threshold of uneven distribution above which partition distribution will be logged.
+     *
+     * The default is '50', that means: warn about nodes with 50+% difference.
+     */
+    public static final String IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD = "IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index c0de08050fe60..e140609aac497 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -20,6 +20,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.lang.Thread.UncaughtExceptionHandler;
 import java.lang.management.ManagementFactory;
 import java.lang.reflect.Constructor;
 import java.net.MalformedURLException;
@@ -88,6 +89,7 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -1764,6 +1766,13 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
 
             validateThreadPoolSize(cfg.getPublicThreadPoolSize(), "public");
 
+            UncaughtExceptionHandler oomeHnd = new UncaughtExceptionHandler() {
+                @Override public void uncaughtException(Thread t, Throwable e) {
+                    if (grid != null && X.hasCause(e, OutOfMemoryError.class))
+                        grid.context().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+                }
+            };
+
             execSvc = new IgniteThreadPoolExecutor(
                 "pub",
                 cfg.getIgniteInstanceName(),
@@ -1771,7 +1780,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 cfg.getPublicThreadPoolSize(),
                 DFLT_THREAD_KEEP_ALIVE_TIME,
                 new LinkedBlockingQueue<Runnable>(),
-                GridIoPolicy.PUBLIC_POOL);
+                GridIoPolicy.PUBLIC_POOL,
+                oomeHnd);
 
             execSvc.allowCoreThreadTimeOut(true);
 
@@ -1784,7 +1794,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 cfg.getServiceThreadPoolSize(),
                 DFLT_THREAD_KEEP_ALIVE_TIME,
                 new LinkedBlockingQueue<Runnable>(),
-                GridIoPolicy.SERVICE_POOL);
+                GridIoPolicy.SERVICE_POOL,
+                oomeHnd);
 
             svcExecSvc.allowCoreThreadTimeOut(true);
 
@@ -1797,7 +1808,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 cfg.getSystemThreadPoolSize(),
                 DFLT_THREAD_KEEP_ALIVE_TIME,
                 new LinkedBlockingQueue<Runnable>(),
-                GridIoPolicy.SYSTEM_POOL);
+                GridIoPolicy.SYSTEM_POOL,
+                oomeHnd);
 
             sysExecSvc.allowCoreThreadTimeOut(true);
 
@@ -1828,7 +1840,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 cfg.getManagementThreadPoolSize(),
                 DFLT_THREAD_KEEP_ALIVE_TIME,
                 new LinkedBlockingQueue<Runnable>(),
-                GridIoPolicy.MANAGEMENT_POOL);
+                GridIoPolicy.MANAGEMENT_POOL,
+                oomeHnd);
 
             mgmtExecSvc.allowCoreThreadTimeOut(true);
 
@@ -1844,7 +1857,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 cfg.getPeerClassLoadingThreadPoolSize(),
                 DFLT_THREAD_KEEP_ALIVE_TIME,
                 new LinkedBlockingQueue<Runnable>(),
-                GridIoPolicy.P2P_POOL);
+                GridIoPolicy.P2P_POOL,
+                oomeHnd);
 
             p2pExecSvc.allowCoreThreadTimeOut(true);
 
@@ -1879,7 +1893,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
             callbackExecSvc = new IgniteStripedThreadPoolExecutor(
                 cfg.getAsyncCallbackPoolSize(),
                 cfg.getIgniteInstanceName(),
-                "callback");
+                "callback",
+                oomeHnd);
 
             if (myCfg.getConnectorConfiguration() != null) {
                 validateThreadPoolSize(myCfg.getConnectorConfiguration().getThreadPoolSize(), "connector");
@@ -1890,7 +1905,9 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                     myCfg.getConnectorConfiguration().getThreadPoolSize(),
                     myCfg.getConnectorConfiguration().getThreadPoolSize(),
                     DFLT_THREAD_KEEP_ALIVE_TIME,
-                    new LinkedBlockingQueue<Runnable>()
+                    new LinkedBlockingQueue<Runnable>(),
+                    GridIoPolicy.UNDEFINED,
+                    oomeHnd
                 );
 
                 restExecSvc.allowCoreThreadTimeOut(true);
@@ -1905,7 +1922,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 myCfg.getUtilityCacheThreadPoolSize(),
                 myCfg.getUtilityCacheKeepAliveTime(),
                 new LinkedBlockingQueue<Runnable>(),
-                GridIoPolicy.UTILITY_CACHE_POOL);
+                GridIoPolicy.UTILITY_CACHE_POOL,
+                oomeHnd);
 
             utilityCacheExecSvc.allowCoreThreadTimeOut(true);
 
@@ -1916,7 +1934,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 1,
                 DFLT_THREAD_KEEP_ALIVE_TIME,
                 new LinkedBlockingQueue<Runnable>(),
-                GridIoPolicy.AFFINITY_POOL);
+                GridIoPolicy.AFFINITY_POOL,
+                oomeHnd);
 
             affExecSvc.allowCoreThreadTimeOut(true);
 
@@ -1930,7 +1949,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                     cpus * 2,
                     3000L,
                     new LinkedBlockingQueue<Runnable>(1000),
-                    GridIoPolicy.IDX_POOL
+                    GridIoPolicy.IDX_POOL,
+                    oomeHnd
                 );
             }
 
@@ -1943,7 +1963,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 cfg.getQueryThreadPoolSize(),
                 DFLT_THREAD_KEEP_ALIVE_TIME,
                 new LinkedBlockingQueue<Runnable>(),
-                GridIoPolicy.QUERY_POOL);
+                GridIoPolicy.QUERY_POOL,
+                oomeHnd);
 
             qryExecSvc.allowCoreThreadTimeOut(true);
 
@@ -1954,7 +1975,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 2,
                 DFLT_THREAD_KEEP_ALIVE_TIME,
                 new LinkedBlockingQueue<Runnable>(),
-                GridIoPolicy.SCHEMA_POOL);
+                GridIoPolicy.SCHEMA_POOL,
+                oomeHnd);
 
             schemaExecSvc.allowCoreThreadTimeOut(true);
 
@@ -1970,7 +1992,9 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                         execCfg.getSize(),
                         execCfg.getSize(),
                         DFLT_THREAD_KEEP_ALIVE_TIME,
-                        new LinkedBlockingQueue<Runnable>());
+                        new LinkedBlockingQueue<Runnable>(),
+                        GridIoPolicy.UNDEFINED,
+                        oomeHnd);
 
                     customExecSvcs.put(execCfg.getName(), exec);
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 400bb5fd28742..77c96573d0a63 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -130,6 +130,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThread;
+import org.apache.ignite.thread.OomExceptionHandler;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -924,6 +925,8 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) {
 
             segChkThread = new IgniteThread(segChkWrk);
 
+            segChkThread.setUncaughtExceptionHandler(new OomExceptionHandler(ctx));
+
             segChkThread.start();
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
index 0ac699f5a8fee..64a6819826357 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.thread.OomExceptionHandler;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 
@@ -473,7 +474,12 @@ public void onProposeExchange(WalStateProposeMessage msg) {
                                     // not-yet-flushed dirty pages have been logged.
                                     WalStateChangeWorker worker = new WalStateChangeWorker(msg, cpFut);
 
-                                    new IgniteThread(worker).start();
+                                    IgniteThread thread = new IgniteThread(worker);
+
+                                    thread.setUncaughtExceptionHandler(new OomExceptionHandler(
+                                        cctx.kernalContext()));
+
+                                    thread.start();
                                 }
                                 else {
                                     // Disable: not-yet-flushed operations are not logged, so wait for them
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index cebe4b177e300..2d48b7d9b16fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -88,6 +88,7 @@
 import org.apache.ignite.spi.discovery.DiscoveryDataBag.GridDiscoveryData;
 import org.apache.ignite.spi.discovery.DiscoveryDataBag.JoiningNodeDiscoveryData;
 import org.apache.ignite.thread.IgniteThread;
+import org.apache.ignite.thread.OomExceptionHandler;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
@@ -1727,6 +1728,8 @@ private boolean registerHandler(final UUID nodeId,
                     }
                 });
 
+                checker.setUncaughtExceptionHandler(new OomExceptionHandler(ctx));
+
                 bufCheckThreads.put(routineId, checker);
 
                 checker.start();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
index 8b984c05c1988..e63d7d4d28bc1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessor.java
@@ -44,6 +44,7 @@
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.stream.StreamReceiver;
 import org.apache.ignite.thread.IgniteThread;
+import org.apache.ignite.thread.OomExceptionHandler;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.Collection;
@@ -125,6 +126,8 @@ public DataStreamProcessor(GridKernalContext ctx) {
             }
         });
 
+        flusher.setUncaughtExceptionHandler(new OomExceptionHandler(ctx));
+
         flusher.start();
 
         if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
index 615fb9f55456f..0234e84345580 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
@@ -19,12 +19,14 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.failure.FailureHandler;
 import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
@@ -40,6 +42,9 @@ public class FailureProcessor extends GridProcessorAdapter {
     /** Failure context. */
     private volatile FailureContext failureCtx;
 
+    /** Reserve buffer, which can be dropped to handle OOME. */
+    private volatile byte[] reserveBuf;
+
     /**
      * @param ctx Context.
      */
@@ -56,6 +61,9 @@ public FailureProcessor(GridKernalContext ctx) {
         if (hnd == null)
             hnd = getDefaultFailureHandler();
 
+        reserveBuf = new byte[IgniteSystemProperties.getInteger(
+            IgniteSystemProperties.IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE, 64 * 1024)];
+
         assert hnd != null;
 
         this.hnd = hnd;
@@ -102,6 +110,9 @@ public synchronized void process(FailureContext failureCtx, FailureHandler hnd)
         U.error(ignite.log(), "Critical failure. Will be handled accordingly to configured handler [hnd=" +
             hnd.getClass() + ", failureCtx=" + failureCtx + ']', failureCtx.error());
 
+        if (reserveBuf != null && X.hasCause(failureCtx.error(), OutOfMemoryError.class))
+            reserveBuf = null;
+
         boolean invalidated = hnd.onFailure(ignite, failureCtx);
 
         if (invalidated) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
index 6d2e621c3b9af..f7c07f516f413 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobWorker.java
@@ -36,6 +36,8 @@
 import org.apache.ignite.compute.ComputeJobMasterLeaveAware;
 import org.apache.ignite.compute.ComputeUserUndeclaredException;
 import org.apache.ignite.events.JobEvent;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.igfs.IgfsOutOfSpaceException;
 import org.apache.ignite.internal.GridInternalException;
 import org.apache.ignite.internal.GridJobContextImpl;
@@ -603,9 +605,13 @@ else if (X.hasCause(e, GridServiceNotFoundException.class) ||
                         X.hasCause(e, ClusterTopologyCheckedException.class))
                         // Should be throttled, because GridServiceProxy continuously retry getting service.
                         LT.error(log, e, "Failed to execute job [jobId=" + ses.getJobId() + ", ses=" + ses + ']');
-                    else
+                    else {
                         U.error(log, "Failed to execute job [jobId=" + ses.getJobId() + ", ses=" + ses + ']', e);
 
+                        if (X.hasCause(e, OutOfMemoryError.class))
+                            ctx.failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+                    }
+
                     ex = e;
                 }
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index ff68e7290b99b..63f50273c88ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.service;
 
+import java.lang.Thread.UncaughtExceptionHandler;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -103,6 +104,7 @@
 import org.apache.ignite.services.ServiceDeploymentException;
 import org.apache.ignite.services.ServiceDescriptor;
 import org.apache.ignite.thread.IgniteThreadFactory;
+import org.apache.ignite.thread.OomExceptionHandler;
 import org.apache.ignite.transactions.Transaction;
 import org.jetbrains.annotations.Nullable;
 import java.util.concurrent.ConcurrentHashMap;
@@ -112,7 +114,6 @@
 import static org.apache.ignite.configuration.DeploymentMode.ISOLATED;
 import static org.apache.ignite.configuration.DeploymentMode.PRIVATE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SERVICES_COMPATIBILITY_MODE;
-import static org.apache.ignite.internal.processors.cache.GridCacheUtils.UTILITY_CACHE_NAME;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
 import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
 import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
@@ -154,8 +155,12 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
     /** Busy lock. */
     private volatile GridSpinBusyLock busyLock = new GridSpinBusyLock();
 
+    /** Uncaught exception handler for thread pools. */
+    private final UncaughtExceptionHandler oomeHnd = new OomExceptionHandler(ctx);
+
     /** Thread factory. */
-    private ThreadFactory threadFactory = new IgniteThreadFactory(ctx.igniteInstanceName(), "service");
+    private ThreadFactory threadFactory = new IgniteThreadFactory(ctx.igniteInstanceName(), "service",
+        oomeHnd);
 
     /** Thread local for service name. */
     private ThreadLocal<String> svcName = new ThreadLocal<>();
@@ -175,7 +180,8 @@ public class GridServiceProcessor extends GridProcessorAdapter implements Ignite
     public GridServiceProcessor(GridKernalContext ctx) {
         super(ctx);
 
-        depExe = Executors.newSingleThreadExecutor(new IgniteThreadFactory(ctx.igniteInstanceName(), "srvc-deploy"));
+        depExe = Executors.newSingleThreadExecutor(new IgniteThreadFactory(ctx.igniteInstanceName(),
+            "srvc-deploy", oomeHnd));
 
         String servicesCompatibilityMode = getString(IGNITE_SERVICES_COMPATIBILITY_MODE);
 
@@ -373,7 +379,8 @@ private IgniteInternalCache<Object, Object> serviceCache() {
 
         busyLock = new GridSpinBusyLock();
 
-        depExe = Executors.newSingleThreadExecutor(new IgniteThreadFactory(ctx.igniteInstanceName(), "srvc-deploy"));
+        depExe = Executors.newSingleThreadExecutor(new IgniteThreadFactory(ctx.igniteInstanceName(),
+            "srvc-deploy", oomeHnd));
 
         start();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteStripedThreadPoolExecutor.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteStripedThreadPoolExecutor.java
index 3cd7484567b8e..418812f8a20c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteStripedThreadPoolExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteStripedThreadPoolExecutor.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.thread;
 
+import java.lang.Thread.UncaughtExceptionHandler;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -45,10 +46,11 @@ public class IgniteStripedThreadPoolExecutor implements ExecutorService {
      * @param igniteInstanceName Node name.
      * @param threadNamePrefix Thread name prefix.
      */
-    public IgniteStripedThreadPoolExecutor(int concurrentLvl, String igniteInstanceName, String threadNamePrefix) {
+    public IgniteStripedThreadPoolExecutor(int concurrentLvl, String igniteInstanceName, String threadNamePrefix,
+        UncaughtExceptionHandler eHnd) {
         execs = new ExecutorService[concurrentLvl];
 
-        ThreadFactory factory = new IgniteThreadFactory(igniteInstanceName, threadNamePrefix);
+        ThreadFactory factory = new IgniteThreadFactory(igniteInstanceName, threadNamePrefix, eHnd);
 
         for (int i = 0; i < concurrentLvl; i++)
             execs[i] = Executors.newSingleThreadExecutor(factory);
@@ -173,4 +175,4 @@ public int threadId(int idx) {
     @Override public String toString() {
         return S.toString(IgniteStripedThreadPoolExecutor.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
index 062c973ba1220..23bf14df1b685 100644
--- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadFactory.java
@@ -17,9 +17,9 @@
 
 package org.apache.ignite.thread;
 
+import java.lang.Thread.UncaughtExceptionHandler;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.atomic.AtomicInteger;
-
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.NotNull;
@@ -41,6 +41,9 @@ public class IgniteThreadFactory implements ThreadFactory {
     /** */
     private final byte plc;
 
+    /** Exception handler. */
+    private final UncaughtExceptionHandler eHnd;
+
     /**
      * Constructs new thread factory for given grid. All threads will belong
      * to the same default thread group.
@@ -49,7 +52,19 @@ public class IgniteThreadFactory implements ThreadFactory {
      * @param threadName Thread name.
      */
     public IgniteThreadFactory(String igniteInstanceName, String threadName) {
-        this(igniteInstanceName, threadName, GridIoPolicy.UNDEFINED);
+        this(igniteInstanceName, threadName, null);
+    }
+
+    /**
+     * Constructs new thread factory for given grid. All threads will belong
+     * to the same default thread group.
+     *
+     * @param igniteInstanceName Ignite instance name.
+     * @param threadName Thread name.
+     * @param eHnd Uncaught exception handler.
+     */
+    public IgniteThreadFactory(String igniteInstanceName, String threadName, UncaughtExceptionHandler eHnd) {
+        this(igniteInstanceName, threadName, GridIoPolicy.UNDEFINED, eHnd);
     }
 
     /**
@@ -59,16 +74,23 @@ public IgniteThreadFactory(String igniteInstanceName, String threadName) {
      * @param igniteInstanceName Ignite instance name.
      * @param threadName Thread name.
      * @param plc {@link GridIoPolicy} for thread pool.
+     * @param eHnd Uncaught exception handler.
      */
-    public IgniteThreadFactory(String igniteInstanceName, String threadName, byte plc) {
+    public IgniteThreadFactory(String igniteInstanceName, String threadName, byte plc, UncaughtExceptionHandler eHnd) {
         this.igniteInstanceName = igniteInstanceName;
         this.threadName = threadName;
         this.plc = plc;
+        this.eHnd = eHnd;
     }
 
     /** {@inheritDoc} */
     @Override public Thread newThread(@NotNull Runnable r) {
-        return new IgniteThread(igniteInstanceName, threadName, r, idxGen.incrementAndGet(), -1, plc);
+        Thread thread = new IgniteThread(igniteInstanceName, threadName, r, idxGen.incrementAndGet(), -1, plc);
+
+        if (eHnd != null)
+            thread.setUncaughtExceptionHandler(eHnd);
+
+        return thread;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java
index 83c64c3363000..fed77adb055b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThreadPoolExecutor.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.thread;
 
+import java.lang.Thread.UncaughtExceptionHandler;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ThreadFactory;
@@ -53,7 +54,8 @@ public IgniteThreadPoolExecutor(
             maxPoolSize,
             keepAliveTime,
             workQ,
-            GridIoPolicy.UNDEFINED);
+            GridIoPolicy.UNDEFINED,
+            null);
     }
 
     /**
@@ -68,6 +70,7 @@ public IgniteThreadPoolExecutor(
      * @param workQ The queue to use for holding tasks before they are executed. This queue will hold only
      *      runnable tasks submitted by the {@link #execute(Runnable)} method.
      * @param plc {@link GridIoPolicy} for thread pool.
+     * @param eHnd Uncaught exception handler for thread pool.
      */
     public IgniteThreadPoolExecutor(
         String threadNamePrefix,
@@ -76,14 +79,15 @@ public IgniteThreadPoolExecutor(
         int maxPoolSize,
         long keepAliveTime,
         BlockingQueue<Runnable> workQ,
-        byte plc) {
+        byte plc,
+        UncaughtExceptionHandler eHnd) {
         super(
             corePoolSize,
             maxPoolSize,
             keepAliveTime,
             TimeUnit.MILLISECONDS,
             workQ,
-            new IgniteThreadFactory(igniteInstanceName, threadNamePrefix, plc)
+            new IgniteThreadFactory(igniteInstanceName, threadNamePrefix, plc, eHnd)
         );
     }
 
@@ -114,4 +118,4 @@ public IgniteThreadPoolExecutor(
             new AbortPolicy()
         );
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/OomExceptionHandler.java b/modules/core/src/main/java/org/apache/ignite/thread/OomExceptionHandler.java
new file mode 100644
index 0000000000000..3a62ad87e5de9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/thread/OomExceptionHandler.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.thread;
+
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.X;
+
+/**
+ * OOM exception handler for system threads.
+ */
+public class OomExceptionHandler implements Thread.UncaughtExceptionHandler {
+    /** Context. */
+    private final GridKernalContext ctx;
+
+    /**
+     * @param ctx Context.
+     */
+    public OomExceptionHandler(GridKernalContext ctx) {
+        this.ctx = ctx;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void uncaughtException(Thread t, Throwable e) {
+        if (X.hasCause(e, OutOfMemoryError.class))
+            ctx.failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java
new file mode 100644
index 0000000000000..2af94b88ee3e4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Out of memory error failure handler test.
+ */
+public class OomFailureHandlerTest extends AbstractFailureHandlerTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setCacheConfiguration(new CacheConfiguration()
+            .setName(DEFAULT_CACHE_NAME)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setBackups(0)
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * Test OOME in IgniteCompute.
+     */
+    public void testComputeOomError() throws Exception {
+        IgniteEx ignite0 = startGrid(0);
+        IgniteEx ignite1 = startGrid(1);
+
+        try {
+            IgniteFuture<Boolean> res = ignite0.compute(ignite0.cluster().forNodeId(ignite1.cluster().localNode().id()))
+                .callAsync(new IgniteCallable<Boolean>() {
+                    @Override public Boolean call() throws Exception {
+                        throw new OutOfMemoryError();
+                    }
+                });
+
+            res.get();
+        }
+        catch (Throwable ignore) {
+            // Expected.
+        }
+
+        assertFailureState(ignite0, ignite1);
+    }
+
+    /**
+     * Test OOME in EntryProcessor.
+     */
+    public void testEntryProcessorOomError() throws Exception {
+        IgniteEx ignite0 = startGrid(0);
+        IgniteEx ignite1 = startGrid(1);
+
+        IgniteCache<Integer, Integer> cache0 = ignite0.getOrCreateCache(DEFAULT_CACHE_NAME);
+        IgniteCache<Integer, Integer> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        awaitPartitionMapExchange();
+
+        Integer key = primaryKey(cache1);
+
+        cache1.put(key, key);
+
+        try {
+            IgniteFuture fut = cache0.invokeAsync(key, new EntryProcessor<Integer, Integer, Object>() {
+                @Override public Object process(MutableEntry<Integer, Integer> entry,
+                    Object... arguments) throws EntryProcessorException {
+                    throw new OutOfMemoryError();
+                }
+            });
+
+            fut.get();
+        }
+        catch (Throwable ignore) {
+            // Expected.
+        }
+
+        assertFailureState(ignite0, ignite1);
+    }
+
+    /**
+     * Test OOME in service method invocation.
+     */
+    public void testServiceInvokeOomError() throws Exception {
+        IgniteEx ignite0 = startGrid(0);
+        IgniteEx ignite1 = startGrid(1);
+
+        IgniteCache<Integer, Integer> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        awaitPartitionMapExchange();
+
+        Integer key = primaryKey(cache1);
+
+        ignite0.services().deployKeyAffinitySingleton("fail-invoke-service", new FailServiceImpl(false),
+            DEFAULT_CACHE_NAME, key);
+
+        FailService svc = ignite0.services().serviceProxy("fail-invoke-service", FailService.class, false);
+
+        try {
+            svc.fail();
+        }
+        catch (Throwable ignore) {
+            // Expected.
+        }
+
+        assertFailureState(ignite0, ignite1);
+    }
+
+    /**
+     * Test OOME in service execute.
+     */
+    public void testServiceExecuteOomError() throws Exception {
+        IgniteEx ignite0 = startGrid(0);
+        IgniteEx ignite1 = startGrid(1);
+
+        IgniteCache<Integer, Integer> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        awaitPartitionMapExchange();
+
+        Integer key = primaryKey(cache1);
+
+        ignite0.services().deployKeyAffinitySingleton("fail-execute-service", new FailServiceImpl(true),
+            DEFAULT_CACHE_NAME, key);
+
+        assertFailureState(ignite0, ignite1);
+    }
+
+    /**
+     * Test OOME in event listener.
+     */
+    public void testEventListenerOomError() throws Exception {
+        IgniteEx ignite0 = startGrid(0);
+        IgniteEx ignite1 = startGrid(1);
+
+        IgniteCache<Integer, Integer> cache0 = ignite0.getOrCreateCache(DEFAULT_CACHE_NAME);
+        IgniteCache<Integer, Integer> cache1 = ignite1.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        awaitPartitionMapExchange();
+
+        ignite1.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                throw new OutOfMemoryError();
+            }
+        }, EventType.EVT_CACHE_OBJECT_PUT);
+
+        Integer key = primaryKey(cache1);
+
+        try {
+            cache0.put(key, key);
+        }
+        catch (Throwable ignore) {
+            // Expected.
+        }
+
+        assertFailureState(ignite0, ignite1);
+    }
+
+    /**
+     * @param igniteWork Working ignite instance.
+     * @param igniteFail Failed ignite instance.
+     */
+    private static void assertFailureState(Ignite igniteWork, Ignite igniteFail) throws IgniteInterruptedCheckedException {
+        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return dummyFailureHandler(igniteFail).failure();
+            }
+        }, 5000L));
+
+        assertFalse(dummyFailureHandler(igniteWork).failure());
+    }
+
+    /**
+     *
+     */
+    private interface FailService extends Service {
+        /**
+         * Fail.
+         */
+        void fail();
+    }
+
+    /**
+     *
+     */
+    private static class FailServiceImpl implements FailService {
+        /** Fail on execute. */
+        private final boolean failOnExec;
+
+        /**
+         * @param failOnExec Fail on execute.
+         */
+        private FailServiceImpl(boolean failOnExec) {
+            this.failOnExec = failOnExec;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void fail() {
+            throw new OutOfMemoryError();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancel(ServiceContext ctx) {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void init(ServiceContext ctx) throws Exception {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute(ServiceContext ctx) throws Exception {
+            if (failOnExec)
+                throw new OutOfMemoryError();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index c4b7d9227f4ee..c388f1dd4d5c7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.GridSuppressedExceptionSelfTest;
 import org.apache.ignite.failure.FailureHandlerTriggeredTest;
 import org.apache.ignite.failure.IoomFailureHandlerTest;
+import org.apache.ignite.failure.OomFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest;
 import org.apache.ignite.internal.ClassSetTest;
@@ -199,6 +200,7 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(StopNodeFailureHandlerTest.class);
         suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
         suite.addTestSuite(IoomFailureHandlerTest.class);
+        suite.addTestSuite(OomFailureHandlerTest.class);
 
         return suite;
     }

From 687194461f445be9902752f38f873d321cde1d85 Mon Sep 17 00:00:00 2001
From: Ilya Borisov <klaster1@gmail.com>
Date: Fri, 6 Apr 2018 11:17:05 +0700
Subject: [PATCH 030/543] IGNITE-7996 Move configuration form templates.

(cherry picked from commit c2c03a9)
---
 .../cache-edit-form/template.tpl.pug          | 22 +++----
 .../cache-edit-form/templates}/affinity.pug   |  0
 .../templates}/concurrency.pug                |  0
 .../cache-edit-form/templates}/general.pug    |  0
 .../cache-edit-form/templates}/memory.pug     |  0
 .../templates}/near-cache-client.pug          |  0
 .../templates}/near-cache-server.pug          |  0
 .../templates}/node-filter.pug                |  0
 .../cache-edit-form/templates}/query.pug      |  0
 .../cache-edit-form/templates}/rebalance.pug  |  0
 .../cache-edit-form/templates}/statistics.pug |  0
 .../cache-edit-form/templates}/store.pug      |  0
 .../cluster-edit-form/template.tpl.pug        | 62 +++++++++----------
 .../cluster-edit-form/templates}/atomic.pug   |  0
 .../templates}/attributes.pug                 |  0
 .../cluster-edit-form/templates}/binary.pug   |  0
 .../templates}/cache-key-cfg.pug              |  0
 .../templates}/checkpoint.pug                 |  0
 .../templates}/checkpoint/fs.pug              |  0
 .../templates}/checkpoint/jdbc.pug            |  0
 .../templates}/checkpoint/s3.pug              |  0
 .../templates}/client-connector.pug           |  0
 .../templates}/collision.pug                  |  0
 .../templates}/collision/custom.pug           |  0
 .../templates}/collision/fifo-queue.pug       |  0
 .../templates}/collision/job-stealing.pug     |  0
 .../templates}/collision/priority-queue.pug   |  0
 .../templates}/communication.pug              |  0
 .../templates}/connector.pug                  |  0
 .../templates}/data-storage.pug               |  0
 .../templates}/deployment.pug                 |  0
 .../templates}/discovery.pug                  |  0
 .../cluster-edit-form/templates}/events.pug   |  0
 .../cluster-edit-form/templates}/failover.pug |  0
 .../cluster-edit-form/templates}/general.pug  |  0
 .../templates}/general/discovery/cloud.pug    |  0
 .../templates}/general/discovery/google.pug   |  0
 .../templates}/general/discovery/jdbc.pug     |  0
 .../general/discovery/kubernetes.pug          |  0
 .../general/discovery/multicast.pug           |  0
 .../templates}/general/discovery/s3.pug       |  0
 .../templates}/general/discovery/shared.pug   |  0
 .../templates}/general/discovery/vm.pug       |  0
 .../general/discovery/zookeeper.pug           |  0
 .../bounded-exponential-backoff.pug           |  0
 .../zookeeper/retrypolicy/custom.pug          |  0
 .../retrypolicy/exponential-backoff.pug       |  0
 .../zookeeper/retrypolicy/forever.pug         |  0
 .../zookeeper/retrypolicy/n-times.pug         |  0
 .../zookeeper/retrypolicy/one-time.pug        |  0
 .../zookeeper/retrypolicy/until-elapsed.pug   |  0
 .../cluster-edit-form/templates}/hadoop.pug   |  0
 .../cluster-edit-form/templates}/igfs.pug     |  0
 .../templates}/load-balancing.pug             |  0
 .../cluster-edit-form/templates}/logger.pug   |  0
 .../templates}/logger/custom.pug              |  0
 .../templates}/logger/log4j.pug               |  0
 .../templates}/logger/log4j2.pug              |  0
 .../templates}/marshaller.pug                 |  0
 .../cluster-edit-form/templates}/memory.pug   |  0
 .../cluster-edit-form/templates}/metrics.pug  |  0
 .../cluster-edit-form/templates}/misc.pug     |  0
 .../cluster-edit-form/templates}/odbc.pug     |  0
 .../templates}/persistence.pug                |  0
 .../cluster-edit-form/templates}/service.pug  |  0
 .../templates}/sql-connector.pug              |  0
 .../cluster-edit-form/templates}/ssl.pug      |  0
 .../cluster-edit-form/templates}/swap.pug     |  0
 .../cluster-edit-form/templates}/thread.pug   |  0
 .../cluster-edit-form/templates}/time.pug     |  0
 .../templates}/transactions.pug               |  0
 .../igfs-edit-form/template.tpl.pug           | 12 ++--
 .../igfs-edit-form/templates}/dual.pug        |  0
 .../templates}/fragmentizer.pug               |  0
 .../igfs-edit-form/templates}/general.pug     |  0
 .../igfs-edit-form/templates}/ipc.pug         |  0
 .../igfs-edit-form/templates}/misc.pug        |  0
 .../igfs-edit-form/templates}/secondary.pug   |  0
 .../model-edit-form/template.tpl.pug          |  6 +-
 .../model-edit-form/templates}/general.pug    |  0
 .../model-edit-form/templates}/query.pug      |  0
 .../model-edit-form/templates}/store.pug      |  0
 .../page-configure-basic/template.pug         | 18 +++---
 83 files changed, 60 insertions(+), 60 deletions(-)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/affinity.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/concurrency.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/general.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/memory.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/near-cache-client.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/near-cache-server.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/node-filter.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/query.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/rebalance.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/statistics.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/caches => components/page-configure-advanced/components/cache-edit-form/templates}/store.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/atomic.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/attributes.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/binary.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/cache-key-cfg.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/checkpoint.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/checkpoint/fs.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/checkpoint/jdbc.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/checkpoint/s3.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/client-connector.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/collision.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/collision/custom.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/collision/fifo-queue.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/collision/job-stealing.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/collision/priority-queue.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/communication.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/connector.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/data-storage.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/deployment.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/discovery.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/events.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/failover.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/cloud.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/google.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/jdbc.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/kubernetes.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/multicast.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/s3.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/shared.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/vm.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/zookeeper.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/zookeeper/retrypolicy/custom.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/zookeeper/retrypolicy/forever.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/zookeeper/retrypolicy/n-times.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/zookeeper/retrypolicy/one-time.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/general/discovery/zookeeper/retrypolicy/until-elapsed.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/hadoop.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/igfs.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/load-balancing.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/logger.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/logger/custom.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/logger/log4j.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/logger/log4j2.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/marshaller.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/memory.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/metrics.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/misc.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/odbc.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/persistence.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/service.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/sql-connector.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/ssl.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/swap.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/thread.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/time.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/clusters => components/page-configure-advanced/components/cluster-edit-form/templates}/transactions.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/igfs => components/page-configure-advanced/components/igfs-edit-form/templates}/dual.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/igfs => components/page-configure-advanced/components/igfs-edit-form/templates}/fragmentizer.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/igfs => components/page-configure-advanced/components/igfs-edit-form/templates}/general.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/igfs => components/page-configure-advanced/components/igfs-edit-form/templates}/ipc.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/igfs => components/page-configure-advanced/components/igfs-edit-form/templates}/misc.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/igfs => components/page-configure-advanced/components/igfs-edit-form/templates}/secondary.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/domains => components/page-configure-advanced/components/model-edit-form/templates}/general.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/domains => components/page-configure-advanced/components/model-edit-form/templates}/query.pug (100%)
 rename modules/web-console/frontend/app/{modules/states/configuration/domains => components/page-configure-advanced/components/model-edit-form/templates}/store.pug (100%)

diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/template.tpl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/template.tpl.pug
index 70cb445812596..a8ae2f2a93e2e 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/template.tpl.pug
@@ -20,18 +20,18 @@ form(
     novalidate
     ng-submit='$ctrl.save($ctrl.clonedCache)'
 )
-    include /app/modules/states/configuration/caches/general
-    include /app/modules/states/configuration/caches/memory
-    include /app/modules/states/configuration/caches/query
-    include /app/modules/states/configuration/caches/store
+    include ./templates/general
+    include ./templates/memory
+    include ./templates/query
+    include ./templates/store
 
-    include /app/modules/states/configuration/caches/affinity
-    include /app/modules/states/configuration/caches/concurrency
-    include /app/modules/states/configuration/caches/near-cache-client
-    include /app/modules/states/configuration/caches/near-cache-server
-    include /app/modules/states/configuration/caches/node-filter
-    include /app/modules/states/configuration/caches/rebalance
-    include /app/modules/states/configuration/caches/statistics
+    include ./templates/affinity
+    include ./templates/concurrency
+    include ./templates/near-cache-client
+    include ./templates/near-cache-server
+    include ./templates/node-filter
+    include ./templates/rebalance
+    include ./templates/statistics
 
 .pc-form-actions-panel
     .pc-form-actions-panel__right-after
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/affinity.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/affinity.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/affinity.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/concurrency.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/concurrency.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/concurrency.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/general.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/general.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/general.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/general.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/memory.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/memory.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/memory.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/memory.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-client.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-client.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-client.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-server.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/near-cache-server.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-server.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/node-filter.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/node-filter.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/node-filter.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/query.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/query.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/query.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/query.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/rebalance.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/rebalance.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/rebalance.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/statistics.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/statistics.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/statistics.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/statistics.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/caches/store.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/store.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/caches/store.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/store.pug
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug
index 4dd0e17159867..c2bfd68a57668 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug
@@ -18,57 +18,57 @@ include /app/helpers/jade/mixins
 
 form(id='cluster' name='ui.inputForm' novalidate ng-submit='$ctrl.save()')
     .panel-group
-        include /app/modules/states/configuration/clusters/general
+        include ./templates/general
 
-        include /app/modules/states/configuration/clusters/atomic
-        include /app/modules/states/configuration/clusters/binary
-        include /app/modules/states/configuration/clusters/cache-key-cfg
-        include /app/modules/states/configuration/clusters/checkpoint
+        include ./templates/atomic
+        include ./templates/binary
+        include ./templates/cache-key-cfg
+        include ./templates/checkpoint
 
         //- Since ignite 2.3
-        include /app/modules/states/configuration/clusters/client-connector
+        include ./templates/client-connector
 
-        include /app/modules/states/configuration/clusters/collision
-        include /app/modules/states/configuration/clusters/communication
-        include /app/modules/states/configuration/clusters/connector
-        include /app/modules/states/configuration/clusters/deployment
+        include ./templates/collision
+        include ./templates/communication
+        include ./templates/connector
+        include ./templates/deployment
 
         //- Since ignite 2.3
-        include /app/modules/states/configuration/clusters/data-storage
+        include ./templates/data-storage
 
-        include /app/modules/states/configuration/clusters/discovery
-        include /app/modules/states/configuration/clusters/events
-        include /app/modules/states/configuration/clusters/failover
-        include /app/modules/states/configuration/clusters/hadoop
-        include /app/modules/states/configuration/clusters/load-balancing
-        include /app/modules/states/configuration/clusters/logger
-        include /app/modules/states/configuration/clusters/marshaller
+        include ./templates/discovery
+        include ./templates/events
+        include ./templates/failover
+        include ./templates/hadoop
+        include ./templates/load-balancing
+        include ./templates/logger
+        include ./templates/marshaller
 
         //- Since ignite 2.0, deprecated in ignite 2.3
-        include /app/modules/states/configuration/clusters/memory
+        include ./templates/memory
 
-        include /app/modules/states/configuration/clusters/misc
-        include /app/modules/states/configuration/clusters/metrics
+        include ./templates/misc
+        include ./templates/metrics
 
         //- Deprecated in ignite 2.1
-        include /app/modules/states/configuration/clusters/odbc
+        include ./templates/odbc
 
         //- Since ignite 2.1, deprecated in ignite 2.3
-        include /app/modules/states/configuration/clusters/persistence
+        include ./templates/persistence
 
         //- Deprecated in ignite 2.3
-        include /app/modules/states/configuration/clusters/sql-connector
+        include ./templates/sql-connector
 
-        include /app/modules/states/configuration/clusters/service
-        include /app/modules/states/configuration/clusters/ssl
+        include ./templates/service
+        include ./templates/ssl
 
         //- Removed in ignite 2.0
-        include /app/modules/states/configuration/clusters/swap
+        include ./templates/swap
 
-        include /app/modules/states/configuration/clusters/thread
-        include /app/modules/states/configuration/clusters/time
-        include /app/modules/states/configuration/clusters/transactions
-        include /app/modules/states/configuration/clusters/attributes
+        include ./templates/thread
+        include ./templates/time
+        include ./templates/transactions
+        include ./templates/attributes
 
 .pc-form-actions-panel(n_g-show='$ctrl.$scope.selectedItem')
     button-preview-project(cluster='$ctrl.cluster' ng-hide='$ctrl.isNew')
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/atomic.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/atomic.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/atomic.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/attributes.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/attributes.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/attributes.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/binary.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/binary.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/binary.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/cache-key-cfg.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/cache-key-cfg.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/cache-key-cfg.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/fs.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/fs.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/fs.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/jdbc.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/jdbc.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/jdbc.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/s3.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/checkpoint/s3.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/s3.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/client-connector.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/client-connector.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/client-connector.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/client-connector.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/custom.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/custom.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/custom.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/fifo-queue.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/fifo-queue.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/fifo-queue.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/job-stealing.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/job-stealing.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/job-stealing.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/priority-queue.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/collision/priority-queue.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/priority-queue.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/communication.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/communication.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/communication.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/connector.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/connector.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/connector.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/connector.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/data-storage.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/data-storage.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/deployment.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/deployment.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/deployment.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/discovery.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/discovery.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/discovery.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/events.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/events.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/events.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/events.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/failover.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/failover.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/cloud.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/cloud.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/cloud.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/google.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/google.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/google.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/jdbc.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/jdbc.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/jdbc.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/kubernetes.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/kubernetes.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/kubernetes.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/kubernetes.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/multicast.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/s3.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/s3.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/s3.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/shared.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/shared.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/shared.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/vm.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/vm.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/vm.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/custom.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/custom.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/custom.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/forever.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/forever.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/forever.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/n-times.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/n-times.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/n-times.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/one-time.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/one-time.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/one-time.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/hadoop.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/hadoop.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/hadoop.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/hadoop.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/igfs.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/igfs.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/igfs.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/load-balancing.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/load-balancing.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/load-balancing.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/custom.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger/custom.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/custom.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j2.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/logger/log4j2.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j2.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/marshaller.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/marshaller.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/marshaller.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/memory.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/memory.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/memory.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/memory.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/metrics.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/metrics.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/metrics.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/misc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/misc.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/misc.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/misc.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/odbc.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/odbc.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/odbc.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/persistence.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/persistence.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/persistence.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/service.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/service.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/sql-connector.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/sql-connector.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/sql-connector.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/sql-connector.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/ssl.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/ssl.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/ssl.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/swap.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/swap.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/swap.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/swap.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/thread.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/thread.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/thread.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/thread.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/time.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/time.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/time.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/time.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/transactions.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/clusters/transactions.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/transactions.pug
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/template.tpl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/template.tpl.pug
index fe8b2182f53b1..f505e589562df 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/template.tpl.pug
@@ -15,15 +15,15 @@
     limitations under the License.
 
 form(id='igfs' name='ui.inputForm' novalidate ng-submit='$ctrl.save()')
-    include /app/modules/states/configuration/igfs/general
+    include ./templates/general
 
-    include /app/modules/states/configuration/igfs/secondary
-    include /app/modules/states/configuration/igfs/ipc
-    include /app/modules/states/configuration/igfs/fragmentizer
+    include ./templates/secondary
+    include ./templates/ipc
+    include ./templates/fragmentizer
 
     //- Removed in ignite 2.0
-    include /app/modules/states/configuration/igfs/dual
-    include /app/modules/states/configuration/igfs/misc
+    include ./templates/dual
+    include ./templates/misc
 
 .pc-form-actions-panel
     .pc-form-actions-panel__right-after
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/dual.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/dual.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/dual.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/dual.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/fragmentizer.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/fragmentizer.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/fragmentizer.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/general.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/general.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/general.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/general.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/ipc.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/ipc.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/ipc.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/misc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/misc.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/misc.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/misc.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/secondary.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/igfs/secondary.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/secondary.pug
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/template.tpl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/template.tpl.pug
index 8ebd11cdc7466..78ae7693dcadb 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/template.tpl.pug
@@ -15,9 +15,9 @@
     limitations under the License.
 
 form(id='model' name='ui.inputForm' novalidate ng-submit='$ctrl.save()')
-    include /app/modules/states/configuration/domains/general
-    include /app/modules/states/configuration/domains/query
-    include /app/modules/states/configuration/domains/store
+    include ./templates/general
+    include ./templates/query
+    include ./templates/store
 
 .pc-form-actions-panel
     .pc-form-actions-panel__right-after
diff --git a/modules/web-console/frontend/app/modules/states/configuration/domains/general.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/general.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/domains/general.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/general.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/domains/query.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/query.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/domains/query.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/query.pug
diff --git a/modules/web-console/frontend/app/modules/states/configuration/domains/store.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/store.pug
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/domains/store.pug
rename to modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/store.pug
diff --git a/modules/web-console/frontend/app/components/page-configure-basic/template.pug b/modules/web-console/frontend/app/components/page-configure-basic/template.pug
index 7714c8144c12d..3bcb216dd2861 100644
--- a/modules/web-console/frontend/app/components/page-configure-basic/template.pug
+++ b/modules/web-console/frontend/app/components/page-configure-basic/template.pug
@@ -15,15 +15,15 @@
     limitations under the License.
 
 include /app/helpers/jade/mixins
-include /app/modules/states/configuration/clusters/general/discovery/cloud
-include /app/modules/states/configuration/clusters/general/discovery/google
-include /app/modules/states/configuration/clusters/general/discovery/jdbc
-include /app/modules/states/configuration/clusters/general/discovery/multicast
-include /app/modules/states/configuration/clusters/general/discovery/s3
-include /app/modules/states/configuration/clusters/general/discovery/shared
-include /app/modules/states/configuration/clusters/general/discovery/vm
-include /app/modules/states/configuration/clusters/general/discovery/zookeeper
-include /app/modules/states/configuration/clusters/general/discovery/kubernetes
+include ./../page-configure-advanced/components/cluster-edit-form/templates/general/discovery/cloud
+include ./../page-configure-advanced/components/cluster-edit-form/templates/general/discovery/google
+include ./../page-configure-advanced/components/cluster-edit-form/templates/general/discovery/jdbc
+include ./../page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast
+include ./../page-configure-advanced/components/cluster-edit-form/templates/general/discovery/s3
+include ./../page-configure-advanced/components/cluster-edit-form/templates/general/discovery/shared
+include ./../page-configure-advanced/components/cluster-edit-form/templates/general/discovery/vm
+include ./../page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper
+include ./../page-configure-advanced/components/cluster-edit-form/templates/general/discovery/kubernetes
 
 - const model = '$ctrl.clonedCluster'
 - const modelDiscoveryKind = `${model}.discovery.kind`

From 9955728a72e0f9c11faa313a521d4566b5a93dc1 Mon Sep 17 00:00:00 2001
From: Ilya Borisov <klaster1@gmail.com>
Date: Fri, 6 Apr 2018 11:19:07 +0700
Subject: [PATCH 031/543] IGNITE-7996 Move config state module index.

(cherry picked from commit d5e0be0)
---
 .../states => components/page-configure}/configuration.state.js   | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename modules/web-console/frontend/app/{modules/states => components/page-configure}/configuration.state.js (100%)

diff --git a/modules/web-console/frontend/app/modules/states/configuration.state.js b/modules/web-console/frontend/app/components/page-configure/configuration.state.js
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration.state.js
rename to modules/web-console/frontend/app/components/page-configure/configuration.state.js

From 536d8b2080b93505e67206a240385baced150674 Mon Sep 17 00:00:00 2001
From: Ilya Borisov <klaster1@gmail.com>
Date: Fri, 6 Apr 2018 11:20:13 +0700
Subject: [PATCH 032/543] IGNITE-7996 Use configuration.state for state
 registration only.

(cherry picked from commit 2800ef0)
---
 .../page-configure/configuration.state.js     | 490 +++++++++---------
 1 file changed, 233 insertions(+), 257 deletions(-)

diff --git a/modules/web-console/frontend/app/components/page-configure/configuration.state.js b/modules/web-console/frontend/app/components/page-configure/configuration.state.js
index 1a0a598905888..f8bb4dc2457f1 100644
--- a/modules/web-console/frontend/app/components/page-configure/configuration.state.js
+++ b/modules/web-console/frontend/app/components/page-configure/configuration.state.js
@@ -17,20 +17,11 @@
 
 import angular from 'angular';
 
-import {default as ActivitiesData} from 'app/core/activities/Activities.data';
-
-// Common directives.
-import previewPanel from './configuration/preview-panel.directive.js';
-
-// Summary screen.
-import ConfigurationResource from './configuration/Configuration.resource';
-import IgniteSummaryZipper from './configuration/summary/summary-zipper.service';
-
 import base2 from 'views/base2.pug';
-import pageConfigureAdvancedClusterComponent from 'app/components/page-configure-advanced/components/page-configure-advanced-cluster/component';
-import pageConfigureAdvancedModelsComponent from 'app/components/page-configure-advanced/components/page-configure-advanced-models/component';
-import pageConfigureAdvancedCachesComponent from 'app/components/page-configure-advanced/components/page-configure-advanced-caches/component';
-import pageConfigureAdvancedIGFSComponent from 'app/components/page-configure-advanced/components/page-configure-advanced-igfs/component';
+import pageConfigureAdvancedClusterComponent from '../page-configure-advanced/components/page-configure-advanced-cluster/component';
+import pageConfigureAdvancedModelsComponent from '../page-configure-advanced/components/page-configure-advanced-models/component';
+import pageConfigureAdvancedCachesComponent from '../page-configure-advanced/components/page-configure-advanced-caches/component';
+import pageConfigureAdvancedIGFSComponent from '../page-configure-advanced/components/page-configure-advanced-igfs/component';
 
 import get from 'lodash/get';
 import {Observable} from 'rxjs/Observable';
@@ -47,251 +38,236 @@ const shortCachesResolve = ['ConfigSelectors', 'ConfigureState', 'ConfigEffects'
     .toPromise();
 }];
 
-/**
- * @param {ActivitiesData} ActivitiesData
- * @param {uirouter.UIRouter} $uiRouter
- */
-function initConfiguration(ActivitiesData, $uiRouter) {
-    $uiRouter.transitionService.onSuccess({to: 'base.configuration.**'}, (transition) => {
-        ActivitiesData.post({group: 'configuration', action: transition.targetState().name()});
+function registerStates($stateProvider) {
+    // Setup the states.
+    $stateProvider
+    .state('base.configuration', {
+        abstract: true,
+        permission: 'configuration',
+        url: '/configuration',
+        onEnter: ['ConfigureState', (ConfigureState) => ConfigureState.dispatchAction({type: 'PRELOAD_STATE', state: {}})],
+        views: {
+            '@': {
+                template: base2
+            }
+        },
+        resolve: {
+            _shortClusters: ['ConfigEffects', ({etp}) => {
+                return etp('LOAD_USER_CLUSTERS');
+            }]
+        },
+        resolvePolicy: {
+            async: 'NOWAIT'
+        }
+    })
+    .state('base.configuration.overview', {
+        url: '/overview',
+        component: 'pageConfigureOverview',
+        permission: 'configuration',
+        tfMetaTags: {
+            title: 'Configuration'
+        }
+    })
+    .state('base.configuration.edit', {
+        url: `/{clusterID:${idRegex}}`,
+        permission: 'configuration',
+        component: 'pageConfigure',
+        resolve: {
+            _cluster: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
+                return $transition$.injector().getAsync('_shortClusters').then(() => {
+                    return etp('LOAD_AND_EDIT_CLUSTER', {clusterID: $transition$.params().clusterID});
+                });
+            }]
+        },
+        data: {
+            errorState: 'base.configuration.overview'
+        },
+        redirectTo: ($transition$) => {
+            const [ConfigureState, ConfigSelectors] = ['ConfigureState', 'ConfigSelectors'].map((t) => $transition$.injector().get(t));
+            const waitFor = ['_cluster', '_shortClusters'].map((t) => $transition$.injector().getAsync(t));
+            return Observable.fromPromise(Promise.all(waitFor)).switchMap(() => {
+                return Observable.combineLatest(
+                    ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1),
+                    ConfigureState.state$.let(ConfigSelectors.selectShortClusters()).take(1)
+                );
+            })
+            .map(([cluster = {caches: []}, clusters]) => {
+                return (clusters.value.size > 10 || cluster.caches.length > 5)
+                    ? 'base.configuration.edit.advanced'
+                    : 'base.configuration.edit.basic';
+            })
+            .toPromise();
+        },
+        failState: 'signin',
+        tfMetaTags: {
+            title: 'Configuration'
+        }
+    })
+    .state('base.configuration.edit.basic', {
+        url: '/basic',
+        component: 'pageConfigureBasic',
+        permission: 'configuration',
+        resolve: {
+            _shortCaches: shortCachesResolve
+        },
+        resolvePolicy: {
+            async: 'NOWAIT'
+        },
+        tfMetaTags: {
+            title: 'Basic Configuration'
+        }
+    })
+    .state('base.configuration.edit.advanced', {
+        url: '/advanced',
+        component: 'pageConfigureAdvanced',
+        permission: 'configuration',
+        redirectTo: 'base.configuration.edit.advanced.cluster'
+    })
+    .state('base.configuration.edit.advanced.cluster', {
+        url: '/cluster',
+        component: pageConfigureAdvancedClusterComponent.name,
+        permission: 'configuration',
+        resolve: {
+            _shortCaches: shortCachesResolve
+        },
+        resolvePolicy: {
+            async: 'NOWAIT'
+        },
+        tfMetaTags: {
+            title: 'Configure Cluster'
+        }
+    })
+    .state('base.configuration.edit.advanced.caches', {
+        url: '/caches',
+        permission: 'configuration',
+        component: pageConfigureAdvancedCachesComponent.name,
+        resolve: {
+            _shortCachesAndModels: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
+                if ($transition$.params().clusterID === 'new') return Promise.resolve();
+                return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
+                .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+                .map((cluster) => {
+                    return Promise.all([
+                        etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
+                        etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id}),
+                        etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
+                    ]);
+                })
+                .toPromise();
+            }]
+        },
+        resolvePolicy: {
+            async: 'NOWAIT'
+        },
+        tfMetaTags: {
+            title: 'Configure Caches'
+        }
+    })
+    .state('base.configuration.edit.advanced.caches.cache', {
+        url: `/{cacheID:${idRegex}}`,
+        permission: 'configuration',
+        resolve: {
+            _cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
+                const {clusterID, cacheID} = $transition$.params();
+                if (cacheID === 'new') return Promise.resolve();
+                return etp('LOAD_CACHE', {cacheID});
+            }]
+        },
+        data: {
+            errorState: 'base.configuration.edit.advanced.caches'
+        },
+        resolvePolicy: {
+            async: 'NOWAIT'
+        },
+        tfMetaTags: {
+            title: 'Configure Caches'
+        }
+    })
+    .state('base.configuration.edit.advanced.models', {
+        url: '/models',
+        component: pageConfigureAdvancedModelsComponent.name,
+        permission: 'configuration',
+        resolve: {
+            _shortCachesAndModels: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
+                if ($transition$.params().clusterID === 'new') return Promise.resolve();
+                return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
+                .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+                .map((cluster) => {
+                    return Promise.all([
+                        etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
+                        etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id})
+                    ]);
+                })
+                .toPromise();
+            }]
+        },
+        resolvePolicy: {
+            async: 'NOWAIT'
+        },
+        tfMetaTags: {
+            title: 'Configure SQL Schemes'
+        }
+    })
+    .state('base.configuration.edit.advanced.models.model', {
+        url: `/{modelID:${idRegex}}`,
+        resolve: {
+            _cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
+                const {clusterID, modelID} = $transition$.params();
+                if (modelID === 'new') return Promise.resolve();
+                return etp('LOAD_MODEL', {modelID});
+            }]
+        },
+        data: {
+            errorState: 'base.configuration.edit.advanced.models'
+        },
+        permission: 'configuration',
+        resolvePolicy: {
+            async: 'NOWAIT'
+        }
+    })
+    .state('base.configuration.edit.advanced.igfs', {
+        url: '/igfs',
+        component: pageConfigureAdvancedIGFSComponent.name,
+        permission: 'configuration',
+        resolve: {
+            _shortIGFSs: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
+                if ($transition$.params().clusterID === 'new') return Promise.resolve();
+                return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
+                .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+                .map((cluster) => {
+                    return Promise.all([
+                        etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
+                    ]);
+                })
+                .toPromise();
+            }]
+        },
+        resolvePolicy: {
+            async: 'NOWAIT'
+        },
+        tfMetaTags: {
+            title: 'Configure IGFS'
+        }
+    })
+    .state('base.configuration.edit.advanced.igfs.igfs', {
+        url: `/{igfsID:${idRegex}}`,
+        permission: 'configuration',
+        resolve: {
+            _igfs: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
+                const {clusterID, igfsID} = $transition$.params();
+                if (igfsID === 'new') return Promise.resolve();
+                return etp('LOAD_IGFS', {igfsID});
+            }]
+        },
+        data: {
+            errorState: 'base.configuration.edit.advanced.igfs'
+        },
+        resolvePolicy: {
+            async: 'NOWAIT'
+        }
     });
 }
 
-initConfiguration.$inject = ['IgniteActivitiesData', '$uiRouter'];
+registerStates.$inject = ['$stateProvider'];
 
-angular.module('ignite-console.states.configuration', ['ui.router'])
-    .directive(...previewPanel)
-    // Services.
-    .service('IgniteSummaryZipper', IgniteSummaryZipper)
-    .service('IgniteConfigurationResource', ConfigurationResource)
-    .run(initConfiguration)
-    // Configure state provider.
-    .config(['$stateProvider', ($stateProvider) => {
-        // Setup the states.
-        $stateProvider
-            .state('base.configuration', {
-                abstract: true,
-                permission: 'configuration',
-                url: '/configuration',
-                onEnter: ['ConfigureState', (ConfigureState) => ConfigureState.dispatchAction({type: 'PRELOAD_STATE', state: {}})],
-                views: {
-                    '@': {
-                        template: base2
-                    }
-                },
-                resolve: {
-                    _shortClusters: ['ConfigEffects', ({etp}) => {
-                        return etp('LOAD_USER_CLUSTERS');
-                    }]
-                },
-                resolvePolicy: {
-                    async: 'NOWAIT'
-                }
-            })
-            .state('base.configuration.overview', {
-                url: '/overview',
-                component: 'pageConfigureOverview',
-                permission: 'configuration',
-                tfMetaTags: {
-                    title: 'Configuration'
-                }
-            })
-            .state('base.configuration.edit', {
-                url: `/{clusterID:${idRegex}}`,
-                permission: 'configuration',
-                component: 'pageConfigure',
-                resolve: {
-                    _cluster: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
-                        return $transition$.injector().getAsync('_shortClusters').then(() => {
-                            return etp('LOAD_AND_EDIT_CLUSTER', {clusterID: $transition$.params().clusterID});
-                        });
-                    }]
-                },
-                data: {
-                    errorState: 'base.configuration.overview'
-                },
-                redirectTo: ($transition$) => {
-                    const [ConfigureState, ConfigSelectors] = ['ConfigureState', 'ConfigSelectors'].map((t) => $transition$.injector().get(t));
-                    const waitFor = ['_cluster', '_shortClusters'].map((t) => $transition$.injector().getAsync(t));
-                    return Observable.fromPromise(Promise.all(waitFor)).switchMap(() => {
-                        return Observable.combineLatest(
-                            ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1),
-                            ConfigureState.state$.let(ConfigSelectors.selectShortClusters()).take(1)
-                        );
-                    })
-                    .map(([cluster = {caches: []}, clusters]) => {
-                        return (clusters.value.size > 10 || cluster.caches.length > 5)
-                            ? 'base.configuration.edit.advanced'
-                            : 'base.configuration.edit.basic';
-                    })
-                    .toPromise();
-                },
-                failState: 'signin',
-                tfMetaTags: {
-                    title: 'Configuration'
-                }
-            })
-            .state('base.configuration.edit.basic', {
-                url: '/basic',
-                component: 'pageConfigureBasic',
-                permission: 'configuration',
-                resolve: {
-                    _shortCaches: shortCachesResolve
-                },
-                resolvePolicy: {
-                    async: 'NOWAIT'
-                },
-                tfMetaTags: {
-                    title: 'Basic Configuration'
-                }
-            })
-            .state('base.configuration.edit.advanced', {
-                url: '/advanced',
-                component: 'pageConfigureAdvanced',
-                permission: 'configuration',
-                redirectTo: 'base.configuration.edit.advanced.cluster'
-            })
-            .state('base.configuration.edit.advanced.cluster', {
-                url: '/cluster',
-                component: pageConfigureAdvancedClusterComponent.name,
-                permission: 'configuration',
-                resolve: {
-                    _shortCaches: shortCachesResolve
-                },
-                resolvePolicy: {
-                    async: 'NOWAIT'
-                },
-                tfMetaTags: {
-                    title: 'Configure Cluster'
-                }
-            })
-            .state('base.configuration.edit.advanced.caches', {
-                url: '/caches',
-                permission: 'configuration',
-                component: pageConfigureAdvancedCachesComponent.name,
-                resolve: {
-                    _shortCachesAndModels: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
-                        if ($transition$.params().clusterID === 'new') return Promise.resolve();
-                        return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
-                        .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
-                        .map((cluster) => {
-                            return Promise.all([
-                                etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
-                                etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id}),
-                                etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
-                            ]);
-                        })
-                        .toPromise();
-                    }]
-                },
-                resolvePolicy: {
-                    async: 'NOWAIT'
-                },
-                tfMetaTags: {
-                    title: 'Configure Caches'
-                }
-            })
-            .state('base.configuration.edit.advanced.caches.cache', {
-                url: `/{cacheID:${idRegex}}`,
-                permission: 'configuration',
-                resolve: {
-                    _cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
-                        const {clusterID, cacheID} = $transition$.params();
-                        if (cacheID === 'new') return Promise.resolve();
-                        return etp('LOAD_CACHE', {cacheID});
-                    }]
-                },
-                data: {
-                    errorState: 'base.configuration.edit.advanced.caches'
-                },
-                resolvePolicy: {
-                    async: 'NOWAIT'
-                },
-                tfMetaTags: {
-                    title: 'Configure Caches'
-                }
-            })
-            .state('base.configuration.edit.advanced.models', {
-                url: '/models',
-                component: pageConfigureAdvancedModelsComponent.name,
-                permission: 'configuration',
-                resolve: {
-                    _shortCachesAndModels: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
-                        if ($transition$.params().clusterID === 'new') return Promise.resolve();
-                        return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
-                        .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
-                        .map((cluster) => {
-                            return Promise.all([
-                                etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
-                                etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id})
-                            ]);
-                        })
-                        .toPromise();
-                    }]
-                },
-                resolvePolicy: {
-                    async: 'NOWAIT'
-                },
-                tfMetaTags: {
-                    title: 'Configure SQL Schemes'
-                }
-            })
-            .state('base.configuration.edit.advanced.models.model', {
-                url: `/{modelID:${idRegex}}`,
-                resolve: {
-                    _cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
-                        const {clusterID, modelID} = $transition$.params();
-                        if (modelID === 'new') return Promise.resolve();
-                        return etp('LOAD_MODEL', {modelID});
-                    }]
-                },
-                data: {
-                    errorState: 'base.configuration.edit.advanced.models'
-                },
-                permission: 'configuration',
-                resolvePolicy: {
-                    async: 'NOWAIT'
-                }
-            })
-            .state('base.configuration.edit.advanced.igfs', {
-                url: '/igfs',
-                component: pageConfigureAdvancedIGFSComponent.name,
-                permission: 'configuration',
-                resolve: {
-                    _shortIGFSs: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
-                        if ($transition$.params().clusterID === 'new') return Promise.resolve();
-                        return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
-                        .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
-                        .map((cluster) => {
-                            return Promise.all([
-                                etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
-                            ]);
-                        })
-                        .toPromise();
-                    }]
-                },
-                resolvePolicy: {
-                    async: 'NOWAIT'
-                },
-                tfMetaTags: {
-                    title: 'Configure IGFS'
-                }
-            })
-            .state('base.configuration.edit.advanced.igfs.igfs', {
-                url: `/{igfsID:${idRegex}}`,
-                permission: 'configuration',
-                resolve: {
-                    _igfs: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
-                        const {clusterID, igfsID} = $transition$.params();
-                        if (igfsID === 'new') return Promise.resolve();
-                        return etp('LOAD_IGFS', {igfsID});
-                    }]
-                },
-                data: {
-                    errorState: 'base.configuration.edit.advanced.igfs'
-                },
-                resolvePolicy: {
-                    async: 'NOWAIT'
-                }
-            });
-    }]);
+export {registerStates};

From a2d9f97c0ae77e81344849bc2c3cba2d3964cf01 Mon Sep 17 00:00:00 2001
From: Ilya Borisov <klaster1@gmail.com>
Date: Fri, 6 Apr 2018 11:20:39 +0700
Subject: [PATCH 033/543] IGNITE-7996 Rename configuration.state to states.

(cherry picked from commit 14dd2df)
---
 .../page-configure/{configuration.state.js => states.js}          | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename modules/web-console/frontend/app/components/page-configure/{configuration.state.js => states.js} (100%)

diff --git a/modules/web-console/frontend/app/components/page-configure/configuration.state.js b/modules/web-console/frontend/app/components/page-configure/states.js
similarity index 100%
rename from modules/web-console/frontend/app/components/page-configure/configuration.state.js
rename to modules/web-console/frontend/app/components/page-configure/states.js

From 1d5f45b4eb900a3c4cc0bedb8919b35f2a0032c8 Mon Sep 17 00:00:00 2001
From: Ilya Borisov <klaster1@gmail.com>
Date: Fri, 6 Apr 2018 11:22:20 +0700
Subject: [PATCH 034/543] IGNITE-7996 Move configuration assets into
 page-configure module.

(cherry picked from commit d02e87b)
---
 modules/web-console/frontend/app/app.js       |  2 --
 .../components/preview-panel/directive.js}    | 11 ++++++--
 .../components/preview-panel/index.js         | 23 +++++++++++++++++
 .../app/components/page-configure/index.js    | 25 ++++++++++++++++++-
 .../services/ConfigurationResource.js}        | 11 ++++++--
 .../page-configure/services/SummaryZipper.js} |  9 +++++--
 .../services}/summary.worker.js               |  0
 7 files changed, 72 insertions(+), 9 deletions(-)
 rename modules/web-console/frontend/app/{modules/states/configuration/preview-panel.directive.js => components/page-configure/components/preview-panel/directive.js} (96%)
 create mode 100644 modules/web-console/frontend/app/components/page-configure/components/preview-panel/index.js
 rename modules/web-console/frontend/app/{modules/states/configuration/Configuration.resource.js => components/page-configure/services/ConfigurationResource.js} (90%)
 rename modules/web-console/frontend/app/{modules/states/configuration/summary/summary-zipper.service.js => components/page-configure/services/SummaryZipper.js} (90%)
 rename modules/web-console/frontend/app/{modules/states/configuration/summary => components/page-configure/services}/summary.worker.js (100%)

diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 757be229e55bc..692acc54f3d64 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -27,7 +27,6 @@ import './modules/nodes/nodes.module';
 import './modules/demo/Demo.module';
 
 import './modules/states/logout.state';
-import './modules/states/configuration.state';
 import './modules/states/admin.state';
 import './modules/states/errors.state';
 
@@ -192,7 +191,6 @@ angular.module('ignite-console', [
     'ignite-console.demo',
     // States.
     'ignite-console.states.logout',
-    'ignite-console.states.configuration',
     'ignite-console.states.admin',
     'ignite-console.states.errors',
     // Common modules.
diff --git a/modules/web-console/frontend/app/modules/states/configuration/preview-panel.directive.js b/modules/web-console/frontend/app/components/page-configure/components/preview-panel/directive.js
similarity index 96%
rename from modules/web-console/frontend/app/modules/states/configuration/preview-panel.directive.js
rename to modules/web-console/frontend/app/components/page-configure/components/preview-panel/directive.js
index be7bf1ed52ec4..b7519ce1804cb 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/preview-panel.directive.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/preview-panel/directive.js
@@ -16,8 +16,13 @@
  */
 
 import ace from 'brace';
+import _ from 'lodash';
 
-export default ['previewPanel', ['$interval', '$timeout', ($interval, $timeout) => {
+/**
+ * @param {ng.IIntervalService} $interval
+ * @param {ng.ITimeoutService} $timeout
+ */
+export default function previewPanelDirective($interval, $timeout) {
     let animation = {editor: null, stage: 0, start: 0, stop: 0};
     let prevContent = [];
 
@@ -236,4 +241,6 @@ export default ['previewPanel', ['$interval', '$timeout', ($interval, $timeout)
         link,
         require: ['?igniteUiAceTabs', '?^igniteUiAceTabs']
     };
-}]];
+}
+
+previewPanelDirective.$inject = ['$interval', '$timeout'];
diff --git a/modules/web-console/frontend/app/components/page-configure/components/preview-panel/index.js b/modules/web-console/frontend/app/components/page-configure/components/preview-panel/index.js
new file mode 100644
index 0000000000000..ff1367b2dbe17
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-configure/components/preview-panel/index.js
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import directive from './directive';
+
+export default angular
+    .module('ignite-console.page-configure.preview-panel', [])
+    .directive('previewPanel', directive);
diff --git a/modules/web-console/frontend/app/components/page-configure/index.js b/modules/web-console/frontend/app/components/page-configure/index.js
index 5874df51dcd73..3209edeb4094f 100644
--- a/modules/web-console/frontend/app/components/page-configure/index.js
+++ b/modules/web-console/frontend/app/components/page-configure/index.js
@@ -28,6 +28,8 @@ import PageConfigure from './services/PageConfigure';
 import ConfigurationDownload from './services/ConfigurationDownload';
 import ConfigChangesGuard from './services/ConfigChangesGuard';
 import ConfigSelectionManager from './services/ConfigSelectionManager';
+import SummaryZipper from './services/SummaryZipper';
+import ConfigurationResource from './services/ConfigurationResource';
 import selectors from './store/selectors';
 import effects from './store/effects';
 
@@ -43,8 +45,10 @@ import modalImportModels from './components/modal-import-models';
 import buttonImportModels from './components/button-import-models';
 import buttonDownloadProject from './components/button-download-project';
 import buttonPreviewProject from './components/button-preview-project';
+import previewPanel from './components/preview-panel';
 
 import {errorState} from './transitionHooks/errorState';
+import {default as ActivitiesData} from 'app/core/activities/Activities.data';
 
 import 'rxjs/add/operator/withLatestFrom';
 import 'rxjs/add/operator/skip';
@@ -73,10 +77,24 @@ import {
     shortIGFSsActionTypes,
     refsReducer
 } from './reducer';
+
 import {reducer as reduxDevtoolsReducer, devTools} from './reduxDevtoolsIntegration';
+import {registerStates} from './states';
+
+/**
+ * @param {ActivitiesData} ActivitiesData
+ * @param {uirouter.UIRouter} $uiRouter
+ */
+function registerActivitiesHook(ActivitiesData, $uiRouter) {
+    $uiRouter.transitionService.onSuccess({to: 'base.configuration.**'}, (transition) => {
+        ActivitiesData.post({group: 'configuration', action: transition.targetState().name()});
+    });
+}
+registerActivitiesHook.$inject = ['IgniteActivitiesData', '$uiRouter'];
 
 export default angular
     .module('ignite-console.page-configure', [
+        'ui.router',
         'asyncFilter',
         uiValidate,
         pcFormFieldSize.name,
@@ -87,11 +105,14 @@ export default angular
         modalImportModels.name,
         buttonImportModels.name,
         buttonDownloadProject.name,
-        buttonPreviewProject.name
+        buttonPreviewProject.name,
+        previewPanel.name
     ])
+    .config(registerStates)
     .config(['DefaultStateProvider', (DefaultState) => {
         DefaultState.setRedirectTo(() => 'base.configuration.overview');
     }])
+    .run(registerActivitiesHook)
     .run(['ConfigEffects', 'ConfigureState', '$uiRouter', (ConfigEffects, ConfigureState, $uiRouter) => {
         $uiRouter.plugin(UIRouterRx);
         // $uiRouter.plugin(Visualizer);
@@ -153,6 +174,8 @@ export default angular
     .directive(fakeUiCanExit.name, fakeUiCanExit)
     .directive(formUICanExitGuard.name, formUICanExitGuard)
     .factory('configSelectionManager', ConfigSelectionManager)
+    .service('IgniteSummaryZipper', SummaryZipper)
+    .service('IgniteConfigurationResource', ConfigurationResource)
     .service('ConfigSelectors', selectors)
     .service('ConfigEffects', effects)
     .service('ConfigChangesGuard', ConfigChangesGuard)
diff --git a/modules/web-console/frontend/app/modules/states/configuration/Configuration.resource.js b/modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.js
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/Configuration.resource.js
rename to modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.js
index 0582d5c6e8f76..2dab8a3d2334e 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/Configuration.resource.js
+++ b/modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.js
@@ -15,7 +15,12 @@
  * limitations under the License.
  */
 
-export default ['$http', ($http) => {
+import _ from 'lodash';
+
+/**
+ * @param {ng.IHttpService} $http
+ */
+export default function ConfigurationResourceService($http) {
     return {
         read() {
             return $http.get('/api/v1/configuration/list')
@@ -39,4 +44,6 @@ export default ['$http', ($http) => {
             return Promise.resolve({spaces, clusters, caches, igfss, domains});
         }
     };
-}];
+}
+
+ConfigurationResourceService.$inject = ['$http'];
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary-zipper.service.js b/modules/web-console/frontend/app/components/page-configure/services/SummaryZipper.js
similarity index 90%
rename from modules/web-console/frontend/app/modules/states/configuration/summary/summary-zipper.service.js
rename to modules/web-console/frontend/app/components/page-configure/services/SummaryZipper.js
index 119fb529e8e3f..8bc16b46203ed 100644
--- a/modules/web-console/frontend/app/modules/states/configuration/summary/summary-zipper.service.js
+++ b/modules/web-console/frontend/app/components/page-configure/services/SummaryZipper.js
@@ -17,7 +17,10 @@
 
 import Worker from './summary.worker';
 
-export default ['$q', function($q) {
+/**
+ * @param {ng.IQService} $q
+ */
+export default function SummaryZipperService($q) {
     return function(message) {
         const defer = $q.defer();
         const worker = new Worker();
@@ -36,4 +39,6 @@ export default ['$q', function($q) {
 
         return defer.promise;
     };
-}];
+}
+
+SummaryZipperService.$inject = ['$q'];
diff --git a/modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js b/modules/web-console/frontend/app/components/page-configure/services/summary.worker.js
similarity index 100%
rename from modules/web-console/frontend/app/modules/states/configuration/summary/summary.worker.js
rename to modules/web-console/frontend/app/components/page-configure/services/summary.worker.js

From a5a83d211a572b8419866fabbc31975168157729 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Thu, 12 Apr 2018 11:21:23 +0700
Subject: [PATCH 035/543] IGNITE-7996 Merge with master.

---
 .../frontend/app/components/page-configure/states.js           | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/modules/web-console/frontend/app/components/page-configure/states.js b/modules/web-console/frontend/app/components/page-configure/states.js
index f8bb4dc2457f1..a75e851174ffe 100644
--- a/modules/web-console/frontend/app/components/page-configure/states.js
+++ b/modules/web-console/frontend/app/components/page-configure/states.js
@@ -15,15 +15,12 @@
  * limitations under the License.
  */
 
-import angular from 'angular';
-
 import base2 from 'views/base2.pug';
 import pageConfigureAdvancedClusterComponent from '../page-configure-advanced/components/page-configure-advanced-cluster/component';
 import pageConfigureAdvancedModelsComponent from '../page-configure-advanced/components/page-configure-advanced-models/component';
 import pageConfigureAdvancedCachesComponent from '../page-configure-advanced/components/page-configure-advanced-caches/component';
 import pageConfigureAdvancedIGFSComponent from '../page-configure-advanced/components/page-configure-advanced-igfs/component';
 
-import get from 'lodash/get';
 import {Observable} from 'rxjs/Observable';
 
 const idRegex = `new|[a-z0-9]+`;

From dbf2d722b0563a9637cc72fe2fcdf3dbd07291fc Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Thu, 12 Apr 2018 10:37:36 +0300
Subject: [PATCH 036/543] IGNITE-8042: .NET thin client: authentication
 support. This closes #3790.

---
 .../client/ClientAuthenticationException.java |  16 +-
 .../client/thin/TcpClientChannel.java         |   9 +-
 .../odbc/ClientListenerNioListener.java       |  19 +++
 .../client/ClientConnectionContext.java       |   8 +-
 .../platform/client/ClientStatus.java         |   5 +-
 .../Client/ClientConnectionTest.cs            | 161 ++++++++++++++++++
 .../Client/ClientStatusCode.cs                |  12 +-
 .../Client/IgniteClientConfiguration.cs       |  13 ++
 .../IgniteClientConfigurationSection.xsd      |  10 ++
 .../Impl/Client/ClientProtocolVersion.cs      |  22 ++-
 .../Impl/Client/ClientSocket.cs               |  85 ++++++++-
 11 files changed, 329 insertions(+), 31 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
index 0c24db8880a4f..526690a69c8b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
+++ b/modules/core/src/main/java/org/apache/ignite/client/ClientAuthenticationException.java
@@ -24,22 +24,10 @@ public class ClientAuthenticationException extends ClientException {
     /** Serial version uid. */
     private static final long serialVersionUID = 0L;
 
-    /** Message. */
-    private static final String MSG = "Invalid user name or password";
-
     /**
      * Default constructor.
      */
-    public ClientAuthenticationException() {
-        super(MSG);
-    }
-
-    /**
-     * Constructs a new exception with the specified cause.
-     *
-     * @param cause the cause.
-     */
-    public ClientAuthenticationException(Throwable cause) {
-        super(MSG, cause);
+    public ClientAuthenticationException(String msg) {
+        super(msg);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
index 8e8294f1ef6f2..10dc8652071f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpClientChannel.java
@@ -272,8 +272,13 @@ private void handshakeRes(String user, String pwd)
             try (BinaryReaderExImpl r = new BinaryReaderExImpl(null, res, null, true)) {
                 String err = r.readString();
 
-                if (err != null && err.toUpperCase().matches(".*USER.*INCORRECT.*"))
-                    throw new ClientAuthenticationException();
+                int errCode = ClientStatus.FAILED;
+
+                if (res.remaining() > 0)
+                    errCode = r.readInt();
+
+                if (errCode == ClientStatus.AUTH_FAILED)
+                    throw new ClientAuthenticationException(err);
                 else if (ver.equals(srvVer))
                     throw new ClientProtocolError(err);
                 else if (!supportedVers.contains(srvVer) ||
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java
index 53b14d7cdb268..407c1a02efc34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java
@@ -26,9 +26,11 @@
 import org.apache.ignite.internal.binary.streams.BinaryHeapInputStream;
 import org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream;
 import org.apache.ignite.internal.binary.streams.BinaryInputStream;
+import org.apache.ignite.internal.processors.authentication.IgniteAccessControlException;
 import org.apache.ignite.internal.processors.odbc.jdbc.JdbcConnectionContext;
 import org.apache.ignite.internal.processors.odbc.odbc.OdbcConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
+import org.apache.ignite.internal.processors.platform.client.ClientStatus;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
 import org.apache.ignite.internal.util.nio.GridNioServerListenerAdapter;
 import org.apache.ignite.internal.util.nio.GridNioSession;
@@ -228,6 +230,18 @@ private void onHandshake(GridNioSession ses, byte[] msg) {
 
             connCtx.handler().writeHandshake(writer);
         }
+        catch (IgniteAccessControlException authEx) {
+            writer.writeBoolean(false);
+
+            writer.writeShort((short)0);
+            writer.writeShort((short)0);
+            writer.writeShort((short)0);
+
+            writer.doWriteString(authEx.getMessage());
+
+            if (ver.compareTo(ClientConnectionContext.VER_1_1_0) >= 0)
+                writer.writeInt(ClientStatus.AUTH_FAILED);
+        }
         catch (IgniteCheckedException e) {
             U.warn(log, "Error during handshake [rmtAddr=" + ses.remoteAddress() + ", msg=" + e.getMessage() + ']');
 
@@ -239,10 +253,15 @@ private void onHandshake(GridNioSession ses, byte[] msg) {
                 currVer = connCtx.currentVersion();
 
             writer.writeBoolean(false);
+
             writer.writeShort(currVer.major());
             writer.writeShort(currVer.minor());
             writer.writeShort(currVer.maintenance());
+
             writer.doWriteString(e.getMessage());
+
+            if (ver.compareTo(ClientConnectionContext.VER_1_1_0) >= 0)
+                writer.writeInt(ClientStatus.FAILED);
         }
 
         ses.send(writer.array());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
index 061aab32c0459..056ea8306fa53 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientConnectionContext.java
@@ -44,10 +44,10 @@
  */
 public class ClientConnectionContext implements ClientListenerConnectionContext {
     /** Version 1.0.0. */
-    private static final ClientListenerProtocolVersion VER_1_0_0 = ClientListenerProtocolVersion.create(1, 0, 0);
+    public static final ClientListenerProtocolVersion VER_1_0_0 = ClientListenerProtocolVersion.create(1, 0, 0);
 
     /** Version 1.1.0. */
-    private static final ClientListenerProtocolVersion VER_1_1_0 = ClientListenerProtocolVersion.create(1, 1, 0);
+    public static final ClientListenerProtocolVersion VER_1_1_0 = ClientListenerProtocolVersion.create(1, 1, 0);
 
     /** Supported versions. */
     private static final Collection<ClientListenerProtocolVersion> SUPPORTED_VERS = Arrays.asList(VER_1_1_0, VER_1_0_0);
@@ -144,12 +144,12 @@ public GridKernalContext kernalContext() {
             authCtx = thirdPartyAuthentication(user, pwd).authorizationContext();
         else if (kernalCtx.authentication().enabled()) {
             if (user == null || user.length() == 0)
-                throw new IgniteCheckedException("Unauthenticated sessions are prohibited.");
+                throw new IgniteAccessControlException("Unauthenticated sessions are prohibited.");
 
             authCtx = kernalCtx.authentication().authenticate(user, pwd);
 
             if (authCtx == null)
-                throw new IgniteCheckedException("Unknown authentication error.");
+                throw new IgniteAccessControlException("Unknown authentication error.");
         }
 
         handler = new ClientRequestHandler(this, authCtx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
index b8dfb1fa64b9e..e63812c991b89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientStatus.java
@@ -49,6 +49,9 @@ private ClientStatus (){
     /** Resource does not exist. */
     public static final int RESOURCE_DOES_NOT_EXIST = 1011;
 
-    /** Resource does not exist. */
+    /** Authorization failure. */
     public static final int SECURITY_VIOLATION = 1012;
+
+    /** Authentication failed. */
+    public static final int AUTH_FAILED = 2000;
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
index 9da9a0310b7c3..2ea17a8739227 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
@@ -19,13 +19,17 @@ namespace Apache.Ignite.Core.Tests.Client
 {
     using System;
     using System.Collections.Generic;
+    using System.IO;
     using System.Linq;
     using System.Net;
     using System.Net.Sockets;
     using System.Text.RegularExpressions;
     using System.Threading;
     using System.Threading.Tasks;
+    using Apache.Ignite.Core.Cache.Configuration;
+    using Apache.Ignite.Core.Cache.Query;
     using Apache.Ignite.Core.Client;
+    using Apache.Ignite.Core.Client.Cache;
     using Apache.Ignite.Core.Configuration;
     using NUnit.Framework;
 
@@ -34,6 +38,18 @@ namespace Apache.Ignite.Core.Tests.Client
     /// </summary>
     public class ClientConnectionTest
     {
+        /** Temp dir for WAL. */
+        private readonly string _tempDir = TestUtils.GetTempDirectoryName();
+
+        /// <summary>
+        /// Sets up the test.
+        /// </summary>
+        [SetUp]
+        public void SetUp()
+        {
+            TestUtils.ClearWorkDir();
+        }
+
         /// <summary>
         /// Fixture tear down.
         /// </summary>
@@ -41,6 +57,13 @@ public class ClientConnectionTest
         public void TearDown()
         {
             Ignition.StopAll(true);
+
+            if (Directory.Exists(_tempDir))
+            {
+                Directory.Delete(_tempDir, true);
+            }
+
+            TestUtils.ClearWorkDir();
         }
 
         /// <summary>
@@ -54,6 +77,107 @@ public void TestNoServerConnectionRefused()
             Assert.AreEqual(SocketError.ConnectionRefused, socketEx.SocketErrorCode);
         }
 
+        /// <summary>
+        /// Tests that empty username or password are not allowed.
+        /// </summary>
+        [Test]
+        public void TestAuthenticationEmptyCredentials()
+        {
+            using (Ignition.Start(SecureServerConfig()))
+            {
+                var cliCfg = SecureClientConfig();
+
+                cliCfg.Password = null;
+                var ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+                Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Password cannot be null"));
+
+                cliCfg.Password = "";
+                ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+                Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Password cannot be empty"));
+
+                cliCfg.Password = "ignite";
+
+                cliCfg.Username = null;
+                ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+                Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Username cannot be null"));
+
+                cliCfg.Username = "";
+                ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+                Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Username cannot be empty"));
+            }
+        }
+
+        /// <summary>
+        /// Test invalid username or password.
+        /// </summary>
+        [Test]
+        public void TestAuthenticationInvalidCredentials()
+        {
+            using (Ignition.Start(SecureServerConfig()))
+            {
+                var cliCfg = SecureClientConfig();
+
+                cliCfg.Username = "invalid";
+
+                var ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+                Assert.True(ex.StatusCode == ClientStatusCode.AuthenticationFailed);
+
+                cliCfg.Username = "ignite";
+                cliCfg.Password = "invalid";
+
+                ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
+                Assert.True(ex.StatusCode == ClientStatusCode.AuthenticationFailed);
+            }
+        }
+
+        /// <summary>
+        /// Test authentication.
+        /// </summary>
+        [Test]
+        public void TestAuthentication()
+        {
+            using (var srv = Ignition.Start(SecureServerConfig()))
+            {
+                srv.GetCluster().SetActive(true);
+
+                using (var cli = Ignition.StartClient(SecureClientConfig()))
+                {
+                    CacheClientConfiguration ccfg = new CacheClientConfiguration()
+                    {
+                        Name = "TestCache",
+                        QueryEntities = new[]
+                        {
+                            new QueryEntity
+                            {
+                                KeyType = typeof(string),
+                                ValueType = typeof(string),
+                            },
+                        },
+                    };
+
+                    ICacheClient<string, string> cache = cli.GetOrCreateCache<string, string>(ccfg);
+
+                    cache.Put("key1", "val1");
+
+                    cache.Query(new SqlFieldsQuery("CREATE USER \"my_User\" WITH PASSWORD 'my_Password'")).GetAll();
+                }
+
+                var cliCfg = SecureClientConfig();
+
+                cliCfg.Username = "my_User";
+                cliCfg.Password = "my_Password";
+
+                using (var cli = Ignition.StartClient(cliCfg))
+                {
+                    ICacheClient<string, string> cache = cli.GetCache<string, string>("TestCache");
+
+                    string val = cache.Get("key1");
+
+                    Assert.True(val == "val1");
+                }
+            }
+        }
+
         /// <summary>
         /// Tests that multiple clients can connect to one server.
         /// </summary>
@@ -374,5 +498,42 @@ private static SocketException GetSocketException(Exception ex)
             
             throw new Exception("SocketException not found.", origEx);
         }
+
+        /// <summary>
+        /// Create server configuration with enabled authentication.
+        /// </summary>
+        /// <returns>Server configuration.</returns>
+        private IgniteConfiguration SecureServerConfig()
+        {
+            return new IgniteConfiguration(TestUtils.GetTestConfiguration())
+            {
+                AuthenticationEnabled = true,
+                DataStorageConfiguration = new DataStorageConfiguration()
+                {
+                    StoragePath = Path.Combine(_tempDir, "Store"),
+                    WalPath = Path.Combine(_tempDir, "WalStore"),
+                    WalArchivePath = Path.Combine(_tempDir, "WalArchive"),
+                    DefaultDataRegionConfiguration = new DataRegionConfiguration()
+                    {
+                        Name = "default",
+                        PersistenceEnabled = true
+                    }
+                }
+            };
+        }
+
+        /// <summary>
+        /// Create client configuration with enabled authentication.
+        /// </summary>
+        /// <returns>Client configuration.</returns>
+        private static IgniteClientConfiguration SecureClientConfig()
+        {
+            return new IgniteClientConfiguration()
+            {
+                Host = "localhost",
+                Username = "ignite",
+                Password = "ignite"
+            };
+        }
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Client/ClientStatusCode.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Client/ClientStatusCode.cs
index 3f5ee8eb4bcee..3bdd9e1ac0d6c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Client/ClientStatusCode.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Client/ClientStatusCode.cs
@@ -52,6 +52,16 @@ public enum ClientStatusCode
         /// <summary>
         /// The too many cursors (see <see cref="ClientConnectorConfiguration.MaxOpenCursorsPerConnection"/>).
         /// </summary>
-        TooManyCursors = 1010
+        TooManyCursors = 1010,
+
+        /// <summary>
+        /// Authorization failure.
+        /// </summary>
+        SecurityViolation = 1012,
+
+        /// <summary>
+        /// Authentication failed.
+        /// </summary>
+        AuthenticationFailed = 2000
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs
index 8730f39340965..32524955ee4c2 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs
@@ -90,6 +90,9 @@ public IgniteClientConfiguration(IgniteClientConfiguration cfg) : this()
 
             BinaryProcessor = cfg.BinaryProcessor;
             SslStreamFactory = cfg.SslStreamFactory;
+
+            Username = cfg.Username;
+            Password = cfg.Password;
         }
 
         /// <summary>
@@ -145,6 +148,16 @@ public IgniteClientConfiguration(IgniteClientConfiguration cfg) : this()
         /// </summary>
         public ISslStreamFactory SslStreamFactory { get; set; }
 
+        /// <summary>
+        /// Username to be used to connect to secured cluster.
+        /// </summary>
+        public string Username { get; set; }
+
+        /// <summary>
+        /// Password to be used to connect to secured cluster.
+        /// </summary>
+        public string Password { get; set; }
+
         /// <summary>
         /// Gets or sets custom binary processor. Internal property for tests.
         /// </summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd
index 569ee6fd2d5bd..7e6caff6c4d19 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd
@@ -237,6 +237,16 @@
                     <xs:documentation>Socket operation timeout. Zero or negative for infinite timeout.</xs:documentation>
                 </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="username" type="xs:string">
+                <xs:annotation>
+                    <xs:documentation>Username to be used to connect to secured cluster.</xs:documentation>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="password" type="xs:string">
+                <xs:annotation>
+                    <xs:documentation>Password to be used to connect to secured cluster.</xs:documentation>
+                </xs:annotation>
+            </xs:attribute>
         </xs:complexType>
     </xs:element>
 </xs:schema>
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientProtocolVersion.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientProtocolVersion.cs
index bfdf5a309a876..4fe5c712a3da6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientProtocolVersion.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientProtocolVersion.cs
@@ -22,7 +22,7 @@ namespace Apache.Ignite.Core.Impl.Client
     /// <summary>
     /// Client protocol version.
     /// </summary>
-    internal struct ClientProtocolVersion : IEquatable<ClientProtocolVersion>
+    internal struct ClientProtocolVersion : IEquatable<ClientProtocolVersion>, IComparable<ClientProtocolVersion>
     {
         /** */
         private readonly short _major;
@@ -67,6 +67,26 @@ public short Maintenance
             get { return _maintenance; }
         }
 
+        /// <summary>
+        /// Compare this version to other version.
+        /// </summary>
+        /// <param name="other"></param>
+        /// <returns></returns>
+        public int CompareTo(ClientProtocolVersion other)
+        {
+            int res = Major - other.Major;
+
+            if (res == 0)
+            {
+                res = Minor - other.Minor;
+
+                if (res == 0)
+                    res = Maintenance - other.Maintenance;
+            }
+
+            return res;
+        }
+
         /// <summary>
         /// Returns a value indicating whether specified instance equals to current.
         /// </summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
index fca5dab55062b..27d8f0bea092a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
@@ -28,6 +28,7 @@ namespace Apache.Ignite.Core.Impl.Client
     using System.Net.Sockets;
     using System.Threading;
     using System.Threading.Tasks;
+    using System.Xml.Schema;
     using Apache.Ignite.Core.Client;
     using Apache.Ignite.Core.Common;
     using Apache.Ignite.Core.Impl.Binary;
@@ -38,8 +39,14 @@ namespace Apache.Ignite.Core.Impl.Client
     /// </summary>
     internal sealed class ClientSocket : IDisposable
     {
+        /** Version 1.0.0. */
+        private static readonly ClientProtocolVersion Ver100 = new ClientProtocolVersion(1, 0, 0);
+
+        /** Version 1.1.0. */
+        private static readonly ClientProtocolVersion Ver110 = new ClientProtocolVersion(1, 1, 0);
+
         /** Current version. */
-        private static readonly ClientProtocolVersion CurrentProtocolVersion = new ClientProtocolVersion(1, 0, 0);
+        private static readonly ClientProtocolVersion CurrentProtocolVersion = Ver110;
 
         /** Handshake opcode. */
         private const byte OpHandshake = 1;
@@ -98,7 +105,9 @@ public ClientSocket(IgniteClientConfiguration clientConfiguration, ClientProtoco
             _socket = Connect(clientConfiguration);
             _stream = GetSocketStream(_socket, clientConfiguration);
 
-            Handshake(version ?? CurrentProtocolVersion);
+            Validate(clientConfiguration);
+
+            Handshake(clientConfiguration, version ?? CurrentProtocolVersion);
 
             // Check periodically if any request has timed out.
             if (_timeout > TimeSpan.Zero)
@@ -111,6 +120,31 @@ public ClientSocket(IgniteClientConfiguration clientConfiguration, ClientProtoco
             Task.Factory.StartNew(WaitForMessages);
         }
 
+        /// <summary>
+        /// Validate configuration.
+        /// </summary>
+        /// <param name="cfg">Configuration.</param>
+        private void Validate(IgniteClientConfiguration cfg)
+        {
+            if (cfg.Username != null)
+            {
+                if (cfg.Username.Length == 0)
+                    throw new IgniteClientException("IgniteClientConfiguration.Username cannot be empty.");
+
+                if (cfg.Password == null)
+                    throw new IgniteClientException("IgniteClientConfiguration.Password cannot be null when Username is set.");
+            }
+
+            if (cfg.Password != null)
+            {
+                if (cfg.Password.Length == 0)
+                    throw new IgniteClientException("IgniteClientConfiguration.Password cannot be empty.");
+
+                if (cfg.Username == null)
+                    throw new IgniteClientException("IgniteClientConfiguration.Username cannot be null when Password is set.");
+            }
+        }
+
         /// <summary>
         /// Performs a send-receive operation.
         /// </summary>
@@ -226,8 +260,10 @@ private static T DecodeResponse<T>(BinaryHeapStream stream, Func<IBinaryStream,
         /// <summary>
         /// Performs client protocol handshake.
         /// </summary>
-        private void Handshake(ClientProtocolVersion version)
+        private void Handshake(IgniteClientConfiguration clientConfiguration, ClientProtocolVersion version)
         {
+            bool auth = version.CompareTo(Ver110) >= 0 && clientConfiguration.Username != null;
+
             // Send request.
             int messageLen;
             var buf = WriteMessage(stream =>
@@ -242,10 +278,19 @@ private void Handshake(ClientProtocolVersion version)
 
                 // Client type: platform.
                 stream.WriteByte(ClientType);
-            }, 12, out messageLen);
 
-            Debug.Assert(messageLen == 12);
+                // Authentication data.
+                if (auth)
+                {
+                    var writer = BinaryUtils.Marshaller.StartMarshal(stream);
+
+                    writer.WriteString(clientConfiguration.Username);
+                    writer.WriteString(clientConfiguration.Password);
 
+                    BinaryUtils.Marshaller.FinishMarshal(writer);
+                }
+            }, 12, out messageLen);
+            
             _stream.Write(buf, 0, messageLen);
 
             // Decode response.
@@ -253,6 +298,7 @@ private void Handshake(ClientProtocolVersion version)
 
             using (var stream = new BinaryHeapStream(res))
             {
+                // Read input.
                 var success = stream.ReadBool();
 
                 if (success)
@@ -265,9 +311,32 @@ private void Handshake(ClientProtocolVersion version)
 
                 var errMsg = BinaryUtils.Marshaller.Unmarshal<string>(stream);
 
-                throw new IgniteClientException(string.Format(
-                    "Client handshake failed: '{0}'. Client version: {1}. Server version: {2}",
-                    errMsg, version, serverVersion));
+                ClientStatusCode errCode = ClientStatusCode.Fail;
+
+                if (stream.Remaining > 0)
+                {
+                    errCode = (ClientStatusCode) stream.ReadInt();
+                }
+
+                // Authentication error is handled immediately.
+                if (errCode == ClientStatusCode.AuthenticationFailed)
+                {
+                    throw new IgniteClientException(errMsg, null, ClientStatusCode.AuthenticationFailed);
+                }
+
+                // Re-try if possible.
+                bool retry = serverVersion.CompareTo(version) < 0 && serverVersion.Equals(Ver100);
+
+                if (retry)
+                {
+                    Handshake(clientConfiguration, serverVersion);
+                }
+                else
+                {
+                    throw new IgniteClientException(string.Format(
+                        "Client handshake failed: '{0}'. Client version: {1}. Server version: {2}",
+                        errMsg, version, serverVersion), null, errCode);
+                }
             }
         }
 

From 72259b01e0c6d72794eca4c28c9d9d848b0ff97f Mon Sep 17 00:00:00 2001
From: dmitrievanthony <dmitrievanthony@gmail.com>
Date: Thu, 12 Apr 2018 11:16:22 +0300
Subject: [PATCH 037/543] IGNITE-8176: Integrate gradient descent linear
 regression with partition based dataset

this closes #3787

(cherry picked from commit df6356d)
---
 .../ml/knn/KNNClassificationExample.java      |  11 +-
 .../examples/ml/nn/MLPTrainerExample.java     |   4 +-
 .../preprocessing/NormalizationExample.java   |  17 +--
 ...ithLSQRTrainerAndNormalizationExample.java |  23 ++--
 ...inearRegressionWithLSQRTrainerExample.java |  14 +--
 ...dLinearRegressionWithQRTrainerExample.java |   9 +-
 ...LinearRegressionWithSGDTrainerExample.java |  78 +++++++++---
 .../SVMBinaryClassificationExample.java       |  11 +-
 .../SVMMultiClassClassificationExample.java   |  24 ++--
 ...isionTreeClassificationTrainerExample.java |   7 +-
 .../DecisionTreeRegressionTrainerExample.java |   4 +-
 .../org/apache/ignite/ml/nn/Activators.java   |  20 +++
 .../org/apache/ignite/ml/nn/MLPTrainer.java   |  46 +++++--
 .../preprocessing/PreprocessingTrainer.java   |  41 +++++-
 .../normalization/NormalizationTrainer.java   |  35 ++++--
 .../linear/FeatureExtractorWrapper.java       |  55 ++++++++
 .../linear/LinearRegressionLSQRTrainer.java   |  38 +-----
 .../linear/LinearRegressionSGDTrainer.java    | 118 ++++++++++++------
 .../ignite/ml/trainers/DatasetTrainer.java    |  46 +++++++
 .../ignite/ml/knn/KNNClassificationTest.java  |  20 +--
 .../ml/nn/MLPTrainerIntegrationTest.java      |  14 +--
 .../apache/ignite/ml/nn/MLPTrainerTest.java   |  22 ++--
 .../MLPTrainerMnistIntegrationTest.java       |   7 +-
 .../nn/performance/MLPTrainerMnistTest.java   |  11 +-
 .../NormalizationTrainerTest.java             |  10 +-
 .../ml/regressions/RegressionsTestSuite.java  |  15 +--
 ...ributedLinearRegressionSGDTrainerTest.java |  35 ------
 ...ributedLinearRegressionSGDTrainerTest.java |  35 ------
 ...reAbstractLinearRegressionTrainerTest.java |   3 +
 .../LinearRegressionLSQRTrainerTest.java      |  14 ++-
 .../LinearRegressionSGDTrainerTest.java       |  94 ++++++++++++++
 .../LocalLinearRegressionSGDTrainerTest.java  |  35 ------
 .../ignite/ml/svm/SVMBinaryTrainerTest.java   |  11 +-
 .../ml/svm/SVMMultiClassTrainerTest.java      |  11 +-
 ...eClassificationTrainerIntegrationTest.java |   9 +-
 ...DecisionTreeClassificationTrainerTest.java |  12 +-
 ...nTreeRegressionTrainerIntegrationTest.java |   9 +-
 .../DecisionTreeRegressionTrainerTest.java    |  12 +-
 .../DecisionTreeMNISTIntegrationTest.java     |   7 +-
 .../performance/DecisionTreeMNISTTest.java    |  11 +-
 40 files changed, 612 insertions(+), 386 deletions(-)
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/FeatureExtractorWrapper.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionSGDTrainerTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionSGDTrainerTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionSGDTrainerTest.java

diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
index f3cdbbefc2a2a..39a8431a18761 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
@@ -17,9 +17,6 @@
 
 package org.apache.ignite.examples.ml.knn;
 
-import java.util.Arrays;
-import java.util.UUID;
-import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
@@ -27,7 +24,6 @@
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
 import org.apache.ignite.ml.knn.classification.KNNClassificationTrainer;
 import org.apache.ignite.ml.knn.classification.KNNStrategy;
@@ -35,6 +31,10 @@
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 import org.apache.ignite.thread.IgniteThread;
 
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
 /**
  * Run kNN multi-class classification trainer over distributed dataset.
  *
@@ -56,7 +56,8 @@ public static void main(String[] args) throws InterruptedException {
                 KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
                 KNNClassificationModel knnMdl = trainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
+                    ignite,
+                    dataCache,
                     (k, v) -> Arrays.copyOfRange(v, 1, v.length),
                     (k, v) -> v[0]
                 ).withK(3)
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
index efa1ba73d0401..ce44cc64b935e 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
@@ -23,7 +23,6 @@
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.ml.math.Matrix;
 import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
 import org.apache.ignite.ml.nn.Activators;
@@ -99,7 +98,8 @@ public static void main(String[] args) throws InterruptedException {
 
                 // Train neural network and get multilayer perceptron model.
                 MultilayerPerceptron mlp = trainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+                    ignite,
+                    trainingSet,
                     (k, v) -> new double[] {v.x, v.y},
                     (k, v) -> new double[] {v.lb}
                 );
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/NormalizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/NormalizationExample.java
index e0bcd089b674b..b2c4e128da332 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/NormalizationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/preprocessing/NormalizationExample.java
@@ -17,21 +17,19 @@
 
 package org.apache.ignite.examples.ml.preprocessing;
 
-import java.util.Arrays;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.examples.ml.dataset.model.Person;
-import org.apache.ignite.ml.dataset.DatasetBuilder;
 import org.apache.ignite.ml.dataset.DatasetFactory;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.ml.dataset.primitive.SimpleDataset;
 import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.preprocessing.normalization.NormalizationPreprocessor;
 import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
 
+import java.util.Arrays;
+
 /**
  * Example that shows how to use normalization preprocessor to normalize data.
  *
@@ -47,8 +45,6 @@ public static void main(String[] args) throws Exception {
 
             IgniteCache<Integer, Person> persons = createCache(ignite);
 
-            DatasetBuilder<Integer, Person> builder = new CacheBasedDatasetBuilder<>(ignite, persons);
-
             // Defines first preprocessor that extracts features from an upstream data.
             IgniteBiFunction<Integer, Person, double[]> featureExtractor = (k, v) -> new double[] {
                 v.getAge(),
@@ -56,14 +52,11 @@ public static void main(String[] args) throws Exception {
             };
 
             // Defines second preprocessor that normalizes features.
-            NormalizationPreprocessor<Integer, Person> preprocessor = new NormalizationTrainer<Integer, Person>()
-                .fit(builder, featureExtractor, 2);
+            IgniteBiFunction<Integer, Person, double[]> preprocessor = new NormalizationTrainer<Integer, Person>()
+                .fit(ignite, persons, featureExtractor);
 
             // Creates a cache based simple dataset containing features and providing standard dataset API.
-            try (SimpleDataset<?> dataset = DatasetFactory.createSimpleDataset(
-                builder,
-                preprocessor
-            )) {
+            try (SimpleDataset<?> dataset = DatasetFactory.createSimpleDataset(ignite, persons, preprocessor)) {
                 // Calculation of the mean value. This calculation will be performed in map-reduce manner.
                 double[] mean = dataset.mean();
                 System.out.println("Mean \n\t" + Arrays.toString(mean));
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
index 567a59975a385..99e657781b131 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
@@ -17,9 +17,6 @@
 
 package org.apache.ignite.examples.ml.regression.linear;
 
-import java.util.Arrays;
-import java.util.UUID;
-import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
@@ -28,7 +25,7 @@
 import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 import org.apache.ignite.ml.preprocessing.normalization.NormalizationPreprocessor;
 import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
@@ -36,6 +33,10 @@
 import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
 import org.apache.ignite.thread.IgniteThread;
 
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
 /**
  * Run linear regression model over distributed matrix.
  *
@@ -119,21 +120,17 @@ public static void main(String[] args) throws InterruptedException {
                 NormalizationTrainer<Integer, double[]> normalizationTrainer = new NormalizationTrainer<>();
 
                 System.out.println(">>> Perform the training to get the normalization preprocessor.");
-                NormalizationPreprocessor<Integer, double[]> preprocessor = normalizationTrainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
-                    (k, v) -> Arrays.copyOfRange(v, 1, v.length),
-                    4
+                IgniteBiFunction<Integer, double[], double[]> preprocessor = normalizationTrainer.fit(
+                    ignite,
+                    dataCache,
+                    (k, v) -> Arrays.copyOfRange(v, 1, v.length)
                 );
 
                 System.out.println(">>> Create new linear regression trainer object.");
                 LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
 
                 System.out.println(">>> Perform the training to get the model.");
-                LinearRegressionModel mdl = trainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
-                    preprocessor,
-                    (k, v) -> v[0]
-                );
+                LinearRegressionModel mdl = trainer.fit(ignite, dataCache, preprocessor, (k, v) -> v[0]);
 
                 System.out.println(">>> Linear regression model: " + mdl);
 
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java
index a853092f92a08..25aec0cbba400 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java
@@ -17,9 +17,6 @@
 
 package org.apache.ignite.examples.ml.regression.linear;
 
-import java.util.Arrays;
-import java.util.UUID;
-import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
@@ -27,13 +24,15 @@
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainer;
 import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
 import org.apache.ignite.thread.IgniteThread;
 
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
 /**
  * Run linear regression model over distributed matrix.
  *
@@ -108,7 +107,7 @@ public static void main(String[] args) throws InterruptedException {
             // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
             // because we create ignite cache internally.
             IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                SparseDistributedMatrixExample.class.getSimpleName(), () -> {
+                DistributedLinearRegressionWithLSQRTrainerExample.class.getSimpleName(), () -> {
                 IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
 
                 System.out.println(">>> Create new linear regression trainer object.");
@@ -116,7 +115,8 @@ public static void main(String[] args) throws InterruptedException {
 
                 System.out.println(">>> Perform the training to get the model.");
                 LinearRegressionModel mdl = trainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
+                    ignite,
+                    dataCache,
                     (k, v) -> Arrays.copyOfRange(v, 1, v.length),
                     (k, v) -> v[0]
                 );
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java
index 2b45aa26b87bb..98d5e4e84ced1 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.examples.ml.regression.linear;
 
-import java.util.Arrays;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
@@ -30,6 +29,8 @@
 import org.apache.ignite.ml.regressions.linear.LinearRegressionQRTrainer;
 import org.apache.ignite.thread.IgniteThread;
 
+import java.util.Arrays;
+
 /**
  * Run linear regression model over distributed matrix.
  *
@@ -113,15 +114,15 @@ public static void main(String[] args) throws InterruptedException {
                 Trainer<LinearRegressionModel, Matrix> trainer = new LinearRegressionQRTrainer();
 
                 System.out.println(">>> Perform the training to get the model.");
-                LinearRegressionModel model = trainer.train(distributedMatrix);
-                System.out.println(">>> Linear regression model: " + model);
+                LinearRegressionModel mdl = trainer.train(distributedMatrix);
+                System.out.println(">>> Linear regression model: " + mdl);
 
                 System.out.println(">>> ---------------------------------");
                 System.out.println(">>> | Prediction\t| Ground Truth\t|");
                 System.out.println(">>> ---------------------------------");
                 for (double[] observation : data) {
                     Vector inputs = new SparseDistributedVector(Arrays.copyOfRange(observation, 1, observation.length));
-                    double prediction = model.apply(inputs);
+                    double prediction = mdl.apply(inputs);
                     double groundTruth = observation[0];
                     System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
                 }
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java
index f3b2655167dc4..44366e1aafc33 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java
@@ -17,20 +17,26 @@
 
 package org.apache.ignite.examples.ml.regression.linear;
 
-import java.util.Arrays;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
-import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
 import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
 import org.apache.ignite.ml.regressions.linear.LinearRegressionQRTrainer;
 import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
+import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
 import org.apache.ignite.thread.IgniteThread;
 
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
 /**
  * Run linear regression model over distributed matrix.
  *
@@ -104,28 +110,43 @@ public static void main(String[] args) throws InterruptedException {
             // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
             // because we create ignite cache internally.
             IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                SparseDistributedMatrixExample.class.getSimpleName(), () -> {
+                DistributedLinearRegressionWithSGDTrainerExample.class.getSimpleName(), () -> {
 
-                // Create SparseDistributedMatrix, new cache will be created automagically.
-                System.out.println(">>> Create new SparseDistributedMatrix inside IgniteThread.");
-                SparseDistributedMatrix distributedMatrix = new SparseDistributedMatrix(data);
+                IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
 
                 System.out.println(">>> Create new linear regression trainer object.");
-                Trainer<LinearRegressionModel, Matrix> trainer = new LinearRegressionSGDTrainer(100_000, 1e-12);
+                LinearRegressionSGDTrainer<?> trainer = new LinearRegressionSGDTrainer<>(new UpdatesStrategy<>(
+                    new RPropUpdateCalculator(),
+                    RPropParameterUpdate::sumLocal,
+                    RPropParameterUpdate::avg
+                ), 100000,  10, 100, 123L);
 
                 System.out.println(">>> Perform the training to get the model.");
-                LinearRegressionModel model = trainer.train(distributedMatrix);
-                System.out.println(">>> Linear regression model: " + model);
+                LinearRegressionModel mdl = trainer.fit(
+                    ignite,
+                    dataCache,
+                    (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+                    (k, v) -> v[0]
+                );
+
+                System.out.println(">>> Linear regression model: " + mdl);
 
                 System.out.println(">>> ---------------------------------");
                 System.out.println(">>> | Prediction\t| Ground Truth\t|");
                 System.out.println(">>> ---------------------------------");
-                for (double[] observation : data) {
-                    Vector inputs = new SparseDistributedVector(Arrays.copyOfRange(observation, 1, observation.length));
-                    double prediction = model.apply(inputs);
-                    double groundTruth = observation[0];
-                    System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+
+                try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+                    for (Cache.Entry<Integer, double[]> observation : observations) {
+                        double[] val = observation.getValue();
+                        double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+                        double groundTruth = val[0];
+
+                        double prediction = mdl.apply(new DenseLocalOnHeapVector(inputs));
+
+                        System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+                    }
                 }
+
                 System.out.println(">>> ---------------------------------");
             });
 
@@ -134,4 +155,23 @@ public static void main(String[] args) throws InterruptedException {
             igniteThread.join();
         }
     }
+
+    /**
+     * Fills cache with data and returns it.
+     *
+     * @param ignite Ignite instance.
+     * @return Filled Ignite Cache.
+     */
+    private static IgniteCache<Integer, double[]> getTestCache(Ignite ignite) {
+        CacheConfiguration<Integer, double[]> cacheConfiguration = new CacheConfiguration<>();
+        cacheConfiguration.setName("TEST_" + UUID.randomUUID());
+        cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+        IgniteCache<Integer, double[]> cache = ignite.createCache(cacheConfiguration);
+
+        for (int i = 0; i < data.length; i++)
+            cache.put(i, data[i]);
+
+        return cache;
+    }
 }
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
index f8bf521637010..ce37112978dd5 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
@@ -17,9 +17,6 @@
 
 package org.apache.ignite.examples.ml.svm.binary;
 
-import java.util.Arrays;
-import java.util.UUID;
-import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
@@ -27,12 +24,15 @@
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
 import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
 import org.apache.ignite.thread.IgniteThread;
 
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
 /**
  * Run SVM binary-class classification model over distributed dataset.
  *
@@ -54,7 +54,8 @@ public static void main(String[] args) throws InterruptedException {
                 SVMLinearBinaryClassificationTrainer trainer = new SVMLinearBinaryClassificationTrainer();
 
                 SVMLinearBinaryClassificationModel mdl = trainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
+                    ignite,
+                    dataCache,
                     (k, v) -> Arrays.copyOfRange(v, 1, v.length),
                     (k, v) -> v[0]
                 );
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
index f8281e489f69d..4054201ee6b96 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
@@ -17,9 +17,6 @@
 
 package org.apache.ignite.examples.ml.svm.multiclass;
 
-import java.util.Arrays;
-import java.util.UUID;
-import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
@@ -27,14 +24,17 @@
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.preprocessing.normalization.NormalizationPreprocessor;
 import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
 import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel;
 import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationTrainer;
 import org.apache.ignite.thread.IgniteThread;
 
+import javax.cache.Cache;
+import java.util.Arrays;
+import java.util.UUID;
+
 /**
  * Run SVM multi-class classification trainer over distributed dataset to build two models:
  * one with normalization and one without normalization.
@@ -57,7 +57,8 @@ public static void main(String[] args) throws InterruptedException {
                 SVMLinearMultiClassClassificationTrainer trainer = new SVMLinearMultiClassClassificationTrainer();
 
                 SVMLinearMultiClassClassificationModel mdl = trainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
+                    ignite,
+                    dataCache,
                     (k, v) -> Arrays.copyOfRange(v, 1, v.length),
                     (k, v) -> v[0]
                 );
@@ -67,14 +68,15 @@ public static void main(String[] args) throws InterruptedException {
 
                 NormalizationTrainer<Integer, double[]> normalizationTrainer = new NormalizationTrainer<>();
 
-                NormalizationPreprocessor<Integer, double[]> preprocessor = normalizationTrainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
-                    (k, v) -> Arrays.copyOfRange(v, 1, v.length),
-                    5
+                IgniteBiFunction<Integer, double[], double[]> preprocessor = normalizationTrainer.fit(
+                    ignite,
+                    dataCache,
+                    (k, v) -> Arrays.copyOfRange(v, 1, v.length)
                 );
 
                 SVMLinearMultiClassClassificationModel mdlWithNormalization = trainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
+                    ignite,
+                    dataCache,
                     preprocessor,
                     (k, v) -> v[0]
                 );
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
index cef63683cb4ec..1ecf460148ddb 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
@@ -17,17 +17,17 @@
 
 package org.apache.ignite.examples.ml.tree;
 
-import java.util.Random;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
 import org.apache.ignite.ml.tree.DecisionTreeNode;
 import org.apache.ignite.thread.IgniteThread;
 
+import java.util.Random;
+
 /**
  * Example of using distributed {@link DecisionTreeClassificationTrainer}.
  */
@@ -65,7 +65,8 @@ public static void main(String... args) throws InterruptedException {
 
                 // Train decision tree model.
                 DecisionTreeNode mdl = trainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+                    ignite,
+                    trainingSet,
                     (k, v) -> new double[]{v.x, v.y},
                     (k, v) -> v.lb
                 );
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
index 61ba5f9dca471..19b15f3bbf3ca 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
@@ -22,7 +22,6 @@
 import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.ml.tree.DecisionTreeNode;
 import org.apache.ignite.ml.tree.DecisionTreeRegressionTrainer;
 import org.apache.ignite.thread.IgniteThread;
@@ -61,7 +60,8 @@ public static void main(String... args) throws InterruptedException {
 
                 // Train decision tree model.
                 DecisionTreeNode mdl = trainer.fit(
-                    new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+                    ignite,
+                    trainingSet,
                     (k, v) -> new double[] {v.x},
                     (k, v) -> v.y
                 );
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java
index f05bde8311440..4c34cd2677247 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/Activators.java
@@ -58,4 +58,24 @@ public class Activators {
             return Math.max(val, 0);
         }
     };
+
+    /**
+     * Linear unit activation function.
+     */
+    public static IgniteDifferentiableDoubleToDoubleFunction LINEAR = new IgniteDifferentiableDoubleToDoubleFunction() {
+        /** {@inheritDoc} */
+        @Override public double differential(double pnt) {
+            return 1.0;
+        }
+
+        /**
+         * Differential of linear at pnt.
+         *
+         * @param pnt Point to differentiate at.
+         * @return Differential at pnt.
+         */
+        @Override public Double apply(double pnt) {
+            return pnt;
+        }
+    };
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
index 47d20226991d2..fe955cbe696de 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
@@ -17,11 +17,6 @@
 
 package org.apache.ignite.ml.nn;
 
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-import org.apache.ignite.ml.trainers.MultiLabelDatasetTrainer;
 import org.apache.ignite.ml.dataset.Dataset;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
 import org.apache.ignite.ml.dataset.primitive.builder.context.EmptyContextBuilder;
@@ -37,17 +32,23 @@
 import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
 import org.apache.ignite.ml.nn.initializers.RandomInitializer;
 import org.apache.ignite.ml.optimization.updatecalculators.ParameterUpdateCalculator;
+import org.apache.ignite.ml.trainers.MultiLabelDatasetTrainer;
 import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
 import org.apache.ignite.ml.util.Utils;
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
 /**
  * Multilayer perceptron trainer based on partition based {@link Dataset}.
  *
  * @param <P> Type of model update used in this trainer.
  */
 public class MLPTrainer<P extends Serializable> implements MultiLabelDatasetTrainer<MultilayerPerceptron> {
-    /** Multilayer perceptron architecture that defines layers and activators. */
-    private final MLPArchitecture arch;
+    /** Multilayer perceptron architecture supplier that defines layers and activators. */
+    private final IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier;
 
     /** Loss function to be minimized during the training. */
     private final IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss;
@@ -81,7 +82,25 @@ public class MLPTrainer<P extends Serializable> implements MultiLabelDatasetTrai
     public MLPTrainer(MLPArchitecture arch, IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
         UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy, int maxIterations, int batchSize,
         int locIterations, long seed) {
-        this.arch = arch;
+        this(dataset -> arch, loss, updatesStgy, maxIterations, batchSize, locIterations, seed);
+    }
+
+    /**
+     * Constructs a new instance of multilayer perceptron trainer.
+     *
+     * @param archSupplier Multilayer perceptron architecture supplier that defines layers and activators.
+     * @param loss Loss function to be minimized during the training.
+     * @param updatesStgy Update strategy that defines how to update model parameters during the training.
+     * @param maxIterations Maximal number of iterations before the training will be stopped.
+     * @param batchSize Batch size (per every partition).
+     * @param locIterations Maximal number of local iterations before synchronization.
+     * @param seed Random initializer seed.
+     */
+    public MLPTrainer(IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier,
+        IgniteFunction<Vector, IgniteDifferentiableVectorToDoubleFunction> loss,
+        UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy, int maxIterations, int batchSize,
+        int locIterations, long seed) {
+        this.archSupplier = archSupplier;
         this.loss = loss;
         this.updatesStgy = updatesStgy;
         this.maxIterations = maxIterations;
@@ -94,13 +113,14 @@ public MLPTrainer(MLPArchitecture arch, IgniteFunction<Vector, IgniteDifferentia
     public <K, V> MultilayerPerceptron fit(DatasetBuilder<K, V> datasetBuilder,
         IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, double[]> lbExtractor) {
 
-        MultilayerPerceptron mdl = new MultilayerPerceptron(arch, new RandomInitializer(seed));
-        ParameterUpdateCalculator<? super MultilayerPerceptron, P> updater = updatesStgy.getUpdatesCalculator();
-
         try (Dataset<EmptyContext, SimpleLabeledDatasetData> dataset = datasetBuilder.build(
             new EmptyContextBuilder<>(),
             new SimpleLabeledDatasetDataBuilder<>(featureExtractor, lbExtractor)
         )) {
+            MLPArchitecture arch = archSupplier.apply(dataset);
+            MultilayerPerceptron mdl = new MultilayerPerceptron(arch, new RandomInitializer(seed));
+            ParameterUpdateCalculator<? super MultilayerPerceptron, P> updater = updatesStgy.getUpdatesCalculator();
+
             for (int i = 0; i < maxIterations; i += locIterations) {
 
                 MultilayerPerceptron finalMdl = mdl;
@@ -163,12 +183,12 @@ else if (b == null)
                 P update = updatesStgy.allUpdatesReducer().apply(totUp);
                 mdl = updater.update(mdl, update);
             }
+
+            return mdl;
         }
         catch (Exception e) {
             throw new RuntimeException(e);
         }
-
-        return mdl;
     }
 
     /**
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
index f5a6bb0ceb132..1886ee557da75 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/PreprocessingTrainer.java
@@ -17,9 +17,15 @@
 
 package org.apache.ignite.ml.preprocessing;
 
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.math.functions.IgniteBiFunction;
 
+import java.util.Map;
+
 /**
  * Trainer for preprocessor.
  *
@@ -34,9 +40,40 @@ public interface PreprocessingTrainer<K, V, T, R> {
      *
      * @param datasetBuilder Dataset builder.
      * @param basePreprocessor Base preprocessor.
-     * @param cols Number of columns.
      * @return Preprocessor.
      */
     public IgniteBiFunction<K, V, R> fit(DatasetBuilder<K, V> datasetBuilder,
-        IgniteBiFunction<K, V, T> basePreprocessor, int cols);
+        IgniteBiFunction<K, V, T> basePreprocessor);
+
+    /**
+     * Fits preprocessor.
+     *
+     * @param ignite Ignite instance.
+     * @param cache Ignite cache.
+     * @param basePreprocessor Base preprocessor.
+     * @return Preprocessor.
+     */
+    public default IgniteBiFunction<K, V, R> fit(Ignite ignite, IgniteCache<K, V> cache,
+        IgniteBiFunction<K, V, T> basePreprocessor) {
+        return fit(
+            new CacheBasedDatasetBuilder<>(ignite, cache),
+            basePreprocessor
+        );
+    }
+
+    /**
+     * Fits preprocessor.
+     *
+     * @param data Data.
+     * @param parts Number of partitions.
+     * @param basePreprocessor Base preprocessor.
+     * @return Preprocessor.
+     */
+    public default IgniteBiFunction<K, V, R> fit(Map<K, V> data, int parts,
+        IgniteBiFunction<K, V, T> basePreprocessor) {
+        return fit(
+            new LocalDatasetBuilder<>(data, parts),
+            basePreprocessor
+        );
+    }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainer.java
index 16623ba5748a7..57acbad54345c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainer.java
@@ -33,33 +33,48 @@
 public class NormalizationTrainer<K, V> implements PreprocessingTrainer<K, V, double[], double[]> {
     /** {@inheritDoc} */
     @Override public NormalizationPreprocessor<K, V> fit(DatasetBuilder<K, V> datasetBuilder,
-        IgniteBiFunction<K, V, double[]> basePreprocessor, int cols) {
+        IgniteBiFunction<K, V, double[]> basePreprocessor) {
         try (Dataset<EmptyContext, NormalizationPartitionData> dataset = datasetBuilder.build(
             (upstream, upstreamSize) -> new EmptyContext(),
             (upstream, upstreamSize, ctx) -> {
-                double[] min = new double[cols];
-                double[] max = new double[cols];
-
-                for (int i = 0; i < cols; i++) {
-                    min[i] = Double.MAX_VALUE;
-                    max[i] = -Double.MAX_VALUE;
-                }
+                double[] min = null;
+                double[] max = null;
 
                 while (upstream.hasNext()) {
                     UpstreamEntry<K, V> entity = upstream.next();
                     double[] row = basePreprocessor.apply(entity.getKey(), entity.getValue());
-                    for (int i = 0; i < cols; i++) {
+
+                    if (min == null) {
+                        min = new double[row.length];
+                        for (int i = 0; i < min.length; i++)
+                            min[i] = Double.MAX_VALUE;
+                    }
+                    else
+                        assert min.length == row.length : "Base preprocessor must return exactly " + min.length
+                            + " features";
+
+                    if (max == null) {
+                        max = new double[row.length];
+                        for (int i = 0; i < max.length; i++)
+                            max[i] = -Double.MAX_VALUE;
+                    }
+                    else
+                        assert max.length == row.length : "Base preprocessor must return exactly " + min.length
+                            + " features";
+
+                    for (int i = 0; i < row.length; i++) {
                         if (row[i] < min[i])
                             min[i] = row[i];
                         if (row[i] > max[i])
                             max[i] = row[i];
                     }
                 }
+
                 return new NormalizationPartitionData(min, max);
             }
         )) {
             double[][] minMax = dataset.compute(
-                data -> new double[][]{ data.getMin(), data.getMax() },
+                data -> data.getMin() != null ? new double[][]{ data.getMin(), data.getMax() } : null,
                 (a, b) -> {
                     if (a == null)
                         return b;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/FeatureExtractorWrapper.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/FeatureExtractorWrapper.java
new file mode 100644
index 0000000000000..8e8f467488efc
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/FeatureExtractorWrapper.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.regressions.linear;
+
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+
+import java.util.Arrays;
+
+/**
+ * Feature extractor wrapper that adds additional column filled by 1.
+ *
+ * @param <K> Type of a key in {@code upstream} data.
+ * @param <V> Type of a value in {@code upstream} data.
+ */
+public class FeatureExtractorWrapper<K, V> implements IgniteBiFunction<K, V, double[]> {
+    /** */
+    private static final long serialVersionUID = -2686524650955735635L;
+
+    /** Underlying feature extractor. */
+    private final IgniteBiFunction<K, V, double[]> featureExtractor;
+
+    /**
+     * Constructs a new instance of feature extractor wrapper.
+     *
+     * @param featureExtractor Underlying feature extractor.
+     */
+    FeatureExtractorWrapper(IgniteBiFunction<K, V, double[]> featureExtractor) {
+        this.featureExtractor = featureExtractor;
+    }
+
+    /** {@inheritDoc} */
+    @Override public double[] apply(K k, V v) {
+        double[] featureRow = featureExtractor.apply(k, v);
+        double[] row = Arrays.copyOf(featureRow, featureRow.length + 1);
+
+        row[featureRow.length] = 1.0;
+
+        return row;
+    }
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
index ae15f2f3a888e..9526db1e366ae 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
@@ -17,8 +17,6 @@
 
 package org.apache.ignite.ml.regressions.linear;
 
-import java.util.Arrays;
-import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
 import org.apache.ignite.ml.math.Vector;
 import org.apache.ignite.ml.math.functions.IgniteBiFunction;
@@ -27,6 +25,9 @@
 import org.apache.ignite.ml.math.isolve.lsqr.AbstractLSQR;
 import org.apache.ignite.ml.math.isolve.lsqr.LSQROnHeap;
 import org.apache.ignite.ml.math.isolve.lsqr.LSQRResult;
+import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
+
+import java.util.Arrays;
 
 /**
  * Trainer of the linear regression model based on LSQR algorithm.
@@ -55,37 +56,4 @@ public class LinearRegressionLSQRTrainer implements SingleLabelDatasetTrainer<Li
 
         return new LinearRegressionModel(weights, x[x.length - 1]);
     }
-
-    /**
-     * Feature extractor wrapper that adds additional column filled by 1.
-     *
-     * @param <K> Type of a key in {@code upstream} data.
-     * @param <V> Type of a value in {@code upstream} data.
-     */
-    private static class FeatureExtractorWrapper<K, V> implements IgniteBiFunction<K, V, double[]> {
-        /** */
-        private static final long serialVersionUID = -2686524650955735635L;
-
-        /** Underlying feature extractor. */
-        private final IgniteBiFunction<K, V, double[]> featureExtractor;
-
-        /**
-         * Constructs a new instance of feature extractor wrapper.
-         *
-         * @param featureExtractor Underlying feature extractor.
-         */
-        FeatureExtractorWrapper(IgniteBiFunction<K, V, double[]> featureExtractor) {
-            this.featureExtractor = featureExtractor;
-        }
-
-        /** {@inheritDoc} */
-        @Override public double[] apply(K k, V v) {
-            double[] featureRow = featureExtractor.apply(k, v);
-            double[] row = Arrays.copyOf(featureRow, featureRow.length + 1);
-
-            row[featureRow.length] = 1.0;
-
-            return row;
-        }
-    }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
index aad4c7a73fd16..9be3fdd2a8aa2 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
@@ -17,51 +17,99 @@
 
 package org.apache.ignite.ml.regressions.linear;
 
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.optimization.BarzilaiBorweinUpdater;
-import org.apache.ignite.ml.optimization.GradientDescent;
-import org.apache.ignite.ml.optimization.LeastSquaresGradientFunction;
-import org.apache.ignite.ml.optimization.SimpleUpdater;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.dataset.primitive.data.SimpleLabeledDatasetData;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.functions.IgniteFunction;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.nn.Activators;
+import org.apache.ignite.ml.nn.MLPTrainer;
+import org.apache.ignite.ml.nn.MultilayerPerceptron;
+import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
+import org.apache.ignite.ml.optimization.LossFunctions;
+import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
+import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
+
+import java.io.Serializable;
+import java.util.Arrays;
 
 /**
- * Linear regression trainer based on least squares loss function and gradient descent optimization algorithm.
+ * Trainer of the linear regression model based on stochastic gradient descent algorithm.
  */
-public class LinearRegressionSGDTrainer implements Trainer<LinearRegressionModel, Matrix> {
-    /**
-     * Gradient descent optimizer.
-     */
-    private final GradientDescent gradientDescent;
+public class LinearRegressionSGDTrainer<P extends Serializable> implements SingleLabelDatasetTrainer<LinearRegressionModel> {
+    /** Update strategy. */
+    private final UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy;
 
-    /** */
-    public LinearRegressionSGDTrainer(GradientDescent gradientDescent) {
-        this.gradientDescent = gradientDescent;
-    }
+    /** Max number of iteration. */
+    private final int maxIterations;
 
-    /** */
-    public LinearRegressionSGDTrainer(int maxIterations, double convergenceTol) {
-        this.gradientDescent = new GradientDescent(new LeastSquaresGradientFunction(), new BarzilaiBorweinUpdater())
-            .withMaxIterations(maxIterations)
-            .withConvergenceTol(convergenceTol);
-    }
+    /** Batch size. */
+    private final int batchSize;
 
-    /** */
-    public LinearRegressionSGDTrainer(int maxIterations, double convergenceTol, double learningRate) {
-        this.gradientDescent = new GradientDescent(new LeastSquaresGradientFunction(), new SimpleUpdater(learningRate))
-            .withMaxIterations(maxIterations)
-            .withConvergenceTol(convergenceTol);
-    }
+    /** Number of local iterations. */
+    private final int locIterations;
+
+    /** Seed for random generator. */
+    private final long seed;
 
     /**
-     * {@inheritDoc}
+     * Constructs a new instance of linear regression SGD trainer.
+     *
+     * @param updatesStgy Update strategy.
+     * @param maxIterations Max number of iteration.
+     * @param batchSize Batch size.
+     * @param locIterations Number of local iterations.
+     * @param seed Seed for random generator.
      */
-    @Override public LinearRegressionModel train(Matrix data) {
-        Vector variables = gradientDescent.optimize(data, data.likeVector(data.columnSize()));
-        Vector weights = variables.viewPart(1, variables.size() - 1);
+    public LinearRegressionSGDTrainer(UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy, int maxIterations,
+        int batchSize, int locIterations, long seed) {
+        this.updatesStgy = updatesStgy;
+        this.maxIterations = maxIterations;
+        this.batchSize = batchSize;
+        this.locIterations = locIterations;
+        this.seed = seed;
+    }
+
+    /** {@inheritDoc} */
+    @Override public <K, V> LinearRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
+        IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+
+        IgniteFunction<Dataset<EmptyContext, SimpleLabeledDatasetData>, MLPArchitecture> archSupplier = dataset -> {
+
+            int cols = dataset.compute(data -> {
+                if (data.getFeatures() == null)
+                    return null;
+                return data.getFeatures().length / data.getRows();
+            }, (a, b) -> a == null ? b : a);
+
+            MLPArchitecture architecture = new MLPArchitecture(cols);
+            architecture = architecture.withAddedLayer(1, true, Activators.LINEAR);
+
+            return architecture;
+        };
+
+        MLPTrainer<?> trainer = new MLPTrainer<>(
+            archSupplier,
+            LossFunctions.MSE,
+            updatesStgy,
+            maxIterations,
+            batchSize,
+            locIterations,
+            seed
+        );
+
+        IgniteBiFunction<K, V, double[]> lbE = new IgniteBiFunction<K, V, double[]>() {
+            @Override public double[] apply(K k, V v) {
+                return new double[]{lbExtractor.apply(k, v)};
+            }
+        };
+
+        MultilayerPerceptron mlp = trainer.fit(datasetBuilder, featureExtractor, lbE);
 
-        double intercept = variables.get(0);
+        double[] p = mlp.parameters().getStorage().data();
 
-        return new LinearRegressionModel(weights, intercept);
+        return new LinearRegressionModel(new DenseLocalOnHeapVector(Arrays.copyOf(p, p.length - 1)), p[p.length - 1]);
     }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
index 8119a2996f36e..fcde3f57ca45c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/DatasetTrainer.java
@@ -17,10 +17,16 @@
 
 package org.apache.ignite.ml.trainers;
 
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.ml.Model;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.math.functions.IgniteBiFunction;
 
+import java.util.Map;
+
 /**
  * Interface for trainers. Trainer is just a function which produces model from the data.
  *
@@ -40,4 +46,44 @@ public interface DatasetTrainer<M extends Model, L> {
      */
     public <K, V> M fit(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, double[]> featureExtractor,
         IgniteBiFunction<K, V, L> lbExtractor);
+
+    /**
+     * Trains model based on the specified data.
+     *
+     * @param ignite Ignite instance.
+     * @param cache Ignite cache.
+     * @param featureExtractor Feature extractor.
+     * @param lbExtractor Label extractor.
+     * @param <K> Type of a key in {@code upstream} data.
+     * @param <V> Type of a value in {@code upstream} data.
+     * @return Model.
+     */
+    public default <K, V> M fit(Ignite ignite, IgniteCache<K, V> cache, IgniteBiFunction<K, V, double[]> featureExtractor,
+        IgniteBiFunction<K, V, L> lbExtractor) {
+        return fit(
+            new CacheBasedDatasetBuilder<>(ignite, cache),
+            featureExtractor,
+            lbExtractor
+        );
+    }
+
+    /**
+     * Trains model based on the specified data.
+     *
+     * @param data Data.
+     * @param parts Number of partitions.
+     * @param featureExtractor Feature extractor.
+     * @param lbExtractor Label extractor.
+     * @param <K> Type of a key in {@code upstream} data.
+     * @param <V> Type of a value in {@code upstream} data.
+     * @return Model.
+     */
+    public default <K, V> M fit(Map<K, V> data, int parts, IgniteBiFunction<K, V, double[]> featureExtractor,
+        IgniteBiFunction<K, V, L> lbExtractor) {
+        return fit(
+            new LocalDatasetBuilder<>(data, parts),
+            featureExtractor,
+            lbExtractor
+        );
+    }
 }
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
index b5a4b540a2232..b27fcba20d246 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
@@ -17,11 +17,7 @@
 
 package org.apache.ignite.ml.knn;
 
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
 import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
 import org.apache.ignite.ml.knn.classification.KNNClassificationTrainer;
 import org.apache.ignite.ml.knn.classification.KNNStrategy;
@@ -29,6 +25,10 @@
 import org.apache.ignite.ml.math.distances.EuclideanDistance;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
 /** Tests behaviour of KNNClassificationTest. */
 public class KNNClassificationTest extends BaseKNNTest {
     /** */
@@ -46,7 +46,8 @@ public void testBinaryClassificationTest() {
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
         KNNClassificationModel knnMdl = trainer.fit(
-            new LocalDatasetBuilder<>(data, 2),
+            data,
+            2,
             (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
             (k, v) -> v[2]
         ).withK(3)
@@ -74,7 +75,8 @@ public void testBinaryClassificationWithSmallestKTest() {
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
         KNNClassificationModel knnMdl = trainer.fit(
-            new LocalDatasetBuilder<>(data, 2),
+            data,
+            2,
             (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
             (k, v) -> v[2]
         ).withK(1)
@@ -102,7 +104,8 @@ public void testBinaryClassificationFarPointsWithSimpleStrategy() {
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
         KNNClassificationModel knnMdl = trainer.fit(
-            new LocalDatasetBuilder<>(data, 2),
+            data,
+            2,
             (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
             (k, v) -> v[2]
         ).withK(3)
@@ -128,7 +131,8 @@ public void testBinaryClassificationFarPointsWithWeightedStrategy() {
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
         KNNClassificationModel knnMdl = trainer.fit(
-            new LocalDatasetBuilder<>(data, 2),
+            data,
+            2,
             (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
             (k, v) -> v[2]
         ).withK(3)
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java
index 5ca661fdbc4b2..038b880f5cebc 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.ml.nn;
 
-import java.io.Serializable;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
@@ -25,22 +24,18 @@
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.ml.math.Matrix;
 import org.apache.ignite.ml.math.Tracer;
 import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
 import org.apache.ignite.ml.optimization.LossFunctions;
-import org.apache.ignite.ml.optimization.updatecalculators.NesterovParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.NesterovUpdateCalculator;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
-import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.*;
 import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import java.io.Serializable;
+
 /**
  * Tests for {@link MLPTrainer} that require to start the whole Ignite infrastructure.
  */
@@ -137,7 +132,8 @@ private <P extends Serializable> void xorTest(UpdatesStrategy<? super Multilayer
             );
 
             MultilayerPerceptron mlp = trainer.fit(
-                new CacheBasedDatasetBuilder<>(ignite, xorCache),
+                ignite,
+                xorCache,
                 (k, v) -> new double[]{ v.x, v.y },
                 (k, v) -> new double[]{ v.lb}
             );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
index 6906424959906..c53f6f104d84e 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
@@ -17,24 +17,13 @@
 
 package org.apache.ignite.ml.nn;
 
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.math.Matrix;
 import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
 import org.apache.ignite.ml.optimization.LossFunctions;
-import org.apache.ignite.ml.optimization.updatecalculators.NesterovParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.NesterovUpdateCalculator;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
-import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
+import org.apache.ignite.ml.optimization.updatecalculators.*;
 import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
 import org.junit.Before;
 import org.junit.Test;
@@ -42,6 +31,12 @@
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 /**
  * Tests for {@link MLPTrainer} that don't require to start the whole Ignite infrastructure.
  */
@@ -140,7 +135,8 @@ private <P extends Serializable> void xorTest(UpdatesStrategy<? super Multilayer
             );
 
             MultilayerPerceptron mlp = trainer.fit(
-                new LocalDatasetBuilder<>(xorData, parts),
+                xorData,
+                parts,
                 (k, v) -> v[0],
                 (k, v) -> v[1]
             );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java
index c787a47da53c4..a64af9b5cc545 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java
@@ -17,13 +17,11 @@
 
 package org.apache.ignite.ml.nn.performance;
 
-import java.io.IOException;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.ml.math.Matrix;
 import org.apache.ignite.ml.math.VectorUtils;
 import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
@@ -38,6 +36,8 @@
 import org.apache.ignite.ml.util.MnistUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import java.io.IOException;
+
 /**
  * Tests {@link MLPTrainer} on the MNIST dataset that require to start the whole Ignite infrastructure.
  */
@@ -104,7 +104,8 @@ public void testMNIST() throws IOException {
         System.out.println("Start training...");
         long start = System.currentTimeMillis();
         MultilayerPerceptron mdl = trainer.fit(
-            new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+            ignite,
+            trainingSet,
             (k, v) -> v.getPixels(),
             (k, v) -> VectorUtils.num2Vec(v.getLabel(), 10).getStorage().data()
         );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java
index 354af2cb68e58..d966484d548c6 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java
@@ -17,10 +17,6 @@
 
 package org.apache.ignite.ml.nn.performance;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.math.Matrix;
 import org.apache.ignite.ml.math.VectorUtils;
 import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
@@ -35,6 +31,10 @@
 import org.apache.ignite.ml.util.MnistUtils;
 import org.junit.Test;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -74,7 +74,8 @@ public void testMNIST() throws IOException {
         System.out.println("Start training...");
         long start = System.currentTimeMillis();
         MultilayerPerceptron mdl = trainer.fit(
-            new LocalDatasetBuilder<>(trainingSet, 1),
+            trainingSet,
+            1,
             (k, v) -> v.getPixels(),
             (k, v) -> VectorUtils.num2Vec(v.getLabel(), 10).getStorage().data()
         );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainerTest.java
index 15482539f22e3..e7a0d47b1f861 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/preprocessing/normalization/NormalizationTrainerTest.java
@@ -17,15 +17,16 @@
 
 package org.apache.ignite.ml.preprocessing.normalization;
 
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
 import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
 import static org.junit.Assert.assertArrayEquals;
 
 /**
@@ -66,8 +67,7 @@ public void testFit() {
 
         NormalizationPreprocessor<Integer, double[]> preprocessor = standardizationTrainer.fit(
             datasetBuilder,
-            (k, v) -> v,
-            3
+            (k, v) -> v
         );
 
         assertArrayEquals(new double[] {0, 4, 1}, preprocessor.getMin(), 1e-8);
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java
index 82b3a1b24df63..b3c9368194366 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java
@@ -17,14 +17,7 @@
 
 package org.apache.ignite.ml.regressions;
 
-import org.apache.ignite.ml.regressions.linear.BlockDistributedLinearRegressionQRTrainerTest;
-import org.apache.ignite.ml.regressions.linear.BlockDistributedLinearRegressionSGDTrainerTest;
-import org.apache.ignite.ml.regressions.linear.DistributedLinearRegressionQRTrainerTest;
-import org.apache.ignite.ml.regressions.linear.DistributedLinearRegressionSGDTrainerTest;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainerTest;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionModelTest;
-import org.apache.ignite.ml.regressions.linear.LocalLinearRegressionQRTrainerTest;
-import org.apache.ignite.ml.regressions.linear.LocalLinearRegressionSGDTrainerTest;
+import org.apache.ignite.ml.regressions.linear.*;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 
@@ -35,12 +28,10 @@
 @Suite.SuiteClasses({
     LinearRegressionModelTest.class,
     LocalLinearRegressionQRTrainerTest.class,
-    LocalLinearRegressionSGDTrainerTest.class,
     DistributedLinearRegressionQRTrainerTest.class,
-    DistributedLinearRegressionSGDTrainerTest.class,
     BlockDistributedLinearRegressionQRTrainerTest.class,
-    BlockDistributedLinearRegressionSGDTrainerTest.class,
-    LinearRegressionLSQRTrainerTest.class
+    LinearRegressionLSQRTrainerTest.class,
+    LinearRegressionSGDTrainerTest.class
 })
 public class RegressionsTestSuite {
     // No-op.
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionSGDTrainerTest.java
deleted file mode 100644
index 58037e2e8d883..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionSGDTrainerTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.SparseBlockDistributedVector;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link SparseBlockDistributedMatrix}.
- */
-public class BlockDistributedLinearRegressionSGDTrainerTest extends GridAwareAbstractLinearRegressionTrainerTest {
-    /** */
-    public BlockDistributedLinearRegressionSGDTrainerTest() {
-        super(
-            new LinearRegressionSGDTrainer(100_000, 1e-12),
-            SparseBlockDistributedMatrix::new,
-            SparseBlockDistributedVector::new,
-            1e-2);
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionSGDTrainerTest.java
deleted file mode 100644
index 71d3b3ba6a15d..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionSGDTrainerTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link SparseDistributedMatrix}.
- */
-public class DistributedLinearRegressionSGDTrainerTest extends GridAwareAbstractLinearRegressionTrainerTest {
-    /** */
-    public DistributedLinearRegressionSGDTrainerTest() {
-        super(
-            new LinearRegressionSGDTrainer(100_000, 1e-12),
-            SparseDistributedMatrix::new,
-            SparseDistributedVector::new,
-            1e-2);
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java
index 1a60b80addf49..9b75bd41392a6 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java
@@ -26,6 +26,9 @@
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.junit.Test;
 
+/**
+ * Grid aware abstract linear regression trainer test.
+ */
 public abstract class GridAwareAbstractLinearRegressionTrainerTest extends GridCommonAbstractTest {
     /** Number of nodes in grid */
     private static final int NODE_COUNT = 3;
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
index e3f60ec9772a3..2414236a33198 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainerTest.java
@@ -17,14 +17,14 @@
 
 package org.apache.ignite.ml.regressions.linear;
 
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
@@ -72,7 +72,8 @@ public void testSmallDataFit() {
         LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
 
         LinearRegressionModel mdl = trainer.fit(
-            new LocalDatasetBuilder<>(data, parts),
+            data,
+            parts,
             (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
             (k, v) -> v[4]
         );
@@ -110,7 +111,8 @@ public void testBigDataFit() {
         LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
 
         LinearRegressionModel mdl = trainer.fit(
-            new LocalDatasetBuilder<>(data, parts),
+            data,
+            parts,
             (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
             (k, v) -> v[coef.length]
         );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
new file mode 100644
index 0000000000000..fa8fac408b112
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.regressions.linear;
+
+import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
+import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link LinearRegressionSGDTrainer}.
+ */
+@RunWith(Parameterized.class)
+public class LinearRegressionSGDTrainerTest {
+    /** Parameters. */
+    @Parameterized.Parameters(name = "Data divided on {0} partitions")
+    public static Iterable<Integer[]> data() {
+        return Arrays.asList(
+            new Integer[] {1},
+            new Integer[] {2},
+            new Integer[] {3},
+            new Integer[] {5},
+            new Integer[] {7},
+            new Integer[] {100}
+        );
+    }
+
+    /** Number of partitions. */
+    @Parameterized.Parameter
+    public int parts;
+
+    /**
+     * Tests {@code fit()} method on a simple small dataset.
+     */
+    @Test
+    public void testSmallDataFit() {
+        Map<Integer, double[]> data = new HashMap<>();
+        data.put(0, new double[] {-1.0915526, 1.81983527, -0.91409478, 0.70890712, -24.55724107});
+        data.put(1, new double[] {-0.61072904, 0.37545517, 0.21705352, 0.09516495, -26.57226867});
+        data.put(2, new double[] {0.05485406, 0.88219898, -0.80584547, 0.94668307, 61.80919728});
+        data.put(3, new double[] {-0.24835094, -0.34000053, -1.69984651, -1.45902635, -161.65525991});
+        data.put(4, new double[] {0.63675392, 0.31675535, 0.38837437, -1.1221971, -14.46432611});
+        data.put(5, new double[] {0.14194017, 2.18158997, -0.28397346, -0.62090588, -3.2122197});
+        data.put(6, new double[] {-0.53487507, 1.4454797, 0.21570443, -0.54161422, -46.5469012});
+        data.put(7, new double[] {-1.58812173, -0.73216803, -2.15670676, -1.03195988, -247.23559889});
+        data.put(8, new double[] {0.20702671, 0.92864654, 0.32721202, -0.09047503, 31.61484949});
+        data.put(9, new double[] {-0.37890345, -0.04846179, -0.84122753, -1.14667474, -124.92598583});
+
+        LinearRegressionSGDTrainer<?> trainer = new LinearRegressionSGDTrainer<>(new UpdatesStrategy<>(
+            new RPropUpdateCalculator(),
+            RPropParameterUpdate::sumLocal,
+            RPropParameterUpdate::avg
+        ), 100000,  10, 100, 123L);
+
+        LinearRegressionModel mdl = trainer.fit(
+            data,
+            parts,
+            (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
+            (k, v) -> v[4]
+        );
+
+        assertArrayEquals(
+            new double[] {72.26948107, 15.95144674, 24.07403921, 66.73038781},
+            mdl.getWeights().getStorage().data(),
+            1e-1
+        );
+
+        assertEquals(2.8421709430404007e-14, mdl.getIntercept(), 1e-1);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionSGDTrainerTest.java
deleted file mode 100644
index bea164d603c48..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionSGDTrainerTest.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/**
- * Tests for {@link LinearRegressionSGDTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class LocalLinearRegressionSGDTrainerTest extends GenericLinearRegressionTrainerTest {
-    /** */
-    public LocalLinearRegressionSGDTrainerTest() {
-        super(
-            new LinearRegressionSGDTrainer(100_000, 1e-12),
-            DenseLocalOnHeapMatrix::new,
-            DenseLocalOnHeapVector::new,
-            1e-2);
-    }
-}
\ No newline at end of file
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
index 26ba2fbba117e..0befd9b503831 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMBinaryTrainerTest.java
@@ -17,14 +17,14 @@
 
 package org.apache.ignite.ml.svm;
 
+import org.apache.ignite.ml.TestUtils;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.junit.Test;
+
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.junit.Test;
 
 /**
  * Tests for {@link SVMLinearBinaryClassificationTrainer}.
@@ -62,7 +62,8 @@ public void testTrainWithTheLinearlySeparableCase() {
         SVMLinearBinaryClassificationTrainer trainer = new SVMLinearBinaryClassificationTrainer();
 
         SVMLinearBinaryClassificationModel mdl = trainer.fit(
-            new LocalDatasetBuilder<>(data, 10),
+            data,
+            10,
             (k, v) -> Arrays.copyOfRange(v, 1, v.length),
             (k, v) -> v[0]
         );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
index ad95eb4542c4c..31ab4d7d7a368 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/svm/SVMMultiClassTrainerTest.java
@@ -17,14 +17,14 @@
 
 package org.apache.ignite.ml.svm;
 
+import org.apache.ignite.ml.TestUtils;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.junit.Test;
+
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.junit.Test;
 
 /**
  * Tests for {@link SVMLinearBinaryClassificationTrainer}.
@@ -65,7 +65,8 @@ public void testTrainWithTheLinearlySeparableCase() {
             .withAmountOfIterations(20);
 
         SVMLinearMultiClassClassificationModel mdl = trainer.fit(
-            new LocalDatasetBuilder<>(data, 10),
+            data,
+            10,
             (k, v) -> Arrays.copyOfRange(v, 1, v.length),
             (k, v) -> v[0]
         );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
index 94bca3f83e2f6..d5b0b86229e0d 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerIntegrationTest.java
@@ -17,16 +17,16 @@
 
 package org.apache.ignite.ml.tree;
 
-import java.util.Arrays;
-import java.util.Random;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import java.util.Arrays;
+import java.util.Random;
+
 /**
  * Tests for {@link DecisionTreeClassificationTrainer} that require to start the whole Ignite infrastructure.
  */
@@ -77,7 +77,8 @@ public void testFit() {
         DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(1, 0);
 
         DecisionTreeNode tree = trainer.fit(
-            new CacheBasedDatasetBuilder<>(ignite, data),
+            ignite,
+            data,
             (k, v) -> Arrays.copyOf(v, v.length - 1),
             (k, v) -> v[v.length - 1]
         );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
index 2599bfe2b17e1..12ef698c2df06 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeClassificationTrainerTest.java
@@ -17,17 +17,12 @@
 
 package org.apache.ignite.ml.tree;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import java.util.*;
+
 import static junit.framework.TestCase.assertEquals;
 import static junit.framework.TestCase.assertTrue;
 
@@ -68,7 +63,8 @@ public void testFit() {
         DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(1, 0);
 
         DecisionTreeNode tree = trainer.fit(
-            new LocalDatasetBuilder<>(data, parts),
+            data,
+            parts,
             (k, v) -> Arrays.copyOf(v, v.length - 1),
             (k, v) -> v[v.length - 1]
         );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
index 754ff20f8046e..c2a463844c202 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerIntegrationTest.java
@@ -17,16 +17,16 @@
 
 package org.apache.ignite.ml.tree;
 
-import java.util.Arrays;
-import java.util.Random;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import java.util.Arrays;
+import java.util.Random;
+
 /**
  * Tests for {@link DecisionTreeRegressionTrainer} that require to start the whole Ignite infrastructure.
  */
@@ -77,7 +77,8 @@ public void testFit() {
         DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0);
 
         DecisionTreeNode tree = trainer.fit(
-            new CacheBasedDatasetBuilder<>(ignite, data),
+            ignite,
+            data,
             (k, v) -> Arrays.copyOf(v, v.length - 1),
             (k, v) -> v[v.length - 1]
         );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
index 3bdbf60675a3f..bcfb53fc6b320 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/DecisionTreeRegressionTrainerTest.java
@@ -17,17 +17,12 @@
 
 package org.apache.ignite.ml.tree;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import java.util.*;
+
 import static junit.framework.TestCase.assertEquals;
 import static junit.framework.TestCase.assertTrue;
 
@@ -68,7 +63,8 @@ public void testFit() {
         DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(1, 0);
 
         DecisionTreeNode tree = trainer.fit(
-            new LocalDatasetBuilder<>(data, parts),
+            data,
+            parts,
             (k, v) -> Arrays.copyOf(v, v.length - 1),
             (k, v) -> v[v.length - 1]
         );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
index b259ec9700de5..35f805e37e4a4 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTIntegrationTest.java
@@ -17,13 +17,11 @@
 
 package org.apache.ignite.ml.tree.performance;
 
-import java.io.IOException;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
 import org.apache.ignite.ml.nn.performance.MnistMLPTestUtil;
 import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
 import org.apache.ignite.ml.tree.DecisionTreeNode;
@@ -31,6 +29,8 @@
 import org.apache.ignite.ml.util.MnistUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import java.io.IOException;
+
 /**
  * Tests {@link DecisionTreeClassificationTrainer} on the MNIST dataset that require to start the whole Ignite
  * infrastructure. For manual run.
@@ -81,7 +81,8 @@ public void testMNIST() throws IOException {
             new SimpleStepFunctionCompressor<>());
 
         DecisionTreeNode mdl = trainer.fit(
-            new CacheBasedDatasetBuilder<>(ignite, trainingSet),
+            ignite,
+            trainingSet,
             (k, v) -> v.getPixels(),
             (k, v) -> (double) v.getLabel()
         );
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
index 6dbd44c5919b3..b40c7ac2bfe6f 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/tree/performance/DecisionTreeMNISTTest.java
@@ -17,10 +17,6 @@
 
 package org.apache.ignite.ml.tree.performance;
 
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.nn.performance.MnistMLPTestUtil;
 import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
 import org.apache.ignite.ml.tree.DecisionTreeNode;
@@ -28,6 +24,10 @@
 import org.apache.ignite.ml.util.MnistUtils;
 import org.junit.Test;
 
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
 import static junit.framework.TestCase.assertTrue;
 
 /**
@@ -50,7 +50,8 @@ public void testMNIST() throws IOException {
             new SimpleStepFunctionCompressor<>());
 
         DecisionTreeNode mdl = trainer.fit(
-            new LocalDatasetBuilder<>(trainingSet, 10),
+            trainingSet,
+            10,
             (k, v) -> v.getPixels(),
             (k, v) -> (double) v.getLabel()
         );

From f143ad0057ddb326f6d8199bf660b354913e6b61 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Thu, 12 Apr 2018 15:02:57 +0300
Subject: [PATCH 038/543] IGNITE-8135: SQL: authentication for CREATE TABLE and
 DROP TABLE commands. This closes #3801.

---
 .../apache/ignite/client/ClientException.java |  3 +-
 .../client/thin/ClientQueryCursor.java        |  6 ++-
 .../platform/client/ClientRequestHandler.java |  7 ++-
 .../ClientCacheSqlFieldsQueryRequest.java     | 19 +++++--
 .../security/SecurityContextHolder.java       | 53 +++++++++++++++++++
 .../query/h2/ddl/DdlStatementsProcessor.java  |  9 ++++
 6 files changed, 91 insertions(+), 6 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java

diff --git a/modules/core/src/main/java/org/apache/ignite/client/ClientException.java b/modules/core/src/main/java/org/apache/ignite/client/ClientException.java
index 05556358fb53c..b0d9f6cc5cfee 100644
--- a/modules/core/src/main/java/org/apache/ignite/client/ClientException.java
+++ b/modules/core/src/main/java/org/apache/ignite/client/ClientException.java
@@ -20,7 +20,7 @@
 /**
  * Common thin client checked exception.
  */
-public class ClientException extends Exception {
+public class ClientException extends RuntimeException {
     /** Serial version uid. */
     private static final long serialVersionUID = 0L;
 
@@ -28,6 +28,7 @@ public class ClientException extends Exception {
      * Constructs a new exception with {@code null} as its detail message.
      */
     public ClientException() {
+        // No-op.
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientQueryCursor.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientQueryCursor.java
index 9367cfd52a0af..086fab875bbf3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientQueryCursor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientQueryCursor.java
@@ -54,6 +54,7 @@ class ClientQueryCursor<T> implements QueryCursor<T> {
             pager.close();
         }
         catch (Exception ignored) {
+            // No-op.
         }
     }
 
@@ -76,7 +77,10 @@ class ClientQueryCursor<T> implements QueryCursor<T> {
                         currPageIt = currPage.iterator();
                     }
                     catch (ClientException e) {
-                        throw new RuntimeException("Failed to retrieve query results", e);
+                        throw e;
+                    }
+                    catch (Exception e) {
+                        throw new ClientException("Failed to retrieve query results", e);
                     }
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java
index faa50bcd0f094..5ed0d38d8eac5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
 import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler;
 import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
+import org.apache.ignite.internal.processors.security.SecurityContextHolder;
 
 /**
  * Thin client request handler.
@@ -47,8 +48,10 @@ public class ClientRequestHandler implements ClientListenerRequestHandler {
 
     /** {@inheritDoc} */
     @Override public ClientListenerResponse handle(ClientListenerRequest req) {
-        if (authCtx != null)
+        if (authCtx != null) {
             AuthorizationContext.context(authCtx);
+            SecurityContextHolder.set(ctx.securityContext());
+        }
 
         try {
             return ((ClientRequest)req).process(ctx);
@@ -56,6 +59,8 @@ public class ClientRequestHandler implements ClientListenerRequestHandler {
         finally {
             if (authCtx != null)
                 AuthorizationContext.clear();
+
+            SecurityContextHolder.clear();
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
index 3aa95bf2a1a37..53f6353cc2a50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java
@@ -28,8 +28,11 @@
 import org.apache.ignite.internal.processors.platform.cache.PlatformCache;
 import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext;
 import org.apache.ignite.internal.processors.platform.client.ClientResponse;
+import org.apache.ignite.internal.processors.platform.client.ClientStatus;
+import org.apache.ignite.internal.processors.platform.client.IgniteClientException;
 import org.apache.ignite.internal.processors.query.QueryUtils;
-import org.apache.ignite.plugin.security.SecurityPermission;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.plugin.security.SecurityException;
 
 /**
  * Sql query request.
@@ -95,7 +98,7 @@ public ClientCacheSqlFieldsQueryRequest(BinaryRawReaderEx reader) {
 
                 if (qry.getSchema() == null) {
                     String schema = QueryUtils.normalizeSchemaName(desc.cacheName(),
-                            desc.cacheConfiguration().getSqlSchema());
+                        desc.cacheConfiguration().getSqlSchema());
 
                     qry.setSchema(schema);
                 }
@@ -108,7 +111,7 @@ public ClientCacheSqlFieldsQueryRequest(BinaryRawReaderEx reader) {
             FieldsQueryCursor cur = curs.get(0);
 
             ClientCacheFieldsQueryCursor cliCur = new ClientCacheFieldsQueryCursor(
-                    cur, qry.getPageSize(), ctx);
+                cur, qry.getPageSize(), ctx);
 
             long cursorId = ctx.resources().put(cliCur);
 
@@ -119,6 +122,16 @@ public ClientCacheSqlFieldsQueryRequest(BinaryRawReaderEx reader) {
         catch (Exception e) {
             ctx.decrementCursors();
 
+            SecurityException securityEx = X.cause(e, SecurityException.class);
+
+            if (securityEx != null) {
+                throw new IgniteClientException(
+                    ClientStatus.SECURITY_VIOLATION,
+                    "Client is not authorized to perform this operation",
+                    securityEx
+                );
+            }
+
             throw e;
         }
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java
new file mode 100644
index 0000000000000..14d70c97cbfdb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/security/SecurityContextHolder.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.security;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Thread-local security context.
+ */
+public class SecurityContextHolder {
+    /** Context. */
+    private static final ThreadLocal<SecurityContext> CTX = new ThreadLocal<>();
+
+    /**
+     * Get security context.
+     *
+     * @return Security context.
+     */
+    @Nullable public static SecurityContext get() {
+        return CTX.get();
+    }
+
+    /**
+     * Set security context.
+     *
+     * @param ctx Context.
+     */
+    public static void set(@Nullable SecurityContext ctx) {
+        CTX.set(ctx);
+    }
+
+    /**
+     * Clear security context.
+     */
+    public static void clear() {
+        set(null);
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
index b14896991ba01..bc5c1e0208848 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
@@ -34,6 +34,8 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.authentication.AuthorizationContext;
+import org.apache.ignite.internal.processors.authentication.UserManagementOperation;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
@@ -56,6 +58,8 @@
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement;
 import org.apache.ignite.internal.processors.query.schema.SchemaOperationException;
+import org.apache.ignite.internal.processors.security.SecurityContext;
+import org.apache.ignite.internal.processors.security.SecurityContextHolder;
 import org.apache.ignite.internal.sql.command.SqlAlterTableCommand;
 import org.apache.ignite.internal.sql.command.SqlAlterUserCommand;
 import org.apache.ignite.internal.sql.command.SqlCommand;
@@ -67,6 +71,7 @@
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.plugin.security.SecurityPermission;
 import org.h2.command.Prepared;
 import org.h2.command.ddl.AlterTableAlterColumn;
 import org.h2.command.ddl.CreateIndex;
@@ -316,6 +321,8 @@ else if (stmt0 instanceof GridSqlDropIndex) {
                 }
             }
             else if (stmt0 instanceof GridSqlCreateTable) {
+                ctx.security().authorize(null, SecurityPermission.CACHE_CREATE, SecurityContextHolder.get());
+
                 GridSqlCreateTable cmd = (GridSqlCreateTable)stmt0;
 
                 if (!F.eq(QueryUtils.DFLT_SCHEMA, cmd.schemaName()))
@@ -349,6 +356,8 @@ else if (stmt0 instanceof GridSqlCreateTable) {
                 }
             }
             else if (stmt0 instanceof GridSqlDropTable) {
+                ctx.security().authorize(null, SecurityPermission.CACHE_DESTROY, SecurityContextHolder.get());
+
                 GridSqlDropTable cmd = (GridSqlDropTable)stmt0;
 
                 if (!F.eq(QueryUtils.DFLT_SCHEMA, cmd.schemaName()))

From 2c4a7a2e366d7308485ae5cc95d4b60e66b09589 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Thu, 12 Apr 2018 15:13:51 +0300
Subject: [PATCH 039/543] IGNITE-8230: SQL: Fixed backup number propagation in
 CREATE TABLE command. This closes #3803.

---
 .../processors/query/GridQueryProcessor.java  |  7 ++--
 .../query/h2/sql/GridSqlCreateTable.java      |  7 ++--
 .../cache/index/H2DynamicTableSelfTest.java   | 37 +++++++++++++++++++
 3 files changed, 45 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index bde9427677963..03e5254183d19 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -1483,10 +1483,10 @@ else if (op instanceof SchemaAlterTableDropColumnOperation) {
     @SuppressWarnings("unchecked")
     public void dynamicTableCreate(String schemaName, QueryEntity entity, String templateName, String cacheName,
         String cacheGroup, @Nullable String dataRegion, String affinityKey, @Nullable CacheAtomicityMode atomicityMode,
-        @Nullable CacheWriteSynchronizationMode writeSyncMode, int backups, boolean ifNotExists)
+        @Nullable CacheWriteSynchronizationMode writeSyncMode, @Nullable Integer backups, boolean ifNotExists)
         throws IgniteCheckedException {
         assert !F.isEmpty(templateName);
-        assert backups >= 0;
+        assert backups == null || backups >= 0;
 
         CacheConfiguration<?, ?> ccfg = ctx.cache().getConfigFromTemplate(templateName);
 
@@ -1525,7 +1525,8 @@ else if (QueryUtils.TEMPLATE_REPLICATED.equalsIgnoreCase(templateName))
         if (writeSyncMode != null)
             ccfg.setWriteSynchronizationMode(writeSyncMode);
 
-        ccfg.setBackups(backups);
+        if (backups != null)
+            ccfg.setBackups(backups);
 
         ccfg.setSqlSchema(schemaName);
         ccfg.setSqlEscapeAll(true);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlCreateTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlCreateTable.java
index 3608aedb40dc7..de86d6aa71e8c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlCreateTable.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlCreateTable.java
@@ -22,6 +22,7 @@
 import java.util.List;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * CREATE TABLE statement.
@@ -57,7 +58,7 @@ public class GridSqlCreateTable extends GridSqlStatement {
     private CacheWriteSynchronizationMode writeSyncMode;
 
     /** Backups number for new cache. */
-    private int backups;
+    private Integer backups;
 
     /** Quietly ignore this command if table already exists. */
     private boolean ifNotExists;
@@ -184,14 +185,14 @@ public void writeSynchronizationMode(CacheWriteSynchronizationMode writeSyncMode
     /**
      * @return Backups number for new cache.
      */
-    public int backups() {
+    @Nullable public Integer backups() {
         return backups;
     }
 
     /**
      * @param backups Backups number for new cache.
      */
-    public void backups(int backups) {
+    public void backups(Integer backups) {
         this.backups = backups;
     }
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
index b20bb59611afe..82247114fc062 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
@@ -88,7 +88,14 @@ public class H2DynamicTableSelfTest extends AbstractSchemaSelfTest {
     /** Bad data region name. */
     public static final String DATA_REGION_NAME_BAD = "my_data_region_bad";
 
+    /** Cache with backups. */
+    private static final String CACHE_NAME_BACKUPS = CACHE_NAME + "_backups";
+
+    /** Number of backups for backup test. */
+    private static final int DFLT_BACKUPS = 2;
+
     /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
@@ -98,6 +105,8 @@ public class H2DynamicTableSelfTest extends AbstractSchemaSelfTest {
         client().addCacheConfiguration(cacheConfiguration());
         client().addCacheConfiguration(cacheConfiguration().setName(CACHE_NAME + "_async")
             .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_ASYNC));
+
+        client().addCacheConfiguration(cacheConfiguration().setName(CACHE_NAME_BACKUPS).setBackups(DFLT_BACKUPS));
     }
 
     /** {@inheritDoc} */
@@ -108,6 +117,7 @@ public class H2DynamicTableSelfTest extends AbstractSchemaSelfTest {
     }
 
     /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
 
@@ -120,6 +130,7 @@ public class H2DynamicTableSelfTest extends AbstractSchemaSelfTest {
         execute("DROP TABLE IF EXISTS PUBLIC.\"Person\"");
         execute("DROP TABLE IF EXISTS PUBLIC.\"City\"");
         execute("DROP TABLE IF EXISTS PUBLIC.\"NameTest\"");
+        execute("DROP TABLE IF EXISTS PUBLIC.\"BackupTest\"");
 
         super.afterTest();
     }
@@ -495,6 +506,32 @@ private void doTestCreateTable(String tplCacheName, String cacheGrp, CacheMode c
         doTestCreateTable(tplCacheName, cacheGrp, cacheMode, writeSyncMode, false, additionalParams);
     }
 
+    /**
+     * Test backups propagation.
+     *
+     * @throws Exception If failed.
+     */
+    @SuppressWarnings("unchecked")
+    public void testBackups() throws Exception {
+        String cacheName = "BackupTestCache";
+
+        execute("CREATE TABLE \"BackupTest\" (id BIGINT PRIMARY KEY, name VARCHAR) WITH \"template=" +
+            CACHE_NAME_BACKUPS + ", cache_name=" + cacheName + "\"");
+
+        CacheConfiguration ccfg = grid(0).cache(cacheName).getConfiguration(CacheConfiguration.class);
+
+        assertEquals(DFLT_BACKUPS, ccfg.getBackups());
+
+        execute("DROP TABLE PUBLIC.\"BackupTest\"");
+
+        execute("CREATE TABLE \"BackupTest\" (id BIGINT PRIMARY KEY, name VARCHAR) WITH \"template=" +
+            CACHE_NAME_BACKUPS + ", cache_name=" + cacheName + ", backups=1\"");
+
+        ccfg = grid(0).cache(cacheName).getConfiguration(CacheConfiguration.class);
+
+        assertEquals(1, ccfg.getBackups());
+    }
+
     /**
      * Test that {@code CREATE TABLE} with given template cache name actually creates new cache,
      * H2 table and type descriptor on all nodes, optionally with cache type check.

From 80f4340f61742988aaf9437eb08ed76644a1c8ca Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 12 Apr 2018 14:29:43 +0300
Subject: [PATCH 040/543] IGNITE-7871 Fixed condition for cache partitions
 validation. - Fixes #3804.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit 7a1d0ea)
---
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index dd4a57157a1e7..af5acd64257e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -2760,7 +2760,7 @@ private void validatePartitionsState() {
                     || grpCtx.config().isReadThrough()
                     || grpCtx.config().isWriteThrough()
                     || grpCtx.config().getCacheStoreFactory() != null
-                    || grpCtx.config().getRebalanceDelay() != -1
+                    || grpCtx.config().getRebalanceDelay() == -1
                     || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE)
                 continue;
 

From dfe17074593d9d12cbab7b60aa73e73c37bbffb7 Mon Sep 17 00:00:00 2001
From: Anton Kurbanov <antkr.dev@gmail.com>
Date: Thu, 12 Apr 2018 20:31:50 +0300
Subject: [PATCH 041/543] IGNITE-8110 GridCacheWriteBehindStore.Flusher thread
 uses the wrong transformation from milliseconds to nanoseconds. - Fixes
 #3742.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit adaedb4)
---
 .../store/GridCacheWriteBehindStore.java      |  2 +-
 .../GridCacheWriteBehindStoreSelfTest.java    | 41 ++++++++++++++++++-
 2 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
index 44cadd6b0e12a..82ff3aaabac22 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStore.java
@@ -895,7 +895,7 @@ private class Flusher extends GridWorker {
         protected Thread thread;
 
         /** Cache flushing frequence in nanos. */
-        protected long cacheFlushFreqNanos = cacheFlushFreq * 1000;
+        protected long cacheFlushFreqNanos = cacheFlushFreq * 1000 * 1000;
 
         /** Writer lock. */
         private final Lock flusherWriterLock = new ReentrantLock();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java
index 9a487a4268726..af21fc831c0c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/store/GridCacheWriteBehindStoreSelfTest.java
@@ -106,6 +106,43 @@ public void testSimpleStoreWithoutCoalescing() throws Exception {
         testSimpleStore(false);
     }
 
+    /**
+     * Checks that write behind cache flush frequency was correctly adjusted to nanos expecting putAllCnt to be
+     * less or equal than elapsed time divided by flush frequency.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSimpleStoreFlushFrequencyWithoutCoalescing() throws Exception {
+        initStore(1, false);
+
+        long writeBehindFlushFreqNanos = FLUSH_FREQUENCY * 1000 * 1000;
+
+        int threshold = store.getWriteBehindStoreBatchSize() / 10;
+
+        try {
+            long start = System.nanoTime();
+
+            for (int i = 0; i < threshold / 2; i++)
+                store.write(new CacheEntryImpl<>(i, "v" + i));
+
+            U.sleep(FLUSH_FREQUENCY + 300);
+
+            for (int i = threshold / 2; i < threshold; i++)
+                store.write(new CacheEntryImpl<>(i, "v" + i));
+
+            long elapsed = System.nanoTime() - start;
+
+            U.sleep(FLUSH_FREQUENCY + 300);
+
+            int expFlushOps = (int)(1 + elapsed / writeBehindFlushFreqNanos);
+
+            assertTrue(delegate.getPutAllCount() <= expFlushOps);
+        }
+        finally {
+            shutdownStore();
+        }
+    }
+
     /**
      * Simple store test.
      *
@@ -254,7 +291,6 @@ private void testContinuousPut(boolean writeCoalescing) throws Exception {
 
             int delegatePutCnt = delegate.getPutAllCount();
 
-
             fut.get();
 
             log().info(">>> [putCnt = " + actualPutCnt.get() + ", delegatePutCnt=" + delegatePutCnt + "]");
@@ -262,7 +298,8 @@ private void testContinuousPut(boolean writeCoalescing) throws Exception {
             assertTrue("No puts were made to the underlying store", delegatePutCnt > 0);
             if (store.getWriteCoalescing()) {
                 assertTrue("Too many puts were made to the underlying store", delegatePutCnt < actualPutCnt.get() / 10);
-            } else {
+            }
+            else {
                 assertTrue("Too few puts cnt=" + actualPutCnt.get() + " << storePutCnt=" + delegatePutCnt, delegatePutCnt > actualPutCnt.get() / 2);
             }
         }

From fe99497528cd040ab3b4d5f7bfc40e788393a5ed Mon Sep 17 00:00:00 2001
From: Andrey Kuznetsov <stkuzma@gmail.com>
Date: Thu, 12 Apr 2018 21:23:28 +0300
Subject: [PATCH 042/543] IGNITE-7983: NPE fixed in transactions

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../cache/distributed/near/GridNearTxLocal.java          | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 33f84f030db71..fc8a9a3fb2f6c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -3916,6 +3916,15 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                         throw new GridClosureException(e);
                     }
 
+                    if (isRollbackOnly()) {
+                        if (timedOut())
+                            throw new GridClosureException(new IgniteTxTimeoutCheckedException(
+                                "Transaction has been timed out: " + GridNearTxLocal.this));
+                        else
+                            throw new GridClosureException(new IgniteTxRollbackCheckedException(
+                                "Transaction has been rolled back: " + GridNearTxLocal.this));
+                    }
+
                     return map;
                 }
             },

From 6a77dd8b182091fe4e38850098c6334597c14a6d Mon Sep 17 00:00:00 2001
From: zaleslaw <zaleslaw.sin@gmail.com>
Date: Fri, 13 Apr 2018 12:49:56 +0300
Subject: [PATCH 043/543] IGNITE-7829: Adopt kNN regression example to the new
 Partitioned Dataset

this closes #3798

(cherry picked from commit 8550d61)
---
 .../ml/knn/KNNClassificationExample.java      |   4 +-
 .../examples/ml/knn/KNNRegressionExample.java | 310 ++++++++++++++++++
 .../org/apache/ignite/ml/knn/KNNUtils.java    |  10 +-
 .../KNNClassificationModel.java               |   9 +-
 .../knn/partitions/KNNPartitionContext.java   |  28 --
 .../ml/knn/partitions/package-info.java       |  22 --
 .../ml/knn/regression/KNNRegressionModel.java |   7 +-
 .../partition/LabelPartitionContext.java      |  28 --
 .../LabelPartitionDataBuilderOnHeap.java      |   1 -
 .../SVMLinearBinaryClassificationModel.java   |   3 +
 .../SVMLinearBinaryClassificationTrainer.java |   9 +-
 ...VMLinearMultiClassClassificationModel.java |   3 +
 ...LinearMultiClassClassificationTrainer.java |   8 +-
 .../ignite/ml/svm/SVMPartitionContext.java    |  28 --
 .../ignite/ml/knn/KNNClassificationTest.java  | 110 ++++---
 .../ignite/ml/knn/KNNRegressionTest.java      | 104 +++---
 ...KNNTest.java => LabeledDatasetHelper.java} |  10 +-
 .../ignite/ml/knn/LabeledDatasetTest.java     |   2 +-
 18 files changed, 453 insertions(+), 243 deletions(-)
 create mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/knn/partitions/KNNPartitionContext.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/knn/partitions/package-info.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/structures/partition/LabelPartitionContext.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java
 rename modules/ml/src/test/java/org/apache/ignite/ml/knn/{BaseKNNTest.java => LabeledDatasetHelper.java} (93%)

diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
index 39a8431a18761..15375a13ff306 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
@@ -80,7 +80,7 @@ public static void main(String[] args) throws InterruptedException {
                         double prediction = knnMdl.apply(new DenseLocalOnHeapVector(inputs));
 
                         totalAmount++;
-                        if(groundTruth != prediction)
+                        if (groundTruth != prediction)
                             amountOfErrors++;
 
                         System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
@@ -89,7 +89,7 @@ public static void main(String[] args) throws InterruptedException {
                     System.out.println(">>> ---------------------------------");
 
                     System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
-                    System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+                    System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double) totalAmount));
                 }
             });
 
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java
new file mode 100644
index 0000000000000..76a07cd9ce244
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.knn;
+
+import java.util.Arrays;
+import java.util.UUID;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.knn.classification.KNNClassificationTrainer;
+import org.apache.ignite.ml.knn.classification.KNNStrategy;
+import org.apache.ignite.ml.knn.regression.KNNRegressionModel;
+import org.apache.ignite.ml.knn.regression.KNNRegressionTrainer;
+import org.apache.ignite.ml.math.distances.ManhattanDistance;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Run kNN regression trainer over distributed dataset.
+ *
+ * @see KNNClassificationTrainer
+ */
+public class KNNRegressionExample {
+    /** Run example. */
+    public static void main(String[] args) throws InterruptedException {
+        System.out.println();
+        System.out.println(">>> kNN regression over cached dataset usage example started.");
+        // Start ignite grid.
+        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+            System.out.println(">>> Ignite grid started.");
+
+            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+                KNNRegressionExample.class.getSimpleName(), () -> {
+                IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
+
+                KNNRegressionTrainer trainer = new KNNRegressionTrainer();
+
+                KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
+                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
+                    (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+                    (k, v) -> v[0]
+                ).withK(5)
+                    .withDistanceMeasure(new ManhattanDistance())
+                    .withStrategy(KNNStrategy.WEIGHTED);
+
+                int totalAmount = 0;
+                // Calculate mean squared error (MSE)
+                double mse = 0.0;
+                // Calculate mean absolute error (MAE)
+                double mae = 0.0;
+
+                try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+                    for (Cache.Entry<Integer, double[]> observation : observations) {
+                        double[] val = observation.getValue();
+                        double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+                        double groundTruth = val[0];
+
+                        double prediction = knnMdl.apply(new DenseLocalOnHeapVector(inputs));
+
+                        mse += Math.pow(prediction - groundTruth, 2.0);
+                        mae += Math.abs(prediction - groundTruth);
+
+                        totalAmount++;
+                    }
+
+                    mse = mse / totalAmount;
+                    System.out.println("\n>>> Mean squared error (MSE) " + mse);
+
+                    mae = mae / totalAmount;
+                    System.out.println("\n>>> Mean absolute error (MAE) " + mae);
+                }
+            });
+
+            igniteThread.start();
+            igniteThread.join();
+        }
+    }
+
+    /**
+     * Fills cache with data and returns it.
+     *
+     * @param ignite Ignite instance.
+     * @return Filled Ignite Cache.
+     */
+    private static IgniteCache<Integer, double[]> getTestCache(Ignite ignite) {
+        CacheConfiguration<Integer, double[]> cacheConfiguration = new CacheConfiguration<>();
+        cacheConfiguration.setName("TEST_" + UUID.randomUUID());
+        cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+        IgniteCache<Integer, double[]> cache = ignite.createCache(cacheConfiguration);
+
+        for (int i = 0; i < data.length; i++)
+            cache.put(i, data[i]);
+
+        return cache;
+    }
+
+    /** The Iris dataset. */
+    private static final double[][] data = {
+        {199, 125, 256, 6000, 256, 16, 128},
+        {253, 29, 8000, 32000, 32, 8, 32},
+        {132, 29, 8000, 16000, 32, 8, 16},
+        {290, 26, 8000, 32000, 64, 8, 32},
+        {381, 23, 16000, 32000, 64, 16, 32},
+        {749, 23, 16000, 64000, 64, 16, 32},
+        {1238, 23, 32000, 64000, 128, 32, 64},
+        {23, 400, 1000, 3000, 0, 1, 2},
+        {24, 400, 512, 3500, 4, 1, 6},
+        {70, 60, 2000, 8000, 65, 1, 8},
+        {117, 50, 4000, 16000, 65, 1, 8},
+        {15, 350, 64, 64, 0, 1, 4},
+        {64, 200, 512, 16000, 0, 4, 32},
+        {23, 167, 524, 2000, 8, 4, 15},
+        {29, 143, 512, 5000, 0, 7, 32},
+        {22, 143, 1000, 2000, 0, 5, 16},
+        {124, 110, 5000, 5000, 142, 8, 64},
+        {35, 143, 1500, 6300, 0, 5, 32},
+        {39, 143, 3100, 6200, 0, 5, 20},
+        {40, 143, 2300, 6200, 0, 6, 64},
+        {45, 110, 3100, 6200, 0, 6, 64},
+        {28, 320, 128, 6000, 0, 1, 12},
+        {21, 320, 512, 2000, 4, 1, 3},
+        {28, 320, 256, 6000, 0, 1, 6},
+        {22, 320, 256, 3000, 4, 1, 3},
+        {28, 320, 512, 5000, 4, 1, 5},
+        {27, 320, 256, 5000, 4, 1, 6},
+        {102, 25, 1310, 2620, 131, 12, 24},
+        {74, 50, 2620, 10480, 30, 12, 24},
+        {138, 56, 5240, 20970, 30, 12, 24},
+        {136, 64, 5240, 20970, 30, 12, 24},
+        {23, 50, 500, 2000, 8, 1, 4},
+        {29, 50, 1000, 4000, 8, 1, 5},
+        {44, 50, 2000, 8000, 8, 1, 5},
+        {30, 50, 1000, 4000, 8, 3, 5},
+        {41, 50, 1000, 8000, 8, 3, 5},
+        {74, 50, 2000, 16000, 8, 3, 5},
+        {54, 133, 1000, 12000, 9, 3, 12},
+        {41, 133, 1000, 8000, 9, 3, 12},
+        {18, 810, 512, 512, 8, 1, 1},
+        {28, 810, 1000, 5000, 0, 1, 1},
+        {36, 320, 512, 8000, 4, 1, 5},
+        {38, 200, 512, 8000, 8, 1, 8},
+        {34, 700, 384, 8000, 0, 1, 1},
+        {19, 700, 256, 2000, 0, 1, 1},
+        {72, 140, 1000, 16000, 16, 1, 3},
+        {36, 200, 1000, 8000, 0, 1, 2},
+        {30, 110, 1000, 4000, 16, 1, 2},
+        {56, 110, 1000, 12000, 16, 1, 2},
+        {42, 220, 1000, 8000, 16, 1, 2},
+        {34, 800, 256, 8000, 0, 1, 4},
+        {19, 125, 512, 1000, 0, 8, 20},
+        {75, 75, 2000, 8000, 64, 1, 38},
+        {113, 75, 2000, 16000, 64, 1, 38},
+        {157, 75, 2000, 16000, 128, 1, 38},
+        {18, 90, 256, 1000, 0, 3, 10},
+        {20, 105, 256, 2000, 0, 3, 10},
+        {28, 105, 1000, 4000, 0, 3, 24},
+        {33, 105, 2000, 4000, 8, 3, 19},
+        {47, 75, 2000, 8000, 8, 3, 24},
+        {54, 75, 3000, 8000, 8, 3, 48},
+        {20, 175, 256, 2000, 0, 3, 24},
+        {23, 300, 768, 3000, 0, 6, 24},
+        {25, 300, 768, 3000, 6, 6, 24},
+        {52, 300, 768, 12000, 6, 6, 24},
+        {27, 300, 768, 4500, 0, 1, 24},
+        {50, 300, 384, 12000, 6, 1, 24},
+        {18, 300, 192, 768, 6, 6, 24},
+        {53, 180, 768, 12000, 6, 1, 31},
+        {23, 330, 1000, 3000, 0, 2, 4},
+        {30, 300, 1000, 4000, 8, 3, 64},
+        {73, 300, 1000, 16000, 8, 2, 112},
+        {20, 330, 1000, 2000, 0, 1, 2},
+        {25, 330, 1000, 4000, 0, 3, 6},
+        {28, 140, 2000, 4000, 0, 3, 6},
+        {29, 140, 2000, 4000, 0, 4, 8},
+        {32, 140, 2000, 4000, 8, 1, 20},
+        {175, 140, 2000, 32000, 32, 1, 20},
+        {57, 140, 2000, 8000, 32, 1, 54},
+        {181, 140, 2000, 32000, 32, 1, 54},
+        {32, 140, 2000, 4000, 8, 1, 20},
+        {82, 57, 4000, 16000, 1, 6, 12},
+        {171, 57, 4000, 24000, 64, 12, 16},
+        {361, 26, 16000, 32000, 64, 16, 24},
+        {350, 26, 16000, 32000, 64, 8, 24},
+        {220, 26, 8000, 32000, 0, 8, 24},
+        {113, 26, 8000, 16000, 0, 8, 16},
+        {15, 480, 96, 512, 0, 1, 1},
+        {21, 203, 1000, 2000, 0, 1, 5},
+        {35, 115, 512, 6000, 16, 1, 6},
+        {18, 1100, 512, 1500, 0, 1, 1},
+        {20, 1100, 768, 2000, 0, 1, 1},
+        {20, 600, 768, 2000, 0, 1, 1},
+        {28, 400, 2000, 4000, 0, 1, 1},
+        {45, 400, 4000, 8000, 0, 1, 1},
+        {18, 900, 1000, 1000, 0, 1, 2},
+        {17, 900, 512, 1000, 0, 1, 2},
+        {26, 900, 1000, 4000, 4, 1, 2},
+        {28, 900, 1000, 4000, 8, 1, 2},
+        {28, 900, 2000, 4000, 0, 3, 6},
+        {31, 225, 2000, 4000, 8, 3, 6},
+        {42, 180, 2000, 8000, 8, 1, 6},
+        {76, 185, 2000, 16000, 16, 1, 6},
+        {76, 180, 2000, 16000, 16, 1, 6},
+        {26, 225, 1000, 4000, 2, 3, 6},
+        {59, 25, 2000, 12000, 8, 1, 4},
+        {65, 25, 2000, 12000, 16, 3, 5},
+        {101, 17, 4000, 16000, 8, 6, 12},
+        {116, 17, 4000, 16000, 32, 6, 12},
+        {18, 1500, 768, 1000, 0, 0, 0},
+        {20, 1500, 768, 2000, 0, 0, 0},
+        {20, 800, 768, 2000, 0, 0, 0},
+        {30, 50, 2000, 4000, 0, 3, 6},
+        {44, 50, 2000, 8000, 8, 3, 6},
+        {82, 50, 2000, 16000, 24, 1, 6},
+        {128, 50, 8000, 16000, 48, 1, 10},
+        {37, 100, 1000, 8000, 0, 2, 6},
+        {46, 100, 1000, 8000, 24, 2, 6},
+        {46, 100, 1000, 8000, 24, 3, 6},
+        {80, 50, 2000, 16000, 12, 3, 16},
+        {88, 50, 2000, 16000, 24, 6, 16},
+        {33, 150, 512, 4000, 0, 8, 128},
+        {46, 115, 2000, 8000, 16, 1, 3},
+        {29, 115, 2000, 4000, 2, 1, 5},
+        {53, 92, 2000, 8000, 32, 1, 6},
+        {41, 92, 2000, 8000, 4, 1, 6},
+        {86, 75, 4000, 16000, 16, 1, 6},
+        {95, 60, 4000, 16000, 32, 1, 6},
+        {107, 60, 2000, 16000, 64, 5, 8},
+        {117, 60, 4000, 16000, 64, 5, 8},
+        {119, 50, 4000, 16000, 64, 5, 10},
+        {120, 72, 4000, 16000, 64, 8, 16},
+        {48, 72, 2000, 8000, 16, 6, 8},
+        {126, 40, 8000, 16000, 32, 8, 16},
+        {266, 40, 8000, 32000, 64, 8, 24},
+        {270, 35, 8000, 32000, 64, 8, 24},
+        {426, 38, 16000, 32000, 128, 16, 32},
+        {151, 48, 4000, 24000, 32, 8, 24},
+        {267, 38, 8000, 32000, 64, 8, 24},
+        {603, 30, 16000, 32000, 256, 16, 24},
+        {19, 112, 1000, 1000, 0, 1, 4},
+        {21, 84, 1000, 2000, 0, 1, 6},
+        {26, 56, 1000, 4000, 0, 1, 6},
+        {35, 56, 2000, 6000, 0, 1, 8},
+        {41, 56, 2000, 8000, 0, 1, 8},
+        {47, 56, 4000, 8000, 0, 1, 8},
+        {62, 56, 4000, 12000, 0, 1, 8},
+        {78, 56, 4000, 16000, 0, 1, 8},
+        {80, 38, 4000, 8000, 32, 16, 32},
+        {142, 38, 8000, 16000, 64, 4, 8},
+        {281, 38, 8000, 24000, 160, 4, 8},
+        {190, 38, 4000, 16000, 128, 16, 32},
+        {21, 200, 1000, 2000, 0, 1, 2},
+        {25, 200, 1000, 4000, 0, 1, 4},
+        {67, 200, 2000, 8000, 64, 1, 5},
+        {24, 250, 512, 4000, 0, 1, 7},
+        {24, 250, 512, 4000, 0, 4, 7},
+        {64, 250, 1000, 16000, 1, 1, 8},
+        {25, 160, 512, 4000, 2, 1, 5},
+        {20, 160, 512, 2000, 2, 3, 8},
+        {29, 160, 1000, 4000, 8, 1, 14},
+        {43, 160, 1000, 8000, 16, 1, 14},
+        {53, 160, 2000, 8000, 32, 1, 13},
+        {19, 240, 512, 1000, 8, 1, 3},
+        {22, 240, 512, 2000, 8, 1, 5},
+        {31, 105, 2000, 4000, 8, 3, 8},
+        {41, 105, 2000, 6000, 16, 6, 16},
+        {47, 105, 2000, 8000, 16, 4, 14},
+        {99, 52, 4000, 16000, 32, 4, 12},
+        {67, 70, 4000, 12000, 8, 6, 8},
+        {81, 59, 4000, 12000, 32, 6, 12},
+        {149, 59, 8000, 16000, 64, 12, 24},
+        {183, 26, 8000, 24000, 32, 8, 16},
+        {275, 26, 8000, 32000, 64, 12, 16},
+        {382, 26, 8000, 32000, 128, 24, 32},
+        {56, 116, 2000, 8000, 32, 5, 28},
+        {182, 50, 2000, 32000, 24, 6, 26},
+        {227, 50, 2000, 32000, 48, 26, 52},
+        {341, 50, 2000, 32000, 112, 52, 104},
+        {360, 50, 4000, 32000, 112, 52, 104},
+        {919, 30, 8000, 64000, 96, 12, 176},
+        {978, 30, 8000, 64000, 128, 12, 176},
+        {24, 180, 262, 4000, 0, 1, 3},
+        {37, 124, 1000, 8000, 0, 1, 8},
+        {50, 98, 1000, 8000, 32, 2, 8},
+        {41, 125, 2000, 8000, 0, 2, 14},
+        {47, 480, 512, 8000, 32, 0, 0},
+        {25, 480, 1000, 4000, 0, 0, 0}
+    };
+}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java
index 88fa70f8c9ea2..716eb526c8fab 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/KNNUtils.java
@@ -20,7 +20,7 @@
 import org.apache.ignite.ml.dataset.Dataset;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
 import org.apache.ignite.ml.dataset.PartitionDataBuilder;
-import org.apache.ignite.ml.knn.partitions.KNNPartitionContext;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
 import org.apache.ignite.ml.math.functions.IgniteBiFunction;
 import org.apache.ignite.ml.structures.LabeledDataset;
 import org.apache.ignite.ml.structures.LabeledVector;
@@ -39,18 +39,18 @@ public class KNNUtils {
      * @param lbExtractor Label extractor.
      * @return Dataset.
      */
-    @Nullable public static <K, V> Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> buildDataset(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
-        PartitionDataBuilder<K, V, KNNPartitionContext, LabeledDataset<Double, LabeledVector>> partDataBuilder
+    @Nullable public static <K, V> Dataset<EmptyContext, LabeledDataset<Double, LabeledVector>> buildDataset(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+        PartitionDataBuilder<K, V, EmptyContext, LabeledDataset<Double, LabeledVector>> partDataBuilder
             = new LabeledDatasetPartitionDataBuilderOnHeap<>(
             featureExtractor,
             lbExtractor
         );
 
-        Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> dataset = null;
+        Dataset<EmptyContext, LabeledDataset<Double, LabeledVector>> dataset = null;
 
         if (datasetBuilder != null) {
             dataset = datasetBuilder.build(
-                (upstream, upstreamSize) -> new KNNPartitionContext(),
+                (upstream, upstreamSize) -> new EmptyContext(),
                 partDataBuilder
             );
         }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java
index 373f822d7276a..693b81df0ecff 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java
@@ -32,7 +32,7 @@
 import org.apache.ignite.ml.Exporter;
 import org.apache.ignite.ml.Model;
 import org.apache.ignite.ml.dataset.Dataset;
-import org.apache.ignite.ml.knn.partitions.KNNPartitionContext;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
 import org.apache.ignite.ml.math.Vector;
 import org.apache.ignite.ml.math.distances.DistanceMeasure;
 import org.apache.ignite.ml.math.distances.EuclideanDistance;
@@ -44,6 +44,9 @@
  * kNN algorithm model to solve multi-class classification task.
  */
 public class KNNClassificationModel<K, V> implements Model<Vector, Double>, Exportable<KNNModelFormat> {
+    /** */
+    private static final long serialVersionUID = -127386523291350345L;
+
     /** Amount of nearest neighbors. */
     protected int k = 5;
 
@@ -54,13 +57,13 @@ public class KNNClassificationModel<K, V> implements Model<Vector, Double>, Expo
     protected KNNStrategy stgy = KNNStrategy.SIMPLE;
 
     /** Dataset. */
-    private Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> dataset;
+    private Dataset<EmptyContext, LabeledDataset<Double, LabeledVector>> dataset;
 
     /**
      * Builds the model via prepared dataset.
      * @param dataset Specially prepared object to run algorithm over it.
      */
-    public KNNClassificationModel(Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> dataset) {
+    public KNNClassificationModel(Dataset<EmptyContext, LabeledDataset<Double, LabeledVector>> dataset) {
         this.dataset = dataset;
     }
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/partitions/KNNPartitionContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/partitions/KNNPartitionContext.java
deleted file mode 100644
index 0081612754e37..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/partitions/KNNPartitionContext.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.knn.partitions;
-
-import java.io.Serializable;
-
-/**
- * Partition context of the kNN classification algorithm.
- */
-public class KNNPartitionContext implements Serializable {
-    /** */
-    private static final long serialVersionUID = -7212307112344430126L;
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/partitions/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/partitions/package-info.java
deleted file mode 100644
index 951a8490444df..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/partitions/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains helper classes for kNN classification algorithms.
- */
-package org.apache.ignite.ml.knn.partitions;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java
index cabc1438e1d77..f5def43634bfe 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/regression/KNNRegressionModel.java
@@ -17,8 +17,8 @@
 package org.apache.ignite.ml.knn.regression;
 
 import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
 import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
-import org.apache.ignite.ml.knn.partitions.KNNPartitionContext;
 import org.apache.ignite.ml.math.Vector;
 import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
 import org.apache.ignite.ml.structures.LabeledDataset;
@@ -38,11 +38,14 @@
  * </ul>
  */
 public class KNNRegressionModel<K,V> extends KNNClassificationModel<K,V> {
+    /** */
+    private static final long serialVersionUID = -721836321291120543L;
+
     /**
      * Builds the model via prepared dataset.
      * @param dataset Specially prepared object to run algorithm over it.
      */
-    public KNNRegressionModel(Dataset<KNNPartitionContext, LabeledDataset<Double, LabeledVector>> dataset) {
+    public KNNRegressionModel(Dataset<EmptyContext, LabeledDataset<Double, LabeledVector>> dataset) {
         super(dataset);
     }
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/structures/partition/LabelPartitionContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/structures/partition/LabelPartitionContext.java
deleted file mode 100644
index 1069ff8ab00c5..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/structures/partition/LabelPartitionContext.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.structures.partition;
-
-import java.io.Serializable;
-
-/**
- * Base partition context.
- */
-public class LabelPartitionContext implements Serializable {
-    /** */
-    private static final long serialVersionUID = -7412302212344430126L;
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/structures/partition/LabelPartitionDataBuilderOnHeap.java b/modules/ml/src/main/java/org/apache/ignite/ml/structures/partition/LabelPartitionDataBuilderOnHeap.java
index 14c053e24acb0..4fba0289d447f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/structures/partition/LabelPartitionDataBuilderOnHeap.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/structures/partition/LabelPartitionDataBuilderOnHeap.java
@@ -22,7 +22,6 @@
 import org.apache.ignite.ml.dataset.PartitionDataBuilder;
 import org.apache.ignite.ml.dataset.UpstreamEntry;
 import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.structures.LabeledDataset;
 
 /**
  * Partition data builder that builds {@link LabelPartitionDataOnHeap}.
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
index dace8c6b3bcad..f806fb8da28a1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationModel.java
@@ -28,6 +28,9 @@
  * Base class for SVM linear classification model.
  */
 public class SVMLinearBinaryClassificationModel implements Model<Vector, Double>, Exportable<SVMLinearBinaryClassificationModel>, Serializable {
+    /** */
+    private static final long serialVersionUID = -996984622291440226L;
+
     /** Output label format. -1 and +1 for false value and raw distances from the separating hyperplane otherwise. */
     private boolean isKeepingRawLabels = false;
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
index 7f11e209b1fd6..d56848c9d6d45 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.ml.svm;
 
 import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
 import org.apache.ignite.ml.structures.partition.LabeledDatasetPartitionDataBuilderOnHeap;
 import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
 import org.apache.ignite.ml.dataset.Dataset;
@@ -59,15 +60,15 @@ public class SVMLinearBinaryClassificationTrainer implements SingleLabelDatasetT
 
         assert datasetBuilder != null;
 
-        PartitionDataBuilder<K, V, SVMPartitionContext, LabeledDataset<Double, LabeledVector>> partDataBuilder = new LabeledDatasetPartitionDataBuilderOnHeap<>(
+        PartitionDataBuilder<K, V, EmptyContext, LabeledDataset<Double, LabeledVector>> partDataBuilder = new LabeledDatasetPartitionDataBuilderOnHeap<>(
             featureExtractor,
             lbExtractor
         );
 
         Vector weights;
 
-        try(Dataset<SVMPartitionContext, LabeledDataset<Double, LabeledVector>> dataset = datasetBuilder.build(
-            (upstream, upstreamSize) -> new SVMPartitionContext(),
+        try(Dataset<EmptyContext, LabeledDataset<Double, LabeledVector>> dataset = datasetBuilder.build(
+            (upstream, upstreamSize) -> new EmptyContext(),
             partDataBuilder
         )) {
             final int cols = dataset.compute(data -> data.colSize(), (a, b) -> a == null ? b : a);
@@ -90,7 +91,7 @@ public class SVMLinearBinaryClassificationTrainer implements SingleLabelDatasetT
     }
 
     /** */
-    private Vector calculateUpdates(Vector weights, Dataset<SVMPartitionContext, LabeledDataset<Double, LabeledVector>> dataset) {
+    private Vector calculateUpdates(Vector weights, Dataset<EmptyContext, LabeledDataset<Double, LabeledVector>> dataset) {
         return dataset.compute(data -> {
             Vector copiedWeights = weights.copy();
             Vector deltaWeights = initializeWeightsWithZeros(weights.size());
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
index 5879ef095214a..bbec791e56652 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationModel.java
@@ -29,6 +29,9 @@
 
 /** Base class for multi-classification model for set of SVM classifiers. */
 public class SVMLinearMultiClassClassificationModel implements Model<Vector, Double>, Exportable<SVMLinearMultiClassClassificationModel>, Serializable {
+    /** */
+    private static final long serialVersionUID = -667986511191350227L;
+
     /** List of models associated with each class. */
     private Map<Double, SVMLinearBinaryClassificationModel> models;
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
index 88c342d78a201..4e081c6123e6a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
@@ -24,12 +24,12 @@
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
 import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
 import org.apache.ignite.ml.dataset.Dataset;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
 import org.apache.ignite.ml.dataset.PartitionDataBuilder;
 import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.structures.partition.LabelPartitionContext;
 import org.apache.ignite.ml.structures.partition.LabelPartitionDataBuilderOnHeap;
 import org.apache.ignite.ml.structures.partition.LabelPartitionDataOnHeap;
 
@@ -89,12 +89,12 @@ public class SVMLinearMultiClassClassificationTrainer
     private <K, V> List<Double> extractClassLabels(DatasetBuilder<K, V> datasetBuilder, IgniteBiFunction<K, V, Double> lbExtractor) {
         assert datasetBuilder != null;
 
-        PartitionDataBuilder<K, V, LabelPartitionContext, LabelPartitionDataOnHeap> partDataBuilder = new LabelPartitionDataBuilderOnHeap<>(lbExtractor);
+        PartitionDataBuilder<K, V, EmptyContext, LabelPartitionDataOnHeap> partDataBuilder = new LabelPartitionDataBuilderOnHeap<>(lbExtractor);
 
         List<Double> res = new ArrayList<>();
 
-        try (Dataset<LabelPartitionContext, LabelPartitionDataOnHeap> dataset = datasetBuilder.build(
-            (upstream, upstreamSize) -> new LabelPartitionContext(),
+        try (Dataset<EmptyContext, LabelPartitionDataOnHeap> dataset = datasetBuilder.build(
+            (upstream, upstreamSize) -> new EmptyContext(),
             partDataBuilder
         )) {
             final Set<Double> clsLabels = dataset.compute(data -> {
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java
deleted file mode 100644
index 0aee0fbf140cb..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMPartitionContext.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.svm;
-
-import java.io.Serializable;
-
-/**
- * Partition context of the SVM classification algorithm.
- */
-public class SVMPartitionContext implements Serializable {
-    /** */
-    private static final long serialVersionUID = -7212307112344430126L;
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
index b27fcba20d246..0877fc07cd16c 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
@@ -17,31 +17,35 @@
 
 package org.apache.ignite.ml.knn;
 
-import org.apache.ignite.internal.util.IgniteUtils;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import org.junit.Assert;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
 import org.apache.ignite.ml.knn.classification.KNNClassificationTrainer;
 import org.apache.ignite.ml.knn.classification.KNNStrategy;
 import org.apache.ignite.ml.math.Vector;
 import org.apache.ignite.ml.math.distances.EuclideanDistance;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
+import org.junit.Test;
 
 /** Tests behaviour of KNNClassificationTest. */
-public class KNNClassificationTest extends BaseKNNTest {
+public class KNNClassificationTest {
+    /** Precision in test checks. */
+    private static final double PRECISION = 1e-2;
+
     /** */
-    public void testBinaryClassificationTest() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
+    @Test
+    public void binaryClassificationTest() {
 
         Map<Integer, double[]> data = new HashMap<>();
-        data.put(0, new double[] {1.0, 1.0, 1.0});
-        data.put(1, new double[] {1.0, 2.0, 1.0});
-        data.put(2, new double[] {2.0, 1.0, 1.0});
-        data.put(3, new double[] {-1.0, -1.0, 2.0});
-        data.put(4, new double[] {-1.0, -2.0, 2.0});
-        data.put(5, new double[] {-2.0, -1.0, 2.0});
+        data.put(0, new double[]{1.0, 1.0, 1.0});
+        data.put(1, new double[]{1.0, 2.0, 1.0});
+        data.put(2, new double[]{2.0, 1.0, 1.0});
+        data.put(3, new double[]{-1.0, -1.0, 2.0});
+        data.put(4, new double[]{-1.0, -2.0, 2.0});
+        data.put(5, new double[]{-2.0, -1.0, 2.0});
 
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
@@ -54,23 +58,23 @@ public void testBinaryClassificationTest() {
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector firstVector = new DenseLocalOnHeapVector(new double[] {2.0, 2.0});
-        assertEquals(knnMdl.apply(firstVector), 1.0);
-        Vector secondVector = new DenseLocalOnHeapVector(new double[] {-2.0, -2.0});
-        assertEquals(knnMdl.apply(secondVector), 2.0);
+        Vector firstVector = new DenseLocalOnHeapVector(new double[]{2.0, 2.0});
+        Assert.assertEquals(knnMdl.apply(firstVector), 1.0, PRECISION);
+        Vector secondVector = new DenseLocalOnHeapVector(new double[]{-2.0, -2.0});
+        Assert.assertEquals(knnMdl.apply(secondVector), 2.0, PRECISION);
     }
 
     /** */
-    public void testBinaryClassificationWithSmallestKTest() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
+    @Test
+    public void binaryClassificationWithSmallestKTest() {
         Map<Integer, double[]> data = new HashMap<>();
-        data.put(0, new double[] {1.0, 1.0, 1.0});
-        data.put(1, new double[] {1.0, 2.0, 1.0});
-        data.put(2, new double[] {2.0, 1.0, 1.0});
-        data.put(3, new double[] {-1.0, -1.0, 2.0});
-        data.put(4, new double[] {-1.0, -2.0, 2.0});
-        data.put(5, new double[] {-2.0, -1.0, 2.0});
+
+        data.put(0, new double[]{1.0, 1.0, 1.0});
+        data.put(1, new double[]{1.0, 2.0, 1.0});
+        data.put(2, new double[]{2.0, 1.0, 1.0});
+        data.put(3, new double[]{-1.0, -1.0, 2.0});
+        data.put(4, new double[]{-1.0, -2.0, 2.0});
+        data.put(5, new double[]{-2.0, -1.0, 2.0});
 
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
@@ -83,23 +87,23 @@ public void testBinaryClassificationWithSmallestKTest() {
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector firstVector = new DenseLocalOnHeapVector(new double[] {2.0, 2.0});
-        assertEquals(knnMdl.apply(firstVector), 1.0);
-        Vector secondVector = new DenseLocalOnHeapVector(new double[] {-2.0, -2.0});
-        assertEquals(knnMdl.apply(secondVector), 2.0);
+        Vector firstVector = new DenseLocalOnHeapVector(new double[]{2.0, 2.0});
+        Assert.assertEquals(knnMdl.apply(firstVector), 1.0, PRECISION);
+        Vector secondVector = new DenseLocalOnHeapVector(new double[]{-2.0, -2.0});
+        Assert.assertEquals(knnMdl.apply(secondVector), 2.0, PRECISION);
     }
 
     /** */
-    public void testBinaryClassificationFarPointsWithSimpleStrategy() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
+    @Test
+    public void binaryClassificationFarPointsWithSimpleStrategy() {
         Map<Integer, double[]> data = new HashMap<>();
-        data.put(0, new double[] {10.0, 10.0, 1.0});
-        data.put(1, new double[] {10.0, 20.0, 1.0});
-        data.put(2, new double[] {-1, -1, 1.0});
-        data.put(3, new double[] {-2, -2, 2.0});
-        data.put(4, new double[] {-1.0, -2.0, 2.0});
-        data.put(5, new double[] {-2.0, -1.0, 2.0});
+
+        data.put(0, new double[]{10.0, 10.0, 1.0});
+        data.put(1, new double[]{10.0, 20.0, 1.0});
+        data.put(2, new double[]{-1, -1, 1.0});
+        data.put(3, new double[]{-2, -2, 2.0});
+        data.put(4, new double[]{-1.0, -2.0, 2.0});
+        data.put(5, new double[]{-2.0, -1.0, 2.0});
 
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
@@ -112,21 +116,21 @@ public void testBinaryClassificationFarPointsWithSimpleStrategy() {
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector vector = new DenseLocalOnHeapVector(new double[] {-1.01, -1.01});
-        assertEquals(knnMdl.apply(vector), 2.0);
+        Vector vector = new DenseLocalOnHeapVector(new double[]{-1.01, -1.01});
+        Assert.assertEquals(knnMdl.apply(vector), 2.0, PRECISION);
     }
 
     /** */
-    public void testBinaryClassificationFarPointsWithWeightedStrategy() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
+    @Test
+    public void binaryClassificationFarPointsWithWeightedStrategy() {
         Map<Integer, double[]> data = new HashMap<>();
-        data.put(0, new double[] {10.0, 10.0, 1.0});
-        data.put(1, new double[] {10.0, 20.0, 1.0});
-        data.put(2, new double[] {-1, -1, 1.0});
-        data.put(3, new double[] {-2, -2, 2.0});
-        data.put(4, new double[] {-1.0, -2.0, 2.0});
-        data.put(5, new double[] {-2.0, -1.0, 2.0});
+
+        data.put(0, new double[]{10.0, 10.0, 1.0});
+        data.put(1, new double[]{10.0, 20.0, 1.0});
+        data.put(2, new double[]{-1, -1, 1.0});
+        data.put(3, new double[]{-2, -2, 2.0});
+        data.put(4, new double[]{-1.0, -2.0, 2.0});
+        data.put(5, new double[]{-2.0, -1.0, 2.0});
 
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
@@ -139,7 +143,7 @@ public void testBinaryClassificationFarPointsWithWeightedStrategy() {
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.WEIGHTED);
 
-        Vector vector = new DenseLocalOnHeapVector(new double[] {-1.01, -1.01});
-        assertEquals(knnMdl.apply(vector), 1.0);
+        Vector vector = new DenseLocalOnHeapVector(new double[]{-1.01, -1.01});
+        Assert.assertEquals(knnMdl.apply(vector), 1.0, PRECISION);
     }
 }
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
index 66dbca9ff038e..ce9cae59cda80 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.ml.knn;
 
-import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.knn.classification.KNNStrategy;
 import org.apache.ignite.ml.knn.regression.KNNRegressionModel;
@@ -30,28 +29,23 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
+import org.junit.Test;
 
 /**
  * Tests for {@link KNNRegressionTrainer}.
  */
-public class KNNRegressionTest extends BaseKNNTest {
+public class KNNRegressionTest {
     /** */
-    private double[] y;
-
-    /** */
-    private double[][] x;
-
-    /** */
-    public void testSimpleRegressionWithOneNeighbour() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
+    @Test
+    public void simpleRegressionWithOneNeighbour() {
         Map<Integer, double[]> data = new HashMap<>();
-        data.put(0, new double[] {11.0, 0, 0, 0, 0, 0});
-        data.put(1, new double[] {12.0, 2.0, 0, 0, 0, 0});
-        data.put(2, new double[] {13.0, 0, 3.0, 0, 0, 0});
-        data.put(3, new double[] {14.0, 0, 0, 4.0, 0, 0});
-        data.put(4, new double[] {15.0, 0, 0, 0, 5.0, 0});
-        data.put(5, new double[] {16.0, 0, 0, 0, 0, 6.0});
+
+        data.put(0, new double[]{11.0, 0, 0, 0, 0, 0});
+        data.put(1, new double[]{12.0, 2.0, 0, 0, 0, 0});
+        data.put(2, new double[]{13.0, 0, 3.0, 0, 0, 0});
+        data.put(3, new double[]{14.0, 0, 0, 4.0, 0, 0});
+        data.put(4, new double[]{15.0, 0, 0, 0, 5.0, 0});
+        data.put(5, new double[]{16.0, 0, 0, 0, 0, 6.0});
 
         KNNRegressionTrainer trainer = new KNNRegressionTrainer();
 
@@ -63,32 +57,31 @@ public void testSimpleRegressionWithOneNeighbour() {
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector vector = new DenseLocalOnHeapVector(new double[] {0, 0, 0, 5.0, 0.0});
+        Vector vector = new DenseLocalOnHeapVector(new double[]{0, 0, 0, 5.0, 0.0});
         System.out.println(knnMdl.apply(vector));
         Assert.assertEquals(15, knnMdl.apply(vector), 1E-12);
     }
 
     /** */
-    public void testLongly() {
-
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
+    @Test
+    public void longly() {
         Map<Integer, double[]> data = new HashMap<>();
-        data.put(0, new double[] {60323, 83.0, 234289, 2356, 1590, 107608, 1947});
-        data.put(1, new double[] {61122, 88.5, 259426, 2325, 1456, 108632, 1948});
-        data.put(2, new double[] {60171, 88.2, 258054, 3682, 1616, 109773, 1949});
-        data.put(3, new double[] {61187, 89.5, 284599, 3351, 1650, 110929, 1950});
-        data.put(4, new double[] {63221, 96.2, 328975, 2099, 3099, 112075, 1951});
-        data.put(5, new double[] {63639, 98.1, 346999, 1932, 3594, 113270, 1952});
-        data.put(6, new double[] {64989, 99.0, 365385, 1870, 3547, 115094, 1953});
-        data.put(7, new double[] {63761, 100.0, 363112, 3578, 3350, 116219, 1954});
-        data.put(8, new double[] {66019, 101.2, 397469, 2904, 3048, 117388, 1955});
-        data.put(9, new double[] {68169, 108.4, 442769, 2936, 2798, 120445, 1957});
-        data.put(10, new double[] {66513, 110.8, 444546, 4681, 2637, 121950, 1958});
-        data.put(11, new double[] {68655, 112.6, 482704, 3813, 2552, 123366, 1959});
-        data.put(12, new double[] {69564, 114.2, 502601, 3931, 2514, 125368, 1960});
-        data.put(13, new double[] {69331, 115.7, 518173, 4806, 2572, 127852, 1961});
-        data.put(14, new double[] {70551, 116.9, 554894, 4007, 2827, 130081, 1962});
+
+        data.put(0, new double[]{60323, 83.0, 234289, 2356, 1590, 107608, 1947});
+        data.put(1, new double[]{61122, 88.5, 259426, 2325, 1456, 108632, 1948});
+        data.put(2, new double[]{60171, 88.2, 258054, 3682, 1616, 109773, 1949});
+        data.put(3, new double[]{61187, 89.5, 284599, 3351, 1650, 110929, 1950});
+        data.put(4, new double[]{63221, 96.2, 328975, 2099, 3099, 112075, 1951});
+        data.put(5, new double[]{63639, 98.1, 346999, 1932, 3594, 113270, 1952});
+        data.put(6, new double[]{64989, 99.0, 365385, 1870, 3547, 115094, 1953});
+        data.put(7, new double[]{63761, 100.0, 363112, 3578, 3350, 116219, 1954});
+        data.put(8, new double[]{66019, 101.2, 397469, 2904, 3048, 117388, 1955});
+        data.put(9, new double[]{68169, 108.4, 442769, 2936, 2798, 120445, 1957});
+        data.put(10, new double[]{66513, 110.8, 444546, 4681, 2637, 121950, 1958});
+        data.put(11, new double[]{68655, 112.6, 482704, 3813, 2552, 123366, 1959});
+        data.put(12, new double[]{69564, 114.2, 502601, 3931, 2514, 125368, 1960});
+        data.put(13, new double[]{69331, 115.7, 518173, 4806, 2572, 127852, 1961});
+        data.put(14, new double[]{70551, 116.9, 554894, 4007, 2827, 130081, 1962});
 
         KNNRegressionTrainer trainer = new KNNRegressionTrainer();
 
@@ -100,31 +93,30 @@ public void testLongly() {
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector vector = new DenseLocalOnHeapVector(new double[] {104.6, 419180, 2822, 2857, 118734, 1956});
+        Vector vector = new DenseLocalOnHeapVector(new double[]{104.6, 419180, 2822, 2857, 118734, 1956});
         System.out.println(knnMdl.apply(vector));
         Assert.assertEquals(67857, knnMdl.apply(vector), 2000);
     }
 
     /** */
     public void testLonglyWithWeightedStrategy() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         Map<Integer, double[]> data = new HashMap<>();
-        data.put(0, new double[] {60323, 83.0, 234289, 2356, 1590, 107608, 1947});
-        data.put(1, new double[] {61122, 88.5, 259426, 2325, 1456, 108632, 1948});
-        data.put(2, new double[] {60171, 88.2, 258054, 3682, 1616, 109773, 1949});
-        data.put(3, new double[] {61187, 89.5, 284599, 3351, 1650, 110929, 1950});
-        data.put(4, new double[] {63221, 96.2, 328975, 2099, 3099, 112075, 1951});
-        data.put(5, new double[] {63639, 98.1, 346999, 1932, 3594, 113270, 1952});
-        data.put(6, new double[] {64989, 99.0, 365385, 1870, 3547, 115094, 1953});
-        data.put(7, new double[] {63761, 100.0, 363112, 3578, 3350, 116219, 1954});
-        data.put(8, new double[] {66019, 101.2, 397469, 2904, 3048, 117388, 1955});
-        data.put(9, new double[] {68169, 108.4, 442769, 2936, 2798, 120445, 1957});
-        data.put(10, new double[] {66513, 110.8, 444546, 4681, 2637, 121950, 1958});
-        data.put(11, new double[] {68655, 112.6, 482704, 3813, 2552, 123366, 1959});
-        data.put(12, new double[] {69564, 114.2, 502601, 3931, 2514, 125368, 1960});
-        data.put(13, new double[] {69331, 115.7, 518173, 4806, 2572, 127852, 1961});
-        data.put(14, new double[] {70551, 116.9, 554894, 4007, 2827, 130081, 1962});
+
+        data.put(0, new double[]{60323, 83.0, 234289, 2356, 1590, 107608, 1947});
+        data.put(1, new double[]{61122, 88.5, 259426, 2325, 1456, 108632, 1948});
+        data.put(2, new double[]{60171, 88.2, 258054, 3682, 1616, 109773, 1949});
+        data.put(3, new double[]{61187, 89.5, 284599, 3351, 1650, 110929, 1950});
+        data.put(4, new double[]{63221, 96.2, 328975, 2099, 3099, 112075, 1951});
+        data.put(5, new double[]{63639, 98.1, 346999, 1932, 3594, 113270, 1952});
+        data.put(6, new double[]{64989, 99.0, 365385, 1870, 3547, 115094, 1953});
+        data.put(7, new double[]{63761, 100.0, 363112, 3578, 3350, 116219, 1954});
+        data.put(8, new double[]{66019, 101.2, 397469, 2904, 3048, 117388, 1955});
+        data.put(9, new double[]{68169, 108.4, 442769, 2936, 2798, 120445, 1957});
+        data.put(10, new double[]{66513, 110.8, 444546, 4681, 2637, 121950, 1958});
+        data.put(11, new double[]{68655, 112.6, 482704, 3813, 2552, 123366, 1959});
+        data.put(12, new double[]{69564, 114.2, 502601, 3931, 2514, 125368, 1960});
+        data.put(13, new double[]{69331, 115.7, 518173, 4806, 2572, 127852, 1961});
+        data.put(14, new double[]{70551, 116.9, 554894, 4007, 2827, 130081, 1962});
 
         KNNRegressionTrainer trainer = new KNNRegressionTrainer();
 
@@ -136,7 +128,7 @@ public void testLonglyWithWeightedStrategy() {
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector vector = new DenseLocalOnHeapVector(new double[] {104.6, 419180, 2822, 2857, 118734, 1956});
+        Vector vector = new DenseLocalOnHeapVector(new double[]{104.6, 419180, 2822, 2857, 118734, 1956});
         System.out.println(knnMdl.apply(vector));
         Assert.assertEquals(67857, knnMdl.apply(vector), 2000);
     }
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/BaseKNNTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetHelper.java
similarity index 93%
rename from modules/ml/src/test/java/org/apache/ignite/ml/knn/BaseKNNTest.java
rename to modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetHelper.java
index aeac2cf2e8b8d..a25b303480780 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/BaseKNNTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetHelper.java
@@ -29,7 +29,7 @@
 /**
  * Base class for decision trees test.
  */
-public class BaseKNNTest extends GridCommonAbstractTest {
+public class LabeledDatasetHelper extends GridCommonAbstractTest {
     /** Count of nodes. */
     private static final int NODE_COUNT = 4;
 
@@ -42,7 +42,7 @@ public class BaseKNNTest extends GridCommonAbstractTest {
     /**
      * Default constructor.
      */
-    public BaseKNNTest() {
+    public LabeledDatasetHelper() {
         super(false);
     }
 
@@ -75,12 +75,10 @@ LabeledDataset loadDatasetFromTxt(String rsrcPath, boolean isFallOnBadData) {
             Path path = Paths.get(this.getClass().getClassLoader().getResource(rsrcPath).toURI());
             try {
                 return LabeledDatasetLoader.loadFromTxtFile(path, SEPARATOR, false, isFallOnBadData);
-            }
-            catch (IOException e) {
+            } catch (IOException e) {
                 e.printStackTrace();
             }
-        }
-        catch (URISyntaxException e) {
+        } catch (URISyntaxException e) {
             e.printStackTrace();
             return null;
         }
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java
index cdd5dc414fc03..77d40a61a0d4f 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java
@@ -34,7 +34,7 @@
 import org.apache.ignite.ml.structures.preprocessing.LabeledDatasetLoader;
 
 /** Tests behaviour of KNNClassificationTest. */
-public class LabeledDatasetTest extends BaseKNNTest implements ExternalizableTest<LabeledDataset> {
+public class LabeledDatasetTest extends LabeledDatasetHelper implements ExternalizableTest<LabeledDataset> {
     /** */
     private static final String KNN_IRIS_TXT = "datasets/knn/iris.txt";
 

From 687ae653bd66745c49ba9f85a169e27191ddc16c Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Fri, 13 Apr 2018 12:28:19 +0300
Subject: [PATCH 044/543] IGNITE-8240 .NET: Use default scheduler when starting
 Tasks

This closes #3812
---
 .gitignore                                    |  1 +
 .../IgniteSessionStateStoreProviderTest.cs    |  7 +-
 .../BenchmarkRunner.cs                        |  1 -
 .../Binary/BinaryDynamicRegistrationTest.cs   |  2 +-
 .../Cache/CacheAbstractTransactionalTest.cs   |  5 +-
 .../Client/ClientConnectionTest.cs            |  3 +-
 .../Apache.Ignite.Core.Tests/EventsTest.cs    |  9 +--
 .../ExceptionsTest.cs                         |  3 +-
 .../IgniteStartStopTest.cs                    |  5 +-
 .../Apache.Ignite.Core.Tests/MessagingTest.cs |  5 +-
 .../Apache.Ignite.Core.csproj                 |  1 +
 .../Impl/Client/ClientSocket.cs               |  6 +-
 .../Impl/Common/TaskRunner.cs                 | 70 +++++++++++++++++++
 .../Impl/Datastream/DataStreamerBatch.cs      |  2 +-
 .../Impl/Datastream/DataStreamerImpl.cs       |  2 +-
 .../Apache.Ignite.Core/Impl/Events/Events.cs  |  2 +-
 .../Impl/Transactions/TransactionImpl.cs      |  3 +-
 17 files changed, 103 insertions(+), 24 deletions(-)
 create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/TaskRunner.cs

diff --git a/.gitignore b/.gitignore
index 535a8ff093818..47220b27df9b8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ git-patch-prop-local.sh
 **/dotnet/libs/
 *.classname*
 *.exe
+.mvn/
 
 #Visual Studio files
 *.[Oo]bj
diff --git a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/IgniteSessionStateStoreProviderTest.cs b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/IgniteSessionStateStoreProviderTest.cs
index 25700c6aa3a5f..08c44a6b0f261 100644
--- a/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/IgniteSessionStateStoreProviderTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.AspNet.Tests/IgniteSessionStateStoreProviderTest.cs
@@ -28,6 +28,7 @@ namespace Apache.Ignite.AspNet.Tests
     using System.Web.SessionState;
     using Apache.Ignite.Core;
     using Apache.Ignite.Core.Common;
+    using Apache.Ignite.Core.Impl.Common;
     using Apache.Ignite.Core.Tests;
     using NUnit.Framework;
 
@@ -265,7 +266,7 @@ public void TestCaching()
             Assert.AreEqual(SessionStateActions.None, actions);
 
             // Try to get it in a different thread.
-            Task.Factory.StartNew(() =>
+            TaskRunner.Run(() =>
             {
                 object lockId1;   // do not overwrite lockId
                 res = provider.GetItem(HttpContext, Id, out locked, out lockAge, out lockId1, out actions);
@@ -277,7 +278,7 @@ public void TestCaching()
             }).Wait();
 
             // Try to get it in a different thread.
-            Task.Factory.StartNew(() =>
+            TaskRunner.Run(() =>
             {
                 object lockId1;   // do not overwrite lockId
                 res = provider.GetItemExclusive(HttpContext, Id, out locked, out lockAge, out lockId1, out actions);
@@ -292,7 +293,7 @@ public void TestCaching()
             provider.ReleaseItemExclusive(HttpContext, Id, lockId);
 
             // Make sure it is accessible in a different thread.
-            Task.Factory.StartNew(() =>
+            TaskRunner.Run(() =>
             {
                 res = provider.GetItem(HttpContext, Id, out locked, out lockAge, out lockId, out actions);
                 Assert.IsNotNull(res);
diff --git a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/BenchmarkRunner.cs b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/BenchmarkRunner.cs
index e152ffb8a81a8..fb2fbd28063f5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Benchmarks/BenchmarkRunner.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Benchmarks/BenchmarkRunner.cs
@@ -22,7 +22,6 @@ namespace Apache.Ignite.Benchmarks
     using System.IO;
     using System.Text;
     using Apache.Ignite.Benchmarks.Interop;
-    using Apache.Ignite.Benchmarks.ThinClient;
 
     /// <summary>
     /// Benchmark runner.
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryDynamicRegistrationTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryDynamicRegistrationTest.cs
index e635bd12cdb20..272a0cae819f6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryDynamicRegistrationTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Binary/BinaryDynamicRegistrationTest.cs
@@ -399,7 +399,7 @@ public void TestRegistrationMultithreaded([Values(true, false)] bool useTypeName
                         };
 
                         var tasks = Enumerable.Range(0, threads)
-                            .Select(x => Task.Factory.StartNew(registerType))
+                            .Select(x => TaskRunner.Run(registerType))
                             .ToArray();
 
                         Task.WaitAll(tasks);
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
index 2602a02efb75c..3d0168c08d6cb 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
@@ -25,6 +25,7 @@ namespace Apache.Ignite.Core.Tests.Cache
     using System.Transactions;
     using Apache.Ignite.Core.Cache;
     using Apache.Ignite.Core.Cache.Configuration;
+    using Apache.Ignite.Core.Impl.Common;
     using Apache.Ignite.Core.Transactions;
     using NUnit.Framework;
 
@@ -563,8 +564,8 @@ public void TestTxDeadlockDetection()
             var aex = Assert.Throws<AggregateException>(() =>
                 Task.WaitAll(new[]
                     {
-                        Task.Factory.StartNew(() => increment(keys0)),
-                        Task.Factory.StartNew(() => increment(keys0.Reverse().ToArray()))
+                        TaskRunner.Run(() => increment(keys0)),
+                        TaskRunner.Run(() => increment(keys0.Reverse().ToArray()))
                     },
                     TimeSpan.FromSeconds(40)));
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
index 2ea17a8739227..cb30f40c874be 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
@@ -31,6 +31,7 @@ namespace Apache.Ignite.Core.Tests.Client
     using Apache.Ignite.Core.Client;
     using Apache.Ignite.Core.Client.Cache;
     using Apache.Ignite.Core.Configuration;
+    using Apache.Ignite.Core.Impl.Common;
     using NUnit.Framework;
 
     /// <summary>
@@ -310,7 +311,7 @@ public void TestServerConnectionAborted()
             var evt = new ManualResetEventSlim();
             var ignite = Ignition.Start(TestUtils.GetTestConfiguration());
 
-            var putGetTask = Task.Factory.StartNew(() =>
+            var putGetTask = TaskRunner.Run(() =>
             {
                 using (var client = StartClient())
                 {
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs
index a7c05344e963f..e9bac023a41bf 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/EventsTest.cs
@@ -34,6 +34,7 @@ namespace Apache.Ignite.Core.Tests
     using Apache.Ignite.Core.Common;
     using Apache.Ignite.Core.Events;
     using Apache.Ignite.Core.Impl;
+    using Apache.Ignite.Core.Impl.Common;
     using Apache.Ignite.Core.Impl.Events;
     using Apache.Ignite.Core.Resource;
     using Apache.Ignite.Core.Tests.Compute;
@@ -385,14 +386,14 @@ public void TestWaitForLocal()
         /// </summary>
         private static IEnumerable<Func<IEventFilter<IEvent>, int[], Task<IEvent>>> GetWaitTasks(IEvents events)
         {
-            yield return (filter, types) => Task.Factory.StartNew(() => events.WaitForLocal(types));
-            yield return (filter, types) => Task.Factory.StartNew(() => events.WaitForLocal(types.ToList()));
+            yield return (filter, types) => TaskRunner.Run(() => events.WaitForLocal(types));
+            yield return (filter, types) => TaskRunner.Run(() => events.WaitForLocal(types.ToList()));
 
             yield return (filter, types) => events.WaitForLocalAsync(types);
             yield return (filter, types) => events.WaitForLocalAsync(types.ToList());
 
-            yield return (filter, types) => Task.Factory.StartNew(() => events.WaitForLocal(filter, types));
-            yield return (filter, types) => Task.Factory.StartNew(() => events.WaitForLocal(filter, types.ToList()));
+            yield return (filter, types) => TaskRunner.Run(() => events.WaitForLocal(filter, types));
+            yield return (filter, types) => TaskRunner.Run(() => events.WaitForLocal(filter, types.ToList()));
 
             yield return (filter, types) => events.WaitForLocalAsync(filter, types);
             yield return (filter, types) => events.WaitForLocalAsync(filter, types.ToList());
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExceptionsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExceptionsTest.cs
index f7568ef19e38f..0b06ea34bde97 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExceptionsTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ExceptionsTest.cs
@@ -29,6 +29,7 @@ namespace Apache.Ignite.Core.Tests
     using Apache.Ignite.Core.Cluster;
     using Apache.Ignite.Core.Common;
     using Apache.Ignite.Core.Compute;
+    using Apache.Ignite.Core.Impl.Common;
     using Apache.Ignite.Core.Services;
     using Apache.Ignite.Core.Transactions;
     using NUnit.Framework;
@@ -348,7 +349,7 @@ private static void TestPartialUpdateException<TK>(bool async, Func<int, IIgnite
                     cache = cache.WithKeepBinary<TK, int>();
 
                 // Do cache puts in parallel
-                var putTask = Task.Factory.StartNew(() =>
+                var putTask = TaskRunner.Run(() =>
                 {
                     try
                     {
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
index 792b33d484cda..f9c1cad8055cd 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/IgniteStartStopTest.cs
@@ -23,6 +23,7 @@ namespace Apache.Ignite.Core.Tests
     using System.Threading;
     using System.Threading.Tasks;
     using Apache.Ignite.Core.Common;
+    using Apache.Ignite.Core.Impl.Common;
     using Apache.Ignite.Core.Messaging;
     using Apache.Ignite.Core.Tests.Process;
     using NUnit.Framework;
@@ -207,7 +208,7 @@ public void TestStartStopLeak()
 
                 if (i % 2 == 0) // Try to stop ignite from another thread.
                 {
-                    Task.Factory.StartNew(() => grid.Dispose()).Wait();
+                    TaskRunner.Run(() => grid.Dispose()).Wait();
                 }
                 else
                 {
@@ -306,7 +307,7 @@ public void TestProcessorInit()
 
             // Spam message subscriptions on a separate thread 
             // to test race conditions during processor init on remote node
-            var listenTask = Task.Factory.StartNew(() =>
+            var listenTask = TaskRunner.Run(() =>
             {
                 var filter = new MessageListener();
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/MessagingTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/MessagingTest.cs
index e644e317d5f97..7db4eef09f44a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/MessagingTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/MessagingTest.cs
@@ -27,6 +27,7 @@ namespace Apache.Ignite.Core.Tests
     using Apache.Ignite.Core.Cache.Configuration;
     using Apache.Ignite.Core.Cluster;
     using Apache.Ignite.Core.Common;
+    using Apache.Ignite.Core.Impl.Common;
     using Apache.Ignite.Core.Messaging;
     using Apache.Ignite.Core.Resource;
     using Apache.Ignite.Core.Tests.Cache;
@@ -252,7 +253,7 @@ public void TestLocalListenMultithreaded()
 
             var messaging = _grid1.GetMessaging();
 
-            var senders = Task.Factory.StartNew(() => TestUtils.RunMultiThreaded(() =>
+            var senders = TaskRunner.Run(() => TestUtils.RunMultiThreaded(() =>
             {
                 messaging.Send(NextMessage());
                 Thread.Sleep(50);
@@ -423,7 +424,7 @@ public void TestRemoteListenMultithreaded()
 
             var messaging = _grid1.GetMessaging();
 
-            var senders = Task.Factory.StartNew(() => TestUtils.RunMultiThreaded(() =>
+            var senders = TaskRunner.Run(() => TestUtils.RunMultiThreaded(() =>
             {
                 MessagingTestHelper.ClearReceived(int.MaxValue);
                 messaging.Send(NextMessage());
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
index ec84a38fa77a7..93c45c34d4caa 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Apache.Ignite.Core.csproj
@@ -72,6 +72,7 @@
     <Compile Include="Client\IgniteClientException.cs" />
     <Compile Include="Client\IIgniteClient.cs" />
     <Compile Include="Common\ExceptionFactory.cs" />
+    <Compile Include="Impl\Common\TaskRunner.cs" />
     <Compile Include="Ssl\ISslContextFactory.cs" />
     <Compile Include="Configuration\Package-Info.cs" />
     <Compile Include="Configuration\ClientConnectorConfiguration.cs" />
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
index 27d8f0bea092a..bce681f8ca557 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
@@ -28,11 +28,11 @@ namespace Apache.Ignite.Core.Impl.Client
     using System.Net.Sockets;
     using System.Threading;
     using System.Threading.Tasks;
-    using System.Xml.Schema;
     using Apache.Ignite.Core.Client;
     using Apache.Ignite.Core.Common;
     using Apache.Ignite.Core.Impl.Binary;
     using Apache.Ignite.Core.Impl.Binary.IO;
+    using Apache.Ignite.Core.Impl.Common;
 
     /// <summary>
     /// Wrapper over framework socket for Ignite thin client operations.
@@ -117,7 +117,7 @@ public ClientSocket(IgniteClientConfiguration clientConfiguration, ClientProtoco
             }
 
             // Continuously and asynchronously wait for data from server.
-            Task.Factory.StartNew(WaitForMessages);
+            TaskRunner.Run(WaitForMessages);
         }
 
         /// <summary>
@@ -174,7 +174,7 @@ public Task<T> DoOutInOpAsync<T>(ClientOp opId, Action<IBinaryStream> writeActio
             var task = SendRequestAsync(ref reqMsg);
 
             // Decode.
-            return task.ContinueWith(responseTask => DecodeResponse(responseTask.Result, readFunc, errorFunc));
+            return task.ContWith(responseTask => DecodeResponse(responseTask.Result, readFunc, errorFunc));
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/TaskRunner.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/TaskRunner.cs
new file mode 100644
index 0000000000000..51a7c6a98bd43
--- /dev/null
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Common/TaskRunner.cs
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+namespace Apache.Ignite.Core.Impl.Common
+{
+    using System;
+    using System.Threading;
+    using System.Threading.Tasks;
+
+    /// <summary>
+    /// Extensions for <see cref="Task"/> classes.
+    /// Fixes the issue with <see cref="TaskScheduler.Current"/> being used by defaut by system APIs.
+    /// </summary>
+    internal static class TaskRunner
+    {
+        /// <summary>
+        /// ContinueWith using default scheduler.
+        /// </summary>
+        public static Task<TNewResult> ContWith<TResult, TNewResult>(this Task<TResult> task,
+            Func<Task<TResult>, TNewResult> continuationFunction)
+        {
+            IgniteArgumentCheck.NotNull(task, "task");
+            
+            return task.ContinueWith(continuationFunction, TaskScheduler.Default);
+        }
+        
+        /// <summary>
+        /// ContinueWith using default scheduler.
+        /// </summary>
+        public static Task ContWith(this Task task,
+            Action<Task> continuationFunction)
+        {
+            IgniteArgumentCheck.NotNull(task, "task");
+            
+            return task.ContinueWith(continuationFunction, TaskScheduler.Default);
+        }
+
+        /// <summary>
+        /// Run new task using default scheduler.
+        /// </summary>
+        public static Task Run(Action action)
+        {
+            return Task.Factory.StartNew(action, CancellationToken.None, TaskCreationOptions.None, 
+                TaskScheduler.Default);
+        }
+        
+        /// <summary>
+        /// Run new task using default scheduler.
+        /// </summary>
+        public static Task<TResult> Run<TResult>(Func<TResult> func)
+        {
+            return Task.Factory.StartNew(func, CancellationToken.None, TaskCreationOptions.None, 
+                TaskScheduler.Default);
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerBatch.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerBatch.cs
index 38a8ea885ac38..002670113071c 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerBatch.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerBatch.cs
@@ -69,7 +69,7 @@ public DataStreamerBatch(DataStreamerBatch<TK, TV> prev)
             if (prev != null)
                 Thread.MemoryBarrier(); // Prevent "prev" field escape.
 
-            _fut.Task.ContinueWith(x => ParentsCompleted());
+            _fut.Task.ContWith(x => ParentsCompleted());
         }
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs
index 555c6e6c05b7c..7aaa84a8009e9 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Datastream/DataStreamerImpl.cs
@@ -897,7 +897,7 @@ public void Stop()
             /// </summary>
             public void RunThread()
             {
-                Task.Factory.StartNew(Run);
+                TaskRunner.Run(Run);
             }
         }
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs
index a81523a3e13f2..04cc210a3370e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Events/Events.cs
@@ -241,7 +241,7 @@ public Task<T> WaitForLocalAsync<T>(IEventFilter<T> filter, params int[] types)
                 if (hnd != null)
                 {
                     // Dispose handle as soon as future ends.
-                    task.ContinueWith(x => Ignite.HandleRegistry.Release(hnd.Value));
+                    task.ContWith(x => Ignite.HandleRegistry.Release(hnd.Value));
                 }
 
                 return task;
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionImpl.cs
index 0b04a6803a975..c80085969c093 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Transactions/TransactionImpl.cs
@@ -21,6 +21,7 @@ namespace Apache.Ignite.Core.Impl.Transactions
     using System.Globalization;
     using System.Threading;
     using System.Threading.Tasks;
+    using Apache.Ignite.Core.Impl.Common;
     using Apache.Ignite.Core.Transactions;
 
     /// <summary>
@@ -457,7 +458,7 @@ private void ThrowIfClosed()
         /// </summary>
         private Task CloseWhenComplete(Task task)
         {
-            return task.ContinueWith(x => Close());
+            return task.ContWith(x => Close());
         }
 
         /** <inheritdoc /> */

From 0a64e4affae9ec2e16c3a99128985f6bd9c788cb Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Fri, 13 Apr 2018 12:44:17 +0300
Subject: [PATCH 045/543] IGNITE-8042: .NET: Thin client: authentication
 support - fix naming and inspections

---
 .../Client/ClientConnectionTest.cs                 | 12 ++++++------
 .../Client/IgniteClientConfiguration.cs            |  4 ++--
 .../IgniteClientConfigurationSection.xsd           |  2 +-
 .../Apache.Ignite.Core/Impl/Client/ClientSocket.cs | 14 +++++++-------
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
index cb30f40c874be..0a6b1a76a71eb 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
@@ -98,11 +98,11 @@ public void TestAuthenticationEmptyCredentials()
 
                 cliCfg.Password = "ignite";
 
-                cliCfg.Username = null;
+                cliCfg.UserName = null;
                 ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
                 Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Username cannot be null"));
 
-                cliCfg.Username = "";
+                cliCfg.UserName = "";
                 ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
                 Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Username cannot be empty"));
             }
@@ -118,12 +118,12 @@ public void TestAuthenticationInvalidCredentials()
             {
                 var cliCfg = SecureClientConfig();
 
-                cliCfg.Username = "invalid";
+                cliCfg.UserName = "invalid";
 
                 var ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
                 Assert.True(ex.StatusCode == ClientStatusCode.AuthenticationFailed);
 
-                cliCfg.Username = "ignite";
+                cliCfg.UserName = "ignite";
                 cliCfg.Password = "invalid";
 
                 ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
@@ -165,7 +165,7 @@ public void TestAuthentication()
 
                 var cliCfg = SecureClientConfig();
 
-                cliCfg.Username = "my_User";
+                cliCfg.UserName = "my_User";
                 cliCfg.Password = "my_Password";
 
                 using (var cli = Ignition.StartClient(cliCfg))
@@ -532,7 +532,7 @@ private static IgniteClientConfiguration SecureClientConfig()
             return new IgniteClientConfiguration()
             {
                 Host = "localhost",
-                Username = "ignite",
+                UserName = "ignite",
                 Password = "ignite"
             };
         }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs
index 32524955ee4c2..80f26cfa5e7d6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs
@@ -91,7 +91,7 @@ public IgniteClientConfiguration(IgniteClientConfiguration cfg) : this()
             BinaryProcessor = cfg.BinaryProcessor;
             SslStreamFactory = cfg.SslStreamFactory;
 
-            Username = cfg.Username;
+            UserName = cfg.UserName;
             Password = cfg.Password;
         }
 
@@ -151,7 +151,7 @@ public IgniteClientConfiguration(IgniteClientConfiguration cfg) : this()
         /// <summary>
         /// Username to be used to connect to secured cluster.
         /// </summary>
-        public string Username { get; set; }
+        public string UserName { get; set; }
 
         /// <summary>
         /// Password to be used to connect to secured cluster.
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd
index 7e6caff6c4d19..b9a04b8e0d75d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd
@@ -237,7 +237,7 @@
                     <xs:documentation>Socket operation timeout. Zero or negative for infinite timeout.</xs:documentation>
                 </xs:annotation>
             </xs:attribute>
-            <xs:attribute name="username" type="xs:string">
+            <xs:attribute name="userName" type="xs:string">
                 <xs:annotation>
                     <xs:documentation>Username to be used to connect to secured cluster.</xs:documentation>
                 </xs:annotation>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
index bce681f8ca557..11d79420ed365 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientSocket.cs
@@ -124,11 +124,11 @@ public ClientSocket(IgniteClientConfiguration clientConfiguration, ClientProtoco
         /// Validate configuration.
         /// </summary>
         /// <param name="cfg">Configuration.</param>
-        private void Validate(IgniteClientConfiguration cfg)
+        private static void Validate(IgniteClientConfiguration cfg)
         {
-            if (cfg.Username != null)
+            if (cfg.UserName != null)
             {
-                if (cfg.Username.Length == 0)
+                if (cfg.UserName.Length == 0)
                     throw new IgniteClientException("IgniteClientConfiguration.Username cannot be empty.");
 
                 if (cfg.Password == null)
@@ -140,8 +140,8 @@ private void Validate(IgniteClientConfiguration cfg)
                 if (cfg.Password.Length == 0)
                     throw new IgniteClientException("IgniteClientConfiguration.Password cannot be empty.");
 
-                if (cfg.Username == null)
-                    throw new IgniteClientException("IgniteClientConfiguration.Username cannot be null when Password is set.");
+                if (cfg.UserName == null)
+                    throw new IgniteClientException("IgniteClientConfiguration.UserName cannot be null when Password is set.");
             }
         }
 
@@ -262,7 +262,7 @@ private static T DecodeResponse<T>(BinaryHeapStream stream, Func<IBinaryStream,
         /// </summary>
         private void Handshake(IgniteClientConfiguration clientConfiguration, ClientProtocolVersion version)
         {
-            bool auth = version.CompareTo(Ver110) >= 0 && clientConfiguration.Username != null;
+            bool auth = version.CompareTo(Ver110) >= 0 && clientConfiguration.UserName != null;
 
             // Send request.
             int messageLen;
@@ -284,7 +284,7 @@ private void Handshake(IgniteClientConfiguration clientConfiguration, ClientProt
                 {
                     var writer = BinaryUtils.Marshaller.StartMarshal(stream);
 
-                    writer.WriteString(clientConfiguration.Username);
+                    writer.WriteString(clientConfiguration.UserName);
                     writer.WriteString(clientConfiguration.Password);
 
                     BinaryUtils.Marshaller.FinishMarshal(writer);

From 6072af825ca1507ad9d8143ca73e556539960e1d Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Fri, 13 Apr 2018 13:36:20 +0300
Subject: [PATCH 046/543] IGNITE-8042: .NET: Thin client: authentication
 support - fix TestAuthenticationEmptyCredentials

---
 .../Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
index 0a6b1a76a71eb..67d1c52cfe001 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/ClientConnectionTest.cs
@@ -100,7 +100,7 @@ public void TestAuthenticationEmptyCredentials()
 
                 cliCfg.UserName = null;
                 ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });
-                Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.Username cannot be null"));
+                Assert.IsTrue(ex.Message.StartsWith("IgniteClientConfiguration.UserName cannot be null"));
 
                 cliCfg.UserName = "";
                 ex = Assert.Throws<IgniteClientException>(() => { Ignition.StartClient(cliCfg); });

From 78e7414cf568ef7e3f7567fdb74334012896b632 Mon Sep 17 00:00:00 2001
From: Dmitriy Shabalin <dshabalin@gridgain.com>
Date: Fri, 13 Apr 2018 17:55:02 +0700
Subject: [PATCH 047/543] IGNITE-8245 Fixed input appearance position with
 error.

(cherry picked from commit 56e3f43)
---
 .../frontend/app/primitives/form-field/index.scss    | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/modules/web-console/frontend/app/primitives/form-field/index.scss b/modules/web-console/frontend/app/primitives/form-field/index.scss
index 7d9ea1ffd655d..1035adecfc6c9 100644
--- a/modules/web-console/frontend/app/primitives/form-field/index.scss
+++ b/modules/web-console/frontend/app/primitives/form-field/index.scss
@@ -240,6 +240,18 @@
                 box-shadow: none;
             }
         }
+
+        // Added right offset to appearance of input for invalid password
+        & > input[type='password'].ng-invalid.ng-touched {
+            padding-right: 36px;
+        }
+
+        // Added right offset to appearance of dropdown for invalid data
+        & > button.select-toggle.ng-invalid.ng-touched {
+            &:after {
+                right: 36px;
+            }
+        }
     }
 
     &__errors {

From db3099e079022a8acc081b284f96583280742567 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 13 Apr 2018 18:08:35 +0700
Subject: [PATCH 048/543] IGNITE-8248 Fixed npe in Web Console agent int case
 of self-signed certificate.

(cherry picked from commit 8a42f64)
---
 .../apache/ignite/console/agent/AgentLauncher.java    | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 4db26bacc05b6..385ce08a12289 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -32,6 +32,7 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.UnknownHostException;
+import java.security.cert.X509Certificate;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -122,18 +123,16 @@ private static TrustManager[] getTrustManagers() {
         return new TrustManager[] {
             new X509TrustManager() {
                 /** {@inheritDoc} */
-                @Override public java.security.cert.X509Certificate[] getAcceptedIssuers() {
-                    return null;
+                @Override public X509Certificate[] getAcceptedIssuers() {
+                    return new X509Certificate[0];
                 }
 
                 /** {@inheritDoc} */
-                @Override public void checkClientTrusted(
-                    java.security.cert.X509Certificate[] certs, String authType) {
+                @Override public void checkClientTrusted(X509Certificate[] certs, String authType) {
                 }
 
                 /** {@inheritDoc} */
-                @Override public void checkServerTrusted(
-                    java.security.cert.X509Certificate[] certs, String authType) {
+                @Override public void checkServerTrusted(X509Certificate[] certs, String authType) {
                 }
             }};
     }

From e904b7b12a9bf4e038d13e39e7d85eaccfb40d65 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Fri, 13 Apr 2018 18:01:00 +0300
Subject: [PATCH 049/543] IGNITE-8256 Fixed simulated node failure in the test

---
 .../cache/distributed/dht/TxRecoveryStoreEnabledTest.java      | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
index 060af21f8d363..30ac83d5f50b9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/TxRecoveryStoreEnabledTest.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
@@ -81,6 +82,8 @@ public class TxRecoveryStoreEnabledTest extends GridCommonAbstractTest {
 
         cfg.setCacheConfiguration(ccfg);
 
+        cfg.setFailureHandler(new StopNodeFailureHandler());
+
         return cfg;
     }
 

From 9be980d084b19a604d7f811bf2c73a28b35b0cd6 Mon Sep 17 00:00:00 2001
From: dmitrievanthony <dmitrievanthony@gmail.com>
Date: Fri, 13 Apr 2018 18:02:37 +0300
Subject: [PATCH 050/543] IGNITE-8233: KNN and SVM algorithms don't work when
 partition doesn't contain data.

this closes #3807

(cherry picked from commit ee9ca06)
---
 .../dataset/impl/cache/CacheBasedDataset.java |  14 +-
 .../dataset/impl/cache/util/ComputeUtils.java |   9 +-
 .../ml/dataset/impl/local/LocalDataset.java   |  16 ++-
 .../impl/local/LocalDatasetBuilder.java       |   8 +-
 .../KNNClassificationModel.java               |  32 +++--
 .../impl/cache/CacheBasedDatasetTest.java     |   9 +-
 .../ignite/ml/knn/KNNClassificationTest.java  | 120 +++++++++--------
 .../ignite/ml/knn/KNNRegressionTest.java      | 122 ++++++++++--------
 .../ignite/ml/knn/LabeledDatasetHelper.java   |  45 +------
 .../ignite/ml/knn/LabeledDatasetTest.java     |  54 ++++----
 10 files changed, 226 insertions(+), 203 deletions(-)

diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDataset.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDataset.java
index 463d4964d09b1..7428faf28fc6c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDataset.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDataset.java
@@ -101,12 +101,16 @@ public CacheBasedDataset(Ignite ignite, IgniteCache<K, V> upstreamCache,
                 partDataBuilder
             );
 
-            R res = map.apply(ctx, data, part);
+            if (data != null) {
+                R res = map.apply(ctx, data, part);
 
-            // Saves partition context after update.
-            ComputeUtils.saveContext(Ignition.localIgnite(), datasetCacheName, part, ctx);
+                // Saves partition context after update.
+                ComputeUtils.saveContext(Ignition.localIgnite(), datasetCacheName, part, ctx);
 
-            return res;
+                return res;
+            }
+
+            return null;
         }, reduce, identity);
     }
 
@@ -125,7 +129,7 @@ public CacheBasedDataset(Ignite ignite, IgniteCache<K, V> upstreamCache,
                 partDataBuilder
             );
 
-            return map.apply(data, part);
+            return data != null ? map.apply(data, part) : null;
         }, reduce, identity);
     }
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtils.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtils.java
index 0785db28711bb..ce2fcfdcf8440 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtils.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/cache/util/ComputeUtils.java
@@ -163,9 +163,14 @@ public static <K, V, C extends Serializable, D extends AutoCloseable> D getData(
             qry.setPartition(part);
 
             long cnt = upstreamCache.localSizeLong(part);
-            try (QueryCursor<Cache.Entry<K, V>> cursor = upstreamCache.query(qry)) {
-                return partDataBuilder.build(new UpstreamCursorAdapter<>(cursor.iterator(), cnt), cnt, ctx);
+
+            if (cnt > 0) {
+                try (QueryCursor<Cache.Entry<K, V>> cursor = upstreamCache.query(qry)) {
+                    return partDataBuilder.build(new UpstreamCursorAdapter<>(cursor.iterator(), cnt), cnt, ctx);
+                }
             }
+
+            return null;
         });
     }
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDataset.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDataset.java
index c08b7dedaba17..e312b202a1259 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDataset.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDataset.java
@@ -55,8 +55,12 @@ public class LocalDataset<C extends Serializable, D extends AutoCloseable> imple
         R identity) {
         R res = identity;
 
-        for (int part = 0; part < ctx.size(); part++)
-            res = reduce.apply(res, map.apply(ctx.get(part), data.get(part), part));
+        for (int part = 0; part < ctx.size(); part++) {
+            D partData = data.get(part);
+
+            if (partData != null)
+                res = reduce.apply(res, map.apply(ctx.get(part), partData, part));
+        }
 
         return res;
     }
@@ -65,8 +69,12 @@ public class LocalDataset<C extends Serializable, D extends AutoCloseable> imple
     @Override public <R> R compute(IgniteBiFunction<D, Integer, R> map, IgniteBinaryOperator<R> reduce, R identity) {
         R res = identity;
 
-        for (int part = 0; part < data.size(); part++)
-            res = reduce.apply(res, map.apply(data.get(part), part));
+        for (int part = 0; part < data.size(); part++) {
+            D partData = data.get(part);
+
+            if (partData != null)
+                res = reduce.apply(res, map.apply(partData, part));
+        }
 
         return res;
     }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilder.java b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilder.java
index 0dc1ed6b5ecbd..cfc1801f26282 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilder.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/dataset/impl/local/LocalDatasetBuilder.java
@@ -69,16 +69,16 @@ public LocalDatasetBuilder(Map<K, V> upstreamMap, int partitions) {
         for (int part = 0; part < partitions; part++) {
             int cnt = part == partitions - 1 ? upstreamMap.size() - ptr : Math.min(partSize, upstreamMap.size() - ptr);
 
-            C ctx = partCtxBuilder.build(
+            C ctx = cnt > 0 ? partCtxBuilder.build(
                 new IteratorWindow<>(firstKeysIter, k -> new UpstreamEntry<>(k, upstreamMap.get(k)), cnt),
                 cnt
-            );
+            ) : null;
 
-            D data = partDataBuilder.build(
+            D data = cnt > 0 ? partDataBuilder.build(
                 new IteratorWindow<>(secondKeysIter, k -> new UpstreamEntry<>(k, upstreamMap.get(k)), cnt),
                 cnt,
                 ctx
-            );
+            ) : null;
 
             ctxList.add(ctx);
             dataList.add(data);
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java
index 693b81df0ecff..0f0cc9fed544c 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/classification/KNNClassificationModel.java
@@ -151,19 +151,29 @@ private LabeledDataset<Double, LabeledVector> buildLabeledDatasetOnListOfVectors
      */
     @NotNull private LabeledVector[] getKClosestVectors(LabeledDataset<Double, LabeledVector> trainingData,
         TreeMap<Double, Set<Integer>> distanceIdxPairs) {
-        LabeledVector[] res = new LabeledVector[k];
-        int i = 0;
-        final Iterator<Double> iter = distanceIdxPairs.keySet().iterator();
-        while (i < k) {
-            double key = iter.next();
-            Set<Integer> idxs = distanceIdxPairs.get(key);
-            for (Integer idx : idxs) {
-                res[i] = trainingData.getRow(idx);
-                i++;
-                if (i >= k)
-                    break; // go to next while-loop iteration
+        LabeledVector[] res;
+
+        if (trainingData.rowSize() <= k) {
+            res = new LabeledVector[trainingData.rowSize()];
+            for (int i = 0; i < trainingData.rowSize(); i++)
+                res[i] = trainingData.getRow(i);
+        }
+        else {
+            res = new LabeledVector[k];
+            int i = 0;
+            final Iterator<Double> iter = distanceIdxPairs.keySet().iterator();
+            while (i < k) {
+                double key = iter.next();
+                Set<Integer> idxs = distanceIdxPairs.get(key);
+                for (Integer idx : idxs) {
+                    res[i] = trainingData.getRow(idx);
+                    i++;
+                    if (i >= k)
+                        break; // go to next while-loop iteration
+                }
             }
         }
+
         return res;
     }
 
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java
index dc0e160e3ca19..16ba04403310d 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.ml.dataset.primitive.data.SimpleDatasetData;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -81,9 +82,9 @@ public void testPartitionExchangeDuringComputeCall() {
 
         CacheBasedDatasetBuilder<Integer, String> builder = new CacheBasedDatasetBuilder<>(ignite, upstreamCache);
 
-        CacheBasedDataset<Integer, String, Long, AutoCloseable> dataset = builder.build(
+        CacheBasedDataset<Integer, String, Long, SimpleDatasetData> dataset = builder.build(
             (upstream, upstreamSize) -> upstreamSize,
-            (upstream, upstreamSize, ctx) -> null
+            (upstream, upstreamSize, ctx) -> new SimpleDatasetData(new double[0], 0)
         );
 
         assertTrue("Before computation all partitions should not be reserved",
@@ -133,9 +134,9 @@ public void testPartitionExchangeDuringComputeWithCtxCall() {
 
         CacheBasedDatasetBuilder<Integer, String> builder = new CacheBasedDatasetBuilder<>(ignite, upstreamCache);
 
-        CacheBasedDataset<Integer, String, Long, AutoCloseable> dataset = builder.build(
+        CacheBasedDataset<Integer, String, Long, SimpleDatasetData> dataset = builder.build(
             (upstream, upstreamSize) -> upstreamSize,
-            (upstream, upstreamSize, ctx) -> null
+            (upstream, upstreamSize, ctx) -> new SimpleDatasetData(new double[0], 0)
         );
 
         assertTrue("Before computation all partitions should not be reserved",
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
index 0877fc07cd16c..004718e238ba1 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNClassificationTest.java
@@ -17,11 +17,11 @@
 
 package org.apache.ignite.ml.knn;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import org.junit.Assert;
-import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
 import org.apache.ignite.ml.knn.classification.KNNClassificationTrainer;
 import org.apache.ignite.ml.knn.classification.KNNStrategy;
@@ -29,121 +29,137 @@
 import org.apache.ignite.ml.math.distances.EuclideanDistance;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import static junit.framework.TestCase.assertEquals;
 
 /** Tests behaviour of KNNClassificationTest. */
+@RunWith(Parameterized.class)
 public class KNNClassificationTest {
-    /** Precision in test checks. */
-    private static final double PRECISION = 1e-2;
+    /** Number of parts to be tested. */
+    private static final int[] partsToBeTested = new int[] {1, 2, 3, 4, 5, 7, 100};
+
+    /** Number of partitions. */
+    @Parameterized.Parameter
+    public int parts;
+
+    /** Parameters. */
+    @Parameterized.Parameters(name = "Data divided on {0} partitions, training with batch size {1}")
+    public static Iterable<Integer[]> data() {
+        List<Integer[]> res = new ArrayList<>();
+
+        for (int part : partsToBeTested)
+            res.add(new Integer[] {part});
+
+        return res;
+    }
 
     /** */
     @Test
-    public void binaryClassificationTest() {
-
+    public void testBinaryClassificationTest() {
         Map<Integer, double[]> data = new HashMap<>();
-        data.put(0, new double[]{1.0, 1.0, 1.0});
-        data.put(1, new double[]{1.0, 2.0, 1.0});
-        data.put(2, new double[]{2.0, 1.0, 1.0});
-        data.put(3, new double[]{-1.0, -1.0, 2.0});
-        data.put(4, new double[]{-1.0, -2.0, 2.0});
-        data.put(5, new double[]{-2.0, -1.0, 2.0});
+        data.put(0, new double[] {1.0, 1.0, 1.0});
+        data.put(1, new double[] {1.0, 2.0, 1.0});
+        data.put(2, new double[] {2.0, 1.0, 1.0});
+        data.put(3, new double[] {-1.0, -1.0, 2.0});
+        data.put(4, new double[] {-1.0, -2.0, 2.0});
+        data.put(5, new double[] {-2.0, -1.0, 2.0});
 
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
         KNNClassificationModel knnMdl = trainer.fit(
             data,
-            2,
+            parts,
             (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
             (k, v) -> v[2]
         ).withK(3)
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector firstVector = new DenseLocalOnHeapVector(new double[]{2.0, 2.0});
-        Assert.assertEquals(knnMdl.apply(firstVector), 1.0, PRECISION);
-        Vector secondVector = new DenseLocalOnHeapVector(new double[]{-2.0, -2.0});
-        Assert.assertEquals(knnMdl.apply(secondVector), 2.0, PRECISION);
+        Vector firstVector = new DenseLocalOnHeapVector(new double[] {2.0, 2.0});
+        assertEquals(knnMdl.apply(firstVector), 1.0);
+        Vector secondVector = new DenseLocalOnHeapVector(new double[] {-2.0, -2.0});
+        assertEquals(knnMdl.apply(secondVector), 2.0);
     }
 
     /** */
     @Test
-    public void binaryClassificationWithSmallestKTest() {
+    public void testBinaryClassificationWithSmallestKTest() {
         Map<Integer, double[]> data = new HashMap<>();
-
-        data.put(0, new double[]{1.0, 1.0, 1.0});
-        data.put(1, new double[]{1.0, 2.0, 1.0});
-        data.put(2, new double[]{2.0, 1.0, 1.0});
-        data.put(3, new double[]{-1.0, -1.0, 2.0});
-        data.put(4, new double[]{-1.0, -2.0, 2.0});
-        data.put(5, new double[]{-2.0, -1.0, 2.0});
+        data.put(0, new double[] {1.0, 1.0, 1.0});
+        data.put(1, new double[] {1.0, 2.0, 1.0});
+        data.put(2, new double[] {2.0, 1.0, 1.0});
+        data.put(3, new double[] {-1.0, -1.0, 2.0});
+        data.put(4, new double[] {-1.0, -2.0, 2.0});
+        data.put(5, new double[] {-2.0, -1.0, 2.0});
 
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
         KNNClassificationModel knnMdl = trainer.fit(
             data,
-            2,
+            parts,
             (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
             (k, v) -> v[2]
         ).withK(1)
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector firstVector = new DenseLocalOnHeapVector(new double[]{2.0, 2.0});
-        Assert.assertEquals(knnMdl.apply(firstVector), 1.0, PRECISION);
-        Vector secondVector = new DenseLocalOnHeapVector(new double[]{-2.0, -2.0});
-        Assert.assertEquals(knnMdl.apply(secondVector), 2.0, PRECISION);
+        Vector firstVector = new DenseLocalOnHeapVector(new double[] {2.0, 2.0});
+        assertEquals(knnMdl.apply(firstVector), 1.0);
+        Vector secondVector = new DenseLocalOnHeapVector(new double[] {-2.0, -2.0});
+        assertEquals(knnMdl.apply(secondVector), 2.0);
     }
 
     /** */
     @Test
-    public void binaryClassificationFarPointsWithSimpleStrategy() {
+    public void testBinaryClassificationFarPointsWithSimpleStrategy() {
         Map<Integer, double[]> data = new HashMap<>();
-
-        data.put(0, new double[]{10.0, 10.0, 1.0});
-        data.put(1, new double[]{10.0, 20.0, 1.0});
-        data.put(2, new double[]{-1, -1, 1.0});
-        data.put(3, new double[]{-2, -2, 2.0});
-        data.put(4, new double[]{-1.0, -2.0, 2.0});
-        data.put(5, new double[]{-2.0, -1.0, 2.0});
+        data.put(0, new double[] {10.0, 10.0, 1.0});
+        data.put(1, new double[] {10.0, 20.0, 1.0});
+        data.put(2, new double[] {-1, -1, 1.0});
+        data.put(3, new double[] {-2, -2, 2.0});
+        data.put(4, new double[] {-1.0, -2.0, 2.0});
+        data.put(5, new double[] {-2.0, -1.0, 2.0});
 
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
         KNNClassificationModel knnMdl = trainer.fit(
             data,
-            2,
+            parts,
             (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
             (k, v) -> v[2]
         ).withK(3)
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector vector = new DenseLocalOnHeapVector(new double[]{-1.01, -1.01});
-        Assert.assertEquals(knnMdl.apply(vector), 2.0, PRECISION);
+        Vector vector = new DenseLocalOnHeapVector(new double[] {-1.01, -1.01});
+        assertEquals(knnMdl.apply(vector), 2.0);
     }
 
     /** */
     @Test
-    public void binaryClassificationFarPointsWithWeightedStrategy() {
+    public void testBinaryClassificationFarPointsWithWeightedStrategy() {
         Map<Integer, double[]> data = new HashMap<>();
-
-        data.put(0, new double[]{10.0, 10.0, 1.0});
-        data.put(1, new double[]{10.0, 20.0, 1.0});
-        data.put(2, new double[]{-1, -1, 1.0});
-        data.put(3, new double[]{-2, -2, 2.0});
-        data.put(4, new double[]{-1.0, -2.0, 2.0});
-        data.put(5, new double[]{-2.0, -1.0, 2.0});
+        data.put(0, new double[] {10.0, 10.0, 1.0});
+        data.put(1, new double[] {10.0, 20.0, 1.0});
+        data.put(2, new double[] {-1, -1, 1.0});
+        data.put(3, new double[] {-2, -2, 2.0});
+        data.put(4, new double[] {-1.0, -2.0, 2.0});
+        data.put(5, new double[] {-2.0, -1.0, 2.0});
 
         KNNClassificationTrainer trainer = new KNNClassificationTrainer();
 
         KNNClassificationModel knnMdl = trainer.fit(
             data,
-            2,
+            parts,
             (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
             (k, v) -> v[2]
         ).withK(3)
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.WEIGHTED);
 
-        Vector vector = new DenseLocalOnHeapVector(new double[]{-1.01, -1.01});
-        Assert.assertEquals(knnMdl.apply(vector), 1.0, PRECISION);
+        Vector vector = new DenseLocalOnHeapVector(new double[] {-1.01, -1.01});
+        assertEquals(knnMdl.apply(vector), 1.0);
     }
 }
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
index ce9cae59cda80..0c26ba9d4a771 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/KNNRegressionTest.java
@@ -17,6 +17,11 @@
 
 package org.apache.ignite.ml.knn;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.knn.classification.KNNStrategy;
 import org.apache.ignite.ml.knn.regression.KNNRegressionModel;
@@ -25,110 +30,125 @@
 import org.apache.ignite.ml.math.distances.EuclideanDistance;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 import org.junit.Assert;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
 import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 /**
  * Tests for {@link KNNRegressionTrainer}.
  */
+@RunWith(Parameterized.class)
 public class KNNRegressionTest {
+    /** Number of parts to be tested. */
+    private static final int[] partsToBeTested = new int[] {1, 2, 3, 4, 5, 7, 100};
+
+    /** Number of partitions. */
+    @Parameterized.Parameter
+    public int parts;
+
+    /** Parameters. */
+    @Parameterized.Parameters(name = "Data divided on {0} partitions, training with batch size {1}")
+    public static Iterable<Integer[]> data() {
+        List<Integer[]> res = new ArrayList<>();
+
+        for (int part : partsToBeTested)
+            res.add(new Integer[] {part});
+
+        return res;
+    }
+
     /** */
     @Test
-    public void simpleRegressionWithOneNeighbour() {
+    public void testSimpleRegressionWithOneNeighbour() {
         Map<Integer, double[]> data = new HashMap<>();
-
-        data.put(0, new double[]{11.0, 0, 0, 0, 0, 0});
-        data.put(1, new double[]{12.0, 2.0, 0, 0, 0, 0});
-        data.put(2, new double[]{13.0, 0, 3.0, 0, 0, 0});
-        data.put(3, new double[]{14.0, 0, 0, 4.0, 0, 0});
-        data.put(4, new double[]{15.0, 0, 0, 0, 5.0, 0});
-        data.put(5, new double[]{16.0, 0, 0, 0, 0, 6.0});
+        data.put(0, new double[] {11.0, 0, 0, 0, 0, 0});
+        data.put(1, new double[] {12.0, 2.0, 0, 0, 0, 0});
+        data.put(2, new double[] {13.0, 0, 3.0, 0, 0, 0});
+        data.put(3, new double[] {14.0, 0, 0, 4.0, 0, 0});
+        data.put(4, new double[] {15.0, 0, 0, 0, 5.0, 0});
+        data.put(5, new double[] {16.0, 0, 0, 0, 0, 6.0});
 
         KNNRegressionTrainer trainer = new KNNRegressionTrainer();
 
         KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
-            new LocalDatasetBuilder<>(data, 2),
+            new LocalDatasetBuilder<>(data, parts),
             (k, v) -> Arrays.copyOfRange(v, 1, v.length),
             (k, v) -> v[0]
         ).withK(1)
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector vector = new DenseLocalOnHeapVector(new double[]{0, 0, 0, 5.0, 0.0});
+        Vector vector = new DenseLocalOnHeapVector(new double[] {0, 0, 0, 5.0, 0.0});
         System.out.println(knnMdl.apply(vector));
         Assert.assertEquals(15, knnMdl.apply(vector), 1E-12);
     }
 
     /** */
     @Test
-    public void longly() {
+    public void testLongly() {
         Map<Integer, double[]> data = new HashMap<>();
-
-        data.put(0, new double[]{60323, 83.0, 234289, 2356, 1590, 107608, 1947});
-        data.put(1, new double[]{61122, 88.5, 259426, 2325, 1456, 108632, 1948});
-        data.put(2, new double[]{60171, 88.2, 258054, 3682, 1616, 109773, 1949});
-        data.put(3, new double[]{61187, 89.5, 284599, 3351, 1650, 110929, 1950});
-        data.put(4, new double[]{63221, 96.2, 328975, 2099, 3099, 112075, 1951});
-        data.put(5, new double[]{63639, 98.1, 346999, 1932, 3594, 113270, 1952});
-        data.put(6, new double[]{64989, 99.0, 365385, 1870, 3547, 115094, 1953});
-        data.put(7, new double[]{63761, 100.0, 363112, 3578, 3350, 116219, 1954});
-        data.put(8, new double[]{66019, 101.2, 397469, 2904, 3048, 117388, 1955});
-        data.put(9, new double[]{68169, 108.4, 442769, 2936, 2798, 120445, 1957});
-        data.put(10, new double[]{66513, 110.8, 444546, 4681, 2637, 121950, 1958});
-        data.put(11, new double[]{68655, 112.6, 482704, 3813, 2552, 123366, 1959});
-        data.put(12, new double[]{69564, 114.2, 502601, 3931, 2514, 125368, 1960});
-        data.put(13, new double[]{69331, 115.7, 518173, 4806, 2572, 127852, 1961});
-        data.put(14, new double[]{70551, 116.9, 554894, 4007, 2827, 130081, 1962});
+        data.put(0, new double[] {60323, 83.0, 234289, 2356, 1590, 107608, 1947});
+        data.put(1, new double[] {61122, 88.5, 259426, 2325, 1456, 108632, 1948});
+        data.put(2, new double[] {60171, 88.2, 258054, 3682, 1616, 109773, 1949});
+        data.put(3, new double[] {61187, 89.5, 284599, 3351, 1650, 110929, 1950});
+        data.put(4, new double[] {63221, 96.2, 328975, 2099, 3099, 112075, 1951});
+        data.put(5, new double[] {63639, 98.1, 346999, 1932, 3594, 113270, 1952});
+        data.put(6, new double[] {64989, 99.0, 365385, 1870, 3547, 115094, 1953});
+        data.put(7, new double[] {63761, 100.0, 363112, 3578, 3350, 116219, 1954});
+        data.put(8, new double[] {66019, 101.2, 397469, 2904, 3048, 117388, 1955});
+        data.put(9, new double[] {68169, 108.4, 442769, 2936, 2798, 120445, 1957});
+        data.put(10, new double[] {66513, 110.8, 444546, 4681, 2637, 121950, 1958});
+        data.put(11, new double[] {68655, 112.6, 482704, 3813, 2552, 123366, 1959});
+        data.put(12, new double[] {69564, 114.2, 502601, 3931, 2514, 125368, 1960});
+        data.put(13, new double[] {69331, 115.7, 518173, 4806, 2572, 127852, 1961});
+        data.put(14, new double[] {70551, 116.9, 554894, 4007, 2827, 130081, 1962});
 
         KNNRegressionTrainer trainer = new KNNRegressionTrainer();
 
         KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
-            new LocalDatasetBuilder<>(data, 2),
+            new LocalDatasetBuilder<>(data, parts),
             (k, v) -> Arrays.copyOfRange(v, 1, v.length),
             (k, v) -> v[0]
         ).withK(3)
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector vector = new DenseLocalOnHeapVector(new double[]{104.6, 419180, 2822, 2857, 118734, 1956});
+        Vector vector = new DenseLocalOnHeapVector(new double[] {104.6, 419180, 2822, 2857, 118734, 1956});
         System.out.println(knnMdl.apply(vector));
         Assert.assertEquals(67857, knnMdl.apply(vector), 2000);
     }
 
     /** */
+    @Test
     public void testLonglyWithWeightedStrategy() {
         Map<Integer, double[]> data = new HashMap<>();
-
-        data.put(0, new double[]{60323, 83.0, 234289, 2356, 1590, 107608, 1947});
-        data.put(1, new double[]{61122, 88.5, 259426, 2325, 1456, 108632, 1948});
-        data.put(2, new double[]{60171, 88.2, 258054, 3682, 1616, 109773, 1949});
-        data.put(3, new double[]{61187, 89.5, 284599, 3351, 1650, 110929, 1950});
-        data.put(4, new double[]{63221, 96.2, 328975, 2099, 3099, 112075, 1951});
-        data.put(5, new double[]{63639, 98.1, 346999, 1932, 3594, 113270, 1952});
-        data.put(6, new double[]{64989, 99.0, 365385, 1870, 3547, 115094, 1953});
-        data.put(7, new double[]{63761, 100.0, 363112, 3578, 3350, 116219, 1954});
-        data.put(8, new double[]{66019, 101.2, 397469, 2904, 3048, 117388, 1955});
-        data.put(9, new double[]{68169, 108.4, 442769, 2936, 2798, 120445, 1957});
-        data.put(10, new double[]{66513, 110.8, 444546, 4681, 2637, 121950, 1958});
-        data.put(11, new double[]{68655, 112.6, 482704, 3813, 2552, 123366, 1959});
-        data.put(12, new double[]{69564, 114.2, 502601, 3931, 2514, 125368, 1960});
-        data.put(13, new double[]{69331, 115.7, 518173, 4806, 2572, 127852, 1961});
-        data.put(14, new double[]{70551, 116.9, 554894, 4007, 2827, 130081, 1962});
+        data.put(0, new double[] {60323, 83.0, 234289, 2356, 1590, 107608, 1947});
+        data.put(1, new double[] {61122, 88.5, 259426, 2325, 1456, 108632, 1948});
+        data.put(2, new double[] {60171, 88.2, 258054, 3682, 1616, 109773, 1949});
+        data.put(3, new double[] {61187, 89.5, 284599, 3351, 1650, 110929, 1950});
+        data.put(4, new double[] {63221, 96.2, 328975, 2099, 3099, 112075, 1951});
+        data.put(5, new double[] {63639, 98.1, 346999, 1932, 3594, 113270, 1952});
+        data.put(6, new double[] {64989, 99.0, 365385, 1870, 3547, 115094, 1953});
+        data.put(7, new double[] {63761, 100.0, 363112, 3578, 3350, 116219, 1954});
+        data.put(8, new double[] {66019, 101.2, 397469, 2904, 3048, 117388, 1955});
+        data.put(9, new double[] {68169, 108.4, 442769, 2936, 2798, 120445, 1957});
+        data.put(10, new double[] {66513, 110.8, 444546, 4681, 2637, 121950, 1958});
+        data.put(11, new double[] {68655, 112.6, 482704, 3813, 2552, 123366, 1959});
+        data.put(12, new double[] {69564, 114.2, 502601, 3931, 2514, 125368, 1960});
+        data.put(13, new double[] {69331, 115.7, 518173, 4806, 2572, 127852, 1961});
+        data.put(14, new double[] {70551, 116.9, 554894, 4007, 2827, 130081, 1962});
 
         KNNRegressionTrainer trainer = new KNNRegressionTrainer();
 
         KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
-            new LocalDatasetBuilder<>(data, 2),
+            new LocalDatasetBuilder<>(data, parts),
             (k, v) -> Arrays.copyOfRange(v, 1, v.length),
             (k, v) -> v[0]
         ).withK(3)
             .withDistanceMeasure(new EuclideanDistance())
             .withStrategy(KNNStrategy.SIMPLE);
 
-        Vector vector = new DenseLocalOnHeapVector(new double[]{104.6, 419180, 2822, 2857, 118734, 1956});
+        Vector vector = new DenseLocalOnHeapVector(new double[] {104.6, 419180, 2822, 2857, 118734, 1956});
         System.out.println(knnMdl.apply(vector));
         Assert.assertEquals(67857, knnMdl.apply(vector), 2000);
     }
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetHelper.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetHelper.java
index a25b303480780..dbcdb99e62b34 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetHelper.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetHelper.java
@@ -21,64 +21,33 @@
 import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import org.apache.ignite.Ignite;
 import org.apache.ignite.ml.structures.LabeledDataset;
 import org.apache.ignite.ml.structures.preprocessing.LabeledDatasetLoader;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
  * Base class for decision trees test.
  */
-public class LabeledDatasetHelper extends GridCommonAbstractTest {
-    /** Count of nodes. */
-    private static final int NODE_COUNT = 4;
-
+public class LabeledDatasetHelper {
     /** Separator. */
     private static final String SEPARATOR = "\t";
 
-    /** Grid instance. */
-    protected Ignite ignite;
-
-    /**
-     * Default constructor.
-     */
-    public LabeledDatasetHelper() {
-        super(false);
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override protected void beforeTest() throws Exception {
-        ignite = grid(NODE_COUNT);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        for (int i = 1; i <= NODE_COUNT; i++)
-            startGrid(i);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        stopAllGrids();
-    }
-
     /**
      * Loads labeled dataset from file with .txt extension.
      *
      * @param rsrcPath path to dataset.
      * @return null if path is incorrect.
      */
-    LabeledDataset loadDatasetFromTxt(String rsrcPath, boolean isFallOnBadData) {
+    public static LabeledDataset loadDatasetFromTxt(String rsrcPath, boolean isFallOnBadData) {
         try {
-            Path path = Paths.get(this.getClass().getClassLoader().getResource(rsrcPath).toURI());
+            Path path = Paths.get(LabeledDatasetHelper.class.getClassLoader().getResource(rsrcPath).toURI());
             try {
                 return LabeledDatasetLoader.loadFromTxtFile(path, SEPARATOR, false, isFallOnBadData);
-            } catch (IOException e) {
+            }
+            catch (IOException e) {
                 e.printStackTrace();
             }
-        } catch (URISyntaxException e) {
+        }
+        catch (URISyntaxException e) {
             e.printStackTrace();
             return null;
         }
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java
index 77d40a61a0d4f..e986740a378c9 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/knn/LabeledDatasetTest.java
@@ -21,7 +21,6 @@
 import java.net.URISyntaxException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.ml.math.ExternalizableTest;
 import org.apache.ignite.ml.math.Vector;
 import org.apache.ignite.ml.math.exceptions.CardinalityException;
@@ -32,9 +31,13 @@
 import org.apache.ignite.ml.structures.LabeledDatasetTestTrainPair;
 import org.apache.ignite.ml.structures.LabeledVector;
 import org.apache.ignite.ml.structures.preprocessing.LabeledDatasetLoader;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.fail;
 
 /** Tests behaviour of KNNClassificationTest. */
-public class LabeledDatasetTest extends LabeledDatasetHelper implements ExternalizableTest<LabeledDataset> {
+public class LabeledDatasetTest implements ExternalizableTest<LabeledDataset> {
     /** */
     private static final String KNN_IRIS_TXT = "datasets/knn/iris.txt";
 
@@ -51,9 +54,8 @@ public class LabeledDatasetTest extends LabeledDatasetHelper implements External
     private static final String IRIS_MISSED_DATA = "datasets/knn/missed_data.txt";
 
     /** */
+    @Test
     public void testFeatureNames() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         double[][] mtx =
             new double[][] {
                 {1.0, 1.0},
@@ -71,9 +73,8 @@ public void testFeatureNames() {
     }
 
     /** */
+    @Test
     public void testAccessMethods() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         double[][] mtx =
             new double[][] {
                 {1.0, 1.0},
@@ -98,9 +99,8 @@ public void testAccessMethods() {
     }
 
     /** */
+    @Test
     public void testFailOnYNull() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         double[][] mtx =
             new double[][] {
                 {1.0, 1.0},
@@ -122,9 +122,8 @@ public void testFailOnYNull() {
     }
 
     /** */
+    @Test
     public void testFailOnXNull() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         double[][] mtx =
             new double[][] {};
         double[] lbs = new double[] {1.0, 1.0, 1.0, 2.0, 2.0, 2.0};
@@ -140,18 +139,17 @@ public void testFailOnXNull() {
     }
 
     /** */
+    @Test
     public void testLoadingCorrectTxtFile() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        LabeledDataset training = loadDatasetFromTxt(KNN_IRIS_TXT, false);
+        LabeledDataset training = LabeledDatasetHelper.loadDatasetFromTxt(KNN_IRIS_TXT, false);
         assertEquals(training.rowSize(), 150);
     }
 
     /** */
+    @Test
     public void testLoadingEmptyFile() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         try {
-            loadDatasetFromTxt(EMPTY_TXT, false);
+            LabeledDatasetHelper.loadDatasetFromTxt(EMPTY_TXT, false);
             fail("EmptyFileException");
         }
         catch (EmptyFileException e) {
@@ -161,11 +159,10 @@ public void testLoadingEmptyFile() {
     }
 
     /** */
+    @Test
     public void testLoadingFileWithFirstEmptyRow() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         try {
-            loadDatasetFromTxt(NO_DATA_TXT, false);
+            LabeledDatasetHelper.loadDatasetFromTxt(NO_DATA_TXT, false);
             fail("NoDataException");
         }
         catch (NoDataException e) {
@@ -175,19 +172,17 @@ public void testLoadingFileWithFirstEmptyRow() {
     }
 
     /** */
+    @Test
     public void testLoadingFileWithIncorrectData() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        LabeledDataset training = loadDatasetFromTxt(IRIS_INCORRECT_TXT, false);
+        LabeledDataset training = LabeledDatasetHelper.loadDatasetFromTxt(IRIS_INCORRECT_TXT, false);
         assertEquals(149, training.rowSize());
     }
 
     /** */
+    @Test
     public void testFailOnLoadingFileWithIncorrectData() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         try {
-            loadDatasetFromTxt(IRIS_INCORRECT_TXT, true);
+            LabeledDatasetHelper.loadDatasetFromTxt(IRIS_INCORRECT_TXT, true);
             fail("FileParsingException");
         }
         catch (FileParsingException e) {
@@ -198,9 +193,8 @@ public void testFailOnLoadingFileWithIncorrectData() {
     }
 
     /** */
+    @Test
     public void testLoadingFileWithMissedData() throws URISyntaxException, IOException {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         Path path = Paths.get(this.getClass().getClassLoader().getResource(IRIS_MISSED_DATA).toURI());
 
         LabeledDataset training = LabeledDatasetLoader.loadFromTxtFile(path, ",", false, false);
@@ -209,9 +203,8 @@ public void testLoadingFileWithMissedData() throws URISyntaxException, IOExcepti
     }
 
     /** */
+    @Test
     public void testSplitting() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         double[][] mtx =
             new double[][] {
                 {1.0, 1.0},
@@ -246,9 +239,8 @@ public void testSplitting() {
     }
 
     /** */
+    @Test
     public void testLabels() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         double[][] mtx =
             new double[][] {
                 {1.0, 1.0},
@@ -267,8 +259,6 @@ public void testLabels() {
 
     /** */
     @Override public void testExternalization() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
         double[][] mtx =
             new double[][] {
                 {1.0, 1.0},

From a9ec54be9c51fe8ff04ca5d21160c27255640007 Mon Sep 17 00:00:00 2001
From: dmitrievanthony <dmitrievanthony@gmail.com>
Date: Fri, 13 Apr 2018 18:08:08 +0300
Subject: [PATCH 051/543] IGNITE-8232: ML package cleanup for 2.5 release

this closes #3806

(cherry picked from commit 47cfdc2)
---
 .../examples/ml/nn/MLPTrainerExample.java     |   2 +-
 ...dLinearRegressionWithQRTrainerExample.java | 137 ------
 ...> LinearRegressionLSQRTrainerExample.java} |   4 +-
 ...nLSQRTrainerWithNormalizationExample.java} |   2 +-
 ...=> LinearRegressionSGDTrainerExample.java} |   9 +-
 .../java/org/apache/ignite/ml/Trainer.java    |  36 --
 .../ignite/ml/estimators/Estimators.java      |  50 ---
 .../ignite/ml/estimators/package-info.java    |  22 -
 .../ml/math/functions/IgniteBiFunction.java   |   8 +-
 .../LinSysPartitionDataBuilderOnHeap.java     |  86 ----
 .../isolve/LinSysPartitionDataOnHeap.java     |  65 ---
 .../ml/math/isolve/lsqr/AbstractLSQR.java     |   3 +-
 .../ml/math/isolve/lsqr/LSQROnHeap.java       |  27 +-
 .../org/apache/ignite/ml/nn/MLPTrainer.java   |   1 -
 .../group => nn}/UpdatesStrategy.java         |   2 +-
 .../ml/optimization/GradientDescent.java      | 202 ---------
 .../ml/optimization/GradientFunction.java     |  31 --
 .../LeastSquaresGradientFunction.java         |  33 --
 .../SparseDistributedMatrixMapReducer.java    |  84 ----
 .../ml/optimization/util/package-info.java    |  22 -
 .../linear/LinearRegressionLSQRTrainer.java   |  10 +-
 .../linear/LinearRegressionQRTrainer.java     |  72 ----
 .../linear/LinearRegressionSGDTrainer.java    |   7 +-
 .../apache/ignite/ml/trainers/Trainer.java    |  33 --
 .../trainers/group/BaseLocalProcessorJob.java | 146 -------
 .../ignite/ml/trainers/group/ConstModel.java  |  46 --
 .../ml/trainers/group/GroupTrainer.java       | 208 ---------
 .../group/GroupTrainerBaseProcessorTask.java  | 144 -------
 .../trainers/group/GroupTrainerCacheKey.java  | 125 ------
 .../GroupTrainerEntriesProcessorTask.java     |  64 ---
 .../ml/trainers/group/GroupTrainerInput.java  |  37 --
 .../group/GroupTrainerKeysProcessorTask.java  |  62 ---
 .../trainers/group/GroupTrainingContext.java  |  98 -----
 .../group/LocalEntriesProcessorJob.java       |  85 ----
 .../trainers/group/LocalKeysProcessorJob.java |  78 ----
 .../ml/trainers/group/Metaoptimizer.java      |  93 ----
 .../group/MetaoptimizerDistributedStep.java   |  97 -----
 .../group/MetaoptimizerGroupTrainer.java      | 132 ------
 .../ml/trainers/group/ResultAndUpdates.java   | 178 --------
 .../ml/trainers/group/UpdateStrategies.java   |  47 --
 .../ml/trainers/group/chain/Chains.java       |  56 ---
 .../group/chain/ComputationsChain.java        | 246 -----------
 .../chain/DistributedEntryProcessingStep.java |  34 --
 .../chain/DistributedKeyProcessingStep.java   |  33 --
 .../trainers/group/chain/DistributedStep.java |  70 ---
 .../trainers/group/chain/EntryAndContext.java |  70 ---
 .../trainers/group/chain/HasTrainingUUID.java |  32 --
 .../trainers/group/chain/KeyAndContext.java   |  67 ---
 .../ml/trainers/group/chain/package-info.java |  22 -
 .../ml/trainers/group/package-info.java       |  22 -
 .../apache/ignite/ml/IgniteMLTestSuite.java   |   4 -
 .../ml/math/isolve/lsqr/LSQROnHeapTest.java   |  14 +-
 .../ml/nn/MLPTrainerIntegrationTest.java      |   1 -
 .../apache/ignite/ml/nn/MLPTrainerTest.java   |   1 -
 .../MLPTrainerMnistIntegrationTest.java       |   2 +-
 .../nn/performance/MLPTrainerMnistTest.java   |   2 +-
 .../ml/optimization/GradientDescentTest.java  |  64 ---
 .../optimization/OptimizationTestSuite.java   |  33 --
 ...SparseDistributedMatrixMapReducerTest.java | 135 ------
 .../ml/regressions/RegressionsTestSuite.java  |   3 -
 .../linear/ArtificialRegressionDatasets.java  | 404 ------------------
 ...tributedLinearRegressionQRTrainerTest.java |  36 --
 ...tributedLinearRegressionQRTrainerTest.java |  36 --
 .../GenericLinearRegressionTrainerTest.java   | 206 ---------
 ...reAbstractLinearRegressionTrainerTest.java | 127 ------
 .../LinearRegressionSGDTrainerTest.java       |   2 +-
 .../LocalLinearRegressionQRTrainerTest.java   |  36 --
 .../group/DistributedWorkersChainTest.java    | 189 --------
 .../ml/trainers/group/GroupTrainerTest.java   |  90 ----
 .../group/SimpleGroupTrainerInput.java        |  63 ---
 .../ml/trainers/group/TestGroupTrainer.java   | 144 -------
 .../group/TestGroupTrainerLocalContext.java   |  85 ----
 .../group/TestGroupTrainingCache.java         |  70 ---
 .../group/TestGroupTrainingSecondCache.java   |  56 ---
 .../ml/trainers/group/TestLocalContext.java   |  51 ---
 .../trainers/group/TestTrainingLoopStep.java  |  65 ---
 .../group/TrainersGroupTestSuite.java         |  32 --
 ...eOLSMultipleLinearRegressionBenchmark.java |  69 ---
 .../yardstick/ml/regression/package-info.java |  22 -
 79 files changed, 54 insertions(+), 5228 deletions(-)
 delete mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java
 rename examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/{DistributedLinearRegressionWithLSQRTrainerExample.java => LinearRegressionLSQRTrainerExample.java} (97%)
 rename examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/{DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java => LinearRegressionLSQRTrainerWithNormalizationExample.java} (99%)
 rename examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/{DistributedLinearRegressionWithSGDTrainerExample.java => LinearRegressionSGDTrainerExample.java} (95%)
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/estimators/package-info.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/LinSysPartitionDataBuilderOnHeap.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/LinSysPartitionDataOnHeap.java
 rename modules/ml/src/main/java/org/apache/ignite/ml/{trainers/group => nn}/UpdatesStrategy.java (98%)
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientDescent.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientFunction.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/optimization/LeastSquaresGradientFunction.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/optimization/util/SparseDistributedMatrixMapReducer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/optimization/util/package-info.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionQRTrainer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/Trainer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ConstModel.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerBaseProcessorTask.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerCacheKey.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerEntriesProcessorTask.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerInput.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerKeysProcessorTask.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainingContext.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/LocalEntriesProcessorJob.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/LocalKeysProcessorJob.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/Metaoptimizer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/MetaoptimizerDistributedStep.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/MetaoptimizerGroupTrainer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ResultAndUpdates.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdateStrategies.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/Chains.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/ComputationsChain.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedEntryProcessingStep.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedKeyProcessingStep.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedStep.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/EntryAndContext.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/HasTrainingUUID.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/KeyAndContext.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/package-info.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/package-info.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/optimization/GradientDescentTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/optimization/OptimizationTestSuite.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/optimization/util/SparseDistributedMatrixMapReducerTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/ArtificialRegressionDatasets.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionQRTrainerTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionQRTrainerTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GenericLinearRegressionTrainerTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionQRTrainerTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/DistributedWorkersChainTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/GroupTrainerTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/SimpleGroupTrainerInput.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainer.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainerLocalContext.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainingCache.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainingSecondCache.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestLocalContext.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestTrainingLoopStep.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TrainersGroupTestSuite.java
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/regression/IgniteOLSMultipleLinearRegressionBenchmark.java
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/regression/package-info.java

diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
index ce44cc64b935e..5d1ac38a4d3cb 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
@@ -32,7 +32,7 @@
 import org.apache.ignite.ml.optimization.LossFunctions;
 import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
 import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
-import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
+import org.apache.ignite.ml.nn.UpdatesStrategy;
 import org.apache.ignite.thread.IgniteThread;
 
 /**
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java
deleted file mode 100644
index 98d5e4e84ced1..0000000000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithQRTrainerExample.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.regression.linear;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionQRTrainer;
-import org.apache.ignite.thread.IgniteThread;
-
-import java.util.Arrays;
-
-/**
- * Run linear regression model over distributed matrix.
- *
- * @see LinearRegressionQRTrainer
- */
-public class DistributedLinearRegressionWithQRTrainerExample {
-    /** */
-    private static final double[][] data = {
-        {8, 78, 284, 9.100000381, 109},
-        {9.300000191, 68, 433, 8.699999809, 144},
-        {7.5, 70, 739, 7.199999809, 113},
-        {8.899999619, 96, 1792, 8.899999619, 97},
-        {10.19999981, 74, 477, 8.300000191, 206},
-        {8.300000191, 111, 362, 10.89999962, 124},
-        {8.800000191, 77, 671, 10, 152},
-        {8.800000191, 168, 636, 9.100000381, 162},
-        {10.69999981, 82, 329, 8.699999809, 150},
-        {11.69999981, 89, 634, 7.599999905, 134},
-        {8.5, 149, 631, 10.80000019, 292},
-        {8.300000191, 60, 257, 9.5, 108},
-        {8.199999809, 96, 284, 8.800000191, 111},
-        {7.900000095, 83, 603, 9.5, 182},
-        {10.30000019, 130, 686, 8.699999809, 129},
-        {7.400000095, 145, 345, 11.19999981, 158},
-        {9.600000381, 112, 1357, 9.699999809, 186},
-        {9.300000191, 131, 544, 9.600000381, 177},
-        {10.60000038, 80, 205, 9.100000381, 127},
-        {9.699999809, 130, 1264, 9.199999809, 179},
-        {11.60000038, 140, 688, 8.300000191, 80},
-        {8.100000381, 154, 354, 8.399999619, 103},
-        {9.800000191, 118, 1632, 9.399999619, 101},
-        {7.400000095, 94, 348, 9.800000191, 117},
-        {9.399999619, 119, 370, 10.39999962, 88},
-        {11.19999981, 153, 648, 9.899999619, 78},
-        {9.100000381, 116, 366, 9.199999809, 102},
-        {10.5, 97, 540, 10.30000019, 95},
-        {11.89999962, 176, 680, 8.899999619, 80},
-        {8.399999619, 75, 345, 9.600000381, 92},
-        {5, 134, 525, 10.30000019, 126},
-        {9.800000191, 161, 870, 10.39999962, 108},
-        {9.800000191, 111, 669, 9.699999809, 77},
-        {10.80000019, 114, 452, 9.600000381, 60},
-        {10.10000038, 142, 430, 10.69999981, 71},
-        {10.89999962, 238, 822, 10.30000019, 86},
-        {9.199999809, 78, 190, 10.69999981, 93},
-        {8.300000191, 196, 867, 9.600000381, 106},
-        {7.300000191, 125, 969, 10.5, 162},
-        {9.399999619, 82, 499, 7.699999809, 95},
-        {9.399999619, 125, 925, 10.19999981, 91},
-        {9.800000191, 129, 353, 9.899999619, 52},
-        {3.599999905, 84, 288, 8.399999619, 110},
-        {8.399999619, 183, 718, 10.39999962, 69},
-        {10.80000019, 119, 540, 9.199999809, 57},
-        {10.10000038, 180, 668, 13, 106},
-        {9, 82, 347, 8.800000191, 40},
-        {10, 71, 345, 9.199999809, 50},
-        {11.30000019, 118, 463, 7.800000191, 35},
-        {11.30000019, 121, 728, 8.199999809, 86},
-        {12.80000019, 68, 383, 7.400000095, 57},
-        {10, 112, 316, 10.39999962, 57},
-        {6.699999809, 109, 388, 8.899999619, 94}
-    };
-
-    /** Run example. */
-    public static void main(String[] args) throws InterruptedException {
-        System.out.println();
-        System.out.println(">>> Linear regression model over sparse distributed matrix API usage example started.");
-        // Start ignite grid.
-        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
-            System.out.println(">>> Ignite grid started.");
-            // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
-            // because we create ignite cache internally.
-            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                SparseDistributedMatrixExample.class.getSimpleName(), () -> {
-
-                // Create SparseDistributedMatrix, new cache will be created automagically.
-                System.out.println(">>> Create new SparseDistributedMatrix inside IgniteThread.");
-                SparseDistributedMatrix distributedMatrix = new SparseDistributedMatrix(data);
-
-                System.out.println(">>> Create new linear regression trainer object.");
-                Trainer<LinearRegressionModel, Matrix> trainer = new LinearRegressionQRTrainer();
-
-                System.out.println(">>> Perform the training to get the model.");
-                LinearRegressionModel mdl = trainer.train(distributedMatrix);
-                System.out.println(">>> Linear regression model: " + mdl);
-
-                System.out.println(">>> ---------------------------------");
-                System.out.println(">>> | Prediction\t| Ground Truth\t|");
-                System.out.println(">>> ---------------------------------");
-                for (double[] observation : data) {
-                    Vector inputs = new SparseDistributedVector(Arrays.copyOfRange(observation, 1, observation.length));
-                    double prediction = mdl.apply(inputs);
-                    double groundTruth = observation[0];
-                    System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
-                }
-                System.out.println(">>> ---------------------------------");
-            });
-
-            igniteThread.start();
-
-            igniteThread.join();
-        }
-    }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java
similarity index 97%
rename from examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java
rename to examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java
index 25aec0cbba400..276d43fcd67ef 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java
@@ -38,7 +38,7 @@
  *
  * @see LinearRegressionLSQRTrainer
  */
-public class DistributedLinearRegressionWithLSQRTrainerExample {
+public class LinearRegressionLSQRTrainerExample {
     /** */
     private static final double[][] data = {
         {8, 78, 284, 9.100000381, 109},
@@ -107,7 +107,7 @@ public static void main(String[] args) throws InterruptedException {
             // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
             // because we create ignite cache internally.
             IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                DistributedLinearRegressionWithLSQRTrainerExample.class.getSimpleName(), () -> {
+                LinearRegressionLSQRTrainerExample.class.getSimpleName(), () -> {
                 IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
 
                 System.out.println(">>> Create new linear regression trainer object.");
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithNormalizationExample.java
similarity index 99%
rename from examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
rename to examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithNormalizationExample.java
index 99e657781b131..0358f44135e00 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithNormalizationExample.java
@@ -44,7 +44,7 @@
  * @see NormalizationTrainer
  * @see NormalizationPreprocessor
  */
-public class DistributedLinearRegressionWithLSQRTrainerAndNormalizationExample {
+public class LinearRegressionLSQRTrainerWithNormalizationExample {
     /** */
     private static final double[][] data = {
         {8, 78, 284, 9.100000381, 109},
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java
similarity index 95%
rename from examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java
rename to examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java
index 44366e1aafc33..ce6ad3b4df287 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/DistributedLinearRegressionWithSGDTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java
@@ -28,9 +28,8 @@
 import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
 import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
 import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionQRTrainer;
 import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
-import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
+import org.apache.ignite.ml.nn.UpdatesStrategy;
 import org.apache.ignite.thread.IgniteThread;
 
 import javax.cache.Cache;
@@ -40,9 +39,9 @@
 /**
  * Run linear regression model over distributed matrix.
  *
- * @see LinearRegressionQRTrainer
+ * @see LinearRegressionSGDTrainer
  */
-public class DistributedLinearRegressionWithSGDTrainerExample {
+public class LinearRegressionSGDTrainerExample {
     /** */
     private static final double[][] data = {
         {8, 78, 284, 9.100000381, 109},
@@ -110,7 +109,7 @@ public static void main(String[] args) throws InterruptedException {
             // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
             // because we create ignite cache internally.
             IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                DistributedLinearRegressionWithSGDTrainerExample.class.getSimpleName(), () -> {
+                LinearRegressionSGDTrainerExample.class.getSimpleName(), () -> {
 
                 IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
deleted file mode 100644
index f53b80110cb4c..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/Trainer.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml;
-
-/**
- * Interface for Trainers. Trainer is just a function which produces model from the data.
- *
- * @param <M> Type of produced model.
- * @param <T> Type of data needed for model producing.
- */
-// TODO: IGNITE-7659: Reduce multiple Trainer interfaces to one
-@Deprecated
-public interface Trainer<M extends Model, T> {
-    /**
-     * Returns model based on data
-     *
-     * @param data data to build model
-     * @return model
-     */
-    M train(T data);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java b/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java
deleted file mode 100644
index b2731ffd0879e..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/estimators/Estimators.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.estimators;
-
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Function;
-import java.util.stream.Stream;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.functions.IgniteTriFunction;
-
-/** Estimators. */
-public class Estimators {
-    /** Simple implementation of mean squared error estimator. */
-    public static <T, V> IgniteTriFunction<Model<T, V>, Stream<IgniteBiTuple<T, V>>, Function<V, Double>, Double> MSE() {
-        return (model, stream, f) -> stream.mapToDouble(dp -> {
-            double diff = f.apply(dp.get2()) - f.apply(model.apply(dp.get1()));
-            return diff * diff;
-        }).average().orElse(0);
-    }
-
-    /** Simple implementation of errors percentage estimator. */
-    public static <T, V> IgniteTriFunction<Model<T, V>, Stream<IgniteBiTuple<T, V>>, Function<V, Double>, Double> errorsPercentage() {
-        return (model, stream, f) -> {
-            AtomicLong total = new AtomicLong(0);
-
-            long cnt = stream.
-                peek((ib) -> total.incrementAndGet()).
-                filter(dp -> !model.apply(dp.get1()).equals(dp.get2())).
-                count();
-
-            return (double)cnt / total.get();
-        };
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/estimators/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/estimators/package-info.java
deleted file mode 100644
index c03827f3e32e0..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/estimators/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains estimation algorithms.
- */
-package org.apache.ignite.ml.estimators;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteBiFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteBiFunction.java
index dc49739426881..45fd035710e79 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteBiFunction.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteBiFunction.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.ml.math.functions;
 
 import java.io.Serializable;
+import java.util.Objects;
 import java.util.function.BiFunction;
 
 /**
@@ -25,5 +26,10 @@
  *
  * @see java.util.function.BiFunction
  */
-public interface IgniteBiFunction<A, B, T> extends BiFunction<A, B, T>, Serializable {
+public interface IgniteBiFunction<T, U, R> extends BiFunction<T, U, R>, Serializable {
+    /** {@inheritDoc} */
+    default <V> IgniteBiFunction<T, U, V> andThen(IgniteFunction<? super R, ? extends V> after) {
+        Objects.requireNonNull(after);
+        return (T t, U u) -> after.apply(apply(t, u));
+    }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/LinSysPartitionDataBuilderOnHeap.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/LinSysPartitionDataBuilderOnHeap.java
deleted file mode 100644
index e80b935a165ca..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/LinSysPartitionDataBuilderOnHeap.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.math.isolve;
-
-import java.io.Serializable;
-import java.util.Iterator;
-import org.apache.ignite.ml.dataset.PartitionDataBuilder;
-import org.apache.ignite.ml.dataset.UpstreamEntry;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-
-/**
- * Linear system partition data builder that builds {@link LinSysPartitionDataOnHeap}.
- *
- * @param <K> Type of a key in <tt>upstream</tt> data.
- * @param <V> Type of a value in <tt>upstream</tt> data.
- * @param <C> Type of a partition <tt>context</tt>.
- */
-public class LinSysPartitionDataBuilderOnHeap<K, V, C extends Serializable>
-    implements PartitionDataBuilder<K, V, C, LinSysPartitionDataOnHeap> {
-    /** */
-    private static final long serialVersionUID = -7820760153954269227L;
-
-    /** Extractor of X matrix row. */
-    private final IgniteBiFunction<K, V, double[]> xExtractor;
-
-    /** Extractor of Y vector value. */
-    private final IgniteBiFunction<K, V, Double> yExtractor;
-
-    /**
-     * Constructs a new instance of linear system partition data builder.
-     *
-     * @param xExtractor Extractor of X matrix row.
-     * @param yExtractor Extractor of Y vector value.
-     */
-    public LinSysPartitionDataBuilderOnHeap(IgniteBiFunction<K, V, double[]> xExtractor,
-        IgniteBiFunction<K, V, Double> yExtractor) {
-        this.xExtractor = xExtractor;
-        this.yExtractor = yExtractor;
-    }
-
-    /** {@inheritDoc} */
-    @Override public LinSysPartitionDataOnHeap build(Iterator<UpstreamEntry<K, V>> upstreamData, long upstreamDataSize,
-        C ctx) {
-        // Prepares the matrix of features in flat column-major format.
-        int xCols = -1;
-        double[] x = null;//new double[Math.toIntExact(upstreamDataSize * cols)];
-        double[] y = new double[Math.toIntExact(upstreamDataSize)];
-
-        int ptr = 0;
-        while (upstreamData.hasNext()) {
-            UpstreamEntry<K, V> entry = upstreamData.next();
-            double[] row = xExtractor.apply(entry.getKey(), entry.getValue());
-
-            if (xCols < 0) {
-                xCols = row.length;
-                x = new double[Math.toIntExact(upstreamDataSize * xCols)];
-            }
-            else
-                assert row.length == xCols : "X extractor must return exactly " + xCols + " columns";
-
-            for (int i = 0; i < xCols; i++)
-                x[Math.toIntExact(i * upstreamDataSize) + ptr] = row[i];
-
-            y[ptr] = yExtractor.apply(entry.getKey(), entry.getValue());
-
-            ptr++;
-        }
-
-        return new LinSysPartitionDataOnHeap(x, y, Math.toIntExact(upstreamDataSize));
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/LinSysPartitionDataOnHeap.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/LinSysPartitionDataOnHeap.java
deleted file mode 100644
index 89c8e441f92fe..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/LinSysPartitionDataOnHeap.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.math.isolve;
-
-/**
- * On Heap partition data that keeps part of a linear system.
- */
-public class LinSysPartitionDataOnHeap implements AutoCloseable {
-    /** Part of X matrix. */
-    private final double[] x;
-
-    /** Part of Y vector. */
-    private final double[] y;
-
-    /** Number of rows. */
-    private final int rows;
-
-    /**
-     * Constructs a new instance of linear system partition data.
-     *
-     * @param x Part of X matrix.
-     * @param y Part of Y vector.
-     * @param rows Number of rows.
-     */
-    public LinSysPartitionDataOnHeap(double[] x, double[] y, int rows) {
-        this.x = x;
-        this.rows = rows;
-        this.y = y;
-    }
-
-    /** */
-    public double[] getX() {
-        return x;
-    }
-
-    /** */
-    public int getRows() {
-        return rows;
-    }
-
-    /** */
-    public double[] getY() {
-        return y;
-    }
-
-    /** {@inheritDoc} */
-    @Override public void close() {
-        // Do nothing, GC will clean up.
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/AbstractLSQR.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/AbstractLSQR.java
index 8d190cd74373a..d1d3219f2b1f5 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/AbstractLSQR.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/AbstractLSQR.java
@@ -19,6 +19,7 @@
 
 import com.github.fommil.netlib.BLAS;
 import java.util.Arrays;
+import org.apache.ignite.ml.math.Precision;
 
 /**
  * Basic implementation of the LSQR algorithm without assumptions about dataset storage format or data processing
@@ -30,7 +31,7 @@
 // TODO: IGNITE-7660: Refactor LSQR algorithm
 public abstract class AbstractLSQR {
     /** The smallest representable positive number such that 1.0 + eps != 1.0. */
-    private static final double eps = Double.longBitsToDouble(Double.doubleToLongBits(1.0) | 1) - 1.0;
+    private static final double eps = Precision.EPSILON;
 
     /** BLAS (Basic Linear Algebra Subprograms) instance. */
     private static BLAS blas = BLAS.getInstance();
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java
index b1cc4c957e1b1..e138cf3ff3db9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeap.java
@@ -22,14 +22,14 @@
 import org.apache.ignite.ml.dataset.Dataset;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
 import org.apache.ignite.ml.dataset.PartitionDataBuilder;
-import org.apache.ignite.ml.math.isolve.LinSysPartitionDataOnHeap;
+import org.apache.ignite.ml.dataset.primitive.data.SimpleLabeledDatasetData;
 
 /**
  * Distributed implementation of LSQR algorithm based on {@link AbstractLSQR} and {@link Dataset}.
  */
 public class LSQROnHeap<K, V> extends AbstractLSQR implements AutoCloseable {
     /** Dataset. */
-    private final Dataset<LSQRPartitionContext, LinSysPartitionDataOnHeap> dataset;
+    private final Dataset<LSQRPartitionContext, SimpleLabeledDatasetData> dataset;
 
     /**
      * Constructs a new instance of OnHeap LSQR algorithm implementation.
@@ -38,7 +38,7 @@ public class LSQROnHeap<K, V> extends AbstractLSQR implements AutoCloseable {
      * @param partDataBuilder Partition data builder.
      */
     public LSQROnHeap(DatasetBuilder<K, V> datasetBuilder,
-        PartitionDataBuilder<K, V, LSQRPartitionContext, LinSysPartitionDataOnHeap> partDataBuilder) {
+        PartitionDataBuilder<K, V, LSQRPartitionContext, SimpleLabeledDatasetData> partDataBuilder) {
         this.dataset = datasetBuilder.build(
             (upstream, upstreamSize) -> new LSQRPartitionContext(),
             partDataBuilder
@@ -48,20 +48,20 @@ public LSQROnHeap(DatasetBuilder<K, V> datasetBuilder,
     /** {@inheritDoc} */
     @Override protected double bnorm() {
         return dataset.computeWithCtx((ctx, data) -> {
-            ctx.setU(Arrays.copyOf(data.getY(), data.getY().length));
+            ctx.setU(Arrays.copyOf(data.getLabels(), data.getLabels().length));
 
-            return BLAS.getInstance().dnrm2(data.getY().length, data.getY(), 1);
+            return BLAS.getInstance().dnrm2(data.getLabels().length, data.getLabels(), 1);
         }, (a, b) -> a == null ? b : b == null ? a : Math.sqrt(a * a + b * b));
     }
 
     /** {@inheritDoc} */
     @Override protected double beta(double[] x, double alfa, double beta) {
         return dataset.computeWithCtx((ctx, data) -> {
-            if (data.getX() == null)
+            if (data.getFeatures() == null)
                 return null;
 
-            int cols = data.getX().length / data.getRows();
-            BLAS.getInstance().dgemv("N", data.getRows(), cols, alfa, data.getX(),
+            int cols = data.getFeatures().length / data.getRows();
+            BLAS.getInstance().dgemv("N", data.getRows(), cols, alfa, data.getFeatures(),
                 Math.max(1, data.getRows()), x, 1, beta, ctx.getU(), 1);
 
             return BLAS.getInstance().dnrm2(ctx.getU().length, ctx.getU(), 1);
@@ -71,13 +71,13 @@ public LSQROnHeap(DatasetBuilder<K, V> datasetBuilder,
     /** {@inheritDoc} */
     @Override protected double[] iter(double bnorm, double[] target) {
         double[] res = dataset.computeWithCtx((ctx, data) -> {
-            if (data.getX() == null)
+            if (data.getFeatures() == null)
                 return null;
 
-            int cols =  data.getX().length / data.getRows();
+            int cols =  data.getFeatures().length / data.getRows();
             BLAS.getInstance().dscal(ctx.getU().length, 1 / bnorm, ctx.getU(), 1);
             double[] v = new double[cols];
-            BLAS.getInstance().dgemv("T", data.getRows(), cols, 1.0, data.getX(),
+            BLAS.getInstance().dgemv("T", data.getRows(), cols, 1.0, data.getFeatures(),
                 Math.max(1, data.getRows()), ctx.getU(), 1, 0, v, 1);
 
             return v;
@@ -101,7 +101,10 @@ else if (b == null)
      * @return number of columns
      */
     @Override protected int getColumns() {
-        return dataset.compute(data -> data.getX() == null ? null :  data.getX().length / data.getRows(), (a, b) -> a == null ? b : a);
+        return dataset.compute(
+            data -> data.getFeatures() == null ? null : data.getFeatures().length / data.getRows(),
+            (a, b) -> a == null ? b : a
+        );
     }
 
     /** {@inheritDoc} */
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
index fe955cbe696de..d12a276b2bc0e 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/MLPTrainer.java
@@ -33,7 +33,6 @@
 import org.apache.ignite.ml.nn.initializers.RandomInitializer;
 import org.apache.ignite.ml.optimization.updatecalculators.ParameterUpdateCalculator;
 import org.apache.ignite.ml.trainers.MultiLabelDatasetTrainer;
-import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
 import org.apache.ignite.ml.util.Utils;
 
 import java.io.Serializable;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdatesStrategy.java b/modules/ml/src/main/java/org/apache/ignite/ml/nn/UpdatesStrategy.java
similarity index 98%
rename from modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdatesStrategy.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/nn/UpdatesStrategy.java
index 5288dbfbf6c1c..e48d94665d3da 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdatesStrategy.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/nn/UpdatesStrategy.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml.trainers.group;
+package org.apache.ignite.ml.nn;
 
 import java.io.Serializable;
 import java.util.List;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientDescent.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientDescent.java
deleted file mode 100644
index 15ed914e4ee16..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientDescent.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.optimization;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.math.impls.vector.FunctionVector;
-import org.apache.ignite.ml.optimization.util.SparseDistributedMatrixMapReducer;
-
-/**
- * Gradient descent optimizer.
- */
-public class GradientDescent {
-    /**
-     * Function which computes gradient of the loss function at any given point.
-     */
-    private final GradientFunction lossGradient;
-
-    /**
-     * Weights updater applied on every gradient descent step to decide how weights should be changed.
-     */
-    private final Updater updater;
-
-    /**
-     * Max number of gradient descent iterations.
-     */
-    private int maxIterations = 1000;
-
-    /**
-     * Convergence tolerance is condition which decides iteration termination.
-     */
-    private double convergenceTol = 1e-8;
-
-    /**
-     * New gradient descent instance based of loss function and updater.
-     *
-     * @param lossGradient Function which computes gradient of the loss function at any given point
-     * @param updater Weights updater applied on every gradient descent step to decide how weights should be changed
-     */
-    public GradientDescent(GradientFunction lossGradient, Updater updater) {
-        this.lossGradient = lossGradient;
-        this.updater = updater;
-    }
-
-    /**
-     * Sets max number of gradient descent iterations.
-     *
-     * @param maxIterations Max number of gradient descent iterations
-     * @return This gradient descent instance
-     */
-    public GradientDescent withMaxIterations(int maxIterations) {
-        assert maxIterations >= 0;
-
-        this.maxIterations = maxIterations;
-
-        return this;
-    }
-
-    /**
-     * Sets convergence tolerance.
-     *
-     * @param convergenceTol Condition which decides iteration termination
-     * @return This gradient descent instance
-     */
-    public GradientDescent withConvergenceTol(double convergenceTol) {
-        assert convergenceTol >= 0;
-
-        this.convergenceTol = convergenceTol;
-
-        return this;
-    }
-
-    /**
-     * Computes point where loss function takes minimal value.
-     *
-     * @param data Inputs parameters of loss function
-     * @param initWeights Initial weights
-     * @return Point where loss function takes minimal value
-     */
-    public Vector optimize(Matrix data, Vector initWeights) {
-        Vector weights = initWeights, oldWeights = null, oldGradient = null;
-        IgniteFunction<Vector, Vector> gradientFunction = getLossGradientFunction(data);
-
-        for (int iteration = 0; iteration < maxIterations; iteration++) {
-            Vector gradient = gradientFunction.apply(weights);
-            Vector newWeights = updater.compute(oldWeights, oldGradient, weights, gradient, iteration);
-
-            if (isConverged(weights, newWeights))
-                return newWeights;
-            else {
-                oldGradient = gradient;
-                oldWeights = weights;
-                weights = newWeights;
-            }
-        }
-        return weights;
-    }
-
-    /**
-     * Calculates gradient based in distributed matrix using {@link SparseDistributedMatrixMapReducer}.
-     *
-     * @param data Distributed matrix
-     * @param weights Point to calculate gradient
-     * @return Gradient
-     */
-    private Vector calculateDistributedGradient(SparseDistributedMatrix data, Vector weights) {
-        SparseDistributedMatrixMapReducer mapReducer = new SparseDistributedMatrixMapReducer(data);
-        return mapReducer.mapReduce(
-            (matrix, args) -> {
-                Matrix inputs = extractInputs(matrix);
-                Vector groundTruth = extractGroundTruth(matrix);
-
-                return lossGradient.compute(inputs, groundTruth, args);
-            },
-            gradients -> {
-                int cnt = 0;
-                Vector resGradient = new DenseLocalOnHeapVector(data.columnSize());
-
-                for (Vector gradient : gradients) {
-                    if (gradient != null) {
-                        resGradient = resGradient.plus(gradient);
-                        cnt++;
-                    }
-                }
-
-                return resGradient.divide(cnt);
-            },
-            weights);
-    }
-
-    /**
-     * Tests if gradient descent process converged.
-     *
-     * @param weights Weights
-     * @param newWeights New weights
-     * @return {@code true} if process has converged, otherwise {@code false}
-     */
-    private boolean isConverged(Vector weights, Vector newWeights) {
-        if (convergenceTol == 0)
-            return false;
-        else {
-            double solutionVectorDiff = weights.minus(newWeights).kNorm(2.0);
-            return solutionVectorDiff < convergenceTol * Math.max(newWeights.kNorm(2.0), 1.0);
-        }
-    }
-
-    /**
-     * Extracts first column with ground truth from the data set matrix.
-     *
-     * @param data data to build model
-     * @return Ground truth vector
-     */
-    private Vector extractGroundTruth(Matrix data) {
-        return data.getCol(0);
-    }
-
-    /**
-     * Extracts all inputs from data set matrix and updates matrix so that first column contains value 1.0.
-     *
-     * @param data data to build model
-     * @return Inputs matrix
-     */
-    private Matrix extractInputs(Matrix data) {
-        data = data.copy();
-        data.assignColumn(0, new FunctionVector(data.rowSize(), row -> 1.0));
-        return data;
-    }
-
-    /** Makes carrying of the gradient function and fixes data matrix. */
-    private IgniteFunction<Vector, Vector> getLossGradientFunction(Matrix data) {
-        if (data instanceof SparseDistributedMatrix) {
-            SparseDistributedMatrix distributedMatrix = (SparseDistributedMatrix)data;
-
-            if (distributedMatrix.getStorage().storageMode() == StorageConstants.ROW_STORAGE_MODE)
-                return weights -> calculateDistributedGradient(distributedMatrix, weights);
-        }
-
-        Matrix inputs = extractInputs(data);
-        Vector groundTruth = extractGroundTruth(data);
-
-        return weights -> lossGradient.compute(inputs, groundTruth, weights);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientFunction.java
deleted file mode 100644
index a6a1e71a38701..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/GradientFunction.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.optimization;
-
-import java.io.Serializable;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Function which computes gradient of the loss function at any given point.
- */
-@FunctionalInterface
-public interface GradientFunction extends Serializable {
-    /** */
-    Vector compute(Matrix inputs, Vector groundTruth, Vector pnt);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/LeastSquaresGradientFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/LeastSquaresGradientFunction.java
deleted file mode 100644
index 4d90e3b994a43..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/LeastSquaresGradientFunction.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.optimization;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-
-/**
- * Function which computes gradient of least square loss function.
- */
-public class LeastSquaresGradientFunction implements GradientFunction {
-    /**
-     * {@inheritDoc}
-     */
-    @Override public Vector compute(Matrix inputs, Vector groundTruth, Vector pnt) {
-        return inputs.transpose().times(inputs.times(pnt).minus(groundTruth));
-    }
-}
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/util/SparseDistributedMatrixMapReducer.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/util/SparseDistributedMatrixMapReducer.java
deleted file mode 100644
index 20f861e84305e..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/util/SparseDistributedMatrixMapReducer.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.optimization.util;
-
-import java.util.Collection;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.distributed.keys.RowColMatrixKey;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
-
-/**
- * Wrapper of {@link SparseDistributedMatrix} which allow to perform computation on every node containing a part of the
- * distributed matrix, get results and then reduce them.
- */
-public class SparseDistributedMatrixMapReducer {
-    /** */
-    private final SparseDistributedMatrix distributedMatrix;
-
-    /** */
-    public SparseDistributedMatrixMapReducer(
-        SparseDistributedMatrix distributedMatrix) {
-        this.distributedMatrix = distributedMatrix;
-    }
-
-    /** */
-    public <R, T> R mapReduce(IgniteBiFunction<Matrix, T, R> mapper, IgniteFunction<Collection<R>, R> reducer, T args) {
-        Ignite ignite = Ignition.localIgnite();
-        SparseDistributedMatrixStorage storage = (SparseDistributedMatrixStorage)distributedMatrix.getStorage();
-
-        int colSize = distributedMatrix.columnSize();
-
-        Collection<R> results = ignite
-            .compute(ignite.cluster().forDataNodes(storage.cacheName()))
-            .broadcast(arguments -> {
-                Ignite locIgnite = Ignition.localIgnite();
-
-                Affinity<RowColMatrixKey> affinity = locIgnite.affinity(storage.cacheName());
-                ClusterNode locNode = locIgnite.cluster().localNode();
-
-                Map<ClusterNode, Collection<RowColMatrixKey>> keys = affinity.mapKeysToNodes(storage.getAllKeys());
-                Collection<RowColMatrixKey> locKeys = keys.get(locNode);
-
-                if (locKeys != null) {
-                    int idx = 0;
-                    Matrix locMatrix = new DenseLocalOnHeapMatrix(locKeys.size(), colSize);
-
-                    for (RowColMatrixKey key : locKeys) {
-                        Map<Integer, Double> row = storage.cache().get(key);
-
-                        for (Map.Entry<Integer, Double> cell : row.entrySet())
-                            locMatrix.set(idx, cell.getKey(), cell.getValue());
-
-                        idx++;
-                    }
-                    return mapper.apply(locMatrix, arguments);
-                }
-                return null;
-            }, args);
-        return reducer.apply(results);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/util/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/optimization/util/package-info.java
deleted file mode 100644
index cb01ab6268a90..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/optimization/util/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains util classes used in optimization package.
- */
-package org.apache.ignite.ml.optimization.util;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
index 9526db1e366ae..095aa31354ef6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionLSQRTrainer.java
@@ -17,18 +17,17 @@
 
 package org.apache.ignite.ml.regressions.linear;
 
+import java.util.Arrays;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.primitive.builder.data.SimpleLabeledDatasetDataBuilder;
 import org.apache.ignite.ml.math.Vector;
 import org.apache.ignite.ml.math.functions.IgniteBiFunction;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.math.isolve.LinSysPartitionDataBuilderOnHeap;
 import org.apache.ignite.ml.math.isolve.lsqr.AbstractLSQR;
 import org.apache.ignite.ml.math.isolve.lsqr.LSQROnHeap;
 import org.apache.ignite.ml.math.isolve.lsqr.LSQRResult;
 import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
 
-import java.util.Arrays;
-
 /**
  * Trainer of the linear regression model based on LSQR algorithm.
  *
@@ -43,7 +42,10 @@ public class LinearRegressionLSQRTrainer implements SingleLabelDatasetTrainer<Li
 
         try (LSQROnHeap<K, V> lsqr = new LSQROnHeap<>(
             datasetBuilder,
-            new LinSysPartitionDataBuilderOnHeap<>(new FeatureExtractorWrapper<>(featureExtractor), lbExtractor)
+            new SimpleLabeledDatasetDataBuilder<>(
+                new FeatureExtractorWrapper<>(featureExtractor),
+                lbExtractor.andThen(e -> new double[]{e})
+            )
         )) {
             res = lsqr.solve(0, 1e-12, 1e-12, 1e8, -1, false, null);
         }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionQRTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionQRTrainer.java
deleted file mode 100644
index 5de3cda5aee0d..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionQRTrainer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.decompositions.QRDSolver;
-import org.apache.ignite.ml.math.decompositions.QRDecomposition;
-import org.apache.ignite.ml.math.impls.vector.FunctionVector;
-
-/**
- * Linear regression trainer based on least squares loss function and QR decomposition.
- */
-public class LinearRegressionQRTrainer implements Trainer<LinearRegressionModel, Matrix> {
-    /**
-     * {@inheritDoc}
-     */
-    @Override public LinearRegressionModel train(Matrix data) {
-        Vector groundTruth = extractGroundTruth(data);
-        Matrix inputs = extractInputs(data);
-
-        QRDecomposition decomposition = new QRDecomposition(inputs);
-        QRDSolver solver = new QRDSolver(decomposition.getQ(), decomposition.getR());
-
-        Vector variables = solver.solve(groundTruth);
-        Vector weights = variables.viewPart(1, variables.size() - 1);
-
-        double intercept = variables.get(0);
-
-        return new LinearRegressionModel(weights, intercept);
-    }
-
-    /**
-     * Extracts first column with ground truth from the data set matrix.
-     *
-     * @param data data to build model
-     * @return Ground truth vector
-     */
-    private Vector extractGroundTruth(Matrix data) {
-        return data.getCol(0);
-    }
-
-    /**
-     * Extracts all inputs from data set matrix and updates matrix so that first column contains value 1.0.
-     *
-     * @param data data to build model
-     * @return Inputs matrix
-     */
-    private Matrix extractInputs(Matrix data) {
-        data = data.copy();
-
-        data.assignColumn(0, new FunctionVector(data.rowSize(), row -> 1.0));
-
-        return data;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
index 9be3fdd2a8aa2..98b888587e1b1 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainer.java
@@ -30,7 +30,7 @@
 import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
 import org.apache.ignite.ml.optimization.LossFunctions;
 import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
-import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
+import org.apache.ignite.ml.nn.UpdatesStrategy;
 
 import java.io.Serializable;
 import java.util.Arrays;
@@ -110,6 +110,9 @@ public LinearRegressionSGDTrainer(UpdatesStrategy<? super MultilayerPerceptron,
 
         double[] p = mlp.parameters().getStorage().data();
 
-        return new LinearRegressionModel(new DenseLocalOnHeapVector(Arrays.copyOf(p, p.length - 1)), p[p.length - 1]);
+        return new LinearRegressionModel(new DenseLocalOnHeapVector(
+            Arrays.copyOf(p, p.length - 1)),
+            p[p.length - 1]
+        );
     }
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/Trainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/Trainer.java
deleted file mode 100644
index b4f83d9e23acb..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/Trainer.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers;
-
-import org.apache.ignite.ml.Model;
-
-/** Trainer interface. */
-@Deprecated
-// TODO: IGNITE-7659: Reduce multiple Trainer interfaces to one
-public interface Trainer<M extends Model, T> {
-    /**
-     * Train the model based on provided data.
-     *
-     * @param data Data for training.
-     * @return Trained model.
-     */
-    public M train(T data);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java
deleted file mode 100644
index e20a55a9507e6..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/BaseLocalProcessorJob.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Objects;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.compute.ComputeJob;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-
-/**
- * Base job for group training.
- * It's purpose is to apply worker to each element (cache key or cache entry) of given cache specified
- * by keySupplier. Worker produces {@link ResultAndUpdates} object which contains 'side effects' which are updates
- * needed to apply to caches and computation result.
- * After we get all {@link ResultAndUpdates} we merge all 'update' parts of them for each node
- * and apply them on corresponding node, also we reduce all 'result' by some given reducer.
- *
- * @param <K> Type of keys of cache used for group trainer.
- * @param <V> Type of values of cache used for group trainer.
- * @param <T> Type of elements to which workers are applier.
- * @param <R> Type of result of worker.
- */
-public abstract class BaseLocalProcessorJob<K, V, T, R extends Serializable> implements ComputeJob {
-    /**
-     * UUID of group training.
-     */
-    protected UUID trainingUUID;
-
-    /**
-     * Worker.
-     */
-    protected IgniteFunction<T, ResultAndUpdates<R>> worker;
-
-    /**
-     * Supplier of keys determining elements to which worker should be applied.
-     */
-    protected IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keySupplier;
-
-    /**
-     * Operator used to reduce results from worker.
-     */
-    protected IgniteFunction<List<R>, R> reducer;
-
-    /**
-     * Name of cache used for training.
-     */
-    protected String cacheName;
-
-    /**
-     * Construct instance of this class with given arguments.
-     *
-     * @param worker Worker.
-     * @param keySupplier Supplier of keys.
-     * @param reducer Reducer.
-     * @param trainingUUID UUID of training.
-     * @param cacheName Name of cache used for training.
-     */
-    public BaseLocalProcessorJob(
-        IgniteFunction<T, ResultAndUpdates<R>> worker,
-        IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keySupplier,
-        IgniteFunction<List<R>, R> reducer,
-        UUID trainingUUID, String cacheName) {
-        this.worker = worker;
-        this.keySupplier = keySupplier;
-        this.reducer = reducer;
-        this.trainingUUID = trainingUUID;
-        this.cacheName = cacheName;
-    }
-
-    /** {@inheritDoc} */
-    @Override public void cancel() {
-        // NO-OP.
-    }
-
-    /** {@inheritDoc} */
-    @Override public R execute() throws IgniteException {
-        List<ResultAndUpdates<R>> resultsAndUpdates = toProcess().
-            map(worker).
-            collect(Collectors.toList());
-
-        ResultAndUpdates<R> totalRes = ResultAndUpdates.sum(reducer, resultsAndUpdates.stream().filter(Objects::nonNull).collect(Collectors.toList()));
-
-        totalRes.applyUpdates(ignite());
-
-        return totalRes.result();
-    }
-
-    /**
-     * Get stream of elements to process.
-     *
-     * @return Stream of elements to process.
-     */
-    protected abstract Stream<T> toProcess();
-
-    /**
-     * Ignite instance.
-     *
-     * @return Ignite instance.
-     */
-    protected static Ignite ignite() {
-        return Ignition.localIgnite();
-    }
-
-    /**
-     * Get cache used for training.
-     *
-     * @return Cache used for training.
-     */
-    protected IgniteCache<GroupTrainerCacheKey<K>, V> cache() {
-        return ignite().getOrCreateCache(cacheName);
-    }
-
-    /**
-     * Get affinity function for cache used in group training.
-     *
-     * @return Affinity function for cache used in group training.
-     */
-    protected Affinity<GroupTrainerCacheKey> affinity() {
-        return ignite().affinity(cacheName);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ConstModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ConstModel.java
deleted file mode 100644
index 75f817915a09c..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ConstModel.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import org.apache.ignite.ml.Model;
-
-/**
- * Model which outputs given constant.
- *
- * @param <T> Type of constant.
- */
-public class ConstModel<T> implements Model<T, T> {
-    /**
-     * Constant to be returned by this model.
-     */
-    private T c;
-
-    /**
-     * Create instance of this class specified by input parameters.
-     *
-     * @param c Constant to be returned by this model.
-     */
-    public ConstModel(T c) {
-        this.c = c;
-    }
-
-    /** {@inheritDoc} */
-    @Override public T apply(T val) {
-        return c;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainer.java
deleted file mode 100644
index fb34bf7443211..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainer.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.Trainer;
-import org.apache.ignite.ml.trainers.group.chain.ComputationsChain;
-import org.apache.ignite.ml.trainers.group.chain.EntryAndContext;
-import org.apache.ignite.ml.trainers.group.chain.HasTrainingUUID;
-
-/**
- * Class encapsulating synchronous distributed group training.
- * Training is performed by following scheme:
- * 1. For specified set of keys distributed initialization is done. For each key some initialization result is returned.
- * 2. All initialization results are processed locally and reduced into some object of type I.
- * 3. While 'shouldContinue' condition is true, training loop step is executed.
- * 4. After loop is finished, data from each key from final key set is collected.
- * 5. Data collected on previous step is transformed into a model which is returned as final result.
- * Note that all methods returning functions, suppliers etc should return values with minimal dependencies because they are serialized
- * with all dependent objects.
- *
- * @param <LC> Type of local context of the training.
- * @param <K> Type of data in {@link GroupTrainerCacheKey} keys on which the training is done.
- * @param <V> Type of cache values on which the training is done.
- * @param <IN> Type of data returned after initializing of distributed context.
- * @param <R> Type of result returned after training from each node.
- * @param <I> Type of data which is fed into each training loop step and returned from it.
- * @param <M> Type of model returned after training.
- * @param <T> Type of input to this trainer.
- * @param <G> Type of distributed context which is needed for forming final result which is send from each node to trainer for final model creation.
- */
-abstract class GroupTrainer<LC extends HasTrainingUUID, K, V, IN extends Serializable, R extends Serializable, I extends Serializable, M extends Model, T extends GroupTrainerInput<K>, G> implements Trainer<M, T> {
-    /**
-     * Cache on which training is performed. For example it can be cache of neural networks.
-     */
-    protected IgniteCache<GroupTrainerCacheKey<K>, V> cache;
-
-    /**
-     * Ignite instance.
-     */
-    protected Ignite ignite;
-
-    /**
-     * Construct an instance of this class.
-     *
-     * @param cache Cache on which training is performed.
-     * @param ignite Ignite instance.
-     */
-    GroupTrainer(
-        IgniteCache<GroupTrainerCacheKey<K>, V> cache,
-        Ignite ignite) {
-        this.cache = cache;
-        this.ignite = ignite;
-    }
-
-    /** {@inheritDoc} */
-    @Override public final M train(T data) {
-        UUID trainingUUID = UUID.randomUUID();
-        LC locCtx = initialLocalContext(data, trainingUUID);
-
-        GroupTrainingContext<K, V, LC> ctx = new GroupTrainingContext<>(locCtx, cache, ignite);
-        ComputationsChain<LC, K, V, T, T> chain = (i, c) -> i;
-        IgniteFunction<GroupTrainerCacheKey<K>, ResultAndUpdates<IN>> distributedInitializer
-            = distributedInitializer(data);
-
-        init(data, trainingUUID);
-
-        M res = chain.
-            thenDistributedForKeys(distributedInitializer, (t, lc) -> data.initialKeys(trainingUUID),
-                reduceDistributedInitData()).
-            thenLocally(this::locallyProcessInitData).
-            thenWhile(this::shouldContinue, trainingLoopStep()).
-            thenDistributedForEntries(this::extractContextForFinalResultCreation, finalResultsExtractor(),
-                this::finalResultKeys, finalResultsReducer()).
-            thenLocally(this::mapFinalResult).
-            process(data, ctx);
-
-        cleanup(locCtx);
-
-        return res;
-    }
-
-    /**
-     * Create initial local context from data given as input to trainer.
-     *
-     * @param data Data given as input to this trainer.
-     * @param trainingUUID UUID of this training.
-     * @return Initial local context.
-     */
-    protected abstract LC initialLocalContext(T data, UUID trainingUUID);
-
-    /** Override in subclasses if needed. */
-    protected void init(T data, UUID trainingUUID) {
-    }
-
-    /**
-     * Get function for initialization for each of keys specified in initial key set.
-     *
-     * @param data Data given to this trainer as input.
-     * @return Function for initialization for each of keys specified in initial key set.
-     */
-    protected abstract IgniteFunction<GroupTrainerCacheKey<K>, ResultAndUpdates<IN>> distributedInitializer(T data);
-
-    /**
-     * Get reducer to reduce data collected from initialization of each key specified in initial key set.
-     *
-     * @return Reducer to reduce data collected from initialization of each key specified in initial key set.
-     */
-    protected abstract IgniteFunction<List<IN>, IN> reduceDistributedInitData();
-
-    /**
-     * Transform data from initialization step into data which is fed as input to first step of training loop.
-     *
-     * @param data Data from initialization step.
-     * @param locCtx Local context.
-     * @return Data which is fed as input to first step of training loop.
-     */
-    protected abstract I locallyProcessInitData(IN data, LC locCtx);
-
-    /**
-     * Training loop step.
-     *
-     * @return Result of training loop step.
-     */
-    protected abstract ComputationsChain<LC, K, V, I, I> trainingLoopStep();
-
-    /**
-     * Condition specifying if training loop should continue.
-     *
-     * @param data First time, data returned by locallyProcessInitData then data returned by last step of loop.
-     * @param locCtx Local context.
-     * @return Boolean value indicating if training loop should continue.
-     */
-    protected abstract boolean shouldContinue(I data, LC locCtx);
-
-    /**
-     * Extract context for final result creation. Each key from the final keys set will be processed with
-     * finalResultsExtractor. While entry data (i.e. key and value) for each key varies, some data can be common for all
-     * processed entries. This data is called context.
-     *
-     * @param data Data returned from last training loop step.
-     * @param locCtx Local context.
-     * @return Context.
-     */
-    protected abstract IgniteSupplier<G> extractContextForFinalResultCreation(I data, LC locCtx);
-
-    /**
-     * Keys for final result creation.
-     *
-     * @param data Data returned from the last training loop step.
-     * @param locCtx Local context.
-     * @return Stream of keys for final result creation.
-     */
-    protected abstract IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> finalResultKeys(I data, LC locCtx);
-
-    /**
-     * Get function for extracting final result from each key specified in finalResultKeys.
-     *
-     * @return Function for extracting final result from each key specified in finalResultKeys.
-     */
-    protected abstract IgniteFunction<EntryAndContext<K, V, G>, ResultAndUpdates<R>> finalResultsExtractor();
-
-    /**
-     * Get function for reducing final results.
-     *
-     * @return Function for reducing final results.
-     */
-    protected abstract IgniteFunction<List<R>, R> finalResultsReducer();
-
-    /**
-     * Map final result to model which is returned by trainer.
-     *
-     * @param res Final result.
-     * @param locCtx Local context.
-     * @return Model resulted from training.
-     */
-    protected abstract M mapFinalResult(R res, LC locCtx);
-
-    /**
-     * Performs cleanups of temporary objects created by this trainer.
-     *
-     * @param locCtx Local context.
-     */
-    protected abstract void cleanup(LC locCtx);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerBaseProcessorTask.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerBaseProcessorTask.java
deleted file mode 100644
index b192f421f1a88..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerBaseProcessorTask.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cache.affinity.Affinity;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.compute.ComputeJob;
-import org.apache.ignite.compute.ComputeJobResult;
-import org.apache.ignite.compute.ComputeTaskAdapter;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * Base task for group trainer.
- *
- * @param <K> Type of cache keys of cache used for training.
- * @param <V> Type of cache values of cache used for training.
- * @param <C> Type of context (common part of data needed for computation).
- * @param <T> Type of arguments of workers.
- * @param <R> Type of computation result.
- */
-public abstract class GroupTrainerBaseProcessorTask<K, V, C, T, R extends Serializable> extends ComputeTaskAdapter<Void, R> {
-    /**
-     * Context supplier.
-     */
-    protected final IgniteSupplier<C> ctxSupplier;
-
-    /**
-     * UUID of training.
-     */
-    protected final UUID trainingUUID;
-
-    /**
-     * Worker.
-     */
-    protected IgniteFunction<T, ResultAndUpdates<R>> worker;
-
-    /**
-     * Reducer used for reducing of computations on specified keys.
-     */
-    protected final IgniteFunction<List<R>, R> reducer;
-
-    /**
-     * Name of cache on which training is done.
-     */
-    protected final String cacheName;
-
-    /**
-     * Supplier of keys on which worker should be executed.
-     */
-    protected final IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keysSupplier;
-
-    /**
-     * Ignite instance.
-     */
-    protected final Ignite ignite;
-
-    /**
-     * Construct an instance of this class with specified parameters.
-     *
-     * @param trainingUUID UUID of training.
-     * @param ctxSupplier Supplier of context.
-     * @param worker Function calculated on each of specified keys.
-     * @param keysSupplier Supplier of keys on which training is done.
-     * @param reducer Reducer used for reducing results of computation performed on each of specified keys.
-     * @param cacheName Name of cache on which training is done.
-     * @param ignite Ignite instance.
-     */
-    public GroupTrainerBaseProcessorTask(UUID trainingUUID,
-        IgniteSupplier<C> ctxSupplier,
-        IgniteFunction<T, ResultAndUpdates<R>> worker,
-        IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keysSupplier,
-        IgniteFunction<List<R>, R> reducer,
-        String cacheName,
-        Ignite ignite) {
-        this.trainingUUID = trainingUUID;
-        this.ctxSupplier = ctxSupplier;
-        this.worker = worker;
-        this.keysSupplier = keysSupplier;
-        this.reducer = reducer;
-        this.cacheName = cacheName;
-        this.ignite = ignite;
-    }
-
-    /** {@inheritDoc} */
-    @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(List<ClusterNode> subgrid,
-        @Nullable Void arg) throws IgniteException {
-        Map<ComputeJob, ClusterNode> res = new HashMap<>();
-
-        for (ClusterNode node : subgrid) {
-            BaseLocalProcessorJob<K, V, T, R> job = createJob();
-            res.put(job, node);
-        }
-
-        return res;
-    }
-
-    /** {@inheritDoc} */
-    @Nullable @Override public R reduce(List<ComputeJobResult> results) throws IgniteException {
-        return reducer.apply(results.stream().map(res -> (R)res.getData()).filter(Objects::nonNull).collect(Collectors.toList()));
-    }
-
-    /**
-     * Create job for execution on subgrid.
-     *
-     * @return Job for execution on subgrid.
-     */
-    protected abstract BaseLocalProcessorJob<K, V, T, R> createJob();
-
-    /**
-     * Get affinity function of cache on which training is done.
-     *
-     * @return Affinity function of cache on which training is done.
-     */
-    protected Affinity<GroupTrainerCacheKey> affinity() {
-        return ignite.affinity(cacheName);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerCacheKey.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerCacheKey.java
deleted file mode 100644
index 5e4cb76b6a663..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerCacheKey.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.UUID;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-
-/**
- * Class used as a key for caches on which {@link GroupTrainer} works.
- * Structurally it is a triple: (nodeLocalEntityIndex, trainingUUID, data);
- * nodeLocalEntityIndex is used to map key to node;
- * trainingUUID is id of training;
- * data is some custom data stored in this key, for example if we want to store three neural networks on one node
- * for training with training UUID == trainingUUID, we can use keys
- * (1, trainingUUID, networkIdx1), (1, trainingUUID, networkIdx2), (1, trainingUUID, networkIdx3).
- *
- * @param <K> Type of data part of this key.
- */
-public class GroupTrainerCacheKey<K> {
-    /**
-     * Part of key for key-to-node affinity.
-     */
-    @AffinityKeyMapped
-    private Long nodeLocEntityIdx;
-
-    /**
-     * UUID of training.
-     */
-    private UUID trainingUUID;
-
-    /**
-     * Data.
-     */
-    K data;
-
-    /**
-     * Construct instance of this class.
-     *
-     * @param nodeLocEntityIdx Part of key for key-to-node affinity.
-     * @param data Data.
-     * @param trainingUUID Training UUID.
-     */
-    public GroupTrainerCacheKey(long nodeLocEntityIdx, K data, UUID trainingUUID) {
-        this.nodeLocEntityIdx = nodeLocEntityIdx;
-        this.trainingUUID = trainingUUID;
-        this.data = data;
-    }
-
-    /**
-     * Construct instance of this class.
-     *
-     * @param nodeLocEntityIdx Part of key for key-to-node affinity.
-     * @param data Data.
-     * @param trainingUUID Training UUID.
-     */
-    public GroupTrainerCacheKey(int nodeLocEntityIdx, K data, UUID trainingUUID) {
-        this((long)nodeLocEntityIdx, data, trainingUUID);
-    }
-
-    /**
-     * Get part of key used for key-to-node affinity.
-     *
-     * @return Part of key used for key-to-node affinity.
-     */
-    public Long nodeLocalEntityIndex() {
-        return nodeLocEntityIdx;
-    }
-
-    /**
-     * Get UUID of training.
-     *
-     * @return UUID of training.
-     */
-    public UUID trainingUUID() {
-        return trainingUUID;
-    }
-
-    /**
-     * Get data.
-     *
-     * @return Data.
-     */
-    public K data() {
-        return data;
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean equals(Object o) {
-        if (this == o)
-            return true;
-        if (o == null || getClass() != o.getClass())
-            return false;
-
-        GroupTrainerCacheKey<?> key = (GroupTrainerCacheKey<?>)o;
-
-        if (nodeLocEntityIdx != null ? !nodeLocEntityIdx.equals(key.nodeLocEntityIdx) : key.nodeLocEntityIdx != null)
-            return false;
-        if (trainingUUID != null ? !trainingUUID.equals(key.trainingUUID) : key.trainingUUID != null)
-            return false;
-        return data != null ? data.equals(key.data) : key.data == null;
-    }
-
-    /** {@inheritDoc} */
-    @Override public int hashCode() {
-        int res = nodeLocEntityIdx != null ? nodeLocEntityIdx.hashCode() : 0;
-        res = 31 * res + (trainingUUID != null ? trainingUUID.hashCode() : 0);
-        res = 31 * res + (data != null ? data.hashCode() : 0);
-        return res;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerEntriesProcessorTask.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerEntriesProcessorTask.java
deleted file mode 100644
index daa396fac5dc4..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerEntriesProcessorTask.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.chain.EntryAndContext;
-
-/**
- * Task for processing entries of cache used for training.
- *
- * @param <K> Type of cache keys of cache used for training.
- * @param <V> Type of cache values of cache used for training.
- * @param <C> Type of context (common part of data needed for computation).
- * @param <R> Type of computation result.
- */
-public class GroupTrainerEntriesProcessorTask<K, V, C, R extends Serializable>
-    extends GroupTrainerBaseProcessorTask<K, V, C, EntryAndContext<K, V, C>, R> {
-    /**
-     * Construct instance of this class with given parameters.
-     *
-     * @param trainingUUID UUID of training.
-     * @param ctxSupplier Supplier of context.
-     * @param worker Function calculated on each of specified keys.
-     * @param keysSupplier Supplier of keys on which training is done.
-     * @param reducer Reducer used for reducing results of computation performed on each of specified keys.
-     * @param cacheName Name of cache on which training is done.
-     * @param ignite Ignite instance.
-     */
-    public GroupTrainerEntriesProcessorTask(UUID trainingUUID,
-        IgniteSupplier<C> ctxSupplier,
-        IgniteFunction<EntryAndContext<K, V, C>, ResultAndUpdates<R>> worker,
-        IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keysSupplier,
-        IgniteFunction<List<R>, R> reducer,
-        String cacheName,
-        Ignite ignite) {
-        super(trainingUUID, ctxSupplier, worker, keysSupplier, reducer, cacheName, ignite);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected BaseLocalProcessorJob<K, V, EntryAndContext<K, V, C>, R> createJob() {
-        return new LocalEntriesProcessorJob<>(ctxSupplier, worker, keysSupplier, reducer, trainingUUID, cacheName);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerInput.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerInput.java
deleted file mode 100644
index ae75f16091b2c..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerInput.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.UUID;
-import java.util.stream.Stream;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-
-/**
- * Interface for {@link GroupTrainer} inputs.
- *
- * @param <K> Types of cache keys used for group training.
- */
-public interface GroupTrainerInput<K> {
-    /**
-     * Get supplier of stream of keys used for initialization of {@link GroupTrainer}.
-     *
-     * @param trainingUUID UUID of training.
-     * @return Supplier of stream of keys used for initialization of {@link GroupTrainer}.
-     */
-    IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> initialKeys(UUID trainingUUID);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerKeysProcessorTask.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerKeysProcessorTask.java
deleted file mode 100644
index 7ac18f8881ec2..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainerKeysProcessorTask.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.chain.KeyAndContext;
-
-/**
- * Task for processing entries of cache used for training.
- *
- * @param <K> Type of cache keys of cache used for training.
- * @param <C> Type of context (common part of data needed for computation).
- * @param <R> Type of computation result.
- */
-public class GroupTrainerKeysProcessorTask<K, C, R extends Serializable> extends GroupTrainerBaseProcessorTask<K, Object, C, KeyAndContext<K, C>, R> {
-    /**
-     * Construct instance of this class with specified parameters.
-     *
-     * @param trainingUUID UUID of training.
-     * @param ctxSupplier Context supplier.
-     * @param worker Function calculated on each of specified keys.
-     * @param keysSupplier Supplier of keys on which computations should be done.
-     * @param reducer Reducer used for reducing results of computation performed on each of specified keys.
-     * @param cacheName Name of cache on which training is done.
-     * @param ignite Ignite instance.
-     */
-    public GroupTrainerKeysProcessorTask(UUID trainingUUID,
-        IgniteSupplier<C> ctxSupplier,
-        IgniteFunction<KeyAndContext<K, C>, ResultAndUpdates<R>> worker,
-        IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keysSupplier,
-        IgniteFunction<List<R>, R> reducer,
-        String cacheName,
-        Ignite ignite) {
-        super(trainingUUID, ctxSupplier, worker, keysSupplier, reducer, cacheName, ignite);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected BaseLocalProcessorJob<K, Object, KeyAndContext<K, C>, R> createJob() {
-        return new LocalKeysProcessorJob<>(ctxSupplier, worker, keysSupplier, reducer, trainingUUID, cacheName);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainingContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainingContext.java
deleted file mode 100644
index cbd04b24fb475..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/GroupTrainingContext.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.ml.trainers.group.chain.HasTrainingUUID;
-
-/**
- * Context for group training.
- *
- * @param <K> Type of keys of cache used for group training.
- * @param <V> Type of values of cache used for group training.
- * @param <L> Type of local context used for training.
- */
-public class GroupTrainingContext<K, V, L extends HasTrainingUUID> {
-    /**
-     * Local context.
-     */
-    private L locCtx;
-
-    /**
-     * Cache used for training.
-     */
-    private IgniteCache<GroupTrainerCacheKey<K>, V> cache;
-
-    /**
-     * Ignite instance.
-     */
-    private Ignite ignite;
-
-    /**
-     * Construct instance of this class.
-     *
-     * @param locCtx Local context.
-     * @param cache Information about cache used for training.
-     * @param ignite Ignite instance.
-     */
-    public GroupTrainingContext(L locCtx, IgniteCache<GroupTrainerCacheKey<K>, V> cache, Ignite ignite) {
-        this.locCtx = locCtx;
-        this.cache = cache;
-        this.ignite = ignite;
-    }
-
-    /**
-     * Construct new training context with same parameters but with new cache.
-     *
-     * @param newCache New cache.
-     * @param <K1> Type of keys of new cache.
-     * @param <V1> Type of values of new cache.
-     * @return New training context with same parameters but with new cache.
-     */
-    public <K1, V1> GroupTrainingContext<K1, V1, L> withCache(IgniteCache<GroupTrainerCacheKey<K1>, V1> newCache) {
-        return new GroupTrainingContext<>(locCtx, newCache, ignite);
-    }
-
-    /**
-     * Get local context.
-     *
-     * @return Local context.
-     */
-    public L localContext() {
-        return locCtx;
-    }
-
-    /**
-     * Get cache used for training.
-     *
-     * @return Cache used for training.
-     */
-    public IgniteCache<GroupTrainerCacheKey<K>, V> cache() {
-        return cache;
-    }
-
-    /**
-     * Get ignite instance.
-     *
-     * @return Ignite instance.
-     */
-    public Ignite ignite() {
-        return ignite;
-    }
-}
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/LocalEntriesProcessorJob.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/LocalEntriesProcessorJob.java
deleted file mode 100644
index d035aa5b26822..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/LocalEntriesProcessorJob.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.chain.EntryAndContext;
-
-/**
- * {@link BaseLocalProcessorJob} specified to entry processing.
- *
- * @param <K> Type of cache used for group training.
- * @param <V> Type of values used for group training.
- * @param <C> Type of context.
- * @param <R> Type of result returned by worker.
- */
-public class LocalEntriesProcessorJob<K, V, C, R extends Serializable> extends BaseLocalProcessorJob<K, V, EntryAndContext<K, V, C>, R> {
-    /**
-     * Supplier of context for worker.
-     */
-    private final IgniteSupplier<C> ctxSupplier;
-
-    /**
-     * Construct an instance of this class.
-     *
-     * @param ctxSupplier Supplier for context for worker.
-     * @param worker Worker.
-     * @param keySupplier Supplier of keys.
-     * @param reducer Reducer.
-     * @param trainingUUID UUID for training.
-     * @param cacheName Name of cache used for training.
-     */
-    public LocalEntriesProcessorJob(IgniteSupplier<C> ctxSupplier,
-        IgniteFunction<EntryAndContext<K, V, C>, ResultAndUpdates<R>> worker,
-        IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keySupplier,
-        IgniteFunction<List<R>, R> reducer,
-        UUID trainingUUID, String cacheName) {
-        super(worker, keySupplier, reducer, trainingUUID, cacheName);
-        this.ctxSupplier = ctxSupplier;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected Stream<EntryAndContext<K, V, C>> toProcess() {
-        C ctx = ctxSupplier.get();
-
-        return selectLocalEntries().map(e -> new EntryAndContext<>(e, ctx));
-    }
-
-    /**
-     * Select entries for processing by worker.
-     *
-     * @return Entries for processing by worker.
-     */
-    private Stream<Map.Entry<GroupTrainerCacheKey<K>, V>> selectLocalEntries() {
-        Set<GroupTrainerCacheKey<K>> keys = keySupplier.get().
-            filter(k -> Objects.requireNonNull(affinity().mapKeyToNode(k)).isLocal()).
-            filter(k -> k.trainingUUID().equals(trainingUUID)).
-            collect(Collectors.toSet());
-
-        return cache().getAll(keys).entrySet().stream();
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/LocalKeysProcessorJob.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/LocalKeysProcessorJob.java
deleted file mode 100644
index cad53c937bd66..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/LocalKeysProcessorJob.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.Objects;
-import java.util.UUID;
-import java.util.stream.Stream;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.chain.KeyAndContext;
-
-/**
- * {@link BaseLocalProcessorJob} specified to keys processing.
- *
- * @param <K> Type of cache used for group training.
- * @param <V> Type of values used for group training.
- * @param <C> Type of context.
- * @param <R> Type of result returned by worker.
- */
-public class LocalKeysProcessorJob<K, V, C, R extends Serializable> extends BaseLocalProcessorJob<K, V, KeyAndContext<K, C>, R> {
-    /**
-     * Supplier of worker context.
-     */
-    private final IgniteSupplier<C> ctxSupplier;
-
-    /**
-     * Construct instance of this class with given arguments.
-     *
-     * @param worker Worker.
-     * @param keySupplier Supplier of keys.
-     * @param reducer Reducer.
-     * @param trainingUUID UUID of training.
-     * @param cacheName Name of cache used for training.
-     */
-    public LocalKeysProcessorJob(IgniteSupplier<C> ctxSupplier,
-        IgniteFunction<KeyAndContext<K, C>, ResultAndUpdates<R>> worker,
-        IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keySupplier,
-        IgniteFunction<List<R>, R> reducer,
-        UUID trainingUUID, String cacheName) {
-        super(worker, keySupplier, reducer, trainingUUID, cacheName);
-        this.ctxSupplier = ctxSupplier;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected Stream<KeyAndContext<K, C>> toProcess() {
-        C ctx = ctxSupplier.get();
-
-        return selectLocalKeys().map(k -> new KeyAndContext<>(k, ctx));
-    }
-
-    /**
-     * Get subset of keys provided by keySupplier which are mapped to node on which code is executed.
-     *
-     * @return Subset of keys provided by keySupplier which are mapped to node on which code is executed.
-     */
-    private Stream<GroupTrainerCacheKey<K>> selectLocalKeys() {
-        return keySupplier.get().
-            filter(k -> Objects.requireNonNull(affinity().mapKeyToNode(k)).isLocal()).
-            filter(k -> k.trainingUUID().equals(trainingUUID));
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/Metaoptimizer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/Metaoptimizer.java
deleted file mode 100644
index 0ab6d32eb0184..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/Metaoptimizer.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.List;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-
-/**
- * Class encapsulating data transformations in group training in {@link MetaoptimizerGroupTrainer}, which is adapter of
- * {@link GroupTrainer}.
- *
- * @param <LC> Local context of {@link GroupTrainer}.
- * @param <X> Type of data which is processed in training loop step.
- * @param <Y> Type of data returned by training loop step data processor.
- * @param <I> Type of data to which data returned by distributed initialization is mapped.
- * @param <D> Type of data returned by initialization.
- * @param <O> Type of data to which data returned by data processor is mapped.
- */
-public interface Metaoptimizer<LC, X, Y, I, D, O> {
-    /**
-     * Get function used to reduce distributed initialization results.
-     *
-     * @return Function used to reduce distributed initialization results.
-     */
-    IgniteFunction<List<D>, D> initialReducer();
-
-    /**
-     * Maps data returned by distributed initialization to data consumed by training loop step.
-     *
-     * @param data Data returned by distributed initialization.
-     * @param locCtx Local context.
-     * @return Mapping of data returned by distributed initialization to data consumed by training loop step.
-     */
-    I locallyProcessInitData(D data, LC locCtx);
-
-    /**
-     * Preprocess data for {@link MetaoptimizerGroupTrainer#dataProcessor()}.
-     *
-     * @return Preprocessed data for {@link MetaoptimizerGroupTrainer#dataProcessor()}.
-     */
-    default IgniteFunction<X, X> distributedPreprocessor() {
-        return x -> x;
-    }
-
-    /**
-     * Get function used to map values returned by {@link MetaoptimizerGroupTrainer#dataProcessor()}.
-     *
-     * @return Function used to map values returned by {@link MetaoptimizerGroupTrainer#dataProcessor()}.
-     */
-    IgniteFunction<Y, O> distributedPostprocessor();
-
-    /**
-     * Get binary operator used for reducing results returned by distributedPostprocessor.
-     *
-     * @return Binary operator used for reducing results returned by distributedPostprocessor.
-     */
-    IgniteFunction<List<O>, O> postProcessReducer();
-
-    /**
-     * Transform data returned by distributed part of training loop step into input fed into distributed part of training
-     * loop step.
-     *
-     * @param input Type of output of distributed part of training loop step.
-     * @param locCtx Local context.
-     * @return Result of transform data returned by distributed part of training loop step into input fed into distributed part of training
-     * loop step.
-     */
-    I localProcessor(O input, LC locCtx);
-
-    /**
-     * Returns value of predicate 'should training loop continue given previous step output and local context'.
-     *
-     * @param input Input of previous step.
-     * @param locCtx Local context.
-     * @return Value of predicate 'should training loop continue given previous step output and local context'.
-     */
-    boolean shouldContinue(I input, LC locCtx);
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/MetaoptimizerDistributedStep.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/MetaoptimizerDistributedStep.java
deleted file mode 100644
index 08e1f47d5ccb0..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/MetaoptimizerDistributedStep.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.stream.Stream;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.chain.DistributedEntryProcessingStep;
-import org.apache.ignite.ml.trainers.group.chain.EntryAndContext;
-import org.apache.ignite.ml.trainers.group.chain.HasTrainingUUID;
-
-/**
- * Distributed step based on {@link Metaoptimizer}.
- *
- * @param <L> Type of local context.
- * @param <K> Type of data in {@link GroupTrainerCacheKey}.
- * @param <V> Type of values of cache on which training is done.
- * @param <G> Type of distributed context.
- * @param <I> Type of data to which data returned by distributed initialization is mapped (see {@link Metaoptimizer}).
- * @param <O> Type of data to which data returned by data processor is mapped (see {@link Metaoptimizer}).
- * @param <X> Type of data which is processed in training loop step (see {@link Metaoptimizer}).
- * @param <Y> Type of data returned by training loop step data processor (see {@link Metaoptimizer}).
- * @param <D> Type of data returned by initialization (see {@link Metaoptimizer}).
- */
-class MetaoptimizerDistributedStep<L extends HasTrainingUUID, K, V, G, I extends Serializable, O extends Serializable,
-    X, Y, D extends Serializable> implements DistributedEntryProcessingStep<L, K, V, G, I, O> {
-    /**
-     * {@link Metaoptimizer}.
-     */
-    private final Metaoptimizer<L, X, Y, I, D, O> metaoptimizer;
-
-    /**
-     * {@link MetaoptimizerGroupTrainer} for which this distributed step is used.
-     */
-    private final MetaoptimizerGroupTrainer<L, K, V, D, ?, I, ?, ?, G, O, X, Y> trainer;
-
-    /**
-     * Construct instance of this class with given parameters.
-     *
-     * @param metaoptimizer Metaoptimizer.
-     * @param trainer {@link MetaoptimizerGroupTrainer} for which this distributed step is used.
-     */
-    public MetaoptimizerDistributedStep(Metaoptimizer<L, X, Y, I, D, O> metaoptimizer,
-        MetaoptimizerGroupTrainer<L, K, V, D, ?, I, ?, ?, G, O, X, Y> trainer) {
-        this.metaoptimizer = metaoptimizer;
-        this.trainer = trainer;
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteSupplier<G> remoteContextSupplier(I input, L locCtx) {
-        return trainer.remoteContextExtractor(input, locCtx);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteFunction<EntryAndContext<K, V, G>, ResultAndUpdates<O>> worker() {
-        IgniteFunction<X, ResultAndUpdates<Y>> dataProcessor = trainer.dataProcessor();
-        IgniteFunction<X, X> preprocessor = metaoptimizer.distributedPreprocessor();
-        IgniteFunction<Y, O> postprocessor = metaoptimizer.distributedPostprocessor();
-        IgniteFunction<EntryAndContext<K, V, G>, X> ctxExtractor = trainer.trainingLoopStepDataExtractor();
-
-        return entryAndCtx -> {
-            X apply = ctxExtractor.apply(entryAndCtx);
-            preprocessor.apply(apply);
-            ResultAndUpdates<Y> res = dataProcessor.apply(apply);
-            O postprocessRes = postprocessor.apply(res.result());
-
-            return ResultAndUpdates.of(postprocessRes).setUpdates(res.updates());
-        };
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keys(I input, L locCtx) {
-        return trainer.keysToProcessInTrainingLoop(locCtx);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteFunction<List<O>, O> reducer() {
-        return metaoptimizer.postProcessReducer();
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/MetaoptimizerGroupTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/MetaoptimizerGroupTrainer.java
deleted file mode 100644
index bebfe3edcd817..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/MetaoptimizerGroupTrainer.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.chain.Chains;
-import org.apache.ignite.ml.trainers.group.chain.ComputationsChain;
-import org.apache.ignite.ml.trainers.group.chain.EntryAndContext;
-import org.apache.ignite.ml.trainers.group.chain.HasTrainingUUID;
-
-/**
- * Group trainer using {@link Metaoptimizer}.
- * Main purpose of this trainer is to extract various transformations (normalizations for example) of data which is processed
- * in the training loop step into distinct entity called metaoptimizer and only fix the main part of logic in
- * trainers extending this class. This way we'll be able to quickly switch between this transformations by using different metaoptimizers
- * without touching main logic.
- *
- * @param <LC> Type of local context.
- * @param <K> Type of data in {@link GroupTrainerCacheKey} keys on which the training is done.
- * @param <V> Type of values of cache used in group training.
- * @param <IN> Data type which is returned by distributed initializer.
- * @param <R> Type of final result returned by nodes on which training is done.
- * @param <I> Type of data which is fed into each training loop step and returned from it.
- * @param <M> Type of model returned after training.
- * @param <T> Type of input of this trainer.
- * @param <G> Type of distributed context which is needed for forming final result which is send from each node to trainer for final model creation.
- * @param <O> Type of output of postprocessor.
- * @param <X> Type of data which is processed by dataProcessor.
- * @param <Y> Type of data which is returned by postprocessor.
- */
-public abstract class MetaoptimizerGroupTrainer<LC extends HasTrainingUUID, K, V, IN extends Serializable,
-    R extends Serializable, I extends Serializable,
-    M extends Model, T extends GroupTrainerInput<K>,
-    G, O extends Serializable, X, Y> extends
-    GroupTrainer<LC, K, V, IN, R, I, M, T, G> {
-    /**
-     * Metaoptimizer.
-     */
-    private Metaoptimizer<LC, X, Y, I, IN, O> metaoptimizer;
-
-    /**
-     * Construct instance of this class.
-     *
-     * @param cache Cache on which group trainer is done.
-     * @param ignite Ignite instance.
-     */
-    public MetaoptimizerGroupTrainer(Metaoptimizer<LC, X, Y, I, IN, O> metaoptimizer,
-        IgniteCache<GroupTrainerCacheKey<K>, V> cache,
-        Ignite ignite) {
-        super(cache, ignite);
-        this.metaoptimizer = metaoptimizer;
-    }
-
-    /**
-     * Get function used to map EntryAndContext to type which is processed by dataProcessor.
-     *
-     * @return Function used to map EntryAndContext to type which is processed by dataProcessor.
-     */
-    protected abstract IgniteFunction<EntryAndContext<K, V, G>, X> trainingLoopStepDataExtractor();
-
-    /**
-     * Get supplier of keys which should be processed by training loop.
-     *
-     * @param locCtx Local text.
-     * @return Supplier of keys which should be processed by training loop.
-     */
-    protected abstract IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keysToProcessInTrainingLoop(LC locCtx);
-
-    /**
-     * Get supplier of context used in training loop step.
-     *
-     * @param input Input.
-     * @param ctx Local context.
-     * @return Supplier of context used in training loop step.
-     */
-    protected abstract IgniteSupplier<G> remoteContextExtractor(I input, LC ctx);
-
-    /** {@inheritDoc} */
-    @Override protected void init(T data, UUID trainingUUID) {
-    }
-
-    /**
-     * Get function used to process data in training loop step.
-     *
-     * @return Function used to process data in training loop step.
-     */
-    protected abstract IgniteFunction<X, ResultAndUpdates<Y>> dataProcessor();
-
-    /** {@inheritDoc} */
-    @Override protected ComputationsChain<LC, K, V, I, I> trainingLoopStep() {
-        ComputationsChain<LC, K, V, I, O> chain = Chains.create(new MetaoptimizerDistributedStep<>(metaoptimizer, this));
-        return chain.thenLocally(metaoptimizer::localProcessor);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected I locallyProcessInitData(IN data, LC locCtx) {
-        return metaoptimizer.locallyProcessInitData(data, locCtx);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected boolean shouldContinue(I data, LC locCtx) {
-        return metaoptimizer.shouldContinue(data, locCtx);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected IgniteFunction<List<IN>, IN> reduceDistributedInitData() {
-        return metaoptimizer.initialReducer();
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ResultAndUpdates.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ResultAndUpdates.java
deleted file mode 100644
index 9ed18af5242e9..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/ResultAndUpdates.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-
-/**
- * Class containing result of computation and updates which should be made for caches.
- * Purpose of this class is mainly performance optimization: suppose we have multiple computations which run in parallel
- * and do some updates to caches. It is more efficient to collect all changes from all this computations and perform them
- * in batch.
- *
- * @param <R> Type of computation result.
- */
-public class ResultAndUpdates<R> {
-    /**
-     * Result of computation.
-     */
-    private R res;
-
-    /**
-     * Updates in the form cache name -> (key -> new value).
-     */
-    private Map<String, Map> updates = new ConcurrentHashMap<>();
-
-    /**
-     * Construct an instance of this class.
-     *
-     * @param res Computation result.
-     */
-    public ResultAndUpdates(R res) {
-        this.res = res;
-    }
-
-    /**
-     * Construct an instance of this class.
-     *
-     * @param res Computation result.
-     * @param updates Map of updates in the form cache name -> (key -> new value).
-     */
-    ResultAndUpdates(R res, Map<String, Map> updates) {
-        this.res = res;
-        this.updates = updates;
-    }
-
-    /**
-     * Construct an empty result.
-     *
-     * @param <R> Result type.
-     * @return Empty result.
-     */
-    public static <R> ResultAndUpdates<R> empty() {
-        return new ResultAndUpdates<>(null);
-    }
-
-    /**
-     * Construct {@link ResultAndUpdates} object from given result.
-     *
-     * @param res Result of computation.
-     * @param <R> Type of result of computation.
-     * @return ResultAndUpdates object.
-     */
-    public static <R> ResultAndUpdates<R> of(R res) {
-        return new ResultAndUpdates<>(res);
-    }
-
-    /**
-     * Add a cache update to this object.
-     *
-     * @param cache Cache to be updated.
-     * @param key Key of cache to be updated.
-     * @param val New value.
-     * @param <K> Type of key of cache to be updated.
-     * @param <V> New value.
-     * @return This object.
-     */
-    @SuppressWarnings("unchecked")
-    public <K, V> ResultAndUpdates<R> updateCache(IgniteCache<K, V> cache, K key, V val) {
-        String name = cache.getName();
-
-        updates.computeIfAbsent(name, s -> new ConcurrentHashMap());
-        updates.get(name).put(key, val);
-
-        return this;
-    }
-
-    /**
-     * Get result of computation.
-     *
-     * @return Result of computation.
-     */
-    public R result() {
-        return res;
-    }
-
-    /**
-     * Sum collection of ResultAndUpdate into one: results are reduced by specified binary operator and updates are merged.
-     *
-     * @param reducer Reducer used to combine computation results.
-     * @param resultsAndUpdates ResultAndUpdates to be combined with.
-     * @param <R> Type of computation result.
-     * @return Sum of collection ResultAndUpdate objects.
-     */
-    @SuppressWarnings("unchecked")
-    static <R> ResultAndUpdates<R> sum(IgniteFunction<List<R>, R> reducer,
-        Collection<ResultAndUpdates<R>> resultsAndUpdates) {
-        Map<String, Map> allUpdates = new HashMap<>();
-
-        for (ResultAndUpdates<R> ru : resultsAndUpdates) {
-            for (String cacheName : ru.updates.keySet()) {
-                allUpdates.computeIfAbsent(cacheName, s -> new HashMap());
-
-                allUpdates.get(cacheName).putAll(ru.updates.get(cacheName));
-            }
-        }
-
-        List<R> results = resultsAndUpdates.stream().map(ResultAndUpdates::result).filter(Objects::nonNull).collect(Collectors.toList());
-
-        return new ResultAndUpdates<>(reducer.apply(results), allUpdates);
-    }
-
-    /**
-     * Get updates map.
-     *
-     * @return Updates map.
-     */
-    public Map<String, Map> updates() {
-        return updates;
-    }
-
-    /**
-     * Set updates map.
-     *
-     * @param updates New updates map.
-     * @return This object.
-     */
-    ResultAndUpdates<R> setUpdates(Map<String, Map> updates) {
-        this.updates = updates;
-        return this;
-    }
-
-    /**
-     * Apply updates to caches.
-     *
-     * @param ignite Ignite instance.
-     */
-    void applyUpdates(Ignite ignite) {
-        for (Map.Entry<String, Map> entry : updates.entrySet()) {
-            IgniteCache<Object, Object> cache = ignite.getOrCreateCache(entry.getKey());
-
-            cache.putAll(entry.getValue());
-        }
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdateStrategies.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdateStrategies.java
deleted file mode 100644
index 33ec96a5ecd45..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/UpdateStrategies.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import org.apache.ignite.ml.optimization.SmoothParametrized;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
-import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
-import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
-
-/**
- * Holder class for various update strategies.
- */
-public class UpdateStrategies {
-    /**
-     * Simple GD update strategy.
-     *
-     * @return GD update strategy.
-     */
-    public static UpdatesStrategy<SmoothParametrized, SimpleGDParameterUpdate> GD() {
-        return new UpdatesStrategy<>(new SimpleGDUpdateCalculator(), SimpleGDParameterUpdate::sumLocal, SimpleGDParameterUpdate::avg);
-    }
-
-    /**
-     * RProp update strategy.
-     *
-     * @return RProp update strategy.
-     */
-    public static UpdatesStrategy<SmoothParametrized, RPropParameterUpdate> RProp() {
-        return new UpdatesStrategy<>(new RPropUpdateCalculator(), RPropParameterUpdate::sumLocal, RPropParameterUpdate::avg);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/Chains.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/Chains.java
deleted file mode 100644
index db4f13f96c4d8..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/Chains.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group.chain;
-
-import java.io.Serializable;
-
-/**
- * Class containing methods creating {@link ComputationsChain}.
- */
-public class Chains {
-    /**
-     * Create computation chain consisting of one returning its input as output.
-     *
-     * @param <L> Type of local context of created chain.
-     * @param <K> Type of keys of cache used in computation chain.
-     * @param <V> Type of values of cache used in computation chain.
-     * @param <I> Type of input to computation chain.
-     * @return Computation chain consisting of one returning its input as output.
-     */
-    public static <L extends HasTrainingUUID, K, V, I> ComputationsChain<L, K, V, I, I> create() {
-        return (input, context) -> input;
-    }
-
-    /**
-     * Create {@link ComputationsChain} from {@link DistributedEntryProcessingStep}.
-     *
-     * @param step Distributed chain step.
-     * @param <L> Type of local context of created chain.
-     * @param <K> Type of keys of cache used in computation chain.
-     * @param <V> Type of values of cache used in computation chain.
-     * @param <C> Type of context used by worker in {@link DistributedEntryProcessingStep}.
-     * @param <I> Type of input to computation chain.
-     * @param <O> Type of output of computation chain.
-     * @return Computation created from {@link DistributedEntryProcessingStep}.
-     */
-    public static <L extends HasTrainingUUID, K, V, C, I, O extends Serializable> ComputationsChain<L, K, V, I, O> create(
-        DistributedEntryProcessingStep<L, K, V, C, I, O> step) {
-        ComputationsChain<L, K, V, I, I> chain = create();
-        return chain.thenDistributedForEntries(step);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/ComputationsChain.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/ComputationsChain.java
deleted file mode 100644
index 3c3bdab833863..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/ComputationsChain.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group.chain;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Stream;
-import javax.cache.processor.EntryProcessor;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.cluster.ClusterGroup;
-import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.ml.math.functions.Functions;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.GroupTrainerCacheKey;
-import org.apache.ignite.ml.trainers.group.GroupTrainerEntriesProcessorTask;
-import org.apache.ignite.ml.trainers.group.GroupTrainerKeysProcessorTask;
-import org.apache.ignite.ml.trainers.group.GroupTrainingContext;
-import org.apache.ignite.ml.trainers.group.ResultAndUpdates;
-
-/**
- * This class encapsulates convenient way for creating computations chain for distributed model training.
- * Chain is meant in the sense that output of each non-final computation is fed as input to next computation.
- * Chain is basically a bi-function from context and input to output, context is separated from input
- * because input is specific to each individual step and context is something which is convenient to have access to in each of steps.
- * Context is separated into two parts: local context and remote context.
- * There are two kinds of computations: local and distributed.
- * Local steps are just functions from two arguments: input and local context.
- * Distributed steps are more sophisticated, but basically can be thought as functions of form
- * localContext -> (function of remote context -> output), locally we fix local context and get function
- * (function of remote context -> output) which is executed distributed.
- * Chains are composable through 'then' method.
- *
- * @param <L> Type of local context.
- * @param <K> Type of cache keys.
- * @param <V> Type of cache values.
- * @param <I> Type of input of this chain.
- * @param <O> Type of output of this chain.
- * // TODO: IGNITE-7405 check if it is possible to integrate with {@link EntryProcessor}.
- */
-@FunctionalInterface
-public interface ComputationsChain<L extends HasTrainingUUID, K, V, I, O> {
-    /**
-     * Process given input and {@link GroupTrainingContext}.
-     *
-     * @param input Computation chain input.
-     * @param ctx {@link GroupTrainingContext}.
-     * @return Result of processing input and context.
-     */
-    O process(I input, GroupTrainingContext<K, V, L> ctx);
-
-    /**
-     * Add a local step to this chain.
-     *
-     * @param locStep Local step.
-     * @param <O1> Output of local step.
-     * @return Composition of this chain and local step.
-     */
-    default <O1> ComputationsChain<L, K, V, I, O1> thenLocally(IgniteBiFunction<O, L, O1> locStep) {
-        ComputationsChain<L, K, V, O, O1> nextStep = (input, context) -> locStep.apply(input, context.localContext());
-        return then(nextStep);
-    }
-
-    /**
-     * Add a distributed step which works in the following way:
-     * 1. apply local context and input to local context extractor and keys supplier to get corresponding suppliers;
-     * 2. on each node_n
-     * 2.1. get context object.
-     * 2.2. for each entry_i e located on node_n with key_i from keys stream compute worker((context, entry_i)) and get
-     * (cachesUpdates_i, result_i).
-     * 2.3. for all i on node_n merge cacheUpdates_i and apply them.
-     * 2.4. for all i on node_n, reduce result_i into result_n.
-     * 3. get all result_n, reduce them into result and return result.
-     *
-     * @param <O1> Type of worker output.
-     * @param <G> Type of context used by worker.
-     * @param workerCtxExtractor Extractor of context for worker.
-     * @param worker Function computed on each entry of cache used for training. Second argument is context:
-     * common part of data which is independent from key.
-     * @param ks Function from chain input and local context to supplier of keys for worker.
-     * @param reducer Function used for reducing results of worker.
-     * @return Combination of this chain and distributed step specified by given parameters.
-     */
-    default <O1 extends Serializable, G> ComputationsChain<L, K, V, I, O1> thenDistributedForEntries(
-        IgniteBiFunction<O, L, IgniteSupplier<G>> workerCtxExtractor,
-        IgniteFunction<EntryAndContext<K, V, G>, ResultAndUpdates<O1>> worker,
-        IgniteBiFunction<O, L, IgniteSupplier<Stream<GroupTrainerCacheKey<K>>>> ks,
-        IgniteFunction<List<O1>, O1> reducer) {
-        ComputationsChain<L, K, V, O, O1> nextStep = (input, context) -> {
-            L locCtx = context.localContext();
-            IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keysSupplier = ks.apply(input, locCtx);
-
-            Ignite ignite = context.ignite();
-            UUID trainingUUID = context.localContext().trainingUUID();
-            String cacheName = context.cache().getName();
-            ClusterGroup grp = ignite.cluster().forDataNodes(cacheName);
-
-            // Apply first two arguments locally because it is common for all nodes.
-            IgniteSupplier<G> extractor = Functions.curry(workerCtxExtractor).apply(input).apply(locCtx);
-
-            return ignite.compute(grp).execute(new GroupTrainerEntriesProcessorTask<>(trainingUUID, extractor, worker, keysSupplier, reducer, cacheName, ignite), null);
-        };
-        return then(nextStep);
-    }
-
-    /**
-     * Add a distributed step which works in the following way:
-     * 1. apply local context and input to local context extractor and keys supplier to get corresponding suppliers;
-     * 2. on each node_n
-     * 2.1. get context object.
-     * 2.2. for each key_i from keys stream such that key_i located on node_n compute worker((context, entry_i)) and get
-     * (cachesUpdates_i, result_i).
-     * 2.3. for all i on node_n merge cacheUpdates_i and apply them.
-     * 2.4. for all i on node_n, reduce result_i into result_n.
-     * 3. get all result_n, reduce them into result and return result.
-     *
-     * @param <O1> Type of worker output.
-     * @param <G> Type of context used by worker.
-     * @param workerCtxExtractor Extractor of context for worker.
-     * @param worker Function computed on each entry of cache used for training. Second argument is context:
-     * common part of data which is independent from key.
-     * @param keysSupplier Function from chain input and local context to supplier of keys for worker.
-     * @param reducer Function used for reducing results of worker.
-     * @return Combination of this chain and distributed step specified by given parameters.
-     */
-    default <O1 extends Serializable, G> ComputationsChain<L, K, V, I, O1> thenDistributedForKeys(
-        IgniteBiFunction<O, L, IgniteSupplier<G>> workerCtxExtractor,
-        IgniteFunction<KeyAndContext<K, G>, ResultAndUpdates<O1>> worker,
-        IgniteBiFunction<O, L, IgniteSupplier<Stream<GroupTrainerCacheKey<K>>>> keysSupplier,
-        IgniteFunction<List<O1>, O1> reducer) {
-        ComputationsChain<L, K, V, O, O1> nextStep = (input, context) -> {
-            L locCtx = context.localContext();
-            IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> ks = keysSupplier.apply(input, locCtx);
-
-            Ignite ignite = context.ignite();
-            UUID trainingUUID = context.localContext().trainingUUID();
-            String cacheName = context.cache().getName();
-            ClusterGroup grp = ignite.cluster().forDataNodes(cacheName);
-
-            // Apply first argument locally because it is common for all nodes.
-            IgniteSupplier<G> extractor = Functions.curry(workerCtxExtractor).apply(input).apply(locCtx);
-
-            return ignite.compute(grp).execute(new GroupTrainerKeysProcessorTask<>(trainingUUID, extractor, worker, ks, reducer, cacheName, ignite), null);
-        };
-        return then(nextStep);
-    }
-
-    /**
-     * Add a distributed step specified by {@link DistributedEntryProcessingStep}.
-     *
-     * @param step Distributed step.
-     * @param <O1> Type of output of distributed step.
-     * @param <G> Type of context of distributed step.
-     * @return Combination of this chain and distributed step specified by input.
-     */
-    default <O1 extends Serializable, G> ComputationsChain<L, K, V, I, O1> thenDistributedForEntries(
-        DistributedEntryProcessingStep<L, K, V, G, O, O1> step) {
-        return thenDistributedForEntries(step::remoteContextSupplier, step.worker(), step::keys, step.reducer());
-    }
-
-    /**
-     * Add a distributed step specified by {@link DistributedKeyProcessingStep}.
-     *
-     * @param step Distributed step.
-     * @param <O1> Type of output of distributed step.
-     * @param <G> Type of context of distributed step.
-     * @return Combination of this chain and distributed step specified by input.
-     */
-    default <O1 extends Serializable, G> ComputationsChain<L, K, V, I, O1> thenDistributedForKeys(
-        DistributedKeyProcessingStep<L, K, G, O, O1> step) {
-        return thenDistributedForKeys(step::remoteContextSupplier, step.worker(), step::keys, step.reducer());
-    }
-
-    /**
-     * Version of 'thenDistributedForKeys' where worker does not depend on context.
-     *
-     * @param worker Worker.
-     * @param kf Function providing supplier
-     * @param reducer Function from chain input and local context to supplier of keys for worker.
-     * @param <O1> Type of worker output.
-     * @return Combination of this chain and distributed step specified by given parameters.
-     */
-    default <O1 extends Serializable> ComputationsChain<L, K, V, I, O1> thenDistributedForKeys(
-        IgniteFunction<GroupTrainerCacheKey<K>, ResultAndUpdates<O1>> worker,
-        IgniteBiFunction<O, L, IgniteSupplier<Stream<GroupTrainerCacheKey<K>>>> kf,
-        IgniteFunction<List<O1>, O1> reducer) {
-
-        return thenDistributedForKeys((o, lc) -> () -> o, (context) -> worker.apply(context.key()), kf, reducer);
-    }
-
-    /**
-     * Combine this computation chain with other computation chain in the following way:
-     * 1. perform this calculations chain and get result r.
-     * 2. while 'cond(r)' is true, r = otherChain(r, context)
-     * 3. return r.
-     *
-     * @param cond Condition checking if 'while' loop should continue.
-     * @param otherChain Chain to be combined with this chain.
-     * @return Combination of this chain and otherChain.
-     */
-    default ComputationsChain<L, K, V, I, O> thenWhile(IgniteBiPredicate<O, L> cond,
-        ComputationsChain<L, K, V, O, O> otherChain) {
-        ComputationsChain<L, K, V, I, O> me = this;
-        return (input, context) -> {
-            O res = me.process(input, context);
-
-            while (cond.apply(res, context.localContext()))
-                res = otherChain.process(res, context);
-
-            return res;
-        };
-    }
-
-    /**
-     * Combine this chain with other: feed this chain as input to other, pass same context as second argument to both chains
-     * process method.
-     *
-     * @param next Next chain.
-     * @param <O1> Type of next chain output.
-     * @return Combined chain.
-     */
-    default <O1> ComputationsChain<L, K, V, I, O1> then(ComputationsChain<L, K, V, O, O1> next) {
-        ComputationsChain<L, K, V, I, O> me = this;
-        return (input, context) -> {
-            O myRes = me.process(input, context);
-            return next.process(myRes, context);
-        };
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedEntryProcessingStep.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedEntryProcessingStep.java
deleted file mode 100644
index 8fd126497e11f..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedEntryProcessingStep.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group.chain;
-
-import java.io.Serializable;
-
-/**
- * {@link DistributedStep} specialized to {@link EntryAndContext}.
- *
- * @param <L> Local context.
- * @param <K> Type of keys of cache used for group training.
- * @param <V> Type of values of cache used for group training.
- * @param <C> Context used by worker.
- * @param <I> Type of input to this step.
- * @param <O> Type of output of this step.
- */
-public interface DistributedEntryProcessingStep<L, K, V, C, I, O extends Serializable> extends
-    DistributedStep<EntryAndContext<K, V, C>, L, K, C, I, O> {
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedKeyProcessingStep.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedKeyProcessingStep.java
deleted file mode 100644
index fb8d867f3ef9b..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedKeyProcessingStep.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group.chain;
-
-import java.io.Serializable;
-
-/**
- * {@link DistributedStep} specialized to {@link KeyAndContext}.
- *
- * @param <L> Local context.
- * @param <K> Type of keys of cache used for group training.
- * @param <C> Context used by worker.
- * @param <I> Type of input to this step.
- * @param <O> Type of output of this step.
- */
-public interface DistributedKeyProcessingStep<L, K, C, I, O extends Serializable> extends
-    DistributedStep<KeyAndContext<K, C>, L, K, C, I, O> {
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedStep.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedStep.java
deleted file mode 100644
index 804a886298a99..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/DistributedStep.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group.chain;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.stream.Stream;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.GroupTrainerCacheKey;
-import org.apache.ignite.ml.trainers.group.ResultAndUpdates;
-
-/**
- * Class encapsulating logic of distributed step in {@link ComputationsChain}.
- *
- * @param <T> Type of elements to be processed by worker.
- * @param <L> Local context.
- * @param <K> Type of keys of cache used for group training.
- * @param <C> Context used by worker.
- * @param <I> Type of input to this step.
- * @param <O> Type of output of this step.
- */
-public interface DistributedStep<T, L, K, C, I, O extends Serializable> {
-    /**
-     * Create supplier of context used by worker.
-     *
-     * @param input Input.
-     * @param locCtx Local context.
-     * @return Context used by worker.
-     */
-    IgniteSupplier<C> remoteContextSupplier(I input, L locCtx);
-
-    /**
-     * Get function applied to each cache element specified by keys.
-     *
-     * @return Function applied to each cache entry specified by keys..
-     */
-    IgniteFunction<T, ResultAndUpdates<O>> worker();
-
-    /**
-     * Get supplier of keys for worker.
-     *
-     * @param input Input to this step.
-     * @param locCtx Local context.
-     * @return Keys for worker.
-     */
-    IgniteSupplier<Stream<GroupTrainerCacheKey<K>>> keys(I input, L locCtx);
-
-    /**
-     * Get function used to reduce results returned by worker.
-     *
-     * @return Function used to reduce results returned by worker..
-     */
-    IgniteFunction<List<O>, O> reducer();
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/EntryAndContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/EntryAndContext.java
deleted file mode 100644
index 59c3b34f63107..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/EntryAndContext.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group.chain;
-
-import java.util.Map;
-import org.apache.ignite.ml.trainers.group.GroupTrainerCacheKey;
-
-/**
- * Entry of cache used for group training and context.
- * This class is used as input for workers of distributed steps of {@link ComputationsChain}.
- *
- * @param <K> Type of cache keys used for training.
- * @param <V> Type of cache values used for training.
- * @param <C> Type of context.
- */
-public class EntryAndContext<K, V, C> {
-    /**
-     * Entry of cache used for training.
-     */
-    private Map.Entry<GroupTrainerCacheKey<K>, V> entry;
-
-    /**
-     * Context.
-     */
-    private C ctx;
-
-    /**
-     * Construct instance of this class.
-     *
-     * @param entry Entry of cache used for training.
-     * @param ctx Context.
-     */
-    public EntryAndContext(Map.Entry<GroupTrainerCacheKey<K>, V> entry, C ctx) {
-        this.entry = entry;
-        this.ctx = ctx;
-    }
-
-    /**
-     * Get entry of cache used for training.
-     *
-     * @return Entry of cache used for training.
-     */
-    public Map.Entry<GroupTrainerCacheKey<K>, V> entry() {
-        return entry;
-    }
-
-    /**
-     * Get context.
-     *
-     * @return Context.
-     */
-    public C context() {
-        return ctx;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/HasTrainingUUID.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/HasTrainingUUID.java
deleted file mode 100644
index d855adf52302e..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/HasTrainingUUID.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group.chain;
-
-import java.util.UUID;
-
-/**
- * Interface for classes which contain UUID of training.
- */
-public interface HasTrainingUUID {
-    /**
-     * Get training UUID.
-     *
-     * @return Training UUID.
-     */
-    UUID trainingUUID();
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/KeyAndContext.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/KeyAndContext.java
deleted file mode 100644
index ba36e0e542fd6..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/KeyAndContext.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group.chain;
-
-import org.apache.ignite.ml.trainers.group.GroupTrainerCacheKey;
-
-/**
- * Class containing key and remote context (see explanation of remote context in {@link ComputationsChain}).
- *
- * @param <K> Cache key type.
- * @param <C> Remote context.
- */
-public class KeyAndContext<K, C> {
-    /**
-     * Key of group trainer.
-     */
-    private GroupTrainerCacheKey<K> key;
-
-    /**
-     * Remote context.
-     */
-    private C ctx;
-
-    /**
-     * Construct instance of this class.
-     *
-     * @param key Cache key.
-     * @param ctx Remote context.
-     */
-    public KeyAndContext(GroupTrainerCacheKey<K> key, C ctx) {
-        this.key = key;
-        this.ctx = ctx;
-    }
-
-    /**
-     * Get group trainer cache key.
-     *
-     * @return Group trainer cache key.
-     */
-    public GroupTrainerCacheKey<K> key() {
-        return key;
-    }
-
-    /**
-     * Get remote context.
-     *
-     * @return Remote context.
-     */
-    public C context() {
-        return ctx;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/package-info.java
deleted file mode 100644
index 46dcc6b9e84c5..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/chain/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains classes related to computations chain.
- */
-package org.apache.ignite.ml.trainers.group.chain;
\ No newline at end of file
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/package-info.java b/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/package-info.java
deleted file mode 100644
index 9b7f7cd9c96ac..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/trainers/group/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * Contains group trainers.
- */
-package org.apache.ignite.ml.trainers.group;
\ No newline at end of file
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
index 9900f854be9f4..0c3408e1697fd 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/IgniteMLTestSuite.java
@@ -23,11 +23,9 @@
 import org.apache.ignite.ml.knn.KNNTestSuite;
 import org.apache.ignite.ml.math.MathImplMainTestSuite;
 import org.apache.ignite.ml.nn.MLPTestSuite;
-import org.apache.ignite.ml.optimization.OptimizationTestSuite;
 import org.apache.ignite.ml.preprocessing.PreprocessingTestSuite;
 import org.apache.ignite.ml.regressions.RegressionsTestSuite;
 import org.apache.ignite.ml.svm.SVMTestSuite;
-import org.apache.ignite.ml.trainers.group.TrainersGroupTestSuite;
 import org.apache.ignite.ml.tree.DecisionTreeTestSuite;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
@@ -45,8 +43,6 @@
     KNNTestSuite.class,
     LocalModelsTest.class,
     MLPTestSuite.class,
-    TrainersGroupTestSuite.class,
-    OptimizationTestSuite.class,
     DatasetTestSuite.class,
     PreprocessingTestSuite.class,
     GAGridTestSuite.class
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeapTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeapTest.java
index ec9fdaaca5498..bdd1eeae8fd65 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeapTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/math/isolve/lsqr/LSQROnHeapTest.java
@@ -22,7 +22,7 @@
 import java.util.Map;
 import org.apache.ignite.ml.dataset.DatasetBuilder;
 import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
-import org.apache.ignite.ml.math.isolve.LinSysPartitionDataBuilderOnHeap;
+import org.apache.ignite.ml.dataset.primitive.builder.data.SimpleLabeledDatasetDataBuilder;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -64,9 +64,9 @@ public void testSolveLinearSystem() {
 
         LSQROnHeap<Integer, double[]> lsqr = new LSQROnHeap<>(
             datasetBuilder,
-            new LinSysPartitionDataBuilderOnHeap<>(
+            new SimpleLabeledDatasetDataBuilder<>(
                 (k, v) -> Arrays.copyOf(v, v.length - 1),
-                (k, v) -> v[3]
+                (k, v) -> new double[]{v[3]}
             )
         );
 
@@ -87,9 +87,9 @@ public void testSolveLinearSystemWithX0() {
 
         LSQROnHeap<Integer, double[]> lsqr = new LSQROnHeap<>(
             datasetBuilder,
-            new LinSysPartitionDataBuilderOnHeap<>(
+            new SimpleLabeledDatasetDataBuilder<>(
                 (k, v) -> Arrays.copyOf(v, v.length - 1),
-                (k, v) -> v[3]
+                (k, v) -> new double[]{v[3]}
             )
         );
 
@@ -118,9 +118,9 @@ public void testSolveLeastSquares() throws Exception {
 
         try (LSQROnHeap<Integer, double[]> lsqr = new LSQROnHeap<>(
             datasetBuilder,
-            new LinSysPartitionDataBuilderOnHeap<>(
+            new SimpleLabeledDatasetDataBuilder<>(
                 (k, v) -> Arrays.copyOf(v, v.length - 1),
-                (k, v) -> v[4]
+                (k, v) -> new double[]{v[4]}
             )
         )) {
             LSQRResult res = lsqr.solve(0, 1e-12, 1e-12, 1e8, -1, false, null);
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java
index 038b880f5cebc..654ebe0030e22 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerIntegrationTest.java
@@ -31,7 +31,6 @@
 import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
 import org.apache.ignite.ml.optimization.LossFunctions;
 import org.apache.ignite.ml.optimization.updatecalculators.*;
-import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import java.io.Serializable;
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
index c53f6f104d84e..db1488139ff70 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/MLPTrainerTest.java
@@ -24,7 +24,6 @@
 import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
 import org.apache.ignite.ml.optimization.LossFunctions;
 import org.apache.ignite.ml.optimization.updatecalculators.*;
-import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.runners.Enclosed;
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java
index a64af9b5cc545..3b65a28b49329 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistIntegrationTest.java
@@ -32,7 +32,7 @@
 import org.apache.ignite.ml.optimization.LossFunctions;
 import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
 import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
-import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
+import org.apache.ignite.ml.nn.UpdatesStrategy;
 import org.apache.ignite.ml.util.MnistUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java
index d966484d548c6..406331234722f 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/nn/performance/MLPTrainerMnistTest.java
@@ -27,7 +27,7 @@
 import org.apache.ignite.ml.optimization.LossFunctions;
 import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
 import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
-import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
+import org.apache.ignite.ml.nn.UpdatesStrategy;
 import org.apache.ignite.ml.util.MnistUtils;
 import org.junit.Test;
 
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/optimization/GradientDescentTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/optimization/GradientDescentTest.java
deleted file mode 100644
index f6f4775be78eb..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/optimization/GradientDescentTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.optimization;
-
-import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.junit.Test;
-
-/**
- * Tests for {@link GradientDescent}.
- */
-public class GradientDescentTest {
-    /** */
-    private static final double PRECISION = 1e-6;
-
-    /**
-     * Test gradient descent optimization on function y = x^2 with gradient function 2 * x.
-     */
-    @Test
-    public void testOptimize() {
-        GradientDescent gradientDescent = new GradientDescent(
-            (inputs, groundTruth, point) -> point.times(2),
-            new SimpleUpdater(0.01)
-        );
-
-        Vector res = gradientDescent.optimize(new DenseLocalOnHeapMatrix(new double[1][1]),
-            new DenseLocalOnHeapVector(new double[]{ 2.0 }));
-
-        TestUtils.assertEquals(0, res.get(0), PRECISION);
-    }
-
-    /**
-     * Test gradient descent optimization on function y = (x - 2)^2 with gradient function 2 * (x - 2).
-     */
-    @Test
-    public void testOptimizeWithOffset() {
-        GradientDescent gradientDescent = new GradientDescent(
-            (inputs, groundTruth, point) -> point.minus(new DenseLocalOnHeapVector(new double[]{ 2.0 })).times(2.0),
-            new SimpleUpdater(0.01)
-        );
-
-        Vector res = gradientDescent.optimize(new DenseLocalOnHeapMatrix(new double[1][1]),
-            new DenseLocalOnHeapVector(new double[]{ 2.0 }));
-
-        TestUtils.assertEquals(2, res.get(0), PRECISION);
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/optimization/OptimizationTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/optimization/OptimizationTestSuite.java
deleted file mode 100644
index 0ae6e4c70bbe4..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/optimization/OptimizationTestSuite.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.optimization;
-
-import org.apache.ignite.ml.optimization.util.SparseDistributedMatrixMapReducerTest;
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/**
- * Test suite for group trainer tests.
- */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
-    GradientDescentTest.class,
-    SparseDistributedMatrixMapReducerTest.class
-})
-public class OptimizationTestSuite {
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/optimization/util/SparseDistributedMatrixMapReducerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/optimization/util/SparseDistributedMatrixMapReducerTest.java
deleted file mode 100644
index 9017c431808fb..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/optimization/util/SparseDistributedMatrixMapReducerTest.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.optimization.util;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Tests for {@link SparseDistributedMatrixMapReducer}.
- */
-public class SparseDistributedMatrixMapReducerTest extends GridCommonAbstractTest {
-    /** Number of nodes in grid */
-    private static final int NODE_COUNT = 2;
-
-    /** */
-    private Ignite ignite;
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        for (int i = 1; i <= NODE_COUNT; i++)
-            startGrid(i);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() {
-        stopAllGrids();
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override protected void beforeTest() throws Exception {
-        /* Grid instance. */
-        ignite = grid(NODE_COUNT);
-        ignite.configuration().setPeerClassLoadingEnabled(true);
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-    }
-
-    /**
-     * Tests that matrix 100x100 filled by "1.0" and distributed across nodes successfully processed (calculate sum of
-     * all elements) via {@link SparseDistributedMatrixMapReducer}.
-     */
-    public void testMapReduce() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        SparseDistributedMatrix distributedMatrix = new SparseDistributedMatrix(100, 100);
-        for (int i = 0; i < 100; i++)
-            for (int j = 0; j < 100; j++)
-                distributedMatrix.set(i, j, 1);
-        SparseDistributedMatrixMapReducer mapReducer = new SparseDistributedMatrixMapReducer(distributedMatrix);
-        double total = mapReducer.mapReduce(
-            (matrix, args) -> {
-                double partialSum = 0.0;
-                for (int i = 0; i < matrix.rowSize(); i++)
-                    for (int j = 0; j < matrix.columnSize(); j++)
-                        partialSum += matrix.get(i, j);
-                return partialSum;
-            },
-            sums -> {
-                double totalSum = 0;
-                for (Double partialSum : sums)
-                    if (partialSum != null)
-                        totalSum += partialSum;
-                return totalSum;
-            }, 0.0);
-        assertEquals(100.0 * 100.0, total, 1e-18);
-    }
-
-    /**
-     * Tests that matrix 100x100 filled by "1.0" and distributed across nodes successfully processed via
-     * {@link SparseDistributedMatrixMapReducer} even when mapping function returns {@code null}.
-     */
-    public void testMapReduceWithNullValues() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        SparseDistributedMatrix distributedMatrix = new SparseDistributedMatrix(100, 100);
-        for (int i = 0; i < 100; i++)
-            for (int j = 0; j < 100; j++)
-                distributedMatrix.set(i, j, 1);
-        SparseDistributedMatrixMapReducer mapReducer = new SparseDistributedMatrixMapReducer(distributedMatrix);
-        double total = mapReducer.mapReduce(
-            (matrix, args) -> null,
-            sums -> {
-                double totalSum = 0;
-                for (Double partialSum : sums)
-                    if (partialSum != null)
-                        totalSum += partialSum;
-                return totalSum;
-            }, 0.0);
-        assertEquals(0, total, 1e-18);
-    }
-
-    /**
-     * Tests that matrix 1x100 filled by "1.0" and distributed across nodes successfully processed (calculate sum of
-     * all elements) via {@link SparseDistributedMatrixMapReducer} even when not all nodes contains data.
-     */
-    public void testMapReduceWithOneEmptyNode() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        SparseDistributedMatrix distributedMatrix = new SparseDistributedMatrix(1, 100);
-        for (int j = 0; j < 100; j++)
-            distributedMatrix.set(0, j, 1);
-        SparseDistributedMatrixMapReducer mapReducer = new SparseDistributedMatrixMapReducer(distributedMatrix);
-        double total = mapReducer.mapReduce(
-            (matrix, args) -> {
-                double partialSum = 0.0;
-                for (int i = 0; i < matrix.rowSize(); i++)
-                    for (int j = 0; j < matrix.columnSize(); j++)
-                        partialSum += matrix.get(i, j);
-                return partialSum;
-            },
-            sums -> {
-                double totalSum = 0;
-                for (Double partialSum : sums)
-                    if (partialSum != null)
-                        totalSum += partialSum;
-                return totalSum;
-            }, 0.0);
-        assertEquals(100.0, total, 1e-18);
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java
index b3c9368194366..5005ef25f3e48 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/RegressionsTestSuite.java
@@ -27,9 +27,6 @@
 @RunWith(Suite.class)
 @Suite.SuiteClasses({
     LinearRegressionModelTest.class,
-    LocalLinearRegressionQRTrainerTest.class,
-    DistributedLinearRegressionQRTrainerTest.class,
-    BlockDistributedLinearRegressionQRTrainerTest.class,
     LinearRegressionLSQRTrainerTest.class,
     LinearRegressionSGDTrainerTest.class
 })
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/ArtificialRegressionDatasets.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/ArtificialRegressionDatasets.java
deleted file mode 100644
index ed6bf36417410..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/ArtificialRegressionDatasets.java
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-/**
- * Artificial regression datasets to be used in regression trainers tests. These datasets were generated by scikit-learn
- * tools, {@code sklearn.datasets.make_regression} procedure.
- */
-public class ArtificialRegressionDatasets {
-    /**
-     * Artificial dataset with 10 observations described by 1 feature.
-     */
-    public static final TestDataset regression10x1 = new TestDataset(new double[][] {
-        {1.97657990214, 0.197725444973},
-        {-5.0835948878, -0.279921224228},
-        {-5.09032600779, -0.352291245969},
-        {9.67660993007, 0.755464872441},
-        {4.95927629958, 0.451981771462},
-        {29.2635107429, 2.2277440173},
-        {-18.3122588459, -1.25363275369},
-        {-3.61729307199, -0.273362913982},
-        {-7.19042139249, -0.473846634967},
-        {3.68008403347, 0.353883097536}
-    }, new double[] {13.554054703}, -0.808655936776);
-
-    /**
-     * Artificial dataset with 10 observations described by 5 features.
-     */
-    public static final TestDataset regression10x5 = new TestDataset(new double[][] {
-        {118.635647237, 0.687593385888, -1.18956185502, -0.305420702986, 1.98794097418, -0.776629036361},
-        {-18.2808432286, -0.165921853684, -0.156162539573, 1.56284391134, -0.198876782109, -0.0921618505605},
-        {22.6110523992, 0.0268106268606, 0.702141470035, -0.41503615392, -1.09726502337, 1.30830482813},
-        {209.820435262, 0.379809113402, -0.192097238579, -1.27460497119, 2.48052002019, -0.574430888865},
-        {-253.750024054, -1.48044570917, -0.331747484523, 0.387993627712, 0.372583756237, -2.27404065923},
-        {-24.6467766166, -0.66991474156, 0.269042238935, -0.271412703096, -0.561166818525, 1.37067541854},
-        {-311.903650717, 0.268274438122, -1.10491275353, -1.06738703543, -2.24387799735, -0.207431467989},
-        {74.2055323536, -0.329489531894, -0.493350762533, -0.644851462227, 0.661220945573, 1.65950140864},
-        {57.0312289904, -1.07266578457, 0.80375035572, -0.45207210139, 1.69314420969, -1.10526080856},
-        {12.149399645, 1.46504629281, -1.05843246079, 0.266225365277, -0.0113100353869, -0.983495425471}
-    }, new double[] {99.8393653561, 82.4948224094, 20.2087724072, 97.3306384162, 55.7502297387}, 3.98444039189);
-
-    /**
-     * Artificial dataset with 100 observations described by 5 features.
-     */
-    public static final TestDataset regression100x5 = new TestDataset(new double[][] {
-        {-44.2310642946, -0.0331360137605, -0.5290800706, -0.634340342338, -0.428433927151, 0.830582347183},
-        {76.2539139721, -0.216200869652, 0.513212019048, -0.693404511747, 0.132995973133, 1.28470259833},
-        {293.369799914, 2.90735870802, 0.457740818846, -0.490470696097, -0.442343455187, 0.584038258781},
-        {124.258807314, 1.64158129148, 0.0616936820145, 1.24082841519, -1.20126518593, -0.542298907742},
-        {13.6610807249, -1.10834821778, 0.545508208111, 1.81361288715, -0.786543112444, 0.250772626496},
-        {101.924582305, -0.433526394969, 0.257594734335, 1.22333193911, 0.76626554927, -0.0400734567005},
-        {25.5963186303, -0.202003301507, 0.717101151637, -0.486881225605, 1.15215024807, -0.921615554612},
-        {75.7959681263, -0.604173187402, 0.0364386836472, 1.67544714536, 0.394743148877, 0.0237966550759},
-        {-97.539357166, -0.774517689169, -0.0966902473883, -0.152250704254, -0.325472625458, 0.0720711851256},
-        {0.394748999236, -0.559303402754, -0.0493339259273, -1.10840277768, -0.0800969523557, 1.80939282066},
-        {-62.0138166431, 0.062614716778, -0.844143618016, 0.55269949861, -2.32580899335, 1.58020577369},
-        {584.427692931, 2.13184767906, 1.22222461994, 1.71894070494, 2.69512281718, 0.294123497874},
-        {-59.8323709765, 1.00006112818, -1.54481230765, -0.781282316493, 0.0255925284853, -0.0821173744608},
-        {101.565711925, -0.38699836725, 1.06934591441, -0.260429311097, 1.02628949564, 0.0431473245174},
-        {-141.592607814, 0.993279116267, -0.371768203378, -0.851483217286, -1.96241293548, -0.612279404296},
-        {34.8038723379, -0.0182719243972, 0.306367604506, -0.650526589206, 1.30693112283, -0.587465952557},
-        {-16.9554534069, -0.703006786668, -0.770718401931, 0.748423272307, 0.502544067819, 0.346625621533},
-        {-76.2896177709, -0.16440174812, -1.77431555198, 0.195326723837, 2.01240994405, -1.19559207119},
-        {-3.23827624818, -0.674138419631, -1.62238580284, 2.02235607862, 0.679194838679, 0.150203732584},
-        {-21.962456854, -0.766271014206, 0.958599712131, -0.313045794728, 0.232655576106, -0.360950549871},
-        {349.583669646, 1.75976166947, 1.47271612346, 0.0346005603489, 0.474907228495, 0.61379496381},
-        {-418.397356757, -1.83395936566, -0.911702678716, -0.532478094882, -2.03835348133, -0.423005552518},
-        {55.0298153952, -0.0301384716096, -0.0137929430966, -0.348583692759, 0.986486580719, 0.154436524434},
-        {127.150063206, 1.92682560465, -0.434844790414, 0.1082898967, -0.00723338222402, -0.513199251824},
-        {89.6172507626, 1.02463790902, 0.744369837717, 1.250323683, -1.58252612128, -0.588242778808},
-        {92.5124829355, -0.403298547743, 0.0422774545428, -0.175000467434, 1.61110066857, 0.422330077287},
-        {-303.040366788, 0.611569308879, -1.21926246291, -2.49250330276, -0.789166929605, -1.30166501196},
-        {-17.4020602839, 1.72337202371, -1.83540537288, 0.731588761841, -0.338642535062, -1.11053518125},
-        {114.918701324, 0.437385758628, 0.975885170381, 0.439444038872, 1.51666514156, -1.93095020264},
-        {-8.43548064928, -0.799507968686, -0.00842968328782, -0.154994093964, 1.09169753491, -0.0114818657732},
-        {109.209286025, 2.56472965015, -2.07047248035, -0.46764001177, 0.845267147375, -0.236767841427},
-        {61.5259982971, -0.379391870148, -0.131017762354, -0.220275015864, 1.82097825699, -0.0568354876403},
-        {-71.3872099588, 0.642138455414, -1.00242489879, 0.536780074488, 0.350977275771, -1.8204862883},
-        {-21.2768078629, -0.454268998895, 0.0992324274219, 0.0363496803224, 0.281940751723, -0.198435570828},
-        {-8.07838891387, -0.331642089041, -0.494067341253, 0.386035842816, -0.738221128298, 1.18236299649},
-        {30.4818041751, 0.099206096537, 0.150688905006, 0.332932621949, 0.194845631964, -0.446717875795},
-        {237.209150991, 1.12560447042, 0.448488431264, -0.724623711259, 0.401868257097, 1.67129001163},
-        {185.172816475, 0.36594142556, -0.0796476435741, 0.473836257, 1.30890722633, 0.592415068693},
-        {19.8830237044, 1.52497319332, 0.466906090264, -0.716635613964, -1.19532276745, -0.697663531684},
-        {209.396793626, 0.368478789658, 0.699162303982, 1.96702434462, -0.815379139879, 0.863369634396},
-        {-215.100514168, -1.83902416164, -1.14966820385, -1.01044860587, 1.76881340629, -0.32165916241},
-        {-33.4687353426, -0.0451102002703, 0.642212950033, 0.580822065219, -1.02341504063, -0.781229325942},
-        {150.251474823, 0.220170650298, 0.224858901011, 0.541299425328, 1.15151550963, 0.0329044069571},
-        {92.2160506097, 1.86450932451, -0.991150940533, -1.49137866968, 1.02113774105, 0.0544762857136},
-        {41.2138467595, -0.778892265105, 0.714957464344, 1.79833618993, -0.335322825621, -0.397548301803},
-        {13.151262759, 0.301745607362, 0.129778280739, 0.260094818273, -0.10587841585, -0.599330307629},
-        {-367.864703951, -1.68695981263, -0.611957677512, -0.0362971579679, -1.2169760515, -1.43224375134},
-        {-57.218869838, 0.428806849751, 0.654302177028, -1.31651788496, 0.363857431276, -1.49953703016},
-        {53.0877462955, -0.411907760185, -0.192634094071, -0.275879375023, 0.603562526571, 1.16508196734},
-        {-8.11860742896, 1.00263982158, -0.157031169267, -1.11795623393, 0.35711440521, -0.851124640982},
-        {-49.1878248403, -0.0253797866589, -0.574767070714, 0.200339045636, -0.0107042446803, -0.351288977927},
-        {-73.8835407053, -2.07980276724, 1.12235566491, -0.917150593536, 0.741384768556, 0.56229424235},
-        {143.163604045, 0.33627769945, 1.07948757447, 0.894869929963, 1.18688316974, -1.54722487849},
-        {92.7045830908, 0.944091525689, 0.693296229491, 0.700097596814, -1.23666276942, -0.203890113084},
-        {79.1878852355, -0.221973023853, -0.566066329011, 1.57683748648, 0.52854717911, 0.147924782476},
-        {30.6547392801, -1.03466213359, 0.606784904328, -0.298096511956, 0.83332987683, 0.636339018254},
-        {-329.128386019, -1.41363866598, -1.34966434823, -0.989010564149, 0.46889477248, -1.20493210784},
-        {121.190205512, 0.0393914245697, 1.98392444232, -0.65310705226, -0.385899987099, 0.444982471471},
-        {-97.0333075649, 0.264325871992, -0.43074811924, -1.14737761316, -0.453134140655, -0.038507405311},
-        {158.273624516, 0.302255432981, -0.292046617818, 1.0704087606, 0.815965268115, 0.470631083546},
-        {8.24795061818, -1.15155524496, 1.29538707184, -0.4650881541, 0.805123486308, -0.134706887329},
-        {87.1140049059, -0.103540823781, -0.192259440773, 1.79648860085, -1.07525447993, 1.06985127941},
-        {-25.1300772481, -0.97140742052, 0.033393948794, -0.698311192672, 0.74417168942, 0.752776770225},
-        {-285.477057638, -0.480612406803, -1.46081500036, -1.92518386336, -0.426454066275, -0.0539099489597},
-        {-65.1269988498, -1.22733468764, 0.121538452336, 0.752958777557, -0.40643211762, 0.257674949803},
-        {-17.1813504942, 0.823753836891, 0.445142465255, 0.185644700144, -1.99733367514, -0.247899323048},
-        {-46.7543447303, 0.183482778928, -0.934858705943, -1.21961947396, 0.460921844744, 0.571388077177},
-        {-1.7536190499, -0.107517908181, 0.0334282610968, -0.556676121428, -0.485957577159, 0.943570398164},
-        {-42.8460452689, 0.944999215632, 0.00530052154909, -0.348526283976, -1.724125354, -0.122649339813},
-        {62.6291497267, 0.249619894002, 1.3139125969, -1.5644227783, 0.117605482783, 0.304844650662},
-        {97.4552176343, 1.59332799639, -1.17868305562, 1.02998378902, -0.31959491258, -0.183038322076},
-        {-6.19358885758, 0.437951016253, 0.373339269494, -0.204072768495, 0.477969349931, -1.52176449389},
-        {34.0350630099, 0.839319087287, -0.610157662489, 1.73881448393, -1.89200107709, 0.204946415522},
-        {54.9790822536, -0.191792583114, 0.989791127554, -0.502154080064, 0.469939512389, -0.102304071079},
-        {58.8272402843, 0.0769623906454, 0.501297284297, -0.410054999243, 0.595712387781, -0.0968329050729},
-        {95.3620983209, 0.0661481959314, 0.0935137309086, 1.11823292347, -0.612960777903, 0.767865072757},
-        {62.4278196648, 0.78350610065, -1.09977017652, 0.526824784479, 1.41310104196, -0.887902707319},
-        {57.6298676729, 0.60084172954, -0.785932027202, 0.0271301584637, -0.134109499719, 0.877256170191},
-        {5.14112905382, -0.738359365006, 1.40242539359, -0.852833010305, -0.68365080837, 0.88561193696},
-        {11.6057244034, -0.958911227571, 1.15715937023, 1.20108425431, 0.882980929338, -1.77404120156},
-        {-265.758185272, -1.2092434823, -0.0550151798639, 0.00703735243613, -1.01767244359, -1.40616581707},
-        {180.625928828, -0.139091127126, 0.243250756129, 2.17509702585, -0.541735827898, 1.2109459934},
-        {-183.604103216, -0.324555097769, -1.71317286749, 1.03645005723, 0.497569347608, -1.96688185911},
-        {9.93237328848, 0.825483591345, 0.910287997312, -1.64938108528, 0.98964075968, -1.65748940528},
-        {-88.6846949813, -0.0759295112746, -0.593311990101, -0.578711915019, 0.256298822361, -0.429322890198},
-        {175.367391479, 0.9361754906, -0.0172852897292, 1.04078658833, 0.919566407184, -0.554923019093},
-        {-175.538247146, -1.43498590417, 0.37233438556, -0.897205352198, -0.339309952316, -0.0321624527843},
-        {-126.331680318, 0.160446617623, 0.816642363249, -1.39863371652, 0.199747744327, -2.13493607457},
-        {116.677107593, 1.19300905847, -0.404409346893, 0.646338976096, -0.534204093869, 0.36692724765},
-        {-181.675962893, -1.57613169533, -0.41549571451, -0.956673746013, 0.35723782515, 0.318317395128},
-        {-55.1457877823, 0.63723030991, -0.324480386466, 0.296028333894, -1.68117515658, -0.131945601375},
-        {25.2534791013, 0.594818219911, -0.0247380403547, -0.101492246071, -0.0745619242015, -0.370837128867},
-        {63.6006283756, -1.53493473818, 0.946464097439, 0.637741397831, 0.938866921166, 0.54405291856},
-        {-69.6245547661, 0.328482934094, -0.776881060846, -0.285133098443, -1.06107824512, 0.49952182341},
-        {233.425957233, 3.10582399189, -0.0854710508706, 0.455873479133, -0.0974589364949, -1.18914783551},
-        {-86.5564290626, -0.819839276484, 0.584745927593, -0.544737106102, -1.21927675581, 0.758502626434},
-        {425.357285631, 1.70712253847, 1.19892647853, 1.60619661301, 0.36832665241, 0.880791322709},
-        {111.797225426, 0.558940594145, -0.746492420236, 1.90172101792, 0.853590062366, -0.867970723941},
-        {-253.616801014, -0.426513440051, 0.0388582291888, -1.18576061365, -2.70895868242, 0.26982210287},
-        {-394.801501024, -1.65087241498, 0.735525201393, -2.02413077052, -0.96492749037, -1.89014065613}
-    }, new double[] {93.3843533037, 72.3610889215, 57.5295295915, 63.7287541653, 65.2263084024}, 6.85683020686);
-
-    /**
-     * Artificial dataset with 100 observations described by 10 features.
-     */
-    public static final TestDataset regression100x10 = new TestDataset(new double[][] {
-        {69.5794204114, -0.684238565877, 0.175665643732, 0.882115894035, 0.612844187624,
-            -0.685301720572, -0.8266500007, -0.0383407025118, 1.7105205222, 0.457436379836, -0.291563926494},
-        {80.1390102826, -1.80708821811, 0.811271788195, 0.30248512861, 0.910658009566,
-            -1.61869762501, -0.148325085362, -0.0714164596509, 0.671646742271, 2.15160094956, -0.0495754979721},
-        {-156.975447515, 0.170702943934, -0.973403372054, -0.093974528453, 1.54577255871,
-            -0.0969022857972, -1.10639617368, 1.51752480948, -2.86016865032, 1.24063030602, -0.521785751026},
-        {-158.134931891, 0.0890071395055, -0.0811824442353, -0.737354274843, -1.7575255492,
-            0.265777246641, 0.0745347238144, -0.457603542683, -1.37034043839, 1.86011799875, 0.651214189491},
-        {-131.465820263, 0.0767565260375, 0.651724194978, 0.142113799753, 0.244367469855,
-            -0.334395162837, -0.069092305876, -0.691806779713, -1.28386786177, -1.43647491141, 0.00721053414234},
-        {-125.468890054, 0.43361925912, -0.800231440065, -0.576001094593, 0.0783664516431,
-            -1.33613252233, -0.968385062126, -1.22077801286, 0.193456109638, -3.09372314386, 0.817979620215},
-        {-44.1113403874, -0.595796803171, 1.29482131972, -0.784513985654, 0.364702038003,
-            -3.2452492093, -0.451605560847, 0.988546607514, 0.492096628873, -0.343018842342, -0.519231306954},
-        {61.2269707872, -0.0289059337716, -1.00409238976, 0.329908621635, 1.41965097539,
-            0.0395065997587, -0.477939549336, 0.842336765911, -0.808790019648, 1.70241718768, -0.117194118865},
-        {301.434286126, 0.430005308515, 1.01290089725, -0.228221561554, 0.463405921629,
-            -0.602413489517, 1.13832440088, 0.930949226185, -0.196440161506, 1.46304624346, 1.23831509056},
-        {-270.454814681, -1.43805412632, -0.256309572507, -0.358047601174, 0.265151660237,
-            1.07087986377, -1.93784654681, -0.854440691754, 0.665691996289, -1.87508012738, -0.387092423365},
-        {-97.6198688184, -1.67658167161, -0.170246709551, -2.26863722189, 0.280289356338,
-            -0.690038347855, -1.69282684019, 0.978606053022, 1.28237852256, -1.2941998486, 0.766405365374},
-        {-29.5630902399, -1.75615633921, 0.633927486329, -1.24117311555, -0.15884687004,
-            0.31296863712, -1.29513272039, 0.344090683606, 1.19598425093, -1.96195019104, 1.81415061059},
-        {-130.896377427, 0.577719366939, -0.087267771748, -0.060088767013, 0.469803880788,
-            -1.03078212088, -1.41547398887, 1.38980586981, -0.37118000595, -1.81689513712, -0.3099432567},
-        {79.6300698059, 1.23408625633, 1.06464588017, 1.23403332691, -1.10993859098,
-            0.874825200577, 0.589337796957, -1.10266185141, 0.842960469618, -0.89231962021, 0.284074900504},
-        {-154.712112815, -1.64474237898, -0.328581696933, 0.38834343178, 0.02682160335,
-            -0.251167527796, -0.199330632103, -0.0405837345525, -0.908200250794, -1.3283756975, 0.540894408264},
-        {233.447381562, 0.395156450609, 0.156412599781, 0.126453148554, 2.40829068933,
-            1.01623530754, -0.0856520211145, -0.874970377099, 0.280617145254, -0.307070438514, 0.4599616054},
-        {209.012380432, -0.848646647675, 0.558383548084, -0.259628264419, 1.1624126549,
-            -0.0755949979572, -0.373930759448, 0.985903312667, 0.435839508011, -0.760916312668, 1.89847574116},
-        {-39.8987262091, 0.176656582642, 0.508538223618, 0.995038391204, -2.08809409812,
-            0.743926580134, 0.246007971514, -0.458288599906, -0.579976479473, 0.0591577146017, 1.64321662761},
-        {222.078510236, -0.24031989218, -0.168104260522, -0.727838425954, 0.557181757624,
-            -0.164906646307, 2.01559331734, 0.897263594222, 0.0921535309562, 0.351910490325, -0.018228500121},
-        {-250.916272061, -2.71504637339, 0.498966191294, -3.16410707344, -0.842488891776,
-            1.27425275951, 0.0141733666756, 0.695942743199, 0.0917995810179, -0.501447196978, -0.355738068451},
-        {134.07259088, 0.0845637591619, 0.237410106679, -0.291458113729, 1.39418566986,
-            -1.18813057956, -0.683117067763, -0.518910379335, 1.35998426879, -1.28404562245, 0.489131754943},
-        {104.988440209, 0.00770925058526, 0.47113239214, -0.606231247854, 0.310679840217,
-            0.146297599928, 0.732013998647, -0.284544010865, 0.402622530153, -0.0217367745613, 0.0742970687987},
-        {155.558071031, 1.11171654653, 0.726629222799, -0.195820863177, 0.801333855535,
-            0.744034755544, 1.11377275513, -0.75673532139, -0.114117607244, -0.158966474923, -0.29701120385},
-        {90.7600194013, -0.104364079622, -0.0165109945217, 0.933002972987, -1.80652594466,
-            -1.34760892883, -0.304511906801, 0.0584734540581, 1.5332169392, 0.478835797824, 1.71534051065},
-        {-313.910553214, 0.149908925551, 0.232806828559, -0.0708920471592, -0.0649553559745,
-            0.377753357707, -0.957292311668, 0.545360522582, -1.37905464371, -0.940702110994, -1.53620430047},
-        {-80.9380113754, 0.135586606896, 0.95759558815, -1.36879020479, 0.735413996144,
-            0.637984100201, -1.79563152885, 1.55025691631, 0.634702068786, -0.203690334141, -0.83954824721},
-        {-244.336816695, -0.179127343947, -2.12396005014, -0.431179356484, -0.860562153749,
-            -1.10270688639, -0.986886012982, -0.945091656162, -0.445428453767, 1.32269756209, -0.223712672168},
-        {123.069612745, 0.703857129626, 0.291605144784, 1.40233051946, 0.278603787802,
-            -0.693567967466, -0.15587953395, 2.10213915684, 0.130663329174, -0.393184478882, 0.0874812844555},
-        {-148.274944223, 1.66294967732, 0.0830002694123, 0.32492930502, 1.11864359687,
-            -0.381901627785, -1.06367037132, -0.392583620174, -1.16283326187, 0.104931461025, -1.64719611405},
-        {-82.0018788235, 0.497118817453, 0.731125358012, -0.00976413646786, -0.0178930713492,
-            -0.814978582886, 0.0602834712523, -0.661940479055, -0.957902899386, -1.34489251111, 0.22166518707},
-        {-35.742996986, 0.0661349516701, -0.204314495629, 1.17101314753, -2.53846825562,
-            -0.560282479298, -0.393442894828, 0.988953809491, -0.911281277704, 0.86862242698, 2.59576940486},
-        {-109.588885664, -0.0793151346628, -0.408962434518, -0.598817776528, 0.0277205469561,
-            0.116291018958, 0.0280416838086, -0.72544170676, -0.669302814774, 0.0751898759816, -0.311002356179},
-        {57.8285173441, 0.53753903532, 0.676340503752, -2.10608342721, 0.477714987751,
-            0.465695114442, 0.245966562421, -1.05230350808, -0.309794163113, -1.12067331828, 1.07841453304},
-        {204.660622582, -0.717565166685, 0.295179660279, -0.377579912697, 1.88425526905,
-            0.251875238436, -0.900214103232, -1.02877401105, 0.291693915093, 1.24889067987, 1.78506220081},
-        {350.949109103, 2.82276814452, -0.429358342127, 1.12140362367, 1.18120725208,
-            -1.63913834939, 1.61441562446, -0.364003766916, -0.258752942225, -0.808124680189, 0.556463488303},
-        {170.960252153, 0.147245922081, 0.3257117575, 0.211749283649, -0.0150701808404,
-            -0.888523132148, 0.777862088798, 0.296729270892, -0.332927550718, 0.888968144245, 1.20913118467},
-        {112.192270383, 0.129846138824, -0.934371449036, -0.595825303214, 1.74749214629,
-            -0.0500069421443, -0.161976298602, -2.54100791613, 1.99632530735, -0.0691582773758, -0.863939367415},
-        {-56.7847711121, 0.0950532853751, -0.467349228201, -0.26457152362, -0.422134692317,
-            -0.0734763062127, 0.90128235602, -1.68470856275, -0.0699692697335, -0.463335845504, -0.301754321169},
-        {-37.9223252258, -1.40835827778, 0.566142056244, -3.22393318933, 0.228823495106,
-            -1.8480727782, 0.129468321643, -1.77392686536, 0.0112549619662, 0.146433267822, 1.29379901303},
-        {-59.7303066136, 0.835675535576, -0.552173157548, 1.90730898966, -0.520145317195,
-            1.55174485912, -1.37531768692, -0.408165743742, 0.0939675842223, 0.318004128812, 0.324378038446},
-        {-0.916090786983, 0.425763794043, -0.295541268984, -0.066619586336, 2.03494974978,
-            -0.197109278058, -0.823307883209, 0.895531446352, -0.276435938737, -1.54580056755, -0.820051830246},
-        {-20.3601082842, 0.56420556369, 0.741234589387, -0.565853617392, -0.311399905686,
-            2.24066463251, -0.071704904286, -1.22796531596, 0.186020404046, -0.786874824874, 0.23140277151},
-        {-22.9342855182, -0.0682789648279, -1.30680909143, 0.0486490588348, 0.890275695028,
-            -0.257961411112, -0.381531755985, 1.56251482581, -2.11808219232, 0.741828675202, 0.696388901165},
-        {-157.251026807, -2.3120966502, 0.183734662375, 1.02192264962, 0.591272941061,
-            -0.0132855098339, -1.02016546348, 1.19642432892, 0.867653154846, -1.37600041722, -1.08542822792},
-        {-68.6110752055, -1.2429968179, -0.950064269349, -0.332379873336, 0.25793632341,
-            0.145780713577, -0.512109283074, -0.477887632032, 0.448960776324, -0.190215737958, 0.219578347563},
-        {-56.1204152481, -0.811729480846, -0.647410362207, 0.934547463984, -0.390943346216,
-            -0.409981308474, 0.0923465893049, 1.9281242912, -0.624713581674, -0.0599353282306, -0.0188591746808},
-        {348.530651658, 2.51721790231, 0.7560998114, -2.69620396681, 0.5174276585,
-            0.403570816695, 0.901648571306, 0.269313230294, 1.07811463589, 0.986649559679, 0.514710327657},
-        {-105.719065924, 0.679016972998, 0.341319363316, -0.515209647377, 0.800000866847,
-            -0.795474442628, -0.866849274801, -1.32927961486, 0.17679343917, -1.93744422464, -0.476447619273},
-        {-197.389429553, -1.98585668879, -0.962610549884, -2.48860863254, -0.545990524642,
-            -0.13005685654, -1.23413782366, 1.17443427507, 1.4785554038, -0.193717671824, -0.466403609229},
-        {-23.9625285402, -0.392164367603, 1.07583388583, -0.412686712477, -0.89339030785,
-            -0.774862334739, -0.186491999529, -0.300162444329, 0.177377235999, 0.134038296039, 0.957945226616},
-        {-91.145725943, -0.154640540119, 0.732911957939, -0.206326119636, -0.569816760116,
-            0.249393336416, -1.02762332953, 0.25096708081, 0.386927162941, -0.346382299592, 0.243099162109},
-        {-80.7295722208, -1.72670707303, 0.138139045677, 0.0648055728598, 0.186182854422,
-            1.07226527747, -1.26133459043, 0.213883744163, 1.47115466163, -1.54791582859, 0.170924664865},
-        {-317.060323531, -0.349785690206, -0.740759426066, -0.407970845617, -0.689282767277,
-            -1.25608665316, -0.772546119412, -2.02925712813, 0.132949072522, -0.191465137244, -1.29079690284},
-        {-252.491508279, -1.24643122869, 1.55335609203, 0.356613424877, 0.817434495353,
-            -1.74503747683, -0.818046363088, -1.58284235058, 0.357919389759, -1.18942962791, -1.91728745247},
-        {-66.8121363157, -0.584246455697, -0.104254351782, 1.17911687508, -0.29288167882,
-            0.891836132692, 0.232853863255, 0.423294355343, -0.669493690103, -1.15783890498, 0.188213983735},
-        {140.681464689, 1.33156046873, -1.8847915949, -0.666528837988, -0.513356191443,
-            0.281290031669, -1.07815005006, 1.22384196227, 1.39093631269, 0.527644817197, 1.21595221509},
-        {-174.22326767, 0.475428766034, 0.856847216768, -0.734282773151, -0.923514989791,
-            0.917510828772, 0.674878068543, 0.0644776431114, -0.607796192908, 0.867740011912, -1.97799769281},
-        {74.3899799579, 0.00915743526294, 0.553578683413, 1.66930486354, 0.15562803404,
-            1.8455840688, -0.371704942927, 1.11228894843, -0.37464389118, -0.48789151589, 0.79553866342},
-        {70.1167175897, 0.154877045187, 1.47803572976, -0.0355743163524, -2.47914644675,
-            0.672384381837, 1.63160379529, 1.81874583854, 1.22797339421, -0.0131258061634, -0.390265963676},
-        {-11.0364788877, 0.173049156249, -1.78140521797, -1.29982707214, -0.48025663179,
-            -0.469112922302, -1.98718063269, 0.585086542043, 0.264611327837, 1.48855512579, 2.00672263496},
-        {-112.711292736, -1.59239636827, -0.600613018822, -0.0209667499746, -1.81872893331,
-            -0.739893084955, 0.140261888569, -0.498107678308, 2.53664045504, -0.536385019089, -0.608755809378},
-        {-198.064468217, 0.737175509877, -2.01835515547, -2.18045950065, 0.428584922529,
-            -1.01848835019, -0.470645361539, -0.00703630153547, -2.2341302754, 1.51483167022, -0.410184418418},
-        {70.2747963991, 1.49474111532, -0.19517712503, 0.7392852909, -0.326060871666,
-            -0.566710349675, 0.14053094122, -0.562830341306, 0.22931613446, -0.0344439061448, 0.175150510551},
-        {207.909021337, 0.839887009159, 0.268826583246, -0.313047158862, 1.12009996015,
-            0.214209976971, -0.396147338251, 2.16039704403, 0.699141312749, 0.756192350992, -0.145368196901},
-        {169.428609429, -1.13702350819, 1.23964530597, -0.864443556622, -0.885630795949,
-            -0.523872327352, 0.467159824748, 0.476596383923, 0.4343735578, 1.4075417896, 2.22939328991},
-        {-176.909833405, 0.0875512760866, -0.455542269288, 0.539742307764, -0.762003092788,
-            0.41829123457, -0.818116139644, -2.01761645956, 0.557395073218, 1.5823271814, -1.0168826293},
-        {-27.734298611, -0.841257541979, 0.348961259301, 1.36935991472, -0.0694528057586,
-            -1.27303784913, 0.152155656569, 1.9279466651, 0.9589415766, -1.76634370106, -1.08831026428},
-        {-55.8416853588, 0.927711536927, 0.157856746063, -0.295628714893, 0.0296602829783,
-            1.75198587897, -0.38285446366, -0.253287154535, -1.64032395229, -0.842089054965, 1.00493779183},
-        {56.0899797005, 0.326117761734, -1.93514762146, 1.0229172721, 0.125568968732,
-            2.37760000658, -0.498532972011, -0.733375842271, -0.757445726993, -0.49515057432, 2.01559891524},
-        {-176.220234909, 1.571129843, -0.867707605929, -0.709690799512, -1.51535538937,
-            1.27424225477, -0.109513704468, -1.46822183, 0.281077088939, -1.97084024232, -0.322309524179},
-        {37.7155152941, 0.363383774219, -0.0240881298641, -1.60692745228, -1.26961656439,
-            -0.41299134216, 1.2890099968, -1.34101694629, -0.455387485256, -0.14055003482, 1.5407059956},
-        {-102.163416997, -2.05927378316, -0.470182865756, -0.875528863204, 0.0361720859253,
-            -1.03713912263, 0.417362606334, 0.707587625276, -0.0591627772581, -2.58905252006, 0.516573345216},
-        {-206.47095321, 0.270030584651, 1.85544202116, -0.144189208964, -0.696400687327,
-            0.0226388634283, -0.490952489106, -1.69209527849, 0.00973614309272, -0.484105876992, -0.991474668217},
-        {201.50637416, 0.513659215697, -0.335630132208, -0.140006500483, 0.149679720127,
-            -1.89526167503, -0.0614973894156, 0.0813221153552, 0.630952530848, 2.40201011339, 0.997708264073},
-        {-72.0667371571, 0.0841570292899, -0.216125859013, -1.77155215764, 2.15081767322,
-            0.00953341785443, -1.0826077946, -0.791135571106, -0.989393577892, -0.791485083644, -0.063560999686},
-        {-162.903837815, -0.273764637097, 0.282387854873, -1.39881596931, 0.554941097854,
-            -0.88790718926, -0.693189960902, 0.398762630571, -1.61878562893, -0.345976341096, 0.138298909959},
-        {-34.3291926715, -0.499883755911, -0.847296893019, -0.323673126437, 0.531205373462,
-            -0.0204345595983, 0.284954510306, 0.565031773028, -0.272049818708, -0.130369799738, -0.617572026201},
-        {76.1272883187, -0.908810282403, -1.04139421904, 0.890678872055, 1.32990256154,
-            -0.0150445428835, 0.593918101047, 0.356897732999, 0.824651162423, -1.54544256217, -0.795703905296},
-        {171.833705285, -0.0425219657568, -0.884042952325, 1.91202504537, 0.381908223898,
-            -0.205693527739, 1.53656598237, 0.534880398015, 0.291950716831, -1.1258051056, -0.0612803476297},
-        {-235.445792009, 0.261252102941, -0.170931758001, 1.67878144235, 0.0278283741792,
-            -1.23194408479, -0.190931886594, 1.0000157972, -2.18792142659, -0.230654984288, -1.36626493512},
-        {348.968834231, 1.35713154434, 0.950377770072, 0.0700577471848, 0.96907140156,
-            2.00890422081, 0.0896405239806, 0.614309607351, 1.07723409067, 2.58506968136, 0.202889806148},
-        {-61.0128039201, 0.465438505031, -1.31448530533, 0.374781933416, -0.0118298606041,
-            -0.477338357738, -0.587656108109, 1.66449545077, 0.435836048385, -0.287027953004, -1.06613472784},
-        {-50.687090469, 0.382331825989, -0.597140322197, 1.1276065465, -1.35593777887,
-            1.14949964423, -0.858742432885, -0.563211485633, -0.57167161928, 0.0294891749132, 1.9571639493},
-        {-186.653649045, -0.00981380006029, 1.0371088941, -1.25319048981, -0.694043021068,
-            1.7280802541, -0.191210409232, -0.866039238001, -0.0791927416078, -0.232228656558, -0.93723545053},
-        {34.5395591744, 0.680943971029, -0.075875481801, -0.144408300848, -0.869070791528,
-            0.496870904214, 1.0940401388, -0.510489750436, -0.47562728601, 0.951406841944, 0.12983846382},
-        {-23.7618645627, 0.527032820313, -0.58295129357, -0.3894567306, -0.0547905472556,
-            -1.86103603537, 0.0506988360667, 1.02778539291, -0.0613720063422, 0.411280841442, -0.665810811374},
-        {116.007776415, 0.441750249008, 0.549342185228, 0.731558201455, -0.903624700864,
-            -2.13208328824, 0.381223328983, 0.283479210749, 1.17705098922, -2.38800904207, 1.32108350152},
-        {-148.479593311, -0.814604260049, -0.821204361946, -1.08768677334, -0.0659445766599,
-            0.583741297405, 0.669345853296, -0.0935352010726, -0.254906787938, -0.394599725657, -1.26305927257},
-        {244.865845084, 0.776784257443, 0.267205388558, 2.37746488031, -0.379275360853,
-            -0.157454754411, -0.359580726073, 0.886887721861, 1.53707627973, 0.634390546684, 0.984864824122},
-        {-81.9954096721, 0.594841146008, -1.22273253129, 0.532466794358, 1.69864239257,
-            -0.12293671327, -2.06645974171, 0.611808231703, -1.32291985291, 0.722066660478, -0.0021343848511},
-        {-245.715046329, -1.77850303496, -0.176518810079, 1.20463434525, -0.597826204963,
-            -1.45842350123, -0.765730251727, -2.17764204443, 0.12996635702, -0.705509516482, 0.170639846082},
-        {123.011946043, -0.909707162714, 0.92357208515, 0.373251929121, 1.24629576577,
-            0.0662688299998, -0.372240547929, -0.739353735168, 0.323495756066, 0.954154005738, 0.69606859977},
-        {-70.4564963177, 0.650682297051, 0.378131376232, 1.37860253614, -0.924042783872,
-            0.802851073842, -0.450299927542, 0.235646185302, -0.148779896161, 1.01308126122, -0.48206889502},
-        {21.5288687935, 0.290876355386, 0.0765702960599, 0.905225489744, 0.252841861521,
-            1.26729272819, 0.315397441908, -2.00317261368, -0.250990653758, 0.425615332405, 0.0875320802483},
-        {231.370169905, 0.535138021352, -1.07151617232, 0.824383756287, 1.84428896701,
-            -0.890892034494, 0.0480296332924, -0.59251208055, 0.267564961845, -0.230698441998, 0.857077278291},
-        {38.8318274023, 2.63547217711, -0.585553060394, 0.430550920323, -0.532619160993,
-            1.25335488136, -1.65265278435, 0.0433880112291, -0.166143379872, 0.534066441314, 1.18929937797},
-        {116.362219013, -0.275949982433, 0.468069787645, -0.879814121059, 0.862799331322,
-            1.18464846725, 0.747084253268, 1.39202500691, -1.23374181275, 0.0949815110503, 0.696546907194},
-        {260.540154731, 1.13798788241, -0.0991903174656, 0.1241636043, -0.201415073037,
-            1.57683389508, 1.81535629587, 1.07873616646, -0.355800782882, 2.18333193195, 0.0711071144615},
-        {-165.835194521, -2.76613178307, 0.805314338858, 0.81526046683, -0.710489036197,
-            -1.20189542317, -0.692110074722, -0.117239516622, 1.0431459458, -0.111898596299, -0.0775811519297},
-        {-341.189958588, 0.668555635008, -1.0940034941, -0.497881262778, -0.603682823779,
-            -0.396875163796, -0.849144848521, 0.403936807183, -1.82076277475, -0.137500972546, -1.22769896568}
-    }, new double[] {45.8685095528, 11.9400336005, 16.3984976652, 79.9069814034, 5.65486853464,
-        83.6427296424, 27.4571268153, 73.5881193584, 27.1465364511, 79.4095449062}, -5.14077007134);
-
-    /** */
-    public static class TestDataset {
-
-        /** */
-        private final double[][] data;
-
-        /** */
-        private final double[] expWeights;
-
-        /** */
-        private final double expIntercept;
-
-        /** */
-        TestDataset(double[][] data, double[] expWeights, double expIntercept) {
-            this.data = data;
-            this.expWeights = expWeights;
-            this.expIntercept = expIntercept;
-        }
-
-        /** */
-        public double[][] getData() {
-            return data;
-        }
-
-        /** */
-        public double[] getExpWeights() {
-            return expWeights;
-        }
-
-        /** */
-        public double getExpIntercept() {
-            return expIntercept;
-        }
-    }
-}
\ No newline at end of file
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionQRTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionQRTrainerTest.java
deleted file mode 100644
index 0c09d75dd8314..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/BlockDistributedLinearRegressionQRTrainerTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.ml.math.impls.matrix.SparseBlockDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.SparseBlockDistributedVector;
-
-/**
- * Tests for {@link LinearRegressionQRTrainer} on {@link SparseBlockDistributedMatrix}.
- */
-public class BlockDistributedLinearRegressionQRTrainerTest extends GridAwareAbstractLinearRegressionTrainerTest {
-    /** */
-    public BlockDistributedLinearRegressionQRTrainerTest() {
-        super(
-            new LinearRegressionQRTrainer(),
-            SparseBlockDistributedMatrix::new,
-            SparseBlockDistributedVector::new,
-            1e-6
-        );
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionQRTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionQRTrainerTest.java
deleted file mode 100644
index 2a506d9410c86..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/DistributedLinearRegressionQRTrainerTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.SparseDistributedVector;
-
-/**
- * Tests for {@link LinearRegressionQRTrainer} on {@link SparseDistributedMatrix}.
- */
-public class DistributedLinearRegressionQRTrainerTest extends GridAwareAbstractLinearRegressionTrainerTest {
-    /** */
-    public DistributedLinearRegressionQRTrainerTest() {
-        super(
-            new LinearRegressionQRTrainer(),
-            SparseDistributedMatrix::new,
-            SparseDistributedVector::new,
-            1e-6
-        );
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GenericLinearRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GenericLinearRegressionTrainerTest.java
deleted file mode 100644
index a55623ca15200..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GenericLinearRegressionTrainerTest.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import java.util.Scanner;
-import org.apache.ignite.ml.TestUtils;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.junit.Test;
-
-/**
- * Base class for all linear regression trainers.
- */
-public class GenericLinearRegressionTrainerTest {
-    /** */
-    private final Trainer<LinearRegressionModel, Matrix> trainer;
-
-    /** */
-    private final IgniteFunction<double[][], Matrix> matrixCreator;
-
-    /** */
-    private final IgniteFunction<double[], Vector> vectorCreator;
-
-    /** */
-    private final double precision;
-
-    /** */
-    public GenericLinearRegressionTrainerTest(
-        Trainer<LinearRegressionModel, Matrix> trainer,
-        IgniteFunction<double[][], Matrix> matrixCreator,
-        IgniteFunction<double[], Vector> vectorCreator,
-        double precision) {
-        this.trainer = trainer;
-        this.matrixCreator = matrixCreator;
-        this.vectorCreator = vectorCreator;
-        this.precision = precision;
-    }
-
-    /**
-     * Test trainer on regression model y = 2 * x.
-     */
-    @Test
-    public void testTrainWithoutIntercept() {
-        Matrix data = matrixCreator.apply(new double[][] {
-            {2.0, 1.0},
-            {4.0, 2.0}
-        });
-
-        LinearRegressionModel mdl = trainer.train(data);
-
-        TestUtils.assertEquals(4, mdl.apply(vectorCreator.apply(new double[] {2})), precision);
-        TestUtils.assertEquals(6, mdl.apply(vectorCreator.apply(new double[] {3})), precision);
-        TestUtils.assertEquals(8, mdl.apply(vectorCreator.apply(new double[] {4})), precision);
-    }
-
-    /**
-     * Test trainer on regression model y = -1 * x + 1.
-     */
-    @Test
-    public void testTrainWithIntercept() {
-        Matrix data = matrixCreator.apply(new double[][] {
-            {1.0, 0.0},
-            {0.0, 1.0}
-        });
-
-        LinearRegressionModel mdl = trainer.train(data);
-
-        TestUtils.assertEquals(0.5, mdl.apply(vectorCreator.apply(new double[] {0.5})), precision);
-        TestUtils.assertEquals(2, mdl.apply(vectorCreator.apply(new double[] {-1})), precision);
-        TestUtils.assertEquals(-1, mdl.apply(vectorCreator.apply(new double[] {2})), precision);
-    }
-
-    /**
-     * Test trainer on diabetes dataset.
-     */
-    @Test
-    public void testTrainOnDiabetesDataset() {
-        Matrix data = loadDataset("datasets/regression/diabetes.csv", 442, 10);
-
-        LinearRegressionModel mdl = trainer.train(data);
-
-        Vector expWeights = vectorCreator.apply(new double[] {
-            -10.01219782, -239.81908937, 519.83978679, 324.39042769, -792.18416163,
-            476.74583782, 101.04457032, 177.06417623, 751.27932109, 67.62538639
-        });
-
-        double expIntercept = 152.13348416;
-
-        TestUtils.assertEquals("Wrong weights", expWeights, mdl.getWeights(), precision);
-        TestUtils.assertEquals("Wrong intercept", expIntercept, mdl.getIntercept(), precision);
-    }
-
-    /**
-     * Test trainer on boston dataset.
-     */
-    @Test
-    public void testTrainOnBostonDataset() {
-        Matrix data = loadDataset("datasets/regression/boston.csv", 506, 13);
-
-        LinearRegressionModel mdl = trainer.train(data);
-
-        Vector expWeights = vectorCreator.apply(new double[] {
-            -1.07170557e-01, 4.63952195e-02, 2.08602395e-02, 2.68856140e+00, -1.77957587e+01, 3.80475246e+00,
-            7.51061703e-04, -1.47575880e+00, 3.05655038e-01, -1.23293463e-02, -9.53463555e-01, 9.39251272e-03,
-            -5.25466633e-01
-        });
-
-        double expIntercept = 36.4911032804;
-
-        TestUtils.assertEquals("Wrong weights", expWeights, mdl.getWeights(), precision);
-        TestUtils.assertEquals("Wrong intercept", expIntercept, mdl.getIntercept(), precision);
-    }
-
-    /**
-     * Tests trainer on artificial dataset with 10 observations described by 1 feature.
-     */
-    @Test
-    public void testTrainOnArtificialDataset10x1() {
-        ArtificialRegressionDatasets.TestDataset dataset = ArtificialRegressionDatasets.regression10x1;
-
-        LinearRegressionModel mdl = trainer.train(matrixCreator.apply(dataset.getData()));
-
-        TestUtils.assertEquals("Wrong weights", dataset.getExpWeights(), mdl.getWeights(), precision);
-        TestUtils.assertEquals("Wrong intercept", dataset.getExpIntercept(), mdl.getIntercept(), precision);
-    }
-
-    /**
-     * Tests trainer on artificial dataset with 10 observations described by 5 features.
-     */
-    @Test
-    public void testTrainOnArtificialDataset10x5() {
-        ArtificialRegressionDatasets.TestDataset dataset = ArtificialRegressionDatasets.regression10x5;
-
-        LinearRegressionModel mdl = trainer.train(matrixCreator.apply(dataset.getData()));
-
-        TestUtils.assertEquals("Wrong weights", dataset.getExpWeights(), mdl.getWeights(), precision);
-        TestUtils.assertEquals("Wrong intercept", dataset.getExpIntercept(), mdl.getIntercept(), precision);
-    }
-
-    /**
-     * Tests trainer on artificial dataset with 100 observations described by 5 features.
-     */
-    @Test
-    public void testTrainOnArtificialDataset100x5() {
-        ArtificialRegressionDatasets.TestDataset dataset = ArtificialRegressionDatasets.regression100x5;
-
-        LinearRegressionModel mdl = trainer.train(matrixCreator.apply(dataset.getData()));
-
-        TestUtils.assertEquals("Wrong weights", dataset.getExpWeights(), mdl.getWeights(), precision);
-        TestUtils.assertEquals("Wrong intercept", dataset.getExpIntercept(), mdl.getIntercept(), precision);
-    }
-
-    /**
-     * Tests trainer on artificial dataset with 100 observations described by 10 features.
-     */
-    @Test
-    public void testTrainOnArtificialDataset100x10() {
-        ArtificialRegressionDatasets.TestDataset dataset = ArtificialRegressionDatasets.regression100x10;
-
-        LinearRegressionModel mdl = trainer.train(matrixCreator.apply(dataset.getData()));
-
-        TestUtils.assertEquals("Wrong weights", dataset.getExpWeights(), mdl.getWeights(), precision);
-        TestUtils.assertEquals("Wrong intercept", dataset.getExpIntercept(), mdl.getIntercept(), precision);
-    }
-
-    /**
-     * Loads dataset file and returns corresponding matrix.
-     *
-     * @param fileName Dataset file name
-     * @param nobs Number of observations
-     * @param nvars Number of features
-     * @return Data matrix
-     */
-    private Matrix loadDataset(String fileName, int nobs, int nvars) {
-        double[][] matrix = new double[nobs][nvars + 1];
-        Scanner scanner = new Scanner(this.getClass().getClassLoader().getResourceAsStream(fileName));
-        int i = 0;
-        while (scanner.hasNextLine()) {
-            String row = scanner.nextLine();
-            int j = 0;
-            for (String feature : row.split(",")) {
-                matrix[i][j] = Double.parseDouble(feature);
-                j++;
-            }
-            i++;
-        }
-        return matrixCreator.apply(matrix);
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java
deleted file mode 100644
index 9b75bd41392a6..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/GridAwareAbstractLinearRegressionTrainerTest.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.Trainer;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.junit.Test;
-
-/**
- * Grid aware abstract linear regression trainer test.
- */
-public abstract class GridAwareAbstractLinearRegressionTrainerTest extends GridCommonAbstractTest {
-    /** Number of nodes in grid */
-    private static final int NODE_COUNT = 3;
-
-    /**
-     * Delegate actually performs tests.
-     */
-    private final GenericLinearRegressionTrainerTest delegate;
-
-    /** */
-    private Ignite ignite;
-
-    /** */
-    public GridAwareAbstractLinearRegressionTrainerTest(
-        Trainer<LinearRegressionModel, Matrix> trainer,
-        IgniteFunction<double[][], Matrix> matrixCreator,
-        IgniteFunction<double[], Vector> vectorCreator,
-        double precision) {
-        delegate = new GenericLinearRegressionTrainerTest(trainer, matrixCreator, vectorCreator, precision);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        for (int i = 1; i <= NODE_COUNT; i++)
-            startGrid(i);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() {
-        stopAllGrids();
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override protected void beforeTest() throws Exception {
-        /* Grid instance. */
-        ignite = grid(NODE_COUNT);
-        ignite.configuration().setPeerClassLoadingEnabled(true);
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-    }
-
-    /**
-     * Test trainer on regression model y = 2 * x.
-     */
-    @Test
-    public void testTrainWithoutIntercept() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        delegate.testTrainWithoutIntercept();
-    }
-
-    /**
-     * Test trainer on regression model y = -1 * x + 1.
-     */
-    @Test
-    public void testTrainWithIntercept() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        delegate.testTrainWithIntercept();
-    }
-
-    /**
-     * Tests trainer on artificial dataset with 10 observations described by 1 feature.
-     */
-    @Test
-    public void testTrainOnArtificialDataset10x1() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        delegate.testTrainOnArtificialDataset10x1();
-    }
-
-    /**
-     * Tests trainer on artificial dataset with 10 observations described by 5 features.
-     */
-    @Test
-    public void testTrainOnArtificialDataset10x5() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        delegate.testTrainOnArtificialDataset10x5();
-    }
-
-    /**
-     * Tests trainer on artificial dataset with 100 observations described by 5 features.
-     */
-    @Test
-    public void testTrainOnArtificialDataset100x5() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        delegate.testTrainOnArtificialDataset100x5();
-    }
-
-    /**
-     * Tests trainer on artificial dataset with 100 observations described by 10 features.
-     */
-    @Test
-    public void testTrainOnArtificialDataset100x10() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-        delegate.testTrainOnArtificialDataset100x10();
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
index fa8fac408b112..c62cca5e9ea90 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LinearRegressionSGDTrainerTest.java
@@ -19,7 +19,7 @@
 
 import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
 import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
-import org.apache.ignite.ml.trainers.group.UpdatesStrategy;
+import org.apache.ignite.ml.nn.UpdatesStrategy;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionQRTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionQRTrainerTest.java
deleted file mode 100644
index f37d71d0c8b9f..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/linear/LocalLinearRegressionQRTrainerTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.regressions.linear;
-
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/**
- * Tests for {@link LinearRegressionQRTrainer} on {@link DenseLocalOnHeapMatrix}.
- */
-public class LocalLinearRegressionQRTrainerTest extends GenericLinearRegressionTrainerTest {
-    /** */
-    public LocalLinearRegressionQRTrainerTest() {
-        super(
-            new LinearRegressionQRTrainer(),
-            DenseLocalOnHeapMatrix::new,
-            DenseLocalOnHeapVector::new,
-            1e-6
-        );
-    }
-}
\ No newline at end of file
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/DistributedWorkersChainTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/DistributedWorkersChainTest.java
deleted file mode 100644
index 7ad59d168e099..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/DistributedWorkersChainTest.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.chain.Chains;
-import org.apache.ignite.ml.trainers.group.chain.ComputationsChain;
-import org.apache.ignite.ml.trainers.group.chain.EntryAndContext;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.junit.Assert;
-
-/** */
-public class DistributedWorkersChainTest extends GridCommonAbstractTest {
-    /** Count of nodes. */
-    private static final int NODE_COUNT = 3;
-
-    /** Grid instance. */
-    protected Ignite ignite;
-
-    /**
-     * Default constructor.
-     */
-    public DistributedWorkersChainTest() {
-        super(false);
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override protected void beforeTest() throws Exception {
-        ignite = grid(NODE_COUNT);
-        TestGroupTrainingCache.getOrCreate(ignite).removeAll();
-        TestGroupTrainingSecondCache.getOrCreate(ignite).removeAll();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        for (int i = 1; i <= NODE_COUNT; i++)
-            startGrid(i);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        stopAllGrids();
-    }
-
-    /** */
-    public void testId() {
-        ComputationsChain<TestLocalContext, Double, Integer, Integer, Integer> chain = Chains.create();
-
-        UUID trainingUUID = UUID.randomUUID();
-        Integer res = chain.process(1, new GroupTrainingContext<>(new TestLocalContext(0, trainingUUID), TestGroupTrainingCache.getOrCreate(ignite), ignite));
-
-        Assert.assertEquals(1L, (long)res);
-    }
-
-    /** */
-    public void testSimpleLocal() {
-        ComputationsChain<TestLocalContext, Double, Integer, Integer, Integer> chain = Chains.create();
-
-        IgniteCache<GroupTrainerCacheKey<Double>, Integer> cache = TestGroupTrainingCache.getOrCreate(ignite);
-        int init = 1;
-        int initLocCtxData = 0;
-        UUID trainingUUID = UUID.randomUUID();
-        TestLocalContext locCtx = new TestLocalContext(initLocCtxData, trainingUUID);
-
-        Integer res = chain.
-            thenLocally((prev, lc) -> prev + 1).
-            process(init, new GroupTrainingContext<>(locCtx, cache, ignite));
-
-        Assert.assertEquals(init + 1, (long)res);
-        Assert.assertEquals(initLocCtxData, locCtx.data());
-    }
-
-    /** */
-    public void testChainLocal() {
-        ComputationsChain<TestLocalContext, Double, Integer, Integer, Integer> chain = Chains.create();
-
-        IgniteCache<GroupTrainerCacheKey<Double>, Integer> cache = TestGroupTrainingCache.getOrCreate(ignite);
-        int init = 1;
-        int initLocCtxData = 0;
-        UUID trainingUUID = UUID.randomUUID();
-        TestLocalContext locCtx = new TestLocalContext(initLocCtxData, trainingUUID);
-
-        Integer res = chain.
-            thenLocally((prev, lc) -> prev + 1).
-            thenLocally((prev, lc) -> prev * 5).
-            process(init, new GroupTrainingContext<>(locCtx, cache, ignite));
-
-        Assert.assertEquals((init + 1) * 5, (long)res);
-        Assert.assertEquals(initLocCtxData, locCtx.data());
-    }
-
-    /** */
-    public void testChangeLocalContext() {
-        ComputationsChain<TestLocalContext, Double, Integer, Integer, Integer> chain = Chains.create();
-        IgniteCache<GroupTrainerCacheKey<Double>, Integer> cache = TestGroupTrainingCache.getOrCreate(ignite);
-        int init = 1;
-        int newData = 10;
-        UUID trainingUUID = UUID.randomUUID();
-        TestLocalContext locCtx = new TestLocalContext(0, trainingUUID);
-
-        Integer res = chain.
-            thenLocally((prev, lc) -> { lc.setData(newData); return prev;}).
-            process(init, new GroupTrainingContext<>(locCtx, cache, ignite));
-
-        Assert.assertEquals(newData, locCtx.data());
-        Assert.assertEquals(init, res.intValue());
-    }
-
-    /** */
-    public void testDistributed() {
-        ComputationsChain<TestLocalContext, Double, Integer, Integer, Integer> chain = Chains.create();
-        IgniteCache<GroupTrainerCacheKey<Double>, Integer> cache = TestGroupTrainingCache.getOrCreate(ignite);
-        int init = 1;
-        UUID trainingUUID = UUID.randomUUID();
-        TestLocalContext locCtx = new TestLocalContext(0, trainingUUID);
-
-        Map<GroupTrainerCacheKey<Double>, Integer> m = new HashMap<>();
-        m.put(new GroupTrainerCacheKey<>(0L, 1.0, trainingUUID), 1);
-        m.put(new GroupTrainerCacheKey<>(1L, 2.0, trainingUUID), 2);
-        m.put(new GroupTrainerCacheKey<>(2L, 3.0, trainingUUID), 3);
-        m.put(new GroupTrainerCacheKey<>(3L, 4.0, trainingUUID), 4);
-
-        Stream<GroupTrainerCacheKey<Double>> keys = m.keySet().stream();
-
-        cache.putAll(m);
-
-        IgniteBiFunction<Integer, TestLocalContext, IgniteSupplier<Stream<GroupTrainerCacheKey<Double>>>> function = (o, l) -> () -> keys;
-        IgniteFunction<List<Integer>, Integer> max = ints -> ints.stream().mapToInt(x -> x).max().orElse(Integer.MIN_VALUE);
-
-        Integer res = chain.
-            thenDistributedForEntries((integer, context) -> () -> null, this::readAndIncrement, function, max).
-            process(init, new GroupTrainingContext<>(locCtx, cache, ignite));
-
-        int localMax = m.values().stream().max(Comparator.comparingInt(i -> i)).orElse(Integer.MIN_VALUE);
-
-        assertEquals((long)localMax, (long)res);
-
-        for (GroupTrainerCacheKey<Double> key : m.keySet())
-            m.compute(key, (k, v) -> v + 1);
-
-        assertMapEqualsCache(m, cache);
-    }
-
-    /** */
-    private ResultAndUpdates<Integer> readAndIncrement(EntryAndContext<Double, Integer, Void> ec) {
-        Integer val = ec.entry().getValue();
-
-        ResultAndUpdates<Integer> res = ResultAndUpdates.of(val);
-        res.updateCache(TestGroupTrainingCache.getOrCreate(Ignition.localIgnite()), ec.entry().getKey(), val + 1);
-
-        return res;
-    }
-
-    /** */
-    private <K, V> void assertMapEqualsCache(Map<K, V> m, IgniteCache<K, V> cache) {
-        assertEquals(m.size(), cache.size());
-
-        for (K k : m.keySet())
-            assertEquals(m.get(k), cache.get(k));
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/GroupTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/GroupTrainerTest.java
deleted file mode 100644
index 5bb9a4706b7b7..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/GroupTrainerTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Test of {@link GroupTrainer}.
- */
-public class GroupTrainerTest extends GridCommonAbstractTest {
-    /** Count of nodes. */
-    private static final int NODE_COUNT = 3;
-
-    /** Grid instance. */
-    private Ignite ignite;
-
-    /**
-     * Default constructor.
-     */
-    public GroupTrainerTest() {
-        super(false);
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override protected void beforeTest() throws Exception {
-        ignite = grid(NODE_COUNT);
-        TestGroupTrainingCache.getOrCreate(ignite).removeAll();
-        TestGroupTrainingSecondCache.getOrCreate(ignite).removeAll();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        for (int i = 1; i <= NODE_COUNT; i++)
-            startGrid(i);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        stopAllGrids();
-    }
-
-    /** */
-    public void testGroupTrainer() {
-        TestGroupTrainer trainer = new TestGroupTrainer(ignite);
-
-        int limit = 5;
-        int eachNumCnt = 3;
-        int iterCnt = 2;
-
-        ConstModel<Integer> mdl = trainer.train(new SimpleGroupTrainerInput(limit, eachNumCnt, iterCnt));
-        int locRes = computeLocally(limit, eachNumCnt, iterCnt);
-        assertEquals(locRes, (int)mdl.apply(10));
-    }
-
-    /** */
-    private int computeLocally(int limit, int eachNumCnt, int iterCnt) {
-        Map<GroupTrainerCacheKey<Double>, Integer> m = new HashMap<>();
-
-        for (int i = 0; i < limit; i++) {
-            for (int j = 0; j < eachNumCnt; j++)
-                m.put(new GroupTrainerCacheKey<>(i, (double)j, null), i);
-        }
-
-        for (int i = 0; i < iterCnt; i++)
-            for (GroupTrainerCacheKey<Double> key : m.keySet())
-                m.compute(key, (key1, integer) -> integer * integer);
-
-        return m.values().stream().filter(x -> x % 2 == 0).mapToInt(i -> i).sum();
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/SimpleGroupTrainerInput.java b/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/SimpleGroupTrainerInput.java
deleted file mode 100644
index db1adc74b81cf..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/SimpleGroupTrainerInput.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.UUID;
-import java.util.stream.IntStream;
-import java.util.stream.Stream;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-
-public class SimpleGroupTrainerInput implements GroupTrainerInput<Double> {
-    /** */
-    private int limit;
-
-    /** */
-    private int eachNumberCount;
-
-    /** */
-    private int iterCnt;
-
-    /** */
-    public SimpleGroupTrainerInput(int limit, int eachNumCnt, int iterCnt) {
-        this.limit = limit;
-        this.eachNumberCount = eachNumCnt;
-        this.iterCnt = iterCnt;
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteSupplier<Stream<GroupTrainerCacheKey<Double>>> initialKeys(UUID trainingUUID) {
-        int lim = limit;
-        UUID uuid = trainingUUID;
-        return () -> IntStream.range(0, lim).mapToObj(i -> new GroupTrainerCacheKey<>(i, 0.0, uuid));
-    }
-
-    /** */
-    public int limit() {
-        return limit;
-    }
-
-    /** */
-    public int iterCnt() {
-        return iterCnt;
-    }
-
-    /** */
-    public int eachNumberCount() {
-        return eachNumberCount;
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainer.java b/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainer.java
deleted file mode 100644
index 0a49fe0fd7b74..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainer.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.chain.Chains;
-import org.apache.ignite.ml.trainers.group.chain.ComputationsChain;
-import org.apache.ignite.ml.trainers.group.chain.EntryAndContext;
-
-/**
- * Test group trainer.
- */
-public class TestGroupTrainer extends GroupTrainer<TestGroupTrainerLocalContext, Double, Integer, Integer, Integer,
-    Double, ConstModel<Integer>, SimpleGroupTrainerInput, Void> {
-    /**
-     * Construct instance of this class with given parameters.
-     *
-     * @param ignite Ignite instance.
-     */
-    public TestGroupTrainer(Ignite ignite) {
-        super(TestGroupTrainingCache.getOrCreate(ignite), ignite);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected TestGroupTrainerLocalContext initialLocalContext(SimpleGroupTrainerInput data,
-        UUID trainingUUID) {
-        return new TestGroupTrainerLocalContext(data.iterCnt(), data.eachNumberCount(), data.limit(), trainingUUID);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected IgniteFunction<GroupTrainerCacheKey<Double>, ResultAndUpdates<Integer>> distributedInitializer(
-        SimpleGroupTrainerInput data) {
-        return key -> {
-            long i = key.nodeLocalEntityIndex();
-            UUID trainingUUID = key.trainingUUID();
-            IgniteCache<GroupTrainerCacheKey<Double>, Integer> cache
-                = TestGroupTrainingCache.getOrCreate(Ignition.localIgnite());
-
-            long sum = i * data.eachNumberCount();
-
-            ResultAndUpdates<Integer> res = ResultAndUpdates.of((int)sum);
-
-            for (int j = 0; j < data.eachNumberCount(); j++)
-                res.updateCache(cache, new GroupTrainerCacheKey<>(i, (double)j, trainingUUID), (int)i);
-
-            return res;
-        };
-    }
-
-    /** {@inheritDoc} */
-    @Override protected IgniteFunction<List<Integer>, Integer> reduceDistributedInitData() {
-        return id -> id.stream().mapToInt(x -> x).sum();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected Double locallyProcessInitData(Integer data, TestGroupTrainerLocalContext locCtx) {
-        return data.doubleValue();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected ComputationsChain<TestGroupTrainerLocalContext,
-        Double, Integer, Double, Double> trainingLoopStep() {
-        // TODO:IGNITE-7405 here we should explicitly create variable because we cannot infer context type, think about it.
-        ComputationsChain<TestGroupTrainerLocalContext, Double, Integer, Double, Double> chain = Chains.
-            create(new TestTrainingLoopStep());
-        return chain.
-            thenLocally((aDouble, context) -> {
-                context.incCnt();
-                return aDouble;
-            });
-    }
-
-    /** {@inheritDoc} */
-    @Override protected boolean shouldContinue(Double data, TestGroupTrainerLocalContext locCtx) {
-        return locCtx.cnt() < locCtx.maxCnt();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected IgniteSupplier<Void> extractContextForFinalResultCreation(Double data,
-        TestGroupTrainerLocalContext locCtx) {
-        // No context is needed.
-        return () -> null;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected IgniteSupplier<Stream<GroupTrainerCacheKey<Double>>> finalResultKeys(Double data,
-        TestGroupTrainerLocalContext locCtx) {
-        int limit = locCtx.limit();
-        int cnt = locCtx.eachNumberCnt();
-        UUID uuid = locCtx.trainingUUID();
-
-        return () -> TestGroupTrainingCache.allKeys(limit, cnt, uuid);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected IgniteFunction<EntryAndContext<Double, Integer, Void>,
-        ResultAndUpdates<Integer>> finalResultsExtractor() {
-        return entryAndCtx -> {
-            Integer val = entryAndCtx.entry().getValue();
-            return ResultAndUpdates.of(val % 2 == 0 ? val : 0);
-        };
-    }
-
-    /** {@inheritDoc} */
-    @Override protected IgniteFunction<List<Integer>, Integer> finalResultsReducer() {
-        return id -> id.stream().mapToInt(x -> x).sum();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected ConstModel<Integer> mapFinalResult(Integer res, TestGroupTrainerLocalContext locCtx) {
-        return new ConstModel<>(res);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void cleanup(TestGroupTrainerLocalContext locCtx) {
-        Stream<GroupTrainerCacheKey<Double>> toRemote = TestGroupTrainingCache.allKeys(locCtx.limit(),
-            locCtx.eachNumberCnt(), locCtx.trainingUUID());
-
-        TestGroupTrainingCache.getOrCreate(ignite).removeAll(toRemote.collect(Collectors.toSet()));
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainerLocalContext.java b/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainerLocalContext.java
deleted file mode 100644
index e1a533b16761f..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainerLocalContext.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.UUID;
-import org.apache.ignite.ml.trainers.group.chain.HasTrainingUUID;
-
-/** */
-public class TestGroupTrainerLocalContext implements HasTrainingUUID {
-    /** */
-    private int cnt = 0;
-
-    /** */
-    private int maxCnt;
-
-    /** */
-    private int eachNumberCnt;
-
-    /** */
-    private int limit;
-
-    /** */
-    private UUID trainingUUID;
-
-    /** */
-    public TestGroupTrainerLocalContext(int maxCnt, int eachNumberCnt, int limit, UUID trainingUUID) {
-        this.maxCnt = maxCnt;
-        this.eachNumberCnt = eachNumberCnt;
-        this.limit = limit;
-        this.trainingUUID = trainingUUID;
-        this.cnt = 0;
-    }
-
-    /** */
-    public int cnt() {
-        return cnt;
-    }
-
-    /** */
-    public void setCnt(int cnt) {
-        this.cnt = cnt;
-    }
-
-    /** */
-    public TestGroupTrainerLocalContext incCnt() {
-        this.cnt++;
-
-        return this;
-    }
-
-    /** */
-    public int maxCnt() {
-        return maxCnt;
-    }
-
-    /** */
-    public int eachNumberCnt() {
-        return eachNumberCnt;
-    }
-
-    /** */
-    public int limit() {
-        return limit;
-    }
-
-    /** {@inheritDoc} */
-    @Override public UUID trainingUUID() {
-        return trainingUUID;
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainingCache.java b/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainingCache.java
deleted file mode 100644
index afee674528c69..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainingCache.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.Arrays;
-import java.util.UUID;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-
-/** */
-public class TestGroupTrainingCache {
-    /** */
-    public static String CACHE_NAME = "TEST_GROUP_TRAINING_CACHE";
-
-    /** */
-    public static IgniteCache<GroupTrainerCacheKey<Double>, Integer> getOrCreate(Ignite ignite) {
-        CacheConfiguration<GroupTrainerCacheKey<Double>, Integer> cfg = new CacheConfiguration<>();
-
-        // Write to primary.
-        cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
-        // Atomic transactions only.
-        cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
-        // No copying of values.
-        cfg.setCopyOnRead(false);
-
-        // Cache is partitioned.
-        cfg.setCacheMode(CacheMode.PARTITIONED);
-
-        cfg.setBackups(0);
-
-        cfg.setOnheapCacheEnabled(true);
-
-        cfg.setName(CACHE_NAME);
-
-        return ignite.getOrCreateCache(cfg);
-    }
-
-    /** */
-    public static Stream<GroupTrainerCacheKey<Double>> allKeys(int limit, int eachNumberCnt, UUID trainingUUID) {
-        GroupTrainerCacheKey<Double>[] a =new GroupTrainerCacheKey[limit * eachNumberCnt];
-
-        for (int num = 0; num < limit; num++)
-            for (int i = 0; i < eachNumberCnt; i++)
-                a[num * eachNumberCnt + i] = new GroupTrainerCacheKey<>(num, (double)i, trainingUUID);
-
-        return Arrays.stream(a);
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainingSecondCache.java b/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainingSecondCache.java
deleted file mode 100644
index e16ed7c74ada2..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestGroupTrainingSecondCache.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-
-/** */
-public class TestGroupTrainingSecondCache {
-    /** */
-    public static String CACHE_NAME = "TEST_GROUP_TRAINING_SECOND_CACHE";
-
-    /** */
-    public static IgniteCache<GroupTrainerCacheKey<Character>, Integer> getOrCreate(Ignite ignite) {
-        CacheConfiguration<GroupTrainerCacheKey<Character>, Integer> cfg = new CacheConfiguration<>();
-
-        // Write to primary.
-        cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
-
-        // Atomic transactions only.
-        cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
-        // No copying of values.
-        cfg.setCopyOnRead(false);
-
-        // Cache is partitioned.
-        cfg.setCacheMode(CacheMode.PARTITIONED);
-
-        cfg.setBackups(0);
-
-        cfg.setOnheapCacheEnabled(true);
-
-        cfg.setName(CACHE_NAME);
-
-        return ignite.getOrCreateCache(cfg);
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestLocalContext.java b/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestLocalContext.java
deleted file mode 100644
index 3f0237fc1c9fa..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestLocalContext.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.UUID;
-import org.apache.ignite.ml.trainers.group.chain.HasTrainingUUID;
-
-/** */
-public class TestLocalContext implements HasTrainingUUID {
-    /** */
-    private final UUID trainingUUID;
-
-    /** */
-    private int data;
-
-    /** */
-    public TestLocalContext(int data, UUID trainingUUID) {
-        this.data = data;
-        this.trainingUUID = trainingUUID;
-    }
-
-    /** */
-    public int data() {
-        return data;
-    }
-
-    /** */
-    public void setData(int data) {
-        this.data = data;
-    }
-
-    /** {@inheritDoc} */
-    @Override public UUID trainingUUID() {
-        return trainingUUID;
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestTrainingLoopStep.java b/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestTrainingLoopStep.java
deleted file mode 100644
index caf92f6b53ce8..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TestTrainingLoopStep.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Stream;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.ml.math.functions.IgniteFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.trainers.group.chain.DistributedEntryProcessingStep;
-import org.apache.ignite.ml.trainers.group.chain.EntryAndContext;
-
-/** */
-public class TestTrainingLoopStep implements DistributedEntryProcessingStep<TestGroupTrainerLocalContext,
-    Double, Integer, Void, Double, Double> {
-    /** {@inheritDoc} */
-    @Override public IgniteSupplier<Void> remoteContextSupplier(Double input, TestGroupTrainerLocalContext locCtx) {
-        // No context is needed.
-        return () -> null;
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteFunction<EntryAndContext<Double, Integer, Void>, ResultAndUpdates<Double>> worker() {
-        return entryAndContext -> {
-            Integer oldVal = entryAndContext.entry().getValue();
-            double v = oldVal * oldVal;
-            ResultAndUpdates<Double> res = ResultAndUpdates.of(v);
-            res.updateCache(TestGroupTrainingCache.getOrCreate(Ignition.localIgnite()),
-                entryAndContext.entry().getKey(), (int)v);
-            return res;
-        };
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteSupplier<Stream<GroupTrainerCacheKey<Double>>> keys(Double input,
-        TestGroupTrainerLocalContext locCtx) {
-        // Copying here because otherwise locCtx will be serialized with supplier returned in result.
-        int limit = locCtx.limit();
-        int cnt = locCtx.eachNumberCnt();
-        UUID uuid = locCtx.trainingUUID();
-
-        return () -> TestGroupTrainingCache.allKeys(limit, cnt, uuid);
-    }
-
-    /** {@inheritDoc} */
-    @Override public IgniteFunction<List<Double>, Double> reducer() {
-        return doubles -> doubles.stream().mapToDouble(x -> x).sum();
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TrainersGroupTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TrainersGroupTestSuite.java
deleted file mode 100644
index 0ec5afb2ce559..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/trainers/group/TrainersGroupTestSuite.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.trainers.group;
-
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
-
-/**
- * Test suite for group trainer tests.
- */
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
-    DistributedWorkersChainTest.class,
-    GroupTrainerTest.class
-})
-public class TrainersGroupTestSuite {
-}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/regression/IgniteOLSMultipleLinearRegressionBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/regression/IgniteOLSMultipleLinearRegressionBenchmark.java
deleted file mode 100644
index 89b54716261f8..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/regression/IgniteOLSMultipleLinearRegressionBenchmark.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.regression;
-
-import java.util.Map;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
-import org.apache.ignite.ml.regressions.linear.LinearRegressionQRTrainer;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteOLSMultipleLinearRegressionBenchmark extends IgniteAbstractBenchmark {
-    /** {@inheritDoc} */
-    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
-        runLongly();
-
-        return true;
-    }
-
-    /**
-     * Based on OLSMultipleLinearRegressionTest#testLongly.
-     */
-    private void runLongly() {
-        // Y values are first, then independent vars
-        // Each row is one observation
-        double[][] data = new double[][] {
-            { 60323, 83.0, 234289, 2356, 1590, 107608, 1947 },
-            { 61122, 88.5, 259426, 2325, 1456, 108632, 1948 },
-            { 60171, 88.2, 258054, 3682, 1616, 109773, 1949 },
-            { 61187, 89.5, 284599, 3351, 1650, 110929, 1950 },
-            { 63221, 96.2, 328975, 2099, 3099, 112075, 1951 },
-            { 63639, 98.1, 346999, 1932, 3594, 113270, 1952 },
-            { 64989, 99.0, 365385, 1870, 3547, 115094, 1953 },
-            { 63761, 100.0, 363112, 3578, 3350, 116219, 1954 },
-            { 66019, 101.2, 397469, 2904, 3048, 117388, 1955 },
-            { 67857, 104.6, 419180, 2822, 2857, 118734, 1956 },
-            { 68169, 108.4, 442769, 2936, 2798, 120445, 1957 },
-            { 66513, 110.8, 444546, 4681, 2637, 121950, 1958 },
-            { 68655, 112.6, 482704, 3813, 2552, 123366, 1959 },
-            { 69564, 114.2, 502601, 3931, 2514, 125368, 1960 },
-            { 69331, 115.7, 518173, 4806, 2572, 127852, 1961 },
-            { 70551, 116.9, 554894, 4007, 2827, 130081, 1962 }
-        };
-
-        final int nobs = 16;
-        final int nvars = 6;
-
-        LinearRegressionQRTrainer trainer = new LinearRegressionQRTrainer();
-        LinearRegressionModel model = trainer.train(new DenseLocalOnHeapMatrix(data));
-    }
-}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/regression/package-info.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/regression/package-info.java
deleted file mode 100644
index 0a5dc1a0b6bf1..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/regression/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * ML Grid regression benchmarks.
- */
-package org.apache.ignite.yardstick.ml.regression;
\ No newline at end of file

From 8acee1da7fc8475e7d2ffb25b179055d93906034 Mon Sep 17 00:00:00 2001
From: YuriBabak <y.chief@gmail.com>
Date: Fri, 13 Apr 2018 20:27:15 +0300
Subject: [PATCH 052/543] IGNITE-8232: ML package cleanup for 2.5 release

this closes #3823

(cherry picked from commit c6ab036)
---
 .../org/apache/ignite/ml/math/functions/IgniteBiFunction.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteBiFunction.java b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteBiFunction.java
index 45fd035710e79..560be4ba35d7b 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteBiFunction.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/math/functions/IgniteBiFunction.java
@@ -27,7 +27,7 @@
  * @see java.util.function.BiFunction
  */
 public interface IgniteBiFunction<T, U, R> extends BiFunction<T, U, R>, Serializable {
-    /** {@inheritDoc} */
+    /** */
     default <V> IgniteBiFunction<T, U, V> andThen(IgniteFunction<? super R, ? extends V> after) {
         Objects.requireNonNull(after);
         return (T t, U u) -> after.apply(apply(t, u));

From c51f2465113d0933336a36d0f173051f68c91be7 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Fri, 13 Apr 2018 14:50:11 -0700
Subject: [PATCH 053/543] IGNITE-2766 - Opportunistically reopen cache after
 client reconnect - Fixes #3417

Signed-off-by: Valentin Kulichenko <valentin.kulichenko@gmail.com>
---
 .../cache/GatewayProtectedCacheProxy.java     | 676 +++++++-----------
 .../processors/cache/GridCacheGateway.java    |   7 +
 .../cache/IgniteCacheProxyImpl.java           |  31 +-
 .../ignite/spi/discovery/tcp/ClientImpl.java  |   6 +-
 .../IgniteCacheQueryCacheDestroySelfTest.java |   4 +
 ...lientReconnectAfterClusterRestartTest.java |  33 +-
 6 files changed, 316 insertions(+), 441 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java
index 27fc39563e36a..2e8120ba7bde2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java
@@ -36,6 +36,7 @@
 import javax.cache.processor.EntryProcessor;
 import javax.cache.processor.EntryProcessorResult;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheEntry;
 import org.apache.ignite.cache.CacheEntryProcessor;
 import org.apache.ignite.cache.CacheMetrics;
@@ -48,6 +49,9 @@
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.internal.AsyncSupportAdapter;
+import org.apache.ignite.internal.GridKernalState;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteClosure;
@@ -138,15 +142,13 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
 
     /** {@inheritDoc} */
     @Override public GatewayProtectedCacheProxy<K, V> withExpiryPolicy(ExpiryPolicy plc) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return new GatewayProtectedCacheProxy<>(delegate, opCtx.withExpiryPolicy(plc), lock);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
@@ -157,9 +159,7 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
 
     /** {@inheritDoc} */
     @Override public GatewayProtectedCacheProxy<K, V> skipStore() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             boolean skip = opCtx.skipStore();
@@ -170,15 +170,13 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
             return new GatewayProtectedCacheProxy<>(delegate, opCtx.setSkipStore(true), lock);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public GatewayProtectedCacheProxy<K, V> withNoRetries() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             boolean noRetries = opCtx.noRetries();
@@ -189,15 +187,13 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
             return new GatewayProtectedCacheProxy<>(delegate, opCtx.setNoRetries(true), lock);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public GatewayProtectedCacheProxy<K, V> withPartitionRecover() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             boolean recovery = opCtx.recovery();
@@ -208,7 +204,7 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
             return new GatewayProtectedCacheProxy<>(delegate, opCtx.setRecovery(true), lock);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
@@ -219,23 +215,19 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
 
     /** {@inheritDoc} */
     @Override public <K1, V1> GatewayProtectedCacheProxy<K1, V1> keepBinary() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return new GatewayProtectedCacheProxy<>((IgniteCacheProxy<K1, V1>) delegate, opCtx.keepBinary(), lock);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public GatewayProtectedCacheProxy<K, V> withDataCenterId(byte dataCenterId) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             Byte prevDataCenterId = opCtx.dataCenterId();
@@ -246,91 +238,79 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
             return new GatewayProtectedCacheProxy<>(delegate, opCtx.setDataCenterId(dataCenterId), lock);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void loadCache(@Nullable IgniteBiPredicate<K, V> p, @Nullable Object... args) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.loadCache(p, args);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Void> loadCacheAsync(@Nullable IgniteBiPredicate<K, V> p, @Nullable Object... args) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.loadCacheAsync(p, args);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void localLoadCache(@Nullable IgniteBiPredicate<K, V> p, @Nullable Object... args) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.localLoadCache(p, args);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Void> localLoadCacheAsync(@Nullable IgniteBiPredicate<K, V> p, @Nullable Object... args) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.localLoadCacheAsync(p, args);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public V getAndPutIfAbsent(K key, V val) throws CacheException, TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAndPutIfAbsent(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<V> getAndPutIfAbsentAsync(K key, V val) throws CacheException, TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAndPutIfAbsentAsync(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
@@ -346,1093 +326,937 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
 
     /** {@inheritDoc} */
     @Override public boolean isLocalLocked(K key, boolean byCurrThread) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.isLocalLocked(key, byCurrThread);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <R> QueryCursor<R> query(Query<R> qry) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.query(qry);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public FieldsQueryCursor<List<?>> query(SqlFieldsQuery qry) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.query(qry);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public List<FieldsQueryCursor<List<?>>> queryMultipleStatements(SqlFieldsQuery qry) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.queryMultipleStatements(qry);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T, R> QueryCursor<R> query(Query<T> qry, IgniteClosure<T, R> transformer) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.query(qry, transformer);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public Iterable<Entry<K, V>> localEntries(CachePeekMode... peekModes) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.localEntries(peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public QueryMetrics queryMetrics() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.queryMetrics();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void resetQueryMetrics() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.resetQueryMetrics();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public Collection<? extends QueryDetailMetrics> queryDetailMetrics() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.queryDetailMetrics();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void resetQueryDetailMetrics() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.resetQueryDetailMetrics();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void localEvict(Collection<? extends K> keys) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.localEvict(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public V localPeek(K key, CachePeekMode... peekModes) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.localPeek(key, peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public int size(CachePeekMode... peekModes) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.size(peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Integer> sizeAsync(CachePeekMode... peekModes) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.sizeAsync(peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public long sizeLong(CachePeekMode... peekModes) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.sizeLong(peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Long> sizeLongAsync(CachePeekMode... peekModes) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.sizeLongAsync(peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public long sizeLong(int partition, CachePeekMode... peekModes) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.sizeLong(partition, peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Long> sizeLongAsync(int partition, CachePeekMode... peekModes) throws CacheException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.sizeLongAsync(partition, peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public int localSize(CachePeekMode... peekModes) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.localSize(peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public long localSizeLong(CachePeekMode... peekModes) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.localSizeLong(peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public long localSizeLong(int partition, CachePeekMode... peekModes) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.localSizeLong(partition, peekModes);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(Map<? extends K, ? extends EntryProcessor<K, V, T>> map, Object... args) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.invokeAll(map, args);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T> IgniteFuture<Map<K, EntryProcessorResult<T>>> invokeAllAsync(Map<? extends K, ? extends EntryProcessor<K, V, T>> map, Object... args) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.invokeAllAsync(map, args);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public V get(K key) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.get(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<V> getAsync(K key) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAsync(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public CacheEntry<K, V> getEntry(K key) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getEntry(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<CacheEntry<K, V>> getEntryAsync(K key) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getEntryAsync(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public Map<K, V> getAll(Set<? extends K> keys) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAll(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Map<K, V>> getAllAsync(Set<? extends K> keys) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAllAsync(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public Collection<CacheEntry<K, V>> getEntries(Set<? extends K> keys) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getEntries(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Collection<CacheEntry<K, V>>> getEntriesAsync(Set<? extends K> keys) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getEntriesAsync(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public Map<K, V> getAllOutTx(Set<? extends K> keys) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAllOutTx(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Map<K, V>> getAllOutTxAsync(Set<? extends K> keys) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAllOutTxAsync(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public boolean containsKey(K key) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.containsKey(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void loadAll(Set<? extends K> keys, boolean replaceExisting, CompletionListener completionListener) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.loadAll(keys, replaceExisting, completionListener);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Boolean> containsKeyAsync(K key) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.containsKeyAsync(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public boolean containsKeys(Set<? extends K> keys) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.containsKeys(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Boolean> containsKeysAsync(Set<? extends K> keys) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.containsKeysAsync(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void put(K key, V val) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.put(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Void> putAsync(K key, V val) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.putAsync(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public V getAndPut(K key, V val) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAndPut(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<V> getAndPutAsync(K key, V val) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAndPutAsync(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void putAll(Map<? extends K, ? extends V> map) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.putAll(map);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Void> putAllAsync(Map<? extends K, ? extends V> map) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.putAllAsync(map);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public boolean putIfAbsent(K key, V val) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.putIfAbsent(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Boolean> putIfAbsentAsync(K key, V val) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.putIfAbsentAsync(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public boolean remove(K key) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.remove(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Boolean> removeAsync(K key) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.removeAsync(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public boolean remove(K key, V oldVal) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.remove(key, oldVal);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Boolean> removeAsync(K key, V oldVal) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.removeAsync(key, oldVal);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public V getAndRemove(K key) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAndRemove(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<V> getAndRemoveAsync(K key) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAndRemoveAsync(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public boolean replace(K key, V oldVal, V newVal) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.replace(key, oldVal, newVal);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Boolean> replaceAsync(K key, V oldVal, V newVal) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.replaceAsync(key, oldVal, newVal);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public boolean replace(K key, V val) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.replace(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Boolean> replaceAsync(K key, V val) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.replaceAsync(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public V getAndReplace(K key, V val) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAndReplace(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<V> getAndReplaceAsync(K key, V val) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.getAndReplaceAsync(key, val);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void removeAll(Set<? extends K> keys) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.removeAll(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Void> removeAllAsync(Set<? extends K> keys) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.removeAllAsync(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void removeAll() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.removeAll();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Void> removeAllAsync() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.removeAllAsync();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void clear() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.clear();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Void> clearAsync() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.clearAsync();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void clear(K key) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.clear(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Void> clearAsync(K key) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.clearAsync(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void clearAll(Set<? extends K> keys) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.clearAll(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public IgniteFuture<Void> clearAllAsync(Set<? extends K> keys) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.clearAllAsync(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void localClear(K key) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.localClear(key);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void localClearAll(Set<? extends K> keys) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.localClearAll(keys);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T> T invoke(K key, EntryProcessor<K, V, T> entryProcessor, Object... arguments) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.invoke(key, entryProcessor, arguments);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T> IgniteFuture<T> invokeAsync(K key, EntryProcessor<K, V, T> entryProcessor, Object... arguments) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.invokeAsync(key, entryProcessor, arguments);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T> T invoke(K key, CacheEntryProcessor<K, V, T> entryProcessor, Object... arguments) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.invoke(key, entryProcessor, arguments);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T> IgniteFuture<T> invokeAsync(K key, CacheEntryProcessor<K, V, T> entryProcessor, Object... arguments) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.invokeAsync(key, entryProcessor, arguments);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(Set<? extends K> keys, EntryProcessor<K, V, T> entryProcessor, Object... args) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.invokeAll(keys, entryProcessor, args);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T> IgniteFuture<Map<K, EntryProcessorResult<T>>> invokeAllAsync(Set<? extends K> keys, EntryProcessor<K, V, T> entryProcessor, Object... args) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.invokeAllAsync(keys, entryProcessor, args);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T> Map<K, EntryProcessorResult<T>> invokeAll(Set<? extends K> keys, CacheEntryProcessor<K, V, T> entryProcessor, Object... args) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.invokeAll(keys, entryProcessor, args);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public <T> IgniteFuture<Map<K, EntryProcessorResult<T>>> invokeAllAsync(Set<? extends K> keys, CacheEntryProcessor<K, V, T> entryProcessor, Object... args) throws TransactionException {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.invokeAllAsync(keys, entryProcessor, args);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
@@ -1443,43 +1267,37 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
 
     /** {@inheritDoc} */
     @Override public void registerCacheEntryListener(CacheEntryListenerConfiguration<K, V> cacheEntryListenerConfiguration) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.registerCacheEntryListener(cacheEntryListenerConfiguration);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void deregisterCacheEntryListener(CacheEntryListenerConfiguration<K, V> cacheEntryListenerConfiguration) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.deregisterCacheEntryListener(cacheEntryListenerConfiguration);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public Iterator<Entry<K, V>> iterator() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.iterator();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
@@ -1550,99 +1368,85 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
 
     /** {@inheritDoc} */
     @Override public CacheMetrics metrics() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.metrics();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public CacheMetrics metrics(ClusterGroup grp) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.metrics(grp);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public CacheMetrics localMetrics() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.localMetrics();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public CacheMetricsMXBean mxBean() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.mxBean();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public CacheMetricsMXBean localMxBean() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.localMxBean();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public Collection<Integer> lostPartitions() {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             return delegate.lostPartitions();
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
     /** {@inheritDoc} */
     @Override public void enableStatistics(boolean enabled) {
-        GridCacheGateway<K, V> gate = gate();
-
-        CacheOperationContext prev = onEnter(gate, opCtx);
+        CacheOperationGate opGate = onEnter();
 
         try {
             delegate.enableStatistics(enabled);
         }
         finally {
-            onLeave(gate, prev);
+            onLeave(opGate);
         }
     }
 
@@ -1662,26 +1466,49 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
      *
      * @param gate Cache gateway.
      */
-    private void checkProxyIsValid(@Nullable GridCacheGateway<K, V> gate) {
+    private GridCacheGateway<K, V> checkProxyIsValid(@Nullable GridCacheGateway<K, V> gate, boolean tryRestart) {
         if (isProxyClosed())
             throw new IllegalStateException("Cache has been closed: " + context().name());
 
-        if (delegate instanceof IgniteCacheProxyImpl)
+        boolean isCacheProxy = delegate instanceof IgniteCacheProxyImpl;
+
+        if (isCacheProxy)
             ((IgniteCacheProxyImpl) delegate).checkRestart();
 
         if (gate == null)
             throw new IllegalStateException("Gateway is unavailable. Probably cache has been destroyed, but proxy is not closed.");
+
+        if (isCacheProxy && tryRestart && gate.isStopped() &&
+            context().kernalContext().gateway().getState() == GridKernalState.STARTED) {
+            IgniteCacheProxyImpl proxyImpl = (IgniteCacheProxyImpl) delegate;
+
+            try {
+                IgniteInternalCache<K, V> cache = context().kernalContext().cache().<K, V>publicJCache(context().name()).internalProxy();
+
+                GridFutureAdapter<Void> fut = proxyImpl.opportunisticRestart();
+
+                if (fut == null)
+                    proxyImpl.onRestarted(cache.context(), cache.context().cache());
+                else
+                    new IgniteFutureImpl<>(fut).get();
+
+                return gate();
+            } catch (IgniteCheckedException ice) {
+                // Opportunity didn't work out.
+            }
+        }
+
+        return gate;
     }
 
     /**
-     * @param gate Cache gateway.
-     * @param opCtx Cache operation context to guard.
      * @return Previous projection set on this thread.
      */
-    private CacheOperationContext onEnter(@Nullable GridCacheGateway<K, V> gate, CacheOperationContext opCtx) {
-        checkProxyIsValid(gate);
+    private CacheOperationGate onEnter() {
+        GridCacheGateway<K, V> gate = checkProxyIsValid(gate(), true);
 
-        return lock ? gate.enter(opCtx) : gate.enterNoLock(opCtx);
+        return new CacheOperationGate(gate,
+            lock ? gate.enter(opCtx) : gate.enterNoLock(opCtx));
     }
 
     /**
@@ -1690,7 +1517,7 @@ private CacheOperationContext onEnter(@Nullable GridCacheGateway<K, V> gate, Cac
      */
     private boolean onEnterIfNoStop(@Nullable GridCacheGateway<K, V> gate) {
         try {
-            checkProxyIsValid(gate);
+            checkProxyIsValid(gate, false);
         }
         catch (Exception e) {
             return false;
@@ -1700,14 +1527,13 @@ private boolean onEnterIfNoStop(@Nullable GridCacheGateway<K, V> gate) {
     }
 
     /**
-     * @param gate Cache gateway.
-     * @param opCtx Operation context to guard.
+     * @param opGate Operation context to guard.
      */
-    private void onLeave(GridCacheGateway<K, V> gate, CacheOperationContext opCtx) {
+    private void onLeave(CacheOperationGate opGate) {
         if (lock)
-            gate.leave(opCtx);
+            opGate.gate.leave(opGate.prev);
         else
-            gate.leaveNoLock(opCtx);
+            opGate.gate.leaveNoLock(opGate.prev);
     }
 
     /**
@@ -1774,4 +1600,28 @@ private void onLeave(GridCacheGateway<K, V> gate) {
     @Override public int hashCode() {
         return delegate.hashCode();
     }
+
+    /**
+     * Holder for gate being entered and operation context to restore.
+     */
+    private class CacheOperationGate {
+        /**
+         * Gate being entered in this operation.
+         */
+        public final GridCacheGateway<K, V> gate;
+
+        /**
+         * Operation context to restore after current operation completes.
+         */
+        public final CacheOperationContext prev;
+
+        /**
+         * @param gate Gate being entered in this operation.
+         * @param prev Operation context to restore after current operation completes.
+         */
+        public CacheOperationGate(GridCacheGateway<K, V> gate, CacheOperationContext prev) {
+            this.gate = gate;
+            this.prev = prev;
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGateway.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGateway.java
index b9a4b257b529f..658ca2a8de039 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGateway.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGateway.java
@@ -253,6 +253,13 @@ private void onEnter() {
             ctx.deploy().onEnter();
     }
 
+    /**
+     *
+     */
+    public boolean isStopped() {
+        return !checkState(false, false);
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index be4b0dbffaf5a..68e5b850aafed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -1824,8 +1824,10 @@ private <R> void setFuture(IgniteInternalFuture<R> fut) {
      * Throws {@code IgniteCacheRestartingException} if proxy is restarting.
      */
     public void checkRestart() {
-        if (isRestarting())
-            throw new IgniteCacheRestartingException(new IgniteFutureImpl<>(restartFut.get()), "Cache is restarting: " +
+        GridFutureAdapter<Void> currentFut = this.restartFut.get();
+
+        if (currentFut != null)
+            throw new IgniteCacheRestartingException(new IgniteFutureImpl<>(currentFut), "Cache is restarting: " +
                     context().name());
     }
 
@@ -1833,13 +1835,13 @@ public void checkRestart() {
      * @return True if proxy is restarting, false in other case.
      */
     public boolean isRestarting() {
-        return restartFut != null && restartFut.get() != null;
+        return restartFut.get() != null;
     }
 
     /**
      * Restarts this cache proxy.
      */
-    public void restart() {
+    public boolean restart() {
         GridFutureAdapter<Void> restartFut = new GridFutureAdapter<>();
 
         final GridFutureAdapter<Void> curFut = this.restartFut.get();
@@ -1855,6 +1857,27 @@ public void restart() {
                         curFut.onDone();
                 }
             });
+
+        return changed;
+    }
+
+    /**
+     * If proxy is already being restarted, returns future to wait on, else restarts this cache proxy.
+     *
+     * @return Future to wait on, or null.
+     */
+    public GridFutureAdapter<Void> opportunisticRestart() {
+        GridFutureAdapter<Void> restartFut = new GridFutureAdapter<>();
+
+        while (true) {
+            if (this.restartFut.compareAndSet(null, restartFut))
+                return null;
+
+            GridFutureAdapter<Void> curFut = this.restartFut.get();
+
+            if (curFut != null)
+                return curFut;
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 16c5d3a4829b2..b22a3970f2710 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -1616,8 +1616,6 @@ else if (msg == SPI_RECONNECT) {
 
                             onDisconnected();
 
-                            notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
-
                             UUID newId = UUID.randomUUID();
 
                             U.quietAndWarn(log, "Local node will try to reconnect to cluster with new id due " +
@@ -1716,8 +1714,6 @@ else if (msg == SPI_RECONNECT_FAILED) {
                                 }
 
                                 onDisconnected();
-
-                                notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
                             }
 
                             UUID newId = UUID.randomUUID();
@@ -1820,6 +1816,8 @@ private void onDisconnected() {
 
             delayDiscoData.clear();
 
+            notifyDiscovery(EVT_CLIENT_NODE_DISCONNECTED, topVer, locNode, allVisibleNodes());
+
             IgniteClientDisconnectedCheckedException err =
                 new IgniteClientDisconnectedCheckedException(null, "Failed to ping node, " +
                     "client node disconnected.");
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IgniteCacheQueryCacheDestroySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IgniteCacheQueryCacheDestroySelfTest.java
index dea491c50be06..d0d392b50b604 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IgniteCacheQueryCacheDestroySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IgniteCacheQueryCacheDestroySelfTest.java
@@ -48,6 +48,10 @@ public class IgniteCacheQueryCacheDestroySelfTest extends GridCommonAbstractTest
     /** */
     public static final int GRID_CNT = 3;
 
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
     /**
      * The main test code.
      */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClientReconnectAfterClusterRestartTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClientReconnectAfterClusterRestartTest.java
index 392cdc771bd6b..505d373ad0386 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClientReconnectAfterClusterRestartTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClientReconnectAfterClusterRestartTest.java
@@ -17,8 +17,10 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import javax.cache.CacheException;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryObjectBuilder;
@@ -31,9 +33,8 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
-import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.NotNull;
 
@@ -119,6 +120,8 @@ public void testReconnectClient() throws Exception {
 
             checkTopology(2);
 
+            IgniteCache<Long, BinaryObject> cache = client.getOrCreateCache(CACHE_PARAMS).withKeepBinary();
+
             client.events().localListen(new IgnitePredicate<Event>() {
 
                 @Override public boolean apply(Event event) {
@@ -161,27 +164,17 @@ public void testReconnectClient() throws Exception {
 
             startGrid(0);
 
-            assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
-                @Override public boolean apply() {
-                    try {
-                        checkTopology(2);
-
-                        return true;
-                    } catch (Exception ex) {
-                        return false;
-                    }
-                }
-            }, 30_000);
+            try {
+                assertNull(cache.get(1L));
+            } catch (CacheException ce) {
+                IgniteClientDisconnectedException icde = (IgniteClientDisconnectedException)ce.getCause();
 
-            info("Pre-insert");
+                icde.reconnectFuture().get();
 
-            streamer = client.dataStreamer("PPRB_PARAMS");
-            streamer.allowOverwrite(true);
-            streamer.keepBinary(true);
-            streamer.perNodeBufferSize(10000);
-            streamer.perNodeParallelOperations(100);
+                assertNull(cache.get(1L));
+            }
 
-            IgniteCache<Long, BinaryObject> cache = client.getOrCreateCache(CACHE_PARAMS).withKeepBinary();
+            info("Pre-insert");
 
             builder = client.binary().builder("PARAMS");
             builder.setField("ID", 2L);

From c43049d4fc154e532497320e178d5acb468c15d9 Mon Sep 17 00:00:00 2001
From: tledkov-gridgain <tledkov@gridgain.com>
Date: Mon, 16 Apr 2018 11:28:39 +0300
Subject: [PATCH 054/543] IGNITE-8129: MTCGA: setup default SSL context in
 JdbcthinConnectionSSLTest (because sometimes default SSL context may be setup
 by build system). This closes #3795.

---
 .../jdbc/thin/JdbcThinConnectionSSLTest.java  | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSSLTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSSLTest.java
index cc71f51772400..355a198c56672 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSSLTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSSLTest.java
@@ -164,7 +164,14 @@ public void testConnectionUseIgniteFactory() throws Exception {
      * @throws Exception If failed.
      */
     public void testDefaultContext() throws Exception {
+        // Store exists default SSL context to restore after test.
+        final SSLContext dfltSslCtx = SSLContext.getDefault();
+
+        // Setup default context
+        SSLContext.setDefault(getTestSslContextFactory().create());
+
         setSslCtxFactoryToCli = true;
+
         // Factory return default SSL context
         sslCtxFactory = new Factory<SSLContext>() {
             @Override public SSLContext create() {
@@ -177,23 +184,16 @@ public void testDefaultContext() throws Exception {
             }
         };
 
-        System.setProperty("javax.net.ssl.keyStore", CLI_KEY_STORE_PATH);
-        System.setProperty("javax.net.ssl.keyStorePassword", "123456");
-        System.setProperty("javax.net.ssl.trustStore", TRUST_KEY_STORE_PATH);
-        System.setProperty("javax.net.ssl.trustStorePassword", "123456");
-
         startGrids(1);
 
         try (Connection conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1/?sslMode=require")) {
             checkConnection(conn);
         }
         finally {
-            System.getProperties().remove("javax.net.ssl.keyStore");
-            System.getProperties().remove("javax.net.ssl.keyStorePassword");
-            System.getProperties().remove("javax.net.ssl.trustStore");
-            System.getProperties().remove("javax.net.ssl.trustStorePassword");
-
             stopAllGrids();
+
+            // Restore SSL context.
+            SSLContext.setDefault(dfltSslCtx);
         }
     }
 

From b3f252666785fddf41ef07846e034d55ae0dda71 Mon Sep 17 00:00:00 2001
From: Alexey Kukushkin <alexeykukushkin@yahoo.com>
Date: Mon, 16 Apr 2018 11:47:19 +0300
Subject: [PATCH 055/543] IGNITE-8097: Java thin client: throw handshake
 exception eagerly on connect phase in case of failure. This closes #3822.

---
 .../internal/client/thin/ReliableChannel.java |  2 +
 .../apache/ignite/client/FunctionalTest.java  | 37 +++++++++++++++----
 .../apache/ignite/client/SecurityTest.java    | 22 +++++------
 3 files changed, 42 insertions(+), 19 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ReliableChannel.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ReliableChannel.java
index 392b8f87ab63c..dac4320bb68a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ReliableChannel.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ReliableChannel.java
@@ -81,6 +81,8 @@ final class ReliableChannel implements AutoCloseable {
 
         primary = addrs.get(new Random().nextInt(addrs.size())); // we already verified there is at least one address
 
+        ch = chFactory.apply(new ClientChannelConfiguration(clientCfg).setAddress(primary)).get();
+
         for (InetSocketAddress a : addrs)
             if (a != primary)
                 this.backups.add(a);
diff --git a/modules/core/src/test/java/org/apache/ignite/client/FunctionalTest.java b/modules/core/src/test/java/org/apache/ignite/client/FunctionalTest.java
index d69ac4d16ed03..b49f7e3a186f8 100644
--- a/modules/core/src/test/java/org/apache/ignite/client/FunctionalTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/client/FunctionalTest.java
@@ -39,21 +39,16 @@
 import org.apache.ignite.cache.PartitionLossPolicy;
 import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.cache.QueryIndex;
-import org.apache.ignite.client.ClientCache;
-import org.apache.ignite.client.ClientCacheConfiguration;
-import org.apache.ignite.client.Comparers;
-import org.apache.ignite.client.Config;
-import org.apache.ignite.client.IgniteClient;
 import org.apache.ignite.configuration.ClientConfiguration;
-import org.apache.ignite.client.LocalIgniteCluster;
-import org.apache.ignite.client.Person;
 import org.junit.Test;
 
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * Thin client functional tests.
@@ -380,6 +375,34 @@ public void testRemoveReplace() throws Exception {
         }
     }
 
+    /**
+     * Test client fails on start if server is unavailable
+     */
+    @Test
+    public void testClientFailsOnStart() {
+        ClientConnectionException expEx = null;
+
+        try (IgniteClient ignored = Ignition.startClient(getClientConfiguration())) {
+            // No-op.
+        }
+        catch (ClientConnectionException connEx) {
+            expEx = connEx;
+        }
+        catch (Exception ex) {
+            fail(String.format(
+                "%s expected but %s was received: %s",
+                ClientConnectionException.class.getName(),
+                ex.getClass().getName(),
+                ex
+            ));
+        }
+
+        assertNotNull(
+            String.format("%s expected but no exception was received", ClientConnectionException.class.getName()),
+            expEx
+        );
+    }
+
     /** */
     private static ClientConfiguration getClientConfiguration() {
         return new ClientConfiguration()
diff --git a/modules/indexing/src/test/java/org/apache/ignite/client/SecurityTest.java b/modules/indexing/src/test/java/org/apache/ignite/client/SecurityTest.java
index dc57f0ca524d6..e2b11db2f66e0 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/client/SecurityTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/client/SecurityTest.java
@@ -127,25 +127,23 @@ public void testEncryption() throws Exception {
 
     /** Test valid user authentication. */
     @Test
-    public void testInvalidUserAuthentication() throws Exception {
+    public void testInvalidUserAuthentication() {
+        Exception authError = null;
+
         try (Ignite ignored = igniteWithAuthentication();
              IgniteClient client = Ignition.startClient(new ClientConfiguration().setAddresses(Config.SERVER)
                  .setUserName("JOE")
                  .setUserPassword("password")
              )
         ) {
-            Exception authError = null;
-
-            try {
-                client.getOrCreateCache("testAuthentication");
-            }
-            catch (Exception e) {
-                authError = e;
-            }
-
-            assertNotNull("Authentication with invalid credentials succeeded", authError);
-            assertTrue("Invalid type of authentication error", authError instanceof ClientAuthenticationException);
+            client.getOrCreateCache("testAuthentication");
+        }
+        catch (Exception e) {
+            authError = e;
         }
+
+        assertNotNull("Authentication with invalid credentials succeeded", authError);
+        assertTrue("Invalid type of authentication error", authError instanceof ClientAuthenticationException);
     }
 
     /** Test valid user authentication. */

From 7173b0c48dab05be0b8e7825ae527824fcdbbc83 Mon Sep 17 00:00:00 2001
From: zaleslaw <zaleslaw.sin@gmail.com>
Date: Mon, 16 Apr 2018 20:20:49 +0300
Subject: [PATCH 056/543] IGNITE-8169: [ML] Adopt KMeans to the new Partitioned
 Dataset and cleanup old code

this closes #3817

(cherry picked from commit 9e21cec)
---
 .../DatasetWithObviousStructure.java          | 105 ----
 .../ml/clustering/FuzzyCMeansExample.java     | 134 -----
 .../clustering/FuzzyCMeansLocalExample.java   |  95 ----
 .../KMeansClusterizationExample.java          | 226 ++++++++
 .../KMeansDistributedClustererExample.java    |  97 ----
 .../KMeansLocalClustererExample.java          | 106 ----
 .../ignite/ml/FuzzyCMeansModelFormat.java     |  76 ---
 .../clustering/BaseFuzzyCMeansClusterer.java  |  90 ---
 .../ml/clustering/BaseKMeansClusterer.java    |  96 ----
 .../FuzzyCMeansDistributedClusterer.java      | 512 ------------------
 .../clustering/FuzzyCMeansLocalClusterer.java | 254 ---------
 .../ml/clustering/FuzzyCMeansModel.java       |  88 ---
 .../KMeansDistributedClusterer.java           | 306 -----------
 .../ml/clustering/KMeansLocalClusterer.java   | 177 ------
 .../ml/clustering/WeightedClusterer.java      |  38 --
 .../ml/clustering/{ => kmeans}/Clusterer.java |   2 +-
 .../{ => kmeans}/ClusterizationModel.java     |   4 +-
 .../clustering/{ => kmeans}/KMeansModel.java  |  31 +-
 .../kmeans}/KMeansModelFormat.java            |   4 +-
 .../ml/clustering/kmeans/KMeansTrainer.java   | 320 +++++++++++
 .../ml/clustering/kmeans/package-info.java}   |  21 +-
 .../preprocessing/LabellingMachine.java       |  41 --
 .../structures/preprocessing/Normalizer.java  |  80 ---
 .../org/apache/ignite/ml/LocalModelsTest.java |  26 +-
 .../ml/clustering/ClusteringTestSuite.java    |   7 +-
 .../FuzzyCMeansDistributedClustererTest.java  | 180 ------
 .../FuzzyCMeansLocalClustererTest.java        | 202 -------
 ...eansDistributedClustererTestMultiNode.java | 138 -----
 ...ansDistributedClustererTestSingleNode.java | 198 -------
 .../clustering/KMeansLocalClustererTest.java  |  46 --
 .../ignite/ml/clustering/KMeansModelTest.java |  63 +++
 .../ml/clustering/KMeansTrainerTest.java      |  73 +++
 32 files changed, 728 insertions(+), 3108 deletions(-)
 delete mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/clustering/DatasetWithObviousStructure.java
 delete mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/clustering/FuzzyCMeansExample.java
 delete mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/clustering/FuzzyCMeansLocalExample.java
 create mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
 delete mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansDistributedClustererExample.java
 delete mode 100644 examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansLocalClustererExample.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/FuzzyCMeansModelFormat.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseFuzzyCMeansClusterer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansDistributedClusterer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansLocalClusterer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java
 rename modules/ml/src/main/java/org/apache/ignite/ml/clustering/{ => kmeans}/Clusterer.java (95%)
 rename modules/ml/src/main/java/org/apache/ignite/ml/clustering/{ => kmeans}/ClusterizationModel.java (92%)
 rename modules/ml/src/main/java/org/apache/ignite/ml/clustering/{ => kmeans}/KMeansModel.java (77%)
 rename modules/ml/src/main/java/org/apache/ignite/ml/{ => clustering/kmeans}/KMeansModelFormat.java (94%)
 create mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
 rename modules/ml/src/{test/java/org/apache/ignite/ml/clustering/KMeansUtil.java => main/java/org/apache/ignite/ml/clustering/kmeans/package-info.java} (62%)
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/structures/preprocessing/LabellingMachine.java
 delete mode 100644 modules/ml/src/main/java/org/apache/ignite/ml/structures/preprocessing/Normalizer.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/clustering/FuzzyCMeansDistributedClustererTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/clustering/FuzzyCMeansLocalClustererTest.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTestMultiNode.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTestSingleNode.java
 delete mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansModelTest.java
 create mode 100644 modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java

diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/DatasetWithObviousStructure.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/DatasetWithObviousStructure.java
deleted file mode 100644
index 5cd0e099c1a13..0000000000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/DatasetWithObviousStructure.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.clustering;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorUtils;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/**
- * See KMeansDistributedClustererTestSingleNode#testClusterizationOnDatasetWithObviousStructure.
- */
-class DatasetWithObviousStructure {
-    /** */
-    private final Random rnd = new Random(123456L);
-
-    /** Let centers be in the vertices of square. */
-    private final Map<Integer, Vector> centers = new HashMap<>();
-
-    /** Square side length. */
-    private final int squareSideLen;
-
-    /** */
-    DatasetWithObviousStructure(int squareSideLen) {
-        this.squareSideLen = squareSideLen;
-        centers.put(100, new DenseLocalOnHeapVector(new double[] {0.0, 0.0}));
-        centers.put(900, new DenseLocalOnHeapVector(new double[] {squareSideLen, 0.0}));
-        centers.put(3000, new DenseLocalOnHeapVector(new double[] {0.0, squareSideLen}));
-        centers.put(6000, new DenseLocalOnHeapVector(new double[] {squareSideLen, squareSideLen}));
-    }
-
-    /** */
-    List<Vector> generate(Matrix points) {
-        int ptsCnt = points.rowSize();
-
-        // Mass centers of dataset.
-        List<Vector> massCenters = new ArrayList<>();
-
-        int centersCnt = centers.size();
-
-        List<Integer> permutation = IntStream.range(0, ptsCnt).boxed().collect(Collectors.toList());
-        Collections.shuffle(permutation, rnd);
-
-        Vector[] mc = new Vector[centersCnt];
-        Arrays.fill(mc, VectorUtils.zeroes(2));
-
-        int totalCnt = 0;
-
-        int centIdx = 0;
-        massCenters.clear();
-
-        for (Integer count : centers.keySet()) {
-            for (int i = 0; i < count; i++) {
-                Vector pnt = getPoint(count);
-
-                mc[centIdx] = mc[centIdx].plus(pnt);
-
-                points.assignRow(permutation.get(totalCnt), pnt);
-
-                totalCnt++;
-            }
-            massCenters.add(mc[centIdx].times(1 / (double)count));
-            centIdx++;
-        }
-
-        return massCenters;
-    }
-
-    /** */
-    Map<Integer, Vector> centers() {
-        return centers;
-    }
-
-    /** */
-    private Vector getPoint(Integer cnt) {
-        Vector pnt = new DenseLocalOnHeapVector(2).assign(centers.get(cnt));
-        // Perturbate point on random value.
-        pnt.map(val -> val + rnd.nextDouble() * squareSideLen / 100);
-        return pnt;
-    }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/FuzzyCMeansExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/FuzzyCMeansExample.java
deleted file mode 100644
index 23aeed7abc2f7..0000000000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/FuzzyCMeansExample.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.clustering;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.ml.clustering.BaseFuzzyCMeansClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansDistributedClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansModel;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.thread.IgniteThread;
-
-/**
- * <p>
- * This example shows how to use {@link FuzzyCMeansDistributedClusterer}.</p>
- * <p>
- * Remote nodes should always be started with special configuration file which
- * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
- * with {@code examples/config/example-ignite.xml} configuration.</p>
- */
-public final class FuzzyCMeansExample {
-    /**
-     * Executes example.
-     *
-     * @param args Command line arguments, none required.
-     */
-    public static void main(String[] args) throws InterruptedException {
-        System.out.println(">>> Fuzzy C-Means usage example started.");
-
-        // Start ignite grid.
-        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
-            System.out.println(">>> Ignite grid started.");
-
-            // Start new Ignite thread.
-            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                FuzzyCMeansExample.class.getSimpleName(),
-                () -> {
-                    // Distance measure that computes distance between two points.
-                    DistanceMeasure distanceMeasure = new EuclideanDistance();
-
-                    // "Fuzziness" - specific constant that is used in membership calculation (1.0+-eps ~ K-Means).
-                    double exponentialWeight = 2.0;
-
-                    // Condition that indicated when algorithm must stop.
-                    // In this example algorithm stops if memberships have changed insignificantly.
-                    BaseFuzzyCMeansClusterer.StopCondition stopCond =
-                        BaseFuzzyCMeansClusterer.StopCondition.STABLE_MEMBERSHIPS;
-
-                    // Maximum difference between new and old membership values with which algorithm will continue to work.
-                    double maxDelta = 0.01;
-
-                    // The maximum number of FCM iterations.
-                    int maxIterations = 50;
-
-                    // Value that is used to initialize random numbers generator. You can choose it randomly.
-                    Long seed = null;
-
-                    // Number of steps of primary centers selection (more steps more candidates).
-                    int initializationSteps = 2;
-
-                    // Number of K-Means iteration that is used to choose required number of primary centers from candidates.
-                    int kMeansMaxIterations = 50;
-
-                    // Create new distributed clusterer with parameters described above.
-                    System.out.println(">>> Create new Distributed Fuzzy C-Means clusterer.");
-                    FuzzyCMeansDistributedClusterer clusterer = new FuzzyCMeansDistributedClusterer(
-                        distanceMeasure, exponentialWeight, stopCond, maxDelta, maxIterations,
-                        seed, initializationSteps, kMeansMaxIterations);
-
-                    // Create sample data.
-                    double[][] points = new double[][] {
-                        {-10, -10}, {-9, -11}, {-10, -9}, {-11, -9},
-                        {10, 10}, {9, 11}, {10, 9}, {11, 9},
-                        {-10, 10}, {-9, 11}, {-10, 9}, {-11, 9},
-                        {10, -10}, {9, -11}, {10, -9}, {11, -9}};
-
-                    // Initialize matrix of data points. Each row contains one point.
-                    int rows = points.length;
-                    int cols = points[0].length;
-
-                    System.out.println(">>> Create the matrix that contains sample points.");
-                    SparseDistributedMatrix pntMatrix = new SparseDistributedMatrix(rows, cols,
-                        StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
-                    // Store points into matrix.
-                    pntMatrix.assign(points);
-
-                    // Call clusterization method with some number of centers.
-                    // It returns model that can predict results for new points.
-                    System.out.println(">>> Perform clusterization.");
-                    int numCenters = 4;
-                    FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, numCenters);
-
-                    // You can also get centers of clusters that is computed by Fuzzy C-Means algorithm.
-                    Vector[] centers = mdl.centers();
-
-                    String res = ">>> Results:\n"
-                        + ">>> 1st center: " + centers[0].get(0) + " " + centers[0].get(1) + "\n"
-                        + ">>> 2nd center: " + centers[1].get(0) + " " + centers[1].get(1) + "\n"
-                        + ">>> 3rd center: " + centers[2].get(0) + " " + centers[2].get(1) + "\n"
-                        + ">>> 4th center: " + centers[3].get(0) + " " + centers[3].get(1) + "\n";
-
-                    System.out.println(res);
-
-                    pntMatrix.destroy();
-                });
-
-            igniteThread.start();
-            igniteThread.join();
-        }
-    }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/FuzzyCMeansLocalExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/FuzzyCMeansLocalExample.java
deleted file mode 100644
index 5c1753ad37ffb..0000000000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/FuzzyCMeansLocalExample.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.clustering;
-
-import org.apache.ignite.ml.clustering.BaseFuzzyCMeansClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansLocalClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansModel;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-
-/**
- * This example shows how to use {@link FuzzyCMeansLocalClusterer}.
- */
-public final class FuzzyCMeansLocalExample {
-    /**
-     * Executes example.
-     *
-     * @param args Command line arguments, none required.
-     */
-    public static void main(String[] args) {
-        System.out.println(">>> Local Fuzzy C-Means usage example started.");
-
-        // Distance measure that computes distance between two points.
-        DistanceMeasure distanceMeasure = new EuclideanDistance();
-
-        // "Fuzziness" - specific constant that is used in membership calculation (1.0+-eps ~ K-Means).
-        double exponentialWeight = 2.0;
-
-        // Condition that indicated when algorithm must stop.
-        // In this example algorithm stops if memberships have changed insignificantly.
-        BaseFuzzyCMeansClusterer.StopCondition stopCond =
-            BaseFuzzyCMeansClusterer.StopCondition.STABLE_MEMBERSHIPS;
-
-        // Maximum difference between new and old membership values with which algorithm will continue to work.
-        double maxDelta = 0.01;
-
-        // The maximum number of FCM iterations.
-        int maxIterations = 50;
-
-        // Value that is used to initialize random numbers generator. You can choose it randomly.
-        Long seed = null;
-
-        // Create new distributed clusterer with parameters described above.
-        System.out.println(">>> Create new Local Fuzzy C-Means clusterer.");
-        FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(distanceMeasure,
-            exponentialWeight, stopCond,
-            maxDelta, maxIterations, seed);
-
-        // Create sample data.
-        double[][] points = new double[][] {
-            {-10, -10}, {-9, -11}, {-10, -9}, {-11, -9},
-            {10, 10}, {9, 11}, {10, 9}, {11, 9},
-            {-10, 10}, {-9, 11}, {-10, 9}, {-11, 9},
-            {10, -10}, {9, -11}, {10, -9}, {11, -9}};
-
-        // Initialize matrix of data points. Each row contains one point.
-        System.out.println(">>> Create the matrix that contains sample points.");
-        // Store points into matrix.
-        DenseLocalOnHeapMatrix pntMatrix = new DenseLocalOnHeapMatrix(points);
-
-        // Call clusterization method with some number of centers.
-        // It returns model that can predict results for new points.
-        System.out.println(">>> Perform clusterization.");
-        int numCenters = 4;
-        FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, numCenters);
-
-        // You can also get centers of clusters that is computed by Fuzzy C-Means algorithm.
-        Vector[] centers = mdl.centers();
-
-        String res = ">>> Results:\n"
-            + ">>> 1st center: " + centers[0].get(0) + " " + centers[0].get(1) + "\n"
-            + ">>> 2nd center: " + centers[1].get(0) + " " + centers[1].get(1) + "\n"
-            + ">>> 3rd center: " + centers[2].get(0) + " " + centers[2].get(1) + "\n"
-            + ">>> 4th center: " + centers[3].get(0) + " " + centers[3].get(1) + "\n";
-
-        System.out.println(res);
-    }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
new file mode 100644
index 0000000000000..8825ebbd4a913
--- /dev/null
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.examples.ml.clustering;
+
+import java.util.Arrays;
+import java.util.UUID;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.ml.dataset.impl.cache.CacheBasedDatasetBuilder;
+import org.apache.ignite.ml.knn.classification.KNNClassificationTrainer;
+import org.apache.ignite.ml.math.Tracer;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.clustering.kmeans.KMeansModel;
+import org.apache.ignite.ml.clustering.kmeans.KMeansTrainer;
+import org.apache.ignite.thread.IgniteThread;
+
+/**
+ * Run kNN multi-class classification trainer over distributed dataset.
+ *
+ * @see KNNClassificationTrainer
+ */
+public class KMeansClusterizationExample {
+    /** Run example. */
+    public static void main(String[] args) throws InterruptedException {
+        System.out.println();
+        System.out.println(">>> KMeans clustering algorithm over cached dataset usage example started.");
+        // Start ignite grid.
+        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
+            System.out.println(">>> Ignite grid started.");
+
+            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
+                KMeansClusterizationExample.class.getSimpleName(), () -> {
+                IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
+
+                KMeansTrainer trainer = new KMeansTrainer()
+                    .withSeed(7867L);
+
+                KMeansModel mdl = trainer.fit(
+                    new CacheBasedDatasetBuilder<>(ignite, dataCache),
+                    (k, v) -> Arrays.copyOfRange(v, 1, v.length),
+                    (k, v) -> v[0]
+                );
+
+                System.out.println(">>> KMeans centroids");
+                Tracer.showAscii(mdl.centers()[0]);
+                Tracer.showAscii(mdl.centers()[1]);
+                System.out.println(">>>");
+
+                System.out.println(">>> -----------------------------------");
+                System.out.println(">>> | Predicted cluster\t| Real Label\t|");
+                System.out.println(">>> -----------------------------------");
+
+                int amountOfErrors = 0;
+                int totalAmount = 0;
+
+                try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+                    for (Cache.Entry<Integer, double[]> observation : observations) {
+                        double[] val = observation.getValue();
+                        double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+                        double groundTruth = val[0];
+
+                        double prediction = mdl.apply(new DenseLocalOnHeapVector(inputs));
+
+                        totalAmount++;
+                        if (groundTruth != prediction)
+                            amountOfErrors++;
+
+                        System.out.printf(">>> | %.4f\t\t\t| %.4f\t\t|\n", prediction, groundTruth);
+                    }
+
+                    System.out.println(">>> ---------------------------------");
+
+                    System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+                    System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+                }
+            });
+
+            igniteThread.start();
+            igniteThread.join();
+        }
+    }
+
+    /**
+     * Fills cache with data and returns it.
+     *
+     * @param ignite Ignite instance.
+     * @return Filled Ignite Cache.
+     */
+    private static IgniteCache<Integer, double[]> getTestCache(Ignite ignite) {
+        CacheConfiguration<Integer, double[]> cacheConfiguration = new CacheConfiguration<>();
+        cacheConfiguration.setName("TEST_" + UUID.randomUUID());
+        cacheConfiguration.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+        IgniteCache<Integer, double[]> cache = ignite.createCache(cacheConfiguration);
+
+        for (int i = 0; i < data.length; i++)
+            cache.put(i, data[i]);
+
+        return cache;
+    }
+
+    /** The Iris dataset. */
+    private static final double[][] data = {
+        {0, 5.1, 3.5, 1.4, 0.2},
+        {0, 4.9, 3, 1.4, 0.2},
+        {0, 4.7, 3.2, 1.3, 0.2},
+        {0, 4.6, 3.1, 1.5, 0.2},
+        {0, 5, 3.6, 1.4, 0.2},
+        {0, 5.4, 3.9, 1.7, 0.4},
+        {0, 4.6, 3.4, 1.4, 0.3},
+        {0, 5, 3.4, 1.5, 0.2},
+        {0, 4.4, 2.9, 1.4, 0.2},
+        {0, 4.9, 3.1, 1.5, 0.1},
+        {0, 5.4, 3.7, 1.5, 0.2},
+        {0, 4.8, 3.4, 1.6, 0.2},
+        {0, 4.8, 3, 1.4, 0.1},
+        {0, 4.3, 3, 1.1, 0.1},
+        {0, 5.8, 4, 1.2, 0.2},
+        {0, 5.7, 4.4, 1.5, 0.4},
+        {0, 5.4, 3.9, 1.3, 0.4},
+        {0, 5.1, 3.5, 1.4, 0.3},
+        {0, 5.7, 3.8, 1.7, 0.3},
+        {0, 5.1, 3.8, 1.5, 0.3},
+        {0, 5.4, 3.4, 1.7, 0.2},
+        {0, 5.1, 3.7, 1.5, 0.4},
+        {0, 4.6, 3.6, 1, 0.2},
+        {0, 5.1, 3.3, 1.7, 0.5},
+        {0, 4.8, 3.4, 1.9, 0.2},
+        {0, 5, 3, 1.6, 0.2},
+        {0, 5, 3.4, 1.6, 0.4},
+        {0, 5.2, 3.5, 1.5, 0.2},
+        {0, 5.2, 3.4, 1.4, 0.2},
+        {0, 4.7, 3.2, 1.6, 0.2},
+        {0, 4.8, 3.1, 1.6, 0.2},
+        {0, 5.4, 3.4, 1.5, 0.4},
+        {0, 5.2, 4.1, 1.5, 0.1},
+        {0, 5.5, 4.2, 1.4, 0.2},
+        {0, 4.9, 3.1, 1.5, 0.1},
+        {0, 5, 3.2, 1.2, 0.2},
+        {0, 5.5, 3.5, 1.3, 0.2},
+        {0, 4.9, 3.1, 1.5, 0.1},
+        {0, 4.4, 3, 1.3, 0.2},
+        {0, 5.1, 3.4, 1.5, 0.2},
+        {0, 5, 3.5, 1.3, 0.3},
+        {0, 4.5, 2.3, 1.3, 0.3},
+        {0, 4.4, 3.2, 1.3, 0.2},
+        {0, 5, 3.5, 1.6, 0.6},
+        {0, 5.1, 3.8, 1.9, 0.4},
+        {0, 4.8, 3, 1.4, 0.3},
+        {0, 5.1, 3.8, 1.6, 0.2},
+        {0, 4.6, 3.2, 1.4, 0.2},
+        {0, 5.3, 3.7, 1.5, 0.2},
+        {0, 5, 3.3, 1.4, 0.2},
+        {1, 7, 3.2, 4.7, 1.4},
+        {1, 6.4, 3.2, 4.5, 1.5},
+        {1, 6.9, 3.1, 4.9, 1.5},
+        {1, 5.5, 2.3, 4, 1.3},
+        {1, 6.5, 2.8, 4.6, 1.5},
+        {1, 5.7, 2.8, 4.5, 1.3},
+        {1, 6.3, 3.3, 4.7, 1.6},
+        {1, 4.9, 2.4, 3.3, 1},
+        {1, 6.6, 2.9, 4.6, 1.3},
+        {1, 5.2, 2.7, 3.9, 1.4},
+        {1, 5, 2, 3.5, 1},
+        {1, 5.9, 3, 4.2, 1.5},
+        {1, 6, 2.2, 4, 1},
+        {1, 6.1, 2.9, 4.7, 1.4},
+        {1, 5.6, 2.9, 3.6, 1.3},
+        {1, 6.7, 3.1, 4.4, 1.4},
+        {1, 5.6, 3, 4.5, 1.5},
+        {1, 5.8, 2.7, 4.1, 1},
+        {1, 6.2, 2.2, 4.5, 1.5},
+        {1, 5.6, 2.5, 3.9, 1.1},
+        {1, 5.9, 3.2, 4.8, 1.8},
+        {1, 6.1, 2.8, 4, 1.3},
+        {1, 6.3, 2.5, 4.9, 1.5},
+        {1, 6.1, 2.8, 4.7, 1.2},
+        {1, 6.4, 2.9, 4.3, 1.3},
+        {1, 6.6, 3, 4.4, 1.4},
+        {1, 6.8, 2.8, 4.8, 1.4},
+        {1, 6.7, 3, 5, 1.7},
+        {1, 6, 2.9, 4.5, 1.5},
+        {1, 5.7, 2.6, 3.5, 1},
+        {1, 5.5, 2.4, 3.8, 1.1},
+        {1, 5.5, 2.4, 3.7, 1},
+        {1, 5.8, 2.7, 3.9, 1.2},
+        {1, 6, 2.7, 5.1, 1.6},
+        {1, 5.4, 3, 4.5, 1.5},
+        {1, 6, 3.4, 4.5, 1.6},
+        {1, 6.7, 3.1, 4.7, 1.5},
+        {1, 6.3, 2.3, 4.4, 1.3},
+        {1, 5.6, 3, 4.1, 1.3},
+        {1, 5.5, 2.5, 4, 1.3},
+        {1, 5.5, 2.6, 4.4, 1.2},
+        {1, 6.1, 3, 4.6, 1.4},
+        {1, 5.8, 2.6, 4, 1.2},
+        {1, 5, 2.3, 3.3, 1},
+        {1, 5.6, 2.7, 4.2, 1.3},
+        {1, 5.7, 3, 4.2, 1.2},
+        {1, 5.7, 2.9, 4.2, 1.3},
+        {1, 6.2, 2.9, 4.3, 1.3},
+        {1, 5.1, 2.5, 3, 1.1},
+        {1, 5.7, 2.8, 4.1, 1.3},
+    };
+}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansDistributedClustererExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansDistributedClustererExample.java
deleted file mode 100644
index f8709e6da45b6..0000000000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansDistributedClustererExample.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.clustering;
-
-import java.util.Arrays;
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.examples.ExampleNodeStartup;
-import org.apache.ignite.examples.ml.math.matrix.SparseDistributedMatrixExample;
-import org.apache.ignite.ml.clustering.KMeansDistributedClusterer;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Tracer;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.thread.IgniteThread;
-
-/**
- * <p>
- * Example of using {@link KMeansDistributedClusterer}.</p>
- * <p>
- * Note that in this example we cannot guarantee order in which nodes return results of intermediate
- * computations and therefore algorithm can return different results.</p>
- * <p>
- * Remote nodes should always be started with special configuration file which
- * enables P2P class loading: {@code 'ignite.{sh|bat} examples/config/example-ignite.xml'}.</p>
- * <p>
- * Alternatively you can run {@link ExampleNodeStartup} in another JVM which will start node
- * with {@code examples/config/example-ignite.xml} configuration.</p>
- */
-public class KMeansDistributedClustererExample {
-    /**
-     * Executes example.
-     *
-     * @param args Command line arguments, none required.
-     */
-    public static void main(String[] args) throws InterruptedException {
-        // IMPL NOTE based on KMeansDistributedClustererTestSingleNode#testClusterizationOnDatasetWithObviousStructure
-        System.out.println(">>> K-means distributed clusterer example started.");
-
-        // Start ignite grid.
-        try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
-            System.out.println(">>> Ignite grid started.");
-
-            // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
-            // because we create ignite cache internally.
-            IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-                SparseDistributedMatrixExample.class.getSimpleName(), () -> {
-
-                int ptsCnt = 10000;
-
-                SparseDistributedMatrix points = new SparseDistributedMatrix(ptsCnt, 2,
-                    StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
-                DatasetWithObviousStructure dataset = new DatasetWithObviousStructure(10000);
-
-                List<Vector> massCenters = dataset.generate(points);
-
-                EuclideanDistance dist = new EuclideanDistance();
-
-                KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(dist, 3, 100, 1L);
-
-                Vector[] resCenters = clusterer.cluster(points, 4).centers();
-
-                System.out.println("Mass centers:");
-                massCenters.forEach(Tracer::showAscii);
-
-                System.out.println("Cluster centers:");
-                Arrays.asList(resCenters).forEach(Tracer::showAscii);
-
-                points.destroy();
-
-                System.out.println("\n>>> K-means distributed clusterer example completed.");
-            });
-
-            igniteThread.start();
-
-            igniteThread.join();
-        }
-    }
-}
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansLocalClustererExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansLocalClustererExample.java
deleted file mode 100644
index 28ca9d940d029..0000000000000
--- a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansLocalClustererExample.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.examples.ml.clustering;
-
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-import org.apache.ignite.ml.clustering.KMeansLocalClusterer;
-import org.apache.ignite.ml.clustering.KMeansModel;
-import org.apache.ignite.ml.math.Tracer;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.functions.Functions;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-
-/**
- * Example of using {@link KMeansLocalClusterer}.
- */
-public class KMeansLocalClustererExample {
-    /**
-     * Executes example.
-     *
-     * @param args Command line arguments, none required.
-     */
-    public static void main(String[] args) {
-        // IMPL NOTE based on KMeansDistributedClustererTestSingleNode#testClusterizationOnDatasetWithObviousStructure
-        System.out.println(">>> K-means local clusterer example started.");
-
-        int ptsCnt = 10000;
-        DenseLocalOnHeapMatrix points = new DenseLocalOnHeapMatrix(ptsCnt, 2);
-
-        DatasetWithObviousStructure dataset = new DatasetWithObviousStructure(10000);
-
-        List<Vector> massCenters = dataset.generate(points);
-
-        EuclideanDistance dist = new EuclideanDistance();
-        OrderedNodesComparator comp = new OrderedNodesComparator(
-            dataset.centers().values().toArray(new Vector[] {}), dist);
-
-        massCenters.sort(comp);
-
-        KMeansLocalClusterer clusterer = new KMeansLocalClusterer(dist, 100, 1L);
-
-        KMeansModel mdl = clusterer.cluster(points, 4);
-        Vector[] resCenters = mdl.centers();
-        Arrays.sort(resCenters, comp);
-
-        System.out.println("Mass centers:");
-        massCenters.forEach(Tracer::showAscii);
-
-        System.out.println("Cluster centers:");
-        Arrays.asList(resCenters).forEach(Tracer::showAscii);
-
-        System.out.println("\n>>> K-means local clusterer example completed.");
-    }
-
-    /** */
-    private static class OrderedNodesComparator implements Comparator<Vector> {
-        /** */
-        private final DistanceMeasure measure;
-
-        /** */
-        List<Vector> orderedNodes;
-
-        /** */
-        OrderedNodesComparator(Vector[] orderedNodes, DistanceMeasure measure) {
-            this.orderedNodes = Arrays.asList(orderedNodes);
-            this.measure = measure;
-        }
-
-        /** */
-        private int findClosestNodeIndex(Vector v) {
-            return Functions.argmin(orderedNodes, v1 -> measure.compute(v1, v)).get1();
-        }
-
-        /** */
-        @Override public int compare(Vector v1, Vector v2) {
-            int ind1 = findClosestNodeIndex(v1);
-            int ind2 = findClosestNodeIndex(v2);
-
-            int signum = (int)Math.signum(ind1 - ind2);
-
-            if (signum != 0)
-                return signum;
-
-            return (int)Math.signum(orderedNodes.get(ind1).minus(v1).kNorm(2) -
-                orderedNodes.get(ind2).minus(v2).kNorm(2));
-        }
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/FuzzyCMeansModelFormat.java b/modules/ml/src/main/java/org/apache/ignite/ml/FuzzyCMeansModelFormat.java
deleted file mode 100644
index cc3d9b3cf4309..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/FuzzyCMeansModelFormat.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-
-/** Fuzzy C-Means model representation. */
-public class FuzzyCMeansModelFormat implements Serializable {
-    /** Centers of clusters. */
-    private final Vector[] centers;
-
-    /** Distance measure. */
-    private final DistanceMeasure measure;
-
-    /**
-     * Constructor that retains result of clusterization and distance measure.
-     *
-     * @param centers Centers found while clusterization.
-     * @param measure Distance measure.
-     */
-    public FuzzyCMeansModelFormat(Vector[] centers, DistanceMeasure measure) {
-        this.centers = centers;
-        this.measure = measure;
-    }
-
-    /** Distance measure used while clusterization. */
-    public DistanceMeasure getMeasure() {
-        return measure;
-    }
-
-    /** Get cluster centers. */
-    public Vector[] getCenters() {
-        return centers;
-    }
-
-    /** {@inheritDoc} */
-    @Override public int hashCode() {
-        int res = 1;
-
-        res = res * 37 + measure.hashCode();
-        res = res * 37 + Arrays.hashCode(centers);
-
-        return res;
-    }
-
-    /** {@inheritDoc} */
-    @Override public boolean equals(Object obj) {
-        if (this == obj)
-            return true;
-
-        if (obj == null || getClass() != obj.getClass())
-            return false;
-
-        FuzzyCMeansModelFormat that = (FuzzyCMeansModelFormat) obj;
-
-        return measure.equals(that.measure) && Arrays.deepEquals(centers, that.centers);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseFuzzyCMeansClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseFuzzyCMeansClusterer.java
deleted file mode 100644
index 2b2febfcd2c75..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseFuzzyCMeansClusterer.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.exceptions.ConvergenceException;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-
-/** The abstract class that defines the basic interface of Fuzzy C-Means clusterers */
-public abstract class BaseFuzzyCMeansClusterer<T extends Matrix> implements Clusterer<T, FuzzyCMeansModel> {
-    /** Distance measure. */
-    protected DistanceMeasure measure;
-
-    /** Specific constant which is used in calculating of membership matrix. */
-    protected double exponentialWeight;
-
-    /** The maximum distance between old and new centers or the maximum difference between new and old membership matrix
-     *  elements for which algorithm must stop. */
-    protected double maxDelta;
-
-    /** The flag that tells when algorithm should stop. */
-    protected StopCondition stopCond;
-
-    /**
-     * Constructor that stores some required parameters.
-     *
-     * @param measure Distance measure.
-     * @param exponentialWeight Specific constant which is used in calculating of membership matrix.
-     * @param stopCond Flag that tells when algorithm should stop.
-     * @param maxDelta The maximum distance between old and new centers or maximum difference between new and old
-     *                 membership matrix elements for which algorithm must stop.
-     */
-    protected BaseFuzzyCMeansClusterer(DistanceMeasure measure, double exponentialWeight, StopCondition stopCond,
-                                       double maxDelta) {
-        this.measure = measure;
-        this.exponentialWeight = exponentialWeight;
-        this.stopCond = stopCond;
-        this.maxDelta = maxDelta;
-    }
-
-    /**
-     * Perform a cluster analysis on the given set of points.
-     *
-     * @param points The set of points.
-     * @return A list of clusters.
-     * @throws MathIllegalArgumentException If points are null or the number of data points is not compatible with this
-     *                                      clusterer.
-     * @throws ConvergenceException If the algorithm has not yet converged after the maximum number of iterations has
-     *                              been exceeded.
-     */
-    public abstract FuzzyCMeansModel cluster(T points, int k);
-
-    /**
-     * Calculates the distance between two vectors. * with the configured {@link DistanceMeasure}.
-     *
-     * @return The distance between two points.
-     */
-    protected double distance(final Vector v1, final Vector v2) {
-        return measure.compute(v1, v2);
-    }
-
-    /** Enumeration that contains different conditions under which algorithm must stop. */
-    public enum StopCondition {
-        /** Algorithm stops if the maximum distance between new and old centers is less than {@link #maxDelta}. */
-        STABLE_CENTERS,
-
-        /**
-         * Algorithm stops if the maximum difference between elements of new and old membership matrix is less than
-         * {@link #maxDelta}.
-         */
-        STABLE_MEMBERSHIPS
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java
deleted file mode 100644
index 521437c71b2f6..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/BaseKMeansClusterer.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.util.List;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.exceptions.ConvergenceException;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-
-/**
- * This class is partly based on the corresponding class from Apache Common Math lib.
- */
-public abstract class BaseKMeansClusterer<T extends Matrix> implements Clusterer<T, KMeansModel> {
-    /** The distance measure to use. */
-    private DistanceMeasure measure;
-
-    /**
-     * Build a new clusterer with the given {@link DistanceMeasure}.
-     *
-     * @param measure the distance measure to use
-     */
-    protected BaseKMeansClusterer(final DistanceMeasure measure) {
-        this.measure = measure;
-    }
-
-    /**
-     * Perform a cluster analysis on the given set of points.
-     *
-     * @param points the set of points
-     * @return a {@link List} of clusters
-     * @throws MathIllegalArgumentException if points are null or the number of data points is not compatible with this
-     * clusterer
-     * @throws ConvergenceException if the algorithm has not yet converged after the maximum number of iterations has
-     * been exceeded
-     */
-    public abstract KMeansModel cluster(T points, int k)
-        throws MathIllegalArgumentException, ConvergenceException;
-
-    /**
-     * Returns the {@link DistanceMeasure} instance used by this clusterer.
-     *
-     * @return the distance measure
-     */
-    public DistanceMeasure getDistanceMeasure() {
-        return measure;
-    }
-
-    /**
-     * Calculates the distance between two vectors.
-     * with the configured {@link DistanceMeasure}.
-     *
-     * @return the distance between the two clusterables
-     */
-    protected double distance(final Vector v1, final Vector v2) {
-        return measure.compute(v1, v2);
-    }
-
-    /**
-     * Find the closest cluster center index and distance to it from a given point.
-     *
-     * @param centers Centers to look in.
-     * @param pnt Point.
-     */
-    protected IgniteBiTuple<Integer, Double> findClosest(Vector[] centers, Vector pnt) {
-        double bestDistance = Double.POSITIVE_INFINITY;
-        int bestInd = 0;
-
-        for (int i = 0; i < centers.length; i++) {
-            double dist = distance(centers[i], pnt);
-            if (dist < bestDistance) {
-                bestDistance = dist;
-                bestInd = i;
-            }
-        }
-
-        return new IgniteBiTuple<>(bestInd, bestDistance);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansDistributedClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansDistributedClusterer.java
deleted file mode 100644
index 8823c10676485..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansDistributedClusterer.java
+++ /dev/null
@@ -1,512 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-import javax.cache.Cache;
-import org.apache.ignite.internal.util.GridArgumentCheck;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorUtils;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distributed.CacheUtils;
-import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey;
-import org.apache.ignite.ml.math.exceptions.ConvergenceException;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.math.functions.Functions;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.functions.IgniteSupplier;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.ml.math.util.MatrixUtil;
-
-/** This class implements distributed version of Fuzzy C-Means clusterization of equal-weighted points. */
-public class FuzzyCMeansDistributedClusterer extends BaseFuzzyCMeansClusterer<SparseDistributedMatrix> {
-    /** Random numbers generator which is used in centers selection. */
-    private Random rnd;
-
-    /** The value that is used to initialize random numbers generator. */
-    private long seed;
-
-    /** The number of initialization steps each of which adds some number of candidates for being a center. */
-    private int initSteps;
-
-    /** The maximum number of iterations of K-Means algorithm which selects the required number of centers. */
-    private int kMeansMaxIterations;
-
-    /** The maximum number of FCM iterations. */
-    private int cMeansMaxIterations;
-
-    /**
-     * Constructor that retains all required parameters.
-     *
-     * @param measure Distance measure.
-     * @param exponentialWeight Specific constant which is used in calculating of membership matrix.
-     * @param stopCond Flag that tells when algorithm should stop.
-     * @param maxDelta The maximum distance between old and new centers or maximum difference between new and old
-     *                 membership matrix elements for which algorithm must stop.
-     * @param cMeansMaxIterations The maximum number of FCM iterations.
-     * @param seed Seed for random numbers generator.
-     * @param initSteps Number of steps of primary centers selection (the more steps, the more candidates).
-     * @param kMeansMaxIterations The maximum number of K-Means iteration in primary centers selection.
-     */
-    public FuzzyCMeansDistributedClusterer(DistanceMeasure measure, double exponentialWeight,
-                                           StopCondition stopCond, double maxDelta, int cMeansMaxIterations,
-                                           Long seed, int initSteps, int kMeansMaxIterations) {
-        super(measure, exponentialWeight, stopCond, maxDelta);
-
-        this.seed = seed != null ? seed : new Random().nextLong();
-        this.initSteps = initSteps;
-        this.cMeansMaxIterations = cMeansMaxIterations;
-        this.kMeansMaxIterations = kMeansMaxIterations;
-        rnd = new Random(this.seed);
-    }
-
-    /** {@inheritDoc} */
-    @Override public FuzzyCMeansModel cluster(SparseDistributedMatrix points, int k)
-            throws MathIllegalArgumentException, ConvergenceException {
-        GridArgumentCheck.notNull(points, "points");
-
-        if (k < 2)
-            throw new MathIllegalArgumentException("The number of clusters is less than 2");
-
-        Vector[] centers = initializeCenters(points, k);
-
-        MembershipsAndSums membershipsAndSums = null;
-
-        int iteration = 0;
-        boolean finished = false;
-        while (!finished && iteration < cMeansMaxIterations) {
-            MembershipsAndSums newMembershipsAndSums = calculateMembership(points, centers);
-            Vector[] newCenters = calculateNewCenters(points, newMembershipsAndSums, k);
-
-            if (stopCond == StopCondition.STABLE_CENTERS)
-                finished = isFinished(centers, newCenters);
-            else
-                finished = isFinished(membershipsAndSums, newMembershipsAndSums);
-
-            centers = newCenters;
-            membershipsAndSums = newMembershipsAndSums;
-
-            iteration++;
-        }
-
-        if (iteration == cMeansMaxIterations)
-            throw new ConvergenceException("Fuzzy C-Means algorithm has not converged after " +
-                    Integer.toString(iteration) + " iterations");
-
-        return new FuzzyCMeansModel(centers, measure);
-    }
-
-    /**
-     * Choose k primary centers from source points.
-     *
-     * @param points Matrix with source points.
-     * @param k Number of centers.
-     * @return Array of primary centers.
-     */
-    private Vector[] initializeCenters(SparseDistributedMatrix points, int k) {
-        int pointsNum = points.rowSize();
-
-        Vector firstCenter = points.viewRow(rnd.nextInt(pointsNum));
-
-        List<Vector> centers = new ArrayList<>();
-        List<Vector> newCenters = new ArrayList<>();
-
-        centers.add(firstCenter);
-        newCenters.add(firstCenter);
-
-        ConcurrentHashMap<Integer, Double> costs = new ConcurrentHashMap<>();
-
-        int step = 0;
-        UUID uuid = points.getUUID();
-        String cacheName = ((SparseDistributedMatrixStorage) points.getStorage()).cacheName();
-
-        while(step < initSteps) {
-            ConcurrentHashMap<Integer, Double> newCosts = getNewCosts(cacheName, uuid, newCenters);
-
-            for (Integer key : newCosts.keySet())
-                costs.merge(key, newCosts.get(key), Math::min);
-
-            double costsSum = costs.values().stream().mapToDouble(Double::valueOf).sum();
-
-            newCenters = getNewCenters(cacheName, uuid, costs, costsSum, k);
-            centers.addAll(newCenters);
-
-            step++;
-        }
-
-        return chooseKCenters(cacheName, uuid, centers, k);
-    }
-
-    /**
-     * Calculate new distances from each point to the nearest center.
-     *
-     * @param cacheName Cache name of point matrix.
-     * @param uuid Uuid of point matrix.
-     * @param newCenters The list of centers that was added on previous step.
-     * @return Hash map with distances.
-     */
-    private ConcurrentHashMap<Integer, Double> getNewCosts(String cacheName, UUID uuid,
-                                                           List<Vector> newCenters) {
-        return CacheUtils.distributedFold(cacheName,
-                (IgniteBiFunction<Cache.Entry<SparseMatrixKey, ConcurrentHashMap<Integer, Double>>,
-                        ConcurrentHashMap<Integer, Double>,
-                        ConcurrentHashMap<Integer, Double>>)(vectorWithIndex, map) -> {
-                    Vector vector = VectorUtils.fromMap(vectorWithIndex.getValue(), false);
-
-                    for (Vector center : newCenters)
-                        map.merge(vectorWithIndex.getKey().index(), distance(vector, center), Functions.MIN);
-
-                    return map;
-                },
-                key -> key.dataStructureId().equals(uuid),
-                (map1, map2) -> {
-                    map1.putAll(map2);
-                    return map1;
-                },
-                ConcurrentHashMap::new);
-    }
-
-    /**
-     * Choose some number of center candidates from source points according to their costs.
-     *
-     * @param cacheName Cache name of point matrix.
-     * @param uuid Uuid of point matrix.
-     * @param costs Hash map with costs (distances to nearest center).
-     * @param costsSum The sum of costs.
-     * @param k The estimated number of centers.
-     * @return The list of new candidates.
-     */
-    private List<Vector> getNewCenters(String cacheName, UUID uuid,
-                                       ConcurrentHashMap<Integer, Double> costs, double costsSum, int k) {
-        return CacheUtils.distributedFold(cacheName,
-                (IgniteBiFunction<Cache.Entry<SparseMatrixKey, Map<Integer, Double>>,
-                                  List<Vector>,
-                                  List<Vector>>)(vectorWithIndex, centers) -> {
-                    Integer idx = vectorWithIndex.getKey().index();
-                    Vector vector = VectorUtils.fromMap(vectorWithIndex.getValue(), false);
-
-                    double probability = (costs.get(idx) * 2.0 * k) / costsSum;
-
-                    if (rnd.nextDouble() < probability)
-                        centers.add(vector);
-
-                    return centers;
-                },
-                key -> key.dataStructureId().equals(uuid),
-                (list1, list2) -> {
-                    list1.addAll(list2);
-                    return list1;
-                },
-                ArrayList::new);
-    }
-
-    /**
-     * Weight candidates and use K-Means to choose required number of them.
-     *
-     * @param cacheName Cache name of the point matrix.
-     * @param uuid Uuid of the point matrix.
-     * @param centers The list of candidates.
-     * @param k The estimated number of centers.
-     * @return {@code k} centers.
-     */
-    private Vector[] chooseKCenters(String cacheName, UUID uuid, List<Vector> centers, int k) {
-        centers = centers.stream().distinct().collect(Collectors.toList());
-
-        ConcurrentHashMap<Integer, Integer> weightsMap = weightCenters(cacheName, uuid, centers);
-
-        List<Double> weights = new ArrayList<>(centers.size());
-
-        for (int i = 0; i < centers.size(); i++)
-            weights.add(i, Double.valueOf(weightsMap.getOrDefault(i, 0)));
-
-        DenseLocalOnHeapMatrix centersMatrix = MatrixUtil.fromList(centers, true);
-
-        KMeansLocalClusterer clusterer = new KMeansLocalClusterer(measure, kMeansMaxIterations, seed);
-        return clusterer.cluster(centersMatrix, k, weights).centers();
-    }
-
-    /**
-     * Weight each center with number of points for which this center is the nearest.
-     *
-     * @param cacheName Cache name of the point matrix.
-     * @param uuid Uuid of the point matrix.
-     * @param centers The list of centers.
-     * @return Hash map with weights.
-     */
-    public ConcurrentHashMap<Integer, Integer> weightCenters(String cacheName, UUID uuid, List<Vector> centers) {
-        if (centers.size() == 0)
-            return new ConcurrentHashMap<>();
-
-        return CacheUtils.distributedFold(cacheName,
-                (IgniteBiFunction<Cache.Entry<SparseMatrixKey, ConcurrentHashMap<Integer, Double>>,
-                                  ConcurrentHashMap<Integer, Integer>,
-                                  ConcurrentHashMap<Integer, Integer>>)(vectorWithIndex, counts) -> {
-                    Vector vector = VectorUtils.fromMap(vectorWithIndex.getValue(), false);
-
-                    int nearest = 0;
-                    double minDistance = distance(centers.get(nearest), vector);
-
-                    for (int i = 0; i < centers.size(); i++) {
-                        double currDistance = distance(centers.get(i), vector);
-                        if (currDistance < minDistance) {
-                            minDistance = currDistance;
-                            nearest = i;
-                        }
-                    }
-
-                    counts.compute(nearest, (index, value) -> value == null ? 1 : value + 1);
-
-                    return counts;
-                },
-                key -> key.dataStructureId().equals(uuid),
-                (map1, map2) -> {
-                    map1.putAll(map2);
-                    return map1;
-                },
-                ConcurrentHashMap::new);
-    }
-
-    /**
-     * Calculate matrix of membership coefficients for each point and each center.
-     *
-     * @param points Matrix with source points.
-     * @param centers Array of current centers.
-     * @return Membership matrix and sums of membership coefficients for each center.
-     */
-    private MembershipsAndSums calculateMembership(SparseDistributedMatrix points, Vector[] centers) {
-        String cacheName = ((SparseDistributedMatrixStorage) points.getStorage()).cacheName();
-        UUID uuid = points.getUUID();
-        double fuzzyMembershipCoefficient = 2 / (exponentialWeight - 1);
-
-        MembershipsAndSumsSupplier supplier = new MembershipsAndSumsSupplier(centers.length);
-
-        return CacheUtils.distributedFold(cacheName,
-                (IgniteBiFunction<Cache.Entry<SparseMatrixKey, ConcurrentHashMap<Integer, Double>>,
-                        MembershipsAndSums,
-                        MembershipsAndSums>)(vectorWithIndex, membershipsAndSums) -> {
-                    Integer idx = vectorWithIndex.getKey().index();
-                    Vector pnt = VectorUtils.fromMap(vectorWithIndex.getValue(), false);
-                    Vector distances = new DenseLocalOnHeapVector(centers.length);
-                    Vector pntMemberships = new DenseLocalOnHeapVector(centers.length);
-
-                    for (int i = 0; i < centers.length; i++)
-                        distances.setX(i, distance(centers[i], pnt));
-
-                    for (int i = 0; i < centers.length; i++) {
-                        double invertedFuzzyWeight = 0.0;
-
-                        for (int j = 0; j < centers.length; j++) {
-                            double val = Math.pow(distances.getX(i) / distances.getX(j), fuzzyMembershipCoefficient);
-                            if (Double.isNaN(val))
-                                val = 1.0;
-
-                            invertedFuzzyWeight += val;
-                        }
-
-                        double membership = Math.pow(1.0 / invertedFuzzyWeight, exponentialWeight);
-                        pntMemberships.setX(i, membership);
-                    }
-
-                    membershipsAndSums.memberships.put(idx, pntMemberships);
-                    membershipsAndSums.membershipSums = membershipsAndSums.membershipSums.plus(pntMemberships);
-
-                    return membershipsAndSums;
-                },
-                key -> key.dataStructureId().equals(uuid),
-                (mem1, mem2) -> {
-                    mem1.merge(mem2);
-                    return mem1;
-                },
-                supplier);
-    }
-
-    /**
-     * Calculate new centers according to membership matrix.
-     *
-     * @param points Matrix with source points.
-     * @param membershipsAndSums Membership matrix and sums of membership coefficient for each center.
-     * @param k The number of centers.
-     * @return Array of new centers.
-     */
-    private Vector[] calculateNewCenters(SparseDistributedMatrix points, MembershipsAndSums membershipsAndSums, int k) {
-        String cacheName = ((SparseDistributedMatrixStorage) points.getStorage()).cacheName();
-        UUID uuid = points.getUUID();
-
-        CentersArraySupplier supplier = new CentersArraySupplier(k, points.columnSize());
-
-        Vector[] centers = CacheUtils.distributedFold(cacheName,
-                (IgniteBiFunction<Cache.Entry<SparseMatrixKey, ConcurrentHashMap<Integer, Double>>,
-                                  Vector[],
-                                  Vector[]>)(vectorWithIndex, centerSums) -> {
-                    Integer idx = vectorWithIndex.getKey().index();
-                    Vector pnt = MatrixUtil.localCopyOf(VectorUtils.fromMap(vectorWithIndex.getValue(), false));
-                    Vector pntMemberships = membershipsAndSums.memberships.get(idx);
-
-                    for (int i = 0; i < k; i++) {
-                        Vector weightedPnt = pnt.times(pntMemberships.getX(i));
-                        centerSums[i] = centerSums[i].plus(weightedPnt);
-                    }
-
-                    return centerSums;
-                },
-                key -> key.dataStructureId().equals(uuid),
-                (sums1, sums2) -> {
-                    for (int i = 0; i < k; i++)
-                        sums1[i] = sums1[i].plus(sums2[i]);
-
-                    return sums1;
-                },
-                supplier);
-
-        for (int i = 0; i < k; i++)
-            centers[i] = centers[i].divide(membershipsAndSums.membershipSums.getX(i));
-
-        return centers;
-    }
-
-    /**
-     * Check if centers have moved insignificantly.
-     *
-     * @param centers Old centers.
-     * @param newCenters New centers.
-     * @return The result of comparison.
-     */
-    private boolean isFinished(Vector[] centers, Vector[] newCenters) {
-        int numCenters = centers.length;
-
-        for (int i = 0; i < numCenters; i++)
-            if (distance(centers[i], newCenters[i]) > maxDelta)
-                return false;
-
-        return true;
-    }
-
-    /**
-     * Check memberships difference.
-     *
-     * @param membershipsAndSums Old memberships.
-     * @param newMembershipsAndSums New memberships.
-     * @return The result of comparison.
-     */
-    private boolean isFinished(MembershipsAndSums membershipsAndSums, MembershipsAndSums newMembershipsAndSums) {
-        if (membershipsAndSums == null)
-            return false;
-
-        double currMaxDelta = 0.0;
-        for (Integer key : membershipsAndSums.memberships.keySet()) {
-            double distance = measure.compute(membershipsAndSums.memberships.get(key),
-                                              newMembershipsAndSums.memberships.get(key));
-            if (distance > currMaxDelta)
-                currMaxDelta = distance;
-        }
-
-        return currMaxDelta <= maxDelta;
-    }
-
-    /** Service class used to optimize counting of membership sums. */
-    private class MembershipsAndSums {
-        /** Membership matrix. */
-        public ConcurrentHashMap<Integer, Vector> memberships = new ConcurrentHashMap<>();
-
-        /** Membership sums. */
-        public Vector membershipSums;
-
-        /**
-         * Default constructor.
-         *
-         * @param k The number of centers.
-         */
-        public MembershipsAndSums(int k) {
-            membershipSums = new DenseLocalOnHeapVector(k);
-        }
-
-        /**
-         * Merge results of calculation for different parts of points.
-         * @param another Another part of memberships and sums.
-         */
-        public void merge(MembershipsAndSums another) {
-            memberships.putAll(another.memberships);
-            membershipSums = membershipSums.plus(another.membershipSums);
-        }
-    }
-
-    /** Service class that is used to create new {@link MembershipsAndSums} instances. */
-    private class MembershipsAndSumsSupplier implements IgniteSupplier<MembershipsAndSums> {
-        /** The number of centers */
-        int k;
-
-        /**
-         * Constructor that retains the number of centers.
-         *
-         * @param k The number of centers.
-         */
-        public MembershipsAndSumsSupplier(int k) {
-            this.k = k;
-        }
-
-        /**
-         * Create new instance of {@link MembershipsAndSums}.
-         *
-         * @return {@link MembershipsAndSums} object.
-         */
-        @Override public MembershipsAndSums get() {
-            return new MembershipsAndSums(k);
-        }
-    }
-
-    /** Service class that is used to create new arrays of vectors. */
-    private class CentersArraySupplier implements IgniteSupplier<Vector[]> {
-        /** The number of centers. */
-        int k;
-
-        /** The number of coordinates. */
-        int dim;
-
-        /**
-         * Constructor that retains all required parameters.
-         *
-         * @param k The number of centers.
-         * @param dim The number of coordinates.
-         */
-        public CentersArraySupplier(int k, int dim) {
-            this.k = k;
-            this.dim = dim;
-        }
-
-        /**
-         * Create new array of vectors.
-         *
-         * @return Array of vectors.
-         */
-        @Override public Vector[] get() {
-            DenseLocalOnHeapVector[] centerSumsArr = new DenseLocalOnHeapVector[k];
-            for (int i = 0; i < k; i++)
-                centerSumsArr[i] = new DenseLocalOnHeapVector(dim);
-            return centerSumsArr;
-        }
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansLocalClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansLocalClusterer.java
deleted file mode 100644
index a1b6d3faa1134..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansLocalClusterer.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import org.apache.ignite.internal.util.GridArgumentCheck;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.exceptions.ConvergenceException;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-
-/** Implements the local version of Fuzzy C-Means algorithm for weighted points. */
-public class FuzzyCMeansLocalClusterer extends BaseFuzzyCMeansClusterer<DenseLocalOnHeapMatrix> implements
-        WeightedClusterer<DenseLocalOnHeapMatrix, FuzzyCMeansModel> {
-    /** The maximum number of iterations. */
-    private int maxIterations;
-
-    /** The random numbers generator that is used to choose primary centers. */
-    private Random rnd;
-
-    /**
-     * Constructor that retains all required parameters.
-     *
-     * @param measure Distance measure.
-     * @param exponentialWeight Specific constant which is used in calculating of membership matrix.
-     * @param stopCond Flag that tells when algorithm should stop.
-     * @param maxDelta The maximum distance between old and new centers or maximum difference between new and old
-     *                 membership matrix elements for which algorithm must stop.
-     * @param maxIterations The maximum number of FCM iterations.
-     */
-    public FuzzyCMeansLocalClusterer(DistanceMeasure measure, double exponentialWeight, StopCondition stopCond,
-                                     double maxDelta, int maxIterations, Long seed) {
-        super(measure, exponentialWeight, stopCond, maxDelta);
-        this.maxIterations = maxIterations;
-        rnd = seed != null ? new Random(seed) : new Random();
-    }
-
-    /** {@inheritDoc} */
-    @Override public FuzzyCMeansModel cluster(DenseLocalOnHeapMatrix points, int k) {
-        List<Double> ones = new ArrayList<>(Collections.nCopies(points.rowSize(), 1.0));
-        return cluster(points, k, ones);
-    }
-
-    /** {@inheritDoc} */
-    @Override public FuzzyCMeansModel cluster(DenseLocalOnHeapMatrix points, int k, List<Double> weights)
-            throws MathIllegalArgumentException, ConvergenceException {
-        GridArgumentCheck.notNull(points, "points");
-        GridArgumentCheck.notNull(weights, "weights");
-
-        if (points.rowSize() != weights.size())
-            throw new MathIllegalArgumentException("The number of points and the number of weights are not equal");
-
-        if (k < 2)
-            throw new MathIllegalArgumentException("The number of clusters is less than 2");
-
-        Matrix centers = new DenseLocalOnHeapMatrix(k, points.columnSize());
-        Matrix distances = new DenseLocalOnHeapMatrix(k, points.rowSize());
-        Matrix membership = new DenseLocalOnHeapMatrix(k, points.rowSize());
-        Vector weightsVector = new DenseLocalOnHeapVector(weights.size());
-        for (int i = 0; i < weights.size(); i++)
-            weightsVector.setX(i, weights.get(i));
-
-        initializeCenters(centers, points, k, weightsVector);
-
-        int iteration = 0;
-        boolean finished = false;
-        while (iteration < maxIterations && !finished) {
-            calculateDistances(distances, points, centers);
-            Matrix newMembership = calculateMembership(distances, weightsVector);
-            Matrix newCenters = calculateNewCenters(points, newMembership);
-
-            if (this.stopCond == StopCondition.STABLE_CENTERS)
-                finished = areCentersStable(centers, newCenters);
-            else
-                finished = areMembershipStable(membership, newMembership);
-
-            centers = newCenters;
-            membership = newMembership;
-            iteration++;
-        }
-
-        if (iteration == maxIterations)
-            throw new ConvergenceException("Fuzzy C-Means algorithm has not converged after " +
-                    Integer.toString(iteration) + " iterations");
-
-        Vector[] centersArr = new Vector[k];
-        for (int i = 0; i < k; i++)
-            centersArr[i] = centers.getRow(i);
-
-        return new FuzzyCMeansModel(centersArr, measure);
-    }
-
-    /**
-     * Choose {@code k} centers according to their weights.
-     *
-     * @param centers Output matrix containing primary centers.
-     * @param points Matrix of source points.
-     * @param k The number of centers.
-     * @param weights Vector of weights.
-     */
-    private void initializeCenters(Matrix centers, Matrix points, int k, Vector weights) {
-        //int dimensions = points.columnSize();
-        int numPoints = points.rowSize();
-
-        Vector firstCenter = points.viewRow(rnd.nextInt(numPoints));
-        centers.setRow(0, firstCenter.getStorage().data());
-
-        Vector costs = points.foldRows(vector -> distance(vector, firstCenter));
-        costs = costs.times(weights);
-
-        double sum = costs.sum();
-
-        for (int i = 1; i < k; i++) {
-            double probe = rnd.nextDouble() * sum;
-            double cntr = 0;
-            int id = 0;
-
-            for (int j = 0; j < numPoints; j++) {
-                cntr += costs.getX(j);
-                if (cntr >= probe) {
-                    id = j;
-                    break;
-                }
-            }
-
-            centers.setRow(i, points.viewRow(id).getStorage().data());
-            sum -= costs.get(id);
-            costs.set(id, 0.0);
-        }
-    }
-
-    /**
-     * Calculate matrix of distances form each point to each center.
-     *
-     * @param distances Output matrix.
-     * @param points Matrix that contains source points.
-     * @param centers Matrix that contains centers.
-     */
-    private void calculateDistances(Matrix distances, Matrix points, Matrix centers) {
-        int numPoints = points.rowSize();
-        int numCenters = centers.rowSize();
-
-        for (int i = 0; i < numCenters; i++)
-            for (int j = 0; j < numPoints; j++)
-                distances.set(i, j, distance(centers.viewRow(i), points.viewRow(j)));
-    }
-
-    /**
-     * Calculate membership matrix.
-     *
-     * @param distances Matrix of distances.
-     * @param weights Vector of weights.
-     * @
-     */
-    private Matrix calculateMembership(Matrix distances, Vector weights) {
-        Matrix newMembership = new DenseLocalOnHeapMatrix(distances.rowSize(), distances.columnSize());
-        int numPoints = distances.columnSize();
-        int numCenters = distances.rowSize();
-        double fuzzyMembershipCoefficient = 2 / (exponentialWeight - 1);
-
-        for (int i = 0; i < numCenters; i++) {
-            for (int j = 0; j < numPoints; j++) {
-                double invertedFuzzyWeight = 0.0;
-
-                for (int k = 0; k < numCenters; k++) {
-                    double val = Math.pow(distances.get(i, j) / distances.get(k, j),
-                            fuzzyMembershipCoefficient);
-                    if (Double.isNaN(val))
-                        val = 1.0;
-
-                    invertedFuzzyWeight += val;
-                }
-
-                double weight = 1.0 / invertedFuzzyWeight * weights.getX(j);
-                newMembership.setX(i, j, Math.pow(weight, exponentialWeight));
-            }
-        }
-        return newMembership;
-    }
-
-    /**
-     * Calculate new centers using membership matrix.
-     *
-     * @param points Matrix of source points.
-     * @param membership Matrix that contains membership coefficients.
-     * @return Matrix that contains new centers.
-     */
-    private Matrix calculateNewCenters(Matrix points, Matrix membership) {
-        Vector membershipSums = membership.foldRows(Vector::sum);
-        Matrix newCenters = membership.times(points);
-
-        int numCenters = newCenters.rowSize();
-        for (int i = 0; i < numCenters; i++)
-            newCenters.viewRow(i).divide(membershipSums.getX(i));
-
-        return newCenters;
-    }
-
-    /**
-     * Check if centers have moved insignificantly.
-     *
-     * @param centers Old centers.
-     * @param newCenters New centers.
-     * @return The result of comparison.
-     */
-    private boolean areCentersStable(Matrix centers, Matrix newCenters) {
-        int numCenters = centers.rowSize();
-        for (int i = 0; i < numCenters; i++)
-            if (distance(centers.viewRow(i), newCenters.viewRow(i)) > maxDelta)
-                return false;
-
-        return true;
-    }
-
-    /**
-     * Check if membership matrix has changed insignificantly.
-     *
-     * @param membership Old membership matrix.
-     * @param newMembership New membership matrix.
-     * @return The result of comparison.
-     */
-    private boolean areMembershipStable(Matrix membership, Matrix newMembership) {
-        int numCenters = membership.rowSize();
-        int numPoints = membership.columnSize();
-
-        for (int i = 0; i < numCenters; i++)
-            for (int j = 0; j < numPoints; j++)
-                if (Math.abs(newMembership.getX(i, j) - membership.getX(i, j)) > maxDelta)
-                    return false;
-
-        return true;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java
deleted file mode 100644
index 70009cb91379b..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/FuzzyCMeansModel.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.util.Arrays;
-import org.apache.ignite.ml.Exportable;
-import org.apache.ignite.ml.Exporter;
-import org.apache.ignite.ml.FuzzyCMeansModelFormat;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-
-/** This class incapsulates result of clusterization. */
-public class FuzzyCMeansModel implements ClusterizationModel<Vector, Integer>, Exportable<FuzzyCMeansModelFormat> {
-    /** Centers of clusters. */
-    private Vector[] centers;
-
-    /** Distance measure. */
-    private DistanceMeasure measure;
-
-    /**
-     * Constructor that creates FCM model by centers and measure.
-     *
-     * @param centers Array of centers.
-     * @param measure Distance measure.
-     */
-    public FuzzyCMeansModel(Vector[] centers, DistanceMeasure measure) {
-        this.centers = Arrays.copyOf(centers, centers.length);
-        this.measure = measure;
-    }
-
-    /** Distance measure used while clusterization. */
-    public DistanceMeasure distanceMeasure() {
-        return measure;
-    }
-
-    /** @inheritDoc */
-    @Override public int clustersCount() {
-        return centers.length;
-    }
-
-    /** @inheritDoc */
-    @Override public Vector[] centers() {
-        return Arrays.copyOf(centers, centers.length);
-    }
-
-    /**
-     * Predict closest center index for a given vector.
-     *
-     * @param val Vector.
-     * @return Index of the closest center or -1 if it can't be found.
-     */
-    @Override public Integer apply(Vector val) {
-        int idx = -1;
-        double minDistance = Double.POSITIVE_INFINITY;
-
-        for (int i = 0; i < centers.length; i++) {
-            double currDistance = measure.compute(val, centers[i]);
-            if (currDistance < minDistance) {
-                minDistance = currDistance;
-                idx = i;
-            }
-        }
-
-        return idx;
-    }
-
-    /** {@inheritDoc} */
-    @Override public <P> void saveModel(Exporter<FuzzyCMeansModelFormat, P> exporter, P path) {
-        FuzzyCMeansModelFormat mdlData = new FuzzyCMeansModelFormat(centers, measure);
-
-        exporter.save(mdlData, path);
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
deleted file mode 100644
index 5595b4cd3b83c..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansDistributedClusterer.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.stream.Collectors;
-import javax.cache.Cache;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorUtils;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distributed.CacheUtils;
-import org.apache.ignite.ml.math.distributed.keys.impl.SparseMatrixKey;
-import org.apache.ignite.ml.math.exceptions.ConvergenceException;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.math.functions.Functions;
-import org.apache.ignite.ml.math.functions.IgniteBiFunction;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.storage.matrix.SparseDistributedMatrixStorage;
-import org.apache.ignite.ml.math.util.MapUtil;
-import org.apache.ignite.ml.math.util.MatrixUtil;
-
-import static org.apache.ignite.ml.math.distributed.CacheUtils.distributedFold;
-import static org.apache.ignite.ml.math.util.MatrixUtil.localCopyOf;
-
-/**
- * Clustering algorithm based on Bahmani et al. paper and Apache Spark class with corresponding functionality.
- *
- * TODO: IGNITE-6059, add block matrix support.
- *
- * @see <a href="http://theory.stanford.edu/~sergei/papers/vldb12-kmpar.pdf">Scalable K-Means++(wikipedia)</a>
- */
-public class KMeansDistributedClusterer extends BaseKMeansClusterer<SparseDistributedMatrix> {
-    /** */
-    private final int maxIterations;
-
-    /** */
-    private Random rnd;
-
-    /** */
-    private int initSteps;
-
-    /** */
-    private long seed;
-
-    /** */
-    private double epsilon = 1e-4;
-
-    /** */
-    public KMeansDistributedClusterer(DistanceMeasure measure, int initSteps, int maxIterations, Long seed) {
-        super(measure);
-        this.initSteps = initSteps;
-
-        this.seed = seed != null ? seed : new Random().nextLong();
-
-        this.maxIterations = maxIterations;
-        rnd = new Random(this.seed);
-    }
-
-    /** */
-    @Override public KMeansModel cluster(SparseDistributedMatrix points, int k) throws
-        MathIllegalArgumentException, ConvergenceException {
-        SparseDistributedMatrix pointsCp = (SparseDistributedMatrix)points.like(points.rowSize(), points.columnSize());
-
-        String cacheName = ((SparseDistributedMatrixStorage)points.getStorage()).cacheName();
-
-        // TODO: IGNITE-5825, this copy is very ineffective, just for POC. Immutability of data should be guaranteed by other methods
-        // such as logical locks for example.
-        pointsCp.assign(points);
-
-        Vector[] centers = initClusterCenters(pointsCp, k);
-
-        boolean converged = false;
-        int iteration = 0;
-        int dim = pointsCp.viewRow(0).size();
-        UUID uid = pointsCp.getUUID();
-
-        // Execute iterations of Lloyd's algorithm until converged
-        while (iteration < maxIterations && !converged) {
-            SumsAndCounts stats = getSumsAndCounts(centers, dim, uid, cacheName);
-
-            converged = true;
-
-            for (Integer ind : stats.sums.keySet()) {
-                Vector massCenter = stats.sums.get(ind).times(1.0 / stats.counts.get(ind));
-
-                if (converged && distance(massCenter, centers[ind]) > epsilon * epsilon)
-                    converged = false;
-
-                centers[ind] = massCenter;
-            }
-
-            iteration++;
-        }
-
-        pointsCp.destroy();
-
-        return new KMeansModel(centers, getDistanceMeasure());
-    }
-
-    /** Initialize cluster centers. */
-    private Vector[] initClusterCenters(SparseDistributedMatrix points, int k) {
-        // Initialize empty centers and point costs.
-        int ptsCnt = points.rowSize();
-
-        String cacheName = ((SparseDistributedMatrixStorage)points.getStorage()).cacheName();
-
-        // Initialize the first center to a random point.
-        Vector sample = localCopyOf(points.viewRow(rnd.nextInt(ptsCnt)));
-
-        List<Vector> centers = new ArrayList<>();
-        List<Vector> newCenters = new ArrayList<>();
-        newCenters.add(sample);
-        centers.add(sample);
-
-        final ConcurrentHashMap<Integer, Double> costs = new ConcurrentHashMap<>();
-
-        // On each step, sample 2 * k points on average with probability proportional
-        // to their squared distance from the centers. Note that only distances between points
-        // and new centers are computed in each iteration.
-        int step = 0;
-        UUID uid = points.getUUID();
-
-        while (step < initSteps) {
-            // We assume here that costs can fit into memory of one node.
-            ConcurrentHashMap<Integer, Double> newCosts = getNewCosts(points, newCenters, cacheName);
-
-            // Merge costs with new costs.
-            for (Integer ind : newCosts.keySet())
-                costs.merge(ind, newCosts.get(ind), Math::min);
-
-            double sumCosts = costs.values().stream().mapToDouble(Double::valueOf).sum();
-
-            newCenters = getNewCenters(k, costs, uid, sumCosts, cacheName);
-            centers.addAll(newCenters);
-
-            step++;
-        }
-
-        List<Vector> distinctCenters = centers.stream().distinct().collect(Collectors.toList());
-
-        if (distinctCenters.size() <= k)
-            return distinctCenters.toArray(new Vector[] {});
-        else {
-            // Finally, we might have a set of more than k distinct candidate centers; weight each
-            // candidate by the number of points in the dataset mapping to it and run a local k-means++
-            // on the weighted centers to pick k of them
-            ConcurrentHashMap<Integer, Integer> centerInd2Weight = weightCenters(uid, distinctCenters, cacheName);
-
-            List<Double> weights = new ArrayList<>(centerInd2Weight.size());
-
-            for (int i = 0; i < distinctCenters.size(); i++)
-                weights.add(i, Double.valueOf(centerInd2Weight.getOrDefault(i, 0)));
-
-            DenseLocalOnHeapMatrix dCenters = MatrixUtil.fromList(distinctCenters, true);
-
-            return new KMeansLocalClusterer(getDistanceMeasure(), 30, seed).cluster(dCenters, k, weights).centers();
-        }
-    }
-
-    /** */
-    private List<Vector> getNewCenters(int k, ConcurrentHashMap<Integer, Double> costs, UUID uid,
-        double sumCosts, String cacheName) {
-        return distributedFold(cacheName,
-            (IgniteBiFunction<Cache.Entry<SparseMatrixKey, Map<Integer, Double>>,
-                List<Vector>,
-                List<Vector>>)(vectorWithIndex, list) -> {
-                Integer ind = vectorWithIndex.getKey().index();
-
-                double prob = costs.get(ind) * 2.0 * k / sumCosts;
-
-                if (new Random(seed ^ ind).nextDouble() < prob)
-                    list.add(VectorUtils.fromMap(vectorWithIndex.getValue(), false));
-
-                return list;
-            },
-            key -> key.dataStructureId().equals(uid),
-            (list1, list2) -> {
-                list1.addAll(list2);
-                return list1;
-            }, ArrayList::new
-        );
-    }
-
-    /** */
-    private ConcurrentHashMap<Integer, Double> getNewCosts(SparseDistributedMatrix points, List<Vector> newCenters,
-        String cacheName) {
-        return distributedFold(cacheName,
-            (IgniteBiFunction<Cache.Entry<SparseMatrixKey, ConcurrentHashMap<Integer, Double>>,
-                ConcurrentHashMap<Integer, Double>,
-                ConcurrentHashMap<Integer, Double>>)(vectorWithIndex, map) -> {
-                for (Vector center : newCenters)
-                    map.merge(vectorWithIndex.getKey().index(), distance(vectorWithIndex.getValue(), center), Functions.MIN);
-
-                return map;
-            },
-            key -> key.dataStructureId().equals(points.getUUID()),
-            (map1, map2) -> {
-                map1.putAll(map2);
-                return map1;
-            }, ConcurrentHashMap::new);
-    }
-
-    /** */
-    private ConcurrentHashMap<Integer, Integer> weightCenters(UUID uid, List<Vector> distinctCenters,
-        String cacheName) {
-        return distributedFold(cacheName,
-            (IgniteBiFunction<Cache.Entry<SparseMatrixKey, Map<Integer, Double>>,
-                ConcurrentHashMap<Integer, Integer>,
-                ConcurrentHashMap<Integer, Integer>>)(vectorWithIndex, countMap) -> {
-                Integer resInd = -1;
-                Double resDist = Double.POSITIVE_INFINITY;
-
-                int i = 0;
-                for (Vector cent : distinctCenters) {
-                    double curDist = distance(vectorWithIndex.getValue(), cent);
-
-                    if (resDist > curDist) {
-                        resDist = curDist;
-                        resInd = i;
-                    }
-
-                    i++;
-                }
-
-                countMap.compute(resInd, (ind, v) -> v != null ? v + 1 : 1);
-                return countMap;
-            },
-            key -> key.dataStructureId().equals(uid),
-            (map1, map2) -> MapUtil.mergeMaps(map1, map2, (integer, integer2) -> integer2 + integer,
-                ConcurrentHashMap::new),
-            ConcurrentHashMap::new);
-    }
-
-    /** */
-    private double distance(Map<Integer, Double> vecMap, Vector vector) {
-        return distance(VectorUtils.fromMap(vecMap, false), vector);
-    }
-
-    /** */
-    private SumsAndCounts getSumsAndCounts(Vector[] centers, int dim, UUID uid, String cacheName) {
-        return CacheUtils.distributedFold(cacheName,
-            (IgniteBiFunction<Cache.Entry<SparseMatrixKey, Map<Integer, Double>>, SumsAndCounts, SumsAndCounts>)(entry, counts) -> {
-                Map<Integer, Double> vec = entry.getValue();
-
-                IgniteBiTuple<Integer, Double> closest = findClosest(centers, VectorUtils.fromMap(vec, false));
-                int bestCenterIdx = closest.get1();
-
-                counts.totalCost += closest.get2();
-                counts.sums.putIfAbsent(bestCenterIdx, VectorUtils.zeroes(dim));
-
-                counts.sums.compute(bestCenterIdx,
-                    (IgniteBiFunction<Integer, Vector, Vector>)(ind, v) -> v.plus(VectorUtils.fromMap(vec, false)));
-
-                counts.counts.merge(bestCenterIdx, 1,
-                    (IgniteBiFunction<Integer, Integer, Integer>)(i1, i2) -> i1 + i2);
-
-                return counts;
-            },
-            key -> key.dataStructureId().equals(uid),
-            SumsAndCounts::merge, SumsAndCounts::new
-        );
-    }
-
-    /** Service class used for statistics. */
-    private static class SumsAndCounts {
-        /** */
-        public double totalCost;
-
-        /** */
-        public ConcurrentHashMap<Integer, Vector> sums = new ConcurrentHashMap<>();
-
-        /** Count of points closest to the center with a given index. */
-        public ConcurrentHashMap<Integer, Integer> counts = new ConcurrentHashMap<>();
-
-        /** Merge current */
-        public SumsAndCounts merge(SumsAndCounts other) {
-            this.totalCost += totalCost;
-            MapUtil.mergeMaps(sums, other.sums, Vector::plus, ConcurrentHashMap::new);
-            MapUtil.mergeMaps(counts, other.counts, (i1, i2) -> i1 + i2, ConcurrentHashMap::new);
-            return this;
-        }
-    }
-
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java
deleted file mode 100644
index 8a50e65436ccd..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansLocalClusterer.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import org.apache.ignite.internal.util.GridArgumentCheck;
-import org.apache.ignite.ml.math.Matrix;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorUtils;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.exceptions.ConvergenceException;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-
-import static org.apache.ignite.ml.math.util.MatrixUtil.localCopyOf;
-
-/**
- * Perform clusterization on local data.
- * This class is based on Apache Spark class with corresponding functionality.
- */
-public class KMeansLocalClusterer extends BaseKMeansClusterer<DenseLocalOnHeapMatrix> implements
-    WeightedClusterer<DenseLocalOnHeapMatrix, KMeansModel> {
-    /** */
-    private int maxIterations;
-
-    /** */
-    private Random rand;
-
-    /**
-     * Build a new clusterer with the given {@link DistanceMeasure}.
-     *
-     * @param measure Distance measure to use.
-     * @param maxIterations maximal number of iterations.
-     * @param seed Seed used in random parts of algorithm.
-     */
-    public KMeansLocalClusterer(DistanceMeasure measure, int maxIterations, Long seed) {
-        super(measure);
-        this.maxIterations = maxIterations;
-        rand = seed != null ? new Random(seed) : new Random();
-    }
-
-    /** {@inheritDoc} */
-    @Override public KMeansModel cluster(
-        DenseLocalOnHeapMatrix points, int k) throws MathIllegalArgumentException, ConvergenceException {
-        List<Double> ones = new ArrayList<>(Collections.nCopies(points.rowSize(), 1.0));
-        return cluster(points, k, ones);
-    }
-
-    /** {@inheritDoc} */
-    @Override public KMeansModel cluster(DenseLocalOnHeapMatrix points, int k,
-        List<Double> weights) throws MathIllegalArgumentException, ConvergenceException {
-
-        GridArgumentCheck.notNull(points, "points");
-
-        int dim = points.columnSize();
-        Vector[] centers = new Vector[k];
-
-        centers[0] = pickWeighted(points, weights);
-
-        Vector costs = points.foldRows(row -> distance(row,
-            centers[0]));
-
-        for (int i = 0; i < k; i++) {
-            double weightedSum = weightedSum(costs, weights);
-
-            double r = rand.nextDouble() * weightedSum;
-            double s = 0.0;
-            int j = 0;
-
-            while (j < points.rowSize() && s < r) {
-                s += weights.get(j) * costs.get(j);
-                j++;
-            }
-
-            if (j == 0)
-                // TODO: IGNITE-5825, Process this case more carefully
-                centers[i] = localCopyOf(points.viewRow(0));
-            else
-                centers[i] = localCopyOf(points.viewRow(j - 1));
-
-            for (int p = 0; p < points.rowSize(); p++)
-                costs.setX(p, Math.min(getDistanceMeasure().compute(localCopyOf(points.viewRow(p)), centers[i]),
-                    costs.get(p)));
-        }
-
-        int[] oldClosest = new int[points.rowSize()];
-        Arrays.fill(oldClosest, -1);
-        int iter = 0;
-        boolean moved = true;
-
-        while (moved && iter < maxIterations) {
-            moved = false;
-
-            double[] counts = new double[k];
-            Arrays.fill(counts, 0.0);
-            Vector[] sums = new Vector[k];
-
-            Arrays.fill(sums, VectorUtils.zeroes(dim));
-
-            int i = 0;
-
-            while (i < points.rowSize()) {
-                Vector p = localCopyOf(points.viewRow(i));
-
-                int ind = findClosest(centers, p).get1();
-                sums[ind] = sums[ind].plus(p.times(weights.get(i)));
-
-                counts[ind] += weights.get(i);
-                if (ind != oldClosest[i]) {
-                    moved = true;
-                    oldClosest[i] = ind;
-                }
-                i++;
-            }
-            // Update centers
-            int j = 0;
-            while (j < k) {
-                if (counts[j] == 0.0) {
-                    // Assign center to a random point
-                    centers[j] = points.viewRow(rand.nextInt(points.rowSize()));
-                }
-                else {
-                    sums[j] = sums[j].times(1.0 / counts[j]);
-                    centers[j] = sums[j];
-                }
-                j++;
-            }
-            iter++;
-        }
-
-        return new KMeansModel(centers, getDistanceMeasure());
-    }
-
-    /** Pick a random vector with a probability proportional to the corresponding weight. */
-    private Vector pickWeighted(Matrix points, List<Double> weights) {
-        double r = rand.nextDouble() * weights.stream().mapToDouble(Double::valueOf).sum();
-
-        int i = 0;
-        double curWeight = 0.0;
-
-        while (i < points.rowSize() && curWeight < r) {
-            curWeight += weights.get(i);
-            i += 1;
-        }
-
-        return localCopyOf(points.viewRow(i - 1));
-    }
-
-    /** Get a weighted sum of a vector v. */
-    private double weightedSum(Vector v, List<Double> weights) {
-        double res = 0.0;
-
-        for (int i = 0; i < v.size(); i++)
-            res += v.getX(i) * weights.get(i);
-
-        return res;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java
deleted file mode 100644
index 16880874eee95..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/WeightedClusterer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.util.List;
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.math.exceptions.ConvergenceException;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-
-/**
- * Support of clusterization with given weights.
- */
-public interface WeightedClusterer<P, M extends Model> extends Clusterer<P, M> {
-    /**
-     * Perform clusterization of given points weighted by given weights.
-     *
-     * @param points Points.
-     * @param k count of centers.
-     * @param weights Weights.
-     */
-    public M cluster(P points, int k, List<Double> weights) throws
-        MathIllegalArgumentException, ConvergenceException;
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/Clusterer.java
similarity index 95%
rename from modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/Clusterer.java
index 204e28a7ab8f5..9930f23bce5e8 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/Clusterer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/Clusterer.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml.clustering;
+package org.apache.ignite.ml.clustering.kmeans;
 
 import org.apache.ignite.ml.Model;
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/ClusterizationModel.java
similarity index 92%
rename from modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/ClusterizationModel.java
index 99afec59cd9dc..474a463eb7550 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/ClusterizationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/ClusterizationModel.java
@@ -15,14 +15,14 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml.clustering;
+package org.apache.ignite.ml.clustering.kmeans;
 
 import org.apache.ignite.ml.Model;
 
 /** Base interface for all clusterization models. */
 public interface ClusterizationModel<P, V> extends Model<P, V> {
     /** Gets the clusters count. */
-    public int clustersCount();
+    public int amountOfClusters();
 
     /** Get cluster centers. */
     public P[] centers();
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java
similarity index 77%
rename from modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java
index e1d783f7d2daf..c900efd6cbe1f 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/KMeansModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModel.java
@@ -15,43 +15,42 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml.clustering;
+package org.apache.ignite.ml.clustering.kmeans;
 
 import java.util.Arrays;
 import org.apache.ignite.ml.Exportable;
 import org.apache.ignite.ml.Exporter;
-import org.apache.ignite.ml.KMeansModelFormat;
 import org.apache.ignite.ml.math.Vector;
 import org.apache.ignite.ml.math.distances.DistanceMeasure;
 
 /**
- * This class encapsulates result of clusterization.
+ * This class encapsulates result of clusterization by KMeans algorithm.
  */
 public class KMeansModel implements ClusterizationModel<Vector, Integer>, Exportable<KMeansModelFormat> {
     /** Centers of clusters. */
     private final Vector[] centers;
 
     /** Distance measure. */
-    private final DistanceMeasure distance;
+    private final DistanceMeasure distanceMeasure;
 
     /**
-     * Construct KMeans model with given centers and distance measure.
+     * Construct KMeans model with given centers and distanceMeasure measure.
      *
      * @param centers Centers.
-     * @param distance Distance measure.
+     * @param distanceMeasure Distance measure.
      */
-    public KMeansModel(Vector[] centers, DistanceMeasure distance) {
+    public KMeansModel(Vector[] centers, DistanceMeasure distanceMeasure) {
         this.centers = centers;
-        this.distance = distance;
+        this.distanceMeasure = distanceMeasure;
     }
 
-    /** Distance measure used while clusterization */
+    /** Distance measure. */
     public DistanceMeasure distanceMeasure() {
-        return distance;
+        return distanceMeasure;
     }
 
-    /** Count of centers in clusterization. */
-    @Override public int clustersCount() {
+    /** Amount of centers in clusterization. */
+    @Override public int amountOfClusters() {
         return centers.length;
     }
 
@@ -70,7 +69,7 @@ public Integer apply(Vector vec) {
         double minDist = Double.POSITIVE_INFINITY;
 
         for (int i = 0; i < centers.length; i++) {
-            double curDist = distance.compute(centers[i], vec);
+            double curDist = distanceMeasure.compute(centers[i], vec);
             if (curDist < minDist) {
                 minDist = curDist;
                 res = i;
@@ -82,7 +81,7 @@ public Integer apply(Vector vec) {
 
     /** {@inheritDoc} */
     @Override public <P> void saveModel(Exporter<KMeansModelFormat, P> exporter, P path) {
-        KMeansModelFormat mdlData = new KMeansModelFormat(centers, distance);
+        KMeansModelFormat mdlData = new KMeansModelFormat(centers, distanceMeasure);
 
         exporter.save(mdlData, path);
     }
@@ -91,7 +90,7 @@ public Integer apply(Vector vec) {
     @Override public int hashCode() {
         int res = 1;
 
-        res = res * 37 + distance.hashCode();
+        res = res * 37 + distanceMeasure.hashCode();
         res = res * 37 + Arrays.hashCode(centers);
 
         return res;
@@ -107,7 +106,7 @@ public Integer apply(Vector vec) {
 
         KMeansModel that = (KMeansModel)obj;
 
-        return distance.equals(that.distance) && Arrays.deepEquals(centers, that.centers);
+        return distanceMeasure.equals(that.distanceMeasure) && Arrays.deepEquals(centers, that.centers);
     }
 
 }
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/KMeansModelFormat.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModelFormat.java
similarity index 94%
rename from modules/ml/src/main/java/org/apache/ignite/ml/KMeansModelFormat.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModelFormat.java
index c013198af9808..266370115f1c6 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/KMeansModelFormat.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansModelFormat.java
@@ -15,10 +15,12 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml;
+package org.apache.ignite.ml.clustering.kmeans;
 
 import java.io.Serializable;
 import java.util.Arrays;
+import org.apache.ignite.ml.Exportable;
+import org.apache.ignite.ml.Exporter;
 import org.apache.ignite.ml.math.Vector;
 import org.apache.ignite.ml.math.distances.DistanceMeasure;
 
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
new file mode 100644
index 0000000000000..f65a3fed4cf2d
--- /dev/null
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/KMeansTrainer.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering.kmeans;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.ml.dataset.Dataset;
+import org.apache.ignite.ml.dataset.DatasetBuilder;
+import org.apache.ignite.ml.dataset.PartitionDataBuilder;
+import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.VectorUtils;
+import org.apache.ignite.ml.math.distances.DistanceMeasure;
+import org.apache.ignite.ml.math.distances.EuclideanDistance;
+import org.apache.ignite.ml.math.functions.IgniteBiFunction;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.math.util.MapUtil;
+import org.apache.ignite.ml.structures.LabeledDataset;
+import org.apache.ignite.ml.structures.LabeledVector;
+import org.apache.ignite.ml.structures.partition.LabeledDatasetPartitionDataBuilderOnHeap;
+import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
+
+/**
+ * The trainer for KMeans algorithm.
+ */
+public class KMeansTrainer implements SingleLabelDatasetTrainer<KMeansModel> {
+    /** Amount of clusters. */
+    private int k = 2;
+
+    /** Amount of iterations. */
+    private int maxIterations = 10;
+
+    /** Delta of convergence. */
+    private double epsilon = 1e-4;
+
+    /** Distance measure. */
+    private DistanceMeasure distance = new EuclideanDistance();
+
+    /** KMeans initializer. */
+    private long seed;
+
+    /**
+     * Trains model based on the specified data.
+     *
+     * @param datasetBuilder Dataset builder.
+     * @param featureExtractor Feature extractor.
+     * @param lbExtractor Label extractor.
+     * @return Model.
+     */
+    @Override public <K, V> KMeansModel fit(DatasetBuilder<K, V> datasetBuilder,
+        IgniteBiFunction<K, V, double[]> featureExtractor, IgniteBiFunction<K, V, Double> lbExtractor) {
+        assert datasetBuilder != null;
+
+        PartitionDataBuilder<K, V, EmptyContext, LabeledDataset<Double, LabeledVector>> partDataBuilder = new LabeledDatasetPartitionDataBuilderOnHeap<>(
+            featureExtractor,
+            lbExtractor
+        );
+
+        Vector[] centers;
+
+        try (Dataset<EmptyContext, LabeledDataset<Double, LabeledVector>> dataset = datasetBuilder.build(
+            (upstream, upstreamSize) -> new EmptyContext(),
+            partDataBuilder
+        )) {
+            final int cols = dataset.compute(org.apache.ignite.ml.structures.Dataset::colSize, (a, b) -> a == null ? b : a);
+            centers = initClusterCentersRandomly(dataset, k);
+
+            boolean converged = false;
+            int iteration = 0;
+
+            while (iteration < maxIterations && !converged) {
+                Vector[] newCentroids = new DenseLocalOnHeapVector[k];
+
+                TotalCostAndCounts totalRes = calcDataForNewCentroids(centers, dataset, cols);
+
+                converged = true;
+
+                for (Integer ind : totalRes.sums.keySet()) {
+                    Vector massCenter = totalRes.sums.get(ind).times(1.0 / totalRes.counts.get(ind));
+
+                    if (converged && distance.compute(massCenter, centers[ind]) > epsilon * epsilon)
+                        converged = false;
+
+                    newCentroids[ind] = massCenter;
+                }
+
+                iteration++;
+                centers = newCentroids;
+            }
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return new KMeansModel(centers, distance);
+    }
+
+    /**
+     * Prepares the data to define new centroids on current iteration.
+     *
+     * @param centers Current centers on the current iteration.
+     * @param dataset Dataset.
+     * @param cols Amount of columns.
+     * @return Helper data to calculate the new centroids.
+     */
+    private TotalCostAndCounts calcDataForNewCentroids(Vector[] centers,
+        Dataset<EmptyContext, LabeledDataset<Double, LabeledVector>> dataset, int cols) {
+        final Vector[] finalCenters = centers;
+
+        return dataset.compute(data -> {
+
+            TotalCostAndCounts res = new TotalCostAndCounts();
+
+            for (int i = 0; i < data.rowSize(); i++) {
+                final IgniteBiTuple<Integer, Double> closestCentroid = findClosestCentroid(finalCenters, data.getRow(i));
+
+                int centroidIdx = closestCentroid.get1();
+
+                data.setLabel(i, centroidIdx);
+
+                res.totalCost += closestCentroid.get2();
+                res.sums.putIfAbsent(centroidIdx, VectorUtils.zeroes(cols));
+
+                int finalI = i;
+                res.sums.compute(centroidIdx,
+                    (IgniteBiFunction<Integer, Vector, Vector>)(ind, v) -> v.plus(data.getRow(finalI).features()));
+
+                res.counts.merge(centroidIdx, 1,
+                    (IgniteBiFunction<Integer, Integer, Integer>)(i1, i2) -> i1 + i2);
+            }
+            return res;
+        }, (a, b) -> a == null ? b : a.merge(b));
+    }
+
+    /**
+     * Find the closest cluster center index and distance to it from a given point.
+     *
+     * @param centers Centers to look in.
+     * @param pnt Point.
+     */
+    private IgniteBiTuple<Integer, Double> findClosestCentroid(Vector[] centers, LabeledVector pnt) {
+        double bestDistance = Double.POSITIVE_INFINITY;
+        int bestInd = 0;
+
+        for (int i = 0; i < centers.length; i++) {
+            double dist = distance.compute(centers[i], pnt.features());
+            if (dist < bestDistance) {
+                bestDistance = dist;
+                bestInd = i;
+            }
+        }
+        return new IgniteBiTuple<>(bestInd, bestDistance);
+    }
+
+    /**
+     * K cluster centers are initialized randomly.
+     *
+     * @param dataset The dataset to pick up random centers.
+     * @param k Amount of clusters.
+     * @return K cluster centers.
+     */
+    private Vector[] initClusterCentersRandomly(Dataset<EmptyContext, LabeledDataset<Double, LabeledVector>> dataset,
+        int k) {
+
+        Vector[] initCenters = new DenseLocalOnHeapVector[k];
+
+        List<LabeledVector> rndPnts = dataset.compute(data -> {
+            List<LabeledVector> rndPnt = new ArrayList<>();
+            rndPnt.add(data.getRow(new Random(seed).nextInt(data.rowSize())));
+            return rndPnt;
+        }, (a, b) -> a == null ? b : Stream.concat(a.stream(), b.stream()).collect(Collectors.toList()));
+
+        for (int i = 0; i < k; i++) {
+            final LabeledVector rndPnt = rndPnts.get(new Random(seed).nextInt(rndPnts.size()));
+            rndPnts.remove(rndPnt);
+            initCenters[i] = rndPnt.features();
+        }
+
+        return initCenters;
+    }
+
+    /** Service class used for statistics. */
+    private static class TotalCostAndCounts {
+        /** */
+        double totalCost;
+
+        /** */
+        ConcurrentHashMap<Integer, Vector> sums = new ConcurrentHashMap<>();
+
+        /** Count of points closest to the center with a given index. */
+        ConcurrentHashMap<Integer, Integer> counts = new ConcurrentHashMap<>();
+
+        /** Merge current */
+        TotalCostAndCounts merge(TotalCostAndCounts other) {
+            this.totalCost += totalCost;
+            this.sums = MapUtil.mergeMaps(sums, other.sums, Vector::plus, ConcurrentHashMap::new);
+            this.counts = MapUtil.mergeMaps(counts, other.counts, (i1, i2) -> i1 + i2, ConcurrentHashMap::new);
+            return this;
+        }
+    }
+
+    /**
+     * Gets the amount of clusters.
+     *
+     * @return The parameter value.
+     */
+    public int getK() {
+        return k;
+    }
+
+    /**
+     * Set up the amount of clusters.
+     *
+     * @param k The parameter value.
+     * @return Model with new amount of clusters parameter value.
+     */
+    public KMeansTrainer withK(int k) {
+        this.k = k;
+        return this;
+    }
+
+    /**
+     * Gets the max number of iterations before convergence.
+     *
+     * @return The parameter value.
+     */
+    public int getMaxIterations() {
+        return maxIterations;
+    }
+
+    /**
+     * Set up the max number of iterations before convergence.
+     *
+     * @param maxIterations The parameter value.
+     * @return Model with new max number of iterations before convergence parameter value.
+     */
+    public KMeansTrainer withMaxIterations(int maxIterations) {
+        this.maxIterations = maxIterations;
+        return this;
+    }
+
+    /**
+     * Gets the epsilon.
+     *
+     * @return The parameter value.
+     */
+    public double getEpsilon() {
+        return epsilon;
+    }
+
+    /**
+     * Set up the epsilon.
+     *
+     * @param epsilon The parameter value.
+     * @return Model with new epsilon parameter value.
+     */
+    public KMeansTrainer withEpsilon(double epsilon) {
+        this.epsilon = epsilon;
+        return this;
+    }
+
+    /**
+     * Gets the distance.
+     *
+     * @return The parameter value.
+     */
+    public DistanceMeasure getDistance() {
+        return distance;
+    }
+
+    /**
+     * Set up the distance.
+     *
+     * @param distance The parameter value.
+     * @return Model with new distance parameter value.
+     */
+    public KMeansTrainer withDistance(DistanceMeasure distance) {
+        this.distance = distance;
+        return this;
+    }
+
+    /**
+     * Gets the seed number.
+     *
+     * @return The parameter value.
+     */
+    public long getSeed() {
+        return seed;
+    }
+
+    /**
+     * Set up the seed.
+     *
+     * @param seed The parameter value.
+     * @return Model with new seed parameter value.
+     */
+    public KMeansTrainer withSeed(long seed) {
+        this.seed = seed;
+        return this;
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/package-info.java
similarity index 62%
rename from modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java
rename to modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/package-info.java
index 420678fe1a4c4..4d27b6e878b95 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansUtil.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/clustering/kmeans/package-info.java
@@ -15,19 +15,8 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.ml.clustering;
-
-import org.apache.ignite.ml.math.Vector;
-
-import static org.junit.Assert.assertTrue;
-
-/** Utilities for k-means tests. */
-class KMeansUtil {
-    /** */
-    static void checkIsInEpsilonNeighbourhood(Vector[] v1s, Vector[] v2s, double epsilon) {
-        for (int i = 0; i < v1s.length; i++) {
-            assertTrue("Not in epsilon neighbourhood (index " + i + ") ",
-                v1s[i].minus(v2s[i]).kNorm(2) < epsilon);
-        }
-    }
-}
+/**
+ * <!-- Package description. -->
+ * Contains kMeans clustering algorithm.
+ */
+package org.apache.ignite.ml.clustering.kmeans;
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/structures/preprocessing/LabellingMachine.java b/modules/ml/src/main/java/org/apache/ignite/ml/structures/preprocessing/LabellingMachine.java
deleted file mode 100644
index e8d88ada91d1d..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/structures/preprocessing/LabellingMachine.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.structures.preprocessing;
-
-import org.apache.ignite.ml.Model;
-import org.apache.ignite.ml.structures.LabeledDataset;
-
-/** Data pre-processing step which assigns labels to all observations according model. */
-public class LabellingMachine {
-    /**
-     * Set labels to each observation according passed model.
-     * <p>
-     * NOTE: In-place operation.
-     * </p>
-     * @param ds The given labeled dataset.
-     * @param mdl The given model.
-     * @return Dataset with predicted labels.
-     */
-    public static LabeledDataset assignLabels(LabeledDataset ds, Model mdl) {
-        for (int i = 0; i < ds.rowSize(); i++) {
-            double predictedCls = (double) mdl.apply(ds.getRow(i).features());
-            ds.setLabel(i, predictedCls);
-        }
-        return ds;
-    }
-}
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/structures/preprocessing/Normalizer.java b/modules/ml/src/main/java/org/apache/ignite/ml/structures/preprocessing/Normalizer.java
deleted file mode 100644
index 161ec355d673c..0000000000000
--- a/modules/ml/src/main/java/org/apache/ignite/ml/structures/preprocessing/Normalizer.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.structures.preprocessing;
-
-import org.apache.ignite.ml.math.exceptions.UnsupportedOperationException;
-import org.apache.ignite.ml.structures.Dataset;
-import org.apache.ignite.ml.structures.DatasetRow;
-
-/** Data pre-processing step which scales features according normalization algorithms. */
-public class Normalizer {
-    /**
-     * Scales features in dataset with MiniMax algorithm x'=(x-MIN[X])/(MAX[X]-MIN[X]). This is an in-place operation.
-     * <p>
-     * NOTE: Complexity 2*N^2.
-     * </p>
-     * @param ds The given dataset.
-     * @return Transformed dataset.
-     */
-    public static Dataset normalizeWithMiniMax(Dataset ds) {
-        int colSize = ds.colSize();
-        double[] mins = new double[colSize];
-        double[] maxs = new double[colSize];
-
-        int rowSize = ds.rowSize();
-        DatasetRow[] data = ds.data();
-        for (int j = 0; j < colSize; j++) {
-            double maxInCurrCol = Double.MIN_VALUE;
-            double minInCurrCol = Double.MAX_VALUE;
-
-            for (int i = 0; i < rowSize; i++) {
-                double e = data[i].features().get(j);
-                maxInCurrCol = Math.max(e, maxInCurrCol);
-                minInCurrCol = Math.min(e, minInCurrCol);
-            }
-
-            mins[j] = minInCurrCol;
-            maxs[j] = maxInCurrCol;
-        }
-
-        for (int j = 0; j < colSize; j++) {
-            double div = maxs[j] - mins[j];
-            if(div == 0)
-                continue;
-
-            for (int i = 0; i < rowSize; i++) {
-                double oldVal = data[i].features().get(j);
-                double newVal = (oldVal - mins[j]) / div;
-                // x'=(x-MIN[X])/(MAX[X]-MIN[X])
-                data[i].features().set(j, newVal);
-            }
-        }
-
-        return ds;
-    }
-
-    /**
-     * Scales features in dataset with Z-Normalization algorithm x'=(x-M[X])/\sigma [X]. This is an in-place operation.
-     *
-     * @param ds The given dataset.
-     * @return Transformed dataset.
-     */
-    public static Dataset normalizeWithZNormalization(Dataset ds) {
-        throw new UnsupportedOperationException("Z-normalization is not supported yet");
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
index 3f12bdcbd7560..353cc227377b0 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/LocalModelsTest.java
@@ -20,14 +20,18 @@
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.function.Function;
-import org.apache.ignite.ml.clustering.KMeansLocalClusterer;
-import org.apache.ignite.ml.clustering.KMeansModel;
+import org.apache.ignite.ml.clustering.kmeans.KMeansModel;
+import org.apache.ignite.ml.clustering.kmeans.KMeansModelFormat;
+import org.apache.ignite.ml.clustering.kmeans.KMeansTrainer;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
 import org.apache.ignite.ml.knn.classification.KNNClassificationModel;
 import org.apache.ignite.ml.knn.classification.KNNModelFormat;
 import org.apache.ignite.ml.knn.classification.KNNStrategy;
 import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
 import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
 import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
 import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
@@ -140,14 +144,20 @@ private void executeModelTest(Function<String, Void> code) throws IOException {
 
     /** */
     private KMeansModel getClusterModel() {
-        KMeansLocalClusterer clusterer = new KMeansLocalClusterer(new EuclideanDistance(), 1, 1L);
+        Map<Integer, double[]> data = new HashMap<>();
+        data.put(0, new double[] {1.0, 1959, 325100});
+        data.put(1, new double[] {1.0, 1960, 373200});
 
-        double[] v1 = new double[] {1959, 325100};
-        double[] v2 = new double[] {1960, 373200};
+        KMeansTrainer trainer = new KMeansTrainer()
+            .withK(1);
 
-        DenseLocalOnHeapMatrix points = new DenseLocalOnHeapMatrix(new double[][] {v1, v2});
+        KMeansModel knnMdl = trainer.fit(
+            new LocalDatasetBuilder<>(data, 2),
+            (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
+            (k, v) -> v[2]
+        );
 
-        return clusterer.cluster(points, 1);
+        return knnMdl;
     }
 
     /** */
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTestSuite.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTestSuite.java
index 85f61fa2c0cfc..80538a0cb89ea 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTestSuite.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/ClusteringTestSuite.java
@@ -25,11 +25,8 @@
  */
 @RunWith(Suite.class)
 @Suite.SuiteClasses({
-    KMeansDistributedClustererTestSingleNode.class,
-    KMeansDistributedClustererTestMultiNode.class,
-    KMeansLocalClustererTest.class,
-    FuzzyCMeansDistributedClustererTest.class,
-    FuzzyCMeansLocalClustererTest.class
+    KMeansTrainerTest.class,
+    KMeansModelTest.class
 })
 public class ClusteringTestSuite {
 }
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/FuzzyCMeansDistributedClustererTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/FuzzyCMeansDistributedClustererTest.java
deleted file mode 100644
index 4b415bb99b723..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/FuzzyCMeansDistributedClustererTest.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.Random;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/** Tests that checks distributed Fuzzy C-Means clusterer. */
-public class FuzzyCMeansDistributedClustererTest extends GridCommonAbstractTest {
-    /** Number of nodes in grid. */
-    private static final int NODE_COUNT = 3;
-
-    /** Grid instance. */
-    private Ignite ignite;
-
-    /** Default constructor. */
-    public FuzzyCMeansDistributedClustererTest() {
-        super(false);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTest() throws Exception {
-        ignite = grid(NODE_COUNT);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        for (int i = 1; i <= NODE_COUNT; i++)
-            startGrid(i);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        stopAllGrids();
-    }
-
-    /** Test that algorithm gives correct results on a small sample - 4 centers on the plane. */
-    public void testTwoDimensionsLittleData() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        FuzzyCMeansDistributedClusterer clusterer = new FuzzyCMeansDistributedClusterer(new EuclideanDistance(),
-                2, BaseFuzzyCMeansClusterer.StopCondition.STABLE_MEMBERSHIPS,
-                0.01, 500, null, 2, 50);
-
-        double[][] points = new double[][]{{-10, -10}, {-9, -11}, {-10, -9}, {-11, -9},
-                {10, 10},   {9, 11},   {10, 9},   {11, 9},
-                {-10, 10},  {-9, 11},  {-10, 9},  {-11, 9},
-                {10, -10},  {9, -11},  {10, -9},  {11, -9}};
-
-        SparseDistributedMatrix pntMatrix = new SparseDistributedMatrix(16, 2,
-                StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-        for (int i = 0; i < 16; i++)
-            pntMatrix.setRow(i, points[i]);
-
-        FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, 4);
-
-        Vector[] centers = mdl.centers();
-        Arrays.sort(centers, Comparator.comparing(vector -> Math.atan2(vector.get(1), vector.get(0))));
-
-        DistanceMeasure measure = mdl.distanceMeasure();
-
-        assertEquals(0, measure.compute(centers[0], new DenseLocalOnHeapVector(new double[]{-10, -10})), 1);
-        assertEquals(0, measure.compute(centers[1], new DenseLocalOnHeapVector(new double[]{10, -10})), 1);
-        assertEquals(0, measure.compute(centers[2], new DenseLocalOnHeapVector(new double[]{10, 10})), 1);
-        assertEquals(0, measure.compute(centers[3], new DenseLocalOnHeapVector(new double[]{-10, 10})), 1);
-
-        pntMatrix.destroy();
-    }
-
-    /** Perform N tests each of which contains M random points placed around K centers on the plane. */
-    public void testTwoDimensionsRandomlyPlacedPointsAndCenters() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        final int numOfTests = 5;
-
-        final double exponentialWeight = 2.0;
-        final double maxCentersDelta = 0.01;
-        final int maxIterations = 500;
-        final Long seed = 1L;
-
-        DistanceMeasure measure = new EuclideanDistance();
-        FuzzyCMeansDistributedClusterer distributedClusterer = new FuzzyCMeansDistributedClusterer(measure,
-                exponentialWeight, BaseFuzzyCMeansClusterer.StopCondition.STABLE_CENTERS,
-                maxCentersDelta, maxIterations, seed, 2, 50);
-
-        for (int i = 0; i < numOfTests; i++)
-            performRandomTest(distributedClusterer, i);
-    }
-
-    /**
-     * Test given clusterer on points placed randomly around vertexes of a regular polygon.
-     *
-     * @param distributedClusterer Tested clusterer.
-     * @param seed Seed for the random numbers generator.
-     */
-    private void performRandomTest(FuzzyCMeansDistributedClusterer distributedClusterer, long seed) {
-        final int minNumCenters = 2;
-        final int maxNumCenters = 5;
-        final double maxRadius = 1000;
-        final int maxPoints = 1000;
-        final int minPoints = 300;
-
-        Random random = new Random(seed);
-
-        int numCenters = random.nextInt(maxNumCenters - minNumCenters) + minNumCenters;
-
-        double[][] centers = new double[numCenters][2];
-
-        for (int i = 0; i < numCenters; i++) {
-            double angle = Math.PI * 2.0 * i / numCenters;
-
-            centers[i][0] = Math.cos(angle) * maxRadius;
-            centers[i][1] = Math.sin(angle) * maxRadius;
-        }
-
-        int numPoints = minPoints + random.nextInt(maxPoints - minPoints);
-
-        double[][] points = new double[numPoints][2];
-
-        for (int i = 0; i < numPoints; i++) {
-            int center = random.nextInt(numCenters);
-            double randomDouble = random.nextDouble();
-            double radius = randomDouble * randomDouble * maxRadius / 10;
-            double angle = random.nextDouble() * Math.PI * 2.0;
-
-            points[i][0] = centers[center][0] + Math.cos(angle) * radius;
-            points[i][1] = centers[center][1] + Math.sin(angle) * radius;
-        }
-
-        SparseDistributedMatrix pntMatrix = new SparseDistributedMatrix(numPoints, 2,
-                StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
-        for (int i = 0; i < numPoints; i++)
-            pntMatrix.setRow(i, points[i]);
-
-        FuzzyCMeansModel mdl = distributedClusterer.cluster(pntMatrix, numCenters);
-        Vector[] computedCenters = mdl.centers();
-        DistanceMeasure measure = mdl.distanceMeasure();
-
-        int cntr = numCenters;
-
-        for (int i = 0; i < numCenters; i++) {
-            for (int j = 0; j < numCenters; j++) {
-                if (measure.compute(computedCenters[i], new DenseLocalOnHeapVector(centers[j])) < 100) {
-                    cntr--;
-                    break;
-                }
-            }
-        }
-
-        assertEquals(0, cntr);
-
-        pntMatrix.destroy();
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/FuzzyCMeansLocalClustererTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/FuzzyCMeansLocalClustererTest.java
deleted file mode 100644
index 4fe1eee3fa078..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/FuzzyCMeansLocalClustererTest.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.exceptions.MathIllegalArgumentException;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/** Tests that checks local Fuzzy C-Means clusterer. */
-public class FuzzyCMeansLocalClustererTest {
-    /** Test FCM on points that forms three clusters on the line. */
-    @Test
-    public void equalWeightsOneDimension() {
-        FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(new EuclideanDistance(),
-                2, BaseFuzzyCMeansClusterer.StopCondition.STABLE_CENTERS,
-                0.01, 10, null);
-
-        double[][] points = new double[][]{{-10}, {-9}, {-8}, {-7},
-                                           {7},   {8},  {9},  {10},
-                                           {-1},  {0},  {1}};
-
-        DenseLocalOnHeapMatrix pntMatrix = new DenseLocalOnHeapMatrix(points);
-
-        FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, 3);
-
-        Vector[] centers = mdl.centers();
-        Arrays.sort(centers, Comparator.comparing(vector -> vector.getX(0)));
-        assertEquals(-8.5, centers[0].getX(0), 2);
-        assertEquals(0, centers[1].getX(0), 2);
-        assertEquals(8.5, centers[2].getX(0), 2);
-    }
-
-    /** Test FCM on points that forms four clusters on the plane. */
-    @Test
-    public void equalWeightsTwoDimensions() {
-        FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(new EuclideanDistance(),
-                2, BaseFuzzyCMeansClusterer.StopCondition.STABLE_CENTERS,
-                0.01, 20, null);
-
-        double[][] points = new double[][]{{-10, -10}, {-9, -11}, {-10, -9}, {-11, -9},
-                                           {10, 10},   {9, 11},   {10, 9},   {11, 9},
-                                           {-10, 10},  {-9, 11},  {-10, 9},  {-11, 9},
-                                           {10, -10},  {9, -11},  {10, -9},  {11, -9}};
-
-        DenseLocalOnHeapMatrix pntMatrix = new DenseLocalOnHeapMatrix(points);
-
-        FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, 4);
-        Vector[] centers = mdl.centers();
-        Arrays.sort(centers, Comparator.comparing(vector -> Math.atan2(vector.get(1), vector.get(0))));
-
-        DistanceMeasure measure = mdl.distanceMeasure();
-
-        assertEquals(0, measure.compute(centers[0], new DenseLocalOnHeapVector(new double[]{-10, -10})), 1);
-        assertEquals(0, measure.compute(centers[1], new DenseLocalOnHeapVector(new double[]{10, -10})), 1);
-        assertEquals(0, measure.compute(centers[2], new DenseLocalOnHeapVector(new double[]{10, 10})), 1);
-        assertEquals(0, measure.compute(centers[3], new DenseLocalOnHeapVector(new double[]{-10, 10})), 1);
-    }
-
-    /** Test FCM on points which have the equal coordinates. */
-    @Test
-    public void checkCentersOfTheSamePointsTwoDimensions() {
-        FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(new EuclideanDistance(),
-                2, BaseFuzzyCMeansClusterer.StopCondition.STABLE_MEMBERSHIPS, 0.01, 10, null);
-
-        double[][] points = new double[][] {{3.3, 10}, {3.3, 10}, {3.3, 10}, {3.3, 10}, {3.3, 10}};
-
-        DenseLocalOnHeapMatrix pntMatrix = new DenseLocalOnHeapMatrix(points);
-
-        int k = 2;
-        FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, k);
-        Vector exp = new DenseLocalOnHeapVector(new double[] {3.3, 10});
-        for (int i = 0; i < k; i++) {
-            Vector center = mdl.centers()[i];
-
-            for (int j = 0; j < 2; j++)
-                assertEquals(exp.getX(j), center.getX(j), 1);
-        }
-    }
-
-    /** Test FCM on points located on the circle. */
-    @Test
-    public void checkCentersLocationOnSphere() {
-        FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(new EuclideanDistance(),
-                2, BaseFuzzyCMeansClusterer.StopCondition.STABLE_CENTERS, 0.01, 100, null);
-
-        int numOfPoints = 650;
-        double radius = 100.0;
-        double[][] points = new double [numOfPoints][2];
-
-        for (int i = 0; i < numOfPoints; i++) {
-            points[i][0] = Math.cos(Math.PI * 2 * i / numOfPoints) * radius;
-            points[i][1] = Math.sin(Math.PI * 2 * i / numOfPoints) * radius;
-        }
-
-        DenseLocalOnHeapMatrix pntMatrix = new DenseLocalOnHeapMatrix(points);
-
-        int k = 10;
-        FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, k);
-
-        Vector sum = mdl.centers()[0];
-        for (int i = 1; i < k; i++)
-            sum = sum.plus(mdl.centers()[i]);
-
-        assertEquals(0, sum.kNorm(1), 1);
-    }
-
-    /** Test FCM on points that forms the line located on the plane. */
-    @Test
-    public void test2DLineClustering() {
-        FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(new EuclideanDistance(),
-                2, BaseFuzzyCMeansClusterer.StopCondition.STABLE_CENTERS, 0.01, 50, null);
-
-        double[][] points = new double[][]{{1, 2}, {3, 6}, {5, 10}};
-
-        DenseLocalOnHeapMatrix pntMatrix = new DenseLocalOnHeapMatrix(points);
-
-        int k = 2;
-        FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, k);
-        Vector[] centers = mdl.centers();
-        Arrays.sort(centers, Comparator.comparing(vector -> vector.getX(0)));
-
-        Vector[] exp = {new DenseLocalOnHeapVector(new double[]{1.5, 3}),
-                        new DenseLocalOnHeapVector(new double[]{4.5, 9})};
-
-        for (int i = 0; i < k; i++) {
-            Vector center = centers[i];
-
-            for (int j = 0; j < 2; j++)
-                assertEquals(exp[i].getX(j), center.getX(j), 0.5);
-        }
-    }
-
-    /** Test FCM on points that have different weights. */
-    @Test
-    public void differentWeightsOneDimension() {
-        FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(new EuclideanDistance(),
-                2, BaseFuzzyCMeansClusterer.StopCondition.STABLE_CENTERS,
-                0.01, 10, null);
-
-        double[][] points = new double[][]{{1}, {2}, {3}, {4}, {5}, {6}};
-
-        DenseLocalOnHeapMatrix pntMatrix = new DenseLocalOnHeapMatrix(points);
-        ArrayList<Double> weights = new ArrayList<>();
-        Collections.addAll(weights, 3.0, 2.0, 1.0, 1.0, 1.0, 1.0);
-
-        Vector[] centers1 = clusterer.cluster(pntMatrix, 2).centers();
-        Vector[] centers2 = clusterer.cluster(pntMatrix, 2, weights).centers();
-        Arrays.sort(centers1, Comparator.comparing(vector -> vector.getX(0)));
-        Arrays.sort(centers2, Comparator.comparing(vector -> vector.getX(0)));
-
-        assertTrue(centers1[0].get(0) - centers2[0].get(0) > 0.5);
-    }
-
-    /** Test FCM on illegal number of clusters. */
-    @Test(expected = MathIllegalArgumentException.class)
-    public void testIllegalNumberOfClusters() {
-        FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(new EuclideanDistance(),
-                2, BaseFuzzyCMeansClusterer.StopCondition.STABLE_CENTERS, 0.01, 10, null);
-        double[][] points = new double[][]{{1}, {2}, {3}, {4}};
-
-        clusterer.cluster(new DenseLocalOnHeapMatrix(points), 1);
-    }
-
-    /** Test FCM on different numbers of points and weights. */
-    @Test(expected = MathIllegalArgumentException.class)
-    public void testDifferentAmountsOfPointsAndWeights(){
-        FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(new EuclideanDistance(),
-                2, BaseFuzzyCMeansClusterer.StopCondition.STABLE_CENTERS, 0.01, 10, null);
-        double[][] points = new double[][]{{1}, {2}, {3}, {4}};
-
-        ArrayList<Double> weights = new ArrayList<>();
-        Collections.addAll(weights, 1.0, 34.0, 2.5, 5.0, 0.5);
-
-        clusterer.cluster(new DenseLocalOnHeapMatrix(points), 2, weights);
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTestMultiNode.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTestMultiNode.java
deleted file mode 100644
index 71be8bef77e58..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTestMultiNode.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * This test is made to make sure that K-Means distributed clustering does not crash on distributed environment.
- * In {@link KMeansDistributedClustererTestSingleNode} we check logic of clustering (checks for clusters structures).
- * In this class we just check that clusterer does not crash. There are two separate tests because we cannot
- * guarantee order in which nodes return results of intermediate computations and therefore algorithm can return
- * different results.
- */
-public class KMeansDistributedClustererTestMultiNode extends GridCommonAbstractTest {
-    /** Number of nodes in grid. */
-    private static final int NODE_COUNT = 3;
-
-    /** Grid instance. */
-    private Ignite ignite;
-
-    /**
-     * Default constructor.
-     */
-    public KMeansDistributedClustererTestMultiNode() {
-        super(false);
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override protected void beforeTest() throws Exception {
-        ignite = grid(NODE_COUNT);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        for (int i = 1; i <= NODE_COUNT; i++)
-            startGrid(i);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        stopAllGrids();
-    }
-
-    /** */
-    public void testPerformClusterAnalysisDegenerate() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(new EuclideanDistance(), 1, 1, 1L);
-
-        double[] v1 = new double[] {1959, 325100};
-        double[] v2 = new double[] {1960, 373200};
-
-        SparseDistributedMatrix points = new SparseDistributedMatrix(2, 2, StorageConstants.ROW_STORAGE_MODE,
-            StorageConstants.RANDOM_ACCESS_MODE);
-
-        points.setRow(0, v1);
-        points.setRow(1, v2);
-
-        clusterer.cluster(points, 1);
-
-        points.destroy();
-    }
-
-    /** */
-    public void testClusterizationOnDatasetWithObviousStructure() throws IOException {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        int ptsCnt = 10000;
-        int squareSideLen = 10000;
-
-        Random rnd = new Random(123456L);
-
-        // Let centers be in the vertices of square.
-        Map<Integer, Vector> centers = new HashMap<>();
-        centers.put(100, new DenseLocalOnHeapVector(new double[] {0.0, 0.0}));
-        centers.put(900, new DenseLocalOnHeapVector(new double[] {squareSideLen, 0.0}));
-        centers.put(3000, new DenseLocalOnHeapVector(new double[] {0.0, squareSideLen}));
-        centers.put(6000, new DenseLocalOnHeapVector(new double[] {squareSideLen, squareSideLen}));
-
-        SparseDistributedMatrix points = new SparseDistributedMatrix(ptsCnt, 2, StorageConstants.ROW_STORAGE_MODE,
-            StorageConstants.RANDOM_ACCESS_MODE);
-
-        List<Integer> permutation = IntStream.range(0, ptsCnt).boxed().collect(Collectors.toList());
-        Collections.shuffle(permutation, rnd);
-
-        int totalCnt = 0;
-
-        for (Integer count : centers.keySet()) {
-            for (int i = 0; i < count; i++) {
-                Vector pnt = new DenseLocalOnHeapVector(2).assign(centers.get(count));
-                // Perturbate point on random value.
-                pnt.map(val -> val + rnd.nextDouble() * squareSideLen / 100);
-                points.assignRow(permutation.get(totalCnt), pnt);
-                totalCnt++;
-            }
-        }
-
-        EuclideanDistance dist = new EuclideanDistance();
-
-        KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(dist, 3, 100, 1L);
-
-        clusterer.cluster(points, 4);
-
-        points.destroy();
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTestSingleNode.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTestSingleNode.java
deleted file mode 100644
index 705db7ad35157..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansDistributedClustererTestSingleNode.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.util.IgniteUtils;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.Vector;
-import org.apache.ignite.ml.math.VectorUtils;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.functions.Functions;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.junit.Assert;
-
-import static org.apache.ignite.ml.clustering.KMeansUtil.checkIsInEpsilonNeighbourhood;
-
-/**
- * This test checks logic of clustering (checks for clusters structures).
- */
-public class KMeansDistributedClustererTestSingleNode extends GridCommonAbstractTest {
-    /**
-     * Number of nodes in grid. We should use 1 in this test because otherwise algorithm will be unstable
-     * (We cannot guarantee the order in which results are returned from each node).
-     */
-    private static final int NODE_COUNT = 1;
-
-    /** Grid instance. */
-    private Ignite ignite;
-
-    /**
-     * Default constructor.
-     */
-    public KMeansDistributedClustererTestSingleNode() {
-        super(false);
-    }
-
-    /**
-     * {@inheritDoc}
-     */
-    @Override protected void beforeTest() throws Exception {
-        ignite = grid(NODE_COUNT);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        for (int i = 1; i <= NODE_COUNT; i++)
-            startGrid(i);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        stopAllGrids();
-    }
-
-    /** */
-    public void testPerformClusterAnalysisDegenerate() {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(new EuclideanDistance(), 1, 1, 1L);
-
-        double[] v1 = new double[] {1959, 325100};
-        double[] v2 = new double[] {1960, 373200};
-
-        SparseDistributedMatrix points = new SparseDistributedMatrix(2, 2, StorageConstants.ROW_STORAGE_MODE,
-            StorageConstants.RANDOM_ACCESS_MODE);
-
-        points.setRow(0, v1);
-        points.setRow(1, v2);
-
-        KMeansModel mdl = clusterer.cluster(points, 1);
-
-        Assert.assertEquals(1, mdl.centers().length);
-        Assert.assertEquals(2, mdl.centers()[0].size());
-    }
-
-    /** */
-    public void testClusterizationOnDatasetWithObviousStructure() throws IOException {
-        IgniteUtils.setCurrentIgniteName(ignite.configuration().getIgniteInstanceName());
-
-        int ptsCnt = 10000;
-        int squareSideLen = 10000;
-
-        Random rnd = new Random(123456L);
-
-        // Let centers be in the vertices of square.
-        Map<Integer, Vector> centers = new HashMap<>();
-        centers.put(100, new DenseLocalOnHeapVector(new double[] {0.0, 0.0}));
-        centers.put(900, new DenseLocalOnHeapVector(new double[] {squareSideLen, 0.0}));
-        centers.put(3000, new DenseLocalOnHeapVector(new double[] {0.0, squareSideLen}));
-        centers.put(6000, new DenseLocalOnHeapVector(new double[] {squareSideLen, squareSideLen}));
-
-        int centersCnt = centers.size();
-
-        SparseDistributedMatrix points = new SparseDistributedMatrix(ptsCnt, 2, StorageConstants.ROW_STORAGE_MODE,
-            StorageConstants.RANDOM_ACCESS_MODE);
-
-        List<Integer> permutation = IntStream.range(0, ptsCnt).boxed().collect(Collectors.toList());
-        Collections.shuffle(permutation, rnd);
-
-        Vector[] mc = new Vector[centersCnt];
-        Arrays.fill(mc, VectorUtils.zeroes(2));
-
-        int centIdx = 0;
-        int totalCnt = 0;
-
-        List<Vector> massCenters = new ArrayList<>();
-
-        for (Integer count : centers.keySet()) {
-            for (int i = 0; i < count; i++) {
-                Vector pnt = new DenseLocalOnHeapVector(2).assign(centers.get(count));
-                // Perturbate point on random value.
-                pnt.map(val -> val + rnd.nextDouble() * squareSideLen / 100);
-                mc[centIdx] = mc[centIdx].plus(pnt);
-                points.assignRow(permutation.get(totalCnt), pnt);
-                totalCnt++;
-            }
-            massCenters.add(mc[centIdx].times(1 / (double)count));
-            centIdx++;
-        }
-
-        EuclideanDistance dist = new EuclideanDistance();
-        OrderedNodesComparator comp = new OrderedNodesComparator(centers.values().toArray(new Vector[] {}), dist);
-
-        massCenters.sort(comp);
-        KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(dist, 3, 100, 1L);
-
-        KMeansModel mdl = clusterer.cluster(points, 4);
-        Vector[] resCenters = mdl.centers();
-        Arrays.sort(resCenters, comp);
-
-        checkIsInEpsilonNeighbourhood(resCenters, massCenters.toArray(new Vector[] {}), 30.0);
-
-        points.destroy();
-    }
-
-    /** */
-    private static class OrderedNodesComparator implements Comparator<Vector> {
-        /** */
-        private final DistanceMeasure measure;
-
-        /** */
-        List<Vector> orderedNodes;
-
-        /** */
-        OrderedNodesComparator(Vector[] orderedNodes, DistanceMeasure measure) {
-            this.orderedNodes = Arrays.asList(orderedNodes);
-            this.measure = measure;
-        }
-
-        /** */
-        private int findClosestNodeIndex(Vector v) {
-            return Functions.argmin(orderedNodes, v1 -> measure.compute(v1, v)).get1();
-        }
-
-        /** */
-        @Override public int compare(Vector v1, Vector v2) {
-            int ind1 = findClosestNodeIndex(v1);
-            int ind2 = findClosestNodeIndex(v2);
-
-            int signum = (int)Math.signum(ind1 - ind2);
-
-            if (signum != 0)
-                return signum;
-
-            return (int)Math.signum(orderedNodes.get(ind1).minus(v1).kNorm(2) -
-                orderedNodes.get(ind2).minus(v2).kNorm(2));
-        }
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java
deleted file mode 100644
index cd9b2ede9d5b5..0000000000000
--- a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansLocalClustererTest.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.ml.clustering;
-
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.junit.Assert;
-import org.junit.Test;
-
-/** */
-public class KMeansLocalClustererTest {
-    /**
-     * Two points, one cluster, one iteration
-     */
-    @Test
-    public void testPerformClusterAnalysisDegenerate() {
-        KMeansLocalClusterer clusterer = new KMeansLocalClusterer(new EuclideanDistance(), 1, 1L);
-
-        double[] v1 = new double[] {1959, 325100};
-        double[] v2 = new double[] {1960, 373200};
-
-        DenseLocalOnHeapMatrix points = new DenseLocalOnHeapMatrix(new double[][] {
-            v1,
-            v2});
-
-        KMeansModel mdl = clusterer.cluster(points, 1);
-
-        Assert.assertEquals(1, mdl.centers().length);
-        Assert.assertEquals(2, mdl.centers()[0].size());
-    }
-}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansModelTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansModelTest.java
new file mode 100644
index 0000000000000..5b3ad85040482
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansModelTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import org.apache.ignite.ml.TestUtils;
+import org.apache.ignite.ml.clustering.kmeans.KMeansModel;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.distances.DistanceMeasure;
+import org.apache.ignite.ml.math.distances.EuclideanDistance;
+import org.apache.ignite.ml.math.exceptions.CardinalityException;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
+import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
+import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Tests for {@link KMeansModel}.
+ */
+public class KMeansModelTest {
+    /** Precision in test checks. */
+    private static final double PRECISION = 1e-6;
+
+    /** */
+    @Test
+    public void predictClusters() {
+        DistanceMeasure distanceMeasure = new EuclideanDistance();
+
+        Vector[] centers = new DenseLocalOnHeapVector[4];
+
+        centers[0] = new DenseLocalOnHeapVector(new double[]{1.0, 1.0});
+        centers[1] = new DenseLocalOnHeapVector(new double[]{-1.0, 1.0});
+        centers[2] = new DenseLocalOnHeapVector(new double[]{1.0, -1.0});
+        centers[3] = new DenseLocalOnHeapVector(new double[]{-1.0, -1.0});
+
+        KMeansModel mdl = new KMeansModel(centers, distanceMeasure);
+
+        Assert.assertEquals(mdl.apply(new DenseLocalOnHeapVector(new double[]{1.1, 1.1})), 0.0, PRECISION);
+        Assert.assertEquals(mdl.apply(new DenseLocalOnHeapVector(new double[]{-1.1, 1.1})), 1.0, PRECISION);
+        Assert.assertEquals(mdl.apply(new DenseLocalOnHeapVector(new double[]{1.1, -1.1})), 2.0, PRECISION);
+        Assert.assertEquals(mdl.apply(new DenseLocalOnHeapVector(new double[]{-1.1, -1.1})), 3.0, PRECISION);
+
+        Assert.assertEquals(mdl.distanceMeasure(), distanceMeasure);
+        Assert.assertEquals(mdl.amountOfClusters(), 4);
+        Assert.assertArrayEquals(mdl.centers(), centers);
+    }
+}
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java
new file mode 100644
index 0000000000000..846d0de86013e
--- /dev/null
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/clustering/KMeansTrainerTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ml.clustering;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.ml.clustering.kmeans.KMeansModel;
+import org.apache.ignite.ml.clustering.kmeans.KMeansTrainer;
+import org.apache.ignite.ml.dataset.impl.local.LocalDatasetBuilder;
+import org.apache.ignite.ml.math.Vector;
+import org.apache.ignite.ml.math.distances.EuclideanDistance;
+import org.apache.ignite.ml.math.impls.vector.DenseLocalOnHeapVector;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Tests for {@link KMeansTrainer}.
+ */
+public class KMeansTrainerTest {
+    /** Precision in test checks. */
+    private static final double PRECISION = 1e-2;
+
+    /**
+     * A few points, one cluster, one iteration
+     */
+    @Test
+    public void findOneClusters() {
+
+        Map<Integer, double[]> data = new HashMap<>();
+        data.put(0, new double[] {1.0, 1.0, 1.0});
+        data.put(1, new double[] {1.0, 2.0, 1.0});
+        data.put(2, new double[] {2.0, 1.0, 1.0});
+        data.put(3, new double[] {-1.0, -1.0, 2.0});
+        data.put(4, new double[] {-1.0, -2.0, 2.0});
+        data.put(5, new double[] {-2.0, -1.0, 2.0});
+
+        KMeansTrainer trainer = new KMeansTrainer()
+            .withDistance(new EuclideanDistance())
+            .withK(1)
+            .withMaxIterations(1)
+            .withEpsilon(PRECISION);
+
+        KMeansModel knnMdl = trainer.fit(
+            new LocalDatasetBuilder<>(data, 2),
+            (k, v) -> Arrays.copyOfRange(v, 0, v.length - 1),
+            (k, v) -> v[2]
+        );
+
+        Vector firstVector = new DenseLocalOnHeapVector(new double[] {2.0, 2.0});
+        assertEquals(knnMdl.apply(firstVector), 0.0, PRECISION);
+        Vector secondVector = new DenseLocalOnHeapVector(new double[] {-2.0, -2.0});
+        assertEquals(knnMdl.apply(secondVector), 0.0, PRECISION);
+        assertEquals(trainer.getMaxIterations(), 1);
+        assertEquals(trainer.getEpsilon(), PRECISION, PRECISION);
+    }
+}

From 6ee5e9fe8c07353a55a2a02b84f7b94c43b5c49f Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 17 Apr 2018 11:46:45 +0700
Subject: [PATCH 057/543] IGNITE-8201 REST: Added AUTHENTICATE command. Fixed
 session tokens. Added new tests.

(cherry picked from commit 1cfc989)
---
 .../client/suite/IgniteClientTestSuite.java   |  6 +-
 .../JettyRestProcessorAbstractSelfTest.java   | 60 +++++++++-----
 ...tyRestProcessorAuthenticationSelfTest.java | 45 ++---------
 ...cessorAuthenticationWithCredsSelfTest.java | 32 ++++++++
 ...cessorAuthenticationWithTokenSelfTest.java | 80 +++++++++++++++++++
 .../processors/rest/GridRestCommand.java      |  3 +
 .../processors/rest/GridRestProcessor.java    | 26 +++---
 .../auth/AuthenticationCommandHandler.java    | 70 ++++++++++++++++
 .../rest/handlers/auth/package-info.java      | 22 +++++
 .../http/jetty/GridJettyRestHandler.java      | 55 ++++++++++---
 10 files changed, 322 insertions(+), 77 deletions(-)
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationWithCredsSelfTest.java
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationWithTokenSelfTest.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/auth/AuthenticationCommandHandler.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/auth/package-info.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
index 79fcf38c67e1a..163f89acb7a35 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
@@ -49,7 +49,8 @@
 import org.apache.ignite.internal.client.util.ClientConsistentHashSelfTest;
 import org.apache.ignite.internal.client.util.ClientJavaHasherSelfTest;
 import org.apache.ignite.internal.processors.rest.ClientMemcachedProtocolSelfTest;
-import org.apache.ignite.internal.processors.rest.JettyRestProcessorAuthenticationSelfTest;
+import org.apache.ignite.internal.processors.rest.JettyRestProcessorAuthenticationWithCredsSelfTest;
+import org.apache.ignite.internal.processors.rest.JettyRestProcessorAuthenticationWithTokenSelfTest;
 import org.apache.ignite.internal.processors.rest.JettyRestProcessorSignedSelfTest;
 import org.apache.ignite.internal.processors.rest.JettyRestProcessorUnsignedSelfTest;
 import org.apache.ignite.internal.processors.rest.RestBinaryProtocolSelfTest;
@@ -87,7 +88,8 @@ public static TestSuite suite() {
         // Test jetty rest processor
         suite.addTestSuite(JettyRestProcessorSignedSelfTest.class);
         suite.addTestSuite(JettyRestProcessorUnsignedSelfTest.class);
-        suite.addTestSuite(JettyRestProcessorAuthenticationSelfTest.class);
+        suite.addTestSuite(JettyRestProcessorAuthenticationWithCredsSelfTest.class);
+        suite.addTestSuite(JettyRestProcessorAuthenticationWithTokenSelfTest.class);
 
         // Test TCP rest processor with original memcache client.
         suite.addTestSuite(ClientMemcachedProtocolSelfTest.class);
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
index 5dc44c4145516..e36447bc3e0c4 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
@@ -294,14 +294,19 @@ protected void assertResponseContainsError(String content, String err) throws IO
 
     /**
      * @param content Content to check.
+     * @return JSON node with actual response.
      */
-    private JsonNode jsonCacheOperationResponse(String content, boolean bulk) throws IOException {
+    protected JsonNode assertResponseSucceeded(String content, boolean bulk) throws IOException {
         assertNotNull(content);
         assertFalse(content.isEmpty());
 
         JsonNode node = JSON_MAPPER.readTree(content);
 
-        assertEquals(bulk, node.get("affinityNodeId").isNull());
+        JsonNode affNode = node.get("affinityNodeId");
+
+        if (affNode != null)
+            assertEquals(bulk, affNode.isNull());
+
         assertEquals(STATUS_SUCCESS, node.get("successStatus").asInt());
         assertTrue(node.get("error").isNull());
 
@@ -315,7 +320,7 @@ private JsonNode jsonCacheOperationResponse(String content, boolean bulk) throws
      * @param res Response.
      */
     private void assertCacheOperation(String content, Object res) throws IOException {
-        JsonNode ret = jsonCacheOperationResponse(content, false);
+        JsonNode ret = assertResponseSucceeded(content, false);
 
         assertEquals(String.valueOf(res), ret.isObject() ? ret.toString() : ret.asText());
     }
@@ -325,7 +330,7 @@ private void assertCacheOperation(String content, Object res) throws IOException
      * @param res Response.
      */
     private void assertCacheBulkOperation(String content, Object res) throws IOException {
-        JsonNode ret = jsonCacheOperationResponse(content, true);
+        JsonNode ret = assertResponseSucceeded(content, true);
 
         assertEquals(String.valueOf(res), ret.asText());
     }
@@ -334,7 +339,7 @@ private void assertCacheBulkOperation(String content, Object res) throws IOExcep
      * @param content Content to check.
      */
     private void assertCacheMetrics(String content) throws IOException {
-        JsonNode ret = jsonCacheOperationResponse(content, true);
+        JsonNode ret = assertResponseSucceeded(content, true);
 
         assertTrue(ret.isObject());
     }
@@ -349,7 +354,7 @@ protected JsonNode jsonResponse(String content) throws IOException {
 
         JsonNode node = JSON_MAPPER.readTree(content);
 
-        assertEquals(0, node.get("successStatus").asInt());
+        assertEquals(STATUS_SUCCESS, node.get("successStatus").asInt());
         assertTrue(node.get("error").isNull());
 
         assertNotSame(securityEnabled(), node.get("sessionToken").isNull());
@@ -367,7 +372,7 @@ protected JsonNode jsonTaskResult(String content) throws IOException {
 
         JsonNode node = JSON_MAPPER.readTree(content);
 
-        assertEquals(0, node.get("successStatus").asInt());
+        assertEquals(STATUS_SUCCESS, node.get("successStatus").asInt());
         assertTrue(node.get("error").isNull());
         assertFalse(node.get("response").isNull());
 
@@ -403,7 +408,7 @@ public void testGet() throws Exception {
      * @throws IOException If failed.
      */
     private void checkJson(String json, Person p) throws IOException {
-        JsonNode res = jsonCacheOperationResponse(json, false);
+        JsonNode res = assertResponseSucceeded(json, false);
 
         assertEquals(p.id.intValue(), res.get("id").asInt());
         assertEquals(p.getOrganizationId().intValue(), res.get("orgId").asInt());
@@ -455,7 +460,7 @@ public void testGetBinaryObjects() throws Exception {
 
         info("Get command result: " + ret);
 
-        JsonNode res = jsonCacheOperationResponse(ret, false);
+        JsonNode res = assertResponseSucceeded(ret, false);
 
         assertEquals("Alex", res.get("NAME").asText());
         assertEquals(300, res.get("SALARY").asInt());
@@ -476,7 +481,7 @@ public void testGetBinaryObjects() throws Exception {
 
         info("Get command result: " + ret);
 
-        JsonNode json = jsonCacheOperationResponse(ret, false);
+        JsonNode json = assertResponseSucceeded(ret, false);
         assertEquals(ref1.name, json.get("name").asText());
 
         ref2.ref(ref1);
@@ -552,7 +557,7 @@ public void testSimpleObject() throws Exception {
 
         info("Get command result: " + ret);
 
-        JsonNode res = jsonCacheOperationResponse(ret, false);
+        JsonNode res = assertResponseSucceeded(ret, false);
 
         assertEquals(p.id, res.get("id").asInt());
         assertEquals(p.name, res.get("name").asText());
@@ -637,7 +642,7 @@ public void testTuple() throws Exception {
 
         info("Get command result: " + ret);
 
-        JsonNode res = jsonCacheOperationResponse(ret, false);
+        JsonNode res = assertResponseSucceeded(ret, false);
 
         assertEquals(t.getKey(), res.get("key").asText());
         assertEquals(t.getValue(), res.get("value").asText());
@@ -775,11 +780,11 @@ public void testGetAll() throws Exception {
 
         info("Get all command result: " + ret);
 
-        JsonNode res = jsonCacheOperationResponse(ret, true);
+        JsonNode res = assertResponseSucceeded(ret, true);
 
         assertTrue(res.isObject());
 
-        assertTrue(entries.equals(JSON_MAPPER.treeToValue(res, Map.class)));
+        assertEquals(entries, JSON_MAPPER.treeToValue(res, Map.class));
     }
 
     /**
@@ -973,10 +978,25 @@ public void testPut() throws Exception {
         assertCacheOperation(ret, true);
     }
 
+    /** */
+    private void failIgnite_5874() {
+        DataStorageConfiguration dsCfg = ignite(0).configuration().getDataStorageConfiguration();
+
+        if (dsCfg.getDefaultDataRegionConfiguration().isPersistenceEnabled())
+            fail("IGNITE-5874");
+
+        for (DataRegionConfiguration dataRegCfg : dsCfg.getDataRegionConfigurations()) {
+            if (dataRegCfg.isPersistenceEnabled())
+                fail("IGNITE-5874");
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
     public void testPutWithExpiration() throws Exception {
+        failIgnite_5874();
+
         String ret = content(DEFAULT_CACHE_NAME, GridRestCommand.CACHE_PUT,
             "key", "putKey",
             "val", "putVal",
@@ -1013,6 +1033,8 @@ public void testAdd() throws Exception {
      * @throws Exception If failed.
      */
     public void testAddWithExpiration() throws Exception {
+        failIgnite_5874();
+
         String ret = content(DEFAULT_CACHE_NAME, GridRestCommand.CACHE_ADD,
             "key", "addKey",
             "val", "addVal",
@@ -1100,7 +1122,7 @@ public void testRemoveAll() throws Exception {
         assertNull(jcache().localPeek("rmvKey2"));
         assertNull(jcache().localPeek("rmvKey3"));
         assertNull(jcache().localPeek("rmvKey4"));
-        assertTrue(jcache().localSize() == 0);
+        assertEquals(0, jcache().localSize());
 
         assertCacheBulkOperation(ret, true);
     }
@@ -1152,6 +1174,8 @@ public void testReplace() throws Exception {
      * @throws Exception If failed.
      */
     public void testReplaceWithExpiration() throws Exception {
+        failIgnite_5874();
+
         jcache().put("replaceKey", "replaceVal");
 
         assertEquals("replaceVal", jcache().get("replaceKey"));
@@ -1353,20 +1377,20 @@ private void testMetadata(Collection<GridCacheSqlMetadata> metas, JsonNode arr)
             assertNotNull(keyClasses);
             assertFalse(keyClasses.isNull());
 
-            assertTrue(meta.keyClasses().equals(JSON_MAPPER.treeToValue(keyClasses, Map.class)));
+            assertEquals(meta.keyClasses(), JSON_MAPPER.treeToValue(keyClasses, Map.class));
 
             JsonNode valClasses = item.get("valClasses");
 
             assertNotNull(valClasses);
             assertFalse(valClasses.isNull());
 
-            assertTrue(meta.valClasses().equals(JSON_MAPPER.treeToValue(valClasses, Map.class)));
+            assertEquals(meta.valClasses(), JSON_MAPPER.treeToValue(valClasses, Map.class));
 
             JsonNode fields = item.get("fields");
 
             assertNotNull(fields);
             assertFalse(fields.isNull());
-            assertTrue(meta.fields().equals(JSON_MAPPER.treeToValue(fields, Map.class)));
+            assertEquals(meta.fields(), JSON_MAPPER.treeToValue(fields, Map.class));
 
             JsonNode indexesByType = item.get("indexes");
 
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationSelfTest.java
index ca62091a2ecee..27b8c03ff696d 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationSelfTest.java
@@ -24,7 +24,6 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.processors.authentication.IgniteAccessControlException;
 import org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor;
-import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -33,18 +32,12 @@
 /**
  * Test REST with enabled authentication.
  */
-public class JettyRestProcessorAuthenticationSelfTest extends JettyRestProcessorUnsignedSelfTest {
+public abstract class JettyRestProcessorAuthenticationSelfTest extends JettyRestProcessorUnsignedSelfTest {
     /** */
-    private static final String DFLT_LOGIN = "ignite";
+    protected static final String DFLT_USER = "ignite";
 
     /** */
-    private static final String DFLT_PWD = "ignite";
-
-    /** */
-    private String login = DFLT_LOGIN;
-
-    /** */
-    private String pwd = DFLT_PWD;
+    protected static final String DFLT_PWD = "ignite";
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
@@ -54,11 +47,8 @@ public class JettyRestProcessorAuthenticationSelfTest extends JettyRestProcessor
     }
 
     /** {@inheritDoc} */
-    @Override protected void beforeTest() throws Exception {
-        super.beforeTest();
-
-        login = DFLT_LOGIN;
-        pwd = DFLT_PWD;
+    @Override protected boolean securityEnabled() {
+        return true;
     }
 
     /** {@inheritDoc} */
@@ -97,32 +87,13 @@ public class JettyRestProcessorAuthenticationSelfTest extends JettyRestProcessor
         return cfg;
     }
 
-    /** {@inheritDoc} */
-    @Override protected String restUrl() {
-        String url = super.restUrl();
-
-        if (!F.isEmpty(login)) {
-            url += "ignite.login=" + login;
-
-            if (!F.isEmpty(pwd))
-                url += "&ignite.password=" + pwd;
-
-            url += '&';
-        }
-
-        return url;
-    }
-
     /**
      * @throws Exception If failed.
      */
-    public void testMissingCredentials() throws Exception {
-        login = null;
-        pwd = null;
-
-        String ret = content(null, GridRestCommand.VERSION);
+    public void testAuthenticationCommand() throws Exception {
+        String ret = content(null, GridRestCommand.AUTHENTICATE);
 
-        assertResponseContainsError(ret, "The user name or password is incorrect");
+        assertResponseSucceeded(ret, false);
     }
 
     /**
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationWithCredsSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationWithCredsSelfTest.java
new file mode 100644
index 0000000000000..c75e8a9797a1c
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationWithCredsSelfTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.rest;
+
+/**
+ * Test REST with enabled authentication and credentials in each request.
+ */
+public class JettyRestProcessorAuthenticationWithCredsSelfTest extends JettyRestProcessorAuthenticationSelfTest {
+    /** {@inheritDoc} */
+    @Override protected String restUrl() {
+        String url = super.restUrl();
+
+        url += "ignite.login=" + DFLT_USER + "&ignite.password=" + DFLT_PWD + "&";
+
+        return url;
+    }
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationWithTokenSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationWithTokenSelfTest.java
new file mode 100644
index 0000000000000..5c046af320394
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAuthenticationWithTokenSelfTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.rest;
+
+import org.apache.ignite.internal.util.typedef.F;
+
+/**
+ * Test REST with enabled authentication and token.
+ */
+public class JettyRestProcessorAuthenticationWithTokenSelfTest extends JettyRestProcessorAuthenticationSelfTest {
+    /** */
+    private String tok = "";
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        // Authenticate and extract token.
+        if (F.isEmpty(tok)) {
+            String ret = content(null, GridRestCommand.AUTHENTICATE,
+                "user", DFLT_USER,
+                "password", DFLT_PWD);
+
+            int p1 = ret.indexOf("sessionToken");
+            int p2 = ret.indexOf('"', p1 + 16);
+
+            tok = ret.substring(p1 + 15, p2);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected String restUrl() {
+        String url = super.restUrl();
+
+        if (!F.isEmpty(tok))
+            url += "sessionToken=" + tok + "&";
+
+        return url;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testInvalidSessionToken() throws Exception {
+        tok = null;
+
+        String ret = content(null, GridRestCommand.VERSION);
+
+        assertResponseContainsError(ret, "Failed to handle request - session token not found or invalid");
+
+        tok = "InvalidToken";
+
+        ret = content(null, GridRestCommand.VERSION);
+
+        assertResponseContainsError(ret, "Failed to handle request - session token not found or invalid");
+
+        tok = "26BE027D32CC42329DEC92D517B44E9E";
+
+        ret = content(null, GridRestCommand.VERSION);
+
+        assertResponseContainsError(ret, "Failed to handle request - unknown session token (maybe expired session)");
+
+        tok = null; // Cleanup token for next tests.
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
index 0b9a66212c52d..265fe4024c7c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
@@ -168,6 +168,9 @@ public enum GridRestCommand {
     /** */
     CLUSTER_CURRENT_STATE("currentstate"),
 
+    /** */
+    AUTHENTICATE("authenticate"),
+
     /** */
     ADD_USER("adduser"),
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
index 4b8497eccf28d..da5e5c205525a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
@@ -45,6 +45,7 @@
 import org.apache.ignite.internal.processors.authentication.AuthorizationContext;
 import org.apache.ignite.internal.processors.rest.client.message.GridClientTaskResultBean;
 import org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandler;
+import org.apache.ignite.internal.processors.rest.handlers.auth.AuthenticationCommandHandler;
 import org.apache.ignite.internal.processors.rest.handlers.cache.GridCacheCommandHandler;
 import org.apache.ignite.internal.processors.rest.handlers.cluster.GridChangeStateCommandHandler;
 import org.apache.ignite.internal.processors.rest.handlers.datastructures.DataStructuresCommandHandler;
@@ -81,6 +82,7 @@
 import org.apache.ignite.thread.IgniteThread;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_REST_START_ON_CLIENT;
+import static org.apache.ignite.internal.processors.rest.GridRestCommand.AUTHENTICATE;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_AUTH_FAILED;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_FAILED;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_SECURITY_CHECK_FAILED;
@@ -230,10 +232,11 @@ private IgniteInternalFuture<GridRestResponse> handleRequest(final GridRestReque
             try {
                 ses = session(req);
             }
+            catch (IgniteAuthenticationException e) {
+                return new GridFinishedFuture<>(new GridRestResponse(STATUS_AUTH_FAILED, e.getMessage()));
+            }
             catch (IgniteCheckedException e) {
-                GridRestResponse res = new GridRestResponse(STATUS_FAILED, e.getMessage());
-
-                return new GridFinishedFuture<>(res);
+                return new GridFinishedFuture<>(new GridRestResponse(STATUS_FAILED, e.getMessage()));
             }
 
             assert ses != null;
@@ -284,9 +287,9 @@ private IgniteInternalFuture<GridRestResponse> handleRequest(final GridRestReque
                             throw new IgniteAuthenticationException("The user name or password is incorrect");
 
                         ses.authCtx = ctx.authentication().authenticate(login, pwd);
-
-                        req.authorizationContext(ses.authCtx);
                     }
+
+                    req.authorizationContext(ses.authCtx);
                 }
                 catch (IgniteCheckedException e) {
                     return new GridFinishedFuture<>(new GridRestResponse(STATUS_AUTH_FAILED, e.getMessage()));
@@ -341,7 +344,7 @@ private IgniteInternalFuture<GridRestResponse> handleRequest(final GridRestReque
 
                 assert res != null;
 
-                if (ctx.security().enabled() && !failed)
+                if ((authenticationEnabled || securityEnabled) && !failed)
                     res.sessionTokenBytes(req.sessionToken());
 
                 interceptResponse(res, req);
@@ -362,6 +365,10 @@ private Session session(final GridRestRequest req) throws IgniteCheckedException
 
         while (true) {
             if (F.isEmpty(sesTok) && clientId == null) {
+                // TODO: In IGNITE 3.0 we should check credentials only for AUTHENTICATE command.
+                if (ctx.authentication().enabled() && req.command() != AUTHENTICATE && req.credentials() == null)
+                    throw new IgniteAuthenticationException("Failed to handle request - session token not found or invalid");
+
                 Session ses = Session.random();
 
                 UUID oldSesId = clientId2SesId.put(ses.clientId, ses.sesId);
@@ -451,10 +458,7 @@ public GridRestProcessor(GridKernalContext ctx) {
         try {
             sesExpTime = System.getProperty(IgniteSystemProperties.IGNITE_REST_SESSION_TIMEOUT);
 
-            if (sesExpTime != null)
-                sesExpTime0 = Long.valueOf(sesExpTime) * 1000;
-            else
-                sesExpTime0 = DEFAULT_SES_TIMEOUT;
+            sesExpTime0 = sesExpTime != null ? Long.valueOf(sesExpTime) * 1000 : DEFAULT_SES_TIMEOUT;
         }
         catch (NumberFormatException ignore) {
             U.warn(log, "Failed parsing IGNITE_REST_SESSION_TIMEOUT system variable [IGNITE_REST_SESSION_TIMEOUT="
@@ -504,6 +508,7 @@ public GridRestProcessor(GridKernalContext ctx) {
             addHandler(new QueryCommandHandler(ctx));
             addHandler(new GridLogCommandHandler(ctx));
             addHandler(new GridChangeStateCommandHandler(ctx));
+            addHandler(new AuthenticationCommandHandler(ctx));
             addHandler(new UserActionCommandHandler(ctx));
 
             // Start protocols.
@@ -860,6 +865,7 @@ private void authorize(GridRestRequest req, SecurityContext sCtx) throws Securit
             case CLUSTER_CURRENT_STATE:
             case CLUSTER_ACTIVE:
             case CLUSTER_INACTIVE:
+            case AUTHENTICATE:
             case ADD_USER:
             case REMOVE_USER:
             case UPDATE_USER:
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/auth/AuthenticationCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/auth/AuthenticationCommandHandler.java
new file mode 100644
index 0000000000000..aa9bbbdacd50b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/auth/AuthenticationCommandHandler.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.rest.handlers.auth;
+
+import java.util.Collection;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.rest.GridRestCommand;
+import org.apache.ignite.internal.processors.rest.GridRestResponse;
+import org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandlerAdapter;
+import org.apache.ignite.internal.processors.rest.request.GridRestRequest;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static org.apache.ignite.internal.processors.rest.GridRestCommand.AUTHENTICATE;
+
+/**
+ * Authentication handler.
+ */
+public class AuthenticationCommandHandler extends GridRestCommandHandlerAdapter {
+    /** Commands. */
+    private static final Collection<GridRestCommand> SUPPORTED_COMMANDS = U.sealList(AUTHENTICATE);
+
+    /**
+     * @param ctx Context.
+     */
+    public AuthenticationCommandHandler(GridKernalContext ctx) {
+        super(ctx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Collection<GridRestCommand> supportedCommands() {
+        return SUPPORTED_COMMANDS;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<GridRestResponse> handleAsync(GridRestRequest req) {
+        assert req != null;
+
+        if (log.isDebugEnabled())
+            log.debug("Handling topology REST request: " + req);
+
+        try {
+            if (log.isDebugEnabled())
+                log.debug("Handled topology REST request [req=" + req + ']');
+
+            return new GridFinishedFuture<>(new GridRestResponse(true));
+        }
+        catch (Throwable e) {
+            log.error("Failed to handle REST request [req=" + req + ']', e);
+
+            return new GridFinishedFuture<>(e);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/auth/package-info.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/auth/package-info.java
new file mode 100644
index 0000000000000..1d58218422ed1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/auth/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * <!-- Package description. -->
+ * REST authentication command.
+ */
+package org.apache.ignite.internal.processors.rest.handlers.auth;
diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
index b3fbddd93ff4b..99a884411cfd9 100644
--- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
+++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
@@ -85,6 +85,12 @@ public class GridJettyRestHandler extends AbstractHandler {
     /** Used to sent request charset. */
     private static final String CHARSET = StandardCharsets.UTF_8.name();
 
+    /** */
+    private static final String  USER_PARAM = "user";
+
+    /** */
+    private static final String  PWD_PARAM = "password";
+
     /** */
     private static final String  CACHE_NAME_PARAM = "cacheName";
 
@@ -100,10 +106,10 @@ public class GridJettyRestHandler extends AbstractHandler {
     /** */
     private static final String WRITE_SYNCHRONIZATION_MODE_PARAM = "writeSynchronizationMode";
 
-    /** */
+    /**@deprecated Should be replaced with AUTHENTICATION + token in IGNITE 3.0 */
     private static final String IGNITE_LOGIN = "ignite.login";
 
-    /** */
+    /**@deprecated Should be replaced with AUTHENTICATION + token in IGNITE 3.0 */
     private static final String IGNITE_PASSWORD = "ignite.password";
 
     /** */
@@ -731,6 +737,12 @@ private Object convert(String type, Object obj) throws IgniteCheckedException {
                 break;
             }
 
+            case AUTHENTICATE: {
+                restReq = new GridRestRequest();
+
+                break;
+            }
+
             case ADD_USER:
             case REMOVE_USER:
             case UPDATE_USER: {
@@ -840,12 +852,9 @@ private Object convert(String type, Object obj) throws IgniteCheckedException {
 
         restReq.command(cmd);
 
-        if (params.containsKey(IGNITE_LOGIN) || params.containsKey(IGNITE_PASSWORD)) {
-            SecurityCredentials cred = new SecurityCredentials(
-                (String)params.get(IGNITE_LOGIN), (String)params.get(IGNITE_PASSWORD));
-
-            restReq.credentials(cred);
-        }
+        // TODO: In IGNITE 3.0 we should check credentials only for AUTHENTICATE command.
+        if (!credentials(params, IGNITE_LOGIN, IGNITE_PASSWORD, restReq))
+            credentials(params, USER_PARAM, PWD_PARAM, restReq);
 
         String clientId = (String)params.get("clientId");
 
@@ -870,8 +879,13 @@ private Object convert(String type, Object obj) throws IgniteCheckedException {
         String sesTokStr = (String)params.get("sessionToken");
 
         try {
-            if (sesTokStr != null)
-                restReq.sessionToken(U.hexString2ByteArray(sesTokStr));
+            if (sesTokStr != null) {
+                // Token is a UUID encoded as 16 bytes as HEX.
+                byte[] bytes = U.hexString2ByteArray(sesTokStr);
+
+                if (bytes.length == 16)
+                    restReq.sessionToken(bytes);
+            }
         }
         catch (IllegalArgumentException ignored) {
             // Ignore invalid session token.
@@ -880,6 +894,27 @@ private Object convert(String type, Object obj) throws IgniteCheckedException {
         return restReq;
     }
 
+    /**
+     *
+     * @param params Parameters.
+     * @param userParam Parameter name to take user name.
+     * @param pwdParam Parameter name to take password.
+     * @param restReq Request to add credentials if any.
+     * @return {@code true} If params contains credentials.
+     */
+    private boolean credentials(Map<String, Object> params, String userParam, String pwdParam, GridRestRequest restReq) {
+        boolean hasCreds = params.containsKey(userParam) || params.containsKey(pwdParam);
+
+        if (hasCreds) {
+            SecurityCredentials cred = new SecurityCredentials((String)params.get(userParam),
+                (String)params.get(pwdParam));
+
+            restReq.credentials(cred);
+        }
+
+        return hasCreds;
+    }
+
     /**
      * Gets values referenced by sequential keys, e.g. {@code key1...keyN}.
      *

From 7457fd319a372d54de68271be7fddbb634cb6070 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Tue, 17 Apr 2018 10:30:52 +0300
Subject: [PATCH 058/543] IGNITE-8048 Store dynamic indexes to cache data on
 node join - Fixes #3719.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../org/apache/ignite/cache/QueryEntity.java  | 175 +++++-
 .../apache/ignite/cache/QueryEntityPatch.java | 118 ++++
 .../cache/CacheJoinNodeDiscoveryData.java     |  15 +-
 .../processors/cache/ClusterCachesInfo.java   | 428 ++++++++++----
 .../cache/DynamicCacheDescriptor.java         |  28 +
 .../processors/cache/GridCacheProcessor.java  |  94 +++-
 .../cluster/GridClusterStateProcessor.java    |   8 +-
 .../processors/query/QuerySchema.java         |  84 ++-
 .../processors/query/QuerySchemaPatch.java    |  96 ++++
 ...ActivateDeactivateTestWithPersistence.java |  18 +-
 .../cache/IgniteDynamicSqlRestoreTest.java    | 529 ++++++++++++++++++
 ...amicColumnsAbstractConcurrentSelfTest.java |   3 +-
 .../IgniteCacheQuerySelfTestSuite.java        |   2 +
 13 files changed, 1470 insertions(+), 128 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/cache/QueryEntityPatch.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/query/QuerySchemaPatch.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java b/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java
index 0065bae959d73..81fd50b8ee5fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/QueryEntity.java
@@ -26,9 +26,11 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.UUID;
 import javax.cache.CacheException;
 import org.apache.ignite.cache.query.annotations.QueryGroupIndex;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
@@ -36,12 +38,17 @@
 import org.apache.ignite.internal.processors.cache.query.QueryEntityClassProperty;
 import org.apache.ignite.internal.processors.cache.query.QueryEntityTypeDescriptor;
 import org.apache.ignite.internal.processors.query.GridQueryIndexDescriptor;
+import org.apache.ignite.internal.processors.query.QueryField;
 import org.apache.ignite.internal.processors.query.QueryUtils;
+import org.apache.ignite.internal.processors.query.schema.operation.SchemaAbstractOperation;
+import org.apache.ignite.internal.processors.query.schema.operation.SchemaAlterTableAddColumnOperation;
+import org.apache.ignite.internal.processors.query.schema.operation.SchemaIndexCreateOperation;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -143,6 +150,172 @@ public QueryEntity(Class<?> keyCls, Class<?> valCls) {
         this(convert(processKeyAndValueClasses(keyCls, valCls)));
     }
 
+    /**
+     * Make query entity patch. This patch can only add properties to entity and can't remove them.
+     * Other words, the patch will contain only add operations(e.g. add column, create index) and not remove ones.
+     *
+     * @param target Query entity to which this entity should be expanded.
+     * @return Patch which contains operations for expanding this entity.
+     */
+    @NotNull public QueryEntityPatch makePatch(QueryEntity target) {
+        if (target == null)
+            return QueryEntityPatch.empty();
+
+        StringBuilder conflicts = new StringBuilder();
+
+        checkEquals(conflicts, "keyType", keyType, target.keyType);
+        checkEquals(conflicts, "valType", valType, target.valType);
+        checkEquals(conflicts, "keyFieldName", keyFieldName, target.keyFieldName);
+        checkEquals(conflicts, "valueFieldName", valueFieldName, target.valueFieldName);
+        checkEquals(conflicts, "tableName", tableName, target.tableName);
+
+        List<QueryField> queryFieldsToAdd = checkFields(target, conflicts);
+
+        Collection<QueryIndex> indexesToAdd = checkIndexes(target, conflicts);
+
+        if (conflicts.length() != 0)
+            return QueryEntityPatch.conflict(tableName + " conflict: \n" + conflicts.toString());
+
+        Collection<SchemaAbstractOperation> patchOperations = new ArrayList<>();
+
+        if (!queryFieldsToAdd.isEmpty())
+            patchOperations.add(new SchemaAlterTableAddColumnOperation(
+                UUID.randomUUID(),
+                null,
+                null,
+                tableName,
+                queryFieldsToAdd,
+                true,
+                true
+            ));
+
+        if (!indexesToAdd.isEmpty()) {
+            for (QueryIndex index : indexesToAdd) {
+                patchOperations.add(new SchemaIndexCreateOperation(
+                    UUID.randomUUID(),
+                    null,
+                    null,
+                    tableName,
+                    index,
+                    true,
+                    0
+                ));
+            }
+        }
+
+        return QueryEntityPatch.patch(patchOperations);
+    }
+
+    /**
+     * Comparing local fields and target fields.
+     *
+     * @param target Query entity for check.
+     * @param conflicts Storage of conflicts.
+     * @return Indexes which exist in target and not exist in local.
+     */
+    @NotNull private Collection<QueryIndex> checkIndexes(QueryEntity target, StringBuilder conflicts) {
+        HashSet<QueryIndex> indexesToAdd = new HashSet<>();
+
+        Map<String, QueryIndex> currentIndexes = new HashMap<>();
+
+        for (QueryIndex index : getIndexes()) {
+            if (currentIndexes.put(index.getName(), index) != null)
+                throw new IllegalStateException("Duplicate key");
+        }
+
+        for (QueryIndex queryIndex : target.getIndexes()) {
+            if(currentIndexes.containsKey(queryIndex.getName())) {
+                checkEquals(
+                    conflicts,
+                    "index " + queryIndex.getName(),
+                    currentIndexes.get(queryIndex.getName()),
+                    queryIndex
+                );
+            }
+            else
+                indexesToAdd.add(queryIndex);
+        }
+        return indexesToAdd;
+    }
+
+    /**
+     * Comparing local entity fields and target entity fields.
+     *
+     * @param target Query entity for check.
+     * @param conflicts Storage of conflicts.
+     * @return Fields which exist in target and not exist in local.
+     */
+    private List<QueryField> checkFields(QueryEntity target, StringBuilder conflicts) {
+        List<QueryField> queryFieldsToAdd = new ArrayList<>();
+
+        for (Map.Entry<String, String> targetField : target.getFields().entrySet()) {
+            String targetFieldName = targetField.getKey();
+            String targetFieldType = targetField.getValue();
+
+            if (getFields().containsKey(targetFieldName)) {
+                checkEquals(
+                    conflicts,
+                    "fieldType of " + targetFieldName,
+                    getFields().get(targetFieldName),
+                    targetFieldType
+                );
+
+                checkEquals(
+                    conflicts,
+                    "nullable of " + targetFieldName,
+                    contains(getNotNullFields(), targetFieldName),
+                    contains(target.getNotNullFields(), targetFieldName)
+                );
+
+                checkEquals(
+                    conflicts,
+                    "default value of " + targetFieldName,
+                    getFromMap(getDefaultFieldValues(), targetFieldName),
+                    getFromMap(target.getDefaultFieldValues(), targetFieldName)
+                );
+            }
+            else {
+                queryFieldsToAdd.add(new QueryField(
+                    targetFieldName,
+                    targetFieldType,
+                    !contains(target.getNotNullFields(),targetFieldName),
+                    getFromMap(target.getDefaultFieldValues(), targetFieldName)
+                ));
+            }
+        }
+
+        return queryFieldsToAdd;
+    }
+
+    /**
+     * @param collection Collection for checking.
+     * @param elementToCheck Element for checking to containing in collection.
+     * @return {@code true} if collection contain elementToCheck.
+     */
+    private static boolean contains(Collection<String> collection, String elementToCheck) {
+        return collection != null && collection.contains(elementToCheck);
+    }
+
+    /**
+     * @return Value from sourceMap or null if map is null.
+     */
+    private static Object getFromMap(Map<String, Object> sourceMap, String key) {
+        return sourceMap == null ? null : sourceMap.get(key);
+    }
+
+    /**
+     * Comparing two objects and add formatted text to conflicts if needed.
+     *
+     * @param conflicts Storage of conflicts resulting error message.
+     * @param name Name of comparing object.
+     * @param local Local object.
+     * @param received Received object.
+     */
+    private void checkEquals(StringBuilder conflicts, String name, Object local, Object received) {
+        if (!Objects.equals(local, received))
+            conflicts.append(String.format("%s is different: local=%s, received=%s\n", name, local, received));
+    }
+
     /**
      * Gets key type for this query pair.
      *
@@ -310,7 +483,7 @@ public QueryEntity setValueFieldName(String valueFieldName) {
      *
      * @return Collection of index entities.
      */
-    public Collection<QueryIndex> getIndexes() {
+    @NotNull public Collection<QueryIndex> getIndexes() {
         return idxs == null ? Collections.<QueryIndex>emptyList() : idxs;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/cache/QueryEntityPatch.java b/modules/core/src/main/java/org/apache/ignite/cache/QueryEntityPatch.java
new file mode 100644
index 0000000000000..38e1b2acdef28
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/cache/QueryEntityPatch.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache;
+
+import java.util.Collection;
+import java.util.Objects;
+import org.apache.ignite.internal.processors.query.schema.operation.SchemaAbstractOperation;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Query entity patch which contain {@link SchemaAbstractOperation} operations for changing query entity.
+ * This patch can only add properties to entity and can't remove them.
+ * Other words, the patch will contain only add operations
+ * (e.g.:
+ * {@link org.apache.ignite.internal.processors.query.schema.operation.SchemaAlterTableAddColumnOperation},
+ * {@link org.apache.ignite.internal.processors.query.schema.operation.SchemaIndexCreateOperation}
+ * ) and not remove ones.
+ *
+ * It contain only add operation because at the moment we don't have history of schema operations
+ * and by current state we can't understand some property was already deleted or it has not been added yet.
+ */
+public class QueryEntityPatch {
+    /** Empty query entity patch. */
+    private static final QueryEntityPatch EMPTY_QUERY_ENTITY_PATCH = new QueryEntityPatch(null, null);
+
+    /** Message which described conflicts during creating this patch. */
+    private String conflictsMessage;
+
+    /** Operations for modification query entity. */
+    private Collection<SchemaAbstractOperation> patchOperations;
+
+    /**
+     * Create patch.
+     */
+    private QueryEntityPatch(String conflictsMessage, Collection<SchemaAbstractOperation> patchOperations) {
+        this.conflictsMessage = conflictsMessage;
+        this.patchOperations = patchOperations;
+    }
+
+    /**
+     * Builder method for patch with conflicts.
+     *
+     * @param conflicts Conflicts.
+     * @return Query entity patch with conflicts.
+     */
+    public static QueryEntityPatch conflict(String conflicts) {
+        return new QueryEntityPatch(conflicts, null);
+    }
+
+    /**
+     * Builder method for empty patch.
+     *
+     * @return Query entity patch.
+     */
+    public static QueryEntityPatch empty() {
+        return EMPTY_QUERY_ENTITY_PATCH;
+    }
+
+    /**
+     * Builder method for patch with operations.
+     *
+     * @param patchOperations Operations for modification.
+     * @return Query entity patch which contain {@link SchemaAbstractOperation} operations for changing query entity.
+     */
+    public static QueryEntityPatch patch(Collection<SchemaAbstractOperation> patchOperations) {
+        return new QueryEntityPatch(null, patchOperations);
+    }
+
+    /**
+     * Check for conflict in this patch.
+     *
+     * @return {@code true} if patch has conflict.
+     */
+    public boolean hasConflict() {
+        return conflictsMessage != null;
+    }
+
+    /**
+     * @return {@code true} if patch is empty and can't be applying.
+     */
+    public boolean isEmpty() {
+        return patchOperations == null || patchOperations.isEmpty();
+    }
+
+    /**
+     * @return Conflicts.
+     */
+    public String getConflictsMessage() {
+        return conflictsMessage;
+    }
+
+    /**
+     * @return Patch operations for applying.
+     */
+    public Collection<SchemaAbstractOperation> getPatchOperations() {
+        return patchOperations;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(QueryEntityPatch.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
index 6d2688c948adb..a3902de71b7ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
@@ -112,17 +112,23 @@ public static class CacheInfo implements Serializable {
         /** Flags added for future usage. */
         private final long flags;
 
+        /** Statically configured flag */
+        private final boolean staticallyConfigured;
+
         /**
          * @param cacheData Cache data.
          * @param cacheType Cache type.
          * @param sql SQL flag - {@code true} if cache was created with {@code CREATE TABLE}.
          * @param flags Flags (for future usage).
+         * @param staticallyConfigured {@code true} if it was configured by static config and {@code false} otherwise.
          */
-        public CacheInfo(StoredCacheData cacheData, CacheType cacheType, boolean sql, long flags) {
+        public CacheInfo(StoredCacheData cacheData, CacheType cacheType, boolean sql, long flags,
+            boolean staticallyConfigured) {
             this.cacheData = cacheData;
             this.cacheType = cacheType;
             this.sql = sql;
             this.flags = flags;
+            this.staticallyConfigured = staticallyConfigured;
         }
 
         /**
@@ -146,6 +152,13 @@ public boolean sql() {
             return sql;
         }
 
+        /**
+         * @return {@code true} if it was configured by static config and {@code false} otherwise.
+         */
+        public boolean isStaticallyConfigured() {
+            return staticallyConfigured;
+        }
+
         /** {@inheritDoc} */
         @Override public String toString() {
             return S.toString(CacheInfo.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 2b2fb559c182e..975617ee8dc0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -35,31 +35,35 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheExistsException;
+import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.GridCachePluginContext;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
 import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.processors.query.QuerySchema;
+import org.apache.ignite.internal.processors.query.QuerySchemaPatch;
 import org.apache.ignite.internal.processors.query.QueryUtils;
 import org.apache.ignite.internal.processors.query.schema.SchemaOperationException;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.lang.GridFunc;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.CachePluginContext;
 import org.apache.ignite.plugin.CachePluginProvider;
 import org.apache.ignite.plugin.PluginProvider;
 import org.apache.ignite.spi.discovery.DiscoveryDataBag;
-import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.cache.CacheMode.LOCAL;
@@ -71,6 +75,9 @@
  * Logic related to cache discovery data processing.
  */
 class ClusterCachesInfo {
+    /** Version since which merge of config is supports. */
+    private static final IgniteProductVersion V_MERGE_CONFIG_SINCE = IgniteProductVersion.fromString("2.5.0");
+
     /** */
     private final GridKernalContext ctx;
 
@@ -987,54 +994,77 @@ public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) {
         // CacheGroup configurations that were created from local node configuration.
         Map<Integer, CacheGroupDescriptor> locCacheGrps = new HashMap<>(registeredCacheGroups());
 
-        // Replace locally registered data with actual data received from cluster.
-        registeredCaches.clear();
-        registeredCacheGrps.clear();
-        ctx.discovery().cleanCachesAndGroups();
+        //Replace locally registered data with actual data received from cluster.
+        cleanCachesAndGroups();
 
-        for (CacheGroupData grpData : cachesData.cacheGroups().values()) {
-            CacheGroupDescriptor grpDesc = new CacheGroupDescriptor(
-                grpData.config(),
-                grpData.groupName(),
-                grpData.groupId(),
-                grpData.receivedFrom(),
-                grpData.startTopologyVersion(),
-                grpData.deploymentId(),
-                grpData.caches(),
-                grpData.persistenceEnabled(),
-                grpData.walEnabled(),
-                grpData.walChangeRequests());
+        registerReceivedCacheGroups(cachesData, locCacheGrps);
 
-            if (locCacheGrps.containsKey(grpDesc.groupId())) {
-                CacheGroupDescriptor locGrpCfg = locCacheGrps.get(grpDesc.groupId());
+        registerReceivedCacheTemplates(cachesData);
 
-                grpDesc.mergeWith(locGrpCfg);
-            }
+        registerReceivedCaches(cachesData);
 
-            CacheGroupDescriptor old = registeredCacheGrps.put(grpDesc.groupId(), grpDesc);
+        addReceivedClientNodesToDiscovery(cachesData);
 
-            assert old == null : old;
+        String conflictErr = validateRegisteredCaches();
 
-            ctx.discovery().addCacheGroup(grpDesc,
-                grpData.config().getNodeFilter(),
-                grpData.config().getCacheMode());
+        gridData = new GridData(joinDiscoData, cachesData, conflictErr);
+
+        if (cachesOnDisconnect == null || cachesOnDisconnect.clusterActive())
+            initStartCachesForLocalJoin(false, disconnectedState());
+    }
+
+    /**
+     * Validation {@link #registeredCaches} on conflicts.
+     *
+     * @return Error message if conflicts was found.
+     */
+    @Nullable private String validateRegisteredCaches() {
+        String conflictErr = null;
+
+        if (joinDiscoData != null) {
+            for (Map.Entry<String, CacheJoinNodeDiscoveryData.CacheInfo> e : joinDiscoData.caches().entrySet()) {
+                if (!registeredCaches.containsKey(e.getKey())) {
+                    conflictErr = checkCacheConflict(e.getValue().cacheData().config());
+
+                    if (conflictErr != null) {
+                        conflictErr = "Failed to start configured cache due to conflict with started caches. " +
+                            conflictErr;
+
+                        break;
+                    }
+                }
+            }
         }
 
-        for (CacheData cacheData : cachesData.templates().values()) {
-            DynamicCacheDescriptor desc = new DynamicCacheDescriptor(
-                ctx,
-                cacheData.cacheConfiguration(),
-                cacheData.cacheType(),
-                null,
-                true,
-                cacheData.receivedFrom(),
-                cacheData.staticallyConfigured(),
-                false,
-                cacheData.deploymentId(),
-                cacheData.schema());
+        return conflictErr;
+    }
 
-            registeredTemplates.put(cacheData.cacheConfiguration().getName(), desc);
+    /**
+     * Adding received client nodes to discovery if needed.
+     *
+     * @param cachesData Data received from cluster.
+     */
+    private void addReceivedClientNodesToDiscovery(CacheNodeCommonDiscoveryData cachesData) {
+        if (!F.isEmpty(cachesData.clientNodesMap())) {
+            for (Map.Entry<String, Map<UUID, Boolean>> entry : cachesData.clientNodesMap().entrySet()) {
+                String cacheName = entry.getKey();
+
+                for (Map.Entry<UUID, Boolean> tup : entry.getValue().entrySet())
+                    ctx.discovery().addClientNode(cacheName, tup.getKey(), tup.getValue());
+            }
         }
+    }
+
+    /**
+     * Register caches received from cluster.
+     *
+     * @param cachesData Data received from cluster.
+     */
+    private void registerReceivedCaches(CacheNodeCommonDiscoveryData cachesData) {
+        Map<DynamicCacheDescriptor, QuerySchemaPatch> patchesToApply = new HashMap<>();
+        Collection<DynamicCacheDescriptor> cachesToSave = new HashSet<>();
+
+        boolean hasSchemaPatchConflict = false;
 
         for (CacheData cacheData : cachesData.caches().values()) {
             CacheGroupDescriptor grpDesc = registeredCacheGrps.get(cacheData.groupId());
@@ -1053,7 +1083,22 @@ public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) {
                 cacheData.staticallyConfigured(),
                 cacheData.sql(),
                 cacheData.deploymentId(),
-                cacheData.schema());
+                new QuerySchema(cacheData.schema().entities())
+            );
+
+            Collection<QueryEntity> localQueryEntities = getLocalQueryEntities(cfg.getName());
+
+            QuerySchemaPatch schemaPatch = desc.makeSchemaPatch(localQueryEntities);
+
+            if (schemaPatch.hasConflicts()) {
+                hasSchemaPatchConflict = true;
+
+                log.warning("Skipping apply patch because conflicts : " + schemaPatch.getConflictsMessage());
+            }
+            else if (!schemaPatch.isEmpty())
+                patchesToApply.put(desc, schemaPatch);
+            else if (!GridFunc.eqNotOrdered(desc.schema().entities(), localQueryEntities))
+                cachesToSave.add(desc); //received config is different of local config - need to resave
 
             desc.receivedOnDiscovery(true);
 
@@ -1066,36 +1111,140 @@ public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) {
                 cfg.getNearConfiguration() != null);
         }
 
-        if (!F.isEmpty(cachesData.clientNodesMap())) {
-            for (Map.Entry<String, Map<UUID, Boolean>> entry : cachesData.clientNodesMap().entrySet()) {
-                String cacheName = entry.getKey();
+        updateRegisteredCachesIfNeeded(patchesToApply, cachesToSave, hasSchemaPatchConflict);
+    }
 
-                for (Map.Entry<UUID, Boolean> tup : entry.getValue().entrySet())
-                    ctx.discovery().addClientNode(cacheName, tup.getKey(), tup.getValue());
+    /**
+     * Merging config or resaving it if it needed.
+     *
+     * @param patchesToApply Patches which need to apply.
+     * @param cachesToSave Caches which need to resave.
+     * @param hasSchemaPatchConflict {@code true} if we have conflict during making patch.
+     */
+    private void updateRegisteredCachesIfNeeded(Map<DynamicCacheDescriptor, QuerySchemaPatch> patchesToApply,
+        Collection<DynamicCacheDescriptor> cachesToSave, boolean hasSchemaPatchConflict) {
+        //Skip merge of config if least one conflict was found.
+        if (!hasSchemaPatchConflict && isMergeConfigSupports(ctx.discovery().localNode())) {
+            boolean isClusterActive = ctx.state().clusterState().active();
+
+            //Merge of config for cluster only for inactive grid.
+            if (!isClusterActive && !patchesToApply.isEmpty()) {
+                for (Map.Entry<DynamicCacheDescriptor, QuerySchemaPatch> entry : patchesToApply.entrySet()) {
+                    if (entry.getKey().applySchemaPatch(entry.getValue()))
+                        saveCacheConfiguration(entry.getKey());
+                }
+
+                for (DynamicCacheDescriptor descriptor : cachesToSave) {
+                    saveCacheConfiguration(descriptor);
+                }
+            }
+            else if (patchesToApply.isEmpty()) {
+                for (DynamicCacheDescriptor descriptor : cachesToSave) {
+                    saveCacheConfiguration(descriptor);
+                }
             }
         }
+    }
 
-        String conflictErr = null;
+    /**
+     * Register cache templates received from cluster.
+     *
+     * @param cachesData Data received from cluster.
+     */
+    private void registerReceivedCacheTemplates(CacheNodeCommonDiscoveryData cachesData) {
+        for (CacheData cacheData : cachesData.templates().values()) {
+            DynamicCacheDescriptor desc = new DynamicCacheDescriptor(
+                ctx,
+                cacheData.cacheConfiguration(),
+                cacheData.cacheType(),
+                null,
+                true,
+                cacheData.receivedFrom(),
+                cacheData.staticallyConfigured(),
+                false,
+                cacheData.deploymentId(),
+                cacheData.schema());
 
-        if (joinDiscoData != null) {
-            for (Map.Entry<String, CacheJoinNodeDiscoveryData.CacheInfo> e : joinDiscoData.caches().entrySet()) {
-                if (!registeredCaches.containsKey(e.getKey())) {
-                    conflictErr = checkCacheConflict(e.getValue().cacheData().config());
+            registeredTemplates.put(cacheData.cacheConfiguration().getName(), desc);
+        }
+    }
 
-                    if (conflictErr != null) {
-                        conflictErr = "Failed to start configured cache due to conflict with started caches. " +
-                            conflictErr;
+    /**
+     * Register cache groups received from cluster.
+     *
+     * @param cachesData Data received from cluster.
+     * @param locCacheGrps Current local cache groups.
+     */
+    private void registerReceivedCacheGroups(CacheNodeCommonDiscoveryData cachesData,
+        Map<Integer, CacheGroupDescriptor> locCacheGrps) {
+        for (CacheGroupData grpData : cachesData.cacheGroups().values()) {
+            CacheGroupDescriptor grpDesc = new CacheGroupDescriptor(
+                grpData.config(),
+                grpData.groupName(),
+                grpData.groupId(),
+                grpData.receivedFrom(),
+                grpData.startTopologyVersion(),
+                grpData.deploymentId(),
+                grpData.caches(),
+                grpData.persistenceEnabled(),
+                grpData.walEnabled(),
+                grpData.walChangeRequests());
 
-                        break;
-                    }
-                }
+            if (locCacheGrps.containsKey(grpDesc.groupId())) {
+                CacheGroupDescriptor locGrpCfg = locCacheGrps.get(grpDesc.groupId());
+
+                grpDesc.mergeWith(locGrpCfg);
             }
+
+            CacheGroupDescriptor old = registeredCacheGrps.put(grpDesc.groupId(), grpDesc);
+
+            assert old == null : old;
+
+            ctx.discovery().addCacheGroup(grpDesc,
+                grpData.config().getNodeFilter(),
+                grpData.config().getCacheMode());
         }
+    }
 
-        gridData = new GridData(joinDiscoData, cachesData, conflictErr);
+    /**
+     * Clean local registered caches and groups
+     */
+    private void cleanCachesAndGroups() {
+        registeredCaches.clear();
+        registeredCacheGrps.clear();
+        ctx.discovery().cleanCachesAndGroups();
+    }
 
-        if (cachesOnDisconnect == null || cachesOnDisconnect.clusterActive())
-            initStartCachesForLocalJoin(false, disconnectedState());
+    /**
+     * Save dynamic cache descriptor on disk.
+     *
+     * @param desc Cache to save.
+     */
+    private void saveCacheConfiguration(DynamicCacheDescriptor desc) {
+        try {
+            ctx.cache().saveCacheConfiguration(desc);
+        }
+        catch (IgniteCheckedException e) {
+            log.error("Error while saving cache configuration to disk, cfg = " + desc.cacheConfiguration(), e);
+        }
+    }
+
+    /**
+     * Get started node query entities by cacheName.
+     *
+     * @param cacheName Cache for which query entities will be returned.
+     * @return Local query entities.
+     */
+    private Collection<QueryEntity> getLocalQueryEntities(String cacheName) {
+        if (joinDiscoData == null)
+            return Collections.emptyList();
+
+        CacheJoinNodeDiscoveryData.CacheInfo cacheInfo = joinDiscoData.caches().get(cacheName);
+
+        if (cacheInfo == null)
+            return Collections.emptyList();
+
+        return cacheInfo.cacheData().queryEntities();
     }
 
     /**
@@ -1144,7 +1293,7 @@ private void initStartCachesForLocalJoin(boolean firstNode, boolean reconnect) {
                         desc.staticallyConfigured(),
                         desc.sql(),
                         desc.deploymentId(),
-                        new QuerySchema(locCfg.cacheData().queryEntities()));
+                        desc.schema().copy());
 
                     desc0.startTopologyVersion(desc.startTopologyVersion());
                     desc0.receivedFromStartVersion(desc.receivedFromStartVersion());
@@ -1385,26 +1534,14 @@ private String checkCacheConflict(CacheConfiguration<?, ?> cfg) {
      * @return Configuration conflict error.
      */
     private String processJoiningNode(CacheJoinNodeDiscoveryData joinData, UUID nodeId, boolean locJoin) {
-        for (CacheJoinNodeDiscoveryData.CacheInfo cacheInfo : joinData.templates().values()) {
-            CacheConfiguration<?, ?> cfg = cacheInfo.cacheData().config();
+        registerNewCacheTemplates(joinData, nodeId);
 
-            if (!registeredTemplates.containsKey(cfg.getName())) {
-                DynamicCacheDescriptor desc = new DynamicCacheDescriptor(ctx,
-                    cfg,
-                    cacheInfo.cacheType(),
-                    null,
-                    true,
-                    nodeId,
-                    true,
-                    false,
-                    joinData.cacheDeploymentId(),
-                    new QuerySchema(cacheInfo.cacheData().queryEntities()));
+        Map<DynamicCacheDescriptor, QuerySchemaPatch> patchesToApply = new HashMap<>();
 
-                DynamicCacheDescriptor old = registeredTemplates.put(cfg.getName(), desc);
+        boolean hasSchemaPatchConflict = false;
+        boolean active = ctx.state().clusterState().active();
 
-                assert old == null : old;
-            }
-        }
+        boolean isMergeConfigSupport = isMergeConfigSupports(null);
 
         for (CacheJoinNodeDiscoveryData.CacheInfo cacheInfo : joinData.caches().values()) {
             CacheConfiguration<?, ?> cfg = cacheInfo.cacheData().config();
@@ -1421,49 +1558,138 @@ private String processJoiningNode(CacheJoinNodeDiscoveryData joinData, UUID node
                     continue;
                 }
 
-                int cacheId = CU.cacheId(cfg.getName());
+                registerNewCache(joinData, nodeId, cacheInfo);
+            }
+            else if (!active && isMergeConfigSupport) {
+                DynamicCacheDescriptor desc = registeredCaches.get(cfg.getName());
+
+                QuerySchemaPatch schemaPatch = desc.makeSchemaPatch(cacheInfo.cacheData().queryEntities());
 
-                CacheGroupDescriptor grpDesc = registerCacheGroup(null,
-                    null,
-                    cfg,
-                    cacheId,
+                if (schemaPatch.hasConflicts()) {
+                    hasSchemaPatchConflict = true;
+
+                    log.error("Error during making schema patch : " + schemaPatch.getConflictsMessage());
+                }
+                else if (!schemaPatch.isEmpty() && !hasSchemaPatchConflict)
+                    patchesToApply.put(desc, schemaPatch);
+            }
+
+            ctx.discovery().addClientNode(cfg.getName(), nodeId, cfg.getNearConfiguration() != null);
+        }
+
+        //If conflict was detected we don't merge config and we leave existed config.
+        if (!hasSchemaPatchConflict && !patchesToApply.isEmpty())
+            for(Map.Entry<DynamicCacheDescriptor, QuerySchemaPatch> entry: patchesToApply.entrySet()){
+                if (entry.getKey().applySchemaPatch(entry.getValue()))
+                    saveCacheConfiguration(entry.getKey());
+            }
+
+        if (joinData.startCaches()) {
+            for (DynamicCacheDescriptor desc : registeredCaches.values()) {
+                ctx.discovery().addClientNode(desc.cacheName(),
                     nodeId,
-                    joinData.cacheDeploymentId());
+                    desc.cacheConfiguration().getNearConfiguration() != null);
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * Register new cache received from joining node.
+     *
+     * @param joinData Data from joining node.
+     * @param nodeId Joining node id.
+     * @param cacheInfo Cache info of new node.
+     */
+    private void registerNewCache(
+        CacheJoinNodeDiscoveryData joinData,
+        UUID nodeId,
+        CacheJoinNodeDiscoveryData.CacheInfo cacheInfo) {
+        CacheConfiguration<?, ?> cfg = cacheInfo.cacheData().config();
+
+        int cacheId = CU.cacheId(cfg.getName());
+
+        CacheGroupDescriptor grpDesc = registerCacheGroup(null,
+            null,
+            cfg,
+            cacheId,
+            nodeId,
+            joinData.cacheDeploymentId());
+
+        ctx.discovery().setCacheFilter(
+            cacheId,
+            grpDesc.groupId(),
+            cfg.getName(),
+            cfg.getNearConfiguration() != null);
+
+        DynamicCacheDescriptor desc = new DynamicCacheDescriptor(ctx,
+            cfg,
+            cacheInfo.cacheType(),
+            grpDesc,
+            false,
+            nodeId,
+            cacheInfo.isStaticallyConfigured(),
+            cacheInfo.sql(),
+            joinData.cacheDeploymentId(),
+            new QuerySchema(cacheInfo.cacheData().queryEntities()));
+
+        DynamicCacheDescriptor old = registeredCaches.put(cfg.getName(), desc);
 
-                ctx.discovery().setCacheFilter(
-                    cacheId,
-                    grpDesc.groupId(),
-                    cfg.getName(),
-                    cfg.getNearConfiguration() != null);
+        assert old == null : old;
+    }
+
+    /**
+     * Register new cache templates received from joining node.
+     *
+     * @param joinData Data from joining node.
+     * @param nodeId Joining node id.
+     */
+    private void registerNewCacheTemplates(CacheJoinNodeDiscoveryData joinData, UUID nodeId) {
+        for (CacheJoinNodeDiscoveryData.CacheInfo cacheInfo : joinData.templates().values()) {
+            CacheConfiguration<?, ?> cfg = cacheInfo.cacheData().config();
 
+            if (!registeredTemplates.containsKey(cfg.getName())) {
                 DynamicCacheDescriptor desc = new DynamicCacheDescriptor(ctx,
                     cfg,
                     cacheInfo.cacheType(),
-                    grpDesc,
-                    false,
+                    null,
+                    true,
                     nodeId,
                     true,
-                    cacheInfo.sql(),
+                    false,
                     joinData.cacheDeploymentId(),
                     new QuerySchema(cacheInfo.cacheData().queryEntities()));
 
-                DynamicCacheDescriptor old = registeredCaches.put(cfg.getName(), desc);
+                DynamicCacheDescriptor old = registeredTemplates.put(cfg.getName(), desc);
 
                 assert old == null : old;
             }
-
-            ctx.discovery().addClientNode(cfg.getName(), nodeId, cfg.getNearConfiguration() != null);
         }
+    }
 
-        if (joinData.startCaches()) {
-            for (DynamicCacheDescriptor desc : registeredCaches.values()) {
-                ctx.discovery().addClientNode(desc.cacheName(),
-                    nodeId,
-                    desc.cacheConfiguration().getNearConfiguration() != null);
-            }
+    /**
+     * @return {@code true} if grid supports merge of config and {@code False} otherwise.
+     */
+    public boolean isMergeConfigSupports(ClusterNode joiningNode) {
+        DiscoCache discoCache = ctx.discovery().discoCache();
+
+        if (discoCache == null)
+            return true;
+
+        if (joiningNode != null && joiningNode.version().compareToIgnoreTimestamp(V_MERGE_CONFIG_SINCE) < 0)
+            return false;
+
+        Collection<ClusterNode> nodes = discoCache.allNodes();
+
+        for (ClusterNode node : nodes) {
+            IgniteProductVersion version = node.version();
+
+            if (version.compareToIgnoreTimestamp(V_MERGE_CONFIG_SINCE) < 0)
+                return false;
         }
 
-        return null;
+        return true;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
index cad84144b519f..93882a253bf65 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheDescriptor.java
@@ -17,14 +17,17 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.Collection;
 import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.query.QuerySchema;
+import org.apache.ignite.internal.processors.query.QuerySchemaPatch;
 import org.apache.ignite.internal.processors.query.schema.message.SchemaFinishDiscoveryMessage;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -345,6 +348,31 @@ public void schemaChangeFinish(SchemaFinishDiscoveryMessage msg) {
         }
     }
 
+    /**
+     * Make schema patch for this cache.
+     *
+     * @param target Query entity list which current schema should be expanded to.
+     * @return Patch which contains operations for expanding schema of this cache.
+     * @see QuerySchemaPatch
+     */
+    public QuerySchemaPatch makeSchemaPatch(Collection<QueryEntity> target) {
+        synchronized (schemaMux) {
+            return schema.makePatch(target);
+        }
+    }
+
+    /**
+     * Apply query schema patch for changing current schema.
+     *
+     * @param patch patch to apply.
+     * @return {@code true} if applying was success and {@code false} otherwise.
+     */
+    public boolean applySchemaPatch(QuerySchemaPatch patch) {
+        synchronized (schemaMux) {
+            return schema.applyPatch(patch);
+        }
+    }
+
     /**
      * Form a {@link StoredCacheData} with all data to correctly restore cache params when its configuration is read
      * from page store. Essentially, this method takes from {@link DynamicCacheDescriptor} all that's needed to start
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 3aa6603a47d96..36edd72dd952f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -117,6 +117,7 @@
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
 import org.apache.ignite.internal.processors.plugin.CachePluginManager;
 import org.apache.ignite.internal.processors.query.QuerySchema;
+import org.apache.ignite.internal.processors.query.QuerySchemaPatch;
 import org.apache.ignite.internal.processors.query.QueryUtils;
 import org.apache.ignite.internal.processors.query.schema.SchemaExchangeWorkerTask;
 import org.apache.ignite.internal.processors.query.schema.SchemaNodeLeaveExchangeWorkerTask;
@@ -183,6 +184,14 @@
  */
 @SuppressWarnings({"unchecked", "TypeMayBeWeakened", "deprecation"})
 public class GridCacheProcessor extends GridProcessorAdapter {
+    /** Template of message of conflicts during configuration merge*/
+    private static final String MERGE_OF_CONFIG_CONFLICTS_MESSAGE =
+        "Conflicts during configuration merge for cache '%s' : \n%s";
+
+    /** Template of message of node join was fail because it requires to merge of config */
+    private static final String MERGE_OF_CONFIG_REQUIRED_MESSAGE = "Failed to join node to the active cluster " +
+        "(the config of the cache '%s' has to be merged which is impossible on active grid). " +
+        "Deactivate grid and retry node join or clean the joining node.";
     /** */
     private final boolean startClientCaches =
         IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_START_CACHES_ON_JOIN, false);
@@ -742,15 +751,29 @@ private void addCacheOnJoin(CacheConfiguration<?, ?> cfg, boolean sql,
             if (cacheType != CacheType.USER && cfg.getDataRegionName() == null)
                 cfg.setDataRegionName(sharedCtx.database().systemDateRegionName());
 
-            if (!cacheType.userCache())
-                stopSeq.addLast(cacheName);
-            else
-                stopSeq.addFirst(cacheName);
-
-            caches.put(cacheName, new CacheJoinNodeDiscoveryData.CacheInfo(cacheData, cacheType, cacheData.sql(), 0));
+            addStoredCache(caches, cacheData, cacheName, cacheType, true);
         }
         else
-            templates.put(cacheName, new CacheJoinNodeDiscoveryData.CacheInfo(cacheData, CacheType.USER, false, 0));
+            templates.put(cacheName, new CacheInfo(cacheData, CacheType.USER, false, 0, true));
+    }
+
+    /**
+     * Add stored cache data to caches storage.
+     *
+     * @param caches Cache storage.
+     * @param cacheData Cache data to add.
+     * @param cacheName Cache name.
+     * @param cacheType Cache type.
+     * @param isStaticalyConfigured Statically configured flag.
+     */
+    private void addStoredCache(Map<String, CacheInfo> caches, StoredCacheData cacheData, String cacheName,
+        CacheType cacheType, boolean isStaticalyConfigured) {
+        if (!cacheType.userCache())
+            stopSeq.addLast(cacheName);
+        else
+            stopSeq.addFirst(cacheName);
+
+        caches.put(cacheName, new CacheInfo(cacheData, cacheType, cacheData.sql(), 0, isStaticalyConfigured));
     }
 
     /**
@@ -774,6 +797,19 @@ private void addCacheOnJoinFromConfig(
 
             addCacheOnJoin(cfg, false, caches, templates);
         }
+
+        if (CU.isPersistenceEnabled(ctx.config()) && ctx.cache().context().pageStore() != null) {
+            Map<String, StoredCacheData> storedCaches = ctx.cache().context().pageStore().readCacheConfigurations();
+
+            if (!F.isEmpty(storedCaches))
+                for (StoredCacheData storedCacheData : storedCaches.values()) {
+                    String cacheName = storedCacheData.config().getName();
+
+                    //Ignore stored caches if it already added by static config(static config has higher priority).
+                    if (!caches.containsKey(cacheName))
+                        addStoredCache(caches, storedCacheData, cacheName, cacheType(cacheName), false);
+                }
+        }
     }
 
     /**
@@ -2439,6 +2475,50 @@ private GridCacheSharedContext createSharedContext(GridKernalContext kernalCtx,
         sharedCtx.walState().onCachesInfoCollected();
     }
 
+    /** {@inheritDoc} */
+    @Nullable @Override public IgniteNodeValidationResult validateNode(
+        ClusterNode node, JoiningNodeDiscoveryData discoData
+    ) {
+        if(!cachesInfo.isMergeConfigSupports(node))
+            return null;
+
+        if (discoData.hasJoiningNodeData() && discoData.joiningNodeData() instanceof CacheJoinNodeDiscoveryData) {
+            CacheJoinNodeDiscoveryData nodeData = (CacheJoinNodeDiscoveryData)discoData.joiningNodeData();
+
+            boolean isGridActive = ctx.state().clusterState().active();
+
+            StringBuilder errorMessage = new StringBuilder();
+
+            for (CacheJoinNodeDiscoveryData.CacheInfo cacheInfo : nodeData.caches().values()) {
+                DynamicCacheDescriptor localDesc = cacheDescriptor(cacheInfo.cacheData().config().getName());
+
+                if (localDesc == null)
+                    continue;
+
+                QuerySchemaPatch schemaPatch = localDesc.makeSchemaPatch(cacheInfo.cacheData().queryEntities());
+
+                if (schemaPatch.hasConflicts() || (isGridActive && !schemaPatch.isEmpty())) {
+                    if (errorMessage.length() > 0)
+                        errorMessage.append("\n");
+
+                    if (schemaPatch.hasConflicts())
+                        errorMessage.append(String.format(MERGE_OF_CONFIG_CONFLICTS_MESSAGE,
+                            localDesc.cacheName(), schemaPatch.getConflictsMessage()));
+                    else
+                        errorMessage.append(String.format(MERGE_OF_CONFIG_REQUIRED_MESSAGE, localDesc.cacheName()));
+                }
+            }
+
+            if (errorMessage.length() > 0) {
+                String msg = errorMessage.toString();
+
+                return new IgniteNodeValidationResult(node.id(), msg, msg);
+            }
+        }
+
+        return null;
+    }
+
     /**
      * @param msg Message.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 81a5b4e5c2f3c..2700a20f46f20 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -1035,7 +1035,13 @@ private boolean sendComputeCheckGlobalState() {
                 ", client=" + ctx.clientNode() +
                 ", daemon" + ctx.isDaemon() + "]");
         }
-        IgniteCompute comp = ((ClusterGroupAdapter)ctx.cluster().get().forServers()).compute();
+
+        ClusterGroupAdapter clusterGroupAdapter = (ClusterGroupAdapter)ctx.cluster().get().forServers();
+
+        if (F.isEmpty(clusterGroupAdapter.nodes()))
+            return false;
+
+        IgniteCompute comp = clusterGroupAdapter.compute();
 
         return comp.call(new IgniteCallable<Boolean>() {
             @IgniteInstanceResource
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QuerySchema.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QuerySchema.java
index 5cbae29d01a6d..569a02e0b2cbc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QuerySchema.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QuerySchema.java
@@ -20,12 +20,14 @@
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
-
 import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryEntityPatch;
 import org.apache.ignite.cache.QueryIndex;
 import org.apache.ignite.internal.processors.query.schema.message.SchemaFinishDiscoveryMessage;
 import org.apache.ignite.internal.processors.query.schema.operation.SchemaAbstractOperation;
@@ -84,15 +86,91 @@ public QuerySchema copy() {
         }
     }
 
+    /**
+     * Make query schema patch.
+     *
+     * @param target Query entity list to which current schema should be expanded.
+     * @return Patch to achieve entity which is a result of merging current one and target.
+     * @see QuerySchemaPatch
+     */
+    public QuerySchemaPatch makePatch(Collection<QueryEntity> target) {
+        synchronized (mux) {
+            Map<String, QueryEntity> localEntities = new HashMap<>();
+
+            for (QueryEntity entity : entities) {
+                if (localEntities.put(entity.getTableName(), entity) != null)
+                    throw new IllegalStateException("Duplicate key");
+            }
+
+            Collection<SchemaAbstractOperation> patchOperations = new ArrayList<>();
+            Collection<QueryEntity> entityToAdd = new ArrayList<>();
+
+            StringBuilder conflicts = new StringBuilder();
+
+            for (QueryEntity queryEntity : target) {
+                if (localEntities.containsKey(queryEntity.getTableName())) {
+                    QueryEntity localEntity = localEntities.get(queryEntity.getTableName());
+
+                    QueryEntityPatch entityPatch = localEntity.makePatch(queryEntity);
+
+                    if (entityPatch.hasConflict()) {
+                        if (conflicts.length() > 0)
+                            conflicts.append("\n");
+
+                        conflicts.append(entityPatch.getConflictsMessage());
+                    }
+
+                    if (!entityPatch.isEmpty())
+                        patchOperations.addAll(entityPatch.getPatchOperations());
+                }
+                else
+                    entityToAdd.add(QueryUtils.copy(queryEntity));
+            }
+
+            return new QuerySchemaPatch(patchOperations, entityToAdd, conflicts.toString());
+        }
+    }
+
+    /**
+     * Apply query schema patch for changing this schema.
+     *
+     * @param patch Patch to apply.
+     * @return {@code true} if applying was success and {@code false} otherwise.
+     */
+    public boolean applyPatch(QuerySchemaPatch patch) {
+        synchronized (mux) {
+            if (patch.hasConflicts())
+                return false;
+
+            if (patch.isEmpty())
+                return true;
+
+            for (SchemaAbstractOperation operation : patch.getPatchOperations()) {
+                finish(operation);
+            }
+
+            entities.addAll(patch.getEntityToAdd());
+
+            return true;
+        }
+    }
+
     /**
      * Process finish message.
      *
      * @param msg Message.
      */
     public void finish(SchemaFinishDiscoveryMessage msg) {
-        synchronized (mux) {
-            SchemaAbstractOperation op = msg.operation();
+        finish(msg.operation());
+    }
 
+    /**
+     * Process operation.
+     *
+     * @param op Operation for handle.
+     */
+    public void finish(SchemaAbstractOperation op) {
+        synchronized (mux) {
             if (op instanceof SchemaIndexCreateOperation) {
                 SchemaIndexCreateOperation op0 = (SchemaIndexCreateOperation)op;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QuerySchemaPatch.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QuerySchemaPatch.java
new file mode 100644
index 0000000000000..68beb049da9ee
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QuerySchemaPatch.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import java.util.Collection;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryEntityPatch;
+import org.apache.ignite.internal.processors.query.schema.operation.SchemaAbstractOperation;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Query schema patch which contains {@link SchemaAbstractOperation} operations for changing query entities.
+ * This patch is high level path on {@link org.apache.ignite.cache.QueryEntityPatch} but
+ * it has operations for all {@link QueryEntity} in schema
+ * and also contains {@link QueryEntity} for adding to schema by whole.
+ *
+ * @see org.apache.ignite.cache.QueryEntityPatch
+ */
+public class QuerySchemaPatch {
+    /** Message which described conflicts during creating this patch. */
+    private String conflictsMessage;
+
+    /** Operations for modification query entity. */
+    private Collection<SchemaAbstractOperation> patchOperations;
+
+    /** Entities which should be added by whole. */
+    private Collection<QueryEntity> entityToAdd;
+
+    /**
+     * Create patch.
+     */
+    public QuerySchemaPatch(
+        @NotNull Collection<SchemaAbstractOperation> patchOperations,
+        @NotNull Collection<QueryEntity> entityToAdd,
+        String conflictsMessage) {
+        this.patchOperations = patchOperations;
+        this.entityToAdd = entityToAdd;
+        this.conflictsMessage = conflictsMessage;
+    }
+
+    /**
+     * @return {@code true} if patch has conflict.
+     */
+    public boolean hasConflicts() {
+        return conflictsMessage != null && !conflictsMessage.isEmpty();
+    }
+
+    /**
+     * @return Conflicts message.
+     */
+    public String getConflictsMessage() {
+        return conflictsMessage;
+    }
+
+    /**
+     * @return {@code true} if patch is empty and can't be applying.
+     */
+    public boolean isEmpty() {
+        return patchOperations.isEmpty() && entityToAdd.isEmpty();
+    }
+
+    /**
+     * @return Patch operations for applying.
+     */
+    @NotNull public Collection<SchemaAbstractOperation> getPatchOperations() {
+        return patchOperations;
+    }
+
+    /**
+     * @return Entities which should be added by whole.
+     */
+    @NotNull public Collection<QueryEntity> getEntityToAdd() {
+        return entityToAdd;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(QuerySchemaPatch.class, this);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
index 58511ee306fdd..8bae136107ca0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
@@ -23,12 +23,13 @@
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.testframework.GridTestUtils;
 
@@ -253,22 +254,15 @@ public void testActivateCacheRestoreConfigurationConflict() throws Exception {
 
         ccfg.setGroupName(DEFAULT_CACHE_NAME);
 
-        ccfgs = new CacheConfiguration[]{ccfg};
-
-        startGrids(SRVS);
+        ccfgs = new CacheConfiguration[] {ccfg};
 
         try {
-            ignite(0).active(true);
+            startGrids(SRVS);
 
             fail();
         }
-        catch (IgniteException e) {
-            // Expected error.
+        catch (IgniteCheckedException e) {
+            assertTrue(X.getCause(e).getMessage().contains("Failed to start configured cache."));
         }
-
-        for (int i = 0; i < SRVS; i++)
-            assertFalse(ignite(i).active());
-
-        checkNoCaches(SRVS);
     }
 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java
new file mode 100644
index 0000000000000..f7dc7b41ba6de
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicSqlRestoreTest.java
@@ -0,0 +1,529 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+/**
+ *
+ */
+public class IgniteDynamicSqlRestoreTest extends GridCommonAbstractTest implements Serializable {
+
+    public static final String TEST_CACHE_NAME = "test";
+    public static final String TEST_INDEX_OBJECT = "TestIndexObject";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setAutoActivationEnabled(false);
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration().setMaxSize(200 * 1024 * 1024).setPersistenceEnabled(true))
+            .setWalMode(WALMode.LOG_ONLY);
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        cfg.setConsistentId(gridName);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testMergeChangedConfigOnCoordinator() throws Exception {
+        {
+            //given: two started nodes with test table
+            Ignite ig = startGrid(0);
+            startGrid(1);
+
+            ig.cluster().active(true);
+
+            IgniteCache cache = ig.getOrCreateCache(getTestTableConfiguration());
+
+            fillTestData(ig);
+
+            //when: stop one node and create indexes on other node
+            stopGrid(1);
+
+            cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a)")).getAll();
+            cache.query(new SqlFieldsQuery("create index myindexb on TestIndexObject(b)")).getAll();
+            cache.query(new SqlFieldsQuery("alter table TestIndexObject add column (c int)")).getAll();
+
+            //and: stop all grid
+            stopAllGrids();
+        }
+
+        {
+            //and: start cluster from node without index
+            IgniteEx ig = startGrid(1);
+            startGrid(0);
+
+            ig.cluster().active(true);
+
+            //and: change data
+            try (IgniteDataStreamer<Object, Object> s = ig.dataStreamer(TEST_CACHE_NAME)) {
+                s.allowOverwrite(true);
+                for (int i = 0; i < 5_000; i++)
+                    s.addData(i, null);
+            }
+
+            stopAllGrids();
+        }
+
+        {
+            //when: start node from first node
+            IgniteEx ig0 = startGrid(0);
+            IgniteEx ig1 = startGrid(1);
+
+            ig0.cluster().active(true);
+
+            //then: everything is ok
+            try (IgniteDataStreamer<Object, Object> s = ig1.dataStreamer(TEST_CACHE_NAME)) {
+                s.allowOverwrite(true);
+                for (int i = 0; i < 50_000; i++) {
+                    BinaryObject bo = ig1.binary().builder(TEST_INDEX_OBJECT)
+                        .setField("a", i, Object.class)
+                        .setField("b", String.valueOf(i), Object.class)
+                        .setField("c", i, Object.class)
+                        .build();
+
+                    s.addData(i, bo);
+                }
+            }
+
+            IgniteCache<Object, Object> cache = ig1.cache(TEST_CACHE_NAME);
+
+            assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where a > 5"), containsString("myindexa"));
+            assertFalse(cache.query(new SqlFieldsQuery("SELECT a,b,c FROM TestIndexObject limit 1")).getAll().isEmpty());
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testTakeConfigFromJoiningNodeOnInactiveGrid() throws Exception {
+        {
+            //given: two started nodes with test table
+            Ignite ig = startGrid(0);
+            startGrid(1);
+
+            ig.cluster().active(true);
+
+            IgniteCache cache = ig.getOrCreateCache(getTestTableConfiguration());
+
+            fillTestData(ig);
+
+            stopGrid(1);
+
+            cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a)")).getAll();
+            cache.query(new SqlFieldsQuery("create index myindexb on TestIndexObject(b)")).getAll();
+            cache.query(new SqlFieldsQuery("alter table TestIndexObject add column (c int)")).getAll();
+
+            stopAllGrids();
+        }
+
+        {
+            //and: start cluster from node without cache
+            IgniteEx ig = startGrid(1);
+            startGrid(0);
+
+            ig.cluster().active(true);
+
+            //then: config for cache was applying successful
+            IgniteCache<Object, Object> cache = ig.cache(TEST_CACHE_NAME);
+
+            assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where a > 5"), containsString("myindexa"));
+            assertFalse(cache.query(new SqlFieldsQuery("SELECT a,b,c FROM TestIndexObject limit 1")).getAll().isEmpty());
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testResaveConfigAfterMerge() throws Exception {
+        {
+            //given: two started nodes with test table
+            Ignite ig = startGrid(0);
+            startGrid(1);
+
+            ig.cluster().active(true);
+
+            IgniteCache cache = ig.getOrCreateCache(getTestTableConfiguration());
+
+            fillTestData(ig);
+
+            stopGrid(1);
+
+            cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a)")).getAll();
+            cache.query(new SqlFieldsQuery("create index myindexb on TestIndexObject(b)")).getAll();
+            cache.query(new SqlFieldsQuery("alter table TestIndexObject add column (c int)")).getAll();
+
+            stopAllGrids();
+        }
+
+        {
+            //when: start cluster from node without cache
+            IgniteEx ig = startGrid(1);
+            startGrid(0);
+
+            ig.cluster().active(true);
+
+            stopAllGrids();
+        }
+
+        {
+            //then: start only one node which originally was without index
+            IgniteEx ig = startGrid(1);
+
+            ig.cluster().active(true);
+
+            IgniteCache<Object, Object> cache = ig.cache(TEST_CACHE_NAME);
+
+            assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where a > 5"), containsString("myindexa"));
+            assertFalse(cache.query(new SqlFieldsQuery("SELECT a,b,c FROM TestIndexObject limit 1")).getAll().isEmpty());
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testMergeChangedConfigOnInactiveGrid() throws Exception {
+        {
+            //given: two started nodes with test table
+            Ignite ig = startGrid(0);
+            startGrid(1);
+
+            ig.cluster().active(true);
+            LinkedHashMap<String, String> fields = new LinkedHashMap<>();
+            fields.put("A", "java.lang.Integer");
+            fields.put("B", "java.lang.String");
+
+            CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(TEST_CACHE_NAME);
+
+            ccfg.setQueryEntities(Arrays.asList(
+                new QueryEntity()
+                    .setKeyType("java.lang.Integer")
+                    .setValueType("TestIndexObject")
+                    .setFields(fields)
+            ));
+
+            IgniteCache cache = ig.getOrCreateCache(ccfg);
+
+            fillTestData(ig);
+
+            cache.query(new SqlFieldsQuery("create index myindexb on TestIndexObject(b)")).getAll();
+
+            //and: stop one node and create index on other node
+            stopGrid(1);
+
+            cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a)")).getAll();
+            cache.query(new SqlFieldsQuery("drop index myindexb")).getAll();
+            cache.query(new SqlFieldsQuery("alter table TestIndexObject drop column b")).getAll();
+
+            //and: stop all grid
+            stopAllGrids();
+        }
+
+        {
+            //and: start cluster
+            IgniteEx ig0 = startGrid(0);
+            IgniteEx ig1 = startGrid(1);
+
+            ig0.cluster().active(true);
+
+            //then: config should be merged
+            try (IgniteDataStreamer<Object, Object> s = ig1.dataStreamer(TEST_CACHE_NAME)) {
+                s.allowOverwrite(true);
+                for (int i = 0; i < 5_000; i++) {
+                    BinaryObject bo = ig1.binary().builder("TestIndexObject")
+                        .setField("a", i, Object.class)
+                        .setField("b", String.valueOf(i), Object.class)
+                        .build();
+
+                    s.addData(i, bo);
+                }
+            }
+            IgniteCache<Object, Object> cache = ig1.cache(TEST_CACHE_NAME);
+
+            //then: index "myindexa" and column "b" restored from node "1"
+            assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where a > 5"), containsString("myindexa"));
+            assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where b > 5"), containsString("myindexb"));
+            assertFalse(cache.query(new SqlFieldsQuery("SELECT a,b FROM TestIndexObject limit 1")).getAll().isEmpty());
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testTakeChangedConfigOnActiveGrid() throws Exception {
+        {
+            //given: two started nodes with test table
+            Ignite ig = startGrid(0);
+            startGrid(1);
+
+            ig.cluster().active(true);
+
+            IgniteCache cache = ig.getOrCreateCache(getTestTableConfiguration());
+
+            fillTestData(ig);
+
+            //stop one node and create index on other node
+            stopGrid(1);
+
+            cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a)")).getAll();
+            cache.query(new SqlFieldsQuery("create index myindexb on TestIndexObject(b)")).getAll();
+            cache.query(new SqlFieldsQuery("alter table TestIndexObject add column (c int)")).getAll();
+
+            stopAllGrids();
+        }
+
+        {
+            //and: start cluster
+            IgniteEx ig = startGrid(0);
+            ig.cluster().active(true);
+
+            ig = startGrid(1);
+
+            //then: config should be merged
+            try (IgniteDataStreamer<Object, Object> s = ig.dataStreamer(TEST_CACHE_NAME)) {
+                s.allowOverwrite(true);
+                for (int i = 0; i < 5_000; i++) {
+                    BinaryObject bo = ig.binary().builder("TestIndexObject")
+                        .setField("a", i, Object.class)
+                        .setField("b", String.valueOf(i), Object.class)
+                        .setField("c", i, Object.class)
+                        .build();
+
+                    s.addData(i, bo);
+                }
+            }
+            IgniteCache<Object, Object> cache = ig.getOrCreateCache(TEST_CACHE_NAME);
+
+            cache.indexReadyFuture().get();
+
+            assertThat(doExplainPlan(cache, "explain select * from TestIndexObject where a > 5"), containsString("myindexa"));
+            assertFalse(cache.query(new SqlFieldsQuery("SELECT a,b,c FROM TestIndexObject limit 1")).getAll().isEmpty());
+        }
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testFailJoiningNodeBecauseDifferentSql() throws Exception {
+        {
+            //given: two started nodes with test table
+            Ignite ig = startGrid(0);
+            startGrid(1);
+
+            ig.cluster().active(true);
+
+            IgniteCache cache = ig.getOrCreateCache(getTestTableConfiguration());
+
+            cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a)")).getAll();
+
+            //stop one node and create index on other node
+            stopGrid(1);
+
+            cache.query(new SqlFieldsQuery("drop index myindexa")).getAll();
+            cache.query(new SqlFieldsQuery("alter table TestIndexObject drop column b")).getAll();
+            cache.query(new SqlFieldsQuery("alter table TestIndexObject add column (b int)")).getAll();
+            cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(b)")).getAll();
+
+            //and: stopped all grid
+            stopAllGrids();
+        }
+
+        {
+            //and: start cluster
+            startGrid(0);
+            try {
+                startGrid(1);
+
+                fail("Node should start with fail");
+            }
+            catch (Exception e) {
+                String cause = X.cause(e, IgniteSpiException.class).getMessage();
+                assertThat(cause, containsString("fieldType of B is different"));
+                assertThat(cause, containsString("index MYINDEXA is different"));
+            }
+        }
+
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testFailJoiningNodeBecauseFieldInlineSizeIsDifferent() throws Exception {
+        {
+            //given: two started nodes with test table
+            Ignite ig = startGrid(0);
+            startGrid(1);
+
+            ig.cluster().active(true);
+
+            IgniteCache cache = ig.getOrCreateCache(getTestTableConfiguration());
+
+            cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a) INLINE_SIZE 1000")).getAll();
+
+            //stop one node and create index on other node
+            stopGrid(1);
+
+            cache.query(new SqlFieldsQuery("drop index myindexa")).getAll();
+            cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a) INLINE_SIZE 2000")).getAll();
+
+            //and: stopped all grid
+            stopAllGrids();
+        }
+
+        {
+            //and: start cluster
+            startGrid(0);
+            try {
+                startGrid(1);
+
+                fail("Node should start with fail");
+            }
+            catch (Exception e) {
+                assertThat(X.cause(e, IgniteSpiException.class).getMessage(), containsString("index MYINDEXA is different"));
+            }
+        }
+
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testFailJoiningNodeBecauseNeedConfigUpdateOnActiveGrid() throws Exception {
+        {
+            startGrid(0);
+            startGrid(1);
+
+            CacheConfiguration<Object, Object> ccfg = getTestTableConfiguration();
+
+            Ignite ig = ignite(0);
+
+            ig.cluster().active(true);
+
+            IgniteCache cache = ig.getOrCreateCache(ccfg);
+
+            fillTestData(ig);
+
+            stopGrid(1);
+
+            cache.query(new SqlFieldsQuery("create index myindexa on TestIndexObject(a)")).getAll();
+            cache.query(new SqlFieldsQuery("create index myindexb on TestIndexObject(b)")).getAll();
+
+            stopGrid(0);
+        }
+
+        {
+            IgniteEx ig = startGrid(1);
+            ig.cluster().active(true);
+
+            try {
+                startGrid(0);
+
+                fail("Node should start with fail");
+            }
+            catch (Exception e) {
+                assertThat(X.cause(e, IgniteSpiException.class).getMessage(), containsString("Failed to join node to the active cluster"));
+            }
+        }
+    }
+
+    /**
+     * @return result of explain plan
+     */
+    @NotNull private String doExplainPlan(IgniteCache<Object, Object> cache, String sql) {
+        return cache.query(new SqlFieldsQuery(sql)).getAll().get(0).get(0).toString().toLowerCase();
+    }
+
+    /**
+     * fill data by default
+     */
+    private void fillTestData(Ignite ig) {
+        try (IgniteDataStreamer<Object, Object> s = ig.dataStreamer(TEST_CACHE_NAME)) {
+            for (int i = 0; i < 50_000; i++) {
+                BinaryObject bo = ig.binary().builder("TestIndexObject")
+                    .setField("a", i, Object.class)
+                    .setField("b", String.valueOf(i), Object.class)
+                    .build();
+
+                s.addData(i, bo);
+            }
+        }
+    }
+
+    /**
+     * @return cache configuration with test table
+     */
+    @NotNull private CacheConfiguration<Object, Object> getTestTableConfiguration() {
+        LinkedHashMap<String, String> fields = new LinkedHashMap<>();
+        fields.put("a", "java.lang.Integer");
+        fields.put("B", "java.lang.String");
+
+        CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(TEST_CACHE_NAME);
+
+        ccfg.setQueryEntities(Collections.singletonList(
+            new QueryEntity()
+                .setKeyType("java.lang.Integer")
+                .setValueType("TestIndexObject")
+                .setFields(fields)
+        ));
+        return ccfg;
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractConcurrentSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractConcurrentSelfTest.java
index dcb3722e9a4db..3f090620287d2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractConcurrentSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractConcurrentSelfTest.java
@@ -315,7 +315,7 @@ public void testNodeJoinOnPendingDropOperation() throws Exception {
      * @throws Exception If failed.
      */
     private void checkNodeJoinOnPendingOperation(boolean addOrRemove) throws Exception {
-        CountDownLatch finishLatch = new CountDownLatch(4);
+        CountDownLatch finishLatch = new CountDownLatch(3);
 
         IgniteEx srv1 = ignitionStart(serverConfiguration(1), finishLatch);
 
@@ -334,7 +334,6 @@ private void checkNodeJoinOnPendingOperation(boolean addOrRemove) throws Excepti
 
         ignitionStart(serverConfiguration(2), finishLatch);
         ignitionStart(serverConfiguration(3, true), finishLatch);
-        ignitionStart(clientConfiguration(4), finishLatch);
 
         assertFalse(idxFut.isDone());
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index e10fff19a8d87..8a88602768177 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -74,6 +74,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheUpdateSqlQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCheckClusterStateBeforeExecuteQueryTest;
 import org.apache.ignite.internal.processors.cache.IgniteCrossCachesJoinsQueryTest;
+import org.apache.ignite.internal.processors.cache.IgniteDynamicSqlRestoreTest;
 import org.apache.ignite.internal.processors.cache.IncorrectQueryEntityTest;
 import org.apache.ignite.internal.processors.cache.QueryEntityCaseMismatchTest;
 import org.apache.ignite.internal.processors.cache.SqlFieldsQuerySelfTest;
@@ -229,6 +230,7 @@ public static TestSuite suite() throws Exception {
         // Config.
         suite.addTestSuite(IgniteCacheDuplicateEntityConfigurationSelfTest.class);
         suite.addTestSuite(IncorrectQueryEntityTest.class);
+        suite.addTestSuite(IgniteDynamicSqlRestoreTest.class);
 
         // Queries tests.
         suite.addTestSuite(LazyQuerySelfTest.class);

From 2883ff4e958747916e7d9eec671100c366cad66b Mon Sep 17 00:00:00 2001
From: Ilya Borisov <klaster1@gmail.com>
Date: Tue, 17 Apr 2018 15:01:36 +0700
Subject: [PATCH 059/543] IGNITE-8291 Web Console: Fixed Docker file
 generation.

(cherry picked from commit 5614621)
---
 .../configuration/generator/Docker.service.js |  94 ++-
 .../generator/Docker.service.spec.js          | 133 +++++
 .../web-console/frontend/package-lock.json    | 550 +++++++++---------
 modules/web-console/frontend/package.json     |   1 +
 4 files changed, 475 insertions(+), 303 deletions(-)
 create mode 100644 modules/web-console/frontend/app/modules/configuration/generator/Docker.service.spec.js

diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.js
index ea49c41927828..8b03e9aa43e8c 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.js
@@ -15,6 +15,15 @@
  * limitations under the License.
  */
 
+import {outdent} from 'outdent/lib';
+import VersionService from 'app/services/Version.service';
+import POM_DEPENDENCIES from 'app/data/pom-dependencies.json';
+import get from 'lodash/get';
+
+const version = new VersionService();
+
+const ALPINE_DOCKER_SINCE = '2.1.0';
+
 /**
  * Docker file generation entry point.
  */
@@ -29,10 +38,10 @@ export default class IgniteDockerGenerator {
      * @returns {String}
      */
     from(cluster, targetVer) {
-        return [
-            '# Start from Apache Ignite image.',
-            `FROM apacheignite/ignite:${targetVer.ignite}`
-        ].join('\n');
+        return outdent`
+            # Start from Apache Ignite image.',
+            FROM apacheignite/ignite:${targetVer.ignite}
+        `;
     }
 
     /**
@@ -42,36 +51,59 @@ export default class IgniteDockerGenerator {
      * @param {Object} targetVer Target version.
      */
     generate(cluster, targetVer) {
+        return outdent`
+            ${this.from(cluster, targetVer)}
+
+            # Set config uri for node.
+            ENV CONFIG_URI ${this.escapeFileName(cluster.name)}-server.xml
+
+            # Copy optional libs.
+            ENV OPTION_LIBS ${this.optionLibs(cluster, targetVer).join(',')}
+
+            # Update packages and install maven.
+            ${this.packages(cluster, targetVer)}
+            
+            # Append project to container.
+            ADD . ${cluster.name}
+
+            # Build project in container.
+            RUN mvn -f ${cluster.name}/pom.xml clean package -DskipTests
+
+            # Copy project jars to node classpath.
+            RUN mkdir $IGNITE_HOME/libs/${cluster.name} && \\
+               find ${cluster.name}/target -name "*.jar" -type f -exec cp {} $IGNITE_HOME/libs/${cluster.name} \\;
+        `;
+    }
+
+    optionLibs(cluster, targetVer) {
         return [
-            this.from(cluster, targetVer),
-            '',
-            '# Set config uri for node.',
-            `ENV CONFIG_URI ${this.escapeFileName(cluster.name)}-server.xml`,
-            '',
-            '# Copy ignite-http-rest from optional.',
-            'ENV OPTION_LIBS ignite-rest-http',
-            '',
-            '# Update packages and install maven.',
-            'RUN \\',
-            '   apt-get update &&\\',
-            '   apt-get install -y maven',
-            '',
-            '# Append project to container.',
-            `ADD . ${cluster.name}`,
-            '',
-            '# Build project in container.',
-            `RUN mvn -f ${cluster.name}/pom.xml clean package -DskipTests`,
-            '',
-            '# Copy project jars to node classpath.',
-            `RUN mkdir $IGNITE_HOME/libs/${cluster.name} && \\`,
-            `   find ${cluster.name}/target -name "*.jar" -type f -exec cp {} $IGNITE_HOME/libs/${cluster.name} \\;`
-        ].join('\n');
+            'ignite-rest-http',
+            get(POM_DEPENDENCIES, [get(cluster, 'discovery.kind'), 'artifactId'])
+        ].filter(Boolean);
+    }
+
+    packages(cluster, targetVer) {
+        return version.since(targetVer.ignite, ALPINE_DOCKER_SINCE)
+            ? outdent`
+                RUN set -x \\
+                    && apk add --no-cache \\
+                        openjdk8
+
+                RUN apk --update add \\
+                    maven \\
+                    && rm -rfv /var/cache/apk/*
+            `
+            : outdent`
+                RUN \\
+                   apt-get update &&\\
+                   apt-get install -y maven
+            `;
     }
 
     ignoreFile() {
-        return [
-            'target',
-            'Dockerfile'
-        ].join('\n');
+        return outdent`
+            target
+            Dockerfile
+        `;
     }
 }
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.spec.js b/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.spec.js
new file mode 100644
index 0000000000000..becc35988dc30
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.spec.js
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DockerGenerator from './Docker.service';
+import {assert} from 'chai';
+import {outdent} from 'outdent/lib';
+
+suite.only('Dockerfile generator', () => {
+    const generator = new DockerGenerator();
+
+    test('Target 2.0', () => {
+        const cluster = {
+            name: 'FooBar'
+        };
+
+        const version = {ignite: '2.0.0'};
+
+        assert.equal(
+            generator.generate(cluster, version),
+            outdent`
+                # Start from Apache Ignite image.',
+                FROM apacheignite/ignite:2.0.0
+
+                # Set config uri for node.
+                ENV CONFIG_URI FooBar-server.xml
+
+                # Copy optional libs.
+                ENV OPTION_LIBS ignite-rest-http
+
+                # Update packages and install maven.
+                RUN \\
+                   apt-get update &&\\
+                   apt-get install -y maven
+
+                # Append project to container.
+                ADD . FooBar
+
+                # Build project in container.
+                RUN mvn -f FooBar/pom.xml clean package -DskipTests
+
+                # Copy project jars to node classpath.
+                RUN mkdir $IGNITE_HOME/libs/FooBar && \\
+                   find FooBar/target -name "*.jar" -type f -exec cp {} $IGNITE_HOME/libs/FooBar \\;
+            `
+        );
+    });
+    test('Target 2.1', () => {
+        const cluster = {
+            name: 'FooBar'
+        };
+        const version = {ignite: '2.1.0'};
+        assert.equal(
+            generator.generate(cluster, version),
+            outdent`
+                # Start from Apache Ignite image.',
+                FROM apacheignite/ignite:2.1.0
+
+                # Set config uri for node.
+                ENV CONFIG_URI FooBar-server.xml
+
+                # Copy optional libs.
+                ENV OPTION_LIBS ignite-rest-http
+
+                # Update packages and install maven.
+                RUN set -x \\
+                    && apk add --no-cache \\
+                        openjdk8
+
+                RUN apk --update add \\
+                    maven \\
+                    && rm -rfv /var/cache/apk/*
+
+                # Append project to container.
+                ADD . FooBar
+
+                # Build project in container.
+                RUN mvn -f FooBar/pom.xml clean package -DskipTests
+
+                # Copy project jars to node classpath.
+                RUN mkdir $IGNITE_HOME/libs/FooBar && \\
+                   find FooBar/target -name "*.jar" -type f -exec cp {} $IGNITE_HOME/libs/FooBar \\;
+            `
+        );
+    });
+
+    test('Discovery optional libs', () => {
+        const generateWithDiscovery = (discovery) => generator.generate({name: 'foo', discovery: {kind: discovery}}, {ignite: '2.1.0'});
+
+        assert.include(
+            generateWithDiscovery('Cloud'),
+            `ENV OPTION_LIBS ignite-rest-http,ignite-cloud`,
+            'Adds Apache jclouds lib'
+        );
+
+        assert.include(
+            generateWithDiscovery('S3'),
+            `ENV OPTION_LIBS ignite-rest-http,ignite-aws`,
+            'Adds Amazon AWS lib'
+        );
+
+        assert.include(
+            generateWithDiscovery('GoogleStorage'),
+            `ENV OPTION_LIBS ignite-rest-http,ignite-gce`,
+            'Adds Google Cloud Engine lib'
+        );
+
+        assert.include(
+            generateWithDiscovery('ZooKeeper'),
+            `ENV OPTION_LIBS ignite-rest-http,ignite-zookeeper`,
+            'Adds Zookeeper lib'
+        );
+
+        assert.include(
+            generateWithDiscovery('Kubernetes'),
+            `ENV OPTION_LIBS ignite-rest-http,ignite-kubernetes`,
+            'Adds Kubernetes lib'
+        );
+    });
+});
diff --git a/modules/web-console/frontend/package-lock.json b/modules/web-console/frontend/package-lock.json
index 1834621f6b316..e28ef4d01c123 100644
--- a/modules/web-console/frontend/package-lock.json
+++ b/modules/web-console/frontend/package-lock.json
@@ -83,9 +83,9 @@
       "dev": true
     },
     "@types/lodash": {
-      "version": "4.14.106",
-      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.106.tgz",
-      "integrity": "sha512-tOSvCVrvSqFZ4A/qrqqm6p37GZoawsZtoR0SJhlF7EonNZUgrn8FfT+RNQ11h+NUpMt6QVe36033f3qEKBwfWA==",
+      "version": "4.14.107",
+      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.107.tgz",
+      "integrity": "sha512-afvjfP2rl3yvtv2qrCRN23zIQcDinF+munMJCoHEw2BXF22QJogTlVfNPTACQ6ieDyA6VnyKT4WLuN/wK368ng==",
       "dev": true
     },
     "@types/mocha": {
@@ -95,9 +95,9 @@
       "dev": true
     },
     "@types/node": {
-      "version": "7.0.60",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-7.0.60.tgz",
-      "integrity": "sha512-ZfCUDgCOPBDn0aAsyBOcNh1nLksuGp3LAL+8GULccZN2IkMBG2KfiwFIRrIuQkLKg1W1dIB9kQZ9MIF3IgAqlw==",
+      "version": "9.6.5",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.5.tgz",
+      "integrity": "sha512-NOLEgsT6UiDTjnWG5Hd2Mg25LRyz/oe8ql3wbjzgSFeRzRROhPmtlsvIrei4B46UjERF0td9SZ1ZXPLOdcrBHg==",
       "dev": true
     },
     "@types/sinon": {
@@ -107,9 +107,9 @@
       "dev": true
     },
     "@types/tapable": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.1.tgz",
-      "integrity": "sha512-zRc13uGALq6rmLOYmpdI8X5TK6ATuf9jITC7iKTxaHqb/se7vBdiC8BEp1vM2VJQVSt3N53kDDzJOYeVkUKO/Q==",
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.2.tgz",
+      "integrity": "sha512-42zEJkBpNfMEAvWR5WlwtTH22oDzcMjFsL9gDGExwF8X8WvAiw7Vwop7hPw03QT8TKfec83LwbHj6SvpqM4ELQ==",
       "dev": true
     },
     "@types/uglify-js": {
@@ -145,8 +145,8 @@
       "integrity": "sha512-NoGVTCumOsyFfuy3934f3ktiJi+wcXHJFxT47tby3iCpuo6M/WjFA9VqT5bYO+FE46i3R0N00RpJX75HxHKDaQ==",
       "dev": true,
       "requires": {
-        "@types/node": "7.0.60",
-        "@types/tapable": "1.0.1",
+        "@types/node": "9.6.5",
+        "@types/tapable": "1.0.2",
         "@types/uglify-js": "3.0.2",
         "source-map": "0.6.1"
       },
@@ -200,7 +200,7 @@
       "resolved": "https://registry.npmjs.org/@uirouter/visualizer/-/visualizer-4.0.2.tgz",
       "integrity": "sha512-95T0g9HHAjEa+sqwzfSbF6HxBG3shp2oTeGvqYk3VcLEHzrgNopEKJojd+3GNcVznQ+MUAaX4EDHXrzaHKJT6Q==",
       "requires": {
-        "d3-hierarchy": "1.1.5",
+        "d3-hierarchy": "1.1.6",
         "d3-interpolate": "1.1.6",
         "preact": "7.2.1"
       }
@@ -933,7 +933,7 @@
       "requires": {
         "bn.js": "4.11.8",
         "inherits": "2.0.3",
-        "minimalistic-assert": "1.0.0"
+        "minimalistic-assert": "1.0.1"
       }
     },
     "assert": {
@@ -1022,7 +1022,7 @@
       "integrity": "sha1-Hb0cg1ZY41zj+ZhAmdsAWFx4IBQ=",
       "requires": {
         "browserslist": "1.7.7",
-        "caniuse-db": "1.0.30000827",
+        "caniuse-db": "1.0.30000830",
         "normalize-range": "0.1.2",
         "num2fraction": "1.2.2",
         "postcss": "5.2.18",
@@ -2097,7 +2097,7 @@
         "querystring-es3": "0.2.1",
         "read-only-stream": "2.0.0",
         "readable-stream": "2.3.6",
-        "resolve": "1.7.0",
+        "resolve": "1.7.1",
         "shasum": "1.0.2",
         "shell-quote": "1.6.1",
         "stream-browserify": "2.0.1",
@@ -2202,26 +2202,26 @@
       "requires": {
         "buffer-xor": "1.0.3",
         "cipher-base": "1.0.4",
-        "create-hash": "1.1.3",
+        "create-hash": "1.2.0",
         "evp_bytestokey": "1.0.3",
         "inherits": "2.0.3",
         "safe-buffer": "5.1.1"
       }
     },
     "browserify-cipher": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.0.tgz",
-      "integrity": "sha1-mYgkSHS/XtTijalWZtzWasj8Njo=",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz",
+      "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==",
       "requires": {
         "browserify-aes": "1.2.0",
-        "browserify-des": "1.0.0",
+        "browserify-des": "1.0.1",
         "evp_bytestokey": "1.0.3"
       }
     },
     "browserify-des": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.0.tgz",
-      "integrity": "sha1-2qJ3cXRwki7S/hhZQRihdUOXId0=",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.1.tgz",
+      "integrity": "sha512-zy0Cobe3hhgpiOM32Tj7KQ3Vl91m0njwsjzZQK1L+JDf11dzP9qIvjreVinsvXrgfjhStXwUWAEpB9D7Gwmayw==",
       "requires": {
         "cipher-base": "1.0.4",
         "des.js": "1.0.0",
@@ -2244,11 +2244,11 @@
       "requires": {
         "bn.js": "4.11.8",
         "browserify-rsa": "4.0.1",
-        "create-hash": "1.1.3",
-        "create-hmac": "1.1.6",
+        "create-hash": "1.2.0",
+        "create-hmac": "1.1.7",
         "elliptic": "6.4.0",
         "inherits": "2.0.3",
-        "parse-asn1": "5.1.0"
+        "parse-asn1": "5.1.1"
       }
     },
     "browserify-zlib": {
@@ -2264,7 +2264,7 @@
       "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-1.7.7.tgz",
       "integrity": "sha1-C9dnBCWL6CmyOYu1Dkti0aFmsLk=",
       "requires": {
-        "caniuse-db": "1.0.30000827",
+        "caniuse-db": "1.0.30000830",
         "electron-to-chromium": "1.3.42"
       }
     },
@@ -2476,15 +2476,15 @@
       "integrity": "sha1-tTTnxzTE+B7F++isoq0kNUuWLGw=",
       "requires": {
         "browserslist": "1.7.7",
-        "caniuse-db": "1.0.30000827",
+        "caniuse-db": "1.0.30000830",
         "lodash.memoize": "4.1.2",
         "lodash.uniq": "4.5.0"
       }
     },
     "caniuse-db": {
-      "version": "1.0.30000827",
-      "resolved": "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30000827.tgz",
-      "integrity": "sha1-vSg53Rlgk7RMKMF/k1ExQMnZJYg="
+      "version": "1.0.30000830",
+      "resolved": "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30000830.tgz",
+      "integrity": "sha1-bkUlWzRWSf0V/1kHLaHhK7PeLxM="
     },
     "caseless": {
       "version": "0.11.0",
@@ -2674,9 +2674,9 @@
       "integrity": "sha1-4qdQQqlVGQi+vSW4Uj1fl2nXkYE="
     },
     "chrome-trace-event": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-0.1.2.tgz",
-      "integrity": "sha1-kPNohdU0WlBiEzLwcXtZWIPV2YI="
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-0.1.3.tgz",
+      "integrity": "sha512-sjndyZHrrWiu4RY7AkHgjn80GfAM2ZSzUkZLV/Js59Ldmh6JDThf0SUmOHU53rFu2rVxxfCzJ30Ukcfch3Gb/A=="
     },
     "cipher-base": {
       "version": "1.0.4",
@@ -3234,32 +3234,33 @@
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
     "create-ecdh": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.0.tgz",
-      "integrity": "sha1-iIxyNZbN92EvZJgjPuvXo1MBc30=",
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.1.tgz",
+      "integrity": "sha512-iZvCCg8XqHQZ1ioNBTzXS/cQSkqkqcPs8xSX4upNB+DAk9Ht3uzQf2J32uAHNCne8LDmKr29AgZrEs4oIrwLuQ==",
       "requires": {
         "bn.js": "4.11.8",
         "elliptic": "6.4.0"
       }
     },
     "create-hash": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.1.3.tgz",
-      "integrity": "sha1-YGBCrIuSYnUPSDyt2rD1gZFy2P0=",
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
+      "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==",
       "requires": {
         "cipher-base": "1.0.4",
         "inherits": "2.0.3",
+        "md5.js": "1.3.4",
         "ripemd160": "2.0.1",
         "sha.js": "2.4.11"
       }
     },
     "create-hmac": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.6.tgz",
-      "integrity": "sha1-rLniIaThe9sHbpBlfEK5PjcmzwY=",
+      "version": "1.1.7",
+      "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
+      "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==",
       "requires": {
         "cipher-base": "1.0.4",
-        "create-hash": "1.1.3",
+        "create-hash": "1.2.0",
         "inherits": "2.0.3",
         "ripemd160": "2.0.1",
         "safe-buffer": "5.1.1",
@@ -3289,15 +3290,15 @@
       "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz",
       "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==",
       "requires": {
-        "browserify-cipher": "1.0.0",
+        "browserify-cipher": "1.0.1",
         "browserify-sign": "4.0.4",
-        "create-ecdh": "4.0.0",
-        "create-hash": "1.1.3",
-        "create-hmac": "1.1.6",
-        "diffie-hellman": "5.0.2",
+        "create-ecdh": "4.0.1",
+        "create-hash": "1.2.0",
+        "create-hmac": "1.1.7",
+        "diffie-hellman": "5.0.3",
         "inherits": "2.0.3",
         "pbkdf2": "3.0.14",
-        "public-encrypt": "4.0.0",
+        "public-encrypt": "4.0.2",
         "randombytes": "2.0.6",
         "randomfill": "1.0.4"
       }
@@ -3424,7 +3425,7 @@
       "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha25.tgz",
       "integrity": "sha512-XC6xLW/JqIGirnZuUWHXCHRaAjje2b3OIB0Vj5RIJo6mIi/AdJo30quQl5LxUl0gkXDIrTrFGbMlcZjyFplz1A==",
       "requires": {
-        "mdn-data": "1.1.0",
+        "mdn-data": "1.1.1",
         "source-map": "0.5.7"
       }
     },
@@ -3521,9 +3522,9 @@
       "integrity": "sha1-vHZD/KjlOoNH4vva/6I2eWtYUJs="
     },
     "d3-hierarchy": {
-      "version": "1.1.5",
-      "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-1.1.5.tgz",
-      "integrity": "sha1-ochFxC+Eoga88cAcAQmOpN2qeiY="
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-1.1.6.tgz",
+      "integrity": "sha512-nn4bhBnwWnMSoZgkBXD7vRyZ0xVUsNMQRKytWYHhP1I4qHw+qzApCTgSQTZqMdf4XXZbTMqA59hFusga+THA/g=="
     },
     "d3-interpolate": {
       "version": "1.1.6",
@@ -3759,7 +3760,7 @@
       "integrity": "sha1-wHTS4qpqipoH29YfmhXCzYPsjsw=",
       "requires": {
         "inherits": "2.0.3",
-        "minimalistic-assert": "1.0.0"
+        "minimalistic-assert": "1.0.1"
       }
     },
     "destroy": {
@@ -3807,9 +3808,9 @@
       "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA=="
     },
     "diffie-hellman": {
-      "version": "5.0.2",
-      "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.2.tgz",
-      "integrity": "sha1-tYNXOScM/ias9jIJn97SoH8gnl4=",
+      "version": "5.0.3",
+      "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
+      "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==",
       "requires": {
         "bn.js": "4.11.8",
         "miller-rabin": "4.0.1",
@@ -3995,7 +3996,7 @@
         "hash.js": "1.1.3",
         "hmac-drbg": "1.0.1",
         "inherits": "2.0.3",
-        "minimalistic-assert": "1.0.0",
+        "minimalistic-assert": "1.0.1",
         "minimalistic-crypto-utils": "1.0.1"
       }
     },
@@ -5104,9 +5105,9 @@
       "integrity": "sha1-2uRqnXj74lKSJYzB54CkHZXAN4I="
     },
     "flow-parser": {
-      "version": "0.69.0",
-      "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.69.0.tgz",
-      "integrity": "sha1-N4tRKNbQtVSosvFqTKPhq5ZJ8A4="
+      "version": "0.70.0",
+      "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.70.0.tgz",
+      "integrity": "sha512-gGdyVUZWswG5jcINrVDHd3RY4nJptBTAx9mR9thGsrGGmAUR7omgJXQSpR+fXrLtxSTAea3HpAZNU/yzRJc2Cg=="
     },
     "flush-write-stream": {
       "version": "1.0.3",
@@ -5727,11 +5728,12 @@
       }
     },
     "hash-base": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-2.0.2.tgz",
-      "integrity": "sha1-ZuodhW206KVHDK32/OI65SRO8uE=",
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz",
+      "integrity": "sha1-X8hoaEfs1zSZQDMZprCj8/auSRg=",
       "requires": {
-        "inherits": "2.0.3"
+        "inherits": "2.0.3",
+        "safe-buffer": "5.1.1"
       }
     },
     "hash.js": {
@@ -5740,7 +5742,7 @@
       "integrity": "sha512-/UETyP0W22QILqS+6HowevwhEFJ3MBJnwTf75Qob9Wz9t0DPuisL8kW8YZMK62dHAKE1c1p+gY1TtOLY+USEHA==",
       "requires": {
         "inherits": "2.0.3",
-        "minimalistic-assert": "1.0.0"
+        "minimalistic-assert": "1.0.1"
       }
     },
     "hawk": {
@@ -5765,7 +5767,7 @@
       "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=",
       "requires": {
         "hash.js": "1.1.3",
-        "minimalistic-assert": "1.0.0",
+        "minimalistic-assert": "1.0.1",
         "minimalistic-crypto-utils": "1.0.1"
       }
     },
@@ -5830,9 +5832,9 @@
       }
     },
     "html-minifier": {
-      "version": "3.5.14",
-      "resolved": "https://registry.npmjs.org/html-minifier/-/html-minifier-3.5.14.tgz",
-      "integrity": "sha512-sZjw6zhQgyUnIlIPU+W80XpRjWjdxHtNcxjfyOskOsCTDKytcfLY04wsQY/83Yqb4ndoiD2FtauiL7Yg6uUQFQ==",
+      "version": "3.5.15",
+      "resolved": "https://registry.npmjs.org/html-minifier/-/html-minifier-3.5.15.tgz",
+      "integrity": "sha512-OZa4rfb6tZOZ3Z8Xf0jKxXkiDcFWldQePGYFDcgKqES2sXeWaEv9y6QQvWUtX3ySI3feApQi5uCsHLINQ6NoAw==",
       "requires": {
         "camel-case": "3.0.0",
         "clean-css": "4.1.11",
@@ -5840,7 +5842,7 @@
         "he": "1.1.1",
         "param-case": "2.1.1",
         "relateurl": "0.2.7",
-        "uglify-js": "3.3.20"
+        "uglify-js": "3.3.21"
       },
       "dependencies": {
         "source-map": {
@@ -5849,9 +5851,9 @@
           "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
         },
         "uglify-js": {
-          "version": "3.3.20",
-          "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.20.tgz",
-          "integrity": "sha512-WpLkWCf9sGvGZnIvBV0PNID9BATQNT/IXKAmqegfKzIPcTmTV3FP8NQpoogQkt/Y402x2sOFdaHUmqFY9IZp+g==",
+          "version": "3.3.21",
+          "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.21.tgz",
+          "integrity": "sha512-uy82472lH8tshK3jS3c5IFb5MmNKd/5qyBd0ih8sM42L3jWvxnE339U9gZU1zufnLVs98Stib9twq8dLm2XYCA==",
           "requires": {
             "commander": "2.15.1",
             "source-map": "0.6.1"
@@ -5864,7 +5866,7 @@
       "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz",
       "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=",
       "requires": {
-        "html-minifier": "3.5.14",
+        "html-minifier": "3.5.15",
         "loader-utils": "0.2.17",
         "lodash": "4.17.5",
         "pretty-error": "2.1.1",
@@ -5902,7 +5904,7 @@
         "posthtml": "0.11.3",
         "posthtml-render": "1.1.3",
         "svgo": "1.0.5",
-        "uglify-js": "3.3.20"
+        "uglify-js": "3.3.21"
       },
       "dependencies": {
         "coa": {
@@ -5926,7 +5928,7 @@
               "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.27.tgz",
               "integrity": "sha512-BAYp9FyN4jLXjfvRpTDchBllDptqlK9I7OsagXCG9Am5C+5jc8eRZHgqb9x500W2OKS14MMlpQc/nmh/aA7TEQ==",
               "requires": {
-                "mdn-data": "1.1.0",
+                "mdn-data": "1.1.1",
                 "source-map": "0.5.7"
               }
             }
@@ -5962,15 +5964,15 @@
             "mkdirp": "0.5.1",
             "object.values": "1.0.4",
             "sax": "1.2.4",
-            "stable": "0.1.6",
+            "stable": "0.1.7",
             "unquote": "1.1.1",
             "util.promisify": "1.0.0"
           }
         },
         "uglify-js": {
-          "version": "3.3.20",
-          "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.20.tgz",
-          "integrity": "sha512-WpLkWCf9sGvGZnIvBV0PNID9BATQNT/IXKAmqegfKzIPcTmTV3FP8NQpoogQkt/Y402x2sOFdaHUmqFY9IZp+g==",
+          "version": "3.3.21",
+          "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.21.tgz",
+          "integrity": "sha512-uy82472lH8tshK3jS3c5IFb5MmNKd/5qyBd0ih8sM42L3jWvxnE339U9gZU1zufnLVs98Stib9twq8dLm2XYCA==",
           "requires": {
             "commander": "2.15.1",
             "source-map": "0.6.1"
@@ -6104,13 +6106,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "has-flag": {
@@ -6123,9 +6125,9 @@
           "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.21.tgz",
           "integrity": "sha512-y/bKfbQz2Nn/QBC08bwvYUxEFOVGfPIUOTsJ2CK5inzlXW9SdYR1x4pEsG9blRAF/PX+wRNdOah+gx/hv4q7dw==",
           "requires": {
-            "chalk": "2.3.2",
+            "chalk": "2.4.0",
             "source-map": "0.6.1",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "source-map": {
@@ -6134,9 +6136,9 @@
           "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -6245,7 +6247,7 @@
       "integrity": "sha512-h+xtnyk4EwKvFWHrUYsWErEVR+igKtLdchu+o0Z1RL7VU/jVMFbYir2bp6bAj8efFNxWqHX0dIss6fJQ+/+qeQ==",
       "requires": {
         "ansi-escapes": "3.1.0",
-        "chalk": "2.3.2",
+        "chalk": "2.4.0",
         "cli-cursor": "2.1.0",
         "cli-width": "2.2.0",
         "external-editor": "2.2.0",
@@ -6274,13 +6276,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "has-flag": {
@@ -6297,9 +6299,9 @@
           }
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -6793,10 +6795,10 @@
         "babel-register": "6.26.0",
         "babylon": "7.0.0-beta.44",
         "colors": "1.1.2",
-        "flow-parser": "0.69.0",
+        "flow-parser": "0.70.0",
         "lodash": "4.17.5",
         "micromatch": "2.3.11",
-        "neo-async": "2.5.0",
+        "neo-async": "2.5.1",
         "node-dir": "0.1.8",
         "nomnom": "1.8.1",
         "recast": "0.14.7",
@@ -7626,7 +7628,7 @@
       "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz",
       "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==",
       "requires": {
-        "chalk": "2.3.2"
+        "chalk": "2.4.0"
       },
       "dependencies": {
         "ansi-styles": {
@@ -7638,13 +7640,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "has-flag": {
@@ -7653,9 +7655,9 @@
           "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -7706,7 +7708,7 @@
       "integrity": "sha512-YL/qpTxYtK0iWWbuKCrevDZz5lh+OjyHHD+mICqpjnYGKdNRBvPeh/1uYjkKUemT1CSO4wwLOwphWMpKAnD9kw==",
       "dev": true,
       "requires": {
-        "circular-json": "0.5.1",
+        "circular-json": "0.5.3",
         "date-format": "1.2.0",
         "debug": "3.1.0",
         "semver": "5.5.0",
@@ -7714,9 +7716,9 @@
       },
       "dependencies": {
         "circular-json": {
-          "version": "0.5.1",
-          "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.5.1.tgz",
-          "integrity": "sha512-UjgcRlTAhAkLeXmDe2wK7ktwy/tgAqxiSndTIPiFZuIPLZmzHzWMwUIe9h9m/OokypG7snxCDEuwJshGBdPvaw==",
+          "version": "0.5.3",
+          "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.5.3.tgz",
+          "integrity": "sha512-YlxLOimeIoQGHnMe3kbf8qIV2Bj7uXLbljMPRguNT49GmSAzooNfS9EJ91rSJKbLBOOzM5agvtx0WyechZN/Hw==",
           "dev": true
         },
         "debug": {
@@ -7838,23 +7840,12 @@
       "requires": {
         "hash-base": "3.0.4",
         "inherits": "2.0.3"
-      },
-      "dependencies": {
-        "hash-base": {
-          "version": "3.0.4",
-          "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz",
-          "integrity": "sha1-X8hoaEfs1zSZQDMZprCj8/auSRg=",
-          "requires": {
-            "inherits": "2.0.3",
-            "safe-buffer": "5.1.1"
-          }
-        }
       }
     },
     "mdn-data": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-1.1.0.tgz",
-      "integrity": "sha512-jC6B3BFC07cCOU8xx1d+sQtDkVIpGKWv4TzK7pN7PyObdbwlIFJbHYk8ofvr0zrU8SkV1rSi87KAHhWCdLGw1Q=="
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-1.1.1.tgz",
+      "integrity": "sha512-2J5JENcb4yD5AzBI4ilTakiq2P9gHSsi4LOygnMu/bkchgTiA63AjsHAhDc+3U36AJHRfcz30Qv6Tb7i/Qsiew=="
     },
     "media-typer": {
       "version": "0.3.0",
@@ -8058,9 +8049,9 @@
       "integrity": "sha1-3z02Uqc/3ta5sLJBRub9BSNTRY4="
     },
     "minimalistic-assert": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.0.tgz",
-      "integrity": "sha1-cCvi3aazf0g2vLP121ZkG2Sh09M="
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz",
+      "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A=="
     },
     "minimalistic-crypto-utils": {
       "version": "1.0.1",
@@ -8244,7 +8235,7 @@
         "inherits": "2.0.3",
         "parents": "1.0.1",
         "readable-stream": "2.3.6",
-        "resolve": "1.7.0",
+        "resolve": "1.7.1",
         "stream-combiner2": "1.1.1",
         "subarg": "1.0.0",
         "through2": "2.0.3",
@@ -8460,9 +8451,9 @@
       "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk="
     },
     "neo-async": {
-      "version": "2.5.0",
-      "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.5.0.tgz",
-      "integrity": "sha512-nJmSswG4As/MkRq7QZFuH/sf/yuv8ODdMZrY4Bedjp77a5MK4A6s7YbBB64c9u79EBUOfXUXBvArmvzTD0X+6g=="
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.5.1.tgz",
+      "integrity": "sha512-3KL3fvuRkZ7s4IFOMfztb7zJp3QaVWnBeGoJlgB38XnCRPj/0tLzzLG5IB8NYOHbJ8g8UGrgZv44GLDk6CxTxA=="
     },
     "nice-try": {
       "version": "1.0.4",
@@ -8547,7 +8538,7 @@
         "stream-browserify": "2.0.1",
         "stream-http": "2.8.1",
         "string_decoder": "1.1.1",
-        "timers-browserify": "2.0.6",
+        "timers-browserify": "2.0.7",
         "tty-browserify": "0.0.0",
         "url": "0.11.0",
         "util": "0.10.3",
@@ -9027,6 +9018,11 @@
         "os-tmpdir": "1.0.2"
       }
     },
+    "outdent": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/outdent/-/outdent-0.5.0.tgz",
+      "integrity": "sha512-/jHxFIzoMXdqPzTaCpFzAAWhpkSjZPF4Vsn6jAfNpmbH/ymsmd7Qc6VE9BGn0L6YMj6uwpQLxCECpus4ukKS9Q=="
+    },
     "p-cancelable": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz",
@@ -9127,13 +9123,13 @@
       }
     },
     "parse-asn1": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.0.tgz",
-      "integrity": "sha1-N8T5t+06tlx0gXtfJICTf7+XxxI=",
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz",
+      "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==",
       "requires": {
         "asn1.js": "4.10.1",
         "browserify-aes": "1.2.0",
-        "create-hash": "1.1.3",
+        "create-hash": "1.2.0",
         "evp_bytestokey": "1.0.3",
         "pbkdf2": "3.0.14"
       }
@@ -9261,8 +9257,8 @@
       "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.14.tgz",
       "integrity": "sha512-gjsZW9O34fm0R7PaLHRJmLLVfSoesxztjPjE9o6R+qtVJij90ltg1joIovN9GKrRW3t1PzhDDG3UMEMFfZ+1wA==",
       "requires": {
-        "create-hash": "1.1.3",
-        "create-hmac": "1.1.6",
+        "create-hash": "1.2.0",
+        "create-hmac": "1.1.7",
         "ripemd160": "2.0.1",
         "safe-buffer": "5.1.1",
         "sha.js": "2.4.11"
@@ -9533,13 +9529,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "has-flag": {
@@ -9552,9 +9548,9 @@
           "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.21.tgz",
           "integrity": "sha512-y/bKfbQz2Nn/QBC08bwvYUxEFOVGfPIUOTsJ2CK5inzlXW9SdYR1x4pEsG9blRAF/PX+wRNdOah+gx/hv4q7dw==",
           "requires": {
-            "chalk": "2.3.2",
+            "chalk": "2.4.0",
             "source-map": "0.6.1",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "source-map": {
@@ -9563,9 +9559,9 @@
           "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -9590,13 +9586,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "has-flag": {
@@ -9609,9 +9605,9 @@
           "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.21.tgz",
           "integrity": "sha512-y/bKfbQz2Nn/QBC08bwvYUxEFOVGfPIUOTsJ2CK5inzlXW9SdYR1x4pEsG9blRAF/PX+wRNdOah+gx/hv4q7dw==",
           "requires": {
-            "chalk": "2.3.2",
+            "chalk": "2.4.0",
             "source-map": "0.6.1",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "source-map": {
@@ -9620,9 +9616,9 @@
           "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -9647,13 +9643,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "has-flag": {
@@ -9666,9 +9662,9 @@
           "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.21.tgz",
           "integrity": "sha512-y/bKfbQz2Nn/QBC08bwvYUxEFOVGfPIUOTsJ2CK5inzlXW9SdYR1x4pEsG9blRAF/PX+wRNdOah+gx/hv4q7dw==",
           "requires": {
-            "chalk": "2.3.2",
+            "chalk": "2.4.0",
             "source-map": "0.6.1",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "source-map": {
@@ -9677,9 +9673,9 @@
           "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -9704,13 +9700,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "has-flag": {
@@ -9723,9 +9719,9 @@
           "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.21.tgz",
           "integrity": "sha512-y/bKfbQz2Nn/QBC08bwvYUxEFOVGfPIUOTsJ2CK5inzlXW9SdYR1x4pEsG9blRAF/PX+wRNdOah+gx/hv4q7dw==",
           "requires": {
-            "chalk": "2.3.2",
+            "chalk": "2.4.0",
             "source-map": "0.6.1",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "source-map": {
@@ -9734,9 +9730,9 @@
           "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -9873,9 +9869,9 @@
       }
     },
     "posthtml-rename-id": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/posthtml-rename-id/-/posthtml-rename-id-1.0.3.tgz",
-      "integrity": "sha512-zaaHJSTihw1fsx2L81npO6gmDYu4yZuHfRX89IsJDhcRIV1P8SKJY5m1xDRZQh542flidwNS+70/pVAK8yMYOA==",
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/posthtml-rename-id/-/posthtml-rename-id-1.0.4.tgz",
+      "integrity": "sha512-bxsGN02JGqcihc9eztWu8Qlj2P/If9sY0ckYmEL+6hqrWRvwJw4RvnXSnlKmjS4yDBcT4cSpJdMy+xsSuHDvZw==",
       "requires": {
         "escape-string-regexp": "1.0.5"
       }
@@ -9937,9 +9933,9 @@
       "integrity": "sha1-gV7R9uvGWSb4ZbMQwHE7yzMVzks="
     },
     "prettier": {
-      "version": "1.11.1",
-      "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.11.1.tgz",
-      "integrity": "sha512-T/KD65Ot0PB97xTrG8afQ46x3oiVhnfGjGESSI9NWYcG92+OUPZKkwHqGWXH2t9jK1crnQjubECW0FuOth+hxw=="
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.12.1.tgz",
+      "integrity": "sha1-wa0g6APndJ+vkFpAnSNn4Gu+cyU="
     },
     "pretty-bytes": {
       "version": "4.0.2",
@@ -10037,14 +10033,14 @@
       "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
     },
     "public-encrypt": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.0.tgz",
-      "integrity": "sha1-OfaZ86RlYN1eusvKaTyvfGXBjMY=",
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.2.tgz",
+      "integrity": "sha512-4kJ5Esocg8X3h8YgJsKAuoesBgB7mqH3eowiDzMUPKiRDDE7E/BqqZD1hnTByIaAFiwAw246YEltSq7tdrOH0Q==",
       "requires": {
         "bn.js": "4.11.8",
         "browserify-rsa": "4.0.1",
-        "create-hash": "1.1.3",
-        "parse-asn1": "5.1.0",
+        "create-hash": "1.2.0",
+        "parse-asn1": "5.1.1",
         "randombytes": "2.0.6"
       }
     },
@@ -10103,7 +10099,7 @@
         "jstransformer": "1.0.0",
         "pug-error": "1.3.2",
         "pug-walk": "1.1.7",
-        "resolve": "1.7.0",
+        "resolve": "1.7.1",
         "uglify-js": "2.8.29"
       }
     },
@@ -10164,7 +10160,7 @@
       "requires": {
         "loader-utils": "1.1.0",
         "pug-walk": "1.1.7",
-        "resolve": "1.7.0"
+        "resolve": "1.7.1"
       }
     },
     "pug-parser": {
@@ -10482,7 +10478,7 @@
       "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
       "integrity": "sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q=",
       "requires": {
-        "resolve": "1.7.0"
+        "resolve": "1.7.1"
       }
     },
     "redent": {
@@ -10791,9 +10787,9 @@
       "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
     },
     "resolve": {
-      "version": "1.7.0",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.7.0.tgz",
-      "integrity": "sha512-QdgZ5bjR1WAlpLaO5yHepFvC+o3rCr6wpfE2tpJNMkXdulf2jKomQBdNRQITF3ZKHNlT71syG98yQP03gasgnA==",
+      "version": "1.7.1",
+      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.7.1.tgz",
+      "integrity": "sha512-c7rwLofp8g1U+h1KNyHL/jicrKg1Ek4q+Lr33AL65uZTinUZHe30D5HlyN5V9NW0JX1D5dXQ4jqW5l7Sy/kGfw==",
       "requires": {
         "path-parse": "1.0.5"
       }
@@ -10921,6 +10917,16 @@
       "requires": {
         "hash-base": "2.0.2",
         "inherits": "2.0.3"
+      },
+      "dependencies": {
+        "hash-base": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-2.0.2.tgz",
+          "integrity": "sha1-ZuodhW206KVHDK32/OI65SRO8uE=",
+          "requires": {
+            "inherits": "2.0.3"
+          }
+        }
       }
     },
     "roboto-font": {
@@ -10944,7 +10950,7 @@
         "acorn": "5.5.3",
         "estree-walker": "0.5.1",
         "magic-string": "0.22.5",
-        "resolve": "1.7.0",
+        "resolve": "1.7.1",
         "rollup-pluginutils": "2.0.1"
       }
     },
@@ -10955,7 +10961,7 @@
       "requires": {
         "builtin-modules": "2.0.0",
         "is-module": "1.0.0",
-        "resolve": "1.7.0"
+        "resolve": "1.7.1"
       }
     },
     "rollup-plugin-progress": {
@@ -11173,7 +11179,7 @@
         "clone-deep": "2.0.2",
         "loader-utils": "1.1.0",
         "lodash.tail": "4.1.1",
-        "neo-async": "2.5.0",
+        "neo-async": "2.5.1",
         "pify": "3.0.0"
       }
     },
@@ -11924,9 +11930,9 @@
       }
     },
     "stable": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.6.tgz",
-      "integrity": "sha1-kQ9dKu17Ugxud3SZwfMuE5/eyxA="
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.7.tgz",
+      "integrity": "sha512-LmxBix+nUtyihSBpxXAhRakYEy49fan2suysdS1fUZcKjI+krXmH8DCZJ3yfngfrOnIFNU8O73EgNTzO2jI53w=="
     },
     "static-extend": {
       "version": "0.1.2",
@@ -12191,7 +12197,7 @@
         "micromatch": "3.1.0",
         "postcss": "5.2.18",
         "postcss-prefix-selector": "1.6.0",
-        "posthtml-rename-id": "1.0.3",
+        "posthtml-rename-id": "1.0.4",
         "posthtml-svg-mode": "1.0.2",
         "query-string": "4.3.4",
         "traverse": "0.6.6"
@@ -12330,9 +12336,9 @@
       }
     },
     "svg-baker-runtime": {
-      "version": "1.3.3",
-      "resolved": "https://registry.npmjs.org/svg-baker-runtime/-/svg-baker-runtime-1.3.3.tgz",
-      "integrity": "sha512-yDnHhVM+nGxLu+Oj/zG07yUPXmJ7XLmekU2XQqL0jaLUazLjxj61uO8IMyww2roUjdMQo4x+J+KIlAp37qIbZQ==",
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/svg-baker-runtime/-/svg-baker-runtime-1.3.5.tgz",
+      "integrity": "sha512-BKxJT/Zz9M+K043zXbZf7CA3c10NKWByxobAukO30VLv71OvmpagjG32Z0UIay6ctMaOUmywOKHuceiSDqwUOA==",
       "requires": {
         "deepmerge": "1.3.2",
         "mitt": "1.1.2",
@@ -12350,7 +12356,7 @@
         "escape-string-regexp": "1.0.5",
         "loader-utils": "1.1.0",
         "svg-baker": "1.2.17",
-        "svg-baker-runtime": "1.3.3",
+        "svg-baker-runtime": "1.3.5",
         "url-slug": "2.0.0"
       }
     },
@@ -12389,7 +12395,7 @@
       "requires": {
         "ajv": "6.4.0",
         "ajv-keywords": "3.1.0",
-        "chalk": "2.3.2",
+        "chalk": "2.4.0",
         "lodash": "4.17.5",
         "slice-ansi": "1.0.0",
         "string-width": "2.1.1"
@@ -12415,13 +12421,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "has-flag": {
@@ -12430,9 +12436,9 @@
           "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -12522,9 +12528,9 @@
       "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8="
     },
     "timers-browserify": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.6.tgz",
-      "integrity": "sha512-HQ3nbYRAowdVd0ckGFvmJPPCOH/CHleFN/Y0YQCX1DVaB7t+KFvisuyN09fuP8Jtp1CpfSh8O8bMkHbdbPe6Pw==",
+      "version": "2.0.7",
+      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.7.tgz",
+      "integrity": "sha512-U7DtjfsHeYjNAyEz4MdCLGZMY3ySyHIgZZp6ba9uxZlMRMiK5yTHUYc2XfGQHKFgxGcmvBF2jafoNtQYvlDpOw==",
       "requires": {
         "setimmediate": "1.0.5"
       }
@@ -13205,7 +13211,7 @@
       "requires": {
         "chokidar": "2.0.3",
         "graceful-fs": "4.1.11",
-        "neo-async": "2.5.0"
+        "neo-async": "2.5.1"
       }
     },
     "wbuf": {
@@ -13213,7 +13219,7 @@
       "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz",
       "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==",
       "requires": {
-        "minimalistic-assert": "1.0.0"
+        "minimalistic-assert": "1.0.1"
       }
     },
     "webpack": {
@@ -13225,7 +13231,7 @@
         "acorn-dynamic-import": "3.0.0",
         "ajv": "6.4.0",
         "ajv-keywords": "3.1.0",
-        "chrome-trace-event": "0.1.2",
+        "chrome-trace-event": "0.1.3",
         "enhanced-resolve": "4.0.0",
         "eslint-scope": "3.7.1",
         "loader-runner": "2.3.0",
@@ -13233,7 +13239,7 @@
         "memory-fs": "0.4.1",
         "micromatch": "3.1.10",
         "mkdirp": "0.5.1",
-        "neo-async": "2.5.0",
+        "neo-async": "2.5.1",
         "node-libs-browser": "2.1.0",
         "schema-utils": "0.4.5",
         "tapable": "1.0.0",
@@ -13566,7 +13572,7 @@
             "babel-register": "6.26.0",
             "babylon": "6.18.0",
             "colors": "1.1.2",
-            "flow-parser": "0.69.0",
+            "flow-parser": "0.70.0",
             "lodash": "4.17.5",
             "micromatch": "2.3.11",
             "node-dir": "0.1.8",
@@ -13600,7 +13606,7 @@
       "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-2.0.14.tgz",
       "integrity": "sha512-gRoWaxSi2JWiYsn1QgOTb6ENwIeSvN1YExZ+kJ0STsTZK7bWPElW+BBBv1UnTbvcPC3v7E17mK8hlFX8DOYSGw==",
       "requires": {
-        "chalk": "2.3.2",
+        "chalk": "2.4.0",
         "cross-spawn": "6.0.5",
         "diff": "3.5.0",
         "enhanced-resolve": "4.0.0",
@@ -13619,13 +13625,13 @@
         "mkdirp": "0.5.1",
         "p-each-series": "1.0.0",
         "p-lazy": "1.0.0",
-        "prettier": "1.11.1",
-        "supports-color": "5.3.0",
+        "prettier": "1.12.1",
+        "supports-color": "5.4.0",
         "v8-compile-cache": "1.1.2",
         "webpack-addons": "1.1.5",
         "yargs": "11.1.0",
         "yeoman-environment": "2.0.6",
-        "yeoman-generator": "2.0.3"
+        "yeoman-generator": "2.0.4"
       },
       "dependencies": {
         "ansi-regex": {
@@ -13647,13 +13653,13 @@
           "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0="
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "cliui": {
@@ -13689,7 +13695,7 @@
           "integrity": "sha512-E9BmnJbAKLPGonz0HeWHtbKf+EeSP93paWO3ZYoUpq/aowXvYGjjCSuashhXPpzbArIjBbji39THkxTz9ZeEUQ==",
           "requires": {
             "ansi-escapes": "3.1.0",
-            "chalk": "2.3.2",
+            "chalk": "2.4.0",
             "cli-cursor": "2.1.0",
             "cli-width": "2.2.0",
             "external-editor": "2.2.0",
@@ -13697,7 +13703,7 @@
             "lodash": "4.17.5",
             "mute-stream": "0.0.7",
             "run-async": "2.3.0",
-            "rxjs": "5.5.8",
+            "rxjs": "5.5.10",
             "string-width": "2.1.1",
             "strip-ansi": "4.0.0",
             "through": "2.3.8"
@@ -13714,9 +13720,9 @@
           }
         },
         "rxjs": {
-          "version": "5.5.8",
-          "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-5.5.8.tgz",
-          "integrity": "sha512-Bz7qou7VAIoGiglJZbzbXa4vpX5BmTTN2Dj/se6+SwADtw4SihqBIiEa7VmTXJ8pynvq0iFr5Gx9VLyye1rIxQ==",
+          "version": "5.5.10",
+          "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-5.5.10.tgz",
+          "integrity": "sha512-SRjimIDUHJkon+2hFo7xnvNC4ZEHGzCRwh9P7nzX3zPkCGFEg/tuElrNR7L/rZMagnK2JeH2jQwPRpmyXyLB6A==",
           "requires": {
             "symbol-observable": "1.0.1"
           }
@@ -13730,9 +13736,9 @@
           }
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -13788,7 +13794,7 @@
       "requires": {
         "loud-rejection": "1.6.0",
         "memory-fs": "0.4.1",
-        "mime": "2.2.2",
+        "mime": "2.3.1",
         "path-is-absolute": "1.0.1",
         "range-parser": "1.2.0",
         "url-join": "4.0.0",
@@ -13796,9 +13802,9 @@
       },
       "dependencies": {
         "mime": {
-          "version": "2.2.2",
-          "resolved": "https://registry.npmjs.org/mime/-/mime-2.2.2.tgz",
-          "integrity": "sha512-A7PDg4s48MkqFEcYg2b069m3DXOEq7hx+9q9rIFrSSYfzsh35GX+LOVMQ8Au0ko7d8bSQCIAuzkjp0vCtwENlQ=="
+          "version": "2.3.1",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz",
+          "integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg=="
         }
       }
     },
@@ -13831,7 +13837,7 @@
         "sockjs-client": "1.1.4",
         "spdy": "3.4.7",
         "strip-ansi": "3.0.1",
-        "supports-color": "5.3.0",
+        "supports-color": "5.4.0",
         "webpack-dev-middleware": "3.0.1",
         "webpack-log": "1.2.0",
         "yargs": "9.0.1"
@@ -13883,9 +13889,9 @@
           "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -13897,7 +13903,7 @@
       "resolved": "https://registry.npmjs.org/webpack-log/-/webpack-log-1.2.0.tgz",
       "integrity": "sha512-U9AnICnu50HXtiqiDxuli5gLB5PGBo7VvcHx36jRZHwK4vzOYLbImqT4lwWwoMHdQWwEKw736fCHEekokTEKHA==",
       "requires": {
-        "chalk": "2.3.2",
+        "chalk": "2.4.0",
         "log-symbols": "2.2.0",
         "loglevelnext": "1.0.4",
         "uuid": "3.2.1"
@@ -13912,13 +13918,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "has-flag": {
@@ -13927,9 +13933,9 @@
           "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -14307,7 +14313,7 @@
       "resolved": "https://registry.npmjs.org/yeoman-environment/-/yeoman-environment-2.0.6.tgz",
       "integrity": "sha512-jzHBTTy8EPI4ImV8dpUMt+Q5zELkSU5xvGpndHcHudQ4tqN6YgIWaCGmRFl+HDchwRUkcgyjQ+n6/w5zlJBCPg==",
       "requires": {
-        "chalk": "2.3.2",
+        "chalk": "2.4.0",
         "debug": "3.1.0",
         "diff": "3.5.0",
         "escape-string-regexp": "1.0.5",
@@ -14331,13 +14337,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "debug": {
@@ -14371,9 +14377,9 @@
           "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
@@ -14381,12 +14387,12 @@
       }
     },
     "yeoman-generator": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/yeoman-generator/-/yeoman-generator-2.0.3.tgz",
-      "integrity": "sha512-mODmrZ26a94djmGZZuIiomSGlN4wULdou29ZwcySupb2e9FdvoCl7Ps2FqHFjEHio3kOl/iBeaNqrnx3C3NwWg==",
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/yeoman-generator/-/yeoman-generator-2.0.4.tgz",
+      "integrity": "sha512-Sgvz3MAkOpEIobcpW3rjEl6bOTNnl8SkibP9z7hYKfIGIlw0QDC2k0MAeXvyE2pLqc2M0Duql+6R7/W9GrJojg==",
       "requires": {
         "async": "2.6.0",
-        "chalk": "2.3.2",
+        "chalk": "2.4.0",
         "cli-table": "0.3.1",
         "cross-spawn": "5.1.0",
         "dargs": "5.1.0",
@@ -14421,13 +14427,13 @@
           }
         },
         "chalk": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
-          "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
+          "version": "2.4.0",
+          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
+          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
           "requires": {
             "ansi-styles": "3.2.1",
             "escape-string-regexp": "1.0.5",
-            "supports-color": "5.3.0"
+            "supports-color": "5.4.0"
           }
         },
         "debug": {
@@ -14493,9 +14499,9 @@
           "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM="
         },
         "supports-color": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
-          "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
+          "version": "5.4.0",
+          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
+          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
           "requires": {
             "has-flag": "3.0.0"
           }
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 3b42a61947f8a..87a516f584d78 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -94,6 +94,7 @@
     "natural-compare-lite": "^1.4.0",
     "node-sass": "4.8.3",
     "nvd3": "1.8.6",
+    "outdent": "^0.5.0",
     "pako": "1.0.6",
     "progress-bar-webpack-plugin": "1.11.0",
     "pug-html-loader": "1.1.0",

From d0997d7740ea1114b4c8236f225d989de98e2f10 Mon Sep 17 00:00:00 2001
From: YuriBabak <y.chief@gmail.com>
Date: Tue, 17 Apr 2018 11:22:14 +0300
Subject: [PATCH 060/543] IGNITE-8292: Broken yardstick compilation.

this closes #3838

(cherry picked from commit e76fcb4)
---
 ...zyCMeansDistributedClustererBenchmark.java | 130 ------------------
 ...iteFuzzyCMeansLocalClustererBenchmark.java |  93 -------------
 2 files changed, 223 deletions(-)
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java
deleted file mode 100644
index e356746ebf094..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansDistributedClustererBenchmark.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.clustering;
-
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.clustering.BaseFuzzyCMeansClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansDistributedClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansModel;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-import org.apache.ignite.yardstick.ml.DataChanger;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteFuzzyCMeansDistributedClustererBenchmark extends IgniteAbstractBenchmark {
-    /** */
-    @IgniteInstanceResource
-    private Ignite ignite;
-
-    /** {@inheritDoc} */
-    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
-        // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
-        // because we create ignite cache internally.
-        IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-            this.getClass().getSimpleName(), new Runnable() {
-            /** {@inheritDoc} */
-            @Override public void run() {
-                // IMPL NOTE originally taken from FuzzyCMeansExample.
-                // Distance measure that computes distance between two points.
-                DistanceMeasure distanceMeasure = new EuclideanDistance();
-
-                // "Fuzziness" - specific constant that is used in membership calculation (1.0+-eps ~ K-Means).
-                double exponentialWeight = 2.0;
-
-                // Condition that indicated when algorithm must stop.
-                // In this example algorithm stops if memberships have changed insignificantly.
-                BaseFuzzyCMeansClusterer.StopCondition stopCond =
-                    BaseFuzzyCMeansClusterer.StopCondition.STABLE_MEMBERSHIPS;
-
-                // Maximum difference between new and old membership values with which algorithm will continue to work.
-                double maxDelta = 0.01;
-
-                // The maximum number of FCM iterations.
-                int maxIterations = 50;
-
-                // Number of steps of primary centers selection (more steps more candidates).
-                int initializationSteps = 2;
-
-                // Number of K-Means iteration that is used to choose required number of primary centers from candidates.
-                int kMeansMaxIterations = 50;
-
-                // Create new distributed clusterer with parameters described above.
-                FuzzyCMeansDistributedClusterer clusterer = new FuzzyCMeansDistributedClusterer(
-                    distanceMeasure, exponentialWeight, stopCond, maxDelta, maxIterations,
-                    null, initializationSteps, kMeansMaxIterations);
-
-                // Create sample data.
-                double[][] points = shuffle((int)(DataChanger.next()));
-
-                // Initialize matrix of data points. Each row contains one point.
-                int rows = points.length;
-                int cols = points[0].length;
-
-                // Create the matrix that contains sample points.
-                SparseDistributedMatrix pntMatrix = new SparseDistributedMatrix(rows, cols,
-                    StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
-                // Store points into matrix.
-                pntMatrix.assign(points);
-
-                // Call clusterization method with some number of centers.
-                // It returns model that can predict results for new points.
-                int numCenters = 4;
-                FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, numCenters);
-
-                // Get centers of clusters that is computed by Fuzzy C-Means algorithm.
-                mdl.centers();
-
-                pntMatrix.destroy();
-            }
-        });
-
-        igniteThread.start();
-
-        igniteThread.join();
-
-        return true;
-    }
-
-    /** */
-    private double[][] shuffle(int off) {
-        final double[][] points = new double[][] {
-            {-10, -10}, {-9, -11}, {-10, -9}, {-11, -9},
-            {10, 10}, {9, 11}, {10, 9}, {11, 9},
-            {-10, 10}, {-9, 11}, {-10, 9}, {-11, 9},
-            {10, -10}, {9, -11}, {10, -9}, {11, -9}};
-
-        final int size = points.length;
-
-        final double[][] res = new double[size][];
-
-        for (int i = 0; i < size; i++)
-            res[i] = points[(i + off) % size];
-
-        return res;
-    }
-}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java
deleted file mode 100644
index 8c4c9ce7529d1..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteFuzzyCMeansLocalClustererBenchmark.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.clustering;
-
-import java.util.Map;
-import org.apache.ignite.ml.clustering.BaseFuzzyCMeansClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansLocalClusterer;
-import org.apache.ignite.ml.clustering.FuzzyCMeansModel;
-import org.apache.ignite.ml.math.distances.DistanceMeasure;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-import org.apache.ignite.yardstick.ml.DataChanger;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteFuzzyCMeansLocalClustererBenchmark extends IgniteAbstractBenchmark {
-    /** {@inheritDoc} */
-    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
-        // IMPL NOTE originally taken from FuzzyLocalCMeansExample.
-        // Distance measure that computes distance between two points.
-        DistanceMeasure distanceMeasure = new EuclideanDistance();
-
-        // "Fuzziness" - specific constant that is used in membership calculation (1.0+-eps ~ K-Means).
-        double exponentialWeight = 2.0;
-
-        // Condition that indicated when algorithm must stop.
-        // In this example algorithm stops if memberships have changed insignificantly.
-        BaseFuzzyCMeansClusterer.StopCondition stopCond =
-            BaseFuzzyCMeansClusterer.StopCondition.STABLE_MEMBERSHIPS;
-
-        // Maximum difference between new and old membership values with which algorithm will continue to work.
-        double maxDelta = 0.01;
-
-        // The maximum number of FCM iterations.
-        int maxIterations = 50;
-
-        // Create new local clusterer with parameters described above.
-        FuzzyCMeansLocalClusterer clusterer = new FuzzyCMeansLocalClusterer(distanceMeasure,
-            exponentialWeight, stopCond, maxDelta, maxIterations, null);
-
-        // Create sample data.
-        double[][] points = shuffle((int)(DataChanger.next()));
-
-        // Create the matrix that contains sample points.
-        DenseLocalOnHeapMatrix pntMatrix = new DenseLocalOnHeapMatrix(points);
-
-        // Call clusterization method with some number of centers.
-        // It returns model that can predict results for new points.
-        int numCenters = 4;
-        FuzzyCMeansModel mdl = clusterer.cluster(pntMatrix, numCenters);
-
-        // Get centers of clusters that is computed by Fuzzy C-Means algorithm.
-        mdl.centers();
-
-        return true;
-    }
-
-    /** */
-    private double[][] shuffle(int off) {
-        final double[][] points = new double[][] {
-            {-10, -10}, {-9, -11}, {-10, -9}, {-11, -9},
-            {10, 10}, {9, 11}, {10, 9}, {11, 9},
-            {-10, 10}, {-9, 11}, {-10, 9}, {-11, 9},
-            {10, -10}, {9, -11}, {10, -9}, {11, -9}};
-
-        final int size = points.length;
-
-        final double[][] res = new double[size][];
-
-        for (int i = 0; i < size; i++)
-            res[i] = points[(i + off) % size];
-
-        return res;
-    }
-}

From 3d2556bc73eff6c5ccd52af1bea88b6016358db8 Mon Sep 17 00:00:00 2001
From: Ilya Borisov <klaster1@gmail.com>
Date: Tue, 17 Apr 2018 15:46:10 +0700
Subject: [PATCH 061/543] IGNITE-8285 Web console: Removed debug output.

(cherry picked from commit 8c80dce)
---
 .../web-console/frontend/app/components/page-configure/index.js  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/web-console/frontend/app/components/page-configure/index.js b/modules/web-console/frontend/app/components/page-configure/index.js
index 3209edeb4094f..34b8cfe3f0f6f 100644
--- a/modules/web-console/frontend/app/components/page-configure/index.js
+++ b/modules/web-console/frontend/app/components/page-configure/index.js
@@ -164,7 +164,6 @@ export default angular
                     state: actionsWindow.filter((a) => !actions.includes(a)).reduce(ConfigureState._combinedReducer, {})
                 };
             })
-            .debug('UNDOED')
             .do((a) => ConfigureState.dispatchAction(a))
             .subscribe();
         ConfigEffects.connect();

From 4846e967e4cb7a174880a2956e807505a78fd441 Mon Sep 17 00:00:00 2001
From: YuriBabak <y.chief@gmail.com>
Date: Tue, 17 Apr 2018 11:54:41 +0300
Subject: [PATCH 062/543] IGNITE-8292: Broken yardstick compilation.

this closes #3840

(cherry picked from commit 3cebf91)
---
 ...teKMeansDistributedClustererBenchmark.java | 75 -------------------
 .../IgniteKMeansLocalClustererBenchmark.java  | 50 -------------
 .../yardstick/ml/clustering/package-info.java | 22 ------
 3 files changed, 147 deletions(-)
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansDistributedClustererBenchmark.java
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansLocalClustererBenchmark.java
 delete mode 100644 modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/package-info.java

diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansDistributedClustererBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansDistributedClustererBenchmark.java
deleted file mode 100644
index de928e88e7733..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansDistributedClustererBenchmark.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.clustering;
-
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.ml.clustering.KMeansDistributedClusterer;
-import org.apache.ignite.ml.math.StorageConstants;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.SparseDistributedMatrix;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-import org.apache.ignite.yardstick.ml.DataChanger;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteKMeansDistributedClustererBenchmark extends IgniteAbstractBenchmark {
-    /** */
-    @IgniteInstanceResource
-    private Ignite ignite;
-
-    /** {@inheritDoc} */
-    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
-        final DataChanger.Scale scale = new DataChanger.Scale();
-
-        // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
-        // because we create ignite cache internally.
-        IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
-            this.getClass().getSimpleName(), new Runnable() {
-            /** {@inheritDoc} */
-            @Override public void run() {
-                // IMPL NOTE originally taken from KMeansDistributedClustererTest
-                KMeansDistributedClusterer clusterer = new KMeansDistributedClusterer(
-                    new EuclideanDistance(), 1, 1, 1L);
-
-                double[] v1 = scale.mutate(new double[] {1959, 325100});
-                double[] v2 = scale.mutate(new double[] {1960, 373200});
-
-                SparseDistributedMatrix points = new SparseDistributedMatrix(
-                    2, 2, StorageConstants.ROW_STORAGE_MODE, StorageConstants.RANDOM_ACCESS_MODE);
-
-                points.setRow(0, v1);
-                points.setRow(1, v2);
-
-                clusterer.cluster(points, 1);
-
-                points.destroy();
-            }
-        });
-
-        igniteThread.start();
-
-        igniteThread.join();
-
-        return true;
-    }
-}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansLocalClustererBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansLocalClustererBenchmark.java
deleted file mode 100644
index d68fc6d19f342..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/IgniteKMeansLocalClustererBenchmark.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.yardstick.ml.clustering;
-
-import java.util.Map;
-import org.apache.ignite.ml.clustering.KMeansLocalClusterer;
-import org.apache.ignite.ml.math.distances.EuclideanDistance;
-import org.apache.ignite.ml.math.impls.matrix.DenseLocalOnHeapMatrix;
-import org.apache.ignite.yardstick.IgniteAbstractBenchmark;
-import org.apache.ignite.yardstick.ml.DataChanger;
-
-/**
- * Ignite benchmark that performs ML Grid operations.
- */
-@SuppressWarnings("unused")
-public class IgniteKMeansLocalClustererBenchmark extends IgniteAbstractBenchmark {
-    /** {@inheritDoc} */
-    @Override public boolean test(Map<Object, Object> ctx) throws Exception {
-        final DataChanger.Scale scale = new DataChanger.Scale();
-
-        // IMPL NOTE originally taken from KMeansLocalClustererTest
-        KMeansLocalClusterer clusterer = new KMeansLocalClusterer(new EuclideanDistance(), 1, 1L);
-
-        double[] v1 = scale.mutate(new double[] {1959, 325100});
-        double[] v2 = scale.mutate(new double[] {1960, 373200});
-
-        DenseLocalOnHeapMatrix points = new DenseLocalOnHeapMatrix(new double[][] {
-            v1,
-            v2});
-
-        clusterer.cluster(points, 1);
-
-        return true;
-    }
-}
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/package-info.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/package-info.java
deleted file mode 100644
index af217d24644a2..0000000000000
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/ml/clustering/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * <!-- Package description. -->
- * ML Grid clustering benchmarks.
- */
-package org.apache.ignite.yardstick.ml.clustering;
\ No newline at end of file

From 733a62bcb6c0d9381a496f07417c10c7edea6d7c Mon Sep 17 00:00:00 2001
From: Ilya Borisov <iborisov@gridgain.com>
Date: Tue, 17 Apr 2018 14:12:39 +0700
Subject: [PATCH 063/543] IGNITE-8287 Change position on signup inputs on
 page-sign-in.

(cherry picked from commit e5c3f89)
---
 .../app/components/page-signin/style.scss     | 10 +++++
 .../app/components/page-signin/template.pug   | 39 +++++++++----------
 2 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/modules/web-console/frontend/app/components/page-signin/style.scss b/modules/web-console/frontend/app/components/page-signin/style.scss
index 7e13ffe502efd..8ea143af440cd 100644
--- a/modules/web-console/frontend/app/components/page-signin/style.scss
+++ b/modules/web-console/frontend/app/components/page-signin/style.scss
@@ -35,4 +35,14 @@ page-sign-in {
         background-color: #ffffff;
         color: #444444;
     }
+
+    .ps-grid {
+        display: grid;
+        grid-gap: 10px;
+        grid-template-columns: 1fr 1fr;
+
+        .ps-grid-full-width {
+            grid-column: 1 / 3;
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-signin/template.pug b/modules/web-console/frontend/app/components/page-signin/template.pug
index 9a8b3badd6f08..58d85715dc0a5 100644
--- a/modules/web-console/frontend/app/components/page-signin/template.pug
+++ b/modules/web-console/frontend/app/components/page-signin/template.pug
@@ -27,10 +27,9 @@ section
                 .row
                     .col-xs-12.col-md-11
                         -var form = '$ctrl.form_signup'
-                        form(name=form novalidate)
-                            .settings-row
-                                h3 Don't Have An Account?
-                            .settings-row
+                        h3 Don't Have An Account?
+                        form.ps-grid(name=form novalidate)
+                            .ps-grid-full-width
                                 +form-field__email({
                                     label: 'Email:',
                                     model: '$ctrl.data.signup.email',
@@ -42,7 +41,7 @@ section
                                     ng-model-options='{allowInvalid: true}'
                                 )
                                     +form-field__error({error: 'server', message: `{{$ctrl.serverErrors.signup}}`})
-                            .settings-row
+                            div
                                 +form-field__password({
                                     label: 'Password:',
                                     model: '$ctrl.data.signup.password',
@@ -52,7 +51,7 @@ section
                                 })(
                                     ignite-on-enter-focus-move='confirmInput'
                                 )
-                            .settings-row
+                            div
                                 +form-field__password({
                                     label: 'Confirm:',
                                     model: 'confirm',
@@ -63,7 +62,7 @@ section
                                     ignite-on-enter-focus-move='firstNameInput'
                                     ignite-match='$ctrl.data.signup.password'
                                 )
-                            .settings-row
+                            div
                                 +form-field__text({
                                     label: 'First name:',
                                     model: '$ctrl.data.signup.firstName',
@@ -73,7 +72,7 @@ section
                                 })(
                                     ignite-on-enter-focus-move='lastNameInput'
                                 )
-                            .settings-row
+                            div
                                 +form-field__text({
                                     label: 'Last name:',
                                     model: '$ctrl.data.signup.lastName',
@@ -83,17 +82,7 @@ section
                                 })(
                                     ignite-on-enter-focus-move='companyInput'
                                 )
-                            .settings-row
-                                +form-field__text({
-                                    label: 'Company:',
-                                    model: '$ctrl.data.signup.company',
-                                    name: '"company"',
-                                    placeholder: 'Input company name',
-                                    required: true
-                                })(
-                                    ignite-on-enter-focus-move='countryInput'
-                                )
-                            .settings-row
+                            .ps-grid-full-width
                                 +form-field__dropdown({
                                     label: 'Country:',
                                     model: '$ctrl.data.signup.country',
@@ -104,7 +93,17 @@ section
                                 })(
                                     ignite-on-enter-focus-move='signup_submit'
                                 )
-                            .login-footer
+                            .ps-grid-full-width
+                                +form-field__text({
+                                    label: 'Company:',
+                                    model: '$ctrl.data.signup.company',
+                                    name: '"company"',
+                                    placeholder: 'Input company name',
+                                    required: true
+                                })(
+                                    ignite-on-enter-focus-move='countryInput'
+                                )
+                            .login-footer.ps-grid-full-width
                                 button#signup_submit.btn-ignite.btn-ignite--primary(
                                     ng-click='$ctrl.signup()'
                                     ng-disabled=`!$ctrl.canSubmitForm(${form})`

From 83e54311fce1d46279c6ddd687ced6f7c9f17ff6 Mon Sep 17 00:00:00 2001
From: Ilya Borisov <klaster1@gmail.com>
Date: Tue, 17 Apr 2018 17:15:57 +0700
Subject: [PATCH 064/543] IGNITE-8200 Web Console: Override clonedCluster in
 cluster-edit-form if caches or models have changed.     This improves interop
 with "import from DB" feature, which might update caches/models of cluster
 currently opened for editing.     The import dialog works as a separate
 state, so the form change detection mechanism ensures that any changes to the
 original     cluster are safe and won't interfere with changes made by user
 in cluster edit form.

(cherry picked from commit 7731669)
---
 .../cluster-edit-form/controller.js           | 24 +++++-
 .../cluster-edit-form/controller.spec.js      | 81 +++++++++++++++++++
 2 files changed, 102 insertions(+), 3 deletions(-)
 create mode 100644 modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.spec.js

diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
index 35b43e0727440..0207729821379 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
@@ -17,6 +17,7 @@
 
 import cloneDeep from 'lodash/cloneDeep';
 import get from 'lodash/get';
+import isEqual from 'lodash/isEqual';
 import _ from 'lodash';
 
 export default class ClusterEditFormController {
@@ -29,9 +30,11 @@ export default class ClusterEditFormController {
     constructor(IgniteLegacyUtils, IgniteEventGroups, IgniteConfirm, IgniteVersion, $scope, Clusters, IgniteFormUtils) {
         Object.assign(this, {IgniteLegacyUtils, IgniteEventGroups, IgniteConfirm, IgniteVersion, $scope, Clusters, IgniteFormUtils});
     }
+
     $onDestroy() {
         this.subscription.unsubscribe();
     }
+
     $onInit() {
         this.available = this.IgniteVersion.available.bind(this.IgniteVersion);
 
@@ -87,10 +90,9 @@ export default class ClusterEditFormController {
         this.$scope.ui = this.IgniteFormUtils.formUI();
         this.$scope.ui.loadedPanels = ['checkpoint', 'serviceConfiguration', 'odbcConfiguration'];
     }
+
     $onChanges(changes) {
-        if (
-            'cluster' in changes && get(this.clonedCluster, '_id') !== get(this.cluster, '_id')
-        ) {
+        if ('cluster' in changes && this.shouldOverwriteValue(this.cluster, this.clonedCluster)) {
             this.clonedCluster = cloneDeep(changes.cluster.currentValue);
             if (this.$scope.ui && this.$scope.ui.inputForm) {
                 this.$scope.ui.inputForm.$setPristine();
@@ -100,14 +102,30 @@ export default class ClusterEditFormController {
         if ('caches' in changes)
             this.cachesMenu = (changes.caches.currentValue || []).map((c) => ({label: c.name, value: c._id}));
     }
+
+    /**
+     * The form should accept incoming cluster value if:
+     * 1. It has different _id ("new" to real id).
+     * 2. Different caches or models (imported from DB).
+     * @param {Object} a Incoming value.
+     * @param {Object} b Current value.
+     */
+    shouldOverwriteValue(a, b) {
+        return get(a, '_id') !== get(b, '_id') ||
+            !isEqual(get(a, 'caches'), get(b, 'caches')) ||
+            !isEqual(get(a, 'models'), get(b, 'models'));
+    }
+
     getValuesToCompare() {
         return [this.cluster, this.clonedCluster].map(this.Clusters.normalize);
     }
+
     save() {
         if (this.$scope.ui.inputForm.$invalid)
             return this.IgniteFormUtils.triggerValidation(this.$scope.ui.inputForm, this.$scope);
         this.onSave({$event: cloneDeep(this.clonedCluster)});
     }
+
     reset = () => this.clonedCluster = cloneDeep(this.cluster);
     confirmAndReset() {
         return this.IgniteConfirm.confirm('Are you sure you want to undo all changes for current cluster?')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.spec.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.spec.js
new file mode 100644
index 0000000000000..cac888f769120
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.spec.js
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import 'mocha';
+import {assert} from 'chai';
+import {spy} from 'sinon';
+import Controller from './controller';
+
+suite('cluster-edit-form controller', () => {
+    test('cluster binding changes', () => {
+        const $scope = {
+            ui: {
+                inputForm: {
+                    $setPristine: spy(),
+                    $setUntouched: spy()
+                }
+            }
+        };
+
+        const mocks = Controller.$inject.map((token) => {
+            switch (token) {
+                case '$scope': return $scope;
+                default: return null;
+            }
+        });
+
+        const changeBoundCluster = ($ctrl, cluster) => {
+            $ctrl.cluster = cluster;
+            $ctrl.$onChanges({
+                cluster: {
+                    currentValue: cluster
+                }
+            });
+        };
+
+        const $ctrl = new Controller(...mocks);
+
+        const cluster1 = {_id: 1, caches: [1, 2, 3]};
+        const cluster2 = {_id: 1, caches: [1, 2, 3, 4, 5, 6], models: [1, 2, 3]};
+        const cluster3 = {_id: 1, caches: [1, 2, 3, 4, 5, 6], models: [1, 2, 3], name: 'Foo'};
+
+        changeBoundCluster($ctrl, cluster1);
+
+        assert.notEqual($ctrl.clonedCluster, cluster1, 'Cloned cluster is really cloned');
+        assert.deepEqual($ctrl.clonedCluster, cluster1, 'Cloned cluster is really a clone of incloming value');
+        assert.equal(1, $scope.ui.inputForm.$setPristine.callCount, 'Sets form pristine when cluster value changes');
+        assert.equal(1, $scope.ui.inputForm.$setUntouched.callCount, 'Sets form untouched when cluster value changes');
+
+        changeBoundCluster($ctrl, cluster2);
+
+        assert.deepEqual(
+            $ctrl.clonedCluster,
+            cluster2,
+            'Overrides clonedCluster if incoming cluster has same id but different caches or models'
+        );
+        assert.equal(2, $scope.ui.inputForm.$setPristine.callCount, 'Sets form pristine when bound cluster caches/models change');
+        assert.equal(2, $scope.ui.inputForm.$setUntouched.callCount, 'Sets form untouched when bound cluster caches/models change');
+
+        changeBoundCluster($ctrl, cluster3);
+
+        assert.deepEqual(
+            $ctrl.clonedCluster,
+            cluster2,
+            'Does not change cloned cluster value if fields other than id, chaches and models change'
+        );
+    });
+});

From 86d3f196e436095f277bb9b3e2c32293185db634 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Tue, 17 Apr 2018 14:28:47 +0300
Subject: [PATCH 065/543] IGNITE-8210 Fixed custom event handling for baseline
 topology change - Fixes #3814.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../affinity/GridAffinityAssignmentCache.java |  2 +-
 .../CacheBaselineTopologyTest.java            | 94 +++++++++++++++++++
 2 files changed, 95 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index 18edd028a2eef..427d60328b7fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -315,7 +315,7 @@ public List<List<ClusterNode>> calculate(
             for (DiscoveryEvent event : events.events()) {
                 boolean affinityNode = CU.affinityNode(event.eventNode(), nodeFilter);
 
-                if (affinityNode) {
+                if (affinityNode || event.type() == EVT_DISCOVERY_CUSTOM_EVT) {
                     skipCalculation = false;
 
                     break;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index 26502ed62391e..0d59a2d79a13d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cache.affinity.AffinityFunctionContext;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
@@ -54,6 +55,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -80,6 +82,12 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
     /** */
     private boolean delayRebalance;
 
+    /** */
+    private Map<String, Object> userAttrs;
+
+    /** */
+    private static final String DATA_NODE = "dataNodeUserAttr";
+
     /** */
     private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
@@ -129,6 +137,9 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
             .setWalMode(WALMode.LOG_ONLY)
         );
 
+        if (userAttrs != null)
+            cfg.setUserAttributes(userAttrs);
+
         if (client)
             cfg.setClientMode(true);
 
@@ -138,6 +149,89 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /**
+     * Verifies that rebalance on cache with Node Filter happens when BaselineTopology changes.
+     *
+     * @throws Exception
+     */
+    public void testRebalanceForCacheWithNodeFilter() throws Exception {
+        try {
+            final int EMPTY_NODE_IDX = 2;
+
+            userAttrs = U.newHashMap(1);
+            userAttrs.put(DATA_NODE, true);
+
+            startGrids(2);
+
+            userAttrs.put(DATA_NODE, false);
+
+            IgniteEx ignite = startGrid(2);
+
+            ignite.cluster().active(true);
+
+            awaitPartitionMapExchange();
+
+            IgniteCache<Integer, Integer> cache =
+                ignite.createCache(
+                    new CacheConfiguration<Integer, Integer>()
+                        .setName(CACHE_NAME)
+                        .setCacheMode(PARTITIONED)
+                        .setBackups(1)
+                        .setPartitionLossPolicy(READ_ONLY_SAFE)
+                        .setAffinity(new RendezvousAffinityFunction(32, null))
+                        .setNodeFilter(new DataNodeFilter())
+                );
+
+            for (int k = 0; k < 10_000; k++)
+                cache.put(k, k);
+
+            Thread.sleep(500);
+
+            printSizesDataNodes(NODE_COUNT - 1, EMPTY_NODE_IDX);
+
+            userAttrs.put(DATA_NODE, true);
+
+            startGrid(3);
+
+            ignite.cluster().setBaselineTopology(ignite.cluster().topologyVersion());
+
+            awaitPartitionMapExchange();
+
+            Thread.sleep(500);
+
+            printSizesDataNodes(NODE_COUNT, EMPTY_NODE_IDX);
+        }
+        finally {
+            userAttrs = null;
+        }
+    }
+
+    /** */
+    private void printSizesDataNodes(int nodesCnt, int emptyNodeIdx) {
+        for (int i = 0; i < nodesCnt; i++) {
+            IgniteEx ig = grid(i);
+
+            int locSize = ig.cache(CACHE_NAME).localSize(CachePeekMode.PRIMARY);
+
+            if (i == emptyNodeIdx)
+                assertEquals("Cache local size on "
+                    + i
+                    + " node is expected to be zero", 0, locSize);
+            else
+                assertTrue("Cache local size on "
+                    + i
+                    + " node is expected to be non zero", locSize > 0);
+        }
+    }
+
+    /** */
+    private static class DataNodeFilter implements IgnitePredicate<ClusterNode> {
+
+        @Override public boolean apply(ClusterNode clusterNode) {
+            return clusterNode.attribute(DATA_NODE);
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From a7dbea16064bbd52907a770bb40c3a2445313db2 Mon Sep 17 00:00:00 2001
From: Dmitriy Sorokin <sbt.sorokin.dvl@gmail.com>
Date: Tue, 17 Apr 2018 14:48:44 +0300
Subject: [PATCH 066/543] IGNITE-8255 Possible name collisions in
 WorkersRegistry.

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../org/apache/ignite/internal/worker/WorkersRegistry.java    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
index e8d46fb2ebddb..16676c8ffa93a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersRegistry.java
@@ -36,7 +36,7 @@ public class WorkersRegistry implements GridWorkerListener {
      * @param w Worker.
      */
     public void register(GridWorker w) {
-        if (registeredWorkers.putIfAbsent(w.name(), w) != null)
+        if (registeredWorkers.putIfAbsent(w.runner().getName(), w) != null)
             throw new IllegalStateException("Worker is already registered [worker=" + w + ']');
     }
 
@@ -75,6 +75,6 @@ public GridWorker worker(String name) {
 
     /** {@inheritDoc} */
     @Override public void onStopped(GridWorker w) {
-        unregister(w.name());
+        unregister(w.runner().getName());
     }
 }

From b762d681b97ea121a8321eb66bf02f89a1d177cd Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Tue, 17 Apr 2018 15:56:36 +0300
Subject: [PATCH 067/543] IGNITE-8166 PME hangs when error occurs during
 checkpoint

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../persistence/GridCacheDatabaseSharedManager.java      | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 5beaafc585c19..16d32924da847 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -3606,7 +3606,14 @@ private static class CheckpointProgress {
         private GridFutureAdapter cpBeginFut = new GridFutureAdapter<>();
 
         /** */
-        private GridFutureAdapter cpFinishFut = new GridFutureAdapter<>();
+        private GridFutureAdapter cpFinishFut = new GridFutureAdapter<Void>() {
+            @Override protected boolean onDone(@Nullable Void res, @Nullable Throwable err, boolean cancel) {
+                if (err != null && !cpBeginFut.isDone())
+                    cpBeginFut.onDone(err);
+
+                return super.onDone(res, err, cancel);
+            }
+        };
 
         /** */
         private volatile boolean nextSnapshot;

From 8428b0e63e97c10277f6d9e1640e16528a772270 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Tue, 17 Apr 2018 18:05:42 +0300
Subject: [PATCH 068/543] IGNITE-8021 Delete cache config files when cache is
 destroyed - Fixes #3697.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../pagemem/store/IgnitePageStoreManager.java |   9 +
 .../processors/cache/GridCacheProcessor.java  |  11 +
 .../file/FilePageStoreManager.java            |  47 +++
 ...onfigurationDataAfterDestroyCacheTest.java | 326 ++++++++++++++++++
 .../pagemem/NoOpPageStoreManager.java         |   5 +
 .../ignite/testsuites/IgnitePdsTestSuite.java |   2 +
 6 files changed, 400 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index 1b46bf990c540..0fc9f94b41978 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -193,6 +193,15 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cac
      * @throws IgniteCheckedException If failed.
      */
     public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException;
+
+    /**
+     * Remove cache configuration data file.
+     *
+     * @param cacheData Cache configuration.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void removeCacheData(StoredCacheData cacheData) throws IgniteCheckedException;
+
     /**
      * @param grpId Cache group ID.
      * @return {@code True} if index store for given cache group existed before node started.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 36edd72dd952f..bceb8c70dd235 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1284,6 +1284,17 @@ private void stopCache(GridCacheAdapter<?, ?> cache, boolean cancel, boolean des
 
             U.stopLifecycleAware(log, lifecycleAwares(ctx.group(), cache.configuration(), ctx.store().configuredStore()));
 
+            IgnitePageStoreManager pageStore;
+
+            if (destroy && (pageStore = sharedCtx.pageStore()) != null) {
+                try {
+                    pageStore.removeCacheData(new StoredCacheData(ctx.config()));
+                } catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to delete cache configuration data while destroying cache" +
+                            "[cache=" + ctx.name() + "]", e);
+                }
+            }
+
             if (log.isInfoEnabled()) {
                 if (ctx.group().sharedGroup())
                     log.info("Stopped cache [cacheName=" + cache.name() + ", group=" + ctx.group().name() + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 6313eac008195..837f3d01b2b2c 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -26,6 +26,7 @@
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
@@ -281,6 +282,9 @@ public FilePageStoreManager(GridKernalContext ctx) {
 
         IgniteCheckedException ex = shutdown(old, /*clean files if destroy*/destroy, null);
 
+        if (destroy)
+            removeCacheGroupConfigurationData(grp);
+
         if (ex != null)
             throw ex;
     }
@@ -745,6 +749,49 @@ private IgniteCheckedException shutdown(CacheStoreHolder holder, boolean cleanFi
         return aggr;
     }
 
+    /**
+     * Delete caches' configuration data files of cache group.
+     *
+     * @param ctx Cache group context.
+     * @throws IgniteCheckedException If fails.
+     */
+    private void removeCacheGroupConfigurationData(CacheGroupContext ctx) throws IgniteCheckedException {
+        File cacheGrpDir = cacheWorkDir(ctx.sharedGroup(), ctx.cacheOrGroupName());
+
+        if (cacheGrpDir != null && cacheGrpDir.exists()) {
+            DirectoryStream.Filter<Path> cacheCfgFileFilter = new DirectoryStream.Filter<Path>() {
+                @Override public boolean accept(Path path) {
+                    return Files.isRegularFile(path) && path.getFileName().toString().endsWith(CACHE_DATA_FILENAME);
+                }
+            };
+
+            try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(cacheGrpDir.toPath(), cacheCfgFileFilter)) {
+                for(Path path: dirStream)
+                    Files.deleteIfExists(path);
+            }
+            catch (IOException e) {
+                throw new IgniteCheckedException("Failed to delete cache configurations of group: " + ctx.toString(), e);
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeCacheData(StoredCacheData cacheData) throws IgniteCheckedException {
+        CacheConfiguration cacheCfg = cacheData.config();
+        File cacheWorkDir = cacheWorkDir(cacheCfg);
+        File file;
+
+        if (cacheData.config().getGroupName() != null)
+            file = new File(cacheWorkDir, cacheCfg.getName() + CACHE_DATA_FILENAME);
+        else
+            file = new File(cacheWorkDir, CACHE_DATA_FILENAME);
+
+        if (file.exists()) {
+            if (!file.delete())
+                throw new IgniteCheckedException("Failed to delete cache configuration:" + cacheCfg.getName());
+        }
+    }
+
     /**
      * @param store Store to shutdown.
      * @param cleanFile {@code True} if files should be cleaned.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java
new file mode 100644
index 0000000000000..d2767d465506b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test correct clean up cache configuration data after destroying cache.
+ */
+public class IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int CACHES = 3;
+
+    /** */
+    private static final int NODES = 3;
+
+    /** */
+    private static final int NUM_OF_KEYS = 100;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        return cfg.setDiscoverySpi(new TcpDiscoverySpi()
+                        .setIpFinder(IP_FINDER))
+                        .setDataStorageConfiguration(new DataStorageConfiguration()
+                            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                                .setMaxSize(200 * 1024 * 1024)
+                                .setPersistenceEnabled(true)));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        super.afterTest();
+    }
+
+    /**
+     * {@inheritDoc}
+     * @returns always {@code true} in order to be able to kill nodes when checkpointer thread hangs.
+     */
+    @Override protected boolean isMultiJvm() {
+        return true;
+    }
+
+    /**
+     *  Test destroy non grouped caches.
+     *
+     *  @throws Exception If failed.
+     */
+    public void testDestroyCaches() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startCachesDynamically(ignite);
+
+        checkDestroyCaches(ignite);
+    }
+
+    /**
+     *  Test destroy grouped caches.
+     *
+     *  @throws Exception If failed.
+     */
+    public void testDestroyGroupCaches() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startGroupCachesDynamically(ignite);
+
+        checkDestroyCaches(ignite);
+    }
+
+    /**
+     * Test destroy caches with disabled checkpoints.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDestroyCachesAbruptlyWithoutCheckpoints() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startCachesDynamically(ignite);
+
+        enableCheckpoints(false);
+
+        checkDestroyCachesAbruptly(ignite);
+    }
+
+    /**
+     * Test destroy group caches with disabled checkpoints.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDestroyGroupCachesAbruptlyWithoutCheckpoints() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startGroupCachesDynamically(ignite);
+
+        enableCheckpoints(false);
+
+        checkDestroyCachesAbruptly(ignite);
+    }
+
+    /**
+     * Test destroy caches abruptly with checkpoints.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDestroyCachesAbruptly() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startCachesDynamically(ignite);
+
+        checkDestroyCachesAbruptly(ignite);
+    }
+
+
+    /**
+     * Test destroy group caches abruptly with checkpoints.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDestroyGroupCachesAbruptly() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startGroupCachesDynamically(ignite);
+
+        checkDestroyCachesAbruptly(ignite);
+    }
+
+    /**
+     * @param ignite Ignite.
+     */
+    private void loadCaches(Ignite ignite) {
+        for (int i = 0; i < CACHES; i++) {
+            try (IgniteDataStreamer<Object, Object> s = ignite.dataStreamer(cacheName(i))) {
+                s.allowOverwrite(true);
+
+                for (int j = 0; j < NUM_OF_KEYS; j++)
+                    s.addData(j, "cache: " + i + " data: " + j);
+
+                s.flush();
+            }
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     */
+    private void checkDestroyCaches(Ignite ignite) throws Exception {
+        loadCaches(ignite);
+
+        log.warning("destroying caches....");
+
+        ignite.cache(cacheName(0)).destroy();
+        ignite.cache(cacheName(1)).destroy();
+
+        assertEquals(CACHES - 2, ignite.cacheNames().size());
+
+        log.warning("Stopping grid");
+
+        stopAllGrids();
+
+        log.warning("Grid stopped");
+
+        log.warning("Starting grid");
+
+        ignite = startGrids(NODES);
+
+        log.warning("Grid started");
+
+        assertEquals("Check that caches don't survived", CACHES - 2, ignite.cacheNames().size());
+
+        for(Ignite ig: G.allGrids()) {
+            IgniteCache cache = ig.cache(cacheName(2));
+
+            for (int j = 0; j < NUM_OF_KEYS; j++)
+                assertNotNull("Check that cache2 contains key: " + j + " node: " + ignite.name(), cache.get(j));
+        }
+    }
+
+
+    /**
+     * @param ignite Ignite instance.
+     */
+    private void checkDestroyCachesAbruptly(Ignite ignite) throws Exception {
+        loadCaches(ignite);
+
+        log.warning("Destroying caches");
+
+        ((GatewayProtectedCacheProxy)ignite.cache(cacheName(0))).destroyAsync();
+        ((GatewayProtectedCacheProxy)ignite.cache(cacheName(1))).destroyAsync();
+
+        log.warning("Stopping grid");
+
+        stopAllGrids();
+
+        log.warning("Grid stopped");
+
+        log.warning("Starting grid");
+
+        startGrids(NODES);
+
+        log.warning("Grid started");
+
+        for(Ignite ig: G.allGrids()) {
+            assertTrue(ig.cacheNames().contains(cacheName(2)));
+
+            IgniteCache cache = ig.cache(cacheName(2));
+
+            for (int j = 0; j < NUM_OF_KEYS; j++)
+                assertNotNull("Check that survived cache cache2 contains key: " + j + " node: " + ig.name(), cache.get(j));
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     */
+    private void startCachesDynamically(Ignite ignite) {
+        List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
+
+        for (int i = 0; i < CACHES; i++)
+            ccfg.add(new CacheConfiguration<>(cacheName(i))
+                    .setBackups(1)
+                    .setAffinity(new RendezvousAffinityFunction(false, 32)));
+
+        ignite.createCaches(ccfg);
+    }
+
+    /**
+     * @param ignite Ignite instance.
+     */
+    private void startGroupCachesDynamically(Ignite ignite) {
+        List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
+
+        for (int i = 0; i < CACHES; i++)
+            ccfg.add(new CacheConfiguration<>(cacheName(i))
+                    .setGroupName(i % 2 == 0 ? "grp-even" : "grp-odd")
+                    .setBackups(1)
+                    .setAffinity(new RendezvousAffinityFunction(false, 32)));
+
+        ignite.createCaches(ccfg);
+    }
+
+
+    /**
+     * Generate cache name from idx.
+     *
+     * @param idx Index.
+     */
+    private String cacheName(int idx) {
+        return "cache" + idx;
+    }
+
+    /**
+     * Enable/disable checkpoints on multi JVM nodes only.
+     *
+     * @param enabled Enabled flag.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void enableCheckpoints(boolean enabled) throws IgniteCheckedException {
+        for (Ignite ignite : G.allGrids()) {
+            assert !ignite.cluster().localNode().isClient();
+
+            GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+                    .cache().context().database();
+
+            dbMgr.enableCheckpoints(enabled).get();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index 64acf0211ddcf..be40c90f7c15c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -186,6 +186,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void removeCacheData(StoredCacheData cacheData) throws IgniteCheckedException {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public boolean hasIndexStore(int grpId) {
         return false;
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index a9668e7fc5852..af0b7adc1e68b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest;
@@ -113,6 +114,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
 
         suite.addTestSuite(IgnitePdsCacheRestoreTest.class);
         suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class);
+        suite.addTestSuite(IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.class);
 
         suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
 

From cd59c8e64f05ca03c7da8dc35d027a14fcebf250 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <Plehanov.Alex@gmail.com>
Date: Tue, 17 Apr 2018 18:27:53 +0300
Subject: [PATCH 069/543] IGNITE-8033 Fixed flaky failure of
 TxOptimisticDeadlockDetectionCrossCacheTest

Signed-off-by: Andrey Gura <agura@apache.org>
---
 ...misticDeadlockDetectionCrossCacheTest.java | 147 ++++++------------
 1 file changed, 50 insertions(+), 97 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionCrossCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionCrossCacheTest.java
index 5d1374c0918d8..056b093f7fba6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionCrossCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticDeadlockDetectionCrossCacheTest.java
@@ -18,30 +18,21 @@
 package org.apache.ignite.internal.processors.cache.transactions;
 
 import java.util.Collection;
-import java.util.Set;
-import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.managers.communication.GridIoMessage;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareResponse;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.util.GridConcurrentHashSet;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteInClosure;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.spi.IgniteSpiException;
-import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -57,9 +48,6 @@
  *
  */
 public class TxOptimisticDeadlockDetectionCrossCacheTest extends GridCommonAbstractTest {
-    /** Nodes count. */
-    private static final int NODES_CNT = 2;
-
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -73,10 +61,6 @@ public class TxOptimisticDeadlockDetectionCrossCacheTest extends GridCommonAbstr
             cfg.setDiscoverySpi(discoSpi);
         }
 
-        TcpCommunicationSpi commSpi = new TestCommunicationSpi();
-
-        cfg.setCommunicationSpi(commSpi);
-
         CacheConfiguration ccfg0 = defaultCacheConfiguration();
 
         ccfg0.setName("cache0");
@@ -96,42 +80,46 @@ public class TxOptimisticDeadlockDetectionCrossCacheTest extends GridCommonAbstr
         return cfg;
     }
 
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        super.beforeTestsStarted();
-
-        startGrids(NODES_CNT);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        super.afterTestsStopped();
-
-        stopAllGrids();
-    }
-
     /**
      * @throws Exception If failed.
      */
     public void testDeadlock() throws Exception {
-        // Sometimes boh transactions perform commit, so we repeat attempt.
-        while (!doTestDeadlock()) {}
+        startGrids(2);
+
+        try {
+            doTestDeadlock();
+        }
+        finally {
+            stopAllGrids();
+        }
     }
 
     /**
      * @throws Exception If failed.
      */
     private boolean doTestDeadlock() throws Exception {
-        TestCommunicationSpi.init(2);
-
-        final CyclicBarrier barrier = new CyclicBarrier(2);
-
         final AtomicInteger threadCnt = new AtomicInteger();
 
         final AtomicBoolean deadlock = new AtomicBoolean();
 
         final AtomicInteger commitCnt = new AtomicInteger();
 
+        grid(0).events().localListen(new CacheLocksListener(), EventType.EVT_CACHE_OBJECT_LOCKED);
+
+        AffinityTopologyVersion waitTopVer = new AffinityTopologyVersion(2, 1);
+
+        IgniteInternalFuture<?> exchFut = grid(0).context().cache().context().exchange().affinityReadyFuture(waitTopVer);
+
+        if (exchFut != null && !exchFut.isDone()) {
+            log.info("Waiting for topology exchange future [waitTopVer=" + waitTopVer + ", curTopVer="
+                + grid(0).context().cache().context().exchange().readyAffinityVersion() + ']');
+
+            exchFut.get();
+        }
+
+        log.info("Finished topology exchange future [curTopVer="
+            + grid(0).context().cache().context().exchange().readyAffinityVersion() + ']');
+
         IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
             @Override public void run() {
                 int threadNum = threadCnt.getAndIncrement();
@@ -152,8 +140,6 @@ private boolean doTestDeadlock() throws Exception {
 
                     cache1.put(key1, 0);
 
-                    barrier.await();
-
                     int key2 = primaryKey(cache2);
 
                     log.info(">>> Performs put [node=" + ((IgniteKernal)ignite).localNode() +
@@ -171,23 +157,23 @@ private boolean doTestDeadlock() throws Exception {
                         hasCause(e, TransactionDeadlockException.class)
                         ) {
                         if (deadlock.compareAndSet(false, true))
-                            U.error(log, "At least one stack trace should contain " +
-                                TransactionDeadlockException.class.getSimpleName(), e);
+                            log.info("Successfully set deadlock flag");
+                        else
+                            log.info("Deadlock flag was already set");
                     }
+                    else
+                        log.warning("Got not deadlock exception", e);
                 }
             }
         }, 2, "tx-thread");
 
         fut.get();
 
-        if (commitCnt.get() == 2)
-            return false;
+        assertFalse("Commits must fail", commitCnt.get() == 2);
 
         assertTrue(deadlock.get());
 
-        for (int i = 0; i < NODES_CNT ; i++) {
-            Ignite ignite = ignite(i);
-
+        for (Ignite ignite : G.allGrids()) {
             IgniteTxManager txMgr = ((IgniteKernal)ignite).context().cache().context().tm();
 
             Collection<IgniteInternalFuture<?>> futs = txMgr.deadlockDetectionFutures();
@@ -199,59 +185,26 @@ private boolean doTestDeadlock() throws Exception {
     }
 
     /**
+     * Listener for cache lock events.
      *
+     * To ensure deadlock this listener blocks transaction thread until both threads acquire first lock.
      */
-    private static class TestCommunicationSpi extends TcpCommunicationSpi {
-        /** Tx count. */
-        private static volatile int TX_CNT;
-
-        /** Tx ids. */
-        private static final Set<GridCacheVersion> TX_IDS = new GridConcurrentHashSet<>();
-
-        /**
-         * @param txCnt Tx count.
-         */
-        private static void init(int txCnt) {
-            TX_CNT = txCnt;
-            TX_IDS.clear();
-        }
+    private static class CacheLocksListener implements IgnitePredicate<Event> {
+        /** Latch. */
+        private final CountDownLatch latch = new CountDownLatch(2);
 
         /** {@inheritDoc} */
-        @Override public void sendMessage(
-            final ClusterNode node,
-            final Message msg,
-            final IgniteInClosure<IgniteException> ackC
-        ) throws IgniteSpiException {
-            if (msg instanceof GridIoMessage) {
-                Message msg0 = ((GridIoMessage)msg).message();
-
-                if (msg0 instanceof GridNearTxPrepareRequest) {
-                    final GridNearTxPrepareRequest req = (GridNearTxPrepareRequest)msg0;
-
-                    GridCacheVersion txId = req.version();
-
-                    if (TX_IDS.contains(txId)) {
-                        while (TX_IDS.size() < TX_CNT) {
-                            try {
-                                U.sleep(50);
-                            }
-                            catch (IgniteInterruptedCheckedException e) {
-                                e.printStackTrace();
-                            }
-                        }
-                    }
-                }
-                else if (msg0 instanceof GridNearTxPrepareResponse) {
-                    GridNearTxPrepareResponse res = (GridNearTxPrepareResponse)msg0;
+        @Override public boolean apply(Event evt) {
+            latch.countDown();
 
-                    GridCacheVersion txId = res.version();
-
-                    TX_IDS.add(txId);
-                }
+            try {
+                latch.await();
+            }
+            catch (InterruptedException e) {
+                e.printStackTrace();
             }
 
-            super.sendMessage(node, msg, ackC);
+            return true;
         }
     }
-
 }

From acfef907db8204ac93fc235770f36bf7f61269c3 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Tue, 17 Apr 2018 19:50:51 +0300
Subject: [PATCH 070/543] IGNITE-2766 Fix .net test. - Fixes #3853.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit 96cb795)
---
 .../dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs        | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs
index 7e6222f026f6c..274439e4e3be6 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ReconnectTest.cs
@@ -98,10 +98,8 @@ public void TestClusterRestart()
             cache1[1] = 2;
             Assert.AreEqual(2, cache1[1]);
 
-            // Check that old cache instance does not work.
-            var cacheEx1 = Assert.Throws<InvalidOperationException>(() => cache.Get(1));
-            Assert.IsTrue(cacheEx1.Message.EndsWith("Failed to perform cache operation (cache is stopped): cache"), 
-                cacheEx1.Message);
+            // Check that old cache instance still works.
+            Assert.AreEqual(2, cache.Get(1));
         }
 
         /// <summary>

From 6cea78e4e13fe43555b78dcd683366f54c6816ff Mon Sep 17 00:00:00 2001
From: Andrey Kuznetsov <stkuzma@gmail.com>
Date: Tue, 17 Apr 2018 19:58:43 +0300
Subject: [PATCH 071/543] IGNITE-7770 Test testRandomMixedTxConfigurations
 partialy fixed

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../processors/cache/GridCacheAdapter.java    |  3 +-
 ...OptimisticSerializableTxPrepareFuture.java |  2 +-
 .../GridNearOptimisticTxPrepareFuture.java    |  2 +-
 .../GridNearPessimisticTxPrepareFuture.java   |  2 +-
 .../near/GridNearTxFastFinishFuture.java      | 10 +++-
 .../distributed/near/GridNearTxLocal.java     | 57 ++++++++++++-------
 .../cache/transactions/IgniteTxManager.java   |  5 +-
 .../transactions/TxRollbackOnTimeoutTest.java |  2 +
 8 files changed, 55 insertions(+), 28 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index c2d0f427fc917..bd613a191f407 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -104,6 +104,7 @@
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
+import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -4283,7 +4284,7 @@ public T applyx(IgniteInternalFuture<T> tFut) throws IgniteCheckedException {
                         }
                         catch (IgniteCheckedException e1) {
                             try {
-                                tx0.rollbackNearTxLocalAsync();
+                                tx0.rollbackNearTxLocalAsync(e1 instanceof IgniteTxTimeoutCheckedException);
                             }
                             catch (Throwable e2) {
                                 if (e2 != e1)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index beb1e160c7ad6..3e2c84a740af5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -294,7 +294,7 @@ private boolean onComplete() {
         boolean txStateCheck = remap ? tx.state() == PREPARING : tx.state(PREPARING);
 
         if (!txStateCheck) {
-            if (tx.setRollbackOnly()) {
+            if (tx.isRollbackOnly() || tx.setRollbackOnly()) {
                 if (tx.timedOut())
                     onError(null, new IgniteTxTimeoutCheckedException("Transaction timed out and " +
                         "was rolled back: " + this));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 8d8c0b28b8b5f..2afb09601ebac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -327,7 +327,7 @@ private boolean onComplete() {
             boolean txStateCheck = remap ? tx.state() == PREPARING : tx.state(PREPARING);
 
             if (!txStateCheck) {
-                if (tx.setRollbackOnly()) {
+                if (tx.isRollbackOnly() || tx.setRollbackOnly()) {
                     if (tx.remainingTime() == -1)
                         onError(new IgniteTxTimeoutCheckedException("Transaction timed out and " +
                             "was rolled back: " + this), false);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index 4ab6863ad03b2..54ae85c539cc7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -155,7 +155,7 @@ private MiniFuture miniFuture(int miniId) {
     /** {@inheritDoc} */
     @Override public void prepare() {
         if (!tx.state(PREPARING)) {
-            if (tx.setRollbackOnly()) {
+            if (tx.isRollbackOnly() || tx.setRollbackOnly()) {
                 if (tx.remainingTime() == -1)
                     onDone(new IgniteTxTimeoutCheckedException("Transaction timed out and was rolled back: " + tx));
                 else
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
index 72226973009c5..95e4deded214f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
@@ -37,13 +37,17 @@ public class GridNearTxFastFinishFuture extends GridFutureAdapter<IgniteInternal
     /** */
     private final boolean commit;
 
+    /** */
+    private final boolean onTimeout;
+
     /**
      * @param tx Transaction.
      * @param commit Commit flag.
      */
-    GridNearTxFastFinishFuture(GridNearTxLocal tx, boolean commit) {
+    GridNearTxFastFinishFuture(GridNearTxLocal tx, boolean commit, boolean onTimeout) {
         this.tx = tx;
         this.commit = commit;
+        this.onTimeout = onTimeout;
     }
 
     /** {@inheritDoc} */
@@ -61,7 +65,7 @@ public void finish() {
                 tx.state(PREPARED);
                 tx.state(COMMITTING);
 
-                tx.context().tm().fastFinishTx(tx, true);
+                tx.context().tm().fastFinishTx(tx, true, true);
 
                 tx.state(COMMITTED);
             }
@@ -70,7 +74,7 @@ public void finish() {
                 tx.state(PREPARED);
                 tx.state(ROLLING_BACK);
 
-                tx.context().tm().fastFinishTx(tx, false);
+                tx.context().tm().fastFinishTx(tx, false, !onTimeout);
 
                 tx.state(ROLLED_BACK);
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index fc8a9a3fb2f6c..e3397505bec8d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -111,6 +111,7 @@
 import static org.apache.ignite.transactions.TransactionState.COMMITTED;
 import static org.apache.ignite.transactions.TransactionState.COMMITTING;
 import static org.apache.ignite.transactions.TransactionState.MARKED_ROLLBACK;
+import static org.apache.ignite.transactions.TransactionState.PREPARED;
 import static org.apache.ignite.transactions.TransactionState.PREPARING;
 import static org.apache.ignite.transactions.TransactionState.ROLLED_BACK;
 import static org.apache.ignite.transactions.TransactionState.ROLLING_BACK;
@@ -165,7 +166,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
     protected boolean transform;
 
     /** */
-    private boolean trackTimeout;
+    private final boolean trackTimeout;
 
     /** */
     @GridToStringExclude
@@ -175,7 +176,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
      * Empty constructor required for {@link Externalizable}.
      */
     public GridNearTxLocal() {
-        // No-op.
+        this.trackTimeout = false;
     }
 
     /**
@@ -228,8 +229,7 @@ public GridNearTxLocal(
 
         initResult();
 
-        if (timeout() > 0 && !implicit())
-            trackTimeout = cctx.time().addTimeoutObject(this);
+        trackTimeout = timeout() > 0 && !implicit() && cctx.time().addTimeoutObject(this);
     }
 
     /** {@inheritDoc} */
@@ -3155,8 +3155,13 @@ public IgniteInternalFuture<?> prepareNearTxLocal() {
             if (!PREP_FUT_UPD.compareAndSet(this, null, fut))
                 return prepFut;
 
-            if (trackTimeout)
-                removeTimeoutHandler();
+            if (trackTimeout) {
+                prepFut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
+                    @Override public void apply(IgniteInternalFuture<?> f) {
+                        GridNearTxLocal.this.removeTimeoutHandler();
+                    }
+                });
+            }
 
             if (timeout == -1) {
                 fut.onDone(this, timeoutException());
@@ -3215,7 +3220,7 @@ public IgniteInternalFuture<IgniteInternalTx> commitNearTxLocalAsync() {
         if (fastFinish()) {
             GridNearTxFastFinishFuture fut0;
 
-            if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, true)))
+            if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, true, false)))
                 return chainFinishFuture(finishFut, true);
 
             fut0.finish();
@@ -3238,7 +3243,9 @@ public IgniteInternalFuture<IgniteInternalTx> commitNearTxLocalAsync() {
                     // Make sure that here are no exceptions.
                     prepareFut.get();
 
-                    fut0.finish(true, true);
+                    TransactionState state = state();
+
+                    fut0.finish(state == PREPARED || state == COMMITTING || state == COMMITTED, true);
                 }
                 catch (Error | RuntimeException e) {
                     COMMIT_ERR_UPD.compareAndSet(GridNearTxLocal.this, null, e);
@@ -3282,7 +3289,7 @@ public IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync() {
      * @param onTimeout {@code True} if rolled back asynchronously on timeout.
      * @return Rollback future.
      */
-    private IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final boolean onTimeout) {
+    public IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final boolean onTimeout) {
         if (log.isDebugEnabled())
             log.debug("Rolling back near tx: " + this);
 
@@ -3292,13 +3299,13 @@ private IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final bo
         NearTxFinishFuture fut = finishFut;
 
         if (fut != null)
-            return chainFinishFuture(finishFut, false);
+            return chainFinishFuture(finishFut, false, !onTimeout);
 
         if (fastFinish()) {
             GridNearTxFastFinishFuture fut0;
 
-            if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, false)))
-                return chainFinishFuture(finishFut, false);
+            if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, false, onTimeout)))
+                return chainFinishFuture(finishFut, false, !onTimeout);
 
             fut0.finish();
 
@@ -3308,7 +3315,7 @@ private IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final bo
         final GridNearTxFinishFuture fut0;
 
         if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFinishFuture<>(cctx, this, false)))
-            return chainFinishFuture(finishFut, false);
+            return chainFinishFuture(finishFut, false, !onTimeout);
 
         cctx.mvcc().addFuture(fut0, fut0.futureId());
 
@@ -3327,7 +3334,7 @@ private IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final bo
 
             fut0.finish(false, !onTimeout);
         }
-        else {
+        else if (!onTimeout) {
             prepFut.listen(new CI1<IgniteInternalFuture<?>>() {
                 @Override public void apply(IgniteInternalFuture<?> f) {
                     try {
@@ -3339,10 +3346,11 @@ private IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final bo
                             log.debug("Got optimistic tx failure [tx=" + this + ", err=" + e + ']');
                     }
 
-                    fut0.finish(false, !onTimeout);
+                    fut0.finish(false, true);
                 }
             });
-        }
+        } else
+            fut0.finish(false, false);
 
         return fut0;
     }
@@ -3352,12 +3360,17 @@ private IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final bo
         return rollbackNearTxLocalAsync();
     }
 
+    /** */
+    private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFinishFuture fut, final boolean commit) {
+        return chainFinishFuture(fut, commit, true);
+    }
+
     /**
      * @param fut Already started finish future.
      * @param commit Commit flag.
      * @return Finish future.
      */
-    private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFinishFuture fut, final boolean commit) {
+    private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFinishFuture fut, final boolean commit, final boolean clearThreadMap) {
         assert fut != null;
 
         if (fut.commit() != commit) {
@@ -3381,7 +3394,7 @@ private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFin
                                 if (!cctx.mvcc().addFuture(rollbackFut, rollbackFut.futureId()))
                                     return;
 
-                                rollbackFut.finish(false, true);
+                                rollbackFut.finish(false, clearThreadMap);
                             }
                         }
                     }
@@ -4161,7 +4174,13 @@ public boolean addTimeoutHandler() {
             }
         }
 
-        if (state(MARKED_ROLLBACK, true) || (state() == MARKED_ROLLBACK)) {
+        boolean proceed;
+
+        synchronized (this) {
+            proceed = state() != PREPARED && state(MARKED_ROLLBACK, true);
+        }
+
+        if (proceed || (state() == MARKED_ROLLBACK)) {
             cctx.kernalContext().closure().runLocalSafe(new Runnable() {
                 @Override public void run() {
                     // Note: if rollback asynchronously on timeout should not clear thread map
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 9fb87770df64d..7fa31bf027cbc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -1356,7 +1356,7 @@ else if (log.isDebugEnabled())
      * @param tx Transaction to finish.
      * @param commit {@code True} if transaction is committed, {@code false} if rolled back.
      */
-    public void fastFinishTx(GridNearTxLocal tx, boolean commit) {
+    public void fastFinishTx(GridNearTxLocal tx, boolean commit, boolean clearThreadMap) {
         assert tx != null;
         assert tx.writeMap().isEmpty();
         assert tx.optimistic() || tx.readMap().isEmpty();
@@ -1377,7 +1377,8 @@ public void fastFinishTx(GridNearTxLocal tx, boolean commit) {
             removeObsolete(tx);
 
             // 4. Remove from per-thread storage.
-            clearThreadMap(tx);
+            if (clearThreadMap)
+                clearThreadMap(tx);
 
             // 5. Clear context.
             resetContext();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
index c5be40ee414ee..97de81edc781d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
@@ -439,6 +439,8 @@ public void testRandomMixedTxConfigurations() throws Exception {
                         if (delay > 0)
                             sleep(delay);
 
+                        assert v != null;
+
                         node.cache(CACHE_NAME).put(k, v + 1);
 
                         tx.commit();

From e394693a7389b4daff328827abdb1dcd28783f66 Mon Sep 17 00:00:00 2001
From: Maxim Muzafarov <maxmuzaf@gmail.com>
Date: Tue, 17 Apr 2018 21:18:36 +0300
Subject: [PATCH 072/543] IGNITE-8301 testReconnectCacheDestroyedAndCreated
 should excpect recreated client cache - Fixes #3856.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit 56be24b)
---
 .../ignite/internal/IgniteClientReconnectCacheTest.java     | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
index 3cb82e07cab14..ec5eab2561b63 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientReconnectCacheTest.java
@@ -894,12 +894,6 @@ public void testReconnectCacheDestroyedAndCreated() throws Exception {
             }
         });
 
-        GridTestUtils.assertThrows(log, new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                return clientCache.get(1);
-            }
-        }, IllegalStateException.class, null);
-
         checkCacheDiscoveryData(srv, client, DEFAULT_CACHE_NAME, true, false, false);
 
         IgniteCache<Object, Object> clientCache0 = client.cache(DEFAULT_CACHE_NAME);

From 4685ebe5f5dda4023980398806e222fada895e26 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Wed, 18 Apr 2018 10:44:44 +0700
Subject: [PATCH 073/543] IGNITE-8140 Web Console: Fixed code generation for
 large numbers in configuration params.

(cherry picked from commit eda5fe7)
---
 .../generator/ConfigurationGenerator.js       | 106 +++++++++---------
 1 file changed, 53 insertions(+), 53 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index 45d9ad10aacdf..c5f82d3b80841 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -720,8 +720,8 @@ export default class IgniteConfigurationGenerator {
                         .intProperty('connectionTimeout')
                         .intProperty('requestTimeout')
                         .stringProperty('signerOverride')
-                        .intProperty('connectionTTL')
-                        .intProperty('connectionMaxIdleMillis')
+                        .longProperty('connectionTTL')
+                        .longProperty('connectionMaxIdleMillis')
                         .emptyBeanProperty('dnsResolver')
                         .intProperty('responseMetadataCacheSize')
                         .emptyBeanProperty('secureRandom')
@@ -790,7 +790,7 @@ export default class IgniteConfigurationGenerator {
         if (!available('2.3.0'))
             return cfg;
 
-        cfg.intProperty('longQueryWarningTimeout');
+        cfg.longProperty('longQueryWarningTimeout');
 
         if (_.get(cluster, 'clientConnectorConfiguration.enabled') !== true)
             return cfg;
@@ -837,7 +837,7 @@ export default class IgniteConfigurationGenerator {
 
                 colSpi.intProperty('activeJobsThreshold')
                     .intProperty('waitJobsThreshold')
-                    .intProperty('messageExpireTime')
+                    .longProperty('messageExpireTime')
                     .intProperty('maximumStealingAttempts')
                     .boolProperty('stealingEnabled')
                     .emptyBeanProperty('externalCollisionListener')
@@ -892,9 +892,9 @@ export default class IgniteConfigurationGenerator {
             .intProperty('sharedMemoryPort')
             .intProperty('directBuffer')
             .intProperty('directSendBuffer')
-            .intProperty('idleConnectionTimeout')
-            .intProperty('connectTimeout')
-            .intProperty('maxConnectTimeout')
+            .longProperty('idleConnectionTimeout')
+            .longProperty('connectTimeout')
+            .longProperty('maxConnectTimeout')
             .intProperty('reconnectCount')
             .intProperty('socketSendBuffer')
             .intProperty('socketReceiveBuffer')
@@ -903,19 +903,19 @@ export default class IgniteConfigurationGenerator {
             .intProperty('tcpNoDelay')
             .intProperty('ackSendThreshold')
             .intProperty('unacknowledgedMessagesBufferSize')
-            .intProperty('socketWriteTimeout')
+            .longProperty('socketWriteTimeout')
             .intProperty('selectorsCount')
             .emptyBeanProperty('addressResolver');
 
         if (commSpi.nonEmpty())
             cfg.beanProperty('communicationSpi', commSpi);
 
-        cfg.intProperty('networkTimeout')
-            .intProperty('networkSendRetryDelay')
+        cfg.longProperty('networkTimeout')
+            .longProperty('networkSendRetryDelay')
             .intProperty('networkSendRetryCount');
 
         if (available(['1.0.0', '2.3.0']))
-            cfg.intProperty('discoveryStartupDelay');
+            cfg.longProperty('discoveryStartupDelay');
 
         return cfg;
     }
@@ -930,9 +930,9 @@ export default class IgniteConfigurationGenerator {
                 .stringProperty('host')
                 .intProperty('port')
                 .intProperty('portRange')
-                .intProperty('idleTimeout')
-                .intProperty('idleQueryCursorTimeout')
-                .intProperty('idleQueryCursorCheckFrequency')
+                .longProperty('idleTimeout')
+                .longProperty('idleQueryCursorTimeout')
+                .longProperty('idleQueryCursorCheckFrequency')
                 .intProperty('receiveBufferSize')
                 .intProperty('sendBufferSize')
                 .intProperty('sendQueueLimit')
@@ -1020,11 +1020,11 @@ export default class IgniteConfigurationGenerator {
             .intProperty('localPort')
             .intProperty('localPortRange')
             .emptyBeanProperty('addressResolver')
-            .intProperty('socketTimeout')
-            .intProperty('ackTimeout')
-            .intProperty('maxAckTimeout')
-            .intProperty('networkTimeout')
-            .intProperty('joinTimeout')
+            .longProperty('socketTimeout')
+            .longProperty('ackTimeout')
+            .longProperty('maxAckTimeout')
+            .longProperty('networkTimeout')
+            .longProperty('joinTimeout')
             .intProperty('threadPriority');
 
         // Removed in ignite 2.0
@@ -1034,13 +1034,13 @@ export default class IgniteConfigurationGenerator {
                 .intProperty('maxMissedClientHeartbeats');
         }
 
-        discoSpi.intProperty('topHistorySize')
+        discoSpi.longProperty('topHistorySize')
             .emptyBeanProperty('listener')
             .emptyBeanProperty('dataExchange')
             .emptyBeanProperty('metricsProvider')
             .intProperty('reconnectCount')
-            .intProperty('statisticsPrintFrequency')
-            .intProperty('ipFinderCleanFrequency')
+            .longProperty('statisticsPrintFrequency')
+            .longProperty('ipFinderCleanFrequency')
             .emptyBeanProperty('authenticator')
             .intProperty('forceServerMode')
             .intProperty('clientReconnectDisabled');
@@ -1090,8 +1090,8 @@ export default class IgniteConfigurationGenerator {
             case 'Memory':
                 eventStorageBean = new Bean('org.apache.ignite.spi.eventstorage.memory.MemoryEventStorageSpi', 'eventStorage', eventStorage.Memory, clusterDflts.eventStorage.Memory);
 
-                eventStorageBean.intProperty('expireAgeMs')
-                    .intProperty('expireCount')
+                eventStorageBean.longProperty('expireAgeMs')
+                    .longProperty('expireCount')
                     .emptyBeanProperty('filter');
 
                 break;
@@ -1128,8 +1128,8 @@ export default class IgniteConfigurationGenerator {
 
         // Since ignite 2.0
         if (available('2.0.0')) {
-            cfg.intProperty('failureDetectionTimeout')
-                .intProperty('clientFailureDetectionTimeout');
+            cfg.longProperty('failureDetectionTimeout')
+                .longProperty('clientFailureDetectionTimeout');
         }
 
         _.forEach(cluster.failoverSpi, (spi) => {
@@ -1210,7 +1210,7 @@ export default class IgniteConfigurationGenerator {
         if (plannerBean)
             hadoopBean.beanProperty('mapReducePlanner', plannerBean);
 
-        hadoopBean.intProperty('finishedJobInfoTtl')
+        hadoopBean.longProperty('finishedJobInfoTtl')
             .intProperty('maxParallelTasks')
             .intProperty('maxTaskQueueSize')
             .arrayProperty('nativeLibraryNames', 'nativeLibraryNames', _.get(hadoop, 'nativeLibraryNames'));
@@ -1368,12 +1368,12 @@ export default class IgniteConfigurationGenerator {
 
         memoryBean.intProperty('pageSize')
             .intProperty('concurrencyLevel')
-            .intProperty('systemCacheInitialSize')
-            .intProperty('systemCacheMaxSize')
+            .longProperty('systemCacheInitialSize')
+            .longProperty('systemCacheMaxSize')
             .stringProperty('defaultMemoryPolicyName');
 
         if (memoryBean.valueOf('defaultMemoryPolicyName') === 'default')
-            memoryBean.intProperty('defaultMemoryPolicySize');
+            memoryBean.longProperty('defaultMemoryPolicySize');
 
         const policies = [];
 
@@ -1438,8 +1438,8 @@ export default class IgniteConfigurationGenerator {
 
         storageBean.intProperty('pageSize')
             .intProperty('concurrencyLevel')
-            .intProperty('systemRegionInitialSize')
-            .intProperty('systemRegionMaxSize');
+            .longProperty('systemRegionInitialSize')
+            .longProperty('systemRegionMaxSize');
 
         const dfltDataRegionCfg = this.dataRegionConfiguration(_.get(dataStorageCfg, 'defaultDataRegionConfiguration'));
 
@@ -1461,7 +1461,7 @@ export default class IgniteConfigurationGenerator {
             storageBean.varArgProperty('dataRegionConfigurations', 'dataRegionConfigurations', dataRegionCfgs, 'org.apache.ignite.configuration.DataRegionConfiguration');
 
         storageBean.stringProperty('storagePath')
-            .intProperty('checkpointFrequency')
+            .longProperty('checkpointFrequency')
             .intProperty('checkpointThreads')
             .enumProperty('checkpointWriteOrder')
             .enumProperty('walMode')
@@ -1480,7 +1480,7 @@ export default class IgniteConfigurationGenerator {
             .longProperty('lockWaitTime')
             .intProperty('walThreadLocalBufferSize')
             .intProperty('metricsSubIntervalCount')
-            .intProperty('metricsRateTimeInterval')
+            .longProperty('metricsRateTimeInterval')
             .longProperty('walAutoArchiveAfterInactivity')
             .boolProperty('metricsEnabled')
             .boolProperty('alwaysWriteFullPages')
@@ -1594,13 +1594,13 @@ export default class IgniteConfigurationGenerator {
 
     // Generate metrics group.
     static clusterMetrics(cluster, available, cfg = this.igniteConfigurationBean(cluster)) {
-        cfg.intProperty('metricsExpireTime')
+        cfg.longProperty('metricsExpireTime')
             .intProperty('metricsHistorySize')
-            .intProperty('metricsLogFrequency');
+            .longProperty('metricsLogFrequency');
 
         // Since ignite 2.0
         if (available('2.0.0'))
-            cfg.intProperty('metricsUpdateFrequency');
+            cfg.longProperty('metricsUpdateFrequency');
 
         return cfg;
     }
@@ -1630,7 +1630,7 @@ export default class IgniteConfigurationGenerator {
         if (!available('2.1.0'))
             return cfg;
 
-        cfg.intProperty('longQueryWarningTimeout');
+        cfg.longProperty('longQueryWarningTimeout');
 
         if (_.get(cluster, 'sqlConnectorConfiguration.enabled') !== true)
             return cfg;
@@ -1663,7 +1663,7 @@ export default class IgniteConfigurationGenerator {
         bean.stringProperty('persistentStorePath')
             .boolProperty('metricsEnabled')
             .boolProperty('alwaysWriteFullPages')
-            .intProperty('checkpointingFrequency')
+            .longProperty('checkpointingFrequency')
             .longProperty('checkpointingPageBufferSize')
             .intProperty('checkpointingThreads')
             .stringProperty('walStorePath')
@@ -1675,7 +1675,7 @@ export default class IgniteConfigurationGenerator {
             .longProperty('walFsyncDelayNanos')
             .intProperty('walRecordIteratorBufferSize')
             .longProperty('lockWaitTime')
-            .intProperty('rateTimeInterval')
+            .longProperty('rateTimeInterval')
             .intProperty('tlbSize')
             .intProperty('subIntervals');
 
@@ -1782,7 +1782,7 @@ export default class IgniteConfigurationGenerator {
             .intProperty('igfsThreadPoolSize')
             .intProperty('rebalanceThreadPoolSize')
             .intProperty('utilityCacheThreadPoolSize', 'utilityCachePoolSize')
-            .intProperty('utilityCacheKeepAliveTime')
+            .longProperty('utilityCacheKeepAliveTime')
             .intProperty('asyncCallbackPoolSize')
             .intProperty('stripedPoolSize');
 
@@ -1817,7 +1817,7 @@ export default class IgniteConfigurationGenerator {
 
         bean.enumProperty('defaultTxConcurrency')
             .enumProperty('defaultTxIsolation')
-            .intProperty('defaultTxTimeout')
+            .longProperty('defaultTxTimeout')
             .intProperty('pessimisticTxLogLinger')
             .intProperty('pessimisticTxLogSize')
             .boolProperty('txSerializableEnabled')
@@ -2070,7 +2070,7 @@ export default class IgniteConfigurationGenerator {
             ccfg.enumProperty('memoryMode');
 
             if (ccfg.valueOf('memoryMode') !== 'OFFHEAP_VALUES')
-                ccfg.intProperty('offHeapMaxMemory');
+                ccfg.longProperty('offHeapMaxMemory');
         }
 
         // Since ignite 2.0
@@ -2105,7 +2105,7 @@ export default class IgniteConfigurationGenerator {
         if (available(['1.0.0', '2.0.0']))
             ccfg.intProperty('sqlOnheapRowCacheSize');
 
-        ccfg.intProperty('longQueryWarningTimeout')
+        ccfg.longProperty('longQueryWarningTimeout')
             .arrayProperty('indexedTypes', 'indexedTypes', indexedTypes, 'java.lang.Class')
             .intProperty('queryDetailMetricsSize')
             .arrayProperty('sqlFunctionClasses', 'sqlFunctionClasses', cache.sqlFunctionClasses, 'java.lang.Class');
@@ -2225,7 +2225,7 @@ export default class IgniteConfigurationGenerator {
             ccfg.boolProperty('writeBehindEnabled')
                 .intProperty('writeBehindBatchSize')
                 .intProperty('writeBehindFlushSize')
-                .intProperty('writeBehindFlushFrequency')
+                .longProperty('writeBehindFlushFrequency')
                 .intProperty('writeBehindFlushThreadCount');
 
             // Since ignite 2.0
@@ -2239,7 +2239,7 @@ export default class IgniteConfigurationGenerator {
     // Generate cache concurrency control group.
     static cacheConcurrency(cache, available, ccfg = this.cacheConfigurationBean(cache)) {
         ccfg.intProperty('maxConcurrentAsyncOperations')
-            .intProperty('defaultLockTimeout');
+            .longProperty('defaultLockTimeout');
 
         // Removed in ignite 2.0
         if (available(['1.0.0', '2.0.0']))
@@ -2297,11 +2297,11 @@ export default class IgniteConfigurationGenerator {
             ccfg.enumProperty('rebalanceMode')
                 .intProperty('rebalanceThreadPoolSize')
                 .intProperty('rebalanceBatchSize')
-                .intProperty('rebalanceBatchesPrefetchCount')
+                .longProperty('rebalanceBatchesPrefetchCount')
                 .intProperty('rebalanceOrder')
-                .intProperty('rebalanceDelay')
-                .intProperty('rebalanceTimeout')
-                .intProperty('rebalanceThrottle');
+                .longProperty('rebalanceDelay')
+                .longProperty('rebalanceTimeout')
+                .longProperty('rebalanceThrottle');
         }
 
         if (ccfg.includes('igfsAffinnityGroupSize')) {
@@ -2456,8 +2456,8 @@ export default class IgniteConfigurationGenerator {
     static igfsFragmentizer(igfs, cfg = this.igfsConfigurationBean(igfs)) {
         if (igfs.fragmentizerEnabled) {
             cfg.intProperty('fragmentizerConcurrentFiles')
-                .intProperty('fragmentizerThrottlingBlockLength')
-                .intProperty('fragmentizerThrottlingDelay');
+                .longProperty('fragmentizerThrottlingBlockLength')
+                .longProperty('fragmentizerThrottlingDelay');
         }
         else
             cfg.boolProperty('fragmentizerEnabled');
@@ -2490,7 +2490,7 @@ export default class IgniteConfigurationGenerator {
         if (available(['1.0.0', '2.0.0']))
             cfg.intProperty('maxSpaceSize');
 
-        cfg.intProperty('maximumTaskRangeLength')
+        cfg.longProperty('maximumTaskRangeLength')
             .intProperty('managementPort')
             .intProperty('perNodeBatchSize')
             .intProperty('perNodeParallelBatchCount')

From 5efc589fcabffdb29cd6dfe0e7323bc91db47703 Mon Sep 17 00:00:00 2001
From: Ilya Borisov <klaster1@gmail.com>
Date: Wed, 18 Apr 2018 11:39:41 +0700
Subject: [PATCH 074/543] IGNITE-8294 Web Console: Move "Beta" ribbon to the
 left.

(cherry picked from commit 69606e4)
---
 .../web-console/frontend/public/stylesheets/style.scss | 10 ++++++++++
 modules/web-console/frontend/views/index.pug           |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index ae1e58c41a769..0c978e9ca43aa 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -1938,6 +1938,16 @@ treecontrol.tree-classic {
     }
 }
 
+.ribbon-wrapper.left {
+    overflow: visible;
+
+    .ribbon {
+        transform: rotate(-45deg);
+        left: -75px;
+        top: 10px;
+    }
+}
+
 html, body {
     width: 100%;
     min-height: 100vh;
diff --git a/modules/web-console/frontend/views/index.pug b/modules/web-console/frontend/views/index.pug
index e91af9bda889f..6384592156c77 100644
--- a/modules/web-console/frontend/views/index.pug
+++ b/modules/web-console/frontend/views/index.pug
@@ -39,7 +39,7 @@ html(ng-app='ignite-console' id='app' ng-strict-di)
 
                 .splash-wellcome Loading...
 
-        .ribbon-wrapper.right(ng-if='!IgniteDemoMode')
+        .ribbon-wrapper.left(ng-if='!IgniteDemoMode')
             .ribbon
                 label Beta
 

From af46856d7f7ec88aa663865a108900abb8314ffe Mon Sep 17 00:00:00 2001
From: oleg-ostanin <oostanin@gridagin.com>
Date: Wed, 18 Apr 2018 00:58:53 +0700
Subject: [PATCH 075/543] IGNITE-8274 sqlline.sh script uses JAVA_HOME now

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit c3ff274)
---
 modules/sqlline/bin/sqlline.sh | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/modules/sqlline/bin/sqlline.sh b/modules/sqlline/bin/sqlline.sh
index 5745aea489a1c..552440de5d185 100644
--- a/modules/sqlline/bin/sqlline.sh
+++ b/modules/sqlline/bin/sqlline.sh
@@ -35,6 +35,11 @@ SCRIPTS_HOME="${IGNITE_HOME_TMP}/bin"
 
 source "${SCRIPTS_HOME}"/include/functions.sh
 
+#
+# Discover path to Java executable and check it's version.
+#
+checkJava
+
 #
 # Discover IGNITE_HOME environment variable.
 #
@@ -51,4 +56,4 @@ CP="${IGNITE_LIBS}"
 
 CP="${CP}${SEP}${IGNITE_HOME_TMP}/bin/include/sqlline/*"
 
-java -cp ${CP} sqlline.SqlLine -d org.apache.ignite.IgniteJdbcThinDriver $@
\ No newline at end of file
+"$JAVA" -cp ${CP} sqlline.SqlLine -d org.apache.ignite.IgniteJdbcThinDriver $@
\ No newline at end of file

From ba4e337068ebfcc4bbb93509166f118225fe0cb4 Mon Sep 17 00:00:00 2001
From: Dmitriy Shabalin <dmitriyff@gmail.com>
Date: Wed, 18 Apr 2018 18:43:13 +0700
Subject: [PATCH 076/543] IGNITE-8298 Web Console: Fixed tables UI issues.

(cherry picked from commit a050436)
---
 .../app/primitives/ui-grid-header/index.scss  | 33 +++++++-
 .../app/primitives/ui-grid/index.scss         | 11 ++-
 .../web-console/frontend/package-lock.json    | 80 +++++++++----------
 3 files changed, 80 insertions(+), 44 deletions(-)

diff --git a/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss b/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss
index ac58707233bd2..1ff27b29c05dd 100644
--- a/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss
+++ b/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss
@@ -33,6 +33,9 @@
     }
 
     .ui-grid-header-cell {
+        // Workaround: Fixed cell header offset in IE11.
+        vertical-align: top;
+
         .ui-grid-cell-contents > span:not(.ui-grid-header-cell-label) {
             right: 3px;
         }
@@ -44,7 +47,7 @@
 
     .ui-grid-header-cell [role="columnheader"] {
         display: flex;
-        
+
         flex-wrap: wrap;
         align-items: center;
         justify-content: center;
@@ -77,15 +80,29 @@
         .ui-grid-column-resizer.right {
             top: -100px;
         }
+
         .ng-hide + .ui-grid-header-cell-row .ui-grid-column-resizer.right {
             bottom: 0;
         }
 
+        &.ui-grid-header-cell:not(:first-child) {
+            left: 0;
+            box-shadow: -1px 0px 0 0 #d4d4d4;
+        }
+
         &.ui-grid-header-cell .ui-grid-header-cell .ui-grid-column-resizer.right {
             border-right-width: 0;
         }
+
         &.ui-grid-header-cell .ui-grid-header-cell:last-child .ui-grid-column-resizer.right {
-            border-right-width: 1px;
+            // Hide all right borders, and fix cell offset.
+            right: -1px;
+            border: none;
+        }
+
+        &.ui-grid-header-cell [ng-show] .ui-grid-cell-contents {
+            text-indent: -20px;
+            margin-right: -20px;
         }
 
         & > div > .ui-grid-cell-contents {
@@ -97,3 +114,15 @@
         line-height: 21px;
     }
 }
+
+.ui-grid[ui-grid-selection][ui-grid-grouping] {
+    .ui-grid-pinned-container-left {
+        .ui-grid-header--subcategories {
+            .ui-grid-header-span {
+                &.ui-grid-header-cell {
+                    box-shadow: none;
+                }
+            }
+        }
+    }
+}
diff --git a/modules/web-console/frontend/app/primitives/ui-grid/index.scss b/modules/web-console/frontend/app/primitives/ui-grid/index.scss
index b19d5f7af3bbf..2a5c5872a5917 100644
--- a/modules/web-console/frontend/app/primitives/ui-grid/index.scss
+++ b/modules/web-console/frontend/app/primitives/ui-grid/index.scss
@@ -148,12 +148,12 @@
 
     .ui-grid-header--subcategories {
         .ui-grid-header-span.ui-grid-header-cell {
+            background: initial;
+
             .ui-grid-cell-contents {
                 padding: 8px 20px;
             }
 
-            background: initial;
-
             [ng-show] .ui-grid-cell-contents {
                 text-align: center;
             }
@@ -464,6 +464,7 @@
         color: #393939;
         text-align: left;
         font-size: 14px;
+        font-weight: normal;
         line-height: 1.42857;
 
         box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075);
@@ -538,3 +539,9 @@
     font-style: italic;
     line-height: 16px;
 }
+
+.ui-grid {
+    input[type="text"].ui-grid-filter-input {
+        font-weight: normal;
+    }
+}
diff --git a/modules/web-console/frontend/package-lock.json b/modules/web-console/frontend/package-lock.json
index e28ef4d01c123..1fec909e3860e 100644
--- a/modules/web-console/frontend/package-lock.json
+++ b/modules/web-console/frontend/package-lock.json
@@ -401,9 +401,9 @@
       "integrity": "sha512-/2xvG6vDC+Us8h0baSa1siDKwPj5R2A7LldxxhK2339HInc09bq9shMVCUy9zqnuvwnDUJ/DSgkSaBoSHSZrqg=="
     },
     "angular-mocks": {
-      "version": "1.6.9",
-      "resolved": "https://registry.npmjs.org/angular-mocks/-/angular-mocks-1.6.9.tgz",
-      "integrity": "sha512-5aEwXmfd5DQvb64pOgP2W2D3ozAQSARkB6q+6NQfUJvJs9bD2YcExrUc1P4EbiIuyWag2OQM+pIKUNojVi3SBg==",
+      "version": "1.6.10",
+      "resolved": "https://registry.npmjs.org/angular-mocks/-/angular-mocks-1.6.10.tgz",
+      "integrity": "sha512-1865/NmqHNogibNoglY1MGBjx882iu2hI46BBhYDWyz0C4TDM5ER8H8SnYwQKUUG4RXMDsJizszEQ2BEoYKV9w==",
       "dev": true
     },
     "angular-motion": {
@@ -1859,9 +1859,9 @@
       "integrity": "sha1-c5JncZI7Whl0etZmqlzUv5xunOg="
     },
     "base64-js": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.2.3.tgz",
-      "integrity": "sha512-MsAhsUW1GxCdgYSO6tAfZrNapmUKk7mWx/k5mFY/A1gBtkaCaNapTg+FExCw1r9yeaZhqx/xPg43xgTFH6KL5w=="
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz",
+      "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw=="
     },
     "base64id": {
       "version": "1.0.0",
@@ -2120,7 +2120,7 @@
           "integrity": "sha512-YkIRgwsZwJWTnyQrsBTWefizHh+8GYj3kbL1BTiAQ/9pwpino0G7B2gp5tx/FUBqUlvtxV85KNR3mwfAtv15Yw==",
           "dev": true,
           "requires": {
-            "base64-js": "1.2.3",
+            "base64-js": "1.3.0",
             "ieee754": "1.1.11"
           }
         },
@@ -2278,7 +2278,7 @@
       "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
       "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
       "requires": {
-        "base64-js": "1.2.3",
+        "base64-js": "1.3.0",
         "ieee754": "1.1.11",
         "isarray": "1.0.0"
       }
@@ -3006,9 +3006,9 @@
       }
     },
     "commander": {
-      "version": "2.15.1",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz",
-      "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag=="
+      "version": "2.13.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.13.0.tgz",
+      "integrity": "sha512-MVuS359B+YzaWqjCL/c+22gfryv+mCBPHAv3zyVI2GN8EY6IRP8VwtasXn8jyyhvvq84R4ImN1OKRtcbIasjYA=="
     },
     "commondir": {
       "version": "1.0.1",
@@ -3192,7 +3192,7 @@
         "loader-utils": "1.1.0",
         "minimatch": "3.0.4",
         "p-limit": "1.2.0",
-        "serialize-javascript": "1.4.0"
+        "serialize-javascript": "1.5.0"
       },
       "dependencies": {
         "globby": {
@@ -3517,9 +3517,9 @@
       "integrity": "sha1-vEZ0gAQ3iyGjYMn8fPUjF5B2L7g="
     },
     "d3-color": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.0.3.tgz",
-      "integrity": "sha1-vHZD/KjlOoNH4vva/6I2eWtYUJs="
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.1.0.tgz",
+      "integrity": "sha512-IZVcqX5yYFvR2NUBbSfIfbgNcSgAtZ7JbgQWqDXf4CywtN7agvI7Kw6+Q1ETvlHOHWJT55Kyuzt0C3I0GVtRHQ=="
     },
     "d3-hierarchy": {
       "version": "1.1.6",
@@ -3531,7 +3531,7 @@
       "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-1.1.6.tgz",
       "integrity": "sha512-mOnv5a+pZzkNIHtw/V6I+w9Lqm9L5bG3OTXPM5A+QO0yyVMQ4W1uZhR+VOJmazaOZXri2ppbiZ5BUNWT0pFM9A==",
       "requires": {
-        "d3-color": "1.0.3"
+        "d3-color": "1.1.0"
       }
     },
     "dargs": {
@@ -5568,7 +5568,7 @@
       "integrity": "sha1-zcvAgYgmWtEZtqWnyKtw7s+10n0=",
       "requires": {
         "chalk": "1.1.3",
-        "commander": "2.15.1",
+        "commander": "2.13.0",
         "is-my-json-valid": "2.17.2",
         "pinkie-promise": "2.0.1"
       },
@@ -5825,7 +5825,7 @@
       "integrity": "sha512-KcuaIRWTU0kFjOJCs32a3JsGNCWkeOak0/F/uvJNp3x/N4McXdqHpcK64cYTozK7QLPKKtUqb9h7wR9K9rYRkg==",
       "requires": {
         "@posthtml/esm": "1.0.0",
-        "htmlnano": "0.1.7",
+        "htmlnano": "0.1.8",
         "loader-utils": "1.1.0",
         "posthtml": "0.11.3",
         "schema-utils": "0.4.5"
@@ -5845,6 +5845,11 @@
         "uglify-js": "3.3.21"
       },
       "dependencies": {
+        "commander": {
+          "version": "2.15.1",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz",
+          "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag=="
+        },
         "source-map": {
           "version": "0.6.1",
           "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
@@ -5895,16 +5900,16 @@
       "dev": true
     },
     "htmlnano": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/htmlnano/-/htmlnano-0.1.7.tgz",
-      "integrity": "sha512-t8Gy+r/loFP2VXAJl6ClaNIomGI609oyQcT7O3IoJE6VcDCLR6PYWXaSh+hfd/dnoZ6KPbpgPek/Crm3havqig==",
+      "version": "0.1.8",
+      "resolved": "https://registry.npmjs.org/htmlnano/-/htmlnano-0.1.8.tgz",
+      "integrity": "sha512-wwmDRJn5OQ9BqFYy5vWaufUQTKj7Ct6xTv+od7QNNJzJM7K3yqR4lJ8SHSOTcBahlXMO5EzddUdsS+fmdGvXpw==",
       "requires": {
         "cssnano": "3.10.0",
         "object-assign": "4.1.1",
         "posthtml": "0.11.3",
         "posthtml-render": "1.1.3",
         "svgo": "1.0.5",
-        "uglify-js": "3.3.21"
+        "uglify-es": "3.3.9"
       },
       "dependencies": {
         "coa": {
@@ -5969,12 +5974,12 @@
             "util.promisify": "1.0.0"
           }
         },
-        "uglify-js": {
-          "version": "3.3.21",
-          "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.21.tgz",
-          "integrity": "sha512-uy82472lH8tshK3jS3c5IFb5MmNKd/5qyBd0ih8sM42L3jWvxnE339U9gZU1zufnLVs98Stib9twq8dLm2XYCA==",
+        "uglify-es": {
+          "version": "3.3.9",
+          "resolved": "https://registry.npmjs.org/uglify-es/-/uglify-es-3.3.9.tgz",
+          "integrity": "sha512-r+MU0rfv4L/0eeW3xZrd16t4NZfK8Ld4SWVglYBb7ez5uXFWHuVRs6xCTrf1yirs9a4j4Y27nn7SRfO6v67XsQ==",
           "requires": {
-            "commander": "2.15.1",
+            "commander": "2.13.0",
             "source-map": "0.6.1"
           },
           "dependencies": {
@@ -8538,7 +8543,7 @@
         "stream-browserify": "2.0.1",
         "stream-http": "2.8.1",
         "string_decoder": "1.1.1",
-        "timers-browserify": "2.0.7",
+        "timers-browserify": "2.0.9",
         "tty-browserify": "0.0.0",
         "url": "0.11.0",
         "util": "0.10.3",
@@ -11273,9 +11278,9 @@
       }
     },
     "serialize-javascript": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.4.0.tgz",
-      "integrity": "sha1-fJWFFNtqwkQ6irwGLcn3iGp/YAU="
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.5.0.tgz",
+      "integrity": "sha512-Ga8c8NjAAp46Br4+0oZ2WxJCwIzwP60Gq1YPgU+39PiTVxyed/iKE/zyZI6+UlVYH5Q4PaQdHhcegIFPZTUfoQ=="
     },
     "serve-index": {
       "version": "1.9.1",
@@ -12528,9 +12533,9 @@
       "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8="
     },
     "timers-browserify": {
-      "version": "2.0.7",
-      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.7.tgz",
-      "integrity": "sha512-U7DtjfsHeYjNAyEz4MdCLGZMY3ySyHIgZZp6ba9uxZlMRMiK5yTHUYc2XfGQHKFgxGcmvBF2jafoNtQYvlDpOw==",
+      "version": "2.0.9",
+      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.9.tgz",
+      "integrity": "sha512-2DhyvVpCWwY7gk8UmKhYvgHQl9XTlO0Dg0/2UZcLgPnpulhdm2aGIlFy5rU5igmOCA51w6jPHqLRA4UH1YmhcA==",
       "requires": {
         "setimmediate": "1.0.5"
       }
@@ -12790,18 +12795,13 @@
         "cacache": "10.0.4",
         "find-cache-dir": "1.0.0",
         "schema-utils": "0.4.5",
-        "serialize-javascript": "1.4.0",
+        "serialize-javascript": "1.5.0",
         "source-map": "0.6.1",
         "uglify-es": "3.3.9",
         "webpack-sources": "1.1.0",
         "worker-farm": "1.6.0"
       },
       "dependencies": {
-        "commander": {
-          "version": "2.13.0",
-          "resolved": "https://registry.npmjs.org/commander/-/commander-2.13.0.tgz",
-          "integrity": "sha512-MVuS359B+YzaWqjCL/c+22gfryv+mCBPHAv3zyVI2GN8EY6IRP8VwtasXn8jyyhvvq84R4ImN1OKRtcbIasjYA=="
-        },
         "source-map": {
           "version": "0.6.1",
           "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",

From 25b25f271bc89d77013e1cda2bad30d941e1c8ad Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 18 Apr 2018 15:57:45 +0300
Subject: [PATCH 077/543] IGNITE-8122 Restore partition state from WAL if no
 checkpoints are done - Fixes #3745.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../affinity/GridAffinityAssignment.java      |   4 +-
 .../processors/cache/ExchangeActions.java     |   2 +-
 .../dht/GridDhtPartitionTopology.java         |   3 +-
 .../dht/GridDhtPartitionTopologyImpl.java     | 220 ++++++++----------
 .../dht/GridDhtPartitionsStateValidator.java  |  61 ++++-
 .../GridDhtPartitionsExchangeFuture.java      |   6 +-
 .../GridDhtPartitionsSingleMessage.java       |  18 +-
 .../dht/preloader/GridDhtPreloader.java       |   4 +-
 .../GridCacheDatabaseSharedManager.java       |  89 +++----
 .../CacheBaselineTopologyTest.java            |  16 +-
 ...cheLoadingConcurrentGridStartSelfTest.java |   3 +
 ...ridCachePartitionsStateValidationTest.java |   4 +
 ...CachePartitionsStateValidatorSelfTest.java |  47 ++--
 .../testsuites/IgniteStandByClusterSuite.java |   2 +-
 .../IgniteCacheQueryNodeRestartSelfTest.java  |   2 +
 ...BaselineCacheQueryNodeRestartSelfTest.java |   4 +-
 16 files changed, 258 insertions(+), 227 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
index 6da6aaa5c2d6d..cbec1a1852eba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignment.java
@@ -149,7 +149,7 @@ public AffinityTopologyVersion topologyVersion() {
      * @param part Partition.
      * @return Affinity nodes.
      */
-    public List<ClusterNode> get(int part) {
+    @Override public List<ClusterNode> get(int part) {
         assert part >= 0 && part < assignment.size() : "Affinity partition is out of range" +
             " [part=" + part + ", partitions=" + assignment.size() + ']';
 
@@ -162,7 +162,7 @@ public List<ClusterNode> get(int part) {
      * @param part Partition.
      * @return Affinity nodes IDs.
      */
-    public HashSet<UUID> getIds(int part) {
+    @Override public HashSet<UUID> getIds(int part) {
         assert part >= 0 && part < assignment.size() : "Affinity partition is out of range" +
             " [part=" + part + ", partitions=" + assignment.size() + ']';
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
index bcf3f408c222f..c289b6e31e7f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
@@ -409,7 +409,7 @@ public DynamicCacheDescriptor descriptor() {
     /**
      *
      */
-    static class CacheGroupActionData {
+    public static class CacheGroupActionData {
         /** */
         private final CacheGroupDescriptor desc;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 6f68dbbaeb511..d586a94d53203 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -386,8 +386,9 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
      * State of all current owners that aren't contained in the set will be reset to MOVING.
      *
      * @param p Partition ID.
-     * @param updateSeq If should increment sequence when updated.
      * @param owners Set of new owners.
+     * @param haveHistory {@code True} if there is WAL history to rebalance given partition.
+     * @param updateSeq If should increment sequence when updated.
      * @return Set of node IDs that should reload partitions.
      */
     public Set<UUID> setOwners(int p, Set<UUID> owners, boolean haveHistory, boolean updateSeq);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 740903e415baf..164f0bf451286 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht;
 
-import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -31,8 +30,6 @@
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReferenceArray;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
@@ -43,7 +40,6 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
-import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
@@ -55,8 +51,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.GridAtomicLong;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
@@ -324,7 +318,7 @@ private String mapString(GridDhtPartitionMap map) {
 
                 long updateSeq = this.updateSeq.incrementAndGet();
 
-                needRefresh = initPartitions0(affVer, exchFut, updateSeq);
+                needRefresh = initPartitions(affVer, grp.affinity().readyAssignments(affVer), exchFut, updateSeq);
 
                 consistencyCheck();
             }
@@ -340,14 +334,15 @@ private String mapString(GridDhtPartitionMap map) {
     }
 
     /**
+     * Creates and initializes partitions using given {@code affVer} and {@code affAssignment}.
+     *
      * @param affVer Affinity version to use.
+     * @param affAssignment Affinity assignment to use.
      * @param exchFut Exchange future.
      * @param updateSeq Update sequence.
      * @return {@code True} if partitions must be refreshed.
      */
-    private boolean initPartitions0(AffinityTopologyVersion affVer, GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
-        List<List<ClusterNode>> aff = grp.affinity().readyAssignments(affVer);
-
+    private boolean initPartitions(AffinityTopologyVersion affVer, List<List<ClusterNode>> affAssignment, GridDhtPartitionsExchangeFuture exchFut, long updateSeq) {
         boolean needRefresh = false;
 
         if (grp.affinityNode()) {
@@ -357,32 +352,24 @@ private boolean initPartitions0(AffinityTopologyVersion affVer, GridDhtPartition
 
             GridDhtPartitionExchangeId exchId = exchFut.exchangeId();
 
-            assert grp.affinity().lastVersion().equals(affVer) :
-                "Invalid affinity [topVer=" + grp.affinity().lastVersion() +
-                ", grp=" + grp.cacheOrGroupName() +
-                ", affVer=" + affVer +
-                ", fut=" + exchFut + ']';
-
             int num = grp.affinity().partitions();
 
             if (grp.rebalanceEnabled()) {
                 boolean added = exchFut.cacheGroupAddedOnExchange(grp.groupId(), grp.receivedFrom());
 
-                boolean first = added || (loc.equals(oldest) && loc.id().equals(exchId.nodeId()) && exchId.isJoined());
+                boolean first = added || (loc.equals(oldest) && loc.id().equals(exchId.nodeId()) && exchId.isJoined()) || exchFut.activateCluster();
 
                 if (first) {
-                    assert exchId.isJoined() || added;
+                    assert exchId.isJoined() || added || exchFut.activateCluster();
 
                     for (int p = 0; p < num; p++) {
-                        if (localNode(p, aff) || initLocalPartition(p, discoCache)) {
-                            GridDhtLocalPartition locPart = createPartition(p);
+                        if (localNode(p, affAssignment)) {
+                            // Partition is created first time, so it's safe to own it.
+                            boolean shouldOwn = locParts.get(p) == null;
 
-                            if (grp.persistenceEnabled()) {
-                                GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)grp.shared().database();
+                            GridDhtLocalPartition locPart = getOrCreatePartition(p);
 
-                                locPart.restoreState(db.readPartitionState(grp, locPart.id()));
-                            }
-                            else {
+                            if (shouldOwn) {
                                 boolean owned = locPart.own();
 
                                 assert owned : "Failed to own partition for oldest node [grp=" + grp.cacheOrGroupName() +
@@ -390,7 +377,7 @@ private boolean initPartitions0(AffinityTopologyVersion affVer, GridDhtPartition
 
                                 if (log.isDebugEnabled())
                                     log.debug("Owned partition for oldest node [grp=" + grp.cacheOrGroupName() +
-                                    ", part=" + locPart + ']');
+                                        ", part=" + locPart + ']');
                             }
 
                             needRefresh = true;
@@ -400,15 +387,15 @@ private boolean initPartitions0(AffinityTopologyVersion affVer, GridDhtPartition
                     }
                 }
                 else
-                    createPartitions(affVer, aff, updateSeq);
+                    createPartitions(affVer, affAssignment, updateSeq);
             }
             else {
                 // If preloader is disabled, then we simply clear out
                 // the partitions this node is not responsible for.
                 for (int p = 0; p < num; p++) {
-                    GridDhtLocalPartition locPart = localPartition0(p, affVer, false, true, false);
+                    GridDhtLocalPartition locPart = localPartition0(p, affVer, false, true);
 
-                    boolean belongs = localNode(p, aff);
+                    boolean belongs = localNode(p, affAssignment);
 
                     if (locPart != null) {
                         if (!belongs) {
@@ -429,7 +416,7 @@ private boolean initPartitions0(AffinityTopologyVersion affVer, GridDhtPartition
                             locPart.own();
                     }
                     else if (belongs) {
-                        locPart = createPartition(p);
+                        locPart = getOrCreatePartition(p);
 
                         locPart.own();
 
@@ -439,27 +426,14 @@ else if (belongs) {
             }
         }
 
-        updateRebalanceVersion(aff);
+        updateRebalanceVersion(affAssignment);
 
         return needRefresh;
     }
 
     /**
-     * @param p Partition ID to restore.
-     * @param discoCache Disco cache to use.
-     * @return {@code True} if should restore local partition.
-     */
-    private boolean initLocalPartition(int p, DiscoCache discoCache) {
-        IgnitePageStoreManager storeMgr = ctx.pageStore();
-
-        return
-            grp.persistenceEnabled() &&
-            storeMgr instanceof FilePageStoreManager &&
-            discoCache.baselineNode(ctx.localNodeId()) &&
-            Files.exists(((FilePageStoreManager)storeMgr).getPath(grp.sharedGroup(), grp.cacheOrGroupName(), p));
-    }
-
-    /**
+     * Creates non-existing partitions belong to given affinity {@code aff}.
+     *
      * @param affVer Affinity version.
      * @param aff Affinity assignments.
      * @param updateSeq Update sequence.
@@ -475,7 +449,7 @@ private void createPartitions(AffinityTopologyVersion affVer, List<List<ClusterN
                 if (localNode(p, aff)) {
                     // This will make sure that all non-existing partitions
                     // will be created in MOVING state.
-                    GridDhtLocalPartition locPart = createPartition(p);
+                    GridDhtLocalPartition locPart = getOrCreatePartition(p);
 
                     updateSeq = updateLocal(p, locPart.state(), updateSeq, affVer);
                 }
@@ -483,7 +457,7 @@ private void createPartitions(AffinityTopologyVersion affVer, List<List<ClusterN
             // If this node's map is empty, we pre-create local partitions,
             // so local map will be sent correctly during exchange.
             else if (localNode(p, aff))
-                createPartition(p);
+                getOrCreatePartition(p);
         }
     }
 
@@ -588,18 +562,29 @@ else if (!node2part.nodeId().equals(loc.id())) {
                         if (grpStarted ||
                             exchFut.firstEvent().type() == EVT_DISCOVERY_CUSTOM_EVT ||
                             exchFut.serverNodeDiscoveryEvent()) {
+
+                            AffinityTopologyVersion affVer;
+                            List<List<ClusterNode>> affAssignment;
+
                             if (affReady) {
-                                assert grp.affinity().lastVersion().equals(evts.topologyVersion());
+                                affVer = evts.topologyVersion();
 
-                                initPartitions0(evts.topologyVersion(), exchFut, updateSeq);
+                                assert grp.affinity().lastVersion().equals(affVer) :
+                                        "Invalid affinity [topVer=" + grp.affinity().lastVersion() +
+                                                ", grp=" + grp.cacheOrGroupName() +
+                                                ", affVer=" + affVer +
+                                                ", fut=" + exchFut + ']';
+
+                                affAssignment = grp.affinity().readyAssignments(affVer);
                             }
                             else {
                                 assert !exchFut.context().mergeExchanges();
 
-                                List<List<ClusterNode>> aff = grp.affinity().idealAssignment();
-
-                                createPartitions(exchFut.initialVersion(), aff, updateSeq);
+                                affVer = exchFut.initialVersion();
+                                affAssignment = grp.affinity().idealAssignment();
                             }
+
+                            initPartitions(affVer, affAssignment, exchFut, updateSeq);
                         }
                     }
 
@@ -695,19 +680,12 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
                 long updateSeq = this.updateSeq.incrementAndGet();
 
                 for (int p = 0; p < num; p++) {
-                    GridDhtLocalPartition locPart = localPartition0(p, topVer, false, false, false);
+                    GridDhtLocalPartition locPart = localPartition0(p, topVer, false, true);
 
                     if (partitionLocalNode(p, topVer)) {
-                        // This partition will be created during next topology event,
-                        // which obviously has not happened at this point.
-                        if (locPart == null) {
-                            if (log.isDebugEnabled()) {
-                                log.debug("Skipping local partition afterExchange (will not create) [" +
-                                    "grp=" + grp.cacheOrGroupName() + ", p=" + p + ']');
-                            }
-
-                            continue;
-                        }
+                        // Prepare partition to rebalance if it's not happened on full map update phase.
+                        if (locPart == null || locPart.state() == RENTING || locPart.state() == EVICTED)
+                            locPart = rebalancePartition(p, false);
 
                         GridDhtPartitionState state = locPart.state();
 
@@ -793,20 +771,23 @@ else if (log.isDebugEnabled())
     @Nullable @Override public GridDhtLocalPartition localPartition(int p, AffinityTopologyVersion topVer,
         boolean create)
         throws GridDhtInvalidPartitionException {
-        return localPartition0(p, topVer, create, false, true);
+        return localPartition0(p, topVer, create, false);
     }
 
     /** {@inheritDoc} */
     @Nullable @Override public GridDhtLocalPartition localPartition(int p, AffinityTopologyVersion topVer,
         boolean create, boolean showRenting) throws GridDhtInvalidPartitionException {
-        return localPartition0(p, topVer, create, showRenting, true);
+        return localPartition0(p, topVer, create, showRenting);
     }
 
     /**
+     * Creates partition with id {@code p} if it doesn't exist or evicted.
+     * In other case returns existing partition.
+     *
      * @param p Partition number.
      * @return Partition.
      */
-    private GridDhtLocalPartition createPartition(int p) {
+    private GridDhtLocalPartition getOrCreatePartition(int p) {
         assert lock.isWriteLockedByCurrentThread();
 
         assert ctx.database().checkpointLockIsHeldByThread();
@@ -865,16 +846,15 @@ private GridDhtLocalPartition createPartition(int p) {
     /**
      * @param p Partition number.
      * @param topVer Topology version.
-     * @param create Create flag.
-     * @param updateSeq Update sequence.
+     * @param create If {@code true} create partition if it doesn't exists or evicted.
+     * @param showRenting If {@code true} return partition in RENTING state if exists.
      * @return Local partition.
      */
     @SuppressWarnings("TooBroadScope")
     private GridDhtLocalPartition localPartition0(int p,
         AffinityTopologyVersion topVer,
         boolean create,
-        boolean showRenting,
-        boolean updateSeq) {
+        boolean showRenting) {
         GridDhtLocalPartition loc;
 
         loc = locParts.get(p);
@@ -1345,6 +1325,8 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
                 if (incomeCntrMap != null) {
                     // update local counters in partitions
                     for (int i = 0; i < locParts.length(); i++) {
+                        cntrMap.updateCounter(i, incomeCntrMap.updateCounter(i));
+
                         GridDhtLocalPartition part = locParts.get(i);
 
                         if (part == null)
@@ -1511,51 +1493,11 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
                             }
                         }
                         else if (state == MOVING) {
-                            GridDhtLocalPartition locPart = locParts.get(p);
-
-                            if (!partsToReload.contains(p)) {
-                                if (locPart == null || locPart.state() == EVICTED)
-                                    locPart = createPartition(p);
-
-                                if (locPart.state() == OWNING) {
-                                    locPart.moving();
+                            boolean haveHistory = !partsToReload.contains(p);
 
-                                    changed = true;
-                                }
-                            }
-                            else {
-                                if (locPart == null || locPart.state() == EVICTED) {
-                                    createPartition(p);
+                            rebalancePartition(p, haveHistory);
 
-                                    changed = true;
-                                }
-                                else if (locPart.state() == OWNING || locPart.state() == MOVING) {
-                                    if (locPart.state() == OWNING)
-                                        locPart.moving();
-                                    locPart.clearAsync();
-
-                                    changed = true;
-                                }
-                                else if (locPart.state() == RENTING) {
-                                    // Try to prevent partition eviction.
-                                    if (locPart.reserve()) {
-                                        try {
-                                            locPart.moving();
-                                            locPart.clearAsync();
-                                        } finally {
-                                            locPart.release();
-                                        }
-                                    }
-                                    // In other case just recreate it.
-                                    else {
-                                        assert locPart.state() == EVICTED;
-
-                                        createPartition(p);
-                                    }
-
-                                    changed = true;
-                                }
-                            }
+                            changed = true;
                         }
                     }
                 }
@@ -2113,7 +2055,7 @@ else if (plc != PartitionLossPolicy.IGNORE) {
     }
 
     /** {@inheritDoc} */
-    @Override public Set<UUID> setOwners(int p, Set<UUID> owners, boolean haveHistory, boolean updateSeq) {
+    @Override public Set<UUID> setOwners(int p, Set<UUID> ownersByUpdCounters, boolean haveHistory, boolean updateSeq) {
         Set<UUID> result = haveHistory ? Collections.<UUID>emptySet() : new HashSet<UUID>();
 
         ctx.database().checkpointReadLock();
@@ -2125,17 +2067,15 @@ else if (plc != PartitionLossPolicy.IGNORE) {
                 GridDhtLocalPartition locPart = locParts.get(p);
 
                 if (locPart != null) {
-                    if (locPart.state() == OWNING && !owners.contains(ctx.localNodeId())) {
-                        locPart.moving();
+                    if (locPart.state() == OWNING && !ownersByUpdCounters.contains(ctx.localNodeId())) {
+                        rebalancePartition(p, haveHistory);
 
-                        if (!haveHistory) {
-                            locPart.clearAsync();
+                        if (!haveHistory)
                             result.add(ctx.localNodeId());
-                        }
 
                         U.warn(log, "Partition has been scheduled for rebalancing due to outdated update counter " +
                             "[nodeId=" + ctx.localNodeId() + ", grp=" + grp.cacheOrGroupName() +
-                            ", partId=" + locPart.id() + ", haveHistory=" + haveHistory + "]");
+                            ", partId=" + p + ", haveHistory=" + haveHistory + "]");
                     }
                 }
 
@@ -2146,7 +2086,7 @@ else if (plc != PartitionLossPolicy.IGNORE) {
                     if (!partMap.containsKey(p))
                         continue;
 
-                    if (partMap.get(p) == OWNING && !owners.contains(remoteNodeId)) {
+                    if (partMap.get(p) == OWNING && !ownersByUpdCounters.contains(remoteNodeId)) {
                         partMap.put(p, MOVING);
 
                         if (!haveHistory)
@@ -2179,6 +2119,42 @@ else if (plc != PartitionLossPolicy.IGNORE) {
         return result;
     }
 
+    /**
+     * Prepares given partition {@code p} for rebalance.
+     * Changes partition state to MOVING and starts clearing if needed.
+     * Prevents ongoing renting if required.
+     *
+     * @param p Partition id.
+     * @param haveHistory If {@code true} there is WAL history to rebalance partition,
+     *                    in other case partition will be cleared for full rebalance.
+     */
+    private GridDhtLocalPartition rebalancePartition(int p, boolean haveHistory) {
+        GridDhtLocalPartition part = getOrCreatePartition(p);
+
+        // Prevent renting.
+        if (part.state() == RENTING) {
+            if (part.reserve()) {
+                part.moving();
+                part.release();
+            }
+            else {
+                assert part.state() == EVICTED : part;
+
+                part = getOrCreatePartition(p);
+            }
+        }
+
+        if (part.state() != MOVING)
+            part.moving();
+
+        if (!haveHistory)
+            part.clearAsync();
+
+        assert part.state() == MOVING : part;
+
+        return part;
+    }
+
     /**
      * Finds local partitions which don't belong to affinity and runs eviction process for such partitions.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
index 92a05848e3d00..cc0542c5d0e5b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.lang.IgniteProductVersion;
+import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 
@@ -71,9 +72,9 @@ public GridDhtPartitionsStateValidator(GridCacheSharedContext<?, ?> cctx) {
     public void validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture fut,
                                                   GridDhtPartitionTopology top,
                                                   Map<UUID, GridDhtPartitionsSingleMessage> messages) throws IgniteCheckedException {
-        // Ignore just joined nodes.
         final Set<UUID> ignoringNodes = new HashSet<>();
 
+        // Ignore just joined nodes.
         for (DiscoveryEvent evt : fut.events().events())
             if (evt.type() == EVT_NODE_JOINED)
                 ignoringNodes.add(evt.eventNode().id());
@@ -98,6 +99,46 @@ public void validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture fu
             throw new IgniteCheckedException("Partitions cache sizes are inconsistent for " + fold(topVer, result));
     }
 
+    /**
+     * Checks what partitions from given {@code singleMsg} message should be excluded from validation.
+     *
+     * @param top Topology to validate.
+     * @param nodeId Node which sent single message.
+     * @param singleMsg Single message.
+     * @return Set of partition ids should be excluded from validation.
+     */
+    @Nullable private Set<Integer> shouldIgnore(GridDhtPartitionTopology top, UUID nodeId, GridDhtPartitionsSingleMessage singleMsg) {
+        CachePartitionPartialCountersMap countersMap = singleMsg.partitionUpdateCounters(top.groupId(), top.partitions());
+        Map<Integer, Long> sizesMap = singleMsg.partitionSizes(top.groupId());
+
+        Set<Integer> ignore = null;
+
+        for (int p = 0; p < top.partitions(); p++) {
+            if (top.partitionState(nodeId, p) != GridDhtPartitionState.OWNING) {
+                if (ignore == null)
+                    ignore = new HashSet<>();
+
+                ignore.add(p);
+
+                continue;
+            }
+
+            int partIdx = countersMap.partitionIndex(p);
+            long updateCounter = partIdx >= 0 ? countersMap.updateCounterAt(partIdx) : 0;
+            long size = sizesMap.containsKey(p) ? sizesMap.get(p) : 0;
+
+            // Do not validate partitions with zero update counter and size.
+            if (updateCounter == 0 && size == 0) {
+                if (ignore == null)
+                    ignore = new HashSet<>();
+
+                ignore.add(p);
+            }
+        }
+
+        return ignore;
+    }
+
     /**
      * Validate partitions update counters for given {@code top}.
      *
@@ -117,7 +158,10 @@ Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
 
         // Populate counters statistics from local node partitions.
         for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
-            if (top.partitionState(cctx.localNodeId(), part.id()) != GridDhtPartitionState.OWNING)
+            if (part.state() != GridDhtPartitionState.OWNING)
+                continue;
+
+            if (part.updateCounter() == 0 && part.fullSize() == 0)
                 continue;
 
             updateCountersAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.updateCounter()));
@@ -133,8 +177,10 @@ Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
 
             CachePartitionPartialCountersMap countersMap = e.getValue().partitionUpdateCounters(top.groupId(), partitions);
 
+            Set<Integer> ignorePartitions = shouldIgnore(top, nodeId, e.getValue());
+
             for (int part = 0; part < partitions; part++) {
-                if (top.partitionState(nodeId, part) != GridDhtPartitionState.OWNING)
+                if (ignorePartitions != null && ignorePartitions.contains(part))
                     continue;
 
                 int partIdx = countersMap.partitionIndex(part);
@@ -166,7 +212,10 @@ Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
 
         // Populate sizes statistics from local node partitions.
         for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
-            if (top.partitionState(cctx.localNodeId(), part.id()) != GridDhtPartitionState.OWNING)
+            if (part.state() != GridDhtPartitionState.OWNING)
+                continue;
+
+            if (part.updateCounter() == 0 && part.fullSize() == 0)
                 continue;
 
             sizesAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.fullSize()));
@@ -182,8 +231,10 @@ Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
 
             Map<Integer, Long> sizesMap = e.getValue().partitionSizes(top.groupId());
 
+            Set<Integer> ignorePartitions = shouldIgnore(top, nodeId, e.getValue());
+
             for (int part = 0; part < partitions; part++) {
-                if (top.partitionState(nodeId, part) != GridDhtPartitionState.OWNING)
+                if (ignorePartitions != null && ignorePartitions.contains(part))
                     continue;
 
                 long currentSize = sizesMap.containsKey(part) ? sizesMap.get(part) : 0L;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index af5acd64257e7..0d57d483f71a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -58,6 +58,7 @@
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord;
+import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -2244,6 +2245,8 @@ private void onAffinityInitialized(IgniteInternalFuture<Map<Integer, Map<Integer
     }
 
     /**
+     * Collects and determines new owners of partitions for all nodes for given {@code top}.
+     *
      * @param top Topology to assign.
      */
     private void assignPartitionStates(GridDhtPartitionTopology top) {
@@ -2585,6 +2588,7 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
                     detectLostPartitions(resTopVer);
             }
 
+            // Recalculate new affinity based on partitions availability.
             if (!exchCtx.mergeExchanges() && forceAffReassignment)
                 idealAffDiff = cctx.affinity().onCustomEventWithEnforcedAffinityReassignment(this);
 
@@ -2768,7 +2772,7 @@ private void validatePartitionsState() {
                 validator.validatePartitionCountersAndSizes(this, top, msgs);
             }
             catch (IgniteCheckedException ex) {
-                log.warning("Partition states validation was failed for cache " + grpDesc.cacheOrGroupName(), ex);
+                log.warning("Partition states validation has failed for group: " + grpDesc.cacheOrGroupName() + ". " + ex.getMessage());
                 // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 6ebafac22611e..b60070e6b4dd2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -73,7 +73,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
     private Map<Integer, Map<Integer, Long>> partSizes;
 
     /** Serialized partitions counters. */
-    private byte[] partSizesBytes;
+    private byte[] partsSizesBytes;
 
     /** Partitions history reservation counters. */
     @GridToStringInclude
@@ -324,7 +324,7 @@ public void setError(Exception ex) {
         boolean marshal = (parts != null && partsBytes == null) ||
             (partCntrs != null && partCntrsBytes == null) ||
             (partHistCntrs != null && partHistCntrsBytes == null) ||
-            (partSizes != null && partSizesBytes == null) ||
+            (partSizes != null && partsSizesBytes == null) ||
             (err != null && errBytes == null);
 
         if (marshal) {
@@ -343,7 +343,7 @@ public void setError(Exception ex) {
             if (partHistCntrs != null && partHistCntrsBytes == null)
                 partHistCntrsBytes0 = U.marshal(ctx, partHistCntrs);
 
-            if (partSizes != null && partSizesBytes == null)
+            if (partSizes != null && partsSizesBytes == null)
                 partSizesBytes0 = U.marshal(ctx, partSizes);
 
             if (err != null && errBytes == null)
@@ -375,7 +375,7 @@ public void setError(Exception ex) {
             partsBytes = partsBytes0;
             partCntrsBytes = partCntrsBytes0;
             partHistCntrsBytes = partHistCntrsBytes0;
-            partSizesBytes = partSizesBytes0;
+            partsSizesBytes = partSizesBytes0;
             errBytes = errBytes0;
         }
     }
@@ -405,11 +405,11 @@ public void setError(Exception ex) {
                 partHistCntrs = U.unmarshal(ctx, partHistCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
         }
 
-        if (partSizesBytes != null && partSizes == null) {
+        if (partsSizesBytes != null && partSizes == null) {
             if (compressed())
-                partSizes = U.unmarshalZip(ctx.marshaller(), partSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+                partSizes = U.unmarshalZip(ctx.marshaller(), partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
             else
-                partSizes = U.unmarshal(ctx, partSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+                partSizes = U.unmarshal(ctx, partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
         }
 
         if (errBytes != null && err == null) {
@@ -504,7 +504,7 @@ public void setError(Exception ex) {
                 writer.incrementState();
 
             case 13:
-                if (!writer.writeByteArray("partsSizesBytes", partSizesBytes))
+                if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes))
                     return false;
 
                 writer.incrementState();
@@ -589,7 +589,7 @@ public void setError(Exception ex) {
                 reader.incrementState();
 
             case 13:
-                partSizesBytes = reader.readByteArray("partsSizesBytes");
+                partsSizesBytes = reader.readByteArray("partsSizesBytes");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 6ec6ad33991da..ddcb81e238965 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -202,7 +202,7 @@ private IgniteCheckedException stopError() {
 
             // If partition belongs to local node.
             if (aff.get(p).contains(ctx.localNode())) {
-                GridDhtLocalPartition part = top.localPartition(p, topVer, true, true);
+                GridDhtLocalPartition part = top.localPartition(p);
 
                 assert part != null;
                 assert part.id() == p;
@@ -226,7 +226,7 @@ private IgniteCheckedException stopError() {
                     part = top.localPartition(p, topVer, true);
                 }
 
-                assert part != null && part.state() == MOVING : "Partition has invalid state for rebalance " + part;
+                assert part.state() == MOVING : "Partition has invalid state for rebalance " + aff.topologyVersion() + " " + part;
 
                 ClusterNode histSupplier = null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 16d32924da847..3009febae9b44 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -59,6 +59,7 @@
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import javax.management.ObjectName;
 import org.apache.ignite.DataStorageMetrics;
 import org.apache.ignite.IgniteCheckedException;
@@ -1151,9 +1152,18 @@ private void shutdownCheckpointer(boolean cancel) {
 
         boolean clusterInTransitionStateToActive = fut.activateCluster();
 
-        // Before local node join event.
-        if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode))
+        // In case of cluster activation or local join restore, restore whole manager state.
+        if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode)) {
             restoreState();
+        }
+        // In case of starting groups, restore partition states only for these groups.
+        else if (fut.exchangeActions() != null && !F.isEmpty(fut.exchangeActions().cacheGroupsToStart())) {
+            Set<Integer> restoreGroups = fut.exchangeActions().cacheGroupsToStart().stream()
+                .map(actionData -> actionData.descriptor().groupId())
+                .collect(Collectors.toSet());
+
+            restorePartitionStates(Collections.emptyMap(), restoreGroups);
+        }
 
         if (cctx.kernalContext().query().moduleEnabled()) {
             ExchangeActions acts = fut.exchangeActions();
@@ -1393,6 +1403,8 @@ private boolean safeToUpdatePageMemories() {
     }
 
     /**
+     * Restores from last checkpoint and applies WAL changes since this checkpoint.
+     *
      * @throws IgniteCheckedException If failed to restore database status from WAL.
      */
     private void restoreState() throws IgniteCheckedException {
@@ -2154,7 +2166,7 @@ else if (log != null)
         checkpointReadLock();
 
         try {
-            restorePartitionState(partStates, Collections.emptySet());
+            restorePartitionStates(partStates, null);
         }
         finally {
             checkpointReadUnlock();
@@ -2264,7 +2276,7 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
             }
 
             if (!metastoreOnly)
-                restorePartitionState(partStates, ignoreGrps);
+                restorePartitionStates(partStates, null);
         }
         finally {
             if (!metastoreOnly)
@@ -2277,15 +2289,17 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
     }
 
     /**
-     * @param partStates Partition states.
-     * @throws IgniteCheckedException If failed to restore.
+     * Initializes not empty partitions and restores their state from page memory or WAL.
+     * Partition states presented in page memory may be overriden by states restored from WAL {@code partStates}.
+     *
+     * @param partStates Partition states restored from WAL.
+     * @param onlyForGroups If not {@code null} restore states only for specified cache groups.
+     * @throws IgniteCheckedException If failed to restore partition states.
      */
-    private void restorePartitionState(
-        Map<T2<Integer, Integer>, T2<Integer, Long>> partStates,
-        Collection<Integer> ignoreGrps
-    ) throws IgniteCheckedException {
+    private void restorePartitionStates(Map<T2<Integer, Integer>, T2<Integer, Long>> partStates,
+                                       @Nullable Set<Integer> onlyForGroups) throws IgniteCheckedException {
         for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
-            if (grp.isLocal() || !grp.affinityNode() || ignoreGrps.contains(grp.groupId())) {
+            if (grp.isLocal() || !grp.affinityNode()) {
                 // Local cache has no partitions and its states.
                 continue;
             }
@@ -2293,6 +2307,9 @@ private void restorePartitionState(
             if (!grp.dataRegion().config().isPersistenceEnabled())
                 continue;
 
+            if (onlyForGroups != null && !onlyForGroups.contains(grp.groupId()))
+                continue;
+
             int grpId = grp.groupId();
 
             PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
@@ -2375,56 +2392,6 @@ else if (restore != null) {
         }
     }
 
-    /**
-     * @param grpCtx Group context.
-     * @param partId Partition ID.
-     * @return Partition state.
-     */
-    public GridDhtPartitionState readPartitionState(CacheGroupContext grpCtx, int partId) {
-        int grpId = grpCtx.groupId();
-        PageMemoryEx pageMem = (PageMemoryEx)grpCtx.dataRegion().pageMemory();
-
-        try {
-            if (storeMgr.exists(grpId, partId)) {
-                storeMgr.ensure(grpId, partId);
-
-                if (storeMgr.pages(grpId, partId) > 1) {
-                    long partMetaId = pageMem.partitionMetaPageId(grpId, partId);
-                    long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
-
-                    try {
-                        long pageAddr = pageMem.readLock(grpId, partMetaId, partMetaPage);
-
-                        try {
-                            if (PageIO.getType(pageAddr) == PageIO.T_PART_META) {
-                                PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
-
-                                GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal((int)io.getPartitionState(pageAddr));
-
-                                if (state == null)
-                                    state = GridDhtPartitionState.MOVING;
-
-                                return state;
-                            }
-                        }
-                        finally {
-                            pageMem.readUnlock(grpId, partMetaId, partMetaPage);
-                        }
-                    }
-                    finally {
-                        pageMem.releasePage(grpId, partMetaId, partMetaPage);
-                    }
-                }
-            }
-        }
-        catch (IgniteCheckedException e) {
-            U.error(log, "Failed to read partition state (will default to MOVING) [grp=" + grpCtx +
-                ", partId=" + partId + "]", e);
-        }
-
-        return GridDhtPartitionState.MOVING;
-    }
-
     /**
      * Wal truncate callBack.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index 0d59a2d79a13d..c3d404b50fd7c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -82,6 +82,9 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
     /** */
     private boolean delayRebalance;
 
+    /** */
+    private boolean disableAutoActivation;
+
     /** */
     private Map<String, Object> userAttrs;
 
@@ -107,6 +110,8 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
         cleanPersistenceDir();
 
         client = false;
+
+        disableAutoActivation = false;
     }
 
     /** {@inheritDoc} */
@@ -120,6 +125,9 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
 
         cfg.setConsistentId(igniteInstanceName);
 
+        if (disableAutoActivation)
+            cfg.setAutoActivationEnabled(false);
+
         cfg.setDataStorageConfiguration(
             new DataStorageConfiguration().setDefaultDataRegionConfiguration(
                 new DataRegionConfiguration()
@@ -884,11 +892,15 @@ public void testAffinityAssignmentChangedAfterRestart() throws Exception {
 
         delayRebalance = true;
 
+        /* There is a problem with handling simultaneous auto activation after restart and manual activation.
+           To properly catch the moment when cluster activation has finished we temporary disable auto activation. */
+        disableAutoActivation = true;
+
         startGrids(4);
 
         ig = grid(0);
 
-        ig.active(true);
+        ig.cluster().active(true);
 
         cache = ig.cache(cacheName);
 
@@ -990,7 +1002,7 @@ public TestAffinityFunction(AffinityFunction delegate) {
 
         /** {@inheritDoc} */
         @Override public void reset() {
-            delegate.reset();;
+            delegate.reset();
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java
index 1e046d41f4b05..ebc804f1c885d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLoadingConcurrentGridStartSelfTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cache.store.CacheStoreAdapter;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -93,6 +94,8 @@ public class CacheLoadingConcurrentGridStartSelfTest extends GridCommonAbstractT
 
         ccfg.setCacheStoreFactory(new FactoryBuilder.SingletonFactory(new TestCacheStoreAdapter()));
 
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 64));
+
         if (getTestIgniteInstanceName(0).equals(igniteInstanceName)) {
             if (client)
                 cfg.setClientMode(true);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
index 63d772a0077c0..fc617bb8f97e2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
@@ -106,6 +106,10 @@ public void testValidationIfPartitionCountersAreInconsistent() throws Exception
 
         awaitPartitionMapExchange();
 
+        // Populate cache to increment update counters.
+        for (int i = 0; i < 1000; i++)
+            ignite.cache(CACHE_NAME).put(i, i);
+
         // Modify update counter for some partition.
         for (GridDhtLocalPartition partition : ignite.cachex(CACHE_NAME).context().topology().localPartitions()) {
             partition.updateCounter(100500L);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
index 9ed8d54080a37..43a23031dd99a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
 import org.junit.Assert;
 import org.mockito.Matchers;
 import org.mockito.Mockito;
@@ -70,24 +71,21 @@ private GridDhtLocalPartition partitionMock(int id, long updateCounter, long siz
         Mockito.when(partitionMock.id()).thenReturn(id);
         Mockito.when(partitionMock.updateCounter()).thenReturn(updateCounter);
         Mockito.when(partitionMock.fullSize()).thenReturn(size);
+        Mockito.when(partitionMock.state()).thenReturn(GridDhtPartitionState.OWNING);
         return partitionMock;
     }
 
     /**
-     * @return Message containing specified {@code countersMap}.
+     * @param countersMap Update counters map.
+     * @param sizesMap Sizes map.
+     * @return Message with specified {@code countersMap} and {@code sizeMap}.
      */
-    private GridDhtPartitionsSingleMessage fromUpdateCounters(Map<Integer, T2<Long, Long>> countersMap) {
+    private GridDhtPartitionsSingleMessage from(@Nullable Map<Integer, T2<Long, Long>> countersMap, @Nullable Map<Integer, Long> sizesMap) {
         GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage();
-        msg.addPartitionUpdateCounters(0, countersMap);
-        return msg;
-    }
-
-    /**
-     * @return Message containing specified {@code sizesMap}.
-     */
-    private GridDhtPartitionsSingleMessage fromCacheSizes(Map<Integer, Long> sizesMap) {
-        GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage();
-        msg.addPartitionSizes(0, sizesMap);
+        if (countersMap != null)
+            msg.addPartitionUpdateCounters(0, countersMap);
+        if (sizesMap != null)
+            msg.addPartitionSizes(0, sizesMap);
         return msg;
     }
 
@@ -98,15 +96,22 @@ public void testPartitionCountersValidation() {
         UUID remoteNode = UUID.randomUUID();
         UUID ignoreNode = UUID.randomUUID();
 
-        // For partitions 0 and 2 (zero counter) we have inconsistent update counters.
+        // For partitions 0 and 2 we have inconsistent update counters.
         Map<Integer, T2<Long, Long>> updateCountersMap = new HashMap<>();
         updateCountersMap.put(0, new T2<>(2L, 2L));
         updateCountersMap.put(1, new T2<>(2L, 2L));
+        updateCountersMap.put(2, new T2<>(5L, 5L));
+
+        // For partitions 0 and 2 we have inconsistent cache sizes.
+        Map<Integer, Long> cacheSizesMap = new HashMap<>();
+        cacheSizesMap.put(0, 2L);
+        cacheSizesMap.put(1, 2L);
+        cacheSizesMap.put(2, 2L);
 
         // Form single messages map.
         Map<UUID, GridDhtPartitionsSingleMessage> messages = new HashMap<>();
-        messages.put(remoteNode, fromUpdateCounters(updateCountersMap));
-        messages.put(ignoreNode, fromUpdateCounters(updateCountersMap));
+        messages.put(remoteNode, from(updateCountersMap, cacheSizesMap));
+        messages.put(ignoreNode, from(updateCountersMap, cacheSizesMap));
 
         GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(cctxMock);
 
@@ -120,7 +125,7 @@ public void testPartitionCountersValidation() {
         Assert.assertTrue(result.get(0).get(localNodeId) == 1L);
         Assert.assertTrue(result.get(0).get(remoteNode) == 2L);
         Assert.assertTrue(result.get(2).get(localNodeId) == 3L);
-        Assert.assertTrue(result.get(2).get(remoteNode) == 0L);
+        Assert.assertTrue(result.get(2).get(remoteNode) == 5L);
     }
 
     /**
@@ -130,6 +135,12 @@ public void testPartitionCacheSizesValidation() {
         UUID remoteNode = UUID.randomUUID();
         UUID ignoreNode = UUID.randomUUID();
 
+        // For partitions 0 and 2 we have inconsistent update counters.
+        Map<Integer, T2<Long, Long>> updateCountersMap = new HashMap<>();
+        updateCountersMap.put(0, new T2<>(2L, 2L));
+        updateCountersMap.put(1, new T2<>(2L, 2L));
+        updateCountersMap.put(2, new T2<>(5L, 5L));
+
         // For partitions 0 and 2 we have inconsistent cache sizes.
         Map<Integer, Long> cacheSizesMap = new HashMap<>();
         cacheSizesMap.put(0, 2L);
@@ -138,8 +149,8 @@ public void testPartitionCacheSizesValidation() {
 
         // Form single messages map.
         Map<UUID, GridDhtPartitionsSingleMessage> messages = new HashMap<>();
-        messages.put(remoteNode, fromCacheSizes(cacheSizesMap));
-        messages.put(ignoreNode, fromCacheSizes(cacheSizesMap));
+        messages.put(remoteNode, from(updateCountersMap, cacheSizesMap));
+        messages.put(ignoreNode, from(updateCountersMap, cacheSizesMap));
 
         GridDhtPartitionsStateValidator validator = new GridDhtPartitionsStateValidator(cctxMock);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
index 6039ae37a1539..fd124b72b1796 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
@@ -47,7 +47,7 @@ public class IgniteStandByClusterSuite extends TestSuite {
      * @return Test suite.
      */
     public static TestSuite suite() {
-        TestSuite suite = new TestSuite("Ignite Activate/DeActivate Cluster Test Suit");
+        TestSuite suite = new TestSuite("Ignite Activate/DeActivate Cluster Test Suite");
 
         suite.addTestSuite(IgniteClusterActivateDeactivateTest.class);
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
index fc1cea69653b4..dd495cfb216a9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartSelfTest.java
@@ -73,6 +73,8 @@ public class IgniteCacheQueryNodeRestartSelfTest extends GridCacheAbstractSelfTe
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
 
+        c.setConsistentId(igniteInstanceName);
+
         TcpDiscoverySpi disco = new TcpDiscoverySpi();
 
         disco.setIpFinder(ipFinder);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
index 8e049acc7d4fd..3ee19d5904478 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/baseline/IgniteChangingBaselineCacheQueryNodeRestartSelfTest.java
@@ -58,7 +58,7 @@ public class IgniteChangingBaselineCacheQueryNodeRestartSelfTest extends IgniteC
 
         initStoreStrategy();
 
-        grid(0).active(true);
+        grid(0).cluster().active(true);
 
         awaitPartitionMapExchange();
     }
@@ -74,7 +74,7 @@ public class IgniteChangingBaselineCacheQueryNodeRestartSelfTest extends IgniteC
     @Override protected IgniteInternalFuture createRestartAction(final AtomicBoolean done, final AtomicInteger restartCnt) throws Exception {
         return multithreadedAsync(new Callable<Object>() {
             /** */
-            private final long baselineTopChangeInterval = 30 * 1000;
+            private final long baselineTopChangeInterval = 10 * 1000;
 
             /** */
             private final int logFreq = 50;

From 4ab86f13cbc7fef68f5fd39354e242fe6cca285a Mon Sep 17 00:00:00 2001
From: skalashnikov <skalashnikov@gridgain.com>
Date: Wed, 18 Apr 2018 16:37:20 +0300
Subject: [PATCH 078/543] IGNITE-7512 Check for null before validateKeyAndValue
 in GridDhtAtomicCache.updateWithBatch - Fixes #3429.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../dht/atomic/GridDhtAtomicCache.java        |  3 +-
 .../local/atomic/GridLocalAtomicCache.java    |  2 +-
 .../transactions/IgniteTxLocalAdapter.java    |  5 +-
 .../query/IgniteSqlNotNullConstraintTest.java | 71 ++++++++++++++++++-
 4 files changed, 77 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index e3ad38205783c..44f2b153db3ad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -2071,7 +2071,8 @@ private DhtAtomicUpdateResult updateWithBatch(
 
                             validation = true;
 
-                            ctx.validateKeyAndValue(entry.key(), updated);
+                            if (updated != null)
+                                ctx.validateKeyAndValue(entry.key(), updated);
                         }
                     }
                     catch (Exception e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index 6cb50f20733c5..dad105231e36e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -1074,7 +1074,7 @@ private Map<K, EntryProcessorResult> updateWithBatch(
                             if (computed != null)
                                 invokeRes = CacheInvokeResult.fromResult(ctx.unwrapTemporary(computed));
 
-                            if (invokeEntry.modified()) {
+                            if (invokeEntry.modified() && updated != null) {
                                 validation = true;
 
                                 ctx.validateKeyAndValue(entry.key(), updated);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index c2a284239ee38..7a7f65f7fb9e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1256,7 +1256,10 @@ protected final void addInvokeResult(IgniteTxEntry txEntry,
                 key0 = invokeEntry.key();
             }
 
-            ctx.validateKeyAndValue(txEntry.key(), ctx.toCacheObject(val0));
+            val0 = ctx.toCacheObject(val0);
+
+            if (val0 != null)
+                ctx.validateKeyAndValue(txEntry.key(), (CacheObject)val0);
 
             if (res != null)
                 ret.addEntryProcessResult(ctx, txEntry.key(), key0, res, null, txEntry.keepBinary());
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java
index 1f4e018889a3b..183138b82dc60 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlNotNullConstraintTest.java
@@ -680,6 +680,72 @@ key1, new TestEntryProcessor(okValue),
         });
     }
 
+    /** */
+    public void testAtomicOrImplicitTxInvokeDelete() throws Exception {
+        executeWithAllCaches(new TestClosure() {
+            @Override public void run() throws Exception {
+                cache.put(key1, okValue);
+
+                cache.invoke(key1, new TestEntryProcessor(null));
+
+                assertEquals(0, cache.size());
+            }
+        });
+    }
+
+    /** */
+    public void testAtomicOrImplicitTxInvokeAllDelete() throws Exception {
+        executeWithAllCaches(new TestClosure() {
+            @Override public void run() throws Exception {
+                cache.put(key1, okValue);
+                cache.put(key2, okValue);
+
+                cache.invokeAll(F.asMap(
+                    key1, new TestEntryProcessor(null),
+                    key2, new TestEntryProcessor(null)));
+
+                assertEquals(0, cache.size());
+            }
+        });
+    }
+
+    /** */
+    public void testTxInvokeDelete() throws Exception {
+        executeWithAllTxCaches(new TestClosure() {
+            @Override public void run() throws Exception {
+                cache.put(key1, okValue);
+
+                try (Transaction tx = ignite.transactions().txStart(concurrency, isolation)) {
+                    cache.invoke(key1, new TestEntryProcessor(null));
+
+                    tx.commit();
+                }
+
+                assertEquals(0, cache.size());
+            }
+        });
+    }
+
+    /** */
+    public void testTxInvokeAllDelete() throws Exception {
+        executeWithAllTxCaches(new TestClosure() {
+            @Override public void run() throws Exception {
+                cache.put(key1, okValue);
+                cache.put(key2, okValue);
+
+                try (Transaction tx = ignite.transactions().txStart(concurrency, isolation)) {
+                    cache.invokeAll(F.asMap(
+                        key1, new TestEntryProcessor(null),
+                        key2, new TestEntryProcessor(null)));
+
+                    tx.commit();
+                }
+
+                assertEquals(0, cache.size());
+            }
+        });
+    }
+
     /** */
     public void testDynamicTableCreateNotNullFieldsAllowed() throws Exception {
         executeSql("CREATE TABLE test(id INT PRIMARY KEY, field INT NOT NULL)");
@@ -1137,7 +1203,10 @@ public TestEntryProcessor(Person value) {
         @Override public Object process(MutableEntry<Integer, Person> entry,
             Object... objects) throws EntryProcessorException {
 
-            entry.setValue(value);
+            if (value == null)
+                entry.remove();
+            else
+                entry.setValue(value);
 
             return null;
         }

From 7f318da5b357cca1a7e483a37d2556fbd56fc2a7 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Wed, 18 Apr 2018 17:56:56 +0300
Subject: [PATCH 079/543] IGNITE-8017 Disable WAL during initial rebalancing

---
 .../apache/ignite/IgniteSystemProperties.java |   7 +
 .../processors/cache/CacheGroupContext.java   |  54 +-
 .../processors/cache/WalStateManager.java     | 160 ++++-
 .../dht/GridClientPartitionTopology.java      |   5 +
 .../dht/GridDhtPartitionTopology.java         |   7 +
 .../dht/GridDhtPartitionTopologyImpl.java     |  27 +
 .../preloader/GridDhtPartitionDemander.java   |  24 +-
 .../GridDhtPartitionsExchangeFuture.java      |   2 +
 .../GridCacheDatabaseSharedManager.java       |  63 +-
 .../IgniteCacheDatabaseSharedManager.java     |   4 +-
 .../file/FilePageStoreManager.java            |  18 +-
 ...alModeChangeDuringRebalancingSelfTest.java | 566 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 13 files changed, 890 insertions(+), 50 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 437f49f6a0a9c..32fed05b369ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -877,6 +877,13 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD = "IGNITE_PART_DISTRIBUTION_WARN_THRESHOLD";
 
+    /**
+     * When set to {@code true}, WAL will be automatically disabled during rebalancing if there is no partition in
+     * OWNING state.
+     * Default is {@code false}.
+     */
+    public static final String IGNITE_DISABLE_WAL_DURING_REBALANCING = "IGNITE_DISABLE_WAL_DURING_REBALANCING";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 12636f3bd0ef5..849ecc8196eeb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -152,7 +152,10 @@ public class CacheGroupContext {
     private CacheGroupMetricsMXBean mxBean;
 
     /** */
-    private volatile boolean walEnabled;
+    private volatile boolean localWalEnabled;
+
+    /** */
+    private volatile boolean globalWalEnabled;
 
     /**
      * @param grpId Group ID.
@@ -196,9 +199,10 @@ public class CacheGroupContext {
         this.reuseList = reuseList;
         this.locStartVer = locStartVer;
         this.cacheType = cacheType;
-        this.walEnabled = walEnabled;
+        this.globalWalEnabled = walEnabled;
+        this.localWalEnabled = true;
 
-        persistWalState(walEnabled);
+        persistGlobalWalState(walEnabled);
 
         ioPlc = cacheType.ioPolicy();
 
@@ -1021,22 +1025,52 @@ public CacheGroupMetricsMXBean mxBean() {
      * WAL enabled flag.
      */
     public boolean walEnabled() {
-        return walEnabled;
+        return localWalEnabled && globalWalEnabled;
+    }
+
+    /**
+     * Local WAL enabled flag.
+     */
+    public boolean localWalEnabled() {
+        return localWalEnabled;
+    }
+
+    /**
+     * @Global WAL enabled flag.
+     */
+    public boolean globalWalEnabled() {
+        return globalWalEnabled;
     }
 
     /**
-     * @param enabled WAL enabled flag.
+     * @param enabled Global WAL enabled flag.
      */
-    public void walEnabled(boolean enabled) {
-        persistWalState(enabled);
+    public void globalWalEnabled(boolean enabled) {
+        persistGlobalWalState(enabled);
 
-        this.walEnabled = enabled;
+        this.globalWalEnabled = enabled;
+    }
+
+    /**
+     * @param enabled Local WAL enabled flag.
+     */
+    public void localWalEnabled(boolean enabled) {
+        persistLocalWalState(enabled);
+
+        this.localWalEnabled = enabled;
+    }
+
+    /**
+     * @param enabled Enabled flag..
+     */
+    private void persistGlobalWalState(boolean enabled) {
+        shared().database().walEnabled(grpId, enabled, false);
     }
 
     /**
      * @param enabled Enabled flag..
      */
-    private void persistWalState(boolean enabled) {
-        shared().database().walEnabled(grpId, enabled);
+    private void persistLocalWalState(boolean enabled) {
+        shared().database().walEnabled(grpId, enabled, true);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
index 64a6819826357..4a14730964197 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -18,7 +18,9 @@
 package org.apache.ignite.internal.processors.cache;
 
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -26,10 +28,13 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointFuture;
 import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -44,15 +49,18 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 
 import static org.apache.ignite.internal.GridTopic.TOPIC_WAL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
 
 /**
  * Write-ahead log state manager. Manages WAL enable and disable.
@@ -102,6 +110,9 @@ public class WalStateManager extends GridCacheSharedManagerAdapter {
     /** Disconnected flag. */
     private boolean disconnected;
 
+    /** Holder for groups with temporary disabled WAL. */
+    private volatile TemporaryDisabledWal tmpDisabledWal;
+
     /**
      * Constructor.
      *
@@ -327,6 +338,126 @@ else if (!F.eq(grpDesc.deploymentId(), curGrpDesc.deploymentId())) {
         }
     }
 
+    /**
+     * Change local WAL state before exchange is done. This method will disable WAL for groups without partitions
+     * in OWNING state if such feature is enabled.
+     *
+     * @param topVer Topology version.
+     */
+    public void changeLocalStatesOnExchangeDone(AffinityTopologyVersion topVer) {
+        if (!IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING, false))
+            return;
+
+        Set<Integer> grpsToEnableWal = new HashSet<>();
+        Set<Integer> grpsToDisableWal = new HashSet<>();
+        Set<Integer> grpsWithWalDisabled = new HashSet<>();
+
+        boolean hasNonEmptyOwning = false;
+
+        for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+            if (grp.isLocal() || !grp.affinityNode() || !grp.persistenceEnabled())
+                continue;
+
+            boolean hasOwning = false;
+
+            for (GridDhtLocalPartition locPart : grp.topology().currentLocalPartitions()) {
+                if (locPart.state() == OWNING) {
+                    hasOwning = true;
+
+                    if (hasNonEmptyOwning)
+                        break;
+
+                    if (locPart.updateCounter() > 0) {
+                        hasNonEmptyOwning = true;
+
+                        break;
+                    }
+                }
+            }
+
+            if (hasOwning && !grp.localWalEnabled()) {
+                grpsToEnableWal.add(grp.groupId());
+            }
+            else if (!hasOwning && grp.localWalEnabled()) {
+                grpsToDisableWal.add(grp.groupId());
+
+                grpsWithWalDisabled.add(grp.groupId());
+            }
+            else if (!grp.localWalEnabled())
+                grpsWithWalDisabled.add(grp.groupId());
+        }
+
+        tmpDisabledWal = new TemporaryDisabledWal(grpsWithWalDisabled, topVer);
+
+        if (grpsToEnableWal.isEmpty() && grpsToDisableWal.isEmpty())
+            return;
+
+        try {
+            if (hasNonEmptyOwning && !grpsToEnableWal.isEmpty())
+                triggerCheckpoint(0).finishFuture().get();
+        }
+        catch (IgniteCheckedException ex) {
+            throw new IgniteException(ex);
+        }
+
+        for (Integer grpId : grpsToEnableWal)
+            cctx.cache().cacheGroup(grpId).localWalEnabled(true);
+
+        for (Integer grpId : grpsToDisableWal)
+            cctx.cache().cacheGroup(grpId).localWalEnabled(false);
+    }
+
+    /**
+     * Callback when group rebalancing is finished. If there are no pending groups, it should trigger checkpoint and
+     * change partition states.
+     * @param grpId Group ID.
+     * @param topVer Topology version.
+     */
+    public void onGroupRebalanceFinished(int grpId, AffinityTopologyVersion topVer) {
+        TemporaryDisabledWal session0 = tmpDisabledWal;
+
+        if (session0 == null || !session0.topVer.equals(topVer))
+            return;
+
+        session0.remainingGrps.remove(grpId);
+
+        if (session0.remainingGrps.isEmpty()) {
+            synchronized (mux) {
+                if (tmpDisabledWal != session0)
+                    return;
+
+                for (Integer grpId0 : session0.disabledGrps) {
+                    CacheGroupContext grp = cctx.cache().cacheGroup(grpId0);
+
+                    assert grp != null;
+
+                    if (!grp.localWalEnabled())
+                        grp.localWalEnabled(true);
+                }
+
+                tmpDisabledWal = null;
+            }
+
+            CheckpointFuture cpFut = triggerCheckpoint(0);
+
+            assert cpFut != null;
+
+            cpFut.finishFuture().listen(new IgniteInClosureX<IgniteInternalFuture>() {
+                @Override public void applyx(IgniteInternalFuture future) {
+                    for (Integer grpId0 : session0.disabledGrps) {
+                        CacheGroupContext grp = cctx.cache().cacheGroup(grpId0);
+
+                        assert grp != null;
+
+                        grp.topology().ownMoving(session0.topVer);
+                    }
+
+                    cctx.exchange().refreshPartitions();
+                }
+            });
+        }
+    }
+
     /**
      * Handle propose message in discovery thread.
      *
@@ -455,7 +586,7 @@ public void onProposeExchange(WalStateProposeMessage msg) {
                         "no longer exist: " + msg.caches().keySet());
                 }
                 else {
-                    if (F.eq(msg.enable(), grpCtx.walEnabled()))
+                    if (F.eq(msg.enable(), grpCtx.globalWalEnabled()))
                         // Nothing changed -> no-op.
                         res = new WalStateResult(msg, false);
                     else {
@@ -468,7 +599,7 @@ public void onProposeExchange(WalStateProposeMessage msg) {
                                 cpFut.beginFuture().get();
 
                                 if (msg.enable()) {
-                                    grpCtx.walEnabled(true);
+                                    grpCtx.globalWalEnabled(true);
 
                                     // Enable: it is enough to release cache operations once mark is finished because
                                     // not-yet-flushed dirty pages have been logged.
@@ -489,7 +620,7 @@ public void onProposeExchange(WalStateProposeMessage msg) {
 
                                     // WAL state is persisted after checkpoint if finished. Otherwise in case of crash
                                     // and restart we will think that WAL is enabled, but data might be corrupted.
-                                    grpCtx.walEnabled(false);
+                                    grpCtx.globalWalEnabled(false);
                                 }
                             }
                             catch (Exception e) {
@@ -917,4 +1048,27 @@ private WalStateChangeWorker(WalStateProposeMessage msg, CheckpointFuture cpFut)
             onCompletedLocally(res);
         }
     }
+
+    /**
+     *
+     */
+    private static class TemporaryDisabledWal {
+        /** Groups with disabled WAL. */
+        private final Set<Integer> disabledGrps;
+
+        /** Remaining groups. */
+        private final Set<Integer> remainingGrps;
+
+        /** Topology version*/
+        private final AffinityTopologyVersion topVer;
+
+        /** */
+        public TemporaryDisabledWal(
+            Set<Integer> disabledGrps,
+            AffinityTopologyVersion topVer) {
+            this.disabledGrps = Collections.unmodifiableSet(disabledGrps);
+            this.remainingGrps = new HashSet<>(disabledGrps);
+            this.topVer = topVer;
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index 3e3bb0db9b9bb..dcb8b96db26ce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -1103,6 +1103,11 @@ private void removeNode(UUID nodeId) {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public void ownMoving(AffinityTopologyVersion topVer) {
+        // No-op
+    }
+
     /** {@inheritDoc} */
     @Override public void onEvicted(GridDhtLocalPartition part, boolean updateSeq) {
         assert updateSeq || lock.isWriteLockedByCurrentThread();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index d586a94d53203..2df2e8960afc9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -356,6 +356,13 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
      */
     public boolean own(GridDhtLocalPartition part);
 
+    /**
+     * Owns all moving partitions for the given topology version.
+     *
+     * @param topVer Topology version.
+     */
+    public void ownMoving(AffinityTopologyVersion topVer);
+
     /**
      * @param part Evicted partition.
      * @param updateSeq Update sequence increment flag.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 164f0bf451286..68104a5349283 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -2409,6 +2409,33 @@ private void removeNode(UUID nodeId) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void ownMoving(AffinityTopologyVersion topVer) {
+        lock.writeLock().lock();
+
+        try {
+            for (GridDhtLocalPartition locPart : grp.topology().currentLocalPartitions()) {
+                if (locPart.state() == MOVING) {
+                    boolean reserved = locPart.reserve();
+
+                    try {
+                        if (reserved && locPart.state() == MOVING && lastTopChangeVer.equals(topVer))
+                            grp.topology().own(locPart);
+                        else // topology changed, rebalancing must be restarted
+                            return;
+                    }
+                    finally {
+                        if (reserved)
+                            locPart.release();
+                    }
+                }
+            }
+        }
+        finally {
+            lock.writeLock().unlock();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void onEvicted(GridDhtLocalPartition part, boolean updateSeq) {
         ctx.database().checkpointReadLock();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 337553b28cfdc..dc4bfe9bc1720 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -57,6 +57,7 @@
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -282,6 +283,14 @@ Runnable addAssignments(
 
             final RebalanceFuture fut = new RebalanceFuture(grp, assignments, log, rebalanceId);
 
+            if (!grp.localWalEnabled())
+                fut.listen(new IgniteInClosureX<IgniteInternalFuture<Boolean>>() {
+                    @Override public void applyx(IgniteInternalFuture<Boolean> future) throws IgniteCheckedException {
+                        if (future.get())
+                            ctx.walState().onGroupRebalanceFinished(grp.groupId(), assignments.topologyVersion());
+                    }
+                });
+
             if (!oldFut.isInitial())
                 oldFut.cancel();
             else
@@ -722,9 +731,7 @@ public void handleSupplyMessage(
                                 // If message was last for this partition,
                                 // then we take ownership.
                                 if (last) {
-                                    top.own(part);
-
-                                    fut.partitionDone(nodeId, p);
+                                    fut.partitionDone(nodeId, p, true);
 
                                     if (log.isDebugEnabled())
                                         log.debug("Finished rebalancing partition: " + part);
@@ -737,14 +744,14 @@ public void handleSupplyMessage(
                         }
                         else {
                             if (last)
-                                fut.partitionDone(nodeId, p);
+                                fut.partitionDone(nodeId, p, false);
 
                             if (log.isDebugEnabled())
                                 log.debug("Skipping rebalancing partition (state is not MOVING): " + part);
                         }
                     }
                     else {
-                        fut.partitionDone(nodeId, p);
+                        fut.partitionDone(nodeId, p, false);
 
                         if (log.isDebugEnabled())
                             log.debug("Skipping rebalancing partition (it does not belong on current node): " + p);
@@ -762,7 +769,7 @@ public void handleSupplyMessage(
             }
 
             for (Integer miss : supply.missed())
-                fut.partitionDone(nodeId, miss);
+                fut.partitionDone(nodeId, miss, false);
 
             GridDhtPartitionDemandMessage d = new GridDhtPartitionDemandMessage(
                 supply.rebalanceId(),
@@ -1064,8 +1071,11 @@ private void cleanupRemoteContexts(UUID nodeId) {
          * @param nodeId Node id.
          * @param p Partition number.
          */
-        private void partitionDone(UUID nodeId, int p) {
+        private void partitionDone(UUID nodeId, int p, boolean updateState) {
             synchronized (this) {
+                if (updateState && grp.localWalEnabled())
+                    grp.topology().own(grp.topology().localPartition(p));
+
                 if (isDone())
                     return;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 0d57d483f71a9..a21d98e67d320 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1702,6 +1702,8 @@ public void finishMerged() {
                 if (!grp.isLocal())
                     grp.topology().onExchangeDone(this, grp.affinity().readyAffinity(res), false);
             }
+
+            cctx.walState().changeLocalStatesOnExchangeDone(exchId.topologyVersion());
         }
 
         if (super.onDone(res, err)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 3009febae9b44..59aad5f83ade0 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -21,6 +21,7 @@
 import java.io.FileFilter;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.io.Serializable;
 import java.lang.ref.SoftReference;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -279,7 +280,13 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     private static final String MBEAN_GROUP = "Persistent Store";
 
     /** WAL marker prefix for meta store. */
-    private static final String WAL_KEY_PREFIX = "grp-wal-disabled-";
+    private static final String WAL_KEY_PREFIX = "grp-wal-";
+
+    /** WAL marker prefix for meta store. */
+    private static final String WAL_GLOBAL_KEY_PREFIX = WAL_KEY_PREFIX + "disabled-";
+
+    /** WAL marker prefix for meta store. */
+    private static final String WAL_LOCAL_KEY_PREFIX = WAL_KEY_PREFIX + "local-disabled-";
 
     /** WAL marker predicate for meta store. */
     private static final IgnitePredicate<String> WAL_KEY_PREFIX_PRED = new IgnitePredicate<String>() {
@@ -379,7 +386,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     private List<MetastorageLifecycleListener> metastorageLifecycleLsnrs;
 
     /** Initially disabled cache groups. */
-    private Collection<Integer> initiallyWalDisabledGrps;
+    private Collection<Integer> initiallyGlobalWalDisabledGrps = new HashSet<>();
+
+    private Collection<Integer> initiallyLocalWalDisabledGrps = new HashSet<>();
 
     /**
      * @param ctx Kernal context.
@@ -563,7 +572,7 @@ private void readMetastore() throws IgniteCheckedException {
 
                 applyLastUpdates(status, true);
 
-                initiallyWalDisabledGrps = walDisabledGroups();
+                fillWalDisabledGroups();
 
                 notifyMetastorageReadyForRead();
             }
@@ -1935,7 +1944,8 @@ private WALPointer restoreMemory(CheckpointStatus status, boolean storeOnly,
         int applied = 0;
         WALPointer lastRead = null;
 
-        Collection<Integer> ignoreGrps = storeOnly ? Collections.emptySet() : initiallyWalDisabledGrps;
+        Collection<Integer> ignoreGrps = storeOnly ? Collections.emptySet() :
+            F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps);
 
         try (WALIterator it = cctx.wal().replay(status.endPtr)) {
             while (it.hasNextX()) {
@@ -2189,7 +2199,8 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
         long start = U.currentTimeMillis();
         int applied = 0;
 
-        Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() : initiallyWalDisabledGrps;
+        Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() :
+            F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps);
 
         try (WALIterator it = cctx.wal().replay(status.startPtr)) {
             Map<T2<Integer, Integer>, T2<Integer, Long>> partStates = new HashMap<>();
@@ -4340,13 +4351,16 @@ public DataStorageMetricsImpl persistentStoreMetricsImpl() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean walEnabled(int grpId) {
-        return !initiallyWalDisabledGrps.contains(grpId);
+    @Override public boolean walEnabled(int grpId, boolean local) {
+        if (local)
+            return !initiallyLocalWalDisabledGrps.contains(grpId);
+        else
+            return !initiallyGlobalWalDisabledGrps.contains(grpId);
     }
 
     /** {@inheritDoc} */
-    @Override public void walEnabled(int grpId, boolean enabled) {
-        String key = walGroupIdToKey(grpId);
+    @Override public void walEnabled(int grpId, boolean enabled, boolean local) {
+        String key = walGroupIdToKey(grpId, local);
 
         checkpointReadLock();
 
@@ -4366,27 +4380,26 @@ public DataStorageMetricsImpl persistentStoreMetricsImpl() {
     }
 
     /**
-     * @return List of initially WAL-disabled groups.
+     *
      */
-    private Collection<Integer> walDisabledGroups() {
+    private void fillWalDisabledGroups() {
         MetaStorage meta = cctx.database().metaStorage();
 
         try {
             Set<String> keys = meta.readForPredicate(WAL_KEY_PREFIX_PRED).keySet();
 
             if (keys.isEmpty())
-                return Collections.emptySet();
-
-            HashSet<Integer> res = new HashSet<>(keys.size());
+                return;
 
             for (String key : keys) {
-                int grpId = walKeyToGroupId(key);
+                T2<Integer, Boolean> t2 = walKeyToGroupIdAndLocalFlag(key);
 
-                res.add(grpId);
+                if (t2.get2())
+                    initiallyLocalWalDisabledGrps.add(t2.get1());
+                else
+                    initiallyGlobalWalDisabledGrps.add(t2.get1());
             }
 
-            return res;
-
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException("Failed to read cache groups WAL state.", e);
@@ -4399,8 +4412,11 @@ private Collection<Integer> walDisabledGroups() {
      * @param grpId Group ID.
      * @return Key.
      */
-    private static String walGroupIdToKey(int grpId) {
-        return WAL_KEY_PREFIX + grpId;
+    private static String walGroupIdToKey(int grpId, boolean local) {
+        if (local)
+            return WAL_LOCAL_KEY_PREFIX + grpId;
+        else
+            return WAL_GLOBAL_KEY_PREFIX + grpId;
     }
 
     /**
@@ -4409,7 +4425,10 @@ private static String walGroupIdToKey(int grpId) {
      * @param key Key.
      * @return Group ID.
      */
-    private static int walKeyToGroupId(String key) {
-        return Integer.parseInt(key.substring(WAL_KEY_PREFIX.length()));
+    private static T2<Integer, Boolean> walKeyToGroupIdAndLocalFlag(String key) {
+        if (key.startsWith(WAL_LOCAL_KEY_PREFIX))
+            return new T2<>(Integer.parseInt(key.substring(WAL_LOCAL_KEY_PREFIX.length())), true);
+        else
+            return new T2<>(Integer.parseInt(key.substring(WAL_GLOBAL_KEY_PREFIX.length())), false);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 8746dca71977b..bf080b6736d0d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -1035,7 +1035,7 @@ public MetaStorage metaStorage() {
      * @param grpId Group ID.
      * @return WAL enabled flag.
      */
-    public boolean walEnabled(int grpId) {
+    public boolean walEnabled(int grpId, boolean local) {
         return false;
     }
 
@@ -1045,7 +1045,7 @@ public boolean walEnabled(int grpId) {
      * @param grpId Group id.
      * @param enabled flag.
      */
-    public void walEnabled(int grpId, boolean enabled) {
+    public void walEnabled(int grpId, boolean enabled, boolean local) {
         // No-op.
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 837f3d01b2b2c..1c1b3e24e175d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -845,16 +845,22 @@ public PageStore getStore(int grpId, int partId) throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public void beforeCacheGroupStart(CacheGroupDescriptor grpDesc) {
-        if (grpDesc.persistenceEnabled() && !cctx.database().walEnabled(grpDesc.groupId())) {
-            File dir = cacheWorkDir(grpDesc.config());
+        if (grpDesc.persistenceEnabled()) {
+            boolean localEnabled = cctx.database().walEnabled(grpDesc.groupId(), true);
+            boolean globalEnabled = cctx.database().walEnabled(grpDesc.groupId(), false);
 
-            assert dir.exists();
+            if (!localEnabled || !globalEnabled) {
+                File dir = cacheWorkDir(grpDesc.config());
 
-            boolean res = IgniteUtils.delete(dir);
+                assert dir.exists();
 
-            assert res;
+                boolean res = IgniteUtils.delete(dir);
 
-            grpDesc.walEnabled(false);
+                assert res;
+
+                if (!globalEnabled)
+                    grpDesc.walEnabled(false);
+            }
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
new file mode 100644
index 0000000000000..aa2613fa3a1a2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -0,0 +1,566 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.file.OpenOption;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static boolean disableWalDuringRebalancing = true;
+
+    /** */
+    private static final AtomicReference<CountDownLatch> supplyMessageLatch = new AtomicReference<>();
+
+    /** */
+    private static final AtomicReference<CountDownLatch> fileIOLatch = new AtomicReference<>();
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setMaxSize(200 * 1024 * 1024)
+                        .setInitialSize(200 * 1024 * 1024)
+                )
+                // Test verifies checkpoint count, so it is essencial that no checkpoint is triggered by timeout
+                .setCheckpointFrequency(999_999_999_999L)
+                .setFileIOFactory(new TestFileIOFactory(new DataStorageConfiguration().getFileIOFactory()))
+        );
+
+        cfg.setCacheConfiguration(
+            new CacheConfiguration(DEFAULT_CACHE_NAME)
+                // Test checks internal state before and after rebalance, so it is configured to be triggered manually
+                .setRebalanceDelay(-1)
+        );
+
+        cfg.setCommunicationSpi(new TcpCommunicationSpi() {
+            @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+                if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof GridDhtPartitionSupplyMessage) {
+                    int grpId = ((GridDhtPartitionSupplyMessage)((GridIoMessage)msg).message()).groupId();
+
+                    if (grpId == CU.cacheId(DEFAULT_CACHE_NAME)) {
+                        CountDownLatch latch0 = supplyMessageLatch.get();
+
+                        if (latch0 != null)
+                            try {
+                                latch0.await();
+                            }
+                            catch (InterruptedException ex) {
+                                throw new IgniteException(ex);
+                            }
+                    }
+                }
+
+                super.sendMessage(node, msg);
+            }
+
+            @Override public void sendMessage(ClusterNode node, Message msg,
+                IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+                if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof GridDhtPartitionSupplyMessage) {
+                    int grpId = ((GridDhtPartitionSupplyMessage)((GridIoMessage)msg).message()).groupId();
+
+                    if (grpId == CU.cacheId(DEFAULT_CACHE_NAME)) {
+                        CountDownLatch latch0 = supplyMessageLatch.get();
+
+                        if (latch0 != null)
+                            try {
+                                latch0.await();
+                            }
+                            catch (InterruptedException ex) {
+                                throw new IgniteException(ex);
+                            }
+                    }
+                }
+
+                super.sendMessage(node, msg, ackC);
+            }
+        });
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        System.setProperty(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING,
+            Boolean.toString(disableWalDuringRebalancing));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        CountDownLatch msgLatch = supplyMessageLatch.get();
+
+        if (msgLatch != null) {
+            while (msgLatch.getCount() > 0)
+                msgLatch.countDown();
+
+            supplyMessageLatch.set(null);
+        }
+
+        CountDownLatch fileLatch = fileIOLatch.get();
+
+        if (fileLatch != null) {
+            while (fileLatch.getCount() > 0)
+                fileLatch.countDown();
+
+            fileIOLatch.set(null);
+        }
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        disableWalDuringRebalancing = true;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWalDisabledDuringRebalancing() throws Exception {
+        doTestSimple();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWalNotDisabledIfParameterSetToFalse() throws Exception {
+        disableWalDuringRebalancing = false;
+
+        doTestSimple();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void doTestSimple() throws Exception {
+        Ignite ignite = startGrids(3);
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        for (int k = 0; k < 10_000; k++)
+            cache.put(k, k);
+
+        IgniteEx newIgnite = startGrid(3);
+
+        final GridCacheDatabaseSharedManager.CheckpointHistory cpHistory =
+            ((GridCacheDatabaseSharedManager)newIgnite.context().cache().context().database()).checkpointHistory();
+
+        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return !cpHistory.checkpoints().isEmpty();
+            }
+        }, 10_000);
+
+        U.sleep(10); // To ensure timestamp granularity.
+
+        long newIgniteStartedTimestamp = System.currentTimeMillis();
+
+        ignite.cluster().setBaselineTopology(4);
+
+        CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
+
+        assertEquals(!disableWalDuringRebalancing, grpCtx.walEnabled());
+
+        U.sleep(10); // To ensure timestamp granularity.
+
+        long rebalanceStartedTimestamp = System.currentTimeMillis();
+
+        for (Ignite g : G.allGrids())
+            g.cache(DEFAULT_CACHE_NAME).rebalance();
+
+        awaitPartitionMapExchange();
+
+        assertTrue(grpCtx.walEnabled());
+
+        U.sleep(10); // To ensure timestamp granularity.
+
+        long rebalanceFinishedTimestamp = System.currentTimeMillis();
+
+        for (Integer k = 0; k < 1000; k++)
+            assertEquals("k=" + k, k, cache.get(k));
+
+        int checkpointsBeforeNodeStarted = 0;
+        int checkpointsBeforeRebalance = 0;
+        int checkpointsAfterRebalance = 0;
+
+        for (Long timestamp : cpHistory.checkpoints()) {
+            if (timestamp < newIgniteStartedTimestamp)
+                checkpointsBeforeNodeStarted++;
+            else if (timestamp >= newIgniteStartedTimestamp && timestamp < rebalanceStartedTimestamp)
+                checkpointsBeforeRebalance++;
+            else if (timestamp >= rebalanceStartedTimestamp && timestamp <= rebalanceFinishedTimestamp)
+                checkpointsAfterRebalance++;
+        }
+
+        assertEquals(1, checkpointsBeforeNodeStarted); // checkpoint on start
+        assertEquals(0, checkpointsBeforeRebalance);
+        assertEquals(disableWalDuringRebalancing ? 1 : 0, checkpointsAfterRebalance); // checkpoint if WAL was re-activated
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLocalAndGlobalWalStateInterdependence() throws Exception {
+        Ignite ignite = startGrids(3);
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        for (int k = 0; k < 10_000; k++)
+            cache.put(k, k);
+
+        IgniteEx newIgnite = startGrid(3);
+
+        ignite.cluster().setBaselineTopology(ignite.cluster().nodes());
+
+        CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
+
+        assertFalse(grpCtx.walEnabled());
+
+        ignite.cluster().disableWal(DEFAULT_CACHE_NAME);
+
+        for (Ignite g : G.allGrids())
+            g.cache(DEFAULT_CACHE_NAME).rebalance();
+
+        awaitPartitionMapExchange();
+
+        assertFalse(grpCtx.walEnabled()); // WAL is globally disabled
+
+        ignite.cluster().enableWal(DEFAULT_CACHE_NAME);
+
+        assertTrue(grpCtx.walEnabled());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testParallelExchangeDuringRebalance() throws Exception {
+        doTestParallelExchange(supplyMessageLatch);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testParallelExchangeDuringCheckpoint() throws Exception {
+        doTestParallelExchange(fileIOLatch);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void doTestParallelExchange(AtomicReference<CountDownLatch> latchRef) throws Exception {
+        Ignite ignite = startGrids(3);
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        for (int k = 0; k < 10_000; k++)
+            cache.put(k, k);
+
+        IgniteEx newIgnite = startGrid(3);
+
+        CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        latchRef.set(latch);
+
+        ignite.cluster().setBaselineTopology(ignite.cluster().nodes());
+
+        for (Ignite g : G.allGrids())
+            g.cache(DEFAULT_CACHE_NAME).rebalance();
+
+        assertFalse(grpCtx.walEnabled());
+
+        // TODO : test with client node as well
+        startGrid(4); // Trigger exchange
+
+        assertFalse(grpCtx.walEnabled());
+
+        latch.countDown();
+
+        assertFalse(grpCtx.walEnabled());
+
+        for (Ignite g : G.allGrids())
+            g.cache(DEFAULT_CACHE_NAME).rebalance();
+
+        awaitPartitionMapExchange();
+
+        assertTrue(grpCtx.walEnabled());
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDataClearedAfterRestartWithDisabledWal() throws Exception {
+        Ignite ignite = startGrid(0);
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        for (int k = 0; k < 10_000; k++)
+            cache.put(k, k);
+
+        IgniteEx newIgnite = startGrid(1);
+
+        ignite.cluster().setBaselineTopology(2);
+
+        CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
+
+        assertFalse(grpCtx.localWalEnabled());
+
+        stopGrid(1);
+        stopGrid(0);
+
+        newIgnite = startGrid(1);
+
+        newIgnite.cluster().active(true);
+
+        newIgnite.cluster().setBaselineTopology(newIgnite.cluster().nodes());
+
+        cache = newIgnite.cache(DEFAULT_CACHE_NAME);
+
+        for (int k = 0; k < 10_000; k++)
+            assertFalse("k=" + k +", v=" + cache.get(k), cache.containsKey(k));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWalNotDisabledAfterShrinkingBaselineTopology() throws Exception {
+        Ignite ignite = startGrids(4);
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        for (int k = 0; k < 10_000; k++)
+            cache.put(k, k);
+
+        for (Ignite g : G.allGrids()) {
+            CacheGroupContext grpCtx = ((IgniteEx)g).cachex(DEFAULT_CACHE_NAME).context().group();
+
+            assertTrue(grpCtx.walEnabled());
+        }
+
+        stopGrid(2);
+
+        ignite.cluster().setBaselineTopology(5);
+
+        for (Ignite g : G.allGrids()) {
+            CacheGroupContext grpCtx = ((IgniteEx)g).cachex(DEFAULT_CACHE_NAME).context().group();
+
+            assertTrue(grpCtx.walEnabled());
+
+            g.cache(DEFAULT_CACHE_NAME).rebalance();
+        }
+
+        awaitPartitionMapExchange();
+
+        for (Ignite g : G.allGrids()) {
+            CacheGroupContext grpCtx = ((IgniteEx)g).cachex(DEFAULT_CACHE_NAME).context().group();
+
+            assertTrue(grpCtx.walEnabled());
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestFileIOFactory implements FileIOFactory {
+        /** */
+        private final FileIOFactory delegate;
+
+        /**
+         * @param delegate Delegate.
+         */
+        TestFileIOFactory(FileIOFactory delegate) {
+            this.delegate = delegate;
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            return new TestFileIO(delegate.create(file));
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+            return new TestFileIO(delegate.create(file, modes));
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestFileIO implements FileIO {
+        /** */
+        private final FileIO delegate;
+
+        /**
+         * @param delegate Delegate.
+         */
+        TestFileIO(FileIO delegate) {
+            this.delegate = delegate;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long position() throws IOException {
+            return delegate.position();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void position(long newPosition) throws IOException {
+            delegate.position(newPosition);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int read(ByteBuffer destBuf) throws IOException {
+            return delegate.read(destBuf);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int read(ByteBuffer destBuf, long position) throws IOException {
+            return delegate.read(destBuf, position);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int read(byte[] buf, int off, int len) throws IOException {
+            return delegate.read(buf, off, len);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int write(ByteBuffer srcBuf) throws IOException {
+            CountDownLatch latch = fileIOLatch.get();
+
+            if (latch != null && Thread.currentThread().getName().contains("checkpoint"))
+                try {
+                    latch.await();
+                }
+                catch (InterruptedException ex) {
+                    throw new IgniteException(ex);
+                }
+
+            return delegate.write(srcBuf);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int write(ByteBuffer srcBuf, long position) throws IOException {
+            CountDownLatch latch = fileIOLatch.get();
+
+            if (latch != null && Thread.currentThread().getName().contains("checkpoint"))
+                try {
+                    latch.await();
+                }
+                catch (InterruptedException ex) {
+                    throw new IgniteException(ex);
+                }
+
+            return delegate.write(srcBuf, position);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void write(byte[] buf, int off, int len) throws IOException {
+            CountDownLatch latch = fileIOLatch.get();
+
+            if (latch != null && Thread.currentThread().getName().contains("checkpoint"))
+                try {
+                    latch.await();
+                }
+                catch (InterruptedException ex) {
+                    throw new IgniteException(ex);
+                }
+
+            delegate.write(buf, off, len);
+        }
+
+        /** {@inheritDoc} */
+        @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
+            return delegate.map(maxWalSegmentSize);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void force() throws IOException {
+            delegate.force();
+        }
+
+        /** {@inheritDoc} */
+        @Override public long size() throws IOException {
+            return delegate.size();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void clear() throws IOException {
+            delegate.clear();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void close() throws IOException {
+            delegate.close();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index f955b118d7e92..ede537ebb6df1 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
+import org.apache.ignite.internal.processors.cache.persistence.LocalWalModeChangeDuringRebalancingSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAllBaselineNodesOnlineFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOfflineBaselineNodeFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest;
@@ -135,5 +136,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgniteCheckpointDirtyPagesForLowLoadTest.class);
 
         suite.addTestSuite(IgnitePdsCorruptedStoreTest.class);
+
+        suite.addTestSuite(LocalWalModeChangeDuringRebalancingSelfTest.class);
     }
 }

From b80fdbedc11b48aa89c6f29146363cec6f552abb Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Wed, 18 Apr 2018 19:04:39 +0300
Subject: [PATCH 080/543] IGNITE-8276 Fixed incorrect assertion during
 initialValue - Fixes #3827.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../ignite/internal/processors/cache/GridCacheMapEntry.java   | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index a6ef0d284d6b4..9f3686aad47ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -3571,12 +3571,10 @@ private IgniteTxLocalAdapter currentTx() {
      * @param oldRow Old row if available.
      * @throws IgniteCheckedException If update failed.
      */
-    protected boolean storeValue(CacheObject val,
+    protected boolean storeValue(@Nullable CacheObject val,
         long expireTime,
         GridCacheVersion ver,
         @Nullable CacheDataRow oldRow) throws IgniteCheckedException {
-        assert val != null : "null values in update for key: " + key;
-
         return storeValue(val, expireTime, ver, oldRow, null);
     }
 

From 5cd32329fe5f303eaf369519771ee22f2a2cf822 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 18 Apr 2018 19:41:44 +0300
Subject: [PATCH 081/543] IGNITE-8116 Fixed historical rebalance from WAL

---
 .../cache/IgniteCacheOffheapManagerImpl.java  |  28 +--
 .../preloader/GridDhtPartitionDemander.java   | 118 ++++++++-----
 .../preloader/GridDhtPartitionSupplier.java   |  54 ++++--
 .../dht/preloader/GridDhtPreloader.java       |   4 +-
 .../IgniteDhtPartitionsToReloadMap.java       |   2 +-
 .../persistence/GridCacheOffheapManager.java  |  82 +++++++--
 modules/core/src/test/config/log4j-test.xml   |   6 -
 ...sAtomicCacheHistoricalRebalancingTest.java |  40 +++++
 ...IgnitePdsCacheRebalancingAbstractTest.java |  32 ++--
 ...tePdsTxCacheHistoricalRebalancingTest.java |  39 +++++
 .../db/wal/IgniteWalRebalanceTest.java        | 164 ++++++++++++++++++
 ...entQueryReplicatedNodeRestartSelfTest.java |   1 -
 .../IgnitePdsWithIndexingCoreTestSuite.java   |   7 +
 13 files changed, 467 insertions(+), 110 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index f8cc86f8cecdb..5c78eb5b4b4a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -861,7 +861,8 @@ private long allocateForTree() throws IgniteCheckedException {
             }
 
             @Override protected void onClose() throws IgniteCheckedException {
-                assert loc != null && loc.state() == OWNING && loc.reservations() > 0;
+                assert loc != null && loc.state() == OWNING && loc.reservations() > 0
+                    : "Partition should be in OWNING state and has at least 1 reservation: " + loc;
 
                 loc.release();
             }
@@ -874,36 +875,37 @@ private long allocateForTree() throws IgniteCheckedException {
         throws IgniteCheckedException {
 
         final TreeMap<Integer, GridCloseableIterator<CacheDataRow>> iterators = new TreeMap<>();
-        Set<Integer> missing = null;
+
+        Set<Integer> missing = new HashSet<>();
 
         for (Integer p : parts.fullSet()) {
             GridCloseableIterator<CacheDataRow> partIter = reservedIterator(p, topVer);
 
             if (partIter == null) {
-                if (missing == null)
-                    missing = new HashSet<>();
-
                 missing.add(p);
+
+                continue;
             }
-            else
-                iterators.put(p, partIter);
+
+            iterators.put(p, partIter);
         }
 
-        IgniteRebalanceIterator iter = new IgniteRebalanceIteratorImpl(iterators, historicalIterator(parts.historicalMap()));
+        IgniteHistoricalIterator historicalIterator = historicalIterator(parts.historicalMap(), missing);
 
-        if (missing != null) {
-            for (Integer p : missing)
-                iter.setPartitionMissing(p);
-        }
+        IgniteRebalanceIterator iter = new IgniteRebalanceIteratorImpl(iterators, historicalIterator);
+
+        for (Integer p : missing)
+            iter.setPartitionMissing(p);
 
         return iter;
     }
 
     /**
      * @param partCntrs Partition counters map.
+     * @param missing Set of partitions need to populate if partition is missing or failed to reserve.
      * @return Historical iterator.
      */
-    @Nullable protected IgniteHistoricalIterator historicalIterator(CachePartitionPartialCountersMap partCntrs)
+    @Nullable protected IgniteHistoricalIterator historicalIterator(CachePartitionPartialCountersMap partCntrs, Set<Integer> missing)
         throws IgniteCheckedException {
         return null;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index dc4bfe9bc1720..c94f511d91887 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -459,7 +459,9 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
                         + ", topology=" + fut.topologyVersion() + ", rebalanceId=" + fut.rebalanceId + "]");
             }
 
-            int stripes = ctx.gridConfig().getRebalanceThreadPoolSize();
+            int totalStripes = ctx.gridConfig().getRebalanceThreadPoolSize();
+
+            int stripes = totalStripes;
 
             final List<IgniteDhtDemandedPartitionsMap> stripePartitions = new ArrayList<>(stripes);
             for (int i = 0; i < stripes; i++)
@@ -467,7 +469,7 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
 
             // Reserve one stripe for historical partitions.
             if (parts.hasHistorical()) {
-                stripePartitions.add(stripes - 1, new IgniteDhtDemandedPartitionsMap(parts.historicalMap(), null));
+                stripePartitions.set(stripes - 1, new IgniteDhtDemandedPartitionsMap(parts.historicalMap(), null));
 
                 if (stripes > 1)
                     stripes--;
@@ -478,7 +480,7 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
             for (int i = 0; it.hasNext(); i++)
                 stripePartitions.get(i % stripes).addFull(it.next());
 
-            for (int stripe = 0; stripe < stripes; stripe++) {
+            for (int stripe = 0; stripe < totalStripes; stripe++) {
                 if (!stripePartitions.get(stripe).isEmpty()) {
                     // Create copy of demand message with new striped partitions map.
                     final GridDhtPartitionDemandMessage demandMsg = d.withNewPartitionsMap(stripePartitions.get(stripe));
@@ -489,23 +491,27 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
 
                     final int topicId = stripe;
 
-                    Runnable initDemandRequestTask = () -> {
+                    IgniteInternalFuture<?> clearAllFuture = clearFullPartitions(fut, demandMsg.partitions().fullSet());
+
+                    // Start rebalancing after clearing full partitions is finished.
+                    clearAllFuture.listen(f -> ctx.kernalContext().closure().runLocalSafe(() -> {
+                        if (fut.isDone())
+                            return;
+
                         try {
-                            if (!fut.isDone()) {
-                                ctx.io().sendOrderedMessage(node, rebalanceTopics.get(topicId),
-                                        demandMsg.convertIfNeeded(node.version()), grp.ioPolicy(), demandMsg.timeout());
-
-                                // Cleanup required in case partitions demanded in parallel with cancellation.
-                                synchronized (fut) {
-                                    if (fut.isDone())
-                                        fut.cleanupRemoteContexts(node.id());
-                                }
+                            ctx.io().sendOrderedMessage(node, rebalanceTopics.get(topicId),
+                                demandMsg.convertIfNeeded(node.version()), grp.ioPolicy(), demandMsg.timeout());
 
-                                if (log.isDebugEnabled())
-                                    log.debug("Requested rebalancing [from node=" + node.id() + ", listener index=" +
-                                            topicId + ", partitions count=" + stripePartitions.get(topicId).size() +
-                                            " (" + stripePartitions.get(topicId).partitionsList() + ")]");
+                            // Cleanup required in case partitions demanded in parallel with cancellation.
+                            synchronized (fut) {
+                                if (fut.isDone())
+                                    fut.cleanupRemoteContexts(node.id());
                             }
+
+                            if (log.isDebugEnabled())
+                                log.debug("Requested rebalancing [from node=" + node.id() + ", listener index=" +
+                                    topicId + " " + demandMsg.rebalanceId() + ", partitions count=" + stripePartitions.get(topicId).size() +
+                                    " (" + stripePartitions.get(topicId).partitionsList() + ")]");
                         }
                         catch (IgniteCheckedException e1) {
                             ClusterTopologyCheckedException cause = e1.getCause(ClusterTopologyCheckedException.class);
@@ -522,31 +528,26 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
 
                             fut.cancel();
                         }
-                    };
-
-                    awaitClearingAndStartRebalance(fut, demandMsg, initDemandRequestTask);
+                    }, true));
                 }
             }
         }
     }
 
     /**
-     * Awaits partitions clearing for full partitions and sends initial demand request
-     * after all partitions are cleared and safe to consume data.
+     * Creates future which will be completed when all {@code fullPartitions} are cleared.
      *
      * @param fut Rebalance future.
-     * @param demandMessage Initial demand message which contains set of full partitions to await.
-     * @param initDemandRequestTask Task which sends initial demand request.
+     * @param fullPartitions Set of full partitions need to be cleared.
+     * @return Future which will be completed when given partitions are cleared.
      */
-    private void awaitClearingAndStartRebalance(RebalanceFuture fut,
-                                                GridDhtPartitionDemandMessage demandMessage,
-                                                Runnable initDemandRequestTask) {
-        Set<Integer> fullPartitions = demandMessage.partitions().fullSet();
+    private IgniteInternalFuture<?> clearFullPartitions(RebalanceFuture fut, Set<Integer> fullPartitions) {
+        final GridFutureAdapter clearAllFuture = new GridFutureAdapter();
 
         if (fullPartitions.isEmpty()) {
-            ctx.kernalContext().closure().runLocalSafe(initDemandRequestTask, true);
+            clearAllFuture.onDone();
 
-            return;
+            return clearAllFuture;
         }
 
         for (GridCacheContext cctx : grp.caches()) {
@@ -560,16 +561,19 @@ private void awaitClearingAndStartRebalance(RebalanceFuture fut,
         final AtomicInteger clearingPartitions = new AtomicInteger(fullPartitions.size());
 
         for (int partId : fullPartitions) {
-            if (fut.isDone())
-                return;
+            if (fut.isDone()) {
+                clearAllFuture.onDone();
+
+                return clearAllFuture;
+            }
 
             GridDhtLocalPartition part = grp.topology().localPartition(partId);
 
             if (part != null && part.state() == MOVING) {
                 part.onClearFinished(f -> {
-                    // Cancel rebalance if partition clearing was failed.
-                    if (f.error() != null) {
-                        if (!fut.isDone()) {
+                    if (!fut.isDone()) {
+                        // Cancel rebalance if partition clearing was failed.
+                        if (f.error() != null) {
                             for (GridCacheContext cctx : grp.caches()) {
                                 if (cctx.statisticsEnabled()) {
                                     final CacheMetricsImpl metrics = cctx.cache().metrics0();
@@ -581,30 +585,54 @@ private void awaitClearingAndStartRebalance(RebalanceFuture fut,
                             log.error("Unable to await partition clearing " + part, f.error());
 
                             fut.cancel();
+
+                            clearAllFuture.onDone(f.error());
                         }
-                    }
-                    else {
-                        if (!fut.isDone()) {
-                            int existed = clearingPartitions.decrementAndGet();
+                        else {
+                            int remaining = clearingPartitions.decrementAndGet();
 
                             for (GridCacheContext cctx : grp.caches()) {
                                 if (cctx.statisticsEnabled()) {
                                     final CacheMetricsImpl metrics = cctx.cache().metrics0();
 
-                                    metrics.rebalanceClearingPartitions(existed);
+                                    metrics.rebalanceClearingPartitions(remaining);
                                 }
                             }
 
-                            // If all partitions are cleared send initial demand message.
-                            if (existed == 0)
-                                ctx.kernalContext().closure().runLocalSafe(initDemandRequestTask, true);
+                            if (log.isDebugEnabled())
+                                log.debug("Remaining clearing partitions [grp=" + grp.cacheOrGroupName()
+                                    + ", remaining=" + remaining + "]");
+
+                            if (remaining == 0)
+                                clearAllFuture.onDone();
                         }
                     }
+                    else {
+                        clearAllFuture.onDone();
+                    }
                 });
             }
-            else
-                clearingPartitions.decrementAndGet();
+            else {
+                int remaining = clearingPartitions.decrementAndGet();
+
+                for (GridCacheContext cctx : grp.caches()) {
+                    if (cctx.statisticsEnabled()) {
+                        final CacheMetricsImpl metrics = cctx.cache().metrics0();
+
+                        metrics.rebalanceClearingPartitions(remaining);
+                    }
+                }
+
+                if (log.isDebugEnabled())
+                    log.debug("Remaining clearing partitions [grp=" + grp.cacheOrGroupName()
+                        + ", remaining=" + remaining + "]");
+
+                if (remaining == 0)
+                    clearAllFuture.onDone();
+            }
         }
+
+        return clearAllFuture;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 6d2f526d73780..a3ee305406207 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -173,7 +173,8 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
 
         if (curTop.compareTo(demTop) > 0) {
             if (log.isDebugEnabled())
-                log.debug("Demand request outdated [currentTopVer=" + curTop
+                log.debug("Demand request outdated [grp=" + grp.cacheOrGroupName()
+                        + ", currentTopVer=" + curTop
                         + ", demandTopVer=" + demTop
                         + ", from=" + nodeId
                         + ", topicId=" + topicId + "]");
@@ -189,10 +190,19 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
 
                 if (sctx != null && sctx.rebalanceId == -d.rebalanceId()) {
                     clearContext(scMap.remove(contextId), log);
+
+                    if (log.isDebugEnabled())
+                        log.debug("Supply context cleaned [grp=" + grp.cacheOrGroupName()
+                            + ", from=" + nodeId
+                            + ", demandMsg=" + d
+                            + ", supplyContext=" + sctx + "]");
                 }
                 else {
                     if (log.isDebugEnabled())
-                        log.debug("Stale context cleanup message " + d + ", supplyContext=" + sctx);
+                        log.debug("Stale supply context cleanup message [grp=" + grp.cacheOrGroupName()
+                            + ", from=" + nodeId
+                            + ", demandMsg=" + d
+                            + ", supplyContext=" + sctx + "]");
                 }
 
                 return;
@@ -200,13 +210,16 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
         }
 
         if (log.isDebugEnabled())
-            log.debug("Demand request accepted [current=" + curTop + ", demanded=" + demTop +
-                ", from=" + nodeId + ", topicId=" + topicId + "]");
+            log.debug("Demand request accepted [grp=" + grp.cacheOrGroupName()
+                + ", from=" + nodeId
+                + ", currentVer=" + curTop
+                + ", demandedVer=" + demTop
+                + ", topicId=" + topicId + "]");
 
         ClusterNode node = grp.shared().discovery().node(nodeId);
 
         if (node == null)
-            return; // Context will be cleaned at topology change.
+            return;
 
         try {
             SupplyContext sctx;
@@ -217,13 +230,27 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                 if (sctx != null && d.rebalanceId() < sctx.rebalanceId) {
                     // Stale message, return context back and return.
                     scMap.put(contextId, sctx);
+
+                    if (log.isDebugEnabled())
+                        log.debug("Stale demand message [grp=" + grp.cacheOrGroupName()
+                            + ", actualContext=" + sctx
+                            + ", from=" + nodeId
+                            + ", demandMsg=" + d + "]");
+
                     return;
                 }
             }
 
             // Demand request should not contain empty partitions if no supply context is associated with it.
-            if (sctx == null && (d.partitions() == null || d.partitions().isEmpty()))
+            if (sctx == null && (d.partitions() == null || d.partitions().isEmpty())) {
+                if (log.isDebugEnabled())
+                    log.debug("Empty demand message [grp=" + grp.cacheOrGroupName()
+                        + ", from=" + nodeId
+                        + ", topicId=" + topicId
+                        + ", demandMsg=" + d + "]");
+
                 return;
+            }
 
             assert !(sctx != null && !d.partitions().isEmpty());
 
@@ -271,7 +298,8 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
 
                     GridDhtLocalPartition loc = top.localPartition(part, d.topologyVersion(), false);
 
-                    assert loc != null && loc.state() == GridDhtPartitionState.OWNING;
+                    assert loc != null && loc.state() == GridDhtPartitionState.OWNING
+                        : "Partition should be in OWNING state: " + loc;
 
                     s.addEstimatedKeysCount(grp.offheap().totalPartitionEntriesCount(part));
                 }
@@ -323,7 +351,8 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
 
                 GridDhtLocalPartition loc = top.localPartition(part, d.topologyVersion(), false);
 
-                assert (loc != null && loc.state() == OWNING && loc.reservations() > 0) || iter.isPartitionMissing(part) : loc;
+                assert (loc != null && loc.state() == OWNING && loc.reservations() > 0) || iter.isPartitionMissing(part)
+                    : "Partition should be in OWNING state and has at least 1 reservation " + loc;
 
                 if (iter.isPartitionMissing(part) && remainingParts.contains(part)) {
                     s.missed(part);
@@ -361,9 +390,6 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
 
                     remainingParts.remove(part);
                 }
-
-                // Need to manually prepare cache message.
-                // TODO GG-11141.
             }
 
             Iterator<Integer> remainingIter = remainingParts.iterator();
@@ -374,7 +400,8 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                 if (iter.isPartitionDone(p)) {
                     GridDhtLocalPartition loc = top.localPartition(p, d.topologyVersion(), false);
 
-                    assert loc != null;
+                    assert loc != null
+                        : "Supply partition is gone: grp=" + grp.cacheOrGroupName() + ", p=" + p;
 
                     s.last(p, loc.updateCounter());
 
@@ -387,7 +414,8 @@ else if (iter.isPartitionMissing(p)) {
                 }
             }
 
-            assert remainingParts.isEmpty();
+            assert remainingParts.isEmpty()
+                : "Partitions after rebalance should be either done or missing: " + remainingParts;
 
             if (sctx != null)
                 clearContext(sctx, log);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index ddcb81e238965..700f0cf98b7a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -187,7 +187,7 @@ private IgniteCheckedException stopError() {
 
         AffinityAssignment aff = grp.affinity().cachedAffinity(topVer);
 
-        CachePartitionFullCountersMap cntrMap = top.fullUpdateCounters();
+        CachePartitionFullCountersMap countersMap = grp.topology().fullUpdateCounters();
 
         for (int p = 0; p < partCnt; p++) {
             if (ctx.exchange().hasPendingExchange()) {
@@ -251,7 +251,7 @@ private IgniteCheckedException stopError() {
                         );
                     }
 
-                    msg.partitions().addHistorical(p, cntrMap.initialUpdateCounter(p), cntrMap.updateCounter(p), partCnt);
+                    msg.partitions().addHistorical(p, part.initialUpdateCounter(), countersMap.updateCounter(p), partCnt);
                 }
                 else {
                     Collection<ClusterNode> picked = pickOwners(p, topVer);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
index 7066e0d72ac3d..8515004c3fdf4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionsToReloadMap.java
@@ -90,7 +90,7 @@ public synchronized void put(UUID nodeId, int cacheId, int partId) {
     /**
      * @return {@code True} if empty.
      */
-    public boolean isEmpty() {
+    public synchronized boolean isEmpty() {
         return map == null || map.isEmpty();
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 68ec83db1b4b6..5feaa252dd3fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -755,8 +755,8 @@ private Metas getOrAllocateCacheMetas() throws IgniteCheckedException {
     }
 
     /** {@inheritDoc} */
-    @Override @Nullable protected IgniteHistoricalIterator historicalIterator(
-        CachePartitionPartialCountersMap partCntrs) throws IgniteCheckedException {
+    @Override @Nullable protected WALHistoricalIterator historicalIterator(
+        CachePartitionPartialCountersMap partCntrs, Set<Integer> missing) throws IgniteCheckedException {
         if (partCntrs == null || partCntrs.isEmpty())
             return null;
 
@@ -773,13 +773,18 @@ private Metas getOrAllocateCacheMetas() throws IgniteCheckedException {
             if (startPtr == null)
                 throw new IgniteCheckedException("Could not find start pointer for partition [part=" + p + ", partCntrSince=" + initCntr + "]");
 
-            if (minPtr == null || startPtr.compareTo(minPtr) == -1)
+            if (minPtr == null || startPtr.compareTo(minPtr) < 0)
                 minPtr = startPtr;
         }
 
         WALIterator it = grp.shared().wal().replay(minPtr);
 
-        return new WALIteratorAdapter(grp, partCntrs, it);
+        WALHistoricalIterator iterator = new WALHistoricalIterator(grp, partCntrs, it);
+
+        // Add historical partitions which are unabled to reserve to missing set.
+        missing.addAll(iterator.missingParts);
+
+        return iterator;
     }
 
     /**
@@ -807,7 +812,7 @@ long freeSpace() {
     /**
      *
      */
-    private static class WALIteratorAdapter implements IgniteHistoricalIterator {
+    private static class WALHistoricalIterator implements IgniteHistoricalIterator {
         /** */
         private static final long serialVersionUID = 0L;
 
@@ -817,6 +822,9 @@ private static class WALIteratorAdapter implements IgniteHistoricalIterator {
         /** Partition counters map. */
         private final CachePartitionPartialCountersMap partMap;
 
+        /** Partitions marked as missing (unable to reserve or partition is not in OWNING state). */
+        private final Set<Integer> missingParts = new HashSet<>();
+
         /** Partitions marked as done. */
         private final Set<Integer> doneParts = new HashSet<>();
 
@@ -830,19 +838,24 @@ private static class WALIteratorAdapter implements IgniteHistoricalIterator {
         private Iterator<DataEntry> entryIt;
 
         /** */
-        private CacheDataRow next;
+        private DataEntry next;
+
+        /** Flag indicates that partition belongs to current {@link #next} is finished and no longer needs to rebalance. */
+        private boolean reachedPartitionEnd;
 
         /**
          * @param grp Cache context.
          * @param walIt WAL iterator.
          */
-        private WALIteratorAdapter(CacheGroupContext grp, CachePartitionPartialCountersMap partMap, WALIterator walIt) {
+        private WALHistoricalIterator(CacheGroupContext grp, CachePartitionPartialCountersMap partMap, WALIterator walIt) {
             this.grp = grp;
             this.partMap = partMap;
             this.walIt = walIt;
 
             cacheIds = grp.cacheIds();
 
+            reservePartitions();
+
             advance();
         }
 
@@ -859,6 +872,7 @@ private WALIteratorAdapter(CacheGroupContext grp, CachePartitionPartialCountersM
         /** {@inheritDoc} */
         @Override public void close() throws IgniteCheckedException {
             walIt.close();
+            releasePartitions();
         }
 
         /** {@inheritDoc} */
@@ -896,7 +910,13 @@ private WALIteratorAdapter(CacheGroupContext grp, CachePartitionPartialCountersM
             if (next == null)
                 throw new NoSuchElementException();
 
-            CacheDataRow val = next;
+            CacheDataRow val = new DataEntryRow(next);
+
+            if (reachedPartitionEnd) {
+                doneParts.add(next.partitionId());
+
+                reachedPartitionEnd = false;
+            }
 
             advance();
 
@@ -908,6 +928,46 @@ private WALIteratorAdapter(CacheGroupContext grp, CachePartitionPartialCountersM
             throw new UnsupportedOperationException();
         }
 
+        /**
+         * Reserve historical partitions.
+         * If partition is unable to reserve, id of that partition is placed to {@link #missingParts} set.
+         */
+        private void reservePartitions() {
+            for (int i = 0; i < partMap.size(); i++) {
+                int p = partMap.partitionAt(i);
+                GridDhtLocalPartition part = grp.topology().localPartition(p);
+
+                if (part == null || !part.reserve()) {
+                    missingParts.add(p);
+                    continue;
+                }
+
+                if (part.state() != OWNING) {
+                    part.release();
+                    missingParts.add(p);
+                }
+            }
+        }
+
+        /**
+         * Release historical partitions.
+         */
+        private void releasePartitions() {
+            for (int i = 0; i < partMap.size(); i++) {
+                int p = partMap.partitionAt(i);
+
+                if (missingParts.contains(p))
+                    continue;
+
+                GridDhtLocalPartition part = grp.topology().localPartition(p);
+
+                assert part != null && part.state() == OWNING && part.reservations() > 0
+                    : "Partition should in OWNING state and has at least 1 reservation";
+
+                part.release();
+            }
+        }
+
         /**
          *
          */
@@ -922,7 +982,7 @@ private void advance() {
                         if (cacheIds.contains(entry.cacheId())) {
                             int idx = partMap.partitionIndex(entry.partitionId());
 
-                            if (idx < 0)
+                            if (idx < 0 || missingParts.contains(idx))
                                 continue;
 
                             long from = partMap.initialUpdateCounterAt(idx);
@@ -930,9 +990,9 @@ private void advance() {
 
                             if (entry.partitionCounter() >= from && entry.partitionCounter() <= to) {
                                 if (entry.partitionCounter() == to)
-                                    doneParts.add(entry.partitionId());
+                                    reachedPartitionEnd = true;
 
-                                next = new DataEntryRow(entry);
+                                next = entry;
 
                                 return;
                             }
diff --git a/modules/core/src/test/config/log4j-test.xml b/modules/core/src/test/config/log4j-test.xml
index 9138c02e00248..b0b08e7d1a394 100755
--- a/modules/core/src/test/config/log4j-test.xml
+++ b/modules/core/src/test/config/log4j-test.xml
@@ -78,12 +78,6 @@
     <!--
         Uncomment to enable Ignite query execution debugging.
     -->
-    <!--
-    <category name="org.apache.ignite.cache.query">
-        <level value="DEBUG"/>
-    </category>
-    -->
-
     <!--
     <category name="org.apache.ignite.internal.processors.query">
         <level value="DEBUG"/>
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
new file mode 100644
index 0000000000000..a090381ff9863
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteSystemProperties;
+
+/**
+ *
+ */
+public class IgnitePdsAtomicCacheHistoricalRebalancingTest extends IgnitePdsAtomicCacheRebalancingTest {
+    /** {@inheritDoc */
+    @Override protected void beforeTest() throws Exception {
+        // Use rebalance from WAL if possible.
+        System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc */
+    @Override protected void afterTest() throws Exception {
+        System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
+
+        super.afterTest();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
index 10f9b037aaaf7..0dd9c788b7f6c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -78,12 +78,15 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
 
         cfg.setConsistentId(gridName);
 
+        cfg.setRebalanceThreadPoolSize(2);
+
         CacheConfiguration ccfg1 = cacheConfiguration(cacheName)
             .setPartitionLossPolicy(PartitionLossPolicy.READ_WRITE_SAFE)
-            .setBackups(2)
+            .setBackups(1)
             .setRebalanceMode(CacheRebalanceMode.ASYNC)
             .setIndexedTypes(Integer.class, Integer.class)
             .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setRebalanceBatchesPrefetchCount(2)
             .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
 
         CacheConfiguration ccfg2 = cacheConfiguration("indexed");
@@ -172,8 +175,6 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
         stopAllGrids();
 
         cleanPersistenceDir();
-
-        System.clearProperty(IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE);
     }
 
     /**
@@ -184,7 +185,7 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
     public void testRebalancingOnRestart() throws Exception {
         Ignite ignite0 = startGrid(0);
 
-        ignite0.active(true);
+        ignite0.cluster().active(true);
 
         startGrid(1);
 
@@ -233,8 +234,6 @@ public void testRebalancingOnRestart() throws Exception {
      * @throws Exception If fails.
      */
     public void testRebalancingOnRestartAfterCheckpoint() throws Exception {
-        fail("IGNITE-5302");
-
         IgniteEx ignite0 = startGrid(0);
 
         IgniteEx ignite1 = startGrid(1);
@@ -242,7 +241,7 @@ public void testRebalancingOnRestartAfterCheckpoint() throws Exception {
         IgniteEx ignite2 = startGrid(2);
         IgniteEx ignite3 = startGrid(3);
 
-        ignite0.active(true);
+        ignite0.cluster().active(true);
 
         ignite0.cache(cacheName).rebalance().get();
         ignite1.cache(cacheName).rebalance().get();
@@ -264,6 +263,8 @@ public void testRebalancingOnRestartAfterCheckpoint() throws Exception {
         ignite2.close();
         ignite3.close();
 
+        resetBaselineTopology();
+
         ignite0.resetLostPartitions(Collections.singletonList(cache1.getName()));
 
         assert cache1.lostPartitions().isEmpty();
@@ -306,7 +307,7 @@ public void testDataCorrectnessAfterRestart() throws Exception {
         IgniteEx ignite3 = (IgniteEx)G.start(getConfiguration("test3"));
         IgniteEx ignite4 = (IgniteEx)G.start(getConfiguration("test4"));
 
-        ignite1.active(true);
+        ignite1.cluster().active(true);
 
         awaitPartitionMapExchange();
 
@@ -325,7 +326,7 @@ public void testDataCorrectnessAfterRestart() throws Exception {
         ignite3 = (IgniteEx)G.start(getConfiguration("test3"));
         ignite4 = (IgniteEx)G.start(getConfiguration("test4"));
 
-        ignite1.active(true);
+        ignite1.cluster().active(true);
 
         awaitPartitionMapExchange();
 
@@ -348,13 +349,13 @@ public void testDataCorrectnessAfterRestart() throws Exception {
      * @throws Exception If fails.
      */
     public void testPartitionLossAndRecover() throws Exception {
-        fail("IGNITE-5302");
-
         Ignite ignite1 = startGrid(0);
         Ignite ignite2 = startGrid(1);
         Ignite ignite3 = startGrid(2);
         Ignite ignite4 = startGrid(3);
 
+        ignite1.cluster().active(true);
+
         awaitPartitionMapExchange();
 
         IgniteCache<String, String> cache1 = ignite1.cache(cacheName);
@@ -537,7 +538,7 @@ private void testForceRebalance(String cacheName) throws Exception {
 
         final Ignite ig = grid(1);
 
-        ig.active(true);
+        ig.cluster().active(true);
 
         awaitPartitionMapExchange();
 
@@ -574,8 +575,6 @@ private void testForceRebalance(String cacheName) throws Exception {
      * @throws Exception If failed
      */
     public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception {
-        System.setProperty(IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, "1");
-
         final Ignite ig = startGrids(4);
 
         ig.cluster().active(true);
@@ -595,9 +594,8 @@ public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception
                 try {
                     stopGrid(3);
 
-                    // Clear checkpoint history to avoid rebalance from WAL.
-                    forceCheckpoint();
                     forceCheckpoint();
+
                     U.sleep(500); // Wait for data load.
 
                     IgniteEx ig0 = startGrid(3);
@@ -609,8 +607,6 @@ public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception
 
                         awaitPartitionMapExchange();
 
-                        // Clear checkpoint history to avoid rebalance from WAL.
-                        forceCheckpoint();
                         forceCheckpoint();
 
                         startGrid(2);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java
new file mode 100644
index 0000000000000..179c8e0fcf399
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteSystemProperties;
+
+/**
+ *
+ */
+public class IgnitePdsTxCacheHistoricalRebalancingTest extends IgnitePdsTxCacheRebalancingTest {
+    /** {@inheritDoc */
+    @Override protected void beforeTest() throws Exception {
+        // Use rebalance from WAL if possible.
+        System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc */
+    @Override protected void afterTest() throws Exception {
+        System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
+
+        super.afterTest();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
new file mode 100644
index 0000000000000..6387dac1a3ef0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+
+/**
+ * Historic WAL rebalance base test.
+ */
+public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
+    /** Cache name. */
+    private static final String CACHE_NAME = "cache";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0"); //to make all rebalance wal-based
+
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+        ccfg.setRebalanceMode(CacheRebalanceMode.ASYNC);
+
+        ccfg.setCacheMode(CacheMode.REPLICATED);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+        cfg.setCacheConfiguration(ccfg);
+
+        DataStorageConfiguration dbCfg = new DataStorageConfiguration()
+                    .setWalHistorySize(Integer.MAX_VALUE)
+                    .setWalMode(WALMode.LOG_ONLY)
+                    .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true));
+
+        cfg.setDataStorageConfiguration(dbCfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        System.clearProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void test() throws Exception {
+        IgniteEx ig0 = startGrid(0);
+        IgniteEx ig1 = startGrid(1);
+        final int entryCnt = 10_000;
+
+        ig0.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = ig0.cache(CACHE_NAME);
+
+        for (int k = 0; k < entryCnt; k++)
+            cache.put(k, new IndexedObject(k));
+
+        forceCheckpoint();
+
+        stopGrid(1, false);
+
+        for (int k = 0; k < entryCnt; k++)
+            cache.put(k, new IndexedObject(k + 1));
+
+        forceCheckpoint();
+
+        ig1 = startGrid(1);
+
+        IgniteCache<Object, Object> cache1 = ig1.cache(CACHE_NAME);
+
+        cache1.rebalance().get(2, TimeUnit.MINUTES);
+
+        for (int k = 0; k < entryCnt; k++)
+            assertEquals(new IndexedObject(k + 1), cache.get(k));
+    }
+
+    /**
+     *
+     */
+    private static class IndexedObject {
+        /** */
+        @QuerySqlField(index = true)
+        private int iVal;
+
+        /** */
+        private byte[] payload = new byte[1024];
+
+        /**
+         * @param iVal Integer value.
+         */
+        private IndexedObject(int iVal) {
+            this.iVal = iVal;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (!(o instanceof IndexedObject))
+                return false;
+
+            IndexedObject that = (IndexedObject)o;
+
+            return iVal == that.iVal;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return iVal;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(IndexedObject.class, this);
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
index de7ee5f30c38d..996d6a7c583de 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheClientQueryReplicatedNodeRestartSelfTest.java
@@ -209,7 +209,6 @@ private void assertClient(IgniteCache<?,?> c, boolean client) {
      * @throws Exception If failed.
      */
     public void testRestarts() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-7946");
         int duration = 90 * 1000;
         int qryThreadNum = 5;
         int restartThreadsNum = 2; // 2 of 4 data nodes
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index 447b622f451ee..943d43ff7dff3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -17,10 +17,12 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsAtomicCacheHistoricalRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsAtomicCacheRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsBinaryMetadataOnClusterRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsBinarySortObjectFieldsTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsMarshallerMappingRestoreOnNodeStartTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheHistoricalRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreCacheGroupsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest;
@@ -30,6 +32,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsDiskErrorsRecoveringTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsNoActualWalHistoryTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsThreadInterruptionTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRebalanceTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryPPCTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryWithCompactionTest;
@@ -60,6 +63,10 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgnitePdsAtomicCacheRebalancingTest.class);
         suite.addTestSuite(IgnitePdsTxCacheRebalancingTest.class);
 
+        suite.addTestSuite(IgnitePdsAtomicCacheHistoricalRebalancingTest.class);
+        suite.addTestSuite(IgnitePdsTxCacheHistoricalRebalancingTest.class);
+        suite.addTestSuite(IgniteWalRebalanceTest.class);
+
         suite.addTestSuite(IgniteWalRecoveryPPCTest.class);
         suite.addTestSuite(IgnitePdsDiskErrorsRecoveringTest.class);
         suite.addTestSuite(IgnitePdsCacheDestroyDuringCheckpointTest.class);

From 442d87d5799c27ef7258c7ac36e3e0b312676713 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 18 Apr 2018 21:03:32 +0300
Subject: [PATCH 082/543] IGNITE-8243 Fixed possible memory leak. Added latch
 manager to diagnostic messages. - Fixes #3850.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit 6c6f094)
---
 .../GridCachePartitionExchangeManager.java    |  2 ++
 .../preloader/latch/ExchangeLatchManager.java | 27 +++++++++++++------
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 20a3ccbf4b2f3..7ea161acb688f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -1633,6 +1633,8 @@ public void dumpDebugInfo(@Nullable GridDhtPartitionsExchangeFuture exchFut) thr
             }
         }
 
+        U.warn(diagnosticLog, "Latch manager state: " + latchMgr);
+
         dumpPendingObjects(exchTopVer, diagCtx);
 
         for (CacheGroupContext grp : cctx.cache().cacheGroups())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index 404f88f9f0695..b9c7dee7ba6b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -64,21 +64,26 @@ public class ExchangeLatchManager {
     private final GridKernalContext ctx;
 
     /** Discovery manager. */
+    @GridToStringExclude
     private final GridDiscoveryManager discovery;
 
     /** IO manager. */
+    @GridToStringExclude
     private final GridIoManager io;
 
     /** Current coordinator. */
-    private volatile ClusterNode coordinator;
+    @GridToStringExclude
+    private volatile ClusterNode crd;
 
     /** Pending acks collection. */
     private final ConcurrentMap<T2<String, AffinityTopologyVersion>, Set<UUID>> pendingAcks = new ConcurrentHashMap<>();
 
     /** Server latches collection. */
+    @GridToStringInclude
     private final ConcurrentMap<T2<String, AffinityTopologyVersion>, ServerLatch> serverLatches = new ConcurrentHashMap<>();
 
     /** Client latches collection. */
+    @GridToStringInclude
     private final ConcurrentMap<T2<String, AffinityTopologyVersion>, ClientLatch> clientLatches = new ConcurrentHashMap<>();
 
     /** Lock. */
@@ -97,15 +102,14 @@ public ExchangeLatchManager(GridKernalContext ctx) {
 
         if (!ctx.clientNode()) {
             ctx.io().addMessageListener(GridTopic.TOPIC_EXCHANGE, (nodeId, msg, plc) -> {
-                if (msg instanceof LatchAckMessage) {
+                if (msg instanceof LatchAckMessage)
                     processAck(nodeId, (LatchAckMessage) msg);
-                }
             });
 
             // First coordinator initialization.
             ctx.discovery().localJoinFuture().listen(f -> {
                 if (f.error() == null)
-                    this.coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+                    this.crd = getLatchCoordinator(AffinityTopologyVersion.NONE);
             });
 
             ctx.event().addDiscoveryEventListener((e, cache) -> {
@@ -288,6 +292,8 @@ else if (!coordinator.isLocal()) {
                     pendingAcks.computeIfAbsent(latchId, (id) -> new GridConcurrentHashSet<>());
                     pendingAcks.get(latchId).add(from);
                 }
+                else if (coordinator.isLocal())
+                    serverLatches.remove(latchId);
             } else {
                 if (log.isDebugEnabled())
                     log.debug("Process ack [latch=" + latchId + ", from=" + from + "]");
@@ -319,7 +325,7 @@ else if (!coordinator.isLocal()) {
      */
     private void becomeNewCoordinator() {
         if (log.isInfoEnabled())
-            log.info("Become new coordinator " + coordinator.id());
+            log.info("Become new coordinator " + crd.id());
 
         List<T2<String, AffinityTopologyVersion>> latchesToRestore = new ArrayList<>();
         latchesToRestore.addAll(pendingAcks.keySet());
@@ -347,7 +353,7 @@ private void becomeNewCoordinator() {
      * @param left Left node.
      */
     private void processNodeLeft(ClusterNode left) {
-        assert this.coordinator != null : "Coordinator is not initialized";
+        assert this.crd != null : "Coordinator is not initialized";
 
         lock.lock();
 
@@ -402,8 +408,8 @@ private void processNodeLeft(ClusterNode left) {
             }
 
             // Coordinator is changed.
-            if (coordinator.isLocal() && this.coordinator.id() != coordinator.id()) {
-                this.coordinator = coordinator;
+            if (coordinator.isLocal() && this.crd.id() != coordinator.id()) {
+                this.crd = coordinator;
 
                 becomeNewCoordinator();
             }
@@ -693,4 +699,9 @@ String latchId() {
             return S.toString(CompletableLatch.class, this);
         }
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ExchangeLatchManager.class, this);
+    }
 }

From 5bc9fe9fa4e51c9af677f8120528746f345d990f Mon Sep 17 00:00:00 2001
From: dpavlov <dpavlov@apache.org>
Date: Wed, 18 Apr 2018 21:15:17 +0300
Subject: [PATCH 083/543] IGNITE-8302 Reduce test IO consumption in case Direct
 IO is enabled, fixes flakyness of testPageRecoveryAfterFileCorruption - Fixes
 #3865.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit d853ebb)
---
 ...itePdsRecoveryAfterFileCorruptionTest.java | 29 +++++++++++--------
 .../testsuites/IgnitePdsTestSuite2.java       |  7 +++--
 ...eIoPdsRecoveryAfterFileCorruptionTest.java | 12 ++++++++
 .../IgnitePdsNativeIoTestSuite2.java          |  4 +++
 4 files changed, 37 insertions(+), 15 deletions(-)
 create mode 100644 modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
index 5dc0da9abc4ee..e03cf52035768 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
@@ -54,14 +54,14 @@
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
- *
+ * This test generates WAL & Page Store with N pages, then rewrites pages with zeroes and tries to acquire all pages.
  */
 public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstractTest {
     /** Ip finder. */
     private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
 
     /** Total pages. */
-    private static final int totalPages = 1024;
+    private static final int totalPages = 512;
 
     /** Cache name. */
     private final String cacheName = "cache";
@@ -120,7 +120,7 @@ public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstract
     public void testPageRecoveryAfterFileCorruption() throws Exception {
         IgniteEx ig = startGrid(0);
 
-        ig.active(true);
+        ig.cluster().active(true);
 
         IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
 
@@ -144,13 +144,15 @@ public void testPageRecoveryAfterFileCorruption() throws Exception {
 
         int cacheId = sharedCtx.cache().cache(cacheName).context().cacheId();
 
-        FullPageId[] pages = new FullPageId[totalPages];
+        int pagesCnt = getTotalPagesToTest();
+
+        FullPageId[] pages = new FullPageId[pagesCnt];
 
         // Get lock to prevent assertion. A new page should be allocated under checkpoint lock.
         psMgr.checkpointReadLock();
 
         try {
-            for (int i = 0; i < totalPages; i++) {
+            for (int i = 0; i < pagesCnt; i++) {
                 pages[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
 
                 initPage(mem, pageIO, pages[i]);
@@ -174,11 +176,18 @@ public void testPageRecoveryAfterFileCorruption() throws Exception {
 
         ig = startGrid(0);
 
-        ig.active(true);
+        ig.cluster().active(true);
 
         checkRestore(ig, pages);
     }
 
+    /**
+     * @return count of pages to test. Note complexity of test is N^2.
+     */
+    protected int getTotalPagesToTest() {
+        return totalPages;
+    }
+
     /**
      * Initializes page.
      * @param mem page memory implementation.
@@ -287,9 +296,7 @@ private void generateWal(
 
         wal.flush(start, false);
 
-        for (int i = 0; i < totalPages; i++) {
-            FullPageId fullId = pages[i];
-
+        for (FullPageId fullId : pages) {
             long page = mem.acquirePage(fullId.groupId(), fullId.pageId());
 
             try {
@@ -325,9 +332,7 @@ private void generateWal(
 
             long write = 0;
 
-            for (int i = 0; i < totalPages; i++) {
-                FullPageId fullId = pages[i];
-
+            for (FullPageId fullId : pages) {
                 if (pageIds.contains(fullId)) {
                     long cpStart = System.nanoTime();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index ede537ebb6df1..6d953cdd7b2de 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -77,7 +77,7 @@ public static TestSuite suite() {
      *
      * @param suite suite to add tests into.
      */
-    public static void addRealPageStoreTestsLongRunning(TestSuite suite) {
+    private static void addRealPageStoreTestsLongRunning(TestSuite suite) {
         suite.addTestSuite(IgnitePdsTransactionsHangTest.class);
 
         suite.addTestSuite(IgnitePdsPageEvictionDuringPartitionClearTest.class);
@@ -87,6 +87,9 @@ public static void addRealPageStoreTestsLongRunning(TestSuite suite) {
         suite.addTestSuite(IgnitePdsContinuousRestartTest2.class);
 
         suite.addTestSuite(IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes.class);
+
+        // Integrity test.
+        suite.addTestSuite(IgnitePdsRecoveryAfterFileCorruptionTest.class);
     }
 
     /**
@@ -95,8 +98,6 @@ public static void addRealPageStoreTestsLongRunning(TestSuite suite) {
      * @param suite suite to add tests into.
      */
     public static void addRealPageStoreTests(TestSuite suite) {
-        // Integrity test.
-        suite.addTestSuite(IgnitePdsRecoveryAfterFileCorruptionTest.class);
         suite.addTestSuite(IgnitePdsPageSizesTest.class);
 
         // Metrics test.
diff --git a/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
new file mode 100644
index 0000000000000..4d65bf346e5fd
--- /dev/null
+++ b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
@@ -0,0 +1,12 @@
+package org.apache.ignite.internal.processors.cache.persistence;
+
+/**
+ * Native IO tests can't use page store, so this test limit pages count to 128.
+ * 524288 bytes to be read 128 times, 64Mbytes total read load.
+ */
+public class IgniteNativeIoPdsRecoveryAfterFileCorruptionTest extends IgnitePdsRecoveryAfterFileCorruptionTest {
+    /** {@inheritDoc} */
+    @Override protected int getTotalPagesToTest() {
+        return 128;
+    }
+}
diff --git a/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java b/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java
index 54dd7d3689ab7..679e3f7e161d0 100644
--- a/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java
+++ b/modules/direct-io/src/test/java/org/apache/ignite/testsuites/IgnitePdsNativeIoTestSuite2.java
@@ -17,6 +17,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteNativeIoPdsRecoveryAfterFileCorruptionTest;
 
 /**
  * Same as {@link IgnitePdsTestSuite2} but is started with direct-oi jar in classpath.
@@ -31,6 +32,9 @@ public static TestSuite suite() throws Exception {
 
         IgnitePdsTestSuite2.addRealPageStoreTests(suite);
 
+        //Integrity test with reduced count of pages.
+        suite.addTestSuite(IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.class);
+
         return suite;
     }
 }

From 3696c064f02567f471420ced50e2786a4f83d8c9 Mon Sep 17 00:00:00 2001
From: dpavlov <dpavlov@apache.org>
Date: Wed, 18 Apr 2018 21:16:35 +0300
Subject: [PATCH 084/543] IGNITE-8302 Licenses fix for Reduce test IO
 consumption in case Direct IO

(cherry picked from commit e7716f4)
---
 ...tiveIoPdsRecoveryAfterFileCorruptionTest.java | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
index 4d65bf346e5fd..41b6c791cd07a 100644
--- a/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
+++ b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteNativeIoPdsRecoveryAfterFileCorruptionTest.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.ignite.internal.processors.cache.persistence;
 
 /**

From 061a5a82c95f4f926c7789764901b70383152df1 Mon Sep 17 00:00:00 2001
From: shroman <rshtykh@yahoo.com>
Date: Wed, 18 Apr 2018 19:38:52 +0900
Subject: [PATCH 085/543] IGNITE-8143: Modified fetching EXTERNAL_LIBS for
 docker container. - Fixes #3751.

Signed-off-by: shroman <rshtykh@yahoo.com>
---
 modules/docker/run.sh | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/modules/docker/run.sh b/modules/docker/run.sh
index dbf287118aa41..72cfdd807451d 100644
--- a/modules/docker/run.sh
+++ b/modules/docker/run.sh
@@ -29,12 +29,8 @@ if [ ! -z "$EXTERNAL_LIBS" ]; then
   IFS=, LIBS_LIST=("$EXTERNAL_LIBS")
 
   for lib in ${LIBS_LIST[@]}; do
-    echo $lib >> temp
+    wget $lib -P $IGNITE_HOME/libs
   done
-
-  wget -i temp -P $IGNITE_HOME/libs
-
-  rm temp
 fi
 
 QUIET=""

From 83d488b083706e1d04200b4e603db1dc337d8fb4 Mon Sep 17 00:00:00 2001
From: Dmitriy Shabalin <dmitriyff@gmail.com>
Date: Thu, 19 Apr 2018 15:16:18 +0700
Subject: [PATCH 086/543] IGNITE-8298 Web Console: Fixed loader under Safari.

(cherry picked from commit 0897309)
---
 .../web-console/frontend/app/modules/loading/loading.scss   | 2 +-
 .../frontend/app/modules/loading/loading.service.js         | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/web-console/frontend/app/modules/loading/loading.scss b/modules/web-console/frontend/app/modules/loading/loading.scss
index 87bbc6a0360b0..0584c25189fad 100644
--- a/modules/web-console/frontend/app/modules/loading/loading.scss
+++ b/modules/web-console/frontend/app/modules/loading/loading.scss
@@ -29,7 +29,7 @@
     opacity: 0;
     visibility: hidden;
     background-color: white;
-    transition: visibility 0s 0.5s, opacity 0.5s linear;
+    transition: opacity 0.5s linear;
 }
 
 .loading-active {
diff --git a/modules/web-console/frontend/app/modules/loading/loading.service.js b/modules/web-console/frontend/app/modules/loading/loading.service.js
index ec31c1accc7ce..bdc80b8f5360a 100644
--- a/modules/web-console/frontend/app/modules/loading/loading.service.js
+++ b/modules/web-console/frontend/app/modules/loading/loading.service.js
@@ -15,11 +15,11 @@
  * limitations under the License.
  */
 
-export default ['IgniteLoading', ['$timeout', ($timeout) => {
+export default ['IgniteLoading', [() => {
     const _overlays = {};
 
     const start = (key) => {
-        $timeout(() => {
+        setTimeout(() => {
             const loadingOverlay = _overlays[key];
 
             loadingOverlay && loadingOverlay.addClass('loading-active');
@@ -27,7 +27,7 @@ export default ['IgniteLoading', ['$timeout', ($timeout) => {
     };
 
     const finish = (key) => {
-        $timeout(() => {
+        setTimeout(() => {
             const loadingOverlay = _overlays[key];
 
             loadingOverlay && loadingOverlay.removeClass('loading-active');

From f9a0ee18f9d88b8f278d27dddc5f7cd3a52ef5c2 Mon Sep 17 00:00:00 2001
From: Nikita Amelchev <nsamelchev@gmail.com>
Date: Thu, 19 Apr 2018 12:57:06 +0300
Subject: [PATCH 087/543] IGNITE-6666: Support Time data type in
 BinaryObjectImpl.writeFieldByOrder. This closes #2878.

---
 .../internal/binary/BinaryFieldImpl.java      |  9 +++++
 .../internal/binary/BinaryObjectImpl.java     |  1 +
 .../binary/BinaryFieldExtractionSelfTest.java | 35 +++++++++++++++++++
 3 files changed, 45 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java
index 883576c9faad3..de0b2d0d3d3cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldImpl.java
@@ -21,6 +21,7 @@
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.sql.Time;
 import java.sql.Timestamp;
 import java.util.Date;
 import java.util.UUID;
@@ -211,6 +212,14 @@ public int fieldId() {
                     break;
                 }
 
+                case GridBinaryMarshaller.TIME: {
+                    long time = buf.getLong();
+
+                    val = new Time(time);
+
+                    break;
+                }
+
                 case GridBinaryMarshaller.UUID: {
                     long most = buf.getLong();
                     long least = buf.getLong();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
index 4f91e254aa989..7ab8b3563acbc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryObjectImpl.java
@@ -545,6 +545,7 @@ else if (fieldOffsetLen == BinaryUtils.OFFSET_2)
             case GridBinaryMarshaller.LONG:
             case GridBinaryMarshaller.DOUBLE:
             case GridBinaryMarshaller.DATE:
+            case GridBinaryMarshaller.TIME:
                 totalLen = 9;
 
                 break;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldExtractionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldExtractionSelfTest.java
index 9f3974693091e..a050591b2d693 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldExtractionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/BinaryFieldExtractionSelfTest.java
@@ -19,6 +19,7 @@
 
 import java.math.BigDecimal;
 import java.nio.ByteBuffer;
+import java.sql.Time;
 import java.util.concurrent.ThreadLocalRandom;
 import org.apache.ignite.configuration.BinaryConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -97,6 +98,27 @@ public void testPrimitiveMarshalling() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTimeMarshalling() throws Exception {
+        BinaryMarshaller marsh = createMarshaller();
+
+        TimeValue obj = new TimeValue(11111L);
+
+        BinaryObjectImpl binObj = toBinary(obj, marsh);
+
+        BinaryFieldEx field = (BinaryFieldEx)binObj.type().field("time");
+
+        ByteBuffer buf = ByteBuffer.allocate(16);
+
+        field.writeField(binObj, buf);
+
+        buf.flip();
+
+        assertEquals(field.value(binObj), field.<Time>readField(buf));
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -194,6 +216,19 @@ private TestObject(long seed) {
         }
     }
 
+    /** */
+    private static class TimeValue {
+        /** */
+        private Time time;
+
+        /**
+         * @param time Time.
+         */
+        TimeValue(long time) {
+            this.time = new Time(time);
+        }
+    }
+
     /**
      *
      */

From a095b98e1123f9562cbbc85d1c1c4b4221542c71 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 19 Apr 2018 15:25:23 +0300
Subject: [PATCH 088/543] IGNITE-8021 Fixed tests - Fixes #3864.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 ...=> IgnitePdsDestroyCacheAbstractTest.java} | 149 ++----------------
 .../IgnitePdsDestroyCacheTest.java            |  85 ++++++++++
 ...PdsDestroyCacheWithoutCheckpointsTest.java |  86 ++++++++++
 .../ignite/testsuites/IgnitePdsTestSuite.java |   7 +-
 4 files changed, 191 insertions(+), 136 deletions(-)
 rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/{IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java => IgnitePdsDestroyCacheAbstractTest.java} (62%)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheWithoutCheckpointsTest.java

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheAbstractTest.java
similarity index 62%
rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java
rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheAbstractTest.java
index d2767d465506b..434007ece5f8e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheAbstractTest.java
@@ -21,14 +21,12 @@
 import java.util.List;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -37,17 +35,17 @@
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
- * Test correct clean up cache configuration data after destroying cache.
+ * Base class for  {@link IgnitePdsDestroyCacheTest} and {@link IgnitePdsDestroyCacheWithoutCheckpointsTest}
  */
-public class IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest extends GridCommonAbstractTest {
+public abstract class IgnitePdsDestroyCacheAbstractTest extends GridCommonAbstractTest {
     /** */
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
     /** */
-    private static final int CACHES = 3;
+    protected static final int CACHES = 3;
 
     /** */
-    private static final int NODES = 3;
+    protected static final int NODES = 3;
 
     /** */
     private static final int NUM_OF_KEYS = 100;
@@ -80,109 +78,6 @@ public class IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest extends
         super.afterTest();
     }
 
-    /**
-     * {@inheritDoc}
-     * @returns always {@code true} in order to be able to kill nodes when checkpointer thread hangs.
-     */
-    @Override protected boolean isMultiJvm() {
-        return true;
-    }
-
-    /**
-     *  Test destroy non grouped caches.
-     *
-     *  @throws Exception If failed.
-     */
-    public void testDestroyCaches() throws Exception {
-        Ignite ignite = startGrids(NODES);
-
-        ignite.cluster().active(true);
-
-        startCachesDynamically(ignite);
-
-        checkDestroyCaches(ignite);
-    }
-
-    /**
-     *  Test destroy grouped caches.
-     *
-     *  @throws Exception If failed.
-     */
-    public void testDestroyGroupCaches() throws Exception {
-        Ignite ignite = startGrids(NODES);
-
-        ignite.cluster().active(true);
-
-        startGroupCachesDynamically(ignite);
-
-        checkDestroyCaches(ignite);
-    }
-
-    /**
-     * Test destroy caches with disabled checkpoints.
-     *
-     * @throws Exception If failed.
-     */
-    public void testDestroyCachesAbruptlyWithoutCheckpoints() throws Exception {
-        Ignite ignite = startGrids(NODES);
-
-        ignite.cluster().active(true);
-
-        startCachesDynamically(ignite);
-
-        enableCheckpoints(false);
-
-        checkDestroyCachesAbruptly(ignite);
-    }
-
-    /**
-     * Test destroy group caches with disabled checkpoints.
-     *
-     * @throws Exception If failed.
-     */
-    public void testDestroyGroupCachesAbruptlyWithoutCheckpoints() throws Exception {
-        Ignite ignite = startGrids(NODES);
-
-        ignite.cluster().active(true);
-
-        startGroupCachesDynamically(ignite);
-
-        enableCheckpoints(false);
-
-        checkDestroyCachesAbruptly(ignite);
-    }
-
-    /**
-     * Test destroy caches abruptly with checkpoints.
-     *
-     * @throws Exception If failed.
-     */
-    public void testDestroyCachesAbruptly() throws Exception {
-        Ignite ignite = startGrids(NODES);
-
-        ignite.cluster().active(true);
-
-        startCachesDynamically(ignite);
-
-        checkDestroyCachesAbruptly(ignite);
-    }
-
-
-    /**
-     * Test destroy group caches abruptly with checkpoints.
-     *
-     * @throws Exception If failed.
-     */
-    public void testDestroyGroupCachesAbruptly() throws Exception {
-        Ignite ignite = startGrids(NODES);
-
-        ignite.cluster().active(true);
-
-        startGroupCachesDynamically(ignite);
-
-        checkDestroyCachesAbruptly(ignite);
-    }
-
     /**
      * @param ignite Ignite.
      */
@@ -202,7 +97,7 @@ private void loadCaches(Ignite ignite) {
     /**
      * @param ignite Ignite.
      */
-    private void checkDestroyCaches(Ignite ignite) throws Exception {
+    protected void checkDestroyCaches(Ignite ignite) throws Exception {
         loadCaches(ignite);
 
         log.warning("destroying caches....");
@@ -214,7 +109,7 @@ private void checkDestroyCaches(Ignite ignite) throws Exception {
 
         log.warning("Stopping grid");
 
-        stopAllGrids();
+        stopAllGrids(false);
 
         log.warning("Grid stopped");
 
@@ -222,6 +117,8 @@ private void checkDestroyCaches(Ignite ignite) throws Exception {
 
         ignite = startGrids(NODES);
 
+        ignite.cluster().active(true);
+
         log.warning("Grid started");
 
         assertEquals("Check that caches don't survived", CACHES - 2, ignite.cacheNames().size());
@@ -238,7 +135,7 @@ private void checkDestroyCaches(Ignite ignite) throws Exception {
     /**
      * @param ignite Ignite instance.
      */
-    private void checkDestroyCachesAbruptly(Ignite ignite) throws Exception {
+    protected void checkDestroyCachesAbruptly(Ignite ignite) throws Exception {
         loadCaches(ignite);
 
         log.warning("Destroying caches");
@@ -254,7 +151,9 @@ private void checkDestroyCachesAbruptly(Ignite ignite) throws Exception {
 
         log.warning("Starting grid");
 
-        startGrids(NODES);
+        ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
 
         log.warning("Grid started");
 
@@ -271,7 +170,7 @@ private void checkDestroyCachesAbruptly(Ignite ignite) throws Exception {
     /**
      * @param ignite Ignite.
      */
-    private void startCachesDynamically(Ignite ignite) {
+    protected void startCachesDynamically(Ignite ignite) {
         List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
 
         for (int i = 0; i < CACHES; i++)
@@ -285,7 +184,7 @@ private void startCachesDynamically(Ignite ignite) {
     /**
      * @param ignite Ignite instance.
      */
-    private void startGroupCachesDynamically(Ignite ignite) {
+    protected void startGroupCachesDynamically(Ignite ignite) {
         List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
 
         for (int i = 0; i < CACHES; i++)
@@ -297,30 +196,12 @@ private void startGroupCachesDynamically(Ignite ignite) {
         ignite.createCaches(ccfg);
     }
 
-
     /**
      * Generate cache name from idx.
      *
      * @param idx Index.
      */
-    private String cacheName(int idx) {
+    protected String cacheName(int idx) {
         return "cache" + idx;
     }
-
-    /**
-     * Enable/disable checkpoints on multi JVM nodes only.
-     *
-     * @param enabled Enabled flag.
-     * @throws IgniteCheckedException If failed.
-     */
-    private void enableCheckpoints(boolean enabled) throws IgniteCheckedException {
-        for (Ignite ignite : G.allGrids()) {
-            assert !ignite.cluster().localNode().isClient();
-
-            GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
-                    .cache().context().database();
-
-            dbMgr.enableCheckpoints(enabled).get();
-        }
-    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java
new file mode 100644
index 0000000000000..99d6f01c07d5d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.Ignite;
+
+/**
+ * Test correct clean up cache configuration data after destroying cache.
+ */
+public class IgnitePdsDestroyCacheTest extends IgnitePdsDestroyCacheAbstractTest {
+    /**
+     *  Test destroy non grouped caches.
+     *
+     *  @throws Exception If failed.
+     */
+    public void testDestroyCaches() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startCachesDynamically(ignite);
+
+        checkDestroyCaches(ignite);
+    }
+
+    /**
+     *  Test destroy grouped caches.
+     *
+     *  @throws Exception If failed.
+     */
+    public void testDestroyGroupCaches() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startGroupCachesDynamically(ignite);
+
+        checkDestroyCaches(ignite);
+    }
+
+    /**
+     * Test destroy caches abruptly with checkpoints.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDestroyCachesAbruptly() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startCachesDynamically(ignite);
+
+        checkDestroyCachesAbruptly(ignite);
+    }
+    
+    /**
+     * Test destroy group caches abruptly with checkpoints.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDestroyGroupCachesAbruptly() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startGroupCachesDynamically(ignite);
+
+        checkDestroyCachesAbruptly(ignite);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheWithoutCheckpointsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheWithoutCheckpointsTest.java
new file mode 100644
index 0000000000000..1bb6f5d8de8ba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheWithoutCheckpointsTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.G;
+
+/**
+ * Check that cluster survives after destroy caches abruptly with disabled checkpoints.
+ */
+public class IgnitePdsDestroyCacheWithoutCheckpointsTest extends IgnitePdsDestroyCacheAbstractTest {
+    /**
+     * {@inheritDoc}
+     * @returns {@code true} in order to be able to kill nodes when checkpointer thread hangs.
+     */
+    @Override protected boolean isMultiJvm() {
+        return true;
+    }
+
+    /**
+     * Test destroy caches with disabled checkpoints.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDestroyCachesAbruptlyWithoutCheckpoints() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startCachesDynamically(ignite);
+
+        disableCheckpoints();
+
+        checkDestroyCachesAbruptly(ignite);
+    }
+
+    /**
+     * Test destroy group caches with disabled checkpoints.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDestroyGroupCachesAbruptlyWithoutCheckpoints() throws Exception {
+        Ignite ignite = startGrids(NODES);
+
+        ignite.cluster().active(true);
+
+        startGroupCachesDynamically(ignite);
+
+        disableCheckpoints();
+
+        checkDestroyCachesAbruptly(ignite);
+    }
+
+    /**
+     * Disable checkpoints on nodes.
+     *
+     * @throws IgniteCheckedException If failed.
+     */
+    private void disableCheckpoints() throws IgniteCheckedException {
+        for (Ignite ignite : G.allGrids()) {
+            assert !ignite.cluster().localNode().isClient();
+
+            GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+                    .cache().context().database();
+
+            dbMgr.enableCheckpoints(false).get();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index af0b7adc1e68b..d11ceb36a863d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -19,7 +19,8 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheWithoutCheckpointsTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest;
@@ -114,7 +115,9 @@ public static void addRealPageStoreTests(TestSuite suite) {
 
         suite.addTestSuite(IgnitePdsCacheRestoreTest.class);
         suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class);
-        suite.addTestSuite(IgnitePdsDeleteCacheConfigurationDataAfterDestroyCacheTest.class);
+
+        suite.addTestSuite(IgnitePdsDestroyCacheTest.class);
+        suite.addTestSuite(IgnitePdsDestroyCacheWithoutCheckpointsTest.class);
 
         suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
 

From 27385e73183fe4bbc2f366f975facaefbcad2a6e Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Wed, 18 Apr 2018 10:59:46 +0300
Subject: [PATCH 089/543] IGNITE-8201 Fixed NPE

---
 .../rest/JettyRestProcessorAbstractSelfTest.java          | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
index e36447bc3e0c4..0285f3af530ed 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
@@ -985,9 +985,11 @@ private void failIgnite_5874() {
         if (dsCfg.getDefaultDataRegionConfiguration().isPersistenceEnabled())
             fail("IGNITE-5874");
 
-        for (DataRegionConfiguration dataRegCfg : dsCfg.getDataRegionConfigurations()) {
-            if (dataRegCfg.isPersistenceEnabled())
-                fail("IGNITE-5874");
+        if (!F.isEmpty(dsCfg.getDataRegionConfigurations())) {
+            for (DataRegionConfiguration dataRegCfg : dsCfg.getDataRegionConfigurations()) {
+                if (dataRegCfg.isPersistenceEnabled())
+                    fail("IGNITE-5874");
+            }
         }
     }
 

From b2e5e66c21676d833b92cdb0420946b155076077 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 19 Apr 2018 15:44:26 +0300
Subject: [PATCH 090/543] IGNITE-8082 Added ZookeeperDiscoverySpi MBean - Fixes
 #3862.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../spi/discovery/DiscoverySpiMBean.java      | 84 +++++++++++++++++
 .../spi/discovery/tcp/TcpDiscoverySpi.java    | 10 ++
 .../discovery/tcp/TcpDiscoverySpiMBean.java   | 13 +--
 .../discovery/zk/ZookeeperDiscoverySpi.java   | 93 +++++++++++++++++--
 .../zk/ZookeeperDiscoverySpiMBean.java        | 67 +++++++++++++
 .../zk/internal/ZookeeperClient.java          |  7 ++
 .../zk/internal/ZookeeperDiscoveryImpl.java   | 36 ++++++-
 .../ZookeeperDiscoveryStatistics.java         | 48 ++++++++++
 .../internal/ZookeeperDiscoverySpiTest.java   | 40 +++++++-
 9 files changed, 376 insertions(+), 22 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMBean.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMBean.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMBean.java
new file mode 100644
index 0000000000000..d86a49703b5f2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiMBean.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery;
+
+import java.util.UUID;
+import org.apache.ignite.mxbean.MXBeanDescription;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Generic MBean interface to monitor DiscoverySpi subsystem.
+ */
+public interface DiscoverySpiMBean {
+    /**
+     * Gets current SPI state.
+     *
+     * @return Current SPI state.
+     */
+    @MXBeanDescription("SPI state.")
+    public String getSpiState();
+
+    /**
+     * Gets joined nodes count.
+     *
+     * @return Nodes joined count.
+     */
+    @MXBeanDescription("Nodes joined count.")
+    public long getNodesJoined();
+
+    /**
+     * Gets failed nodes count.
+     *
+     * @return Failed nodes count.
+     */
+    @MXBeanDescription("Nodes failed count.")
+    public long getNodesFailed();
+
+    /**
+     * Gets left nodes count.
+     *
+     * @return Left nodes count.
+     */
+    @MXBeanDescription("Nodes left count.")
+    public long getNodesLeft();
+
+    /**
+     * Gets current coordinator.
+     *
+     * @return Gets current coordinator.
+     * @deprecated Use {@link #getCoordinatorNodeFormatted()} instead.
+     */
+    @Deprecated
+    @MXBeanDescription("Coordinator node ID.")
+    @Nullable public UUID getCoordinator();
+
+    /**
+     * Gets current coordinator node formatted as a string.
+     *
+     * @return Current coordinator string representation.
+     */
+    @MXBeanDescription("Coordinator node formatted as a string.")
+    @Nullable public String getCoordinatorNodeFormatted();
+
+    /**
+     * Gets local node formatted as a string.
+     *
+     * @return Local node string representation.
+     */
+    @MXBeanDescription("Local node formatted as a string.")
+    public String getLocalNodeFormatted();
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index f0a5186f0ae08..7c73b2879428d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -2349,6 +2349,16 @@ private class TcpDiscoverySpiMBeanImpl extends IgniteSpiMBeanAdapter implements
             return impl.getCoordinator();
         }
 
+        /** {@inheritDoc} */
+        @Nullable @Override public String getCoordinatorNodeFormatted() {
+            return String.valueOf(impl.getNode(impl.getCoordinator()));
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getLocalNodeFormatted() {
+            return String.valueOf(getLocalNode());
+        }
+
         /** {@inheritDoc} */
         @Override public void dumpDebugInfo() {
             impl.dumpDebugInfo(log);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
index 8d51d773cd45a..cb0fd362c4a54 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
@@ -23,20 +23,21 @@
 import org.apache.ignite.mxbean.MXBeanParametersDescriptions;
 import org.apache.ignite.mxbean.MXBeanParametersNames;
 import org.apache.ignite.spi.IgniteSpiManagementMBean;
+import org.apache.ignite.spi.discovery.DiscoverySpiMBean;
 import org.jetbrains.annotations.Nullable;
 
 /**
  * Management bean for {@link TcpDiscoverySpi}.
  */
 @MXBeanDescription("MBean provide access to TCP-based discovery SPI.")
-public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
+public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean, DiscoverySpiMBean {
     /**
      * Gets current SPI state.
      *
      * @return Current SPI state.
      */
     @MXBeanDescription("SPI state.")
-    public String getSpiState();
+    @Override public String getSpiState();
 
     /**
      * Gets {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} (string representation).
@@ -116,7 +117,7 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
      * @return Nodes joined count.
      */
     @MXBeanDescription("Nodes joined count.")
-    public long getNodesJoined();
+    @Override public long getNodesJoined();
 
     /**
      * Gets left nodes count.
@@ -124,7 +125,7 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
      * @return Left nodes count.
      */
     @MXBeanDescription("Nodes left count.")
-    public long getNodesLeft();
+    @Override public long getNodesLeft();
 
     /**
      * Gets failed nodes count.
@@ -132,7 +133,7 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
      * @return Failed nodes count.
      */
     @MXBeanDescription("Nodes failed count.")
-    public long getNodesFailed();
+    @Override public long getNodesFailed();
 
     /**
      * Gets pending messages registered count.
@@ -212,7 +213,7 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean {
      * @return Gets current coordinator.
      */
     @MXBeanDescription("Coordinator node ID.")
-    @Nullable public UUID getCoordinator();
+    @Override @Nullable public UUID getCoordinator();
 
     /**
      * Gets message acknowledgement timeout.
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
index 860c71c16e972..52f77fcc8a871 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi.java
@@ -41,13 +41,12 @@
 import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.spi.IgniteSpiAdapter;
 import org.apache.ignite.spi.IgniteSpiConfiguration;
-import org.apache.ignite.spi.IgniteSpiContext;
 import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.IgniteSpiMBeanAdapter;
 import org.apache.ignite.spi.IgniteSpiMultipleInstancesSupport;
 import org.apache.ignite.spi.communication.CommunicationSpi;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
-import org.apache.ignite.spi.discovery.DiscoverySpi;
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
 import org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport;
@@ -57,6 +56,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
 import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClusterNode;
 import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryStatistics;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CONSISTENT_ID_BY_HOST_WITHOUT_PORT;
@@ -69,7 +69,7 @@
 @DiscoverySpiOrderSupport(true)
 @DiscoverySpiHistorySupport(true)
 @DiscoverySpiMutableCustomMessageSupport(false)
-public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements DiscoverySpi, IgniteDiscoverySpi {
+public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements IgniteDiscoverySpi {
     /** */
     public static final String DFLT_ROOT_PATH = "/apacheIgnite";
 
@@ -137,6 +137,9 @@ public class ZookeeperDiscoverySpi extends IgniteSpiAdapter implements Discovery
     /** */
     private IgniteDiscoverySpiInternalListener internalLsnr;
 
+    /** */
+    private final ZookeeperDiscoveryStatistics stats = new ZookeeperDiscoveryStatistics();
+
     /**
      * @return Base path in ZK for znodes created by SPI.
      */
@@ -384,7 +387,7 @@ private void initAddresses() {
 
     /** {@inheritDoc} */
     @Override public void setAuthenticator(DiscoverySpiNodeAuthenticator auth) {
-        this.nodeAuth = auth;
+        nodeAuth = auth;
     }
 
     /**
@@ -421,13 +424,10 @@ public DiscoverySpiNodeAuthenticator getAuthenticator() {
         return impl.localNode().isClient();
     }
 
-    /** {@inheritDoc} */
-    @Override protected void onContextInitialized0(IgniteSpiContext spiCtx) throws IgniteSpiException {
-        super.onContextInitialized0(spiCtx);
-    }
-
     /** {@inheritDoc} */
     @Override public void spiStart(@Nullable String igniteInstanceName) throws IgniteSpiException {
+        startStopwatch();
+
         if (sesTimeout == 0)
             sesTimeout = ignite.configuration().getFailureDetectionTimeout().intValue();
 
@@ -465,7 +465,10 @@ public DiscoverySpiNodeAuthenticator getAuthenticator() {
             locNode,
             lsnr,
             exchange,
-            internalLsnr);
+            internalLsnr,
+            stats);
+
+        registerMBean(igniteInstanceName, new ZookeeperDiscoverySpiMBeanImpl(this), ZookeeperDiscoverySpiMBean.class);
 
         try {
             impl.startJoinAndWait();
@@ -492,6 +495,8 @@ public DiscoverySpiNodeAuthenticator getAuthenticator() {
 
     /** {@inheritDoc} */
     @Override public void spiStop() throws IgniteSpiException {
+        unregisterMBean();
+
         if (impl != null)
             impl.stop();
     }
@@ -554,4 +559,72 @@ private ZookeeperDiscoverySpi cloneSpiConfiguration() {
     @Override public String toString() {
         return S.toString(ZookeeperDiscoverySpi.class, this);
     }
+
+    /** */
+    private class ZookeeperDiscoverySpiMBeanImpl extends IgniteSpiMBeanAdapter implements ZookeeperDiscoverySpiMBean {
+        /** {@inheritDoc} */
+        public ZookeeperDiscoverySpiMBeanImpl(IgniteSpiAdapter spiAdapter) {
+            super(spiAdapter);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getSpiState() {
+            return impl.getSpiState();
+        }
+
+        /** {@inheritDoc} */
+        @Override public long getNodesJoined() {
+            return stats.joinedNodesCnt();
+        }
+
+        /** {@inheritDoc} */
+        @Override public long getNodesLeft() {
+            return 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long getNodesFailed() {
+            return stats.failedNodesCnt();
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public UUID getCoordinator() {
+            return impl.getCoordinator();
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public String getCoordinatorNodeFormatted() {
+            return String.valueOf(impl.node(impl.getCoordinator()));
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getLocalNodeFormatted() {
+            return String.valueOf(getLocalNode());
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getZkConnectionString() {
+            return zkConnectionString;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long getZkSessionTimeout() {
+            return sesTimeout;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getZkSessionId() {
+            return impl.getZkSessionId();
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getZkRootPath() {
+            return zkRootPath;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long getNodeOrder() {
+            return getLocalNode().order();
+        }
+    }
 }
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java
new file mode 100644
index 0000000000000..1eed0b41d098d
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiMBean.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk;
+
+import org.apache.ignite.mxbean.MXBeanDescription;
+import org.apache.ignite.spi.IgniteSpiManagementMBean;
+import org.apache.ignite.spi.discovery.DiscoverySpiMBean;
+
+/**
+ * Management bean for {@link ZookeeperDiscoverySpi}.
+ */
+@MXBeanDescription("MBean provide access to Zookeeper-based discovery SPI.")
+public interface ZookeeperDiscoverySpiMBean extends IgniteSpiManagementMBean, DiscoverySpiMBean {
+    /**
+     * Gets node join order.
+     *
+     * @return Node Join Order.
+     */
+    @MXBeanDescription("Node Join Order.")
+    public long getNodeOrder();
+
+    /**
+     * Gets connection string used to connect to ZooKeeper cluster.
+     *
+     * @return Zk Connection String.
+     */
+    @MXBeanDescription("Zk Connection String.")
+    public String getZkConnectionString();
+
+    /**
+     * Gets session timeout used by Zk client of local Ignite node.
+     *
+     * @return Zk Session Timeout.
+     */
+    @MXBeanDescription("Zk Session Timeout (milliseconds).")
+    public long getZkSessionTimeout();
+
+    /**
+     * Gets session id of Zk client established with ZooKeeper cluster.
+     *
+     * @return Zk Session Id.
+     */
+    @MXBeanDescription("Zk Session Id.")
+    public String getZkSessionId();
+
+    /**
+     * Gets root path in ZooKeeper cluster Zk client uses to put data to.
+     *
+     * @return Zk Root Path.
+     */
+    @MXBeanDescription("Zk Root Path.")
+    public String getZkRootPath();
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
index 21703c66a5518..cc525d31eb7ee 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -162,6 +162,13 @@ boolean connected() {
         }
     }
 
+    /** */
+    String state() {
+        synchronized (stateMux) {
+            return state.toString();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void process(WatchedEvent evt) {
         if (closing)
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 7708358f687dc..3010d1652f331 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -59,8 +59,6 @@
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.security.SecurityContext;
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.GridSpinBusyLock;
@@ -71,7 +69,6 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.MarshallerUtils;
@@ -191,6 +188,9 @@ public class ZookeeperDiscoveryImpl {
     /** */
     private long prevSavedEvtsTopVer;
 
+    /** */
+    private final ZookeeperDiscoveryStatistics stats;
+
     /**
      * @param spi Discovery SPI.
      * @param igniteInstanceName Instance name.
@@ -200,6 +200,7 @@ public class ZookeeperDiscoveryImpl {
      * @param lsnr Discovery events listener.
      * @param exchange Discovery data exchange.
      * @param internalLsnr Internal listener (used for testing only).
+     * @param stats Zookeeper DiscoverySpi statistics collector.
      */
     public ZookeeperDiscoveryImpl(
         ZookeeperDiscoverySpi spi,
@@ -209,7 +210,8 @@ public ZookeeperDiscoveryImpl(
         ZookeeperClusterNode locNode,
         DiscoverySpiListener lsnr,
         DiscoverySpiDataExchange exchange,
-        IgniteDiscoverySpiInternalListener internalLsnr) {
+        IgniteDiscoverySpiInternalListener internalLsnr,
+        ZookeeperDiscoveryStatistics stats) {
         assert locNode.id() != null && locNode.isLocal() : locNode;
 
         MarshallerUtils.setNodeName(marsh, igniteInstanceName);
@@ -235,6 +237,8 @@ public ZookeeperDiscoveryImpl(
 
         if (internalLsnr != null)
             this.internalLsnr = internalLsnr;
+
+        this.stats = stats;
     }
 
     /**
@@ -2184,6 +2188,8 @@ private void addJoinedNode(
         joinCtx.addJoinedNode(nodeEvtData, commonData);
 
         rtState.evtsData.onNodeJoin(joinedNode);
+
+        stats.onNodeJoined();
     }
 
     /**
@@ -3485,6 +3491,8 @@ private void notifyNodeFail(long nodeInternalOrder, long topVer) {
             topSnapshot,
             Collections.<Long, Collection<ClusterNode>>emptyMap(),
             null);
+
+        stats.onNodeFailed();
     }
 
     /**
@@ -4461,4 +4469,24 @@ enum ConnectionState {
         /** */
         STOPPED
     }
+
+    /** */
+    public UUID getCoordinator() {
+        Map.Entry<Long, ZookeeperClusterNode> e = rtState.top.nodesByOrder.firstEntry();
+
+        return e != null ? e.getValue().id() : null;
+    }
+
+    /** */
+    public String getSpiState() {
+        return rtState.zkClient.state();
+    }
+
+    /** */
+    public String getZkSessionId() {
+        if (rtState.zkClient != null && rtState.zkClient.zk() != null)
+            return Long.toHexString(rtState.zkClient.zk().getSessionId());
+        else
+            return null;
+    }
 }
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java
new file mode 100644
index 0000000000000..ef4c37e0766b3
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spi.discovery.zk.internal;
+
+/**
+ *
+ */
+public class ZookeeperDiscoveryStatistics {
+    /** */
+    private int joinedNodesCnt;
+
+    /** */
+    private int failedNodesCnt;
+
+    /** */
+    public int joinedNodesCnt() {
+        return joinedNodesCnt;
+    }
+
+    /** */
+    public int failedNodesCnt() {
+        return failedNodesCnt;
+    }
+
+    /** */
+    public void onNodeJoined() {
+        joinedNodesCnt++;
+    }
+
+    /** */
+    public void onNodeFailed() {
+        failedNodesCnt++;
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index fb12c3a5e9801..408af30c87e05 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.spi.discovery.zk.internal;
 
 import java.io.Serializable;
+import java.lang.management.ManagementFactory;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
@@ -40,6 +41,9 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import javax.management.JMX;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
 import org.apache.curator.test.TestingCluster;
 import org.apache.curator.test.TestingZooKeeperServer;
 import org.apache.ignite.Ignite;
@@ -60,7 +64,7 @@
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.DiscoverySpiTestListener;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgnitionEx;
@@ -71,7 +75,6 @@
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.managers.discovery.DiscoveryLocalJoinData;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
-import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpiInternalListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
@@ -107,6 +110,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiMBean;
 import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiTestSuite2;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -559,6 +563,38 @@ public void testDefaultConsistentId() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception if failed.
+     */
+    public void testMbean() throws Exception {
+        startGrids(3);
+
+        MBeanServer srv = ManagementFactory.getPlatformMBeanServer();
+
+        UUID crdNodeId = grid(0).localNode().id();
+
+        try {
+            for (int i = 0; i < 3; i++) {
+                IgniteEx grid = grid(i);
+
+                ObjectName spiName = U.makeMBeanName(grid.context().igniteInstanceName(), "SPIs",
+                    ZookeeperDiscoverySpi.class.getSimpleName());
+
+                ZookeeperDiscoverySpiMBean bean = JMX.newMBeanProxy(srv, spiName, ZookeeperDiscoverySpiMBean.class);
+
+                assertNotNull(bean);
+
+                assertEquals(String.valueOf(grid.cluster().node(crdNodeId)), bean.getCoordinatorNodeFormatted());
+                assertEquals(String.valueOf(grid.cluster().localNode()), bean.getLocalNodeFormatted());
+                assertEquals(zkCluster.getConnectString(), bean.getZkConnectionString());
+                assertEquals((long)grid.configuration().getFailureDetectionTimeout(), bean.getZkSessionTimeout());
+            }
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From 7da1cb14ef4c06a6c4d29f04194b1f841bd3754d Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Thu, 19 Apr 2018 17:53:26 +0300
Subject: [PATCH 091/543] IGNITE-8327 Removed delaying communication SPI from
 tests

---
 .../CacheBaselineTopologyTest.java            | 35 +------------------
 1 file changed, 1 insertion(+), 34 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index c3d404b50fd7c..22cf205d7a1c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -46,17 +46,12 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.TestDelayingCommunicationSpi;
-import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -79,9 +74,6 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
     /** */
     private static final int NODE_COUNT = 4;
 
-    /** */
-    private boolean delayRebalance;
-
     /** */
     private boolean disableAutoActivation;
 
@@ -151,16 +143,13 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
         if (client)
             cfg.setClientMode(true);
 
-        if (delayRebalance)
-            cfg.setCommunicationSpi(new DelayRebalanceCommunicationSpi());
-
         return cfg;
     }
 
     /**
      * Verifies that rebalance on cache with Node Filter happens when BaselineTopology changes.
      *
-     * @throws Exception
+     * @throws Exception If failed.
      */
     public void testRebalanceForCacheWithNodeFilter() throws Exception {
         try {
@@ -846,8 +835,6 @@ public void testNonPersistentCachesIgnoreBaselineTopology() throws Exception {
      * @throws Exception if failed.
      */
     public void testAffinityAssignmentChangedAfterRestart() throws Exception {
-        delayRebalance = false;
-
         int parts = 32;
 
         final List<Integer> partMapping = new ArrayList<>();
@@ -890,8 +877,6 @@ public void testAffinityAssignmentChangedAfterRestart() throws Exception {
 
         Collections.shuffle(TestAffinityFunction.partsAffMapping, new Random(1));
 
-        delayRebalance = true;
-
         /* There is a problem with handling simultaneous auto activation after restart and manual activation.
            To properly catch the moment when cluster activation has finished we temporary disable auto activation. */
         disableAutoActivation = true;
@@ -1035,22 +1020,4 @@ public TestAffinityFunction(AffinityFunction delegate) {
             delegate.removeNode(nodeId);
         }
     }
-
-    /**
-     *
-     */
-    private static class DelayRebalanceCommunicationSpi extends TestDelayingCommunicationSpi {
-        /** {@inheritDoc} */
-        @Override protected boolean delayMessage(Message msg, GridIoMessage ioMsg) {
-            if (msg != null && (msg instanceof GridDhtPartitionDemandMessage || msg instanceof GridDhtPartitionSupplyMessage))
-                return true;
-
-            return false;
-        }
-
-        /** {@inheritDoc} */
-        @Override protected int delayMillis() {
-            return 1_000_000;
-        }
-    }
 }

From bf3e0b5ddec5e3b1fa145e9a5eb72b1fb6569b54 Mon Sep 17 00:00:00 2001
From: Ivan Rakov <ivan.glukos@gmail.com>
Date: Thu, 19 Apr 2018 19:04:08 +0300
Subject: [PATCH 092/543] IGNITE-8325 Compressor thread may miss notification
 on stop - Fixes #3877.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../cache/persistence/wal/FileWriteAheadLogManager.java        | 3 +++
 .../persistence/wal/FsyncModeFileWriteAheadLogManager.java     | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index a40811bfb6de4..9917e85f46fe6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1832,6 +1832,9 @@ private long tryReserveNextSegmentOrWait() throws InterruptedException, IgniteCh
             long segmentToCompress = lastCompressedIdx + 1;
 
             synchronized (this) {
+                if (stopped)
+                    return -1;
+
                 while (segmentToCompress > Math.min(lastAllowedToCompressIdx, archivedMonitor.lastArchivedAbsoluteIndex())) {
                     wait();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index c7d2c115ef7e4..2305389509f5d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -1666,6 +1666,9 @@ private long tryReserveNextSegmentOrWait() throws InterruptedException, IgniteCh
             long segmentToCompress = lastCompressedIdx + 1;
 
             synchronized (this) {
+                if (stopped)
+                    return -1;
+
                 while (segmentToCompress > Math.min(lastAllowedToCompressIdx, archiver.lastArchivedAbsoluteIndex())) {
                     wait();
 

From 7a4e804dbd9691f71b15bee49fe5c1858eae7b0b Mon Sep 17 00:00:00 2001
From: dpavlov <dpavlov@apache.org>
Date: Thu, 19 Apr 2018 15:17:58 +0300
Subject: [PATCH 093/543] IGNITE-8017 Javadoc fix for Disable WAL during
 initial rebalancing

(cherry picked from commit 02a0865)
---
 .../ignite/internal/processors/cache/CacheGroupContext.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 849ecc8196eeb..5f750d5996416 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -1036,7 +1036,7 @@ public boolean localWalEnabled() {
     }
 
     /**
-     * @Global WAL enabled flag.
+     * @return Global WAL enabled flag.
      */
     public boolean globalWalEnabled() {
         return globalWalEnabled;

From 26d78660809a84ebf959dd18ef4f8aef7e6cacbf Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Fri, 20 Apr 2018 15:28:31 +0300
Subject: [PATCH 094/543] IGNITE-8258 Race in page memory in
 copyPageForCheckpoint, Ignite PDS 1 suite, test failed suite
 IgnitePdsPageReplacementTest.testPageReplacement. - Fixes #3821.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit 8caeb3d)
---
 .../cache/persistence/pagemem/PageMemoryImpl.java    | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 4463224b64686..c684ce155a2a6 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1144,7 +1144,7 @@ private boolean copyPageForCheckpoint(
 
             assert success : "Page was pin when we resolve abs pointer, it can not be evicted";
 
-            if (tmpRelPtr != INVALID_REL_PTR){
+            if (tmpRelPtr != INVALID_REL_PTR) {
                 PageHeader.tempBufferPointer(absPtr, INVALID_REL_PTR);
 
                 long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr);
@@ -1158,9 +1158,6 @@ private boolean copyPageForCheckpoint(
 
                 checkpointPool.releaseFreePage(tmpRelPtr);
 
-                // We pinned the page when allocated the temp buffer, release it now.
-                PageHeader.releasePage(absPtr);
-
                 // Need release again because we pin page when resolve abs pointer,
                 // and page did not have tmp buffer page.
                 if (!pageSingleAcquire)
@@ -1171,9 +1168,6 @@ private boolean copyPageForCheckpoint(
                 copyInBuffer(absPtr, outBuf);
 
                 PageHeader.dirty(absPtr, false);
-
-                // We pinned the page when resolve abs pointer.
-                PageHeader.releasePage(absPtr);
             }
 
             assert PageIO.getType(outBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
@@ -1183,6 +1177,10 @@ private boolean copyPageForCheckpoint(
         }
         finally {
             rwLock.writeUnlock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
+
+            // We pinned the page either when allocated the temp buffer, or when resolved abs pointer.
+            // Must release the page only after write unlock.
+            PageHeader.releasePage(absPtr);
         }
     }
 

From e67ac4c170eaf74c4544a4408a58967fe066f371 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Fri, 20 Apr 2018 16:00:00 +0300
Subject: [PATCH 095/543] IGNITE-8078 Introduced additional data storage
 metrics - Fixes #3845.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../org/apache/ignite/DataRegionMetrics.java  |  35 ++++
 .../org/apache/ignite/DataStorageMetrics.java |  70 +++++++
 .../apache/ignite/internal/IgniteKernal.java  |   1 +
 .../cache/CacheGroupMetricsMXBeanImpl.java    |  86 +++++++-
 .../persistence/DataRegionMetricsImpl.java    |  98 ++++++++-
 .../DataRegionMetricsMXBeanImpl.java          |  25 +++
 .../DataRegionMetricsSnapshot.java            |  45 ++++
 .../persistence/DataStorageMetricsImpl.java   | 195 +++++++++++++++++-
 .../DataStorageMetricsSnapshot.java           |  90 ++++++++
 .../GridCacheDatabaseSharedManager.java       |   7 +-
 .../IgniteCacheDatabaseSharedManager.java     |  37 +++-
 .../file/FilePageStoreManager.java            |  17 +-
 .../persistence/pagemem/PageMemoryImpl.java   |   6 +
 .../wal/FileWriteAheadLogManager.java         |  24 ++-
 .../FsyncModeFileWriteAheadLogManager.java    |  22 +-
 .../mxbean/CacheGroupMetricsMXBean.java       |  24 +++
 .../mxbean/DataRegionMetricsMXBean.java       |  20 ++
 .../mxbean/DataStorageMetricsMXBean.java      |  40 ++++
 .../SwapPathConstructionSelfTest.java         |   4 +-
 .../ApiParity/DataRegionMetricsParityTest.cs  |   8 +-
 .../ApiParity/DataStorageMetricsParityTest.cs |  14 +-
 21 files changed, 849 insertions(+), 19 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java b/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
index dc48f115c4ec4..ca2fc66818ee1 100644
--- a/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
@@ -165,4 +165,39 @@ public interface DataRegionMetrics {
      * @return Page size in bytes.
      */
     public int getPageSize();
+
+    /**
+     * The number of read pages from last restart.
+     *
+     * @return The number of read pages from last restart.
+     */
+    public long getPagesRead();
+
+    /**
+     * The number of written pages from last restart.
+     *
+     * @return The number of written pages from last restart.
+     */
+    public long getPagesWritten();
+
+    /**
+     * The number of replaced pages from last restart .
+     *
+     * @return The number of replaced pages from last restart .
+     */
+    public long getPagesReplaced();
+
+    /**
+     * Total offheap size in bytes.
+     *
+     * @return Total offheap size in bytes.
+     */
+    public long getOffHeapSize();
+
+    /**
+     * Total used offheap size in bytes.
+     *
+     * @return Total used offheap size in bytes.
+     */
+    public long getOffheapUsedSize();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java b/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java
index 680caba62b8f9..5fb2b1e088700 100644
--- a/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java
@@ -66,6 +66,27 @@ public interface DataStorageMetrics {
      */
     public long getWalBuffPollSpinsRate();
 
+    /**
+     * Total size in bytes for storage wal files.
+     *
+     * @return Total size in bytes for storage wal files.
+     */
+    public long getWalTotalSize();
+
+    /**
+     * Time of the last WAL segment rollover.
+     *
+     * @return  Time of the last WAL segment rollover.
+     */
+    public long getWalLastRollOverTime();
+
+    /**
+     * Total checkpoint time from last restart.
+     *
+     * @return  Total checkpoint time from last restart.
+     */
+    public long getCheckpointTotalTime();
+
     /**
      * Gets the duration of the last checkpoint in milliseconds.
      *
@@ -121,4 +142,53 @@ public interface DataStorageMetrics {
      * @return Total number of pages copied to a temporary checkpoint buffer during the last checkpoint.
      */
     public long getLastCheckpointCopiedOnWritePagesNumber();
+
+    /**
+     * Total dirty pages for the next checkpoint.
+     *
+     * @return  Total dirty pages for the next checkpoint.
+     */
+    public long getDirtyPages();
+
+    /**
+     * The number of read pages from last restart.
+     *
+     * @return The number of read pages from last restart.
+     */
+    public long getPagesRead();
+
+    /**
+     * The number of written pages from last restart.
+     *
+     * @return The number of written pages from last restart.
+     */
+    public long getPagesWritten();
+
+    /**
+     * The number of replaced pages from last restart.
+     *
+     * @return The number of replaced pages from last restart.
+     */
+    public long getPagesReplaced();
+
+    /**
+     * Total offheap size in bytes.
+     *
+     * @return Total offheap size in bytes.
+     */
+    public long getOffHeapSize();
+
+    /**
+     * Total used offheap size in bytes.
+     *
+     * @return Total used offheap size in bytes.
+     */
+    public long getOffheapUsedSize();
+
+    /**
+     * Total size of memory allocated in bytes.
+     *
+     * @return Total size of memory allocated in bytes.
+     */
+    public long getTotalAllocatedSize();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 1cb07b91e82b3..2dc3df90c5a76 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -104,6 +104,7 @@
 import org.apache.ignite.internal.binary.BinaryUtils;
 import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
 import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.managers.GridManager;
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
index 0788ee9126a6a..e9a2736f34433 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
@@ -25,12 +25,18 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.atomic.LongAdder;
+import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
+import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
 import org.apache.ignite.mxbean.CacheGroupMetricsMXBean;
 
 /**
@@ -40,6 +46,9 @@ public class CacheGroupMetricsMXBeanImpl implements CacheGroupMetricsMXBean {
     /** Cache group context. */
     private final CacheGroupContext ctx;
 
+    /** */
+    private final GroupAllocationTrucker groupPageAllocationTracker;
+
     /** Interface describing a predicate of two integers. */
     private interface IntBiPredicate {
         /**
@@ -52,12 +61,58 @@ private interface IntBiPredicate {
     }
 
     /**
-     * Creates MBean;
+     *
+     */
+    public static class GroupAllocationTrucker implements AllocatedPageTracker {
+        /** */
+        private final LongAdder totalAllocatedPages = new LongAdder();
+
+        /** */
+        private final AllocatedPageTracker delegate;
+
+        /**
+         * @param delegate Delegate allocation trucker.
+         */
+        public GroupAllocationTrucker(AllocatedPageTracker delegate) {
+            this.delegate = delegate;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void updateTotalAllocatedPages(long delta) {
+            totalAllocatedPages.add(delta);
+
+            delegate.updateTotalAllocatedPages(delta);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class NoopAllocationTrucker implements AllocatedPageTracker{
+        /** {@inheritDoc} */
+        @Override public void updateTotalAllocatedPages(long delta) {
+            // No-op.
+        }
+    }
+
+    /**
+     * Creates Group metrics MBean.
      *
      * @param ctx Cache group context.
      */
     public CacheGroupMetricsMXBeanImpl(CacheGroupContext ctx) {
         this.ctx = ctx;
+
+        DataRegion region = ctx.dataRegion();
+
+        // On client node, region is null.
+        if (region != null) {
+            DataRegionMetricsImpl dataRegionMetrics = ctx.dataRegion().memoryMetrics();
+
+            this.groupPageAllocationTracker = dataRegionMetrics.getOrAllocateGroupPageAllocationTracker(ctx.groupId());
+        }
+        else
+            this.groupPageAllocationTracker = new GroupAllocationTrucker(new NoopAllocationTrucker());
     }
 
     /** {@inheritDoc} */
@@ -252,4 +307,33 @@ private Map<Integer, Set<String>> clusterPartitionsMapByState(GridDhtPartitionSt
 
         return assignmentMap;
     }
+
+    /** {@inheritDoc} */
+    @Override public String getType() {
+        CacheMode type = ctx.config().getCacheMode();
+
+        return String.valueOf(type);
+    }
+
+    /** {@inheritDoc} */
+    @Override public List<Integer> getPartitionIds() {
+        List<GridDhtLocalPartition> parts = ctx.topology().localPartitions();
+
+        List<Integer> partsRes = new ArrayList<>(parts.size());
+
+        for (GridDhtLocalPartition part : parts)
+            partsRes.add(part.id());
+
+        return partsRes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTotalAllocatedPages() {
+        return groupPageAllocationTracker.totalAllocatedPages.longValue();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTotalAllocatedSize() {
+        return getTotalAllocatedPages() * ctx.dataRegion().pageMemory().pageSize();
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
index 7d6b4785ed484..cb13747b775a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
@@ -16,10 +16,13 @@
  */
 package org.apache.ignite.internal.processors.cache.persistence;
 
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.LongAdder;
 import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMXBeanImpl.GroupAllocationTrucker;
 import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteOutClosure;
@@ -35,13 +38,28 @@ public class DataRegionMetricsImpl implements DataRegionMetrics, AllocatedPageTr
     /** */
     private final LongAdder totalAllocatedPages = new LongAdder();
 
+    /** */
+    private final ConcurrentHashMap<Integer, GroupAllocationTrucker> groupAllocationTruckers = new ConcurrentHashMap<>();
+
     /**
      * Counter for number of pages occupied by large entries (one entry is larger than one page).
      */
     private final LongAdder largeEntriesPages = new LongAdder();
 
     /** Counter for number of dirty pages. */
-    private LongAdder dirtyPages = new LongAdder();
+    private final LongAdder dirtyPages = new LongAdder();
+
+    /** */
+    private final LongAdder readPages = new LongAdder();
+
+    /** */
+    private final LongAdder writtenPages = new LongAdder();
+
+    /** */
+    private final LongAdder replacedPages = new LongAdder();
+
+    /** */
+    private final AtomicLong offHeapSize = new AtomicLong();
 
     /** */
     private volatile boolean metricsEnabled;
@@ -212,7 +230,7 @@ public DataRegionMetricsImpl(DataRegionConfiguration memPlcCfg, @Nullable Ignite
     }
 
     /** {@inheritDoc} */
-    public int getPageSize() {
+    @Override public int getPageSize() {
         if (!metricsEnabled)
             return 0;
 
@@ -221,6 +239,54 @@ public int getPageSize() {
         return pageMem.pageSize();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getPagesRead() {
+        if (!metricsEnabled)
+            return 0;
+
+        return readPages.longValue();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesWritten() {
+        if (!metricsEnabled)
+            return 0;
+
+        return writtenPages.longValue();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesReplaced() {
+        if (!metricsEnabled)
+            return 0;
+
+        return replacedPages.longValue();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOffHeapSize() {
+        if (!metricsEnabled)
+            return 0;
+
+        return offHeapSize.get();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOffheapUsedSize() {
+        if (!metricsEnabled)
+            return 0;
+
+        return pageMem.loadedPages() * pageMem.pageSize();
+    }
+
+    /**
+     *
+     * @param size Region size.
+     */
+    public void updateOffHeapSize(long size) {
+        this.offHeapSize.addAndGet(size);
+    }
+
     /**
      * Updates pageReplaceRate metric.
      */
@@ -229,9 +295,27 @@ public void updatePageReplaceRate(long pageAge) {
             pageReplaceRate.onHit();
 
             pageReplaceAge.onHits(pageAge);
+
+            replacedPages.increment();
         }
     }
 
+    /**
+     * Updates page read.
+     */
+    public void onPageRead(){
+        if (metricsEnabled)
+            readPages.increment();
+    }
+
+    /**
+     * Updates page written.
+     */
+    public void onPageWritten(){
+        if (metricsEnabled)
+            writtenPages.increment();
+    }
+
     /**
      * Increments dirtyPages counter.
      */
@@ -273,6 +357,16 @@ public void incrementTotalAllocatedPages() {
         }
     }
 
+    /**
+     * Get or allocate group allocation trucker.
+     *
+     * @param grpId Group id.
+     * @return Group allocation trucker.
+     */
+    public GroupAllocationTrucker getOrAllocateGroupPageAllocationTracker(int grpId) {
+        return groupAllocationTruckers.getOrDefault(grpId, new GroupAllocationTrucker(this));
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
index fc746fe3be3ac..9ab682487512c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
@@ -158,4 +158,29 @@ class DataRegionMetricsMXBeanImpl implements DataRegionMetricsMXBean {
     @Override public String getSwapPath() {
         return dataRegCfg.getSwapPath();
     }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesRead() {
+        return memMetrics.getPagesRead();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesWritten() {
+        return memMetrics.getPagesWritten();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesReplaced() {
+        return memMetrics.getPagesReplaced();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOffHeapSize() {
+        return memMetrics.getOffHeapSize();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOffheapUsedSize() {
+        return memMetrics.getOffheapUsedSize();
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
index d715e82caea63..6126c4baa5c1f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
@@ -68,6 +68,21 @@ public class DataRegionMetricsSnapshot implements DataRegionMetrics {
     /** */
     private int pageSize;
 
+    /** */
+    private long readPages;
+
+    /** */
+    private long writtenPages;
+
+    /** */
+    private long replacedPage;
+
+    /** */
+    private long offHeapSize;
+
+    /** */
+    private long offHeapUsedSize;
+
     /**
      * @param metrics Metrics instance to take a copy.
      */
@@ -87,6 +102,11 @@ public DataRegionMetricsSnapshot(DataRegionMetrics metrics) {
         checkpointBufferPages = metrics.getCheckpointBufferPages();
         checkpointBufferSize = metrics.getCheckpointBufferSize();
         pageSize = metrics.getPageSize();
+        readPages = metrics.getPagesRead();
+        writtenPages = metrics.getPagesWritten();
+        replacedPage = metrics.getPagesReplaced();
+        offHeapSize = metrics.getOffHeapSize();
+        offHeapUsedSize = metrics.getOffheapUsedSize();
     }
 
     /** {@inheritDoc} */
@@ -163,4 +183,29 @@ public DataRegionMetricsSnapshot(DataRegionMetrics metrics) {
     @Override public int getPageSize() {
         return pageSize;
     }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesRead() {
+        return readPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesWritten() {
+        return writtenPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesReplaced() {
+        return replacedPage;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOffHeapSize() {
+        return offHeapSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOffheapUsedSize() {
+        return offHeapUsedSize;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
index 4f32c25321e6e..b7f99e5038924 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
@@ -16,8 +16,14 @@
  */
 package org.apache.ignite.internal.processors.cache.persistence;
 
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteOutClosure;
 import org.apache.ignite.mxbean.DataStorageMetricsMXBean;
 
 /**
@@ -73,7 +79,19 @@ public class DataStorageMetricsImpl implements DataStorageMetricsMXBean {
     private volatile boolean metricsEnabled;
 
     /** */
-    private IgniteWriteAheadLogManager wal;
+    private volatile IgniteWriteAheadLogManager wal;
+
+    /** */
+    private volatile IgniteOutClosure<Long> walSizeProvider;
+
+    /** */
+    private volatile long lastWalSegmentRollOverTime;
+
+    /** */
+    private final AtomicLong totalCheckpointTime = new AtomicLong();
+
+    /** */
+    private volatile Collection<DataRegionMetrics> regionMetrics;
 
     /**
      * @param metricsEnabled Metrics enabled flag.
@@ -226,6 +244,158 @@ public DataStorageMetricsImpl(
         resetRates();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getWalTotalSize() {
+        if (!metricsEnabled)
+            return 0;
+
+        IgniteOutClosure<Long> walSize = this.walSizeProvider;
+
+        return walSize != null ? walSize.apply() : 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getWalLastRollOverTime() {
+        if (!metricsEnabled)
+            return 0;
+
+        return lastWalSegmentRollOverTime;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCheckpointTotalTime() {
+        if (!metricsEnabled)
+            return 0;
+
+        return totalCheckpointTime.get();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getDirtyPages() {
+        if (!metricsEnabled)
+            return 0;
+
+        Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+        if (F.isEmpty(regionMetrics0))
+            return 0;
+
+        long dirtyPages = 0L;
+
+        for (DataRegionMetrics rm : regionMetrics0)
+            dirtyPages += rm.getDirtyPages();
+
+        return dirtyPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesRead() {
+        if (!metricsEnabled)
+            return 0;
+
+        Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+        if (F.isEmpty(regionMetrics0))
+            return 0;
+
+        long readPages = 0L;
+
+        for (DataRegionMetrics rm : regionMetrics0)
+            readPages += rm.getPagesRead();
+
+        return readPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesWritten() {
+        if (!metricsEnabled)
+            return 0;
+
+        Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+        if (F.isEmpty(regionMetrics0))
+            return 0;
+
+        long writtenPages = 0L;
+
+        for (DataRegionMetrics rm : regionMetrics0)
+            writtenPages += rm.getPagesWritten();
+
+        return writtenPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesReplaced() {
+        if (!metricsEnabled)
+            return 0;
+
+        Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+        if (F.isEmpty(regionMetrics0))
+            return 0;
+
+        long replacedPages = 0L;
+
+        for (DataRegionMetrics rm : regionMetrics0)
+            replacedPages += rm.getPagesReplaced();
+
+        return replacedPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOffHeapSize() {
+        if (!metricsEnabled)
+            return 0;
+
+        Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+        if (F.isEmpty(regionMetrics0))
+            return 0;
+
+        long offHeapSize = 0L;
+
+        for (DataRegionMetrics rm : regionMetrics0)
+            offHeapSize += rm.getOffHeapSize();
+
+        return offHeapSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOffheapUsedSize() {
+        if (!metricsEnabled)
+            return 0;
+
+        Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+        if (F.isEmpty(regionMetrics0))
+            return 0;
+
+        long offHeapUsedSize = 0L;
+
+        for (DataRegionMetrics rm : regionMetrics0)
+            offHeapUsedSize += rm.getOffheapUsedSize();
+
+        return offHeapUsedSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTotalAllocatedSize() {
+        if (!metricsEnabled)
+            return 0;
+
+        Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+        if (F.isEmpty(regionMetrics0))
+            return 0;
+
+        long totalAllocatedSize = 0L;
+
+        for (DataRegionMetrics rm : regionMetrics0)
+            totalAllocatedSize += rm.getTotalAllocatedSize();
+
+        return totalAllocatedSize;
+    }
+
     /**
      * @param wal Write-ahead log manager.
      */
@@ -233,6 +403,27 @@ public void wal(IgniteWriteAheadLogManager wal) {
         this.wal = wal;
     }
 
+    /**
+     * @param walSizeProvider Wal size provider.
+     */
+    public void setWalSizeProvider(IgniteOutClosure<Long> walSizeProvider){
+        this.walSizeProvider = walSizeProvider;
+    }
+
+    /**
+     *
+     */
+    public void onWallRollOver() {
+        this.lastWalSegmentRollOverTime = U.currentTimeMillis();
+    }
+
+    /**
+     *
+     */
+    public void regionMetrics(Collection<DataRegionMetrics> regionMetrics){
+        this.regionMetrics = regionMetrics;
+    }
+
     /**
      * @return Metrics enabled flag.
      */
@@ -269,6 +460,8 @@ public void onCheckpoint(
             lastCpTotalPages = totalPages;
             lastCpDataPages = dataPages;
             lastCpCowPages = cowPages;
+
+            totalCheckpointTime.addAndGet(duration);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java
index c67eeeb1f2bb7..cb522791652c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java
@@ -62,6 +62,36 @@ public class DataStorageMetricsSnapshot implements DataStorageMetrics {
     /** */
     private long lastCpCowPages;
 
+    /** */
+    private long walTotalSize;
+
+    /** */
+    private long walLastRollOverTime;
+
+    /** */
+    private long checkpointTotalSize;
+
+    /** */
+    private long dirtyPages;
+
+    /** */
+    private long readPages;
+
+    /** */
+    private long writtenPages;
+
+    /** */
+    private long replacedPages;
+
+    /** */
+    private long offHeapSize;
+
+    /** */
+    private long offHeadUsedSize;
+
+    /** */
+    private long totalAllocatedSize;
+
     /**
      * @param metrics Metrics.
      */
@@ -79,6 +109,16 @@ public DataStorageMetricsSnapshot(DataStorageMetrics metrics) {
         lastCpTotalPages = metrics.getLastCheckpointTotalPagesNumber();
         lastCpDataPages = metrics.getLastCheckpointDataPagesNumber();
         lastCpCowPages = metrics.getLastCheckpointCopiedOnWritePagesNumber();
+        walTotalSize = metrics.getWalTotalSize();
+        walLastRollOverTime = metrics.getWalLastRollOverTime();
+        checkpointTotalSize = metrics.getCheckpointTotalTime();
+        dirtyPages = metrics.getDirtyPages();
+        readPages = metrics.getPagesRead();
+        writtenPages = metrics.getPagesWritten();
+        replacedPages = metrics.getPagesReplaced();
+        offHeapSize = metrics.getOffHeapSize();
+        offHeadUsedSize = metrics.getOffheapUsedSize();
+        totalAllocatedSize = metrics.getTotalAllocatedSize();
     }
 
     /** {@inheritDoc} */
@@ -146,6 +186,56 @@ public DataStorageMetricsSnapshot(DataStorageMetrics metrics) {
         return lastCpCowPages;
     }
 
+    /** {@inheritDoc} */
+    @Override public long getWalTotalSize() {
+        return walTotalSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getWalLastRollOverTime() {
+        return walLastRollOverTime;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCheckpointTotalTime() {
+        return checkpointTotalSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getDirtyPages() {
+        return dirtyPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesRead() {
+        return readPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesWritten() {
+        return writtenPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getPagesReplaced() {
+        return replacedPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOffHeapSize() {
+        return offHeapSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOffheapUsedSize() {
+        return offHeadUsedSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTotalAllocatedSize() {
+        return totalAllocatedSize;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(DataStorageMetricsSnapshot.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 59aad5f83ade0..f24f5d87fc08a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -85,6 +85,7 @@
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.DirectMemoryRegion;
 import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
 import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.FullPageId;
@@ -459,6 +460,8 @@ public IgniteInternalFuture<Void> enableCheckpoints(boolean enable) {
             createDataRegionConfiguration(memCfg),
             false
         );
+
+        persStoreMetrics.regionMetrics(memMetricsMap.values());
     }
 
     /**
@@ -970,7 +973,7 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize, long chpBufSi
             changeTracker = null;
 
         PageMemoryImpl pageMem = new PageMemoryImpl(
-            memProvider,
+            wrapMetricsMemoryProvider(memProvider, memMetrics),
             calculateFragmentSizes(
                 memCfg.getConcurrencyLevel(),
                 cacheSize,
@@ -979,6 +982,8 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize, long chpBufSi
             cctx,
             memCfg.getPageSize(),
             (fullId, pageBuf, tag) -> {
+                memMetrics.onPageWritten();
+
                 // First of all, write page to disk.
                 storeMgr.write(fullId.groupId(), fullId.pageId(), pageBuf, tag);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index bf080b6736d0d..a251234683342 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.DirectMemoryRegion;
 import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
 import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.PageMemory;
@@ -247,6 +248,7 @@ protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteChe
             CU.isPersistenceEnabled(memCfg)
         );
 
+
         dataRegionsInitialized = true;
     }
 
@@ -957,7 +959,7 @@ protected PageMemory createPageMemory(
 
         PageMemory pageMem = new PageMemoryNoStoreImpl(
             log,
-            memProvider,
+            wrapMetricsMemoryProvider(memProvider, memMetrics),
             cctx,
             memCfg.getPageSize(),
             memPlcCfg,
@@ -970,6 +972,39 @@ protected PageMemory createPageMemory(
         return pageMem;
     }
 
+    /**
+     * @param memoryProvider0 Memory provider.
+     * @param memMetrics Memory metrics.
+     * @return Wrapped memory provider.
+     */
+    protected DirectMemoryProvider wrapMetricsMemoryProvider(
+        final DirectMemoryProvider memoryProvider0,
+        final DataRegionMetricsImpl memMetrics
+    ) {
+        return new DirectMemoryProvider() {
+            private final DirectMemoryProvider memProvider = memoryProvider0;
+
+            @Override public void initialize(long[] chunkSizes) {
+                memProvider.initialize(chunkSizes);
+            }
+
+            @Override public void shutdown() {
+                memProvider.shutdown();
+            }
+
+            @Override public DirectMemoryRegion nextRegion() {
+                DirectMemoryRegion nextMemoryRegion = memProvider.nextRegion();
+
+                if (nextMemoryRegion == null)
+                    return null;
+
+                memMetrics.updateOffHeapSize(nextMemoryRegion.size());
+
+                return nextMemoryRegion;
+            }
+        };
+    }
+
     /**
      * @param path Path to the working directory.
      * @param consId Consistent ID of the local node.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 1c1b3e24e175d..661694d72101b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -52,10 +52,12 @@
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
 import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.marshaller.jdk.JdkMarshaller;
@@ -409,13 +411,20 @@ private CacheStoreHolder initForCache(CacheGroupDescriptor grpDesc, CacheConfigu
 
         File cacheWorkDir = cacheWorkDir(ccfg);
 
-        AllocatedPageTracker allocatedTracker =
-            cctx.database().dataRegion(grpDesc.config().getDataRegionName()).memoryMetrics();
+        String dataRegionName = grpDesc.config().getDataRegionName();
 
-        return initDir(cacheWorkDir,
+        DataRegionMetricsImpl regionMetrics = cctx.database().dataRegion(dataRegionName).memoryMetrics();
+
+        int grpId = CU.cacheId(grpDesc.cacheOrGroupName());
+
+        AllocatedPageTracker allocatedTracker = regionMetrics.getOrAllocateGroupPageAllocationTracker(grpId);
+
+        return initDir(
+            cacheWorkDir,
             grpDesc.groupId(),
             grpDesc.config().getAffinity().partitions(),
-            allocatedTracker);
+            allocatedTracker
+        );
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index c684ce155a2a6..10d0a3f4cb047 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -777,6 +777,8 @@ else if (relPtr == OUTDATED_REL_PTR) {
 
                 try {
                     storeMgr.read(grpId, pageId, buf);
+
+                    memMetrics.onPageRead();
                 }
                 catch (IgniteDataIntegrityViolationException ignore) {
                     U.warn(log, "Failed to read page (data integrity violation encountered, will try to " +
@@ -785,6 +787,8 @@ else if (relPtr == OUTDATED_REL_PTR) {
                     buf.rewind();
 
                     tryToRestorePage(fullId, buf);
+
+                    memMetrics.onPageRead();
                 }
                 finally {
                     rwLock.writeUnlock(lockedPageAbsPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
@@ -1173,6 +1177,8 @@ private boolean copyPageForCheckpoint(
             assert PageIO.getType(outBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
             assert PageIO.getVersion(outBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
 
+            memMetrics.onPageWritten();
+
             return true;
         }
         finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 9917e85f46fe6..a28b73b9ab2a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -66,7 +66,6 @@
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.events.WalSegmentArchivedEvent;
 import org.apache.ignite.failure.FailureContext;
-import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -101,6 +100,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.CIX1;
+import org.apache.ignite.internal.util.typedef.CO;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -367,14 +367,14 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
 
             checkWalConfiguration();
 
-            walWorkDir = initDirectory(
+            final File walWorkDir0 = walWorkDir = initDirectory(
                 dsCfg.getWalPath(),
                 DataStorageConfiguration.DFLT_WAL_PATH,
                 resolveFolders.folderName(),
                 "write ahead log work directory"
             );
 
-            walArchiveDir = initDirectory(
+            final File walArchiveDir0 = walArchiveDir = initDirectory(
                 dsCfg.getWalArchivePath(),
                 DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH,
                 resolveFolders.folderName(),
@@ -389,6 +389,21 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
 
             checkOrPrepareFiles();
 
+            if (metrics != null)
+                metrics.setWalSizeProvider(new CO<Long>() {
+                    @Override public Long apply() {
+                        long size = 0;
+
+                        for (File f : walWorkDir0.listFiles())
+                            size += f.length();
+
+                        for (File f : walArchiveDir0.listFiles())
+                            size += f.length();
+
+                        return size;
+                    }
+                });
+
             IgniteBiTuple<Long, Long> tup = scanMinMaxArchiveIndices();
 
             lastTruncatedArchiveIdx = tup == null ? -1 : tup.get1() - 1;
@@ -1042,6 +1057,9 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, I
             return hnd;
 
         if (hnd.close(true)) {
+            if (metrics.metricsEnabled())
+                metrics.onWallRollOver();
+
             FileWriteHandle next = initNextWriteHandle(cur);
 
             next.writeHeader();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 2305389509f5d..e354b439ef2b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -94,6 +94,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.CIX1;
+import org.apache.ignite.internal.util.typedef.CO;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.SB;
@@ -320,14 +321,14 @@ public FsyncModeFileWriteAheadLogManager(@NotNull final GridKernalContext ctx) {
 
             checkWalConfiguration();
 
-            walWorkDir = initDirectory(
+            final File walWorkDir0 = walWorkDir = initDirectory(
                 dsCfg.getWalPath(),
                 DataStorageConfiguration.DFLT_WAL_PATH,
                 resolveFolders.folderName(),
                 "write ahead log work directory"
             );
 
-            walArchiveDir = initDirectory(
+            final File walArchiveDir0 = walArchiveDir = initDirectory(
                 dsCfg.getWalArchivePath(),
                 DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH,
                 resolveFolders.folderName(),
@@ -342,6 +343,20 @@ public FsyncModeFileWriteAheadLogManager(@NotNull final GridKernalContext ctx) {
 
             checkOrPrepareFiles();
 
+            metrics.setWalSizeProvider(new CO<Long>() {
+                @Override public Long apply() {
+                    long size = 0;
+
+                    for (File f : walWorkDir0.listFiles())
+                        size += f.length();
+
+                    for (File f : walArchiveDir0.listFiles())
+                        size += f.length();
+
+                    return size;
+                }
+            });
+
             IgniteBiTuple<Long, Long> tup = scanMinMaxArchiveIndices();
 
             lastTruncatedArchiveIdx = tup == null ? -1 : tup.get1() - 1;
@@ -944,6 +959,9 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, I
             return hnd;
 
         if (hnd.close(true)) {
+            if (metrics.metricsEnabled())
+                metrics.onWallRollOver();
+
             FileWriteHandle next = initNextWriteHandle(cur.idx);
 
             boolean swapped = currentHndUpd.compareAndSet(this, hnd, next);
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/CacheGroupMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/CacheGroupMetricsMXBean.java
index 46b99b14c1bb1..9973d84280df7 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/CacheGroupMetricsMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/CacheGroupMetricsMXBean.java
@@ -139,4 +139,28 @@ public interface CacheGroupMetricsMXBean {
      */
     @MXBeanDescription("Affinity partitions assignment map.")
     public Map<Integer, List<String>> getAffinityPartitionsAssignmentMap();
+
+    /**
+     * Cache group type.
+     */
+    @MXBeanDescription("Cache group type.")
+    public String getType();
+
+    /**
+     * Local partition ids.
+     */
+    @MXBeanDescription("Local partition ids.")
+    public List<Integer> getPartitionIds();
+
+    /**
+     * Cache group total allocated pages.
+     */
+    @MXBeanDescription("Cache group total allocated pages.")
+    public long getTotalAllocatedPages();
+
+    /**
+     * Total size of memory allocated for group, in bytes.
+     */
+    @MXBeanDescription("Total size of memory allocated for group, in bytes.")
+    public long getTotalAllocatedSize();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/DataRegionMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/DataRegionMetricsMXBean.java
index 5b2697abb0ee7..9bb1af610e567 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/DataRegionMetricsMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/DataRegionMetricsMXBean.java
@@ -89,6 +89,26 @@ public interface DataRegionMetricsMXBean extends DataRegionMetrics {
     @MXBeanDescription("Number of pages residing in physical RAM.")
     @Override public long getPhysicalMemoryPages();
 
+    /** {@inheritDoc} */
+    @MXBeanDescription("Number of pages read from last restart.")
+    @Override public long getPagesRead();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Number of pages written from last restart.")
+    @Override public long getPagesWritten();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Number of pages replaced from last restart.")
+    @Override public long getPagesReplaced();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Offheap size in bytes.")
+    @Override public long getOffHeapSize();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Offheap used size in bytes.")
+    @Override public long getOffheapUsedSize();
+
     /**
      * Enables memory metrics collection on an Apache Ignite node.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java
index 2051da39b41e4..2450874e64df9 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java
@@ -44,6 +44,18 @@ public interface DataStorageMetricsMXBean extends DataStorageMetrics {
     @MXBeanDescription("WAL buffer poll spins number over the last time interval.")
     @Override long getWalBuffPollSpinsRate();
 
+    /** {@inheritDoc} */
+    @MXBeanDescription("Total size in bytes for storage wal files.")
+    @Override long getWalTotalSize();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Time of the last WAL segment rollover.")
+    @Override long getWalLastRollOverTime();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Total checkpoint time from last restart.")
+    @Override long getCheckpointTotalTime();
+
     /** {@inheritDoc} */
     @MXBeanDescription("Duration of the last checkpoint in milliseconds.")
     @Override long getLastCheckpointDuration();
@@ -76,6 +88,34 @@ public interface DataStorageMetricsMXBean extends DataStorageMetrics {
     @MXBeanDescription("Number of pages copied to a temporary checkpoint buffer during the last checkpoint.")
     @Override long getLastCheckpointCopiedOnWritePagesNumber();
 
+    /** {@inheritDoc} */
+    @MXBeanDescription("Total dirty pages for the next checkpoint.")
+    @Override long getDirtyPages();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("The number of read pages from last restart.")
+    @Override long getPagesRead();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("The number of written pages from last restart.")
+    @Override long getPagesWritten();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("The number of replaced pages from last restart.")
+    @Override long getPagesReplaced();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Total offheap size in bytes.")
+    @Override long getOffHeapSize();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Total used offheap size in bytes.")
+    @Override long getOffheapUsedSize();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Total size of memory allocated in bytes.")
+    @Override long getTotalAllocatedSize();
+
     /**
      * Enables persistence metrics collection on an Apache Ignite node.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java
index 416087b2b922f..177bcbedff4ae 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/SwapPathConstructionSelfTest.java
@@ -124,7 +124,9 @@ private String extractDefaultPageMemoryAllocPath(GridKernalContext context) {
 
         Object memProvider = U.field(pageMem, "directMemoryProvider");
 
-        return ((File) U.field(memProvider, "allocationPath")).getAbsolutePath();
+        Object memProvider0 = U.field(memProvider, "memProvider");
+
+        return ((File) U.field(memProvider0, "allocationPath")).getAbsolutePath();
     }
 
     /**
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
index bea31b12c62dd..22b8986f812f8 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
@@ -34,7 +34,13 @@ public class DataRegionMetricsParityTest
         };
 
         /** Properties that are missing on .NET side. */
-        private static readonly string[] MissingProperties = {};
+        private static readonly string[] MissingProperties = {
+            "PagesRead",
+            "PagesWritten",
+            "PagesReplaced",
+            "OffHeapSize",
+            "OffheapUsedSize"
+        };
 
         /// <summary>
         /// Tests the API parity.
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs
index 22068910faf72..300ab25c3f433 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs
@@ -1,4 +1,4 @@
-/*
+/*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
@@ -28,7 +28,17 @@ public class DataStorageMetricsParityTest
         private static readonly string[] MissingProperties =
         {
             // IGNITE-7305
-            "WalBuffPollSpinsRate"
+            "WalBuffPollSpinsRate",
+            "WalTotalSize",
+            "WalLastRollOverTime",
+            "CheckpointTotalTime",
+            "DirtyPages",
+            "PagesRead",
+            "PagesWritten",
+            "PagesReplaced",
+            "OffHeapSize",
+            "OffheapUsedSize",
+            "TotalAllocatedSize"
         };
 
         /// <summary>

From 553e0ff7c2fdecd5d4a64afd88d93e3ccffbe37a Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Fri, 20 Apr 2018 16:20:00 +0300
Subject: [PATCH 096/543] IGNITE-8077 Introduced additional transaction metrics
 - Fixes #3879.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../apache/ignite/internal/IgniteKernal.java  |  56 ++--
 .../TransactionMetricsMxBeanImpl.java         | 104 ++++++
 .../cache/GridCacheSharedContext.java         |   6 +-
 .../TransactionMetricsAdapter.java            | 206 +++++++++++-
 .../mxbean/TransactionMetricsMxBean.java      |  86 +++++
 .../transactions/TransactionMetrics.java      |  52 +++
 .../TransactionMetricsMxBeanImplTest.java     | 302 ++++++++++++++++++
 .../IgniteCacheMetricsSelfTestSuite.java      |   3 +
 .../ApiParity/TransactionMetricsParityTest.cs |  14 +-
 9 files changed, 786 insertions(+), 43 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/TransactionMetricsMxBeanImpl.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/mxbean/TransactionMetricsMxBean.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 2dc3df90c5a76..79e0c955cba01 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -201,6 +201,7 @@
 import org.apache.ignite.mxbean.StripedExecutorMXBean;
 import org.apache.ignite.mxbean.WorkersControlMXBean;
 import org.apache.ignite.mxbean.ThreadPoolMXBean;
+import org.apache.ignite.mxbean.TransactionMetricsMxBean;
 import org.apache.ignite.plugin.IgnitePlugin;
 import org.apache.ignite.plugin.PluginNotFoundException;
 import org.apache.ignite.plugin.PluginProvider;
@@ -650,7 +651,7 @@ public IgniteKernal(@Nullable GridSpringResourceContext rsrcCtx) {
     }
 
     /**
-     * @param name  New attribute name.
+     * @param name New attribute name.
      * @param val New attribute value.
      * @throws IgniteCheckedException If duplicated SPI name found.
      */
@@ -750,8 +751,7 @@ public void start(
         @Nullable final Map<String, ? extends ExecutorService> customExecSvcs,
         GridAbsClosure errHnd
     )
-        throws IgniteCheckedException
-    {
+        throws IgniteCheckedException {
         gw.compareAndSet(null, new GridKernalGatewayImpl(cfg.getIgniteInstanceName()));
 
         GridKernalGateway gw = this.gw.get();
@@ -954,7 +954,7 @@ public void start(
             try {
                 startProcessor(new PdsConsistentIdProcessor(ctx));
                 startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx));
-                startProcessor(new  GridAffinityProcessor(ctx));
+                startProcessor(new GridAffinityProcessor(ctx));
                 startProcessor(createComponent(GridSegmentationProcessor.class, ctx));
                 startProcessor(createComponent(IgniteCacheObjectProcessor.class, ctx));
                 startProcessor(createComponent(IGridClusterStateProcessor.class, ctx));
@@ -1720,6 +1720,7 @@ private void startProcessor(GridProcessor proc) throws IgniteCheckedException {
 
     /**
      * Returns class name filter for marshaller.
+     *
      * @return Class name filter for marshaller.
      */
     private IgnitePredicate<String> classNameFilter() throws IgniteCheckedException {
@@ -1774,9 +1775,9 @@ private ClassSet classBlackList() throws IgniteCheckedException {
         return clsSet;
     }
 
-
     /**
      * Reads class names from resource referred by given system property name and returns set of classes.
+     *
      * @param fileName File name containing list of classes.
      * @param clsSet Class set for update.
      * @return Set of classes.
@@ -1805,7 +1806,7 @@ private void addClassNames(String fileName, ClassSet clsSet) throws IgniteChecke
                     }
                     catch (IllegalArgumentException e) {
                         throw new IgniteCheckedException("Exception occurred while reading list of classes" +
-                        "[path=" + fileName + ", row=" + i + ", line=" + s + ']', e);
+                            "[path=" + fileName + ", row=" + i + ", line=" + s + ']', e);
                     }
                 }
             }
@@ -1836,7 +1837,6 @@ private String onOff(boolean b) {
     }
 
     /**
-     *
      * @return Whether or not REST is enabled.
      */
     private boolean isRestEnabled() {
@@ -1903,7 +1903,6 @@ private void ackConfigUrl() {
             log.info("Config URL: " + System.getProperty(IGNITE_CONFIG_URL, "n/a"));
     }
 
-
     /**
      * Acks configuration.
      */
@@ -1921,7 +1920,7 @@ private void ackLogger() {
         assert log != null;
 
         if (log.isInfoEnabled())
-            log.info("Logger: " + log.getLoggerInfo() );
+            log.info("Logger: " + log.getLoggerInfo());
     }
 
     /**
@@ -2345,8 +2344,8 @@ else if (state == STARTING)
      * USED ONLY FOR TESTING.
      *
      * @param name Cache name.
-     * @param <K>  Key type.
-     * @param <V>  Value type.
+     * @param <K> Key type.
+     * @param <V> Value type.
      * @return Internal cache instance.
      */
     /*@java.test.only*/
@@ -2410,7 +2409,6 @@ private void ackDaemon() {
     }
 
     /**
-     *
      * @return {@code True} is this node is daemon.
      */
     private boolean isDaemon() {
@@ -2420,11 +2418,8 @@ private boolean isDaemon() {
     }
 
     /**
-     * Whether or not remote JMX management is enabled for this node. Remote JMX management is
-     * enabled when the following system property is set:
-     * <ul>
-     *     <li>{@code com.sun.management.jmxremote}</li>
-     * </ul>
+     * Whether or not remote JMX management is enabled for this node. Remote JMX management is enabled when the
+     * following system property is set: <ul> <li>{@code com.sun.management.jmxremote}</li> </ul>
      *
      * @return {@code True} if remote JMX management is enabled - {@code false} otherwise.
      */
@@ -2433,9 +2428,9 @@ private boolean isDaemon() {
     }
 
     /**
-     * Whether or not node restart is enabled. Node restart us supported when this node was started
-     * with {@code bin/ignite.{sh|bat}} script using {@code -r} argument. Node can be
-     * programmatically restarted using {@link Ignition#restart(boolean)}} method.
+     * Whether or not node restart is enabled. Node restart us supported when this node was started with {@code
+     * bin/ignite.{sh|bat}} script using {@code -r} argument. Node can be programmatically restarted using {@link
+     * Ignition#restart(boolean)}} method.
      *
      * @return {@code True} if restart mode is enabled, {@code false} otherwise.
      * @see Ignition#restart(boolean)
@@ -2812,7 +2807,6 @@ public <K, V> IgniteInternalCache<K, V> getCache(String name) {
         }
     }
 
-
     /** {@inheritDoc} */
     @Override public Collection<IgniteCache> createCaches(Collection<CacheConfiguration> cacheCfgs) {
         A.notNull(cacheCfgs, "cacheCfgs");
@@ -2886,12 +2880,12 @@ public <K, V> IgniteInternalCache<K, V> getCache(String name) {
             if (ctx.cache().cache(cacheCfg.getName()) == null) {
                 res =
                     sql ? ctx.cache().dynamicStartSqlCache(cacheCfg).get() :
-                    ctx.cache().dynamicStartCache(cacheCfg,
-                    cacheCfg.getName(),
-                    null,
-                    false,
-                    true,
-                    true).get();
+                        ctx.cache().dynamicStartCache(cacheCfg,
+                            cacheCfg.getName(),
+                            null,
+                            false,
+                            true,
+                            true).get();
             }
 
             return new IgniteBiTuple<>((IgniteCache<K, V>)ctx.cache().publicJCache(cacheCfg.getName()), res);
@@ -4022,7 +4016,6 @@ private static String componentClassName(Class<?> cls) {
 
     /**
      * @return IgniteKernal instance.
-     *
      * @throws ObjectStreamException If failed.
      */
     protected Object readResolve() throws ObjectStreamException {
@@ -4159,7 +4152,6 @@ private class MBeansManager {
          * @param qryExecSvc Query executor service
          * @param schemaExecSvc Schema executor service
          * @param customExecSvcs Custom named executors
-         *
          * @throws IgniteCheckedException if fails to register any of the MBeans
          */
         private void registerAllMBeans(
@@ -4193,6 +4185,10 @@ private void registerAllMBeans(
             ClusterMetricsMXBean metricsBean = new ClusterMetricsMXBeanImpl(cluster());
             registerMBean("Kernal", metricsBean.getClass().getSimpleName(), metricsBean, ClusterMetricsMXBean.class);
 
+            // Transactions
+            TransactionMetricsMxBean txMXBean = new TransactionMetricsMxBeanImpl(ctx.cache().transactions().metrics());
+            registerMBean("Transactions", txMXBean.getClass().getSimpleName(), txMXBean, TransactionMetricsMxBean.class);
+
             // Executors
             registerExecutorMBean("GridUtilityCacheExecutor", utilityCachePool);
             registerExecutorMBean("GridExecutionExecutor", execSvc);
@@ -4239,7 +4235,6 @@ private void registerAllMBeans(
          *
          * @param name name of the bean to register
          * @param exec executor to register a bean for
-         *
          * @throws IgniteCheckedException if registration fails.
          */
         private void registerExecutorMBean(String name, ExecutorService exec) throws IgniteCheckedException {
@@ -4254,7 +4249,6 @@ private void registerExecutorMBean(String name, ExecutorService exec) throws Ign
          * @param impl bean implementation
          * @param itf bean interface
          * @param <T> bean type
-         *
          * @throws IgniteCheckedException if registration fails
          */
         private <T> void registerMBean(String grp, String name, T impl, Class<T> itf) throws IgniteCheckedException {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/TransactionMetricsMxBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/TransactionMetricsMxBeanImpl.java
new file mode 100644
index 0000000000000..715bd273114e0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/TransactionMetricsMxBeanImpl.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.Map;
+import org.apache.ignite.mxbean.TransactionMetricsMxBean;
+import org.apache.ignite.transactions.TransactionMetrics;
+
+/**
+ * Transactions MXBean implementation.
+ */
+public class TransactionMetricsMxBeanImpl implements TransactionMetricsMxBean {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final TransactionMetrics transactionMetrics;
+
+    /**
+     * Create TransactionMetricsMxBeanImpl.
+     */
+    public TransactionMetricsMxBeanImpl() {
+        this(null);
+    }
+
+    /**
+     * @param transactionMetrics Transaction metrics.
+     */
+    TransactionMetricsMxBeanImpl(TransactionMetrics transactionMetrics) {
+        this.transactionMetrics = transactionMetrics;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long commitTime() {
+        return transactionMetrics.commitTime();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long rollbackTime() {
+        return transactionMetrics.rollbackTime();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int txCommits() {
+        return transactionMetrics.txCommits();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int txRollbacks() {
+        return transactionMetrics.txRollbacks();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map<String, String> getAllOwnerTransactions() {
+        return transactionMetrics.getAllOwnerTransactions();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map<String, String> getLongRunningOwnerTransactions(int duration) {
+        return transactionMetrics.getLongRunningOwnerTransactions(duration);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTransactionsCommittedNumber() {
+        return transactionMetrics.getTransactionsCommittedNumber();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTransactionsRolledBackNumber() {
+        return transactionMetrics.getTransactionsRolledBackNumber();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTransactionsHoldingLockNumber() {
+        return transactionMetrics.getTransactionsHoldingLockNumber();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getLockedKeysNumber() {
+        return transactionMetrics.getLockedKeysNumber();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOwnerTransactionsNumber() {
+        return transactionMetrics.getOwnerTransactionsNumber();
+    }
+}
+
+
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index b3b4f0dc7b432..b5c409699cc09 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -129,7 +129,7 @@ public class GridCacheSharedContext<K, V> {
     private ConcurrentHashMap<Integer, GridCacheContext<K, V>> ctxMap;
 
     /** Tx metrics. */
-    private volatile TransactionMetricsAdapter txMetrics;
+    private final TransactionMetricsAdapter txMetrics;
 
     /** Store session listeners. */
     private Collection<CacheStoreSessionListener> storeSesLsnrs;
@@ -208,7 +208,7 @@ public GridCacheSharedContext(
 
         this.storeSesLsnrs = storeSesLsnrs;
 
-        txMetrics = new TransactionMetricsAdapter();
+        txMetrics = new TransactionMetricsAdapter(kernalCtx);
 
         ctxMap = new ConcurrentHashMap<>();
 
@@ -621,7 +621,7 @@ public TransactionMetricsAdapter txMetrics() {
      * Resets tx metrics.
      */
     public void resetTxMetrics() {
-        txMetrics = new TransactionMetricsAdapter();
+        txMetrics.reset();
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
index 783796b8204c3..c537bb8e8ca82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionMetricsAdapter.java
@@ -21,14 +21,32 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.GridKernalContextImpl;
+import org.apache.ignite.internal.processors.cache.GridCacheMvccManager;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.util.GridStringBuilder;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.transactions.TransactionMetrics;
+import org.apache.ignite.transactions.TransactionState;
 
 /**
  * Tx metrics adapter.
  */
 public class TransactionMetricsAdapter implements TransactionMetrics, Externalizable {
+    /** Grid kernal context. */
+    private final GridKernalContext gridKernalCtx;
+
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -45,20 +63,17 @@ public class TransactionMetricsAdapter implements TransactionMetrics, Externaliz
     private volatile long rollbackTime;
 
     /**
-     *
+     * Create TransactionMetricsAdapter.
      */
     public TransactionMetricsAdapter() {
-
+        this(null);
     }
 
     /**
-     * @param m Transaction metrics to copy.
+     * @param ctx Kernal context.
      */
-    public TransactionMetricsAdapter(TransactionMetrics m) {
-        commitTime = m.commitTime();
-        rollbackTime = m.rollbackTime();
-        txCommits = m.txCommits();
-        txRollbacks = m.txRollbacks();
+    public TransactionMetricsAdapter(GridKernalContext ctx) {
+        gridKernalCtx = ctx;
     }
 
     /** {@inheritDoc} */
@@ -81,6 +96,41 @@ public TransactionMetricsAdapter(TransactionMetrics m) {
         return txRollbacks;
     }
 
+    /** {@inheritDoc} */
+    @Override public Map<String, String> getAllOwnerTransactions() {
+        return getNearTxs(0);
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map<String, String> getLongRunningOwnerTransactions(final int duration) {
+        return getNearTxs(duration);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTransactionsCommittedNumber() {
+        return gridKernalCtx.cache().context().txMetrics().txCommits();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTransactionsRolledBackNumber() {
+        return gridKernalCtx.cache().context().txMetrics().txRollbacks();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTransactionsHoldingLockNumber() {
+        return txHoldingLockNum();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getLockedKeysNumber() {
+        return txLockedKeysNum();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getOwnerTransactionsNumber() {
+        return nearTxNum();
+    }
+
     /**
      * Transaction commit callback.
      */
@@ -99,6 +149,146 @@ public void onTxRollback() {
         txRollbacks++;
     }
 
+    /**
+     * Reset.
+     */
+    public void reset() {
+        commitTime = 0;
+        txCommits = 0;
+        rollbackTime = 0;
+        txRollbacks = 0;
+    }
+
+    /**
+     * @param duration Duration.
+     */
+    private Map<String, String> getNearTxs(long duration) {
+        final Collection<GridNearTxLocal> txs = nearTxs(duration);
+
+        final HashMap<String, String> res = new HashMap<>(txs.size());
+
+        for (GridNearTxLocal tx : txs)
+            res.put(tx.xid().toString(), composeTx(tx));
+
+        return res;
+    }
+
+    /**
+     * @param id Id.
+     */
+    private String composeNodeInfo(final UUID id) {
+        final ClusterNode node = gridKernalCtx.discovery().node(id);
+        if (node == null)
+            return "";
+
+        return String.format("%s %s",
+            node.id(),
+            node.hostNames());
+    }
+
+    /**
+     * @param ids Ids.
+     */
+    private String composeNodeInfo(final Set<UUID> ids) {
+        final GridStringBuilder sb = new GridStringBuilder();
+
+        sb.a("[");
+
+        String delim = "";
+
+        for (UUID id : ids) {
+            sb
+                .a(delim)
+                .a(composeNodeInfo(id));
+            delim = ", ";
+        }
+
+        sb.a("]");
+
+        return sb.toString();
+    }
+
+    /**
+     * @param tx Transaction.
+     */
+    private String composeTx(final GridNearTxLocal tx) {
+        final TransactionState txState = tx.state();
+
+        String top = txState + ", NEAR, ";
+
+        if (txState == TransactionState.PREPARING) {
+            final Map<UUID, Collection<UUID>> transactionNodes = tx.transactionNodes();
+            if (!F.isEmpty(transactionNodes)) {
+                final Set<UUID> primaryNodes = transactionNodes.keySet();
+                if (!F.isEmpty(primaryNodes))
+                    top += "PRIMARY: " + composeNodeInfo(primaryNodes) + ", ";
+            }
+        }
+
+        final Long duration = System.currentTimeMillis() - tx.startTime();
+
+        return top + "DURATION: " + duration;
+    }
+
+    /**
+     *
+     */
+    private Collection<GridNearTxLocal> nearTxs(long duration) {
+        final long start = System.currentTimeMillis();
+        IgniteClosure<IgniteInternalTx, GridNearTxLocal> c = new IgniteClosure<IgniteInternalTx, GridNearTxLocal>() {
+            @Override public GridNearTxLocal apply(IgniteInternalTx tx) {
+                return ((GridNearTxLocal)tx);
+            }
+        };
+
+        IgnitePredicate<IgniteInternalTx> pred = new IgnitePredicate<IgniteInternalTx>() {
+            @Override public boolean apply(IgniteInternalTx tx) {
+                return tx.local() && tx.near() && start - tx.startTime() >= duration;
+            }
+        };
+
+        return F.viewReadOnly(gridKernalCtx.cache().context().tm().activeTransactions(), c, pred);
+    }
+
+    /**
+     *
+     */
+    private long nearTxNum() {
+        IgnitePredicate<IgniteInternalTx> pred = new IgnitePredicate<IgniteInternalTx>() {
+            @Override public boolean apply(IgniteInternalTx tx) {
+                return tx.local() && tx.near();
+            }
+        };
+
+        return F.size(gridKernalCtx.cache().context().tm().activeTransactions(), pred);
+    }
+
+    /**
+     * Count total number of holding locks on local node.
+     */
+    private long txHoldingLockNum() {
+        long holdingLockCounter = 0;
+
+        IgniteTxManager tm = gridKernalCtx.cache().context().tm();
+        for (IgniteInternalTx tx : tm.activeTransactions()) {
+            if ((tx.optimistic() && tx.state() == TransactionState.ACTIVE) || tx.empty() || !tx.local())
+                continue;
+
+            holdingLockCounter++;
+        }
+
+        return holdingLockCounter;
+    }
+
+    /**
+     * Count total number of locked keys on local node.
+     */
+    private long txLockedKeysNum() {
+        GridCacheMvccManager mvccManager = gridKernalCtx.cache().context().mvcc();
+
+        return mvccManager.lockedKeys().size() + mvccManager.nearLockedKeys().size();
+    }
+
     /** {@inheritDoc} */
     @Override public void writeExternal(ObjectOutput out) throws IOException {
         out.writeLong(commitTime);
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionMetricsMxBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionMetricsMxBean.java
new file mode 100644
index 0000000000000..effe28f5a5759
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionMetricsMxBean.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mxbean;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.transactions.TransactionMetrics;
+
+/**
+ * Transactions MXBean interface.
+ */
+@MXBeanDescription("MBean that provides access to Ignite transactions.")
+public interface TransactionMetricsMxBean extends TransactionMetrics {
+    /**
+     * All near transactions
+     *
+     * @return near transactions.
+     */
+    @MXBeanDescription("All near transactions.")
+    @Override public Map<String, String> getAllOwnerTransactions();
+
+    /**
+     * Long running near transactions
+     *
+     * @return near transactions.
+     */
+    @MXBeanDescription("Long running near transactions.")
+    @MXBeanParametersNames("duration")
+    @MXBeanParametersDescriptions("Duration, at least (ms).")
+    @Override public Map<String, String> getLongRunningOwnerTransactions(int duration);
+
+    /**
+     * The number of transactions which were committed.
+     *
+     * @return number of transactions which were committed.
+     */
+    @MXBeanDescription("The number of transactions which were committed.")
+    @Override public long getTransactionsCommittedNumber();
+
+    /**
+     * The number of transactions which were rollback.
+     *
+     * @return number of transactions which were rollback.
+     */
+    @MXBeanDescription("The number of transactions which were rollback.")
+    @Override public long getTransactionsRolledBackNumber();
+
+    /**
+     * The number of active transactions holding at least one key lock.
+     *
+     * @return number of active transactions holding at least one key lock.
+     */
+    @MXBeanDescription("The number of active transactions holding at least one key lock.")
+    @Override public long getTransactionsHoldingLockNumber();
+
+    /**
+     *  The number of keys locked on the node.
+     *
+     * @return number of keys locked on the node.
+     */
+    @MXBeanDescription("The number of keys locked on the node.")
+    @Override public long getLockedKeysNumber();
+
+    /**
+     * The number of active transactions for which this node is the initiator.
+     *
+     * @return number of active transactions for which this node is the initiator.
+     */
+    @MXBeanDescription("The number of active transactions for which this node is the initiator.")
+    @Override public long getOwnerTransactionsNumber();
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/transactions/TransactionMetrics.java b/modules/core/src/main/java/org/apache/ignite/transactions/TransactionMetrics.java
index a5af51ef382f2..fa2ec22f85986 100644
--- a/modules/core/src/main/java/org/apache/ignite/transactions/TransactionMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/transactions/TransactionMetrics.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.transactions;
 
 import java.io.Serializable;
+import java.util.Map;
 
 /**
  * Transaction metrics, shared across all caches.
@@ -50,4 +51,55 @@ public interface TransactionMetrics extends Serializable {
      * @return Number of transaction rollbacks.
      */
     public int txRollbacks();
+
+    /**
+     * Gets a map of all transactions for which the local node is the originating node.
+     *
+     * @return Map of local node owning transactions.
+     */
+    public Map<String, String> getAllOwnerTransactions();
+
+    /**
+     * Gets a map of all transactions for which the local node is the originating node and which duration
+     * exceeds the given duration.
+     *
+     * @return Map of local node owning transactions which duration is longer than {@code duration}.
+     */
+    public Map<String, String> getLongRunningOwnerTransactions(int duration);
+
+    /**
+     * The number of transactions which were committed on the local node.
+     *
+     * @return The number of transactions which were committed on the local node.
+     */
+    public long getTransactionsCommittedNumber();
+
+    /**
+     * The number of transactions which were rolled back on the local node.
+     *
+     * @return The number of transactions which were rolled back on the local node.
+     */
+    public long getTransactionsRolledBackNumber();
+
+    /**
+     * The number of active transactions on the local node holding at least one key lock.
+     *
+     * @return The number of active transactions holding at least one key lock.
+     */
+    public long getTransactionsHoldingLockNumber();
+
+    /**
+     * The number of keys locked on the node.
+     *
+     * @return The number of keys locked on the node.
+     */
+    public long getLockedKeysNumber();
+
+    /**
+     * The number of active transactions for which this node is the initiator. Effectively, this method is
+     * semantically equivalent to {@code getAllOwnerTransactions.size()}.
+     *
+     * @return The number of active transactions for which this node is the initiator.
+     */
+    public long getOwnerTransactionsNumber();
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java
new file mode 100644
index 0000000000000..80b0dc066300a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.ObjectName;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.mxbean.TransactionMetricsMxBean;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ *
+ */
+public class TransactionMetricsMxBeanImplTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int TRANSACTIONS = 10;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(name);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+        cfg.setLocalHost("127.0.0.1");
+
+        final CacheConfiguration cCfg = new CacheConfiguration()
+            .setName(DEFAULT_CACHE_NAME)
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1)
+            .setAtomicityMode(TRANSACTIONAL)
+            .setRebalanceMode(CacheRebalanceMode.ASYNC)
+            .setWriteSynchronizationMode(FULL_SYNC);
+
+        cfg.setCacheConfiguration(cCfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+        super.afterTest();
+    }
+
+    /**
+     *
+     */
+    public void testTxMetric() throws Exception {
+        //given:
+        int keysNumber = 10;
+
+        IgniteEx ignite = startGrid(0);
+
+        startGrid(1);
+
+        IgniteConfiguration clientConf = getConfiguration(getTestIgniteInstanceName(2));
+
+        clientConf.setClientMode(true);
+
+        IgniteEx client = startGrid(clientConf);
+
+        awaitPartitionMapExchange();
+
+        TransactionMetricsMxBean txMXBean = txMXBean(0);
+
+        final IgniteCache<Integer, String> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        //when: one transaction commit
+        ignite.transactions().txStart().commit();
+
+        //then:
+        assertEquals(1, txMXBean.getTransactionsCommittedNumber());
+
+        //when: transaction is opening
+        final Transaction tx1 = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ);
+
+        int localKeysNum = 0;
+
+        for (int i = 0; i < keysNumber; i++) {
+            cache.put(i, "");
+
+            if (affinity(cache).isPrimary(ignite.localNode(), i))
+                localKeysNum++;
+        }
+
+        //then:
+        assertEquals(localKeysNum, txMXBean.getLockedKeysNumber());
+        assertEquals(1, txMXBean.getTransactionsHoldingLockNumber());
+        assertEquals(1, txMXBean.getOwnerTransactionsNumber());
+
+        //when: transaction rollback
+        tx1.rollback();
+
+        //then:
+        assertEquals(1, txMXBean.getTransactionsRolledBackNumber());
+        assertEquals(0, txMXBean.getLockedKeysNumber());
+        assertEquals(0, txMXBean.getTransactionsHoldingLockNumber());
+        assertEquals(0, txMXBean.getOwnerTransactionsNumber());
+
+        //when: keysNumber transactions from MXbean owner node + keysNumber transactions from client.
+        CountDownLatch commitAllower = new CountDownLatch(1);
+        CountDownLatch transactionStarter = new CountDownLatch(keysNumber + keysNumber);
+
+        int txNumFromOwner = 0;
+
+        for (int i = 0; i < keysNumber; i++) {
+            new Thread(new TxThread(commitAllower, transactionStarter, ignite, i, i)).start();
+
+            if (affinity(cache).isPrimary(ignite.localNode(), i))
+                txNumFromOwner++;
+        }
+
+        int txNumFromClient = 0;
+
+        for (int i = keysNumber; i < keysNumber * 2; i++) {
+            new Thread(new TxThread(commitAllower, transactionStarter, client, i, i)).start();
+
+            if (affinity(cache).isPrimary(ignite.localNode(), i))
+                txNumFromClient++;
+        }
+
+        transactionStarter.await();
+
+        //then:
+        assertEquals(txNumFromOwner + txNumFromClient, txMXBean.getLockedKeysNumber());
+        assertEquals(keysNumber + txNumFromClient, txMXBean.getTransactionsHoldingLockNumber());
+        assertEquals(keysNumber, txMXBean.getOwnerTransactionsNumber());
+
+        commitAllower.countDown();
+    }
+
+    /**
+     *
+     */
+    public void testNearTxInfo() throws Exception {
+        IgniteEx primaryNode1 = startGrid(0);
+        IgniteEx primaryNode2 = startGrid(1);
+        IgniteEx nearNode = startGrid(2);
+
+        TransactionMetricsMxBean txMXBeanBackup = txMXBean(2);
+
+        awaitPartitionMapExchange();
+
+        final IgniteCache<Integer, String> primaryCache1 = primaryNode1.cache(DEFAULT_CACHE_NAME);
+        final IgniteCache<Integer, String> primaryCache2 = primaryNode2.cache(DEFAULT_CACHE_NAME);
+
+        final List<Integer> primaryKeys1 = primaryKeys(primaryCache1, TRANSACTIONS);
+        final List<Integer> primaryKeys2 = primaryKeys(primaryCache2, TRANSACTIONS);
+
+        CountDownLatch commitAllower = new CountDownLatch(1);
+        CountDownLatch transactionStarter = new CountDownLatch(primaryKeys1.size());
+
+        for (int i = 0; i < primaryKeys1.size(); i++)
+            new Thread(new TxThread(
+                commitAllower,
+                transactionStarter,
+                nearNode,
+                primaryKeys1.get(i),
+                primaryKeys2.get(i)
+            )).start();
+
+        transactionStarter.await();
+
+        final Map<String, String> transactions = txMXBeanBackup.getAllOwnerTransactions();
+
+        assertEquals(TRANSACTIONS, transactions.size());
+
+        int match = 0;
+
+        for (String txInfo : transactions.values()) {
+            if (txInfo.contains("ACTIVE")
+                && txInfo.contains("NEAR")
+                && !txInfo.contains("REMOTE"))
+                match++;
+        }
+
+        assertEquals(TRANSACTIONS, match);
+
+        commitAllower.countDown();
+    }
+
+    /**
+     *
+     */
+    private static class TxThread implements Runnable {
+        /** */
+        private CountDownLatch commitAllowLatch;
+        /** */
+        private CountDownLatch transactionStartLatch;
+        /** */
+        private Ignite ignite;
+        /** */
+        private int key1;
+        /** */
+        private int key2;
+
+        /**
+         * Create TxThread.
+         */
+        private TxThread(
+            CountDownLatch commitAllowLatch,
+            CountDownLatch transactionStartLatch,
+            final Ignite ignite,
+            final int key1,
+            final int key2
+        ) {
+            this.commitAllowLatch = commitAllowLatch;
+            this.transactionStartLatch = transactionStartLatch;
+            this.ignite = ignite;
+            this.key1 = key1;
+            this.key2 = key2;
+        }
+
+        /**
+         * @param ignite Ignite.
+         * @param key1 key 1.
+         * @param key2 key 2.
+         */
+        private TxThread(final Ignite ignite, final int key1, final int key2) {
+            commitAllowLatch = new CountDownLatch(0);
+            transactionStartLatch = new CountDownLatch(1);
+
+            this.ignite = ignite;
+            this.key1 = key1;
+            this.key2 = key2;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                ignite.cache(DEFAULT_CACHE_NAME).put(key1, Thread.currentThread().getName());
+                ignite.cache(DEFAULT_CACHE_NAME).put(key2, Thread.currentThread().getName());
+
+                transactionStartLatch.countDown();
+
+                commitAllowLatch.await();
+
+                tx.commit();
+            }
+            catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private TransactionMetricsMxBean txMXBean(int igniteInt) throws Exception {
+        ObjectName mbeanName = U.makeMBeanName(
+            getTestIgniteInstanceName(igniteInt),
+            "Transactions",
+            TransactionMetricsMxBeanImpl.class.getSimpleName()
+        );
+
+        MBeanServer mbeanSrv = ManagementFactory.getPlatformMBeanServer();
+
+        if (!mbeanSrv.isRegistered(mbeanName))
+            fail("MBean is not registered: " + mbeanName.getCanonicalName());
+
+        return MBeanServerInvocationHandler.newProxyInstance(mbeanSrv, mbeanName, TransactionMetricsMxBean.class, true);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
index ac7dca2e61f17..237f7e1189282 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.TransactionMetricsMxBeanImplTest;
 import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMBeanTest;
 import org.apache.ignite.internal.processors.cache.CacheGroupsMetricsRebalanceTest;
 import org.apache.ignite.internal.processors.cache.CacheMetricsEnableRuntimeTest;
@@ -74,6 +75,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheMetricsForClusterGroupSelfTest.class);
         suite.addTestSuite(OffheapCacheMetricsForClusterGroupSelfTest.class);
 
+        suite.addTestSuite(TransactionMetricsMxBeanImplTest.class);
+
         return suite;
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionMetricsParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionMetricsParityTest.cs
index 0327e3aec1d08..6740eb9fe987d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionMetricsParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionMetricsParityTest.cs
@@ -25,6 +25,18 @@ namespace Apache.Ignite.Core.Tests.ApiParity
     /// </summary>
     public class TransactionMetricsParityTest
     {
+            /** Properties that are missing on .NET side. */
+            private static readonly string[] MissingProperties =
+            {
+                "AllOwnerTransactions",
+                "LongRunningOwnerTransactions",
+                "TransactionsCommittedNumber",
+                "TransactionsRolledBackNumber",
+                "TransactionsHoldingLockNumber",
+                "LockedKeysNumber",
+                "OwnerTransactionsNumber",
+            };
+
         /// <summary>
         /// Tests the API parity.
         /// </summary>
@@ -33,7 +45,7 @@ public void TestTransactionMetrics()
         {
             ParityTest.CheckInterfaceParity(
                 @"modules\core\src\main\java\org\apache\ignite\transactions\TransactionMetrics.java",
-                typeof(ITransactionMetrics));
+                typeof(ITransactionMetrics),knownMissingMembers: MissingProperties);
         }
     }
 }
\ No newline at end of file

From d2332c4865445f372f6ce0ba70b7da1fec987a8f Mon Sep 17 00:00:00 2001
From: Andrey Kuznetsov <stkuzma@gmail.com>
Date: Fri, 20 Apr 2018 19:51:59 +0300
Subject: [PATCH 097/543] IGNITE-8303 Avoided failure handler invocation when
 exchange-worker terminates before reconnect.

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../cache/GridCachePartitionExchangeManager.java    |  7 ++++++-
 .../ignite/internal/IgniteClientRejoinTest.java     | 13 ++++++++++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 7ea161acb688f..35f14da8c94ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -2092,6 +2092,9 @@ private class ExchangeWorker extends GridWorker {
         /** */
         private boolean stop;
 
+        /** Indicates that worker terminates because the node needs to reconnect to the cluster. */
+        private boolean reconnectNeeded;
+
         /**
          * Constructor.
          */
@@ -2303,7 +2306,7 @@ void dumpExchangeDebugInfo() {
                 err = e;
             }
             finally {
-                if (err == null && !stop)
+                if (err == null && !stop && !reconnectNeeded)
                     err = new IllegalStateException("Thread " + name() + " is terminated unexpectedly");
 
                 if (err instanceof OutOfMemoryError)
@@ -2602,6 +2605,8 @@ else if (r != null) {
 
                     cctx.discovery().reconnect();
 
+                    reconnectNeeded = true;
+
                     return;
                 }
                 catch (IgniteCheckedException e) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
index 8edbb52babae1..462c8ab6197d8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.Ignition;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -209,7 +210,17 @@ public void testClientsReconnect() throws Exception {
                 @Override public Ignite call() throws Exception {
                     latch.await();
 
-                    return startGrid("client" + idx);
+                    String nodeName = "client" + idx;
+
+                    IgniteConfiguration cfg = getConfiguration(nodeName)
+                        .setFailureHandler((ignite, failureCtx) -> {
+                            // This should _not_ fire when exchange-worker terminates before reconnect.
+                            Runtime.getRuntime().halt(Ignition.KILL_EXIT_CODE);
+
+                            return false;
+                        });
+
+                    return startGrid(nodeName, optimize(cfg), null);
                 }
             });
 

From 1d42f48cc8a39c3de9cac0e434f527ab4041479b Mon Sep 17 00:00:00 2001
From: Pavel Tupitsyn <ptupitsyn@apache.org>
Date: Wed, 11 Apr 2018 11:51:45 +0300
Subject: [PATCH 098/543] IGNITE-7222 .NET: Ignore missing
 IgniteConfiguration.CommunicationFailureResolver

---
 .../ApiParity/IgniteConfigurationParityTest.cs                 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
index d68083f3b900b..bf34fc0214b46 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
@@ -63,7 +63,8 @@ public class IgniteConfigurationParityTest
             "ClassLoader",
             "CacheStoreSessionListenerFactories",
             "PlatformConfiguration",
-            "ExecutorConfiguration"
+            "ExecutorConfiguration",
+            "CommunicationFailureResolver"
         };
 
         /** Properties that are missing on .NET side. */

From 744662e071f9c963cf494f78ad54b503851e5c4a Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Sun, 22 Apr 2018 20:07:28 +0300
Subject: [PATCH 099/543] IGNITE-8353 Fixed serialVersionUID for
 AbstractEvictionPolicy

---
 .../apache/ignite/cache/eviction/AbstractEvictionPolicy.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicy.java b/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicy.java
index d58077f5b6e0c..bd1c4ca616bc2 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicy.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/eviction/AbstractEvictionPolicy.java
@@ -28,6 +28,9 @@
  * Common functionality implementation for eviction policies with max size/max memory and batch eviction support.
  */
 public abstract class AbstractEvictionPolicy<K, V> implements EvictionPolicy<K, V>, Externalizable {
+    /** */
+    private static final long serialVersionUID = 4358725333474509598L;
+
     /** Max memory size occupied by elements in container. */
     private volatile long maxMemSize;
 

From bfed389fb6f3437eee20bc6baf2b7e725f9fb6a7 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Mon, 23 Apr 2018 11:04:28 +0300
Subject: [PATCH 100/543] IGNITE-8077 Do not attempt to clean up work directory
 on running grid in ConfigVariationTests

---
 .../junits/IgniteConfigVariationsAbstractTest.java        | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteConfigVariationsAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteConfigVariationsAbstractTest.java
index 8365848a0133b..0e454a366473d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteConfigVariationsAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteConfigVariationsAbstractTest.java
@@ -81,15 +81,15 @@ protected boolean isSafeTopology() {
     @Override protected void beforeTestsStarted() throws Exception {
         assert testsCfg != null;
 
-        FileUtils.deleteDirectory(workDir);
-
-        info("Ignite's 'work' directory has been cleaned.");
-
         if (Ignition.allGrids().size() != testsCfg.gridCount()) {
             info("All nodes will be stopped, new " + testsCfg.gridCount() + " nodes will be started.");
 
             Ignition.stopAll(true);
 
+            FileUtils.deleteDirectory(workDir);
+
+            info("Ignite's 'work' directory has been cleaned.");
+
             startGrids(testsCfg.gridCount());
 
             for (int i = 0; i < testsCfg.gridCount(); i++)

From 628f3fb33416e6e5c963450d5544ac32b6ecfe4a Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Mon, 23 Apr 2018 12:32:18 +0300
Subject: [PATCH 101/543] IGNITE-8190 Print out an information message when
 local node is not in baseline

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../cluster/GridClusterStateProcessor.java    | 22 +++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 2700a20f46f20..d4d0eb1a76e95 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -321,6 +321,9 @@ private void writeBaselineTopology(BaselineTopology blt, BaselineTopologyHistory
     @Override @Nullable public IgniteInternalFuture<Boolean> onLocalJoin(DiscoCache discoCache) {
         final DiscoveryDataClusterState state = globalState;
 
+        if (state.active())
+            checkLocalNodeInBaseline(state.baselineTopology());
+
         if (state.transition()) {
             joinFut = new TransitionOnJoinWaitFuture(state, discoCache);
 
@@ -336,6 +339,23 @@ && isBaselineSatisfied(state.baselineTopology(), discoCache.serverNodes()))
         return null;
     }
 
+    /**
+     * Checks whether local node participating in Baseline Topology and warn if not.
+     */
+    private void checkLocalNodeInBaseline(BaselineTopology blt) {
+        if (blt == null || blt.consistentIds() == null || ctx.clientNode() || ctx.isDaemon())
+            return;
+
+        if (!CU.isPersistenceEnabled(ctx.config()))
+            return;
+
+        if (!blt.consistentIds().contains(ctx.discovery().localNode().consistentId())) {
+            U.quietAndInfo(log, "Local node is not included in Baseline Topology and will not be used " +
+                "for persistent data storage. Use control.(sh|bat) script or IgniteCluster interface to include " +
+                "the node to Baseline Topology.");
+        }
+    }
+
     /**
      * Checks whether all conditions to meet BaselineTopology are satisfied.
      */
@@ -1101,6 +1121,8 @@ private boolean sendComputeCheckGlobalState() {
     private void onFinalActivate(final StateChangeRequest req) {
         ctx.dataStructures().onBeforeActivate();
 
+        checkLocalNodeInBaseline(globalState.baselineTopology());
+
         ctx.closure().runLocalSafe(new Runnable() {
             @Override public void run() {
                 boolean client = ctx.clientNode();

From c2a8bbc6b7ca500a2b7e485c16664b440439e602 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Mon, 23 Apr 2018 12:55:25 +0300
Subject: [PATCH 102/543] IGNITE-8191 Print out information when cluster is not
 activated

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../main/java/org/apache/ignite/internal/IgniteKernal.java   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 79e0c955cba01..c256ac43f6b85 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2021,6 +2021,11 @@ private void ackStart(RuntimeMXBean rtBean) {
 
             log.info(str);
         }
+
+        if (!cluster().active()) {
+            U.quietAndInfo(log, ">>> Ignite cluster is not active (limited functionality available). " +
+                "Use control.(sh|bat) script or IgniteCluster interface to activate.");
+        }
     }
 
     /**

From 3c3a24e84a9a9f07d0833b29524de244930634c3 Mon Sep 17 00:00:00 2001
From: Nikolay Izhikov <nizhikov@apache.org>
Date: Fri, 20 Apr 2018 11:23:42 +0300
Subject: [PATCH 103/543] IGNITE-7077: Implementation of Spark query
 optimization. - Fixes #3397.

Signed-off-by: Nikolay Izhikov <nizhikov@apache.org>
---
 .../spark/IgniteDataFrameSettings.scala       |   9 +
 .../org/apache/ignite/spark/IgniteRDD.scala   |   7 +-
 .../spark/impl/IgniteRelationProvider.scala   |  20 +-
 .../impl/IgniteSQLAccumulatorRelation.scala   |  98 ++++
 .../spark/impl/IgniteSQLDataFrameRDD.scala    |  16 +-
 .../ignite/spark/impl/IgniteSQLRelation.scala |  67 +--
 .../optimization/AggregateExpressions.scala   | 114 ++++
 .../optimization/ConditionExpressions.scala   | 160 ++++++
 .../impl/optimization/DateExpressions.scala   | 127 ++++
 .../optimization/IgniteQueryContext.scala     |  52 ++
 .../impl/optimization/MathExpressions.scala   | 263 +++++++++
 .../impl/optimization/SimpleExpressions.scala | 180 ++++++
 .../impl/optimization/StringExpressions.scala | 154 +++++
 .../optimization/SupportedExpressions.scala   |  42 ++
 .../impl/optimization/SystemExpressions.scala | 122 ++++
 .../accumulator/JoinSQLAccumulator.scala      | 222 +++++++
 .../accumulator/QueryAccumulator.scala        |  70 +++
 .../accumulator/SelectAccumulator.scala       |  70 +++
 .../SingleTableSQLAccumulator.scala           | 124 ++++
 .../accumulator/UnionSQLAccumulator.scala     |  63 ++
 .../spark/impl/optimization/package.scala     | 230 ++++++++
 .../apache/ignite/spark/impl/package.scala    |  48 +-
 .../spark/sql/ignite/IgniteOptimization.scala | 436 ++++++++++++++
 .../spark/sql/ignite/IgniteSparkSession.scala |  10 +-
 .../ignite/spark/AbstractDataFrameSpec.scala  |  68 ++-
 .../ignite/spark/IgniteCatalogSpec.scala      |   7 +-
 .../spark/IgniteDataFrameSchemaSpec.scala     |   5 +-
 .../ignite/spark/IgniteDataFrameSuite.scala   |   9 +-
 ...gniteOptimizationAggregationFuncSpec.scala | 189 ++++++
 .../IgniteOptimizationDateFuncSpec.scala      | 230 ++++++++
 .../IgniteOptimizationDisableEnableSpec.scala | 127 ++++
 .../spark/IgniteOptimizationJoinSpec.scala    | 543 ++++++++++++++++++
 .../IgniteOptimizationMathFuncSpec.scala      | 358 ++++++++++++
 .../ignite/spark/IgniteOptimizationSpec.scala | 305 ++++++++++
 .../IgniteOptimizationStringFuncSpec.scala    | 313 ++++++++++
 .../IgniteOptimizationSystemFuncSpec.scala    | 147 +++++
 36 files changed, 4924 insertions(+), 81 deletions(-)
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLAccumulatorRelation.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/AggregateExpressions.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/ConditionExpressions.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/DateExpressions.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/IgniteQueryContext.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/MathExpressions.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/StringExpressions.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SupportedExpressions.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SystemExpressions.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/JoinSQLAccumulator.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/QueryAccumulator.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SelectAccumulator.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SingleTableSQLAccumulator.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/UnionSQLAccumulator.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/package.scala
 create mode 100644 modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteOptimization.scala
 create mode 100644 modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationAggregationFuncSpec.scala
 create mode 100644 modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDateFuncSpec.scala
 create mode 100644 modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDisableEnableSpec.scala
 create mode 100644 modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationJoinSpec.scala
 create mode 100644 modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationMathFuncSpec.scala
 create mode 100644 modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
 create mode 100644 modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationStringFuncSpec.scala
 create mode 100644 modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSystemFuncSpec.scala

diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteDataFrameSettings.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteDataFrameSettings.scala
index 6bff476e1ac85..9daaec4c4b7e1 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteDataFrameSettings.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteDataFrameSettings.scala
@@ -153,4 +153,13 @@ object IgniteDataFrameSettings {
       * @see [[org.apache.ignite.IgniteDataStreamer#perNodeParallelOperations(int)]]
       */
     val OPTION_STREAMER_PER_NODE_PARALLEL_OPERATIONS = "streamerPerNodeParallelOperations"
+
+    /**
+      * Option for a [[org.apache.spark.sql.SparkSession]] configuration.
+      * If `true` then all Ignite optimization of Spark SQL statements will be disabled.
+      * Default value is `false`.
+      *
+      * @see [[org.apache.spark.sql.ignite.IgniteOptimization]]
+      */
+    val OPTION_DISABLE_SPARK_SQL_OPTIMIZATION = "ignite.disableSparkSQLOptimization"
 }
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
index d87ea0a206d88..5fb81b6f08c44 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/IgniteRDD.scala
@@ -343,6 +343,11 @@ class IgniteRDD[K, V] (
 }
 
 object IgniteRDD {
+    /**
+      * Default decimal type.
+      */
+    private[spark] val DECIMAL = DecimalType(DecimalType.MAX_PRECISION, 3)
+
     /**
       * Gets Spark data type based on type name.
       *
@@ -357,7 +362,7 @@ object IgniteRDD {
         case "java.lang.Long" ⇒ LongType
         case "java.lang.Float" ⇒ FloatType
         case "java.lang.Double" ⇒ DoubleType
-        case "java.math.BigDecimal" ⇒ DataTypes.createDecimalType()
+        case "java.math.BigDecimal" ⇒ DECIMAL
         case "java.lang.String" ⇒ StringType
         case "java.util.Date" ⇒ DateType
         case "java.sql.Date" ⇒ DateType
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteRelationProvider.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteRelationProvider.scala
index a9f9f8947f335..e4fa9f73c4844 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteRelationProvider.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteRelationProvider.scala
@@ -26,6 +26,7 @@ import org.apache.ignite.spark.IgniteDataFrameSettings._
 import org.apache.ignite.spark.impl.QueryHelper.{createTable, dropTable, ensureCreateTableOptions, saveTable}
 import org.apache.spark.sql.SaveMode.{Append, Overwrite}
 import org.apache.spark.sql.ignite.IgniteExternalCatalog.{IGNITE_PROTOCOL, OPTION_GRID}
+import org.apache.spark.sql.ignite.IgniteOptimization
 import org.apache.spark.sql.sources._
 import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode}
 
@@ -176,11 +177,28 @@ class IgniteRelationProvider extends RelationProvider
       * @param sqlCtx SQL context.
       * @return Ignite SQL relation.
       */
-    private def createRelation(igniteCtx: IgniteContext, tblName: String, sqlCtx: SQLContext): BaseRelation =
+    private def createRelation(igniteCtx: IgniteContext, tblName: String, sqlCtx: SQLContext): BaseRelation = {
+        val optimizationDisabled =
+            sqlCtx.sparkSession.conf.get(OPTION_DISABLE_SPARK_SQL_OPTIMIZATION, "false").toBoolean
+
+        val experimentalMethods = sqlCtx.sparkSession.sessionState.experimentalMethods
+
+        if (optimizationDisabled) {
+            experimentalMethods.extraOptimizations = 
+                experimentalMethods.extraOptimizations.filter(_ != IgniteOptimization)
+        } 
+        else {
+            val optimizationExists = experimentalMethods.extraOptimizations.contains(IgniteOptimization)
+
+            if (!optimizationExists)
+                experimentalMethods.extraOptimizations = experimentalMethods.extraOptimizations :+ IgniteOptimization
+        }
+
         IgniteSQLRelation(
             igniteCtx,
             tblName,
             sqlCtx)
+    }
 
     /**
       * @param params Params.
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLAccumulatorRelation.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLAccumulatorRelation.scala
new file mode 100644
index 0000000000000..6eb600a040c29
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLAccumulatorRelation.scala
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl
+
+import org.apache.ignite.spark.impl
+import org.apache.ignite.spark.impl.optimization.accumulator.{JoinSQLAccumulator, QueryAccumulator}
+import org.apache.ignite.spark.impl.optimization.isSimpleTableAcc
+import org.apache.spark.Partition
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.{Row, SQLContext}
+import org.apache.spark.sql.sources.{BaseRelation, TableScan}
+import org.apache.spark.sql.types.{Metadata, StructField, StructType}
+
+/**
+  * Relation to query data from query generated by <code>QueryAccumulator</code>.
+  * <code>QueryAccumulator</code> is generated by <code>IgniteOptimization</code>.
+  *
+  * @see IgniteOptimization
+  */
+class IgniteSQLAccumulatorRelation[K, V](val acc: QueryAccumulator)
+    (@transient val sqlContext: SQLContext) extends BaseRelation with TableScan {
+
+    /** @inheritdoc */
+    override def schema: StructType =
+        StructType(acc.output.map { c ⇒
+            StructField(
+                name = c.name,
+                dataType = c.dataType,
+                nullable = c.nullable,
+                metadata = Metadata.empty)
+        })
+
+    /** @inheritdoc */
+    override def buildScan(): RDD[Row] =
+        IgniteSQLDataFrameRDD[K, V](
+            acc.igniteQueryContext.igniteContext,
+            acc.igniteQueryContext.cacheName,
+            schema,
+            acc.compileQuery(),
+            List.empty,
+            calcPartitions,
+            isDistributeJoin(acc)
+        )
+
+    /** @inheritdoc */
+    override def toString: String =
+        s"IgniteSQLAccumulatorRelation(columns=[${acc.output.map(_.name).mkString(", ")}], qry=${acc.compileQuery()})"
+
+    /**
+      * @return Collection of spark partition.
+      */
+    private def calcPartitions: Array[Partition] =
+        //If accumulator stores some complex query(join, aggregation, limit, order, etc.).
+        //we has to load data from Ignite as a single Spark partition.
+        if (!isSimpleTableAcc(acc)){
+            val aff = acc.igniteQueryContext.igniteContext.ignite().affinity(acc.igniteQueryContext.cacheName)
+
+            val parts = aff.partitions()
+
+            Array(IgniteDataFramePartition(0, primary = null, igniteParts = (0 until parts).toList))
+        }
+        else
+            impl.calcPartitions(acc.igniteQueryContext.igniteContext, acc.igniteQueryContext.cacheName)
+
+    /**
+      * @param acc Plan.
+      * @return True if plan of one or its children are `JoinSQLAccumulator`, false otherwise.
+      */
+    private def isDistributeJoin(acc: LogicalPlan): Boolean =
+        acc match {
+            case _: JoinSQLAccumulator ⇒
+                true
+
+            case _ ⇒
+                acc.children.exists(isDistributeJoin)
+        }
+}
+
+object IgniteSQLAccumulatorRelation {
+    def apply[K, V](acc: QueryAccumulator): IgniteSQLAccumulatorRelation[K, V] =
+        new IgniteSQLAccumulatorRelation[K, V](acc)(acc.igniteQueryContext.sqlContext)
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLDataFrameRDD.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLDataFrameRDD.scala
index 93ef52904bed5..ec502fc854455 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLDataFrameRDD.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLDataFrameRDD.scala
@@ -36,7 +36,8 @@ class IgniteSQLDataFrameRDD[K, V](
     schema: StructType,
     qryStr: String,
     args: List[_],
-    parts: Array[Partition]) extends
+    parts: Array[Partition],
+    distributedJoin: Boolean) extends
     IgniteSqlRDD[Row, JList[_], K, V](
         ic,
         cacheName,
@@ -56,13 +57,17 @@ class IgniteSQLDataFrameRDD[K, V](
     override def compute(partition: Partition, context: TaskContext): Iterator[Row] = {
         val qry0 = new SqlFieldsQuery(qryStr)
 
+        qry0.setDistributedJoins(distributedJoin)
+
         if (args.nonEmpty)
             qry0.setArgs(args.map(_.asInstanceOf[Object]): _*)
 
         val ccfg = ic.ignite().cache[K, V](cacheName).getConfiguration(classOf[CacheConfiguration[K, V]])
 
-        if (ccfg.getCacheMode != CacheMode.REPLICATED)
-            qry0.setPartitions(partition.asInstanceOf[IgniteDataFramePartition].igniteParts: _*)
+        val ignitePartition = partition.asInstanceOf[IgniteDataFramePartition]
+
+        if (ccfg.getCacheMode != CacheMode.REPLICATED && ignitePartition.igniteParts.nonEmpty && !distributedJoin)
+            qry0.setPartitions(ignitePartition.igniteParts: _*)
 
         qry = qry0
 
@@ -76,7 +81,8 @@ object IgniteSQLDataFrameRDD {
         schema: StructType,
         qryStr: String,
         args: List[_],
-        parts: Array[Partition] = Array(IgnitePartition(0))) = {
-        new IgniteSQLDataFrameRDD(ic, cacheName, schema, qryStr, args, parts)
+        parts: Array[Partition] = Array(IgnitePartition(0)),
+        distributedJoin: Boolean = false): IgniteSQLDataFrameRDD[K, V] = {
+        new IgniteSQLDataFrameRDD[K, V](ic, cacheName, schema, qryStr, args, parts, distributedJoin)
     }
 }
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLRelation.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLRelation.scala
index 1fb8de73bb424..485ddf62ea34a 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLRelation.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/IgniteSQLRelation.scala
@@ -18,10 +18,8 @@
 package org.apache.ignite.spark.impl
 
 import org.apache.ignite.IgniteException
-import org.apache.ignite.cache.{CacheMode, QueryEntity}
-import org.apache.ignite.cluster.ClusterNode
-import org.apache.ignite.configuration.CacheConfiguration
-import org.apache.ignite.spark.{IgniteContext, IgniteRDD}
+import org.apache.ignite.cache.QueryEntity
+import org.apache.ignite.spark.{IgniteContext, IgniteRDD, impl}
 import org.apache.spark.Partition
 import org.apache.spark.internal.Logging
 import org.apache.spark.rdd.RDD
@@ -30,14 +28,13 @@ import org.apache.spark.sql.types._
 import org.apache.spark.sql.{Row, SQLContext}
 
 import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
 
 /**
   * Apache Ignite implementation of Spark BaseRelation with PrunedFilteredScan for Ignite SQL Tables
   */
 class IgniteSQLRelation[K, V](
-    private[spark] val ic: IgniteContext,
-    private[spark] val tableName: String)
+    private[apache] val ic: IgniteContext,
+    private[apache] val tableName: String)
     (@transient val sqlContext: SQLContext) extends BaseRelation with PrunedFilteredScan with Logging {
 
     /**
@@ -92,41 +89,13 @@ class IgniteSQLRelation[K, V](
         qryAndArgs
     }
 
-    private def calcPartitions(filters: Array[Filter]): Array[Partition] = {
-        val cache = ic.ignite().cache[K, V](cacheName)
-
-        val ccfg = cache.getConfiguration(classOf[CacheConfiguration[K, V]])
-
-        if (ccfg.getCacheMode == CacheMode.REPLICATED) {
-            val serverNodes = ic.ignite().cluster().forCacheNodes(cacheName).forServers().nodes()
-
-            Array(IgniteDataFramePartition(0, serverNodes.head, Stream.from(0).take(1024).toList))
-        }
-        else {
-            val aff = ic.ignite().affinity(cacheName)
-
-            val parts = aff.partitions()
-
-            val nodesToParts = (0 until parts).foldLeft(Map[ClusterNode, ArrayBuffer[Int]]()) {
-                case (nodeToParts, ignitePartIdx) ⇒
-                    val primary = aff.mapPartitionToPrimaryAndBackups(ignitePartIdx).head
-
-                    if (nodeToParts.contains(primary)) {
-                        nodeToParts(primary) += ignitePartIdx
-
-                        nodeToParts
-                    }
-                    else
-                        nodeToParts + (primary → ArrayBuffer[Int](ignitePartIdx))
-            }
-
-            val partitions = nodesToParts.zipWithIndex.map { case ((node, nodesParts), i) ⇒
-                IgniteDataFramePartition(i, node, nodesParts.toList)
-            }
-
-            partitions.toArray
-        }
-    }
+    /**
+      * Computes spark partitions for this relation.
+      *
+      * @return Array of IgniteDataFramPartition.
+      */
+    private def calcPartitions(filters: Array[Filter]): Array[Partition] =
+        impl.calcPartitions(ic, cacheName)
 
     /**
       * Cache name for a table name.
@@ -134,20 +103,6 @@ class IgniteSQLRelation[K, V](
     private lazy val cacheName: String =
         sqlCacheName(ic.ignite(), tableName)
             .getOrElse(throw new IgniteException(s"Unknown table $tableName"))
-
-    /**
-      * Utility method to add clause to sql WHERE string.
-      *
-      * @param filterStr Current filter string
-      * @param clause Clause to add.
-      * @return Filter string.
-      */
-    private def addStrClause(filterStr: String, clause: String) =
-        if (filterStr.isEmpty)
-            clause
-        else
-            filterStr + " AND " + clause
-
 }
 
 object IgniteSQLRelation {
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/AggregateExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/AggregateExpressions.scala
new file mode 100644
index 0000000000000..421a9a9447118
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/AggregateExpressions.scala
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.aggregate._
+import org.apache.spark.sql.catalyst.expressions.Expression
+import org.apache.spark.sql.types._
+
+/**
+  * Object to support aggregate expressions like `sum` or `avg`.
+  */
+private[optimization] object AggregateExpressions extends SupportedExpressions {
+    /** @inheritdoc */
+    def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean = expr match {
+        case AggregateExpression(aggregateFunction, _, _, _) ⇒
+            checkChild(aggregateFunction)
+
+        case Average(child) ⇒
+            checkChild(child)
+
+        case Count(children) ⇒
+            children.forall(checkChild)
+
+        case Max(child) ⇒
+            checkChild(child)
+
+        case Min(child) ⇒
+            checkChild(child)
+
+        case Sum(child) ⇒
+            checkChild(child)
+
+        case _ ⇒
+            false
+    }
+
+    /** @inheritdoc */
+    override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+        useAlias: Boolean): Option[String] = expr match {
+        case AggregateExpression(aggregateFunction, _, isDistinct, _) ⇒
+            aggregateFunction match {
+                case Count(children) ⇒
+                    if (isDistinct)
+                        Some(s"COUNT(DISTINCT ${children.map(childToString(_)).mkString(" ")})")
+                    else
+                        Some(s"COUNT(${children.map(childToString(_)).mkString(" ")})")
+
+                case sum: Sum ⇒
+                    if (isDistinct)
+                        Some(castSum(
+                            s"SUM(DISTINCT ${sum.children.map(childToString(_)).mkString(" ")})", sum.dataType))
+                    else
+                        Some(castSum(s"SUM(${sum.children.map(childToString(_)).mkString(" ")})", sum.dataType))
+
+                case _ ⇒
+                    Some(childToString(aggregateFunction))
+            }
+
+        case Average(child) ⇒
+            child.dataType match {
+                case DecimalType() | DoubleType ⇒
+                    Some(s"AVG(${childToString(child)})")
+
+                case _ ⇒
+                    //Spark `AVG` return type is always a double or a decimal.
+                    //See [[org.apache.spark.sql.catalyst.expressions.aggregate.Average]]
+                    //But Ignite `AVG` return type for a integral types is integral.
+                    //To preserve query correct results has to cast column to double.
+                    Some(s"AVG(CAST(${childToString(child)} AS DOUBLE))")
+            }
+
+
+        case Count(children) ⇒
+            Some(s"COUNT(${children.map(childToString(_)).mkString(" ")})")
+
+        case Max(child) ⇒
+            Some(s"MAX(${childToString(child)})")
+
+        case Min(child) ⇒
+            Some(s"MIN(${childToString(child)})")
+
+        case sum: Sum ⇒
+            Some(castSum(s"SUM(${childToString(sum.child)})", sum.dataType))
+
+        case _ ⇒
+            None
+    }
+
+    /**
+      * Ignite returns BigDecimal but Spark expects BIGINT.
+      */
+    private def castSum(sumSql: String, dataType: DataType): String = dataType match {
+        case LongType ⇒
+            s"CAST($sumSql AS BIGINT)"
+
+        case _ ⇒
+            s"$sumSql"
+    }
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/ConditionExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/ConditionExpressions.scala
new file mode 100644
index 0000000000000..fbfbd640bf7f5
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/ConditionExpressions.scala
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, _}
+
+/**
+  * Object to support condition expression. Like `and` or `in` operators.
+  */
+private[optimization] object ConditionExpressions extends SupportedExpressions {
+    /** @inheritdoc */
+    def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean = expr match {
+        case EqualTo(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case EqualNullSafe(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case GreaterThan(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case GreaterThanOrEqual(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case LessThan(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case LessThanOrEqual(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case InSet(child, set) if set.forall(_.isInstanceOf[Literal]) ⇒
+            checkChild(child)
+
+        case In(child, list) if list.forall(_.isInstanceOf[Literal]) ⇒
+            checkChild(child)
+
+        case IsNull(child) ⇒
+            checkChild(child)
+
+        case IsNotNull(child) ⇒
+            checkChild(child)
+
+        case And(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Or(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Not(child) ⇒
+            checkChild(child)
+
+        case StartsWith(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case EndsWith(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Contains(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case _ ⇒
+            false
+    }
+
+    /** @inheritdoc */
+    override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+        useAlias: Boolean): Option[String] = expr match {
+        case EqualTo(left, right) ⇒
+            Some(s"${childToString(left)} = ${childToString(right)}")
+
+        case EqualNullSafe(left, right) ⇒
+            Some(s"(${childToString(left)} IS NULL OR ${childToString(left)} = ${childToString(right)})")
+
+        case GreaterThan(left, right) ⇒
+            Some(s"${childToString(left)} > ${childToString(right)}")
+
+        case GreaterThanOrEqual(left, right) ⇒
+            Some(s"${childToString(left)} >= ${childToString(right)}")
+
+        case LessThan(left, right) ⇒
+            Some(s"${childToString(left)} < ${childToString(right)}")
+
+        case LessThanOrEqual(left, right) ⇒
+            Some(s"${childToString(left)} <= ${childToString(right)}")
+
+        case In(attr, values) ⇒
+            Some(s"${childToString(attr)} IN (${values.map(childToString(_)).mkString(", ")})")
+
+        case IsNull(child) ⇒
+            Some(s"${childToString(child)} IS NULL")
+
+        case IsNotNull(child) ⇒
+            Some(s"${childToString(child)} IS NOT NULL")
+
+        case And(left, right) ⇒
+            Some(s"${childToString(left)} AND ${childToString(right)}")
+
+        case Or(left, right) ⇒
+            Some(s"${childToString(left)} OR ${childToString(right)}")
+
+        case Not(child) ⇒
+            Some(s"NOT ${childToString(child)}")
+
+        case StartsWith(attr, value) ⇒ {
+            //Expecting string literal here.
+            //To add % sign it's required to remove quotes.
+            val valStr = removeQuotes(childToString(value))
+
+            Some(s"${childToString(attr)} LIKE '$valStr%'")
+        }
+
+        case EndsWith(attr, value) ⇒ {
+            //Expecting string literal here.
+            //To add % sign it's required to remove quotes.
+            val valStr = removeQuotes(childToString(value))
+
+            Some(s"${childToString(attr)} LIKE '%$valStr'")
+        }
+
+        case Contains(attr, value) ⇒ {
+            //Expecting string literal here.
+            //To add % signs it's required to remove quotes.
+            val valStr = removeQuotes(childToString(value))
+
+            Some(s"${childToString(attr)} LIKE '%$valStr%'")
+        }
+
+        case _ ⇒
+            None
+    }
+
+    /**
+      * @param str String to process.
+      * @return Str without surrounding quotes.
+      */
+    private def removeQuotes(str: String): String =
+        if (str.length < 2)
+            str
+        else
+            str match {
+                case quoted if quoted.startsWith("'") && quoted.endsWith("'") ⇒
+                    quoted.substring(1, quoted.length-1)
+
+                case _ ⇒ str
+            }
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/DateExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/DateExpressions.scala
new file mode 100644
index 0000000000000..d075bf0bcf689
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/DateExpressions.scala
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, _}
+
+/**
+  * Object to support expressions to work with date/timestamp.
+  */
+private[optimization] object DateExpressions extends SupportedExpressions {
+    /** @inheritdoc */
+    def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean = expr match {
+        case CurrentDate(None) ⇒
+            true
+
+        case CurrentTimestamp() ⇒
+            true
+
+        case DateAdd(startDate, days) ⇒
+            checkChild(startDate) && checkChild(days)
+
+        case DateDiff(date1, date2) ⇒
+            checkChild(date1) && checkChild(date2)
+
+        case DayOfMonth(date) ⇒
+            checkChild(date)
+
+        case DayOfYear(date) ⇒
+            checkChild(date)
+
+        case Hour(date, _) ⇒
+            checkChild(date)
+
+        case Minute(date, _) ⇒
+            checkChild(date)
+
+        case Month(date) ⇒
+            checkChild(date)
+
+        case ParseToDate(left, format, child) ⇒
+            checkChild(left) && (format.isEmpty || checkChild(format.get)) && checkChild(child)
+
+        case Quarter(date) ⇒
+            checkChild(date)
+
+        case Second(date, _) ⇒
+            checkChild(date)
+
+        case WeekOfYear(date) ⇒
+            checkChild(date)
+
+        case Year(date) ⇒
+            checkChild(date)
+
+        case _ ⇒
+            false
+    }
+
+    /** @inheritdoc */
+    override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+        useAlias: Boolean): Option[String] = expr match {
+        case CurrentDate(_) ⇒
+            Some(s"CURRENT_DATE()")
+
+        case CurrentTimestamp() ⇒
+            Some(s"CURRENT_TIMESTAMP()")
+
+        case DateAdd(startDate, days) ⇒
+            Some(s"CAST(DATEADD('DAY', ${childToString(days)}, ${childToString(startDate)}) AS DATE)")
+
+        case DateDiff(date1, date2) ⇒
+            Some(s"CAST(DATEDIFF('DAY', ${childToString(date1)}, ${childToString(date2)}) AS INT)")
+
+        case DayOfMonth(date) ⇒
+            Some(s"DAY_OF_MONTH(${childToString(date)})")
+
+        case DayOfYear(date) ⇒
+            Some(s"DAY_OF_YEAR(${childToString(date)})")
+
+        case Hour(date, _) ⇒
+            Some(s"HOUR(${childToString(date)})")
+
+        case Minute(date, _) ⇒
+            Some(s"MINUTE(${childToString(date)})")
+
+        case Month(date) ⇒
+            Some(s"MINUTE(${childToString(date)})")
+
+        case ParseToDate(left, formatOption, _) ⇒
+            formatOption match {
+                case Some(format) ⇒
+                    Some(s"PARSEDATETIME(${childToString(left)}, ${childToString(format)})")
+                case None ⇒
+                    Some(s"PARSEDATETIME(${childToString(left)})")
+            }
+
+        case Quarter(date) ⇒
+            Some(s"QUARTER(${childToString(date)})")
+
+        case Second(date, _) ⇒
+            Some(s"SECOND(${childToString(date)})")
+
+        case WeekOfYear(date) ⇒
+            Some(s"WEEK(${childToString(date)})")
+
+        case Year(date) ⇒
+            Some(s"YEAR(${childToString(date)})")
+
+        case _ ⇒
+            None
+    }
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/IgniteQueryContext.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/IgniteQueryContext.scala
new file mode 100644
index 0000000000000..c5a7f34c72700
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/IgniteQueryContext.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.ignite.spark.IgniteContext
+import org.apache.spark.sql.SQLContext
+import org.apache.spark.sql.catalyst.catalog.CatalogTable
+import org.apache.spark.sql.catalyst.expressions.NamedExpression
+
+/**
+  * Class to store Ignite query info during optimization process.
+  *
+  * @param igniteContext IgniteContext.
+  * @param sqlContext SQLContext.
+  * @param cacheName Cache name.
+  * @param aliasIndex Iterator to generate indexes for auto-generated aliases.
+  * @param catalogTable CatalogTable from source relation.
+  */
+case class IgniteQueryContext(
+    igniteContext: IgniteContext,
+    sqlContext: SQLContext,
+    cacheName: String,
+    aliasIndex: Iterator[Int],
+    catalogTable: Option[CatalogTable] = None,
+    distributeJoin: Boolean = false
+) {
+    /**
+      * @return Unique table alias.
+      */
+    def uniqueTableAlias: String = "table" + aliasIndex.next
+
+    /**
+      * @param col Column
+      * @return Unique column alias.
+      */
+    def uniqueColumnAlias(col: NamedExpression): String = col.name + "_" + aliasIndex.next
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/MathExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/MathExpressions.scala
new file mode 100644
index 0000000000000..dc05e95c090da
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/MathExpressions.scala
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, _}
+
+/**
+  * Object to support math expressions.
+  */
+private[optimization] object MathExpressions extends SupportedExpressions {
+    /** @inheritdoc */
+    def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean = expr match {
+        case Abs(child) ⇒
+            checkChild(child)
+
+        case Acos(child) ⇒
+            checkChild(child)
+
+        case Asin(child) ⇒
+            checkChild(child)
+
+        case Atan(child) ⇒
+            checkChild(child)
+
+        case Cos(child) ⇒
+            checkChild(child)
+
+        case Cosh(child) ⇒
+            checkChild(child)
+
+        case Sin(child) ⇒
+            checkChild(child)
+
+        case Sinh(child) ⇒
+            checkChild(child)
+
+        case Tan(child) ⇒
+            checkChild(child)
+
+        case Tanh(child) ⇒
+            checkChild(child)
+
+        case Atan2(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case BitwiseAnd(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case BitwiseOr(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case BitwiseXor(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Ceil(child) ⇒
+            checkChild(child)
+
+        case ToDegrees(child) ⇒
+            checkChild(child)
+
+        case Exp(child) ⇒
+            checkChild(child)
+
+        case Floor(child) ⇒
+            checkChild(child)
+
+        case Log(child) ⇒
+            checkChild(child)
+
+        case Log10(child) ⇒
+            checkChild(child)
+
+        case Logarithm(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case ToRadians(child) ⇒
+            checkChild(child)
+
+        case Sqrt(child) ⇒
+            checkChild(child)
+
+        case _: Pi ⇒
+            true
+
+        case _: EulerNumber ⇒
+            true
+
+        case Pow(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Rand(child) ⇒
+            checkChild(child)
+
+        case Round(child, scale) ⇒
+            checkChild(child) && checkChild(scale)
+
+        case Signum(child) ⇒
+            checkChild(child)
+
+        case Remainder(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Divide(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Multiply(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Subtract(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Add(left, right) ⇒
+            checkChild(left) && checkChild(right)
+
+        case UnaryMinus(child) ⇒
+            checkChild(child)
+
+        case UnaryPositive(child) ⇒
+            checkChild(child)
+
+        case _ ⇒ false
+    }
+
+    /** @inheritdoc */
+    override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+        useAlias: Boolean): Option[String] = expr match {
+        case Abs(child) ⇒
+            Some(s"ABS(${childToString(child)})")
+
+        case Acos(child) ⇒
+            Some(s"ACOS(${childToString(child)})")
+
+        case Asin(child) ⇒
+            Some(s"ASIN(${childToString(child)})")
+
+        case Atan(child) ⇒
+            Some(s"ATAN(${childToString(child)})")
+
+        case Cos(child) ⇒
+            Some(s"COS(${childToString(child)})")
+
+        case Cosh(child) ⇒
+            Some(s"COSH(${childToString(child)})")
+
+        case Sin(child) ⇒
+            Some(s"SIN(${childToString(child)})")
+
+        case Sinh(child) ⇒
+            Some(s"SINH(${childToString(child)})")
+
+        case Tan(child) ⇒
+            Some(s"TAN(${childToString(child)})")
+
+        case Tanh(child) ⇒
+            Some(s"TANH(${childToString(child)})")
+
+        case Atan2(left, right) ⇒
+            Some(s"ATAN2(${childToString(left)}, ${childToString(right)})")
+
+        case BitwiseAnd(left, right) ⇒
+            Some(s"BITAND(${childToString(left)}, ${childToString(right)})")
+
+        case BitwiseOr(left, right) ⇒
+            Some(s"BITOR(${childToString(left)}, ${childToString(right)})")
+
+        case BitwiseXor(left, right) ⇒
+            Some(s"BITXOR(${childToString(left)}, ${childToString(right)})")
+
+        case Ceil(child) ⇒
+            Some(s"CAST(CEIL(${childToString(child)}) AS LONG)")
+
+        case ToDegrees(child) ⇒
+            Some(s"DEGREES(${childToString(child)})")
+
+        case Exp(child) ⇒
+            Some(s"EXP(${childToString(child)})")
+
+        case Floor(child) ⇒
+            Some(s"CAST(FLOOR(${childToString(child)}) AS LONG)")
+
+        case Log(child) ⇒
+            Some(s"LOG(${childToString(child)})")
+
+        case Log10(child) ⇒
+            Some(s"LOG10(${childToString(child)})")
+
+        case Logarithm(base, arg) ⇒
+            childToString(base) match {
+                //Spark internally converts LN(XXX) to LOG(2.718281828459045, XXX).
+                //Because H2 doesn't have builtin function for a free base logarithm
+                //I want to prevent usage of log(a, b) = ln(a)/ln(b) when possible.
+                case "2.718281828459045" ⇒
+                    Some(s"LOG(${childToString(arg)})")
+                case "10" ⇒
+                    Some(s"LOG10(${childToString(arg)})")
+                case argStr ⇒
+                    Some(s"(LOG(${childToString(arg)})/LOG($argStr))")
+            }
+
+        case ToRadians(child) ⇒
+            Some(s"RADIANS(${childToString(child)})")
+
+        case Sqrt(child) ⇒
+            Some(s"SQRT(${childToString(child)})")
+
+        case _: Pi ⇒
+            Some("PI()")
+
+        case _: EulerNumber ⇒
+            Some("E()")
+
+        case Pow(left, right) ⇒
+            Some(s"POWER(${childToString(left)}, ${childToString(right)})")
+
+        case Rand(child) ⇒
+            Some(s"RAND(${childToString(child)})")
+
+        case Round(child, scale) ⇒
+            Some(s"ROUND(${childToString(child)}, ${childToString(scale)})")
+
+        case Signum(child) ⇒
+            Some(s"SIGN(${childToString(child)})")
+
+        case Remainder(left, right) ⇒
+            Some(s"${childToString(left)} % ${childToString(right)}")
+
+        case Divide(left, right) ⇒
+            Some(s"${childToString(left)} / ${childToString(right)}")
+
+        case Multiply(left, right) ⇒
+            Some(s"${childToString(left)} * ${childToString(right)}")
+
+        case Subtract(left, right) ⇒
+            Some(s"${childToString(left)} - ${childToString(right)}")
+
+        case Add(left, right) ⇒
+            Some(s"${childToString(left)} + ${childToString(right)}")
+
+        case UnaryMinus(child) ⇒
+            Some(s"-${childToString(child)}")
+
+        case UnaryPositive(child) ⇒
+            Some(s"+${childToString(child)}")
+
+        case _ ⇒
+            None
+    }
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
new file mode 100644
index 0000000000000..a1c945843ed58
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
@@ -0,0 +1,180 @@
+package org.apache.ignite.spark.impl.optimization
+
+import java.text.SimpleDateFormat
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, _}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
+import org.apache.spark.sql.types._
+
+/**
+  * Object to support some 'simple' expressions like aliases.
+  */
+private[optimization] object SimpleExpressions extends SupportedExpressions {
+    /** @inheritdoc */
+    override def apply(expr: Expression, checkChild: Expression ⇒ Boolean): Boolean = expr match {
+        case Literal(_, _) ⇒
+            true
+
+        case _: Attribute ⇒
+            true
+
+        case Alias(child, _) ⇒
+            checkChild(child)
+
+        case Cast(child, dataType, _) ⇒
+            checkChild(child) && castSupported(from = child.dataType, to = dataType)
+
+        case _ ⇒
+            false
+    }
+
+    /** @inheritdoc */
+    override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+        useAlias: Boolean): Option[String] = expr match {
+        case l: Literal ⇒ l.dataType match {
+            case StringType ⇒
+                Some("'" + l.value.toString + "'")
+
+            case TimestampType ⇒
+                l.value match {
+                    //Internal representation of TimestampType is Long.
+                    //So we converting from internal spark representation to CAST call.
+                    case date: Long ⇒
+                        Some(s"CAST('${timestampFormat.get.format(DateTimeUtils.toJavaTimestamp(date))}' AS TIMESTAMP)")
+
+                    case _ ⇒
+                        Some(l.value.toString)
+                }
+
+            case DateType ⇒
+                l.value match {
+                    //Internal representation of DateType is Int.
+                    //So we converting from internal spark representation to CAST call.
+                    case days: Integer ⇒
+                        val date = new java.util.Date(DateTimeUtils.daysToMillis(days))
+
+                        Some(s"CAST('${dateFormat.get.format(date)}' AS DATE)")
+
+                    case _ ⇒
+                        Some(l.value.toString)
+                }
+
+            case _ ⇒
+                if (l.value == null)
+                    Some("null")
+                else
+                    Some(l.value.toString)
+        }
+
+        case ar: AttributeReference ⇒
+            val name =
+                if (useQualifier)
+                    ar.qualifier.map(_ + "." + ar.name).getOrElse(ar.name)
+                else
+                    ar.name
+
+            if (ar.metadata.contains(ALIAS) && !isAliasEqualColumnName(ar.metadata.getString(ALIAS), ar.name) && useAlias)
+                Some(aliasToString(name, ar.metadata.getString(ALIAS)))
+            else
+                Some(name)
+
+        case Alias(child, name) ⇒
+            if (useAlias)
+                Some(childToString(child)).map(aliasToString(_, name))
+            else
+                Some(childToString(child))
+
+        case Cast(child, dataType, _) ⇒
+            Some(s"CAST(${childToString(child)} AS ${toSqlType(dataType)})")
+
+        case SortOrder(child, direction, _, _) ⇒
+            Some(s"${childToString(child)}${if(direction==Descending) " DESC" else ""}")
+
+        case _ ⇒
+            None
+    }
+
+    /**
+      * @param column Column name.
+      * @param alias Alias.
+      * @return SQL String for column with alias.
+      */
+    private def aliasToString(column: String, alias: String): String =
+        if (isAliasEqualColumnName(alias, column))
+            column
+        else if (alias.matches("[A-Za-z_][0-9A-Za-z_]*"))
+            s"$column AS $alias"
+        else
+            s"""$column AS "$alias""""
+
+    /**
+      * @param alias Alias.
+      * @param column Column.
+      * @return True if name equals to alias, false otherwise.
+      */
+    private def isAliasEqualColumnName(alias: String, column: String): Boolean =
+        alias.compareToIgnoreCase(column.replaceAll("'", "")) == 0
+
+    /**
+      * @param from From type conversion.
+      * @param to To type conversion.
+      * @return True if cast support for types, false otherwise.
+      */
+    private def castSupported(from: DataType, to: DataType): Boolean = from match {
+        case BooleanType ⇒
+            Set[DataType](BooleanType, StringType)(to)
+
+        case ByteType ⇒
+            Set(ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType, StringType, DecimalType(_, _), StringType)(to)
+
+        case ShortType ⇒
+            Set(ShortType, IntegerType, LongType, FloatType, DoubleType, StringType, DecimalType(_, _))(to)
+
+        case IntegerType ⇒
+            Set(IntegerType, LongType, FloatType, DoubleType, StringType, DecimalType(_, _))(to)
+
+        case LongType ⇒
+            Set(LongType, FloatType, DoubleType, StringType, DecimalType(_, _))(to)
+
+        case FloatType ⇒
+            Set(FloatType, DoubleType, StringType, DecimalType(_, _))(to)
+
+        case DoubleType ⇒
+            Set(DoubleType, StringType, DecimalType(_, _))(to)
+
+        case DecimalType() ⇒
+            Set(StringType, DecimalType(_, _))(to)
+
+        case DateType ⇒
+            Set[DataType](DateType, StringType, LongType, TimestampType)(to)
+
+        case TimestampType ⇒
+            Set[DataType](TimestampType, DateType, StringType, LongType)(to)
+
+        case StringType ⇒
+            Set(BooleanType, ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType,
+                DecimalType(_, _), DateType, TimestampType, StringType)(to)
+
+        case BinaryType ⇒
+            false
+
+        case ArrayType(_, _) ⇒
+            false
+    }
+
+    /**
+      * Date format built-in Ignite.
+      */
+    private val dateFormat: ThreadLocal[SimpleDateFormat] = new ThreadLocal[SimpleDateFormat] {
+        override def initialValue(): SimpleDateFormat =
+            new SimpleDateFormat("yyyy-MM-dd")
+    }
+
+    /**
+      * Timestamp format built-in Ignite.
+      */
+    private val timestampFormat: ThreadLocal[SimpleDateFormat] = new ThreadLocal[SimpleDateFormat] {
+        override def initialValue(): SimpleDateFormat =
+            new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS")
+    }
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/StringExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/StringExpressions.scala
new file mode 100644
index 0000000000000..1ecab2ca0d16b
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/StringExpressions.scala
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.{Expression, _}
+
+/**
+  * Object to support expressions to work with strings like `length` or `trim`.
+  */
+private[optimization] object StringExpressions extends SupportedExpressions {
+    /** @inheritdoc */
+    def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean = expr match {
+        case Ascii(child) ⇒
+            checkChild(child)
+
+        case Length(child) ⇒
+            checkChild(child)
+
+        case Concat(children) ⇒
+            children.forall(checkChild)
+
+        case ConcatWs(children) ⇒
+            children.forall(checkChild)
+
+        case StringInstr(str, substr) ⇒
+            checkChild(str) && checkChild(substr)
+
+        case Lower(child) ⇒
+            checkChild(child)
+
+        case Upper(child) ⇒
+            checkChild(child)
+
+        case StringLocate(substr, str, start) ⇒
+            checkChild(substr) && checkChild(str) && checkChild(start)
+
+        case StringLPad(str, len, pad) ⇒
+            checkChild(str) && checkChild(len) && checkChild(pad)
+
+        case StringRPad(str, len, pad) ⇒
+            checkChild(str) && checkChild(len) && checkChild(pad)
+
+        case StringTrimLeft(child) ⇒
+            checkChild(child)
+
+        case StringTrimRight(child) ⇒
+            checkChild(child)
+
+        case StringTrim(child) ⇒
+            checkChild(child)
+
+        case RegExpReplace(subject, regexp, rep) ⇒
+            checkChild(subject) && checkChild(regexp) && checkChild(rep)
+
+        case StringRepeat(str, times) ⇒
+            checkChild(str) && checkChild(times)
+
+        case SoundEx(child) ⇒
+            checkChild(child)
+
+        case StringSpace(child) ⇒
+            checkChild(child)
+
+        case Substring(str, pos, len) ⇒
+            checkChild(str) && checkChild(pos) && checkChild(len)
+
+        case Substring(str, pos, len) ⇒
+            checkChild(str) && checkChild(pos) && checkChild(len)
+
+        case StringTranslate(str, strMatch, strReplace) ⇒
+            checkChild(str) && checkChild(strMatch) && checkChild(strReplace)
+
+        case _ ⇒ false
+    }
+
+    /** @inheritdoc */
+    override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+        useAlias: Boolean): Option[String] = expr match {
+        case Ascii(child) ⇒
+            Some(s"ASCII(${childToString(child)})")
+
+        case Length(child) ⇒
+            Some(s"CAST(LENGTH(${childToString(child)}) AS INTEGER)")
+
+        case Concat(children) ⇒
+            Some(s"CONCAT(${children.map(childToString(_)).mkString(", ")})")
+
+        case ConcatWs(children) ⇒
+            Some(s"CONCAT_WS(${children.map(childToString(_)).mkString(", ")})")
+
+        case StringInstr(str, substr) ⇒
+            Some(s"POSITION(${childToString(substr)}, ${childToString(str)})")
+
+        case Lower(child) ⇒
+            Some(s"LOWER(${childToString(child)})")
+
+        case Upper(child) ⇒
+            Some(s"UPPER(${childToString(child)})")
+
+        case StringLocate(substr, str, start) ⇒
+            Some(s"LOCATE(${childToString(substr)}, ${childToString(str)}, ${childToString(start)})")
+
+        case StringLPad(str, len, pad) ⇒
+            Some(s"LPAD(${childToString(str)}, ${childToString(len)}, ${childToString(pad)})")
+
+        case StringRPad(str, len, pad) ⇒
+            Some(s"RPAD(${childToString(str)}, ${childToString(len)}, ${childToString(pad)})")
+
+        case StringTrimLeft(child) ⇒
+            Some(s"LTRIM(${childToString(child)})")
+
+        case StringTrimRight(child) ⇒
+            Some(s"RTRIM(${childToString(child)})")
+
+        case StringTrim(child) ⇒
+            Some(s"TRIM(${childToString(child)})")
+
+        case RegExpReplace(subject, regexp, rep) ⇒
+            Some(s"REGEXP_REPLACE(${childToString(subject)}, ${childToString(regexp)}, ${childToString(rep)})")
+
+        case StringRepeat(str, times) ⇒
+            Some(s"REPEAT(${childToString(str)}, ${childToString(times)})")
+
+        case SoundEx(child) ⇒
+            Some(s"SOUND_EX(${childToString(child)})")
+
+        case StringSpace(child) ⇒
+            Some(s"SPACE(${childToString(child)})")
+
+        case Substring(str, pos, len) ⇒
+            Some(s"SUBSTR(${childToString(str)}, ${childToString(pos)}, ${childToString(len)})")
+
+        case StringTranslate(str, strMatch, strReplace) ⇒
+            Some(s"TRANSLATE(${childToString(str)}, ${childToString(strMatch)}, ${childToString(strReplace)})")
+
+        case _ ⇒
+            None
+    }
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SupportedExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SupportedExpressions.scala
new file mode 100644
index 0000000000000..f46eb72bb1adc
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SupportedExpressions.scala
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+/**
+  * Provides methods to work with Spark SQL expression that supported by Ignite SQL syntax.
+  */
+private[optimization] trait SupportedExpressions {
+    /**
+      * @param expr Expression to check.
+      * @param checkChild Closure to check child expression.
+      * @return True if `expr` are supported, false otherwise.
+      */
+    def apply(expr: Expression, checkChild: (Expression) ⇒ Boolean): Boolean
+
+    /**
+      * @param expr Expression to convert to string.
+      * @param childToString Closure to convert children expressions.
+      * @param useQualifier If true `expr` should be printed using qualifier. `Table1.id` for example.
+      * @param useAlias If true `expr` should be printed with alias. `name as person_name` for example.
+      * @return SQL representation of `expr` if it supported. `None` otherwise.
+      */
+    def toString(expr: Expression, childToString: (Expression) ⇒ String, useQualifier: Boolean,
+        useAlias: Boolean): Option[String]
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SystemExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SystemExpressions.scala
new file mode 100644
index 0000000000000..40e4e290659ff
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SystemExpressions.scala
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization
+
+import org.apache.ignite.IgniteException
+import org.apache.spark.sql.catalyst.expressions.{Coalesce, EqualTo, Expression, Greatest, If, IfNull, IsNotNull, IsNull, Least, Literal, NullIf, Nvl2}
+
+/**
+  * Object to support some built-in expressions like `nvl2` or `coalesce`.
+  */
+private[optimization] object SystemExpressions extends SupportedExpressions {
+    /** @inheritdoc */
+    override def apply(expr: Expression, checkChild: Expression ⇒ Boolean): Boolean = expr match {
+        case Coalesce(children) ⇒
+            children.forall(checkChild)
+
+        case Greatest(children) ⇒
+            children.forall(checkChild)
+
+        case IfNull(left, right, _) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Least(children) ⇒
+            children.forall(checkChild)
+
+        case NullIf(left, right, _) ⇒
+            checkChild(left) && checkChild(right)
+
+        case Nvl2(expr1, expr2, expr3, _) ⇒
+            checkChild(expr1) && checkChild(expr2) && checkChild(expr3)
+
+        case If(predicate, trueValue, falseValue) ⇒
+            predicate match {
+                case IsNotNull(child) ⇒
+                    checkChild(child) && checkChild(trueValue) && checkChild(falseValue)
+
+                case IsNull(child) ⇒
+                    checkChild(child) && checkChild(trueValue) && checkChild(falseValue)
+
+                case EqualTo(left, right) ⇒
+                    trueValue match {
+                        case Literal(null, _) ⇒
+                            (left == falseValue || right == falseValue) && checkChild(left) && checkChild(right)
+
+                        case _ ⇒
+                            false
+                    }
+
+                case _ ⇒
+                    false
+            }
+
+        case _ ⇒
+            false
+    }
+
+    /** @inheritdoc */
+    override def toString(expr: Expression, childToString: Expression ⇒ String, useQualifier: Boolean,
+        useAlias: Boolean): Option[String] = expr match {
+        case Coalesce(children) ⇒
+            Some(s"COALESCE(${children.map(childToString(_)).mkString(", ")})")
+
+        case Greatest(children) ⇒
+            Some(s"GREATEST(${children.map(childToString(_)).mkString(", ")})")
+
+        case IfNull(left, right, _) ⇒
+            Some(s"IFNULL(${childToString(left)}, ${childToString(right)})")
+
+        case Least(children) ⇒
+            Some(s"LEAST(${children.map(childToString(_)).mkString(", ")})")
+
+        case NullIf(left, right, _) ⇒
+            Some(s"NULLIF(${childToString(left)}, ${childToString(right)})")
+
+        case Nvl2(expr1, expr2, expr3, _) ⇒
+            Some(s"NVL2(${childToString(expr1)}, ${childToString(expr2)}, ${childToString(expr3)})")
+
+        case If(predicate, trueValue, falseValue) ⇒
+            predicate match {
+                case IsNotNull(child) ⇒
+                    Some(s"NVL2(${childToString(child)}, ${childToString(trueValue)}, ${childToString(falseValue)})")
+
+                case IsNull(child) ⇒
+                    Some(s"NVL2(${childToString(child)}, ${childToString(falseValue)}, ${childToString(trueValue)})")
+
+                case EqualTo(left, right) ⇒
+                    trueValue match {
+                        case Literal(null, _) ⇒
+                            if (left == falseValue)
+                                Some(s"NULLIF(${childToString(left)}, ${childToString(right)})")
+                            else if (right == falseValue)
+                                Some(s"NULLIF(${childToString(right)}, ${childToString(left)})")
+                            else
+                                throw new IgniteException(s"Expression not supported. $expr")
+
+                        case _ ⇒
+                            throw new IgniteException(s"Expression not supported. $expr")
+                    }
+
+                case _ ⇒
+                    throw new IgniteException(s"Expression not supported. $expr")
+            }
+
+        case _ ⇒
+            None
+    }
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/JoinSQLAccumulator.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/JoinSQLAccumulator.scala
new file mode 100644
index 0000000000000..7ae5e70a994be
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/JoinSQLAccumulator.scala
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization.accumulator
+
+import org.apache.ignite.IgniteException
+import org.apache.ignite.spark.impl.optimization._
+import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression, NamedExpression, SortOrder}
+import org.apache.spark.sql.catalyst.plans.logical.BinaryNode
+import org.apache.spark.sql.catalyst.plans.{Inner, JoinType, LeftOuter, RightOuter}
+
+/**
+  * Accumulator to store information about join query.
+  */
+private[apache] case class JoinSQLAccumulator(
+    igniteQueryContext: IgniteQueryContext,
+    left: QueryAccumulator,
+    right: QueryAccumulator,
+    joinType: JoinType,
+    outputExpressions: Seq[NamedExpression],
+    condition: Option[Expression],
+    leftAlias: Option[String],
+    rightAlias: Option[String],
+    distinct: Boolean = false,
+    where: Option[Seq[Expression]] = None,
+    groupBy: Option[Seq[Expression]] = None,
+    having: Option[Seq[Expression]] = None,
+    limit: Option[Expression] = None,
+    localLimit: Option[Expression] = None,
+    orderBy: Option[Seq[SortOrder]] = None
+) extends BinaryNode with SelectAccumulator {
+    /** @inheritdoc */
+    override def compileQuery(prettyPrint: Boolean = false): String = {
+        val delim = if (prettyPrint) "\n" else " "
+        val tab = if (prettyPrint) "  " else ""
+
+        var sql = s"SELECT$delim$tab" +
+            s"${fixQualifier(outputExpressions).map(exprToString(_, useQualifier = true)).mkString(", ")}$delim" +
+            s"FROM$delim$tab$compileJoinExpr"
+
+        if (allFilters.nonEmpty)
+            sql += s"${delim}WHERE$delim$tab" +
+                s"${fixQualifier(allFilters).map(exprToString(_, useQualifier = true)).mkString(s" AND$delim$tab")}"
+
+        if (groupBy.exists(_.nonEmpty))
+            sql += s"${delim}GROUP BY " +
+                s"${fixQualifier(groupBy.get).map(exprToString(_, useQualifier = true)).mkString(s",$delim$tab")}"
+
+        if (having.exists(_.nonEmpty))
+            sql += s"${delim}HAVING " +
+                s"${fixQualifier(having.get).map(exprToString(_, useQualifier = true)).mkString(s" AND$delim$tab")}"
+
+        if (orderBy.exists(_.nonEmpty))
+            sql += s"${delim}ORDER BY " +
+                s"${fixQualifier(orderBy.get).map(exprToString(_, useQualifier = true)).mkString(s",$delim$tab")}"
+
+        if (limit.isDefined)
+            sql += s" LIMIT ${exprToString(fixQualifier0(limit.get), useQualifier = true)}"
+
+        sql
+    }
+
+    /**
+      * @return Filters for this query.
+      */
+    private def allFilters: Seq[Expression] = {
+        val leftFilters =
+            if (isSimpleTableAcc(left))
+                left.asInstanceOf[SingleTableSQLAccumulator].where.getOrElse(Seq.empty)
+            else
+                Seq.empty
+
+        val rightFilters =
+            if (isSimpleTableAcc(right))
+                right.asInstanceOf[SingleTableSQLAccumulator].where.getOrElse(Seq.empty)
+            else Seq.empty
+
+        where.getOrElse(Seq.empty) ++ leftFilters ++ rightFilters
+    }
+
+    /**
+      * @return `table1 LEFT JOIN (SELECT....FROM...) table2` part of join query.
+      */
+    private def compileJoinExpr: String = {
+        val leftJoinSql =
+            if (isSimpleTableAcc(left))
+                left.asInstanceOf[SingleTableSQLAccumulator].table.get
+            else
+                s"(${left.compileQuery()}) ${leftAlias.get}"
+
+        val rightJoinSql = {
+            val leftTableName =
+                if (isSimpleTableAcc(left))
+                    left.qualifier
+                else
+                    leftAlias.get
+
+            if (isSimpleTableAcc(right)) {
+                val rightTableName = right.asInstanceOf[SingleTableSQLAccumulator].table.get
+
+                if (leftTableName == rightTableName)
+                    s"$rightTableName as ${rightAlias.get}"
+                else
+                    rightTableName
+            } else
+                s"(${right.compileQuery()}) ${rightAlias.get}"
+        }
+
+        s"$leftJoinSql $joinTypeSQL $rightJoinSql" +
+            s"${condition.map(expr ⇒ s" ON ${exprToString(fixQualifier0(expr), useQualifier = true)}").getOrElse("")}"
+    }
+
+    /**
+      * @return SQL string representing specific join type.
+      */
+    private def joinTypeSQL = joinType match {
+        case Inner ⇒
+            "JOIN"
+        case LeftOuter ⇒
+            "LEFT JOIN"
+
+        case RightOuter ⇒
+            "RIGHT JOIN"
+
+        case _ ⇒
+            throw new IgniteException(s"Unsupported join type $joinType")
+    }
+
+    /**
+      * Changes table qualifier in case of embedded query.
+      *
+      * @param exprs Expressions to fix.
+      * @tparam T type of input expression.
+      * @return copy of `exprs` with fixed qualifier.
+      */
+    private def fixQualifier[T <: Expression](exprs: Seq[T]): Seq[T] =
+        exprs.map(fixQualifier0)
+
+    /**
+      * Changes table qualifier for single expression.
+      *
+      * @param expr Expression to fix.
+      * @tparam T type of input expression.
+      * @return copy of `expr` with fixed qualifier.
+      */
+    private def fixQualifier0[T <: Expression](expr: T): T = expr match {
+        case attr: AttributeReference ⇒
+            attr.withQualifier(Some(findQualifier(attr))).asInstanceOf[T]
+
+        case _ ⇒
+            expr.withNewChildren(fixQualifier(expr.children)).asInstanceOf[T]
+    }
+
+    /**
+      * Find right qualifier for a `attr`.
+      *
+      * @param attr Attribute to fix qualifier in
+      * @return Right qualifier for a `attr`
+      */
+    private def findQualifier(attr: AttributeReference): String = {
+        val leftTableName =
+            if (isSimpleTableAcc(left))
+                left.qualifier
+            else
+                leftAlias.get
+
+        if (left.outputExpressions.exists(_.exprId == attr.exprId))
+            leftTableName
+        else if (isSimpleTableAcc(right) && right.qualifier != leftTableName)
+            right.qualifier
+        else
+            rightAlias.get
+    }
+
+    /** @inheritdoc */
+    override def simpleString: String =
+        s"JoinSQLAccumulator(joinType: $joinType, columns: $outputExpressions, condition: $condition)"
+
+    /** @inheritdoc */
+    override def withOutputExpressions(outputExpressions: Seq[NamedExpression]): SelectAccumulator = copy(outputExpressions= outputExpressions)
+
+    /** @inheritdoc */
+    override def withDistinct(distinct: Boolean): JoinSQLAccumulator = copy(distinct = distinct)
+
+    /** @inheritdoc */
+    override def withWhere(where: Seq[Expression]): JoinSQLAccumulator = copy(where = Some(where))
+
+    /** @inheritdoc */
+    override def withGroupBy(groupBy: Seq[Expression]): JoinSQLAccumulator = copy(groupBy = Some(groupBy))
+
+    /** @inheritdoc */
+    override def withHaving(having: Seq[Expression]): JoinSQLAccumulator = copy(having = Some(having))
+
+    /** @inheritdoc */
+    override def withLimit(limit: Expression): JoinSQLAccumulator = copy(limit = Some(limit))
+
+    /** @inheritdoc */
+    override def withLocalLimit(localLimit: Expression): JoinSQLAccumulator = copy(localLimit = Some(localLimit))
+
+    /** @inheritdoc */
+    override def withOrderBy(orderBy: Seq[SortOrder]): JoinSQLAccumulator = copy(orderBy = Some(orderBy))
+
+    /** @inheritdoc */
+    override def output: Seq[Attribute] = outputExpressions.map(toAttributeReference(_, Seq.empty))
+
+    /** @inheritdoc */
+    override lazy val qualifier: String = igniteQueryContext.uniqueTableAlias
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/QueryAccumulator.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/QueryAccumulator.scala
new file mode 100644
index 0000000000000..133d355f9a644
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/QueryAccumulator.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization.accumulator
+
+import org.apache.ignite.spark.impl.optimization.IgniteQueryContext
+import org.apache.spark.sql.catalyst.expressions.{NamedExpression, SortOrder}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+/**
+  * Generic query info accumulator interface.
+  */
+private[apache] trait QueryAccumulator extends LogicalPlan {
+    /**
+      * @return Ignite query context.
+      */
+    def igniteQueryContext: IgniteQueryContext
+
+    /**
+      * @return Generated output.
+      */
+    def outputExpressions: Seq[NamedExpression]
+
+    /**
+      * @return Ordering info.
+      */
+    def orderBy: Option[Seq[SortOrder]]
+
+    /**
+      * @param outputExpressions New output expressions.
+      * @return Copy of this accumulator with new output.
+      */
+    def withOutputExpressions(outputExpressions: Seq[NamedExpression]): QueryAccumulator
+
+    /**
+      * @param orderBy New ordering.
+      * @return Copy of this accumulator with new order.
+      */
+    def withOrderBy(orderBy: Seq[SortOrder]): QueryAccumulator
+
+    /**
+      * @param prettyPrint If true human readable query will be generated.
+      * @return SQL query.
+      */
+    def compileQuery(prettyPrint: Boolean = false): String
+
+    /**
+      * @return Qualifier that should be use to select data from this accumulator.
+      */
+    def qualifier: String
+
+    /**
+      * All expressions are resolved when extra optimization executed.
+      */
+    override lazy val resolved = true
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SelectAccumulator.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SelectAccumulator.scala
new file mode 100644
index 0000000000000..c1db6f9fd31f3
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SelectAccumulator.scala
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization.accumulator
+
+import org.apache.spark.sql.catalyst.expressions.Expression
+
+/**
+  * Generic interface for a SELECT query.
+  */
+private[apache] trait SelectAccumulator extends QueryAccumulator {
+    /**
+      * @return Expression for HAVING part of query.
+      */
+    def having: Option[Seq[Expression]]
+
+    /**
+      * @return Expression for WHERE part of query.
+      */
+    def where: Option[Seq[Expression]]
+
+    /**
+      * @return Expression for GROUP BY part of query.
+      */
+    def groupBy: Option[Seq[Expression]]
+
+    /**
+      * @return Copy of this accumulator with `distinct` flag.
+      */
+    def withDistinct(distinct: Boolean): SelectAccumulator
+
+    /**
+      * @return Copy of this accumulator with `where` expressions.
+      */
+    def withWhere(where: Seq[Expression]): SelectAccumulator
+
+    /**
+      * @return Copy of this accumulator with `groupBy` expressions.
+      */
+    def withGroupBy(groupBy: Seq[Expression]): SelectAccumulator
+
+    /**
+      * @return Copy of this accumulator with `having` expressions.
+      */
+    def withHaving(having: Seq[Expression]): SelectAccumulator
+
+    /**
+      * @return Copy of this accumulator with `limit` expression.
+      */
+    def withLimit(limit: Expression): SelectAccumulator
+
+    /**
+      * @return Copy of this accumulator with `localLimit` expression.
+      */
+    def withLocalLimit(localLimit: Expression): SelectAccumulator
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SingleTableSQLAccumulator.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SingleTableSQLAccumulator.scala
new file mode 100644
index 0000000000000..47035b9626e1d
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/SingleTableSQLAccumulator.scala
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization.accumulator
+
+import org.apache.ignite.IgniteException
+import org.apache.ignite.spark.impl.optimization._
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, NamedExpression, SortOrder}
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+/**
+  * Class for accumulating parts of SQL query to a single Ignite table.
+  *
+  * See <a href="http://www.h2database.com/html/grammar.html#select">select syntax of H2</a>.
+  */
+private[apache] case class SingleTableSQLAccumulator(
+    igniteQueryContext: IgniteQueryContext,
+    table: Option[String],
+    tableExpression: Option[(QueryAccumulator, String)],
+    outputExpressions: Seq[NamedExpression],
+    distinct: Boolean = false,
+    all: Boolean = false,
+    where: Option[Seq[Expression]] = None,
+    groupBy: Option[Seq[Expression]] = None,
+    having: Option[Seq[Expression]] = None,
+    limit: Option[Expression] = None,
+    localLimit: Option[Expression] = None,
+    orderBy: Option[Seq[SortOrder]] = None
+) extends SelectAccumulator {
+    /** @inheritdoc */
+    override def compileQuery(prettyPrint: Boolean = false): String = {
+        val delim = if (prettyPrint) "\n" else " "
+        val tab = if (prettyPrint) "  " else ""
+
+        var sql = s"SELECT$delim$tab${outputExpressions.map(exprToString(_)).mkString(", ")}${delim}" +
+            s"FROM$delim$tab$compiledTableExpression"
+
+        if (where.exists(_.nonEmpty))
+            sql += s"${delim}WHERE$delim$tab${where.get.map(exprToString(_)).mkString(s" AND$delim$tab")}"
+
+        if (groupBy.exists(_.nonEmpty))
+            sql += s"${delim}GROUP BY ${groupBy.get.map(exprToString(_)).mkString(s",$delim$tab")}"
+
+        if (having.exists(_.nonEmpty))
+            sql += s"${delim}HAVING ${having.get.map(exprToString(_)).mkString(s" AND$delim$tab")}"
+
+        if (orderBy.exists(_.nonEmpty))
+            sql += s"${delim}ORDER BY ${orderBy.get.map(exprToString(_)).mkString(s",$delim$tab")}"
+
+        if (limit.isDefined)
+            sql += s" LIMIT ${limit.map(exprToString(_)).get}"
+
+        sql
+    }
+
+    /**
+      * @return From table SQL query part.
+      */
+    private def compiledTableExpression: String = table match {
+        case Some(tableName) ⇒
+            tableName
+
+        case None ⇒ tableExpression match {
+            case Some((acc, alias)) ⇒
+                s"(${acc.compileQuery()}) $alias"
+
+            case None ⇒
+                throw new IgniteException("Unknown table.")
+        }
+    }
+
+    /** @inheritdoc */
+    override def simpleString: String =
+        s"IgniteSQLAccumulator(table: $table, columns: $outputExpressions, distinct: $distinct, all: $all, " +
+            s"where: $where, groupBy: $groupBy, having: $having, limit: $limit, orderBy: $orderBy)"
+
+    /** @inheritdoc */
+    override def withOutputExpressions(outputExpressions: Seq[NamedExpression]): SelectAccumulator =
+        copy(outputExpressions= outputExpressions)
+
+    /** @inheritdoc */
+    override def withDistinct(distinct: Boolean): SingleTableSQLAccumulator = copy(distinct = distinct)
+
+    /** @inheritdoc */
+    override def withWhere(where: Seq[Expression]): SingleTableSQLAccumulator = copy(where = Some(where))
+
+    /** @inheritdoc */
+    override def withGroupBy(groupBy: Seq[Expression]): SingleTableSQLAccumulator = copy(groupBy = Some(groupBy))
+
+    /** @inheritdoc */
+    override def withHaving(having: Seq[Expression]): SingleTableSQLAccumulator = copy(having = Some(having))
+
+    /** @inheritdoc */
+    override def withLimit(limit: Expression): SingleTableSQLAccumulator = copy(limit = Some(limit))
+
+    /** @inheritdoc */
+    override def withLocalLimit(localLimit: Expression): SingleTableSQLAccumulator = copy(localLimit = Some(localLimit))
+
+    /** @inheritdoc */
+    override def withOrderBy(orderBy: Seq[SortOrder]): SingleTableSQLAccumulator = copy(orderBy = Some(orderBy))
+
+    /** @inheritdoc */
+    override def output: Seq[Attribute] = outputExpressions.map(toAttributeReference(_, Seq.empty))
+
+    /** @inheritdoc */
+    override def qualifier: String = table.getOrElse(tableExpression.get._2)
+
+    /** @inheritdoc */
+    override def children: Seq[LogicalPlan] = tableExpression.map(te ⇒ Seq(te._1)).getOrElse(Nil)
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/UnionSQLAccumulator.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/UnionSQLAccumulator.scala
new file mode 100644
index 0000000000000..723e17af3894a
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/accumulator/UnionSQLAccumulator.scala
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl.optimization.accumulator
+
+import org.apache.ignite.spark.impl.optimization.{IgniteQueryContext, exprToString, toAttributeReference}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, NamedExpression, SortOrder}
+
+/**
+  * Accumulator to store info about UNION query.
+  */
+private[apache] case class UnionSQLAccumulator(
+    igniteQueryContext: IgniteQueryContext,
+    children: Seq[QueryAccumulator],
+    outputExpressions: Seq[NamedExpression],
+    orderBy: Option[Seq[SortOrder]] = None
+) extends QueryAccumulator {
+    /** @inheritdoc */
+    override def compileQuery(prettyPrint: Boolean = false): String = {
+        val delim = if (prettyPrint) "\n" else " "
+        val tab = if (prettyPrint) "  " else ""
+
+        val query = children.map(_.compileQuery(prettyPrint)).mkString(s"${delim}UNION$delim")
+
+        orderBy match {
+            case Some(sortOrders) ⇒
+                query + s"${delim}ORDER BY ${sortOrders.map(exprToString(_)).mkString(s",$delim$tab")}"
+
+            case None ⇒ query
+        }
+    }
+
+    /** @inheritdoc */
+    override def simpleString: String =
+        s"UnionSQLAccumulator(orderBy: ${orderBy.map(_.map(exprToString(_)).mkString(", ")).getOrElse("[]")})"
+
+    /** @inheritdoc */
+    override def withOutputExpressions(outputExpressions: Seq[NamedExpression]): QueryAccumulator =
+        copy(outputExpressions= outputExpressions)
+
+    /** @inheritdoc */
+    override def withOrderBy(orderBy: Seq[SortOrder]): QueryAccumulator = copy(orderBy = Some(orderBy))
+
+    /** @inheritdoc */
+    override def output: Seq[Attribute] = outputExpressions.map(toAttributeReference(_, Seq.empty))
+
+    /** @inheritdoc */
+    override lazy val qualifier: String = igniteQueryContext.uniqueTableAlias
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/package.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/package.scala
new file mode 100644
index 0000000000000..4e168f41dfc61
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/package.scala
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark.impl
+
+
+import org.apache.ignite.IgniteException
+import org.apache.ignite.spark.impl.optimization.accumulator.{QueryAccumulator, SingleTableSQLAccumulator}
+import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count}
+import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference, ExprId, Expression, NamedExpression}
+import org.apache.spark.sql.types._
+
+import scala.annotation.tailrec
+
+/**
+  */
+package object optimization {
+    /**
+      * Constant to store alias in column metadata.
+      */
+    private[optimization] val ALIAS: String = "alias"
+
+    /**
+      * All `SupportedExpression` implementations.
+      */
+    private val SUPPORTED_EXPRESSIONS: List[SupportedExpressions] = List (
+        SimpleExpressions,
+        SystemExpressions,
+        AggregateExpressions,
+        ConditionExpressions,
+        DateExpressions,
+        MathExpressions,
+        StringExpressions
+    )
+
+    /**
+      * @param expr Expression.
+      * @param useQualifier If true outputs attributes of `expr` with qualifier.
+      * @param useAlias If true outputs `expr` with alias.
+      * @return String representation of expression.
+      */
+    def exprToString(expr: Expression, useQualifier: Boolean = false, useAlias: Boolean = true): String = {
+        @tailrec
+        def exprToString0(expr: Expression, supportedExpressions: List[SupportedExpressions]): Option[String] =
+            if (supportedExpressions.nonEmpty) {
+                val exprStr = supportedExpressions.head.toString(
+                    expr,
+                    exprToString(_, useQualifier, useAlias = false),
+                    useQualifier,
+                    useAlias)
+
+                exprStr match {
+                    case res: Some[String] ⇒
+                        res
+                    case None ⇒
+                        exprToString0(expr, supportedExpressions.tail)
+                }
+            }
+            else
+                None
+
+        exprToString0(expr, SUPPORTED_EXPRESSIONS) match {
+            case Some(str) ⇒ str
+
+            case None ⇒
+                throw new IgniteException("Unsupporte expression " + expr)
+        }
+    }
+
+    /**
+      * @param exprs Expressions to check.
+      * @return True if `exprs` contains only allowed(i.e. can be pushed down to Ignite) expressions false otherwise.
+      */
+    def exprsAllowed(exprs: Seq[Expression]): Boolean =
+        exprs.forall(exprsAllowed)
+
+    /**
+      * @param expr Expression to check.
+      * @return True if `expr` allowed(i.e. can be pushed down to Ignite) false otherwise.
+      *
+      */
+    def exprsAllowed(expr: Expression): Boolean =
+        SUPPORTED_EXPRESSIONS.exists(_(expr, exprsAllowed))
+
+    /**
+      * Converts `input` into `AttributeReference`.
+      *
+      * @param input Expression to convert.
+      * @param existingOutput Existing output.
+      * @param exprId Optional expression ID to use.
+      * @param alias Optional alias for a result.
+      * @return Converted expression.
+      */
+    def toAttributeReference(input: Expression, existingOutput: Seq[NamedExpression], exprId: Option[ExprId] = None,
+        alias: Option[String] = None): AttributeReference = {
+
+        input match {
+            case attr: AttributeReference ⇒
+                val toCopy = existingOutput.find(_.exprId == attr.exprId).getOrElse(attr)
+
+                AttributeReference(
+                    name = toCopy.name,
+                    dataType = toCopy.dataType,
+                    metadata = alias
+                        .map(new MetadataBuilder().withMetadata(toCopy.metadata).putString(ALIAS, _).build())
+                        .getOrElse(toCopy.metadata)
+                )(exprId = exprId.getOrElse(toCopy.exprId), qualifier = toCopy.qualifier, isGenerated = toCopy.isGenerated)
+
+            case a: Alias ⇒
+                toAttributeReference(a.child, existingOutput, Some(a.exprId), Some(alias.getOrElse(a.name)))
+
+            case agg: AggregateExpression ⇒
+                agg.aggregateFunction match {
+                    case c: Count ⇒
+                        if (agg.isDistinct)
+                            AttributeReference(
+                                name = s"COUNT(DISTINCT ${c.children.map(exprToString(_)).mkString(" ")})",
+                                dataType = LongType,
+                                metadata = alias
+                                    .map(new MetadataBuilder().putString(ALIAS, _).build())
+                                    .getOrElse(Metadata.empty)
+                            )(exprId = exprId.getOrElse(agg.resultId))
+                        else
+                            AttributeReference(
+                                name = s"COUNT(${c.children.map(exprToString(_)).mkString(" ")})",
+                                dataType = LongType,
+                                metadata = alias
+                                    .map(new MetadataBuilder().putString(ALIAS, _).build())
+                                    .getOrElse(Metadata.empty)
+                            )(exprId = exprId.getOrElse(agg.resultId))
+
+                    case _ ⇒
+                        toAttributeReference(agg.aggregateFunction, existingOutput, Some(exprId.getOrElse(agg.resultId)), alias)
+                }
+
+            case ne: NamedExpression ⇒
+                AttributeReference(
+                    name = exprToString(input),
+                    dataType = input.dataType,
+                    metadata = alias
+                        .map(new MetadataBuilder().withMetadata(ne.metadata).putString(ALIAS, _).build())
+                        .getOrElse(Metadata.empty)
+                )(exprId = exprId.getOrElse(ne.exprId))
+
+            case _ if exprsAllowed(input) ⇒
+                AttributeReference(
+                    name = exprToString(input),
+                    dataType = input.dataType,
+                    metadata = alias
+                        .map(new MetadataBuilder().putString(ALIAS, _).build())
+                        .getOrElse(Metadata.empty)
+                )(exprId = exprId.getOrElse(NamedExpression.newExprId))
+
+            case _  ⇒
+                throw new IgniteException(s"Unsupported column expression $input")
+        }
+    }
+
+    /**
+      * @param dataType Spark data type.
+      * @return SQL data type.
+      */
+    def toSqlType(dataType: DataType): String = dataType match {
+        case BooleanType ⇒ "BOOLEAN"
+        case IntegerType ⇒ "INT"
+        case ByteType ⇒ "TINYINT"
+        case ShortType ⇒ "SMALLINT"
+        case LongType ⇒ "BIGINT"
+        case DecimalType() ⇒ "DECIMAL"
+        case DoubleType ⇒ "DOUBLE"
+        case FloatType ⇒ "REAL"
+        case DateType ⇒ "DATE"
+        case TimestampType ⇒ "TIMESTAMP"
+        case StringType ⇒ "VARCHAR"
+        case BinaryType ⇒ "BINARY"
+        case ArrayType(_, _) ⇒ "ARRAY"
+        case _ ⇒
+            throw new IgniteException(s"$dataType not supported!")
+    }
+
+    /**
+      * @param expr Expression
+      * @return True if expression or some of it children is AggregateExpression, false otherwise.
+      */
+    def hasAggregateInside(expr: Expression): Boolean = {
+        def hasAggregateInside0(expr: Expression): Boolean = expr match {
+            case AggregateExpression(_, _, _, _) ⇒
+                true
+
+            case e: Expression ⇒
+                e.children.exists(hasAggregateInside0)
+        }
+
+        hasAggregateInside0(expr)
+    }
+
+    /**
+      * Check if `acc` representing simple query.
+      * Simple is `SELECT ... FROM table WHERE ... ` like query.
+      * Without aggregation, limits, order, embedded select expressions.
+      *
+      * @param acc Accumulator to check.
+      * @return True if accumulator stores simple query info, false otherwise.
+      */
+    def isSimpleTableAcc(acc: QueryAccumulator): Boolean = acc match {
+        case acc: SingleTableSQLAccumulator if acc.table.isDefined ⇒
+            acc.groupBy.isEmpty &&
+                acc.localLimit.isEmpty &&
+                acc.orderBy.isEmpty &&
+                !acc.distinct &&
+                !acc.outputExpressions.exists(hasAggregateInside)
+
+        case _ ⇒
+            false
+    }
+}
diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/package.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/package.scala
index 4634a97b4a753..6502c0ffccc50 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/package.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/package.scala
@@ -19,12 +19,15 @@ package org.apache.ignite.spark
 
 import org.apache.commons.lang.StringUtils.equalsIgnoreCase
 import org.apache.ignite.{Ignite, IgniteException, IgniteState, Ignition}
-import org.apache.ignite.cache.QueryEntity
+import org.apache.ignite.cache.{CacheMode, QueryEntity}
+import org.apache.ignite.cluster.ClusterNode
 import org.apache.ignite.configuration.CacheConfiguration
 import org.apache.ignite.internal.util.lang.GridFunc.contains
+import org.apache.spark.Partition
 import org.apache.spark.sql.catalyst.catalog.SessionCatalog
 
 import scala.collection.JavaConversions._
+import scala.collection.mutable.ArrayBuffer
 
 package object impl {
     /**
@@ -129,4 +132,47 @@ package object impl {
       */
     def isKeyColumn(table: QueryEntity, column: String): Boolean =
         contains(table.getKeyFields, column) || equalsIgnoreCase(table.getKeyFieldName, column)
+
+    /**
+      * Computes spark partitions for a given cache.
+      *
+      * @param ic Ignite context.
+      * @param cacheName Cache name
+      * @return Array of IgniteDataFramPartition
+      */
+    def calcPartitions(ic: IgniteContext, cacheName: String): Array[Partition] = {
+        val cache = ic.ignite().cache[Any, Any](cacheName)
+
+        val ccfg = cache.getConfiguration(classOf[CacheConfiguration[Any, Any]])
+
+        if (ccfg.getCacheMode == CacheMode.REPLICATED) {
+            val serverNodes = ic.ignite().cluster().forCacheNodes(cacheName).forServers().nodes()
+
+            Array(IgniteDataFramePartition(0, serverNodes.head, Stream.from(0).take(1024).toList))
+        }
+        else {
+            val aff = ic.ignite().affinity(cacheName)
+
+            val parts = aff.partitions()
+
+            val nodesToParts = (0 until parts).foldLeft(Map[ClusterNode, ArrayBuffer[Int]]()) {
+                case (nodeToParts, ignitePartIdx) ⇒
+                    val primary = aff.mapPartitionToPrimaryAndBackups(ignitePartIdx).head
+
+                    if (nodeToParts.contains(primary)) {
+                        nodeToParts(primary) += ignitePartIdx
+
+                        nodeToParts
+                    }
+                    else
+                        nodeToParts + (primary → ArrayBuffer[Int](ignitePartIdx))
+            }
+
+            val partitions = nodesToParts.zipWithIndex.map { case ((node, nodesParts), i) ⇒
+                IgniteDataFramePartition(i, node, nodesParts.toList)
+            }
+
+            partitions.toArray
+        }
+    }
 }
diff --git a/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteOptimization.scala b/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteOptimization.scala
new file mode 100644
index 0000000000000..b23cd6fc690cb
--- /dev/null
+++ b/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteOptimization.scala
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.ignite
+
+import org.apache.ignite.IgniteException
+import org.apache.ignite.spark.impl.{IgniteSQLAccumulatorRelation, IgniteSQLRelation, sqlCacheName}
+import org.apache.ignite.spark.impl.optimization.{accumulator, _}
+import org.apache.ignite.spark.impl.optimization.accumulator._
+import org.apache.spark.internal.Logging
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.execution.datasources.LogicalRelation
+
+/**
+  * Query plan optimization for a Ignite based queries.
+  */
+object IgniteOptimization extends Rule[LogicalPlan] with Logging {
+    /** @inheritdoc */
+    override def apply(plan: LogicalPlan): LogicalPlan = {
+        logDebug("")
+        logDebug("== Plan Before Ignite Operator Push Down ==")
+        logDebug(plan.toString())
+
+        val transformed = fixAmbiguousOutput(pushDownOperators(plan))
+
+        logDebug("")
+        logDebug("== Plan After Ignite Operator Push Down ==")
+        logDebug(transformed.toString())
+
+        makeIgniteAccRelation(transformed)
+    }
+
+    /**
+      * Change query plan by accumulating query parts supported by Ignite into `QueryAccumulator`.
+      *
+      * @param plan Query plan.
+      * @return Transformed plan.
+      */
+    private def pushDownOperators(plan: LogicalPlan): LogicalPlan = {
+        val aliasIndexIterator = Stream.from(1).iterator
+
+        //Flag to indicate that some step was skipped due to unsupported expression.
+        //When it true we has to skip entire transformation of higher level Nodes.
+        var stepSkipped = true
+
+        //Applying optimization rules from bottom to up tree nodes.
+        plan.transformUp {
+            //We found basic node to transform.
+            //We create new accumulator and going to the upper layers.
+            case LogicalRelation(igniteSqlRelation: IgniteSQLRelation[_, _], output, _catalogTable) ⇒
+                //Clear flag to optimize each statement separately
+                stepSkipped = false
+
+                val igniteQueryContext = IgniteQueryContext(
+                    igniteContext = igniteSqlRelation.ic,
+                    sqlContext = igniteSqlRelation.sqlContext,
+                    catalogTable = _catalogTable,
+                    aliasIndex = aliasIndexIterator,
+                    cacheName =
+                        sqlCacheName(igniteSqlRelation.ic.ignite(), igniteSqlRelation.tableName)
+                            .getOrElse(throw new IgniteException("Unknown table")))
+
+                //Logical Relation is bottomest TreeNode in LogicalPlan.
+                //We replace it with accumulator.
+                //We push all supported SQL operator into it on the higher tree levels.
+                SingleTableSQLAccumulator(
+                    igniteQueryContext = igniteQueryContext,
+                    table = Some(igniteSqlRelation.tableName),
+                    tableExpression = None,
+                    outputExpressions = output.map(attr ⇒ attr.withQualifier(Some(igniteSqlRelation.tableName))))
+
+            case project: Project if !stepSkipped && exprsAllowed(project.projectList) ⇒
+                //Project layer just changes output of current query.
+                project.child match {
+                    case acc: SelectAccumulator ⇒
+                        acc.withOutputExpressions(
+                            substituteExpressions(project.projectList, acc.outputExpressions))
+
+                    case _ ⇒
+                        throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+                }
+
+            case sort: Sort if !stepSkipped && isSortPushDownAllowed(sort.order, sort.global) ⇒
+                sort.child match {
+                    case acc: QueryAccumulator ⇒
+                        acc.withOrderBy(sort.order)
+
+                    case _ ⇒
+                        throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+                }
+
+            case filter: Filter if !stepSkipped && exprsAllowed(filter.condition) ⇒
+
+                filter.child match {
+                    case acc: SelectAccumulator ⇒
+                        if (hasAggregateInside(filter.condition) || acc.groupBy.isDefined)
+                            acc.withHaving(acc.having.getOrElse(Nil) :+ filter.condition)
+                        else
+                            acc.withWhere(acc.where.getOrElse(Nil) :+ filter.condition)
+
+                    case _ ⇒
+                        throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+                }
+
+            case agg: Aggregate
+                if !stepSkipped && exprsAllowed(agg.groupingExpressions) && exprsAllowed(agg.aggregateExpressions) ⇒
+
+                agg.child match {
+                    case acc: SelectAccumulator ⇒
+                        if (acc.groupBy.isDefined) {
+                            val tableAlias = acc.igniteQueryContext.uniqueTableAlias
+
+                            accumulator.SingleTableSQLAccumulator(
+                                igniteQueryContext = acc.igniteQueryContext,
+                                table = None,
+                                tableExpression = Some((acc, tableAlias)),
+                                outputExpressions = agg.aggregateExpressions)
+                        }
+                        else
+                            acc
+                                .withGroupBy(agg.groupingExpressions)
+                                .withOutputExpressions(
+                                    substituteExpressions(agg.aggregateExpressions, acc.outputExpressions))
+
+                    case acc: QueryAccumulator ⇒
+                        val tableAlias = acc.igniteQueryContext.uniqueTableAlias
+
+                        accumulator.SingleTableSQLAccumulator(
+                            igniteQueryContext = acc.igniteQueryContext,
+                            table = None,
+                            tableExpression = Some((acc, tableAlias)),
+                            outputExpressions = agg.aggregateExpressions)
+
+                    case _ ⇒
+                        throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+                }
+
+            case limit: LocalLimit if !stepSkipped && exprsAllowed(limit.limitExpr) ⇒
+                limit.child match {
+                    case acc: SelectAccumulator ⇒
+                        acc.withLocalLimit(limit.limitExpr)
+
+                    case _ ⇒
+                        throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+                }
+
+            case limit: GlobalLimit if !stepSkipped && exprsAllowed(limit.limitExpr) ⇒
+                limit.child.transformUp {
+                    case acc: SelectAccumulator ⇒
+                        acc.withLimit(limit.limitExpr)
+
+                    case _ ⇒
+                        throw new IgniteException("stepSkipped == true but child is not SelectAccumulator")
+                }
+
+            case union: Union if !stepSkipped && isAllChildrenOptimized(union.children) ⇒
+                val first = union.children.head.asInstanceOf[QueryAccumulator]
+
+                val subQueries = union.children.map(_.asInstanceOf[QueryAccumulator])
+
+                UnionSQLAccumulator(
+                    first.igniteQueryContext,
+                    subQueries,
+                    subQueries.head.output)
+
+            case join: Join
+                if !stepSkipped && isAllChildrenOptimized(Seq(join.left, join.right)) &&
+                    join.condition.forall(exprsAllowed) ⇒
+
+                val left = join.left.asInstanceOf[QueryAccumulator]
+
+                val (leftOutput, leftAlias) =
+                    if (!isSimpleTableAcc(left)) {
+                        val tableAlias = left.igniteQueryContext.uniqueTableAlias
+
+                        (left.output, Some(tableAlias))
+                    }
+                    else
+                        (left.output, None)
+
+                val right = join.right.asInstanceOf[QueryAccumulator]
+
+                val (rightOutput, rightAlias) =
+                    if (!isSimpleTableAcc(right) ||
+                        leftAlias.getOrElse(left.qualifier) == right.qualifier) {
+                        val tableAlias = right.igniteQueryContext.uniqueTableAlias
+
+                        (right.output, Some(tableAlias))
+                    }
+                    else
+                        (right.output, None)
+
+                JoinSQLAccumulator(
+                    left.igniteQueryContext,
+                    left,
+                    right,
+                    join.joinType,
+                    leftOutput ++ rightOutput,
+                    join.condition,
+                    leftAlias,
+                    rightAlias)
+
+            case unknown ⇒
+                stepSkipped = true
+
+                unknown
+        }
+    }
+
+    /**
+      * Changes qualifiers for an ambiguous columns names.
+      *
+      * @param plan Query plan.
+      * @return Transformed plan.
+      */
+    private def fixAmbiguousOutput(plan: LogicalPlan): LogicalPlan = plan.transformDown {
+        case acc: SingleTableSQLAccumulator if acc.children.exists(_.isInstanceOf[JoinSQLAccumulator]) ⇒
+            val fixedChildOutput =
+                fixAmbiguousOutput(acc.tableExpression.get._1.outputExpressions, acc.igniteQueryContext)
+
+            val newOutput = substituteExpressions(acc.outputExpressions, fixedChildOutput, changeOnlyName = true)
+
+            acc.copy(
+                outputExpressions = newOutput,
+                where = acc.where.map(
+                    substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+                groupBy = acc.groupBy.map(
+                    substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+                having = acc.having.map(
+                    substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+                limit = acc.limit.map(
+                    substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+                localLimit = acc.localLimit.map(
+                    substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+                orderBy = acc.orderBy.map(
+                    substituteExpressions(_, fixedChildOutput, changeOnlyName = true)))
+
+            acc
+
+        case acc: JoinSQLAccumulator
+            if acc.left.isInstanceOf[JoinSQLAccumulator] || acc.right.isInstanceOf[JoinSQLAccumulator] ⇒
+            val leftFixed = acc.left match {
+                case leftJoin: JoinSQLAccumulator ⇒
+                    val fixedChildOutput = fixAmbiguousOutput(acc.left.outputExpressions, acc.igniteQueryContext)
+
+                    val newOutput =
+                        substituteExpressions(acc.outputExpressions, fixedChildOutput, changeOnlyName = true)
+
+                    acc.copy(
+                        outputExpressions = newOutput,
+                        left = leftJoin.copy(outputExpressions = fixedChildOutput),
+                        condition = acc.condition.map(
+                            substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+                        where = acc.where.map(
+                            substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+                        groupBy = acc.groupBy.map(
+                            substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+                        having = acc.having.map(
+                            substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+                        limit = acc.limit.map(
+                            substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+                        localLimit = acc.localLimit.map(
+                            substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+                        orderBy = acc.orderBy.map(
+                            substituteExpressions(_, fixedChildOutput, changeOnlyName = true)))
+
+                case _ ⇒ acc
+            }
+
+            val fixed = leftFixed.right match {
+                case rightJoin: JoinSQLAccumulator ⇒
+                    val fixedChildOutput =
+                        fixAmbiguousOutput(leftFixed.outputExpressions, leftFixed.igniteQueryContext)
+
+                    val newOutput = substituteExpressions(leftFixed.outputExpressions, fixedChildOutput)
+
+                    leftFixed.copy(
+                        outputExpressions = newOutput,
+                        right = rightJoin.copy(outputExpressions = fixedChildOutput),
+                        condition = acc.condition.map(
+                            substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+                        where = acc.where.map(
+                            substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+                        groupBy = acc.groupBy.map(
+                            substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+                        having = acc.having.map(
+                            substituteExpressions(_, fixedChildOutput, changeOnlyName = true)),
+                        limit = acc.limit.map(
+                            substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+                        localLimit = acc.localLimit.map(
+                            substituteExpression(_, fixedChildOutput, changeOnlyName = true)),
+                        orderBy = acc.orderBy.map(
+                            substituteExpressions(_, fixedChildOutput, changeOnlyName = true)))
+
+                case _ ⇒ leftFixed
+            }
+
+            fixed.copy(
+                condition = acc.condition.map(
+                    substituteExpression(_, acc.outputExpressions, changeOnlyName = true)),
+                where = acc.where.map(
+                    substituteExpressions(_, acc.outputExpressions, changeOnlyName = true)),
+                groupBy = acc.groupBy.map(
+                    substituteExpressions(_, acc.outputExpressions, changeOnlyName = true)),
+                having = acc.having.map(
+                    substituteExpressions(_, acc.outputExpressions, changeOnlyName = true)),
+                limit = acc.limit.map(
+                    substituteExpression(_, acc.outputExpressions, changeOnlyName = true)),
+                localLimit = acc.localLimit.map(
+                    substituteExpression(_, acc.outputExpressions, changeOnlyName = true)),
+                orderBy = acc.orderBy.map(
+                    substituteExpressions(_, acc.outputExpressions, changeOnlyName = true)))
+
+        case unknown ⇒
+            unknown
+    }
+
+    private def fixAmbiguousOutput(exprs: Seq[NamedExpression], ctx: IgniteQueryContext): Seq[NamedExpression] =
+        exprs.foldLeft((Set[String](), Set[NamedExpression]())) {
+            case ((uniqueNames, fixed), next) ⇒
+                if (uniqueNames(next.name))
+                    (uniqueNames, fixed + Alias(next, ctx.uniqueColumnAlias(next))(exprId = next.exprId))
+                else
+                    (uniqueNames + next.name, fixed + next)
+        }._2.toSeq
+
+    /**
+      * Substitutes each `QueryAccumulator` with a `LogicalRelation` contains `IgniteSQLAccumulatorRelation`.
+      *
+      * @param plan Query plan.
+      * @return Transformed plan.
+      */
+    private def makeIgniteAccRelation(plan: LogicalPlan): LogicalPlan =
+        plan.transformDown {
+            case acc: QueryAccumulator ⇒
+                new LogicalRelation (
+                    relation = IgniteSQLAccumulatorRelation(acc),
+                    output = acc.outputExpressions.map(toAttributeReference(_, Seq.empty)),
+                    catalogTable = acc.igniteQueryContext.catalogTable)
+        }
+
+    /**
+      * @param order Order.
+      * @param global True if order applied to entire result set false if ordering per-partition.
+      * @return True if sort can be pushed down to Ignite, false otherwise.
+      */
+    private def isSortPushDownAllowed(order: Seq[SortOrder], global: Boolean): Boolean =
+        global && order.map(_.child).forall(exprsAllowed)
+
+    /**
+      * @param children Plans to check.
+      * @return True is all plan are `QueryAccumulator`, false otherwise.
+      */
+    private def isAllChildrenOptimized(children: Seq[LogicalPlan]): Boolean =
+        children.forall {
+            case _: QueryAccumulator ⇒
+                true
+
+            case _ ⇒
+                false
+        }
+
+    /**
+      * Changes expression from `exprs` collection to expression with same `exprId` from `substitution`.
+      *
+      * @param exprs Expressions to substitute.
+      * @param substitution Expressions for substitution
+      * @param changeOnlyName If true substitute only expression name.
+      * @tparam T Concrete expression type.
+      * @return Substituted expressions.
+      */
+    private def substituteExpressions[T <: Expression](exprs: Seq[T], substitution: Seq[NamedExpression],
+        changeOnlyName: Boolean = false): Seq[T] = {
+
+        exprs.map(substituteExpression(_, substitution, changeOnlyName))
+    }
+
+    private def substituteExpression[T <: Expression](expr: T, substitution: Seq[NamedExpression],
+        changeOnlyName: Boolean): T = expr match {
+        case ne: NamedExpression ⇒
+            substitution.find(_.exprId == ne.exprId) match {
+                case Some(found) ⇒
+                    if (!changeOnlyName)
+                        found.asInstanceOf[T]
+                    else ne match {
+                        case alias: Alias ⇒
+                            Alias(
+                                AttributeReference(
+                                    found.name,
+                                    found.dataType,
+                                    nullable = found.nullable,
+                                    metadata = found.metadata)(
+                                    exprId = found.exprId,
+                                    qualifier = found.qualifier,
+                                    isGenerated = found.isGenerated),
+                                alias.name) (
+                                exprId = alias.exprId,
+                                qualifier = alias.qualifier,
+                                explicitMetadata = alias.explicitMetadata,
+                                isGenerated = alias.isGenerated).asInstanceOf[T]
+
+                        case attr: AttributeReference ⇒
+                            attr.copy(name = found.name)(
+                                exprId = found.exprId,
+                                qualifier = found.qualifier,
+                                isGenerated = found.isGenerated).asInstanceOf[T]
+
+                        case _ ⇒ ne.asInstanceOf[T]
+                    }
+
+                case None ⇒
+                    expr.withNewChildren(
+                        substituteExpressions(expr.children, substitution, changeOnlyName)).asInstanceOf[T]
+            }
+
+        case _ ⇒
+            expr.withNewChildren(
+                substituteExpressions(expr.children, substitution, changeOnlyName)).asInstanceOf[T]
+    }
+}
diff --git a/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteSparkSession.scala b/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteSparkSession.scala
index 8860590795d68..1fccc3ab43026 100644
--- a/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteSparkSession.scala
+++ b/modules/spark/src/main/scala/org/apache/spark/sql/ignite/IgniteSparkSession.scala
@@ -66,8 +66,14 @@ class IgniteSparkSession private(ic: IgniteContext, proxy: SparkSession) extends
         new IgniteSharedState(ic, sparkContext)
 
     /** @inheritdoc */
-    @transient override lazy val sessionState: SessionState =
-        new SessionStateBuilder(self, None).build()
+    @transient override lazy val sessionState: SessionState = {
+        val sessionState = new SessionStateBuilder(self, None).build()
+
+        sessionState.experimentalMethods.extraOptimizations =
+            sessionState.experimentalMethods.extraOptimizations :+ IgniteOptimization
+
+        sessionState
+    }
 
     /** @inheritdoc */
     @transient override lazy val conf: RuntimeConfig = proxy.conf
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/AbstractDataFrameSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/AbstractDataFrameSpec.scala
index 8613592cfaa2a..29a4e6f28acfb 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/AbstractDataFrameSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/AbstractDataFrameSpec.scala
@@ -19,13 +19,16 @@ package org.apache.ignite.spark
 
 import org.apache.ignite.{Ignite, Ignition}
 import org.apache.ignite.configuration.{CacheConfiguration, IgniteConfiguration}
-import org.apache.spark.sql.SparkSession
-import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll, FunSpec, Matchers}
+import org.apache.spark.sql.{DataFrame, Row, SparkSession}
+import org.scalatest._
 import java.lang.{Long ⇒ JLong}
 
 import org.apache.ignite.cache.query.SqlFieldsQuery
 import org.apache.ignite.cache.query.annotations.QuerySqlField
-import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.internal.IgnitionEx.loadConfiguration
+import org.apache.ignite.spark.AbstractDataFrameSpec.configuration
+import org.apache.ignite.spark.impl.IgniteSQLAccumulatorRelation
+import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.ignite.spark.AbstractDataFrameSpec._
 
 import scala.annotation.meta.field
@@ -33,12 +36,13 @@ import scala.reflect.ClassTag
 
 /**
   */
-abstract class AbstractDataFrameSpec extends FunSpec with Matchers with BeforeAndAfterAll with BeforeAndAfter {
+abstract class AbstractDataFrameSpec extends FunSpec with Matchers with BeforeAndAfterAll with BeforeAndAfter
+    with Assertions {
     var spark: SparkSession = _
 
     var client: Ignite = _
 
-    override protected def beforeAll() = {
+    override protected def beforeAll(): Unit = {
         for (i ← 0 to 3)
             Ignition.start(configuration("grid-" + i, client = false))
 
@@ -47,7 +51,7 @@ abstract class AbstractDataFrameSpec extends FunSpec with Matchers with BeforeAn
         createSparkSession()
     }
 
-    override protected def afterAll() = {
+    override protected def afterAll(): Unit = {
         Ignition.stop("client", false)
 
         for (i ← 0 to 3)
@@ -108,6 +112,7 @@ abstract class AbstractDataFrameSpec extends FunSpec with Matchers with BeforeAn
         cache.query(qry.setArgs(1L.asInstanceOf[JLong], "Forest Hill")).getAll
         cache.query(qry.setArgs(2L.asInstanceOf[JLong], "Denver")).getAll
         cache.query(qry.setArgs(3L.asInstanceOf[JLong], "St. Petersburg")).getAll
+        cache.query(qry.setArgs(4L.asInstanceOf[JLong], "St. Petersburg")).getAll
     }
 
     def createEmployeeCache(client: Ignite, cacheName: String): Unit = {
@@ -119,6 +124,31 @@ abstract class AbstractDataFrameSpec extends FunSpec with Matchers with BeforeAn
         cache.put("key2", Employee(2, "Sarah Connor", 32, 10000))
         cache.put("key3", Employee(3, "Arnold Schwarzenegger", 27, 1000))
     }
+
+    def checkQueryData[T](res: DataFrame, expectedRes: Product)
+        (implicit ord: T ⇒ Ordered[T]): Unit =
+        checkQueryData(res, expectedRes, _.getAs[T](0))
+
+    def checkQueryData[Ordered](res: DataFrame, expectedRes: Product, sorter: Row => Ordered)
+        (implicit ord: Ordering[Ordered]): Unit = {
+        val data = res.rdd.collect.sortBy(sorter)
+
+        for(i ← 0 until expectedRes.productArity) {
+            val row = data(i)
+
+            if (row.size == 1)
+                assert(row(0) == expectedRes.productElement(i), s"row[$i, 0] = ${row(0)} should be equal ${expectedRes.productElement(i)}")
+            else {
+                val expectedRow: Product = expectedRes.productElement(i).asInstanceOf[Product]
+
+                assert(expectedRow.productArity == row.size, s"Rows size should be equal, but expected.size=${expectedRow.productArity} " +
+                    s"and row.size=${row.size}")
+
+                for (j ← 0 until expectedRow.productArity)
+                    assert(row(j) == expectedRow.productElement(j), s"row[$i, $j] = ${row(j)} should be equal ${expectedRow.productElement(j)}")
+            }
+        }
+    }
 }
 
 object AbstractDataFrameSpec {
@@ -135,7 +165,7 @@ object AbstractDataFrameSpec {
     val PERSON_TBL_NAME_2 = "person2"
 
     def configuration(igniteInstanceName: String, client: Boolean): IgniteConfiguration = {
-        val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+        val cfg = loadConfiguration(TEST_CONFIG_FILE).get1()
 
         cfg.setClientMode(client)
 
@@ -166,6 +196,30 @@ object AbstractDataFrameSpec {
         ccfg
     }
 
+    /**
+      * @param df Data frame.
+      * @param qry SQL Query.
+      */
+    def checkOptimizationResult(df: DataFrame, qry: String = ""): Unit = {
+        df.explain(true)
+
+        val plan = df.queryExecution.optimizedPlan
+
+        val cnt = plan.collectLeaves.count {
+            case LogicalRelation(relation: IgniteSQLAccumulatorRelation[_, _], _, _) ⇒
+                if (qry != "")
+                    assert(qry.toLowerCase == relation.acc.compileQuery().toLowerCase,
+                        s"Generated query should be equal to expected.\nexpected  - $qry\ngenerated - ${relation.acc.compileQuery()}")
+
+                true
+
+            case _ ⇒
+                false
+        }
+
+        assert(cnt != 0, s"Plan should contains IgniteSQLAccumulatorRelation")
+    }
+
     /**
       * Enclose some closure, so it doesn't on outer object(default scala behaviour) while serializing.
       */
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteCatalogSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteCatalogSpec.scala
index 6077211882d6d..d87d23410b0a1 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteCatalogSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteCatalogSpec.scala
@@ -58,7 +58,7 @@ class IgniteCatalogSpec extends AbstractDataFrameSpec {
         it("Should provide ability to query SQL table without explicit registration") {
             val res = igniteSession.sql("SELECT id, name FROM city").rdd
 
-            res.count should equal(3)
+            res.count should equal(4)
 
             val cities = res.collect.sortBy(_.getAs[JLong]("id"))
 
@@ -66,7 +66,8 @@ class IgniteCatalogSpec extends AbstractDataFrameSpec {
                 Array(
                     (1, "Forest Hill"),
                     (2, "Denver"),
-                    (3, "St. Petersburg")
+                    (3, "St. Petersburg"),
+                    (4, "St. Petersburg")
                 )
             )
         }
@@ -136,7 +137,7 @@ class IgniteCatalogSpec extends AbstractDataFrameSpec {
 
         createEmployeeCache(client, EMPLOYEE_CACHE_NAME)
 
-        val configProvider = enclose(null) (x ⇒ () ⇒ {
+        val configProvider = enclose(null) (_ ⇒ () ⇒ {
             val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
 
             cfg.setClientMode(true)
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSchemaSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSchemaSpec.scala
index cdd26cd5a6a63..c5df90125cfc7 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSchemaSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSchemaSpec.scala
@@ -17,9 +17,6 @@
 
 package org.apache.ignite.spark
 
-import java.lang.{Integer ⇒ JInteger, String ⇒ JString}
-
-import org.apache.ignite.Ignite
 import org.apache.ignite.spark.AbstractDataFrameSpec._
 import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.types._
@@ -45,7 +42,7 @@ class IgniteDataFrameSchemaSpec extends AbstractDataFrameSpec {
                     ("IS_RESIDENT", BooleanType, true),
                     ("SALARY", DoubleType, true),
                     ("PENSION", DoubleType, true),
-                    ("ACCOUNT", DecimalType(10, 0), true),
+                    ("ACCOUNT", IgniteRDD.DECIMAL, true),
                     ("AGE", IntegerType, true),
                     ("ID", LongType, false),
                     ("CITY_ID", LongType, false))
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSuite.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSuite.scala
index 2ceb44a254faf..b3f702647ce3a 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSuite.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteDataFrameSuite.scala
@@ -28,5 +28,12 @@ class IgniteDataFrameSuite extends Suites (
     new IgniteSQLDataFrameWriteSpec,
     new IgniteSQLDataFrameIgniteSessionWriteSpec,
     new IgniteDataFrameWrongConfigSpec,
-    new IgniteCatalogSpec
+    new IgniteCatalogSpec,
+    new IgniteOptimizationSpec,
+    new IgniteOptimizationStringFuncSpec,
+    new IgniteOptimizationMathFuncSpec,
+    new IgniteOptimizationAggregationFuncSpec,
+    new IgniteOptimizationSystemFuncSpec,
+    new IgniteOptimizationJoinSpec,
+    new IgniteOptimizationDisableEnableSpec
 )
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationAggregationFuncSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationAggregationFuncSpec.scala
new file mode 100644
index 0000000000000..d2527c801758d
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationAggregationFuncSpec.scala
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import java.lang.{Double ⇒ JDouble, Long ⇒ JLong}
+
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+import org.apache.spark.sql.ignite.IgniteSparkSession
+
+/**
+  */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationAggregationFuncSpec extends AbstractDataFrameSpec {
+    var igniteSession: IgniteSparkSession = _
+
+    describe("Supported optimized aggregation functions") {
+        it("COUNT") {
+            val df = igniteSession.sql("SELECT count(*) FROM numbers")
+
+            checkOptimizationResult(df, "SELECT count(1) FROM numbers")
+
+            val data = Tuple1(21)
+
+            checkQueryData(df, data)
+        }
+
+        it("AVG - DECIMAL") {
+            //TODO: write me
+        }
+
+        it("AVG - DOUBLE") {
+            val df = igniteSession.sql("SELECT AVG(val) FROM numbers WHERE id <= 3")
+
+            checkOptimizationResult(df, "SELECT AVG(val) FROM numbers WHERE id IS NOT NULL and id <= 3")
+
+            val data = Tuple1(.5)
+
+            checkQueryData(df, data)
+        }
+
+        it("MIN - DOUBLE") {
+            val df = igniteSession.sql("SELECT MIN(val) FROM numbers")
+
+            checkOptimizationResult(df, "SELECT MIN(val) FROM numbers")
+
+            val data = Tuple1(-1.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("MAX - DOUBLE") {
+            val df = igniteSession.sql("SELECT MAX(val) FROM numbers")
+
+            checkOptimizationResult(df, "SELECT MAX(val) FROM numbers")
+
+            val data = Tuple1(180.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("SUM - DOUBLE") {
+            val df = igniteSession.sql("SELECT SUM(val) FROM numbers WHERE id <= 3")
+
+            checkOptimizationResult(df, "SELECT SUM(val) FROM numbers WHERE id IS NOT NULL and id <= 3")
+
+            val data = Tuple1(1.5)
+
+            checkQueryData(df, data)
+        }
+
+        it("SUM - DECIMAL - 1") {
+            val df = igniteSession.sql("SELECT SUM(decimal_val) FROM numbers WHERE id IN (18, 19, 20)")
+
+            checkOptimizationResult(df, "SELECT SUM(decimal_val) FROM numbers WHERE id IN (18, 19, 20)")
+
+            df.printSchema()
+
+            val data = Tuple1(new java.math.BigDecimal(10.5).setScale(3))
+
+            checkQueryData(df, data)
+        }
+
+        it("SUM - DECIMAL - 2") {
+            val df = igniteSession.sql("SELECT SUM(decimal_val) FROM numbers WHERE id IN (18, 19, 20, 21)")
+
+            checkOptimizationResult(df, "SELECT SUM(decimal_val) FROM numbers WHERE id IN (18, 19, 20, 21)")
+
+            val data = Tuple1(new java.math.BigDecimal(15).setScale(3))
+
+            checkQueryData(df, data)
+        }
+
+        it("SUM - LONG") {
+            val df = igniteSession.sql("SELECT SUM(int_val) FROM numbers WHERE id in (15, 16, 17)")
+
+            checkOptimizationResult(df, "SELECT CAST(SUM(int_val) AS BIGINT) as \"SUM(int_val)\" " +
+                "FROM numbers WHERE id in (15, 16, 17)")
+
+            val data = Tuple1(6L)
+
+            checkQueryData(df, data)
+        }
+    }
+
+    def createNumberTable(client: Ignite, cacheName: String): Unit = {
+        val cache = client.cache(cacheName)
+
+        cache.query(new SqlFieldsQuery(
+            """
+              | CREATE TABLE numbers (
+              |    id LONG,
+              |    val DOUBLE,
+              |    int_val LONG,
+              |    decimal_val DECIMAL(5, 5),
+              |    PRIMARY KEY (id)) WITH "backups=1"
+            """.stripMargin)).getAll
+
+        var qry = new SqlFieldsQuery("INSERT INTO numbers (id, val) values (?, ?)")
+
+        cache.query(qry.setArgs(1L.asInstanceOf[JLong], .0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(2L.asInstanceOf[JLong], .5.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(3L.asInstanceOf[JLong], 1.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(4L.asInstanceOf[JLong], 2.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(5L.asInstanceOf[JLong], 4.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(6L.asInstanceOf[JLong], -0.5.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(7L.asInstanceOf[JLong], -1.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(8L.asInstanceOf[JLong], 42.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(9L.asInstanceOf[JLong], .51.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(10L.asInstanceOf[JLong], .49.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(11L.asInstanceOf[JLong], 100.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(12L.asInstanceOf[JLong], (Math.E*Math.E).asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(13L.asInstanceOf[JLong], Math.PI.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(14L.asInstanceOf[JLong], 180.0.asInstanceOf[JDouble])).getAll
+
+        qry = new SqlFieldsQuery("INSERT INTO numbers (id, int_val) values (?, ?)")
+
+        cache.query(qry.setArgs(15L.asInstanceOf[JLong], 1L.asInstanceOf[JLong])).getAll
+        cache.query(qry.setArgs(16L.asInstanceOf[JLong], 2L.asInstanceOf[JLong])).getAll
+        cache.query(qry.setArgs(17L.asInstanceOf[JLong], 3L.asInstanceOf[JLong])).getAll
+
+        qry = new SqlFieldsQuery("INSERT INTO numbers (id, decimal_val) values (?, ?)")
+
+        cache.query(qry.setArgs(18L.asInstanceOf[JLong], new java.math.BigDecimal(2.5))).getAll
+        cache.query(qry.setArgs(19L.asInstanceOf[JLong], new java.math.BigDecimal(3.5))).getAll
+        cache.query(qry.setArgs(20L.asInstanceOf[JLong], new java.math.BigDecimal(4.5))).getAll
+        cache.query(qry.setArgs(21L.asInstanceOf[JLong], new java.math.BigDecimal(4.5))).getAll
+    }
+
+    override protected def beforeAll(): Unit = {
+        super.beforeAll()
+
+        createNumberTable(client, DEFAULT_CACHE)
+
+        val configProvider = enclose(null) (x ⇒ () ⇒ {
+            val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+            cfg.setClientMode(true)
+
+            cfg.setIgniteInstanceName("client-2")
+
+            cfg
+        })
+
+        igniteSession = IgniteSparkSession.builder()
+            .config(spark.sparkContext.getConf)
+            .igniteConfigProvider(configProvider)
+            .getOrCreate()
+    }
+}
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDateFuncSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDateFuncSpec.scala
new file mode 100644
index 0000000000000..7912cd0307d5b
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDateFuncSpec.scala
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import java.lang.{Long ⇒ JLong}
+import java.util.{Date ⇒ JDate}
+import java.text.SimpleDateFormat
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.TimeUnit.DAYS
+
+/**
+  */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationDateFuncSpec extends AbstractDataFrameSpec {
+    var igniteSession: IgniteSparkSession = _
+
+    val format = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss")
+
+    describe("Supported optimized date functions") {
+        it(" - CURRENT_TIMESTAMP") {
+            val df = igniteSession.sql("SELECT id, CURRENT_TIMESTAMP() FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = df.rdd.collect
+
+            assert(data(0).getAs[JLong]("id") == 1L)
+
+            val date: JDate = data(0).getAs[JDate]("current_timestamp()")
+            val millisDiff = new JDate().getTime - date.getTime
+
+            assert(millisDiff <= 30000)
+        }
+
+        it(" - CURRENT_DATE") {
+            val df = igniteSession.sql("SELECT id, CURRENT_DATE() FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = df.rdd.collect
+
+            assert(data(0).getAs[JLong]("id") == 1L)
+
+            val date: JDate = data(0).getAs[JDate]("current_date()")
+            val dayDiff = DAYS.convert(new JDate().getTime - date.getTime, TimeUnit.MILLISECONDS)
+
+            assert(dayDiff <= 1)
+        }
+
+        it(" - HOUR") {
+            val df = igniteSession.sql("SELECT HOUR(val) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(0)
+
+            checkQueryData(df, data)
+        }
+
+        it(" - MINUTE") {
+            val df = igniteSession.sql("SELECT MINUTE(val) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(0)
+
+            checkQueryData(df, data)
+        }
+
+        it(" - SECOND") {
+            val df = igniteSession.sql("SELECT SECOND(val) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(0)
+
+            checkQueryData(df, data)
+        }
+
+        it(" - MONTH") {
+            val df = igniteSession.sql("SELECT MONTH(val) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(0)
+
+            checkQueryData(df, data)
+        }
+
+        it(" - YEAR") {
+            val df = igniteSession.sql("SELECT YEAR(val) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(2017)
+
+            checkQueryData(df, data)
+        }
+
+        it(" - QUARTER") {
+            val df = igniteSession.sql("SELECT QUARTER(val) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(1)
+
+            checkQueryData(df, data)
+        }
+
+        it(" - WEEK") {
+            val df = igniteSession.sql("SELECT WEEKOFYEAR(val) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(1)
+
+            checkQueryData(df, data)
+        }
+
+        it(" - DAY_OF_MONTH") {
+            val df = igniteSession.sql("SELECT DAYOFMONTH(val) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(1)
+
+            checkQueryData(df, data)
+        }
+
+        it(" - DAY_OF_YEAR") {
+            val df = igniteSession.sql("SELECT DAYOFYEAR(val) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(1)
+
+            checkQueryData(df, data)
+        }
+
+        it(" - DATE_ADD") {
+            val df = igniteSession.sql("SELECT DATE_ADD(val, 2) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(format.parse("03.01.2017 00:00:00"))
+
+            checkQueryData(df, data)
+        }
+
+        it(" - DATEDIFF") {
+            val df = igniteSession.sql("SELECT " +
+                "DATEDIFF(val, TO_DATE('2017-01-02 00:00:00.000', 'yyyy-MM-dd HH:mm:ss.SSS')) FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1(1)
+
+            checkQueryData(df, data)
+        }
+
+        it(" - FORMATDATETIME") {
+            val df = igniteSession.sql("SELECT DATE_FORMAT(val, 'yyyy-MM-dd HH:mm:ss.SSS') FROM dates WHERE id = 1")
+
+            checkOptimizationResult(df)
+
+            val data = Tuple1("2017-01-01 00:00:00.000")
+
+            checkQueryData(df, data)
+        }
+    }
+
+    def createDateTable(client: Ignite, cacheName: String): Unit = {
+        val cache = client.cache(cacheName)
+
+        cache.query(new SqlFieldsQuery(
+            """
+              | CREATE TABLE dates (
+              |    id LONG,
+              |    val DATE,
+              |    PRIMARY KEY (id)) WITH "backups=1"
+            """.stripMargin)).getAll
+
+        val qry = new SqlFieldsQuery("INSERT INTO dates(id, val) values (?, ?)")
+
+        cache.query(qry.setArgs(1L.asInstanceOf[JLong], format.parse("01.01.2017 00:00:00"))).getAll
+    }
+
+    override protected def beforeAll(): Unit = {
+        super.beforeAll()
+
+        createDateTable(client, DEFAULT_CACHE)
+
+        val configProvider = enclose(null) (x ⇒ () ⇒ {
+            val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+            cfg.setClientMode(true)
+
+            cfg.setIgniteInstanceName("client-2")
+
+            cfg
+        })
+
+        igniteSession = IgniteSparkSession.builder()
+            .config(spark.sparkContext.getConf)
+            .igniteConfigProvider(configProvider)
+            .getOrCreate()
+    }
+}
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDisableEnableSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDisableEnableSpec.scala
new file mode 100644
index 0000000000000..033af74e41904
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationDisableEnableSpec.scala
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.spark.AbstractDataFrameSpec.TEST_CONFIG_FILE
+import org.apache.ignite.spark.IgniteDataFrameSettings._
+import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.ignite.IgniteOptimization
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+/**
+  */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationDisableEnableSpec extends AbstractDataFrameSpec {
+    var personDataFrame: DataFrame = _
+
+    describe("Ignite Optimization Disabling/Enabling") {
+        it("should add Ignite Optimization to a session on a first query") {
+            if (spark.sparkContext.isStopped)
+                createSparkSession()
+
+            assert(!igniteOptimizationExists(spark), "Session shouldn't contains IgniteOptimization")
+
+            personDataFrame = spark.read
+                .format(FORMAT_IGNITE)
+                .option(OPTION_CONFIG_FILE, TEST_CONFIG_FILE)
+                .option(OPTION_TABLE, "person")
+                .load()
+
+            assert(igniteOptimizationExists(spark),
+                "Session should contains IgniteOptimization after executing query over Ignite Data Frame")
+
+            spark.stop()
+        }
+
+        it("should remove Ignite Optimization if it disabled at runtime") {
+            if (!spark.sparkContext.isStopped)
+                spark.stop()
+
+            val newSession = SparkSession.builder()
+                .appName("Ignite Optimization check")
+                .master("local")
+                .config("spark.executor.instances", "2")
+                .getOrCreate()
+
+            assert(!igniteOptimizationExists(newSession), "Session shouldn't contains IgniteOptimization")
+
+            var newPersonDataFrame = newSession.read
+                .format(FORMAT_IGNITE)
+                .option(OPTION_CONFIG_FILE, TEST_CONFIG_FILE)
+                .option(OPTION_TABLE, "person")
+                .load()
+
+            assert(igniteOptimizationExists(newSession),
+                "Session should contains IgniteOptimization after executing query over Ignite Data Frame")
+
+
+            newSession.conf.set(OPTION_DISABLE_SPARK_SQL_OPTIMIZATION, "true")
+
+            newPersonDataFrame = newSession.read
+                .format(FORMAT_IGNITE)
+                .option(OPTION_CONFIG_FILE, TEST_CONFIG_FILE)
+                .option(OPTION_TABLE, "person")
+                .load()
+
+            assert(!igniteOptimizationExists(newSession),
+                "Session shouldn't contains IgniteOptimization")
+
+            newSession.close()
+        }
+
+        it("shouldn't add Ignite Optimization to a session when it's disabled") {
+            if (!spark.sparkContext.isStopped)
+                spark.stop()
+
+            val newSession = SparkSession.builder()
+                .appName("Ignite Optimization check")
+                .master("local")
+                .config("spark.executor.instances", "2")
+                .config(OPTION_DISABLE_SPARK_SQL_OPTIMIZATION, "true")
+                .getOrCreate()
+
+            assert(!igniteOptimizationExists(newSession), "Session shouldn't contains IgniteOptimization")
+
+            val newPersonDataFrame = newSession.read
+                .format(FORMAT_IGNITE)
+                .option(OPTION_CONFIG_FILE, TEST_CONFIG_FILE)
+                .option(OPTION_TABLE, "person")
+                .load()
+
+            newPersonDataFrame.createOrReplaceTempView("person")
+
+            val res = newSession.sqlContext.sql("SELECT name FROM person WHERE id = 2").rdd
+
+            res.count should equal(1)
+
+            assert(!igniteOptimizationExists(newSession), "Session shouldn't contains IgniteOptimization")
+
+            newSession.close()
+        }
+    }
+
+    def igniteOptimizationExists(session: SparkSession): Boolean =
+        session.sessionState.experimentalMethods.extraOptimizations.contains(IgniteOptimization)
+
+    override protected def beforeAll(): Unit = {
+        super.beforeAll()
+
+        createPersonTable(client, "cache1")
+    }
+}
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationJoinSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationJoinSpec.scala
new file mode 100644
index 0000000000000..b4b36a8627a6e
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationJoinSpec.scala
@@ -0,0 +1,543 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+
+import java.lang.{Long ⇒ JLong}
+
+/**
+  */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationJoinSpec extends AbstractDataFrameSpec {
+    var igniteSession: IgniteSparkSession = _
+
+    describe("Optimized join queries") {
+        it("UNION") {
+            val qry =
+                """
+                  | SELECT id, val1 as val FROM jt1 UNION
+                  | SELECT id, val2 as val FROM jt2 UNION
+                  | SELECT id, val3 as val FROM jt3
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df,
+                "SELECT id, val FROM (SELECT id, val1 as val FROM jt1 UNION " +
+                    "SELECT id, val2 as val FROM jt2 UNION " +
+                    "SELECT id, val3 as val FROM jt3) table1")
+
+            val data = (
+                (1L, "A"),
+                (1L, "B"),
+                (2L, "B"),
+                (2L, "C"),
+                (2L, "D"),
+                (3L, "C"),
+                (3L, "D"),
+                (3L, "E"))
+
+            checkQueryData(df, data, row ⇒ (row.getAs[JLong](0), row.getAs[String](1)))
+        }
+
+        it("UNION ALL") {
+            val qry =
+                """
+                  | SELECT id, val1 as val FROM jt1 UNION ALL
+                  | SELECT id, val2 as val FROM jt2 UNION ALL
+                  | SELECT id, val3 as val FROM jt3
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df,
+                "SELECT id, val1 as val FROM jt1 UNION " +
+                    "SELECT id, val2 as val FROM jt2 UNION " +
+                    "SELECT id, val3 as val FROM jt3")
+
+            val data = (
+                (1L, "A"),
+                (1L, "B"),
+                (2L, "B"),
+                (2L, "C"),
+                (2L, "D"),
+                (3L, "C"),
+                (3L, "D"),
+                (3L, "E"))
+
+            checkQueryData(df, data, row ⇒ (row.getAs[JLong](0), row.getAs[String](1)))
+        }
+
+        it("UNION ALL ORDER") {
+            val qry =
+                """
+                  | SELECT id, val1 as val FROM jt1 UNION ALL
+                  | SELECT id, val2 as val FROM jt2 UNION ALL
+                  | SELECT id, val3 as val FROM jt3
+                  | ORDER BY id DESC, val
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df,
+                "SELECT id, val1 as val FROM jt1 UNION " +
+                    "SELECT id, val2 as val FROM jt2 UNION " +
+                    "SELECT id, val3 as val FROM jt3 " +
+                    "ORDER BY id DESC, val")
+
+            val data = (
+                (3L, "C"),
+                (3L, "D"),
+                (3L, "E"),
+                (2L, "B"),
+                (2L, "C"),
+                (2L, "D"),
+                (1L, "A"),
+                (1L, "B")
+            )
+
+            checkQueryData(df, data, _ ⇒ 0)
+        }
+
+        it("UNION WITH AGGREGATE") {
+            val qry =
+                """
+                  | SELECT VAL, COUNT(*) FROM (
+                  |     SELECT id, val1 as val FROM jt1 UNION
+                  |     SELECT id, val2 as val FROM jt2 UNION
+                  |     SELECT id, val3 as val FROM jt3 ) t1
+                  | GROUP BY val HAVING COUNT(*) > 1
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df,
+                "SELECT VAL, count(1) FROM (" +
+                    "SELECT id, val1 AS val FROM JT1 UNION " +
+                    "SELECT id, val2 AS val FROM JT2 UNION " +
+                    "SELECT id, val3 AS val FROM JT3" +
+                ") table1 GROUP BY val HAVING count(1) > 1")
+
+            val data = (
+                ("B", 2L),
+                ("C", 2L),
+                ("D", 2L)
+            )
+
+            checkQueryData(df, data)
+        }
+
+        it("AGGREGATE ON AGGREGATE RESULT") {
+            val qry =
+                """
+                  | SELECT SUM(cnt) FROM (
+                  |     SELECT VAL, COUNT(*) as CNT FROM (
+                  |         SELECT id, val1 as val FROM jt1 UNION
+                  |         SELECT id, val2 as val FROM jt2 UNION
+                  |         SELECT id, val3 as val FROM jt3 ) t1
+                  |     GROUP BY val HAVING COUNT(*) > 1
+                  | ) t1
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df,
+                "SELECT CAST(SUM(cnt) as BIGINT) as \"SUM(cnt)\" FROM (" +
+                    "SELECT count(1) as cnt FROM (" +
+                        "SELECT id, val1 as val FROM jt1 UNION " +
+                        "SELECT id, val2 as val FROM jt2 UNION " +
+                        "SELECT id, val3 as val FROM jt3" +
+                    ") table1 GROUP BY val HAVING count(1) > 1) table2")
+
+            val data = Tuple1(6.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("SELF INNER JOIN") {
+            val qry =
+                """
+                  |SELECT
+                  | jt1.id,
+                  | jt1.val1,
+                  | jt2.id,
+                  | jt2.val1
+                  |FROM
+                  |     jt1 JOIN
+                  |     jt1 as jt2 ON jt1.val1 = jt2.val1
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df, "SELECT JT1.ID, JT1.VAL1, table1.ID, table1.VAL1 " +
+                "FROM JT1 JOIN JT1 AS table1 ON jt1.val1 = table1.val1 " +
+                "WHERE jt1.val1 IS NOT NULL AND table1.val1 IS NOT NULL")
+
+            val data = (
+                (1, "A", 1, "A"),
+                (2, "B", 2, "B"),
+                (3, "C", 3, "C")
+            )
+
+            checkQueryData(df, data)
+        }
+
+
+        it("SELF INNER JOIN WITH WHERE") {
+            val qry =
+                """
+                  |SELECT
+                  | jt1.id,
+                  | jt1.val1,
+                  | jt2.id,
+                  | jt2.val1
+                  |FROM
+                  |     jt1 JOIN
+                  |     jt1 as jt2 ON jt1.val1 = jt2.val1
+                  |WHERE jt2.val1 = 'A'
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+/*            checkOptimizationResult(df, "SELECT JT1.ID, JT1.VAL1, table1.ID, table1.VAL1 " +
+                "FROM JT1 JOIN JT1 as table1 ON JT1.val1 = table1.val1 " +
+                "WHERE JT1.val1 = 'A' AND JT1.val1 IS NOT NULL AND table1.val1 IS NOT NULL AND table1.val1 = 'A'")*/
+
+            val data = Tuple1(
+                (1, "A", 1, "A")
+            )
+
+            checkQueryData(df, data)
+        }
+
+
+        it("INNER JOIN") {
+            val qry =
+                """
+                  |SELECT
+                  | jt1.id as id1,
+                  | jt1.val1,
+                  | jt2.id as id2,
+                  | jt2.val2
+                  |FROM
+                  |     jt1 JOIN
+                  |     jt2 ON jt1.val1 = jt2.val2
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df, "SELECT JT1.ID AS id1, JT1.VAL1, JT2.ID AS id2, JT2.VAL2 " +
+                "FROM JT1 JOIN JT2 ON jt1.val1 = jt2.val2 " +
+                "WHERE jt1.val1 IS NOT NULL AND jt2.val2 IS NOT NULL")
+
+            val data = (
+                (2, "B", 1, "B"),
+                (3, "C", 2, "C")
+            )
+
+            checkQueryData(df, data)
+        }
+
+        it("INNER JOIN WITH WHERE") {
+            val qry =
+                """
+                  |SELECT
+                  | jt1.id as id1,
+                  | jt1.val1,
+                  | jt2.id as id2,
+                  | jt2.val2
+                  |FROM
+                  |     jt1 JOIN
+                  |     jt2 ON jt1.val1 = jt2.val2
+                  |WHERE
+                  | jt1.id < 10
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df, "SELECT jt1.id as id1, jt1.val1, jt2.id as id2, jt2.val2 " +
+                "FROM jt1 JOIN jt2 ON jt1.val1 = jt2.val2 " +
+                "WHERE jt1.id IS NOT NULL AND jt1.id < 10 AND jt1.val1 IS NOT NULL and jt2.val2 IS NOT NULL")
+
+            val data = (
+                (2, "B", 1, "B"),
+                (3, "C", 2, "C")
+            )
+
+            checkQueryData(df, data)
+        }
+
+        it("LEFT JOIN") {
+            val qry =
+                """
+                  |SELECT
+                  | jt1.id as id1,
+                  | jt1.val1,
+                  | jt2.id as id2,
+                  | jt2.val2
+                  |FROM
+                  | jt1 LEFT JOIN
+                  | jt2 ON jt1.val1 = jt2.val2
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df, "SELECT jt1.id as id1, jt1.val1, jt2.id as id2, jt2.val2 " +
+                "FROM jt1 LEFT JOIN jt2 ON jt1.val1 = jt2.val2")
+
+            val data = (
+                (1, "A", null, null),
+                (2, "B", 1, "B"),
+                (3, "C", 2, "C")
+            )
+
+            checkQueryData(df, data)
+        }
+
+        it("RIGHT JOIN") {
+            val qry =
+                """
+                  |SELECT
+                  | jt1.id as id1,
+                  | jt1.val1,
+                  | jt2.id as id2,
+                  | jt2.val2
+                  |FROM
+                  | jt1 RIGHT JOIN
+                  | jt2 ON jt1.val1 = jt2.val2
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df, "SELECT jt1.id as id1, jt1.val1, jt2.id as id2, jt2.val2 " +
+                "FROM jt1 RIGHT JOIN jt2 ON jt1.val1 = jt2.val2")
+
+            val data = (
+                (2, "B", 1, "B"),
+                (3, "C", 2, "C"),
+                (null, null, 3, "D")
+            )
+
+            checkQueryData(df, data, r ⇒ if (r.get(0) == null) 100L else r.getAs[Long](0))
+        }
+
+        it("JOIN 3 TABLE") {
+            val qry =
+                """
+                  |SELECT
+                  | jt1.id as id1,
+                  | jt1.val1 as val1,
+                  | jt2.id as id2,
+                  | jt2.val2 as val2,
+                  | jt3.id as id3,
+                  | jt3.val3 as val3
+                  |FROM
+                  | jt1 LEFT JOIN
+                  | jt2 ON jt1.val1 = jt2.val2 LEFT JOIN
+                  | jt3 ON jt1.val1 = jt3.val3
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df,
+                "SELECT table1.id as id1, table1.val1, table1.id_2 as id2, table1.val2, jt3.id as id3, jt3.val3 " +
+                    "FROM (" +
+                    "SELECT jt1.val1, jt1.id, jt2.val2, jt2.id as id_2 " +
+                    "FROM JT1 LEFT JOIN jt2 ON jt1.val1 = jt2.val2) table1 LEFT JOIN " +
+                    "jt3 ON table1.val1 = jt3.val3")
+
+            val data = (
+                (1, "A", null, null, 1, "A"),
+                (2, "B", 1, "B", null, null),
+                (3, "C", 2, "C", null, null))
+
+            checkQueryData(df, data)
+        }
+
+        it("JOIN 3 TABLE AND AGGREGATE") {
+            val qry =
+                """
+                  |SELECT SUM(id1) FROM (
+                  | SELECT
+                  |     jt1.id as id1,
+                  |     jt1.val1 as val1,
+                  |     jt2.id as id2,
+                  |     jt2.val2 as val2,
+                  |     jt3.id as id3,
+                  |     jt3.val3 as val3
+                  |FROM
+                  |     jt1 LEFT JOIN
+                  |     jt2 ON jt1.val1 = jt2.val2 LEFT JOIN
+                  |     jt3 ON jt1.val1 = jt3.val3
+                  |) WHERE CONCAT(val1, val2) = 'BB' OR CONCAT(val1, val3) = 'AA'
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df,
+            "SELECT CAST(SUM(table1.ID) AS BIGINT) AS \"sum(id1)\" FROM " +
+                "(SELECT JT1.VAL1, JT1.ID, JT2.VAL2 FROM JT1 LEFT JOIN JT2 ON JT1.val1 = JT2.val2) table1 LEFT JOIN " +
+                "JT3 ON table1.val1 = JT3.val3 " +
+                "WHERE CONCAT(table1.val1, table1.val2) = 'BB' OR CONCAT(table1.val1, JT3.val3) = 'AA'")
+
+            val data = Tuple1(3)
+
+            checkQueryData(df, data, _ ⇒ 0)
+        }
+
+        it("INNER JOIN SUBQUERY") {
+            val qry =
+                """
+                  |SELECT sum_id, val1, val2 FROM (
+                  |  SELECT
+                  |    jt1.id + jt2.id as sum_id,
+                  |    jt1.val1 as val1,
+                  |    jt2.val2 as val2
+                  |  FROM
+                  |     jt1 JOIN
+                  |     jt2 ON jt1.val1 = jt2.val2
+                  |) t1 WHERE sum_id != 15
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df,
+                "SELECT jt1.id + jt2.id as sum_id, jt1.val1, jt2.val2 FROM " +
+                    "jt1 JOIN jt2 ON NOT jt1.id + jt2.id = 15 AND jt1.val1 = jt2.val2 " +
+                    "WHERE " +
+                    "jt1.id IS NOT NULL AND " +
+                    "jt1.val1 IS NOT NULL AND " +
+                    "jt2.id IS NOT NULL AND " +
+                    "jt2.val2 IS NOT NULL"
+            )
+
+            val data = (
+                (3, "B", "B"),
+                (5, "C", "C")
+            )
+
+            checkQueryData(df, data)
+        }
+
+        it("INNER JOIN SUBQUERY - 2") {
+            val qry =
+                """
+                  |SELECT SUM(sum_id) FROM (
+                  |  SELECT
+                  |    jt1.id + jt2.id as sum_id
+                  |  FROM
+                  |     jt1 JOIN
+                  |     jt2 ON jt1.val1 = jt2.val2
+                  |) t1 WHERE sum_id != 15
+                  |""".stripMargin
+
+            val df = igniteSession.sql(qry)
+
+            checkOptimizationResult(df,
+                "SELECT CAST(SUM(JT1.ID + JT2.ID) AS BIGINT) AS \"sum(sum_id)\" " +
+                    "FROM JT1 JOIN JT2 ON NOT JT1.id + JT2.id = 15 AND JT1.val1 = JT2.val2 " +
+                    "WHERE JT1.id IS NOT NULL AND JT1.val1 IS NOT NULL AND JT2.id IS NOT NULL AND JT2.val2 IS NOT NULL")
+
+            val data = Tuple1(8)
+
+            checkQueryData(df, data)
+        }
+    }
+
+    def createJoinedTables(client: Ignite, cacheName: String): Unit = {
+        val cache = client.cache(cacheName)
+
+        cache.query(new SqlFieldsQuery(
+            """
+              | CREATE TABLE jt1 (
+              |    id LONG,
+              |    val1 VARCHAR,
+              |    PRIMARY KEY (id)) WITH "backups=1"
+            """.stripMargin)).getAll
+
+        cache.query(new SqlFieldsQuery(
+            """
+              | CREATE TABLE jt2 (
+              |    id LONG,
+              |    val2 VARCHAR,
+              |    PRIMARY KEY (id)) WITH "backups=1"
+            """.stripMargin)).getAll
+
+        cache.query(new SqlFieldsQuery(
+            """
+              | CREATE TABLE jt3 (
+              |    id LONG,
+              |    val3 VARCHAR,
+              |    PRIMARY KEY (id)) WITH "backups=1"
+            """.stripMargin)).getAll
+
+        var qry = new SqlFieldsQuery("INSERT INTO jt1 (id, val1) values (?, ?)")
+
+        cache.query(qry.setArgs(1L.asInstanceOf[JLong], "A")).getAll
+        cache.query(qry.setArgs(2L.asInstanceOf[JLong], "B")).getAll
+        cache.query(qry.setArgs(3L.asInstanceOf[JLong], "C")).getAll
+
+        qry = new SqlFieldsQuery("INSERT INTO jt2 (id, val2) values (?, ?)")
+
+        cache.query(qry.setArgs(1L.asInstanceOf[JLong], "B")).getAll
+        cache.query(qry.setArgs(2L.asInstanceOf[JLong], "C")).getAll
+        cache.query(qry.setArgs(3L.asInstanceOf[JLong], "D")).getAll
+
+        qry = new SqlFieldsQuery("INSERT INTO jt3 (id, val3) values (?, ?)")
+
+        cache.query(qry.setArgs(1L.asInstanceOf[JLong], "A")).getAll
+        cache.query(qry.setArgs(2L.asInstanceOf[JLong], "D")).getAll
+        cache.query(qry.setArgs(3L.asInstanceOf[JLong], "E")).getAll
+
+        cache.query(new SqlFieldsQuery("CREATE INDEX idx1 ON jt1(val1)")).getAll
+        cache.query(new SqlFieldsQuery("CREATE INDEX idx2 ON jt2(val2)")).getAll
+        cache.query(new SqlFieldsQuery("CREATE INDEX idx3 ON jt3(val3)")).getAll
+    }
+
+    override protected def beforeAll(): Unit = {
+        super.beforeAll()
+
+        createPersonTable(client, DEFAULT_CACHE)
+
+        createCityTable(client, DEFAULT_CACHE)
+
+        createJoinedTables(client, DEFAULT_CACHE)
+
+        val configProvider = enclose(null) (x ⇒ () ⇒ {
+            val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+            cfg.setClientMode(true)
+
+            cfg.setIgniteInstanceName("client-2")
+
+            cfg
+        })
+
+        igniteSession = IgniteSparkSession.builder()
+            .config(spark.sparkContext.getConf)
+            .igniteConfigProvider(configProvider)
+            .getOrCreate()
+    }
+}
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationMathFuncSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationMathFuncSpec.scala
new file mode 100644
index 0000000000000..02793c94f9570
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationMathFuncSpec.scala
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import java.lang.{Double ⇒ JDouble, Long ⇒ JLong}
+
+/**
+  */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationMathFuncSpec extends AbstractDataFrameSpec {
+    var igniteSession: IgniteSparkSession = _
+
+    describe("Supported optimized string functions") {
+        it("ABS") {
+            val df = igniteSession.sql("SELECT ABS(val) FROM numbers WHERE id = 6")
+
+            checkOptimizationResult(df, "SELECT ABS(val) FROM numbers WHERE id is not null AND id = 6")
+
+            val data = Tuple1(.5)
+
+            checkQueryData(df, data)
+        }
+
+        it("ACOS") {
+            val df = igniteSession.sql("SELECT ACOS(val) FROM numbers WHERE id = 7")
+
+            checkOptimizationResult(df, "SELECT ACOS(val) FROM numbers WHERE id is not null AND id = 7")
+
+            val data = Tuple1(Math.PI)
+
+            checkQueryData(df, data)
+        }
+
+        it("ASIN") {
+            val df = igniteSession.sql("SELECT ASIN(val) FROM numbers WHERE id = 7")
+
+            checkOptimizationResult(df, "SELECT ASIN(val) FROM numbers WHERE id is not null AND id = 7")
+
+            val data = Tuple1(-Math.PI/2)
+
+            checkQueryData(df, data)
+        }
+
+        it("ATAN") {
+            val df = igniteSession.sql("SELECT ATAN(val) FROM numbers WHERE id = 7")
+
+            checkOptimizationResult(df, "SELECT ATAN(val) FROM numbers WHERE id is not null AND id = 7")
+
+            val data = Tuple1(-Math.PI/4)
+
+            checkQueryData(df, data)
+        }
+
+        it("COS") {
+            val df = igniteSession.sql("SELECT COS(val) FROM numbers WHERE id = 1")
+
+            checkOptimizationResult(df, "SELECT COS(val) FROM numbers WHERE id is not null AND id = 1")
+
+            val data = Tuple1(1.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("SIN") {
+            val df = igniteSession.sql("SELECT SIN(val) FROM numbers WHERE id = 1")
+
+            checkOptimizationResult(df, "SELECT SIN(val) FROM numbers WHERE id is not null AND id = 1")
+
+            val data = Tuple1(.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("TAN") {
+            val df = igniteSession.sql("SELECT TAN(val) FROM numbers WHERE id = 1")
+
+            checkOptimizationResult(df, "SELECT TAN(val) FROM numbers WHERE id is not null AND id = 1")
+
+            val data = Tuple1(.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("COSH") {
+            val df = igniteSession.sql("SELECT COSH(val) FROM numbers WHERE id = 1")
+
+            checkOptimizationResult(df, "SELECT COSH(val) FROM numbers WHERE id is not null AND id = 1")
+
+            val data = Tuple1(1.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("SINH") {
+            val df = igniteSession.sql("SELECT SINH(val) FROM numbers WHERE id = 1")
+
+            checkOptimizationResult(df, "SELECT SINH(val) FROM numbers WHERE id is not null AND id = 1")
+
+            val data = Tuple1(.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("TANH") {
+            val df = igniteSession.sql("SELECT TANH(val) FROM numbers WHERE id = 1")
+
+            checkOptimizationResult(df, "SELECT TANH(val) FROM numbers WHERE id is not null AND id = 1")
+
+            val data = Tuple1(.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("ATAN2") {
+            val df = igniteSession.sql("SELECT ATAN2(val, 0.0) FROM numbers WHERE id = 1")
+
+            checkOptimizationResult(df, "SELECT ATAN2(val, 0.0) AS \"ATAN2(val, CAST(0.0 AS DOUBLE))\" " +
+                "FROM numbers WHERE id is not null AND id = 1")
+
+            val data = Tuple1(.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("MOD") {
+            val df = igniteSession.sql("SELECT val % 9 FROM numbers WHERE id = 8")
+
+            checkOptimizationResult(df, "SELECT val % 9.0 as \"(val % CAST(9 AS DOUBLE))\" " +
+                "FROM numbers WHERE id is not null AND id = 8")
+
+            val data = Tuple1(6.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("CEIL") {
+            val df = igniteSession.sql("SELECT CEIL(val) FROM numbers WHERE id = 2")
+
+            checkOptimizationResult(df, "SELECT CAST(CEIL(val) AS LONG) as \"CEIL(val)\" " +
+                "FROM numbers WHERE id is not null AND id = 2")
+
+            val data = Tuple1(1)
+
+            checkQueryData(df, data)
+        }
+
+        it("ROUND") {
+        val df = igniteSession.sql("SELECT id, ROUND(val) FROM numbers WHERE  id IN (2, 9, 10)")
+
+            checkOptimizationResult(df, "SELECT id, ROUND(val, 0) FROM numbers WHERE id IN (2, 9, 10)")
+
+            val data = (
+                (2, 1.0),
+                (9, 1.0),
+                (10, 0.0))
+
+            checkQueryData(df, data)
+        }
+
+        it("FLOOR") {
+            val df = igniteSession.sql("SELECT FLOOR(val) FROM numbers WHERE id = 2")
+
+            checkOptimizationResult(df, "SELECT CAST(FLOOR(val) AS LONG) as \"FLOOR(val)\" FROM numbers " +
+                "WHERE id is not null AND id = 2")
+
+            val data = Tuple1(0)
+
+            checkQueryData(df, data)
+        }
+
+        it("POWER") {
+            val df = igniteSession.sql("SELECT POWER(val, 3) FROM numbers WHERE id = 4")
+
+            checkOptimizationResult(df, "SELECT POWER(val, 3.0) as \"POWER(val, CAST(3 AS DOUBLE))\" FROM numbers " +
+                "WHERE id is not null AND id = 4")
+
+            val data = Tuple1(8.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("EXP") {
+            val df = igniteSession.sql("SELECT id, EXP(val) FROM numbers WHERE id IN (1, 3)")
+
+            checkOptimizationResult(df, "SELECT id, EXP(val) FROM numbers WHERE id IN (1, 3)")
+
+            val data = (
+                (1, 1),
+                (3, Math.E))
+
+            checkQueryData(df, data)
+        }
+
+        it("LOG") {
+            val df = igniteSession.sql("SELECT LOG(val) FROM numbers WHERE id = 12")
+
+            checkOptimizationResult(df, "SELECT LOG(val) as \"LOG(E(), val)\" FROM numbers " +
+                "WHERE id IS NOT NULL AND id = 12")
+
+            val data = Tuple1(2.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("LOG10") {
+            val df = igniteSession.sql("SELECT LOG10(val) FROM numbers WHERE id = 11")
+
+            checkOptimizationResult(df, "SELECT LOG10(val) FROM numbers WHERE id IS NOT NULL AND id = 11")
+
+            val data = Tuple1(2.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("DEGREES") {
+            val df = igniteSession.sql("SELECT DEGREES(val) FROM numbers WHERE id = 13")
+
+            checkOptimizationResult(df, "SELECT DEGREES(val) FROM numbers WHERE id IS NOT NULL AND id = 13")
+
+            val data = Tuple1(180.0)
+
+            checkQueryData(df, data)
+        }
+
+        it("RADIANS") {
+            val df = igniteSession.sql("SELECT RADIANS(val) FROM numbers WHERE id = 14")
+
+            checkOptimizationResult(df, "SELECT RADIANS(val) FROM numbers WHERE id IS NOT NULL AND id = 14")
+
+            val data = Tuple1(Math.PI)
+
+            checkQueryData(df, data)
+        }
+
+        it("BITAND") {
+            val df = igniteSession.sql("SELECT int_val&1 FROM numbers WHERE id = 15")
+
+            checkOptimizationResult(df, "SELECT BITAND(int_val, 1) as \"(int_val & CAST(1 AS BIGINT))\" FROM numbers " +
+                "WHERE id IS NOT NULL AND id = 15")
+
+            val data = Tuple1(1)
+
+            checkQueryData(df, data)
+        }
+
+        it("BITOR") {
+            val df = igniteSession.sql("SELECT int_val|1 FROM numbers WHERE id = 16")
+
+            checkOptimizationResult(df, "SELECT BITOR(int_val, 1) as \"(int_val | CAST(1 AS BIGINt))\" FROM numbers " +
+                "WHERE id IS NOT NULL AND id = 16")
+
+            val data = Tuple1(3)
+
+            checkQueryData(df, data)
+        }
+
+        it("BITXOR") {
+            val df = igniteSession.sql("SELECT int_val^1 FROM numbers WHERE id = 17")
+
+            checkOptimizationResult(df, "SELECT BITXOR(int_val, 1) AS \"(int_val ^ CAST(1 AS BIGINT))\" FROM numbers " +
+                "WHERE id IS NOT NULL AND id = 17")
+
+            val data = Tuple1(2)
+
+            checkQueryData(df, data)
+        }
+
+        it("RAND") {
+            val df = igniteSession.sql("SELECT id, RAND(1) FROM numbers WHERE id = 17")
+
+            checkOptimizationResult(df, "SELECT id, RAND(1) FROM numbers WHERE id IS NOT NULL AND id = 17")
+
+            val data = df.rdd.collect
+
+            assert(data(0).getAs[JLong]("id") == 17L)
+            assert(data(0).getAs[JDouble]("rand(1)") != null)
+        }
+    }
+
+    def createNumberTable(client: Ignite, cacheName: String): Unit = {
+        val cache = client.cache(cacheName)
+
+        cache.query(new SqlFieldsQuery(
+            """
+              | CREATE TABLE numbers (
+              |    id LONG,
+              |    val DOUBLE,
+              |    int_val LONG,
+              |    PRIMARY KEY (id)) WITH "backups=1"
+            """.stripMargin)).getAll
+
+        var qry = new SqlFieldsQuery("INSERT INTO numbers (id, val) values (?, ?)")
+
+        cache.query(qry.setArgs(1L.asInstanceOf[JLong], .0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(2L.asInstanceOf[JLong], .5.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(3L.asInstanceOf[JLong], 1.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(4L.asInstanceOf[JLong], 2.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(5L.asInstanceOf[JLong], 4.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(6L.asInstanceOf[JLong], -0.5.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(7L.asInstanceOf[JLong], -1.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(8L.asInstanceOf[JLong], 42.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(9L.asInstanceOf[JLong], .51.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(10L.asInstanceOf[JLong], .49.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(11L.asInstanceOf[JLong], 100.0.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(12L.asInstanceOf[JLong], (Math.E*Math.E).asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(13L.asInstanceOf[JLong], Math.PI.asInstanceOf[JDouble])).getAll
+        cache.query(qry.setArgs(14L.asInstanceOf[JLong], 180.0.asInstanceOf[JDouble])).getAll
+
+        qry = new SqlFieldsQuery("INSERT INTO numbers (id, int_val) values (?, ?)")
+
+        cache.query(qry.setArgs(15L.asInstanceOf[JLong], 1L.asInstanceOf[JLong])).getAll
+        cache.query(qry.setArgs(16L.asInstanceOf[JLong], 2L.asInstanceOf[JLong])).getAll
+        cache.query(qry.setArgs(17L.asInstanceOf[JLong], 3L.asInstanceOf[JLong])).getAll
+    }
+
+    override protected def beforeAll(): Unit = {
+        super.beforeAll()
+
+        createNumberTable(client, DEFAULT_CACHE)
+
+        val configProvider = enclose(null) (x ⇒ () ⇒ {
+            val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+            cfg.setClientMode(true)
+
+            cfg.setIgniteInstanceName("client-2")
+
+            cfg
+        })
+
+        igniteSession = IgniteSparkSession.builder()
+            .config(spark.sparkContext.getConf)
+            .igniteConfigProvider(configProvider)
+            .getOrCreate()
+    }
+}
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
new file mode 100644
index 0000000000000..00075f46925e6
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expdfs or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.internal.util.IgniteUtils.resolveIgnitePath
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+
+/**
+  */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationSpec extends AbstractDataFrameSpec {
+    var igniteSession: IgniteSparkSession = _
+
+    describe("Optimized queries") {
+        it("SELECT name as city_name FROM city") {
+            val df = igniteSession.sql("SELECT name as city_name FROM city")
+
+            checkOptimizationResult(df, "SELECT name as city_name FROM city")
+        }
+
+        it("SELECT count(*) as city_count FROM city") {
+            val df = igniteSession.sql("SELECT count(1) as city_count FROM city")
+
+            checkOptimizationResult(df, "SELECT count(1) as city_count FROM city")
+        }
+
+        it("SELECT count(*), city_id FROM person p GROUP BY city_id") {
+            val df = igniteSession.sql("SELECT city_id, count(*) FROM person GROUP BY city_id")
+
+            checkOptimizationResult(df, "SELECT city_id, count(1) FROM person GROUP BY city_id")
+
+            val data = (
+                (1, 1),
+                (2, 3),
+                (3, 1)
+            )
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT id, name FROM person WHERE id > 3 ORDER BY id") {
+            val df = igniteSession.sql("SELECT id, name FROM person WHERE id > 3 ORDER BY id")
+
+            checkOptimizationResult(df, "SELECT id, name FROM person WHERE id IS NOT NULL AND id > 3 ORDER BY id")
+
+            val data = (
+                (4, "Richard Miles"),
+                (5, null))
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT id, name FROM person WHERE id > 3 ORDER BY id DESC") {
+            val df = igniteSession.sql("SELECT id, name FROM person WHERE id > 3 ORDER BY id DESC")
+
+            checkOptimizationResult(df, "SELECT id, name FROM person WHERE id IS NOT NULL AND id > 3 ORDER BY id DESC")
+
+            val data = (
+                (5, null),
+                (4, "Richard Miles"))
+
+            checkQueryData(df, data, -_.getAs[Long]("id"))
+        }
+
+        it("SELECT id, test_reverse(name) FROM city ORDER BY id") {
+            igniteSession.udf.register("test_reverse", (str: String) ⇒ str.reverse)
+
+            val df = igniteSession.sql("SELECT id, test_reverse(name) FROM city ORDER BY id")
+
+            checkOptimizationResult(df, "SELECT name, id FROM city")
+
+            val data = (
+                (1, "Forest Hill".reverse),
+                (2, "Denver".reverse),
+                (3, "St. Petersburg".reverse),
+                (4, "St. Petersburg".reverse))
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT count(*), city_id FROM person p GROUP BY city_id HAVING count(*) > 1") {
+            val df = igniteSession.sql("SELECT city_id, count(*) FROM person p GROUP BY city_id HAVING count(*) > 1")
+
+            checkOptimizationResult(df, "SELECT city_id, count(1) FROM person GROUP BY city_id HAVING count(1) > 1")
+
+            val data = Tuple1(
+                (2, 3))
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT id FROM city HAVING id > 1") {
+            val df = igniteSession.sql("SELECT id FROM city HAVING id > 1")
+
+            checkOptimizationResult(df, "SELECT id FROM city WHERE id IS NOT NULL AND id > 1")
+
+            val data = (2, 3, 4)
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT DISTINCT name FROM city ORDER BY name") {
+            val df = igniteSession.sql("SELECT DISTINCT name FROM city ORDER BY name")
+
+            checkOptimizationResult(df, "SELECT name FROM city GROUP BY name ORDER BY name")
+
+            val data = ("Denver", "Forest Hill", "St. Petersburg")
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT id, name FROM city ORDER BY id, name") {
+            val df = igniteSession.sql("SELECT id, name FROM city ORDER BY id, name")
+
+            checkOptimizationResult(df, "SELECT id, name FROM city ORDER BY id, name")
+
+            val data = (
+                (1, "Forest Hill"),
+                (2, "Denver"),
+                (3, "St. Petersburg"),
+                (4, "St. Petersburg"))
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT id, name FROM city WHERE id > 1 ORDER BY id") {
+            val df = igniteSession.sql("SELECT id, name FROM city WHERE id > 1 ORDER BY id")
+
+            checkOptimizationResult(df, "SELECT id, name FROM city WHERE id IS NOT NULL and id > 1 ORDER BY id")
+
+            val data = (
+                (2, "Denver"),
+                (3, "St. Petersburg"),
+                (4, "St. Petersburg"))
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT count(*) FROM city") {
+            val df = igniteSession.sql("SELECT count(*) FROM city")
+
+            checkOptimizationResult(df, "SELECT count(1) FROM city")
+
+            val data = Tuple1(4)
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT count(DISTINCT name)  FROM city") {
+            val df = igniteSession.sql("SELECT count(DISTINCT name) FROM city")
+
+            checkOptimizationResult(df, "SELECT count(DISTINCT name) FROM city")
+
+            val data = Tuple1(3)
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT id FROM city LIMIT 2") {
+            val df = igniteSession.sql("SELECT id FROM city LIMIT 2")
+
+            checkOptimizationResult(df, "SELECT id FROM city LIMIT 2")
+
+            val data = (1, 2)
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT CAST(id AS STRING) FROM city") {
+            val df = igniteSession.sql("SELECT CAST(id AS STRING) FROM city")
+
+            checkOptimizationResult(df, "SELECT CAST(id AS varchar) as id FROM city")
+
+            val data = ("1", "2", "3", "4")
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT SQRT(id) FROM city WHERE id = 4 OR id = 1") {
+            val df = igniteSession.sql("SELECT SQRT(id) FROM city WHERE id = 4 OR id = 1")
+
+            checkOptimizationResult(df,
+                "SELECT SQRT(cast(id as double)) FROM city WHERE id = 4 OR id = 1")
+
+            val data = (1, 2)
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT CONCAT(id, \" - this is ID\") FROM city") {
+            val df = igniteSession.sql("SELECT CONCAT(id, \" - this is ID\") FROM city")
+
+            checkOptimizationResult(df,
+                "SELECT CONCAT(cast(id AS VARCHAR), ' - this is ID') as \"CONCAT(cast(id AS STRING),  - this is ID)\" " +
+                    "FROM city")
+
+            val data = (
+                "1 - this is ID",
+                "2 - this is ID",
+                "3 - this is ID",
+                "4 - this is ID")
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT id FROM city WHERE CONCAT(id, \" - this is ID\") = \"1 - this is ID\"") {
+            val df = igniteSession.sql("SELECT id FROM city WHERE CONCAT(id, \" - this is ID\") = \"1 - this is ID\"")
+
+            checkOptimizationResult(df,
+                "SELECT id FROM city WHERE CONCAT(CAST(id AS VARCHAR), ' - this is ID') = '1 - this is ID'")
+
+            val data = Tuple1(1)
+
+            checkQueryData(df, data)
+        }
+    }
+
+    describe("Not Optimized Queries") {
+        it("SELECT id, name FROM json_cities") {
+            val citiesDataFrame = igniteSession.read.json(
+                resolveIgnitePath("modules/spark/src/test/resources/cities.json").getAbsolutePath)
+
+            citiesDataFrame.createOrReplaceTempView("JSON_CITIES")
+
+            val df = igniteSession.sql("SELECT id, name FROM json_cities")
+
+            val data = (
+                (1, "Forest Hill"),
+                (2, "Denver"),
+                (3, "St. Petersburg"))
+
+            checkQueryData(df, data)
+        }
+
+        it("SELECT id, test_reverse(name) tr FROM city WHERE test_reverse(name) = 'revneD' ORDER BY id") {
+            val df = igniteSession.sql("SELECT id, test_reverse(name) tr " +
+                "FROM city WHERE test_reverse(name) = 'revneD' ORDER BY id")
+
+            checkOptimizationResult(df)
+        }
+
+        it("SELECT id, test_reverse(name) tr FROM city WHERE test_reverse(name) = 'revneD' and id > 0 ORDER BY id") {
+            val df = igniteSession.sql("SELECT id, test_reverse(name) tr " +
+                "FROM city WHERE test_reverse(name) = 'revneD' and id > 0 ORDER BY id")
+
+            checkOptimizationResult(df)
+        }
+
+        it("SELECT id, test_reverse(name) tr FROM city ORDER BY tr") {
+            val df = igniteSession.sql("SELECT id, test_reverse(name) tr FROM city ORDER BY tr")
+
+            checkOptimizationResult(df)
+        }
+
+        it("SELECT count(*), test_reverse(name) tr FROM city GROUP BY test_reverse(name)") {
+            val df = igniteSession.sql("SELECT count(*), test_reverse(name) tr FROM city GROUP BY test_reverse(name)")
+
+            checkOptimizationResult(df)
+        }
+    }
+
+    override protected def beforeAll(): Unit = {
+        super.beforeAll()
+
+        createPersonTable(client, DEFAULT_CACHE)
+
+        createCityTable(client, DEFAULT_CACHE)
+
+        val configProvider = enclose(null) (x ⇒ () ⇒ {
+            val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+            cfg.setClientMode(true)
+
+            cfg.setIgniteInstanceName("client-2")
+
+            cfg
+        })
+
+        igniteSession = IgniteSparkSession.builder()
+            .config(spark.sparkContext.getConf)
+            .igniteConfigProvider(configProvider)
+            .getOrCreate()
+
+        igniteSession.udf.register("test_reverse", (str: String) ⇒ str.reverse)
+    }
+}
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationStringFuncSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationStringFuncSpec.scala
new file mode 100644
index 0000000000000..db106f2473b2b
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationStringFuncSpec.scala
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import java.lang.{Long ⇒ JLong}
+
+/**
+  * === Doesn't supported by Spark ===
+  * CHAR
+  * DIFFERENCE
+  * HEXTORAW
+  * RAWTOHEX
+  * REGEXP_LIKE
+  * SOUNDEX
+  * STRINGDECODE
+  * STRINGENCODE
+  * STRINGTOUTF8
+  * UTF8TOSTRING
+  * XMLATTR
+  * XMLNODE
+  * XMLCOMMENT
+  * XMLCDATA
+  * XMLSTARTDOC
+  * XMLTEXT
+  * TO_CHAR - The function that can format a timestamp, a number, or text.
+  * ====== This functions in spark master but not in release =====
+  * LEFT
+  * RIGHT
+  * INSERT
+  * REPLACE
+  */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationStringFuncSpec extends AbstractDataFrameSpec {
+    var igniteSession: IgniteSparkSession = _
+
+    describe("Supported optimized string functions") {
+        it("LENGTH") {
+            val df = igniteSession.sql("SELECT LENGTH(str) FROM strings WHERE id <= 3")
+
+            checkOptimizationResult(df, "SELECT CAST(LENGTH(str) AS INTEGER) as \"length(str)\" FROM strings " +
+                "WHERE id is not null AND id <= 3")
+
+            val data = (3, 3, 6)
+
+            checkQueryData(df, data)
+        }
+
+        it("RTRIM") {
+            val df = igniteSession.sql("SELECT RTRIM(str) FROM strings WHERE id = 3")
+
+            checkOptimizationResult(df, "SELECT RTRIM(str) FROM strings WHERE id is not null AND id = 3")
+
+            val data = Tuple1("AAA")
+
+            checkQueryData(df, data)
+        }
+
+        it("LTRIM") {
+            val df = igniteSession.sql("SELECT LTRIM(str) FROM strings WHERE id = 4")
+
+            checkOptimizationResult(df, "SELECT LTRIM(str) FROM strings WHERE id is not null AND id = 4")
+
+            val data = Tuple1("AAA")
+
+            checkQueryData(df, data)
+        }
+
+        it("TRIM") {
+            val df = igniteSession.sql("SELECT TRIM(str) FROM strings WHERE id = 5")
+
+            checkOptimizationResult(df, "SELECT TRIM(str) FROM strings WHERE id is not null AND id = 5")
+
+            val data = Tuple1("AAA")
+
+            checkQueryData(df, data)
+        }
+
+        it("LOWER") {
+            val df = igniteSession.sql("SELECT LOWER(str) FROM strings WHERE id = 2")
+
+            checkOptimizationResult(df, "SELECT LOWER(str) FROM strings WHERE id is not null AND id = 2")
+
+            val data = Tuple1("aaa")
+
+            checkQueryData(df, data)
+        }
+
+        it("UPPER") {
+            val df = igniteSession.sql("SELECT UPPER(str) FROM strings WHERE id = 1")
+
+            checkOptimizationResult(df, "SELECT UPPER(str) FROM strings WHERE id is not null AND id = 1")
+
+            val data = Tuple1("AAA")
+
+            checkQueryData(df, data)
+        }
+
+        it("LOWER(RTRIM)") {
+            val df = igniteSession.sql("SELECT LOWER(RTRIM(str)) FROM strings WHERE id = 3")
+
+            checkOptimizationResult(df, "SELECT LOWER(RTRIM(str)) FROM strings WHERE id is not null AND id = 3")
+
+            val data = Tuple1("aaa")
+
+            checkQueryData(df, data)
+        }
+
+        it("LOCATE") {
+            val df = igniteSession.sql("SELECT LOCATE('D', str) FROM strings WHERE id = 6")
+
+            checkOptimizationResult(df, "SELECT LOCATE('D', str, 1) FROM strings WHERE id is not null AND id = 6")
+
+            val data = Tuple1(4)
+
+            checkQueryData(df, data)
+        }
+
+        it("LOCATE - 2") {
+            val df = igniteSession.sql("SELECT LOCATE('A', str) FROM strings WHERE id = 6")
+
+            checkOptimizationResult(df, "SELECT LOCATE('A', str, 1) FROM strings WHERE id is not null AND id = 6")
+
+            val data = Tuple1(1)
+
+            checkQueryData(df, data)
+        }
+
+        it("POSITION") {
+            val df = igniteSession.sql("SELECT instr(str, 'BCD') FROM strings WHERE id = 6")
+
+            checkOptimizationResult(df, "SELECT POSITION('BCD', str) as \"instr(str, BCD)\" FROM strings " +
+                "WHERE id is not null AND id = 6")
+
+            val data = Tuple1(2)
+
+            checkQueryData(df, data)
+        }
+
+        it("CONCAT") {
+            val df = igniteSession.sql("SELECT concat(str, 'XXX') FROM strings WHERE id = 6")
+
+            checkOptimizationResult(df, "SELECT concat(str, 'XXX') FROM strings WHERE id is not null AND id = 6")
+
+            val data = Tuple1("ABCDEFXXX")
+
+            checkQueryData(df, data)
+        }
+
+        it("RPAD") {
+            val df = igniteSession.sql("SELECT RPAD(str, 10, 'X') FROM strings WHERE id = 6")
+
+            checkOptimizationResult(df, "SELECT RPAD(str, 10, 'X') FROM strings WHERE id is not null AND id = 6")
+
+            val data = Tuple1("ABCDEFXXXX")
+
+            checkQueryData(df, data)
+        }
+
+        it("LPAD") {
+            val df = igniteSession.sql("SELECT LPAD(str, 10, 'X') FROM strings WHERE id = 6")
+
+            checkOptimizationResult(df, "SELECT LPAD(str, 10, 'X') FROM strings WHERE id is not null AND id = 6")
+
+            val data = Tuple1("XXXXABCDEF")
+
+            checkQueryData(df, data)
+        }
+
+        it("REPEAT") {
+            val df = igniteSession.sql("SELECT REPEAT(str, 2) FROM strings WHERE id = 6")
+
+            checkOptimizationResult(df, "SELECT REPEAT(str, 2) FROM strings WHERE id is not null AND id = 6")
+
+            val data = Tuple1("ABCDEFABCDEF")
+
+            checkQueryData(df, data)
+        }
+
+        it("SUBSTRING") {
+            val df = igniteSession.sql("SELECT SUBSTRING(str, 4, 3) FROM strings WHERE id = 6")
+
+            checkOptimizationResult(df, "SELECT SUBSTR(str, 4, 3) as \"SUBSTRING(str, 4, 3)\" FROM strings " +
+                "WHERE id is not null AND id = 6")
+
+            val data = Tuple1("DEF")
+
+            checkQueryData(df, data)
+        }
+
+        it("SPACE") {
+            val df = igniteSession.sql("SELECT SPACE(LENGTH(str)) FROM strings WHERE id = 1")
+
+            checkOptimizationResult(df, "SELECT SPACE(CAST(LENGTH(str) AS INTEGER)) as \"SPACE(LENGTH(str))\" " +
+                "FROM strings WHERE id is not null AND id = 1")
+
+            val data = Tuple1("   ")
+
+            checkQueryData(df, data)
+        }
+
+        it("ASCII") {
+            val df = igniteSession.sql("SELECT ASCII(str) FROM strings WHERE id = 7")
+
+            checkOptimizationResult(df, "SELECT ASCII(str) FROM strings WHERE id is not null AND id = 7")
+
+            val data = Tuple1(50)
+
+            checkQueryData(df, data)
+        }
+
+        it("REGEXP_REPLACE") {
+            val df = igniteSession.sql("SELECT REGEXP_REPLACE(str, '(\\\\d+)', 'num') FROM strings WHERE id = 7")
+
+            checkOptimizationResult(df, "SELECT REGEXP_REPLACE(str, '(\\d+)', 'num') FROM strings " +
+                "WHERE id is not null AND id = 7")
+
+            val data = Tuple1("num")
+
+            checkQueryData(df, data)
+        }
+
+        it("CONCAT_WS") {
+            val df = igniteSession.sql("SELECT id, CONCAT_WS(', ', str, 'after') FROM strings " +
+                "WHERE id >= 7 AND id <= 8")
+
+            checkOptimizationResult(df, "SELECT id, CONCAT_WS(', ', str, 'after') FROM strings " +
+                "WHERE id is not null AND id >= 7 AND id <= 8")
+
+            val data = (
+                (7, "222, after"),
+                (8, "after"))
+
+            checkQueryData(df, data)
+        }
+
+        it("TRANSLATE") {
+            val df = igniteSession.sql("SELECT id, TRANSLATE(str, 'DEF', 'ABC') FROM strings WHERE id = 6")
+
+            checkOptimizationResult(df, "SELECT id, TRANSLATE(str, 'DEF', 'ABC') FROM strings " +
+                "WHERE id is not null AND id = 6")
+
+            val data = Tuple1((6, "ABCABC"))
+
+            checkQueryData(df, data)
+        }
+    }
+
+    def createStringTable(client: Ignite, cacheName: String): Unit = {
+        val cache = client.cache(cacheName)
+
+        cache.query(new SqlFieldsQuery(
+            """
+              | CREATE TABLE strings (
+              |    id LONG,
+              |    str VARCHAR,
+              |    PRIMARY KEY (id)) WITH "backups=1"
+            """.stripMargin)).getAll
+
+        val qry = new SqlFieldsQuery("INSERT INTO strings (id, str) values (?, ?)")
+
+        cache.query(qry.setArgs(1L.asInstanceOf[JLong], "aaa")).getAll
+        cache.query(qry.setArgs(2L.asInstanceOf[JLong], "AAA")).getAll
+        cache.query(qry.setArgs(3L.asInstanceOf[JLong], "AAA   ")).getAll
+        cache.query(qry.setArgs(4L.asInstanceOf[JLong], "   AAA")).getAll
+        cache.query(qry.setArgs(5L.asInstanceOf[JLong], "   AAA   ")).getAll
+        cache.query(qry.setArgs(6L.asInstanceOf[JLong], "ABCDEF")).getAll
+        cache.query(qry.setArgs(7L.asInstanceOf[JLong], "222")).getAll
+        cache.query(qry.setArgs(8L.asInstanceOf[JLong], null)).getAll
+    }
+
+    override protected def beforeAll(): Unit = {
+        super.beforeAll()
+
+        createStringTable(client, DEFAULT_CACHE)
+
+        val configProvider = enclose(null) (x ⇒ () ⇒ {
+            val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+            cfg.setClientMode(true)
+
+            cfg.setIgniteInstanceName("client-2")
+
+            cfg
+        })
+
+        igniteSession = IgniteSparkSession.builder()
+            .config(spark.sparkContext.getConf)
+            .igniteConfigProvider(configProvider)
+            .getOrCreate()
+    }
+}
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSystemFuncSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSystemFuncSpec.scala
new file mode 100644
index 0000000000000..282a45fc840ac
--- /dev/null
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSystemFuncSpec.scala
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spark
+
+import org.apache.ignite.Ignite
+import org.apache.ignite.cache.query.SqlFieldsQuery
+import org.apache.spark.sql.ignite.IgniteSparkSession
+import org.junit.runner.RunWith
+import org.scalatest.junit.JUnitRunner
+import java.lang.{Double ⇒ JDouble, Long ⇒ JLong}
+
+import org.apache.ignite.internal.IgnitionEx
+import org.apache.ignite.spark.AbstractDataFrameSpec.{DEFAULT_CACHE, TEST_CONFIG_FILE, checkOptimizationResult, enclose}
+
+/**
+  */
+@RunWith(classOf[JUnitRunner])
+class IgniteOptimizationSystemFuncSpec extends AbstractDataFrameSpec {
+    var igniteSession: IgniteSparkSession = _
+
+    describe("Supported optimized system functions") {
+        it("COALESCE") {
+            val df = igniteSession.sql("SELECT COALESCE(int_val1, int_val2) FROM numbers WHERE id IN (1, 2, 3)")
+
+            checkOptimizationResult(df, "SELECT COALESCE(int_val1, int_val2) FROM numbers WHERE id IN (1, 2, 3)")
+
+            val data = (1, 2, 3)
+
+            checkQueryData(df, data)
+        }
+
+        it("GREATEST") {
+            val df = igniteSession.sql("SELECT GREATEST(int_val1, int_val2) FROM numbers WHERE id IN (4, 5)")
+
+            checkOptimizationResult(df, "SELECT GREATEST(int_val1, int_val2) FROM numbers WHERE id IN (4, 5)")
+
+            val data = (4, 6)
+
+            checkQueryData(df, data)
+        }
+
+        it("LEAST") {
+            val df = igniteSession.sql("SELECT LEAST(int_val1, int_val2) FROM numbers WHERE id IN (4, 5)")
+
+            checkOptimizationResult(df, "SELECT LEAST(int_val1, int_val2) FROM numbers WHERE id IN (4, 5)")
+
+            val data = (3, 5)
+
+            checkQueryData(df, data)
+        }
+
+        it("IFNULL") {
+            val df = igniteSession.sql("SELECT IFNULL(int_val1, int_val2) FROM numbers WHERE id IN (1, 2, 3)")
+
+            checkOptimizationResult(df, "SELECT COALESCE(int_val1, int_val2) as \"ifnull(numbers.`int_val1`, numbers.`int_val2`)\" FROM numbers WHERE id IN (1, 2, 3)")
+
+            val data = (1, 2, 3)
+
+            checkQueryData(df, data)
+        }
+
+        it("NULLIF") {
+            val df = igniteSession.sql("SELECT id, NULLIF(int_val1, int_val2) FROM numbers WHERE id IN (6, 7)")
+
+            checkOptimizationResult(df)
+
+            val data = (
+                (6, null),
+                (7, 8))
+
+            checkQueryData(df, data)
+        }
+
+        it("NVL2") {
+            val df = igniteSession.sql("SELECT id, NVL2(int_val1, 'not null', 'null') FROM numbers WHERE id IN (1, 2, 3)")
+
+            checkOptimizationResult(df)
+
+            val data = (
+                (1, "not null"),
+                (2, "null"),
+                (3, "not null"))
+
+            checkQueryData(df, data)
+        }
+    }
+
+    def createNumberTable(client: Ignite, cacheName: String): Unit = {
+        val cache = client.cache(cacheName)
+
+        cache.query(new SqlFieldsQuery(
+            """
+              | CREATE TABLE numbers (
+              |    id LONG,
+              |    int_val1 LONG,
+              |    int_val2 LONG,
+              |    PRIMARY KEY (id)) WITH "backups=1"
+            """.stripMargin)).getAll
+
+
+        val qry = new SqlFieldsQuery("INSERT INTO numbers (id, int_val1, int_val2) values (?, ?, ?)")
+
+        cache.query(qry.setArgs(1L.asInstanceOf[JLong], 1L.asInstanceOf[JLong], null)).getAll
+        cache.query(qry.setArgs(2L.asInstanceOf[JLong], null, 2L.asInstanceOf[JLong])).getAll
+        cache.query(qry.setArgs(3L.asInstanceOf[JLong], 3L.asInstanceOf[JLong], null)).getAll
+        cache.query(qry.setArgs(4L.asInstanceOf[JLong], 3L.asInstanceOf[JLong], 4L.asInstanceOf[JLong])).getAll
+        cache.query(qry.setArgs(5L.asInstanceOf[JLong], 6L.asInstanceOf[JLong], 5L.asInstanceOf[JLong])).getAll
+        cache.query(qry.setArgs(6L.asInstanceOf[JLong], 7L.asInstanceOf[JLong], 7L.asInstanceOf[JLong])).getAll
+        cache.query(qry.setArgs(7L.asInstanceOf[JLong], 8L.asInstanceOf[JLong], 9L.asInstanceOf[JLong])).getAll
+    }
+
+    override protected def beforeAll(): Unit = {
+        super.beforeAll()
+
+        createNumberTable(client, DEFAULT_CACHE)
+
+        val configProvider = enclose(null) (x ⇒ () ⇒ {
+            val cfg = IgnitionEx.loadConfiguration(TEST_CONFIG_FILE).get1()
+
+            cfg.setClientMode(true)
+
+            cfg.setIgniteInstanceName("client-2")
+
+            cfg
+        })
+
+        igniteSession = IgniteSparkSession.builder()
+            .config(spark.sparkContext.getConf)
+            .igniteConfigProvider(configProvider)
+            .getOrCreate()
+    }
+}

From d7962f83f14292a571016b79af594ceefe632d83 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Fri, 20 Apr 2018 19:17:32 +0300
Subject: [PATCH 104/543] IGNITE-7077 Fixed header licenses

---
 .../impl/optimization/SimpleExpressions.scala   | 17 +++++++++++++++++
 .../ignite/spark/IgniteOptimizationSpec.scala   |  2 +-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
index a1c945843ed58..10d021a5adb2b 100644
--- a/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
+++ b/modules/spark/src/main/scala/org/apache/ignite/spark/impl/optimization/SimpleExpressions.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.ignite.spark.impl.optimization
 
 import java.text.SimpleDateFormat
diff --git a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
index 00075f46925e6..128f625e5eb5b 100644
--- a/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
+++ b/modules/spark/src/test/scala/org/apache/ignite/spark/IgniteOptimizationSpec.scala
@@ -10,7 +10,7 @@
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either expdfs or implied.
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

From 2497fe376ffa7df65381f2c91151b6f7fccaf212 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Mon, 23 Apr 2018 13:27:51 +0300
Subject: [PATCH 105/543] IGNITE-8192 Print out information on how many nodes
 left until auto-activation

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../apache/ignite/internal/IgniteKernal.java  |  5 +-
 .../discovery/GridDiscoveryManager.java       | 65 +++++++++++++++----
 .../ignite/internal/util/lang/GridFunc.java   | 14 ++--
 3 files changed, 65 insertions(+), 19 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index c256ac43f6b85..565a8ce31a804 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -99,12 +99,12 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.MemoryConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.binary.BinaryEnumCache;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.binary.BinaryUtils;
 import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
 import org.apache.ignite.internal.cluster.IgniteClusterEx;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.managers.GridManager;
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
@@ -1301,7 +1301,8 @@ private long checkPoolStarvation(
         ackStart(rtBean);
 
         if (!isDaemon())
-            ctx.discovery().ackTopology(ctx.discovery().localJoin().joinTopologyVersion().topologyVersion());
+            ctx.discovery().ackTopology(ctx.discovery().localJoin().joinTopologyVersion().topologyVersion(),
+                EventType.EVT_NODE_JOINED, localNode());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 77c96573d0a63..6af3293935a62 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -1372,18 +1372,22 @@ private static int cpus(Collection<ClusterNode> nodes) {
      * Prints the latest topology info into log taking into account logging/verbosity settings.
      *
      * @param topVer Topology version.
+     * @param evtType Event type.
+     * @param evtNode Event node.
      */
-    public void ackTopology(long topVer) {
-        ackTopology(topVer, false);
+    public void ackTopology(long topVer, int evtType, ClusterNode evtNode) {
+        ackTopology(topVer, evtType, evtNode, false);
     }
 
     /**
      * Logs grid size for license compliance.
      *
      * @param topVer Topology version.
+     * @param evtType Event type.
+     * @param evtNode Event node.
      * @param throttle Suppress printing if this topology was already printed.
      */
-    private void ackTopology(long topVer, boolean throttle) {
+    private void ackTopology(long topVer, int evtType, ClusterNode evtNode, boolean throttle) {
         assert !isLocDaemon;
 
         DiscoCache discoCache = discoCacheHist.get(new AffinityTopologyVersion(topVer));
@@ -1430,7 +1434,7 @@ else if (log.isInfoEnabled())
 
                     return null;
                 }
-            }, topVer, srvNodes.size(), clientNodes.size(), totalCpus, heap, offheap);
+            }, topVer, discoCache, evtType, evtNode, srvNodes.size(), clientNodes.size(), totalCpus, heap, offheap);
 
         if (log.isDebugEnabled()) {
             String dbg = "";
@@ -1481,7 +1485,7 @@ else if (log.isInfoEnabled())
 
                     return null;
                 }
-            }, topVer, srvNodes.size(), clientNodes.size(), totalCpus, heap, offheap);
+            }, topVer, discoCache, evtType, evtNode, srvNodes.size(), clientNodes.size(), totalCpus, heap, offheap);
     }
 
     /**
@@ -1557,14 +1561,18 @@ private String dataRegionConfigurationMessage(DataRegionConfiguration regCfg) {
     /**
      * @param clo Wrapper of logger.
      * @param topVer Topology version.
+     * @param discoCache Discovery cache.
+     * @param evtType Event type.
+     * @param evtNode Event node.
      * @param srvNodesNum Server nodes number.
      * @param clientNodesNum Client nodes number.
      * @param totalCpus Total cpu number.
      * @param heap Heap size.
      * @param offheap Offheap size.
      */
-    private void topologySnapshotMessage(IgniteClosure<String, Void> clo, long topVer, int srvNodesNum,
-        int clientNodesNum, int totalCpus, double heap, double offheap) {
+    private void topologySnapshotMessage(IgniteClosure<String, Void> clo, long topVer, DiscoCache discoCache,
+        int evtType, ClusterNode evtNode, int srvNodesNum, int clientNodesNum, int totalCpus, double heap,
+        double offheap) {
         String summary = PREFIX + " [" +
             (discoOrdered ? "ver=" + topVer + ", " : "") +
             "servers=" + srvNodesNum +
@@ -1575,6 +1583,41 @@ private void topologySnapshotMessage(IgniteClosure<String, Void> clo, long topVe
 
         clo.apply(summary);
 
+        DiscoveryDataClusterState state = discoCache.state();
+
+        clo.apply("  ^-- Node [id=" + discoCache.localNode().id().toString().toUpperCase() + ", clusterState="
+            + (state.active() ? "ACTIVE" : "INACTIVE") + ']');
+
+        BaselineTopology blt = state.baselineTopology();
+
+        if (blt != null && discoCache.baselineNodes() != null) {
+            int bltSize = discoCache.baselineNodes().size();
+            int bltOnline = discoCache.aliveBaselineNodes().size();
+            int bltOffline = bltSize - bltOnline;
+
+            clo.apply("  ^-- Baseline [id=" + blt.id() + ", size=" + bltSize + ", online=" + bltOnline
+                + ", offline=" + bltOffline + ']');
+
+            if (!state.active() && ctx.config().isAutoActivationEnabled()) {
+                String offlineConsistentIds = "";
+
+                if (bltOffline > 0 && bltOffline <= 5) {
+                    Collection<BaselineNode> offlineNodes = new HashSet<>(discoCache.baselineNodes());
+
+                    offlineNodes.removeAll(discoCache.aliveBaselineNodes());
+
+                    offlineConsistentIds = ' ' + F.nodeConsistentIds(offlineNodes).toString();
+                }
+
+                if (bltOffline == 0) {
+                    if (evtType == EVT_NODE_JOINED && discoCache.baselineNode(evtNode))
+                        clo.apply("  ^-- All baseline nodes are online, will start auto-activation");
+                }
+                else
+                    clo.apply("  ^-- " + bltOffline + " nodes left for auto-activation" + offlineConsistentIds);
+            }
+        }
+
         DataStorageConfiguration memCfg = ctx.config().getDataStorageConfiguration();
 
         if (memCfg == null)
@@ -2742,7 +2785,7 @@ private void body0() throws InterruptedException {
                             if (log.isInfoEnabled())
                                 log.info("Added new node to topology: " + node);
 
-                            ackTopology(topVer.topologyVersion(), true);
+                            ackTopology(topVer.topologyVersion(), type, node, true);
                         }
                         else if (log.isDebugEnabled())
                             log.debug("Added new node to topology: " + node);
@@ -2763,7 +2806,7 @@ else if (log.isDebugEnabled())
                             if (log.isInfoEnabled())
                                 log.info("Node left topology: " + node);
 
-                            ackTopology(topVer.topologyVersion(), true);
+                            ackTopology(topVer.topologyVersion(), type, node, true);
                         }
                         else if (log.isDebugEnabled())
                             log.debug("Node left topology: " + node);
@@ -2785,7 +2828,7 @@ else if (log.isDebugEnabled())
                         log.info("Client node reconnected to topology: " + node);
 
                     if (!isLocDaemon)
-                        ackTopology(topVer.topologyVersion(), true);
+                        ackTopology(topVer.topologyVersion(), type, node, true);
 
                     break;
                 }
@@ -2799,7 +2842,7 @@ else if (log.isDebugEnabled())
                         if (!isLocDaemon) {
                             U.warn(log, "Node FAILED: " + node);
 
-                            ackTopology(topVer.topologyVersion(), true);
+                            ackTopology(topVer.topologyVersion(), type, node, true);
                         }
                         else if (log.isDebugEnabled())
                             log.debug("Node FAILED: " + node);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
index ef32e3114d72e..f9c4d9d75388d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
@@ -43,6 +43,7 @@
 import javax.cache.Cache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -164,11 +165,12 @@ public class GridFunc {
     private static final IgniteClosure<ClusterNode, UUID> NODE2ID = new ClusterNodeGetIdClosure();
 
     /** */
-    private static final IgniteClosure<ClusterNode, Object> NODE2CONSISTENTID = new IgniteClosure<ClusterNode, Object>() {
-        @Override public Object apply(ClusterNode node) {
-            return node.consistentId();
-        }
-    };
+    private static final IgniteClosure<BaselineNode, Object> NODE2CONSISTENTID =
+        new IgniteClosure<BaselineNode, Object>() {
+            @Override public Object apply(BaselineNode node) {
+                return node.consistentId();
+            }
+        };
 
     /**
      * Gets predicate that evaluates to {@code true} only for given local node ID.
@@ -335,7 +337,7 @@ public static Collection<UUID> nodeIds(@Nullable Collection<? extends ClusterNod
      * @param nodes Collection of grid nodes.
      * @return Collection of node consistent IDs for given collection of grid nodes.
      */
-    public static Collection<Object> nodeConsistentIds(@Nullable Collection<? extends ClusterNode> nodes) {
+    public static Collection<Object> nodeConsistentIds(@Nullable Collection<? extends BaselineNode> nodes) {
         if (nodes == null || nodes.isEmpty())
             return Collections.emptyList();
 

From 9e3daf6aafd48ad93086f8e7968fb376f03f4519 Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Mon, 23 Apr 2018 16:31:57 +0300
Subject: [PATCH 106/543] IGNITE-8228 Print out information about configured
 failure handler

---
 .../java/org/apache/ignite/failure/NoOpFailureHandler.java  | 6 ++++++
 .../apache/ignite/failure/RestartProcessFailureHandler.java | 6 ++++++
 .../org/apache/ignite/failure/StopNodeFailureHandler.java   | 6 ++++++
 .../apache/ignite/failure/StopNodeOrHaltFailureHandler.java | 6 ++++++
 .../internal/processors/failure/FailureProcessor.java       | 6 ++++--
 5 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java
index 2ec645eb1d5ea..b998d62dd88c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/failure/NoOpFailureHandler.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.failure;
 
 import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
  * Just ignores any failure. It's useful for tests and debugging.
@@ -27,4 +28,9 @@ public class NoOpFailureHandler implements FailureHandler {
     @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
         return false;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(NoOpFailureHandler.class, this);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java
index 4f4432fc924c9..fe65b28cb2db7 100644
--- a/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/failure/RestartProcessFailureHandler.java
@@ -20,6 +20,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
@@ -42,4 +43,9 @@ public class RestartProcessFailureHandler implements FailureHandler {
 
         return true;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(RestartProcessFailureHandler.class, this);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java
index f05cc760f2a7d..861d220bced56 100644
--- a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeFailureHandler.java
@@ -19,6 +19,7 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
@@ -40,4 +41,9 @@ public class StopNodeFailureHandler implements FailureHandler {
 
         return true;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(StopNodeFailureHandler.class, this);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
index 3ce4ff6f27ef6..879ddb0d798c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/failure/StopNodeOrHaltFailureHandler.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
@@ -99,4 +100,9 @@ public StopNodeOrHaltFailureHandler(boolean tryStop, long timeout) {
 
         return true;
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(StopNodeOrHaltFailureHandler.class, this);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
index 0234e84345580..b11cb95126674 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
@@ -67,6 +67,8 @@ public FailureProcessor(GridKernalContext ctx) {
         assert hnd != null;
 
         this.hnd = hnd;
+
+        U.quietAndInfo(log, "Configured failure handler: [hnd=" + hnd + ']');
     }
 
     /**
@@ -107,8 +109,8 @@ public synchronized void process(FailureContext failureCtx, FailureHandler hnd)
         if (this.failureCtx != null) // Node already terminating, no reason to process more errors.
             return;
 
-        U.error(ignite.log(), "Critical failure. Will be handled accordingly to configured handler [hnd=" +
-            hnd.getClass() + ", failureCtx=" + failureCtx + ']', failureCtx.error());
+        U.error(ignite.log(), "Critical system error detected. Will be handled accordingly to configured handler " +
+            "[hnd=" + hnd.getClass() + ", failureCtx=" + failureCtx + ']', failureCtx.error());
 
         if (reserveBuf != null && X.hasCause(failureCtx.error(), OutOfMemoryError.class))
             reserveBuf = null;

From ee1e089a8cded02e51063602cb8c470a2aad4fa7 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Mon, 23 Apr 2018 17:39:57 +0300
Subject: [PATCH 107/543] IGNITE-8323 Fixed discovery event listener calls -
 Fixes #3890.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../internal/managers/discovery/GridDiscoveryManager.java | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 6af3293935a62..e9b6cfea6eeea 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -687,11 +687,15 @@ else if (customMsg instanceof ChangeGlobalStateFinishMessage) {
                     }
 
                     nextTopVer = new AffinityTopologyVersion(topVer, minorTopVer);
+
+                    if (incMinorTopVer)
+                        ctx.cache().onDiscoveryEvent(type, customMsg, node, nextTopVer, ctx.state().clusterState());
                 }
-                else
+                else {
                     nextTopVer = new AffinityTopologyVersion(topVer, minorTopVer);
 
-                ctx.cache().onDiscoveryEvent(type, customMsg, node, nextTopVer, ctx.state().clusterState());
+                    ctx.cache().onDiscoveryEvent(type, customMsg, node, nextTopVer, ctx.state().clusterState());
+                }
 
                 if (type == EVT_DISCOVERY_CUSTOM_EVT) {
                     for (Class cls = customMsg.getClass(); cls != null; cls = cls.getSuperclass()) {

From 5a981c9f38abfc37934d7f3c19e87f8cd5f586c2 Mon Sep 17 00:00:00 2001
From: tledkov-gridgain <tledkov@gridgain.com>
Date: Mon, 23 Apr 2018 18:20:21 +0300
Subject: [PATCH 108/543] IGNITE-8355 Fixed NPE on concurrent nodes start -
 Fixes #3899.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../discovery/GridDiscoveryManager.java       |  2 +-
 .../IgniteAuthenticationProcessor.java        | 19 +++-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  2 +-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    | 13 +++
 ...AuthenticationProcessorNPEOnStartTest.java | 90 +++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite6.java     |  2 +
 6 files changed, 125 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNPEOnStartTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index e9b6cfea6eeea..261e73db97786 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2140,7 +2140,7 @@ public Serializable consistentId() {
      *
      * @return Wrapped DiscoverySpi SPI.
      */
-    private DiscoverySpi getInjectedDiscoverySpi() {
+    public DiscoverySpi getInjectedDiscoverySpi() {
         try {
             inject();
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
index 93b3a5bc2cce4..ac713c3dce875 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
@@ -68,6 +68,8 @@
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.spi.IgniteNodeValidationResult;
 import org.apache.ignite.spi.discovery.DiscoveryDataBag;
+import org.apache.ignite.spi.discovery.DiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.thread.IgniteThreadPoolExecutor;
 import org.jetbrains.annotations.Nullable;
 
@@ -451,7 +453,7 @@ public void updateUser(String login, String passwd) throws IgniteCheckedExceptio
     @Override public void collectGridNodeData(DiscoveryDataBag dataBag) {
         // 1. Collect users info only on coordinator
         // 2. Doesn't collect users info to send on client node due to security reason.
-        if (!isEnabled || !F.eq(ctx.localNodeId(), coordinator().id()) || dataBag.isJoiningNodeClient())
+        if (!isEnabled || !isLocalNodeCoordinator() || dataBag.isJoiningNodeClient())
             return;
 
         synchronized (mux) {
@@ -466,6 +468,21 @@ public void updateUser(String login, String passwd) throws IgniteCheckedExceptio
         }
     }
 
+    /**
+     * Checks whether local node is coordinator. Nodes that are leaving or failed
+     * (but are still in topology) are removed from search.
+     *
+     * @return {@code true} if local node is coordinator.
+     */
+    private boolean isLocalNodeCoordinator() {
+        DiscoverySpi spi = ctx.discovery().getInjectedDiscoverySpi();
+
+        if (spi instanceof TcpDiscoverySpi)
+            return ((TcpDiscoverySpi)spi).isLocalNodeCoordinator();
+        else
+            return F.eq(ctx.localNodeId(), coordinator().id());
+    }
+
     /** {@inheritDoc} */
     @Override public void onGridDataReceived(DiscoveryDataBag.GridDiscoveryData data) {
         initUsrs = (InitialUsersData)data.commonData();
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 6d3864e704215..6b2293a1d275c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1473,7 +1473,7 @@ private static <T extends R, R> Collection<R> upcast(Collection<T> c) {
      *
      * @return {@code true} if local node is coordinator.
      */
-    private boolean isLocalNodeCoordinator() {
+    public boolean isLocalNodeCoordinator() {
         synchronized (mux) {
             boolean crd = spiState == CONNECTED && locNode.equals(resolveCoordinator());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 7c73b2879428d..801f2b6565521 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -2235,6 +2235,19 @@ public void brakeConnection() {
         impl.brakeConnection();
     }
 
+    /**
+     * Checks whether local node is coordinator. Nodes that are leaving or failed
+     * (but are still in topology) are removed from search.
+     *
+     * @return {@code true} if local node is coordinator.
+     */
+    public boolean isLocalNodeCoordinator() {
+        if (impl instanceof ServerImpl)
+            return ((ServerImpl)impl).isLocalNodeCoordinator();
+
+        return false;
+    }
+
     /**
      * @return Marshaller.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNPEOnStartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNPEOnStartTest.java
new file mode 100644
index 0000000000000..9748db772747a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNPEOnStartTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.authentication;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for NPE on start node simultaneous.
+ */
+public class AuthenticationProcessorNPEOnStartTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi spi = new TcpDiscoverySpi();
+
+        spi.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(spi);
+
+        cfg.setAuthenticationEnabled(true);
+
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setPersistenceEnabled(true)));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", true);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void test() throws Exception {
+        final AtomicInteger nodeIdx = new AtomicInteger();
+
+        GridTestUtils.runMultiThreaded(new Runnable() {
+            @Override public void run() {
+                try {
+                    startGrid(nodeIdx.getAndIncrement());
+                }
+                catch (Exception e) {
+                    e.printStackTrace();
+
+                    fail("Unexpected exception");
+                }
+            }
+        }, 10, "auth-grid-starter");
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 415479dae4138..b730697e91ffa 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.authentication.AuthenticationConfigurationClusterTest;
 import org.apache.ignite.internal.processors.authentication.AuthenticationOnNotActiveClusterTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNPEOnStartTest;
 import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNodeRestartTest;
 import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorSelfTest;
 import org.apache.ignite.internal.processors.cache.PartitionedAtomicCacheGetsDistributionTest;
@@ -86,6 +87,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(AuthenticationProcessorSelfTest.class);
         suite.addTestSuite(AuthenticationOnNotActiveClusterTest.class);
         suite.addTestSuite(AuthenticationProcessorNodeRestartTest.class);
+        suite.addTestSuite(AuthenticationProcessorNPEOnStartTest.class);
 
         suite.addTestSuite(ReplicatedAtomicCacheGetsDistributionTest.class);
         suite.addTestSuite(ReplicatedTransactionalOptimisticCacheGetsDistributionTest.class);

From 46bff2b143db5c6819222477348c9c7b539934ac Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Mon, 23 Apr 2018 18:59:04 +0300
Subject: [PATCH 109/543] IGNITE-6827 Configurable rollback for long running
 transactions before partition exchange IGNITE-7910 Improved transaction
 debugging support

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../ignite/tests/utils/TestTransaction.java   |   5 +
 .../java/org/apache/ignite/IgniteCluster.java |   7 +
 .../org/apache/ignite/IgniteTransactions.java |  23 +-
 .../TransactionConfiguration.java             |  41 +
 .../apache/ignite/internal/IgniteKernal.java  |   9 +-
 .../internal/TransactionsMXBeanImpl.java      | 162 +++
 .../cluster/IgniteClusterAsyncImpl.java       |   5 +
 .../internal/cluster/IgniteClusterImpl.java   |  15 +
 .../internal/commandline/Arguments.java       |  16 +-
 .../ignite/internal/commandline/Command.java  |   5 +-
 .../internal/commandline/CommandHandler.java  | 345 ++++++-
 .../optimized/OptimizedMarshallerUtils.java   |   2 +-
 .../processors/cache/GridCacheAdapter.java    |  96 +-
 .../GridCachePartitionExchangeManager.java    |  29 +-
 .../processors/cache/GridCacheProcessor.java  | 103 +-
 .../cache/GridCacheSharedContext.java         |  19 +-
 .../processors/cache/GridCacheUtils.java      |   4 +-
 ...utOnPartitionMapExchangeChangeMessage.java | 128 +++
 ...meoutOnPartitionMapExchangeChangeTask.java |  57 ++
 .../distributed/GridCacheTxFinishSync.java    |   3 +-
 .../distributed/GridDistributedTxMapping.java |   2 +-
 .../GridDistributedTxRemoteAdapter.java       |  23 +-
 .../distributed/dht/GridDhtLockFuture.java    | 363 ++++---
 .../dht/GridDhtTransactionalCacheAdapter.java |  48 +-
 .../cache/distributed/dht/GridDhtTxLocal.java |  29 +-
 .../dht/GridDhtTxLocalAdapter.java            | 187 +++-
 .../dht/GridDhtTxPrepareFuture.java           |   2 +-
 .../dht/GridDhtTxPrepareRequest.java          |  36 +-
 .../dht/colocated/GridDhtColocatedCache.java  |   4 +-
 .../colocated/GridDhtColocatedLockFuture.java | 121 ++-
 .../GridDhtPartitionsExchangeFuture.java      |  25 +-
 .../distributed/near/GridNearLockFuture.java  | 115 +--
 ...OptimisticSerializableTxPrepareFuture.java |   8 +-
 .../GridNearOptimisticTxPrepareFuture.java    |  13 +-
 .../GridNearPessimisticTxPrepareFuture.java   |   7 +-
 .../near/GridNearTxFastFinishFuture.java      |  12 +-
 .../near/GridNearTxFinishFuture.java          |  96 +-
 .../distributed/near/GridNearTxLocal.java     | 398 +++++---
 .../cache/local/GridLocalLockFuture.java      |  80 +-
 .../GridCacheDatabaseSharedManager.java       |   1 -
 .../store/GridCacheStoreManagerAdapter.java   |   5 +
 .../transactions/IgniteTransactionsImpl.java  |  36 +-
 .../cache/transactions/IgniteTxAdapter.java   |  19 +-
 .../cache/transactions/IgniteTxHandler.java   |   7 +-
 .../transactions/IgniteTxLocalAdapter.java    |  20 +-
 .../cache/transactions/IgniteTxManager.java   | 121 ++-
 .../transactions/TransactionProxyImpl.java    |   9 +
 .../TransactionProxyRollbackOnlyImpl.java     |  80 ++
 .../internal/util/GridPartitionStateMap.java  |   2 +-
 .../ignite/internal/visor/tx/VisorTxInfo.java | 174 ++++
 .../internal/visor/tx/VisorTxOperation.java   |  43 +
 .../internal/visor/tx/VisorTxProjection.java  |  42 +
 .../internal/visor/tx/VisorTxSortOrder.java   |  55 +
 .../ignite/internal/visor/tx/VisorTxTask.java | 248 +++++
 .../internal/visor/tx/VisorTxTaskArg.java     | 205 ++++
 .../internal/visor/tx/VisorTxTaskResult.java  |  80 ++
 .../ignite/mxbean/TransactionsMXBean.java     |  99 ++
 .../ignite/transactions/Transaction.java      |  18 +-
 .../resources/META-INF/classnames.properties  |   7 +
 .../TestRecordingCommunicationSpi.java        |   3 +-
 .../TransactionMetricsMxBeanImplTest.java     |   8 +-
 .../internal/TransactionsMXBeanImplTest.java  | 118 +++
 .../CommandHandlerParsingTest.java            | 100 ++
 ...ntryPessimisticRepeatableReadSeltTest.java |   6 +-
 ...etTxTimeoutOnPartitionMapExchangeTest.java | 166 ++++
 .../CacheLateAffinityAssignmentTest.java      |   5 -
 .../IgniteTxRemoveTimeoutObjectsNearTest.java |  30 +
 .../IgniteTxRemoveTimeoutObjectsTest.java     |   6 +
 ...rimaryNodeFailureRecoveryAbstractTest.java |   2 +-
 .../IgniteCacheTxRecoveryRollbackTest.java    |   2 +-
 .../GridCacheNearTxMultiNodeSelfTest.java     |   2 +-
 ...GridCachePartitionedTxSalvageSelfTest.java |   8 +-
 ...GridCacheLocalTxMultiThreadedSelfTest.java |   2 +-
 .../cache/transactions/TxLabelTest.java       |  63 ++
 .../TxMultiCacheAsyncOpsTest.java             | 140 +++
 ...timisticPrepareOnUnstableTopologyTest.java | 250 +++++
 .../TxPessimisticDeadlockDetectionTest.java   |   2 +-
 .../TxRollbackAsyncNearCacheTest.java         |  28 +
 .../transactions/TxRollbackAsyncTest.java     | 938 ++++++++++++++++++
 .../TxRollbackAsyncWithPersistenceTest.java   |  59 ++
 ...lbackOnTimeoutNoDeadlockDetectionTest.java |   6 +-
 .../transactions/TxRollbackOnTimeoutTest.java | 118 ++-
 .../TxRollbackOnTopologyChangeTest.java       | 228 +++++
 .../igfs/IgfsDataManagerSelfTest.java         |   6 +-
 .../cache/GridAbstractCacheStoreSelfTest.java |   7 +-
 .../multijvm/IgniteClusterProcessProxy.java   |   5 +
 .../testsuites/IgniteBasicTestSuite.java      |   4 +
 .../testsuites/IgniteCacheTestSuite3.java     |   2 +
 .../testsuites/IgniteCacheTestSuite6.java     |  17 +
 .../ignite/util/GridCommandHandlerTest.java   | 247 +++++
 .../processors/cache/jta/CacheJtaManager.java |   3 +-
 .../ApiParity/ClusterParityTest.cs            |   3 +-
 .../ApiParity/TransactionsParityTest.cs       |  13 +-
 .../Cache/CacheAbstractTransactionalTest.cs   |  11 +-
 94 files changed, 5695 insertions(+), 827 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxTimeoutOnPartitionMapExchangeChangeMessage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxTimeoutOnPartitionMapExchangeChangeTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyRollbackOnlyImpl.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxOperation.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxProjection.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxSortOrder.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/mxbean/TransactionsMXBean.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/TransactionsMXBeanImplTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsNearTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxLabelTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxMultiCacheAsyncOpsTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticPrepareOnUnstableTopologyTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncNearCacheTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncWithPersistenceTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java

diff --git a/modules/cassandra/store/src/test/java/org/apache/ignite/tests/utils/TestTransaction.java b/modules/cassandra/store/src/test/java/org/apache/ignite/tests/utils/TestTransaction.java
index e587bd7862282..be2211f801546 100644
--- a/modules/cassandra/store/src/test/java/org/apache/ignite/tests/utils/TestTransaction.java
+++ b/modules/cassandra/store/src/test/java/org/apache/ignite/tests/utils/TestTransaction.java
@@ -146,6 +146,11 @@ public class TestTransaction implements Transaction {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Nullable @Override public String label() {
+        return null;
+    }
+
     /** {@inheritDoc} */
     @Override public void resume() throws IgniteException {
         // No-op.
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java b/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
index 7329d682ebea4..b50133334dc9e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteCluster.java
@@ -420,6 +420,13 @@ public IgniteFuture<Collection<ClusterStartNodeResult>> startNodesAsync(Collecti
      */
     public void enableStatistics(Collection<String> caches, boolean enabled);
 
+    /**
+     * Sets transaction timeout on partition map exchange.
+     *
+     * @param timeout Transaction timeout on partition map exchange in milliseconds.
+     */
+    public void setTxTimeoutOnPartitionMapExchange(long timeout);
+
     /**
      * If local client node disconnected from cluster returns future
      * that will be completed when client reconnected.
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteTransactions.java b/modules/core/src/main/java/org/apache/ignite/IgniteTransactions.java
index dfe6a1a98c5ea..2bb7101a71928 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteTransactions.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteTransactions.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite;
 
+import java.util.Collection;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.configuration.TransactionConfiguration;
 import org.apache.ignite.transactions.Transaction;
@@ -103,4 +104,24 @@ public Transaction txStart(TransactionConcurrency concurrency, TransactionIsolat
      * Resets transaction metrics.
      */
     public void resetMetrics();
-}
\ No newline at end of file
+
+    /**
+     * Returns a list of active transactions initiated by this node.
+     * <p>
+     * Note: returned transaction handle will only support getters, {@link Transaction#close()},
+     * {@link Transaction#rollback()}, {@link Transaction#rollbackAsync()} methods.
+     * Trying to invoke other methods will lead to UnsupportedOperationException.
+     *
+     * @return Transactions started on local node.
+     */
+    public Collection<Transaction> localActiveTransactions();
+
+    /**
+     * Returns instance of Ignite Transactions to mark a transaction with a special label.
+     *
+     * @param lb label.
+     * @return {@code This} for chaining.
+     * @throws NullPointerException if label is null.
+     */
+    public IgniteTransactions withLabel(String lb);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/TransactionConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/TransactionConfiguration.java
index 0063afc158fd1..818f8235cfc7f 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/TransactionConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/TransactionConfiguration.java
@@ -42,6 +42,9 @@ public class TransactionConfiguration implements Serializable {
     /** Default transaction timeout. */
     public static final long DFLT_TRANSACTION_TIMEOUT = 0;
 
+    /** Transaction timeout on partition map synchronization. */
+    public static final long TX_TIMEOUT_ON_PARTITION_MAP_EXCHANGE = 0;
+
     /** Default size of pessimistic transactions log. */
     public static final int DFLT_PESSIMISTIC_TX_LOG_LINGER = 10_000;
 
@@ -57,6 +60,9 @@ public class TransactionConfiguration implements Serializable {
     /** Default transaction timeout. */
     private long dfltTxTimeout = DFLT_TRANSACTION_TIMEOUT;
 
+    /** Transaction timeout on partition map exchange. */
+    private volatile long txTimeoutOnPartitionMapExchange = TX_TIMEOUT_ON_PARTITION_MAP_EXCHANGE;
+
     /** Pessimistic tx log size. */
     private int pessimisticTxLogSize;
 
@@ -89,6 +95,7 @@ public TransactionConfiguration(TransactionConfiguration cfg) {
         dfltConcurrency = cfg.getDefaultTxConcurrency();
         dfltIsolation = cfg.getDefaultTxIsolation();
         dfltTxTimeout = cfg.getDefaultTxTimeout();
+        txTimeoutOnPartitionMapExchange = cfg.getTxTimeoutOnPartitionMapExchange();
         pessimisticTxLogLinger = cfg.getPessimisticTxLogLinger();
         pessimisticTxLogSize = cfg.getPessimisticTxLogSize();
         txSerEnabled = cfg.isTxSerializableEnabled();
@@ -191,6 +198,40 @@ public TransactionConfiguration setDefaultTxTimeout(long dfltTxTimeout) {
         return this;
     }
 
+    /**
+     * Some Ignite operations provoke partition map exchange process within Ignite to ensure the partitions distribution
+     * state is synchronized cluster-wide. Topology update events and a start of a new distributed cache are examples
+     * of those operations.
+     * <p>
+     * When the partition map exchange starts, Ignite acquires a global lock at a particular stage. The lock can't be
+     * obtained until pending transactions are running in parallel. If there is a transaction that runs for a while,
+     * then it will prevent the partition map exchange process from the start freezing some operations such as a new
+     * node join process.
+     * <p>
+     * This property allows to rollback such long transactions to let Ignite acquire the lock faster and initiate the
+     * partition map exchange process. The timeout is enforced only at the time of the partition map exchange process.
+     * <p>
+     * If not set, default value is {@link #TX_TIMEOUT_ON_PARTITION_MAP_EXCHANGE} which means transactions will never be
+     * rolled back on partition map exchange.
+     *
+     * @return Transaction timeout for partition map synchronization in milliseconds.
+     */
+    public long getTxTimeoutOnPartitionMapExchange() {
+        return txTimeoutOnPartitionMapExchange;
+    }
+
+    /**
+     * Sets the transaction timeout that will be enforced if the partition map exchange process starts.
+     *
+     * @param txTimeoutOnPartitionMapExchange Transaction timeout value in milliseconds.
+     * @return {@code this} for chaining.
+     */
+    public TransactionConfiguration setTxTimeoutOnPartitionMapExchange(long txTimeoutOnPartitionMapExchange) {
+        this.txTimeoutOnPartitionMapExchange = txTimeoutOnPartitionMapExchange;
+
+        return this;
+    }
+
     /**
      * Gets size of pessimistic transactions log stored on node in order to recover transaction commit if originating
      * node has left grid before it has sent all messages to transaction nodes.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 565a8ce31a804..f4c7ccea9eab0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -201,6 +201,7 @@
 import org.apache.ignite.mxbean.StripedExecutorMXBean;
 import org.apache.ignite.mxbean.WorkersControlMXBean;
 import org.apache.ignite.mxbean.ThreadPoolMXBean;
+import org.apache.ignite.mxbean.TransactionsMXBean;
 import org.apache.ignite.mxbean.TransactionMetricsMxBean;
 import org.apache.ignite.plugin.IgnitePlugin;
 import org.apache.ignite.plugin.PluginNotFoundException;
@@ -4191,9 +4192,13 @@ private void registerAllMBeans(
             ClusterMetricsMXBean metricsBean = new ClusterMetricsMXBeanImpl(cluster());
             registerMBean("Kernal", metricsBean.getClass().getSimpleName(), metricsBean, ClusterMetricsMXBean.class);
 
+            // Transaction metrics
+            TransactionMetricsMxBean txMetricsMXBean = new TransactionMetricsMxBeanImpl(ctx.cache().transactions().metrics());
+            registerMBean("TransactionMetrics", txMetricsMXBean.getClass().getSimpleName(), txMetricsMXBean, TransactionMetricsMxBean.class);
+
             // Transactions
-            TransactionMetricsMxBean txMXBean = new TransactionMetricsMxBeanImpl(ctx.cache().transactions().metrics());
-            registerMBean("Transactions", txMXBean.getClass().getSimpleName(), txMXBean, TransactionMetricsMxBean.class);
+            TransactionsMXBean txMXBean = new TransactionsMXBeanImpl(ctx);
+            registerMBean("Transactions", txMXBean.getClass().getSimpleName(), txMXBean, TransactionsMXBean.class);
 
             // Executors
             registerExecutorMBean("GridUtilityCacheExecutor", utilityCachePool);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
new file mode 100644
index 0000000000000..6937ebd146c4a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.tx.VisorTxInfo;
+import org.apache.ignite.internal.visor.tx.VisorTxOperation;
+import org.apache.ignite.internal.visor.tx.VisorTxProjection;
+import org.apache.ignite.internal.visor.tx.VisorTxSortOrder;
+import org.apache.ignite.internal.visor.tx.VisorTxTask;
+import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
+import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
+import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.mxbean.TransactionsMXBean;
+
+/**
+ * TransactionsMXBean implementation.
+ */
+public class TransactionsMXBeanImpl implements TransactionsMXBean {
+    /** */
+    private final GridKernalContextImpl ctx;
+
+    /**
+     * @param ctx Context.
+     */
+    TransactionsMXBeanImpl(GridKernalContextImpl ctx) {
+        this.ctx = ctx;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String getActiveTransactions(Long minDuration, Integer minSize, String prj, String consistentIds,
+        String xid, String lbRegex, Integer limit, String order, boolean detailed, boolean kill) {
+        try {
+            IgniteCompute compute = ctx.cluster().get().compute();
+
+            VisorTxProjection proj = null;
+
+            if (prj != null) {
+                if ("clients".equals(prj))
+                    proj = VisorTxProjection.CLIENT;
+                else if ("servers".equals(prj))
+                    proj = VisorTxProjection.SERVER;
+            }
+
+            List<String> consIds = null;
+
+            if (consistentIds != null)
+                consIds = Arrays.stream(consistentIds.split(",")).collect(Collectors.toList());
+
+            VisorTxSortOrder sortOrder = null;
+
+            if (order != null) {
+                if ("DURATION".equals(order))
+                    sortOrder = VisorTxSortOrder.DURATION;
+                else if ("SIZE".equals(order))
+                    sortOrder = VisorTxSortOrder.SIZE;
+            }
+
+            VisorTxTaskArg arg = new VisorTxTaskArg(kill ? VisorTxOperation.KILL : VisorTxOperation.LIST,
+                limit, minDuration == null ? null : minDuration * 1000, minSize, null, proj, consIds, xid, lbRegex, sortOrder);
+
+            Map<ClusterNode, VisorTxTaskResult> res = compute.execute(new VisorTxTask(),
+                new VisorTaskArgument<>(ctx.cluster().get().localNode().id(), arg, false));
+
+            if (detailed) {
+                StringWriter sw = new StringWriter();
+
+                PrintWriter w = new PrintWriter(sw);
+
+                for (Map.Entry<ClusterNode, VisorTxTaskResult> entry : res.entrySet()) {
+                    if (entry.getValue().getInfos().isEmpty())
+                        continue;
+
+                    ClusterNode key = entry.getKey();
+
+                    w.println(key.toString());
+
+                    for (VisorTxInfo info : entry.getValue().getInfos())
+                        w.println("    Tx: [xid=" + info.getXid() +
+                            ", label=" + info.getLabel() +
+                            ", state=" + info.getState() +
+                            ", duration=" + info.getDuration() / 1000 +
+                            ", isolation=" + info.getIsolation() +
+                            ", concurrency=" + info.getConcurrency() +
+                            ", timeout=" + info.getTimeout() +
+                            ", size=" + info.getSize() +
+                            ", dhtNodes=" + F.transform(info.getPrimaryNodes(), new IgniteClosure<UUID, String>() {
+                            @Override public String apply(UUID id) {
+                                return U.id8(id);
+                            }
+                        }) +
+                            ']');
+                }
+
+                w.flush();
+
+                return sw.toString();
+            }
+            else {
+                int cnt = 0;
+
+                for (VisorTxTaskResult result : res.values())
+                    cnt += result.getInfos().size();
+
+                return Integer.toString(cnt);
+            }
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e.getMessage());
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getTxTimeoutOnPartitionMapExchange() {
+        return ctx.config().getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setTxTimeoutOnPartitionMapExchange(long timeout) {
+        try {
+            ctx.grid().context().cache().setTxTimeoutOnPartitionMapExchange(timeout);
+        }
+        catch (Exception e) {
+            throw new RuntimeException(e.getMessage());
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(TransactionsMXBeanImpl.class, this);
+    }
+}
+
+
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
index 43e97b5e4ac94..98fb8ce0d841e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterAsyncImpl.java
@@ -171,6 +171,11 @@ public IgniteClusterAsyncImpl(IgniteClusterImpl cluster) {
         cluster.enableStatistics(caches, enabled);
     }
 
+    /** {@inheritDoc} */
+    @Override public void setTxTimeoutOnPartitionMapExchange(long timeout) {
+        cluster.setTxTimeoutOnPartitionMapExchange(timeout);
+    }
+
     /** {@inheritDoc} */
     @Override public Ignite ignite() {
         return cluster.ignite();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
index b69923b5bb783..e28342cd6f3a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java
@@ -501,6 +501,21 @@ private Collection<Object> getConsistentIds(Collection<? extends BaselineNode> n
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void setTxTimeoutOnPartitionMapExchange(long timeout) {
+        guard();
+
+        try {
+            ctx.cache().setTxTimeoutOnPartitionMapExchange(timeout);
+        }
+        catch (IgniteCheckedException e) {
+            throw U.convertException(e);
+        }
+        finally {
+            unguard();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public IgniteCluster withAsync() {
         return new IgniteClusterAsyncImpl(this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index 23c8eecdca664..660edf4350ee4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.commandline;
 
+import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
+
 /**
  * Bean with all parsed and validated arguments.
  */
@@ -49,6 +51,9 @@ public class Arguments {
      */
     private String baselineArgs;
 
+    /** Transaction arguments. */
+    private final VisorTxTaskArg txArg;
+
     /**
      * @param cmd Command.
      * @param host Host.
@@ -57,11 +62,12 @@ public class Arguments {
      * @param pwd Password.
      * @param baselineAct Baseline action.
      * @param baselineArgs Baseline args.
+     * @param txArg TX arg.
      * @param force Force flag.
      */
     public Arguments(Command cmd, String host, String port, String user, String pwd,
         String baselineAct, String baselineArgs,
-        boolean force
+        VisorTxTaskArg txArg, boolean force
     ) {
         this.cmd = cmd;
         this.host = host;
@@ -71,6 +77,7 @@ public Arguments(Command cmd, String host, String port, String user, String pwd,
         this.baselineAct = baselineAct;
         this.baselineArgs = baselineArgs;
         this.force = force;
+        this.txArg = txArg;
     }
 
     /**
@@ -122,6 +129,13 @@ public String baselineArguments() {
         return baselineArgs;
     }
 
+    /**
+     * @return Transaction arguments.
+     */
+    public VisorTxTaskArg transactionArguments() {
+        return txArg;
+    }
+
     /**
      * @return Force option.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
index e73a24fe104ba..c8c7db5cafad5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
@@ -31,7 +31,10 @@ public enum Command {
     STATE("--state"),
 
     /** */
-    BASELINE("--baseline");
+    BASELINE("--baseline"),
+
+    /** */
+    TX("--tx");
 
     /** */
     private final String text;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 5993f593c7342..3bc5c1a7e8c66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -24,6 +24,11 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
+import java.util.UUID;
+import java.util.logging.Logger;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.client.GridClient;
 import org.apache.ignite.internal.client.GridClientAuthenticationException;
 import org.apache.ignite.internal.client.GridClientClosedException;
@@ -39,15 +44,23 @@
 import org.apache.ignite.internal.client.impl.connection.GridClientConnectionResetException;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.apache.ignite.internal.visor.baseline.VisorBaselineNode;
 import org.apache.ignite.internal.visor.baseline.VisorBaselineOperation;
 import org.apache.ignite.internal.visor.baseline.VisorBaselineTask;
 import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskArg;
 import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskResult;
+import org.apache.ignite.internal.visor.tx.VisorTxInfo;
+import org.apache.ignite.internal.visor.tx.VisorTxOperation;
+import org.apache.ignite.internal.visor.tx.VisorTxProjection;
+import org.apache.ignite.internal.visor.tx.VisorTxSortOrder;
+import org.apache.ignite.internal.visor.tx.VisorTxTask;
+import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
+import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
+import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
-import org.jetbrains.annotations.NotNull;
 
 import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR;
 import static org.apache.ignite.internal.IgniteVersionUtils.COPYRIGHT;
@@ -55,6 +68,7 @@
 import static org.apache.ignite.internal.commandline.Command.BASELINE;
 import static org.apache.ignite.internal.commandline.Command.DEACTIVATE;
 import static org.apache.ignite.internal.commandline.Command.STATE;
+import static org.apache.ignite.internal.commandline.Command.TX;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.ADD;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.COLLECT;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.REMOVE;
@@ -65,6 +79,9 @@
  * Class that execute several commands passed via command line.
  */
 public class CommandHandler {
+    /** Logger. */
+    private static final Logger log = Logger.getLogger(CommandHandler.class.getName());
+
     /** */
     static final String DFLT_HOST = "127.0.0.1";
 
@@ -86,6 +103,9 @@ public class CommandHandler {
     /** */
     private static final String CMD_USER = "--user";
 
+    /** */
+    public static final String CONFIRM_MSG = "y";
+
     /** */
     private static final String BASELINE_ADD = "add";
 
@@ -125,12 +145,45 @@ public class CommandHandler {
     /** */
     private static final Scanner IN = new Scanner(System.in);
 
+    /** */
+    private static final String TX_LIMIT = "limit";
+
+    /** */
+    private static final String TX_ORDER = "order";
+
+    /** */
+    private static final String TX_SERVERS = "servers";
+
+    /** */
+    private static final String TX_CLIENTS = "clients";
+
+    /** */
+    private static final String TX_DURATION = "minDuration";
+
+    /** */
+    private static final String TX_SIZE = "minSize";
+
+    /** */
+    private static final String TX_LABEL = "label";
+
+    /** */
+    private static final String TX_NODES = "nodes";
+
+    /** */
+    private static final String TX_XID = "xid";
+
+    /** */
+    private static final String TX_KILL = "kill";
+
     /** */
     private Iterator<String> argsIt;
 
     /** */
     private String peekedArg;
 
+    /** */
+    private Object lastOperationResult;
+
     /**
      * Output specified string to console.
      *
@@ -198,7 +251,7 @@ private boolean confirm(Arguments args) {
         if (prompt == null)
             return true;
 
-        return "y".equalsIgnoreCase(readLine(prompt));
+        return CONFIRM_MSG.equalsIgnoreCase(readLine(prompt));
     }
 
     /**
@@ -219,9 +272,15 @@ private String confirmationPrompt(Arguments args) {
             case BASELINE:
                 if (!BASELINE_COLLECT.equals(args.baselineAction()))
                     str = "Warning: the command will perform changes in baseline.";
+                break;
+
+            case TX:
+                if (args.transactionArguments().getOperation() == VisorTxOperation.KILL)
+                    str = "Warning: the command will kill some transactions.";
+                break;
         }
 
-        return str == null ? null : str + "\nPress 'y' to continue...";
+        return str == null ? null : str + "\nPress '" + CONFIRM_MSG + "' to continue . . . ";
     }
 
     /**
@@ -300,15 +359,40 @@ private void state(GridClient client) throws Throwable {
         }
     }
 
+    /**
+     * @param client Client.
+     * @param arg Task argument.
+     * @return Task result.
+     * @throws GridClientException If failed to execute task.
+     */
+    private Map<UUID, VisorTxTaskResult> executeTransactionsTask(GridClient client,
+        VisorTxTaskArg arg) throws GridClientException {
+
+        return executeTask(client, VisorTxTask.class, arg);
+    }
+
     /**
      *
-     * @param client Client
+     * @param client Client.
+     * @param taskCls Task class.
+     * @param taskArgs Task arguments.
      * @return Task result.
      * @throws GridClientException If failed to execute task.
      */
     private <R> R executeTask(GridClient client, Class<?> taskCls, Object taskArgs) throws GridClientException {
         GridClientCompute compute = client.compute();
 
+        GridClientNode node = getBalancedNode(compute);
+
+        return compute.execute(taskCls.getName(),
+            new VisorTaskArgument<>(node.nodeId(), taskArgs, false));
+    }
+
+    /**
+     * @param compute instance
+     * @return balanced node
+     */
+    private GridClientNode getBalancedNode(GridClientCompute compute) throws GridClientException {
         List<GridClientNode> nodes = new ArrayList<>();
 
         for (GridClientNode node : compute.nodes())
@@ -318,10 +402,7 @@ private <R> R executeTask(GridClient client, Class<?> taskCls, Object taskArgs)
         if (F.isEmpty(nodes))
             throw new GridClientDisconnectedException("Connectable node not found", null);
 
-        GridClientNode node = compute.balancer().balancedNode(nodes);
-
-        return compute.projection(node).execute(taskCls.getName(),
-            new VisorTaskArgument<>(node.nodeId(), taskArgs, false));
+        return compute.balancer().balancedNode(nodes);
     }
 
     /**
@@ -368,13 +449,7 @@ private VisorBaselineTaskArg arg(VisorBaselineOperation op, String s) {
             case ADD:
             case REMOVE:
             case SET:
-                if(F.isEmpty(s))
-                    throw new IllegalArgumentException("Empty list of consistent IDs");
-
-                List<String> consistentIds = new ArrayList<>();
-
-                for (String consistentId : s.split(","))
-                    consistentIds.add(consistentId.trim());
+                List<String> consistentIds = getConsistentIds(s);
 
                 return new VisorBaselineTaskArg(op, -1, consistentIds);
 
@@ -393,6 +468,22 @@ private VisorBaselineTaskArg arg(VisorBaselineOperation op, String s) {
         }
     }
 
+    /**
+     * @param s String of consisted ids delimited by comma.
+     * @return List of consistent ids.
+     */
+    private List<String> getConsistentIds(String s) {
+        if (F.isEmpty(s))
+            throw new IllegalArgumentException("Empty list of consistent IDs");
+
+        List<String> consistentIds = new ArrayList<>();
+
+        for (String consistentId : s.split(","))
+            consistentIds.add(consistentId.trim());
+
+        return consistentIds;
+    }
+
     /**
      * Print baseline topology.
      *
@@ -404,6 +495,7 @@ private void baselinePrint0(VisorBaselineTaskResult res) {
         nl();
 
         Map<String, VisorBaselineNode> baseline = res.getBaseline();
+
         Map<String, VisorBaselineNode> servers = res.getServers();
 
         if (F.isEmpty(baseline))
@@ -531,6 +623,57 @@ private void baselineVersion(GridClient client, String arg) throws GridClientExc
         }
     }
 
+    /**
+     * Dump transactions information.
+     *
+     * @param client Client.
+     * @param arg Transaction search arguments
+     */
+    private void transactions(GridClient client, VisorTxTaskArg arg) throws GridClientException {
+        try {
+            Map<ClusterNode, VisorTxTaskResult> res = executeTask(client, VisorTxTask.class, arg);
+
+            lastOperationResult = res;
+
+            if (res.isEmpty())
+                log("Nothing found.");
+            else if (arg.getOperation() == VisorTxOperation.KILL)
+                log("Killed transactions:");
+            else
+                log("Matching transactions:");
+
+            for (Map.Entry<ClusterNode, VisorTxTaskResult> entry : res.entrySet()) {
+                if (entry.getValue().getInfos().isEmpty())
+                    continue;
+
+                ClusterNode key = entry.getKey();
+
+                log(key.toString());
+
+                for (VisorTxInfo info : entry.getValue().getInfos())
+                    log("    Tx: [xid=" + info.getXid() +
+                        ", label=" + info.getLabel() +
+                        ", state=" + info.getState() +
+                        ", duration=" + info.getDuration() / 1000 +
+                        ", isolation=" + info.getIsolation() +
+                        ", concurrency=" + info.getConcurrency() +
+                        ", timeout=" + info.getTimeout() +
+                        ", size=" + info.getSize() +
+                        ", dhtNodes=" + F.transform(info.getPrimaryNodes(), new IgniteClosure<UUID, String>() {
+                        @Override public String apply(UUID id) {
+                            return U.id8(id);
+                        }
+                    }) +
+                        ']');
+            }
+        }
+        catch (Throwable e) {
+            log("Failed to perform operation.");
+
+            throw e;
+        }
+    }
+
     /**
      * @param e Exception to check.
      * @return {@code true} if specified exception is {@link GridClientAuthenticationException}.
@@ -603,7 +746,7 @@ private String peekNextArg() {
      * @return Arguments bean.
      * @throws IllegalArgumentException In case arguments aren't valid.
      */
-    @NotNull Arguments parseAndValidate(List<String> rawArgs) {
+    Arguments parseAndValidate(List<String> rawArgs) {
         String host = DFLT_HOST;
 
         String port = DFLT_PORT;
@@ -622,6 +765,8 @@ private String peekNextArg() {
 
         initArgIterator(rawArgs);
 
+        VisorTxTaskArg txArgs = null;
+
         while (hasNextArg()) {
             String str = nextArg("").toLowerCase();
 
@@ -632,7 +777,14 @@ private String peekNextArg() {
                     case ACTIVATE:
                     case DEACTIVATE:
                     case STATE:
-                        commands.add(Command.of(str));
+                        commands.add(cmd);
+                        break;
+
+                    case TX:
+                        commands.add(TX);
+
+                        txArgs = parseTransactionArguments();
+
                         break;
 
                     case BASELINE:
@@ -652,6 +804,11 @@ private String peekNextArg() {
                                 baselineArgs = nextArg("Expected baseline arguments");
                             }
                         }
+
+                        break;
+
+                    default:
+                        throw new IllegalArgumentException("Unexpected command: " + str);
                 }
             }
             else {
@@ -685,6 +842,7 @@ private String peekNextArg() {
                     case CMD_FORCE:
                         force = true;
                         break;
+
                     default:
                         throw new IllegalArgumentException("Unexpected argument: " + str);
                 }
@@ -707,7 +865,138 @@ private String peekNextArg() {
         if (hasUsr != hasPwd)
             throw new IllegalArgumentException("Both user and password should be specified");
 
-        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, force);
+        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, force);
+    }
+
+    /**
+     * @return Transaction arguments.
+     */
+    private VisorTxTaskArg parseTransactionArguments() {
+        VisorTxProjection proj = null;
+
+        Integer limit = null;
+
+        VisorTxSortOrder sortOrder = null;
+
+        Long duration = null;
+
+        Integer size = null;
+
+        String lbRegex = null;
+
+        List<String> consistentIds = null;
+
+        VisorTxOperation op = VisorTxOperation.LIST;
+
+        String xid = null;
+
+        boolean end = false;
+
+        do {
+            String str = peekNextArg();
+
+            if (str == null)
+                break;
+
+            switch (str) {
+                case TX_LIMIT:
+                    nextArg("");
+
+                    limit = (int) nextLongArg(TX_LIMIT);
+                    break;
+
+                case TX_ORDER:
+                    nextArg("");
+
+                    sortOrder = VisorTxSortOrder.fromString(nextArg(TX_ORDER));
+
+                    break;
+
+                case TX_SERVERS:
+                    nextArg("");
+
+                    proj = VisorTxProjection.SERVER;
+                    break;
+
+                case TX_CLIENTS:
+                    nextArg("");
+
+                    proj = VisorTxProjection.CLIENT;
+                    break;
+
+                case TX_NODES:
+                    nextArg("");
+
+                    consistentIds = getConsistentIds(nextArg(TX_NODES));
+                    break;
+
+                case TX_DURATION:
+                    nextArg("");
+
+                    duration = nextLongArg(TX_DURATION) * 1000L;
+                    break;
+
+                case TX_SIZE:
+                    nextArg("");
+
+                    size = (int) nextLongArg(TX_SIZE);
+                    break;
+
+                case TX_LABEL:
+                    nextArg("");
+
+                    lbRegex = nextArg(TX_LABEL);
+
+                    try {
+                        Pattern.compile(lbRegex);
+                    }
+                    catch (PatternSyntaxException e) {
+                        throw new IllegalArgumentException("Illegal regex syntax");
+                    }
+
+                    break;
+
+                case TX_XID:
+                    nextArg("");
+
+                    xid = nextArg(TX_XID);
+                    break;
+
+                case TX_KILL:
+                    nextArg("");
+
+                    op = VisorTxOperation.KILL;
+                    break;
+
+                default:
+                    end = true;
+            }
+        }
+        while (!end);
+
+        if (proj != null && consistentIds != null)
+            throw new IllegalArgumentException("Projection can't be used together with list of consistent ids.");
+
+        return new VisorTxTaskArg(op, limit, duration, size, null, proj, consistentIds, xid, lbRegex, sortOrder);
+    }
+
+    /**
+     * @return Numeric value.
+     */
+    private long nextLongArg(String lb) {
+        String str = nextArg("Expecting " + lb);
+
+        try {
+            long val = Long.parseLong(str);
+
+            if (val < 0)
+                throw new IllegalArgumentException("Invalid value for " + lb + ": " + val);
+
+            return val;
+        }
+        catch (NumberFormatException e) {
+            throw new IllegalArgumentException("Invalid value for " + lb + ": " + str);
+        }
     }
 
     /**
@@ -734,8 +1023,11 @@ public int execute(List<String> rawArgs) {
                 usage("  Remove nodes from baseline topology:", BASELINE, " remove consistentId1[,consistentId2,....,consistentIdN] [--force]");
                 usage("  Set baseline topology:", BASELINE, " set consistentId1[,consistentId2,....,consistentIdN] [--force]");
                 usage("  Set baseline topology based on version:", BASELINE, " version topologyVersion [--force]");
+                usage("  List or kill transactions:", TX, " [xid XID] [minDuration SECONDS] " +
+                    "[minSize SIZE] [label PATTERN_REGEX] [servers|clients] " +
+                    "[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE] [kill] [--force]");
 
-                log("By default cluster deactivation and changes in baseline topology commands request interactive confirmation. ");
+                log("By default commands affecting the cluster require interactive confirmation. ");
                 log("  --force option can be used to execute commands without prompting for confirmation.");
                 nl();
 
@@ -757,7 +1049,7 @@ public int execute(List<String> rawArgs) {
             Arguments args = parseAndValidate(rawArgs);
 
             if (!confirm(args)) {
-                log("Operation canceled.");
+                log("Operation cancelled.");
 
                 return EXIT_CODE_OK;
             }
@@ -789,6 +1081,10 @@ public int execute(List<String> rawArgs) {
                     case BASELINE:
                         baseline(client, args.baselineAction(), args.baselineArguments());
                         break;
+
+                    case TX:
+                        transactions(client, args.transactionArguments());
+                        break;
                 }
             }
 
@@ -816,5 +1112,14 @@ public static void main(String[] args) {
 
         System.exit(hnd.execute(Arrays.asList(args)));
     }
+
+    /**
+     * Used for tests.
+     * @return Last operation result;
+     */
+    @SuppressWarnings("unchecked")
+    public <T> T getLastOperationResult() {
+        return (T)lastOperationResult;
+    }
 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerUtils.java
index aa4bfd6a2cd66..4626df72055f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerUtils.java
@@ -203,7 +203,7 @@ static OptimizedClassDescriptor classDescriptor(
             try {
                 registered = ctx.registerClassName(JAVA_ID, typeId, cls.getName());
             }
-            catch (IgniteCheckedException e) {
+            catch (Exception e) {
                 throw new IOException("Failed to register class: " + cls.getName(), e);
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index bd613a191f407..567cc6e71083f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -89,6 +89,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
@@ -4081,7 +4082,8 @@ public void awaitLastFut() {
                     READ_COMMITTED,
                     tCfg.getDefaultTxTimeout(),
                     !ctx.skipStore(),
-                    0
+                    0,
+                    null
                 );
 
                 assert tx != null;
@@ -4105,7 +4107,8 @@ public void awaitLastFut() {
                                 tx.xid(), e);
                         }
                         catch (IgniteCheckedException | AssertionError | RuntimeException e1) {
-                            U.error(log, "Failed to rollback transaction (cache may contain stale locks): " + tx, e1);
+                            U.error(log, "Failed to rollback transaction (cache may contain stale locks): " +
+                                CU.txString(tx), e1);
 
                             if (e != e1)
                                 e.addSuppressed(e1);
@@ -4180,7 +4183,8 @@ private <T> IgniteInternalFuture<T> asyncOp(final AsyncOp<T> op) {
                     READ_COMMITTED,
                     txCfg.getDefaultTxTimeout(),
                     !skipStore,
-                    0);
+                    0,
+                    null);
 
                 return asyncOp(tx, op, opCtx, /*retry*/false);
             }
@@ -4224,6 +4228,31 @@ protected <T> IgniteInternalFuture<T> asyncOp(
 
             final GridNearTxLocal tx0 = tx;
 
+            final CX1 clo = new CX1<IgniteInternalFuture<T>, T>() {
+                @Override public T applyx(IgniteInternalFuture<T> tFut) throws IgniteCheckedException {
+                    try {
+                        return tFut.get();
+                    }
+                    catch (IgniteTxTimeoutCheckedException | IgniteTxRollbackCheckedException | NodeStoppingException e) {
+                        throw e;
+                    }
+                    catch (IgniteCheckedException e1) {
+                        try {
+                            tx0.rollbackNearTxLocalAsync();
+                        }
+                        catch (Throwable e2) {
+                            if (e1 != e2)
+                                e1.addSuppressed(e2);
+                        }
+
+                        throw e1;
+                    }
+                    finally {
+                        ctx.shared().txContextReset();
+                    }
+                }
+            };
+
             if (fut != null && !fut.isDone()) {
                 IgniteInternalFuture<T> f = new GridEmbeddedFuture(fut,
                     new IgniteOutClosure<IgniteInternalFuture>() {
@@ -4233,31 +4262,7 @@ protected <T> IgniteInternalFuture<T> asyncOp(
                                     new IgniteCheckedException("Operation has been cancelled (node is stopping)."));
 
                             try {
-                                return op.op(tx0, opCtx).chain(new CX1<IgniteInternalFuture<T>, T>() {
-                                    @Override
-                                    public T applyx(IgniteInternalFuture<T> tFut) throws IgniteCheckedException {
-                                        try {
-                                            return tFut.get();
-                                        }
-                                        catch (IgniteTxRollbackCheckedException | NodeStoppingException e) {
-                                            throw e;
-                                        }
-                                        catch (IgniteCheckedException e1) {
-                                            try {
-                                                tx0.rollbackNearTxLocalAsync();
-                                            }
-                                            catch (Throwable e2) {
-                                                if (e1 != e2)
-                                                    e1.addSuppressed(e2);
-                                            }
-
-                                            throw e1;
-                                        }
-                                        finally {
-                                            ctx.shared().txContextReset();
-                                        }
-                                    }
-                                });
+                                return op.op(tx0, opCtx).chain(clo);
                             }
                             finally {
                                 // It is necessary to clear tx context in this thread as well.
@@ -4271,33 +4276,17 @@ public T applyx(IgniteInternalFuture<T> tFut) throws IgniteCheckedException {
                 return f;
             }
 
+            /**
+             * Wait for concurrent tx operation to finish.
+             * See {@link GridDhtTxLocalAdapter#updateLockFuture(IgniteInternalFuture, IgniteInternalFuture)}
+             */
+            if (!tx0.txState().implicitSingle())
+                tx0.txState().awaitLastFuture(ctx.shared());
+
             IgniteInternalFuture<T> f;
 
             try {
-                f = op.op(tx, opCtx).chain(new CX1<IgniteInternalFuture<T>, T>() {
-                    @Override public T applyx(IgniteInternalFuture<T> tFut) throws IgniteCheckedException {
-                        try {
-                            return tFut.get();
-                        }
-                        catch (IgniteTxRollbackCheckedException | NodeStoppingException e) {
-                            throw e;
-                        }
-                        catch (IgniteCheckedException e1) {
-                            try {
-                                tx0.rollbackNearTxLocalAsync(e1 instanceof IgniteTxTimeoutCheckedException);
-                            }
-                            catch (Throwable e2) {
-                                if (e2 != e1)
-                                    e1.addSuppressed(e2);
-                            }
-
-                            throw e1;
-                        }
-                        finally {
-                            ctx.shared().txContextReset();
-                        }
-                    }
-                });
+                f = op.op(tx, opCtx).chain(clo);
             }
             finally {
                 // It is necessary to clear tx context in this thread as well.
@@ -4858,7 +4847,8 @@ public void execute(boolean retry) {
                 READ_COMMITTED,
                 CU.transactionConfiguration(ctx, ctx.kernalContext().config()).getDefaultTxTimeout(),
                 opCtx == null || !opCtx.skipStore(),
-                0);
+                0,
+                null);
 
             IgniteInternalFuture<T> fut = asyncOp(tx, op, opCtx, retry);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 35f14da8c94ae..28d5d207f59c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -36,7 +36,6 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.LinkedBlockingDeque;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
@@ -47,6 +46,7 @@
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
@@ -2420,23 +2420,28 @@ else if (task instanceof ForceRebalanceExchangeTask) {
 
                             int dumpCnt = 0;
 
-                            final long futTimeout = 2 * cctx.gridConfig().getNetworkTimeout();
+                            IgniteConfiguration cfg = cctx.gridConfig();
+
+                            long rollbackTimeout = cfg.getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange();
+
+                            final long dumpTimeout = 2 * cctx.gridConfig().getNetworkTimeout();
 
                             long nextDumpTime = 0;
 
                             while (true) {
                                 try {
-                                    resVer = exchFut.get(futTimeout, TimeUnit.MILLISECONDS);
+                                    resVer = exchFut.get(rollbackTimeout > 0 ? rollbackTimeout : dumpTimeout);
 
                                     break;
                                 }
                                 catch (IgniteFutureTimeoutCheckedException ignored) {
-                                    U.warn(diagnosticLog, "Failed to wait for partition map exchange [" +
-                                        "topVer=" + exchFut.initialVersion() +
-                                        ", node=" + cctx.localNodeId() + "]. " +
-                                        "Dumping pending objects that might be the cause: ");
-
                                     if (nextDumpTime <= U.currentTimeMillis()) {
+                                        U.warn(diagnosticLog, "Failed to wait for partition map exchange [" +
+                                            "topVer=" + exchFut.initialVersion() +
+                                            ", node=" + cctx.localNodeId() + "]. " +
+                                            (rollbackTimeout == 0 ? "Consider changing TransactionConfiguration.txTimeoutOnPartitionMapSynchronization to non default value to avoid this message. " : "") +
+                                            "Dumping pending objects that might be the cause: ");
+
                                         try {
                                             dumpDebugInfo(exchFut);
                                         }
@@ -2444,7 +2449,13 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                             U.error(diagnosticLog, "Failed to dump debug information: " + e, e);
                                         }
 
-                                        nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, futTimeout);
+                                        nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, dumpTimeout);
+                                    }
+
+                                    if (rollbackTimeout > 0) {
+                                        rollbackTimeout = 0; // Try automatic rollback only once.
+
+                                        cctx.tm().rollbackOnTopologyChange(exchFut.initialVersion());
                                     }
                                 }
                                 catch (Exception e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index bceb8c70dd235..39c7e715c06f6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -229,6 +229,10 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     /** Enable/disable cache statistics futures. */
     private ConcurrentMap<UUID, EnableStatisticsFuture> enableStatisticsFuts = new ConcurrentHashMap<>();
 
+    /** The futures for changing transaction timeout on partition map exchange. */
+    private ConcurrentMap<UUID, TxTimeoutOnPartitionMapExchangeChangeFuture> txTimeoutOnPartitionMapExchangeFuts =
+        new ConcurrentHashMap<>();
+
     /** */
     private ClusterCachesInfo cachesInfo;
 
@@ -380,6 +384,11 @@ else if (task instanceof CacheStatisticsModeChangeTask) {
 
             processStatisticsModeChange(task0.message());
         }
+        else if (task instanceof TxTimeoutOnPartitionMapExchangeChangeTask) {
+            TxTimeoutOnPartitionMapExchangeChangeTask task0 = (TxTimeoutOnPartitionMapExchangeChangeTask)task;
+
+            processTxTimeoutOnPartitionMapExchangeChange(task0.message());
+        }
         else if (task instanceof StopCachesOnClientReconnectExchangeTask) {
             StopCachesOnClientReconnectExchangeTask task0 = (StopCachesOnClientReconnectExchangeTask)task;
 
@@ -683,7 +692,7 @@ private void cleanup(CacheConfiguration cfg, @Nullable Object rsrc, boolean near
 
         sharedCtx = createSharedContext(ctx, sessionListeners);
 
-        transactions = new IgniteTransactionsImpl(sharedCtx);
+        transactions = new IgniteTransactionsImpl(sharedCtx, null);
 
         // Start shared managers.
         for (GridCacheSharedManager mgr : sharedCtx.managers())
@@ -1057,6 +1066,9 @@ public void onKernalStopCaches(boolean cancel) {
         for (EnableStatisticsFuture fut : enableStatisticsFuts.values())
             fut.onDone(err);
 
+        for (TxTimeoutOnPartitionMapExchangeChangeFuture fut : txTimeoutOnPartitionMapExchangeFuts.values())
+            fut.onDone(err);
+
         for (CacheGroupContext grp : cacheGrps.values())
             grp.onDisconnected(reconnectFut);
 
@@ -2617,6 +2629,43 @@ public void processStatisticsModeChange(CacheStatisticsModeChangeMessage msg) {
         }
     }
 
+    /**
+     * Callback invoked from discovery thread when discovery custom message is received.
+     *
+     * @param msg Discovery message for changing transaction timeout on partition map exchange.
+     */
+    public void onTxTimeoutOnPartitionMapExchangeChange(TxTimeoutOnPartitionMapExchangeChangeMessage msg) {
+        assert msg != null;
+
+        if (msg.isInit()) {
+            TransactionConfiguration cfg = ctx.config().getTransactionConfiguration();
+
+            if (cfg.getTxTimeoutOnPartitionMapExchange() != msg.getTimeout())
+                cfg.setTxTimeoutOnPartitionMapExchange(msg.getTimeout());
+        }
+        else {
+            TxTimeoutOnPartitionMapExchangeChangeFuture fut = txTimeoutOnPartitionMapExchangeFuts.get(
+                msg.getRequestId());
+
+            if (fut != null)
+                fut.onDone();
+        }
+    }
+
+    /**
+     * The task for changing transaction timeout on partition map exchange processed by exchange worker.
+     *
+     * @param msg Message.
+     */
+    public void processTxTimeoutOnPartitionMapExchangeChange(TxTimeoutOnPartitionMapExchangeChangeMessage msg) {
+        assert msg != null;
+
+        long timeout = ctx.config().getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange();
+
+        if (timeout != msg.getTimeout())
+            ctx.config().getTransactionConfiguration().setTxTimeoutOnPartitionMapExchange(msg.getTimeout());
+    }
+
     /**
      * @param stoppedCaches Stopped caches.
      */
@@ -3413,6 +3462,9 @@ else if (msg0 instanceof WalStateFinishMessage)
         if (msg instanceof CacheStatisticsModeChangeMessage)
             onCacheStatisticsModeChange((CacheStatisticsModeChangeMessage)msg);
 
+        if (msg instanceof TxTimeoutOnPartitionMapExchangeChangeMessage)
+            onTxTimeoutOnPartitionMapExchangeChange((TxTimeoutOnPartitionMapExchangeChangeMessage)msg);
+
         return false;
     }
 
@@ -3963,6 +4015,9 @@ private void cancelFutures() {
 
         for (EnableStatisticsFuture fut : enableStatisticsFuts.values())
             fut.onDone(err);
+
+        for (TxTimeoutOnPartitionMapExchangeChangeFuture fut : txTimeoutOnPartitionMapExchangeFuts.values())
+            fut.onDone(err);
     }
 
     /**
@@ -4391,6 +4446,26 @@ public void enableStatistics(Collection<String> caches, boolean enabled) throws
         fut.get();
     }
 
+    /**
+     * Sets transaction timeout on partition map exchange.
+     *
+     * @param timeout Transaction timeout on partition map exchange in milliseconds.
+     */
+    public void setTxTimeoutOnPartitionMapExchange(long timeout) throws IgniteCheckedException {
+        UUID requestId = UUID.randomUUID();
+
+        TxTimeoutOnPartitionMapExchangeChangeFuture fut = new TxTimeoutOnPartitionMapExchangeChangeFuture(requestId);
+
+        txTimeoutOnPartitionMapExchangeFuts.put(requestId, fut);
+
+        TxTimeoutOnPartitionMapExchangeChangeMessage msg = new TxTimeoutOnPartitionMapExchangeChangeMessage(
+            requestId, timeout);
+
+        ctx.grid().context().discovery().sendCustomEvent(msg);
+
+        fut.get();
+    }
+
     /**
      * @param obj Object to clone.
      * @return Object copy.
@@ -4624,4 +4699,30 @@ private EnableStatisticsFuture(UUID id) {
             return S.toString(EnableStatisticsFuture.class, this);
         }
     }
+
+    /**
+     * The future for changing transaction timeout on partition map exchange.
+     */
+    private class TxTimeoutOnPartitionMapExchangeChangeFuture extends GridFutureAdapter<Void> {
+        /** */
+        private UUID id;
+
+        /**
+         * @param id Future ID.
+         */
+        private TxTimeoutOnPartitionMapExchangeChangeFuture(UUID id) {
+            this.id = id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean onDone(@Nullable Void res, @Nullable Throwable err) {
+            txTimeoutOnPartitionMapExchangeFuts.remove(id, this);
+            return super.onDone(res, err);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(TxTimeoutOnPartitionMapExchangeChangeFuture.class, this);
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index b5c409699cc09..b1955085e29db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -71,6 +71,7 @@
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_LOCAL_STORE_KEEPS_PRIMARY_ONLY;
+import static org.apache.ignite.transactions.TransactionState.MARKED_ROLLBACK;
 
 /**
  * Shared context.
@@ -955,9 +956,14 @@ public void cleanup() {
      * @throws IgniteCheckedException If failed.
      */
     public void endTx(GridNearTxLocal tx) throws IgniteCheckedException {
-        tx.txState().awaitLastFuture(this);
+        boolean clearThreadMap = txMgr.threadLocalTx(null) == tx;
+
+        if (clearThreadMap)
+            tx.txState().awaitLastFuture(this);
+        else
+            tx.state(MARKED_ROLLBACK);
 
-        tx.close();
+        tx.close(clearThreadMap);
     }
 
     /**
@@ -983,9 +989,14 @@ public IgniteInternalFuture<IgniteInternalTx> commitTxAsync(GridNearTxLocal tx)
      * @return Rollback future.
      */
     public IgniteInternalFuture rollbackTxAsync(GridNearTxLocal tx) throws IgniteCheckedException {
-        tx.txState().awaitLastFuture(this);
+        boolean clearThreadMap = txMgr.threadLocalTx(null) == tx;
+
+        if (clearThreadMap)
+            tx.txState().awaitLastFuture(this);
+        else
+            tx.state(MARKED_ROLLBACK);
 
-        return tx.rollbackNearTxLocalAsync();
+        return tx.rollbackNearTxLocalAsync(clearThreadMap, false);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index e244c75ad4a0e..bb64cc6f2e73e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -817,13 +817,15 @@ public static String txString(@Nullable IgniteInternalTx tx) {
         if (tx == null)
             return "null";
 
-        return tx.getClass().getSimpleName() + "[id=" + tx.xid() +
+        return tx.getClass().getSimpleName() + "[xid=" + tx.xid() +
+            ", xidVersion=" + tx.xidVersion() +
             ", concurrency=" + tx.concurrency() +
             ", isolation=" + tx.isolation() +
             ", state=" + tx.state() +
             ", invalidate=" + tx.isInvalidate() +
             ", rollbackOnly=" + tx.isRollbackOnly() +
             ", nodeId=" + tx.nodeId() +
+            ", timeout=" + tx.timeout() +
             ", duration=" + (U.currentTimeMillis() - tx.startTime()) + ']';
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxTimeoutOnPartitionMapExchangeChangeMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxTimeoutOnPartitionMapExchangeChangeMessage.java
new file mode 100644
index 0000000000000..5589c94fdfba7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxTimeoutOnPartitionMapExchangeChangeMessage.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.UUID;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Discovery message for changing transaction timeout on partition map exchange.
+ */
+public class TxTimeoutOnPartitionMapExchangeChangeMessage implements DiscoveryCustomMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final IgniteUuid id = IgniteUuid.randomUuid();
+
+    /** Request ID. */
+    private final UUID reqId;
+
+    /** Transaction timeout on partition map exchange in milliseconds. */
+    private final long timeout;
+
+    /** Init flag. */
+    private final boolean isInit;
+
+    /**
+     * Constructor for response.
+     *
+     * @param req Request message.
+     */
+    public TxTimeoutOnPartitionMapExchangeChangeMessage(TxTimeoutOnPartitionMapExchangeChangeMessage req) {
+        this.reqId = req.reqId;
+        this.timeout = req.timeout;
+        this.isInit = false;
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param reqId Request ID.
+     * @param timeout Transaction timeout on partition map exchange in milliseconds.
+     */
+    public TxTimeoutOnPartitionMapExchangeChangeMessage(UUID reqId, long timeout) {
+        this.reqId = reqId;
+        this.timeout = timeout;
+        this.isInit = true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteUuid id() {
+        return id;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public DiscoveryCustomMessage ackMessage() {
+        return isInit() ? new TxTimeoutOnPartitionMapExchangeChangeMessage(this) : null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMutable() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
+        DiscoCache discoCache) {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * Gets request ID.
+     *
+     * @return Request ID.
+     */
+    public UUID getRequestId() {
+        return reqId;
+    }
+
+    /**
+     * Gets transaction timeout on partition map exchange in milliseconds.
+     *
+     * @return Transaction timeout on partition map exchange in milliseconds.
+     */
+    public long getTimeout() {
+        return timeout;
+    }
+
+    /**
+     * Gets init flag.
+     *
+     * @return Init flag.
+     */
+    public boolean isInit() {
+        return isInit;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(TxTimeoutOnPartitionMapExchangeChangeMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxTimeoutOnPartitionMapExchangeChangeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxTimeoutOnPartitionMapExchangeChangeTask.java
new file mode 100644
index 0000000000000..6edfa586e2f45
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/TxTimeoutOnPartitionMapExchangeChangeTask.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * The task for changing transaction timeout on partition map exchange.
+ */
+public class TxTimeoutOnPartitionMapExchangeChangeTask implements CachePartitionExchangeWorkerTask {
+    /** Discovery message. */
+    private final TxTimeoutOnPartitionMapExchangeChangeMessage msg;
+
+    /**
+     * Constructor.
+     *
+     * @param msg Discovery message.
+     */
+    public TxTimeoutOnPartitionMapExchangeChangeTask(TxTimeoutOnPartitionMapExchangeChangeMessage msg) {
+        assert msg != null;
+        this.msg = msg;
+    }
+
+    /**
+     * Gets discovery message.
+     *
+     * @return Discovery message.
+     */
+    public TxTimeoutOnPartitionMapExchangeChangeMessage message() {
+        return msg;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean skipForExchangeMerge() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(TxTimeoutOnPartitionMapExchangeChangeTask.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxFinishSync.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxFinishSync.java
index 02d1e8ee36be2..7cc368a999b66 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxFinishSync.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxFinishSync.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.F;
@@ -338,4 +339,4 @@ public void onDisconnected(IgniteFuture<?> reconnectFut) {
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxMapping.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxMapping.java
index 45903aa5dc3cd..481c95486a278 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxMapping.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxMapping.java
@@ -41,7 +41,7 @@ public class GridDistributedTxMapping {
 
     /** Entries. */
     @GridToStringInclude
-    private Collection<IgniteTxEntry> entries;
+    private final Collection<IgniteTxEntry> entries;
 
     /** Explicit lock flag. */
     private boolean explicitLock;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index 27044ebee8e23..a692b2e8dc15d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -117,6 +117,9 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
     @GridToStringInclude
     protected IgniteTxRemoteState txState;
 
+    /** {@code True} if tx should skip adding itself to completed version map on finish. */
+    private boolean skipCompletedVers;
+
     /**
      * Empty constructor required for {@link Externalizable}.
      */
@@ -228,8 +231,7 @@ public GridDistributedTxRemoteAdapter(
     @Override public GridTuple<CacheObject> peek(GridCacheContext cacheCtx,
         boolean failFast,
         KeyCacheObject key)
-        throws GridCacheFilterFailedException
-    {
+        throws GridCacheFilterFailedException {
         assert false : "Method peek can only be called on user transaction: " + this;
 
         throw new IllegalStateException("Method peek can only be called on user transaction: " + this);
@@ -869,7 +871,7 @@ public void forceCommit() throws IgniteCheckedException {
             // Note that we don't evict near entries here -
             // they will be deleted by their corresponding transactions.
             if (state(ROLLING_BACK) || state() == UNKNOWN) {
-                cctx.tm().rollbackTx(this, false);
+                cctx.tm().rollbackTx(this, false, skipCompletedVers);
 
                 state(ROLLED_BACK);
             }
@@ -898,6 +900,20 @@ public void forceCommit() throws IgniteCheckedException {
         // No-op.
     }
 
+    /**
+     * @return {@code True} if tx should skip adding itself to completed version map on finish.
+     */
+    public boolean skipCompletedVersions() {
+        return skipCompletedVers;
+    }
+
+    /**
+     * @param skipCompletedVers {@code True} if tx should skip adding itself to completed version map on finish.
+     */
+    public void skipCompletedVersions(boolean skipCompletedVers) {
+        this.skipCompletedVers = skipCompletedVers;
+    }
+
     /**
      * Adds explicit version if there is one.
      *
@@ -925,4 +941,5 @@ protected void addExplicit(IgniteTxEntry e) {
     @Override public String toString() {
         return GridToStringBuilder.toString(GridDistributedTxRemoteAdapter.class, this, "super", super.toString());
     }
+
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
index b4447b725fbc9..2869bb6441652 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
@@ -52,6 +52,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheMappedVersion;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
@@ -67,6 +68,7 @@
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.NotNull;
@@ -259,6 +261,38 @@ public GridDhtLockFuture(
             msgLog = cctx.shared().txLockMessageLogger();
             log = U.logger(cctx.kernalContext(), logRef, GridDhtLockFuture.class);
         }
+
+        if (tx != null) {
+            while(true) {
+                IgniteInternalFuture<Boolean> fut = tx.lockFut;
+
+                if (fut != null) {
+                    if (fut == GridDhtTxLocalAdapter.ROLLBACK_FUT)
+                        onError(tx.timedOut() ? tx.timeoutException() : tx.rollbackException());
+                    else {
+                        // Wait for collocated lock future.
+                        assert fut instanceof GridDhtColocatedLockFuture : fut;
+
+                        // Terminate this future if parent(collocated) future is terminated by rollback.
+                        fut.listen(new IgniteInClosure<IgniteInternalFuture<Boolean>>() {
+                            @Override public void apply(IgniteInternalFuture<Boolean> fut) {
+                                try {
+                                    fut.get();
+                                }
+                                catch (IgniteCheckedException e) {
+                                    onError(e);
+                                }
+                            }
+                        });
+                    }
+
+                    return;
+                }
+
+                if (tx.updateLockFuture(null, this))
+                    return;
+            }
+        }
     }
 
     /** {@inheritDoc} */
@@ -650,19 +684,23 @@ private boolean filter(GridCacheEntryEx cached) {
      * @param entry Entry whose lock ownership changed.
      */
     @Override public boolean onOwnerChanged(GridCacheEntryEx entry, GridCacheMvccCandidate owner) {
-        if (isDone() || (inTx() && tx.remainingTime() == -1))
+        if (isDone() || (inTx() && (tx.remainingTime() == -1 || tx.isRollbackOnly())))
             return false; // Check other futures.
 
         if (log.isDebugEnabled())
             log.debug("Received onOwnerChanged() callback [entry=" + entry + ", owner=" + owner + "]");
 
         if (owner != null && owner.version().equals(lockVer)) {
+            boolean isEmpty;
+
             synchronized (this) {
                 if (!pendingLocks.remove(entry.key()))
                     return false;
+
+                isEmpty = pendingLocks.isEmpty();
             }
 
-            if (checkLocks())
+            if (isEmpty)
                 map(entries());
 
             return true;
@@ -722,7 +760,7 @@ private synchronized boolean checkLocks() {
      * @param unlock {@code True} if locks should be released.
      * @return {@code True} if complete by this operation.
      */
-    private boolean onComplete(boolean success, boolean stopping, boolean unlock) {
+    private synchronized boolean onComplete(boolean success, boolean stopping, boolean unlock) {
         if (log.isDebugEnabled())
             log.debug("Received onComplete(..) callback [success=" + success + ", fut=" + this + ']');
 
@@ -735,6 +773,9 @@ private boolean onComplete(boolean success, boolean stopping, boolean unlock) {
             cctx.tm().txContext(tx);
 
             set = cctx.tm().setTxTopologyHint(tx.topologyVersionSnapshot());
+
+            if (success)
+                tx.clearLockFuture(this);
         }
 
         try {
@@ -774,13 +815,28 @@ public void map() {
 
         readyLocks();
 
-        if (timeout > 0) {
+        if (timeout > 0 && !isDone()) { // Prevent memory leak if future is completed by call to readyLocks.
             timeoutObj = new LockTimeoutObject();
 
             cctx.time().addTimeoutObject(timeoutObj);
         }
     }
 
+    /**
+     *
+     * @return {@code True} if future is done.
+     */
+    private boolean checkDone() {
+        if (isDone()) {
+            if (log.isDebugEnabled())
+                log.debug("Mapping won't proceed because future is done: " + this);
+
+            return true;
+        }
+
+        return false;
+    }
+
     /**
      * @param entries Entries.
      */
@@ -835,139 +891,146 @@ private void map(Iterable<GridDhtCacheEntry> entries) {
                 }
             }
 
-            if (isDone()) {
-                if (log.isDebugEnabled())
-                    log.debug("Mapping won't proceed because future is done: " + this);
-
+            if (checkDone())
                 return;
-            }
 
             if (log.isDebugEnabled())
                 log.debug("Mapped DHT lock future [dhtMap=" + F.nodeIds(dhtMap.keySet()) + ", dhtLockFut=" + this + ']');
 
             long timeout = inTx() ? tx.remainingTime() : this.timeout;
 
-            // Create mini futures.
-            for (Map.Entry<ClusterNode, List<GridDhtCacheEntry>> mapped : dhtMap.entrySet()) {
-                ClusterNode n = mapped.getKey();
+            synchronized (this) { // Prevents entry removal on concurrent rollback.
+                if (checkDone())
+                    return;
+
+                // Create mini futures.
+                for (Map.Entry<ClusterNode, List<GridDhtCacheEntry>> mapped : dhtMap.entrySet()) {
+                    ClusterNode n = mapped.getKey();
 
-                List<GridDhtCacheEntry> dhtMapping = mapped.getValue();
+                    List<GridDhtCacheEntry> dhtMapping = mapped.getValue();
 
-                int cnt = F.size(dhtMapping);
+                    int cnt = F.size(dhtMapping);
 
-                if (cnt > 0) {
-                    assert !n.id().equals(cctx.localNodeId());
+                    if (cnt > 0) {
+                        assert !n.id().equals(cctx.localNodeId());
 
-                    if (inTx() && tx.remainingTime() == -1)
-                        return;
+                        if (inTx() && tx.remainingTime() == -1)
+                            return;
 
-                    MiniFuture fut = new MiniFuture(n, dhtMapping);
-
-                    GridDhtLockRequest req = new GridDhtLockRequest(
-                        cctx.cacheId(),
-                        nearNodeId,
-                        inTx() ? tx.nearXidVersion() : null,
-                        threadId,
-                        futId,
-                        fut.futureId(),
-                        lockVer,
-                        topVer,
-                        inTx(),
-                        read,
-                        isolation(),
-                        isInvalidate(),
-                        timeout,
-                        cnt,
-                        0,
-                        inTx() ? tx.size() : cnt,
-                        inTx() ? tx.subjectId() : null,
-                        inTx() ? tx.taskNameHash() : 0,
-                        read ? accessTtl : -1L,
-                        skipStore,
-                        cctx.store().configured(),
-                        keepBinary,
-                        cctx.deploymentEnabled());
-
-                    try {
-                        for (ListIterator<GridDhtCacheEntry> it = dhtMapping.listIterator(); it.hasNext();) {
-                            GridDhtCacheEntry e = it.next();
-
-                            boolean needVal = false;
+                        MiniFuture fut = new MiniFuture(n, dhtMapping);
+
+                        GridDhtLockRequest req = new GridDhtLockRequest(
+                            cctx.cacheId(),
+                            nearNodeId,
+                            inTx() ? tx.nearXidVersion() : null,
+                            threadId,
+                            futId,
+                            fut.futureId(),
+                            lockVer,
+                            topVer,
+                            inTx(),
+                            read,
+                            isolation(),
+                            isInvalidate(),
+                            timeout,
+                            cnt,
+                            0,
+                            inTx() ? tx.size() : cnt,
+                            inTx() ? tx.subjectId() : null,
+                            inTx() ? tx.taskNameHash() : 0,
+                            read ? accessTtl : -1L,
+                            skipStore,
+                            cctx.store().configured(),
+                            keepBinary,
+                            cctx.deploymentEnabled());
 
-                            try {
-                                // Must unswap entry so that isNewLocked returns correct value.
-                                e.unswap(false);
+                        try {
+                            for (ListIterator<GridDhtCacheEntry> it = dhtMapping.listIterator(); it.hasNext(); ) {
+                                GridDhtCacheEntry e = it.next();
 
-                                needVal = e.isNewLocked();
+                                boolean needVal = false;
 
-                                if (needVal) {
-                                    List<ClusterNode> owners = cctx.topology().owners(e.partition(),
-                                        tx != null ? tx.topologyVersion() : cctx.affinity().affinityTopologyVersion());
+                                try {
+                                    // Must unswap entry so that isNewLocked returns correct value.
+                                    e.unswap(false);
 
-                                    // Do not preload if local node is partition owner.
-                                    if (owners.contains(cctx.localNode()))
-                                        needVal = false;
-                                }
-                            }
-                            catch (GridCacheEntryRemovedException ex) {
-                                assert false : "Entry cannot become obsolete when DHT local candidate is added " +
-                                    "[e=" + e + ", ex=" + ex + ']';
-                            }
+                                    needVal = e.isNewLocked();
+
+                                    if (needVal) {
+                                        List<ClusterNode> owners = cctx.topology().owners(e.partition(),
+                                            tx != null ? tx.topologyVersion() : cctx.affinity().affinityTopologyVersion());
+
+                                        // Do not preload if local node is partition owner.
+                                        if (owners.contains(cctx.localNode()))
+                                            needVal = false;
+                                    }
 
-                            // Skip entry if it is not new and is not present in updated mapping.
-                            if (tx != null && !needVal)
-                                continue;
+                                    // Skip entry if it is not new and is not present in updated mapping.
+                                    if (tx != null && !needVal)
+                                        continue;
 
-                            boolean invalidateRdr = e.readerId(n.id()) != null;
+                                    boolean invalidateRdr = e.readerId(n.id()) != null;
 
-                            req.addDhtKey(e.key(), invalidateRdr, cctx);
+                                    req.addDhtKey(e.key(), invalidateRdr, cctx);
 
-                            if (needVal) {
-                                // Mark last added key as needed to be preloaded.
-                                req.markLastKeyForPreload();
+                                    if (needVal) {
+                                        // Mark last added key as needed to be preloaded.
+                                        req.markLastKeyForPreload();
 
-                                if (tx != null) {
-                                    IgniteTxEntry txEntry = tx.entry(e.txKey());
+                                        if (tx != null) {
+                                            IgniteTxEntry txEntry = tx.entry(e.txKey());
 
-                                    // NOOP entries will be sent to backups on prepare step.
-                                    if (txEntry.op() == GridCacheOperation.READ)
-                                        txEntry.op(GridCacheOperation.NOOP);
+                                            // NOOP entries will be sent to backups on prepare step.
+                                            if (txEntry.op() == GridCacheOperation.READ)
+                                                txEntry.op(GridCacheOperation.NOOP);
+                                        }
+                                    }
+
+                                    GridCacheMvccCandidate added = e.candidate(lockVer);
+
+                                    assert added != null;
+                                    assert added.dhtLocal();
+
+                                    if (added.ownerVersion() != null)
+                                        req.owned(e.key(), added.ownerVersion());
+                                }
+                                catch (GridCacheEntryRemovedException ex) {
+                                    assert false : "Entry cannot become obsolete when DHT local candidate is added " +
+                                        "[e=" + e + ", ex=" + ex + ']';
                                 }
                             }
 
-                            it.set(addOwned(req, e));
-                        }
-
-                        if (!F.isEmpty(req.keys())) {
-                            if (tx != null)
-                                tx.addLockTransactionNode(n);
+                            if (!F.isEmpty(req.keys())) {
+                                if (tx != null)
+                                    tx.addLockTransactionNode(n);
 
-                            add(fut); // Append new future.
+                                add(fut); // Append new future.
 
-                            cctx.io().send(n, req, cctx.ioPolicy());
+                                cctx.io().send(n, req, cctx.ioPolicy());
 
-                            if (msgLog.isDebugEnabled()) {
-                                msgLog.debug("DHT lock fut, sent request [txId=" + nearLockVer +
-                                    ", dhtTxId=" + lockVer +
-                                    ", inTx=" + inTx() +
-                                    ", nodeId=" + n.id() + ']');
+                                if (msgLog.isDebugEnabled()) {
+                                    msgLog.debug("DHT lock fut, sent request [txId=" + nearLockVer +
+                                        ", dhtTxId=" + lockVer +
+                                        ", inTx=" + inTx() +
+                                        ", nodeId=" + n.id() + ']');
+                                }
                             }
                         }
-                    }
-                    catch (IgniteCheckedException e) {
-                        // Fail the whole thing.
-                        if (e instanceof ClusterTopologyCheckedException)
-                            fut.onResult();
-                        else {
-                            if (msgLog.isDebugEnabled()) {
-                                msgLog.debug("DHT lock fut, failed to send request [txId=" + nearLockVer +
-                                    ", dhtTxId=" + lockVer +
-                                    ", inTx=" + inTx() +
-                                    ", node=" + n.id() +
-                                    ", err=" + e + ']');
-                            }
+                        catch (IgniteCheckedException e) {
+                            // Fail the whole thing.
+                            if (e instanceof ClusterTopologyCheckedException)
+                                fut.onResult();
+                            else {
+                                if (msgLog.isDebugEnabled()) {
+                                    msgLog.debug("DHT lock fut, failed to send request [txId=" + nearLockVer +
+                                        ", dhtTxId=" + lockVer +
+                                        ", inTx=" + inTx() +
+                                        ", node=" + n.id() +
+                                        ", err=" + e + ']');
+                                }
 
-                            fut.onResult(e);
+                                fut.onResult(e);
+                            }
                         }
                     }
                 }
@@ -978,35 +1041,6 @@ private void map(Iterable<GridDhtCacheEntry> entries) {
         }
     }
 
-    /**
-     * @param req Request.
-     * @param e Entry.
-     * @return Entry.
-     */
-    private GridDhtCacheEntry addOwned(GridDhtLockRequest req, GridDhtCacheEntry e) {
-        while (true) {
-            try {
-                GridCacheMvccCandidate added = e.candidate(lockVer);
-
-                assert added != null;
-                assert added.dhtLocal();
-
-                if (added.ownerVersion() != null)
-                    req.owned(e.key(), added.ownerVersion());
-
-                break;
-            }
-            catch (GridCacheEntryRemovedException ignore) {
-                if (log.isDebugEnabled())
-                    log.debug("Got removed entry when creating DHT lock request (will retry): " + e);
-
-                e = cctx.dht().entryExx(e.key(), topVer);
-            }
-        }
-
-        return e;
-    }
-
     /** {@inheritDoc} */
     @Override public int hashCode() {
         return futId.hashCode();
@@ -1260,38 +1294,43 @@ void onResult(GridDhtLockResponse res) {
                 if (cache0.isNear())
                     cache0 = ((GridNearCacheAdapter)cache0).dht();
 
-                for (GridCacheEntryInfo info : res.preloadEntries()) {
-                    try {
-                        GridCacheEntryEx entry = cache0.entryEx(info.key(), topVer);
-
-                        cctx.shared().database().checkpointReadLock();
+                synchronized (GridDhtLockFuture.this) { // Prevents entry re-creation on concurrent rollback.
+                    if (GridDhtLockFuture.this.checkDone())
+                        return;
 
+                    for (GridCacheEntryInfo info : res.preloadEntries()) {
                         try {
-                            if (entry.initialValue(info.value(),
-                                info.version(),
-                                info.ttl(),
-                                info.expireTime(),
-                                true, topVer,
-                                replicate ? DR_PRELOAD : DR_NONE,
-                                false)) {
-                                if (rec && !entry.isInternal())
-                                    cctx.events().addEvent(entry.partition(), entry.key(), cctx.localNodeId(),
-                                        (IgniteUuid)null, null, EVT_CACHE_REBALANCE_OBJECT_LOADED, info.value(), true, null,
-                                        false, null, null, null, false);
+                            GridCacheEntryEx entry = cache0.entryEx(info.key(), topVer);
+
+                            cctx.shared().database().checkpointReadLock();
+
+                            try {
+                                if (entry.initialValue(info.value(),
+                                    info.version(),
+                                    info.ttl(),
+                                    info.expireTime(),
+                                    true, topVer,
+                                    replicate ? DR_PRELOAD : DR_NONE,
+                                    false)) {
+                                    if (rec && !entry.isInternal())
+                                        cctx.events().addEvent(entry.partition(), entry.key(), cctx.localNodeId(),
+                                            (IgniteUuid)null, null, EVT_CACHE_REBALANCE_OBJECT_LOADED, info.value(), true, null,
+                                            false, null, null, null, false);
+                                }
+                            }
+                            finally {
+                                cctx.shared().database().checkpointReadUnlock();
                             }
                         }
-                        finally {
-                            cctx.shared().database().checkpointReadUnlock();
-                        }
-                    }
-                    catch (IgniteCheckedException e) {
-                        onDone(e);
+                        catch (IgniteCheckedException e) {
+                            onDone(e);
 
-                        return;
-                    }
-                    catch (GridCacheEntryRemovedException e) {
-                        assert false : "Entry cannot become obsolete when DHT local candidate is added " +
-                            "[e=" + e + ", ex=" + e + ']';
+                            return;
+                        }
+                        catch (GridCacheEntryRemovedException e) {
+                            assert false : "Entry cannot become obsolete when DHT local candidate is added " +
+                                "[e=" + e + ", ex=" + e + ']';
+                        }
                     }
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index 27e5d6fa2f7db..e3e29b01e03cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheLockTimeoutException;
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheReturn;
+import org.apache.ignite.internal.processors.cache.GridCacheVersionedFuture;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
@@ -61,6 +62,7 @@
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
+import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -74,6 +76,7 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.thread.IgniteThread;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
@@ -671,11 +674,16 @@ private void processNearLockRequest0(ClusterNode nearNode, GridNearLockRequest r
             f = lockAllAsync(ctx, nearNode, req, null);
 
         // Register listener just so we print out errors.
-        // Exclude lock timeout exception since it's not a fatal exception.
+        // Exclude lock timeout and rollback exceptions since it's not a fatal exception.
         f.listen(CU.errorLogger(log, GridCacheLockTimeoutException.class,
-            GridDistributedLockCancelledException.class));
+            GridDistributedLockCancelledException.class, IgniteTxTimeoutCheckedException.class,
+            IgniteTxRollbackCheckedException.class));
     }
 
+    /**
+     * @param node Node.
+     * @param req Request.
+     */
     private boolean waitForExchangeFuture(final ClusterNode node, final GridNearLockRequest req) {
         assert req.firstClientRequest() : req;
 
@@ -824,6 +832,9 @@ public GridDhtFuture<Boolean> lockAllAsyncInternal(@Nullable Collection<KeyCache
             skipStore,
             keepBinary);
 
+        if (fut.isDone()) // Possible in case of cancellation or timeout or rollback.
+            return fut;
+
         for (KeyCacheObject key : keys) {
             try {
                 while (true) {
@@ -832,7 +843,7 @@ public GridDhtFuture<Boolean> lockAllAsyncInternal(@Nullable Collection<KeyCache
                     try {
                         fut.addEntry(entry);
 
-                        // Possible in case of cancellation or time out.
+                        // Possible in case of cancellation or time out or rollback.
                         if (fut.isDone())
                             return fut;
 
@@ -859,9 +870,11 @@ public GridDhtFuture<Boolean> lockAllAsyncInternal(@Nullable Collection<KeyCache
             }
         }
 
-        ctx.mvcc().addFuture(fut);
+        if (!fut.isDone()) {
+            ctx.mvcc().addFuture(fut);
 
-        fut.map();
+            fut.map();
+        }
 
         return fut;
     }
@@ -962,10 +975,10 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                 }
             }
 
-            boolean timedout = false;
+            boolean timedOut = false;
 
             for (KeyCacheObject key : keys) {
-                if (timedout)
+                if (timedOut)
                     break;
 
                 while (true) {
@@ -980,7 +993,7 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                             fut.addEntry(key == null ? null : entry);
 
                             if (fut.isDone()) {
-                                timedout = true;
+                                timedOut = true;
 
                                 break;
                             }
@@ -1079,7 +1092,7 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                             if (tx != null)
                                 tx.rollbackDhtLocal();
 
-                            return new GridDhtFinishedFuture<>(new IgniteCheckedException(msg));
+                            return new GridDhtFinishedFuture<>(new IgniteTxRollbackCheckedException(msg));
                         }
 
                         tx.topologyVersion(req.topologyVersion());
@@ -1117,7 +1130,8 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                             if (e != null)
                                 e = U.unwrap(e);
 
-                            assert !t.empty();
+                            // Transaction can be emptied by asynchronous rollback.
+                            assert e != null || !t.empty();
 
                             // Create response while holding locks.
                             final GridNearLockResponse resp = createLockReply(nearNode,
@@ -1296,13 +1310,11 @@ private GridNearLockResponse createLockReply(
                                         null,
                                         req.keepBinary());
 
-                                assert e.lockedBy(mappedVer) ||
-                                    (ctx.mvcc().isRemoved(e.context(), mappedVer) && req.timeout() > 0) :
+                                assert e.lockedBy(mappedVer) || ctx.mvcc().isRemoved(e.context(), mappedVer) :
                                     "Entry does not own lock for tx [locNodeId=" + ctx.localNodeId() +
                                         ", entry=" + e +
                                         ", mappedVer=" + mappedVer + ", ver=" + ver +
-                                        ", tx=" + tx + ", req=" + req +
-                                        ", err=" + err + ']';
+                                        ", tx=" + tx + ", req=" + req + ']';
 
                                 boolean filterPassed = false;
 
@@ -1384,12 +1396,14 @@ private void sendLockReply(
         Throwable err = res.error();
 
         // Log error before sending reply.
-        if (err != null && !(err instanceof GridCacheLockTimeoutException) && !ctx.kernalContext().isStopping())
+        if (err != null && !(err instanceof GridCacheLockTimeoutException) &&
+            !(err instanceof IgniteTxRollbackCheckedException) && !ctx.kernalContext().isStopping())
             U.error(log, "Failed to acquire lock for request: " + req, err);
 
         try {
-            // Don't send reply message to this node or if lock was cancelled.
-            if (!nearNode.id().equals(ctx.nodeId()) && !X.hasCause(err, GridDistributedLockCancelledException.class)) {
+            // Don't send reply message to this node or if lock was cancelled or tx was rolled back asynchronously.
+            if (!nearNode.id().equals(ctx.nodeId()) && !X.hasCause(err, GridDistributedLockCancelledException.class) &&
+                !X.hasCause(err, IgniteTxRollbackCheckedException.class)) {
                 ctx.io().send(nearNode, res, ctx.ioPolicy());
 
                 if (txLockMsgLog.isDebugEnabled()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
index 0609f04017643..2e19df2911509 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
@@ -48,6 +48,7 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
@@ -395,13 +396,13 @@ public final IgniteInternalFuture<GridNearTxPrepareResponse> prepareAsync(GridNe
             else
                 fut.prepare(req);
         }
-        catch (IgniteTxTimeoutCheckedException | IgniteTxOptimisticCheckedException e) {
+        catch (IgniteTxTimeoutCheckedException | IgniteTxRollbackCheckedException | IgniteTxOptimisticCheckedException e) {
             fut.onError(e);
         }
         catch (IgniteCheckedException e) {
             setRollbackOnly();
 
-            fut.onError(new IgniteTxRollbackCheckedException("Failed to prepare transaction: " + this, e));
+            fut.onError(new IgniteTxRollbackCheckedException("Failed to prepare transaction: " + CU.txString(this), e));
         }
 
         return chainOnePhasePrepare(fut);
@@ -419,6 +420,30 @@ private void finishTx(boolean commit, @Nullable IgniteInternalFuture prepFut, Gr
 
         IgniteCheckedException err = null;
 
+        if (!commit) {
+            final IgniteInternalFuture<?> lockFut = tryRollbackAsync();
+
+            if (lockFut != null) {
+                if (lockFut instanceof GridDhtLockFuture)
+                    ((GridDhtLockFuture)lockFut).onError(rollbackException());
+                else {
+                    /**
+                     * Prevents race with {@link GridDhtTransactionalCacheAdapter#lockAllAsync
+                     * (GridCacheContext, ClusterNode, GridNearLockRequest, CacheEntryPredicate[])}
+                     */
+                    final IgniteInternalFuture finalPrepFut = prepFut;
+
+                    lockFut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
+                        @Override public void apply(IgniteInternalFuture<?> ignored) {
+                            finishTx(false, finalPrepFut, fut);
+                        }
+                    });
+
+                    return;
+                }
+            }
+        }
+
         if (!commit && prepFut != null) {
             try {
                 prepFut.get();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
index bed92430426af..604fe0655801e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
@@ -25,6 +25,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -46,7 +47,9 @@
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.CX1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -59,7 +62,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.NOOP;
-import static org.apache.ignite.internal.processors.cache.GridCacheOperation.READ;
 import static org.apache.ignite.transactions.TransactionState.COMMITTED;
 import static org.apache.ignite.transactions.TransactionState.COMMITTING;
 import static org.apache.ignite.transactions.TransactionState.PREPARED;
@@ -75,6 +77,13 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
     /** */
     private static final long serialVersionUID = 0L;
 
+    /** Asynchronous rollback marker for lock futures. */
+    protected static final IgniteInternalFuture<Boolean> ROLLBACK_FUT = new GridFutureAdapter<>();
+
+    /** Lock future updater. */
+    private static final AtomicReferenceFieldUpdater<GridDhtTxLocalAdapter, IgniteInternalFuture> LOCK_FUT_UPD =
+        AtomicReferenceFieldUpdater.newUpdater(GridDhtTxLocalAdapter.class, IgniteInternalFuture.class, "lockFut");
+
     /** Near mappings. */
     protected Map<UUID, GridDistributedTxMapping> nearMap = new ConcurrentHashMap<>();
 
@@ -96,6 +105,11 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
     /** Nodes where transactions were started on lock step. */
     private Set<ClusterNode> lockTxNodes;
 
+    /** Enlist or lock future what is currently in progress. */
+    @SuppressWarnings("UnusedDeclaration")
+    @GridToStringExclude
+    protected volatile IgniteInternalFuture<Boolean> lockFut;
+
     /**
      * Empty constructor required for {@link Externalizable}.
      */
@@ -561,69 +575,79 @@ IgniteInternalFuture<GridCacheReturn> lockAllAsync(
         onePhaseCommit(onePhaseCommit);
 
         try {
-            Set<KeyCacheObject> skipped = null;
+            GridFutureAdapter<GridCacheReturn> enlistFut = new GridFutureAdapter<>();
 
-            AffinityTopologyVersion topVer = topologyVersion();
+            if (!updateLockFuture(null, enlistFut))
+                return finishFuture(enlistFut, timedOut() ? timeoutException() : rollbackException(), false);
 
-            GridDhtCacheAdapter dhtCache = cacheCtx.isNear() ? cacheCtx.near().dht() : cacheCtx.dht();
+            Set<KeyCacheObject> skipped = null;
 
-            // Enlist locks into transaction.
-            for (int i = 0; i < entries.size(); i++) {
-                GridCacheEntryEx entry = entries.get(i);
+            try {
+                AffinityTopologyVersion topVer = topologyVersion();
 
-                KeyCacheObject key = entry.key();
+                GridDhtCacheAdapter dhtCache = cacheCtx.isNear() ? cacheCtx.near().dht() : cacheCtx.dht();
 
-                IgniteTxEntry txEntry = entry(entry.txKey());
+                // Enlist locks into transaction.
+                for (int i = 0; i < entries.size(); i++) {
+                    GridCacheEntryEx entry = entries.get(i);
 
-                // First time access.
-                if (txEntry == null) {
-                    GridDhtCacheEntry cached;
+                    KeyCacheObject key = entry.key();
 
-                    while (true) {
-                        try {
-                            cached = dhtCache.entryExx(key, topVer);
+                    IgniteTxEntry txEntry = entry(entry.txKey());
 
-                            cached.unswap(read);
+                    // First time access.
+                    if (txEntry == null) {
+                        GridDhtCacheEntry cached;
 
-                            break;
-                        }
-                        catch (GridCacheEntryRemovedException ignore) {
-                            if (log.isDebugEnabled())
-                                log.debug("Get removed entry: " + key);
-                        }
-                    }
-
-                    addActiveCache(dhtCache.context(), false);
+                        while (true) {
+                            try {
+                                cached = dhtCache.entryExx(key, topVer);
 
-                    txEntry = addEntry(NOOP,
-                        null,
-                        null,
-                        null,
-                        cached,
-                        null,
-                        CU.empty0(),
-                        false,
-                        -1L,
-                        -1L,
-                        null,
-                        skipStore,
-                        keepBinary,
-                        nearCache);
-
-                    if (read)
-                        txEntry.ttl(accessTtl);
+                                cached.unswap(read);
 
-                    txEntry.cached(cached);
+                                break;
+                            }
+                            catch (GridCacheEntryRemovedException ignore) {
+                                if (log.isDebugEnabled())
+                                    log.debug("Get removed entry: " + key);
+                            }
+                        }
 
-                    addReader(msgId, cached, txEntry, topVer);
-                }
-                else {
-                    if (skipped == null)
-                        skipped = new GridLeanSet<>();
+                        addActiveCache(dhtCache.context(), false);
+
+                        txEntry = addEntry(NOOP,
+                            null,
+                            null,
+                            null,
+                            cached,
+                            null,
+                            CU.empty0(),
+                            false,
+                            -1L,
+                            -1L,
+                            null,
+                            skipStore,
+                            keepBinary,
+                            nearCache);
+
+                        if (read)
+                            txEntry.ttl(accessTtl);
+
+                        txEntry.cached(cached);
+
+                        addReader(msgId, cached, txEntry, topVer);
+                    }
+                    else {
+                        if (skipped == null)
+                            skipped = new GridLeanSet<>();
 
-                    skipped.add(key);
+                        skipped.add(key);
+                    }
                 }
             }
+            finally {
+                finishFuture(enlistFut, null, true);
+            }
 
             assert pessimistic();
 
@@ -690,6 +714,9 @@ private IgniteInternalFuture<GridCacheReturn> obtainLockAsync(
         if (timeout == -1)
             return new GridFinishedFuture<>(timeoutException());
 
+        if (isRollbackOnly())
+            return new GridFinishedFuture<>(rollbackException());
+
         IgniteInternalFuture<Boolean> fut = dhtCache.lockAllAsyncInternal(passedKeys,
             timeout,
             this,
@@ -825,6 +852,68 @@ public final boolean commitOnPrepare() {
         return onePhaseCommit() && !near() && !nearOnOriginatingNode;
     }
 
+    /**
+     * Atomically updates lock future.
+     *
+     * @param oldFut Old future.
+     * @param newFut New future.
+     *
+     * @return {@code true} If future was changed.
+     */
+    public boolean updateLockFuture(IgniteInternalFuture<?> oldFut, IgniteInternalFuture<?> newFut) {
+        return LOCK_FUT_UPD.compareAndSet(this, oldFut, newFut);
+    }
+
+    /**
+     * Clears lock future.
+     *
+     * @param cond Clear lock condition.
+     */
+    public void clearLockFuture(@Nullable IgniteInternalFuture cond) {
+        IgniteInternalFuture f = lockFut;
+
+        if (cond != null && f != cond)
+            return;
+
+        lockFut = null;
+    }
+
+    /**
+     *
+     * @param f Future to finish.
+     * @param err Error.
+     * @param clearLockFut {@code True} if need to clear lock future.
+     *
+     * @return Finished future.
+     */
+    public <T> GridFutureAdapter<T> finishFuture(GridFutureAdapter<T> f, Throwable err, boolean clearLockFut) {
+        if (clearLockFut)
+            clearLockFuture(null);
+
+        f.onDone(err);
+
+        return f;
+    }
+
+    /**
+     * Prepare async rollback.
+     *
+     * @return Current lock future or null if it's safe to roll back.
+     */
+    public @Nullable IgniteInternalFuture<?> tryRollbackAsync() {
+        IgniteInternalFuture<Boolean> fut;
+
+        while(true) {
+            fut = lockFut;
+
+            if (fut != null)
+                return fut == ROLLBACK_FUT ? null : fut;
+
+            if (updateLockFuture(null, ROLLBACK_FUT))
+                return null;
+        }
+    }
+
     /**
      * @param prepFut Prepare future.
      * @return If transaction if finished on prepare step returns future which is completed after transaction finish.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index bde15c254fc9a..622774dd58f6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -1040,7 +1040,7 @@ public void prepare(GridNearTxPrepareRequest req) {
 
         readyLocks();
 
-        if (timeoutObj != null) {
+        if (timeoutObj != null && !isDone()) {
             // Start timeout tracking after 'readyLocks' to avoid race with timeout processing.
             cctx.time().addTimeoutObject(timeoutObj);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
index d3348500517e4..12819bf67215a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
@@ -103,6 +103,9 @@ public class GridDhtTxPrepareRequest extends GridDistributedTxPrepareRequest {
     @GridDirectTransient
     private List<IgniteTxKey> nearWritesCacheMissed;
 
+    /** {@code True} if remote tx should skip adding itself to completed versions map on finish. */
+    private boolean skipCompletedVers;
+
     /**
      * Empty constructor required for {@link Externalizable}.
      */
@@ -169,6 +172,8 @@ public GridDhtTxPrepareRequest(
         invalidateNearEntries = new BitSet(dhtWrites == null ? 0 : dhtWrites.size());
 
         nearNodeId = tx.nearNodeId();
+
+        skipCompletedVers = tx.xidVersion() == tx.nearXidVersion();
     }
 
     /**
@@ -292,6 +297,13 @@ public Map<IgniteTxKey, GridCacheVersion> owned() {
         return owned;
     }
 
+    /**
+     * @return {@code True} if remote tx should skip adding itself to completed versions map on finish.
+     */
+    public boolean skipCompletedVersion() {
+        return skipCompletedVers;
+    }
+
     /**
      * {@inheritDoc}
      *
@@ -443,18 +455,24 @@ public Map<IgniteTxKey, GridCacheVersion> owned() {
                 writer.incrementState();
 
             case 29:
-                if (!writer.writeUuid("subjId", subjId))
+                if (!writer.writeBoolean("skipCompletedVers", skipCompletedVers))
                     return false;
 
                 writer.incrementState();
 
             case 30:
-                if (!writer.writeInt("taskNameHash", taskNameHash))
+                if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
             case 31:
+                if (!writer.writeInt("taskNameHash", taskNameHash))
+                    return false;
+
+                writer.incrementState();
+
+            case 32:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -549,7 +567,7 @@ public Map<IgniteTxKey, GridCacheVersion> owned() {
                 reader.incrementState();
 
             case 29:
-                subjId = reader.readUuid("subjId");
+                skipCompletedVers = reader.readBoolean("skipCompletedVers");
 
                 if (!reader.isLastRead())
                     return false;
@@ -557,7 +575,7 @@ public Map<IgniteTxKey, GridCacheVersion> owned() {
                 reader.incrementState();
 
             case 30:
-                taskNameHash = reader.readInt("taskNameHash");
+                subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
                     return false;
@@ -565,6 +583,14 @@ public Map<IgniteTxKey, GridCacheVersion> owned() {
                 reader.incrementState();
 
             case 31:
+                taskNameHash = reader.readInt("taskNameHash");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 32:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -584,6 +610,6 @@ public Map<IgniteTxKey, GridCacheVersion> owned() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 32;
+        return 33;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 607f363395220..14d386648ef1c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -194,7 +194,7 @@ public GridDistributedCacheEntry entryExx(
         if (tx != null && !tx.implicit() && !skipTx) {
             return asyncOp(tx, new AsyncOp<V>() {
                 @Override public IgniteInternalFuture<V> op(GridNearTxLocal tx, AffinityTopologyVersion readyTopVer) {
-                    IgniteInternalFuture<Map<Object, Object>>  fut = tx.getAllAsync(ctx,
+                    IgniteInternalFuture<Map<Object, Object>> fut = tx.getAllAsync(ctx,
                         readyTopVer,
                         Collections.singleton(ctx.toCacheKeyObject(key)),
                         deserializeBinary,
@@ -425,7 +425,7 @@ public final IgniteInternalFuture<Map<K, V>> loadAsync(
         if (expiryPlc == null)
             expiryPlc = expiryPolicy(null);
 
-        // Optimisation: try to resolve value locally and escape 'get future' creation.
+        // Optimization: try to resolve value locally and escape 'get future' creation.
         if (!forcePrimary && ctx.affinityNode()) {
             try {
                 Map<K, V> locVals = null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index c10f2ca7777f3..da0858f61cc5d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -23,6 +23,7 @@
 import java.util.Deque;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
@@ -56,7 +57,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
-import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.transactions.TxDeadlock;
@@ -237,6 +237,12 @@ public GridDhtColocatedLockFuture(
         }
 
         valMap = new ConcurrentHashMap<>();
+
+        if (tx != null && !tx.updateLockFuture(null, this)) {
+            onError(tx.timedOut() ? tx.timeoutException() : tx.rollbackException());
+
+            onComplete(false, false);
+        }
     }
 
     /** {@inheritDoc} */
@@ -405,7 +411,7 @@ else if (log.isDebugEnabled())
      * @param success Success flag.
      */
     public void complete(boolean success) {
-        onComplete(success, true, true);
+        onComplete(success, true);
     }
 
     /**
@@ -450,8 +456,9 @@ void onResult(UUID nodeId, GridNearLockResponse res) {
                 return;
             }
 
+            //  This warning can be triggered by deadlock detection code which clears pending futures.
             U.warn(msgLog, "Collocated lock fut, failed to find mini future [txId=" + lockVer +
-                ", inTx=" + inTx() +
+                ", tx=" + (inTx() ? CU.txString(tx) : "N/A") +
                 ", node=" + nodeId +
                 ", res=" + res +
                 ", fut=" + this + ']');
@@ -536,12 +543,14 @@ private synchronized void onError(Throwable t) {
             err = t;
     }
 
-    /** {@inheritDoc} */
+    /**
+     * Cancellation has special meaning for lock futures. It's called then lock must be released on rollback.
+     */
     @Override public boolean cancel() {
-        if (onCancelled())
-            onComplete(false, true, true);
+        if (inTx())
+            onError(tx.rollbackException());
 
-        return isCancelled();
+        return onComplete(false, true);
     }
 
     /** {@inheritDoc} */
@@ -562,7 +571,7 @@ private synchronized void onError(Throwable t) {
         if (err != null)
             success = false;
 
-        return onComplete(success, true, true);
+        return onComplete(success, true);
     }
 
     /**
@@ -570,10 +579,9 @@ private synchronized void onError(Throwable t) {
      *
      * @param success {@code True} if lock was acquired.
      * @param distribute {@code True} if need to distribute lock removal in case of failure.
-     * @param restoreTimeout {@code True} if need restore tx timeout callback.
      * @return {@code True} if complete by this operation.
      */
-    private boolean onComplete(boolean success, boolean distribute, boolean restoreTimeout) {
+    private boolean onComplete(boolean success, boolean distribute) {
         if (log.isDebugEnabled()) {
             log.debug("Received onComplete(..) callback [success=" + success + ", distribute=" + distribute +
                 ", fut=" + this + ']');
@@ -588,12 +596,8 @@ private boolean onComplete(boolean success, boolean distribute, boolean restoreT
         if (tx != null) {
             cctx.tm().txContext(tx);
 
-            if (restoreTimeout && tx.trackTimeout()) {
-                // Need restore timeout before onDone is called and next tx operation can proceed.
-                boolean add = tx.addTimeoutHandler();
-
-                assert add;
-            }
+            if (success)
+                tx.clearLockFuture(this);
         }
 
         if (super.onDone(success, err)) {
@@ -694,23 +698,8 @@ private boolean isMini(IgniteInternalFuture<?> f) {
      * part. Note that if primary node leaves grid, the future will fail and transaction will be rolled back.
      */
     void map() {
-        if (tx != null && tx.trackTimeout()) {
-            if (!tx.removeTimeoutHandler()) {
-                tx.finishFuture().listen(new IgniteInClosure<IgniteInternalFuture<IgniteInternalTx>>() {
-                    @Override public void apply(IgniteInternalFuture<IgniteInternalTx> fut) {
-                        IgniteTxTimeoutCheckedException err = new IgniteTxTimeoutCheckedException("Failed to " +
-                            "acquire lock, transaction was rolled back on timeout [timeout=" + tx.timeout() +
-                            ", tx=" + tx + ']');
-
-                        onError(err);
-
-                        onComplete(false, false, false);
-                    }
-                });
-
-                return;
-            }
-        }
+        if (isDone()) // Possible due to async rollback.
+            return;
 
         if (timeout > 0) {
             timeoutObj = new LockTimeoutObject();
@@ -973,7 +962,7 @@ private synchronized void map0(
                             if (log.isDebugEnabled())
                                 log.debug("Entry being locked did not pass filter (will not lock): " + entry);
 
-                            onComplete(false, false, true);
+                            onComplete(false, false);
 
                             return;
                         }
@@ -1350,7 +1339,7 @@ private boolean addLocalKey(
             if (log.isDebugEnabled())
                 log.debug("Entry being locked did not pass filter (will not lock): " + entry);
 
-            onComplete(false, false, true);
+            onComplete(false, false);
 
             return false;
         }
@@ -1430,44 +1419,48 @@ private class LockTimeoutObject extends GridTimeoutObjectAdapter {
             if (log.isDebugEnabled())
                 log.debug("Timed out waiting for lock response: " + this);
 
-            if (inTx() && cctx.tm().deadlockDetectionEnabled()) {
-                synchronized (GridDhtColocatedLockFuture.this) {
-                    requestedKeys = requestedKeys0();
+            if (inTx()) {
+                if (cctx.tm().deadlockDetectionEnabled()) {
+                    synchronized (GridDhtColocatedLockFuture.this) {
+                        requestedKeys = requestedKeys0();
 
-                    clear(); // Stop response processing.
-                }
+                        clear(); // Stop response processing.
+                    }
 
-                Set<IgniteTxKey> keys = new HashSet<>();
+                    Set<IgniteTxKey> keys = new HashSet<>();
 
-                for (IgniteTxEntry txEntry : tx.allEntries()) {
-                    if (!txEntry.locked())
-                        keys.add(txEntry.txKey());
-                }
+                    for (IgniteTxEntry txEntry : tx.allEntries()) {
+                        if (!txEntry.locked())
+                            keys.add(txEntry.txKey());
+                    }
 
-                IgniteInternalFuture<TxDeadlock> fut = cctx.tm().detectDeadlock(tx, keys);
+                    IgniteInternalFuture<TxDeadlock> fut = cctx.tm().detectDeadlock(tx, keys);
 
-                fut.listen(new IgniteInClosure<IgniteInternalFuture<TxDeadlock>>() {
-                    @Override public void apply(IgniteInternalFuture<TxDeadlock> fut) {
-                        try {
-                            TxDeadlock deadlock = fut.get();
+                    fut.listen(new IgniteInClosure<IgniteInternalFuture<TxDeadlock>>() {
+                        @Override public void apply(IgniteInternalFuture<TxDeadlock> fut) {
+                            try {
+                                TxDeadlock deadlock = fut.get();
 
-                            if (deadlock != null)
-                                err = new IgniteTxTimeoutCheckedException("Failed to acquire lock within provided timeout for " +
-                                        "transaction [timeout=" + tx.timeout() + ", tx=" + tx + ']',
-                                        new TransactionDeadlockException(deadlock.toString(cctx.shared())));
-                        }
-                        catch (IgniteCheckedException e) {
-                            err = e;
+                                err = new IgniteTxTimeoutCheckedException("Failed to acquire lock within provided " +
+                                    "timeout for transaction [timeout=" + tx.timeout() + ", tx=" + CU.txString(tx) + ']',
+                                    deadlock != null ? new TransactionDeadlockException(deadlock.toString(cctx.shared())) :
+                                        null);
+                            }
+                            catch (IgniteCheckedException e) {
+                                err = e;
 
-                            U.warn(log, "Failed to detect deadlock.", e);
-                        }
+                                U.warn(log, "Failed to detect deadlock.", e);
+                            }
 
-                        onComplete(false, true, true);
-                    }
-                });
+                            onComplete(false, true);
+                        }
+                    });
+                }
+                else
+                    err = tx.timeoutException();
             }
             else
-                onComplete(false, true, true);
+                onComplete(false, true);
         }
 
         /** {@inheritDoc} */
@@ -1716,4 +1709,4 @@ private void remap() {
             return S.toString(MiniFuture.class, this, "node", node.id(), "super", super.toString());
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index a21d98e67d320..c8e0034aa4536 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
@@ -1243,11 +1244,17 @@ private void waitPartitionRelease(int phase) throws IgniteCheckedException {
 
         long nextDumpTime = 0;
 
-        long futTimeout = 2 * cctx.gridConfig().getNetworkTimeout();
+        IgniteConfiguration cfg = cctx.gridConfig();
+
+        boolean rollbackEnabled = cfg.getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange() > 0;
+
+        long waitTimeout = 2 * cfg.getNetworkTimeout();
 
         while (true) {
             try {
-                partReleaseFut.get(futTimeout, TimeUnit.MILLISECONDS);
+                partReleaseFut.get(rollbackEnabled ?
+                    cfg.getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange() :
+                    waitTimeout, TimeUnit.MILLISECONDS);
 
                 break;
             }
@@ -1256,7 +1263,13 @@ private void waitPartitionRelease(int phase) throws IgniteCheckedException {
                 if (nextDumpTime <= U.currentTimeMillis()) {
                     dumpPendingObjects(partReleaseFut);
 
-                    nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, futTimeout);
+                    nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, waitTimeout);
+                }
+
+                if (rollbackEnabled) {
+                    rollbackEnabled = false;
+
+                    cctx.tm().rollbackOnTopologyChange(initialVersion());
                 }
             }
             catch (IgniteCheckedException e) {
@@ -1286,7 +1299,7 @@ private void waitPartitionRelease(int phase) throws IgniteCheckedException {
 
         while (true) {
             try {
-                locksFut.get(futTimeout, TimeUnit.MILLISECONDS);
+                locksFut.get(waitTimeout, TimeUnit.MILLISECONDS);
 
                 break;
             }
@@ -1309,7 +1322,7 @@ private void waitPartitionRelease(int phase) throws IgniteCheckedException {
                     for (Map.Entry<IgniteTxKey, Collection<GridCacheMvccCandidate>> e : locks.entrySet())
                         U.warn(log, "Awaited locked entry [key=" + e.getKey() + ", mvcc=" + e.getValue() + ']');
 
-                    nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, futTimeout);
+                    nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, waitTimeout);
 
                     if (getBoolean(IGNITE_THREAD_DUMP_ON_EXCHANGE_TIMEOUT, false))
                         U.dumpThreads(log);
@@ -1326,7 +1339,7 @@ private void waitPartitionRelease(int phase) throws IgniteCheckedException {
             try {
                 while (true) {
                     try {
-                        releaseLatch.await(futTimeout, TimeUnit.MILLISECONDS);
+                        releaseLatch.await(waitTimeout, TimeUnit.MILLISECONDS);
 
                         if (log.isInfoEnabled())
                             log.info("Finished waiting for partitions release latch: " + releaseLatch);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index 6ff5e6541f37a..4b46bda560319 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -52,7 +52,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTransactionalCacheAdapter;
-import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.transactions.TxDeadlock;
@@ -239,6 +238,12 @@ public GridNearLockFuture(
             log = U.logger(cctx.kernalContext(), logRef, GridNearLockFuture.class);
 
         valMap = new ConcurrentHashMap<>();
+
+        if (tx != null && !tx.updateLockFuture(null, this)) {
+            err = tx.timedOut() ? tx.timeoutException() : tx.rollbackException();
+
+            onComplete(false, false);
+        }
     }
 
     /** {@inheritDoc} */
@@ -436,7 +441,7 @@ private void onFailed(boolean dist) {
      * @param success Success flag.
      */
     public void complete(boolean success) {
-        onComplete(success, true, true);
+        onComplete(success, true);
     }
 
     /**
@@ -657,7 +662,7 @@ private boolean checkLocks() {
                     log.debug("Local lock acquired for entries [fut=" + this + ", entries=" + entries + "]");
             }
 
-            onComplete(true, true, true);
+            onComplete(true, true);
 
             return true;
         }
@@ -665,12 +670,14 @@ private boolean checkLocks() {
         return false;
     }
 
-    /** {@inheritDoc} */
+    /**
+     * Cancellation has special meaning for lock futures. It's called then lock must be released on rollback.
+     */
     @Override public boolean cancel() {
-        if (onCancelled())
-            onComplete(false, true, true);
+        if (inTx())
+            onError(tx.rollbackException());
 
-        return isCancelled();
+        return onComplete(false, true);
     }
 
     /** {@inheritDoc} */
@@ -692,7 +699,7 @@ private boolean checkLocks() {
         if (err != null)
             success = false;
 
-        return onComplete(success, true, true);
+        return onComplete(success, true);
     }
 
     /**
@@ -700,10 +707,9 @@ private boolean checkLocks() {
      *
      * @param success {@code True} if lock was acquired.
      * @param distribute {@code True} if need to distribute lock removal in case of failure.
-     * @param restoreTimeout {@code True} if need restore tx timeout callback.
      * @return {@code True} if complete by this operation.
      */
-    private boolean onComplete(boolean success, boolean distribute, boolean restoreTimeout) {
+    private boolean onComplete(boolean success, boolean distribute) {
         if (log.isDebugEnabled()) {
             log.debug("Received onComplete(..) callback [success=" + success + ", distribute=" + distribute +
                 ", fut=" + this + ']');
@@ -718,12 +724,8 @@ private boolean onComplete(boolean success, boolean distribute, boolean restoreT
         if (tx != null) {
             cctx.tm().txContext(tx);
 
-            if (restoreTimeout && tx.trackTimeout()) {
-                // Need restore timeout before onDone is called and next tx operation can proceed.
-                boolean add = tx.addTimeoutHandler();
-
-                assert add;
-            }
+            if (success)
+                tx.clearLockFuture(this);
         }
 
         if (super.onDone(success, err)) {
@@ -785,23 +787,8 @@ private boolean isMini(IgniteInternalFuture<?> f) {
      * part. Note that if primary node leaves grid, the future will fail and transaction will be rolled back.
      */
     void map() {
-        if (tx != null && tx.trackTimeout()) {
-            if (!tx.removeTimeoutHandler()) {
-                tx.finishFuture().listen(new IgniteInClosure<IgniteInternalFuture<IgniteInternalTx>>() {
-                    @Override public void apply(IgniteInternalFuture<IgniteInternalTx> fut) {
-                        IgniteTxTimeoutCheckedException err = new IgniteTxTimeoutCheckedException("Failed to " +
-                            "acquire lock, transaction was rolled back on timeout [timeout=" + tx.timeout() +
-                            ", tx=" + tx + ']');
-
-                        onError(err);
-
-                        onComplete(false, false, false);
-                    }
-                });
-
-                return;
-            }
-        }
+        if (isDone()) // Possible due to async rollback.
+            return;
 
         if (timeout > 0) {
             timeoutObj = new LockTimeoutObject();
@@ -1020,7 +1007,7 @@ private void map(Iterable<KeyCacheObject> keys, boolean remap, boolean topLocked
                                     if (log.isDebugEnabled())
                                         log.debug("Entry being locked did not pass filter (will not lock): " + entry);
 
-                                    onComplete(false, false, true);
+                                    onComplete(false, false);
 
                                     return;
                                 }
@@ -1476,44 +1463,48 @@ private class LockTimeoutObject extends GridTimeoutObjectAdapter {
 
             timedOut = true;
 
-            if (inTx() && cctx.tm().deadlockDetectionEnabled()) {
-                synchronized (GridNearLockFuture.this) {
-                    requestedKeys = requestedKeys0();
+            if (inTx()) {
+                if (cctx.tm().deadlockDetectionEnabled()) {
+                    synchronized (GridNearLockFuture.this) {
+                        requestedKeys = requestedKeys0();
 
-                    clear(); // Stop response processing.
-                }
+                        clear(); // Stop response processing.
+                    }
 
-                Set<IgniteTxKey> keys = new HashSet<>();
+                    Set<IgniteTxKey> keys = new HashSet<>();
 
-                for (IgniteTxEntry txEntry : tx.allEntries()) {
-                    if (!txEntry.locked())
-                        keys.add(txEntry.txKey());
-                }
+                    for (IgniteTxEntry txEntry : tx.allEntries()) {
+                        if (!txEntry.locked())
+                            keys.add(txEntry.txKey());
+                    }
 
-                IgniteInternalFuture<TxDeadlock> fut = cctx.tm().detectDeadlock(tx, keys);
+                    IgniteInternalFuture<TxDeadlock> fut = cctx.tm().detectDeadlock(tx, keys);
 
-                fut.listen(new IgniteInClosure<IgniteInternalFuture<TxDeadlock>>() {
-                    @Override public void apply(IgniteInternalFuture<TxDeadlock> fut) {
-                        try {
-                            TxDeadlock deadlock = fut.get();
+                    fut.listen(new IgniteInClosure<IgniteInternalFuture<TxDeadlock>>() {
+                        @Override public void apply(IgniteInternalFuture<TxDeadlock> fut) {
+                            try {
+                                TxDeadlock deadlock = fut.get();
 
-                            if (deadlock != null)
-                                err = new IgniteTxTimeoutCheckedException("Failed to acquire lock within provided timeout for " +
-                                        "transaction [timeout=" + tx.timeout() + ", tx=" + tx + ']',
-                                        new TransactionDeadlockException(deadlock.toString(cctx.shared())));
-                        }
-                        catch (IgniteCheckedException e) {
-                            err = e;
+                                err = new IgniteTxTimeoutCheckedException("Failed to acquire lock within provided " +
+                                    "timeout for transaction [timeout=" + tx.timeout() + ", tx=" + CU.txString(tx) + ']',
+                                    deadlock != null ? new TransactionDeadlockException(deadlock.toString(cctx.shared())) :
+                                        null);
+                            }
+                            catch (IgniteCheckedException e) {
+                                err = e;
 
-                            U.warn(log, "Failed to detect deadlock.", e);
-                        }
+                                U.warn(log, "Failed to detect deadlock.", e);
+                            }
 
-                        onComplete(false, true, true);
-                    }
-                });
+                            onComplete(false, true);
+                        }
+                    });
+                }
+                else
+                    err = tx.timeoutException();
             }
             else
-                onComplete(false, true, true);
+                onComplete(false, true);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index 3e2c84a740af5..974de6b9ef30c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -296,14 +296,12 @@ private boolean onComplete() {
         if (!txStateCheck) {
             if (tx.isRollbackOnly() || tx.setRollbackOnly()) {
                 if (tx.timedOut())
-                    onError(null, new IgniteTxTimeoutCheckedException("Transaction timed out and " +
-                        "was rolled back: " + this));
+                    onDone(null, tx.timeoutException());
                 else
-                    onError(null, new IgniteCheckedException("Invalid transaction state for prepare " +
-                        "[state=" + tx.state() + ", tx=" + this + ']'));
+                    onDone(null, tx.rollbackException());
             }
             else
-                onError(null, new IgniteTxRollbackCheckedException("Invalid transaction state for " +
+                onDone(null, new IgniteCheckedException("Invalid transaction state for " +
                     "prepare [state=" + tx.state() + ", tx=" + this + ']'));
 
             return;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 2afb09601ebac..247af8496ed10 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -59,6 +59,7 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiClosure;
@@ -329,15 +330,13 @@ private boolean onComplete() {
             if (!txStateCheck) {
                 if (tx.isRollbackOnly() || tx.setRollbackOnly()) {
                     if (tx.remainingTime() == -1)
-                        onError(new IgniteTxTimeoutCheckedException("Transaction timed out and " +
-                            "was rolled back: " + this), false);
+                        onDone(tx.timeoutException());
                     else
-                        onError(new IgniteCheckedException("Invalid transaction state for prepare " +
-                            "[state=" + tx.state() + ", tx=" + this + ']'), false);
+                        onDone(tx.rollbackException());
                 }
                 else
-                    onError(new IgniteTxRollbackCheckedException("Invalid transaction state for " +
-                        "prepare [state=" + tx.state() + ", tx=" + this + ']'), false);
+                    onDone(new IgniteCheckedException("Invalid transaction state for " +
+                        "prepare [state=" + tx.state() + ", tx=" + this + ']'));
 
                 return;
             }
@@ -768,7 +767,7 @@ private void onTimeout() {
                         U.warn(log, "Failed to detect deadlock.", e);
                     else {
                         e = new IgniteTxTimeoutCheckedException("Failed to acquire lock within provided timeout for " +
-                            "transaction [timeout=" + tx.timeout() + ", tx=" + tx + ']',
+                            "transaction [timeout=" + tx.timeout() + ", tx=" + CU.txString(tx) + ']',
                             deadlock != null ? new TransactionDeadlockException(deadlock.toString(cctx)) : null);
                     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index 54ae85c539cc7..da9bdac22739a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -157,13 +157,12 @@ private MiniFuture miniFuture(int miniId) {
         if (!tx.state(PREPARING)) {
             if (tx.isRollbackOnly() || tx.setRollbackOnly()) {
                 if (tx.remainingTime() == -1)
-                    onDone(new IgniteTxTimeoutCheckedException("Transaction timed out and was rolled back: " + tx));
+                    onDone(tx.timeoutException());
                 else
-                    onDone(new IgniteCheckedException("Invalid transaction state for prepare " +
-                        "[state=" + tx.state() + ", tx=" + this + ']'));
+                    onDone(tx.rollbackException());
             }
             else
-                onDone(new IgniteTxRollbackCheckedException("Invalid transaction state for prepare " +
+                onDone(new IgniteCheckedException("Invalid transaction state for prepare " +
                     "[state=" + tx.state() + ", tx=" + this + ']'));
 
             return;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
index 95e4deded214f..53d901aafb93a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
@@ -37,17 +37,13 @@ public class GridNearTxFastFinishFuture extends GridFutureAdapter<IgniteInternal
     /** */
     private final boolean commit;
 
-    /** */
-    private final boolean onTimeout;
-
     /**
      * @param tx Transaction.
      * @param commit Commit flag.
      */
-    GridNearTxFastFinishFuture(GridNearTxLocal tx, boolean commit, boolean onTimeout) {
+    GridNearTxFastFinishFuture(GridNearTxLocal tx, boolean commit) {
         this.tx = tx;
         this.commit = commit;
-        this.onTimeout = onTimeout;
     }
 
     /** {@inheritDoc} */
@@ -56,9 +52,9 @@ public class GridNearTxFastFinishFuture extends GridFutureAdapter<IgniteInternal
     }
 
     /**
-     *
+     * @param clearThreadMap {@code True} if need remove tx from thread map.
      */
-    public void finish() {
+    public void finish(boolean clearThreadMap) {
         try {
             if (commit) {
                 tx.state(PREPARING);
@@ -74,7 +70,7 @@ public void finish() {
                 tx.state(PREPARED);
                 tx.state(ROLLING_BACK);
 
-                tx.context().tm().fastFinishTx(tx, false, !onTimeout);
+                tx.context().tm().fastFinishTx(tx, false, clearThreadMap);
 
                 tx.state(ROLLED_BACK);
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index b6a885553f86b..da8c6c8a9a570 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheReturn;
 import org.apache.ignite.internal.processors.cache.GridCacheReturnCompletableWrapper;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.GridCacheVersionedFuture;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishResponse;
@@ -55,6 +56,7 @@
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.transactions.TransactionRollbackException;
 
@@ -120,14 +122,8 @@ public GridNearTxFinishFuture(GridCacheSharedContext<K, V> cctx, GridNearTxLocal
 
         futId = IgniteUuid.randomUuid();
 
-        CacheWriteSynchronizationMode syncMode;
-
         if (tx.explicitLock())
-            syncMode = FULL_SYNC;
-        else
-            syncMode = tx.syncMode();
-
-        tx.syncMode(syncMode);
+            tx.syncMode(FULL_SYNC);
 
         if (log == null) {
             msgLog = cctx.txFinishMessageLogger();
@@ -399,9 +395,10 @@ private boolean isMini(IgniteInternalFuture<?> fut) {
      *
      * @param commit Commit flag.
      * @param clearThreadMap If {@code true} removes {@link GridNearTxLocal} from thread map.
+     * @param onTimeout If {@code true} called from timeout handler.
      */
     @SuppressWarnings("ForLoopReplaceableByForEach")
-    public void finish(boolean commit, boolean clearThreadMap) {
+    public void finish(final boolean commit, final boolean clearThreadMap, final boolean onTimeout) {
         if (tx.onNeedCheckBackup()) {
             assert tx.onePhaseCommit();
 
@@ -414,6 +411,58 @@ public void finish(boolean commit, boolean clearThreadMap) {
             return;
         }
 
+        if (!commit && !clearThreadMap)
+            tryRollbackAsync(onTimeout); // Asynchronous rollback.
+        else
+            doFinish(commit, clearThreadMap);
+    }
+
+    /**
+     * Does async rollback when it's safe.
+     * If current future is not lock future (enlist future) waits until completion and tries again.
+     * Else terminates or waits for lock future depending on rollback mode.
+     *
+     * @param onTimeout If {@code true} called from timeout handler.
+     */
+    private void tryRollbackAsync(boolean onTimeout) {
+        IgniteInternalFuture<?> curFut = tx.tryRollbackAsync();
+
+        if (curFut == null) { // Safe to rollback.
+            doFinish(false, false);
+
+            return;
+        }
+
+        if (curFut instanceof GridCacheVersionedFuture && !onTimeout) {
+            try {
+                curFut.cancel(); // Force cancellation.
+            }
+            catch (IgniteCheckedException e) {
+                log.error("Failed to cancel lock for the transaction: " + CU.txString(tx), e);
+            }
+        }
+
+        curFut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
+            @Override public void apply(IgniteInternalFuture<?> fut) {
+                try {
+                    fut.get();
+
+                    tryRollbackAsync(onTimeout);
+                }
+                catch (IgniteCheckedException e) {
+                    doFinish(false, false);
+                }
+            }
+        });
+    }
+
+    /**
+     * Finishes a transaction.
+     *
+     * @param commit Commit.
+     * @param clearThreadMap Clear thread map.
+     */
+    private void doFinish(boolean commit, boolean clearThreadMap) {
         try {
             if (tx.localFinish(commit, clearThreadMap) || (!commit && tx.state() == UNKNOWN)) {
                 if ((tx.onePhaseCommit() && needFinishOnePhase(commit)) || (!tx.onePhaseCommit() && mappings != null)) {
@@ -423,17 +472,18 @@ public void finish(boolean commit, boolean clearThreadMap) {
                         if (mapping != null) {
                             assert !hasFutures() : futures();
 
-                            finish(1, mapping, commit);
+                            finish(1, mapping, commit, !clearThreadMap);
                         }
                     }
                     else
-                        finish(mappings.mappings(), commit);
+                        finish(mappings.mappings(), commit, !clearThreadMap);
                 }
 
                 markInitialized();
             }
             else
-                onDone(new IgniteCheckedException("Failed to commit transaction: " + CU.txString(tx)));
+                onDone(new IgniteCheckedException("Failed to " + (commit ? "commit" : "rollback") +
+                    " transaction: " + CU.txString(tx)));
         }
         catch (Error | RuntimeException e) {
             onDone(e);
@@ -679,26 +729,28 @@ private void readyNearMappingFromBackup(GridDistributedTxMapping mapping) {
     /**
      * @param mappings Mappings.
      * @param commit Commit flag.
+     * @param {@code true} If need to add completed version on finish.
      */
-    private void finish(Iterable<GridDistributedTxMapping> mappings, boolean commit) {
+    private void finish(Iterable<GridDistributedTxMapping> mappings, boolean commit, boolean useCompletedVer) {
         assert !hasFutures() : futures();
 
         int miniId = 0;
 
         // Create mini futures.
         for (GridDistributedTxMapping m : mappings)
-            finish(++miniId, m, commit);
+            finish(++miniId, m, commit, useCompletedVer);
     }
 
     /**
      * @param miniId Mini future ID.
      * @param m Mapping.
      * @param commit Commit flag.
+     * @param useCompletedVer {@code True} if need to add completed version on finish.
      */
-    private void finish(int miniId, GridDistributedTxMapping m, boolean commit) {
+    private void finish(int miniId, GridDistributedTxMapping m, boolean commit, boolean useCompletedVer) {
         ClusterNode n = m.primary();
 
-        assert !m.empty() : m;
+        assert !m.empty() : m + " " + tx.state();
 
         CacheWriteSynchronizationMode syncMode = tx.syncMode();
 
@@ -706,7 +758,7 @@ private void finish(int miniId, GridDistributedTxMapping m, boolean commit) {
             syncMode = FULL_SYNC;
 
         // Version to be added in completed versions on primary node.
-        GridCacheVersion completedVer = !commit && tx.timeout() > 0 ? tx.xidVersion() : null;
+        GridCacheVersion completedVer = !commit && useCompletedVer ? tx.xidVersion() : null;
 
         GridNearTxFinishRequest req = new GridNearTxFinishRequest(
             futId,
@@ -746,7 +798,7 @@ private void finish(int miniId, GridDistributedTxMapping m, boolean commit) {
 
             add(fut); // Append new future.
 
-            if (tx.pessimistic())
+            if (tx.pessimistic() && !useCompletedVer)
                 cctx.tm().beforeFinishRemote(n.id(), tx.threadId());
 
             try {
@@ -1000,7 +1052,15 @@ boolean onNodeLeft(UUID nodeId, boolean discoThread) {
          */
         void onNearFinishResponse(GridNearTxFinishResponse res) {
             if (res.error() != null)
-                onDone(res.error());
+                if (res.error() instanceof IgniteTxRollbackCheckedException) {
+                    // This exception is expected on asynchronous rollback.
+                    if (log.isDebugEnabled())
+                        log.debug("Transaction was rolled back: " + tx);
+
+                    onDone(tx);
+                }
+                else
+                    onDone(res.error());
             else
                 onDone(tx);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index e3397505bec8d..9c0b220000c0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -34,7 +34,6 @@
 import javax.cache.expiry.ExpiryPolicy;
 import javax.cache.processor.EntryProcessor;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.NodeStoppingException;
@@ -67,6 +66,7 @@
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.transactions.TransactionProxy;
 import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
+import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyRollbackOnlyImpl;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
@@ -166,17 +166,24 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
     protected boolean transform;
 
     /** */
-    private final boolean trackTimeout;
+    private boolean trackTimeout;
 
     /** */
     @GridToStringExclude
     private TransactionProxyImpl proxy;
 
+    /** */
+    @GridToStringExclude
+    private TransactionProxyImpl rollbackOnlyProxy;
+
+    /** Tx label. */
+    private @Nullable String lb;
+
     /**
      * Empty constructor required for {@link Externalizable}.
      */
     public GridNearTxLocal() {
-        this.trackTimeout = false;
+        // No-op.
     }
 
     /**
@@ -192,6 +199,7 @@ public GridNearTxLocal() {
      * @param txSize Transaction size.
      * @param subjId Subject ID.
      * @param taskNameHash Task name hash code.
+     * @param lb Label.
      */
     public GridNearTxLocal(
         GridCacheSharedContext ctx,
@@ -205,7 +213,8 @@ public GridNearTxLocal(
         boolean storeEnabled,
         int txSize,
         @Nullable UUID subjId,
-        int taskNameHash
+        int taskNameHash,
+        @Nullable String lb
     ) {
         super(
             ctx,
@@ -224,6 +233,7 @@ public GridNearTxLocal(
             txSize,
             subjId,
             taskNameHash);
+        this.lb = lb;
 
         mappings = implicitSingle ? new IgniteTxMappingsSingleImpl() : new IgniteTxMappingsImpl();
 
@@ -342,6 +352,24 @@ public void colocatedLocallyMapped(boolean colocatedLocallyMapped) {
         return entry.detached() || super.ownsLockUnsafe(entry);
     }
 
+    /** {@inheritDoc} */
+    @Override public long timeout(long timeout) {
+        long old = super.timeout(timeout);
+
+        if (old == timeout)
+            return old;
+
+        if (trackTimeout) {
+            if (!cctx.time().removeTimeoutObject(this))
+                return old; // Do nothing, because transaction is started to roll back.
+        }
+
+        if (timeout() > 0) // Method can be called only for explicit transactions.
+            trackTimeout = cctx.time().addTimeoutObject(this);
+
+        return old;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean ownsLock(GridCacheEntryEx entry) throws GridCacheEntryRemovedException {
         return entry.detached() || super.ownsLock(entry);
@@ -550,23 +578,28 @@ private <K, V> IgniteInternalFuture putAsync0(
                 opCtx != null && opCtx.recovery(),
                 dataCenterId);
 
-            if (pessimistic()) {
-                assert loadFut == null || loadFut.isDone() : loadFut;
+            try {
+                loadFut.get();
+            }
+            catch (IgniteCheckedException e) {
+                return new GridFinishedFuture(e);
+            }
+
+            long timeout = remainingTime();
+
+            if (timeout == -1)
+                return new GridFinishedFuture<>(timeoutException());
 
-                if (loadFut != null)
-                    loadFut.get();
+            if (isRollbackOnly())
+                return new GridFinishedFuture<>(rollbackException());
 
+            if (pessimistic()) {
                 final Collection<KeyCacheObject> enlisted = Collections.singleton(cacheKey);
 
                 if (log.isDebugEnabled())
                     log.debug("Before acquiring transaction lock for put on key: " + enlisted);
 
-                long timeout = remainingTime();
-
-                if (timeout == -1)
-                    return new GridFinishedFuture<>(timeoutException());
-
-                IgniteInternalFuture<Boolean> fut = cacheCtx.cache().txLockAsync(enlisted,
+                IgniteInternalFuture<Boolean>fut = cacheCtx.cache().txLockAsync(enlisted,
                     timeout,
                     this,
                     /*read*/entryProcessor != null, // Needed to force load from store.
@@ -726,25 +759,24 @@ private <K, V> IgniteInternalFuture putAllAsync0(
                 opCtx != null && opCtx.recovery(),
                 dataCenterId);
 
-            if (pessimistic()) {
-                assert loadFut == null || loadFut.isDone() : loadFut;
+            try {
+                loadFut.get();
+            }
+            catch (IgniteCheckedException e) {
+                return new GridFinishedFuture(e);
+            }
 
-                if (loadFut != null) {
-                    try {
-                        loadFut.get();
-                    }
-                    catch (IgniteCheckedException e) {
-                        return new GridFinishedFuture(e);
-                    }
-                }
+            long timeout = remainingTime();
 
-                if (log.isDebugEnabled())
-                    log.debug("Before acquiring transaction lock for put on keys: " + enlisted);
+            if (timeout == -1)
+                return new GridFinishedFuture<>(timeoutException());
 
-                long timeout = remainingTime();
+            if (isRollbackOnly())
+                return new GridFinishedFuture<>(rollbackException());
 
-                if (timeout == -1)
-                    return new GridFinishedFuture<>(timeoutException());
+            if (pessimistic()) {
+                if (log.isDebugEnabled())
+                    log.debug("Before acquiring transaction lock for put on keys: " + enlisted);
 
                 IgniteInternalFuture<Boolean> fut = cacheCtx.cache().txLockAsync(enlisted,
                     timeout,
@@ -842,7 +874,12 @@ private <K, V> IgniteInternalFuture<Void> enlistWrite(
         boolean keepBinary,
         boolean recovery,
         Byte dataCenterId) {
+        GridFutureAdapter<Void> enlistFut = new GridFutureAdapter<>();
+
         try {
+            if (!updateLockFuture(null, enlistFut))
+                return finishFuture(enlistFut, timedOut() ? timeoutException() : rollbackException(), false);
+
             addActiveCache(cacheCtx, recovery);
 
             final boolean hasFilters = !F.isEmptyOrNulls(filter) && !F.isAlwaysTrue(filter);
@@ -883,7 +920,7 @@ private <K, V> IgniteInternalFuture<Void> enlistWrite(
                 if (topVer == null)
                     topVer = entryTopVer;
 
-                return loadMissing(cacheCtx,
+                IgniteInternalFuture<Void> loadFut = loadMissing(cacheCtx,
                     topVer != null ? topVer : topologyVersion(),
                     Collections.singleton(cacheKey),
                     filter,
@@ -896,12 +933,29 @@ private <K, V> IgniteInternalFuture<Void> enlistWrite(
                     keepBinary,
                     recovery,
                     expiryPlc);
+
+                loadFut.listen(new IgniteInClosure<IgniteInternalFuture<Void>>() {
+                    @Override public void apply(IgniteInternalFuture<Void> fut) {
+                        try {
+                            fut.get();
+
+                            finishFuture(enlistFut, null, true);
+                        }
+                        catch (IgniteCheckedException e) {
+                            finishFuture(enlistFut, e, true);
+                        }
+                    }
+                });
+
+                return enlistFut;
             }
 
-            return new GridFinishedFuture<>();
+            finishFuture(enlistFut, null, true);
+
+            return enlistFut;
         }
         catch (IgniteCheckedException e) {
-            return new GridFinishedFuture<>(e);
+            return finishFuture(enlistFut, e, true);
         }
     }
 
@@ -925,7 +979,7 @@ private <K, V> IgniteInternalFuture<Void> enlistWrite(
      * @param singleRmv {@code True} for single key remove operation ({@link Cache#remove(Object)}.
      * @param keepBinary Keep binary flag.
      * @param dataCenterId Optional data center ID.
-     * @return Future for missing values loading.
+     * @return Future for enlisting writes.
      */
     private <K, V> IgniteInternalFuture<Void> enlistWrite(
         final GridCacheContext cacheCtx,
@@ -950,11 +1004,16 @@ private <K, V> IgniteInternalFuture<Void> enlistWrite(
     ) {
         assert retval || invokeMap == null;
 
+        GridFutureAdapter<Void> enlistFut = new GridFutureAdapter<>();
+
+        if (!updateLockFuture(null, enlistFut))
+            return finishFuture(enlistFut, timedOut() ? timeoutException() : rollbackException(), false);
+
         try {
             addActiveCache(cacheCtx, recovery);
         }
         catch (IgniteCheckedException e) {
-            return new GridFinishedFuture<>(e);
+            return finishFuture(enlistFut, e, false);
         }
 
         boolean rmv = lookup == null && invokeMap == null;
@@ -971,6 +1030,9 @@ private <K, V> IgniteInternalFuture<Void> enlistWrite(
             Set<KeyCacheObject> missedForLoad = null;
 
             for (Object key : keys) {
+                if (isRollbackOnly())
+                    return finishFuture(enlistFut, timedOut() ? timeoutException() : rollbackException(), false);
+
                 if (key == null) {
                     rollback();
 
@@ -1056,7 +1118,7 @@ else if (dataCenterId != null) {
                 if (topVer == null)
                     topVer = entryTopVer;
 
-                return loadMissing(cacheCtx,
+                IgniteInternalFuture<Void> loadFut = loadMissing(cacheCtx,
                     topVer != null ? topVer : topologyVersion(),
                     missedForLoad,
                     filter,
@@ -1069,12 +1131,27 @@ else if (dataCenterId != null) {
                     keepBinary,
                     recovery,
                     expiryPlc);
+
+                loadFut.listen(new IgniteInClosure<IgniteInternalFuture<Void>>() {
+                    @Override public void apply(IgniteInternalFuture<Void> fut) {
+                        try {
+                            fut.get();
+
+                            finishFuture(enlistFut, null, true);
+                        }
+                        catch (IgniteCheckedException e) {
+                            finishFuture(enlistFut, e, true);
+                        }
+                    }
+                });
+
+                return enlistFut;
             }
 
-            return new GridFinishedFuture<>();
+            return finishFuture(enlistFut, null, true);
         }
         catch (IgniteCheckedException e) {
-            return new GridFinishedFuture<>(e);
+            return finishFuture(enlistFut, e, true);
         }
     }
 
@@ -1537,6 +1614,21 @@ private <K, V> IgniteInternalFuture<GridCacheReturn> removeAllAsync0(
             dataCenterId
         );
 
+        try {
+            loadFut.get();
+        }
+        catch (IgniteCheckedException e) {
+            return new GridFinishedFuture(e);
+        }
+
+        long timeout = remainingTime();
+
+        if (timeout == -1)
+            return new GridFinishedFuture<>(timeoutException());
+
+        if (isRollbackOnly())
+            return new GridFinishedFuture<>(rollbackException());
+
         if (log.isDebugEnabled())
             log.debug("Remove keys: " + enlisted);
 
@@ -1544,25 +1636,9 @@ private <K, V> IgniteInternalFuture<GridCacheReturn> removeAllAsync0(
         // Otherwise, during rollback we will not know whether locks need
         // to be rolled back.
         if (pessimistic()) {
-            assert loadFut == null || loadFut.isDone() : loadFut;
-
-            if (loadFut != null) {
-                try {
-                    loadFut.get();
-                }
-                catch (IgniteCheckedException e) {
-                    return new GridFinishedFuture<>(e);
-                }
-            }
-
             if (log.isDebugEnabled())
                 log.debug("Before acquiring transaction lock for remove on keys: " + enlisted);
 
-            long timeout = remainingTime();
-
-            if (timeout == -1)
-                return new GridFinishedFuture<>(timeoutException());
-
             IgniteInternalFuture<Boolean> fut = cacheCtx.cache().txLockAsync(enlisted,
                 timeout,
                 this,
@@ -1684,6 +1760,11 @@ public <K, V> IgniteInternalFuture<Map<K, V>> getAllAsync(
         try {
             checkValid();
 
+            GridFutureAdapter<?> enlistFut = new GridFutureAdapter<>();
+
+            if (!updateLockFuture(null, enlistFut))
+                return new GridFinishedFuture<>(timedOut() ? timeoutException() : rollbackException());
+
             final Map<K, V> retMap = new GridLeanMap<>(keysCnt);
 
             final Map<KeyCacheObject, GridCacheVersion> missed = new GridLeanMap<>(pessimistic() ? keysCnt : 0);
@@ -1692,19 +1773,32 @@ public <K, V> IgniteInternalFuture<Map<K, V>> getAllAsync(
 
             ExpiryPolicy expiryPlc = opCtx != null ? opCtx.expiry() : null;
 
-            final Collection<KeyCacheObject> lockKeys = enlistRead(cacheCtx,
-                entryTopVer,
-                keys,
-                expiryPlc,
-                retMap,
-                missed,
-                keysCnt,
-                deserializeBinary,
-                skipVals,
-                keepCacheObjects,
-                skipStore,
-                recovery,
-                needVer);
+            final Collection<KeyCacheObject> lockKeys;
+
+            try {
+                lockKeys = enlistRead(cacheCtx,
+                    entryTopVer,
+                    keys,
+                    expiryPlc,
+                    retMap,
+                    missed,
+                    keysCnt,
+                    deserializeBinary,
+                    skipVals,
+                    keepCacheObjects,
+                    skipStore,
+                    recovery,
+                    needVer);
+            }
+            catch (IgniteCheckedException e) {
+                return new GridFinishedFuture<>(e);
+            }
+            finally {
+                finishFuture(enlistFut, null, true);
+            }
+
+            if (isRollbackOnly())
+                return new GridFinishedFuture<>(timedOut() ? timeoutException() : rollbackException());
 
             if (single && missed.isEmpty())
                 return new GridFinishedFuture<>(retMap);
@@ -1996,6 +2090,9 @@ private <K, V> Collection<KeyCacheObject> enlistRead(
         // Transactions that are pessimistic and not read-committed are covered
         // outside of this loop.
         for (KeyCacheObject key : keys) {
+            if (isRollbackOnly())
+                throw timedOut() ? timeoutException() : rollbackException();
+
             if ((pessimistic() || needReadVer) && !readCommitted() && !skipVals)
                 addActiveCache(cacheCtx, recovery);
 
@@ -2383,13 +2480,6 @@ private IgniteInternalFuture optimisticPutFuture(
             // with prepare response, if required.
             assert loadFut.isDone();
 
-            try {
-                loadFut.get();
-            }
-            catch (IgniteCheckedException e) {
-                return new GridFinishedFuture<>(e);
-            }
-
             return nonInterruptable(commitNearTxLocalAsync().chain(
                 new CX1<IgniteInternalFuture<IgniteInternalTx>, GridCacheReturn>() {
                     @Override public GridCacheReturn applyx(IgniteInternalFuture<IgniteInternalTx> txFut)
@@ -2805,7 +2895,7 @@ private void processLoaded(
     /**
      * @return DHT map.
      */
-    IgniteTxMappings mappings() {
+    public IgniteTxMappings mappings() {
         return mappings;
     }
 
@@ -2887,16 +2977,17 @@ public void suspend() throws IgniteCheckedException {
      * @throws IgniteCheckedException If the transaction is in an incorrect state, or timed out.
      */
     public void resume() throws IgniteCheckedException {
-        resume(true);
+        resume(true, Thread.currentThread().getId());
     }
 
     /**
      * Resumes transaction (possibly in another thread) if it was previously suspended.
      *
      * @param checkTimeout Whether timeout should be checked.
+     * @param threadId Thread id to restore.
      * @throws IgniteCheckedException If the transaction is in an incorrect state, or timed out.
      */
-    private void resume(boolean checkTimeout) throws IgniteCheckedException {
+    private void resume(boolean checkTimeout, long threadId) throws IgniteCheckedException {
         if (log.isDebugEnabled())
             log.debug("Resume near local tx: " + this);
 
@@ -2906,7 +2997,7 @@ private void resume(boolean checkTimeout) throws IgniteCheckedException {
         synchronized (this) {
             checkValid(checkTimeout);
 
-            cctx.tm().resumeTx(this);
+            cctx.tm().resumeTx(this, threadId);
         }
     }
 
@@ -3062,7 +3153,8 @@ private void readyNearLock(IgniteTxEntry txEntry,
                 TransactionState state = state();
 
                 if (state != COMMITTING && state != COMMITTED)
-                    throw new IgniteCheckedException("Invalid transaction state for commit [state=" + state() +
+                    throw isRollbackOnly() ? timedOut() ? timeoutException() : rollbackException() :
+                        new IgniteCheckedException("Invalid transaction state for commit [state=" + state() +
                         ", tx=" + this + ']');
                 else {
                     if (log.isDebugEnabled())
@@ -3215,15 +3307,15 @@ public IgniteInternalFuture<IgniteInternalTx> commitNearTxLocalAsync() {
         NearTxFinishFuture fut = finishFut;
 
         if (fut != null)
-            return chainFinishFuture(fut, true);
+            return chainFinishFuture(fut, true, true, false);
 
         if (fastFinish()) {
             GridNearTxFastFinishFuture fut0;
 
-            if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, true, false)))
-                return chainFinishFuture(finishFut, true);
+            if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, true)))
+                return chainFinishFuture(finishFut, true, true, false);
 
-            fut0.finish();
+            fut0.finish(false);
 
             return fut0;
         }
@@ -3231,7 +3323,7 @@ public IgniteInternalFuture<IgniteInternalTx> commitNearTxLocalAsync() {
         final GridNearTxFinishFuture fut0;
 
         if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFinishFuture<>(cctx, this, true)))
-            return chainFinishFuture(finishFut, true);
+            return chainFinishFuture(finishFut, true, true, false);
 
         cctx.mvcc().addFuture(fut0, fut0.futureId());
 
@@ -3243,14 +3335,12 @@ public IgniteInternalFuture<IgniteInternalTx> commitNearTxLocalAsync() {
                     // Make sure that here are no exceptions.
                     prepareFut.get();
 
-                    TransactionState state = state();
-
-                    fut0.finish(state == PREPARED || state == COMMITTING || state == COMMITTED, true);
+                    fut0.finish(true, true, false);
                 }
                 catch (Error | RuntimeException e) {
                     COMMIT_ERR_UPD.compareAndSet(GridNearTxLocal.this, null, e);
 
-                    fut0.finish(false, true);
+                    fut0.finish(false, true, false);
 
                     throw e;
                 }
@@ -3258,7 +3348,7 @@ public IgniteInternalFuture<IgniteInternalTx> commitNearTxLocalAsync() {
                     COMMIT_ERR_UPD.compareAndSet(GridNearTxLocal.this, null, e);
 
                     if (!(e instanceof NodeStoppingException))
-                        fut0.finish(false, true);
+                        fut0.finish(false, true, true);
                 }
             }
         });
@@ -3282,14 +3372,20 @@ public void rollback() throws IgniteCheckedException {
      * @return Rollback future.
      */
     public IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync() {
-        return rollbackNearTxLocalAsync(false);
+        return rollbackNearTxLocalAsync(true, false);
     }
 
     /**
-     * @param onTimeout {@code True} if rolled back asynchronously on timeout.
+     * @param clearThreadMap {@code True} if needed to clear thread map.
+     * @param onTimeout {@code True} if called from timeout handler.
+     * Note: For async rollbacks (from timeouts or another thread) should not clear thread map
+     * since thread started tx still should be able to see this tx to prevent subsequent operations.
+     *
      * @return Rollback future.
      */
-    public IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final boolean onTimeout) {
+    @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+    public IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final boolean clearThreadMap,
+        final boolean onTimeout) {
         if (log.isDebugEnabled())
             log.debug("Rolling back near tx: " + this);
 
@@ -3299,15 +3395,16 @@ public IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final boo
         NearTxFinishFuture fut = finishFut;
 
         if (fut != null)
-            return chainFinishFuture(finishFut, false, !onTimeout);
+            return chainFinishFuture(finishFut, false, clearThreadMap, onTimeout);
 
-        if (fastFinish()) {
+        // Enable fast finish only from tx thread.
+        if (clearThreadMap && fastFinish()) {
             GridNearTxFastFinishFuture fut0;
 
-            if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, false, onTimeout)))
-                return chainFinishFuture(finishFut, false, !onTimeout);
+            if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, false)))
+                return chainFinishFuture(finishFut, false, true, onTimeout);
 
-            fut0.finish();
+            fut0.finish(true);
 
             return fut0;
         }
@@ -3315,7 +3412,7 @@ public IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final boo
         final GridNearTxFinishFuture fut0;
 
         if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFinishFuture<>(cctx, this, false)))
-            return chainFinishFuture(finishFut, false, !onTimeout);
+            return chainFinishFuture(finishFut, false, clearThreadMap, onTimeout);
 
         cctx.mvcc().addFuture(fut0, fut0.futureId());
 
@@ -3332,9 +3429,9 @@ public IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final boo
                     log.debug("Got optimistic tx failure [tx=" + this + ", err=" + e + ']');
             }
 
-            fut0.finish(false, !onTimeout);
+            fut0.finish(false, clearThreadMap, onTimeout);
         }
-        else if (!onTimeout) {
+        else {
             prepFut.listen(new CI1<IgniteInternalFuture<?>>() {
                 @Override public void apply(IgniteInternalFuture<?> f) {
                     try {
@@ -3346,11 +3443,10 @@ else if (!onTimeout) {
                             log.debug("Got optimistic tx failure [tx=" + this + ", err=" + e + ']');
                     }
 
-                    fut0.finish(false, true);
+                    fut0.finish(false, clearThreadMap, onTimeout);
                 }
             });
-        } else
-            fut0.finish(false, false);
+        }
 
         return fut0;
     }
@@ -3360,17 +3456,14 @@ else if (!onTimeout) {
         return rollbackNearTxLocalAsync();
     }
 
-    /** */
-    private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFinishFuture fut, final boolean commit) {
-        return chainFinishFuture(fut, commit, true);
-    }
-
     /**
      * @param fut Already started finish future.
      * @param commit Commit flag.
+     * @param clearThreadMap Clear thread map.
      * @return Finish future.
      */
-    private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFinishFuture fut, final boolean commit, final boolean clearThreadMap) {
+    private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFinishFuture fut, final boolean commit,
+        final boolean clearThreadMap, final boolean onTimeout) {
         assert fut != null;
 
         if (fut.commit() != commit) {
@@ -3394,9 +3487,23 @@ private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFin
                                 if (!cctx.mvcc().addFuture(rollbackFut, rollbackFut.futureId()))
                                     return;
 
-                                rollbackFut.finish(false, clearThreadMap);
+                                rollbackFut.finish(false, clearThreadMap, onTimeout);
                             }
                         }
+                        else {
+                            finishFut.listen(new IgniteInClosure<IgniteInternalFuture<IgniteInternalTx>>() {
+                                @Override public void apply(IgniteInternalFuture<IgniteInternalTx> fut) {
+                                    try {
+                                        fut.get();
+
+                                        rollbackFut.markInitialized();
+                                    }
+                                    catch (IgniteCheckedException e) {
+                                        rollbackFut.onDone(e);
+                                    }
+                                }
+                            });
+                        }
                     }
                 });
 
@@ -3411,7 +3518,7 @@ private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFin
                             fut0.onDone(new IgniteTxTimeoutCheckedException("Failed to commit transaction, " +
                                 "transaction is concurrently rolled back on timeout: " + tx));
                         else
-                            fut0.onDone(new IgniteCheckedException("Failed to commit transaction, " +
+                            fut0.onDone(new IgniteTxRollbackCheckedException("Failed to commit transaction, " +
                                 "transaction is concurrently rolled back: " + tx));
                     }
                 });
@@ -3442,13 +3549,11 @@ public IgniteInternalFuture<GridNearTxPrepareResponse> prepareAsyncLocal(GridNea
 
         if (state() != PREPARING) {
             if (timeout == -1)
-                return new GridFinishedFuture<>(
-                    new IgniteTxTimeoutCheckedException("Transaction timed out: " + this));
+                return new GridFinishedFuture<>(timeoutException());
 
             setRollbackOnly();
 
-            return new GridFinishedFuture<>(
-                new IgniteCheckedException("Invalid transaction state for prepare [state=" + state() + ", tx=" + this + ']'));
+            return new GridFinishedFuture<>(rollbackException());
         }
 
         if (timeout == -1)
@@ -3483,19 +3588,6 @@ public IgniteInternalFuture<GridNearTxPrepareResponse> prepareAsyncLocal(GridNea
             setRollbackOnly();
 
             fut.onError(new IgniteTxRollbackCheckedException("Failed to prepare transaction: " + this, e));
-
-            try {
-                rollback();
-            }
-            catch (IgniteTxOptimisticCheckedException e1) {
-                if (log.isDebugEnabled())
-                    log.debug("Failed optimistically to prepare transaction [tx=" + this + ", e=" + e1 + ']');
-
-                fut.onError(e);
-            }
-            catch (IgniteCheckedException e1) {
-                U.error(log, "Failed to rollback transaction: " + this, e1);
-            }
         }
 
         return chainOnePhasePrepare(fut);
@@ -3788,17 +3880,26 @@ public <K> IgniteInternalFuture<GridCacheReturn> lockAllAsync(GridCacheContext c
 
     /** {@inheritDoc} */
     @Override public void close() throws IgniteCheckedException {
+        close(true);
+    }
+
+    /**
+     * @param clearThreadMap Clear thread map.
+     */
+    public void close(boolean clearThreadMap) throws IgniteCheckedException {
         TransactionState state = state();
 
         try {
             if (state == COMMITTED || state == ROLLED_BACK)
                 return;
 
+            boolean rmv = false;
+
             if (trackTimeout)
-                removeTimeoutHandler();
+                rmv = removeTimeoutHandler();
 
-            if (state != ROLLING_BACK && state != COMMITTING)
-                rollback();
+            if (state != COMMITTING && state != ROLLING_BACK && (!trackTimeout || rmv))
+                rollbackNearTxLocalAsync(clearThreadMap, false).get();
 
             synchronized (this) {
                 try {
@@ -3815,10 +3916,7 @@ public <K> IgniteInternalFuture<GridCacheReturn> lockAllAsync(GridCacheContext c
             }
         }
         finally {
-            // It is possible tx was rolled back asynchronously on timeout and thread map is not cleared yet.
-            boolean cleanup = state == ROLLED_BACK && timedOut();
-
-            if (cleanup)
+            if (clearThreadMap)
                 cctx.tm().clearThreadMap(this);
 
             if (accessMap != null) {
@@ -3890,6 +3988,16 @@ public TransactionProxy proxy() {
         return proxy;
     }
 
+    /**
+     * @return Public API proxy.
+     */
+    public TransactionProxy rollbackOnlyProxy() {
+        if (rollbackOnlyProxy == null)
+            rollbackOnlyProxy = new TransactionProxyRollbackOnlyImpl<>(this, cctx, false);
+
+        return rollbackOnlyProxy;
+    }
+
     /**
      * @param cacheCtx Cache context.
      * @param topVer Topology version.
@@ -3954,14 +4062,6 @@ private <K, V> IgniteInternalFuture<Map<K, V>> checkMissed(
                 expiryPlc,
                 new GridInClosure3<KeyCacheObject, Object, GridCacheVersion>() {
                     @Override public void apply(KeyCacheObject key, Object val, GridCacheVersion loadVer) {
-                        if (isRollbackOnly()) {
-                            if (log.isDebugEnabled())
-                                log.debug("Ignoring loaded value for read because transaction was rolled back: " +
-                                    GridNearTxLocal.this);
-
-                            return;
-                        }
-
                         CacheObject cacheVal = cacheCtx.toCacheObject(val);
 
                         CacheObject visibleVal = cacheVal;
@@ -4163,11 +4263,18 @@ public boolean addTimeoutHandler() {
         return startTime() + timeout();
     }
 
+    /**
+     * @return Tx label.
+     */
+    public String label() {
+        return lb;
+    }
+
     /** {@inheritDoc} */
     @Override public void onTimeout() {
         if (state() == SUSPENDED) {
             try {
-                resume(false);
+                resume(false, threadId());
             }
             catch (IgniteCheckedException e) {
                 log.warning("Error resuming suspended transaction on timeout: " + this, e);
@@ -4185,9 +4292,10 @@ public boolean addTimeoutHandler() {
                 @Override public void run() {
                     // Note: if rollback asynchronously on timeout should not clear thread map
                     // since thread started tx still should be able to see this tx.
-                    rollbackNearTxLocalAsync(true);
+                    rollbackNearTxLocalAsync(false, true);
 
-                    U.warn(log, "Transaction was rolled back because the timeout is reached: " + GridNearTxLocal.this);
+                    U.warn(log, "The transaction was forcibly rolled back because a timeout is reached: " +
+                        CU.txString(GridNearTxLocal.this));
                 }
             });
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java
index 93d82c93773b5..18a84319945f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalLockFuture.java
@@ -36,20 +36,21 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheVersionedFuture;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
 import org.apache.ignite.internal.processors.cache.transactions.TxDeadlock;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.transactions.TransactionDeadlockException;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
-import org.apache.ignite.transactions.TransactionDeadlockException;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -155,6 +156,12 @@ public final class GridLocalLockFuture<K, V> extends GridCacheFutureAdapter<Bool
 
         if (log == null)
             log = U.logger(cctx.kernalContext(), logRef, GridLocalLockFuture.class);
+
+        if (tx != null && tx instanceof GridNearTxLocal && !((GridNearTxLocal)tx).updateLockFuture(null, this)) {
+            GridNearTxLocal tx0 = (GridNearTxLocal)tx;
+
+            onError(tx0.timedOut() ? tx0.timeoutException() : tx0.rollbackException());
+        }
     }
 
     /**
@@ -457,6 +464,9 @@ private void onComplete(boolean success) {
         if (!success)
             undoLocks();
 
+        if (tx != null && success)
+            ((GridNearTxLocal)tx).clearLockFuture(this);
+
         if (onDone(success, err)) {
             if (log.isDebugEnabled())
                 log.debug("Completing future: " + this);
@@ -490,47 +500,51 @@ private class LockTimeoutObject extends GridTimeoutObjectAdapter {
             if (log.isDebugEnabled())
                 log.debug("Timed out waiting for lock response: " + this);
 
-            if (inTx() && cctx.tm().deadlockDetectionEnabled()) {
-                Set<IgniteTxKey> keys = new HashSet<>();
+            if (inTx()) {
+                if (cctx.tm().deadlockDetectionEnabled()) {
+                    Set<IgniteTxKey> keys = new HashSet<>();
 
-                List<GridLocalCacheEntry> entries = entries();
+                    List<GridLocalCacheEntry> entries = entries();
 
-                for (int i = 0; i < entries.size(); i++) {
-                    GridLocalCacheEntry e = entries.get(i);
+                    for (int i = 0; i < entries.size(); i++) {
+                        GridLocalCacheEntry e = entries.get(i);
 
-                    List<GridCacheMvccCandidate> mvcc = e.mvccAllLocal();
+                        List<GridCacheMvccCandidate> mvcc = e.mvccAllLocal();
 
-                    if (mvcc == null)
-                        continue;
+                        if (mvcc == null)
+                            continue;
 
-                    GridCacheMvccCandidate cand = mvcc.get(0);
+                        GridCacheMvccCandidate cand = mvcc.get(0);
 
-                    if (cand.owner() && cand.tx() && !cand.version().equals(tx.xidVersion()))
-                        keys.add(e.txKey());
-                }
+                        if (cand.owner() && cand.tx() && !cand.version().equals(tx.xidVersion()))
+                            keys.add(e.txKey());
+                    }
 
-                IgniteInternalFuture<TxDeadlock> fut = cctx.tm().detectDeadlock(tx, keys);
+                    IgniteInternalFuture<TxDeadlock> fut = cctx.tm().detectDeadlock(tx, keys);
 
-                fut.listen(new IgniteInClosure<IgniteInternalFuture<TxDeadlock>>() {
-                    @Override public void apply(IgniteInternalFuture<TxDeadlock> fut) {
-                        try {
-                            TxDeadlock deadlock = fut.get();
+                    fut.listen(new IgniteInClosure<IgniteInternalFuture<TxDeadlock>>() {
+                        @Override public void apply(IgniteInternalFuture<TxDeadlock> fut) {
+                            try {
+                                TxDeadlock deadlock = fut.get();
 
-                            if (deadlock != null)
-                                ERR_UPD.compareAndSet(GridLocalLockFuture.this, null,
-                                        new IgniteTxTimeoutCheckedException("Failed to acquire lock within provided timeout for " +
-                                                "transaction [timeout=" + tx.timeout() + ", tx=" + tx + ']',
-                                                new TransactionDeadlockException(deadlock.toString(cctx.shared()))));
-                        }
-                        catch (IgniteCheckedException e) {
-                            U.warn(log, "Failed to detect deadlock.", e);
+                                err = new IgniteTxTimeoutCheckedException("Failed to acquire lock within provided " +
+                                    "timeout for transaction [timeout=" + tx.timeout() + ", tx=" + CU.txString(tx) + ']',
+                                    deadlock != null ? new TransactionDeadlockException(deadlock.toString(cctx.shared())) :
+                                        null);
+                            }
+                            catch (IgniteCheckedException e) {
+                                err = e;
 
-                            ERR_UPD.compareAndSet(GridLocalLockFuture.this, null, e);
-                        }
+                                U.warn(log, "Failed to detect deadlock.", e);
+                            }
 
-                        onComplete(false);
-                    }
-                });
+                            onComplete(false);
+                        }
+                    });
+                }
+                else
+                    err = new IgniteTxTimeoutCheckedException("Failed to acquire lock within provided " +
+                        "timeout for transaction [timeout=" + tx.timeout() + ", tx=" + CU.txString(tx) + ']');
             }
             else
                 onComplete(false);
@@ -541,4 +555,4 @@ private class LockTimeoutObject extends GridTimeoutObjectAdapter {
             return S.toString(LockTimeoutObject.class, this);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index f24f5d87fc08a..d5da19998d09f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -2369,7 +2369,6 @@ private void restorePartitionStates(Map<T2<Integer, Integer>, T2<Integer, Long>>
 
                                     if (stateId == GridDhtPartitionState.OWNING.ordinal()
                                         || (stateId == GridDhtPartitionState.MOVING.ordinal()
-
                                         && part.initialUpdateCounter() < restore.get2())) {
                                         part.initialUpdateCounter(restore.get2());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
index 6c5922a92f8bf..64a052d8b3e64 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/store/GridCacheStoreManagerAdapter.java
@@ -1430,6 +1430,11 @@ private static class TxProxy implements Transaction {
             throw new UnsupportedOperationException();
         }
 
+        /** {@inheritDoc} */
+        @Nullable @Override public String label() {
+            return null;
+        }
+
         /** {@inheritDoc} */
         @Override public long timeout() {
             return tx.timeout();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTransactionsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTransactionsImpl.java
index 12f655adc02b3..25ba849b37d1d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTransactionsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTransactionsImpl.java
@@ -17,13 +17,18 @@
 
 package org.apache.ignite.internal.processors.cache.transactions;
 
+import java.util.Collection;
+import org.apache.ignite.IgniteTransactions;
 import org.apache.ignite.configuration.TransactionConfiguration;
 import org.apache.ignite.internal.IgniteTransactionsEx;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
@@ -38,11 +43,15 @@ public class IgniteTransactionsImpl<K, V> implements IgniteTransactionsEx {
     /** Cache shared context. */
     private GridCacheSharedContext<K, V> cctx;
 
+    /** Label. */
+    private String lb;
+
     /**
      * @param cctx Cache shared context.
      */
-    public IgniteTransactionsImpl(GridCacheSharedContext<K, V> cctx) {
+    public IgniteTransactionsImpl(GridCacheSharedContext<K, V> cctx, @Nullable String lb) {
         this.cctx = cctx;
+        this.lb = lb;
     }
 
     /** {@inheritDoc} */
@@ -166,7 +175,8 @@ private GridNearTxLocal txStart0(
                 isolation,
                 timeout,
                 true,
-                txSize
+                txSize,
+                lb
             );
 
             assert tx != null;
@@ -195,6 +205,26 @@ private GridNearTxLocal txStart0(
         cctx.resetTxMetrics();
     }
 
+    /** {@inheritDoc} */
+    @Override public Collection<Transaction> localActiveTransactions() {
+        return F.viewReadOnly(cctx.tm().activeTransactions(), new IgniteClosure<IgniteInternalTx, Transaction>() {
+            @Override public Transaction apply(IgniteInternalTx tx) {
+                return ((GridNearTxLocal)tx).rollbackOnlyProxy();
+            }
+        }, new IgnitePredicate<IgniteInternalTx>() {
+            @Override public boolean apply(IgniteInternalTx tx) {
+                return tx.local() && tx.near();
+            }
+        });
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteTransactions withLabel(String lb) {
+        A.notNull(lb, "label should not be empty.");
+
+        return new IgniteTransactionsImpl<>(cctx, lb);
+    }
+
     /**
      * @param ctx Cache context.
      */
@@ -202,4 +232,4 @@ private void checkTransactional(GridCacheContext ctx) {
         if (!ctx.transactional())
             throw new TransactionException("Failed to start transaction on non-transactional cache: " + ctx.name());
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 945ef486fbe5a..3cf1146e80657 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -65,6 +65,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
+import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.GridSetWrapper;
@@ -706,7 +707,15 @@ public void needReturnValue(boolean needRetVal) {
      */
     public final IgniteCheckedException timeoutException() {
         return new IgniteTxTimeoutCheckedException("Failed to acquire lock within provided timeout " +
-            "for transaction [timeout=" + timeout() + ", tx=" + this + ']');
+            "for transaction [timeout=" + timeout() + ", tx=" + CU.txString(this) + ']');
+    }
+
+    /**
+     * @return Rollback exception.
+     */
+    public final IgniteCheckedException rollbackException() {
+        return new IgniteTxRollbackCheckedException("Failed to finish transaction because it has been rolled back " +
+            "[timeout=" + timeout() + ", tx=" + CU.txString(this) + ']');
     }
 
     /** {@inheritDoc} */
@@ -993,9 +1002,9 @@ public void onePhaseCommit(boolean onePhaseCommit) {
      *
      * @param state State to set.
      * @param timedOut Timeout flag.
+     *
      * @return {@code True} if state changed.
      */
-    @SuppressWarnings({"TooBroadScope"})
     protected final boolean state(TransactionState state, boolean timedOut) {
         boolean valid = false;
 
@@ -1058,7 +1067,7 @@ protected final boolean state(TransactionState state, boolean timedOut) {
                 }
 
                 case MARKED_ROLLBACK: {
-                    valid = prev == ACTIVE || prev == PREPARING || prev == PREPARED || prev == SUSPENDED;
+                    valid = prev == ACTIVE  || prev == PREPARING || prev == PREPARED || prev == SUSPENDED;
 
                     break;
                 }
@@ -1099,11 +1108,9 @@ protected final boolean state(TransactionState state, boolean timedOut) {
                 if (state != ACTIVE && state != SUSPENDED)
                     seal();
 
-                if (cctx.wal() != null && cctx.tm().logTxRecords()) {
+                if (cctx.wal() != null && cctx.tm().logTxRecords() && txNodes != null) {
                     // Log tx state change to WAL.
                     if (state == PREPARED || state == COMMITTED || state == ROLLED_BACK) {
-                        assert txNodes != null || state == ROLLED_BACK : "txNodes=" + txNodes + " state=" + state;
-
                         BaselineTopology baselineTop = cctx.kernalContext().state().clusterState().baselineTopology();
 
                         Map<Short, Collection<Short>> participatingNodes = consistentIdMapper
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 38c877b030016..44482abf0c8b7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -256,9 +256,6 @@ public IgniteInternalFuture<GridNearTxPrepareResponse> prepareColocatedTx(
 
         IgniteInternalFuture<GridNearTxPrepareResponse> fut = locTx.prepareAsyncLocal(req);
 
-        if (locTx.isRollbackOnly())
-            locTx.rollbackNearTxLocalAsync();
-
         return fut.chain(new C1<IgniteInternalFuture<GridNearTxPrepareResponse>, GridNearTxPrepareResponse>() {
             @Override public GridNearTxPrepareResponse apply(IgniteInternalFuture<GridNearTxPrepareResponse> f) {
                 try {
@@ -885,7 +882,7 @@ private IgniteInternalFuture<IgniteInternalTx> finishDhtLocal(UUID nodeId,
                 req.threadId(),
                 req.futureId(),
                 req.miniId(),
-                new IgniteCheckedException("Transaction has been already completed."));
+                new IgniteTxRollbackCheckedException("Transaction has been already completed or not started yet."));
 
             try {
                 ctx.io().send(nodeId, res, req.policy());
@@ -1698,6 +1695,8 @@ private void sendReply(UUID nodeId, GridDhtTxFinishRequest req, boolean committe
             res.invalidPartitionsByCacheId(tx.invalidPartitions());
 
             if (tx.empty() && req.last()) {
+                tx.skipCompletedVersions(req.skipCompletedVersion());
+
                 tx.rollbackRemoteTx();
 
                 return null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 7a7f65f7fb9e7..54b7ae2cbe630 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.internal.processors.cache.transactions;
 
 import java.io.Externalizable;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Map;
@@ -963,7 +962,7 @@ public void tmFinish(boolean commit, boolean nodeStop, boolean clearThreadMap) t
                 if (commit)
                     cctx.tm().commitTx(this);
                 else
-                    cctx.tm().rollbackTx(this, clearThreadMap);
+                    cctx.tm().rollbackTx(this, clearThreadMap, false);
             }
 
             state(commit ? COMMITTED : ROLLED_BACK);
@@ -1028,7 +1027,7 @@ public Collection<GridCacheVersion> rolledbackVersions() {
         }
 
         if (DONE_FLAG_UPD.compareAndSet(this, 0, 1)) {
-            cctx.tm().rollbackTx(this, clearThreadMap);
+            cctx.tm().rollbackTx(this, clearThreadMap, false);
 
             if (!internal()) {
                 Collection<CacheStoreManager> stores = txState.stores(cctx);
@@ -1310,7 +1309,7 @@ protected void checkValid(boolean checkTimeout) throws IgniteCheckedException {
 
         if (isRollbackOnly()) {
             if (remainingTime() == -1)
-                throw new IgniteTxTimeoutCheckedException("Cache transaction timed out: " + this);
+                throw new IgniteTxTimeoutCheckedException("Cache transaction timed out: " + CU.txString(this));
 
             TransactionState state = state();
 
@@ -1322,7 +1321,7 @@ protected void checkValid(boolean checkTimeout) throws IgniteCheckedException {
                 throw new IgniteTxHeuristicCheckedException("Cache transaction is in unknown state " +
                     "(remote transactions will be invalidated): " + this);
 
-            throw new IgniteCheckedException("Cache transaction marked as rollback-only: " + this);
+            throw rollbackException();
         }
     }
 
@@ -1367,12 +1366,6 @@ protected final IgniteTxEntry addEntry(GridCacheOperation op,
 
         checkInternal(key);
 
-        TransactionState state = state();
-
-        assert state == TransactionState.ACTIVE || remainingTime() == -1 :
-            "Invalid tx state for adding entry [op=" + op + ", val=" + val + ", entry=" + entry + ", filter=" +
-                Arrays.toString(filter) + ", txCtx=" + cctx.tm().txContextVersion() + ", tx=" + this + ']';
-
         IgniteTxEntry old = entry(key);
 
         // Keep old filter if already have one (empty filter is always overridden).
@@ -1692,7 +1685,8 @@ protected PostLockClosure1(T arg, boolean commit) {
 
                 final GridClosureException ex = new GridClosureException(
                     new IgniteTxTimeoutCheckedException("Failed to acquire lock within provided timeout " +
-                        "for transaction [timeout=" + timeout() + ", tx=" + this + ']', deadlockErr)
+                        "for transaction [timeout=" + timeout() + ", tx=" + CU.txString(IgniteTxLocalAdapter.this) +
+                        ']', deadlockErr)
                 );
 
                 if (commit && commitAfterLock())
@@ -1773,7 +1767,7 @@ protected abstract class PostLockClosure2<T> implements IgniteBiClosure<Boolean,
                 if (!locked)
                     throw new GridClosureException(new IgniteTxTimeoutCheckedException("Failed to acquire lock " +
                         "within provided timeout for transaction [timeout=" + timeout() +
-                        ", tx=" + IgniteTxLocalAdapter.this + ']'));
+                        ", tx=" + CU.txString(IgniteTxLocalAdapter.this) + ']'));
 
                 IgniteInternalFuture<T> fut = postLock();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 7fa31bf027cbc..dc8bc4668c41f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -293,6 +293,22 @@ public void rollbackTransactionsForCache(int cacheId) {
         rollbackTransactionsForCache(cacheId, idMap);
     }
 
+    /**
+     * Rollback transactions blocking partition map exchange.
+     *
+     * @param topVer Initial exchange version.
+     */
+    public void rollbackOnTopologyChange(AffinityTopologyVersion topVer) {
+        for (IgniteInternalTx tx : activeTransactions()) {
+            if (tx.local() && tx.near() && needWaitTransaction(tx, topVer)) {
+                U.warn(log, "The transaction was forcibly rolled back on partition map exchange because a timeout is " +
+                    "reached: [tx=" + CU.txString(tx) + ", topVer=" + topVer + ']');
+
+                ((GridNearTxLocal)tx).rollbackNearTxLocalAsync(false, false);
+            }
+        }
+    }
+
     /**
      * @param cacheId Cache ID.
      * @param txMap Transactions map.
@@ -303,7 +319,7 @@ private void rollbackTransactionsForCache(int cacheId, ConcurrentMap<?, IgniteIn
 
             for (IgniteTxEntry entry : tx.allEntries()) {
                 if (entry.cacheId() == cacheId) {
-                    rollbackTx(tx, false);
+                    rollbackTx(tx, false, false);
 
                     break;
                 }
@@ -316,9 +332,9 @@ private void rollbackTransactionsForCache(int cacheId, ConcurrentMap<?, IgniteIn
         txFinishSync.onDisconnected(reconnectFut);
 
         for (IgniteInternalTx tx : idMap.values())
-            rollbackTx(tx, true);
+            rollbackTx(tx, true, false);
         for (IgniteInternalTx tx : nearIdMap.values())
-            rollbackTx(tx, true);
+            rollbackTx(tx, true, false);
 
         IgniteClientDisconnectedException err =
             new IgniteClientDisconnectedException(reconnectFut, "Client node disconnected.");
@@ -414,8 +430,9 @@ public int completedVersionsSize() {
     private boolean isCompleted(IgniteInternalTx tx) {
         boolean completed = completedVersHashMap.containsKey(tx.xidVersion());
 
-        // Need check that for tx with timeout rollback message was not received before lock.
-        if (!completed && tx.local() && tx.dht() && tx.timeout() > 0)
+        // Need check that for tx rollback message was not received before lock.
+        // This could happen on timeout or async rollback.
+        if (!completed && tx.local() && tx.dht())
             return completedVersHashMap.containsKey(tx.nearXidVersion());
 
         return completed;
@@ -428,6 +445,7 @@ private boolean isCompleted(IgniteInternalTx tx) {
      * @param isolation Isolation.
      * @param timeout transaction timeout.
      * @param txSize Expected transaction size.
+     * @param lb Label.
      * @return New transaction.
      */
     public GridNearTxLocal newTx(
@@ -438,7 +456,8 @@ public GridNearTxLocal newTx(
         TransactionIsolation isolation,
         long timeout,
         boolean storeEnabled,
-        int txSize
+        int txSize,
+        @Nullable String lb
     ) {
         assert sysCacheCtx == null || sysCacheCtx.systemTx();
 
@@ -458,7 +477,8 @@ public GridNearTxLocal newTx(
             storeEnabled,
             txSize,
             subjId,
-            taskNameHash);
+            taskNameHash,
+            lb);
 
         if (tx.system()) {
             AffinityTopologyVersion topVer = cctx.tm().lockedTopologyVersion(Thread.currentThread().getId(), tx);
@@ -560,10 +580,9 @@ public IgniteInternalFuture<Boolean> finishLocalTxs(AffinityTopologyVersion topV
                     }
                 });
 
-        for (IgniteInternalTx tx : txs()) {
-            if (needWaitTransaction(tx, topVer)) {
+        for (IgniteInternalTx tx : activeTransactions()) {
+            if (needWaitTransaction(tx, topVer))
                 res.add(tx.finishFuture());
-            }
         }
 
         res.markInitialized();
@@ -960,7 +979,7 @@ public void addCommittedTxReturn(IgniteInternalTx tx, GridCacheReturnCompletable
 
     /**
      * @param tx Committed transaction.
-     * @return If transaction was not already present in committed set.
+     * @return If transaction was not already present in completed set.
      */
     public boolean addRolledbackTx(IgniteInternalTx tx) {
         return addRolledbackTx(tx, tx.xidVersion());
@@ -1293,15 +1312,17 @@ else if (log.isDebugEnabled())
      *
      * @param tx Transaction to rollback.
      * @param clearThreadMap {@code True} if need remove tx from thread map.
+     * @param skipCompletedVers {@code True} if tx should skip adding itself to completed versions map on finish.
      */
-    public void rollbackTx(IgniteInternalTx tx, boolean clearThreadMap) {
+    public void rollbackTx(IgniteInternalTx tx, boolean clearThreadMap, boolean skipCompletedVers) {
         assert tx != null;
 
         if (log.isDebugEnabled())
             log.debug("Rolling back from TM [locNodeId=" + cctx.localNodeId() + ", tx=" + tx + ']');
 
         // 1. Record transaction version to avoid duplicates.
-        addRolledbackTx(tx);
+        if (!skipCompletedVers)
+            addRolledbackTx(tx);
 
         ConcurrentMap<GridCacheVersion, IgniteInternalTx> txIdMap = transactionMap(tx);
 
@@ -1355,10 +1376,11 @@ else if (log.isDebugEnabled())
      *
      * @param tx Transaction to finish.
      * @param commit {@code True} if transaction is committed, {@code false} if rolled back.
+     * @param clearThreadMap {@code True} if need remove tx from thread map.
      */
     public void fastFinishTx(GridNearTxLocal tx, boolean commit, boolean clearThreadMap) {
         assert tx != null;
-        assert tx.writeMap().isEmpty();
+        tx.writeMap().isEmpty();
         assert tx.optimistic() || tx.readMap().isEmpty();
 
         ConcurrentMap<GridCacheVersion, IgniteInternalTx> txIdMap = transactionMap(tx);
@@ -1683,7 +1705,14 @@ private boolean lockMultiple(IgniteInternalTx tx, Iterable<IgniteTxEntry> entrie
     private void txUnlock(IgniteInternalTx tx, IgniteTxEntry txEntry) {
         while (true) {
             try {
-                txEntry.cached().txUnlock(tx);
+                GridCacheEntryEx entry = txEntry.cached();
+
+                assert entry != null;
+
+                if (entry.detached())
+                    break;
+
+                entry.txUnlock(tx);
 
                 break;
             }
@@ -1691,14 +1720,7 @@ private void txUnlock(IgniteInternalTx tx, IgniteTxEntry txEntry) {
                 if (log.isDebugEnabled())
                     log.debug("Got removed entry in TM txUnlock(..) method (will retry): " + txEntry);
 
-                try {
-                    txEntry.cached(txEntry.context().cache().entryEx(txEntry.key(), tx.topologyVersion()));
-                }
-                catch (GridDhtInvalidPartitionException e) {
-                    tx.addInvalidPartition(txEntry.context(), e.partition());
-
-                    break;
-                }
+                txEntry.cached(txEntry.context().cache().entryEx(txEntry.key(), tx.topologyVersion()));
             }
         }
     }
@@ -1708,31 +1730,8 @@ private void txUnlock(IgniteInternalTx tx, IgniteTxEntry txEntry) {
      * @param entries Entries to unlock.
      */
     private void unlockMultiple(IgniteInternalTx tx, Iterable<IgniteTxEntry> entries) {
-        for (IgniteTxEntry txEntry : entries) {
-            GridCacheContext cacheCtx = txEntry.context();
-
-            while (true) {
-                try {
-                    GridCacheEntryEx entry = txEntry.cached();
-
-                    assert entry != null;
-
-                    if (entry.detached())
-                        break;
-
-                    entry.txUnlock(tx);
-
-                    break;
-                }
-                catch (GridCacheEntryRemovedException ignored) {
-                    if (log.isDebugEnabled())
-                        log.debug("Got removed entry in TM unlockMultiple(..) method (will retry): " + txEntry);
-
-                    // Renew cache entry.
-                    txEntry.cached(cacheCtx.cache().entryEx(txEntry.key(), tx.topologyVersion()));
-                }
-            }
-        }
+        for (IgniteTxEntry txEntry : entries)
+            txUnlock(tx, txEntry);
     }
 
     /**
@@ -1771,13 +1770,6 @@ public void resetContext() {
         threadCtx.set(null);
     }
 
-    /**
-     * @return All transactions.
-     */
-    public Collection<IgniteInternalTx> txs() {
-        return F.concat(false, idMap.values(), nearIdMap.values());
-    }
-
     /**
      * @return Slow tx warn timeout.
      */
@@ -1859,12 +1851,12 @@ public IgniteInternalFuture<Boolean> txCommitted(GridCacheVersion xidVer) {
      * @return Finish future for related remote transactions.
      */
     @SuppressWarnings("unchecked")
-    public IgniteInternalFuture<IgniteInternalTx> remoteTxFinishFuture(GridCacheVersion nearVer) {
-        GridCompoundFuture<IgniteInternalTx, IgniteInternalTx> fut = new GridCompoundFuture<>();
+    public IgniteInternalFuture<?> remoteTxFinishFuture(GridCacheVersion nearVer) {
+        GridCompoundFuture<Void, Void> fut = new GridCompoundFuture<>();
 
-        for (final IgniteInternalTx tx : txs()) {
+        for (final IgniteInternalTx tx : activeTransactions()) {
             if (!tx.local() && nearVer.equals(tx.nearXidVersion()))
-                fut.add(tx.finishFuture());
+                fut.add((IgniteInternalFuture) tx.finishFuture());
         }
 
         fut.markInitialized();
@@ -1884,7 +1876,7 @@ public IgniteInternalFuture<IgniteInternalTx> remoteTxFinishFuture(GridCacheVers
         @Nullable GridFutureAdapter<Boolean> fut,
         @Nullable Collection<GridCacheVersion> processedVers)
     {
-        for (final IgniteInternalTx tx : txs()) {
+        for (final IgniteInternalTx tx : activeTransactions()) {
             if (nearVer.equals(tx.nearXidVersion())) {
                 IgniteInternalFuture<?> prepFut = tx.currentPrepareFuture();
 
@@ -2308,7 +2300,7 @@ public Collection<IgniteInternalFuture<?>> deadlockDetectionFutures() {
      *
      * @param tx Transaction to be suspended.
      *
-     * @see #resumeTx(GridNearTxLocal)
+     * @see #resumeTx(GridNearTxLocal, long)
      * @see GridNearTxLocal#suspend()
      * @see GridNearTxLocal#resume()
      * @throws IgniteCheckedException If failed to suspend transaction.
@@ -2331,13 +2323,14 @@ public void suspendTx(final GridNearTxLocal tx) throws IgniteCheckedException {
      * Please don't use directly. Use tx.resume() instead.
      *
      * @param tx Transaction to be resumed.
+     * @param threadId Thread id to restore.
      *
      * @see #suspendTx(GridNearTxLocal)
      * @see GridNearTxLocal#suspend()
      * @see GridNearTxLocal#resume()
      * @throws IgniteCheckedException If failed to resume tx.
      */
-    public void resumeTx(GridNearTxLocal tx) throws IgniteCheckedException {
+    public void resumeTx(GridNearTxLocal tx, long threadId) throws IgniteCheckedException {
         assert tx != null && !tx.system() : tx;
 
         if (!tx.state(ACTIVE)) {
@@ -2349,8 +2342,6 @@ public void resumeTx(GridNearTxLocal tx) throws IgniteCheckedException {
         assert !transactionMap(tx).containsValue(tx) : tx;
         assert !haveSystemTxForThread(Thread.currentThread().getId());
 
-        long threadId = Thread.currentThread().getId();
-
         if (threadMap.putIfAbsent(threadId, tx) != null)
             throw new IgniteCheckedException("Thread already has started a transaction.");
 
@@ -2420,7 +2411,7 @@ private void onTimeout0() {
                     log.debug("Processing node failed event [locNodeId=" + cctx.localNodeId() +
                         ", failedNodeId=" + evtNodeId + ']');
 
-                for (final IgniteInternalTx tx : txs()) {
+                for (final IgniteInternalTx tx : activeTransactions()) {
                     if ((tx.near() && !tx.local()) || (tx.storeWriteThrough() && tx.masterNodeIds().contains(evtNodeId))) {
                         // Invalidate transactions.
                         salvageTx(tx, RECOVERY_FINISH);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyImpl.java
index f25fc36b2b996..628b8138222f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyImpl.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.apache.ignite.transactions.TransactionState;
+import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.transactions.TransactionState.SUSPENDED;
 
@@ -232,6 +233,14 @@ private void leave() {
         }
     }
 
+    /** {@inheritDoc} */
+    @Nullable @Override public String label() {
+        if (async)
+            save(tx.label());
+
+        return tx.label();
+    }
+
     /** {@inheritDoc} */
     @Override public long timeout(long timeout) {
         return tx.timeout(timeout);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyRollbackOnlyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyRollbackOnlyImpl.java
new file mode 100644
index 0000000000000..74735f93ee888
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TransactionProxyRollbackOnlyImpl.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.io.Externalizable;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.lang.IgniteFuture;
+
+/**
+ * Cache transaction proxy which support only rollback or close operations and getters.
+ */
+@SuppressWarnings("unchecked")
+public class TransactionProxyRollbackOnlyImpl<K, V> extends TransactionProxyImpl<K, V> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * Empty constructor required for {@link Externalizable}.
+     */
+    public TransactionProxyRollbackOnlyImpl() {
+        // No-op.
+    }
+
+    /**
+     * @param tx Tx.
+     * @param cctx Cctx.
+     * @param async Async.
+     */
+    public TransactionProxyRollbackOnlyImpl(GridNearTxLocal tx,
+        GridCacheSharedContext<K, V> cctx, boolean async) {
+        super(tx, cctx, async);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean setRollbackOnly() {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void commit() {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteFuture<Void> commitAsync() throws IgniteException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void suspend() throws IgniteException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void resume() throws IgniteException {
+        throw new UnsupportedOperationException();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long timeout(long timeout) {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java
index 173ddbe598da7..f5893c02f77b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java
@@ -52,7 +52,7 @@ public class GridPartitionStateMap extends AbstractMap<Integer, GridDhtPartition
     @Override public Set<Entry<Integer, GridDhtPartitionState>> entrySet() {
         return new AbstractSet<Entry<Integer, GridDhtPartitionState>>() {
             @Override public Iterator<Entry<Integer, GridDhtPartitionState>> iterator() {
-                final int size = states.isEmpty() ? 0 : (states.length() - 1)/ BITS + 1;
+                final int size = states.isEmpty() ? 0 : (states.length() - 1) / BITS + 1;
 
                 return new Iterator<Entry<Integer, GridDhtPartitionState>>() {
                     private int next;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
new file mode 100644
index 0000000000000..89bf274653165
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.tx;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.io.Serializable;
+import java.util.Collection;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.apache.ignite.transactions.TransactionState;
+
+/**
+ */
+public class VisorTxInfo extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private IgniteUuid xid;
+
+    /** */
+    private long duration;
+
+    /** */
+    private TransactionIsolation isolation;
+
+    /** */
+    private TransactionConcurrency concurrency;
+
+    /** */
+    private long timeout;
+
+    /** */
+    private String lb;
+
+    /** */
+    private Collection<UUID> primaryNodes;
+
+    /** */
+    private TransactionState state;
+
+    /** */
+    private int size;
+
+    /**
+     * Default constructor.
+     */
+    public VisorTxInfo() {
+        // No-op.
+    }
+
+    /**
+     * @param xid Xid.
+     * @param duration Duration.
+     * @param isolation Isolation.
+     * @param concurrency Concurrency.
+     * @param timeout Timeout.
+     * @param lb Label.
+     * @param primaryNodes Primary nodes.
+     * @param state State.
+     * @param size Size.
+     */
+    public VisorTxInfo(IgniteUuid xid, long duration, TransactionIsolation isolation,
+        TransactionConcurrency concurrency, long timeout, String lb, Collection<UUID> primaryNodes,
+        TransactionState state, int size) {
+        this.xid = xid;
+        this.duration = duration;
+        this.isolation = isolation;
+        this.concurrency = concurrency;
+        this.timeout = timeout;
+        this.lb = lb;
+        this.primaryNodes = primaryNodes;
+        this.state = state;
+        this.size = size;
+    }
+
+    /** */
+    public IgniteUuid getXid() {
+        return xid;
+    }
+
+    /** */
+    public long getDuration() {
+        return duration;
+    }
+
+    /** */
+    public TransactionIsolation getIsolation() {
+        return isolation;
+    }
+
+    /** */
+    public TransactionConcurrency getConcurrency() {
+        return concurrency;
+    }
+
+    /** */
+    public long getTimeout() {
+        return timeout;
+    }
+
+    /** */
+    public String getLabel() {
+        return lb;
+    }
+
+    /** */
+    public Collection<UUID> getPrimaryNodes() {
+        return primaryNodes;
+    }
+
+    /** */
+    public TransactionState getState() {
+        return state;
+    }
+
+    /** */
+    public int getSize() {
+        return size;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeGridUuid(out, xid);
+        out.writeLong(duration);
+        U.writeEnum(out, isolation);
+        U.writeEnum(out, concurrency);
+        out.writeLong(timeout);
+        U.writeString(out, lb);
+        U.writeCollection(out, primaryNodes);
+        U.writeEnum(out, state);
+        out.writeInt(size);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        xid = U.readGridUuid(in);
+        duration = in.readLong();
+        isolation = TransactionIsolation.fromOrdinal(in.readByte());
+        concurrency = TransactionConcurrency.fromOrdinal(in.readByte());
+        timeout = in.readLong();
+        lb = U.readString(in);
+        primaryNodes = U.readCollection(in);
+        state = TransactionState.fromOrdinal(in.readByte());
+        size = in.readInt();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorTxInfo.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxOperation.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxOperation.java
new file mode 100644
index 0000000000000..4aaab8ecd8f59
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxOperation.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.tx;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Defines possible Visor operation on transactions.
+ */
+public enum VisorTxOperation {
+    /** List matching transactions. */
+    LIST,
+    /** Kill matching transactions. */
+    KILL;
+
+    /** Enumerated values. */
+    private static final VisorTxOperation[] VALS = values();
+
+    /**
+     * Efficiently gets enumerated value from its ordinal.
+     *
+     * @param ord Ordinal value.
+     * @return Enumerated value or {@code null} if ordinal out of range.
+     */
+    @Nullable public static VisorTxOperation fromOrdinal(int ord) {
+        return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxProjection.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxProjection.java
new file mode 100644
index 0000000000000..4c81cc6bff7b4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxProjection.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.tx;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ */
+public enum     VisorTxProjection {
+    /** Server nodes. */
+    SERVER,
+    /** Client nodes. */
+    CLIENT;
+
+    /** Enumerated values. */
+    private static final VisorTxProjection[] VALS = values();
+
+    /**
+     * Efficiently gets enumerated value from its ordinal.
+     *
+     * @param ord Ordinal value.
+     * @return Enumerated value or {@code null} if ordinal out of range.
+     */
+    @Nullable public static VisorTxProjection fromOrdinal(int ord) {
+        return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxSortOrder.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxSortOrder.java
new file mode 100644
index 0000000000000..382cf9152207b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxSortOrder.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.tx;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ */
+public enum VisorTxSortOrder {
+    /** Sort by duration. */
+    DURATION,
+    /** Sort by size. */
+    SIZE;
+
+    /** Enumerated values. */
+    private static final VisorTxSortOrder[] VALS = values();
+
+    /**
+     * Efficiently gets enumerated value from its ordinal.
+     *
+     * @param ord Ordinal value.
+     * @return Enumerated value or {@code null} if ordinal out of range.
+     */
+    @Nullable public static VisorTxSortOrder fromOrdinal(int ord) {
+        return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+    }
+
+    /**
+     * @param name Name.
+     */
+    public static VisorTxSortOrder fromString(String name) {
+        if (DURATION.toString().equals(name))
+            return DURATION;
+
+        if (SIZE.toString().equals(name))
+            return SIZE;
+
+        throw new IllegalArgumentException("Sort order is unknown: " + name);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
new file mode 100644
index 0000000000000..72b174050c72c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.tx;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.transactions.Transaction;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+@GridInternal
+public class VisorTxTask extends VisorMultiNodeTask<VisorTxTaskArg, Map<ClusterNode, VisorTxTaskResult>, VisorTxTaskResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<VisorTxTaskArg, VisorTxTaskResult> job(VisorTxTaskArg arg) {
+        return new VisorTxJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Collection<UUID> jobNodes(VisorTaskArgument<VisorTxTaskArg> arg) {
+        final VisorTxTaskArg taskArg = arg.getArgument();
+
+        if (taskArg.getConsistentIds() != null) {
+            return F.transform(ignite.cluster().forPredicate(new IgnitePredicate<ClusterNode>() {
+                @Override public boolean apply(ClusterNode node) {
+                    return taskArg.getConsistentIds().contains((String)node.consistentId().toString());
+                }
+            }).nodes(), new IgniteClosure<ClusterNode, UUID>() {
+                @Override public UUID apply(ClusterNode node) {
+                    return node.id();
+                }
+            });
+        }
+
+        if (taskArg.getProjection() == VisorTxProjection.SERVER) {
+            return F.transform(ignite.cluster().forServers().nodes(), new IgniteClosure<ClusterNode, UUID>() {
+                @Override public UUID apply(ClusterNode node) {
+                    return node.id();
+                }
+            });
+        }
+
+        if (taskArg.getProjection() == VisorTxProjection.CLIENT) {
+            return F.transform(ignite.cluster().forClients().nodes(), new IgniteClosure<ClusterNode, UUID>() {
+                @Override public UUID apply(ClusterNode node) {
+                    return node.id();
+                }
+            });
+        }
+
+        return F.transform(ignite.cluster().nodes(), new IgniteClosure<ClusterNode, UUID>() {
+            @Override public UUID apply(ClusterNode node) {
+                return node.id();
+            }
+        });
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected Map<ClusterNode, VisorTxTaskResult> reduce0(List<ComputeJobResult> results) throws IgniteException {
+        Map<ClusterNode, VisorTxTaskResult> mapRes = new TreeMap<>();
+
+        for (ComputeJobResult result : results) {
+            VisorTxTaskResult data = result.getData();
+
+            if (data == null || data.getInfos().isEmpty())
+                continue;
+
+            mapRes.put(result.getNode(), data);
+        }
+
+        return mapRes;
+    }
+
+    /**
+     *
+     */
+    private static class VisorTxJob extends VisorJob<VisorTxTaskArg, VisorTxTaskResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        public static final int DEFAULT_LIMIT = 50;
+
+        /**
+         * @param arg Formal job argument.
+         * @param debug Debug flag.
+         */
+        private VisorTxJob(VisorTxTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorTxTaskResult run(@Nullable VisorTxTaskArg arg) throws IgniteException {
+            if (arg == null)
+                return new VisorTxTaskResult(Collections.emptyList());
+
+            Collection<Transaction> transactions = ignite.transactions().localActiveTransactions();
+
+            List<VisorTxInfo> infos = new ArrayList<>();
+
+            int limit = arg.getLimit() == null ? DEFAULT_LIMIT : arg.getLimit();
+
+            Pattern lbMatch = null;
+
+            if (arg.getLabelRegex() != null) {
+                try {
+                    lbMatch = Pattern.compile(arg.getLabelRegex());
+                }
+                catch (PatternSyntaxException ignored) {
+                    // No-op.
+                }
+            }
+
+            for (Transaction transaction : transactions) {
+                GridNearTxLocal locTx = ((TransactionProxyImpl)transaction).tx();
+
+                if (arg.getXid() != null && !locTx.xid().toString().equals(arg.getXid()))
+                    continue;
+
+                if (arg.getState() != null && locTx.state() != arg.getState())
+                    continue;
+
+                long duration = U.currentTimeMillis() - transaction.startTime();
+
+                if (arg.getMinDuration() != null &&
+                    duration < arg.getMinDuration())
+                    continue;
+
+                if (arg.getMinSize() != null && locTx.size() < arg.getMinSize())
+                    continue;
+
+                if (lbMatch != null && (locTx.label() == null || !lbMatch.matcher(locTx.label()).matches()))
+                    continue;
+
+                Collection<UUID> mappings = new ArrayList<>();
+
+                int size = 0;
+
+                if (locTx.mappings() != null) {
+                    for (GridDistributedTxMapping mapping : locTx.mappings().mappings()) {
+                        mappings.add(mapping.primary().id());
+
+                        size += mapping.entries().size(); // Entries are not synchronized so no visibility guaranties.
+                    }
+                }
+
+                infos.add(new VisorTxInfo(locTx.xid(), duration, locTx.isolation(), locTx.concurrency(),
+                    locTx.timeout(), locTx.label(), mappings, locTx.state(), size));
+
+                if (arg.getOperation() == VisorTxOperation.KILL)
+                    locTx.rollbackAsync();
+
+                if (infos.size() == limit)
+                    break;
+            }
+
+            Comparator<VisorTxInfo> comp = TxDurationComparator.INSTANCE;
+
+            if (arg.getSortOrder() != null) {
+                switch (arg.getSortOrder()) {
+                    case DURATION:
+                        comp = TxDurationComparator.INSTANCE;
+
+                        break;
+
+                    case SIZE:
+                        comp = TxSizeComparator.INSTANCE;
+
+                        break;
+
+                    default:
+                }
+            }
+
+            Collections.sort(infos, comp);
+
+            return new VisorTxTaskResult(infos);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TxDurationComparator implements Comparator<VisorTxInfo> {
+        /** Instance. */
+        public static final TxDurationComparator INSTANCE = new TxDurationComparator();
+
+        /** {@inheritDoc} */
+        @Override public int compare(VisorTxInfo o1, VisorTxInfo o2) {
+            return Long.compare(o2.getDuration(), o1.getDuration());
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TxSizeComparator implements Comparator<VisorTxInfo> {
+        /** Instance. */
+        public static final TxSizeComparator INSTANCE = new TxSizeComparator();
+
+        /** {@inheritDoc} */
+        @Override public int compare(VisorTxInfo o1, VisorTxInfo o2) {
+            return Long.compare(o2.getSize(), o1.getSize());
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskArg.java
new file mode 100644
index 0000000000000..80bf8b537a8e5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskArg.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.tx;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineTask;
+import org.apache.ignite.transactions.TransactionState;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Argument for {@link VisorTxTask}.
+ */
+public class VisorTxTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private VisorTxOperation op;
+
+    /** */
+    private @Nullable Integer limit;
+
+    /** */
+    private @Nullable Long minDuration;
+
+    /** */
+    private @Nullable Integer minSize;
+
+    /** */
+    private @Nullable TransactionState state;
+
+    /** */
+    private @Nullable VisorTxProjection proj;
+
+    /** */
+    private @Nullable List<String> consistentIds;
+
+    /** */
+    private @Nullable String xid;
+
+    /** */
+    private @Nullable String lbRegex;
+
+    /** */
+    private @Nullable VisorTxSortOrder sortOrder;
+
+    /**
+     * Default constructor.
+     */
+    public VisorTxTaskArg() {
+        // No-op.
+    }
+
+    /**
+     * @param limit Limit to collect.
+     * @param minDuration Min duration.
+     * @param minSize Min size.
+     * @param state State.
+     * @param proj Projection.
+     * @param consistentIds Consistent ids for NODES projection.
+     * @param xid Xid.
+     * @param lbRegex Label regex.
+     * @param sortOrder Sort order.
+     */
+    public VisorTxTaskArg(VisorTxOperation op, @Nullable Integer limit, @Nullable Long minDuration, @Nullable Integer minSize,
+        @Nullable TransactionState state, @Nullable VisorTxProjection proj, @Nullable List<String> consistentIds,
+        @Nullable String xid, @Nullable String lbRegex, @Nullable VisorTxSortOrder sortOrder) {
+        this.op = op;
+        this.limit = limit;
+        this.minDuration = minDuration;
+        this.minSize = minSize;
+        this.state = state;
+        this.proj = proj;
+        this.consistentIds = consistentIds;
+        this.lbRegex = lbRegex;
+        this.xid = xid;
+        this.sortOrder = sortOrder;
+    }
+
+    /** */
+    public VisorTxOperation getOperation() {
+        return op;
+    }
+
+    /** */
+    @Nullable public Integer getLimit() {
+        return limit;
+    }
+
+    /** */
+    @Nullable public Long getMinDuration() {
+        return minDuration;
+    }
+
+    /** */
+    @Nullable public Integer getMinSize() {
+        return minSize;
+    }
+
+    /** */
+    @Nullable public TransactionState getState() {
+        return state;
+    }
+
+    /** */
+    public VisorTxProjection getProjection() {
+        return proj;
+    }
+
+    /** */
+    @Nullable public List<String> getConsistentIds() {
+        return consistentIds;
+    }
+
+    /** */
+    @Nullable public String getLabelRegex() {
+        return lbRegex;
+    }
+
+    /** */
+    @Nullable public String getXid() {
+        return xid;
+    }
+
+    /** */
+    @Nullable public VisorTxSortOrder getSortOrder() {
+        return sortOrder;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeEnum(out, op);
+        out.writeInt(limit == null ? -1 : limit);
+        out.writeLong(minDuration == null ? -1 : minDuration);
+        out.writeInt(minSize == null ? -1 : minSize);
+        U.writeEnum(out, state);
+        U.writeEnum(out, proj);
+        U.writeCollection(out, consistentIds);
+        out.writeUTF(lbRegex == null ? "" : lbRegex);
+        out.writeUTF(xid == null ? "" : xid);
+        U.writeEnum(out, sortOrder);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        op = VisorTxOperation.fromOrdinal(in.readByte());
+        limit = fixNull(in.readInt());
+        minDuration = fixNull(in.readLong());
+        minSize = fixNull(in.readInt());
+        state = TransactionState.fromOrdinal(in.readByte());
+        proj = VisorTxProjection.fromOrdinal(in.readByte());
+        consistentIds = U.readList(in);
+        lbRegex = fixNull(in.readUTF());
+        xid = fixNull(in.readUTF());
+        sortOrder = VisorTxSortOrder.fromOrdinal(in.readByte());
+    }
+
+    /**
+     * @param val Value.
+     */
+    private Integer fixNull(int val) {
+        return val == -1 ? null : val;
+    }
+
+    /**
+     * @param val Value.
+     */
+    private Long fixNull(long val) {
+        return val == -1 ? null : val;
+    }
+
+    /**
+     * @param val Value.
+     */
+    private String fixNull(String val) {
+        return "".equals(val) ? null : val;
+    }
+
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorTxTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskResult.java
new file mode 100644
index 0000000000000..4864a77ab2ad4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTaskResult.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.tx;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineNode;
+import org.apache.ignite.internal.visor.baseline.VisorBaselineTask;
+
+/**
+ * Result for {@link VisorTxTask}.
+ */
+public class VisorTxTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private List<VisorTxInfo> infos;
+
+    /**
+     * Default constructor.
+     */
+    public VisorTxTaskResult() {
+        // No-op.
+    }
+
+    /**
+     * Constructor.
+     *
+     * @param infos Infos.
+     */
+    public VisorTxTaskResult(List<VisorTxInfo> infos) {
+        this.infos = infos;
+    }
+
+    /** */
+    public List<VisorTxInfo> getInfos() {
+        return infos;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeCollection(out, infos);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        infos = U.readList(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorTxTaskResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionsMXBean.java
new file mode 100644
index 0000000000000..2a125fa8e58b2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/TransactionsMXBean.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mxbean;
+
+import org.apache.ignite.configuration.TransactionConfiguration;
+
+/**
+ * Transactions MXBean interface.
+ */
+@MXBeanDescription("MBean that provides access to Ignite transactions.")
+public interface TransactionsMXBean {
+    /**
+     * @param minDuration Minimum duration.
+     * @param minSize Minimum size.
+     * @param prj Projection.
+     * @param consistentIds Consistent ids.
+     * @param xid Xid.
+     * @param lbRegex Label regex.
+     * @param limit Limit.
+     * @param order Order.
+     * @param detailed Detailed.
+     * @param kill Kill.
+     */
+    @MXBeanDescription("Returns or kills transactions matching the filter conditions.")
+    @MXBeanParametersNames(
+        {
+            "minDuration",
+            "minSize",
+            "prj",
+            "consistentIds",
+            "xid",
+            "lbRegex",
+            "limit",
+            "order",
+            "detailed",
+            "kill"
+        }
+    )
+    @MXBeanParametersDescriptions(
+        {
+            "Minimum duration (seconds).",
+            "Minimum size.",
+            "Projection (servers|clients).",
+            "Consistent ids (separated by comma).",
+            "Transaction XID.",
+            "Label regexp.",
+            "Limit a number of transactions collected on each node.",
+            "Order by DURATION|SIZE.",
+            "Show detailed description, otherwise only count.",
+            "Kill matching transactions (be careful)."
+        }
+    )
+    public String getActiveTransactions(Long minDuration, Integer minSize, String prj,
+        String consistentIds, String xid, String lbRegex, Integer limit, String order, boolean detailed, boolean kill);
+
+    /**
+     * Gets transaction timeout on partition map exchange.
+     * <p>
+     * @see TransactionConfiguration#getTxTimeoutOnPartitionMapExchange
+     *
+     * @return Transaction timeout on partition map exchange in milliseconds.
+     */
+    @MXBeanDescription("Returns transaction timeout on partition map exchange in milliseconds.")
+    public long getTxTimeoutOnPartitionMapExchange();
+
+    /**
+     * Sets transaction timeout on partition map exchange.
+     * <p>
+     * If not set, default value is {@link TransactionConfiguration#TX_TIMEOUT_ON_PARTITION_MAP_EXCHANGE} which means
+     * transactions will never be rolled back on partition map exchange.
+     * <p>
+     * @see TransactionConfiguration#setTxTimeoutOnPartitionMapExchange
+     *
+     * @param timeout Transaction timeout on partition map exchange in milliseconds.
+     */
+    @MXBeanDescription("Sets transaction timeout on partition map exchange in milliseconds.")
+    @MXBeanParametersNames(
+        "timeout"
+    )
+    @MXBeanParametersDescriptions(
+        "Transaction timeout on partition map exchange in milliseconds."
+    )
+    public void setTxTimeoutOnPartitionMapExchange(long timeout);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java b/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java
index a1b4d78fc6444..b0f90c1bb05f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java
+++ b/modules/core/src/main/java/org/apache/ignite/transactions/Transaction.java
@@ -24,6 +24,7 @@
 import org.apache.ignite.lang.IgniteAsyncSupported;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Ignite cache transaction. Cache transactions have a default 2PC (two-phase-commit) behavior and
@@ -259,6 +260,7 @@ public interface Transaction extends AutoCloseable, IgniteAsyncSupport {
 
     /**
      * Rolls back this transaction.
+     * Note, that it's allowed to roll back transaction from any thread at any time.
      *
      * @throws IgniteException If rollback failed.
      */
@@ -267,6 +269,7 @@ public interface Transaction extends AutoCloseable, IgniteAsyncSupport {
 
     /**
      * Asynchronously rolls back this transaction.
+     * Note, that it's allowed to roll back transaction from any thread at any time.
      *
      * @return a Future representing pending completion of the rollback.
      * @throws IgniteException If rollback failed.
@@ -274,16 +277,25 @@ public interface Transaction extends AutoCloseable, IgniteAsyncSupport {
     public IgniteFuture<Void> rollbackAsync() throws IgniteException;
 
     /**
-     * Resume transaction if it was previously suspended. <strong>Supported only for optimistic transactions.</strong>
+     * Resume a transaction if it was previously suspended. <strong>Supported only for optimistic transactions.</strong>
      *
      * @throws IgniteException If resume failed.
      */
     public void resume() throws IgniteException;
 
     /**
-     * Suspends transaction. It could be resumed later. <strong>Supported only for optimistic transactions.</strong>
+     * Suspends a transaction. It could be resumed later. <strong>Supported only for optimistic transactions.</strong>
      *
      * @throws IgniteException If suspension failed.
      */
     public void suspend() throws IgniteException;
-}
\ No newline at end of file
+
+    /**
+     * Returns transaction's label.
+     * <p>
+     * Use {@link IgniteTransactions#withLabel(java.lang.String)} to assign a label to a newly created transaction.
+     *
+     * @return Label.
+     */
+    public @Nullable String label();
+}
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 6dc3d85d592b1..d939b02018bbb 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -2069,6 +2069,13 @@ org.apache.ignite.internal.visor.util.VisorClusterGroupEmptyException
 org.apache.ignite.internal.visor.util.VisorEventMapper
 org.apache.ignite.internal.visor.util.VisorExceptionWrapper
 org.apache.ignite.internal.visor.util.VisorTaskUtils$4
+org.apache.ignite.internal.visor.tx.VisorTxInfo
+org.apache.ignite.internal.visor.tx.VisorTxOperation
+org.apache.ignite.internal.visor.tx.VisorTxProjection
+org.apache.ignite.internal.visor.tx.VisorTxSortOrder
+org.apache.ignite.internal.visor.tx.VisorTxTask
+org.apache.ignite.internal.visor.tx.VisorTxTaskArg
+org.apache.ignite.internal.visor.tx.VisorTxTaskResult
 org.apache.ignite.internal.websession.WebSessionAttributeProcessor
 org.apache.ignite.internal.websession.WebSessionEntity
 org.apache.ignite.lang.IgniteBiClosure
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
index cf4f059389eab..5d12d9aea9ff7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
@@ -105,7 +105,7 @@ boolean record = (recordClasses != null && recordClasses.contains(msg0.getClass(
                 }
 
                 if (block) {
-                    ignite.log().info("Block message [node=" + node.id() +
+                    ignite.log().info("Block message [node=" + node.id() + ", order=" + node.order() +
                         ", msg=" + ioMsg.message() + ']');
 
                     blockedMsgs.add(new T2<>(node, ioMsg));
@@ -273,6 +273,7 @@ public void stopBlock(boolean sndMsgs) {
                 for (T2<ClusterNode, GridIoMessage> msg : blockedMsgs) {
                     try {
                         ignite.log().info("Send blocked message [node=" + msg.get1().id() +
+                            ", order=" + msg.get1().order() +
                             ", msg=" + msg.get2().message() + ']');
 
                         super.sendMessage(msg.get1(), msg.get2());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java
index 80b0dc066300a..91bd9bcf98271 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TransactionMetricsMxBeanImplTest.java
@@ -100,7 +100,7 @@ public void testTxMetric() throws Exception {
 
         awaitPartitionMapExchange();
 
-        TransactionMetricsMxBean txMXBean = txMXBean(0);
+        TransactionMetricsMxBean txMXBean = txMetricsMXBean(0);
 
         final IgniteCache<Integer, String> cache = ignite.cache(DEFAULT_CACHE_NAME);
 
@@ -176,7 +176,7 @@ public void testNearTxInfo() throws Exception {
         IgniteEx primaryNode2 = startGrid(1);
         IgniteEx nearNode = startGrid(2);
 
-        TransactionMetricsMxBean txMXBeanBackup = txMXBean(2);
+        TransactionMetricsMxBean txMXBeanBackup = txMetricsMXBean(2);
 
         awaitPartitionMapExchange();
 
@@ -285,10 +285,10 @@ private TxThread(final Ignite ignite, final int key1, final int key2) {
     /**
      *
      */
-    private TransactionMetricsMxBean txMXBean(int igniteInt) throws Exception {
+    private TransactionMetricsMxBean txMetricsMXBean(int igniteInt) throws Exception {
         ObjectName mbeanName = U.makeMBeanName(
             getTestIgniteInstanceName(igniteInt),
-            "Transactions",
+            "TransactionMetrics",
             TransactionMetricsMxBeanImpl.class.getSimpleName()
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TransactionsMXBeanImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/TransactionsMXBeanImplTest.java
new file mode 100644
index 0000000000000..d358c72e79917
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TransactionsMXBeanImplTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.lang.management.ManagementFactory;
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.ObjectName;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.mxbean.TransactionsMXBean;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class TransactionsMXBeanImplTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(name);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        final CacheConfiguration cCfg = new CacheConfiguration()
+            .setName(DEFAULT_CACHE_NAME)
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1)
+            .setAtomicityMode(TRANSACTIONAL)
+            .setRebalanceMode(CacheRebalanceMode.ASYNC)
+            .setWriteSynchronizationMode(FULL_SYNC);
+
+        cfg.setCacheConfiguration(cCfg);
+
+        return cfg;
+    }
+
+    /**
+     *
+     */
+    public void testBasic() throws Exception {
+        IgniteEx ignite = startGrid(0);
+
+        TransactionsMXBean bean = txMXBean(0);
+
+        ignite.transactions().txStart();
+
+        ignite.cache(DEFAULT_CACHE_NAME).put(0, 0);
+
+        String res = bean.getActiveTransactions(null, null, null, null, null, null, null, null, false, false);
+
+        assertEquals("1", res);
+
+        res = bean.getActiveTransactions(null, null, null, null, null, null, null, null, true, false);
+
+        assertTrue(res.indexOf("Tx:") > 0);
+
+        res = bean.getActiveTransactions(null, null, null, null, null, null, null, null, false, true);
+
+        assertEquals("1", res);
+
+        doSleep(500);
+
+        res = bean.getActiveTransactions(null, null, null, null, null, null, null, null, false, false);
+
+        assertEquals("0", res);
+    }
+
+    /**
+     *
+     */
+    private TransactionsMXBean txMXBean(int igniteInt) throws Exception {
+        ObjectName mbeanName = U.makeMBeanName(getTestIgniteInstanceName(igniteInt), "Transactions",
+            TransactionsMXBeanImpl.class.getSimpleName());
+
+        MBeanServer mbeanSrv = ManagementFactory.getPlatformMBeanServer();
+
+        if (!mbeanSrv.isRegistered(mbeanName))
+            fail("MBean is not registered: " + mbeanName.getCanonicalName());
+
+        return MBeanServerInvocationHandler.newProxyInstance(mbeanSrv, mbeanName, TransactionsMXBean.class, true);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index cfbf1a67d7f25..049500abc6853 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -19,7 +19,11 @@
 
 package org.apache.ignite.internal.commandline;
 
+import java.util.Arrays;
 import junit.framework.TestCase;
+import org.apache.ignite.internal.visor.tx.VisorTxProjection;
+import org.apache.ignite.internal.visor.tx.VisorTxSortOrder;
+import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
 
 import static java.util.Arrays.asList;
 import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_HOST;
@@ -109,4 +113,100 @@ public void testHostAndPort() {
             }
         }
     }
+
+    /**
+     * test parsing dump transaction arguments
+     */
+    @SuppressWarnings("Null")
+    public void testTransactionArguments() {
+        CommandHandler hnd = new CommandHandler();
+        Arguments args;
+
+        args = hnd.parseAndValidate(asList("--tx"));
+
+        try {
+            hnd.parseAndValidate(asList("--tx", "minDuration"));
+
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException ignored) {
+        }
+
+        try {
+            hnd.parseAndValidate(asList("--tx", "minDuration", "-1"));
+
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException ignored) {
+        }
+
+        try {
+            hnd.parseAndValidate(asList("--tx", "minSize"));
+
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException ignored) {
+        }
+
+        try {
+            hnd.parseAndValidate(asList("--tx", "minSize", "-1"));
+
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException ignored) {
+        }
+
+        try {
+            hnd.parseAndValidate(asList("--tx", "label"));
+
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException ignored) {
+        }
+
+        try {
+            hnd.parseAndValidate(asList("--tx", "label", "tx123["));
+
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException ignored) {
+        }
+
+        try {
+            hnd.parseAndValidate(asList("--tx", "servers", "nodes", "1,2,3"));
+
+            fail("Expected exception");
+        }
+        catch (IllegalArgumentException ignored) {
+        }
+
+        args = hnd.parseAndValidate(asList("--tx", "minDuration", "120", "minSize", "10", "limit", "100", "order", "SIZE",
+            "servers"));
+
+        VisorTxTaskArg arg = args.transactionArguments();
+
+        assertEquals(Long.valueOf(120 * 1000L), arg.getMinDuration());
+        assertEquals(Integer.valueOf(10), arg.getMinSize());
+        assertEquals(Integer.valueOf(100), arg.getLimit());
+        assertEquals(VisorTxSortOrder.SIZE, arg.getSortOrder());
+        assertEquals(VisorTxProjection.SERVER, arg.getProjection());
+
+        args = hnd.parseAndValidate(asList("--tx", "minDuration", "130", "minSize", "1", "limit", "60", "order", "DURATION",
+            "clients"));
+
+        arg = args.transactionArguments();
+
+        assertEquals(Long.valueOf(130 * 1000L), arg.getMinDuration());
+        assertEquals(Integer.valueOf(1), arg.getMinSize());
+        assertEquals(Integer.valueOf(60), arg.getLimit());
+        assertEquals(VisorTxSortOrder.DURATION, arg.getSortOrder());
+        assertEquals(VisorTxProjection.CLIENT, arg.getProjection());
+
+        args = hnd.parseAndValidate(asList("--tx", "nodes", "1,2,3"));
+
+        arg = args.transactionArguments();
+
+        assertNull(arg.getProjection());
+        assertEquals(Arrays.asList("1", "2", "3"), arg.getConsistentIds());
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSeltTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSeltTest.java
index dac64d9cd596d..4aa693c29c1b3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSeltTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGetEntryPessimisticRepeatableReadSeltTest.java
@@ -33,4 +33,8 @@ public class CacheGetEntryPessimisticRepeatableReadSeltTest extends CacheGetEntr
     @Override protected TransactionIsolation isolation() {
         return TransactionIsolation.REPEATABLE_READ;
     }
-}
\ No newline at end of file
+
+    @Override public void testReplicatedTransactional() throws Exception {
+        super.testReplicatedTransactional();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java
new file mode 100644
index 0000000000000..3152349a30025
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.lang.management.ManagementFactory;
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.ObjectName;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.TransactionConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.TransactionsMXBeanImpl;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.mxbean.TransactionsMXBean;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class SetTxTimeoutOnPartitionMapExchangeTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** Wait condition timeout. */
+    private static final long WAIT_CONDITION_TIMEOUT = 10_000L;
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        return cfg;
+    }
+
+    /**
+     *
+     */
+    public void testDefaultTxTimeoutOnPartitionMapExchange() throws Exception {
+        IgniteEx ig1 = startGrid(1);
+        IgniteEx ig2 = startGrid(2);
+
+        TransactionConfiguration txCfg1 = ig1.configuration().getTransactionConfiguration();
+        TransactionConfiguration txCfg2 = ig2.configuration().getTransactionConfiguration();
+
+        final long expDfltTimeout = TransactionConfiguration.TX_TIMEOUT_ON_PARTITION_MAP_EXCHANGE;
+
+        assertEquals(expDfltTimeout, txCfg1.getTxTimeoutOnPartitionMapExchange());
+        assertEquals(expDfltTimeout, txCfg2.getTxTimeoutOnPartitionMapExchange());
+    }
+
+    /**
+     *
+     */
+    public void testJmxSetTxTimeoutOnPartitionMapExchange() throws Exception {
+        startGrid(1);
+        startGrid(2);
+
+        TransactionsMXBean mxBean1 = txMXBean(1);
+        TransactionsMXBean mxBean2 = txMXBean(2);
+
+        final long expTimeout1 = 20_000L;
+        final long expTimeout2 = 30_000L;
+
+        mxBean1.setTxTimeoutOnPartitionMapExchange(expTimeout1);
+        assertTxTimeoutOnPartitionMapExchange(expTimeout1);
+        assertEquals(expTimeout1, mxBean1.getTxTimeoutOnPartitionMapExchange());
+
+        mxBean2.setTxTimeoutOnPartitionMapExchange(expTimeout2);
+        assertTxTimeoutOnPartitionMapExchange(expTimeout2);
+        assertEquals(expTimeout2, mxBean2.getTxTimeoutOnPartitionMapExchange());
+    }
+
+    /**
+     *
+     */
+    public void testClusterSetTxTimeoutOnPartitionMapExchange() throws Exception {
+        Ignite ig1 = startGrid(1);
+        Ignite ig2 = startGrid(2);
+
+        final long expTimeout1 = 20_000L;
+        final long expTimeout2 = 30_000L;
+
+        ig1.cluster().setTxTimeoutOnPartitionMapExchange(expTimeout1);
+        assertTxTimeoutOnPartitionMapExchange(expTimeout1);
+
+        ig2.cluster().setTxTimeoutOnPartitionMapExchange(expTimeout2);
+        assertTxTimeoutOnPartitionMapExchange(expTimeout2);
+    }
+
+    /**
+     *
+     */
+    private TransactionsMXBean txMXBean(int igniteInt) throws Exception {
+        ObjectName mbeanName = U.makeMBeanName(getTestIgniteInstanceName(igniteInt), "Transactions",
+            TransactionsMXBeanImpl.class.getSimpleName());
+
+        MBeanServer mbeanSrv = ManagementFactory.getPlatformMBeanServer();
+
+        if (!mbeanSrv.isRegistered(mbeanName))
+            fail("MBean is not registered: " + mbeanName.getCanonicalName());
+
+        return MBeanServerInvocationHandler.newProxyInstance(mbeanSrv, mbeanName, TransactionsMXBean.class, true);
+    }
+
+    /**
+     * Checking the transaction timeout on all grids.
+     *
+     * @param expTimeout Expected timeout.
+     * @throws IgniteInterruptedCheckedException If failed.
+     */
+    private void assertTxTimeoutOnPartitionMapExchange(final long expTimeout)
+        throws IgniteInterruptedCheckedException {
+
+        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                for (Ignite ignite : G.allGrids()) {
+                    long actualTimeout = ignite.configuration()
+                        .getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange();
+
+                    if (actualTimeout != expTimeout) {
+                        log.warning(String.format(
+                            "Wrong transaction timeout on partition map exchange [grid=%s, timeout=%d, expected=%d]",
+                            ignite.name(), actualTimeout, expTimeout));
+
+                        return false;
+                    }
+                }
+
+                return true;
+
+            }
+        }, WAIT_CONDITION_TIMEOUT));
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index e4560478daae2..787095e605cf1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -219,11 +219,6 @@ protected AffinityFunction affinityFunction(@Nullable Integer parts) {
             parts == null ? RendezvousAffinityFunction.DFLT_PARTITION_COUNT : parts);
     }
 
-    @Override
-    protected void beforeTest() throws Exception {
-        super.beforeTest();
-    }
-
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         try {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsNearTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsNearTest.java
new file mode 100644
index 0000000000000..fe8b1fb572d94
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsNearTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import org.apache.ignite.configuration.NearCacheConfiguration;
+
+/**
+ * Test correctness of rollback a transaction with timeout during the grid stop.
+ */
+public class IgniteTxRemoveTimeoutObjectsNearTest extends IgniteTxRemoveTimeoutObjectsTest {
+    /** {@inheritDoc} */
+    @Override protected NearCacheConfiguration nearConfiguration() {
+        return new NearCacheConfiguration();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java
index 752d854ba1b25..571362143ea5f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteTxRemoveTimeoutObjectsTest.java
@@ -22,6 +22,7 @@
 import java.util.Set;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
@@ -205,4 +206,9 @@ private Set<GridTimeoutObject> getTimeoutObjects(IgniteEx igniteEx) {
 
         return GridTestUtils.getFieldValue(timeout, timeout.getClass(), "timeoutObjs");
     }
+
+    /** {@inheritDoc} */
+    @Override protected NearCacheConfiguration nearConfiguration() {
+        return null;
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePrimaryNodeFailureRecoveryAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePrimaryNodeFailureRecoveryAbstractTest.java
index d147d31b1b08f..00f972921eb1b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePrimaryNodeFailureRecoveryAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCachePrimaryNodeFailureRecoveryAbstractTest.java
@@ -481,7 +481,7 @@ private void waitPrepared(Ignite ignite) throws Exception {
             @Override public boolean apply() {
                 GridDhtTxLocal locTx = null;
 
-                for (IgniteInternalTx tx : tm.txs()) {
+                for (IgniteInternalTx tx : tm.activeTransactions()) {
                     if (tx instanceof GridDhtTxLocal) {
                         assertNull("Only one tx is expected.", locTx);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheTxRecoveryRollbackTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheTxRecoveryRollbackTest.java
index b644cdaf028df..11c4c67f415c8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheTxRecoveryRollbackTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheTxRecoveryRollbackTest.java
@@ -98,7 +98,7 @@ public class IgniteCacheTxRecoveryRollbackTest extends GridCommonAbstractTest {
     @Override protected void afterTest() throws Exception {
         try {
             for (Ignite node : G.allGrids()) {
-                Collection<IgniteInternalTx> txs = ((IgniteKernal)node).context().cache().context().tm().txs();
+                Collection<IgniteInternalTx> txs = ((IgniteKernal)node).context().cache().context().tm().activeTransactions();
 
                 assertTrue("Unfinished txs [node=" + node.name() + ", txs=" + txs + ']', txs.isEmpty());
             }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxMultiNodeSelfTest.java
index 0ca10f89830f5..07ee991ba17b8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCacheNearTxMultiNodeSelfTest.java
@@ -242,7 +242,7 @@ private void testReadersUpdate(TransactionConcurrency concurrency, TransactionIs
      */
     @SuppressWarnings( {"unchecked"})
     private void checkTm(Ignite g, IgniteTxManager tm) {
-        Collection<IgniteInternalTx> txs = tm.txs();
+        Collection<IgniteInternalTx> txs = tm.activeTransactions();
 
         info(">>> Number of transactions in the set [size=" + txs.size() +
             ", nodeId=" + g.cluster().localNode().id() + ']');
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java
index 256e8da8726e9..2e4ad92351a30 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/GridCachePartitionedTxSalvageSelfTest.java
@@ -181,8 +181,8 @@ private void checkSalvageBeforeTimeout(TransactionConcurrency mode, boolean prep
         List<Integer> dhtSizes = new ArrayList<>(GRID_CNT - 1);
 
         for (int i = 1; i < GRID_CNT; i++) {
-            nearSizes.add(near(i).context().tm().txs().size());
-            dhtSizes.add(dht(i).context().tm().txs().size());
+            nearSizes.add(near(i).context().tm().activeTransactions().size());
+            dhtSizes.add(dht(i).context().tm().activeTransactions().size());
         }
 
         stopNodeAndSleep(SALVAGE_TIMEOUT - DELTA_BEFORE);
@@ -247,7 +247,7 @@ private void stopNodeAndSleep(long timeout) throws Exception {
      * @param ctx Cache context.
      */
     private void checkTxsEmpty(GridCacheContext ctx) {
-        Collection txs = ctx.tm().txs();
+        Collection txs = ctx.tm().activeTransactions();
 
         assert txs.isEmpty() : "Not all transactions were salvaged: " + txs;
     }
@@ -259,7 +259,7 @@ private void checkTxsEmpty(GridCacheContext ctx) {
      * @param exp Expected amount of transactions.
      */
     private void checkTxsNotEmpty(GridCacheContext ctx, int exp) {
-        int size = ctx.tm().txs().size();
+        int size = ctx.tm().activeTransactions().size();
 
         assertEquals("Some transactions were salvaged unexpectedly", exp, size);
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxMultiThreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxMultiThreadedSelfTest.java
index 9e46b40a82104..388a7bfb78958 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxMultiThreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/local/GridCacheLocalTxMultiThreadedSelfTest.java
@@ -73,7 +73,7 @@ public class GridCacheLocalTxMultiThreadedSelfTest extends IgniteTxMultiThreaded
 
     /** {@inheritDoc} */
     @Override protected int threadCount() {
-        return 8;
+        return 2;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxLabelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxLabelTest.java
new file mode 100644
index 0000000000000..d89ba0b6554ec
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxLabelTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
+import org.apache.ignite.transactions.Transaction;
+
+/**
+ * Tests transaction labels.
+ */
+public class TxLabelTest extends GridCacheAbstractSelfTest {
+    /**
+     * Tests transaction labels.
+     */
+    public void testLabel() {
+        testLabel0(grid(0), "lbl0");
+        testLabel0(grid(0), "lbl1");
+
+        try {
+            testLabel0(grid(0), null);
+
+            fail();
+        }
+        catch (Exception e) {
+            // Expected.
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @param lbl Label.
+     */
+    private void testLabel0(Ignite ignite, String lbl) {
+        try(Transaction tx = ignite.transactions().withLabel(lbl).txStart()) {
+            assertEquals(lbl, tx.label());
+
+            ignite.cache(DEFAULT_CACHE_NAME).put(0, 0);
+
+            tx.commit();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 1;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxMultiCacheAsyncOpsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxMultiCacheAsyncOpsTest.java
new file mode 100644
index 0000000000000..261af7049fa7d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxMultiCacheAsyncOpsTest.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class TxMultiCacheAsyncOpsTest extends GridCommonAbstractTest {
+    /** Grid count. */
+    public static final int GRID_COUNT = 3;
+
+    /** Caches count. */
+    public static final int CACHES_CNT = 3;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setCacheConfiguration(cacheConfigurations());
+
+        return cfg;
+    }
+
+    /** */
+    private CacheConfiguration[] cacheConfigurations() {
+        return IntStream.range(0, CACHES_CNT).mapToObj(
+            this::cacheConfiguration).collect(Collectors.toList()).toArray(new CacheConfiguration[CACHES_CNT]);
+    }
+
+    /**
+     * @param idx Index.
+     */
+    private CacheConfiguration cacheConfiguration(int idx) {
+        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME + idx);
+
+        ccfg.setAtomicityMode(TRANSACTIONAL);
+        ccfg.setBackups(2);
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+        ccfg.setOnheapCacheEnabled(false);
+
+        return ccfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGridsMultiThreaded(GRID_COUNT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        stopAllGrids();
+    }
+
+    /**
+     *
+     */
+    public void testCommitAfterAsyncPut() {
+        CacheConfiguration[] caches = cacheConfigurations();
+
+        try {
+            for (int i = 0; i < caches.length; i++)
+                grid(0).cache(caches[i].getName()).put(1, i + 1);
+
+            try (Transaction tx = grid(0).transactions().txStart()) {
+                for (int i = 0; i < caches.length; i++)
+                    grid(0).cache(caches[i].getName()).putAsync(1, (i + 1) * 10);
+
+                tx.commit();
+            }
+            catch (Exception e) {
+                System.out.println();
+            }
+
+            for (int i = 0; i < caches.length; i++)
+                assertEquals((i + 1) * 10, grid(0).cache(caches[i].getName()).get(1));
+        }
+        finally {
+            for (int i = 0; i < caches.length; i++)
+                grid(0).cache(caches[i].getName()).removeAll();
+        }
+    }
+
+    /**
+     *
+     */
+    public void testCommitAfterAsyncGet() {
+        CacheConfiguration[] caches = cacheConfigurations();
+
+        try {
+            for (int i = 0; i < caches.length; i++)
+                grid(0).cache(caches[i].getName()).put(1, i + 1);
+
+            List<IgniteFuture> futs = new ArrayList<>();
+
+            try (Transaction tx = grid(0).transactions().txStart()) {
+                for (int i = 0; i < caches.length; i++)
+                    futs.add(grid(0).cache(caches[i].getName()).getAsync(1));
+
+                tx.commit();
+            }
+
+            for (int i = 0; i < futs.size(); i++)
+                assertEquals(i + 1, futs.get(i).get());
+        }
+        finally {
+            for (int i = 0; i < caches.length; i++)
+                grid(0).cache(caches[i].getName()).removeAll();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticPrepareOnUnstableTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticPrepareOnUnstableTopologyTest.java
new file mode 100644
index 0000000000000..cb371cf2cc583
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOptimisticPrepareOnUnstableTopologyTest.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.TreeMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.PRIMARY_SYNC;
+
+/**
+ * Tests optimistic prepare on unstable topology.
+ */
+public class TxOptimisticPrepareOnUnstableTopologyTest extends GridCommonAbstractTest {
+    /** */
+    public static final String CACHE_NAME = "part_cache";
+
+    /** IP finder. */
+    private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private volatile boolean run = true;
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        assertEquals(0, G.allGrids().size());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(ipFinder);
+
+        c.setDiscoverySpi(disco);
+
+        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+        ccfg.setName(CACHE_NAME);
+        ccfg.setAtomicityMode(TRANSACTIONAL);
+        ccfg.setWriteSynchronizationMode(PRIMARY_SYNC);
+        ccfg.setBackups(2);
+        ccfg.setCacheMode(PARTITIONED);
+
+        c.setCacheConfiguration(ccfg);
+
+        c.setClientMode(client);
+
+        return c;
+    }
+
+    /**
+     *
+     */
+    public void testPrepareOnUnstableTopology() throws Exception {
+        for (TransactionIsolation isolation : TransactionIsolation.values()) {
+            doPrepareOnUnstableTopology(4, false, isolation, 0);
+            doPrepareOnUnstableTopology(4, true, isolation, 0);
+            doPrepareOnUnstableTopology(4, false, isolation, TimeUnit.DAYS.toMillis(1));
+            doPrepareOnUnstableTopology(4, true, isolation, TimeUnit.DAYS.toMillis(1));
+        }
+    }
+
+    /**
+     * @param keys Keys.
+     * @param testClient Test client.
+     * @param isolation Isolation.
+     * @param timeout Timeout.
+     */
+    private void doPrepareOnUnstableTopology(int keys, boolean testClient, TransactionIsolation isolation,
+        long timeout) throws Exception {
+        Collection<Thread> threads = new ArrayList<>();
+
+        try {
+            // Start grid 1.
+            IgniteEx grid1 = startGrid(0);
+
+            assertFalse(grid1.configuration().isClientMode());
+
+            threads.add(runCacheOperations(grid1, isolation, timeout, keys));
+
+            TimeUnit.SECONDS.sleep(3L);
+
+            client = testClient; // If test client start on node in client mode.
+
+            // Start grid 2.
+            IgniteEx grid2 = startGrid(1);
+
+            assertEquals((Object)testClient, grid2.configuration().isClientMode());
+
+            client = false;
+
+            threads.add(runCacheOperations(grid2, isolation, timeout, keys));
+
+            TimeUnit.SECONDS.sleep(3L);
+
+            // Start grid 3.
+            IgniteEx grid3 = startGrid(2);
+
+            assertFalse(grid3.configuration().isClientMode());
+
+            if (testClient)
+                log.info("Started client node: " + grid3.name());
+
+            threads.add(runCacheOperations(grid3, isolation, timeout, keys));
+
+            TimeUnit.SECONDS.sleep(3L);
+
+            // Start grid 4.
+            IgniteEx grid4 = startGrid(3);
+
+            assertFalse(grid4.configuration().isClientMode());
+
+            threads.add(runCacheOperations(grid4, isolation, timeout, keys));
+
+            TimeUnit.SECONDS.sleep(3L);
+
+            stopThreads(threads);
+
+            for (int i = 0; i < 4; i++) {
+                IgniteTxManager tm = ((IgniteKernal)grid(i)).internalCache(CACHE_NAME).context().tm();
+
+                assertEquals("txMap is not empty:" + i, 0, tm.idMapSize());
+            }
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @param threads Thread which will be stopped.
+     */
+    private void stopThreads(Iterable<Thread> threads) {
+        try {
+            run = false;
+
+            for (Thread thread : threads)
+                thread.join();
+        }
+        catch (Exception e) {
+            U.error(log(), "Couldn't stop threads.", e);
+        }
+    }
+
+    /**
+     * @param node Node.
+     * @param isolation Isolation.
+     * @param timeout Timeout.
+     * @param keys Number of keys.
+     * @return Running thread.
+     */
+    @SuppressWarnings("TypeMayBeWeakened")
+    private Thread runCacheOperations(Ignite node, TransactionIsolation isolation, long timeout, final int keys) {
+        Thread t = new Thread() {
+            @Override public void run() {
+                while (run) {
+                    TreeMap<Integer, String> vals = generateValues(keys);
+
+                    try {
+                        try (Transaction tx = node.transactions().txStart(TransactionConcurrency.OPTIMISTIC, isolation,
+                            timeout, keys)){
+
+                            IgniteCache<Object, Object> cache = node.cache(CACHE_NAME);
+
+                            // Put or remove.
+                            if (ThreadLocalRandom.current().nextDouble(1) < 0.65)
+                                cache.putAll(vals);
+                            else
+                                cache.removeAll(vals.keySet());
+
+                            tx.commit();
+                        }
+                        catch (Exception e) {
+                            U.error(log(), "Failed cache operation.", e);
+                        }
+
+                        U.sleep(100);
+                    }
+                    catch (Exception e){
+                        U.error(log(), "Failed unlock.", e);
+                    }
+                }
+            }
+        };
+
+        t.start();
+
+        return t;
+    }
+
+    /**
+     * @param cnt Number of keys to generate.
+     * @return Map.
+     */
+    private TreeMap<Integer, String> generateValues(int cnt) {
+        TreeMap<Integer, String> res = new TreeMap<>();
+
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        while (res.size() < cnt) {
+            int key = rnd.nextInt(0, 100);
+
+            res.put(key, String.valueOf(key));
+        }
+
+        return res;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
index 60f1c96f572d2..aab0b8e854f80 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxPessimisticDeadlockDetectionTest.java
@@ -107,7 +107,7 @@ public class TxPessimisticDeadlockDetectionTest extends AbstractDeadlockDetectio
 
         client = false;
 
-        startGrids(NODES_CNT);
+        startGridsMultiThreaded(NODES_CNT);
 
         client = true;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncNearCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncNearCacheTest.java
new file mode 100644
index 0000000000000..5caa1a06325b8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncNearCacheTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+/**
+ * Tests an ability to async rollback near transactions.
+ */
+public class TxRollbackAsyncNearCacheTest extends TxRollbackAsyncTest {
+    /** {@inheritDoc} */
+    @Override protected boolean nearCacheEnabled() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
new file mode 100644
index 0000000000000..3a8be4941c7dc
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
@@ -0,0 +1,938 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicIntegerArray;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.LongAdder;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteTransactions;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.apache.ignite.transactions.TransactionRollbackException;
+
+import static java.lang.Thread.yield;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
+import static org.apache.ignite.transactions.TransactionState.ROLLED_BACK;
+
+/**
+ * Tests an ability to async rollback near transactions.
+ */
+public class TxRollbackAsyncTest extends GridCommonAbstractTest {
+    /** */
+    public static final int DURATION = 60_000;
+
+    /** */
+    private static final String CACHE_NAME = "test";
+
+    /** IP finder. */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int GRID_CNT = 3;
+
+    /** */
+    public static final long MB = 1024 * 1024;
+
+    /** */
+    public static final String LABEL = "wLockTx";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        boolean client = igniteInstanceName.startsWith("client");
+
+        cfg.setClientMode(client);
+
+        if (persistenceEnabled())
+            cfg.setDataStorageConfiguration(new DataStorageConfiguration().setWalMode(LOG_ONLY).setPageSize(1024).
+                setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true).
+                    setInitialSize(100 * MB).setMaxSize(100 * MB)));
+
+        if (!client) {
+            CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME);
+
+            if (nearCacheEnabled())
+                ccfg.setNearConfiguration(new NearCacheConfiguration());
+
+            ccfg.setAtomicityMode(TRANSACTIONAL);
+            ccfg.setBackups(2);
+            ccfg.setWriteSynchronizationMode(FULL_SYNC);
+            ccfg.setOnheapCacheEnabled(false);
+
+            cfg.setCacheConfiguration(ccfg);
+        }
+
+        return cfg;
+    }
+
+    /**
+     * @return Near cache flag.
+     */
+    protected boolean nearCacheEnabled() {
+        return false;
+    }
+
+    /**
+     *
+     * @return {@code True} if persistence must be enabled for test.
+     */
+    protected boolean persistenceEnabled() { return false; }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        final IgniteEx crd = startGrid(0);
+
+        startGridsMultiThreaded(1, GRID_CNT - 1);
+
+        crd.cluster().active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @return Started client.
+     * @throws Exception If f nodeailed.
+     */
+    private Ignite startClient() throws Exception {
+        Ignite client = startGrid("client");
+
+        assertTrue(client.configuration().isClientMode());
+
+        if (nearCacheEnabled())
+            client.createNearCache(CACHE_NAME, new NearCacheConfiguration<>());
+        else
+            assertNotNull(client.cache(CACHE_NAME));
+
+        return client;
+    }
+
+    /**
+     *
+     */
+    public void testRollbackSimple() throws Exception {
+        startClient();
+
+        for (Ignite ignite : G.allGrids()) {
+            testRollbackSimple0(ignite);
+
+            ignite.cache(CACHE_NAME).clear();
+        }
+    }
+
+    /**
+     *
+     */
+    private void testRollbackSimple0(Ignite near) throws Exception {
+        // Normal rollback after put.
+        Transaction tx = near.transactions().txStart(PESSIMISTIC, READ_COMMITTED);
+
+        near.cache(CACHE_NAME).put(0, 0);
+
+        tx.rollback();
+
+        assertNull(near.cache(CACHE_NAME).get(0));
+
+        // Normal rollback before put.
+        tx = near.transactions().txStart();
+
+        tx.rollback();
+
+        near.cache(CACHE_NAME).put(0, 1);
+
+        assertEquals(1, near.cache(CACHE_NAME).get(0));
+
+        // Normal rollback async after put.
+        tx = near.transactions().txStart();
+
+        near.cache(CACHE_NAME).put(1, 0);
+
+        rollbackAsync(tx).get();
+
+        try {
+            assertNull(near.cache(CACHE_NAME).get(0));
+
+            fail();
+        }
+        catch (Exception e) {
+            // Expected.
+        }
+
+        try {
+            near.cache(CACHE_NAME).put(1, 1);
+
+            fail();
+        }
+        catch (Exception e) {
+            // Expected.
+        }
+
+        try {
+            near.cache(CACHE_NAME).remove(0);
+
+            fail();
+        }
+        catch (Exception e) {
+            // Expected.
+        }
+
+        // Normal rollback async before put.
+        tx = near.transactions().txStart();
+
+        rollbackAsync(tx).get();
+
+        try {
+            assertNull(near.cache(CACHE_NAME).get(0));
+
+            fail();
+        }
+        catch (Exception e) {
+            // Expected.
+        }
+
+        try {
+            near.cache(CACHE_NAME).put(1, 1);
+
+            fail();
+        }
+        catch (Exception e) {
+            // Expected.
+        }
+
+        try {
+            near.cache(CACHE_NAME).remove(0);
+
+            fail();
+        }
+        catch (Exception e) {
+            // Expected.
+        }
+
+        checkFutures();
+    }
+
+    /**
+     *
+     */
+    public void testSynchronousRollback() throws Exception {
+        Ignite client = startClient();
+
+        for (int i = 0; i < GRID_CNT; i++)
+            testSynchronousRollback0(grid(0), grid(i), false);
+
+        testSynchronousRollback0(grid(0), client, false);
+
+        for (int i = 0; i < GRID_CNT; i++)
+            testSynchronousRollback0(grid(0), grid(i), true);
+
+        testSynchronousRollback0(grid(0), client, true);
+
+        for (int i = 0; i < GRID_CNT; i++)
+            testSynchronousRollback0(grid(1), grid(i), false);
+
+        testSynchronousRollback0(grid(1), client, false);
+
+        for (int i = 0; i < GRID_CNT; i++)
+            testSynchronousRollback0(grid(1), grid(i), true);
+
+        testSynchronousRollback0(grid(1), client, true);
+    }
+
+    /**
+     *
+     * @param holdLockNode Node holding the write lock.
+     * @param tryLockNode Node trying to acquire lock.
+     * @param useTimeout {@code True} if need to start tx with timeout.
+     *
+     * @throws Exception If failed.
+     */
+    private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockNode, final boolean useTimeout) throws Exception {
+        final CountDownLatch keyLocked = new CountDownLatch(1);
+
+        CountDownLatch waitCommit = new CountDownLatch(1);
+
+        IgniteInternalFuture<?> lockFut = lockInTx(holdLockNode, keyLocked, waitCommit, 0);
+
+        U.awaitQuiet(keyLocked);
+
+        final CountDownLatch rollbackLatch = new CountDownLatch(1);
+
+        final int txCnt = 10000;
+
+        final IgniteKernal k = (IgniteKernal)tryLockNode;
+
+        final GridCacheSharedContext<Object, Object> ctx = k.context().cache().context();
+
+        final GridCacheContext<Object, Object> cctx = ctx.cacheContext(CU.cacheId(CACHE_NAME));
+
+        final AtomicBoolean stop = new AtomicBoolean();
+
+        IgniteInternalFuture<?> txFut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                U.awaitQuiet(keyLocked);
+
+                for (int i = 0; i < txCnt; i++) {
+                    GridNearTxLocal tx0 = ctx.tm().threadLocalTx(cctx);
+
+                    assertTrue(tx0 == null || tx0.state() == ROLLED_BACK);
+
+                    rollbackLatch.countDown();
+
+                    try (Transaction tx = tryLockNode.transactions().txStart(PESSIMISTIC, REPEATABLE_READ,
+                        useTimeout ? 500 : 0, 1)) {
+
+                        // Will block on lock request until rolled back asynchronously.
+                        Object o = tryLockNode.cache(CACHE_NAME).get(0);
+
+                        assertNull(o); // If rolled back by close, previous get will return null.
+                    }
+                    catch (Exception e) {
+                        // If rolled back by rollback, previous get will throw an exception.
+                    }
+                }
+
+                stop.set(true);
+            }
+        }, 1, "tx-get-thread");
+
+        IgniteInternalFuture<?> rollbackFut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                U.awaitQuiet(rollbackLatch);
+
+                doSleep(50);
+
+                Set<IgniteUuid> rolledBackVers = new HashSet<>();
+
+                int proc = 1;
+
+                while(!stop.get()) {
+                    for (Transaction tx : tryLockNode.transactions().localActiveTransactions()) {
+                        if (rolledBackVers.contains(tx.xid()))
+                            fail("Rollback version is expected");
+
+                        // Skip write transaction.
+                        if (LABEL.equals(tx.label()))
+                            continue;
+
+                        try {
+                            if (proc % 2 == 0)
+                                tx.rollback();
+                            else
+                                tx.close();
+                        }
+                        catch (IgniteException e) {
+                            log.warning("Got exception while rolling back a transaction", e);
+                        }
+
+                        rolledBackVers.add(tx.xid());
+
+                        if (proc % 1000 == 0)
+                            log.info("Rolled back: " + proc);
+
+                        proc++;
+                    }
+                }
+
+                assertEquals("Unexpected size", txCnt, rolledBackVers.size());
+            }
+        }, 1, "tx-rollback-thread");
+
+        rollbackFut.get();
+
+        txFut.get();
+
+        log.info("All transactions are rolled back: holdLockNode=" + holdLockNode + ", tryLockNode=" + tryLockNode);
+
+        waitCommit.countDown();
+
+        lockFut.get();
+
+        assertEquals(0, holdLockNode.cache(CACHE_NAME).get(0));
+
+        checkFutures();
+    }
+
+    /**
+     *
+     */
+    public void testEnlistManyRead() throws Exception {
+        testEnlistMany(false, REPEATABLE_READ, PESSIMISTIC);
+    }
+
+    /**
+     *
+     */
+    public void testEnlistManyWrite() throws Exception {
+        testEnlistMany(true, REPEATABLE_READ, PESSIMISTIC);
+    }
+
+    /**
+     *
+     */
+    public void testEnlistManyReadOptimistic() throws Exception {
+        testEnlistMany(false, SERIALIZABLE, OPTIMISTIC);
+    }
+
+    /**
+     *
+     */
+    public void testEnlistManyWriteOptimistic() throws Exception {
+        testEnlistMany(true, SERIALIZABLE, OPTIMISTIC);
+    }
+
+
+    /**
+     *
+     */
+    private void testEnlistMany(boolean write, TransactionIsolation isolation, TransactionConcurrency conc) throws Exception {
+        final Ignite client = startClient();
+
+        Map<Integer, Integer> entries = new HashMap<>();
+
+        for (int i = 0; i < 1000000; i++)
+            entries.put(i, i);
+
+        IgniteInternalFuture<?> fut = null;
+
+        try(Transaction tx = client.transactions().txStart(conc, isolation, 0, 0)) {
+            fut = rollbackAsync(tx, 200);
+
+            if (write)
+                client.cache(CACHE_NAME).putAll(entries);
+            else
+                client.cache(CACHE_NAME).getAll(entries.keySet());
+
+            tx.commit();
+
+            fail("Commit must fail");
+        }
+        catch (Throwable t) {
+            assertTrue(X.hasCause(t, TransactionRollbackException.class));
+        }
+
+        fut.get();
+
+        assertEquals(0, client.cache(CACHE_NAME).size());
+
+        checkFutures();
+    }
+
+    /**
+     * Rollback tx while near lock request is delayed.
+     */
+    public void testRollbackDelayNearLockRequest() throws Exception {
+        final Ignite client = startClient();
+
+        final Ignite prim = primaryNode(0, CACHE_NAME);
+
+        final TestRecordingCommunicationSpi spi = (TestRecordingCommunicationSpi)client.configuration().getCommunicationSpi();
+
+        spi.blockMessages(GridNearLockRequest.class, prim.name());
+
+        final IgniteInternalFuture<Void> rollbackFut = runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                spi.waitForBlocked();
+
+                client.transactions().localActiveTransactions().iterator().next().rollback();
+
+                return null;
+            }
+        }, "tx-rollback-thread");
+
+        try(final Transaction tx = client.transactions().txStart()) {
+            client.cache(CACHE_NAME).put(0, 0);
+
+            fail();
+        }
+        catch (CacheException e) {
+            assertTrue(X.hasCause(e, TransactionRollbackException.class));
+        }
+
+        rollbackFut.get();
+
+        spi.stopBlock(true);
+
+        doSleep(500);
+
+        checkFutures();
+    }
+
+    /**
+     * Tests rollback with concurrent commit.
+     */
+    public void testRollbackDelayFinishRequest() throws Exception {
+        final Ignite client = startClient();
+
+        final Ignite prim = primaryNode(0, CACHE_NAME);
+
+        final TestRecordingCommunicationSpi spi = (TestRecordingCommunicationSpi)client.configuration().getCommunicationSpi();
+
+        final AtomicReference<Transaction> txRef = new AtomicReference<>();
+
+        // Block commit request to primary node.
+        spi.blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+            @Override public boolean apply(ClusterNode node, Message msg) {
+                if (msg instanceof GridNearTxFinishRequest) {
+                    GridNearTxFinishRequest r = (GridNearTxFinishRequest)msg;
+
+                    return r.commit() && node.equals(prim.cluster().localNode());
+                }
+
+                return false;
+            }
+        });
+
+        final IgniteInternalFuture<Void> rollbackFut = runAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                spi.waitForBlocked();
+
+                final IgniteFuture<?> fut = txRef.get().rollbackAsync();
+
+                doSleep(500);
+
+                spi.stopBlock(true);
+
+                fut.get();
+
+                return null;
+            }
+        }, "tx-rollback-thread");
+
+        try(final Transaction tx = client.transactions().txStart()) {
+            txRef.set(tx);
+
+            client.cache(CACHE_NAME).put(0, 0);
+
+            tx.commit();
+        }
+        catch (CacheException e) {
+            assertTrue(X.hasCause(e, TransactionRollbackException.class));
+        }
+
+        rollbackFut.get();
+
+        doSleep(500);
+
+        checkFutures();
+    }
+
+    /**
+     *
+     */
+    public void testMixedAsyncRollbackTypes() throws Exception {
+        final Ignite client = startClient();
+
+        final AtomicBoolean stop = new AtomicBoolean();
+
+        final int threadCnt = Runtime.getRuntime().availableProcessors() * 2;
+
+        final int txSize = 200;
+
+        for (int k = 0; k < txSize; k++)
+            grid(0).cache(CACHE_NAME).put(k, (long)0);
+
+        final long seed = System.currentTimeMillis();
+
+        final Random r = new Random(seed);
+
+        log.info("Using seed: " + seed);
+
+        final TransactionConcurrency[] TC_VALS = TransactionConcurrency.values();
+        final TransactionIsolation[] TI_VALS = TransactionIsolation.values();
+
+        final LongAdder total = new LongAdder();
+        final LongAdder completed = new LongAdder();
+        final LongAdder failed = new LongAdder();
+        final LongAdder rolledBack = new LongAdder();
+
+        IgniteInternalFuture<?> txFut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                while (!stop.get()) {
+                    int nodeId = r.nextInt(GRID_CNT + 1);
+
+                    // Choose random node to start tx on.
+                    Ignite node = nodeId == GRID_CNT || nearCacheEnabled() ? client : grid(nodeId);
+
+                    TransactionConcurrency conc = TC_VALS[r.nextInt(TC_VALS.length)];
+                    TransactionIsolation isolation = TI_VALS[r.nextInt(TI_VALS.length)];
+
+                    long timeout = r.nextInt(50) + 50; // Timeout is necessary to prevent deadlocks.
+
+                    try (Transaction tx = node.transactions().txStart(conc, isolation, timeout, txSize)) {
+                        int setSize = r.nextInt(txSize / 2) + 1;
+
+                        for (int i = 0; i < setSize; i++) {
+                            switch (r.nextInt(4)) {
+                                case 0:
+                                    node.cache(CACHE_NAME).remove(r.nextInt(txSize));
+
+                                    break;
+
+                                case 1:
+                                    node.cache(CACHE_NAME).get(r.nextInt(txSize));
+
+                                    break;
+
+                                case 2:
+                                    final Integer v = (Integer)node.cache(CACHE_NAME).get(r.nextInt(txSize));
+
+                                    node.cache(CACHE_NAME).put(r.nextInt(txSize), (v == null ? 0 : v) + 1);
+
+                                    break;
+
+                                case 3:
+                                    node.cache(CACHE_NAME).put(r.nextInt(txSize), 0);
+
+                                    break;
+
+                                default:
+                                    fail("Unexpected opcode");
+                            }
+                        }
+
+                        tx.commit();
+
+                        completed.add(1);
+                    }
+                    catch (Throwable e) {
+                        failed.add(1);
+                    }
+
+                    total.add(1);
+                }
+            }
+        }, threadCnt, "tx-thread");
+
+        final AtomicIntegerArray idx = new AtomicIntegerArray(GRID_CNT + 1);
+
+        IgniteInternalFuture<?> rollbackFut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                int concurrentRollbackCnt = 5;
+
+                List<IgniteFuture<?>> futs = new ArrayList<>(concurrentRollbackCnt);
+
+                while (!stop.get()) {
+                    // Choose node randomly.
+                    final int nodeId = r.nextInt(GRID_CNT + 1);
+
+                    // Reserve node.
+                    if (!idx.compareAndSet(nodeId, 0, 1)) {
+                        yield();
+
+                        continue;
+                    }
+
+                    Ignite node = nodeId == GRID_CNT || nearCacheEnabled() ? client : grid(nodeId);
+
+                    Collection<Transaction> transactions = node.transactions().localActiveTransactions();
+
+                    for (Transaction tx : transactions) {
+                        rolledBack.add(1);
+
+                        if (rolledBack.sum() % 1000 == 0)
+                            info("Processed: " + rolledBack.sum());
+
+                        try {
+                            IgniteFuture<Void> rollbackFut = tx.rollbackAsync();
+
+                            rollbackFut.listen(new IgniteInClosure<IgniteFuture<Void>>() {
+                                @Override public void apply(IgniteFuture<Void> fut) {
+                                    tx.close();
+                                }
+                            });
+
+                            futs.add(rollbackFut);
+                        }
+                        catch (Throwable t) {
+                            log.error("Exception on async rollback", t);
+
+                            fail("Exception is not expected");
+                        }
+
+                        if (futs.size() == concurrentRollbackCnt) {
+                            for (IgniteFuture<?> fut : futs)
+                                try {
+                                    fut.get();
+                                }
+                                catch (IgniteException e) {
+                                    log.warning("Future was rolled back with error", e);
+                                }
+
+                            futs.clear();
+                        }
+                    }
+
+                    idx.set(nodeId, 0);
+                }
+
+                for (IgniteFuture<?> fut : futs)
+                    try {
+                        fut.get();
+                    }
+                    catch (Throwable t) {
+                        // No-op.
+                    }
+
+            }
+        }, 3, "rollback-thread"); // Rollback by multiple threads.
+
+        doSleep(DURATION);
+
+        stop.set(true);
+
+        txFut.get();
+
+        rollbackFut.get();
+
+        log.info("total=" + total.sum() + ", completed=" + completed.sum() + ", failed=" + failed.sum() +
+            ", rolledBack=" + rolledBack.sum());
+
+        assertEquals("total != completed + failed", total.sum(), completed.sum() + failed.sum());
+
+        checkFutures();
+    }
+
+    /**
+     * Tests proxy object returned by {@link IgniteTransactions#localActiveTransactions()}
+     */
+    public void testRollbackProxy() throws Exception {
+        final CountDownLatch keyLocked = new CountDownLatch(1);
+
+        CountDownLatch waitCommit = new CountDownLatch(1);
+
+        Ignite ig = ignite(0);
+
+        IgniteInternalFuture<?> lockFut = lockInTx(ig, keyLocked, waitCommit, 0);
+
+        U.awaitQuiet(keyLocked);
+
+        Collection<Transaction> txs = ig.transactions().localActiveTransactions();
+
+        for (Transaction tx : txs) {
+            try {
+                tx.timeout(1);
+
+                fail("timeout");
+            }
+            catch (Exception e) {
+                // No-op.
+            }
+
+            try {
+                tx.setRollbackOnly();
+
+                fail("setRollbackOnly");
+            }
+            catch (Exception e) {
+                // No-op.
+            }
+
+            try {
+                tx.commit();
+
+                fail("commit");
+            }
+            catch (Exception e) {
+                // No-op.
+            }
+
+            try {
+                tx.commitAsync();
+
+                fail("commitAsync");
+            }
+            catch (Exception e) {
+                // No-op.
+            }
+
+            try {
+                tx.suspend();
+
+                fail("suspend");
+            }
+            catch (Exception e) {
+                // No-op.
+            }
+
+            try {
+                tx.resume();
+
+                fail("resume");
+            }
+            catch (Exception e) {
+                // No-op.
+            }
+
+            tx.rollback();
+
+            tx.rollbackAsync().get();
+        }
+
+        waitCommit.countDown();
+
+        try {
+            lockFut.get();
+
+            fail();
+        }
+        catch (IgniteCheckedException e) {
+            // No-op.
+        }
+    }
+
+    /**
+     * Locks entry in tx and delays commit until signalled.
+     *
+     * @param node Near node.
+     * @param keyLocked Latch for notifying until key is locked.
+     * @param waitCommit Latch for waiting until commit is allowed.
+     * @param timeout Timeout.
+     *
+     * @return tx completion future.
+     */
+    private IgniteInternalFuture<?> lockInTx(final Ignite node, final CountDownLatch keyLocked,
+        final CountDownLatch waitCommit, final int timeout) throws Exception {
+        return multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                Transaction tx = node.transactions().withLabel(LABEL).txStart(PESSIMISTIC, REPEATABLE_READ, timeout, 1);
+
+                node.cache(CACHE_NAME).put(0, 0);
+
+                keyLocked.countDown();
+
+                try {
+                    U.await(waitCommit);
+                }
+                catch (IgniteInterruptedCheckedException e) {
+                    fail("Lock thread was interrupted while waiting");
+                }
+
+                tx.commit();
+            }
+        }, 1, "tx-lock-thread");
+    }
+
+    /**
+     * Checks if all tx futures are finished.
+     */
+    private void checkFutures() {
+        for (Ignite ignite : G.allGrids()) {
+            IgniteEx ig = (IgniteEx)ignite;
+
+            final Collection<GridCacheFuture<?>> futs = ig.context().cache().context().mvcc().activeFutures();
+
+            for (GridCacheFuture<?> fut : futs)
+                log.info("Waiting for future: " + fut);
+
+            assertTrue("Expecting no active futures: node=" + ig.localNode().id(), futs.isEmpty());
+        }
+    }
+
+    /**
+     * @param tx Tx to rollback.
+     */
+    private IgniteInternalFuture<?> rollbackAsync(final Transaction tx) throws Exception {
+        return multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                tx.rollback();
+            }
+        }, 1, "tx-rollback-thread");
+    }
+
+    /**
+     * @param tx Tx to rollback.
+     * @param delay Delay in millis.
+     */
+    private IgniteInternalFuture<?> rollbackAsync(final Transaction tx, long delay) throws Exception {
+        return multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                doSleep(delay);
+
+                tx.rollback();
+            }
+        }, 1, "tx-rollback-thread");
+    }
+
+}
+
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncWithPersistenceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncWithPersistenceTest.java
new file mode 100644
index 0000000000000..f1f8d10da5414
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncWithPersistenceTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS;
+
+/**
+ * Tests an ability to rollback near transactions.
+ */
+public class TxRollbackAsyncWithPersistenceTest extends TxRollbackAsyncTest {
+    /** {@inheritDoc} */
+    @Override protected boolean persistenceEnabled() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty(IGNITE_WAL_LOG_TX_RECORDS, "true");
+
+        super.beforeTestsStarted();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IGNITE_WAL_LOG_TX_RECORDS);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        cleanPersistenceDir();
+    }
+}
+
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNoDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNoDeadlockDetectionTest.java
index 51233294f41ff..d888970b481f3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNoDeadlockDetectionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNoDeadlockDetectionTest.java
@@ -44,4 +44,8 @@ public class TxRollbackOnTimeoutNoDeadlockDetectionTest extends TxRollbackOnTime
         assertEquals("TimeoutException is expected",
             TransactionTimeoutException.class, e.getCause().getClass());
     }
-}
\ No newline at end of file
+
+    @Override public void testRandomMixedTxConfigurations() throws Exception {
+        super.testRandomMixedTxConfigurations();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
index 97de81edc781d..d1cbfa72613bd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
@@ -18,10 +18,13 @@
 package org.apache.ignite.internal.processors.cache.transactions;
 
 import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.LongAdder;
 import javax.cache.CacheException;
 import org.apache.ignite.Ignite;
@@ -434,23 +437,19 @@ public void testRandomMixedTxConfigurations() throws Exception {
 
                         final Long v = (Long)node.cache(CACHE_NAME).get(k);
 
+                        assertNotNull("Expecting not null value: " + tx, v);
+
                         final int delay = r.nextInt(400);
 
                         if (delay > 0)
                             sleep(delay);
 
-                        assert v != null;
-
                         node.cache(CACHE_NAME).put(k, v + 1);
 
                         tx.commit();
 
                         cntr1.add(1);
                     }
-                    catch (TransactionOptimisticException | InterruptedException e) {
-                        // Expected.
-                        cntr3.add(1);
-                    }
                     catch (TransactionTimeoutException e) {
                         cntr2.add(1);
                     }
@@ -459,6 +458,9 @@ public void testRandomMixedTxConfigurations() throws Exception {
 
                         cntr2.add(1);
                     }
+                    catch (Exception e) {
+                        cntr3.add(1);
+                    }
                 }
             }
         }, threadsCnt, "tx-async-thread");
@@ -474,7 +476,8 @@ public void testRandomMixedTxConfigurations() throws Exception {
             ", failed=" + cntr3.sum() +
             ", timedOut=" + cntr2.sum());
 
-        assertEquals("Expected finished count same as started count", cntr0.sum(), cntr1.sum() + cntr2.sum() + cntr3.sum());
+        assertEquals("Expected finished count same as started count", cntr0.sum(), cntr1.sum() + cntr2.sum() +
+            cntr3.sum());
     }
 
     /**
@@ -493,6 +496,107 @@ public void testTimeoutOnPrimaryDHTNode() throws Exception {
         }
     }
 
+    /**
+     *
+     */
+    public void testLockRelease() throws Exception {
+        final Ignite client = startClient();
+
+        final AtomicInteger idx = new AtomicInteger();
+
+        final int threadCnt = Runtime.getRuntime().availableProcessors() * 2;
+
+        final CountDownLatch readStartLatch = new CountDownLatch(1);
+
+        final CountDownLatch commitLatch = new CountDownLatch(threadCnt - 1);
+
+        final IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                final int idx0 = idx.getAndIncrement();
+
+                if (idx0 == 0) {
+                    try(final Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, 1)) {
+                        client.cache(CACHE_NAME).put(0, 0); // Lock is owned.
+
+                        readStartLatch.countDown();
+
+                        U.awaitQuiet(commitLatch);
+
+                        tx.commit();
+                    }
+                }
+                else {
+                    try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 300, 1)) {
+                        U.awaitQuiet(readStartLatch);
+
+                        client.cache(CACHE_NAME).get(0); // Lock acquisition is queued.
+                    }
+                    catch (CacheException e) {
+                        assertTrue(e.getMessage(), X.hasCause(e, TransactionTimeoutException.class));
+                    }
+
+                    commitLatch.countDown();
+                }
+            }
+        }, threadCnt, "tx-async");
+
+        fut.get();
+
+        Thread.sleep(500);
+
+        assertEquals(0, client.cache(CACHE_NAME).get(0));
+
+        for (Ignite ignite : G.allGrids()) {
+            IgniteEx ig = (IgniteEx)ignite;
+
+            final IgniteInternalFuture<?> f = ig.context().cache().context().
+                partitionReleaseFuture(new AffinityTopologyVersion(G.allGrids().size() + 1, 0));
+
+            assertTrue("Unexpected incomplete future", f.isDone());
+        }
+
+    }
+
+    /**
+     *
+     */
+    public void testEnlistManyRead() throws Exception {
+        testEnlistMany(false);
+    }
+
+    /**
+     *
+     */
+    public void testEnlistManyWrite() throws Exception {
+        testEnlistMany(true);
+    }
+
+    /**
+     *
+     */
+    private void testEnlistMany(boolean write) throws Exception {
+        final Ignite client = startClient();
+
+        Map<Integer, Integer> entries = new HashMap<>();
+
+        for (int i = 0; i < 1000000; i++)
+            entries.put(i, i);
+
+        try(Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 200, 0)) {
+            if (write)
+                client.cache(CACHE_NAME).putAll(entries);
+            else
+                client.cache(CACHE_NAME).getAll(entries.keySet());
+
+            tx.commit();
+        }
+        catch (Throwable t) {
+            assertTrue(X.hasCause(t, TransactionTimeoutException.class));
+        }
+
+        assertEquals(0, client.cache(CACHE_NAME).size());
+    }
+
     /**
      *
      * @param prim Primary node.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java
new file mode 100644
index 0000000000000..2542cdb5365ef
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerArray;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.TransactionConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.GridCacheFuture;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+
+import static java.lang.Thread.yield;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Tests an ability to rollback transactions on topology change.
+ */
+public class TxRollbackOnTopologyChangeTest extends GridCommonAbstractTest {
+    /** */
+    public static final int ROLLBACK_TIMEOUT = 500;
+
+    /** */
+    private static final String CACHE_NAME = "test";
+
+    /** IP finder. */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int SRV_CNT = 6;
+
+    /** */
+    private static final int CLNT_CNT = 2;
+
+    /** */
+    private static final int TOTAL_CNT = SRV_CNT + CLNT_CNT;
+
+    /** */
+    public static final int ITERATIONS = 100;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setTransactionConfiguration(new TransactionConfiguration().
+            setTxTimeoutOnPartitionMapExchange(ROLLBACK_TIMEOUT));
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        cfg.setClientMode(getTestIgniteInstanceIndex(igniteInstanceName) >= SRV_CNT);
+
+        CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME);
+
+        ccfg.setAtomicityMode(TRANSACTIONAL);
+        ccfg.setBackups(2);
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        startGridsMultiThreaded(TOTAL_CNT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * Tests rollbacks on topology change.
+     */
+    public void testRollbackOnTopologyChange() throws Exception {
+        final AtomicBoolean stop = new AtomicBoolean();
+
+        final long seed = System.currentTimeMillis();
+
+        final Random r = new Random(seed);
+
+        log.info("Using seed: " + seed);
+
+        AtomicIntegerArray reservedIdx = new AtomicIntegerArray(TOTAL_CNT);
+
+        final int keysCnt = SRV_CNT - 1;
+
+        for (int k = 0; k < keysCnt; k++)
+            grid(0).cache(CACHE_NAME).put(k, (long)0);
+
+        final CyclicBarrier b = new CyclicBarrier(keysCnt);
+
+        AtomicInteger idGen = new AtomicInteger();
+
+        final IgniteInternalFuture<?> txFut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                int key = idGen.getAndIncrement();
+
+                List<Integer> keys = new ArrayList<>();
+
+                for (int k = 0; k < keysCnt; k++)
+                    keys.add(k);
+
+                int cntr = 0;
+
+                for (int i = 0; i < ITERATIONS; i++) {
+                    cntr++;
+
+                    int nodeId;
+
+                    while(!reservedIdx.compareAndSet((nodeId = r.nextInt(TOTAL_CNT)), 0, 1))
+                        doSleep(10);
+
+                    U.awaitQuiet(b);
+
+                    final IgniteEx grid = grid(nodeId);
+
+                    try (final Transaction tx = grid.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, 0)) {
+                        reservedIdx.set(nodeId, 0);
+
+                        // Construct deadlock
+                        grid.cache(CACHE_NAME).get(keys.get(key));
+
+                        // Should block.
+                        grid.cache(CACHE_NAME).get(keys.get((key + 1) % keysCnt));
+
+                        fail("Deadlock expected");
+                    }
+                    catch (Throwable t) {
+                        // Expected.
+                    }
+
+                    if (key == 0)
+                        log.info("Rolled back: " + cntr);
+                }
+            }
+        }, keysCnt, "tx-lock-thread");
+
+        final IgniteInternalFuture<?> restartFut = multithreadedAsync(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                while(!stop.get()) {
+                    final int nodeId = r.nextInt(TOTAL_CNT);
+
+                    if (!reservedIdx.compareAndSet(nodeId, 0, 1)) {
+                        yield();
+
+                        continue;
+                    }
+
+                    stopGrid(nodeId);
+
+                    doSleep(500 + r.nextInt(1000));
+
+                    startGrid(nodeId);
+
+                    reservedIdx.set(nodeId, 0);
+                }
+
+                return null;
+            }
+        }, 1, "tx-restart-thread");
+
+        txFut.get(); // Wait for iterations to complete.
+
+        stop.set(true);
+
+        restartFut.get();
+
+        checkFutures();
+    }
+
+    /**
+     * Checks if all tx futures are finished.
+     */
+    private void checkFutures() {
+        for (Ignite ignite : G.allGrids()) {
+            IgniteEx ig = (IgniteEx)ignite;
+
+            final Collection<GridCacheFuture<?>> futs = ig.context().cache().context().mvcc().activeFutures();
+
+            for (GridCacheFuture<?> fut : futs)
+                log.info("Waiting for future: " + fut);
+
+            assertTrue("Expecting no active futures: node=" + ig.localNode().id(), futs.isEmpty());
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
index 774af039f99fa..18abc1c88a19e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
@@ -196,7 +196,7 @@ public void testDataStoring() throws Exception {
                 GridCacheContext<Object, Object> ctx = GridTestUtils.getFieldValue(
                     grid(j).cachex(grid(j).igfsx("igfs").configuration().getDataCacheConfiguration().getName()),
                     "ctx");
-                Collection<IgniteInternalTx> txs = ctx.tm().txs();
+                Collection<IgniteInternalTx> txs = ctx.tm().activeTransactions();
 
                 assert txs.isEmpty() : "Incomplete transactions: " + txs;
             }
@@ -289,7 +289,7 @@ public void testDataStoringRemainder() throws Exception {
                 GridCacheContext<Object, Object> ctx = GridTestUtils.getFieldValue(grid(j).cachex(
                     grid(j).igfsx("igfs").configuration().getDataCacheConfiguration().getName()),
                     "ctx");
-                Collection<IgniteInternalTx> txs = ctx.tm().txs();
+                Collection<IgniteInternalTx> txs = ctx.tm().activeTransactions();
 
                 assert txs.isEmpty() : "Incomplete transactions: " + txs;
             }
@@ -374,7 +374,7 @@ public void testDataStoringFlush() throws Exception {
                 GridCacheContext<Object, Object> ctx = GridTestUtils.getFieldValue(
                     grid(j).cachex(grid(j).igfsx("igfs").configuration().getDataCacheConfiguration().getName()),
                     "ctx");
-                Collection<IgniteInternalTx> txs = ctx.tm().txs();
+                Collection<IgniteInternalTx> txs = ctx.tm().activeTransactions();
 
                 assert txs.isEmpty() : "Incomplete transactions: " + txs;
             }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/cache/GridAbstractCacheStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/cache/GridAbstractCacheStoreSelfTest.java
index f764212fada90..22d93763abf8f 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/cache/GridAbstractCacheStoreSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/cache/GridAbstractCacheStoreSelfTest.java
@@ -583,6 +583,11 @@ public static class DummyTx extends GridMetadataAwareAdapter implements Transact
             // No-op.
         }
 
+        /** {@inheritDoc} */
+        @Nullable @Override public String label() {
+            return null;
+        }
+
         /** {@inheritDoc} */
         @Override public void resume() {
             // No-op.
@@ -593,4 +598,4 @@ public static class DummyTx extends GridMetadataAwareAdapter implements Transact
             return null;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
index 266c1f1800932..e06af68c571da 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteClusterProcessProxy.java
@@ -160,6 +160,11 @@ public IgniteClusterProcessProxy(IgniteProcessProxy proxy) {
         throw new UnsupportedOperationException("Operation is not supported yet.");
     }
 
+    /** {@inheritDoc} */
+    @Override public void setTxTimeoutOnPartitionMapExchange(long timeout) {
+        throw new UnsupportedOperationException("Operation is not supported yet.");
+    }
+
     /** {@inheritDoc} */
     @Override public IgniteCluster withAsync() {
         throw new UnsupportedOperationException("Operation is not supported yet.");
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index c388f1dd4d5c7..e71a56981f09c 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -42,6 +42,7 @@
 import org.apache.ignite.internal.IgniteLocalNodeMapBeforeStartTest;
 import org.apache.ignite.internal.IgniteSlowClientDetectionSelfTest;
 import org.apache.ignite.internal.MarshallerContextLockingSelfTest;
+import org.apache.ignite.internal.TransactionsMXBeanImplTest;
 import org.apache.ignite.internal.managers.IgniteDiagnosticMessagesTest;
 import org.apache.ignite.internal.processors.affinity.GridAffinityProcessorRendezvousSelfTest;
 import org.apache.ignite.internal.processors.cache.GridLocalIgniteSerializationTest;
@@ -51,6 +52,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheClientRequestsMappingOnMissTest;
 import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheConcurrentReadWriteTest;
 import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheFSRestoreTest;
+import org.apache.ignite.internal.processors.cache.SetTxTimeoutOnPartitionMapExchangeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteRejectConnectOnNodeStopTest;
 import org.apache.ignite.internal.processors.closure.GridClosureProcessorRemoteTest;
 import org.apache.ignite.internal.processors.closure.GridClosureProcessorSelfTest;
@@ -153,6 +155,8 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(GridNodeMetricsLogSelfTest.class);
         suite.addTestSuite(GridLocalIgniteSerializationTest.class);
         suite.addTestSuite(GridMBeansTest.class);
+        suite.addTestSuite(TransactionsMXBeanImplTest.class);
+        suite.addTestSuite(SetTxTimeoutOnPartitionMapExchangeTest.class);
 
         suite.addTestSuite(IgniteExceptionInNioWorkerSelfTest.class);
         suite.addTestSuite(IgniteLocalNodeMapBeforeStartTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index bfdf2fd2550b0..b66bf5b5026fa 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.CacheAsyncOperationsTest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheMixedModeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteTxGetAfterStopTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteTxRemoveTimeoutObjectsNearTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteTxRemoveTimeoutObjectsTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheDaemonNodePartitionedSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedOnlyP2PDisabledByteArrayValuesSelfTest;
@@ -206,6 +207,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheAsyncOperationsTest.class);
 
         suite.addTestSuite(IgniteTxRemoveTimeoutObjectsTest.class);
+        suite.addTestSuite(IgniteTxRemoveTimeoutObjectsNearTest.class);
 
         return suite;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index b730697e91ffa..fb0daa9f67396 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -42,10 +42,17 @@
 import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgnitePessimisticTxSuspendResumeTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxLabelTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxMultiCacheAsyncOpsTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticPrepareOnUnstableTopologyTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncWithPersistenceTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticOnPartitionExchangeTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNearCacheTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncNearCacheTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTopologyChangeTest;
 
 /**
  * Test suite.
@@ -71,6 +78,16 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TxRollbackOnTimeoutNoDeadlockDetectionTest.class);
         suite.addTestSuite(TxRollbackOnTimeoutNearCacheTest.class);
         suite.addTestSuite(IgniteCacheThreadLocalTxTest.class);
+        suite.addTestSuite(TxRollbackAsyncTest.class);
+        suite.addTestSuite(TxRollbackAsyncNearCacheTest.class);
+        suite.addTestSuite(TxRollbackAsyncWithPersistenceTest.class);
+        suite.addTestSuite(TxRollbackOnTopologyChangeTest.class);
+
+        suite.addTestSuite(TxOptimisticPrepareOnUnstableTopologyTest.class);
+
+        suite.addTestSuite(TxLabelTest.class);
+
+        suite.addTestSuite(TxMultiCacheAsyncOpsTest.class);
 
         suite.addTestSuite(IgnitePdsCacheAssignmentNodeRestartsTest.class);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index eb18e29680082..c2d9eec4bae58 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -22,20 +22,40 @@
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ConnectorConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.commandline.CommandHandler;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.tx.VisorTxInfo;
+import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionRollbackException;
 
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
 
 /**
  * Command line handler test.
@@ -85,6 +105,8 @@ protected File folder(String folder) throws IgniteCheckedException {
 
         cfg.setConsistentId(igniteInstanceName);
 
+        cfg.setClientMode("client".equals(igniteInstanceName));
+
         return cfg;
     }
 
@@ -122,6 +144,32 @@ protected int execute(ArrayList<String> args) {
         return new CommandHandler().execute(args);
     }
 
+    /**
+     * @param hnd Handler.
+     * @param args Arguments.
+     * @return Result of execution
+     */
+    protected int execute(CommandHandler hnd, ArrayList<String> args) {
+        // Add force to avoid interactive confirmation
+        args.add("--force");
+
+        return hnd.execute(args);
+    }
+
+    /**
+     * @param hnd Handler.
+     * @param args Arguments.
+     * @return Result of execution
+     */
+    protected int execute(CommandHandler hnd, String... args) {
+        ArrayList<String> args0 = new ArrayList<>(Arrays.asList(args));
+
+        // Add force to avoid interactive confirmation
+        args0.add("--force");
+
+        return hnd.execute(args0);
+    }
+
     /**
      * Test deactivation works via control.sh
      *
@@ -312,4 +360,203 @@ public void testBaselineVersion() throws Exception {
 
         assertEquals(2, ignite.cluster().currentBaselineTopology().size());
     }
+
+    /**
+     * Test active transactions.
+     *
+     * @throws Exception If failed.
+     */
+    public void testActiveTransactions() throws Exception {
+        Ignite ignite = startGridsMultiThreaded(2);
+
+        ignite.cluster().active(true);
+
+        Ignite client = startGrid("client");
+
+        IgniteCache<Object, Object> cache = client.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+            .setAtomicityMode(TRANSACTIONAL).setWriteSynchronizationMode(FULL_SYNC));
+
+        for (Ignite ig : G.allGrids())
+            assertNotNull(ig.cache(DEFAULT_CACHE_NAME));
+
+        AtomicInteger idx = new AtomicInteger();
+
+        CountDownLatch lockLatch = new CountDownLatch(1);
+        CountDownLatch unlockLatch = new CountDownLatch(1);
+
+        IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                int id = idx.getAndIncrement();
+
+                switch (id) {
+                    case 0:
+                        try (Transaction tx = grid(0).transactions().txStart()) {
+                            grid(0).cache(DEFAULT_CACHE_NAME).putAll(generate(0, 100));
+
+                            lockLatch.countDown();
+
+                            U.awaitQuiet(unlockLatch);
+
+                            tx.commit();
+
+                            fail("Commit must fail");
+                        }
+                        catch (Exception ignored) {
+                            // No-op.
+                            assertTrue(X.hasCause(ignored, TransactionRollbackException.class));
+                        }
+
+                        break;
+                    case 1:
+                        U.awaitQuiet(lockLatch);
+
+                        doSleep(3000);
+
+                        try (Transaction tx = grid(0).transactions().withLabel("label1").txStart(PESSIMISTIC, READ_COMMITTED, Integer.MAX_VALUE, 0)) {
+                            grid(0).cache(DEFAULT_CACHE_NAME).putAll(generate(200, 110));
+
+                            grid(0).cache(DEFAULT_CACHE_NAME).put(0, 0);
+                        }
+
+                        break;
+                    case 2:
+                        try (Transaction tx = grid(1).transactions().txStart()) {
+                            U.awaitQuiet(lockLatch);
+
+                            grid(1).cache(DEFAULT_CACHE_NAME).put(0, 0);
+                        }
+
+                        break;
+                    case 3:
+                        try (Transaction tx = client.transactions().withLabel("label2").txStart(OPTIMISTIC, READ_COMMITTED, 0, 0)) {
+                            U.awaitQuiet(lockLatch);
+
+                            client.cache(DEFAULT_CACHE_NAME).putAll(generate(100, 10));
+
+                            client.cache(DEFAULT_CACHE_NAME).put(0, 0);
+
+                            tx.commit();
+                        }
+
+                        break;
+                }
+            }
+        }, 4, "tx-thread");
+
+        U.awaitQuiet(lockLatch);
+
+        doSleep(5000);
+
+        CommandHandler h = new CommandHandler();
+
+        final VisorTxInfo[] toKill = {null};
+
+        // Basic test.
+        validate(h, map -> {
+            ClusterNode node = grid(0).cluster().localNode();
+
+            VisorTxTaskResult res = map.get(node);
+
+            for (VisorTxInfo info : res.getInfos()) {
+                if (info.getSize() == 100) {
+                    toKill[0] = info;
+
+                    break;
+                }
+            }
+
+            assertEquals(3, map.size());
+        }, "--tx");
+
+        assertNotNull(toKill);
+
+        // Test filter by label.
+        validate(h, map -> {
+            ClusterNode node = grid(0).cluster().localNode();
+
+            for (Map.Entry<ClusterNode, VisorTxTaskResult> entry : map.entrySet())
+                assertEquals(entry.getKey().equals(node) ? 1 : 0, entry.getValue().getInfos().size());
+        }, "--tx", "label", "label1");
+
+        // Test filter by label regex.
+        validate(h, map -> {
+            ClusterNode node1 = grid(0).cluster().localNode();
+            ClusterNode node2 = grid("client").cluster().localNode();
+
+            for (Map.Entry<ClusterNode, VisorTxTaskResult> entry : map.entrySet()) {
+                if (entry.getKey().equals(node1)) {
+                    assertEquals(1, entry.getValue().getInfos().size());
+
+                    assertEquals("label1", entry.getValue().getInfos().get(0).getLabel());
+                }
+                else if (entry.getKey().equals(node2)) {
+                    assertEquals(1, entry.getValue().getInfos().size());
+
+                    assertEquals("label2", entry.getValue().getInfos().get(0).getLabel());
+                }
+                else
+                    assertTrue(entry.getValue().getInfos().isEmpty());
+
+            }
+        }, "--tx", "label", "^label[0-9]");
+
+        // test order by size.
+        validate(h, map -> {
+            VisorTxTaskResult res = map.get(grid(0).localNode());
+
+            assertTrue(res.getInfos().get(0).getSize() >=  res.getInfos().get(1).getSize());
+
+        }, "--tx", "order", "SIZE");
+
+        // test order by duration.
+        validate(h, map -> {
+            VisorTxTaskResult res = map.get(grid(0).localNode());
+
+            assertTrue(res.getInfos().get(0).getDuration() >=  res.getInfos().get(1).getDuration());
+
+        }, "--tx", "order", "DURATION");
+
+        // Test kill by xid.
+        validate(h, map -> {
+                assertEquals(1, map.size());
+
+                Map.Entry<ClusterNode, VisorTxTaskResult> killedEntry = map.entrySet().iterator().next();
+
+                VisorTxInfo info = killedEntry.getValue().getInfos().get(0);
+
+                assertEquals(toKill[0].getXid(), info.getXid());
+            }, "--tx", "kill",
+            "xid", toKill[0].getXid().toString(),
+            "nodes", grid(0).localNode().consistentId().toString());
+
+        unlockLatch.countDown();
+
+        fut.get();
+    }
+
+    /**
+     * @param h Handler.
+     * @param validateClo Validate clo.
+     * @param args Args.
+     */
+    private void validate(
+        CommandHandler h, IgniteInClosure<Map<ClusterNode, VisorTxTaskResult>> validateClo, String... args)
+        throws IgniteCheckedException {
+        assertEquals(EXIT_CODE_OK, execute(h, args));
+
+        validateClo.apply(h.getLastOperationResult());
+    }
+
+    /**
+     * @param from From.
+     * @param cnt Count.
+     */
+    private Map<Object, Object> generate(int from, int cnt) {
+        Map<Object, Object> map = new TreeMap<>();
+
+        for (int i = 0; i < cnt; i++ )
+            map.put(i + from, i + from);
+
+        return map;
+    }
 }
diff --git a/modules/jta/src/main/java/org/apache/ignite/internal/processors/cache/jta/CacheJtaManager.java b/modules/jta/src/main/java/org/apache/ignite/internal/processors/cache/jta/CacheJtaManager.java
index dd5f6b7e628f1..8dcdd574fa6b8 100644
--- a/modules/jta/src/main/java/org/apache/ignite/internal/processors/cache/jta/CacheJtaManager.java
+++ b/modules/jta/src/main/java/org/apache/ignite/internal/processors/cache/jta/CacheJtaManager.java
@@ -167,7 +167,8 @@ private CacheTmLookup createTmLookup(String tmLookupClsName) throws IgniteChecke
                                 tCfg.getDefaultTxIsolation(),
                                 tCfg.getDefaultTxTimeout(),
                                 /*store enabled*/true,
-                                /*tx size*/0
+                                /*tx size*/0,
+                                null
                             );
                         }
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
index b940d09bc23b7..2267540443abe 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/ClusterParityTest.cs
@@ -38,7 +38,8 @@ public class ClusterParityTest
         /** Members that are missing on .NET side and should be added in future. */
         private static readonly string[] MissingMembers =
         {
-            "enableStatistics"  // IGNITE-7276
+            "enableStatistics",  // IGNITE-7276
+            "setTxTimeoutOnPartitionMapExchange" // IGNITE-8075
         };
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionsParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionsParityTest.cs
index 7c195694929f9..e519ab358ec1d 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionsParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/TransactionsParityTest.cs
@@ -25,6 +25,13 @@ namespace Apache.Ignite.Core.Tests.ApiParity
     /// </summary>
     public class TransactionsParityTest
     {
+        /** Members that are missing on .NET side and should be added in future. */
+        private static readonly string[] MissingMembers =
+        {
+            "localActiveTransactions",  // IGNITE-8075
+            "withLabel"  // IGNITE-8075
+        };
+
         /// <summary>
         /// Tests the API parity.
         /// </summary>
@@ -33,7 +40,9 @@ public void TestTransactions()
         {
             ParityTest.CheckInterfaceParity(
                 @"modules\core\src\main\java\org\apache\ignite\IgniteTransactions.java",
-                typeof(ITransactions));
+                typeof(ITransactions),
+                null,
+                MissingMembers);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
index 3d0168c08d6cb..a414a2fc65fa7 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheAbstractTransactionalTest.cs
@@ -439,7 +439,6 @@ public void TestTxRollbackOnly()
             Assert.AreEqual(TransactionState.MarkedRollback, tx.State);
 
             var ex = Assert.Throws<TransactionRollbackException>(() => tx.Commit());
-            Assert.IsTrue(ex.Message.StartsWith("Invalid transaction state for prepare [state=MARKED_ROLLBACK"));
 
             tx.Dispose();
 
@@ -612,7 +611,7 @@ public void TestTransactionScopeSingleCache()
         }
 
         /// <summary>
-        /// Test Ignite transaction enlistment in ambient <see cref="TransactionScope"/> 
+        /// Test Ignite transaction enlistment in ambient <see cref="TransactionScope"/>
         /// with multiple participating caches.
         /// </summary>
         [Test]
@@ -668,7 +667,7 @@ public void TestTransactionScopeMultiCache([Values(true, false)] bool async)
         }
 
         /// <summary>
-        /// Test Ignite transaction enlistment in ambient <see cref="TransactionScope"/> 
+        /// Test Ignite transaction enlistment in ambient <see cref="TransactionScope"/>
         /// when Ignite tx is started manually.
         /// </summary>
         [Test]
@@ -680,7 +679,7 @@ public void TestTransactionScopeWithManualIgniteTx()
             cache[1] = 1;
 
             // When Ignite tx is started manually, it won't be enlisted in TransactionScope.
-            using (var tx = transactions.TxStart())            
+            using (var tx = transactions.TxStart())
             {
                 using (new TransactionScope())
                 {
@@ -747,7 +746,7 @@ public void TestNestedTransactionScope()
 
                     cache[1] = 5;
                 }
-                
+
                 // In case with Required option there is a single tx
                 // that gets aborted, second put executes outside the tx.
                 Assert.AreEqual(option == TransactionScopeOption.Required ? 5 : 3, cache[1], option.ToString());
@@ -927,4 +926,4 @@ public int Process(IMutableCacheEntry<int, int> entry, int arg)
             }
         }
     }
-}
\ No newline at end of file
+}

From 9cbe870642941a68a688d4a1654e7302a48e79bd Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Mon, 23 Apr 2018 19:39:01 +0300
Subject: [PATCH 110/543] IGNITE-6083 Null value may be passed to the entry
 processor on existing key

---
 .../distributed/near/GridNearTxLocal.java     |   3 +
 .../transactions/IgniteTxLocalAdapter.java    |   2 +
 ...CacheEntryProcessorSequentialCallTest.java | 311 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite.java      |   2 +
 4 files changed, 318 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/cache/IgniteCacheEntryProcessorSequentialCallTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 9c0b220000c0e..16653e0170f56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -1350,6 +1350,9 @@ private boolean enlistWriteEntry(GridCacheContext cacheCtx,
                         keepBinary,
                         CU.isNearEnabled(cacheCtx));
 
+                    if (op == TRANSFORM && txEntry.value() == null && old != null)
+                        txEntry.value(cacheCtx.toCacheObject(old), false, false);
+
                     if (enlisted != null)
                         enlisted.add(cacheKey);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 54b7ae2cbe630..61650ccefa393 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1152,6 +1152,8 @@ protected final void postLockWrite(
 
                         if (txEntry.op() == TRANSFORM) {
                             if (computeInvoke) {
+                                txEntry.readValue(v);
+
                                 GridCacheVersion ver;
 
                                 try {
diff --git a/modules/core/src/test/java/org/apache/ignite/cache/IgniteCacheEntryProcessorSequentialCallTest.java b/modules/core/src/test/java/org/apache/ignite/cache/IgniteCacheEntryProcessorSequentialCallTest.java
new file mode 100644
index 0000000000000..592449d053ed0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/cache/IgniteCacheEntryProcessorSequentialCallTest.java
@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.cache;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.transactions.TransactionOptimisticException;
+
+/**
+ */
+public class IgniteCacheEntryProcessorSequentialCallTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrids(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        CacheConfiguration cacheCfg = new CacheConfiguration("cache");
+
+        cacheCfg.setCacheMode(CacheMode.PARTITIONED);
+        cacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+        cacheCfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+        cacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+        cacheCfg.setMaxConcurrentAsyncOperations(0);
+        cacheCfg.setBackups(0);
+
+        cfg.setCacheConfiguration(cacheCfg);
+
+        return cfg;
+    }
+
+    /**
+     *
+     */
+    public void testOptimisticSerializableTxInvokeSequentialCall() throws Exception {
+        transactionInvokeSequentialCallOnPrimaryNode(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.SERIALIZABLE);
+
+        transactionInvokeSequentialCallOnNearNode(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.SERIALIZABLE);
+    }
+
+    /**
+     *
+     */
+    public void testOptimisticRepeatableReadTxInvokeSequentialCall() throws Exception {
+        transactionInvokeSequentialCallOnPrimaryNode(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.REPEATABLE_READ);
+
+        transactionInvokeSequentialCallOnNearNode(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.REPEATABLE_READ);
+    }
+
+    /**
+     *
+     */
+    public void testOptimisticReadCommittedTxInvokeSequentialCall() throws Exception {
+        transactionInvokeSequentialCallOnPrimaryNode(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
+
+        transactionInvokeSequentialCallOnNearNode(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
+    }
+
+    /**
+     *
+     */
+    public void testPessimisticSerializableTxInvokeSequentialCall() throws Exception {
+        transactionInvokeSequentialCallOnPrimaryNode(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.SERIALIZABLE);
+
+        transactionInvokeSequentialCallOnNearNode(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.SERIALIZABLE);
+    }
+
+    /**
+     *
+     */
+    public void testPessimisticRepeatableReadTxInvokeSequentialCall() throws Exception {
+        transactionInvokeSequentialCallOnPrimaryNode(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ);
+
+        transactionInvokeSequentialCallOnNearNode(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ);
+    }
+
+    /**
+     *
+     */
+    public void testPessimisticReadCommittedTxInvokeSequentialCall() throws Exception {
+        transactionInvokeSequentialCallOnPrimaryNode(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.READ_COMMITTED);
+
+        transactionInvokeSequentialCallOnNearNode(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.READ_COMMITTED);
+    }
+
+    /**
+     * Test for sequential entry processor invoking not null value on primary cache.
+     * In this test entry processor gets value from local node.
+     *
+     * @param transactionConcurrency Transaction concurrency.
+     * @param transactionIsolation Transaction isolation.
+     */
+    public void transactionInvokeSequentialCallOnPrimaryNode(TransactionConcurrency transactionConcurrency,
+        TransactionIsolation transactionIsolation) throws Exception {
+        TestKey key = new TestKey(1L);
+        TestValue val = new TestValue();
+        val.value("1");
+
+        Ignite primaryIgnite;
+
+        if (ignite(0).affinity("cache").isPrimary(ignite(0).cluster().localNode(), key))
+            primaryIgnite = ignite(0);
+        else
+            primaryIgnite = ignite(1);
+
+        IgniteCache<TestKey, TestValue> cache = primaryIgnite.cache("cache");
+
+        cache.put(key, val);
+
+        NotNullCacheEntryProcessor cacheEntryProcessor = new NotNullCacheEntryProcessor();
+
+        try (Transaction transaction = primaryIgnite.transactions().txStart(transactionConcurrency,
+            transactionIsolation)) {
+
+            cache.invoke(key, cacheEntryProcessor);
+            cache.invoke(key, cacheEntryProcessor);
+
+            transaction.commit();
+        }
+
+        cache.remove(key);
+    }
+
+    /**
+     * Test for sequential entry processor invoking not null value on near cache.
+     * In this test entry processor fetches value from remote node.
+     *
+     * @param transactionConcurrency Transaction concurrency.
+     * @param transactionIsolation Transaction isolation.
+     */
+    public void transactionInvokeSequentialCallOnNearNode(TransactionConcurrency transactionConcurrency,
+        TransactionIsolation transactionIsolation) throws Exception {
+        TestKey key = new TestKey(1L);
+        TestValue val = new TestValue();
+        val.value("1");
+
+        Ignite nearIgnite;
+        Ignite primaryIgnite;
+
+        if (ignite(0).affinity("cache").isPrimary(ignite(0).cluster().localNode(), key)) {
+            primaryIgnite = ignite(0);
+
+            nearIgnite = ignite(1);
+        }
+        else {
+            primaryIgnite = ignite(1);
+
+            nearIgnite = ignite(0);
+        }
+
+        primaryIgnite.cache("cache").put(key, val);
+
+        IgniteCache<TestKey, TestValue> nearCache = nearIgnite.cache("cache");
+
+        NotNullCacheEntryProcessor cacheEntryProcessor = new NotNullCacheEntryProcessor();
+
+        try (Transaction transaction = nearIgnite.transactions().txStart(transactionConcurrency,
+            transactionIsolation)) {
+
+            nearCache.invoke(key, cacheEntryProcessor);
+            nearCache.invoke(key, cacheEntryProcessor);
+
+            transaction.commit();
+        }
+
+        primaryIgnite.cache("cache").remove(key);
+    }
+
+    /**
+     * Test for sequential entry processor invocation. During transaction value is changed externally, which leads to
+     * optimistic conflict exception.
+     */
+    public void testTxInvokeSequentialOptimisticConflict() throws Exception {
+        TestKey key = new TestKey(1L);
+
+        IgniteCache<TestKey, TestValue> cache = ignite(0).cache("cache");
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        cache.put(key, new TestValue("1"));
+
+        multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    latch.await();
+                }
+                catch (InterruptedException e) {
+                    fail();
+                }
+
+                cache.put(key, new TestValue("2"));
+            }
+        }, 1);
+
+        Transaction tx = ignite(0).transactions().txStart(TransactionConcurrency.OPTIMISTIC, TransactionIsolation.SERIALIZABLE);
+
+        cache.invoke(key, new NotNullCacheEntryProcessor());
+
+        latch.countDown();
+
+        Thread.sleep(1_000);
+
+        cache.invoke(key, new NotNullCacheEntryProcessor());
+
+        GridTestUtils.assertThrowsWithCause(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                tx.commit();
+
+                return null;
+            }
+        }, TransactionOptimisticException.class);
+
+        cache.remove(key);
+    }
+
+    /**
+     * Cache entry processor checking whether entry has got non-null value.
+     */
+    public static class NotNullCacheEntryProcessor implements CacheEntryProcessor<TestKey, TestValue, Object> {
+        /** {@inheritDoc} */
+        public Object process(MutableEntry entry, Object... arguments) throws EntryProcessorException {
+            assertNotNull(entry.getValue());
+
+            return null;
+        }
+    }
+
+    /**
+     *
+     */
+    public static class TestKey {
+        /** Value. */
+        private final Long val;
+
+        /**
+         * @param val Value.
+         */
+        public TestKey(Long val) {
+            this.val = val;
+        }
+    }
+
+    /**
+     *
+     */
+    public static class TestValue {
+        /** Value. */
+        private String val;
+
+        /**
+         * Default constructor.
+         */
+        public TestValue() {
+        }
+
+        /**
+         * @param val Value.
+         */
+        public TestValue(String val) {
+            this.val = val;
+        }
+
+        /**
+         * @return Value.
+         */
+        public String value() {
+            return val;
+        }
+
+        /**
+         * @param val New value.
+         */
+        public void value(String val) {
+            this.val = val;
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 06126151271e9..1bf65e0bc7ad7 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -19,6 +19,7 @@
 
 import java.util.Set;
 import junit.framework.TestSuite;
+import org.apache.ignite.cache.IgniteCacheEntryProcessorSequentialCallTest;
 import org.apache.ignite.cache.IgniteWarmupClosureSelfTest;
 import org.apache.ignite.cache.store.CacheStoreReadFromBackupTest;
 import org.apache.ignite.cache.store.CacheTransactionalStoreReadFromBackupTest;
@@ -191,6 +192,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(IgniteCacheTxNearEnabledInvokeTest.class);
         suite.addTestSuite(IgniteCacheTxLocalInvokeTest.class);
         suite.addTestSuite(IgniteCrossCacheTxStoreSelfTest.class);
+        suite.addTestSuite(IgniteCacheEntryProcessorSequentialCallTest.class);
 
         // TODO GG-11148: include test when implemented.
         // Test fails due to incorrect handling of CacheConfiguration#getCopyOnRead() and

From 3425ef5e1d0e03a30a4060e5e3f3bcb0076c99c9 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Mon, 23 Apr 2018 20:37:09 +0300
Subject: [PATCH 111/543] IGNITE-7786 Added ping timeout and ping interval to
 control.sh

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../GridClientNioTcpConnection.java           | 12 ++--
 .../internal/commandline/Arguments.java       | 35 ++++++++++-
 .../internal/commandline/CommandHandler.java  | 62 ++++++++++++++++++-
 .../CommandHandlerParsingTest.java            | 29 +++++++--
 4 files changed, 123 insertions(+), 15 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
index 6e55df8ba2621..ecc1cdf32154a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/impl/connection/GridClientNioTcpConnection.java
@@ -212,11 +212,11 @@ public class GridClientNioTcpConnection extends GridClientConnection {
             sock.setTcpNoDelay(tcpNoDelay);
             sock.setKeepAlive(true);
 
-            final long startConnTime = U.currentTimeMillis();
+            final long startConnTime = System.currentTimeMillis();
 
             sock.connect(srvAddr, connectTimeout);
 
-            final long connTimeoutRest = connectTimeout - (U.currentTimeMillis() - startConnTime);
+            final long connTimeoutRest = connectTimeout - (System.currentTimeMillis() - startConnTime);
 
             GridClientFuture<?> handshakeFut = new GridClientFutureAdapter<>();
 
@@ -433,7 +433,7 @@ private <R> GridClientFutureAdapter<R> makeRequest(GridClientMessage msg, final
         assert msg != null;
 
         if (msg instanceof GridClientPingPacket) {
-            long now = U.currentTimeMillis();
+            long now = System.currentTimeMillis();
 
             if (Math.min(now, lastPingRcvTime) - lastPingSndTime >= pingTimeout)
                 close(FAILED, false,
@@ -472,7 +472,7 @@ else if (now - lastPingSndTime > pingInterval && lastPingRcvTime != Long.MAX_VAL
 
             GridNioFuture<?> sndFut = ses.send(msg);
 
-            lastMsgSndTime = U.currentTimeMillis();
+            lastMsgSndTime = System.currentTimeMillis();
 
             if (routeMode) {
                 sndFut.listen(new CI1<IgniteInternalFuture<?>>() {
@@ -506,7 +506,7 @@ else if (now - lastPingSndTime > pingInterval && lastPingRcvTime != Long.MAX_VAL
      * Handles ping response.
      */
     void handlePingResponse() {
-        lastPingRcvTime = U.currentTimeMillis();
+        lastPingRcvTime = System.currentTimeMillis();
     }
 
     /**
@@ -517,7 +517,7 @@ void handlePingResponse() {
      */
     @SuppressWarnings({"unchecked", "TooBroadScope"})
     void handleResponse(GridClientMessage res) throws IOException {
-        lastMsgRcvTime = U.currentTimeMillis();
+        lastMsgRcvTime = System.currentTimeMillis();
 
         TcpClientFuture fut = pendingReqs.get(res.requestId());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index 660edf4350ee4..d64e8ba515a08 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.commandline;
 
+import org.apache.ignite.internal.client.GridClientConfiguration;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
 
 /**
@@ -51,6 +52,12 @@ public class Arguments {
      */
     private String baselineArgs;
 
+    /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}.*/
+    private long pingTimeout;
+
+    /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}.*/
+    private long pingInterval;
+
     /** Transaction arguments. */
     private final VisorTxTaskArg txArg;
 
@@ -64,11 +71,13 @@ public class Arguments {
      * @param baselineArgs Baseline args.
      * @param txArg TX arg.
      * @param force Force flag.
+     * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
+     * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
+     * @param force Force flag.
      */
     public Arguments(Command cmd, String host, String port, String user, String pwd,
-        String baselineAct, String baselineArgs,
-        VisorTxTaskArg txArg, boolean force
-    ) {
+                     String baselineAct, String baselineArgs, long pingTimeout,
+                     long pingInterval, VisorTxTaskArg txArg, boolean force) {
         this.cmd = cmd;
         this.host = host;
         this.port = port;
@@ -76,6 +85,8 @@ public Arguments(Command cmd, String host, String port, String user, String pwd,
         this.pwd = pwd;
         this.baselineAct = baselineAct;
         this.baselineArgs = baselineArgs;
+        this.pingTimeout = pingTimeout;
+        this.pingInterval = pingInterval;
         this.force = force;
         this.txArg = txArg;
     }
@@ -129,6 +140,24 @@ public String baselineArguments() {
         return baselineArgs;
     }
 
+    /**
+     * See {@link GridClientConfiguration#pingTimeout}.
+     *
+     * @return Ping timeout.
+     */
+    public long pingTimeout() {
+        return pingTimeout;
+    }
+
+    /**
+     * See {@link GridClientConfiguration#pingInterval}.
+     *
+     * @return Ping interval.
+     */
+    public long pingInterval() {
+        return pingInterval;
+    }
+
     /**
      * @return Transaction arguments.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 3bc5c1a7e8c66..08488eacd4192 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -103,6 +103,12 @@ public class CommandHandler {
     /** */
     private static final String CMD_USER = "--user";
 
+    /** */
+    protected static final String CMD_PING_INTERVAL = "--ping-interval";
+
+    /** */
+    protected static final String CMD_PING_TIMEOUT = "--ping-timeout";
+
     /** */
     public static final String CONFIRM_MSG = "y";
 
@@ -142,6 +148,12 @@ public class CommandHandler {
     /** */
     public static final int EXIT_CODE_UNEXPECTED_ERROR = 4;
 
+    /** */
+    private static final long DFLT_PING_INTERVAL = 5000L;
+
+    /** */
+    private static final long DFLT_PING_TIMEOUT = 30_000L;
+
     /** */
     private static final Scanner IN = new Scanner(System.in);
 
@@ -702,7 +714,8 @@ private boolean isConnectionError(Throwable e) {
      */
     private void usage(String desc, Command cmd, String... args) {
         log(desc);
-        log("    control.sh [--host HOST_OR_IP] [--port PORT] [--user USER] [--password PASSWORD] " + cmd.text() + String.join("", args));
+        log("    control.sh [--host HOST_OR_IP] [--port PORT] [--user USER] [--password PASSWORD] " +
+                " [--ping-interval PING_INTERVAL] [--ping-timeout PING_TIMEOUT] " + cmd.text() + String.join("", args));
         nl();
     }
 
@@ -759,6 +772,10 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         String baselineArgs = "";
 
+        Long pingInterval = DFLT_PING_INTERVAL;
+
+        Long pingTimeout = DFLT_PING_TIMEOUT;
+
         boolean force = false;
 
         List<Command> commands = new ArrayList<>();
@@ -815,6 +832,7 @@ Arguments parseAndValidate(List<String> rawArgs) {
                 switch (str) {
                     case CMD_HOST:
                         host = nextArg("Expected host name");
+
                         break;
 
                     case CMD_PORT:
@@ -829,6 +847,17 @@ Arguments parseAndValidate(List<String> rawArgs) {
                         catch (NumberFormatException ignored) {
                             throw new IllegalArgumentException("Invalid value for port: " + port);
                         }
+
+                        break;
+
+                    case CMD_PING_INTERVAL:
+                        pingInterval = getPingParam("Expected ping interval", "Invalid value for ping interval");
+
+                        break;
+
+                    case CMD_PING_TIMEOUT:
+                        pingTimeout = getPingParam("Expected ping timeout", "Invalid value for ping timeout");
+
                         break;
 
                     case CMD_USER:
@@ -865,7 +894,30 @@ Arguments parseAndValidate(List<String> rawArgs) {
         if (hasUsr != hasPwd)
             throw new IllegalArgumentException("Both user and password should be specified");
 
-        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, force);
+        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs,
+            pingTimeout, pingInterval, txArgs, force);
+    }
+
+    /**
+     * Get ping param for grid client.
+     *
+     * @param nextArgErr Argument extraction error message.
+     * @param invalidErr Param validation error message.
+     */
+    private Long getPingParam(String nextArgErr, String invalidErr) {
+        String raw = nextArg(nextArgErr);
+
+        try {
+            long val = Long.valueOf(raw);
+
+            if (val <= 0)
+                throw new IllegalArgumentException(invalidErr + ": " + val);
+            else
+                return val;
+        }
+        catch (NumberFormatException ignored) {
+            throw new IllegalArgumentException(invalidErr + ": " + raw);
+        }
     }
 
     /**
@@ -1034,6 +1086,8 @@ public int execute(List<String> rawArgs) {
                 log("Default values:");
                 log("    HOST_OR_IP=" + DFLT_HOST);
                 log("    PORT=" + DFLT_PORT);
+                log("    PING_INTERVAL=" + DFLT_PING_INTERVAL);
+                log("    PING_TIMEOUT=" + DFLT_PING_TIMEOUT);
                 nl();
 
                 log("Exit codes:");
@@ -1056,6 +1110,10 @@ public int execute(List<String> rawArgs) {
 
             GridClientConfiguration cfg = new GridClientConfiguration();
 
+            cfg.setPingInterval(args.pingInterval());
+
+            cfg.setPingTimeout(args.pingTimeout());
+
             cfg.setServers(Collections.singletonList(args.host() + ":" + args.port()));
 
             if (!F.isEmpty(args.user())) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 049500abc6853..fda7e6054ffbc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -34,7 +34,7 @@
  */
 public class CommandHandlerParsingTest extends TestCase {
     /**
-     * Test parsing and validation for user and password arguments
+     * Test parsing and validation for user and password arguments.
      */
     public void testParseAndValidateUserAndPassword() {
         CommandHandler hnd = new CommandHandler();
@@ -85,9 +85,9 @@ public void testParseAndValidateUserAndPassword() {
     }
 
     /**
-     * tests host and port arguments
+     * Tests connection settings arguments.
      */
-    public void testHostAndPort() {
+    public void testConnectionSettings() {
         CommandHandler hnd = new CommandHandler();
 
         for (Command cmd : Command.values()) {
@@ -97,11 +97,14 @@ public void testHostAndPort() {
             assertEquals(DFLT_HOST, args.host());
             assertEquals(DFLT_PORT, args.port());
 
-            args = hnd.parseAndValidate(asList("--port", "12345", "--host", "test-host", cmd.text()));
+            args = hnd.parseAndValidate(asList("--port", "12345", "--host", "test-host", "--ping-interval", "5000",
+                    "--ping-timeout", "40000", cmd.text()));
 
             assertEquals(cmd, args.command());
             assertEquals("test-host", args.host());
             assertEquals("12345", args.port());
+            assertEquals(5000, args.pingInterval());
+            assertEquals(40000, args.pingTimeout());
 
             try {
                 hnd.parseAndValidate(asList("--port", "wrong-port", cmd.text()));
@@ -111,6 +114,24 @@ public void testHostAndPort() {
             catch (IllegalArgumentException e) {
                 e.printStackTrace();
             }
+
+            try {
+                hnd.parseAndValidate(asList("--ping-interval", "-10", cmd.text()));
+
+                fail("expected exception: Ping interval must be specified");
+            }
+            catch (IllegalArgumentException e) {
+                e.printStackTrace();
+            }
+
+            try {
+                hnd.parseAndValidate(asList("--ping-timeout", "-20", cmd.text()));
+
+                fail("expected exception: Ping timeout must be specified");
+            }
+            catch (IllegalArgumentException e) {
+                e.printStackTrace();
+            }
         }
     }
 

From 1eaa39c2fbbdcc116b39d11317d067da3cbca270 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 24 Apr 2018 10:42:55 +0300
Subject: [PATCH 112/543] IGNITE-8369 Fixed ZookeeperDiscoveryImpl to use
 j.u.c.ConcurrentHashMap

---
 .../spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 3010d1652f331..74670acfc9647 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -33,6 +33,7 @@
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
@@ -89,7 +90,6 @@
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.data.Stat;
-import org.jboss.netty.util.internal.ConcurrentHashMap;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;

From dd356ddbd9e37f78c8312d770ca8578a15d96ec9 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 24 Apr 2018 12:33:58 +0300
Subject: [PATCH 113/543] IGNITE-7786 Fixed javadoc

---
 .../java/org/apache/ignite/internal/commandline/Arguments.java   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index d64e8ba515a08..83a272b6ec12f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -73,7 +73,6 @@ public class Arguments {
      * @param force Force flag.
      * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
      * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
-     * @param force Force flag.
      */
     public Arguments(Command cmd, String host, String port, String user, String pwd,
                      String baselineAct, String baselineArgs, long pingTimeout,

From 0ad44bc27bc79630b03748b060aa2503e0830d31 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 24 Apr 2018 13:26:12 +0300
Subject: [PATCH 114/543] IGNITE-8369 Fixed ZkCommunicationErrorProcessFuture
 to use j.u.c.ConcurrentHashMap

---
 .../zk/internal/ZkCommunicationErrorProcessFuture.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
index accda6e5fb6c1..da064a535b4f7 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationErrorProcessFuture.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -35,7 +36,6 @@
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.spi.IgniteSpiTimeoutObject;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
-import org.jboss.netty.util.internal.ConcurrentHashMap;
 import org.jetbrains.annotations.Nullable;
 
 /**

From 198d6d2a6c66100a38212373e61014e746b364e3 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Tue, 24 Apr 2018 16:08:21 +0300
Subject: [PATCH 115/543] IGNITE-8339 Fixed partition state detection on
 activation - Fixes #3885.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../processors/cache/GridCacheContext.java    |   2 +-
 .../dht/GridDhtPartitionTopologyImpl.java     | 110 +++++++++--------
 .../GridDhtPartitionsExchangeFuture.java      |  10 +-
 ...ActivateDeactivateTestWithPersistence.java | 113 +++++++++++++++++-
 .../CacheLateAffinityAssignmentTest.java      |   2 +-
 ...IgnitePdsCacheRebalancingAbstractTest.java |   4 +-
 6 files changed, 180 insertions(+), 61 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index f194423320737..c5f4d3c9c993a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -2072,7 +2072,7 @@ public boolean reserveForFastLocalGet(int part, AffinityTopologyVersion topVer)
             topology().partitionState(localNodeId(), part) == OWNING :
             "result=" + result + ", persistenceEnabled=" + group().persistenceEnabled() +
                 ", partitionState=" + topology().partitionState(localNodeId(), part) +
-                ", replicated=" + isReplicated();
+                ", replicated=" + isReplicated() + ", part=" + part;
 
         return result;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 68104a5349283..71d43c8ebc55c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -370,10 +370,7 @@ private boolean initPartitions(AffinityTopologyVersion affVer, List<List<Cluster
                             GridDhtLocalPartition locPart = getOrCreatePartition(p);
 
                             if (shouldOwn) {
-                                boolean owned = locPart.own();
-
-                                assert owned : "Failed to own partition for oldest node [grp=" + grp.cacheOrGroupName() +
-                                    ", part=" + locPart + ']';
+                                locPart.own();
 
                                 if (log.isDebugEnabled())
                                     log.debug("Owned partition for oldest node [grp=" + grp.cacheOrGroupName() +
@@ -466,8 +463,6 @@ else if (localNode(p, aff))
         boolean affReady,
         boolean updateMoving)
         throws IgniteCheckedException {
-        ClusterNode loc = ctx.localNode();
-
         ctx.database().checkpointReadLock();
 
         try {
@@ -495,8 +490,6 @@ else if (localNode(p, aff))
                         lastTopChangeVer = readyTopVer = evts.topologyVersion();
                     }
 
-                    ClusterNode oldest = discoCache.oldestAliveServerNode();
-
                     if (log.isDebugEnabled()) {
                         log.debug("Partition map beforeExchange [grp=" + grp.cacheOrGroupName() +
                             ", exchId=" + exchFut.exchangeId() + ", fullMap=" + fullMapString() + ']');
@@ -506,46 +499,9 @@ else if (localNode(p, aff))
 
                     cntrMap.clear();
 
-                    boolean grpStarted = exchFut.cacheGroupAddedOnExchange(grp.groupId(), grp.receivedFrom());
-
-                    // If this is the oldest node.
-                    if (oldest != null && (loc.equals(oldest) || grpStarted)) {
-                        if (node2part == null) {
-                            node2part = new GridDhtPartitionFullMap(oldest.id(), oldest.order(), updateSeq);
-
-                            if (log.isDebugEnabled()) {
-                                log.debug("Created brand new full topology map on oldest node [" +
-                                    "grp=" + grp.cacheOrGroupName() + ", exchId=" + exchFut.exchangeId() +
-                                    ", fullMap=" + fullMapString() + ']');
-                            }
-                        }
-                        else if (!node2part.valid()) {
-                            node2part = new GridDhtPartitionFullMap(oldest.id(),
-                                oldest.order(),
-                                updateSeq,
-                                node2part,
-                                false);
-
-                            if (log.isDebugEnabled()) {
-                                log.debug("Created new full topology map on oldest node [" +
-                                    "grp=" +  grp.cacheOrGroupName() + ", exchId=" + exchFut.exchangeId() +
-                                    ", fullMap=" + node2part + ']');
-                            }
-                        }
-                        else if (!node2part.nodeId().equals(loc.id())) {
-                            node2part = new GridDhtPartitionFullMap(oldest.id(),
-                                oldest.order(),
-                                updateSeq,
-                                node2part,
-                                false);
+                    initializeFullMap(updateSeq);
 
-                            if (log.isDebugEnabled()) {
-                                log.debug("Copied old map into new map on oldest node (previous oldest node left) [" +
-                                    "grp=" + grp.cacheOrGroupName() + ", exchId=" + exchFut.exchangeId() +
-                                    ", fullMap=" + fullMapString() + ']');
-                            }
-                        }
-                    }
+                    boolean grpStarted = exchFut.cacheGroupAddedOnExchange(grp.groupId(), grp.receivedFrom());
 
                     if (evts.hasServerLeft()) {
                         List<DiscoveryEvent> evts0 = evts.events();
@@ -611,6 +567,61 @@ else if (!node2part.nodeId().equals(loc.id())) {
         }
     }
 
+    /**
+     * Initializes full map if current full map is empty or invalid in case of coordinator or cache groups start.
+     *
+     * @param updateSeq Update sequence to initialize full map.
+     */
+    private void initializeFullMap(long updateSeq) {
+        if (!(topReadyFut instanceof GridDhtPartitionsExchangeFuture))
+            return;
+
+        GridDhtPartitionsExchangeFuture exchFut = (GridDhtPartitionsExchangeFuture) topReadyFut;
+
+        boolean grpStarted = exchFut.cacheGroupAddedOnExchange(grp.groupId(), grp.receivedFrom());
+
+        ClusterNode oldest = discoCache.oldestAliveServerNode();
+
+        // If this is the oldest node.
+        if (oldest != null && (ctx.localNode().equals(oldest) || grpStarted)) {
+            if (node2part == null) {
+                node2part = new GridDhtPartitionFullMap(oldest.id(), oldest.order(), updateSeq);
+
+                if (log.isDebugEnabled()) {
+                    log.debug("Created brand new full topology map on oldest node [" +
+                        "grp=" + grp.cacheOrGroupName() + ", exchId=" + exchFut.exchangeId() +
+                        ", fullMap=" + fullMapString() + ']');
+                }
+            }
+            else if (!node2part.valid()) {
+                node2part = new GridDhtPartitionFullMap(oldest.id(),
+                    oldest.order(),
+                    updateSeq,
+                    node2part,
+                    false);
+
+                if (log.isDebugEnabled()) {
+                    log.debug("Created new full topology map on oldest node [" +
+                        "grp=" +  grp.cacheOrGroupName() + ", exchId=" + exchFut.exchangeId() +
+                        ", fullMap=" + node2part + ']');
+                }
+            }
+            else if (!node2part.nodeId().equals(ctx.localNode().id())) {
+                node2part = new GridDhtPartitionFullMap(oldest.id(),
+                    oldest.order(),
+                    updateSeq,
+                    node2part,
+                    false);
+
+                if (log.isDebugEnabled()) {
+                    log.debug("Copied old map into new map on oldest node (previous oldest node left) [" +
+                        "grp=" + grp.cacheOrGroupName() + ", exchId=" + exchFut.exchangeId() +
+                        ", fullMap=" + fullMapString() + ']');
+                }
+            }
+        }
+    }
+
     /**
      * @param p Partition number.
      * @param topVer Topology version.
@@ -625,11 +636,10 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
         lock.writeLock().lock();
 
         try {
-            if (node2part == null)
-                return;
-
             long updateSeq = this.updateSeq.incrementAndGet();
 
+            initializeFullMap(updateSeq);
+
             for (int p = 0; p < grp.affinity().partitions(); p++) {
                 GridDhtLocalPartition locPart = locParts.get(p);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index c8e0034aa4536..56907b2bf5b2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1127,6 +1127,12 @@ private void distributedExchange() throws IgniteCheckedException {
             }
         }
 
+        /* It is necessary to run database callback before all topology callbacks.
+           In case of persistent store is enabled we first restore partitions presented on disk.
+           We need to guarantee that there are no partition state changes logged to WAL before this callback
+           to make sure that we correctly restored last actual states. */
+        cctx.database().beforeExchange(this);
+
         if (!exchCtx.mergeExchanges()) {
             for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                 if (grp.isLocal() || cacheGroupStopping(grp.groupId()))
@@ -1138,10 +1144,6 @@ private void distributedExchange() throws IgniteCheckedException {
             }
         }
 
-        // It is necessary to run database callback after all topology callbacks, so partition states could be
-        // correctly restored from the persistent store.
-        cctx.database().beforeExchange(this);
-
         changeWalModeIfNeeded();
 
         if (crd.isLocal()) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
index 8bae136107ca0..5a403b8bca44f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistence.java
@@ -20,18 +20,27 @@
 import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Assert;
 
 /**
  *
@@ -131,7 +140,7 @@ private void activateCachesRestore(int srvs, boolean withNewCaches) throws Excep
 
         checkNoCaches(srvs);
 
-        srv.active(true);
+        srv.cluster().active(true);
 
         final int CACHES = withNewCaches ? 4 : 2;
 
@@ -212,7 +221,7 @@ public void testClientJoinsWhenActivationIsInProgress() throws Exception {
         }, "client-starter-thread");
 
         clientStartLatch.countDown();
-        srv.active(true);
+        srv.cluster().active(true);
 
         clStartFut.get();
     }
@@ -242,7 +251,7 @@ public void testActivateCacheRestoreConfigurationConflict() throws Exception {
 
         Ignite srv = startGrids(SRVS);
 
-        srv.active(true);
+        srv.cluster().active(true);
 
         CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
 
@@ -265,4 +274,102 @@ public void testActivateCacheRestoreConfigurationConflict() throws Exception {
             assertTrue(X.getCause(e).getMessage().contains("Failed to start configured cache."));
         }
     }
+
+    /**
+     * Test that after deactivation during eviction and rebalance and activation again after
+     * all data in cache is consistent.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDeactivateDuringEvictionAndRebalance() throws Exception {
+        IgniteEx srv = (IgniteEx) startGrids(3);
+
+        srv.cluster().active(true);
+
+        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME)
+            .setBackups(1)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setIndexedTypes(Integer.class, Integer.class)
+            .setAffinity(new RendezvousAffinityFunction(false, 64));
+
+        IgniteCache cache = srv.createCache(ccfg);
+
+        // High number of keys triggers long partition eviction.
+        final int keysCount = 100_000;
+
+        try (IgniteDataStreamer ds = srv.dataStreamer(DEFAULT_CACHE_NAME)) {
+            log.info("Writing initial data...");
+
+            ds.allowOverwrite(true);
+            for (int k = 1; k <= keysCount; k++) {
+                ds.addData(k, k);
+
+                if (k % 50_000 == 0)
+                    log.info("Written " + k + " entities.");
+            }
+
+            log.info("Writing initial data finished.");
+        }
+
+        AtomicInteger keyCounter = new AtomicInteger(keysCount);
+        AtomicBoolean stop = new AtomicBoolean(false);
+
+        Set<Integer> addedKeys = new GridConcurrentHashSet<>();
+
+        IgniteInternalFuture cacheLoadFuture = GridTestUtils.runMultiThreadedAsync(() -> {
+            while (!stop.get()) {
+                int key = keyCounter.incrementAndGet();
+                try {
+                    cache.put(key, key);
+
+                    addedKeys.add(key);
+
+                    Thread.sleep(10);
+                }
+                catch (Exception ignored) { }
+            }
+        }, 2, "cache-load");
+
+        stopGrid(2);
+
+        // Wait for some data.
+        Thread.sleep(3000);
+
+        startGrid(2);
+
+        log.info("Stop load...");
+
+        stop.set(true);
+
+        cacheLoadFuture.get();
+
+        // Deactivate and activate again.
+        srv.cluster().active(false);
+
+        srv.cluster().active(true);
+
+        awaitPartitionMapExchange();
+
+        log.info("Checking data...");
+
+        for (Ignite ignite : G.allGrids()) {
+            IgniteCache cache1 = ignite.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+            for (int k = 1; k <= keysCount; k++) {
+                Object val = cache1.get(k);
+
+                Assert.assertNotNull("node=" + ignite.name() + ", key=" + k, val);
+
+                Assert.assertTrue("node=" + ignite.name() + ", key=" + k + ", val=" + val, (int) val == k);
+            }
+
+            for (int k : addedKeys) {
+                Object val = cache1.get(k);
+
+                Assert.assertNotNull("node=" + ignite.name() + ", key=" + k, val);
+
+                Assert.assertTrue("node=" + ignite.name() + ", key=" + k + ", val=" + val, (int) val == k);
+            }
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 787095e605cf1..0b6fd83c60c26 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -186,7 +186,7 @@ public class CacheLateAffinityAssignmentTest extends GridCommonAbstractTest {
 
         DataStorageConfiguration cfg1 = new DataStorageConfiguration();
 
-        cfg1.setDefaultDataRegionConfiguration(new DataRegionConfiguration().setMaxSize(150 * 1024 * 1024L));
+        cfg1.setDefaultDataRegionConfiguration(new DataRegionConfiguration().setMaxSize(512 * 1024 * 1024L));
 
         cfg.setDataStorageConfiguration(cfg1);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
index 0dd9c788b7f6c..c0c2be9f86943 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -584,7 +584,7 @@ public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception
         try (IgniteDataStreamer ds = ig.dataStreamer(cacheName)) {
             ds.allowOverwrite(true);
 
-            for (int k0 = k; k < k0 + 50_000; k++)
+            for (int k0 = k; k < k0 + 10_000; k++)
                 ds.addData(k, k);
         }
 
@@ -631,7 +631,7 @@ public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception
                     for (;k < k0 + 3; k++)
                         ds.addData(k, k);
 
-                    U.sleep(1);
+                    U.sleep(10);
                 }
             }
             catch (Exception e) {

From b4bf53f81a4ed06b61ca207ce33ba5e859e6ce40 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 24 Apr 2018 20:57:56 +0300
Subject: [PATCH 116/543] IGNITE-8082 Fixed build (added javadoc)

---
 .../spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java
index ef4c37e0766b3..082203a383f42 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryStatistics.java
@@ -17,7 +17,7 @@
 package org.apache.ignite.spi.discovery.zk.internal;
 
 /**
- *
+ * Zookeeper discovery statistics.
  */
 public class ZookeeperDiscoveryStatistics {
     /** */

From b96445247a547e538d923f6898b85bde1b8b17b9 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Wed, 25 Apr 2018 12:30:09 +0300
Subject: [PATCH 117/543] ignite-2.5.1 Fixed version for backward compatibility

---
 .../dht/preloader/GridDhtPartitionDemandMessage.java            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
index 4fba9179bcfe2..5044b915ccc60 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
@@ -40,7 +40,7 @@ public class GridDhtPartitionDemandMessage extends GridCacheGroupIdMessage {
     private static final long serialVersionUID = 0L;
 
     /** */
-    public static final IgniteProductVersion VERSION_SINCE = IgniteProductVersion.fromString("2.4.0");
+    public static final IgniteProductVersion VERSION_SINCE = IgniteProductVersion.fromString("2.4.2");
 
     /** Rebalance id. */
     private long rebalanceId;

From f6cb457a713bd482b202f98a795f711aa9243638 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Wed, 25 Apr 2018 14:57:15 +0300
Subject: [PATCH 118/543] ignite-2.5.1 Fixed version for backward compatibility

---
 .../dht/preloader/GridDhtPartitionDemandMessage.java            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
index 5044b915ccc60..0d2cc4bebf5fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
@@ -40,7 +40,7 @@ public class GridDhtPartitionDemandMessage extends GridCacheGroupIdMessage {
     private static final long serialVersionUID = 0L;
 
     /** */
-    public static final IgniteProductVersion VERSION_SINCE = IgniteProductVersion.fromString("2.4.2");
+    public static final IgniteProductVersion VERSION_SINCE = IgniteProductVersion.fromString("2.4.4");
 
     /** Rebalance id. */
     private long rebalanceId;

From 6c87080156c39f7dd70692df2f46eff8b9846a8e Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 25 Apr 2018 17:39:02 +0300
Subject: [PATCH 119/543] IGNITE-8358 Destroy partition inside evictor to
 prevent possible deadlock. This closes #3911.

---
 .../dht/GridDhtLocalPartition.java            | 21 ++++++++++++-------
 .../dht/GridDhtPartitionsEvictor.java         |  6 +++++-
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index ea20dbf4b1184..be74eff5cf674 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -31,7 +31,6 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
@@ -164,6 +163,9 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
     /** Set if partition must be cleared in MOVING state. */
     private volatile boolean clear;
 
+    /** Set if topology update sequence should be updated on partition destroy. */
+    private boolean updateSeqOnDestroy;
+
     /**
      * @param ctx Context.
      * @param grp Cache group.
@@ -664,6 +666,12 @@ && getReservations(state) == 0 && !groupReserved()) {
             if (partState == RENTING && casState(state, EVICTED) || clearingRequested) {
                 clearFuture.finish();
 
+                if (state() == EVICTED && markForDestroy()) {
+                    updateSeqOnDestroy = updateSeq;
+
+                    destroy();
+                }
+
                 return;
             }
         }
@@ -744,7 +752,7 @@ private boolean clearEvicting() {
     /**
      * @return {@code True} if partition is safe to destroy.
      */
-    private boolean markForDestroy() {
+    public boolean markForDestroy() {
         while (true) {
             int cnt = evictGuard.get();
 
@@ -771,17 +779,14 @@ private void finishEviction(boolean updateSeq) {
             if (log.isDebugEnabled())
                 log.debug("Evicted partition: " + this);
 
-            if (markForDestroy())
-                finishDestroy(updateSeq);
+            updateSeqOnDestroy = updateSeq;
         }
     }
 
     /**
      * Destroys partition data store and invokes appropriate callbacks.
-     *
-     * @param updateSeq If {@code true} increment update sequence on cache group topology after successful destroy.
      */
-    private void finishDestroy(boolean updateSeq) {
+    public void destroy() {
         assert state() == EVICTED : this;
         assert evictGuard.get() == -1;
 
@@ -791,7 +796,7 @@ private void finishDestroy(boolean updateSeq) {
 
         rent.onDone();
 
-        ((GridDhtPreloader)grp.preloader()).onPartitionEvicted(this, updateSeq);
+        ((GridDhtPreloader)grp.preloader()).onPartitionEvicted(this, updateSeqOnDestroy);
 
         clearDeferredDeletes();
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java
index 3e51ff3065385..2a289212ca30c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java
@@ -94,8 +94,12 @@ public void evictPartitionAsync(GridDhtLocalPartition part) {
                                 try {
                                     boolean success = part.tryClear();
 
-                                    if (success)
+                                    if (success) {
                                         evictionQueue.remove(part.id());
+
+                                        if (part.state() == GridDhtPartitionState.EVICTED && part.markForDestroy())
+                                            part.destroy();
+                                    }
                                 }
                                 catch (Throwable ex) {
                                     if (ctx.kernalContext().isStopping()) {

From 3b93a70025f0295f1d016b981603592d3ba7a404 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Wed, 25 Apr 2018 18:40:46 +0300
Subject: [PATCH 120/543] IGNITE-8374 Fixed hanging of state transition
 completeion on cluster().activate()

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../src/main/java/org/apache/ignite/internal/IgniteKernal.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index f4c7ccea9eab0..dc1307cd18dc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2024,7 +2024,7 @@ private void ackStart(RuntimeMXBean rtBean) {
             log.info(str);
         }
 
-        if (!cluster().active()) {
+        if (!ctx.state().clusterState().active()) {
             U.quietAndInfo(log, ">>> Ignite cluster is not active (limited functionality available). " +
                 "Use control.(sh|bat) script or IgniteCluster interface to activate.");
         }

From bae07ead40dea7f1a44688f0567f265ca6b20294 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Wed, 25 Apr 2018 20:59:25 +0300
Subject: [PATCH 121/543] IGNITE-8372 ZookeeperClusterNode was made
 Externalizable to preserve serializing local node's metrics - Fixes #3907.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../zk/internal/ZookeeperClusterNode.java     | 56 ++++++++++++++++++-
 .../zk/internal/ZookeeperDiscoveryImpl.java   |  5 --
 2 files changed, 55 insertions(+), 6 deletions(-)

diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
index 3cb5fad3129cf..2fe3052fa9d23 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
@@ -17,6 +17,10 @@
 
 package org.apache.ignite.spi.discovery.zk.internal;
 
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
 import java.io.Serializable;
 import java.util.Collection;
 import java.util.Collections;
@@ -25,6 +29,7 @@
 import java.util.UUID;
 import org.apache.ignite.cache.CacheMetrics;
 import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.internal.ClusterMetricsSnapshot;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
@@ -41,7 +46,7 @@
 /**
  * Zookeeper Cluster Node.
  */
-public class ZookeeperClusterNode implements IgniteClusterNode, Serializable, Comparable<ZookeeperClusterNode> {
+public class ZookeeperClusterNode implements IgniteClusterNode, Externalizable, Comparable<ZookeeperClusterNode> {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -99,6 +104,11 @@ public class ZookeeperClusterNode implements IgniteClusterNode, Serializable, Co
     @GridToStringExclude
     private transient volatile boolean daemonInit;
 
+    /** */
+    public ZookeeperClusterNode() {
+        //No-op
+    }
+
     /**
      * @param id Node ID.
      * @param addrs Node addresses.
@@ -325,6 +335,50 @@ public void local(boolean loc) {
         return (CLIENT_NODE_MASK & flags) != 0;
     }
 
+    /** {@inheritDoc} */
+    @Override public void writeExternal(ObjectOutput out) throws IOException {
+        U.writeUuid(out, id);
+        out.writeObject(consistentId);
+        out.writeLong(internalId);
+        out.writeLong(order);
+        out.writeObject(ver);
+        U.writeMap(out, attrs);
+        U.writeCollection(out, addrs);
+        U.writeCollection(out, hostNames);
+        out.writeLong(sesTimeout);
+        out.writeByte(flags);
+
+        // Cluster metrics
+        byte[] mtr = null;
+
+        ClusterMetrics metrics = this.metrics;
+
+        if (metrics != null)
+            mtr = ClusterMetricsSnapshot.serialize(metrics);
+
+        U.writeByteArray(out, mtr);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        id = U.readUuid(in);
+        consistentId = (Serializable) in.readObject();
+        internalId = in.readLong();
+        order = in.readLong();
+        ver = (IgniteProductVersion) in.readObject();
+        attrs = U.sealMap(U.readMap(in));
+        addrs = U.readCollection(in);
+        hostNames = U.readCollection(in);
+        sesTimeout = in.readLong();
+        flags = in.readByte();
+
+        // Cluster metrics
+        byte[] mtr = U.readByteArray(in);
+
+        if (mtr != null)
+            metrics = ClusterMetricsSnapshot.deserialize(mtr, 0);
+    }
+
     /** {@inheritDoc} */
     @Override public int compareTo(@Nullable ZookeeperClusterNode node) {
         if (node == null)
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 74670acfc9647..0604458e461b6 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -51,7 +51,6 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CommunicationFailureResolver;
 import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.ClusterMetricsSnapshot;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -2955,8 +2954,6 @@ private void processLocalJoin(ZkDiscoveryEventsData evtsData,
                 if (node.order() >= locNode.order())
                     break;
 
-                node.setMetrics(new ClusterMetricsSnapshot());
-
                 rtState.top.addNode(node);
             }
 
@@ -3448,8 +3445,6 @@ private void notifyNodeJoin(ZkJoinedNodeEvtData joinedEvtData, ZkJoiningNodeData
         joinedNode.order(joinedEvtData.topVer);
         joinedNode.internalId(joinedEvtData.joinedInternalId);
 
-        joinedNode.setMetrics(new ClusterMetricsSnapshot());
-
         rtState.top.addNode(joinedNode);
 
         final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();

From 3f62a454a27588b49983c6e00a17568d7a2db634 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 26 Apr 2018 16:06:52 +0300
Subject: [PATCH 122/543] IGNITE-8390 Correct assertion for historical
 rebalance - Fixes #3917.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../preloader/GridDhtPartitionSupplier.java   |  2 +-
 .../GridDhtPartitionSupplyMessage.java        |  5 +-
 .../db/wal/IgniteWalRebalanceTest.java        | 63 +++++++++++++++++--
 3 files changed, 62 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index a3ee305406207..84e6828df4dc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -378,7 +378,7 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                 info.cacheId(row.cacheId());
 
                 if (preloadPred == null || preloadPred.apply(info))
-                    s.addEntry0(part, info, grp.shared(), grp.cacheObjectContext());
+                    s.addEntry0(part, iter.historical(part), info, grp.shared(), grp.cacheObjectContext());
                 else {
                     if (log.isDebugEnabled())
                         log.debug("Rebalance predicate evaluated to false (will not send " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
index 4ae5acdbaf2c9..77baa38cc52c3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
@@ -209,15 +209,16 @@ int messageSize() {
 
     /**
      * @param p Partition.
+     * @param historical {@code True} if partition rebalancing using WAL history.
      * @param info Entry to add.
      * @param ctx Cache shared context.
      * @param cacheObjCtx Cache object context.
      * @throws IgniteCheckedException If failed.
      */
-    void addEntry0(int p, GridCacheEntryInfo info, GridCacheSharedContext ctx, CacheObjectContext cacheObjCtx) throws IgniteCheckedException {
+    void addEntry0(int p, boolean historical, GridCacheEntryInfo info, GridCacheSharedContext ctx, CacheObjectContext cacheObjCtx) throws IgniteCheckedException {
         assert info != null;
         assert info.key() != null : info;
-        assert info.value() != null : info;
+        assert info.value() != null || historical : info;
 
         // Need to call this method to initialize info properly.
         marshalInfo(info, ctx, cacheObjCtx);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
index 6387dac1a3ef0..23dda265fb5eb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
@@ -17,7 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.db.wal;
 
-import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
@@ -30,6 +30,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -48,6 +49,8 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
 
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
+        cfg.setConsistentId(gridName);
+
         CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(CACHE_NAME);
 
         ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
@@ -87,9 +90,11 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
     }
 
     /**
+     * Test simple WAL historical rebalance.
+     *
      * @throws Exception if failed.
      */
-    public void test() throws Exception {
+    public void testSimple() throws Exception {
         IgniteEx ig0 = startGrid(0);
         IgniteEx ig1 = startGrid(1);
         final int entryCnt = 10_000;
@@ -112,12 +117,60 @@ public void test() throws Exception {
 
         ig1 = startGrid(1);
 
-        IgniteCache<Object, Object> cache1 = ig1.cache(CACHE_NAME);
+        awaitPartitionMapExchange();
+
+        for (Ignite ig : G.allGrids()) {
+            IgniteCache<Object, Object> cache1 = ig.cache(CACHE_NAME);
+
+            for (int k = 0; k < entryCnt; k++)
+                assertEquals(new IndexedObject(k + 1), cache1.get(k));
+        }
+    }
+
+    /**
+     * Test that cache entry removes are rebalanced properly using WAL.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRebalanceRemoves() throws Exception {
+        IgniteEx ig0 = startGrid(0);
+        IgniteEx ig1 = startGrid(1);
+        final int entryCnt = 10_000;
+
+        ig0.cluster().active(true);
 
-        cache1.rebalance().get(2, TimeUnit.MINUTES);
+        IgniteCache<Object, Object> cache = ig0.cache(CACHE_NAME);
 
         for (int k = 0; k < entryCnt; k++)
-            assertEquals(new IndexedObject(k + 1), cache.get(k));
+            cache.put(k, new IndexedObject(k));
+
+        forceCheckpoint();
+
+        stopGrid(1, false);
+
+        for (int k = 0; k < entryCnt; k++) {
+            if (k % 3 != 2)
+                cache.put(k, new IndexedObject(k + 1));
+            else // Spread removes across all partitions.
+                cache.remove(k);
+        }
+
+        forceCheckpoint();
+
+        ig1 = startGrid(1);
+
+        awaitPartitionMapExchange();
+
+        for (Ignite ig : G.allGrids()) {
+            IgniteCache<Object, Object> cache1 = ig.cache(CACHE_NAME);
+
+            for (int k = 0; k < entryCnt; k++) {
+                if (k % 3 != 2)
+                    assertEquals(new IndexedObject(k + 1), cache1.get(k));
+                else
+                    assertNull(cache1.get(k));
+            }
+        }
     }
 
     /**

From 2f2ad0fd35c2928fb117ad8f0b0b5e05ff8a9074 Mon Sep 17 00:00:00 2001
From: Ivan Rakov <ivan.glukos@gmail.com>
Date: Thu, 26 Apr 2018 18:57:38 +0300
Subject: [PATCH 123/543] IGNITE-8277 Added utilities to check and display
 cache info

---
 .../internal/commandline/Arguments.java       |  21 +-
 .../ignite/internal/commandline/Command.java  |   5 +-
 .../internal/commandline/CommandHandler.java  | 433 +++++++++++++++++-
 .../commandline/cache/CacheArguments.java     | 163 +++++++
 .../commandline/cache/CacheCommand.java       |  93 ++++
 .../processors/cache/verify/CacheInfo.java    | 322 +++++++++++++
 .../cache/verify/ContentionClosure.java       | 162 +++++++
 .../cache/verify/ContentionInfo.java          |  72 +++
 .../cache/verify/ViewCacheClosure.java        | 185 ++++++++
 .../visor/verify/IndexValidationIssue.java    |  88 ++++
 .../ValidateIndexesPartitionResult.java       | 145 ++++++
 .../verify/VisorContentionJobResult.java      |  80 ++++
 .../visor/verify/VisorContentionTask.java     | 100 ++++
 .../visor/verify/VisorContentionTaskArg.java  |  84 ++++
 .../verify/VisorContentionTaskResult.java     | 100 ++++
 .../visor/verify/VisorIdleAnalyzeTask.java    | 124 +++++
 .../visor/verify/VisorIdleAnalyzeTaskArg.java |  88 ++++
 .../verify/VisorIdleAnalyzeTaskResult.java    |  76 +++
 .../visor/verify/VisorIdleVerifyTask.java     |  97 ++++
 .../visor/verify/VisorIdleVerifyTaskArg.java  |  74 +++
 .../verify/VisorIdleVerifyTaskResult.java     |  76 +++
 .../verify/VisorValidateIndexesJobResult.java |  73 +++
 .../verify/VisorValidateIndexesTaskArg.java   |  74 +++
 .../VisorValidateIndexesTaskResult.java       |  88 ++++
 .../visor/verify/VisorViewCacheCmd.java       |  47 ++
 .../visor/verify/VisorViewCacheTask.java      |  75 +++
 .../visor/verify/VisorViewCacheTaskArg.java   |  86 ++++
 .../verify/VisorViewCacheTaskResult.java      |  74 +++
 .../resources/META-INF/classnames.properties  | 114 ++++-
 .../ignite/util/GridCommandHandlerTest.java   | 312 +++++++++++--
 .../visor/verify/ValidateIndexesClosure.java  | 356 ++++++++++++++
 .../verify/VisorValidateIndexesTask.java      |  99 ++++
 .../util/GridCommandHandlerIndexingTest.java  | 121 +++++
 33 files changed, 4026 insertions(+), 81 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ContentionClosure.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ContentionInfo.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ViewCacheClosure.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexValidationIssue.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionJobResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTaskResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTaskResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheCmd.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskResult.java
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index 83a272b6ec12f..ce7269378fc20 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.commandline;
 
 import org.apache.ignite.internal.client.GridClientConfiguration;
+import org.apache.ignite.internal.commandline.cache.CacheArguments;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
 
 /**
@@ -61,6 +62,11 @@ public class Arguments {
     /** Transaction arguments. */
     private final VisorTxTaskArg txArg;
 
+    /**
+     * Arguments for --cache subcommand.
+     */
+    private CacheArguments cacheArgs;
+
     /**
      * @param cmd Command.
      * @param host Host.
@@ -73,10 +79,11 @@ public class Arguments {
      * @param force Force flag.
      * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
      * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
+     * @param cacheArgs --cache subcommand arguments.
      */
-    public Arguments(Command cmd, String host, String port, String user, String pwd,
-                     String baselineAct, String baselineArgs, long pingTimeout,
-                     long pingInterval, VisorTxTaskArg txArg, boolean force) {
+    public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
+        String baselineArgs, long pingTimeout, long pingInterval, VisorTxTaskArg txArg, boolean force,
+        CacheArguments cacheArgs) {
         this.cmd = cmd;
         this.host = host;
         this.port = port;
@@ -88,6 +95,7 @@ public Arguments(Command cmd, String host, String port, String user, String pwd,
         this.pingInterval = pingInterval;
         this.force = force;
         this.txArg = txArg;
+        this.cacheArgs = cacheArgs;
     }
 
     /**
@@ -170,4 +178,11 @@ public VisorTxTaskArg transactionArguments() {
     public boolean force() {
         return force;
     }
+
+    /**
+     * @return Arguments for --cache subcommand.
+     */
+    public CacheArguments cacheArgs() {
+        return cacheArgs;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
index c8c7db5cafad5..52098d629fb0d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
@@ -34,7 +34,10 @@ public enum Command {
     BASELINE("--baseline"),
 
     /** */
-    TX("--tx");
+    TX("--tx"),
+
+    /** */
+    CACHE("--cache");
 
     /** */
     private final String text;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 08488eacd4192..ff05684bb47a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -19,15 +19,19 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
+import java.util.Set;
 import java.util.UUID;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
+import java.util.stream.Collectors;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.client.GridClient;
 import org.apache.ignite.internal.client.GridClientAuthenticationException;
@@ -42,6 +46,12 @@
 import org.apache.ignite.internal.client.GridClientNode;
 import org.apache.ignite.internal.client.GridServerUnreachableException;
 import org.apache.ignite.internal.client.impl.connection.GridClientConnectionResetException;
+import org.apache.ignite.internal.commandline.cache.CacheArguments;
+import org.apache.ignite.internal.commandline.cache.CacheCommand;
+import org.apache.ignite.internal.processors.cache.verify.CacheInfo;
+import org.apache.ignite.internal.processors.cache.verify.ContentionInfo;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -58,6 +68,20 @@
 import org.apache.ignite.internal.visor.tx.VisorTxTask;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
+import org.apache.ignite.internal.visor.verify.IndexValidationIssue;
+import org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult;
+import org.apache.ignite.internal.visor.verify.VisorContentionTask;
+import org.apache.ignite.internal.visor.verify.VisorContentionTaskArg;
+import org.apache.ignite.internal.visor.verify.VisorContentionTaskResult;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult;
+import org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult;
+import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskArg;
+import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskResult;
+import org.apache.ignite.internal.visor.verify.VisorViewCacheTask;
+import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg;
+import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskResult;
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
@@ -66,6 +90,7 @@
 import static org.apache.ignite.internal.IgniteVersionUtils.COPYRIGHT;
 import static org.apache.ignite.internal.commandline.Command.ACTIVATE;
 import static org.apache.ignite.internal.commandline.Command.BASELINE;
+import static org.apache.ignite.internal.commandline.Command.CACHE;
 import static org.apache.ignite.internal.commandline.Command.DEACTIVATE;
 import static org.apache.ignite.internal.commandline.Command.STATE;
 import static org.apache.ignite.internal.commandline.Command.TX;
@@ -74,6 +99,8 @@
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.REMOVE;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.SET;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.VERSION;
+import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.GROUPS;
+import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.SEQ;
 
 /**
  * Class that execute several commands passed via command line.
@@ -103,6 +130,23 @@ public class CommandHandler {
     /** */
     private static final String CMD_USER = "--user";
 
+    /** Force option is used for auto confirmation. */
+    private static final String CMD_FORCE = "--force";
+
+    /** List of optional auxiliary commands. */
+    private static final Set<String> AUX_COMMANDS = new HashSet<>();
+    static {
+        AUX_COMMANDS.add(CMD_HELP);
+        AUX_COMMANDS.add(CMD_HOST);
+        AUX_COMMANDS.add(CMD_PORT);
+        AUX_COMMANDS.add(CMD_PASSWORD);
+        AUX_COMMANDS.add(CMD_USER);
+        AUX_COMMANDS.add(CMD_FORCE);
+    }
+
+    /** Broadcast uuid. */
+    private static final UUID BROADCAST_UUID = UUID.randomUUID();
+
     /** */
     protected static final String CMD_PING_INTERVAL = "--ping-interval";
 
@@ -130,9 +174,6 @@ public class CommandHandler {
     /** */
     private static final String DELIM = "--------------------------------------------------------------------------------";
 
-    /** Force option is used for auto confirmation. */
-    private static final String CMD_FORCE = "--force";
-
     /** */
     public static final int EXIT_CODE_OK = 0;
 
@@ -157,6 +198,9 @@ public class CommandHandler {
     /** */
     private static final Scanner IN = new Scanner(System.in);
 
+    /** Validate indexes task name. */
+    private static final String VALIDATE_INDEXES_TASK = "org.apache.ignite.internal.visor.verify.VisorValidateIndexesTask";
+
     /** */
     private static final String TX_LIMIT = "limit";
 
@@ -194,7 +238,7 @@ public class CommandHandler {
     private String peekedArg;
 
     /** */
-    private Object lastOperationResult;
+    private Object lastOperationRes;
 
     /**
      * Output specified string to console.
@@ -279,16 +323,22 @@ private String confirmationPrompt(Arguments args) {
         switch (args.command()) {
             case DEACTIVATE:
                 str = "Warning: the command will deactivate a cluster.";
+
                 break;
 
             case BASELINE:
                 if (!BASELINE_COLLECT.equals(args.baselineAction()))
                     str = "Warning: the command will perform changes in baseline.";
+
                 break;
 
             case TX:
                 if (args.transactionArguments().getOperation() == VisorTxOperation.KILL)
                     str = "Warning: the command will kill some transactions.";
+
+                break;
+
+            default:
                 break;
         }
 
@@ -392,12 +442,52 @@ private Map<UUID, VisorTxTaskResult> executeTransactionsTask(GridClient client,
      * @throws GridClientException If failed to execute task.
      */
     private <R> R executeTask(GridClient client, Class<?> taskCls, Object taskArgs) throws GridClientException {
+        return executeTaskByNameOnNode(client, taskCls.getName(), taskArgs, null);
+    }
+
+    /**
+     * @param client Client
+     * @param taskClsName Task class name.
+     * @param taskArgs Task args.
+     * @param nodeId Node ID to execute task at (if null, random node will be chosen by balancer).
+     * @return Task result.
+     * @throws GridClientException If failed to execute task.
+     */
+    private <R> R executeTaskByNameOnNode(GridClient client, String taskClsName, Object taskArgs, UUID nodeId
+    ) throws GridClientException {
         GridClientCompute compute = client.compute();
 
-        GridClientNode node = getBalancedNode(compute);
+        if (nodeId == BROADCAST_UUID) {
+            Collection<GridClientNode> nodes = compute.nodes(GridClientNode::connectable);
+
+            if (F.isEmpty(nodes))
+                throw new GridClientDisconnectedException("Connectable nodes not found", null);
+
+            List<UUID> nodeIds = nodes.stream()
+                .map(GridClientNode::nodeId)
+                .collect(Collectors.toList());
+
+            return client.compute().execute(taskClsName, new VisorTaskArgument<>(nodeIds, taskArgs, false));
+        }
+
+        GridClientNode node = null;
+
+        if (nodeId == null)
+            node = getBalancedNode(compute);
+        else {
+            for (GridClientNode n : compute.nodes()) {
+                if (n.connectable() && nodeId.equals(n.nodeId())) {
+                    node = n;
 
-        return compute.execute(taskCls.getName(),
-            new VisorTaskArgument<>(node.nodeId(), taskArgs, false));
+                    break;
+                }
+            }
+
+            if (node == null)
+                throw new IllegalArgumentException("Node with id=" + nodeId + " not found");
+        }
+
+        return compute.projection(node).execute(taskClsName, new VisorTaskArgument<>(node.nodeId(), taskArgs, false));
     }
 
     /**
@@ -405,11 +495,7 @@ private <R> R executeTask(GridClient client, Class<?> taskCls, Object taskArgs)
      * @return balanced node
      */
     private GridClientNode getBalancedNode(GridClientCompute compute) throws GridClientException {
-        List<GridClientNode> nodes = new ArrayList<>();
-
-        for (GridClientNode node : compute.nodes())
-            if (node.connectable())
-                nodes.add(node);
+        Collection<GridClientNode> nodes = compute.nodes(GridClientNode::connectable);
 
         if (F.isEmpty(nodes))
             throw new GridClientDisconnectedException("Connectable node not found", null);
@@ -417,6 +503,175 @@ private GridClientNode getBalancedNode(GridClientCompute compute) throws GridCli
         return compute.balancer().balancedNode(nodes);
     }
 
+    /**
+     * Executes --cache subcommand.
+     *
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     */
+    private void cache(GridClient client, CacheArguments cacheArgs) throws Throwable {
+        switch (cacheArgs.command()) {
+            case HELP:
+                printCacheHelp();
+
+                break;
+
+            case IDLE_VERIFY:
+                cacheIdleVerify(client, cacheArgs);
+
+                break;
+
+            case VALIDATE_INDEXES:
+                cacheValidateIndexes(client, cacheArgs);
+
+                break;
+
+            case CONTENTION:
+                cacheContention(client, cacheArgs);
+
+                break;
+
+            default:
+                cacheView(client, cacheArgs);
+
+                break;
+        }
+    }
+
+    /**
+     *
+     */
+    private void printCacheHelp() {
+        log("--cache subcommand allows to do the following operations:");
+
+        usage("  Show information about caches, groups or sequences that match a regex:", CACHE, " list regexPattern [groups|seq] [nodeId]");
+        usage("  Show hot keys that are point of contention for multiple transactions:", CACHE, " contention minQueueSize [nodeId] [maxPrint]");
+        usage("  Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [cache1,...,cacheN]");
+        usage("  Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId]");
+
+        log("  If [nodeId] is not specified, cont and validate_indexes commands will be broadcasted to all server nodes.");
+        log("  Another commands where [nodeId] is optional will run on a random server node.");
+        nl();
+    }
+
+    /**
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     */
+    private void cacheContention(GridClient client, CacheArguments cacheArgs) throws GridClientException {
+        VisorContentionTaskArg taskArg = new VisorContentionTaskArg(
+            cacheArgs.minQueueSize(), cacheArgs.maxPrint());
+
+        UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId();
+
+        VisorContentionTaskResult res = executeTaskByNameOnNode(
+            client, VisorContentionTask.class.getName(), taskArg, nodeId);
+
+        if (!F.isEmpty(res.exceptions())) {
+            log("Contention check failed on nodes:");
+
+            for (Map.Entry<UUID, Exception> e : res.exceptions().entrySet()) {
+                log("Node ID = " + e.getKey());
+
+                log("Exception message:");
+                log(e.getValue().getMessage());
+                nl();
+            }
+        }
+
+        for (ContentionInfo info : res.getInfos())
+            info.print();
+    }
+
+    /**
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     */
+    private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) throws GridClientException {
+        VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(cacheArgs.caches());
+
+        UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId();
+
+        VisorValidateIndexesTaskResult taskRes = executeTaskByNameOnNode(
+            client, VALIDATE_INDEXES_TASK, taskArg, nodeId);
+
+        if (!F.isEmpty(taskRes.exceptions())) {
+            log("Index validation failed on nodes:");
+
+            for (Map.Entry<UUID, Exception> e : taskRes.exceptions().entrySet()) {
+                log("Node ID = " + e.getKey());
+
+                log("Exception message:");
+                log(e.getValue().getMessage());
+                nl();
+            }
+        }
+
+        boolean errors = false;
+
+        for (Map.Entry<UUID, VisorValidateIndexesJobResult> nodeEntry : taskRes.results().entrySet()) {
+            Map<PartitionKey, ValidateIndexesPartitionResult> map = nodeEntry.getValue().response();
+
+            for (Map.Entry<PartitionKey, ValidateIndexesPartitionResult> e : map.entrySet()) {
+                ValidateIndexesPartitionResult res = e.getValue();
+
+                if (!res.issues().isEmpty()) {
+                    errors = true;
+
+                    log(e.getKey().toString() + " " + e.getValue().toString());
+
+                    for (IndexValidationIssue is : res.issues())
+                        log(is.toString());
+                }
+            }
+        }
+
+        if (!errors)
+            log("validate_indexes has finished, no issues found.");
+        else
+            log("validate_indexes has finished with errors (listed above).");
+    }
+
+    /**
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     */
+    private void cacheView(GridClient client, CacheArguments cacheArgs) throws GridClientException {
+        VisorViewCacheTaskArg taskArg = new VisorViewCacheTaskArg(cacheArgs.regex(), cacheArgs.cacheCommand());
+
+        VisorViewCacheTaskResult res = executeTaskByNameOnNode(
+            client, VisorViewCacheTask.class.getName(), taskArg, cacheArgs.nodeId());
+
+        for (CacheInfo info : res.cacheInfos())
+            info.print(cacheArgs.cacheCommand());
+    }
+
+    /**
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     */
+    private void cacheIdleVerify(GridClient client, CacheArguments cacheArgs) throws GridClientException {
+        VisorIdleVerifyTaskResult res = executeTask(
+            client, VisorIdleVerifyTask.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
+
+        Map<PartitionKey, List<PartitionHashRecord>> conflicts = res.getConflicts();
+
+        if (conflicts.isEmpty()) {
+            log("idle_verify check has finished, no conflicts have been found.");
+            nl();
+        }
+        else {
+            log ("idle_verify check has finished, found " + conflicts.size() + " conflict partitions.");
+            nl();
+
+            for (Map.Entry<PartitionKey, List<PartitionHashRecord>> entry : conflicts.entrySet()) {
+                log("Conflict partition: " + entry.getKey());
+
+                log("Partition instances: " + entry.getValue());
+            }
+        }
+    }
+
     /**
      * Change baseline.
      *
@@ -508,7 +763,7 @@ private void baselinePrint0(VisorBaselineTaskResult res) {
 
         Map<String, VisorBaselineNode> baseline = res.getBaseline();
 
-        Map<String, VisorBaselineNode> servers = res.getServers();
+        Map<String, VisorBaselineNode> srvs = res.getServers();
 
         if (F.isEmpty(baseline))
             log("Baseline nodes not found.");
@@ -517,7 +772,7 @@ private void baselinePrint0(VisorBaselineTaskResult res) {
 
             for(VisorBaselineNode node : baseline.values()) {
                 log("    ConsistentID=" + node.getConsistentId() + ", STATE=" +
-                    (servers.containsKey(node.getConsistentId()) ? "ONLINE" : "OFFLINE"));
+                    (srvs.containsKey(node.getConsistentId()) ? "ONLINE" : "OFFLINE"));
             }
 
             log(DELIM);
@@ -527,7 +782,7 @@ private void baselinePrint0(VisorBaselineTaskResult res) {
 
             List<VisorBaselineNode> others = new ArrayList<>();
 
-            for (VisorBaselineNode node : servers.values()) {
+            for (VisorBaselineNode node : srvs.values()) {
                 if (!baseline.containsKey(node.getConsistentId()))
                     others.add(node);
             }
@@ -645,7 +900,7 @@ private void transactions(GridClient client, VisorTxTaskArg arg) throws GridClie
         try {
             Map<ClusterNode, VisorTxTaskResult> res = executeTask(client, VisorTxTask.class, arg);
 
-            lastOperationResult = res;
+            lastOperationRes = res;
 
             if (res.isEmpty())
                 log("Nothing found.");
@@ -778,6 +1033,8 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         boolean force = false;
 
+        CacheArguments cacheArgs = null;
+
         List<Command> commands = new ArrayList<>();
 
         initArgIterator(rawArgs);
@@ -824,6 +1081,13 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
+                    case CACHE:
+                        commands.add(CACHE);
+
+                        cacheArgs = parseAndValidateCacheArgs();
+
+                        break;
+
                     default:
                         throw new IllegalArgumentException("Unexpected command: " + str);
                 }
@@ -862,14 +1126,17 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                     case CMD_USER:
                         user = nextArg("Expected user name");
+
                         break;
 
                     case CMD_PASSWORD:
                         pwd = nextArg("Expected password");
+
                         break;
 
                     case CMD_FORCE:
                         force = true;
+
                         break;
 
                     default:
@@ -895,7 +1162,118 @@ Arguments parseAndValidate(List<String> rawArgs) {
             throw new IllegalArgumentException("Both user and password should be specified");
 
         return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs,
-            pingTimeout, pingInterval, txArgs, force);
+            pingTimeout, pingInterval, txArgs, force, cacheArgs);
+    }
+
+    /**
+     * Parses and validates cache arguments.
+     *
+     * @return --cache subcommand arguments in case validation is successful.
+     */
+    private CacheArguments parseAndValidateCacheArgs() {
+        if (!hasNextCacheArg()) {
+            throw new IllegalArgumentException("Arguments are expected for --cache subcommand, " +
+                "run --cache help for more info.");
+        }
+
+        CacheArguments cacheArgs = new CacheArguments();
+
+        String str = nextArg("").toLowerCase();
+
+        CacheCommand cmd = CacheCommand.of(str);
+
+        if (cmd == null)
+            cmd = CacheCommand.HELP;
+
+        cacheArgs.command(cmd);
+
+        switch (cmd) {
+            case HELP:
+                break;
+
+            case IDLE_VERIFY:
+                parseCacheNamesIfPresent(cacheArgs);
+
+                break;
+
+            case CONTENTION:
+                cacheArgs.minQueueSize(Integer.parseInt(nextArg("Min queue size expected")));
+
+                if (hasNextCacheArg())
+                    cacheArgs.nodeId(UUID.fromString(nextArg("")));
+
+                if (hasNextCacheArg())
+                    cacheArgs.maxPrint(Integer.parseInt(nextArg("")));
+                else
+                    cacheArgs.maxPrint(10);
+
+                break;
+
+            case VALIDATE_INDEXES:
+                parseCacheNamesIfPresent(cacheArgs);
+
+                if (hasNextCacheArg())
+                    cacheArgs.nodeId(UUID.fromString(nextArg("")));
+
+                break;
+
+            default:
+                cacheArgs.regex(nextArg("Regex is expected"));
+
+                if (hasNextCacheArg()) {
+                    String tmp = nextArg("");
+
+                    switch (tmp) {
+                        case "groups":
+                            cacheArgs.cacheCommand(GROUPS);
+
+                            break;
+
+                        case "seq":
+                            cacheArgs.cacheCommand(SEQ);
+
+                            break;
+
+                        default:
+                            cacheArgs.nodeId(UUID.fromString(tmp));
+                    }
+                }
+
+                break;
+        }
+
+        if (hasNextCacheArg())
+            throw new IllegalArgumentException("Unexpected argument of --cache subcommand: " + peekNextArg());
+
+        return cacheArgs;
+    }
+
+    /**
+     * @return <code>true</code> if there's next argument for --cache subcommand.
+     */
+    private boolean hasNextCacheArg() {
+        return hasNextArg() && Command.of(peekNextArg()) == null && !AUX_COMMANDS.contains(peekNextArg());
+    }
+
+    /**
+     * @param cacheArgs Cache args.
+     */
+    private void parseCacheNamesIfPresent(CacheArguments cacheArgs) {
+        if (hasNextCacheArg()) {
+            String cacheNames = nextArg("");
+
+            String[] cacheNamesArr = cacheNames.split(",");
+            Set<String> cacheNamesSet = new HashSet<>();
+
+            for (String cacheName : cacheNamesArr) {
+                if (F.isEmpty(cacheName))
+                    throw new IllegalArgumentException("Non-empty cache names expected.");
+
+                cacheNamesSet.add(cacheName.trim());
+            }
+
+            cacheArgs.caches(cacheNamesSet);
+        }
     }
 
     /**
@@ -1002,7 +1380,7 @@ private VisorTxTaskArg parseTransactionArguments() {
                     try {
                         Pattern.compile(lbRegex);
                     }
-                    catch (PatternSyntaxException e) {
+                    catch (PatternSyntaxException ignored) {
                         throw new IllegalArgumentException("Illegal regex syntax");
                     }
 
@@ -1046,7 +1424,7 @@ private long nextLongArg(String lb) {
 
             return val;
         }
-        catch (NumberFormatException e) {
+        catch (NumberFormatException ignored) {
             throw new IllegalArgumentException("Invalid value for " + lb + ": " + str);
         }
     }
@@ -1079,6 +1457,10 @@ public int execute(List<String> rawArgs) {
                     "[minSize SIZE] [label PATTERN_REGEX] [servers|clients] " +
                     "[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE] [kill] [--force]");
 
+                log("The utility has --cache subcommand to view and control state of caches in cluster.");
+                log("  More info:    control.sh --cache help");
+                nl();
+
                 log("By default commands affecting the cluster require interactive confirmation. ");
                 log("  --force option can be used to execute commands without prompting for confirmation.");
                 nl();
@@ -1122,26 +1504,35 @@ public int execute(List<String> rawArgs) {
             }
 
             try (GridClient client = GridClientFactory.start(cfg)) {
-
                 switch (args.command()) {
                     case ACTIVATE:
                         activate(client);
+
                         break;
 
                     case DEACTIVATE:
                         deactivate(client);
+
                         break;
 
                     case STATE:
                         state(client);
+
                         break;
 
                     case BASELINE:
                         baseline(client, args.baselineAction(), args.baselineArguments());
+
                         break;
 
                     case TX:
                         transactions(client, args.transactionArguments());
+
+                        break;
+
+                    case CACHE:
+                        cache(client, args.cacheArgs());
+
                         break;
                 }
             }
@@ -1177,7 +1568,7 @@ public static void main(String[] args) {
      */
     @SuppressWarnings("unchecked")
     public <T> T getLastOperationResult() {
-        return (T)lastOperationResult;
+        return (T)lastOperationRes;
     }
 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
new file mode 100644
index 0000000000000..6f315efe6a732
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
@@ -0,0 +1,163 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.commandline.cache;
+
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.internal.visor.verify.VisorViewCacheCmd;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class CacheArguments {
+    /** Command. */
+    private CacheCommand cmd;
+
+    /** Caches. */
+    private Set<String> caches;
+
+    /** Partition id. */
+    private int partId;
+
+    /** Regex. */
+    private String regex;
+
+    /** Node id. */
+    private UUID nodeId;
+
+    /** Min queue size. */
+    private int minQueueSize;
+
+    /** Max print. */
+    private int maxPrint;
+
+    /** Cache view command. */
+    private @Nullable VisorViewCacheCmd cacheCmd;
+
+    /**
+     * @return Command.
+     */
+    public CacheCommand command() {
+        return cmd;
+    }
+
+    /**
+     * @return Cache view command.
+     */
+    @Nullable public VisorViewCacheCmd cacheCommand() {
+        return cacheCmd;
+    }
+
+    /**
+     * @param cmd Cache view command.
+     */
+    public void cacheCommand(VisorViewCacheCmd cmd) {
+        this.cacheCmd = cmd;
+    }
+
+    /**
+     * @param cmd New command.
+     */
+    public void command(CacheCommand cmd) {
+        this.cmd = cmd;
+    }
+
+    /**
+     * @return Caches.
+     */
+    public Set<String> caches() {
+        return caches;
+    }
+
+    /**
+     * @param caches New caches.
+     */
+    public void caches(Set<String> caches) {
+        this.caches = caches;
+    }
+
+    /**
+     * @return Partition id.
+     */
+    public int partitionId() {
+        return partId;
+    }
+
+    /**
+     * @param partId New partition id.
+     */
+    public void partitionId(int partId) {
+        this.partId = partId;
+    }
+
+    /**
+     * @return Regex.
+     */
+    public String regex() {
+        return regex;
+    }
+
+    /**
+     * @param regex New regex.
+     */
+    public void regex(String regex) {
+        this.regex = regex;
+    }
+
+    /**
+     * @return Node id.
+     */
+    public UUID nodeId() {
+        return nodeId;
+    }
+
+    /**
+     * @param nodeId New node id.
+     */
+    public void nodeId(UUID nodeId) {
+        this.nodeId = nodeId;
+    }
+
+    /**
+     * @return Min queue size.
+     */
+    public int minQueueSize() {
+        return minQueueSize;
+    }
+
+    /**
+     * @param minQueueSize New min queue size.
+     */
+    public void minQueueSize(int minQueueSize) {
+        this.minQueueSize = minQueueSize;
+    }
+
+    /**
+     * @return Max print.
+     */
+    public int maxPrint() {
+        return maxPrint;
+    }
+
+    /**
+     * @param maxPrint New max print.
+     */
+    public void maxPrint(int maxPrint) {
+        this.maxPrint = maxPrint;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java
new file mode 100644
index 0000000000000..6aec6d7e9cfa1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java
@@ -0,0 +1,93 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.commandline.cache;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public enum CacheCommand {
+    /**
+     * Prints out help for the cache command.
+     */
+    HELP("help"),
+
+    /**
+     * Checks consistency of primary and backup partitions assuming no concurrent updates are happening in the cluster.
+     */
+    IDLE_VERIFY("idle_verify"),
+
+    /**
+     * Prints info regarding caches, groups or sequences.
+     */
+    LIST("list"),
+
+    /**
+     * Validates indexes attempting to read each indexed entry.
+     */
+    VALIDATE_INDEXES("validate_indexes"),
+
+    /**
+     * Prints info about contended keys (the keys concurrently locked from multiple transactions).
+     */
+    CONTENTION("contention");
+
+    /** Enumerated values. */
+    private static final CacheCommand[] VALS = values();
+
+    /** Name. */
+    private final String name;
+
+    /**
+     * @param name Name.
+     */
+    CacheCommand(String name) {
+        this.name = name;
+    }
+
+    /**
+     * @param text Command text.
+     * @return Command for the text.
+     */
+    public static CacheCommand of(String text) {
+        for (CacheCommand cmd : CacheCommand.values()) {
+            if (cmd.text().equalsIgnoreCase(text))
+                return cmd;
+        }
+
+        return null;
+    }
+
+    /**
+     * @return Name.
+     */
+    public String text() {
+        return name;
+    }
+
+    /**
+     * Efficiently gets enumerated value from its ordinal.
+     *
+     * @param ord Ordinal value.
+     * @return Enumerated value or {@code null} if ordinal out of range.
+     */
+    @Nullable public static CacheCommand fromOrdinal(int ord) {
+        return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java
new file mode 100644
index 0000000000000..9a090a042e961
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.apache.ignite.internal.visor.verify.VisorViewCacheCmd;
+
+/**
+ * Cache info DTO.
+ */
+public class CacheInfo extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Sequence name. */
+    private String seqName;
+
+    /** Sequence value. */
+    private long seqVal;
+
+    /** Cache name. */
+    private String cacheName;
+
+    /** Cache id. */
+    private int cacheId;
+
+    /** Group name. */
+    private String grpName;
+
+    /** Group id. */
+    private int grpId;
+
+    /** Caches count. */
+    private int cachesCnt;
+
+    /** Partitions. */
+    private int partitions;
+
+    /** Mapped. */
+    private int mapped;
+
+    /** Topology version. */
+    public AffinityTopologyVersion topVer;
+
+    /** Mode. */
+    private CacheMode mode;
+
+    /** Backups count. */
+    private int backupsCnt;
+
+    /** Affinity class name. */
+    private String affinityClsName;
+
+    /** */
+    public String getSeqName() {
+        return seqName;
+    }
+
+    /**
+     * @param seqName Sequence name.
+     */
+    public void setSeqName(String seqName) {
+        this.seqName = seqName;
+    }
+
+    /**
+     *
+     */
+    public String getCacheName() {
+        return cacheName;
+    }
+
+    /**
+     * @param cacheName Cache name.
+     */
+    public void setCacheName(String cacheName) {
+        this.cacheName = cacheName;
+    }
+
+    /**
+     *
+     */
+    public int getCacheId() {
+        return cacheId;
+    }
+
+    /**
+     * @param cacheId Cache id.
+     */
+    public void setCacheId(int cacheId) {
+        this.cacheId = cacheId;
+    }
+
+    /**
+     *
+     */
+    public String getGrpName() {
+        return grpName;
+    }
+
+    /**
+     * @param grpName Group name.
+     */
+    public void setGrpName(String grpName) {
+        this.grpName = grpName;
+    }
+
+    /**
+     *
+     */
+    public int getGrpId() {
+        return grpId;
+    }
+
+    /**
+     * @param grpId Group id.
+     */
+    public void setGrpId(int grpId) {
+        this.grpId = grpId;
+    }
+
+    /**
+     *
+     */
+    public int getCachesCnt() {
+        return cachesCnt;
+    }
+
+    /**
+     * @param cachesCnt Caches count.
+     */
+    public void setCachesCnt(int cachesCnt) {
+        this.cachesCnt = cachesCnt;
+    }
+
+    /**
+     *
+     */
+    public int getPartitions() {
+        return partitions;
+    }
+
+    /**
+     * @param partitions Partitions.
+     */
+    public void setPartitions(int partitions) {
+        this.partitions = partitions;
+    }
+
+    /**
+     *
+     */
+    public int getMapped() {
+        return mapped;
+    }
+
+    /**
+     * @param mapped Mapped.
+     */
+    public void setMapped(int mapped) {
+        this.mapped = mapped;
+    }
+
+    /**
+     *
+     */
+    public AffinityTopologyVersion getTopologyVersion() {
+        return topVer;
+    }
+
+    /**
+     * @param topologyVersion Topology version.
+     */
+    public void setTopologyVersion(AffinityTopologyVersion topologyVersion) {
+        this.topVer = topologyVersion;
+    }
+
+    /**
+     * @param seqVal Sequence value.
+     */
+    public void setSeqVal(long seqVal) {
+        this.seqVal = seqVal;
+    }
+
+    /**
+     *
+     */
+    public long getSeqVal() {
+        return seqVal;
+    }
+
+    /**
+     *
+     */
+    public CacheMode getMode() {
+        return mode;
+    }
+
+    /**
+     * @param mode Mode.
+     */
+    public void setMode(CacheMode mode) {
+        this.mode = mode;
+    }
+
+    /**
+     *
+     */
+    public int getBackupsCnt() {
+        return backupsCnt;
+    }
+
+    /**
+     * @param backupsCnt Backups count.
+     */
+    public void setBackupsCnt(int backupsCnt) {
+        this.backupsCnt = backupsCnt;
+    }
+
+    /**
+     *
+     */
+    public String getAffinityClsName() {
+        return affinityClsName;
+    }
+
+    /**
+     * @param affinityClsName Affinity class name.
+     */
+    public void setAffinityClsName(String affinityClsName) {
+        this.affinityClsName = affinityClsName;
+    }
+
+    /**
+     * @param cmd Command.
+     */
+    public void print(VisorViewCacheCmd cmd) {
+        if (cmd == null)
+            cmd = VisorViewCacheCmd.CACHES;
+
+        switch (cmd) {
+            case SEQ:
+                System.out.println("[seqName=" + getSeqName() + ", curVal=" + seqVal + ']');
+
+                break;
+
+            case GROUPS:
+                System.out.println("[grpName=" + getGrpName() + ", grpId=" + getGrpId() + ", cachesCnt=" + getCachesCnt() +
+                    ", prim=" + getPartitions() + ", mapped=" + getMapped() + ", mode=" + getMode() +
+                    ", backups=" + getBackupsCnt() + ", affCls=" + getAffinityClsName() + ']');
+
+                break;
+
+            default:
+                System.out.println("[cacheName=" + getCacheName() + ", cacheId=" + getCacheId() +
+                    ", grpName=" + getGrpName() + ", grpId=" + getGrpId() + ", prim=" + getPartitions() +
+                    ", mapped=" + getMapped() + ", mode=" + getMode() +
+                    ", backups=" + getBackupsCnt() + ", affCls=" + getAffinityClsName() + ']');
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeString(out, seqName);
+        out.writeLong(seqVal);
+        U.writeString(out, cacheName);
+        out.writeInt(cacheId);
+        U.writeString(out, grpName);
+        out.writeInt(grpId);
+        out.writeInt(partitions);
+        out.writeInt(mapped);
+        out.writeObject(topVer);
+        U.writeEnum(out, mode);
+        out.writeInt(backupsCnt);
+        U.writeString(out, affinityClsName);
+        out.writeInt(cachesCnt);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        seqName = U.readString(in);
+        seqVal = in.readLong();
+        cacheName = U.readString(in);
+        cacheId = in.readInt();
+        grpName = U.readString(in);
+        grpId = in.readInt();
+        partitions = in.readInt();
+        mapped = in.readInt();
+        topVer = (AffinityTopologyVersion)in.readObject();
+        mode = CacheMode.fromOrdinal(in.readByte());
+        backupsCnt = in.readInt();
+        affinityClsName = U.readString(in);
+        cachesCnt = in.readInt();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(CacheInfo.class, this);
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ContentionClosure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ContentionClosure.java
new file mode 100644
index 0000000000000..e97378e9870d6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ContentionClosure.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
+import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
+import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalState;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxState;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxStateImpl;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+
+/**
+ */
+public class ContentionClosure implements IgniteCallable<ContentionInfo> {
+    /** Serial version uid. */
+    private static final long serialVersionUID = 0L;
+
+    /** Ignite. */
+    @IgniteInstanceResource
+    protected transient IgniteEx ignite;
+
+    /** */
+    private int minQueueSize;
+
+    /** */
+    private int maxPrint;
+
+    /**
+     * @param minQueueSize Min candidate queue size to account.
+     * @param maxPrint Max entries to print.
+     */
+    public ContentionClosure(int minQueueSize, int maxPrint) {
+        this.minQueueSize = minQueueSize;
+        this.maxPrint = maxPrint;
+    }
+
+    /** {@inheritDoc} */
+    @Override public ContentionInfo call() throws Exception {
+        final IgniteTxManager tm = ignite.context().cache().context().tm();
+
+        final Collection<IgniteInternalTx> activeTxs = tm.activeTransactions();
+
+        ContentionInfo ci = new ContentionInfo();
+
+        ci.setNode(ignite.localNode());
+        ci.setEntries(new ArrayList<>());
+
+        for (IgniteInternalTx tx : activeTxs) {
+            if (ci.getEntries().size() == maxPrint)
+                break;
+
+            // Show only primary txs.
+            if (tx.local()) {
+                IgniteTxLocalAdapter tx0 = (IgniteTxLocalAdapter)tx;
+
+                final IgniteTxLocalState state0 = tx0.txState();
+
+                if (!(state0 instanceof IgniteTxStateImpl))
+                    continue;
+
+                final IgniteTxStateImpl state = (IgniteTxStateImpl)state0;
+
+                final Collection<IgniteTxEntry> entries = state.allEntriesCopy();
+
+                IgniteTxEntry bad = null;
+
+                int qSize = 0;
+
+                for (IgniteTxEntry entry : entries) {
+                    Collection<GridCacheMvccCandidate> locs;
+
+                    GridCacheEntryEx cached = entry.cached();
+
+                    while(true) {
+                        try {
+                            locs = cached.localCandidates();
+
+                            break;
+                        }
+                        catch (GridCacheEntryRemovedException ignored) {
+                            cached = entry.context().cache().entryEx(entry.key());
+                        }
+                    }
+
+                    if (locs != null)
+                        qSize += locs.size();
+
+                    final Collection<GridCacheMvccCandidate> rmts = cached.remoteMvccSnapshot();
+
+                    if (rmts != null)
+                        qSize += rmts.size();
+
+                    if (qSize >= minQueueSize) {
+                        bad = entry;
+
+                        break;
+                    }
+                    else
+                        qSize = 0;
+                }
+
+                if (bad != null) {
+                    StringBuilder b = new StringBuilder();
+
+                    b.append("TxEntry [cacheId=").append(bad.cacheId()).
+                        append(", key=").append(bad.key()).
+                        append(", queue=").append(qSize).
+                        append(", op=").append(bad.op()).
+                        append(", val=").append(bad.value()).
+                        append(", tx=").append(CU.txString(tx)).
+                        append(", other=[");
+
+                    final IgniteTxState st = tx.txState();
+
+                    if (st instanceof IgniteTxStateImpl) {
+                        IgniteTxStateImpl st0 = (IgniteTxStateImpl)st;
+
+                        final Collection<IgniteTxEntry> cp = st0.allEntriesCopy();
+
+                        for (IgniteTxEntry entry : cp) {
+                            if (entry == bad)
+                                continue;
+
+                            b.append(entry.toString()).append('\n');
+                        }
+                    }
+
+                    b.append("]]");
+
+                    ci.getEntries().add(b.toString());
+                }
+            }
+        }
+
+        return ci;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ContentionInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ContentionInfo.java
new file mode 100644
index 0000000000000..c7bfbebef4316
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ContentionInfo.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ */
+public class ContentionInfo implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private ClusterNode node;
+
+    /** */
+    private List<String> entries;
+
+    /**
+     * @return Node.
+     */
+    public ClusterNode getNode() {
+        return node;
+    }
+
+    /**
+     * @param node Node.
+     */
+    public void setNode(ClusterNode node) {
+        this.node = node;
+    }
+
+    /**
+     * @return Entries.
+     */
+    public List<String> getEntries() {
+        return entries;
+    }
+
+    /**
+     * @param entries Entries.
+     */
+    public void setEntries(List<String> entries) {
+        this.entries = entries;
+    }
+
+    /** */
+    public void print() {
+        System.out.println("[node=" + node + ']');
+
+        for (String entry : entries)
+            System.out.println("    " + entry);
+    }
+}
+
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ViewCacheClosure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ViewCacheClosure.java
new file mode 100644
index 0000000000000..1f363f3643db7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ViewCacheClosure.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.datastructures.AtomicDataStructureValue;
+import org.apache.ignite.internal.processors.datastructures.DataStructureType;
+import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
+import org.apache.ignite.internal.processors.datastructures.GridCacheAtomicSequenceValue;
+import org.apache.ignite.internal.processors.datastructures.GridCacheInternalKey;
+import org.apache.ignite.internal.visor.verify.VisorViewCacheCmd;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+
+/**
+ * View cache closure.
+ */
+public class ViewCacheClosure implements IgniteCallable<List<CacheInfo>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Regex. */
+    private String regex;
+
+    /** {@code true} to skip cache destroying. */
+    private VisorViewCacheCmd cmd;
+
+    @IgniteInstanceResource
+    private Ignite ignite;
+
+    /**
+     * @param regex Regex name for stopping caches.
+     * @param cmd Command.
+     */
+    public ViewCacheClosure(String regex, VisorViewCacheCmd cmd) {
+        this.regex = regex;
+        this.cmd = cmd;
+    }
+
+    /** {@inheritDoc} */
+    @Override public List<CacheInfo> call() throws Exception {
+        Pattern compiled = Pattern.compile(regex);
+
+        List<CacheInfo> cacheInfo = new ArrayList<>();
+
+        IgniteKernal k = (IgniteKernal)ignite;
+
+        if (cmd == null)
+            cmd = VisorViewCacheCmd.CACHES;
+
+        switch (cmd) {
+            case SEQ:
+                collectSequences(k.context(), compiled, cacheInfo);
+
+                return cacheInfo;
+
+            case GROUPS:
+                Collection<CacheGroupContext> contexts = k.context().cache().cacheGroups();
+
+                for (CacheGroupContext context : contexts) {
+                    if (!compiled.matcher(context.cacheOrGroupName()).find())
+                        continue;
+
+                    CacheInfo ci = new CacheInfo();
+                    ci.setGrpName(context.cacheOrGroupName());
+                    ci.setGrpId(context.groupId());
+                    ci.setCachesCnt(context.caches().size());
+                    ci.setPartitions(context.config().getAffinity().partitions());
+                    ci.setBackupsCnt(context.config().getBackups());
+                    ci.setAffinityClsName(context.config().getAffinity().getClass().getSimpleName());
+                    ci.setMode(context.config().getCacheMode());
+                    ci.setMapped(mapped(context.caches().iterator().next().name()));
+
+                    cacheInfo.add(ci);
+                }
+
+                return cacheInfo;
+
+            default:
+                Map<String, DynamicCacheDescriptor> descMap = k.context().cache().cacheDescriptors();
+
+                for (Map.Entry<String, DynamicCacheDescriptor> entry : descMap.entrySet()) {
+
+                    DynamicCacheDescriptor desc = entry.getValue();
+
+                    if (!compiled.matcher(desc.cacheName()).find())
+                        continue;
+
+                    CacheInfo ci = new CacheInfo();
+
+                    ci.setCacheName(desc.cacheName());
+                    ci.setCacheId(desc.cacheId());
+                    ci.setGrpName(desc.groupDescriptor().groupName());
+                    ci.setGrpId(desc.groupDescriptor().groupId());
+                    ci.setPartitions(desc.cacheConfiguration().getAffinity().partitions());
+                    ci.setBackupsCnt(desc.cacheConfiguration().getBackups());
+                    ci.setAffinityClsName(desc.cacheConfiguration().getAffinity().getClass().getSimpleName());
+                    ci.setMode(desc.cacheConfiguration().getCacheMode());
+                    ci.setMapped(mapped(desc.cacheName()));
+
+                    cacheInfo.add(ci);
+                }
+
+                return cacheInfo;
+        }
+    }
+
+    /**
+     * @param cacheName Cache name.
+     */
+    private int mapped(String cacheName) {
+        int mapped = 0;
+
+        ClusterGroup srvs = ignite.cluster().forServers();
+
+        Collection<ClusterNode> nodes = srvs.forDataNodes(cacheName).nodes();
+
+        for (ClusterNode node : nodes)
+            mapped += ignite.affinity(cacheName).primaryPartitions(node).length;
+
+        return mapped;
+    }
+
+    /**
+     * @param ctx Context.
+     * @param compiled Compiled pattern.
+     * @param cacheInfo Cache info.
+     */
+    private void collectSequences(GridKernalContext ctx, Pattern compiled, List<CacheInfo> cacheInfo) throws IgniteCheckedException {
+        String dsCacheName = DataStructuresProcessor.ATOMICS_CACHE_NAME + "@default-ds-group";
+
+        IgniteInternalCache<GridCacheInternalKey, AtomicDataStructureValue> cache0 = ctx.cache().cache(dsCacheName);
+
+        final Iterator<Cache.Entry<GridCacheInternalKey, AtomicDataStructureValue>> iter = cache0.scanIterator(false, null);
+
+        while (iter.hasNext()) {
+            Cache.Entry<GridCacheInternalKey, AtomicDataStructureValue> entry = iter.next();
+
+            final AtomicDataStructureValue val = entry.getValue();
+
+            if (val.type() == DataStructureType.ATOMIC_SEQ) {
+                final String name = entry.getKey().name();
+
+                if (compiled.matcher(name).find()) {
+                    CacheInfo ci = new CacheInfo();
+                    ci.setSeqName(name);
+                    ci.setSeqVal(((GridCacheAtomicSequenceValue)val).get());
+
+                    cacheInfo.add(ci);
+                }
+
+            }
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexValidationIssue.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexValidationIssue.java
new file mode 100644
index 0000000000000..37f936088b439
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexValidationIssue.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *
+ */
+public class IndexValidationIssue extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Key. */
+    private String key;
+
+    /** Cache name. */
+    private String cacheName;
+
+    /** Index name. */
+    private String idxName;
+
+    /** T. */
+    @GridToStringExclude
+    private Throwable t;
+
+    /**
+     *
+     */
+    public IndexValidationIssue() {
+        // Default constructor required for Externalizable.
+    }
+
+    /**
+     * @param key Key.
+     * @param cacheName Cache name.
+     * @param idxName Index name.
+     * @param t T.
+     */
+    public IndexValidationIssue(String key, String cacheName, String idxName, Throwable t) {
+        this.key = key;
+        this.cacheName = cacheName;
+        this.idxName = idxName;
+        this.t = t;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeString(out, key);
+        U.writeString(out, cacheName);
+        U.writeString(out, idxName);
+        out.writeObject(t);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        key = U.readString(in);
+        cacheName = U.readString(in);
+        idxName = U.readString(in);
+        t = (Throwable)in.readObject();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IndexValidationIssue.class, this) + ", " + t.getClass() + ": " + t.getMessage();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
new file mode 100644
index 0000000000000..18899607110fc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
@@ -0,0 +1,145 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *
+ */
+public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Update counter. */
+    private long updateCntr;
+
+    /** Size. */
+    private long size;
+
+    /** Is primary. */
+    private boolean isPrimary;
+
+    /** Consistent id. */
+    @GridToStringInclude
+    private Object consistentId;
+
+    /** Issues. */
+    @GridToStringExclude
+    private List<IndexValidationIssue> issues = new ArrayList<>(10);
+
+    /**
+     *
+     */
+    public ValidateIndexesPartitionResult() {
+        // Empty constructor required for Externalizable.
+    }
+
+    /**
+     * @param updateCntr Update counter.
+     * @param size Size.
+     * @param isPrimary Is primary.
+     * @param consistentId Consistent id.
+     */
+    public ValidateIndexesPartitionResult(long updateCntr, long size, boolean isPrimary, Object consistentId) {
+        this.updateCntr = updateCntr;
+        this.size = size;
+        this.isPrimary = isPrimary;
+        this.consistentId = consistentId;
+    }
+
+    /**
+     *
+     */
+    public long updateCntr() {
+        return updateCntr;
+    }
+
+    /**
+     *
+     */
+    public long size() {
+        return size;
+    }
+
+    /**
+     *
+     */
+    public boolean primary() {
+        return isPrimary;
+    }
+
+    /**
+     *
+     */
+    public Object consistentId() {
+        return consistentId;
+    }
+
+    /**
+     *
+     */
+    public List<IndexValidationIssue> issues() {
+        return issues;
+    }
+
+    /**
+     * @param t Issue.
+     * @return True if there are already enough issues.
+     */
+    public boolean reportIssue(IndexValidationIssue t) {
+        if (issues.size() >= 10)
+            return true;
+
+        issues.add(t);
+
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        out.writeLong(updateCntr);
+        out.writeLong(size);
+        out.writeBoolean(isPrimary);
+        out.writeObject(consistentId);
+        U.writeCollection(out, issues);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        updateCntr = in.readLong();
+        size = in.readLong();
+        isPrimary = in.readBoolean();
+        consistentId = in.readObject();
+        issues = U.readList(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ValidateIndexesPartitionResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionJobResult.java
new file mode 100644
index 0000000000000..427224423c61a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionJobResult.java
@@ -0,0 +1,80 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.processors.cache.verify.ContentionInfo;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *
+ */
+public class VisorContentionJobResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Info. */
+    private ContentionInfo info;
+
+    /**
+     * @param info Info.
+     */
+    public VisorContentionJobResult(ContentionInfo info) {
+        this.info = info;
+    }
+
+    /**
+     * For externalization only.
+     */
+    public VisorContentionJobResult() {
+    }
+
+    /**
+     * @return Contention info.
+     */
+    public ContentionInfo info() {
+        return info;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        out.writeObject(info.getNode());
+        U.writeCollection(out, info.getEntries());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        Object node = in.readObject();
+        List<String> entries = U.readList(in);
+
+        info = new ContentionInfo();
+        info.setNode((ClusterNode)node);
+        info.setEntries(entries);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorContentionJobResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTask.java
new file mode 100644
index 0000000000000..4e7e12c2ff102
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTask.java
@@ -0,0 +1,100 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.cache.verify.ContentionClosure;
+import org.apache.ignite.internal.processors.cache.verify.ContentionInfo;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+@GridInternal
+public class VisorContentionTask extends VisorMultiNodeTask<VisorContentionTaskArg,
+    VisorContentionTaskResult, VisorContentionJobResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected VisorContentionTaskResult reduce0(List<ComputeJobResult> list) throws IgniteException {
+        Map<UUID, Exception> exceptions = new HashMap<>();
+        List<VisorContentionJobResult> infos = new ArrayList<>();
+
+        for (ComputeJobResult res : list) {
+            if (res.getException() != null)
+                exceptions.put(res.getNode().id(), res.getException());
+            else
+                infos.add(res.getData());
+        }
+
+        return new VisorContentionTaskResult(infos, exceptions);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<VisorContentionTaskArg, VisorContentionJobResult> job(VisorContentionTaskArg arg) {
+        return new VisorContentionJob(arg, debug);
+    }
+
+    /**
+     *
+     */
+    private static class VisorContentionJob extends VisorJob<VisorContentionTaskArg, VisorContentionJobResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param arg Argument.
+         * @param debug Debug.
+         */
+        protected VisorContentionJob(@Nullable VisorContentionTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorContentionJobResult run(@Nullable VisorContentionTaskArg arg) throws IgniteException {
+            try {
+                ContentionClosure clo = new ContentionClosure(arg.minQueueSize(), arg.maxPrint());
+
+                ignite.context().resource().injectGeneric(clo);
+
+                ContentionInfo info = clo.call();
+
+                return new VisorContentionJobResult(info);
+            }
+            catch (Exception e) {
+                throw new IgniteException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorContentionJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTaskArg.java
new file mode 100644
index 0000000000000..ecfc9d8340145
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTaskArg.java
@@ -0,0 +1,84 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *
+ */
+public class VisorContentionTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Min queue size. */
+    private int minQueueSize;
+
+    /** Max print size. */
+    private int maxPrint;
+
+    /**
+     * @param minQueueSize Min queue size.
+     * @param maxPrint Max print.
+     */
+    public VisorContentionTaskArg(int minQueueSize, int maxPrint) {
+        this.minQueueSize = minQueueSize;
+        this.maxPrint = maxPrint;
+    }
+
+    /**
+     * For externalization only.
+     */
+    public VisorContentionTaskArg() {
+    }
+
+    /**
+     * @return Min queue size.
+     */
+    public int minQueueSize() {
+        return minQueueSize;
+    }
+
+    /**
+     * @return Max print size.
+     */
+    public int maxPrint() {
+        return maxPrint;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        out.writeInt(minQueueSize);
+        out.writeInt(maxPrint);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        minQueueSize = in.readInt();
+        maxPrint = in.readInt();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorContentionTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTaskResult.java
new file mode 100644
index 0000000000000..5c452caf0e9ce
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorContentionTaskResult.java
@@ -0,0 +1,100 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+import org.apache.ignite.internal.processors.cache.verify.ContentionInfo;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *
+ */
+public class VisorContentionTaskResult extends VisorDataTransferObject {
+    /** Serial version uid. */
+    private static final long serialVersionUID = 0L;
+
+    /** Cluster infos. */
+    private List<VisorContentionJobResult> clusterInfos;
+
+    /** Exceptions. */
+    private Map<UUID, Exception> exceptions;
+
+    /**
+     * @param clusterInfos Cluster infos.
+     * @param exceptions Exceptions.
+     */
+    public VisorContentionTaskResult(List<VisorContentionJobResult> clusterInfos,
+        Map<UUID, Exception> exceptions) {
+        this.clusterInfos = clusterInfos;
+        this.exceptions = exceptions;
+    }
+
+    /**
+     * For externalization only.
+     */
+    public VisorContentionTaskResult() {
+    }
+
+    /**
+     * @return Cluster infos.
+     */
+    public Collection<VisorContentionJobResult> jobResults() {
+        return clusterInfos;
+    }
+
+    /**
+     * @return Collection of {@link ContentionInfo} collected during task execution.
+     */
+    public Collection<ContentionInfo> getInfos() {
+        return clusterInfos.stream().map(VisorContentionJobResult::info).collect(Collectors.toList());
+    }
+
+    /**
+     * @return Exceptions.
+     */
+    public Map<UUID, Exception> exceptions() {
+        return exceptions;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeCollection(out, clusterInfos);
+        U.writeMap(out, exceptions);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in
+    ) throws IOException, ClassNotFoundException {
+        clusterInfos = U.readList(in);
+        exceptions = U.readMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorContentionTaskResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTask.java
new file mode 100644
index 0000000000000..2d86a428d38bd
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTask.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobContext;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysTask;
+import org.apache.ignite.internal.processors.cache.verify.PartitionEntryHashRecord;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
+import org.apache.ignite.internal.processors.cache.verify.RetrieveConflictPartitionValuesTask;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.resources.JobContextResource;
+
+/**
+ * Task to find diverged keys of conflict partition.
+ */
+@GridInternal
+public class VisorIdleAnalyzeTask extends VisorOneNodeTask<VisorIdleAnalyzeTaskArg, VisorIdleAnalyzeTaskResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<VisorIdleAnalyzeTaskArg, VisorIdleAnalyzeTaskResult> job(VisorIdleAnalyzeTaskArg arg) {
+        return new VisorIdleVerifyJob(arg, debug);
+    }
+
+    /**
+     *
+     */
+    private static class VisorIdleVerifyJob extends VisorJob<VisorIdleAnalyzeTaskArg, VisorIdleAnalyzeTaskResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private ComputeTaskFuture<Map<PartitionHashRecord, List<PartitionEntryHashRecord>>> conflictKeysFut;
+
+        /** */
+        private ComputeTaskFuture<Map<PartitionHashRecord, List<PartitionEntryHashRecord>>> conflictValsFut;
+
+        /** Auto-inject job context. */
+        @JobContextResource
+        protected transient ComputeJobContext jobCtx;
+
+        /**
+         * @param arg Argument.
+         * @param debug Debug.
+         */
+        private VisorIdleVerifyJob(VisorIdleAnalyzeTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorIdleAnalyzeTaskResult run(VisorIdleAnalyzeTaskArg arg) throws IgniteException {
+            if (conflictKeysFut == null) {
+                conflictKeysFut = ignite.compute()
+                    .executeAsync(CollectConflictPartitionKeysTask.class, arg.getPartitionKey());
+
+                if (!conflictKeysFut.isDone()) {
+                    jobCtx.holdcc();
+
+                    conflictKeysFut.listen(new IgniteInClosure<IgniteFuture<Map<PartitionHashRecord, List<PartitionEntryHashRecord>>>>() {
+                        @Override public void apply(IgniteFuture<Map<PartitionHashRecord, List<PartitionEntryHashRecord>>> f) {
+                            jobCtx.callcc();
+                        }
+                    });
+
+                    return null;
+                }
+            }
+
+            Map<PartitionHashRecord, List<PartitionEntryHashRecord>> conflictKeys = conflictKeysFut.get();
+
+            if (conflictKeys.isEmpty())
+                return new VisorIdleAnalyzeTaskResult(Collections.emptyMap());
+
+            if (conflictValsFut == null) {
+                conflictValsFut = ignite.compute().executeAsync(RetrieveConflictPartitionValuesTask.class, conflictKeys);
+
+                if (!conflictValsFut.isDone()) {
+                    jobCtx.holdcc();
+
+                    conflictKeysFut.listen(new IgniteInClosure<IgniteFuture<Map<PartitionHashRecord, List<PartitionEntryHashRecord>>>>() {
+                        @Override public void apply(IgniteFuture<Map<PartitionHashRecord, List<PartitionEntryHashRecord>>> f) {
+                            jobCtx.callcc();
+                        }
+                    });
+
+                    return null;
+                }
+            }
+
+            return new VisorIdleAnalyzeTaskResult(conflictValsFut.get());
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorIdleVerifyJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTaskArg.java
new file mode 100644
index 0000000000000..884f9610c42a7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTaskArg.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Arguments for task {@link VisorIdleAnalyzeTask}
+ */
+public class VisorIdleAnalyzeTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Partition key. */
+    private PartitionKey partKey;
+
+    /**
+     * Default constructor.
+     */
+    public VisorIdleAnalyzeTaskArg() {
+        // No-op.
+    }
+
+    /**
+     * @param partKey Partition key.
+     */
+    public VisorIdleAnalyzeTaskArg(PartitionKey partKey) {
+        this.partKey = partKey;
+    }
+
+    /**
+     * @param grpId Group id.
+     * @param partId Partition id.
+     * @param grpName Group name.
+     */
+    public VisorIdleAnalyzeTaskArg(int grpId, int partId, String grpName) {
+        this(new PartitionKey(grpId, partId, grpName));
+    }
+
+    /**
+     * @return Partition key.
+     */
+    public PartitionKey getPartitionKey() {
+        return partKey;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        out.writeInt(partKey.groupId());
+        out.writeInt(partKey.partitionId());
+        U.writeString(out, partKey.groupName());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException {
+        int grpId = in.readInt();
+        int partId = in.readInt();
+        String grpName = U.readString(in);
+
+        partKey = new PartitionKey(grpId, partId, grpName);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorIdleAnalyzeTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTaskResult.java
new file mode 100644
index 0000000000000..e8c129033ef8d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleAnalyzeTaskResult.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.processors.cache.verify.PartitionEntryHashRecord;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Result for task {@link VisorIdleAnalyzeTask}
+ */
+public class VisorIdleAnalyzeTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Results. */
+    private Map<PartitionHashRecord, List<PartitionEntryHashRecord>> divergedEntries;
+
+    /**
+     * Default constructor.
+     */
+    public VisorIdleAnalyzeTaskResult() {
+        // No-op.
+    }
+
+    /**
+     * @param divergedEntries Result.
+     */
+    public VisorIdleAnalyzeTaskResult(Map<PartitionHashRecord, List<PartitionEntryHashRecord>> divergedEntries) {
+        this.divergedEntries = divergedEntries;
+    }
+
+    /**
+     * @return Results.
+     */
+    public Map<PartitionHashRecord, List<PartitionEntryHashRecord>> getDivergedEntries() {
+        return divergedEntries;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, divergedEntries);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        divergedEntries = U.readMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorIdleAnalyzeTaskResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTask.java
new file mode 100644
index 0000000000000..05f2621310366
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTask.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobContext;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.resources.JobContextResource;
+
+/**
+ * Task to verify checksums of backup partitions.
+ */
+@GridInternal
+public class VisorIdleVerifyTask extends VisorOneNodeTask<VisorIdleVerifyTaskArg, VisorIdleVerifyTaskResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<VisorIdleVerifyTaskArg, VisorIdleVerifyTaskResult> job(VisorIdleVerifyTaskArg arg) {
+        return new VisorIdleVerifyJob(arg, debug);
+    }
+
+    /**
+     *
+     */
+    private static class VisorIdleVerifyJob extends VisorJob<VisorIdleVerifyTaskArg, VisorIdleVerifyTaskResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private ComputeTaskFuture<Map<PartitionKey, List<PartitionHashRecord>>> fut;
+
+        /** Auto-inject job context. */
+        @JobContextResource
+        protected transient ComputeJobContext jobCtx;
+
+        /**
+         * @param arg Argument.
+         * @param debug Debug.
+         */
+        private VisorIdleVerifyJob(VisorIdleVerifyTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorIdleVerifyTaskResult run(VisorIdleVerifyTaskArg arg) throws IgniteException {
+            if (fut == null) {
+                fut = ignite.compute().executeAsync(VerifyBackupPartitionsTask.class, arg.getCaches());
+
+                if (!fut.isDone()) {
+                    jobCtx.holdcc();
+
+                    fut.listen(new IgniteInClosure<IgniteFuture<Map<PartitionKey, List<PartitionHashRecord>>>>() {
+                        @Override public void apply(IgniteFuture<Map<PartitionKey, List<PartitionHashRecord>>> f) {
+                            jobCtx.callcc();
+                        }
+                    });
+
+                    return null;
+                }
+            }
+
+            return new VisorIdleVerifyTaskResult(fut.get());
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorIdleVerifyJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
new file mode 100644
index 0000000000000..c82af5878ebb6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Set;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Arguments for task {@link VisorIdleVerifyTask}
+ */
+public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Caches. */
+    private Set<String> caches;
+
+    /**
+     * Default constructor.
+     */
+    public VisorIdleVerifyTaskArg() {
+        // No-op.
+    }
+
+    /**
+     * @param caches Caches.
+     */
+    public VisorIdleVerifyTaskArg(Set<String> caches) {
+        this.caches = caches;
+    }
+
+
+    /**
+     * @return Caches.
+     */
+    public Set<String> getCaches() {
+        return caches;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeCollection(out, caches);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        caches = U.readSet(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorIdleVerifyTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskResult.java
new file mode 100644
index 0000000000000..7ef542f019e62
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskResult.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Result for task {@link VisorIdleVerifyTask}
+ */
+public class VisorIdleVerifyTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Results. */
+    private Map<PartitionKey, List<PartitionHashRecord>> conflicts;
+
+    /**
+     * Default constructor.
+     */
+    public VisorIdleVerifyTaskResult() {
+        // No-op.
+    }
+
+    /**
+     * @param conflicts Result.
+     */
+    public VisorIdleVerifyTaskResult(Map<PartitionKey, List<PartitionHashRecord>> conflicts) {
+        this.conflicts = conflicts;
+    }
+
+    /**
+     * @return Results.
+     */
+    public Map<PartitionKey, List<PartitionHashRecord>> getConflicts() {
+        return conflicts;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, conflicts);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        conflicts = U.readMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorIdleVerifyTaskResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
new file mode 100644
index 0000000000000..25c97b651a4d9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
@@ -0,0 +1,73 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Map;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *
+ */
+public class VisorValidateIndexesJobResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Results of indexes validation from node. */
+    private Map<PartitionKey, ValidateIndexesPartitionResult> res;
+
+    /**
+     * @param res Results of indexes validation from node.
+     */
+    public VisorValidateIndexesJobResult(Map<PartitionKey, ValidateIndexesPartitionResult> res) {
+        this.res = res;
+    }
+
+    /**
+     * For externalization only.
+     */
+    public VisorValidateIndexesJobResult() {
+    }
+
+    /**
+     * @return Results of indexes validation from node.
+     */
+    public Map<PartitionKey, ValidateIndexesPartitionResult> response() {
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, res);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        res = U.readMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorValidateIndexesJobResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
new file mode 100644
index 0000000000000..cf9aff5ac2033
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Set;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Arguments for task {@link VisorIdleVerifyTask}
+ */
+public class VisorValidateIndexesTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Caches. */
+    private Set<String> caches;
+
+    /**
+     * Default constructor.
+     */
+    public VisorValidateIndexesTaskArg() {
+        // No-op.
+    }
+
+    /**
+     * @param caches Caches.
+     */
+    public VisorValidateIndexesTaskArg(Set<String> caches) {
+        this.caches = caches;
+    }
+
+
+    /**
+     * @return Caches.
+     */
+    public Set<String> getCaches() {
+        return caches;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeCollection(out, caches);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        caches = U.readSet(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorValidateIndexesTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskResult.java
new file mode 100644
index 0000000000000..e206448c43ccc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskResult.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *
+ */
+public class VisorValidateIndexesTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Exceptions. */
+    private Map<UUID, Exception> exceptions;
+
+    /** Results from cluster. */
+    private Map<UUID, VisorValidateIndexesJobResult> results;
+
+    /**
+     * @param results Results.
+     * @param exceptions Exceptions.
+     */
+    public VisorValidateIndexesTaskResult(Map<UUID, VisorValidateIndexesJobResult> results,
+        Map<UUID, Exception> exceptions) {
+        this.exceptions = exceptions;
+        this.results = results;
+    }
+
+    /**
+     * For externalization only.
+     */
+    public VisorValidateIndexesTaskResult() {
+    }
+
+    /**
+     * @return Exceptions.
+     */
+    public Map<UUID, Exception> exceptions() {
+        return exceptions;
+    }
+
+    /**
+     * @return Results from cluster.
+     */
+    public Map<UUID, VisorValidateIndexesJobResult> results() {
+        return results;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, exceptions);
+        U.writeMap(out, results);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        exceptions = U.readMap(in);
+        results = U.readMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorValidateIndexesTaskResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheCmd.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheCmd.java
new file mode 100644
index 0000000000000..0d9ce3ae20ad8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheCmd.java
@@ -0,0 +1,47 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.visor.verify;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public enum VisorViewCacheCmd {
+    /** Caches. */
+    CACHES,
+
+    /** Groups. */
+    GROUPS,
+
+    /** Sequence. */
+    SEQ;
+
+    /** Enumerated values. */
+    private static final VisorViewCacheCmd[] VALS = values();
+
+    /**
+     * Efficiently gets enumerated value from its ordinal.
+     *
+     * @param ord Ordinal value.
+     * @return Enumerated value or {@code null} if ordinal out of range.
+     */
+    @Nullable public static VisorViewCacheCmd fromOrdinal(int ord) {
+        return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTask.java
new file mode 100644
index 0000000000000..86931e60c64ae
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTask.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.cache.verify.ViewCacheClosure;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+@GridInternal
+public class VisorViewCacheTask extends VisorOneNodeTask<VisorViewCacheTaskArg, VisorViewCacheTaskResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<VisorViewCacheTaskArg, VisorViewCacheTaskResult> job(VisorViewCacheTaskArg arg) {
+        return new VisorViewCacheJob(arg, debug);
+    }
+
+    /**
+     *
+     */
+    private static class VisorViewCacheJob extends VisorJob<VisorViewCacheTaskArg, VisorViewCacheTaskResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param arg Argument.
+         * @param debug Debug.
+         */
+        protected VisorViewCacheJob(@Nullable VisorViewCacheTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorViewCacheTaskResult run(@Nullable VisorViewCacheTaskArg arg) throws IgniteException {
+            try {
+                ViewCacheClosure clo = new ViewCacheClosure(arg.regex(), arg.command());
+
+                ignite.context().resource().injectGeneric(clo);
+
+                return new VisorViewCacheTaskResult(clo.call());
+            }
+            catch (Exception e) {
+                throw new IgniteException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorViewCacheJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java
new file mode 100644
index 0000000000000..5fcd66d18d082
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java
@@ -0,0 +1,86 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class VisorViewCacheTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Regex. */
+    private String regex;
+
+    /** Type. */
+    private @Nullable VisorViewCacheCmd cmd;
+
+    /**
+     * @param regex Regex.
+     * @param cmd Command.
+     */
+    public VisorViewCacheTaskArg(String regex, @Nullable VisorViewCacheCmd cmd) {
+        this.regex = regex;
+        this.cmd = cmd;
+    }
+
+    /**
+     * For externalization only.
+     */
+    public VisorViewCacheTaskArg() {
+    }
+
+    /**
+     * @return Regex.
+     */
+    public String regex() {
+        return regex;
+    }
+
+    /**
+     * @return Command.
+     */
+    public VisorViewCacheCmd command() {
+        return cmd;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeString(out, regex);
+        U.writeEnum(out, cmd);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        regex = U.readString(in);
+        cmd = VisorViewCacheCmd.fromOrdinal(in.readByte());
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorViewCacheTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskResult.java
new file mode 100644
index 0000000000000..138bf064710b0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskResult.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.internal.processors.cache.verify.CacheInfo;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *
+ */
+public class VisorViewCacheTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Cache infos. */
+    private List<CacheInfo> cacheInfos;
+
+    /**
+     * @param cacheInfos Cache infos.
+     */
+    public VisorViewCacheTaskResult(List<CacheInfo> cacheInfos) {
+        this.cacheInfos = cacheInfos;
+    }
+
+    /**
+     * For externalization only.
+     */
+    public VisorViewCacheTaskResult() {
+    }
+
+    /**
+     * @return Cache infos.
+     */
+    public Collection<CacheInfo> cacheInfos() {
+        return cacheInfos;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeCollection(out, cacheInfos);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        cacheInfos = U.readList(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorViewCacheTaskResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index d939b02018bbb..8ca47c8430830 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -28,6 +28,7 @@ org.apache.ignite.IgniteState
 org.apache.ignite.binary.BinaryInvalidTypeException
 org.apache.ignite.binary.BinaryObject
 org.apache.ignite.binary.BinaryObjectException
+org.apache.ignite.binary.BinaryTypeConfiguration
 org.apache.ignite.cache.CacheAtomicUpdateTimeoutException
 org.apache.ignite.cache.CacheAtomicityMode
 org.apache.ignite.cache.CacheEntryEventSerializableFilter
@@ -112,6 +113,13 @@ org.apache.ignite.cache.store.jdbc.dialect.SQLServerDialect
 org.apache.ignite.cache.store.jdbc.dialect.SQLServerDialect$1
 org.apache.ignite.cache.store.jdbc.dialect.SQLServerDialect$2
 org.apache.ignite.cache.store.jdbc.dialect.SQLServerDialect$3
+org.apache.ignite.client.ClientAuthenticationException
+org.apache.ignite.client.ClientAuthorizationException
+org.apache.ignite.client.ClientCacheConfiguration
+org.apache.ignite.client.ClientConnectionException
+org.apache.ignite.client.ClientException
+org.apache.ignite.client.SslMode
+org.apache.ignite.client.SslProtocol
 org.apache.ignite.cluster.ClusterGroupEmptyException
 org.apache.ignite.cluster.ClusterTopologyException
 org.apache.ignite.compute.ComputeExecutionRejectedException
@@ -135,9 +143,11 @@ org.apache.ignite.compute.gridify.GridifyTaskSplitAdapter
 org.apache.ignite.compute.gridify.aop.GridifyArgumentAdapter
 org.apache.ignite.compute.gridify.aop.GridifyDefaultRangeTask
 org.apache.ignite.compute.gridify.aop.GridifyDefaultTask
+org.apache.ignite.configuration.BinaryConfiguration
 org.apache.ignite.configuration.CacheConfiguration
 org.apache.ignite.configuration.CacheConfiguration$IgniteAllNodesPredicate
 org.apache.ignite.configuration.CheckpointWriteOrder
+org.apache.ignite.configuration.ClientConfiguration
 org.apache.ignite.configuration.CollectionConfiguration
 org.apache.ignite.configuration.DataPageEvictionMode
 org.apache.ignite.configuration.DataRegionConfiguration
@@ -164,6 +174,7 @@ org.apache.ignite.events.IgfsEvent
 org.apache.ignite.events.JobEvent
 org.apache.ignite.events.TaskEvent
 org.apache.ignite.events.WalSegmentArchivedEvent
+org.apache.ignite.failure.FailureType
 org.apache.ignite.hadoop.HadoopInputSplit
 org.apache.ignite.hadoop.HadoopMapReducePlan
 org.apache.ignite.igfs.IgfsConcurrentModificationException
@@ -247,7 +258,7 @@ org.apache.ignite.internal.IgniteMessagingImpl
 org.apache.ignite.internal.IgniteNeedReconnectException
 org.apache.ignite.internal.IgniteSchedulerImpl
 org.apache.ignite.internal.IgniteServicesImpl
-org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$1
+org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$4
 org.apache.ignite.internal.NodeStoppingException
 org.apache.ignite.internal.binary.BinaryEnumObjectImpl
 org.apache.ignite.internal.binary.BinaryFieldMetadata
@@ -286,6 +297,11 @@ org.apache.ignite.internal.client.impl.connection.GridClientNioTcpConnection$6
 org.apache.ignite.internal.client.impl.connection.GridClientNioTcpConnection$7
 org.apache.ignite.internal.client.impl.connection.GridClientTopology$1
 org.apache.ignite.internal.client.impl.connection.GridConnectionIdleClosedException
+org.apache.ignite.internal.client.thin.ClientError
+org.apache.ignite.internal.client.thin.ClientOperation
+org.apache.ignite.internal.client.thin.ClientProtocolError
+org.apache.ignite.internal.client.thin.ClientServerError
+org.apache.ignite.internal.client.thin.ClientUtils$CfgItem
 org.apache.ignite.internal.cluster.ClusterGroupAdapter
 org.apache.ignite.internal.cluster.ClusterGroupAdapter$AgeClusterGroup
 org.apache.ignite.internal.cluster.ClusterGroupAdapter$AttributeFilter
@@ -305,6 +321,8 @@ org.apache.ignite.internal.cluster.IgniteKillTask
 org.apache.ignite.internal.cluster.IgniteKillTask$IgniteKillJob
 org.apache.ignite.internal.cluster.NodeOrderComparator
 org.apache.ignite.internal.cluster.NodeOrderLegacyComparator
+org.apache.ignite.internal.commandline.Command
+org.apache.ignite.internal.commandline.cache.CacheCommand
 org.apache.ignite.internal.compute.ComputeTaskCancelledCheckedException
 org.apache.ignite.internal.compute.ComputeTaskTimeoutCheckedException
 org.apache.ignite.internal.direct.DirectMessageReader$1
@@ -317,7 +335,6 @@ org.apache.ignite.internal.igfs.common.IgfsIpcCommand
 org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl
 org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$BooleanProperty
 org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$ConnectionProperty
-org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$EmptyStringValidator
 org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$IntegerProperty
 org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$NumberProperty
 org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl$PropertyValidator
@@ -348,6 +365,7 @@ org.apache.ignite.internal.managers.discovery.DiscoCache$1
 org.apache.ignite.internal.managers.discovery.DiscoCache$2
 org.apache.ignite.internal.managers.discovery.DiscoCache$3
 org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage
+org.apache.ignite.internal.managers.discovery.DiscoveryServerOnlyCustomMessage
 org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$1
 org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$2
 org.apache.ignite.internal.managers.discovery.GridDiscoveryManager$4$1
@@ -372,6 +390,19 @@ org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion
 org.apache.ignite.internal.processors.affinity.GridAffinityAssignment
 org.apache.ignite.internal.processors.affinity.GridAffinityMessage
 org.apache.ignite.internal.processors.affinity.GridAffinityUtils$AffinityJob
+org.apache.ignite.internal.processors.authentication.IgniteAccessControlException
+org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor$3
+org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor$InitialUsersData
+org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor$RefreshUsersStorageWorker$1
+org.apache.ignite.internal.processors.authentication.User
+org.apache.ignite.internal.processors.authentication.UserAcceptedMessage
+org.apache.ignite.internal.processors.authentication.UserAuthenticateRequestMessage
+org.apache.ignite.internal.processors.authentication.UserAuthenticateResponseMessage
+org.apache.ignite.internal.processors.authentication.UserManagementException
+org.apache.ignite.internal.processors.authentication.UserManagementOperation
+org.apache.ignite.internal.processors.authentication.UserManagementOperation$OperationType
+org.apache.ignite.internal.processors.authentication.UserManagementOperationFinishedMessage
+org.apache.ignite.internal.processors.authentication.UserProposedMessage
 org.apache.ignite.internal.processors.bulkload.BulkLoadCacheWriter
 org.apache.ignite.internal.processors.bulkload.BulkLoadStreamerWriter
 org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage
@@ -528,6 +559,7 @@ org.apache.ignite.internal.processors.cache.GridCacheLoaderWriterStore
 org.apache.ignite.internal.processors.cache.GridCacheLoaderWriterStoreFactory
 org.apache.ignite.internal.processors.cache.GridCacheLockTimeoutException
 org.apache.ignite.internal.processors.cache.GridCacheLogger
+org.apache.ignite.internal.processors.cache.GridCacheMapEntry$1
 org.apache.ignite.internal.processors.cache.GridCacheMessage
 org.apache.ignite.internal.processors.cache.GridCacheMultiTxFuture$1
 org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate
@@ -616,6 +648,7 @@ org.apache.ignite.internal.processors.cache.WalStateAbstractMessage
 org.apache.ignite.internal.processors.cache.WalStateAckMessage
 org.apache.ignite.internal.processors.cache.WalStateFinishMessage
 org.apache.ignite.internal.processors.cache.WalStateManager$2
+org.apache.ignite.internal.processors.cache.WalStateManager$3
 org.apache.ignite.internal.processors.cache.WalStateProposeMessage
 org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityProxy
 org.apache.ignite.internal.processors.cache.binary.BinaryMetadataHolder
@@ -803,17 +836,15 @@ org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CacheParti
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysFuture$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysRequest
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysResponse
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandLegacyMessage
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$1$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$2
-org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$3
-org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$4$1
-org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$5$1
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander$3$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap
-org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplier$SupplyContextPhase
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsAbstractMessage
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$2
@@ -837,6 +868,9 @@ org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtP
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionCountersMap2
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionHistorySuppliersMap
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionsToReloadMap
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteHistoricalIterator
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.LatchAckMessage
 org.apache.ignite.internal.processors.cache.distributed.near.CacheVersionedValue
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearAtomicCache
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearAtomicCache$1
@@ -903,6 +937,7 @@ org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$20
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$21
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$22
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$23
+org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$24
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$3
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$5
@@ -926,11 +961,11 @@ org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$5
 org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$8
 org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$9
 org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter$RowData
-org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$11
+org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$12
 org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$6
-org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$8
+org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$9
 org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$CheckpointEntryType
-org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$RebalanceIteratorAdapter
+org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$WALHistoricalIterator
 org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager$1
 org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory
 org.apache.ignite.internal.processors.cache.persistence.file.FileDownloader$1
@@ -942,6 +977,7 @@ org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl$T
 org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteSpeedBasedThrottle$ThrottleMode
 org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage
 org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperation
+org.apache.ignite.internal.processors.cache.persistence.snapshot.TrackingPageIsCorruptedException
 org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Bool
 org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$DestroyBag
 org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$Result
@@ -1095,8 +1131,13 @@ org.apache.ignite.internal.processors.cache.transactions.TxLock
 org.apache.ignite.internal.processors.cache.transactions.TxLockList
 org.apache.ignite.internal.processors.cache.transactions.TxLocksRequest
 org.apache.ignite.internal.processors.cache.transactions.TxLocksResponse
+org.apache.ignite.internal.processors.cache.verify.CacheInfo
+org.apache.ignite.internal.processors.cache.verify.CacheInfo$1
+org.apache.ignite.internal.processors.cache.verify.CacheInfo$2
 org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysTask
 org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysTask$CollectPartitionEntryHashesJob
+org.apache.ignite.internal.processors.cache.verify.ContentionClosure
+org.apache.ignite.internal.processors.cache.verify.ContentionInfo
 org.apache.ignite.internal.processors.cache.verify.PartitionEntryHashRecord
 org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord
 org.apache.ignite.internal.processors.cache.verify.PartitionKey
@@ -1104,6 +1145,8 @@ org.apache.ignite.internal.processors.cache.verify.RetrieveConflictPartitionValu
 org.apache.ignite.internal.processors.cache.verify.RetrieveConflictPartitionValuesTask$RetrieveConflictValuesJob
 org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask
 org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask$VerifyBackupPartitionsJob
+org.apache.ignite.internal.processors.cache.verify.ViewCacheClosure
+org.apache.ignite.internal.processors.cache.verify.ViewCacheClosure$1
 org.apache.ignite.internal.processors.cache.version.GridCacheRawVersionedEntry
 org.apache.ignite.internal.processors.cache.version.GridCacheVersion
 org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext$State
@@ -1130,15 +1173,16 @@ org.apache.ignite.internal.processors.closure.GridClosureProcessor$T8
 org.apache.ignite.internal.processors.closure.GridClosureProcessor$T9
 org.apache.ignite.internal.processors.closure.GridClosureProcessor$TaskNoReduceAdapter
 org.apache.ignite.internal.processors.closure.GridPeerDeployAwareTaskAdapter
-org.apache.ignite.internal.processors.cluster.ClusterNodeMetrics
 org.apache.ignite.internal.processors.cluster.BaselineTopology
 org.apache.ignite.internal.processors.cluster.BaselineTopologyHistory
 org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem
 org.apache.ignite.internal.processors.cluster.BranchingPointType
 org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage
 org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage
-org.apache.ignite.internal.processors.cluster.ClusterProcessor$3
-org.apache.ignite.internal.processors.cluster.ClusterProcessor$3$1
+org.apache.ignite.internal.processors.cluster.ClusterMetricsUpdateMessage
+org.apache.ignite.internal.processors.cluster.ClusterNodeMetrics
+org.apache.ignite.internal.processors.cluster.ClusterProcessor$4
+org.apache.ignite.internal.processors.cluster.ClusterProcessor$4$1
 org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState
 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$1$1
 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$2
@@ -1150,18 +1194,24 @@ org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$Baseline
 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$CheckGlobalStateComputeRequest
 org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor$ClientChangeGlobalStateComputeRequest
 org.apache.ignite.internal.processors.continuous.AbstractContinuousMessage
+org.apache.ignite.internal.processors.continuous.ContinuousRoutineInfo
+org.apache.ignite.internal.processors.continuous.ContinuousRoutineStartResultMessage
+org.apache.ignite.internal.processors.continuous.ContinuousRoutinesCommonDiscoveryData
+org.apache.ignite.internal.processors.continuous.ContinuousRoutinesJoiningNodeDiscoveryData
 org.apache.ignite.internal.processors.continuous.GridContinuousHandler
 org.apache.ignite.internal.processors.continuous.GridContinuousHandler$RegisterStatus
 org.apache.ignite.internal.processors.continuous.GridContinuousMessage
 org.apache.ignite.internal.processors.continuous.GridContinuousMessageType
-org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$8
-org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$9$1
+org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$11$1
+org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$9
 org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$DiscoveryData
 org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$DiscoveryDataItem
 org.apache.ignite.internal.processors.continuous.GridContinuousProcessor$LocalRoutineInfo
 org.apache.ignite.internal.processors.continuous.StartRequestData
+org.apache.ignite.internal.processors.continuous.StartRequestDataV2
 org.apache.ignite.internal.processors.continuous.StartRoutineAckDiscoveryMessage
 org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessage
+org.apache.ignite.internal.processors.continuous.StartRoutineDiscoveryMessageV2
 org.apache.ignite.internal.processors.continuous.StopRoutineAckDiscoveryMessage
 org.apache.ignite.internal.processors.continuous.StopRoutineDiscoveryMessage
 org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor$3
@@ -1427,6 +1477,7 @@ org.apache.ignite.internal.processors.query.GridQueryProcessor$5
 org.apache.ignite.internal.processors.query.GridQueryProcessor$6
 org.apache.ignite.internal.processors.query.GridQueryProcessor$7
 org.apache.ignite.internal.processors.query.GridQueryProcessor$8
+org.apache.ignite.internal.processors.query.GridQueryProcessor$9
 org.apache.ignite.internal.processors.query.GridQueryProcessor$SchemaOperation$1
 org.apache.ignite.internal.processors.query.IgniteSQLException
 org.apache.ignite.internal.processors.query.QueryEntityEx
@@ -1543,6 +1594,7 @@ org.apache.ignite.internal.processors.service.GridServiceProcessor$ServiceDeploy
 org.apache.ignite.internal.processors.service.GridServiceProcessor$ServiceTopologyCallable
 org.apache.ignite.internal.processors.service.GridServiceProxy
 org.apache.ignite.internal.processors.service.GridServiceProxy$ServiceProxyCallable
+org.apache.ignite.internal.processors.service.GridServiceProxy$ServiceProxyException
 org.apache.ignite.internal.processors.service.LazyServiceConfiguration
 org.apache.ignite.internal.processors.service.ServiceContextImpl
 org.apache.ignite.internal.processors.service.ServiceDescriptorImpl
@@ -1622,6 +1674,7 @@ org.apache.ignite.internal.util.GridSnapshotLock$Sync
 org.apache.ignite.internal.util.GridSpiCloseableIteratorWrapper
 org.apache.ignite.internal.util.GridStringBuilder
 org.apache.ignite.internal.util.GridSynchronizedMap
+org.apache.ignite.internal.util.HostAndPortRange
 org.apache.ignite.internal.util.IgniteExceptionRegistry$ExceptionInfo
 org.apache.ignite.internal.util.IgniteTree$OperationType
 org.apache.ignite.internal.util.IgniteUtils$10
@@ -1860,6 +1913,10 @@ org.apache.ignite.internal.visor.cache.VisorCacheJdbcTypeField
 org.apache.ignite.internal.visor.cache.VisorCacheLoadTask
 org.apache.ignite.internal.visor.cache.VisorCacheLoadTask$VisorCachesLoadJob
 org.apache.ignite.internal.visor.cache.VisorCacheLoadTaskArg
+org.apache.ignite.internal.visor.cache.VisorCacheLostPartitionsTask
+org.apache.ignite.internal.visor.cache.VisorCacheLostPartitionsTask$VisorCacheLostPartitionsJob
+org.apache.ignite.internal.visor.cache.VisorCacheLostPartitionsTaskArg
+org.apache.ignite.internal.visor.cache.VisorCacheLostPartitionsTaskResult
 org.apache.ignite.internal.visor.cache.VisorCacheMetadataTask
 org.apache.ignite.internal.visor.cache.VisorCacheMetadataTask$VisorCacheMetadataJob
 org.apache.ignite.internal.visor.cache.VisorCacheMetadataTaskArg
@@ -1883,6 +1940,9 @@ org.apache.ignite.internal.visor.cache.VisorCacheRebalanceConfiguration
 org.apache.ignite.internal.visor.cache.VisorCacheRebalanceTask
 org.apache.ignite.internal.visor.cache.VisorCacheRebalanceTask$VisorCachesRebalanceJob
 org.apache.ignite.internal.visor.cache.VisorCacheRebalanceTaskArg
+org.apache.ignite.internal.visor.cache.VisorCacheResetLostPartitionsTask
+org.apache.ignite.internal.visor.cache.VisorCacheResetLostPartitionsTask$VisorCacheResetLostPartitionsJob
+org.apache.ignite.internal.visor.cache.VisorCacheResetLostPartitionsTaskArg
 org.apache.ignite.internal.visor.cache.VisorCacheResetMetricsTask
 org.apache.ignite.internal.visor.cache.VisorCacheResetMetricsTask$VisorCacheResetMetricsJob
 org.apache.ignite.internal.visor.cache.VisorCacheResetMetricsTaskArg
@@ -2076,6 +2136,31 @@ org.apache.ignite.internal.visor.tx.VisorTxSortOrder
 org.apache.ignite.internal.visor.tx.VisorTxTask
 org.apache.ignite.internal.visor.tx.VisorTxTaskArg
 org.apache.ignite.internal.visor.tx.VisorTxTaskResult
+org.apache.ignite.internal.visor.verify.VisorViewCacheCmd
+org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult
+org.apache.ignite.internal.visor.verify.VisorContentionJobResult
+org.apache.ignite.internal.visor.verify.VisorContentionTask
+org.apache.ignite.internal.visor.verify.VisorContentionTask$VisorContentionJob
+org.apache.ignite.internal.visor.verify.VisorContentionTaskArg
+org.apache.ignite.internal.visor.verify.VisorContentionTaskResult
+org.apache.ignite.internal.visor.verify.VisorIdleAnalyzeTask
+org.apache.ignite.internal.visor.verify.VisorIdleAnalyzeTask$VisorIdleVerifyJob
+org.apache.ignite.internal.visor.verify.VisorIdleAnalyzeTask$VisorIdleVerifyJob$1
+org.apache.ignite.internal.visor.verify.VisorIdleAnalyzeTask$VisorIdleVerifyJob$2
+org.apache.ignite.internal.visor.verify.VisorIdleAnalyzeTaskArg
+org.apache.ignite.internal.visor.verify.VisorIdleAnalyzeTaskResult
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask$VisorIdleVerifyJob
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask$VisorIdleVerifyJob$1
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult
+org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult
+org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskArg
+org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskResult
+org.apache.ignite.internal.visor.verify.VisorViewCacheTask
+org.apache.ignite.internal.visor.verify.VisorViewCacheTask$VisorViewCacheJob
+org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg
+org.apache.ignite.internal.visor.verify.VisorViewCacheTaskResult
 org.apache.ignite.internal.websession.WebSessionAttributeProcessor
 org.apache.ignite.internal.websession.WebSessionEntity
 org.apache.ignite.lang.IgniteBiClosure
@@ -2143,6 +2228,7 @@ org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$4
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeClosure
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeException
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$HandshakeTimeoutException
+org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture$SingleAddressConnectFuture$1
 org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage
 org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2
 org.apache.ignite.spi.communication.tcp.messages.NodeIdMessage
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index c2d9eec4bae58..dbd6107f737b1 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -22,13 +22,19 @@
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteAtomicSequence;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ConnectorConfiguration;
@@ -36,8 +42,10 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.commandline.CommandHandler;
+import org.apache.ignite.internal.commandline.cache.CacheCommand;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.X;
@@ -49,8 +57,8 @@
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionRollbackException;
 
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheWriteSynchronizationMode.*;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
 import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
@@ -61,6 +69,12 @@
  * Command line handler test.
  */
 public class GridCommandHandlerTest extends GridCommonAbstractTest {
+    /** System out. */
+    protected PrintStream sysOut;
+
+    /** Test out - can be injected via {@link #injectTestSystemOut()} instead of System.out and analyzed in test. */
+    protected ByteArrayOutputStream testOut;
+
     /**
      * @return Folder in work directory.
      * @throws IgniteCheckedException If failed to resolve folder name.
@@ -74,6 +88,10 @@ protected File folder(String folder) throws IgniteCheckedException {
         cleanPersistenceDir();
 
         stopAllGrids();
+
+        sysOut = System.out;
+
+        testOut = new ByteArrayOutputStream(128 * 1024);
     }
 
     /** {@inheritDoc} */
@@ -81,6 +99,18 @@ protected File folder(String folder) throws IgniteCheckedException {
         stopAllGrids();
 
         cleanPersistenceDir();
+
+        System.setOut(sysOut);
+
+        if (testOut != null)
+            System.out.println(testOut.toString());
+    }
+
+    /**
+     *
+     */
+    protected void injectTestSystemOut() {
+        System.setOut(new PrintStream(testOut));
     }
 
     /** {@inheritDoc} */
@@ -262,40 +292,6 @@ public void testBaselineAdd() throws Exception {
         assertEquals(2, ignite.cluster().currentBaselineTopology().size());
     }
 
-    /**
-     * Test baseline add items works via control.sh
-     *
-     * @throws Exception If failed.
-     */
-    public void testBaselineAddOnNotActiveCluster() throws Exception {
-        try {
-            Ignite ignite = startGrid(1);
-
-            assertFalse(ignite.cluster().active());
-
-            String consistentIDs = getTestIgniteInstanceName(1);
-
-            ByteArrayOutputStream out = new ByteArrayOutputStream(4096);
-            System.setOut(new PrintStream(out));
-
-            assertEquals(EXIT_CODE_UNEXPECTED_ERROR, execute("--baseline", "add", consistentIDs));
-
-            assertTrue(out.toString().contains("Changing BaselineTopology on inactive cluster is not allowed."));
-
-            consistentIDs =
-                getTestIgniteInstanceName(1) + ", " +
-                    getTestIgniteInstanceName(2) + "," +
-                    getTestIgniteInstanceName(3);
-
-            assertEquals(EXIT_CODE_UNEXPECTED_ERROR, execute("--baseline", "add", consistentIDs));
-
-            assertTrue(out.toString().contains("Node not found for consistent ID: bltTest2"));
-        }
-        finally {
-            System.setOut(System.out);
-        }
-    }
-
     /**
      * Test baseline remove works via control.sh
      *
@@ -373,7 +369,7 @@ public void testActiveTransactions() throws Exception {
 
         Ignite client = startGrid("client");
 
-        IgniteCache<Object, Object> cache = client.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+        client.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
             .setAtomicityMode(TRANSACTIONAL).setWriteSynchronizationMode(FULL_SYNC));
 
         for (Ignite ig : G.allGrids())
@@ -401,9 +397,9 @@ public void testActiveTransactions() throws Exception {
 
                             fail("Commit must fail");
                         }
-                        catch (Exception ignored) {
+                        catch (Exception e) {
                             // No-op.
-                            assertTrue(X.hasCause(ignored, TransactionRollbackException.class));
+                            assertTrue(X.hasCause(e, TransactionRollbackException.class));
                         }
 
                         break;
@@ -534,14 +530,246 @@ else if (entry.getKey().equals(node2)) {
         fut.get();
     }
 
+    /**
+     * Test baseline add items works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testBaselineAddOnNotActiveCluster() throws Exception {
+        Ignite ignite = startGrid(1);
+
+        assertFalse(ignite.cluster().active());
+
+        String consistentIDs = getTestIgniteInstanceName(1);
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_UNEXPECTED_ERROR, execute("--baseline", "add", consistentIDs));
+
+        assertTrue(testOut.toString().contains("Changing BaselineTopology on inactive cluster is not allowed."));
+
+        consistentIDs =
+            getTestIgniteInstanceName(1) + ", " +
+                getTestIgniteInstanceName(2) + "," +
+                getTestIgniteInstanceName(3);
+
+        assertEquals(EXIT_CODE_UNEXPECTED_ERROR, execute("--baseline", "add", consistentIDs));
+
+        assertTrue(testOut.toString().contains("Node not found for consistent ID: bltTest2"));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCacheHelp() throws Exception {
+        Ignite ignite = startGrids(1);
+
+        ignite.cluster().active(true);
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "help"));
+
+        for (CacheCommand cmd : CacheCommand.values()) {
+            if (cmd != CacheCommand.HELP)
+                assertTrue(cmd.text(), testOut.toString().contains(cmd.text()));
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCacheIdleVerify() throws Exception {
+        Ignite ignite = startGrids(2);
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1)
+            .setName("cacheIV"));
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, i);
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify"));
+
+        assertTrue(testOut.toString().contains("no conflicts have been found"));
+
+        HashSet<Integer> clearKeys = new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6));
+
+        ((IgniteEx)ignite).context().cache().cache("cacheIV").clearLocallyAll(clearKeys, true, true, true);
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify"));
+
+        assertTrue(testOut.toString().contains("conflict partitions"));
+    }
+
+    /**
+     *
+     */
+    public void testCacheContention() throws Exception {
+        int cnt = 10;
+
+        final ExecutorService svc = Executors.newFixedThreadPool(cnt);
+
+        try {
+            Ignite ignite = startGrids(2);
+
+            ignite.cluster().active(true);
+
+            final IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+                .setAffinity(new RendezvousAffinityFunction(false, 32))
+                .setAtomicityMode(TRANSACTIONAL)
+                .setBackups(1)
+                .setName("cacheCont"));
+
+            final CountDownLatch l = new CountDownLatch(1);
+
+            final CountDownLatch l2 = new CountDownLatch(1);
+
+            svc.submit(new Runnable() {
+                @Override public void run() {
+                    try (final Transaction tx = ignite.transactions().txStart()) {
+                        cache.put(0, 0);
+
+                        l.countDown();
+
+                        U.awaitQuiet(l2);
+
+                        tx.commit();
+                    }
+                }
+            });
+
+            for (int i = 0; i < cnt - 1; i++) {
+                svc.submit(new Runnable() {
+                    @Override public void run() {
+                        U.awaitQuiet(l);
+
+                        try (final Transaction tx = ignite.transactions().txStart()) {
+                            cache.get(0);
+
+                            tx.commit();
+                        }
+                    }
+                });
+            }
+
+            U.awaitQuiet(l);
+
+            Thread.sleep(300);
+
+            injectTestSystemOut();
+
+            assertEquals(EXIT_CODE_OK, execute("--cache", "contention", "5"));
+
+            l2.countDown();
+
+            assertTrue(testOut.toString().contains("TxEntry"));
+            assertTrue(testOut.toString().contains("op=READ"));
+            assertTrue(testOut.toString().contains("op=CREATE"));
+            assertTrue(testOut.toString().contains("id=" + ignite(0).cluster().localNode().id()));
+            assertTrue(testOut.toString().contains("id=" + ignite(1).cluster().localNode().id()));
+        }
+        finally {
+            svc.shutdown();
+            svc.awaitTermination(100, TimeUnit.DAYS);
+        }
+    }
+
+    /**
+     *
+     */
+    public void testCacheSequence() throws Exception {
+        Ignite ignite = startGrid();
+
+        ignite.cluster().active(true);
+
+        Ignite client = startGrid("client");
+
+        final IgniteAtomicSequence seq1 = client.atomicSequence("testSeq", 1, true);
+        seq1.get();
+
+        final IgniteAtomicSequence seq2 = client.atomicSequence("testSeq2", 10, true);
+        seq2.get();
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "list", "testSeq.*", "seq"));
+
+        assertTrue(testOut.toString().contains("testSeq"));
+        assertTrue(testOut.toString().contains("testSeq2"));
+    }
+
+    /**
+     *
+     */
+    public void testCacheGroups() throws Exception {
+        Ignite ignite = startGrid();
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Object, Object> cache1 = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1)
+            .setGroupName("G100")
+            .setName("cacheG1"));
+
+        IgniteCache<Object, Object> cache2 = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1)
+            .setGroupName("G100")
+            .setName("cacheG2"));
+
+        for (int i = 0; i < 100; i++) {
+            cache1.put(i, i);
+
+            cache2.put(i, i);
+        }
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "list", ".*", "groups"));
+
+        assertTrue(testOut.toString().contains("G100"));
+    }
+
+    /**
+     *
+     */
+    public void testCacheAffinity() throws Exception {
+        Ignite ignite = startGrid();
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Object, Object> cache1 = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1)
+            .setName("cacheAf"));
+
+        for (int i = 0; i < 100; i++)
+            cache1.put(i, i);
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "list", ".*"));
+
+        assertTrue(testOut.toString().contains("cacheName=cacheAf"));
+        assertTrue(testOut.toString().contains("prim=32"));
+        assertTrue(testOut.toString().contains("mapped=32"));
+        assertTrue(testOut.toString().contains("affCls=RendezvousAffinityFunction"));
+    }
+
     /**
      * @param h Handler.
      * @param validateClo Validate clo.
      * @param args Args.
      */
-    private void validate(
-        CommandHandler h, IgniteInClosure<Map<ClusterNode, VisorTxTaskResult>> validateClo, String... args)
-        throws IgniteCheckedException {
+    private void validate(CommandHandler h, IgniteInClosure<Map<ClusterNode, VisorTxTaskResult>> validateClo,
+        String... args) {
         assertEquals(EXIT_CODE_OK, execute(h, args));
 
         validateClo.apply(h.getLastOperationResult());
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
new file mode 100644
index 0000000000000..373bd15dad315
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -0,0 +1,356 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.visor.verify;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteInterruptedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
+import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.QueryTypeDescriptorImpl;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
+import org.apache.ignite.internal.util.lang.GridIterator;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.h2.engine.Session;
+import org.h2.index.Cursor;
+import org.h2.index.Index;
+
+/**
+ *
+ */
+public class ValidateIndexesClosure implements IgniteCallable<Map<PartitionKey, ValidateIndexesPartitionResult>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Ignite. */
+    @IgniteInstanceResource
+    private transient IgniteEx ignite;
+
+    /** Injected logger. */
+    @LoggerResource
+    private IgniteLogger log;
+
+    /** Cache names. */
+    private Set<String> cacheNames;
+
+    /** Counter of processed partitions. */
+    private final AtomicInteger completionCntr = new AtomicInteger(0);
+
+    /** Calculation executor. */
+    private volatile ExecutorService calcExecutor;
+
+    /**
+     * @param cacheNames Cache names.
+     */
+    public ValidateIndexesClosure(Set<String> cacheNames) {
+        this.cacheNames = cacheNames;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Map<PartitionKey, ValidateIndexesPartitionResult> call() throws Exception {
+        calcExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
+
+        try {
+            return call0();
+        }
+        finally {
+            calcExecutor.shutdown();
+        }
+    }
+
+    /**
+     *
+     */
+    private Map<PartitionKey, ValidateIndexesPartitionResult> call0() throws Exception {
+        Set<Integer> grpIds = new HashSet<>();
+
+        Set<String> missingCaches = new HashSet<>();
+
+        if (cacheNames != null) {
+            for (String cacheName : cacheNames) {
+                DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName);
+
+                if (desc == null) {
+                    missingCaches.add(cacheName);
+
+                    continue;
+                }
+
+                grpIds.add(desc.groupId());
+            }
+
+            if (!missingCaches.isEmpty()) {
+                StringBuilder strBuilder = new StringBuilder("The following caches do not exist: ");
+
+                for (String name : missingCaches)
+                    strBuilder.append(name).append(", ");
+
+                strBuilder.delete(strBuilder.length() - 2, strBuilder.length());
+
+                throw new IgniteException(strBuilder.toString());
+            }
+        }
+        else {
+            Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
+
+            for (CacheGroupContext grp : groups) {
+                if (!grp.systemCache() && !grp.isLocal())
+                    grpIds.add(grp.groupId());
+            }
+        }
+
+        List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>();
+
+        completionCntr.set(0);
+
+        for (Integer grpId : grpIds) {
+            CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
+
+            if (grpCtx == null)
+                continue;
+
+            List<GridDhtLocalPartition> parts = grpCtx.topology().localPartitions();
+
+            for (GridDhtLocalPartition part : parts)
+                procPartFutures.add(processPartitionAsync(grpCtx, part));
+        }
+
+        Map<PartitionKey, ValidateIndexesPartitionResult> res = new HashMap<>();
+
+        long lastProgressLogTs = U.currentTimeMillis();
+
+        for (int i = 0; i < procPartFutures.size(); ) {
+            Future<Map<PartitionKey, ValidateIndexesPartitionResult>> fut = procPartFutures.get(i);
+
+            try {
+                Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get(1, TimeUnit.SECONDS);
+
+                res.putAll(partRes);
+
+                i++;
+            }
+            catch (InterruptedException | ExecutionException e) {
+                for (int j = i + 1; j < procPartFutures.size(); j++)
+                    procPartFutures.get(j).cancel(false);
+
+                if (e instanceof InterruptedException)
+                    throw new IgniteInterruptedException((InterruptedException)e);
+                else if (e.getCause() instanceof IgniteException)
+                    throw (IgniteException)e.getCause();
+                else
+                    throw new IgniteException(e.getCause());
+            }
+            catch (TimeoutException ignored) {
+                if (U.currentTimeMillis() - lastProgressLogTs > 60 * 1000L) {
+                    lastProgressLogTs = U.currentTimeMillis();
+
+                    log.warning("ValidateIndexesClosure is still running, processed " + completionCntr.get() + " of " +
+                        procPartFutures.size() + " local partitions");
+                }
+            }
+        }
+
+        return res;
+    }
+
+    /**
+     * @param grpCtx Group context.
+     * @param part Local partition.
+     */
+    private Future<Map<PartitionKey, ValidateIndexesPartitionResult>> processPartitionAsync(
+        final CacheGroupContext grpCtx,
+        final GridDhtLocalPartition part
+    ) {
+        return calcExecutor.submit(new Callable<Map<PartitionKey, ValidateIndexesPartitionResult>>() {
+            @Override public Map<PartitionKey, ValidateIndexesPartitionResult> call() throws Exception {
+                return processPartition(grpCtx, part);
+            }
+        });
+    }
+
+    /**
+     * @param grpCtx Group context.
+     * @param part Local partition.
+     */
+    private Map<PartitionKey, ValidateIndexesPartitionResult> processPartition(
+        CacheGroupContext grpCtx,
+        GridDhtLocalPartition part
+    ) {
+        if (!part.reserve())
+            return Collections.emptyMap();
+
+        ValidateIndexesPartitionResult partRes;
+
+        try {
+            if (part.state() != GridDhtPartitionState.OWNING)
+                return Collections.emptyMap();
+
+            long updateCntrBefore = part.updateCounter();
+
+            long partSize = part.dataStore().fullSize();
+
+            GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
+
+            Object consId = ignite.context().discovery().localNode().consistentId();
+
+            boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
+
+            partRes = new ValidateIndexesPartitionResult(updateCntrBefore, partSize, isPrimary, consId);
+
+            boolean enoughIssues = false;
+
+            long keysProcessed = 0;
+            long lastProgressLog = U.currentTimeMillis();
+
+            while (it.hasNextX()) {
+                if (enoughIssues)
+                    break;
+
+                CacheDataRow row = it.nextX();
+
+                int cacheId = row.cacheId() == 0 ? grpCtx.groupId() : row.cacheId();
+
+                GridCacheContext cacheCtx = row.cacheId() == 0 ?
+                    grpCtx.singleCacheContext() : grpCtx.shared().cacheContext(row.cacheId());
+
+                if (cacheCtx == null)
+                    throw new IgniteException("Unknown cacheId of CacheDataRow: " + cacheId);
+
+                GridQueryProcessor qryProcessor = ignite.context().query();
+
+                try {
+                    Method m = GridQueryProcessor.class.getDeclaredMethod("typeByValue", String.class,
+                        CacheObjectContext.class, KeyCacheObject.class, CacheObject.class, boolean.class);
+
+                    m.setAccessible(true);
+
+                    QueryTypeDescriptorImpl res = (QueryTypeDescriptorImpl)m.invoke(
+                        qryProcessor, cacheCtx.name(), cacheCtx.cacheObjectContext(), row.key(), row.value(), true);
+
+                    if (res == null)
+                        continue; // Tolerate - (k, v) is just not indexed.
+
+                    IgniteH2Indexing indexing = (IgniteH2Indexing)qryProcessor.getIndexing();
+
+                    GridH2Table gridH2Tbl = indexing.dataTable(cacheCtx.name(), res.tableName());
+
+                    if (gridH2Tbl == null)
+                        continue; // Tolerate - (k, v) is just not indexed.
+
+                    GridH2RowDescriptor gridH2RowDesc = gridH2Tbl.rowDescriptor();
+
+                    GridH2Row h2Row = gridH2RowDesc.createRow(row);
+
+                    ArrayList<Index> indexes = gridH2Tbl.getIndexes();
+
+                    for (Index idx : indexes) {
+                        try {
+                            Cursor cursor = idx.find((Session) null, h2Row, h2Row);
+
+                            if (cursor == null || !cursor.next())
+                                throw new IgniteCheckedException("Key not found.");
+                        }
+                        catch (Throwable t) {
+                            Object o = CacheObjectUtils.unwrapBinaryIfNeeded(
+                                grpCtx.cacheObjectContext(), row.key(), true, true);
+
+                            IndexValidationIssue is = new IndexValidationIssue(
+                                o.toString(), cacheCtx.name(), idx.getName(), t);
+
+                            log.error("Failed to lookup key: " + is.toString());
+
+                            enoughIssues |= partRes.reportIssue(is);
+                        }
+                    }
+                }
+                catch (IllegalAccessException | NoSuchMethodException e) {
+                    log.error("Failed to invoke typeByValue", e);
+
+                    throw new IgniteException(e);
+                }
+                catch (InvocationTargetException e) {
+                    Throwable target = e.getTargetException();
+
+                    log.error("Failed to invoke typeByValue", target);
+
+                    throw new IgniteException(target);
+                }
+                finally {
+                    keysProcessed++;
+
+                    if (U.currentTimeMillis() - lastProgressLog >= 60_000 && partSize > 0) {
+                        log.warning("Processing partition " + part.id() + " (" + (keysProcessed * 100 / partSize) +
+                            "% " + keysProcessed + "/" + partSize + ")");
+
+                        lastProgressLog = U.currentTimeMillis();
+                    }
+                }
+            }
+        }
+        catch (IgniteCheckedException e) {
+            U.error(log, "Failed to process partition [grpId=" + grpCtx.groupId() +
+                ", partId=" + part.id() + "]", e);
+
+            return Collections.emptyMap();
+        }
+        finally {
+            part.release();
+        }
+
+        PartitionKey partKey = new PartitionKey(grpCtx.groupId(), part.id(), grpCtx.cacheOrGroupName());
+
+        completionCntr.incrementAndGet();
+
+        return Collections.singletonMap(partKey, partRes);
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
new file mode 100644
index 0000000000000..1a89c2cadb57e
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
@@ -0,0 +1,99 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+@GridInternal
+public class VisorValidateIndexesTask extends VisorMultiNodeTask<VisorValidateIndexesTaskArg,
+    VisorValidateIndexesTaskResult, VisorValidateIndexesJobResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected VisorValidateIndexesTaskResult reduce0(List<ComputeJobResult> list) throws IgniteException {
+        Map<UUID, Exception> exceptions = new HashMap<>();
+        Map<UUID, VisorValidateIndexesJobResult> jobResults = new HashMap<>();
+
+        for (ComputeJobResult res : list) {
+            if (res.getException() != null)
+                exceptions.put(res.getNode().id(), res.getException());
+            else
+                jobResults.put(res.getNode().id(), res.getData());
+        }
+
+        return new VisorValidateIndexesTaskResult(jobResults, exceptions);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<VisorValidateIndexesTaskArg, VisorValidateIndexesJobResult> job(VisorValidateIndexesTaskArg arg) {
+        return new VisorValidateIndexesJob(arg, debug);
+    }
+
+    /**
+     *
+     */
+    private static class VisorValidateIndexesJob extends VisorJob<VisorValidateIndexesTaskArg, VisorValidateIndexesJobResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param arg Argument.
+         * @param debug Debug.
+         */
+        protected VisorValidateIndexesJob(@Nullable VisorValidateIndexesTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorValidateIndexesJobResult run(@Nullable VisorValidateIndexesTaskArg arg) throws IgniteException {
+            try {
+                ValidateIndexesClosure clo = new ValidateIndexesClosure(arg.getCaches());
+
+                ignite.context().resource().injectGeneric(clo);
+
+                Map<PartitionKey, ValidateIndexesPartitionResult> res = clo.call();
+
+                return new VisorValidateIndexesJobResult(res);
+
+            }
+            catch (Exception e) {
+                throw new IgniteException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorValidateIndexesJob.class, this);
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
new file mode 100644
index 0000000000000..9e9c7770a644c
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -0,0 +1,121 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.util;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.typedef.F;
+
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+
+/**
+ *
+ */
+public class GridCommandHandlerIndexingTest extends GridCommandHandlerTest {
+    /**
+     *
+     */
+    public void testValidateIndexes() throws Exception {
+        Ignite ignite = startGrids(2);
+
+        ignite.cluster().active(true);
+
+        Ignite client = startGrid("client");
+
+        IgniteCache<Integer, Person> personCache = client.getOrCreateCache(new CacheConfiguration<Integer, Person>()
+            .setName("persons-cache-vi")
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setBackups(1)
+            .setQueryEntities(F.asList(personEntity(true, true)))
+            .setAffinity(new RendezvousAffinityFunction(false, 32)));
+
+        ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+        for (int i = 0; i < 1000; i++)
+            personCache.put(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong())));
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", "persons-cache-vi"));
+
+        assertTrue(testOut.toString().contains("validate_indexes has finished, no issues found"));
+    }
+
+    /**
+     * @param idxName Index name.
+     * @param idxOrgId Index org id.
+     */
+    private QueryEntity personEntity(boolean idxName, boolean idxOrgId) {
+        QueryEntity entity = new QueryEntity();
+
+        entity.setKeyType(Integer.class.getName());
+        entity.setValueType(Person.class.getName());
+
+        entity.addQueryField("orgId", Integer.class.getName(), null);
+        entity.addQueryField("name", String.class.getName(), null);
+
+        List<QueryIndex> idxs = new ArrayList<>();
+
+        if (idxName) {
+            QueryIndex idx = new QueryIndex("name");
+
+            idxs.add(idx);
+        }
+
+        if (idxOrgId) {
+            QueryIndex idx = new QueryIndex("orgId");
+
+            idxs.add(idx);
+        }
+
+        entity.setIndexes(idxs);
+
+        return entity;
+    }
+
+    /**
+     *
+     */
+    private static class Person implements Serializable {
+        /** */
+        int orgId;
+
+        /** */
+        String name;
+
+        /**
+         * @param orgId Organization ID.
+         * @param name Name.
+         */
+        public Person(int orgId, String name) {
+            this.orgId = orgId;
+            this.name = name;
+        }
+    }
+}

From 19fd136338a6571ae352df424fa7a06e8c537312 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 26 Apr 2018 19:38:03 +0300
Subject: [PATCH 124/543] IGNITE-8404 Fixed NPE in MappedFileMemoryProvider

---
 .../mem/file/MappedFileMemoryProvider.java     | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java
index 3800214b90f7f..7186b27f08118 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java
@@ -94,14 +94,18 @@ public MappedFileMemoryProvider(IgniteLogger log, File allocationPath) {
 
     /** {@inheritDoc} */
     @Override public void shutdown() {
-        for (MappedFile file : mappedFiles) {
-            try {
-                file.close();
-            }
-            catch (IOException e) {
-                log.error("Failed to close memory-mapped file upon stop (will ignore) [file=" +
-                    file.file() + ", err=" + e.getMessage() + ']');
+        if (mappedFiles != null) {
+            for (MappedFile file : mappedFiles) {
+                try {
+                    file.close();
+                }
+                catch (IOException e) {
+                    log.error("Failed to close memory-mapped file upon stop (will ignore) [file=" +
+                        file.file() + ", err=" + e.getMessage() + ']');
+                }
             }
+
+            mappedFiles = null;
         }
     }
 

From 1676560a844402348ee4882e8c9f9d798866697e Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Fri, 27 Apr 2018 13:08:03 +0300
Subject: [PATCH 125/543] IGNITE-8408 Fixed IgniteUtils.invoke

---
 .../java/org/apache/ignite/internal/util/IgniteUtils.java | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 42e96fb6067f0..2defefad4721e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -8143,10 +8143,10 @@ public static <T> T invoke(@Nullable Class<?> cls, @Nullable Object obj, String
         assert mtdName != null;
 
         try {
-            for (Class<?> c = cls != null ? cls : obj.getClass(); cls != Object.class; cls = cls.getSuperclass()) {
+            for (cls = cls != null ? cls : obj.getClass(); cls != Object.class; cls = cls.getSuperclass()) {
                 Method mtd = null;
 
-                for (Method declaredMtd : c.getDeclaredMethods()) {
+                for (Method declaredMtd : cls.getDeclaredMethods()) {
                     if (declaredMtd.getName().equals(mtdName)) {
                         if (mtd == null)
                             mtd = declaredMtd;
@@ -8202,11 +8202,11 @@ public static <T> T invoke(@Nullable Class<?> cls, @Nullable Object obj, String
         assert mtdName != null;
 
         try {
-            for (Class<?> c = cls != null ? cls : obj.getClass(); cls != Object.class; cls = cls.getSuperclass()) {
+            for (cls = cls != null ? cls : obj.getClass(); cls != Object.class; cls = cls.getSuperclass()) {
                 Method mtd;
 
                 try {
-                    mtd = c.getDeclaredMethod(mtdName, paramTypes);
+                    mtd = cls.getDeclaredMethod(mtdName, paramTypes);
                 }
                 catch (NoSuchMethodException ignored) {
                     continue;

From e10baa510c8855bbe6344c24b4f8c6db5fc086f4 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Wed, 25 Apr 2018 20:46:47 +0300
Subject: [PATCH 126/543] IGNITE-8066 AssertionError while trying to archive
 wal segment. - Fixes #3837.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit 1fb2369)
---
 .../GridCacheDatabaseSharedManager.java       | 11 +++--
 .../wal/IgniteWalHistoryReservationsTest.java | 49 +++++++++++++++++--
 2 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index d5da19998d09f..42c7090a857f8 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -747,6 +747,9 @@ private void unRegistrateMetricsMBean() {
 
             WALPointer restore = restoreMemory(status);
 
+            if (restore == null && status.endPtr != CheckpointStatus.NULL_PTR)
+                throw new StorageException("Restore wal pointer = " + restore + ", while status.endPtr = " + status.endPtr + ".");
+
             // First, bring memory to the last consistent checkpoint state if needed.
             // This method should return a pointer to the last valid record in the WAL.
 
@@ -767,7 +770,7 @@ private void unRegistrateMetricsMBean() {
         catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
-            throw new IgniteCheckedException(e);
+            throw e;
         }
         finally {
             checkpointReadUnlock();
@@ -1915,7 +1918,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
      * @throws IgniteCheckedException If failed.
      * @throws StorageException In case I/O error occurred during operations with storage.
      */
-    private WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException {
+    private @Nullable WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException {
         return restoreMemory(status, false, (PageMemoryEx)metaStorage.pageMemory());
     }
 
@@ -1926,7 +1929,7 @@ private WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedEx
      * @throws IgniteCheckedException If failed.
      * @throws StorageException In case I/O error occurred during operations with storage.
      */
-    private WALPointer restoreMemory(CheckpointStatus status, boolean storeOnly,
+    private @Nullable WALPointer restoreMemory(CheckpointStatus status, boolean storeOnly,
         PageMemoryEx storePageMem) throws IgniteCheckedException {
         assert !storeOnly || storePageMem != null;
 
@@ -2079,7 +2082,7 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
 
         if (status.needRestoreMemory()) {
             if (apply)
-                throw new IgniteCheckedException("Failed to restore memory state (checkpoint marker is present " +
+                throw new StorageException("Failed to restore memory state (checkpoint marker is present " +
                     "on disk, but checkpoint record is missed in WAL) " +
                     "[cpStatus=" + status + ", lastRead=" + lastRead + "]");
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
index 5ae16d2ce722a..e2c8bd1fce568 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
@@ -18,10 +18,10 @@
 package org.apache.ignite.internal.processors.cache.persistence.db.wal;
 
 import java.util.Map;
+import java.util.concurrent.Callable;
 import java.util.concurrent.locks.Lock;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -31,12 +31,11 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
-import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -59,12 +58,15 @@ public class IgniteWalHistoryReservationsTest extends GridCommonAbstractTest {
 
         cfg.setConsistentId("NODE$" + gridName.charAt(gridName.length() - 1));
 
+        cfg.setFailureHandler(new StopNodeFailureHandler());
+
         DataStorageConfiguration memCfg = new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(
                 new DataRegionConfiguration()
                     .setMaxSize(200 * 1024 * 1024)
                     .setPersistenceEnabled(true))
-            .setWalMode(WALMode.LOG_ONLY);
+                .setWalMode(WALMode.LOG_ONLY)
+                .setWalSegmentSize(512 * 1024);
 
         cfg.setDataStorageConfiguration(memCfg);
 
@@ -352,6 +354,45 @@ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWalHistoryPartiallyRemoved() throws Exception {
+        int entryCnt = 10_000;
+
+        IgniteEx ig0 = (IgniteEx) startGrids(2);
+
+        ig0.cluster().active(true);
+
+        IgniteCache<Integer, Integer> cache = ig0.cache("cache1");
+
+        for (int k = 0; k < entryCnt; k++)
+            cache.put(k, k);
+
+        GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                forceCheckpoint();
+
+                return null;
+            }
+        });
+
+        String nodeId0 = U.maskForFileName(ig0.localNode().consistentId().toString());
+
+        String walArchPath = ig0.configuration().getDataStorageConfiguration().getWalArchivePath();
+
+        stopAllGrids();
+
+        U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), walArchPath + "/" +
+                nodeId0, false));
+
+        startGrid(0);
+
+        Ignite ig1 = startGrid(1);
+
+        ig1.cluster().active(true);
+    }
+
     /**
      * @throws Exception If failed.
      */

From 48601cbd224e89e8396e53b4406f7a65e6fc6d9d Mon Sep 17 00:00:00 2001
From: Ivan Rakov <ivan.glukos@gmail.com>
Date: Fri, 27 Apr 2018 15:06:08 +0300
Subject: [PATCH 127/543] IGNITE-8393 Unexpected error during WAL compression
 fixed

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../GridCacheDatabaseSharedManager.java       |  6 +-
 .../wal/FileWriteAheadLogManager.java         | 53 +++++++---
 .../FsyncModeFileWriteAheadLogManager.java    | 59 +++++++----
 .../persistence/db/wal/WalCompactionTest.java | 99 ++++++++++++++++++-
 4 files changed, 180 insertions(+), 37 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 42c7090a857f8..d847132611e5f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -747,8 +747,10 @@ private void unRegistrateMetricsMBean() {
 
             WALPointer restore = restoreMemory(status);
 
-            if (restore == null && status.endPtr != CheckpointStatus.NULL_PTR)
-                throw new StorageException("Restore wal pointer = " + restore + ", while status.endPtr = " + status.endPtr + ".");
+            if (restore == null && status.endPtr != CheckpointStatus.NULL_PTR) {
+                throw new StorageException("Restore wal pointer = " + restore + ", while status.endPtr = " +
+                    status.endPtr + ". Can't restore memory - critical part of WAL archive is missing.");
+            }
 
             // First, bring memory to the last consistent checkpoint state if needed.
             // This method should return a pointer to the last valid record in the WAL.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index a28b73b9ab2a0..779534489a3bc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -39,8 +39,10 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeSet;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.PriorityBlockingQueue;
@@ -1869,17 +1871,28 @@ private long tryReserveNextSegmentOrWait() throws InterruptedException, IgniteCh
         }
 
         /**
-         *
+         * Deletes raw WAL segments if they aren't locked and already have compressed copies of themselves.
          */
         private void deleteObsoleteRawSegments() {
-            FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER));
+            FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
+
+            Set<Long> indices = new HashSet<>();
+            Set<Long> duplicateIndices = new HashSet<>();
+
+            for (FileDescriptor desc : descs) {
+                if (!indices.add(desc.idx))
+                    duplicateIndices.add(desc.idx);
+            }
 
             for (FileDescriptor desc : descs) {
+                if (desc.isCompressed())
+                    continue;
+
                 // Do not delete reserved or locked segment and any segment after it.
                 if (segmentReservedOrLocked(desc.idx))
                     return;
 
-                if (desc.idx < lastCompressedIdx) {
+                if (desc.idx < lastCompressedIdx && duplicateIndices.contains(desc.idx)) {
                     if (!desc.file.delete())
                         U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
                             desc.file.getAbsolutePath() + ", exists: " + desc.file.exists());
@@ -1892,22 +1905,24 @@ private void deleteObsoleteRawSegments() {
             init();
 
             while (!Thread.currentThread().isInterrupted() && !stopped) {
+                long currReservedSegment = -1;
+
                 try {
                     deleteObsoleteRawSegments();
 
-                    long nextSegment = tryReserveNextSegmentOrWait();
-                    if (nextSegment == -1)
+                    currReservedSegment = tryReserveNextSegmentOrWait();
+                    if (currReservedSegment == -1)
                         continue;
 
-                    File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(nextSegment) + ".zip" + ".tmp");
+                    File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + ".zip" + ".tmp");
 
-                    File zip = new File(walArchiveDir, FileDescriptor.fileName(nextSegment) + ".zip");
+                    File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + ".zip");
 
-                    File raw = new File(walArchiveDir, FileDescriptor.fileName(nextSegment));
+                    File raw = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment));
                     if (!Files.exists(raw.toPath()))
                         throw new IgniteCheckedException("WAL archive segment is missing: " + raw);
 
-                    compressSegmentToFile(nextSegment, raw, tmpZip);
+                    compressSegmentToFile(currReservedSegment, raw, tmpZip);
 
                     Files.move(tmpZip.toPath(), zip.toPath());
 
@@ -1917,14 +1932,27 @@ private void deleteObsoleteRawSegments() {
                         }
                     }
 
-                    lastCompressedIdx = nextSegment;
+                    lastCompressedIdx = currReservedSegment;
                 }
                 catch (IgniteCheckedException | IOException e) {
-                    U.error(log, "Unexpected error during WAL compression", e);
+                    U.error(log, "Compression of WAL segment [idx=" + currReservedSegment +
+                        "] was skipped due to unexpected error", e);
+
+                    lastCompressedIdx++;
                 }
                 catch (InterruptedException ignore) {
                     Thread.currentThread().interrupt();
                 }
+                finally {
+                    try {
+                        if (currReservedSegment != -1)
+                            release(new FileWALPointer(currReservedSegment, 0, 0));
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Can't release raw WAL segment [idx=" + currReservedSegment +
+                            "] after compression", e);
+                    }
+                }
             }
         }
 
@@ -1977,9 +2005,6 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
 
                 zos.write(heapBuf.array());
             }
-            finally {
-                release(new FileWALPointer(nextSegment, 0, 0));
-            }
         }
 
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index e354b439ef2b0..dfb1c41e7378a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -34,9 +34,11 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.PriorityBlockingQueue;
@@ -1703,19 +1705,30 @@ private long tryReserveNextSegmentOrWait() throws InterruptedException, IgniteCh
         }
 
         /**
-         *
+         * Deletes raw WAL segments if they aren't locked and already have compressed copies of themselves.
          */
         private void deleteObsoleteRawSegments() {
-            FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER));
+            FsyncModeFileWriteAheadLogManager.FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
+
+            Set<Long> indices = new HashSet<>();
+            Set<Long> duplicateIndices = new HashSet<>();
+
+            for (FsyncModeFileWriteAheadLogManager.FileDescriptor desc : descs) {
+                if (!indices.add(desc.idx))
+                    duplicateIndices.add(desc.idx);
+            }
 
             FileArchiver archiver0 = archiver;
 
-            for (FileDescriptor desc : descs) {
+            for (FsyncModeFileWriteAheadLogManager.FileDescriptor desc : descs) {
+                if (desc.isCompressed())
+                    continue;
+
                 // Do not delete reserved or locked segment and any segment after it.
                 if (archiver0 != null && archiver0.reserved(desc.idx))
                     return;
 
-                if (desc.idx < lastCompressedIdx) {
+                if (desc.idx < lastCompressedIdx && duplicateIndices.contains(desc.idx)) {
                     if (!desc.file.delete())
                         U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
                             desc.file.getAbsolutePath() + ", exists: " + desc.file.exists());
@@ -1728,39 +1741,54 @@ private void deleteObsoleteRawSegments() {
             init();
 
             while (!Thread.currentThread().isInterrupted() && !stopped) {
+                long currReservedSegment = -1;
+
                 try {
                     deleteObsoleteRawSegments();
 
-                    long nextSegment = tryReserveNextSegmentOrWait();
-                    if (nextSegment == -1)
+                    currReservedSegment = tryReserveNextSegmentOrWait();
+                    if (currReservedSegment == -1)
                         continue;
 
-                    File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(nextSegment) + ".zip" + ".tmp");
+                    File tmpZip = new File(walArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(currReservedSegment) + ".zip" + ".tmp");
 
-                    File zip = new File(walArchiveDir, FileDescriptor.fileName(nextSegment) + ".zip");
+                    File zip = new File(walArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(currReservedSegment) + ".zip");
 
-                    File raw = new File(walArchiveDir, FileDescriptor.fileName(nextSegment));
+                    File raw = new File(walArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(currReservedSegment));
                     if (!Files.exists(raw.toPath()))
                         throw new IgniteCheckedException("WAL archive segment is missing: " + raw);
 
-                    compressSegmentToFile(nextSegment, raw, tmpZip);
+                    compressSegmentToFile(currReservedSegment, raw, tmpZip);
 
                     Files.move(tmpZip.toPath(), zip.toPath());
 
-                    if (mode == WALMode.FSYNC) {
+                    if (mode != WALMode.NONE) {
                         try (FileIO f0 = ioFactory.create(zip, CREATE, READ, WRITE)) {
                             f0.force();
                         }
                     }
 
-                    lastCompressedIdx = nextSegment;
+                    lastCompressedIdx = currReservedSegment;
                 }
                 catch (IgniteCheckedException | IOException e) {
-                    U.error(log, "Unexpected error during WAL compression", e);
+                    U.error(log, "Compression of WAL segment [idx=" + currReservedSegment +
+                        "] was skipped due to unexpected error", e);
+
+                    lastCompressedIdx++;
                 }
-                catch (InterruptedException e) {
+                catch (InterruptedException ignore) {
                     Thread.currentThread().interrupt();
                 }
+                finally {
+                    try {
+                        if (currReservedSegment != -1)
+                            release(new FileWALPointer(currReservedSegment, 0, 0));
+                    }
+                    catch (IgniteCheckedException e) {
+                        U.error(log, "Can't release raw WAL segment [idx=" + currReservedSegment +
+                            "] after compression", e);
+                    }
+                }
             }
         }
 
@@ -1804,9 +1832,6 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
                         iter.nextX();
                 }
             }
-            finally {
-                release(new FileWALPointer(nextSegment, 0, 0));
-            }
         }
 
         /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
index 6b79d906a4937..fe0169f70d908 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
@@ -18,6 +18,7 @@
 
 import java.io.File;
 import java.io.FilenameFilter;
+import java.io.RandomAccessFile;
 import java.util.Arrays;
 import java.util.Comparator;
 import org.apache.ignite.IgniteCache;
@@ -57,6 +58,12 @@ public class WalCompactionTest extends GridCommonAbstractTest {
     /** Entries count. */
     public static final int ENTRIES = 1000;
 
+    /** Compaction enabled flag. */
+    private boolean compactionEnabled;
+
+    /** Wal mode. */
+    private WALMode walMode;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(name);
@@ -67,10 +74,10 @@ public class WalCompactionTest extends GridCommonAbstractTest {
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
                 .setPersistenceEnabled(true)
                 .setMaxSize(200 * 1024 * 1024))
-            .setWalMode(WALMode.LOG_ONLY)
+            .setWalMode(walMode)
             .setWalSegmentSize(WAL_SEGMENT_SIZE)
             .setWalHistorySize(500)
-            .setWalCompactionEnabled(true));
+            .setWalCompactionEnabled(compactionEnabled));
 
         CacheConfiguration ccfg = new CacheConfiguration();
 
@@ -91,6 +98,10 @@ public class WalCompactionTest extends GridCommonAbstractTest {
         stopAllGrids();
 
         cleanPersistenceDir();
+
+        compactionEnabled = true;
+
+        walMode = WALMode.LOG_ONLY;
     }
 
     /** {@inheritDoc} */
@@ -125,7 +136,7 @@ public void testApplyingUpdatesFromCompactedWal() throws Exception {
         ig.context().cache().context().database().wakeupForCheckpoint("Forced checkpoint").get();
         ig.context().cache().context().database().wakeupForCheckpoint("Forced checkpoint").get();
 
-        Thread.sleep(15_000); // Allow compressor to archive WAL segments.
+        Thread.sleep(15_000); // Allow compressor to compress WAL segments.
 
         String nodeFolderName = ig.context().pdsFolderResolver().resolveFolders().folderName();
 
@@ -187,6 +198,86 @@ else if (arr[i] != 1) {
         assertFalse(fail);
     }
 
+    /**
+     *
+     */
+    public void testCompressorToleratesEmptyWalSegmentsFsync() throws Exception {
+        testCompressorToleratesEmptyWalSegments(WALMode.FSYNC);
+    }
+
+    /**
+     *
+     */
+    public void testCompressorToleratesEmptyWalSegmentsLogOnly() throws Exception {
+        testCompressorToleratesEmptyWalSegments(WALMode.LOG_ONLY);
+    }
+
+    /**
+     * Tests that WAL compaction won't be stopped by single broken WAL segment.
+     */
+    private void testCompressorToleratesEmptyWalSegments(WALMode walMode) throws Exception {
+        this.walMode = walMode;
+        compactionEnabled = false;
+
+        IgniteEx ig = startGrid(0);
+        ig.cluster().active(true);
+
+        IgniteCache<Integer, byte[]> cache = ig.cache("cache");
+
+        for (int i = 0; i < 2500; i++) { // At least 50MB of raw data in total.
+            final byte[] val = new byte[20000];
+
+            val[i] = 1;
+
+            cache.put(i, val);
+        }
+
+        // WAL archive segment is allowed to be compressed when it's at least one checkpoint away from current WAL head.
+        ig.context().cache().context().database().wakeupForCheckpoint("Forced checkpoint").get();
+        ig.context().cache().context().database().wakeupForCheckpoint("Forced checkpoint").get();
+
+        String nodeFolderName = ig.context().pdsFolderResolver().resolveFolders().folderName();
+
+        stopAllGrids();
+
+        int emptyIdx = 5;
+
+        File dbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false);
+        File walDir = new File(dbDir, "wal");
+        File archiveDir = new File(walDir, "archive");
+        File nodeArchiveDir = new File(archiveDir, nodeFolderName);
+        File walSegment = new File(nodeArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(emptyIdx));
+
+        try (RandomAccessFile raf = new RandomAccessFile(walSegment, "rw")) {
+            raf.setLength(0); // Clear wal segment, but don't delete.
+        }
+
+        compactionEnabled = true;
+
+        ig = startGrid(0);
+        ig.cluster().active(true);
+
+        Thread.sleep(15_000); // Allow compressor to compress WAL segments.
+
+        File[] compressedSegments = nodeArchiveDir.listFiles(new FilenameFilter() {
+            @Override public boolean accept(File dir, String name) {
+                return name.endsWith(".wal.zip");
+            }
+        });
+
+        long maxIdx = -1;
+        for (File f : compressedSegments) {
+            String idxPart = f.getName().substring(0, f.getName().length() - ".wal.zip".length());
+
+            maxIdx = Math.max(maxIdx, Long.parseLong(idxPart));
+        }
+
+        System.out.println("Max compressed index: " + maxIdx);
+        assertTrue(maxIdx > emptyIdx);
+
+        assertTrue(walSegment.exists()); // Failed to compress WAL segment shoudn't be deleted.
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -240,7 +331,7 @@ public void testSeekingStartInCompactedSegment() throws Exception {
         ig.context().cache().context().database().wakeupForCheckpoint("Forced checkpoint").get();
         ig.context().cache().context().database().wakeupForCheckpoint("Forced checkpoint").get();
 
-        Thread.sleep(15_000); // Allow compressor to archive WAL segments.
+        Thread.sleep(15_000); // Allow compressor to compress WAL segments.
 
         File walDir = new File(dbDir, "wal");
         File archiveDir = new File(walDir, "archive");

From 23e802e5b8146677a8e05767d3db93991fb0831a Mon Sep 17 00:00:00 2001
From: Ivan Rakov <ivan.glukos@gmail.com>
Date: Fri, 27 Apr 2018 22:15:15 +0300
Subject: [PATCH 128/543] IGNITE-8416 CommandHandlerParsingTest stably fails
 with parsing error

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../internal/commandline/CommandHandlerParsingTest.java   | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index fda7e6054ffbc..98b8e01c99ea3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -40,6 +40,9 @@ public void testParseAndValidateUserAndPassword() {
         CommandHandler hnd = new CommandHandler();
 
         for (Command cmd : Command.values()) {
+            if (cmd == Command.CACHE)
+                continue; // --cache subcommand requires its own specific arguments.
+
             try {
                 hnd.parseAndValidate(asList("--user"));
 
@@ -91,6 +94,9 @@ public void testConnectionSettings() {
         CommandHandler hnd = new CommandHandler();
 
         for (Command cmd : Command.values()) {
+            if (cmd == Command.CACHE)
+                continue; // --cache subcommand requires its own specific arguments.
+
             Arguments args = hnd.parseAndValidate(asList(cmd.text()));
 
             assertEquals(cmd, args.command());
@@ -98,7 +104,7 @@ public void testConnectionSettings() {
             assertEquals(DFLT_PORT, args.port());
 
             args = hnd.parseAndValidate(asList("--port", "12345", "--host", "test-host", "--ping-interval", "5000",
-                    "--ping-timeout", "40000", cmd.text()));
+                "--ping-timeout", "40000", cmd.text()));
 
             assertEquals(cmd, args.command());
             assertEquals("test-host", args.host());

From 7412288aa5f00533b2b1ece07b245a0029242add Mon Sep 17 00:00:00 2001
From: Ivan Rakov <ivan.glukos@gmail.com>
Date: Tue, 8 May 2018 15:03:00 +0300
Subject: [PATCH 129/543] IGNITE-8429 Unexpected error during incorrect WAL
 segment decompression, causes node termination

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit f3a61e4)

(cherry picked from commit 104b771)
---
 .../preloader/GridDhtPartitionSupplier.java   |  4 +--
 .../wal/FileWriteAheadLogManager.java         | 32 +++++++++++--------
 .../FsyncModeFileWriteAheadLogManager.java    | 32 +++++++++++--------
 3 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 84e6828df4dc3..4946d7e3f0eed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -32,10 +32,10 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
 import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.T3;
 import org.apache.ignite.internal.util.typedef.internal.S;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 779534489a3bc..6ac102fff5e75 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -33,6 +33,7 @@
 import java.nio.ByteOrder;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.ClosedByInterruptException;
+import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.sql.Time;
 import java.util.ArrayList;
@@ -2065,11 +2066,11 @@ private class FileDecompressor extends Thread {
 
         /** {@inheritDoc} */
         @Override public void run() {
-            Throwable err = null;
-
             while (!Thread.currentThread().isInterrupted() && !stopped) {
+                long segmentToDecompress = -1L;
+
                 try {
-                    long segmentToDecompress = segmentsQueue.take();
+                    segmentToDecompress = segmentsQueue.take();
 
                     if (stopped)
                         break;
@@ -2087,7 +2088,16 @@ private class FileDecompressor extends Thread {
                             io.write(arr, 0, bytesRead);
                     }
 
-                    Files.move(unzipTmp.toPath(), unzip.toPath());
+                    try {
+                        Files.move(unzipTmp.toPath(), unzip.toPath());
+                    }
+                    catch (FileAlreadyExistsException e) {
+                        U.error(log, "Can't rename temporary unzipped segment: raw segment is already present " +
+                            "[tmp=" + unzipTmp + ", raw=" + unzip + "]", e);
+
+                        if (!unzipTmp.delete())
+                            U.error(log, "Can't delete temporary unzipped segment [tmp=" + unzipTmp + "]");
+                    }
 
                     synchronized (this) {
                         decompressionFutures.remove(segmentToDecompress).onDone();
@@ -2097,16 +2107,12 @@ private class FileDecompressor extends Thread {
                     Thread.currentThread().interrupt();
                 }
                 catch (Throwable t) {
-                    err = t;
-                }
-                finally {
-                    if (err == null && !stopped)
-                        err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+                    if (!stopped && segmentToDecompress != -1L) {
+                        IgniteCheckedException e = new IgniteCheckedException("Error during WAL segment " +
+                            "decompression [segmentIdx=" + segmentToDecompress + "]", t);
 
-                    if (err instanceof OutOfMemoryError)
-                        cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
-                    else if (err != null)
-                        cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+                        decompressionFutures.remove(segmentToDecompress).onDone(e);
+                    }
                 }
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index dfb1c41e7378a..cf643fde42378 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -28,6 +28,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
 import java.sql.Time;
 import java.util.ArrayList;
@@ -1873,11 +1874,11 @@ private class FileDecompressor extends Thread {
 
         /** {@inheritDoc} */
         @Override public void run() {
-            Throwable err = null;
-
             while (!Thread.currentThread().isInterrupted() && !stopped) {
+                long segmentToDecompress = -1L;
+
                 try {
-                    long segmentToDecompress = segmentsQueue.take();
+                    segmentToDecompress = segmentsQueue.take();
 
                     if (stopped)
                         break;
@@ -1895,7 +1896,16 @@ private class FileDecompressor extends Thread {
                             io.write(arr, 0, bytesRead);
                     }
 
-                    Files.move(unzipTmp.toPath(), unzip.toPath());
+                    try {
+                        Files.move(unzipTmp.toPath(), unzip.toPath());
+                    }
+                    catch (FileAlreadyExistsException e) {
+                        U.error(log, "Can't rename temporary unzipped segment: raw segment is already present " +
+                            "[tmp=" + unzipTmp + ", raw=" + unzip + ']', e);
+
+                        if (!unzipTmp.delete())
+                            U.error(log, "Can't delete temporary unzipped segment [tmp=" + unzipTmp + ']');
+                    }
 
                     synchronized (this) {
                         decompressionFutures.remove(segmentToDecompress).onDone();
@@ -1905,16 +1915,12 @@ private class FileDecompressor extends Thread {
                     Thread.currentThread().interrupt();
                 }
                 catch (Throwable t) {
-                    err = t;
-                }
-                finally {
-                    if (err == null && !stopped)
-                        err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+                    if (!stopped && segmentToDecompress != -1L) {
+                        IgniteCheckedException e = new IgniteCheckedException("Error during WAL segment " +
+                            "decompression [segmentIdx=" + segmentToDecompress + ']', t);
 
-                    if (err instanceof OutOfMemoryError)
-                        cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
-                    else if (err != null)
-                        cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+                        decompressionFutures.remove(segmentToDecompress).onDone(e);
+                    }
                 }
             }
         }

From 6c44ebc51b31d04f392b7248ea479bfd12af06cd Mon Sep 17 00:00:00 2001
From: Ivan Rakov <ivan.glukos@gmail.com>
Date: Tue, 8 May 2018 15:49:59 +0300
Subject: [PATCH 130/543] IGNITE-8429 Unexpected error during incorrect WAL
 segment decompression, causes node termination + synchronized block

(cherry picked from commit 99a07ed)
---
 .../cache/persistence/wal/FileWriteAheadLogManager.java       | 4 +++-
 .../persistence/wal/FsyncModeFileWriteAheadLogManager.java    | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 6ac102fff5e75..8755e1bf31f4b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -2111,7 +2111,9 @@ private class FileDecompressor extends Thread {
                         IgniteCheckedException e = new IgniteCheckedException("Error during WAL segment " +
                             "decompression [segmentIdx=" + segmentToDecompress + "]", t);
 
-                        decompressionFutures.remove(segmentToDecompress).onDone(e);
+                        synchronized (this) {
+                            decompressionFutures.remove(segmentToDecompress).onDone(e);
+                        }
                     }
                 }
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index cf643fde42378..c446f7f228649 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -1919,7 +1919,9 @@ private class FileDecompressor extends Thread {
                         IgniteCheckedException e = new IgniteCheckedException("Error during WAL segment " +
                             "decompression [segmentIdx=" + segmentToDecompress + ']', t);
 
-                        decompressionFutures.remove(segmentToDecompress).onDone(e);
+                        synchronized (this) {
+                            decompressionFutures.remove(segmentToDecompress).onDone(e);
+                        }
                     }
                 }
             }

From 561e7fafa34b638b95f6007c259436a6183f0c47 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Tue, 8 May 2018 16:35:05 +0300
Subject: [PATCH 131/543] IGNITE-8435 StorageException is handled like
 NodeStoppingException during failing transaction commit

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit e8b846b)
---
 .../internal/InvalidEnvironmentException.java |  25 ++
 .../internal/NodeStoppingException.java       |   2 +-
 .../pagemem/wal/StorageException.java         |   3 +-
 .../GridDistributedTxRemoteAdapter.java       |   8 +-
 .../dht/GridDhtTxFinishFuture.java            |   8 +-
 .../transactions/IgniteTxLocalAdapter.java    |   8 +-
 .../ignite/internal/util/typedef/X.java       |   4 +-
 .../AccountTransferTransactionTest.java       | 331 ++++++++++++++++++
 .../testsuites/IgniteBasicTestSuite.java      |   2 +
 9 files changed, 375 insertions(+), 16 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/InvalidEnvironmentException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/failure/AccountTransferTransactionTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/InvalidEnvironmentException.java b/modules/core/src/main/java/org/apache/ignite/internal/InvalidEnvironmentException.java
new file mode 100644
index 0000000000000..d45a443e8afd8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/InvalidEnvironmentException.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+/**
+ * Marker interface of invalid environment exception.
+ */
+public interface InvalidEnvironmentException {
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/NodeStoppingException.java b/modules/core/src/main/java/org/apache/ignite/internal/NodeStoppingException.java
index 75447a14c53fa..cc39b14cd92d6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/NodeStoppingException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/NodeStoppingException.java
@@ -22,7 +22,7 @@
 /**
  *
  */
-public class NodeStoppingException extends IgniteCheckedException {
+public class NodeStoppingException extends IgniteCheckedException implements InvalidEnvironmentException {
     /** */
     private static final long serialVersionUID = 0L;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java
index 2da08b9442965..debc391271a97 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java
@@ -19,12 +19,13 @@
 
 import java.io.IOException;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.InvalidEnvironmentException;
 import org.jetbrains.annotations.NotNull;
 
 /**
  * Exception is needed to distinguish WAL manager & page store critical I/O errors.
  */
-public class StorageException extends IgniteCheckedException {
+public class StorageException extends IgniteCheckedException implements InvalidEnvironmentException {
     /** */
     private static final long serialVersionUID = 0L;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index a692b2e8dc15d..5e3111c1250cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -28,8 +28,8 @@
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.InvalidEnvironmentException;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.pagemem.wal.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
@@ -734,21 +734,21 @@ else if (op == READ) {
                                     }
                                 }
                                 catch (Throwable ex) {
-                                    boolean nodeStopping = X.hasCause(ex, NodeStoppingException.class);
+                                    boolean hasIOIssue = X.hasCause(ex, InvalidEnvironmentException.class);
 
                                     // In case of error, we still make the best effort to commit,
                                     // as there is no way to rollback at this point.
                                     err = new IgniteTxHeuristicCheckedException("Commit produced a runtime exception " +
                                         "(all transaction entries will be invalidated): " + CU.txString(this), ex);
 
-                                    if (nodeStopping) {
+                                    if (hasIOIssue) {
                                         U.warn(log, "Failed to commit transaction, node is stopping [tx=" + this +
                                             ", err=" + ex + ']');
                                     }
                                     else
                                         U.error(log, "Commit failed.", err);
 
-                                    uncommit(nodeStopping);
+                                    uncommit(hasIOIssue);
 
                                     state(UNKNOWN);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index 63807107fcb84..0ed8419b32297 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -26,10 +26,10 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.InvalidEnvironmentException;
 import org.apache.ignite.internal.IgniteDiagnosticAware;
 import org.apache.ignite.internal.IgniteDiagnosticPrepareContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheFuture;
@@ -168,7 +168,7 @@ public void rollbackOnError(Throwable e) {
         if (ERR_UPD.compareAndSet(this, null, e)) {
             tx.setRollbackOnly();
 
-            if (X.hasCause(e, NodeStoppingException.class))
+            if (X.hasCause(e, InvalidEnvironmentException.class))
                 onComplete();
             else
                 finish(false);
@@ -225,9 +225,9 @@ public void onResult(UUID nodeId, GridDhtTxFinishResponse res) {
 
             if (this.tx.onePhaseCommit() && (this.tx.state() == COMMITTING)) {
                 try {
-                    boolean nodeStop = err != null && X.hasCause(err, NodeStoppingException.class);
+                    boolean hasInvalidEnvironmentIssue = X.hasCause(err, InvalidEnvironmentException.class);
 
-                    this.tx.tmFinish(err == null, nodeStop, false);
+                    this.tx.tmFinish(err == null, hasInvalidEnvironmentIssue, false);
                 }
                 catch (IgniteCheckedException finishErr) {
                     U.error(log, "Failed to finish tx: " + tx, e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 61650ccefa393..6f11a57500ec2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -30,8 +30,8 @@
 import javax.cache.processor.EntryProcessor;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.internal.InvalidEnvironmentException;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.pagemem.wal.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
@@ -846,13 +846,13 @@ assert ownsLock(txEntry.cached()):
                             throw ex;
                         }
                         else {
-                            boolean nodeStopping = X.hasCause(ex, NodeStoppingException.class);
+                            boolean hasInvalidEnvironmentIssue = X.hasCause(ex, InvalidEnvironmentException.class);
 
                             IgniteCheckedException err = new IgniteTxHeuristicCheckedException("Failed to locally write to cache " +
                                 "(all transaction entries will be invalidated, however there was a window when " +
                                 "entries for this transaction were visible to others): " + this, ex);
 
-                            if (nodeStopping) {
+                            if (hasInvalidEnvironmentIssue) {
                                 U.warn(log, "Failed to commit transaction, node is stopping " +
                                     "[tx=" + this + ", err=" + ex + ']');
                             }
@@ -865,7 +865,7 @@ assert ownsLock(txEntry.cached()):
 
                             try {
                                 // Courtesy to minimize damage.
-                                uncommit(nodeStopping);
+                                uncommit(hasInvalidEnvironmentIssue);
                             }
                             catch (Throwable ex1) {
                                 U.error(log, "Failed to uncommit transaction: " + this, ex1);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java b/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
index 1a43daa09c918..49732b6817630 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
@@ -433,14 +433,14 @@ private static void cloneField(Map<Integer, Integer> identityIdxs, List<Object>
      *      {@code false} otherwise.
      */
     @SafeVarargs
-    public static boolean hasCause(@Nullable Throwable t, @Nullable Class<? extends Throwable>... cls) {
+    public static boolean hasCause(@Nullable Throwable t, @Nullable Class<?>... cls) {
         if (t == null || F.isEmpty(cls))
             return false;
 
         assert cls != null;
 
         for (Throwable th = t; th != null; th = th.getCause()) {
-            for (Class<? extends Throwable> c : cls) {
+            for (Class<?> c : cls) {
                 if (c.isAssignableFrom(th.getClass()))
                     return true;
             }
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/AccountTransferTransactionTest.java b/modules/core/src/test/java/org/apache/ignite/failure/AccountTransferTransactionTest.java
new file mode 100644
index 0000000000000..8d7cf15608ed2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/AccountTransferTransactionTest.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.ObjectName;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.worker.WorkersControlMXBeanImpl;
+import org.apache.ignite.mxbean.WorkersControlMXBean;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.jetbrains.annotations.NotNull;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Test transfer amount between accounts with enabled {@link StopNodeFailureHandler}.
+ */
+public class AccountTransferTransactionTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+    /** Count of accounts in one thread. */
+    private static final int ACCOUNTS_CNT = 20;
+    /** Count of threads and caches. */
+    private static final int THREADS_CNT = 20;
+    /** Count of nodes to start. */
+    private static final int NODES_CNT = 3;
+    /** Count of transaction on cache. */
+    private static final int TRANSACTION_CNT = 10;
+
+    /** {@inheritDoc} */
+    @Override protected FailureHandler getFailureHandler(String igniteInstanceName) {
+        return new StopNodeFailureHandler();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(name);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+        cfg.setLocalHost("127.0.0.1");
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(50 * 1024 * 1024)
+                .setPersistenceEnabled(true))
+        );
+
+        CacheConfiguration[] cacheConfigurations = new CacheConfiguration[THREADS_CNT];
+        for (int i = 0; i < THREADS_CNT; i++) {
+            cacheConfigurations[i] = new CacheConfiguration()
+                .setName(cacheName(i))
+                .setAffinity(new RendezvousAffinityFunction(false, 32))
+                .setBackups(1)
+                .setAtomicityMode(TRANSACTIONAL)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setWriteSynchronizationMode(FULL_SYNC)
+                .setEvictionPolicy(new FifoEvictionPolicy(1000))
+                .setOnheapCacheEnabled(true);
+        }
+
+        cfg.setCacheConfiguration(cacheConfigurations);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Test transfer amount.
+     */
+    public void testTransferAmount() throws Exception {
+        //given: started some nodes with client.
+        startGrids(NODES_CNT);
+
+        IgniteEx igniteClient = startGrid(getClientConfiguration(NODES_CNT));
+
+        igniteClient.cluster().active(true);
+
+        Random random = new Random();
+
+        long[] initAmount = new long[THREADS_CNT];
+
+        //and: fill all accounts on all caches and calculate total amount for every cache.
+        for (int cachePrefixIdx = 0; cachePrefixIdx < THREADS_CNT; cachePrefixIdx++) {
+            IgniteCache<Object, Object> cache = igniteClient.getOrCreateCache(cacheName(cachePrefixIdx));
+
+            try (Transaction tx = igniteClient.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                for (int accountId = 0; accountId < ACCOUNTS_CNT; accountId++) {
+                    Long amount = (long)random.nextInt(1000);
+
+                    cache.put(accountId, amount);
+
+                    initAmount[cachePrefixIdx] += amount;
+                }
+
+                tx.commit();
+            }
+        }
+
+        //when: start transfer amount from account to account in different threads.
+        CountDownLatch firstTransactionDone = new CountDownLatch(THREADS_CNT);
+
+        ArrayList<Thread> transferThreads = new ArrayList<>();
+
+        for (int i = 0; i < THREADS_CNT; i++) {
+            transferThreads.add(new TransferAmountTxThread(firstTransactionDone, igniteClient, cacheName(i)));
+
+            transferThreads.get(i).start();
+        }
+
+        firstTransactionDone.await(10, TimeUnit.SECONDS);
+
+        //and: terminate disco-event-worker thread on one node.
+        WorkersControlMXBean bean = workersMXBean(1);
+
+        bean.terminateWorker(
+            bean.getWorkerNames().stream()
+                .filter(name -> name.startsWith("disco-event-worker"))
+                .findFirst()
+                .orElse(null)
+        );
+
+        for (Thread thread : transferThreads) {
+            thread.join();
+        }
+
+        long[] resultAmount = new long[THREADS_CNT];
+
+        //then: calculate total amount for every thread.
+        for (int j = 0; j < THREADS_CNT; j++) {
+            String cacheName = cacheName(j);
+
+            IgniteCache<Object, Object> cache = igniteClient.getOrCreateCache(cacheName);
+
+            try (Transaction tx = igniteClient.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+
+                for (int i = 0; i < ACCOUNTS_CNT; i++)
+                    resultAmount[j] += getNotNullValue(cache, i);
+                tx.commit();
+            }
+
+            long diffAmount = initAmount[j] - resultAmount[j];
+
+            //and: check that result amount equal to init amount.
+            assertTrue(
+                String.format("Total amount before and after transfer is not same: diff=%s, cache=%s",
+                    diffAmount, cacheName),
+                diffAmount == 0
+            );
+        }
+    }
+
+    /**
+     * Make test cache name by prefix.
+     */
+    @NotNull private String cacheName(int cachePrefixIdx) {
+        return "cache" + cachePrefixIdx;
+    }
+
+    /**
+     * Ignite configuration for client.
+     */
+    @NotNull private IgniteConfiguration getClientConfiguration(int nodesPrefix) throws Exception {
+        IgniteConfiguration clientConf = getConfiguration(getTestIgniteInstanceName(nodesPrefix));
+
+        clientConf.setClientMode(true);
+
+        return clientConf;
+    }
+
+    /**
+     * Extract not null value from cache.
+     */
+    private long getNotNullValue(IgniteCache<Object, Object> cache, int i) {
+        Object value = cache.get(i);
+
+        return value == null ? 0 : ((Long)value);
+    }
+
+    /**
+     * Configure workers mx bean.
+     */
+    private WorkersControlMXBean workersMXBean(int igniteInt) throws Exception {
+        ObjectName mbeanName = U.makeMBeanName(
+            getTestIgniteInstanceName(igniteInt),
+            "Kernal",
+            WorkersControlMXBeanImpl.class.getSimpleName()
+        );
+
+        MBeanServer mbeanSrv = ManagementFactory.getPlatformMBeanServer();
+
+        if (!mbeanSrv.isRegistered(mbeanName))
+            fail("MBean is not registered: " + mbeanName.getCanonicalName());
+
+        return MBeanServerInvocationHandler.newProxyInstance(mbeanSrv, mbeanName, WorkersControlMXBean.class, true);
+    }
+
+    /**
+     *
+     */
+    private static class TransferAmountTxThread extends Thread {
+        /** */
+        private CountDownLatch firstTransactionLatch;
+        /** */
+        private Ignite ignite;
+        /** */
+        private String cacheName;
+        /** */
+        private Random random = new Random();
+
+        /**
+         * @param ignite Ignite.
+         */
+        private TransferAmountTxThread(CountDownLatch firstTransactionLatch, final Ignite ignite, String cacheName) {
+            this.firstTransactionLatch = firstTransactionLatch;
+            this.ignite = ignite;
+            this.cacheName = cacheName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            for (int i = 0; i < TRANSACTION_CNT; i++) {
+                try {
+                    updateInTransaction(ignite.cache(cacheName));
+                }
+                finally {
+                    if (i == 0)
+                        firstTransactionLatch.countDown();
+                }
+            }
+        }
+
+        /**
+         * @throws IgniteException if fails
+         */
+        @SuppressWarnings("unchecked")
+        private void updateInTransaction(IgniteCache cache) throws IgniteException {
+            int accIdFrom = random.nextInt(ACCOUNTS_CNT);
+            int accIdTo = random.nextInt(ACCOUNTS_CNT);
+
+            if (accIdFrom == accIdTo)
+                accIdTo = (int)getNextAccountId(accIdFrom);
+
+            Long acctFrom;
+            Long acctTo;
+
+            try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                acctFrom = (Long)cache.get(accIdFrom);
+                acctTo = (Long)cache.get(accIdTo);
+
+                long transactionAmount = (long)(random.nextDouble() * acctFrom);
+
+                cache.put(accIdFrom, acctFrom - transactionAmount);
+                cache.put(accIdTo, acctTo + transactionAmount);
+
+                tx.commit();
+            }
+        }
+
+        /**
+         * @param curr current
+         * @return random value
+         */
+        private long getNextAccountId(long curr) {
+            long randomVal;
+
+            do {
+                randomVal = random.nextInt(ACCOUNTS_CNT);
+            }
+            while (curr == randomVal);
+
+            return randomVal;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index e71a56981f09c..21d56b465da32 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -20,6 +20,7 @@
 import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.GridSuppressedExceptionSelfTest;
+import org.apache.ignite.failure.AccountTransferTransactionTest;
 import org.apache.ignite.failure.FailureHandlerTriggeredTest;
 import org.apache.ignite.failure.IoomFailureHandlerTest;
 import org.apache.ignite.failure.OomFailureHandlerTest;
@@ -205,6 +206,7 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
         suite.addTestSuite(IoomFailureHandlerTest.class);
         suite.addTestSuite(OomFailureHandlerTest.class);
+        suite.addTestSuite(AccountTransferTransactionTest.class);
 
         return suite;
     }

From b3ca15c36765652d311502f9e2d0ddb83ef4fc9d Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Tue, 8 May 2018 17:10:53 +0300
Subject: [PATCH 132/543] IGNITE-8430 implemented
 getCurrentCoordinatorFormatted method in IgniteMXBean - Fixes #3957.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit 666f220)
---
 .../apache/ignite/internal/IgniteKernal.java  | 22 +++++++++++++++++++
 .../apache/ignite/mxbean/IgniteMXBean.java    |  6 +++++
 .../ignite/util/mbeans/GridMBeanSelfTest.java | 16 +++++++++++---
 3 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index dc1307cd18dc5..4ceee7f726a58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -105,6 +105,7 @@
 import org.apache.ignite.internal.binary.BinaryUtils;
 import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
 import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.managers.GridManager;
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
@@ -297,6 +298,9 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
     /** Force complete reconnect future. */
     private static final Object STOP_RECONNECT = new Object();
 
+    /** Separator for formatted coordinator properties. */
+    public static final String COORDINATOR_PROPERTIES_SEPARATOR = ",";
+
     static {
         LongJVMPauseDetector.start();
     }
@@ -502,6 +506,24 @@ public IgniteKernal(@Nullable GridSpringResourceContext rsrcCtx) {
         return Arrays.toString(cfg.getCheckpointSpi());
     }
 
+    /** {@inheritDoc} */
+    @Override public String getCurrentCoordinatorFormatted() {
+        ClusterNode node = ctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE);
+
+        if (node == null)
+            return "";
+
+        return new StringBuilder()
+            .append(node.addresses())
+            .append(COORDINATOR_PROPERTIES_SEPARATOR)
+            .append(node.id())
+            .append(COORDINATOR_PROPERTIES_SEPARATOR)
+            .append(node.order())
+            .append(COORDINATOR_PROPERTIES_SEPARATOR)
+            .append(node.hostNames())
+            .toString();
+    }
+
     /** {@inheritDoc} */
     @Override public String getCommunicationSpiFormatted() {
         assert cfg != null;
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
index 308c148d11e8c..bb79334352a2d 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
@@ -432,6 +432,12 @@ public interface IgniteMXBean {
     @MXBeanDescription("Dumps debug information for the current node.")
     public void dumpDebugInfo();
 
+    /**
+     * Gets a formatted properties of current coordinator.
+     */
+    @MXBeanDescription("Formatted properties of current coordinator.")
+    public String getCurrentCoordinatorFormatted();
+
     /**
      * Runs IO latency test against all remote server nodes in cluster.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanSelfTest.java
index 4f5ed19cfacae..4e329f9fc9437 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanSelfTest.java
@@ -22,6 +22,7 @@
 import javax.management.MBeanOperationInfo;
 import javax.management.MBeanParameterInfo;
 import javax.management.StandardMBean;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.mxbean.IgniteStandardMXBean;
 import org.apache.ignite.mxbean.IgniteMXBean;
 import org.apache.ignite.mxbean.MXBeanDescription;
@@ -161,10 +162,19 @@ public void testEmptyNameMBeanInfo() throws Exception {
      */
     public void testIgniteKernalReturnsValidMBeanInfo() throws Exception {
         try {
-            IgniteMXBean ignite = (IgniteMXBean)startGrid();
+            IgniteEx igniteCrd = startGrid(0);
 
-            assertNotNull(ignite.getUserAttributesFormatted());
-            assertNotNull(ignite.getLifecycleBeansFormatted());
+            IgniteMXBean igniteMXBean = (IgniteMXBean)startGrid(1);
+
+            assertNotNull(igniteMXBean.getUserAttributesFormatted());
+            assertNotNull(igniteMXBean.getLifecycleBeansFormatted());
+
+            String coordinatorFormatted = igniteMXBean.getCurrentCoordinatorFormatted();
+
+            assertTrue(coordinatorFormatted.contains(igniteCrd.localNode().addresses().toString()));
+            assertTrue(coordinatorFormatted.contains(igniteCrd.localNode().hostNames().toString()));
+            assertTrue(coordinatorFormatted.contains(Long.toString(igniteCrd.localNode().order())));
+            assertTrue(coordinatorFormatted.contains(igniteCrd.localNode().id().toString()));
         }
         finally {
             stopAllGrids();

From 383c0de3c92003650c59b20246f82df89f098c79 Mon Sep 17 00:00:00 2001
From: ascherbakoff <alexey.scherbakoff@gmail.com>
Date: Thu, 10 May 2018 12:08:17 +0300
Subject: [PATCH 133/543] IGNITE-8437: Control utility fails to connect to
 cluster if zookeeper discovery used. This closes #3950.

(cherry picked from commit 3cdee53)
---
 modules/zookeeper/pom.xml                     | 49 +++++++++++++++++++
 .../resources/META-INF/classnames.properties  | 18 +++++++
 .../zk/ZookeeperDiscoverySpiTestSuite2.java   |  4 +-
 3 files changed, 70 insertions(+), 1 deletion(-)
 create mode 100644 modules/zookeeper/src/main/resources/META-INF/classnames.properties

diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index 2d47ece408719..0777f1e36edbd 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -172,6 +172,55 @@
                 </executions>
             </plugin>
 
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>exec-maven-plugin</artifactId>
+                <version>1.3.2</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>org.apache.ignite</groupId>
+                        <artifactId>ignite-tools</artifactId>
+                        <version>${project.version}</version>
+                    </dependency>
+                </dependencies>
+                <executions>
+                    <execution>
+                        <phase>process-classes</phase>
+                        <goals>
+                            <goal>java</goal>
+                        </goals>
+                        <configuration>
+                            <includePluginDependencies>true</includePluginDependencies>
+                            <mainClass>org.apache.ignite.tools.classgen.ClassesGenerator</mainClass>
+                            <arguments>
+                                <argument>${project.basedir}/target/classes</argument>
+                                <argument>
+                                    <![CDATA[
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+                                    ]]>
+                                </argument>
+                                <argument>org.apache.ignite</argument>
+                            </arguments>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
             <!-- Generate the OSGi MANIFEST.MF for this bundle. -->
             <plugin>
                 <groupId>org.apache.felix</groupId>
diff --git a/modules/zookeeper/src/main/resources/META-INF/classnames.properties b/modules/zookeeper/src/main/resources/META-INF/classnames.properties
new file mode 100644
index 0000000000000..34e842cb4e56d
--- /dev/null
+++ b/modules/zookeeper/src/main/resources/META-INF/classnames.properties
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.ignite.spi.discovery.zk.internal.ZookeeperClusterNode
\ No newline at end of file
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
index 3775aa1df9b3d..ddb003bec5366 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
@@ -21,7 +21,6 @@
 import org.apache.curator.test.TestingCluster;
 import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest;
 import org.apache.ignite.internal.IgniteClientReconnectCacheTest;
-import org.apache.ignite.internal.processors.cache.IgniteCacheEntryListenerAtomicTest;
 import org.apache.ignite.internal.processors.cache.datastructures.IgniteClientDataStructuresTest;
 import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedNodeRestartTxSelfTest;
 import org.apache.ignite.internal.processors.cache.datastructures.partitioned.GridCachePartitionedSequenceApiSelfTest;
@@ -38,6 +37,7 @@
 import org.apache.ignite.internal.processors.cache.multijvm.GridCacheAtomicMultiJvmFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.multijvm.GridCachePartitionedMultiJvmFullApiSelfTest;
 import org.apache.ignite.internal.processors.continuous.GridEventConsumeSelfTest;
+import org.apache.ignite.util.GridCommandHandlerTest;
 
 /**
  * Regular Ignite tests executed with {@link org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi}.
@@ -89,6 +89,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheAtomicMultiJvmFullApiSelfTest.class);
         suite.addTestSuite(GridCachePartitionedMultiJvmFullApiSelfTest.class);
 
+        suite.addTestSuite(GridCommandHandlerTest.class);
+
         return suite;
     }
 }

From 0fa7e8de720c813a2152dfa7526f5234a658e01a Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 10 May 2018 12:05:37 +0300
Subject: [PATCH 134/543] IGNITE-8422: Zookeeper discovery split brain
 detection shouldn't consider client nodes. This closes #3951.

(cherry picked from commit 534fcec)
---
 .../DefaultCommunicationFailureResolver.java  | 367 ++++++++---------
 .../cluster/graph/BitSetIterator.java         |  66 +++
 .../internal/cluster/graph/ClusterGraph.java  | 207 ++++++++++
 .../FullyConnectedComponentSearcher.java      | 341 ++++++++++++++++
 .../tcp/TcpCommunicationSpi.java              |  82 ++--
 .../FullyConnectedComponentSearcherTest.java  | 323 +++++++++++++++
 .../internal/ZookeeperDiscoverySpiTest.java   | 379 ++++++++++++++++++
 7 files changed, 1539 insertions(+), 226 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/BitSetIterator.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/ClusterGraph.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/FullyConnectedComponentSearcher.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/cluster/FullyConnectedComponentSearcherTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
index a4c6da9e9986b..9ccadf39b2933 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
@@ -18,13 +18,21 @@
 package org.apache.ignite.configuration;
 
 import java.util.BitSet;
+import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.cluster.graph.BitSetIterator;
+import org.apache.ignite.internal.cluster.graph.ClusterGraph;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Default Communication Failure Resolver.
@@ -36,266 +44,239 @@ public class DefaultCommunicationFailureResolver implements CommunicationFailure
 
     /** {@inheritDoc} */
     @Override public void resolve(CommunicationFailureContext ctx) {
-        ClusterGraph graph = new ClusterGraph(log, ctx);
+        ClusterPart largestCluster = findLargestConnectedCluster(ctx);
 
-        ClusterSearch cluster = graph.findLargestIndependentCluster();
+        if (largestCluster == null)
+            return;
 
-        List<ClusterNode> nodes = ctx.topologySnapshot();
+        log.info("Communication problem resolver found fully connected independent cluster ["
+            + "serverNodesCnt=" + largestCluster.srvNodesCnt + ", "
+            + "clientNodesCnt=" + largestCluster.connectedClients.size() + ", "
+            + "totalAliveNodes=" + ctx.topologySnapshot().size() + ", "
+            + "serverNodesIds=" + clusterNodeIds(largestCluster.srvNodesSet, ctx.topologySnapshot(), 1000) + "]");
 
-        assert nodes.size() > 0;
-        assert cluster != null;
-
-        if (graph.checkFullyConnected(cluster.nodesBitSet)) {
-            assert cluster.nodeCnt <= nodes.size();
-
-            if (cluster.nodeCnt < nodes.size()) {
-                if (log.isInfoEnabled()) {
-                    log.info("Communication problem resolver found fully connected independent cluster [" +
-                        "clusterSrvCnt=" + cluster.srvCnt +
-                        ", clusterTotalNodes=" + cluster.nodeCnt +
-                        ", totalAliveNodes=" + nodes.size() + "]");
-                }
-
-                for (int i = 0; i < nodes.size(); i++) {
-                    if (!cluster.nodesBitSet.get(i))
-                        ctx.killNode(nodes.get(i));
-                }
-            }
-            else
-                U.warn(log, "All alive nodes are fully connected, this should be resolved automatically.");
-        }
-        else {
-            if (log.isInfoEnabled()) {
-                log.info("Communication problem resolver failed to find fully connected independent cluster.");
-            }
-        }
+        keepCluster(ctx, largestCluster);
     }
 
     /**
-     * @param cluster Cluster nodes mask.
-     * @param nodes Nodes.
-     * @param limit IDs limit.
-     * @return Cluster node IDs string.
+     * Finds largest part of the cluster where each node is able to connect to each other.
+     *
+     * @param ctx Communication failure context.
+     * @return Largest part of the cluster nodes to keep.
      */
-    private static String clusterNodeIds(BitSet cluster, List<ClusterNode> nodes, int limit) {
-        int startIdx = 0;
+    @Nullable private ClusterPart findLargestConnectedCluster(CommunicationFailureContext ctx) {
+        List<ClusterNode> srvNodes = ctx.topologySnapshot()
+            .stream()
+            .filter(node -> !CU.clientNode(node))
+            .collect(Collectors.toList());
 
-        StringBuilder builder = new StringBuilder();
+        // Exclude client nodes from analysis.
+        ClusterGraph graph = new ClusterGraph(ctx, CU::clientNode);
 
-        int cnt = 0;
+        List<BitSet> components = graph.findConnectedComponents();
 
-        for (;;) {
-            int idx = cluster.nextSetBit(startIdx);
+        if (components.isEmpty()) {
+            U.warn(log, "Unable to find at least one alive server node in the cluster " + ctx);
 
-            if (idx == -1)
-                break;
+            return null;
+        }
 
-            startIdx = idx + 1;
+        if (components.size() == 1) {
+            BitSet nodesSet = components.get(0);
+            int nodeCnt = nodesSet.cardinality();
 
-            if (builder.length() == 0) {
-                builder.append('[');
+            boolean fullyConnected = graph.checkFullyConnected(nodesSet);
+
+            if (fullyConnected && nodeCnt == srvNodes.size()) {
+                U.warn(log, "All alive nodes are fully connected, this should be resolved automatically.");
+
+                return null;
             }
-            else
-                builder.append(", ");
 
-            builder.append(nodes.get(idx).id());
+            if (log.isInfoEnabled())
+                log.info("Communication problem resolver detected partial lost for some connections inside cluster. "
+                    + "Will keep largest set of healthy fully-connected nodes. Other nodes will be killed forcibly.");
 
-            if (cnt++ > limit)
-                builder.append(", ...");
+            BitSet fullyConnectedPart = graph.findLargestFullyConnectedComponent(nodesSet);
+            Set<ClusterNode> connectedClients = findConnectedClients(ctx, fullyConnectedPart);
+
+            return new ClusterPart(fullyConnectedPart, connectedClients);
         }
 
-        builder.append(']');
+        // If cluster has splitted on several parts and there are at least 2 parts which aren't single node
+        // It means that split brain has happened.
+        boolean isSplitBrain = components.size() > 1 &&
+            components.stream().filter(cmp -> cmp.size() > 1).count() > 1;
 
-        return builder.toString();
-    }
+        if (isSplitBrain)
+            U.warn(log, "Communication problem resolver detected split brain. "
+                + "Cluster has splitted on " + components.size() + " independent parts. "
+                + "Will keep only one largest fully-connected part. "
+                + "Other nodes will be killed forcibly.");
+        else
+            U.warn(log, "Communication problem resolver detected full lost for some connections inside cluster. "
+                + "Problem nodes will be found and killed forcibly.");
 
-    /**
-     *
-     */
-    private static class ClusterSearch {
-        /** */
-        int srvCnt;
+        // For each part of splitted cluster extract largest fully-connected component.
+        ClusterPart largestCluster = null;
+        for (int i = 0; i < components.size(); i++) {
+            BitSet clusterPart = components.get(i);
 
-        /** */
-        int nodeCnt;
+            BitSet fullyConnectedPart = graph.findLargestFullyConnectedComponent(clusterPart);
+            Set<ClusterNode> connectedClients = findConnectedClients(ctx, fullyConnectedPart);
 
-        /** */
-        final BitSet nodesBitSet;
+            ClusterPart curr = new ClusterPart(fullyConnectedPart, connectedClients);
 
-        /**
-         * @param nodes Total nodes.
-         */
-        ClusterSearch(int nodes) {
-            nodesBitSet = new BitSet(nodes);
+            if (largestCluster == null || curr.compareTo(largestCluster) > 0)
+                largestCluster = curr;
         }
+
+        assert largestCluster != null
+            : "Unable to find at least one alive independent cluster.";
+
+        return largestCluster;
     }
 
     /**
+     * Keeps server cluster nodes presented in given {@code srvNodesSet}.
+     * Client nodes which have connections to presented {@code srvNodesSet} will be also keeped.
+     * Other nodes will be killed forcibly.
      *
+     * @param ctx Communication failure context.
+     * @param clusterPart Set of nodes need to keep in the cluster.
      */
-    private static class ClusterGraph {
-        /** */
-        private final static int WORD_IDX_SHIFT = 6;
-
-        /** */
-        private final IgniteLogger log;
-
-        /** */
-        private final int nodeCnt;
-
-        /** */
-        private final long[] visitBitSet;
-
-        /** */
-        private final CommunicationFailureContext ctx;
-
-        /** */
-        private final List<ClusterNode> nodes;
+    private void keepCluster(CommunicationFailureContext ctx, ClusterPart clusterPart) {
+        List<ClusterNode> allNodes = ctx.topologySnapshot();
 
-        /**
-         * @param log Logger.
-         * @param ctx Context.
-         */
-        ClusterGraph(IgniteLogger log, CommunicationFailureContext ctx) {
-            this.log = log;
-            this.ctx = ctx;
+        // Kill server nodes.
+        for (int idx = 0; idx < allNodes.size(); idx++) {
+            ClusterNode node = allNodes.get(idx);
 
-            nodes = ctx.topologySnapshot();
+            // Client nodes will be processed separately.
+            if (CU.clientNode(node))
+                continue;
 
-            nodeCnt = nodes.size();
+            if (!clusterPart.srvNodesSet.get(idx))
+                ctx.killNode(node);
+        }
 
-            assert nodeCnt > 0;
+        // Kill client nodes unable to connect to the presented part of cluster.
+        for (int idx = 0; idx < allNodes.size(); idx++) {
+            ClusterNode node = allNodes.get(idx);
 
-            visitBitSet = initBitSet(nodeCnt);
+            if (CU.clientNode(node) && !clusterPart.connectedClients.contains(node))
+                ctx.killNode(node);
         }
+    }
 
-        /**
-         * @param bitIndex Bit index.
-         * @return Word index containing bit with given index.
-         */
-        private static int wordIndex(int bitIndex) {
-            return bitIndex >> WORD_IDX_SHIFT;
-        }
+    /**
+     * Finds set of the client nodes which are able to connect to given set of server nodes {@code srvNodesSet}.
+     *
+     * @param ctx Communication failure context.
+     * @param srvNodesSet Server nodes set.
+     * @return Set of client nodes.
+     */
+    private Set<ClusterNode> findConnectedClients(CommunicationFailureContext ctx, BitSet srvNodesSet) {
+        Set<ClusterNode> connectedClients = new HashSet<>();
 
-        /**
-         * @param bitCnt Number of bits.
-         * @return Bit set words.
-         */
-        static long[] initBitSet(int bitCnt) {
-            return new long[wordIndex(bitCnt - 1) + 1];
-        }
+        List<ClusterNode> allNodes = ctx.topologySnapshot();
 
-        /**
-         * @return Cluster nodes bit set.
-         */
-        ClusterSearch findLargestIndependentCluster() {
-            ClusterSearch maxCluster = null;
+        for (ClusterNode node : allNodes) {
+            if (!CU.clientNode(node))
+                continue;
 
-            for (int i = 0; i < nodeCnt; i++) {
-                if (getBit(visitBitSet, i))
-                    continue;
+            boolean hasConnections = true;
 
-                ClusterSearch cluster = new ClusterSearch(nodeCnt);
+            Iterator<Integer> it = new BitSetIterator(srvNodesSet);
+            while (it.hasNext()) {
+                int srvNodeIdx = it.next();
+                ClusterNode srvNode = allNodes.get(srvNodeIdx);
 
-                search(cluster, i);
+                if (!ctx.connectionAvailable(node, srvNode) || !ctx.connectionAvailable(srvNode, node)) {
+                    hasConnections = false;
 
-                if (log.isInfoEnabled()) {
-                    log.info("Communication problem resolver found cluster [srvCnt=" + cluster.srvCnt +
-                        ", totalNodeCnt=" + cluster.nodeCnt +
-                        ", nodeIds=" + clusterNodeIds(cluster.nodesBitSet, nodes, 1000) + "]");
+                    break;
                 }
-
-                if (maxCluster == null || cluster.srvCnt > maxCluster.srvCnt)
-                    maxCluster = cluster;
             }
 
-            return maxCluster;
+            if (hasConnections)
+                connectedClients.add(node);
         }
 
-        /**
-         * @param cluster Cluster nodes bit set.
-         * @return {@code True} if all cluster nodes are able to connect to each other.
-         */
-        boolean checkFullyConnected(BitSet cluster) {
-            int startIdx = 0;
-
-            int clusterNodes = cluster.cardinality();
-
-            for (;;) {
-                int idx = cluster.nextSetBit(startIdx);
+        return connectedClients;
+    }
 
-                if (idx == -1)
-                    break;
+    /**
+     * Class representing part of cluster.
+     */
+    private static class ClusterPart implements Comparable<ClusterPart> {
+        /** Server nodes count. */
+        int srvNodesCnt;
 
-                ClusterNode node1 = nodes.get(idx);
+        /** Server nodes set. */
+        BitSet srvNodesSet;
 
-                for (int i = 0; i < clusterNodes; i++) {
-                    if (!cluster.get(i) || i == idx)
-                        continue;
+        /** Set of client nodes are able to connect to presented part of server nodes. */
+        Set<ClusterNode> connectedClients;
 
-                    ClusterNode node2 = nodes.get(i);
+        /**
+         * Constructor.
+         *
+         * @param srvNodesSet Server nodes set.
+         * @param connectedClients Set of client nodes.
+         */
+        public ClusterPart(BitSet srvNodesSet, Set<ClusterNode> connectedClients) {
+            this.srvNodesSet = srvNodesSet;
+            this.srvNodesCnt = srvNodesSet.cardinality();
+            this.connectedClients = connectedClients;
+        }
 
-                    if (cluster.get(i) && !ctx.connectionAvailable(node1, node2))
-                        return false;
-                }
+        /** {@inheritDoc} */
+        @Override public int compareTo(@NotNull ClusterPart o) {
+            int srvNodesCmp = Integer.compare(srvNodesCnt, o.srvNodesCnt);
 
-                startIdx = idx + 1;
-            }
+            if (srvNodesCmp != 0)
+                return srvNodesCmp;
 
-            return true;
+            return Integer.compare(connectedClients.size(), o.connectedClients.size());
         }
+    }
 
-        /**
-         * @param cluster Current cluster bit set.
-         * @param idx Node index.
-         */
-        void search(ClusterSearch cluster, int idx) {
-            assert !getBit(visitBitSet, idx);
-
-            setBit(visitBitSet, idx);
-
-            cluster.nodesBitSet.set(idx);
-            cluster.nodeCnt++;
+    /**
+     * @param cluster Cluster nodes mask.
+     * @param nodes Nodes.
+     * @param limit IDs limit.
+     * @return Cluster node IDs string.
+     */
+    private static String clusterNodeIds(BitSet cluster, List<ClusterNode> nodes, int limit) {
+        int startIdx = 0;
 
-            ClusterNode node1 = nodes.get(idx);
+        StringBuilder builder = new StringBuilder();
 
-            if (!CU.clientNode(node1))
-                cluster.srvCnt++;
+        int cnt = 0;
 
-            for (int i = 0; i < nodeCnt; i++) {
-                if (i == idx || getBit(visitBitSet, i))
-                    continue;
+        for (;;) {
+            int idx = cluster.nextSetBit(startIdx);
 
-                ClusterNode node2 = nodes.get(i);
+            if (idx == -1)
+                break;
 
-                boolean connected = ctx.connectionAvailable(node1, node2) ||
-                    ctx.connectionAvailable(node2, node1);
+            startIdx = idx + 1;
 
-                if (connected)
-                    search(cluster, i);
-            }
-        }
+            if (builder.length() == 0)
+                builder.append('[');
+            else
+                builder.append(", ");
 
-        /**
-         * @param words Bit set words.
-         * @param bitIndex Bit index.
-         */
-        static void setBit(long words[], int bitIndex) {
-            int wordIndex = wordIndex(bitIndex);
+            builder.append(nodes.get(idx).id());
 
-            words[wordIndex] |= (1L << bitIndex);
+            if (cnt++ > limit)
+                builder.append(", ...");
         }
 
-        /**
-         * @param words Bit set words.
-         * @param bitIndex Bit index.
-         * @return Bit value.
-         */
-        static boolean getBit(long[] words, int bitIndex) {
-            int wordIndex = wordIndex(bitIndex);
+        builder.append(']');
 
-            return (words[wordIndex] & (1L << bitIndex)) != 0;
-        }
+        return builder.toString();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/BitSetIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/BitSetIterator.java
new file mode 100644
index 0000000000000..3a5cf9f21ea70
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/BitSetIterator.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.cluster.graph;
+
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ * Iterator over set bits in {@link BitSet}.
+ */
+public class BitSetIterator implements Iterator<Integer> {
+    /** Bitset. */
+    private final BitSet bitSet;
+
+    /** Current index. */
+    private int idx = -1;
+
+    /**
+     * @param bitSet Bitset.
+     */
+    public BitSetIterator(BitSet bitSet) {
+        this.bitSet = bitSet;
+
+        advance();
+    }
+
+    /**
+     * Find index of the next set bit.
+     */
+    private void advance() {
+        idx = bitSet.nextSetBit(idx + 1);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean hasNext() {
+        return idx != -1;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Integer next() throws NoSuchElementException {
+        if (idx == -1)
+            throw new NoSuchElementException();
+
+        int res = idx;
+
+        advance();
+
+        return res;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/ClusterGraph.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/ClusterGraph.java
new file mode 100644
index 0000000000000..ba56c3386e34e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/ClusterGraph.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.cluster.graph;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.function.Predicate;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CommunicationFailureContext;
+
+/**
+ * Class to represent cluster nodes avalaible connections as graph.
+ * Provides several graph algorithms to analyze cluster nodes connections.
+ */
+public class ClusterGraph {
+    /** Number of all cluster nodes. */
+    private final int nodeCnt;
+
+    /** List of the all cluster nodes. */
+    private final List<ClusterNode> nodes;
+
+    /** Connectivity (adjacency) matrix between cluster nodes. */
+    private final BitSet[] connections;
+
+    /** Fully-connected component searcher. */
+    private final FullyConnectedComponentSearcher fccSearcher;
+
+    /**
+     * Constructor.
+     *
+     * @param ctx Communication failure context.
+     * @param nodeFilterOut Filter to exclude some cluster nodes from graph.
+     */
+    public ClusterGraph(CommunicationFailureContext ctx, Predicate<ClusterNode> nodeFilterOut) {
+        nodes = ctx.topologySnapshot();
+
+        nodeCnt = nodes.size();
+
+        assert nodeCnt > 0;
+
+        connections = buildConnectivityMatrix(ctx, nodeFilterOut);
+
+        fccSearcher = new FullyConnectedComponentSearcher(connections);
+    }
+
+    /**
+     * Builds connectivity matrix (adjacency matrix) for all cluster nodes.
+     *
+     * @param ctx Communication failure context.
+     * @param nodeFilterOut Filter to exclude some cluster nodes from graph.
+     * @return Connections bit set for each node, where set bit means avalable connection.
+     */
+    private BitSet[] buildConnectivityMatrix(CommunicationFailureContext ctx, Predicate<ClusterNode> nodeFilterOut) {
+        BitSet[] connections = new BitSet[nodeCnt];
+
+        for (int i = 0; i < nodeCnt; i++) {
+            ClusterNode node = nodes.get(i);
+
+            if (nodeFilterOut.test(node)) {
+                connections[i] = null;
+                continue;
+            }
+
+            connections[i] = new BitSet(nodeCnt);
+            for (int j = 0; j < nodeCnt; j++) {
+                ClusterNode to = nodes.get(j);
+
+                if (nodeFilterOut.test(to))
+                    continue;
+
+                if (i == j || ctx.connectionAvailable(node, to))
+                    connections[i].set(j);
+            }
+        }
+
+        // Remove unidirectional connections (node A can connect to B, but B can't connect to A).
+        for (int i = 0; i < nodeCnt; i++)
+            for (int j = i + 1; j < nodeCnt; j++) {
+                if (connections[i] == null || connections[j] == null)
+                    continue;
+
+                if (connections[i].get(j) ^ connections[j].get(i)) {
+                    connections[i].set(j, false);
+                    connections[j].set(i, false);
+                }
+            }
+
+        return connections;
+    }
+
+    /**
+     * Finds connected components in cluster graph.
+     *
+     * @return List of set of nodes, each set represents connected component.
+     */
+    public List<BitSet> findConnectedComponents() {
+        List<BitSet> connectedComponets = new ArrayList<>();
+
+        BitSet visitSet = new BitSet(nodeCnt);
+
+        for (int i = 0; i < nodeCnt; i++) {
+            if (visitSet.get(i) || connections[i] == null)
+                continue;
+
+            BitSet currComponent = new BitSet(nodeCnt);
+
+            dfs(i, currComponent, visitSet);
+
+            connectedComponets.add(currComponent);
+        }
+
+        return connectedComponets;
+    }
+
+    /**
+     * Deep-first search to find connected components in connections graph.
+     *
+     * @param nodeIdx Current node index to traverse from.
+     * @param currComponent Current connected component to populate.
+     * @param allVisitSet Set of the visited nodes in whole graph during traversal.
+     */
+    private void dfs(int nodeIdx, BitSet currComponent, BitSet allVisitSet) {
+        assert !allVisitSet.get(nodeIdx)
+            : "Incorrect node visit " + nodeIdx;
+
+        assert connections[nodeIdx] != null
+            : "Incorrect node visit. Node has not passed filter " + nodes.get(nodeIdx);
+
+        allVisitSet.set(nodeIdx);
+
+        currComponent.set(nodeIdx);
+
+        for (int toIdx = 0; toIdx < nodeCnt; toIdx++) {
+            if (toIdx == nodeIdx || allVisitSet.get(toIdx) || connections[toIdx] == null)
+                continue;
+
+            boolean connected = connections[nodeIdx].get(toIdx) && connections[toIdx].get(nodeIdx);
+
+            if (connected)
+                dfs(toIdx, currComponent, allVisitSet);
+        }
+    }
+
+    /**
+     * Finds largest fully-connected component from given {@code nodesSet}.
+     *
+     * @param nodesSet Set of nodes.
+     * @return Set of nodes which forms largest fully-connected component.
+     */
+    public BitSet findLargestFullyConnectedComponent(BitSet nodesSet) {
+        // Check that current set is already fully connected.
+        boolean fullyConnected = checkFullyConnected(nodesSet);
+
+        if (fullyConnected)
+            return nodesSet;
+
+        BitSet res = fccSearcher.findLargest(nodesSet);
+
+        assert checkFullyConnected(res)
+            : "Not fully connected component was found [result=" + res + ", nodesSet=" + nodesSet + "]";
+
+        return res;
+    }
+
+    /**
+     * Checks that given {@code nodesSet} forms fully-connected component.
+     *
+     * @param nodesSet Set of cluster nodes.
+     * @return {@code True} if all given cluster nodes are able to connect to each other.
+     */
+    public boolean checkFullyConnected(BitSet nodesSet) {
+        int maxIdx = nodesSet.length();
+
+        Iterator<Integer> it = new BitSetIterator(nodesSet);
+
+        while (it.hasNext()) {
+            int idx = it.next();
+
+            for (int i = 0; i < maxIdx; i++) {
+                if (i == idx)
+                    continue;
+
+                if (nodesSet.get(i) && !connections[idx].get(i))
+                    return false;
+            }
+        }
+
+        return true;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/FullyConnectedComponentSearcher.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/FullyConnectedComponentSearcher.java
new file mode 100644
index 0000000000000..9a8098eb3de70
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/graph/FullyConnectedComponentSearcher.java
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.cluster.graph;
+
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Comparator;
+import java.util.Iterator;
+
+/**
+ * Class to find (possibly) largest fully-connected component (also can be called as <b>complete subgraph</b>) in graph.
+ * This problem is also known as <b>Clique problem</b> which is NP-complete.
+ *
+ * For small number of nodes simple brute-force algorithm is used which finds such component guaranteed.
+ * For large number of nodes some sort of greedy heuristic is used which works well for real-life scenarios
+ * but doesn't guarantee to find largest component, however very close to ideal result.
+ */
+public class FullyConnectedComponentSearcher {
+    /** The maximal number of nodes when bruteforce algorithm will be used. */
+    private static final int BRUTE_FORCE_THRESHOULD = 24;
+
+    /** Number of nodes in connections graph. */
+    private final int totalNodesCnt;
+
+    /** Adjacency matrix. */
+    private final BitSet[] connections;
+
+    /**
+     * Constructor.
+     *
+     * @param connections Adjacency matrix.
+     */
+    public FullyConnectedComponentSearcher(BitSet[] connections) {
+        this.connections = connections;
+        totalNodesCnt = connections.length;
+    }
+
+    /**
+     * Find largest fully connected component from presented set of the nodes {@code where}.
+     *
+     * @param where Set of nodes where fully connected component must be found.
+     * @return Set of nodes forming fully connected component.
+     */
+    public BitSet findLargest(BitSet where) {
+        int nodesCnt = where.cardinality();
+
+        if (nodesCnt <= BRUTE_FORCE_THRESHOULD)
+            return bruteforce(nodesCnt, where);
+
+        // Return best of the 2 heuristics.
+        BitSet e1 = heuristic1(where);
+        BitSet e2 = heuristic2(where);
+
+        return e1.cardinality() > e2.cardinality() ? e1 : e2;
+    }
+
+    /**
+     * Extract node indexes (set bits) from given {@code selectedSet} to integer array.
+     *
+     * @param selectedNodesCnt Number of nodes.
+     * @param selectedSet Set of nodes.
+     * @return Arrays which contains node indexes.
+     */
+    private Integer[] extractNodeIndexes(int selectedNodesCnt, BitSet selectedSet) {
+        Integer[] indexes = new Integer[selectedNodesCnt];
+        Iterator<Integer> it = new BitSetIterator(selectedSet);
+        int i = 0;
+
+        while (it.hasNext())
+            indexes[i++] = it.next();
+
+        assert i == indexes.length
+            : "Extracted not all indexes [nodesCnt=" + selectedNodesCnt + ", extracted=" + i + ", set=" + selectedSet + "]";
+
+        return indexes;
+    }
+
+    /**
+     * Sorts nodes using {@link ConnectionsComparator}
+     * and runs greedy algorithm {@link #greedyIterative(int, Integer[])} on it.
+     *
+     * @param selectedSet Set of nodes used to form fully-connected component.
+     * @return Subset of given {@code selectedSet} which forms fully connected component.
+     */
+    private BitSet heuristic1(BitSet selectedSet) {
+        int selectedNodesCnt = selectedSet.cardinality();
+        Integer[] nodeIndexes = extractNodeIndexes(selectedNodesCnt, selectedSet);
+
+        Arrays.sort(nodeIndexes, new ConnectionsComparator(totalNodesCnt));
+
+        return greedyIterative(selectedNodesCnt, nodeIndexes);
+    }
+
+    /**
+     * Exactly the same thing as in {@link #heuristic1(BitSet)} but using reversed {@link ConnectionsComparator}.
+     *
+     * @param selectedSet Set of nodes used to form fully-connected component.
+     * @return Subset of given {@code selectedSet} which forms fully connected component.
+     */
+    private BitSet heuristic2(BitSet selectedSet) {
+        int selectedNodesCnt = selectedSet.cardinality();
+        Integer[] nodeIndexes = extractNodeIndexes(selectedNodesCnt, selectedSet);
+
+        Arrays.sort(nodeIndexes, new ConnectionsComparator(totalNodesCnt).reversed());
+
+        return greedyIterative(selectedNodesCnt, nodeIndexes);
+    }
+
+    /**
+     * Finds fully-connected component between given {@code nodeIndexes} and tries to maximize size of it.
+     *
+     * The main idea of the algorithm is that after specific sorting,
+     * nodes able to form fully-connected will be placed closer to each other in given {@code nodeIndexes} array.
+     * While nodes not able to form will be placed further.
+     *
+     * At the begging of algorithm we form global set of nodes can be used to form fully-connected component.
+     * We iterate over this set and try to add each node to current fully-connected component, which is empty at the beginning.
+     *
+     * When we add node to the component we need to check that after adding new component is also fully-connected.
+     * See {@link #joinNode(BitSet, int, Integer[])}.
+     *
+     * After end of iteration we exclude nodes which formed fully-connected from the global set and run iteration again and again
+     * on remaining nodes, while the global set will not be empty.
+     *
+     * Complexity is O(N^2), where N is number of nodes.
+     *
+     * @param selectedNodesCnt Number of nodes.
+     * @param nodeIndexes Node indexes used to form fully-connected component.
+     * @return Subset of given {@code nodeIndexes} which forms fully connected component.
+     */
+    private BitSet greedyIterative(int selectedNodesCnt, Integer[] nodeIndexes) {
+        // Set of the nodes which can be used to form fully connected component.
+        BitSet canUse = new BitSet(selectedNodesCnt);
+        for (int i = 0; i < selectedNodesCnt; i++)
+            canUse.set(i);
+
+        BitSet bestRes = null;
+
+        while (!canUse.isEmpty()) {
+            // Even if we pick all possible nodes, their size will not be greater than current best result.
+            // No needs to run next iteration in this case.
+            if (bestRes != null && canUse.cardinality() <= bestRes.cardinality())
+                break;
+
+            BitSet currRes = new BitSet(selectedNodesCnt);
+
+            Iterator<Integer> canUseIter = new BitSetIterator(canUse);
+            while (canUseIter.hasNext()) {
+                /* Try to add node to the current set that forms fully connected component.
+                   Node will be skipped if after adding, current set loose fully connectivity. */
+                int pickedIdx = canUseIter.next();
+
+                if (joinNode(currRes, pickedIdx, nodeIndexes)) {
+                    currRes.set(pickedIdx);
+                    canUse.set(pickedIdx, false);
+                }
+            }
+
+            if (bestRes == null || currRes.cardinality() > bestRes.cardinality())
+                bestRes = currRes;
+        }
+
+        // Try to improve our best result, if it was formed on second or next iteration.
+        for (int nodeIdx = 0; nodeIdx < selectedNodesCnt; nodeIdx++)
+            if (!bestRes.get(nodeIdx) && joinNode(bestRes, nodeIdx, nodeIndexes))
+                bestRes.set(nodeIdx);
+
+        // Replace relative node indexes (used in indexes) to absolute node indexes (used in whole graph connections).
+        BitSet reindexedBestRes = new BitSet(totalNodesCnt);
+        Iterator<Integer> it = new BitSetIterator(bestRes);
+        while (it.hasNext())
+            reindexedBestRes.set(nodeIndexes[it.next()]);
+
+        return reindexedBestRes;
+    }
+
+    /**
+     * Checks that given {@code nodeIdx} can be joined to current fully-connected component,
+     * so after join result component will be also fully-connected.
+     *
+     * @param currComponent Current fully-connected component.
+     * @param nodeIdx Node relative index.
+     * @param nodeIndexes Node absolute indexes.
+     * @return {@code True} if given node can be joined to {@code currentComponent}.
+     */
+    private boolean joinNode(BitSet currComponent, int nodeIdx, Integer[] nodeIndexes) {
+        boolean fullyConnected = true;
+
+        Iterator<Integer> alreadyUsedIter = new BitSetIterator(currComponent);
+        while (alreadyUsedIter.hasNext()) {
+            int existedIdx = alreadyUsedIter.next();
+
+            // If no connection between existing node and picked node, skip picked node.
+            if (!connections[nodeIndexes[nodeIdx]].get(nodeIndexes[existedIdx])) {
+                fullyConnected = false;
+
+                break;
+            }
+        }
+
+        return fullyConnected;
+    }
+
+    /**
+     * Simple bruteforce implementation which works in O(2^N * N^2), where N is number of nodes.
+     *
+     * @param selectedNodesCnt Nodes count.
+     * @param selectedSet Set of nodes.
+     * @return Subset of given {@code set} of nodes which forms fully connected component.
+     */
+    private BitSet bruteforce(int selectedNodesCnt, BitSet selectedSet) {
+        Integer[] indexes = extractNodeIndexes(selectedNodesCnt, selectedSet);
+
+        int resMask = -1;
+        int maxCardinality = -1;
+
+        // Iterate over all possible combinations of used nodes.
+        for (int mask = (1 << selectedNodesCnt) - 1; mask > 0; mask--) {
+            int cardinality = Integer.bitCount(mask);
+
+            if (cardinality <= maxCardinality)
+                continue;
+
+            // Check that selected set of nodes forms fully connected component.
+            boolean fullyConnected = true;
+
+            for (int i = 0; i < selectedNodesCnt && fullyConnected; i++)
+                if ((mask & (1 << i)) != 0)
+                    for (int j = 0; j < selectedNodesCnt; j++)
+                        if ((mask & (1 << j)) != 0) {
+                            boolean connected = connections[indexes[i]].get(indexes[j]);
+
+                            if (!connected) {
+                                fullyConnected = false;
+
+                                break;
+                            }
+                        }
+
+            if (fullyConnected) {
+                resMask = mask;
+                maxCardinality = cardinality;
+            }
+        }
+
+        BitSet resSet = new BitSet(selectedNodesCnt);
+
+        for (int i = 0; i < selectedNodesCnt; i++) {
+            if ((resMask & (1 << i)) != 0) {
+                int idx = indexes[i];
+
+                assert selectedSet.get(idx)
+                    : "Result contains node which is not presented in income set [nodeIdx" + idx + ", set=" + selectedSet + "]";
+
+                resSet.set(idx);
+            }
+        }
+
+        assert resSet.cardinality() > 0
+            : "No nodes selected as fully connected component [set=" + selectedSet + "]";
+
+        return resSet;
+    }
+
+    /**
+     * Comparator which sorts nodes by their connections array.
+     *
+     * Suppose you have connections matrix:
+     *
+     * 1111
+     * 1101
+     * 1010
+     * 1101
+     *
+     * Each connection row associated with some node.
+     * Comparator will sort node indexes using their connection rows as very big binary numbers, as in example:
+     *
+     * 1111
+     * 1101
+     * 1101
+     * 1011
+     *
+     * Note: Comparator sorts only node indexes, actual connection rows swapping will be not happened.
+     */
+    private class ConnectionsComparator implements Comparator<Integer> {
+        /** Cache each connection long array representation, to avoid creating new object for each comparison. */
+        private final long[][] cachedConnRows;
+
+        /**
+         * Constructor
+         * @param totalNodesCnt Total number of nodes in the cluster.
+         */
+        ConnectionsComparator(int totalNodesCnt) {
+            cachedConnRows = new long[totalNodesCnt][];
+        }
+
+        /**
+         * Returns long array representation of connection row for given node {@code idx}.
+         *
+         * @param idx Node index.
+         * @return Long array connection row representation.
+         */
+        private long[] connectionRow(int idx) {
+            if (cachedConnRows[idx] != null)
+                return cachedConnRows[idx];
+
+            return cachedConnRows[idx] = connections[idx].toLongArray();
+        }
+
+        /** {@inheritDoc} */
+        @Override public int compare(Integer node1, Integer node2) {
+            long[] conn1 = connectionRow(node1);
+            long[] conn2 = connectionRow(node2);
+
+            int len = Math.min(conn1.length, conn2.length);
+            for (int i = 0; i < len; i++) {
+                int res = Long.compare(conn1[i], conn2[i]);
+
+                if (res != 0)
+                    return res;
+            }
+
+            return conn1.length - conn2.length;
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index df37dff1f6f10..f9fd6fd504ad1 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3442,50 +3442,66 @@ else if (X.hasCause(e, SocketTimeoutException.class))
                 break;
         }
 
-        if (client == null) {
-            assert errs != null;
+        if (client == null)
+            processClientCreationError(node, addrs, errs);
 
-            if (X.hasCause(errs, ConnectException.class))
-                LT.warn(log, "Failed to connect to a remote node " +
-                    "(make sure that destination node is alive and " +
-                    "operating system firewall is disabled on local and remote hosts) " +
-                    "[addrs=" + addrs + ']');
+        return client;
+    }
+
+    /**
+     * Process errors if TCP client to remote node hasn't been created.
+     *
+     * @param node Remote node.
+     * @param addrs Remote node addresses.
+     * @param errs TCP client creation errors.
+     *
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void processClientCreationError(
+        ClusterNode node,
+        Collection<InetSocketAddress> addrs,
+        IgniteCheckedException errs
+    ) throws IgniteCheckedException {
+        assert errs != null;
 
-            boolean commErrResolve = false;
+        if (X.hasCause(errs, ConnectException.class))
+            LT.warn(log, "Failed to connect to a remote node " +
+                "(make sure that destination node is alive and " +
+                "operating system firewall is disabled on local and remote hosts) " +
+                "[addrs=" + addrs + ']');
 
-            IgniteSpiContext ctx = getSpiContext();
+        boolean commErrResolve = false;
 
-            if (connectionError(errs) && ctx.communicationFailureResolveSupported()) {
-                commErrResolve = true;
+        IgniteSpiContext ctx = getSpiContext();
 
-                ctx.resolveCommunicationFailure(node, errs);
-            }
+        if (connectionError(errs) && ctx.communicationFailureResolveSupported()) {
+            commErrResolve = true;
 
-            if (!commErrResolve && enableForcibleNodeKill) {
-                if (ctx.node(node.id()) != null
-                    && (CU.clientNode(node) ||  !CU.clientNode(getLocalNode())) &&
-                    connectionError(errs)) {
-                    String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
-                        "cluster [" + "rmtNode=" + node + ']';
+            ctx.resolveCommunicationFailure(node, errs);
+        }
 
-                    if (enableTroubleshootingLog)
-                        U.error(log, msg, errs);
-                    else
-                        U.warn(log, msg);
+        if (!commErrResolve && enableForcibleNodeKill) {
+            if (ctx.node(node.id()) != null
+                && (CU.clientNode(node) ||  !CU.clientNode(getLocalNode())) &&
+                connectionError(errs)) {
+                String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
+                    "cluster [" + "rmtNode=" + node + ']';
 
-                    ctx.failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
-                        "rmtNode=" + node +
-                        ", errs=" + errs +
-                        ", connectErrs=" + X.getSuppressedList(errs) + ']');
-                }
-            }
+                if (enableTroubleshootingLog)
+                    U.error(log, msg, errs);
+                else
+                    U.warn(log, msg);
 
-            if (!X.hasCause(errs, SocketTimeoutException.class, HandshakeTimeoutException.class,
-                IgniteSpiOperationTimeoutException.class))
-                throw errs;
+                ctx.failNode(node.id(), "TcpCommunicationSpi failed to establish connection to node [" +
+                    "rmtNode=" + node +
+                    ", errs=" + errs +
+                    ", connectErrs=" + X.getSuppressedList(errs) + ']');
+            }
         }
 
-        return client;
+        if (!X.hasCause(errs, SocketTimeoutException.class, HandshakeTimeoutException.class,
+            IgniteSpiOperationTimeoutException.class))
+            throw errs;
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/cluster/FullyConnectedComponentSearcherTest.java b/modules/core/src/test/java/org/apache/ignite/internal/cluster/FullyConnectedComponentSearcherTest.java
new file mode 100644
index 0000000000000..d6680cf0867f8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/cluster/FullyConnectedComponentSearcherTest.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.cluster;
+
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Random;
+import org.apache.ignite.internal.cluster.graph.FullyConnectedComponentSearcher;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ * Class to test correctness of fully-connectet component searching algorithm.
+ */
+@RunWith(Parameterized.class)
+public class FullyConnectedComponentSearcherTest {
+    /** Adjacency matrix provider for each test. */
+    private AdjacencyMatrixProvider provider;
+
+    /** Minimul acceptable result of size of fully-connected component for each test. */
+    private int minAcceptableRes;
+
+    /**
+     * @param provider Adjacency matrix.
+     * @param minAcceptableRes Expected result.
+     */
+    public FullyConnectedComponentSearcherTest(AdjacencyMatrixProvider provider, int minAcceptableRes) {
+        this.provider = provider;
+        this.minAcceptableRes = minAcceptableRes;
+    }
+
+    /**
+     *
+     */
+    @Test
+    public void testFind() {
+        BitSet[] matrix = provider.provide();
+
+        int nodes = matrix.length;
+
+        BitSet all = new BitSet(nodes);
+        for (int i = 0; i < nodes; i++)
+            all.set(i);
+
+        FullyConnectedComponentSearcher searcher = new FullyConnectedComponentSearcher(matrix);
+
+        BitSet res = searcher.findLargest(all);
+        int size = res.cardinality();
+
+        Assert.assertTrue("Actual = " + size + ", Expected = " + minAcceptableRes,
+            size >= minAcceptableRes);
+    }
+
+    /**
+     * @return Test dataset.
+     */
+    @Parameterized.Parameters(name = "{index}: search({0}) >= {1}")
+    public static Collection<Object[]> data() {
+        return Arrays.asList(new Object[][]{
+            {new StaticMatrix(new String[] {
+                "100",
+                "010",
+                "001",
+            }), 1},
+            {new StaticMatrix(new String[] {
+                "101",
+                "010",
+                "101",
+            }), 2},
+            {new StaticMatrix(new String[] {
+                "1101",
+                "1111",
+                "0110",
+                "1101",
+            }), 3},
+            {new StaticMatrix(new String[] {
+                "1111001",
+                "1111000",
+                "1111000",
+                "1111000",
+                "0000111",
+                "0000111",
+                "1000111",
+            }), 4},
+            {new AlmostSplittedMatrix(30, 100, 200), 200},
+            {new AlmostSplittedMatrix(500, 1000, 2000), 2000},
+            {new AlmostSplittedMatrix(1000, 2000, 3000), 3000},
+            {new AlmostSplittedMatrix(30, 22, 25, 33, 27), 33},
+            {new AlmostSplittedMatrix(1000, 400, 1000, 800), 1000},
+            {new SeveralConnectionsAreLostMatrix(200, 10), 190},
+            {new SeveralConnectionsAreLostMatrix(2000, 100), 1900},
+            {new SeveralConnectionsAreLostMatrix(2000, 500), 1500},
+            {new SeveralConnectionsAreLostMatrix(4000, 2000), 2000}
+        });
+    }
+
+    /**
+     * Provider for adjacency matrix for each test.
+     */
+    interface AdjacencyMatrixProvider {
+        /**
+         * @return Adjacency matrix.
+         */
+        BitSet[] provide();
+    }
+
+    /**
+     * Static graph represented as array of strings. Each cell (i, j) in such matrix means that there is connection
+     * between node(i) and node(j). Needed mostly to test bruteforce algorithm implementation.
+     */
+    static class StaticMatrix implements AdjacencyMatrixProvider {
+        /** Matrix. */
+        private final BitSet[] matrix;
+
+        /**
+         * @param strMatrix String matrix.
+         */
+        public StaticMatrix(@NotNull String[] strMatrix) {
+            A.ensure(strMatrix.length > 0, "Matrix should not be empty");
+            for (int i = 0; i < strMatrix.length; i++)
+                A.ensure(strMatrix[i].length() == strMatrix.length,
+                    "Matrix should be quadratic. Problem row: " + i);
+
+            int nodes = strMatrix.length;
+
+            matrix = init(nodes);
+
+            for (int i = 0; i < nodes; i++)
+                for (int j = 0; j < nodes; j++)
+                    matrix[i].set(j, strMatrix[i].charAt(j) == '1');
+        }
+
+        /** {@inheritDoc} */
+        @Override public BitSet[] provide() {
+            return matrix;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "StaticMatrix{" +
+                "matrix=" + Arrays.toString(matrix) +
+                '}';
+        }
+    }
+
+    /**
+     * A graph splitted on several isolated fully-connected components,
+     * but each of such component have some connections to another to reach graph connectivity.
+     * Answer is this case should be the size max(Pi), where Pi size of each fully-connected component.
+     */
+    static class AlmostSplittedMatrix implements AdjacencyMatrixProvider {
+        /** Partition sizes. */
+        private final int[] partSizes;
+
+        /** Connections between parts. */
+        private final int connectionsBetweenParts;
+
+        /** Matrix. */
+        private final BitSet[] matrix;
+
+        /**
+         * @param connectionsBetweenParts Connections between parts.
+         * @param partSizes Partition sizes.
+         */
+        public AlmostSplittedMatrix(int connectionsBetweenParts, int... partSizes) {
+            A.ensure(connectionsBetweenParts >= 1 + partSizes.length, "There should be at least 1 connection between parts");
+            A.ensure(partSizes.length >= 2, "The should be at least 2 parts of cluster");
+            for (int i = 0; i < partSizes.length; i++)
+                A.ensure(partSizes[i] > 0, "Part size " + (i + 1) + " shouldn't be empty");
+
+            this.partSizes = partSizes.clone();
+            this.connectionsBetweenParts = connectionsBetweenParts;
+
+            int nodes = 0;
+            for (int i = 0; i < partSizes.length; i++)
+                nodes += partSizes[i];
+
+            matrix = init(nodes);
+
+            int[] startIdx = new int[partSizes.length];
+
+            for (int i = 0, total = 0; i < partSizes.length; i++) {
+                startIdx[i] = total;
+
+                fillAll(matrix, total, total + partSizes[i]);
+
+                total += partSizes[i];
+            }
+
+            Random random = new Random(777);
+
+            for (int i = 0, part1 = 0; i < connectionsBetweenParts; i++) {
+                int part2 = (part1 + 1) % partSizes.length;
+
+                // Pick 2 random nodes from 2 parts and add connection between them.
+                int idx1 = random.nextInt(partSizes[part1]) + startIdx[part1];
+                int idx2 = random.nextInt(partSizes[part2]) + startIdx[part2];
+
+                matrix[idx1].set(idx2);
+                matrix[idx2].set(idx1);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public BitSet[] provide() {
+            return matrix;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "AlmostSplittedGraph{" +
+                "partSizes=" + Arrays.toString(partSizes) +
+                ", connectionsBetweenParts=" + connectionsBetweenParts +
+                '}';
+        }
+    }
+
+    /**
+     * Complete graph with several connections lost choosen randomly.
+     * In worst case each lost connection decreases potential size of maximal fully-connected component.
+     * So answer in this test case should be at least N - L, where N - nodes, L - lost connections.
+     */
+    static class SeveralConnectionsAreLostMatrix implements AdjacencyMatrixProvider {
+        /** Nodes. */
+        private final int nodes;
+
+        /** Lost connections. */
+        private final int lostConnections;
+
+        /** Matrix. */
+        private final BitSet[] matrix;
+
+        /**
+         * @param nodes Nodes.
+         * @param lostConnections Lost connections.
+         */
+        public SeveralConnectionsAreLostMatrix(int nodes, int lostConnections) {
+            A.ensure(nodes > 0, "There should be at least 1 node");
+
+            this.nodes = nodes;
+            this.lostConnections = lostConnections;
+
+            this.matrix = init(nodes);
+
+            fillAll(matrix, 0, nodes);
+
+            Random random = new Random(777);
+
+            for (int i = 0; i < lostConnections; i++) {
+                int idx1 = random.nextInt(nodes);
+                int idx2 = random.nextInt(nodes);
+
+                if (idx1 == idx2)
+                    continue;
+
+                matrix[idx1].set(idx2, false);
+                matrix[idx2].set(idx1, false);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public BitSet[] provide() {
+            return matrix;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "SeveralConnectionsAreLost{" +
+                "nodes=" + nodes +
+                ", lostConnections=" + lostConnections +
+                '}';
+        }
+    }
+
+    /**
+     * Utility method to pre-create adjacency matrix.
+     *
+     * @param nodes Nodes in graph.
+     * @return Adjacency matrix.
+     */
+    private static BitSet[] init(int nodes) {
+        BitSet[] matrix = new BitSet[nodes];
+        for (int i = 0; i < nodes; i++)
+            matrix[i] = new BitSet(nodes);
+
+        return matrix;
+    }
+
+    /**
+     * Utility method to fill all connections between all nodes from {@code fromIdx} and {@code endIdx} exclusive.
+     *
+     * @param matrix Adjacency matrix.
+     * @param fromIdx Lower bound node index inclusive.
+     * @param endIdx Upper bound node index exclusive.
+     */
+    private static void fillAll(BitSet[] matrix, int fromIdx, int endIdx) {
+        for (int i = fromIdx; i < endIdx; i++)
+            for (int j = fromIdx; j < endIdx; j++) {
+                matrix[i].set(j);
+                matrix[j].set(i);
+            }
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index 408af30c87e05..03b874dce1bf9 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -17,8 +17,13 @@
 
 package org.apache.ignite.spi.discovery.zk.internal;
 
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.Serializable;
 import java.lang.management.ManagementFactory;
+import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
@@ -41,6 +46,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 import javax.management.JMX;
 import javax.management.MBeanServer;
 import javax.management.ObjectName;
@@ -84,6 +90,8 @@
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.lang.IgniteInClosure2X;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.T3;
 import org.apache.ignite.internal.util.typedef.X;
@@ -169,6 +177,9 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
     /** */
     private boolean testCommSpi;
 
+    /** */
+    private boolean failCommSpi;
+
     /** */
     private long sesTimeout;
 
@@ -335,6 +346,9 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
         if (testCommSpi)
             cfg.setCommunicationSpi(new ZkTestCommunicationSpi());
 
+        if (failCommSpi)
+            cfg.setCommunicationSpi(new PeerToPeerCommunicationFailureSpi());
+
         if (commFailureRslvr != null)
             cfg.setCommunicationFailureResolver(commFailureRslvr.apply());
 
@@ -3558,6 +3572,369 @@ public void testReconnectServersRestart_2() throws Exception {
         reconnectServersRestart(3);
     }
 
+    /**
+     * A simple split-brain test, where cluster spliited on 2 parts of server nodes (2 and 3).
+     * There is also client which sees both parts of splitted cluster.
+     *
+     * Result cluster should be: 3 server nodes + 1 client.
+     *
+     * @throws Exception If failed.
+     */
+    public void testSimpleSplitBrain() throws Exception {
+        failCommSpi = true;
+
+        startGridsMultiThreaded(5);
+
+        client = true;
+
+        startGridsMultiThreaded(5, 3);
+
+        List<ClusterNode> all = G.allGrids().stream()
+            .map(g -> g.cluster().localNode())
+            .collect(Collectors.toList());;
+
+        List<ClusterNode> part1 = all.subList(0, 3);
+        List<ClusterNode> part2 = all.subList(3, all.size());
+
+        ConnectionsFailureMatrix matrix = ConnectionsFailureMatrix.buildFrom(part1, part2);
+
+        ClusterNode lastClient = startGrid(8).cluster().localNode();
+
+        // Make last client connected to other nodes.
+        for (ClusterNode node : all) {
+            if (node.id().equals(lastClient.id()))
+                continue;
+
+            matrix.addConnection(lastClient, node);
+            matrix.addConnection(node, lastClient);
+        }
+
+        PeerToPeerCommunicationFailureSpi.fail(matrix);
+
+        waitForTopology(4);
+    }
+
+    /**
+     * A simple not actual split-brain test, where some connections between server nodes are lost.
+     * Server nodes: 5.
+     * Client nodes: 5.
+     * Lost connections between server nodes: 2.
+     *
+     * Result cluster should be: 3 server nodes + 5 clients.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNotActualSplitBrain() throws Exception {
+        failCommSpi = true;
+
+        startGridsMultiThreaded(5);
+
+        List<ClusterNode> srvNodes = G.allGrids().stream()
+            .map(g -> g.cluster().localNode())
+            .collect(Collectors.toList());
+
+        client = true;
+
+        startGridsMultiThreaded(5, 3);
+
+        client = false;
+
+        ConnectionsFailureMatrix matrix = new ConnectionsFailureMatrix();
+
+        matrix.addAll(G.allGrids().stream().map(g -> g.cluster().localNode()).collect(Collectors.toList()));
+
+        // Remove 2 connections between server nodes.
+        matrix.removeConnection(srvNodes.get(0), srvNodes.get(1));
+        matrix.removeConnection(srvNodes.get(1), srvNodes.get(0));
+        matrix.removeConnection(srvNodes.get(2), srvNodes.get(3));
+        matrix.removeConnection(srvNodes.get(3), srvNodes.get(2));
+
+        PeerToPeerCommunicationFailureSpi.fail(matrix);
+
+        waitForTopology(8);
+    }
+
+    /**
+     * Almost split-brain test, server nodes splitted on 2 parts and there are some connections between these 2 parts.
+     * Server nodes: 5.
+     * Client nodes: 5.
+     * Splitted on: 3 servers + 2 clients and 3 servers + 2 clients.
+     * Extra connections between server nodes: 3.
+     *
+     * Result cluster should be: 3 server nodes + 2 clients.
+     *
+     * @throws Exception If failed.
+     */
+    public void testAlmostSplitBrain() throws Exception {
+        failCommSpi = true;
+
+        startGridsMultiThreaded(6);
+
+        List<ClusterNode> srvNodes = G.allGrids().stream()
+            .map(g -> g.cluster().localNode())
+            .collect(Collectors.toList());
+
+        List<ClusterNode> srvPart1 = srvNodes.subList(0, 3);
+        List<ClusterNode> srvPart2 = srvNodes.subList(3, srvNodes.size());
+
+        client = true;
+
+        startGridsMultiThreaded(6, 5);
+
+        client = false;
+
+        List<ClusterNode> clientNodes = G.allGrids().stream()
+            .map(g -> g.cluster().localNode())
+            .filter(ClusterNode::isClient)
+            .collect(Collectors.toList());
+
+        List<ClusterNode> clientPart1 = clientNodes.subList(0, 2);
+        List<ClusterNode> clientPart2 = clientNodes.subList(2, 4);
+
+        List<ClusterNode> splittedPart1 = new ArrayList<>();
+        splittedPart1.addAll(srvPart1);
+        splittedPart1.addAll(clientPart1);
+
+        List<ClusterNode> splittedPart2 = new ArrayList<>();
+        splittedPart2.addAll(srvPart2);
+        splittedPart2.addAll(clientPart2);
+
+        ConnectionsFailureMatrix matrix = new ConnectionsFailureMatrix();
+
+        matrix.addAll(splittedPart1);
+        matrix.addAll(splittedPart2);
+
+        matrix.addConnection(srvPart1.get(0), srvPart2.get(1));
+        matrix.addConnection(srvPart2.get(1), srvPart1.get(0));
+
+        matrix.addConnection(srvPart1.get(1), srvPart2.get(2));
+        matrix.addConnection(srvPart2.get(2), srvPart1.get(1));
+
+        matrix.addConnection(srvPart1.get(2), srvPart2.get(0));
+        matrix.addConnection(srvPart2.get(0), srvPart1.get(2));
+
+        PeerToPeerCommunicationFailureSpi.fail(matrix);
+
+        waitForTopology(5);
+    }
+
+    /**
+     * Class represents available connections between cluster nodes.
+     * This is needed to simulate network problems in {@link PeerToPeerCommunicationFailureSpi}.
+     */
+    static class ConnectionsFailureMatrix {
+        /** Available connections per each node id. */
+        private Map<UUID, Set<UUID>> availableConnections = new HashMap<>();
+
+        /**
+         * @param from Cluster node 1.
+         * @param to Cluster node 2.
+         * @return {@code True} if there is connection between nodes {@code from} and {@code to}.
+         */
+        public boolean hasConnection(ClusterNode from, ClusterNode to) {
+            return availableConnections.getOrDefault(from.id(), Collections.emptySet()).contains(to.id());
+        }
+
+        /**
+         * Adds connection between nodes {@code from} and {@code to}.
+         * @param from Cluster node 1.
+         * @param to Cluster node 2.
+         */
+        public void addConnection(ClusterNode from, ClusterNode to) {
+            availableConnections.computeIfAbsent(from.id(), s -> new HashSet<>()).add(to.id());
+        }
+
+        /**
+         * Removes connection between nodes {@code from} and {@code to}.
+         * @param from Cluster node 1.
+         * @param to Cluster node 2.
+         */
+        public void removeConnection(ClusterNode from, ClusterNode to) {
+            availableConnections.getOrDefault(from.id(), Collections.emptySet()).remove(to.id());
+        }
+
+        /**
+         * Adds connections between all nodes presented in given {@code nodeSet}.
+         *
+         * @param nodeSet Set of the cluster nodes.
+         */
+        public void addAll(List<ClusterNode> nodeSet) {
+            for (int i = 0; i < nodeSet.size(); i++) {
+                for (int j = 0; j < nodeSet.size(); j++) {
+                    if (i == j)
+                        continue;
+
+                    addConnection(nodeSet.get(i), nodeSet.get(j));
+                }
+            }
+        }
+
+        /**
+         * Builds connections failure matrix from two part of the cluster nodes.
+         * Each part has all connections inside, but hasn't any connection to another part.
+         *
+         * @param part1 Part 1.
+         * @param part2 Part 2.
+         * @return Connections failure matrix.
+         */
+        static ConnectionsFailureMatrix buildFrom(List<ClusterNode> part1, List<ClusterNode> part2) {
+            ConnectionsFailureMatrix matrix = new ConnectionsFailureMatrix();
+            matrix.addAll(part1);
+            matrix.addAll(part2);
+            return matrix;
+        }
+    }
+
+    /**
+     * Communication SPI with possibility to simulate network problems between some of the cluster nodes.
+     */
+    static class PeerToPeerCommunicationFailureSpi extends TcpCommunicationSpi {
+        /** Flag indicates that connections according to {@code matrix} should be failed. */
+        private static volatile boolean failure;
+
+        /** Connections failure matrix. */
+        private static ConnectionsFailureMatrix matrix;
+
+        /**
+         * Start failing connections according to given matrix {@code with}.
+         * @param with Failure matrix.
+         */
+        public static void fail(ConnectionsFailureMatrix with) {
+            matrix = with;
+            failure = true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteFuture<BitSet> checkConnection(List<ClusterNode> nodes) {
+            // Creates connections statuses according to failure matrix.
+            BitSet bitSet = new BitSet();
+
+            ClusterNode localNode = getLocalNode();
+
+            int idx = 0;
+
+            for (ClusterNode remoteNode : nodes) {
+                if (localNode.id().equals(remoteNode.id()))
+                    bitSet.set(idx);
+                else {
+                    if (matrix.hasConnection(localNode, remoteNode))
+                        bitSet.set(idx);
+                }
+                idx++;
+            }
+
+            return new IgniteFinishedFutureImpl<>(bitSet);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected GridCommunicationClient createTcpClient(
+            ClusterNode node,
+            int connIdx
+        ) throws IgniteCheckedException {
+            if (failure && !matrix.hasConnection(getLocalNode(), node)) {
+                processClientCreationError(node, null, new IgniteCheckedException("Test", new SocketTimeoutException()));
+
+                return null;
+            }
+
+            return new FailingCommunicationClient(getLocalNode(), node,
+                super.createTcpClient(node, connIdx));
+        }
+
+        /**
+         * Communication client with possibility to simulate network error between peers.
+         */
+        class FailingCommunicationClient implements GridCommunicationClient {
+            /** Delegate. */
+            private final GridCommunicationClient delegate;
+
+            /** Local node which sends messages. */
+            private final ClusterNode localNode;
+
+            /** Remote node which receives messages. */
+            private final ClusterNode remoteNode;
+
+            FailingCommunicationClient(ClusterNode localNode, ClusterNode remoteNode, GridCommunicationClient delegate) {
+                this.delegate = delegate;
+                this.localNode = localNode;
+                this.remoteNode = remoteNode;
+            }
+
+            /** {@inheritDoc} */
+            @Override public void doHandshake(IgniteInClosure2X<InputStream, OutputStream> handshakeC) throws IgniteCheckedException {
+                if (failure && !matrix.hasConnection(localNode, remoteNode))
+                    throw new IgniteCheckedException("Test", new SocketTimeoutException());
+
+                delegate.doHandshake(handshakeC);
+            }
+
+            /** {@inheritDoc} */
+            @Override public boolean close() {
+                return delegate.close();
+            }
+
+            /** {@inheritDoc} */
+            @Override public void forceClose() {
+                delegate.forceClose();
+            }
+
+            /** {@inheritDoc} */
+            @Override public boolean closed() {
+                return delegate.closed();
+            }
+
+            /** {@inheritDoc} */
+            @Override public boolean reserve() {
+                return delegate.reserve();
+            }
+
+            /** {@inheritDoc} */
+            @Override public void release() {
+                delegate.release();
+            }
+
+            /** {@inheritDoc} */
+            @Override public long getIdleTime() {
+                return delegate.getIdleTime();
+            }
+
+            /** {@inheritDoc} */
+            @Override public void sendMessage(ByteBuffer data) throws IgniteCheckedException {
+                if (failure && !matrix.hasConnection(localNode, remoteNode))
+                    throw new IgniteCheckedException("Test", new SocketTimeoutException());
+
+                delegate.sendMessage(data);
+            }
+
+            /** {@inheritDoc} */
+            @Override public void sendMessage(byte[] data, int len) throws IgniteCheckedException {
+                if (failure && !matrix.hasConnection(localNode, remoteNode))
+                    throw new IgniteCheckedException("Test", new SocketTimeoutException());
+
+                delegate.sendMessage(data, len);
+            }
+
+            /** {@inheritDoc} */
+            @Override public boolean sendMessage(@Nullable UUID nodeId, Message msg, @Nullable IgniteInClosure<IgniteException> c) throws IgniteCheckedException {
+                // This will enforce SPI to create new client.
+                if (failure && !matrix.hasConnection(localNode, remoteNode))
+                    return true;
+
+                return delegate.sendMessage(nodeId, msg, c);
+            }
+
+            /** {@inheritDoc} */
+            @Override public boolean async() {
+                return delegate.async();
+            }
+
+            /** {@inheritDoc} */
+            @Override public int connectionIndex() {
+                return delegate.connectionIndex();
+            }
+        }
+    }
+
     /**
      * @param srvs Number of server nodes in test.
      * @throws Exception If failed.
@@ -3886,6 +4263,8 @@ private void reset() {
 
         err = false;
 
+        failCommSpi = false;
+
         evts.clear();
 
         try {

From 27ae1a18cbd15b7ff0d0a3f58c1499050a3262e6 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 10 May 2018 12:02:20 +0300
Subject: [PATCH 135/543] IGNITE-7912: control.sh script should show used
 WAL-segments. This closes #3636. This closes #3965.

(cherry picked from commit 07cbe22)
---
 .../apache/ignite/IgniteSystemProperties.java |   3 +
 .../internal/commandline/Arguments.java       |  80 +++--
 .../ignite/internal/commandline/Command.java  |   5 +-
 .../internal/commandline/CommandHandler.java  | 228 +++++++++++-
 .../wal/IgniteWriteAheadLogManager.java       |  10 +
 .../GridCacheDatabaseSharedManager.java       |  20 ++
 .../wal/FileWriteAheadLogManager.java         |  28 ++
 .../FsyncModeFileWriteAheadLogManager.java    |  30 ++
 .../internal/visor/misc/VisorClusterNode.java | 129 +++++++
 .../internal/visor/misc/VisorWalTask.java     | 336 ++++++++++++++++++
 .../internal/visor/misc/VisorWalTaskArg.java  |  98 +++++
 .../visor/misc/VisorWalTaskOperation.java     |  44 +++
 .../visor/misc/VisorWalTaskResult.java        | 113 ++++++
 .../resources/META-INF/classnames.properties  |   6 +
 .../CommandHandlerParsingTest.java            | 119 ++++++-
 .../db/IgnitePdsUnusedWalSegmentsTest.java    | 202 +++++++++++
 .../persistence/pagemem/NoOpWALManager.java   |   7 +-
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 .../testsuites/IgniteUtilSelfTestSuite.java   |   2 +
 .../ignite/util/GridCommandHandlerTest.java   |  72 ++++
 ...GridInternalTaskUnusedWalSegmentsTest.java | 153 ++++++++
 21 files changed, 1634 insertions(+), 54 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorClusterNode.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskOperation.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskResult.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 32fed05b369ab..008974c6ff68e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -277,6 +277,9 @@ public final class IgniteSystemProperties {
     /** System property to hold SSH host for visor-started nodes. */
     public static final String IGNITE_SSH_HOST = "IGNITE_SSH_HOST";
 
+    /** System property to enable experimental commands in control.sh script. */
+    public static final String IGNITE_ENABLE_EXPERIMENTAL_COMMAND = "IGNITE_ENABLE_EXPERIMENTAL_COMMAND";
+
     /** System property to hold SSH user name for visor-started nodes. */
     public static final String IGNITE_SSH_USER_NAME = "IGNITE_SSH_USER_NAME";
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index ce7269378fc20..0d4b38e83851b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -53,12 +53,6 @@ public class Arguments {
      */
     private String baselineArgs;
 
-    /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}.*/
-    private long pingTimeout;
-
-    /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}.*/
-    private long pingInterval;
-
     /** Transaction arguments. */
     private final VisorTxTaskArg txArg;
 
@@ -67,6 +61,22 @@ public class Arguments {
      */
     private CacheArguments cacheArgs;
 
+    /**
+     * Action for WAL command.
+     */
+    private String walAct;
+
+    /**
+     * Arguments for WAL command.
+     */
+    private String walArgs;
+
+    /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}.*/
+    private long pingTimeout;
+
+    /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}.*/
+    private long pingInterval;
+
     /**
      * @param cmd Command.
      * @param host Host.
@@ -76,14 +86,16 @@ public class Arguments {
      * @param baselineAct Baseline action.
      * @param baselineArgs Baseline args.
      * @param txArg TX arg.
-     * @param force Force flag.
+     * @param cacheArgs --cache subcommand arguments.
+     * @param walAct WAL action.
+     * @param walArgs WAL args.
      * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
      * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
-     * @param cacheArgs --cache subcommand arguments.
+     * @param force Force flag.
      */
     public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
-        String baselineArgs, long pingTimeout, long pingInterval, VisorTxTaskArg txArg, boolean force,
-        CacheArguments cacheArgs) {
+                     String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
+                     Long pingTimeout, Long pingInterval, boolean force) {
         this.cmd = cmd;
         this.host = host;
         this.port = port;
@@ -91,11 +103,13 @@ public Arguments(Command cmd, String host, String port, String user, String pwd,
         this.pwd = pwd;
         this.baselineAct = baselineAct;
         this.baselineArgs = baselineArgs;
+        this.txArg = txArg;
+        this.cacheArgs = cacheArgs;
+        this.walAct = walAct;
+        this.walArgs = walArgs;
         this.pingTimeout = pingTimeout;
         this.pingInterval = pingInterval;
         this.force = force;
-        this.txArg = txArg;
-        this.cacheArgs = cacheArgs;
     }
 
     /**
@@ -147,6 +161,34 @@ public String baselineArguments() {
         return baselineArgs;
     }
 
+    /**
+     * @return Transaction arguments.
+     */
+    public VisorTxTaskArg transactionArguments() {
+        return txArg;
+    }
+
+    /**
+     * @return Arguments for --cache subcommand.
+     */
+    public CacheArguments cacheArgs() {
+        return cacheArgs;
+    }
+
+    /**
+     * @return WAL action.
+     */
+    public String walAction() {
+        return walAct;
+    }
+
+    /**
+     * @return WAL arguments.
+     */
+    public String walArguments() {
+        return walArgs;
+    }
+
     /**
      * See {@link GridClientConfiguration#pingTimeout}.
      *
@@ -165,24 +207,10 @@ public long pingInterval() {
         return pingInterval;
     }
 
-    /**
-     * @return Transaction arguments.
-     */
-    public VisorTxTaskArg transactionArguments() {
-        return txArg;
-    }
-
     /**
      * @return Force option.
      */
     public boolean force() {
         return force;
     }
-
-    /**
-     * @return Arguments for --cache subcommand.
-     */
-    public CacheArguments cacheArgs() {
-        return cacheArgs;
-    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
index 52098d629fb0d..c64e488db4ffd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Command.java
@@ -37,7 +37,10 @@ public enum Command {
     TX("--tx"),
 
     /** */
-    CACHE("--cache");
+    CACHE("--cache"),
+
+    /** */
+    WAL("--wal");
 
     /** */
     private final String text;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index ff05684bb47a8..7d457fd1f1265 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -32,6 +32,7 @@
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 import java.util.stream.Collectors;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.client.GridClient;
 import org.apache.ignite.internal.client.GridClientAuthenticationException;
@@ -61,6 +62,11 @@
 import org.apache.ignite.internal.visor.baseline.VisorBaselineTask;
 import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskArg;
 import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskResult;
+import org.apache.ignite.internal.visor.misc.VisorClusterNode;
+import org.apache.ignite.internal.visor.misc.VisorWalTask;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskArg;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskOperation;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskResult;
 import org.apache.ignite.internal.visor.tx.VisorTxInfo;
 import org.apache.ignite.internal.visor.tx.VisorTxOperation;
 import org.apache.ignite.internal.visor.tx.VisorTxProjection;
@@ -86,6 +92,7 @@
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
 import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR;
 import static org.apache.ignite.internal.IgniteVersionUtils.COPYRIGHT;
 import static org.apache.ignite.internal.commandline.Command.ACTIVATE;
@@ -93,6 +100,7 @@
 import static org.apache.ignite.internal.commandline.Command.CACHE;
 import static org.apache.ignite.internal.commandline.Command.DEACTIVATE;
 import static org.apache.ignite.internal.commandline.Command.STATE;
+import static org.apache.ignite.internal.commandline.Command.WAL;
 import static org.apache.ignite.internal.commandline.Command.TX;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.ADD;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.COLLECT;
@@ -133,6 +141,12 @@ public class CommandHandler {
     /** Force option is used for auto confirmation. */
     private static final String CMD_FORCE = "--force";
 
+    /** */
+    protected static final String CMD_PING_INTERVAL = "--ping-interval";
+
+    /** */
+    protected static final String CMD_PING_TIMEOUT = "--ping-timeout";
+
     /** List of optional auxiliary commands. */
     private static final Set<String> AUX_COMMANDS = new HashSet<>();
     static {
@@ -142,17 +156,13 @@ public class CommandHandler {
         AUX_COMMANDS.add(CMD_PASSWORD);
         AUX_COMMANDS.add(CMD_USER);
         AUX_COMMANDS.add(CMD_FORCE);
+        AUX_COMMANDS.add(CMD_PING_INTERVAL);
+        AUX_COMMANDS.add(CMD_PING_TIMEOUT);
     }
 
     /** Broadcast uuid. */
     private static final UUID BROADCAST_UUID = UUID.randomUUID();
 
-    /** */
-    protected static final String CMD_PING_INTERVAL = "--ping-interval";
-
-    /** */
-    protected static final String CMD_PING_TIMEOUT = "--ping-timeout";
-
     /** */
     public static final String CONFIRM_MSG = "y";
 
@@ -172,7 +182,13 @@ public class CommandHandler {
     private static final String BASELINE_SET_VERSION = "version";
 
     /** */
-    private static final String DELIM = "--------------------------------------------------------------------------------";
+    static final String WAL_PRINT = "print";
+
+    /** */
+    static final String WAL_DELETE = "delete";
+
+    /** */
+    static final String DELIM = "--------------------------------------------------------------------------------";
 
     /** */
     public static final int EXIT_CODE_OK = 0;
@@ -240,6 +256,9 @@ public class CommandHandler {
     /** */
     private Object lastOperationRes;
 
+    /** Check if experimental commands are enabled. Default {@code false}. */
+    private final boolean enableExperimental = IgniteSystemProperties.getBoolean(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, false);
+
     /**
      * Output specified string to console.
      *
@@ -332,6 +351,12 @@ private String confirmationPrompt(Arguments args) {
 
                 break;
 
+            case WAL:
+                if (WAL_DELETE.equals(args.walAction()))
+                    str = "Warning: the command will delete unused WAL segments.";
+
+                break;
+
             case TX:
                 if (args.transactionArguments().getOperation() == VisorTxOperation.KILL)
                     str = "Warning: the command will kill some transactions.";
@@ -941,6 +966,147 @@ else if (arg.getOperation() == VisorTxOperation.KILL)
         }
     }
 
+    /**
+     * Execute WAL command.
+     *
+     * @param client Client.
+     * @param walAct WAL action to execute.
+     * @param walArgs WAL args.
+     * @throws Throwable If failed to execute wal action.
+     */
+    private void wal(GridClient client, String walAct, String walArgs) throws Throwable {
+        switch (walAct){
+            case WAL_DELETE:
+                deleteUnusedWalSegments(client, walArgs);
+
+                break;
+
+            case WAL_PRINT:
+            default:
+                printUnusedWalSegments(client, walArgs);
+
+                break;
+        }
+    }
+
+    /**
+     * Execute delete unused WAL segments task.
+     *
+     * @param client Client.
+     * @param walArgs WAL args.
+     */
+    private void deleteUnusedWalSegments(GridClient client, String walArgs) throws Throwable {
+        VisorWalTaskResult res = executeTask(client, VisorWalTask.class,
+                walArg(VisorWalTaskOperation.DELETE_UNUSED_WAL_SEGMENTS, walArgs));
+        printDeleteWalSegments0(res);
+    }
+
+    /**
+     * Execute print unused WAL segments task.
+     *
+     * @param client Client.
+     * @param walArgs Wal args.
+     */
+    private void printUnusedWalSegments(GridClient client, String walArgs) throws Throwable {
+        VisorWalTaskResult res = executeTask(client, VisorWalTask.class,
+                walArg(VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS, walArgs));
+        printUnusedWalSegments0(res);
+    }
+
+    /**
+     * Prepare WAL task argument.
+     *
+     * @param op Operation.
+     * @param s Argument from command line.
+     * @return Task argument.
+     */
+    private VisorWalTaskArg walArg(VisorWalTaskOperation op, String s){
+        List<String> consistentIds = null;
+
+        if (!F.isEmpty(s)) {
+            consistentIds = new ArrayList<>();
+
+            for (String consistentId : s.split(","))
+                consistentIds.add(consistentId.trim());
+        }
+
+        switch (op) {
+            case DELETE_UNUSED_WAL_SEGMENTS:
+            case PRINT_UNUSED_WAL_SEGMENTS:
+                return new VisorWalTaskArg(op, consistentIds);
+
+            default:
+                return new VisorWalTaskArg(VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS, consistentIds);
+        }
+
+    }
+
+    /**
+     * Print list of unused wal segments.
+     *
+     * @param taskRes Task result with baseline topology.
+     */
+    private void printUnusedWalSegments0(VisorWalTaskResult taskRes) {
+        log("Unused wal segments per node:");
+        nl();
+
+        Map<String, Collection<String>> res = taskRes.results();
+        Map<String, Exception> failRes = taskRes.exceptions();
+        Map<String, VisorClusterNode> nodesInfo = taskRes.getNodesInfo();
+
+        for(Map.Entry<String, Collection<String>> entry: res.entrySet()) {
+            VisorClusterNode node = nodesInfo.get(entry.getKey());
+
+            log("Node=" + node.getConsistentId());
+            log("     addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+
+            for(String fileName: entry.getValue())
+                log("   " + fileName);
+
+            nl();
+        }
+
+        for(Map.Entry<String, Exception> entry: failRes.entrySet()) {
+            VisorClusterNode node = nodesInfo.get(entry.getKey());
+
+            log("Node=" + node.getConsistentId());
+            log("     addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+            log("   failed with error: " + entry.getValue().getMessage());
+            nl();
+        }
+    }
+
+    /**
+     * Print list of unused wal segments.
+     *
+     * @param taskRes Task result with baseline topology.
+     */
+    private void printDeleteWalSegments0(VisorWalTaskResult taskRes) {
+        log("WAL segments deleted for nodes:");
+        nl();
+
+        Map<String, Collection<String>> res = taskRes.results();
+        Map<String, Exception> errors = taskRes.exceptions();
+        Map<String, VisorClusterNode> nodesInfo = taskRes.getNodesInfo();
+
+        for(Map.Entry<String, Collection<String>> entry: res.entrySet()) {
+            VisorClusterNode node = nodesInfo.get(entry.getKey());
+
+            log("Node=" + node.getConsistentId());
+            log("     addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+            nl();
+        }
+
+        for(Map.Entry<String, Exception> entry: errors.entrySet()) {
+            VisorClusterNode node = nodesInfo.get(entry.getKey());
+
+            log("Node=" + node.getConsistentId());
+            log("     addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+            log("   failed with error: " + entry.getValue().getMessage());
+            nl();
+        }
+    }
+
     /**
      * @param e Exception to check.
      * @return {@code true} if specified exception is {@link GridClientAuthenticationException}.
@@ -1031,6 +1197,10 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         Long pingTimeout = DFLT_PING_TIMEOUT;
 
+        String walAct = "";
+
+        String walArgs = "";
+
         boolean force = false;
 
         CacheArguments cacheArgs = null;
@@ -1052,6 +1222,7 @@ Arguments parseAndValidate(List<String> rawArgs) {
                     case DEACTIVATE:
                     case STATE:
                         commands.add(cmd);
+
                         break;
 
                     case TX:
@@ -1088,6 +1259,24 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
+                    case WAL:
+                        if (!enableExperimental)
+                            throw new IllegalArgumentException("Experimental command is disabled.");
+
+                        commands.add(WAL);
+
+                        str = nextArg("Expected arguments for " + WAL.text());
+
+                        walAct = str.toLowerCase();
+
+                        if (WAL_PRINT.equals(walAct) || WAL_DELETE.equals(walAct))
+                            walArgs = (str = peekNextArg()) != null && !isCommandOrOption(str)
+                                    ? nextArg("Unexpected argument for " + WAL.text() + ": " + walAct)
+                                    : "";
+                        else
+                            throw new IllegalArgumentException("Unexpected action " + walAct + " for " + WAL.text());
+
+                        break;
                     default:
                         throw new IllegalArgumentException("Unexpected command: " + str);
                 }
@@ -1161,8 +1350,8 @@ Arguments parseAndValidate(List<String> rawArgs) {
         if (hasUsr != hasPwd)
             throw new IllegalArgumentException("Both user and password should be specified");
 
-        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs,
-            pingTimeout, pingInterval, txArgs, force, cacheArgs);
+        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
+                pingTimeout, pingInterval, force);
     }
 
     /**
@@ -1429,6 +1618,15 @@ private long nextLongArg(String lb) {
         }
     }
 
+    /**
+     *  Check if raw arg is command or option.
+     *
+     *  @return {@code true} If raw arg is command, overwise {@code false}.
+     */
+    private boolean isCommandOrOption(String raw) {
+        return raw != null && raw.contains("--");
+    }
+
     /**
      * Parse and execute command.
      *
@@ -1457,6 +1655,13 @@ public int execute(List<String> rawArgs) {
                     "[minSize SIZE] [label PATTERN_REGEX] [servers|clients] " +
                     "[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE] [kill] [--force]");
 
+                if(enableExperimental) {
+                    usage("  Print absolute paths of unused archived wal segments on each node:", WAL,
+                            " print [consistentId1,consistentId2,....,consistentIdN]");
+                    usage("  Delete unused archived wal segments on each node:", WAL,
+                            " delete [consistentId1,consistentId2,....,consistentIdN] [--force]");
+                }
+
                 log("The utility has --cache subcommand to view and control state of caches in cluster.");
                 log("  More info:    control.sh --cache help");
                 nl();
@@ -1533,6 +1738,11 @@ public int execute(List<String> rawArgs) {
                     case CACHE:
                         cache(client, args.cacheArgs());
 
+                        break;
+
+                    case WAL:
+                        wal(client, args.walAction(), args.walArguments());
+
                         break;
                 }
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 8e8a1b3a43e9b..b5c22c92341c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -123,11 +123,21 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
 
     /**
      * Checks if WAL segment is under lock or reserved
+     *
      * @param ptr Pointer to check.
      * @return True if given pointer is located in reserved segment.
      */
     public boolean reserved(WALPointer ptr);
 
+    /**
+     * Checks if WAL segments is under lock or reserved.
+     *
+     * @param low Pointer since which WAL is locked or reserved. If {@code null}, checks from the oldest segment.
+     * @param high Pointer for which WAL is locked or reserved.
+     * @return Number of reserved WAL segments.
+     */
+    public int reserved(WALPointer low, WALPointer high);
+
     /**
      * Checks WAL disabled for cache group.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index d847132611e5f..12cfc05d72e2e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -3721,6 +3721,26 @@ private CheckpointEntry entry(Long cpTs) throws IgniteCheckedException {
             return entry;
         }
 
+        /**
+         * @return First checkpoint entry if exists. Otherwise {@code null}.
+         */
+        private CheckpointEntry firstEntry() {
+            Map.Entry<Long,CheckpointEntry> entry = histMap.firstEntry();
+
+            return entry != null ? entry.getValue() : null;
+        }
+
+        /**
+         * Get WAL pointer to low checkpoint bound.
+         *
+         * @return WAL pointer to low checkpoint bound.
+         */
+        public WALPointer lowCheckpointBound() {
+            CheckpointEntry entry = firstEntry();
+
+            return entry != null ? entry.cpMark : null;
+        }
+
         /**
          * @return Collection of checkpoint timestamps.
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 8755e1bf31f4b..e69cc8040f7e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -946,6 +946,34 @@ private boolean segmentReservedOrLocked(long absIdx) {
         return segmentReservedOrLocked(fPtr.index());
     }
 
+    /** {@inheritDoc} */
+    @Override public int reserved(WALPointer low, WALPointer high) {
+        // It is not clear now how to get the highest WAL pointer. So when high is null method returns 0.
+        if (high == null)
+            return 0;
+
+        assert high instanceof FileWALPointer : high;
+
+        assert low == null || low instanceof FileWALPointer : low;
+
+        FileWALPointer lowPtr = (FileWALPointer)low;
+
+        FileWALPointer highPtr = (FileWALPointer)high;
+
+        long lowIdx = lowPtr != null ? lowPtr.index() : 0;
+
+        long highIdx = highPtr.index();
+
+        while (lowIdx < highIdx) {
+            if (segmentReservedOrLocked(lowIdx))
+                break;
+
+            lowIdx++;
+        }
+
+        return (int)(highIdx - lowIdx + 1);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean disabled(int grpId) {
         CacheGroupContext ctx = cctx.cache().cacheGroup(grpId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index c446f7f228649..7bb2ce4dfc48c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -848,6 +848,36 @@ private boolean hasIndex(long absIdx) {
         return archiver0 != null && archiver0.reserved(fPtr.index());
     }
 
+    /** {@inheritDoc} */
+    @Override public int reserved(WALPointer low, WALPointer high) {
+        // It is not clear now how to get the highest WAL pointer. So when high is null method returns 0.
+        if (high == null)
+            return 0;
+
+        assert high instanceof FileWALPointer : high;
+
+        assert low == null || low instanceof FileWALPointer : low;
+
+        FileWALPointer lowPtr = (FileWALPointer)low;
+
+        FileWALPointer highPtr = (FileWALPointer)high;
+
+        FileArchiver archiver0 = archiver;
+
+        long lowIdx = lowPtr != null ? lowPtr.index() : 0;
+
+        long highIdx = highPtr.index();
+
+        while (lowIdx < highIdx) {
+            if(archiver0 != null && archiver0.reserved(lowIdx))
+                break;
+
+            lowIdx++;
+        }
+
+        return (int)(highIdx - lowIdx + 1);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean disabled(int grpId) {
         CacheGroupContext ctx = cctx.cache().cacheGroup(grpId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorClusterNode.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorClusterNode.java
new file mode 100644
index 0000000000000..5706444e086dd
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorClusterNode.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.misc;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Map;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *  Data transfer object for {@link ClusterNode}.
+ */
+public class VisorClusterNode extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Cluster node consistent id. */
+    @GridToStringInclude
+    private String consistentId;
+
+    /** Cluster node attributes. */
+    private Map<String, Object> attrs;
+
+    /** Cluster node addresses. */
+    @GridToStringInclude
+    private Collection<String> addrs;
+
+    /** Cluster node host names. */
+    @GridToStringInclude
+    private Collection<String> hostNames;
+
+    /**
+     * Default constructor.
+     */
+    public VisorClusterNode() {
+        // No-op.
+    }
+
+    /**
+     * Create data transfer object for baseline node.
+     *
+     * @param node Baseline node.
+     */
+    public VisorClusterNode(ClusterNode node) {
+        consistentId = String.valueOf(node.consistentId());
+        addrs = node.addresses();
+        hostNames = node.hostNames();
+        attrs = node.attributes();
+    }
+
+    /**
+     * Get cluster node consistent id.
+     *
+     * @return Cluster node consistent id.
+     */
+    public String getConsistentId() {
+        return consistentId;
+    }
+
+    /**
+     * Get cluster node attributes.
+     *
+     * @return Cluster node attributes.
+     */
+    public Map<String, Object> getAttributes() {
+        return attrs;
+    }
+
+    /**
+     * Get cluster node addresses.
+     *
+     * @return Node addresses.
+     */
+    public Collection<String> getAddresses() {
+        return addrs;
+    }
+
+    /**
+     * Get cluster node host names.
+     *
+     * @return Node host names.
+     */
+    public Collection<String> getHostNames() {
+        return hostNames;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeString(out, consistentId);
+        U.writeMap(out, attrs);
+        U.writeCollection(out, hostNames);
+        U.writeCollection(out, addrs);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        consistentId = U.readString(in);
+        attrs = U.readMap(in);
+        hostNames = U.readCollection(in);
+        addrs = U.readCollection(in);
+    }
+
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorClusterNode.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java
new file mode 100644
index 0000000000000..7edcea9d459b2
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.misc;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.regex.Pattern;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Performs WAL cleanup clusterwide.
+ */
+@GridInternal
+public class VisorWalTask extends VisorMultiNodeTask<VisorWalTaskArg, VisorWalTaskResult, Collection<String>> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Pattern for segment file names. */
+    private static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal");
+
+    /** Pattern for compacted segment file names. */
+    private static final Pattern WAL_SEGMENT_FILE_COMPACTED_PATTERN = Pattern.compile("\\d{16}\\.wal\\.zip");
+
+    /** WAL archive file filter. */
+    private static final FileFilter WAL_ARCHIVE_FILE_FILTER = new FileFilter() {
+        @Override public boolean accept(File file) {
+            return !file.isDirectory() && (WAL_NAME_PATTERN.matcher(file.getName()).matches() ||
+                    WAL_SEGMENT_FILE_COMPACTED_PATTERN.matcher(file.getName()).matches());
+        }
+    };
+
+    /** {@inheritDoc} */
+    @Override protected VisorWalJob job(VisorWalTaskArg arg) {
+        return new VisorWalJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Collection<UUID> jobNodes(VisorTaskArgument<VisorWalTaskArg> arg) {
+        Collection<ClusterNode> srvNodes = ignite.cluster().forServers().nodes();
+        Collection<UUID> ret = new ArrayList<>(srvNodes.size());
+
+        VisorWalTaskArg taskArg = arg.getArgument();
+
+        Set<String> nodeIds = taskArg.getConsistentIds() != null ? new HashSet<>(arg.getArgument().getConsistentIds())
+                                : null;
+
+        if (nodeIds == null) {
+            for (ClusterNode node : srvNodes)
+                ret.add(node.id());
+        }
+        else {
+            for (ClusterNode node : srvNodes) {
+                if (nodeIds.contains(node.consistentId().toString()))
+                    ret.add(node.id());
+            }
+        }
+
+        return ret;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected VisorWalTaskResult reduce0(List<ComputeJobResult> results) throws IgniteException {
+        Map<String, Exception> exRes = U.newHashMap(0);
+        Map<String, Collection<String>> res = U.newHashMap(results.size());
+        Map<String, VisorClusterNode> nodesInfo = U.newHashMap(results.size());
+
+        for (ComputeJobResult result: results){
+            ClusterNode node = result.getNode();
+
+            String nodeId = node.consistentId().toString();
+
+            if(result.getException() != null)
+                exRes.put(nodeId, result.getException());
+            else if (result.getData() != null) {
+                Collection<String> data = result.getData();
+
+                if(data != null)
+                    res.put(nodeId, data);
+            }
+
+            nodesInfo.put(nodeId, new VisorClusterNode(node));
+        }
+
+        return new VisorWalTaskResult(res, exRes, nodesInfo);
+    }
+
+    /**
+     * Performs WAL cleanup per node.
+     */
+    private static class VisorWalJob extends VisorJob<VisorWalTaskArg, Collection<String>> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** Auto injected logger */
+        @LoggerResource
+        private transient IgniteLogger log;
+
+        /**
+         *  @param arg WAL task argument.
+         *  @param debug Debug flag.
+         */
+        public VisorWalJob(VisorWalTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override protected Collection<String> run(@Nullable VisorWalTaskArg arg) throws IgniteException {
+            try {
+                GridKernalContext cctx = ignite.context();
+
+                GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)cctx.cache().context().database();
+                FileWriteAheadLogManager wal = (FileWriteAheadLogManager)cctx.cache().context().wal();
+
+                if (dbMgr == null || arg == null || wal == null)
+                    return null;
+
+                switch (arg.getOperation()) {
+                    case DELETE_UNUSED_WAL_SEGMENTS:
+                        return deleteUnusedWalSegments(dbMgr, wal);
+
+                    case PRINT_UNUSED_WAL_SEGMENTS:
+                    default:
+                        return getUnusedWalSegments(dbMgr, wal);
+
+                }
+            }
+            catch (IgniteCheckedException e){
+                U.error(log, "Failed to perform WAL task", e);
+
+                throw new IgniteException("Failed to perform WAL task", e);
+            }
+        }
+
+        /**
+         * Get unused wal segments.
+         *
+         * @param  wal Database manager.
+         * @return {@link Collection<String>} of absolute paths of unused WAL segments.
+         * @throws IgniteCheckedException if failed.
+         */
+        Collection<String> getUnusedWalSegments(
+            GridCacheDatabaseSharedManager dbMgr,
+            FileWriteAheadLogManager wal
+        ) throws IgniteCheckedException{
+            WALPointer lowBoundForTruncate = dbMgr.checkpointHistory().lowCheckpointBound();
+
+            if (lowBoundForTruncate == null)
+                return Collections.emptyList();
+
+            int maxIdx = resolveMaxReservedIndex(wal, lowBoundForTruncate);
+
+            File[] walFiles = getWalArchiveDir().listFiles(WAL_ARCHIVE_FILE_FILTER);
+
+            Collection<String> res = new ArrayList<>(walFiles != null && walFiles.length > 0 ? walFiles.length - 1 : 0);
+
+            if(walFiles != null && walFiles.length > 0) {
+                sortWalFiles(walFiles);
+
+                // Obtain index of last archived WAL segment, it will not be deleted.
+                long lastArchIdx = getIndex(walFiles[walFiles.length - 1]);
+
+                for (File f : walFiles) {
+                    long fileIdx = getIndex(f);
+
+                    if (fileIdx < maxIdx && fileIdx < lastArchIdx)
+                        res.add(f.getAbsolutePath());
+                    else
+                        break;
+                }
+            }
+
+            return res;
+        }
+
+        /**
+         * Delete unused wal segments.
+         *
+         * @param dbMgr Database manager.
+         * @return {@link Collection<String>} of deleted WAL segment's files.
+         * @throws IgniteCheckedException if failed.
+         */
+        Collection<String> deleteUnusedWalSegments(
+            GridCacheDatabaseSharedManager dbMgr,
+            FileWriteAheadLogManager wal
+        ) throws IgniteCheckedException {
+            WALPointer lowBoundForTruncate = dbMgr.checkpointHistory().lowCheckpointBound();
+
+            if (lowBoundForTruncate == null)
+                return Collections.emptyList();
+
+            int maxIdx = resolveMaxReservedIndex(wal, lowBoundForTruncate);
+
+            File[] walFiles = getWalArchiveDir().listFiles(WAL_ARCHIVE_FILE_FILTER);
+
+            dbMgr.onWalTruncated(lowBoundForTruncate);
+
+            int num = wal.truncate(null, lowBoundForTruncate);
+
+            if (walFiles != null) {
+                sortWalFiles(walFiles);
+
+                Collection<String> res = new ArrayList<>(num);
+
+                for (File walFile: walFiles) {
+                    if (getIndex(walFile) < maxIdx && num > 0)
+                        res.add(walFile.getAbsolutePath());
+                    else
+                        break;
+
+                    num--;
+                }
+
+                return res;
+            }
+            else
+                return Collections.emptyList();
+
+        }
+
+        /**
+         *
+         */
+        private int resolveMaxReservedIndex(FileWriteAheadLogManager wal, WALPointer lowBoundForTruncate) {
+            FileWALPointer low = (FileWALPointer)lowBoundForTruncate;
+
+            int resCnt = wal.reserved(null, lowBoundForTruncate);
+
+            long highIdx = low.index();
+
+            return (int)(highIdx - resCnt + 1);
+        }
+
+        /**
+         * Get WAL archive directory from configuration.
+         *
+         * @return WAL archive directory.
+         * @throws IgniteCheckedException if failed.
+         */
+        private File getWalArchiveDir() throws IgniteCheckedException {
+            IgniteConfiguration igCfg = ignite.context().config();
+
+            DataStorageConfiguration dsCfg = igCfg.getDataStorageConfiguration();
+
+            PdsFolderSettings resFldrs = ignite.context().pdsFolderResolver().resolveFolders();
+
+            String consId = resFldrs.folderName();
+
+            File dir;
+
+            if (dsCfg.getWalArchivePath() != null) {
+                File workDir0 = new File(dsCfg.getWalArchivePath());
+
+                dir = workDir0.isAbsolute() ?
+                        new File(workDir0, consId) :
+                        new File(U.resolveWorkDirectory(igCfg.getWorkDirectory(), dsCfg.getWalArchivePath(), false),
+                                consId);
+            }
+            else
+                dir = new File(U.resolveWorkDirectory(igCfg.getWorkDirectory(),
+                        DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false), consId);
+
+            if (!dir.exists())
+                throw new IgniteCheckedException("WAL archive directory does not exists" + dir.getAbsolutePath());
+
+            return dir;
+        }
+
+
+        /**
+         * Sort WAL files according their indices.
+         *
+         * @param files Array of WAL segment files.
+         */
+        private void sortWalFiles(File[] files) {
+            Arrays.sort(files, new Comparator<File>() {
+                @Override public int compare(File o1, File o2) {
+                    return Long.compare(getIndex(o1), getIndex(o2));
+                }
+            });
+        }
+    }
+
+    /**
+     * Get index from WAL segment file.
+     *
+     * @param file WAL segment file.
+     * @return Index of WAL segment file.
+     */
+    private static long getIndex(File file) {
+        return Long.parseLong(file.getName().substring(0, 16));
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskArg.java
new file mode 100644
index 0000000000000..afd3b5836fc70
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskArg.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.misc;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Argument for {@link VisorWalTask}.
+ */
+public class VisorWalTaskArg extends VisorDataTransferObject{
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** WAL task operation. */
+    private VisorWalTaskOperation op;
+
+    /** List of nodes' consistent ids. */
+    private List<String> consistentIds;
+
+    /**
+     * Default constructor.
+     */
+    public VisorWalTaskArg() {
+        // No-op.
+    }
+
+    /**
+     * @param op Task operation.
+     */
+    public VisorWalTaskArg(VisorWalTaskOperation op) {
+        this.op = op;
+    }
+
+    /**
+     * @param op WAL task operation.
+     * @param consistentIds Nodes consistent ids.
+     */
+    public VisorWalTaskArg(VisorWalTaskOperation op, List<String> consistentIds) {
+        this.op = op;
+        this.consistentIds = consistentIds;
+    }
+
+    /**
+     *  Get WAL task operation.
+     *
+     *  @return WAL task operation.
+     */
+    public VisorWalTaskOperation getOperation() {
+        return op == null ? VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS : op;
+    }
+
+    /**
+     * Get nodes consistent ids.
+     *
+     * @return Consistent ids.
+     */
+    public List<String> getConsistentIds() {
+        return consistentIds;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeEnum(out, op);
+        U.writeCollection(out, consistentIds);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        op = VisorWalTaskOperation.fromOrdinal(in.readByte());
+        consistentIds = U.readList(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorWalTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskOperation.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskOperation.java
new file mode 100644
index 0000000000000..be2ff18456840
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskOperation.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.misc;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *  WAL Task operation types.
+ */
+public enum VisorWalTaskOperation {
+    /** Print unused wal segments. */
+    PRINT_UNUSED_WAL_SEGMENTS,
+
+    /** Delete unused wal segments. */
+    DELETE_UNUSED_WAL_SEGMENTS;
+
+    /** Enumerated values. */
+    private static final VisorWalTaskOperation[] VALS = values();
+
+    /**
+     * Efficiently gets enumerated value from its ordinal.
+     *
+     * @param ord Ordinal value.
+     * @return Enumerated value or {@code null} if ordinal out of range.
+     */
+    @Nullable public static VisorWalTaskOperation fromOrdinal(int ord) {
+        return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskResult.java
new file mode 100644
index 0000000000000..104c2f3059508
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTaskResult.java
@@ -0,0 +1,113 @@
+/*
+ *  * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  * contributor license agreements.  See the NOTICE file distributed with
+ *  * this work for additional information regarding copyright ownership.
+ *  * The ASF licenses this file to You under the Apache License, Version 2.0
+ *  * (the "License"); you may not use this file except in compliance with
+ *  * the License.  You may obtain a copy of the License at
+ *  *
+ *  *      http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.misc;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Map;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *  Result of {@link VisorWalTask}.
+ */
+public class VisorWalTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Exceptions by node consistent id. */
+    @GridToStringInclude
+    private Map<String, Exception> exceptions;
+
+    /** Archived wal segments path search results by node consistent id. */
+    @GridToStringInclude
+    private Map<String, Collection<String>> results;
+
+    /** Nodes info by node consistent id. */
+    @GridToStringInclude
+    private Map<String, VisorClusterNode> nodesInfo;
+
+    /**
+     * Default constructor.
+     */
+    public VisorWalTaskResult() {
+        // No-op.
+    }
+
+    /**
+     * Create {@link VisorWalTask } result with given parameters.
+     *
+     * @param results List of log search results.
+     * @param exceptions List of exceptions by node id.
+     * @param nodesInfo Nodes info.
+     */
+    public VisorWalTaskResult(Map<String, Collection<String>> results, Map<String, Exception> exceptions,
+                              Map<String, VisorClusterNode> nodesInfo) {
+        this.exceptions = exceptions;
+        this.results = results;
+        this.nodesInfo = nodesInfo;
+    }
+
+    /**
+     *  Get occurred errors by node consistent id.
+     *
+     * @return Exceptions by node consistent id.
+     */
+    public Map<String, Exception> exceptions() {
+        return exceptions;
+    }
+
+    /**
+     * @return List of archived wal segments path search results by node consistent id.
+     */
+    public Map<String, Collection<String>> results() {
+        return results;
+    }
+
+    /**
+     * Get nodes info by node consistent id.
+     *
+     * @return Nodes info.
+     */
+    public Map<String, VisorClusterNode> getNodesInfo() {
+        return nodesInfo;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, exceptions);
+        U.writeMap(out, results);
+        U.writeMap(out, nodesInfo);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        exceptions = U.readMap(in);
+        results = U.readMap(in);
+        nodesInfo = U.readMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorWalTaskResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 8ca47c8430830..f1aec976e4c53 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -2028,6 +2028,12 @@ org.apache.ignite.internal.visor.misc.VisorNopTask
 org.apache.ignite.internal.visor.misc.VisorNopTask$VisorNopJob
 org.apache.ignite.internal.visor.misc.VisorResolveHostNameTask
 org.apache.ignite.internal.visor.misc.VisorResolveHostNameTask$VisorResolveHostNameJob
+org.apache.ignite.internal.visor.misc.VisorClusterNode
+org.apache.ignite.internal.visor.misc.VisorWalTask
+org.apache.ignite.internal.visor.misc.VisorWalTask$VisorWalJob
+org.apache.ignite.internal.visor.misc.VisorWalTaskArg
+org.apache.ignite.internal.visor.misc.VisorWalTaskOperation
+org.apache.ignite.internal.visor.misc.VisorWalTaskResult
 org.apache.ignite.internal.visor.node.VisorAffinityTopologyVersion
 org.apache.ignite.internal.visor.node.VisorAtomicConfiguration
 org.apache.ignite.internal.visor.node.VisorBasicConfiguration
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 98b8e01c99ea3..2fc40ca167944 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -1,46 +1,91 @@
 /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
  *
- *  * Licensed to the Apache Software Foundation (ASF) under one or more
- *  * contributor license agreements.  See the NOTICE file distributed with
- *  * this work for additional information regarding copyright ownership.
- *  * The ASF licenses this file to You under the Apache License, Version 2.0
- *  * (the "License"); you may not use this file except in compliance with
- *  * the License.  You may obtain a copy of the License at
- *  *
- *  *      http://www.apache.org/licenses/LICENSE-2.0
- *  *
- *  * Unless required by applicable law or agreed to in writing, software
- *  * distributed under the License is distributed on an "AS IS" BASIS,
- *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  * See the License for the specific language governing permissions and
- *  * limitations under the License.
+ *      http://www.apache.org/licenses/LICENSE-2.0
  *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 
 package org.apache.ignite.internal.commandline;
 
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.UUID;
 import junit.framework.TestCase;
 import org.apache.ignite.internal.visor.tx.VisorTxProjection;
 import org.apache.ignite.internal.visor.tx.VisorTxSortOrder;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
 
 import static java.util.Arrays.asList;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
+import static org.apache.ignite.internal.commandline.Command.WAL;
 import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_HOST;
 import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_PORT;
+import static org.apache.ignite.internal.commandline.CommandHandler.WAL_DELETE;
+import static org.apache.ignite.internal.commandline.CommandHandler.WAL_PRINT;
 
 /**
  * Tests Command Handler parsing arguments.
  */
 public class CommandHandlerParsingTest extends TestCase {
+    /** {@inheritDoc} */
+    @Override protected void setUp() throws Exception {
+        System.setProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, "true");
+
+        super.setUp();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void tearDown() throws Exception {
+        System.clearProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND);
+
+        super.tearDown();
+    }
+
+    /**
+     * Test that experimental command (i.e. WAL command) is disabled by default.
+     */
+    public void testExperimentalCommandIsDisabled() {
+        System.clearProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND);
+
+        CommandHandler hnd = new CommandHandler();
+
+        try {
+            hnd.parseAndValidate(Arrays.asList(WAL.text(), WAL_PRINT));
+        }
+        catch (Throwable e) {
+            e.printStackTrace();
+
+            assertTrue(e instanceof IllegalArgumentException);
+        }
+
+        try {
+            hnd.parseAndValidate(Arrays.asList(WAL.text(), WAL_DELETE));
+        }
+        catch (Throwable e) {
+            e.printStackTrace();
+
+            assertTrue(e instanceof IllegalArgumentException);
+        }
+    }
+
     /**
-     * Test parsing and validation for user and password arguments.
+     * Tests parsing and validation for user and password arguments.
      */
     public void testParseAndValidateUserAndPassword() {
         CommandHandler hnd = new CommandHandler();
 
         for (Command cmd : Command.values()) {
-            if (cmd == Command.CACHE)
+            if (cmd == Command.CACHE || cmd == Command.WAL)
                 continue; // --cache subcommand requires its own specific arguments.
 
             try {
@@ -88,13 +133,53 @@ public void testParseAndValidateUserAndPassword() {
     }
 
     /**
+     * Tests parsing and validation  of WAL commands.
+     */
+    public void testParseAndValidateWalActions() {
+        CommandHandler hnd = new CommandHandler();
+
+        Arguments args = hnd.parseAndValidate(Arrays.asList(WAL.text(), WAL_PRINT));
+
+        assertEquals(WAL, args.command());
+
+        assertEquals(WAL_PRINT, args.walAction());
+
+        String nodes = UUID.randomUUID().toString() + "," + UUID.randomUUID().toString();
+
+        args = hnd.parseAndValidate(Arrays.asList(WAL.text(), WAL_DELETE, nodes));
+
+        assertEquals(WAL_DELETE, args.walAction());
+
+        assertEquals(nodes, args.walArguments());
+
+        try {
+            hnd.parseAndValidate(Collections.singletonList(WAL.text()));
+
+            fail("expected exception: invalid arguments for --wal command");
+        }
+        catch (IllegalArgumentException e) {
+            e.printStackTrace();
+        }
+
+        try {
+            hnd.parseAndValidate(Arrays.asList(WAL.text(), UUID.randomUUID().toString()));
+
+            fail("expected exception: invalid arguments for --wal command");
+        }
+        catch (IllegalArgumentException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /**
+     * Tests host and port arguments.
      * Tests connection settings arguments.
      */
     public void testConnectionSettings() {
         CommandHandler hnd = new CommandHandler();
 
         for (Command cmd : Command.values()) {
-            if (cmd == Command.CACHE)
+            if (cmd == Command.CACHE || cmd == Command.WAL)
                 continue; // --cache subcommand requires its own specific arguments.
 
             Arguments args = hnd.parseAndValidate(asList(cmd.text()));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
new file mode 100644
index 0000000000000..0f522544107e4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
+
+/**
+ * Test correctness of truncating unused WAL segments.
+ */
+public class IgnitePdsUnusedWalSegmentsTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        System.setProperty(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, "2");
+
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+        cfg.setCacheConfiguration(ccfg);
+
+        DataStorageConfiguration dbCfg = new DataStorageConfiguration();
+
+        dbCfg.setPageSize(4 * 1024);
+
+        cfg.setDataStorageConfiguration(dbCfg);
+
+        dbCfg.setWalSegmentSize(1024 * 1024)
+                .setWalHistorySize(Integer.MAX_VALUE)
+                .setWalSegments(10)
+                .setWalMode(WALMode.LOG_ONLY)
+                .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                        .setMaxSize(100 * 1024 * 1024)
+                        .setPersistenceEnabled(true));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Tests that range reserved method return correct number of reserved WAL segments.
+     *
+     * @throws Exception if failed.
+     */
+    public void testWalManagerRangeReservation() throws Exception {
+        try{
+            IgniteEx ig0 = prepareGrid(4);
+
+            GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ig0.context().cache().context()
+                    .database();
+
+            IgniteWriteAheadLogManager wal = ig0.context().cache().context().wal();
+
+            long resIdx = getReservedWalSegmentIndex(dbMgr);
+
+            assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
+
+            FileWALPointer lowPtr = (FileWALPointer)dbMgr.checkpointHistory().lowCheckpointBound();
+
+            assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
+
+            // Reserve previous WAL segment.
+            wal.reserve(new FileWALPointer(resIdx - 1, 0, 0));
+
+            int resCnt = wal.reserved(new FileWALPointer(resIdx - 1, 0, 0), new FileWALPointer(resIdx, 0, 0));
+
+            assertTrue("Expected resCnt is 2, real is " + resCnt, resCnt == 2);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * Tests that grid cache manager correctly truncates unused WAL segments;
+     *
+     * @throws Exception if failed.
+     */
+    public void testUnusedWalTruncate() throws Exception {
+        try{
+            IgniteEx ig0 = prepareGrid(4);
+
+            GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) ig0.context().cache().context()
+                    .database();
+
+            IgniteWriteAheadLogManager wal = ig0.context().cache().context().wal();
+
+            long resIdx = getReservedWalSegmentIndex(dbMgr);
+
+            assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
+
+            FileWALPointer lowPtr = (FileWALPointer) dbMgr.checkpointHistory().lowCheckpointBound();
+
+            assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
+
+            // Reserve previous WAL segment.
+            wal.reserve(new FileWALPointer(resIdx - 1, 0, 0));
+
+            int numDel = wal.truncate(null, lowPtr);
+
+            int expNumDel = (int)resIdx - 1;
+
+            assertTrue("Expected del segments is " + expNumDel + ", real is " + numDel, expNumDel == numDel);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * Starts grid and populates test data.
+     *
+     * @param cnt Grid count.
+     * @return First started grid.
+     * @throws Exception If failed.
+     */
+    private IgniteEx prepareGrid(int cnt) throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(cnt);
+
+        ig0.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = ig0.cache(DEFAULT_CACHE_NAME);
+
+        for (int k = 0; k < 10_000; k++)
+            cache.put(k, new byte[1024]);
+
+        forceCheckpoint();
+
+        for (int k = 0; k < 1_000; k++)
+            cache.put(k, new byte[1024]);
+
+        forceCheckpoint();
+
+        return ig0;
+    }
+
+
+    /**
+     * Get index of reserved WAL segment by checkpointer.
+     *
+     * @param dbMgr Database shared manager.
+     * @throws Exception If failed.
+     */
+    private long getReservedWalSegmentIndex(GridCacheDatabaseSharedManager dbMgr) throws Exception{
+        GridCacheDatabaseSharedManager.CheckpointHistory cpHist = dbMgr.checkpointHistory();
+
+        Object histMap = GridTestUtils.getFieldValue(cpHist, "histMap");
+
+        Object cpEntry = GridTestUtils.getFieldValue(GridTestUtils.invoke(histMap, "firstEntry"), "value");
+
+        FileWALPointer walPtr = GridTestUtils.getFieldValue(cpEntry, "cpMark");
+
+        return walPtr.index();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index 16d1d0a6872c1..c95d1f4a61340 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -91,6 +91,11 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public int reserved(WALPointer low, WALPointer high) {
+        return 0;
+    }
+
     /** {@inheritDoc} */
     @Override public boolean disabled(int grpId) {
         return false;
@@ -127,7 +132,7 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
     }
 
     /** {@inheritDoc} */
-    @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
+    @Override public void onActivate(GridKernalContext kctx) {
         // No-op.
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 6d953cdd7b2de..71c6c32a983cb 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsUnusedWalSegmentsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.filename.IgniteUidAsConsistentIdMigrationTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushDefaultSelfTest;
@@ -127,6 +128,8 @@ public static void addRealPageStoreTests(TestSuite suite) {
 
         suite.addTestSuite(IgnitePdsExchangeDuringCheckpointTest.class);
 
+        suite.addTestSuite(IgnitePdsUnusedWalSegmentsTest.class);
+
         // new style folders with generated consistent ID test
         suite.addTestSuite(IgniteUidAsConsistentIdMigrationTest.class);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index e3be1e3f22312..124300585b4e7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.thread.IgniteThreadPoolSizeTest;
 import org.apache.ignite.util.GridCommandHandlerTest;
 import org.apache.ignite.util.GridIntListSelfTest;
+import org.apache.ignite.util.GridInternalTaskUnusedWalSegmentsTest;
 import org.apache.ignite.util.GridLongListSelfTest;
 import org.apache.ignite.util.GridMessageCollectionTest;
 import org.apache.ignite.util.GridPartitionMapSelfTest;
@@ -116,6 +117,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         // control.sh
         suite.addTestSuite(CommandHandlerParsingTest.class);
         suite.addTestSuite(GridCommandHandlerTest.class);
+        suite.addTestSuite(GridInternalTaskUnusedWalSegmentsTest.class);
 
         return suite;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index dbd6107f737b1..4daa92cdf000c 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -30,6 +30,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.List;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteAtomicSequence;
 import org.apache.ignite.IgniteCache;
@@ -57,6 +58,7 @@
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionRollbackException;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
@@ -85,6 +87,8 @@ protected File folder(String folder) throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
+        System.setProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, "true");
+
         cleanPersistenceDir();
 
         stopAllGrids();
@@ -100,6 +104,8 @@ protected File folder(String folder) throws IgniteCheckedException {
 
         cleanPersistenceDir();
 
+        System.clearProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND);
+
         System.setOut(sysOut);
 
         if (testOut != null)
@@ -787,4 +793,70 @@ private Map<Object, Object> generate(int from, int cnt) {
 
         return map;
     }
+
+    /**
+     *  Test execution of --wal print command.
+     *
+     *  @throws Exception if failed.
+     */
+    public void testUnusedWalPrint() throws Exception {
+        Ignite ignite = startGrids(2);
+
+        ignite.cluster().active(true);
+
+        List<String> nodes = new ArrayList<>(2);
+
+        for (ClusterNode node: ignite.cluster().forServers().nodes())
+            nodes.add(node.consistentId().toString());
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--wal", "print"));
+
+        for(String id: nodes)
+            assertTrue(testOut.toString().contains(id));
+
+        assertTrue(!testOut.toString().contains("error"));
+
+        testOut.reset();
+
+        assertEquals(EXIT_CODE_OK, execute("--wal", "print", nodes.get(0)));
+
+        assertTrue(!testOut.toString().contains(nodes.get(1)));
+
+        assertTrue(!testOut.toString().contains("error"));
+    }
+
+    /**
+     *  Test execution of --wal delete command.
+     *
+     *  @throws Exception if failed.
+     */
+    public void testUnusedWalDelete() throws Exception {
+        Ignite ignite = startGrids(2);
+
+        ignite.cluster().active(true);
+
+        List<String> nodes = new ArrayList<>(2);
+
+        for (ClusterNode node: ignite.cluster().forServers().nodes())
+            nodes.add(node.consistentId().toString());
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--wal", "delete"));
+
+        for(String id: nodes)
+            assertTrue(testOut.toString().contains(id));
+
+        assertTrue(!testOut.toString().contains("error"));
+
+        testOut.reset();
+
+        assertEquals(EXIT_CODE_OK, execute("--wal", "delete", nodes.get(0)));
+
+        assertTrue(!testOut.toString().contains(nodes.get(1)));
+
+        assertTrue(!testOut.toString().contains("error"));
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
new file mode 100644
index 0000000000000..148ecef37d3f9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.misc.VisorWalTask;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskArg;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskOperation;
+import org.apache.ignite.internal.visor.misc.VisorWalTaskResult;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
+
+/**
+ * Test correctness of VisorWalTask.
+ */
+public class GridInternalTaskUnusedWalSegmentsTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        System.setProperty(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, "2");
+
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+        cfg.setCacheConfiguration(ccfg);
+
+        DataStorageConfiguration dbCfg = new DataStorageConfiguration();
+
+        dbCfg.setPageSize(4 * 1024);
+
+        cfg.setDataStorageConfiguration(dbCfg);
+
+        dbCfg.setWalSegmentSize(1024 * 1024)
+                .setWalHistorySize(Integer.MAX_VALUE)
+                .setWalSegments(10)
+                .setWalMode(WALMode.LOG_ONLY)
+                .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                        .setMaxSize(100 * 1024 * 1024)
+                        .setPersistenceEnabled(true));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Tests correctness of {@link VisorWalTaskOperation}.
+     *
+     * @throws Exception if failed.
+     */
+    public void testCorrectnessOfDeletionTaskSegments() throws Exception {
+        try {
+            IgniteEx ig0 = (IgniteEx)startGrids(4);
+
+            ig0.cluster().active(true);
+
+            IgniteCache<Object, Object> cache = ig0.cache(DEFAULT_CACHE_NAME);
+
+            for (int k = 0; k < 10_000; k++)
+                cache.put(k, new byte[1024]);
+
+            forceCheckpoint();
+
+            for (int k = 0; k < 1_000; k++)
+                cache.put(k, new byte[1024]);
+
+            forceCheckpoint();
+
+            VisorWalTaskResult printRes = ig0.compute().execute(VisorWalTask.class,
+                    new VisorTaskArgument<>(ig0.cluster().node().id(),
+                            new VisorWalTaskArg(VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS), false));
+
+            assertEquals("Check that print task finished without exceptions", printRes.results().size(), 4);
+
+            List<File> walArchives = new ArrayList<>();
+
+            for (Collection<String> pathsPerNode : printRes.results().values()) {
+                for (String path : pathsPerNode)
+                    walArchives.add(Paths.get(path).toFile());
+            }
+
+            VisorWalTaskResult delRes = ig0.compute().execute(VisorWalTask.class,
+                    new VisorTaskArgument<>(ig0.cluster().node().id(),
+                            new VisorWalTaskArg(VisorWalTaskOperation.DELETE_UNUSED_WAL_SEGMENTS), false));
+
+            assertEquals("Check that delete task finished with no exceptions", delRes.results().size(), 4);
+
+            List<File> walDeletedArchives = new ArrayList<>();
+
+            for (Collection<String> pathsPerNode : delRes.results().values()) {
+                for (String path : pathsPerNode)
+                    walDeletedArchives.add(Paths.get(path).toFile());
+            }
+
+            for (File f : walDeletedArchives)
+                assertTrue("Checking existing of deleted WAL archived segments: " + f.getAbsolutePath(), !f.exists());
+
+            for (File f : walArchives)
+                assertTrue("Checking existing of WAL archived segments from print task after delete: " + f.getAbsolutePath(),
+                        !f.exists());
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+}

From 5cc69ed65f66ecd59d3f98f189bba5f42d77748c Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Thu, 10 May 2018 12:58:53 +0300
Subject: [PATCH 136/543] release notes 2.5.1-p1

---
 RELEASE_NOTES.txt | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 77db736468574..79550b0a0584e 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,6 +1,44 @@
 Apache Ignite Release Notes
 ===========================
 
+Changes in GridGain In-Memory Data Fabric Professional Edition 2.5.1
+------------------------------------
+* Introduced ZooKeeper discovery SPI
+* Added ability to disable WAL during rebalancing
+* Added ability to rollback long-running transactions that block partition map exchange
+* Introduced additional storage metrics
+* Added utilities to control.sh script to display cache info
+* Implemented handling of broken segment in WAL compaction
+* Fixed incorrect mapping of smallint cassandra type
+* Fixed the issue that prevented rebalancing on a new baseline node join
+* Added additional data storage metrics
+* Added additional transactional metrics
+* Error while trying to initialize cluster with partially acceptable wal archive
+* Nodes with incompatible SQL metadata will not be able to join grid
+* Added ability to disable WAL during initial data rebalancing
+* Implemented additional synchronization for correct partition counters update on partition map exchange
+* Fixed NullPointerException on batch atomic cache operations with cache store
+* Added partition update counters verification on partition map exchange
+* General approach for critical failures handling implemented
+* Added configurable automatic rollback timeout for pending transactions preventing partition map exchange to complete
+
+* SQL: sqlline.sh Fixed script to use Java from JAVA_HOME
+
+* REST: Added AUTHENTICATE command
+
+* Control utility: Added utilities to control.sh script to display cache info
+* Control utility: Fixed NPE in case of empty base line and not active cluster
+* Control utility: Added utilities to view and kill active transactions
+* Control utility: Fixed timeout on changing baseline
+
+* Web Console: Added export of thread dump in text format
+* Web Console: Fixed Dockerfile generation
+* Web Console: Fixed code generation for large numbers in configuration params
+* Web Console: Fixed demo for non-collocated joins
+* Web Console: Implemented support for comma-separated list of node URIs
+
+* Visor CMD: Added "cache -slp" and "cache -rlp" commands to show and reset lost partitions for specified cache
+
 Apache Ignite In-Memory Data Fabric 2.4
 ---------------------------------------
 Ignite:

From 0690bd492714f35cf01dd58a61623e50a8d3bab9 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Thu, 10 May 2018 13:17:00 +0300
Subject: [PATCH 137/543] release notes 2.5.1-p1 updated

---
 RELEASE_NOTES.txt | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 79550b0a0584e..eb19ae5f48667 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -3,6 +3,7 @@ Apache Ignite Release Notes
 
 Changes in GridGain In-Memory Data Fabric Professional Edition 2.5.1
 ------------------------------------
+Ignite:
 * Introduced ZooKeeper discovery SPI
 * Added ability to disable WAL during rebalancing
 * Added ability to rollback long-running transactions that block partition map exchange
@@ -22,6 +23,7 @@ Changes in GridGain In-Memory Data Fabric Professional Edition 2.5.1
 * General approach for critical failures handling implemented
 * Added configurable automatic rollback timeout for pending transactions preventing partition map exchange to complete
 
+SQL:
 * SQL: sqlline.sh Fixed script to use Java from JAVA_HOME
 
 * REST: Added AUTHENTICATE command
@@ -31,6 +33,7 @@ Changes in GridGain In-Memory Data Fabric Professional Edition 2.5.1
 * Control utility: Added utilities to view and kill active transactions
 * Control utility: Fixed timeout on changing baseline
 
+Web Console:
 * Web Console: Added export of thread dump in text format
 * Web Console: Fixed Dockerfile generation
 * Web Console: Fixed code generation for large numbers in configuration params

From 2b4feaef1e5c84417f45d3efeb2185424375b215 Mon Sep 17 00:00:00 2001
From: dpavlov <dpavlov@apache.org>
Date: Mon, 14 May 2018 13:39:21 +0300
Subject: [PATCH 138/543] IGNITE-8138 Fix for incorrect uptime in Ignite
 metrics for long running server node Cherry-picked from
 4ea7f926e6f3803c616ab51e1d2e79765862efa1

---
 .../ignite/internal/util/typedef/X.java       | 44 ++++++++++++++++---
 .../ignite/spi/IgniteSpiMBeanAdapter.java     |  2 +-
 .../org/apache/ignite/lang/GridXSelfTest.java | 13 ++++++
 3 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java b/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
index 49732b6817630..be69b24821394 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/typedef/X.java
@@ -53,8 +53,24 @@ public final class X {
     /** An empty immutable {@code Object} array. */
     public static final Object[] EMPTY_OBJECT_ARRAY = new Object[0];
 
+    /** Millis in second. */
+    private static final long MILLIS_IN_SECOND = 1000L;
+
+    /** Seconds in minute. */
+    private static final long SECONDS_IN_MINUTE = 60L;
+
+    /** Minuses in hour. */
+    private static final long MINUTES_IN_HOUR = 60L;
+
+    /** Hours in day. */
+    private static final long HOURS_IN_DAY = 24L;
+
     /** Time span dividers. */
-    private static final long[] SPAN_DIVS = new long[] {1000L, 60L, 60L, 60L};
+    private static final long[] SPAN_DIVS = new long[] {
+        MILLIS_IN_SECOND, SECONDS_IN_MINUTE, MINUTES_IN_HOUR, HOURS_IN_DAY};
+
+    /** Millis in day. */
+    private static final long MILLIS_IN_DAY = MILLIS_IN_SECOND * SECONDS_IN_MINUTE * MINUTES_IN_HOUR * HOURS_IN_DAY;
 
     /** The names of methods commonly used to access a wrapped exception. */
     private static final String[] CAUSE_MTD_NAMES = new String[] {
@@ -75,9 +91,6 @@ public final class X {
     /** The Method object for Java 1.4 getCause. */
     private static final Method THROWABLE_CAUSE_METHOD;
 
-    /**
-     *
-     */
     static {
         Method causeMtd;
 
@@ -200,7 +213,7 @@ public static void printerr(@Nullable String s1, @Nullable Object... rest) {
      * Creates string presentation of given time {@code span} in hh:mm:ss.msec {@code HMSM} format.
      *
      * @param span Time span.
-     * @return String presentation.
+     * @return String presentation. If duration if longer than 1 day, days count is ignored.
      */
     public static String timeSpan2HMSM(long span) {
         long[] t = new long[4];
@@ -235,6 +248,27 @@ public static String timeSpan2HMS(long span) {
             (t[1] < 10 ? "0" + t[1] : Long.toString(t[1]));
     }
 
+    /**
+     * Creates string presentation of given time {@code span} in days, hh:mm:ss.mmm {@code HMS} format.
+     *
+     * @param span Time span.
+     * @return String presentation.
+     */
+    public static String timeSpan2DHMSM(long span) {
+        String days = "";
+
+        String hmsm = timeSpan2HMSM(span % MILLIS_IN_DAY);
+
+        long daysCnt = span / MILLIS_IN_DAY;
+
+        if (daysCnt == 1)
+            days = "1 day, ";
+        else if (daysCnt > 1)
+            days = daysCnt + " days, ";
+
+        return days + hmsm;
+    }
+
     /**
      * Clones a passed in object. If parameter {@code deep} is set to {@code true}
      * then this method will use deep cloning algorithm based on deep reflection
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiMBeanAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiMBeanAdapter.java
index 8035333d984a4..f48b7e5839177 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiMBeanAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiMBeanAdapter.java
@@ -46,7 +46,7 @@ public IgniteSpiMBeanAdapter(IgniteSpiAdapter spiAdapter) {
 
     /** {@inheritDoc} */
     @Override public final String getUpTimeFormatted() {
-        return X.timeSpan2HMSM(getUpTime());
+        return X.timeSpan2DHMSM(getUpTime());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/lang/GridXSelfTest.java b/modules/core/src/test/java/org/apache/ignite/lang/GridXSelfTest.java
index e7608b7ea625e..8fd6df638668d 100644
--- a/modules/core/src/test/java/org/apache/ignite/lang/GridXSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/lang/GridXSelfTest.java
@@ -52,6 +52,19 @@ public void testHasCause() {
         assert gridEx.getCause(NumberFormatException.class) == null;
     }
 
+    /**
+     * Tests string presentation of given time.
+     */
+    public void testTimeSpan() {
+        assertEquals(X.timeSpan2DHMSM(86400001L), "1 day, 00:00:00.001");
+
+        assertEquals(X.timeSpan2DHMSM(172800004L), "2 days, 00:00:00.004");
+
+        assertEquals(X.timeSpan2DHMSM(1L), "00:00:00.001");
+
+        assertEquals(X.timeSpan2HMSM(172800004L), "00:00:00.004");
+    }
+
     /**
      *
      */

From bf6cef1468c44ece07cc3190bdff2d515195d12c Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Wed, 16 May 2018 15:25:51 +0300
Subject: [PATCH 139/543] IGNITE-8508 Proper ordering of ZK discovery custom
 events ACKs

---
 .../discovery/zk/internal/ZkDiscoveryEventsData.java  | 11 +++++++++++
 .../discovery/zk/internal/ZookeeperDiscoveryImpl.java |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
index dce861b523e4c..6520b8cb97cde 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryEventsData.java
@@ -21,6 +21,8 @@
 import java.util.Collection;
 import java.util.TreeMap;
 import java.util.UUID;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -118,4 +120,13 @@ void addEvent(Collection<ZookeeperClusterNode> nodes, ZkDiscoveryEventData evt)
 
         evt.initRemainingAcks(nodes);
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(ZkDiscoveryEventsData.class, this,
+            "topVer", topVer,
+            "evtIdGen", evtIdGen,
+            "procCustEvt", procCustEvt,
+            "evts", evts);
+    }
 }
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 0604458e461b6..43d6aeb9f3a6a 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -3658,7 +3658,7 @@ private ZkDiscoveryCustomEventData createAckEvent(
         ZkDiscoveryCustomEventData ackEvtData = new ZkDiscoveryCustomEventData(
             evtId,
             origEvt.eventId(),
-            origEvt.topologyVersion(), // Use topology version from original event.
+            rtState.evtsData.topVer, // Use actual topology version because topology version must be growing.
             locNode.id(),
             null,
             null);

From 1d096b35c8723ea6bbc7d3273134f4094f1fb62b Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Tue, 24 Apr 2018 18:22:52 +0300
Subject: [PATCH 140/543] IGNITE-8313 Add trace logs on exchange phases and
 affinity calculation. - Fixes #3881.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit f4646e4)
---
 .../affinity/GridAffinityAssignmentCache.java | 37 ++++++++++++++
 .../dht/GridDhtPartitionTopologyImpl.java     | 51 +++++++++++++++++++
 2 files changed, 88 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index 427d60328b7fb..5b985c262bd6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -48,6 +48,7 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
@@ -216,6 +217,12 @@ public void initialize(AffinityTopologyVersion topVer, List<List<ClusterNode>> a
         }
 
         onHistoryAdded(assignment);
+
+        if (log.isTraceEnabled()) {
+            log.trace("New affinity assignment [grp=" + cacheOrGrpName
+                + ", topVer=" + topVer
+                + ", aff=" + fold(affAssignment) + "]");
+        }
     }
 
     /**
@@ -768,6 +775,36 @@ public Collection<AffinityTopologyVersion> cachedVersions() {
         return affCache.keySet();
     }
 
+    /**
+     * @param affAssignment Affinity assignment.
+     * @return String representation of given {@code affAssignment}.
+     */
+    private static String fold(List<List<ClusterNode>> affAssignment) {
+        SB sb = new SB();
+
+        for (int p = 0; p < affAssignment.size(); p++) {
+            sb.a("Part [");
+            sb.a("id=" + p + ", ");
+
+            SB partOwners = new SB();
+
+            List<ClusterNode> affOwners = affAssignment.get(p);
+
+            for (ClusterNode node : affOwners) {
+                partOwners.a(node.consistentId());
+                partOwners.a(' ');
+            }
+
+            sb.a("owners=[");
+            sb.a(partOwners);
+            sb.a(']');
+
+            sb.a("] ");
+        }
+
+        return sb.toString();
+    }
+
     /**
      * Affinity ready future. Will remove itself from ready futures map.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 71d43c8ebc55c..3d664f3eafb08 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -60,6 +60,7 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
+import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -556,6 +557,11 @@ else if (localNode(p, aff))
                         log.debug("Partition map after beforeExchange [grp=" + grp.cacheOrGroupName() + ", " +
                             "exchId=" + exchFut.exchangeId() + ", fullMap=" + fullMapString() + ']');
                     }
+
+                    if (log.isTraceEnabled()) {
+                        log.trace("Partition states after beforeExchange [grp=" + grp.cacheOrGroupName()
+                            + ", exchId=" + exchFut.exchangeId() + ", states=" + dumpPartitionStates() + ']');
+                    }
                 }
                 finally {
                     lock.writeLock().unlock();
@@ -687,6 +693,11 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
                         ", fullMap=" + fullMapString() + ']');
                 }
 
+                if (log.isTraceEnabled()) {
+                    log.trace("Partition states before afterExchange [grp=" + grp.cacheOrGroupName()
+                        + ", exchVer=" + exchFut.exchangeId() + ", states=" + dumpPartitionStates() + ']');
+                }
+
                 long updateSeq = this.updateSeq.incrementAndGet();
 
                 for (int p = 0; p < num; p++) {
@@ -765,6 +776,11 @@ else if (log.isDebugEnabled())
                 updateRebalanceVersion(aff.assignment());
 
                 consistencyCheck();
+
+                if (log.isTraceEnabled()) {
+                    log.trace("Partition states after afterExchange [grp=" + grp.cacheOrGroupName()
+                        + ", exchVer=" + exchFut.exchangeId() + ", states=" + dumpPartitionStates() + ']');
+                }
             }
             finally {
                 lock.writeLock().unlock();
@@ -1327,6 +1343,11 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
             lock.writeLock().lock();
 
             try {
+                if (log.isTraceEnabled() && exchangeVer != null) {
+                    log.trace("Partition states before full update [grp=" + grp.cacheOrGroupName()
+                        + ", exchVer=" + exchangeVer + ", states=" + dumpPartitionStates() + ']');
+                }
+
                 if (stopping || !lastTopChangeVer.initialized() ||
                     // Ignore message not-related to exchange if exchange is in progress.
                     (exchangeVer == null && !lastTopChangeVer.equals(readyTopVer)))
@@ -1530,6 +1551,11 @@ else if (state == MOVING) {
                         ", map=" + fullMapString() + ']');
                 }
 
+                if (log.isTraceEnabled() && exchangeVer != null) {
+                    log.trace("Partition states after full update [grp=" + grp.cacheOrGroupName()
+                        + ", exchVer=" + exchangeVer + ", states=" + dumpPartitionStates() + ']');
+                }
+
                 if (changed)
                     ctx.exchange().scheduleResendPartitions();
 
@@ -2734,6 +2760,31 @@ private void consistencyCheck() {
         // no-op
     }
 
+    /**
+     * Collects states of local partitions.
+     *
+     * @return String representation of all local partition states.
+     */
+    private String dumpPartitionStates() {
+        SB sb = new SB();
+
+        for (int p = 0; p < locParts.length(); p++) {
+            GridDhtLocalPartition part = locParts.get(p);
+
+            if (part == null)
+                continue;
+
+            sb.a("Part [");
+            sb.a("id=" + part.id() + ", ");
+            sb.a("state=" + part.state() + ", ");
+            sb.a("initCounter=" + part.initialUpdateCounter() + ", ");
+            sb.a("updCounter=" + part.updateCounter() + ", ");
+            sb.a("size=" + part.fullSize() + "] ");
+        }
+
+        return sb.toString();
+    }
+
     /**
      * Iterator over current local partitions.
      */

From e536f3c1b1e28f4e6ed813148b3b7fae56303279 Mon Sep 17 00:00:00 2001
From: Ivan Rakov <irakov@apache.org>
Date: Wed, 16 May 2018 20:13:30 +0300
Subject: [PATCH 141/543] IGNITE-8499 validate_indexes command doesn't detect
 absent rows in cache data tree

(cherry picked from commit 88a6bfd)
---
 .../internal/commandline/CommandHandler.java  |  21 +-
 .../ValidateIndexesPartitionResult.java       |  31 +-
 .../verify/VisorValidateIndexesJobResult.java |  38 ++-
 .../visor/verify/ValidateIndexesClosure.java  | 264 ++++++++++++++----
 .../verify/VisorValidateIndexesTask.java      |   6 +-
 .../IgniteCacheWithIndexingTestSuite.java     |   3 +
 .../util/GridCommandHandlerIndexingTest.java  | 203 +++++++++++++-
 7 files changed, 477 insertions(+), 89 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 7d457fd1f1265..04578e53f916d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -100,8 +100,8 @@
 import static org.apache.ignite.internal.commandline.Command.CACHE;
 import static org.apache.ignite.internal.commandline.Command.DEACTIVATE;
 import static org.apache.ignite.internal.commandline.Command.STATE;
-import static org.apache.ignite.internal.commandline.Command.WAL;
 import static org.apache.ignite.internal.commandline.Command.TX;
+import static org.apache.ignite.internal.commandline.Command.WAL;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.ADD;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.COLLECT;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.REMOVE;
@@ -635,9 +635,9 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
         boolean errors = false;
 
         for (Map.Entry<UUID, VisorValidateIndexesJobResult> nodeEntry : taskRes.results().entrySet()) {
-            Map<PartitionKey, ValidateIndexesPartitionResult> map = nodeEntry.getValue().response();
+            Map<PartitionKey, ValidateIndexesPartitionResult> partRes = nodeEntry.getValue().partitionResult();
 
-            for (Map.Entry<PartitionKey, ValidateIndexesPartitionResult> e : map.entrySet()) {
+            for (Map.Entry<PartitionKey, ValidateIndexesPartitionResult> e : partRes.entrySet()) {
                 ValidateIndexesPartitionResult res = e.getValue();
 
                 if (!res.issues().isEmpty()) {
@@ -649,6 +649,21 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
                         log(is.toString());
                 }
             }
+
+            Map<String, ValidateIndexesPartitionResult> idxRes = nodeEntry.getValue().indexResult();
+
+            for (Map.Entry<String, ValidateIndexesPartitionResult> e : idxRes.entrySet()) {
+                ValidateIndexesPartitionResult res = e.getValue();
+
+                if (!res.issues().isEmpty()) {
+                    errors = true;
+
+                    log("SQL Index " + e.getKey() + " " + e.getValue().toString());
+
+                    for (IndexValidationIssue is : res.issues())
+                        log(is.toString());
+                }
+            }
         }
 
         if (!errors)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
index 18899607110fc..5d74a57e14ca4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesPartitionResult.java
@@ -29,7 +29,7 @@
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
 
 /**
- *
+ * Encapsulates intermediate results of validation of SQL index (if {@link #sqlIdxName} is present) or partition.
  */
 public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
     /** */
@@ -52,6 +52,10 @@ public class ValidateIndexesPartitionResult extends VisorDataTransferObject {
     @GridToStringExclude
     private List<IndexValidationIssue> issues = new ArrayList<>(10);
 
+    /** Sql index name. */
+    @GridToStringExclude
+    private String sqlIdxName;
+
     /**
      *
      */
@@ -64,12 +68,15 @@ public ValidateIndexesPartitionResult() {
      * @param size Size.
      * @param isPrimary Is primary.
      * @param consistentId Consistent id.
+     * @param sqlIdxName Sql index name (optional).
      */
-    public ValidateIndexesPartitionResult(long updateCntr, long size, boolean isPrimary, Object consistentId) {
+    public ValidateIndexesPartitionResult(long updateCntr, long size, boolean isPrimary, Object consistentId,
+        String sqlIdxName) {
         this.updateCntr = updateCntr;
         this.size = size;
         this.isPrimary = isPrimary;
         this.consistentId = consistentId;
+        this.sqlIdxName = sqlIdxName;
     }
 
     /**
@@ -107,6 +114,13 @@ public List<IndexValidationIssue> issues() {
         return issues;
     }
 
+    /**
+     * @return <code>null</code> for partition validation result, SQL index name for index validation result
+     */
+    public String sqlIndexName() {
+        return sqlIdxName;
+    }
+
     /**
      * @param t Issue.
      * @return True if there are already enough issues.
@@ -120,6 +134,11 @@ public boolean reportIssue(IndexValidationIssue t) {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         out.writeLong(updateCntr);
@@ -127,6 +146,7 @@ public boolean reportIssue(IndexValidationIssue t) {
         out.writeBoolean(isPrimary);
         out.writeObject(consistentId);
         U.writeCollection(out, issues);
+        U.writeString(out, sqlIdxName);
     }
 
     /** {@inheritDoc} */
@@ -136,10 +156,15 @@ public boolean reportIssue(IndexValidationIssue t) {
         isPrimary = in.readBoolean();
         consistentId = in.readObject();
         issues = U.readList(in);
+
+        if (protoVer >= V2)
+            sqlIdxName = U.readString(in);
     }
 
     /** {@inheritDoc} */
     @Override public String toString() {
-        return S.toString(ValidateIndexesPartitionResult.class, this);
+        return sqlIdxName == null ? S.toString(ValidateIndexesPartitionResult.class, this) :
+            ValidateIndexesPartitionResult.class.getSimpleName() + " [consistentId=" + consistentId +
+                ", sqlIdxName=" + sqlIdxName + "]";
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
index 25c97b651a4d9..aa74323898bd8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
@@ -34,13 +34,19 @@ public class VisorValidateIndexesJobResult extends VisorDataTransferObject {
     private static final long serialVersionUID = 0L;
 
     /** Results of indexes validation from node. */
-    private Map<PartitionKey, ValidateIndexesPartitionResult> res;
+    private Map<PartitionKey, ValidateIndexesPartitionResult> partRes;
+
+    /** Results of reverse indexes validation from node. */
+    private Map<String, ValidateIndexesPartitionResult> idxRes;
 
     /**
-     * @param res Results of indexes validation from node.
+     * @param partRes Results of indexes validation from node.
+     * @param idxRes Results of reverse indexes validation from node.
      */
-    public VisorValidateIndexesJobResult(Map<PartitionKey, ValidateIndexesPartitionResult> res) {
-        this.res = res;
+    public VisorValidateIndexesJobResult(Map<PartitionKey, ValidateIndexesPartitionResult> partRes,
+        Map<String, ValidateIndexesPartitionResult> idxRes) {
+        this.partRes = partRes;
+        this.idxRes = idxRes;
     }
 
     /**
@@ -49,21 +55,37 @@ public VisorValidateIndexesJobResult(Map<PartitionKey, ValidateIndexesPartitionR
     public VisorValidateIndexesJobResult() {
     }
 
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
     /**
      * @return Results of indexes validation from node.
      */
-    public Map<PartitionKey, ValidateIndexesPartitionResult> response() {
-        return res;
+    public Map<PartitionKey, ValidateIndexesPartitionResult> partitionResult() {
+        return partRes;
+    }
+
+    /**
+     * @return Results of reverse indexes validation from node.
+     */
+    public Map<String, ValidateIndexesPartitionResult> indexResult() {
+        return idxRes;
     }
 
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
-        U.writeMap(out, res);
+        U.writeMap(out, partRes);
+        U.writeMap(out, idxRes);
     }
 
     /** {@inheritDoc} */
     @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
-        res = U.readMap(in);
+        partRes = U.readMap(in);
+
+        if (protoVer >= V2)
+            idxRes = U.readMap(in);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index 373bd15dad315..e0eff612fad2a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -31,9 +31,8 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteInterruptedException;
@@ -51,12 +50,15 @@
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
 import org.apache.ignite.internal.processors.query.QueryTypeDescriptorImpl;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
 import org.apache.ignite.internal.util.lang.GridIterator;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.resources.IgniteInstanceResource;
@@ -66,9 +68,13 @@
 import org.h2.index.Index;
 
 /**
- *
+ * Closure that locally validates indexes of given caches.
+ * Validation consists of three checks:
+ * 1. If entry is present in cache data tree, it's reachable from all cache SQL indexes
+ * 2. If entry is present in cache SQL index, it can be dereferenced with link from index
+ * 3. If entry is present in cache SQL index, it's present in cache data tree
  */
-public class ValidateIndexesClosure implements IgniteCallable<Map<PartitionKey, ValidateIndexesPartitionResult>> {
+public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndexesJobResult> {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -84,7 +90,19 @@ public class ValidateIndexesClosure implements IgniteCallable<Map<PartitionKey,
     private Set<String> cacheNames;
 
     /** Counter of processed partitions. */
-    private final AtomicInteger completionCntr = new AtomicInteger(0);
+    private final AtomicInteger processedPartitions = new AtomicInteger(0);
+
+    /** Total partitions. */
+    private volatile int totalPartitions;
+
+    /** Counter of processed indexes. */
+    private final AtomicInteger processedIndexes = new AtomicInteger(0);
+
+    /** Total partitions. */
+    private volatile int totalIndexes;
+
+    /** Last progress print timestamp. */
+    private final AtomicLong lastProgressPrintTs = new AtomicLong(0);
 
     /** Calculation executor. */
     private volatile ExecutorService calcExecutor;
@@ -97,7 +115,7 @@ public ValidateIndexesClosure(Set<String> cacheNames) {
     }
 
     /** {@inheritDoc} */
-    @Override public Map<PartitionKey, ValidateIndexesPartitionResult> call() throws Exception {
+    @Override public VisorValidateIndexesJobResult call() throws Exception {
         calcExecutor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
 
         try {
@@ -111,7 +129,7 @@ public ValidateIndexesClosure(Set<String> cacheNames) {
     /**
      *
      */
-    private Map<PartitionKey, ValidateIndexesPartitionResult> call0() throws Exception {
+    private VisorValidateIndexesJobResult call0() throws Exception {
         Set<Integer> grpIds = new HashSet<>();
 
         Set<String> missingCaches = new HashSet<>();
@@ -150,8 +168,9 @@ private Map<PartitionKey, ValidateIndexesPartitionResult> call0() throws Excepti
         }
 
         List<Future<Map<PartitionKey, ValidateIndexesPartitionResult>>> procPartFutures = new ArrayList<>();
-
-        completionCntr.set(0);
+        List<Future<Map<String, ValidateIndexesPartitionResult>>> procIdxFutures = new ArrayList<>();
+        List<T2<CacheGroupContext, GridDhtLocalPartition>> partArgs = new ArrayList<>();
+        List<T2<GridCacheContext, Index>> idxArgs = new ArrayList<>();
 
         for (Integer grpId : grpIds) {
             CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
@@ -162,45 +181,82 @@ private Map<PartitionKey, ValidateIndexesPartitionResult> call0() throws Excepti
             List<GridDhtLocalPartition> parts = grpCtx.topology().localPartitions();
 
             for (GridDhtLocalPartition part : parts)
-                procPartFutures.add(processPartitionAsync(grpCtx, part));
-        }
+                partArgs.add(new T2<>(grpCtx, part));
 
-        Map<PartitionKey, ValidateIndexesPartitionResult> res = new HashMap<>();
+            GridQueryProcessor qry = ignite.context().query();
 
-        long lastProgressLogTs = U.currentTimeMillis();
+            IgniteH2Indexing indexing = (IgniteH2Indexing)qry.getIndexing();
 
-        for (int i = 0; i < procPartFutures.size(); ) {
-            Future<Map<PartitionKey, ValidateIndexesPartitionResult>> fut = procPartFutures.get(i);
+            for (GridCacheContext ctx : grpCtx.caches()) {
+                Collection<GridQueryTypeDescriptor> types = qry.types(ctx.name());
 
-            try {
-                Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get(1, TimeUnit.SECONDS);
+                if (!F.isEmpty(types)) {
+                    for (GridQueryTypeDescriptor type : types) {
+                        GridH2Table gridH2Tbl = indexing.dataTable(ctx.name(), type.tableName());
+
+                        if (gridH2Tbl == null)
+                            continue;
 
-                res.putAll(partRes);
+                        ArrayList<Index> indexes = gridH2Tbl.getIndexes();
 
-                i++;
+                        for (Index idx : indexes)
+                            idxArgs.add(new T2<>(ctx, idx));
+                    }
+                }
             }
-            catch (InterruptedException | ExecutionException e) {
-                for (int j = i + 1; j < procPartFutures.size(); j++)
-                    procPartFutures.get(j).cancel(false);
-
-                if (e instanceof InterruptedException)
-                    throw new IgniteInterruptedException((InterruptedException)e);
-                else if (e.getCause() instanceof IgniteException)
-                    throw (IgniteException)e.getCause();
-                else
-                    throw new IgniteException(e.getCause());
+        }
+
+        // To decrease contention on same indexes.
+        Collections.shuffle(partArgs);
+        Collections.shuffle(idxArgs);
+
+        for (T2<CacheGroupContext, GridDhtLocalPartition> t2 : partArgs)
+            procPartFutures.add(processPartitionAsync(t2.get1(), t2.get2()));
+
+        for (T2<GridCacheContext, Index> t2 : idxArgs)
+            procIdxFutures.add(processIndexAsync(t2.get1(), t2.get2()));
+
+        totalPartitions = procPartFutures.size();
+        totalIndexes = procIdxFutures.size();
+
+        Map<PartitionKey, ValidateIndexesPartitionResult> partResults = new HashMap<>();
+        Map<String, ValidateIndexesPartitionResult> idxResults = new HashMap<>();
+
+        int curPart = 0;
+        int curIdx = 0;
+        try {
+            for (; curPart < procPartFutures.size(); curPart++) {
+                Future<Map<PartitionKey, ValidateIndexesPartitionResult>> fut = procPartFutures.get(curPart);
+
+                Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get();
+
+                partResults.putAll(partRes);
             }
-            catch (TimeoutException ignored) {
-                if (U.currentTimeMillis() - lastProgressLogTs > 60 * 1000L) {
-                    lastProgressLogTs = U.currentTimeMillis();
 
-                    log.warning("ValidateIndexesClosure is still running, processed " + completionCntr.get() + " of " +
-                        procPartFutures.size() + " local partitions");
-                }
+            for (; curIdx < procIdxFutures.size(); curIdx++) {
+                Future<Map<String, ValidateIndexesPartitionResult>> fut = procIdxFutures.get(curIdx);
+
+                Map<String, ValidateIndexesPartitionResult> idxRes = fut.get();
+
+                idxResults.putAll(idxRes);
             }
         }
+        catch (InterruptedException | ExecutionException e) {
+            for (int j = curPart; j < procPartFutures.size(); j++)
+                procPartFutures.get(j).cancel(false);
+
+            for (int j = curIdx; j < procIdxFutures.size(); j++)
+                procIdxFutures.get(j).cancel(false);
+
+            if (e instanceof InterruptedException)
+                throw new IgniteInterruptedException((InterruptedException)e);
+            else if (e.getCause() instanceof IgniteException)
+                throw (IgniteException)e.getCause();
+            else
+                throw new IgniteException(e.getCause());
+        }
 
-        return res;
+        return new VisorValidateIndexesJobResult(partResults, idxResults);
     }
 
     /**
@@ -245,12 +301,24 @@ private Map<PartitionKey, ValidateIndexesPartitionResult> processPartition(
 
             boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
 
-            partRes = new ValidateIndexesPartitionResult(updateCntrBefore, partSize, isPrimary, consId);
+            partRes = new ValidateIndexesPartitionResult(updateCntrBefore, partSize, isPrimary, consId, null);
 
             boolean enoughIssues = false;
 
-            long keysProcessed = 0;
-            long lastProgressLog = U.currentTimeMillis();
+            GridQueryProcessor qryProcessor = ignite.context().query();
+
+            Method m;
+            try {
+                m = GridQueryProcessor.class.getDeclaredMethod("typeByValue", String.class,
+                    CacheObjectContext.class, KeyCacheObject.class, CacheObject.class, boolean.class);
+            }
+            catch (NoSuchMethodException e) {
+                log.error("Failed to invoke typeByValue", e);
+
+                throw new IgniteException(e);
+            }
+
+            m.setAccessible(true);
 
             while (it.hasNextX()) {
                 if (enoughIssues)
@@ -266,14 +334,7 @@ private Map<PartitionKey, ValidateIndexesPartitionResult> processPartition(
                 if (cacheCtx == null)
                     throw new IgniteException("Unknown cacheId of CacheDataRow: " + cacheId);
 
-                GridQueryProcessor qryProcessor = ignite.context().query();
-
                 try {
-                    Method m = GridQueryProcessor.class.getDeclaredMethod("typeByValue", String.class,
-                        CacheObjectContext.class, KeyCacheObject.class, CacheObject.class, boolean.class);
-
-                    m.setAccessible(true);
-
                     QueryTypeDescriptorImpl res = (QueryTypeDescriptorImpl)m.invoke(
                         qryProcessor, cacheCtx.name(), cacheCtx.cacheObjectContext(), row.key(), row.value(), true);
 
@@ -298,7 +359,7 @@ private Map<PartitionKey, ValidateIndexesPartitionResult> processPartition(
                             Cursor cursor = idx.find((Session) null, h2Row, h2Row);
 
                             if (cursor == null || !cursor.next())
-                                throw new IgniteCheckedException("Key not found.");
+                                throw new IgniteCheckedException("Key is present in CacheDataTree, but can't be found in SQL index.");
                         }
                         catch (Throwable t) {
                             Object o = CacheObjectUtils.unwrapBinaryIfNeeded(
@@ -313,7 +374,7 @@ private Map<PartitionKey, ValidateIndexesPartitionResult> processPartition(
                         }
                     }
                 }
-                catch (IllegalAccessException | NoSuchMethodException e) {
+                catch (IllegalAccessException e) {
                     log.error("Failed to invoke typeByValue", e);
 
                     throw new IgniteException(e);
@@ -325,16 +386,6 @@ private Map<PartitionKey, ValidateIndexesPartitionResult> processPartition(
 
                     throw new IgniteException(target);
                 }
-                finally {
-                    keysProcessed++;
-
-                    if (U.currentTimeMillis() - lastProgressLog >= 60_000 && partSize > 0) {
-                        log.warning("Processing partition " + part.id() + " (" + (keysProcessed * 100 / partSize) +
-                            "% " + keysProcessed + "/" + partSize + ")");
-
-                        lastProgressLog = U.currentTimeMillis();
-                    }
-                }
             }
         }
         catch (IgniteCheckedException e) {
@@ -345,12 +396,107 @@ private Map<PartitionKey, ValidateIndexesPartitionResult> processPartition(
         }
         finally {
             part.release();
+
+            printProgressIfNeeded();
         }
 
         PartitionKey partKey = new PartitionKey(grpCtx.groupId(), part.id(), grpCtx.cacheOrGroupName());
 
-        completionCntr.incrementAndGet();
+        processedPartitions.incrementAndGet();
 
         return Collections.singletonMap(partKey, partRes);
     }
+
+    /**
+     *
+     */
+    private void printProgressIfNeeded() {
+        long curTs = U.currentTimeMillis();
+
+        long lastTs = lastProgressPrintTs.get();
+
+        if (curTs - lastTs >= 60_000 && lastProgressPrintTs.compareAndSet(lastTs, curTs)) {
+            log.warning("Current progress of ValidateIndexesClosure: processed " +
+                processedPartitions.get() + " of " + totalPartitions + " partitions, " +
+                processedIndexes.get() + " of " + totalIndexes + " SQL indexes");
+        }
+    }
+
+    /**
+     * @param ctx Context.
+     * @param idx Index.
+     */
+    private Future<Map<String, ValidateIndexesPartitionResult>> processIndexAsync(GridCacheContext ctx, Index idx) {
+        return calcExecutor.submit(new Callable<Map<String, ValidateIndexesPartitionResult>>() {
+            @Override public Map<String, ValidateIndexesPartitionResult> call() throws Exception {
+                return processIndex(ctx, idx);
+            }
+        });
+    }
+
+    /**
+     * @param ctx Context.
+     * @param idx Index.
+     */
+    private Map<String, ValidateIndexesPartitionResult> processIndex(GridCacheContext ctx, Index idx) {
+        Object consId = ignite.context().discovery().localNode().consistentId();
+
+        ValidateIndexesPartitionResult idxValidationRes = new ValidateIndexesPartitionResult(
+            -1, -1, true, consId, idx.getName());
+
+        boolean enoughIssues = false;
+
+        Cursor cursor = null;
+
+        try {
+            cursor = idx.find((Session)null, null, null);
+
+            if (cursor == null)
+                throw new IgniteCheckedException("Can't iterate through index: " + idx);
+        }
+        catch (Throwable t) {
+            IndexValidationIssue is = new IndexValidationIssue(null, ctx.name(), idx.getName(), t);
+
+            log.error("Find in index failed: " + is.toString());
+
+            enoughIssues = true;
+        }
+
+        while (!enoughIssues) {
+            KeyCacheObject h2key = null;
+
+            try {
+                if (!cursor.next())
+                    break;
+
+                GridH2Row h2Row = (GridH2Row)cursor.get();
+
+                h2key = h2Row.key();
+
+                CacheDataRow cacheDataStoreRow = ctx.group().offheap().read(ctx, h2key);
+
+                if (cacheDataStoreRow == null)
+                    throw new IgniteCheckedException("Key is present in SQL index, but can't be found in CacheDataTree.");
+            }
+            catch (Throwable t) {
+                Object o = CacheObjectUtils.unwrapBinaryIfNeeded(
+                    ctx.cacheObjectContext(), h2key, true, true);
+
+                IndexValidationIssue is = new IndexValidationIssue(
+                    String.valueOf(o), ctx.name(), idx.getName(), t);
+
+                log.error("Failed to lookup key: " + is.toString());
+
+                enoughIssues |= idxValidationRes.reportIssue(is);
+            }
+        }
+
+        String uniqueIdxName = "[cache=" + ctx.name() + ", idx=" + idx.getName() + "]";
+
+        processedIndexes.incrementAndGet();
+
+        printProgressIfNeeded();
+
+        return Collections.singletonMap(uniqueIdxName, idxValidationRes);
+    }
 }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
index 1a89c2cadb57e..52b48a58c0340 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
@@ -23,7 +23,6 @@
 import java.util.UUID;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.compute.ComputeJobResult;
-import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.visor.VisorJob;
@@ -81,10 +80,7 @@ protected VisorValidateIndexesJob(@Nullable VisorValidateIndexesTaskArg arg, boo
 
                 ignite.context().resource().injectGeneric(clo);
 
-                Map<PartitionKey, ValidateIndexesPartitionResult> res = clo.call();
-
-                return new VisorValidateIndexesJobResult(res);
-
+                return clo.call();
             }
             catch (Exception e) {
                 throw new IgniteException(e);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index bc99981372491..c89673626e83a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalPartitionedSelfTest;
 import org.apache.ignite.internal.processors.client.IgniteDataStreamerTest;
 import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelperTest;
+import org.apache.ignite.util.GridCommandHandlerIndexingTest;
 
 /**
  * Cache tests using indexing.
@@ -81,6 +82,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteDataStreamerTest.class);
 
+        suite.addTestSuite(GridCommandHandlerIndexingTest.class);
+
         return suite;
     }
 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
index 9e9c7770a644c..62d3fc02bdbaa 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -19,17 +19,32 @@
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.ThreadLocalRandom;
+import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.cache.QueryIndex;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.tree.SearchRow;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.util.lang.GridIterator;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
 
@@ -38,35 +53,201 @@
  */
 public class GridCommandHandlerIndexingTest extends GridCommandHandlerTest {
     /**
-     *
+     * Tests that validation doesn't fail if nothing is broken.
      */
-    public void testValidateIndexes() throws Exception {
+    public void testValidateIndexesNoErrors() throws Exception {
         Ignite ignite = startGrids(2);
 
         ignite.cluster().active(true);
 
         Ignite client = startGrid("client");
 
-        IgniteCache<Integer, Person> personCache = client.getOrCreateCache(new CacheConfiguration<Integer, Person>()
-            .setName("persons-cache-vi")
-            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
-            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
-            .setBackups(1)
-            .setQueryEntities(F.asList(personEntity(true, true)))
-            .setAffinity(new RendezvousAffinityFunction(false, 32)));
+        String cacheName = "persons-cache-vi";
+
+        IgniteCache<Integer, Person> personCache = createPersonCache(client, cacheName);
 
         ThreadLocalRandom rand = ThreadLocalRandom.current();
 
-        for (int i = 0; i < 1000; i++)
+        for (int i = 0; i < 10_000; i++)
             personCache.put(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong())));
 
         injectTestSystemOut();
 
-        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", "persons-cache-vi"));
+        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", cacheName));
 
         assertTrue(testOut.toString().contains("validate_indexes has finished, no issues found"));
     }
 
+    /**
+     * Tests that missing rows in CacheDataTree are detected.
+     */
+    public void testBrokenCacheDataTreeShouldFailValidation() throws Exception {
+        Ignite ignite = startGrids(2);
+
+        ignite.cluster().active(true);
+
+        Ignite client = startGrid("client");
+
+        String cacheName = "persons-cache-vi";
+
+        IgniteCache<Integer, Person> personCache = createPersonCache(client, cacheName);
+
+        ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+        for (int i = 0; i < 10_000; i++)
+            personCache.put(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong())));
+
+        breakCacheDataTree(ignite, cacheName, 1);
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", cacheName));
+
+        assertTrue(testOut.toString().contains("validate_indexes has finished with errors"));
+    }
+
+    /**
+     * Tests that missing rows in H2 indexes are detected.
+     */
+    public void testBrokenSqlIndexShouldFailValidation() throws Exception {
+        Ignite ignite = startGrids(2);
+
+        ignite.cluster().active(true);
+
+        Ignite client = startGrid("client");
+
+        String cacheName = "persons-cache-vi";
+
+        IgniteCache<Integer, Person> personCache = createPersonCache(client, cacheName);
+
+        ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+        for (int i = 0; i < 10_000; i++)
+            personCache.put(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong())));
+
+        breakSqlIndex(ignite, cacheName);
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", cacheName));
+
+        assertTrue(testOut.toString().contains("validate_indexes has finished with errors"));
+    }
+
+    /**
+     * Removes some entries from a partition skipping index update. This effectively breaks the index.
+     */
+    private void breakCacheDataTree(Ignite ig, String cacheName, int partId) {
+        IgniteEx ig0 = (IgniteEx)ig;
+        int cacheId = CU.cacheId(cacheName);
+
+        ScanQuery scanQry = new ScanQuery(partId);
+
+        GridCacheContext<Object, Object> ctx = ig0.context().cache().context().cacheContext(cacheId);
+
+        // Get current update counter
+        String grpName = ig0.context().cache().context().cacheContext(cacheId).config().getGroupName();
+        int cacheGrpId = grpName == null ? cacheName.hashCode() : grpName.hashCode();
+
+        GridDhtLocalPartition locPart = ctx.dht().topology().localPartition(partId);
+        IgniteCacheOffheapManager.CacheDataStore dataStore = ig0.context().cache().context().cache().cacheGroup(cacheGrpId).offheap().dataStore(locPart);
+
+        Iterator<Cache.Entry> it = ig.cache(cacheName).withKeepBinary().query(scanQry).iterator();
+
+        for (int i = 0; i < 5_000; i++) {
+            if (it.hasNext()) {
+                Cache.Entry entry = it.next();
+
+                if (i % 5 == 0) {
+                    // Do update
+                    GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig0.context().cache().context().database();
+
+                    db.checkpointReadLock();
+
+                    try {
+                        IgniteCacheOffheapManager.CacheDataStore innerStore = U.field(dataStore, "delegate");
+
+                        // IgniteCacheOffheapManagerImpl.CacheDataRowStore
+                        Object rowStore = U.field(innerStore, "rowStore");
+
+                        // IgniteCacheOffheapManagerImpl.CacheDataTree
+                        Object dataTree = U.field(innerStore, "dataTree");
+
+                        CacheDataRow oldRow = U.invoke(
+                            dataTree.getClass(),
+                            dataTree,
+                            "remove",
+                            new SearchRow(cacheId, ctx.toCacheKeyObject(entry.getKey())));
+
+                        if (oldRow != null)
+                            U.invoke(rowStore.getClass(), rowStore, "removeRow", oldRow.link());
+                    }
+                    catch (IgniteCheckedException e) {
+                        System.out.println("Failed to remove key skipping indexes: " + entry);
+
+                        e.printStackTrace();
+                    }
+                    finally {
+                        db.checkpointReadUnlock();
+                    }
+                }
+            }
+            else {
+                System.out.println("Early exit for index corruption, keys processed: " + i);
+
+                break;
+            }
+        }
+    }
+
+    /**
+     * Removes some entries from H2 trees skipping partition updates. This effectively breaks the index.
+     */
+    private void breakSqlIndex(Ignite ig, String cacheName) throws Exception {
+        GridQueryProcessor qry = ((IgniteEx)ig).context().query();
+
+        GridCacheContext<Object, Object> ctx = ((IgniteEx)ig).cachex(cacheName).context();
+
+        GridDhtLocalPartition locPart = ctx.topology().localPartitions().get(0);
+
+        GridIterator<CacheDataRow> it = ctx.group().offheap().partitionIterator(locPart.id());
+
+        for (int i = 0; i < 500; i++) {
+            if (!it.hasNextX()) {
+                System.out.println("Early exit for index corruption, keys processed: " + i);
+
+                break;
+            }
+
+            CacheDataRow row = it.nextX();
+
+            ctx.shared().database().checkpointReadLock();
+
+            try {
+                qry.remove(ctx, row);
+            }
+            finally {
+                ctx.shared().database().checkpointReadUnlock();
+            }
+        }
+    }
+
+    /**
+     * Dynamically creates cache with SQL indexes.
+     *
+     * @param ig Client.
+     * @param cacheName Cache name.
+     */
+    private IgniteCache<Integer, Person> createPersonCache(Ignite ig, String cacheName) {
+        return ig.getOrCreateCache(new CacheConfiguration<Integer, Person>()
+            .setName(cacheName)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setBackups(1)
+            .setQueryEntities(F.asList(personEntity(true, true)))
+            .setAffinity(new RendezvousAffinityFunction(false, 32)));
+    }
+
     /**
      * @param idxName Index name.
      * @param idxOrgId Index org id.

From 88479b21652aee66835ac0014c02dd378373e3ff Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 10 May 2018 20:00:12 +0300
Subject: [PATCH 142/543] IGNITE-7896 FilePageStore truncate now actually
 remove redundant partition page file.

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit d154eec)
---
 .../cache/persistence/file/FilePageStore.java |  23 +--
 .../IgnitePdsPartitionFilesTruncateTest.java  | 153 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 3 files changed, 170 insertions(+), 9 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesTruncateTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index fa02f5dcd3348..05f94218a7fc1 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -260,26 +260,31 @@ public void stop(boolean cleanFile) throws PersistentStorageIOException {
     public void truncate(int tag) throws PersistentStorageIOException {
         lock.writeLock().lock();
 
+        long pages = this.pages();
+
         try {
             if (!inited)
                 return;
 
             this.tag = tag;
 
-            fileIO.clear();
-
-            long newAlloc = initFile();
-
-            long delta = newAlloc - allocated.getAndSet(newAlloc);
-
-            assert delta % pageSize == 0;
-
-            allocatedTracker.updateTotalAllocatedPages(delta / pageSize);
+            try {
+                fileIO.close();
+            }
+            finally {
+                cfgFile.delete();
+            }
         }
         catch (IOException e) {
             throw new PersistentStorageIOException(e);
         }
         finally {
+            inited = false;
+
+            allocated.set(0);
+
+            allocatedTracker.updateTotalAllocatedPages(-1L * pages);
+
             lock.writeLock().unlock();
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesTruncateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesTruncateTest.java
new file mode 100644
index 0000000000000..78c2453df08a7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesTruncateTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Checks that evicted partitions doesn't leave files in PDS.
+ */
+public class IgnitePdsPartitionFilesTruncateTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setConsistentId(gridName)
+                .setDataStorageConfiguration(new DataStorageConfiguration()
+                            .setPageSize(1024)
+                            .setWalMode(WALMode.LOG_ONLY)
+                            .setDefaultDataRegionConfiguration(
+                                new DataRegionConfiguration()
+                                        .setPersistenceEnabled(true)))
+                .setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME)
+                        .setBackups(1)
+                        .setAffinity(new RendezvousAffinityFunction(false, 32)));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTruncatingPartitionFilesOnEviction() throws Exception {
+        Ignite ignite0 = startGrids(3);
+
+        ignite0.cluster().active(true);
+
+        try (IgniteDataStreamer<Integer,String> streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) {
+            for (int i = 0; i < 1_000; i++)
+                streamer.addData(i, "Value " + i);
+        }
+
+        assertEquals(1, ignite0.cacheNames().size());
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkPartFiles(0);
+        checkPartFiles(1);
+        checkPartFiles(2);
+
+        stopGrid(2);
+
+        ignite0.cluster().setBaselineTopology(ignite0.cluster().topologyVersion());
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkPartFiles(0);
+        checkPartFiles(1);
+
+        startGrid(2);
+
+        ignite0.cluster().setBaselineTopology(ignite0.cluster().topologyVersion());
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkPartFiles(0);
+        checkPartFiles(1);
+        checkPartFiles(2);
+    }
+
+    /**
+     * @param idx Node index.
+     */
+    private void checkPartFiles(int idx) throws Exception {
+        Ignite ignite = grid(idx);
+
+        int[] parts = ignite.affinity(DEFAULT_CACHE_NAME).allPartitions(ignite.cluster().localNode());
+
+        Path dirPath = Paths.get(U.defaultWorkDirectory(), "db",
+                U.maskForFileName(ignite.configuration().getIgniteInstanceName()), "cache-" + DEFAULT_CACHE_NAME);
+
+        info("Path: " + dirPath.toString());
+
+        assertTrue(Files.exists(dirPath));
+
+        for (Path f : Files.newDirectoryStream(dirPath)) {
+            if (f.getFileName().toString().startsWith("part-"))
+                assertTrue("Node_" + idx +" should contains only partitions " + Arrays.toString(parts)
+                        + ", but the file is redundant: " + f.getFileName(), anyMatch(parts, f));
+        }
+    }
+
+    /** */
+    private boolean anyMatch(int[] parts, Path f) {
+        Pattern ptrn = Pattern.compile("part-(\\d+).bin");
+        Matcher matcher = ptrn.matcher(f.getFileName().toString());
+
+        if (!matcher.find())
+            throw new IllegalArgumentException("File is not a partition:" + f.getFileName());
+
+        int part = Integer.parseInt(matcher.group(1));
+
+        for (int p: parts) {
+            if (p == part)
+                return true;
+        }
+
+        return false;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 71c6c32a983cb..851d942ae8eb6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCorruptedStoreTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchangeDuringCheckpointTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPartitionFilesTruncateTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
 import org.apache.ignite.internal.processors.cache.persistence.LocalWalModeChangeDuringRebalancingSelfTest;
@@ -101,6 +102,8 @@ private static void addRealPageStoreTestsLongRunning(TestSuite suite) {
     public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgnitePdsPageSizesTest.class);
 
+        suite.addTestSuite(IgnitePdsPartitionFilesTruncateTest.class);
+
         // Metrics test.
         suite.addTestSuite(IgniteDataStorageMetricsSelfTest.class);
 

From fac043bd1d06a08bffff1426a6e83a267cbd843f Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 17 May 2018 12:19:36 +0300
Subject: [PATCH 143/543] IGNITE-8320 Partition file can be truncated only
 after checkpoint - Fixes #3985.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 8cb35e1)
---
 .../dht/GridDhtLocalPartition.java            |  70 ++-
 .../GridCacheDatabaseSharedManager.java       | 383 +++++++++++++--
 .../persistence/GridCacheOffheapManager.java  |  44 +-
 .../cache/persistence/file/FilePageStore.java |  43 +-
 .../serializer/RecordDataV2Serializer.java    |   2 +-
 .../IgnitePdsCorruptedIndexTest.java          | 341 ++++++++++++++
 .../IgnitePdsPartitionFilesDestroyTest.java   | 444 ++++++++++++++++++
 .../IgnitePdsPartitionFilesTruncateTest.java  | 153 ------
 .../junits/GridAbstractTest.java              |   9 +-
 .../junits/multijvm/IgniteProcessProxy.java   |  23 +-
 .../ignite/testsuites/IgnitePdsTestSuite.java |   2 +
 .../testsuites/IgnitePdsTestSuite2.java       |   8 +-
 .../query/h2/database/InlineIndexHelper.java  |   1 -
 .../IgnitePdsWithIndexingCoreTestSuite.java   |   3 +
 14 files changed, 1281 insertions(+), 245 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesTruncateTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index be74eff5cf674..398b0d5880f9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.NodeStoppingException;
@@ -89,6 +90,12 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
     /** Maximum size for delete queue. */
     public static final int MAX_DELETE_QUEUE_SIZE = Integer.getInteger(IGNITE_ATOMIC_CACHE_DELETE_HISTORY_SIZE, 200_000);
 
+    /** ONLY FOR TEST PURPOSES: force test checkpoint on partition eviction. */
+    private static boolean forceTestCheckpointOnEviction = IgniteSystemProperties.getBoolean("TEST_CHECKPOINT_ON_EVICTION", false);
+
+    /** ONLY FOR TEST PURPOSES: partition id where test checkpoint was enforced during eviction. */
+    static volatile Integer partWhereTestCheckpointEnforced;
+
     /** Maximum size for {@link #rmvQueue}. */
     private final int rmvQueueMaxSize;
 
@@ -210,6 +217,10 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
         try {
             store = grp.offheap().createCacheDataStore(id);
 
+            // Log partition creation for further crash recovery purposes.
+            if (grp.walEnabled())
+                ctx.wal().log(new PartitionMetaStateRecord(grp.groupId(), id, state(), updateCounter()));
+
             // Inject row cache cleaner on store creation
             // Used in case the cache with enabled SqlOnheapCache is single cache at the cache group
             if (ctx.kernalContext().query().moduleEnabled()) {
@@ -1036,16 +1047,25 @@ private long clearAll() throws NodeStoppingException {
                     catch (GridDhtInvalidPartitionException e) {
                         assert isEmpty() && state() == EVICTED : "Invalid error [e=" + e + ", part=" + this + ']';
 
-                        break; // Partition is already concurrently cleared and evicted.
-                    }
-                    finally {
-                        ctx.database().checkpointReadUnlock();
-                    }
+                    break; // Partition is already concurrently cleared and evicted.
+                }
+                finally {
+                    ctx.database().checkpointReadUnlock();
                 }
             }
-            catch (NodeStoppingException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Failed to get iterator for evicted partition: " + id);
+
+        if (forceTestCheckpointOnEviction) {
+                if (partWhereTestCheckpointEnforced == null && cleared >= fullSize()) {
+                    ctx.database().forceCheckpoint("test").finishFuture().get();
+
+                    log.warning("Forced checkpoint by test reasons for partition: " + this);
+
+                    partWhereTestCheckpointEnforced = id;
+                }
+            }
+        }catch (NodeStoppingException e) {
+            if (log.isDebugEnabled())
+                log.debug("Failed to get iterator for evicted partition: " + id);
 
                 throw e;
             }
@@ -1337,10 +1357,10 @@ long expireTime() {
      */
     class ClearFuture extends GridFutureAdapter<Boolean> {
         /** Flag indicates that eviction callback was registered on the current future. */
-        private volatile boolean evictionCallbackRegistered;
+        private volatile boolean evictionCbRegistered;
 
         /** Flag indicates that clearing callback was registered on the current future. */
-        private volatile boolean clearingCallbackRegistered;
+        private volatile boolean clearingCbRegistered;
 
         /** Flag indicates that future with all callbacks was finished. */
         private volatile boolean finished;
@@ -1359,25 +1379,29 @@ class ClearFuture extends GridFutureAdapter<Boolean> {
          * @param updateSeq If {@code true} update topology sequence after successful eviction.
          */
         private void registerEvictionCallback(boolean updateSeq) {
-            if (evictionCallbackRegistered)
+            if (evictionCbRegistered)
                 return;
 
             synchronized (this) {
                 // Double check
-                if (evictionCallbackRegistered)
+                if (evictionCbRegistered)
                     return;
 
-                evictionCallbackRegistered = true;
+                evictionCbRegistered = true;
 
                 // Initiates partition eviction and destroy.
                 listen(f -> {
-                    if (f.error() != null) {
-                        rent.onDone(f.error());
-                    } else if (f.isDone()) {
+                    try {
+                        // Check for errors.
+                        f.get();
+
                         finishEviction(updateSeq);
                     }
+                    catch (Exception e) {
+                        rent.onDone(e);
+                    }
 
-                    evictionCallbackRegistered = false;
+                    evictionCbRegistered = false;
                 });
             }
         }
@@ -1386,21 +1410,21 @@ private void registerEvictionCallback(boolean updateSeq) {
          * Registers clearing callback on the future.
          */
         private void registerClearingCallback() {
-            if (clearingCallbackRegistered)
+            if (clearingCbRegistered)
                 return;
 
             synchronized (this) {
                 // Double check
-                if (clearingCallbackRegistered)
+                if (clearingCbRegistered)
                     return;
 
-                clearingCallbackRegistered = true;
+                clearingCbRegistered = true;
 
                 // Recreate cache data store in case of allowed fast eviction, and reset clear flag.
                 listen(f -> {
                     clear = false;
 
-                    clearingCallbackRegistered = false;
+                    clearingCbRegistered = false;
                 });
             }
         }
@@ -1481,8 +1505,8 @@ public boolean initialize(boolean updateSeq, boolean evictionRequested) {
                     reset();
 
                     finished = false;
-                    evictionCallbackRegistered = false;
-                    clearingCallbackRegistered = false;
+                    evictionCbRegistered = false;
+                    clearingCbRegistered = false;
                 }
 
                 if (evictionRequested)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 12cfc05d72e2e..a542da3f9970a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -21,7 +21,6 @@
 import java.io.FileFilter;
 import java.io.IOException;
 import java.io.RandomAccessFile;
-import java.io.Serializable;
 import java.lang.ref.SoftReference;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -85,7 +84,6 @@
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.mem.DirectMemoryRegion;
 import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
 import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.FullPageId;
@@ -296,6 +294,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
         }
     };
 
+    /** Timeout between partition file destroy and checkpoint to handle it. */
+    private static final long PARTITION_DESTROY_CHECKPOINT_TIMEOUT = 30 * 1000; // 30 Seconds.
+
     /** Checkpoint thread. Needs to be volatile because it is created in exchange worker. */
     private volatile Checkpointer checkpointer;
 
@@ -389,6 +390,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Initially disabled cache groups. */
     private Collection<Integer> initiallyGlobalWalDisabledGrps = new HashSet<>();
 
+    /** Initially local wal disabled groups. */
     private Collection<Integer> initiallyLocalWalDisabledGrps = new HashSet<>();
 
     /**
@@ -498,6 +500,8 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
         final GridKernalContext kernalCtx = cctx.kernalContext();
 
         if (!kernalCtx.clientNode()) {
+            checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log);
+
             IgnitePageStoreManager store = cctx.pageStore();
 
             assert store instanceof FilePageStoreManager : "Invalid page store manager was created: " + store;
@@ -632,6 +636,9 @@ else if (regCfg.getMaxSize() < 8 * GB)
             registrateMetricsMBean();
         }
 
+        if (checkpointer == null)
+            checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log);
+
         super.onActivate(ctx);
     }
 
@@ -1441,8 +1448,6 @@ private void restoreState() throws IgniteCheckedException {
 
             snapshotMgr.restoreState();
 
-            checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log);
-
             new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
 
             CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started");
@@ -1920,20 +1925,23 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
      * @throws IgniteCheckedException If failed.
      * @throws StorageException In case I/O error occurred during operations with storage.
      */
-    private @Nullable WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException {
+    @Nullable private WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException {
         return restoreMemory(status, false, (PageMemoryEx)metaStorage.pageMemory());
     }
 
     /**
      * @param status Checkpoint status.
-     * @param storeOnly If {@code True} restores Metastorage only.
+     * @param metastoreOnly If {@code True} restores Metastorage only.
      * @param storePageMem Metastore page memory.
      * @throws IgniteCheckedException If failed.
      * @throws StorageException In case I/O error occurred during operations with storage.
      */
-    private @Nullable WALPointer restoreMemory(CheckpointStatus status, boolean storeOnly,
-        PageMemoryEx storePageMem) throws IgniteCheckedException {
-        assert !storeOnly || storePageMem != null;
+    @Nullable private WALPointer restoreMemory(
+        CheckpointStatus status,
+        boolean metastoreOnly,
+        PageMemoryEx storePageMem
+    ) throws IgniteCheckedException {
+        assert !metastoreOnly || storePageMem != null;
 
         if (log.isInfoEnabled())
             log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked="
@@ -1954,7 +1962,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
         int applied = 0;
         WALPointer lastRead = null;
 
-        Collection<Integer> ignoreGrps = storeOnly ? Collections.emptySet() :
+        Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() :
             F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps);
 
         try (WALIterator it = cctx.wal().replay(status.endPtr)) {
@@ -1990,7 +1998,7 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
                             // several repetitive restarts and the same pages may have changed several times.
                             int grpId = pageRec.fullPageId().groupId();
 
-                            if (storeOnly && grpId != METASTORAGE_CACHE_ID)
+                            if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
                                 continue;
 
                             if (!ignoreGrps.contains(grpId)) {
@@ -2020,22 +2028,47 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
 
                         break;
 
+                    case PART_META_UPDATE_STATE:
+                        PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord)rec;
+
+                        {
+                            int grpId = metaStateRecord.groupId();
+
+                            if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
+                                continue;
+
+                            if (ignoreGrps.contains(grpId))
+                                continue;
+
+                            int partId = metaStateRecord.partitionId();
+
+                            GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(metaStateRecord.state());
+
+                            if (state == null || state == GridDhtPartitionState.EVICTED)
+                                schedulePartitionDestroy(grpId, partId);
+                            else
+                                cancelOrWaitPartitionDestroy(grpId, partId);
+                        }
+
+                        break;
+
                     case PARTITION_DESTROY:
-                        PartitionDestroyRecord destroyRec = (PartitionDestroyRecord)rec;
+                        PartitionDestroyRecord destroyRecord = (PartitionDestroyRecord)rec;
 
-                        final int gId = destroyRec.groupId();
+                        {
+                            int grpId = destroyRecord.groupId();
 
-                        if (storeOnly && gId != METASTORAGE_CACHE_ID)
-                            continue;
+                            if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
+                                continue;
+
+                            if (ignoreGrps.contains(grpId))
+                                continue;
 
-                        if (!ignoreGrps.contains(gId)) {
-                            final int pId = destroyRec.partitionId();
+                            PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
 
-                            PageMemoryEx pageMem = gId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(gId);
+                            pageMem.invalidate(grpId, destroyRecord.partitionId());
 
-                            pageMem.clearAsync(
-                                (grpId, pageId) -> grpId == gId && PageIdUtils.partId(pageId) == pId,
-                                true).get();
+                            schedulePartitionDestroy(grpId, destroyRecord.partitionId());
                         }
 
                         break;
@@ -2046,7 +2079,7 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
 
                             int grpId = r.groupId();
 
-                            if (storeOnly && grpId != METASTORAGE_CACHE_ID)
+                            if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
                                 continue;
 
                             if (!ignoreGrps.contains(grpId)) {
@@ -2079,7 +2112,7 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
             }
         }
 
-        if (storeOnly)
+        if (metastoreOnly)
             return null;
 
         if (status.needRestoreMemory()) {
@@ -2691,6 +2724,170 @@ private CheckpointEntry createCheckPointEntry(
         return new CheckpointEntry(cpTs, ptr, cpId, cacheGrpStates);
     }
 
+    /**
+     * Adds given partition to checkpointer destroy queue.
+     *
+     * @param grpId Group ID.
+     * @param partId Partition ID.
+     */
+    public void schedulePartitionDestroy(int grpId, int partId) {
+        Checkpointer cp = checkpointer;
+
+        if (cp != null)
+            cp.schedulePartitionDestroy(cctx.cache().cacheGroup(grpId), grpId, partId);
+    }
+
+    /**
+     * Cancels or wait for partition destroy.
+     *
+     * @param grpId Group ID.
+     * @param partId Partition ID.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void cancelOrWaitPartitionDestroy(int grpId, int partId) throws IgniteCheckedException {
+        Checkpointer cp = checkpointer;
+
+        if (cp != null)
+            cp.cancelOrWaitPartitionDestroy(grpId, partId);
+    }
+
+    /**
+     * Partition destroy queue.
+     */
+    private static class PartitionDestroyQueue {
+        /** */
+        private final ConcurrentMap<T2<Integer, Integer>, PartitionDestroyRequest> pendingReqs =
+            new ConcurrentHashMap<>();
+
+        /**
+         * @param grpCtx Group context.
+         * @param partId Partition ID to destroy.
+         */
+        private void addDestroyRequest(@Nullable CacheGroupContext grpCtx, int grpId, int partId) {
+            PartitionDestroyRequest req = new PartitionDestroyRequest(grpId, partId);
+
+            PartitionDestroyRequest old = pendingReqs.putIfAbsent(new T2<>(grpId, partId), req);
+
+            assert old == null || grpCtx == null : "Must wait for old destroy request to finish before adding a new one "
+                + "[grpId=" + grpId
+                + ", grpName=" + grpCtx.cacheOrGroupName()
+                + ", partId=" + partId + ']';
+        }
+
+        /**
+         * @param destroyId Destroy ID.
+         * @return Destroy request to complete if was not concurrently cancelled.
+         */
+        private PartitionDestroyRequest beginDestroy(T2<Integer, Integer> destroyId) {
+            PartitionDestroyRequest rmvd = pendingReqs.remove(destroyId);
+
+            return rmvd == null ? null : rmvd.beginDestroy() ? rmvd : null;
+        }
+
+        /**
+         * @param grpId Group ID.
+         * @param partId Partition ID.
+         * @return Destroy request to wait for if destroy has begun.
+         */
+        private PartitionDestroyRequest cancelDestroy(int grpId, int partId) {
+            PartitionDestroyRequest rmvd = pendingReqs.remove(new T2<>(grpId, partId));
+
+            return rmvd == null ? null : !rmvd.cancel() ? rmvd : null;
+        }
+    }
+
+    /**
+     * Partition destroy request.
+     */
+    private static class PartitionDestroyRequest {
+        /** */
+        private final int grpId;
+
+        /** */
+        private final int partId;
+
+        /** Destroy cancelled flag. */
+        private boolean cancelled;
+
+        /** Destroy future. Not null if partition destroy has begun. */
+        private GridFutureAdapter<Void> destroyFut;
+
+        /**
+         * @param grpId Group ID.
+         * @param partId Partition ID.
+         */
+        private PartitionDestroyRequest(int grpId, int partId) {
+            this.grpId = grpId;
+            this.partId = partId;
+        }
+
+        /**
+         * Cancels partition destroy request.
+         *
+         * @return {@code False} if this request needs to be waited for.
+         */
+        private synchronized boolean cancel() {
+            if (destroyFut != null) {
+                assert !cancelled;
+
+                return false;
+            }
+
+            cancelled = true;
+
+            return true;
+        }
+
+        /**
+         * Initiates partition destroy.
+         *
+         * @return {@code True} if destroy request should be executed, {@code false} otherwise.
+         */
+        private synchronized boolean beginDestroy() {
+            if (cancelled) {
+                assert destroyFut == null;
+
+                return false;
+            }
+
+            if (destroyFut != null)
+                return false;
+
+            destroyFut = new GridFutureAdapter<>();
+
+            return true;
+        }
+
+        /**
+         *
+         */
+        private synchronized void onDone(Throwable err) {
+            assert destroyFut != null;
+
+            destroyFut.onDone(err);
+        }
+
+        /**
+         *
+         */
+        private void waitCompleted() throws IgniteCheckedException {
+            GridFutureAdapter<Void> fut;
+
+            synchronized (this) {
+                assert destroyFut != null;
+
+                fut = destroyFut;
+            }
+
+            fut.get();
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "PartitionDestroyRequest [grpId=" + grpId + ", partId=" + partId + ']';
+        }
+    }
+
     /**
      * Checkpointer object is used for notification on checkpoint begin, predicate is {@link #scheduledCp}<code>.nextCpTs - now
      * > 0 </code>. Method {@link #wakeupForCheckpoint} uses notify, {@link #waitCheckpointEvent} uses wait
@@ -2704,7 +2901,7 @@ public class Checkpointer extends GridWorker {
         private volatile CheckpointProgress scheduledCp;
 
         /** Current checkpoint. This field is updated only by checkpoint thread. */
-        private volatile CheckpointProgress curCpProgress;
+        @Nullable private volatile CheckpointProgress curCpProgress;
 
         /** Shutdown now. */
         private volatile boolean shutdownNow;
@@ -2850,7 +3047,7 @@ private void doCheckpoint() {
                 syncedPagesCntr = new AtomicInteger();
                 evictedPagesCntr = new AtomicInteger();
 
-                boolean interrupted = true;
+                boolean success = false;
 
                 try {
                     if (chp.hasDelta()) {
@@ -2937,11 +3134,22 @@ private void doCheckpoint() {
 
                     snapshotMgr.afterCheckpointPageWritten();
 
+                    try {
+                        destroyEvictedPartitions();
+                    }
+                    catch (IgniteCheckedException e) {
+                        chp.progress.cpFinishFut.onDone(e);
+
+                        cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+                        return;
+                    }
+
                     // Must mark successful checkpoint only if there are no exceptions or interrupts.
-                    interrupted = false;
+                    success = true;
                 }
                 finally {
-                    if (!interrupted)
+                    if (success)
                         markCheckpointEnd(chp);
                 }
 
@@ -2991,6 +3199,122 @@ private void doCheckpoint() {
             }
         }
 
+        /**
+         * Processes all evicted partitions scheduled for destroy.
+         *
+         * @throws IgniteCheckedException If failed.
+         */
+        private void destroyEvictedPartitions() throws IgniteCheckedException {
+            PartitionDestroyQueue destroyQueue = curCpProgress.destroyQueue;
+
+            if (destroyQueue.pendingReqs.isEmpty())
+                return;
+
+            List<PartitionDestroyRequest> reqs = null;
+
+            for (final PartitionDestroyRequest req : destroyQueue.pendingReqs.values()) {
+                if (!req.beginDestroy())
+                    continue;
+
+                final int grpId = req.grpId;
+                final int partId = req.partId;
+
+                CacheGroupContext grp = cctx.cache().cacheGroup(grpId);
+
+                assert grp != null
+                    : "Cache group is not initialized [grpId=" + grpId + "]";
+                assert grp.offheap() instanceof GridCacheOffheapManager
+                    : "Destroying partition files when persistence is off " + grp.offheap();
+
+                final GridCacheOffheapManager offheap = (GridCacheOffheapManager) grp.offheap();
+
+                Runnable destroyPartTask = () -> {
+                    try {
+                        offheap.destroyPartitionStore(grpId, partId);
+
+                        req.onDone(null);
+
+                        if (log.isDebugEnabled())
+                            log.debug("Partition file has destroyed [grpId=" + grpId + ", partId=" + partId + "]");
+                    }
+                    catch (Exception e) {
+                        req.onDone(new IgniteCheckedException(
+                            "Partition file destroy has failed [grpId=" + grpId + ", partId=" + partId + "]", e));
+                    }
+                };
+
+                if (asyncRunner != null) {
+                    try {
+                        asyncRunner.execute(destroyPartTask);
+                    }
+                    catch (RejectedExecutionException ignore) {
+                        // Run the task synchronously.
+                        destroyPartTask.run();
+                    }
+                }
+                else
+                    destroyPartTask.run();
+
+                if (reqs == null)
+                    reqs = new ArrayList<>();
+
+                reqs.add(req);
+            }
+
+            if (reqs != null)
+                for (PartitionDestroyRequest req : reqs)
+                    req.waitCompleted();
+
+            destroyQueue.pendingReqs.clear();
+        }
+
+        /**
+         * @param grpCtx Group context. Can be {@code null} in case of crash recovery.
+         * @param grpId Group ID.
+         * @param partId Partition ID.
+         */
+        private void schedulePartitionDestroy(@Nullable CacheGroupContext grpCtx, int grpId, int partId) {
+            synchronized (this) {
+                scheduledCp.destroyQueue.addDestroyRequest(grpCtx, grpId, partId);
+            }
+
+            if (log.isDebugEnabled())
+                log.debug("Partition file has been scheduled to destroy [grpId=" + grpId + ", partId=" + partId + "]");
+
+            if (grpCtx != null)
+                wakeupForCheckpoint(PARTITION_DESTROY_CHECKPOINT_TIMEOUT, "partition destroy");
+        }
+
+        /**
+         * @param grpId Group ID.
+         * @param partId Partition ID.
+         */
+        private void cancelOrWaitPartitionDestroy(int grpId, int partId) throws IgniteCheckedException {
+            PartitionDestroyRequest req;
+
+            synchronized (this) {
+                req = scheduledCp.destroyQueue.cancelDestroy(grpId, partId);
+            }
+
+            if (req != null)
+                req.waitCompleted();
+
+            CheckpointProgress cur;
+
+            synchronized (this) {
+                cur = curCpProgress;
+
+                if (cur != null)
+                    req = cur.destroyQueue.cancelDestroy(grpId, partId);
+            }
+
+            if (req != null)
+                req.waitCompleted();
+
+            if (log.isDebugEnabled())
+                log.debug("Partition file destroy has cancelled [grpId=" + grpId + ", partId=" + partId + "]");
+        }
+
         /**
          *
          */
@@ -3136,7 +3460,7 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
                 }
             }
 
-            if (hasPages) {
+            if (hasPages || !curr.destroyQueue.pendingReqs.isEmpty()) {
                 assert cpPtr != null;
 
                 tracker.onWalCpRecordFsyncStart();
@@ -3611,6 +3935,9 @@ private static class CheckpointProgress {
         /** */
         private volatile SnapshotOperation snapshotOperation;
 
+        /** Partitions destroy queue. */
+        private final PartitionDestroyQueue destroyQueue = new PartitionDestroyQueue();
+
         /** Wakeup reason. */
         private String reason;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 5feaa252dd3fa..50c90e1de6860 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -148,8 +148,10 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
     }
 
     /** {@inheritDoc} */
-    @Override protected CacheDataStore createCacheDataStore0(final int p)
-        throws IgniteCheckedException {
+    @Override protected CacheDataStore createCacheDataStore0(int p) throws IgniteCheckedException {
+        if (ctx.database() instanceof GridCacheDatabaseSharedManager)
+            ((GridCacheDatabaseSharedManager) ctx.database()).cancelOrWaitPartitionDestroy(grp.groupId(), p);
+
         boolean exists = ctx.pageStore() != null && ctx.pageStore().exists(grp.groupId(), p);
 
         return new GridCacheDataStore(p, exists);
@@ -575,25 +577,41 @@ private static boolean addPartition(
 
     /** {@inheritDoc} */
     @Override protected void destroyCacheDataStore0(CacheDataStore store) throws IgniteCheckedException {
+        assert ctx.database() instanceof GridCacheDatabaseSharedManager
+            : "Destroying cache data store when persistence is not enabled: " + ctx.database();
+
+        int partId = store.partId();
+
         ctx.database().checkpointReadLock();
 
         try {
-            int p = store.partId();
-
             saveStoreMetadata(store, null, false, true);
-
-            PageMemoryEx pageMemory = (PageMemoryEx)grp.dataRegion().pageMemory();
-
-            int tag = pageMemory.invalidate(grp.groupId(), p);
-
-            if (grp.walEnabled())
-                ctx.wal().log(new PartitionDestroyRecord(grp.groupId(), p));
-
-            ctx.pageStore().onPartitionDestroyed(grp.groupId(), p, tag);
         }
         finally {
             ctx.database().checkpointReadUnlock();
         }
+
+        ((GridCacheDatabaseSharedManager)ctx.database()).schedulePartitionDestroy(grp.groupId(), partId);
+    }
+
+    /**
+     * Invalidates page memory for given partition. Destroys partition store.
+     * <b>NOTE:</b> This method can be invoked only within checkpoint lock or checkpointer thread.
+     *
+     * @param grpId Group ID.
+     * @param partId Partition ID.
+     *
+     * @throws IgniteCheckedException If destroy has failed.
+     */
+    public void destroyPartitionStore(int grpId, int partId) throws IgniteCheckedException {
+        PageMemoryEx pageMemory = (PageMemoryEx)grp.dataRegion().pageMemory();
+
+        int tag = pageMemory.invalidate(grp.groupId(), partId);
+
+        if (grp.walEnabled())
+            ctx.wal().log(new PartitionDestroyRecord(grp.groupId(), partId));
+
+        ctx.pageStore().onPartitionDestroyed(grpId, partId, tag);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 05f94218a7fc1..ae4880d22715c 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.file.Files;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -244,7 +245,7 @@ public void stop(boolean cleanFile) throws PersistentStorageIOException {
             fileIO.close();
 
             if (cleanFile)
-                cfgFile.delete();
+                Files.delete(cfgFile.toPath());
         }
         catch (IOException e) {
             throw new PersistentStorageIOException(e);
@@ -255,35 +256,34 @@ public void stop(boolean cleanFile) throws PersistentStorageIOException {
     }
 
     /**
+     * Truncates and deletes partition file.
      *
+     * @param tag New partition tag.
+     * @throws PersistentStorageIOException If failed
      */
     public void truncate(int tag) throws PersistentStorageIOException {
-        lock.writeLock().lock();
+        init();
 
-        long pages = this.pages();
+        lock.writeLock().lock();
 
         try {
-            if (!inited)
-                return;
-
             this.tag = tag;
 
-            try {
-                fileIO.close();
-            }
-            finally {
-                cfgFile.delete();
-            }
+            fileIO.clear();
+
+            fileIO.close();
+
+            Files.delete(cfgFile.toPath());
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException(e);
+            throw new PersistentStorageIOException("Failed to delete partition file: " + cfgFile.getPath(), e);
         }
         finally {
-            inited = false;
-
             allocated.set(0);
 
-            allocatedTracker.updateTotalAllocatedPages(-1L * pages);
+            allocatedTracker.updateTotalAllocatedPages(-1L * this.pages());
+
+            inited = false;
 
             lock.writeLock().unlock();
         }
@@ -325,7 +325,7 @@ public void finishRecover() throws PersistentStorageIOException {
             recover = false;
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Unable to finish recover", e);
+            throw new PersistentStorageIOException("Failed to finish recover", e);
         }
         finally {
             lock.writeLock().unlock();
@@ -421,9 +421,9 @@ public void finishRecover() throws PersistentStorageIOException {
     }
 
     /**
-     * @throws IgniteCheckedException If failed to initialize store file.
+     * @throws PersistentStorageIOException If failed to initialize store file.
      */
-    private void init() throws IgniteCheckedException {
+    private void init() throws PersistentStorageIOException {
         if (!inited) {
             lock.writeLock().lock();
 
@@ -431,7 +431,7 @@ private void init() throws IgniteCheckedException {
                 if (!inited) {
                     FileIO fileIO = null;
 
-                    IgniteCheckedException err = null;
+                    PersistentStorageIOException err = null;
 
                     try {
                         this.fileIO = fileIO = ioFactory.create(cfgFile, CREATE, READ, WRITE);
@@ -451,7 +451,8 @@ private void init() throws IgniteCheckedException {
                         inited = true;
                     }
                     catch (IOException e) {
-                        err = new PersistentStorageIOException("Could not initialize file: " + cfgFile.getName(), e);
+                        err = new PersistentStorageIOException(
+                            "Failed to initialize partition file: " + cfgFile.getName(), e);
 
                         throw err;
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
index efba611d312cc..b3a00beeb6aa8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
@@ -92,7 +92,7 @@ public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) {
                 return 8 + 1;
 
             case EXCHANGE:
-               return 4 /*type*/ + 8 /*timestamp*/ + 2 /*constId*/;
+                return 4 /*type*/ + 8 /*timestamp*/ + 2 /*constId*/;
 
             case TX_RECORD:
                 return txRecordSerializer.size((TxRecord)rec);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
new file mode 100644
index 0000000000000..a1065f6b4b618
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
@@ -0,0 +1,341 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.OpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.Cache;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy;
+import org.junit.Assert;
+
+/**
+ * Test to reproduce corrupted indexes problem after partition file eviction and truncation.
+ */
+public class IgnitePdsCorruptedIndexTest extends GridCommonAbstractTest {
+    /** Cache name. */
+    private static final String CACHE = "cache";
+
+    /** Flag indicates that {@link HaltOnTruncateFileIO} should be used. */
+    private boolean haltFileIO;
+
+    /** MultiJVM flag. */
+    private boolean multiJvm = true;
+
+    /** Additional remote JVM args. */
+    private List<String> additionalArgs = Collections.emptyList();
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+            .setWalMode(WALMode.LOG_ONLY)
+            .setCheckpointFrequency(10 * 60 * 1000)
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(512 * 1024 * 1024)
+                    .setPersistenceEnabled(true)
+            );
+
+        if (haltFileIO)
+            dsCfg.setFileIOFactory(new HaltOnTruncateFileIOFactory(new RandomAccessFileIOFactory()));
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(CACHE)
+            .setBackups(1)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setIndexedTypes(Integer.class, IndexedObject.class, Long.class, IndexedObject.class)
+            .setAffinity(new RendezvousAffinityFunction(false, 32));
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean isMultiJvm() {
+        return multiJvm;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected List<String> additionalRemoteJvmArgs() {
+        return additionalArgs;
+    }
+
+    /**
+     *
+     */
+    public void testCorruption() throws Exception {
+        final String corruptedNodeName = "corrupted";
+
+        IgniteEx ignite = startGrid(0);
+
+        haltFileIO = true;
+
+        additionalArgs = new ArrayList<>();
+        additionalArgs.add("-D" + "TEST_CHECKPOINT_ON_EVICTION=true");
+        additionalArgs.add("-D" + "IGNITE_QUIET=false");
+
+        IgniteEx corruptedNode = (IgniteEx) startGrid(corruptedNodeName);
+
+        additionalArgs.clear();
+
+        haltFileIO = false;
+
+        startGrid(2);
+
+        ignite.cluster().active(true);
+
+        awaitPartitionMapExchange();
+
+        final int entityCnt = 3_200;
+
+        try (IgniteDataStreamer<Object, Object> streamer = ignite.dataStreamer(CACHE)) {
+            streamer.allowOverwrite(true);
+
+            for (int i = 0; i < entityCnt; i++)
+                streamer.addData(i, new IndexedObject(i));
+        }
+
+        startGrid(3);
+
+        resetBaselineTopology();
+
+        // Corrupted node should be halted during partition destroy.
+        GridTestUtils.waitForCondition(() -> ignite.cluster().nodes().size() == 3, getTestTimeout());
+
+        // Clear remote JVM instance cache.
+        IgniteProcessProxy.kill(corruptedNode.name());
+
+        stopAllGrids();
+
+        // Disable multi-JVM mode and start coordinator and corrupted node in the same JVM.
+        multiJvm = false;
+
+        startGrid(0);
+
+        corruptedNode = (IgniteEx) startGrid(corruptedNodeName);
+
+        corruptedNode.cluster().active(true);
+
+        resetBaselineTopology();
+
+        // If index was corrupted, rebalance or one of the following queries should be failed.
+        awaitPartitionMapExchange();
+
+        for (int k = 0; k < entityCnt; k += entityCnt / 4) {
+            IgniteCache<Integer, IndexedObject> cache1 = corruptedNode.cache(CACHE);
+
+            int l = k;
+            int r = k + entityCnt / 4 - 1;
+
+            log.info("Check range [" + l + "-" + r + "]");
+
+            QueryCursor<Cache.Entry<Long, IndexedObject>> qry =
+                cache1.query(new SqlQuery(IndexedObject.class, "lVal between ? and ?")
+                    .setArgs(l * l, r * r));
+
+            Collection<Cache.Entry<Long, IndexedObject>> queried = qry.getAll();
+
+            log.info("Qry result size = " + queried.size());
+        }
+    }
+
+    /**
+     *
+     */
+    private static class IndexedObject {
+        /** Integer indexed value. */
+        @QuerySqlField(index = true)
+        private int iVal;
+
+        /** Long indexed value. */
+        @QuerySqlField(index = true)
+        private long lVal;
+
+        /** */
+        private byte[] payload = new byte[1024];
+
+        /**
+         * @param iVal Integer value.
+         */
+        private IndexedObject(int iVal) {
+            this.iVal = iVal;
+            this.lVal = (long) iVal * iVal;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (!(o instanceof IndexedObject))
+                return false;
+
+            IndexedObject that = (IndexedObject)o;
+
+            return iVal == that.iVal;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return iVal;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(IndexedObject.class, this);
+        }
+    }
+
+    /**
+     * File I/O which halts JVM after specified file truncation.
+     */
+    private static class HaltOnTruncateFileIO extends FileIODecorator {
+        /** File. */
+        private final File file;
+
+        /** The overall number of file truncations have done. */
+        private static final AtomicInteger truncations = new AtomicInteger();
+
+        /**
+         * @param delegate File I/O delegate
+         */
+        public HaltOnTruncateFileIO(FileIO delegate, File file) {
+            super(delegate);
+            this.file = file;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void clear() throws IOException {
+            super.clear();
+
+            System.err.println("Truncated file: " + file.getAbsolutePath());
+
+            truncations.incrementAndGet();
+
+            Integer checkpointedPart = null;
+            try {
+                Field field = GridDhtLocalPartition.class.getDeclaredField("partWhereTestCheckpointEnforced");
+
+                field.setAccessible(true);
+
+                checkpointedPart = (Integer) field.get(null);
+            }
+            catch (Exception e) {
+                e.printStackTrace();
+            }
+
+            // Wait while more than one file have truncated and checkpoint on partition eviction has done.
+            if (truncations.get() > 1 && checkpointedPart != null) {
+                System.err.println("JVM is going to be crushed for test reasons...");
+
+                Runtime.getRuntime().halt(0);
+            }
+        }
+    }
+
+    /**
+     * I/O Factory which creates {@link HaltOnTruncateFileIO} instances for partition files.
+     */
+    private static class HaltOnTruncateFileIOFactory implements FileIOFactory {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** Delegate factory. */
+        private final FileIOFactory delegateFactory;
+
+        /**
+         * @param delegateFactory Delegate factory.
+         */
+        HaltOnTruncateFileIOFactory(FileIOFactory delegateFactory) {
+            this.delegateFactory = delegateFactory;
+        }
+
+        /**
+         * @param file File.
+         */
+        private static boolean isPartitionFile(File file) {
+            return file.getName().contains("part") && file.getName().endsWith("bin");
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            FileIO delegate = delegateFactory.create(file);
+
+            if (isPartitionFile(file))
+                return new HaltOnTruncateFileIO(delegate, file);
+
+            return delegate;
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+            FileIO delegate = delegateFactory.create(file, modes);
+
+            if (isPartitionFile(file))
+                return new HaltOnTruncateFileIO(delegate, file);
+
+            return delegate;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
new file mode 100644
index 0000000000000..b5afddf542889
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.OpenOption;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ * Test class to check that partition files after eviction are destroyed correctly on next checkpoint or crash recovery.
+ */
+public class IgnitePdsPartitionFilesDestroyTest extends GridCommonAbstractTest {
+    /** Cache name. */
+    private static final String CACHE = "cache";
+
+    /** Partitions count. */
+    private static final int PARTS_CNT = 32;
+
+    /** Set if I/O exception should be thrown on partition file truncation. */
+    private boolean failFileIo;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+            .setWalMode(WALMode.LOG_ONLY)
+            .setCheckpointFrequency(10 * 60 * 1000)
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(512 * 1024 * 1024)
+                    .setPersistenceEnabled(true)
+            );
+
+        if (failFileIo)
+            dsCfg.setFileIOFactory(new FailingFileIOFactory(new RandomAccessFileIOFactory()));
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(CACHE)
+            .setBackups(1)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setAffinity(new RendezvousAffinityFunction(false, PARTS_CNT));
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        failFileIo = false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected FailureHandler getFailureHandler(String igniteInstanceName) {
+        return new StopNodeFailureHandler();
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @param keysCnt Keys count.
+     */
+    private void loadData(IgniteEx ignite, int keysCnt, int multiplier) {
+        log.info("Load data: keys=" + keysCnt);
+
+        try (IgniteDataStreamer streamer = ignite.dataStreamer(CACHE)) {
+            streamer.allowOverwrite(true);
+
+            for (int k = 0; k < keysCnt; k++)
+                streamer.addData(k, k * multiplier);
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @param keysCnt Keys count.
+     */
+    private void checkData(IgniteEx ignite, int keysCnt, int multiplier) {
+        log.info("Check data: " + ignite.name() + ", keys=" + keysCnt);
+
+        IgniteCache<Integer, Integer> cache = ignite.cache(CACHE);
+
+        for (int k = 0; k < keysCnt; k++)
+            Assert.assertEquals("node = " + ignite.name() + ", key = " + k, (Integer) (k * multiplier), cache.get(k));
+    }
+
+    /**
+     * Test that partition files have been deleted correctly on next checkpoint.
+     *
+     * @throws Exception If failed.
+     */
+    public void testPartitionFileDestroyAfterCheckpoint() throws Exception {
+        IgniteEx crd = (IgniteEx) startGrids(2);
+
+        crd.cluster().active(true);
+
+        int keysCnt = 50_000;
+
+        loadData(crd, keysCnt, 1);
+
+        startGridsMultiThreaded(2, 2);
+
+        // Trigger partitions eviction.
+        resetBaselineTopology();
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkPartitionFiles(crd, true);
+
+        // This checkpoint should delete partition files.
+        forceCheckpoint();
+
+        checkPartitionFiles(crd, false);
+
+        for (Ignite ignite : G.allGrids())
+            checkData((IgniteEx) ignite, keysCnt, 1);
+    }
+
+    /**
+     * Test that partition files are reused correctly.
+     *
+     * @throws Exception If failed.
+     */
+    public void testPartitionFileDestroyAndRecreate() throws Exception {
+        IgniteEx crd = startGrid(0);
+
+        IgniteEx node = startGrid(1);
+
+        crd.cluster().active(true);
+
+        int keysCnt = 50_000;
+
+        loadData(crd, keysCnt, 1);
+
+        startGridsMultiThreaded(2, 2);
+
+        // Trigger partitions eviction.
+        resetBaselineTopology();
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkPartitionFiles(node, true);
+
+        // Trigger partitions re-create.
+        stopGrid(2);
+
+        resetBaselineTopology();
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkPartitionFiles(node, true);
+
+        // Rewrite data.
+        loadData(crd, keysCnt, 2);
+
+        // Force checkpoint on all nodes.
+        forceCheckpoint();
+
+        // Check that all unecessary partition files have been deleted.
+        checkPartitionFiles(node, false);
+
+        for (Ignite ignite : G.allGrids())
+            checkData((IgniteEx) ignite, keysCnt, 2);
+    }
+
+    /**
+     * Test that partitions files have been deleted correctly during crash recovery.
+     *
+     * @throws Exception If failed.
+     */
+    public void testPartitionFileDestroyCrashRecovery1() throws Exception {
+        IgniteEx crd = startGrid(0);
+
+        failFileIo = true;
+
+        IgniteEx problemNode = startGrid(1);
+
+        failFileIo = false;
+
+        crd.cluster().active(true);
+
+        int keysCnt = 50_000;
+
+        loadData(crd, keysCnt, 1);
+
+        startGridsMultiThreaded(2, 2);
+
+        // Trigger partitions eviction.
+        resetBaselineTopology();
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkPartitionFiles(problemNode, true);
+
+        try {
+            forceCheckpoint(problemNode);
+
+            Assert.assertTrue("Checkpoint must be failed", false);
+        }
+        catch (Exception expected) {
+            expected.printStackTrace();
+        }
+
+        // Problem node should be stopped after failed checkpoint.
+        waitForTopology(3);
+
+        problemNode = startGrid(1);
+
+        awaitPartitionMapExchange();
+
+        // After recovery all evicted partition files should be deleted from disk.
+        checkPartitionFiles(problemNode, false);
+
+        for (Ignite ignite : G.allGrids())
+            checkData((IgniteEx) ignite, keysCnt, 1);
+    }
+
+    /**
+     * Test that partitions files are not deleted if they were re-created on next time
+     * and no checkpoint has done during this time.
+     *
+     * @throws Exception If failed.
+     */
+    public void testPartitionFileDestroyCrashRecovery2() throws Exception {
+        IgniteEx crd = startGrid(0);
+
+        failFileIo = true;
+
+        IgniteEx problemNode = startGrid(1);
+
+        failFileIo = false;
+
+        crd.cluster().active(true);
+
+        int keysCnt = 50_000;
+
+        loadData(crd, keysCnt, 1);
+
+        // Trigger partitions eviction.
+        startGridsMultiThreaded(2, 2);
+
+        resetBaselineTopology();
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkPartitionFiles(problemNode, true);
+
+        // Trigger partitions re-create.
+        stopGrid(2);
+
+        resetBaselineTopology();
+
+        awaitPartitionMapExchange(true, true, null);
+
+        checkPartitionFiles(problemNode, true);
+
+        try {
+            forceCheckpoint(problemNode);
+
+            Assert.assertTrue("Checkpoint must be failed", false);
+        }
+        catch (Exception expected) {
+            expected.printStackTrace();
+        }
+
+        // Problem node should be stopped after failed checkpoint.
+        waitForTopology(2);
+
+        problemNode = startGrid(1);
+
+        awaitPartitionMapExchange();
+
+        // After recovery all evicted partition files should be deleted from disk.
+        checkPartitionFiles(problemNode, false);
+
+        for (Ignite ignite : G.allGrids())
+            checkData((IgniteEx) ignite, keysCnt, 1);
+    }
+
+    /**
+     * If {@code exists} is {@code true}, checks that all partition files exist
+     * if partition has state EVICTED.
+     *
+     * If {@code exists} is {@code false}, checks that all partition files don't exist
+     * if partition is absent or has state EVICTED.
+     *
+     * @param ignite Node.
+     * @param exists If {@code true} method will check that partition file exists,
+     *               in other case will check that file doesn't exist.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void checkPartitionFiles(IgniteEx ignite, boolean exists) throws IgniteCheckedException {
+        int evicted = 0;
+
+        GridDhtPartitionTopology top = ignite.cachex(CACHE).context().topology();
+
+        for (int p = 0; p < PARTS_CNT; p++) {
+            GridDhtLocalPartition part = top.localPartition(p);
+
+            File partFile = partitionFile(ignite, CACHE, p);
+
+            if (exists) {
+                if (part != null && part.state() == GridDhtPartitionState.EVICTED)
+                    Assert.assertTrue("Partition file has deleted ahead of time: " + partFile, partFile.exists());
+
+                evicted++;
+            }
+            else {
+                if (part == null || part.state() == GridDhtPartitionState.EVICTED)
+                    Assert.assertTrue("Partition file has not deleted: " + partFile, !partFile.exists());
+            }
+        }
+
+        if (exists)
+            Assert.assertTrue("There should be at least 1 eviction", evicted > 0);
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @param cacheName Cache name.
+     * @param partId Partition id.
+     */
+    private static File partitionFile(Ignite ignite, String cacheName, int partId) throws IgniteCheckedException {
+        File dbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false);
+
+        String nodeName = ignite.name().replaceAll("\\.", "_");
+
+        return new File(dbDir, String.format("%s/cache-%s/part-%d.bin", nodeName, cacheName, partId));
+    }
+
+    /**
+     *
+     */
+    static class FailingFileIO extends FileIODecorator {
+        /**
+         * @param delegate File I/O delegate
+         */
+        public FailingFileIO(FileIO delegate) {
+            super(delegate);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void clear() throws IOException {
+            throw new IOException("Test");
+        }
+    }
+
+    /**
+     *
+     */
+    static class FailingFileIOFactory implements FileIOFactory {
+        /** Delegate factory. */
+        private final FileIOFactory delegateFactory;
+
+        /**
+         * @param delegateFactory Delegate factory.
+         */
+        FailingFileIOFactory(FileIOFactory delegateFactory) {
+            this.delegateFactory = delegateFactory;
+        }
+
+        /**
+         * @param file File.
+         */
+        private static boolean isPartitionFile(File file) {
+            return file.getName().contains("part") && file.getName().endsWith("bin");
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            FileIO delegate = delegateFactory.create(file);
+
+            if (isPartitionFile(file))
+                return new FailingFileIO(delegate);
+
+            return delegate;
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+            FileIO delegate = delegateFactory.create(file, modes);
+
+            if (isPartitionFile(file))
+                return new FailingFileIO(delegate);
+
+            return delegate;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesTruncateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesTruncateTest.java
deleted file mode 100644
index 78c2453df08a7..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesTruncateTest.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence;
-
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Checks that evicted partitions doesn't leave files in PDS.
- */
-public class IgnitePdsPartitionFilesTruncateTest extends GridCommonAbstractTest {
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        cfg.setConsistentId(gridName)
-                .setDataStorageConfiguration(new DataStorageConfiguration()
-                            .setPageSize(1024)
-                            .setWalMode(WALMode.LOG_ONLY)
-                            .setDefaultDataRegionConfiguration(
-                                new DataRegionConfiguration()
-                                        .setPersistenceEnabled(true)))
-                .setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME)
-                        .setBackups(1)
-                        .setAffinity(new RendezvousAffinityFunction(false, 32)));
-
-        return cfg;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTest() throws Exception {
-        stopAllGrids();
-
-        cleanPersistenceDir();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        stopAllGrids();
-
-        cleanPersistenceDir();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testTruncatingPartitionFilesOnEviction() throws Exception {
-        Ignite ignite0 = startGrids(3);
-
-        ignite0.cluster().active(true);
-
-        try (IgniteDataStreamer<Integer,String> streamer = ignite0.dataStreamer(DEFAULT_CACHE_NAME)) {
-            for (int i = 0; i < 1_000; i++)
-                streamer.addData(i, "Value " + i);
-        }
-
-        assertEquals(1, ignite0.cacheNames().size());
-
-        awaitPartitionMapExchange(true, true, null);
-
-        checkPartFiles(0);
-        checkPartFiles(1);
-        checkPartFiles(2);
-
-        stopGrid(2);
-
-        ignite0.cluster().setBaselineTopology(ignite0.cluster().topologyVersion());
-
-        awaitPartitionMapExchange(true, true, null);
-
-        checkPartFiles(0);
-        checkPartFiles(1);
-
-        startGrid(2);
-
-        ignite0.cluster().setBaselineTopology(ignite0.cluster().topologyVersion());
-
-        awaitPartitionMapExchange(true, true, null);
-
-        checkPartFiles(0);
-        checkPartFiles(1);
-        checkPartFiles(2);
-    }
-
-    /**
-     * @param idx Node index.
-     */
-    private void checkPartFiles(int idx) throws Exception {
-        Ignite ignite = grid(idx);
-
-        int[] parts = ignite.affinity(DEFAULT_CACHE_NAME).allPartitions(ignite.cluster().localNode());
-
-        Path dirPath = Paths.get(U.defaultWorkDirectory(), "db",
-                U.maskForFileName(ignite.configuration().getIgniteInstanceName()), "cache-" + DEFAULT_CACHE_NAME);
-
-        info("Path: " + dirPath.toString());
-
-        assertTrue(Files.exists(dirPath));
-
-        for (Path f : Files.newDirectoryStream(dirPath)) {
-            if (f.getFileName().toString().startsWith("part-"))
-                assertTrue("Node_" + idx +" should contains only partitions " + Arrays.toString(parts)
-                        + ", but the file is redundant: " + f.getFileName(), anyMatch(parts, f));
-        }
-    }
-
-    /** */
-    private boolean anyMatch(int[] parts, Path f) {
-        Pattern ptrn = Pattern.compile("part-(\\d+).bin");
-        Matcher matcher = ptrn.matcher(f.getFileName().toString());
-
-        if (!matcher.find())
-            throw new IllegalArgumentException("File is not a partition:" + f.getFileName());
-
-        int part = Integer.parseInt(matcher.group(1));
-
-        for (int p: parts) {
-            if (p == part)
-                return true;
-        }
-
-        return false;
-    }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index f5784ebcdccd2..00929262e69c0 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -1015,7 +1015,14 @@ protected Ignite startRemoteGrid(String igniteInstanceName, IgniteConfiguration
             }
         }
 
-        return new IgniteProcessProxy(cfg, log, locNode, resetDiscovery);
+        return new IgniteProcessProxy(cfg, log, locNode, resetDiscovery, additionalRemoteJvmArgs());
+    }
+
+    /**
+     * @return Additional JVM args for remote instances.
+     */
+    protected List<String> additionalRemoteJvmArgs() {
+        return Collections.emptyList();
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
index fb59ae2933320..1eb7ddb09deb8 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
@@ -19,6 +19,8 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -135,6 +137,18 @@ public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Ignite locJ
         this(cfg, log, locJvmGrid, true);
     }
 
+    /**
+     * @param cfg Configuration.
+     * @param log Logger.
+     * @param locJvmGrid Local JVM grid.
+     * @throws Exception On error.
+     */
+    public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Ignite locJvmGrid, boolean discovery)
+        throws Exception {
+        this(cfg, log, locJvmGrid, discovery, Collections.emptyList());
+    }
+
+
     /**
      * @param cfg Configuration.
      * @param log Logger.
@@ -142,7 +156,13 @@ public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Ignite locJ
      * @param resetDiscovery Reset DiscoverySpi at the configuration.
      * @throws Exception On error.
      */
-    public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Ignite locJvmGrid, boolean resetDiscovery)
+    public IgniteProcessProxy(
+        IgniteConfiguration cfg,
+        IgniteLogger log,
+        Ignite locJvmGrid,
+        boolean resetDiscovery,
+        List<String> additionalArgs
+    )
         throws Exception {
         this.cfg = cfg;
         this.locJvmGrid = locJvmGrid;
@@ -151,6 +171,7 @@ public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Ignite locJ
         String params = params(cfg, resetDiscovery);
 
         Collection<String> filteredJvmArgs = filteredJvmArgs();
+        filteredJvmArgs.addAll(additionalArgs);
 
         final CountDownLatch rmtNodeStartedLatch = new CountDownLatch(1);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index d11ceb36a863d..e4c59b3c5109d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -91,6 +91,8 @@ public static void addRealPageStoreTestsLongRunning(TestSuite suite) {
     /**
      * Fills {@code suite} with PDS test subset, which operates with real page store and does actual disk operations.
      *
+     * NOTE: These tests are also executed using I/O plugins.
+     *
      * @param suite suite to add tests into.
      */
     public static void addRealPageStoreTests(TestSuite suite) {
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 851d942ae8eb6..9ce1242686e29 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -25,7 +25,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCorruptedStoreTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchangeDuringCheckpointTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPartitionFilesTruncateTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPartitionFilesDestroyTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
 import org.apache.ignite.internal.processors.cache.persistence.LocalWalModeChangeDuringRebalancingSelfTest;
@@ -92,18 +92,20 @@ private static void addRealPageStoreTestsLongRunning(TestSuite suite) {
 
         // Integrity test.
         suite.addTestSuite(IgnitePdsRecoveryAfterFileCorruptionTest.class);
+
+        suite.addTestSuite(IgnitePdsPartitionFilesDestroyTest.class);
     }
 
     /**
      * Fills {@code suite} with PDS test subset, which operates with real page store and does actual disk operations.
      *
+     * NOTE: These tests are also executed using I/O plugins.
+     *
      * @param suite suite to add tests into.
      */
     public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgnitePdsPageSizesTest.class);
 
-        suite.addTestSuite(IgnitePdsPartitionFilesTruncateTest.class);
-
         // Metrics test.
         suite.addTestSuite(IgniteDataStorageMetricsSelfTest.class);
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java
index 5c42d0d3097b4..34191273694f6 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelper.java
@@ -61,7 +61,6 @@ public class InlineIndexHelper {
         Value.SHORT,
         Value.INT,
         Value.LONG,
-        Value.LONG,
         Value.FLOAT,
         Value.DOUBLE,
         Value.DATE,
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index 943d43ff7dff3..d33b20b6f6cf5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsAtomicCacheRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsBinaryMetadataOnClusterRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsBinarySortObjectFieldsTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCorruptedIndexTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsMarshallerMappingRestoreOnNodeStartTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheHistoricalRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheRebalancingTest;
@@ -76,6 +77,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgnitePdsThreadInterruptionTest.class);
         suite.addTestSuite(IgnitePdsBinarySortObjectFieldsTest.class);
 
+        suite.addTestSuite(IgnitePdsCorruptedIndexTest.class);
+
         return suite;
     }
 }

From 69d9272009cc6cbec07d8bbc99364ec777c851b5 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Thu, 17 May 2018 13:10:01 +0300
Subject: [PATCH 144/543] IGNITE-8474 Fixed WalStateNodeLeaveExchangeTask
 preventing exchange merge - Fixes #3990.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry-picked from commit #ebd669e4c53cfd66708ff18dd59071e4aace38ae)
---
 .../GridCachePartitionExchangeManager.java    | 115 +++++++++++-------
 .../cache/WalStateNodeLeaveExchangeTask.java  |   2 +-
 .../distributed/CacheExchangeMergeTest.java   |  25 +++-
 .../ignite/testframework/GridTestUtils.java   |  11 +-
 4 files changed, 108 insertions(+), 45 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 28d5d207f59c0..c3a0add55dee9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -217,6 +217,9 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     /** For tests only. */
     private volatile AffinityTopologyVersion exchMergeTestWaitVer;
 
+    /** For tests only. */
+    private volatile List mergedEvtsForTest;
+
     /** Distributed latch manager. */
     private ExchangeLatchManager latchMgr;
 
@@ -1879,9 +1882,14 @@ private void dumpDiagnosticInfo(IgniteInternalFuture<?> fut,
      * For testing only.
      *
      * @param exchMergeTestWaitVer Version to wait for.
+     * @param mergedEvtsForTest List to collect discovery events with merged exchanges.
      */
-    public void mergeExchangesTestWaitVersion(AffinityTopologyVersion exchMergeTestWaitVer) {
+    public void mergeExchangesTestWaitVersion(
+        AffinityTopologyVersion exchMergeTestWaitVer,
+        @Nullable List mergedEvtsForTest
+    ) {
         this.exchMergeTestWaitVer = exchMergeTestWaitVer;
+        this.mergedEvtsForTest = mergedEvtsForTest;
     }
 
     /**
@@ -1968,46 +1976,8 @@ public boolean mergeExchangesOnCoordinator(GridDhtPartitionsExchangeFuture curFu
 
         AffinityTopologyVersion exchMergeTestWaitVer = this.exchMergeTestWaitVer;
 
-        if (exchMergeTestWaitVer != null) {
-            if (log.isInfoEnabled()) {
-                log.info("Exchange merge test, waiting for version [exch=" + curFut.initialVersion() +
-                    ", waitVer=" + exchMergeTestWaitVer + ']');
-            }
-
-            long end = U.currentTimeMillis() + 10_000;
-
-            while (U.currentTimeMillis() < end) {
-                boolean found = false;
-
-                for (CachePartitionExchangeWorkerTask task : exchWorker.futQ) {
-                    if (task instanceof GridDhtPartitionsExchangeFuture) {
-                        GridDhtPartitionsExchangeFuture fut = (GridDhtPartitionsExchangeFuture)task;
-
-                        if (exchMergeTestWaitVer.equals(fut.initialVersion())) {
-                            if (log.isInfoEnabled())
-                                log.info("Exchange merge test, found awaited version: " + exchMergeTestWaitVer);
-
-                            found = true;
-
-                            break;
-                        }
-                    }
-                }
-
-                if (found)
-                    break;
-                else {
-                    try {
-                        U.sleep(100);
-                    }
-                    catch (IgniteInterruptedCheckedException e) {
-                        break;
-                    }
-                }
-            }
-
-            this.exchMergeTestWaitVer = null;
-        }
+        if (exchMergeTestWaitVer != null)
+            waitForTestVersion(exchMergeTestWaitVer, curFut);
 
         synchronized (curFut.mutex()) {
             int awaited = 0;
@@ -2048,6 +2018,8 @@ public boolean mergeExchangesOnCoordinator(GridDhtPartitionsExchangeFuture curFu
                             ", evtNodeClient=" + CU.clientNode(fut.firstEvent().eventNode())+ ']');
                     }
 
+                    addDiscoEvtForTest(fut.firstEvent());
+
                     curFut.context().events().addEvent(fut.initialVersion(),
                         fut.firstEvent(),
                         fut.firstEventCache());
@@ -2071,6 +2043,67 @@ public boolean mergeExchangesOnCoordinator(GridDhtPartitionsExchangeFuture curFu
         }
     }
 
+
+    /**
+     * For testing purposes. Stores discovery events with merged exchanges to enable examining them later.
+     *
+     * @param discoEvt Discovery event.
+     */
+    private void addDiscoEvtForTest(DiscoveryEvent discoEvt) {
+        List mergedEvtsForTest = this.mergedEvtsForTest;
+
+        if (mergedEvtsForTest != null)
+            mergedEvtsForTest.add(discoEvt);
+    }
+
+    /**
+     * For testing purposes. Method allows to wait for an exchange future of specific version
+     * to appear in exchange worker queue.
+     *
+     * @param exchMergeTestWaitVer Topology Version to wait for.
+     * @param curFut Current Exchange Future.
+     */
+    private void waitForTestVersion(AffinityTopologyVersion exchMergeTestWaitVer, GridDhtPartitionsExchangeFuture curFut) {
+        if (log.isInfoEnabled()) {
+            log.info("Exchange merge test, waiting for version [exch=" + curFut.initialVersion() +
+                ", waitVer=" + exchMergeTestWaitVer + ']');
+        }
+
+        long end = U.currentTimeMillis() + 10_000;
+
+        while (U.currentTimeMillis() < end) {
+            boolean found = false;
+
+            for (CachePartitionExchangeWorkerTask task : exchWorker.futQ) {
+                if (task instanceof GridDhtPartitionsExchangeFuture) {
+                    GridDhtPartitionsExchangeFuture fut = (GridDhtPartitionsExchangeFuture)task;
+
+                    if (exchMergeTestWaitVer.equals(fut.initialVersion())) {
+                        if (log.isInfoEnabled())
+                            log.info("Exchange merge test, found awaited version: " + exchMergeTestWaitVer);
+
+                        found = true;
+
+                        break;
+                    }
+                }
+            }
+
+            if (found)
+                break;
+            else {
+                try {
+                    U.sleep(100);
+                }
+                catch (IgniteInterruptedCheckedException e) {
+                    break;
+                }
+            }
+        }
+
+        this.exchMergeTestWaitVer = null;
+    }
+
     /**
      * Exchange future thread. All exchanges happen only by one thread and next
      * exchange will not start until previous one completes.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateNodeLeaveExchangeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateNodeLeaveExchangeTask.java
index 3ac12fc3b2ee8..77dfc34e32e7c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateNodeLeaveExchangeTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateNodeLeaveExchangeTask.java
@@ -47,7 +47,7 @@ public ClusterNode node() {
 
     /** {@inheritDoc} */
     @Override public boolean skipForExchangeMerge() {
-        return false;
+        return true;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
index 9660a7649d6a7..a7ceb83e2dab1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
@@ -775,17 +776,37 @@ public void testMergeServersFail1_2() throws Exception {
      * @throws Exception If failed.
      */
     private void mergeServersFail1(boolean waitRebalance) throws Exception {
-        final Ignite srv0 = startGrids(4);
+        final Ignite srv0 = startGrids(5);
 
         if (waitRebalance)
             awaitPartitionMapExchange();
 
-        mergeExchangeWaitVersion(srv0, 6);
+        final List<DiscoveryEvent> mergedEvts = new ArrayList<>();
+
+        mergeExchangeWaitVersion(srv0, 8, mergedEvts);
+
+        UUID grid3Id = grid(3).localNode().id();
+        UUID grid2Id = grid(2).localNode().id();
 
+        stopGrid(getTestIgniteInstanceName(4), true, false);
         stopGrid(getTestIgniteInstanceName(3), true, false);
         stopGrid(getTestIgniteInstanceName(2), true, false);
 
         checkCaches();
+
+        awaitPartitionMapExchange();
+
+        assertTrue("Unexpected number of merged disco events: " + mergedEvts.size(), mergedEvts.size() == 2);
+
+        for (DiscoveryEvent discoEvt : mergedEvts) {
+            ClusterNode evtNode = discoEvt.eventNode();
+
+            assertTrue("eventNode is null for DiscoEvent " + discoEvt, evtNode != null);
+
+            assertTrue("Unexpected eventNode ID: "
+                    + evtNode.id() + " while expecting " + grid2Id + " or " + grid3Id,
+                evtNode.id().equals(grid2Id) || evtNode.id().equals(grid3Id));
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index e6c6657578c4d..9390d6b36fe69 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -1949,6 +1949,15 @@ public static String randomString(Random rnd, int maxLen) {
      */
     public static void mergeExchangeWaitVersion(Ignite node, long topVer) {
         ((IgniteEx)node).context().cache().context().exchange().mergeExchangesTestWaitVersion(
-            new AffinityTopologyVersion(topVer, 0));
+            new AffinityTopologyVersion(topVer, 0), null);
+    }
+
+    /**
+     * @param node Node.
+     * @param topVer Ready exchange version to wait for before trying to merge exchanges.
+     */
+    public static void mergeExchangeWaitVersion(Ignite node, long topVer, List mergedEvts) {
+        ((IgniteEx)node).context().cache().context().exchange().mergeExchangesTestWaitVersion(
+            new AffinityTopologyVersion(topVer, 0), mergedEvts);
     }
 }

From 9b80a49d642de27dd6a69daea3921d1c05919df6 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Sat, 28 Apr 2018 15:09:17 +0300
Subject: [PATCH 145/543] IGNITE-8401: errorneous WalPointers comparation. -
 Fixes #3931.

Signed-off-by: dspavlov <dpavlov@gridgain.com>
---
 .../cache/persistence/GridCacheDatabaseSharedManager.java       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 12cfc05d72e2e..787d7fec78ba3 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -747,7 +747,7 @@ private void unRegistrateMetricsMBean() {
 
             WALPointer restore = restoreMemory(status);
 
-            if (restore == null && status.endPtr != CheckpointStatus.NULL_PTR) {
+            if (restore == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) {
                 throw new StorageException("Restore wal pointer = " + restore + ", while status.endPtr = " +
                     status.endPtr + ". Can't restore memory - critical part of WAL archive is missing.");
             }

From 31f8d7e445b1ef7986e3b58e700e119b8490c734 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Thu, 17 May 2018 20:11:41 +0300
Subject: [PATCH 146/543] IGNITE-8423 Control utility may block when joining
 node is watiting for partition map exchange. - Fixes #4018.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 bin/control.bat                               |   2 +-
 bin/control.sh                                |   2 +-
 .../internal/commandline/CommandHandler.java  |  59 ++++---
 .../ignite/util/GridCommandHandlerTest.java   | 150 +++++++++++-------
 4 files changed, 127 insertions(+), 86 deletions(-)

diff --git a/bin/control.bat b/bin/control.bat
index 8a1e1c84e3c70..15d5e6fcb619b 100644
--- a/bin/control.bat
+++ b/bin/control.bat
@@ -104,7 +104,7 @@ if "%OS%" == "Windows_NT" set PROG_NAME=%~nx0%
 ::
 call "%SCRIPTS_HOME%\include\setenv.bat"
 call "%SCRIPTS_HOME%\include\build-classpath.bat"
-set CP=%IGNITE_LIBS%
+set CP=%IGNITE_LIBS%;%IGNITE_HOME%\libs\optional\ignite-zookeeper\*
 
 ::
 :: Process 'restart'.
diff --git a/bin/control.sh b/bin/control.sh
index ad4b14b05151b..e2b75afbd1a07 100755
--- a/bin/control.sh
+++ b/bin/control.sh
@@ -54,7 +54,7 @@ fi
 #
 . "${SCRIPTS_HOME}"/include/setenv.sh
 . "${SCRIPTS_HOME}"/include/build-classpath.sh # Will be removed in the binary release.
-CP="${IGNITE_LIBS}"
+CP="${IGNITE_LIBS}:${IGNITE_HOME}/libs/optional/ignite-zookeeper/*"
 
 RANDOM_NUMBER=$("$JAVA" -cp "${CP}" org.apache.ignite.startup.cmdline.CommandLineRandomNumberGenerator)
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 04578e53f916d..47cc233f61ebc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -256,6 +256,9 @@ public class CommandHandler {
     /** */
     private Object lastOperationRes;
 
+    /** */
+    private GridClientConfiguration clientCfg;
+
     /** Check if experimental commands are enabled. Default {@code false}. */
     private final boolean enableExperimental = IgniteSystemProperties.getBoolean(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, false);
 
@@ -446,18 +449,6 @@ private void state(GridClient client) throws Throwable {
         }
     }
 
-    /**
-     * @param client Client.
-     * @param arg Task argument.
-     * @return Task result.
-     * @throws GridClientException If failed to execute task.
-     */
-    private Map<UUID, VisorTxTaskResult> executeTransactionsTask(GridClient client,
-        VisorTxTaskArg arg) throws GridClientException {
-
-        return executeTask(client, VisorTxTask.class, arg);
-    }
-
     /**
      *
      * @param client Client.
@@ -497,8 +488,28 @@ private <R> R executeTaskByNameOnNode(GridClient client, String taskClsName, Obj
 
         GridClientNode node = null;
 
-        if (nodeId == null)
-            node = getBalancedNode(compute);
+        if (nodeId == null) {
+            Collection<GridClientNode> nodes = compute.nodes(GridClientNode::connectable);
+
+            // Prefer node from connect string.
+            String origAddr = clientCfg.getServers().iterator().next();
+
+            for (GridClientNode clientNode : nodes) {
+                Iterator<String> it = F.concat(clientNode.tcpAddresses().iterator(), clientNode.tcpHostNames().iterator());
+
+                while (it.hasNext()) {
+                    if (origAddr.equals(it.next() + ":" + clientNode.tcpPort())) {
+                        node = clientNode;
+
+                        break;
+                    }
+                }
+            }
+
+            // Otherwise choose random node.
+            if (node == null)
+                node = getBalancedNode(compute);
+        }
         else {
             for (GridClientNode n : compute.nodes()) {
                 if (n.connectable() && nodeId.equals(n.nodeId())) {
@@ -1677,12 +1688,12 @@ public int execute(List<String> rawArgs) {
                             " delete [consistentId1,consistentId2,....,consistentIdN] [--force]");
                 }
 
-                log("The utility has --cache subcommand to view and control state of caches in cluster.");
-                log("  More info:    control.sh --cache help");
+                log("  View caches information in a cluster. For more details type:");
+                log("    control.sh --cache help");
                 nl();
 
-                log("By default commands affecting the cluster require interactive confirmation. ");
-                log("  --force option can be used to execute commands without prompting for confirmation.");
+                log("By default commands affecting the cluster require interactive confirmation.");
+                log("Use --force option to disable it.");
                 nl();
 
                 log("Default values:");
@@ -1710,20 +1721,20 @@ public int execute(List<String> rawArgs) {
                 return EXIT_CODE_OK;
             }
 
-            GridClientConfiguration cfg = new GridClientConfiguration();
+            clientCfg = new GridClientConfiguration();
 
-            cfg.setPingInterval(args.pingInterval());
+            clientCfg.setPingInterval(args.pingInterval());
 
-            cfg.setPingTimeout(args.pingTimeout());
+            clientCfg.setPingTimeout(args.pingTimeout());
 
-            cfg.setServers(Collections.singletonList(args.host() + ":" + args.port()));
+            clientCfg.setServers(Collections.singletonList(args.host() + ":" + args.port()));
 
             if (!F.isEmpty(args.user())) {
-                cfg.setSecurityCredentialsProvider(
+                clientCfg.setSecurityCredentialsProvider(
                     new SecurityCredentialsBasicProvider(new SecurityCredentials(args.user(), args.password())));
             }
 
-            try (GridClient client = GridClientFactory.start(cfg)) {
+            try (GridClient client = GridClientFactory.start(clientCfg)) {
                 switch (args.command()) {
                     case ACTIVATE:
                         activate(client);
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 4daa92cdf000c..670c22c313b82 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -381,69 +381,10 @@ public void testActiveTransactions() throws Exception {
         for (Ignite ig : G.allGrids())
             assertNotNull(ig.cache(DEFAULT_CACHE_NAME));
 
-        AtomicInteger idx = new AtomicInteger();
-
         CountDownLatch lockLatch = new CountDownLatch(1);
         CountDownLatch unlockLatch = new CountDownLatch(1);
 
-        IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
-            @Override public void run() {
-                int id = idx.getAndIncrement();
-
-                switch (id) {
-                    case 0:
-                        try (Transaction tx = grid(0).transactions().txStart()) {
-                            grid(0).cache(DEFAULT_CACHE_NAME).putAll(generate(0, 100));
-
-                            lockLatch.countDown();
-
-                            U.awaitQuiet(unlockLatch);
-
-                            tx.commit();
-
-                            fail("Commit must fail");
-                        }
-                        catch (Exception e) {
-                            // No-op.
-                            assertTrue(X.hasCause(e, TransactionRollbackException.class));
-                        }
-
-                        break;
-                    case 1:
-                        U.awaitQuiet(lockLatch);
-
-                        doSleep(3000);
-
-                        try (Transaction tx = grid(0).transactions().withLabel("label1").txStart(PESSIMISTIC, READ_COMMITTED, Integer.MAX_VALUE, 0)) {
-                            grid(0).cache(DEFAULT_CACHE_NAME).putAll(generate(200, 110));
-
-                            grid(0).cache(DEFAULT_CACHE_NAME).put(0, 0);
-                        }
-
-                        break;
-                    case 2:
-                        try (Transaction tx = grid(1).transactions().txStart()) {
-                            U.awaitQuiet(lockLatch);
-
-                            grid(1).cache(DEFAULT_CACHE_NAME).put(0, 0);
-                        }
-
-                        break;
-                    case 3:
-                        try (Transaction tx = client.transactions().withLabel("label2").txStart(OPTIMISTIC, READ_COMMITTED, 0, 0)) {
-                            U.awaitQuiet(lockLatch);
-
-                            client.cache(DEFAULT_CACHE_NAME).putAll(generate(100, 10));
-
-                            client.cache(DEFAULT_CACHE_NAME).put(0, 0);
-
-                            tx.commit();
-                        }
-
-                        break;
-                }
-            }
-        }, 4, "tx-thread");
+        IgniteInternalFuture<?> fut = startTransactions(lockLatch, unlockLatch);
 
         U.awaitQuiet(lockLatch);
 
@@ -518,6 +459,23 @@ else if (entry.getKey().equals(node2)) {
 
         }, "--tx", "order", "DURATION");
 
+        // Trigger topology change and test connection.
+        IgniteInternalFuture<?> startFut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    startGrid(2);
+                }
+                catch (Exception e) {
+                    fail();
+                }
+            }
+        }, 1, "start-node-thread");
+
+        doSleep(5000);
+
+        assertEquals(EXIT_CODE_OK, execute(h, "--host", "127.0.0.1", "--port", "11211", "--tx"));
+        assertEquals(EXIT_CODE_OK, execute(h, "--host", "127.0.0.1", "--port", "11212", "--tx"));
+
         // Test kill by xid.
         validate(h, map -> {
                 assertEquals(1, map.size());
@@ -533,6 +491,8 @@ else if (entry.getKey().equals(node2)) {
 
         unlockLatch.countDown();
 
+        startFut.get();
+
         fut.get();
     }
 
@@ -859,4 +819,74 @@ public void testUnusedWalDelete() throws Exception {
 
         assertTrue(!testOut.toString().contains("error"));
     }
+
+    /**
+     *
+     * @param lockLatch Lock latch.
+     * @param unlockLatch Unlock latch.
+     */
+    private IgniteInternalFuture<?> startTransactions(CountDownLatch lockLatch, CountDownLatch unlockLatch) throws Exception {
+        IgniteEx client = grid("client");
+
+        AtomicInteger idx = new AtomicInteger();
+
+        return multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                int id = idx.getAndIncrement();
+
+                switch (id) {
+                    case 0:
+                        try (Transaction tx = grid(0).transactions().txStart()) {
+                            grid(0).cache(DEFAULT_CACHE_NAME).putAll(generate(0, 100));
+
+                            lockLatch.countDown();
+
+                            U.awaitQuiet(unlockLatch);
+
+                            tx.commit();
+
+                            fail("Commit must fail");
+                        }
+                        catch (Exception e) {
+                            // No-op.
+                            assertTrue(X.hasCause(e, TransactionRollbackException.class));
+                        }
+
+                        break;
+                    case 1:
+                        U.awaitQuiet(lockLatch);
+
+                        doSleep(3000);
+
+                        try (Transaction tx = grid(0).transactions().withLabel("label1").txStart(PESSIMISTIC, READ_COMMITTED, Integer.MAX_VALUE, 0)) {
+                            grid(0).cache(DEFAULT_CACHE_NAME).putAll(generate(200, 110));
+
+                            grid(0).cache(DEFAULT_CACHE_NAME).put(0, 0);
+                        }
+
+                        break;
+                    case 2:
+                        try (Transaction tx = grid(1).transactions().txStart()) {
+                            U.awaitQuiet(lockLatch);
+
+                            grid(1).cache(DEFAULT_CACHE_NAME).put(0, 0);
+                        }
+
+                        break;
+                    case 3:
+                        try (Transaction tx = client.transactions().withLabel("label2").txStart(OPTIMISTIC, READ_COMMITTED, 0, 0)) {
+                            U.awaitQuiet(lockLatch);
+
+                            client.cache(DEFAULT_CACHE_NAME).putAll(generate(100, 10));
+
+                            client.cache(DEFAULT_CACHE_NAME).put(0, 0);
+
+                            tx.commit();
+                        }
+
+                        break;
+                }
+            }
+        }, 4, "tx-thread");
+    }
 }

From d9a80ff6c09ac92a78948fcdb292c5e5c28111d0 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Fri, 18 May 2018 11:18:39 +0300
Subject: [PATCH 147/543] IGNITE-8464 Fixed race in WALIterator leading to
 skipped segments - Fixes #3982.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit fdad641)
---
 .../wal/AbstractWalRecordsIterator.java       | 11 ++-
 .../wal/FileWriteAheadLogManager.java         | 86 ++++---------------
 .../FsyncModeFileWriteAheadLogManager.java    | 81 ++++-------------
 .../wal/serializer/RecordV1Serializer.java    | 61 +++++++++++++
 .../wal/serializer/SegmentHeader.java         | 58 +++++++++++++
 .../db/wal/reader/IgniteWalReaderTest.java    | 46 +++++-----
 6 files changed, 185 insertions(+), 158 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/SegmentHeader.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index 65f3a20830109..d9312f6888b2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -33,12 +33,15 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.SegmentHeader;
 import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
 import org.apache.ignite.internal.util.typedef.P2;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader;
+
 /**
  * Iterator over WAL segments. This abstract class provides most functionality for reading records in log.
  * Subclasses are to override segment switching functionality
@@ -265,11 +268,9 @@ protected AbstractReadFileHandle initReadHandle(
             FileIO fileIO = desc.isCompressed() ? new UnzipFileIO(desc.file()) : ioFactory.create(desc.file());
 
             try {
-                IgniteBiTuple<Integer, Boolean> tup = FileWriteAheadLogManager.readSerializerVersionAndCompactedFlag(fileIO);
-
-                int serVer = tup.get1();
+                SegmentHeader segmentHeader = readSegmentHeader(fileIO, curWalSegmIdx);
 
-                boolean isCompacted = tup.get2();
+                boolean isCompacted = segmentHeader.isCompacted();
 
                 if (isCompacted)
                     serializerFactory.skipPositionCheck(true);
@@ -289,6 +290,8 @@ protected AbstractReadFileHandle initReadHandle(
                     }
                 }
 
+                int serVer = segmentHeader.getSerializerVersion();
+
                 return createReadFileHandle(fileIO, desc.idx(), serializerFactory.createSerializer(serVer), in);
             }
             catch (SegmentEofException | EOFException ignore) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index e69cc8040f7e9..e095f6a26b30a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -124,6 +124,8 @@
 import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.SWITCH_SEGMENT_RECORD;
 import static org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBuffer.BufferMode.DIRECT;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.HEADER_RECORD_SIZE;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader;
 import static org.apache.ignite.internal.util.IgniteUtils.findField;
 import static org.apache.ignite.internal.util.IgniteUtils.findNonPublicMethod;
 
@@ -1142,7 +1144,7 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
                 // If we have existing segment, try to read version from it.
                 if (lastReadPtr != null) {
                     try {
-                        serVer = readSerializerVersionAndCompactedFlag(fileIO).get1();
+                        serVer = readSegmentHeader(fileIO, absIdx).getSerializerVersion();
                     }
                     catch (SegmentEofException | EOFException ignore) {
                         serVer = serializerVer;
@@ -1328,17 +1330,28 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
     }
 
     /**
-     * Clears the file, fills with zeros for Default mode.
+     * Clears whole the file, fills with zeros for Default mode.
      *
      * @param file File to format.
      * @throws IgniteCheckedException if formatting failed
      */
     private void formatFile(File file) throws IgniteCheckedException {
+        formatFile(file, dsCfg.getWalSegmentSize());
+    }
+
+    /**
+     * Clears the file, fills with zeros for Default mode.
+     *
+     * @param file File to format.
+     * @param bytesCntToFormat Count of first bytes to format.
+     * @throws IgniteCheckedException if formatting failed
+     */
+    private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedException {
         if (log.isDebugEnabled())
             log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
         try (FileIO fileIO = ioFactory.create(file, CREATE, READ, WRITE)) {
-            int left = dsCfg.getWalSegmentSize();
+            int left = bytesCntToFormat;
 
             if (mode == WALMode.FSYNC) {
                 while (left > 0) {
@@ -1584,13 +1597,7 @@ private synchronized boolean locked(long absIdx) {
                         synchronized (this) {
                             while (locked.containsKey(toArchive) && !stopped)
                                 wait();
-                        }
-
-                        // Firstly, format working file
-                        if (!stopped)
-                            formatFile(res.getOrigWorkFile());
 
-                        synchronized (this) {
                             // Then increase counter to allow rollover on clean working file
                             changeLastArchivedIndexAndNotifyWaiters(toArchive);
 
@@ -1995,15 +2002,13 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
             int segmentSerializerVer;
 
             try (FileIO fileIO = ioFactory.create(raw)) {
-                IgniteBiTuple<Integer, Boolean> tup = readSerializerVersionAndCompactedFlag(fileIO);
-
-                segmentSerializerVer = tup.get1();
+                segmentSerializerVer = readSegmentHeader(fileIO, nextSegment).getSerializerVersion();
             }
 
             try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(zip)))) {
                 zos.putNextEntry(new ZipEntry(""));
 
-                ByteBuffer buf = ByteBuffer.allocate(RecordV1Serializer.HEADER_RECORD_SIZE);
+                ByteBuffer buf = ByteBuffer.allocate(HEADER_RECORD_SIZE);
                 buf.order(ByteOrder.nativeOrder());
 
                 zos.write(prepareSerializerVersionBuffer(nextSegment, segmentSerializerVer, true, buf).array());
@@ -2222,59 +2227,6 @@ else if (create)
         }
     }
 
-    /**
-     * Reads record serializer version from provided {@code io} along with compacted flag.
-     * NOTE: Method mutates position of {@code io}.
-     *
-     * @param io I/O interface for file.
-     * @return Serializer version stored in the file.
-     * @throws IgniteCheckedException If failed to read serializer version.
-     */
-    static IgniteBiTuple<Integer, Boolean> readSerializerVersionAndCompactedFlag(FileIO io)
-            throws IgniteCheckedException, IOException {
-        try (ByteBufferExpander buf = new ByteBufferExpander(RecordV1Serializer.HEADER_RECORD_SIZE, ByteOrder.nativeOrder())) {
-            FileInput in = new FileInput(io, buf);
-
-            in.ensure(RecordV1Serializer.HEADER_RECORD_SIZE);
-
-            int recordType = in.readUnsignedByte();
-
-            if (recordType == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE)
-                throw new SegmentEofException("Reached logical end of the segment", null);
-
-            WALRecord.RecordType type = WALRecord.RecordType.fromOrdinal(recordType - 1);
-
-            if (type != WALRecord.RecordType.HEADER_RECORD)
-                throw new IOException("Can't read serializer version", null);
-
-            // Read file pointer.
-            FileWALPointer ptr = RecordV1Serializer.readPosition(in);
-
-            assert ptr.fileOffset() == 0 : "Header record should be placed at the beginning of file " + ptr;
-
-            long hdrMagicNum = in.readLong();
-
-            boolean compacted;
-
-            if (hdrMagicNum == HeaderRecord.REGULAR_MAGIC)
-                compacted = false;
-            else if (hdrMagicNum == HeaderRecord.COMPACTED_MAGIC)
-                compacted = true;
-            else {
-                throw new IOException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.REGULAR_MAGIC) +
-                    ", actual=" + U.hexLong(hdrMagicNum) + ']');
-            }
-
-            // Read serializer version.
-            int ver = in.readInt();
-
-            // Read and skip CRC.
-            in.readInt();
-
-            return new IgniteBiTuple<>(ver, compacted);
-        }
-    }
-
     /**
      * Needs only for WAL compaction.
      *
@@ -2582,7 +2534,7 @@ private FileWriteHandle(
          * Write serializer version to current handle.
          */
         public void writeHeader() {
-            SegmentedRingByteBuffer.WriteSegment seg = buf.offer(RecordV1Serializer.HEADER_RECORD_SIZE);
+            SegmentedRingByteBuffer.WriteSegment seg = buf.offer(HEADER_RECORD_SIZE);
 
             assert seg != null && seg.position() > 0;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 7bb2ce4dfc48c..f582f193f9198 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -115,6 +115,7 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
 import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
 import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader;
 
 /**
  * File WAL manager.
@@ -1037,7 +1038,7 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
                 // If we have existing segment, try to read version from it.
                 if (lastReadPtr != null) {
                     try {
-                        serVer = readSerializerVersionAndCompactedFlag(fileIO).get1();
+                        serVer = readSegmentHeader(fileIO, absIdx).getSerializerVersion();
                     }
                     catch (SegmentEofException | EOFException ignore) {
                         serVer = serializerVersion;
@@ -1151,16 +1152,28 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
     }
 
     /**
-     * Clears the file with zeros.
+     * Clears whole the file, fills with zeros for Default mode.
      *
      * @param file File to format.
+     * @throws IgniteCheckedException if formatting failed
      */
     private void formatFile(File file) throws IgniteCheckedException {
+        formatFile(file, dsCfg.getWalSegmentSize());
+    }
+
+    /**
+     * Clears the file, fills with zeros for Default mode.
+     *
+     * @param file File to format.
+     * @param bytesCntToFormat Count of first bytes to format.
+     * @throws IgniteCheckedException if formatting failed
+     */
+    private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedException {
         if (log.isDebugEnabled())
             log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
         try (FileIO fileIO = ioFactory.create(file, CREATE, READ, WRITE)) {
-            int left = dsCfg.getWalSegmentSize();
+            int left = bytesCntToFormat;
 
             if (mode == WALMode.FSYNC) {
                 while (left > 0) {
@@ -1424,13 +1437,7 @@ private synchronized void release(long absIdx) {
                         synchronized (this) {
                             while (locked.containsKey(toArchive) && !stopped)
                                 wait();
-                        }
-
-                        // Firstly, format working file
-                        if (!stopped)
-                            formatFile(res.getOrigWorkFile());
 
-                        synchronized (this) {
                             // Then increase counter to allow rollover on clean working file
                             changeLastArchivedIndexAndWakeupCompressor(toArchive);
 
@@ -1833,9 +1840,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
             int segmentSerializerVer;
 
             try (FileIO fileIO = ioFactory.create(raw)) {
-                IgniteBiTuple<Integer, Boolean> tup = readSerializerVersionAndCompactedFlag(fileIO);
-
-                segmentSerializerVer = tup.get1();
+                segmentSerializerVer = readSegmentHeader(fileIO, nextSegment).getSerializerVersion();
             }
 
             try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(zip)))) {
@@ -2029,58 +2034,6 @@ else if (create)
         }
     }
 
-    /**
-     * Reads record serializer version from provided {@code io} along with compacted flag.
-     * NOTE: Method mutates position of {@code io}.
-     *
-     * @param io I/O interface for file.
-     * @return Serializer version stored in the file.
-     * @throws IgniteCheckedException If failed to read serializer version.
-     */
-    public static IgniteBiTuple<Integer, Boolean> readSerializerVersionAndCompactedFlag(FileIO io)
-            throws IgniteCheckedException, IOException {
-        try (ByteBufferExpander buf = new ByteBufferExpander(RecordV1Serializer.HEADER_RECORD_SIZE, ByteOrder.nativeOrder())) {
-            FileInput in = new FileInput(io, buf);
-
-            in.ensure(RecordV1Serializer.HEADER_RECORD_SIZE);
-
-            int recordType = in.readUnsignedByte();
-
-            if (recordType == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE)
-                throw new SegmentEofException("Reached logical end of the segment", null);
-
-            WALRecord.RecordType type = WALRecord.RecordType.fromOrdinal(recordType - 1);
-
-            if (type != WALRecord.RecordType.HEADER_RECORD)
-                throw new IOException("Can't read serializer version", null);
-
-            // Read file pointer.
-            FileWALPointer ptr = RecordV1Serializer.readPosition(in);
-
-            assert ptr.fileOffset() == 0 : "Header record should be placed at the beginning of file " + ptr;
-
-            long hdrMagicNum = in.readLong();
-
-            boolean compacted;
-            if (hdrMagicNum == HeaderRecord.REGULAR_MAGIC)
-                compacted = false;
-            else if (hdrMagicNum == HeaderRecord.COMPACTED_MAGIC)
-                compacted = true;
-            else {
-                throw new IOException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.REGULAR_MAGIC) +
-                    ", actual=" + U.hexLong(hdrMagicNum) + ']');
-            }
-
-            // Read serializer version.
-            int ver = in.readInt();
-
-            // Read and skip CRC.
-            in.readInt();
-
-            return new IgniteBiTuple<>(ver, compacted);
-        }
-    }
-
     /**
      * Writes record serializer version to provided {@code io}.
      * NOTE: Method mutates position of {@code io}.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
index 5ae269b953095..dd0819c859ce8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -29,17 +29,21 @@
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io.RecordIO;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiPredicate;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
@@ -218,6 +222,63 @@ public static void putPosition(ByteBuffer buf, FileWALPointer ptr) {
         buf.putInt(ptr.fileOffset());
     }
 
+    /**
+     * Reads stored record from provided {@code io}.
+     * NOTE: Method mutates position of {@code io}.
+     *
+     * @param io I/O interface for file.
+     * @param expectedIdx Expected WAL segment index for readable record.
+     * @return Instance of {@link SegmentHeader} extracted from the file.
+     * @throws IgniteCheckedException If failed to read serializer version.
+     */
+    public static SegmentHeader readSegmentHeader(FileIO io, long expectedIdx)
+        throws IgniteCheckedException, IOException {
+        try (ByteBufferExpander buf = new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())) {
+            FileInput in = new FileInput(io, buf);
+
+            in.ensure(HEADER_RECORD_SIZE);
+
+            int recordType = in.readUnsignedByte();
+
+            if (recordType == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE)
+                throw new SegmentEofException("Reached logical end of the segment", null);
+
+            WALRecord.RecordType type = WALRecord.RecordType.fromOrdinal(recordType - 1);
+
+            if (type != WALRecord.RecordType.HEADER_RECORD)
+                throw new IOException("Can't read serializer version", null);
+
+            // Read file pointer.
+            FileWALPointer ptr = readPosition(in);
+
+            if (expectedIdx != ptr.index())
+                throw new SegmentEofException("Reached logical end of the segment by pointer", null);
+
+            assert ptr.fileOffset() == 0 : "Header record should be placed at the beginning of file " + ptr;
+
+            long hdrMagicNum = in.readLong();
+
+            boolean compacted;
+
+            if (hdrMagicNum == HeaderRecord.REGULAR_MAGIC)
+                compacted = false;
+            else if (hdrMagicNum == HeaderRecord.COMPACTED_MAGIC)
+                compacted = true;
+            else {
+                throw new IOException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.REGULAR_MAGIC) +
+                    ", actual=" + U.hexLong(hdrMagicNum) + ']');
+            }
+
+            // Read serializer version.
+            int ver = in.readInt();
+
+            // Read and skip CRC.
+            in.readInt();
+
+            return new SegmentHeader(ver, compacted);
+        }
+    }
+
     /**
      * @param in Data input to read pointer from.
      * @return Read file WAL pointer.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/SegmentHeader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/SegmentHeader.java
new file mode 100644
index 0000000000000..8f7e738d6f8fa
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/SegmentHeader.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.serializer;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * WAL segment header info.
+ */
+public class SegmentHeader {
+    /** Serializer version. */
+    private int serializerVersion;
+    /** Compacted flag. */
+    private boolean isCompacted;
+
+    /**
+     * @param serializerVersion Serializer version.
+     * @param isCompacted Compacted flag.
+     */
+    public SegmentHeader(int serializerVersion, boolean isCompacted) {
+        this.serializerVersion = serializerVersion;
+        this.isCompacted = isCompacted;
+    }
+
+    /**
+     * @return Record serializer version.
+     */
+    public int getSerializerVersion() {
+        return serializerVersion;
+    }
+
+    /**
+     * @return Comacted flag.
+     */
+    public boolean isCompacted() {
+        return isCompacted;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(SegmentHeader.class, this);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
index 857c0d0f2983b..28f049aaef5c9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
@@ -23,6 +23,7 @@
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import java.io.Serializable;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumMap;
 import java.util.HashMap;
@@ -213,45 +214,44 @@ public void testFillWalAndReadRecords() throws Exception {
         final File wal = new File(db, "wal");
         final File walArchive = setWalAndArchiveToSameValue ? wal : new File(wal, "archive");
 
-        final MockWalIteratorFactory mockItFactory = new MockWalIteratorFactory(log, PAGE_SIZE, consistentId, subfolderName, WAL_SEGMENTS);
-        final WALIterator it = mockItFactory.iterator(wal, walArchive);
-        final int cntUsingMockIter = iterateAndCount(it, false);
-
-        log.info("Total records loaded " + cntUsingMockIter);
-        assertTrue(cntUsingMockIter > 0);
-        assertTrue(cntUsingMockIter > cacheObjectsToWrite);
+        int[] checkKeyIterArr = new int[cacheObjectsToWrite];
 
         final File walArchiveDirWithConsistentId = new File(walArchive, subfolderName);
         final File walWorkDirWithConsistentId = new File(wal, subfolderName);
         final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+
+        //Check iteratorArchiveDirectory and iteratorArchiveFiles are same.
         final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId));
 
         log.info("Total records loaded using directory : " + cntArchiveDir);
 
-        final int cntArchiveFileByFile = iterateAndCount(
-            factory.iteratorArchiveFiles(
-                walArchiveDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER)));
+        final int cntArchiveFileByFile = iterateAndCount(factory.iteratorArchiveFiles(
+            walArchiveDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER)));
 
         log.info("Total records loaded using archive directory (file-by-file): " + cntArchiveFileByFile);
 
-        assertTrue(cntArchiveFileByFile > cacheObjectsToWrite);
-        assertTrue(cntArchiveDir > cacheObjectsToWrite);
         assertTrue(cntArchiveDir == cntArchiveFileByFile);
-        //really count2 may be less because work dir correct loading is not supported yet
-        assertTrue("Mock based reader loaded " + cntUsingMockIter + " records " +
-                "but standalone has loaded only " + cntArchiveDir,
-            cntUsingMockIter >= cntArchiveDir);
 
-        final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
+        //Check iteratorArchiveFiles + iteratorWorkFiles iterate over all entries.
+        Arrays.fill(checkKeyIterArr, 0);
 
-        final int cntWork = iterateAndCount(factory.iteratorWorkFiles(workFiles));
+        iterateAndCountDataRecord(factory.iteratorArchiveFiles(
+            walArchiveDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER)), new IgniteBiInClosure<Object, Object>() {
+            @Override public void apply(Object o, Object o2) {
+                checkKeyIterArr[(Integer)o]++;
+            }
+        }, null);
 
-        log.info("Total records loaded from work: " + cntWork);
+        final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
+
+        iterateAndCountDataRecord(factory.iteratorWorkFiles(workFiles), new IgniteBiInClosure<Object, Object>() {
+            @Override public void apply(Object o, Object o2) {
+                checkKeyIterArr[(Integer) o]++;
+            }
+        }, null).size();
 
-        assertTrue("Work iterator loaded [" + cntWork + "] " +
-                "Archive iterator loaded [" + cntArchiveFileByFile + "]; " +
-                "mock iterator [" + cntUsingMockIter + "]",
-            cntWork + cntArchiveFileByFile == cntUsingMockIter);
+        for (int i =0 ; i< cacheObjectsToWrite; i++)
+            assertTrue("Iterator didn't find key="+ i, checkKeyIterArr[i] > 0);
     }
 
     /**

From 68bf746c47d445d5a469d29b7a4ae5ce040b3d9b Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Fri, 18 May 2018 19:28:45 +0300
Subject: [PATCH 148/543] IGNITE-8531 Fixed NPE if checkpoint has no pages to
 write, but has partitions to destroy. - Fixes #4026.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 5c8d9ff)
---
 .../GridCacheDatabaseSharedManager.java       | 24 ++++++++++------
 .../IgnitePdsPartitionFilesDestroyTest.java   | 28 +++++++++++++++++++
 2 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 2f95ed68b84c7..ffa7259669775 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -3049,6 +3049,8 @@ private void doCheckpoint() {
 
                 boolean success = false;
 
+                int destroyedPartitionsCnt;
+
                 try {
                     if (chp.hasDelta()) {
                         // Identity stores set.
@@ -3135,7 +3137,7 @@ private void doCheckpoint() {
                     snapshotMgr.afterCheckpointPageWritten();
 
                     try {
-                        destroyEvictedPartitions();
+                        destroyedPartitionsCnt = destroyEvictedPartitions();
                     }
                     catch (IgniteCheckedException e) {
                         chp.progress.cpFinishFut.onDone(e);
@@ -3155,15 +3157,15 @@ private void doCheckpoint() {
 
                 tracker.onEnd();
 
-                if (chp.hasDelta()) {
+                if (chp.hasDelta() || destroyedPartitionsCnt > 0) {
                     if (printCheckpointStats) {
                         if (log.isInfoEnabled())
                             log.info(String.format("Checkpoint finished [cpId=%s, pages=%d, markPos=%s, " +
                                     "walSegmentsCleared=%d, markDuration=%dms, pagesWrite=%dms, fsync=%dms, " +
                                     "total=%dms]",
-                                chp.cpEntry.checkpointId(),
+                                chp.cpEntry != null ? chp.cpEntry.checkpointId() : "",
                                 chp.pagesSize,
-                                chp.cpEntry.checkpointMark(),
+                                chp.cpEntry != null ? chp.cpEntry.checkpointMark() : "",
                                 chp.walFilesDeleted,
                                 tracker.markDuration(),
                                 tracker.pagesWriteDuration(),
@@ -3203,12 +3205,14 @@ private void doCheckpoint() {
          * Processes all evicted partitions scheduled for destroy.
          *
          * @throws IgniteCheckedException If failed.
+         *
+         * @return The number of destroyed partition files.
          */
-        private void destroyEvictedPartitions() throws IgniteCheckedException {
+        private int destroyEvictedPartitions() throws IgniteCheckedException {
             PartitionDestroyQueue destroyQueue = curCpProgress.destroyQueue;
 
             if (destroyQueue.pendingReqs.isEmpty())
-                return;
+                return 0;
 
             List<PartitionDestroyRequest> reqs = null;
 
@@ -3266,6 +3270,8 @@ private void destroyEvictedPartitions() throws IgniteCheckedException {
                     req.waitCompleted();
 
             destroyQueue.pendingReqs.clear();
+
+            return reqs != null ? reqs.size() : 0;
         }
 
         /**
@@ -3434,7 +3440,7 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
 
                 hasPages = hasPageForWrite(cpPagesTuple.get1());
 
-                if (hasPages || curr.nextSnapshot) {
+                if (hasPages || curr.nextSnapshot || !curr.destroyQueue.pendingReqs.isEmpty()) {
                     // No page updates for this checkpoint are allowed from now on.
                     cpPtr = cctx.wal().log(cpRec);
 
@@ -3813,7 +3819,7 @@ private enum CheckpointEntryType {
      */
     private static class Checkpoint {
         /** Checkpoint entry. */
-        private final CheckpointEntry cpEntry;
+        @Nullable private final CheckpointEntry cpEntry;
 
         /** Checkpoint pages. */
         private final GridMultiCollectionWrapper<FullPageId> cpPages;
@@ -3833,7 +3839,7 @@ private static class Checkpoint {
          * @param progress Checkpoint progress status.
          */
         private Checkpoint(
-            CheckpointEntry cpEntry,
+            @Nullable CheckpointEntry cpEntry,
             @NotNull GridMultiCollectionWrapper<FullPageId> cpPages,
             CheckpointProgress progress
         ) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
index b5afddf542889..5e0ccc9ce1fc8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
@@ -19,6 +19,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.OpenOption;
+import java.util.List;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
@@ -332,6 +333,33 @@ public void testPartitionFileDestroyCrashRecovery2() throws Exception {
             checkData((IgniteEx) ignite, keysCnt, 1);
     }
 
+    /**
+     * Test destroy when partition files are empty and there are no pages for checkpoint.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDestroyWhenPartitionsAreEmpty() throws Exception {
+        IgniteEx crd = (IgniteEx) startGrids(2);
+
+        crd.cluster().active(true);
+
+        forceCheckpoint();
+
+        // Evict arbitrary partition.
+        List<GridDhtLocalPartition> parts = crd.cachex(CACHE).context().topology().localPartitions();
+        for (GridDhtLocalPartition part : parts)
+            if (part.state() != GridDhtPartitionState.EVICTED) {
+                part.rent(false).get();
+
+                break;
+            }
+
+        // This checkpoint has no pages to write, but has one partition file to destroy.
+        forceCheckpoint(crd);
+
+        checkPartitionFiles(crd, false);
+    }
+
     /**
      * If {@code exists} is {@code true}, checks that all partition files exist
      * if partition has state EVICTED.

From 18d913c6118b5132859fbc1e5bacfc97392f9bc2 Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Fri, 18 May 2018 16:59:14 +0300
Subject: [PATCH 149/543] IGNITE-8491 Add JMX flag: Is the node in baseline or
 not - Fixes #4010.

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit f8ae30d)
---
 .../apache/ignite/internal/IgniteKernal.java  | 13 +++
 .../apache/ignite/mxbean/IgniteMXBean.java    |  8 ++
 .../util/mbeans/GridMBeanBaselineTest.java    | 96 +++++++++++++++++++
 3 files changed, 117 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanBaselineTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 4ceee7f726a58..07edc7e71f041 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -106,6 +106,7 @@
 import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
 import org.apache.ignite.internal.cluster.IgniteClusterEx;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.managers.GridManager;
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
@@ -524,6 +525,18 @@ public IgniteKernal(@Nullable GridSpringResourceContext rsrcCtx) {
             .toString();
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean isNodeInBaseline() {
+        ClusterNode locNode = localNode();
+
+        if (locNode.isClient() || locNode.isDaemon())
+            return false;
+
+        DiscoveryDataClusterState clusterState = ctx.state().clusterState();
+
+        return clusterState.baselineTopology() != null && CU.baselineNode(locNode, clusterState);
+    }
+
     /** {@inheritDoc} */
     @Override public String getCommunicationSpiFormatted() {
         assert cfg != null;
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
index bb79334352a2d..d121b79162c88 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/IgniteMXBean.java
@@ -438,6 +438,14 @@ public interface IgniteMXBean {
     @MXBeanDescription("Formatted properties of current coordinator.")
     public String getCurrentCoordinatorFormatted();
 
+    /**
+     * Gets a flag whether local node is in baseline. Returns false if baseline topology is not established.
+     *
+     * @return Return a baseline flag.
+     */
+    @MXBeanDescription("Baseline node flag.")
+    public boolean isNodeInBaseline();
+
     /**
      * Runs IO latency test against all remote server nodes in cluster.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanBaselineTest.java
new file mode 100644
index 0000000000000..00dce83f1cca6
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanBaselineTest.java
@@ -0,0 +1,96 @@
+package org.apache.ignite.util.mbeans;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.mxbean.IgniteMXBean;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ *
+ */
+public class GridMBeanBaselineTest extends GridCommonAbstractTest {
+    /** Client index. */
+    private static final int CLIENT_IDX = 33;
+
+    /** Nodes. */
+    public static final int NODES = 2;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        return super.getConfiguration(igniteInstanceName)
+            .setClientMode(igniteInstanceName.equals(getTestIgniteInstanceName(CLIENT_IDX)))
+            .setDataStorageConfiguration(new DataStorageConfiguration()
+                .setCheckpointFrequency(2_000)
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)))
+            .setDataStorageConfiguration(
+                new DataStorageConfiguration()
+                    .setDefaultDataRegionConfiguration(
+                        new DataRegionConfiguration()
+                            .setMaxSize(200L * 1024 * 1024)
+                            .setPersistenceEnabled(true)));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Test ignite kernal node in baseline test.
+     *
+     * @throws Exception Thrown if test fails.
+     */
+    public void testIgniteKernalNodeInBaselineTest() throws Exception {
+        try {
+            IgniteEx ignite0 = (IgniteEx)startGrids(NODES);
+
+            startGrid(CLIENT_IDX);
+
+            ignite0.cluster().active(true);
+
+            checkBaselineInFromMBean(ignite0);
+
+            startGrid(NODES);
+
+            checkBaselineInFromMBean(ignite0);
+
+            ignite0.cluster().setBaselineTopology(ignite0.cluster().topologyVersion());
+
+            checkBaselineInFromMBean(ignite0);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @param ignite Ignite.
+     */
+    private void checkBaselineInFromMBean(IgniteEx ignite) {
+        Set<Object> cIds = ignite.cluster().currentBaselineTopology().stream()
+            .map(BaselineNode::consistentId)
+            .collect(Collectors.toSet());
+
+        for (Ignite ign : Ignition.allGrids()) {
+            IgniteMXBean igniteMXBean = (IgniteMXBean)ign;
+
+            assertEquals(cIds.contains(ign.cluster().localNode().consistentId()),
+                igniteMXBean.isNodeInBaseline());
+        }
+    }
+
+}

From 40bbe65df3cbd8b3f478812b0e204b2973b98d3d Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Mon, 21 May 2018 15:13:37 +0300
Subject: [PATCH 150/543] IGNITE-8521 Do not attempt to unregister continuous
 query if query ID is null

---
 .../cache/query/continuous/CacheContinuousQueryManager.java    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 19225f8ebd984..55c44b489d7a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -703,7 +703,8 @@ private UUID executeQuery0(CacheEntryUpdatedListener locLsnr,
         catch (IgniteCheckedException e) {
             log.warning("Failed to start continuous query.", e);
 
-            cctx.kernalContext().continuous().stopRoutine(id);
+            if (id != null)
+                cctx.kernalContext().continuous().stopRoutine(id);
 
             throw new IgniteCheckedException("Failed to start continuous query.", e);
         }

From affc9d47b986e054fb0c229dcf95e64a38cab13f Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Mon, 21 May 2018 22:33:50 +0300
Subject: [PATCH 151/543] IGNITE-8544 Use exchange result topology version for
 local wal state management. - Fixes #4039.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../GridDhtPartitionsExchangeFuture.java      |  2 +-
 ...alModeChangeDuringRebalancingSelfTest.java | 66 +++++++++++++++++++
 2 files changed, 67 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 56907b2bf5b2e..54ff58d20e282 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1718,7 +1718,7 @@ public void finishMerged() {
                     grp.topology().onExchangeDone(this, grp.affinity().readyAffinity(res), false);
             }
 
-            cctx.walState().changeLocalStatesOnExchangeDone(exchId.topologyVersion());
+            cctx.walState().changeLocalStatesOnExchangeDone(res);
         }
 
         if (super.onDone(res, err)) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
index aa2613fa3a1a2..8be819f27b4e8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -24,10 +24,12 @@
 import java.nio.file.OpenOption;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
+import com.sun.org.apache.regexp.internal.RE;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
@@ -49,6 +51,7 @@
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
 
 /**
  *
@@ -63,6 +66,9 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr
     /** */
     private static final AtomicReference<CountDownLatch> fileIOLatch = new AtomicReference<>();
 
+    /** Replicated cache name. */
+    private static final String REPL_CACHE = "cache";
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -83,7 +89,11 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr
         cfg.setCacheConfiguration(
             new CacheConfiguration(DEFAULT_CACHE_NAME)
                 // Test checks internal state before and after rebalance, so it is configured to be triggered manually
+                .setRebalanceDelay(-1),
+
+            new CacheConfiguration(REPL_CACHE)
                 .setRebalanceDelay(-1)
+                .setCacheMode(CacheMode.REPLICATED)
         );
 
         cfg.setCommunicationSpi(new TcpCommunicationSpi() {
@@ -294,6 +304,62 @@ public void testLocalAndGlobalWalStateInterdependence() throws Exception {
         assertTrue(grpCtx.walEnabled());
     }
 
+    /**
+     * Test that local WAL mode changing works well with exchanges merge.
+     *
+     * @throws Exception If failed.
+     */
+    public void testWithExchangesMerge() throws Exception {
+        final int nodeCnt = 5;
+        final int keyCnt = 10_000;
+
+        Ignite ignite = startGrids(nodeCnt);
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Integer, Integer> cache = ignite.cache(REPL_CACHE);
+
+        for (int k = 0; k < keyCnt; k++)
+            cache.put(k, k);
+
+        stopGrid(2);
+        stopGrid(3);
+        stopGrid(4);
+
+        // Rewrite data to trigger further rebalance.
+        for (int k = 0; k < keyCnt; k++)
+            cache.put(k, k * 2);
+
+        // Start several grids in parallel to trigger exchanges merge.
+        startGridsMultiThreaded(2, 3);
+
+        for (int nodeIdx = 2; nodeIdx < nodeCnt; nodeIdx++) {
+            CacheGroupContext grpCtx = grid(nodeIdx).cachex(REPL_CACHE).context().group();
+
+            assertFalse(grpCtx.walEnabled());
+        }
+
+        // Invoke rebalance manually.
+        for (Ignite g : G.allGrids())
+            g.cache(REPL_CACHE).rebalance();
+
+        awaitPartitionMapExchange();
+
+        for (int nodeIdx = 2; nodeIdx < nodeCnt; nodeIdx++) {
+            CacheGroupContext grpCtx = grid(nodeIdx).cachex(REPL_CACHE).context().group();
+
+            assertTrue(grpCtx.walEnabled());
+        }
+
+        // Check no data loss.
+        for (int nodeIdx = 2; nodeIdx < nodeCnt; nodeIdx++) {
+            IgniteCache<Integer, Integer> cache0 = grid(nodeIdx).cache(REPL_CACHE);
+
+            for (int k = 0; k < keyCnt; k++)
+                Assert.assertEquals("nodeIdx=" + nodeIdx + ", key=" + k, (Integer) (2 * k), cache0.get(k));
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From 3f4b2ae4f594e8c73b5a9c4b86f2351c2bae591e Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Wed, 23 May 2018 12:24:51 +0300
Subject: [PATCH 152/543] IGNITE-8561 SingleSegmentLogicalRecordsIterator is
 broken - Fixes #4045.

(cherry picked from commit 21678bc)
---
 .../wal/SingleSegmentLogicalRecordsIterator.java           | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
index d5c10cf72bc4d..36e5b0e21b3a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
@@ -47,9 +47,6 @@ public class SingleSegmentLogicalRecordsIterator extends AbstractWalRecordsItera
     /** Segment initialized flag. */
     private boolean segmentInitialized;
 
-    /** Archived segment index. */
-    private long archivedSegIdx;
-
     /** Archive directory. */
     private File archiveDir;
 
@@ -76,7 +73,7 @@ public class SingleSegmentLogicalRecordsIterator extends AbstractWalRecordsItera
     ) throws IgniteCheckedException {
         super(log, sharedCtx, initLogicalRecordsSerializerFactory(sharedCtx), ioFactory, bufSize);
 
-        this.archivedSegIdx = archivedSegIdx;
+        curWalSegmIdx = archivedSegIdx;
         this.archiveDir = archiveDir;
         this.advanceC = advanceC;
 
@@ -106,7 +103,7 @@ private static RecordSerializerFactory initLogicalRecordsSerializerFactory(GridC
             segmentInitialized = true;
 
             FileWriteAheadLogManager.FileDescriptor fd = new FileWriteAheadLogManager.FileDescriptor(
-                new File(archiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(archivedSegIdx)));
+                new File(archiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx)));
 
             try {
                 return initReadHandle(fd, null);

From 49929b73a00f758ef099ea99f8c6f54373c30a95 Mon Sep 17 00:00:00 2001
From: EdShangGG <eshangareev@gridgain.com>
Date: Sat, 28 Apr 2018 19:27:27 +0300
Subject: [PATCH 153/543] IGNITE-7628 SqlQuery hangs indefinitely with
 additional not registered in baseline node.

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../discovery/GridDiscoveryManager.java       |  52 +++--
 .../near/IgniteSqlQueryWithBaselineTest.java  | 184 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite2.java       |   2 +
 3 files changed, 221 insertions(+), 17 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteSqlQueryWithBaselineTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 261e73db97786..15badf27a071b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -367,7 +367,7 @@ public void cleanCachesAndGroups() {
      */
     public void addCacheGroup(CacheGroupDescriptor grpDesc, IgnitePredicate<ClusterNode> filter, CacheMode cacheMode) {
         CacheGroupAffinity old = registeredCacheGrps.put(grpDesc.groupId(),
-            new CacheGroupAffinity(grpDesc.cacheOrGroupName(), filter, cacheMode));
+            new CacheGroupAffinity(grpDesc.cacheOrGroupName(), filter, cacheMode, grpDesc.persistenceEnabled()));
 
         assert old == null : old;
     }
@@ -2387,12 +2387,6 @@ else if (node.version().compareTo(minVer) < 0)
         assert !rmtNodes.contains(loc) : "Remote nodes collection shouldn't contain local node" +
             " [rmtNodes=" + rmtNodes + ", loc=" + loc + ']';
 
-        Map<Integer, List<ClusterNode>> allCacheNodes = U.newHashMap(allNodes.size());
-        Map<Integer, List<ClusterNode>> cacheGrpAffNodes = U.newHashMap(allNodes.size());
-        Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(NodeOrderComparator.getInstance());
-
-        fillAffinityNodeCaches(allNodes, allCacheNodes, cacheGrpAffNodes, rmtNodesWithCaches);
-
         BaselineTopology blt = state.baselineTopology();
 
         if (blt != null) {
@@ -2435,6 +2429,13 @@ else if (node.version().compareTo(minVer) < 0)
             baselineNodes = null;
         }
 
+        Map<Integer, List<ClusterNode>> allCacheNodes = U.newHashMap(allNodes.size());
+        Map<Integer, List<ClusterNode>> cacheGrpAffNodes = U.newHashMap(allNodes.size());
+        Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(NodeOrderComparator.getInstance());
+
+        fillAffinityNodeCaches(allNodes, allCacheNodes, cacheGrpAffNodes, rmtNodesWithCaches,
+                nodeIdToConsIdx == null ? null : nodeIdToConsIdx.keySet());
+
         return new DiscoCache(
             topVer,
             state,
@@ -3127,23 +3128,29 @@ private static class CacheGroupAffinity {
         /** Cache mode. */
         private final CacheMode cacheMode;
 
+        /** Persistent cache group or not. */
+        private final boolean persistentCacheGrp;
+
         /**
          * @param name Name.
          * @param cacheFilter Node filter.
          * @param cacheMode Cache mode.
+         * @param persistentCacheGrp Persistence is configured for cache or not.
          */
         CacheGroupAffinity(
-            String name,
-            IgnitePredicate<ClusterNode> cacheFilter,
-            CacheMode cacheMode) {
+                String name,
+                IgnitePredicate<ClusterNode> cacheFilter,
+                CacheMode cacheMode,
+                boolean persistentCacheGrp) {
             this.name = name;
             this.cacheFilter = cacheFilter;
             this.cacheMode = cacheMode;
+            this.persistentCacheGrp = persistentCacheGrp;
         }
 
         /** {@inheritDoc} */
         @Override public String toString() {
-            return "CacheGroupAffinity [name=" + name + ']';
+            return S.toString(CacheGroupAffinity.class, this);
         }
     }
 
@@ -3268,14 +3275,19 @@ private Boolean cacheClientNode(ClusterNode node) {
 
     /**
      * Fills affinity node caches.
-     *
      * @param allNodes All nodes.
      * @param allCacheNodes All cache nodes.
      * @param cacheGrpAffNodes Cache group aff nodes.
      * @param rmtNodesWithCaches Rmt nodes with caches.
-     */
-    private void fillAffinityNodeCaches(List<ClusterNode> allNodes, Map<Integer, List<ClusterNode>> allCacheNodes,
-        Map<Integer, List<ClusterNode>> cacheGrpAffNodes, Set<ClusterNode> rmtNodesWithCaches) {
+     * @param bltNodes Baseline node ids.
+     */
+    private void fillAffinityNodeCaches(
+        List<ClusterNode> allNodes,
+        Map<Integer, List<ClusterNode>> allCacheNodes,
+        Map<Integer, List<ClusterNode>> cacheGrpAffNodes,
+        Set<ClusterNode> rmtNodesWithCaches,
+        Set<UUID> bltNodes
+    ) {
         for (ClusterNode node : allNodes) {
             assert node.order() != 0 : "Invalid node order [locNode=" + localNode() + ", node=" + node + ']';
             assert !node.isDaemon();
@@ -3285,6 +3297,9 @@ private void fillAffinityNodeCaches(List<ClusterNode> allNodes, Map<Integer, Lis
                 Integer grpId = e.getKey();
 
                 if (CU.affinityNode(node, grpAff.cacheFilter)) {
+                    if (grpAff.persistentCacheGrp && bltNodes != null && !bltNodes.contains(node.id())) // Filter out.
+                        continue;
+                    
                     List<ClusterNode> nodes = cacheGrpAffNodes.get(grpId);
 
                     if (nodes == null)
@@ -3324,7 +3339,10 @@ public DiscoCache createDiscoCacheOnCacheChange(
         Map<Integer, List<ClusterNode>> cacheGrpAffNodes = U.newHashMap(allNodes.size());
         Set<ClusterNode> rmtNodesWithCaches = new TreeSet<>(NodeOrderComparator.getInstance());
 
-        fillAffinityNodeCaches(allNodes, allCacheNodes, cacheGrpAffNodes, rmtNodesWithCaches);
+        Map<UUID, Short> nodeIdToConsIdx = discoCache.nodeIdToConsIdx;
+
+        fillAffinityNodeCaches(allNodes, allCacheNodes, cacheGrpAffNodes, rmtNodesWithCaches,
+                nodeIdToConsIdx == null ? null : nodeIdToConsIdx.keySet());
 
         return new DiscoCache(
             topVer,
@@ -3340,7 +3358,7 @@ public DiscoCache createDiscoCacheOnCacheChange(
             cacheGrpAffNodes,
             discoCache.nodeMap,
             discoCache.alives,
-            discoCache.nodeIdToConsIdx,
+            nodeIdToConsIdx,
             discoCache.consIdxToNodeId,
             discoCache.minimumNodeVersion(),
             discoCache.minimumServerNodeVersion());
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteSqlQueryWithBaselineTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteSqlQueryWithBaselineTest.java
new file mode 100644
index 0000000000000..203a319a3c6b7
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteSqlQueryWithBaselineTest.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+
+import java.io.Serializable;
+import java.util.Collection;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import javax.cache.Cache;
+
+/**
+ *
+ */
+public class IgniteSqlQueryWithBaselineTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+                new DataStorageConfiguration()
+                        .setDefaultDataRegionConfiguration(
+                                new DataRegionConfiguration()
+                                        .setMaxSize(200 * 1024 * 1024)
+                                        .setPersistenceEnabled(true)
+                        )
+        );
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** */
+    public static class C1 implements Serializable {
+        /** */
+        private static final long serialVersionUID = 1L;
+
+        /** */
+        public C1(int id) {
+            this.id = id;
+        }
+
+        /** */
+        @QuerySqlField(index = true)
+        protected Integer id;
+    }
+
+    /** */
+    public static class C2 implements Serializable {
+        /** */
+        private static final long serialVersionUID = 1L;
+
+        /** */
+        C2(int id) {
+            this.id = id;
+        }
+
+        /** */
+        @QuerySqlField(index = true)
+        protected Integer id;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testQueryWithNodeNotInBLT() throws Exception {
+        startGrids(2);
+
+        grid(0).cluster().active(true);
+
+        startGrid(2); //Start extra node.
+
+        doQuery();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testQueryWithoutBLTNode() throws Exception {
+        startGrids(2);
+
+        grid(0).cluster().active(true);
+
+        startGrid(2); //Start extra node.
+        stopGrid(1);
+
+        doQuery();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testQueryFromNotBLTNode() throws Exception {
+        startGrid(1);
+
+        grid(1).cluster().active(true);
+
+        startGrid(0); //Start extra node.
+
+        doQuery();
+    }
+
+    /**
+     *
+     */
+    private void doQuery() {
+        CacheConfiguration<Integer, C1> c1Conf = new CacheConfiguration<>("C1");
+        c1Conf.setIndexedTypes(Integer.class, C1.class).setBackups(2);
+
+        CacheConfiguration<Integer, C2> c2Conf = new CacheConfiguration<>("C2");
+        c2Conf.setIndexedTypes(Integer.class, C2.class).setBackups(2);
+
+        final IgniteCache<Integer, C1> cache = grid(0).getOrCreateCache(c1Conf);
+
+        final IgniteCache<Integer, C2> cache1 = grid(0).getOrCreateCache(c2Conf);
+
+        for (int i = 0; i < 100; i++) {
+            cache.put(i, new C1(i));
+
+            cache1.put(i, new C2(i));
+        }
+
+        String sql = "SELECT C1.*" +
+                " from C1 inner join \"C2\".C2 as D on C1.id = D.id" +
+                " order by C1.id asc";
+
+        SqlQuery<Integer, C1> qry = new SqlQuery<>(C1.class, sql);
+
+        qry.setDistributedJoins(true);
+
+        log.info("before query run...");
+
+        Collection<Cache.Entry<Integer, C1>> res = cache.query(qry).getAll();
+
+        log.info("result size: " + res.size());
+    }
+}
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index 11d98e2c451e8..5b888ce7c65f8 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNodeRestartSelfTest2;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNodeRestartTxSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryStopOnCancelOrTimeoutDistributedJoinSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteSqlQueryWithBaselineTest;
 import org.apache.ignite.internal.processors.cache.index.DynamicColumnsConcurrentAtomicPartitionedSelfTest;
 import org.apache.ignite.internal.processors.cache.index.DynamicColumnsConcurrentAtomicReplicatedSelfTest;
 import org.apache.ignite.internal.processors.cache.index.DynamicColumnsConcurrentTransactionalPartitionedSelfTest;
@@ -87,6 +88,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheClientQueryReplicatedNodeRestartSelfTest.class);
         suite.addTestSuite(IgniteCacheQueryNodeFailTest.class);
         suite.addTestSuite(IgniteCacheQueryNodeRestartSelfTest.class);
+        suite.addTestSuite(IgniteSqlQueryWithBaselineTest.class);
         suite.addTestSuite(IgniteChangingBaselineCacheQueryNodeRestartSelfTest.class);
         suite.addTestSuite(IgniteStableBaselineCacheQueryNodeRestartsSelfTest.class);
         suite.addTestSuite(IgniteCacheQueryNodeRestartSelfTest2.class);

From 95b749460b5fa32deecb946865944e854d9745d3 Mon Sep 17 00:00:00 2001
From: Dmitriy Sorokin <d.w.sorokin@gmail.com>
Date: Thu, 24 May 2018 16:03:41 +0300
Subject: [PATCH 154/543] IGNITE-8584 Provide ability to terminate any thread
 with enabled test features.

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../internal/util/nio/GridNioServer.java      |  2 +-
 .../worker/WorkersControlMXBeanImpl.java      | 29 ++++++
 .../ignite/mxbean/WorkersControlMXBean.java   | 30 ++++++
 .../testsuites/IgniteUtilSelfTestSuite.java   |  2 +
 .../util/mbeans/WorkersControlMXBeanTest.java | 98 +++++++++++++++++++
 5 files changed, 160 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/util/mbeans/WorkersControlMXBeanTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 3597a050051f3..da3438e069414 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -1790,7 +1790,7 @@ else if (!closed) {
                     if (err == null)
                         lsnr.onFailure(SYSTEM_WORKER_TERMINATION,
                             new IllegalStateException("Thread " + name() + " is terminated unexpectedly"));
-                    else if (err instanceof InterruptedException)
+                    else
                         lsnr.onFailure(SYSTEM_WORKER_TERMINATION, err);
                 }
                 else if (err != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
index 9e427e8f2ab1c..65f872c0162ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
@@ -19,6 +19,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.mxbean.WorkersControlMXBean;
 
@@ -59,4 +60,32 @@ public WorkersControlMXBeanImpl(WorkersRegistry registry) {
 
         return true;
     }
+
+    /** {@inheritDoc} */
+    @Override public boolean stopThreadByUniqueName(String name) {
+        Thread[] threads = Thread.getAllStackTraces().keySet().stream()
+            .filter(t -> Objects.equals(t.getName(), name))
+            .toArray(Thread[]::new);
+
+        if (threads.length != 1)
+            return false;
+
+        threads[0].stop();
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean stopThreadById(long id) {
+        Thread[] threads = Thread.getAllStackTraces().keySet().stream()
+            .filter(t -> t.getId() == id)
+            .toArray(Thread[]::new);
+
+        if (threads.length != 1)
+            return false;
+
+        threads[0].stop();
+
+        return true;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
index 0f5419b3b4aac..b999ab7d716de 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/WorkersControlMXBean.java
@@ -46,4 +46,34 @@ public interface WorkersControlMXBean {
         "Name of worker to terminate."
     )
     public boolean terminateWorker(String name);
+
+    /**
+     * Stops thread by {@code name}, if exists and unique.
+     *
+     * @param name Thread name.
+     * @return {@code True} if thread has been stopped successfully, {@code false} otherwise.
+     */
+    @MXBeanDescription("Stops thread by unique name.")
+    @MXBeanParametersNames(
+        "name"
+    )
+    @MXBeanParametersDescriptions(
+        "Name of thread to stop."
+    )
+    public boolean stopThreadByUniqueName(String name);
+
+    /**
+     * Stops thread by {@code id}, if exists.
+     *
+     * @param id Thread id.
+     * @return {@code True} if thread has been stopped successfully, {@code false} otherwise.
+     */
+    @MXBeanDescription("Stops thread by id.")
+    @MXBeanParametersNames(
+        "id"
+    )
+    @MXBeanParametersDescriptions(
+        "Id of thread to stop."
+    )
+    public boolean stopThreadById(long id);
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index 124300585b4e7..791621ff598b0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -53,6 +53,7 @@
 import org.apache.ignite.util.mbeans.GridMBeanDisableSelfTest;
 import org.apache.ignite.util.mbeans.GridMBeanExoticNamesSelfTest;
 import org.apache.ignite.util.mbeans.GridMBeanSelfTest;
+import org.apache.ignite.util.mbeans.WorkersControlMXBeanTest;
 
 /**
  * Test suite for Ignite utility classes.
@@ -92,6 +93,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridCacheUtilsSelfTest.class);
         suite.addTestSuite(IgniteExceptionRegistrySelfTest.class);
         suite.addTestSuite(GridMessageCollectionTest.class);
+        suite.addTestSuite(WorkersControlMXBeanTest.class);
 
         // Metrics.
         suite.addTestSuite(ClusterMetricsSnapshotSerializeSelfTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/WorkersControlMXBeanTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/WorkersControlMXBeanTest.java
new file mode 100644
index 0000000000000..c1c2fdae87726
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/WorkersControlMXBeanTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util.mbeans;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.internal.worker.WorkersControlMXBeanImpl;
+import org.apache.ignite.mxbean.WorkersControlMXBean;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * {@link WorkersControlMXBean} test.
+ */
+public class WorkersControlMXBeanTest extends GridCommonAbstractTest {
+    /** Test thread name. */
+    private static final String TEST_THREAD_NAME = "test-thread";
+
+    /**
+     * @throws Exception Thrown if test fails.
+     */
+    public void testStopThreadByUniqueName() throws Exception {
+        WorkersControlMXBean workersCtrlMXBean = new WorkersControlMXBeanImpl(null);
+
+        Thread t = startTestThread();
+
+        assertTrue(workersCtrlMXBean.stopThreadByUniqueName(TEST_THREAD_NAME));
+
+        t.join(500);
+
+        assertFalse(workersCtrlMXBean.stopThreadByUniqueName(TEST_THREAD_NAME));
+
+        Thread t1 = startTestThread();
+        Thread t2 = startTestThread();
+
+        assertFalse(workersCtrlMXBean.stopThreadByUniqueName(TEST_THREAD_NAME));
+
+        t1.stop();
+        t2.stop();
+    }
+
+    /**
+     * @throws Exception Thrown if test fails.
+     */
+    public void testStopThreadById() throws Exception {
+        WorkersControlMXBean workersCtrlMXBean = new WorkersControlMXBeanImpl(null);
+
+        Thread t1 = startTestThread();
+        Thread t2 = startTestThread();
+
+        assertTrue(workersCtrlMXBean.stopThreadById(t1.getId()));
+        assertTrue(workersCtrlMXBean.stopThreadById(t2.getId()));
+
+        t1.join(500);
+        t2.join(500);
+
+        assertFalse(workersCtrlMXBean.stopThreadById(t1.getId()));
+        assertFalse(workersCtrlMXBean.stopThreadById(t2.getId()));
+    }
+
+    /**
+     * @return Started thread.
+     */
+    private static Thread startTestThread() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        Thread t = new Thread(TEST_THREAD_NAME) {
+            public void run() {
+                latch.countDown();
+
+                for (;;)
+                    ;
+            }
+        };
+
+        t.start();
+
+        assertTrue(latch.await(500, TimeUnit.MILLISECONDS));
+
+        assertTrue(t.isAlive());
+
+        return t;
+    }
+}

From 8ac69e68ecfc1c31b02004d33157d3f9b85a7b6e Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Thu, 24 May 2018 16:56:42 +0300
Subject: [PATCH 155/543] IGNITE-8563 Fixed WAL file archiver does not
 propagate file archiving error to error handler

---
 .../wal/FileWriteAheadLogManager.java         | 28 +++++++------------
 .../FsyncModeFileWriteAheadLogManager.java    | 28 +++++++------------
 2 files changed, 20 insertions(+), 36 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index e095f6a26b30a..40ebcf04289fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1591,29 +1591,21 @@ private synchronized boolean locked(long absIdx) {
                     if (stopped)
                         break;
 
-                    try {
-                        final SegmentArchiveResult res = archiveSegment(toArchive);
+                    final SegmentArchiveResult res = archiveSegment(toArchive);
 
-                        synchronized (this) {
-                            while (locked.containsKey(toArchive) && !stopped)
-                                wait();
+                    synchronized (this) {
+                        while (locked.containsKey(toArchive) && !stopped)
+                            wait();
 
-                            // Then increase counter to allow rollover on clean working file
-                            changeLastArchivedIndexAndNotifyWaiters(toArchive);
+                        // Then increase counter to allow rollover on clean working file
+                        changeLastArchivedIndexAndNotifyWaiters(toArchive);
 
-                            notifyAll();
-                        }
-
-                        if (evt.isRecordable(EventType.EVT_WAL_SEGMENT_ARCHIVED))
-                            evt.record(new WalSegmentArchivedEvent(cctx.discovery().localNode(),
-                                res.getAbsIdx(), res.getDstArchiveFile()));
+                        notifyAll();
                     }
-                    catch (IgniteCheckedException e) {
-                        synchronized (this) {
-                            cleanErr = e;
 
-                            notifyAll();
-                        }
+                    if (evt.isRecordable(EventType.EVT_WAL_SEGMENT_ARCHIVED)) {
+                        evt.record(new WalSegmentArchivedEvent(cctx.discovery().localNode(),
+                            res.getAbsIdx(), res.getDstArchiveFile()));
                     }
                 }
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index f582f193f9198..79a3e1966dede 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -1431,29 +1431,21 @@ private synchronized void release(long absIdx) {
                     if (stopped)
                         break;
 
-                    try {
-                        final SegmentArchiveResult res = archiveSegment(toArchive);
+                    final SegmentArchiveResult res = archiveSegment(toArchive);
 
-                        synchronized (this) {
-                            while (locked.containsKey(toArchive) && !stopped)
-                                wait();
+                    synchronized (this) {
+                        while (locked.containsKey(toArchive) && !stopped)
+                            wait();
 
-                            // Then increase counter to allow rollover on clean working file
-                            changeLastArchivedIndexAndWakeupCompressor(toArchive);
+                        // Then increase counter to allow rollover on clean working file
+                        changeLastArchivedIndexAndWakeupCompressor(toArchive);
 
-                            notifyAll();
-                        }
-
-                        if (evt.isRecordable(EventType.EVT_WAL_SEGMENT_ARCHIVED))
-                            evt.record(new WalSegmentArchivedEvent(cctx.discovery().localNode(),
-                                res.getAbsIdx(), res.getDstArchiveFile()));
+                        notifyAll();
                     }
-                    catch (IgniteCheckedException e) {
-                        synchronized (this) {
-                            cleanException = e;
 
-                            notifyAll();
-                        }
+                    if (evt.isRecordable(EventType.EVT_WAL_SEGMENT_ARCHIVED)) {
+                        evt.record(new WalSegmentArchivedEvent(cctx.discovery().localNode(),
+                            res.getAbsIdx(), res.getDstArchiveFile()));
                     }
                 }
             }

From ef63514bfa38c595017ebec62c42a93c90e0a062 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Thu, 24 May 2018 18:04:02 +0300
Subject: [PATCH 156/543] IGNITE-8583 DataStorageMetricsMXBean.getOffHeapSize
 include checkpoint buffer size - Fixes #4054.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit 86c1899)
---
 .../org/apache/ignite/DataRegionMetrics.java  | 11 ++-
 .../org/apache/ignite/DataStorageMetrics.java | 21 ++++
 .../persistence/DataRegionMetricsImpl.java    | 31 ++++--
 .../DataRegionMetricsMXBeanImpl.java          |  9 +-
 .../DataRegionMetricsSnapshot.java            | 17 +++-
 .../persistence/DataStorageMetricsImpl.java   | 54 +++++++++++
 .../DataStorageMetricsSnapshot.java           | 33 ++++++-
 .../GridCacheDatabaseSharedManager.java       | 46 +++++++++
 .../cluster/PlatformClusterGroup.java         |  4 +-
 .../visor/cache/VisorMemoryMetrics.java       |  4 +-
 .../mxbean/DataStorageMetricsMXBean.java      | 12 +++
 .../db/IgnitePdsDataRegionMetricsTest.java    | 97 +++++++++++++++++--
 .../ApiParity/DataRegionMetricsParityTest.cs  |  4 +-
 .../ApiParity/DataStorageMetricsParityTest.cs |  5 +-
 14 files changed, 318 insertions(+), 30 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java b/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
index ca2fc66818ee1..88dcd168e0104 100644
--- a/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/DataRegionMetrics.java
@@ -146,11 +146,18 @@ public interface DataRegionMetrics {
     public long getPhysicalMemorySize();
 
     /**
-     * Gets checkpoint buffer size in pages.
+     * Gets used checkpoint buffer size in pages.
      *
      * @return Checkpoint buffer size in pages.
      */
-    public long getCheckpointBufferPages();
+    public long getUsedCheckpointBufferPages();
+
+    /**
+     * Gets used checkpoint buffer size in bytes.
+     *
+     * @return Checkpoint buffer size in bytes.
+     */
+    public long getUsedCheckpointBufferSize();
 
     /**
      * Gets checkpoint buffer size in bytes.
diff --git a/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java b/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java
index 5fb2b1e088700..cdde0aced29d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/DataStorageMetrics.java
@@ -191,4 +191,25 @@ public interface DataStorageMetrics {
      * @return Total size of memory allocated in bytes.
      */
     public long getTotalAllocatedSize();
+
+    /**
+     * Gets used checkpoint buffer size in pages.
+     *
+     * @return Checkpoint buffer size in pages.
+     */
+    public long getUsedCheckpointBufferPages();
+
+    /**
+     * Gets used checkpoint buffer size in bytes.
+     *
+     * @return Checkpoint buffer size in bytes.
+     */
+    public long getUsedCheckpointBufferSize();
+
+    /**
+     * Checkpoint buffer size in bytes.
+     *
+     * @return Checkpoint buffer size in bytes.
+     */
+    public long getCheckpointBufferSize();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
index cb13747b775a6..a82f73bd7966f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
@@ -61,6 +61,9 @@ public class DataRegionMetricsImpl implements DataRegionMetrics, AllocatedPageTr
     /** */
     private final AtomicLong offHeapSize = new AtomicLong();
 
+    /** */
+    private final AtomicLong checkpointBufferSize = new AtomicLong();
+
     /** */
     private volatile boolean metricsEnabled;
 
@@ -129,7 +132,7 @@ public DataRegionMetricsImpl(DataRegionConfiguration memPlcCfg, @Nullable Ignite
     @Override public long getTotalAllocatedSize() {
         assert pageMem != null;
 
-        return getTotalAllocatedPages() * pageMem.pageSize();
+        return getTotalAllocatedPages() * (persistenceEnabled ? pageMem.pageSize() : pageMem.systemPageSize());
     }
 
     /** {@inheritDoc} */
@@ -211,12 +214,12 @@ public DataRegionMetricsImpl(DataRegionConfiguration memPlcCfg, @Nullable Ignite
 
     /** {@inheritDoc} */
     @Override public long getPhysicalMemorySize() {
-        return getPhysicalMemoryPages() * pageMem.pageSize();
+        return getPhysicalMemoryPages() * pageMem.systemPageSize();
     }
 
     /** {@inheritDoc} */
-    @Override public long getCheckpointBufferPages() {
-        if (!metricsEnabled)
+    @Override public long getUsedCheckpointBufferPages() {
+        if (!metricsEnabled || !persistenceEnabled)
             return 0;
 
         assert pageMem != null;
@@ -224,9 +227,17 @@ public DataRegionMetricsImpl(DataRegionConfiguration memPlcCfg, @Nullable Ignite
         return pageMem.checkpointBufferPagesCount();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getUsedCheckpointBufferSize() {
+        return getUsedCheckpointBufferPages() * pageMem.systemPageSize();
+    }
+
     /** {@inheritDoc} */
     @Override public long getCheckpointBufferSize() {
-        return getCheckpointBufferPages() * pageMem.pageSize();
+        if (!metricsEnabled || !persistenceEnabled)
+            return 0;
+
+        return checkpointBufferSize.get();
     }
 
     /** {@inheritDoc} */
@@ -276,17 +287,23 @@ public DataRegionMetricsImpl(DataRegionConfiguration memPlcCfg, @Nullable Ignite
         if (!metricsEnabled)
             return 0;
 
-        return pageMem.loadedPages() * pageMem.pageSize();
+        return pageMem.loadedPages() * pageMem.systemPageSize();
     }
 
     /**
-     *
      * @param size Region size.
      */
     public void updateOffHeapSize(long size) {
         this.offHeapSize.addAndGet(size);
     }
 
+    /**
+     * @param size Checkpoint buffer size.
+     */
+    public void updateCheckpointBufferSize(long size) {
+        this.checkpointBufferSize.addAndGet(size);
+    }
+
     /**
      * Updates pageReplaceRate metric.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
index 9ab682487512c..f83716816e53b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsMXBeanImpl.java
@@ -97,8 +97,13 @@ class DataRegionMetricsMXBeanImpl implements DataRegionMetricsMXBean {
     }
 
     /** {@inheritDoc} */
-    @Override public long getCheckpointBufferPages() {
-        return memMetrics.getCheckpointBufferPages();
+    @Override public long getUsedCheckpointBufferPages() {
+        return memMetrics.getUsedCheckpointBufferPages();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getUsedCheckpointBufferSize() {
+        return memMetrics.getUsedCheckpointBufferSize();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
index 6126c4baa5c1f..f119419ef8eaa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsSnapshot.java
@@ -60,7 +60,10 @@ public class DataRegionMetricsSnapshot implements DataRegionMetrics {
     private long physicalMemorySize;
 
     /** */
-    private long checkpointBufferPages;
+    private long usedCheckpointBufferPages;
+
+    /** */
+    private long usedCheckpointBufferSize;
 
     /** */
     private long checkpointBufferSize;
@@ -99,7 +102,8 @@ public DataRegionMetricsSnapshot(DataRegionMetrics metrics) {
         pageReplaceAge = metrics.getPagesReplaceAge();
         physicalMemoryPages = metrics.getPhysicalMemoryPages();
         physicalMemorySize = metrics.getPhysicalMemorySize();
-        checkpointBufferPages = metrics.getCheckpointBufferPages();
+        usedCheckpointBufferPages = metrics.getUsedCheckpointBufferPages();
+        usedCheckpointBufferSize = metrics.getUsedCheckpointBufferSize();
         checkpointBufferSize = metrics.getCheckpointBufferSize();
         pageSize = metrics.getPageSize();
         readPages = metrics.getPagesRead();
@@ -170,8 +174,13 @@ public DataRegionMetricsSnapshot(DataRegionMetrics metrics) {
     }
 
     /** {@inheritDoc} */
-    @Override public long getCheckpointBufferPages() {
-        return checkpointBufferPages;
+    @Override public long getUsedCheckpointBufferPages() {
+        return usedCheckpointBufferPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getUsedCheckpointBufferSize() {
+        return usedCheckpointBufferSize;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
index b7f99e5038924..03955a4b799cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsImpl.java
@@ -396,6 +396,60 @@ public DataStorageMetricsImpl(
         return totalAllocatedSize;
     }
 
+    /** {@inheritDoc} */
+    @Override public long getUsedCheckpointBufferPages() {
+        if (!metricsEnabled)
+            return 0;
+
+        Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+        if (F.isEmpty(regionMetrics0))
+            return 0;
+
+        long usedCheckpointBufferPages = 0L;
+
+        for (DataRegionMetrics rm : regionMetrics0)
+            usedCheckpointBufferPages += rm.getUsedCheckpointBufferPages();
+
+        return usedCheckpointBufferPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getUsedCheckpointBufferSize() {
+        if (!metricsEnabled)
+            return 0;
+
+        Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+        if (F.isEmpty(regionMetrics0))
+            return 0;
+
+        long usedCheckpointBufferSize = 0L;
+
+        for (DataRegionMetrics rm : regionMetrics0)
+            usedCheckpointBufferSize += rm.getUsedCheckpointBufferSize();
+
+        return usedCheckpointBufferSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCheckpointBufferSize(){
+        if (!metricsEnabled)
+            return 0;
+
+        Collection<DataRegionMetrics> regionMetrics0 = regionMetrics;
+
+        if (F.isEmpty(regionMetrics0))
+            return 0;
+
+        long checkpointBufferSize = 0L;
+
+        for (DataRegionMetrics rm : regionMetrics0)
+            checkpointBufferSize += rm.getCheckpointBufferSize();
+
+        return checkpointBufferSize;
+    }
+
     /**
      * @param wal Write-ahead log manager.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java
index cb522791652c2..c3bcd5b9d6df9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMetricsSnapshot.java
@@ -69,7 +69,16 @@ public class DataStorageMetricsSnapshot implements DataStorageMetrics {
     private long walLastRollOverTime;
 
     /** */
-    private long checkpointTotalSize;
+    private long checkpointTotalTime;
+
+    /** */
+    private long usedCheckpointBufferSize;
+
+    /** */
+    private long usedCheckpointBufferPages;
+
+    /** */
+    private long checkpointBufferSize;
 
     /** */
     private long dirtyPages;
@@ -111,7 +120,10 @@ public DataStorageMetricsSnapshot(DataStorageMetrics metrics) {
         lastCpCowPages = metrics.getLastCheckpointCopiedOnWritePagesNumber();
         walTotalSize = metrics.getWalTotalSize();
         walLastRollOverTime = metrics.getWalLastRollOverTime();
-        checkpointTotalSize = metrics.getCheckpointTotalTime();
+        checkpointTotalTime = metrics.getCheckpointTotalTime();
+        usedCheckpointBufferSize = metrics.getUsedCheckpointBufferSize();
+        usedCheckpointBufferPages = metrics.getUsedCheckpointBufferPages();
+        checkpointBufferSize = metrics.getCheckpointBufferSize();
         dirtyPages = metrics.getDirtyPages();
         readPages = metrics.getPagesRead();
         writtenPages = metrics.getPagesWritten();
@@ -198,7 +210,7 @@ public DataStorageMetricsSnapshot(DataStorageMetrics metrics) {
 
     /** {@inheritDoc} */
     @Override public long getCheckpointTotalTime() {
-        return checkpointTotalSize;
+        return checkpointTotalTime;
     }
 
     /** {@inheritDoc} */
@@ -236,6 +248,21 @@ public DataStorageMetricsSnapshot(DataStorageMetrics metrics) {
         return totalAllocatedSize;
     }
 
+    /** {@inheritDoc} */
+    @Override public long getUsedCheckpointBufferPages() {
+        return usedCheckpointBufferPages;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getUsedCheckpointBufferSize() {
+        return usedCheckpointBufferSize;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCheckpointBufferSize(){
+        return checkpointBufferSize;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(DataStorageMetricsSnapshot.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index ffa7259669775..6ce743d8d9632 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -84,6 +84,7 @@
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.DirectMemoryRegion;
 import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
 import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.FullPageId;
@@ -1019,6 +1020,51 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize, long chpBufSi
         return pageMem;
     }
 
+    /**
+     * @param memoryProvider0 Memory provider.
+     * @param memMetrics Memory metrics.
+     * @return Wrapped memory provider.
+     */
+    @Override protected DirectMemoryProvider wrapMetricsMemoryProvider(
+            final DirectMemoryProvider memoryProvider0,
+            final DataRegionMetricsImpl memMetrics
+    ) {
+        return new DirectMemoryProvider() {
+            private AtomicInteger checkPointBufferIdxCnt = new AtomicInteger();
+
+            private final DirectMemoryProvider memProvider = memoryProvider0;
+
+            @Override public void initialize(long[] chunkSizes) {
+                memProvider.initialize(chunkSizes);
+
+                checkPointBufferIdxCnt.set(chunkSizes.length);
+            }
+
+            @Override public void shutdown() {
+                memProvider.shutdown();
+            }
+
+            @Override public DirectMemoryRegion nextRegion() {
+                DirectMemoryRegion nextMemoryRegion = memProvider.nextRegion();
+
+                if (nextMemoryRegion == null)
+                    return null;
+
+                int idx = checkPointBufferIdxCnt.decrementAndGet();
+
+                long chunkSize = nextMemoryRegion.size();
+
+                // Checkpoint chunk last in the long[] chunkSizes.
+                if (idx != 0)
+                    memMetrics.updateOffHeapSize(chunkSize);
+                else
+                    memMetrics.updateCheckpointBufferSize(chunkSize);
+
+                return nextMemoryRegion;
+            }
+        };
+    }
+
     /**
      * Resolves throttling policy according to the settings.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java
index f95e69b985e1c..d7a6b565580c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cluster/PlatformClusterGroup.java
@@ -560,8 +560,8 @@ private static void writeDataRegionMetrics(BinaryRawWriter writer, DataRegionMet
         writer.writeFloat(metrics.getPagesReplaceAge());
         writer.writeLong(metrics.getPhysicalMemoryPages());
         writer.writeLong(metrics.getPhysicalMemorySize());
-        writer.writeLong(metrics.getCheckpointBufferPages());
-        writer.writeLong(metrics.getCheckpointBufferSize());
+        writer.writeLong(metrics.getUsedCheckpointBufferPages());
+        writer.writeLong(metrics.getUsedCheckpointBufferSize());
         writer.writeInt(metrics.getPageSize());
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java
index c19fd3607afd8..5b46220d8f7df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorMemoryMetrics.java
@@ -96,8 +96,8 @@ public VisorMemoryMetrics(DataRegionMetrics m) {
         physicalMemoryPages = m.getPhysicalMemoryPages();
         totalAllocatedSz = m.getTotalAllocatedSize();
         physicalMemSz = m.getPhysicalMemorySize();
-        cpBufPages = m.getCheckpointBufferPages();
-        cpBufSz = m.getCheckpointBufferSize();
+        cpBufPages = m.getUsedCheckpointBufferPages();
+        cpBufSz = m.getUsedCheckpointBufferSize();
         pageSize = m.getPageSize();
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java
index 2450874e64df9..2069099957747 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMetricsMXBean.java
@@ -56,6 +56,18 @@ public interface DataStorageMetricsMXBean extends DataStorageMetrics {
     @MXBeanDescription("Total checkpoint time from last restart.")
     @Override long getCheckpointTotalTime();
 
+    /** {@inheritDoc} */
+    @MXBeanDescription("Used checkpoint buffer size in pages.")
+    @Override long getUsedCheckpointBufferPages();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Used checkpoint buffer size in bytes.")
+    @Override long getUsedCheckpointBufferSize();
+
+    /** {@inheritDoc} */
+    @MXBeanDescription("Total size in bytes for checkpoint buffer.")
+    @Override  long getCheckpointBufferSize();
+
     /** {@inheritDoc} */
     @MXBeanDescription("Duration of the last checkpoint in milliseconds.")
     @Override long getLastCheckpointDuration();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
index 4b7d6aeb8b7c1..18a47814755d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
@@ -17,10 +17,11 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.db;
 
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.Random;
+import java.util.HashMap;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.stream.Collectors;
 import org.apache.ignite.DataRegionMetrics;
@@ -32,14 +33,20 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
 
 import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
 
@@ -53,6 +60,9 @@ public class IgnitePdsDataRegionMetricsTest extends GridCommonAbstractTest {
     /** */
     private static final long INIT_REGION_SIZE = 10 << 20;
 
+    /** */
+    private static final long MAX_REGION_SIZE = INIT_REGION_SIZE * 10;
+
     /** */
     private static final int ITERATIONS = 3;
 
@@ -72,11 +82,13 @@ public class IgnitePdsDataRegionMetricsTest extends GridCommonAbstractTest {
         ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
 
         DataStorageConfiguration memCfg = new DataStorageConfiguration()
-            .setDefaultDataRegionConfiguration(
-                new DataRegionConfiguration()
-                    .setInitialSize(INIT_REGION_SIZE)
-                    .setPersistenceEnabled(true)
-                    .setMetricsEnabled(true));
+                .setDefaultDataRegionConfiguration(
+                        new DataRegionConfiguration()
+                                .setInitialSize(INIT_REGION_SIZE)
+                                .setMaxSize(MAX_REGION_SIZE)
+                                .setPersistenceEnabled(true)
+                                .setMetricsEnabled(true))
+                .setCheckpointFrequency(1000);
 
         cfg.setDataStorageConfiguration(memCfg);
 
@@ -194,6 +206,79 @@ public void testMemoryUsageMultipleNodes() throws Exception {
         checkMetricsConsistency(node2, grpIds);
     }
 
+    /**
+     * Test for check checkpoint size metric.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCheckpointBufferSize() throws Exception {
+        IgniteEx ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        DataRegionMetricsImpl regionMetrics = ig.cachex(DEFAULT_CACHE_NAME)
+                .context().group().dataRegion().memoryMetrics();
+
+        Assert.assertTrue(regionMetrics.getCheckpointBufferSize() != 0);
+        Assert.assertTrue(regionMetrics.getCheckpointBufferSize() <= MAX_REGION_SIZE);
+    }
+
+    /**
+     * Test for check used checkpoint size metric.
+     *
+     * @throws Exception If failed.
+     */
+    public void testUsedCheckpointBuffer() throws Exception {
+        IgniteEx ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        final DataRegionMetricsImpl regionMetrics = ig.cachex(DEFAULT_CACHE_NAME)
+                .context().group().dataRegion().memoryMetrics();
+
+        Assert.assertEquals(0, regionMetrics.getUsedCheckpointBufferPages());
+        Assert.assertEquals(0, regionMetrics.getUsedCheckpointBufferSize());
+
+        load(ig);
+
+        GridCacheDatabaseSharedManager psMgr = (GridCacheDatabaseSharedManager) ig.context().cache().context().database();
+
+        GridFutureAdapter<T2<Long, Long>> metricsResult = new GridFutureAdapter<>();
+
+        IgniteInternalFuture chpBeginFut = psMgr.wakeupForCheckpoint(null);
+
+        chpBeginFut.listen((f) -> {
+            load(ig);
+
+            metricsResult.onDone(new T2<>(
+                regionMetrics.getUsedCheckpointBufferPages(),
+                regionMetrics.getUsedCheckpointBufferSize()
+            ));
+        });
+
+        metricsResult.get();
+
+        Assert.assertTrue(metricsResult.get().get1() > 0);
+        Assert.assertTrue(metricsResult.get().get2() > 0);
+    }
+
+    /**
+     * @param ig Ignite.
+     */
+    private void load(Ignite ig){
+        IgniteCache<Integer, byte[]> cache = ig.cache(DEFAULT_CACHE_NAME);
+
+        Random rnd = new Random();
+
+        for (int i = 0; i < 1000; i++) {
+            byte[] payload = new byte[128];
+
+            rnd.nextBytes(payload);
+
+            cache.put(i, payload);
+        }
+    }
+
     /** */
     private static DataRegionMetrics getDfltRegionMetrics(Ignite node) {
         for (DataRegionMetrics m : node.dataRegionMetrics())
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
index 22b8986f812f8..463eaa4b8b204 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataRegionMetricsParityTest.cs
@@ -39,7 +39,9 @@ public class DataRegionMetricsParityTest
             "PagesWritten",
             "PagesReplaced",
             "OffHeapSize",
-            "OffheapUsedSize"
+            "OffheapUsedSize",
+            "UsedCheckpointBufferPages",
+            "UsedCheckpointBufferSize"
         };
 
         /// <summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs
index 300ab25c3f433..58c974ec57ea1 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageMetricsParityTest.cs
@@ -38,7 +38,10 @@ public class DataStorageMetricsParityTest
             "PagesReplaced",
             "OffHeapSize",
             "OffheapUsedSize",
-            "TotalAllocatedSize"
+            "TotalAllocatedSize",
+            "UsedCheckpointBufferPages",
+            "UsedCheckpointBufferSize",
+            "CheckpointBufferSize"
         };
 
         /// <summary>

From 04d00be82369f633f26955656441147f53cf2da1 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Thu, 24 May 2018 18:11:47 +0300
Subject: [PATCH 157/543] IGNITE-8560 Update index validation utility to use
 statistical check approach - Fixes #4051.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry-picked from commit #76e1fe754d7a8bd059d7ab64d17cbefa4913a702)
---
 .../internal/commandline/CommandHandler.java  | 92 +++++++++++++++----
 .../commandline/cache/CacheArguments.java     | 34 +++++++
 .../verify/VisorValidateIndexesTaskArg.java   | 40 +++++++-
 .../CommandHandlerParsingTest.java            | 87 ++++++++++++++++++
 .../visor/verify/ValidateIndexesClosure.java  | 66 ++++++++++++-
 .../verify/VisorValidateIndexesTask.java      |  2 +-
 .../util/GridCommandHandlerIndexingTest.java  |  8 +-
 7 files changed, 305 insertions(+), 24 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 47cc233f61ebc..c59e348f768ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -181,6 +181,12 @@ public class CommandHandler {
     /** */
     private static final String BASELINE_SET_VERSION = "version";
 
+    /** Parameter name for validate_indexes command. */
+    static final String VI_CHECK_FIRST = "checkFirst";
+
+    /** Parameter name for validate_indexes command. */
+    static final String VI_CHECK_THROUGH = "checkThrough";
+
     /** */
     static final String WAL_PRINT = "print";
 
@@ -583,10 +589,12 @@ private void printCacheHelp() {
         usage("  Show information about caches, groups or sequences that match a regex:", CACHE, " list regexPattern [groups|seq] [nodeId]");
         usage("  Show hot keys that are point of contention for multiple transactions:", CACHE, " contention minQueueSize [nodeId] [maxPrint]");
         usage("  Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [cache1,...,cacheN]");
-        usage("  Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId]");
+        usage("  Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId] [checkFirst|checkThrough]");
 
-        log("  If [nodeId] is not specified, cont and validate_indexes commands will be broadcasted to all server nodes.");
+        log("  If [nodeId] is not specified, contention and validate_indexes commands will be broadcasted to all server nodes.");
         log("  Another commands where [nodeId] is optional will run on a random server node.");
+        log("  checkFirst numeric parameter for validate_indexes specifies number of first K keys to be validated.");
+        log("  checkThrough numeric parameter for validate_indexes allows to check each Kth key.");
         nl();
     }
 
@@ -624,7 +632,11 @@ private void cacheContention(GridClient client, CacheArguments cacheArgs) throws
      * @param cacheArgs Cache args.
      */
     private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) throws GridClientException {
-        VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(cacheArgs.caches());
+        VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(
+            cacheArgs.caches(),
+            cacheArgs.checkFirst(),
+            cacheArgs.checkThrough()
+        );
 
         UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId();
 
@@ -1407,7 +1419,8 @@ private CacheArguments parseAndValidateCacheArgs() {
                 break;
 
             case IDLE_VERIFY:
-                parseCacheNamesIfPresent(cacheArgs);
+                if (hasNextCacheArg())
+                    parseCacheNames(nextArg(""), cacheArgs);
 
                 break;
 
@@ -1425,10 +1438,53 @@ private CacheArguments parseAndValidateCacheArgs() {
                 break;
 
             case VALIDATE_INDEXES:
-                parseCacheNamesIfPresent(cacheArgs);
+                int argsCnt = 0;
 
-                if (hasNextCacheArg())
-                    cacheArgs.nodeId(UUID.fromString(nextArg("")));
+                while (hasNextCacheArg() && argsCnt++ < 4) {
+                    String arg = nextArg("");
+
+                    if (VI_CHECK_FIRST.equals(arg) || VI_CHECK_THROUGH.equals(arg)) {
+                        if (!hasNextCacheArg())
+                            throw new IllegalArgumentException("Numeric value for '" + arg + "' parameter expected.");
+
+                        int numVal;
+
+                        String numStr = nextArg("");
+
+                        try {
+                            numVal = Integer.parseInt(numStr);
+                        }
+                        catch (IllegalArgumentException e) {
+                            throw new IllegalArgumentException(
+                                "Not numeric value was passed for '"
+                                    + arg
+                                    + "' parameter: "
+                                    + numStr
+                            );
+                        }
+
+                        if (numVal <= 0)
+                            throw new IllegalArgumentException("Value for '" + arg + "' property should be positive.");
+
+                        if (VI_CHECK_FIRST.equals(arg))
+                            cacheArgs.checkFirst(numVal);
+                        else
+                            cacheArgs.checkThrough(numVal);
+
+                        continue;
+                    }
+
+                    try {
+                        cacheArgs.nodeId(UUID.fromString(arg));
+
+                        continue;
+                    }
+                    catch (IllegalArgumentException ignored) {
+                        //No-op.
+                    }
+
+                    parseCacheNames(arg, cacheArgs);
+                }
 
                 break;
 
@@ -1473,22 +1529,18 @@ private boolean hasNextCacheArg() {
     /**
      * @param cacheArgs Cache args.
      */
-    private void parseCacheNamesIfPresent(CacheArguments cacheArgs) {
-        if (hasNextCacheArg()) {
-            String cacheNames = nextArg("");
-
-            String[] cacheNamesArr = cacheNames.split(",");
-            Set<String> cacheNamesSet = new HashSet<>();
+    private void parseCacheNames(String cacheNames, CacheArguments cacheArgs) {
+        String[] cacheNamesArr = cacheNames.split(",");
+        Set<String> cacheNamesSet = new HashSet<>();
 
-            for (String cacheName : cacheNamesArr) {
-                if (F.isEmpty(cacheName))
-                    throw new IllegalArgumentException("Non-empty cache names expected.");
+        for (String cacheName : cacheNamesArr) {
+            if (F.isEmpty(cacheName))
+                throw new IllegalArgumentException("Non-empty cache names expected.");
 
-                cacheNamesSet.add(cacheName.trim());
-            }
-
-            cacheArgs.caches(cacheNamesSet);
+            cacheNamesSet.add(cacheName.trim());
         }
+
+        cacheArgs.caches(cacheNamesSet);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
index 6f315efe6a732..1411b2a2352ce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
@@ -46,6 +46,12 @@ public class CacheArguments {
     /** Max print. */
     private int maxPrint;
 
+    /** validate_indexes 'checkFirst' argument */
+    private int checkFirst = -1;
+
+    /** validate_indexes 'checkThrough' argument */
+    private int checkThrough = -1;
+
     /** Cache view command. */
     private @Nullable VisorViewCacheCmd cacheCmd;
 
@@ -160,4 +166,32 @@ public int maxPrint() {
     public void maxPrint(int maxPrint) {
         this.maxPrint = maxPrint;
     }
+
+    /**
+     *  @return Max number of entries to be checked.
+     */
+    public int checkFirst() {
+        return checkFirst;
+    }
+
+    /**
+     * @param checkFirst Max number of entries to be checked.
+     */
+    public void checkFirst(int checkFirst) {
+        this.checkFirst = checkFirst;
+    }
+
+    /**
+     * @return Number of entries to check through.
+     */
+    public int checkThrough() {
+        return checkThrough;
+    }
+
+    /**
+     * @param checkThrough Number of entries to check through.
+     */
+    public void checkThrough(int checkThrough) {
+        this.checkThrough = checkThrough;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
index cf9aff5ac2033..aa49977c7a6fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
@@ -35,6 +35,12 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject {
     /** Caches. */
     private Set<String> caches;
 
+    /** Check first K elements. */
+    private int checkFirst;
+
+    /** Check through K element (skip K-1, check Kth). */
+    private int checkThrough;
+
     /**
      * Default constructor.
      */
@@ -45,8 +51,10 @@ public VisorValidateIndexesTaskArg() {
     /**
      * @param caches Caches.
      */
-    public VisorValidateIndexesTaskArg(Set<String> caches) {
+    public VisorValidateIndexesTaskArg(Set<String> caches, int checkFirst, int checkThrough) {
         this.caches = caches;
+        this.checkFirst = checkFirst;
+        this.checkThrough = checkThrough;
     }
 
 
@@ -57,14 +65,44 @@ public Set<String> getCaches() {
         return caches;
     }
 
+    /**
+     * @return checkFirst.
+     */
+    public int getCheckFirst() {
+        return checkFirst;
+    }
+
+    /**
+     * @return checkThrough.
+     */
+    public int getCheckThrough() {
+        return checkThrough;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeCollection(out, caches);
+        out.writeInt(checkFirst);
+        out.writeInt(checkThrough);
     }
 
     /** {@inheritDoc} */
     @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
         caches = U.readSet(in);
+
+        if (protoVer > V1) {
+            checkFirst = in.readInt();
+            checkThrough = in.readInt();
+        }
+        else {
+            checkFirst = -1;
+            checkThrough = -1;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 2fc40ca167944..737c0c736f7ae 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -21,15 +21,20 @@
 import java.util.Collections;
 import java.util.UUID;
 import junit.framework.TestCase;
+import org.apache.ignite.internal.commandline.cache.CacheArguments;
+import org.apache.ignite.internal.commandline.cache.CacheCommand;
 import org.apache.ignite.internal.visor.tx.VisorTxProjection;
 import org.apache.ignite.internal.visor.tx.VisorTxSortOrder;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
 
 import static java.util.Arrays.asList;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
+import static org.apache.ignite.internal.commandline.Command.CACHE;
 import static org.apache.ignite.internal.commandline.Command.WAL;
 import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_HOST;
 import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_PORT;
+import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_FIRST;
+import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_THROUGH;
 import static org.apache.ignite.internal.commandline.CommandHandler.WAL_DELETE;
 import static org.apache.ignite.internal.commandline.CommandHandler.WAL_PRINT;
 
@@ -51,6 +56,88 @@ public class CommandHandlerParsingTest extends TestCase {
         super.tearDown();
     }
 
+    /**
+     * validate_indexes command arguments parsing and validation
+     */
+    public void testValidateIndexArguments() {
+        CommandHandler hnd = new CommandHandler();
+
+        //happy case for all parameters
+        try {
+            int expectedCheckFirst = 10;
+            int expectedCheckThrough = 11;
+            UUID nodeId = UUID.randomUUID();
+
+            CacheArguments args = hnd.parseAndValidate(
+                Arrays.asList(
+                    CACHE.text(),
+                    CacheCommand.VALIDATE_INDEXES.text(),
+                    "cache1, cache2",
+                    nodeId.toString(),
+                    VI_CHECK_FIRST,
+                    Integer.toString(expectedCheckFirst),
+                    VI_CHECK_THROUGH,
+                    Integer.toString(expectedCheckThrough)
+                )
+            ).cacheArgs();
+
+            assertEquals("nodeId parameter unexpected value", nodeId, args.nodeId());
+            assertEquals("checkFirst parameter unexpected value", expectedCheckFirst, args.checkFirst());
+            assertEquals("checkThrough parameter unexpected value", expectedCheckThrough, args.checkThrough());
+        }
+        catch (IllegalArgumentException e) {
+            fail("Unexpected exception: " + e);
+        }
+
+        try {
+            int expectedParam = 11;
+            UUID nodeId = UUID.randomUUID();
+
+            CacheArguments args = hnd.parseAndValidate(
+                Arrays.asList(
+                    CACHE.text(),
+                    CacheCommand.VALIDATE_INDEXES.text(),
+                    nodeId.toString(),
+                    VI_CHECK_THROUGH,
+                    Integer.toString(expectedParam)
+                )
+            ).cacheArgs();
+
+            assertNull("caches weren't specified, null value expected", args.caches());
+            assertEquals("nodeId parameter unexpected value", nodeId, args.nodeId());
+            assertEquals("checkFirst parameter unexpected value", -1, args.checkFirst());
+            assertEquals("checkThrough parameter unexpected value", expectedParam, args.checkThrough());
+        }
+        catch (IllegalArgumentException e) {
+            e.printStackTrace();
+        }
+
+        try {
+            hnd.parseAndValidate(
+                Arrays.asList(
+                    CACHE.text(),
+                    CacheCommand.VALIDATE_INDEXES.text(),
+                    VI_CHECK_FIRST,
+                    "0"
+                )
+            );
+
+            fail("Expected exception hasn't been thrown");
+        }
+        catch (IllegalArgumentException e) {
+            e.printStackTrace();
+        }
+
+        try {
+            hnd.parseAndValidate(Arrays.asList(CACHE.text(), CacheCommand.VALIDATE_INDEXES.text(), VI_CHECK_THROUGH));
+
+            fail("Expected exception hasn't been thrown");
+        }
+        catch (IllegalArgumentException e) {
+            e.printStackTrace();
+        }
+    }
+
     /**
      * Test that experimental command (i.e. WAL command) is disabled by default.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index e0eff612fad2a..e01dca2c92d8f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -89,6 +89,12 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
     /** Cache names. */
     private Set<String> cacheNames;
 
+    /** If provided only first K elements will be validated. */
+    private final int checkFirst;
+
+    /** If provided only each Kth element will be validated. */
+    private final int checkThrough;
+
     /** Counter of processed partitions. */
     private final AtomicInteger processedPartitions = new AtomicInteger(0);
 
@@ -109,9 +115,13 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
 
     /**
      * @param cacheNames Cache names.
+     * @param checkFirst If positive only first K elements will be validated.
+     * @param checkThrough If positive only each Kth element will be validated.
      */
-    public ValidateIndexesClosure(Set<String> cacheNames) {
+    public ValidateIndexesClosure(Set<String> cacheNames, int checkFirst, int checkThrough) {
         this.cacheNames = cacheNames;
+        this.checkFirst = checkFirst;
+        this.checkThrough = checkThrough;
     }
 
     /** {@inheritDoc} */
@@ -320,12 +330,39 @@ private Map<PartitionKey, ValidateIndexesPartitionResult> processPartition(
 
             m.setAccessible(true);
 
+            final boolean skipConditions = checkFirst > 0 || checkThrough > 0;
+            final boolean bothSkipConditions = checkFirst > 0 && checkThrough > 0;
+
+            long current = 0;
+            long processedNumber = 0;
+
             while (it.hasNextX()) {
                 if (enoughIssues)
                     break;
 
                 CacheDataRow row = it.nextX();
 
+                if (skipConditions) {
+                    if (bothSkipConditions) {
+                        if (processedNumber > checkFirst)
+                            break;
+                        else if (current++ % checkThrough > 0)
+                            continue;
+                        else
+                            processedNumber++;
+                    }
+                    else {
+                        if (checkFirst > 0) {
+                            if (current++ > checkFirst)
+                                break;
+                        }
+                        else {
+                            if (current++ % checkThrough > 0)
+                                continue;
+                        }
+                    }
+                }
+
                 int cacheId = row.cacheId() == 0 ? grpCtx.groupId() : row.cacheId();
 
                 GridCacheContext cacheCtx = row.cacheId() == 0 ?
@@ -462,6 +499,12 @@ private Map<String, ValidateIndexesPartitionResult> processIndex(GridCacheContex
             enoughIssues = true;
         }
 
+        final boolean skipConditions = checkFirst > 0 || checkThrough > 0;
+        final boolean bothSkipConditions = checkFirst > 0 && checkThrough > 0;
+
+        long current = 0;
+        long processedNumber = 0;
+
         while (!enoughIssues) {
             KeyCacheObject h2key = null;
 
@@ -471,6 +514,27 @@ private Map<String, ValidateIndexesPartitionResult> processIndex(GridCacheContex
 
                 GridH2Row h2Row = (GridH2Row)cursor.get();
 
+                if (skipConditions) {
+                    if (bothSkipConditions) {
+                        if (processedNumber > checkFirst)
+                            break;
+                        else if (current++ % checkThrough > 0)
+                            continue;
+                        else
+                            processedNumber++;
+                    }
+                    else {
+                        if (checkFirst > 0) {
+                            if (current++ > checkFirst)
+                                break;
+                        }
+                        else {
+                            if (current++ % checkThrough > 0)
+                                continue;
+                        }
+                    }
+                }
+
                 h2key = h2Row.key();
 
                 CacheDataRow cacheDataStoreRow = ctx.group().offheap().read(ctx, h2key);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
index 52b48a58c0340..abb7f7ee55ec1 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
@@ -76,7 +76,7 @@ protected VisorValidateIndexesJob(@Nullable VisorValidateIndexesTaskArg arg, boo
         /** {@inheritDoc} */
         @Override protected VisorValidateIndexesJobResult run(@Nullable VisorValidateIndexesTaskArg arg) throws IgniteException {
             try {
-                ValidateIndexesClosure clo = new ValidateIndexesClosure(arg.getCaches());
+                ValidateIndexesClosure clo = new ValidateIndexesClosure(arg.getCaches(), arg.getCheckFirst(), arg.getCheckThrough());
 
                 ignite.context().resource().injectGeneric(clo);
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
index 62d3fc02bdbaa..ca9aa5372f552 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -101,7 +101,13 @@ public void testBrokenCacheDataTreeShouldFailValidation() throws Exception {
 
         injectTestSystemOut();
 
-        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", cacheName));
+        assertEquals(EXIT_CODE_OK,
+            execute(
+                "--cache",
+                "validate_indexes",
+                cacheName,
+                "checkFirst", "10000",
+                "checkThrough", "10"));
 
         assertTrue(testOut.toString().contains("validate_indexes has finished with errors"));
     }

From c457aa60084e1805b7fdbbc88d870ffdc36d14fa Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Fri, 6 Apr 2018 10:35:17 +0300
Subject: [PATCH 158/543] IGNITE-7933 Checkpoing file markers should be written
 atomically - Fixes #3633.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry-picked from commit #4a0695ceae2f99c4841e8382e723daff4580ea3d)
---
 .../apache/ignite/internal/IgnitionEx.java    |   2 +-
 .../GridCacheDatabaseSharedManager.java       | 145 +++++++----
 .../cache/persistence/file/AsyncFileIO.java   |   9 +-
 .../cache/persistence/file/FileIO.java        |  20 +-
 .../persistence/file/FileIODecorator.java     |   9 +-
 .../persistence/file/RandomAccessFileIO.java  |  13 +-
 .../cache/persistence/file/UnzipFileIO.java   |   7 +-
 .../IgnitePdsDiskErrorsRecoveringTest.java    | 231 ++++++++++++++----
 .../db/wal/IgniteWalFlushFailoverTest.java    |   4 +-
 ...lushMultiNodeFailoverAbstractSelfTest.java |   4 +-
 .../pagemem/PagesWriteThrottleSmokeTest.java  |   4 +-
 .../file/AlignedBuffersDirectFileIO.java      |   7 +-
 12 files changed, 353 insertions(+), 102 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index e140609aac497..b3c3ee8b5a9b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -414,7 +414,7 @@ public void run() {
                             " milliseconds. Killing node...");
 
                     // We are not able to kill only one grid so whole JVM will be stopped.
-                    System.exit(Ignition.KILL_EXIT_CODE);
+                    Runtime.getRuntime().halt(Ignition.KILL_EXIT_CODE);
                 }
             }
         }, timeoutMs, TimeUnit.MILLISECONDS);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 6ce743d8d9632..4096a8d7ba13f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -27,6 +27,7 @@
 import java.nio.channels.FileChannel;
 import java.nio.channels.FileLock;
 import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -118,8 +119,10 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.PersistentStorageIOException;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker;
@@ -215,11 +218,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Checkpoint file name pattern. */
     private static final Pattern CP_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-(.*)-(START|END)\\.bin");
 
+    /** Checkpoint file temporary suffix. This is needed to safe writing checkpoint markers through temporary file and renaming. */
+    public static final String FILE_TMP_SUFFIX = ".tmp";
+
     /** Node started file patter. */
     private static final Pattern NODE_STARTED_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-node-started\\.bin");
 
     /** Node started file suffix. */
-    private static final String NODE_STARTED_FILE_NAME_SUFFIX = "-node-started.bin";
+    public static final String NODE_STARTED_FILE_NAME_SUFFIX = "-node-started.bin";
 
     /** */
     private static final FileFilter CP_FILE_FILTER = new FileFilter() {
@@ -394,6 +400,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Initially local wal disabled groups. */
     private Collection<Integer> initiallyLocalWalDisabledGrps = new HashSet<>();
 
+    /** File I/O factory for writing checkpoint markers. */
+    private final FileIOFactory ioFactory;
+
     /**
      * @param ctx Kernal context.
      */
@@ -418,6 +427,8 @@ public GridCacheDatabaseSharedManager(GridKernalContext ctx) {
 
         maxCpHistMemSize = Math.min(persistenceCfg.getWalHistorySize(),
             IgniteSystemProperties.getInteger(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, 100));
+
+        ioFactory = persistenceCfg.getFileIOFactory();
     }
 
     /** */
@@ -514,6 +525,8 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
             if (!U.mkdirs(cpDir))
                 throw new IgniteCheckedException("Could not create directory for checkpoint metadata: " + cpDir);
 
+            cleanupCheckpointDirectory();
+
             final FileLockHolder preLocked = kernalCtx.pdsFolderResolver()
                 .resolveFolders()
                 .getLockedFileLockHolder();
@@ -527,6 +540,26 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
         }
     }
 
+    /**
+     * Cleanup checkpoint directory from all temporary files {@link #FILE_TMP_SUFFIX}.
+     */
+    private void cleanupCheckpointDirectory() throws IgniteCheckedException {
+        try {
+            try (DirectoryStream<Path> files = Files.newDirectoryStream(cpDir.toPath(), new DirectoryStream.Filter<Path>() {
+                @Override
+                public boolean accept(Path path) throws IOException {
+                    return path.endsWith(FILE_TMP_SUFFIX);
+                }
+            })) {
+                for (Path path : files)
+                    Files.delete(path);
+            }
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Failed to cleanup checkpoint directory: " + cpDir, e);
+        }
+    }
+
     /**
      *
      */
@@ -777,7 +810,7 @@ private void unRegistrateMetricsMBean() {
 
             notifyMetastorageReadyForReadWrite();
         }
-        catch (StorageException e) {
+        catch (StorageException | PersistentStorageIOException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
@@ -788,41 +821,52 @@ private void unRegistrateMetricsMBean() {
     }
 
     /**
+     * Creates file with current timestamp and specific "node-started.bin" suffix
+     * and writes into memory recovery pointer.
+     *
      * @param ptr Memory recovery wal pointer.
      */
     private void nodeStart(WALPointer ptr) throws IgniteCheckedException {
         FileWALPointer p = (FileWALPointer)ptr;
 
-        String fileName = U.currentTimeMillis() + "-node-started.bin";
+        String fileName = U.currentTimeMillis() + NODE_STARTED_FILE_NAME_SUFFIX;
+        String tmpFileName = fileName + FILE_TMP_SUFFIX;
 
         ByteBuffer buf = ByteBuffer.allocate(20);
         buf.order(ByteOrder.nativeOrder());
 
-        try (FileChannel ch = FileChannel.open(
-            Paths.get(cpDir.getAbsolutePath(), fileName),
-            StandardOpenOption.CREATE_NEW, StandardOpenOption.APPEND)
-        ) {
-            buf.putLong(p.index());
+        try {
+            try (FileIO io = ioFactory.create(Paths.get(cpDir.getAbsolutePath(), tmpFileName).toFile(),
+                    StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+                buf.putLong(p.index());
 
-            buf.putInt(p.fileOffset());
+                buf.putInt(p.fileOffset());
 
-            buf.putInt(p.length());
+                buf.putInt(p.length());
 
-            buf.flip();
+                buf.flip();
 
-            ch.write(buf);
+                io.write(buf);
 
-            buf.clear();
+                buf.clear();
+
+                io.force(true);
+            }
 
-            ch.force(true);
+            Files.move(Paths.get(cpDir.getAbsolutePath(), tmpFileName), Paths.get(cpDir.getAbsolutePath(), fileName));
         }
         catch (IOException e) {
-            throw new IgniteCheckedException(e);
+            throw new PersistentStorageIOException("Failed to write node start marker: " + ptr, e);
         }
     }
 
     /**
+     * Collects memory recovery pointers from node started files. See {@link #nodeStart(WALPointer)}.
+     * Each pointer associated with timestamp extracted from file.
+     * Tuples are sorted by timestamp.
      *
+     * @return Sorted list of tuples (node started timestamp, memory recovery pointer).
+     * @throws IgniteCheckedException
      */
     public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedException {
         List<T2<Long, WALPointer>> res = new ArrayList<>();
@@ -834,15 +878,10 @@ public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedExce
                 String n1 = o1.getName();
                 String n2 = o2.getName();
 
-                Long ts1 = Long.valueOf(n1.substring(0, n1.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
-                Long ts2 = Long.valueOf(n2.substring(0, n2.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
+                long ts1 = Long.valueOf(n1.substring(0, n1.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
+                long ts2 = Long.valueOf(n2.substring(0, n2.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
 
-                if (ts1 == ts2)
-                    return 0;
-                else if (ts1 < ts2)
-                    return -1;
-                else
-                    return 1;
+                return Long.compare(ts1, ts2);
             }
         });
 
@@ -854,8 +893,8 @@ else if (ts1 < ts2)
 
             Long ts = Long.valueOf(name.substring(0, name.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
 
-            try (FileChannel ch = FileChannel.open(f.toPath(), READ)) {
-                ch.read(buf);
+            try (FileIO io = ioFactory.create(f, READ)) {
+                io.read(buf);
 
                 buf.flip();
 
@@ -1953,8 +1992,8 @@ else if (type == CheckpointEntryType.END && ts > lastEndTs) {
     private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteCheckedException {
         buf.position(0);
 
-        try (FileChannel ch = FileChannel.open(cpMarkerFile.toPath(), READ)) {
-            ch.read(buf);
+        try (FileIO io = ioFactory.create(cpMarkerFile, READ)) {
+            io.read(buf);
 
             buf.flip();
 
@@ -2652,6 +2691,8 @@ private void finalizeCheckpointOnRecovery(long cpTs, UUID cpId, WALPointer walPt
     }
 
     /**
+     * Writes into specified file checkpoint entry containing WAL pointer to checkpoint record.
+     *
      * @param cpId Checkpoint ID.
      * @param ptr Wal pointer of current checkpoint.
      */
@@ -2668,31 +2709,40 @@ private CheckpointEntry writeCheckpointEntry(
         FileWALPointer filePtr = (FileWALPointer)ptr;
 
         String fileName = checkpointFileName(cpTs, cpId, type);
+        String tmpFileName = fileName + FILE_TMP_SUFFIX;
 
-        try (FileChannel ch = FileChannel.open(Paths.get(cpDir.getAbsolutePath(), fileName),
-            StandardOpenOption.CREATE_NEW, StandardOpenOption.APPEND)) {
+        try {
+            try (FileIO io = ioFactory.create(Paths.get(cpDir.getAbsolutePath(), skipSync ? fileName : tmpFileName).toFile(),
+                    StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
 
-            tmpWriteBuf.rewind();
+                tmpWriteBuf.rewind();
 
-            tmpWriteBuf.putLong(filePtr.index());
+                tmpWriteBuf.putLong(filePtr.index());
 
-            tmpWriteBuf.putInt(filePtr.fileOffset());
+                tmpWriteBuf.putInt(filePtr.fileOffset());
 
-            tmpWriteBuf.putInt(filePtr.length());
+                tmpWriteBuf.putInt(filePtr.length());
 
-            tmpWriteBuf.flip();
+                tmpWriteBuf.flip();
 
-            ch.write(tmpWriteBuf);
+                io.write(tmpWriteBuf);
 
-            tmpWriteBuf.clear();
+                tmpWriteBuf.clear();
+
+                if (!skipSync)
+                    io.force(true);
+            }
 
             if (!skipSync)
-                ch.force(true);
+                Files.move(Paths.get(cpDir.getAbsolutePath(), tmpFileName), Paths.get(cpDir.getAbsolutePath(), fileName));
 
             return createCheckPointEntry(cpTs, ptr, cpId, rec, type);
         }
         catch (IOException e) {
-            throw new IgniteCheckedException(e);
+            throw new PersistentStorageIOException("Failed to write checkpoint entry [ptr=" + filePtr
+                    + ", cpTs=" + cpTs
+                    + ", cpId=" + cpId
+                    + ", type=" + type + "]", e);
         }
     }
 
@@ -2759,8 +2809,6 @@ private CheckpointEntry createCheckPointEntry(
         if (type != CheckpointEntryType.START)
             return null;
 
-        CheckpointEntry entry;
-
         Map<Integer, CacheState> cacheGrpStates = null;
 
         // Create lazy checkpoint entry.
@@ -3085,7 +3133,20 @@ private void doCheckpoint() {
             try {
                 CheckpointMetricsTracker tracker = new CheckpointMetricsTracker();
 
-                Checkpoint chp = markCheckpointBegin(tracker);
+                Checkpoint chp;
+
+                try {
+                    chp = markCheckpointBegin(tracker);
+                }
+                catch (IgniteCheckedException e) {
+                    if (curCpProgress != null)
+                        curCpProgress.cpFinishFut.onDone(e);
+
+                    // In case of checkpoint initialization error node should be invalidated and stopped.
+                    cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+                    return;
+                }
 
                 currCheckpointPagesCnt = chp.pagesSize;
 
@@ -3145,7 +3206,7 @@ private void doCheckpoint() {
                         } catch (IgniteCheckedException e) {
                             chp.progress.cpFinishFut.onDone(e);
 
-                            // In case of writing error node should be invalidated and stopped.
+                            // In case of checkpoint writing error node should be invalidated and stopped.
                             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
                             return;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
index b1db79d706a2e..799a78cb99e60 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
@@ -169,13 +169,18 @@ public AsyncFileIO(File file, ThreadLocal<ChannelOpFuture> holder, OpenOption...
     }
 
     /** {@inheritDoc} */
-    @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
+    @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
         throw new UnsupportedOperationException("AsynchronousFileChannel doesn't support mmap.");
     }
 
     /** {@inheritDoc} */
     @Override public void force() throws IOException {
-        ch.force(false);
+        force(false);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void force(boolean withMetadata) throws IOException {
+        ch.force(withMetadata);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
index 73e44b0d9af1f..822bd66413dc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
@@ -124,7 +124,16 @@ public interface FileIO extends AutoCloseable {
      */
     public void write(byte[] buf, int off, int len) throws IOException;
 
-    public MappedByteBuffer map(int maxWalSegmentSize) throws IOException;
+    /**
+     * Allocates memory mapped buffer for this file with given size.
+     *
+     * @param sizeBytes Size of buffer.
+     *
+     * @return Instance of mapped byte buffer.
+     *
+     * @throws IOException If some I/O error occurs.
+     */
+    public MappedByteBuffer map(int sizeBytes) throws IOException;
 
     /**
      * Forces any updates of this file to be written to the storage
@@ -134,6 +143,15 @@ public interface FileIO extends AutoCloseable {
      */
     public void force() throws IOException;
 
+    /**
+     * Forces any updates of this file to be written to the storage
+     * device that contains it.
+     *
+     * @param withMetadata If {@code true} force also file metadata.
+     * @throws IOException If some I/O error occurs.
+     */
+    public void force(boolean withMetadata) throws IOException;
+
     /**
      * Returns current file size in bytes.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java
index dd563f2d68197..683845bc268ba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java
@@ -77,8 +77,8 @@ public FileIODecorator(FileIO delegate) {
     }
 
     /** {@inheritDoc} */
-    @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
-        return delegate.map(maxWalSegmentSize);
+    @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
+        return delegate.map(sizeBytes);
     }
 
     /** {@inheritDoc} */
@@ -86,6 +86,11 @@ public FileIODecorator(FileIO delegate) {
         delegate.force();
     }
 
+    /** {@inheritDoc} */
+    @Override public void force(boolean withMetadata) throws IOException {
+        delegate.force(withMetadata);
+    }
+
     /** {@inheritDoc} */
     @Override public long size() throws IOException {
         return delegate.size();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
index 23d6ebfeead76..8f7454dcae617 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
@@ -84,8 +84,8 @@ public RandomAccessFileIO(File file, OpenOption... modes) throws IOException {
     }
 
     /** {@inheritDoc} */
-    @Override public void force() throws IOException {
-        ch.force(false);
+    @Override public void force(boolean withMetadata) throws IOException {
+        ch.force(withMetadata);
     }
 
     /** {@inheritDoc} */
@@ -104,7 +104,12 @@ public RandomAccessFileIO(File file, OpenOption... modes) throws IOException {
     }
 
     /** {@inheritDoc} */
-    @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
-        return ch.map(FileChannel.MapMode.READ_WRITE, 0, maxWalSegmentSize);
+    @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
+        return ch.map(FileChannel.MapMode.READ_WRITE, 0, sizeBytes);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void force() throws IOException {
+        force(false);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
index 83ff91ba99da5..469cf3eb7ebbc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
@@ -116,6 +116,11 @@ public UnzipFileIO(File zip) throws IOException {
 
     /** {@inheritDoc} */
     @Override public void force() throws IOException {
+        force(false);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void force(boolean withMetadata) throws IOException {
         throw new UnsupportedOperationException();
     }
 
@@ -130,7 +135,7 @@ public UnzipFileIO(File zip) throws IOException {
     }
 
     /** {@inheritDoc} */
-    @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
+    @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
         throw new UnsupportedOperationException();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
index 3e85c7760478c..c902879cbaae2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
@@ -25,6 +25,7 @@
 import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -39,15 +40,20 @@
 import org.apache.ignite.internal.GridKernalState;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.junit.Assert;
 
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
 
 /**
@@ -60,20 +66,14 @@ public class IgnitePdsDiskErrorsRecoveringTest extends GridCommonAbstractTest {
     /** */
     private static final int WAL_SEGMENT_SIZE = 1024 * PAGE_SIZE;
 
-    /** */
-    private static final long DFLT_DISK_SPACE_BYTES = Long.MAX_VALUE;
-
     /** */
     private static final long STOP_TIMEOUT_MS = 30 * 1000;
 
     /** */
     private static final String CACHE_NAME = "cache";
 
-    /** */
-    private boolean failPageStoreDiskOperations = false;
-
-    /** */
-    private long diskSpaceBytes = DFLT_DISK_SPACE_BYTES;
+    /** Specified i/o factory for particular test. */
+    private FileIOFactory ioFactory;
 
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
@@ -88,8 +88,7 @@ public class IgnitePdsDiskErrorsRecoveringTest extends GridCommonAbstractTest {
 
         cleanPersistenceDir();
 
-        failPageStoreDiskOperations = false;
-        diskSpaceBytes = DFLT_DISK_SPACE_BYTES;
+        ioFactory = null;
         System.clearProperty(IGNITE_WAL_MMAP);
     }
 
@@ -103,10 +102,11 @@ public class IgnitePdsDiskErrorsRecoveringTest extends GridCommonAbstractTest {
                 .setWalMode(WALMode.LOG_ONLY)
                 .setWalCompactionEnabled(false)
                 .setWalSegmentSize(WAL_SEGMENT_SIZE)
+                .setCheckpointFrequency(240 * 60 * 1000)
                 .setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
 
-        if (failPageStoreDiskOperations)
-            dsCfg.setFileIOFactory(new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), diskSpaceBytes));
+        if (ioFactory != null)
+            dsCfg.setFileIOFactory(ioFactory);
 
         cfg.setDataStorageConfiguration(dsCfg);
 
@@ -122,19 +122,17 @@ public class IgnitePdsDiskErrorsRecoveringTest extends GridCommonAbstractTest {
     }
 
     /**
-     *
+     * Test node stopping & recovering on cache initialization fail.
      */
-    public void testRecoveringOnCacheInitError() throws Exception {
-        failPageStoreDiskOperations = true;
-
-        // Two pages is enough to initialize MetaStorage.
-        diskSpaceBytes = 2 * PAGE_SIZE;
+    public void testRecoveringOnCacheInitFail() throws Exception {
+        // Fail to initialize page store. 2 extra pages is needed for MetaStorage.
+        ioFactory = new FilteringFileIOFactory(".bin", new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 2 * PAGE_SIZE));
 
         final IgniteEx grid = startGrid(0);
 
         boolean failed = false;
         try {
-            grid.active(true);
+            grid.cluster().active(true);
         } catch (Exception expected) {
             log.warning("Expected cache error", expected);
 
@@ -147,21 +145,128 @@ public void testRecoveringOnCacheInitError() throws Exception {
         awaitStop(grid);
 
         // Grid should be successfully recovered after stopping.
-        failPageStoreDiskOperations = false;
+        ioFactory = null;
 
         IgniteEx recoveredGrid = startGrid(0);
         recoveredGrid.active(true);
     }
 
     /**
+     * Test node stopping & recovering on start marker writing fail during activation.
      *
+     * @throws Exception If test failed.
      */
-    public void testRecoveringOnCheckpointWritingError() throws Exception {
-        failPageStoreDiskOperations = true;
-        diskSpaceBytes = 1024 * PAGE_SIZE;
+    public void testRecoveringOnNodeStartMarkerWriteFail() throws Exception {
+        // Fail to write node start marker tmp file at the second checkpoint. Pass only initial checkpoint.
+        ioFactory = new FilteringFileIOFactory("started.bin" + GridCacheDatabaseSharedManager.FILE_TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20));
+
+        IgniteEx grid = startGrid(0);
+        grid.cluster().active(true);
+
+        for (int i = 0; i < 1000; i++) {
+            byte payload = (byte) i;
+            byte[] data = new byte[2048];
+            Arrays.fill(data, payload);
+
+            grid.cache(CACHE_NAME).put(i, data);
+        }
+
+        stopAllGrids();
+
+        boolean activationFailed = false;
+        try {
+            grid = startGrid(0);
+            grid.cluster().active(true);
+        }
+        catch (IgniteException e) {
+            log.warning("Activation test exception", e);
+
+            activationFailed = true;
+        }
+
+        Assert.assertTrue("Activation must be failed", activationFailed);
+
+        // Grid should be automatically stopped after checkpoint fail.
+        awaitStop(grid);
+
+        // Grid should be successfully recovered after stopping.
+        ioFactory = null;
+
+        IgniteEx recoveredGrid = startGrid(0);
+        recoveredGrid.cluster().active(true);
+
+        for (int i = 0; i < 1000; i++) {
+            byte payload = (byte) i;
+            byte[] data = new byte[2048];
+            Arrays.fill(data, payload);
+
+            byte[] actualData = (byte[]) recoveredGrid.cache(CACHE_NAME).get(i);
+            Assert.assertArrayEquals(data, actualData);
+        }
+    }
+
+
+    /**
+     * Test node stopping & recovering on checkpoint begin fail.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testRecoveringOnCheckpointBeginFail() throws Exception {
+        // Fail to write checkpoint start marker tmp file at the second checkpoint. Pass only initial checkpoint.
+        ioFactory = new FilteringFileIOFactory("START.bin" + GridCacheDatabaseSharedManager.FILE_TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20));
 
         final IgniteEx grid = startGrid(0);
-        grid.active(true);
+        grid.cluster().active(true);
+
+        for (int i = 0; i < 1000; i++) {
+            byte payload = (byte) i;
+            byte[] data = new byte[2048];
+            Arrays.fill(data, payload);
+
+            grid.cache(CACHE_NAME).put(i, data);
+        }
+
+        String errMsg = "Failed to write checkpoint entry";
+
+        boolean checkpointFailed = false;
+        try {
+            forceCheckpoint();
+        }
+        catch (IgniteCheckedException e) {
+            if (e.getMessage().contains(errMsg))
+                checkpointFailed = true;
+        }
+
+        Assert.assertTrue("Checkpoint must be failed by IgniteCheckedException: " + errMsg, checkpointFailed);
+
+        // Grid should be automatically stopped after checkpoint fail.
+        awaitStop(grid);
+
+        // Grid should be successfully recovered after stopping.
+        ioFactory = null;
+
+        IgniteEx recoveredGrid = startGrid(0);
+        recoveredGrid.cluster().active(true);
+
+        for (int i = 0; i < 1000; i++) {
+            byte payload = (byte) i;
+            byte[] data = new byte[2048];
+            Arrays.fill(data, payload);
+
+            byte[] actualData = (byte[]) recoveredGrid.cache(CACHE_NAME).get(i);
+            Assert.assertArrayEquals(data, actualData);
+        }
+    }
+
+    /**
+     * Test node stopping & recovering on checkpoint pages write fail.
+     */
+    public void testRecoveringOnCheckpointWriteFail() throws Exception {
+        // Fail write partition and index files at the second checkpoint. Pass only initial checkpoint.
+        ioFactory = new FilteringFileIOFactory(".bin", new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 128 * PAGE_SIZE));
+
+        final IgniteEx grid = startGrid(0);
+        grid.cluster().active(true);
 
         for (int i = 0; i < 1000; i++) {
             byte payload = (byte) i;
@@ -187,10 +292,10 @@ public void testRecoveringOnCheckpointWritingError() throws Exception {
         awaitStop(grid);
 
         // Grid should be successfully recovered after stopping.
-        failPageStoreDiskOperations = false;
+        ioFactory = null;
 
         IgniteEx recoveredGrid = startGrid(0);
-        recoveredGrid.active(true);
+        recoveredGrid.cluster().active(true);
 
         for (int i = 0; i < 1000; i++) {
             byte payload = (byte) i;
@@ -203,33 +308,35 @@ public void testRecoveringOnCheckpointWritingError() throws Exception {
     }
 
     /**
-     *
+     * Test node stopping & recovering on WAL writing fail with enabled MMAP (Batch allocation for WAL segments).
      */
-    public void testRecoveringOnWALErrorWithMmap() throws Exception {
-        diskSpaceBytes = WAL_SEGMENT_SIZE;
+    public void testRecoveringOnWALWritingFail1() throws Exception {
+        // Allow to allocate only 1 wal segment, fail on write to second.
+        ioFactory = new FilteringFileIOFactory(".wal", new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), WAL_SEGMENT_SIZE));
         System.setProperty(IGNITE_WAL_MMAP, "true");
-        emulateRecoveringOnWALWritingError();
+        doTestRecoveringOnWALWritingFail();
     }
 
     /**
-     *
+     * Test node stopping & recovering on WAL writing fail with disabled MMAP.
      */
-    public void testRecoveringOnWALErrorWithoutMmap() throws Exception {
-        diskSpaceBytes = 2 * WAL_SEGMENT_SIZE;
+    public void testRecoveringOnWALWritingFail2() throws Exception {
+        // Fail somewhere on the second wal segment.
+        ioFactory = new FilteringFileIOFactory(".wal", new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), (long) (1.5 * WAL_SEGMENT_SIZE)));
         System.setProperty(IGNITE_WAL_MMAP, "false");
-        emulateRecoveringOnWALWritingError();
+        doTestRecoveringOnWALWritingFail();
     }
 
     /**
-     *
+     * Test node stopping & recovery on WAL writing fail.
      */
-    private void emulateRecoveringOnWALWritingError() throws Exception {
+    private void doTestRecoveringOnWALWritingFail() throws Exception {
         final IgniteEx grid = startGrid(0);
 
         FileWriteAheadLogManager wal = (FileWriteAheadLogManager)grid.context().cache().context().wal();
-        wal.setFileIOFactory(new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), diskSpaceBytes));
+        wal.setFileIOFactory(ioFactory);
 
-        grid.active(true);
+        grid.cluster().active(true);
 
         int failedPosition = -1;
 
@@ -254,9 +361,11 @@ private void emulateRecoveringOnWALWritingError() throws Exception {
         // Grid should be automatically stopped after WAL fail.
         awaitStop(grid);
 
+        ioFactory = null;
+
         // Grid should be successfully recovered after stopping.
         IgniteEx recoveredGrid = startGrid(0);
-        recoveredGrid.active(true);
+        recoveredGrid.cluster().active(true);
 
         for (int i = 0; i < failedPosition; i++) {
             byte payload = (byte) i;
@@ -328,11 +437,49 @@ public LimitedSizeFileIO(FileIO delegate, AtomicLong availableSpaceBytes) {
         }
 
         /** {@inheritDoc} */
-        @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
-            availableSpaceBytes.addAndGet(-maxWalSegmentSize);
+        @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
+            availableSpaceBytes.addAndGet(-sizeBytes);
             if (availableSpaceBytes.get() < 0)
                 throw new IOException("Not enough space!");
-            return super.map(maxWalSegmentSize);
+            return super.map(sizeBytes);
+        }
+    }
+
+    /**
+     * Factory to provide custom File I/O interfaces only for files with specified suffix.
+     * For other files {@link RandomAccessFileIO} will be used.
+     */
+    private static class FilteringFileIOFactory implements FileIOFactory {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** Delegate. */
+        private final FileIOFactory delegate;
+
+        /** File suffix pattern. */
+        private final String pattern;
+
+        /**
+         * Constructor.
+         *
+         * @param pattern File suffix pattern.
+         * @param delegate I/O Factory delegate.
+         */
+        FilteringFileIOFactory(String pattern, FileIOFactory delegate) {
+            this.delegate = delegate;
+            this.pattern = pattern;
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            return create(file, CREATE, WRITE, READ);
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+            if (file.getName().endsWith(pattern))
+                return delegate.create(file, modes);
+            return new RandomAccessFileIO(file, modes);
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
index 946b4e80c52d5..042a447d032eb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
@@ -206,8 +206,8 @@ private static class FailingFileIOFactory implements FileIOFactory {
                 }
 
                 /** {@inheritDoc} */
-                @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
-                    return delegate.map(maxWalSegmentSize);
+                @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
+                    return delegate.map(sizeBytes);
                 }
             };
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index 1259c3c489ccc..fe1632817e44c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -250,8 +250,8 @@ private static class FailingFileIOFactory implements FileIOFactory {
                 }
 
                 /** {@inheritDoc} */
-                @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
-                    return delegate.map(maxWalSegmentSize);
+                @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
+                    return delegate.map(sizeBytes);
                 }
             };
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
index 9f1342fd9d3b7..249718b02ec06 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
@@ -319,8 +319,8 @@ private static class SlowCheckpointFileIOFactory implements FileIOFactory {
                 }
 
                 /** {@inheritDoc} */
-                @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
-                    return delegate.map(maxWalSegmentSize);
+                @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
+                    return delegate.map(sizeBytes);
                 }
             };
         }
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
index 3cb4886e4613f..681426cd4c18e 100644
--- a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
@@ -460,12 +460,17 @@ private static String getLastError() {
     }
 
     /** {@inheritDoc} */
-    @Override public MappedByteBuffer map(int maxWalSegmentSize) throws IOException {
+    @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
         throw new UnsupportedOperationException("AsynchronousFileChannel doesn't support mmap.");
     }
 
     /** {@inheritDoc} */
     @Override public void force() throws IOException {
+        force(false);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void force(boolean withMetadata) throws IOException {
         if (IgniteNativeIoLib.fsync(fdCheckOpened()) < 0)
             throw new IOException(String.format("Error fsync()'ing %s, got %s", file, getLastError()));
     }

From 6b28e8d76bc0119917af0a738d9f07ff794cde20 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Fri, 25 May 2018 12:01:10 +0300
Subject: [PATCH 159/543] IGNITE-8540 Fast cleanup of PDS when joining node is
 not in baseline - Fixes #4037.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry-picked from commit #3f14d2b35d7818196598e8541590186e06b8edbb)
---
 .../pagemem/store/IgnitePageStoreManager.java |   8 ++
 .../wal/IgniteWriteAheadLogManager.java       |   5 +
 .../processors/cache/GridCacheProcessor.java  |  16 ++-
 .../GridDhtPartitionsExchangeFuture.java      |  18 +++
 .../GridCacheDatabaseSharedManager.java       |  19 +++-
 .../IgniteCacheDatabaseSharedManager.java     |   7 ++
 .../file/FilePageStoreManager.java            |  23 ++++
 .../wal/FileWriteAheadLogManager.java         |  25 +++++
 .../FsyncModeFileWriteAheadLogManager.java    |  25 +++++
 ...teAbsentEvictionNodeOutOfBaselineTest.java | 106 ++++++++++++++++++
 .../pagemem/NoOpPageStoreManager.java         |   6 +
 .../persistence/pagemem/NoOpWALManager.java   |   5 +
 .../testsuites/IgnitePdsTestSuite2.java       |   2 +
 13 files changed, 262 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index 0fc9f94b41978..7dba8aee94c5e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -20,6 +20,7 @@
 import java.nio.ByteBuffer;
 import java.util.Map;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
@@ -220,4 +221,11 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cac
      * @return number of pages.
      */
     public long pagesAllocated(int grpId);
+
+    /**
+     * Cleanup persistent space for cache.
+     *
+     * @param cacheConfiguration Cache configuration of cache which should be cleanup.
+     */
+    public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws IgniteCheckedException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index b5c22c92341c4..fd5d53b17e1d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -144,4 +144,9 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
      * @param grpId Group id.
      */
     public boolean disabled(int grpId);
+
+    /**
+     * Cleanup all directories relating to WAL (e.g. work WAL dir, archive WAL dir).
+     */
+    public void cleanupWalDirectories() throws IgniteCheckedException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 39c7e715c06f6..1a7c9f581cba0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -178,6 +178,7 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CONSISTENCY_CHECK_SKIPPED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_CONFIG;
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled;
+import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isPersistentCache;
 
 /**
  * Cache processor.
@@ -3279,10 +3280,23 @@ public void saveCacheConfiguration(DynamicCacheDescriptor desc) throws IgniteChe
         assert desc != null;
 
         if (sharedCtx.pageStore() != null && !sharedCtx.kernalContext().clientNode() &&
-            CU.isPersistentCache(desc.cacheConfiguration(), sharedCtx.gridConfig().getDataStorageConfiguration()))
+            isPersistentCache(desc.cacheConfiguration(), sharedCtx.gridConfig().getDataStorageConfiguration()))
             sharedCtx.pageStore().storeCacheData(desc.toStoredData(), true);
     }
 
+    /**
+     * Remove all persistent files for all registered caches.
+     */
+    public void cleanupCachesDirectories() throws IgniteCheckedException {
+        if (sharedCtx.pageStore() == null || sharedCtx.kernalContext().clientNode())
+            return;
+
+        for (DynamicCacheDescriptor desc : cacheDescriptors().values()) {
+            if (isPersistentCache(desc.cacheConfiguration(), sharedCtx.gridConfig().getDataStorageConfiguration()))
+                sharedCtx.pageStore().cleanupPersistentSpace(desc.cacheConfiguration());
+        }
+    }
+
     /**
      * @param reqs Requests.
      * @return Collection of futures.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 54ff58d20e282..32d9cb30b8b18 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -769,6 +769,15 @@ else if (msg instanceof WalStateAbstractMessage)
      * @throws IgniteCheckedException If failed.
      */
     private void initCachesOnLocalJoin() throws IgniteCheckedException {
+        if (isLocalNodeNotInBaseline()) {
+            cctx.cache().cleanupCachesDirectories();
+
+            cctx.database().cleanupCheckpointDirectory();
+
+            if (cctx.wal() != null)
+                cctx.wal().cleanupWalDirectories();
+        }
+
         cctx.activate();
 
         LocalJoinCachesContext locJoinCtx = exchActions == null ? null : exchActions.localJoinContext();
@@ -794,6 +803,15 @@ private void initCachesOnLocalJoin() throws IgniteCheckedException {
         cctx.cache().startCachesOnLocalJoin(locJoinCtx, initialVersion());
     }
 
+    /**
+     * @return {@code true} if local node is not in baseline and {@code false} otherwise.
+     */
+    private boolean isLocalNodeNotInBaseline() {
+        BaselineTopology topology = cctx.discovery().discoCache().state().baselineTopology();
+
+        return topology!= null && !topology.consistentIds().contains(cctx.localNode().consistentId());
+    }
+
     /**
      * @throws IgniteCheckedException If failed.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 4096a8d7ba13f..6064cfe2775cd 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -525,7 +525,7 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
             if (!U.mkdirs(cpDir))
                 throw new IgniteCheckedException("Could not create directory for checkpoint metadata: " + cpDir);
 
-            cleanupCheckpointDirectory();
+            cleanupTempCheckpointDirectory();
 
             final FileLockHolder preLocked = kernalCtx.pdsFolderResolver()
                 .resolveFolders()
@@ -543,7 +543,7 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
     /**
      * Cleanup checkpoint directory from all temporary files {@link #FILE_TMP_SUFFIX}.
      */
-    private void cleanupCheckpointDirectory() throws IgniteCheckedException {
+    private void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
         try {
             try (DirectoryStream<Path> files = Files.newDirectoryStream(cpDir.toPath(), new DirectoryStream.Filter<Path>() {
                 @Override
@@ -555,6 +555,21 @@ public boolean accept(Path path) throws IOException {
                     Files.delete(path);
             }
         }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Failed to cleanup checkpoint directory from temporary files: " + cpDir, e);
+        }
+    }
+
+    /**
+     * Cleanup checkpoint directory.
+     */
+    public void cleanupCheckpointDirectory() throws IgniteCheckedException {
+        try {
+            try (DirectoryStream<Path> files = Files.newDirectoryStream(cpDir.toPath())) {
+                for (Path path : files)
+                    Files.delete(path);
+            }
+        }
         catch (IOException e) {
             throw new IgniteCheckedException("Failed to cleanup checkpoint directory: " + cpDir, e);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index a251234683342..5e8f40706e984 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -732,6 +732,13 @@ public void checkpointReadUnlock() {
         // No-op.
     }
 
+    /**
+     * No-op for non-persistent storage.
+     */
+    public void cleanupCheckpointDirectory() throws IgniteCheckedException {
+        // No-op.
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 661694d72101b..d065ff1e1e54b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -153,6 +153,29 @@ public FilePageStoreManager(GridKernalContext ctx) {
         U.ensureDirectory(storeWorkDir, "page store work directory", log);
     }
 
+    /** {@inheritDoc} */
+    public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws IgniteCheckedException {
+        try {
+            File cacheWorkDir = cacheWorkDir(cacheConfiguration);
+
+            if(!cacheWorkDir.exists())
+                return;
+
+            try (DirectoryStream<Path> files = Files.newDirectoryStream(cacheWorkDir.toPath(),
+                new DirectoryStream.Filter<Path>() {
+                    @Override public boolean accept(Path entry) throws IOException {
+                        return entry.toFile().getName().endsWith(FILE_SUFFIX);
+                    }
+                })) {
+                for (Path path : files)
+                    Files.delete(path);
+            }
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Failed to cleanup persistent directory: ", e);
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void stop0(boolean cancel) {
         if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 40ebcf04289fd..5efd5ee6c44b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -33,8 +33,10 @@
 import java.nio.ByteOrder;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.ClosedByInterruptException;
+import java.nio.file.DirectoryStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.sql.Time;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -1329,6 +1331,29 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
             checkFiles(0, false, null, null);
     }
 
+    /** {@inheritDoc} */
+    public void cleanupWalDirectories() throws IgniteCheckedException {
+        try {
+            try (DirectoryStream<Path> files = Files.newDirectoryStream(walWorkDir.toPath())) {
+                for (Path path : files)
+                    Files.delete(path);
+            }
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Failed to cleanup wal work directory: " + walWorkDir, e);
+        }
+
+        try {
+            try (DirectoryStream<Path> files = Files.newDirectoryStream(walArchiveDir.toPath())) {
+                for (Path path : files)
+                    Files.delete(path);
+            }
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Failed to cleanup wal archive directory: " + walArchiveDir, e);
+        }
+    }
+
     /**
      * Clears whole the file, fills with zeros for Default mode.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 79a3e1966dede..4a642d0193fe3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -28,8 +28,10 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.file.DirectoryStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.sql.Time;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -886,6 +888,29 @@ private boolean hasIndex(long absIdx) {
         return ctx != null && !ctx.walEnabled();
     }
 
+    /** {@inheritDoc} */
+    @Override public void cleanupWalDirectories() throws IgniteCheckedException {
+        try {
+            try (DirectoryStream<Path> files = Files.newDirectoryStream(walWorkDir.toPath())) {
+                for (Path path : files)
+                    Files.delete(path);
+            }
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Failed to cleanup wal work directory: " + walWorkDir, e);
+        }
+
+        try {
+            try (DirectoryStream<Path> files = Files.newDirectoryStream(walArchiveDir.toPath())) {
+                for (Path path : files)
+                    Files.delete(path);
+            }
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Failed to cleanup wal archive directory: " + walArchiveDir, e);
+        }
+    }
+
     /**
      * Lists files in archive directory and returns the index of last archived file.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java
new file mode 100644
index 0000000000000..0c9fb631b39a2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test absenting eviction for joined node if it is out of baseline.
+ */
+public class IgniteAbsentEvictionNodeOutOfBaselineTest extends GridCommonAbstractTest {
+    /** */
+    private static final String TEST_CACHE_NAME = "test";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+            .setWalSegmentSize(512 * 1024)
+            .setWalSegments(4)
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(256 * 1024 * 1024)
+                    .setPersistenceEnabled(true))
+            .setWalMode(WALMode.LOG_ONLY));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Removed partitions if node is out of baseline.
+     */
+    public void testPartitionsRemovedIfJoiningNodeNotInBaseline() throws Exception {
+        //given: start 3 nodes with data
+        Ignite ignite0 = startGrids(3);
+
+        ignite0.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = ignite0.getOrCreateCache(TEST_CACHE_NAME);
+
+        for(int i = 0; i< 100; i++)
+            cache.put(i, i);
+
+        //when: stop one node and reset baseline topology
+        stopGrid(2);
+
+        resetBaselineTopology();
+
+        awaitPartitionMapExchange();
+
+        for(int i = 0; i< 200; i++)
+            cache.put(i, i);
+
+        //then: after returning stopped node to grid its partitions should be removed
+        IgniteEx ignite2 = startGrid(2);
+
+        List<GridDhtLocalPartition> partitions = ignite2.cachex(TEST_CACHE_NAME).context().topology().localPartitions();
+
+        assertTrue("Should be empty : " + partitions, partitions.isEmpty());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index be40c90f7c15c..ba236af929365 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -24,6 +24,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -215,4 +216,9 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
     @Override public long pagesAllocated(int grpId) {
         return 0;
     }
+
+    /** {@inheritDoc} */
+    @Override public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws IgniteCheckedException {
+        // No-op.
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index c95d1f4a61340..0188445463d32 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -101,6 +101,11 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public void cleanupWalDirectories() throws IgniteCheckedException {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public void start(GridCacheSharedContext cctx) throws IgniteCheckedException {
         // No-op.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 9ce1242686e29..40bcf8c3f07d0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
 import org.apache.ignite.internal.processors.cache.persistence.LocalWalModeChangeDuringRebalancingSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAbsentEvictionNodeOutOfBaselineTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAllBaselineNodesOnlineFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOfflineBaselineNodeFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest;
@@ -70,6 +71,7 @@ public static TestSuite suite() {
         suite.addTestSuite(IgniteAllBaselineNodesOnlineFullApiSelfTest.class);
         suite.addTestSuite(IgniteOfflineBaselineNodeFullApiSelfTest.class);
         suite.addTestSuite(IgniteOnlineNodeOutOfBaselineFullApiSelfTest.class);
+        suite.addTestSuite(IgniteAbsentEvictionNodeOutOfBaselineTest.class);
 
         return suite;
     }

From 504a0ef759917b65cdb4428f9e28769b384e080e Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Fri, 25 May 2018 13:29:26 +0300
Subject: [PATCH 160/543] IGNITE-7933 compilation error fix after cherry-pick

---
 .../LocalWalModeChangeDuringRebalancingSelfTest.java         | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
index 8be819f27b4e8..ca46a75e09181 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -614,6 +614,11 @@ private static class TestFileIO implements FileIO {
             delegate.force();
         }
 
+        /** {@inheritDoc} */
+        @Override public void force(boolean withMetadata) throws IOException {
+            delegate.force(withMetadata);
+        }
+
         /** {@inheritDoc} */
         @Override public long size() throws IOException {
             return delegate.size();

From 46277009573aedd13eb5a8eafb1af0e52f5fabc1 Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Mon, 16 Apr 2018 20:43:36 +0300
Subject: [PATCH 161/543] IGNITE-7972 Fixed NPE in TTL manager on unwindEvicts.
 - Fixes #3810.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit 737933e)
---
 .../internal/processors/cache/GridCacheTtlManager.java | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManager.java
index b6f54a14748ee..d36485ab70a7f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheTtlManager.java
@@ -40,8 +40,8 @@ public class GridCacheTtlManager extends GridCacheManagerAdapter {
     /** Entries pending removal. */
     private GridConcurrentSkipListSetEx pendingEntries;
 
-    /** */
-    private boolean eagerTtlEnabled;
+    /** See {@link CacheConfiguration#isEagerTtl()}. */
+    private volatile boolean eagerTtlEnabled;
 
     /** */
     private GridCacheContext dhtCtx;
@@ -166,6 +166,12 @@ public void expire() {
      * @return {@code True} if unprocessed expired entries remains.
      */
     public boolean expire(int amount) {
+        // TTL manager is not initialized or eagerTtl disabled for cache.
+        if (!eagerTtlEnabled)
+            return false;
+
+        assert cctx != null;
+
         long now = U.currentTimeMillis();
 
         try {

From 9f7ec7fdf1d9144fb9211e9334a9dbd8548eac97 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Tue, 29 May 2018 17:38:38 +0300
Subject: [PATCH 162/543] IGNITE-8624 Added reproducer of IGNITE-7972 issue. -
 Fixes #4077.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>

(cherry picked from commit d191cef)
---
 .../transactions/TxOnCachesStartTest.java     | 141 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite6.java     |   9 +-
 2 files changed, 147 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOnCachesStartTest.java

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOnCachesStartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOnCachesStartTest.java
new file mode 100644
index 0000000000000..24044040cac8b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxOnCachesStartTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+
+/**
+ *  Tests transactions closes correctly while other caches start and stop.
+ *  Tests possible {@link NullPointerException} in {@link TransactionProxyImpl#leave} due to race while
+ *  {@link org.apache.ignite.internal.processors.cache.GridCacheTtlManager} initializes (IGNITE-7972).
+ */
+public class TxOnCachesStartTest extends GridCommonAbstractTest {
+    /** */
+    private static int NUM_CACHES = 100;
+
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTransactionCloseOnCachesStartAndStop() throws Exception {
+        Ignite srv =  startGrids(5);
+
+        IgniteEx client1 = startGrid(getConfiguration("client-1").setClientMode(true));
+
+        srv.cluster().active(true);
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        AtomicReference<NullPointerException> ex = new AtomicReference<>(null);
+
+        IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
+            @Override public void run() {
+                for (int i = 0; i < NUM_CACHES; i++) {
+                    IgniteCache cache = client1.getOrCreateCache(testCacheConfiguration(DEFAULT_CACHE_NAME + i));
+
+                    try {
+                        U.sleep(100);
+                    }
+                    catch (Exception e) {
+                        //Ignore.
+                    }
+
+                    cache.destroy();
+                }
+            }
+        }, 1, "tx-thread");
+
+        GridTestUtils.runMultiThreadedAsync(new Runnable() {
+            @Override public void run()  {
+                while (true) {
+                    try (Transaction tx = client1.transactions().txStart()) {
+                        /** Empty transaction, just testing {@link TransactionProxyImpl#leave} */
+                    }
+                    catch (NullPointerException e) {
+                        e.printStackTrace();
+
+                        ex.compareAndSet(null, e);
+
+                        latch.countDown();
+
+                        break;
+                    }
+                }
+            }
+        }, 1, "tx-thread");
+
+        latch.await(5, TimeUnit.SECONDS);
+
+        fut.cancel();
+
+        assertNull("NullPointerException thrown while closing transaction", ex.get());
+    }
+
+    /**
+     * Get cache configuration for tests.
+     *
+     * @param name Name.
+     */
+    private CacheConfiguration testCacheConfiguration(String name) {
+        return new CacheConfiguration()
+                .setGroupName("default-group")
+                .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+                   .setName(name);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index fb0daa9f67396..66c1c488cf447 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -44,14 +44,15 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxLabelTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxMultiCacheAsyncOpsTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxOnCachesStartTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticOnPartitionExchangeTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticPrepareOnUnstableTopologyTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncNearCacheTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncWithPersistenceTest;
-import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticOnPartitionExchangeTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNearCacheTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest;
-import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncNearCacheTest;
-import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTopologyChangeTest;
 
 /**
@@ -89,6 +90,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(TxMultiCacheAsyncOpsTest.class);
 
+        suite.addTestSuite(TxOnCachesStartTest.class);
+
         suite.addTestSuite(IgnitePdsCacheAssignmentNodeRestartsTest.class);
 
         suite.addTestSuite(WalModeChangeSelfTest.class);

From 907e05f8e35a721629690253ce4de272f101b400 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Thu, 31 May 2018 16:38:33 +0300
Subject: [PATCH 163/543] IGNITE-8651 VisrTxTask fails when printing
 transactions having implicit single type - Fixes #4096.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../ignite/internal/visor/tx/VisorTxTask.java | 12 ++++-
 .../ignite/util/GridCommandHandlerTest.java   | 47 +++++++++++++------
 2 files changed, 43 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
index 72b174050c72c..b411e29006757 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
@@ -21,6 +21,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.ConcurrentModificationException;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -32,6 +33,7 @@
 import org.apache.ignite.compute.ComputeJobResult;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxMappings;
 import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.F;
@@ -179,10 +181,16 @@ private VisorTxJob(VisorTxTaskArg arg, boolean debug) {
                 int size = 0;
 
                 if (locTx.mappings() != null) {
-                    for (GridDistributedTxMapping mapping : locTx.mappings().mappings()) {
+                    IgniteTxMappings txMappings = locTx.mappings();
+
+                    for (GridDistributedTxMapping mapping :
+                        txMappings.single() ? Collections.singleton(txMappings.singleMapping()) : txMappings.mappings()) {
+                        if (mapping == null)
+                            continue;
+
                         mappings.add(mapping.primary().id());
 
-                        size += mapping.entries().size(); // Entries are not synchronized so no visibility guaranties.
+                        size += mapping.entries().size(); // Entries are not synchronized so no visibility guaranties for size.
                     }
                 }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 670c22c313b82..385e79b3926fa 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -22,6 +22,7 @@
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.TreeMap;
@@ -47,6 +48,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.commandline.CommandHandler;
 import org.apache.ignite.internal.commandline.cache.CacheCommand;
+import org.apache.ignite.internal.processors.cache.GridCacheFuture;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.X;
@@ -402,7 +404,7 @@ public void testActiveTransactions() throws Exception {
 
             for (VisorTxInfo info : res.getInfos()) {
                 if (info.getSize() == 100) {
-                    toKill[0] = info;
+                    toKill[0] = info; // Store for further use.
 
                     break;
                 }
@@ -411,7 +413,7 @@ public void testActiveTransactions() throws Exception {
             assertEquals(3, map.size());
         }, "--tx");
 
-        assertNotNull(toKill);
+        assertNotNull(toKill[0]);
 
         // Test filter by label.
         validate(h, map -> {
@@ -460,21 +462,18 @@ else if (entry.getKey().equals(node2)) {
         }, "--tx", "order", "DURATION");
 
         // Trigger topology change and test connection.
-        IgniteInternalFuture<?> startFut = multithreadedAsync(new Runnable() {
-            @Override public void run() {
-                try {
-                    startGrid(2);
-                }
-                catch (Exception e) {
-                    fail();
-                }
+        IgniteInternalFuture<?> startFut = multithreadedAsync(() -> {
+            try {
+                startGrid(2);
+            }
+            catch (Exception e) {
+                fail();
             }
         }, 1, "start-node-thread");
 
-        doSleep(5000);
+        doSleep(5000); // Give enough time to reach exchange future.
 
-        assertEquals(EXIT_CODE_OK, execute(h, "--host", "127.0.0.1", "--port", "11211", "--tx"));
-        assertEquals(EXIT_CODE_OK, execute(h, "--host", "127.0.0.1", "--port", "11212", "--tx"));
+        assertEquals(EXIT_CODE_OK, execute(h, "--tx"));
 
         // Test kill by xid.
         validate(h, map -> {
@@ -486,7 +485,7 @@ else if (entry.getKey().equals(node2)) {
 
                 assertEquals(toKill[0].getXid(), info.getXid());
             }, "--tx", "kill",
-            "xid", toKill[0].getXid().toString(),
+            "xid", toKill[0].getXid().toString(), // Use saved on first run value.
             "nodes", grid(0).localNode().consistentId().toString());
 
         unlockLatch.countDown();
@@ -494,6 +493,10 @@ else if (entry.getKey().equals(node2)) {
         startFut.get();
 
         fut.get();
+
+        awaitPartitionMapExchange();
+
+        checkFutures();
     }
 
     /**
@@ -889,4 +892,20 @@ private IgniteInternalFuture<?> startTransactions(CountDownLatch lockLatch, Coun
             }
         }, 4, "tx-thread");
     }
+
+    /**
+     * Checks if all tx futures are finished.
+     */
+    private void checkFutures() {
+        for (Ignite ignite : G.allGrids()) {
+            IgniteEx ig = (IgniteEx)ignite;
+
+            final Collection<GridCacheFuture<?>> futs = ig.context().cache().context().mvcc().activeFutures();
+
+            for (GridCacheFuture<?> fut : futs)
+                log.info("Waiting for future: " + fut);
+
+            assertTrue("Expecting no active futures: node=" + ig.localNode().id(), futs.isEmpty());
+        }
+    }
 }

From 19bef904d53138609a573b3bc71f83fcc434658e Mon Sep 17 00:00:00 2001
From: Ivan Rakov <irakov@apache.org>
Date: Thu, 31 May 2018 16:56:14 +0300
Subject: [PATCH 164/543] IGNITE-8476 AssertionError exception occurs when
 trying to remove node from baseline under loading

(cherry picked from commit 5e6e4e5)
---
 .../apache/ignite/internal/IgniteKernal.java  |  18 +-
 .../ignite/internal/IgniteNodeAttributes.java |   3 +
 .../processors/cache/CacheGroupContext.java   |  13 +-
 .../processors/cache/ClusterCachesInfo.java   |  58 +-
 .../processors/cache/GridCacheProcessor.java  |   1 +
 ...entAffinityAssignmentWithBaselineTest.java | 974 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   2 +
 7 files changed, 1059 insertions(+), 10 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 07edc7e71f041..0d0abfc6ac237 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -105,9 +105,6 @@
 import org.apache.ignite.internal.binary.BinaryUtils;
 import org.apache.ignite.internal.cluster.ClusterGroupAdapter;
 import org.apache.ignite.internal.cluster.IgniteClusterEx;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
-import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.managers.GridManager;
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
 import org.apache.ignite.internal.managers.collision.GridCollisionManager;
@@ -121,6 +118,7 @@
 import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
 import org.apache.ignite.internal.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.internal.processors.GridProcessor;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
 import org.apache.ignite.internal.processors.authentication.IgniteAuthenticationProcessor;
 import org.apache.ignite.internal.processors.cache.CacheConfigurationOverride;
@@ -136,11 +134,13 @@
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.closure.GridClosureProcessor;
 import org.apache.ignite.internal.processors.cluster.ClusterProcessor;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.processors.cluster.GridClusterStateProcessor;
 import org.apache.ignite.internal.processors.cluster.IGridClusterStateProcessor;
 import org.apache.ignite.internal.processors.continuous.GridContinuousProcessor;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.processors.hadoop.Hadoop;
 import org.apache.ignite.internal.processors.hadoop.HadoopProcessorAdapter;
 import org.apache.ignite.internal.processors.job.GridJobProcessor;
@@ -201,10 +201,10 @@
 import org.apache.ignite.mxbean.ClusterMetricsMXBean;
 import org.apache.ignite.mxbean.IgniteMXBean;
 import org.apache.ignite.mxbean.StripedExecutorMXBean;
-import org.apache.ignite.mxbean.WorkersControlMXBean;
 import org.apache.ignite.mxbean.ThreadPoolMXBean;
-import org.apache.ignite.mxbean.TransactionsMXBean;
 import org.apache.ignite.mxbean.TransactionMetricsMxBean;
+import org.apache.ignite.mxbean.TransactionsMXBean;
+import org.apache.ignite.mxbean.WorkersControlMXBean;
 import org.apache.ignite.plugin.IgnitePlugin;
 import org.apache.ignite.plugin.PluginNotFoundException;
 import org.apache.ignite.plugin.PluginProvider;
@@ -1582,7 +1582,13 @@ private void fillNodeAttributes(boolean notifyEnabled) throws IgniteCheckedExcep
 
         // Stick in network context into attributes.
         add(ATTR_IPS, (ips.isEmpty() ? "" : ips));
-        add(ATTR_MACS, (macs.isEmpty() ? "" : macs));
+
+        Map<String, ?> userAttrs = configuration().getUserAttributes();
+
+        if (userAttrs != null && userAttrs.get(IgniteNodeAttributes.ATTR_MACS_OVERRIDE) != null)
+            add(ATTR_MACS, (Serializable)userAttrs.get(IgniteNodeAttributes.ATTR_MACS_OVERRIDE));
+        else
+            add(ATTR_MACS, (macs.isEmpty() ? "" : macs));
 
         // Stick in some system level attributes
         add(ATTR_JIT_NAME, U.getCompilerMx() == null ? "" : U.getCompilerMx().getName());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
index 073369fdd2efb..6a4beebac0cba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
@@ -120,6 +120,9 @@ public final class IgniteNodeAttributes {
     /** Internal attribute name constant. */
     public static final String ATTR_MACS = ATTR_PREFIX + ".macs";
 
+    /** Allows to override {@link #ATTR_MACS} by adding this attribute in the user attributes. */
+    public static final String ATTR_MACS_OVERRIDE = "override." + ATTR_MACS;
+
     /** Internal attribute name constant. */
     public static final String ATTR_PHY_RAM = ATTR_PREFIX + ".phy.ram";
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 5f750d5996416..4f9b7f598d1b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -133,6 +133,9 @@ public class CacheGroupContext {
     /** */
     private final DataRegion dataRegion;
 
+    /** Persistence enabled flag. */
+    private final boolean persistenceEnabled;
+
     /** */
     private final CacheObjectContext cacheObjCtx;
 
@@ -158,8 +161,8 @@ public class CacheGroupContext {
     private volatile boolean globalWalEnabled;
 
     /**
-     * @param grpId Group ID.
      * @param ctx Context.
+     * @param grpId Group ID.
      * @param rcvdFrom Node ID cache group was received from.
      * @param cacheType Cache type.
      * @param ccfg Cache configuration.
@@ -169,6 +172,7 @@ public class CacheGroupContext {
      * @param freeList Free list.
      * @param reuseList Reuse list.
      * @param locStartVer Topology version when group was started on local node.
+     * @param persistenceEnabled Persistence enabled flag.
      * @param walEnabled Wal enabled flag.
      */
     CacheGroupContext(
@@ -183,7 +187,9 @@ public class CacheGroupContext {
         FreeList freeList,
         ReuseList reuseList,
         AffinityTopologyVersion locStartVer,
-        boolean walEnabled) {
+        boolean persistenceEnabled,
+        boolean walEnabled
+    ) {
         assert ccfg != null;
         assert dataRegion != null || !affNode;
         assert grpId != 0 : "Invalid group ID [cache=" + ccfg.getName() + ", grpName=" + ccfg.getGroupName() + ']';
@@ -200,6 +206,7 @@ public class CacheGroupContext {
         this.locStartVer = locStartVer;
         this.cacheType = cacheType;
         this.globalWalEnabled = walEnabled;
+        this.persistenceEnabled = persistenceEnabled;
         this.localWalEnabled = true;
 
         persistGlobalWalState(walEnabled);
@@ -918,7 +925,7 @@ public void start() throws IgniteCheckedException {
      * @return Persistence enabled flag.
      */
     public boolean persistenceEnabled() {
-        return dataRegion != null && dataRegion.config().isPersistenceEnabled();
+        return persistenceEnabled;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 975617ee8dc0a..9ff84e7d510cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -38,10 +38,12 @@
 import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.GridCachePluginContext;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateFinishMessage;
@@ -60,6 +62,7 @@
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.plugin.CachePluginContext;
 import org.apache.ignite.plugin.CachePluginProvider;
 import org.apache.ignite.plugin.PluginProvider;
@@ -1752,7 +1755,7 @@ private CacheGroupDescriptor registerCacheGroup(
 
         Map<String, Integer> caches = Collections.singletonMap(startedCacheCfg.getName(), cacheId);
 
-        boolean persistent = CU.isPersistentCache(startedCacheCfg, ctx.config().getDataStorageConfiguration());
+        boolean persistent = resolvePersistentFlag(exchActions, startedCacheCfg);
 
         CacheGroupDescriptor grpDesc = new CacheGroupDescriptor(
             startedCacheCfg,
@@ -1781,6 +1784,59 @@ private CacheGroupDescriptor registerCacheGroup(
         return grpDesc;
     }
 
+    /**
+     * Resolves persistent flag for new cache group descriptor.
+     *
+     * @param exchActions Optional exchange actions to update if new group was added.
+     * @param startedCacheCfg Started cache configuration.
+     */
+    private boolean resolvePersistentFlag(@Nullable ExchangeActions exchActions, CacheConfiguration<?, ?> startedCacheCfg) {
+        if (!ctx.clientNode()) {
+            // On server, we always can determine whether cache is persistent by local storage configuration.
+            return CU.isPersistentCache(startedCacheCfg, ctx.config().getDataStorageConfiguration());
+        }
+        else if (exchActions == null) {
+            // It's either client local join event or cache is statically configured on another node.
+            // No need to resolve on client - we'll anyway receive group descriptor from server with correct flag.
+            return false;
+        }
+        else {
+            // Dynamic cache start. Initiator of the start may not have known whether cache should be persistent.
+            // On client, we should peek attributes of any affinity server node to get data storage configuration.
+            Collection<ClusterNode> aliveSrvNodes = ctx.discovery().aliveServerNodes();
+
+            assert !aliveSrvNodes.isEmpty() : "No alive server nodes";
+
+            for (ClusterNode srvNode : aliveSrvNodes) {
+                if (CU.affinityNode(srvNode, startedCacheCfg.getNodeFilter())) {
+                    Object dsCfgBytes = srvNode.attribute(IgniteNodeAttributes.ATTR_DATA_STORAGE_CONFIG);
+
+                    if (dsCfgBytes instanceof byte[]) {
+                        try {
+                            DataStorageConfiguration crdDsCfg = new JdkMarshaller().unmarshal(
+                                (byte[])dsCfgBytes, U.resolveClassLoader(ctx.config()));
+
+                            return CU.isPersistentCache(startedCacheCfg, crdDsCfg);
+                        }
+                        catch (IgniteCheckedException e) {
+                            U.error(log, "Failed to unmarshal remote data storage configuration [remoteNode=" +
+                                srvNode + ", cacheName=" + startedCacheCfg.getName() + "]", e);
+                        }
+                    }
+                    else {
+                        U.error(log, "Remote marshalled data storage configuration is absent [remoteNode=" + srvNode +
+                            ", cacheName=" + startedCacheCfg.getName() + ", dsCfg=" + dsCfgBytes + "]");
+                    }
+                }
+            }
+
+            U.error(log, "Failed to find affinity server node with data storage configuration for starting cache " +
+                "[cacheName=" + startedCacheCfg.getName() + ", aliveSrvNodes=" + aliveSrvNodes + "]");
+
+            return false;
+        }
+    }
+
     /**
      * @param ccfg Cache configuration to start.
      * @throws IgniteCheckedException If failed.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 1a7c9f581cba0..8b63cdd1fc8a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2040,6 +2040,7 @@ private CacheGroupContext startCacheGroup(
             freeList,
             reuseList,
             exchTopVer,
+            desc.persistenceEnabled(),
             desc.walEnabled()
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
new file mode 100644
index 0000000000000..15ec41557dfba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
@@ -0,0 +1,974 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.baseline;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.LockSupport;
+import javax.cache.CacheException;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.cluster.ClusterTopologyException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ * Checks that client affinity assignment cache is calculated correctly regardless of current baseline topology.
+ */
+public class ClientAffinityAssignmentWithBaselineTest extends GridCommonAbstractTest {
+    /** Nodes count. */
+    private static final int DEFAULT_NODES_COUNT = 5;
+
+    /** Tx cache name. */
+    private static final String PARTITIONED_TX_CACHE_NAME = "p-tx-cache";
+
+    /** Tx cache name with shifted affinity. */
+    private static final String PARTITIONED_TX_PRIM_SYNC_CACHE_NAME = "prim-sync";
+
+    /** Tx cache name from client static configuration. */
+    private static final String PARTITIONED_TX_CLIENT_CACHE_NAME = "p-tx-client-cache";
+
+    /** Atomic cache name. */
+    private static final String PARTITIONED_ATOMIC_CACHE_NAME = "p-atomic-cache";
+
+    /** Tx cache name. */
+    private static final String REPLICATED_TX_CACHE_NAME = "r-tx-cache";
+
+    /** Atomic cache name. */
+    private static final String REPLICATED_ATOMIC_CACHE_NAME = "r-atomic-cache";
+
+    /** Client grid name. */
+    private static final String CLIENT_GRID_NAME = "client";
+
+    /** Flaky node name */
+    private static final String FLAKY_NODE_NAME = "flaky";
+
+    /** Entries. */
+    private static final int ENTRIES = 3_000;
+
+    /** Flaky node wal path. */
+    public static final String FLAKY_WAL_PATH = "flakywal";
+
+    /** Flaky node wal archive path. */
+    public static final String FLAKY_WAL_ARCHIVE_PATH = "flakywalarchive";
+
+    /** Flaky node storage path. */
+    public static final String FLAKY_STORAGE_PATH = "flakystorage";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (igniteInstanceName.startsWith(CLIENT_GRID_NAME)) {
+            // Intentionally skipping data storage in client configuration.
+            cfg.setClientMode(true);
+        }
+        else {
+            cfg.setDataStorageConfiguration(
+                new DataStorageConfiguration()
+                    .setDefaultDataRegionConfiguration(
+                        new DataRegionConfiguration()
+                            .setPersistenceEnabled(true)
+                            .setMaxSize(200 * 1024 * 1024)
+                    )
+            );
+        }
+
+        if (igniteInstanceName.contains(FLAKY_NODE_NAME)) {
+            File store = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false);
+
+            cfg.getDataStorageConfiguration().setWalPath(new File(store, FLAKY_WAL_PATH).getAbsolutePath());
+            cfg.getDataStorageConfiguration().setWalArchivePath(new File(store, FLAKY_WAL_ARCHIVE_PATH).getAbsolutePath());
+            cfg.getDataStorageConfiguration().setStoragePath(new File(store, FLAKY_STORAGE_PATH).getAbsolutePath());
+        }
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        List<CacheConfiguration> srvConfigs = new ArrayList<>();
+        srvConfigs.add(cacheConfig(PARTITIONED_TX_CACHE_NAME));
+        srvConfigs.add(cacheConfig(PARTITIONED_TX_PRIM_SYNC_CACHE_NAME));
+        srvConfigs.add(cacheConfig(REPLICATED_ATOMIC_CACHE_NAME));
+
+        List<CacheConfiguration> clientConfigs = new ArrayList<>(srvConfigs);
+
+        // Skip some configs in client static configuration to check that clients receive correct cache descriptors.
+        srvConfigs.add(cacheConfig(PARTITIONED_ATOMIC_CACHE_NAME));
+        srvConfigs.add(cacheConfig(REPLICATED_TX_CACHE_NAME));
+
+        // Skip config in server static configuration to check that caches received on client join start correctly.
+        clientConfigs.add(cacheConfig(PARTITIONED_TX_CLIENT_CACHE_NAME));
+
+        if (igniteInstanceName.startsWith(CLIENT_GRID_NAME))
+            cfg.setCacheConfiguration(clientConfigs.toArray(new CacheConfiguration[clientConfigs.size()]));
+        else
+            cfg.setCacheConfiguration(srvConfigs.toArray(new CacheConfiguration[srvConfigs.size()]));
+
+        // Enforce different mac adresses to emulate distributed environment by default.
+        cfg.setUserAttributes(Collections.singletonMap(
+            IgniteNodeAttributes.ATTR_MACS_OVERRIDE, UUID.randomUUID().toString()));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * @param cacheName Cache name.
+     */
+    private CacheConfiguration<Integer, String> cacheConfig(String cacheName) {
+        CacheConfiguration<Integer, String> cfg = new CacheConfiguration<>();
+
+        if (PARTITIONED_ATOMIC_CACHE_NAME.equals(cacheName)) {
+            cfg.setName(PARTITIONED_ATOMIC_CACHE_NAME);
+            cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+            cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+            cfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+            cfg.setBackups(2);
+        }
+        else if (PARTITIONED_TX_CACHE_NAME.equals(cacheName)) {
+            cfg.setName(PARTITIONED_TX_CACHE_NAME);
+            cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+            cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+            cfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+            cfg.setBackups(2);
+        }
+        else if (PARTITIONED_TX_CLIENT_CACHE_NAME.equals(cacheName)) {
+            cfg.setName(PARTITIONED_TX_CLIENT_CACHE_NAME);
+            cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+            cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+            cfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+            cfg.setBackups(2);
+        }
+        else if (PARTITIONED_TX_PRIM_SYNC_CACHE_NAME.equals(cacheName)) {
+            cfg.setName(PARTITIONED_TX_PRIM_SYNC_CACHE_NAME);
+            cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+            cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
+            cfg.setAffinity(new RendezvousAffinityFunction(false, 41)); // To break collocation.
+            cfg.setBackups(2);
+        }
+        else if (REPLICATED_ATOMIC_CACHE_NAME.equals(cacheName)) {
+            cfg.setName(REPLICATED_ATOMIC_CACHE_NAME);
+            cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+            cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+            cfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+            cfg.setCacheMode(CacheMode.REPLICATED);
+        }
+        else if (REPLICATED_TX_CACHE_NAME.equals(cacheName)) {
+            cfg.setName(REPLICATED_TX_CACHE_NAME);
+            cfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+            cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+            cfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+            cfg.setCacheMode(CacheMode.REPLICATED);
+        }
+        else
+            throw new IllegalArgumentException("Unexpected cache name");
+
+        return cfg;
+    }
+
+    /**
+     *
+     */
+    public void testPartitionedAtomicCache() throws Exception {
+        testChangingBaselineDown(PARTITIONED_ATOMIC_CACHE_NAME, false);
+    }
+
+    /**
+     *
+     */
+    public void testPartitionedTxCache() throws Exception {
+        testChangingBaselineDown(PARTITIONED_TX_CACHE_NAME, false);
+    }
+
+    /**
+     * Test that activation after client join won't break cache.
+     */
+    public void testLateActivation() throws Exception {
+        testChangingBaselineDown(PARTITIONED_TX_CACHE_NAME, true);
+    }
+
+    /**
+     *
+     */
+    public void testReplicatedAtomicCache() throws Exception {
+        testChangingBaselineDown(REPLICATED_ATOMIC_CACHE_NAME, false);
+    }
+
+    /**
+     *
+     */
+    public void testReplicatedTxCache() throws Exception {
+        testChangingBaselineDown(REPLICATED_TX_CACHE_NAME, false);
+    }
+
+    /**
+     * Tests that changing baseline down under load won't break cache.
+     */
+    private void testChangingBaselineDown(String cacheName, boolean lateActivation) throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(DEFAULT_NODES_COUNT);
+
+        IgniteEx client1 = null;
+        IgniteEx client2 = null;
+
+        if (lateActivation) {
+            client1 = (IgniteEx)startGrid("client1");
+            client2 = (IgniteEx)startGrid("client2");
+        }
+        else
+            ig0.cluster().active(true);
+
+        AtomicBoolean stopLoad = new AtomicBoolean(false);
+
+        AtomicReference<Throwable> loadError = new AtomicReference<>(null);
+
+        if (lateActivation)
+            ig0.cluster().active(true);
+
+        IgniteCache<Integer, String> cache = ig0.cache(cacheName);
+
+        System.out.println("### Starting preloading");
+
+        for (int i = 0; i < ENTRIES; i++) {
+            ThreadLocalRandom r = ThreadLocalRandom.current();
+
+            byte[] randBytes = new byte[r.nextInt(10, 100)];
+
+            cache.put(r.nextInt(ENTRIES), new String(randBytes));
+        }
+
+        System.out.println("### Preloading is finished");
+
+        if (!lateActivation) {
+            client1 = (IgniteEx)startGrid("client1");
+            client2 = (IgniteEx)startGrid("client2");
+        }
+
+        ConcurrentMap<Long, Long> threadProgressTracker = new ConcurrentHashMap<>();
+
+        startSimpleLoadThread(client1, cacheName, stopLoad, loadError, threadProgressTracker);
+        startSimpleLoadThread(client1, cacheName, stopLoad, loadError, threadProgressTracker);
+        startSimpleLoadThread(client1, cacheName, stopLoad, loadError, threadProgressTracker);
+        startTxLoadThread(client2, cacheName, stopLoad, loadError, threadProgressTracker);
+        startTxLoadThread(client2, cacheName, stopLoad, loadError, threadProgressTracker);
+        startTxLoadThread(client2, cacheName, stopLoad, loadError, threadProgressTracker);
+
+        awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
+
+        List<BaselineNode> fullBlt = new ArrayList<>();
+        for (int i = 0; i < DEFAULT_NODES_COUNT; i++)
+            fullBlt.add(grid(i).localNode());
+
+        stopGrid(DEFAULT_NODES_COUNT - 1, true);
+        stopGrid(DEFAULT_NODES_COUNT - 2, true);
+
+        awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
+
+        tryChangeBaselineDown(ig0, fullBlt, DEFAULT_NODES_COUNT - 1, loadError, threadProgressTracker);
+        tryChangeBaselineDown(ig0, fullBlt, DEFAULT_NODES_COUNT - 2, loadError, threadProgressTracker);
+
+        stopLoad.set(true);
+    }
+
+    /**
+     * Tests that rejoin of baseline node with clear LFS under load won't break cache.
+     */
+    public void testRejoinWithCleanLfs() throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(DEFAULT_NODES_COUNT - 1);
+        startGrid("flaky");
+
+        ig0.cluster().active(true);
+
+        AtomicBoolean stopLoad = new AtomicBoolean(false);
+
+        AtomicReference<Throwable> loadError = new AtomicReference<>(null);
+
+        IgniteCache<Integer, String> cache1 = ig0.cache(PARTITIONED_ATOMIC_CACHE_NAME);
+        IgniteCache<Integer, String> cache2 = ig0.cache(PARTITIONED_TX_CACHE_NAME);
+        IgniteCache<Integer, String> cache3 = ig0.cache(REPLICATED_ATOMIC_CACHE_NAME);
+        IgniteCache<Integer, String> cache4 = ig0.cache(REPLICATED_TX_CACHE_NAME);
+
+        System.out.println("### Starting preloading");
+
+        for (int i = 0; i < ENTRIES; i++) {
+            ThreadLocalRandom r = ThreadLocalRandom.current();
+
+            cache1.put(r.nextInt(ENTRIES), new String(new byte[r.nextInt(10, 100)]));
+            cache2.put(r.nextInt(ENTRIES), new String(new byte[r.nextInt(10, 100)]));
+            cache3.put(r.nextInt(ENTRIES), new String(new byte[r.nextInt(10, 100)]));
+            cache4.put(r.nextInt(ENTRIES), new String(new byte[r.nextInt(10, 100)]));
+        }
+
+        System.out.println("### Preloading is finished");
+
+        IgniteEx client1 = (IgniteEx)startGrid("client1");
+        IgniteEx client2 = (IgniteEx)startGrid("client2");
+
+        ConcurrentMap<Long, Long> threadProgressTracker = new ConcurrentHashMap<>();
+
+        startSimpleLoadThread(client1, PARTITIONED_ATOMIC_CACHE_NAME, stopLoad, loadError, threadProgressTracker);
+        startSimpleLoadThread(client1, PARTITIONED_TX_CACHE_NAME, stopLoad, loadError, threadProgressTracker);
+        startSimpleLoadThread(client1, REPLICATED_ATOMIC_CACHE_NAME, stopLoad, loadError, threadProgressTracker);
+        startTxLoadThread(client2, PARTITIONED_ATOMIC_CACHE_NAME, stopLoad, loadError, threadProgressTracker);
+        startTxLoadThread(client2, PARTITIONED_TX_CACHE_NAME, stopLoad, loadError, threadProgressTracker);
+        startTxLoadThread(client2, REPLICATED_TX_CACHE_NAME, stopLoad, loadError, threadProgressTracker);
+
+        awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
+
+        stopGrid("flaky");
+
+        awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
+
+        File store = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false);
+
+        U.delete(new File(store, FLAKY_WAL_PATH));
+        U.delete(new File(store, FLAKY_WAL_ARCHIVE_PATH));
+        U.delete(new File(store, FLAKY_STORAGE_PATH));
+
+        startGrid("flaky");
+
+        System.out.println("### Starting rebalancing after flaky node join");
+        waitForRebalancing();
+        System.out.println("### Rebalancing is finished after flaky node join");
+
+        awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
+
+        stopLoad.set(true);
+    }
+
+    /**
+     * Test that changing baseline down under cross-cache txs load won't break cache.
+     */
+    public void testCrossCacheTxs() throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(DEFAULT_NODES_COUNT);
+
+        ig0.cluster().active(true);
+
+        AtomicBoolean stopLoad = new AtomicBoolean(false);
+
+        AtomicReference<Throwable> loadError = new AtomicReference<>(null);
+
+        String cacheName1 = PARTITIONED_TX_CACHE_NAME;
+        String cacheName2 = PARTITIONED_TX_PRIM_SYNC_CACHE_NAME;
+
+        IgniteCache<Integer, String> cache1 = ig0.cache(PARTITIONED_TX_CACHE_NAME);
+        IgniteCache<Integer, String> cache2 = ig0.cache(PARTITIONED_TX_PRIM_SYNC_CACHE_NAME);
+
+        System.out.println("### Starting preloading");
+
+        for (int i = 0; i < ENTRIES; i++) {
+            ThreadLocalRandom r = ThreadLocalRandom.current();
+
+            byte[] randBytes1 = new byte[r.nextInt(10, 100)];
+            byte[] randBytes2 = new byte[r.nextInt(10, 100)];
+
+            cache1.put(r.nextInt(ENTRIES), new String(randBytes1));
+            cache2.put(r.nextInt(ENTRIES), new String(randBytes2));
+        }
+
+        System.out.println("### Preloading is finished");
+
+        IgniteEx client1 = (IgniteEx)startGrid("client1");
+        IgniteEx client2 = (IgniteEx)startGrid("client2");
+
+        ConcurrentMap<Long, Long> threadProgressTracker = new ConcurrentHashMap<>();
+
+        startCrossCacheTxLoadThread(client1, cacheName1, cacheName2, stopLoad, loadError, threadProgressTracker);
+        startCrossCacheTxLoadThread(client1, cacheName1, cacheName2, stopLoad, loadError, threadProgressTracker);
+        startCrossCacheTxLoadThread(client1, cacheName2, cacheName1, stopLoad, loadError, threadProgressTracker);
+        startCrossCacheTxLoadThread(client2, cacheName1, cacheName2, stopLoad, loadError, threadProgressTracker);
+        startCrossCacheTxLoadThread(client2, cacheName1, cacheName2, stopLoad, loadError, threadProgressTracker);
+        startCrossCacheTxLoadThread(client2, cacheName2, cacheName1, stopLoad, loadError, threadProgressTracker);
+
+        awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
+
+        List<BaselineNode> fullBlt = new ArrayList<>();
+        for (int i = 0; i < DEFAULT_NODES_COUNT; i++)
+            fullBlt.add(grid(i).localNode());
+
+        stopGrid(DEFAULT_NODES_COUNT - 1, true);
+        stopGrid(DEFAULT_NODES_COUNT - 2, true);
+
+        awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
+
+        tryChangeBaselineDown(ig0, fullBlt, DEFAULT_NODES_COUNT - 1, loadError, threadProgressTracker);
+        tryChangeBaselineDown(ig0, fullBlt, DEFAULT_NODES_COUNT - 2, loadError, threadProgressTracker);
+
+        stopLoad.set(true);
+    }
+
+    /**
+     * Tests that join of non-baseline node while long transactions are running won't break dynamically started cache.
+     */
+    public void testDynamicCacheLongTransactionNodeStart() throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(4);
+
+        ig0.cluster().active(true);
+
+        IgniteEx client = (IgniteEx)startGrid("client");
+
+        CacheConfiguration<Integer, String> dynamicCacheCfg = cacheConfig(REPLICATED_TX_CACHE_NAME);
+        dynamicCacheCfg.setName("dyn");
+
+        IgniteCache<Integer, String> dynamicCache = client.getOrCreateCache(dynamicCacheCfg);
+
+        for (int i = 0; i < ENTRIES; i++)
+            dynamicCache.put(i, "abacaba" + i);
+
+        AtomicBoolean releaseTx = new AtomicBoolean(false);
+        CountDownLatch allTxsDoneLatch = new CountDownLatch(10);
+
+        for (int i = 0; i < 10; i++) {
+            final int i0 = i;
+
+            GridTestUtils.runAsync(new Runnable() {
+                @Override public void run() {
+                    try (Transaction tx = client.transactions().txStart(TransactionConcurrency.PESSIMISTIC,
+                        TransactionIsolation.REPEATABLE_READ)) {
+                        dynamicCache.put(i0, "txtxtxtx" + i0);
+
+                        while (!releaseTx.get())
+                            LockSupport.parkNanos(1_000_000);
+
+                        tx.commit();
+
+                        System.out.println("Tx #" + i0 + " committed");
+                    }
+                    catch (Throwable t) {
+                        System.out.println("Tx #" + i0 + " failed");
+
+                        t.printStackTrace();
+                    }
+                    finally {
+                        allTxsDoneLatch.countDown();
+                    }
+                }
+            });
+        }
+
+        GridTestUtils.runAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    startGrid(4);
+                }
+                catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+        U.sleep(1_000);
+
+        releaseTx.set(true);
+
+        allTxsDoneLatch.await();
+
+        for (int i = 0; i < 10_000; i++)
+            assertEquals("txtxtxtx" + (i % 10), dynamicCache.get(i % 10));
+    }
+
+    /**
+     * Tests that if dynamic cache has no affinity nodes at the moment of start,
+     * it will still work correctly when affinity nodes will appear.
+     */
+    public void testDynamicCacheStartNoAffinityNodes() throws Exception {
+        fail("IGNITE-8652");
+
+        IgniteEx ig0 = startGrid(0);
+
+        ig0.cluster().active(true);
+        
+        IgniteEx client = (IgniteEx)startGrid("client");
+
+        CacheConfiguration<Integer, String> dynamicCacheCfg = new CacheConfiguration<Integer, String>()
+            .setName("dyn")
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(2)
+            .setNodeFilter(new ConsistentIdNodeFilter((Serializable)ig0.localNode().consistentId()));
+
+        IgniteCache<Integer, String> dynamicCache = client.getOrCreateCache(dynamicCacheCfg);
+        
+        for (int i = 1; i < 4; i++)
+            startGrid(i);
+
+        resetBaselineTopology();
+
+        for (int i = 0; i < ENTRIES; i++)
+            dynamicCache.put(i, "abacaba" + i);
+        
+        AtomicBoolean releaseTx = new AtomicBoolean(false);
+        CountDownLatch allTxsDoneLatch = new CountDownLatch(10);
+        
+        for (int i = 0; i < 10; i++) {
+            final int i0 = i;
+            
+            GridTestUtils.runAsync(new Runnable() {
+                @Override public void run() {
+                    try (Transaction tx = client.transactions().txStart(TransactionConcurrency.PESSIMISTIC,
+                        TransactionIsolation.REPEATABLE_READ)) {
+                        dynamicCache.put(i0, "txtxtxtx" + i0);
+                        
+                        while (!releaseTx.get())
+                            LockSupport.parkNanos(1_000_000);
+                        
+                        tx.commit();
+                        
+                        System.out.println("Tx #" + i0 + " committed");
+                    }
+                    catch (Throwable t) {
+                        System.out.println("Tx #" + i0 + " failed");
+                        
+                        t.printStackTrace();
+                    }
+                    finally {
+                        allTxsDoneLatch.countDown();
+                    }
+                }
+            });
+        }
+        
+        GridTestUtils.runAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    startGrid(4);
+                }
+                catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+        U.sleep(1_000);
+
+        releaseTx.set(true);
+
+        allTxsDoneLatch.await();
+
+        for (int i = 0; i < 10_000; i++)
+            assertEquals("txtxtxtx" + (i % 10), dynamicCache.get(i % 10));
+    }
+
+    /**
+     * Tests that join of non-baseline node while long transactions are running won't break cache started on client join.
+     */
+    public void testClientJoinCacheLongTransactionNodeStart() throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(4);
+
+        ig0.cluster().active(true);
+
+        IgniteEx client = (IgniteEx)startGrid("client");
+
+        IgniteCache<Integer, String> clientJoinCache = client.cache(PARTITIONED_TX_CLIENT_CACHE_NAME);
+
+        for (int i = 0; i < ENTRIES; i++)
+            clientJoinCache.put(i, "abacaba" + i);
+
+        AtomicBoolean releaseTx = new AtomicBoolean(false);
+        CountDownLatch allTxsDoneLatch = new CountDownLatch(10);
+
+        for (int i = 0; i < 10; i++) {
+            final int i0 = i;
+
+            GridTestUtils.runAsync(new Runnable() {
+                @Override public void run() {
+                    try (Transaction tx = client.transactions().txStart(TransactionConcurrency.PESSIMISTIC,
+                        TransactionIsolation.REPEATABLE_READ)) {
+                        clientJoinCache.put(i0, "txtxtxtx" + i0);
+
+                        while (!releaseTx.get())
+                            LockSupport.parkNanos(1_000_000);
+
+                        tx.commit();
+
+                        System.out.println("Tx #" + i0 + " committed");
+                    }
+                    catch (Throwable t) {
+                        System.out.println("Tx #" + i0 + " failed");
+
+                        t.printStackTrace();
+                    }
+                    finally {
+                        allTxsDoneLatch.countDown();
+                    }
+                }
+            });
+        }
+
+        GridTestUtils.runAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    startGrid(4);
+                }
+                catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        });
+
+        U.sleep(1_000);
+
+        releaseTx.set(true);
+
+        allTxsDoneLatch.await();
+
+        for (int i = 0; i < 10_000; i++)
+            assertEquals("txtxtxtx" + (i % 10), clientJoinCache.get(i % 10));
+    }
+
+    /**
+     * @param ig0 Ignite.
+     * @param fullBlt Initial BLT list.
+     * @param newBaselineSize New baseline size.
+     * @param threadProgressTracker Thread progress tracker.
+     */
+    private void tryChangeBaselineDown(
+        IgniteEx ig0,
+        List<BaselineNode> fullBlt,
+        int newBaselineSize,
+        AtomicReference<Throwable> loadError,
+        ConcurrentMap<Long, Long> threadProgressTracker
+    ) throws Exception {
+        System.out.println("### Changing BLT: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
+        ig0.cluster().setBaselineTopology(fullBlt.subList(0, newBaselineSize));
+
+        System.out.println("### Starting rebalancing after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
+        waitForRebalancing();
+        System.out.println("### Rebalancing is finished after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
+
+        awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
+
+        if (loadError.get() != null) {
+            loadError.get().printStackTrace();
+
+            fail("Unexpected error in load thread: " + loadError.get().toString());
+        }
+    }
+
+    /**
+     * @param ig Ignite instance.
+     * @param cacheName Cache name.
+     * @param stopFlag Stop flag.
+     * @param loadError Load error reference.
+     * @param threadProgressTracker Progress tracker.
+     */
+    private void startSimpleLoadThread(
+        IgniteEx ig,
+        String cacheName,
+        AtomicBoolean stopFlag,
+        AtomicReference<Throwable> loadError,
+        ConcurrentMap<Long, Long> threadProgressTracker
+    ) {
+        GridTestUtils.runAsync(new Runnable() {
+            @Override public void run() {
+                ThreadLocalRandom r = ThreadLocalRandom.current();
+
+                IgniteCache<Integer, String> cache = ig.cache(cacheName);
+
+                try {
+                    while (!stopFlag.get()) {
+                        try {
+                            int op = r.nextInt(3);
+
+                            switch (op) {
+                                case 0:
+                                    byte[] randBytes = new byte[r.nextInt(10, 100)];
+
+                                    cache.put(r.nextInt(ENTRIES), new String(randBytes));
+
+                                    break;
+                                case 1:
+                                    cache.remove(r.nextInt(ENTRIES));
+
+                                    break;
+                                case 2:
+                                    cache.get(r.nextInt(ENTRIES));
+
+                                    break;
+                            }
+
+                            threadProgressTracker.compute(Thread.currentThread().getId(),
+                                (tId, ops) -> ops == null ? 1 : ops + 1);
+                        }
+                        catch (CacheException e) {
+                            if (e.getCause() instanceof ClusterTopologyException)
+                                ((ClusterTopologyException)e.getCause()).retryReadyFuture().get();
+                        }
+                        catch (ClusterTopologyException e) {
+                            e.retryReadyFuture().get();
+                        }
+                    }
+                }
+                catch (Throwable t) {
+                    loadError.compareAndSet(null, t);
+
+                    stopFlag.set(true);
+                }
+            }
+        });
+    }
+
+    /**
+     * @param ig Ignite instance.
+     * @param cacheName Cache name.
+     * @param stopFlag Stop flag.
+     * @param loadError Load error reference.
+     * @param threadProgressTracker Progress tracker.
+     */
+    private void startTxLoadThread(
+        IgniteEx ig,
+        String cacheName,
+        AtomicBoolean stopFlag,
+        AtomicReference<Throwable> loadError,
+        ConcurrentMap<Long, Long> threadProgressTracker
+    ) {
+        GridTestUtils.runAsync(new Runnable() {
+            @Override public void run() {
+                ThreadLocalRandom r = ThreadLocalRandom.current();
+
+                IgniteCache<Integer, String> cache = ig.cache(cacheName);
+
+                boolean pessimistic = r.nextBoolean();
+
+                boolean rollback = r.nextBoolean();
+
+                try {
+                    while (!stopFlag.get()) {
+                        try (Transaction tx = ig.transactions().txStart(
+                            pessimistic ? TransactionConcurrency.PESSIMISTIC : TransactionConcurrency.OPTIMISTIC,
+                            TransactionIsolation.REPEATABLE_READ
+                        )) {
+                            int key1 = -1;
+                            String val1 = null;
+                            while (val1 == null) {
+                                key1 = r.nextInt(ENTRIES);
+                                val1 = cache.get(key1);
+                            }
+
+                            int key2 = -1;
+                            String val2 = null;
+                            while (val2 == null) {
+                                key2 = r.nextInt(ENTRIES);
+                                val2 = cache.get(key2);
+                            }
+
+                            cache.put(key1, val2);
+                            cache.put(key2, val1);
+
+                            if (rollback)
+                                tx.rollback();
+                            else
+                                tx.commit();
+
+                            threadProgressTracker.compute(Thread.currentThread().getId(),
+                                (tId, ops) -> ops == null ? 1 : ops + 1);
+                        }
+                        catch (CacheException e) {
+                            if (e.getCause() instanceof ClusterTopologyException)
+                                ((ClusterTopologyException)e.getCause()).retryReadyFuture().get();
+                        }
+                        catch (ClusterTopologyException e) {
+                            e.retryReadyFuture().get();
+                        }
+                    }
+                }
+                catch (Throwable t) {
+                    loadError.compareAndSet(null, t);
+
+                    stopFlag.set(true);
+                }
+            }
+        });
+    }
+
+    /**
+     * @param ig Ignite instance.
+     * @param cacheName1 Cache name 1.
+     * @param cacheName2 Cache name 2.
+     * @param stopFlag Stop flag.
+     * @param loadError Load error reference.
+     * @param threadProgressTracker Progress tracker.
+     */
+    private void startCrossCacheTxLoadThread(
+        IgniteEx ig,
+        String cacheName1,
+        String cacheName2,
+        AtomicBoolean stopFlag,
+        AtomicReference<Throwable> loadError,
+        ConcurrentMap<Long, Long> threadProgressTracker
+    ) {
+        GridTestUtils.runAsync(new Runnable() {
+            @Override public void run() {
+                ThreadLocalRandom r = ThreadLocalRandom.current();
+
+                IgniteCache<Integer, String> cache1 = ig.cache(cacheName1);
+                IgniteCache<Integer, String> cache2 = ig.cache(cacheName2);
+
+                boolean pessimistic = r.nextBoolean();
+
+                boolean rollback = r.nextBoolean();
+
+                try {
+                    while (!stopFlag.get()) {
+                        try (Transaction tx = ig.transactions().txStart(
+                            pessimistic ? TransactionConcurrency.PESSIMISTIC : TransactionConcurrency.OPTIMISTIC,
+                            TransactionIsolation.REPEATABLE_READ
+                        )) {
+                            int key1 = -1;
+                            String val1 = null;
+                            while (val1 == null) {
+                                key1 = r.nextInt(ENTRIES);
+                                val1 = cache1.get(key1);
+                            }
+
+                            int key2 = -1;
+                            String val2 = null;
+                            while (val2 == null) {
+                                key2 = r.nextInt(ENTRIES);
+                                val2 = cache2.get(key2);
+                            }
+
+                            cache1.put(key1, val2);
+                            cache2.put(key2, val1);
+
+                            if (rollback)
+                                tx.rollback();
+                            else
+                                tx.commit();
+
+                            threadProgressTracker.compute(Thread.currentThread().getId(),
+                                (tId, ops) -> ops == null ? 1 : ops + 1);
+                        }
+                        catch (CacheException e) {
+                            if (e.getCause() instanceof ClusterTopologyException)
+                                ((ClusterTopologyException)e.getCause()).retryReadyFuture().get();
+                        }
+                        catch (ClusterTopologyException e) {
+                            e.retryReadyFuture().get();
+                        }
+                    }
+                }
+                catch (Throwable t) {
+                    loadError.compareAndSet(null, t);
+
+                    stopFlag.set(true);
+                }
+            }
+        });
+    }
+
+    /**
+     * @param waitMs Wait milliseconds.
+     * @param loadError Load error.
+     * @param threadProgressTracker Thread progress tracker.
+     */
+    private void awaitProgressInAllLoaders(
+        long waitMs,
+        AtomicReference<Throwable> loadError,
+        ConcurrentMap<Long, Long> threadProgressTracker
+    ) throws Exception {
+        Map<Long, Long> view1 = new HashMap<>(threadProgressTracker);
+
+        long startTs = U.currentTimeMillis();
+
+        while (U.currentTimeMillis() < startTs + waitMs) {
+            Map<Long, Long> view2 = new HashMap<>(threadProgressTracker);
+
+            if (loadError.get() != null) {
+                loadError.get().printStackTrace();
+
+                fail("Unexpected error in load thread: " + loadError.get().toString());
+            }
+
+            boolean frozenThreadExists = false;
+
+            for (Map.Entry<Long, Long> entry : view1.entrySet()) {
+                if (entry.getValue().equals(view2.get(entry.getKey())))
+                    frozenThreadExists = true;
+            }
+
+            if (!frozenThreadExists)
+                return;
+
+            U.sleep(100);
+        }
+
+        fail("No progress in load thread");
+    }
+
+    /**
+     * Accepts all nodes except one with specified consistent ID.
+     */
+    private static class ConsistentIdNodeFilter implements IgnitePredicate<ClusterNode> {
+        /** Consistent ID. */
+        private final Serializable consId0;
+
+        /**
+         * @param consId0 Consistent ID.
+         */
+        public ConsistentIdNodeFilter(Serializable consId0) {
+            this.consId0 = consId0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode node) {
+            return !node.consistentId().equals(consId0);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 40bcf8c3f07d0..4e7d8e323e559 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
 import org.apache.ignite.internal.processors.cache.persistence.LocalWalModeChangeDuringRebalancingSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAbsentEvictionNodeOutOfBaselineTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.ClientAffinityAssignmentWithBaselineTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAllBaselineNodesOnlineFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOfflineBaselineNodeFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest;
@@ -71,6 +72,7 @@ public static TestSuite suite() {
         suite.addTestSuite(IgniteAllBaselineNodesOnlineFullApiSelfTest.class);
         suite.addTestSuite(IgniteOfflineBaselineNodeFullApiSelfTest.class);
         suite.addTestSuite(IgniteOnlineNodeOutOfBaselineFullApiSelfTest.class);
+        suite.addTestSuite(ClientAffinityAssignmentWithBaselineTest.class);
         suite.addTestSuite(IgniteAbsentEvictionNodeOutOfBaselineTest.class);
 
         return suite;

From dc1cc39ffc0cdb2abc2767852883d6c8307cf9ea Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Thu, 31 May 2018 16:06:52 +0300
Subject: [PATCH 165/543] IGNITE-8530 fixed onNodeLeft for
 InitNewCoordinatorFuture - Fixes #4086.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 49fe8cd)
---
 .../distributed/dht/preloader/InitNewCoordinatorFuture.java   | 3 +++
 .../java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java  | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
index 42a9ba6891955..5909a05cb6098 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/InitNewCoordinatorFuture.java
@@ -338,6 +338,9 @@ public void onNodeLeft(UUID nodeId) {
 
         synchronized (this) {
             done = awaited.remove(nodeId) && awaited.isEmpty();
+
+            if (done)
+                onAllReceived();
         }
 
         if (done)
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 6b2293a1d275c..35e945208eb65 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -4212,10 +4212,10 @@ else if (!locNodeId.equals(node.id()) && ring.node(node.id()) != null) {
 
                     DiscoveryDataPacket dataPacket = msg.gridDiscoveryData();
 
-                    dataPacket.joiningNodeClient(msg.client());
-
                     assert dataPacket != null : msg;
 
+                    dataPacket.joiningNodeClient(msg.client());
+
                     if (dataPacket.hasJoiningNodeData())
                         spi.onExchange(dataPacket, U.resolveClassLoader(spi.ignite().configuration()));
 

From 383f3ad52d20045477fe4fed61d1d56e97ece35d Mon Sep 17 00:00:00 2001
From: dgladkikh <dgladkikh@gridgain.com>
Date: Fri, 1 Jun 2018 18:34:50 +0300
Subject: [PATCH 166/543] IGNITE-8603 Add JMX-metric to cluster: baseline nodes
 - Fixes #4060.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 1f6266c)
---
 .../ClusterLocalNodeMetricsMXBeanImpl.java    |  21 +++
 .../internal/ClusterMetricsMXBeanImpl.java    |  32 ++++
 .../ignite/mxbean/ClusterMetricsMXBean.java   |  16 ++
 .../ClusterBaselineNodesMetricsSelfTest.java  | 178 ++++++++++++++++++
 .../IgniteComputeGridTestSuite.java           |   2 +
 5 files changed, 249 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/ClusterLocalNodeMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/ClusterLocalNodeMetricsMXBeanImpl.java
index a242345df0fe5..eed501ae65444 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/ClusterLocalNodeMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/ClusterLocalNodeMetricsMXBeanImpl.java
@@ -18,9 +18,11 @@
 package org.apache.ignite.internal;
 
 import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.UUID;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -332,6 +334,25 @@ public ClusterLocalNodeMetricsMXBeanImpl(GridDiscoveryManager discoMgr) {
         return node.metrics().getTotalNodes();
     }
 
+    /** {@inheritDoc} */
+    @Override public int getTotalBaselineNodes() {
+        if (!node.isClient() && !node.isDaemon()) {
+            List<? extends BaselineNode> baselineNodes = discoMgr.baselineNodes(discoMgr.topologyVersionEx());
+
+            if (baselineNodes != null)
+                for (BaselineNode baselineNode : baselineNodes)
+                    if (baselineNode.consistentId().equals(node.consistentId()))
+                        return 1;
+        }
+
+        return 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getActiveBaselineNodes() {
+        return getTotalBaselineNodes();
+    }
+
     /** {@inheritDoc} */
     @Override public int getTotalServerNodes() {
         return !node.isClient() ? 1 : 0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/ClusterMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/ClusterMetricsMXBeanImpl.java
index e09ad3c110dc9..1efb5906f3472 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/ClusterMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/ClusterMetricsMXBeanImpl.java
@@ -18,10 +18,13 @@
 package org.apache.ignite.internal;
 
 import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
 import java.util.UUID;
+import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
@@ -358,6 +361,35 @@ private ClusterMetrics metrics() {
         return metrics().getTotalNodes();
     }
 
+    /** {@inheritDoc} */
+    @Override public int getTotalBaselineNodes() {
+        Collection<BaselineNode> baselineNodes = cluster.ignite().cluster().currentBaselineTopology();
+
+        return baselineNodes != null ? baselineNodes.size() : 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getActiveBaselineNodes() {
+        Collection<BaselineNode> baselineNodes = cluster.ignite().cluster().currentBaselineTopology();
+
+        if (baselineNodes != null && !baselineNodes.isEmpty()) {
+            Set<Object> bltIds = new HashSet<>(baselineNodes.size());
+
+            for (BaselineNode baselineNode : baselineNodes)
+                bltIds.add(baselineNode.consistentId());
+
+            int count = 0;
+
+            for (ClusterNode node : cluster.forServers().nodes())
+                if (bltIds.contains(node.consistentId()))
+                    count++;
+
+            return count;
+        }
+
+        return 0;
+    }
+
     /** {@inheritDoc} */
     @Override public int getTotalServerNodes() {
         return cluster.forServers().nodes().size();
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/ClusterMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/ClusterMetricsMXBean.java
index 6385604e783f1..537cef7dc45da 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/ClusterMetricsMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/ClusterMetricsMXBean.java
@@ -257,6 +257,22 @@ public interface ClusterMetricsMXBean extends ClusterMetrics {
     @MXBeanDescription("Total number of nodes.")
     public int getTotalNodes();
 
+    /**
+     * Get count of total baseline nodes.
+     *
+     * @return Count of total baseline nodes.
+     */
+    @MXBeanDescription("Total baseline nodes count.")
+    public int getTotalBaselineNodes();
+
+    /**
+     * Get count of active baseline nodes.
+     *
+     * @return Count of active baseline nodes.
+     */
+    @MXBeanDescription("Active baseline nodes count.")
+    public int getActiveBaselineNodes();
+
     /**
      * Get count of server nodes.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
new file mode 100644
index 0000000000000..565317720d2a9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.lang.management.ManagementFactory;
+import java.util.Collection;
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.mxbean.ClusterMetricsMXBean;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.common.GridCommonTest;
+
+/**
+ * Baseline nodes metrics self test.
+ */
+@GridCommonTest(group = "Kernal Self")
+public class ClusterBaselineNodesMetricsSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBaselineNodes() throws Exception {
+        // Start 2 server nodes.
+        IgniteEx ignite0 = startGrid(0);
+        startGrid(1);
+
+        // Cluster metrics.
+        ClusterMetricsMXBean mxBeanCluster = mxBean(0, ClusterMetricsMXBeanImpl.class);
+
+        ignite0.cluster().active(true);
+
+        // Added 2 server nodes to baseline.
+        resetBlt();
+
+        // Add server node outside of the baseline.
+        startGrid(2);
+
+        // Start client node.
+        Ignition.setClientMode(true);
+        startGrid(3);
+        Ignition.setClientMode(false);
+
+        Collection<BaselineNode> baselineNodes;
+
+        // State #0: 3 server nodes (2 total baseline nodes, 2 active baseline nodes), 1 client node
+        log.info(String.format(">>> State #0: topology version = %d", ignite0.cluster().topologyVersion()));
+
+        assertEquals(3, mxBeanCluster.getTotalServerNodes());
+        assertEquals(1, mxBeanCluster.getTotalClientNodes());
+        assertEquals(2, mxBeanCluster.getTotalBaselineNodes());
+        assertEquals(2, mxBeanCluster.getActiveBaselineNodes());
+        assertEquals(2, (baselineNodes = ignite0.cluster().currentBaselineTopology()) != null
+            ? baselineNodes.size()
+            : 0);
+
+        stopGrid(1, true);
+
+        // State #1: 2 server nodes (2 total baseline nodes, 1 active baseline node), 1 client node
+        log.info(String.format(">>> State #1: topology version = %d", ignite0.cluster().topologyVersion()));
+
+        assertEquals(2, mxBeanCluster.getTotalServerNodes());
+        assertEquals(1, mxBeanCluster.getTotalClientNodes());
+        assertEquals(2, mxBeanCluster.getTotalBaselineNodes());
+        assertEquals(1, mxBeanCluster.getActiveBaselineNodes());
+        assertEquals(2, (baselineNodes = ignite0.cluster().currentBaselineTopology()) != null
+            ? baselineNodes.size()
+            : 0);
+
+        startGrid(1);
+
+        ClusterMetricsMXBean mxBeanLocalNode1 = mxBean(1, ClusterLocalNodeMetricsMXBeanImpl.class);
+
+        // State #2: 3 server nodes (2 total baseline nodes, 2 active baseline nodes), 1 client node
+        log.info(String.format(">>> State #2: topology version = %d", ignite0.cluster().topologyVersion()));
+
+        assertEquals(3, mxBeanCluster.getTotalServerNodes());
+        assertEquals(1, mxBeanCluster.getTotalClientNodes());
+        assertEquals(2, mxBeanCluster.getTotalBaselineNodes());
+        assertEquals(2, mxBeanCluster.getActiveBaselineNodes());
+        assertEquals(1, mxBeanLocalNode1.getTotalBaselineNodes());
+        assertEquals(2, (baselineNodes = ignite0.cluster().currentBaselineTopology()) != null
+            ? baselineNodes.size()
+            : 0);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(name);
+
+        cfg.setConsistentId(name);
+
+        String storePath = getClass().getSimpleName().toLowerCase() + "/" + getName();
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setWalMode(WALMode.LOG_ONLY)
+                .setStoragePath(storePath)
+                .setWalPath(storePath + "/wal")
+                .setWalArchivePath(storePath + "/archive")
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setMaxSize(2L * 1024 * 1024 * 1024)
+                )
+        );
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void resetBlt() throws Exception {
+        resetBaselineTopology();
+
+        waitForRebalancing();
+        awaitPartitionMapExchange();
+    }
+
+    /**
+     * Gets ClusterMetricsMXBean for given node.
+     *
+     * @param nodeIdx Node index.
+     * @param clazz Class of ClusterMetricsMXBean implementation.
+     * @return MBean instance.
+     */
+    private ClusterMetricsMXBean mxBean(int nodeIdx, Class<? extends ClusterMetricsMXBean> clazz)
+        throws MalformedObjectNameException {
+
+        ObjectName mbeanName = U.makeMBeanName(
+            getTestIgniteInstanceName(nodeIdx),
+            "Kernal",
+            clazz.getSimpleName());
+
+        MBeanServer mbeanSrv = ManagementFactory.getPlatformMBeanServer();
+
+        if (!mbeanSrv.isRegistered(mbeanName))
+            fail("MBean is not registered: " + mbeanName.getCanonicalName());
+
+        return MBeanServerInvocationHandler.newProxyInstance(mbeanSrv, mbeanName, ClusterMetricsMXBean.class, true);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index 14eb296424a47..bab7099e3ebe5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest;
 import org.apache.ignite.internal.ClusterNodeMetricsSelfTest;
 import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest;
 import org.apache.ignite.internal.GridAffinityNoCacheSelfTest;
@@ -124,6 +125,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridTaskInstanceExecutionSelfTest.class);
         suite.addTestSuite(ClusterNodeMetricsSelfTest.class);
         suite.addTestSuite(ClusterNodeMetricsUpdateTest.class);
+        suite.addTestSuite(ClusterBaselineNodesMetricsSelfTest.class);
         suite.addTestSuite(GridNonHistoryMetricsSelfTest.class);
         suite.addTestSuite(GridCancelledJobsMetricsSelfTest.class);
         suite.addTestSuite(GridCollisionJobsContextSelfTest.class);

From a716bbee641a123f313325140ded10dc0c2de9ba Mon Sep 17 00:00:00 2001
From: Sergey Skudnov <serg.skudnov@gmail.com>
Date: Mon, 14 May 2018 13:35:14 +0300
Subject: [PATCH 167/543] IGNITE-8138 Uptime output with days - Fixes #3775.
 Cherry-picked from dfb0b9ee35afeb6adc546160c37b08a85d869f59 Signed-off-by:
 dpavlov <dpavlov@apache.org>

---
 .../java/org/apache/ignite/internal/IgniteKernal.java  | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 0d0abfc6ac237..38a19875b648f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -492,7 +492,7 @@ public IgniteKernal(@Nullable GridSpringResourceContext rsrcCtx) {
 
     /** {@inheritDoc} */
     @Override public String getUpTimeFormatted() {
-        return X.timeSpan2HMSM(U.currentTimeMillis() - startTime);
+        return X.timeSpan2DHMSM(U.currentTimeMillis() - startTime);
     }
 
     /** {@inheritDoc} */
@@ -2327,10 +2327,10 @@ else if (state == STARTING)
 
                 if (!errOnStop)
                     U.quiet(false, "Ignite node stopped OK [" + nodeName + "uptime=" +
-                        X.timeSpan2HMSM(U.currentTimeMillis() - startTime) + ']');
+                        X.timeSpan2DHMSM(U.currentTimeMillis() - startTime) + ']');
                 else
                     U.quiet(true, "Ignite node stopped wih ERRORS [" + nodeName + "uptime=" +
-                        X.timeSpan2HMSM(U.currentTimeMillis() - startTime) + ']');
+                        X.timeSpan2DHMSM(U.currentTimeMillis() - startTime) + ']');
             }
 
             if (log.isInfoEnabled())
@@ -2345,7 +2345,7 @@ else if (state == STARTING)
                         ">>> " + ack + NL +
                         ">>> " + dash + NL +
                         (igniteInstanceName == null ? "" : ">>> Ignite instance name: " + igniteInstanceName + NL) +
-                        ">>> Grid uptime: " + X.timeSpan2HMSM(U.currentTimeMillis() - startTime) +
+                        ">>> Grid uptime: " + X.timeSpan2DHMSM(U.currentTimeMillis() - startTime) +
                         NL +
                         NL);
                 }
@@ -2359,7 +2359,7 @@ else if (state == STARTING)
                         ">>> " + ack + NL +
                         ">>> " + dash + NL +
                         (igniteInstanceName == null ? "" : ">>> Ignite instance name: " + igniteInstanceName + NL) +
-                        ">>> Grid uptime: " + X.timeSpan2HMSM(U.currentTimeMillis() - startTime) +
+                        ">>> Grid uptime: " + X.timeSpan2DHMSM(U.currentTimeMillis() - startTime) +
                         NL +
                         ">>> See log above for detailed error message." + NL +
                         ">>> Note that some errors during stop can prevent grid from" + NL +

From 9c381fd3ef0f75e2fed4562f562904ce5ede6bc3 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 10 May 2018 14:56:35 +0300
Subject: [PATCH 168/543] IGNITE-8424 Add directory sorting and skip duplicated
 config file in order to consistently start nodes.

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit bec3e9b)
---
 .../file/FilePageStoreManager.java            |  21 ++-
 ...tePdsDuplicatedCacheConfigurationTest.java | 161 ++++++++++++++++++
 .../ignite/testsuites/IgnitePdsTestSuite.java |   2 +
 3 files changed, 182 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDuplicatedCacheConfigurationTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index d065ff1e1e54b..e72a0f644bdf1 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -30,6 +30,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -646,6 +647,8 @@ else if (lockF.exists()) {
 
         Map<String, StoredCacheData> ccfgs = new HashMap<>();
 
+        Arrays.sort(files);
+
         for (File file : files) {
             if (file.isDirectory()) {
                 if (file.getName().startsWith(CACHE_DIR_PREFIX)) {
@@ -654,7 +657,14 @@ else if (lockF.exists()) {
                     if (conf.exists() && conf.length() > 0) {
                         StoredCacheData cacheData = readCacheData(conf);
 
-                        ccfgs.put(cacheData.config().getName(), cacheData);
+                        String cacheName = cacheData.config().getName();
+
+                        if (!ccfgs.containsKey(cacheName))
+                            ccfgs.put(cacheName, cacheData);
+                        else {
+                            U.warn(log, "Cache with name=" + cacheName + " is already registered, skipping config file "
+                                    + file.getName());
+                        }
                     }
                 }
                 else if (file.getName().startsWith(CACHE_GRP_DIR_PREFIX))
@@ -680,7 +690,14 @@ private void readCacheGroupCaches(File grpDir, Map<String, StoredCacheData> ccfg
             if (!file.isDirectory() && file.getName().endsWith(CACHE_DATA_FILENAME) && file.length() > 0) {
                 StoredCacheData cacheData = readCacheData(file);
 
-                ccfgs.put(cacheData.config().getName(), cacheData);
+                String cacheName = cacheData.config().getName();
+
+                if (!ccfgs.containsKey(cacheName))
+                    ccfgs.put(cacheName, cacheData);
+                else {
+                    U.warn(log, "Cache with name=" + cacheName + " is already registered, skipping config file "
+                            + file.getName() + " in group directory " + grpDir.getName());
+                }
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDuplicatedCacheConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDuplicatedCacheConfigurationTest.java
new file mode 100644
index 0000000000000..66459bd620d1f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDuplicatedCacheConfigurationTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.StoredCacheData;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests that ignite can start when caches' configurations with same name in different groups stored.
+ */
+public class IgnitePdsDuplicatedCacheConfigurationTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int CACHES = 4;
+
+    /** */
+    private static final int NODES = 4;
+
+    /** */
+    private static final String ODD_GROUP_NAME = "group-odd";
+
+    /** */
+    private static final String EVEN_GROUP_NAME = "group-even";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        return cfg.setDiscoverySpi(new TcpDiscoverySpi()
+                .setIpFinder(IP_FINDER))
+                .setDataStorageConfiguration(new DataStorageConfiguration()
+                        .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                                .setMaxSize(200 * 1024 * 1024)
+                                .setPersistenceEnabled(true)));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        super.afterTest();
+    }
+
+    /**
+     * Tests that ignite can start when caches' configurations with same name in different groups stored.
+     *
+     * @throws Exception If fails.
+     */
+    public void testStartDuplicatedCacheConfigurations() throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(NODES);
+
+        ig0.cluster().active(true);
+
+        startCaches(ig0);
+
+        DynamicCacheDescriptor desc = ig0.context().cache().cacheDescriptor(cacheName(3));
+
+        storeInvalidCacheData(desc);
+
+        stopAllGrids();
+
+        startGrids(NODES);
+
+        desc = ig0.context().cache().cacheDescriptor(cacheName(3));
+
+        assertEquals("expected that group of " + cacheName(3) + " is " + EVEN_GROUP_NAME, EVEN_GROUP_NAME,
+                desc.groupDescriptor().groupName());
+
+    }
+
+    /**
+     * Store cache descriptor to PDS with invalid group name.
+     *
+     * @param cacheDescr Cache descr.
+     * @throws IgniteCheckedException If fails.
+     */
+    private void storeInvalidCacheData(DynamicCacheDescriptor cacheDescr) throws IgniteCheckedException {
+        for (int i = 0; i < NODES; i++) {
+            IgniteEx ig = grid(i);
+
+            GridCacheSharedContext sharedCtx = ig.context().cache().context();
+
+            FilePageStoreManager pageStore = (FilePageStoreManager) sharedCtx.pageStore();
+
+            StoredCacheData corrData = cacheDescr.toStoredData();
+
+            corrData.config().setGroupName(ODD_GROUP_NAME);
+
+            pageStore.storeCacheData(corrData, true);
+        }
+    }
+
+    /**
+     * @param ignite Ignite instance.
+     */
+    private void startCaches(Ignite ignite) {
+        List<CacheConfiguration> ccfg = new ArrayList<>(CACHES);
+
+        for (int i = 0; i < CACHES; i++) {
+            ccfg.add(new CacheConfiguration<>(cacheName(i))
+                    .setGroupName(i % 2 == 0 ? ODD_GROUP_NAME : EVEN_GROUP_NAME)
+                    .setBackups(1)
+                    .setAffinity(new RendezvousAffinityFunction(false, 32)));
+        }
+
+        ignite.createCaches(ccfg);
+    }
+
+    /**
+     * Generate cache name from idx.
+     *
+     * @param idx Index.
+     */
+    private String cacheName(int idx) {
+        return "cache" + idx;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index e4c59b3c5109d..43ce04f4b7aae 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheWithoutCheckpointsTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDuplicatedCacheConfigurationTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest;
@@ -120,6 +121,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
 
         suite.addTestSuite(IgnitePdsDestroyCacheTest.class);
         suite.addTestSuite(IgnitePdsDestroyCacheWithoutCheckpointsTest.class);
+        suite.addTestSuite(IgnitePdsDuplicatedCacheConfigurationTest.class);
 
         suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
 

From dc8ba0932a944f3e966ebbe2af053488d1fe9b28 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Fri, 4 May 2018 15:26:02 +0300
Subject: [PATCH 169/543] IGNITE-8421 new implementation of getChildren method
 is added tolerating KeeperException.NoNodeException - Fixes #3939.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry-picked from commit #02e9ca993178d4aa648d06cb93ce1a9277eb22b1)
---
 .../ZkDistributedCollectDataFuture.java       |  2 +-
 .../zk/internal/ZookeeperClient.java          | 25 +++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
index 174d698fe27e4..e9b28e192216c 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDistributedCollectDataFuture.java
@@ -150,7 +150,7 @@ static void deleteFutureData(ZookeeperClient client,
 
         try {
             client.deleteAll(evtDir,
-                client.getChildren(evtDir),
+                client.getChildrenIfPathExists(evtDir),
                 -1);
         }
         catch (KeeperException.NoNodeException e) {
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
index cc525d31eb7ee..6cc77a5875eaa 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -489,6 +489,31 @@ List<String> getChildren(String path)
         }
     }
 
+    /**
+     * @param path Path.
+     * @return Children nodes.
+     * @throws KeeperException.NoNodeException If provided path does not exist.
+     * @throws ZookeeperClientFailedException If connection to zk was lost.
+     * @throws InterruptedException If interrupted.
+     */
+    List<String> getChildrenIfPathExists(String path) throws
+        KeeperException.NoNodeException, InterruptedException, ZookeeperClientFailedException {
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                return zk.getChildren(path, false);
+            }
+            catch (KeeperException.NoNodeException e) {
+                throw e;
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+        }
+    }
+
+
     /**
      * @param path Path.
      * @throws InterruptedException If interrupted.

From 0d37bd6749829ce07fbd4b0d2344b62588177628 Mon Sep 17 00:00:00 2001
From: Ivan Rakov <irakov@apache.org>
Date: Tue, 5 Jun 2018 14:42:43 +0300
Subject: [PATCH 170/543] IGNITE-8682 Attempt to configure IGFS in persistent
 mode without specific data region ends with AssertionError

(cherry picked from commit 2fe0a10)
---
 .../processors/cache/GridCacheProcessor.java         |  9 ++++++---
 .../internal/processors/cache/GridCacheUtils.java    | 12 +++++++++++-
 .../IgniteCacheDatabaseSharedManager.java            |  2 +-
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 8b63cdd1fc8a9..aca3f2dbdaa37 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2025,22 +2025,25 @@ private CacheGroupContext startCacheGroup(
 
         String memPlcName = cfg.getDataRegionName();
 
-        DataRegion memPlc = sharedCtx.database().dataRegion(memPlcName);
+        DataRegion dataRegion = sharedCtx.database().dataRegion(memPlcName);
         FreeList freeList = sharedCtx.database().freeList(memPlcName);
         ReuseList reuseList = sharedCtx.database().reuseList(memPlcName);
 
+        boolean persistenceEnabled = sharedCtx.localNode().isClient() ? desc.persistenceEnabled() :
+            dataRegion != null && dataRegion.config().isPersistenceEnabled();
+
         CacheGroupContext grp = new CacheGroupContext(sharedCtx,
             desc.groupId(),
             desc.receivedFrom(),
             cacheType,
             cfg,
             affNode,
-            memPlc,
+            dataRegion,
             cacheObjCtx,
             freeList,
             reuseList,
             exchTopVer,
-            desc.persistenceEnabled(),
+            persistenceEnabled,
             desc.walEnabled()
         );
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index bb64cc6f2e73e..4f349a4b5c385 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -119,6 +119,7 @@
 import static org.apache.ignite.configuration.CacheConfiguration.DFLT_CACHE_MODE;
 import static org.apache.ignite.internal.GridTopic.TOPIC_REPLICATION;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.READ;
+import static org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager.SYSTEM_DATA_REGION_NAME;
 
 /**
  * Cache utility methods.
@@ -1800,7 +1801,7 @@ public static boolean isPersistentCache(CacheConfiguration ccfg, DataStorageConf
             return false;
 
         // Special handling for system cache is needed.
-        if (isSystemCache(ccfg.getName())) {
+        if (isSystemCache(ccfg.getName()) || isIgfsCacheInSystemRegion(ccfg)) {
             if (dsCfg.getDefaultDataRegionConfiguration().isPersistenceEnabled())
                 return true;
 
@@ -1829,6 +1830,15 @@ public static boolean isPersistentCache(CacheConfiguration ccfg, DataStorageConf
         return false;
     }
 
+    /**
+     * Checks whether cache configuration represents IGFS cache that will be placed in system memory region.
+     *
+     * @param ccfg Cache config.
+     */
+    private static boolean isIgfsCacheInSystemRegion(CacheConfiguration ccfg) {
+        return IgfsUtils.matchIgfsCacheName(ccfg.getName()) &&
+            (SYSTEM_DATA_REGION_NAME.equals(ccfg.getDataRegionName()) || ccfg.getDataRegionName() == null);
+    }
 
     /**
      * @return {@code true} if persistence is enabled for at least one data region, {@code false} if not.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 5e8f40706e984..e3ce04d8809ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -76,7 +76,7 @@
 public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdapter
     implements IgniteChangeGlobalStateSupport, CheckpointLockStateChecker {
     /** DataRegionConfiguration name reserved for internal caches. */
-    static final String SYSTEM_DATA_REGION_NAME = "sysMemPlc";
+    public static final String SYSTEM_DATA_REGION_NAME = "sysMemPlc";
 
     /** Minimum size of memory chunk */
     private static final long MIN_PAGE_MEMORY_SIZE = 10 * 1024 * 1024;

From 0c20c51025e5a550b7f3d8458acae27ec8cdcef5 Mon Sep 17 00:00:00 2001
From: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Date: Tue, 5 Jun 2018 18:49:19 +0300
Subject: [PATCH 171/543] IGNITE-8602 Add support filter "label null" for
 control.sh tx utility - Fixes #4073.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 3372590)
---
 .../apache/ignite/internal/visor/tx/VisorTxTask.java  |  2 +-
 .../apache/ignite/util/GridCommandHandlerTest.java    | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
index b411e29006757..5a7ffdd2abbca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
@@ -173,7 +173,7 @@ private VisorTxJob(VisorTxTaskArg arg, boolean debug) {
                 if (arg.getMinSize() != null && locTx.size() < arg.getMinSize())
                     continue;
 
-                if (lbMatch != null && (locTx.label() == null || !lbMatch.matcher(locTx.label()).matches()))
+                if (lbMatch != null && !lbMatch.matcher(locTx.label() == null ? "null" : locTx.label()).matches())
                     continue;
 
                 Collection<UUID> mappings = new ArrayList<>();
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 385e79b3926fa..f78f46626f2c7 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -445,6 +445,17 @@ else if (entry.getKey().equals(node2)) {
             }
         }, "--tx", "label", "^label[0-9]");
 
+        // Test filter by empty label.
+        validate(h, map -> {
+            VisorTxTaskResult res = map.get(grid(0).localNode());
+
+            for (VisorTxInfo info:res.getInfos()){
+                assertNull(info.getLabel());
+
+            }
+
+        }, "--tx", "label", "null");
+
         // test order by size.
         validate(h, map -> {
             VisorTxTaskResult res = map.get(grid(0).localNode());

From 455777d2cd2bd6b83d69dbdbbd7e95b4f375bb75 Mon Sep 17 00:00:00 2001
From: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Date: Tue, 5 Jun 2018 19:01:03 +0300
Subject: [PATCH 172/543] IGNITE-8467 Fixed filter minSize for transactions
 utility control.sh. Fixes #4069

(cherry picked from commit d61c068)
---
 .../ignite/internal/visor/tx/VisorTxTask.java     |  6 +++---
 .../ignite/util/GridCommandHandlerTest.java       | 15 +++++++++++++++
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
index 5a7ffdd2abbca..579abbe9261a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
@@ -170,9 +170,6 @@ private VisorTxJob(VisorTxTaskArg arg, boolean debug) {
                     duration < arg.getMinDuration())
                     continue;
 
-                if (arg.getMinSize() != null && locTx.size() < arg.getMinSize())
-                    continue;
-
                 if (lbMatch != null && !lbMatch.matcher(locTx.label() == null ? "null" : locTx.label()).matches())
                     continue;
 
@@ -194,6 +191,9 @@ private VisorTxJob(VisorTxTaskArg arg, boolean debug) {
                     }
                 }
 
+                if (arg.getMinSize() != null && size < arg.getMinSize())
+                    continue;
+
                 infos.add(new VisorTxInfo(locTx.xid(), duration, locTx.isolation(), locTx.concurrency(),
                     locTx.timeout(), locTx.label(), mappings, locTx.state(), size));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index f78f46626f2c7..3ec0617e9164a 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -456,6 +456,21 @@ else if (entry.getKey().equals(node2)) {
 
         }, "--tx", "label", "null");
 
+
+        // test check minSize
+        int minSize=10;
+
+        validate(h, map -> {
+            VisorTxTaskResult res = map.get(grid(0).localNode());
+
+            assertNotNull(res);
+
+            for (VisorTxInfo txInfo : res.getInfos()) {
+                assertTrue(txInfo.getSize() >= minSize);
+
+            }
+        }, "--tx", "minSize", Integer.toString(minSize));
+
         // test order by size.
         validate(h, map -> {
             VisorTxTaskResult res = map.get(grid(0).localNode());

From c40f7659f24b6037485a5ac5e1965b0c6bd573bb Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Wed, 6 Jun 2018 11:19:26 +0300
Subject: [PATCH 173/543] IGNITE-8587 GridToStringBuilder use ConcurrentHashMap
 to avoid global locks on classCache - Fixes #4059.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit ecd8261f1add62099fde39aa7dca49855a866eda)
---
 .../util/tostring/GridToStringBuilder.java    | 38 +++----------------
 1 file changed, 5 insertions(+), 33 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
index 56eef1df14813..dbc33db88c544 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
@@ -32,14 +32,13 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.EventListener;
-import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.Map;
 import java.util.Queue;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_COLLECTION_LIMIT;
@@ -81,10 +80,7 @@
  */
 public class GridToStringBuilder {
     /** */
-    private static final Map<String, GridToStringClassDescriptor> classCache = new HashMap<>();
-
-    /** */
-    private static final ReadWriteLock rwLock = new ReentrantReadWriteLock();
+    private static final Map<String, GridToStringClassDescriptor> classCache = new ConcurrentHashMap<>();
 
     /** {@link IgniteSystemProperties#IGNITE_TO_STRING_INCLUDE_SENSITIVE} */
     public static final boolean INCLUDE_SENSITIVE =
@@ -1013,16 +1009,10 @@ private static <T> String toStringImpl(
         }
         // Specifically catching all exceptions.
         catch (Exception e) {
-            rwLock.writeLock().lock();
 
             // Remove entry from cache to avoid potential memory leak
             // in case new class loader got loaded under the same identity hash.
-            try {
-                classCache.remove(cls.getName() + System.identityHashCode(cls.getClassLoader()));
-            }
-            finally {
-                rwLock.writeLock().unlock();
-            }
+            classCache.remove(cls.getName() + System.identityHashCode(cls.getClassLoader()));
 
             // No other option here.
             throw new IgniteException(e);
@@ -1733,14 +1723,7 @@ private static <T> GridToStringClassDescriptor getClassDescriptor(Class<T> cls)
 
         GridToStringClassDescriptor cd;
 
-        rwLock.readLock().lock();
-
-        try {
-            cd = classCache.get(key);
-        }
-        finally {
-            rwLock.readLock().unlock();
-        }
+        cd = classCache.get(key);
 
         if (cd == null) {
             cd = new GridToStringClassDescriptor(cls);
@@ -1802,18 +1785,7 @@ else if (!f.isAnnotationPresent(GridToStringExclude.class) &&
 
             cd.sortFields();
 
-            /*
-             * Allow multiple puts for the same class - they will simply override.
-             */
-
-            rwLock.writeLock().lock();
-
-            try {
-                classCache.put(key, cd);
-            }
-            finally {
-                rwLock.writeLock().unlock();
-            }
+            classCache.putIfAbsent(key, cd);
         }
 
         return cd;

From af026c7840101e2c57c7dd882baabddb6df2fa82 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Tue, 5 Jun 2018 01:24:37 +0300
Subject: [PATCH 174/543] IGNITE-8693 SQL JOIN between PARTITIONED and
 REPLICATED cache fails - Fixes #4120.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit bc35ce0)
---
 .../h2/twostep/GridReduceQueryExecutor.java   |  15 ++
 ...QueryJoinWithDifferentNodeFiltersTest.java | 163 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite2.java       |   3 +
 3 files changed, 181 insertions(+)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/QueryJoinWithDifferentNodeFiltersTest.java

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 20bf32c5bc0d5..cd76bc1977bc3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -452,6 +452,21 @@ private Map<ClusterNode, IntArray> stableDataNodes(boolean isReplicatedOnly, Aff
         List<Integer> cacheIds, int[] parts) {
         GridCacheContext<?, ?> cctx = cacheContext(cacheIds.get(0));
 
+        // If the first cache is not partitioned, find it (if it's present) and move it to index 0.
+        if (!cctx.isPartitioned()) {
+            for (int cacheId = 1; cacheId < cacheIds.size(); cacheId++) {
+                GridCacheContext<?, ?> currCctx = cacheContext(cacheIds.get(cacheId));
+
+                if (currCctx.isPartitioned()) {
+                    Collections.swap(cacheIds, 0, cacheId);
+
+                    cctx = currCctx;
+
+                    break;
+                }
+            }
+        }
+
         Map<ClusterNode, IntArray> map = stableDataNodesMap(topVer, cctx, parts);
 
         Set<ClusterNode> nodes = map.keySet();
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/QueryJoinWithDifferentNodeFiltersTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/QueryJoinWithDifferentNodeFiltersTest.java
new file mode 100644
index 0000000000000..47666b93cc04b
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/QueryJoinWithDifferentNodeFiltersTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class QueryJoinWithDifferentNodeFiltersTest extends GridCommonAbstractTest {
+    /** */
+    private static final String CACHE_NAME = "cache";
+
+    /** */
+    private static final String CACHE_NAME_2 = "cache2";
+
+    /** */
+    private static final int NODE_COUNT = 4;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setCacheConfiguration(
+            new CacheConfiguration<>(CACHE_NAME)
+                .setBackups(1)
+                .setCacheMode(CacheMode.REPLICATED)
+                .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+                .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+                .setIndexedTypes(Integer.class, Organization.class),
+            new CacheConfiguration<>(CACHE_NAME_2)
+                .setNodeFilter(new TestFilter())
+                .setBackups(1)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+                .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+                .setIndexedTypes(Integer.class, Person.class)
+            );
+
+        if (getTestIgniteInstanceName(0).equals(igniteInstanceName) || getTestIgniteInstanceName(1).equals(igniteInstanceName))
+            cfg.setUserAttributes(F.asMap("DATA", "true"));
+
+        if ("client".equals(igniteInstanceName))
+            cfg.setClientMode(true);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+
+        U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "snapshot", false));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "snapshot", false));
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testSize() throws Exception {
+        startGrids(NODE_COUNT);
+
+        Ignite client = startGrid("client");
+
+        client.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = client.cache(CACHE_NAME);
+        IgniteCache<Object, Object> cache2 = client.cache(CACHE_NAME_2);
+
+        int size = 100;
+
+        for (int i = 0; i < size; i++) {
+            cache.put(i, new Organization(i, "Org-" + i));
+            cache2.put(i, new Person(i, i, "Person-" + i));
+        }
+
+        info(cache2.query(new SqlFieldsQuery("select * from \"cache\".Organization r, \"cache2\".Person p where p.orgId=r.orgId")).getAll().toString());
+    }
+
+    /**
+     *
+     */
+    private static class Organization {
+        /** */
+        @SuppressWarnings("unused") @QuerySqlField(index = true) private int orgId;
+
+        /** */
+        @SuppressWarnings("unused") private String orgName;
+
+        /**
+         *
+         */
+        public Organization(int orgId, String orgName) {
+            this.orgId = orgId;
+            this.orgName = orgName;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Person {
+        /** */
+        @SuppressWarnings("unused") @QuerySqlField(index = true) private int personId;
+
+        /** */
+        @SuppressWarnings("unused") @QuerySqlField(index = true) private int orgId;
+
+        /** */
+        @SuppressWarnings("unused") private String name;
+
+        /**
+         *
+         */
+        public Person(int personId, int orgId, String name) {
+            this.personId = personId;
+            this.orgId = orgId;
+            this.name = name;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestFilter implements IgnitePredicate<ClusterNode> {
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode clusterNode) {
+            return clusterNode.attribute("DATA") != null;
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index 5b888ce7c65f8..1b76283f8d685 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.QueryJoinWithDifferentNodeFiltersTest;
 import org.apache.ignite.internal.processors.cache.CacheScanPartitionQueryFallbackSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheCrossCacheJoinRandomTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheObjectKeyIndexingSelfTest;
@@ -104,6 +105,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheGroupsSqlSegmentedIndexMultiNodeSelfTest.class);
         suite.addTestSuite(IgniteCacheGroupsSqlDistributedJoinSelfTest.class);
 
+        suite.addTestSuite(QueryJoinWithDifferentNodeFiltersTest.class);
+
         return suite;
     }
 }

From a84538a9df8ace432411736289ec8371dee215e2 Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Wed, 6 Jun 2018 12:59:28 +0300
Subject: [PATCH 175/543] IGNITE-8696 control.sh utility does not show
 atomicity mode - Fixes #4127.

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit 78e5d970be74c04b01857123b1a623038aa18440)
---
 .../processors/cache/verify/CacheInfo.java    | 30 +++++++++++++++++--
 .../cache/verify/ViewCacheClosure.java        |  2 ++
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java
index 9a090a042e961..31c0b3fcbe2d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java
@@ -20,6 +20,8 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -67,6 +69,9 @@ public class CacheInfo extends VisorDataTransferObject {
     /** Mode. */
     private CacheMode mode;
 
+    /** Atomicity mode. */
+    private CacheAtomicityMode atomicityMode;
+
     /** Backups count. */
     private int backupsCnt;
 
@@ -225,6 +230,20 @@ public void setMode(CacheMode mode) {
         this.mode = mode;
     }
 
+    /**
+     *
+     */
+    public CacheAtomicityMode getAtomicityMode() {
+        return atomicityMode;
+    }
+
+    /**
+     * @param atomicityMode
+     */
+    public void setAtomicityMode(CacheAtomicityMode atomicityMode) {
+        this.atomicityMode = atomicityMode;
+    }
+
     /**
      *
      */
@@ -269,18 +288,23 @@ public void print(VisorViewCacheCmd cmd) {
             case GROUPS:
                 System.out.println("[grpName=" + getGrpName() + ", grpId=" + getGrpId() + ", cachesCnt=" + getCachesCnt() +
                     ", prim=" + getPartitions() + ", mapped=" + getMapped() + ", mode=" + getMode() +
-                    ", backups=" + getBackupsCnt() + ", affCls=" + getAffinityClsName() + ']');
+                    ", atomicity=" + getAtomicityMode() + ", backups=" + getBackupsCnt() + ", affCls=" + getAffinityClsName() + ']');
 
                 break;
 
             default:
                 System.out.println("[cacheName=" + getCacheName() + ", cacheId=" + getCacheId() +
                     ", grpName=" + getGrpName() + ", grpId=" + getGrpId() + ", prim=" + getPartitions() +
-                    ", mapped=" + getMapped() + ", mode=" + getMode() +
+                    ", mapped=" + getMapped() + ", mode=" + getMode() + ", atomicity=" + getAtomicityMode() +
                     ", backups=" + getBackupsCnt() + ", affCls=" + getAffinityClsName() + ']');
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeString(out, seqName);
@@ -296,6 +320,7 @@ public void print(VisorViewCacheCmd cmd) {
         out.writeInt(backupsCnt);
         U.writeString(out, affinityClsName);
         out.writeInt(cachesCnt);
+        U.writeEnum(out, atomicityMode);
     }
 
     /** {@inheritDoc} */
@@ -313,6 +338,7 @@ public void print(VisorViewCacheCmd cmd) {
         backupsCnt = in.readInt();
         affinityClsName = U.readString(in);
         cachesCnt = in.readInt();
+        atomicityMode = protoVer >= V2 ? CacheAtomicityMode.fromOrdinal(in.readByte()) : null;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ViewCacheClosure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ViewCacheClosure.java
index 1f363f3643db7..9003ac00257f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ViewCacheClosure.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/ViewCacheClosure.java
@@ -99,6 +99,7 @@ public ViewCacheClosure(String regex, VisorViewCacheCmd cmd) {
                     ci.setBackupsCnt(context.config().getBackups());
                     ci.setAffinityClsName(context.config().getAffinity().getClass().getSimpleName());
                     ci.setMode(context.config().getCacheMode());
+                    ci.setAtomicityMode(context.config().getAtomicityMode());
                     ci.setMapped(mapped(context.caches().iterator().next().name()));
 
                     cacheInfo.add(ci);
@@ -126,6 +127,7 @@ public ViewCacheClosure(String regex, VisorViewCacheCmd cmd) {
                     ci.setBackupsCnt(desc.cacheConfiguration().getBackups());
                     ci.setAffinityClsName(desc.cacheConfiguration().getAffinity().getClass().getSimpleName());
                     ci.setMode(desc.cacheConfiguration().getCacheMode());
+                    ci.setAtomicityMode(desc.cacheConfiguration().getAtomicityMode());
                     ci.setMapped(mapped(desc.cacheName()));
 
                     cacheInfo.add(ci);

From fff8a4cc48ae7d8a47f8911763a71d0d59042d87 Mon Sep 17 00:00:00 2001
From: Andrey Kuznetsov <stkuzma@gmail.com>
Date: Wed, 6 Jun 2018 15:02:35 +0300
Subject: [PATCH 176/543] IGNITE-8642 Added thread dumping to FailureProcessor

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../java/org/apache/ignite/IgniteSystemProperties.java   | 7 +++++++
 .../internal/processors/failure/FailureProcessor.java    | 9 +++++++++
 2 files changed, 16 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 008974c6ff68e..6f98d76ed0e10 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -887,6 +887,13 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_DISABLE_WAL_DURING_REBALANCING = "IGNITE_DISABLE_WAL_DURING_REBALANCING";
 
+    /**
+     * Enables threads dumping on critical node failure.
+     *
+     * Default is {@code true}.
+     */
+    public static final String IGNITE_DUMP_THREADS_ON_FAILURE = "IGNITE_DUMP_THREADS_ON_FAILURE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
index b11cb95126674..722de1870c6ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
@@ -29,10 +29,16 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE;
+
 /**
  * General failure processing API
  */
 public class FailureProcessor extends GridProcessorAdapter {
+    /** Value of the system property that enables threads dumping on failure. */
+    private static final boolean IGNITE_DUMP_THREADS_ON_FAILURE =
+        IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE, true);
+
     /** Ignite. */
     private final Ignite ignite;
 
@@ -115,6 +121,9 @@ public synchronized void process(FailureContext failureCtx, FailureHandler hnd)
         if (reserveBuf != null && X.hasCause(failureCtx.error(), OutOfMemoryError.class))
             reserveBuf = null;
 
+        if (IGNITE_DUMP_THREADS_ON_FAILURE)
+            U.dumpThreads(log);
+
         boolean invalidated = hnd.onFailure(ignite, failureCtx);
 
         if (invalidated) {

From 6ea23844cfb0294a0cf3f3c21787aeff30939bcf Mon Sep 17 00:00:00 2001
From: Dmitriy Sorokin <sbt.sorokin.dvl@gmail.com>
Date: Wed, 6 Jun 2018 14:34:04 +0300
Subject: [PATCH 177/543] IGNITE-8311
 IgniteClientRejoinTest.testClientsReconnectDisabled causes exchange-worker to
 terminate via NPE

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../GridCachePartitionExchangeManager.java    |  4 +-
 .../failure/FailureHandlerTriggeredTest.java  | 35 -----------
 .../ignite/failure/TestFailureHandler.java    | 63 +++++++++++++++++++
 .../internal/IgniteClientRejoinTest.java      | 11 +++-
 .../IgniteChangeGlobalStateTest.java          |  2 +
 .../CacheHibernateStoreFactorySelfTest.java   |  2 +
 .../CacheHibernateStoreFactorySelfTest.java   |  2 +
 .../query/IgniteSqlSchemaIndexingTest.java    |  5 +-
 .../CacheJdbcBlobStoreFactorySelfTest.java    |  2 +
 .../CacheJdbcPojoStoreFactorySelfTest.java    |  2 +
 10 files changed, 90 insertions(+), 38 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index c3a0add55dee9..bb66a3b5ada47 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -2352,7 +2352,7 @@ else if (err != null)
         /**
          *
          */
-        private void body0() throws InterruptedException, IgniteInterruptedCheckedException {
+        private void body0() throws InterruptedException, IgniteCheckedException {
             long timeout = cctx.gridConfig().getNetworkTimeout();
 
             long cnt = 0;
@@ -2656,6 +2656,8 @@ else if (r != null) {
                 catch (IgniteCheckedException e) {
                     U.error(log, "Failed to wait for completion of partition map exchange " +
                         "(preloading will not start): " + task, e);
+
+                    throw e;
                 }
             }
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java b/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java
index 5eca7d698af7c..8d56ced75f2cc 100644
--- a/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/failure/FailureHandlerTriggeredTest.java
@@ -19,7 +19,6 @@
 
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
-import org.apache.ignite.Ignite;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.processors.cache.CachePartitionExchangeWorkerTask;
@@ -66,40 +65,6 @@ public void testFailureHandlerTriggeredOnExchangeWorkerTermination() throws Exce
         }
     }
 
-    /**
-     * Test failure handler implementation
-     */
-    private class TestFailureHandler implements FailureHandler {
-        /** Invalidate. */
-        private final boolean invalidate;
-
-        /** Latch. */
-        private final CountDownLatch latch;
-
-        /** Failure context. */
-        volatile FailureContext failureCtx;
-
-        /**
-         * @param invalidate Invalidate.
-         * @param latch Latch.
-         */
-        TestFailureHandler(boolean invalidate, CountDownLatch latch) {
-            this.invalidate = invalidate;
-            this.latch = latch;
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
-            this.failureCtx = failureCtx;
-
-            this.latch.countDown();
-
-            ignite.log().warning("Handled ignite failure: " + failureCtx);
-
-            return invalidate;
-        }
-    }
-
     /**
      * Custom exchange worker task implementation for delaying exchange worker processing.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java b/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
new file mode 100644
index 0000000000000..1159683e6b54f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+
+/**
+ * Test failure handler implementation
+ */
+public class TestFailureHandler implements FailureHandler {
+    /** Invalidate. */
+    private final boolean invalidate;
+
+    /** Latch. */
+    private final CountDownLatch latch;
+
+    /** Failure context. */
+    volatile FailureContext failureCtx;
+
+    /**
+     * @param invalidate Invalidate.
+     * @param latch Latch.
+     */
+    public TestFailureHandler(boolean invalidate, CountDownLatch latch) {
+        this.invalidate = invalidate;
+        this.latch = latch;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+        this.failureCtx = failureCtx;
+
+        if (latch != null)
+            latch.countDown();
+
+        ignite.log().warning("Handled ignite failure: " + failureCtx);
+
+        return invalidate;
+    }
+
+    /**
+     * @return Failure context.
+     */
+    public FailureContext failureContext() {
+        return failureCtx;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
index 462c8ab6197d8..9a98a888aa2cb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteClientRejoinTest.java
@@ -27,6 +27,7 @@
 import java.util.Random;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
@@ -35,6 +36,7 @@
 import org.apache.ignite.Ignition;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.TestFailureHandler;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
@@ -282,6 +284,8 @@ public void testClientsReconnectDisabled() throws Exception {
 
         final int CLIENTS_NUM = 5;
 
+        final CountDownLatch failureHndLatch = new CountDownLatch(CLIENTS_NUM);
+
         for (int i = 0; i < CLIENTS_NUM; i++) {
             final int idx = i;
 
@@ -289,7 +293,10 @@ public void testClientsReconnectDisabled() throws Exception {
                 @Override public Ignite call() throws Exception {
                     latch.await();
 
-                    return startGrid("client" + idx);
+                    String igniteInstanceName = "client" + idx;
+
+                    return startGrid(igniteInstanceName, getConfiguration(igniteInstanceName)
+                        .setFailureHandler(new TestFailureHandler(true, failureHndLatch)));
                 }
             });
 
@@ -309,6 +316,8 @@ public void testClientsReconnectDisabled() throws Exception {
             }, IgniteCheckedException.class, null);
         }
 
+        assertTrue(failureHndLatch.await(1000, TimeUnit.MILLISECONDS));
+
         assertEquals(0, srv1.cluster().forClients().nodes().size());
         assertEquals(0, srv2.cluster().forClients().nodes().size());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
index cbf0ec9256b2e..9152ab90319df 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
@@ -543,6 +543,8 @@ public void testFailGetLock() throws Exception {
      * @throws Exception If failed.
      */
     public void testActivateAfterFailGetLock() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-1094");
+
         Ignite ig1P = primary(0);
         Ignite ig2P = primary(1);
         Ignite ig3P = primary(2);
diff --git a/modules/hibernate-4.2/src/test/java/org/apache/ignite/cache/store/hibernate/CacheHibernateStoreFactorySelfTest.java b/modules/hibernate-4.2/src/test/java/org/apache/ignite/cache/store/hibernate/CacheHibernateStoreFactorySelfTest.java
index 0dab22cbb0748..db0e1bebe8026 100644
--- a/modules/hibernate-4.2/src/test/java/org/apache/ignite/cache/store/hibernate/CacheHibernateStoreFactorySelfTest.java
+++ b/modules/hibernate-4.2/src/test/java/org/apache/ignite/cache/store/hibernate/CacheHibernateStoreFactorySelfTest.java
@@ -83,6 +83,8 @@ public void testXmlConfiguration() throws Exception {
      * @throws Exception If failed.
      */
     public void testIncorrectBeanConfiguration() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-1094");
+
         GridTestUtils.assertThrows(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
                 try(Ignite ignite =
diff --git a/modules/hibernate-5.1/src/test/java/org/apache/ignite/cache/store/hibernate/CacheHibernateStoreFactorySelfTest.java b/modules/hibernate-5.1/src/test/java/org/apache/ignite/cache/store/hibernate/CacheHibernateStoreFactorySelfTest.java
index a329f2b7f5b6f..0ffe52ef1176f 100644
--- a/modules/hibernate-5.1/src/test/java/org/apache/ignite/cache/store/hibernate/CacheHibernateStoreFactorySelfTest.java
+++ b/modules/hibernate-5.1/src/test/java/org/apache/ignite/cache/store/hibernate/CacheHibernateStoreFactorySelfTest.java
@@ -83,6 +83,8 @@ public void testXmlConfiguration() throws Exception {
      * @throws Exception If failed.
      */
     public void testIncorrectBeanConfiguration() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-1094");
+
         GridTestUtils.assertThrows(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
                 try(Ignite ignite =
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSchemaIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSchemaIndexingTest.java
index e375df2cf6e7e..2dee617502424 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSchemaIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/IgniteSqlSchemaIndexingTest.java
@@ -87,7 +87,8 @@ private static CacheConfiguration cacheConfig(String name, boolean partitioned,
      */
     public void testCaseSensitive() throws Exception {
         //TODO rewrite with dynamic cache creation, and GRID start in #beforeTest after resolve of
-        //https://issues.apache.org/jira/browse/IGNITE-1094
+        //TODO https://issues.apache.org/jira/browse/IGNITE-1094
+        fail("https://issues.apache.org/jira/browse/IGNITE-1094");
 
         GridTestUtils.assertThrows(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
@@ -117,6 +118,8 @@ public void testCaseSensitive() throws Exception {
     public void testCustomSchemaMultipleCachesTablesCollision() throws Exception {
         //TODO: Rewrite with dynamic cache creation, and GRID start in #beforeTest after resolve of
         //TODO: https://issues.apache.org/jira/browse/IGNITE-1094
+        fail("https://issues.apache.org/jira/browse/IGNITE-1094");
+
         GridTestUtils.assertThrows(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
                 final CacheConfiguration cfg = cacheConfig("cache1", true, Integer.class, Fact.class)
diff --git a/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcBlobStoreFactorySelfTest.java b/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcBlobStoreFactorySelfTest.java
index bc4f2684dfffd..be2238bd8bf93 100644
--- a/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcBlobStoreFactorySelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcBlobStoreFactorySelfTest.java
@@ -77,6 +77,8 @@ public void testCacheConfiguration() throws Exception {
      * @throws Exception If failed.
      */
     public void testIncorrectBeanConfiguration() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-1094");
+
         GridTestUtils.assertThrows(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
                 try(Ignite ignite = Ignition.start("modules/spring/src/test/config/incorrect-store-cache.xml")) {
diff --git a/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java b/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java
index 0d6d6a28279fe..ba4984d22cab9 100644
--- a/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java
+++ b/modules/spring/src/test/java/org/apache/ignite/cache/store/jdbc/CacheJdbcPojoStoreFactorySelfTest.java
@@ -69,6 +69,8 @@ public void testSerializable() throws Exception {
      * @throws Exception If failed.
      */
     public void testIncorrectBeanConfiguration() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-1094");
+
         GridTestUtils.assertThrowsAnyCause(log, new Callable<Object>() {
             @Override public Object call() throws Exception {
                 try (Ignite ignored = Ignition.start("modules/spring/src/test/config/pojo-incorrect-store-cache.xml")) {

From bad52d3843925d831e6d357a4382731751ae6bb7 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Mon, 4 Jun 2018 17:07:52 +0300
Subject: [PATCH 178/543] GG-13862 : Disabled fast eviction.

(cherry picked from commit 786633d)
---
 .../ignite/internal/processors/cache/CacheGroupContext.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 4f9b7f598d1b9..d3c06b8475541 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -522,9 +522,10 @@ public boolean queriesEnabled() {
 
     /**
      * @return {@code True} if fast eviction is allowed.
+     * @deprecated This mode is now unsafe and should be completely removed in 2.6.
      */
-    public boolean allowFastEviction() {
-        return persistenceEnabled() && !queriesEnabled();
+    @Deprecated public boolean allowFastEviction() {
+        return false;
     }
 
     /**

From 6956097ceedd2e3e9edcb627d97e49afb7f0a26d Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 6 Jun 2018 15:24:39 +0300
Subject: [PATCH 179/543] IGNITE-8482 Skip 2-phase partition release wait in
 case of activation or dynamic caches start - Fixes #4078.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 7c565d2)
---
 .../GridDhtPartitionsExchangeFuture.java      | 32 ++++++++++++++-----
 .../preloader/latch/ExchangeLatchManager.java | 18 ++++++-----
 2 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 32d9cb30b8b18..fa82aa45a0dbe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -515,6 +515,13 @@ public boolean changedBaseline() {
         return exchActions != null && exchActions.changedBaseline();
     }
 
+    /**
+     * @return {@code True} if there are caches to start.
+     */
+    public boolean hasCachesToStart() {
+        return exchActions != null && !exchActions.cacheStartRequests().isEmpty();
+    }
+
     /**
      * @return First event discovery event.
      *
@@ -1127,11 +1134,17 @@ private void distributedExchange() throws IgniteCheckedException {
         // To correctly rebalance when persistence is enabled, it is necessary to reserve history within exchange.
         partHistReserved = cctx.database().reserveHistoryForExchange();
 
-        // On first phase we wait for finishing all local tx updates, atomic updates and lock releases.
-        waitPartitionRelease(1);
+        boolean distributed = true;
+
+        // Do not perform distributed partition release in case of cluster activation or caches start.
+        if (activateCluster() || hasCachesToStart())
+            distributed = false;
+
+        // On first phase we wait for finishing all local tx updates, atomic updates and lock releases on all nodes.
+        waitPartitionRelease(distributed);
 
         // Second phase is needed to wait for finishing all tx updates from primary to backup nodes remaining after first phase.
-        waitPartitionRelease(2);
+        waitPartitionRelease(false);
 
         boolean topChanged = firstDiscoEvt.type() != EVT_DISCOVERY_CUSTOM_EVT || affChangeMsg != null;
 
@@ -1236,15 +1249,14 @@ private void changeWalModeIfNeeded() {
      * For the exact list of the objects being awaited for see
      * {@link GridCacheSharedContext#partitionReleaseFuture(AffinityTopologyVersion)} javadoc.
      *
-     * @param phase Phase of partition release.
+     * @param distributed If {@code true} then node should wait for partition release completion on all other nodes.
      *
      * @throws IgniteCheckedException If failed.
      */
-    private void waitPartitionRelease(int phase) throws IgniteCheckedException {
+    private void waitPartitionRelease(boolean distributed) throws IgniteCheckedException {
         Latch releaseLatch = null;
 
-        // Wait for other nodes only on first phase.
-        if (phase == 1)
+        if (distributed)
             releaseLatch = cctx.exchange().latch().getOrCreate("exchange", initialVersion());
 
         IgniteInternalFuture<?> partReleaseFut = cctx.partitionReleaseFuture(initialVersion());
@@ -2591,7 +2603,11 @@ private void finishExchangeOnCoordinator(@Nullable Collection<ClusterNode> sndRe
                 }
             }
 
-            validatePartitionsState();
+            // Don't validate partitions state in case of caches start.
+            boolean skipValidation = hasCachesToStart();
+
+            if (!skipValidation)
+                validatePartitionsState();
 
             if (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
                 assert firstDiscoEvt instanceof DiscoveryCustomEvent;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index b9c7dee7ba6b2..25424b8822ba7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -452,11 +452,12 @@ class ServerLatch extends CompletableLatch {
                             io.sendToGridTopic(node, GridTopic.TOPIC_EXCHANGE, new LatchAckMessage(id, topVer, true), GridIoPolicy.SYSTEM_POOL);
 
                             if (log.isDebugEnabled())
-                                log.debug("Final ack is ackSent [latch=" + latchId() + ", to=" + node.id() + "]");
+                                log.debug("Final ack has sent [latch=" + latchId() + ", to=" + node.id() + "]");
                         }
-                    } catch (IgniteCheckedException e) {
+                    }
+                    catch (IgniteCheckedException e) {
                         if (log.isDebugEnabled())
-                            log.debug("Unable to send final ack [latch=" + latchId() + ", to=" + node.id() + "]");
+                            log.debug("Failed to send final ack [latch=" + latchId() + ", to=" + node.id() + "]: " + e.getMessage());
                     }
                 }
             });
@@ -500,7 +501,7 @@ private void countDown0(UUID node) {
             int remaining = permits.decrementAndGet();
 
             if (log.isDebugEnabled())
-                log.debug("Count down + [latch=" + latchId() + ", remaining=" + remaining + "]");
+                log.debug("Count down [latch=" + latchId() + ", remaining=" + remaining + "]");
 
             if (remaining == 0)
                 complete();
@@ -563,7 +564,7 @@ private boolean hasCoordinator(UUID node) {
          */
         private void newCoordinator(ClusterNode coordinator) {
             if (log.isDebugEnabled())
-                log.debug("Coordinator is changed [latch=" + latchId() + ", crd=" + coordinator.id() + "]");
+                log.debug("Coordinator is changed [latch=" + latchId() + ", newCrd=" + coordinator.id() + "]");
 
             synchronized (this) {
                 this.coordinator = coordinator;
@@ -585,11 +586,12 @@ private void sendAck() {
                 io.sendToGridTopic(coordinator, GridTopic.TOPIC_EXCHANGE, new LatchAckMessage(id, topVer, false), GridIoPolicy.SYSTEM_POOL);
 
                 if (log.isDebugEnabled())
-                    log.debug("Ack is ackSent + [latch=" + latchId() + ", to=" + coordinator.id() + "]");
-            } catch (IgniteCheckedException e) {
+                    log.debug("Ack has sent [latch=" + latchId() + ", to=" + coordinator.id() + "]");
+            }
+            catch (IgniteCheckedException e) {
                 // Coordinator is unreachable. On coodinator node left discovery event ack will be resent.
                 if (log.isDebugEnabled())
-                    log.debug("Unable to send ack [latch=" + latchId() + ", to=" + coordinator.id() + "]: " + e.getMessage());
+                    log.debug("Failed to send ack [latch=" + latchId() + ", to=" + coordinator.id() + "]: " + e.getMessage());
             }
         }
 

From d777c76eadd0a2ccc50f1031c42626b135da5906 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Wed, 6 Jun 2018 17:20:18 +0300
Subject: [PATCH 180/543] IGNITE-8685 Fixed switch segment record size - Fixes
 #4130.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 2a048bd)
---
 .../wal/record/BaselineTopologyRecord.java    |  76 ----
 .../pagemem/wal/record/WALRecord.java         |  12 +-
 .../wal/AbstractWalRecordsIterator.java       |  15 +-
 .../reader/StandaloneGridKernalContext.java   |   8 +-
 .../BaselineTopologyRecordSerializer.java     | 168 --------
 .../serializer/RecordDataV2Serializer.java    |  16 -
 .../wal/serializer/RecordV1Serializer.java    |  28 +-
 .../wal/serializer/RecordV2Serializer.java    |  27 +-
 .../IgniteWalIteratorSwitchSegmentTest.java   | 386 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 10 files changed, 459 insertions(+), 280 deletions(-)
 delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java
 delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java
deleted file mode 100644
index 48b60b3924362..0000000000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/BaselineTopologyRecord.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.pagemem.wal.record;
-
-import java.util.Map;
-import org.apache.ignite.internal.util.typedef.internal.S;
-
-/**
- * Record for storing baseline topology compact node ID to consistent node ID mapping.
- */
-public class BaselineTopologyRecord extends WALRecord {
-    /** Id. */
-    private int id;
-
-    /** Compact ID to consistent ID mapping. */
-    private Map<Short, Object> mapping;
-
-    /**
-     * Default constructor.
-     */
-    private BaselineTopologyRecord() {
-        // No-op, used from factory methods.
-    }
-
-    /**
-     * @param id Baseline topology ID.
-     * @param mapping Compact ID to consistent ID mapping.
-     */
-    public BaselineTopologyRecord(int id, Map<Short, Object> mapping) {
-        this.id = id;
-        this.mapping = mapping;
-    }
-
-    /** {@inheritDoc} */
-    @Override public RecordType type() {
-        return RecordType.BASELINE_TOP_RECORD;
-    }
-
-    /**
-     * Returns baseline topology ID.
-     *
-     * @return Baseline topology ID.
-     */
-    public int id() {
-        return id;
-    }
-
-    /**
-     * Returns mapping.
-     *
-     * @return Compact ID to consistent ID mapping.
-     */
-    public Map<Short, Object> mapping() {
-        return mapping;
-    }
-
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return S.toString(BaselineTopologyRecord.class, this, "super", super.toString());
-    }
-}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
index 4fae179b63d28..87ba07db7f341 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/WALRecord.java
@@ -153,7 +153,13 @@ public enum RecordType {
         /** Page list meta reset count record. */
         PAGE_LIST_META_RESET_COUNT_RECORD,
 
-        /** Switch segment record. */
+        /** Switch segment record.
+         *  Marker record for indicate end of segment.
+         *  If the next one record is written down exactly at the end of segment,
+         *  SWITCH_SEGMENT_RECORD will not be written, if not then it means that we have more
+         *  that one byte in the end,then we write SWITCH_SEGMENT_RECORD as marker end of segment.
+         *  No need write CRC or WAL pointer for this record. It is byte marker record.
+         *  */
         SWITCH_SEGMENT_RECORD,
 
         /** */
@@ -174,8 +180,8 @@ public enum RecordType {
         /** Exchange record. */
         EXCHANGE,
 
-        /** Baseline topology record. */
-        BASELINE_TOP_RECORD;
+        /** Reserved for future record. */
+        RESERVED;
 
         /** */
         private static final RecordType[] VALS = RecordType.values();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index d9312f6888b2f..e442386c03e87 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -190,18 +190,20 @@ protected void advance() throws IgniteCheckedException {
     /**
      * Switches records iterator to the next WAL segment
      * as result of this method, new reference to segment should be returned.
-     * Null for current handle means stop of iteration
-     * @throws IgniteCheckedException if reading failed
+     * Null for current handle means stop of iteration.
+     *
      * @param curWalSegment current open WAL segment or null if there is no open segment yet
      * @return new WAL segment to read or null for stop iteration
+     * @throws IgniteCheckedException if reading failed
      */
     protected abstract AbstractReadFileHandle advanceSegment(
         @Nullable final AbstractReadFileHandle curWalSegment) throws IgniteCheckedException;
 
     /**
-     * Switches to new record
-     * @param hnd currently opened read handle
-     * @return next advanced record
+     * Switches to new record.
+     *
+     * @param hnd currently opened read handle.
+     * @return next advanced record.
      */
     private IgniteBiTuple<WALPointer, WALRecord> advanceRecord(
         @Nullable final AbstractReadFileHandle hnd
@@ -242,7 +244,8 @@ private IgniteBiTuple<WALPointer, WALRecord> advanceRecord(
     }
 
     /**
-     * Handler for record deserialization exception
+     * Handler for record deserialization exception.
+     *
      * @param e problem from records reading
      * @param ptr file pointer was accessed
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index cb04575c8810a..795d46004f727 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -128,9 +128,11 @@ public class StandaloneGridKernalContext implements GridKernalContext {
      * {@code null} means no specific folder is configured.
      * Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects will be provided <br>
      */
-    StandaloneGridKernalContext(IgniteLogger log,
+    public StandaloneGridKernalContext(
+        IgniteLogger log,
         @Nullable File binaryMetadataFileStoreDir,
-        @Nullable File marshallerMappingFileStoreDir) throws IgniteCheckedException {
+        @Nullable File marshallerMappingFileStoreDir
+    ) throws IgniteCheckedException {
         this.log = log;
 
         try {
@@ -179,7 +181,7 @@ private IgniteCacheObjectProcessor binaryProcessor(
     /**
      * @return Ignite configuration which allows to start requied processors for WAL reader
      */
-    private IgniteConfiguration prepareIgniteConfiguration() {
+    protected IgniteConfiguration prepareIgniteConfiguration() {
         IgniteConfiguration cfg = new IgniteConfiguration();
 
         cfg.setDiscoverySpi(new StandaloneNoopDiscoverySpi());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java
deleted file mode 100644
index 94b51c5cdd8cb..0000000000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/BaselineTopologyRecordSerializer.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.wal.serializer;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.Map;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.wal.record.BaselineTopologyRecord;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- * {@link BaselineTopologyRecord} WAL serializer.
- */
-public class BaselineTopologyRecordSerializer {
-    /** Cache shared context. */
-    private GridCacheSharedContext cctx;
-
-    /** Class loader to unmarshal consistent IDs. */
-    private ClassLoader clsLdr;
-
-    /**
-     * Create an instance of serializer.
-     *
-     * @param cctx Cache shared context.
-     */
-    public BaselineTopologyRecordSerializer(GridCacheSharedContext cctx) {
-        this.cctx = cctx;
-
-        clsLdr = U.resolveClassLoader(cctx.gridConfig());
-    }
-
-    /**
-     * Writes {@link BaselineTopologyRecord} to given buffer.
-     *
-     * @param rec {@link BaselineTopologyRecord} instance.
-     * @param buf Byte buffer.
-     * @throws IgniteCheckedException In case of fail.
-     */
-    public void write(BaselineTopologyRecord rec, ByteBuffer buf) throws IgniteCheckedException {
-        buf.putInt(rec.id());
-
-        Map<Short, Object> mapping = rec.mapping();
-
-        if (mapping != null && !mapping.isEmpty()) {
-            buf.putInt(mapping.size());
-
-            for (Map.Entry<Short, Object> e : mapping.entrySet()) {
-                buf.putShort(e.getKey());
-
-                writeConsistentId(e.getValue(), buf);
-            }
-        }
-        else
-            buf.putInt(0);
-    }
-
-    /**
-     * Reads {@link BaselineTopologyRecord} from given input.
-     *
-     * @param in Input
-     * @return BaselineTopologyRecord instance.
-     * @throws IOException In case of fail.
-     * @throws IgniteCheckedException In case of fail.
-     */
-    public BaselineTopologyRecord read(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
-        int id = in.readInt();
-
-        int size = in.readInt();
-
-        Map<Short, Object> mapping = size > 0 ? U.<Short, Object>newHashMap(size) : null;
-
-        for (int i = 0; i < size; i++) {
-            short compactId = in.readShort();
-
-            Object consistentId = readConsistentId(in);
-
-            mapping.put(compactId, consistentId);
-        }
-
-        return new BaselineTopologyRecord(id, mapping);
-    }
-
-    /**
-     * Returns size of marshalled {@link BaselineTopologyRecord} in bytes.
-     *
-     * @param rec BaselineTopologyRecord instance.
-     * @return Size of BaselineTopologyRecord instance in bytes.
-     * @throws IgniteCheckedException In case of fail.
-     */
-    public int size(BaselineTopologyRecord rec) throws IgniteCheckedException {
-        int size = 0;
-
-        size += /* Baseline topology ID. */ 4;
-
-        size += /* Consistent ID mapping size. */ 4;
-
-        if (rec.mapping() != null) {
-            for (Object consistentId : rec.mapping().values()) {
-                size += /* Compact ID size */ 2;
-
-                size += marshalConsistentId(consistentId).length;
-            }
-        }
-
-        return size;
-    }
-
-    /**
-     * Write consistent id to given buffer.
-     *
-     * @param consistentId Consistent id.
-     * @param buf Byte buffer.
-     * @throws IgniteCheckedException In case of fail.
-     */
-    private void writeConsistentId(Object consistentId, ByteBuffer buf) throws IgniteCheckedException {
-        byte[] content = marshalConsistentId(consistentId);
-
-        buf.putInt(content.length);
-        buf.put(content);
-    }
-
-    /**
-     * Read consistent id from given input.
-     *
-     * @param in Input.
-     * @return Consistent id.
-     * @throws IOException In case of fail.
-     * @throws IgniteCheckedException In case of fail.
-     */
-    private Object readConsistentId(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
-        int len = in.readInt();
-        in.ensure(len);
-
-        byte[] content = new byte[len];
-        in.readFully(content);
-
-        return cctx.marshaller().unmarshal(content, clsLdr);
-    }
-
-    /**
-     * Marshal consistent id to byte array.
-     *
-     * @param consistentId Consistent id.
-     * @return Marshalled byte array.
-     * @throws IgniteCheckedException In case of fail.
-     */
-    private byte[] marshalConsistentId(Object consistentId) throws IgniteCheckedException {
-        return cctx.marshaller().marshal(consistentId);
-    }
-}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
index b3a00beeb6aa8..b760547e6e933 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV2Serializer.java
@@ -27,7 +27,6 @@
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.wal.record.BaselineTopologyRecord;
 import org.apache.ignite.internal.pagemem.wal.record.CacheState;
 import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
@@ -53,9 +52,6 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
     /** Serializer of {@link TxRecord} records. */
     private final TxRecordSerializer txRecordSerializer;
 
-    /** Serializer of {@link BaselineTopologyRecord} records. */
-    private final BaselineTopologyRecordSerializer bltRecSerializer;
-
     /**
      * Create an instance of V2 data serializer.
      *
@@ -64,7 +60,6 @@ public class RecordDataV2Serializer implements RecordDataSerializer {
     public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) {
         this.delegateSerializer = delegateSerializer;
         this.txRecordSerializer = new TxRecordSerializer();
-        this.bltRecSerializer = new BaselineTopologyRecordSerializer(delegateSerializer.cctx());
     }
 
     /** {@inheritDoc} */
@@ -97,9 +92,6 @@ public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) {
             case TX_RECORD:
                 return txRecordSerializer.size((TxRecord)rec);
 
-            case BASELINE_TOP_RECORD:
-                return bltRecSerializer.size((BaselineTopologyRecord)rec);
-
             default:
                 return delegateSerializer.size(rec);
         }
@@ -157,9 +149,6 @@ public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) {
             case TX_RECORD:
                 return txRecordSerializer.read(in);
 
-            case BASELINE_TOP_RECORD:
-                return bltRecSerializer.read(in);
-
             default:
                 return delegateSerializer.readRecord(type, in);
         }
@@ -231,11 +220,6 @@ public RecordDataV2Serializer(RecordDataV1Serializer delegateSerializer) {
 
                 break;
 
-            case BASELINE_TOP_RECORD:
-                bltRecSerializer.write((BaselineTopologyRecord)rec, buf);
-
-                break;
-
             default:
                 delegateSerializer.writeRecord(rec, buf);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
index dd0819c859ce8..caa096294e88b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -47,6 +47,7 @@
 import org.apache.ignite.lang.IgniteBiPredicate;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.SWITCH_SEGMENT_RECORD;
 
 /**
  * Record V1 serializer.
@@ -111,7 +112,13 @@ public class RecordV1Serializer implements RecordSerializer {
 
         /** {@inheritDoc} */
         @Override public int sizeWithHeaders(WALRecord record) throws IgniteCheckedException {
-            return dataSerializer.size(record) + REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE;
+            int recordSize = dataSerializer.size(record);
+
+            int recordSizeWithType = recordSize + REC_TYPE_SIZE;
+
+            // Why this condition here, see SWITCH_SEGMENT_RECORD doc.
+            return record.type() != SWITCH_SEGMENT_RECORD ?
+                recordSizeWithType + FILE_WAL_POINTER_SIZE + CRC_SIZE : recordSizeWithType;
         }
 
         /** {@inheritDoc} */
@@ -160,6 +167,10 @@ else if (marshalledMode) {
             // Write record type.
             putRecordType(buf, rec);
 
+            // SWITCH_SEGMENT_RECORD should have only type, no need to write pointer.
+            if (rec.type() == SWITCH_SEGMENT_RECORD)
+                return;
+
             // Write record file position.
             putPositionOfRecord(buf, rec);
 
@@ -176,8 +187,13 @@ else if (marshalledMode) {
      * @param skipPositionCheck Skip position check mode.
      * @param recordFilter Record type filter. {@link FilteredRecord} is deserialized instead of original record
      */
-    public RecordV1Serializer(RecordDataV1Serializer dataSerializer, boolean writePointer,
-        boolean marshalledMode, boolean skipPositionCheck, IgniteBiPredicate<RecordType, WALPointer> recordFilter) {
+    public RecordV1Serializer(
+        RecordDataV1Serializer dataSerializer,
+        boolean writePointer,
+        boolean marshalledMode,
+        boolean skipPositionCheck,
+        IgniteBiPredicate<RecordType, WALPointer> recordFilter
+    ) {
         this.dataSerializer = dataSerializer;
         this.writePointer = writePointer;
         this.recordFilter = recordFilter;
@@ -376,10 +392,16 @@ static WALRecord readWithCrc(FileInput in0, WALPointer expPtr, RecordIO reader)
     static void writeWithCrc(WALRecord rec, ByteBuffer buf, RecordIO writer) throws IgniteCheckedException {
         assert rec.size() >= 0 && buf.remaining() >= rec.size() : rec.size();
 
+        boolean switchSegmentRec = rec.type() == RecordType.SWITCH_SEGMENT_RECORD;
+
         int startPos = buf.position();
 
         writer.writeWithHeaders(rec, buf);
 
+        // No need calculate and write CRC for SWITCH_SEGMENT_RECORD.
+        if (switchSegmentRec)
+            return;
+
         if (!skipCrc) {
             int curPos = buf.position();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
index feeb810a94320..2b81210e6fd0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.lang.IgniteBiPredicate;
 
 import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.SWITCH_SEGMENT_RECORD;
 import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.CRC_SIZE;
 import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.REC_TYPE_SIZE;
 
@@ -93,7 +94,13 @@ public class RecordV2Serializer implements RecordSerializer {
 
         /** {@inheritDoc} */
         @Override public int sizeWithHeaders(WALRecord record) throws IgniteCheckedException {
-            return dataSerializer.size(record) + REC_TYPE_SIZE + FILE_WAL_POINTER_SIZE + CRC_SIZE;
+            int recordSize = dataSerializer.size(record);
+
+            int recordSizeWithType = recordSize + REC_TYPE_SIZE;
+
+            // Why this condition here, see SWITCH_SEGMENT_RECORD doc.
+            return record.type() != SWITCH_SEGMENT_RECORD ?
+                recordSizeWithType + FILE_WAL_POINTER_SIZE + CRC_SIZE : recordSizeWithType;
         }
 
         /** {@inheritDoc} */
@@ -103,7 +110,7 @@ public class RecordV2Serializer implements RecordSerializer {
         ) throws IOException, IgniteCheckedException {
             WALRecord.RecordType recType = RecordV1Serializer.readRecordType(in);
 
-            if (recType == WALRecord.RecordType.SWITCH_SEGMENT_RECORD)
+            if (recType == SWITCH_SEGMENT_RECORD)
                 throw new SegmentEofException("Reached end of segment", null);
 
             FileWALPointer ptr = readPositionAndCheckPoint(in, expPtr, skipPositionCheck);
@@ -162,6 +169,10 @@ else if (marshalledMode) {
             // Write record type.
             RecordV1Serializer.putRecordType(buf, record);
 
+            // SWITCH_SEGMENT_RECORD should have only type, no need to write pointer.
+            if (record.type() == SWITCH_SEGMENT_RECORD)
+                return;
+
             // Write record file position.
             putPositionOfRecord(buf, record);
 
@@ -172,13 +183,19 @@ else if (marshalledMode) {
 
     /**
      * Create an instance of Record V2 serializer.
+     *
      * @param dataSerializer V2 data serializer.
      * @param marshalledMode Marshalled mode.
      * @param skipPositionCheck Skip position check mode.
-     * @param recordFilter Record type filter. {@link FilteredRecord} is deserialized instead of original record
+     * @param recordFilter Record type filter. {@link FilteredRecord} is deserialized instead of original record.
      */
-    public RecordV2Serializer(RecordDataV2Serializer dataSerializer, boolean writePointer,
-        boolean marshalledMode, boolean skipPositionCheck, IgniteBiPredicate<RecordType, WALPointer> recordFilter) {
+    public RecordV2Serializer(
+        RecordDataV2Serializer dataSerializer,
+        boolean writePointer,
+        boolean marshalledMode,
+        boolean skipPositionCheck,
+        IgniteBiPredicate<RecordType, WALPointer> recordFilter
+    ) {
         this.dataSerializer = dataSerializer;
         this.writePointer = writePointer;
         this.marshalledMode = marshalledMode;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
new file mode 100644
index 0000000000000..b30466e5acf49
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
@@ -0,0 +1,386 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.Random;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheIoManager;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneGridKernalContext;
+import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
+import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
+import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessorImpl;
+import org.apache.ignite.internal.processors.subscription.GridInternalSubscriptionProcessor;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.spi.eventstorage.NoopEventStorageSpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.HEADER_RECORD_SIZE;
+
+/***
+ * Test check correct switch segment if in the tail of segment have garbage.
+ */
+public class IgniteWalIteratorSwitchSegmentTest extends GridCommonAbstractTest {
+    /** Segment file size. */
+    private static final int SEGMENT_SIZE = 1024 * 1024;
+
+    /** WAL segment file sub directory. */
+    private static final String WORK_SUB_DIR = "/NODE/wal";
+
+    /** WAL archive segment file sub directory. */
+    private static final String ARCHIVE_SUB_DIR = "/NODE/walArchive";
+
+    /** Serializer versions for check. */
+    private int[] checkSerializerVers = new int[] {
+        1,
+        2
+    };
+
+    /** FileWriteAheadLogManagers for check. */
+    private Class[] checkWalManagers = new Class[] {
+        FileWriteAheadLogManager.class,
+        FsyncModeFileWriteAheadLogManager.class
+    };
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        U.delete(Paths.get(U.defaultWorkDirectory()));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        U.delete(Paths.get(U.defaultWorkDirectory()));
+    }
+
+    /**
+     * Test for check invariant, size of SWITCH_SEGMENT_RECORD should be 1 byte.
+     *
+     * @throws Exception If some thing failed.
+     */
+    public void testCheckSerializer() throws Exception {
+        for (int serVer : checkSerializerVers) {
+            checkInvariantSwitchSegmentSize(serVer);
+        }
+    }
+
+    /**
+     * @param serVer WAL serializer version.
+     * @throws Exception If some thing failed.
+     */
+    private void checkInvariantSwitchSegmentSize(int serVer) throws Exception {
+        GridKernalContext kctx = new StandaloneGridKernalContext(
+            log, null, null) {
+            @Override public IgniteCacheObjectProcessor cacheObjects() {
+                return new IgniteCacheObjectProcessorImpl(this);
+            }
+        };
+
+        RecordSerializer serializer = new RecordSerializerFactoryImpl(
+            new GridCacheSharedContext<>(
+                kctx,
+                null,
+                null,
+                null,
+                null,
+                null,
+                null,
+                new IgniteCacheDatabaseSharedManager() {
+                    @Override public int pageSize() {
+                        return DataStorageConfiguration.DFLT_PAGE_SIZE;
+                    }
+                },
+                null,
+                null,
+                null,
+                null,
+                null,
+                null,
+                null,
+
+                null)
+        ).createSerializer(serVer);
+
+        SwitchSegmentRecord switchSegmentRecord = new SwitchSegmentRecord();
+
+        int recordSize = serializer.size(switchSegmentRecord);
+
+        Assert.assertEquals(1, recordSize);
+    }
+
+    /**
+     * Test for check invariant, size of SWITCH_SEGMENT_RECORD should be 1 byte.
+     *
+     * @throws Exception If some thing failed.
+     */
+    public void test() throws Exception {
+        for (int serVer : checkSerializerVers) {
+            for (Class walMgrClass : checkWalManagers) {
+                try {
+                    log.info("checking wal manager " + walMgrClass + " with serializer version " + serVer);
+
+                    checkInvariantSwitchSegment(walMgrClass, serVer);
+                }
+                finally {
+                    U.delete(Paths.get(U.defaultWorkDirectory()));
+                }
+            }
+        }
+    }
+
+    /**
+     * @param walMgrClass WAL manager class.
+     * @param serVer WAL serializer version.
+     * @throws Exception If some thing failed.
+     */
+    private void checkInvariantSwitchSegment(Class walMgrClass, int serVer) throws Exception {
+        String workDir = U.defaultWorkDirectory();
+
+        T2<IgniteWriteAheadLogManager, RecordSerializer> initTup = initiate(walMgrClass, serVer, workDir);
+
+        IgniteWriteAheadLogManager walMgr = initTup.get1();
+
+        RecordSerializer recordSerializer = initTup.get2();
+
+        int switchSegmentRecordSize = recordSerializer.size(new SwitchSegmentRecord());
+
+        log.info("switchSegmentRecordSize:" + switchSegmentRecordSize);
+
+        int tailSize = 0;
+
+        /* Initial record payload size. */
+        int payloadSize = 1024;
+
+        int recSize = 0;
+
+        MetastoreDataRecord rec = null;
+
+        /* Record size. */
+        int recordTypeSize = 1;
+
+        /* Record pointer. */
+        int recordPointerSize = 8 + 4 + 4;
+
+        int lowBound = recordTypeSize + recordPointerSize;
+        int highBound = lowBound + /*CRC*/4;
+
+        int attempt = 1000;
+
+        // Try find how many record need for specific tail size.
+        while (true) {
+            if (attempt < 0)
+                throw new IgniteCheckedException("Can not find any payload size for test, " +
+                    "lowBound=" + lowBound + ", highBound=" + highBound);
+
+            if (tailSize >= lowBound && tailSize < highBound)
+                break;
+
+            payloadSize++;
+
+            byte[] payload = new byte[payloadSize];
+
+            // Fake record for payload.
+            rec = new MetastoreDataRecord("0", payload);
+
+            recSize = recordSerializer.size(rec);
+
+            tailSize = (SEGMENT_SIZE - HEADER_RECORD_SIZE) % recSize;
+
+            attempt--;
+        }
+
+        Assert.assertNotNull(rec);
+
+        int recordsToWrite = SEGMENT_SIZE / recSize;
+
+        log.info("records to write " + recordsToWrite + " tail size " +
+            (SEGMENT_SIZE - HEADER_RECORD_SIZE) % recSize);
+
+        // Add more record for rollover to the next segment.
+        recordsToWrite += 100;
+
+        for (int i = 0; i < recordsToWrite; i++) {
+            walMgr.log(new MetastoreDataRecord(rec.key(), rec.value()));
+        }
+
+        walMgr.flush(null, true);
+
+        // Await archiver move segment to WAL archive.
+        Thread.sleep(5000);
+
+        // If switchSegmentRecordSize more that 1, it mean that invariant is broke.
+        // Filling tail some garbage. Simulate tail garbage on rotate segment in WAL work directory.
+        if (switchSegmentRecordSize > 1) {
+            File seg = new File(workDir + ARCHIVE_SUB_DIR + "/0000000000000000.wal");
+
+            FileIOFactory ioFactory = new RandomAccessFileIOFactory();
+
+            FileIO seg0 = ioFactory.create(seg);
+
+            byte[] bytes = new byte[tailSize];
+
+            Random rnd = new Random();
+
+            rnd.nextBytes(bytes);
+
+            // Some record type.
+            bytes[0] = (byte)(METASTORE_DATA_RECORD.ordinal() + 1);
+
+            seg0.position((int)(seg0.size() - tailSize));
+
+            seg0.write(bytes, 0, tailSize);
+
+            seg0.force(true);
+
+            seg0.close();
+        }
+
+        int expectedRecords = recordsToWrite;
+        int actualRecords = 0;
+
+        // Check that switch segment works as expected and all record is reachable.
+        try (WALIterator it = walMgr.replay(null)) {
+            while (it.hasNext()) {
+                IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+                WALRecord rec0 = tup.get2();
+
+                if (rec0.type() == METASTORE_DATA_RECORD)
+                    actualRecords++;
+            }
+        }
+
+        Assert.assertEquals("Not all records read during iteration.", expectedRecords, actualRecords);
+    }
+
+    /***
+     * Initiate WAL manager.
+     *
+     * @param walMgrClass WAL manager class.
+     * @param serVer WAL serializer version.
+     * @param workDir Work directory path.
+     * @return Tuple of WAL manager and WAL record serializer.
+     * @throws IgniteCheckedException If some think failed.
+     */
+    private T2<IgniteWriteAheadLogManager, RecordSerializer> initiate(
+        Class walMgrClass,
+        int serVer,
+        String workDir
+    ) throws IgniteCheckedException {
+
+        GridKernalContext kctx = new StandaloneGridKernalContext(
+            log, null, null
+        ) {
+            @Override protected IgniteConfiguration prepareIgniteConfiguration() {
+                IgniteConfiguration cfg = super.prepareIgniteConfiguration();
+
+                cfg.setDataStorageConfiguration(
+                    new DataStorageConfiguration()
+                        .setWalSegmentSize(SEGMENT_SIZE)
+                        .setWalMode(WALMode.FSYNC)
+                        .setWalPath(workDir + WORK_SUB_DIR)
+                        .setWalArchivePath(workDir + ARCHIVE_SUB_DIR)
+                );
+
+                cfg.setEventStorageSpi(new NoopEventStorageSpi());
+
+                return cfg;
+            }
+
+            @Override public GridInternalSubscriptionProcessor internalSubscriptionProcessor() {
+                return new GridInternalSubscriptionProcessor(this);
+            }
+
+            @Override public GridEventStorageManager event() {
+                return new GridEventStorageManager(this);
+            }
+        };
+
+        IgniteWriteAheadLogManager walMgr = null;
+
+        if (walMgrClass.equals(FileWriteAheadLogManager.class)) {
+            walMgr = new FileWriteAheadLogManager(kctx);
+
+            GridTestUtils.setFieldValue(walMgr, "serializerVer", serVer);
+        }
+        else if (walMgrClass.equals(FsyncModeFileWriteAheadLogManager.class)) {
+            walMgr = new FsyncModeFileWriteAheadLogManager(kctx);
+
+            GridTestUtils.setFieldValue(walMgr, "serializerVersion", serVer);
+        }
+
+        GridCacheSharedContext<?, ?> ctx = new GridCacheSharedContext<>(
+            kctx,
+            null,
+            null,
+            null,
+            null,
+            walMgr,
+            null,
+            new GridCacheDatabaseSharedManager(kctx),
+            null,
+            null,
+            null,
+            null,
+            new GridCacheIoManager(),
+            null,
+            null,
+            null
+        );
+
+        walMgr.start(ctx);
+
+        walMgr.onActivate(kctx);
+
+        walMgr.resumeLogging(null);
+
+        RecordSerializer recordSerializer = new RecordSerializerFactoryImpl(ctx)
+            .createSerializer(walMgr.serializerVersion());
+
+        return new T2<>(walMgr, recordSerializer);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 4e7d8e323e559..8f99c1d148a60 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -46,6 +46,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFailoverTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlySelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorSwitchSegmentTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalSerializerVersionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalCompactionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteDataIntegrityTests;
@@ -151,5 +152,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgnitePdsCorruptedStoreTest.class);
 
         suite.addTestSuite(LocalWalModeChangeDuringRebalancingSelfTest.class);
+
+        suite.addTestSuite(IgniteWalIteratorSwitchSegmentTest.class);
     }
 }

From ef4782d8a60eea870b43933480f2c9695c9fc55d Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 7 Jun 2018 14:55:13 +0300
Subject: [PATCH 181/543] IGNITE-8706 Fixed testMemoryUsageMultipleNodes

---
 .../processors/cache/persistence/file/FilePageStore.java      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index ae4880d22715c..852eb0da870b1 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -279,9 +279,9 @@ public void truncate(int tag) throws PersistentStorageIOException {
             throw new PersistentStorageIOException("Failed to delete partition file: " + cfgFile.getPath(), e);
         }
         finally {
-            allocated.set(0);
+            allocatedTracker.updateTotalAllocatedPages(-1L * pages());
 
-            allocatedTracker.updateTotalAllocatedPages(-1L * this.pages());
+            allocated.set(0);
 
             inited = false;
 

From 6902383e01ee69d7efff9613a68f3fc419152305 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Mon, 14 May 2018 13:14:52 +0300
Subject: [PATCH 182/543] IGNITE-8422 Tests fix for Zookeeper discovery split
 brain detection shouldn't consider client nodes - Fixes #3975.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry-picked from commit #19cbf8058b48a1f8fec60f7ceac70229e7c5391a)
---
 .../internal/ZookeeperDiscoverySpiTest.java   | 33 +++++++++++++++++--
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index 03b874dce1bf9..09db1049211cc 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -127,6 +127,7 @@
 import org.apache.zookeeper.ZkTestClientCnxnSocketNIO;
 import org.apache.zookeeper.ZooKeeper;
 import org.jetbrains.annotations.Nullable;
+import org.junit.Assert;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.SECONDS;
@@ -3589,6 +3590,10 @@ public void testSimpleSplitBrain() throws Exception {
 
         startGridsMultiThreaded(5, 3);
 
+        client = false;
+
+        awaitPartitionMapExchange();
+
         List<ClusterNode> all = G.allGrids().stream()
             .map(g -> g.cluster().localNode())
             .collect(Collectors.toList());;
@@ -3600,6 +3605,8 @@ public void testSimpleSplitBrain() throws Exception {
 
         ClusterNode lastClient = startGrid(8).cluster().localNode();
 
+        awaitPartitionMapExchange();
+
         // Make last client connected to other nodes.
         for (ClusterNode node : all) {
             if (node.id().equals(lastClient.id()))
@@ -3633,15 +3640,21 @@ public void testNotActualSplitBrain() throws Exception {
             .map(g -> g.cluster().localNode())
             .collect(Collectors.toList());
 
+        Assert.assertEquals(5, srvNodes.size());
+
         client = true;
 
         startGridsMultiThreaded(5, 3);
 
         client = false;
 
+        awaitPartitionMapExchange();
+
         ConnectionsFailureMatrix matrix = new ConnectionsFailureMatrix();
 
-        matrix.addAll(G.allGrids().stream().map(g -> g.cluster().localNode()).collect(Collectors.toList()));
+        List<ClusterNode> allNodes = G.allGrids().stream().map(g -> g.cluster().localNode()).collect(Collectors.toList());
+
+        matrix.addAll(allNodes);
 
         // Remove 2 connections between server nodes.
         matrix.removeConnection(srvNodes.get(0), srvNodes.get(1));
@@ -3674,6 +3687,8 @@ public void testAlmostSplitBrain() throws Exception {
             .map(g -> g.cluster().localNode())
             .collect(Collectors.toList());
 
+        Assert.assertEquals(6, srvNodes.size());
+
         List<ClusterNode> srvPart1 = srvNodes.subList(0, 3);
         List<ClusterNode> srvPart2 = srvNodes.subList(3, srvNodes.size());
 
@@ -3683,11 +3698,15 @@ public void testAlmostSplitBrain() throws Exception {
 
         client = false;
 
+        awaitPartitionMapExchange();
+
         List<ClusterNode> clientNodes = G.allGrids().stream()
             .map(g -> g.cluster().localNode())
             .filter(ClusterNode::isClient)
             .collect(Collectors.toList());
 
+        Assert.assertEquals(5, clientNodes.size());
+
         List<ClusterNode> clientPart1 = clientNodes.subList(0, 2);
         List<ClusterNode> clientPart2 = clientNodes.subList(2, 4);
 
@@ -3793,17 +3812,24 @@ static class PeerToPeerCommunicationFailureSpi extends TcpCommunicationSpi {
         private static volatile boolean failure;
 
         /** Connections failure matrix. */
-        private static ConnectionsFailureMatrix matrix;
+        private static volatile ConnectionsFailureMatrix matrix;
 
         /**
          * Start failing connections according to given matrix {@code with}.
          * @param with Failure matrix.
          */
-        public static void fail(ConnectionsFailureMatrix with) {
+        static void fail(ConnectionsFailureMatrix with) {
             matrix = with;
             failure = true;
         }
 
+        /**
+         * Resets failure matrix.
+         */
+        static void unfail() {
+            failure = false;
+        }
+
         /** {@inheritDoc} */
         @Override public IgniteFuture<BitSet> checkConnection(List<ClusterNode> nodes) {
             // Creates connections statuses according to failure matrix.
@@ -4264,6 +4290,7 @@ private void reset() {
         err = false;
 
         failCommSpi = false;
+        PeerToPeerCommunicationFailureSpi.unfail();
 
         evts.clear();
 

From 6eef070f8e0627e2986d870cc09ceb9cb351ee0d Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Mon, 14 May 2018 20:09:33 +0300
Subject: [PATCH 183/543] IGNITE-8487 tmpfs is employed by ZooKeeper tests -
 Fixes #3994.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry-picked from commit #67be050a2c7b9efdef2388c579fc060c68ff8a38)
---
 .../zk/ZookeeperDiscoverySpiAbstractTestSuite.java         | 7 ++++++-
 .../spi/discovery/zk/internal/ZookeeperClientTest.java     | 3 ++-
 .../discovery/zk/internal/ZookeeperDiscoverySpiTest.java   | 5 +++--
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
index 766635cb5580c..c5d34884a8c08 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
@@ -76,7 +76,12 @@ public synchronized static void preprocessConfiguration(IgniteConfiguration cfg)
      * @return Test cluster.
      */
     public static TestingCluster createTestingCluster(int instances) {
-        String tmpDir = System.getProperty("java.io.tmpdir");
+        String tmpDir;
+
+        if (System.getenv("TMPFS_ROOT") != null)
+            tmpDir = System.getenv("TMPFS_ROOT");
+        else
+            tmpDir = System.getProperty("java.io.tmpdir");
 
         List<InstanceSpec> specs = new ArrayList<>();
 
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
index e7cb97a39434f..d228e03070ca7 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiAbstractTestSuite;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.zookeeper.AsyncCallback;
@@ -462,7 +463,7 @@ public void testReconnect4() throws Exception {
     private void startZK(int instances) throws Exception {
         assert zkCluster == null;
 
-        zkCluster = new TestingCluster(instances);
+        zkCluster = ZookeeperDiscoverySpiAbstractTestSuite.createTestingCluster(instances);
 
         zkCluster.start();
     }
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index 09db1049211cc..1585870053b87 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -118,6 +118,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpi;
+import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiAbstractTestSuite;
 import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiMBean;
 import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiTestSuite2;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -421,7 +422,7 @@ private void clearAckEveryEventSystemProperty() {
         super.beforeTest();
 
         if (USE_TEST_CLUSTER && zkCluster == null) {
-            zkCluster = ZookeeperDiscoverySpiTestSuite2.createTestingCluster(ZK_SRVS);
+            zkCluster = ZookeeperDiscoverySpiAbstractTestSuite.createTestingCluster(ZK_SRVS);
 
             zkCluster.start();
         }
@@ -4315,7 +4316,7 @@ private IgniteInternalFuture<?> startRestartZkServers(final long stopTime, final
                 ThreadLocalRandom rnd = ThreadLocalRandom.current();
 
                 while (!stop.get() && System.currentTimeMillis() < stopTime) {
-                    U.sleep(rnd.nextLong(500) + 500);
+                    U.sleep(rnd.nextLong(2500) + 2500);
 
                     int idx = rnd.nextInt(ZK_SRVS);
 

From c9b82cc632ad1cac51661f729f8605ab6c97c1dd Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Fri, 8 Jun 2018 18:24:09 +0300
Subject: [PATCH 184/543] IGNITE-8739 Implemented WA for tcp communication SPI
 hanging on descriptor reservation - Fixes #4148.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 7021651)
---
 .../apache/ignite/IgniteSystemProperties.java |   6 +
 .../util/nio/GridNioRecoveryDescriptor.java   |  52 ++++++-
 .../internal/util/nio/GridNioSessionImpl.java |   5 +
 .../util/nio/GridSelectorNioSessionImpl.java  |  23 +++
 .../tcp/TcpCommunicationSpi.java              |  20 ++-
 ...nectionConcurrentReserveAndRemoveTest.java | 146 ++++++++++++++++++
 .../testsuites/IgniteKernalSelfTestSuite.java |   2 +
 7 files changed, 248 insertions(+), 6 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 6f98d76ed0e10..3f47d76f26584 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -887,6 +887,12 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_DISABLE_WAL_DURING_REBALANCING = "IGNITE_DISABLE_WAL_DURING_REBALANCING";
 
+    /**
+     * Sets timeout for TCP client recovery descriptor reservation.
+     */
+    public static final String IGNITE_NIO_RECOVERY_DESCRIPTOR_RESERVATION_TIMEOUT =
+            "IGNITE_NIO_RECOVERY_DESCRIPTOR_RESERVATION_TIMEOUT";
+
     /**
      * Enables threads dumping on critical node failure.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
index af7b757762982..bd1291aced450 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioRecoveryDescriptor.java
@@ -22,16 +22,24 @@
 import java.util.Deque;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_NIO_RECOVERY_DESCRIPTOR_RESERVATION_TIMEOUT;
+
 /**
  * Recovery information for single node.
  */
 public class GridNioRecoveryDescriptor {
+    /** Timeout for outgoing recovery descriptor reservation. */
+    private static final long DESC_RESERVATION_TIMEOUT =
+        Math.max(1_000, IgniteSystemProperties.getLong(IGNITE_NIO_RECOVERY_DESCRIPTOR_RESERVATION_TIMEOUT, 5_000));
+
     /** Number of acknowledged messages. */
     private long acked;
 
@@ -80,6 +88,10 @@ public class GridNioRecoveryDescriptor {
     /** */
     private final boolean pairedConnections;
 
+    /** Session for the descriptor. */
+    @GridToStringExclude
+    private GridNioSession ses;
+
     /**
      * @param pairedConnections {@code True} if in/out connections pair is used for communication with node.
      * @param queueLimit Maximum size of unacknowledged messages queue.
@@ -271,8 +283,19 @@ public boolean nodeAlive(@Nullable ClusterNode node) {
      */
     public boolean reserve() throws InterruptedException {
         synchronized (this) {
-            while (!connected && reserved)
-                wait();
+            long t0 = System.nanoTime();
+
+            while (!connected && reserved) {
+                wait(DESC_RESERVATION_TIMEOUT);
+
+                if ((System.nanoTime() - t0) / 1_000_000 >= DESC_RESERVATION_TIMEOUT - 100) {
+                    // Dumping a descriptor.
+                    log.error("Failed to wait for recovery descriptor reservation " +
+                        "[desc=" + this + ", ses=" + ses + ']');
+
+                    return false;
+                }
+            }
 
             if (!connected) {
                 reserved = true;
@@ -354,6 +377,8 @@ public void release() {
         SessionWriteRequest[] futs = null;
 
         synchronized (this) {
+            ses = null;
+
             connected = false;
 
             if (handshakeReq != null) {
@@ -439,17 +464,36 @@ public int reserveCount() {
         }
     }
 
+    /**
+     * @return Current session.
+     */
+    public synchronized GridNioSession session() {
+        return ses;
+    }
+
+    /**
+     * @param ses Session.
+     */
+    public synchronized void session(GridNioSession ses) {
+        this.ses = ses;
+    }
+
     /**
      * @param reqs Requests to notify about error.
      */
     private void notifyOnNodeLeft(SessionWriteRequest[] reqs) {
         IOException e = new IOException("Failed to send message, node has left: " + node.id());
+        IgniteException cloErr = null;
 
         for (SessionWriteRequest req : reqs) {
             req.onError(e);
 
-            if (req.ackClosure() != null)
-                req.ackClosure().apply(new IgniteException(e));
+            if (req.ackClosure() != null) {
+                if (cloErr == null)
+                    cloErr = new IgniteException(e);
+
+                req.ackClosure().apply(cloErr);
+            }
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
index 98a22d6c350dd..c6410c40a360a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioSessionImpl.java
@@ -77,6 +77,9 @@ public class GridNioSessionImpl implements GridNioSession {
     /** Accepted flag. */
     private final boolean accepted;
 
+    /** For debug purposes. */
+    private volatile boolean markedForClose;
+
     /**
      * @param filterChain Chain.
      * @param locAddr Local address.
@@ -156,6 +159,8 @@ public GridNioSessionImpl(
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public GridNioFuture<Boolean> close() {
+        markedForClose = true;
+
         try {
             return filterChain.onSessionClose(this);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
index d30b122cb2970..d9c3cae4fa92d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridSelectorNioSessionImpl.java
@@ -26,6 +26,7 @@
 import java.util.concurrent.ConcurrentLinkedDeque;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.LT;
@@ -377,6 +378,8 @@ Collection<SessionWriteRequest> writeQueue() {
         assert recoveryDesc != null;
 
         outRecovery = recoveryDesc;
+
+        outRecovery.session(this);
     }
 
     /** {@inheritDoc} */
@@ -436,6 +439,26 @@ Object systemMessage() {
         return ret;
     }
 
+    /** {@inheritDoc} */
+    @Override public GridNioFuture<Boolean> close() {
+        GridNioFuture<Boolean> fut = super.close();
+
+        if (!fut.isDone()) {
+            fut.listen(fut0 -> {
+                try {
+                    fut0.get();
+                }
+                catch (IgniteCheckedException e) {
+                    log.error("Failed to close session [ses=" + GridSelectorNioSessionImpl.this + ']', e);
+                }
+            });
+        }
+        else if (fut.error() != null)
+            log.error("Failed to close session [ses=" + GridSelectorNioSessionImpl.this + ']', fut.error());
+
+        return fut;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridSelectorNioSessionImpl.class, this, super.toString());
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index f9fd6fd504ad1..7583d96dd4af2 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -2714,8 +2714,16 @@ private void sendMessage0(ClusterNode node, Message msg, IgniteInClosure<IgniteE
                 }
                 while (retry);
             }
-            catch (IgniteCheckedException e) {
-                throw new IgniteSpiException("Failed to send message to remote node: " + node, e);
+            catch (Throwable t) {
+                log.error("Failed to send message to remote node [node=" + node + ", msg=" + msg + ']', t);
+
+                if (t instanceof Error)
+                    throw (Error)t;
+
+                if (t instanceof RuntimeException)
+                    throw (RuntimeException)t;
+
+                throw new IgniteSpiException("Failed to send message to remote node: " + node, t);
             }
             finally {
                 if (client != null && removeNodeClient(node.id(), client))
@@ -3249,6 +3257,14 @@ protected GridCommunicationClient createTcpClient(ClusterNode node, int connIdx)
                     if (!recoveryDesc.reserve()) {
                         U.closeQuiet(ch);
 
+                        // Ensure the session is closed.
+                        GridNioSession ses = recoveryDesc.session();
+
+                        if (ses != null) {
+                            while(ses.closeTime() == 0)
+                                ses.close();
+                        }
+
                         return null;
                     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java
new file mode 100644
index 0000000000000..fb194491b7c6b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/IgniteConnectionConcurrentReserveAndRemoveTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.nio.GridCommunicationClient;
+import org.apache.ignite.internal.util.nio.GridTcpNioCommunicationClient;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.communication.tcp.messages.HandshakeMessage2;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ */
+public class IgniteConnectionConcurrentReserveAndRemoveTest extends GridCommonAbstractTest {
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration c = super.getConfiguration(igniteInstanceName);
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration().setDefaultDataRegionConfiguration(
+            new DataRegionConfiguration().setMaxSize(50 * 1024 * 1024));
+
+        c.setDataStorageConfiguration(memCfg);
+
+        c.setClientMode(igniteInstanceName.startsWith("client"));
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(ipFinder);
+
+        c.setDiscoverySpi(disco);
+
+        TestRecordingCommunicationSpi spi = new TestRecordingCommunicationSpi();
+        spi.setIdleConnectionTimeout(Integer.MAX_VALUE);
+
+        c.setCommunicationSpi(spi);
+
+        return c;
+    }
+
+    /** */
+    private static final class TestClosure implements IgniteCallable<Integer> {
+        /** Serial version uid. */
+        private static final long serialVersionUid = 0L;
+
+        /** {@inheritDoc} */
+        @Override public Integer call() throws Exception {
+            return 1;
+        }
+    }
+
+
+    public void test() throws Exception {
+        IgniteEx svr = startGrid(0);
+
+        Ignite c1 = startGrid("client1");
+
+        assertTrue(c1.configuration().isClientMode());
+
+        Ignite c2 = startGrid("client2");
+
+        assertTrue(c2.configuration().isClientMode());
+
+        TestRecordingCommunicationSpi spi2 = (TestRecordingCommunicationSpi)c1.configuration().getCommunicationSpi();
+
+        spi2.blockMessages(HandshakeMessage2.class, c1.name());
+
+        AtomicInteger cnt = new AtomicInteger();
+
+        cnt.getAndAdd(c1.compute(c1.cluster().forNodeId(c2.cluster().localNode().id())).call(new TestClosure()));
+
+        TcpCommunicationSpi spi1 = (TcpCommunicationSpi)c1.configuration().getCommunicationSpi();
+
+        ConcurrentMap<UUID, GridCommunicationClient[]> clientsMap = U.field(spi1, "clients");
+
+        GridCommunicationClient[] arr = clientsMap.get(c2.cluster().localNode().id());
+
+        GridTcpNioCommunicationClient client = null;
+
+        for (GridCommunicationClient c : arr) {
+            client = (GridTcpNioCommunicationClient)c;
+
+            if(client != null) {
+                assertTrue(client.session().outRecoveryDescriptor().reserved());
+
+                assertFalse(client.session().outRecoveryDescriptor().connected());
+            }
+        }
+
+        assertNotNull(client);
+
+        //spi1.failSend = true;
+
+        IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                doSleep(1000);
+
+                //spi1.failSend = false;
+
+                cnt.getAndAdd(c1.compute(c1.cluster().forNodeId(c2.cluster().localNode().id())).call(new TestClosure()));
+            }
+        }, 1, "hang-thread");
+
+        try {
+            cnt.getAndAdd(c1.compute(c1.cluster().forNodeId(c2.cluster().localNode().id())).call(new TestClosure()));
+
+            //fail();
+        }
+        catch (IgniteException e) {
+            // Expected.
+        }
+
+        fut.get();
+
+        assertEquals(3, cnt.get());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index f26205bf79538..5306aebd7611a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.internal.GridSpiExceptionSelfTest;
 import org.apache.ignite.internal.GridVersionSelfTest;
 import org.apache.ignite.internal.IgniteConcurrentEntryProcessorAccessStopTest;
+import org.apache.ignite.internal.IgniteConnectionConcurrentReserveAndRemoveTest;
 import org.apache.ignite.internal.IgniteUpdateNotifierPerClusterSettingSelfTest;
 import org.apache.ignite.internal.managers.GridManagerStopSelfTest;
 import org.apache.ignite.internal.managers.communication.GridCommunicationSendMessageSelfTest;
@@ -125,6 +126,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(GridLocalEventListenerSelfTest.class);
         suite.addTestSuite(IgniteTopologyPrintFormatSelfTest.class);
         suite.addTestSuite(ComputeJobCancelWithServiceSelfTest.class);
+        suite.addTestSuite(IgniteConnectionConcurrentReserveAndRemoveTest.class);
 
         // Managed Services.
         suite.addTestSuite(GridServiceProcessorSingleNodeSelfTest.class);

From 45953256ef0f09fd5450f6f6c6967a539c5cd395 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 3 May 2018 17:01:20 +0300
Subject: [PATCH 185/543] IGNITE-8226 Improved logging - Fixes #3796.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 8dd9c5d61c86c831b490134c3cd21f2ed47758d2)
---
 .../dht/GridClientPartitionTopology.java      |  6 ++----
 .../dht/GridDhtPartitionTopologyImpl.java     | 20 +++++++++++++------
 .../dht/preloader/GridDhtPartitionMap.java    |  5 +++++
 3 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index dcb8b96db26ce..a9c4b35ba48f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -857,16 +857,14 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
 
     /**
      * Method checks is new partition map more stale than current partition map
-     * New partition map is stale if topology version or update sequence are less than of current map
+     * New partition map is stale if topology version or update sequence are less or equal than of current map
      *
      * @param currentMap Current partition map
      * @param newMap New partition map
      * @return True if new partition map is more stale than current partition map, false in other case
      */
     private boolean isStaleUpdate(GridDhtPartitionMap currentMap, GridDhtPartitionMap newMap) {
-        return currentMap != null &&
-            (newMap.topologyVersion().compareTo(currentMap.topologyVersion()) < 0 ||
-            newMap.topologyVersion().compareTo(currentMap.topologyVersion()) == 0 && newMap.updateSequence() <= currentMap.updateSequence());
+        return currentMap != null && newMap.compareTo(currentMap) <= 0;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 3d664f3eafb08..44ec16bb7ee73 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -1660,16 +1660,14 @@ else if (part.updateCounter() > 0) {
 
     /**
      * Method checks is new partition map more stale than current partition map
-     * New partition map is stale if topology version or update sequence are less than of current map
+     * New partition map is stale if topology version or update sequence are less or equal than of current map
      *
      * @param currentMap Current partition map
      * @param newMap New partition map
      * @return True if new partition map is more stale than current partition map, false in other case
      */
     private boolean isStaleUpdate(GridDhtPartitionMap currentMap, GridDhtPartitionMap newMap) {
-        return currentMap != null &&
-            (newMap.topologyVersion().compareTo(currentMap.topologyVersion()) < 0 ||
-            newMap.topologyVersion().compareTo(currentMap.topologyVersion()) == 0 && newMap.updateSequence() <= currentMap.updateSequence());
+        return currentMap != null && newMap.compareTo(currentMap) <= 0;
     }
 
     /** {@inheritDoc} */
@@ -1725,11 +1723,21 @@ private boolean isStaleUpdate(GridDhtPartitionMap currentMap, GridDhtPartitionMa
                         parts.updateSequence(cur.updateSequence(), cur.topologyVersion());
                 }
                 else if (isStaleUpdate(cur, parts)) {
-                    U.warn(log, "Stale update for single partition map update (will ignore) [" +
+                    assert cur != null;
+
+                    String msg = "Stale update for single partition map update (will ignore) [" +
                         "grp=" + grp.cacheOrGroupName() +
                         ", exchId=" + exchId +
                         ", curMap=" + cur +
-                        ", newMap=" + parts + ']');
+                        ", newMap=" + parts + ']';
+
+                    // This is usual situation when partition maps are equal, just print debug message.
+                    if (cur.compareTo(parts) == 0) {
+                        if (log.isDebugEnabled())
+                            log.debug(msg);
+                    }
+                    else
+                        U.warn(log, msg);
 
                     return false;
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java
index cb697149639fa..28c8c8453faad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java
@@ -243,6 +243,11 @@ public AffinityTopologyVersion topologyVersion() {
     @Override public int compareTo(GridDhtPartitionMap o) {
         assert nodeId.equals(o.nodeId);
 
+        int topVerCompare = top.compareTo(o.top);
+
+        if (topVerCompare != 0)
+            return topVerCompare;
+
         return Long.compare(updateSeq, o.updateSeq);
     }
 

From 77d089d857876f1901ade5a017a92c9835e2976a Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Wed, 13 Jun 2018 17:24:42 +0300
Subject: [PATCH 186/543] IGNITE-8736 Add transaction label to CU.txString()
 method output - Fixes #4152.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../ignite/internal/processors/cache/GridCacheUtils.java      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 4f349a4b5c385..2520f65e54311 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -827,7 +827,9 @@ public static String txString(@Nullable IgniteInternalTx tx) {
             ", rollbackOnly=" + tx.isRollbackOnly() +
             ", nodeId=" + tx.nodeId() +
             ", timeout=" + tx.timeout() +
-            ", duration=" + (U.currentTimeMillis() - tx.startTime()) + ']';
+            ", duration=" + (U.currentTimeMillis() - tx.startTime()) +
+            (tx instanceof GridNearTxLocal ? ", label=" + ((GridNearTxLocal)tx).label() : "") +
+            ']';
     }
 
     /**

From c3c80439945134867e28e8d2c6db5ef9295141b6 Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Sat, 9 Jun 2018 16:37:49 +0300
Subject: [PATCH 187/543] IGNITE-8751 Failure handler accordingly to
 segmentation policy should be invoked on node segmentation instead of
 configured failure handler

---
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 11 ++-
 .../TcpDiscoverySegmentationPolicyTest.java   | 91 +++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |  3 +
 3 files changed, 101 insertions(+), 4 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySegmentationPolicyTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 35e945208eb65..079058b799287 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1687,7 +1687,10 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
 
     /** {@inheritDoc} */
     @Override public void brakeConnection() {
-        throw new UnsupportedOperationException();
+        Socket sock = msgWorker.sock;
+
+        if (sock != null)
+            U.closeQuiet(sock);
     }
 
     /** {@inheritDoc} */
@@ -2621,7 +2624,7 @@ void addMessage(TcpDiscoveryAbstractMessage msg) {
                 super.body();
             }
             catch (InterruptedException e) {
-                if (!spi.isNodeStopping0())
+                if (!spi.isNodeStopping0() && spiStateCopy() != DISCONNECTING)
                     err = e;
 
                 throw e;
@@ -2657,7 +2660,7 @@ void addMessage(TcpDiscoveryAbstractMessage msg) {
                 throw e;
             }
             finally {
-                if (err == null && !spi.isNodeStopping0())
+                if (err == null && !spi.isNodeStopping0() && spiStateCopy() != DISCONNECTING)
                     err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
 
                 FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
@@ -5682,7 +5685,7 @@ private class TcpServer extends IgniteSpiThread {
                 throw t;
             }
             finally {
-                if (err == null && !spi.isNodeStopping0())
+                if (err == null && !spi.isNodeStopping0() && spiStateCopy() != DISCONNECTING)
                     err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
 
                 FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySegmentationPolicyTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySegmentationPolicyTest.java
new file mode 100644
index 0000000000000..df76afc2317e4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySegmentationPolicyTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for segmentation policy and failure handling in {@link TcpDiscoverySpi}.
+ */
+public class TcpDiscoverySegmentationPolicyTest extends GridCommonAbstractTest {
+    /** Nodes count. */
+    private static final int NODES_CNT = 3;
+
+    /** Default failure handler invoked. */
+    private static volatile boolean dfltFailureHndInvoked;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if (igniteInstanceName.endsWith("2"))
+            cfg.setFailureHandler(new TestFailureHandler());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStopOnSegmentation() throws Exception {
+        startGrids(NODES_CNT);
+
+        IgniteEx ignite1 = grid(1);
+        IgniteEx ignite2 = grid(2);
+
+        ((TcpDiscoverySpi)ignite1.configuration().getDiscoverySpi()).brakeConnection();
+        ((TcpDiscoverySpi)ignite2.configuration().getDiscoverySpi()).brakeConnection();
+
+        waitForTopology(2);
+
+        assertFalse(dfltFailureHndInvoked);
+
+        Collection<ClusterNode> nodes = ignite1.cluster().forServers().nodes();
+
+        assertEquals(2, nodes.size());
+        assertTrue(nodes.containsAll(Arrays.asList(((IgniteKernal)ignite(0)).localNode(), ((IgniteKernal)ignite(1)).localNode())));
+
+        System.out.println();
+    }
+
+    /**
+     * Test failure handler.
+     */
+    private static class TestFailureHandler implements FailureHandler {
+        /** {@inheritDoc} */
+        @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+            dfltFailureHndInvoked = true;
+
+            return true;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 1f82316e5a99b..ef582a5df7565 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeConfigConsistentIdSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeConsistentIdSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryRestartTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySegmentationPolicyTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySnapshotHistoryTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiConfigSelfTest;
@@ -96,6 +97,8 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(TcpDiscoveryRestartTest.class));
         suite.addTest(new TestSuite(TcpDiscoveryMultiThreadedTest.class));
 
+        suite.addTest(new TestSuite(TcpDiscoverySegmentationPolicyTest.class));
+
         suite.addTest(new TestSuite(TcpDiscoveryNodeAttributesUpdateOnReconnectTest.class));
         suite.addTest(new TestSuite(AuthenticationRestartTest.class));
 

From d33fe100bc9fdbcb66754d43ec0199e44233190b Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Wed, 13 Jun 2018 17:30:18 +0300
Subject: [PATCH 188/543] IGNITE-8781 GridNioServer accepter threads should
 have different names

---
 .../ignite/internal/util/nio/GridNioServer.java       | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index da3438e069414..03870dbb3e0d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -338,7 +338,16 @@ private GridNioServer(
             // This method will throw exception if address already in use.
             Selector acceptSelector = createSelector(locAddr);
 
-            acceptThread = new IgniteThread(new GridNioAcceptWorker(igniteInstanceName, "nio-acceptor", log, acceptSelector));
+            String threadName;
+
+            if (srvName == null)
+                threadName = "nio-acceptor";
+            else
+                threadName = "nio-acceptor-" + srvName;
+
+            GridNioAcceptWorker w = new GridNioAcceptWorker(igniteInstanceName, threadName, log, acceptSelector);
+
+            acceptThread = new IgniteThread(w);
         }
         else {
             locAddr = null;

From d2c304ca415919f0933461dde92fdf354539529b Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Thu, 14 Jun 2018 20:41:31 +0300
Subject: [PATCH 189/543] IGNITE-8763 java.nio.file.AccessDeniedException is
 not handled with default failure handler

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../GridCacheDatabaseSharedManager.java       |  6 ++-
 .../IgnitePdsCorruptedStoreTest.java          | 50 ++++++++++++++++++-
 2 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 6064cfe2775cd..3ee8f28d6d3de 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -146,6 +146,7 @@
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -825,8 +826,9 @@ private void unRegistrateMetricsMBean() {
 
             notifyMetastorageReadyForReadWrite();
         }
-        catch (StorageException | PersistentStorageIOException e) {
-            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+        catch (IgniteCheckedException e) {
+            if (X.hasCause(e, StorageException.class, PersistentStorageIOException.class, IOException.class))
+                cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
index d4a135dccca5d..ff95f97a1fc78 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.cache.persistence;
 
 import java.io.File;
+import java.io.IOException;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
@@ -33,14 +34,18 @@
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.failure.FailureHandler;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.wal.StorageException;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.PersistentStorageIOException;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -303,13 +308,56 @@ private void corruptTreeRoot(IgniteEx ignite, PageMemoryEx pageMem, int grpId, i
         }
     }
 
+    /**
+     * Test node invalidation when meta store is read only.
+     */
+    public void testReadOnlyMetaStore() throws Exception {
+        IgniteEx ignite0 = startGrid(0);
+
+        ignite0.cluster().active(true);
+
+        IgniteInternalCache cache = ignite0.cachex(CACHE_NAME1);
+
+        cache.put(1, 1);
+
+        ignite0.cluster().active(false);
+
+        FilePageStoreManager storeMgr = ((FilePageStoreManager)ignite0.context().cache().context().pageStore());
+
+        File workDir = storeMgr.workDir();
+        File metaStoreDir = new File(workDir, MetaStorage.METASTORAGE_CACHE_NAME.toLowerCase());
+        File metaStoreFile = new File(metaStoreDir, String.format(FilePageStoreManager.PART_FILE_TEMPLATE, 0));
+
+        metaStoreFile.setWritable(false);
+
+        try {
+            IgniteInternalFuture fut = GridTestUtils.runAsync(new Runnable() {
+                @Override public void run() {
+                    try {
+                        ignite0.cluster().active(true);
+                    }
+                    catch (Exception ignore) {
+                        // No-op.
+                    }
+                }
+            });
+
+            waitFailure(IOException.class);
+
+            fut.cancel();
+        }
+        finally {
+            metaStoreFile.setWritable(true);
+        }
+    }
+
     /**
      * @param expError Expected error.
      */
     private void waitFailure(Class<? extends Throwable> expError) throws IgniteInterruptedCheckedException {
         assertTrue(GridTestUtils.waitForCondition(() -> failureHnd.failure(), 5_000L));
 
-        assertTrue(expError.isInstance(failureHnd.error()));
+        assertTrue(X.hasCause(failureHnd.error(), expError));
     }
 
     /**

From 2545a4d65ae9c0289c9f6faf9d2ee498375aa3c7 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Fri, 15 Jun 2018 11:31:33 +0300
Subject: [PATCH 190/543] IGNITE-8657 Simultaneous start of multiple clients
 may lead to client start hang when exchange history size is too short - Fixes
 #4102.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry-picked from commit #2b928c702b2d6a7c1de8d1b3b0a1e0e65e653f21)
---
 .../apache/ignite/internal/IgniteKernal.java  |   8 +-
 .../IgniteNeedReconnectException.java         |   8 +-
 .../GridCachePartitionExchangeManager.java    |  40 +++-
 .../GridDhtPartitionsExchangeFuture.java      |  40 +++-
 .../IgniteCacheClientReconnectTest.java       | 178 +++++++++++++++++-
 5 files changed, 249 insertions(+), 25 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 38a19875b648f..d24a19e078a6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1104,7 +1104,13 @@ public void start(
                         comp.onKernalStart(active);
                     }
                     catch (IgniteNeedReconnectException e) {
-                        assert ctx.discovery().reconnectSupported();
+                        ClusterNode locNode = ctx.discovery().localNode();
+
+                        assert CU.clientNode(locNode);
+
+                        if (!locNode.isClient())
+                            throw new IgniteCheckedException("Client node in forceServerMode " +
+                                "is not allowed to reconnect to the cluster and will be stopped.");
 
                         if (log.isDebugEnabled())
                             log.debug("Failed to start node components on node start, will wait for reconnect: " + e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
index f3849500d3561..c26f4da03107a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNeedReconnectException.java
@@ -29,12 +29,10 @@ public class IgniteNeedReconnectException extends IgniteCheckedException {
     private static final long serialVersionUID = 0L;
 
     /**
-     * @param locNode Local node.
+     * @param node Node that should reconnect.
      * @param cause Cause.
      */
-    public IgniteNeedReconnectException(ClusterNode locNode, @Nullable Throwable cause) {
-        super("Local node need try to reconnect [locNodeId=" + locNode.id() + ']', cause);
-
-        assert locNode.isClient() : locNode;
+    public IgniteNeedReconnectException(ClusterNode node, @Nullable Throwable cause) {
+        super("Node need try to reconnect [nodeId=" + node.id() + ']', cause);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index bb66a3b5ada47..38ddaf661de92 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -1552,8 +1552,28 @@ else if (!grp.isLocal())
                 if (updated)
                     scheduleResendPartitions();
             }
-            else
-                exchangeFuture(msg.exchangeId(), null, null, null, null).onReceiveSingleMessage(node, msg);
+            else {
+                GridDhtPartitionsExchangeFuture exchFut = exchangeFuture(msg.exchangeId(), null, null, null, null);
+
+                if (log.isDebugEnabled())
+                    log.debug("Notifying exchange future about single message: " + exchFut);
+
+                if (msg.client() && !exchFut.isDone()) {
+                    if (exchFut.initialVersion().compareTo(readyAffinityVersion()) <= 0) {
+                        U.warn(log, "Client node tries to connect but its exchange " +
+                            "info is cleaned up from exchange history." +
+                            " Consider increasing 'IGNITE_EXCHANGE_HISTORY_SIZE' property " +
+                            "or start clients in  smaller batches."
+                        );
+
+                        exchFut.forceClientReconnect(node, msg);
+
+                        return;
+                    }
+                }
+
+                exchFut.onReceiveSingleMessage(node, msg);
+            }
         }
         finally {
             leaveBusy();
@@ -2642,14 +2662,18 @@ else if (r != null) {
                     throw e;
                 }
                 catch (IgniteClientDisconnectedCheckedException | IgniteNeedReconnectException e) {
-                    assert cctx.discovery().reconnectSupported();
+                    if (cctx.discovery().reconnectSupported()) {
+                        U.warn(log, "Local node failed to complete partition map exchange due to " +
+                            "exception, will try to reconnect to cluster: " + e.getMessage(), e);
 
-                    U.warn(log,"Local node failed to complete partition map exchange due to " +
-                        "network issues, will try to reconnect to cluster", e);
+                        cctx.discovery().reconnect();
 
-                    cctx.discovery().reconnect();
-
-                    reconnectNeeded = true;
+                        reconnectNeeded = true;
+                    }
+                    else
+                        U.warn(log, "Local node received IgniteClientDisconnectedCheckedException or " +
+                            " IgniteNeedReconnectException exception but doesn't support reconnect, stopping node: " +
+                            e.getMessage(), e);
 
                     return;
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index fa82aa45a0dbe..ef1980de9f58f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -2035,6 +2035,31 @@ private void processMergedMessage(final ClusterNode node, final GridDhtPartition
             finishExchangeOnCoordinator(null);
     }
 
+    /**
+     * Method is called on coordinator in situation when initial ExchangeFuture created on client join event was preempted
+     * from exchange history because of IGNITE_EXCHANGE_HISTORY_SIZE property.
+     *
+     * @param node Client node that should try to reconnect to the cluster.
+     * @param msg Single message received from the client which didn't find original ExchangeFuture.
+     */
+    public void forceClientReconnect(ClusterNode node, GridDhtPartitionsSingleMessage msg) {
+        Exception e = new IgniteNeedReconnectException(node, null);
+
+        changeGlobalStateExceptions.put(node.id(), e);
+
+        onDone(null, e);
+
+        GridDhtPartitionsFullMessage fullMsg = createPartitionsMessage(true, false);
+
+        fullMsg.setErrorsMap(changeGlobalStateExceptions);
+
+        FinishState finishState0 = new FinishState(cctx.localNodeId(),
+            initialVersion(),
+            fullMsg);
+
+        sendAllPartitionsToNode(finishState0, msg, node.id());
+    }
+
     /**
      * Processing of received single message. Actual processing in future may be delayed if init method was not
      * completed, see {@link #initDone()}
@@ -3113,6 +3138,16 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti
                                 return;
                             }
                             else {
+                                if (!F.isEmpty(msg.getErrorsMap())) {
+                                    Exception e = msg.getErrorsMap().get(cctx.localNodeId());
+
+                                    assert e != null : msg.getErrorsMap();
+
+                                    onDone(e);
+
+                                    return;
+                                }
+
                                 AffinityTopologyVersion resVer = msg.resultTopologyVersion() != null ? msg.resultTopologyVersion() : initialVersion();
 
                                 if (log.isInfoEnabled()) {
@@ -3738,8 +3773,9 @@ private void onBecomeCoordinator(InitNewCoordinatorFuture newCrdFut) {
      * @return {@code True} if local node should try reconnect in case of error.
      */
     public boolean reconnectOnError(Throwable e) {
-        return X.hasCause(e, IOException.class, IgniteClientDisconnectedCheckedException.class) &&
-            cctx.discovery().reconnectSupported();
+        return (e instanceof IgniteNeedReconnectException
+            || X.hasCause(e, IOException.class, IgniteClientDisconnectedCheckedException.class))
+            && cctx.discovery().reconnectSupported();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java
index ced1a7d8a78d7..4beb31a9d92e6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java
@@ -24,9 +24,17 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -48,15 +56,24 @@ public class IgniteCacheClientReconnectTest extends GridCommonAbstractTest {
     /** */
     private static final int SRV_CNT = 3;
 
+    /** */
+    private static final int CLIENTS_CNT = 3;
+
     /** */
     private static final int CACHES = 10;
 
+    /** */
+    private static final int PARTITIONS_CNT = 32;
+
     /** */
     private static final long TEST_TIME = 60_000;
 
     /** */
     private boolean client;
 
+    /** */
+    private boolean forceServerMode;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -69,19 +86,22 @@ public class IgniteCacheClientReconnectTest extends GridCommonAbstractTest {
             CacheConfiguration[] ccfgs = new CacheConfiguration[CACHES];
 
             for (int i = 0; i < CACHES; i++) {
-                CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+                CacheConfiguration ccfg = new CacheConfiguration();
 
                 ccfg.setCacheMode(PARTITIONED);
                 ccfg.setAtomicityMode(TRANSACTIONAL);
                 ccfg.setBackups(1);
                 ccfg.setName("cache-" + i);
                 ccfg.setWriteSynchronizationMode(FULL_SYNC);
+                ccfg.setAffinity(new RendezvousAffinityFunction(PARTITIONS_CNT, null));
 
                 ccfgs[i] = ccfg;
             }
 
             cfg.setCacheConfiguration(ccfgs);
         }
+        else
+            ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setForceServerMode(forceServerMode);
 
         cfg.setClientMode(client);
 
@@ -89,26 +109,166 @@ public class IgniteCacheClientReconnectTest extends GridCommonAbstractTest {
     }
 
     /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        startGrids(SRV_CNT);
+    @Override protected long getTestTimeout() {
+        return TEST_TIME + 60_000;
     }
 
     /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        super.afterTestsStopped();
-
+    @Override protected void afterTest() throws Exception {
         stopAllGrids();
     }
 
-    /** {@inheritDoc} */
-    @Override protected long getTestTimeout() {
-        return TEST_TIME + 60_000;
+    /**
+     * If setting IGNITE_EXCHANGE_HISTORY_SIZE is set to small value
+     * it is possible that bunch of clients simultaneous start (amount > IGNITE_EXCHANGE_HISTORY_SIZE)
+     * may result in ExchangeFuture for some client being flushed from exchange history.
+     *
+     * In that case client attempts to reconnect until success.
+     *
+     * After that it should get correct information about topology version and affinity distribution.
+     *
+     * @throws Exception If failed
+     */
+    public void testClientReconnectOnExchangeHistoryExhaustion() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE, "1");
+
+        try {
+            startGrids(SRV_CNT);
+
+            client = true;
+
+            startGridsMultiThreaded(SRV_CNT, CLIENTS_CNT);
+
+            waitForTopology(SRV_CNT + CLIENTS_CNT);
+
+            verifyPartitionToNodeMappings();
+
+            verifyAffinityTopologyVersions();
+
+            verifyCacheOperationsOnClients();
+        }
+        finally {
+            System.clearProperty(IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE);
+        }
+    }
+
+    /**
+     * Verifies that in case of exchange history exhaustion
+     * (refer to javadoc at {@link #testClientReconnectOnExchangeHistoryExhaustion()} for more info about it)
+     * clients with forceServerMode=true flag don't try to reconnect to the cluster and stop.
+     *
+     * @throws Exception If failed
+     */
+    public void testClientInForceServerModeStopsOnExchangeHistoryExhaustion() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE, "1");
+
+        try {
+            startGrids(SRV_CNT);
+
+            client = true;
+
+            forceServerMode = true;
+
+            int clientNodes = 10;
+
+            try {
+                startGridsMultiThreaded(SRV_CNT, clientNodes);
+            }
+            catch (IgniteCheckedException e) {
+                //Ignored: it is expected to get exception here
+            }
+
+            awaitPartitionMapExchange();
+
+            int topSize = G.allGrids().size();
+
+            assertTrue("Actual size: " + topSize, topSize < SRV_CNT + clientNodes);
+
+        }
+        finally {
+            System.clearProperty(IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE);
+        }
+    }
+
+    /**
+     * Verifies basic cache operations from all clients.
+     */
+    private void verifyCacheOperationsOnClients() {
+        for (int i = SRV_CNT; i < SRV_CNT + CLIENTS_CNT; i++) {
+            IgniteEx cl = grid(i);
+
+            if (!forceServerMode)
+                assertTrue(cl.localNode().isClient());
+
+            for (int j = 0; j < CACHES; j++) {
+                IgniteCache<Object, Object> cache = cl.cache("cache-" + j);
+
+                String keyPrefix = "cl-" + i + "_key-";
+
+                for (int k = 0; k < 1_000; k++)
+                    cache.put(keyPrefix + k, "val-" + k);
+
+                for (int k = 999; k >= 0; k--)
+                    assertEquals("val-" + k, cache.get(keyPrefix + k));
+            }
+        }
+    }
+
+    /**
+     * Verifies that affinity mappings are the same on clients and servers.
+     */
+    private void verifyPartitionToNodeMappings() {
+        IgniteEx refSrv = grid(0);
+        String cacheName;
+
+        for (int i = 0; i < CACHES; i++) {
+            cacheName = "cache-" + i;
+
+            Affinity<Object> refAffinity = refSrv.affinity(cacheName);
+
+            for (int j = 0; j < PARTITIONS_CNT; j++) {
+                ClusterNode refAffNode = refAffinity.mapPartitionToNode(j);
+
+                assertNotNull("Affinity node for " + j + " partition is null", refAffNode);
+
+                for (int k = SRV_CNT; k < SRV_CNT + CLIENTS_CNT; k++) {
+                    ClusterNode clAffNode = grid(k).affinity(cacheName).mapPartitionToNode(j);
+
+                    assertNotNull("Affinity node for " + k + " client and " + j + " partition is null", clAffNode);
+
+                    assertEquals("Affinity node for "
+                        + k
+                        + " client and "
+                        + j
+                        + " partition is different on client",
+                        refAffNode.id(),
+                        clAffNode.id());
+                }
+            }
+        }
+    }
+
+    /**
+     * Verifies {@link AffinityTopologyVersion}s: one obtained from coordinator and all from each client node.
+     */
+    private void verifyAffinityTopologyVersions() {
+        IgniteEx srv = grid(0);
+
+        AffinityTopologyVersion srvTopVer = srv.context().discovery().topologyVersionEx();
+
+        for (int i = SRV_CNT; i < SRV_CNT + CLIENTS_CNT; i++) {
+            AffinityTopologyVersion clntTopVer = grid(i).context().discovery().topologyVersionEx();
+
+            assertTrue(clntTopVer.equals(srvTopVer));
+        }
     }
 
     /**
      * @throws Exception If failed.
      */
     public void testClientReconnect() throws Exception {
+        startGrids(SRV_CNT);
+
         client = true;
 
         final AtomicBoolean stop = new AtomicBoolean(false);

From f4511d921c34fc1fc048afaf3ec10307b0f3bea2 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Fri, 15 Jun 2018 12:48:24 +0300
Subject: [PATCH 191/543] IGNITE-8610 Fixed checkpoint record search for WAL
 delta rebalancing - Fixes #4090.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 10aa02a)
---
 .../dht/GridDhtLocalPartition.java            |   23 +-
 .../distributed/dht/GridDhtLockFuture.java    |    3 +-
 .../GridDhtPartitionDemandMessage.java        |    2 +-
 .../GridDhtPartitionsExchangeFuture.java      |   21 +-
 .../dht/preloader/GridDhtPreloader.java       |    8 +-
 ...IgniteDhtPartitionHistorySuppliersMap.java |   13 +-
 .../GridCacheDatabaseSharedManager.java       | 1389 +++++------------
 .../persistence/GridCacheOffheapManager.java  |    5 +-
 .../IgniteCacheDatabaseSharedManager.java     |   26 +-
 .../checkpoint/CheckpointEntry.java           |  366 +++++
 .../checkpoint/CheckpointEntryType.java       |   29 +
 .../checkpoint/CheckpointHistory.java         |  382 +++++
 .../wal/FileWriteAheadLogManager.java         |   12 +-
 .../internal/visor/misc/VisorWalTask.java     |    4 +-
 ...sAtomicCacheHistoricalRebalancingTest.java |   19 +
 ...IgnitePdsCacheRebalancingAbstractTest.java |  328 ++--
 ...tePdsTxCacheHistoricalRebalancingTest.java |   39 -
 ...alModeChangeDuringRebalancingSelfTest.java |    4 +-
 .../db/IgnitePdsUnusedWalSegmentsTest.java    |   20 +-
 .../db/wal/IgniteWalRebalanceTest.java        |  271 +++-
 .../wal/WalRecoveryTxLogicalRecordsTest.java  |    3 +-
 .../IgnitePdsWithIndexingCoreTestSuite.java   |    2 -
 22 files changed, 1696 insertions(+), 1273 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntry.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntryType.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 398b0d5880f9d..8114ec7e55059 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -32,6 +32,7 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
@@ -816,11 +817,23 @@ public void destroy() {
      * Awaits completion of partition destroy process in case of {@code EVICTED} partition state.
      */
     public void awaitDestroy() {
-        try {
-            if (state() == EVICTED)
-                rent.get();
-        } catch (IgniteCheckedException e) {
-            log.error("Unable to await partition destroy " + this, e);
+        if (state() != EVICTED)
+            return;
+
+        final long timeout = 10_000;
+
+        for (;;) {
+            try {
+                rent.get(timeout);
+
+                break;
+            }
+            catch (IgniteFutureTimeoutCheckedException ignored) {
+                U.warn(log, "Failed to await partition destroy within timeout " + this);
+            }
+            catch (IgniteCheckedException e) {
+                throw new IgniteException("Failed to await partition destroy " + this, e);
+            }
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
index 2869bb6441652..529d96575e827 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
@@ -1309,7 +1309,8 @@ void onResult(GridDhtLockResponse res) {
                                     info.version(),
                                     info.ttl(),
                                     info.expireTime(),
-                                    true, topVer,
+                                    true,
+                                    topVer,
                                     replicate ? DR_PRELOAD : DR_NONE,
                                     false)) {
                                     if (rec && !entry.isInternal())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
index 0d2cc4bebf5fe..dc6162bc321f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
@@ -136,7 +136,7 @@ public GridDhtPartitionDemandMessage withNewPartitionsMap(@NotNull IgniteDhtDema
     /**
      * @return Partition.
      */
-    IgniteDhtDemandedPartitionsMap partitions() {
+    public IgniteDhtDemandedPartitionsMap partitions() {
         return parts;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index ef1980de9f58f..f957a042fec1a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -423,12 +423,15 @@ public void affinityChangeMessage(CacheAffinityChangeMessage affChangeMsg) {
     }
 
     /**
+     * Retreives the node which has WAL history since {@code cntrSince}.
+     *
      * @param grpId Cache group ID.
      * @param partId Partition ID.
+     * @param cntrSince Partition update counter since history supplying is requested.
      * @return ID of history supplier node or null if it doesn't exist.
      */
-    @Nullable public UUID partitionHistorySupplier(int grpId, int partId) {
-        return partHistSuppliers.getSupplier(grpId, partId);
+    @Nullable public UUID partitionHistorySupplier(int grpId, int partId, long cntrSince) {
+        return partHistSuppliers.getSupplier(grpId, partId, cntrSince);
     }
 
     /**
@@ -1162,8 +1165,9 @@ private void distributedExchange() throws IgniteCheckedException {
            In case of persistent store is enabled we first restore partitions presented on disk.
            We need to guarantee that there are no partition state changes logged to WAL before this callback
            to make sure that we correctly restored last actual states. */
-        cctx.database().beforeExchange(this);
+        boolean restored = cctx.database().beforeExchange(this);
 
+        // Pre-create missing partitions using current affinity.
         if (!exchCtx.mergeExchanges()) {
             for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                 if (grp.isLocal() || cacheGroupStopping(grp.groupId()))
@@ -1175,6 +1179,10 @@ private void distributedExchange() throws IgniteCheckedException {
             }
         }
 
+        // After all partitions have been restored and pre-created it's safe to make first checkpoint.
+        if (restored)
+            cctx.database().onStateRestored();
+
         changeWalModeIfNeeded();
 
         if (crd.isLocal()) {
@@ -1740,6 +1748,7 @@ public void finishMerged() {
         }
 
         cctx.database().releaseHistoryForExchange();
+
         cctx.database().rebuildIndexesIfNeeded(this);
 
         if (err == null) {
@@ -2397,8 +2406,6 @@ else if (cntr == maxCntr.cnt)
                 maxCntr.nodes.add(cctx.localNodeId());
         }
 
-        int entryLeft = maxCntrs.size();
-
         Map<Integer, Map<Integer, Long>> partHistReserved0 = partHistReserved;
 
         Map<Integer, Long> localReserved = partHistReserved0 != null ? partHistReserved0.get(top.groupId()) : null;
@@ -2421,7 +2428,7 @@ else if (cntr == maxCntr.cnt)
                 Long localCntr = localReserved.get(p);
 
                 if (localCntr != null && localCntr <= minCntr && maxCntrObj.nodes.contains(cctx.localNodeId())) {
-                    partHistSuppliers.put(cctx.localNodeId(), top.groupId(), p, minCntr);
+                    partHistSuppliers.put(cctx.localNodeId(), top.groupId(), p, localCntr);
 
                     haveHistory.add(p);
 
@@ -2433,7 +2440,7 @@ else if (cntr == maxCntr.cnt)
                 Long histCntr = e0.getValue().partitionHistoryCounters(top.groupId()).get(p);
 
                 if (histCntr != null && histCntr <= minCntr && maxCntrObj.nodes.contains(e0.getKey())) {
-                    partHistSuppliers.put(e0.getKey(), top.groupId(), p, minCntr);
+                    partHistSuppliers.put(e0.getKey(), top.groupId(), p, histCntr);
 
                     haveHistory.add(p);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 700f0cf98b7a1..77f48661d588b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -223,6 +224,8 @@ private IgniteCheckedException stopError() {
 
                 // If partition was destroyed recreate it.
                 if (part.state() == EVICTED) {
+                    part.awaitDestroy();
+
                     part = top.localPartition(p, topVer, true);
                 }
 
@@ -231,7 +234,7 @@ private IgniteCheckedException stopError() {
                 ClusterNode histSupplier = null;
 
                 if (grp.persistenceEnabled() && exchFut != null) {
-                    UUID nodeId = exchFut.partitionHistorySupplier(grp.groupId(), p);
+                    UUID nodeId = exchFut.partitionHistorySupplier(grp.groupId(), p, part.initialUpdateCounter());
 
                     if (nodeId != null)
                         histSupplier = ctx.discovery().node(nodeId);
@@ -288,6 +291,9 @@ private IgniteCheckedException stopError() {
             }
         }
 
+        if (!assignments.isEmpty())
+            ctx.database().lastCheckpointInapplicableForWalRebalance(grp.groupId());
+
         return assignments;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionHistorySuppliersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionHistorySuppliersMap.java
index f8da6a80bf972..6755f287b6974 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionHistorySuppliersMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtPartitionHistorySuppliersMap.java
@@ -47,17 +47,22 @@ public static IgniteDhtPartitionHistorySuppliersMap empty() {
     }
 
     /**
-     * @param grpId Cache group ID.
+     * @param grpId Group ID.
      * @param partId Partition ID.
+     * @param cntrSince Partition update counter since history supplying is requested.
      * @return Supplier UUID.
      */
-    @Nullable public synchronized UUID getSupplier(int grpId, int partId) {
+    @Nullable public synchronized UUID getSupplier(int grpId, int partId, long cntrSince) {
         if (map == null)
             return null;
 
         for (Map.Entry<UUID, Map<T2<Integer, Integer>, Long>> e : map.entrySet()) {
-            if (e.getValue().containsKey(new T2<>(grpId, partId)))
-                return e.getKey();
+            UUID supplierNode = e.getKey();
+
+            Long historyCounter = e.getValue().get(new T2<>(grpId, partId));
+
+            if (historyCounter != null && historyCounter <= cntrSince)
+                return supplierNode;
         }
 
         return null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 3ee8f28d6d3de..7d30181c9bf46 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -18,10 +18,8 @@
 package org.apache.ignite.internal.processors.cache.persistence;
 
 import java.io.File;
-import java.io.FileFilter;
 import java.io.IOException;
 import java.io.RandomAccessFile;
-import java.lang.ref.SoftReference;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.nio.channels.FileChannel;
@@ -42,20 +40,16 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.NavigableMap;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import java.util.concurrent.atomic.LongAdder;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
@@ -118,6 +112,9 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntryType;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
@@ -165,7 +162,6 @@
 import org.jsr166.ConcurrentLinkedHashMap;
 
 import static java.nio.file.StandardOpenOption.READ;
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
 import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
@@ -222,64 +218,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Checkpoint file temporary suffix. This is needed to safe writing checkpoint markers through temporary file and renaming. */
     public static final String FILE_TMP_SUFFIX = ".tmp";
 
-    /** Node started file patter. */
-    private static final Pattern NODE_STARTED_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-node-started\\.bin");
-
     /** Node started file suffix. */
     public static final String NODE_STARTED_FILE_NAME_SUFFIX = "-node-started.bin";
 
-    /** */
-    private static final FileFilter CP_FILE_FILTER = new FileFilter() {
-        @Override public boolean accept(File f) {
-            return CP_FILE_NAME_PATTERN.matcher(f.getName()).matches();
-        }
-    };
-
-    /** */
-    private static final FileFilter NODE_STARTED_FILE_FILTER = new FileFilter() {
-        @Override public boolean accept(File f) {
-            return f.getName().endsWith(NODE_STARTED_FILE_NAME_SUFFIX);
-        }
-    };
-
-    /** */
-    private static final Comparator<GridDhtLocalPartition> ASC_PART_COMPARATOR = new Comparator<GridDhtLocalPartition>() {
-        @Override public int compare(GridDhtLocalPartition a, GridDhtLocalPartition b) {
-            return Integer.compare(a.id(), b.id());
-        }
-    };
-
-    /** */
-    private static final Comparator<File> CP_TS_COMPARATOR = new Comparator<File>() {
-        /** {@inheritDoc} */
-        @Override public int compare(File o1, File o2) {
-            Matcher m1 = CP_FILE_NAME_PATTERN.matcher(o1.getName());
-            Matcher m2 = CP_FILE_NAME_PATTERN.matcher(o2.getName());
-
-            boolean s1 = m1.matches();
-            boolean s2 = m2.matches();
-
-            assert s1 : "Failed to match CP file: " + o1.getAbsolutePath();
-            assert s2 : "Failed to match CP file: " + o2.getAbsolutePath();
-
-            long ts1 = Long.parseLong(m1.group(1));
-            long ts2 = Long.parseLong(m2.group(1));
-
-            int res = Long.compare(ts1, ts2);
-
-            if (res == 0) {
-                CheckpointEntryType type1 = CheckpointEntryType.valueOf(m1.group(3));
-                CheckpointEntryType type2 = CheckpointEntryType.valueOf(m2.group(3));
-
-                assert type1 != type2 : "o1=" + o1.getAbsolutePath() + ", o2=" + o2.getAbsolutePath();
-
-                res = type1 == CheckpointEntryType.START ? -1 : 1;
-            }
-
-            return res;
-        }
-    };
-
     /** */
     private static final String MBEAN_NAME = "DataStorageMetrics";
 
@@ -289,12 +230,15 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** WAL marker prefix for meta store. */
     private static final String WAL_KEY_PREFIX = "grp-wal-";
 
-    /** WAL marker prefix for meta store. */
+    /** Prefix for meta store records which means that WAL was disabled globally for some group. */
     private static final String WAL_GLOBAL_KEY_PREFIX = WAL_KEY_PREFIX + "disabled-";
 
-    /** WAL marker prefix for meta store. */
+    /** Prefix for meta store records which means that WAL was disabled locally for some group. */
     private static final String WAL_LOCAL_KEY_PREFIX = WAL_KEY_PREFIX + "local-disabled-";
 
+    /** Prefix for meta store records which means that checkpoint entry for some group is not applicable for WAL rebalance. */
+    private static final String CHECKPOINT_INAPPLICABLE_FOR_REBALANCE = "cp-wal-rebalance-inapplicable-";
+
     /** WAL marker predicate for meta store. */
     private static final IgnitePredicate<String> WAL_KEY_PREFIX_PRED = new IgnitePredicate<String>() {
         @Override public boolean apply(String key) {
@@ -320,6 +264,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** */
     private long checkpointFreq;
 
+    /** */
+    private CheckpointHistory cpHistory;
+
     /** */
     private FilePageStoreManager storeMgr;
 
@@ -335,9 +282,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** */
     private final Collection<DbCheckpointListener> lsnrs = new CopyOnWriteArrayList<>();
 
-    /** Checkpoint history. */
-    private final CheckpointHistory checkpointHist = new CheckpointHistory();
-
     /** */
     private boolean stopping;
 
@@ -360,7 +304,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     private final long lockWaitTime;
 
     /** */
-    private final int maxCpHistMemSize;
+    private final boolean truncateWalOnCpFinish;
 
     /** */
     private Map</*grpId*/Integer, Map</*partId*/Integer, T2</*updCntr*/Long, WALPointer>>> reservedForExchange;
@@ -416,6 +360,8 @@ public GridCacheDatabaseSharedManager(GridKernalContext ctx) {
 
         checkpointFreq = persistenceCfg.getCheckpointFrequency();
 
+        truncateWalOnCpFinish = persistenceCfg.getWalHistorySize() != Integer.MAX_VALUE;
+
         lockWaitTime = persistenceCfg.getLockWaitTime();
 
         persStoreMetrics = new DataStorageMetricsImpl(
@@ -426,9 +372,6 @@ public GridCacheDatabaseSharedManager(GridKernalContext ctx) {
 
         metastorageLifecycleLsnrs = ctx.internalSubscriptionProcessor().getMetastorageSubscribers();
 
-        maxCpHistMemSize = Math.min(persistenceCfg.getWalHistorySize(),
-            IgniteSystemProperties.getInteger(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, 100));
-
         ioFactory = persistenceCfg.getFileIOFactory();
     }
 
@@ -515,6 +458,8 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
         if (!kernalCtx.clientNode()) {
             checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log);
 
+            cpHistory = new CheckpointHistory(kernalCtx);
+
             IgnitePageStoreManager store = cctx.pageStore();
 
             assert store instanceof FilePageStoreManager : "Invalid page store manager was created: " + store;
@@ -531,6 +476,7 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
             final FileLockHolder preLocked = kernalCtx.pdsFolderResolver()
                 .resolveFolders()
                 .getLockedFileLockHolder();
+
             if (preLocked == null)
                 fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), kernalCtx, log);
 
@@ -546,12 +492,10 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
      */
     private void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
         try {
-            try (DirectoryStream<Path> files = Files.newDirectoryStream(cpDir.toPath(), new DirectoryStream.Filter<Path>() {
-                @Override
-                public boolean accept(Path path) throws IOException {
-                    return path.endsWith(FILE_TMP_SUFFIX);
-                }
-            })) {
+            try (DirectoryStream<Path> files = Files.newDirectoryStream(
+                cpDir.toPath(),
+                path -> path.endsWith(FILE_TMP_SUFFIX))
+            ) {
                 for (Path path : files)
                     Files.delete(path);
             }
@@ -577,18 +521,83 @@ public void cleanupCheckpointDirectory() throws IgniteCheckedException {
     }
 
     /**
+     * Retreives checkpoint history form specified {@code dir}.
      *
+     * @return List of checkpoints.
      */
-    private void initDataBase() {
-        if (persistenceCfg.getCheckpointThreads() > 1)
-            asyncRunner = new IgniteThreadPoolExecutor(
-                "checkpoint-runner",
-                cctx.igniteInstanceName(),
-                persistenceCfg.getCheckpointThreads(),
-                persistenceCfg.getCheckpointThreads(),
-                30_000,
-                new LinkedBlockingQueue<Runnable>()
-            );
+    private List<CheckpointEntry> retreiveHistory() throws IgniteCheckedException {
+        if (!cpDir.exists())
+            return Collections.emptyList();
+
+        try (DirectoryStream<Path> cpFiles = Files.newDirectoryStream(
+            cpDir.toPath(),
+            path -> CP_FILE_NAME_PATTERN.matcher(path.toFile().getName()).matches())
+        ) {
+            List<CheckpointEntry> checkpoints = new ArrayList<>();
+
+            ByteBuffer buf = ByteBuffer.allocate(16);
+            buf.order(ByteOrder.nativeOrder());
+
+            for (Path cpFile : cpFiles) {
+                CheckpointEntry cp = parseFromFile(buf, cpFile.toFile());
+
+                if (cp != null)
+                    checkpoints.add(cp);
+            }
+
+            return checkpoints;
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Failed to load checkpoint history.", e);
+        }
+    }
+
+    /**
+     * Parses checkpoint entry from given file.
+     *
+     * @param buf Temporary byte buffer.
+     * @param file Checkpoint file.
+     */
+    @Nullable private CheckpointEntry parseFromFile(ByteBuffer buf, File file) throws IgniteCheckedException {
+        Matcher matcher = CP_FILE_NAME_PATTERN.matcher(file.getName());
+
+        if (!matcher.matches())
+            return null;
+
+        CheckpointEntryType type = CheckpointEntryType.valueOf(matcher.group(3));
+
+        if (type != CheckpointEntryType.START)
+            return null;
+
+        long cpTs = Long.parseLong(matcher.group(1));
+        UUID cpId = UUID.fromString(matcher.group(2));
+
+        WALPointer ptr = readPointer(file, buf);
+
+        return createCheckPointEntry(cpTs, ptr, cpId, null, CheckpointEntryType.START);
+    }
+
+    /**
+     * Removes checkpoint start/end files belongs to given {@code cpEntry}.
+     *
+     * @param cpEntry Checkpoint entry.
+     *
+     * @throws IgniteCheckedException If failed to delete.
+     */
+    private void removeCheckpointFiles(CheckpointEntry cpEntry) throws IgniteCheckedException {
+        Path startFile = new File(cpDir.getAbsolutePath(), checkpointFileName(cpEntry, CheckpointEntryType.START)).toPath();
+        Path endFile = new File(cpDir.getAbsolutePath(), checkpointFileName(cpEntry, CheckpointEntryType.END)).toPath();
+
+        try {
+            if (Files.exists(startFile))
+                Files.delete(startFile);
+
+            if (Files.exists(endFile))
+                Files.delete(endFile);
+        }
+        catch (IOException e) {
+            throw new PersistentStorageIOException("Failed to delete stale checkpoint files: " + cpEntry, e);
+        }
     }
 
     /** */
@@ -712,6 +721,21 @@ else if (regCfg.getMaxSize() < 8 * GB)
         }
     }
 
+    /**
+     *
+     */
+    private void initDataBase() {
+        if (persistenceCfg.getCheckpointThreads() > 1)
+            asyncRunner = new IgniteThreadPoolExecutor(
+                "checkpoint-runner",
+                cctx.igniteInstanceName(),
+                persistenceCfg.getCheckpointThreads(),
+                persistenceCfg.getCheckpointThreads(),
+                30_000,
+                new LinkedBlockingQueue<Runnable>()
+            );
+    }
+
     /**
      * Try to register Metrics MBean.
      *
@@ -883,49 +907,49 @@ private void nodeStart(WALPointer ptr) throws IgniteCheckedException {
      * Tuples are sorted by timestamp.
      *
      * @return Sorted list of tuples (node started timestamp, memory recovery pointer).
-     * @throws IgniteCheckedException
+     *
+     * @throws IgniteCheckedException If failed.
      */
     public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedException {
         List<T2<Long, WALPointer>> res = new ArrayList<>();
 
-        File[] files = cpDir.listFiles(NODE_STARTED_FILE_FILTER);
-
-        Arrays.sort(files, new Comparator<File>() {
-            @Override public int compare(File o1, File o2) {
-                String n1 = o1.getName();
-                String n2 = o2.getName();
-
-                long ts1 = Long.valueOf(n1.substring(0, n1.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
-                long ts2 = Long.valueOf(n2.substring(0, n2.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
-
-                return Long.compare(ts1, ts2);
-            }
-        });
+        try (DirectoryStream<Path> nodeStartedFiles = Files.newDirectoryStream(
+            cpDir.toPath(),
+            path -> path.toFile().getName().endsWith(NODE_STARTED_FILE_NAME_SUFFIX))
+        ) {
+            ByteBuffer buf = ByteBuffer.allocate(20);
+            buf.order(ByteOrder.nativeOrder());
 
-        ByteBuffer buf = ByteBuffer.allocate(20);
-        buf.order(ByteOrder.nativeOrder());
+            for (Path path : nodeStartedFiles) {
+                File f = path.toFile();
 
-        for (File f : files){
-            String name = f.getName();
+                String name = f.getName();
 
-            Long ts = Long.valueOf(name.substring(0, name.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
+                Long ts = Long.valueOf(name.substring(0, name.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
 
-            try (FileIO io = ioFactory.create(f, READ)) {
-                io.read(buf);
+                try (FileIO io = ioFactory.create(f, READ)) {
+                    io.read(buf);
 
-                buf.flip();
+                    buf.flip();
 
-                FileWALPointer ptr = new FileWALPointer(
-                    buf.getLong(), buf.getInt(), buf.getInt());
+                    FileWALPointer ptr = new FileWALPointer(
+                        buf.getLong(), buf.getInt(), buf.getInt());
 
-                res.add(new T2<Long, WALPointer>(ts, ptr));
+                    res.add(new T2<>(ts, ptr));
 
-                buf.clear();
-            }
-            catch (IOException e) {
-                throw new IgniteCheckedException("Failed to read node started marker file: " + f.getAbsolutePath(), e);
+                    buf.clear();
+                }
+                catch (IOException e) {
+                    throw new PersistentStorageIOException("Failed to read node started marker file: " + f.getAbsolutePath(), e);
+                }
             }
         }
+        catch (IOException e) {
+            throw new PersistentStorageIOException("Failed to retreive node started files.", e);
+        }
+
+        // Sort start markers by file timestamp.
+        res.sort(Comparator.comparingLong(IgniteBiTuple::get1));
 
         return res;
     }
@@ -1182,7 +1206,7 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize, long chpBufSi
      * @param partFile Partition file.
      */
     private int resolvePageSizeFromPartitionFile(Path partFile) throws IOException, IgniteCheckedException {
-        try (FileIO fileIO = persistenceCfg.getFileIOFactory().create(partFile.toFile())) {
+        try (FileIO fileIO = ioFactory.create(partFile.toFile())) {
             int minimalHdr = FilePageStore.HEADER_SIZE;
 
             if (fileIO.size() < minimalHdr)
@@ -1269,7 +1293,7 @@ private void shutdownCheckpointer(boolean cancel) {
     }
 
     /** {@inheritDoc} */
-    @Override public void beforeExchange(GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException {
+    @Override public boolean beforeExchange(GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException {
         DiscoveryEvent discoEvt = fut.firstEvent();
 
         boolean joinEvt = discoEvt.type() == EventType.EVT_NODE_JOINED;
@@ -1280,9 +1304,13 @@ private void shutdownCheckpointer(boolean cancel) {
 
         boolean clusterInTransitionStateToActive = fut.activateCluster();
 
+        boolean restored = false;
+
         // In case of cluster activation or local join restore, restore whole manager state.
         if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode)) {
             restoreState();
+
+            restored = true;
         }
         // In case of starting groups, restore partition states only for these groups.
         else if (fut.exchangeActions() != null && !F.isEmpty(fut.exchangeActions().cacheGroupsToStart())) {
@@ -1307,6 +1335,8 @@ else if (acts.localJoinContext() != null && !F.isEmpty(acts.localJoinContext().c
                 }
             }
         }
+
+        return restored;
     }
 
     /**
@@ -1386,13 +1416,7 @@ private void prepareIndexRebuildFuture(int cacheId) {
 
             PageMemoryEx pageMem = (PageMemoryEx)gctx.dataRegion().pageMemory();
 
-            Collection<Integer> grpIds = destroyed.get(pageMem);
-
-            if (grpIds == null) {
-                grpIds = new HashSet<>();
-
-                destroyed.put(pageMem, grpIds);
-            }
+            Collection<Integer> grpIds = destroyed.computeIfAbsent(pageMem, k -> new HashSet<>());
 
             grpIds.add(tup.get1().groupId());
 
@@ -1549,97 +1573,91 @@ private void restoreState() throws IgniteCheckedException {
             }
 
             snapshotMgr.restoreState();
-
-            new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
-
-            CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started");
-
-            if (chp != null)
-                chp.cpBeginFut.get();
         }
         catch (StorageException e) {
             throw new IgniteCheckedException(e);
         }
     }
 
+    /**
+     * Called when all partitions have been fully restored and pre-created on node start.
+     *
+     * Starts checkpointing process and initiates first checkpoint.
+     *
+     * @throws IgniteCheckedException If first checkpoint has failed.
+     */
+    @Override public void onStateRestored() throws IgniteCheckedException {
+        new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
+
+        CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started");
+
+        if (chp != null)
+            chp.cpBeginFut.get();
+    }
+
     /** {@inheritDoc} */
     @Override public synchronized Map<Integer, Map<Integer, Long>> reserveHistoryForExchange() {
         assert reservedForExchange == null : reservedForExchange;
 
         reservedForExchange = new HashMap<>();
 
-        Map</*grpId*/Integer, Set</*partId*/Integer>> parts4CheckpointHistSearch = partsForCheckpointHistorySearch();
+        Map</*grpId*/Integer, Set</*partId*/Integer>> applicableGroupsAndPartitions = partitionsApplicableForWalRebalance();
 
-        Map</*grpId*/Integer, Map</*partId*/Integer, CheckpointEntry>> lastCheckpointEntry4Grp =
-            searchLastCheckpointEntryPerPartition(parts4CheckpointHistSearch);
+        Map</*grpId*/Integer, Map</*partId*/Integer, CheckpointEntry>> earliestValidCheckpoints;
 
-        Map</*grpId*/Integer, Map</*partId*/Integer, /*updCntr*/Long>> grpPartsWithCnts = new HashMap<>();
+        checkpointReadLock();
 
         try {
-            for (Map.Entry<Integer, Map<Integer, CheckpointEntry>> e : lastCheckpointEntry4Grp.entrySet()) {
-                Integer grpId = e.getKey();
-
-                for (Map.Entry<Integer, CheckpointEntry> e0 : e.getValue().entrySet()) {
-                    CheckpointEntry cpEntry = e0.getValue();
+            earliestValidCheckpoints = cpHistory.searchAndReserveCheckpoints(applicableGroupsAndPartitions);
+        }
+        finally {
+            checkpointReadUnlock();
+        }
 
-                    Integer partId = e0.getKey();
+        Map</*grpId*/Integer, Map</*partId*/Integer, /*updCntr*/Long>> grpPartsWithCnts = new HashMap<>();
 
-                    if (cctx.wal().reserve(cpEntry.cpMark)) {
-                        Map</*partId*/Integer, T2</*updCntr*/Long, WALPointer>> grpChpState = reservedForExchange.get(grpId);
+        for (Map.Entry<Integer, Map<Integer, CheckpointEntry>> e : earliestValidCheckpoints.entrySet()) {
+            int grpId = e.getKey();
 
-                        Map<Integer, Long> grpCnts = grpPartsWithCnts.get(grpId);
+            for (Map.Entry<Integer, CheckpointEntry> e0 : e.getValue().entrySet()) {
+                CheckpointEntry cpEntry = e0.getValue();
 
-                        if (grpChpState == null) {
-                            reservedForExchange.put(grpId, grpChpState = new HashMap<>());
+                int partId = e0.getKey();
 
-                            grpPartsWithCnts.put(grpId, grpCnts = new HashMap<>());
-                        }
+                assert cctx.wal().reserved(cpEntry.checkpointMark())
+                    : "WAL segment for checkpoint " + cpEntry + " has not reserved";
 
-                        Long partCnt = cpEntry.partitionCounter(cctx, grpId, partId);
+                Long updCntr = cpEntry.partitionCounter(cctx, grpId, partId);
 
-                        if (partCnt != null) {
-                            grpChpState.put(partId, new T2<>(partCnt, cpEntry.cpMark));
+                if (updCntr != null) {
+                    reservedForExchange.computeIfAbsent(grpId, k -> new HashMap<>())
+                        .put(partId, new T2<>(updCntr, cpEntry.checkpointMark()));
 
-                            grpCnts.put(partId, partCnt);
-                        }
-                        else
-                            cctx.wal().release(cpEntry.cpMark);
-                    }
+                    grpPartsWithCnts.computeIfAbsent(grpId, k -> new HashMap<>()).put(partId, updCntr);
                 }
             }
         }
-        catch (IgniteCheckedException ex) {
-            U.error(log, "Error while trying to reserve history", ex);
-        }
 
         return grpPartsWithCnts;
     }
 
     /**
-     *
-     * @return Map of group id -> Set parts.
+     * @return Map of group id -> Set of partitions which can be used as suppliers for WAL rebalance.
      */
-    private Map<Integer, Set<Integer>> partsForCheckpointHistorySearch() {
-        Map<Integer, Set<Integer>> part4CheckpointHistSearch = new HashMap<>();
+    private Map<Integer, Set<Integer>> partitionsApplicableForWalRebalance() {
+        Map<Integer, Set<Integer>> res = new HashMap<>();
 
         for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
             if (grp.isLocal())
                 continue;
 
-            for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions()) {
-                if (part.state() != GridDhtPartitionState.OWNING || part.dataStore().fullSize() <= walRebalanceThreshold)
-                    continue;
-
-                Set<Integer> parts = part4CheckpointHistSearch.get(grp.groupId());
-
-                if (parts == null)
-                    part4CheckpointHistSearch.put(grp.groupId(), parts = new HashSet<>());
-
-                parts.add(part.id());
+            for (GridDhtLocalPartition locPart : grp.topology().currentLocalPartitions()) {
+                if (locPart.state() == GridDhtPartitionState.OWNING && locPart.fullSize() > walRebalanceThreshold)
+                    res.computeIfAbsent(grp.groupId(), k -> new HashSet<>()).add(locPart.id());
             }
         }
 
-        return part4CheckpointHistSearch;
+        return res;
     }
 
     /** {@inheritDoc} */
@@ -1647,28 +1665,41 @@ private Map<Integer, Set<Integer>> partsForCheckpointHistorySearch() {
         if (reservedForExchange == null)
             return;
 
+        FileWALPointer earliestPtr = null;
+
         for (Map.Entry<Integer, Map<Integer, T2<Long, WALPointer>>> e : reservedForExchange.entrySet()) {
             for (Map.Entry<Integer, T2<Long, WALPointer>> e0 : e.getValue().entrySet()) {
-                try {
-                    cctx.wal().release(e0.getValue().get2());
-                }
-                catch (IgniteCheckedException ex) {
-                    U.error(log, "Could not release history lock", ex);
-                }
+                FileWALPointer ptr = (FileWALPointer) e0.getValue().get2();
+
+                if (earliestPtr == null || ptr.index() < earliestPtr.index())
+                    earliestPtr = ptr;
             }
         }
 
         reservedForExchange = null;
+
+        if (earliestPtr == null)
+            return;
+
+        assert cctx.wal().reserved(earliestPtr)
+            : "Earliest checkpoint WAL pointer is not reserved for exchange: " + earliestPtr;
+
+        try {
+            cctx.wal().release(earliestPtr);
+        }
+        catch (IgniteCheckedException e) {
+            log.error("Failed to release earliest checkpoint WAL pointer: " + earliestPtr, e);
+        }
     }
 
     /** {@inheritDoc} */
     @Override public boolean reserveHistoryForPreloading(int grpId, int partId, long cntr) {
-        CheckpointEntry cpEntry = searchCheckpointEntry(grpId, partId, cntr);
+        CheckpointEntry cpEntry = cpHistory.searchCheckpointEntry(grpId, partId, cntr);
 
         if (cpEntry == null)
             return false;
 
-        WALPointer ptr = cpEntry.cpMark;
+        WALPointer ptr = cpEntry.checkpointMark();
 
         if (ptr == null)
             return false;
@@ -1706,13 +1737,6 @@ private Map<Integer, Set<Integer>> partsForCheckpointHistorySearch() {
         reservedForPreloading.clear();
     }
 
-    /**
-     * For debugging only. TODO: remove.
-     */
-    public Map<T2<Integer, Integer>, T2<Long, WALPointer>> reservedForPreloading() {
-        return reservedForPreloading;
-    }
-
     /**
      *
      */
@@ -1758,150 +1782,6 @@ public Map<T2<Integer, Integer>, T2<Long, WALPointer>> reservedForPreloading() {
         return cp.wakeupForCheckpoint(0, reason);
     }
 
-    /**
-     * Tries to search for a WAL pointer for the given partition counter start.
-     *
-     * @return Checkpoint entry or {@code null} if failed to search.
-     */
-    private Map<Integer, Map<Integer, CheckpointEntry>> searchLastCheckpointEntryPerPartition(
-        final Map<Integer, Set<Integer>> part4reserve
-    ) {
-        final Map<Integer, Map<Integer, CheckpointEntry>> res = new HashMap<>();
-
-        if (F.isEmpty(part4reserve))
-            return res;
-
-        for (Long cpTs : checkpointHist.checkpoints()) {
-            CheckpointEntry chpEntry = null;
-
-            try {
-                chpEntry = checkpointHist.entry(cpTs);
-
-                Map<Integer, CheckpointEntry.GroupState> grpsState = chpEntry.groupState(cctx);
-
-                if (F.isEmpty(grpsState)) {
-                    res.clear();
-
-                    continue;
-                }
-
-                for (Map.Entry<Integer, Set<Integer>> grps : part4reserve.entrySet()) {
-                    Integer grpId = grps.getKey();
-
-                    Map<Integer, CheckpointEntry> partToCheckPntEntry = res.get(grpId);
-
-                    CheckpointEntry.GroupState grpState = grpsState.get(grpId);
-
-                    if (grpState == null) {
-                        res.remove(grpId);
-
-                        continue;
-                    }
-
-                    if (partToCheckPntEntry == null)
-                        res.put(grpId, partToCheckPntEntry = new HashMap<>());
-
-                    for (Integer partId : grps.getValue()) {
-                        int idx = grpState.indexByPartition(partId);
-
-                        if (idx < 0)
-                            partToCheckPntEntry.remove(partId);
-                        else {
-                            if (partToCheckPntEntry.containsKey(partId))
-                                continue;
-
-                            partToCheckPntEntry.put(partId, chpEntry);
-                        }
-                    }
-                }
-            }
-            catch (IgniteCheckedException ex) {
-                String msg = chpEntry != null ?
-                    ", chpId=" + chpEntry.cpId + " ptr=" + chpEntry.cpMark + " ts=" + chpEntry.cpTs : "";
-
-                U.error(log, "Failed to read checkpoint entry" + msg, ex);
-
-                // Treat exception the same way as a gap.
-                res.clear();
-            }
-        }
-
-        return res;
-    }
-
-    /**
-     * Tries to search for a WAL pointer for the given partition counter start.
-     *
-     * @param grpId Cache group ID.
-     * @param part Partition ID.
-     * @param partCntrSince Partition counter or {@code null} to search for minimal counter.
-     * @return Checkpoint entry or {@code null} if failed to search.
-     */
-    @Nullable public WALPointer searchPartitionCounter(int grpId, int part, @Nullable Long partCntrSince) {
-        CheckpointEntry entry = searchCheckpointEntry(grpId, part, partCntrSince);
-
-        if (entry == null)
-            return null;
-
-        return entry.cpMark;
-    }
-
-    /**
-     * Tries to search for a WAL pointer for the given partition counter start.
-     *
-     * @param grpId Cache group ID.
-     * @param part Partition ID.
-     * @param partCntrSince Partition counter or {@code null} to search for minimal counter.
-     * @return Checkpoint entry or {@code null} if failed to search.
-     */
-    @Nullable private CheckpointEntry searchCheckpointEntry(int grpId, int part, @Nullable Long partCntrSince) {
-        boolean hasGap = false;
-        CheckpointEntry first = null;
-
-        for (Long cpTs : checkpointHist.checkpoints()) {
-            try {
-                CheckpointEntry entry = checkpointHist.entry(cpTs);
-
-                Long foundCntr = entry.partitionCounter(cctx, grpId, part);
-
-                if (foundCntr != null) {
-                    if (partCntrSince == null) {
-                        if (hasGap) {
-                            first = entry;
-
-                            hasGap = false;
-                        }
-
-                        if (first == null)
-                            first = entry;
-                    }
-                    else if (foundCntr <= partCntrSince) {
-                        first = entry;
-
-                        hasGap = false;
-                    }
-                    else
-                        return hasGap ? null : first;
-                }
-                else
-                    hasGap = true;
-            }
-            catch (IgniteCheckedException ignore) {
-                // Treat exception the same way as a gap.
-                hasGap = true;
-            }
-        }
-
-        return hasGap ? null : first;
-    }
-
-    /**
-     * @return Checkpoint history. For tests only.
-     */
-    public CheckpointHistory checkpointHistory() {
-        return checkpointHist;
-    }
-
     /**
      * @return Checkpoint directory.
      */
@@ -1944,19 +1824,7 @@ private CheckpointStatus readCheckpointStatus() throws IgniteCheckedException {
         File dir = cpDir;
 
         if (!dir.exists()) {
-            // TODO: remove excessive logging after GG-12116 fix.
-            File[] files = dir.listFiles();
-
-            if (files != null && files.length > 0) {
-                log.warning("Read checkpoint status: cpDir.exists() is false, cpDir.listFiles() is: " +
-                    Arrays.toString(files));
-            }
-
-            if (Files.exists(dir.toPath()))
-                log.warning("Read checkpoint status: cpDir.exists() is false, Files.exists(cpDir) is true.");
-
-            if (log.isInfoEnabled())
-                log.info("Read checkpoint status: checkpoint directory is not found.");
+            log.warning("Read checkpoint status: checkpoint directory is not found.");
 
             return new CheckpointStatus(0, startId, startPtr, endId, endPtr);
         }
@@ -2230,7 +2098,7 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
                 finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr);
         }
 
-        checkpointHist.loadHistory(cpDir);
+        cpHistory.initialize(retreiveHistory());
 
         return lastRead == null ? null : lastRead.next();
     }
@@ -2552,8 +2420,11 @@ else if (restore != null) {
      *
      * @param highBound WALPointer.
      */
-    public void onWalTruncated(WALPointer highBound) {
-        checkpointHist.onWalTruncated(highBound);
+    public void onWalTruncated(WALPointer highBound) throws IgniteCheckedException {
+        List<CheckpointEntry> removedFromHistory = cpHistory.onWalTruncated(highBound);
+
+        for (CheckpointEntry cp : removedFromHistory)
+            removeCheckpointFiles(cp);
     }
 
     /**
@@ -2686,7 +2557,7 @@ private void finalizeCheckpointOnRecovery(long cpTs, UUID cpId, WALPointer walPt
         for (IgniteBiTuple<PageMemory, Collection<FullPageId>> e : cpEntities)
             ((PageMemoryEx)e.get1()).finishCheckpoint();
 
-        writeCheckpointEntry(
+        CheckpointEntry cp = prepareCheckpointEntry(
             tmpWriteBuf,
             cpTs,
             cpId,
@@ -2694,6 +2565,8 @@ private void finalizeCheckpointOnRecovery(long cpTs, UUID cpId, WALPointer walPt
             null,
             CheckpointEntryType.END);
 
+        writeCheckpointEntry(tmpWriteBuf, cp, CheckpointEntryType.END);
+
         cctx.pageStore().finishRecover();
 
         if (log.isInfoEnabled())
@@ -2708,43 +2581,61 @@ private void finalizeCheckpointOnRecovery(long cpTs, UUID cpId, WALPointer walPt
     }
 
     /**
-     * Writes into specified file checkpoint entry containing WAL pointer to checkpoint record.
+     * Prepares checkpoint entry containing WAL pointer to checkpoint record.
+     * Writes into given {@code ptrBuf} WAL pointer content.
      *
-     * @param cpId Checkpoint ID.
-     * @param ptr Wal pointer of current checkpoint.
+     * @param entryBuf Buffer to fill
+     * @param cpTs Checkpoint timestamp.
+     * @param cpId Checkpoint id.
+     * @param ptr WAL pointer containing record.
+     * @param rec Checkpoint WAL record.
+     * @param type Checkpoint type.
+     * @return Checkpoint entry.
      */
-    private CheckpointEntry writeCheckpointEntry(
-        ByteBuffer tmpWriteBuf,
+    private CheckpointEntry prepareCheckpointEntry(
+        ByteBuffer entryBuf,
         long cpTs,
         UUID cpId,
         WALPointer ptr,
-        CheckpointRecord rec,
+        @Nullable CheckpointRecord rec,
         CheckpointEntryType type
-    ) throws IgniteCheckedException {
+    ) {
         assert ptr instanceof FileWALPointer;
 
         FileWALPointer filePtr = (FileWALPointer)ptr;
 
-        String fileName = checkpointFileName(cpTs, cpId, type);
-        String tmpFileName = fileName + FILE_TMP_SUFFIX;
+        entryBuf.rewind();
 
-        try {
-            try (FileIO io = ioFactory.create(Paths.get(cpDir.getAbsolutePath(), skipSync ? fileName : tmpFileName).toFile(),
-                    StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+        entryBuf.putLong(filePtr.index());
 
-                tmpWriteBuf.rewind();
+        entryBuf.putInt(filePtr.fileOffset());
 
-                tmpWriteBuf.putLong(filePtr.index());
+        entryBuf.putInt(filePtr.length());
 
-                tmpWriteBuf.putInt(filePtr.fileOffset());
+        entryBuf.flip();
 
-                tmpWriteBuf.putInt(filePtr.length());
+        return createCheckPointEntry(cpTs, ptr, cpId, rec, type);
+    }
 
-                tmpWriteBuf.flip();
+    /**
+     * Writes checkpoint entry buffer {@code entryBuf} to specified checkpoint file with 2-phase protocol.
+     *
+     * @param entryBuf Checkpoint entry buffer to write.
+     * @param cp Checkpoint entry.
+     * @param type Checkpoint entry type.
+     * @throws PersistentStorageIOException If failed to write checkpoint entry.
+     */
+    public void writeCheckpointEntry(ByteBuffer entryBuf, CheckpointEntry cp, CheckpointEntryType type) throws PersistentStorageIOException {
+        String fileName = checkpointFileName(cp, type);
+        String tmpFileName = fileName + FILE_TMP_SUFFIX;
+
+        try {
+            try (FileIO io = ioFactory.create(Paths.get(cpDir.getAbsolutePath(), skipSync ? fileName : tmpFileName).toFile(),
+                StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
 
-                io.write(tmpWriteBuf);
+                io.write(entryBuf);
 
-                tmpWriteBuf.clear();
+                entryBuf.clear();
 
                 if (!skipSync)
                     io.force(true);
@@ -2752,14 +2643,12 @@ private CheckpointEntry writeCheckpointEntry(
 
             if (!skipSync)
                 Files.move(Paths.get(cpDir.getAbsolutePath(), tmpFileName), Paths.get(cpDir.getAbsolutePath(), fileName));
-
-            return createCheckPointEntry(cpTs, ptr, cpId, rec, type);
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Failed to write checkpoint entry [ptr=" + filePtr
-                    + ", cpTs=" + cpTs
-                    + ", cpId=" + cpId
-                    + ", type=" + type + "]", e);
+            throw new PersistentStorageIOException("Failed to write checkpoint entry [ptr=" + cp.checkpointMark()
+                + ", cpTs=" + cp.timestamp()
+                + ", cpId=" + cp.checkpointId()
+                + ", type=" + type + "]", e);
         }
     }
 
@@ -2793,6 +2682,14 @@ private static String checkpointFileName(long cpTs, UUID cpId, CheckpointEntryTy
         return cpTs + "-" + cpId + "-" + type + ".bin";
     }
 
+    /**
+     * @param cp Checkpoint entry.
+     * @param type Checkpoint type.
+     */
+    private static String checkpointFileName(CheckpointEntry cp, CheckpointEntryType type) {
+        return checkpointFileName(cp.timestamp(), cp.checkpointId(), type);
+    }
+
     /**
      * Replace thread local with buffers. Thread local should provide direct buffer with one page in length.
      *
@@ -2811,7 +2708,7 @@ public void setThreadBuf(final ThreadLocal<ByteBuffer> threadBuf) {
      *
      * @return Checkpoint entry.
      */
-    private CheckpointEntry createCheckPointEntry(
+    public CheckpointEntry createCheckPointEntry(
         long cpTs,
         WALPointer ptr,
         UUID cpId,
@@ -2823,18 +2720,23 @@ private CheckpointEntry createCheckPointEntry(
         assert cpId != null;
         assert type != null;
 
-        if (type != CheckpointEntryType.START)
-            return null;
-
         Map<Integer, CacheState> cacheGrpStates = null;
 
-        // Create lazy checkpoint entry.
-        if ((checkpointHist.histMap.size() + 1 < maxCpHistMemSize) && rec != null)
+        // Do not hold groups state in-memory if there is no space in the checkpoint history to prevent possible OOM.
+        // In this case the actual group states will be readed from WAL by demand.
+        if (rec != null && cpHistory.hasSpace())
             cacheGrpStates = rec.cacheGroupStates();
 
         return new CheckpointEntry(cpTs, ptr, cpId, cacheGrpStates);
     }
 
+    /**
+     * @return Checkpoint history.
+     */
+    @Nullable public CheckpointHistory checkpointHistory() {
+        return cpHistory;
+    }
+
     /**
      * Adds given partition to checkpointer destroy queue.
      *
@@ -3275,8 +3177,18 @@ private void doCheckpoint() {
                     success = true;
                 }
                 finally {
-                    if (success)
-                        markCheckpointEnd(chp);
+                    if (success) {
+                        try {
+                            markCheckpointEnd(chp);
+                        }
+                        catch (IgniteCheckedException e) {
+                            chp.progress.cpFinishFut.onDone(e);
+
+                            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+                            return;
+                        }
+                    }
                 }
 
                 tracker.onEnd();
@@ -3486,14 +3398,27 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
 
             final CheckpointProgress curr;
 
+            CheckpointEntry cp = null;
+
             IgniteBiTuple<Collection<GridMultiCollectionWrapper<FullPageId>>, Integer> cpPagesTuple;
 
             tracker.onLockWaitStart();
 
             boolean hasPages;
 
+            boolean hasPartitionsToDestroy;
+
             IgniteFuture snapFut = null;
 
+            long cpTs = System.currentTimeMillis();
+
+            // This can happen in an unlikely event of two checkpoints happening
+            // within a currentTimeMillis() granularity window.
+            if (cpTs == lastCpTs)
+                cpTs++;
+
+            lastCpTs = cpTs;
+
             checkpointLock.writeLock().lock();
 
             try {
@@ -3564,13 +3489,27 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
 
                 hasPages = hasPageForWrite(cpPagesTuple.get1());
 
-                if (hasPages || curr.nextSnapshot || !curr.destroyQueue.pendingReqs.isEmpty()) {
+                hasPartitionsToDestroy = !curr.destroyQueue.pendingReqs.isEmpty();
+
+                if (hasPages || curr.nextSnapshot || hasPartitionsToDestroy) {
                     // No page updates for this checkpoint are allowed from now on.
                     cpPtr = cctx.wal().log(cpRec);
 
                     if (cpPtr == null)
                         cpPtr = CheckpointStatus.NULL_PTR;
                 }
+
+                if (hasPages || hasPartitionsToDestroy) {
+                    cp = prepareCheckpointEntry(
+                        tmpWriteBuf,
+                        cpTs,
+                        cpRec.checkpointId(),
+                        cpPtr,
+                        cpRec,
+                        CheckpointEntryType.START);
+
+                    cpHistory.addCheckpoint(cp);
+                }
             }
             finally {
                 checkpointLock.writeLock().unlock();
@@ -3590,8 +3529,9 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
                 }
             }
 
-            if (hasPages || !curr.destroyQueue.pendingReqs.isEmpty()) {
+            if (hasPages || hasPartitionsToDestroy) {
                 assert cpPtr != null;
+                assert cp != null;
 
                 tracker.onWalCpRecordFsyncStart();
 
@@ -3600,24 +3540,7 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
 
                 tracker.onWalCpRecordFsyncEnd();
 
-                long cpTs = System.currentTimeMillis();
-
-                // This can happen in an unlikely event of two checkpoints happening
-                // within a currentTimeMillis() granularity window.
-                if (cpTs == lastCpTs)
-                    cpTs++;
-
-                lastCpTs = cpTs;
-
-                CheckpointEntry cpEntry = writeCheckpointEntry(
-                    tmpWriteBuf,
-                    cpTs,
-                    cpRec.checkpointId(),
-                    cpPtr,
-                    cpRec,
-                    CheckpointEntryType.START);
-
-                checkpointHist.addCheckpointEntry(cpEntry);
+                writeCheckpointEntry(tmpWriteBuf, cp, CheckpointEntryType.START);
 
                 GridMultiCollectionWrapper<FullPageId> cpPages = splitAndSortCpPagesIfNeeded(cpPagesTuple);
 
@@ -3634,7 +3557,7 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
                             curr.reason)
                         );
 
-                return new Checkpoint(cpEntry, cpPages, curr);
+                return new Checkpoint(cp, cpPages, curr);
             }
             else {
                 if (curr.nextSnapshot)
@@ -3714,19 +3637,27 @@ private void markCheckpointEnd(Checkpoint chp) throws IgniteCheckedException {
                     ((PageMemoryEx)memPlc.pageMemory()).finishCheckpoint();
                 }
 
-                if (chp.hasDelta())
-                    writeCheckpointEntry(
-                        tmpWriteBuf,
-                        chp.cpEntry.checkpointTimestamp(),
-                        chp.cpEntry.checkpointId(),
-                        chp.cpEntry.checkpointMark(),
-                        null,
-                        CheckpointEntryType.END);
-
                 currCheckpointPagesCnt = 0;
             }
 
-            checkpointHist.onCheckpointFinished(chp);
+            if (chp.hasDelta()) {
+                CheckpointEntry cp = prepareCheckpointEntry(
+                    tmpWriteBuf,
+                    chp.cpEntry.timestamp(),
+                    chp.cpEntry.checkpointId(),
+                    chp.cpEntry.checkpointMark(),
+                    null,
+                    CheckpointEntryType.END);
+
+                writeCheckpointEntry(tmpWriteBuf, cp, CheckpointEntryType.END);
+
+                cctx.wal().allowCompressionUntil(chp.cpEntry.checkpointMark());
+            }
+
+            List<CheckpointEntry> removedFromHistory = cpHistory.onCheckpointFinished(chp, truncateWalOnCpFinish);
+
+            for (CheckpointEntry cp : removedFromHistory)
+                removeCheckpointFiles(cp);
 
             if (chp.progress != null)
                 chp.progress.cpFinishFut.onDone();
@@ -3930,18 +3861,7 @@ private WriteCheckpointPages(
     /**
      *
      */
-    private enum CheckpointEntryType {
-        /** */
-        START,
-
-        /** */
-        END
-    }
-
-    /**
-     *
-     */
-    private static class Checkpoint {
+    public static class Checkpoint {
         /** Checkpoint entry. */
         @Nullable private final CheckpointEntry cpEntry;
 
@@ -3977,9 +3897,16 @@ private Checkpoint(
         /**
          * @return {@code true} if this checkpoint contains at least one dirty page.
          */
-        private boolean hasDelta() {
+        public boolean hasDelta() {
             return pagesSize != 0;
         }
+
+        /**
+         * @param walFilesDeleted Wal files deleted.
+         */
+        public void walFilesDeleted(int walFilesDeleted) {
+            this.walFilesDeleted = walFilesDeleted;
+        }
     }
 
     /**
@@ -4037,16 +3964,16 @@ public String toString() {
     }
 
     /**
-     *
+     * Data class representing the state of running/scheduled checkpoint.
      */
     private static class CheckpointProgress {
-        /** */
+        /** Scheduled time of checkpoint. */
         private volatile long nextCpTs;
 
-        /** */
+        /** Checkpoint begin phase future. */
         private GridFutureAdapter cpBeginFut = new GridFutureAdapter<>();
 
-        /** */
+        /** Checkpoint finish phase future. */
         private GridFutureAdapter cpFinishFut = new GridFutureAdapter<Void>() {
             @Override protected boolean onDone(@Nullable Void res, @Nullable Throwable err, boolean cancel) {
                 if (err != null && !cpBeginFut.isDone())
@@ -4056,13 +3983,13 @@ private static class CheckpointProgress {
             }
         };
 
-        /** */
+        /** Flag indicates that snapshot operation will be performed after checkpoint. */
         private volatile boolean nextSnapshot;
 
-        /** */
+        /** Flag indicates that checkpoint is started. */
         private volatile boolean started;
 
-        /** */
+        /** Snapshot operation that should be performed if {@link #nextSnapshot} set to true. */
         private volatile SnapshotOperation snapshotOperation;
 
         /** Partitions destroy queue. */
@@ -4110,540 +4037,6 @@ private static class CheckpointProgressSnapshot implements CheckpointFuture {
         }
     }
 
-    /**
-     * Checkpoint history. Holds chronological ordered map with {@link GridCacheDatabaseSharedManager.CheckpointEntry
-     * CheckpointEntries}. Data is loaded from corresponding checkpoint directory. This directory holds files for
-     * checkpoint start and end.
-     */
-    @SuppressWarnings("PublicInnerClass")
-    public class CheckpointHistory {
-        /**
-         * Maps checkpoint's timestamp (from CP file name) to CP entry.
-         * Using TS provides historical order of CP entries in map ( first is oldest )
-         */
-        private final NavigableMap<Long, CheckpointEntry> histMap = new ConcurrentSkipListMap<>();
-
-        /**
-         * Load history form checkpoint directory.
-         *
-         * @param dir Checkpoint state dir.
-         */
-        private void loadHistory(File dir) throws IgniteCheckedException {
-            if (!dir.exists())
-                return;
-
-            File[] files = dir.listFiles(CP_FILE_FILTER);
-
-            if (!F.isEmpty(files)) {
-                Arrays.sort(files, CP_TS_COMPARATOR);
-
-                ByteBuffer buf = ByteBuffer.allocate(16);
-                buf.order(ByteOrder.nativeOrder());
-
-                for (File file : files) {
-                    Matcher matcher = CP_FILE_NAME_PATTERN.matcher(file.getName());
-
-                    if (matcher.matches()) {
-                        CheckpointEntryType type = CheckpointEntryType.valueOf(matcher.group(3));
-
-                        if (type == CheckpointEntryType.START) {
-                            long cpTs = Long.parseLong(matcher.group(1));
-                            UUID cpId = UUID.fromString(matcher.group(2));
-
-                            WALPointer ptr = readPointer(file, buf);
-
-                            if (ptr == null)
-                                continue;
-
-                            CheckpointEntry entry = createCheckPointEntry(cpTs, ptr, cpId, null, type);
-
-                            histMap.put(cpTs, entry);
-                        }
-                    }
-                }
-            }
-        }
-
-        /**
-         * @param cpTs Checkpoint timestamp.
-         * @return Initialized entry.
-         * @throws IgniteCheckedException If failed to initialize entry.
-         */
-        private CheckpointEntry entry(Long cpTs) throws IgniteCheckedException {
-            CheckpointEntry entry = histMap.get(cpTs);
-
-            if (entry == null)
-                throw new IgniteCheckedException("Checkpoint entry was removed: " + cpTs);
-
-            return entry;
-        }
-
-        /**
-         * @return First checkpoint entry if exists. Otherwise {@code null}.
-         */
-        private CheckpointEntry firstEntry() {
-            Map.Entry<Long,CheckpointEntry> entry = histMap.firstEntry();
-
-            return entry != null ? entry.getValue() : null;
-        }
-
-        /**
-         * Get WAL pointer to low checkpoint bound.
-         *
-         * @return WAL pointer to low checkpoint bound.
-         */
-        public WALPointer lowCheckpointBound() {
-            CheckpointEntry entry = firstEntry();
-
-            return entry != null ? entry.cpMark : null;
-        }
-
-        /**
-         * @return Collection of checkpoint timestamps.
-         */
-        public Collection<Long> checkpoints() {
-            return histMap.keySet();
-        }
-
-        /**
-         * Adds checkpoint entry after the corresponding WAL record has been written to WAL. The checkpoint itself
-         * is not finished yet.
-         *
-         * @param entry Entry to ad.
-         */
-        private void addCheckpointEntry(CheckpointEntry entry) {
-            histMap.put(entry.checkpointTimestamp(), entry);
-        }
-
-        /**
-         * Callback on truncate wal.
-         */
-        private void onWalTruncated(WALPointer ptr) {
-            FileWALPointer highBound = (FileWALPointer)ptr;
-
-            List<CheckpointEntry> cpToRemove = new ArrayList<>();
-
-            for (CheckpointEntry cpEntry : histMap.values()) {
-                FileWALPointer cpPnt = (FileWALPointer)cpEntry.checkpointMark();
-
-                if (highBound.compareTo(cpPnt) <= 0)
-                    break;
-
-                if (cctx.wal().reserved(cpEntry.checkpointMark())) {
-                    U.warn(log, "Could not clear historyMap due to WAL reservation on cpEntry " + cpEntry.cpId +
-                        ", history map size is " + histMap.size());
-
-                    break;
-                }
-
-                if (!removeCheckpointFiles(cpEntry))
-                    cpToRemove.add(cpEntry);
-            }
-
-            for (CheckpointEntry cpEntry : cpToRemove)
-                histMap.remove(cpEntry.cpTs);
-        }
-
-        /**
-         * Clears checkpoint history.
-         */
-        private void onCheckpointFinished(Checkpoint chp) {
-            int deleted = 0;
-
-            boolean dropWal = persistenceCfg.getWalHistorySize() != Integer.MAX_VALUE;
-
-            while (histMap.size() > maxCpHistMemSize) {
-                Map.Entry<Long, CheckpointEntry> entry = histMap.firstEntry();
-
-                CheckpointEntry cpEntry = entry.getValue();
-
-                if (cctx.wal().reserved(cpEntry.checkpointMark())) {
-                    U.warn(log, "Could not clear historyMap due to WAL reservation on cpEntry " + cpEntry.cpId +
-                        ", history map size is " + histMap.size());
-
-                    break;
-                }
-
-                boolean fail = removeCheckpointFiles(cpEntry);
-
-                if (!fail) {
-                    if (dropWal)
-                        deleted += cctx.wal().truncate(null, cpEntry.checkpointMark());
-
-                    histMap.remove(entry.getKey());
-                }
-                else
-                    break;
-            }
-
-            chp.walFilesDeleted = deleted;
-
-            if (!chp.cpPages.isEmpty())
-                cctx.wal().allowCompressionUntil(chp.cpEntry.checkpointMark());
-        }
-
-        /**
-         * @param cpEntry Checkpoint entry.
-         * @return {True} if delete fail.
-         */
-        private boolean removeCheckpointFiles(CheckpointEntry cpEntry) {
-            File startFile = new File(cpDir.getAbsolutePath(), cpEntry.startFile());
-            File endFile = new File(cpDir.getAbsolutePath(), cpEntry.endFile());
-
-            boolean rmvdStart = !startFile.exists() || startFile.delete();
-            boolean rmvdEnd = !endFile.exists() || endFile.delete();
-
-            boolean fail = !rmvdStart || !rmvdEnd;
-
-            if (fail) {
-                U.warn(log, "Failed to remove stale checkpoint files [startFile=" + startFile.getAbsolutePath() +
-                    ", endFile=" + endFile.getAbsolutePath() + ']');
-
-                if (histMap.size() > 2 * maxCpHistMemSize) {
-                    U.error(log, "Too many stale checkpoint entries in the map, will truncate WAL archive anyway.");
-
-                    fail = false;
-                }
-            }
-
-            return fail;
-        }
-    }
-
-    /**
-     * Checkpoint entry.
-     */
-    private static class CheckpointEntry {
-        /** Checkpoint timestamp. */
-        private long cpTs;
-
-        /** Checkpoint end mark. */
-        private WALPointer cpMark;
-
-        /** Checkpoint ID. */
-        private UUID cpId;
-
-        /** */
-        private volatile SoftReference<GroupStateLazyStore> grpStateLazyStore;
-
-        /**
-         * Checkpoint entry constructor.
-         *
-         * If {@code grpStates} is null then it will be inited lazy from wal pointer.
-         *
-         * @param cpTs Checkpoint timestamp.
-         * @param cpMark Checkpoint mark pointer.
-         * @param cpId Checkpoint ID.
-         * @param cacheGrpStates Cache groups states.
-         */
-        private CheckpointEntry(
-            long cpTs,
-            WALPointer cpMark,
-            UUID cpId,
-            @Nullable Map<Integer, CacheState> cacheGrpStates
-        ) {
-            this.cpTs = cpTs;
-            this.cpMark = cpMark;
-            this.cpId = cpId;
-            this.grpStateLazyStore = new SoftReference<>(new GroupStateLazyStore(cacheGrpStates));
-        }
-
-        /**
-         * @return Checkpoint timestamp.
-         */
-        private long checkpointTimestamp() {
-            return cpTs;
-        }
-
-        /**
-         * @return Checkpoint ID.
-         */
-        private UUID checkpointId() {
-            return cpId;
-        }
-
-        /**
-         * @return Checkpoint mark.
-         */
-        private WALPointer checkpointMark() {
-            return cpMark;
-        }
-
-        /**
-         * @return Start file name.
-         */
-        private String startFile() {
-            return checkpointFileName(cpTs, cpId, CheckpointEntryType.START);
-        }
-
-        /**
-         * @return End file name.
-         */
-        private String endFile() {
-            return checkpointFileName(cpTs, cpId, CheckpointEntryType.END);
-        }
-
-        /**
-         * @param cctx Cache shred context.
-         */
-        public Map<Integer, GroupState> groupState(
-            GridCacheSharedContext cctx
-        ) throws IgniteCheckedException {
-            GroupStateLazyStore store = initIfNeeded(cctx);
-
-            return store.grpStates;
-        }
-
-        /**
-         * @param cctx Cache shred context.
-         * @return Group lazy store.
-         */
-        private GroupStateLazyStore initIfNeeded(GridCacheSharedContext cctx) throws IgniteCheckedException {
-            GroupStateLazyStore store = grpStateLazyStore.get();
-
-            if (store == null) {
-                store = new GroupStateLazyStore();
-
-                grpStateLazyStore = new SoftReference<>(store);
-            }
-
-            store.initIfNeeded(cctx, cpMark);
-
-            return store;
-        }
-
-        /**
-         * @param cctx Cache shared context.
-         * @param grpId Cache group ID.
-         * @param part Partition ID.
-         * @return Partition counter or {@code null} if not found.
-         */
-        private Long partitionCounter(GridCacheSharedContext cctx, int grpId, int part) {
-            GroupStateLazyStore store;
-
-            try {
-                store = initIfNeeded(cctx);
-            }
-            catch (IgniteCheckedException e) {
-                return null;
-            }
-
-            return store.partitionCounter(grpId, part);
-        }
-
-        /**
-         *
-         */
-        private static class GroupState {
-            /** */
-            private int[] parts;
-
-            /** */
-            private long[] cnts;
-
-            /** */
-            private int idx;
-
-            /**
-             * @param partsCnt Partitions count.
-             */
-            private GroupState(int partsCnt) {
-                parts = new int[partsCnt];
-                cnts = new long[partsCnt];
-            }
-
-            /**
-             * @param partId Partition ID to add.
-             * @param cntr Partition counter.
-             */
-            public void addPartitionCounter(int partId, long cntr) {
-                if (idx == parts.length)
-                    throw new IllegalStateException("Failed to add new partition to the partitions state " +
-                        "(no enough space reserved) [partId=" + partId + ", reserved=" + parts.length + ']');
-
-                if (idx > 0) {
-                    if (parts[idx - 1] >= partId)
-                        throw new IllegalStateException("Adding partition in a wrong order [prev=" + parts[idx - 1] +
-                            ", cur=" + partId + ']');
-                }
-
-                parts[idx] = partId;
-
-                cnts[idx] = cntr;
-
-                idx++;
-            }
-
-            /**
-             * Gets partition counter by partition ID.
-             *
-             * @param partId Partition ID.
-             * @return Partition update counter (will return {@code -1} if partition is not present in the record).
-             */
-            public long counterByPartition(int partId) {
-                int idx = indexByPartition(partId);
-
-                return idx >= 0 ? cnts[idx] : 0;
-            }
-
-            public long size(){
-                return idx;
-            }
-
-            /**
-             * @param partId Partition ID to search.
-             * @return Non-negative index of partition if found or negative value if not found.
-             */
-            private int indexByPartition(int partId) {
-                return Arrays.binarySearch(parts, 0, idx, partId);
-            }
-
-            /** {@inheritDoc} */
-            @Override public String toString() {
-                return "GroupState [cap=" + parts.length + ", size=" + idx + ']';
-            }
-        }
-
-        /**
-         *  Group state lazy store.
-         */
-        private static class GroupStateLazyStore {
-            /** */
-            private static final AtomicIntegerFieldUpdater<GroupStateLazyStore> initGuardUpdater =
-                AtomicIntegerFieldUpdater.newUpdater(GroupStateLazyStore.class, "initGuard");
-
-            /** Cache states. Initialized lazily. */
-            private volatile Map<Integer, GroupState> grpStates;
-
-            /** */
-            private final CountDownLatch latch;
-
-            /** */
-            @SuppressWarnings("unused")
-            private volatile int initGuard;
-
-            /** Initialization exception. */
-            private IgniteCheckedException initEx;
-
-            /**
-             * Default constructor.
-             */
-            private GroupStateLazyStore() {
-                this(null);
-            }
-
-            /**
-             * @param cacheGrpStates Cache group state.
-             */
-            private GroupStateLazyStore(Map<Integer, CacheState> cacheGrpStates) {
-                CountDownLatch latch;
-
-                if (cacheGrpStates != null) {
-                    initGuard = 1;
-
-                    this.latch = new CountDownLatch(0);
-                }
-                else
-                    this.latch = new CountDownLatch(1);
-
-                this.grpStates = remap(cacheGrpStates);
-            }
-
-            /**
-             * @param stateRec Cache group state.
-             */
-            private Map<Integer, GroupState> remap(Map<Integer, CacheState> stateRec) {
-                if (stateRec == null)
-                    return null;
-
-                Map<Integer, GroupState> grpStates = new HashMap<>(stateRec.size());
-
-                for (Integer grpId : stateRec.keySet()) {
-                    CacheState recState = stateRec.get(grpId);
-
-                    GroupState groupState = new GroupState(recState.size());
-
-                    for (int i = 0; i < recState.size(); i++) {
-                        groupState.addPartitionCounter(
-                            recState.partitionByIndex(i),
-                            recState.partitionCounterByIndex(i)
-                        );
-                    }
-
-                    grpStates.put(grpId, groupState);
-                }
-
-                return grpStates;
-            }
-
-            /**
-             * @param grpId Group id.
-             * @param part Partition id.
-             * @return Partition counter.
-             */
-            private Long partitionCounter(int grpId, int part) {
-                assert initGuard != 0 : initGuard;
-
-                if (initEx != null || grpStates == null)
-                    return null;
-
-                GroupState state = grpStates.get(grpId);
-
-                if (state != null) {
-                    long cntr = state.counterByPartition(part);
-
-                    return cntr < 0 ? null : cntr;
-                }
-
-                return null;
-            }
-
-            /**
-             * @param cctx Cache shared context.
-             * @param ptr Checkpoint wal pointer.
-             * @throws IgniteCheckedException If failed to read WAL entry.
-             */
-            private void initIfNeeded(
-                GridCacheSharedContext cctx,
-                WALPointer ptr
-            ) throws IgniteCheckedException {
-                if (initGuardUpdater.compareAndSet(this, 0, 1)) {
-                    try (WALIterator it = cctx.wal().replay(ptr)) {
-                        if (it.hasNextX()) {
-                            IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
-
-                            CheckpointRecord rec = (CheckpointRecord)tup.get2();
-
-                            Map<Integer, CacheState> stateRec = rec.cacheGroupStates();
-
-                            if (stateRec != null)
-                                this.grpStates = remap(stateRec);
-                            else
-                                grpStates = Collections.emptyMap();
-                        }
-                        else
-                            initEx = new IgniteCheckedException(
-                                "Failed to find checkpoint record at the given WAL pointer: " + ptr);
-                    }
-                    catch (IgniteCheckedException e) {
-                        initEx = e;
-
-                        throw e;
-                    }
-                    finally {
-                        latch.countDown();
-                    }
-                }
-                else {
-                    U.await(latch);
-
-                    if (initEx != null)
-                        throw initEx;
-                }
-            }
-        }
-    }
-
     /**
      *
      */
@@ -4853,8 +4246,11 @@ public DataStorageMetricsImpl persistentStoreMetricsImpl() {
         try {
             if (enabled)
                 metaStorage.remove(key);
-            else
+            else {
                 metaStorage.write(key, true);
+
+                lastCheckpointInapplicableForWalRebalance(grpId);
+            }
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException("Failed to write cache group WAL state [grpId=" + grpId +
@@ -4865,6 +4261,41 @@ public DataStorageMetricsImpl persistentStoreMetricsImpl() {
         }
     }
 
+    /**
+     * Checks that checkpoint with timestamp {@code cpTs} is inapplicable as start point for WAL rebalance for given group {@code grpId}.
+     *
+     * @param cpTs Checkpoint timestamp.
+     * @param grpId Group ID.
+     * @return {@code true} if checkpoint {@code cpTs} is inapplicable as start point for WAL rebalance for {@code grpId}.
+     * @throws IgniteCheckedException If failed to check.
+     */
+    public boolean isCheckpointInapplicableForWalRebalance(Long cpTs, int grpId) throws IgniteCheckedException {
+        return metaStorage.read(checkpointInapplicableCpAndGroupIdToKey(cpTs, grpId)) != null;
+    }
+
+    /**
+     * Set last checkpoint as inapplicable for WAL rebalance for given group {@code grpId}.
+     *
+     * @param grpId Group ID.
+     */
+    @Override public void lastCheckpointInapplicableForWalRebalance(int grpId) {
+        checkpointReadLock();
+
+        try {
+            CheckpointEntry lastCp = cpHistory.lastCheckpoint();
+            long lastCpTs = lastCp != null ? lastCp.timestamp() : 0;
+
+            if (lastCpTs != 0)
+                metaStorage.write(checkpointInapplicableCpAndGroupIdToKey(lastCpTs, grpId), true);
+        }
+        catch (IgniteCheckedException e) {
+            log.error("Failed to mark last checkpoint as inapplicable for WAL rebalance for group: " + grpId, e);
+        }
+        finally {
+            checkpointReadUnlock();
+        }
+    }
+
     /**
      *
      */
@@ -4880,6 +4311,9 @@ private void fillWalDisabledGroups() {
             for (String key : keys) {
                 T2<Integer, Boolean> t2 = walKeyToGroupIdAndLocalFlag(key);
 
+                if (t2 == null)
+                    continue;
+
                 if (t2.get2())
                     initiallyLocalWalDisabledGrps.add(t2.get1());
                 else
@@ -4905,6 +4339,17 @@ private static String walGroupIdToKey(int grpId, boolean local) {
             return WAL_GLOBAL_KEY_PREFIX + grpId;
     }
 
+    /**
+     * Convert checkpoint timestamp and cache group ID to key for {@link #CHECKPOINT_INAPPLICABLE_FOR_REBALANCE} metastorage records.
+     *
+     * @param cpTs Checkpoint timestamp.
+     * @param grpId Group ID.
+     * @return Key.
+     */
+    private static String checkpointInapplicableCpAndGroupIdToKey(long cpTs, int grpId) {
+        return CHECKPOINT_INAPPLICABLE_FOR_REBALANCE + cpTs + "-" + grpId;
+    }
+
     /**
      * Convert WAL state key to cache group ID.
      *
@@ -4914,7 +4359,9 @@ private static String walGroupIdToKey(int grpId, boolean local) {
     private static T2<Integer, Boolean> walKeyToGroupIdAndLocalFlag(String key) {
         if (key.startsWith(WAL_LOCAL_KEY_PREFIX))
             return new T2<>(Integer.parseInt(key.substring(WAL_LOCAL_KEY_PREFIX.length())), true);
-        else
+        else if (key.startsWith(WAL_GLOBAL_KEY_PREFIX))
             return new T2<>(Integer.parseInt(key.substring(WAL_GLOBAL_KEY_PREFIX.length())), false);
+        else
+            return null;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 50c90e1de6860..87a06040b1a78 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -786,7 +786,8 @@ private Metas getOrAllocateCacheMetas() throws IgniteCheckedException {
             int p = partCntrs.partitionAt(i);
             long initCntr = partCntrs.initialUpdateCounterAt(i);
 
-            FileWALPointer startPtr = (FileWALPointer)database.searchPartitionCounter(grp.groupId(), p, initCntr);
+            FileWALPointer startPtr = (FileWALPointer)database.checkpointHistory().searchPartitionCounter(
+                grp.groupId(), p, initCntr);
 
             if (startPtr == null)
                 throw new IgniteCheckedException("Could not find start pointer for partition [part=" + p + ", partCntrSince=" + initCntr + "]");
@@ -1006,7 +1007,7 @@ private void advance() {
                             long from = partMap.initialUpdateCounterAt(idx);
                             long to = partMap.updateCounterAt(idx);
 
-                            if (entry.partitionCounter() >= from && entry.partitionCounter() <= to) {
+                            if (entry.partitionCounter() > from && entry.partitionCounter() <= to) {
                                 if (entry.partitionCounter() == to)
                                     reachedPartitionEnd = true;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index e3ce04d8809ac..06f1c5762a983 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -766,14 +766,25 @@ public void waitForCheckpoint(String reason) throws IgniteCheckedException {
 
     /**
      * @param discoEvt Before exchange for the given discovery event.
+     *
+     * @return {@code True} if partitions have been restored from persistent storage.
      */
-    public void beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws IgniteCheckedException {
-        // No-op.
+    public boolean beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws IgniteCheckedException {
+        return false;
     }
 
     /**
-     * @param fut Partition exchange future.
+     * Called when all partitions have been fully restored and pre-created on node start.
+     *
+     * @throws IgniteCheckedException If failed.
      */
+    public void onStateRestored() throws IgniteCheckedException {
+        // No-op.
+    }
+
+        /**
+         * @param fut Partition exchange future.
+         */
     public void rebuildIndexesIfNeeded(GridDhtPartitionsExchangeFuture fut) {
         // No-op.
     }
@@ -1090,4 +1101,13 @@ public boolean walEnabled(int grpId, boolean local) {
     public void walEnabled(int grpId, boolean enabled, boolean local) {
         // No-op.
     }
+
+    /**
+     * Marks last checkpoint as inapplicable for WAL rebalance for given group {@code grpId}.
+     *
+     * @param grpId Group id.
+     */
+    public void lastCheckpointInapplicableForWalRebalance(int grpId) {
+        // No-op.
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntry.java
new file mode 100644
index 0000000000000..f6433e156070b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntry.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.checkpoint;
+
+import java.lang.ref.SoftReference;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.CacheState;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Class represents checkpoint state.
+ */
+public class CheckpointEntry {
+    /** Checkpoint timestamp. */
+    private final long cpTs;
+
+    /** Checkpoint end mark. */
+    private final WALPointer cpMark;
+
+    /** Checkpoint ID. */
+    private final UUID cpId;
+
+    /** State of groups and partitions snapshotted at the checkpoint begin. */
+    private volatile SoftReference<GroupStateLazyStore> grpStateLazyStore;
+
+    /**
+     * Checkpoint entry constructor.
+     *
+     * If {@code grpStates} is null then it will be inited lazy from wal pointer.
+     *
+     * @param cpTs Checkpoint timestamp.
+     * @param cpMark Checkpoint mark pointer.
+     * @param cpId Checkpoint ID.
+     * @param cacheGrpStates Cache groups states.
+     */
+    public CheckpointEntry(
+        long cpTs,
+        WALPointer cpMark,
+        UUID cpId,
+        @Nullable Map<Integer, CacheState> cacheGrpStates
+    ) {
+        this.cpTs = cpTs;
+        this.cpMark = cpMark;
+        this.cpId = cpId;
+        this.grpStateLazyStore = new SoftReference<>(new GroupStateLazyStore(cacheGrpStates));
+    }
+
+    /**
+     * @return Checkpoint timestamp.
+     */
+    public long timestamp() {
+        return cpTs;
+    }
+
+    /**
+     * @return Checkpoint ID.
+     */
+    public UUID checkpointId() {
+        return cpId;
+    }
+
+    /**
+     * @return Checkpoint mark.
+     */
+    public WALPointer checkpointMark() {
+        return cpMark;
+    }
+
+    /**
+     * @param cctx Cache shred context.
+     */
+    public Map<Integer, GroupState> groupState(
+        GridCacheSharedContext cctx
+    ) throws IgniteCheckedException {
+        GroupStateLazyStore store = initIfNeeded(cctx);
+
+        return store.grpStates;
+    }
+
+    /**
+     * @param cctx Cache shred context.
+     * @return Group lazy store.
+     */
+    private GroupStateLazyStore initIfNeeded(GridCacheSharedContext cctx) throws IgniteCheckedException {
+        GroupStateLazyStore store = grpStateLazyStore.get();
+
+        if (store == null) {
+            store = new GroupStateLazyStore();
+
+            grpStateLazyStore = new SoftReference<>(store);
+        }
+
+        store.initIfNeeded(cctx, cpMark);
+
+        return store;
+    }
+
+    /**
+     * @param cctx Cache shared context.
+     * @param grpId Cache group ID.
+     * @param part Partition ID.
+     * @return Partition counter or {@code null} if not found.
+     */
+    public Long partitionCounter(GridCacheSharedContext cctx, int grpId, int part) {
+        GroupStateLazyStore store;
+
+        try {
+            store = initIfNeeded(cctx);
+        }
+        catch (IgniteCheckedException e) {
+            return null;
+        }
+
+        return store.partitionCounter(grpId, part);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return "CheckpointEntry [id=" + cpId + ", timestamp=" + cpTs + ", ptr=" + cpMark + "]";
+    }
+
+    /**
+     *
+     */
+    public static class GroupState {
+        /** */
+        private int[] parts;
+
+        /** */
+        private long[] cnts;
+
+        /** */
+        private int idx;
+
+        /**
+         * @param partsCnt Partitions count.
+         */
+        private GroupState(int partsCnt) {
+            parts = new int[partsCnt];
+            cnts = new long[partsCnt];
+        }
+
+        /**
+         * @param partId Partition ID to add.
+         * @param cntr Partition counter.
+         */
+        public void addPartitionCounter(int partId, long cntr) {
+            if (idx == parts.length)
+                throw new IllegalStateException("Failed to add new partition to the partitions state " +
+                    "(no enough space reserved) [partId=" + partId + ", reserved=" + parts.length + ']');
+
+            if (idx > 0) {
+                if (parts[idx - 1] >= partId)
+                    throw new IllegalStateException("Adding partition in a wrong order [prev=" + parts[idx - 1] +
+                        ", cur=" + partId + ']');
+            }
+
+            parts[idx] = partId;
+
+            cnts[idx] = cntr;
+
+            idx++;
+        }
+
+        /**
+         * Gets partition counter by partition ID.
+         *
+         * @param partId Partition ID.
+         * @return Partition update counter (will return {@code -1} if partition is not present in the record).
+         */
+        public long counterByPartition(int partId) {
+            int idx = indexByPartition(partId);
+
+            return idx >= 0 ? cnts[idx] : -1;
+        }
+
+        /**
+         *
+         */
+        public long size(){
+            return idx;
+        }
+
+        /**
+         * @param partId Partition ID to search.
+         * @return Non-negative index of partition if found or negative value if not found.
+         */
+        public int indexByPartition(int partId) {
+            return Arrays.binarySearch(parts, 0, idx, partId);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "GroupState [cap=" + parts.length + ", size=" + idx + ']';
+        }
+    }
+
+    /**
+     *  Group state lazy store.
+     */
+    public static class GroupStateLazyStore {
+        /** */
+        private static final AtomicIntegerFieldUpdater<GroupStateLazyStore> initGuardUpdater =
+            AtomicIntegerFieldUpdater.newUpdater(GroupStateLazyStore.class, "initGuard");
+
+        /** Cache states. Initialized lazily. */
+        private volatile Map<Integer, GroupState> grpStates;
+
+        /** */
+        private final CountDownLatch latch;
+
+        /** */
+        @SuppressWarnings("unused")
+        private volatile int initGuard;
+
+        /** Initialization exception. */
+        private IgniteCheckedException initEx;
+
+        /**
+         * Default constructor.
+         */
+        private GroupStateLazyStore() {
+            this(null);
+        }
+
+        /**
+         * @param cacheGrpStates Cache group state.
+         */
+        private GroupStateLazyStore(Map<Integer, CacheState> cacheGrpStates) {
+            if (cacheGrpStates != null) {
+                initGuard = 1;
+
+                latch = new CountDownLatch(0);
+            }
+            else
+                latch = new CountDownLatch(1);
+
+            grpStates = remap(cacheGrpStates);
+        }
+
+        /**
+         * @param stateRec Cache group state.
+         */
+        private Map<Integer, GroupState> remap(Map<Integer, CacheState> stateRec) {
+            if (stateRec == null)
+                return Collections.emptyMap();
+
+            Map<Integer, GroupState> grpStates = new HashMap<>(stateRec.size());
+
+            for (Integer grpId : stateRec.keySet()) {
+                CacheState recState = stateRec.get(grpId);
+
+                GroupState grpState = new GroupState(recState.size());
+
+                for (int i = 0; i < recState.size(); i++) {
+                    byte partState = recState.stateByIndex(i);
+
+                    if (GridDhtPartitionState.fromOrdinal(partState) != GridDhtPartitionState.OWNING)
+                        continue;
+
+                    grpState.addPartitionCounter(
+                        recState.partitionByIndex(i),
+                        recState.partitionCounterByIndex(i)
+                    );
+                }
+
+                grpStates.put(grpId, grpState);
+            }
+
+            return grpStates;
+        }
+
+        /**
+         * @param grpId Group id.
+         * @param part Partition id.
+         * @return Partition counter.
+         */
+        private Long partitionCounter(int grpId, int part) {
+            assert initGuard != 0 : initGuard;
+
+            if (initEx != null || grpStates == null)
+                return null;
+
+            GroupState state = grpStates.get(grpId);
+
+            if (state != null) {
+                long cntr = state.counterByPartition(part);
+
+                return cntr < 0 ? null : cntr;
+            }
+
+            return null;
+        }
+
+        /**
+         * @param cctx Cache shared context.
+         * @param ptr Checkpoint wal pointer.
+         * @throws IgniteCheckedException If failed to read WAL entry.
+         */
+        private void initIfNeeded(
+            GridCacheSharedContext cctx,
+            WALPointer ptr
+        ) throws IgniteCheckedException {
+            if (initGuardUpdater.compareAndSet(this, 0, 1)) {
+                try (WALIterator it = cctx.wal().replay(ptr)) {
+                    if (it.hasNextX()) {
+                        IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+
+                        CheckpointRecord rec = (CheckpointRecord)tup.get2();
+
+                        Map<Integer, CacheState> stateRec = rec.cacheGroupStates();
+
+                        grpStates = remap(stateRec);
+                    }
+                    else
+                        initEx = new IgniteCheckedException(
+                            "Failed to find checkpoint record at the given WAL pointer: " + ptr);
+                }
+                catch (IgniteCheckedException e) {
+                    initEx = e;
+
+                    throw e;
+                }
+                finally {
+                    latch.countDown();
+                }
+            }
+            else {
+                U.await(latch);
+
+                if (initEx != null)
+                    throw initEx;
+            }
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntryType.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntryType.java
new file mode 100644
index 0000000000000..66619bda84de8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntryType.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.checkpoint;
+
+/**
+ * Checkpoint entry types.
+ */
+public enum CheckpointEntryType {
+    /** */
+    START,
+
+    /** */
+    END
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
new file mode 100644
index 0000000000000..d6cc297d0fd4f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
@@ -0,0 +1,382 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.checkpoint;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+import java.util.concurrent.ConcurrentSkipListMap;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
+
+/**
+ * Checkpoint history. Holds chronological ordered map with {@link CheckpointEntry CheckpointEntries}.
+ * Data is loaded from corresponding checkpoint directory.
+ * This directory holds files for checkpoint start and end.
+ */
+public class CheckpointHistory {
+    /** Logger. */
+    private final IgniteLogger log;
+
+    /** Cache shared context. */
+    private final GridCacheSharedContext<?, ?> cctx;
+
+    /**
+     * Maps checkpoint's timestamp (from CP file name) to CP entry.
+     * Using TS provides historical order of CP entries in map ( first is oldest )
+     */
+    private final NavigableMap<Long, CheckpointEntry> histMap = new ConcurrentSkipListMap<>();
+
+    /** The maximal number of checkpoints hold in memory. */
+    private final int maxCpHistMemSize;
+
+    /**
+     * Constructor.
+     *
+     * @param ctx Context.
+     */
+    public CheckpointHistory(GridKernalContext ctx) {
+        cctx = ctx.cache().context();
+        log = ctx.log(getClass());
+
+        DataStorageConfiguration dsCfg = ctx.config().getDataStorageConfiguration();
+
+        maxCpHistMemSize = Math.min(dsCfg.getWalHistorySize(),
+            IgniteSystemProperties.getInteger(IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE, 100));
+    }
+
+    /**
+     * @param checkpoints Checkpoints.
+     */
+    public void initialize(List<CheckpointEntry> checkpoints) {
+        for (CheckpointEntry e : checkpoints)
+            histMap.put(e.timestamp(), e);
+    }
+
+    /**
+     * @param cpTs Checkpoint timestamp.
+     * @return Initialized entry.
+     * @throws IgniteCheckedException If failed to initialize entry.
+     */
+    private CheckpointEntry entry(Long cpTs) throws IgniteCheckedException {
+        CheckpointEntry entry = histMap.get(cpTs);
+
+        if (entry == null)
+            throw new IgniteCheckedException("Checkpoint entry was removed: " + cpTs);
+
+        return entry;
+    }
+
+    /**
+     * @return First checkpoint entry if exists. Otherwise {@code null}.
+     */
+    public CheckpointEntry firstCheckpoint() {
+        Map.Entry<Long,CheckpointEntry> entry = histMap.firstEntry();
+
+        return entry != null ? entry.getValue() : null;
+    }
+
+    /**
+     * @return Last checkpoint entry if exists. Otherwise {@code null}.
+     */
+    public CheckpointEntry lastCheckpoint() {
+        Map.Entry<Long,CheckpointEntry> entry = histMap.lastEntry();
+
+        return entry != null ? entry.getValue() : null;
+    }
+
+    /**
+     * @return First checkpoint WAL pointer if exists. Otherwise {@code null}.
+     */
+    public WALPointer firstCheckpointPointer() {
+        CheckpointEntry entry = firstCheckpoint();
+
+        return entry != null ? entry.checkpointMark() : null;
+    }
+
+    /**
+     * @return Collection of checkpoint timestamps.
+     */
+    public Collection<Long> checkpoints(boolean descending) {
+        if (descending)
+            return histMap.descendingKeySet();
+
+        return histMap.keySet();
+    }
+
+    /**
+     *
+     */
+    public Collection<Long> checkpoints() {
+        return checkpoints(false);
+    }
+
+    /**
+     * Adds checkpoint entry after the corresponding WAL record has been written to WAL. The checkpoint itself
+     * is not finished yet.
+     *
+     * @param entry Entry to add.
+     */
+    public void addCheckpoint(CheckpointEntry entry) {
+        histMap.put(entry.timestamp(), entry);
+    }
+
+    /**
+     * @return {@code true} if there is space for next checkpoint.
+     */
+    public boolean hasSpace() {
+        return histMap.size() + 1 <= maxCpHistMemSize;
+    }
+
+    /**
+     * Clears checkpoint history after WAL truncation.
+     *
+     * @return List of checkpoint entries removed from history.
+     */
+    public List<CheckpointEntry> onWalTruncated(WALPointer ptr) {
+        List<CheckpointEntry> removed = new ArrayList<>();
+
+        FileWALPointer highBound = (FileWALPointer)ptr;
+
+        for (CheckpointEntry cpEntry : histMap.values()) {
+            FileWALPointer cpPnt = (FileWALPointer)cpEntry.checkpointMark();
+
+            if (highBound.compareTo(cpPnt) <= 0)
+                break;
+
+            if (cctx.wal().reserved(cpEntry.checkpointMark())) {
+                U.warn(log, "Could not clear historyMap due to WAL reservation on cp: " + cpEntry +
+                    ", history map size is " + histMap.size());
+
+                break;
+            }
+
+            histMap.remove(cpEntry.timestamp());
+
+            removed.add(cpEntry);
+        }
+
+        return removed;
+    }
+
+    /**
+     * Clears checkpoint history after checkpoint finish.
+     *
+     * @return List of checkpoints removed from history.
+     */
+    public List<CheckpointEntry> onCheckpointFinished(GridCacheDatabaseSharedManager.Checkpoint chp, boolean truncateWal) {
+        List<CheckpointEntry> removed = new ArrayList<>();
+
+        int deleted = 0;
+
+        while (histMap.size() > maxCpHistMemSize) {
+            Map.Entry<Long, CheckpointEntry> entry = histMap.firstEntry();
+
+            CheckpointEntry cpEntry = entry.getValue();
+
+            if (cctx.wal().reserved(cpEntry.checkpointMark())) {
+                U.warn(log, "Could not clear historyMap due to WAL reservation on cpEntry " + cpEntry.checkpointId() +
+                    ", history map size is " + histMap.size());
+
+                break;
+            }
+
+            if (truncateWal)
+                deleted += cctx.wal().truncate(null, cpEntry.checkpointMark());
+
+            histMap.remove(entry.getKey());
+
+            removed.add(cpEntry);
+        }
+
+        chp.walFilesDeleted(deleted);
+
+        return removed;
+    }
+
+    /**
+     * Tries to search for a WAL pointer for the given partition counter start.
+     *
+     * @param grpId Cache group ID.
+     * @param part Partition ID.
+     * @param partCntrSince Partition counter or {@code null} to search for minimal counter.
+     * @return Checkpoint entry or {@code null} if failed to search.
+     */
+    @Nullable public WALPointer searchPartitionCounter(int grpId, int part, long partCntrSince) {
+        CheckpointEntry entry = searchCheckpointEntry(grpId, part, partCntrSince);
+
+        if (entry == null)
+            return null;
+
+        return entry.checkpointMark();
+    }
+
+    /**
+     * Tries to search for a WAL pointer for the given partition counter start.
+     *
+     * @param grpId Cache group ID.
+     * @param part Partition ID.
+     * @param partCntrSince Partition counter or {@code null} to search for minimal counter.
+     * @return Checkpoint entry or {@code null} if failed to search.
+     */
+    @Nullable public CheckpointEntry searchCheckpointEntry(int grpId, int part, long partCntrSince) {
+        for (Long cpTs : checkpoints(true)) {
+            try {
+                CheckpointEntry entry = entry(cpTs);
+
+                Long foundCntr = entry.partitionCounter(cctx, grpId, part);
+
+                if (foundCntr != null && foundCntr <= partCntrSince)
+                    return entry;
+            }
+            catch (IgniteCheckedException ignore) {
+                break;
+            }
+        }
+
+        return null;
+    }
+
+    /**
+     * Finds and reserves earliest valid checkpoint for each of given groups and partitions.
+     *
+     * @param groupsAndPartitions Groups and partitions to find and reserve earliest valid checkpoint.
+     *
+     * @return Map (groupId, Map (partitionId, earliest valid checkpoint to history search)).
+     */
+    public Map<Integer, Map<Integer, CheckpointEntry>> searchAndReserveCheckpoints(
+        final Map<Integer, Set<Integer>> groupsAndPartitions
+    ) {
+        if (F.isEmpty(groupsAndPartitions))
+            return Collections.emptyMap();
+
+        final Map<Integer, Map<Integer, CheckpointEntry>> res = new HashMap<>();
+
+        CheckpointEntry prevReserved = null;
+
+        // Iterate over all possible checkpoints starting from latest and moving to earliest.
+        for (Long cpTs : checkpoints(true)) {
+            CheckpointEntry chpEntry = null;
+
+            try {
+                chpEntry = entry(cpTs);
+
+                boolean reserved = cctx.wal().reserve(chpEntry.checkpointMark());
+
+                // If checkpoint WAL history can't be reserved, stop searching.
+                if (!reserved)
+                    break;
+
+                for (Integer grpId : groupsAndPartitions.keySet())
+                    if (!isCheckpointApplicableForGroup(grpId, chpEntry))
+                        groupsAndPartitions.remove(grpId);
+
+                // All groups are no more applicable, release history and stop searching.
+                if (groupsAndPartitions.isEmpty()) {
+                    cctx.wal().release(chpEntry.checkpointMark());
+
+                    break;
+                }
+
+                // Release previous checkpoint marker.
+                if (prevReserved != null)
+                    cctx.wal().release(prevReserved.checkpointMark());
+
+                prevReserved = chpEntry;
+
+                for (Map.Entry<Integer, CheckpointEntry.GroupState> state : chpEntry.groupState(cctx).entrySet()) {
+                    int grpId = state.getKey();
+                    CheckpointEntry.GroupState cpGroupState = state.getValue();
+
+                    Set<Integer> applicablePartitions = groupsAndPartitions.get(grpId);
+
+                    if (F.isEmpty(applicablePartitions))
+                        continue;
+
+                    Set<Integer> inapplicablePartitions = null;
+
+                    for (Integer partId : applicablePartitions) {
+                        int pIdx = cpGroupState.indexByPartition(partId);
+
+                        if (pIdx >= 0)
+                            res.computeIfAbsent(grpId, k -> new HashMap<>()).put(partId, chpEntry);
+                        else {
+                            if (inapplicablePartitions == null)
+                                inapplicablePartitions = new HashSet<>();
+
+                            // Partition is no more applicable for history search, exclude partition from searching.
+                            inapplicablePartitions.add(partId);
+                        }
+                    }
+
+                    if (!F.isEmpty(inapplicablePartitions))
+                        for (Integer partId : inapplicablePartitions)
+                            applicablePartitions.remove(partId);
+                }
+
+                // Remove groups from search with empty set of applicable partitions.
+                for (Map.Entry<Integer, Set<Integer>> e : groupsAndPartitions.entrySet())
+                    if (e.getValue().isEmpty())
+                        groupsAndPartitions.remove(e.getKey());
+            }
+            catch (IgniteCheckedException ex) {
+                U.error(log, "Failed to process checkpoint: " + (chpEntry != null ? chpEntry : "none"), ex);
+            }
+        }
+
+        return res;
+    }
+
+    /**
+     * Checkpoint is not applicable when:
+     * 1) WAL was disabled somewhere after given checkpoint.
+     * 2) Checkpoint doesn't contain specified {@code grpId}.
+     *
+     * @param grpId Group ID.
+     * @param cp Checkpoint.
+     */
+    private boolean isCheckpointApplicableForGroup(int grpId, CheckpointEntry cp) throws IgniteCheckedException {
+        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) cctx.database();
+
+        if (dbMgr.isCheckpointInapplicableForWalRebalance(cp.timestamp(), grpId))
+            return false;
+
+        if (!cp.groupState(cctx).containsKey(grpId))
+            return false;
+
+        return true;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 5efd5ee6c44b2..15b9109a8f01a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -832,7 +832,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
     }
 
     /** {@inheritDoc} */
-    @Override public void release(WALPointer start) throws IgniteCheckedException {
+    @Override public void release(WALPointer start) {
         assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start;
 
         if (mode == WALMode.NONE)
@@ -1997,14 +1997,8 @@ private void deleteObsoleteRawSegments() {
                     Thread.currentThread().interrupt();
                 }
                 finally {
-                    try {
-                        if (currReservedSegment != -1)
-                            release(new FileWALPointer(currReservedSegment, 0, 0));
-                    }
-                    catch (IgniteCheckedException e) {
-                        U.error(log, "Can't release raw WAL segment [idx=" + currReservedSegment +
-                            "] after compression", e);
-                    }
+                    if (currReservedSegment != -1)
+                        release(new FileWALPointer(currReservedSegment, 0, 0));
                 }
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java
index 7edcea9d459b2..bdadf970bbd81 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/misc/VisorWalTask.java
@@ -186,7 +186,7 @@ Collection<String> getUnusedWalSegments(
             GridCacheDatabaseSharedManager dbMgr,
             FileWriteAheadLogManager wal
         ) throws IgniteCheckedException{
-            WALPointer lowBoundForTruncate = dbMgr.checkpointHistory().lowCheckpointBound();
+            WALPointer lowBoundForTruncate = dbMgr.checkpointHistory().firstCheckpointPointer();
 
             if (lowBoundForTruncate == null)
                 return Collections.emptyList();
@@ -227,7 +227,7 @@ Collection<String> deleteUnusedWalSegments(
             GridCacheDatabaseSharedManager dbMgr,
             FileWriteAheadLogManager wal
         ) throws IgniteCheckedException {
-            WALPointer lowBoundForTruncate = dbMgr.checkpointHistory().lowCheckpointBound();
+            WALPointer lowBoundForTruncate = dbMgr.checkpointHistory().firstCheckpointPointer();
 
             if (lowBoundForTruncate == null)
                 return Collections.emptyList();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
index a090381ff9863..f06494b965235 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
@@ -18,11 +18,22 @@
 package org.apache.ignite.internal.processors.cache.persistence;
 
 import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRebalanceTest;
 
 /**
  *
  */
 public class IgnitePdsAtomicCacheHistoricalRebalancingTest extends IgnitePdsAtomicCacheRebalancingTest {
+    /** {@inheritDoc */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCommunicationSpi(new IgniteWalRebalanceTest.WalRebalanceCheckingCommunicationSpi());
+
+        return cfg;
+    }
+
     /** {@inheritDoc */
     @Override protected void beforeTest() throws Exception {
         // Use rebalance from WAL if possible.
@@ -33,8 +44,16 @@ public class IgnitePdsAtomicCacheHistoricalRebalancingTest extends IgnitePdsAtom
 
     /** {@inheritDoc */
     @Override protected void afterTest() throws Exception {
+        boolean walRebalanceInvoked = !IgniteWalRebalanceTest.WalRebalanceCheckingCommunicationSpi.allRebalances()
+            .isEmpty();
+
+        IgniteWalRebalanceTest.WalRebalanceCheckingCommunicationSpi.cleanup();
+
         System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
 
         super.afterTest();
+
+        if (!walRebalanceInvoked)
+            throw new AssertionError("WAL rebalance hasn't been invoked.");
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
index c0c2be9f86943..347412df79965 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -18,9 +18,11 @@
 package org.apache.ignite.internal.processors.cache.persistence;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
@@ -47,7 +49,6 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -66,12 +67,21 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
     /** */
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
-    /** Cache name. */
-    private static final String cacheName = "cache";
+    /** Default cache. */
+    private static final String CACHE = "cache";
+
+    /** Cache with node filter. */
+    private static final String FILTERED_CACHE = "filtered";
+
+    /** Cache with enabled indexes. */
+    private static final String INDEXED_CACHE = "indexed";
 
     /** */
     protected boolean explicitTx;
 
+    /** Set to enable filtered cache on topology. */
+    private boolean filteredCacheEnabled;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -80,18 +90,19 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
 
         cfg.setRebalanceThreadPoolSize(2);
 
-        CacheConfiguration ccfg1 = cacheConfiguration(cacheName)
+        CacheConfiguration ccfg1 = cacheConfiguration(CACHE)
             .setPartitionLossPolicy(PartitionLossPolicy.READ_WRITE_SAFE)
-            .setBackups(1)
+            .setBackups(2)
             .setRebalanceMode(CacheRebalanceMode.ASYNC)
             .setIndexedTypes(Integer.class, Integer.class)
             .setAffinity(new RendezvousAffinityFunction(false, 32))
             .setRebalanceBatchesPrefetchCount(2)
             .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
 
-        CacheConfiguration ccfg2 = cacheConfiguration("indexed");
-        ccfg2.setBackups(1);
-        ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+        CacheConfiguration ccfg2 = cacheConfiguration(INDEXED_CACHE)
+            .setBackups(2)
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
 
         QueryEntity qryEntity = new QueryEntity(Integer.class.getName(), TestValue.class.getName());
 
@@ -108,36 +119,34 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
 
         ccfg2.setQueryEntities(Collections.singleton(qryEntity));
 
-        // Do not start filtered cache on coordinator.
-        if (gridName.endsWith("0")) {
-            cfg.setCacheConfiguration(ccfg1, ccfg2);
-        }
-        else {
-            CacheConfiguration ccfg3 = cacheConfiguration("filtered");
-            ccfg3.setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_SAFE);
-            ccfg3.setBackups(1);
-            ccfg3.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-            ccfg3.setNodeFilter(new CoordinatorNodeFilter());
-
-            cfg.setCacheConfiguration(ccfg1, ccfg2, ccfg3);
-        }
-
-        DataStorageConfiguration memCfg = new DataStorageConfiguration();
+        List<CacheConfiguration> cacheCfgs = new ArrayList<>();
+        cacheCfgs.add(ccfg1);
+        cacheCfgs.add(ccfg2);
 
-        memCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
-        memCfg.setPageSize(1024);
-        memCfg.setWalMode(WALMode.LOG_ONLY);
+        if (filteredCacheEnabled && !gridName.endsWith("0")) {
+            CacheConfiguration ccfg3 = cacheConfiguration(FILTERED_CACHE)
+                .setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_SAFE)
+                .setBackups(2)
+                .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+                .setNodeFilter(new CoordinatorNodeFilter());
 
-        DataRegionConfiguration memPlcCfg = new DataRegionConfiguration();
+            cacheCfgs.add(ccfg3);
+        }
 
-        memPlcCfg.setName("dfltDataRegion");
-        memPlcCfg.setMaxSize(150 * 1024 * 1024);
-        memPlcCfg.setInitialSize(100 * 1024 * 1024);
-        memPlcCfg.setPersistenceEnabled(true);
+        cfg.setCacheConfiguration(asArray(cacheCfgs));
 
-        memCfg.setDefaultDataRegionConfiguration(memPlcCfg);
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+            .setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4)
+            .setPageSize(1024)
+            .setCheckpointFrequency(10 * 1000)
+            .setWalMode(WALMode.LOG_ONLY)
+            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setName("dfltDataRegion")
+                .setPersistenceEnabled(true)
+                .setMaxSize(512 * 1024 * 1024)
+            );
 
-        cfg.setDataStorageConfiguration(memCfg);
+        cfg.setDataStorageConfiguration(dsCfg);
 
         cfg.setDiscoverySpi(
             new TcpDiscoverySpi()
@@ -147,6 +156,17 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
         return cfg;
     }
 
+    /**
+     * @param cacheCfgs Cache cfgs.
+     */
+    private static CacheConfiguration[] asArray(List<CacheConfiguration> cacheCfgs) {
+        CacheConfiguration[] res = new CacheConfiguration[cacheCfgs.size()];
+        for (int i = 0; i < res.length; i++)
+            res[i] = cacheCfgs.get(i);
+
+        return res;
+    }
+
     /** {@inheritDoc} */
     @Override protected long getTestTimeout() {
         return 20 * 60 * 1000;
@@ -185,15 +205,15 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
     public void testRebalancingOnRestart() throws Exception {
         Ignite ignite0 = startGrid(0);
 
-        ignite0.cluster().active(true);
-
         startGrid(1);
 
         IgniteEx ignite2 = startGrid(2);
 
+        ignite0.cluster().active(true);
+
         awaitPartitionMapExchange();
 
-        IgniteCache<Integer, Integer> cache1 = ignite0.cache(cacheName);
+        IgniteCache<Integer, Integer> cache1 = ignite0.cache(CACHE);
 
         for (int i = 0; i < 5000; i++)
             cache1.put(i, i);
@@ -222,7 +242,7 @@ public void testRebalancingOnRestart() throws Exception {
 
         awaitPartitionMapExchange();
 
-        IgniteCache<Integer, Integer> cache3 = ignite2.cache(cacheName);
+        IgniteCache<Integer, Integer> cache3 = ignite2.cache(CACHE);
 
         for (int i = 0; i < 100; i++)
             assertEquals(String.valueOf(i), (Integer)(i * 2), cache3.get(i));
@@ -243,28 +263,21 @@ public void testRebalancingOnRestartAfterCheckpoint() throws Exception {
 
         ignite0.cluster().active(true);
 
-        ignite0.cache(cacheName).rebalance().get();
-        ignite1.cache(cacheName).rebalance().get();
-        ignite2.cache(cacheName).rebalance().get();
-        ignite3.cache(cacheName).rebalance().get();
-
         awaitPartitionMapExchange();
 
-        IgniteCache<Integer, Integer> cache1 = ignite0.cache(cacheName);
+        IgniteCache<Integer, Integer> cache1 = ignite0.cache(CACHE);
 
         for (int i = 0; i < 1000; i++)
             cache1.put(i, i);
 
-        ignite0.context().cache().context().database().waitForCheckpoint("test");
-        ignite1.context().cache().context().database().waitForCheckpoint("test");
+        forceCheckpoint(ignite0);
+        forceCheckpoint(ignite1);
 
         info("++++++++++ After checkpoint");
 
         ignite2.close();
         ignite3.close();
 
-        resetBaselineTopology();
-
         ignite0.resetLostPartitions(Collections.singletonList(cache1.getName()));
 
         assert cache1.lostPartitions().isEmpty();
@@ -281,116 +294,20 @@ public void testRebalancingOnRestartAfterCheckpoint() throws Exception {
 
         info(">>> Done puts...");
 
-        ignite2 = startGrid(2);
-        ignite3 = startGrid(3);
-
-        ignite2.cache(cacheName).rebalance().get();
-        ignite3.cache(cacheName).rebalance().get();
-
-        IgniteCache<Integer, Integer> cache2 = ignite2.cache(cacheName);
-        IgniteCache<Integer, Integer> cache3 = ignite3.cache(cacheName);
-
-        for (int i = 0; i < 100; i++) {
-            assertEquals(String.valueOf(i), (Integer)(i * 2), cache2.get(i));
-            assertEquals(String.valueOf(i), (Integer)(i * 2), cache3.get(i));
-        }
-    }
-
-    /**
-     * Test that all data is correctly restored after non-graceful restart.
-     *
-     * @throws Exception If fails.
-     */
-    public void testDataCorrectnessAfterRestart() throws Exception {
-        IgniteEx ignite1 = (IgniteEx)G.start(getConfiguration("test1"));
-        IgniteEx ignite2 = (IgniteEx)G.start(getConfiguration("test2"));
-        IgniteEx ignite3 = (IgniteEx)G.start(getConfiguration("test3"));
-        IgniteEx ignite4 = (IgniteEx)G.start(getConfiguration("test4"));
-
-        ignite1.cluster().active(true);
+        startGrid(2);
+        startGrid(3);
 
         awaitPartitionMapExchange();
 
-        IgniteCache<Integer, Integer> cache1 = ignite1.cache(cacheName);
-
-        for (int i = 0; i < 100; i++)
-            cache1.put(i, i);
-
-        ignite1.close();
-        ignite2.close();
-        ignite3.close();
-        ignite4.close();
-
-        ignite1 = (IgniteEx)G.start(getConfiguration("test1"));
-        ignite2 = (IgniteEx)G.start(getConfiguration("test2"));
-        ignite3 = (IgniteEx)G.start(getConfiguration("test3"));
-        ignite4 = (IgniteEx)G.start(getConfiguration("test4"));
+        ignite2 = grid(2);
+        ignite3 = grid(3);
 
-        ignite1.cluster().active(true);
-
-        awaitPartitionMapExchange();
-
-        cache1 = ignite1.cache(cacheName);
-        IgniteCache<Integer, Integer> cache2 = ignite2.cache(cacheName);
-        IgniteCache<Integer, Integer> cache3 = ignite3.cache(cacheName);
-        IgniteCache<Integer, Integer> cache4 = ignite4.cache(cacheName);
+        IgniteCache<Integer, Integer> cache2 = ignite2.cache(CACHE);
+        IgniteCache<Integer, Integer> cache3 = ignite3.cache(CACHE);
 
         for (int i = 0; i < 100; i++) {
-            assert cache1.get(i).equals(i);
-            assert cache2.get(i).equals(i);
-            assert cache3.get(i).equals(i);
-            assert cache4.get(i).equals(i);
-        }
-    }
-
-    /**
-     * Test that partitions are marked as lost when all owners leave cluster, but recover after nodes rejoin.
-     *
-     * @throws Exception If fails.
-     */
-    public void testPartitionLossAndRecover() throws Exception {
-        Ignite ignite1 = startGrid(0);
-        Ignite ignite2 = startGrid(1);
-        Ignite ignite3 = startGrid(2);
-        Ignite ignite4 = startGrid(3);
-
-        ignite1.cluster().active(true);
-
-        awaitPartitionMapExchange();
-
-        IgniteCache<String, String> cache1 = ignite1.cache(cacheName);
-
-        final int offset = 10;
-
-        for (int i = 0; i < 100; i++)
-            cache1.put(String.valueOf(i), String.valueOf(i + offset));
-
-        ignite3.close();
-        ignite4.close();
-
-        awaitPartitionMapExchange();
-
-        assert !ignite1.cache(cacheName).lostPartitions().isEmpty();
-
-        ignite3 = startGrid(2);
-        ignite4 = startGrid(3);
-
-        ignite1.resetLostPartitions(Collections.singletonList(cacheName));
-
-        IgniteCache<String, String> cache2 = ignite2.cache(cacheName);
-        IgniteCache<String, String> cache3 = ignite3.cache(cacheName);
-        IgniteCache<String, String> cache4 = ignite4.cache(cacheName);
-
-        //Thread.sleep(5_000);
-
-        for (int i = 0; i < 100; i++) {
-            String key = String.valueOf(i);
-            String expected = String.valueOf(i + offset);
-
-            assertEquals(expected, cache1.get(key));
-            assertEquals(expected, cache2.get(key));
-            assertEquals(expected, cache3.get(key));
-            assertEquals(expected, cache4.get(key));
+            assertEquals(String.valueOf(i), (Integer)(i * 2), cache2.get(i));
+            assertEquals(String.valueOf(i), (Integer)(i * 2), cache3.get(i));
         }
     }
 
@@ -401,26 +318,26 @@ public void testTopologyChangesWithConstantLoad() throws Exception {
         final long timeOut = U.currentTimeMillis() + 10 * 60 * 1000;
 
         final int entriesCnt = 10_000;
-        int maxNodesCount = 4;
-        int topChanges = 20;
-        final String cacheName = "indexed";
+        final int maxNodesCnt = 4;
+        final int topChanges = 50;
 
         final AtomicBoolean stop = new AtomicBoolean();
+        final AtomicBoolean suspend = new AtomicBoolean();
 
         final ConcurrentMap<Integer, TestValue> map = new ConcurrentHashMap<>();
 
-        Ignite ignite = startGrid(0);
+        Ignite ignite = startGridsMultiThreaded(4);
 
         ignite.cluster().active(true);
 
-        IgniteCache<Integer, TestValue> cache = ignite.cache(cacheName);
+        IgniteCache<Integer, TestValue> cache = ignite.cache(INDEXED_CACHE);
 
         for (int i = 0; i < entriesCnt; i++) {
             cache.put(i, new TestValue(i, i));
             map.put(i, new TestValue(i, i));
         }
 
-        final AtomicInteger nodesCnt = new AtomicInteger();
+        final AtomicInteger nodesCnt = new AtomicInteger(4);
 
         IgniteInternalFuture fut = runMultiThreadedAsync(new Callable<Void>() {
             @Override public Void call() throws Exception {
@@ -428,6 +345,12 @@ public void testTopologyChangesWithConstantLoad() throws Exception {
                     if (stop.get())
                         return null;
 
+                    if (suspend.get()) {
+                        U.sleep(10);
+
+                        continue;
+                    }
+
                     int k = ThreadLocalRandom.current().nextInt(entriesCnt);
                     int v1 = ThreadLocalRandom.current().nextInt();
                     int v2 = ThreadLocalRandom.current().nextInt();
@@ -456,7 +379,7 @@ public void testTopologyChangesWithConstantLoad() throws Exception {
                         tx = ignite.transactions().txStart();
 
                     try {
-                        ignite.cache(cacheName).put(k, new TestValue(v1, v2));
+                        ignite.cache(INDEXED_CACHE).put(k, new TestValue(v1, v2));
                     }
                     catch (Exception ignored) {
                         success = false;
@@ -478,8 +401,10 @@ public void testTopologyChangesWithConstantLoad() throws Exception {
             }
         }, 1, "load-runner");
 
+        boolean[] changes = new boolean[] {false, false, true, true};
+
         try {
-            for (int i = 0; i < topChanges; i++) {
+            for (int it = 0; it < topChanges; it++) {
                 if (U.currentTimeMillis() > timeOut)
                     break;
 
@@ -487,21 +412,30 @@ public void testTopologyChangesWithConstantLoad() throws Exception {
 
                 boolean add;
 
-                if (nodesCnt.get() <= maxNodesCount / 2)
+                if (it < changes.length)
+                    add = changes[it];
+                else if (nodesCnt.get() <= maxNodesCnt / 2)
                     add = true;
-                else if (nodesCnt.get() > maxNodesCount)
+                else if (nodesCnt.get() >= maxNodesCnt)
                     add = false;
                 else // More chance that node will be added
                     add = ThreadLocalRandom.current().nextInt(3) <= 1;
 
                 if (add)
-                    startGrid(nodesCnt.incrementAndGet());
+                    startGrid(nodesCnt.getAndIncrement());
                 else
-                    stopGrid(nodesCnt.getAndDecrement());
+                    stopGrid(nodesCnt.decrementAndGet());
 
                 awaitPartitionMapExchange();
 
-                cache.rebalance().get();
+                suspend.set(true);
+
+                U.sleep(200);
+
+                for (Map.Entry<Integer, TestValue> entry : map.entrySet())
+                    assertEquals(it + " " + Integer.toString(entry.getKey()), entry.getValue(), cache.get(entry.getKey()));
+
+                suspend.set(false);
             }
         }
         finally {
@@ -520,14 +454,21 @@ else if (nodesCnt.get() > maxNodesCount)
      * @throws Exception If failed.
      */
     public void testForceRebalance() throws Exception {
-        testForceRebalance(cacheName);
+        testForceRebalance(CACHE);
     }
 
     /**
      * @throws Exception If failed.
      */
     public void testForceRebalanceClientTopology() throws Exception {
-        testForceRebalance("filtered");
+        filteredCacheEnabled = true;
+
+        try {
+            testForceRebalance(FILTERED_CACHE);
+        }
+        finally {
+            filteredCacheEnabled = false;
+        }
     }
 
     /**
@@ -579,63 +520,52 @@ public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception
 
         ig.cluster().active(true);
 
-        int k = 0;
+        int keys = 0;
 
-        try (IgniteDataStreamer ds = ig.dataStreamer(cacheName)) {
+        try (IgniteDataStreamer<Object, Object> ds = ig.dataStreamer(CACHE)) {
             ds.allowOverwrite(true);
 
-            for (int k0 = k; k < k0 + 10_000; k++)
-                ds.addData(k, k);
+            for (; keys < 10_000; keys++)
+                ds.addData(keys, keys);
         }
 
-        for (int t = 0; t < 5; t++) {
-            int t0 = t;
+        for (int it = 0; it < 10; it++) {
+            final int it0 = it;
+
             IgniteInternalFuture fut = GridTestUtils.runAsync(() -> {
                 try {
                     stopGrid(3);
 
-                    forceCheckpoint();
-
                     U.sleep(500); // Wait for data load.
 
-                    IgniteEx ig0 = startGrid(3);
+                    startGrid(3);
 
-                    U.sleep(2000); // Wait for node join.
+                    U.sleep(500); // Wait for data load.
 
-                    if (t0 % 2 == 1) {
+                    if (it0 % 2 != 0) {
                         stopGrid(2);
 
-                        awaitPartitionMapExchange();
-
-                        forceCheckpoint();
+                        U.sleep(500); // Wait for data load.
 
                         startGrid(2);
-
-                        awaitPartitionMapExchange();
                     }
 
-                    ig0.cache(cacheName).rebalance().get();
+                    awaitPartitionMapExchange();
                 }
                 catch (Exception e) {
                     error("Unable to start/stop grid", e);
+
                     throw new RuntimeException(e);
                 }
             });
 
-            try (IgniteDataStreamer ds = ig.dataStreamer(cacheName)) {
-                ds.allowOverwrite(true);
-
-                while (!fut.isDone()) {
-                    int k0 = k;
+            IgniteCache<Object, Object> cache = ig.cache(CACHE);
 
-                    for (;k < k0 + 3; k++)
-                        ds.addData(k, k);
+            while (!fut.isDone()) {
+                int nextKeys = keys + 10;
 
-                    U.sleep(10);
-                }
-            }
-            catch (Exception e) {
-                log.error("Unable to write data", e);
+                for (;keys < nextKeys; keys++)
+                    cache.put(keys, keys);
             }
 
             fut.get();
@@ -647,18 +577,18 @@ public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception
             for (int g = 0; g < 4; g++) {
                 IgniteEx ig0 = grid(g);
 
-                for (GridDhtLocalPartition part : ig0.cachex(cacheName).context().topology().currentLocalPartitions()) {
+                for (GridDhtLocalPartition part : ig0.cachex(CACHE).context().topology().currentLocalPartitions()) {
                     if (cntrs.containsKey(part.id()))
                         assertEquals(String.valueOf(part.id()), (long) cntrs.get(part.id()), part.updateCounter());
                     else
                         cntrs.put(part.id(), part.updateCounter());
                 }
 
-                for (int k0 = 0; k0 < k; k0++)
-                    assertEquals(String.valueOf(k0) + " " + g, k0, ig0.cache(cacheName).get(k0));
+                for (int k0 = 0; k0 < keys; k0++)
+                    assertEquals(String.valueOf(k0) + " " + g, k0, ig0.cache(CACHE).get(k0));
             }
 
-            assertEquals(ig.affinity(cacheName).partitions(), cntrs.size());
+            assertEquals(ig.affinity(CACHE).partitions(), cntrs.size());
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java
deleted file mode 100644
index 179c8e0fcf399..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheHistoricalRebalancingTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal.processors.cache.persistence;
-
-import org.apache.ignite.IgniteSystemProperties;
-
-/**
- *
- */
-public class IgnitePdsTxCacheHistoricalRebalancingTest extends IgnitePdsTxCacheRebalancingTest {
-    /** {@inheritDoc */
-    @Override protected void beforeTest() throws Exception {
-        // Use rebalance from WAL if possible.
-        System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
-
-        super.beforeTest();
-    }
-
-    /** {@inheritDoc */
-    @Override protected void afterTest() throws Exception {
-        System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
-
-        super.afterTest();
-    }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
index ca46a75e09181..f365448d40f75 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -24,7 +24,6 @@
 import java.nio.file.OpenOption;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
-import com.sun.org.apache.regexp.internal.RE;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
@@ -39,6 +38,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -214,7 +214,7 @@ private void doTestSimple() throws Exception {
 
         IgniteEx newIgnite = startGrid(3);
 
-        final GridCacheDatabaseSharedManager.CheckpointHistory cpHistory =
+        final CheckpointHistory cpHistory =
             ((GridCacheDatabaseSharedManager)newIgnite.context().cache().context().database()).checkpointHistory();
 
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
index 0f522544107e4..06a9ec271ef25 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsUnusedWalSegmentsTest.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -49,6 +50,8 @@ public class IgnitePdsUnusedWalSegmentsTest extends GridCommonAbstractTest {
 
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
+        cfg.setConsistentId(gridName);
+
         cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
 
         CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
@@ -102,7 +105,7 @@ public void testWalManagerRangeReservation() throws Exception {
 
             assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
 
-            FileWALPointer lowPtr = (FileWALPointer)dbMgr.checkpointHistory().lowCheckpointBound();
+            FileWALPointer lowPtr = (FileWALPointer)dbMgr.checkpointHistory().firstCheckpointPointer();
 
             assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
 
@@ -136,7 +139,7 @@ public void testUnusedWalTruncate() throws Exception {
 
             assertTrue("Expected that at least resIdx greater than 0, real is " + resIdx, resIdx > 0);
 
-            FileWALPointer lowPtr = (FileWALPointer) dbMgr.checkpointHistory().lowCheckpointBound();
+            FileWALPointer lowPtr = (FileWALPointer) dbMgr.checkpointHistory().firstCheckpointPointer();
 
             assertTrue("Expected that dbMbr returns valid resIdx", lowPtr.index() == resIdx);
 
@@ -186,17 +189,10 @@ private IgniteEx prepareGrid(int cnt) throws Exception {
      * Get index of reserved WAL segment by checkpointer.
      *
      * @param dbMgr Database shared manager.
-     * @throws Exception If failed.
      */
-    private long getReservedWalSegmentIndex(GridCacheDatabaseSharedManager dbMgr) throws Exception{
-        GridCacheDatabaseSharedManager.CheckpointHistory cpHist = dbMgr.checkpointHistory();
-
-        Object histMap = GridTestUtils.getFieldValue(cpHist, "histMap");
-
-        Object cpEntry = GridTestUtils.getFieldValue(GridTestUtils.invoke(histMap, "firstEntry"), "value");
-
-        FileWALPointer walPtr = GridTestUtils.getFieldValue(cpEntry, "cpMark");
+    private long getReservedWalSegmentIndex(GridCacheDatabaseSharedManager dbMgr) {
+        CheckpointHistory cpHist = dbMgr.checkpointHistory();
 
-        return walPtr.index();
+        return ((FileWALPointer) cpHist.firstCheckpointPointer()).index();
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
index 23dda265fb5eb..28801cd6c20bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
@@ -17,32 +17,52 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.db.wal;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
 
 /**
- * Historic WAL rebalance base test.
+ * Historical WAL rebalance base test.
  */
 public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
     /** Cache name. */
     private static final String CACHE_NAME = "cache";
 
+    /** Partitions count. */
+    private static final int PARTS_CNT = 32;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0"); //to make all rebalance wal-based
@@ -51,25 +71,24 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
 
         cfg.setConsistentId(gridName);
 
-        CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(CACHE_NAME);
-
-        ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
-        ccfg.setRebalanceMode(CacheRebalanceMode.ASYNC);
-
-        ccfg.setCacheMode(CacheMode.REPLICATED);
-
-        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+        CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>(CACHE_NAME)
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setRebalanceMode(CacheRebalanceMode.ASYNC)
+            .setCacheMode(CacheMode.REPLICATED)
+            .setAffinity(new RendezvousAffinityFunction(false, PARTS_CNT));
 
         cfg.setCacheConfiguration(ccfg);
 
         DataStorageConfiguration dbCfg = new DataStorageConfiguration()
                     .setWalHistorySize(Integer.MAX_VALUE)
                     .setWalMode(WALMode.LOG_ONLY)
+                    .setCheckpointFrequency(15 * 60 * 1000)
                     .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true));
 
         cfg.setDataStorageConfiguration(dbCfg);
 
+        cfg.setCommunicationSpi(new WalRebalanceCheckingCommunicationSpi());
+
         return cfg;
     }
 
@@ -83,10 +102,19 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         System.clearProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
+        System.clearProperty(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING);
+
+        boolean walRebalanceInvoked = !IgniteWalRebalanceTest.WalRebalanceCheckingCommunicationSpi.allRebalances()
+            .isEmpty();
+
+        IgniteWalRebalanceTest.WalRebalanceCheckingCommunicationSpi.cleanup();
 
         stopAllGrids();
 
         cleanPersistenceDir();
+
+        if (!walRebalanceInvoked)
+            throw new AssertionError("WAL rebalance hasn't been invoked.");
     }
 
     /**
@@ -97,7 +125,8 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
     public void testSimple() throws Exception {
         IgniteEx ig0 = startGrid(0);
         IgniteEx ig1 = startGrid(1);
-        final int entryCnt = 10_000;
+
+        final int entryCnt = PARTS_CNT * 100;
 
         ig0.cluster().active(true);
 
@@ -135,7 +164,8 @@ public void testSimple() throws Exception {
     public void testRebalanceRemoves() throws Exception {
         IgniteEx ig0 = startGrid(0);
         IgniteEx ig1 = startGrid(1);
-        final int entryCnt = 10_000;
+
+        final int entryCnt = PARTS_CNT * 100;
 
         ig0.cluster().active(true);
 
@@ -173,6 +203,164 @@ public void testRebalanceRemoves() throws Exception {
         }
     }
 
+    /**
+     * Test that WAL rebalance is not invoked if there are gaps in WAL history due to temporary WAL disabling.
+     *
+     * @throws Exception If failed.
+     */
+    public void testWithLocalWalChange() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING, "true");
+
+        IgniteEx crd = (IgniteEx) startGrids(4);
+
+        crd.cluster().active(true);
+
+        final int entryCnt = PARTS_CNT * 10;
+
+        {
+            IgniteCache<Object, Object> cache = crd.cache(CACHE_NAME);
+
+            for (int k = 0; k < entryCnt; k++)
+                cache.put(k, new IndexedObject(k - 1));
+        }
+
+        stopAllGrids();
+
+        IgniteEx ig0 = (IgniteEx) startGrids(2);
+
+        ig0.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = ig0.cache(CACHE_NAME);
+
+        int grpId = ig0.cachex(CACHE_NAME).context().groupId();
+
+        for (int k = 0; k < entryCnt; k++)
+            cache.put(k, new IndexedObject(k));
+
+        // This node should rebalance data from other nodes and shouldn't have WAL history.
+        Ignite ignite = startGrid(2);
+
+        awaitPartitionMapExchange();
+
+        Set<Long> topVers = ((WalRebalanceCheckingCommunicationSpi) ignite.configuration().getCommunicationSpi())
+            .walRebalanceVersions(grpId);
+
+        Assert.assertTrue(topVers.contains(ignite.cluster().topologyVersion()));
+
+        // Rewrite some data.
+        for (int k = 0; k < entryCnt; k++) {
+            if (k % 3 == 0)
+                cache.put(k, new IndexedObject(k + 1));
+            else if (k % 3 == 1) // Spread removes across all partitions.
+                cache.remove(k);
+        }
+
+        // Stop grids which have actual WAL history.
+        stopGrid(0);
+
+        stopGrid(1);
+
+        // Start new node which should rebalance all data from node(2) without using WAL,
+        // because node(2) doesn't have full history for rebalance.
+        ignite = startGrid(3);
+
+        awaitPartitionMapExchange();
+
+        topVers = ((WalRebalanceCheckingCommunicationSpi) ignite.configuration().getCommunicationSpi())
+            .walRebalanceVersions(grpId);
+
+        Assert.assertFalse(topVers.contains(ignite.cluster().topologyVersion()));
+
+        // Check data consistency.
+        for (Ignite ig : G.allGrids()) {
+            IgniteCache<Object, Object> cache1 = ig.cache(CACHE_NAME);
+
+            for (int k = 0; k < entryCnt; k++) {
+                if (k % 3 == 0)
+                    assertEquals(new IndexedObject(k + 1), cache1.get(k));
+                else if (k % 3 == 1)
+                    assertNull(cache1.get(k));
+                else
+                    assertEquals(new IndexedObject(k), cache1.get(k));
+            }
+        }
+    }
+
+    /**
+     * Test that WAL rebalance is not invoked if there are gaps in WAL history due to global WAL disabling.
+     *
+     * @throws Exception If failed.
+     */
+    public void testWithGlobalWalChange() throws Exception {
+        // Prepare some data.
+        IgniteEx crd = (IgniteEx) startGrids(3);
+
+        crd.cluster().active(true);
+
+        final int entryCnt = PARTS_CNT * 10;
+
+        {
+            IgniteCache<Object, Object> cache = crd.cache(CACHE_NAME);
+
+            for (int k = 0; k < entryCnt; k++)
+                cache.put(k, new IndexedObject(k - 1));
+        }
+
+        stopAllGrids();
+
+        // Rewrite data with globally disabled WAL.
+        crd = (IgniteEx) startGrids(2);
+
+        crd.cluster().active(true);
+
+        crd.cluster().disableWal(CACHE_NAME);
+
+        IgniteCache<Object, Object> cache = crd.cache(CACHE_NAME);
+
+        int grpId = crd.cachex(CACHE_NAME).context().groupId();
+
+        for (int k = 0; k < entryCnt; k++)
+            cache.put(k, new IndexedObject(k));
+
+        crd.cluster().enableWal(CACHE_NAME);
+
+        // This node shouldn't rebalance data using WAL, because it was disabled on other nodes.
+        IgniteEx ignite = startGrid(2);
+
+        awaitPartitionMapExchange();
+
+        Set<Long> topVers = ((WalRebalanceCheckingCommunicationSpi) ignite.configuration().getCommunicationSpi())
+            .walRebalanceVersions(grpId);
+
+        Assert.assertFalse(topVers.contains(ignite.cluster().topologyVersion()));
+
+        stopGrid(2);
+
+        // Fix actual state to have start point in WAL to rebalance from.
+        forceCheckpoint();
+
+        // After another rewriting data with enabled WAL, node should rebalance this diff using WAL rebalance.
+        for (int k = 0; k < entryCnt; k++)
+            cache.put(k, new IndexedObject(k + 1));
+
+        ignite = startGrid(2);
+
+        awaitPartitionMapExchange();
+
+        topVers = ((WalRebalanceCheckingCommunicationSpi) ignite.configuration().getCommunicationSpi())
+            .walRebalanceVersions(grpId);
+
+        Assert.assertTrue(topVers.contains(ignite.cluster().topologyVersion()));
+
+        // Check data consistency.
+        for (Ignite ig : G.allGrids()) {
+            IgniteCache<Object, Object> cache1 = ig.cache(CACHE_NAME);
+
+            for (int k = 0; k < entryCnt; k++)
+                assertEquals(new IndexedObject(k + 1), cache1.get(k));
+        }
+    }
+
     /**
      *
      */
@@ -214,4 +402,63 @@ private IndexedObject(int iVal) {
             return S.toString(IndexedObject.class, this);
         }
     }
+
+    /**
+     * Wrapper of communication spi to detect on what topology versions WAL rebalance has happened.
+     */
+    public static class WalRebalanceCheckingCommunicationSpi extends TcpCommunicationSpi {
+        /** (Group ID, Set of topology versions). */
+        private static final Map<Integer, Set<Long>> topVers = new HashMap<>();
+
+        /** Lock object. */
+        private static final Object mux = new Object();
+
+        /**
+         * @param grpId Group ID.
+         * @return Set of topology versions where WAL history has been used for rebalance.
+         */
+        Set<Long> walRebalanceVersions(int grpId) {
+            synchronized (mux) {
+                return Collections.unmodifiableSet(topVers.getOrDefault(grpId, Collections.emptySet()));
+            }
+        }
+
+        /**
+         * @return All topology versions for all groups where WAL rebalance has been used.
+         */
+        public static Map<Integer, Set<Long>> allRebalances() {
+            synchronized (mux) {
+                return Collections.unmodifiableMap(topVers);
+            }
+        }
+
+        /**
+         * Cleans all rebalances history.
+         */
+        public static void cleanup() {
+            synchronized (mux) {
+                topVers.clear();
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (((GridIoMessage)msg).message() instanceof GridDhtPartitionDemandMessage) {
+                GridDhtPartitionDemandMessage demandMsg = (GridDhtPartitionDemandMessage) ((GridIoMessage)msg).message();
+
+                IgniteDhtDemandedPartitionsMap map = demandMsg.partitions();
+
+                if (!map.historicalMap().isEmpty()) {
+                    int grpId = demandMsg.groupId();
+                    long topVer = demandMsg.topologyVersion().topologyVersion();
+
+                    synchronized (mux) {
+                        topVers.computeIfAbsent(grpId, v -> new HashSet<>()).add(topVer);
+                    }
+                }
+            }
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
index 7a50c0d55a44b..808e737244101 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
@@ -54,6 +54,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
@@ -500,7 +501,7 @@ public void testCheckpointHistory() throws Exception {
                 dbMgr.waitForCheckpoint("test");
             }
 
-            GridCacheDatabaseSharedManager.CheckpointHistory hist = dbMgr.checkpointHistory();
+            CheckpointHistory hist = dbMgr.checkpointHistory();
 
             assertTrue(hist.checkpoints().size() <= WAL_HIST_SIZE);
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index d33b20b6f6cf5..571576010995b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -23,7 +23,6 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsBinarySortObjectFieldsTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCorruptedIndexTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsMarshallerMappingRestoreOnNodeStartTest;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheHistoricalRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreCacheGroupsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest;
@@ -65,7 +64,6 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgnitePdsTxCacheRebalancingTest.class);
 
         suite.addTestSuite(IgnitePdsAtomicCacheHistoricalRebalancingTest.class);
-        suite.addTestSuite(IgnitePdsTxCacheHistoricalRebalancingTest.class);
         suite.addTestSuite(IgniteWalRebalanceTest.class);
 
         suite.addTestSuite(IgniteWalRecoveryPPCTest.class);

From 1a8e9fcafa3a6d53ab01901be798cc8957252fe6 Mon Sep 17 00:00:00 2001
From: EdShangGG <eshangareev@gridgain.com>
Date: Fri, 15 Jun 2018 19:35:00 +0300
Subject: [PATCH 192/543] IGNITE-8610 Fix compilation

(cherry picked from commit 55af7d1)
---
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java          | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index f957a042fec1a..07a785c6aad03 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -2406,6 +2406,8 @@ else if (cntr == maxCntr.cnt)
                 maxCntr.nodes.add(cctx.localNodeId());
         }
 
+        int entryLeft = maxCntrs.size();
+
         Map<Integer, Map<Integer, Long>> partHistReserved0 = partHistReserved;
 
         Map<Integer, Long> localReserved = partHistReserved0 != null ? partHistReserved0.get(top.groupId()) : null;

From 298ca37aea710e098d033fcc5750e408b4a07c24 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Wed, 23 May 2018 18:02:23 +0300
Subject: [PATCH 193/543] IGNITE-7809 fixed stable failures of
 IgniteWalFlushDefaultSelfTest in Ignite PDS 2 & PDS 2 Direct IO, new tests
 were added. - Fixes #3569.

Signed-off-by: dpavlov <dpavlov@apache.org>

(cherry picked from commit fed2c02)
---
 .../wal/FileWriteAheadLogManager.java         |  6 +-
 .../FsyncModeFileWriteAheadLogManager.java    | 11 +++-
 ...FlushBackgroundWithMmapBufferSelfTest.java | 28 ++++++++++
 ....java => IgniteWalFlushFsyncSelfTest.java} |  2 +-
 ...FlushFsyncWithDedicatedWorkerSelfTest.java | 39 +++++++++++++
 ...teWalFlushFsyncWithMmapBufferSelfTest.java | 28 ++++++++++
 ...WalFlushLogOnlyWithMmapBufferSelfTest.java | 28 ++++++++++
 ...lushMultiNodeFailoverAbstractSelfTest.java | 56 ++++++++++++-------
 .../testsuites/IgnitePdsTestSuite2.java       | 16 +++++-
 9 files changed, 189 insertions(+), 25 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java
 rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/{IgniteWalFlushDefaultSelfTest.java => IgniteWalFlushFsyncSelfTest.java} (91%)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncWithDedicatedWorkerSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncWithMmapBufferSelfTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 15b9109a8f01a..cb80961215301 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -362,7 +362,11 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) {
         evt = ctx.event();
     }
 
-    /** For test purposes only. */
+    /**
+     * For test purposes only.
+     *
+     * @param ioFactory IO factory.
+     */
     public void setFileIOFactory(FileIOFactory ioFactory) {
         this.ioFactory = ioFactory;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 4a642d0193fe3..867f13f69e5c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -230,7 +230,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
     private volatile long lastTruncatedArchiveIdx = -1L;
 
     /** Factory to provide I/O interfaces for read/write operations with files */
-    private final FileIOFactory ioFactory;
+    private FileIOFactory ioFactory;
 
     /** Updater for {@link #currentHnd}, used for verify there are no concurrent update for current log segment handle */
     private static final AtomicReferenceFieldUpdater<FsyncModeFileWriteAheadLogManager, FileWriteHandle> currentHndUpd =
@@ -320,6 +320,15 @@ public FsyncModeFileWriteAheadLogManager(@NotNull final GridKernalContext ctx) {
         assert mode == WALMode.FSYNC : dsCfg;
     }
 
+    /**
+     * For test purposes only.
+     *
+     * @param ioFactory IO factory.
+     */
+    public void setFileIOFactory(FileIOFactory ioFactory) {
+        this.ioFactory = ioFactory;
+    }
+
     /** {@inheritDoc} */
     @Override public void start0() throws IgniteCheckedException {
         if (!cctx.kernalContext().clientNode()) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java
new file mode 100644
index 0000000000000..4f021cb9bd665
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+/**
+ *
+ */
+public class IgniteWalFlushBackgroundWithMmapBufferSelfTest extends IgniteWalFlushBackgroundSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean mmap() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushDefaultSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncSelfTest.java
similarity index 91%
rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushDefaultSelfTest.java
rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncSelfTest.java
index 94e7e258b298b..453064a3e407d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushDefaultSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncSelfTest.java
@@ -22,7 +22,7 @@
 /**
  *
  */
-public class IgniteWalFlushDefaultSelfTest extends IgniteWalFlushMultiNodeFailoverAbstractSelfTest {
+public class IgniteWalFlushFsyncSelfTest extends IgniteWalFlushMultiNodeFailoverAbstractSelfTest {
     /** {@inheritDoc} */
     @Override protected int gridCount() {
         return 1;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncWithDedicatedWorkerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncWithDedicatedWorkerSelfTest.java
new file mode 100644
index 0000000000000..4360fe5221599
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncWithDedicatedWorkerSelfTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import org.apache.ignite.IgniteSystemProperties;
+
+/**
+ *
+ */
+public class IgniteWalFlushFsyncWithDedicatedWorkerSelfTest extends IgniteWalFlushFsyncSelfTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER, "true");
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncWithMmapBufferSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncWithMmapBufferSelfTest.java
new file mode 100644
index 0000000000000..63872bace6ce8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFsyncWithMmapBufferSelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+/**
+ *
+ */
+public class IgniteWalFlushFsyncWithMmapBufferSelfTest extends IgniteWalFlushFsyncSelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean mmap() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java
new file mode 100644
index 0000000000000..d1f5075b56b4a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+/**
+ *
+ */
+public class IgniteWalFlushLogOnlyWithMmapBufferSelfTest extends IgniteWalFlushLogOnlySelfTest {
+    /** {@inheritDoc} */
+    @Override protected boolean mmap() {
+        return true;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index fe1632817e44c..9b5b642272ff0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -24,20 +24,20 @@
 import java.nio.MappedByteBuffer;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -72,9 +72,13 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
 
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
         stopAllGrids();
 
         cleanPersistenceDir();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_WAL_MMAP, Boolean.toString(mmap()));
     }
 
     /** {@inheritDoc} */
@@ -82,6 +86,10 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
         stopAllGrids();
 
         cleanPersistenceDir();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_WAL_MMAP);
+
+        super.afterTest();
     }
 
     /** {@inheritDoc} */
@@ -89,9 +97,18 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
         return 60_000;
     }
 
-    /** {@inheritDoc} */
+    /**
+     * @return WAL mode used in test.
+     */
     protected abstract WALMode walMode();
 
+    /**
+     * @return {@code True} if test should use MMAP buffer mode.
+     */
+    protected boolean mmap() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -138,14 +155,7 @@ public void testFailAfterStart() throws Exception {
     public void failWhilePut(boolean failWhileStart) throws Exception {
         final Ignite grid = startGridsMultiThreaded(gridCount());
 
-        IgniteWriteAheadLogManager wal = ((IgniteKernal)grid).context().cache().context().wal();
-
-        boolean mmap = GridTestUtils.getFieldValue(wal, "mmap");
-
-        if (mmap)
-            return;
-
-        grid.active(true);
+        grid.cluster().active(true);
 
         IgniteCache<Object, Object> cache = grid.cache(TEST_CACHE);
 
@@ -170,9 +180,7 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
 
                     startGrid(gridCount());
 
-                    FileWriteAheadLogManager wal0 = (FileWriteAheadLogManager)grid(gridCount()).context().cache().context().wal();
-
-                    wal0.setFileIOFactory(new FailingFileIOFactory(canFail));
+                    setFileIOFactory(grid(gridCount()).context().cache().context().wal());
 
                     grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
 
@@ -186,7 +194,6 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
                 canFail.set(true);
         }
 
-
         // We should await successful stop of node.
         GridTestUtils.waitForCondition(new GridAbsPredicate() {
             @Override public boolean apply() {
@@ -200,11 +207,9 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
 
         Ignite grid0 = startGrids(gridCount() + 1);
 
-        FileWriteAheadLogManager wal0 = (FileWriteAheadLogManager)grid(gridCount()).context().cache().context().wal();
+        setFileIOFactory(grid(gridCount()).context().cache().context().wal());
 
-        wal0.setFileIOFactory(new FailingFileIOFactory(canFail));
-
-        grid0.active(true);
+        grid0.cluster().active(true);
 
         cache = grid0.cache(TEST_CACHE);
 
@@ -212,6 +217,16 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
             assertEquals(cache.get(i), "testValue" + i);
     }
 
+    /** */
+    private void setFileIOFactory(IgniteWriteAheadLogManager wal) {
+        if (wal instanceof FileWriteAheadLogManager)
+            ((FileWriteAheadLogManager)wal).setFileIOFactory(new FailingFileIOFactory(canFail));
+        else if (wal instanceof FsyncModeFileWriteAheadLogManager)
+            ((FsyncModeFileWriteAheadLogManager)wal).setFileIOFactory(new FailingFileIOFactory(canFail));
+        else
+            fail(wal.getClass().toString());
+    }
+
     /**
      * Create File I/O which fails after second attempt to write to File
      */
@@ -220,7 +235,7 @@ private static class FailingFileIOFactory implements FileIOFactory {
         private static final long serialVersionUID = 0L;
 
         /** */
-        private AtomicBoolean fail;
+        private final AtomicBoolean fail;
 
         /** */
         private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
@@ -242,6 +257,7 @@ private static class FailingFileIOFactory implements FileIOFactory {
             return new FileIODecorator(delegate) {
                 int writeAttempts = 2;
 
+                /** {@inheritDoc} */
                 @Override public int write(ByteBuffer srcBuf) throws IOException {
                     if (--writeAttempts <= 0 && fail != null && fail.get())
                         throw new IOException("No space left on device");
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 8f99c1d148a60..7c82f24110af4 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -42,9 +42,13 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsUnusedWalSegmentsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.filename.IgniteUidAsConsistentIdMigrationTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundSelfTest;
-import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushDefaultSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundWithMmapBufferSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFailoverTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncWithDedicatedWorkerSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncWithMmapBufferSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlySelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlyWithMmapBufferSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorSwitchSegmentTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalSerializerVersionTest;
@@ -127,12 +131,20 @@ public static void addRealPageStoreTests(TestSuite suite) {
         // Failover test
         suite.addTestSuite(IgniteWalFlushFailoverTest.class);
 
-        suite.addTestSuite(IgniteWalFlushDefaultSelfTest.class);
+        suite.addTestSuite(IgniteWalFlushFsyncSelfTest.class);
+
+        suite.addTestSuite(IgniteWalFlushFsyncWithDedicatedWorkerSelfTest.class);
+
+        suite.addTestSuite(IgniteWalFlushFsyncWithMmapBufferSelfTest.class);
 
         suite.addTestSuite(IgniteWalFlushBackgroundSelfTest.class);
 
+        suite.addTestSuite(IgniteWalFlushBackgroundWithMmapBufferSelfTest.class);
+
         suite.addTestSuite(IgniteWalFlushLogOnlySelfTest.class);
 
+        suite.addTestSuite(IgniteWalFlushLogOnlyWithMmapBufferSelfTest.class);
+
         // Test suite uses Standalone WAL iterator to verify PDS content.
         suite.addTestSuite(IgniteWalReaderTest.class);
 

From 3a1645135cf474f5760319a3c2dd7712041cc6f4 Mon Sep 17 00:00:00 2001
From: dpavlov <dpavlov@apache.org>
Date: Fri, 25 May 2018 14:48:04 +0300
Subject: [PATCH 194/543] IGNITE-7809 Corrected failure handler to avoid suite
 timeout

(cherry picked from commit 0c3a7a6)
---
 .../IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index 9b5b642272ff0..c54ad884f40a1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -20,8 +20,9 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-
 import java.nio.MappedByteBuffer;
+import java.nio.file.OpenOption;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteSystemProperties;
@@ -31,6 +32,7 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
@@ -45,9 +47,6 @@
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
 
-import java.nio.file.OpenOption;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 import static java.nio.file.StandardOpenOption.CREATE;
 import static java.nio.file.StandardOpenOption.READ;
 import static java.nio.file.StandardOpenOption.WRITE;
@@ -128,6 +127,8 @@ protected boolean mmap() {
 
         cfg.setDataStorageConfiguration(memCfg);
 
+        cfg.setFailureHandler(new StopNodeFailureHandler());
+
         return cfg;
     }
 

From 3c44891906171d0668fee02bc5c9e9b2d4c7b432 Mon Sep 17 00:00:00 2001
From: dpavlov <dpavlov@apache.org>
Date: Wed, 6 Jun 2018 21:09:25 +0300
Subject: [PATCH 195/543] IGNITE-8727: Disabling failed tests

(cherry picked from commit 095f564)
---
 ...teWalFlushBackgroundWithMmapBufferSelfTest.java | 14 ++++++++++++++
 ...gniteWalFlushLogOnlyWithMmapBufferSelfTest.java | 14 ++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java
index 4f021cb9bd665..15d2b7a7258a5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java
@@ -25,4 +25,18 @@ public class IgniteWalFlushBackgroundWithMmapBufferSelfTest extends IgniteWalFlu
     @Override protected boolean mmap() {
         return true;
     }
+
+    /** {@inheritDoc} */
+    @Override public void testFailWhileStart() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8727");
+
+        super.testFailWhileStart();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void testFailAfterStart() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8727");
+
+        super.testFailAfterStart();
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java
index d1f5075b56b4a..ce06b7dc35061 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java
@@ -25,4 +25,18 @@ public class IgniteWalFlushLogOnlyWithMmapBufferSelfTest extends IgniteWalFlushL
     @Override protected boolean mmap() {
         return true;
     }
+
+    /** {@inheritDoc} */
+    @Override public void testFailWhileStart() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8727");
+
+        super.testFailWhileStart();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void testFailAfterStart() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8727");
+
+        super.testFailAfterStart();
+    }
 }

From db9ee3e14f02ccab4954c539cb40e1f8faf8b34b Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Fri, 15 Jun 2018 17:49:12 +0300
Subject: [PATCH 196/543] IGNITE-8727 Fixed WalFlush with MMap tests.

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit 41954cc)
---
 .../file/FilePageStoreManager.java            |  1 -
 .../file/FileVersionCheckingFactory.java      |  7 ++++--
 .../wal/FileWriteAheadLogManager.java         |  8 +++----
 .../FsyncModeFileWriteAheadLogManager.java    | 17 ++++++--------
 ...FlushBackgroundWithMmapBufferSelfTest.java | 14 -----------
 ...WalFlushLogOnlyWithMmapBufferSelfTest.java | 14 -----------
 ...lushMultiNodeFailoverAbstractSelfTest.java | 23 +++++++++++++------
 .../testsuites/IgnitePdsTestSuite2.java       | 14 ++++++-----
 8 files changed, 40 insertions(+), 58 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index e72a0f644bdf1..3e882595a0454 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -956,5 +956,4 @@ public CacheStoreHolder(FilePageStore idxStore, FilePageStore[] partStores) {
             this.partStores = partStores;
         }
     }
-
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
index c7aaf1bb401fd..ab36d7cbd5201 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
@@ -52,8 +52,11 @@ public class FileVersionCheckingFactory implements FilePageStoreFactory {
      * @param fileIOFactoryStoreV1 File IO factory for V1 page store and for version checking.
      * @param memCfg Memory configuration.
      */
-    public FileVersionCheckingFactory(FileIOFactory fileIOFactory, FileIOFactory fileIOFactoryStoreV1,
-        DataStorageConfiguration memCfg) {
+    public FileVersionCheckingFactory(
+        FileIOFactory fileIOFactory,
+        FileIOFactory fileIOFactoryStoreV1,
+        DataStorageConfiguration memCfg
+    ) {
         this.fileIOFactory = fileIOFactory;
         this.fileIOFactoryStoreV1 = fileIOFactoryStoreV1;
         this.memCfg = memCfg;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index cb80961215301..59fcf4e05a234 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -277,7 +277,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     private volatile long lastTruncatedArchiveIdx = -1L;
 
     /** Factory to provide I/O interfaces for read/write operations with files */
-    private FileIOFactory ioFactory;
+    private volatile FileIOFactory ioFactory;
 
     /** Next WAL segment archived monitor. Manages last archived index, emulates archivation in no-archiver mode. */
     private final SegmentArchivedMonitor archivedMonitor = new SegmentArchivedMonitor();
@@ -3291,13 +3291,13 @@ else if (pos == FILE_FORCE)
                         }
                     }
                 }
-
-                unparkWaiters(Long.MAX_VALUE);
             }
             catch (Throwable t) {
                 err = t;
             }
             finally {
+                unparkWaiters(Long.MAX_VALUE);
+
                 if (err == null && !shutdown)
                     err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
 
@@ -3458,7 +3458,7 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, Igni
                 assert hdl.written == hdl.fileIO.position();
             }
             catch (IOException e) {
-                StorageException se = new StorageException("Unable to write", e);
+                StorageException se = new StorageException("Failed to write buffer.", e);
 
                 cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 867f13f69e5c1..c58ab3d22c7c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -230,7 +230,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
     private volatile long lastTruncatedArchiveIdx = -1L;
 
     /** Factory to provide I/O interfaces for read/write operations with files */
-    private FileIOFactory ioFactory;
+    private volatile FileIOFactory ioFactory;
 
     /** Updater for {@link #currentHnd}, used for verify there are no concurrent update for current log segment handle */
     private static final AtomicReferenceFieldUpdater<FsyncModeFileWriteAheadLogManager, FileWriteHandle> currentHndUpd =
@@ -266,9 +266,6 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
     /** Current log segment handle */
     private volatile FileWriteHandle currentHnd;
 
-    /** Environment failure. */
-    private volatile Throwable envFailed;
-
     /**
      * Positive (non-0) value indicates WAL can be archived even if not complete<br>
      * See {@link DataStorageConfiguration#setWalAutoArchiveAfterInactivity(long)}<br>
@@ -2412,9 +2409,9 @@ private FileWriteHandle(
          * Write serializer version to current handle.
          * NOTE: Method mutates {@code fileIO} position, written and lastFsyncPos fields.
          *
-         * @throws IgniteCheckedException If fail to write serializer version.
+         * @throws IOException If fail to write serializer version.
          */
-        public void writeSerializerVersion() throws IgniteCheckedException {
+        public void writeSerializerVersion() throws IOException {
             try {
                 assert fileIO.position() == 0 : "Serializer version can be written only at the begin of file " +
                     fileIO.position();
@@ -2427,7 +2424,7 @@ public void writeSerializerVersion() throws IgniteCheckedException {
                 head.set(new FakeRecord(new FileWALPointer(idx, (int)updatedPosition, 0), false));
             }
             catch (IOException e) {
-                throw new IgniteCheckedException("Unable to write serializer version for segment " + idx, e);
+                throw new IOException("Unable to write serializer version for segment " + idx, e);
             }
         }
 
@@ -2542,7 +2539,7 @@ else if (stop) {
             lock.lock();
 
             try {
-                while (written < expWritten && envFailed == null)
+                while (written < expWritten && !cctx.kernalContext().invalid())
                     U.awaitQuiet(writeComplete);
             }
             finally {
@@ -2858,7 +2855,7 @@ private void signalNextAvailable() {
             try {
                 WALRecord rec = head.get();
 
-                if (envFailed == null) {
+                if (!cctx.kernalContext().invalid()) {
                     assert rec instanceof FakeRecord : "Expected head FakeRecord, actual head "
                     + (rec != null ? rec.getClass().getSimpleName() : "null");
 
@@ -2882,7 +2879,7 @@ private void awaitNext() throws IgniteCheckedException {
             lock.lock();
 
             try {
-                while (fileIO != null)
+                while (fileIO != null && !cctx.kernalContext().invalid())
                     U.awaitQuiet(nextSegment);
             }
             finally {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java
index 15d2b7a7258a5..4f021cb9bd665 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushBackgroundWithMmapBufferSelfTest.java
@@ -25,18 +25,4 @@ public class IgniteWalFlushBackgroundWithMmapBufferSelfTest extends IgniteWalFlu
     @Override protected boolean mmap() {
         return true;
     }
-
-    /** {@inheritDoc} */
-    @Override public void testFailWhileStart() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-8727");
-
-        super.testFailWhileStart();
-    }
-
-    /** {@inheritDoc} */
-    @Override public void testFailAfterStart() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-8727");
-
-        super.testFailAfterStart();
-    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java
index ce06b7dc35061..d1f5075b56b4a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushLogOnlyWithMmapBufferSelfTest.java
@@ -25,18 +25,4 @@ public class IgniteWalFlushLogOnlyWithMmapBufferSelfTest extends IgniteWalFlushL
     @Override protected boolean mmap() {
         return true;
     }
-
-    /** {@inheritDoc} */
-    @Override public void testFailWhileStart() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-8727");
-
-        super.testFailWhileStart();
-    }
-
-    /** {@inheritDoc} */
-    @Override public void testFailAfterStart() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-8727");
-
-        super.testFailAfterStart();
-    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index c54ad884f40a1..d585a82e66a2f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -27,6 +27,8 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -59,7 +61,7 @@ public abstract class IgniteWalFlushMultiNodeFailoverAbstractSelfTest extends Gr
     private static final String TEST_CACHE = "testCache";
 
     /** */
-    private static final int ITRS = 1000;
+    private static final int ITRS = 2000;
 
     /** */
     private AtomicBoolean canFail = new AtomicBoolean();
@@ -112,9 +114,13 @@ protected boolean mmap() {
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
+        cfg.setConsistentId(gridName);
+
         CacheConfiguration cacheCfg = new CacheConfiguration(TEST_CACHE)
                 .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
-                .setBackups(1);
+                .setBackups(1)
+                .setRebalanceMode(CacheRebalanceMode.SYNC)
+                .setAffinity(new RendezvousAffinityFunction(false, 32));
 
         cfg.setCacheConfiguration(cacheCfg);
 
@@ -169,7 +175,8 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
                     tx.commit();
 
                     break;
-                } catch (Exception expected) {
+                }
+                catch (Exception expected) {
                     // Expected exception.
                 }
             }
@@ -186,7 +193,8 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
                     grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
 
                     waitForRebalancing();
-                } catch (Exception expected) {
+                }
+                catch (Throwable expected) {
                     // There can be any exception. Do nothing.
                 }
             }
@@ -256,11 +264,9 @@ private static class FailingFileIOFactory implements FileIOFactory {
             final FileIO delegate = delegateFactory.create(file, modes);
 
             return new FileIODecorator(delegate) {
-                int writeAttempts = 2;
-
                 /** {@inheritDoc} */
                 @Override public int write(ByteBuffer srcBuf) throws IOException {
-                    if (--writeAttempts <= 0 && fail != null && fail.get())
+                    if (fail != null && fail.get())
                         throw new IOException("No space left on device");
 
                     return super.write(srcBuf);
@@ -268,6 +274,9 @@ private static class FailingFileIOFactory implements FileIOFactory {
 
                 /** {@inheritDoc} */
                 @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
+                    if (fail != null && fail.get())
+                        throw new IOException("No space left on deive");
+
                     return delegate.map(sizeBytes);
                 }
             };
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 7c82f24110af4..c5a381eb6e476 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -103,6 +103,14 @@ private static void addRealPageStoreTestsLongRunning(TestSuite suite) {
         suite.addTestSuite(IgnitePdsRecoveryAfterFileCorruptionTest.class);
 
         suite.addTestSuite(IgnitePdsPartitionFilesDestroyTest.class);
+
+        suite.addTestSuite(LocalWalModeChangeDuringRebalancingSelfTest.class);
+
+        suite.addTestSuite(IgniteWalFlushFsyncSelfTest.class);
+
+        suite.addTestSuite(IgniteWalFlushFsyncWithDedicatedWorkerSelfTest.class);
+
+        suite.addTestSuite(IgniteWalFlushFsyncWithMmapBufferSelfTest.class);
     }
 
     /**
@@ -131,12 +139,6 @@ public static void addRealPageStoreTests(TestSuite suite) {
         // Failover test
         suite.addTestSuite(IgniteWalFlushFailoverTest.class);
 
-        suite.addTestSuite(IgniteWalFlushFsyncSelfTest.class);
-
-        suite.addTestSuite(IgniteWalFlushFsyncWithDedicatedWorkerSelfTest.class);
-
-        suite.addTestSuite(IgniteWalFlushFsyncWithMmapBufferSelfTest.class);
-
         suite.addTestSuite(IgniteWalFlushBackgroundSelfTest.class);
 
         suite.addTestSuite(IgniteWalFlushBackgroundWithMmapBufferSelfTest.class);

From ca43e1b61151f6bd2b3d1b08b35eff67a105e3b5 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Wed, 6 Jun 2018 17:11:12 +0300
Subject: [PATCH 197/543] GG-13865 filling wal segments with non-zero values

---
 .../wal/FileWriteAheadLogManager.java          | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 59fcf4e05a234..141f4f7d2e065 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -165,6 +165,24 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     /** */
     private static final byte[] FILL_BUF = new byte[1024 * 1024];
 
+    static {
+        for (int i = 0; i < 32; i++)
+            FILL_BUF[i] = 0;
+
+        for (int i = 32; i < FILL_BUF.length; i++) {
+            if (i % 8 == 0 || i % 8 == 3)
+                FILL_BUF[i] = 0xD;
+            else if (i % 8 == 1 || i % 8 == 5 || i % 8 == 6)
+                FILL_BUF[i] = 0xE;
+            else if (i % 8 == 2)
+                FILL_BUF[i] = 0xA;
+            else if (i % 8 == 4)
+                FILL_BUF[i] = 0xB;
+            else if (i % 8 == 7)
+                FILL_BUF[i] = 0xF;
+        }
+    }
+
     /** Pattern for segment file names */
     private static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal");
 

From 039833c86dcd44de57278344b567bc277647ffdd Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Thu, 7 Jun 2018 12:51:57 +0300
Subject: [PATCH 198/543] gg-13865 JVM crash if there is no space for wal
 storage

---
 .../processors/cache/persistence/file/AsyncFileIO.java       | 4 +++-
 .../internal/processors/cache/persistence/file/FileIO.java   | 2 +-
 .../processors/cache/persistence/file/FileIODecorator.java   | 4 ++--
 .../cache/persistence/file/RandomAccessFileIO.java           | 4 ++--
 .../processors/cache/persistence/file/UnzipFileIO.java       | 2 +-
 .../cache/persistence/wal/FileWriteAheadLogManager.java      | 3 ++-
 .../LocalWalModeChangeDuringRebalancingSelfTest.java         | 4 ++--
 .../db/file/IgnitePdsDiskErrorsRecoveringTest.java           | 5 +++--
 .../persistence/pagemem/PagesWriteThrottleSmokeTest.java     | 4 ++--
 .../cache/persistence/file/AlignedBuffersDirectFileIO.java   | 4 ++--
 10 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
index 799a78cb99e60..3e75228618912 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
@@ -154,7 +154,7 @@ public AsyncFileIO(File file, ThreadLocal<ChannelOpFuture> holder, OpenOption...
     }
 
     /** {@inheritDoc} */
-    @Override public void write(byte[] buf, int off, int len) throws IOException {
+    @Override public int write(byte[] buf, int off, int len) throws IOException {
         ChannelOpFuture fut = holder.get();
         fut.reset();
 
@@ -166,6 +166,8 @@ public AsyncFileIO(File file, ThreadLocal<ChannelOpFuture> holder, OpenOption...
         catch (IgniteCheckedException e) {
             throw new IOException(e);
         }
+
+        return 0;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
index 822bd66413dc3..5eae54355f244 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
@@ -122,7 +122,7 @@ public interface FileIO extends AutoCloseable {
      *
      * @throws IOException If some I/O error occurs.
      */
-    public void write(byte[] buf, int off, int len) throws IOException;
+    public int write(byte[] buf, int off, int len) throws IOException;
 
     /**
      * Allocates memory mapped buffer for this file with given size.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java
index 683845bc268ba..9c389851ef248 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java
@@ -72,8 +72,8 @@ public FileIODecorator(FileIO delegate) {
     }
 
     /** {@inheritDoc} */
-    @Override public void write(byte[] buf, int off, int len) throws IOException {
-        delegate.write(buf, off, len);
+    @Override public int write(byte[] buf, int off, int len) throws IOException {
+        return delegate.write(buf, off, len);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
index 8f7454dcae617..018ed276de7ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
@@ -79,8 +79,8 @@ public RandomAccessFileIO(File file, OpenOption... modes) throws IOException {
     }
 
     /** {@inheritDoc} */
-    @Override public void write(byte[] buf, int off, int len) throws IOException {
-        ch.write(ByteBuffer.wrap(buf, off, len));
+    @Override public int write(byte[] buf, int off, int len) throws IOException {
+        return ch.write(ByteBuffer.wrap(buf, off, len));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
index 469cf3eb7ebbc..8194ba36eed14 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
@@ -110,7 +110,7 @@ public UnzipFileIO(File zip) throws IOException {
     }
 
     /** {@inheritDoc} */
-    @Override public void write(byte[] buf, int off, int len) throws IOException {
+    @Override public int write(byte[] buf, int off, int len) throws IOException {
         throw new UnsupportedOperationException();
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 141f4f7d2e065..bab1057eda2c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1404,7 +1404,8 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                 while (left > 0) {
                     int toWrite = Math.min(FILL_BUF.length, left);
 
-                    fileIO.write(FILL_BUF, 0, toWrite);
+                    if (fileIO.write(FILL_BUF, 0, toWrite) < toWrite)
+                        throw new IgniteCheckedException("Can't extend file: " + file.getName());
 
                     left -= toWrite;
                 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
index f365448d40f75..41a18ec2e8c59 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -590,7 +590,7 @@ private static class TestFileIO implements FileIO {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(byte[] buf, int off, int len) throws IOException {
+        @Override public int write(byte[] buf, int off, int len) throws IOException {
             CountDownLatch latch = fileIOLatch.get();
 
             if (latch != null && Thread.currentThread().getName().contains("checkpoint"))
@@ -601,7 +601,7 @@ private static class TestFileIO implements FileIO {
                     throw new IgniteException(ex);
                 }
 
-            delegate.write(buf, off, len);
+            return delegate.write(buf, off, len);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
index c902879cbaae2..6e9ed1fe9318b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
@@ -429,11 +429,12 @@ public LimitedSizeFileIO(FileIO delegate, AtomicLong availableSpaceBytes) {
         }
 
         /** {@inheritDoc} */
-        @Override public void write(byte[] buf, int off, int len) throws IOException {
-            super.write(buf, off, len);
+        @Override public int write(byte[] buf, int off, int len) throws IOException {
+            final int num = super.write(buf, off, len);
             availableSpaceBytes.addAndGet(-len);
             if (availableSpaceBytes.get() < 0)
                 throw new IOException("Not enough space!");
+            return num;
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
index 249718b02ec06..48ff980d37e46 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
@@ -311,11 +311,11 @@ private static class SlowCheckpointFileIOFactory implements FileIOFactory {
                     return delegate.write(srcBuf, position);
                 }
 
-                @Override public void write(byte[] buf, int off, int len) throws IOException {
+                @Override public int write(byte[] buf, int off, int len) throws IOException {
                     if (slowCheckpointEnabled.get() && Thread.currentThread().getName().contains("checkpoint"))
                         LockSupport.parkNanos(5_000_000);
 
-                    delegate.write(buf, off, len);
+                    return delegate.write(buf, off, len);
                 }
 
                 /** {@inheritDoc} */
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
index 681426cd4c18e..88d77e017f4c0 100644
--- a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
@@ -455,8 +455,8 @@ private static String getLastError() {
     }
 
     /** {@inheritDoc} */
-    @Override public void write(byte[] buf, int off, int len) throws IOException {
-        write(ByteBuffer.wrap(buf, off, len));
+    @Override public int write(byte[] buf, int off, int len) throws IOException {
+        return write(ByteBuffer.wrap(buf, off, len));
     }
 
     /** {@inheritDoc} */

From 96a1e230c40e15da4f3eb6bd7919902dbda55531 Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Thu, 7 Jun 2018 13:13:03 +0300
Subject: [PATCH 199/543] gg-13865 JVM crash if there is no space for wal
 storage

---
 .../internal/processors/cache/persistence/file/FileIO.java      | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
index 5eae54355f244..50568aff0f14e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
@@ -120,6 +120,8 @@ public interface FileIO extends AutoCloseable {
      * @param off Start offset in the {@code buffer}.
      * @param len Number of bytes to write.
      *
+     * @return Number of written bytes.
+     * 
      * @throws IOException If some I/O error occurs.
      */
     public int write(byte[] buf, int off, int len) throws IOException;

From dfaaf2bcf5afb639e7af57ba5cf8fab24caf36b5 Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Sat, 9 Jun 2018 15:37:51 +0300
Subject: [PATCH 200/543] Add failure handler

---
 .../wal/FileWriteAheadLogManager.java           | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index bab1057eda2c1..20889824e220d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -71,6 +71,8 @@
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.events.WalSegmentArchivedEvent;
 import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -98,6 +100,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutProcessor;
 import org.apache.ignite.internal.util.GridUnsafe;
@@ -272,6 +275,9 @@ else if (i % 8 == 7)
     /** Events service */
     private final GridEventStorageManager evt;
 
+    /** Failure processor */
+    private final FailureProcessor failureProcessor;
+
     /** */
     private IgniteConfiguration igCfg;
 
@@ -378,6 +384,7 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) {
         ioFactory = new RandomAccessFileIOFactory();
         walAutoArchiveAfterInactivity = dsCfg.getWalAutoArchiveAfterInactivity();
         evt = ctx.event();
+        failureProcessor = ctx.failure();
     }
 
     /**
@@ -1404,8 +1411,11 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                 while (left > 0) {
                     int toWrite = Math.min(FILL_BUF.length, left);
 
-                    if (fileIO.write(FILL_BUF, 0, toWrite) < toWrite)
+                    if (fileIO.write(FILL_BUF, 0, toWrite) < toWrite) {
+                        if (failureProcessor != null)
+                            failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, null), new StopNodeFailureHandler());
                         throw new IgniteCheckedException("Can't extend file: " + file.getName());
+                    }
 
                     left -= toWrite;
                 }
@@ -2152,7 +2162,10 @@ private class FileDecompressor extends Thread {
 
                         int bytesRead;
                         while ((bytesRead = zis.read(arr)) > 0)
-                            io.write(arr, 0, bytesRead);
+                            if (io.write(arr, 0, bytesRead) < bytesRead) {
+                                failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, null), new StopNodeFailureHandler());
+                                throw new IgniteCheckedException("Can't extend file: " + unzipTmp.getName());
+                            }
                     }
 
                     try {

From 3698d5534dc108f5e910a99ecaf038f7f3d809ba Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Thu, 14 Jun 2018 15:56:38 +0300
Subject: [PATCH 201/543] ignite-8748 Review fix

---
 .../wal/FileWriteAheadLogManager.java         | 24 ++++---------------
 1 file changed, 4 insertions(+), 20 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 20889824e220d..a161b7177f492 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -168,24 +168,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     /** */
     private static final byte[] FILL_BUF = new byte[1024 * 1024];
 
-    static {
-        for (int i = 0; i < 32; i++)
-            FILL_BUF[i] = 0;
-
-        for (int i = 32; i < FILL_BUF.length; i++) {
-            if (i % 8 == 0 || i % 8 == 3)
-                FILL_BUF[i] = 0xD;
-            else if (i % 8 == 1 || i % 8 == 5 || i % 8 == 6)
-                FILL_BUF[i] = 0xE;
-            else if (i % 8 == 2)
-                FILL_BUF[i] = 0xA;
-            else if (i % 8 == 4)
-                FILL_BUF[i] = 0xB;
-            else if (i % 8 == 7)
-                FILL_BUF[i] = 0xF;
-        }
-    }
-
     /** Pattern for segment file names */
     private static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal");
 
@@ -1412,9 +1394,11 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                     int toWrite = Math.min(FILL_BUF.length, left);
 
                     if (fileIO.write(FILL_BUF, 0, toWrite) < toWrite) {
+                        final IgniteCheckedException ex = new IgniteCheckedException("Failed to extend WAL segment file: " +
+                            file.getName() + ". Probably disk is too busy, please check your device.");
                         if (failureProcessor != null)
-                            failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, null), new StopNodeFailureHandler());
-                        throw new IgniteCheckedException("Can't extend file: " + file.getName());
+                            failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex), new StopNodeFailureHandler());
+                        throw ex;
                     }
 
                     left -= toWrite;

From 0c040504cf526517e4e101def1b83d73c51d0b8a Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Thu, 14 Jun 2018 17:34:37 +0300
Subject: [PATCH 202/543] ignite-8748 Review fix

---
 .../cache/persistence/wal/FileWriteAheadLogManager.java    | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index a161b7177f492..c23c9493393a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -2147,8 +2147,11 @@ private class FileDecompressor extends Thread {
                         int bytesRead;
                         while ((bytesRead = zis.read(arr)) > 0)
                             if (io.write(arr, 0, bytesRead) < bytesRead) {
-                                failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, null), new StopNodeFailureHandler());
-                                throw new IgniteCheckedException("Can't extend file: " + unzipTmp.getName());
+                                final IgniteCheckedException ex = new IgniteCheckedException("Failed to extend file: " +
+                                    unzipTmp.getName() + ". Probably disk is too busy, please check your device.");
+                                if (failureProcessor != null)
+                                    failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex), new StopNodeFailureHandler());
+                                throw ex;
                             }
                     }
 

From d1272a6338b12f7357c899b574afef60d0ff059b Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Thu, 14 Jun 2018 18:03:24 +0300
Subject: [PATCH 203/543] ignite-8748 Code style fix

---
 .../cache/persistence/wal/FileWriteAheadLogManager.java       | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index c23c9493393a9..99ef6b961b71b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1396,8 +1396,10 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                     if (fileIO.write(FILL_BUF, 0, toWrite) < toWrite) {
                         final IgniteCheckedException ex = new IgniteCheckedException("Failed to extend WAL segment file: " +
                             file.getName() + ". Probably disk is too busy, please check your device.");
+
                         if (failureProcessor != null)
                             failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex), new StopNodeFailureHandler());
+
                         throw ex;
                     }
 
@@ -2149,8 +2151,10 @@ private class FileDecompressor extends Thread {
                             if (io.write(arr, 0, bytesRead) < bytesRead) {
                                 final IgniteCheckedException ex = new IgniteCheckedException("Failed to extend file: " +
                                     unzipTmp.getName() + ". Probably disk is too busy, please check your device.");
+
                                 if (failureProcessor != null)
                                     failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex), new StopNodeFailureHandler());
+
                                 throw ex;
                             }
                     }

From 03c17110954b70406ce7f1a1bd9120fecd125488 Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Thu, 14 Jun 2018 18:14:17 +0300
Subject: [PATCH 204/543] ignite-8748 Use configured failure handler

---
 .../cache/persistence/wal/FileWriteAheadLogManager.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 99ef6b961b71b..2a77c3311e9e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1398,7 +1398,7 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                             file.getName() + ". Probably disk is too busy, please check your device.");
 
                         if (failureProcessor != null)
-                            failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex), new StopNodeFailureHandler());
+                            failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
 
                         throw ex;
                     }
@@ -2153,7 +2153,7 @@ private class FileDecompressor extends Thread {
                                     unzipTmp.getName() + ". Probably disk is too busy, please check your device.");
 
                                 if (failureProcessor != null)
-                                    failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex), new StopNodeFailureHandler());
+                                    failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
 
                                 throw ex;
                             }

From 9e91b79c329e4f570dffb9062cc668bf21b4123a Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Fri, 15 Jun 2018 21:05:11 +0300
Subject: [PATCH 205/543] ignite-8740 Return

---
 .../processors/cache/persistence/file/AsyncFileIO.java        | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
index 3e75228618912..76142bb509626 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
@@ -161,13 +161,11 @@ public AsyncFileIO(File file, ThreadLocal<ChannelOpFuture> holder, OpenOption...
         ch.write(ByteBuffer.wrap(buf, off, len), position, this, fut);
 
         try {
-            fut.getUninterruptibly();
+            return fut.getUninterruptibly();
         }
         catch (IgniteCheckedException e) {
             throw new IOException(e);
         }
-
-        return 0;
     }
 
     /** {@inheritDoc} */

From 5674e1ae9eb0cb4d88a8a91696afba9d23c320ca Mon Sep 17 00:00:00 2001
From: Andrey Kuznetsov <stkuzma@gmail.com>
Date: Mon, 18 Jun 2018 12:47:25 +0300
Subject: [PATCH 206/543] IGNITE-8562 Turn system-critical Ignite threads into
 GridWorkers

Signed-off-by: agura <agura@apache.org>
---
 .../internal/GridKernalContextImpl.java       |   7 +-
 .../apache/ignite/internal/IgniteKernal.java  |   7 +-
 .../apache/ignite/internal/IgnitionEx.java    |  21 +-
 .../GridCacheDatabaseSharedManager.java       |   2 +-
 .../wal/FileWriteAheadLogManager.java         |  83 +++--
 .../FsyncModeFileWriteAheadLogManager.java    |  51 ++-
 .../ignite/internal/util/IgniteUtils.java     |   5 +-
 .../ignite/internal/util/StripedExecutor.java | 164 +++++----
 .../internal/util/nio/GridNioServer.java      |  90 +++--
 .../tcp/TcpCommunicationSpi.java              |  98 +++---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  87 +++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 310 ++++++++++++------
 .../IgniteDiagnosticMessagesTest.java         |   2 +
 .../internal/util/StripedExecutorTest.java    |   8 +-
 .../discovery/tcp/TcpDiscoverySelfTest.java   |   7 +-
 .../junits/GridTestKernalContext.java         |   1 +
 16 files changed, 594 insertions(+), 349 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index ac4970859d2b9..2f4ecbc7ddfab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -363,7 +363,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
 
     /** */
     @GridToStringExclude
-    private final WorkersRegistry workersRegistry = new WorkersRegistry();
+    private WorkersRegistry workersRegistry;
 
     /** */
     private IgniteEx grid;
@@ -431,6 +431,7 @@ public GridKernalContextImpl() {
      * @param schemaExecSvc Schema executor service.
      * @param customExecSvcs Custom named executors.
      * @param plugins Plugin providers.
+     * @param workerRegistry Worker registry.
      */
     @SuppressWarnings("TypeMayBeWeakened")
     protected GridKernalContextImpl(
@@ -455,7 +456,8 @@ protected GridKernalContextImpl(
         ExecutorService schemaExecSvc,
         @Nullable Map<String, ? extends ExecutorService> customExecSvcs,
         List<PluginProvider> plugins,
-        IgnitePredicate<String> clsFilter
+        IgnitePredicate<String> clsFilter,
+        WorkersRegistry workerRegistry
     ) {
         assert grid != null;
         assert cfg != null;
@@ -480,6 +482,7 @@ protected GridKernalContextImpl(
         this.qryExecSvc = qryExecSvc;
         this.schemaExecSvc = schemaExecSvc;
         this.customExecSvcs = customExecSvcs;
+        this.workersRegistry = workerRegistry;
 
         marshCtx = new MarshallerContextImpl(plugins, clsFilter);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index d24a19e078a6f..2e860a0de1c85 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -764,6 +764,7 @@ private void notifyLifecycleBeansEx(LifecycleEventType evt) {
      * @param schemaExecSvc Schema executor service.
      * @param customExecSvcs Custom named executors.
      * @param errHnd Error handler to use for notification about startup problems.
+     * @param workerRegistry Worker registry.
      * @throws IgniteCheckedException Thrown in case of any errors.
      */
     @SuppressWarnings({"CatchGenericClass", "unchecked"})
@@ -785,7 +786,8 @@ public void start(
         ExecutorService qryExecSvc,
         ExecutorService schemaExecSvc,
         @Nullable final Map<String, ? extends ExecutorService> customExecSvcs,
-        GridAbsClosure errHnd
+        GridAbsClosure errHnd,
+        WorkersRegistry workerRegistry
     )
         throws IgniteCheckedException {
         gw.compareAndSet(null, new GridKernalGatewayImpl(cfg.getIgniteInstanceName()));
@@ -901,7 +903,8 @@ public void start(
                 schemaExecSvc,
                 customExecSvcs,
                 plugins,
-                classNameFilter()
+                classNameFilter(),
+                workerRegistry
             );
 
             cfg.getMarshaller().setContext(ctx.marshallerContext());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index b3c3ee8b5a9b1..4d1724d5afa97 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -93,7 +93,9 @@
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.worker.WorkersRegistry;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.logger.LoggerNodeIdAware;
 import org.apache.ignite.logger.java.JavaLogger;
 import org.apache.ignite.marshaller.Marshaller;
@@ -1815,17 +1817,20 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
 
             validateThreadPoolSize(cfg.getStripedPoolSize(), "stripedPool");
 
+            WorkersRegistry workerRegistry = new WorkersRegistry();
+
             stripedExecSvc = new StripedExecutor(
                 cfg.getStripedPoolSize(),
                 cfg.getIgniteInstanceName(),
                 "sys",
                 log,
-                new Thread.UncaughtExceptionHandler() {
-                    @Override public void uncaughtException(Thread thread, Throwable t) {
+                new IgniteInClosure<Throwable>() {
+                    @Override public void apply(Throwable t) {
                         if (grid != null)
                             grid.context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, t));
                     }
-                });
+                },
+                workerRegistry);
 
             // Note that since we use 'LinkedBlockingQueue', number of
             // maximum threads has no effect.
@@ -1867,13 +1872,14 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                 cfg.getIgniteInstanceName(),
                 "data-streamer",
                 log,
-                new Thread.UncaughtExceptionHandler() {
-                    @Override public void uncaughtException(Thread thread, Throwable t) {
+                new IgniteInClosure<Throwable>() {
+                    @Override public void apply(Throwable t) {
                         if (grid != null)
                             grid.context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, t));
                     }
                 },
-                true);
+                true,
+                workerRegistry);
 
             // Note that we do not pre-start threads here as igfs pool may not be needed.
             validateThreadPoolSize(cfg.getIgfsThreadPoolSize(), "IGFS");
@@ -2033,7 +2039,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                         @Override public void apply() {
                             startLatch.countDown();
                         }
-                    }
+                    },
+                    workerRegistry
                 );
 
                 state = STARTED;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 7d30181c9bf46..4a44f23fe693a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -2928,7 +2928,7 @@ public class Checkpointer extends GridWorker {
          * @param log Logger.
          */
         protected Checkpointer(@Nullable String gridName, String name, IgniteLogger log) {
-            super(gridName, name, log);
+            super(gridName, name, log, cctx.kernalContext().workersRegistry());
 
             scheduledCp = new CheckpointProgress(U.currentTimeMillis() + checkpointFreq);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 2a77c3311e9e2..9b39987edf42a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -112,10 +112,12 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -429,7 +431,7 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
             long lastAbsArchivedIdx = tup == null ? -1 : tup.get2();
 
             if (isArchiverEnabled())
-                archiver = new FileArchiver(lastAbsArchivedIdx);
+                archiver = new FileArchiver(lastAbsArchivedIdx, log);
             else
                 archiver = null;
 
@@ -439,10 +441,10 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
             if (dsCfg.isWalCompactionEnabled()) {
                 compressor = new FileCompressor();
 
-                decompressor = new FileDecompressor();
+                decompressor = new FileDecompressor(log);
 
                 if (!cctx.kernalContext().clientNode())
-                    decompressor.start();
+                    new IgniteThread(decompressor).start();
             }
 
             if (mode != WALMode.NONE) {
@@ -572,7 +574,7 @@ private void checkWalConfiguration() throws IgniteCheckedException {
             if (isArchiverEnabled()) {
                 assert archiver != null;
 
-                archiver.start();
+                new IgniteThread(archiver).start();
             }
 
             if (compressor != null)
@@ -609,10 +611,10 @@ private void checkWalConfiguration() throws IgniteCheckedException {
 
             FileWALPointer filePtr = (FileWALPointer)lastPtr;
 
-            walWriter = new WALWriter();
+            walWriter = new WALWriter(log);
 
             if (!mmap)
-                walWriter.start();
+                new IgniteThread(walWriter).start();
 
             currHnd = restoreWriteHandle(filePtr);
 
@@ -1523,7 +1525,7 @@ public long maxWalSegmentSize() {
      * FileArchiver#lastAbsArchivedIdx})</li> <li>some WAL index was removed from {@link FileArchiver#locked} map</li>
      * </ul>
      */
-    private class FileArchiver extends Thread {
+    private class FileArchiver extends GridWorker {
         /** Exception which occurred during initial creation of files or during archiving WAL segment */
         private IgniteCheckedException cleanErr;
 
@@ -1551,8 +1553,9 @@ private class FileArchiver extends Thread {
         /**
          *
          */
-        private FileArchiver(long lastAbsArchivedIdx) {
-            super("wal-file-archiver%" + cctx.igniteInstanceName());
+        private FileArchiver(long lastAbsArchivedIdx, IgniteLogger log) {
+            super(cctx.igniteInstanceName(), "wal-file-archiver%" + cctx.igniteInstanceName(), log,
+                cctx.kernalContext().workersRegistry());
 
             this.lastAbsArchivedIdx = lastAbsArchivedIdx;
         }
@@ -1567,7 +1570,7 @@ private void shutdown() throws IgniteInterruptedCheckedException {
                 notifyAll();
             }
 
-            U.join(this);
+            U.join(runner());
         }
 
         /**
@@ -1592,7 +1595,7 @@ private synchronized boolean locked(long absIdx) {
         }
 
         /** {@inheritDoc} */
-        @Override public void run() {
+        @Override protected void body() {
             try {
                 allocateRemainingFiles();
             }
@@ -1661,7 +1664,7 @@ private synchronized boolean locked(long absIdx) {
             }
             finally {
                 if (err == null && !stopped)
-                    err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+                    err = new IllegalStateException("Worker " + name() + " is terminated unexpectedly");
 
                 if (err instanceof OutOfMemoryError)
                     cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
@@ -2107,10 +2110,7 @@ private void shutdown() throws IgniteInterruptedCheckedException {
     /**
      * Responsible for decompressing previously compressed segments of WAL archive if they are needed for replay.
      */
-    private class FileDecompressor extends Thread {
-        /** Current thread stopping advice. */
-        private volatile boolean stopped;
-
+    private class FileDecompressor extends GridWorker {
         /** Decompression futures. */
         private Map<Long, GridFutureAdapter<Void>> decompressionFutures = new HashMap<>();
 
@@ -2121,21 +2121,21 @@ private class FileDecompressor extends Thread {
         private byte[] arr = new byte[BUF_SIZE];
 
         /**
-         *
+         * @param log Logger.
          */
-        FileDecompressor() {
-            super("wal-file-decompressor%" + cctx.igniteInstanceName());
+        FileDecompressor(IgniteLogger log) {
+            super(cctx.igniteInstanceName(), "wal-file-decompressor%" + cctx.igniteInstanceName(), log);
         }
 
         /** {@inheritDoc} */
-        @Override public void run() {
-            while (!Thread.currentThread().isInterrupted() && !stopped) {
+        @Override protected void body() {
+            while (!isCancelled()) {
                 long segmentToDecompress = -1L;
 
                 try {
                     segmentToDecompress = segmentsQueue.take();
 
-                    if (stopped)
+                    if (isCancelled())
                         break;
 
                     File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".zip");
@@ -2143,7 +2143,7 @@ private class FileDecompressor extends Thread {
                     File unzip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress));
 
                     try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
-                        FileIO io = ioFactory.create(unzipTmp)) {
+                         FileIO io = ioFactory.create(unzipTmp)) {
                         zis.getNextEntry();
 
                         int bytesRead;
@@ -2178,7 +2178,7 @@ private class FileDecompressor extends Thread {
                     Thread.currentThread().interrupt();
                 }
                 catch (Throwable t) {
-                    if (!stopped && segmentToDecompress != -1L) {
+                    if (!isCancelled && segmentToDecompress != -1L) {
                         IgniteCheckedException e = new IgniteCheckedException("Error during WAL segment " +
                             "decompression [segmentIdx=" + segmentToDecompress + "]", t);
 
@@ -2218,13 +2218,13 @@ synchronized IgniteInternalFuture<Void> decompressFile(long idx) {
          */
         private void shutdown() throws IgniteInterruptedCheckedException {
             synchronized (this) {
-                stopped = true;
+                U.cancel(this);
 
                 // Put fake -1 to wake thread from queue.take()
                 segmentsQueue.put(-1L);
             }
 
-            U.join(this);
+            U.join(this, log);
         }
     }
 
@@ -3212,7 +3212,7 @@ private void doFlush() {
      * WAL writer worker.
      */
     @SuppressWarnings("ForLoopReplaceableByForEach")
-    private class WALWriter extends Thread {
+    private class WALWriter extends GridWorker {
         /** Unconditional flush. */
         private static final long UNCONDITIONAL_FLUSH = -1L;
 
@@ -3222,9 +3222,6 @@ private class WALWriter extends Thread {
         /** File force. */
         private static final long FILE_FORCE = -3L;
 
-        /** Shutdown. */
-        private volatile boolean shutdown;
-
         /** Err. */
         private volatile Throwable err;
 
@@ -3234,19 +3231,22 @@ private class WALWriter extends Thread {
 
         /**
          * Default constructor.
+         *
+         * @param log Logger.
          */
-        WALWriter() {
-            super("wal-write-worker%" + cctx.igniteInstanceName());
+        WALWriter(IgniteLogger log) {
+            super(cctx.igniteInstanceName(), "wal-write-worker%" + cctx.igniteInstanceName(), log,
+                cctx.kernalContext().workersRegistry());
         }
 
         /** {@inheritDoc} */
-        @Override public void run() {
+        @Override protected void body() {
             Throwable err = null;
 
             try {
-                while (!shutdown && !Thread.currentThread().isInterrupted()) {
+                while (!isCancelled()) {
                     while (waiters.isEmpty()) {
-                        if (!shutdown)
+                        if (!isCancelled())
                             LockSupport.park();
                         else {
                             unparkWaiters(Long.MAX_VALUE);
@@ -3321,8 +3321,8 @@ else if (pos == FILE_FORCE)
             finally {
                 unparkWaiters(Long.MAX_VALUE);
 
-                if (err == null && !shutdown)
-                    err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+                if (err == null && !isCancelled)
+                    err = new IllegalStateException("Worker " + name() + " is terminated unexpectedly");
 
                 if (err instanceof OutOfMemoryError)
                     cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
@@ -3335,11 +3335,11 @@ else if (err != null)
          * Shutdowns thread.
          */
         public void shutdown() throws IgniteInterruptedCheckedException {
-            shutdown = true;
+            U.cancel(this);
 
-            LockSupport.unpark(this);
+            LockSupport.unpark(runner());
 
-            U.join(this);
+            U.join(runner());
         }
 
         /**
@@ -3386,7 +3386,6 @@ void flushAll() throws IgniteCheckedException {
         /**
          * @param expPos Expected position.
          */
-        @SuppressWarnings("ForLoopReplaceableByForEach")
         void flushBuffer(long expPos) throws StorageException, IgniteCheckedException {
             if (mmap)
                 return;
@@ -3403,7 +3402,7 @@ void flushBuffer(long expPos) throws StorageException, IgniteCheckedException {
 
             waiters.put(t, expPos);
 
-            LockSupport.unpark(walWriter);
+            LockSupport.unpark(walWriter.runner());
 
             while (true) {
                 Long val = waiters.get(t);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index c58ab3d22c7c8..49fbc73b9a48c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -104,10 +104,12 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -373,12 +375,12 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
 
             lastTruncatedArchiveIdx = tup == null ? -1 : tup.get1() - 1;
 
-            archiver = new FileArchiver(tup == null ? -1 : tup.get2());
+            archiver = new FileArchiver(tup == null ? -1 : tup.get2(), log);
 
             if (dsCfg.isWalCompactionEnabled()) {
                 compressor = new FileCompressor();
 
-                decompressor = new FileDecompressor();
+                decompressor = new FileDecompressor(log);
             }
 
             if (mode != WALMode.NONE) {
@@ -452,13 +454,13 @@ private void checkWalConfiguration() throws IgniteCheckedException {
         if (!cctx.kernalContext().clientNode()) {
             assert archiver != null;
 
-            archiver.start();
+            new IgniteThread(archiver).start();
 
             if (compressor != null)
                 compressor.start();
 
             if (decompressor != null)
-                decompressor.start();
+                new IgniteThread(decompressor).start();
         }
     }
 
@@ -1314,7 +1316,7 @@ private void checkNode() throws StorageException {
      * <li>some WAL index was removed from {@link FileArchiver#locked} map</li>
      * </ul>
      */
-    private class FileArchiver extends Thread {
+    private class FileArchiver extends GridWorker {
         /** Exception which occurred during initial creation of files or during archiving WAL segment */
         private IgniteCheckedException cleanException;
 
@@ -1345,8 +1347,9 @@ private class FileArchiver extends Thread {
         /**
          *
          */
-        private FileArchiver(long lastAbsArchivedIdx) {
-            super("wal-file-archiver%" + cctx.igniteInstanceName());
+        private FileArchiver(long lastAbsArchivedIdx, IgniteLogger log) {
+            super(cctx.igniteInstanceName(), "wal-file-archiver%" + cctx.igniteInstanceName(), log,
+                cctx.kernalContext().workersRegistry());
 
             this.lastAbsArchivedIdx = lastAbsArchivedIdx;
         }
@@ -1368,7 +1371,7 @@ private void shutdown() throws IgniteInterruptedCheckedException {
                 notifyAll();
             }
 
-            U.join(this);
+            U.join(runner());
         }
 
         /**
@@ -1419,7 +1422,7 @@ private synchronized void release(long absIdx) {
         }
 
         /** {@inheritDoc} */
-        @Override public void run() {
+        @Override protected void body() {
             try {
                 allocateRemainingFiles();
             }
@@ -1468,7 +1471,6 @@ private synchronized void release(long absIdx) {
                         while (locked.containsKey(toArchive) && !stopped)
                             wait();
 
-                        // Then increase counter to allow rollover on clean working file
                         changeLastArchivedIndexAndWakeupCompressor(toArchive);
 
                         notifyAll();
@@ -1488,7 +1490,7 @@ private synchronized void release(long absIdx) {
             }
             finally {
                 if (err == null && !stopped)
-                    err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly");
+                    err = new IllegalStateException("Worker " + name() + " is terminated unexpectedly");
 
                 if (err instanceof OutOfMemoryError)
                     cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
@@ -1910,10 +1912,7 @@ private void shutdown() throws IgniteInterruptedCheckedException {
     /**
      * Responsible for decompressing previously compressed segments of WAL archive if they are needed for replay.
      */
-    private class FileDecompressor extends Thread {
-        /** Current thread stopping advice. */
-        private volatile boolean stopped;
-
+    private class FileDecompressor extends GridWorker {
         /** Decompression futures. */
         private Map<Long, GridFutureAdapter<Void>> decompressionFutures = new HashMap<>();
 
@@ -1924,21 +1923,21 @@ private class FileDecompressor extends Thread {
         private byte[] arr = new byte[tlbSize];
 
         /**
-         *
+         * @param log Logger.
          */
-        FileDecompressor() {
-            super("wal-file-decompressor%" + cctx.igniteInstanceName());
+        FileDecompressor(IgniteLogger log) {
+            super(cctx.igniteInstanceName(), "wal-file-decompressor%" + cctx.igniteInstanceName(), log);
         }
 
         /** {@inheritDoc} */
-        @Override public void run() {
-            while (!Thread.currentThread().isInterrupted() && !stopped) {
+        @Override protected void body() {
+            while (!isCancelled()) {
                 long segmentToDecompress = -1L;
 
                 try {
                     segmentToDecompress = segmentsQueue.take();
 
-                    if (stopped)
+                    if (isCancelled())
                         break;
 
                     File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".zip");
@@ -1946,7 +1945,7 @@ private class FileDecompressor extends Thread {
                     File unzip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress));
 
                     try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
-                        FileIO io = ioFactory.create(unzipTmp)) {
+                         FileIO io = ioFactory.create(unzipTmp)) {
                         zis.getNextEntry();
 
                         int bytesRead;
@@ -1973,7 +1972,7 @@ private class FileDecompressor extends Thread {
                     Thread.currentThread().interrupt();
                 }
                 catch (Throwable t) {
-                    if (!stopped && segmentToDecompress != -1L) {
+                    if (!isCancelled && segmentToDecompress != -1L) {
                         IgniteCheckedException e = new IgniteCheckedException("Error during WAL segment " +
                             "decompression [segmentIdx=" + segmentToDecompress + ']', t);
 
@@ -2011,15 +2010,15 @@ synchronized IgniteInternalFuture<Void> decompressFile(long idx) {
         /**
          * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
          */
-        private void shutdown() throws IgniteInterruptedCheckedException {
+        private void shutdown() {
             synchronized (this) {
-                stopped = true;
+                U.cancel(this);
 
                 // Put fake -1 to wake thread from queue.take()
                 segmentsQueue.put(-1L);
             }
 
-            U.join(this);
+            U.join(this, log);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 2defefad4721e..a76f20c21512b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -7683,7 +7683,10 @@ public static <T> T get(Future<T> fut) throws IgniteCheckedException {
      * @param t Thread.
      * @throws org.apache.ignite.internal.IgniteInterruptedCheckedException Wrapped {@link InterruptedException}.
      */
-    public static void join(Thread t) throws IgniteInterruptedCheckedException {
+    public static void join(@Nullable Thread t) throws IgniteInterruptedCheckedException {
+        if (t == null)
+            return;
+
         try {
             t.join();
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
index c6383ee41319e..904b8d1a5f48b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedExecutor.java
@@ -35,14 +35,16 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.locks.LockSupport;
-import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.internal.util.worker.GridWorkerListener;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.NotNull;
 
@@ -64,11 +66,18 @@ public class StripedExecutor implements ExecutorService {
      * @param igniteInstanceName Node name.
      * @param poolName Pool name.
      * @param log Logger.
-     * @param errHnd Exception handler.
+     * @param errHnd Critical failure handler.
+     * @param gridWorkerLsnr listener to link with every stripe worker.
      */
-    public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log,
-        Thread.UncaughtExceptionHandler errHnd) {
-        this(cnt, igniteInstanceName, poolName, log, errHnd, false);
+    public StripedExecutor(
+        int cnt,
+        String igniteInstanceName,
+        String poolName,
+        final IgniteLogger log,
+        IgniteInClosure<Throwable> errHnd,
+        GridWorkerListener gridWorkerLsnr
+    ) {
+        this(cnt, igniteInstanceName, poolName, log, errHnd, false, gridWorkerLsnr);
     }
 
     /**
@@ -76,11 +85,19 @@ public StripedExecutor(int cnt, String igniteInstanceName, String poolName, fina
      * @param igniteInstanceName Node name.
      * @param poolName Pool name.
      * @param log Logger.
-     * @param errHnd Exception handler.
+     * @param errHnd Critical failure handler.
      * @param stealTasks {@code True} to steal tasks.
+     * @param gridWorkerLsnr listener to link with every stripe worker.
      */
-    public StripedExecutor(int cnt, String igniteInstanceName, String poolName, final IgniteLogger log,
-        Thread.UncaughtExceptionHandler errHnd, boolean stealTasks) {
+    public StripedExecutor(
+        int cnt,
+        String igniteInstanceName,
+        String poolName,
+        final IgniteLogger log,
+        IgniteInClosure<Throwable> errHnd,
+        boolean stealTasks,
+        GridWorkerListener gridWorkerLsnr
+    ) {
         A.ensure(cnt > 0, "cnt > 0");
 
         boolean success = false;
@@ -96,8 +113,8 @@ public StripedExecutor(int cnt, String igniteInstanceName, String poolName, fina
         try {
             for (int i = 0; i < cnt; i++) {
                 stripes[i] = stealTasks
-                    ? new StripeConcurrentQueue(igniteInstanceName, poolName, i, log, stripes, errHnd)
-                    : new StripeConcurrentQueue(igniteInstanceName, poolName, i, log, errHnd);
+                    ? new StripeConcurrentQueue(igniteInstanceName, poolName, i, log, stripes, errHnd, gridWorkerLsnr)
+                    : new StripeConcurrentQueue(igniteInstanceName, poolName, i, log, errHnd, gridWorkerLsnr);
             }
 
             for (int i = 0; i < cnt; i++)
@@ -112,15 +129,11 @@ public StripedExecutor(int cnt, String igniteInstanceName, String poolName, fina
         }
         finally {
             if (!success) {
-                for (Stripe stripe : stripes) {
-                    if (stripe != null)
-                        stripe.signalStop();
-                }
+                for (Stripe stripe : stripes)
+                    U.cancel(stripe);
 
-                for (Stripe stripe : stripes) {
-                    if (stripe != null)
-                        stripe.awaitStop();
-                }
+                for (Stripe stripe : stripes)
+                    U.join(stripe, log);
             }
         }
     }
@@ -221,7 +234,7 @@ public void execute(int idx, Runnable cmd) {
     /** {@inheritDoc} */
     @Override public boolean isShutdown() {
         for (Stripe stripe : stripes) {
-            if (stripe != null && stripe.stopping)
+            if (stripe != null && stripe.isCancelled())
                 return true;
         }
 
@@ -252,15 +265,15 @@ public void stop() {
      */
     private void signalStop() {
         for (Stripe stripe : stripes)
-            stripe.signalStop();
+            U.cancel(stripe);
     }
 
     /**
-     * @throws IgniteInterruptedException If interrupted.
+     * Waits for all stripes to stop.
      */
-    private void awaitStop() throws IgniteInterruptedException {
+    private void awaitStop() {
         for (Stripe stripe : stripes)
-            stripe.awaitStop();
+            U.join(stripe, log);
     }
 
     /**
@@ -407,22 +420,16 @@ public int[] stripesQueueSizes() {
     /**
      * Stripe.
      */
-    private static abstract class Stripe implements Runnable {
+    private static abstract class Stripe extends GridWorker {
         /** */
         private final String igniteInstanceName;
 
-        /** */
-        private final String poolName;
-
         /** */
         protected final int idx;
 
         /** */
         private final IgniteLogger log;
 
-        /** Stopping flag. */
-        private volatile boolean stopping;
-
         /** */
         private volatile long completedCnt;
 
@@ -432,8 +439,8 @@ private static abstract class Stripe implements Runnable {
         /** Thread executing the loop. */
         protected Thread thread;
 
-        /** Exception handler. */
-        private Thread.UncaughtExceptionHandler errHnd;
+        /** Critical failure handler. */
+        private IgniteInClosure<Throwable> errHnd;
 
         /**
          * @param igniteInstanceName Ignite instance name.
@@ -441,16 +448,19 @@ private static abstract class Stripe implements Runnable {
          * @param idx Stripe index.
          * @param log Logger.
          * @param errHnd Exception handler.
+         * @param gridWorkerLsnr listener to link with stripe worker.
          */
         public Stripe(
             String igniteInstanceName,
             String poolName,
             int idx,
             IgniteLogger log,
-            Thread.UncaughtExceptionHandler errHnd
+            IgniteInClosure<Throwable> errHnd,
+            GridWorkerListener gridWorkerLsnr
         ) {
+            super(igniteInstanceName, poolName + "-stripe-" + idx, log, gridWorkerLsnr);
+
             this.igniteInstanceName = igniteInstanceName;
-            this.poolName = poolName;
             this.idx = idx;
             this.log = log;
             this.errHnd = errHnd;
@@ -461,44 +471,19 @@ public Stripe(
          */
         void start() {
             thread = new IgniteThread(igniteInstanceName,
-                poolName + "-stripe-" + idx,
+                name(),
                 this,
                 IgniteThread.GRP_IDX_UNASSIGNED,
                 idx,
                 GridIoPolicy.UNDEFINED);
 
-            thread.setUncaughtExceptionHandler(errHnd);
-
             thread.start();
         }
 
-        /**
-         * Stop the stripe.
-         */
-        void signalStop() {
-            stopping = true;
-
-            U.interrupt(thread);
-        }
-
-        /**
-         * Await thread stop.
-         */
-        void awaitStop() {
-            try {
-                if (thread != null)
-                    thread.join();
-            }
-            catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-
-                throw new IgniteInterruptedException(e);
-            }
-        }
-
         /** {@inheritDoc} */
-        @Override public void run() {
-            while (!stopping) {
+        @SuppressWarnings("NonAtomicOperationOnVolatileField")
+        @Override public void body() {
+            while (!isCancelled()) {
                 Runnable cmd;
 
                 try {
@@ -517,25 +502,21 @@ void awaitStop() {
                     }
                 }
                 catch (InterruptedException ignored) {
-                    stopping = true;
-
                     Thread.currentThread().interrupt();
 
-                    return;
+                    break;
                 }
                 catch (Throwable e) {
-                    if (e instanceof OutOfMemoryError) {
-                        // Re-throwing to exploit uncaught exception handler.
-                        throw e;
-                    }
+                    if (e instanceof OutOfMemoryError)
+                        errHnd.apply(e);
 
                     U.error(log, "Failed to execute runnable.", e);
                 }
             }
 
-            if (!stopping) {
-                throw new IllegalStateException("Thread " + Thread.currentThread().getName() +
-                    " is terminated unexpectedly");
+            if (!isCancelled) {
+                errHnd.apply(new IllegalStateException("Thread " + Thread.currentThread().getName() +
+                    " is terminated unexpectedly"));
             }
         }
 
@@ -592,16 +573,18 @@ private static class StripeConcurrentQueue extends Stripe {
          * @param poolName Pool name.
          * @param idx Stripe index.
          * @param log Logger.
-         * @param errHnd Exception handler.
+         * @param errHnd Critical failure handler.
+         * @param gridWorkerLsnr listener to link with stripe worker.
          */
         StripeConcurrentQueue(
             String igniteInstanceName,
             String poolName,
             int idx,
             IgniteLogger log,
-            Thread.UncaughtExceptionHandler errHnd
+            IgniteInClosure<Throwable> errHnd,
+            GridWorkerListener gridWorkerLsnr
         ) {
-            this(igniteInstanceName, poolName, idx, log, null, errHnd);
+            this(igniteInstanceName, poolName, idx, log, null, errHnd, gridWorkerLsnr);
         }
 
         /**
@@ -609,7 +592,8 @@ private static class StripeConcurrentQueue extends Stripe {
          * @param poolName Pool name.
          * @param idx Stripe index.
          * @param log Logger.
-         * @param errHnd Exception handler.
+         * @param errHnd Critical failure handler.
+         * @param gridWorkerLsnr listener to link with stripe worker.
          */
         StripeConcurrentQueue(
             String igniteInstanceName,
@@ -617,14 +601,16 @@ private static class StripeConcurrentQueue extends Stripe {
             int idx,
             IgniteLogger log,
             Stripe[] others,
-            Thread.UncaughtExceptionHandler errHnd
+            IgniteInClosure<Throwable> errHnd,
+            GridWorkerListener gridWorkerLsnr
         ) {
             super(
                 igniteInstanceName,
                 poolName,
                 idx,
                 log,
-                errHnd);
+                errHnd,
+                gridWorkerLsnr);
 
             this.others = others;
 
@@ -723,20 +709,23 @@ private static class StripeConcurrentQueueNoPark extends Stripe {
          * @param poolName Pool name.
          * @param idx Stripe index.
          * @param log Logger.
-         * @param errHnd Exception handler.
+         * @param errHnd Critical failure handler.
+         * @param gridWorkerLsnr listener to link with stripe worker.
          */
         public StripeConcurrentQueueNoPark(
             String igniteInstanceName,
             String poolName,
             int idx,
             IgniteLogger log,
-            Thread.UncaughtExceptionHandler errHnd
+            IgniteInClosure<Throwable> errHnd,
+            GridWorkerListener gridWorkerLsnr
         ) {
             super(igniteInstanceName,
                 poolName,
                 idx,
                 log,
-                errHnd);
+                errHnd,
+                gridWorkerLsnr);
         }
 
         /** {@inheritDoc} */
@@ -782,20 +771,23 @@ private static class StripeConcurrentBlockingQueue extends Stripe {
          * @param poolName Pool name.
          * @param idx Stripe index.
          * @param log Logger.
-         * @param errHnd Exception handler.
+         * @param errHnd Critical failure handler.
+         * @param gridWorkerLsnr listener to link with stripe worker.
          */
         public StripeConcurrentBlockingQueue(
             String igniteInstanceName,
             String poolName,
             int idx,
             IgniteLogger log,
-            Thread.UncaughtExceptionHandler errHnd
+            IgniteInClosure<Throwable> errHnd,
+            GridWorkerListener gridWorkerLsnr
         ) {
             super(igniteInstanceName,
                 poolName,
                 idx,
                 log,
-                errHnd);
+                errHnd,
+                gridWorkerLsnr);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
index 03870dbb3e0d9..85332f4d85235 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/nio/GridNioServer.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.internal.util.worker.GridWorkerListener;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -142,9 +143,9 @@ public class GridNioServer<T> {
     /** Defines how many times selector should do {@code selectNow()} before doing {@code select(long)}. */
     private long selectorSpins;
 
-    /** Accept worker thread. */
+    /** Accept worker. */
     @GridToStringExclude
-    private final IgniteThread acceptThread;
+    private final GridNioAcceptWorker acceptWorker;
 
     /** Read worker threads. */
     private final IgniteThread[] clientThreads;
@@ -267,6 +268,7 @@ public class GridNioServer<T> {
      * @param skipRecoveryPred Skip recovery predicate.
      * @param msgQueueLsnr Message queue size listener.
      * @param readWriteSelectorsAssign If {@code true} then in/out connections are assigned to even/odd workers.
+     * @param workerLsnr Worker lifecycle listener.
      * @param filters Filters for this server.
      * @throws IgniteCheckedException If failed.
      */
@@ -292,6 +294,7 @@ private GridNioServer(
         IgnitePredicate<Message> skipRecoveryPred,
         IgniteBiInClosure<GridNioSession, Integer> msgQueueLsnr,
         boolean readWriteSelectorsAssign,
+        @Nullable GridWorkerListener workerLsnr,
         GridNioFilter... filters
     ) throws IgniteCheckedException {
         if (port != -1)
@@ -345,13 +348,11 @@ private GridNioServer(
             else
                 threadName = "nio-acceptor-" + srvName;
 
-            GridNioAcceptWorker w = new GridNioAcceptWorker(igniteInstanceName, threadName, log, acceptSelector);
-
-            acceptThread = new IgniteThread(w);
+            acceptWorker = new GridNioAcceptWorker(igniteInstanceName, threadName, log, acceptSelector, workerLsnr);
         }
         else {
             locAddr = null;
-            acceptThread = null;
+            acceptWorker = null;
         }
 
         clientWorkers = new ArrayList<>(selectorCnt);
@@ -366,8 +367,8 @@ private GridNioServer(
                 threadName = "grid-nio-worker-" + srvName + "-" + i;
 
             AbstractNioClientWorker worker = directMode ?
-                new DirectNioClientWorker(i, igniteInstanceName, threadName, log) :
-                new ByteBufferNioClientWorker(i, igniteInstanceName, threadName, log);
+                new DirectNioClientWorker(i, igniteInstanceName, threadName, log, workerLsnr) :
+                new ByteBufferNioClientWorker(i, igniteInstanceName, threadName, log, workerLsnr);
 
             clientWorkers.add(worker);
 
@@ -437,8 +438,8 @@ public static <T> Builder<T> builder() {
     public void start() {
         filterChain.start();
 
-        if (acceptThread != null)
-            acceptThread.start();
+        if (acceptWorker != null)
+            new IgniteThread(acceptWorker).start();
 
         for (IgniteThread thread : clientThreads)
             thread.start();
@@ -452,8 +453,8 @@ public void stop() {
             closed = true;
 
             // Make sure to entirely stop acceptor if any.
-            U.interrupt(acceptThread);
-            U.join(acceptThread, log);
+            U.cancel(acceptWorker);
+            U.join(acceptWorker, log);
 
             U.cancel(clientWorkers);
             U.join(clientWorkers, log);
@@ -1054,11 +1055,17 @@ private class ByteBufferNioClientWorker extends AbstractNioClientWorker {
          * @param igniteInstanceName Ignite instance name.
          * @param name Worker name.
          * @param log Logger.
+         * @param workerLsnr Worker lifecycle listener.
          * @throws IgniteCheckedException If selector could not be created.
          */
-        protected ByteBufferNioClientWorker(int idx, @Nullable String igniteInstanceName, String name, IgniteLogger log)
-            throws IgniteCheckedException {
-            super(idx, igniteInstanceName, name, log);
+        protected ByteBufferNioClientWorker(
+            int idx,
+            @Nullable String igniteInstanceName,
+            String name,
+            IgniteLogger log,
+            @Nullable GridWorkerListener workerLsnr
+        ) throws IgniteCheckedException {
+            super(idx, igniteInstanceName, name, log, workerLsnr);
 
             readBuf = directBuf ? ByteBuffer.allocateDirect(8 << 10) : ByteBuffer.allocate(8 << 10);
 
@@ -1224,11 +1231,17 @@ private class DirectNioClientWorker extends AbstractNioClientWorker {
          * @param igniteInstanceName Ignite instance name.
          * @param name Worker name.
          * @param log Logger.
+         * @param workerLsnr Worker lifecycle listener.
          * @throws IgniteCheckedException If selector could not be created.
          */
-        protected DirectNioClientWorker(int idx, @Nullable String igniteInstanceName, String name, IgniteLogger log)
-            throws IgniteCheckedException {
-            super(idx, igniteInstanceName, name, log);
+        protected DirectNioClientWorker(
+            int idx,
+            @Nullable String igniteInstanceName,
+            String name,
+            IgniteLogger log,
+            @Nullable GridWorkerListener workerLsnr
+        ) throws IgniteCheckedException {
+            super(idx, igniteInstanceName, name, log, workerLsnr);
         }
 
         /**
@@ -1747,11 +1760,17 @@ private abstract class AbstractNioClientWorker extends GridWorker implements Gri
          * @param igniteInstanceName Ignite instance name.
          * @param name Worker name.
          * @param log Logger.
+         * @param workerLsnr Worker lifecycle listener.
          * @throws IgniteCheckedException If selector could not be created.
          */
-        protected AbstractNioClientWorker(int idx, @Nullable String igniteInstanceName, String name, IgniteLogger log)
-            throws IgniteCheckedException {
-            super(igniteInstanceName, name, log);
+        AbstractNioClientWorker(
+            int idx,
+            @Nullable String igniteInstanceName,
+            String name,
+            IgniteLogger log,
+            @Nullable GridWorkerListener workerLsnr
+        ) throws IgniteCheckedException {
+            super(igniteInstanceName, name, log, workerLsnr);
 
             createSelector();
 
@@ -2118,6 +2137,10 @@ private void bodyInternal() throws IgniteCheckedException, InterruptedException
                             else
                                 processSelectedKeysOptimized(selectedKeys.flip());
                         }
+
+                        // select() call above doesn't throw on interruption; checking it here to propagate timely.
+                        if (!closed && !isCancelled && Thread.interrupted())
+                            throw new InterruptedException();
                     }
                     finally {
                         select = false;
@@ -2807,11 +2830,16 @@ private class GridNioAcceptWorker extends GridWorker {
          * @param name Thread name.
          * @param log Log.
          * @param selector Which will accept incoming connections.
+         * @param workerLsnr Worker lifecycle listener.
          */
         protected GridNioAcceptWorker(
-            @Nullable String igniteInstanceName, String name, IgniteLogger log, Selector selector
+            @Nullable String igniteInstanceName,
+            String name,
+            IgniteLogger log,
+            Selector selector,
+            @Nullable GridWorkerListener workerLsnr
         ) {
-            super(igniteInstanceName, name, log);
+            super(igniteInstanceName, name, log, workerLsnr);
 
             this.selector = selector;
         }
@@ -2823,7 +2851,7 @@ protected GridNioAcceptWorker(
             try {
                 boolean reset = false;
 
-                while (!closed && !Thread.currentThread().isInterrupted()) {
+                while (!closed && !isCancelled()) {
                     try {
                         if (reset)
                             selector = createSelector(locAddr);
@@ -3605,6 +3633,9 @@ public static class Builder<T> {
         /** */
         private boolean readWriteSelectorsAssign;
 
+        /** Worker lifecycle listener to be used by server's worker threads. */
+        private GridWorkerListener workerLsnr;
+
         /**
          * Finishes building the instance.
          *
@@ -3634,6 +3665,7 @@ public GridNioServer<T> build() throws IgniteCheckedException {
                 skipRecoveryPred,
                 msgQueueLsnr,
                 readWriteSelectorsAssign,
+                workerLsnr,
                 filters != null ? Arrays.copyOf(filters, filters.length) : EMPTY_FILTERS
             );
 
@@ -3888,6 +3920,16 @@ public Builder<T> messageQueueSizeListener(IgniteBiInClosure<GridNioSession, Int
 
             return this;
         }
+
+        /**
+         * @param workerLsnr Worker lifecycle listener.
+         * @return This for chaining.
+         */
+        public Builder<T> workerListener(GridWorkerListener workerLsnr) {
+            this.workerLsnr = workerLsnr;
+
+            return this;
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 7583d96dd4af2..3eab09bb24d5b 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -805,7 +805,8 @@ else if (connKey.dummy()) {
 
             /** {@inheritDoc} */
             @Override public void onFailure(FailureType failureType, Throwable failure) {
-                ((IgniteEx)ignite).context().failure().process(new FailureContext(failureType, failure));
+                if (ignite instanceof IgniteEx)
+                    ((IgniteEx)ignite).context().failure().process(new FailureContext(failureType, failure));
             }
 
             /**
@@ -2219,9 +2220,13 @@ public int boundPort() {
 
         nioSrvr.start();
 
-        commWorker = new CommunicationWorker(igniteInstanceName);
+        commWorker = new CommunicationWorker(igniteInstanceName, log);
 
-        commWorker.start();
+        new IgniteSpiThread(igniteInstanceName, commWorker.name(), log) {
+            @Override protected void body() {
+                commWorker.run();
+            }
+        }.start();
 
         // Ack start.
         if (log.isDebugEnabled())
@@ -2367,31 +2372,37 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
                         new GridConnectionBytesVerifyFilter(log)
                     };
 
-                GridNioServer<Message> srvr =
-                    GridNioServer.<Message>builder()
-                        .address(locHost)
-                        .port(port)
-                        .listener(srvLsnr)
-                        .logger(log)
-                        .selectorCount(selectorsCnt)
-                        .igniteInstanceName(igniteInstanceName)
-                        .serverName("tcp-comm")
-                        .tcpNoDelay(tcpNoDelay)
-                        .directBuffer(directBuf)
-                        .byteOrder(ByteOrder.nativeOrder())
-                        .socketSendBufferSize(sockSndBuf)
-                        .socketReceiveBufferSize(sockRcvBuf)
-                        .sendQueueLimit(msgQueueLimit)
-                        .directMode(true)
-                        .metricsListener(metricsLsnr)
-                        .writeTimeout(sockWriteTimeout)
-                        .selectorSpins(selectorSpins)
-                        .filters(filters)
-                        .writerFactory(writerFactory)
-                        .skipRecoveryPredicate(skipRecoveryPred)
-                        .messageQueueSizeListener(queueSizeMonitor)
-                        .readWriteSelectorsAssign(usePairedConnections)
-                        .build();
+                GridNioServer.Builder<Message> builder = GridNioServer.<Message>builder()
+                    .address(locHost)
+                    .port(port)
+                    .listener(srvLsnr)
+                    .logger(log)
+                    .selectorCount(selectorsCnt)
+                    .igniteInstanceName(igniteInstanceName)
+                    .serverName("tcp-comm")
+                    .tcpNoDelay(tcpNoDelay)
+                    .directBuffer(directBuf)
+                    .byteOrder(ByteOrder.nativeOrder())
+                    .socketSendBufferSize(sockSndBuf)
+                    .socketReceiveBufferSize(sockRcvBuf)
+                    .sendQueueLimit(msgQueueLimit)
+                    .directMode(true)
+                    .metricsListener(metricsLsnr)
+                    .writeTimeout(sockWriteTimeout)
+                    .selectorSpins(selectorSpins)
+                    .filters(filters)
+                    .writerFactory(writerFactory)
+                    .skipRecoveryPredicate(skipRecoveryPred)
+                    .messageQueueSizeListener(queueSizeMonitor)
+                    .readWriteSelectorsAssign(usePairedConnections);
+
+                if (ignite instanceof IgniteEx) {
+                    IgniteEx igniteEx = (IgniteEx)ignite;
+
+                    builder.workerListener(igniteEx.context().workersRegistry());
+                }
+
+                GridNioServer<Message> srvr = builder.build();
 
                 boundTcpPort = port;
 
@@ -2492,7 +2503,7 @@ private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
         if (nioSrvr != null)
             nioSrvr.stop();
 
-        U.interrupt(commWorker);
+        U.cancel(commWorker);
         U.join(commWorker, log);
 
         U.cancel(shmemAcceptWorker);
@@ -3856,7 +3867,8 @@ public void simulateNodeFailure() {
         if (nioSrvr != null)
             nioSrvr.stop();
 
-        U.interrupt(commWorker);
+        if (commWorker != null)
+            U.interrupt(commWorker.runner());
 
         U.join(commWorker, log);
 
@@ -4216,15 +4228,17 @@ private ShmemWorker(IpcEndpoint endpoint) {
     /**
      *
      */
-    private class CommunicationWorker extends IgniteSpiThread {
+    private class CommunicationWorker extends GridWorker {
         /** */
         private final BlockingQueue<DisconnectedSessionInfo> q = new LinkedBlockingQueue<>();
 
         /**
          * @param igniteInstanceName Ignite instance name.
+         * @param log Logger.
          */
-        private CommunicationWorker(String igniteInstanceName) {
-            super(igniteInstanceName, "tcp-comm-worker", log);
+        private CommunicationWorker(String igniteInstanceName, IgniteLogger log) {
+            super(igniteInstanceName, "tcp-comm-worker", log,
+                ignite instanceof IgniteEx ? ((IgniteEx)ignite).context().workersRegistry() : null);
         }
 
         /** {@inheritDoc} */
@@ -4235,7 +4249,7 @@ private CommunicationWorker(String igniteInstanceName) {
             Throwable err = null;
 
             try {
-                while (!isInterrupted()) {
+                while (!isCancelled()) {
                     DisconnectedSessionInfo disconnectData = q.poll(idleConnTimeout, TimeUnit.MILLISECONDS);
 
                     if (disconnectData != null)
@@ -4251,13 +4265,15 @@ private CommunicationWorker(String igniteInstanceName) {
                 throw t;
             }
             finally {
-                if (err == null && !stopping)
-                    err = new IllegalStateException("Thread  " + getName() + " is terminated unexpectedly.");
-
-                if (err instanceof OutOfMemoryError)
-                    ((IgniteEx)ignite).context().failure().process(new FailureContext(CRITICAL_ERROR, err));
-                else if (err != null)
-                    ((IgniteEx)ignite).context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+                if (ignite instanceof IgniteEx) {
+                    if (err == null && !stopping)
+                        err = new IllegalStateException("Thread  " + getName() + " is terminated unexpectedly.");
+
+                    if (err instanceof OutOfMemoryError)
+                        ((IgniteEx)ignite).context().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                    else if (err != null)
+                        ((IgniteEx)ignite).context().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+                }
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index b22a3970f2710..dc62bf3331fb2 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -26,7 +26,6 @@
 import java.net.SocketTimeoutException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
@@ -42,11 +41,13 @@
 import java.util.TreeSet;
 import java.util.UUID;
 import java.util.concurrent.BlockingDeque;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.net.ssl.SSLException;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
@@ -55,7 +56,9 @@
 import org.apache.ignite.cache.CacheMetrics;
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
@@ -68,6 +71,8 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.internal.worker.WorkersRegistry;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.spi.IgniteSpiContext;
@@ -104,7 +109,6 @@
 import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryServerOnlyCustomEventMessage;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
-import java.util.concurrent.ConcurrentHashMap;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY;
@@ -115,6 +119,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.events.EventType.EVT_NODE_METRICS_UPDATED;
 import static org.apache.ignite.events.EventType.EVT_NODE_SEGMENTED;
+import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
 import static org.apache.ignite.spi.discovery.tcp.ClientImpl.State.CONNECTED;
 import static org.apache.ignite.spi.discovery.tcp.ClientImpl.State.DISCONNECTED;
@@ -207,7 +212,7 @@ class ClientImpl extends TcpDiscoveryImpl {
 
         b.append("Internal threads: ").append(U.nl());
 
-        b.append("    Message worker: ").append(threadStatus(msgWorker)).append(U.nl());
+        b.append("    Message worker: ").append(threadStatus(msgWorker.runner())).append(U.nl());
         b.append("    Socket reader: ").append(threadStatus(sockReader)).append(U.nl());
         b.append("    Socket writer: ").append(threadStatus(sockWriter)).append(U.nl());
 
@@ -264,8 +269,13 @@ class ClientImpl extends TcpDiscoveryImpl {
         if (spi.ipFinder.isShared())
             registerLocalNodeAddress();
 
-        msgWorker = new MessageWorker();
-        msgWorker.start();
+        msgWorker = new MessageWorker(log);
+
+        new IgniteSpiThread(msgWorker.igniteInstanceName(), msgWorker.name(), log) {
+            @Override protected void body() {
+                msgWorker.run();
+            }
+        }.start();
 
         try {
             joinLatch.await();
@@ -289,7 +299,7 @@ class ClientImpl extends TcpDiscoveryImpl {
 
     /** {@inheritDoc} */
     @Override public void spiStop() throws IgniteSpiException {
-        if (msgWorker != null && msgWorker.isAlive()) { // Should always be alive
+        if (msgWorker != null && !msgWorker.isDone()) { // Should always be alive
             msgWorker.addMessage(SPI_STOP);
 
             try {
@@ -306,11 +316,15 @@ class ClientImpl extends TcpDiscoveryImpl {
 
         rmtNodes.clear();
 
-        U.interrupt(msgWorker);
+        if (msgWorker != null)
+            U.interrupt(msgWorker.runner());
+
         U.interrupt(sockWriter);
         U.interrupt(sockReader);
 
-        U.join(msgWorker, log);
+        if (msgWorker != null)
+            U.join(msgWorker.runner(), log);
+
         U.join(sockWriter, log);
         U.join(sockReader, log);
 
@@ -405,11 +419,15 @@ else if (state == DISCONNECTED) {
 
     /** {@inheritDoc} */
     @Override public void disconnect() throws IgniteSpiException {
-        U.interrupt(msgWorker);
+        if (msgWorker != null)
+            U.interrupt(msgWorker.runner());
+
         U.interrupt(sockWriter);
         U.interrupt(sockReader);
 
-        U.join(msgWorker, log);
+        if (msgWorker != null)
+            U.join(msgWorker.runner(), log);
+
         U.join(sockWriter, log);
         U.join(sockReader, log);
 
@@ -846,12 +864,14 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
         U.warn(log, "Simulating client node failure: " + getLocalNodeId());
 
         U.interrupt(sockWriter);
-        U.interrupt(msgWorker);
+
+        if (msgWorker != null)
+            U.interrupt(msgWorker.runner());
 
         U.join(sockWriter, log);
-        U.join(
-            msgWorker,
-            log);
+
+        if (msgWorker != null)
+            U.join(msgWorker.runner(), log);
     }
 
     /** {@inheritDoc} */
@@ -879,7 +899,20 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
 
     /** {@inheritDoc} */
     @Override protected Collection<IgniteSpiThread> threads() {
-        return Arrays.asList(sockWriter, msgWorker);
+        ArrayList<IgniteSpiThread> res = new ArrayList<>();
+
+        res.add(sockWriter);
+
+        MessageWorker msgWorker0 = msgWorker;
+
+        if (msgWorker0 != null) {
+            Thread runner = msgWorker0.runner();
+
+            if (runner instanceof IgniteSpiThread)
+                res.add((IgniteSpiThread)runner);
+        }
+
+        return res;
     }
 
     /**
@@ -889,7 +922,7 @@ private NavigableSet<ClusterNode> allVisibleNodes() {
     public void waitForClientMessagePrecessed() {
         Object last = msgWorker.queue.peekLast();
 
-        while (last != null && msgWorker.isAlive() && msgWorker.queue.contains(last)) {
+        while (last != null && !msgWorker.isDone() && msgWorker.queue.contains(last)) {
             try {
                 Thread.sleep(10);
             }
@@ -910,6 +943,13 @@ private void joinError(IgniteSpiException err) {
         joinLatch.countDown();
     }
 
+    /** */
+    private WorkersRegistry getWorkersRegistry() {
+        Ignite ignite = spi.ignite();
+
+        return ignite instanceof IgniteEx ? ((IgniteEx)ignite).context().workersRegistry() : null;
+    }
+
     /**
      * Metrics sender.
      */
@@ -1530,7 +1570,7 @@ else if (spi.ensured(msg)) {
     /**
      * Message worker.
      */
-    protected class MessageWorker extends IgniteSpiThread {
+    protected class MessageWorker extends GridWorker {
         /** Message queue. */
         private final BlockingDeque<Object> queue = new LinkedBlockingDeque<>();
 
@@ -1544,10 +1584,10 @@ protected class MessageWorker extends IgniteSpiThread {
         private boolean nodeAdded;
 
         /**
-         *
+         * @param log Logger.
          */
-        private MessageWorker() {
-            super(spi.ignite().name(), "tcp-client-disco-msg-worker", log);
+        private MessageWorker(IgniteLogger log) {
+            super(spi.ignite().name(), "tcp-client-disco-msg-worker", log, getWorkersRegistry());
         }
 
         /** {@inheritDoc} */
@@ -1789,6 +1829,13 @@ else if (discoMsg instanceof TcpDiscoveryCheckFailedMessage)
                     }
                 }
             }
+            catch (InterruptedException ignored) {
+                Thread.currentThread().interrupt();
+            }
+            catch (Throwable t) {
+                if (spi.ignite() instanceof IgniteEx)
+                    ((IgniteEx)spi.ignite()).context().failure().process(new FailureContext(CRITICAL_ERROR, t));
+            }
             finally {
                 SocketStream currSock = this.currSock;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 079058b799287..170c1badf3d54 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -94,6 +94,8 @@
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.internal.util.worker.GridWorkerListener;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteProductVersion;
@@ -108,7 +110,6 @@
 import org.apache.ignite.spi.IgniteSpiThread;
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
-import org.apache.ignite.spi.discovery.IgniteDiscoveryThread;
 import org.apache.ignite.spi.discovery.tcp.internal.DiscoveryDataPacket;
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNodesRing;
@@ -305,7 +306,7 @@ class ServerImpl extends TcpDiscoveryImpl {
     /** {@inheritDoc} */
     @Override public int boundPort() throws IgniteSpiException {
         if (tcpSrvr == null)
-            tcpSrvr = new TcpServer();
+            tcpSrvr = new TcpServer(log);
 
         return tcpSrvr.port;
     }
@@ -337,18 +338,19 @@ class ServerImpl extends TcpDiscoveryImpl {
         fromAddrs.clear();
         noResAddrs.clear();
 
-        msgWorker = new RingMessageWorker();
-        msgWorker.start();
+        msgWorker = new RingMessageWorker(log);
+
+        new MessageWorkerThread(msgWorker, log).start();
 
         if (tcpSrvr == null)
-            tcpSrvr = new TcpServer();
+            tcpSrvr = new TcpServer(log);
 
         spi.initLocalNode(tcpSrvr.port, true);
 
         locNode = spi.locNode;
 
         // Start TCP server thread after local node is initialized.
-        tcpSrvr.start();
+        new TcpServerThread(tcpSrvr, log).start();
 
         ring.localNode(locNode);
 
@@ -412,7 +414,7 @@ private void spiStop0(boolean disconnect) throws IgniteSpiException {
             }
         }
 
-        if (msgWorker != null && msgWorker.isAlive() && !disconnect) {
+        if (msgWorker != null && msgWorker.runner() != null && msgWorker.runner().isAlive() && !disconnect) {
             // Send node left message only if it is final stop.
             msgWorker.addMessage(new TcpDiscoveryNodeLeftMessage(locNode.id()));
 
@@ -446,7 +448,7 @@ else if (log.isInfoEnabled()) {
             }
         }
 
-        U.interrupt(tcpSrvr);
+        U.cancel(tcpSrvr);
         U.join(tcpSrvr, log);
 
         tcpSrvr = null;
@@ -463,12 +465,14 @@ else if (log.isInfoEnabled()) {
         U.interrupt(ipFinderCleaner);
         U.join(ipFinderCleaner, log);
 
-        U.interrupt(msgWorker);
+        U.cancel(msgWorker);
         U.join(msgWorker, log);
 
         for (ClientMessageWorker clientWorker : clientMsgWorkers.values()) {
-            U.interrupt(clientWorker);
-            U.join(clientWorker, log);
+            if (clientWorker != null) {
+                U.interrupt(clientWorker.runner());
+                U.join(clientWorker.runner(), log);
+            }
         }
 
         clientMsgWorkers.clear();
@@ -1657,7 +1661,7 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
     @Override void simulateNodeFailure() {
         U.warn(log, "Simulating node failure: " + getLocalNodeId());
 
-        U.interrupt(tcpSrvr);
+        U.cancel(tcpSrvr);
         U.join(tcpSrvr, log);
 
         U.interrupt(ipFinderCleaner);
@@ -1672,13 +1676,14 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
         U.interrupt(tmp);
         U.joinThreads(tmp, log);
 
-        U.interrupt(msgWorker);
+        U.cancel(msgWorker);
         U.join(msgWorker, log);
 
         for (ClientMessageWorker msgWorker : clientMsgWorkers.values()) {
-            U.interrupt(msgWorker);
-
-            U.join(msgWorker, log);
+            if (msgWorker != null) {
+                U.interrupt(msgWorker.runner());
+                U.join(msgWorker.runner(), log);
+            }
         }
 
         U.interrupt(statsPrinter);
@@ -1707,10 +1712,36 @@ private void clearNodeAddedMessage(TcpDiscoveryAbstractMessage msg) {
             threads.addAll(readers);
         }
 
-        threads.addAll(clientMsgWorkers.values());
-        threads.add(tcpSrvr);
+        for (ClientMessageWorker wrk : clientMsgWorkers.values()) {
+            Thread t = wrk.runner();
+
+            assert t instanceof IgniteSpiThread;
+
+            threads.add((IgniteSpiThread)t);
+        }
+
+        TcpServer tcpSrvr0 = tcpSrvr;
+
+        if (tcpSrvr0 != null) {
+            Thread tcpServerThread = tcpSrvr0.runner();
+
+            if (tcpServerThread != null) {
+                assert tcpServerThread instanceof IgniteSpiThread;
+
+                threads.add((IgniteSpiThread)tcpServerThread);
+            }
+        }
+
         threads.add(ipFinderCleaner);
-        threads.add(msgWorker);
+
+        Thread msgWorkerThread = msgWorker.runner();
+
+        if (msgWorkerThread != null) {
+            assert msgWorkerThread instanceof IgniteSpiThread;
+
+            threads.add((IgniteSpiThread)msgWorkerThread);
+        }
+
         threads.add(statsPrinter);
 
         threads.removeAll(Collections.<IgniteSpiThread>singleton(null));
@@ -1776,7 +1807,7 @@ TcpDiscoveryNodesRing ring() {
 
             b.append("Internal threads: ").append(U.nl());
 
-            b.append("    Message worker: ").append(threadStatus(msgWorker)).append(U.nl());
+            b.append("    Message worker: ").append(threadStatus(msgWorker.runner())).append(U.nl());
 
             b.append("    IP finder cleaner: ").append(threadStatus(ipFinderCleaner)).append(U.nl());
             b.append("    Stats printer: ").append(threadStatus(statsPrinter)).append(U.nl());
@@ -2534,9 +2565,9 @@ private void advance() {
     }
 
     /**
-     * Message worker thread for messages processing.
+     * Message worker for discovery messages processing.
      */
-    private class RingMessageWorker extends MessageWorkerAdapter<TcpDiscoveryAbstractMessage> implements IgniteDiscoveryThread {
+    private class RingMessageWorker extends MessageWorker<TcpDiscoveryAbstractMessage> {
         /** Next node. */
         @SuppressWarnings({"FieldAccessedSynchronizedAndUnsynchronized"})
         private TcpDiscoveryNode next;
@@ -2581,9 +2612,11 @@ private class RingMessageWorker extends MessageWorkerAdapter<TcpDiscoveryAbstrac
         private long lastRingMsgTime;
 
         /**
+         * @param log Logger.
          */
-        RingMessageWorker() {
-            super("tcp-disco-msg-worker", 10);
+        private RingMessageWorker(IgniteLogger log) {
+            super("tcp-disco-msg-worker", log, 10,
+                spi.ignite() instanceof IgniteEx ? ((IgniteEx)spi.ignite()).context().workersRegistry() : null);
 
             initConnectionCheckFrequency();
         }
@@ -2660,15 +2693,17 @@ void addMessage(TcpDiscoveryAbstractMessage msg) {
                 throw e;
             }
             finally {
-                if (err == null && !spi.isNodeStopping0() && spiStateCopy() != DISCONNECTING)
-                    err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
+                if (spi.ignite() instanceof IgniteEx) {
+                    if (err == null && !spi.isNodeStopping0()&& spiStateCopy() != DISCONNECTING)
+                        err = new IllegalStateException("Worker " + name() + " is terminated unexpectedly.");
 
-                FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
+                    FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
 
-                if (err instanceof OutOfMemoryError)
-                    failure.process(new FailureContext(CRITICAL_ERROR, err));
-                else if (err != null)
-                    failure.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+                    if (err instanceof OutOfMemoryError)
+                        failure.process(new FailureContext(CRITICAL_ERROR, err));
+                    else if (err != null)
+                        failure.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+                }
             }
         }
 
@@ -4856,8 +4891,8 @@ private void processNodeFailedMessage(TcpDiscoveryNodeFailedMessage msg) {
                     if (!msg.force()) { // ClientMessageWorker will stop after sending force fail message.
                         ClientMessageWorker worker = clientMsgWorkers.remove(failedNode.id());
 
-                        if (worker != null)
-                            worker.interrupt();
+                        if (worker != null && worker.runner() != null)
+                            worker.runner().interrupt();
                     }
                 }
 
@@ -5572,13 +5607,43 @@ private void checkConnection() {
         }
     }
 
+    /** Thread that executes {@link TcpServer}'s code. */
+    private class TcpServerThread extends IgniteSpiThread {
+        /** */
+        private final TcpServer worker;
+
+        /**
+         * @param worker Worker to be executed by this thread.
+         * @param log Logger.
+         */
+        private TcpServerThread(TcpServer worker, IgniteLogger log) {
+            super(worker.igniteInstanceName(), worker.name(), log);
+
+            setPriority(spi.threadPri);
+
+            this.worker = worker;
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void body() throws InterruptedException {
+            worker.run();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void interrupt() {
+            super.interrupt();
+
+            worker.onInterruption();
+        }
+    }
+
     /**
-     * Thread that accepts incoming TCP connections.
+     * Worker that accepts incoming TCP connections.
      * <p>
      * Tcp server will call provided closure when accepts incoming connection.
      * From that moment server is no more responsible for the socket.
      */
-    private class TcpServer extends IgniteSpiThread {
+    private class TcpServer extends GridWorker {
         /** Socket TCP server listens to. */
         private ServerSocket srvrSock;
 
@@ -5586,14 +5651,12 @@ private class TcpServer extends IgniteSpiThread {
         private int port;
 
         /**
-         * Constructor.
-         *
+         * @param log Logger.
          * @throws IgniteSpiException In case of error.
          */
-        TcpServer() throws IgniteSpiException {
-            super(spi.ignite().name(), "tcp-disco-srvr", log);
-
-            setPriority(spi.threadPri);
+        TcpServer(IgniteLogger log) throws IgniteSpiException {
+            super(spi.ignite().name(), "tcp-disco-srvr", log,
+                spi.ignite() instanceof IgniteEx ? ((IgniteEx)spi.ignite()).context().workersRegistry() : null);
 
             int lastPort = spi.locPortRange == 0 ? spi.locPort : spi.locPort + spi.locPortRange - 1;
 
@@ -5640,7 +5703,7 @@ private class TcpServer extends IgniteSpiThread {
             Throwable err = null;
 
             try {
-                while (!isInterrupted()) {
+                while (!isCancelled()) {
                     Socket sock = srvrSock.accept();
 
                     long tstamp = U.currentTimeMillis();
@@ -5670,7 +5733,7 @@ private class TcpServer extends IgniteSpiThread {
 
                 onException("Failed to accept TCP connection.", e);
 
-                if (!isInterrupted()) {
+                if (!runner().isInterrupted()) {
                     err = e;
 
                     if (U.isMacInvalidArgumentError(e))
@@ -5685,24 +5748,24 @@ private class TcpServer extends IgniteSpiThread {
                 throw t;
             }
             finally {
-                if (err == null && !spi.isNodeStopping0() && spiStateCopy() != DISCONNECTING)
-                    err = new IllegalStateException("Thread " + getName() + " is terminated unexpectedly.");
+                if (spi.ignite() instanceof IgniteEx) {
+                    if (err == null && !spi.isNodeStopping0() && spiStateCopy() != DISCONNECTING)
+                        err = new IllegalStateException("Worker " + name() + " is terminated unexpectedly.");
 
-                FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
+                    FailureProcessor failure = ((IgniteEx)spi.ignite()).context().failure();
 
-                if (err instanceof OutOfMemoryError)
-                    failure.process(new FailureContext(CRITICAL_ERROR, err));
-                else if (err != null)
-                    failure.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+                    if (err instanceof OutOfMemoryError)
+                        failure.process(new FailureContext(CRITICAL_ERROR, err));
+                    else if (err != null)
+                        failure.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+                }
 
                 U.closeQuiet(srvrSock);
             }
         }
 
-        /** {@inheritDoc} */
-        @Override public void interrupt() {
-            super.interrupt();
-
+        /** */
+        public void onInterruption() {
             U.close(srvrSock, log);
         }
     }
@@ -5874,7 +5937,7 @@ else if (log.isDebugEnabled())
                     }
 
                     if (req.client()) {
-                        ClientMessageWorker clientMsgWrk0 = new ClientMessageWorker(sock, nodeId);
+                        ClientMessageWorker clientMsgWrk0 = new ClientMessageWorker(sock, nodeId, log);
 
                         while (true) {
                             ClientMessageWorker old = clientMsgWorkers.putIfAbsent(nodeId, clientMsgWrk0);
@@ -5882,13 +5945,14 @@ else if (log.isDebugEnabled())
                             if (old == null)
                                 break;
 
-                            if (old.isInterrupted()) {
+                            if (old.isDone() || (old.runner() != null && old.runner().isInterrupted())) {
                                 clientMsgWorkers.remove(nodeId, old);
 
                                 continue;
                             }
 
-                            old.join(500);
+                            if (old.runner() != null)
+                                old.runner().join(500);
 
                             old = clientMsgWorkers.putIfAbsent(nodeId, clientMsgWrk0);
 
@@ -6020,8 +6084,8 @@ else if (msg instanceof TcpDiscoveryClientReconnectMessage) {
                             if (state == CONNECTED) {
                                 spi.writeToSocket(msg, sock, RES_OK, sockTimeout);
 
-                                if (clientMsgWrk != null && clientMsgWrk.getState() == State.NEW)
-                                    clientMsgWrk.start();
+                                if (clientMsgWrk != null && clientMsgWrk.runner() == null && !clientMsgWrk.isDone())
+                                    new MessageWorkerThreadWithCleanup<>(clientMsgWrk, log).start();
 
                                 processClientReconnectMessage((TcpDiscoveryClientReconnectMessage)msg);
 
@@ -6262,7 +6326,7 @@ else if (msg instanceof TcpDiscoveryRingLatencyCheckMessage) {
 
                     clientMsgWorkers.remove(nodeId, clientMsgWrk);
 
-                    U.interrupt(clientMsgWrk);
+                    U.interrupt(clientMsgWrk.runner());
                 }
 
                 U.closeQuiet(sock);
@@ -6408,10 +6472,10 @@ private boolean processJoinRequestMessage(TcpDiscoveryJoinRequestMessage msg,
 
                 msg.responded(true);
 
-                if (clientMsgWrk != null && clientMsgWrk.getState() == State.NEW) {
+                if (clientMsgWrk != null && clientMsgWrk.runner() == null && !clientMsgWrk.isDone()) {
                     clientMsgWrk.clientVersion(U.productVersion(msg.node()));
 
-                    clientMsgWrk.start();
+                    new MessageWorkerThreadWithCleanup<>(clientMsgWrk, log).start();
                 }
 
                 msgWorker.addMessage(msg);
@@ -6508,9 +6572,8 @@ private class StatisticsPrinter extends IgniteSpiThread {
         }
     }
 
-    /**
-     */
-    private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAbstractMessage, byte[]>> {
+    /** */
+    private class ClientMessageWorker extends MessageWorker<T2<TcpDiscoveryAbstractMessage, byte[]>> {
         /** Node ID. */
         private final UUID clientNodeId;
 
@@ -6529,9 +6592,10 @@ private class ClientMessageWorker extends MessageWorkerAdapter<T2<TcpDiscoveryAb
         /**
          * @param sock Socket.
          * @param clientNodeId Node ID.
+         * @param log Logger.
          */
-        protected ClientMessageWorker(Socket sock, UUID clientNodeId) throws IOException {
-            super("tcp-disco-client-message-worker", 2000);
+        private ClientMessageWorker(Socket sock, UUID clientNodeId, IgniteLogger log) {
+            super("tcp-disco-client-message-worker", log, 2000, null);
 
             this.sock = sock;
             this.clientNodeId = clientNodeId;
@@ -6649,7 +6713,7 @@ else if (msgLog.isDebugEnabled())
                 if (!success) {
                     clientMsgWorkers.remove(clientNodeId, this);
 
-                    U.interrupt(this);
+                    U.interrupt(runner());
 
                     U.closeQuiet(sock);
                 }
@@ -6729,39 +6793,100 @@ public boolean ping(IgniteSpiOperationTimeoutHelper timeoutHelper) throws Interr
             }
         }
 
+        /** {@inheritDoc} */
+        @Override protected void tearDown() {
+            pingResult(false);
+
+            U.closeQuiet(sock);
+        }
+    }
+
+    /** */
+    private class MessageWorkerThreadWithCleanup<T> extends MessageWorkerThread {
+        /** */
+        private final MessageWorker worker;
+
+        /** {@inheritDoc} */
+        private MessageWorkerThreadWithCleanup(MessageWorker<T> worker, IgniteLogger log) {
+            super(worker, log);
+
+            this.worker = worker;
+        }
+
         /** {@inheritDoc} */
         @Override protected void cleanup() {
             super.cleanup();
 
-            pingResult(false);
+            worker.tearDown();
+        }
+    }
 
-            U.closeQuiet(sock);
+    /**
+     * Slightly modified {@link IgniteSpiThread} intended to use with message workers.
+     */
+    private class MessageWorkerThread extends IgniteSpiThread {
+        /**
+         * Backed interrupted flag, once set, it is not affected by further {@link Thread#interrupted()} calls.
+         */
+        private volatile boolean interrupted;
+
+        /** */
+        private final GridWorker worker;
+
+        /** {@inheritDoc} */
+        private MessageWorkerThread(GridWorker worker, IgniteLogger log) {
+            super(worker.igniteInstanceName(), worker.name(), log);
+
+            this.worker = worker;
+
+            setPriority(spi.threadPri);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void body() throws InterruptedException {
+            worker.run();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void interrupt() {
+            interrupted = true;
+
+            super.interrupt();
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isInterrupted() {
+            return interrupted || super.isInterrupted();
         }
     }
 
     /**
-     * Base class for message workers.
+     * Superclass for all message workers.
+     *
+     * @param <T> Message type.
      */
-    protected abstract class MessageWorkerAdapter<T> extends IgniteSpiThread {
+    private abstract class MessageWorker<T> extends GridWorker {
         /** Message queue. */
         protected final BlockingDeque<T> queue = new LinkedBlockingDeque<>();
 
-        /** Backed interrupted flag. */
-        private volatile boolean interrupted;
-
         /** Polling timeout. */
         private final long pollingTimeout;
 
         /**
-         * @param name Thread name.
+         * @param name Worker name.
+         * @param log Logger.
          * @param pollingTimeout Messages polling timeout.
+         * @param lsnr Listener for life-cycle events.
          */
-        protected MessageWorkerAdapter(String name, long pollingTimeout) {
-            super(spi.ignite().name(), name, log);
+        protected MessageWorker(
+            String name,
+            IgniteLogger log,
+            long pollingTimeout,
+            @Nullable GridWorkerListener lsnr
+        ) {
+            super(spi.ignite().name(), name, log, lsnr);
 
             this.pollingTimeout = pollingTimeout;
-
-            setPriority(spi.threadPri);
         }
 
         /** {@inheritDoc} */
@@ -6769,7 +6894,7 @@ protected MessageWorkerAdapter(String name, long pollingTimeout) {
             if (log.isDebugEnabled())
                 log.debug("Message worker started [locNodeId=" + getConfiguredNodeId() + ']');
 
-            while (!isInterrupted()) {
+            while (!isCancelled()) {
                 T msg = queue.poll(pollingTimeout, TimeUnit.MILLISECONDS);
 
                 if (msg == null)
@@ -6779,26 +6904,16 @@ protected MessageWorkerAdapter(String name, long pollingTimeout) {
             }
         }
 
-        /** {@inheritDoc} */
-        @Override public void interrupt() {
-            interrupted = true;
-
-            super.interrupt();
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean isInterrupted() {
-            return interrupted || super.isInterrupted();
-        }
-
         /**
-         * @return Current queue size.
+         * @return Current message queue size.
          */
         int queueSize() {
             return queue.size();
         }
 
         /**
+         * Processes succeeding message.
+         *
          * @param msg Message.
          */
         protected abstract void processMessage(T msg);
@@ -6809,6 +6924,13 @@ int queueSize() {
         protected void noMessageLoop() {
             // No-op.
         }
+
+        /**
+         * Actions to be done before worker termination.
+         */
+        protected void tearDown() {
+            // No-op.
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/IgniteDiagnosticMessagesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/IgniteDiagnosticMessagesTest.java
index 572f3562778d0..11dceef77836a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/IgniteDiagnosticMessagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/IgniteDiagnosticMessagesTest.java
@@ -305,6 +305,8 @@ public void testLongRunningTx() throws Exception {
 
             GridStringLogger strLog = this.strLog = new GridStringLogger();
 
+            strLog.logLength(65536);
+
             startGrid(1);
 
             awaitPartitionMapExchange();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
index 3fca7afed7ff5..9a4bf0619c4df 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/StripedExecutorTest.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.util;
 
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.logger.java.JavaLogger;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -29,7 +30,10 @@ public class StripedExecutorTest extends GridCommonAbstractTest {
 
     /** {@inheritDoc} */
     @Override public void beforeTest() {
-        stripedExecSvc = new StripedExecutor(3, "foo name", "pool name", new JavaLogger(), (thread, t) -> {});
+        stripedExecSvc = new StripedExecutor(3, "foo name", "pool name", new JavaLogger(),
+            new IgniteInClosure<Throwable>() {
+                @Override public void apply(Throwable throwable) {}
+            }, null);
     }
 
     /** {@inheritDoc} */
@@ -165,4 +169,4 @@ public TestRunnable(boolean infinitely) {
     private void sleepASec() throws InterruptedException {
         Thread.sleep(1000);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
index d6d484ca5956c..d50a967aae172 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySelfTest.java
@@ -220,10 +220,15 @@ else if (igniteInstanceName.contains("testNoRingMessageWorkerAbnormalFailureSegm
             cfg.setFailureDetectionTimeout(6_000);
 
             cfg.setGridLogger(strLog = new GridStringLogger());
+
+            strLog.logLength(300_000);
         }
-        else if (igniteInstanceName.contains("testNodeShutdownOnRingMessageWorkerFailureFailedNode"))
+        else if (igniteInstanceName.contains("testNodeShutdownOnRingMessageWorkerFailureFailedNode")) {
             cfg.setGridLogger(strLog = new GridStringLogger());
 
+            strLog.logLength(300_000);
+        }
+
         cfg.setClientMode(client);
 
         return cfg;
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
index 5b853856bcde5..d1de34797cc70 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
@@ -79,6 +79,7 @@ public GridTestKernalContext(IgniteLogger log, IgniteConfiguration cfg) {
                 null,
                 null,
                 U.allPluginProviders(),
+                null,
                 null
         );
 

From fd13deec3156af81e4f26f420b2ff2b18a2b0686 Mon Sep 17 00:00:00 2001
From: Dmitriy Sorokin <d.w.sorokin@gmail.com>
Date: Mon, 18 Jun 2018 13:48:11 +0300
Subject: [PATCH 207/543] IGNITE-8749 Exception for no space left situation
 should be propagated to FailureHandler.

Signed-off-by: agura <agura@apache.org>
---
 .../wal/FileWriteAheadLogManager.java         | 167 ++++++------
 .../FsyncModeFileWriteAheadLogManager.java    | 215 ++++++++-------
 .../ignite/failure/TestFailureHandler.java    |  19 ++
 .../wal/IgniteWalFormatFileFailoverTest.java  | 258 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 5 files changed, 483 insertions(+), 179 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 9b39987edf42a..09a08c9d4bce5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -605,48 +605,43 @@ private void checkWalConfiguration() throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
-        try {
-            assert currHnd == null;
-            assert lastPtr == null || lastPtr instanceof FileWALPointer;
+        assert currHnd == null;
+        assert lastPtr == null || lastPtr instanceof FileWALPointer;
 
-            FileWALPointer filePtr = (FileWALPointer)lastPtr;
+        FileWALPointer filePtr = (FileWALPointer)lastPtr;
 
             walWriter = new WALWriter(log);
 
             if (!mmap)
                 new IgniteThread(walWriter).start();
 
-            currHnd = restoreWriteHandle(filePtr);
+        currHnd = restoreWriteHandle(filePtr);
 
-            // For new handle write serializer version to it.
-            if (filePtr == null)
-                currHnd.writeHeader();
+        // For new handle write serializer version to it.
+        if (filePtr == null)
+            currHnd.writeHeader();
 
-            if (currHnd.serializer.version() != serializer.version()) {
-                if (log.isInfoEnabled())
-                    log.info("Record serializer version change detected, will start logging with a new WAL record " +
-                        "serializer to a new WAL segment [curFile=" + currHnd + ", newVer=" + serializer.version() +
-                        ", oldVer=" + currHnd.serializer.version() + ']');
+        if (currHnd.serializer.version() != serializer.version()) {
+            if (log.isInfoEnabled())
+                log.info("Record serializer version change detected, will start logging with a new WAL record " +
+                    "serializer to a new WAL segment [curFile=" + currHnd + ", newVer=" + serializer.version() +
+                    ", oldVer=" + currHnd.serializer.version() + ']');
 
-                rollOver(currHnd);
-            }
+            rollOver(currHnd);
+        }
 
-            currHnd.resume = false;
+        currHnd.resume = false;
 
-            if (mode == WALMode.BACKGROUND) {
-                backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
-                    @Override public void run() {
-                        doFlush();
-                    }
-                }, flushFreq, flushFreq);
-            }
-
-            if (walAutoArchiveAfterInactivity > 0)
-                scheduleNextInactivityPeriodElapsedCheck();
-        }
-        catch (StorageException e) {
-            throw new IgniteCheckedException(e);
+        if (mode == WALMode.BACKGROUND) {
+            backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
+                @Override public void run() {
+                    doFlush();
+                }
+            }, flushFreq, flushFreq);
         }
+
+        if (walAutoArchiveAfterInactivity > 0)
+            scheduleNextInactivityPeriodElapsedCheck();
     }
 
     /**
@@ -1131,9 +1126,9 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, I
     /**
      * @param lastReadPtr Last read WAL file pointer.
      * @return Initialized file write handle.
-     * @throws IgniteCheckedException If failed to initialize WAL write handle.
+     * @throws StorageException If failed to initialize WAL write handle.
      */
-    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws IgniteCheckedException {
+    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws StorageException {
         long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
 
         @Nullable FileArchiver archiver0 = archiver;
@@ -1175,14 +1170,9 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
                 SegmentedRingByteBuffer rbuf;
 
                 if (mmap) {
-                    try {
-                        MappedByteBuffer buf = fileIO.map((int)maxWalSegmentSize);
+                    MappedByteBuffer buf = fileIO.map((int)maxWalSegmentSize);
 
-                        rbuf = new SegmentedRingByteBuffer(buf, metrics);
-                    }
-                    catch (IOException e) {
-                        throw new IgniteCheckedException(e);
-                    }
+                    rbuf = new SegmentedRingByteBuffer(buf, metrics);
                 }
                 else
                     rbuf = new SegmentedRingByteBuffer(dsCfg.getWalBufferSize(), maxWalSegmentSize, DIRECT, metrics);
@@ -1206,13 +1196,21 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
                 return hnd;
             }
             catch (IgniteCheckedException | IOException e) {
-                fileIO.close();
+                try {
+                    fileIO.close();
+                }
+                catch (IOException suppressed) {
+                    e.addSuppressed(suppressed);
+                }
 
-                throw e;
+                if (e instanceof StorageException)
+                    throw (StorageException) e;
+
+                throw e instanceof IOException ? (IOException) e : new IOException(e);
             }
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
+            throw new StorageException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
         }
     }
 
@@ -1223,9 +1221,8 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
      * @param cur Current file write handle released by WAL writer
      * @return Initialized file handle.
      * @throws StorageException If IO exception occurred.
-     * @throws IgniteCheckedException If failed.
      */
-    private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageException, IgniteCheckedException {
+    private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageException {
         try {
             File nextFile = pollNextFile(cur.idx);
 
@@ -1310,8 +1307,10 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageE
 
     /**
      * Deletes temp files, creates and prepares new; Creates first segment if necessary
+     *
+     * @throws StorageException If failed.
      */
-    private void checkOrPrepareFiles() throws IgniteCheckedException {
+    private void checkOrPrepareFiles() throws StorageException {
         // Clean temp files.
         {
             File[] tmpFiles = walWorkDir.listFiles(WAL_SEGMENT_TEMP_FILE_FILTER);
@@ -1321,7 +1320,7 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
                     boolean deleted = tmp.delete();
 
                     if (!deleted)
-                        throw new IgniteCheckedException("Failed to delete previously created temp file " +
+                        throw new StorageException("Failed to delete previously created temp file " +
                             "(make sure Ignite process has enough rights): " + tmp.getAbsolutePath());
                 }
             }
@@ -1331,7 +1330,7 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
 
         if(isArchiverEnabled())
             if (allFiles.length != 0 && allFiles.length > dsCfg.getWalSegments())
-                throw new IgniteCheckedException("Failed to initialize wal (work directory contains " +
+                throw new StorageException("Failed to initialize wal (work directory contains " +
                     "incorrect number of segments) [cur=" + allFiles.length + ", expected=" + dsCfg.getWalSegments() + ']');
 
         // Allocate the first segment synchronously. All other segments will be allocated by archiver in background.
@@ -1371,9 +1370,9 @@ public void cleanupWalDirectories() throws IgniteCheckedException {
      * Clears whole the file, fills with zeros for Default mode.
      *
      * @param file File to format.
-     * @throws IgniteCheckedException if formatting failed
+     * @throws StorageException if formatting failed
      */
-    private void formatFile(File file) throws IgniteCheckedException {
+    private void formatFile(File file) throws StorageException {
         formatFile(file, dsCfg.getWalSegmentSize());
     }
 
@@ -1382,9 +1381,9 @@ private void formatFile(File file) throws IgniteCheckedException {
      *
      * @param file File to format.
      * @param bytesCntToFormat Count of first bytes to format.
-     * @throws IgniteCheckedException if formatting failed
+     * @throws StorageException if formatting failed
      */
-    private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedException {
+    private void formatFile(File file, int bytesCntToFormat) throws StorageException {
         if (log.isDebugEnabled())
             log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1396,7 +1395,7 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                     int toWrite = Math.min(FILL_BUF.length, left);
 
                     if (fileIO.write(FILL_BUF, 0, toWrite) < toWrite) {
-                        final IgniteCheckedException ex = new IgniteCheckedException("Failed to extend WAL segment file: " +
+                        final StorageException ex = new StorageException("Failed to extend WAL segment file: " +
                             file.getName() + ". Probably disk is too busy, please check your device.");
 
                         if (failureProcessor != null)
@@ -1414,7 +1413,7 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                 fileIO.clear();
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
+            throw new StorageException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
         }
     }
 
@@ -1422,9 +1421,9 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
      * Creates a file atomically with temp file.
      *
      * @param file File to create.
-     * @throws IgniteCheckedException If failed.
+     * @throws StorageException If failed.
      */
-    private void createFile(File file) throws IgniteCheckedException {
+    private void createFile(File file) throws StorageException {
         if (log.isDebugEnabled())
             log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1436,7 +1435,7 @@ private void createFile(File file) throws IgniteCheckedException {
             Files.move(tmp.toPath(), file.toPath());
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to move temp file to a regular WAL segment file: " +
+            throw new StorageException("Failed to move temp file to a regular WAL segment file: " +
                 file.getAbsolutePath(), e);
         }
 
@@ -1449,9 +1448,9 @@ private void createFile(File file) throws IgniteCheckedException {
      *
      * @param curIdx Current absolute WAL segment index.
      * @return File ready for use as new WAL segment.
-     * @throws IgniteCheckedException If failed.
+     * @throws StorageException If exception occurred in the archiver thread.
      */
-    private File pollNextFile(long curIdx) throws IgniteCheckedException {
+    private File pollNextFile(long curIdx) throws StorageException {
         FileArchiver archiver0 = archiver;
 
         if (archiver0 == null) {
@@ -1527,7 +1526,7 @@ public long maxWalSegmentSize() {
      */
     private class FileArchiver extends GridWorker {
         /** Exception which occurred during initial creation of files or during archiving WAL segment */
-        private IgniteCheckedException cleanErr;
+        private StorageException cleanErr;
 
         /**
          * Absolute current segment index WAL Manager writes to. Guarded by <code>this</code>. Incremented during
@@ -1599,15 +1598,17 @@ private synchronized boolean locked(long absIdx) {
             try {
                 allocateRemainingFiles();
             }
-            catch (IgniteCheckedException e) {
+            catch (StorageException e) {
                 synchronized (this) {
                     // Stop the thread and report to starter.
                     cleanErr = e;
 
                     notifyAll();
-
-                    return;
                 }
+
+                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
+
+                return;
             }
 
             Throwable err = null;
@@ -1691,9 +1692,9 @@ private void changeLastArchivedIndexAndNotifyWaiters(long idx) {
          *
          * @param curIdx Current absolute index that we want to increment.
          * @return Next index (curWalSegmIdx+1) when it is ready to be written.
-         * @throws IgniteCheckedException If failed (if interrupted or if exception occurred in the archiver thread).
+         * @throws StorageException If exception occurred in the archiver thread.
          */
-        private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException {
+        private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException {
             synchronized (this) {
                 if (cleanErr != null)
                     throw cleanErr;
@@ -1708,6 +1709,9 @@ private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException
                 while (curAbsWalIdx - lastAbsArchivedIdx > dsCfg.getWalSegments() && cleanErr == null) {
                     try {
                         wait();
+
+                        if (cleanErr != null)
+                            throw cleanErr;
                     }
                     catch (InterruptedException ignore) {
                         interrupted.set(true);
@@ -1715,9 +1719,12 @@ private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException
                 }
 
                 // Wait for formatter so that we do not open an empty file in DEFAULT mode.
-                while (curAbsWalIdx % dsCfg.getWalSegments() > formatted)
+                while (curAbsWalIdx % dsCfg.getWalSegments() > formatted && cleanErr == null)
                     try {
                         wait();
+
+                        if (cleanErr != null)
+                            throw cleanErr;
                     }
                     catch (InterruptedException ignore) {
                         interrupted.set(true);
@@ -1789,7 +1796,7 @@ private void releaseWorkSegment(long absIdx) {
          *
          * @param absIdx Absolute index to archive.
          */
-        private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedException {
+        private SegmentArchiveResult archiveSegment(long absIdx) throws StorageException {
             long segIdx = absIdx % dsCfg.getWalSegments();
 
             File origFile = new File(walWorkDir, FileDescriptor.fileName(segIdx));
@@ -1818,7 +1825,7 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedExc
                 }
             }
             catch (IOException e) {
-                throw new IgniteCheckedException("Failed to archive WAL segment [" +
+                throw new StorageException("Failed to archive WAL segment [" +
                     "srcFile=" + origFile.getAbsolutePath() +
                     ", dstFile=" + dstTmpFile.getAbsolutePath() + ']', e);
             }
@@ -1841,7 +1848,7 @@ private boolean checkStop() {
          * Background creation of all segments except first. First segment was created in main thread by {@link
          * FileWriteAheadLogManager#checkOrPrepareFiles()}
          */
-        private void allocateRemainingFiles() throws IgniteCheckedException {
+        private void allocateRemainingFiles() throws StorageException {
             checkFiles(
                 1,
                 true,
@@ -2235,23 +2242,23 @@ private void shutdown() throws IgniteInterruptedCheckedException {
      * @param startWith Start with.
      * @param create Flag create file.
      * @param p Predicate Exit condition.
-     * @throws IgniteCheckedException if validation or create file fail.
+     * @throws StorageException if validation or create file fail.
      */
     private void checkFiles(
         int startWith,
         boolean create,
         @Nullable IgnitePredicate<Integer> p,
         @Nullable IgniteInClosure<Integer> completionCallback
-    ) throws IgniteCheckedException {
+    ) throws StorageException {
         for (int i = startWith; i < dsCfg.getWalSegments() && (p == null || p.apply(i)); i++) {
             File checkFile = new File(walWorkDir, FileDescriptor.fileName(i));
 
             if (checkFile.exists()) {
                 if (checkFile.isDirectory())
-                    throw new IgniteCheckedException("Failed to initialize WAL log segment (a directory with " +
+                    throw new StorageException("Failed to initialize WAL log segment (a directory with " +
                         "the same name already exists): " + checkFile.getAbsolutePath());
                 else if (checkFile.length() != dsCfg.getWalSegmentSize() && mode == WALMode.FSYNC)
-                    throw new IgniteCheckedException("Failed to initialize WAL log segment " +
+                    throw new StorageException("Failed to initialize WAL log segment " +
                         "(WAL segment size change is not supported in 'DEFAULT' WAL mode) " +
                         "[filePath=" + checkFile.getAbsolutePath() +
                         ", fileSize=" + checkFile.length() +
@@ -2651,9 +2658,8 @@ public void writeHeader() {
          * Flush or wait for concurrent flush completion.
          *
          * @param ptr Pointer.
-         * @throws IgniteCheckedException If failed.
          */
-        private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException {
+        private void flushOrWait(FileWALPointer ptr) {
             if (ptr != null) {
                 // If requested obsolete file index, it must be already flushed by close.
                 if (ptr.index() != idx)
@@ -2665,10 +2671,8 @@ private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException {
 
         /**
          * @param ptr Pointer.
-         * @throws IgniteCheckedException If failed.
-         * @throws StorageException If failed.
          */
-        private void flush(FileWALPointer ptr) throws IgniteCheckedException, StorageException {
+        private void flush(FileWALPointer ptr) {
             if (ptr == null) { // Unconditional flush.
                 walWriter.flushAll();
 
@@ -2884,7 +2888,7 @@ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageEx
                         }
                     }
                     catch (IOException e) {
-                        throw new IgniteCheckedException(e);
+                        throw new StorageException(e);
                     }
 
                     if (log.isDebugEnabled())
@@ -3365,28 +3369,29 @@ private void unparkWaiters(long pos) {
         /**
          * Forces all made changes to the file.
          */
-        void force() throws IgniteCheckedException {
+        void force() {
             flushBuffer(FILE_FORCE);
         }
 
         /**
          * Closes file.
          */
-        void close() throws IgniteCheckedException {
+        void close() {
             flushBuffer(FILE_CLOSE);
         }
 
         /**
          * Flushes all data from the buffer.
          */
-        void flushAll() throws IgniteCheckedException {
+        void flushAll() {
             flushBuffer(UNCONDITIONAL_FLUSH);
         }
 
         /**
          * @param expPos Expected position.
          */
-        void flushBuffer(long expPos) throws StorageException, IgniteCheckedException {
+        @SuppressWarnings("ForLoopReplaceableByForEach")
+        void flushBuffer(long expPos) {
             if (mmap)
                 return;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 49fbc73b9a48c..6f676fcf43284 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -487,37 +487,32 @@ private void checkWalConfiguration() throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
-        try {
-            assert currentHnd == null;
-            assert lastPtr == null || lastPtr instanceof FileWALPointer;
-
-            FileWALPointer filePtr = (FileWALPointer)lastPtr;
+        assert currentHnd == null;
+        assert lastPtr == null || lastPtr instanceof FileWALPointer;
 
-            currentHnd = restoreWriteHandle(filePtr);
+        FileWALPointer filePtr = (FileWALPointer)lastPtr;
 
-            if (currentHnd.serializer.version() != serializer.version()) {
-                if (log.isInfoEnabled())
-                    log.info("Record serializer version change detected, will start logging with a new WAL record " +
-                        "serializer to a new WAL segment [curFile=" + currentHnd + ", newVer=" + serializer.version() +
-                        ", oldVer=" + currentHnd.serializer.version() + ']');
+        currentHnd = restoreWriteHandle(filePtr);
 
-                rollOver(currentHnd);
-            }
+        if (currentHnd.serializer.version() != serializer.version()) {
+            if (log.isInfoEnabled())
+                log.info("Record serializer version change detected, will start logging with a new WAL record " +
+                    "serializer to a new WAL segment [curFile=" + currentHnd + ", newVer=" + serializer.version() +
+                    ", oldVer=" + currentHnd.serializer.version() + ']');
 
-            if (mode == WALMode.BACKGROUND) {
-                backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
-                    @Override public void run() {
-                        doFlush();
-                    }
-                }, flushFreq, flushFreq);
-            }
-
-            if (walAutoArchiveAfterInactivity > 0)
-                scheduleNextInactivityPeriodElapsedCheck();
+            rollOver(currentHnd);
         }
-        catch (StorageException e) {
-            throw new IgniteCheckedException(e);
+
+        if (mode == WALMode.BACKGROUND) {
+            backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
+                @Override public void run() {
+                    doFlush();
+                }
+            }, flushFreq, flushFreq);
         }
+
+        if (walAutoArchiveAfterInactivity > 0)
+            scheduleNextInactivityPeriodElapsedCheck();
     }
 
     /**
@@ -1019,7 +1014,7 @@ private FileWriteHandle currentHandle() {
      * @param cur Handle that failed to fit the given entry.
      * @return Handle that will fit the entry.
      */
-    private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, IgniteCheckedException {
+    private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, IgniteInterruptedCheckedException {
         FileWriteHandle hnd = currentHandle();
 
         if (hnd != cur)
@@ -1050,9 +1045,9 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, I
     /**
      * @param lastReadPtr Last read WAL file pointer.
      * @return Initialized file write handle.
-     * @throws IgniteCheckedException If failed to initialize WAL write handle.
+     * @throws StorageException If failed to initialize WAL write handle.
      */
-    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws IgniteCheckedException {
+    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws StorageException {
         long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
 
         long segNo = absIdx % dsCfg.getWalSegments();
@@ -1100,13 +1095,21 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
                 return hnd;
             }
             catch (IgniteCheckedException | IOException e) {
-                fileIO.close();
+                try {
+                    fileIO.close();
+                }
+                catch (IOException suppressed) {
+                    e.addSuppressed(suppressed);
+                }
 
-                throw e;
+                if (e instanceof StorageException)
+                    throw (StorageException) e;
+
+                throw e instanceof IOException ? (IOException) e : new IOException(e);
             }
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
+            throw new StorageException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
         }
     }
 
@@ -1118,9 +1121,9 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
      * @param curIdx current absolute segment released by WAL writer
      * @return Initialized file handle.
      * @throws StorageException If IO exception occurred.
-     * @throws IgniteCheckedException If failed.
+     * @throws IgniteInterruptedCheckedException If interrupted.
      */
-    private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException, IgniteCheckedException {
+    private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException, IgniteInterruptedCheckedException {
         try {
             File nextFile = pollNextFile(curIdx);
 
@@ -1150,9 +1153,11 @@ private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException
     }
 
     /**
-     * Deletes temp files, creates and prepares new; Creates first segment if necessary
+     * Deletes temp files, creates and prepares new; Creates first segment if necessary.
+     *
+     * @throws StorageException If failed.
      */
-    private void checkOrPrepareFiles() throws IgniteCheckedException {
+    private void checkOrPrepareFiles() throws StorageException {
         // Clean temp files.
         {
             File[] tmpFiles = walWorkDir.listFiles(WAL_SEGMENT_TEMP_FILE_FILTER);
@@ -1162,7 +1167,7 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
                     boolean deleted = tmp.delete();
 
                     if (!deleted)
-                        throw new IgniteCheckedException("Failed to delete previously created temp file " +
+                        throw new StorageException("Failed to delete previously created temp file " +
                             "(make sure Ignite process has enough rights): " + tmp.getAbsolutePath());
                 }
             }
@@ -1171,7 +1176,7 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
         File[] allFiles = walWorkDir.listFiles(WAL_SEGMENT_FILE_FILTER);
 
         if (allFiles.length != 0 && allFiles.length > dsCfg.getWalSegments())
-            throw new IgniteCheckedException("Failed to initialize wal (work directory contains " +
+            throw new StorageException("Failed to initialize wal (work directory contains " +
                 "incorrect number of segments) [cur=" + allFiles.length + ", expected=" + dsCfg.getWalSegments() + ']');
 
         // Allocate the first segment synchronously. All other segments will be allocated by archiver in background.
@@ -1188,9 +1193,9 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
      * Clears whole the file, fills with zeros for Default mode.
      *
      * @param file File to format.
-     * @throws IgniteCheckedException if formatting failed
+     * @throws StorageException if formatting failed.
      */
-    private void formatFile(File file) throws IgniteCheckedException {
+    private void formatFile(File file) throws StorageException {
         formatFile(file, dsCfg.getWalSegmentSize());
     }
 
@@ -1199,9 +1204,9 @@ private void formatFile(File file) throws IgniteCheckedException {
      *
      * @param file File to format.
      * @param bytesCntToFormat Count of first bytes to format.
-     * @throws IgniteCheckedException if formatting failed
+     * @throws StorageException If formatting failed.
      */
-    private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedException {
+    private void formatFile(File file, int bytesCntToFormat) throws StorageException {
         if (log.isDebugEnabled())
             log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1223,7 +1228,7 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                 fileIO.clear();
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
+            throw new StorageException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
         }
     }
 
@@ -1231,9 +1236,9 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
      * Creates a file atomically with temp file.
      *
      * @param file File to create.
-     * @throws IgniteCheckedException If failed.
+     * @throws StorageException If failed.
      */
-    private void createFile(File file) throws IgniteCheckedException {
+    private void createFile(File file) throws StorageException {
         if (log.isDebugEnabled())
             log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1245,7 +1250,7 @@ private void createFile(File file) throws IgniteCheckedException {
             Files.move(tmp.toPath(), file.toPath());
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to move temp file to a regular WAL segment file: " +
+            throw new StorageException("Failed to move temp file to a regular WAL segment file: " +
                 file.getAbsolutePath(), e);
         }
 
@@ -1259,9 +1264,10 @@ private void createFile(File file) throws IgniteCheckedException {
      *
      * @param curIdx Current absolute WAL segment index.
      * @return File ready for use as new WAL segment.
-     * @throws IgniteCheckedException If failed.
+     * @throws StorageException If exception occurred in the archiver thread.
+     * @throws IgniteInterruptedCheckedException If interrupted.
      */
-    private File pollNextFile(long curIdx) throws IgniteCheckedException {
+    private File pollNextFile(long curIdx) throws StorageException, IgniteInterruptedCheckedException {
         // Signal to archiver that we are done with the segment and it can be archived.
         long absNextIdx = archiver.nextAbsoluteSegmentIndex(curIdx);
 
@@ -1318,7 +1324,7 @@ private void checkNode() throws StorageException {
      */
     private class FileArchiver extends GridWorker {
         /** Exception which occurred during initial creation of files or during archiving WAL segment */
-        private IgniteCheckedException cleanException;
+        private StorageException cleanException;
 
         /**
          * Absolute current segment index WAL Manager writes to. Guarded by <code>this</code>.
@@ -1426,15 +1432,17 @@ private synchronized void release(long absIdx) {
             try {
                 allocateRemainingFiles();
             }
-            catch (IgniteCheckedException e) {
+            catch (StorageException e) {
                 synchronized (this) {
                     // Stop the thread and report to starter.
                     cleanException = e;
 
                     notifyAll();
-
-                    return;
                 }
+
+                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
+
+                return;
             }
 
             Throwable err = null;
@@ -1515,9 +1523,10 @@ private void changeLastArchivedIndexAndWakeupCompressor(long idx) {
          *
          * @param curIdx Current absolute index that we want to increment.
          * @return Next index (curWalSegmIdx+1) when it is ready to be written.
-         * @throws IgniteCheckedException If failed (if interrupted or if exception occurred in the archiver thread).
+         * @throws StorageException If exception occurred in the archiver thread.
+         * @throws IgniteInterruptedCheckedException If interrupted.
          */
-        private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException {
+        private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException, IgniteInterruptedCheckedException {
             try {
                 synchronized (this) {
                     if (cleanException != null)
@@ -1535,10 +1544,16 @@ private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException
                     while ((curAbsWalIdx - lastAbsArchivedIdx > segments && cleanException == null))
                         wait();
 
+                    if (cleanException != null)
+                        throw cleanException;
+
                     // Wait for formatter so that we do not open an empty file in DEFAULT mode.
-                    while (curAbsWalIdx % dsCfg.getWalSegments() > formatted)
+                    while (curAbsWalIdx % dsCfg.getWalSegments() > formatted && cleanException == null)
                         wait();
 
+                    if (cleanException != null)
+                        throw cleanException;
+
                     return curAbsWalIdx;
                 }
             }
@@ -1664,7 +1679,7 @@ private boolean checkStop() {
          * Background creation of all segments except first. First segment was created in main thread by
          * {@link FsyncModeFileWriteAheadLogManager#checkOrPrepareFiles()}
          */
-        private void allocateRemainingFiles() throws IgniteCheckedException {
+        private void allocateRemainingFiles() throws StorageException {
             final FileArchiver archiver = this;
 
             checkFiles(1,
@@ -2029,23 +2044,23 @@ private void shutdown() {
      * @param startWith Start with.
      * @param create Flag create file.
      * @param p Predicate Exit condition.
-     * @throws IgniteCheckedException if validation or create file fail.
+     * @throws StorageException if validation or create file fail.
      */
     private void checkFiles(
         int startWith,
         boolean create,
         @Nullable IgnitePredicate<Integer> p,
         @Nullable IgniteInClosure<Integer> completionCallback
-    ) throws IgniteCheckedException {
+    ) throws StorageException {
         for (int i = startWith; i < dsCfg.getWalSegments() && (p == null || (p != null && p.apply(i))); i++) {
             File checkFile = new File(walWorkDir, FileDescriptor.fileName(i));
 
             if (checkFile.exists()) {
                 if (checkFile.isDirectory())
-                    throw new IgniteCheckedException("Failed to initialize WAL log segment (a directory with " +
+                    throw new StorageException("Failed to initialize WAL log segment (a directory with " +
                         "the same name already exists): " + checkFile.getAbsolutePath());
                 else if (checkFile.length() != dsCfg.getWalSegmentSize() && mode == WALMode.FSYNC)
-                    throw new IgniteCheckedException("Failed to initialize WAL log segment " +
+                    throw new StorageException("Failed to initialize WAL log segment " +
                         "(WAL segment size change is not supported):" + checkFile.getAbsolutePath());
             }
             else if (create)
@@ -2408,9 +2423,9 @@ private FileWriteHandle(
          * Write serializer version to current handle.
          * NOTE: Method mutates {@code fileIO} position, written and lastFsyncPos fields.
          *
-         * @throws IOException If fail to write serializer version.
+         * @throws StorageException If fail to write serializer version.
          */
-        public void writeSerializerVersion() throws IOException {
+        private void writeSerializerVersion() throws StorageException {
             try {
                 assert fileIO.position() == 0 : "Serializer version can be written only at the begin of file " +
                     fileIO.position();
@@ -2423,7 +2438,7 @@ public void writeSerializerVersion() throws IOException {
                 head.set(new FakeRecord(new FileWALPointer(idx, (int)updatedPosition, 0), false));
             }
             catch (IOException e) {
-                throw new IOException("Unable to write serializer version for segment " + idx, e);
+                throw new StorageException("Unable to write serializer version for segment " + idx, e);
             }
         }
 
@@ -2448,9 +2463,8 @@ private boolean stopped(WALRecord record) {
          * @param rec Record to be added to record chain as new {@link #head}
          * @return Pointer or null if roll over to next segment is required or already started by other thread.
          * @throws StorageException If failed.
-         * @throws IgniteCheckedException If failed.
          */
-        @Nullable private WALPointer addRecord(WALRecord rec) throws StorageException, IgniteCheckedException {
+        @Nullable private WALPointer addRecord(WALRecord rec) throws StorageException {
             assert rec.size() > 0 || rec.getClass() == FakeRecord.class;
 
             boolean flushed = false;
@@ -2503,9 +2517,9 @@ private long nextPosition(WALRecord rec) {
          * Flush or wait for concurrent flush completion.
          *
          * @param ptr Pointer.
-         * @throws IgniteCheckedException If failed.
+         * @throws StorageException If failed.
          */
-        private void flushOrWait(FileWALPointer ptr, boolean stop) throws IgniteCheckedException {
+        private void flushOrWait(FileWALPointer ptr, boolean stop) throws StorageException {
             long expWritten;
 
             if (ptr != null) {
@@ -2549,10 +2563,9 @@ else if (stop) {
         /**
          * @param ptr Pointer.
          * @return {@code true} If the flush really happened.
-         * @throws IgniteCheckedException If failed.
          * @throws StorageException If failed.
          */
-        private boolean flush(FileWALPointer ptr, boolean stop) throws IgniteCheckedException, StorageException {
+        private boolean flush(FileWALPointer ptr, boolean stop) throws StorageException {
             if (ptr == null) { // Unconditional flush.
                 for (; ; ) {
                     WALRecord expHead = head.get();
@@ -2594,10 +2607,9 @@ private long chainBeginPosition(WALRecord h) {
 
         /**
          * @param expHead Expected head of chain. If head was changed, flush is not performed in this thread
-         * @throws IgniteCheckedException If failed.
          * @throws StorageException If failed.
          */
-        private boolean flush(WALRecord expHead, boolean stop) throws StorageException, IgniteCheckedException {
+        private boolean flush(WALRecord expHead, boolean stop) throws StorageException {
             if (expHead.previous() == null) {
                 FakeRecord frHead = (FakeRecord)expHead;
 
@@ -2643,7 +2655,8 @@ private boolean flush(WALRecord expHead, boolean stop) throws StorageException,
                 return true;
             }
             catch (Throwable e) {
-                StorageException se = new StorageException("Unable to write", new IOException(e));
+                StorageException se = e instanceof StorageException ? (StorageException) e :
+                    new StorageException("Unable to write", new IOException(e));
 
                 cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
 
@@ -2725,8 +2738,9 @@ private FileWALPointer position() {
         /**
          * @param ptr Pointer to sync.
          * @throws StorageException If failed.
+         * @throws IgniteInterruptedCheckedException If interrupted.
          */
-        private void fsync(FileWALPointer ptr, boolean stop) throws StorageException, IgniteCheckedException {
+        private void fsync(FileWALPointer ptr, boolean stop) throws StorageException, IgniteInterruptedCheckedException {
             lock.lock();
 
             try {
@@ -2780,10 +2794,9 @@ private void fsync(FileWALPointer ptr, boolean stop) throws StorageException, Ig
 
         /**
          * @return {@code true} If this thread actually closed the segment.
-         * @throws IgniteCheckedException If failed.
          * @throws StorageException If failed.
          */
-        private boolean close(boolean rollOver) throws IgniteCheckedException, StorageException {
+        private boolean close(boolean rollOver) throws StorageException {
             if (stop.compareAndSet(false, true)) {
                 lock.lock();
 
@@ -2793,43 +2806,49 @@ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageEx
                     assert stopped() : "Segment is not closed after close flush: " + head.get();
 
                     try {
-                        RecordSerializer backwardSerializer = new RecordSerializerFactoryImpl(cctx)
-                            .createSerializer(serializerVersion);
+                        try {
+                            RecordSerializer backwardSerializer = new RecordSerializerFactoryImpl(cctx)
+                                .createSerializer(serializerVersion);
 
-                        SwitchSegmentRecord segmentRecord = new SwitchSegmentRecord();
+                            SwitchSegmentRecord segmentRecord = new SwitchSegmentRecord();
 
-                        int switchSegmentRecSize = backwardSerializer.size(segmentRecord);
+                            int switchSegmentRecSize = backwardSerializer.size(segmentRecord);
 
-                        if (rollOver && written < (maxSegmentSize - switchSegmentRecSize)) {
-                            final ByteBuffer buf = ByteBuffer.allocate(switchSegmentRecSize);
+                            if (rollOver && written < (maxSegmentSize - switchSegmentRecSize)) {
+                                final ByteBuffer buf = ByteBuffer.allocate(switchSegmentRecSize);
 
-                            segmentRecord.position(new FileWALPointer(idx, (int)written, switchSegmentRecSize));
-                            backwardSerializer.writeRecord(segmentRecord, buf);
+                                segmentRecord.position(new FileWALPointer(idx, (int)written, switchSegmentRecSize));
+                                backwardSerializer.writeRecord(segmentRecord, buf);
 
-                            buf.rewind();
+                                buf.rewind();
 
-                            int rem = buf.remaining();
+                                int rem = buf.remaining();
 
-                            while (rem > 0) {
-                                int written0 = fileIO.write(buf, written);
+                                while (rem > 0) {
+                                    int written0 = fileIO.write(buf, written);
 
-                                written += written0;
+                                    written += written0;
 
-                                rem -= written0;
+                                    rem -= written0;
+                                }
                             }
                         }
+                        catch (IgniteCheckedException e) {
+                            throw new IOException(e);
+                        }
+                        finally {
+                            // Do the final fsync.
+                            if (mode == WALMode.FSYNC) {
+                                fileIO.force();
 
-                        // Do the final fsync.
-                        if (mode == WALMode.FSYNC) {
-                            fileIO.force();
+                                lastFsyncPos = written;
+                            }
 
-                            lastFsyncPos = written;
+                            fileIO.close();
                         }
-
-                        fileIO.close();
                     }
                     catch (IOException e) {
-                        throw new IgniteCheckedException(e);
+                        throw new StorageException(e);
                     }
 
                     if (log.isDebugEnabled())
@@ -2872,9 +2891,9 @@ private void signalNextAvailable() {
         }
 
         /**
-         * @throws IgniteCheckedException If failed.
+         *
          */
-        private void awaitNext() throws IgniteCheckedException {
+        private void awaitNext() {
             lock.lock();
 
             try {
@@ -2894,7 +2913,7 @@ private void awaitNext() throws IgniteCheckedException {
          * @throws IgniteCheckedException If failed.
          */
         @SuppressWarnings("TooBroadScope")
-        private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, IgniteCheckedException {
+        private void writeBuffer(long pos, ByteBuffer buf) throws StorageException {
             boolean interrupted = false;
 
             lock.lock();
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java b/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
index 1159683e6b54f..545c9ea1176d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
+++ b/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.failure;
 
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import org.apache.ignite.Ignite;
 
 /**
@@ -33,6 +34,13 @@ public class TestFailureHandler implements FailureHandler {
     /** Failure context. */
     volatile FailureContext failureCtx;
 
+    /**
+     * @param invalidate Invalidate.
+     */
+    public TestFailureHandler(boolean invalidate) {
+        this(invalidate, new CountDownLatch(1));
+    }
+
     /**
      * @param invalidate Invalidate.
      * @param latch Latch.
@@ -60,4 +68,15 @@ public TestFailureHandler(boolean invalidate, CountDownLatch latch) {
     public FailureContext failureContext() {
         return failureCtx;
     }
+
+    /**
+     * @param millis Millis.
+
+     * @return Failure context.
+     */
+    public FailureContext awaitFailure(long millis) throws InterruptedException {
+        latch.await(millis, TimeUnit.MILLISECONDS);
+
+        return failureCtx;
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
new file mode 100644
index 0000000000000..379b8c32cda89
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.OpenOption;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.failure.TestFailureHandler;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+/**
+ *
+ */
+public class IgniteWalFormatFileFailoverTest extends GridCommonAbstractTest {
+    /** */
+    private static final String TEST_CACHE = "testCache";
+
+    /** */
+    private static final String formatFile = "formatFile";
+
+    /** Fail method name reference. */
+    private final AtomicReference<String> failMtdNameRef = new AtomicReference<>();
+
+    /** */
+    private boolean fsync;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCacheConfiguration(new CacheConfiguration(TEST_CACHE)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL));
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(2048L * 1024 * 1024)
+                .setPersistenceEnabled(true))
+            .setWalMode(fsync ? WALMode.FSYNC : WALMode.BACKGROUND)
+            .setWalBufferSize(1024 * 1024)
+            .setWalSegmentSize(512 * 1024)
+            .setFileIOFactory(new FailingFileIOFactory(failMtdNameRef));
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        cfg.setFailureHandler(new TestFailureHandler(false));
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNodeStartFailedFsync() throws Exception {
+        fsync = true;
+
+        failMtdNameRef.set(formatFile);
+
+        checkCause(GridTestUtils.assertThrows(log, () -> startGrid(0), IgniteCheckedException.class, null));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testFailureHandlerTriggeredFsync() throws Exception {
+        fsync = true;
+
+        failFormatFileOnClusterActivate();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testFailureHandlerTriggered() throws Exception {
+        fsync = false;
+
+        failFormatFileOnClusterActivate();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void failFormatFileOnClusterActivate() throws Exception {
+        failMtdNameRef.set(null);
+
+        startGrid(0);
+        startGrid(1);
+
+        if (!fsync) {
+            setFileIOFactory(grid(0).context().cache().context().wal());
+            setFileIOFactory(grid(1).context().cache().context().wal());
+        }
+
+        failMtdNameRef.set(formatFile);
+
+        grid(0).cluster().active(true);
+
+        checkCause(failureHandler(0).awaitFailure(2000).error());
+        checkCause(failureHandler(1).awaitFailure(2000).error());
+    }
+
+    /**
+     * @param mtdName Method name.
+     */
+    private static boolean isCalledFrom(String mtdName) {
+        return isCalledFrom(Thread.currentThread().getStackTrace(), mtdName);
+    }
+
+    /**
+     * @param stackTrace Stack trace.
+     * @param mtdName Method name.
+     */
+    private static boolean isCalledFrom(StackTraceElement[] stackTrace, String mtdName) {
+        return Arrays.stream(stackTrace).map(StackTraceElement::getMethodName).anyMatch(mtdName::equals);
+    }
+
+    /**
+     * @param gridIdx Grid index.
+     * @return Failure handler configured for grid with given index.
+     */
+    private TestFailureHandler failureHandler(int gridIdx) {
+        FailureHandler hnd = grid(gridIdx).configuration().getFailureHandler();
+
+        assertTrue(hnd instanceof TestFailureHandler);
+
+        return (TestFailureHandler)hnd;
+    }
+
+    /**
+     * @param t Throwable.
+     */
+    private void checkCause(Throwable t) {
+        StorageException e = X.cause(t, StorageException.class);
+
+        assertNotNull(e);
+        assertNotNull(e.getMessage());
+        assertTrue(e.getMessage().contains("Failed to format WAL segment file"));
+
+        IOException ioe = X.cause(e, IOException.class);
+
+        assertNotNull(ioe);
+        assertNotNull(ioe.getMessage());
+        assertTrue(ioe.getMessage().contains("No space left on device"));
+
+        assertTrue(isCalledFrom(ioe.getStackTrace(), formatFile));
+    }
+
+    /** */
+    private void setFileIOFactory(IgniteWriteAheadLogManager wal) {
+        if (wal instanceof FileWriteAheadLogManager)
+            ((FileWriteAheadLogManager)wal).setFileIOFactory(new FailingFileIOFactory(failMtdNameRef));
+        else
+            fail(wal.getClass().toString());
+    }
+
+    /**
+     * Create File I/O which fails if specific method call present in stack trace.
+     */
+    private static class FailingFileIOFactory implements FileIOFactory {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** Delegate factory. */
+        private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
+
+        /** Fail method name reference. */
+        private final AtomicReference<String> failMtdNameRef;
+
+        /**
+         * @param failMtdNameRef Fail method name reference.
+         */
+        FailingFileIOFactory(AtomicReference<String> failMtdNameRef) {
+            assertNotNull(failMtdNameRef);
+
+            this.failMtdNameRef = failMtdNameRef;
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            return create(file, CREATE, READ, WRITE);
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+            final FileIO delegate = delegateFactory.create(file, modes);
+
+            return new FileIODecorator(delegate) {
+                @Override public int write(byte[] buf, int off, int len) throws IOException {
+                    conditionalFail();
+
+                    return super.write(buf, off, len);
+                }
+
+                @Override public void clear() throws IOException {
+                    conditionalFail();
+
+                    super.clear();
+                }
+
+                private void conditionalFail() throws IOException {
+                    String failMtdName = failMtdNameRef.get();
+
+                    if (failMtdName != null && isCalledFrom(failMtdName))
+                        throw new IOException("No space left on device");
+                }
+            };
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index c5a381eb6e476..7732cee672799 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -49,6 +49,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncWithMmapBufferSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlySelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlyWithMmapBufferSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFormatFileFailoverTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorSwitchSegmentTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalSerializerVersionTest;
@@ -147,6 +148,8 @@ public static void addRealPageStoreTests(TestSuite suite) {
 
         suite.addTestSuite(IgniteWalFlushLogOnlyWithMmapBufferSelfTest.class);
 
+        suite.addTestSuite(IgniteWalFormatFileFailoverTest.class);
+
         // Test suite uses Standalone WAL iterator to verify PDS content.
         suite.addTestSuite(IgniteWalReaderTest.class);
 

From 3d355ab0f0b75d010604775bb7f5458732087a4d Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Mon, 18 Jun 2018 18:53:43 +0300
Subject: [PATCH 208/543] IGNITE-8707 DataStorageMetrics.getTotalAllocatedSize
 metric does not account reserved partition page header - Fixes #4202.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit ceade87)
---
 .../cache/persistence/file/FilePageStore.java |  8 +-
 .../db/IgnitePdsDataRegionMetricsTest.java    | 95 ++++++++++---------
 2 files changed, 50 insertions(+), 53 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 852eb0da870b1..f614032c58b15 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -279,7 +279,7 @@ public void truncate(int tag) throws PersistentStorageIOException {
             throw new PersistentStorageIOException("Failed to delete partition file: " + cfgFile.getPath(), e);
         }
         finally {
-            allocatedTracker.updateTotalAllocatedPages(-1L * pages());
+            allocatedTracker.updateTotalAllocatedPages(-1L * allocated.get() / pageSize);
 
             allocated.set(0);
 
@@ -440,11 +440,7 @@ private void init() throws PersistentStorageIOException {
 
                         assert allocated.get() == 0;
 
-                        long delta = newSize - headerSize();
-
-                        assert delta % pageSize == 0;
-
-                        allocatedTracker.updateTotalAllocatedPages(delta / pageSize);
+                        allocatedTracker.updateTotalAllocatedPages(newSize / pageSize);
 
                         allocated.set(newSize);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
index 18a47814755d1..268d2fb53dda3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
@@ -17,13 +17,14 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.db;
 
+import java.io.File;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Path;
+import java.util.HashMap;
 import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
 import java.util.Random;
-import java.util.HashMap;
+import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
-import java.util.stream.Collectors;
 import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
@@ -34,20 +35,19 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.junit.Assert;
 
+import static java.nio.file.Files.newDirectoryStream;
 import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
 
 /**
@@ -136,9 +136,6 @@ public void testMemoryUsageSingleNode() throws Exception {
 
             final IgniteCache<String, String> cache = node.getOrCreateCache(DEFAULT_CACHE_NAME);
 
-            final Set<Integer> grpIds = node.context().cache().cacheGroups()
-                .stream().map(CacheGroupContext::groupId).collect(Collectors.toSet());
-
             Map<String, String> map = new HashMap<>();
 
             for (int batch = 0; batch < BATCHES; batch++) {
@@ -149,7 +146,9 @@ public void testMemoryUsageSingleNode() throws Exception {
 
                 cache.putAll(map);
 
-                checkMetricsConsistency(node, grpIds);
+                forceCheckpoint();
+
+                checkMetricsConsistency(node, DEFAULT_CACHE_NAME);
             }
 
             currMetrics = getDfltRegionMetrics(node);
@@ -169,22 +168,21 @@ public void testMemoryUsageMultipleNodes() throws Exception {
 
         node0.cluster().active(true);
 
-        final IgniteCache<String, String> cache = node0.getOrCreateCache(DEFAULT_CACHE_NAME);
+        final IgniteCache<Integer, String> cache = node0.getOrCreateCache(DEFAULT_CACHE_NAME);
 
-        final Set<Integer> grpIds = node0.context().cache().cacheGroups()
-            .stream().map(CacheGroupContext::groupId).collect(Collectors.toSet());
-
-        Map<String, String> map = new HashMap<>();
+        Map<Integer, String> map = new HashMap<>();
 
         for (int i = 0; i < 10_000; i++)
-            map.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
+            map.put(i, UUID.randomUUID().toString());
 
         cache.putAll(map);
 
         awaitPartitionMapExchange(true, true, null);
 
-        checkMetricsConsistency(node0, grpIds);
-        checkMetricsConsistency(node1, grpIds);
+        forceCheckpoint();
+
+        checkMetricsConsistency(node0, DEFAULT_CACHE_NAME);
+        checkMetricsConsistency(node1, DEFAULT_CACHE_NAME);
 
         IgniteEx node2 = startGrid(2);
 
@@ -192,9 +190,11 @@ public void testMemoryUsageMultipleNodes() throws Exception {
 
         awaitPartitionMapExchange(true, true, null);
 
-        checkMetricsConsistency(node0, grpIds);
-        checkMetricsConsistency(node1, grpIds);
-        checkMetricsConsistency(node2, grpIds);
+        forceCheckpoint();
+
+        checkMetricsConsistency(node0, DEFAULT_CACHE_NAME);
+        checkMetricsConsistency(node1, DEFAULT_CACHE_NAME);
+        checkMetricsConsistency(node2, DEFAULT_CACHE_NAME);
 
         stopGrid(1, true);
 
@@ -202,8 +202,10 @@ public void testMemoryUsageMultipleNodes() throws Exception {
 
         awaitPartitionMapExchange(true, true, null);
 
-        checkMetricsConsistency(node0, grpIds);
-        checkMetricsConsistency(node2, grpIds);
+        forceCheckpoint();
+
+        checkMetricsConsistency(node0, DEFAULT_CACHE_NAME);
+        checkMetricsConsistency(node2, DEFAULT_CACHE_NAME);
     }
 
     /**
@@ -289,29 +291,28 @@ private static DataRegionMetrics getDfltRegionMetrics(Ignite node) {
     }
 
     /** */
-    private static void checkMetricsConsistency(
-        final IgniteEx node,
-        final Set<Integer> grpIds) throws Exception {
-        boolean storageMatches = GridTestUtils.waitForCondition((PA)() -> {
-            long pagesInStore = 0;
-            long allocated = 0;
-
-            for (int grpId : grpIds) {
-                DataRegion region = node.context().cache().cacheGroup(grpId).dataRegion();
-
-                if (!region.config().isMetricsEnabled())
-                    continue;
-
-                pagesInStore += node.context().cache().context().pageStore().pagesAllocated(grpId);
-                allocated += region.memoryMetrics().getTotalAllocatedPages();
-            }
-
-            assert 0 != pagesInStore;
-            assert 0 != allocated;
+    private void checkMetricsConsistency(final IgniteEx node, String cacheName) throws Exception {
+        FilePageStoreManager pageStoreManager = (FilePageStoreManager)node.context().cache().context().pageStore();
+
+        long totalPersistanceSize = 0;
+        File cacheWorkDir = pageStoreManager.cacheWorkDir(
+            node.getOrCreateCache(cacheName).getConfiguration(CacheConfiguration.class)
+        );
+
+        try (DirectoryStream<Path> files = newDirectoryStream(
+            cacheWorkDir.toPath(), entry -> entry.toFile().getName().endsWith(".bin"))
+        ) {
+            for (Path path : files)
+                totalPersistanceSize += path.toFile().length();
+        }
 
-            return allocated == pagesInStore;
-        }, 1000);
+        long totalAllocatedPagesFromMetrics = node.context().cache().context()
+            .cacheContext(CU.cacheId(DEFAULT_CACHE_NAME))
+            .group()
+            .dataRegion()
+            .memoryMetrics()
+            .getTotalAllocatedPages();
 
-        assertTrue(storageMatches);
+        assertEquals(totalPersistanceSize / pageStoreManager.pageSize(), totalAllocatedPagesFromMetrics);
     }
 }

From db84d6b81db717b39ef41df7101789856ad83645 Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Mon, 18 Jun 2018 18:48:05 +0300
Subject: [PATCH 209/543] IGNITE-8815 Ease enabling WAL management in
 control.sh - Fixes #4212.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit b80482d125d08a4779c8f7b1ea75e4dd1a4c4bde)
---
 bin/control.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/bin/control.sh b/bin/control.sh
index e2b75afbd1a07..7f84696831c50 100755
--- a/bin/control.sh
+++ b/bin/control.sh
@@ -92,6 +92,11 @@ if [ -z "$JVM_OPTS" ] ; then
     fi
 fi
 
+#
+# Uncomment to enable experimental commands [--wal]
+#
+# JVM_OPTS="${JVM_OPTS} -DIGNITE_ENABLE_EXPERIMENTAL_COMMAND=true"
+
 #
 # Uncomment the following GC settings if you see spikes in your throughput due to Garbage Collection.
 #

From 93d10329458a2d990fbcebe44ff1f2fbf2692382 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Fri, 15 Jun 2018 11:43:01 +0300
Subject: [PATCH 210/543] IGNITE-8800: Binary: print type name and existing
 schema IDs in case of exception due to missing schema. This closes #4190.

(cherry picked from commit 7a39efb)
---
 .../internal/binary/BinaryReaderExImpl.java   | 31 ++++++++++++++-----
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
index 2b7964c609a3d..f88e3c3234b41 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
@@ -23,8 +23,10 @@
 import java.math.BigDecimal;
 import java.sql.Time;
 import java.sql.Timestamp;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
+import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.binary.BinaryCollectionFactory;
@@ -2004,21 +2006,36 @@ public BinarySchema getOrCreateSchema() {
             if (fieldIdLen != BinaryUtils.FIELD_ID_LEN) {
                 BinaryTypeImpl type = (BinaryTypeImpl) ctx.metadata(typeId, schemaId);
 
-                if (type == null || type.metadata() == null)
+                BinaryMetadata meta = type != null ? type.metadata() : null;
+
+                if (type == null || meta == null)
                     throw new BinaryObjectException("Cannot find metadata for object with compact footer: " +
                         typeId);
 
-                for (BinarySchema typeSchema : type.metadata().schemas()) {
-                    if (schemaId == typeSchema.schemaId()) {
-                        schema = typeSchema;
+                Collection<BinarySchema> existingSchemas = meta.schemas();
+
+                for (BinarySchema existingSchema : existingSchemas) {
+                    if (schemaId == existingSchema.schemaId()) {
+                        schema = existingSchema;
 
                         break;
                     }
                 }
 
-                if (schema == null)
-                    throw new BinaryObjectException("Cannot find schema for object with compact footer [" +
-                        "typeId=" + typeId + ", schemaId=" + schemaId + ']');
+                if (schema == null) {
+                    List<Integer> existingSchemaIds = new ArrayList<>(existingSchemas.size());
+
+                    for (BinarySchema existingSchema : existingSchemas)
+                        existingSchemaIds.add(existingSchema.schemaId());
+
+
+                    throw new BinaryObjectException("Cannot find schema for object with compact footer" +
+                        " [typeName=" + type.typeName() +
+                        ", typeId=" + typeId +
+                        ", missingSchemaId=" + schemaId +
+                        ", existingSchemaIds=" + existingSchemaIds + ']'
+                    );
+                }
             }
             else
                 schema = createSchema();

From d13a7bef47a05e1fc2e2aafd0479916f728285cd Mon Sep 17 00:00:00 2001
From: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Date: Mon, 18 Jun 2018 19:11:32 +0300
Subject: [PATCH 211/543] IGNITE-8601 Add to control.sh utility information
 about transaction start time - Fixes #4070.

(cherry picked from commit 4776a1a)
---
 .../internal/TransactionsMXBeanImpl.java      |  2 +-
 .../internal/commandline/CommandHandler.java  |  6 ++-
 .../ignite/internal/visor/tx/VisorTxInfo.java | 37 +++++++++++++++--
 .../internal/visor/tx/VisorTxSortOrder.java   |  8 +++-
 .../ignite/internal/visor/tx/VisorTxTask.java | 20 ++++++++-
 .../ignite/util/GridCommandHandlerTest.java   | 41 +++++++++++--------
 6 files changed, 89 insertions(+), 25 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
index 6937ebd146c4a..210a32008d313 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
@@ -25,7 +25,6 @@
 import java.util.UUID;
 import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCompute;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -107,6 +106,7 @@ else if ("SIZE".equals(order))
                         w.println("    Tx: [xid=" + info.getXid() +
                             ", label=" + info.getLabel() +
                             ", state=" + info.getState() +
+                            ", startTime=" + info.getFormattedStartTime() +
                             ", duration=" + info.getDuration() / 1000 +
                             ", isolation=" + info.getIsolation() +
                             ", concurrency=" + info.getConcurrency() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index c59e348f768ae..3a26537260f54 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -229,6 +229,9 @@ public class CommandHandler {
     /** */
     private static final String TX_ORDER = "order";
 
+    /** */
+    public static final String CMD_TX_ORDER_START_TIME="START_TIME";
+
     /** */
     private static final String TX_SERVERS = "servers";
 
@@ -984,6 +987,7 @@ else if (arg.getOperation() == VisorTxOperation.KILL)
                     log("    Tx: [xid=" + info.getXid() +
                         ", label=" + info.getLabel() +
                         ", state=" + info.getState() +
+                        ", startTime=" + info.getFormattedStartTime() +
                         ", duration=" + info.getDuration() / 1000 +
                         ", isolation=" + info.getIsolation() +
                         ", concurrency=" + info.getConcurrency() +
@@ -1731,7 +1735,7 @@ public int execute(List<String> rawArgs) {
                 usage("  Set baseline topology based on version:", BASELINE, " version topologyVersion [--force]");
                 usage("  List or kill transactions:", TX, " [xid XID] [minDuration SECONDS] " +
                     "[minSize SIZE] [label PATTERN_REGEX] [servers|clients] " +
-                    "[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE] [kill] [--force]");
+                    "[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE|", CMD_TX_ORDER_START_TIME, "] [kill] [--force]");
 
                 if(enableExperimental) {
                     usage("  Print absolute paths of unused archived wal segments on each node:", WAL,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
index 89bf274653165..ecf3e0d79fb4c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
@@ -20,8 +20,11 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
-import java.io.Serializable;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.Collection;
+import java.util.TimeZone;
 import java.util.UUID;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -37,9 +40,17 @@ public class VisorTxInfo extends VisorDataTransferObject {
     /** */
     private static final long serialVersionUID = 0L;
 
+    /** */
+    private static DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
+
     /** */
     private IgniteUuid xid;
 
+    /**
+     * Transaction start time.
+     */
+    private long startTime;
+
     /** */
     private long duration;
 
@@ -73,6 +84,7 @@ public VisorTxInfo() {
 
     /**
      * @param xid Xid.
+     * @param startTime Start time of transaction.
      * @param duration Duration.
      * @param isolation Isolation.
      * @param concurrency Concurrency.
@@ -82,10 +94,11 @@ public VisorTxInfo() {
      * @param state State.
      * @param size Size.
      */
-    public VisorTxInfo(IgniteUuid xid, long duration, TransactionIsolation isolation,
+    public VisorTxInfo(IgniteUuid xid, long startTime, long duration, TransactionIsolation isolation,
         TransactionConcurrency concurrency, long timeout, String lb, Collection<UUID> primaryNodes,
         TransactionState state, int size) {
         this.xid = xid;
+        this.startTime = startTime;
         this.duration = duration;
         this.isolation = isolation;
         this.concurrency = concurrency;
@@ -101,6 +114,16 @@ public IgniteUuid getXid() {
         return xid;
     }
 
+    /** */
+    public long getStartTime() {
+        return startTime;
+    }
+
+    /** */
+    public String getFormattedStartTime() {
+        return dateTimeFormatter.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(startTime), TimeZone.getDefault().toZoneId()));
+    }
+
     /** */
     public long getDuration() {
         return duration;
@@ -141,6 +164,11 @@ public int getSize() {
         return size;
     }
 
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeGridUuid(out, xid);
@@ -152,10 +180,12 @@ public int getSize() {
         U.writeCollection(out, primaryNodes);
         U.writeEnum(out, state);
         out.writeInt(size);
+        out.writeLong(startTime);
     }
 
     /** {@inheritDoc} */
-    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+    @Override protected void readExternalData(byte protoVer,
+        ObjectInput in) throws IOException, ClassNotFoundException {
         xid = U.readGridUuid(in);
         duration = in.readLong();
         isolation = TransactionIsolation.fromOrdinal(in.readByte());
@@ -165,6 +195,7 @@ public int getSize() {
         primaryNodes = U.readCollection(in);
         state = TransactionState.fromOrdinal(in.readByte());
         size = in.readInt();
+        startTime = protoVer >= V2 ? in.readLong() : 0L;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxSortOrder.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxSortOrder.java
index 382cf9152207b..9a18882c24958 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxSortOrder.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxSortOrder.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.visor.tx;
 
+import org.apache.ignite.internal.commandline.CommandHandler;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -25,7 +26,9 @@ public enum VisorTxSortOrder {
     /** Sort by duration. */
     DURATION,
     /** Sort by size. */
-    SIZE;
+    SIZE,
+    /** Sort by startTime */
+    START_TIME;
 
     /** Enumerated values. */
     private static final VisorTxSortOrder[] VALS = values();
@@ -50,6 +53,9 @@ public static VisorTxSortOrder fromString(String name) {
         if (SIZE.toString().equals(name))
             return SIZE;
 
+        if (CommandHandler.CMD_TX_ORDER_START_TIME.equals(name))
+            return START_TIME;
+
         throw new IllegalArgumentException("Sort order is unknown: " + name);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
index 579abbe9261a9..25a69d1d67e06 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
@@ -194,7 +194,7 @@ private VisorTxJob(VisorTxTaskArg arg, boolean debug) {
                 if (arg.getMinSize() != null && size < arg.getMinSize())
                     continue;
 
-                infos.add(new VisorTxInfo(locTx.xid(), duration, locTx.isolation(), locTx.concurrency(),
+                infos.add(new VisorTxInfo(locTx.xid(), locTx.startTime(), duration, locTx.isolation(), locTx.concurrency(),
                     locTx.timeout(), locTx.label(), mappings, locTx.state(), size));
 
                 if (arg.getOperation() == VisorTxOperation.KILL)
@@ -218,6 +218,11 @@ private VisorTxJob(VisorTxTaskArg arg, boolean debug) {
 
                         break;
 
+                    case START_TIME:
+                        comp = TxStartTimeComparator.INSTANCE;
+
+                        break;
+
                     default:
                 }
             }
@@ -228,6 +233,19 @@ private VisorTxJob(VisorTxTaskArg arg, boolean debug) {
         }
     }
 
+    /**
+     *
+     */
+    private static class TxStartTimeComparator implements Comparator<VisorTxInfo> {
+        /** Instance. */
+        public static final TxStartTimeComparator INSTANCE = new TxStartTimeComparator();
+
+        /** {@inheritDoc} */
+        @Override public int compare(VisorTxInfo o1, VisorTxInfo o2) {
+            return Long.compare(o2.getStartTime(), o1.getStartTime());
+        }
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 3ec0617e9164a..30d80f65d212e 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -449,16 +449,13 @@ else if (entry.getKey().equals(node2)) {
         validate(h, map -> {
             VisorTxTaskResult res = map.get(grid(0).localNode());
 
-            for (VisorTxInfo info:res.getInfos()){
+            for (VisorTxInfo info : res.getInfos())
                 assertNull(info.getLabel());
 
-            }
-
         }, "--tx", "label", "null");
 
-
         // test check minSize
-        int minSize=10;
+        int minSize = 10;
 
         validate(h, map -> {
             VisorTxTaskResult res = map.get(grid(0).localNode());
@@ -475,7 +472,7 @@ else if (entry.getKey().equals(node2)) {
         validate(h, map -> {
             VisorTxTaskResult res = map.get(grid(0).localNode());
 
-            assertTrue(res.getInfos().get(0).getSize() >=  res.getInfos().get(1).getSize());
+            assertTrue(res.getInfos().get(0).getSize() >= res.getInfos().get(1).getSize());
 
         }, "--tx", "order", "SIZE");
 
@@ -483,10 +480,18 @@ else if (entry.getKey().equals(node2)) {
         validate(h, map -> {
             VisorTxTaskResult res = map.get(grid(0).localNode());
 
-            assertTrue(res.getInfos().get(0).getDuration() >=  res.getInfos().get(1).getDuration());
+            assertTrue(res.getInfos().get(0).getDuration() >= res.getInfos().get(1).getDuration());
 
         }, "--tx", "order", "DURATION");
 
+        // test order by start_time.
+        validate(h, map -> {
+            VisorTxTaskResult res = map.get(grid(0).localNode());
+
+            for (int i = res.getInfos().size() - 1; i > 1; i--)
+                assertTrue(res.getInfos().get(i - 1).getStartTime() >= res.getInfos().get(i).getStartTime());
+        }, "--tx", "order", CommandHandler.CMD_TX_ORDER_START_TIME);
+
         // Trigger topology change and test connection.
         IgniteInternalFuture<?> startFut = multithreadedAsync(() -> {
             try {
@@ -777,16 +782,16 @@ private void validate(CommandHandler h, IgniteInClosure<Map<ClusterNode, VisorTx
     private Map<Object, Object> generate(int from, int cnt) {
         Map<Object, Object> map = new TreeMap<>();
 
-        for (int i = 0; i < cnt; i++ )
+        for (int i = 0; i < cnt; i++)
             map.put(i + from, i + from);
 
         return map;
     }
 
     /**
-     *  Test execution of --wal print command.
+     * Test execution of --wal print command.
      *
-     *  @throws Exception if failed.
+     * @throws Exception if failed.
      */
     public void testUnusedWalPrint() throws Exception {
         Ignite ignite = startGrids(2);
@@ -795,14 +800,14 @@ public void testUnusedWalPrint() throws Exception {
 
         List<String> nodes = new ArrayList<>(2);
 
-        for (ClusterNode node: ignite.cluster().forServers().nodes())
+        for (ClusterNode node : ignite.cluster().forServers().nodes())
             nodes.add(node.consistentId().toString());
 
         injectTestSystemOut();
 
         assertEquals(EXIT_CODE_OK, execute("--wal", "print"));
 
-        for(String id: nodes)
+        for (String id : nodes)
             assertTrue(testOut.toString().contains(id));
 
         assertTrue(!testOut.toString().contains("error"));
@@ -817,9 +822,9 @@ public void testUnusedWalPrint() throws Exception {
     }
 
     /**
-     *  Test execution of --wal delete command.
+     * Test execution of --wal delete command.
      *
-     *  @throws Exception if failed.
+     * @throws Exception if failed.
      */
     public void testUnusedWalDelete() throws Exception {
         Ignite ignite = startGrids(2);
@@ -828,14 +833,14 @@ public void testUnusedWalDelete() throws Exception {
 
         List<String> nodes = new ArrayList<>(2);
 
-        for (ClusterNode node: ignite.cluster().forServers().nodes())
+        for (ClusterNode node : ignite.cluster().forServers().nodes())
             nodes.add(node.consistentId().toString());
 
         injectTestSystemOut();
 
         assertEquals(EXIT_CODE_OK, execute("--wal", "delete"));
 
-        for(String id: nodes)
+        for (String id : nodes)
             assertTrue(testOut.toString().contains(id));
 
         assertTrue(!testOut.toString().contains("error"));
@@ -850,11 +855,11 @@ public void testUnusedWalDelete() throws Exception {
     }
 
     /**
-     *
      * @param lockLatch Lock latch.
      * @param unlockLatch Unlock latch.
      */
-    private IgniteInternalFuture<?> startTransactions(CountDownLatch lockLatch, CountDownLatch unlockLatch) throws Exception {
+    private IgniteInternalFuture<?> startTransactions(CountDownLatch lockLatch,
+        CountDownLatch unlockLatch) throws Exception {
         IgniteEx client = grid("client");
 
         AtomicInteger idx = new AtomicInteger();

From 12bbdc5550634d21501943fa03dd2b3864429e4b Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Fri, 15 Jun 2018 17:12:09 +0300
Subject: [PATCH 212/543] IGNITE-8789 Invoke failure processor in case of
 message processing error

---
 .../processors/cache/GridCacheIoManager.java  |  16 +-
 .../igfs/IgfsPrimaryMultiNodeSelfTest.java    |   7 +
 ...ryRelaxedConsistencyMultiNodeSelfTest.java |   7 +
 .../GridCacheMessageSelfTest.java             | 248 ++++++++++++++++++
 4 files changed, 275 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 3182192678cbc..01344211f36e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -34,6 +34,8 @@
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.binary.BinaryObjectException;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
@@ -1057,10 +1059,18 @@ private void processMessage(UUID nodeId, GridCacheMessage msg, IgniteBiInClosure
                 log.debug("Finished processing cache communication message [nodeId=" + nodeId + ", msg=" + msg + ']');
         }
         catch (Throwable e) {
-            U.error(log, "Failed processing message [senderId=" + nodeId + ", msg=" + msg + ']', e);
+            try {
+                U.error(log, "Failed processing message [senderId=" + nodeId + ", msg=" + msg + ']', e);
+            }
+            catch (Throwable e0) {
+                U.error(log, "Failed processing message [senderId=" + nodeId + ", msg=(failed to log message)", e);
 
-            if (e instanceof Error)
-                throw e;
+                U.error(log, "Failed to log message due to an error: ", e0);
+            }
+
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+            throw e;
         }
         finally {
             onMessageProcessed(msg);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryMultiNodeSelfTest.java
index 1ee6d5ac500e8..f004d4069bbe2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryMultiNodeSelfTest.java
@@ -25,4 +25,11 @@ public class IgfsPrimaryMultiNodeSelfTest extends IgfsPrimarySelfTest {
     @Override protected int nodeCount() {
         return 4;
     }
+
+    /**
+     * @throws Exception If failed.
+     */
+    @Override public void testCreateConsistencyMultithreaded() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8823");
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryRelaxedConsistencyMultiNodeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryRelaxedConsistencyMultiNodeSelfTest.java
index 73a14c3c2470b..d35237cacd0fb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryRelaxedConsistencyMultiNodeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsPrimaryRelaxedConsistencyMultiNodeSelfTest.java
@@ -25,4 +25,11 @@ public class IgfsPrimaryRelaxedConsistencyMultiNodeSelfTest extends IgfsPrimaryR
     @Override protected int nodeCount() {
         return 4;
     }
+
+    @Override
+    public void testCreateConsistencyMultithreaded() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-8823");
+
+        super.testCreateConsistencyMultithreaded();
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
index 435af8f4b0e9c..587be71d25e2e 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java
@@ -22,10 +22,14 @@
 import java.util.Collection;
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureHandler;
 import org.apache.ignite.internal.GridDirectCollection;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.managers.communication.GridIoManager;
@@ -33,6 +37,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.processors.cache.GridCacheMessage;
+import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.CO;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
@@ -56,6 +61,9 @@ public class GridCacheMessageSelfTest extends GridCommonAbstractTest {
     /** Sample count. */
     private static final int SAMPLE_CNT = 1;
 
+    /** Latch on failure processor. */
+    private static CountDownLatch failureLatch;
+
     /** */
     public static final String TEST_BODY = "Test body";
 
@@ -86,6 +94,12 @@ public class GridCacheMessageSelfTest extends GridCommonAbstractTest {
                 return new TestMessage2();
             }
         });
+
+        GridIoMessageFactory.registerCustom(TestBadMessage.DIRECT_TYPE, new CO<Message>() {
+            @Override public Message apply() {
+                return new TestBadMessage();
+            }
+        });
     }
 
     /** {@inheritDoc} */
@@ -100,6 +114,8 @@ public class GridCacheMessageSelfTest extends GridCommonAbstractTest {
 
         cfg.setIncludeEventTypes((int[])null);
 
+        cfg.setFailureHandler(new TestFailureHandler());
+
         CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
 
         ccfg.setCacheMode(CacheMode.PARTITIONED);
@@ -113,6 +129,11 @@ public class GridCacheMessageSelfTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        failureLatch = new CountDownLatch(1);
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -127,6 +148,42 @@ public void testSendMessage() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSendBadMessage() throws Exception {
+        try {
+            startGrids(2);
+
+            Ignite ignite0 = grid(0);
+            Ignite ignite1 = grid(1);
+
+            ((IgniteKernal)ignite0).context().cache().context().io().addCacheHandler(
+                0, TestBadMessage.class, new CI2<UUID, GridCacheMessage>() {
+                @Override public void apply(UUID nodeId, GridCacheMessage msg) {
+                    throw new RuntimeException("Test bad message exception");
+                }
+            });
+
+            ((IgniteKernal)ignite1).context().cache().context().io().addCacheHandler(
+                0, TestBadMessage.class, new CI2<UUID, GridCacheMessage>() {
+                    @Override public void apply(UUID nodeId, GridCacheMessage msg) {
+                        throw new RuntimeException("Test bad message exception");
+                    }
+                });
+
+            ((IgniteKernal)ignite0).context().cache().context().io().send(
+                ((IgniteKernal)ignite1).localNode().id(), new TestBadMessage(), (byte)2);
+
+            boolean res = failureLatch.await(5, TimeUnit.SECONDS);
+
+            assertTrue(res);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -506,6 +563,185 @@ public int id() {
             return 7;
         }
 
+        /** {@inheritDoc} */
+        @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+            writer.setBuffer(buf);
+
+            if (!super.writeTo(buf, writer))
+                return false;
+
+            if (!writer.isHeaderWritten()) {
+                if (!writer.writeHeader(directType(), fieldsCount()))
+                    return false;
+
+                writer.onHeaderWritten();
+            }
+
+            switch (writer.state()) {
+                case 3:
+                    if (!writer.writeUuid("nodeId", nodeId))
+                        return false;
+
+                    writer.incrementState();
+
+                case 4:
+                    if (!writer.writeInt("id", id))
+                        return false;
+
+                    writer.incrementState();
+
+                case 5:
+                    if (!writer.writeString("body", body))
+                        return false;
+
+                    writer.incrementState();
+
+                case 6:
+                    if (!writer.writeMessage("msg", msg))
+                        return false;
+
+                    writer.incrementState();
+            }
+
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+            reader.setBuffer(buf);
+
+            if (!reader.beforeMessageRead())
+                return false;
+
+            if (!super.readFrom(buf, reader))
+                return false;
+
+            switch (reader.state()) {
+                case 3:
+                    nodeId = reader.readUuid("nodeId");
+
+                    if (!reader.isLastRead())
+                        return false;
+
+                    reader.incrementState();
+
+                case 4:
+                    id = reader.readInt("id");
+
+                    if (!reader.isLastRead())
+                        return false;
+
+                    reader.incrementState();
+
+                case 5:
+                    body = reader.readString("body");
+
+                    if (!reader.isLastRead())
+                        return false;
+
+                    reader.incrementState();
+
+                case 6:
+                    msg = reader.readMessage("msg");
+
+                    if (!reader.isLastRead())
+                        return false;
+
+                    reader.incrementState();
+
+            }
+
+            return true;
+        }
+    }
+
+    /**
+     * Test message class.
+     */
+    static class TestBadMessage extends GridCacheMessage {
+        /** */
+        public static final short DIRECT_TYPE = 204;
+
+        /** Node id. */
+        private UUID nodeId;
+
+        /** Integer field. */
+        private int id;
+
+        /** Body. */
+        private String body;
+
+        /** */
+        private Message msg;
+
+        /**
+         * @param mes Message.
+         */
+        public void init(Message mes, UUID nodeId, int id, String body) {
+            this.nodeId = nodeId;
+            this.id = id;
+            this.msg = mes;
+            this.body = body;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int handlerId() {
+            return 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean cacheGroupMessage() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean addDeploymentInfo() {
+            return false;
+        }
+
+        /**
+         * @return Body.
+         */
+        public String body() {
+            return body;
+        }
+
+        /**
+         * @return Message.
+         */
+        public Message message() {
+            return msg;
+        }
+
+        /**
+         * @return Node id.
+         */
+        public UUID nodeId() {
+            return nodeId;
+        }
+
+        /**
+         * @return Id.
+         */
+        public int id() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public short directType() {
+            return DIRECT_TYPE;
+        }
+
+        /** {@inheritDoc} */
+        @Override public byte fieldsCount() {
+            return 7;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            throw new RuntimeException("Exception while log message");
+        }
+
         /** {@inheritDoc} */
         @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
             writer.setBuffer(buf);
@@ -598,4 +834,16 @@ public int id() {
             return true;
         }
     }
+
+    /**
+     *
+     */
+    private static class TestFailureHandler implements FailureHandler {
+        /** {@inheritDoc} */
+        @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+            failureLatch.countDown();
+
+            return false;
+        }
+    }
 }

From 1f8457b0d913973d74098047d7236d036fae37a8 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Mon, 18 Jun 2018 20:44:00 +0300
Subject: [PATCH 213/543] IGNITE-8755 Throw a correct error message when client
 optimized marshaller is trying to serialize too large object

(cherry picked from commit 61b3897)
---
 .../util/io/GridUnsafeDataOutput.java         | 36 ++++++++-
 .../optimized/OptimizedMarshallerTest.java    | 80 +++++++++++++++++++
 2 files changed, 114 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
index c45b8fdd9fe6c..ad94889e52498 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/io/GridUnsafeDataOutput.java
@@ -120,7 +120,12 @@ public void bytes(byte[] bytes, int off) {
     /**
      * @param size Size.
      */
-    private void requestFreeSize(int size) {
+    private void requestFreeSize(int size) throws IOException {
+        // If arithmetic overflow occurs, off + size should be less than size.
+        if (off + size < size)
+            throw new IOException("Failed to allocate required memory (arithmetic overflow detected) " +
+                "[length=" + size + ", offset=" + off + ']');
+
         size = off + size;
 
         maxOff = Math.max(maxOff, size);
@@ -128,7 +133,7 @@ private void requestFreeSize(int size) {
         long now = U.currentTimeMillis();
 
         if (size > bytes.length) {
-            byte[] newBytes = new byte[size << 1]; // Grow.
+            byte[] newBytes = new byte[Math.max(size << 1, size)]; // Grow.
 
             System.arraycopy(bytes, 0, newBytes, 0, off);
 
@@ -185,6 +190,8 @@ private void onWrite(int size) throws IOException {
 
         int bytesToCp = arr.length << 3;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "double");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -217,6 +224,8 @@ private void onWrite(int size) throws IOException {
 
         int bytesToCp = arr.length << 1;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "char");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -240,6 +249,8 @@ private void onWrite(int size) throws IOException {
 
         int bytesToCp = arr.length << 3;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "long");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -263,6 +274,8 @@ private void onWrite(int size) throws IOException {
 
         int bytesToCp = arr.length << 2;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "float");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -304,6 +317,8 @@ private void onWrite(int size) throws IOException {
 
         int bytesToCp = arr.length << 1;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "short");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -327,6 +342,8 @@ private void onWrite(int size) throws IOException {
 
         int bytesToCp = arr.length << 2;
 
+        checkArrayAllocationOverflow(bytesToCp, arr.length, "int");
+
         requestFreeSize(bytesToCp);
 
         if (BIG_ENDIAN) {
@@ -471,6 +488,21 @@ private void onWrite(int size) throws IOException {
         writeUTF(s, utfLength(s));
     }
 
+    /**
+     * Check for possible arithmetic overflow when trying to serialize a humongous array.
+     *
+     * @param bytesToAlloc Bytes to allocate.
+     * @param arrLen Array length.
+     * @param type Type of an array.
+     * @throws IOException If oveflow presents and data corruption can occur.
+     */
+    private void checkArrayAllocationOverflow(int bytesToAlloc, int arrLen, String type) throws IOException {
+        // If arithmetic overflow occurs, bytesToAlloc should be less than arrLen.
+        if (bytesToAlloc < arrLen)
+            throw new IOException("Failed to allocate required memory for " + type + " array " +
+                "(arithmetic overflow detected) [length=" + arrLen + ']');
+    }
+
     /**
      *
      * Returns the length in bytes of the UTF encoding of the given string.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
index 14b49a4f8b2ae..79496ae1163c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
@@ -28,6 +28,7 @@
 import java.net.InetSocketAddress;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentMap;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
@@ -41,6 +42,7 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.Nullable;
 
@@ -386,6 +388,84 @@ private void checkPerformance(int cnt, int tries) throws Exception {
         info(">>> Finished performance check <<<");
     }
 
+    /**
+     * Tests checks for arithmetic overflow when trying to serialize huge object.
+     * WARNING! Requires a lot of heap space. Should not be run on CI.
+     */
+    public void _testAllocationOverflow() {
+        allocationOverflowCheck(() -> marshaller().marshal(new HugeObject()));
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new short[1<<30]);
+            marshaller().marshal(new short[1<<30]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new char[1<<30]);
+            marshaller().marshal(new char[1<<30]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new int[1<<30]);
+            marshaller().marshal(new int[1<<30]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new int[1<<30]);
+            marshaller().marshal(new int[1<<30]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new float[1<<29]);
+            marshaller().marshal(new float[1<<29]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new long[1<<29]);
+            marshaller().marshal(new long[1<<29]);
+            return null;
+        });
+
+        allocationOverflowCheck(() -> {
+            marshaller().marshal(new double[1<<29]);
+            marshaller().marshal(new double[1<<29]);
+            return null;
+        });
+    }
+
+    /**
+     * Asserts that {@link IOException} will be thrown.
+     *
+     * @param call Callable that cause allocation overflow.
+     */
+    private void allocationOverflowCheck(Callable<?> call) {
+        GridTestUtils.assertThrowsAnyCause(log, call, IOException.class, "Impossible to allocate required memory");
+    }
+
+    /**
+     *
+     */
+    public static class HugeObject implements Externalizable {
+
+        /** {@inheritDoc} */
+        @Override public void writeExternal(ObjectOutput out) throws IOException {
+            out.write(new byte[1 << 31 - 2]);
+            out.write(new byte[1 << 31 - 2]);
+            out.write(new byte[1 << 31 - 2]);
+            out.write(new byte[1 << 31 - 2]);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+
+        }
+    }
+
     /**
      * Some non-serializable class.
      */

From 2a32267504fae0812fb9441fcf70470894c30a85 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Thu, 26 Apr 2018 18:31:47 +0300
Subject: [PATCH 214/543] GG-13652 PITR : handle situation when WAL got
 temporarily disabled for rebalancing

Signed-off-by: EdShangGG <eshangareev@gridgain.com>

(cherry picked from commit 92dc88d)
Signed-off-by: EdShangGG <eshangareev@gridgain.com>
---
 .../cache/persistence/GridCacheDatabaseSharedManager.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 4a44f23fe693a..e2fe57260e35e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -4332,7 +4332,7 @@ private void fillWalDisabledGroups() {
      * @param grpId Group ID.
      * @return Key.
      */
-    private static String walGroupIdToKey(int grpId, boolean local) {
+    public static String walGroupIdToKey(int grpId, boolean local) {
         if (local)
             return WAL_LOCAL_KEY_PREFIX + grpId;
         else
@@ -4354,9 +4354,9 @@ private static String checkpointInapplicableCpAndGroupIdToKey(long cpTs, int grp
      * Convert WAL state key to cache group ID.
      *
      * @param key Key.
-     * @return Group ID.
+     * @return Group ID or {@code null} if key is not WAL state key.
      */
-    private static T2<Integer, Boolean> walKeyToGroupIdAndLocalFlag(String key) {
+    @Nullable  public static T2<Integer, Boolean> walKeyToGroupIdAndLocalFlag(String key) {
         if (key.startsWith(WAL_LOCAL_KEY_PREFIX))
             return new T2<>(Integer.parseInt(key.substring(WAL_LOCAL_KEY_PREFIX.length())), true);
         else if (key.startsWith(WAL_GLOBAL_KEY_PREFIX))

From a6dd24b1933e1a44f6779ed797e8a3b4a545ba30 Mon Sep 17 00:00:00 2001
From: Ivan Rakov <irakov@apache.org>
Date: Tue, 19 Jun 2018 12:46:10 +0300
Subject: [PATCH 215/543] IGNITE-8749 Exception for no space left situation
 should be propagated to FailureHandler - rollback

---
 .../wal/FileWriteAheadLogManager.java         | 167 ++++++------
 .../FsyncModeFileWriteAheadLogManager.java    | 215 +++++++--------
 .../ignite/failure/TestFailureHandler.java    |  19 --
 .../wal/IgniteWalFormatFileFailoverTest.java  | 258 ------------------
 .../testsuites/IgnitePdsTestSuite2.java       |   3 -
 5 files changed, 179 insertions(+), 483 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 09a08c9d4bce5..9b39987edf42a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -605,43 +605,48 @@ private void checkWalConfiguration() throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
-        assert currHnd == null;
-        assert lastPtr == null || lastPtr instanceof FileWALPointer;
+        try {
+            assert currHnd == null;
+            assert lastPtr == null || lastPtr instanceof FileWALPointer;
 
-        FileWALPointer filePtr = (FileWALPointer)lastPtr;
+            FileWALPointer filePtr = (FileWALPointer)lastPtr;
 
             walWriter = new WALWriter(log);
 
             if (!mmap)
                 new IgniteThread(walWriter).start();
 
-        currHnd = restoreWriteHandle(filePtr);
+            currHnd = restoreWriteHandle(filePtr);
 
-        // For new handle write serializer version to it.
-        if (filePtr == null)
-            currHnd.writeHeader();
+            // For new handle write serializer version to it.
+            if (filePtr == null)
+                currHnd.writeHeader();
 
-        if (currHnd.serializer.version() != serializer.version()) {
-            if (log.isInfoEnabled())
-                log.info("Record serializer version change detected, will start logging with a new WAL record " +
-                    "serializer to a new WAL segment [curFile=" + currHnd + ", newVer=" + serializer.version() +
-                    ", oldVer=" + currHnd.serializer.version() + ']');
+            if (currHnd.serializer.version() != serializer.version()) {
+                if (log.isInfoEnabled())
+                    log.info("Record serializer version change detected, will start logging with a new WAL record " +
+                        "serializer to a new WAL segment [curFile=" + currHnd + ", newVer=" + serializer.version() +
+                        ", oldVer=" + currHnd.serializer.version() + ']');
 
-            rollOver(currHnd);
-        }
+                rollOver(currHnd);
+            }
 
-        currHnd.resume = false;
+            currHnd.resume = false;
 
-        if (mode == WALMode.BACKGROUND) {
-            backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
-                @Override public void run() {
-                    doFlush();
-                }
-            }, flushFreq, flushFreq);
-        }
+            if (mode == WALMode.BACKGROUND) {
+                backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
+                    @Override public void run() {
+                        doFlush();
+                    }
+                }, flushFreq, flushFreq);
+            }
 
-        if (walAutoArchiveAfterInactivity > 0)
-            scheduleNextInactivityPeriodElapsedCheck();
+            if (walAutoArchiveAfterInactivity > 0)
+                scheduleNextInactivityPeriodElapsedCheck();
+        }
+        catch (StorageException e) {
+            throw new IgniteCheckedException(e);
+        }
     }
 
     /**
@@ -1126,9 +1131,9 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, I
     /**
      * @param lastReadPtr Last read WAL file pointer.
      * @return Initialized file write handle.
-     * @throws StorageException If failed to initialize WAL write handle.
+     * @throws IgniteCheckedException If failed to initialize WAL write handle.
      */
-    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws StorageException {
+    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws IgniteCheckedException {
         long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
 
         @Nullable FileArchiver archiver0 = archiver;
@@ -1170,9 +1175,14 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
                 SegmentedRingByteBuffer rbuf;
 
                 if (mmap) {
-                    MappedByteBuffer buf = fileIO.map((int)maxWalSegmentSize);
+                    try {
+                        MappedByteBuffer buf = fileIO.map((int)maxWalSegmentSize);
 
-                    rbuf = new SegmentedRingByteBuffer(buf, metrics);
+                        rbuf = new SegmentedRingByteBuffer(buf, metrics);
+                    }
+                    catch (IOException e) {
+                        throw new IgniteCheckedException(e);
+                    }
                 }
                 else
                     rbuf = new SegmentedRingByteBuffer(dsCfg.getWalBufferSize(), maxWalSegmentSize, DIRECT, metrics);
@@ -1196,21 +1206,13 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
                 return hnd;
             }
             catch (IgniteCheckedException | IOException e) {
-                try {
-                    fileIO.close();
-                }
-                catch (IOException suppressed) {
-                    e.addSuppressed(suppressed);
-                }
-
-                if (e instanceof StorageException)
-                    throw (StorageException) e;
+                fileIO.close();
 
-                throw e instanceof IOException ? (IOException) e : new IOException(e);
+                throw e;
             }
         }
         catch (IOException e) {
-            throw new StorageException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
+            throw new IgniteCheckedException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
         }
     }
 
@@ -1221,8 +1223,9 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
      * @param cur Current file write handle released by WAL writer
      * @return Initialized file handle.
      * @throws StorageException If IO exception occurred.
+     * @throws IgniteCheckedException If failed.
      */
-    private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageException {
+    private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageException, IgniteCheckedException {
         try {
             File nextFile = pollNextFile(cur.idx);
 
@@ -1307,10 +1310,8 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageE
 
     /**
      * Deletes temp files, creates and prepares new; Creates first segment if necessary
-     *
-     * @throws StorageException If failed.
      */
-    private void checkOrPrepareFiles() throws StorageException {
+    private void checkOrPrepareFiles() throws IgniteCheckedException {
         // Clean temp files.
         {
             File[] tmpFiles = walWorkDir.listFiles(WAL_SEGMENT_TEMP_FILE_FILTER);
@@ -1320,7 +1321,7 @@ private void checkOrPrepareFiles() throws StorageException {
                     boolean deleted = tmp.delete();
 
                     if (!deleted)
-                        throw new StorageException("Failed to delete previously created temp file " +
+                        throw new IgniteCheckedException("Failed to delete previously created temp file " +
                             "(make sure Ignite process has enough rights): " + tmp.getAbsolutePath());
                 }
             }
@@ -1330,7 +1331,7 @@ private void checkOrPrepareFiles() throws StorageException {
 
         if(isArchiverEnabled())
             if (allFiles.length != 0 && allFiles.length > dsCfg.getWalSegments())
-                throw new StorageException("Failed to initialize wal (work directory contains " +
+                throw new IgniteCheckedException("Failed to initialize wal (work directory contains " +
                     "incorrect number of segments) [cur=" + allFiles.length + ", expected=" + dsCfg.getWalSegments() + ']');
 
         // Allocate the first segment synchronously. All other segments will be allocated by archiver in background.
@@ -1370,9 +1371,9 @@ public void cleanupWalDirectories() throws IgniteCheckedException {
      * Clears whole the file, fills with zeros for Default mode.
      *
      * @param file File to format.
-     * @throws StorageException if formatting failed
+     * @throws IgniteCheckedException if formatting failed
      */
-    private void formatFile(File file) throws StorageException {
+    private void formatFile(File file) throws IgniteCheckedException {
         formatFile(file, dsCfg.getWalSegmentSize());
     }
 
@@ -1381,9 +1382,9 @@ private void formatFile(File file) throws StorageException {
      *
      * @param file File to format.
      * @param bytesCntToFormat Count of first bytes to format.
-     * @throws StorageException if formatting failed
+     * @throws IgniteCheckedException if formatting failed
      */
-    private void formatFile(File file, int bytesCntToFormat) throws StorageException {
+    private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedException {
         if (log.isDebugEnabled())
             log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1395,7 +1396,7 @@ private void formatFile(File file, int bytesCntToFormat) throws StorageException
                     int toWrite = Math.min(FILL_BUF.length, left);
 
                     if (fileIO.write(FILL_BUF, 0, toWrite) < toWrite) {
-                        final StorageException ex = new StorageException("Failed to extend WAL segment file: " +
+                        final IgniteCheckedException ex = new IgniteCheckedException("Failed to extend WAL segment file: " +
                             file.getName() + ". Probably disk is too busy, please check your device.");
 
                         if (failureProcessor != null)
@@ -1413,7 +1414,7 @@ private void formatFile(File file, int bytesCntToFormat) throws StorageException
                 fileIO.clear();
         }
         catch (IOException e) {
-            throw new StorageException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
+            throw new IgniteCheckedException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
         }
     }
 
@@ -1421,9 +1422,9 @@ private void formatFile(File file, int bytesCntToFormat) throws StorageException
      * Creates a file atomically with temp file.
      *
      * @param file File to create.
-     * @throws StorageException If failed.
+     * @throws IgniteCheckedException If failed.
      */
-    private void createFile(File file) throws StorageException {
+    private void createFile(File file) throws IgniteCheckedException {
         if (log.isDebugEnabled())
             log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1435,7 +1436,7 @@ private void createFile(File file) throws StorageException {
             Files.move(tmp.toPath(), file.toPath());
         }
         catch (IOException e) {
-            throw new StorageException("Failed to move temp file to a regular WAL segment file: " +
+            throw new IgniteCheckedException("Failed to move temp file to a regular WAL segment file: " +
                 file.getAbsolutePath(), e);
         }
 
@@ -1448,9 +1449,9 @@ private void createFile(File file) throws StorageException {
      *
      * @param curIdx Current absolute WAL segment index.
      * @return File ready for use as new WAL segment.
-     * @throws StorageException If exception occurred in the archiver thread.
+     * @throws IgniteCheckedException If failed.
      */
-    private File pollNextFile(long curIdx) throws StorageException {
+    private File pollNextFile(long curIdx) throws IgniteCheckedException {
         FileArchiver archiver0 = archiver;
 
         if (archiver0 == null) {
@@ -1526,7 +1527,7 @@ public long maxWalSegmentSize() {
      */
     private class FileArchiver extends GridWorker {
         /** Exception which occurred during initial creation of files or during archiving WAL segment */
-        private StorageException cleanErr;
+        private IgniteCheckedException cleanErr;
 
         /**
          * Absolute current segment index WAL Manager writes to. Guarded by <code>this</code>. Incremented during
@@ -1598,17 +1599,15 @@ private synchronized boolean locked(long absIdx) {
             try {
                 allocateRemainingFiles();
             }
-            catch (StorageException e) {
+            catch (IgniteCheckedException e) {
                 synchronized (this) {
                     // Stop the thread and report to starter.
                     cleanErr = e;
 
                     notifyAll();
-                }
-
-                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
 
-                return;
+                    return;
+                }
             }
 
             Throwable err = null;
@@ -1692,9 +1691,9 @@ private void changeLastArchivedIndexAndNotifyWaiters(long idx) {
          *
          * @param curIdx Current absolute index that we want to increment.
          * @return Next index (curWalSegmIdx+1) when it is ready to be written.
-         * @throws StorageException If exception occurred in the archiver thread.
+         * @throws IgniteCheckedException If failed (if interrupted or if exception occurred in the archiver thread).
          */
-        private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException {
+        private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException {
             synchronized (this) {
                 if (cleanErr != null)
                     throw cleanErr;
@@ -1709,9 +1708,6 @@ private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException {
                 while (curAbsWalIdx - lastAbsArchivedIdx > dsCfg.getWalSegments() && cleanErr == null) {
                     try {
                         wait();
-
-                        if (cleanErr != null)
-                            throw cleanErr;
                     }
                     catch (InterruptedException ignore) {
                         interrupted.set(true);
@@ -1719,12 +1715,9 @@ private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException {
                 }
 
                 // Wait for formatter so that we do not open an empty file in DEFAULT mode.
-                while (curAbsWalIdx % dsCfg.getWalSegments() > formatted && cleanErr == null)
+                while (curAbsWalIdx % dsCfg.getWalSegments() > formatted)
                     try {
                         wait();
-
-                        if (cleanErr != null)
-                            throw cleanErr;
                     }
                     catch (InterruptedException ignore) {
                         interrupted.set(true);
@@ -1796,7 +1789,7 @@ private void releaseWorkSegment(long absIdx) {
          *
          * @param absIdx Absolute index to archive.
          */
-        private SegmentArchiveResult archiveSegment(long absIdx) throws StorageException {
+        private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedException {
             long segIdx = absIdx % dsCfg.getWalSegments();
 
             File origFile = new File(walWorkDir, FileDescriptor.fileName(segIdx));
@@ -1825,7 +1818,7 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws StorageException
                 }
             }
             catch (IOException e) {
-                throw new StorageException("Failed to archive WAL segment [" +
+                throw new IgniteCheckedException("Failed to archive WAL segment [" +
                     "srcFile=" + origFile.getAbsolutePath() +
                     ", dstFile=" + dstTmpFile.getAbsolutePath() + ']', e);
             }
@@ -1848,7 +1841,7 @@ private boolean checkStop() {
          * Background creation of all segments except first. First segment was created in main thread by {@link
          * FileWriteAheadLogManager#checkOrPrepareFiles()}
          */
-        private void allocateRemainingFiles() throws StorageException {
+        private void allocateRemainingFiles() throws IgniteCheckedException {
             checkFiles(
                 1,
                 true,
@@ -2242,23 +2235,23 @@ private void shutdown() throws IgniteInterruptedCheckedException {
      * @param startWith Start with.
      * @param create Flag create file.
      * @param p Predicate Exit condition.
-     * @throws StorageException if validation or create file fail.
+     * @throws IgniteCheckedException if validation or create file fail.
      */
     private void checkFiles(
         int startWith,
         boolean create,
         @Nullable IgnitePredicate<Integer> p,
         @Nullable IgniteInClosure<Integer> completionCallback
-    ) throws StorageException {
+    ) throws IgniteCheckedException {
         for (int i = startWith; i < dsCfg.getWalSegments() && (p == null || p.apply(i)); i++) {
             File checkFile = new File(walWorkDir, FileDescriptor.fileName(i));
 
             if (checkFile.exists()) {
                 if (checkFile.isDirectory())
-                    throw new StorageException("Failed to initialize WAL log segment (a directory with " +
+                    throw new IgniteCheckedException("Failed to initialize WAL log segment (a directory with " +
                         "the same name already exists): " + checkFile.getAbsolutePath());
                 else if (checkFile.length() != dsCfg.getWalSegmentSize() && mode == WALMode.FSYNC)
-                    throw new StorageException("Failed to initialize WAL log segment " +
+                    throw new IgniteCheckedException("Failed to initialize WAL log segment " +
                         "(WAL segment size change is not supported in 'DEFAULT' WAL mode) " +
                         "[filePath=" + checkFile.getAbsolutePath() +
                         ", fileSize=" + checkFile.length() +
@@ -2658,8 +2651,9 @@ public void writeHeader() {
          * Flush or wait for concurrent flush completion.
          *
          * @param ptr Pointer.
+         * @throws IgniteCheckedException If failed.
          */
-        private void flushOrWait(FileWALPointer ptr) {
+        private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException {
             if (ptr != null) {
                 // If requested obsolete file index, it must be already flushed by close.
                 if (ptr.index() != idx)
@@ -2671,8 +2665,10 @@ private void flushOrWait(FileWALPointer ptr) {
 
         /**
          * @param ptr Pointer.
+         * @throws IgniteCheckedException If failed.
+         * @throws StorageException If failed.
          */
-        private void flush(FileWALPointer ptr) {
+        private void flush(FileWALPointer ptr) throws IgniteCheckedException, StorageException {
             if (ptr == null) { // Unconditional flush.
                 walWriter.flushAll();
 
@@ -2888,7 +2884,7 @@ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageEx
                         }
                     }
                     catch (IOException e) {
-                        throw new StorageException(e);
+                        throw new IgniteCheckedException(e);
                     }
 
                     if (log.isDebugEnabled())
@@ -3369,29 +3365,28 @@ private void unparkWaiters(long pos) {
         /**
          * Forces all made changes to the file.
          */
-        void force() {
+        void force() throws IgniteCheckedException {
             flushBuffer(FILE_FORCE);
         }
 
         /**
          * Closes file.
          */
-        void close() {
+        void close() throws IgniteCheckedException {
             flushBuffer(FILE_CLOSE);
         }
 
         /**
          * Flushes all data from the buffer.
          */
-        void flushAll() {
+        void flushAll() throws IgniteCheckedException {
             flushBuffer(UNCONDITIONAL_FLUSH);
         }
 
         /**
          * @param expPos Expected position.
          */
-        @SuppressWarnings("ForLoopReplaceableByForEach")
-        void flushBuffer(long expPos) {
+        void flushBuffer(long expPos) throws StorageException, IgniteCheckedException {
             if (mmap)
                 return;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 6f676fcf43284..49fbc73b9a48c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -487,32 +487,37 @@ private void checkWalConfiguration() throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
-        assert currentHnd == null;
-        assert lastPtr == null || lastPtr instanceof FileWALPointer;
+        try {
+            assert currentHnd == null;
+            assert lastPtr == null || lastPtr instanceof FileWALPointer;
 
-        FileWALPointer filePtr = (FileWALPointer)lastPtr;
+            FileWALPointer filePtr = (FileWALPointer)lastPtr;
 
-        currentHnd = restoreWriteHandle(filePtr);
+            currentHnd = restoreWriteHandle(filePtr);
 
-        if (currentHnd.serializer.version() != serializer.version()) {
-            if (log.isInfoEnabled())
-                log.info("Record serializer version change detected, will start logging with a new WAL record " +
-                    "serializer to a new WAL segment [curFile=" + currentHnd + ", newVer=" + serializer.version() +
-                    ", oldVer=" + currentHnd.serializer.version() + ']');
+            if (currentHnd.serializer.version() != serializer.version()) {
+                if (log.isInfoEnabled())
+                    log.info("Record serializer version change detected, will start logging with a new WAL record " +
+                        "serializer to a new WAL segment [curFile=" + currentHnd + ", newVer=" + serializer.version() +
+                        ", oldVer=" + currentHnd.serializer.version() + ']');
 
-            rollOver(currentHnd);
-        }
+                rollOver(currentHnd);
+            }
 
-        if (mode == WALMode.BACKGROUND) {
-            backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
-                @Override public void run() {
-                    doFlush();
-                }
-            }, flushFreq, flushFreq);
-        }
+            if (mode == WALMode.BACKGROUND) {
+                backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
+                    @Override public void run() {
+                        doFlush();
+                    }
+                }, flushFreq, flushFreq);
+            }
 
-        if (walAutoArchiveAfterInactivity > 0)
-            scheduleNextInactivityPeriodElapsedCheck();
+            if (walAutoArchiveAfterInactivity > 0)
+                scheduleNextInactivityPeriodElapsedCheck();
+        }
+        catch (StorageException e) {
+            throw new IgniteCheckedException(e);
+        }
     }
 
     /**
@@ -1014,7 +1019,7 @@ private FileWriteHandle currentHandle() {
      * @param cur Handle that failed to fit the given entry.
      * @return Handle that will fit the entry.
      */
-    private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, IgniteInterruptedCheckedException {
+    private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, IgniteCheckedException {
         FileWriteHandle hnd = currentHandle();
 
         if (hnd != cur)
@@ -1045,9 +1050,9 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, I
     /**
      * @param lastReadPtr Last read WAL file pointer.
      * @return Initialized file write handle.
-     * @throws StorageException If failed to initialize WAL write handle.
+     * @throws IgniteCheckedException If failed to initialize WAL write handle.
      */
-    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws StorageException {
+    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws IgniteCheckedException {
         long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
 
         long segNo = absIdx % dsCfg.getWalSegments();
@@ -1095,21 +1100,13 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
                 return hnd;
             }
             catch (IgniteCheckedException | IOException e) {
-                try {
-                    fileIO.close();
-                }
-                catch (IOException suppressed) {
-                    e.addSuppressed(suppressed);
-                }
-
-                if (e instanceof StorageException)
-                    throw (StorageException) e;
+                fileIO.close();
 
-                throw e instanceof IOException ? (IOException) e : new IOException(e);
+                throw e;
             }
         }
         catch (IOException e) {
-            throw new StorageException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
+            throw new IgniteCheckedException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
         }
     }
 
@@ -1121,9 +1118,9 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
      * @param curIdx current absolute segment released by WAL writer
      * @return Initialized file handle.
      * @throws StorageException If IO exception occurred.
-     * @throws IgniteInterruptedCheckedException If interrupted.
+     * @throws IgniteCheckedException If failed.
      */
-    private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException, IgniteInterruptedCheckedException {
+    private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException, IgniteCheckedException {
         try {
             File nextFile = pollNextFile(curIdx);
 
@@ -1153,11 +1150,9 @@ private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException
     }
 
     /**
-     * Deletes temp files, creates and prepares new; Creates first segment if necessary.
-     *
-     * @throws StorageException If failed.
+     * Deletes temp files, creates and prepares new; Creates first segment if necessary
      */
-    private void checkOrPrepareFiles() throws StorageException {
+    private void checkOrPrepareFiles() throws IgniteCheckedException {
         // Clean temp files.
         {
             File[] tmpFiles = walWorkDir.listFiles(WAL_SEGMENT_TEMP_FILE_FILTER);
@@ -1167,7 +1162,7 @@ private void checkOrPrepareFiles() throws StorageException {
                     boolean deleted = tmp.delete();
 
                     if (!deleted)
-                        throw new StorageException("Failed to delete previously created temp file " +
+                        throw new IgniteCheckedException("Failed to delete previously created temp file " +
                             "(make sure Ignite process has enough rights): " + tmp.getAbsolutePath());
                 }
             }
@@ -1176,7 +1171,7 @@ private void checkOrPrepareFiles() throws StorageException {
         File[] allFiles = walWorkDir.listFiles(WAL_SEGMENT_FILE_FILTER);
 
         if (allFiles.length != 0 && allFiles.length > dsCfg.getWalSegments())
-            throw new StorageException("Failed to initialize wal (work directory contains " +
+            throw new IgniteCheckedException("Failed to initialize wal (work directory contains " +
                 "incorrect number of segments) [cur=" + allFiles.length + ", expected=" + dsCfg.getWalSegments() + ']');
 
         // Allocate the first segment synchronously. All other segments will be allocated by archiver in background.
@@ -1193,9 +1188,9 @@ private void checkOrPrepareFiles() throws StorageException {
      * Clears whole the file, fills with zeros for Default mode.
      *
      * @param file File to format.
-     * @throws StorageException if formatting failed.
+     * @throws IgniteCheckedException if formatting failed
      */
-    private void formatFile(File file) throws StorageException {
+    private void formatFile(File file) throws IgniteCheckedException {
         formatFile(file, dsCfg.getWalSegmentSize());
     }
 
@@ -1204,9 +1199,9 @@ private void formatFile(File file) throws StorageException {
      *
      * @param file File to format.
      * @param bytesCntToFormat Count of first bytes to format.
-     * @throws StorageException If formatting failed.
+     * @throws IgniteCheckedException if formatting failed
      */
-    private void formatFile(File file, int bytesCntToFormat) throws StorageException {
+    private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedException {
         if (log.isDebugEnabled())
             log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1228,7 +1223,7 @@ private void formatFile(File file, int bytesCntToFormat) throws StorageException
                 fileIO.clear();
         }
         catch (IOException e) {
-            throw new StorageException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
+            throw new IgniteCheckedException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
         }
     }
 
@@ -1236,9 +1231,9 @@ private void formatFile(File file, int bytesCntToFormat) throws StorageException
      * Creates a file atomically with temp file.
      *
      * @param file File to create.
-     * @throws StorageException If failed.
+     * @throws IgniteCheckedException If failed.
      */
-    private void createFile(File file) throws StorageException {
+    private void createFile(File file) throws IgniteCheckedException {
         if (log.isDebugEnabled())
             log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1250,7 +1245,7 @@ private void createFile(File file) throws StorageException {
             Files.move(tmp.toPath(), file.toPath());
         }
         catch (IOException e) {
-            throw new StorageException("Failed to move temp file to a regular WAL segment file: " +
+            throw new IgniteCheckedException("Failed to move temp file to a regular WAL segment file: " +
                 file.getAbsolutePath(), e);
         }
 
@@ -1264,10 +1259,9 @@ private void createFile(File file) throws StorageException {
      *
      * @param curIdx Current absolute WAL segment index.
      * @return File ready for use as new WAL segment.
-     * @throws StorageException If exception occurred in the archiver thread.
-     * @throws IgniteInterruptedCheckedException If interrupted.
+     * @throws IgniteCheckedException If failed.
      */
-    private File pollNextFile(long curIdx) throws StorageException, IgniteInterruptedCheckedException {
+    private File pollNextFile(long curIdx) throws IgniteCheckedException {
         // Signal to archiver that we are done with the segment and it can be archived.
         long absNextIdx = archiver.nextAbsoluteSegmentIndex(curIdx);
 
@@ -1324,7 +1318,7 @@ private void checkNode() throws StorageException {
      */
     private class FileArchiver extends GridWorker {
         /** Exception which occurred during initial creation of files or during archiving WAL segment */
-        private StorageException cleanException;
+        private IgniteCheckedException cleanException;
 
         /**
          * Absolute current segment index WAL Manager writes to. Guarded by <code>this</code>.
@@ -1432,17 +1426,15 @@ private synchronized void release(long absIdx) {
             try {
                 allocateRemainingFiles();
             }
-            catch (StorageException e) {
+            catch (IgniteCheckedException e) {
                 synchronized (this) {
                     // Stop the thread and report to starter.
                     cleanException = e;
 
                     notifyAll();
-                }
 
-                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
-
-                return;
+                    return;
+                }
             }
 
             Throwable err = null;
@@ -1523,10 +1515,9 @@ private void changeLastArchivedIndexAndWakeupCompressor(long idx) {
          *
          * @param curIdx Current absolute index that we want to increment.
          * @return Next index (curWalSegmIdx+1) when it is ready to be written.
-         * @throws StorageException If exception occurred in the archiver thread.
-         * @throws IgniteInterruptedCheckedException If interrupted.
+         * @throws IgniteCheckedException If failed (if interrupted or if exception occurred in the archiver thread).
          */
-        private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException, IgniteInterruptedCheckedException {
+        private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException {
             try {
                 synchronized (this) {
                     if (cleanException != null)
@@ -1544,16 +1535,10 @@ private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException, Igni
                     while ((curAbsWalIdx - lastAbsArchivedIdx > segments && cleanException == null))
                         wait();
 
-                    if (cleanException != null)
-                        throw cleanException;
-
                     // Wait for formatter so that we do not open an empty file in DEFAULT mode.
-                    while (curAbsWalIdx % dsCfg.getWalSegments() > formatted && cleanException == null)
+                    while (curAbsWalIdx % dsCfg.getWalSegments() > formatted)
                         wait();
 
-                    if (cleanException != null)
-                        throw cleanException;
-
                     return curAbsWalIdx;
                 }
             }
@@ -1679,7 +1664,7 @@ private boolean checkStop() {
          * Background creation of all segments except first. First segment was created in main thread by
          * {@link FsyncModeFileWriteAheadLogManager#checkOrPrepareFiles()}
          */
-        private void allocateRemainingFiles() throws StorageException {
+        private void allocateRemainingFiles() throws IgniteCheckedException {
             final FileArchiver archiver = this;
 
             checkFiles(1,
@@ -2044,23 +2029,23 @@ private void shutdown() {
      * @param startWith Start with.
      * @param create Flag create file.
      * @param p Predicate Exit condition.
-     * @throws StorageException if validation or create file fail.
+     * @throws IgniteCheckedException if validation or create file fail.
      */
     private void checkFiles(
         int startWith,
         boolean create,
         @Nullable IgnitePredicate<Integer> p,
         @Nullable IgniteInClosure<Integer> completionCallback
-    ) throws StorageException {
+    ) throws IgniteCheckedException {
         for (int i = startWith; i < dsCfg.getWalSegments() && (p == null || (p != null && p.apply(i))); i++) {
             File checkFile = new File(walWorkDir, FileDescriptor.fileName(i));
 
             if (checkFile.exists()) {
                 if (checkFile.isDirectory())
-                    throw new StorageException("Failed to initialize WAL log segment (a directory with " +
+                    throw new IgniteCheckedException("Failed to initialize WAL log segment (a directory with " +
                         "the same name already exists): " + checkFile.getAbsolutePath());
                 else if (checkFile.length() != dsCfg.getWalSegmentSize() && mode == WALMode.FSYNC)
-                    throw new StorageException("Failed to initialize WAL log segment " +
+                    throw new IgniteCheckedException("Failed to initialize WAL log segment " +
                         "(WAL segment size change is not supported):" + checkFile.getAbsolutePath());
             }
             else if (create)
@@ -2423,9 +2408,9 @@ private FileWriteHandle(
          * Write serializer version to current handle.
          * NOTE: Method mutates {@code fileIO} position, written and lastFsyncPos fields.
          *
-         * @throws StorageException If fail to write serializer version.
+         * @throws IOException If fail to write serializer version.
          */
-        private void writeSerializerVersion() throws StorageException {
+        public void writeSerializerVersion() throws IOException {
             try {
                 assert fileIO.position() == 0 : "Serializer version can be written only at the begin of file " +
                     fileIO.position();
@@ -2438,7 +2423,7 @@ private void writeSerializerVersion() throws StorageException {
                 head.set(new FakeRecord(new FileWALPointer(idx, (int)updatedPosition, 0), false));
             }
             catch (IOException e) {
-                throw new StorageException("Unable to write serializer version for segment " + idx, e);
+                throw new IOException("Unable to write serializer version for segment " + idx, e);
             }
         }
 
@@ -2463,8 +2448,9 @@ private boolean stopped(WALRecord record) {
          * @param rec Record to be added to record chain as new {@link #head}
          * @return Pointer or null if roll over to next segment is required or already started by other thread.
          * @throws StorageException If failed.
+         * @throws IgniteCheckedException If failed.
          */
-        @Nullable private WALPointer addRecord(WALRecord rec) throws StorageException {
+        @Nullable private WALPointer addRecord(WALRecord rec) throws StorageException, IgniteCheckedException {
             assert rec.size() > 0 || rec.getClass() == FakeRecord.class;
 
             boolean flushed = false;
@@ -2517,9 +2503,9 @@ private long nextPosition(WALRecord rec) {
          * Flush or wait for concurrent flush completion.
          *
          * @param ptr Pointer.
-         * @throws StorageException If failed.
+         * @throws IgniteCheckedException If failed.
          */
-        private void flushOrWait(FileWALPointer ptr, boolean stop) throws StorageException {
+        private void flushOrWait(FileWALPointer ptr, boolean stop) throws IgniteCheckedException {
             long expWritten;
 
             if (ptr != null) {
@@ -2563,9 +2549,10 @@ else if (stop) {
         /**
          * @param ptr Pointer.
          * @return {@code true} If the flush really happened.
+         * @throws IgniteCheckedException If failed.
          * @throws StorageException If failed.
          */
-        private boolean flush(FileWALPointer ptr, boolean stop) throws StorageException {
+        private boolean flush(FileWALPointer ptr, boolean stop) throws IgniteCheckedException, StorageException {
             if (ptr == null) { // Unconditional flush.
                 for (; ; ) {
                     WALRecord expHead = head.get();
@@ -2607,9 +2594,10 @@ private long chainBeginPosition(WALRecord h) {
 
         /**
          * @param expHead Expected head of chain. If head was changed, flush is not performed in this thread
+         * @throws IgniteCheckedException If failed.
          * @throws StorageException If failed.
          */
-        private boolean flush(WALRecord expHead, boolean stop) throws StorageException {
+        private boolean flush(WALRecord expHead, boolean stop) throws StorageException, IgniteCheckedException {
             if (expHead.previous() == null) {
                 FakeRecord frHead = (FakeRecord)expHead;
 
@@ -2655,8 +2643,7 @@ private boolean flush(WALRecord expHead, boolean stop) throws StorageException {
                 return true;
             }
             catch (Throwable e) {
-                StorageException se = e instanceof StorageException ? (StorageException) e :
-                    new StorageException("Unable to write", new IOException(e));
+                StorageException se = new StorageException("Unable to write", new IOException(e));
 
                 cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
 
@@ -2738,9 +2725,8 @@ private FileWALPointer position() {
         /**
          * @param ptr Pointer to sync.
          * @throws StorageException If failed.
-         * @throws IgniteInterruptedCheckedException If interrupted.
          */
-        private void fsync(FileWALPointer ptr, boolean stop) throws StorageException, IgniteInterruptedCheckedException {
+        private void fsync(FileWALPointer ptr, boolean stop) throws StorageException, IgniteCheckedException {
             lock.lock();
 
             try {
@@ -2794,9 +2780,10 @@ private void fsync(FileWALPointer ptr, boolean stop) throws StorageException, Ig
 
         /**
          * @return {@code true} If this thread actually closed the segment.
+         * @throws IgniteCheckedException If failed.
          * @throws StorageException If failed.
          */
-        private boolean close(boolean rollOver) throws StorageException {
+        private boolean close(boolean rollOver) throws IgniteCheckedException, StorageException {
             if (stop.compareAndSet(false, true)) {
                 lock.lock();
 
@@ -2806,49 +2793,43 @@ private boolean close(boolean rollOver) throws StorageException {
                     assert stopped() : "Segment is not closed after close flush: " + head.get();
 
                     try {
-                        try {
-                            RecordSerializer backwardSerializer = new RecordSerializerFactoryImpl(cctx)
-                                .createSerializer(serializerVersion);
+                        RecordSerializer backwardSerializer = new RecordSerializerFactoryImpl(cctx)
+                            .createSerializer(serializerVersion);
 
-                            SwitchSegmentRecord segmentRecord = new SwitchSegmentRecord();
+                        SwitchSegmentRecord segmentRecord = new SwitchSegmentRecord();
 
-                            int switchSegmentRecSize = backwardSerializer.size(segmentRecord);
+                        int switchSegmentRecSize = backwardSerializer.size(segmentRecord);
 
-                            if (rollOver && written < (maxSegmentSize - switchSegmentRecSize)) {
-                                final ByteBuffer buf = ByteBuffer.allocate(switchSegmentRecSize);
+                        if (rollOver && written < (maxSegmentSize - switchSegmentRecSize)) {
+                            final ByteBuffer buf = ByteBuffer.allocate(switchSegmentRecSize);
 
-                                segmentRecord.position(new FileWALPointer(idx, (int)written, switchSegmentRecSize));
-                                backwardSerializer.writeRecord(segmentRecord, buf);
+                            segmentRecord.position(new FileWALPointer(idx, (int)written, switchSegmentRecSize));
+                            backwardSerializer.writeRecord(segmentRecord, buf);
 
-                                buf.rewind();
+                            buf.rewind();
 
-                                int rem = buf.remaining();
+                            int rem = buf.remaining();
 
-                                while (rem > 0) {
-                                    int written0 = fileIO.write(buf, written);
+                            while (rem > 0) {
+                                int written0 = fileIO.write(buf, written);
 
-                                    written += written0;
+                                written += written0;
 
-                                    rem -= written0;
-                                }
+                                rem -= written0;
                             }
                         }
-                        catch (IgniteCheckedException e) {
-                            throw new IOException(e);
-                        }
-                        finally {
-                            // Do the final fsync.
-                            if (mode == WALMode.FSYNC) {
-                                fileIO.force();
 
-                                lastFsyncPos = written;
-                            }
+                        // Do the final fsync.
+                        if (mode == WALMode.FSYNC) {
+                            fileIO.force();
 
-                            fileIO.close();
+                            lastFsyncPos = written;
                         }
+
+                        fileIO.close();
                     }
                     catch (IOException e) {
-                        throw new StorageException(e);
+                        throw new IgniteCheckedException(e);
                     }
 
                     if (log.isDebugEnabled())
@@ -2891,9 +2872,9 @@ private void signalNextAvailable() {
         }
 
         /**
-         *
+         * @throws IgniteCheckedException If failed.
          */
-        private void awaitNext() {
+        private void awaitNext() throws IgniteCheckedException {
             lock.lock();
 
             try {
@@ -2913,7 +2894,7 @@ private void awaitNext() {
          * @throws IgniteCheckedException If failed.
          */
         @SuppressWarnings("TooBroadScope")
-        private void writeBuffer(long pos, ByteBuffer buf) throws StorageException {
+        private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, IgniteCheckedException {
             boolean interrupted = false;
 
             lock.lock();
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java b/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
index 545c9ea1176d1..1159683e6b54f 100644
--- a/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
+++ b/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.failure;
 
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
 import org.apache.ignite.Ignite;
 
 /**
@@ -34,13 +33,6 @@ public class TestFailureHandler implements FailureHandler {
     /** Failure context. */
     volatile FailureContext failureCtx;
 
-    /**
-     * @param invalidate Invalidate.
-     */
-    public TestFailureHandler(boolean invalidate) {
-        this(invalidate, new CountDownLatch(1));
-    }
-
     /**
      * @param invalidate Invalidate.
      * @param latch Latch.
@@ -68,15 +60,4 @@ public TestFailureHandler(boolean invalidate, CountDownLatch latch) {
     public FailureContext failureContext() {
         return failureCtx;
     }
-
-    /**
-     * @param millis Millis.
-
-     * @return Failure context.
-     */
-    public FailureContext awaitFailure(long millis) throws InterruptedException {
-        latch.await(millis, TimeUnit.MILLISECONDS);
-
-        return failureCtx;
-    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
deleted file mode 100644
index 379b8c32cda89..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.persistence.db.wal;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.OpenOption;
-import java.util.Arrays;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.failure.FailureHandler;
-import org.apache.ignite.failure.TestFailureHandler;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
-import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
-import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
-import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
-import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static java.nio.file.StandardOpenOption.CREATE;
-import static java.nio.file.StandardOpenOption.READ;
-import static java.nio.file.StandardOpenOption.WRITE;
-
-/**
- *
- */
-public class IgniteWalFormatFileFailoverTest extends GridCommonAbstractTest {
-    /** */
-    private static final String TEST_CACHE = "testCache";
-
-    /** */
-    private static final String formatFile = "formatFile";
-
-    /** Fail method name reference. */
-    private final AtomicReference<String> failMtdNameRef = new AtomicReference<>();
-
-    /** */
-    private boolean fsync;
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTest() throws Exception {
-        cleanPersistenceDir();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        stopAllGrids();
-
-        cleanPersistenceDir();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        cfg.setCacheConfiguration(new CacheConfiguration(TEST_CACHE)
-            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL));
-
-        DataStorageConfiguration memCfg = new DataStorageConfiguration()
-            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
-                .setMaxSize(2048L * 1024 * 1024)
-                .setPersistenceEnabled(true))
-            .setWalMode(fsync ? WALMode.FSYNC : WALMode.BACKGROUND)
-            .setWalBufferSize(1024 * 1024)
-            .setWalSegmentSize(512 * 1024)
-            .setFileIOFactory(new FailingFileIOFactory(failMtdNameRef));
-
-        cfg.setDataStorageConfiguration(memCfg);
-
-        cfg.setFailureHandler(new TestFailureHandler(false));
-
-        return cfg;
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testNodeStartFailedFsync() throws Exception {
-        fsync = true;
-
-        failMtdNameRef.set(formatFile);
-
-        checkCause(GridTestUtils.assertThrows(log, () -> startGrid(0), IgniteCheckedException.class, null));
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testFailureHandlerTriggeredFsync() throws Exception {
-        fsync = true;
-
-        failFormatFileOnClusterActivate();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testFailureHandlerTriggered() throws Exception {
-        fsync = false;
-
-        failFormatFileOnClusterActivate();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    private void failFormatFileOnClusterActivate() throws Exception {
-        failMtdNameRef.set(null);
-
-        startGrid(0);
-        startGrid(1);
-
-        if (!fsync) {
-            setFileIOFactory(grid(0).context().cache().context().wal());
-            setFileIOFactory(grid(1).context().cache().context().wal());
-        }
-
-        failMtdNameRef.set(formatFile);
-
-        grid(0).cluster().active(true);
-
-        checkCause(failureHandler(0).awaitFailure(2000).error());
-        checkCause(failureHandler(1).awaitFailure(2000).error());
-    }
-
-    /**
-     * @param mtdName Method name.
-     */
-    private static boolean isCalledFrom(String mtdName) {
-        return isCalledFrom(Thread.currentThread().getStackTrace(), mtdName);
-    }
-
-    /**
-     * @param stackTrace Stack trace.
-     * @param mtdName Method name.
-     */
-    private static boolean isCalledFrom(StackTraceElement[] stackTrace, String mtdName) {
-        return Arrays.stream(stackTrace).map(StackTraceElement::getMethodName).anyMatch(mtdName::equals);
-    }
-
-    /**
-     * @param gridIdx Grid index.
-     * @return Failure handler configured for grid with given index.
-     */
-    private TestFailureHandler failureHandler(int gridIdx) {
-        FailureHandler hnd = grid(gridIdx).configuration().getFailureHandler();
-
-        assertTrue(hnd instanceof TestFailureHandler);
-
-        return (TestFailureHandler)hnd;
-    }
-
-    /**
-     * @param t Throwable.
-     */
-    private void checkCause(Throwable t) {
-        StorageException e = X.cause(t, StorageException.class);
-
-        assertNotNull(e);
-        assertNotNull(e.getMessage());
-        assertTrue(e.getMessage().contains("Failed to format WAL segment file"));
-
-        IOException ioe = X.cause(e, IOException.class);
-
-        assertNotNull(ioe);
-        assertNotNull(ioe.getMessage());
-        assertTrue(ioe.getMessage().contains("No space left on device"));
-
-        assertTrue(isCalledFrom(ioe.getStackTrace(), formatFile));
-    }
-
-    /** */
-    private void setFileIOFactory(IgniteWriteAheadLogManager wal) {
-        if (wal instanceof FileWriteAheadLogManager)
-            ((FileWriteAheadLogManager)wal).setFileIOFactory(new FailingFileIOFactory(failMtdNameRef));
-        else
-            fail(wal.getClass().toString());
-    }
-
-    /**
-     * Create File I/O which fails if specific method call present in stack trace.
-     */
-    private static class FailingFileIOFactory implements FileIOFactory {
-        /** Serial version uid. */
-        private static final long serialVersionUID = 0L;
-
-        /** Delegate factory. */
-        private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
-
-        /** Fail method name reference. */
-        private final AtomicReference<String> failMtdNameRef;
-
-        /**
-         * @param failMtdNameRef Fail method name reference.
-         */
-        FailingFileIOFactory(AtomicReference<String> failMtdNameRef) {
-            assertNotNull(failMtdNameRef);
-
-            this.failMtdNameRef = failMtdNameRef;
-        }
-
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return create(file, CREATE, READ, WRITE);
-        }
-
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
-            final FileIO delegate = delegateFactory.create(file, modes);
-
-            return new FileIODecorator(delegate) {
-                @Override public int write(byte[] buf, int off, int len) throws IOException {
-                    conditionalFail();
-
-                    return super.write(buf, off, len);
-                }
-
-                @Override public void clear() throws IOException {
-                    conditionalFail();
-
-                    super.clear();
-                }
-
-                private void conditionalFail() throws IOException {
-                    String failMtdName = failMtdNameRef.get();
-
-                    if (failMtdName != null && isCalledFrom(failMtdName))
-                        throw new IOException("No space left on device");
-                }
-            };
-        }
-    }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 7732cee672799..c5a381eb6e476 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -49,7 +49,6 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncWithMmapBufferSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlySelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlyWithMmapBufferSelfTest;
-import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFormatFileFailoverTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorSwitchSegmentTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalSerializerVersionTest;
@@ -148,8 +147,6 @@ public static void addRealPageStoreTests(TestSuite suite) {
 
         suite.addTestSuite(IgniteWalFlushLogOnlyWithMmapBufferSelfTest.class);
 
-        suite.addTestSuite(IgniteWalFormatFileFailoverTest.class);
-
         // Test suite uses Standalone WAL iterator to verify PDS content.
         suite.addTestSuite(IgniteWalReaderTest.class);
 

From 1536e13fecedb693cd1753024bcd98119a1e7bd2 Mon Sep 17 00:00:00 2001
From: Dmitriy Sorokin <d.w.sorokin@gmail.com>
Date: Tue, 19 Jun 2018 15:35:51 +0300
Subject: [PATCH 216/543] IGNITE-8769 JVM crash in Basic1 suite in master
 branch on TC - Fixes #4206.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit b37f8a2)
---
 .../cache/persistence/freelist/PagesList.java | 23 +++++++++++++++----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
index ed77674a8e024..78dc91f512c57 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
@@ -635,8 +635,19 @@ protected final void put(
                         continue;
                 }
 
-                assert PageIO.getPageId(tailAddr) == tailId : "pageId = " + PageIO.getPageId(tailAddr) + ", tailId = " + tailId;
-                assert PageIO.getType(tailAddr) == PageIO.T_PAGE_LIST_NODE;
+                if (stripe.tailId != tailId) {
+                    // Another thread took the last page.
+                    writeUnlock(tailId, tailPage, tailAddr, false);
+
+                    lockAttempt--; // Ignore current attempt.
+
+                    continue;
+                }
+
+                assert PageIO.getPageId(tailAddr) == tailId
+                    : "tailId = " + U.hexLong(tailId) + ", pageId = " + U.hexLong(PageIO.getPageId(tailAddr));
+                assert PageIO.getType(tailAddr) == PageIO.T_PAGE_LIST_NODE
+                    : "tailId = " + U.hexLong(tailId) + ", type = " + PageIO.getType(tailAddr);
 
                 boolean ok = false;
 
@@ -1032,7 +1043,7 @@ protected final long takeEmptyPage(int bucket, @Nullable IOVersions initIoVers)
                 if (tailAddr == 0L)
                     continue;
 
-                if (stripe.empty) {
+                if (stripe.empty || stripe.tailId != tailId) {
                     // Another thread took the last page.
                     writeUnlock(tailId, tailPage, tailAddr, false);
 
@@ -1045,8 +1056,10 @@ protected final long takeEmptyPage(int bucket, @Nullable IOVersions initIoVers)
                         return 0L;
                 }
 
-                assert PageIO.getPageId(tailAddr) == tailId : "tailId = " + tailId + ", tailPageId = " + PageIO.getPageId(tailAddr);
-                assert PageIO.getType(tailAddr) == PageIO.T_PAGE_LIST_NODE;
+                assert PageIO.getPageId(tailAddr) == tailId
+                    : "tailId = " + U.hexLong(tailId) + ", pageId = " + U.hexLong(PageIO.getPageId(tailAddr));
+                assert PageIO.getType(tailAddr) == PageIO.T_PAGE_LIST_NODE
+                    : "tailId = " + U.hexLong(tailId) + ", type = " + PageIO.getType(tailAddr);
 
                 boolean dirty = false;
                 long dataPageId;

From a8c74bada9b4f3c74fa92279a3b3d6ade3dc1549 Mon Sep 17 00:00:00 2001
From: ezagumennov <ezagumennov@gridgain.com>
Date: Tue, 19 Jun 2018 18:04:34 +0300
Subject: [PATCH 217/543] IGNITE-8798 Move transaction recovery logging to INFO
 level

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 49a565c)
Signed-off-by: EdShangGG <eshangareev@gridgain.com>
---
 .../GridCacheTxRecoveryFuture.java            | 112 +++++++++---------
 .../cache/transactions/IgniteTxHandler.java   |   8 +-
 .../cache/transactions/IgniteTxManager.java   |  20 ++--
 3 files changed, 72 insertions(+), 68 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java
index d6b45b342fc18..3eeb0dbe45fcb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryFuture.java
@@ -120,8 +120,8 @@ public GridCacheTxRecoveryFuture(GridCacheSharedContext<?, ?> cctx,
 
                 if (node != null)
                     nodes.put(node.id(), node);
-                else if (log.isDebugEnabled())
-                    log.debug("Transaction node left (will ignore) " + e.getKey());
+                else if (log.isInfoEnabled())
+                    log.info("Transaction node left (will ignore) " + e.getKey());
             }
 
             for (UUID nodeId : e.getValue()) {
@@ -130,8 +130,8 @@ else if (log.isDebugEnabled())
 
                     if (node != null)
                         nodes.put(node.id(), node);
-                    else if (log.isDebugEnabled())
-                        log.debug("Transaction node left (will ignore) " + e.getKey());
+                    else if (log.isInfoEnabled())
+                        log.info("Transaction node left (will ignore) " + e.getKey());
                 }
             }
         }
@@ -179,21 +179,21 @@ public void prepare() {
                 try {
                     cctx.io().send(nearNodeId, req, tx.ioPolicy());
 
-                    if (msgLog.isDebugEnabled()) {
-                        msgLog.debug("Recovery fut, sent request near tx [txId=" + tx.nearXidVersion() +
-                            ", dhtTxId=" + tx.xidVersion() +
-                            ", node=" + nearNodeId + ']');
+                    if (msgLog.isInfoEnabled()) {
+                        msgLog.info("Recovery fut, sent request near tx [txId=" + tx.nearXidVersion() +
+                                ", dhtTxId=" + tx.xidVersion() +
+                                ", node=" + nearNodeId + ']');
                     }
                 }
                 catch (ClusterTopologyCheckedException ignore) {
                     fut.onNodeLeft(nearNodeId);
                 }
                 catch (IgniteCheckedException e) {
-                    if (msgLog.isDebugEnabled()) {
-                        msgLog.debug("Recovery fut, failed to send request near tx [txId=" + tx.nearXidVersion() +
-                            ", dhtTxId=" + tx.xidVersion() +
-                            ", node=" + nearNodeId +
-                            ", err=" + e + ']');
+                    if (msgLog.isInfoEnabled()) {
+                        msgLog.info("Recovery fut, failed to send request near tx [txId=" + tx.nearXidVersion() +
+                                ", dhtTxId=" + tx.xidVersion() +
+                                ", node=" + nearNodeId +
+                                ", err=" + e + ']');
                     }
 
                     fut.onError(e);
@@ -298,21 +298,21 @@ private void proceedPrepare() {
                     try {
                         cctx.io().send(id, req, tx.ioPolicy());
 
-                        if (msgLog.isDebugEnabled()) {
-                            msgLog.debug("Recovery fut, sent request to backup [txId=" + tx.nearXidVersion() +
-                                ", dhtTxId=" + tx.xidVersion() +
-                                ", node=" + id + ']');
+                        if (msgLog.isInfoEnabled()) {
+                            msgLog.info("Recovery fut, sent request to backup [txId=" + tx.nearXidVersion() +
+                                    ", dhtTxId=" + tx.xidVersion() +
+                                    ", node=" + id + ']');
                         }
                     }
                     catch (ClusterTopologyCheckedException ignored) {
                         fut.onNodeLeft(id);
                     }
                     catch (IgniteCheckedException e) {
-                        if (msgLog.isDebugEnabled()) {
-                            msgLog.debug("Recovery fut, failed to send request to backup [txId=" + tx.nearXidVersion() +
-                                ", dhtTxId=" + tx.xidVersion() +
-                                ", node=" + id +
-                                ", err=" + e + ']');
+                        if (msgLog.isInfoEnabled()) {
+                            msgLog.info("Recovery fut, failed to send request to backup [txId=" + tx.nearXidVersion() +
+                                    ", dhtTxId=" + tx.xidVersion() +
+                                    ", node=" + id +
+                                    ", err=" + e + ']');
                         }
 
                         fut.onError(e);
@@ -337,21 +337,21 @@ private void proceedPrepare() {
                 try {
                     cctx.io().send(nodeId, req, tx.ioPolicy());
 
-                    if (msgLog.isDebugEnabled()) {
-                        msgLog.debug("Recovery fut, sent request to primary [txId=" + tx.nearXidVersion() +
-                            ", dhtTxId=" + tx.xidVersion() +
-                            ", node=" + nodeId + ']');
+                    if (msgLog.isInfoEnabled()) {
+                        msgLog.info("Recovery fut, sent request to primary [txId=" + tx.nearXidVersion() +
+                                ", dhtTxId=" + tx.xidVersion() +
+                                ", node=" + nodeId + ']');
                     }
                 }
                 catch (ClusterTopologyCheckedException ignored) {
                     fut.onNodeLeft(nodeId);
                 }
                 catch (IgniteCheckedException e) {
-                    if (msgLog.isDebugEnabled()) {
-                        msgLog.debug("Recovery fut, failed to send request to primary [txId=" + tx.nearXidVersion() +
-                            ", dhtTxId=" + tx.xidVersion() +
-                            ", node=" + nodeId +
-                            ", err=" + e + ']');
+                    if (msgLog.isInfoEnabled()) {
+                        msgLog.info("Recovery fut, failed to send request to primary [txId=" + tx.nearXidVersion() +
+                                ", dhtTxId=" + tx.xidVersion() +
+                                ", node=" + nodeId +
+                                ", err=" + e + ']');
                     }
 
                     fut.onError(e);
@@ -398,22 +398,24 @@ public void onResult(UUID nodeId, GridCacheTxRecoveryResponse res) {
                 mini.onResult(res);
             }
             else {
-                if (msgLog.isDebugEnabled()) {
-                    msgLog.debug("Tx recovery fut, failed to find mini future [txId=" + tx.nearXidVersion() +
+                if (msgLog.isInfoEnabled()) {
+                    msgLog.info("Tx recovery fut, failed to find mini future [txId=" + tx.nearXidVersion() +
+                            ", dhtTxId=" + tx.xidVersion() +
+                            ", node=" + nodeId +
+                            ", res=" + res +
+                            ", fut=" + this + ']');
+                }
+            }
+        }
+        else {
+            if (msgLog.isInfoEnabled()) {
+                msgLog.info("Tx recovery fut, response for finished future [txId=" + tx.nearXidVersion() +
                         ", dhtTxId=" + tx.xidVersion() +
                         ", node=" + nodeId +
                         ", res=" + res +
                         ", fut=" + this + ']');
-                }
             }
         }
-        else {
-            msgLog.debug("Tx recovery fut, response for finished future [txId=" + tx.nearXidVersion() +
-                ", dhtTxId=" + tx.xidVersion() +
-                ", node=" + nodeId +
-                ", res=" + res +
-                ", fut=" + this + ']');
-        }
     }
 
     /**
@@ -502,14 +504,16 @@ public IgniteInternalTx tx() {
             }
             else {
                 if (err instanceof ClusterTopologyCheckedException && nearTxCheck) {
-                    if (log.isDebugEnabled())
-                        log.debug("Failed to check transaction on near node, " +
-                            "ignoring [err=" + err + ", tx=" + tx + ']');
+                    if (log.isInfoEnabled()) {
+                        log.info("Failed to check transaction on near node, " +
+                                "ignoring [err=" + err + ", tx=" + tx + ']');
+                    }
                 }
                 else {
-                    if (log.isDebugEnabled())
-                        log.debug("Failed to check prepared transactions, " +
-                            "invalidating transaction [err=" + err + ", tx=" + tx + ']');
+                    if (log.isInfoEnabled()) {
+                        log.info("Failed to check prepared transactions, " +
+                                "invalidating transaction [err=" + err + ", tx=" + tx + ']');
+                    }
 
                     cctx.tm().salvageTx(tx);
                 }
@@ -577,8 +581,8 @@ private IgniteUuid futureId() {
          * @param e Error.
          */
         private void onError(Throwable e) {
-            if (log.isDebugEnabled())
-                log.debug("Failed to get future result [fut=" + this + ", err=" + e + ']');
+            if (log.isInfoEnabled())
+                log.info("Failed to get future result [fut=" + this + ", err=" + e + ']');
 
             onDone(e);
         }
@@ -587,11 +591,11 @@ private void onError(Throwable e) {
          * @param nodeId Failed node ID.
          */
         private void onNodeLeft(UUID nodeId) {
-            if (msgLog.isDebugEnabled()) {
-                msgLog.debug("Tx recovery fut, mini future node left [txId=" + tx.nearXidVersion() +
-                    ", dhtTxId=" + tx.xidVersion() +
-                    ", node=" + nodeId +
-                    ", nearTxCheck=" + nearTxCheck + ']');
+            if (msgLog.isInfoEnabled()) {
+                msgLog.info("Tx recovery fut, mini future node left [txId=" + tx.nearXidVersion() +
+                        ", dhtTxId=" + tx.xidVersion() +
+                        ", node=" + nodeId +
+                        ", nearTxCheck=" + nearTxCheck + ']');
             }
 
             if (nearTxCheck) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 44482abf0c8b7..6bfa1e8c61979 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -1877,8 +1877,8 @@ private void sendCheckPreparedResponse(UUID nodeId,
      * @param res Response.
      */
     protected void processCheckPreparedTxResponse(UUID nodeId, GridCacheTxRecoveryResponse res) {
-        if (txRecoveryMsgLog.isDebugEnabled()) {
-            txRecoveryMsgLog.debug("Received tx recovery response [txId=" + res.version() +
+        if (txRecoveryMsgLog.isInfoEnabled()) {
+            txRecoveryMsgLog.info("Received tx recovery response [txId=" + res.version() +
                 ", node=" + nodeId +
                 ", res=" + res + ']');
         }
@@ -1886,8 +1886,8 @@ protected void processCheckPreparedTxResponse(UUID nodeId, GridCacheTxRecoveryRe
         GridCacheTxRecoveryFuture fut = (GridCacheTxRecoveryFuture)ctx.mvcc().future(res.futureId());
 
         if (fut == null) {
-            if (txRecoveryMsgLog.isDebugEnabled()) {
-                txRecoveryMsgLog.debug("Failed to find future for tx recovery response [txId=" + res.version() +
+            if (txRecoveryMsgLog.isInfoEnabled()) {
+                txRecoveryMsgLog.info("Failed to find future for tx recovery response [txId=" + res.version() +
                     ", node=" + nodeId + ", res=" + res + ']');
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index dc8bc4668c41f..d15dbdcdb7238 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -372,16 +372,16 @@ private void salvageTx(IgniteInternalTx tx, IgniteInternalTx.FinalizationStatus
 
         if (state == ACTIVE || state == PREPARING || state == PREPARED || state == MARKED_ROLLBACK) {
             if (!tx.markFinalizing(status)) {
-                if (log.isDebugEnabled())
-                    log.debug("Will not try to commit invalidate transaction (could not mark finalized): " + tx);
+                if (log.isInfoEnabled())
+                    log.info("Will not try to commit invalidate transaction (could not mark finalized): " + tx);
 
                 return;
             }
 
             tx.salvageTx();
 
-            if (log.isDebugEnabled())
-                log.debug("Invalidated transaction because originating node left grid: " + CU.txString(tx));
+            if (log.isInfoEnabled())
+                log.info("Invalidated transaction because originating node left grid: " + CU.txString(tx));
         }
     }
 
@@ -2001,12 +2001,12 @@ public IgniteInternalFuture<?> remoteTxFinishFuture(GridCacheVersion nearVer) {
      * @param commit Whether transaction should be committed or rolled back.
      */
     public void finishTxOnRecovery(final IgniteInternalTx tx, boolean commit) {
-        if (log.isDebugEnabled())
-            log.debug("Finishing prepared transaction [tx=" + tx + ", commit=" + commit + ']');
+        if (log.isInfoEnabled())
+            log.info("Finishing prepared transaction [tx=" + tx + ", commit=" + commit + ']');
 
         if (!tx.markFinalizing(RECOVERY_FINISH)) {
-            if (log.isDebugEnabled())
-                log.debug("Will not try to commit prepared transaction (could not mark finalized): " + tx);
+            if (log.isInfoEnabled())
+                log.info("Will not try to commit prepared transaction (could not mark finalized): " + tx);
 
             return;
         }
@@ -2046,8 +2046,8 @@ public void commitIfPrepared(IgniteInternalTx tx, Set<UUID> failedNodeIds) {
 
         cctx.mvcc().addFuture(fut, fut.futureId());
 
-        if (log.isDebugEnabled())
-            log.debug("Checking optimistic transaction state on remote nodes [tx=" + tx + ", fut=" + fut + ']');
+        if (log.isInfoEnabled())
+            log.info("Checking optimistic transaction state on remote nodes [tx=" + tx + ", fut=" + fut + ']');
 
         fut.prepare();
     }

From 4a5d53a4ba2886e9e54dbf6a55f07d0d4b4dbf5d Mon Sep 17 00:00:00 2001
From: Alexander Menshikov <sharplermc@gmail.com>
Date: Wed, 25 Apr 2018 14:03:24 +0300
Subject: [PATCH 218/543] IGNITE-6565 Use long type for size and keySize in
 cache metrics

Signed-off-by: Anton Vinogradov <av@apache.org>

(cherry picked from commit 0dc906f)

(cherry picked from commit 687d5f8)
---
 .../org/apache/ignite/cache/CacheMetrics.java | 11 ++++++
 .../cache/CacheClusterMetricsMXBeanImpl.java  |  5 +++
 .../cache/CacheLocalMetricsMXBeanImpl.java    |  5 +++
 .../processors/cache/CacheMetricsImpl.java    | 30 +++++++++++++-
 .../cache/CacheMetricsSnapshot.java           | 10 +++++
 .../platform/cache/PlatformCache.java         |  1 +
 .../visor/cache/VisorCacheMetrics.java        | 17 ++++++++
 .../ignite/mxbean/CacheMetricsMXBean.java     |  4 ++
 .../GridCacheAbstractMetricsSelfTest.java     | 39 +++++++++++++++++++
 .../PlatformCacheWriteMetricsTask.java        |  5 +++
 .../Cache/CacheMetricsTest.cs                 |  7 ++++
 .../Apache.Ignite.Core/Cache/ICacheMetrics.cs |  8 ++++
 .../Impl/Cache/CacheMetricsImpl.cs            |  7 ++++
 13 files changed, 148 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
index 0b1cb87c3a301..eae7989b0e8c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
@@ -240,14 +240,25 @@ public interface CacheMetrics {
      * Gets number of non-{@code null} values in the cache.
      *
      * @return Number of non-{@code null} values in the cache.
+     * @deprecated Can overflow. Use {@link CacheMetrics#getCacheSize()} instead.
      */
+    @Deprecated
     public int getSize();
 
+    /**
+     * Gets number of non-{@code null} values in the cache as a long value.
+     *
+     * @return Number of non-{@code null} values in the cache.
+     */
+    public long getCacheSize();
+
     /**
      * Gets number of keys in the cache, possibly with {@code null} values.
      *
      * @return Number of keys in the cache.
+     * @deprecated Can overflow. Use {@link CacheMetrics#getCacheSize()} instead.
      */
+    @Deprecated
     public int getKeySize();
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java
index ce6416fa0c766..32603cbf940d5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java
@@ -114,6 +114,11 @@ class CacheClusterMetricsMXBeanImpl implements CacheMetricsMXBean {
         return cache.clusterMetrics().getSize();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getCacheSize() {
+        return cache.clusterMetrics().getCacheSize();
+    }
+
     /** {@inheritDoc} */
     @Override public int getKeySize() {
         return cache.clusterMetrics().getKeySize();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java
index 438c8c666c01b..d3060d3ab3835 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java
@@ -115,6 +115,11 @@ class CacheLocalMetricsMXBeanImpl implements CacheMetricsMXBean {
         return cache.metrics0().getSize();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getCacheSize() {
+        return cache.metrics0().getCacheSize();
+    }
+
     /** {@inheritDoc} */
     @Override public int getKeySize() {
         return cache.metrics0().getKeySize();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
index b402ff2bb9170..96f40bfd72ea3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
@@ -254,6 +254,11 @@ public void delegate(CacheMetricsImpl delegate) {
         return getEntriesStat().size();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getCacheSize() {
+        return getEntriesStat().cacheSize();
+    }
+
     /** {@inheritDoc} */
     @Override public int getKeySize() {
         return getEntriesStat().keySize();
@@ -754,6 +759,7 @@ public EntriesStatMetrics getEntriesStat() {
         long offHeapBackupEntriesCnt = 0L;
         long heapEntriesCnt = 0L;
         int size = 0;
+        long sizeLong = 0L;
         boolean isEmpty;
 
         try {
@@ -765,8 +771,9 @@ public EntriesStatMetrics getEntriesStat() {
                     offHeapBackupEntriesCnt = offHeapEntriesCnt;
 
                     size = cctx.cache().size();
+                    sizeLong = cctx.cache().sizeLong();
 
-                    heapEntriesCnt = size;
+                    heapEntriesCnt = sizeLong;
                 }
             }
             else {
@@ -806,6 +813,8 @@ public EntriesStatMetrics getEntriesStat() {
 
                     heapEntriesCnt += part.publicSize(cctx.cacheId());
                 }
+
+                sizeLong = offHeapEntriesCnt;
             }
         }
         catch (Exception e) {
@@ -816,6 +825,7 @@ public EntriesStatMetrics getEntriesStat() {
             offHeapBackupEntriesCnt = -1L;
             heapEntriesCnt = -1L;
             size = -1;
+            sizeLong = -1L;
         }
 
         isEmpty = (offHeapEntriesCnt == 0);
@@ -827,6 +837,7 @@ public EntriesStatMetrics getEntriesStat() {
         stat.offHeapBackupEntriesCount(offHeapBackupEntriesCnt);
         stat.heapEntriesCount(heapEntriesCnt);
         stat.size(size);
+        stat.cacheSize(sizeLong);
         stat.keySize(size);
         stat.isEmpty(isEmpty);
         stat.totalPartitionsCount(owningPartCnt + movingPartCnt);
@@ -1039,6 +1050,9 @@ public static class EntriesStatMetrics {
         /** Size. */
         private int size;
 
+        /** Long size. */
+        private long cacheSize;
+
         /** Key size. */
         private int keySize;
 
@@ -1157,6 +1171,20 @@ public void keySize(int keySize) {
             this.keySize = keySize;
         }
 
+        /**
+         * @return Long size.
+         */
+        public long cacheSize() {
+            return cacheSize;
+        }
+
+        /**
+         * @param cacheSize Size long.
+         */
+        public void cacheSize(long cacheSize) {
+            this.cacheSize = cacheSize;
+        }
+
         /**
          * @return Is empty.
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java
index e69372001f215..539ad59a818a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java
@@ -110,6 +110,9 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable {
     /** Number of non-{@code null} values in the cache. */
     private int size;
 
+    /** Number of non-{@code null} values in the cache as long value as a long value. */
+    private long cacheSize;
+
     /** Number of keys in the cache, possibly with {@code null} values. */
     private int keySize;
 
@@ -286,6 +289,7 @@ public CacheMetricsSnapshot(CacheMetricsImpl m) {
         offHeapAllocatedSize = m.getOffHeapAllocatedSize();
 
         size = entriesStat.size();
+        cacheSize = entriesStat.cacheSize();
         keySize = entriesStat.keySize();
         isEmpty = entriesStat.isEmpty();
 
@@ -351,6 +355,7 @@ public CacheMetricsSnapshot(CacheMetrics loc, Collection<CacheMetrics> metrics)
         writeBehindStoreBatchSize = loc.getWriteBehindStoreBatchSize();
         writeBehindBufSize = loc.getWriteBehindBufferSize();
         size = loc.getSize();
+        cacheSize = loc.getCacheSize();
         keySize = loc.getKeySize();
 
         keyType = loc.getKeyType();
@@ -633,6 +638,11 @@ public CacheMetricsSnapshot(CacheMetrics loc, Collection<CacheMetrics> metrics)
         return size;
     }
 
+    /** {@inheritDoc} */
+    @Override public long getCacheSize() {
+        return cacheSize;
+    }
+
     /** {@inheritDoc} */
     @Override public int getKeySize() {
         return keySize;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
index 19dac83758d23..3a11fc535a661 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
@@ -1483,6 +1483,7 @@ public static void writeCacheMetrics(BinaryRawWriter writer, CacheMetrics metric
         writer.writeLong(metrics.getEstimatedRebalancingFinishTime());
         writer.writeLong(metrics.getRebalancingStartTime());
         writer.writeLong(metrics.getRebalanceClearingPartitionsLeft());
+        writer.writeLong(metrics.getCacheSize());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java
index 5d8bc8151cfbf..59f16b2a5e7cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java
@@ -55,6 +55,9 @@ public class VisorCacheMetrics extends VisorDataTransferObject {
     /** Gets number of keys in the cache, possibly with {@code null} values. */
     private int keySize;
 
+    /** Number of non-{@code null} values in the cache as a long value. */
+    private long cacheSize;
+
     /** Total number of reads of the owning entity (either cache or entry). */
     private long reads;
 
@@ -224,6 +227,8 @@ public VisorCacheMetrics(IgniteEx ignite, String cacheName) {
         size = m.getSize();
         keySize = m.getKeySize();
 
+        cacheSize = m.getCacheSize();
+
         reads = m.getCacheGets();
         writes = m.getCachePuts() + m.getCacheRemovals();
         hits = m.getCacheHits();
@@ -456,6 +461,13 @@ public int getKeySize() {
         return keySize;
     }
 
+    /**
+     * @return Number of non-{@code null} values in the cache as a long value.
+     */
+    public long getCacheSize() {
+        return cacheSize;
+    }
+
     /**
      * @return Gets query metrics for cache.
      */
@@ -694,6 +706,8 @@ public long getRebalancingBytesRate() {
         out.writeLong(rebalancingBytesRate);
 
         out.writeObject(qryMetrics);
+
+        out.writeLong(cacheSize);
     }
 
     /** {@inheritDoc} */
@@ -751,6 +765,9 @@ public long getRebalancingBytesRate() {
         rebalancingBytesRate = in.readLong();
 
         qryMetrics = (VisorQueryMetrics)in.readObject();
+
+        if (in.available() > 0)
+            cacheSize = in.readLong();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/CacheMetricsMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/CacheMetricsMXBean.java
index ae03a5a8fc91f..16bdedeb2c471 100644
--- a/modules/core/src/main/java/org/apache/ignite/mxbean/CacheMetricsMXBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/CacheMetricsMXBean.java
@@ -151,6 +151,10 @@ public interface CacheMetricsMXBean extends CacheStatisticsMXBean, CacheMXBean,
     @MXBeanDescription("Number of non-null values in the cache.")
     public int getSize();
 
+    /** {@inheritDoc} */
+    @MXBeanDescription("Number of non-null values in the cache as a long value.")
+    public long getCacheSize();
+
     /** {@inheritDoc} */
     @MXBeanDescription("Number of keys in the cache (possibly with null values).")
     public int getKeySize();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java
index e20715826e7b9..7948569bf9327 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractMetricsSelfTest.java
@@ -674,6 +674,45 @@ public void testRemoves() throws Exception {
         assertEquals(1L, cache.localMetrics().getCacheRemovals());
     }
 
+    /**
+     * Test {@link CacheMetrics#getSize()} and {@link CacheMetrics#getCacheSize()} work equally.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCacheSizeWorksAsSize() throws Exception {
+        IgniteCache<Integer, Integer> cache = grid(0).cache(DEFAULT_CACHE_NAME);
+
+        assertEquals(cache.metrics().getSize(), cache.metrics().getCacheSize());
+
+        for (int i = 0; i < KEY_CNT; i++) {
+            cache.put(i, i);
+
+            CacheMetrics metrics = cache.metrics();
+
+            assertEquals(metrics.getSize(), metrics.getCacheSize());
+
+            CacheMetrics localMetrics = cache.localMetrics();
+
+            assertEquals(localMetrics.getSize(), localMetrics.getCacheSize());
+        }
+
+        for (int i = 0; i < KEY_CNT / 2; i++) {
+            cache.remove(i, i);
+
+            CacheMetrics metrics = cache.metrics();
+
+            assertEquals(metrics.getSize(), metrics.getCacheSize());
+
+            CacheMetrics localMetrics = cache.localMetrics();
+
+            assertEquals(localMetrics.getSize(), localMetrics.getCacheSize());
+        }
+
+        cache.removeAll();
+
+        assertEquals(cache.metrics().getSize(), cache.metrics().getCacheSize());
+    }
+
     /**
      * @throws Exception If failed.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java b/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java
index 3b7bb8905c2eb..44a15d500ca44 100644
--- a/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java
+++ b/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java
@@ -468,5 +468,10 @@ private static class TestCacheMetrics implements CacheMetrics {
         @Override public long getRebalanceClearingPartitionsLeft() {
             return 64;
         }
+
+        /** {@inheritDoc} */
+        @Override public long getCacheSize() {
+            return 65;
+        }
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheMetricsTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheMetricsTest.cs
index 3f671d9d71150..129b4b5de662e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheMetricsTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Cache/CacheMetricsTest.cs
@@ -67,10 +67,12 @@ public void TestLocalMetrics()
             var remoteMetrics = metrics.Item2;
 
             Assert.AreEqual(1, localMetrics.Size);
+            Assert.AreEqual(1, localMetrics.CacheSize);
             Assert.AreEqual(1, localMetrics.CacheGets);
             Assert.AreEqual(1, localMetrics.CachePuts);
 
             Assert.AreEqual(0, remoteMetrics.Size);
+            Assert.AreEqual(0, remoteMetrics.CacheSize);
             Assert.AreEqual(0, remoteMetrics.CacheGets);
             Assert.AreEqual(0, remoteMetrics.CachePuts);
         }
@@ -87,10 +89,12 @@ public void TestGlobalMetrics()
             var remoteMetrics = metrics.Item2;
 
             Assert.AreEqual(1, localMetrics.Size);
+            Assert.AreEqual(1, localMetrics.CacheSize);
             Assert.AreEqual(1, localMetrics.CacheGets);
             Assert.AreEqual(1, localMetrics.CachePuts);
 
             Assert.AreEqual(0, remoteMetrics.Size);
+            Assert.AreEqual(0, remoteMetrics.CacheSize);
             Assert.AreEqual(1, remoteMetrics.CacheGets);
             Assert.AreEqual(1, remoteMetrics.CachePuts);
         }
@@ -111,10 +115,12 @@ public void TestClusterGroupMetrics()
             var remoteMetrics = metrics.Item1;
 
             Assert.AreEqual(1, localMetrics.Size);
+            Assert.AreEqual(1, localMetrics.CacheSize);
             Assert.AreEqual(1, localMetrics.CacheGets);
             Assert.AreEqual(1, localMetrics.CachePuts);
 
             Assert.AreEqual(1, remoteMetrics.Size);
+            Assert.AreEqual(1, remoteMetrics.CacheSize);
             Assert.AreEqual(0, remoteMetrics.CacheGets);
             Assert.AreEqual(0, remoteMetrics.CachePuts);
         }
@@ -203,6 +209,7 @@ public void TestMetricsPropagation()
                 Assert.AreEqual(59, metrics.HeapEntriesCount);
                 Assert.AreEqual(62, metrics.EstimatedRebalancingFinishTime);
                 Assert.AreEqual(63, metrics.RebalancingStartTime);
+                Assert.AreEqual(65, metrics.CacheSize);
                 Assert.AreEqual("foo", metrics.KeyType);
                 Assert.AreEqual("bar", metrics.ValueType);
                 Assert.AreEqual(true, metrics.IsStoreByValue);
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
index a328cf5adc8ac..d775c05691076 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
@@ -256,6 +256,14 @@ public interface ICacheMetrics
         /// </returns>
         int Size { get; }
 
+        /// <summary>
+        /// Gets number of non-null values in the cache.
+        /// </summary>
+        /// <returns>
+        /// Number of non-null values in the cache.
+        /// </returns>
+        long CacheSize { get; }
+
         /// <summary>
         /// Gets number of keys in the cache, possibly with null values.
         /// </summary>
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
index fbc5d4cbba7cd..1fdc8771f832e 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
@@ -115,6 +115,9 @@ internal class CacheMetricsImpl : ICacheMetrics
         /** */
         private readonly int _keySize;
 
+        /** */
+        private readonly long _cacheSize;
+
         /** */
         private readonly bool _isEmpty;
 
@@ -323,6 +326,7 @@ public CacheMetricsImpl(IBinaryRawReader reader)
             _estimatedRebalancingFinishTime = reader.ReadLong();
             _rebalancingStartTime = reader.ReadLong();
             _rebalancingClearingPartitionsLeft = reader.ReadLong();
+            _cacheSize = reader.ReadLong();
         }
 
         /** <inheritDoc /> */
@@ -412,6 +416,9 @@ public CacheMetricsImpl(IBinaryRawReader reader)
         /** <inheritDoc /> */
         public int Size { get { return _size; } }
 
+        /** <inheritDoc /> */
+        public long CacheSize { get { return _cacheSize; } }
+
         /** <inheritDoc /> */
         public int KeySize { get { return _keySize; } }
 

From 9eb7f1cb8b62f9a85291a15cd24579625b5da0d6 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Tue, 19 Jun 2018 19:33:24 +0300
Subject: [PATCH 219/543] IGNITE-8554 Cache metrics: expose metrics with
 rebalance info about keys - Fixes #4094.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit cf09e76)

(cherry picked from commit b4b4764)
---
 .../org/apache/ignite/cache/CacheMetrics.java | 10 +++
 .../cache/CacheAffinitySharedManager.java     |  3 +-
 .../cache/CacheClusterMetricsMXBeanImpl.java  | 10 +++
 .../cache/CacheLocalMetricsMXBeanImpl.java    |  8 ++
 .../processors/cache/CacheMetricsImpl.java    | 15 +++-
 .../cache/CacheMetricsSnapshot.java           | 30 +++++++
 .../GridCachePartitionExchangeManager.java    | 13 ++-
 .../dht/GridClientPartitionTopology.java      | 35 ++++++++
 .../dht/GridDhtPartitionTopology.java         | 11 +++
 .../dht/GridDhtPartitionTopologyImpl.java     | 34 ++++++++
 .../preloader/GridDhtPartitionDemander.java   | 28 +++++--
 .../GridDhtPartitionSupplyMessage.java        |  9 +-
 .../GridDhtPartitionsExchangeFuture.java      | 61 ++++++++++++--
 .../GridDhtPartitionsFullMessage.java         | 80 ++++++++++++++++--
 .../GridDhtPartitionsSingleMessage.java       | 24 +++---
 .../platform/cache/PlatformCache.java         |  2 +
 .../visor/cache/VisorCacheMetrics.java        | 35 ++++++++
 .../visor/node/VisorNodeDataCollectorJob.java | 20 ++++-
 .../CacheGroupsMetricsRebalanceTest.java      | 82 +++++++++++--------
 .../PlatformCacheWriteMetricsTask.java        | 10 +++
 .../Apache.Ignite.Core/Cache/ICacheMetrics.cs | 18 ++++
 .../Impl/Cache/CacheMetricsImpl.cs            | 14 ++++
 22 files changed, 468 insertions(+), 84 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
index eae7989b0e8c9..15e56c789c946 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/CacheMetrics.java
@@ -501,6 +501,16 @@ public interface CacheMetrics {
      */
     public int getRebalancingPartitionsCount();
 
+    /**
+     * @return Number of already rebalanced keys.
+     */
+    public long getRebalancedKeys();
+
+    /**
+     * @return Number estimated to rebalance keys.
+     */
+    public long getEstimatedRebalancingKeys();
+
     /**
      * @return Estimated number of keys to be rebalanced on current node.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 92b8d3ea87019..08eb43f5291c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -472,6 +472,7 @@ void onCacheGroupCreated(CacheGroupContext grp) {
                                     clientTop.partitionMap(true),
                                     clientTop.fullUpdateCounters(),
                                     Collections.<Integer>emptySet(),
+                                    null,
                                     null);
                             }
 
@@ -530,7 +531,7 @@ else if (!fetchFuts.containsKey(grp.groupId())) {
 
                 grp.topology().updateTopologyVersion(topFut, discoCache, -1, false);
 
-                grp.topology().update(topVer, partMap, null, Collections.<Integer>emptySet(), null);
+                grp.topology().update(topVer, partMap, null, Collections.<Integer>emptySet(), null, null);
 
                 topFut.validate(grp, discoCache.allNodes());
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java
index 32603cbf940d5..3d5278cc96496 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClusterMetricsMXBeanImpl.java
@@ -369,6 +369,16 @@ class CacheClusterMetricsMXBeanImpl implements CacheMetricsMXBean {
         return cache.clusterMetrics().getTotalPartitionsCount();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getRebalancedKeys() {
+        return cache.clusterMetrics().getRebalancedKeys();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getEstimatedRebalancingKeys() {
+        return cache.clusterMetrics().getEstimatedRebalancingKeys();
+    }
+
     /** {@inheritDoc} */
     @Override public int getRebalancingPartitionsCount() {
         return cache.clusterMetrics().getRebalancingPartitionsCount();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java
index d3060d3ab3835..212c7a07c4620 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheLocalMetricsMXBeanImpl.java
@@ -370,6 +370,14 @@ class CacheLocalMetricsMXBeanImpl implements CacheMetricsMXBean {
         return cache.metrics0().getTotalPartitionsCount();
     }
 
+    @Override public long getRebalancedKeys() {
+        return cache.metrics0().getRebalancedKeys();
+    }
+
+    @Override public long getEstimatedRebalancingKeys() {
+        return cache.metrics0().getEstimatedRebalancingKeys();
+    }
+
     /** {@inheritDoc} */
     @Override public int getRebalancingPartitionsCount() {
         return cache.metrics0().getRebalancingPartitionsCount();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
index 96f40bfd72ea3..0f6d06f7edd56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
@@ -856,6 +856,16 @@ public EntriesStatMetrics getEntriesStat() {
         return getEntriesStat().rebalancingPartitionsCount();
     }
 
+    /** {@inheritDoc} */
+    @Override public long getRebalancedKeys() {
+        return rebalancedKeys.get();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getEstimatedRebalancingKeys() {
+        return estimatedRebalancingKeys.get();
+    }
+
     /** {@inheritDoc} */
     @Override public long getKeysToRebalanceLeft() {
         return Math.max(0, estimatedRebalancingKeys.get() - rebalancedKeys.get());
@@ -935,7 +945,10 @@ public void rebalanceClearingPartitions(int partitions) {
      * First rebalance supply message callback.
      * @param keysCnt Estimated number of keys.
      */
-    public void onRebalancingKeysCountEstimateReceived(long keysCnt) {
+    public void onRebalancingKeysCountEstimateReceived(Long keysCnt) {
+        if (keysCnt == null)
+            return;
+
         estimatedRebalancingKeys.addAndGet(keysCnt);
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java
index 539ad59a818a1..8a0f0e4b326ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsSnapshot.java
@@ -197,6 +197,12 @@ public class CacheMetricsSnapshot implements CacheMetrics, Externalizable {
     /** Rebalancing partitions count. */
     private int rebalancingPartitionsCnt;
 
+    /** Number of already rebalanced keys. */
+    private long rebalancedKeys;
+
+    /** Number estimated to rebalance keys. */
+    private long estimatedRebalancingKeys;
+
     /** Keys to rebalance left. */
     private long keysToRebalanceLeft;
 
@@ -331,6 +337,8 @@ public CacheMetricsSnapshot(CacheMetricsImpl m) {
         totalPartitionsCnt = entriesStat.totalPartitionsCount();
         rebalancingPartitionsCnt = entriesStat.rebalancingPartitionsCount();
 
+        rebalancedKeys = m.getRebalancedKeys();
+        estimatedRebalancingKeys = m.getEstimatedRebalancingKeys();
         keysToRebalanceLeft = m.getKeysToRebalanceLeft();
         rebalancingBytesRate = m.getRebalancingBytesRate();
         rebalancingKeysRate = m.getRebalancingKeysRate();
@@ -459,6 +467,8 @@ public CacheMetricsSnapshot(CacheMetrics loc, Collection<CacheMetrics> metrics)
             else
                 writeBehindErrorRetryCnt = -1;
 
+            rebalancedKeys += e.getRebalancedKeys();
+            estimatedRebalancingKeys += e.getEstimatedRebalancingKeys();
             totalPartitionsCnt += e.getTotalPartitionsCount();
             rebalancingPartitionsCnt += e.getRebalancingPartitionsCount();
             keysToRebalanceLeft += e.getKeysToRebalanceLeft();
@@ -733,6 +743,14 @@ public CacheMetricsSnapshot(CacheMetrics loc, Collection<CacheMetrics> metrics)
         return totalPartitionsCnt;
     }
 
+    @Override public long getRebalancedKeys() {
+        return rebalancedKeys;
+    }
+
+    @Override public long getEstimatedRebalancingKeys() {
+        return estimatedRebalancingKeys;
+    }
+
     /** {@inheritDoc} */
     @Override public int getRebalancingPartitionsCount() {
         return rebalancingPartitionsCnt;
@@ -926,6 +944,12 @@ public CacheMetricsSnapshot(CacheMetrics loc, Collection<CacheMetrics> metrics)
         out.writeLong(keysToRebalanceLeft);
         out.writeLong(rebalancingBytesRate);
         out.writeLong(rebalancingKeysRate);
+
+        out.writeLong(rebalancedKeys);
+        out.writeLong(estimatedRebalancingKeys);
+        out.writeLong(rebalanceStartTime);
+        out.writeLong(rebalanceFinishTime);
+        out.writeLong(rebalanceClearingPartitionsLeft);
     }
 
     /** {@inheritDoc} */
@@ -981,5 +1005,11 @@ public CacheMetricsSnapshot(CacheMetrics loc, Collection<CacheMetrics> metrics)
         keysToRebalanceLeft = in.readLong();
         rebalancingBytesRate = in.readLong();
         rebalancingKeysRate = in.readLong();
+
+        rebalancedKeys = in.readLong();
+        estimatedRebalancingKeys = in.readLong();
+        rebalanceStartTime = in.readLong();
+        rebalanceFinishTime = in.readLong();
+        rebalanceClearingPartitionsLeft = in.readLong();
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 38ddaf661de92..715c2907b2506 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -1121,6 +1121,8 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage(
                         affCache.similarAffinityKey());
                 }
 
+                m.addPartitionSizes(grp.groupId(), grp.topology().globalPartSizes());
+
                 if (exchId != null) {
                     CachePartitionFullCountersMap cntrsMap = grp.topology().fullUpdateCounters();
 
@@ -1154,6 +1156,8 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage(
                     m.addPartitionUpdateCounters(top.groupId(), cntrsMap);
                 else
                     m.addPartitionUpdateCounters(top.groupId(), CachePartitionFullCountersMap.toCountersMap(cntrsMap));
+
+                m.addPartitionSizes(top.groupId(), top.globalPartSizes());
             }
         }
 
@@ -1264,9 +1268,9 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(
 
                     m.addPartitionUpdateCounters(grp.groupId(),
                         newCntrMap ? cntrsMap : CachePartitionPartialCountersMap.toCountersMap(cntrsMap));
-
-                    m.addPartitionSizes(grp.groupId(), grp.topology().partitionSizes());
                 }
+
+                m.addPartitionSizes(grp.groupId(), grp.topology().partitionSizes());
             }
         }
 
@@ -1288,9 +1292,9 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(
 
                 m.addPartitionUpdateCounters(top.groupId(),
                     newCntrMap ? cntrsMap : CachePartitionPartialCountersMap.toCountersMap(cntrsMap));
-
-                m.addPartitionSizes(top.groupId(), top.partitionSizes());
             }
+
+            m.addPartitionSizes(top.groupId(), top.partitionSizes());
         }
 
         return m;
@@ -1482,6 +1486,7 @@ else if (!grp.isLocal())
                             entry.getValue(),
                             null,
                             msg.partsToReload(cctx.localNodeId(), grpId),
+                            msg.partitionSizes(grpId),
                             msg.topologyVersion());
                     }
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
index a9c4b35ba48f1..0d8a9f7cdf467 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
@@ -119,6 +119,9 @@ public class GridClientPartitionTopology implements GridDhtPartitionTopology {
     /** */
     private final int parts;
 
+    /** */
+    private volatile Map<Integer, Long> globalPartSizes;
+
     /**
      * @param cctx Context.
      * @param discoCache Discovery data cache.
@@ -711,6 +714,7 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
         GridDhtPartitionFullMap partMap,
         @Nullable CachePartitionFullCountersMap cntrMap,
         Set<Integer> partsToReload,
+        @Nullable Map<Integer, Long> partSizes,
         @Nullable AffinityTopologyVersion msgTopVer) {
         if (log.isDebugEnabled())
             log.debug("Updating full partition map [exchVer=" + exchangeVer + ", parts=" + fullMapString() + ']');
@@ -814,6 +818,9 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
             if (cntrMap != null)
                 this.cntrMap = new CachePartitionFullCountersMap(cntrMap);
 
+            if (partSizes != null)
+                this.globalPartSizes = partSizes;
+
             consistencyCheck();
 
             if (log.isDebugEnabled())
@@ -1203,6 +1210,34 @@ private void removeNode(UUID nodeId) {
         return Collections.emptyMap();
     }
 
+    /** {@inheritDoc} */
+    @Override @Nullable public Map<Integer, Long> globalPartSizes() {
+        lock.readLock().lock();
+
+        try {
+            if (globalPartSizes == null)
+                return Collections.emptyMap();
+
+            return Collections.unmodifiableMap(globalPartSizes);
+        }
+        finally {
+            lock.readLock().unlock();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void globalPartSizes(@Nullable Map<Integer, Long> partSizes) {
+        lock.writeLock().lock();
+
+        try {
+            this.globalPartSizes = partSizes;
+        }
+        finally {
+            lock.writeLock().unlock();
+        }
+    }
+
+
     /** {@inheritDoc} */
     @Override public boolean rebalanceFinished(AffinityTopologyVersion topVer) {
         assert false : "Should not be called on non-affinity node";
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
index 2df2e8960afc9..d77d9c35b265b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
@@ -287,6 +287,7 @@ public boolean update(
         GridDhtPartitionFullMap partMap,
         @Nullable CachePartitionFullCountersMap cntrMap,
         Set<Integer> partsToReload,
+        @Nullable Map<Integer, Long> partSizes,
         @Nullable AffinityTopologyVersion msgTopVer);
 
     /**
@@ -382,6 +383,16 @@ public boolean update(@Nullable GridDhtPartitionExchangeId exchId,
      */
     public void printMemoryStats(int threshold);
 
+    /**
+     * @return Sizes of up-to-date partition versions in topology.
+     */
+    Map<Integer, Long> globalPartSizes();
+
+    /**
+     * @param partSizes Sizes of up-to-date partition versions in topology.
+     */
+    void globalPartSizes(@Nullable Map<Integer, Long> partSizes);
+
     /**
      * @param topVer Topology version.
      * @return {@code True} if rebalance process finished.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 44ec16bb7ee73..c79121987f688 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -135,6 +135,9 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
     /** Partition update counter. */
     private final CachePartitionFullCountersMap cntrMap;
 
+    /** */
+    private volatile Map<Integer, Long> globalPartSizes;
+
     /** */
     private volatile AffinityTopologyVersion rebalancedTopVer = AffinityTopologyVersion.NONE;
 
@@ -1329,6 +1332,7 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
         GridDhtPartitionFullMap partMap,
         @Nullable CachePartitionFullCountersMap incomeCntrMap,
         Set<Integer> partsToReload,
+        @Nullable Map<Integer, Long> partSizes,
         @Nullable AffinityTopologyVersion msgTopVer) {
         if (log.isDebugEnabled()) {
             log.debug("Updating full partition map [grp=" + grp.cacheOrGroupName() + ", exchVer=" + exchangeVer +
@@ -1544,6 +1548,9 @@ else if (state == MOVING) {
                     updateRebalanceVersion(aff.assignment());
                 }
 
+                if (partSizes != null)
+                    this.globalPartSizes = partSizes;
+
                 consistencyCheck();
 
                 if (log.isDebugEnabled()) {
@@ -2596,6 +2603,33 @@ private void removeNode(UUID nodeId) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public Map<Integer, Long> globalPartSizes() {
+        lock.readLock().lock();
+
+        try {
+            if (globalPartSizes == null)
+                return Collections.emptyMap();
+
+            return Collections.unmodifiableMap(globalPartSizes);
+        }
+        finally {
+            lock.readLock().unlock();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void globalPartSizes(@Nullable Map<Integer, Long> partSizes) {
+        lock.writeLock().lock();
+
+        try {
+            this.globalPartSizes = partSizes;
+        }
+        finally {
+            lock.writeLock().unlock();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public boolean rebalanceFinished(AffinityTopologyVersion topVer) {
         AffinityTopologyVersion curTopVer = this.readyTopVer;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index c94f511d91887..3cfc25f6b7fbf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -307,9 +307,22 @@ Runnable addAssignments(
 
                     metrics.clearRebalanceCounters();
 
-                    metrics.startRebalance(0);
+                    for (GridDhtPartitionDemandMessage msg : assignments.values()) {
+                        for (Integer partId : msg.partitions().fullSet()) {
+                            metrics.onRebalancingKeysCountEstimateReceived(grp.topology().globalPartSizes().get(partId));
+                        }
+
+                        CachePartitionPartialCountersMap histMap = msg.partitions().historicalMap();
 
-                    rebalanceFut.listen(f -> metrics.clearRebalanceCounters());
+                        for (int i = 0; i < histMap.size(); i++) {
+                            long from = histMap.initialUpdateCounterAt(i);
+                            long to = histMap.updateCounterAt(i);
+
+                            metrics.onRebalancingKeysCountEstimateReceived(to - from);
+                        }
+                    }
+
+                    metrics.startRebalance(0);
                 }
             }
 
@@ -714,8 +727,6 @@ public void handleSupplyMessage(
         try {
             AffinityAssignment aff = grp.affinity().cachedAffinity(topVer);
 
-            GridCacheContext cctx = grp.sharedGroup() ? null : grp.singleCacheContext();
-
             ctx.database().checkpointReadLock();
 
             try {
@@ -749,11 +760,10 @@ public void handleSupplyMessage(
                                         break;
                                     }
 
-                                    if (grp.sharedGroup() && (cctx == null || cctx.cacheId() != entry.cacheId()))
-                                        cctx = ctx.cacheContext(entry.cacheId());
-
-                                    if (cctx != null && cctx.statisticsEnabled())
-                                        cctx.cache().metrics0().onRebalanceKeyReceived();
+                                    for (GridCacheContext cctx : grp.caches()) {
+                                        if (cctx.statisticsEnabled())
+                                            cctx.cache().metrics0().onRebalanceKeyReceived();
+                                    }
                                 }
 
                                 // If message was last for this partition,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
index 77baa38cc52c3..4ecffc492c7b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
@@ -443,7 +443,7 @@ public int size() {
      * @return Estimated keys count.
      */
     public long estimatedKeysCount() {
-        return estimatedKeysCnt;
+        return -1;
     }
 
     /**
@@ -457,12 +457,7 @@ public void addEstimatedKeysCount(long cnt) {
      * @return Estimated keys count for a given cache ID.
      */
     public long keysForCache(int cacheId) {
-        if (this.keysPerCache == null)
-            return -1;
-
-        Long cnt = this.keysPerCache.get(cacheId);
-
-        return cnt != null ? cnt : 0;
+        return -1;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 07a785c6aad03..1f03ff01fad59 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -868,6 +868,7 @@ private void updateTopologies(boolean crd) throws IgniteCheckedException {
                         clientTop.partitionMap(true),
                         clientTop.fullUpdateCounters(),
                         Collections.emptySet(),
+                        null,
                         null);
                 }
             }
@@ -2325,6 +2326,37 @@ private void onAffinityInitialized(IgniteInternalFuture<Map<Integer, Map<Integer
         }
     }
 
+    /**
+     * @param top Topology.
+     */
+    private void assignPartitionSizes(GridDhtPartitionTopology top) {
+        Map<Integer, Long> partSizes = new HashMap<>();
+
+        for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> e : msgs.entrySet()) {
+            GridDhtPartitionsSingleMessage singleMsg = e.getValue();
+
+            GridDhtPartitionMap partMap = singleMsg.partitions().get(top.groupId());
+
+            if (partMap == null)
+                continue;
+
+            for (Map.Entry<Integer, GridDhtPartitionState> e0 : partMap.entrySet()) {
+                int p = e0.getKey();
+                GridDhtPartitionState state = e0.getValue();
+
+                if (state == GridDhtPartitionState.OWNING)
+                    partSizes.put(p, singleMsg.partitionSizes(top.groupId()).get(p));
+            }
+        }
+
+        for (GridDhtLocalPartition locPart : top.currentLocalPartitions()) {
+            if (locPart.state() == GridDhtPartitionState.OWNING)
+                partSizes.put(locPart.id(), locPart.fullSize());
+        }
+
+        top.globalPartSizes(partSizes);
+    }
+
     /**
      * Collects and determines new owners of partitions for all nodes for given {@code top}.
      *
@@ -2365,7 +2397,7 @@ private void assignPartitionStates(GridDhtPartitionTopology top) {
                 CounterWithNodes maxCntr = maxCntrs.get(p);
 
                 if (maxCntr == null || cntr > maxCntr.cnt)
-                    maxCntrs.put(p, new CounterWithNodes(cntr, uuid));
+                    maxCntrs.put(p, new CounterWithNodes(cntr, e.getValue().partitionSizes(top.groupId()).get(p), uuid));
                 else if (cntr == maxCntr.cnt)
                     maxCntr.nodes.add(uuid);
             }
@@ -2391,7 +2423,7 @@ else if (cntr == maxCntr.cnt)
             CounterWithNodes maxCntr = maxCntrs.get(part.id());
 
             if (maxCntr == null && cntr == 0) {
-                CounterWithNodes cntrObj = new CounterWithNodes(0, cctx.localNodeId());
+                CounterWithNodes cntrObj = new CounterWithNodes(0, 0L, cctx.localNodeId());
 
                 for (UUID nodeId : msgs.keySet()) {
                     if (top.partitionState(nodeId, part.id()) == GridDhtPartitionState.OWNING)
@@ -2401,7 +2433,7 @@ else if (cntr == maxCntr.cnt)
                 maxCntrs.put(part.id(), cntrObj);
             }
             else if (maxCntr == null || cntr > maxCntr.cnt)
-                maxCntrs.put(part.id(), new CounterWithNodes(cntr, cctx.localNodeId()));
+                maxCntrs.put(part.id(), new CounterWithNodes(cntr, part.fullSize(), cctx.localNodeId()));
             else if (cntr == maxCntr.cnt)
                 maxCntr.nodes.add(cctx.localNodeId());
         }
@@ -2465,6 +2497,13 @@ else if (cntr == maxCntr.cnt)
             for (UUID nodeId : nodesToReload)
                 partsToReload.put(nodeId, top.groupId(), p);
         }
+
+        Map<Integer, Long> partSizes = new HashMap<>(maxCntrs.size());
+
+        for (Map.Entry<Integer, CounterWithNodes> e : maxCntrs.entrySet())
+            partSizes.put(e.getKey(), e.getValue().size);
+
+        top.globalPartSizes(partSizes);
     }
 
     /**
@@ -2872,16 +2911,16 @@ private void assignPartitionsStates() {
             if (grpDesc.config().getCacheMode() == CacheMode.LOCAL)
                 continue;
 
-            if (!CU.isPersistentCache(grpDesc.config(), cctx.gridConfig().getDataStorageConfiguration()))
-                continue;
-
             CacheGroupContext grpCtx = cctx.cache().cacheGroup(e.getKey());
 
             GridDhtPartitionTopology top = grpCtx != null ?
                 grpCtx.topology() :
                 cctx.exchange().clientTopology(e.getKey(), events().discoveryCache());
 
-            assignPartitionStates(top);
+            if (!CU.isPersistentCache(grpDesc.config(), cctx.gridConfig().getDataStorageConfiguration()))
+                assignPartitionSizes(top);
+            else
+                assignPartitionStates(top);
         }
     }
 
@@ -3268,6 +3307,7 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa
                     entry.getValue(),
                     cntrMap,
                     msg.partsToReload(cctx.localNodeId(), grpId),
+                    msg.partitionSizes(grpId),
                     null);
             }
             else {
@@ -3283,6 +3323,7 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa
                         entry.getValue(),
                         cntrMap,
                         Collections.emptySet(),
+                        null,
                         null);
                 }
             }
@@ -3867,6 +3908,9 @@ private static class CounterWithNodes {
         /** */
         private final long cnt;
 
+        /** */
+        private final long size;
+
         /** */
         private final Set<UUID> nodes = new HashSet<>();
 
@@ -3874,8 +3918,9 @@ private static class CounterWithNodes {
          * @param cnt Count.
          * @param firstNode Node ID.
          */
-        private CounterWithNodes(long cnt, UUID firstNode) {
+        private CounterWithNodes(long cnt, @Nullable Long size, UUID firstNode) {
             this.cnt = cnt;
+            this.size = size != null ? size : 0;
 
             nodes.add(firstNode);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index 4a449d18a09d8..59624687b9606 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -93,6 +93,14 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
     /** Serialized partitions that must be cleared and re-loaded. */
     private byte[] partsToReloadBytes;
 
+    /** Partitions sizes. */
+    @GridToStringInclude
+    @GridDirectTransient
+    private Map<Integer, Map<Integer, Long>> partsSizes;
+
+    /** Serialized partitions sizes. */
+    private byte[] partsSizesBytes;
+
     /** Topology version. */
     private AffinityTopologyVersion topVer;
 
@@ -164,6 +172,8 @@ public GridDhtPartitionsFullMessage(@Nullable GridDhtPartitionExchangeId id,
         cp.partHistSuppliersBytes = partHistSuppliersBytes;
         cp.partsToReload = partsToReload;
         cp.partsToReloadBytes = partsToReloadBytes;
+        cp.partsSizes = partsSizes;
+        cp.partsSizesBytes = partsSizesBytes;
         cp.topVer = topVer;
         cp.errs = errs;
         cp.errsBytes = errsBytes;
@@ -331,6 +341,9 @@ public IgniteDhtPartitionHistorySuppliersMap partitionHistorySuppliers() {
         return partHistSuppliers;
     }
 
+    /**
+     *
+     */
     public Set<Integer> partsToReload(UUID nodeId, int grpId) {
         if (partsToReload == null)
             return Collections.emptySet();
@@ -338,6 +351,35 @@ public Set<Integer> partsToReload(UUID nodeId, int grpId) {
         return partsToReload.get(nodeId, grpId);
     }
 
+    /**
+     * Adds partition sizes map for specified {@code grpId} to the current message.
+     *
+     * @param grpId Group id.
+     * @param partSizesMap Partition sizes map.
+     */
+    public void addPartitionSizes(int grpId, Map<Integer, Long> partSizesMap) {
+        if (partSizesMap.isEmpty())
+            return;
+
+        if (partsSizes == null)
+            partsSizes = new HashMap<>();
+
+        partsSizes.put(grpId, partSizesMap);
+    }
+
+    /**
+     * Returns partition sizes map for specified {@code grpId}.
+     *
+     * @param grpId Group id.
+     * @return Partition sizes map (partId, partSize).
+     */
+    public Map<Integer, Long> partitionSizes(int grpId) {
+        if (partsSizes == null)
+            return Collections.emptyMap();
+
+        return partsSizes.getOrDefault(grpId, Collections.emptyMap());
+    }
+
     /**
      * @return Errors map.
      */
@@ -369,6 +411,7 @@ void setErrorsMap(Map<UUID, Exception> errs) {
             byte[] partCntrsBytes20 = null;
             byte[] partHistSuppliersBytes0 = null;
             byte[] partsToReloadBytes0 = null;
+            byte[] partsSizesBytes0 = null;
             byte[] errsBytes0 = null;
 
             if (!F.isEmpty(parts) && partsBytes == null)
@@ -386,6 +429,9 @@ void setErrorsMap(Map<UUID, Exception> errs) {
             if (partsToReload != null && partsToReloadBytes == null)
                 partsToReloadBytes0 = U.marshal(ctx, partsToReload);
 
+            if (partsSizes != null && partsSizesBytes == null)
+                partsSizesBytes0 = U.marshal(ctx, partsSizes);
+
             if (!F.isEmpty(errs) && errsBytes == null)
                 errsBytes0 = U.marshal(ctx, errs);
 
@@ -398,6 +444,7 @@ void setErrorsMap(Map<UUID, Exception> errs) {
                     byte[] partCntrsBytes2Zip = U.zip(partCntrsBytes20);
                     byte[] partHistSuppliersBytesZip = U.zip(partHistSuppliersBytes0);
                     byte[] partsToReloadBytesZip = U.zip(partsToReloadBytes0);
+                    byte[] partsSizesBytesZip = U.zip(partsSizesBytes0);
                     byte[] exsBytesZip = U.zip(errsBytes0);
 
                     partsBytes0 = partsBytesZip;
@@ -405,6 +452,7 @@ void setErrorsMap(Map<UUID, Exception> errs) {
                     partCntrsBytes20 = partCntrsBytes2Zip;
                     partHistSuppliersBytes0 = partHistSuppliersBytesZip;
                     partsToReloadBytes0 = partsToReloadBytesZip;
+                    partsSizesBytes0 = partsSizesBytesZip;
                     errsBytes0 = exsBytesZip;
 
                     compressed(true);
@@ -419,6 +467,7 @@ void setErrorsMap(Map<UUID, Exception> errs) {
             partCntrsBytes2 = partCntrsBytes20;
             partHistSuppliersBytes = partHistSuppliersBytes0;
             partsToReloadBytes = partsToReloadBytes0;
+            partsSizesBytes = partsSizesBytes0;
             errsBytes = errsBytes0;
         }
     }
@@ -506,6 +555,13 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
                 partsToReload = U.unmarshal(ctx, partsToReloadBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
         }
 
+        if (partsSizesBytes != null && partsSizes == null) {
+            if (compressed())
+                partsSizes = U.unmarshalZip(ctx.marshaller(), partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+            else
+                partsSizes = U.unmarshal(ctx, partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+        }
+
         if (partCntrs == null)
             partCntrs = new IgniteDhtPartitionCountersMap();
 
@@ -584,18 +640,24 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
                 writer.incrementState();
 
             case 13:
-                if (!writer.writeByteArray("partsToReloadBytes", partsToReloadBytes))
+                if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes))
                     return false;
 
                 writer.incrementState();
 
             case 14:
-                if (!writer.writeMessage("resTopVer", resTopVer))
+                if (!writer.writeByteArray("partsToReloadBytes", partsToReloadBytes))
                     return false;
 
                 writer.incrementState();
 
             case 15:
+                if (!writer.writeMessage("resTopVer", resTopVer))
+                    return false;
+
+                writer.incrementState();
+
+            case 16:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -682,7 +744,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
                 reader.incrementState();
 
             case 13:
-                partsToReloadBytes = reader.readByteArray("partsToReloadBytes");
+                partsSizesBytes = reader.readByteArray("partsSizesBytes");
 
                 if (!reader.isLastRead())
                     return false;
@@ -690,7 +752,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
                 reader.incrementState();
 
             case 14:
-                resTopVer = reader.readMessage("resTopVer");
+                partsToReloadBytes = reader.readByteArray("partsToReloadBytes");
 
                 if (!reader.isLastRead())
                     return false;
@@ -698,6 +760,14 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
                 reader.incrementState();
 
             case 15:
+                resTopVer = reader.readMessage("resTopVer");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 16:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -717,7 +787,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 16;
+        return 17;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index b60070e6b4dd2..804cc030489f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -70,7 +70,7 @@ public class GridDhtPartitionsSingleMessage extends GridDhtPartitionsAbstractMes
     /** Partitions sizes. */
     @GridToStringInclude
     @GridDirectTransient
-    private Map<Integer, Map<Integer, Long>> partSizes;
+    private Map<Integer, Map<Integer, Long>> partsSizes;
 
     /** Serialized partitions counters. */
     private byte[] partsSizesBytes;
@@ -237,10 +237,10 @@ public void addPartitionSizes(int grpId, Map<Integer, Long> partSizesMap) {
         if (partSizesMap.isEmpty())
             return;
 
-        if (partSizes == null)
-            partSizes = new HashMap<>();
+        if (partsSizes == null)
+            partsSizes = new HashMap<>();
 
-        partSizes.put(grpId, partSizesMap);
+        partsSizes.put(grpId, partSizesMap);
     }
 
     /**
@@ -250,10 +250,10 @@ public void addPartitionSizes(int grpId, Map<Integer, Long> partSizesMap) {
      * @return Partition sizes map (partId, partSize).
      */
     public Map<Integer, Long> partitionSizes(int grpId) {
-        if (partSizes == null)
+        if (partsSizes == null)
             return Collections.emptyMap();
 
-        return partSizes.getOrDefault(grpId, Collections.emptyMap());
+        return partsSizes.getOrDefault(grpId, Collections.emptyMap());
     }
 
     /**
@@ -324,7 +324,7 @@ public void setError(Exception ex) {
         boolean marshal = (parts != null && partsBytes == null) ||
             (partCntrs != null && partCntrsBytes == null) ||
             (partHistCntrs != null && partHistCntrsBytes == null) ||
-            (partSizes != null && partsSizesBytes == null) ||
+            (partsSizes != null && partsSizesBytes == null) ||
             (err != null && errBytes == null);
 
         if (marshal) {
@@ -343,8 +343,8 @@ public void setError(Exception ex) {
             if (partHistCntrs != null && partHistCntrsBytes == null)
                 partHistCntrsBytes0 = U.marshal(ctx, partHistCntrs);
 
-            if (partSizes != null && partsSizesBytes == null)
-                partSizesBytes0 = U.marshal(ctx, partSizes);
+            if (partsSizes != null && partsSizesBytes == null)
+                partSizesBytes0 = U.marshal(ctx, partsSizes);
 
             if (err != null && errBytes == null)
                 errBytes0 = U.marshal(ctx, err);
@@ -405,11 +405,11 @@ public void setError(Exception ex) {
                 partHistCntrs = U.unmarshal(ctx, partHistCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
         }
 
-        if (partsSizesBytes != null && partSizes == null) {
+        if (partsSizesBytes != null && partsSizes == null) {
             if (compressed())
-                partSizes = U.unmarshalZip(ctx.marshaller(), partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+                partsSizes = U.unmarshalZip(ctx.marshaller(), partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
             else
-                partSizes = U.unmarshal(ctx, partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+                partsSizes = U.unmarshal(ctx, partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
         }
 
         if (errBytes != null && err == null) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
index 3a11fc535a661..87e129bcf9b77 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/cache/PlatformCache.java
@@ -1484,6 +1484,8 @@ public static void writeCacheMetrics(BinaryRawWriter writer, CacheMetrics metric
         writer.writeLong(metrics.getRebalancingStartTime());
         writer.writeLong(metrics.getRebalanceClearingPartitionsLeft());
         writer.writeLong(metrics.getCacheSize());
+        writer.writeLong(metrics.getRebalancedKeys());
+        writer.writeLong(metrics.getEstimatedRebalancingKeys());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java
index 59f16b2a5e7cd..854bbd7ad1e2c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheMetrics.java
@@ -175,6 +175,12 @@ public class VisorCacheMetrics extends VisorDataTransferObject {
     /** Total number of partitions on current node. */
     private int totalPartsCnt;
 
+    /** Number of already rebalanced keys. */
+    private long rebalancedKeys;
+
+    /** Number estimated to rebalance keys. */
+    private long estimatedRebalancingKeys;
+
     /** Number of currently rebalancing partitions on current node. */
     private int rebalancingPartsCnt;
 
@@ -276,6 +282,8 @@ public VisorCacheMetrics(IgniteEx ignite, String cacheName) {
         offHeapPrimaryEntriesCnt = m.getOffHeapPrimaryEntriesCount();
 
         totalPartsCnt = m.getTotalPartitionsCount();
+        rebalancedKeys = m.getRebalancedKeys();
+        estimatedRebalancingKeys = m.getEstimatedRebalancingKeys();
         rebalancingPartsCnt = m.getRebalancingPartitionsCount();
         keysToRebalanceLeft = m.getKeysToRebalanceLeft();
         rebalancingKeysRate = m.getRebalancingKeysRate();
@@ -622,6 +630,20 @@ public int getTotalPartitionsCount() {
         return totalPartsCnt;
     }
 
+    /**
+     * @return Number of already rebalanced keys.
+     */
+    public long getRebalancedKeys() {
+        return rebalancedKeys;
+    }
+
+    /**
+     * @return Number estimated to rebalance keys.
+     */
+    public long getEstimatedRebalancingKeys() {
+        return estimatedRebalancingKeys;
+    }
+
     /**
      * @return Number of currently rebalancing partitions on current node.
      */
@@ -650,6 +672,11 @@ public long getRebalancingBytesRate() {
         return rebalancingBytesRate;
     }
 
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeString(out, name);
@@ -708,6 +735,9 @@ public long getRebalancingBytesRate() {
         out.writeObject(qryMetrics);
 
         out.writeLong(cacheSize);
+
+        out.writeLong(rebalancedKeys);
+        out.writeLong(estimatedRebalancingKeys);
     }
 
     /** {@inheritDoc} */
@@ -768,6 +798,11 @@ public long getRebalancingBytesRate() {
 
         if (in.available() > 0)
             cacheSize = in.readLong();
+
+        if (protoVer > V1) {
+            rebalancedKeys = in.readLong();
+            estimatedRebalancingKeys = in.readLong();
+        }
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
index fda23a2a917c6..14b928164978c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
@@ -183,8 +183,9 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto
 
             List<VisorCache> resCaches = res.getCaches();
 
+            int partitions = 0;
             double total = 0;
-            double moving = 0;
+            double ready = 0;
 
             for (String cacheName : cacheProc.cacheNames()) {
                 if (proxyCache(cacheName))
@@ -201,8 +202,16 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto
 
                         CacheMetrics cm = ca.localMetrics();
 
-                        total += cm.getTotalPartitionsCount();
-                        moving += cm.getRebalancingPartitionsCount();
+                        partitions += cm.getTotalPartitionsCount();
+
+                        long partTotal = cm.getEstimatedRebalancingKeys();
+                        long partReady = cm.getRebalancedKeys();
+
+                        if (partReady >= partTotal)
+                            partReady = Math.max(partTotal - 1, 0);
+
+                        total += partTotal;
+                        ready += partReady;
 
                         resCaches.add(new VisorCache(ignite, ca, arg.isCollectCacheMetrics()));
                     }
@@ -217,7 +226,10 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto
                 }
             }
 
-            res.setRebalance(total > 0 ? (total - moving) / total : -1);
+            if (partitions == 0)
+                res.setRebalance(-1);
+            else
+                res.setRebalance(total > 0 ? ready / total : 1);
         }
         catch (Exception e) {
             res.setCachesEx(new VisorExceptionWrapper(e));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
index ceb98522d5a25..b81bdff154d13 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupsMetricsRebalanceTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.events.CacheRebalancingEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -159,12 +160,12 @@ public void testRebalance() throws Exception {
         assertTrue(rate1 > 0);
         assertTrue(rate2 > 0);
 
-        // rate1 has to be roughly twice more than rate2.
-        double ratio = ((double)rate2 / rate1) * 100;
+        // rate1 has to be roughly the same as rate2
+        double ratio = ((double)rate2 / rate1);
 
         log.info("Ratio: " + ratio);
 
-        assertTrue(ratio > 40 && ratio < 60);
+        assertTrue(ratio > 0.9 && ratio < 1.1);
     }
 
     /**
@@ -177,8 +178,6 @@ public void testRebalanceEstimateFinishTime() throws Exception {
 
         final int KEYS = 4_000_000;
 
-        IgniteCache<Object, Object> cache1 = ig1.cache(CACHE1);
-
         try (IgniteDataStreamer<Integer, String> st = ig1.dataStreamer(CACHE1)) {
             for (int i = 0; i < KEYS; i++)
                 st.addData(i, CACHE1 + "-" + i);
@@ -211,10 +210,10 @@ public void testRebalanceEstimateFinishTime() throws Exception {
         CacheMetrics metrics = ig2.cache(CACHE1).localMetrics();
 
         long startTime = metrics.getRebalancingStartTime();
+        long currTime = U.currentTimeMillis();
 
-        assertTrue(startTime > 0);
-        assertTrue((U.currentTimeMillis() - startTime) < 5000);
-        assertTrue((U.currentTimeMillis() - startTime) > 0);
+        assertTrue("Invalid start time [startTime=" + startTime + ", currTime=" + currTime + ']',
+            startTime > 0L && (currTime - startTime) >= 0L && (currTime - startTime) <= 5000L);
 
         final CountDownLatch latch = new CountDownLatch(1);
 
@@ -223,47 +222,62 @@ public void testRebalanceEstimateFinishTime() throws Exception {
                 // Waiting 25% keys will be rebalanced.
                 int partKeys = KEYS / 2;
 
-                final long keysLine = (long)(partKeys - (partKeys * 0.25));
+                final long keysLine = partKeys * 3L / 4L;
 
                 System.out.println("Wait until keys left will be less " + keysLine);
 
-                try {
-                    while (finishRebalanceLatch.getCount() != 0) {
-                        CacheMetrics m = ig2.cache(CACHE1).localMetrics();
+                while (true) {
+                    CacheMetrics m = ig2.cache(CACHE1).localMetrics();
 
-                        long keyLeft = m.getKeysToRebalanceLeft();
+                    long keyLeft = m.getKeysToRebalanceLeft();
 
-                        if (keyLeft > 0 && keyLeft < keysLine)
-                            latch.countDown();
+                    if (keyLeft > 0 && keyLeft < keysLine) {
+                        latch.countDown();
 
-                        System.out.println("Keys left: " + m.getKeysToRebalanceLeft());
+                        break;
+                    }
 
-                        try {
-                            Thread.sleep(1_000);
-                        }
-                        catch (InterruptedException e) {
-                            System.out.println("Interrupt thread: " + e.getMessage());
+                    System.out.println("Keys left: " + m.getKeysToRebalanceLeft());
 
-                            Thread.currentThread().interrupt();
-                        }
+                    try {
+                        Thread.sleep(1_000);
+                    }
+                    catch (InterruptedException e) {
+                        System.out.println("Interrupt thread: " + e.getMessage());
+
+                        Thread.currentThread().interrupt();
                     }
-                }
-                finally {
-                    latch.countDown();
                 }
             }
         });
 
         assertTrue(latch.await(getTestTimeout(), TimeUnit.MILLISECONDS));
 
+        waitForCondition(new PA() {
+            @Override public boolean apply() {
+                return ig2.cache(CACHE1).localMetrics().getEstimatedRebalancingFinishTime() != -1L;
+            }
+        }, 5_000L);
+
         long finishTime = ig2.cache(CACHE1).localMetrics().getEstimatedRebalancingFinishTime();
 
-        assertTrue(finishTime > 0);
+        assertTrue("Not a positive estimation of rebalancing finish time: " + finishTime,
+            finishTime > 0L);
 
-        long timePassed = U.currentTimeMillis() - startTime;
-        long timeLeft = finishTime - System.currentTimeMillis();
+        currTime = U.currentTimeMillis();
 
-        assertTrue(finishRebalanceLatch.await(timeLeft + 2_000, TimeUnit.SECONDS));
+        long timePassed = currTime - startTime;
+        long timeLeft = finishTime - currTime;
+
+        // TODO: finishRebalanceLatch gets countdown much earlier because of ForceRebalanceExchangeTask triggered by cache with delay
+//        assertTrue("Got timeout while waiting for rebalancing. Estimated left time: " + timeLeft,
+//            finishRebalanceLatch.await(timeLeft + 10_000L, TimeUnit.MILLISECONDS));
+
+        waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return ig2.cache(CACHE1).localMetrics().getKeysToRebalanceLeft() == 0;
+            }
+        }, timeLeft + 10_000L);
 
         System.out.println(
             "TimePassed:" + timePassed +
@@ -275,11 +289,13 @@ public void testRebalanceEstimateFinishTime() throws Exception {
 
         System.clearProperty(IGNITE_REBALANCE_STATISTICS_TIME_INTERVAL);
 
-        System.out.println("Rebalance time:" + (U.currentTimeMillis() - startTime));
+        currTime = U.currentTimeMillis();
+
+        log.info("Rebalance time: " + (currTime - startTime));
 
-        long diff = finishTime - U.currentTimeMillis();
+        long diff = finishTime - currTime;
 
-        assertTrue("Expected less 5000, Actual:" + diff, Math.abs(diff) < 10_000);
+        assertTrue("Expected less than 10000, but actual: " + diff, Math.abs(diff) < 10_000L);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java b/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java
index 44a15d500ca44..fe61e35b11dd1 100644
--- a/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java
+++ b/modules/core/src/test/java/org/apache/ignite/platform/PlatformCacheWriteMetricsTask.java
@@ -473,5 +473,15 @@ private static class TestCacheMetrics implements CacheMetrics {
         @Override public long getCacheSize() {
             return 65;
         }
+
+        /** {@inheritDoc} */
+        @Override public long getRebalancedKeys() {
+            return 66;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long getEstimatedRebalancingKeys() {
+            return 67;
+        }
     }
 }
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
index d775c05691076..e0e7301c4d233 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Cache/ICacheMetrics.cs
@@ -654,5 +654,23 @@ public interface ICacheMetrics
         /// Number of clearing partitions for rebalance.
         /// </returns>
         long RebalanceClearingPartitionsLeft { get; }
+
+        /// <summary>
+        /// Gets number of already rebalanced keys.
+        /// need to be cleared before actual rebalance start.
+        /// </summary>
+        /// <returns>
+        /// Number of already rebalanced keys.
+        /// </returns>
+        long RebalancedKeys { get; }
+
+        /// <summary>
+        /// Gets number of estimated keys to rebalance.
+        /// need to be cleared before actual rebalance start.
+        /// </summary>
+        /// <returns>
+        /// Number of estimated keys to rebalance.
+        /// </returns>
+        long EstimatedRebalancingKeys { get; }
     }
 }
\ No newline at end of file
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
index 1fdc8771f832e..be6980d7cdd97 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Cache/CacheMetricsImpl.cs
@@ -247,6 +247,12 @@ internal class CacheMetricsImpl : ICacheMetrics
         /** */
         private readonly long _rebalancingClearingPartitionsLeft;
 
+        /** */
+        private readonly long _rebalancedKeys;
+
+        /** */
+        private readonly long _estimatedRebalancedKeys;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="CacheMetricsImpl"/> class.
         /// </summary>
@@ -327,6 +333,8 @@ public CacheMetricsImpl(IBinaryRawReader reader)
             _rebalancingStartTime = reader.ReadLong();
             _rebalancingClearingPartitionsLeft = reader.ReadLong();
             _cacheSize = reader.ReadLong();
+            _rebalancedKeys = reader.ReadLong();
+            _estimatedRebalancedKeys = reader.ReadLong();
         }
 
         /** <inheritDoc /> */
@@ -550,5 +558,11 @@ public CacheMetricsImpl(IBinaryRawReader reader)
 
         /** <inheritDoc /> */
         public long RebalanceClearingPartitionsLeft { get { return _rebalancingClearingPartitionsLeft; } }
+
+        /** <inheritDoc /> */
+        public long RebalancedKeys { get { return _rebalancedKeys; } }
+        
+        /** <inheritDoc /> */
+        public long EstimatedRebalancingKeys { get { return _estimatedRebalancedKeys; } }
     }
 }
\ No newline at end of file

From 1af9c4c4f62b828145df8ca44e5bc96746091af4 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Tue, 19 Jun 2018 20:12:13 +0300
Subject: [PATCH 220/543] IGNITE-8808 Improve control.sh --tx command to show
 local and remote transactions. - Fixes #4209.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../internal/commandline/CommandHandler.java  |  18 +-
 .../ignite/internal/visor/tx/VisorTxInfo.java |  41 ++-
 .../ignite/internal/visor/tx/VisorTxTask.java | 212 ++++++++++--
 .../TestRecordingCommunicationSpi.java        |  42 ++-
 .../ignite/util/GridCommandHandlerTest.java   | 310 +++++++++++++++++-
 5 files changed, 566 insertions(+), 57 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 3a26537260f54..77ee6f14d2d2c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -993,11 +993,19 @@ else if (arg.getOperation() == VisorTxOperation.KILL)
                         ", concurrency=" + info.getConcurrency() +
                         ", timeout=" + info.getTimeout() +
                         ", size=" + info.getSize() +
-                        ", dhtNodes=" + F.transform(info.getPrimaryNodes(), new IgniteClosure<UUID, String>() {
-                        @Override public String apply(UUID id) {
-                            return U.id8(id);
-                        }
-                    }) +
+                        ", dhtNodes=" + (info.getPrimaryNodes() == null ? "N/A" :
+                        F.transform(info.getPrimaryNodes(), new IgniteClosure<UUID, String>() {
+                            @Override public String apply(UUID id) {
+                                return U.id8(id);
+                            }
+                        })) +
+                        ", nearXid=" + info.getNearXid() +
+                        ", parentNodeIds=" + (info.getMasterNodeIds() == null ? "N/A" :
+                        F.transform(info.getMasterNodeIds(), new IgniteClosure<UUID, String>() {
+                            @Override public String apply(UUID id) {
+                                return U.id8(id);
+                            }
+                        })) +
                         ']');
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
index ecf3e0d79fb4c..03de5b04a9c1a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.apache.ignite.transactions.TransactionState;
+import org.jetbrains.annotations.Nullable;
 
 /**
  */
@@ -75,6 +76,12 @@ public class VisorTxInfo extends VisorDataTransferObject {
     /** */
     private int size;
 
+    /** */
+    private IgniteUuid nearXid;
+
+    /** */
+    private Collection<UUID> masterNodeIds;
+
     /**
      * Default constructor.
      */
@@ -96,7 +103,7 @@ public VisorTxInfo() {
      */
     public VisorTxInfo(IgniteUuid xid, long startTime, long duration, TransactionIsolation isolation,
         TransactionConcurrency concurrency, long timeout, String lb, Collection<UUID> primaryNodes,
-        TransactionState state, int size) {
+        TransactionState state, int size, IgniteUuid nearXid, Collection<UUID> masterNodeIds) {
         this.xid = xid;
         this.startTime = startTime;
         this.duration = duration;
@@ -107,6 +114,13 @@ public VisorTxInfo(IgniteUuid xid, long startTime, long duration, TransactionIso
         this.primaryNodes = primaryNodes;
         this.state = state;
         this.size = size;
+        this.nearXid = nearXid;
+        this.masterNodeIds = masterNodeIds;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
     }
 
     /** */
@@ -164,9 +178,14 @@ public int getSize() {
         return size;
     }
 
-    /** {@inheritDoc} */
-    @Override public byte getProtocolVersion() {
-        return V2;
+    /** */
+    public @Nullable IgniteUuid getNearXid() {
+        return nearXid;
+    }
+
+    /** */
+    public @Nullable Collection<UUID> getMasterNodeIds() {
+        return masterNodeIds;
     }
 
     /** {@inheritDoc} */
@@ -180,12 +199,13 @@ public int getSize() {
         U.writeCollection(out, primaryNodes);
         U.writeEnum(out, state);
         out.writeInt(size);
+        U.writeGridUuid(out, nearXid);
+        U.writeCollection(out, masterNodeIds);
         out.writeLong(startTime);
     }
 
     /** {@inheritDoc} */
-    @Override protected void readExternalData(byte protoVer,
-        ObjectInput in) throws IOException, ClassNotFoundException {
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
         xid = U.readGridUuid(in);
         duration = in.readLong();
         isolation = TransactionIsolation.fromOrdinal(in.readByte());
@@ -195,7 +215,14 @@ public int getSize() {
         primaryNodes = U.readCollection(in);
         state = TransactionState.fromOrdinal(in.readByte());
         size = in.readInt();
-        startTime = protoVer >= V2 ? in.readLong() : 0L;
+        if (protoVer >= V2) {
+            nearXid = U.readGridUuid(in);
+
+            masterNodeIds = U.readCollection(in);
+
+            startTime = in.readLong();
+        }
+
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
index 25a69d1d67e06..9919b7d654458 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
@@ -21,7 +21,8 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.ConcurrentModificationException;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
@@ -31,22 +32,32 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxRemote;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxMappings;
-import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxRemoteEx;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorMultiNodeTask;
 import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.lang.IgniteBiClosure;
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.lang.IgniteUuid;
-import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionState;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.transactions.TransactionState.COMMITTED;
+import static org.apache.ignite.transactions.TransactionState.COMMITTING;
+
 /**
  *
  */
@@ -103,6 +114,8 @@ public class VisorTxTask extends VisorMultiNodeTask<VisorTxTaskArg, Map<ClusterN
     @Nullable @Override protected Map<ClusterNode, VisorTxTaskResult> reduce0(List<ComputeJobResult> results) throws IgniteException {
         Map<ClusterNode, VisorTxTaskResult> mapRes = new TreeMap<>();
 
+        Map<UUID, ClusterNode> nodeMap = new HashMap<>();
+
         for (ComputeJobResult result : results) {
             VisorTxTaskResult data = result.getData();
 
@@ -110,6 +123,47 @@ public class VisorTxTask extends VisorMultiNodeTask<VisorTxTaskArg, Map<ClusterN
                 continue;
 
             mapRes.put(result.getNode(), data);
+
+            nodeMap.put(result.getNode().id(), result.getNode());
+        }
+
+        // Remove local and remote txs for which near txs are present.
+        for (VisorTxTaskResult result : mapRes.values()) {
+            List<VisorTxInfo> infos = result.getInfos();
+
+            Iterator<VisorTxInfo> it = infos.iterator();
+
+            while (it.hasNext()) {
+                VisorTxInfo info = it.next();
+
+                if (!info.getXid().equals(info.getNearXid())) {
+                    UUID nearNodeId = info.getMasterNodeIds().iterator().next();
+
+                    // Try find id.
+                    ClusterNode node = nodeMap.get(nearNodeId);
+
+                    if (node == null)
+                        continue;
+
+                    VisorTxTaskResult res0 = mapRes.get(node);
+
+                    if (res0 == null)
+                        continue;
+
+                    boolean exists = false;
+
+                    for (VisorTxInfo txInfo : res0.getInfos()) {
+                        if (txInfo.getXid().equals(info.getNearXid())) {
+                            exists = true;
+
+                            break;
+                        }
+                    }
+
+                    if (exists)
+                        it.remove();
+                }
+            }
         }
 
         return mapRes;
@@ -123,7 +177,16 @@ private static class VisorTxJob extends VisorJob<VisorTxTaskArg, VisorTxTaskResu
         private static final long serialVersionUID = 0L;
 
         /** */
-        public static final int DEFAULT_LIMIT = 50;
+        private static final int DEFAULT_LIMIT = 50;
+
+        /** */
+        private static final TxKillClosure NEAR_KILL_CLOSURE = new NearKillClosure();
+
+        /** */
+        private static final TxKillClosure LOCAL_KILL_CLOSURE = NEAR_KILL_CLOSURE;
+
+        /** */
+        private static final TxKillClosure REMOTE_KILL_CLOSURE = new RemoteKillClosure();
 
         /**
          * @param arg Formal job argument.
@@ -138,7 +201,9 @@ private VisorTxJob(VisorTxTaskArg arg, boolean debug) {
             if (arg == null)
                 return new VisorTxTaskResult(Collections.emptyList());
 
-            Collection<Transaction> transactions = ignite.transactions().localActiveTransactions();
+            IgniteTxManager tm = ignite.context().cache().context().tm();
+
+            Collection<IgniteInternalTx> transactions = tm.activeTransactions();
 
             List<VisorTxInfo> infos = new ArrayList<>();
 
@@ -155,50 +220,102 @@ private VisorTxJob(VisorTxTaskArg arg, boolean debug) {
                 }
             }
 
-            for (Transaction transaction : transactions) {
-                GridNearTxLocal locTx = ((TransactionProxyImpl)transaction).tx();
-
+            for (IgniteInternalTx locTx : transactions) {
                 if (arg.getXid() != null && !locTx.xid().toString().equals(arg.getXid()))
                     continue;
 
                 if (arg.getState() != null && locTx.state() != arg.getState())
                     continue;
 
-                long duration = U.currentTimeMillis() - transaction.startTime();
+                long duration = U.currentTimeMillis() - locTx.startTime();
 
-                if (arg.getMinDuration() != null &&
-                    duration < arg.getMinDuration())
+                if (arg.getMinDuration() != null && duration < arg.getMinDuration())
                     continue;
 
-                if (lbMatch != null && !lbMatch.matcher(locTx.label() == null ? "null" : locTx.label()).matches())
-                    continue;
+                String lb = null;
+                int size = 0;
+                Collection<UUID> mappings = null;
+                TxKillClosure killClo = null;
 
-                Collection<UUID> mappings = new ArrayList<>();
+                // This filter conditions have meaning only for near txs, so we skip dht because it will never match.
+                boolean skip = arg.getMinSize() != null || lbMatch != null;
 
-                int size = 0;
+                if (locTx instanceof GridNearTxLocal) {
+                    GridNearTxLocal locTx0 = (GridNearTxLocal)locTx;
+
+                    lb = locTx0.label();
+
+                    if (lbMatch != null && !lbMatch.matcher(lb == null ? "null" : lb).matches())
+                        continue;
+
+                    mappings = new ArrayList<>();
+
+                    if (locTx0.mappings() != null) {
+                        IgniteTxMappings txMappings = locTx0.mappings();
+
+                        for (GridDistributedTxMapping mapping :
+                            txMappings.single() ? Collections.singleton(txMappings.singleMapping()) : txMappings.mappings()) {
+                            if (mapping == null)
+                                continue;
 
-                if (locTx.mappings() != null) {
-                    IgniteTxMappings txMappings = locTx.mappings();
+                            mappings.add(mapping.primary().id());
 
-                    for (GridDistributedTxMapping mapping :
-                        txMappings.single() ? Collections.singleton(txMappings.singleMapping()) : txMappings.mappings()) {
-                        if (mapping == null)
-                            continue;
+                            size += mapping.entries().size(); // Entries are not synchronized so no visibility guaranties for size.
+                        }
+                    }
+
+                    if (arg.getMinSize() != null && size < arg.getMinSize())
+                        continue;
+
+                    killClo = NEAR_KILL_CLOSURE;
+                }
+                else if (locTx instanceof GridDhtTxLocal) {
+                    if (skip)
+                        continue;
+
+                    GridDhtTxLocal locTx0 = (GridDhtTxLocal)locTx;
+
+                    Map<UUID, GridDistributedTxMapping> dhtMap = U.field(locTx0, "dhtMap");
+
+                    mappings = new ArrayList<>();
+
+                    if (dhtMap != null) {
+                        for (GridDistributedTxMapping mapping : dhtMap.values()) {
+                            mappings.add(mapping.primary().id());
+
+                            size += mapping.entries().size();
+                        }
+                    }
+
+                    Map<UUID, GridDistributedTxMapping> nearMap = U.field(locTx, "nearMap");
 
-                        mappings.add(mapping.primary().id());
+                    if (nearMap != null) {
+                        for (GridDistributedTxMapping mapping : nearMap.values()) {
+                            mappings.add(mapping.primary().id());
 
-                        size += mapping.entries().size(); // Entries are not synchronized so no visibility guaranties for size.
+                            size += mapping.entries().size();
+                        }
                     }
+
+                    killClo = LOCAL_KILL_CLOSURE;
                 }
+                else if (locTx instanceof GridDhtTxRemote) {
+                    if (skip)
+                        continue;
 
-                if (arg.getMinSize() != null && size < arg.getMinSize())
-                    continue;
+                    GridDhtTxRemote locTx0 = (GridDhtTxRemote)locTx;
+
+                    size = locTx0.readMap().size() + locTx.writeMap().size();
+
+                    killClo = REMOTE_KILL_CLOSURE;
+                }
 
                 infos.add(new VisorTxInfo(locTx.xid(), locTx.startTime(), duration, locTx.isolation(), locTx.concurrency(),
-                    locTx.timeout(), locTx.label(), mappings, locTx.state(), size));
+                    locTx.timeout(), lb, mappings, locTx.state(),
+                    size, locTx.nearXidVersion().asGridUuid(), locTx.masterNodeIds()));
 
                 if (arg.getOperation() == VisorTxOperation.KILL)
-                    locTx.rollbackAsync();
+                    killClo.apply(locTx, tm);
 
                 if (infos.size() == limit)
                     break;
@@ -271,4 +388,43 @@ private static class TxSizeComparator implements Comparator<VisorTxInfo> {
             return Long.compare(o2.getSize(), o1.getSize());
         }
     }
+
+    /** Type shortcut. */
+    private interface TxKillClosure extends
+        IgniteBiClosure<IgniteInternalTx, IgniteTxManager, IgniteInternalFuture<IgniteInternalTx>> {
+    }
+
+    /** Kills near or local tx. */
+    private static class NearKillClosure implements TxKillClosure {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public IgniteInternalFuture<IgniteInternalTx> apply(IgniteInternalTx tx, IgniteTxManager tm) {
+            return tx.isRollbackOnly() || tx.state() == COMMITTING || tx.state() == COMMITTED ?
+                new GridFinishedFuture<>() : tx.rollbackAsync();
+        }
+    }
+
+    /** Kills remote tx. */
+    private static class RemoteKillClosure implements TxKillClosure {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public IgniteInternalFuture<IgniteInternalTx> apply(IgniteInternalTx tx, IgniteTxManager tm) {
+            IgniteTxRemoteEx remote = (IgniteTxRemoteEx)tx;
+
+            if (tx.isRollbackOnly() || tx.state() == COMMITTING || tx.state() == COMMITTED)
+                return new GridFinishedFuture<>();
+
+            if (tx.state() == TransactionState.PREPARED)
+                remote.doneRemote(tx.xidVersion(),
+                    Collections.<GridCacheVersion>emptyList(),
+                    Collections.<GridCacheVersion>emptyList(),
+                    Collections.<GridCacheVersion>emptyList());
+
+            return tx.rollbackAsync();
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
index 5d12d9aea9ff7..b36bf16dc9fdb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -29,13 +30,16 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
 
@@ -198,6 +202,18 @@ public void waitForBlocked() throws InterruptedException {
         }
     }
 
+    /**
+     * @param cnt Number of messages to wait.
+     *
+     * @throws InterruptedException If interrupted.
+     */
+    public void waitForBlocked(int cnt) throws InterruptedException {
+        synchronized (this) {
+            while (blockedMsgs.size() < cnt)
+                wait();
+        }
+    }
+
     /**
      * @throws InterruptedException If interrupted.
      */
@@ -251,26 +267,38 @@ public void blockMessages(Class<?> cls, String nodeName) {
     }
 
     /**
-     * Stops block messages and can sends all already blocked messages.
+     * Stops block messages and sends all already blocked messages.
      */
     public void stopBlock() {
-        stopBlock(true);
+        stopBlock(true, null);
     }
 
     /**
      * Stops block messages and sends all already blocked messages if sndMsgs is 'true'.
      *
-     * @param sndMsgs If {@code true} sends blocked messages.
+     * @param sndMsgs {@code True} to send blocked messages.
      */
     public void stopBlock(boolean sndMsgs) {
-        synchronized (this) {
-            blockP = null;
+        stopBlock(sndMsgs, null);
+    }
 
+    /**
+     * Stops block messages and sends all already blocked messages if sndMsgs is 'true' optionally filtered
+     * by unblockPred.
+     *
+     * @param sndMsgs If {@code true} sends blocked messages.
+     * @param unblockPred If not null unblocks only messages allowed by predicate.
+     */
+    public void stopBlock(boolean sndMsgs, @Nullable IgnitePredicate<T2<ClusterNode, GridIoMessage>> unblockPred) {
+        synchronized (this) {
             blockCls.clear();
             blockP = null;
 
+            Collection<T2<ClusterNode, GridIoMessage>> msgs =
+                unblockPred == null ? blockedMsgs : F.view(blockedMsgs, unblockPred);
+
             if (sndMsgs) {
-                for (T2<ClusterNode, GridIoMessage> msg : blockedMsgs) {
+                for (T2<ClusterNode, GridIoMessage> msg : msgs) {
                     try {
                         ignite.log().info("Send blocked message [node=" + msg.get1().id() +
                             ", order=" + msg.get1().order() +
@@ -284,7 +312,7 @@ public void stopBlock(boolean sndMsgs) {
                 }
             }
 
-            blockedMsgs.clear();
+            msgs.clear();
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 30d80f65d212e..c2c5ff46edb61 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -24,6 +24,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.concurrent.CountDownLatch;
@@ -31,7 +32,10 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.List;
+import java.util.concurrent.atomic.LongAdder;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteAtomicSequence;
 import org.apache.ignite.IgniteCache;
@@ -46,19 +50,45 @@
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.commandline.CommandHandler;
 import org.apache.ignite.internal.commandline.cache.CacheCommand;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.processors.cache.CacheObjectImpl;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
+import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
+import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
+import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.tx.VisorTxInfo;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
+import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionRollbackException;
+import org.apache.ignite.transactions.TransactionTimeoutException;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
@@ -130,10 +160,12 @@ protected void injectTestSystemOut() {
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
         cfg.setConnectorConfiguration(new ConnectorConfiguration());
 
         DataStorageConfiguration memCfg = new DataStorageConfiguration().setDefaultDataRegionConfiguration(
-            new DataRegionConfiguration().setMaxSize(100 * 1024 * 1024));
+            new DataRegionConfiguration().setMaxSize(100L * 1024 * 1024));
 
         cfg.setDataStorageConfiguration(memCfg);
 
@@ -143,7 +175,7 @@ protected void injectTestSystemOut() {
 
         cfg.setConsistentId(igniteInstanceName);
 
-        cfg.setClientMode("client".equals(igniteInstanceName));
+        cfg.setClientMode(igniteInstanceName.startsWith("client"));
 
         return cfg;
     }
@@ -462,26 +494,22 @@ else if (entry.getKey().equals(node2)) {
 
             assertNotNull(res);
 
-            for (VisorTxInfo txInfo : res.getInfos()) {
+            for (VisorTxInfo txInfo : res.getInfos())
                 assertTrue(txInfo.getSize() >= minSize);
-
-            }
         }, "--tx", "minSize", Integer.toString(minSize));
 
         // test order by size.
         validate(h, map -> {
             VisorTxTaskResult res = map.get(grid(0).localNode());
 
-            assertTrue(res.getInfos().get(0).getSize() >= res.getInfos().get(1).getSize());
-
+            assertTrue(res.getInfos().get(0).getSize() >=  res.getInfos().get(1).getSize());
         }, "--tx", "order", "SIZE");
 
         // test order by duration.
         validate(h, map -> {
             VisorTxTaskResult res = map.get(grid(0).localNode());
 
-            assertTrue(res.getInfos().get(0).getDuration() >= res.getInfos().get(1).getDuration());
-
+            assertTrue(res.getInfos().get(0).getDuration() >=  res.getInfos().get(1).getDuration());
         }, "--tx", "order", "DURATION");
 
         // test order by start_time.
@@ -530,6 +558,251 @@ else if (entry.getKey().equals(node2)) {
         checkFutures();
     }
 
+    /**
+     *
+     */
+    public void testKillHangingLocalTransactions() throws Exception {
+        Ignite ignite = startGridsMultiThreaded(2);
+
+        ignite.cluster().active(true);
+
+        Ignite client = startGrid("client");
+
+        client.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME).
+            setAtomicityMode(TRANSACTIONAL).
+            setWriteSynchronizationMode(FULL_SYNC).
+            setAffinity(new RendezvousAffinityFunction(false, 64)));
+
+        Ignite prim = primaryNode(0L, DEFAULT_CACHE_NAME);
+
+        // Blocks lock response to near node.
+        TestRecordingCommunicationSpi.spi(prim).blockMessages(GridNearLockResponse.class, client.name());
+
+        TestRecordingCommunicationSpi.spi(client).blockMessages(GridNearTxFinishRequest.class, prim.name());
+
+        GridNearTxLocal clientTx = null;
+
+        try(Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 2000, 1)) {
+            clientTx = ((TransactionProxyImpl)tx).tx();
+
+            client.cache(DEFAULT_CACHE_NAME).put(0L, 0L);
+
+            fail();
+        }
+        catch (Exception e) {
+            assertTrue(X.hasCause(e, TransactionTimeoutException.class));
+        }
+
+        assertNotNull(clientTx);
+
+        IgniteEx primEx = (IgniteEx)prim;
+
+        IgniteInternalTx tx0 = primEx.context().cache().context().tm().activeTransactions().iterator().next();
+
+        assertNotNull(tx0);
+
+        CommandHandler h = new CommandHandler();
+
+        validate(h, map -> {
+            ClusterNode node = grid(0).cluster().localNode();
+
+            VisorTxTaskResult res = map.get(node);
+
+            for (VisorTxInfo info : res.getInfos())
+                assertEquals(tx0.xid(), info.getXid());
+
+            assertEquals(1, map.size());
+        }, "--tx", "kill");
+
+        tx0.finishFuture().get();
+
+        TestRecordingCommunicationSpi.spi(prim).stopBlock();
+
+        TestRecordingCommunicationSpi.spi(client).stopBlock();
+
+        IgniteInternalFuture<?> nearFinFut = U.field(clientTx, "finishFut");
+
+        nearFinFut.get();
+
+        checkFutures();
+    }
+
+    /**
+     * Simulate uncommitted backup transactions and test rolling back using utility.
+     */
+    public void testKillHangingRemoteTransactions() throws Exception {
+        final int cnt = 3;
+
+        startGridsMultiThreaded(cnt);
+
+        Ignite[] clients = new Ignite[] {
+            startGrid("client1"),
+            startGrid("client2"),
+            startGrid("client3"),
+            startGrid("client4")
+        };
+
+        clients[0].getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME).
+            setBackups(2).
+            setAtomicityMode(TRANSACTIONAL).
+            setWriteSynchronizationMode(FULL_SYNC).
+            setAffinity(new RendezvousAffinityFunction(false, 64)));
+
+        for (Ignite client : clients) {
+            assertTrue(client.configuration().isClientMode());
+
+            assertNotNull(client.cache(DEFAULT_CACHE_NAME));
+        }
+
+        LongAdder progress = new LongAdder();
+
+        AtomicInteger idx = new AtomicInteger();
+
+        int tc = clients.length;
+
+        CountDownLatch lockLatch = new CountDownLatch(1);
+        CountDownLatch commitLatch = new CountDownLatch(1);
+
+        Ignite prim = primaryNode(0L, DEFAULT_CACHE_NAME);
+
+        TestRecordingCommunicationSpi primSpi = TestRecordingCommunicationSpi.spi(prim);
+
+        primSpi.blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+            @Override public boolean apply(ClusterNode node, Message message) {
+                return message instanceof GridDhtTxFinishRequest;
+            }
+        });
+
+        IgniteInternalFuture<?> fut = multithreadedAsync(new Runnable() {
+            @Override public void run() {
+                int id = idx.getAndIncrement();
+
+                Ignite client = clients[id];
+
+                try (Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 0, 1)) {
+                    IgniteCache<Long, Long> cache = client.cache(DEFAULT_CACHE_NAME);
+
+                    if (id != 0)
+                        U.awaitQuiet(lockLatch);
+
+                    cache.invoke(0L, new IncrementClosure(), null);
+
+                    if (id == 0) {
+                        lockLatch.countDown();
+
+                        U.awaitQuiet(commitLatch);
+
+                        doSleep(500); // Wait until candidates will enqueue.
+                    }
+
+                    tx.commit();
+                }
+                catch (Exception e) {
+                    assertTrue(X.hasCause(e, TransactionTimeoutException.class));
+                }
+
+                progress.increment();
+
+            }
+        }, tc, "invoke-thread");
+
+        U.awaitQuiet(lockLatch);
+
+        commitLatch.countDown();
+
+        primSpi.waitForBlocked(clients.length);
+
+        // Unblock only finish messages from clients from 2 to 4.
+        primSpi.stopBlock(true, new IgnitePredicate<T2<ClusterNode,GridIoMessage>>() {
+            @Override public boolean apply(T2<ClusterNode, GridIoMessage> objects) {
+                GridIoMessage iom = objects.get2();
+
+                Message m = iom.message();
+
+                if (m instanceof GridDhtTxFinishRequest) {
+                    GridDhtTxFinishRequest r = (GridDhtTxFinishRequest)m;
+
+                    if (r.nearNodeId().equals(clients[0].cluster().localNode().id()))
+                        return false;
+                }
+
+                return true;
+            }
+        });
+
+        // Wait until queue is stable
+        for (Ignite ignite : G.allGrids()) {
+            if (ignite.configuration().isClientMode())
+                continue;
+
+            Collection<IgniteInternalTx> txs = ((IgniteEx)ignite).context().cache().context().tm().activeTransactions();
+
+            GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    for (IgniteInternalTx tx : txs)
+                        if (!tx.local()) {
+                            IgniteTxEntry entry = tx.writeEntries().iterator().next();
+
+                            GridCacheEntryEx cached = entry.cached();
+
+                            Collection<GridCacheMvccCandidate> candidates = cached.remoteMvccSnapshot();
+
+                            if (candidates.size() != clients.length)
+                                return false;
+                        }
+
+                    return true;
+                }
+            }, 10_000);
+        }
+
+        CommandHandler h = new CommandHandler();
+
+        // Check listing.
+        validate(h, map -> {
+            for (int i = 0; i < cnt; i++) {
+                IgniteEx grid = grid(i);
+
+                // Skip primary.
+                if (grid.localNode().id().equals(prim.cluster().localNode().id()))
+                    continue;
+
+                VisorTxTaskResult res = map.get(grid.localNode());
+
+                // Validate queue length on backups.
+                assertEquals(clients.length, res.getInfos().size());
+            }
+        }, "--tx");
+
+        // Check kill.
+        validate(h, map -> {
+            // No-op.
+        }, "--tx", "kill");
+
+        // Wait for all remote txs to finish.
+        for (Ignite ignite : G.allGrids()) {
+            if (ignite.configuration().isClientMode())
+                continue;
+
+            Collection<IgniteInternalTx> txs = ((IgniteEx)ignite).context().cache().context().tm().activeTransactions();
+
+            for (IgniteInternalTx tx : txs)
+                if (!tx.local())
+                    tx.finishFuture().get();
+        }
+
+        // Unblock finish message from client1.
+        primSpi.stopBlock(true);
+
+        fut.get();
+
+        Long cur = (Long)clients[0].cache(DEFAULT_CACHE_NAME).get(0L);
+
+        assertEquals(tc - 1, cur.longValue());
+
+        checkFutures();
+    }
+
     /**
      * Test baseline add items works via control.sh
      *
@@ -937,6 +1210,23 @@ private void checkFutures() {
                 log.info("Waiting for future: " + fut);
 
             assertTrue("Expecting no active futures: node=" + ig.localNode().id(), futs.isEmpty());
+
+            Collection<IgniteInternalTx> txs = ig.context().cache().context().tm().activeTransactions();
+
+            for (IgniteInternalTx tx : txs)
+                log.info("Waiting for tx: " + tx);
+
+            assertTrue("Expecting no active transactions: node=" + ig.localNode().id(), txs.isEmpty());
+        }
+    }
+
+    /** */
+    private static class IncrementClosure implements EntryProcessor<Long, Long, Void> {
+        /** {@inheritDoc} */
+        @Override public Void process(MutableEntry<Long, Long> entry, Object... arguments) throws EntryProcessorException {
+            entry.setValue(entry.exists() ? entry.getValue() + 1 : 0);
+
+            return null;
         }
     }
 }

From 6517f2409f45567a119fadf298811fc8de6ede53 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Wed, 20 Jun 2018 14:05:53 +0300
Subject: [PATCH 221/543] IGNITE-8491 Add JMX flag: Is the node in baseline or
 not - fixed license header

(cherry picked from commit d58f368)
---
 .../util/mbeans/GridMBeanBaselineTest.java      | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanBaselineTest.java
index 00dce83f1cca6..31c012c8b2a49 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanBaselineTest.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.ignite.util.mbeans;
 
 import org.apache.ignite.Ignite;

From d054ab588e33b579dca72411b6096881f771a3d2 Mon Sep 17 00:00:00 2001
From: Dmitriy Sorokin <d.w.sorokin@gmail.com>
Date: Wed, 20 Jun 2018 16:41:46 +0300
Subject: [PATCH 222/543] IGNITE-8749 Exception for "no space left" situation
 should be propagated to FailureHandler - Fixes #4200.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 3fff8a8)
Signed-off-by: Ivan Rakov <irakov@apache.org>
---
 .../wal/FileWriteAheadLogManager.java         | 190 +++++++------
 .../FsyncModeFileWriteAheadLogManager.java    | 235 +++++++++-------
 .../ignite/failure/TestFailureHandler.java    |  19 ++
 .../wal/IgniteWalFormatFileFailoverTest.java  | 258 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 5 files changed, 513 insertions(+), 192 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 9b39987edf42a..3f406629f9e49 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -605,48 +605,43 @@ private void checkWalConfiguration() throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
-        try {
-            assert currHnd == null;
-            assert lastPtr == null || lastPtr instanceof FileWALPointer;
+        assert currHnd == null;
+        assert lastPtr == null || lastPtr instanceof FileWALPointer;
 
-            FileWALPointer filePtr = (FileWALPointer)lastPtr;
+        FileWALPointer filePtr = (FileWALPointer)lastPtr;
 
-            walWriter = new WALWriter(log);
+        walWriter = new WALWriter(log);
 
-            if (!mmap)
-                new IgniteThread(walWriter).start();
+        if (!mmap)
+            new IgniteThread(walWriter).start();
 
-            currHnd = restoreWriteHandle(filePtr);
+        currHnd = restoreWriteHandle(filePtr);
 
-            // For new handle write serializer version to it.
-            if (filePtr == null)
-                currHnd.writeHeader();
+        // For new handle write serializer version to it.
+        if (filePtr == null)
+            currHnd.writeHeader();
 
-            if (currHnd.serializer.version() != serializer.version()) {
-                if (log.isInfoEnabled())
-                    log.info("Record serializer version change detected, will start logging with a new WAL record " +
-                        "serializer to a new WAL segment [curFile=" + currHnd + ", newVer=" + serializer.version() +
-                        ", oldVer=" + currHnd.serializer.version() + ']');
+        if (currHnd.serializer.version() != serializer.version()) {
+            if (log.isInfoEnabled())
+                log.info("Record serializer version change detected, will start logging with a new WAL record " +
+                    "serializer to a new WAL segment [curFile=" + currHnd + ", newVer=" + serializer.version() +
+                    ", oldVer=" + currHnd.serializer.version() + ']');
 
-                rollOver(currHnd);
-            }
+            rollOver(currHnd);
+        }
 
-            currHnd.resume = false;
+        currHnd.resume = false;
 
-            if (mode == WALMode.BACKGROUND) {
-                backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
-                    @Override public void run() {
-                        doFlush();
-                    }
-                }, flushFreq, flushFreq);
-            }
-
-            if (walAutoArchiveAfterInactivity > 0)
-                scheduleNextInactivityPeriodElapsedCheck();
-        }
-        catch (StorageException e) {
-            throw new IgniteCheckedException(e);
+        if (mode == WALMode.BACKGROUND) {
+            backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
+                @Override public void run() {
+                    doFlush();
+                }
+            }, flushFreq, flushFreq);
         }
+
+        if (walAutoArchiveAfterInactivity > 0)
+            scheduleNextInactivityPeriodElapsedCheck();
     }
 
     /**
@@ -1131,9 +1126,9 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, I
     /**
      * @param lastReadPtr Last read WAL file pointer.
      * @return Initialized file write handle.
-     * @throws IgniteCheckedException If failed to initialize WAL write handle.
+     * @throws StorageException If failed to initialize WAL write handle.
      */
-    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws IgniteCheckedException {
+    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws StorageException {
         long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
 
         @Nullable FileArchiver archiver0 = archiver;
@@ -1175,14 +1170,9 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
                 SegmentedRingByteBuffer rbuf;
 
                 if (mmap) {
-                    try {
-                        MappedByteBuffer buf = fileIO.map((int)maxWalSegmentSize);
+                    MappedByteBuffer buf = fileIO.map((int)maxWalSegmentSize);
 
-                        rbuf = new SegmentedRingByteBuffer(buf, metrics);
-                    }
-                    catch (IOException e) {
-                        throw new IgniteCheckedException(e);
-                    }
+                    rbuf = new SegmentedRingByteBuffer(buf, metrics);
                 }
                 else
                     rbuf = new SegmentedRingByteBuffer(dsCfg.getWalBufferSize(), maxWalSegmentSize, DIRECT, metrics);
@@ -1206,13 +1196,21 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
                 return hnd;
             }
             catch (IgniteCheckedException | IOException e) {
-                fileIO.close();
+                try {
+                    fileIO.close();
+                }
+                catch (IOException suppressed) {
+                    e.addSuppressed(suppressed);
+                }
 
-                throw e;
+                if (e instanceof StorageException)
+                    throw (StorageException) e;
+
+                throw e instanceof IOException ? (IOException) e : new IOException(e);
             }
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
+            throw new StorageException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
         }
     }
 
@@ -1222,10 +1220,11 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
      *
      * @param cur Current file write handle released by WAL writer
      * @return Initialized file handle.
-     * @throws StorageException If IO exception occurred.
-     * @throws IgniteCheckedException If failed.
+     * @throws IgniteCheckedException If exception occurred.
      */
-    private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageException, IgniteCheckedException {
+    private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws IgniteCheckedException {
+        IgniteCheckedException error = null;
+
         try {
             File nextFile = pollNextFile(cur.idx);
 
@@ -1299,19 +1298,24 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws StorageE
 
             return hnd;
         }
+        catch (IgniteCheckedException e) {
+            throw error = e;
+        }
         catch (IOException e) {
-            StorageException se = new StorageException("Unable to initialize WAL segment", e);
-
-            cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
-
-            throw se;
+            throw error = new StorageException("Unable to initialize WAL segment", e);
+        }
+        finally {
+            if (error != null)
+                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, error));
         }
     }
 
     /**
      * Deletes temp files, creates and prepares new; Creates first segment if necessary
+     *
+     * @throws StorageException If failed.
      */
-    private void checkOrPrepareFiles() throws IgniteCheckedException {
+    private void checkOrPrepareFiles() throws StorageException {
         // Clean temp files.
         {
             File[] tmpFiles = walWorkDir.listFiles(WAL_SEGMENT_TEMP_FILE_FILTER);
@@ -1321,7 +1325,7 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
                     boolean deleted = tmp.delete();
 
                     if (!deleted)
-                        throw new IgniteCheckedException("Failed to delete previously created temp file " +
+                        throw new StorageException("Failed to delete previously created temp file " +
                             "(make sure Ignite process has enough rights): " + tmp.getAbsolutePath());
                 }
             }
@@ -1331,7 +1335,7 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
 
         if(isArchiverEnabled())
             if (allFiles.length != 0 && allFiles.length > dsCfg.getWalSegments())
-                throw new IgniteCheckedException("Failed to initialize wal (work directory contains " +
+                throw new StorageException("Failed to initialize wal (work directory contains " +
                     "incorrect number of segments) [cur=" + allFiles.length + ", expected=" + dsCfg.getWalSegments() + ']');
 
         // Allocate the first segment synchronously. All other segments will be allocated by archiver in background.
@@ -1371,9 +1375,9 @@ public void cleanupWalDirectories() throws IgniteCheckedException {
      * Clears whole the file, fills with zeros for Default mode.
      *
      * @param file File to format.
-     * @throws IgniteCheckedException if formatting failed
+     * @throws StorageException if formatting failed
      */
-    private void formatFile(File file) throws IgniteCheckedException {
+    private void formatFile(File file) throws StorageException {
         formatFile(file, dsCfg.getWalSegmentSize());
     }
 
@@ -1382,9 +1386,9 @@ private void formatFile(File file) throws IgniteCheckedException {
      *
      * @param file File to format.
      * @param bytesCntToFormat Count of first bytes to format.
-     * @throws IgniteCheckedException if formatting failed
+     * @throws StorageException if formatting failed
      */
-    private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedException {
+    private void formatFile(File file, int bytesCntToFormat) throws StorageException {
         if (log.isDebugEnabled())
             log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1396,7 +1400,7 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                     int toWrite = Math.min(FILL_BUF.length, left);
 
                     if (fileIO.write(FILL_BUF, 0, toWrite) < toWrite) {
-                        final IgniteCheckedException ex = new IgniteCheckedException("Failed to extend WAL segment file: " +
+                        final StorageException ex = new StorageException("Failed to extend WAL segment file: " +
                             file.getName() + ". Probably disk is too busy, please check your device.");
 
                         if (failureProcessor != null)
@@ -1414,7 +1418,7 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                 fileIO.clear();
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
+            throw new StorageException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
         }
     }
 
@@ -1422,9 +1426,9 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
      * Creates a file atomically with temp file.
      *
      * @param file File to create.
-     * @throws IgniteCheckedException If failed.
+     * @throws StorageException If failed.
      */
-    private void createFile(File file) throws IgniteCheckedException {
+    private void createFile(File file) throws StorageException {
         if (log.isDebugEnabled())
             log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1436,7 +1440,7 @@ private void createFile(File file) throws IgniteCheckedException {
             Files.move(tmp.toPath(), file.toPath());
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to move temp file to a regular WAL segment file: " +
+            throw new StorageException("Failed to move temp file to a regular WAL segment file: " +
                 file.getAbsolutePath(), e);
         }
 
@@ -1449,9 +1453,9 @@ private void createFile(File file) throws IgniteCheckedException {
      *
      * @param curIdx Current absolute WAL segment index.
      * @return File ready for use as new WAL segment.
-     * @throws IgniteCheckedException If failed.
+     * @throws StorageException If exception occurred in the archiver thread.
      */
-    private File pollNextFile(long curIdx) throws IgniteCheckedException {
+    private File pollNextFile(long curIdx) throws StorageException {
         FileArchiver archiver0 = archiver;
 
         if (archiver0 == null) {
@@ -1527,7 +1531,7 @@ public long maxWalSegmentSize() {
      */
     private class FileArchiver extends GridWorker {
         /** Exception which occurred during initial creation of files or during archiving WAL segment */
-        private IgniteCheckedException cleanErr;
+        private StorageException cleanErr;
 
         /**
          * Absolute current segment index WAL Manager writes to. Guarded by <code>this</code>. Incremented during
@@ -1599,15 +1603,17 @@ private synchronized boolean locked(long absIdx) {
             try {
                 allocateRemainingFiles();
             }
-            catch (IgniteCheckedException e) {
+            catch (StorageException e) {
                 synchronized (this) {
                     // Stop the thread and report to starter.
                     cleanErr = e;
 
                     notifyAll();
-
-                    return;
                 }
+
+                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
+
+                return;
             }
 
             Throwable err = null;
@@ -1691,9 +1697,9 @@ private void changeLastArchivedIndexAndNotifyWaiters(long idx) {
          *
          * @param curIdx Current absolute index that we want to increment.
          * @return Next index (curWalSegmIdx+1) when it is ready to be written.
-         * @throws IgniteCheckedException If failed (if interrupted or if exception occurred in the archiver thread).
+         * @throws StorageException If exception occurred in the archiver thread.
          */
-        private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException {
+        private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException {
             synchronized (this) {
                 if (cleanErr != null)
                     throw cleanErr;
@@ -1708,6 +1714,9 @@ private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException
                 while (curAbsWalIdx - lastAbsArchivedIdx > dsCfg.getWalSegments() && cleanErr == null) {
                     try {
                         wait();
+
+                        if (cleanErr != null)
+                            throw cleanErr;
                     }
                     catch (InterruptedException ignore) {
                         interrupted.set(true);
@@ -1715,9 +1724,12 @@ private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException
                 }
 
                 // Wait for formatter so that we do not open an empty file in DEFAULT mode.
-                while (curAbsWalIdx % dsCfg.getWalSegments() > formatted)
+                while (curAbsWalIdx % dsCfg.getWalSegments() > formatted && cleanErr == null)
                     try {
                         wait();
+
+                        if (cleanErr != null)
+                            throw cleanErr;
                     }
                     catch (InterruptedException ignore) {
                         interrupted.set(true);
@@ -1789,7 +1801,7 @@ private void releaseWorkSegment(long absIdx) {
          *
          * @param absIdx Absolute index to archive.
          */
-        private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedException {
+        private SegmentArchiveResult archiveSegment(long absIdx) throws StorageException {
             long segIdx = absIdx % dsCfg.getWalSegments();
 
             File origFile = new File(walWorkDir, FileDescriptor.fileName(segIdx));
@@ -1818,7 +1830,7 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedExc
                 }
             }
             catch (IOException e) {
-                throw new IgniteCheckedException("Failed to archive WAL segment [" +
+                throw new StorageException("Failed to archive WAL segment [" +
                     "srcFile=" + origFile.getAbsolutePath() +
                     ", dstFile=" + dstTmpFile.getAbsolutePath() + ']', e);
             }
@@ -1841,7 +1853,7 @@ private boolean checkStop() {
          * Background creation of all segments except first. First segment was created in main thread by {@link
          * FileWriteAheadLogManager#checkOrPrepareFiles()}
          */
-        private void allocateRemainingFiles() throws IgniteCheckedException {
+        private void allocateRemainingFiles() throws StorageException {
             checkFiles(
                 1,
                 true,
@@ -2235,23 +2247,23 @@ private void shutdown() throws IgniteInterruptedCheckedException {
      * @param startWith Start with.
      * @param create Flag create file.
      * @param p Predicate Exit condition.
-     * @throws IgniteCheckedException if validation or create file fail.
+     * @throws StorageException if validation or create file fail.
      */
     private void checkFiles(
         int startWith,
         boolean create,
         @Nullable IgnitePredicate<Integer> p,
         @Nullable IgniteInClosure<Integer> completionCallback
-    ) throws IgniteCheckedException {
+    ) throws StorageException {
         for (int i = startWith; i < dsCfg.getWalSegments() && (p == null || p.apply(i)); i++) {
             File checkFile = new File(walWorkDir, FileDescriptor.fileName(i));
 
             if (checkFile.exists()) {
                 if (checkFile.isDirectory())
-                    throw new IgniteCheckedException("Failed to initialize WAL log segment (a directory with " +
+                    throw new StorageException("Failed to initialize WAL log segment (a directory with " +
                         "the same name already exists): " + checkFile.getAbsolutePath());
                 else if (checkFile.length() != dsCfg.getWalSegmentSize() && mode == WALMode.FSYNC)
-                    throw new IgniteCheckedException("Failed to initialize WAL log segment " +
+                    throw new StorageException("Failed to initialize WAL log segment " +
                         "(WAL segment size change is not supported in 'DEFAULT' WAL mode) " +
                         "[filePath=" + checkFile.getAbsolutePath() +
                         ", fileSize=" + checkFile.length() +
@@ -2651,9 +2663,8 @@ public void writeHeader() {
          * Flush or wait for concurrent flush completion.
          *
          * @param ptr Pointer.
-         * @throws IgniteCheckedException If failed.
          */
-        private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException {
+        private void flushOrWait(FileWALPointer ptr) {
             if (ptr != null) {
                 // If requested obsolete file index, it must be already flushed by close.
                 if (ptr.index() != idx)
@@ -2665,10 +2676,8 @@ private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException {
 
         /**
          * @param ptr Pointer.
-         * @throws IgniteCheckedException If failed.
-         * @throws StorageException If failed.
          */
-        private void flush(FileWALPointer ptr) throws IgniteCheckedException, StorageException {
+        private void flush(FileWALPointer ptr) {
             if (ptr == null) { // Unconditional flush.
                 walWriter.flushAll();
 
@@ -2884,7 +2893,7 @@ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageEx
                         }
                     }
                     catch (IOException e) {
-                        throw new IgniteCheckedException(e);
+                        throw new StorageException("Failed to close WAL write handle [idx=" + idx + "]", e);
                     }
 
                     if (log.isDebugEnabled())
@@ -3365,28 +3374,29 @@ private void unparkWaiters(long pos) {
         /**
          * Forces all made changes to the file.
          */
-        void force() throws IgniteCheckedException {
+        void force() {
             flushBuffer(FILE_FORCE);
         }
 
         /**
          * Closes file.
          */
-        void close() throws IgniteCheckedException {
+        void close() {
             flushBuffer(FILE_CLOSE);
         }
 
         /**
          * Flushes all data from the buffer.
          */
-        void flushAll() throws IgniteCheckedException {
+        void flushAll() {
             flushBuffer(UNCONDITIONAL_FLUSH);
         }
 
         /**
          * @param expPos Expected position.
          */
-        void flushBuffer(long expPos) throws StorageException, IgniteCheckedException {
+        @SuppressWarnings("ForLoopReplaceableByForEach")
+        void flushBuffer(long expPos) {
             if (mmap)
                 return;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 49fbc73b9a48c..6e59ad3614dec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -487,37 +487,32 @@ private void checkWalConfiguration() throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
-        try {
-            assert currentHnd == null;
-            assert lastPtr == null || lastPtr instanceof FileWALPointer;
-
-            FileWALPointer filePtr = (FileWALPointer)lastPtr;
+        assert currentHnd == null;
+        assert lastPtr == null || lastPtr instanceof FileWALPointer;
 
-            currentHnd = restoreWriteHandle(filePtr);
-
-            if (currentHnd.serializer.version() != serializer.version()) {
-                if (log.isInfoEnabled())
-                    log.info("Record serializer version change detected, will start logging with a new WAL record " +
-                        "serializer to a new WAL segment [curFile=" + currentHnd + ", newVer=" + serializer.version() +
-                        ", oldVer=" + currentHnd.serializer.version() + ']');
+        FileWALPointer filePtr = (FileWALPointer)lastPtr;
 
-                rollOver(currentHnd);
-            }
+        currentHnd = restoreWriteHandle(filePtr);
 
-            if (mode == WALMode.BACKGROUND) {
-                backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
-                    @Override public void run() {
-                        doFlush();
-                    }
-                }, flushFreq, flushFreq);
-            }
+        if (currentHnd.serializer.version() != serializer.version()) {
+            if (log.isInfoEnabled())
+                log.info("Record serializer version change detected, will start logging with a new WAL record " +
+                    "serializer to a new WAL segment [curFile=" + currentHnd + ", newVer=" + serializer.version() +
+                    ", oldVer=" + currentHnd.serializer.version() + ']');
 
-            if (walAutoArchiveAfterInactivity > 0)
-                scheduleNextInactivityPeriodElapsedCheck();
+            rollOver(currentHnd);
         }
-        catch (StorageException e) {
-            throw new IgniteCheckedException(e);
+
+        if (mode == WALMode.BACKGROUND) {
+            backgroundFlushSchedule = cctx.time().schedule(new Runnable() {
+                @Override public void run() {
+                    doFlush();
+                }
+            }, flushFreq, flushFreq);
         }
+
+        if (walAutoArchiveAfterInactivity > 0)
+            scheduleNextInactivityPeriodElapsedCheck();
     }
 
     /**
@@ -1019,7 +1014,7 @@ private FileWriteHandle currentHandle() {
      * @param cur Handle that failed to fit the given entry.
      * @return Handle that will fit the entry.
      */
-    private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, IgniteCheckedException {
+    private FileWriteHandle rollOver(FileWriteHandle cur) throws IgniteCheckedException {
         FileWriteHandle hnd = currentHandle();
 
         if (hnd != cur)
@@ -1050,9 +1045,9 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, I
     /**
      * @param lastReadPtr Last read WAL file pointer.
      * @return Initialized file write handle.
-     * @throws IgniteCheckedException If failed to initialize WAL write handle.
+     * @throws StorageException If failed to initialize WAL write handle.
      */
-    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws IgniteCheckedException {
+    private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws StorageException {
         long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
 
         long segNo = absIdx % dsCfg.getWalSegments();
@@ -1100,13 +1095,21 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
                 return hnd;
             }
             catch (IgniteCheckedException | IOException e) {
-                fileIO.close();
+                try {
+                    fileIO.close();
+                }
+                catch (IOException suppressed) {
+                    e.addSuppressed(suppressed);
+                }
 
-                throw e;
+                if (e instanceof StorageException)
+                    throw (StorageException) e;
+
+                throw e instanceof IOException ? (IOException) e : new IOException(e);
             }
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
+            throw new StorageException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
         }
     }
 
@@ -1117,10 +1120,11 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws Ig
      *
      * @param curIdx current absolute segment released by WAL writer
      * @return Initialized file handle.
-     * @throws StorageException If IO exception occurred.
-     * @throws IgniteCheckedException If failed.
+     * @throws IgniteCheckedException If exception occurred.
      */
-    private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException, IgniteCheckedException {
+    private FileWriteHandle initNextWriteHandle(long curIdx) throws IgniteCheckedException {
+        IgniteCheckedException error = null;
+
         try {
             File nextFile = pollNextFile(curIdx);
 
@@ -1140,19 +1144,24 @@ private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException
 
             return hnd;
         }
+        catch (IgniteCheckedException e) {
+            throw error = e;
+        }
         catch (IOException e) {
-            StorageException se = new StorageException("Unable to initialize WAL segment", e);
-
-            cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
-
-            throw se;
+            throw error = new StorageException("Unable to initialize WAL segment", e);
+        }
+        finally {
+            if (error != null)
+                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, error));
         }
     }
 
     /**
-     * Deletes temp files, creates and prepares new; Creates first segment if necessary
+     * Deletes temp files, creates and prepares new; Creates first segment if necessary.
+     *
+     * @throws StorageException If failed.
      */
-    private void checkOrPrepareFiles() throws IgniteCheckedException {
+    private void checkOrPrepareFiles() throws StorageException {
         // Clean temp files.
         {
             File[] tmpFiles = walWorkDir.listFiles(WAL_SEGMENT_TEMP_FILE_FILTER);
@@ -1162,7 +1171,7 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
                     boolean deleted = tmp.delete();
 
                     if (!deleted)
-                        throw new IgniteCheckedException("Failed to delete previously created temp file " +
+                        throw new StorageException("Failed to delete previously created temp file " +
                             "(make sure Ignite process has enough rights): " + tmp.getAbsolutePath());
                 }
             }
@@ -1171,7 +1180,7 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
         File[] allFiles = walWorkDir.listFiles(WAL_SEGMENT_FILE_FILTER);
 
         if (allFiles.length != 0 && allFiles.length > dsCfg.getWalSegments())
-            throw new IgniteCheckedException("Failed to initialize wal (work directory contains " +
+            throw new StorageException("Failed to initialize wal (work directory contains " +
                 "incorrect number of segments) [cur=" + allFiles.length + ", expected=" + dsCfg.getWalSegments() + ']');
 
         // Allocate the first segment synchronously. All other segments will be allocated by archiver in background.
@@ -1188,9 +1197,9 @@ private void checkOrPrepareFiles() throws IgniteCheckedException {
      * Clears whole the file, fills with zeros for Default mode.
      *
      * @param file File to format.
-     * @throws IgniteCheckedException if formatting failed
+     * @throws StorageException if formatting failed.
      */
-    private void formatFile(File file) throws IgniteCheckedException {
+    private void formatFile(File file) throws StorageException {
         formatFile(file, dsCfg.getWalSegmentSize());
     }
 
@@ -1199,9 +1208,9 @@ private void formatFile(File file) throws IgniteCheckedException {
      *
      * @param file File to format.
      * @param bytesCntToFormat Count of first bytes to format.
-     * @throws IgniteCheckedException if formatting failed
+     * @throws StorageException If formatting failed.
      */
-    private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedException {
+    private void formatFile(File file, int bytesCntToFormat) throws StorageException {
         if (log.isDebugEnabled())
             log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1223,7 +1232,7 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
                 fileIO.clear();
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
+            throw new StorageException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
         }
     }
 
@@ -1231,9 +1240,9 @@ private void formatFile(File file, int bytesCntToFormat) throws IgniteCheckedExc
      * Creates a file atomically with temp file.
      *
      * @param file File to create.
-     * @throws IgniteCheckedException If failed.
+     * @throws StorageException If failed.
      */
-    private void createFile(File file) throws IgniteCheckedException {
+    private void createFile(File file) throws StorageException {
         if (log.isDebugEnabled())
             log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
@@ -1245,7 +1254,7 @@ private void createFile(File file) throws IgniteCheckedException {
             Files.move(tmp.toPath(), file.toPath());
         }
         catch (IOException e) {
-            throw new IgniteCheckedException("Failed to move temp file to a regular WAL segment file: " +
+            throw new StorageException("Failed to move temp file to a regular WAL segment file: " +
                 file.getAbsolutePath(), e);
         }
 
@@ -1259,9 +1268,10 @@ private void createFile(File file) throws IgniteCheckedException {
      *
      * @param curIdx Current absolute WAL segment index.
      * @return File ready for use as new WAL segment.
-     * @throws IgniteCheckedException If failed.
+     * @throws StorageException If exception occurred in the archiver thread.
+     * @throws IgniteInterruptedCheckedException If interrupted.
      */
-    private File pollNextFile(long curIdx) throws IgniteCheckedException {
+    private File pollNextFile(long curIdx) throws StorageException, IgniteInterruptedCheckedException {
         // Signal to archiver that we are done with the segment and it can be archived.
         long absNextIdx = archiver.nextAbsoluteSegmentIndex(curIdx);
 
@@ -1318,7 +1328,7 @@ private void checkNode() throws StorageException {
      */
     private class FileArchiver extends GridWorker {
         /** Exception which occurred during initial creation of files or during archiving WAL segment */
-        private IgniteCheckedException cleanException;
+        private StorageException cleanException;
 
         /**
          * Absolute current segment index WAL Manager writes to. Guarded by <code>this</code>.
@@ -1426,15 +1436,17 @@ private synchronized void release(long absIdx) {
             try {
                 allocateRemainingFiles();
             }
-            catch (IgniteCheckedException e) {
+            catch (StorageException e) {
                 synchronized (this) {
                     // Stop the thread and report to starter.
                     cleanException = e;
 
                     notifyAll();
-
-                    return;
                 }
+
+                cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, e));
+
+                return;
             }
 
             Throwable err = null;
@@ -1515,9 +1527,10 @@ private void changeLastArchivedIndexAndWakeupCompressor(long idx) {
          *
          * @param curIdx Current absolute index that we want to increment.
          * @return Next index (curWalSegmIdx+1) when it is ready to be written.
-         * @throws IgniteCheckedException If failed (if interrupted or if exception occurred in the archiver thread).
+         * @throws StorageException If exception occurred in the archiver thread.
+         * @throws IgniteInterruptedCheckedException If interrupted.
          */
-        private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException {
+        private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException, IgniteInterruptedCheckedException {
             try {
                 synchronized (this) {
                     if (cleanException != null)
@@ -1535,10 +1548,16 @@ private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException
                     while ((curAbsWalIdx - lastAbsArchivedIdx > segments && cleanException == null))
                         wait();
 
+                    if (cleanException != null)
+                        throw cleanException;
+
                     // Wait for formatter so that we do not open an empty file in DEFAULT mode.
-                    while (curAbsWalIdx % dsCfg.getWalSegments() > formatted)
+                    while (curAbsWalIdx % dsCfg.getWalSegments() > formatted && cleanException == null)
                         wait();
 
+                    if (cleanException != null)
+                        throw cleanException;
+
                     return curAbsWalIdx;
                 }
             }
@@ -1664,7 +1683,7 @@ private boolean checkStop() {
          * Background creation of all segments except first. First segment was created in main thread by
          * {@link FsyncModeFileWriteAheadLogManager#checkOrPrepareFiles()}
          */
-        private void allocateRemainingFiles() throws IgniteCheckedException {
+        private void allocateRemainingFiles() throws StorageException {
             final FileArchiver archiver = this;
 
             checkFiles(1,
@@ -2029,23 +2048,23 @@ private void shutdown() {
      * @param startWith Start with.
      * @param create Flag create file.
      * @param p Predicate Exit condition.
-     * @throws IgniteCheckedException if validation or create file fail.
+     * @throws StorageException if validation or create file fail.
      */
     private void checkFiles(
         int startWith,
         boolean create,
         @Nullable IgnitePredicate<Integer> p,
         @Nullable IgniteInClosure<Integer> completionCallback
-    ) throws IgniteCheckedException {
+    ) throws StorageException {
         for (int i = startWith; i < dsCfg.getWalSegments() && (p == null || (p != null && p.apply(i))); i++) {
             File checkFile = new File(walWorkDir, FileDescriptor.fileName(i));
 
             if (checkFile.exists()) {
                 if (checkFile.isDirectory())
-                    throw new IgniteCheckedException("Failed to initialize WAL log segment (a directory with " +
+                    throw new StorageException("Failed to initialize WAL log segment (a directory with " +
                         "the same name already exists): " + checkFile.getAbsolutePath());
                 else if (checkFile.length() != dsCfg.getWalSegmentSize() && mode == WALMode.FSYNC)
-                    throw new IgniteCheckedException("Failed to initialize WAL log segment " +
+                    throw new StorageException("Failed to initialize WAL log segment " +
                         "(WAL segment size change is not supported):" + checkFile.getAbsolutePath());
             }
             else if (create)
@@ -2410,7 +2429,7 @@ private FileWriteHandle(
          *
          * @throws IOException If fail to write serializer version.
          */
-        public void writeSerializerVersion() throws IOException {
+        private void writeSerializerVersion() throws IOException {
             try {
                 assert fileIO.position() == 0 : "Serializer version can be written only at the begin of file " +
                     fileIO.position();
@@ -2448,9 +2467,8 @@ private boolean stopped(WALRecord record) {
          * @param rec Record to be added to record chain as new {@link #head}
          * @return Pointer or null if roll over to next segment is required or already started by other thread.
          * @throws StorageException If failed.
-         * @throws IgniteCheckedException If failed.
          */
-        @Nullable private WALPointer addRecord(WALRecord rec) throws StorageException, IgniteCheckedException {
+        @Nullable private WALPointer addRecord(WALRecord rec) throws StorageException {
             assert rec.size() > 0 || rec.getClass() == FakeRecord.class;
 
             boolean flushed = false;
@@ -2503,9 +2521,9 @@ private long nextPosition(WALRecord rec) {
          * Flush or wait for concurrent flush completion.
          *
          * @param ptr Pointer.
-         * @throws IgniteCheckedException If failed.
+         * @throws StorageException If failed.
          */
-        private void flushOrWait(FileWALPointer ptr, boolean stop) throws IgniteCheckedException {
+        private void flushOrWait(FileWALPointer ptr, boolean stop) throws StorageException {
             long expWritten;
 
             if (ptr != null) {
@@ -2549,10 +2567,9 @@ else if (stop) {
         /**
          * @param ptr Pointer.
          * @return {@code true} If the flush really happened.
-         * @throws IgniteCheckedException If failed.
          * @throws StorageException If failed.
          */
-        private boolean flush(FileWALPointer ptr, boolean stop) throws IgniteCheckedException, StorageException {
+        private boolean flush(FileWALPointer ptr, boolean stop) throws StorageException {
             if (ptr == null) { // Unconditional flush.
                 for (; ; ) {
                     WALRecord expHead = head.get();
@@ -2594,10 +2611,9 @@ private long chainBeginPosition(WALRecord h) {
 
         /**
          * @param expHead Expected head of chain. If head was changed, flush is not performed in this thread
-         * @throws IgniteCheckedException If failed.
          * @throws StorageException If failed.
          */
-        private boolean flush(WALRecord expHead, boolean stop) throws StorageException, IgniteCheckedException {
+        private boolean flush(WALRecord expHead, boolean stop) throws StorageException {
             if (expHead.previous() == null) {
                 FakeRecord frHead = (FakeRecord)expHead;
 
@@ -2643,7 +2659,8 @@ private boolean flush(WALRecord expHead, boolean stop) throws StorageException,
                 return true;
             }
             catch (Throwable e) {
-                StorageException se = new StorageException("Unable to write", new IOException(e));
+                StorageException se = e instanceof StorageException ? (StorageException) e :
+                    new StorageException("Unable to write", new IOException(e));
 
                 cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, se));
 
@@ -2725,8 +2742,9 @@ private FileWALPointer position() {
         /**
          * @param ptr Pointer to sync.
          * @throws StorageException If failed.
+         * @throws IgniteInterruptedCheckedException If interrupted.
          */
-        private void fsync(FileWALPointer ptr, boolean stop) throws StorageException, IgniteCheckedException {
+        private void fsync(FileWALPointer ptr, boolean stop) throws StorageException, IgniteInterruptedCheckedException {
             lock.lock();
 
             try {
@@ -2780,10 +2798,9 @@ private void fsync(FileWALPointer ptr, boolean stop) throws StorageException, Ig
 
         /**
          * @return {@code true} If this thread actually closed the segment.
-         * @throws IgniteCheckedException If failed.
          * @throws StorageException If failed.
          */
-        private boolean close(boolean rollOver) throws IgniteCheckedException, StorageException {
+        private boolean close(boolean rollOver) throws StorageException {
             if (stop.compareAndSet(false, true)) {
                 lock.lock();
 
@@ -2793,43 +2810,49 @@ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageEx
                     assert stopped() : "Segment is not closed after close flush: " + head.get();
 
                     try {
-                        RecordSerializer backwardSerializer = new RecordSerializerFactoryImpl(cctx)
-                            .createSerializer(serializerVersion);
+                        try {
+                            RecordSerializer backwardSerializer = new RecordSerializerFactoryImpl(cctx)
+                                .createSerializer(serializerVersion);
 
-                        SwitchSegmentRecord segmentRecord = new SwitchSegmentRecord();
+                            SwitchSegmentRecord segmentRecord = new SwitchSegmentRecord();
 
-                        int switchSegmentRecSize = backwardSerializer.size(segmentRecord);
+                            int switchSegmentRecSize = backwardSerializer.size(segmentRecord);
 
-                        if (rollOver && written < (maxSegmentSize - switchSegmentRecSize)) {
-                            final ByteBuffer buf = ByteBuffer.allocate(switchSegmentRecSize);
+                            if (rollOver && written < (maxSegmentSize - switchSegmentRecSize)) {
+                                final ByteBuffer buf = ByteBuffer.allocate(switchSegmentRecSize);
 
-                            segmentRecord.position(new FileWALPointer(idx, (int)written, switchSegmentRecSize));
-                            backwardSerializer.writeRecord(segmentRecord, buf);
+                                segmentRecord.position(new FileWALPointer(idx, (int)written, switchSegmentRecSize));
+                                backwardSerializer.writeRecord(segmentRecord, buf);
 
-                            buf.rewind();
+                                buf.rewind();
 
-                            int rem = buf.remaining();
+                                int rem = buf.remaining();
 
-                            while (rem > 0) {
-                                int written0 = fileIO.write(buf, written);
+                                while (rem > 0) {
+                                    int written0 = fileIO.write(buf, written);
 
-                                written += written0;
+                                    written += written0;
 
-                                rem -= written0;
+                                    rem -= written0;
+                                }
                             }
                         }
+                        catch (IgniteCheckedException e) {
+                            throw new IOException(e);
+                        }
+                        finally {
+                            assert mode == WALMode.FSYNC;
 
-                        // Do the final fsync.
-                        if (mode == WALMode.FSYNC) {
+                            // Do the final fsync.
                             fileIO.force();
 
                             lastFsyncPos = written;
-                        }
 
-                        fileIO.close();
+                            fileIO.close();
+                        }
                     }
                     catch (IOException e) {
-                        throw new IgniteCheckedException(e);
+                        throw new StorageException("Failed to close WAL write handle [idx=" + idx + "]", e);
                     }
 
                     if (log.isDebugEnabled())
@@ -2860,9 +2883,17 @@ private void signalNextAvailable() {
 
                     assert written == lastFsyncPos || mode != WALMode.FSYNC :
                     "fsync [written=" + written + ", lastFsync=" + lastFsyncPos + ']';
-                }
 
-                fileIO = null;
+                    fileIO = null;
+                }
+                else {
+                    try {
+                        fileIO.close();
+                    }
+                    catch (IOException e) {
+                        U.error(log, "Failed to close WAL file [idx=" + idx + ", fileIO=" + fileIO + "]", e);
+                    }
+                }
 
                 nextSegment.signalAll();
             }
@@ -2872,9 +2903,9 @@ private void signalNextAvailable() {
         }
 
         /**
-         * @throws IgniteCheckedException If failed.
+         *
          */
-        private void awaitNext() throws IgniteCheckedException {
+        private void awaitNext() {
             lock.lock();
 
             try {
@@ -2894,7 +2925,7 @@ private void awaitNext() throws IgniteCheckedException {
          * @throws IgniteCheckedException If failed.
          */
         @SuppressWarnings("TooBroadScope")
-        private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, IgniteCheckedException {
+        private void writeBuffer(long pos, ByteBuffer buf) throws StorageException {
             boolean interrupted = false;
 
             lock.lock();
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java b/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
index 1159683e6b54f..545c9ea1176d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
+++ b/modules/core/src/test/java/org/apache/ignite/failure/TestFailureHandler.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.failure;
 
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import org.apache.ignite.Ignite;
 
 /**
@@ -33,6 +34,13 @@ public class TestFailureHandler implements FailureHandler {
     /** Failure context. */
     volatile FailureContext failureCtx;
 
+    /**
+     * @param invalidate Invalidate.
+     */
+    public TestFailureHandler(boolean invalidate) {
+        this(invalidate, new CountDownLatch(1));
+    }
+
     /**
      * @param invalidate Invalidate.
      * @param latch Latch.
@@ -60,4 +68,15 @@ public TestFailureHandler(boolean invalidate, CountDownLatch latch) {
     public FailureContext failureContext() {
         return failureCtx;
     }
+
+    /**
+     * @param millis Millis.
+
+     * @return Failure context.
+     */
+    public FailureContext awaitFailure(long millis) throws InterruptedException {
+        latch.await(millis, TimeUnit.MILLISECONDS);
+
+        return failureCtx;
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
new file mode 100644
index 0000000000000..d30289682cbba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.OpenOption;
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.failure.TestFailureHandler;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+/**
+ *
+ */
+public class IgniteWalFormatFileFailoverTest extends GridCommonAbstractTest {
+    /** */
+    private static final String TEST_CACHE = "testCache";
+
+    /** */
+    private static final String formatFile = "formatFile";
+
+    /** Fail method name reference. */
+    private final AtomicReference<String> failMtdNameRef = new AtomicReference<>();
+
+    /** */
+    private boolean fsync;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCacheConfiguration(new CacheConfiguration(TEST_CACHE)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL));
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(2048L * 1024 * 1024)
+                .setPersistenceEnabled(true))
+            .setWalMode(fsync ? WALMode.FSYNC : WALMode.BACKGROUND)
+            .setWalBufferSize(1024 * 1024)
+            .setWalSegmentSize(512 * 1024)
+            .setFileIOFactory(new FailingFileIOFactory(failMtdNameRef));
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        cfg.setFailureHandler(new TestFailureHandler(false));
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNodeStartFailedFsync() throws Exception {
+        fsync = true;
+
+        failMtdNameRef.set(formatFile);
+
+        checkCause(GridTestUtils.assertThrows(log, () -> startGrid(0), IgniteCheckedException.class, null));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testFailureHandlerTriggeredFsync() throws Exception {
+        fsync = true;
+
+        failFormatFileOnClusterActivate();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testFailureHandlerTriggered() throws Exception {
+        fsync = false;
+
+        failFormatFileOnClusterActivate();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void failFormatFileOnClusterActivate() throws Exception {
+        failMtdNameRef.set(null);
+
+        startGrid(0);
+        startGrid(1);
+
+        if (!fsync) {
+            setFileIOFactory(grid(0).context().cache().context().wal());
+            setFileIOFactory(grid(1).context().cache().context().wal());
+        }
+
+        failMtdNameRef.set(formatFile);
+
+        grid(0).cluster().active(true);
+
+        checkCause(failureHandler(0).awaitFailure(2000).error());
+        checkCause(failureHandler(1).awaitFailure(2000).error());
+    }
+
+    /**
+     * @param mtdName Method name.
+     */
+    private static boolean isCalledFrom(String mtdName) {
+        return isCalledFrom(Thread.currentThread().getStackTrace(), mtdName);
+    }
+
+    /**
+     * @param stackTrace Stack trace.
+     * @param mtdName Method name.
+     */
+    private static boolean isCalledFrom(StackTraceElement[] stackTrace, String mtdName) {
+        return Arrays.stream(stackTrace).map(StackTraceElement::getMethodName).anyMatch(mtdName::equals);
+    }
+
+    /**
+     * @param gridIdx Grid index.
+     * @return Failure handler configured for grid with given index.
+     */
+    private TestFailureHandler failureHandler(int gridIdx) {
+        FailureHandler hnd = grid(gridIdx).configuration().getFailureHandler();
+
+        assertTrue(hnd instanceof TestFailureHandler);
+
+        return (TestFailureHandler)hnd;
+    }
+
+    /**
+     * @param t Throwable.
+     */
+    private void checkCause(Throwable t) {
+        StorageException e = X.cause(t, StorageException.class);
+
+        assertNotNull(e);
+        assertNotNull(e.getMessage());
+        assertTrue(e.getMessage().contains("Failed to format WAL segment file"));
+
+        IOException ioe = X.cause(e, IOException.class);
+
+        assertNotNull(ioe);
+        assertNotNull(ioe.getMessage());
+        assertTrue(ioe.getMessage().contains("No space left on device"));
+
+        assertTrue(isCalledFrom(ioe.getStackTrace(), formatFile));
+    }
+
+    /** */
+    private void setFileIOFactory(IgniteWriteAheadLogManager wal) {
+        if (wal instanceof FileWriteAheadLogManager)
+            ((FileWriteAheadLogManager)wal).setFileIOFactory(new FailingFileIOFactory(failMtdNameRef));
+        else
+            fail(wal.getClass().toString());
+    }
+
+    /**
+     * Create File I/O which fails if specific method call present in stack trace.
+     */
+    private static class FailingFileIOFactory implements FileIOFactory {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** Delegate factory. */
+        private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
+
+        /** Fail method name reference. */
+        private final AtomicReference<String> failMtdNameRef;
+
+        /**
+         * @param failMtdNameRef Fail method name reference.
+         */
+        FailingFileIOFactory(AtomicReference<String> failMtdNameRef) {
+            assertNotNull(failMtdNameRef);
+
+            this.failMtdNameRef = failMtdNameRef;
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            return create(file, CREATE, READ, WRITE);
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+            final FileIO delegate = delegateFactory.create(file, modes);
+
+            return new FileIODecorator(delegate) {
+                @Override public int write(byte[] buf, int off, int len) throws IOException {
+                    conditionalFail();
+
+                    return super.write(buf, off, len);
+                }
+
+                @Override public void clear() throws IOException {
+                    conditionalFail();
+
+                    super.clear();
+                }
+
+                private void conditionalFail() throws IOException {
+                    String failMtdName = failMtdNameRef.get();
+
+                    if (failMtdName != null && isCalledFrom(failMtdName))
+                        throw new IOException("No space left on device");
+                }
+            };
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index c5a381eb6e476..7732cee672799 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -49,6 +49,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncWithMmapBufferSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlySelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlyWithMmapBufferSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFormatFileFailoverTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorSwitchSegmentTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalSerializerVersionTest;
@@ -147,6 +148,8 @@ public static void addRealPageStoreTests(TestSuite suite) {
 
         suite.addTestSuite(IgniteWalFlushLogOnlyWithMmapBufferSelfTest.class);
 
+        suite.addTestSuite(IgniteWalFormatFileFailoverTest.class);
+
         // Test suite uses Standalone WAL iterator to verify PDS content.
         suite.addTestSuite(IgniteWalReaderTest.class);
 

From c26eeca8f9225cfc819733e75e65ac616403068f Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Fri, 22 Jun 2018 16:43:53 +0300
Subject: [PATCH 223/543] IGNITE-8831 Fix of MarshallerMappingFileStore:
 Incorrect locks on files. - Fixes #4224.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
---
 .../internal/MarshallerMappingFileStore.java  | 85 +++++++++++--------
 1 file changed, 50 insertions(+), 35 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerMappingFileStore.java b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerMappingFileStore.java
index 6fb1371f10ea3..a01981b88ed74 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/MarshallerMappingFileStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/MarshallerMappingFileStore.java
@@ -45,6 +45,9 @@
  * when a classname is requested but is not presented in local cache of {@link MarshallerContextImpl}.
  */
 final class MarshallerMappingFileStore {
+    /** File lock timeout in milliseconds. */
+    private static final int FILE_LOCK_TIMEOUT_MS = 5000;
+
     /** */
     private static final GridStripedLock fileLock = new GridStripedLock(32);
 
@@ -92,14 +95,12 @@ void writeMapping(byte platformId, int typeId, String typeName) {
             File file = new File(workDir, fileName);
 
             try (FileOutputStream out = new FileOutputStream(file)) {
-                FileLock fileLock = fileLock(out.getChannel(), false);
-
-                assert fileLock != null : fileName;
-
                 try (Writer writer = new OutputStreamWriter(out, StandardCharsets.UTF_8)) {
-                    writer.write(typeName);
+                    try (FileLock ignored = fileLock(out.getChannel(), false)) {
+                        writer.write(typeName);
 
-                    writer.flush();
+                        writer.flush();
+                    }
                 }
             }
             catch (IOException e) {
@@ -120,11 +121,10 @@ void writeMapping(byte platformId, int typeId, String typeName) {
     }
 
     /**
-     * @param platformId Platform id.
-     * @param typeId Type id.
+     * @param fileName File name.
      */
-    String readMapping(byte platformId, int typeId) throws IgniteCheckedException {
-        String fileName = getFileName(platformId, typeId);
+    private String readMapping(String fileName) throws IgniteCheckedException {
+        ThreadLocalRandom rnd = null;
 
         Lock lock = fileLock(fileName);
 
@@ -133,17 +133,30 @@ String readMapping(byte platformId, int typeId) throws IgniteCheckedException {
         try {
             File file = new File(workDir, fileName);
 
-            try (FileInputStream in = new FileInputStream(file)) {
-                FileLock fileLock = fileLock(in.getChannel(), true);
+            long time = 0;
+
+            while (true) {
+                try (FileInputStream in = new FileInputStream(file)) {
+                    try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
+                        try (FileLock ignored = fileLock(in.getChannel(), true)) {
+                            if (file.length() > 0)
+                                return reader.readLine();
+
+                            if (rnd == null)
+                                rnd = ThreadLocalRandom.current();
 
-                assert fileLock != null : fileName;
+                            if (time == 0)
+                                time = U.currentTimeMillis();
+                            else if ((U.currentTimeMillis() - time) >= FILE_LOCK_TIMEOUT_MS)
+                                return null;
 
-                try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
-                    return reader.readLine();
+                            U.sleep(rnd.nextLong(50));
+                        }
+                    }
+                }
+                catch (IOException ignored) {
+                    return null;
                 }
-            }
-            catch (IOException ignored) {
-                return null;
             }
         }
         finally {
@@ -151,6 +164,14 @@ String readMapping(byte platformId, int typeId) throws IgniteCheckedException {
         }
     }
 
+    /**
+     * @param platformId Platform id.
+     * @param typeId Type id.
+     */
+    String readMapping(byte platformId, int typeId) throws IgniteCheckedException {
+        return readMapping(getFileName(platformId, typeId));
+    }
+
     /**
      * Restores all mappings available in file system to marshaller context.
      * This method should be used only on node startup.
@@ -165,22 +186,16 @@ void restoreMappings(MarshallerContext marshCtx) throws IgniteCheckedException {
 
             int typeId = getTypeId(name);
 
-            try (FileInputStream in = new FileInputStream(file)) {
-                try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
-                    String clsName = reader.readLine();
+            String clsName = readMapping(name);
 
-                    if (clsName == null) {
-                        throw new IgniteCheckedException("Class name is null for [platformId=" + platformId +
-                            ", typeId=" + typeId + "], marshaller mappings storage is broken. " +
-                            "Clean up marshaller directory (<work_dir>/marshaller) and restart the node.");
-                    }
-
-                    marshCtx.registerClassNameLocally(platformId, typeId, clsName);
-                }
-            }
-            catch (IOException e) {
-                throw new IgniteCheckedException("Reading marshaller mapping from file " + name + " failed.", e);
+            if (clsName == null) {
+                throw new IgniteCheckedException("Class name is null for [platformId=" + platformId +
+                    ", typeId=" + typeId + "], marshaller mappings storage is broken. " +
+                    "Clean up marshaller directory (<work_dir>/marshaller) and restart the node. File name: " + name +
+                    ", FileSize: " + file.length());
             }
+
+            marshCtx.registerClassNameLocally(platformId, typeId, clsName);
         }
     }
 
@@ -276,10 +291,10 @@ private static FileLock fileLock(
         while (true) {
             FileLock fileLock = ch.tryLock(0L, Long.MAX_VALUE, shared);
 
-            if (fileLock == null)
-                U.sleep(rnd.nextLong(50));
-            else
+            if (fileLock != null)
                 return fileLock;
+
+            U.sleep(rnd.nextLong(50));
         }
     }
 }

From 8c18efb7439d298e07b23b603bd4fc4f716a0ab8 Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Mon, 25 Jun 2018 17:04:15 +0300
Subject: [PATCH 224/543] IGNITE-8594 Make error messages in validate_indexes
 command report more informative - Fixes #4197.

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit 594df60)
---
 .../visor/verify/ValidateIndexesClosure.java      | 15 +++++++++++++--
 .../util/GridCommandHandlerIndexingTest.java      |  3 +++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index e01dca2c92d8f..e3aebc3121919 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -505,12 +505,21 @@ private Map<String, ValidateIndexesPartitionResult> processIndex(GridCacheContex
         long current = 0;
         long processedNumber = 0;
 
+        KeyCacheObject previousKey = null;
+
         while (!enoughIssues) {
             KeyCacheObject h2key = null;
 
             try {
-                if (!cursor.next())
-                    break;
+                try {
+                    if (!cursor.next())
+                        break;
+                }
+                catch (IllegalStateException e) {
+                    throw new IgniteCheckedException("Key is present in SQL index, but is missing in corresponding " +
+                        "data page. Previous successfully read key: " +
+                        CacheObjectUtils.unwrapBinaryIfNeeded(ctx.cacheObjectContext(), previousKey, true, true), e);
+                }
 
                 GridH2Row h2Row = (GridH2Row)cursor.get();
 
@@ -541,6 +550,8 @@ else if (current++ % checkThrough > 0)
 
                 if (cacheDataStoreRow == null)
                     throw new IgniteCheckedException("Key is present in SQL index, but can't be found in CacheDataTree.");
+
+                previousKey = h2key;
             }
             catch (Throwable t) {
                 Object o = CacheObjectUtils.unwrapBinaryIfNeeded(
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
index ca9aa5372f552..1a274e5fcc7f0 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -110,6 +110,9 @@ public void testBrokenCacheDataTreeShouldFailValidation() throws Exception {
                 "checkThrough", "10"));
 
         assertTrue(testOut.toString().contains("validate_indexes has finished with errors"));
+
+        assertTrue(testOut.toString().contains(
+            "Key is present in SQL index, but is missing in corresponding data page."));
     }
 
     /**

From 6466262c473592e7781c0726b4023cda65d59446 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Tue, 26 Jun 2018 18:50:08 +0300
Subject: [PATCH 225/543] IGNITE-8768 Fixed JVM crash caused by an in-progress
 partition eviction during cache stop - Fixes #4227.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 56975c266e7019f307bb9da42333a6db4e47365e)
---
 .../processors/cache/CacheGroupContext.java   |   2 +
 .../distributed/dht/EvictionContext.java      |  28 ++
 .../dht/GridDhtLocalPartition.java            |  44 ++-
 .../dht/GridDhtPartitionsEvictor.java         | 326 +++++++++++++-----
 4 files changed, 301 insertions(+), 99 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/EvictionContext.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index d3c06b8475541..1ebe253fce3c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -730,6 +730,8 @@ void stopGroup() {
         IgniteCheckedException err =
             new IgniteCheckedException("Failed to wait for topology update, cache (or node) is stopping.");
 
+        evictor.stop();
+
         aff.cancelFutures(err);
 
         preldr.onKernalStop();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/EvictionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/EvictionContext.java
new file mode 100644
index 0000000000000..0964c3c3cc5c1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/EvictionContext.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+/**
+ * Additional context for partition eviction process.
+ */
+public interface EvictionContext {
+    /**
+     * @return {@code true} If eviction process should be stopped.
+     */
+    public boolean shouldStop();
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 8114ec7e55059..ae36dd4d9e115 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -854,14 +854,14 @@ public boolean isClearing() {
     }
 
     /**
-     * Tries to start partition clear process {@link GridDhtLocalPartition#clearAll()}).
+     * Tries to start partition clear process {@link GridDhtLocalPartition#clearAll(EvictionContext)}).
      * Only one thread is allowed to do such process concurrently.
      * At the end of clearing method completes {@code clearFuture}.
      *
      * @return {@code false} if clearing is not started due to existing reservations.
      * @throws NodeStoppingException If node is stopping.
      */
-    public boolean tryClear() throws NodeStoppingException {
+    public boolean tryClear(EvictionContext evictionCtx) throws NodeStoppingException {
         if (clearFuture.isDone())
             return true;
 
@@ -873,7 +873,7 @@ public boolean tryClear() throws NodeStoppingException {
         if (addEvicting()) {
             try {
                 // Attempt to evict partition entries from cache.
-                long clearedEntities = clearAll();
+                long clearedEntities = clearAll(evictionCtx);
 
                 if (log.isDebugEnabled())
                     log.debug("Partition is cleared [clearedEntities=" + clearedEntities + ", part=" + this + "]");
@@ -989,7 +989,7 @@ public long fullSize() {
      * @return Number of rows cleared from page memory.
      * @throws NodeStoppingException If node stopping.
      */
-    private long clearAll() throws NodeStoppingException {
+    private long clearAll(EvictionContext evictionCtx) throws NodeStoppingException {
         GridCacheVersion clearVer = ctx.versions().next();
 
         GridCacheObsoleteEntryExtras extras = new GridCacheObsoleteEntryExtras(clearVer);
@@ -1005,6 +1005,8 @@ private long clearAll() throws NodeStoppingException {
 
         long cleared = 0;
 
+        final long stopCheckFreq = 1000;
+
         if (!grp.allowFastEviction()) {
             CacheMapHolder hld = grp.sharedGroup() ? null : singleCacheEntryMap;
 
@@ -1056,34 +1058,38 @@ private long clearAll() throws NodeStoppingException {
 
                             cleared++;
                         }
+
+                        if (cleared % stopCheckFreq == 0 && evictionCtx.shouldStop())
+                            return cleared;
                     }
                     catch (GridDhtInvalidPartitionException e) {
                         assert isEmpty() && state() == EVICTED : "Invalid error [e=" + e + ", part=" + this + ']';
 
-                    break; // Partition is already concurrently cleared and evicted.
-                }
-                finally {
-                    ctx.database().checkpointReadUnlock();
+                        break; // Partition is already concurrently cleared and evicted.
+                    }
+                    finally {
+                        ctx.database().checkpointReadUnlock();
+                    }
                 }
-            }
 
-        if (forceTestCheckpointOnEviction) {
-                if (partWhereTestCheckpointEnforced == null && cleared >= fullSize()) {
-                    ctx.database().forceCheckpoint("test").finishFuture().get();
+                if (forceTestCheckpointOnEviction) {
+                    if (partWhereTestCheckpointEnforced == null && cleared >= fullSize()) {
+                        ctx.database().forceCheckpoint("test").finishFuture().get();
 
-                    log.warning("Forced checkpoint by test reasons for partition: " + this);
+                        log.warning("Forced checkpoint by test reasons for partition: " + this);
 
-                    partWhereTestCheckpointEnforced = id;
+                        partWhereTestCheckpointEnforced = id;
+                    }
                 }
             }
-        }catch (NodeStoppingException e) {
-            if (log.isDebugEnabled())
-                log.debug("Failed to get iterator for evicted partition: " + id);
+            catch (NodeStoppingException e) {
+                if (log.isDebugEnabled())
+                    log.debug("Failed to get iterator for evicted partition: " + id);
 
-                throw e;
+                    throw e;
             }
             catch (IgniteCheckedException e) {
-                U.error(log, "Failed to get iterator for evicted partition: " + id, e);
+                    U.error(log, "Failed to get iterator for evicted partition: " + id, e);
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java
index 2a289212ca30c..72063974b6d13 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java
@@ -16,21 +16,32 @@
  */
 package org.apache.ignite.internal.processors.cache.distributed.dht;
 
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.function.Function;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.util.typedef.internal.GPC;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
- * Class that serves asynchronous partition eviction process.
+ * Class that serves asynchronous part eviction process.
+ * Only one partition from group can be evicted at the moment.
  */
 public class GridDhtPartitionsEvictor {
-    /** Show eviction progress frequency in ms. */
-    private static final int SHOW_EVICTION_PROGRESS_FREQ_MS = 2 * 60 * 1000; // 2 Minutes.
+    /** Default eviction progress show frequency. */
+    private static final int DEFAULT_SHOW_EVICTION_PROGRESS_FREQ_MS = 2 * 60 * 1000; // 2 Minutes.
+
+    /** Eviction progress frequency property name. */
+    private static final String SHOW_EVICTION_PROGRESS_FREQ = "SHOW_EVICTION_PROGRESS_FREQ";
 
     /** */
     private final GridCacheSharedContext<?, ?> ctx;
@@ -41,11 +52,31 @@ public class GridDhtPartitionsEvictor {
     /** */
     private final IgniteLogger log;
 
+    /** Lock object. */
+    private final Object mux = new Object();
+
     /** Queue contains partitions scheduled for eviction. */
-    private final ConcurrentHashMap<Integer, GridDhtLocalPartition> evictionQueue = new ConcurrentHashMap<>();
+    private final DeduplicationQueue<Integer, GridDhtLocalPartition> evictionQueue = new DeduplicationQueue<>(GridDhtLocalPartition::id);
+
+    /**
+     * Flag indicates that eviction process is running at the moment.
+     * This is needed to schedule partition eviction if there are no currently running self-scheduling eviction tasks.
+     * Guarded by {@link #mux}.
+     */
+    private boolean evictionRunning;
 
-    /** Flag indicates that eviction process is running at the moment, false in other case. */
-    private final AtomicBoolean evictionRunning = new AtomicBoolean();
+    /** Flag indicates that eviction process has stopped. */
+    private volatile boolean stop;
+
+    /** Future for currently running partition eviction task. */
+    private volatile GridFutureAdapter<Boolean> evictionFut;
+
+    /** Eviction progress frequency in ms. */
+    private final long evictionProgressFreqMs = IgniteSystemProperties.getLong(SHOW_EVICTION_PROGRESS_FREQ,
+        DEFAULT_SHOW_EVICTION_PROGRESS_FREQ_MS);
+
+    /** Next time of show eviction progress. */
+    private long nextShowProgressTime;
 
     /**
      * Constructor.
@@ -67,78 +98,213 @@ public GridDhtPartitionsEvictor(CacheGroupContext grp) {
      * @param part Partition to evict.
      */
     public void evictPartitionAsync(GridDhtLocalPartition part) {
-        evictionQueue.putIfAbsent(part.id(), part);
-
-        if (evictionRunning.compareAndSet(false, true)) {
-            ctx.kernalContext().closure().callLocalSafe(new GPC<Boolean>() {
-                @Override public Boolean call() {
-                    boolean locked = true;
-
-                    long nextShowProgressTime = U.currentTimeMillis() + SHOW_EVICTION_PROGRESS_FREQ_MS;
-
-                    while (locked || !evictionQueue.isEmpty()) {
-                        if (!locked && !evictionRunning.compareAndSet(false, true))
-                            return false;
-
-                        try {
-                            for (GridDhtLocalPartition part : evictionQueue.values()) {
-                                // Show progress of currently evicting partitions.
-                                if (U.currentTimeMillis() >= nextShowProgressTime) {
-                                    if (log.isInfoEnabled())
-                                        log.info("Eviction in progress [grp=" + grp.cacheOrGroupName()
-                                                + ", remainingCnt=" + evictionQueue.size() + "]");
-
-                                    nextShowProgressTime = U.currentTimeMillis() + SHOW_EVICTION_PROGRESS_FREQ_MS;
-                                }
-
-                                try {
-                                    boolean success = part.tryClear();
-
-                                    if (success) {
-                                        evictionQueue.remove(part.id());
-
-                                        if (part.state() == GridDhtPartitionState.EVICTED && part.markForDestroy())
-                                            part.destroy();
-                                    }
-                                }
-                                catch (Throwable ex) {
-                                    if (ctx.kernalContext().isStopping()) {
-                                        LT.warn(log, ex, "Partition eviction failed (current node is stopping).",
-                                                false,
-                                                true);
-
-                                        evictionQueue.clear();
-
-                                        return true;
-                                    }
-                                    else
-                                        LT.error(log, ex, "Partition eviction failed, this can cause grid hang.");
-                                }
-                            }
-                        }
-                        finally {
-                            if (!evictionQueue.isEmpty()) {
-                                if (ctx.kernalContext().isStopping()) {
-                                    evictionQueue.clear();
-
-                                    locked = false;
-                                }
-                                else
-                                    locked = true;
-                            }
-                            else {
-                                boolean res = evictionRunning.compareAndSet(true, false);
-
-                                assert res;
-
-                                locked = false;
-                            }
-                        }
-                    }
-
-                    return true;
+        if (stop)
+            return;
+
+        boolean added = evictionQueue.offer(part);
+
+        if (!added)
+            return;
+
+        synchronized (mux) {
+            if (!evictionRunning) {
+                nextShowProgressTime = U.currentTimeMillis() + evictionProgressFreqMs;
+
+                scheduleNextPartitionEviction();
+            }
+        }
+    }
+
+    /**
+     * Stops eviction process.
+     * Method awaits last offered partition eviction.
+     */
+    public void stop() {
+        stop = true;
+
+        synchronized (mux) {
+            // Wait for last offered partition eviction completion.
+            IgniteInternalFuture<Boolean> evictionFut0 = evictionFut;
+
+            if (evictionFut0 != null) {
+                try {
+                    evictionFut0.get();
+                }
+                catch (IgniteCheckedException e) {
+                    if (log.isDebugEnabled())
+                        log.warning("Failed to await partition eviction during stopping", e);
+                }
+            }
+        }
+    }
+
+    /**
+     * Gets next partition from the queue and schedules it for eviction.
+     */
+    private void scheduleNextPartitionEviction() {
+        if (stop)
+            return;
+
+        synchronized (mux) {
+            GridDhtLocalPartition next = evictionQueue.poll();
+
+            if (next != null) {
+                showProgress();
+
+                evictionFut = new GridFutureAdapter<>();
+
+                ctx.kernalContext().closure().callLocalSafe(new PartitionEvictionTask(next, () -> stop), true);
+            }
+            else
+                evictionRunning = false;
+        }
+    }
+
+    /**
+     * Shows progress of eviction.
+     */
+    private void showProgress() {
+        if (U.currentTimeMillis() >= nextShowProgressTime) {
+            int size = evictionQueue.size() + 1; // Queue size plus current partition.
+
+            if (log.isInfoEnabled())
+                log.info("Eviction in progress [grp=" + grp.cacheOrGroupName()
+                    + ", remainingPartsCnt=" + size + "]");
+
+            nextShowProgressTime = U.currentTimeMillis() + evictionProgressFreqMs;
+        }
+    }
+
+    /**
+     * Task for self-scheduled partition eviction / clearing.
+     */
+    private class PartitionEvictionTask implements Callable<Boolean> {
+        /** Partition to evict. */
+        private final GridDhtLocalPartition part;
+
+        /** Eviction context. */
+        private final EvictionContext evictionCtx;
+
+        /**
+         * @param part Partition.
+         * @param evictionCtx Eviction context.
+         */
+        public PartitionEvictionTask(GridDhtLocalPartition part, EvictionContext evictionCtx) {
+            this.part = part;
+            this.evictionCtx = evictionCtx;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Boolean call() throws Exception {
+            if (stop) {
+                evictionFut.onDone();
+
+                return false;
+            }
+
+            try {
+                boolean success = part.tryClear(evictionCtx);
+
+                if (success) {
+                    if (part.state() == GridDhtPartitionState.EVICTED && part.markForDestroy())
+                        part.destroy();
                 }
-            }, /*system pool*/ true);
+                else // Re-offer partition if clear was unsuccessful due to partition reservation.
+                    evictionQueue.offer(part);
+
+                // Complete eviction future before schedule new to prevent deadlock with
+                // simultaneous eviction stopping and scheduling new eviction.
+                evictionFut.onDone();
+
+                scheduleNextPartitionEviction();
+
+                return true;
+            }
+            catch (Throwable ex) {
+                evictionFut.onDone(ex);
+
+                if (ctx.kernalContext().isStopping()) {
+                    LT.warn(log, ex, "Partition eviction failed (current node is stopping).",
+                        false,
+                        true);
+                }
+                else
+                    LT.error(log, ex, "Partition eviction failed, this can cause grid hang.");
+            }
+
+            return false;
+        }
+    }
+
+    /**
+     * Thread-safe blocking queue with items deduplication.
+     *
+     * @param <K> Key type of item used for deduplication.
+     * @param <V> Queue item type.
+     */
+    private static class DeduplicationQueue<K, V> {
+        /** Queue. */
+        private final Queue<V> queue;
+
+        /** Unique items set. */
+        private final Set<K> uniqueItems;
+
+        /** Key mapping function. */
+        private final Function<V, K> keyMappingFunction;
+
+        /**
+         * Constructor.
+         *
+         * @param keyExtractor Function to extract a key from a queue item.
+         *                     This key is used for deduplication if some item has offered twice.
+         */
+        public DeduplicationQueue(Function<V, K> keyExtractor) {
+            keyMappingFunction = keyExtractor;
+            queue = new LinkedBlockingQueue<>();
+            uniqueItems = new GridConcurrentHashSet<>();
+        }
+
+        /**
+         * Offers item to the queue.
+         *
+         * @param item Item.
+         * @return {@code true} if item has been successfully offered to the queue,
+         *         {@code false} if item was rejected because already exists in the queue.
+         */
+        public boolean offer(V item) {
+            K key = keyMappingFunction.apply(item);
+
+            if (uniqueItems.add(key)) {
+                queue.offer(item);
+
+                return true;
+            }
+
+            return false;
+        }
+
+        /**
+         * Polls next item from queue.
+         *
+         * @return Next item or {@code null} if queue is empty.
+         */
+        public V poll() {
+            V item = queue.poll();
+
+            if (item != null) {
+                K key = keyMappingFunction.apply(item);
+
+                uniqueItems.remove(key);
+            }
+
+            return item;
+        }
+
+        /**
+         * @return Size of queue.
+         */
+        public int size() {
+            return queue.size();
         }
     }
 }

From 87cb41668fe1b1a395a1eccab0feb6d7bc179ead Mon Sep 17 00:00:00 2001
From: Andrey Kuznetsov <stkuzma@gmail.com>
Date: Thu, 28 Jun 2018 18:38:22 +0300
Subject: [PATCH 226/543] IGNITE-8860 Returned IgniteDiscoveryThread to
 RingMessageWorker. - Fixes #4248.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry-picked from commit#219bc81b730ea3ba078b61f96c9dab354496c22b)
---
 .../apache/ignite/spi/discovery/tcp/ServerImpl.java   | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 170c1badf3d54..bb76895bd204c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -110,6 +110,7 @@
 import org.apache.ignite.spi.IgniteSpiThread;
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.ignite.spi.discovery.IgniteDiscoveryThread;
 import org.apache.ignite.spi.discovery.tcp.internal.DiscoveryDataPacket;
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNodesRing;
@@ -340,7 +341,7 @@ class ServerImpl extends TcpDiscoveryImpl {
 
         msgWorker = new RingMessageWorker(log);
 
-        new MessageWorkerThread(msgWorker, log).start();
+        new MessageWorkerDiscoveryThread(msgWorker, log).start();
 
         if (tcpSrvr == null)
             tcpSrvr = new TcpServer(log);
@@ -6821,6 +6822,14 @@ private MessageWorkerThreadWithCleanup(MessageWorker<T> worker, IgniteLogger log
         }
     }
 
+    /** */
+    private class MessageWorkerDiscoveryThread extends MessageWorkerThread implements IgniteDiscoveryThread {
+        /** {@inheritDoc} */
+        private MessageWorkerDiscoveryThread(GridWorker worker, IgniteLogger log) {
+            super(worker, log);
+        }
+    }
+
     /**
      * Slightly modified {@link IgniteSpiThread} intended to use with message workers.
      */

From 02bbb7da7e091534dd734da5e84ce89d5a199420 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Thu, 28 Jun 2018 17:39:38 +0300
Subject: [PATCH 227/543] IGNITE-8661 WALIterator should be stopped if it fails
 to deserialize a record - Fixes #4155.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit d6ab2ae)
---
 .../wal/IgniteWriteAheadLogManager.java       |    5 +
 .../GridCacheDatabaseSharedManager.java       |  221 +++-
 .../wal/AbstractWalRecordsIterator.java       |   27 +-
 .../wal/FileWriteAheadLogManager.java         |   39 +-
 .../FsyncModeFileWriteAheadLogManager.java    |    5 +
 .../SingleSegmentLogicalRecordsIterator.java  |    4 +-
 .../wal/reader/IgniteWalIteratorFactory.java  |  617 ++++++++--
 .../reader/StandaloneWalRecordsIterator.java  |  246 ++--
 .../RecordSerializerFactoryImpl.java          |   39 +-
 .../wal/serializer/RecordV1Serializer.java    |   27 +-
 .../wal/serializer/RecordV2Serializer.java    |    3 +
 ...iteWalIteratorExceptionDuringReadTest.java |  150 +++
 .../db/wal/reader/IgniteWalReaderTest.java    | 1050 +++++++++--------
 .../persistence/pagemem/NoOpWALManager.java   |    5 +
 .../testsuites/IgnitePdsTestSuite2.java       |    3 +
 .../development/utils/IgniteWalConverter.java |   10 +-
 16 files changed, 1589 insertions(+), 862 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorExceptionDuringReadTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index fd5d53b17e1d4..2b6358b6edaa7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -121,6 +121,11 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
      */
     public int walArchiveSegments();
 
+    /**
+     * @return Last archived segment index.
+     */
+    public long lastArchivedSegment();
+
     /**
      * Checks if WAL segment is under lock or reserved
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index e2fe57260e35e..a410010f9896a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -131,6 +131,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.processors.port.GridPortRecord;
 import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
@@ -166,6 +167,7 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
 import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
 import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD;
 import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID;
 
 /**
@@ -1929,39 +1931,26 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
             cctx.wal().allowCompressionUntil(status.startPtr);
 
         long start = U.currentTimeMillis();
-        int applied = 0;
-        WALPointer lastRead = null;
+
+        long lastArchivedSegment = cctx.wal().lastArchivedSegment();
+
+        RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, lastArchivedSegment, log);
 
         Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() :
             F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps);
 
+        int applied = 0;
+
         try (WALIterator it = cctx.wal().replay(status.endPtr)) {
             while (it.hasNextX()) {
-                IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
-
-                WALRecord rec = tup.get2();
+                WALRecord rec = restoreBinaryState.next(it);
 
-                lastRead = tup.get1();
+                if (rec == null)
+                    break;
 
                 switch (rec.type()) {
-                    case CHECKPOINT_RECORD:
-                        CheckpointRecord cpRec = (CheckpointRecord)rec;
-
-                        // We roll memory up until we find a checkpoint start record registered in the status.
-                        if (F.eq(cpRec.checkpointId(), status.cpStartId)) {
-                            log.info("Found last checkpoint marker [cpId=" + cpRec.checkpointId() +
-                                ", pos=" + tup.get1() + ']');
-
-                            apply = false;
-                        }
-                        else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
-                            U.warn(log, "Found unexpected checkpoint marker, skipping [cpId=" + cpRec.checkpointId() +
-                                ", expCpId=" + status.cpStartId + ", pos=" + tup.get1() + ']');
-
-                        break;
-
                     case PAGE_RECORD:
-                        if (apply) {
+                        if (restoreBinaryState.needApplyBinaryUpdate()) {
                             PageSnapshot pageRec = (PageSnapshot)rec;
 
                             // Here we do not require tag check because we may be applying memory changes after
@@ -2044,7 +2033,7 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
                         break;
 
                     default:
-                        if (apply && rec instanceof PageDeltaRecord) {
+                        if (restoreBinaryState.needApplyBinaryUpdate() && rec instanceof PageDeltaRecord) {
                             PageDeltaRecord r = (PageDeltaRecord)rec;
 
                             int grpId = r.groupId();
@@ -2085,11 +2074,13 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
         if (metastoreOnly)
             return null;
 
+        WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer();
+
         if (status.needRestoreMemory()) {
-            if (apply)
+            if (restoreBinaryState.needApplyBinaryUpdate())
                 throw new StorageException("Failed to restore memory state (checkpoint marker is present " +
                     "on disk, but checkpoint record is missed in WAL) " +
-                    "[cpStatus=" + status + ", lastRead=" + lastRead + "]");
+                    "[cpStatus=" + status + ", lastRead=" + lastReadPtr + "]");
 
             log.info("Finished applying memory changes [changesApplied=" + applied +
                 ", time=" + (U.currentTimeMillis() - start) + "ms]");
@@ -2100,7 +2091,7 @@ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
 
         cpHistory.initialize(retreiveHistory());
 
-        return lastRead == null ? null : lastRead.next();
+        return lastReadPtr == null ? null : lastReadPtr.next();
     }
 
     /**
@@ -2209,6 +2200,10 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
         if (!metastoreOnly)
             cctx.kernalContext().query().skipFieldLookup(true);
 
+        long lastArchivedSegment = cctx.wal().lastArchivedSegment();
+
+        RestoreLogicalState restoreLogicalState = new RestoreLogicalState(lastArchivedSegment, log);
+
         long start = U.currentTimeMillis();
         int applied = 0;
 
@@ -2219,9 +2214,10 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
             Map<T2<Integer, Integer>, T2<Integer, Long>> partStates = new HashMap<>();
 
             while (it.hasNextX()) {
-                IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
+                WALRecord rec = restoreLogicalState.next(it);
 
-                WALRecord rec = next.get2();
+                if (rec == null)
+                    break;
 
                 switch (rec.type()) {
                     case DATA_RECORD:
@@ -4364,4 +4360,171 @@ else if (key.startsWith(WAL_GLOBAL_KEY_PREFIX))
         else
             return null;
     }
+
+    /**
+     * Abstract class for create restore context.
+     */
+    public abstract static class RestoreStateContext {
+        /** */
+        protected final IgniteLogger log;
+
+        /** Last archived segment. */
+        protected final long lastArchivedSegment;
+
+        /** Last read record WAL pointer. */
+        protected FileWALPointer lastRead;
+
+        /**
+         * @param lastArchivedSegment Last archived segment index.
+         * @param log Ignite logger.
+         */
+        public RestoreStateContext(long lastArchivedSegment, IgniteLogger log) {
+            this.lastArchivedSegment = lastArchivedSegment;
+            this.log = log;
+        }
+
+        /**
+         * Advance iterator to the next record.
+         *
+         * @param it WAL iterator.
+         * @return WALRecord entry.
+         * @throws IgniteCheckedException If CRC check fail during binary recovery state or another exception occurring.
+         */
+        public WALRecord next(WALIterator it) throws IgniteCheckedException {
+            try {
+                IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+
+                WALRecord rec = tup.get2();
+
+                WALPointer ptr = tup.get1();
+
+                lastRead = (FileWALPointer)ptr;
+
+                rec.position(ptr);
+
+                return rec;
+            }
+            catch (IgniteCheckedException e) {
+                boolean throwsCRCError = throwsCRCError();
+
+                if (X.hasCause(e, IgniteDataIntegrityViolationException.class)) {
+                    if (throwsCRCError)
+                        throw e;
+                    else
+                        return null;
+                }
+
+                log.error("Catch error during restore state, throwsCRCError=" + throwsCRCError, e);
+
+                throw e;
+            }
+        }
+
+        /**
+         *
+         * @return Last read WAL record pointer.
+         */
+        public WALPointer lastReadRecordPointer() {
+            return lastRead;
+        }
+
+        /**
+         *
+         * @return Flag indicates need throws CRC exception or not.
+         */
+        public boolean throwsCRCError(){
+            FileWALPointer lastReadPtr = lastRead;
+
+            return lastReadPtr != null && lastReadPtr.index() <= lastArchivedSegment;
+        }
+    }
+
+    /**
+     * Restore memory context. Tracks the safety of binary recovery.
+     */
+    public static class RestoreBinaryState extends RestoreStateContext {
+        /** Checkpoint status. */
+        private final CheckpointStatus status;
+
+        /** The flag indicates need to apply the binary update or no needed. */
+        private boolean needApplyBinaryUpdates;
+
+        /**
+         * @param status Checkpoint status.
+         * @param lastArchivedSegment Last archived segment index.
+         * @param log Ignite logger.
+         */
+        public RestoreBinaryState(CheckpointStatus status, long lastArchivedSegment, IgniteLogger log) {
+            super(lastArchivedSegment, log);
+
+            this.status = status;
+            needApplyBinaryUpdates = status.needRestoreMemory();
+        }
+
+        /**
+         * Advance iterator to the next record.
+         *
+         * @param it WAL iterator.
+         * @return WALRecord entry.
+         * @throws IgniteCheckedException If CRC check fail during binary recovery state or another exception occurring.
+         */
+        @Override public WALRecord next(WALIterator it) throws IgniteCheckedException {
+            WALRecord rec = super.next(it);
+
+            if (rec == null)
+                return null;
+
+            if (rec.type() == CHECKPOINT_RECORD) {
+                CheckpointRecord cpRec = (CheckpointRecord)rec;
+
+                // We roll memory up until we find a checkpoint start record registered in the status.
+                if (F.eq(cpRec.checkpointId(), status.cpStartId)) {
+                    log.info("Found last checkpoint marker [cpId=" + cpRec.checkpointId() +
+                        ", pos=" + rec.position() + ']');
+
+                    needApplyBinaryUpdates = false;
+                }
+                else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
+                    U.warn(log, "Found unexpected checkpoint marker, skipping [cpId=" + cpRec.checkpointId() +
+                        ", expCpId=" + status.cpStartId + ", pos=" + rec.position() + ']');
+            }
+
+            return rec;
+        }
+
+        /**
+         *
+         * @return Flag indicates need apply binary record or not.
+         */
+        public boolean needApplyBinaryUpdate() {
+            return needApplyBinaryUpdates;
+        }
+
+        /**
+         *
+         * @return Flag indicates need throws CRC exception or not.
+         */
+        @Override public boolean throwsCRCError() {
+            log.info("Throws CRC error check, needApplyBinaryUpdates=" + needApplyBinaryUpdates +
+                ", lastArchivedSegment=" + lastArchivedSegment + ", lastRead=" + lastRead);
+
+            if (needApplyBinaryUpdates)
+                return true;
+
+            return super.throwsCRCError();
+        }
+    }
+
+    /**
+     * Restore logical state context. Tracks the safety of logical recovery.
+     */
+    public static class RestoreLogicalState extends RestoreStateContext {
+        /**
+         * @param lastArchivedSegment Last archived segment index.
+         * @param log Ignite logger.
+         */
+        public RestoreLogicalState(long lastArchivedSegment, IgniteLogger log) {
+            super(lastArchivedSegment, log);
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index e442386c03e87..01b093399fbc1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -91,21 +91,21 @@ public abstract class AbstractWalRecordsIterator
      * @param sharedCtx Shared context.
      * @param serializerFactory Serializer of current version to read headers.
      * @param ioFactory ioFactory for file IO access.
-     * @param bufSize buffer for reading records size.
+     * @param initialReadBufferSize buffer for reading records size.
      */
     protected AbstractWalRecordsIterator(
         @NotNull final IgniteLogger log,
         @NotNull final GridCacheSharedContext sharedCtx,
         @NotNull final RecordSerializerFactory serializerFactory,
         @NotNull final FileIOFactory ioFactory,
-        final int bufSize
+        final int initialReadBufferSize
     ) {
         this.log = log;
         this.sharedCtx = sharedCtx;
         this.serializerFactory = serializerFactory;
         this.ioFactory = ioFactory;
 
-        buf = new ByteBufferExpander(bufSize, ByteOrder.nativeOrder());
+        buf = new ByteBufferExpander(initialReadBufferSize, ByteOrder.nativeOrder());
     }
 
     /** {@inheritDoc} */
@@ -225,8 +225,12 @@ private IgniteBiTuple<WALPointer, WALRecord> advanceRecord(
             if (e instanceof WalSegmentTailReachedException)
                 throw (WalSegmentTailReachedException)e;
 
-            if (!(e instanceof SegmentEofException))
-                handleRecordException(e, actualFilePtr);
+            if (!(e instanceof SegmentEofException) && !(e instanceof EOFException)) {
+                IgniteCheckedException e0 = handleRecordException(e, actualFilePtr);
+
+                if (e0 != null)
+                    throw e0;
+            }
 
             return null;
         }
@@ -248,12 +252,15 @@ private IgniteBiTuple<WALPointer, WALRecord> advanceRecord(
      *
      * @param e problem from records reading
      * @param ptr file pointer was accessed
+     *
+     * @return {@code null} if the error was handled and we can go ahead,
+     *  {@code IgniteCheckedException} if the error was not handled, and we should stop the iteration.
      */
-    protected void handleRecordException(
-        @NotNull final Exception e,
-        @Nullable final FileWALPointer ptr) {
+    protected IgniteCheckedException handleRecordException(@NotNull final Exception e, @Nullable final FileWALPointer ptr) {
         if (log.isInfoEnabled())
             log.info("Stopping WAL iteration due to an exception: " + e.getMessage() + ", ptr=" + ptr);
+
+        return new IgniteCheckedException(e);
     }
 
     /**
@@ -265,8 +272,8 @@ protected void handleRecordException(
      */
     protected AbstractReadFileHandle initReadHandle(
         @NotNull final AbstractFileDescriptor desc,
-        @Nullable final FileWALPointer start)
-        throws IgniteCheckedException, FileNotFoundException {
+        @Nullable final FileWALPointer start
+    ) throws IgniteCheckedException, FileNotFoundException {
         try {
             FileIO fileIO = desc.isCompressed() ? new UnzipFileIO(desc.file()) : ioFactory.create(desc.file());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 3f406629f9e49..4a696e47a7abc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -94,6 +94,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator.AbstractFileDescriptor;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
@@ -127,6 +128,7 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
 import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
+import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED;
 import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
 import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.SWITCH_SEGMENT_RECORD;
@@ -171,7 +173,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     private static final byte[] FILL_BUF = new byte[1024 * 1024];
 
     /** Pattern for segment file names */
-    private static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal");
+    public static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal");
 
     /** */
     private static final Pattern WAL_TEMP_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal\\.tmp");
@@ -191,7 +193,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     };
 
     /** */
-    private static final Pattern WAL_SEGMENT_FILE_COMPACTED_PATTERN = Pattern.compile("\\d{16}\\.wal\\.zip");
+    public static final Pattern WAL_SEGMENT_FILE_COMPACTED_PATTERN = Pattern.compile("\\d{16}\\.wal\\.zip");
 
     /** WAL segment file filter, see {@link #WAL_NAME_PATTERN} */
     public static final FileFilter WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER = new FileFilter() {
@@ -951,6 +953,11 @@ private boolean segmentReservedOrLocked(long absIdx) {
         return res >= 0 ? res : 0;
     }
 
+    /** {@inheritDoc} */
+    @Override public long lastArchivedSegment() {
+        return archivedMonitor.lastArchivedAbsoluteIndex();
+    }
+
     /** {@inheritDoc} */
     @Override public boolean reserved(WALPointer ptr) {
         FileWALPointer fPtr = (FileWALPointer)ptr;
@@ -1656,9 +1663,12 @@ private synchronized boolean locked(long absIdx) {
                         notifyAll();
                     }
 
-                    if (evt.isRecordable(EventType.EVT_WAL_SEGMENT_ARCHIVED)) {
-                        evt.record(new WalSegmentArchivedEvent(cctx.discovery().localNode(),
-                            res.getAbsIdx(), res.getDstArchiveFile()));
+                    if (evt.isRecordable(EVT_WAL_SEGMENT_ARCHIVED)) {
+                        evt.record(new WalSegmentArchivedEvent(
+                                cctx.discovery().localNode(),
+                                res.getAbsIdx(),
+                                res.getDstArchiveFile())
+                        );
                     }
                 }
             }
@@ -1912,7 +1922,7 @@ private void init() {
             FileDescriptor[] alreadyCompressed = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_COMPACTED_FILTER));
 
             if (alreadyCompressed.length > 0)
-                lastCompressedIdx = alreadyCompressed[alreadyCompressed.length - 1].getIdx();
+                lastCompressedIdx = alreadyCompressed[alreadyCompressed.length - 1].idx();
         }
 
         /**
@@ -2320,7 +2330,8 @@ else if (create)
     /**
      * WAL file descriptor.
      */
-    public static class FileDescriptor implements Comparable<FileDescriptor>, AbstractWalRecordsIterator.AbstractFileDescriptor {
+    public static class FileDescriptor implements
+        Comparable<FileDescriptor>, AbstractFileDescriptor {
         /** */
         protected final File file;
 
@@ -2390,20 +2401,6 @@ public static String fileName(long segment) {
             return (int)(idx ^ (idx >>> 32));
         }
 
-        /**
-         * @return Absolute WAL segment file index
-         */
-        public long getIdx() {
-            return idx;
-        }
-
-        /**
-         * @return absolute pathname string of this file descriptor pathname.
-         */
-        public String getAbsolutePath() {
-            return file.getAbsolutePath();
-        }
-
         /**
          * @return True if segment is ZIP compressed.
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 6e59ad3614dec..5db21d25ae4a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -845,6 +845,11 @@ private boolean hasIndex(long absIdx) {
         return res >= 0 ? res : 0;
     }
 
+    /** {@inheritDoc} */
+    @Override public long lastArchivedSegment() {
+        return archiver.lastArchivedAbsoluteIndex();
+    }
+
     /** {@inheritDoc} */
     @Override public boolean reserved(WALPointer ptr) {
         FileWALPointer fPtr = (FileWALPointer)ptr;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
index 36e5b0e21b3a9..f688bb4b5f783 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
@@ -86,9 +86,7 @@ public class SingleSegmentLogicalRecordsIterator extends AbstractWalRecordsItera
     private static RecordSerializerFactory initLogicalRecordsSerializerFactory(GridCacheSharedContext sharedCtx)
         throws IgniteCheckedException {
 
-        return new RecordSerializerFactoryImpl(sharedCtx)
-            .recordDeserializeFilter(new LogicalRecordsFilter())
-            .marshalledMode(true);
+        return new RecordSerializerFactoryImpl(sharedCtx, new LogicalRecordsFilter()).marshalledMode(true);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index 0c7bbb323656f..2bfc22d79ac54 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -17,17 +17,49 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.wal.reader;
 
+import java.io.DataInput;
 import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.ByteOrder;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.TreeSet;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiPredicate;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import static java.lang.System.arraycopy;
+import static java.nio.file.Files.walkFileTree;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.WAL_NAME_PATTERN;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.WAL_SEGMENT_FILE_COMPACTED_PATTERN;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.HEADER_RECORD_SIZE;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readPosition;
+
 /**
  * Factory for creating iterator over WAL files
  */
@@ -35,189 +67,522 @@ public class IgniteWalIteratorFactory {
     /** Logger. */
     private final IgniteLogger log;
 
-    /** Page size, in standalone iterator mode this value can't be taken from memory configuration. */
-    private final int pageSize;
-
-    /**
-     * Folder specifying location of metadata File Store. {@code null} means no specific folder is configured. <br>
-     * This folder should be specified for converting data entries into BinaryObjects
-     */
-    @Nullable private File binaryMetadataFileStoreDir;
-
     /**
-     * Folder specifying location of marshaller mapping file store. {@code null} means no specific folder is configured.
-     * <br> This folder should be specified for converting data entries into BinaryObjects. Providing {@code null} will
-     * disable unmarshall for non primitive objects, BinaryObjects will be provided
+     * Creates WAL files iterator factory.
+     * WAL iterator supports automatic converting from CacheObjects and KeyCacheObject into BinaryObjects
      */
-    @Nullable private File marshallerMappingFileStoreDir;
-
-    /** Keep binary. This flag disables converting of non primitive types (BinaryObjects) */
-    private boolean keepBinary;
-
-    /** Factory to provide I/O interfaces for read/write operations with files */
-    private FileIOFactory ioFactory;
-
-    /** Wal records iterator buffer size */
-    private int bufSize = StandaloneWalRecordsIterator.DFLT_BUF_SIZE;
+    public IgniteWalIteratorFactory() {
+        this(ConsoleLogger.INSTANCE);
+    }
 
     /**
      * Creates WAL files iterator factory.
      * WAL iterator supports automatic converting from CacheObjects and KeyCacheObject into BinaryObjects
      *
      * @param log Logger.
-     * @param pageSize Page size which was used in Ignite Persistent Data store to read WAL from, size is validated
-     * according its boundaries.
-     * @param binaryMetadataFileStoreDir folder specifying location of metadata File Store. Should include "binary_meta"
-     * subfolder and consistent ID subfolder. Note Consistent ID should be already masked and should not contain special
-     * symbols. Providing {@code null} means no specific folder is configured. <br>
-     * @param marshallerMappingFileStoreDir Folder specifying location of marshaller mapping file store. Should include
-     * "marshaller" subfolder. Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects
-     * will be provided
-     * @param keepBinary {@code true} disables complex object unmarshall into source classes
      */
-    public IgniteWalIteratorFactory(
-        @NotNull final IgniteLogger log,
-        final int pageSize,
-        @Nullable final File binaryMetadataFileStoreDir,
-        @Nullable final File marshallerMappingFileStoreDir,
-        final boolean keepBinary) {
+    public IgniteWalIteratorFactory(@NotNull final IgniteLogger log) {
         this.log = log;
-        this.pageSize = pageSize;
-        this.binaryMetadataFileStoreDir = binaryMetadataFileStoreDir;
-        this.marshallerMappingFileStoreDir = marshallerMappingFileStoreDir;
-        this.keepBinary = keepBinary;
-        this.ioFactory = new DataStorageConfiguration().getFileIOFactory();
-        new DataStorageConfiguration().setPageSize(pageSize); // just for validate
     }
 
     /**
-     * Creates WAL files iterator factory.
-     * WAL iterator supports automatic converting from CacheObjects and KeyCacheObject into BinaryObjects
+     * Creates iterator for file by file scan mode.
+     * This method may be used for work folder, file indexes are scanned from the file context.
+     * In this mode only provided WAL segments will be scanned. New WAL files created during iteration will be ignored.
      *
-     * @param log Logger.
-     * @param pageSize Page size which was used in Ignite Persistent Data store to read WAL from, size is validated
-     * according its boundaries.
-     * @param binaryMetadataFileStoreDir folder specifying location of metadata File Store. Should include "binary_meta"
-     * subfolder and consistent ID subfolder. Note Consistent ID should be already masked and should not contain special
-     * symbols. Providing {@code null} means no specific folder is configured. <br>
-     * @param marshallerMappingFileStoreDir Folder specifying location of marshaller mapping file store. Should include
-     * "marshaller" subfolder. Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects
-     * will be provided
+     * @param filesOrDirs files to scan. A file can be the path to '.wal' file, or directory with '.wal' files.
+     * Order is not important, but it is significant to provide all segments without omissions.
+     * Path should not contain special symbols. Special symbols should be already masked.
+     * @return closable WAL records iterator, should be closed when non needed.
+     * @throws IgniteCheckedException if failed to read files
+     * @throws IllegalArgumentException If parameter validation failed.
      */
-    public IgniteWalIteratorFactory(
-        @NotNull final IgniteLogger log,
-        final int pageSize,
-        @Nullable final File binaryMetadataFileStoreDir,
-        @Nullable final File marshallerMappingFileStoreDir) {
-        this(log, pageSize, binaryMetadataFileStoreDir, marshallerMappingFileStoreDir, false);
+    public WALIterator iterator(
+        @NotNull File... filesOrDirs
+    ) throws IgniteCheckedException, IllegalArgumentException {
+        return iterator(new IteratorParametersBuilder().filesOrDirs(filesOrDirs));
     }
 
     /**
-     * Creates WAL files iterator factory. This constructor does not allow WAL iterators access to data entries key and value.
+     * Creates iterator for file by file scan mode.
+     * This method may be used for work folder, file indexes are scanned from the file context.
+     * In this mode only provided WAL segments will be scanned. New WAL files created during iteration will be ignored.
      *
-     * @param log Logger.
-     * @param ioFactory Custom factory for non-standard file API to be used in WAL reading.
-     * @param pageSize Page size which was used in Ignite Persistent Data store to read WAL from, size is validated
-     * according its boundaries.
+     * @param filesOrDirs paths to scan. A path can be direct to '.wal' file, or directory with '.wal' files.
+     * Order is not important, but it is significant to provide all segments without omissions.
+     * Path should not contain special symbols. Special symbols should be already masked.
+     * @return closable WAL records iterator, should be closed when non needed.
+     * @throws IgniteCheckedException If failed to read files.
+     * @throws IllegalArgumentException If parameter validation failed.
      */
-    public IgniteWalIteratorFactory(@NotNull final IgniteLogger log, @NotNull final FileIOFactory ioFactory, int pageSize) {
-        this.log = log;
-        this.pageSize = pageSize;
-        this.ioFactory = ioFactory;
-        new DataStorageConfiguration().setPageSize(pageSize); // just for validate
+    public WALIterator iterator(
+        @NotNull String... filesOrDirs
+    ) throws IgniteCheckedException, IllegalArgumentException {
+        return iterator(new IteratorParametersBuilder().filesOrDirs(filesOrDirs));
     }
 
     /**
-     * Creates WAL files iterator factory. This constructor does not allow WAL iterators access to data entries key and
-     * value.
-     *
-     * @param log Logger.
-     * @param pageSize Page size which was used in Ignite Persistent Data store to read WAL from, size is validated
-     * according its boundaries.
+     * @param iteratorParametersBuilder Iterator parameters builder.
+     * @return closable WAL records iterator, should be closed when non needed
      */
-    public IgniteWalIteratorFactory(@NotNull final IgniteLogger log, int pageSize) {
-        this(log, new DataStorageConfiguration().getFileIOFactory(), pageSize);
+    public WALIterator iterator(
+        @NotNull IteratorParametersBuilder iteratorParametersBuilder
+    ) throws IgniteCheckedException, IllegalArgumentException {
+        iteratorParametersBuilder.validate();
+
+        return new StandaloneWalRecordsIterator(log,
+            prepareSharedCtx(iteratorParametersBuilder),
+            iteratorParametersBuilder.ioFactory,
+            resolveWalFiles(
+                iteratorParametersBuilder.filesOrDirs,
+                iteratorParametersBuilder
+            ),
+            iteratorParametersBuilder.filter,
+            iteratorParametersBuilder.keepBinary,
+            iteratorParametersBuilder.bufferSize
+        );
     }
 
     /**
-     * Creates iterator for (archive) directory scan mode.
-     * Note in this mode total scanned files at end of iteration may be wider that initial files in directory.
-     * This mode does not support work directory scan because work directory contains unpredictable number in file name.
-     * Such file may broke iteration.
+     * Find WAL gaps, for example:
+     * 0 1 2 3 4 7 8 10 - WAL segment files in directory, this method will return
+     * List with two tuples [(4,7),(8,10)].
      *
-     * @param walDirWithConsistentId directory with WAL files. Should already contain node consistent ID as subfolder.
-     * Note: 'Consistent ID'-based subfolder name (if any) should not contain special symbols.
-     * @return closable WAL records iterator, should be closed when non needed
-     * @throws IgniteCheckedException if failed to read folder
+     * @param filesOrDirs Paths to files or directories for scan.
+     * @return List of tuples, low and high index segments with gap.
      */
-    public WALIterator iteratorArchiveDirectory(
-        @NotNull final File walDirWithConsistentId) throws IgniteCheckedException {
-        return new StandaloneWalRecordsIterator(
-            walDirWithConsistentId, log, prepareSharedCtx(), ioFactory, keepBinary, bufSize);
+    public List<T2<Long, Long>> hasGaps(
+        @NotNull String... filesOrDirs
+    ) throws IllegalArgumentException {
+        return hasGaps(new IteratorParametersBuilder().filesOrDirs(filesOrDirs));
     }
 
     /**
-     * Creates iterator for file by file scan mode.
-     * This method may be used only for archive folder (not for work).
-     * In this mode only provided WAL segments will be scanned. New WAL files created during iteration will be ignored
+     * Find WAL gaps, for example:
+     * 0 1 2 3 4 7 8 10 - WAL segment files in directory, this method will return
+     * List with two tuples [(4,7),(8,10)].
      *
-     * @param files files to scan. Order is not important, but it is significant to provide all segments without omissions.
-     * Parameter should contain direct file links to '.wal' files from archive directory.
-     * 'Consistent ID'-based subfolder name (if any) should not contain special symbols.
-     * Special symbols should be already masked.
-     *
-     * @return closable WAL records iterator, should be closed when non needed
-     * @throws IgniteCheckedException if failed to read files
+     * @param filesOrDirs Files or directories to scan.
+     * @return List of tuples, low and high index segments with gap.
      */
-    public WALIterator iteratorArchiveFiles(@NotNull final File... files) throws IgniteCheckedException {
-        return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, false, keepBinary, bufSize, files);
+    public List<T2<Long, Long>> hasGaps(
+        @NotNull File... filesOrDirs
+    ) throws IllegalArgumentException {
+        return hasGaps(new IteratorParametersBuilder().filesOrDirs(filesOrDirs));
     }
 
     /**
-     * Creates iterator for file by file scan mode.
-     * This method may be used for work folder, file indexes are scanned from the file context.
-     * In this mode only provided WAL segments will be scanned. New WAL files created during iteration will be ignored.
-     *
-     * @param files files to scan. Order is not important, but it is significant to provide all segments without omissions.
-     * Parameter should contain direct file links to '.wal' files from work directory.
-     * 'Consistent ID'-based subfolder name (if any) should not contain special symbols.
-     * Special symbols should be already masked.
+     * @param iteratorParametersBuilder Iterator parameters builder.
+     * @return List of tuples, low and high index segments with gap.
+     */
+    public List<T2<Long, Long>> hasGaps(
+        @NotNull IteratorParametersBuilder iteratorParametersBuilder
+    ) throws IllegalArgumentException {
+        iteratorParametersBuilder.validate();
+
+        List<T2<Long, Long>> gaps = new ArrayList<>();
+
+        List<FileDescriptor> descriptors = resolveWalFiles(
+            iteratorParametersBuilder.filesOrDirs,
+            iteratorParametersBuilder
+        );
+
+        Iterator<FileDescriptor> it = descriptors.iterator();
+
+        FileDescriptor prevFd = null;
+
+        while (it.hasNext()) {
+            FileDescriptor nextFd = it.next();
+
+            if (prevFd == null) {
+                prevFd = nextFd;
+
+                continue;
+            }
+
+            if (prevFd.idx() + 1 != nextFd.idx())
+                gaps.add(new T2<>(prevFd.idx(), nextFd.idx()));
+
+            prevFd = nextFd;
+        }
+
+        return gaps;
+    }
+
+    /**
+     * This methods checks all provided files to be correct WAL segment.
+     * Header record and its position is checked. WAL position is used to determine real index.
+     * File index from file name is ignored.
      *
-     * @return closable WAL records iterator, should be closed when non needed
-     * @throws IgniteCheckedException if failed to read files
+     * @param iteratorParametersBuilder IteratorParametersBuilder.
+     * @return list of file descriptors with checked header records, having correct file index is set
+     */
+    private List<FileDescriptor> resolveWalFiles(
+        File[] filesOrDirs,
+        IteratorParametersBuilder iteratorParametersBuilder
+    ) {
+        if (filesOrDirs == null || filesOrDirs.length == 0)
+            return Collections.emptyList();
+
+        final FileIOFactory ioFactory = iteratorParametersBuilder.ioFactory;
+
+        final TreeSet<FileDescriptor> descriptors = new TreeSet<>();
+
+        for (File file : filesOrDirs) {
+            if (file.isDirectory()) {
+                try {
+                    walkFileTree(file.toPath(), new SimpleFileVisitor<Path>() {
+                        @Override
+                        public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
+                            addFileDescriptor(path.toFile(), ioFactory, descriptors);
+
+                            return FileVisitResult.CONTINUE;
+                        }
+                    });
+                }
+                catch (IOException e) {
+                    U.error(log, "Failed to walk directories from root [" + file + "]. Skipping this directory.", e);
+                }
+
+                continue;
+            }
+
+            addFileDescriptor(file, ioFactory, descriptors);
+        }
+
+        return new ArrayList<>(descriptors);
+    }
+
+    /**
+     * @param file File.
+     * @param ioFactory IO factory.
+     * @param descriptors List of descriptors.
+     */
+    private void addFileDescriptor(File file, FileIOFactory ioFactory, TreeSet<FileDescriptor> descriptors) {
+        if (file.length() < HEADER_RECORD_SIZE)
+            return; // Filter out this segment as it is too short.
+
+        String fileName = file.getName();
+
+        if (!WAL_NAME_PATTERN.matcher(fileName).matches() &&
+            !WAL_SEGMENT_FILE_COMPACTED_PATTERN.matcher(fileName).matches())
+            return;  // Filter out this because it is not segment file.
+
+        FileDescriptor desc = readFileDescriptor(file, ioFactory);
+
+        if (desc != null)
+            descriptors.add(desc);
+    }
+
+    /**
+     * @param file File to read.
+     * @param ioFactory IO factory.
      */
-    public WALIterator iteratorWorkFiles(@NotNull final File... files) throws IgniteCheckedException {
-        return new StandaloneWalRecordsIterator(log, prepareSharedCtx(), ioFactory, true, keepBinary, bufSize, files);
+    private FileDescriptor readFileDescriptor(File file, FileIOFactory ioFactory) {
+        FileDescriptor ds = new FileDescriptor(file);
+
+        try (
+            FileIO fileIO = ds.isCompressed() ? new UnzipFileIO(file) : ioFactory.create(file);
+            ByteBufferExpander buf = new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())
+        ) {
+            final DataInput in = new FileInput(fileIO, buf);
+
+            // Header record must be agnostic to the serializer version.
+            final int type = in.readUnsignedByte();
+
+            if (type == RecordType.STOP_ITERATION_RECORD_TYPE) {
+                if (log.isInfoEnabled())
+                    log.info("Reached logical end of the segment for file " + file);
+
+                return null;
+            }
+
+            FileWALPointer ptr = readPosition(in);
+
+            return new FileDescriptor(file, ptr.index());
+        }
+        catch (IOException e) {
+            U.warn(log, "Failed to scan index from file [" + file + "]. Skipping this file during iteration", e);
+
+            return null;
+        }
     }
 
     /**
-     * @return fake shared context required for create minimal services for record reading
+     * @return Fake shared context required for create minimal services for record reading.
      */
-    @NotNull private GridCacheSharedContext prepareSharedCtx() throws IgniteCheckedException {
-        final GridKernalContext kernalCtx = new StandaloneGridKernalContext(log, binaryMetadataFileStoreDir, marshallerMappingFileStoreDir);
+    @NotNull private GridCacheSharedContext prepareSharedCtx(
+        IteratorParametersBuilder iteratorParametersBuilder
+    ) throws IgniteCheckedException {
+        GridKernalContext kernalCtx = new StandaloneGridKernalContext(log,
+            iteratorParametersBuilder.binaryMetadataFileStoreDir,
+            iteratorParametersBuilder.marshallerMappingFileStoreDir
+        );
 
-        final StandaloneIgniteCacheDatabaseSharedManager dbMgr = new StandaloneIgniteCacheDatabaseSharedManager();
+        StandaloneIgniteCacheDatabaseSharedManager dbMgr = new StandaloneIgniteCacheDatabaseSharedManager();
 
-        dbMgr.setPageSize(pageSize);
+        dbMgr.setPageSize(iteratorParametersBuilder.pageSize);
 
         return new GridCacheSharedContext<>(
             kernalCtx, null, null, null,
             null, null, null, dbMgr, null,
             null, null, null, null,
-            null, null, null);
+            null, null, null
+        );
     }
 
     /**
-     * @param ioFactory New factory to provide I/O interfaces for read/write operations with files
+     * Wal iterator parameter builder.
      */
-    public void ioFactory(FileIOFactory ioFactory) {
-        this.ioFactory = ioFactory;
+    public static class IteratorParametersBuilder {
+        /** */
+        private File[] filesOrDirs;
+
+        /** */
+        private int pageSize = DataStorageConfiguration.DFLT_PAGE_SIZE;
+
+        /** Wal records iterator buffer size. */
+        private int bufferSize = StandaloneWalRecordsIterator.DFLT_BUF_SIZE;
+
+        /** Keep binary. This flag disables converting of non primitive types (BinaryObjects). */
+        private boolean keepBinary;
+
+        /** Factory to provide I/O interfaces for read/write operations with files. */
+        private FileIOFactory ioFactory = new DataStorageConfiguration().getFileIOFactory();
+
+        /**
+         * Folder specifying location of metadata File Store. {@code null} means no specific folder is configured. <br>
+         * This folder should be specified for converting data entries into BinaryObjects
+         */
+        @Nullable private File binaryMetadataFileStoreDir;
+
+        /**
+         * Folder specifying location of marshaller mapping file store. {@code null} means no specific folder is configured.
+         * <br> This folder should be specified for converting data entries into BinaryObjects. Providing {@code null} will
+         * disable unmarshall for non primitive objects, BinaryObjects will be provided
+         */
+        @Nullable private File marshallerMappingFileStoreDir;
+
+        /** */
+        @Nullable private IgniteBiPredicate<RecordType, WALPointer> filter;
+
+        /**
+         * @param filesOrDirs Paths to files or directories.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder filesOrDirs(String... filesOrDirs) {
+            File[] filesOrDirs0 = new File[filesOrDirs.length];
+
+            for (int i = 0; i < filesOrDirs.length; i++) {
+                filesOrDirs0[i] = new File(filesOrDirs[i]);
+            }
+
+            return filesOrDirs(filesOrDirs0);
+        }
+
+        /**
+         * @param filesOrDirs Files or directories.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder filesOrDirs(File... filesOrDirs) {
+            if (this.filesOrDirs == null)
+                this.filesOrDirs = filesOrDirs;
+            else
+                this.filesOrDirs = merge(this.filesOrDirs, filesOrDirs);
+
+            return this;
+        }
+
+        /**
+         * @param pageSize Page size.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder pageSize(int pageSize) {
+            this.pageSize = pageSize;
+
+            return this;
+        }
+
+        /**
+         * @param bufferSize Initial size of buffer for reading segments.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder bufferSize(int bufferSize) {
+            this.bufferSize = bufferSize;
+
+            return this;
+        }
+
+        /**
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder keepBinary(boolean keepBinary) {
+            this.keepBinary = keepBinary;
+
+            return this;
+        }
+
+        /**
+         * @param ioFactory Custom IO factory for reading files.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder ioFactory(FileIOFactory ioFactory) {
+            this.ioFactory = ioFactory;
+
+            return this;
+        }
+
+        /**
+         * @param binaryMetadataFileStoreDir Path to the binary metadata.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder binaryMetadataFileStoreDir(File binaryMetadataFileStoreDir) {
+            this.binaryMetadataFileStoreDir = binaryMetadataFileStoreDir;
+
+            return this;
+        }
+
+        /**
+         * @param marshallerMappingFileStoreDir Path to the marshaller mapping.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder marshallerMappingFileStoreDir(File marshallerMappingFileStoreDir) {
+            this.marshallerMappingFileStoreDir = marshallerMappingFileStoreDir;
+
+            return this;
+        }
+
+        /**
+         * @param filter Record filter for skip records during iteration.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder filter(IgniteBiPredicate<RecordType, WALPointer> filter) {
+            this.filter = filter;
+
+            return this;
+        }
+
+        /**
+         * Copy current state of builder to new instance.
+         *
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder copy() {
+            return new IteratorParametersBuilder()
+                .filesOrDirs(filesOrDirs)
+                .pageSize(pageSize)
+                .bufferSize(bufferSize)
+                .keepBinary(keepBinary)
+                .ioFactory(ioFactory)
+                .binaryMetadataFileStoreDir(binaryMetadataFileStoreDir)
+                .marshallerMappingFileStoreDir(marshallerMappingFileStoreDir)
+                .filter(filter);
+        }
+
+        /**
+         * @throws IllegalArgumentException If validation failed.
+         */
+        public void validate() throws IllegalArgumentException {
+            A.ensure(pageSize >= 1024 && pageSize <= 16 * 1024, "Page size must be between 1kB and 16kB.");
+            A.ensure(U.isPow2(pageSize), "Page size must be a power of 2.");
+
+            A.ensure(bufferSize >= pageSize * 2, "Buffer to small.");
+        }
+
+        /**
+         * Merge file arrays.
+         *
+         * @param f1 Files array one.
+         * @param f2 Files array two.
+         * @return Merged arrays from one and two arrays.
+         */
+        private File[] merge(File[] f1, File[] f2) {
+            File[] merged = new File[f1.length + f2.length];
+
+            arraycopy(f1, 0, merged, 0, f1.length);
+            arraycopy(f2, 0, merged, f1.length, f2.length);
+
+            return merged;
+        }
     }
 
     /**
-     * @param bufSize New wal records iterator buffer size
+     *
      */
-    public void bufferSize(int bufSize) {
-        this.bufSize = bufSize;
+    private static class ConsoleLogger implements IgniteLogger {
+        /** */
+        private static final ConsoleLogger INSTANCE = new ConsoleLogger();
+
+        /** */
+        private static final PrintStream OUT = System.out;
+
+        /** */
+        private static final PrintStream ERR = System.err;
+
+        /** */
+        private ConsoleLogger() {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteLogger getLogger(Object ctgr) {
+            return this;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void trace(String msg) {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void debug(String msg) {
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void info(String msg) {
+            OUT.println(msg);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void warning(String msg, @Nullable Throwable e) {
+            OUT.println(msg);
+
+            if (e != null)
+                e.printStackTrace(OUT);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void error(String msg, @Nullable Throwable e) {
+            ERR.println(msg);
+
+            if (e != null)
+                e.printStackTrace(ERR);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isTraceEnabled() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isDebugEnabled() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isInfoEnabled() {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isQuiet() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String fileName() {
+            return "SYSTEM.OUT";
+        }
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index 712517bbbf90c..9df446836198d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -17,44 +17,41 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.wal.reader;
 
-import java.io.DataInput;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.nio.ByteOrder;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.UnwrapDataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheObjectContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator;
-import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.ReadFileHandle;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
-import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiPredicate;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.HEADER_RECORD_SIZE;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader;
 
 /**
  * WAL reader iterator, for creation in standalone WAL reader tool
@@ -66,78 +63,52 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
 
     /** */
     private static final long serialVersionUID = 0L;
-
-    /**
-     * WAL files directory. Should already contain 'consistent ID' as subfolder.
-     * <code>null</code> value means file-by-file iteration mode
-     */
-    @Nullable
-    private File walFilesDir;
-
     /**
      * File descriptors remained to scan.
      * <code>null</code> value means directory scan mode
      */
     @Nullable
-    private List<FileWriteAheadLogManager.FileDescriptor> walFileDescriptors;
+    private final List<FileDescriptor> walFileDescriptors;
+
+    /** */
+    private int curIdx = -1;
 
     /** Keep binary. This flag disables converting of non primitive types (BinaryObjects) */
     private boolean keepBinary;
 
-    /**
-     * Creates iterator in directory scan mode
-     * @param walFilesDir Wal files directory. Should already contain node consistent ID as subfolder
-     * @param log Logger.
-     * @param sharedCtx Shared context. Cache processor is to be configured if Cache Object Key & Data Entry is required.
-     * @param ioFactory File I/O factory.
-     * @param keepBinary  Keep binary. This flag disables converting of non primitive types
-     * @param bufSize Buffer size.
-     */
-    StandaloneWalRecordsIterator(
-        @NotNull File walFilesDir,
-        @NotNull IgniteLogger log,
-        @NotNull GridCacheSharedContext sharedCtx,
-        @NotNull FileIOFactory ioFactory,
-        boolean keepBinary,
-        int bufSize
-    ) throws IgniteCheckedException {
-        super(log,
-            sharedCtx,
-            new RecordSerializerFactoryImpl(sharedCtx),
-            ioFactory,
-            bufSize);
-        this.keepBinary = keepBinary;
-        init(walFilesDir, false, null);
-        advance();
-    }
-
     /**
      * Creates iterator in file-by-file iteration mode. Directory
      * @param log Logger.
      * @param sharedCtx Shared context. Cache processor is to be configured if Cache Object Key & Data Entry is
      * required.
      * @param ioFactory File I/O factory.
-     * @param workDir Work directory is scanned, false - archive
      * @param keepBinary Keep binary. This flag disables converting of non primitive types
      * (BinaryObjects will be used instead)
      * @param walFiles Wal files.
      */
     StandaloneWalRecordsIterator(
-            @NotNull IgniteLogger log,
-            @NotNull GridCacheSharedContext sharedCtx,
-            @NotNull FileIOFactory ioFactory,
-            boolean workDir,
-            boolean keepBinary,
-            int bufSize,
-            @NotNull File... walFiles) throws IgniteCheckedException {
-        super(log,
+        @NotNull IgniteLogger log,
+        @NotNull GridCacheSharedContext sharedCtx,
+        @NotNull FileIOFactory ioFactory,
+        @NotNull List<FileDescriptor> walFiles,
+        IgniteBiPredicate<RecordType, WALPointer> readTypeFilter,
+        boolean keepBinary,
+        int initialReadBufferSize
+    ) throws IgniteCheckedException {
+        super(
+            log,
             sharedCtx,
-            new RecordSerializerFactoryImpl(sharedCtx),
+            new RecordSerializerFactoryImpl(sharedCtx, readTypeFilter),
             ioFactory,
-            bufSize);
+            initialReadBufferSize
+        );
 
         this.keepBinary = keepBinary;
-        init(null, workDir, walFiles);
+
+        walFileDescriptors = walFiles;
+
+        init(walFiles);
+
         advance();
     }
 
@@ -145,119 +116,42 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
      * For directory mode sets oldest file as initial segment,
      * for file by file mode, converts all files to descriptors and gets oldest as initial.
      *
-     * @param walFilesDir directory for directory scan mode
-     * @param workDir work directory, only for file-by-file mode
      * @param walFiles files for file-by-file iteration mode
      */
-    private void init(
-        @Nullable final File walFilesDir,
-        final boolean workDir,
-        @Nullable final File[] walFiles) throws IgniteCheckedException {
-        if (walFilesDir != null) {
-            FileWriteAheadLogManager.FileDescriptor[] descs = FileWriteAheadLogManager.loadFileDescriptors(walFilesDir);
-            curWalSegmIdx = !F.isEmpty(descs) ? descs[0].getIdx() : 0;
-            this.walFilesDir = walFilesDir;
-        }
-        else {
+    private void init(List<FileDescriptor> walFiles) {
+        if (walFiles == null || walFiles.isEmpty())
+            return;
 
-            if (workDir)
-                walFileDescriptors = scanIndexesFromFileHeaders(walFiles);
-            else
-                walFileDescriptors = new ArrayList<>(Arrays.asList(FileWriteAheadLogManager.scan(walFiles)));
-
-            curWalSegmIdx = !walFileDescriptors.isEmpty() ? walFileDescriptors.get(0).getIdx() : 0;
-        }
-        curWalSegmIdx--;
+        curWalSegmIdx = walFiles.get(curIdx + 1).idx() - 1;
 
         if (log.isDebugEnabled())
             log.debug("Initialized WAL cursor [curWalSegmIdx=" + curWalSegmIdx + ']');
     }
 
-    /**
-     * This methods checks all provided files to be correct WAL segment.
-     * Header record and its position is checked. WAL position is used to determine real index.
-     * File index from file name is ignored.
-     *
-     * @param allFiles files to scan.
-     * @return list of file descriptors with checked header records, having correct file index is set
-     */
-    private List<FileWriteAheadLogManager.FileDescriptor> scanIndexesFromFileHeaders(
-        @Nullable final File[] allFiles) {
-        if (allFiles == null || allFiles.length == 0)
-            return Collections.emptyList();
-
-        final List<FileWriteAheadLogManager.FileDescriptor> resultingDescs = new ArrayList<>();
-
-        for (File file : allFiles) {
-            if (file.length() < HEADER_RECORD_SIZE)
-                continue;  //filter out this segment as it is too short
-
-            FileWALPointer ptr;
-
-            try (
-                FileIO fileIO = ioFactory.create(file);
-                ByteBufferExpander buf = new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())
-            ) {
-                final DataInput in = new FileInput(fileIO, buf);
-
-                // Header record must be agnostic to the serializer version.
-                final int type = in.readUnsignedByte();
-
-                if (type == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE) {
-                    if (log.isInfoEnabled())
-                        log.info("Reached logical end of the segment for file " + file);
-
-                    continue; //filter out this segment
-                }
-                ptr = RecordV1Serializer.readPosition(in);
-            }
-            catch (IOException e) {
-                U.warn(log, "Failed to scan index from file [" + file + "]. Skipping this file during iteration", e);
-
-                continue; //filter out this segment
-            }
-
-            resultingDescs.add(new FileWriteAheadLogManager.FileDescriptor(file, ptr.index()));
-        }
-        Collections.sort(resultingDescs);
-
-        return resultingDescs;
-    }
-
     /** {@inheritDoc} */
     @Override protected AbstractReadFileHandle advanceSegment(
-        @Nullable final AbstractReadFileHandle curWalSegment) throws IgniteCheckedException {
+        @Nullable final AbstractReadFileHandle curWalSegment
+    ) throws IgniteCheckedException {
 
         if (curWalSegment != null)
             curWalSegment.close();
 
         curWalSegmIdx++;
-        // curHandle.workDir is false
-        final FileWriteAheadLogManager.FileDescriptor fd;
-
-        if (walFilesDir != null) {
-            File segmentFile = new File(walFilesDir,
-                FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx));
 
-            if (!segmentFile.exists())
-                segmentFile = new File(walFilesDir,
-                    FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx) + ".zip");
+        curIdx++;
 
-            fd = new FileWriteAheadLogManager.FileDescriptor(segmentFile);
-        }
-        else {
-            if (walFileDescriptors.isEmpty())
-                return null; //no files to read, stop iteration
+        if (curIdx >= walFileDescriptors.size())
+            return null;
 
-            fd = walFileDescriptors.remove(0);
-        }
+        FileDescriptor fd = walFileDescriptors.get(curIdx);
 
         if (log.isDebugEnabled())
-            log.debug("Reading next file [absIdx=" + curWalSegmIdx + ", file=" + fd.getAbsolutePath() + ']');
+            log.debug("Reading next file [absIdx=" + curWalSegmIdx + ", file=" + fd.file().getAbsolutePath() + ']');
 
         assert fd != null;
 
         curRec = null;
+
         try {
             return initReadHandle(fd, null);
         }
@@ -269,12 +163,43 @@ private List<FileWriteAheadLogManager.FileDescriptor> scanIndexesFromFileHeaders
         }
     }
 
+    /** {@inheritDoc} */
+    @Override protected AbstractReadFileHandle initReadHandle(
+        @NotNull AbstractFileDescriptor desc,
+        @Nullable FileWALPointer start
+    ) throws IgniteCheckedException, FileNotFoundException {
+
+        AbstractFileDescriptor fd = desc;
+
+        while (true) {
+            try {
+                FileIO fileIO = fd.isCompressed() ? new UnzipFileIO(fd.file()) : ioFactory.create(fd.file());
+
+                readSegmentHeader(fileIO, curWalSegmIdx);
+
+                break;
+            }
+            catch (IOException | IgniteCheckedException e) {
+                log.error("Failed to init segment curWalSegmIdx=" + curWalSegmIdx + ", curIdx=" + curIdx, e);
+
+                curIdx++;
+
+                if (curIdx >= walFileDescriptors.size())
+                    return null;
+
+                fd = walFileDescriptors.get(curIdx);
+            }
+        }
+
+        return super.initReadHandle(fd, start);
+    }
+
     /** {@inheritDoc} */
     @NotNull @Override protected WALRecord postProcessRecord(@NotNull final WALRecord rec) {
-        final GridKernalContext kernalCtx = sharedCtx.kernalContext();
-        final IgniteCacheObjectProcessor processor = kernalCtx.cacheObjects();
+         GridKernalContext kernalCtx = sharedCtx.kernalContext();
+         IgniteCacheObjectProcessor processor = kernalCtx.cacheObjects();
 
-        if (processor != null && rec.type() == WALRecord.RecordType.DATA_RECORD) {
+        if (processor != null && rec.type() == RecordType.DATA_RECORD) {
             try {
                 return postProcessDataRecord((DataRecord)rec, kernalCtx, processor);
             }
@@ -296,11 +221,12 @@ private List<FileWriteAheadLogManager.FileDescriptor> scanIndexesFromFileHeaders
      * @throws IgniteCheckedException if failed.
      */
     @NotNull private WALRecord postProcessDataRecord(
-        @NotNull final DataRecord dataRec,
-        final GridKernalContext kernalCtx,
-        final IgniteCacheObjectProcessor processor) throws IgniteCheckedException {
-        final CacheObjectContext fakeCacheObjCtx = new CacheObjectContext(kernalCtx,
-            null, null, false, false, false);
+        @NotNull DataRecord dataRec,
+        GridKernalContext kernalCtx,
+        IgniteCacheObjectProcessor processor
+    ) throws IgniteCheckedException {
+        final CacheObjectContext fakeCacheObjCtx = new CacheObjectContext(
+            kernalCtx, null, null, false, false, false);
 
         final List<DataEntry> entries = dataRec.writeEntries();
         final List<DataEntry> postProcessedEntries = new ArrayList<>(entries.size());
@@ -327,8 +253,7 @@ private List<FileWriteAheadLogManager.FileDescriptor> scanIndexesFromFileHeaders
      * @return post precessed entry
      * @throws IgniteCheckedException if failed
      */
-    @NotNull
-    private DataEntry postProcessDataEntry(
+    @NotNull private DataEntry postProcessDataEntry(
         final IgniteCacheObjectProcessor processor,
         final CacheObjectContext fakeCacheObjCtx,
         final DataEntry dataEntry) throws IgniteCheckedException {
@@ -383,8 +308,9 @@ private DataEntry postProcessDataEntry(
     }
 
     /** {@inheritDoc} */
-    @Override protected AbstractReadFileHandle createReadFileHandle(FileIO fileIO, long idx,
-        RecordSerializer ser, FileInput in) {
-        return new FileWriteAheadLogManager.ReadFileHandle(fileIO, idx, ser, in);
+    @Override protected AbstractReadFileHandle createReadFileHandle(
+        FileIO fileIO, long idx, RecordSerializer ser, FileInput in
+    ) {
+        return new ReadFileHandle(fileIO, idx, ser, in);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializerFactoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializerFactoryImpl.java
index 468392a2631bb..2e2e2f8cbe942 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializerFactoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializerFactoryImpl.java
@@ -31,7 +31,7 @@ public class RecordSerializerFactoryImpl implements RecordSerializerFactory {
     private GridCacheSharedContext cctx;
 
     /** Write pointer. */
-    private boolean writePointer;
+    private boolean needWritePointer;
 
     /** Read record filter. */
     private IgniteBiPredicate<WALRecord.RecordType, WALPointer> recordDeserializeFilter;
@@ -49,7 +49,17 @@ public class RecordSerializerFactoryImpl implements RecordSerializerFactory {
      * @param cctx Cctx.
      */
     public RecordSerializerFactoryImpl(GridCacheSharedContext cctx) {
+        this(cctx, null);
+    }
+    /**
+     * @param cctx Cctx.
+     */
+    public RecordSerializerFactoryImpl(
+        GridCacheSharedContext cctx,
+        IgniteBiPredicate<WALRecord.RecordType, WALPointer> readTypeFilter
+    ) {
         this.cctx = cctx;
+        this.recordDeserializeFilter = readTypeFilter;
     }
 
     /** {@inheritDoc} */
@@ -59,14 +69,24 @@ public RecordSerializerFactoryImpl(GridCacheSharedContext cctx) {
 
         switch (ver) {
             case 1:
-                return new RecordV1Serializer(new RecordDataV1Serializer(cctx),
-                    writePointer, marshalledMode, skipPositionCheck, recordDeserializeFilter);
+                return new RecordV1Serializer(
+                    new RecordDataV1Serializer(cctx),
+                    needWritePointer,
+                    marshalledMode,
+                    skipPositionCheck,
+                    recordDeserializeFilter);
 
             case 2:
-                RecordDataV2Serializer dataV2Serializer = new RecordDataV2Serializer(new RecordDataV1Serializer(cctx));
+                RecordDataV2Serializer dataV2Serializer = new RecordDataV2Serializer(
+                    new RecordDataV1Serializer(cctx));
 
-                return new RecordV2Serializer(dataV2Serializer,
-                    writePointer, marshalledMode, skipPositionCheck, recordDeserializeFilter);
+                return new RecordV2Serializer(
+                    dataV2Serializer,
+                    needWritePointer,
+                    marshalledMode,
+                    skipPositionCheck,
+                    recordDeserializeFilter
+                );
 
             default:
                 throw new IgniteCheckedException("Failed to create a serializer with the given version " +
@@ -78,12 +98,12 @@ public RecordSerializerFactoryImpl(GridCacheSharedContext cctx) {
      * @return Write pointer.
      */
     public boolean writePointer() {
-        return writePointer;
+        return needWritePointer;
     }
 
     /** {@inheritDoc} */
     @Override public RecordSerializerFactoryImpl writePointer(boolean writePointer) {
-        this.writePointer = writePointer;
+        this.needWritePointer = writePointer;
 
         return this;
     }
@@ -97,7 +117,8 @@ public IgniteBiPredicate<WALRecord.RecordType, WALPointer> recordDeserializeFilt
 
     /** {@inheritDoc} */
     @Override public RecordSerializerFactoryImpl recordDeserializeFilter(
-        IgniteBiPredicate<WALRecord.RecordType, WALPointer> readTypeFilter) {
+        IgniteBiPredicate<WALRecord.RecordType, WALPointer> readTypeFilter
+    ) {
         this.recordDeserializeFilter = readTypeFilter;
 
         return this;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
index caa096294e88b..ca484ce112031 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -134,6 +134,9 @@ public class RecordV1Serializer implements RecordSerializer {
                 throw new SegmentEofException("WAL segment rollover detected (will end iteration) [expPtr=" + expPtr +
                         ", readPtr=" + ptr + ']', null);
 
+            if (recType == null)
+                throw new IOException("Unknown record type: " + recType);
+
             final WALRecord rec = dataSerializer.readRecord(recType, in);
 
             rec.position(ptr);
@@ -341,12 +344,7 @@ static RecordType readRecordType(DataInput in) throws IgniteCheckedException, IO
         if (type == WALRecord.RecordType.STOP_ITERATION_RECORD_TYPE)
             throw new SegmentEofException("Reached logical end of the segment", null);
 
-        RecordType recType = RecordType.fromOrdinal(type - 1);
-
-        if (recType == null)
-            throw new IOException("Unknown record type: " + type);
-
-        return recType;
+        return RecordType.fromOrdinal(type - 1);
     }
 
     /**
@@ -359,7 +357,11 @@ static RecordType readRecordType(DataInput in) throws IgniteCheckedException, IO
      * @throws EOFException In case of end of file.
      * @throws IgniteCheckedException If it's unable to read record.
      */
-    static WALRecord readWithCrc(FileInput in0, WALPointer expPtr, RecordIO reader) throws EOFException, IgniteCheckedException {
+    static WALRecord readWithCrc(
+        FileInput in0,
+        WALPointer expPtr,
+        RecordIO reader
+    ) throws EOFException, IgniteCheckedException {
         long startPos = -1;
 
         try (FileInput.Crc32CheckingFileInput in = in0.startRead(skipCrc)) {
@@ -377,7 +379,16 @@ static WALRecord readWithCrc(FileInput in0, WALPointer expPtr, RecordIO reader)
             throw e;
         }
         catch (Exception e) {
-            throw new IgniteCheckedException("Failed to read WAL record at position: " + startPos, e);
+            long size = -1;
+
+            try {
+                size = in0.io().size();
+            }
+            catch (IOException ignore) {
+                // No-op. It just for information. Fail calculate file size.
+            }
+
+            throw new IgniteCheckedException("Failed to read WAL record at position: " + startPos + " size: " + size, e);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
index 2b81210e6fd0a..2c65ebe1d2c96 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
@@ -115,6 +115,9 @@ public class RecordV2Serializer implements RecordSerializer {
 
             FileWALPointer ptr = readPositionAndCheckPoint(in, expPtr, skipPositionCheck);
 
+            if (recType == null)
+                throw new IOException("Unknown record type: " + recType);
+
             if (recordFilter != null && !recordFilter.apply(recType, ptr)) {
                 int toSkip = ptr.length() - REC_TYPE_SIZE - FILE_WAL_POINTER_SIZE - CRC_SIZE;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorExceptionDuringReadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorExceptionDuringReadTest.java
new file mode 100644
index 0000000000000..ccd889aaaaf42
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorExceptionDuringReadTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class IgniteWalIteratorExceptionDuringReadTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int WAL_SEGMENT_SIZE = 1024 * 1024 * 20;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setWalSegmentSize(WAL_SEGMENT_SIZE)
+                .setWalMode(WALMode.LOG_ONLY)
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void test() throws Exception {
+        IgniteEx ig = (IgniteEx)startGrid();
+
+        ig.cluster().active(true);
+
+        IgniteCache<Integer, byte[]> cache = ig.cache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < 20 * 4; i++)
+            cache.put(i, new byte[1024 * 1024]);
+
+        ig.cluster().active(false);
+
+        IgniteWalIteratorFactory iteratorFactory = new IgniteWalIteratorFactory(log);
+
+        FileWALPointer failOnPtr = new FileWALPointer(3, 1024 * 1024 * 5, 0);
+
+        String failMessage = "test fail message";
+
+        IteratorParametersBuilder builder = new IteratorParametersBuilder()
+            .filesOrDirs(U.defaultWorkDirectory())
+            .filter((r, ptr) -> {
+                FileWALPointer ptr0 = (FileWALPointer)ptr;
+
+                if (ptr0.compareTo(failOnPtr) >= 0)
+                    throw new TestRuntimeException(failMessage);
+
+                return true;
+            });
+
+        try (WALIterator it = iteratorFactory.iterator(builder)) {
+            FileWALPointer ptr = null;
+
+            boolean failed = false;
+
+            while (it.hasNext()) {
+                try {
+                    IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+                    ptr = (FileWALPointer)tup.get1();
+                }
+                catch (IgniteException e) {
+                    Assert.assertNotNull(ptr);
+                    Assert.assertEquals(failOnPtr.index(), ptr.index());
+                    Assert.assertTrue(ptr.compareTo(failOnPtr) < 0);
+
+                    failed = X.hasCause(e, TestRuntimeException.class);
+
+                    break;
+                }
+            }
+
+            assertTrue(failed);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestRuntimeException extends IgniteException {
+        /**
+         * @param msg Exception message.
+         */
+        private TestRuntimeException(String msg) {
+            super(msg);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
index 28f049aaef5c9..a6183a9073b9a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
@@ -23,7 +23,6 @@
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import java.io.Serializable;
-import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumMap;
 import java.util.HashMap;
@@ -48,11 +47,10 @@
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.events.WalSegmentArchivedEvent;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
@@ -66,24 +64,27 @@
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteInClosure;
-import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.logger.NullLogger;
-import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.transactions.Transaction;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.junit.Assert;
 
+import static java.util.Arrays.fill;
 import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_RECORD;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.TX_RECORD;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.CREATE;
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
@@ -93,6 +94,9 @@
  * Test suite for WAL segments reader and event generator.
  */
 public class IgniteWalReaderTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
     /** Wal segments count */
     private static final int WAL_SEGMENTS = 10;
 
@@ -103,10 +107,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
     private static final String CACHE_ADDL_NAME = "cache1";
 
     /** Dump records to logger. Should be false for non local run. */
-    private static final boolean dumpRecords = false;
-
-    /** Page size to set. */
-    public static final int PAGE_SIZE = 4 * 1024;
+    private static final boolean DUMP_RECORDS = true;
 
     /**
      * Field for transferring setting from test to getConfig method.
@@ -125,9 +126,11 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        final IgniteConfiguration cfg = super.getConfiguration(gridName);
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
 
-        final CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(CACHE_NAME);
+        CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(CACHE_NAME);
 
         ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
         ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
@@ -140,8 +143,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
 
         DataStorageConfiguration dsCfg = new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(
-                new DataRegionConfiguration().setMaxSize(1024 * 1024 * 1024).setPersistenceEnabled(true))
-            .setPageSize(PAGE_SIZE)
+                new DataRegionConfiguration()
+                    .setMaxSize(1024 * 1024 * 1024)
+                    .setPersistenceEnabled(true))
             .setWalHistorySize(1)
             .setWalSegmentSize(1024 * 1024)
             .setWalSegments(WAL_SEGMENTS)
@@ -150,12 +154,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
         if (archiveIncompleteSegmentAfterInactivityMs > 0)
             dsCfg.setWalAutoArchiveAfterInactivity(archiveIncompleteSegmentAfterInactivityMs);
 
-        final String workDir = U.defaultWorkDirectory();
-        final File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
-        final File wal = new File(db, "wal");
+        String workDir = U.defaultWorkDirectory();
+        File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
+        File wal = new File(db, "wal");
 
         if(setWalAndArchiveToSameValue) {
-            final String walAbsPath = wal.getAbsolutePath();
+            String walAbsPath = wal.getAbsolutePath();
 
             dsCfg.setWalPath(walAbsPath);
             dsCfg.setWalArchivePath(walAbsPath);
@@ -173,85 +177,69 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
     @Override protected void beforeTest() throws Exception {
         stopAllGrids();
 
-        deleteWorkFiles();
+        cleanPersistenceDir();
     }
 
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         stopAllGrids();
 
+        cleanPersistenceDir();
+
         if (clearProperties)
             System.clearProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS);
     }
 
-    /**
-     * @throws IgniteCheckedException If failed.
-     */
-    private void deleteWorkFiles() throws Exception {
-        cleanPersistenceDir();
-    }
-
     /**
      * @throws Exception if failed.
      */
     public void testFillWalAndReadRecords() throws Exception {
         setWalAndArchiveToSameValue = false;
-        final int cacheObjectsToWrite = 10000;
 
-        final Ignite ignite0 = startGrid("node0");
+        Ignite ignite0 = startGrid();
 
-        ignite0.active(true);
+        ignite0.cluster().active(true);
 
-        final Serializable consistentId = (Serializable)ignite0.cluster().localNode().consistentId();
-        final String subfolderName = genNewStyleSubfolderName(0, (UUID)consistentId);
+        Serializable consistentId = (Serializable)ignite0.cluster().localNode().consistentId();
 
-        putDummyRecords(ignite0, cacheObjectsToWrite);
+        String subfolderName = genNewStyleSubfolderName(0, (UUID)consistentId);
 
-        stopGrid("node0");
+        int cacheObjectsToWrite = 10_000;
 
-        final String workDir = U.defaultWorkDirectory();
-        final File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
-        final File wal = new File(db, "wal");
-        final File walArchive = setWalAndArchiveToSameValue ? wal : new File(wal, "archive");
+        putDummyRecords(ignite0, cacheObjectsToWrite);
 
-        int[] checkKeyIterArr = new int[cacheObjectsToWrite];
+        stopGrid();
 
-        final File walArchiveDirWithConsistentId = new File(walArchive, subfolderName);
-        final File walWorkDirWithConsistentId = new File(wal, subfolderName);
-        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+        String workDir = U.defaultWorkDirectory();
 
-        //Check iteratorArchiveDirectory and iteratorArchiveFiles are same.
-        final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId));
+        File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
 
-        log.info("Total records loaded using directory : " + cntArchiveDir);
+        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
 
-        final int cntArchiveFileByFile = iterateAndCount(factory.iteratorArchiveFiles(
-            walArchiveDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER)));
+        IteratorParametersBuilder params =
+            createIteratorParametersBuilder(workDir, subfolderName)
+                .filesOrDirs(db);
 
-        log.info("Total records loaded using archive directory (file-by-file): " + cntArchiveFileByFile);
+        // Check iteratorArchiveDirectory and iteratorArchiveFiles are same.
+        int cntArchiveDir = iterateAndCount(factory.iterator(params));
 
-        assertTrue(cntArchiveDir == cntArchiveFileByFile);
+        log.info("Total records loaded using directory : " + cntArchiveDir);
 
-        //Check iteratorArchiveFiles + iteratorWorkFiles iterate over all entries.
-        Arrays.fill(checkKeyIterArr, 0);
+        assertTrue(cntArchiveDir > 0);
 
-        iterateAndCountDataRecord(factory.iteratorArchiveFiles(
-            walArchiveDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER)), new IgniteBiInClosure<Object, Object>() {
-            @Override public void apply(Object o, Object o2) {
-                checkKeyIterArr[(Integer)o]++;
-            }
-        }, null);
+        // Check iteratorArchiveFiles + iteratorWorkFiles iterate over all entries.
+        int[] checkKeyIterArr = new int[cacheObjectsToWrite];
 
-        final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
+        fill(checkKeyIterArr, 0);
 
-        iterateAndCountDataRecord(factory.iteratorWorkFiles(workFiles), new IgniteBiInClosure<Object, Object>() {
-            @Override public void apply(Object o, Object o2) {
-                checkKeyIterArr[(Integer) o]++;
-            }
-        }, null).size();
+        iterateAndCountDataRecord(
+            factory.iterator(params),
+            (o1, o2) -> checkKeyIterArr[(Integer)o1]++,
+            null
+        );
 
-        for (int i =0 ; i< cacheObjectsToWrite; i++)
-            assertTrue("Iterator didn't find key="+ i, checkKeyIterArr[i] > 0);
+        for (int i = 0; i < cacheObjectsToWrite; i++)
+            assertTrue("Iterator didn't find key=" + i, checkKeyIterArr[i] > 0);
     }
 
     /**
@@ -262,35 +250,29 @@ public void testFillWalAndReadRecords() throws Exception {
      * @throws IgniteCheckedException if failed to iterate.
      */
     private int iterateAndCount(WALIterator walIter) throws IgniteCheckedException {
-        return iterateAndCount(walIter, true);
-    }
-
-    /**
-     * Iterates on records and closes iterator.
-     *
-     * @param walIter iterator to count, will be closed.
-     * @param touchEntries access data within entries.
-     * @return count of records.
-     * @throws IgniteCheckedException if failed to iterate.
-     */
-    private int iterateAndCount(WALIterator walIter, boolean touchEntries) throws IgniteCheckedException {
         int cnt = 0;
 
         try (WALIterator it = walIter) {
             while (it.hasNextX()) {
-                final IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
-                final WALRecord walRecord = next.get2();
-                if (touchEntries && walRecord.type() == WALRecord.RecordType.DATA_RECORD) {
-                    final DataRecord record = (DataRecord)walRecord;
+                IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+
+                WALRecord walRecord = tup.get2();
+
+                if (walRecord.type() == DATA_RECORD) {
+                    DataRecord record = (DataRecord)walRecord;
+
                     for (DataEntry entry : record.writeEntries()) {
-                        final KeyCacheObject key = entry.key();
-                        final CacheObject val = entry.value();
-                        if (dumpRecords)
+                        KeyCacheObject key = entry.key();
+                        CacheObject val = entry.value();
+
+                        if (DUMP_RECORDS)
                             log.info("Op: " + entry.op() + ", Key: " + key + ", Value: " + val);
                     }
                 }
-                if (dumpRecords)
+
+                if (DUMP_RECORDS)
                     log.info("Record: " + walRecord);
+
                 cnt++;
             }
         }
@@ -303,128 +285,148 @@ private int iterateAndCount(WALIterator walIter, boolean touchEntries) throws Ig
      * @throws Exception if failed.
      */
     public void testArchiveCompletedEventFired() throws Exception {
-        final AtomicBoolean evtRecorded = new AtomicBoolean();
+        AtomicBoolean evtRecorded = new AtomicBoolean();
 
-        final Ignite ignite = startGrid("node0");
+        Ignite ignite = startGrid();
 
-        ignite.active(true);
+        ignite.cluster().active(true);
 
         final IgniteEvents evts = ignite.events();
 
         if (!evts.isEnabled(EVT_WAL_SEGMENT_ARCHIVED))
-            assertTrue("nothing to test", false);
+            fail("nothing to test");
 
-        evts.localListen(new IgnitePredicate<Event>() {
-            @Override public boolean apply(Event e) {
-                WalSegmentArchivedEvent archComplEvt = (WalSegmentArchivedEvent)e;
-                long idx = archComplEvt.getAbsWalSegmentIdx();
-                log.info("Finished archive for segment [" + idx + ", " +
-                    archComplEvt.getArchiveFile() + "]: [" + e + "]");
+        evts.localListen(e -> {
+            WalSegmentArchivedEvent archComplEvt = (WalSegmentArchivedEvent)e;
 
-                evtRecorded.set(true);
-                return true;
-            }
+            long idx = archComplEvt.getAbsWalSegmentIdx();
+
+            log.info("Finished archive for segment [" +
+                idx + ", " + archComplEvt.getArchiveFile() + "]: [" + e + "]");
+
+            evtRecorded.set(true);
+
+            return true;
         }, EVT_WAL_SEGMENT_ARCHIVED);
 
         putDummyRecords(ignite, 500);
 
-        stopGrid("node0");
+        stopGrid();
+
         assertTrue(evtRecorded.get());
     }
 
     /**
-     * Puts provided number of records to fill WAL.
+     * Tests time out based WAL segment archiving.
      *
-     * @param ignite ignite instance.
-     * @param recordsToWrite count.
+     * @throws Exception if failure occurs.
      */
-    private void putDummyRecords(Ignite ignite, int recordsToWrite) {
-        IgniteCache<Object, Object> cache0 = ignite.cache(CACHE_NAME);
+    public void testArchiveIncompleteSegmentAfterInactivity() throws Exception {
+        AtomicBoolean waitingForEvt = new AtomicBoolean();
 
-        for (int i = 0; i < recordsToWrite; i++)
-            cache0.put(i, new IndexedObject(i));
-    }
+        CountDownLatch archiveSegmentForInactivity = new CountDownLatch(1);
 
-    /**
-     * Puts provided number of records to fill WAL.
-     *
-     * @param ignite ignite instance.
-     * @param recordsToWrite count.
-     */
-    private void putAllDummyRecords(Ignite ignite, int recordsToWrite) {
-        IgniteCache<Object, Object> cache0 = ignite.cache(CACHE_NAME);
+        archiveIncompleteSegmentAfterInactivityMs = 1000;
 
-        Map<Object, Object> values = new HashMap<>();
+        Ignite ignite = startGrid();
 
-        for (int i = 0; i < recordsToWrite; i++)
-            values.put(i, new IndexedObject(i));
+        ignite.cluster().active(true);
 
-        cache0.putAll(values);
-    }
+        IgniteEvents evts = ignite.events();
 
-    /**
-     * Puts provided number of records to fill WAL under transactions.
-     *
-     * @param ignite ignite instance.
-     * @param recordsToWrite count.
-     * @param txCnt transactions to run. If number is less then records count, txCnt records will be written.
-     */
-    private IgniteCache<Object, Object> txPutDummyRecords(Ignite ignite, int recordsToWrite, int txCnt) {
-        IgniteCache<Object, Object> cache0 = ignite.cache(CACHE_NAME);
-        int keysPerTx = recordsToWrite / txCnt;
-        if (keysPerTx == 0)
-            keysPerTx = 1;
-        for (int t = 0; t < txCnt; t++) {
-            try (Transaction tx = ignite.transactions().txStart()) {
-                for (int i = t * keysPerTx; i < (t + 1) * keysPerTx; i++)
-                    cache0.put(i, new IndexedObject(i));
+        evts.localListen(e -> {
+            WalSegmentArchivedEvent archComplEvt = (WalSegmentArchivedEvent)e;
 
-                tx.commit();
-            }
-        }
-        return cache0;
+            long idx = archComplEvt.getAbsWalSegmentIdx();
+
+            log.info("Finished archive for segment [" + idx + ", " +
+                archComplEvt.getArchiveFile() + "]: [" + e + "]");
+
+            if (waitingForEvt.get())
+                archiveSegmentForInactivity.countDown();
+
+            return true;
+        }, EVT_WAL_SEGMENT_ARCHIVED);
+
+        putDummyRecords(ignite, 100);
+
+        waitingForEvt.set(true); // Flag for skipping regular log() and rollOver().
+
+        log.info("Wait for archiving segment for inactive grid started");
+
+        boolean recordedAfterSleep = archiveSegmentForInactivity.await(
+            archiveIncompleteSegmentAfterInactivityMs + 1001, TimeUnit.MILLISECONDS);
+
+        stopGrid();
+
+        assertTrue(recordedAfterSleep);
     }
 
     /**
-     * Tests time out based WAL segment archiving.
+     * Tests archive completed event is fired.
      *
-     * @throws Exception if failure occurs.
+     * @throws Exception if failed.
      */
-    public void testArchiveIncompleteSegmentAfterInactivity() throws Exception {
-        final AtomicBoolean waitingForEvt = new AtomicBoolean();
-        final CountDownLatch archiveSegmentForInactivity = new CountDownLatch(1);
+    public void testFillWalForExactSegmentsCount() throws Exception {
+        customWalMode = WALMode.FSYNC;
 
-        archiveIncompleteSegmentAfterInactivityMs = 1000;
+        CountDownLatch reqSegments = new CountDownLatch(15);
 
-        final Ignite ignite = startGrid("node0");
+        Ignite ignite = startGrid();
 
-        ignite.active(true);
+        ignite.cluster().active(true);
 
         final IgniteEvents evts = ignite.events();
 
-        evts.localListen(new IgnitePredicate<Event>() {
-            @Override public boolean apply(Event e) {
-                WalSegmentArchivedEvent archComplEvt = (WalSegmentArchivedEvent)e;
-                long idx = archComplEvt.getAbsWalSegmentIdx();
-                log.info("Finished archive for segment [" + idx + ", " +
-                    archComplEvt.getArchiveFile() + "]: [" + e + "]");
+        if (!evts.isEnabled(EVT_WAL_SEGMENT_ARCHIVED))
+            fail("nothing to test");
 
-                if (waitingForEvt.get())
-                    archiveSegmentForInactivity.countDown();
-                return true;
-            }
+        evts.localListen(e -> {
+            WalSegmentArchivedEvent archComplEvt = (WalSegmentArchivedEvent)e;
+
+            long idx = archComplEvt.getAbsWalSegmentIdx();
+
+            log.info("Finished archive for segment [" + idx + ", " +
+                archComplEvt.getArchiveFile() + "]: [" + e + "]");
+
+            reqSegments.countDown();
+
+            return true;
         }, EVT_WAL_SEGMENT_ARCHIVED);
 
-        putDummyRecords(ignite, 100);
-        waitingForEvt.set(true); //flag for skipping regular log() and rollOver()
+        int totalEntries = 0;
 
-        log.info("Wait for archiving segment for inactive grid started");
+        while (reqSegments.getCount() > 0) {
+            int write = 500;
 
-        boolean recordedAfterSleep =
-            archiveSegmentForInactivity.await(archiveIncompleteSegmentAfterInactivityMs + 1001, TimeUnit.MILLISECONDS);
+            putAllDummyRecords(ignite, write);
 
-        stopGrid("node0");
-        assertTrue(recordedAfterSleep);
+            totalEntries += write;
+
+            Assert.assertTrue("Too much entries generated, but segments was not become available",
+                totalEntries < 10000);
+        }
+
+        String subfolderName = genDbSubfolderName(ignite, 0);
+
+        stopGrid();
+
+        String workDir = U.defaultWorkDirectory();
+
+        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
+
+        IteratorParametersBuilder iteratorParametersBuilder = createIteratorParametersBuilder(workDir, subfolderName);
+
+        iteratorParametersBuilder.filesOrDirs(workDir);
+
+        scanIterateAndCount(
+            factory,
+            iteratorParametersBuilder,
+            totalEntries,
+            0,
+            null,
+            null
+        );
     }
 
     /**
@@ -450,51 +452,50 @@ private boolean remove(Map m, Object key, Object val) {
      * @throws Exception if failed.
      */
     public void testTxFillWalAndExtractDataRecords() throws Exception {
-        final int cntEntries = 1000;
-        final int txCnt = 100;
+        Ignite ignite0 = startGrid();
 
-        final Ignite ignite0 = startGrid("node0");
+        ignite0.cluster().active(true);
 
-        ignite0.active(true);
+        int cntEntries = 1000;
+        int txCnt = 100;
 
-        final IgniteCache<Object, Object> entries = txPutDummyRecords(ignite0, cntEntries, txCnt);
+        IgniteCache<Object, Object> entries = txPutDummyRecords(ignite0, cntEntries, txCnt);
 
-        final Map<Object, Object> ctrlMap = new HashMap<>();    for (Cache.Entry<Object, Object> next : entries)
-                ctrlMap.put(next.getKey(), next.getValue());
+        Map<Object, Object> ctrlMap = new HashMap<>();
 
+        for (Cache.Entry<Object, Object> next : entries)
+            ctrlMap.put(next.getKey(), next.getValue());
 
-        final String subfolderName = genDbSubfolderName(ignite0, 0);
-        stopGrid("node0");
+        String subfolderName = genDbSubfolderName(ignite0, 0);
 
-        final String workDir = U.defaultWorkDirectory();
-        final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
-        final File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
-        final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false);
+        stopGrid();
 
-        final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log,
-            PAGE_SIZE,
-            binaryMetaWithConsId,
-            marshallerMapping);
+        String workDir = U.defaultWorkDirectory();
 
-        final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object, Object>() {
-            @Override public void apply(Object key, Object val) {
-                boolean rmv = remove(ctrlMap, key, val);
-                if (!rmv)
-                    log.error("Unable to remove Key and value from control Map K:[" + key + "] V: [" + val + "]");
+        IteratorParametersBuilder params = createIteratorParametersBuilder(workDir,subfolderName);
 
-                if (val instanceof IndexedObject) {
-                    IndexedObject indexedObj = (IndexedObject)val;
+        params.filesOrDirs(workDir);
 
-                    assertEquals(indexedObj.iVal, indexedObj.jVal);
-                    assertEquals(indexedObj.iVal, key);
+        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
 
-                    for (byte datum : indexedObj.getData())
-                        assertTrue(datum >= 'A' && datum <= 'A' + 10);
-                }
+        IgniteBiInClosure<Object, Object> objConsumer = (key, val) -> {
+            boolean rmv = remove(ctrlMap, key, val);
+
+            if (!rmv)
+                log.error("Unable to remove Key and value from control Map K:[" + key + "] V: [" + val + "]");
+
+            if (val instanceof IndexedObject) {
+                IndexedObject indexedObj = (IndexedObject)val;
+
+                assertEquals(indexedObj.iVal, indexedObj.jVal);
+                assertEquals(indexedObj.iVal, key);
+
+                for (byte datum : indexedObj.getData())
+                    assertTrue(datum >= 'A' && datum <= 'A' + 10);
             }
         };
 
-        scanIterateAndCount(factory, workDir, subfolderName, cntEntries, txCnt, objConsumer, null);
+        scanIterateAndCount(factory, params, cntEntries, txCnt, objConsumer, null);
 
         assertTrue(" Control Map is not empty after reading entries " + ctrlMap, ctrlMap.isEmpty());
     }
@@ -514,8 +515,6 @@ public void testTxFillWalAndExtractDataRecords() throws Exception {
      * Scan WAL and WAL archive for logical records and its entries.
      *
      * @param factory WAL iterator factory.
-     * @param workDir Ignite work directory.
-     * @param subfolderName DB subfolder name based on consistent ID.
      * @param minCntEntries minimum expected entries count to find.
      * @param minTxCnt minimum expected transaction count to find.
      * @param objConsumer object handler, called for each object found in logical data records.
@@ -523,67 +522,43 @@ public void testTxFillWalAndExtractDataRecords() throws Exception {
      * @throws IgniteCheckedException if failed.
      */
     private void scanIterateAndCount(
-        final IgniteWalIteratorFactory factory,
-        final String workDir,
-        final String subfolderName,
-        final int minCntEntries,
-        final int minTxCnt,
-        @Nullable final IgniteBiInClosure<Object, Object> objConsumer,
-        @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException {
-
-        final File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
-        final File wal = new File(db, "wal");
-        final File walArchive = new File(wal, "archive");
-
-        final File walArchiveDirWithConsistentId = new File(walArchive, subfolderName);
-
-        final File[] files = walArchiveDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
-        A.notNull(files, "Can't iterate over files [" + walArchiveDirWithConsistentId + "] Directory is N/A");
-        final WALIterator iter = factory.iteratorArchiveFiles(files);
+        IgniteWalIteratorFactory factory,
+        IteratorParametersBuilder itParamBuilder,
+        int minCntEntries,
+        int minTxCnt,
+        @Nullable IgniteBiInClosure<Object, Object> objConsumer,
+        @Nullable IgniteInClosure<DataRecord> dataRecordHnd
+    ) throws IgniteCheckedException {
+        WALIterator iter = factory.iterator(itParamBuilder);
 
-        final Map<GridCacheVersion, Integer> cntArch = iterateAndCountDataRecord(iter, objConsumer, dataRecordHnd);
+        Map<GridCacheVersion, Integer> cntArch = iterateAndCountDataRecord(iter, objConsumer, dataRecordHnd);
 
         int txCntObservedArch = cntArch.size();
-        if (cntArch.containsKey(null))
-            txCntObservedArch -= 1; // exclude non transactional updates
-        final int entriesArch = valuesSum(cntArch.values());
 
-        log.info("Total tx found loaded using archive directory (file-by-file): " + txCntObservedArch);
+        if (cntArch.containsKey(null))
+            txCntObservedArch -= 1; // Exclude non transactional updates.
 
-        final File walWorkDirWithNodeSubDir = new File(wal, subfolderName);
-        final File[] workFiles = walWorkDirWithNodeSubDir.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
+        int entries = valuesSum(cntArch.values());
 
-        final WALIterator tuples = factory.iteratorWorkFiles(workFiles);
-        final Map<GridCacheVersion, Integer> cntWork = iterateAndCountDataRecord(tuples, objConsumer, dataRecordHnd);
-        int txCntObservedWork = cntWork.size();
-        if (cntWork.containsKey(null))
-            txCntObservedWork -= 1; // exclude non transactional updates
+        log.info("Total tx found loaded using archive directory (file-by-file): " + txCntObservedArch);
 
-        final int entriesWork = valuesSum(cntWork.values());
-        log.info("Archive directory: Tx found " + txCntObservedWork + " entries " + entriesWork);
+        assertTrue("txCntObservedArch=" + txCntObservedArch + " >= minTxCnt=" + minTxCnt,
+            txCntObservedArch >= minTxCnt);
 
-        assertTrue("entriesArch=" + entriesArch + " + entriesWork=" + entriesWork
-                + " >= minCntEntries=" + minCntEntries,
-            entriesArch + entriesWork >= minCntEntries);
-        assertTrue("txCntObservedWork=" + txCntObservedWork + " + txCntObservedArch=" + txCntObservedArch
-                + " >= minTxCnt=" + minTxCnt,
-            txCntObservedWork + txCntObservedArch >= minTxCnt);
+        assertTrue("entries=" + entries + " >= minCntEntries=" + minCntEntries,
+            entries >= minCntEntries);
     }
 
     /**
      * @throws Exception if failed.
      */
     public void testFillWalWithDifferentTypes() throws Exception {
-        int cntEntries;
+        Ignite ig = startGrid();
 
-        final Map<Object, Object> ctrlMap = new HashMap<>();
-        final Map<Object, Object> ctrlMapForBinaryObjects = new HashMap<>();
-        final Collection<String> ctrlStringsToSearch = new HashSet<>();
-        final Collection<String> ctrlStringsForBinaryObjSearch = new HashSet<>();
-        final Ignite ignite0 = startGrid("node0");
-        ignite0.active(true);
+        ig.cluster().active(true);
+
+        IgniteCache<Object, Object> addlCache = ig.getOrCreateCache(CACHE_ADDL_NAME);
 
-        final IgniteCache<Object, Object> addlCache = ignite0.getOrCreateCache(CACHE_ADDL_NAME);
         addlCache.put("1", "2");
         addlCache.put(1, 2);
         addlCache.put(1L, 2L);
@@ -597,200 +572,173 @@ public void testFillWalWithDifferentTypes() throws Exception {
         addlCache.put(new TestExternalizable(42), "Externalizable_As_Key");
         addlCache.put(292, new IndexedObject(292));
 
-        final String search1 = "SomeUnexpectedStringValueAsKeyToSearch";
+        String search1 = "SomeUnexpectedStringValueAsKeyToSearch";
+
+        Collection<String> ctrlStringsToSearch = new HashSet<>();
+
         ctrlStringsToSearch.add(search1);
+
+        Collection<String> ctrlStringsForBinaryObjSearch = new HashSet<>();
+
         ctrlStringsForBinaryObjSearch.add(search1);
+
         addlCache.put(search1, "SearchKey");
 
         String search2 = "SomeTestStringContainerToBePrintedLongLine";
-        final TestStringContainerToBePrinted val = new TestStringContainerToBePrinted(search2);
+
+        TestStringContainerToBePrinted val = new TestStringContainerToBePrinted(search2);
+
         ctrlStringsToSearch.add(val.toString()); //will validate original toString() was called
         ctrlStringsForBinaryObjSearch.add(search2);
+
         addlCache.put("SearchValue", val);
 
         String search3 = "SomeTestStringContainerToBePrintedLongLine2";
-        final TestStringContainerToBePrinted key = new TestStringContainerToBePrinted(search3);
+
+        TestStringContainerToBePrinted key = new TestStringContainerToBePrinted(search3);
         ctrlStringsToSearch.add(key.toString()); //will validate original toString() was called
         ctrlStringsForBinaryObjSearch.add(search3); //validate only string itself
+
         addlCache.put(key, "SearchKey");
 
-            cntEntries = addlCache.size();
-            for (Cache.Entry<Object, Object> next : addlCache)
-                ctrlMap.put(next.getKey(), next.getValue());
+        int cntEntries = addlCache.size();
 
-            for (Cache.Entry<Object, Object> next : addlCache)
-                ctrlMapForBinaryObjects.put(next.getKey(), next.getValue());
+        Map<Object, Object> ctrlMap = new HashMap<>();
 
+        for (Cache.Entry<Object, Object> next : addlCache)
+            ctrlMap.put(next.getKey(), next.getValue());
 
-        final String subfolderName = genDbSubfolderName(ignite0, 0);
+        Map<Object, Object> ctrlMapForBinaryObjects = new HashMap<>();
 
-        stopGrid("node0");
+        for (Cache.Entry<Object, Object> next : addlCache)
+            ctrlMapForBinaryObjects.put(next.getKey(), next.getValue());
 
-        final String workDir = U.defaultWorkDirectory();
+        String subfolderName = genDbSubfolderName(ig, 0);
 
-        final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
-        final File binaryMetaWithNodeSubfolder = new File(binaryMeta, subfolderName);
-        final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false);
+        // Wait async allocation wal segment file by archiver.
+        Thread.sleep(1000);
 
-        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
-        final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object, Object>() {
-            @Override public void apply(Object key, Object val) {
-                log.info("K: [" + key + ", " +
-                    (key != null ? key.getClass().getName() : "?") + "]" +
-                    " V: [" + val + ", " +
-                    (val != null ? val.getClass().getName() : "?") + "]");
-                boolean rmv = remove(ctrlMap, key, val);
-                if (!rmv) {
-                    String msg = "Unable to remove pair from control map " + "K: [" + key + "] V: [" + val + "]";
-                    log.error(msg);
-                }
-                assertFalse(val instanceof BinaryObject);
+        stopGrid("node0", false);
+
+        String workDir = U.defaultWorkDirectory();
+
+        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
+
+        IteratorParametersBuilder params0 = createIteratorParametersBuilder(workDir, subfolderName);
+
+        params0.filesOrDirs(workDir);
+
+        IgniteBiInClosure<Object, Object> objConsumer = (key12, val1) -> {
+            log.info("K: [" + key12 + ", " +
+                (key12 != null ? key12.getClass().getName() : "?") + "]" +
+                " V: [" + val1 + ", " +
+                (val1 != null ? val1.getClass().getName() : "?") + "]");
+            boolean rmv = remove(ctrlMap, key12, val1);
+            if (!rmv) {
+                String msg = "Unable to remove pair from control map " + "K: [" + key12 + "] V: [" + val1 + "]";
+                log.error(msg);
             }
+            assertFalse(val1 instanceof BinaryObject);
         };
 
-        final IgniteInClosure<DataRecord> toStrChecker = new IgniteInClosure<DataRecord>() {
-            @Override public void apply(DataRecord record) {
-                String strRepresentation = record.toString();
-                for (Iterator<String> iter = ctrlStringsToSearch.iterator(); iter.hasNext(); ) {
-                    final String next = iter.next();
-                    if (strRepresentation.contains(next)) {
-                        iter.remove();
-                        break;
-                    }
+        IgniteInClosure<DataRecord> toStrChecker = record -> {
+            String strRepresentation = record.toString();
+
+            for (Iterator<String> iter = ctrlStringsToSearch.iterator(); iter.hasNext(); ) {
+                final String next = iter.next();
+                if (strRepresentation.contains(next)) {
+                    iter.remove();
+                    break;
                 }
             }
         };
-        scanIterateAndCount(factory, workDir, subfolderName, cntEntries, 0, objConsumer, toStrChecker);
+
+        scanIterateAndCount(factory, params0, cntEntries, 0, objConsumer, toStrChecker);
 
         assertTrue(" Control Map is not empty after reading entries: " + ctrlMap, ctrlMap.isEmpty());
         assertTrue(" Control Map for strings in entries is not empty after" +
             " reading records: " + ctrlStringsToSearch, ctrlStringsToSearch.isEmpty());
 
-        //Validate same WAL log with flag binary objects only
-        final IgniteWalIteratorFactory keepBinFactory = new IgniteWalIteratorFactory(log, PAGE_SIZE,
-            binaryMetaWithNodeSubfolder,
-            marshallerMapping,
-            true);
-        final IgniteBiInClosure<Object, Object> binObjConsumer = new IgniteBiInClosure<Object, Object>() {
-            @Override public void apply(Object key, Object val) {
-                log.info("K(KeepBinary): [" + key + ", " +
-                    (key != null ? key.getClass().getName() : "?") + "]" +
-                    " V(KeepBinary): [" + val + ", " +
-                    (val != null ? val.getClass().getName() : "?") + "]");
-                boolean rmv = remove(ctrlMapForBinaryObjects, key, val);
-                if (!rmv) {
-                    if (key instanceof BinaryObject) {
-                        BinaryObject keyBinObj = (BinaryObject)key;
-                        String binaryObjTypeName = keyBinObj.type().typeName();
-                        if (Objects.equals(TestStringContainerToBePrinted.class.getName(), binaryObjTypeName)) {
-                            String data = keyBinObj.field("data");
-                            rmv = ctrlMapForBinaryObjects.remove(new TestStringContainerToBePrinted(data)) != null;
-                        }
-                        else if (Objects.equals(TestSerializable.class.getName(), binaryObjTypeName)) {
-                            Integer iVal = keyBinObj.field("iVal");
-                            rmv = ctrlMapForBinaryObjects.remove(new TestSerializable(iVal)) != null;
-                        }
-                        else if (Objects.equals(TestEnum.class.getName(), binaryObjTypeName)) {
-                            TestEnum key1 = TestEnum.values()[keyBinObj.enumOrdinal()];
-                            rmv = ctrlMapForBinaryObjects.remove(key1) != null;
-                        }
+        IgniteBiInClosure<Object, Object> binObjConsumer = (key13, val12) -> {
+            log.info("K(KeepBinary): [" + key13 + ", " +
+                (key13 != null ? key13.getClass().getName() : "?") + "]" +
+                " V(KeepBinary): [" + val12 + ", " +
+                (val12 != null ? val12.getClass().getName() : "?") + "]");
+
+            boolean rmv = remove(ctrlMapForBinaryObjects, key13, val12);
+
+            if (!rmv) {
+                if (key13 instanceof BinaryObject) {
+                    BinaryObject keyBinObj = (BinaryObject)key13;
+                    String binaryObjTypeName = keyBinObj.type().typeName();
+
+                    if (Objects.equals(TestStringContainerToBePrinted.class.getName(), binaryObjTypeName)) {
+                        String data = keyBinObj.field("data");
+                        rmv = ctrlMapForBinaryObjects.remove(new TestStringContainerToBePrinted(data)) != null;
                     }
-                    else if (val instanceof BinaryObject) {
-                        //don't compare BO values, just remove by key
-                        rmv = ctrlMapForBinaryObjects.remove(key) != null;
+                    else if (Objects.equals(TestSerializable.class.getName(), binaryObjTypeName)) {
+                        Integer iVal = keyBinObj.field("iVal");
+                        rmv = ctrlMapForBinaryObjects.remove(new TestSerializable(iVal)) != null;
                     }
-                }
-                if (!rmv)
-                    log.error("Unable to remove pair from control map " + "K: [" + key + "] V: [" + val + "]");
-
-                if (val instanceof BinaryObject) {
-                    BinaryObject binaryObj = (BinaryObject)val;
-                    String binaryObjTypeName = binaryObj.type().typeName();
-                    if (Objects.equals(IndexedObject.class.getName(), binaryObjTypeName)) {
-                        assertEquals(binaryObj.field("iVal").toString(),
-                            binaryObj.field("jVal").toString());
-
-                        byte data[] = binaryObj.field("data");
-                        for (byte datum : data)
-                            assertTrue(datum >= 'A' && datum <= 'A' + 10);
+                    else if (Objects.equals(TestEnum.class.getName(), binaryObjTypeName)) {
+                        TestEnum key1 = TestEnum.values()[keyBinObj.enumOrdinal()];
+                        rmv = ctrlMapForBinaryObjects.remove(key1) != null;
                     }
                 }
+                else if (val12 instanceof BinaryObject) {
+                    //don't compare BO values, just remove by key
+                    rmv = ctrlMapForBinaryObjects.remove(key13) != null;
+                }
             }
-        };
+            if (!rmv)
+                log.error("Unable to remove pair from control map " + "K: [" + key13 + "] V: [" + val12 + "]");
 
-        final IgniteInClosure<DataRecord> binObjToStrChecker = new IgniteInClosure<DataRecord>() {
-            @Override public void apply(DataRecord record) {
-                String strRepresentation = record.toString();
+            if (val12 instanceof BinaryObject) {
+                BinaryObject binaryObj = (BinaryObject)val12;
+                String binaryObjTypeName = binaryObj.type().typeName();
 
-                for (Iterator<String> iter = ctrlStringsForBinaryObjSearch.iterator(); iter.hasNext(); ) {
-                    final String next = iter.next();
+                if (Objects.equals(IndexedObject.class.getName(), binaryObjTypeName)) {
+                    assertEquals(
+                        binaryObj.field("iVal").toString(),
+                        binaryObj.field("jVal").toString()
+                    );
 
-                    if (strRepresentation.contains(next)) {
-                        iter.remove();
+                    byte data[] = binaryObj.field("data");
 
-                        break;
-                    }
+                    for (byte datum : data)
+                        assertTrue(datum >= 'A' && datum <= 'A' + 10);
                 }
             }
         };
 
-        scanIterateAndCount(keepBinFactory, workDir, subfolderName, cntEntries, 0, binObjConsumer, binObjToStrChecker);
-
-        assertTrue(" Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects,
-            ctrlMapForBinaryObjects.isEmpty());
-        assertTrue(" Control Map for strings in entries is not empty after" +
-                " reading records: " + ctrlStringsForBinaryObjSearch,
-            ctrlStringsForBinaryObjSearch.isEmpty());
-
-    }
-
-    /**
-     * Tests archive completed event is fired.
-     *
-     * @throws Exception if failed.
-     */
-    public void testFillWalForExactSegmentsCount() throws Exception {
-        customWalMode = WALMode.FSYNC;
-
-        final CountDownLatch reqSegments = new CountDownLatch(15);
-        final Ignite ignite = startGrid("node0");
-
-        ignite.active(true);
+        IgniteInClosure<DataRecord> binObjToStrChecker = record -> {
+            String strRepresentation = record.toString();
 
-        final IgniteEvents evts = ignite.events();
-
-        if (!evts.isEnabled(EVT_WAL_SEGMENT_ARCHIVED))
-            assertTrue("nothing to test", false);
+            for (Iterator<String> iter = ctrlStringsForBinaryObjSearch.iterator(); iter.hasNext(); ) {
+                final String next = iter.next();
 
-        evts.localListen(new IgnitePredicate<Event>() {
-            @Override public boolean apply(Event e) {
-                WalSegmentArchivedEvent archComplEvt = (WalSegmentArchivedEvent)e;
-                long idx = archComplEvt.getAbsWalSegmentIdx();
-                log.info("Finished archive for segment [" + idx + ", " +
-                    archComplEvt.getArchiveFile() + "]: [" + e + "]");
+                if (strRepresentation.contains(next)) {
+                    iter.remove();
 
-                reqSegments.countDown();
-                return true;
+                    break;
+                }
             }
-        }, EVT_WAL_SEGMENT_ARCHIVED);
+        };
 
+        IteratorParametersBuilder params1 = createIteratorParametersBuilder(workDir, subfolderName);
 
-        int totalEntries = 0;
-        while (reqSegments.getCount() > 0) {
-            final int write = 500;
-            putAllDummyRecords(ignite, write);
-            totalEntries += write;
-            Assert.assertTrue("Too much entries generated, but segments was not become available",
-                totalEntries < 10000);
-        }
-        final String subfolderName = genDbSubfolderName(ignite, 0);
+        params1.filesOrDirs(workDir).keepBinary(true);
 
-        stopGrid("node0");
+        //Validate same WAL log with flag binary objects only
+        IgniteWalIteratorFactory keepBinFactory = new IgniteWalIteratorFactory(log);
 
-        final String workDir = U.defaultWorkDirectory();
-        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+        scanIterateAndCount(keepBinFactory, params1, cntEntries, 0, binObjConsumer, binObjToStrChecker);
 
-        scanIterateAndCount(factory, workDir, subfolderName, totalEntries, 0, null, null);
+        assertTrue(" Control Map is not empty after reading entries: " +
+            ctrlMapForBinaryObjects, ctrlMapForBinaryObjects.isEmpty());
+
+        assertTrue(" Control Map for strings in entries is not empty after" +
+                " reading records: " + ctrlStringsForBinaryObjSearch, ctrlStringsForBinaryObjSearch.isEmpty());
     }
 
     /**
@@ -801,19 +749,32 @@ public void testFillWalForExactSegmentsCount() throws Exception {
     public void testReadEmptyWal() throws Exception {
         customWalMode = WALMode.FSYNC;
 
-        final Ignite ignite = startGrid("node0");
+        Ignite ignite = startGrid();
 
-        ignite.active(true);
-        ignite.active(false);
+        ignite.cluster().active(true);
+
+        ignite.cluster().active(false);
 
         final String subfolderName = genDbSubfolderName(ignite, 0);
 
-        stopGrid("node0");
+        stopGrid();
+
+        String workDir = U.defaultWorkDirectory();
+
+        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
 
-        final String workDir = U.defaultWorkDirectory();
-        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+        IteratorParametersBuilder iteratorParametersBuilder =
+            createIteratorParametersBuilder(workDir, subfolderName)
+            .filesOrDirs(workDir);
 
-        scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, null);
+        scanIterateAndCount(
+            factory,
+            iteratorParametersBuilder,
+            0,
+            0,
+            null,
+            null
+        );
     }
 
     /**
@@ -872,54 +833,72 @@ public void testRemoveOperationPresentedForDataEntryForAtomic() throws Exception
      * @param mode Cache Atomicity Mode.
      */
     private void runRemoveOperationTest(CacheAtomicityMode mode) throws Exception {
-        final Ignite ignite = startGrid("node0");
+        Ignite ignite = startGrid();
+
+        ignite.cluster().active(true);
 
-        ignite.active(true);
         createCache2(ignite, mode);
-        ignite.active(false);
 
-        final String subfolderName = genDbSubfolderName(ignite, 0);
+        ignite.cluster().active(false);
+
+        String subfolderName = genDbSubfolderName(ignite, 0);
+
+        stopGrid();
+
+        String workDir = U.defaultWorkDirectory();
+
+        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
 
-        stopGrid("node0");
+        IteratorParametersBuilder params = createIteratorParametersBuilder(workDir, subfolderName);
 
-        final String workDir = U.defaultWorkDirectory();
-        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+        params.filesOrDirs(workDir);
 
-        final StringBuilder builder = new StringBuilder();
-        final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
+        StringBuilder sb = new StringBuilder();
 
-        scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new IgniteInClosure<DataRecord>() {
-            @Override public void apply(DataRecord dataRecord) {
+        Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
+
+        scanIterateAndCount(
+            factory,
+            params,
+            0,
+            0,
+            null,
+            dataRecord -> {
                 final List<DataEntry> entries = dataRecord.writeEntries();
 
-                builder.append("{");
+                sb.append("{");
+
                 for (DataEntry entry : entries) {
-                    final GridCacheOperation op = entry.op();
-                    final Integer cnt = operationsFound.get(op);
+                    GridCacheOperation op = entry.op();
+                    Integer cnt = operationsFound.get(op);
 
                     operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
 
                     if (entry instanceof UnwrapDataEntry) {
-                        final UnwrapDataEntry entry1 = (UnwrapDataEntry)entry;
+                        UnwrapDataEntry entry1 = (UnwrapDataEntry)entry;
 
-                        builder.append(entry1.op()).append(" for ").append(entry1.unwrappedKey());
-                        final GridCacheVersion ver = entry.nearXidVersion();
+                        sb.append(entry1.op())
+                            .append(" for ")
+                            .append(entry1.unwrappedKey());
 
-                        builder.append(", ");
+                        GridCacheVersion ver = entry.nearXidVersion();
+
+                        sb.append(", ");
 
                         if (ver != null)
-                            builder.append("tx=").append(ver).append(", ");
+                            sb.append("tx=")
+                                .append(ver)
+                                .append(", ");
                     }
                 }
 
-                builder.append("}\n");
-            }
-        });
+                sb.append("}\n");
+            });
 
         final Integer deletesFound = operationsFound.get(DELETE);
 
         if (log.isInfoEnabled())
-            log.info(builder.toString());
+            log.info(sb.toString());
 
         assertTrue("Delete operations should be found in log: " + operationsFound,
             deletesFound != null && deletesFound > 0);
@@ -927,109 +906,144 @@ private void runRemoveOperationTest(CacheAtomicityMode mode) throws Exception {
 
     /**
      * Tests transaction generation and WAL for putAll cache operation.
+     *
      * @throws Exception if failed.
      */
     public void testPutAllTxIntoTwoNodes() throws Exception {
-        final Ignite ignite = startGrid("node0");
-        final Ignite ignite1 = startGrid(1);
+        Ignite ignite = startGrid("node0");
+        Ignite ignite1 = startGrid(1);
+
+        ignite.cluster().active(true);
 
-        ignite.active(true);
+        Map<Object, IndexedObject> map = new TreeMap<>();
 
-        final Map<Object, IndexedObject> map = new TreeMap<>();
+        int cntEntries = 1000;
 
-        final int cntEntries = 1000;
         for (int i = 0; i < cntEntries; i++)
             map.put(i, new IndexedObject(i));
 
         ignite.cache(CACHE_NAME).putAll(map);
 
-        ignite.active(false);
+        ignite.cluster().active(false);
 
-        final String subfolderName = genDbSubfolderName(ignite, 0);
-        final String subfolderName1 = genDbSubfolderName(ignite1, 1);
+        String subfolderName1 = genDbSubfolderName(ignite, 0);
+        String subfolderName2 = genDbSubfolderName(ignite1, 1);
 
         stopAllGrids();
 
-        final String workDir = U.defaultWorkDirectory();
-        final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+        String workDir = U.defaultWorkDirectory();
 
-        final StringBuilder builder = new StringBuilder();
-        final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
+        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log);
 
-        final IgniteInClosure<DataRecord> drHnd = new IgniteInClosure<DataRecord>() {
-            @Override public void apply(DataRecord dataRecord) {
-                final List<DataEntry> entries = dataRecord.writeEntries();
+        StringBuilder sb = new StringBuilder();
 
-                builder.append("{");
-                for (DataEntry entry : entries) {
-                    final GridCacheOperation op = entry.op();
-                    final Integer cnt = operationsFound.get(op);
+        Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
 
-                    operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
+        IgniteInClosure<DataRecord> drHnd = dataRecord -> {
+            List<DataEntry> entries = dataRecord.writeEntries();
 
-                    if (entry instanceof UnwrapDataEntry) {
-                        final UnwrapDataEntry entry1 = (UnwrapDataEntry)entry;
+            sb.append("{");
 
-                        builder.append(entry1.op()).append(" for ").append(entry1.unwrappedKey());
-                        final GridCacheVersion ver = entry.nearXidVersion();
+            for (DataEntry entry : entries) {
+                GridCacheOperation op = entry.op();
+                Integer cnt = operationsFound.get(op);
 
-                        builder.append(", ");
+                operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
 
-                        if (ver != null)
-                            builder.append("tx=").append(ver).append(", ");
-                    }
-                }
+                if (entry instanceof UnwrapDataEntry) {
+                    final UnwrapDataEntry entry1 = (UnwrapDataEntry)entry;
+
+                    sb.append(entry1.op()).append(" for ").append(entry1.unwrappedKey());
+                    final GridCacheVersion ver = entry.nearXidVersion();
+
+                    sb.append(", ");
 
-                builder.append("}\n");
+                    if (ver != null)
+                        sb.append("tx=").append(ver).append(", ");
+                }
             }
+
+            sb.append("}\n");
         };
-        scanIterateAndCount(factory, workDir, subfolderName, 1, 1, null, drHnd);
-        scanIterateAndCount(factory, workDir, subfolderName1, 1, 1, null, drHnd);
 
-        final Integer createsFound = operationsFound.get(CREATE);
+        scanIterateAndCount(
+            factory,
+            createIteratorParametersBuilder(workDir, subfolderName1)
+                .filesOrDirs(
+                    workDir + "/db/wal/" + subfolderName1,
+                    workDir + "/db/wal/archive/" + subfolderName1
+                ),
+            1,
+            1,
+            null, drHnd
+        );
+
+        scanIterateAndCount(
+            factory,
+            createIteratorParametersBuilder(workDir, subfolderName2)
+                .filesOrDirs(
+                    workDir + "/db/wal/" + subfolderName2,
+                    workDir + "/db/wal/archive/" + subfolderName2
+                ),
+            1,
+            1,
+            null,
+            drHnd
+        );
+
+        Integer createsFound = operationsFound.get(CREATE);
 
         if (log.isInfoEnabled())
-            log.info(builder.toString());
+            log.info(sb.toString());
 
         assertTrue("Create operations should be found in log: " + operationsFound,
             createsFound != null && createsFound > 0);
 
         assertTrue("Create operations count should be at least " + cntEntries + " in log: " + operationsFound,
             createsFound != null && createsFound >= cntEntries);
-
     }
 
     /**
      * Tests transaction generation and WAL for putAll cache operation.
+     *
      * @throws Exception if failed.
      */
     public void testTxRecordsReadWoBinaryMeta() throws Exception {
         clearProperties = true;
+
         System.setProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS, "true");
 
-        final Ignite ignite = startGrid("node0");
-        ignite.active(true);
+        Ignite ignite = startGrid("node0");
 
-        final Map<Object, IndexedObject> map = new TreeMap<>();
+        ignite.cluster().active(true);
+
+        Map<Object, IndexedObject> map = new TreeMap<>();
 
         for (int i = 0; i < 1000; i++)
             map.put(i, new IndexedObject(i));
 
         ignite.cache(CACHE_NAME).putAll(map);
 
-        ignite.active(false);
+        ignite.cluster().active(false);
+
+        String workDir = U.defaultWorkDirectory();
+
+        String subfolderName = genDbSubfolderName(ignite, 0);
 
-        final String workDir = U.defaultWorkDirectory();
-        final String subfolderName = genDbSubfolderName(ignite, 0);
         stopAllGrids();
 
-        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(),
-            PAGE_SIZE,
-            null,
-            null,
-            false);
+        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger());
+
+        IteratorParametersBuilder params = createIteratorParametersBuilder(workDir, subfolderName);
 
-        scanIterateAndCount(factory, workDir, subfolderName, 1000, 1, null, null);
+        scanIterateAndCount(
+            factory,
+            params.filesOrDirs(workDir),
+            1000,
+            1,
+            null,
+            null
+        );
     }
 
     /**
@@ -1038,19 +1052,17 @@ public void testTxRecordsReadWoBinaryMeta() throws Exception {
      * @return WAL iterator factory.
      * @throws IgniteCheckedException If failed.
      */
-    @NotNull private IgniteWalIteratorFactory createWalIteratorFactory(
-        final String workDir,
-        final String subfolderName
+    @NotNull private IteratorParametersBuilder createIteratorParametersBuilder(
+        String workDir,
+        String subfolderName
     ) throws IgniteCheckedException {
-        final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
-        final File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
-        final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false);
-
-        return new IgniteWalIteratorFactory(log,
-            PAGE_SIZE,
-            binaryMetaWithConsId,
-            marshallerMapping,
-            false);
+        File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
+        File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
+        File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false);
+
+        return new IteratorParametersBuilder()
+            .binaryMetadataFileStoreDir(binaryMetaWithConsId)
+            .marshallerMappingFileStoreDir(marshallerMapping);
     }
 
     /**
@@ -1074,28 +1086,33 @@ private int valuesSum(Iterable<Integer> values) {
      * @throws IgniteCheckedException if failure.
      */
     private Map<GridCacheVersion, Integer> iterateAndCountDataRecord(
-        final WALIterator walIter,
-        @Nullable final IgniteBiInClosure<Object, Object> cacheObjHnd,
-        @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException {
+        WALIterator walIter,
+        @Nullable IgniteBiInClosure<Object, Object> cacheObjHnd,
+        @Nullable IgniteInClosure<DataRecord> dataRecordHnd
+    ) throws IgniteCheckedException {
 
-        final Map<GridCacheVersion, Integer> entriesUnderTxFound = new HashMap<>();
+        Map<GridCacheVersion, Integer> entriesUnderTxFound = new HashMap<>();
 
         try (WALIterator stIt = walIter) {
             while (stIt.hasNextX()) {
-                final IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
-                final WALRecord walRecord = next.get2();
+                IgniteBiTuple<WALPointer, WALRecord> tup = stIt.nextX();
 
-                if (walRecord.type() == WALRecord.RecordType.DATA_RECORD && walRecord instanceof DataRecord) {
-                    final DataRecord dataRecord = (DataRecord)walRecord;
+                WALRecord walRecord = tup.get2();
+
+                if (walRecord.type() == DATA_RECORD && walRecord instanceof DataRecord) {
+                    DataRecord dataRecord = (DataRecord)walRecord;
 
                     if (dataRecordHnd != null)
                         dataRecordHnd.apply(dataRecord);
-                    final List<DataEntry> entries = dataRecord.writeEntries();
+
+                    List<DataEntry> entries = dataRecord.writeEntries();
 
                     for (DataEntry entry : entries) {
-                        final GridCacheVersion globalTxId = entry.nearXidVersion();
+                        GridCacheVersion globalTxId = entry.nearXidVersion();
+
                         Object unwrappedKeyObj;
                         Object unwrappedValObj;
+
                         if (entry instanceof UnwrapDataEntry) {
                             UnwrapDataEntry unwrapDataEntry = (UnwrapDataEntry)entry;
                             unwrappedKeyObj = unwrapDataEntry.unwrappedKey();
@@ -1116,7 +1133,7 @@ else if (entry instanceof LazyDataEntry) {
                             unwrappedKeyObj = key instanceof BinaryObject ? key : key.value(null, false);
                         }
 
-                        if (dumpRecords)
+                        if (DUMP_RECORDS)
                             log.info("//Entry operation " + entry.op() + "; cache Id" + entry.cacheId() + "; " +
                                 "under transaction: " + globalTxId +
                                 //; entry " + entry +
@@ -1126,23 +1143,78 @@ else if (entry instanceof LazyDataEntry) {
                         if (cacheObjHnd != null && (unwrappedKeyObj != null || unwrappedValObj != null))
                             cacheObjHnd.apply(unwrappedKeyObj, unwrappedValObj);
 
-                        final Integer entriesUnderTx = entriesUnderTxFound.get(globalTxId);
+                        Integer entriesUnderTx = entriesUnderTxFound.get(globalTxId);
+
                         entriesUnderTxFound.put(globalTxId, entriesUnderTx == null ? 1 : entriesUnderTx + 1);
                     }
                 }
-                else if (walRecord.type() == WALRecord.RecordType.TX_RECORD && walRecord instanceof TxRecord) {
-                    final TxRecord txRecord = (TxRecord)walRecord;
-                    final GridCacheVersion globalTxId = txRecord.nearXidVersion();
+                else if (walRecord.type() == TX_RECORD && walRecord instanceof TxRecord) {
+                    TxRecord txRecord = (TxRecord)walRecord;
+                    GridCacheVersion globalTxId = txRecord.nearXidVersion();
 
-                    if (dumpRecords)
+                    if (DUMP_RECORDS)
                         log.info("//Tx Record, state: " + txRecord.state() +
                             "; nearTxVersion" + globalTxId);
                 }
             }
         }
+
         return entriesUnderTxFound;
     }
 
+    /**
+     * Puts provided number of records to fill WAL.
+     *
+     * @param ignite ignite instance.
+     * @param recordsToWrite count.
+     */
+    private void putDummyRecords(Ignite ignite, int recordsToWrite) {
+        IgniteCache<Object, Object> cache0 = ignite.cache(CACHE_NAME);
+
+        for (int i = 0; i < recordsToWrite; i++)
+            cache0.put(i, new IndexedObject(i));
+    }
+
+    /**
+     * Puts provided number of records to fill WAL.
+     *
+     * @param ignite ignite instance.
+     * @param recordsToWrite count.
+     */
+    private void putAllDummyRecords(Ignite ignite, int recordsToWrite) {
+        IgniteCache<Object, Object> cache0 = ignite.cache(CACHE_NAME);
+
+        Map<Object, Object> values = new HashMap<>();
+
+        for (int i = 0; i < recordsToWrite; i++)
+            values.put(i, new IndexedObject(i));
+
+        cache0.putAll(values);
+    }
+
+    /**
+     * Puts provided number of records to fill WAL under transactions.
+     *
+     * @param ignite ignite instance.
+     * @param recordsToWrite count.
+     * @param txCnt transactions to run. If number is less then records count, txCnt records will be written.
+     */
+    private IgniteCache<Object, Object> txPutDummyRecords(Ignite ignite, int recordsToWrite, int txCnt) {
+        IgniteCache<Object, Object> cache0 = ignite.cache(CACHE_NAME);
+        int keysPerTx = recordsToWrite / txCnt;
+        if (keysPerTx == 0)
+            keysPerTx = 1;
+        for (int t = 0; t < txCnt; t++) {
+            try (Transaction tx = ignite.transactions().txStart()) {
+                for (int i = t * keysPerTx; i < (t + 1) * keysPerTx; i++)
+                    cache0.put(i, new IndexedObject(i));
+
+                tx.commit();
+            }
+        }
+        return cache0;
+    }
+
     /** Enum for cover binaryObject enum save/load. */
     enum TestEnum {
         /** */A, /** */B, /** */C
@@ -1257,7 +1329,7 @@ static class TestStringContainerToBePrinted {
          *
          * @param data value to be searched in to String
          */
-        public TestStringContainerToBePrinted(String data) {
+        TestStringContainerToBePrinted(String data) {
             this.data = data;
         }
 
@@ -1297,7 +1369,7 @@ private static class Organization {
          * @param key Key.
          * @param name Name.
          */
-        public Organization(int key, String name) {
+        Organization(int key, String name) {
             this.key = key;
             this.name = name;
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index 0188445463d32..0a240ea6ae36f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -150,4 +150,9 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
     @Override public int walArchiveSegments() {
         return 0;
     }
+
+    /** {@inheritDoc} */
+    @Override public long lastArchivedSegment() {
+        return -1L;
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 7732cee672799..52a32ed9abe35 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -51,6 +51,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlyWithMmapBufferSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFormatFileFailoverTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorExceptionDuringReadTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalIteratorSwitchSegmentTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalSerializerVersionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalCompactionTest;
@@ -171,5 +172,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(LocalWalModeChangeDuringRebalancingSelfTest.class);
 
         suite.addTestSuite(IgniteWalIteratorSwitchSegmentTest.class);
+
+        suite.addTestSuite(IgniteWalIteratorExceptionDuringReadTest.class);
     }
 }
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
index 2da1aa30fadcb..eee193ab34965 100644
--- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
@@ -54,11 +54,7 @@ public static void main(String[] args) throws Exception {
         boolean printRecords = IgniteSystemProperties.getBoolean("PRINT_RECORDS", false);
         boolean printStat = IgniteSystemProperties.getBoolean("PRINT_STAT", true);
 
-        final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(),
-            Integer.parseInt(args[0]),
-            null,
-            null,
-            false);
+        final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger());
 
         final File walWorkDirWithConsistentId = new File(args[1]);
 
@@ -69,7 +65,7 @@ public static void main(String[] args) throws Exception {
 
         @Nullable final WalStat stat = printStat ? new WalStat() : null;
 
-        try (WALIterator stIt = factory.iteratorWorkFiles(workFiles)) {
+        try (WALIterator stIt = factory.iterator(workFiles)) {
             while (stIt.hasNextX()) {
                 IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
 
@@ -87,7 +83,7 @@ public static void main(String[] args) throws Exception {
         if (args.length >= 3) {
             final File walArchiveDirWithConsistentId = new File(args[2]);
 
-            try (WALIterator stIt = factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId)) {
+            try (WALIterator stIt = factory.iterator(walArchiveDirWithConsistentId)) {
                 while (stIt.hasNextX()) {
                     IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
 

From cb80440f68ce43237505269043dd643a5af340c7 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Thu, 28 Jun 2018 17:29:20 +0300
Subject: [PATCH 228/543] IGNITE-8857 HashMap is returned instead of filtering
 wrapper. - Fixes #4258.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>

(cherry-picked from commit#e1a3398b509a16f0db3b60751d0ef52e1b763a87)
---
 .../zk/internal/ZookeeperClusterNode.java     |  4 ++--
 .../internal/ZookeeperDiscoverySpiTest.java   | 22 +++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
index 2fe3052fa9d23..02e412326f3a9 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
@@ -243,11 +243,11 @@ public void setCacheMetrics(Map<Integer, CacheMetrics> cacheMetrics) {
     /** {@inheritDoc} */
     @Override public Map<String, Object> attributes() {
         // Even though discovery SPI removes this attribute after authentication, keep this check for safety.
-        return F.view(attrs, new IgnitePredicate<String>() {
+        return new HashMap<>(F.view(attrs, new IgnitePredicate<String>() {
             @Override public boolean apply(String s) {
                 return !IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS.equals(s);
             }
-        });
+        }));
     }
 
     /** {@inheritDoc} */
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index 1585870053b87..92706bf98220d 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -470,6 +470,28 @@ private void checkInternalStructuresCleanup() throws Exception {
         }
     }
 
+    /**
+     * Verifies that node attributes returned through public API are presented in standard form.
+     *
+     * It means there is no exotic classes that may unnecessary capture other classes from the context.
+     *
+     * For more information about the problem refer to
+     * <a href="https://issues.apache.org/jira/browse/IGNITE-8857">IGNITE-8857</a>.
+     */
+    public void testNodeAttributesNotReferencingZookeeperClusterNode() throws Exception {
+        userAttrs = new HashMap<>();
+        userAttrs.put("testAttr", "testAttr");
+
+        try {
+            IgniteEx ignite = startGrid(0);
+
+            assertTrue(ignite.cluster().localNode().attributes() instanceof HashMap);
+        }
+        finally {
+            userAttrs = null;
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From 590b97938443f44cafde839b879da715b9ce1a49 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Mon, 2 Jul 2018 14:36:25 +0300
Subject: [PATCH 229/543] IGNITE-8857 new IgnitePredicate filtering credential
 attribute introduced, HashMap was removed - Fixes #4272.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry-picked from commit#a67b08cc0e76dfd1ee3810972bfe6781698550d4)
---
 ...ecurityCredentialsAttrFilterPredicate.java | 39 +++++++++++++++++++
 .../zk/internal/ZookeeperClusterNode.java     |  8 +---
 .../internal/ZookeeperDiscoverySpiTest.java   | 14 ++++++-
 3 files changed, 54 insertions(+), 7 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/SecurityCredentialsAttrFilterPredicate.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/SecurityCredentialsAttrFilterPredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/SecurityCredentialsAttrFilterPredicate.java
new file mode 100644
index 0000000000000..2f774bdcf4d32
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/SecurityCredentialsAttrFilterPredicate.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgnitePredicate;
+
+/**
+ * Predicate to filter out security credentials attribute by its name.
+ */
+public class SecurityCredentialsAttrFilterPredicate implements IgnitePredicate<String> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override public boolean apply(String s) {
+        return !IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS.equals(s);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(SecurityCredentialsAttrFilterPredicate.class, this);
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
index 02e412326f3a9..1c2a589d934dc 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
@@ -31,11 +31,11 @@
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.internal.ClusterMetricsSnapshot;
 import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.SecurityCredentialsAttrFilterPredicate;
 import org.apache.ignite.internal.managers.discovery.IgniteClusterNode;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.spi.discovery.DiscoveryMetricsProvider;
 import org.jetbrains.annotations.Nullable;
@@ -243,11 +243,7 @@ public void setCacheMetrics(Map<Integer, CacheMetrics> cacheMetrics) {
     /** {@inheritDoc} */
     @Override public Map<String, Object> attributes() {
         // Even though discovery SPI removes this attribute after authentication, keep this check for safety.
-        return new HashMap<>(F.view(attrs, new IgnitePredicate<String>() {
-            @Override public boolean apply(String s) {
-                return !IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS.equals(s);
-            }
-        }));
+        return F.view(attrs, new SecurityCredentialsAttrFilterPredicate());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index 92706bf98220d..fbacebfd5dff7 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -74,6 +74,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.SecurityCredentialsAttrFilterPredicate;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.managers.discovery.CustomEventListener;
@@ -91,6 +92,7 @@
 import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.lang.IgniteInClosure2X;
+import org.apache.ignite.internal.util.lang.gridfunc.PredicateMapView;
 import org.apache.ignite.internal.util.nio.GridCommunicationClient;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.T3;
@@ -485,7 +487,17 @@ public void testNodeAttributesNotReferencingZookeeperClusterNode() throws Except
         try {
             IgniteEx ignite = startGrid(0);
 
-            assertTrue(ignite.cluster().localNode().attributes() instanceof HashMap);
+            Map<String, Object> attrs = ignite.cluster().localNode().attributes();
+
+            assertTrue(attrs instanceof PredicateMapView);
+
+            IgnitePredicate[] preds = GridTestUtils.getFieldValue(attrs, "preds");
+
+            assertNotNull(preds);
+
+            assertTrue(preds.length == 1);
+
+            assertTrue(preds[0] instanceof SecurityCredentialsAttrFilterPredicate);
         }
         finally {
             userAttrs = null;

From a3a08312a0476fd7d2b4eb3773911a8d4b8cf865 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <Plehanov.Alex@gmail.com>
Date: Mon, 2 Jul 2018 14:47:13 +0300
Subject: [PATCH 230/543] IGNITE-8203 Handle ClosedByInterruptionException in
 FilePageStore - Fixes #4211.

---
 .../cache/persistence/file/FilePageStore.java | 341 ++++++++++++-----
 .../IgnitePdsTaskCancelingTest.java           | 352 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 3 files changed, 610 insertions(+), 86 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index f614032c58b15..2d9e37f3b50a9 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -21,6 +21,8 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.channels.ClosedByInterruptException;
+import java.nio.channels.ClosedChannelException;
 import java.nio.file.Files;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReadWriteLock;
@@ -67,7 +69,7 @@ public class FilePageStore implements PageStore {
     private final FileIOFactory ioFactory;
 
     /** I/O interface for read/write operations with file */
-    private FileIO fileIO;
+    private volatile FileIO fileIO;
 
     /** */
     private final AtomicLong allocated;
@@ -159,74 +161,77 @@ public ByteBuffer header(byte type, int pageSize) {
      * @return Next available position in the file to store a data.
      * @throws IOException If initialization is failed.
      */
-    private long initFile() throws IOException {
-        ByteBuffer hdr = header(type, dbCfg.getPageSize());
+    private long initFile(FileIO fileIO) throws IOException {
+        try {
+            ByteBuffer hdr = header(type, dbCfg.getPageSize());
 
-        while (hdr.remaining() > 0)
-            fileIO.write(hdr);
+            while (hdr.remaining() > 0)
+                fileIO.write(hdr);
+
+            //there is 'super' page in every file
+            return headerSize() + dbCfg.getPageSize();
+        }
+        catch (ClosedByInterruptException e) {
+            // If thread was interrupted written header can be inconsistent.
+            Files.delete(cfgFile.toPath());
 
-        //there is 'super' page in every file
-        return headerSize() + dbCfg.getPageSize();
+            throw e;
+        }
     }
 
     /**
      * Checks that file store has correct header and size.
      *
      * @return Next available position in the file to store a data.
-     * @throws PersistentStorageIOException If check is failed.
+     * @throws IOException If check is failed.
      */
-    private long checkFile() throws PersistentStorageIOException {
-        try {
-            ByteBuffer hdr = ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN);
+    private long checkFile(FileIO fileIO) throws IOException {
+        ByteBuffer hdr = ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN);
 
-            while (hdr.remaining() > 0)
-                fileIO.read(hdr);
+        while (hdr.remaining() > 0)
+            fileIO.read(hdr);
 
-            hdr.rewind();
+        hdr.rewind();
 
-            long signature = hdr.getLong();
+        long signature = hdr.getLong();
 
-            if (SIGNATURE != signature)
-                throw new IOException("Failed to verify store file (invalid file signature)" +
-                    " [expectedSignature=" + U.hexLong(SIGNATURE) +
-                    ", actualSignature=" + U.hexLong(signature) + ']');
+        if (SIGNATURE != signature)
+            throw new IOException("Failed to verify store file (invalid file signature)" +
+                " [expectedSignature=" + U.hexLong(SIGNATURE) +
+                ", actualSignature=" + U.hexLong(signature) + ']');
 
-            int ver = hdr.getInt();
+        int ver = hdr.getInt();
 
-            if (version() != ver)
-                throw new IOException("Failed to verify store file (invalid file version)" +
-                    " [expectedVersion=" + version() +
-                    ", fileVersion=" + ver + "]");
+        if (version() != ver)
+            throw new IOException("Failed to verify store file (invalid file version)" +
+                " [expectedVersion=" + version() +
+                ", fileVersion=" + ver + "]");
 
-            byte type = hdr.get();
+        byte type = hdr.get();
 
-            if (this.type != type)
-                throw new IOException("Failed to verify store file (invalid file type)" +
-                    " [expectedFileType=" + this.type +
-                    ", actualFileType=" + type + "]");
+        if (this.type != type)
+            throw new IOException("Failed to verify store file (invalid file type)" +
+                " [expectedFileType=" + this.type +
+                ", actualFileType=" + type + "]");
 
-            int pageSize = hdr.getInt();
+        int pageSize = hdr.getInt();
 
-            if (dbCfg.getPageSize() != pageSize)
-                throw new IOException("Failed to verify store file (invalid page size)" +
-                    " [expectedPageSize=" + dbCfg.getPageSize() +
-                    ", filePageSize=" + pageSize + "]");
+        if (dbCfg.getPageSize() != pageSize)
+            throw new IOException("Failed to verify store file (invalid page size)" +
+                " [expectedPageSize=" + dbCfg.getPageSize() +
+                ", filePageSize=" + pageSize + "]");
 
-            long fileSize = cfgFile.length();
+        long fileSize = cfgFile.length();
 
-            if (fileSize == headerSize()) // Every file has a special meta page.
-                fileSize = pageSize + headerSize();
+        if (fileSize == headerSize()) // Every file has a special meta page.
+            fileSize = pageSize + headerSize();
 
-            if ((fileSize - headerSize()) % pageSize != 0)
-                throw new IOException("Failed to verify store file (invalid file size)" +
-                    " [fileSize=" + U.hexLong(fileSize) +
-                    ", pageSize=" + U.hexLong(pageSize) + ']');
+        if ((fileSize - headerSize()) % pageSize != 0)
+            throw new IOException("Failed to verify store file (invalid file size)" +
+                " [fileSize=" + U.hexLong(fileSize) +
+                ", pageSize=" + U.hexLong(pageSize) + ']');
 
-            return fileSize;
-        }
-        catch (IOException e) {
-            throw new PersistentStorageIOException("File check failed", e);
-        }
+        return fileSize;
     }
 
     /**
@@ -244,6 +249,8 @@ public void stop(boolean cleanFile) throws PersistentStorageIOException {
 
             fileIO.close();
 
+            fileIO = null;
+
             if (cleanFile)
                 Files.delete(cfgFile.toPath());
         }
@@ -273,6 +280,8 @@ public void truncate(int tag) throws PersistentStorageIOException {
 
             fileIO.close();
 
+            fileIO = null;
+
             Files.delete(cfgFile.toPath());
         }
         catch (IOException e) {
@@ -348,7 +357,7 @@ public void finishRecover() throws PersistentStorageIOException {
             int len = pageSize;
 
             do {
-                int n = fileIO.read(pageBuf, off);
+                int n = readWithFailover(pageBuf, off);
 
                 // If page was not written yet, nothing to read.
                 if (n < 0) {
@@ -403,7 +412,7 @@ public void finishRecover() throws PersistentStorageIOException {
             long off = 0;
 
             do {
-                int n = fileIO.read(buf, off);
+                int n = readWithFailover(buf, off);
 
                 // If page was not written yet, nothing to read.
                 if (n < 0)
@@ -433,10 +442,28 @@ private void init() throws PersistentStorageIOException {
 
                     PersistentStorageIOException err = null;
 
+                    long newSize;
+
                     try {
-                        this.fileIO = fileIO = ioFactory.create(cfgFile, CREATE, READ, WRITE);
+                        boolean interrupted = false;
+
+                        while (true) {
+                            try {
+                                this.fileIO = fileIO = ioFactory.create(cfgFile, CREATE, READ, WRITE);
+
+                                newSize = cfgFile.length() == 0 ? initFile(fileIO) : checkFile(fileIO);
 
-                        long newSize = cfgFile.length() == 0 ? initFile() : checkFile();
+                                if (interrupted)
+                                    Thread.currentThread().interrupt();
+
+                                break;
+                            }
+                            catch (ClosedByInterruptException e) {
+                                interrupted = true;
+
+                                Thread.interrupted();
+                            }
+                        }
 
                         assert allocated.get() == 0;
 
@@ -469,59 +496,158 @@ private void init() throws PersistentStorageIOException {
         }
     }
 
-    /** {@inheritDoc} */
-    @Override public void write(long pageId, ByteBuffer pageBuf, int tag, boolean calculateCrc) throws IgniteCheckedException {
-        init();
+    /**
+     * Reinit page store after file channel was closed by thread interruption.
+     *
+     * @param fileIO Old fileIO.
+     */
+    private void reinit(FileIO fileIO) throws IOException {
+        if (!inited)
+            return;
 
-        lock.readLock().lock();
+        if (fileIO != this.fileIO)
+            return;
+
+        lock.writeLock().lock();
 
         try {
-            if (tag < this.tag)
+            if (fileIO != this.fileIO)
                 return;
 
-            long off = pageOffset(pageId);
+            try {
+                boolean interrupted = false;
 
-            assert (off >= 0 && off + headerSize() <= allocated.get() ) || recover :
-                "off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) + ", pageId=" + U.hexLong(pageId);
+                while (true) {
+                    try {
+                        fileIO = null;
 
-            assert pageBuf.capacity() == pageSize;
-            assert pageBuf.position() == 0;
-            assert pageBuf.order() == ByteOrder.nativeOrder() : "Page buffer order " + pageBuf.order()
-                + " should be same with " + ByteOrder.nativeOrder();
-            assert PageIO.getType(pageBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(pageId);
-            assert PageIO.getVersion(pageBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(pageId);
+                        fileIO = ioFactory.create(cfgFile, CREATE, READ, WRITE);
+
+                        checkFile(fileIO);
 
-            if (calculateCrc && !skipCrc) {
-                assert PageIO.getCrc(pageBuf) == 0 : U.hexLong(pageId);
+                        this.fileIO = fileIO;
 
-                PageIO.setCrc(pageBuf, calcCrc32(pageBuf, pageSize));
+                        if (interrupted)
+                            Thread.currentThread().interrupt();
+
+                        break;
+                    }
+                    catch (ClosedByInterruptException e) {
+                        interrupted = true;
+
+                        Thread.interrupted();
+                    }
+                }
             }
+            catch (IOException e) {
+                try {
+                    if (fileIO != null)
+                        fileIO.close();
+                }
+                catch (IOException e0) {
+                    e.addSuppressed(e0);
+                }
 
-            // Check whether crc was calculated somewhere above the stack if it is forcibly skipped.
-            assert skipCrc || PageIO.getCrc(pageBuf) != 0 || calcCrc32(pageBuf, pageSize) == 0 :
-                    "CRC hasn't been calculated, crc=0";
+                throw e;
+            }
+        }
+        finally {
+            lock.writeLock().unlock();
+        }
+    }
 
-            assert pageBuf.position() == 0 : pageBuf.position();
+    /** {@inheritDoc} */
+    @Override public void write(long pageId, ByteBuffer pageBuf, int tag, boolean calculateCrc) throws IgniteCheckedException {
+        init();
 
-            int len = pageSize;
+        boolean interrupted = false;
 
-            do {
-                int n = fileIO.write(pageBuf, off);
+        while (true) {
+            FileIO fileIO = this.fileIO;
 
-                off += n;
+            try {
+                lock.readLock().lock();
 
-                len -= n;
+                try {
+                    if (tag < this.tag)
+                        return;
+
+                    long off = pageOffset(pageId);
+
+                    assert (off >= 0 && off + headerSize() <= allocated.get()) || recover :
+                        "off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) + ", pageId=" + U.hexLong(pageId);
+
+                    assert pageBuf.capacity() == pageSize;
+                    assert pageBuf.position() == 0;
+                    assert pageBuf.order() == ByteOrder.nativeOrder() : "Page buffer order " + pageBuf.order()
+                        + " should be same with " + ByteOrder.nativeOrder();
+                    assert PageIO.getType(pageBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(pageId);
+                    assert PageIO.getVersion(pageBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(pageId);
+
+                    if (calculateCrc && !skipCrc) {
+                        assert PageIO.getCrc(pageBuf) == 0 : U.hexLong(pageId);
+
+                        PageIO.setCrc(pageBuf, calcCrc32(pageBuf, pageSize));
+                    }
+
+                    // Check whether crc was calculated somewhere above the stack if it is forcibly skipped.
+                    assert skipCrc || PageIO.getCrc(pageBuf) != 0 || calcCrc32(pageBuf, pageSize) == 0 :
+                        "CRC hasn't been calculated, crc=0";
+
+                    assert pageBuf.position() == 0 : pageBuf.position();
+
+                    int len = pageSize;
+
+                    if (fileIO == null)
+                        throw new IOException("FileIO has stopped");
+
+                    do {
+                        int n = fileIO.write(pageBuf, off);
+
+                        off += n;
+
+                        len -= n;
+                    }
+                    while (len > 0);
+
+                    PageIO.setCrc(pageBuf, 0);
+
+                    if (interrupted)
+                        Thread.currentThread().interrupt();
+
+                    return;
+                }
+                finally {
+                    lock.readLock().unlock();
+                }
             }
-            while (len > 0);
+            catch (IOException e) {
+                if (e instanceof ClosedChannelException) {
+                    try {
+                        if (e instanceof ClosedByInterruptException) {
+                            interrupted = true;
 
-            PageIO.setCrc(pageBuf, 0);
-        }
-        catch (IOException e) {
-            throw new PersistentStorageIOException("Failed to write the page to the file store [pageId=" + pageId +
-                ", file=" + cfgFile.getAbsolutePath() + ']', e);
-        }
-        finally {
-            lock.readLock().unlock();
+                            Thread.interrupted();
+                        }
+
+                        reinit(fileIO);
+
+                        pageBuf.position(0);
+
+                        PageIO.setCrc(pageBuf, 0);
+
+                        continue;
+                    }
+                    catch (IOException e0) {
+                        e0.addSuppressed(e);
+
+                        e = e0;
+                    }
+                }
+
+                throw new PersistentStorageIOException("Failed to write the page to the file store [pageId=" + pageId
+                    + ", file=" + cfgFile.getAbsolutePath() + ']', e);
+            }
         }
     }
 
@@ -552,7 +678,10 @@ private static int calcCrc32(ByteBuffer pageBuf, int pageSize) {
         try {
             init();
 
-            fileIO.force();
+            FileIO fileIO = this.fileIO;
+
+            if (fileIO != null)
+                fileIO.force();
         }
         catch (IOException e) {
             throw new PersistentStorageIOException("Sync error", e);
@@ -603,4 +732,44 @@ private long allocPage() {
 
         return (int)((allocated.get() - headerSize()) / pageSize);
     }
+
+    /**
+     * @param destBuf Destination buffer.
+     * @param position Position.
+     * @return Number of read bytes.
+     */
+    private int readWithFailover(ByteBuffer destBuf, long position) throws IOException {
+        boolean interrupted = false;
+
+        int bufPos = destBuf.position();
+
+        while (true) {
+            FileIO fileIO = this.fileIO;
+
+            if (fileIO == null)
+                throw new IOException("FileIO has stopped");
+
+            try {
+                assert destBuf.remaining() > 0;
+
+                int bytesRead = fileIO.read(destBuf, position);
+
+                if (interrupted)
+                    Thread.currentThread().interrupt();
+
+                return bytesRead;
+            }
+            catch (ClosedChannelException e) {
+                destBuf.position(bufPos);
+
+                if (e instanceof ClosedByInterruptException) {
+                    interrupted = true;
+
+                    Thread.interrupted();
+                }
+
+                reinit(fileIO);
+            }
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java
new file mode 100644
index 0000000000000..d42b7885add49
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.OpenOption;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.LockSupport;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCountDownLatch;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+/**
+ * Test handle of task canceling with PDS enabled.
+ */
+public class IgnitePdsTaskCancelingTest extends GridCommonAbstractTest {
+    /** Slow file IO enabled. */
+    private static final AtomicBoolean slowFileIoEnabled = new AtomicBoolean(false);
+
+    /** Node failure occurs. */
+    private static final AtomicBoolean failure = new AtomicBoolean(false);
+
+    /** Number of executing tasks. */
+    private static final int NUM_TASKS = 16;
+
+    /** Page size. */
+    private static final int PAGE_SIZE = 2048;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setFailureHandler(new FailureHandler() {
+            @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+                failure.set(true);
+
+                return true;
+            }
+        });
+
+        cfg.setCacheConfiguration(new CacheConfiguration().setName(DEFAULT_CACHE_NAME).setAffinity(
+            new RendezvousAffinityFunction(false, NUM_TASKS / 2)
+        ));
+
+        cfg.setDataStorageConfiguration(getDataStorageConfiguration());
+
+        return cfg;
+    }
+
+    /**
+     * Default data storage configuration.
+     */
+    private DataStorageConfiguration getDataStorageConfiguration() {
+        DataStorageConfiguration dbCfg = new DataStorageConfiguration();
+
+        dbCfg.setPageSize(PAGE_SIZE);
+
+        dbCfg.setFileIOFactory(new SlowIOFactory());
+
+        dbCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+            .setMaxSize(100 * 1024 * 1024)
+            .setPersistenceEnabled(true));
+
+        return dbCfg;
+    }
+
+    /**
+     * Checks that tasks canceling does not lead to node failure.
+     */
+    public void testFailNodesOnCanceledTask() throws Exception {
+        cleanPersistenceDir();
+
+        failure.set(false);
+
+        try {
+            Ignite ig0 = startGrids(4);
+
+            ig0.cluster().active(true);
+
+            Collection<IgniteFuture> cancelFutures = new ArrayList<>(NUM_TASKS);
+
+            IgniteCountDownLatch latch = ig0.countDownLatch("latch", NUM_TASKS, false, true);
+
+            for (int i = 0; i < NUM_TASKS; i++) {
+                final Integer key = i;
+
+                cancelFutures.add(ig0.compute().affinityRunAsync(DEFAULT_CACHE_NAME, key,
+                    new IgniteRunnable() {
+                        @IgniteInstanceResource
+                        Ignite ig;
+
+                        @Override public void run() {
+                            latch.countDown();
+
+                            latch.await();
+
+                            ig.cache(DEFAULT_CACHE_NAME).put(key, new byte[1024]);
+                        }
+                    }));
+            }
+
+            slowFileIoEnabled.set(true);
+
+            latch.await();
+
+            for (IgniteFuture future: cancelFutures)
+                future.cancel();
+
+            slowFileIoEnabled.set(false);
+
+            for (int i = 0; i < NUM_TASKS; i++) {
+                final Integer key = i;
+
+                ig0.compute().affinityRun(DEFAULT_CACHE_NAME, key,
+                    new IgniteRunnable() {
+                        @IgniteInstanceResource
+                        Ignite ig;
+
+                        @Override public void run() {
+                            ig.cache(DEFAULT_CACHE_NAME).put(key, new byte[1024]);
+                        }
+                    });
+            }
+
+            assertFalse(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+                @Override public boolean apply() {
+                    return failure.get();
+                }
+            }, 5_000L));
+        }
+        finally {
+            stopAllGrids();
+
+            cleanPersistenceDir();
+        }
+    }
+
+    /**
+     * Test FilePageStore with multiple interrupted threads.
+     */
+    public void testFilePageStoreInterruptThreads() throws Exception {
+        failure.set(false);
+
+        FileIOFactory factory = new RandomAccessFileIOFactory();
+
+        File file = new File(U.defaultWorkDirectory(), "file.bin");
+
+        file.deleteOnExit();
+
+        DataStorageConfiguration dbCfg = getDataStorageConfiguration();
+
+        FilePageStore pageStore = new FilePageStore(PageMemory.FLAG_DATA, file, factory, dbCfg,
+            new AllocatedPageTracker() {
+                @Override public void updateTotalAllocatedPages(long delta) {
+                    // No-op.
+                }
+            });
+
+        int pageSize = dbCfg.getPageSize();
+
+        PageIO pageIO = PageIO.getPageIO(PageIO.T_DATA, 1);
+
+        long ptr = GridUnsafe.allocateMemory(NUM_TASKS * pageSize);
+
+        try {
+            List<Thread> threadList = new ArrayList<>(NUM_TASKS);
+
+            AtomicBoolean stopThreads = new AtomicBoolean(false);
+
+            for (int i = 0; i < NUM_TASKS; i++) {
+                long pageId = PageIdUtils.pageId(0, PageMemory.FLAG_DATA, (int)pageStore.allocatePage());
+
+                long pageAdr = ptr + i * pageSize;
+
+                pageIO.initNewPage(pageAdr, pageId, pageSize);
+
+                ByteBuffer buf = GridUnsafe.wrapPointer(pageAdr, pageSize);
+
+                pageStore.write(pageId, buf, 0, true);
+
+                threadList.add(new Thread(new Runnable() {
+                    @Override public void run() {
+                        Random random = new Random();
+
+                        while (!stopThreads.get()) {
+                            buf.position(0);
+
+                            try {
+                                if (random.nextBoolean()) {
+                                    log.info(">>> Read page " + U.hexLong(pageId));
+
+                                    pageStore.read(pageId, buf, false);
+                                }
+                                else {
+                                    log.info(">>> Write page " + U.hexLong(pageId));
+
+                                    pageStore.write(pageId, buf, 0, true);
+                                }
+
+                                Thread.interrupted();
+                            }
+                            catch (Exception e) {
+                                log.error("Error while reading/writing page", e);
+
+                                failure.set(true);
+                            }
+                        }
+                    }
+                }));
+            }
+
+            for (Thread thread : threadList)
+                thread.start();
+
+            for (int i = 0; i < 10; i++) {
+                for (Thread thread : threadList) {
+                    doSleep(10L);
+
+                    log.info("Interrupting " + thread.getName());
+
+                    thread.interrupt();
+                }
+            }
+
+            stopThreads.set(true);
+
+            for (Thread thread : threadList)
+                thread.join();
+
+            assertFalse(failure.get());
+        }
+        finally {
+            GridUnsafe.freeMemory(ptr);
+        }
+    }
+
+    /**
+     * Decorated FileIOFactory with slow IO operations.
+     */
+    private static class SlowIOFactory implements FileIOFactory {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            return create(file, CREATE, READ, WRITE);
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... openOption) throws IOException {
+            final FileIO delegate = delegateFactory.create(file, openOption);
+
+            final boolean slow = file.getName().contains(".bin");
+
+            return new FileIODecorator(delegate) {
+                @Override public int write(ByteBuffer srcBuf) throws IOException {
+                    parkForAWhile();
+
+                    return super.write(srcBuf);
+                }
+
+                @Override public int write(ByteBuffer srcBuf, long position) throws IOException {
+                    parkForAWhile();
+
+                    return super.write(srcBuf, position);
+                }
+
+                @Override public int write(byte[] buf, int off, int len) throws IOException {
+                    parkForAWhile();
+
+                    return super.write(buf, off, len);
+                }
+
+                @Override public int read(ByteBuffer destBuf) throws IOException {
+                    parkForAWhile();
+
+                    return super.read(destBuf);
+                }
+
+                @Override public int read(ByteBuffer destBuf, long position) throws IOException {
+                    parkForAWhile();
+
+                    return super.read(destBuf, position);
+                }
+
+                @Override public int read(byte[] buf, int off, int len) throws IOException {
+                    parkForAWhile();
+
+                    return super.read(buf, off, len);
+                }
+
+                private void parkForAWhile() {
+                    if(slowFileIoEnabled.get() && slow)
+                        LockSupport.parkNanos(1_000_000_000L);
+                }
+            };
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 52a32ed9abe35..f66ff20ba8d9c 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPartitionFilesDestroyTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
 import org.apache.ignite.internal.processors.cache.persistence.LocalWalModeChangeDuringRebalancingSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAbsentEvictionNodeOutOfBaselineTest;
@@ -82,6 +83,8 @@ public static TestSuite suite() {
         suite.addTestSuite(ClientAffinityAssignmentWithBaselineTest.class);
         suite.addTestSuite(IgniteAbsentEvictionNodeOutOfBaselineTest.class);
 
+        suite.addTestSuite(IgnitePdsTaskCancelingTest.class);
+
         return suite;
     }
 

From afbad007cc7dd1d2795ab3e04ffdb580763ca99d Mon Sep 17 00:00:00 2001
From: Dmitriy Sorokin <d.w.sorokin@gmail.com>
Date: Wed, 4 Jul 2018 13:16:21 +0300
Subject: [PATCH 231/543] IGNITE-8910 PagesList.takeEmptyPage may fail with
 AssertionError: type = 1 - Fixes #4294.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit e6f44ad)
---
 .../cache/persistence/freelist/PagesList.java    | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
index 78dc91f512c57..f8400ca7430fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
@@ -469,8 +469,12 @@ private boolean updateTail(int bucket, long oldTailId, long newTailId) {
                 else
                     newTails = null; // Drop the bucket completely.
 
-                if (casBucket(bucket, tails, newTails))
+                if (casBucket(bucket, tails, newTails)) {
+                    // Reset tailId for invalidation of locking when stripe was taken concurrently.
+                    tails[idx].tailId = 0L;
+
                     return true;
+                }
             }
             else {
                 // It is safe to assign new tail since we do it only when write lock on tail is held.
@@ -622,6 +626,11 @@ protected final void put(
                 return;
 
             final long tailId = stripe.tailId;
+
+            // Stripe was removed from bucket concurrently.
+            if (tailId == 0L)
+                continue;
+
             final long tailPage = acquirePage(tailId);
 
             try {
@@ -1035,6 +1044,11 @@ protected final long takeEmptyPage(int bucket, @Nullable IOVersions initIoVers)
                 return 0L;
 
             final long tailId = stripe.tailId;
+
+            // Stripe was removed from bucket concurrently.
+            if (tailId == 0L)
+                continue;
+
             final long tailPage = acquirePage(tailId);
 
             try {

From 84ce1a3452fdd0195c3f7c291141a54ced76b125 Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Wed, 4 Jul 2018 17:15:28 +0300
Subject: [PATCH 232/543] IGNITE-8780 File I/O operations must be retried if
 buffer hasn't read/written completely

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../GridCacheDatabaseSharedManager.java       |  19 +-
 .../persistence/file/AbstractFileIO.java      | 146 +++++++++
 .../cache/persistence/file/AsyncFileIO.java   |   2 +-
 .../cache/persistence/file/FileIO.java        |  76 +++++
 .../persistence/file/FileIODecorator.java     |   2 +-
 .../cache/persistence/file/FilePageStore.java |  20 +-
 .../file/FileVersionCheckingFactory.java      |   3 +-
 .../persistence/file/RandomAccessFileIO.java  |   2 +-
 .../cache/persistence/file/UnzipFileIO.java   |   2 +-
 .../cache/persistence/wal/FileWALPointer.java |   3 +
 .../wal/FileWriteAheadLogManager.java         |  46 +--
 .../FsyncModeFileWriteAheadLogManager.java    |  34 +-
 ...itePdsRecoveryAfterFileCorruptionTest.java |   2 +-
 ...alModeChangeDuringRebalancingSelfTest.java |   3 +-
 .../db/wal/crc/IgniteDataIntegrityTests.java  |   6 +-
 .../file/AlignedBuffersDirectFileIO.java      |   2 +-
 .../persistence/file/IgniteFileIOTest.java    | 304 ++++++++++++++++++
 17 files changed, 572 insertions(+), 100 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AbstractFileIO.java
 create mode 100644 modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteFileIOTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index a410010f9896a..ce78d656d93da 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -537,7 +537,7 @@ private List<CheckpointEntry> retreiveHistory() throws IgniteCheckedException {
         ) {
             List<CheckpointEntry> checkpoints = new ArrayList<>();
 
-            ByteBuffer buf = ByteBuffer.allocate(16);
+            ByteBuffer buf = ByteBuffer.allocate(FileWALPointer.POINTER_SIZE);
             buf.order(ByteOrder.nativeOrder());
 
             for (Path cpFile : cpFiles) {
@@ -875,7 +875,7 @@ private void nodeStart(WALPointer ptr) throws IgniteCheckedException {
         String fileName = U.currentTimeMillis() + NODE_STARTED_FILE_NAME_SUFFIX;
         String tmpFileName = fileName + FILE_TMP_SUFFIX;
 
-        ByteBuffer buf = ByteBuffer.allocate(20);
+        ByteBuffer buf = ByteBuffer.allocate(FileWALPointer.POINTER_SIZE);
         buf.order(ByteOrder.nativeOrder());
 
         try {
@@ -889,7 +889,7 @@ private void nodeStart(WALPointer ptr) throws IgniteCheckedException {
 
                 buf.flip();
 
-                io.write(buf);
+                io.writeFully(buf);
 
                 buf.clear();
 
@@ -919,7 +919,7 @@ public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedExce
             cpDir.toPath(),
             path -> path.toFile().getName().endsWith(NODE_STARTED_FILE_NAME_SUFFIX))
         ) {
-            ByteBuffer buf = ByteBuffer.allocate(20);
+            ByteBuffer buf = ByteBuffer.allocate(FileWALPointer.POINTER_SIZE);
             buf.order(ByteOrder.nativeOrder());
 
             for (Path path : nodeStartedFiles) {
@@ -930,7 +930,7 @@ public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedExce
                 Long ts = Long.valueOf(name.substring(0, name.length() - NODE_STARTED_FILE_NAME_SUFFIX.length()));
 
                 try (FileIO io = ioFactory.create(f, READ)) {
-                    io.read(buf);
+                    io.readFully(buf);
 
                     buf.flip();
 
@@ -1216,8 +1216,7 @@ private int resolvePageSizeFromPartitionFile(Path partFile) throws IOException,
 
             ByteBuffer hdr = ByteBuffer.allocate(minimalHdr).order(ByteOrder.LITTLE_ENDIAN);
 
-            while (hdr.remaining() > 0)
-                fileIO.read(hdr);
+            fileIO.readFully(hdr);
 
             hdr.rewind();
 
@@ -1854,7 +1853,7 @@ else if (type == CheckpointEntryType.END && ts > lastEndTs) {
             }
         }
 
-        ByteBuffer buf = ByteBuffer.allocate(20);
+        ByteBuffer buf = ByteBuffer.allocate(FileWALPointer.POINTER_SIZE);
         buf.order(ByteOrder.nativeOrder());
 
         if (startFile != null)
@@ -1880,7 +1879,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
         buf.position(0);
 
         try (FileIO io = ioFactory.create(cpMarkerFile, READ)) {
-            io.read(buf);
+            io.readFully(buf);
 
             buf.flip();
 
@@ -2629,7 +2628,7 @@ public void writeCheckpointEntry(ByteBuffer entryBuf, CheckpointEntry cp, Checkp
             try (FileIO io = ioFactory.create(Paths.get(cpDir.getAbsolutePath(), skipSync ? fileName : tmpFileName).toFile(),
                 StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
 
-                io.write(entryBuf);
+                io.writeFully(entryBuf);
 
                 entryBuf.clear();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AbstractFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AbstractFileIO.java
new file mode 100644
index 0000000000000..47236446b086f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AbstractFileIO.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public abstract class AbstractFileIO implements FileIO {
+    /** Max io timeout milliseconds. */
+    private static final int MAX_IO_TIMEOUT_MS = 2000;
+
+    /**
+     *
+     */
+    private interface IOOperation {
+        /**
+         * @param offs Offset.
+         *
+         * @return Number of bytes operated.
+         */
+        public int run(int offs) throws IOException;
+    }
+
+    /**
+     * @param operation IO operation.
+     *
+     * @param num Number of bytes to operate.
+     */
+    private int fully(IOOperation operation, int num, boolean write) throws IOException {
+        if (num > 0) {
+            long time = 0;
+
+            for (int i = 0; i < num; ) {
+                int n = operation.run(i);
+
+                if (n > 0) {
+                    i += n;
+                    time = 0;
+                }
+                else if (n == 0) {
+                    if (time == 0)
+                        time = U.currentTimeMillis();
+                    else if ((U.currentTimeMillis() - time) >= MAX_IO_TIMEOUT_MS)
+                        throw new IOException(write && position() == size() ? "Failed to extend file." :
+                            "Probably disk is too busy, please check your device.");
+                }
+                else
+                    throw new EOFException("EOF at position [" + position() + "] expected to read [" + num + "] bytes.");
+            }
+        }
+
+        return num;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int readFully(final ByteBuffer destBuf) throws IOException {
+        return fully(new IOOperation() {
+            @Override public int run(int offs) throws IOException {
+                return read(destBuf);
+            }
+        }, available(destBuf.remaining()), false);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int readFully(final ByteBuffer destBuf, final long position) throws IOException {
+        return fully(new IOOperation() {
+            @Override public int run(int offs) throws IOException {
+                return read(destBuf, position + offs);
+            }
+        }, available(destBuf.remaining(), position), false);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int readFully(final byte[] buf, final int off, final int len) throws IOException {
+        return fully(new IOOperation() {
+            @Override public int run(int offs) throws IOException {
+                return read(buf, off + offs, len - offs);
+            }
+        }, len, false);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int writeFully(final ByteBuffer srcBuf) throws IOException {
+        return fully(new IOOperation() {
+            @Override public int run(int offs) throws IOException {
+                return write(srcBuf);
+            }
+        }, srcBuf.remaining(), true);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int writeFully(final ByteBuffer srcBuf, final long position) throws IOException {
+        return fully(new IOOperation() {
+            @Override public int run(int offs) throws IOException {
+                return write(srcBuf, position + offs);
+            }
+        }, srcBuf.remaining(), true);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int writeFully(final byte[] buf, final int off, final int len) throws IOException {
+        return fully(new IOOperation() {
+            @Override public int run(int offs) throws IOException {
+                return write(buf, off + offs, len - offs);
+            }
+        }, len, true);
+    }
+
+    /**
+     * @param requested Requested.
+     */
+    private int available(int requested) throws IOException {
+        return available(requested, position());
+    }
+
+    /**
+     * @param requested Requested.
+     * @param position Position.
+     */
+    private int available(int requested, long position) throws IOException {
+        long avail = size() - position;
+
+        return requested > avail ? (int) avail : requested;
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
index 76142bb509626..fd00e255fc75f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
@@ -31,7 +31,7 @@
 /**
  * File I/O implementation based on {@link AsynchronousFileChannel}.
  */
-public class AsyncFileIO implements FileIO {
+public class AsyncFileIO extends AbstractFileIO {
     /**
      * File channel associated with {@code file}
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
index 50568aff0f14e..6f32d015ae410 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIO.java
@@ -59,6 +59,17 @@ public interface FileIO extends AutoCloseable {
      */
     public int read(ByteBuffer destBuf) throws IOException;
 
+    /**
+     * Reads a sequence of bytes from this file into the {@code destinationBuffer}.
+     *
+     * @param destBuf Destination byte buffer.
+     *
+     * @return Number of written bytes.
+     *
+     * @throws IOException If some I/O error occurs.
+     */
+    public int readFully(ByteBuffer destBuf) throws IOException;
+
     /**
      * Reads a sequence of bytes from this file into the {@code destinationBuffer}
      * starting from specified file {@code position}.
@@ -74,6 +85,19 @@ public interface FileIO extends AutoCloseable {
      */
     public int read(ByteBuffer destBuf, long position) throws IOException;
 
+    /**
+     * Reads a sequence of bytes from this file into the {@code destinationBuffer}
+     * starting from specified file {@code position}.
+     *
+     * @param destBuf Destination byte buffer.
+     * @param position Starting position of file.
+     *
+     * @return Number of written bytes.
+     *
+     * @throws IOException If some I/O error occurs.
+     */
+    public int readFully(ByteBuffer destBuf, long position) throws IOException;
+
     /**
      * Reads a up to {@code length} bytes from this file into the {@code buffer}.
      *
@@ -88,6 +112,20 @@ public interface FileIO extends AutoCloseable {
      */
     public int read(byte[] buf, int off, int len) throws IOException;
 
+    /**
+     * Reads a up to {@code length} bytes from this file into the {@code buffer}.
+     *
+     * @param buf Destination byte array.
+     * @param off The start offset in array {@code b}
+     *               at which the data is written.
+     * @param len Number of bytes read.
+     *
+     * @return Number of written bytes.
+     *
+     * @throws IOException If some I/O error occurs.
+     */
+    public int readFully(byte[] buf, int off, int len) throws IOException;
+
     /**
      * Writes a sequence of bytes to this file from the {@code sourceBuffer}.
      *
@@ -99,6 +137,17 @@ public interface FileIO extends AutoCloseable {
      */
     public int write(ByteBuffer srcBuf) throws IOException;
 
+    /**
+     * Writes a sequence of bytes to this file from the {@code sourceBuffer}.
+     *
+     * @param srcBuf Source buffer.
+     *
+     * @return Number of written bytes.
+     *
+     * @throws IOException If some I/O error occurs.
+     */
+    public int writeFully(ByteBuffer srcBuf) throws IOException;
+
     /**
      * Writes a sequence of bytes to this file from the {@code sourceBuffer}
      * starting from specified file {@code position}
@@ -112,6 +161,19 @@ public interface FileIO extends AutoCloseable {
      */
     public int write(ByteBuffer srcBuf, long position) throws IOException;
 
+    /**
+     * Writes a sequence of bytes to this file from the {@code sourceBuffer}
+     * starting from specified file {@code position}
+     *
+     * @param srcBuf Source buffer.
+     * @param position Starting file position.
+     *
+     * @return Number of written bytes.
+     *
+     * @throws IOException If some I/O error occurs.
+     */
+    public int writeFully(ByteBuffer srcBuf, long position) throws IOException;
+
     /**
      * Writes {@code length} bytes from the {@code buffer}
      * starting at offset {@code off} to this file.
@@ -126,6 +188,20 @@ public interface FileIO extends AutoCloseable {
      */
     public int write(byte[] buf, int off, int len) throws IOException;
 
+    /**
+     * Writes {@code length} bytes from the {@code buffer}
+     * starting at offset {@code off} to this file.
+     *
+     * @param buf Source byte array.
+     * @param off Start offset in the {@code buffer}.
+     * @param len Number of bytes to write.
+     *
+     * @return Number of written bytes.
+     *
+     * @throws IOException If some I/O error occurs.
+     */
+    public int writeFully(byte[] buf, int off, int len) throws IOException;
+
     /**
      * Allocates memory mapped buffer for this file with given size.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java
index 9c389851ef248..8e79b54accbb5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIODecorator.java
@@ -24,7 +24,7 @@
 /**
  * Decorator class for File I/O
  */
-public class FileIODecorator implements FileIO {
+public class FileIODecorator extends AbstractFileIO {
     /** File I/O delegate */
     private final FileIO delegate;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 2d9e37f3b50a9..d2d5506f6cb01 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -165,8 +165,7 @@ private long initFile(FileIO fileIO) throws IOException {
         try {
             ByteBuffer hdr = header(type, dbCfg.getPageSize());
 
-            while (hdr.remaining() > 0)
-                fileIO.write(hdr);
+        fileIO.writeFully(hdr);
 
             //there is 'super' page in every file
             return headerSize() + dbCfg.getPageSize();
@@ -188,8 +187,7 @@ private long initFile(FileIO fileIO) throws IOException {
     private long checkFile(FileIO fileIO) throws IOException {
         ByteBuffer hdr = ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN);
 
-        while (hdr.remaining() > 0)
-            fileIO.read(hdr);
+        fileIO.readFully(hdr);
 
         hdr.rewind();
 
@@ -596,19 +594,7 @@ private void reinit(FileIO fileIO) throws IOException {
 
                     assert pageBuf.position() == 0 : pageBuf.position();
 
-                    int len = pageSize;
-
-                    if (fileIO == null)
-                        throw new IOException("FileIO has stopped");
-
-                    do {
-                        int n = fileIO.write(pageBuf, off);
-
-                        off += n;
-
-                        len -= n;
-                    }
-                    while (len > 0);
+                    fileIO.writeFully(pageBuf, off);
 
                     PageIO.setCrc(pageBuf, 0);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
index ab36d7cbd5201..bc938a57912fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
@@ -86,8 +86,7 @@ public FileVersionCheckingFactory(FileIOFactory fileIOFactory, DataStorageConfig
 
             ByteBuffer hdr = ByteBuffer.allocate(minHdr).order(ByteOrder.LITTLE_ENDIAN);
 
-            while (hdr.remaining() > 0)
-                fileIO.read(hdr);
+            fileIO.readFully(hdr);
 
             hdr.rewind();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
index 018ed276de7ab..ef4a3df37be4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
@@ -27,7 +27,7 @@
 /**
  * File I/O implementation based on {@link FileChannel}.
  */
-public class RandomAccessFileIO implements FileIO {
+public class RandomAccessFileIO extends AbstractFileIO {
     /**
      * File channel.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
index 8194ba36eed14..6345b1fc0aab0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/UnzipFileIO.java
@@ -30,7 +30,7 @@
  * Doesn't allow random access and setting {@link FileIO#position()} backwards.
  * Allows sequential reads including setting {@link FileIO#position()} forward.
  */
-public class UnzipFileIO implements FileIO {
+public class UnzipFileIO extends AbstractFileIO {
     /** Zip input stream. */
     private final ZipInputStream zis;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java
index 0e095fa24f17f..6ea7e002b6e6d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java
@@ -28,6 +28,9 @@ public class FileWALPointer implements WALPointer, Comparable<FileWALPointer> {
     /** Serial version uid. */
     private static final long serialVersionUID = 0L;
 
+    /** Pointer serialized size. */
+    public static final int POINTER_SIZE = 16;
+
     /** Absolute WAL segment file index (incrementing counter) */
     private final long idx;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 4a696e47a7abc..44003039549b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1402,22 +1402,9 @@ private void formatFile(File file, int bytesCntToFormat) throws StorageException
         try (FileIO fileIO = ioFactory.create(file, CREATE, READ, WRITE)) {
             int left = bytesCntToFormat;
 
-            if (mode == WALMode.FSYNC) {
-                while (left > 0) {
-                    int toWrite = Math.min(FILL_BUF.length, left);
-
-                    if (fileIO.write(FILL_BUF, 0, toWrite) < toWrite) {
-                        final StorageException ex = new StorageException("Failed to extend WAL segment file: " +
-                            file.getName() + ". Probably disk is too busy, please check your device.");
-
-                        if (failureProcessor != null)
-                            failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
-
-                        throw ex;
-                    }
-
-                    left -= toWrite;
-                }
+            if (mode == WALMode.FSYNC || mmap) {
+                while ((left -= fileIO.writeFully(FILL_BUF, 0, Math.min(FILL_BUF.length, left))) > 0)
+                    ;
 
                 fileIO.force();
             }
@@ -1425,7 +1412,12 @@ private void formatFile(File file, int bytesCntToFormat) throws StorageException
                 fileIO.clear();
         }
         catch (IOException e) {
-            throw new StorageException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
+            StorageException ex = new StorageException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
+
+            if (failureProcessor != null)
+                failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
+
+            throw ex;
         }
     }
 
@@ -2168,17 +2160,8 @@ private class FileDecompressor extends GridWorker {
                          FileIO io = ioFactory.create(unzipTmp)) {
                         zis.getNextEntry();
 
-                        int bytesRead;
-                        while ((bytesRead = zis.read(arr)) > 0)
-                            if (io.write(arr, 0, bytesRead) < bytesRead) {
-                                final IgniteCheckedException ex = new IgniteCheckedException("Failed to extend file: " +
-                                    unzipTmp.getName() + ". Probably disk is too busy, please check your device.");
-
-                                if (failureProcessor != null)
-                                    failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
-
-                                throw ex;
-                            }
+                        while (io.writeFully(arr, 0, zis.read(arr)) > 0)
+                            ;
                     }
 
                     try {
@@ -3475,12 +3458,7 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, Igni
             try {
                 assert hdl.written == hdl.fileIO.position();
 
-                do {
-                    hdl.fileIO.write(buf);
-                }
-                while (buf.hasRemaining());
-
-                hdl.written += size;
+                hdl.written += hdl.fileIO.writeFully(buf);
 
                 metrics.onWalBytesWritten(size);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 5db21d25ae4a3..45cb1a72b28bc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -1223,13 +1223,8 @@ private void formatFile(File file, int bytesCntToFormat) throws StorageException
             int left = bytesCntToFormat;
 
             if (mode == WALMode.FSYNC) {
-                while (left > 0) {
-                    int toWrite = Math.min(FILL_BUF.length, left);
-
-                    fileIO.write(FILL_BUF, 0, toWrite);
-
-                    left -= toWrite;
-                }
+                while ((left -= fileIO.writeFully(FILL_BUF, 0, Math.min(FILL_BUF.length, left))) > 0)
+                    ;
 
                 fileIO.force();
             }
@@ -1972,9 +1967,8 @@ private class FileDecompressor extends GridWorker {
                          FileIO io = ioFactory.create(unzipTmp)) {
                         zis.getNextEntry();
 
-                        int bytesRead;
-                        while ((bytesRead = zis.read(arr)) > 0)
-                            io.write(arr, 0, bytesRead);
+                        while (io.writeFully(arr, 0, zis.read(arr)) > 0)
+                            ;
                     }
 
                     try {
@@ -2093,10 +2087,7 @@ else if (create)
     public static long writeSerializerVersion(FileIO io, long idx, int version, WALMode mode) throws IOException {
         ByteBuffer buffer = prepareSerializerVersionBuffer(idx, version, false);
 
-        do {
-            io.write(buffer);
-        }
-        while (buffer.hasRemaining());
+        io.writeFully(buffer);
 
         // Flush
         if (mode == WALMode.FSYNC)
@@ -2831,15 +2822,7 @@ private boolean close(boolean rollOver) throws StorageException {
 
                                 buf.rewind();
 
-                                int rem = buf.remaining();
-
-                                while (rem > 0) {
-                                    int written0 = fileIO.write(buf, written);
-
-                                    written += written0;
-
-                                    rem -= written0;
-                                }
+                                written += fileIO.writeFully(buf, written);
                             }
                         }
                         catch (IgniteCheckedException e) {
@@ -2981,10 +2964,7 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException {
                 try {
                     assert written == fileIO.position();
 
-                    do {
-                        fileIO.write(buf);
-                    }
-                    while (buf.hasRemaining());
+                    fileIO.writeFully(buf);
 
                     written += size;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
index e03cf52035768..f7299b962eeaa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
@@ -234,7 +234,7 @@ private void eraseDataFromDisk(
 
         long size = fileIO.size();
 
-        fileIO.write(ByteBuffer.allocate((int)size - filePageStore.headerSize()), filePageStore.headerSize());
+        fileIO.writeFully(ByteBuffer.allocate((int)size - filePageStore.headerSize()), filePageStore.headerSize());
 
         fileIO.force();
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
index 41a18ec2e8c59..d6c244367cc10 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
 import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory;
+import org.apache.ignite.internal.processors.cache.persistence.file.AbstractFileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -523,7 +524,7 @@ private static class TestFileIOFactory implements FileIOFactory {
     /**
      *
      */
-    private static class TestFileIO implements FileIO {
+    private static class TestFileIO extends AbstractFileIO {
         /** */
         private final FileIO delegate;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
index 3d5250703de5a..c077b27cb8ee3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
@@ -70,7 +70,7 @@ public class IgniteDataIntegrityTests extends TestCase {
 
         buf.rewind();
 
-        fileInput.io().write(buf);
+        fileInput.io().writeFully(buf);
         fileInput.io().force();
     }
 
@@ -180,12 +180,12 @@ private void toggleOneRandomBit(int rangeFrom, int rangeTo) throws IOException {
 
         byte[] buf = new byte[1];
 
-        fileInput.io().read(buf, 0, 1);
+        fileInput.io().readFully(buf, 0, 1);
 
         buf[0] ^= (1 << 3);
 
         fileInput.io().position(pos);
-        fileInput.io().write(buf, 0, 1);
+        fileInput.io().writeFully(buf, 0, 1);
         fileInput.io().force();
     }
 
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
index 88d77e017f4c0..2a7504cc0bc11 100644
--- a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIO.java
@@ -42,7 +42,7 @@
  *
  * Works only for Linux
  */
-public class AlignedBuffersDirectFileIO implements FileIO {
+public class AlignedBuffersDirectFileIO extends AbstractFileIO {
     /** Negative value for file offset: read/write starting from current file position */
     private static final int FILE_POS_USE_CURRENT = -1;
 
diff --git a/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteFileIOTest.java b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteFileIOTest.java
new file mode 100644
index 0000000000000..9620eb0fca683
--- /dev/null
+++ b/modules/direct-io/src/test/java/org/apache/ignite/internal/processors/cache/persistence/file/IgniteFileIOTest.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.util.concurrent.ThreadLocalRandom;
+import junit.framework.TestCase;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * File IO tests.
+ */
+public class IgniteFileIOTest extends TestCase {
+    /** Test data size. */
+    private static final int TEST_DATA_SIZE = 16 * 1024 * 1024;
+
+    /**
+     *
+     */
+    private static class TestFileIO extends AbstractFileIO {
+        /** Data. */
+        private final byte[] data;
+        /** Position. */
+        private int position;
+
+        /**
+         * @param maxSize Maximum size.
+         */
+        TestFileIO(int maxSize) {
+            this.data = new byte[maxSize];
+        }
+
+        /**
+         * @param data Initial data.
+         */
+        TestFileIO(byte[] data) {
+            this.data = data;
+        }
+
+        /** {@inheritDoc} */
+        @Override public long position() throws IOException {
+            return position;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void position(long newPosition) throws IOException {
+            checkPosition(newPosition);
+
+            this.position = (int)newPosition;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int read(ByteBuffer destBuf) throws IOException {
+            final int len = Math.min(destBuf.remaining(), data.length - position);
+
+            destBuf.put(data, position, len);
+
+            position += len;
+
+            return len;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int read(ByteBuffer destBuf, long position) throws IOException {
+            checkPosition(position);
+
+            final int len = Math.min(destBuf.remaining(), data.length - (int)position);
+
+            destBuf.put(data, (int)position, len);
+
+            return len;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int read(byte[] buf, int off, int maxLen) throws IOException {
+            final int len = Math.min(maxLen, data.length - position);
+
+            System.arraycopy(data, position, buf, off, len);
+
+            position += len;
+
+            return len;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int write(ByteBuffer srcBuf) throws IOException {
+            final int len = Math.min(srcBuf.remaining(), data.length - position);
+
+            srcBuf.get(data, position, len);
+
+            position += len;
+
+            return len;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int write(ByteBuffer srcBuf, long position) throws IOException {
+            checkPosition(position);
+
+            final int len = Math.min(srcBuf.remaining(), data.length - (int)position);
+
+            srcBuf.get(data, (int)position, len);
+
+            return len;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int write(byte[] buf, int off, int maxLen) throws IOException {
+            final int len = Math.min(maxLen, data.length - position);
+
+            System.arraycopy(buf, off, data, position, len);
+
+            position += len;
+
+            return len;
+        }
+
+        /** {@inheritDoc} */
+        @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void force() throws IOException {
+        }
+
+        /** {@inheritDoc} */
+        @Override public void force(boolean withMetadata) throws IOException {
+        }
+
+        /** {@inheritDoc} */
+        @Override public long size() throws IOException {
+            return data.length;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void clear() throws IOException {
+            position = 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void close() throws IOException {
+        }
+
+        /**
+         * @param position Position.
+         */
+        private void checkPosition(long position) throws IOException {
+            if (position < 0 || position >= data.length)
+                throw new IOException("Invalid position: " + position);
+        }
+    }
+
+    /**
+     * test for 'full read' functionality.
+     */
+    public void testReadFully() throws Exception {
+        byte[] arr = new byte[TEST_DATA_SIZE];
+
+        fillRandomArray(arr);
+
+        ByteBuffer buf = ByteBuffer.allocate(TEST_DATA_SIZE);
+
+        TestFileIO fileIO = new TestFileIO(arr) {
+            @Override public int read(ByteBuffer destBuf) throws IOException {
+                if (destBuf.remaining() < 2)
+                    return super.read(destBuf);
+
+                int oldLimit = destBuf.limit();
+
+                destBuf.limit(destBuf.position() + (destBuf.remaining() >> 1));
+
+                try {
+                    return super.read(destBuf);
+                }
+                finally {
+                    destBuf.limit(oldLimit);
+                }
+            }
+        };
+
+        fileIO.readFully(buf);
+
+        assert buf.remaining() == 0;
+
+        assert compareArrays(arr, buf.array());
+    }
+
+    /**
+     * test for 'full read' functionality.
+     */
+    public void testReadFullyArray() throws Exception {
+        byte[] arr = new byte[TEST_DATA_SIZE];
+
+        byte[] arrDst = new byte[TEST_DATA_SIZE];
+
+        fillRandomArray(arr);
+
+        TestFileIO fileIO = new TestFileIO(arr) {
+            @Override public int read(byte[] buf, int off, int len) throws IOException {
+                return super.read(buf, off, len < 2 ? len : (len >> 1));
+            }
+        };
+
+        fileIO.readFully(arrDst, 0, arrDst.length);
+
+        assert compareArrays(arr, arrDst);
+    }
+
+    /**
+     * test for 'full write' functionality.
+     */
+    public void testWriteFully() throws Exception {
+        byte[] arr = new byte[TEST_DATA_SIZE];
+
+        ByteBuffer buf = ByteBuffer.allocate(TEST_DATA_SIZE);
+
+        fillRandomArray(buf.array());
+
+        TestFileIO fileIO = new TestFileIO(arr) {
+            @Override public int write(ByteBuffer destBuf) throws IOException {
+                if (destBuf.remaining() < 2)
+                    return super.write(destBuf);
+
+                int oldLimit = destBuf.limit();
+
+                destBuf.limit(destBuf.position() + (destBuf.remaining() >> 1));
+
+                try {
+                    return super.write(destBuf);
+                }
+                finally {
+                    destBuf.limit(oldLimit);
+                }
+            }
+        };
+
+        fileIO.writeFully(buf);
+
+        assert buf.remaining() == 0;
+
+        assert compareArrays(arr, buf.array());
+    }
+
+    /**
+     * test for 'full write' functionality.
+     */
+    public void testWriteFullyArray() throws Exception {
+        byte[] arr = new byte[TEST_DATA_SIZE];
+
+        byte[] arrSrc = new byte[TEST_DATA_SIZE];
+
+        fillRandomArray(arrSrc);
+
+        TestFileIO fileIO = new TestFileIO(arr) {
+            @Override public int write(byte[] buf, int off, int len) throws IOException {
+                return super.write(buf, off, len < 2 ? len : (len >> 1));
+            }
+        };
+
+        fileIO.writeFully(arrSrc, 0, arrSrc.length);
+
+        assert compareArrays(arr, arrSrc);
+    }
+
+    /**
+     * @param arr Array.
+     */
+    private static void fillRandomArray(@NotNull final byte[] arr) {
+        ThreadLocalRandom.current().nextBytes(arr);
+    }
+
+    /**
+     * @param arr1 Array 1.
+     * @param arr2 Array 2.
+     */
+    private static boolean compareArrays(@NotNull final byte[] arr1, @NotNull final byte[] arr2) {
+        if (arr1.length != arr2.length)
+            return false;
+
+        for (int i = 0; i < arr1.length; i++)
+            if (arr1[i] != arr2[i])
+                return false;
+
+        return true;
+    }
+}

From ab298bc8b61ef176709398f96eb1f17a19e1965a Mon Sep 17 00:00:00 2001
From: AMedvedev <AMedvedev@gridgain.com>
Date: Thu, 5 Jul 2018 13:19:23 +0300
Subject: [PATCH 233/543] IGNITE-8737 Improve checkpoint logging information -
 Fixes #4244.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry-picked from commit#a727a4c7135e57415de0756e8fdc235ba191109a)
---
 .../GridCacheDatabaseSharedManager.java       |  13 ++-
 .../checkpoint/CheckpointHistory.java         |  28 ++++-
 .../wal/FileWriteAheadLogManager.java         |  12 +-
 .../FsyncModeFileWriteAheadLogManager.java    |   8 +-
 .../checkpoint/IgniteMassLoadSandboxTest.java | 110 +++++++++++++++++-
 5 files changed, 154 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index ce78d656d93da..a4553042e5e37 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -3192,12 +3192,13 @@ private void doCheckpoint() {
                     if (printCheckpointStats) {
                         if (log.isInfoEnabled())
                             log.info(String.format("Checkpoint finished [cpId=%s, pages=%d, markPos=%s, " +
-                                    "walSegmentsCleared=%d, markDuration=%dms, pagesWrite=%dms, fsync=%dms, " +
+                                    "walSegmentsCleared=%d, walSegmentsCovered=%s, markDuration=%dms, pagesWrite=%dms, fsync=%dms, " +
                                     "total=%dms]",
                                 chp.cpEntry != null ? chp.cpEntry.checkpointId() : "",
                                 chp.pagesSize,
                                 chp.cpEntry != null ? chp.cpEntry.checkpointMark() : "",
                                 chp.walFilesDeleted,
+                                chp.walSegmentsCovered,
                                 tracker.markDuration(),
                                 tracker.pagesWriteDuration(),
                                 tracker.fsyncDuration(),
@@ -3869,6 +3870,9 @@ public static class Checkpoint {
         /** Number of deleted WAL files. */
         private int walFilesDeleted;
 
+        /** WAL segments fully covered by this checkpoint. */
+        private List<Long> walSegmentsCovered;
+
         /** */
         private final int pagesSize;
 
@@ -3902,6 +3906,13 @@ public boolean hasDelta() {
         public void walFilesDeleted(int walFilesDeleted) {
             this.walFilesDeleted = walFilesDeleted;
         }
+
+        /**
+         * @param walSegmentsCovered WAL segments fully covered by this checkpoint.
+         */
+        public void walSegmentsCovered(final List<Long> walSegmentsCovered) {
+            this.walSegmentsCovered = walSegmentsCovered;
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
index d6cc297d0fd4f..cef2093ab9e21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
@@ -193,13 +193,39 @@ public List<CheckpointEntry> onWalTruncated(WALPointer ptr) {
     }
 
     /**
-     * Clears checkpoint history after checkpoint finish.
+     * Logs and clears checkpoint history after checkpoint finish.
      *
      * @return List of checkpoints removed from history.
      */
     public List<CheckpointEntry> onCheckpointFinished(GridCacheDatabaseSharedManager.Checkpoint chp, boolean truncateWal) {
         List<CheckpointEntry> removed = new ArrayList<>();
 
+        final Map.Entry<Long, CheckpointEntry> lastEntry = histMap.lastEntry();
+
+        assert lastEntry != null;
+
+        final Map.Entry<Long, CheckpointEntry> previousEntry = histMap.lowerEntry(lastEntry.getKey());
+
+        final WALPointer lastWALPointer = lastEntry.getValue().checkpointMark();
+
+        long lastIdx = 0;
+
+        long prevIdx = 0;
+
+        final ArrayList<Long> walSegmentsCovered = new ArrayList<>();
+
+        if (lastWALPointer instanceof FileWALPointer) {
+            lastIdx = ((FileWALPointer)lastWALPointer).index();
+
+            if (previousEntry != null)
+                prevIdx = ((FileWALPointer)previousEntry.getValue().checkpointMark()).index();
+        }
+
+        for (long walCovered = prevIdx; walCovered < lastIdx; walCovered++)
+            walSegmentsCovered.add(walCovered);
+
+        chp.walSegmentsCovered(walSegmentsCovered);
+
         int deleted = 0;
 
         while (histMap.size() > maxCpHistMemSize) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 44003039549b6..2bb6cd9ce7ef6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -745,8 +745,8 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
 
                 currWrHandle = rollOver(currWrHandle);
 
-                if (log != null && log.isDebugEnabled())
-                    log.debug("Rollover segment [" + idx + " to " + currWrHandle.idx + "], recordType=" + rec.type());
+                if (log != null && log.isInfoEnabled())
+                    log.info("Rollover segment [" + idx + " to " + currWrHandle.idx + "], recordType=" + rec.type());
             }
 
             WALPointer ptr = currWrHandle.addRecord(rec);
@@ -1814,8 +1814,8 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws StorageException
 
             File dstFile = new File(walArchiveDir, name);
 
-            if (log.isDebugEnabled())
-                log.debug("Starting to copy WAL segment [absIdx=" + absIdx + ", segIdx=" + segIdx +
+            if (log.isInfoEnabled())
+                log.info("Starting to copy WAL segment [absIdx=" + absIdx + ", segIdx=" + segIdx +
                     ", origFile=" + origFile.getAbsolutePath() + ", dstFile=" + dstFile.getAbsolutePath() + ']');
 
             try {
@@ -1837,8 +1837,8 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws StorageException
                     ", dstFile=" + dstTmpFile.getAbsolutePath() + ']', e);
             }
 
-            if (log.isDebugEnabled())
-                log.debug("Copied file [src=" + origFile.getAbsolutePath() +
+            if (log.isInfoEnabled())
+                log.info("Copied file [src=" + origFile.getAbsolutePath() +
                     ", dst=" + dstFile.getAbsolutePath() + ']');
 
             return new SegmentArchiveResult(absIdx, origFile, dstFile);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 45cb1a72b28bc..a75dd3100bf3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -1642,8 +1642,8 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedExc
 
             File dstFile = new File(walArchiveDir, name);
 
-            if (log.isDebugEnabled())
-                log.debug("Starting to copy WAL segment [absIdx=" + absIdx + ", segIdx=" + segIdx +
+            if (log.isInfoEnabled())
+                log.info("Starting to copy WAL segment [absIdx=" + absIdx + ", segIdx=" + segIdx +
                     ", origFile=" + origFile.getAbsolutePath() + ", dstFile=" + dstFile.getAbsolutePath() + ']');
 
             try {
@@ -1665,8 +1665,8 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedExc
                     ", dstFile=" + dstTmpFile.getAbsolutePath() + ']', e);
             }
 
-            if (log.isDebugEnabled())
-                log.debug("Copied file [src=" + origFile.getAbsolutePath() +
+            if (log.isInfoEnabled())
+                log.info("Copied file [src=" + origFile.getAbsolutePath() +
                     ", dst=" + dstFile.getAbsolutePath() + ']');
 
             return new SegmentArchiveResult(absIdx, origFile, dstFile);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
index 76cd5bdc7bf89..7fb277ca9f11e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
@@ -19,6 +19,7 @@
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Map;
 import java.util.Random;
@@ -29,6 +30,8 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import javax.cache.Cache;
 import junit.framework.TestCase;
 import org.apache.ignite.Ignite;
@@ -47,9 +50,11 @@
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridStringLogger;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -165,6 +170,7 @@ public class IgniteMassLoadSandboxTest extends GridCommonAbstractTest {
 
     /**
      * Runs multithreaded put scenario (no data streamer). Load is generated to page store and to WAL.
+     *
      * @throws Exception if failed.
      */
     public void testContinuousPutMultithreaded() throws Exception {
@@ -224,6 +230,7 @@ public void testContinuousPutMultithreaded() throws Exception {
 
     /**
      * Runs multithreaded put scenario (no data streamer). Load is generated to page store and to WAL.
+     *
      * @throws Exception if failed.
      */
     public void testDataStreamerContinuousPutMultithreaded() throws Exception {
@@ -234,7 +241,6 @@ public void testDataStreamerContinuousPutMultithreaded() throws Exception {
             System.setProperty(IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED, "speed");
             System.setProperty(IgniteSystemProperties.IGNITE_DELAYED_REPLACED_PAGE_WRITE, "true");
 
-
             setWalArchAndWorkToSameVal = true;
 
             customWalMode = WALMode.BACKGROUND;
@@ -243,7 +249,8 @@ public void testDataStreamerContinuousPutMultithreaded() throws Exception {
 
             ignite.active(true);
 
-            final int threads = 1; Runtime.getRuntime().availableProcessors();
+            final int threads = 1;
+            Runtime.getRuntime().availableProcessors();
 
             final int recsPerThread = CONTINUOUS_PUT_RECS_CNT / threads;
 
@@ -297,6 +304,100 @@ public void testDataStreamerContinuousPutMultithreaded() throws Exception {
         }
     }
 
+    /**
+     * Test that WAL segments that are fully covered by checkpoint are logged
+     *
+     * @throws Exception if failed.
+     */
+    public void testCoveredWalLogged() throws Exception {
+        GridStringLogger log0 = null;
+
+        try {
+            log0 = new GridStringLogger();
+
+            final IgniteConfiguration cfg = getConfiguration("testCoveredWalLogged");
+
+            cfg.setGridLogger(log0);
+
+            cfg.getDataStorageConfiguration().setWalAutoArchiveAfterInactivity(10);
+
+            final Ignite ignite = G.start(cfg);
+
+            ignite.cluster().active(true);
+
+            final IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+            cache.put(1, new byte[cfg.getDataStorageConfiguration().getWalSegmentSize() - 1024]);
+
+            forceCheckpoint();
+
+            cache.put(1, new byte[cfg.getDataStorageConfiguration().getWalSegmentSize() - 1024]);
+
+            forceCheckpoint();
+
+            cache.put(1, new byte[cfg.getDataStorageConfiguration().getWalSegmentSize() - 1024]);
+
+            forceCheckpoint();
+
+            Thread.sleep(200); // needed by GridStringLogger
+
+            final String log = log0.toString();
+
+            final String lines[] = log.split("\\r?\\n");
+
+            final Pattern chPtrn = Pattern.compile("Checkpoint finished");
+
+            final Pattern idxPtrn = Pattern.compile("idx=([0-9]+),");
+
+            final Pattern covererdPtrn = Pattern.compile("walSegmentsCovered=\\[(.+)\\], ");
+
+            boolean hasCheckpoint = false;
+
+            long nextCovered = 0;
+
+            for (String line : lines) {
+                if (!chPtrn.matcher(line).find())
+                    continue;
+
+                hasCheckpoint = true;
+
+                final Matcher idxMatcher = idxPtrn.matcher(line);
+
+                assertTrue(idxMatcher.find());
+
+                final long idx = Long.valueOf(idxMatcher.group(1));
+
+                final Matcher coveredMatcher = covererdPtrn.matcher(line);
+
+                if (!coveredMatcher.find()) { // no wal segments are covered by checkpoint
+                    assertEquals(nextCovered, idx);
+                    continue;
+                }
+
+                final String coveredMatcherGrp = coveredMatcher.group(1);
+
+                final long[] covered = coveredMatcherGrp.length() > 0 ?
+                    Arrays.stream(coveredMatcherGrp.split(",")).mapToLong(e -> Integer.valueOf(e.trim())).toArray() :
+                    new long[0];
+
+                assertEquals(nextCovered, covered[0]);
+
+                final long lastCovered = covered[covered.length - 1];
+
+                assertEquals(idx - 1, lastCovered);  // current wal is excluded
+
+                nextCovered = lastCovered + 1;
+            }
+
+            assertTrue(hasCheckpoint);
+
+        }
+        finally {
+            System.out.println(log0 != null ? log0.toString() : "Error initializing GridStringLogger");
+
+            stopAllGrids();
+        }
+    }
 
     /**
      * Verifies data from storage.
@@ -457,14 +558,13 @@ public void testPutRemoveMultithreaded() throws Exception {
         }
     }
 
-
     /** {@inheritDoc} */
     @Override protected long getTestTimeout() {
         return TimeUnit.MINUTES.toMillis(20);
     }
 
     /** Object with additional 40 000 bytes of payload */
-    public static class HugeIndexedObject   {
+    public static class HugeIndexedObject {
         /** Data. */
         private byte[] data;
         /** */
@@ -514,4 +614,4 @@ public byte[] data() {
             return S.toString(HugeIndexedObject.class, this);
         }
     }
-}
\ No newline at end of file
+}

From 6971f3d09bcf67e76efde290154b6c36470b20cb Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Fri, 6 Jul 2018 23:00:25 +0300
Subject: [PATCH 234/543] IGNITE-8904 Add rebalanceThreadPoolSize to nodes
 configuration consistency check

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit b490982)
Signed-off-by: Ivan Rakov <irakov@apache.org>
---
 .../apache/ignite/internal/IgniteKernal.java  |  2 +
 .../ignite/internal/IgniteNodeAttributes.java |  3 +
 .../processors/cache/GridCacheProcessor.java  | 24 ++++++++
 .../CacheRebalanceConfigValidationTest.java   | 55 +++++++++++++++++++
 .../testsuites/IgniteBasicTestSuite.java      |  3 +
 5 files changed, 87 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalanceConfigValidationTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 2e860a0de1c85..343c621b73e0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -261,6 +261,7 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PREFIX;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REBALANCE_POOL_SIZE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_RESTART_ENABLED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REST_PORT_RANGE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SPI_CLASS;
@@ -1524,6 +1525,7 @@ private void suggestOptimizations(IgniteConfiguration cfg) {
      */
     @SuppressWarnings({"SuspiciousMethodCalls", "unchecked", "TypeMayBeWeakened"})
     private void fillNodeAttributes(boolean notifyEnabled) throws IgniteCheckedException {
+        ctx.addNodeAttribute(ATTR_REBALANCE_POOL_SIZE, configuration().getRebalanceThreadPoolSize());
         ctx.addNodeAttribute(ATTR_DATA_STREAMER_POOL_SIZE, configuration().getDataStreamerThreadPoolSize());
 
         final String[] incProps = cfg.getIncludeProperties();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
index 6a4beebac0cba..663a6f9ad244e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
@@ -199,6 +199,9 @@ public final class IgniteNodeAttributes {
     /** User authentication enabled flag. */
     public static final String ATTR_AUTHENTICATION_ENABLED = ATTR_PREFIX + ".authentication.enabled";
 
+    /** Rebalance thread pool size. */
+    public static final String ATTR_REBALANCE_POOL_SIZE = ATTR_PREFIX + ".rebalance.pool.size";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index aca3f2dbdaa37..a0e61b5fb88e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -926,6 +926,8 @@ private void checkConsistency() throws IgniteCheckedException {
             if (Boolean.TRUE.equals(n.attribute(ATTR_CONSISTENCY_CHECK_SKIPPED)))
                 continue;
 
+            checkRebalanceConfiguration(n);
+
             checkTransactionConfiguration(n);
 
             checkMemoryConfiguration(n);
@@ -3646,6 +3648,28 @@ private void checkMemoryConfiguration(ClusterNode rmt) throws IgniteCheckedExcep
         }
     }
 
+    /**
+     * @param rmt Remote node to check.
+     * @throws IgniteCheckedException If check failed.
+     */
+    private void checkRebalanceConfiguration(ClusterNode rmt) throws IgniteCheckedException {
+        ClusterNode locNode = ctx.discovery().localNode();
+
+        if (ctx.config().isClientMode() || locNode.isDaemon() || rmt.isClient() || rmt.isDaemon())
+            return;
+
+        Integer rebalanceThreadPoolSize = rmt.attribute(IgniteNodeAttributes.ATTR_REBALANCE_POOL_SIZE);
+
+        if (rebalanceThreadPoolSize != null && rebalanceThreadPoolSize != ctx.config().getRebalanceThreadPoolSize()) {
+            throw new IgniteCheckedException("Rebalance configuration mismatch (fix configuration or set -D" +
+                IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK + "=true system property)." +
+                " Different values of such parameter may lead to rebalance process instability and hanging. " +
+                " [rmtNodeId=" + rmt.id() +
+                ", locRebalanceThreadPoolSize = " + ctx.config().getRebalanceThreadPoolSize() +
+                ", rmtRebalanceThreadPoolSize = " + rebalanceThreadPoolSize + "]");
+        }
+    }
+
     /**
      * @param cfg Cache configuration.
      * @return Query manager.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalanceConfigValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalanceConfigValidationTest.java
new file mode 100644
index 0000000000000..2f31b31372396
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheRebalanceConfigValidationTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class CacheRebalanceConfigValidationTest extends GridCommonAbstractTest {
+    /** Rebalance pool size. */
+    private int rebalancePoolSize;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setRebalanceThreadPoolSize(rebalancePoolSize);
+
+        return cfg;
+    }
+
+    /**
+     * Checks that node is not allowed to join to cluster if has different value of {@link IgniteConfiguration#rebalanceThreadPoolSize}.
+     *
+     * @throws Exception If failed.
+     */
+    public void testParameterConsistency() throws Exception {
+        rebalancePoolSize = 2;
+
+        startGrid(0);
+
+        rebalancePoolSize = 1;
+
+        GridTestUtils.assertThrows(log, () -> startGrid(1), IgniteCheckedException.class, "Rebalance configuration mismatch");
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index 21d56b465da32..eaaff3f24369b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -46,6 +46,7 @@
 import org.apache.ignite.internal.TransactionsMXBeanImplTest;
 import org.apache.ignite.internal.managers.IgniteDiagnosticMessagesTest;
 import org.apache.ignite.internal.processors.affinity.GridAffinityProcessorRendezvousSelfTest;
+import org.apache.ignite.internal.processors.cache.CacheRebalanceConfigValidationTest;
 import org.apache.ignite.internal.processors.cache.GridLocalIgniteSerializationTest;
 import org.apache.ignite.internal.processors.cache.GridProjectionForCachesOnDaemonNodeSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteDaemonNodeMarshallerCacheTest;
@@ -208,6 +209,8 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(OomFailureHandlerTest.class);
         suite.addTestSuite(AccountTransferTransactionTest.class);
 
+        suite.addTestSuite(CacheRebalanceConfigValidationTest.class);
+
         return suite;
     }
 }

From 6c0f32fe6fdcc5d0b17388a9e1290371499368ce Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Fri, 6 Jul 2018 18:11:55 +0300
Subject: [PATCH 235/543] IGNITE-8754 Node outside of baseline does not start
 when service configured

Signed-off-by: Andrey Gura <agura@apache.org>
(cherry picked from commit b1832673eba37919ad33f9d1c42b9b6bfa7e5a34)
---
 .../internal/processors/service/GridServiceProcessor.java  | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 63f50273c88ae..6915e810995c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -71,6 +71,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.query.CacheQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
@@ -1517,7 +1518,11 @@ private Iterator<Cache.Entry<Object, Object>> serviceEntries(IgniteBiPredicate<O
 
             qry.keepAll(false);
 
-            if (!cache.context().affinityNode()) {
+            DiscoveryDataClusterState clusterState = ctx.state().clusterState();
+
+            if ((clusterState.baselineTopology() != null
+                && !CU.baselineNode(ctx.cluster().get().localNode(), clusterState))
+                || !cache.context().affinityNode()) {
                 ClusterNode oldestSrvNode =
                     ctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE);
 

From 6ac7204f1c66e9691504ef7cd25e8d904bdf36c4 Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Mon, 9 Jul 2018 20:15:49 +0300
Subject: [PATCH 236/543] IGNITE-8898 Renamed command argument '--force' to
 '--yes' for control.sh

Signed-off-by: Andrey Gura <agura@apache.org>
(cherry picked from commit 413b4a8a1f905e31e3a8770d2c78ca953d39a60f)
---
 .../internal/commandline/Arguments.java       | 14 ++---
 .../internal/commandline/CommandHandler.java  | 35 +++++------
 .../CommandHandlerParsingTest.java            | 61 +++++++++++++++++++
 .../ignite/util/GridCommandHandlerTest.java   |  9 ++-
 4 files changed, 90 insertions(+), 29 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index 0d4b38e83851b..5b8a0dcd6c005 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -41,7 +41,7 @@ public class Arguments {
     private String pwd;
 
     /** Force option is used for auto confirmation. */
-    private boolean force;
+    private boolean autoConfirmation;
 
     /**
      * Action for baseline command.
@@ -91,11 +91,11 @@ public class Arguments {
      * @param walArgs WAL args.
      * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
      * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
-     * @param force Force flag.
+     * @param autoConfirmation Auto confirmation flag.
      */
     public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
                      String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
-                     Long pingTimeout, Long pingInterval, boolean force) {
+                     Long pingTimeout, Long pingInterval, boolean autoConfirmation) {
         this.cmd = cmd;
         this.host = host;
         this.port = port;
@@ -109,7 +109,7 @@ public Arguments(Command cmd, String host, String port, String user, String pwd,
         this.walArgs = walArgs;
         this.pingTimeout = pingTimeout;
         this.pingInterval = pingInterval;
-        this.force = force;
+        this.autoConfirmation = autoConfirmation;
     }
 
     /**
@@ -208,9 +208,9 @@ public long pingInterval() {
     }
 
     /**
-     * @return Force option.
+     * @return Auto confirmation option.
      */
-    public boolean force() {
-        return force;
+    public boolean autoConfirmation() {
+        return autoConfirmation;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 77ee6f14d2d2c..fb0a5dfc54491 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -138,8 +138,8 @@ public class CommandHandler {
     /** */
     private static final String CMD_USER = "--user";
 
-    /** Force option is used for auto confirmation. */
-    private static final String CMD_FORCE = "--force";
+    /** Option is used for auto confirmation. */
+    private static final String CMD_AUTO_CONFIRMATION = "--yes";
 
     /** */
     protected static final String CMD_PING_INTERVAL = "--ping-interval";
@@ -155,7 +155,7 @@ public class CommandHandler {
         AUX_COMMANDS.add(CMD_PORT);
         AUX_COMMANDS.add(CMD_PASSWORD);
         AUX_COMMANDS.add(CMD_USER);
-        AUX_COMMANDS.add(CMD_FORCE);
+        AUX_COMMANDS.add(CMD_AUTO_CONFIRMATION);
         AUX_COMMANDS.add(CMD_PING_INTERVAL);
         AUX_COMMANDS.add(CMD_PING_TIMEOUT);
     }
@@ -346,9 +346,6 @@ private boolean confirm(Arguments args) {
      * @return Prompt text if confirmation needed, otherwise {@code null}.
      */
     private String confirmationPrompt(Arguments args) {
-        if (args.force())
-            return null;
-
         String str = null;
 
         switch (args.command()) {
@@ -1251,7 +1248,7 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         String walArgs = "";
 
-        boolean force = false;
+        boolean autoConfirmation = false;
 
         CacheArguments cacheArgs = null;
 
@@ -1373,8 +1370,8 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
-                    case CMD_FORCE:
-                        force = true;
+                    case CMD_AUTO_CONFIRMATION:
+                        autoConfirmation = true;
 
                         break;
 
@@ -1401,7 +1398,7 @@ Arguments parseAndValidate(List<String> rawArgs) {
             throw new IllegalArgumentException("Both user and password should be specified");
 
         return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
-                pingTimeout, pingInterval, force);
+                pingTimeout, pingInterval, autoConfirmation);
     }
 
     /**
@@ -1734,22 +1731,22 @@ public int execute(List<String> rawArgs) {
                 log("This utility can do the following commands:");
 
                 usage("  Activate cluster:", ACTIVATE);
-                usage("  Deactivate cluster:", DEACTIVATE, " [--force]");
+                usage("  Deactivate cluster:", DEACTIVATE, " [" + CMD_AUTO_CONFIRMATION + "]");
                 usage("  Print current cluster state:", STATE);
                 usage("  Print cluster baseline topology:", BASELINE);
-                usage("  Add nodes into baseline topology:", BASELINE, " add consistentId1[,consistentId2,....,consistentIdN] [--force]");
-                usage("  Remove nodes from baseline topology:", BASELINE, " remove consistentId1[,consistentId2,....,consistentIdN] [--force]");
-                usage("  Set baseline topology:", BASELINE, " set consistentId1[,consistentId2,....,consistentIdN] [--force]");
-                usage("  Set baseline topology based on version:", BASELINE, " version topologyVersion [--force]");
+                usage("  Add nodes into baseline topology:", BASELINE, " add consistentId1[,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
+                usage("  Remove nodes from baseline topology:", BASELINE, " remove consistentId1[,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
+                usage("  Set baseline topology:", BASELINE, " set consistentId1[,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
+                usage("  Set baseline topology based on version:", BASELINE, " version topologyVersion [" + CMD_AUTO_CONFIRMATION + "]");
                 usage("  List or kill transactions:", TX, " [xid XID] [minDuration SECONDS] " +
                     "[minSize SIZE] [label PATTERN_REGEX] [servers|clients] " +
-                    "[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE|", CMD_TX_ORDER_START_TIME, "] [kill] [--force]");
+                    "[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE|", CMD_TX_ORDER_START_TIME, "] [kill] [" + CMD_AUTO_CONFIRMATION + "]");
 
                 if(enableExperimental) {
                     usage("  Print absolute paths of unused archived wal segments on each node:", WAL,
                             " print [consistentId1,consistentId2,....,consistentIdN]");
                     usage("  Delete unused archived wal segments on each node:", WAL,
-                            " delete [consistentId1,consistentId2,....,consistentIdN] [--force]");
+                            " delete [consistentId1,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
                 }
 
                 log("  View caches information in a cluster. For more details type:");
@@ -1757,7 +1754,7 @@ public int execute(List<String> rawArgs) {
                 nl();
 
                 log("By default commands affecting the cluster require interactive confirmation.");
-                log("Use --force option to disable it.");
+                log("Use " + CMD_AUTO_CONFIRMATION + " option to disable it.");
                 nl();
 
                 log("Default values:");
@@ -1779,7 +1776,7 @@ public int execute(List<String> rawArgs) {
 
             Arguments args = parseAndValidate(rawArgs);
 
-            if (!confirm(args)) {
+            if (!args.autoConfirmation() && !confirm(args)) {
                 log("Operation cancelled.");
 
                 return EXIT_CODE_OK;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 737c0c736f7ae..0ac5d1a4483b0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -23,6 +23,7 @@
 import junit.framework.TestCase;
 import org.apache.ignite.internal.commandline.cache.CacheArguments;
 import org.apache.ignite.internal.commandline.cache.CacheCommand;
+import org.apache.ignite.internal.visor.tx.VisorTxOperation;
 import org.apache.ignite.internal.visor.tx.VisorTxProjection;
 import org.apache.ignite.internal.visor.tx.VisorTxSortOrder;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
@@ -258,6 +259,66 @@ public void testParseAndValidateWalActions() {
         }
     }
 
+    /**
+     * Tests that the auto confirmation flag was correctly parsed.
+     */
+    public void testParseAutoConfirmationFlag() {
+        CommandHandler hnd = new CommandHandler();
+
+        for (Command cmd : Command.values()) {
+            if (cmd != Command.DEACTIVATE
+                && cmd != Command.BASELINE
+                && cmd != Command.TX)
+                continue;
+
+            Arguments args = hnd.parseAndValidate(asList(cmd.text()));
+
+            assertEquals(cmd, args.command());
+            assertEquals(DFLT_HOST, args.host());
+            assertEquals(DFLT_PORT, args.port());
+            assertEquals(false, args.autoConfirmation());
+
+            switch (cmd) {
+                case DEACTIVATE: {
+                    args = hnd.parseAndValidate(asList(cmd.text(), "--yes"));
+
+                    assertEquals(cmd, args.command());
+                    assertEquals(DFLT_HOST, args.host());
+                    assertEquals(DFLT_PORT, args.port());
+                    assertEquals(true, args.autoConfirmation());
+
+                    break;
+                }
+                case BASELINE: {
+                    for (String baselineAct : asList("add", "remove", "set")) {
+                        args = hnd.parseAndValidate(asList(cmd.text(), baselineAct, "c_id1,c_id2", "--yes"));
+
+                        assertEquals(cmd, args.command());
+                        assertEquals(DFLT_HOST, args.host());
+                        assertEquals(DFLT_PORT, args.port());
+                        assertEquals(baselineAct, args.baselineAction());
+                        assertEquals("c_id1,c_id2", args.baselineArguments());
+                        assertEquals(true, args.autoConfirmation());
+                    }
+
+                    break;
+                }
+                case TX: {
+                    args = hnd.parseAndValidate(asList(cmd.text(), "xid", "xid1", "minDuration", "10", "kill", "--yes"));
+
+                    assertEquals(cmd, args.command());
+                    assertEquals(DFLT_HOST, args.host());
+                    assertEquals(DFLT_PORT, args.port());
+                    assertEquals(true, args.autoConfirmation());
+
+                    assertEquals("xid1", args.transactionArguments().getXid());
+                    assertEquals(10_000, args.transactionArguments().getMinDuration().longValue());
+                    assertEquals(VisorTxOperation.KILL, args.transactionArguments().getOperation());
+                }
+            }
+        }
+    }
+
     /**
      * Tests host and port arguments.
      * Tests connection settings arguments.
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index c2c5ff46edb61..1432171485a00 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -109,6 +109,9 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
     /** Test out - can be injected via {@link #injectTestSystemOut()} instead of System.out and analyzed in test. */
     protected ByteArrayOutputStream testOut;
 
+    /** Option is used for auto confirmation. */
+    private static final String CMD_AUTO_CONFIRMATION = "--yes";
+
     /**
      * @return Folder in work directory.
      * @throws IgniteCheckedException If failed to resolve folder name.
@@ -209,7 +212,7 @@ protected int execute(String... args) {
      */
     protected int execute(ArrayList<String> args) {
         // Add force to avoid interactive confirmation
-        args.add("--force");
+        args.add(CMD_AUTO_CONFIRMATION);
 
         return new CommandHandler().execute(args);
     }
@@ -221,7 +224,7 @@ protected int execute(ArrayList<String> args) {
      */
     protected int execute(CommandHandler hnd, ArrayList<String> args) {
         // Add force to avoid interactive confirmation
-        args.add("--force");
+        args.add(CMD_AUTO_CONFIRMATION);
 
         return hnd.execute(args);
     }
@@ -235,7 +238,7 @@ protected int execute(CommandHandler hnd, String... args) {
         ArrayList<String> args0 = new ArrayList<>(Arrays.asList(args));
 
         // Add force to avoid interactive confirmation
-        args0.add("--force");
+        args0.add(CMD_AUTO_CONFIRMATION);
 
         return hnd.execute(args0);
     }

From d3b140f763433686dbdd3f156edfeec1561df531 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Wed, 4 Jul 2018 17:11:00 +0300
Subject: [PATCH 237/543] IGNITE-8925: SQL: Limit default number of threads for
 CREATE INDEX to 4. This closes #4301.

(cherry picked from commit 5cc41df7ac33ce4664f7db1c1cf8b3cf33d71cc8)
---
 .../processors/query/schema/SchemaIndexCacheVisitorImpl.java   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java
index b99fb6d3975ae..1775c79f72a4e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java
@@ -49,7 +49,8 @@
 @SuppressWarnings("ForLoopReplaceableByForEach")
 public class SchemaIndexCacheVisitorImpl implements SchemaIndexCacheVisitor {
     /** Default degree of parallelism. */
-    private static final int DFLT_PARALLELISM = Math.max(1, Runtime.getRuntime().availableProcessors() / 4);
+    private static final int DFLT_PARALLELISM =
+        Math.min(4, Math.max(1, Runtime.getRuntime().availableProcessors() / 4));
 
     /** Count of rows, being processed within a single checkpoint lock. */
     private static final int BATCH_SIZE = 1000;

From 72848848268c6c0facbd96ebff7ed1f6de10de29 Mon Sep 17 00:00:00 2001
From: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Date: Fri, 6 Jul 2018 18:31:23 +0300
Subject: [PATCH 238/543] IGNITE-8620 Remove intOrder and loc keys from node
 info in control.sh --tx utility

Signed-off-by: Andrey Gura <agura@apache.org>
(cherry picked from commit 532dc79a1459befa757849487f7aef2cb8608cee)
---
 .../ignite/internal/commandline/CommandHandler.java       | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index fb0a5dfc54491..69811e1cfc8d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -978,7 +978,13 @@ else if (arg.getOperation() == VisorTxOperation.KILL)
 
                 ClusterNode key = entry.getKey();
 
-                log(key.toString());
+                log(key.getClass().getSimpleName() + " [id=" + key.id() +
+                    ", addrs=" + key.addresses() +
+                    ", order=" + key.order() +
+                    ", ver=" + key.version() +
+                    ", isClient=" + key.isClient() +
+                    ", consistentId=" + key.consistentId() +
+                    "]");
 
                 for (VisorTxInfo info : entry.getValue().getInfos())
                     log("    Tx: [xid=" + info.getXid() +

From f9fbf2a5b50ba623599292e4cc1bab8893e698d8 Mon Sep 17 00:00:00 2001
From: ascherbakoff <alexey.scherbakoff@gmail.com>
Date: Wed, 11 Jul 2018 17:09:05 +0300
Subject: [PATCH 239/543] IGNITE-8942 In some cases grid cannot be deactivated
 because of hanging CQ internal cleanup. - Fixes #4329.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 08f98e3)
---
 .../managers/discovery/DiscoCache.java        |   4 +-
 .../discovery/GridDiscoveryManager.java       |  10 +-
 .../IgniteSequenceInternalCleanupTest.java    | 147 ++++++++++++++++++
 ...gniteCacheDataStructuresSelfTestSuite.java |   3 +
 4 files changed, 157 insertions(+), 7 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteSequenceInternalCleanupTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index fef44fad30bb7..0bb01f3f2c3ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -164,10 +164,8 @@ public class DiscoCache {
         this.consIdxToNodeId = consIdxToNodeId;
 
         aliveBaselineNodePred = new P1<BaselineNode>() {
-            @Override
-            public boolean apply(BaselineNode node) {
+            @Override public boolean apply(BaselineNode node) {
                 return node instanceof ClusterNode && alives.contains(((ClusterNode)node).id());
-
             }
         };
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 15badf27a071b..4122fd62363d7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -1085,6 +1085,10 @@ public DiscoveryMetricsProvider createMetricsProvider() {
             /** {@inheritDoc} */
             @Override public Map<Integer, CacheMetrics> cacheMetrics() {
                 try {
+                    /** Caches should not be accessed while state transition is in progress. */
+                    if (ctx.state().clusterState().transition())
+                        return Collections.emptyMap();
+
                     Collection<GridCacheAdapter<?, ?>> caches = ctx.cache().internalCaches();
 
                     if (!F.isEmpty(caches)) {
@@ -1093,10 +1097,8 @@ public DiscoveryMetricsProvider createMetricsProvider() {
                         for (GridCacheAdapter<?, ?> cache : caches) {
                             if (cache.context().statisticsEnabled() &&
                                 cache.context().started() &&
-                                cache.context().affinity().affinityTopologyVersion().topologyVersion() > 0) {
-
+                                cache.context().affinity().affinityTopologyVersion().topologyVersion() > 0)
                                 metrics.put(cache.context().cacheId(), cache.localMetrics());
-                            }
                         }
 
                         return metrics;
@@ -3299,7 +3301,7 @@ private void fillAffinityNodeCaches(
                 if (CU.affinityNode(node, grpAff.cacheFilter)) {
                     if (grpAff.persistentCacheGrp && bltNodes != null && !bltNodes.contains(node.id())) // Filter out.
                         continue;
-                    
+
                     List<ClusterNode> nodes = cacheGrpAffNodes.get(grpId);
 
                     if (nodes == null)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteSequenceInternalCleanupTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteSequenceInternalCleanupTest.java
new file mode 100644
index 0000000000000..d5a76f859266b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteSequenceInternalCleanupTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.datastructures;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteAtomicSequence;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.AtomicConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ */
+public class IgniteSequenceInternalCleanupTest extends GridCommonAbstractTest {
+    /** */
+    public static final int GRIDS_CNT = 5;
+
+    /** */
+    public static final int SEQ_RESERVE = 50_000;
+
+    /** */
+    public static final int CACHES_CNT = 10;
+
+    /** */
+    protected static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setClientMode("client".equals(igniteInstanceName));
+
+        cfg.setMetricsUpdateFrequency(10);
+
+        cfg.setActiveOnStart(false);
+
+        TcpDiscoverySpi spi = new TcpDiscoverySpi();
+
+        spi.setIpFinder(ipFinder);
+
+        cfg.setDiscoverySpi(spi);
+
+        AtomicConfiguration atomicCfg = atomicConfiguration();
+
+        assertNotNull(atomicCfg);
+
+        cfg.setAtomicConfiguration(atomicCfg);
+
+        List<CacheConfiguration> cacheCfg = new ArrayList<>();
+
+        for (int i = 0; i < CACHES_CNT; i++) {
+            cacheCfg.add(new CacheConfiguration("test" + i).
+                setStatisticsEnabled(true).
+                setCacheMode(PARTITIONED).
+                setAtomicityMode(TRANSACTIONAL).
+                setAffinity(new RendezvousAffinityFunction(false, 16)));
+        }
+
+        cfg.setCacheConfiguration(cacheCfg.toArray(new CacheConfiguration[cacheCfg.size()]));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    protected AtomicConfiguration atomicConfiguration() {
+        AtomicConfiguration cfg = new AtomicConfiguration();
+
+        cfg.setCacheMode(PARTITIONED);
+        cfg.setBackups(1);
+        cfg.setAtomicSequenceReserveSize(SEQ_RESERVE);
+
+        return cfg;
+    }
+
+    /** */
+    public void testDeactivate() throws Exception {
+        try {
+            Ignite ignite = startGridsMultiThreaded(GRIDS_CNT);
+
+            ignite.cache("test0").put(0, 0);
+
+            int id = 0;
+
+            for (Ignite ig : G.allGrids()) {
+                IgniteAtomicSequence seq = ig.atomicSequence("testSeq", 0, true);
+
+                long id0 = seq.getAndIncrement();
+
+                assertEquals(id0, id);
+
+                id += SEQ_RESERVE;
+            }
+
+            doSleep(1000);
+
+            long puts = ignite.cache("test0").metrics().getCachePuts();
+
+            assertEquals(1, puts);
+
+            grid(GRIDS_CNT - 1).cluster().active(false);
+
+            ignite.cluster().active(true);
+
+            long putsAfter = ignite.cache("test0").metrics().getCachePuts();
+
+            assertEquals(0, putsAfter);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
index 9583143a35253..80e8319bd335c 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.processors.cache.datastructures.IgniteClientDiscoveryDataStructuresTest;
 import org.apache.ignite.internal.processors.cache.datastructures.IgniteDataStructureUniqueNameTest;
 import org.apache.ignite.internal.processors.cache.datastructures.IgniteDataStructureWithJobTest;
+import org.apache.ignite.internal.processors.cache.datastructures.IgniteSequenceInternalCleanupTest;
 import org.apache.ignite.internal.processors.cache.datastructures.SemaphoreFailoverNoWaitingAcquirerTest;
 import org.apache.ignite.internal.processors.cache.datastructures.SemaphoreFailoverSafeReleasePermitsTest;
 import org.apache.ignite.internal.processors.cache.datastructures.local.GridCacheLocalAtomicQueueApiSelfTest;
@@ -173,6 +174,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTest(new TestSuite(IgnitePartitionedQueueNoBackupsTest.class));
 
+        suite.addTest(new TestSuite(IgniteSequenceInternalCleanupTest.class));
+
         suite.addTestSuite(AtomicCacheAffinityConfigurationTest.class);
 
         return suite;

From 3e83d6ac0457811a212bc61e82ae00bddbd3fc52 Mon Sep 17 00:00:00 2001
From: Pereslegin Pavel <sbt-pereslegin-pa1@mail.ca.sbrf.ru>
Date: Wed, 11 Jul 2018 17:25:34 +0300
Subject: [PATCH 240/543] IGNITE-7366 Affinity assignment exception in service
 processor during multiple nodes join - Fixes #4321.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit efa3269)
---
 .../service/GridServiceProcessor.java         | 61 +++++++++++---
 .../GridServiceReassignmentSelfTest.java      | 45 ++++++++---
 .../IgniteServiceReassignmentTest.java        | 79 +++++++++++++++++++
 3 files changed, 162 insertions(+), 23 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 6915e810995c4..ff701f4b56d45 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -68,6 +68,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.binary.MetadataUpdateAcceptedMessage;
 import org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.query.CacheQuery;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
@@ -1728,6 +1729,47 @@ private class TopologyListener implements DiscoveryEventListener {
         /** */
         private volatile AffinityTopologyVersion currTopVer = null;
 
+        /**
+         * Check that listening-in topology version is the latest and wait until exchange is finished.
+         *
+         * @param initTopVer listening-in topology version.
+         * @return {@code True} if current event is not last and should be skipped.
+         */
+        private boolean skipExchange(AffinityTopologyVersion initTopVer) {
+            AffinityTopologyVersion pendingTopVer = null;
+            AffinityTopologyVersion newTopVer = currTopVer;
+
+            if (!initTopVer.equals(newTopVer))
+                pendingTopVer = newTopVer;
+            else {
+                GridDhtTopologyFuture fut = ctx.cache().context().exchange().lastTopologyFuture();
+
+                if (!fut.isDone() && !fut.isCancelled()) {
+                    try {
+                        fut.get();
+                    }
+                    catch (IgniteCheckedException e) {
+                        throw U.convertException(e);
+                    }
+                }
+
+                AffinityTopologyVersion lastTopVer;
+
+                // If exchange already moved forward - skip current version.
+                if (fut.exchangeDone() && newTopVer.compareTo(lastTopVer = fut.topologyVersion()) < 0)
+                    pendingTopVer = lastTopVer;
+            }
+
+            if (pendingTopVer != null && log.isInfoEnabled()) {
+                log.info("Service processor detected a topology change during " +
+                    "assignments calculation (will abort current iteration and " +
+                    "re-calculate on the newer version): " +
+                    "[topVer=" + initTopVer + ", newTopVer=" + pendingTopVer + ']');
+            }
+
+            return pendingTopVer != null;
+        }
+
         /** {@inheritDoc} */
         @Override public void onEvent(final DiscoveryEvent evt, final DiscoCache discoCache) {
             GridSpinBusyLock busyLock = GridServiceProcessor.this.busyLock;
@@ -1781,17 +1823,8 @@ else if (msg instanceof DynamicCacheChangeBatch) {
 
                                 while (it.hasNext()) {
                                     // If topology changed again, let next event handle it.
-                                    AffinityTopologyVersion currTopVer0 = currTopVer;
-
-                                    if (currTopVer0 != topVer) {
-                                        if (log.isInfoEnabled())
-                                            log.info("Service processor detected a topology change during " +
-                                                "assignments calculation (will abort current iteration and " +
-                                                "re-calculate on the newer version): " +
-                                                "[topVer=" + topVer + ", newTopVer=" + currTopVer0 + ']');
-
+                                    if (skipExchange(topVer))
                                         return;
-                                    }
 
                                     Cache.Entry<Object, Object> e = it.next();
 
@@ -1800,12 +1833,10 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                                     try {
                                         svcName.set(dep.configuration().getName());
 
-                                        ctx.cache().context().exchange().affinityReadyFuture(topVer).get();
-
                                         reassign(dep, topVer);
                                     }
                                     catch (IgniteCheckedException ex) {
-                                        if (!(e instanceof ClusterTopologyCheckedException))
+                                        if (!(ex instanceof ClusterTopologyCheckedException))
                                             LT.error(log, ex, "Failed to do service reassignment (will retry): " +
                                                 dep.configuration().getName());
 
@@ -1827,6 +1858,10 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                         // Clean up zombie assignments.
                         IgniteInternalCache<Object, Object> cache = serviceCache();
 
+                        // If topology changed again, let next event handle it.
+                        if (skipExchange(topVer))
+                            return;
+
                         while (it.hasNext()) {
                             Cache.Entry<Object, Object> e = it.next();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceReassignmentSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceReassignmentSelfTest.java
index 0f5d595ca481d..e44c8eabde0c7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceReassignmentSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/GridServiceReassignmentSelfTest.java
@@ -29,11 +29,15 @@
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
 
 /**
  * Tests service reassignment.
  */
 public class GridServiceReassignmentSelfTest extends GridServiceProcessorAbstractSelfTest {
+    /** */
+    private static final String SERVICE_NAME = "testService";
+
     /** {@inheritDoc} */
     @Override protected int nodeCount() {
         return 1;
@@ -71,7 +75,7 @@ public void testLimited2() throws Exception {
      * @throws Exception If failed.
      */
     private CounterService proxy(Ignite g) throws Exception {
-        return g.services().serviceProxy("testService", CounterService.class, false);
+        return g.services().serviceProxy(SERVICE_NAME, CounterService.class, false);
     }
 
     /**
@@ -82,9 +86,9 @@ private CounterService proxy(Ignite g) throws Exception {
     private void checkReassigns(int total, int maxPerNode) throws Exception {
         CountDownLatch latch = new CountDownLatch(nodeCount());
 
-        DummyService.exeLatch("testService", latch);
+        DummyService.exeLatch(SERVICE_NAME, latch);
 
-        grid(0).services().deployMultiple("testService", new CounterServiceImpl(), total, maxPerNode);
+        grid(0).services().deployMultiple(SERVICE_NAME, new CounterServiceImpl(), total, maxPerNode);
 
         for (int i = 0; i < 10; i++)
             proxy(randomGrid()).increment();
@@ -104,11 +108,7 @@ private void checkReassigns(int total, int maxPerNode) throws Exception {
                 if (grow) {
                     assert startedGrids.size() < maxTopSize;
 
-                    int gridIdx = nextAvailableIdx(startedGrids, maxTopSize, rnd);
-
-                    startGrid(gridIdx);
-
-                    startedGrids.add(gridIdx);
+                    startRandomNodesMultithreaded(maxTopSize, rnd, startedGrids);
 
                     if (startedGrids.size() == maxTopSize)
                         grow = false;
@@ -135,7 +135,7 @@ private void checkReassigns(int total, int maxPerNode) throws Exception {
             }
         }
         finally {
-            grid(F.first(startedGrids)).services().cancel("testService");
+            grid(F.first(startedGrids)).services().cancel(SERVICE_NAME);
 
             stopAllGrids();
 
@@ -158,7 +158,7 @@ private boolean checkServices(int total, int maxPerNode, int gridIdx, boolean la
 
         IgniteInternalCache<GridServiceAssignmentsKey, GridServiceAssignments> cache = grid.utilityCache();
 
-        GridServiceAssignments assignments = cache.get(new GridServiceAssignmentsKey("testService"));
+        GridServiceAssignments assignments = cache.get(new GridServiceAssignmentsKey(SERVICE_NAME));
 
         Collection<UUID> nodes = F.viewReadOnly(grid.cluster().nodes(), F.node2id());
 
@@ -186,6 +186,8 @@ private boolean checkServices(int total, int maxPerNode, int gridIdx, boolean la
         if (total > 0)
             assertTrue("Total number of services limit exceeded [sum=" + sum +
                 ", assigns=" + assignments.assigns() + ']', sum <= total);
+        else
+            assertEquals("Reassign per node failed.", nodes.size(), assignments.assigns().size());
 
         if (!lastTry && proxy(grid).get() != 10)
             return false;
@@ -195,6 +197,29 @@ private boolean checkServices(int total, int maxPerNode, int gridIdx, boolean la
         return true;
     }
 
+    /**
+     * Start 1, 2 or 3 random nodes simultaneously.
+     *
+     * @param limit Cluster size limit.
+     * @param rnd Randmo generator.
+     * @param grids Collection with indexes of running nodes.
+     * @throws Exception If failed.
+     */
+    private void startRandomNodesMultithreaded(int limit, Random rnd, Collection<Integer> grids) throws Exception {
+        int cnt = rnd.nextInt(Math.min(limit - grids.size(), 3)) + 1;
+
+        for (int i = 1; i <= cnt; i++) {
+            int gridIdx = nextAvailableIdx(grids, limit, rnd);
+
+            if (i == cnt)
+                startGrid(gridIdx);
+            else
+                GridTestUtils.runAsync(() -> startGrid(gridIdx));
+
+            grids.add(gridIdx);
+        }
+    }
+
     /**
      * Gets next available index.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java
index 8116d1b05bc57..865f1213e3ccc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java
@@ -17,10 +17,16 @@
 
 package org.apache.ignite.internal.processors.service;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
 import java.util.concurrent.ThreadLocalRandom;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.services.Service;
@@ -29,6 +35,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridStringLogger;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -42,6 +49,12 @@ public class IgniteServiceReassignmentTest extends GridCommonAbstractTest {
     /** */
     private ServiceConfiguration srvcCfg;
 
+    /** */
+    private boolean useStrLog;
+
+    /** */
+    private List<IgniteLogger> strLoggers = new ArrayList<>();
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -51,6 +64,16 @@ public class IgniteServiceReassignmentTest extends GridCommonAbstractTest {
         if (srvcCfg != null)
             cfg.setServiceConfiguration(srvcCfg);
 
+        if (useStrLog) {
+            GridStringLogger strLog = new GridStringLogger(false, cfg.getGridLogger());
+
+            strLog.logLength(100 * 1024);
+
+            cfg.setGridLogger(strLog);
+
+            strLoggers.add(strLog);
+        }
+
         return cfg;
     }
 
@@ -164,6 +187,62 @@ public void testNodeRestartRandom() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testZombieAssignmentsCleanup() throws Exception {
+        useStrLog = true;
+
+        final int nodesCnt = 2;
+        final int maxSvc = 30;
+
+        try {
+            startGridsMultiThreaded(nodesCnt);
+
+            IgniteEx ignite = grid(0);
+
+            IgniteInternalCache<GridServiceAssignmentsKey, Object> sysCache = ignite.utilityCache();
+
+            List<GridServiceAssignmentsKey> zombieAssignmentsKeys = new ArrayList<>(maxSvc);
+
+            // Adding some assignments without deployments.
+            for (int i = 0; i < maxSvc; i++) {
+                String name = "svc-" + i;
+
+                ServiceConfiguration svcCfg = new ServiceConfiguration();
+
+                svcCfg.setName(name);
+
+                GridServiceAssignmentsKey key = new GridServiceAssignmentsKey(name);
+
+                UUID nodeId = grid(i % nodesCnt).localNode().id();
+
+                sysCache.put(key, new GridServiceAssignments(svcCfg, nodeId, ignite.cluster().topologyVersion()));
+
+                zombieAssignmentsKeys.add(key);
+            }
+
+            // Simulate exchange with merge.
+            GridTestUtils.runAsync(() -> startGrid(nodesCnt));
+            GridTestUtils.runAsync(() -> startGrid(nodesCnt + 1));
+            startGrid(nodesCnt + 2);
+
+            awaitPartitionMapExchange();
+
+            // Checking that all our assignments was removed.
+            for (GridServiceAssignmentsKey key : zombieAssignmentsKeys)
+                assertNull("Found assignment for undeployed service " + key.name(), sysCache.get(key));
+
+            for (IgniteLogger logger : strLoggers)
+                assertFalse(logger.toString().contains("Getting affinity for topology version earlier than affinity is " +
+                    "calculated"));
+        } finally {
+            useStrLog = false;
+
+            strLoggers.clear();
+        }
+    }
+
     /**
      * @param node Node.
      * @throws Exception If failed.

From bc920ccb37f3bd31e98a61fc43c89038f804d12a Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Wed, 11 Jul 2018 17:33:57 +0300
Subject: [PATCH 241/543] IGNITE-8945 Stored cache data files corruption when
 node stops abruptly. - Fixes #4319.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit fff979a)
---
 .../file/FilePageStoreManager.java            |  36 ++++-
 ...onfigurationFileConsistencyCheckTest.java} | 130 +++++++++++++++++-
 .../ignite/testsuites/IgnitePdsTestSuite.java |   4 +-
 3 files changed, 162 insertions(+), 8 deletions(-)
 rename modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/{IgnitePdsDuplicatedCacheConfigurationTest.java => IgnitePdsCacheConfigurationFileConsistencyCheckTest.java} (56%)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 3e882595a0454..4dd275ddfe249 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -22,6 +22,7 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
+import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -90,6 +91,9 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
     /** */
     public static final String CACHE_DATA_FILENAME = "cache_data.dat";
 
+    /** */
+    public static final String CACHE_DATA_TMP_FILENAME = CACHE_DATA_FILENAME + ".tmp";
+
     /** */
     public static final String DFLT_STORE_DIR = "db";
 
@@ -144,6 +148,7 @@ public FilePageStoreManager(GridKernalContext ctx) {
     /** {@inheritDoc} */
     @Override public void start0() throws IgniteCheckedException {
         final GridKernalContext ctx = cctx.kernalContext();
+
         if (ctx.clientNode())
             return;
 
@@ -284,12 +289,19 @@ public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws
 
         if (overwrite || !file.exists() || file.length() == 0) {
             try {
-                file.createNewFile();
+                File tmp = new File(file.getParent(), file.getName() + ".tmp");
+
+                tmp.createNewFile();
 
                 // Pre-existing file will be truncated upon stream open.
-                try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(file))) {
+                try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(tmp))) {
                     marshaller.marshal(cacheData, stream);
                 }
+
+                if (file.exists())
+                    file.delete();
+
+                Files.move(tmp.toPath(), file.toPath());
             }
             catch (IOException ex) {
                 throw new IgniteCheckedException("Failed to persist cache configuration: " + cacheData.config().getName(), ex);
@@ -651,6 +663,20 @@ else if (lockF.exists()) {
 
         for (File file : files) {
             if (file.isDirectory()) {
+                File[] tmpFiles = file.listFiles(new FilenameFilter() {
+                    @Override public boolean accept(File dir, String name) {
+                        return name.endsWith(CACHE_DATA_TMP_FILENAME);
+                    }
+                });
+
+                if (tmpFiles != null) {
+                    for (File tmpFile: tmpFiles) {
+                        if (!tmpFile.delete())
+                            log.warning("Failed to delete temporary cache config file" +
+                                    "(make sure Ignite process has enough rights):" + file.getName());
+                    }
+                }
+
                 if (file.getName().startsWith(CACHE_DIR_PREFIX)) {
                     File conf = new File(file, CACHE_DATA_FILENAME);
 
@@ -711,9 +737,9 @@ private StoredCacheData readCacheData(File conf) throws IgniteCheckedException {
         try (InputStream stream = new BufferedInputStream(new FileInputStream(conf))) {
             return marshaller.unmarshal(stream, U.resolveClassLoader(igniteCfg));
         }
-        catch (IOException e) {
-            throw new IgniteCheckedException("Failed to read cache configuration from disk for cache: " +
-                conf.getAbsolutePath(), e);
+        catch (IgniteCheckedException | IOException e) {
+                throw new IgniteCheckedException("An error occurred during cache configuration loading from file [file=" +
+                    conf.getAbsolutePath() + "]", e);
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDuplicatedCacheConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheConfigurationFileConsistencyCheckTest.java
similarity index 56%
rename from modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDuplicatedCacheConfigurationTest.java
rename to modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheConfigurationFileConsistencyCheckTest.java
index 66459bd620d1f..74a3950239195 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDuplicatedCacheConfigurationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheConfigurationFileConsistencyCheckTest.java
@@ -17,6 +17,12 @@
 
 package org.apache.ignite.internal.processors.cache.persistence;
 
+import java.io.BufferedOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 import org.apache.ignite.Ignite;
@@ -31,15 +37,21 @@
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DATA_FILENAME;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DATA_TMP_FILENAME;
+
 /**
  * Tests that ignite can start when caches' configurations with same name in different groups stored.
  */
-public class IgnitePdsDuplicatedCacheConfigurationTest extends GridCommonAbstractTest {
+public class IgnitePdsCacheConfigurationFileConsistencyCheckTest extends GridCommonAbstractTest {
     /** */
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
@@ -112,6 +124,67 @@ public void testStartDuplicatedCacheConfigurations() throws Exception {
 
     }
 
+    /**
+     * Check that cache_data.dat.tmp files are deleted after node restarts.
+     *
+     * @throws Exception If failed.
+     */
+    public void testTmpCacheConfigurationsDelete() throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(NODES);
+
+        ig0.cluster().active(true);
+
+        startCaches(ig0);
+
+        DynamicCacheDescriptor desc = ig0.context().cache().cacheDescriptor(cacheName(3));
+
+        storeTmpCacheData(desc);
+
+        stopAllGrids();
+
+        startGrids(NODES);
+
+        for (int i = 0; i < NODES; i++) {
+            IgniteEx ig = grid(i);
+
+            GridCacheSharedContext sharedCtx = ig.context().cache().context();
+
+            FilePageStoreManager pageStore = (FilePageStoreManager) sharedCtx.pageStore();
+
+            File[] tmpFile = pageStore.cacheWorkDir(true, ODD_GROUP_NAME).listFiles(new FilenameFilter() {
+                @Override public boolean accept(File dir, String name) {
+                    return name.endsWith(CACHE_DATA_TMP_FILENAME);
+                }
+            });
+
+            assertNotNull(tmpFile);
+
+            assertEquals(0, tmpFile.length);
+        }
+    }
+
+    /**
+     * Check that exception contains proper filename when trying to read corrupted cache configuration file.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCorruptedCacheConfigurationsValidation() throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(NODES);
+
+        ig0.cluster().active(true);
+
+        startCaches(ig0);
+
+        DynamicCacheDescriptor desc = ig0.context().cache().cacheDescriptor(cacheName(2));
+
+        corruptCacheData(desc);
+
+        stopAllGrids();
+
+        GridTestUtils.assertThrowsAnyCause(log, () -> startGrids(NODES), IgniteCheckedException.class,
+                desc.cacheName() + CACHE_DATA_FILENAME);
+    }
+
     /**
      * Store cache descriptor to PDS with invalid group name.
      *
@@ -134,6 +207,61 @@ private void storeInvalidCacheData(DynamicCacheDescriptor cacheDescr) throws Ign
         }
     }
 
+    /**
+     * Store temp cache descriptor to PDS.
+     *
+     * @param cacheDescr Cache descr.
+     * @throws IgniteCheckedException If fails.
+     */
+    private void storeTmpCacheData(DynamicCacheDescriptor cacheDescr) throws Exception {
+        Marshaller marshaller = new JdkMarshaller();
+
+        for (int i = 0; i < NODES; i++) {
+            IgniteEx ig = grid(i);
+
+            GridCacheSharedContext sharedCtx = ig.context().cache().context();
+
+            FilePageStoreManager pageStore = (FilePageStoreManager) sharedCtx.pageStore();
+
+            StoredCacheData data = cacheDescr.toStoredData();
+
+            data.config().setGroupName(ODD_GROUP_NAME);
+
+            File tmp = new File(pageStore.cacheWorkDir(true, ODD_GROUP_NAME), data.config().getName() + CACHE_DATA_TMP_FILENAME);
+
+            try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(tmp))) {
+                marshaller.marshal(data, stream);
+            }
+        }
+    }
+
+    /**
+     * Store temp cache descriptor to PDS.
+     *
+     * @param cacheDescr Cache descr.
+     * @throws IgniteCheckedException If fails.
+     */
+    private void corruptCacheData(DynamicCacheDescriptor cacheDescr) throws Exception {
+        for (int i = 0; i < NODES; i++) {
+            IgniteEx ig = grid(i);
+
+            GridCacheSharedContext sharedCtx = ig.context().cache().context();
+
+            FilePageStoreManager pageStore = (FilePageStoreManager) sharedCtx.pageStore();
+
+            StoredCacheData data = cacheDescr.toStoredData();
+
+            data.config().setGroupName(ODD_GROUP_NAME);
+
+            File config = new File(pageStore.cacheWorkDir(true, ODD_GROUP_NAME), data.config().getName() + CACHE_DATA_FILENAME);
+
+            try (DataOutputStream os = new DataOutputStream(new FileOutputStream(config))) {
+                os.writeLong(-1L);
+            }
+
+        }
+    }
+
     /**
      * @param ignite Ignite instance.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index 43ce04f4b7aae..17a3e41f35987 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -21,7 +21,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheWithoutCheckpointsTest;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDuplicatedCacheConfigurationTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheConfigurationFileConsistencyCheckTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest;
@@ -121,7 +121,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
 
         suite.addTestSuite(IgnitePdsDestroyCacheTest.class);
         suite.addTestSuite(IgnitePdsDestroyCacheWithoutCheckpointsTest.class);
-        suite.addTestSuite(IgnitePdsDuplicatedCacheConfigurationTest.class);
+        suite.addTestSuite(IgnitePdsCacheConfigurationFileConsistencyCheckTest.class);
 
         suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
 

From 86b465b61591b0b00b58c387c03e2cba86eb36e3 Mon Sep 17 00:00:00 2001
From: Ivan Rakov <irakov@apache.org>
Date: Wed, 11 Jul 2018 17:57:53 +0300
Subject: [PATCH 242/543] IGNITE-8965 Add logs in SegmentReservationStorage on
 exchange process

(cherry picked from commit ee909a3)
---
 .../internal/processors/cache/WalStateManager.java    | 11 +++++++++--
 .../persistence/wal/SegmentReservationStorage.java    |  2 +-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
index 4a14730964197..a3b73b9d58476 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -360,6 +360,8 @@ public void changeLocalStatesOnExchangeDone(AffinityTopologyVersion topVer) {
 
             boolean hasOwning = false;
 
+            int parts = 0;
+
             for (GridDhtLocalPartition locPart : grp.topology().currentLocalPartitions()) {
                 if (locPart.state() == OWNING) {
                     hasOwning = true;
@@ -373,11 +375,16 @@ public void changeLocalStatesOnExchangeDone(AffinityTopologyVersion topVer) {
                         break;
                     }
                 }
+
+                parts++;
             }
 
-            if (hasOwning && !grp.localWalEnabled()) {
+            log.info("Prepare change WAL state, grp=" + grp.cacheOrGroupName() +
+                ", grpId=" + grp.groupId() + ", hasOwning=" + hasOwning +
+                ", WALState=" + grp.walEnabled() + ", parts=" + parts);
+
+            if (hasOwning && !grp.localWalEnabled())
                 grpsToEnableWal.add(grp.groupId());
-            }
             else if (!hasOwning && grp.localWalEnabled()) {
                 grpsToDisableWal.add(grp.groupId());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentReservationStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentReservationStorage.java
index 17da96deec659..12c4b4f2ffd96 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentReservationStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentReservationStorage.java
@@ -51,7 +51,7 @@ synchronized boolean reserved(long absIdx) {
     synchronized void release(long absIdx) {
         Integer cur = reserved.get(absIdx);
 
-        assert cur != null && cur >= 1 : cur;
+        assert cur != null && cur >= 1 : "cur=" + cur + ", absIdx=" + absIdx;
 
         if (cur == 1)
             reserved.remove(absIdx);

From 02c3419e7d39a0db6ae0dcc6e84c5e9e89b54863 Mon Sep 17 00:00:00 2001
From: Ivan Rakov <irakov@apache.org>
Date: Wed, 11 Jul 2018 18:45:31 +0300
Subject: [PATCH 243/543] IGNITE-8946 AssertionError can occur during release
 of WAL history that was reserved for historical rebalance

(cherry picked from commit 54055ec)
---
 .../checkpoint/CheckpointHistory.java         |  35 +--
 ...owHistoricalRebalanceSmallHistoryTest.java | 236 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 3 files changed, 257 insertions(+), 17 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
index cef2093ab9e21..95f150b2f79f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
@@ -326,26 +326,13 @@ public Map<Integer, Map<Integer, CheckpointEntry>> searchAndReserveCheckpoints(
                 if (!reserved)
                     break;
 
-                for (Integer grpId : groupsAndPartitions.keySet())
+                for (Integer grpId : new HashSet<>(groupsAndPartitions.keySet()))
                     if (!isCheckpointApplicableForGroup(grpId, chpEntry))
                         groupsAndPartitions.remove(grpId);
 
-                // All groups are no more applicable, release history and stop searching.
-                if (groupsAndPartitions.isEmpty()) {
-                    cctx.wal().release(chpEntry.checkpointMark());
-
-                    break;
-                }
-
-                // Release previous checkpoint marker.
-                if (prevReserved != null)
-                    cctx.wal().release(prevReserved.checkpointMark());
-
-                prevReserved = chpEntry;
-
                 for (Map.Entry<Integer, CheckpointEntry.GroupState> state : chpEntry.groupState(cctx).entrySet()) {
                     int grpId = state.getKey();
-                    CheckpointEntry.GroupState cpGroupState = state.getValue();
+                    CheckpointEntry.GroupState cpGrpState = state.getValue();
 
                     Set<Integer> applicablePartitions = groupsAndPartitions.get(grpId);
 
@@ -355,7 +342,7 @@ public Map<Integer, Map<Integer, CheckpointEntry>> searchAndReserveCheckpoints(
                     Set<Integer> inapplicablePartitions = null;
 
                     for (Integer partId : applicablePartitions) {
-                        int pIdx = cpGroupState.indexByPartition(partId);
+                        int pIdx = cpGrpState.indexByPartition(partId);
 
                         if (pIdx >= 0)
                             res.computeIfAbsent(grpId, k -> new HashMap<>()).put(partId, chpEntry);
@@ -374,9 +361,23 @@ public Map<Integer, Map<Integer, CheckpointEntry>> searchAndReserveCheckpoints(
                 }
 
                 // Remove groups from search with empty set of applicable partitions.
-                for (Map.Entry<Integer, Set<Integer>> e : groupsAndPartitions.entrySet())
+                for (Map.Entry<Integer, Set<Integer>> e : new HashSet<>(groupsAndPartitions.entrySet()))
                     if (e.getValue().isEmpty())
                         groupsAndPartitions.remove(e.getKey());
+
+                // All groups are no more applicable, release history and stop searching.
+                if (groupsAndPartitions.isEmpty()) {
+                    cctx.wal().release(chpEntry.checkpointMark());
+
+                    break;
+                }
+                else {
+                    // Release previous checkpoint marker.
+                    if (prevReserved != null)
+                        cctx.wal().release(prevReserved.checkpointMark());
+
+                    prevReserved = chpEntry;
+                }
             }
             catch (IgniteCheckedException ex) {
                 U.error(log, "Failed to process checkpoint: " + (chpEntry != null ? chpEntry : "none"), ex);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
new file mode 100644
index 0000000000000..8f2e7389f8bb3
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
@@ -0,0 +1,236 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class SlowHistoricalRebalanceSmallHistoryTest extends GridCommonAbstractTest {
+    /** Ip finder. */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** Slow rebalance cache name. */
+    private static final String SLOW_REBALANCE_CACHE = "b13813ce";
+
+    /** Regular cache name. */
+    private static final String REGULAR_CACHE = "another-cache";
+
+    /** Supply message latch. */
+    private static final AtomicReference<CountDownLatch> SUPPLY_MESSAGE_LATCH = new AtomicReference<>();
+
+    /** Wal history size. */
+    private static final int WAL_HISTORY_SIZE = 5;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(name);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setWalHistorySize(WAL_HISTORY_SIZE)
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                )
+                .setWalSegmentSize(512 * 1024)
+        );
+
+        cfg.setFailureHandler(new StopNodeFailureHandler());
+
+        cfg.setCommunicationSpi(new RebalanceBlockingSPI());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        SUPPLY_MESSAGE_LATCH.set(new CountDownLatch(1));
+
+        System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "1000");
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        SUPPLY_MESSAGE_LATCH.get().countDown();
+
+        SUPPLY_MESSAGE_LATCH.set(null);
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Checks that we reserve and release the same WAL index on exchange.
+     */
+    public void testReservation() throws Exception {
+        IgniteEx ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        ig.getOrCreateCache(new CacheConfiguration<>()
+            .setName(SLOW_REBALANCE_CACHE)
+            .setAffinity(new RendezvousAffinityFunction(false, 1))
+            .setBackups(1)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setRebalanceBatchSize(100));
+
+        try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(SLOW_REBALANCE_CACHE)) {
+            for (int i = 0; i < 3_000; i++)
+                streamer.addData(i, new byte[5 * 1000]);
+
+            streamer.flush();
+        }
+
+        startGrid(1);
+
+        resetBaselineTopology();
+
+        IgniteCache<Object, Object> anotherCache = ig.getOrCreateCache(new CacheConfiguration<>()
+            .setName(REGULAR_CACHE)
+            .setAffinity(new RendezvousAffinityFunction(false, 1))
+            .setBackups(1)
+            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+            .setRebalanceBatchSize(100));
+
+        Thread.sleep(7_000); // To let distributedExchange() finish.
+
+        for (int i = 0; i < WAL_HISTORY_SIZE; i++) {
+            for (int j = 0; i < 500; i++)
+                anotherCache.put(j, new byte[5 * 1000]);
+
+            forceCheckpoint(); // Checkpoints where partition is OWNING on grid(0), MOVING on grid(1)
+
+            for (int j = 0; i < 500; i++)
+                anotherCache.put(j, new byte[5 * 1000]);
+        }
+
+        SUPPLY_MESSAGE_LATCH.get().countDown();
+
+        waitForRebalancing(); // Partition is OWNING on grid(0) and grid(1)
+
+        for (int i = 0; i < 2; i++) {
+            for (int j = 0; i < 500; i++)
+                anotherCache.put(j, new byte[5 * 1000]);
+
+            forceCheckpoint(); // A few more checkpoints when partition is OWNING everywhere
+
+            for (int j = 0; i < 500; i++)
+                anotherCache.put(j, new byte[5 * 1000]);
+        }
+
+        stopGrid(0);
+
+        IgniteCache<Object, Object> anotherCacheGrid1 = grid(1).cache(REGULAR_CACHE);
+
+        for (int i = 0; i < 500; i++)
+            anotherCacheGrid1.put(i, new byte[5 * 1000]);
+
+        startGrid(0);
+
+        waitForRebalancing();
+
+        assertEquals(2, grid(1).context().discovery().aliveServerNodes().size());
+    }
+
+    /**
+     *
+     */
+    private static class RebalanceBlockingSPI extends TcpCommunicationSpi {
+        /** */
+        public static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+            if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof GridDhtPartitionSupplyMessage) {
+                int grpId = ((GridCacheGroupIdMessage)((GridIoMessage)msg).message()).groupId();
+
+                if (grpId == CU.cacheId(SLOW_REBALANCE_CACHE)) {
+                    CountDownLatch latch0 = SUPPLY_MESSAGE_LATCH.get();
+
+                    if (latch0 != null)
+                        try {
+                            latch0.await();
+                        }
+                        catch (InterruptedException ex) {
+                            throw new IgniteException(ex);
+                        }
+                }
+            }
+
+            super.sendMessage(node, msg);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+            IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof GridDhtPartitionSupplyMessage) {
+                int grpId = ((GridCacheGroupIdMessage)((GridIoMessage)msg).message()).groupId();
+
+                if (grpId == CU.cacheId(SLOW_REBALANCE_CACHE)) {
+                    CountDownLatch latch0 = SUPPLY_MESSAGE_LATCH.get();
+
+                    if (latch0 != null)
+                        try {
+                            latch0.await();
+                        }
+                        catch (InterruptedException ex) {
+                            throw new IgniteException(ex);
+                        }
+                }
+            }
+
+            super.sendMessage(node, msg, ackC);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index f66ff20ba8d9c..9779ca58801e8 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.SlowHistoricalRebalanceSmallHistoryTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsUnusedWalSegmentsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.filename.IgniteUidAsConsistentIdMigrationTest;
@@ -139,6 +140,8 @@ public static void addRealPageStoreTests(TestSuite suite) {
         // Rebalancing test
         suite.addTestSuite(IgniteWalHistoryReservationsTest.class);
 
+        suite.addTestSuite(SlowHistoricalRebalanceSmallHistoryTest.class);
+
         suite.addTestSuite(IgnitePersistentStoreDataStructuresTest.class);
 
         // Failover test

From 3d727d316c4187c46cf7499255a328a2e265f91d Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Wed, 11 Jul 2018 19:50:58 +0300
Subject: [PATCH 244/543] IGNITE-8827 Fixed failing test

---
 .../ignite/internal/processors/cache/WalStateManager.java     | 4 +++-
 .../db/wal/IgniteWalIteratorSwitchSegmentTest.java            | 3 ++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
index a3b73b9d58476..3729ec610a3b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -122,7 +122,9 @@ public WalStateManager(GridKernalContext kernalCtx) {
         if (kernalCtx != null) {
             IgniteConfiguration cfg = kernalCtx.config();
 
-            srv = !cfg.isClientMode() && !cfg.isDaemon();
+            boolean client = cfg.isClientMode() != null && cfg.isClientMode();
+
+            srv = !client && !cfg.isDaemon();
 
             log = kernalCtx.log(WalStateManager.class);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
index b30466e5acf49..00ed6f166dcb0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
@@ -34,6 +34,7 @@
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.processors.cache.GridCacheIoManager;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.WalStateManager;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
@@ -360,7 +361,7 @@ else if (walMgrClass.equals(FsyncModeFileWriteAheadLogManager.class)) {
             null,
             null,
             walMgr,
-            null,
+            new WalStateManager(kctx),
             new GridCacheDatabaseSharedManager(kctx),
             null,
             null,

From 5c269ebfaf91ce56e8825bf36f41db88fb791cd0 Mon Sep 17 00:00:00 2001
From: Eduard Shangareev <eshangareev@gridgain.com>
Date: Wed, 11 Jul 2018 19:43:19 +0300
Subject: [PATCH 245/543] IGNITE-8955 Checkpoint can't get write lock if
 massive eviction on node start started

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit a0fa79a)
---
 .../IgniteAuthenticationProcessor.java        |   7 +-
 .../preloader/GridDhtPartitionDemander.java   | 107 +++---
 .../GridCacheDatabaseSharedManager.java       | 169 ++++++---
 .../persistence/GridCacheOffheapManager.java  |   4 +-
 .../persistence/pagemem/PageMemoryImpl.java   |  18 +-
 .../pagemem/PagesWriteSpeedBasedThrottle.java |  16 +-
 .../pagemem/PagesWriteThrottle.java           |  19 +-
 .../pagemem/PagesWriteThrottlePolicy.java     |   5 +
 .../db/CheckpointBufferDeadlockTest.java      | 358 ++++++++++++++++++
 9 files changed, 577 insertions(+), 126 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
index ac713c3dce875..ded37e79e7ab1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/IgniteAuthenticationProcessor.java
@@ -1292,9 +1292,10 @@ private UserOperationWorker(UserManagementOperation op, UserOperationFinishFutur
                 // Remove failed operation from active operations.
                 activeOps.remove(op.id());
             }
-
-            if (sharedCtx != null)
-                sharedCtx.database().checkpointReadUnlock();
+            finally {
+                if (sharedCtx != null)
+                    sharedCtx.database().checkpointReadUnlock();
+            }
 
             curOpFinishMsg = msg0;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 3cfc25f6b7fbf..1eeebaef3a79e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -727,77 +727,86 @@ public void handleSupplyMessage(
         try {
             AffinityAssignment aff = grp.affinity().cachedAffinity(topVer);
 
-            ctx.database().checkpointReadLock();
+            // Preload.
+            for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
+                int p = e.getKey();
 
-            try {
-                // Preload.
-                for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
-                    int p = e.getKey();
+                if (aff.get(p).contains(ctx.localNode())) {
+                    GridDhtLocalPartition part = top.localPartition(p, topVer, true);
 
-                    if (aff.get(p).contains(ctx.localNode())) {
-                        GridDhtLocalPartition part = top.localPartition(p, topVer, true);
+                    assert part != null;
 
-                        assert part != null;
+                    boolean last = supply.last().containsKey(p);
 
-                        boolean last = supply.last().containsKey(p);
+                    if (part.state() == MOVING) {
+                        boolean reserved = part.reserve();
 
-                        if (part.state() == MOVING) {
-                            boolean reserved = part.reserve();
+                        assert reserved : "Failed to reserve partition [igniteInstanceName=" +
+                            ctx.igniteInstanceName() + ", grp=" + grp.cacheOrGroupName() + ", part=" + part + ']';
 
-                            assert reserved : "Failed to reserve partition [igniteInstanceName=" +
-                                ctx.igniteInstanceName() + ", grp=" + grp.cacheOrGroupName() + ", part=" + part + ']';
+                        part.lock();
 
-                            part.lock();
+                        try {
+                            Iterator<GridCacheEntryInfo> infos = e.getValue().infos().iterator();
 
-                            try {
-                                // Loop through all received entries and try to preload them.
-                                for (GridCacheEntryInfo entry : e.getValue().infos()) {
-                                    if (!preloadEntry(node, p, entry, topVer)) {
-                                        if (log.isDebugEnabled())
-                                            log.debug("Got entries for invalid partition during " +
-                                                "preloading (will skip) [p=" + p + ", entry=" + entry + ']');
-
-                                        break;
-                                    }
+                            // Loop through all received entries and try to preload them.
+                            while (infos.hasNext()) {
+                                ctx.database().checkpointReadLock();
 
-                                    for (GridCacheContext cctx : grp.caches()) {
-                                        if (cctx.statisticsEnabled())
-                                            cctx.cache().metrics0().onRebalanceKeyReceived();
-                                    }
-                                }
+                                try {
+                                    for (int i = 0; i < 100; i++) {
+                                        if (!infos.hasNext())
+                                            break;
+
+                                        GridCacheEntryInfo entry = infos.next();
 
-                                // If message was last for this partition,
-                                // then we take ownership.
-                                if (last) {
-                                    fut.partitionDone(nodeId, p, true);
+                                        if (!preloadEntry(node, p, entry, topVer)) {
+                                            if (log.isDebugEnabled())
+                                                log.debug("Got entries for invalid partition during " +
+                                                        "preloading (will skip) [p=" + p + ", entry=" + entry + ']');
 
-                                    if (log.isDebugEnabled())
-                                        log.debug("Finished rebalancing partition: " + part);
+                                            break;
+                                        }
+
+                                        for (GridCacheContext cctx : grp.caches()) {
+                                            if (cctx.statisticsEnabled())
+                                                cctx.cache().metrics0().onRebalanceKeyReceived();
+                                        }
+                                    }
+                                }
+                                finally {
+                                    ctx.database().checkpointReadUnlock();
                                 }
                             }
-                            finally {
-                                part.unlock();
-                                part.release();
+
+                            // If message was last for this partition,
+                            // then we take ownership.
+                            if (last) {
+                                fut.partitionDone(nodeId, p, true);
+
+                                if (log.isDebugEnabled())
+                                    log.debug("Finished rebalancing partition: " + part);
                             }
                         }
-                        else {
-                            if (last)
-                                fut.partitionDone(nodeId, p, false);
-
-                            if (log.isDebugEnabled())
-                                log.debug("Skipping rebalancing partition (state is not MOVING): " + part);
+                        finally {
+                            part.unlock();
+                            part.release();
                         }
                     }
                     else {
-                        fut.partitionDone(nodeId, p, false);
+                        if (last)
+                            fut.partitionDone(nodeId, p, false);
 
                         if (log.isDebugEnabled())
-                            log.debug("Skipping rebalancing partition (it does not belong on current node): " + p);
+                            log.debug("Skipping rebalancing partition (state is not MOVING): " + part);
                     }
                 }
-            }
-            finally {
-                ctx.database().checkpointReadUnlock();
+                else {
+                    fut.partitionDone(nodeId, p, false);
+
+                    if (log.isDebugEnabled())
+                        log.debug("Skipping rebalancing partition (it does not belong on current node): " + p);
+                }
             }
 
             // Only request partitions based on latest topology version.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index a4553042e5e37..7280f96384d3e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1476,7 +1476,7 @@ private void prepareIndexRebuildFuture(int cacheId) {
                 throw new RuntimeException("Failed to perform cache update: node is stopping.");
             }
 
-            if (safeToUpdatePageMemories() || checkpointLock.getReadHoldCount() > 1)
+            if (checkpointLock.getReadHoldCount() > 1 || safeToUpdatePageMemories())
                 break;
             else {
                 checkpointLock.readLock().unlock();
@@ -2530,6 +2530,9 @@ private void finalizeCheckpointOnRecovery(long cpTs, UUID cpId, WALPointer walPt
 
                 Integer tag = pageMem.getForCheckpoint(fullId, tmpWriteBuf, null);
 
+                assert tag == null || tag != PageMemoryImpl.TRY_AGAIN_TAG :
+                        "Lock is held by other thread for page " + fullId;
+
                 if (tag != null) {
                     tmpWriteBuf.rewind();
 
@@ -3091,7 +3094,8 @@ private void doCheckpoint() {
                                     chp.cpPages.innerCollection(i),
                                     updStores,
                                     doneWriteFut,
-                                    totalPagesToWriteCnt
+                                    totalPagesToWriteCnt,
+                                    asyncRunner
                                 );
 
                                 try {
@@ -3105,11 +3109,13 @@ private void doCheckpoint() {
                         }
                         else {
                             // Single-threaded checkpoint.
-                            Runnable write = new WriteCheckpointPages(tracker,
+                            Runnable write = new WriteCheckpointPages(
+                                tracker,
                                 chp.cpPages,
                                 updStores,
                                 doneWriteFut,
-                                totalPagesToWriteCnt);
+                                totalPagesToWriteCnt,
+                                null);
 
                             write.run();
                         }
@@ -3740,117 +3746,158 @@ private GridMultiCollectionWrapper<FullPageId> splitAndSortCpPagesIfNeeded(
     /** Pages write task */
     private class WriteCheckpointPages implements Runnable {
         /** */
-        private CheckpointMetricsTracker tracker;
+        private final CheckpointMetricsTracker tracker;
 
         /** Collection of page IDs to write under this task. Overall pages to write may be greater than this collection */
-        private Collection<FullPageId> writePageIds;
+        private final Collection<FullPageId> writePageIds;
 
         /** */
-        private ConcurrentLinkedHashMap<PageStore, LongAdder> updStores;
+        private final ConcurrentLinkedHashMap<PageStore, LongAdder> updStores;
 
         /** */
-        private CountDownFuture doneFut;
+        private final CountDownFuture doneFut;
 
         /** Total pages to write, counter may be greater than {@link #writePageIds} size */
         private final int totalPagesToWrite;
 
+        /** If any pages were skipped, new task with remaining pages will be submitted here. */
+        private final ExecutorService retryWriteExecutor;
+
         /**
-         * Creates task for write pages
-         *
-         * @param tracker
-         * @param writePageIds Collection of page IDs to write.
-         * @param updStores
-         * @param doneFut
-         * @param totalPagesToWrite total pages to be written under this checkpoint
+         * @param tracker Tracker.
+         * @param writePageIds Write page ids.
+         * @param updStores Upd stores.
+         * @param doneFut Done future.
+         * @param totalPagesToWrite Total pages to write.
+         * @param retryWriteExecutor Retry write executor.
          */
         private WriteCheckpointPages(
             final CheckpointMetricsTracker tracker,
             final Collection<FullPageId> writePageIds,
             final ConcurrentLinkedHashMap<PageStore, LongAdder> updStores,
             final CountDownFuture doneFut,
-            final int totalPagesToWrite) {
+            final int totalPagesToWrite,
+            final ExecutorService retryWriteExecutor
+        ) {
             this.tracker = tracker;
             this.writePageIds = writePageIds;
             this.updStores = updStores;
             this.doneFut = doneFut;
             this.totalPagesToWrite = totalPagesToWrite;
+            this.retryWriteExecutor = retryWriteExecutor;
         }
 
         /** {@inheritDoc} */
         @Override public void run() {
+            snapshotMgr.beforeCheckpointPageWritten();
+
+            Collection<FullPageId> writePageIds = this.writePageIds;
+
+            try {
+                List<FullPageId> pagesToRetry = writePages(writePageIds);
+
+                if (pagesToRetry.isEmpty())
+                    doneFut.onDone((Void)null);
+                else {
+                    if (retryWriteExecutor == null) {
+                        while (!pagesToRetry.isEmpty())
+                            pagesToRetry = writePages(pagesToRetry);
+
+                        doneFut.onDone((Void)null);
+                    }
+                    else {
+                        // Submit current retry pages to the end of the queue to avoid starvation.
+                        WriteCheckpointPages retryWritesTask = new WriteCheckpointPages(
+                            tracker, pagesToRetry, updStores, doneFut, totalPagesToWrite, retryWriteExecutor);
+
+                        retryWriteExecutor.submit(retryWritesTask);
+                    }
+                }
+            }
+            catch (Throwable e) {
+                doneFut.onDone(e);
+            }
+        }
+
+        /**
+         * @param writePageIds Collections of pages to write.
+         * @return pagesToRetry Pages which should be retried.
+         */
+        private List<FullPageId> writePages(Collection<FullPageId> writePageIds) throws IgniteCheckedException {
             ByteBuffer tmpWriteBuf = threadBuf.get();
 
             long writeAddr = GridUnsafe.bufferAddress(tmpWriteBuf);
 
-            snapshotMgr.beforeCheckpointPageWritten();
+            List<FullPageId> pagesToRetry = new ArrayList<>();
 
-            try {
-                for (FullPageId fullId : writePageIds) {
-                    if (checkpointer.shutdownNow)
-                        break;
-
-                    tmpWriteBuf.rewind();
+            for (FullPageId fullId : writePageIds) {
+                if (checkpointer.shutdownNow)
+                    break;
 
-                    snapshotMgr.beforePageWrite(fullId);
+                tmpWriteBuf.rewind();
 
-                    int grpId = fullId.groupId();
+                snapshotMgr.beforePageWrite(fullId);
 
-                    PageMemoryEx pageMem;
+                int grpId = fullId.groupId();
 
-                    if (grpId != MetaStorage.METASTORAGE_CACHE_ID) {
-                        CacheGroupContext grp = context().cache().cacheGroup(grpId);
+                PageMemoryEx pageMem;
 
-                        if (grp == null)
-                            continue;
+                if (grpId != MetaStorage.METASTORAGE_CACHE_ID) {
+                    CacheGroupContext grp = context().cache().cacheGroup(grpId);
 
-                        if (!grp.dataRegion().config().isPersistenceEnabled())
-                            continue;
+                    if (grp == null)
+                        continue;
 
-                        pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
-                    }
-                    else
-                        pageMem = (PageMemoryEx)metaStorage.pageMemory();
+                    if (!grp.dataRegion().config().isPersistenceEnabled())
+                        continue;
 
+                    pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
+                }
+                else
+                    pageMem = (PageMemoryEx)metaStorage.pageMemory();
 
-                    Integer tag = pageMem.getForCheckpoint(
-                        fullId, tmpWriteBuf, persStoreMetrics.metricsEnabled() ? tracker : null);
 
-                    if (tag != null) {
-                        assert PageIO.getType(tmpWriteBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
-                        assert PageIO.getVersion(tmpWriteBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
+                Integer tag = pageMem.getForCheckpoint(
+                    fullId, tmpWriteBuf, persStoreMetrics.metricsEnabled() ? tracker : null);
 
-                        tmpWriteBuf.rewind();
+                if (tag != null) {
+                    if (tag == PageMemoryImpl.TRY_AGAIN_TAG) {
+                        pagesToRetry.add(fullId);
 
-                        if (persStoreMetrics.metricsEnabled()) {
-                            int pageType = PageIO.getType(tmpWriteBuf);
+                        continue;
+                    }
 
-                            if (PageIO.isDataPageType(pageType))
-                                tracker.onDataPageWritten();
-                        }
+                    assert PageIO.getType(tmpWriteBuf) != 0 : "Invalid state. Type is 0! pageId = " + U.hexLong(fullId.pageId());
+                    assert PageIO.getVersion(tmpWriteBuf) != 0 : "Invalid state. Version is 0! pageId = " + U.hexLong(fullId.pageId());
 
-                        if (!skipCrc) {
-                            PageIO.setCrc(writeAddr, PureJavaCrc32.calcCrc32(tmpWriteBuf, pageSize()));
+                    tmpWriteBuf.rewind();
 
-                            tmpWriteBuf.rewind();
-                        }
+                    if (persStoreMetrics.metricsEnabled()) {
+                        int pageType = PageIO.getType(tmpWriteBuf);
 
-                        int curWrittenPages = writtenPagesCntr.incrementAndGet();
+                        if (PageIO.isDataPageType(pageType))
+                            tracker.onDataPageWritten();
+                    }
 
-                        snapshotMgr.onPageWrite(fullId, tmpWriteBuf, curWrittenPages, totalPagesToWrite);
+                    if (!skipCrc) {
+                        PageIO.setCrc(writeAddr, PureJavaCrc32.calcCrc32(tmpWriteBuf, pageSize()));
 
                         tmpWriteBuf.rewind();
+                    }
 
-                        PageStore store = storeMgr.writeInternal(grpId, fullId.pageId(), tmpWriteBuf, tag, false);
+                    int curWrittenPages = writtenPagesCntr.incrementAndGet();
 
-                        updStores.computeIfAbsent(store, k -> new LongAdder()).increment();
-                    }
-                }
+                    snapshotMgr.onPageWrite(fullId, tmpWriteBuf, curWrittenPages, totalPagesToWrite);
 
-                doneFut.onDone((Void)null);
-            }
-            catch (Throwable e) {
-                doneFut.onDone(e);
+                    tmpWriteBuf.rewind();
+
+                    PageStore store = storeMgr.writeInternal(grpId, fullId.pageId(), tmpWriteBuf, tag, false);
+
+                    updStores.computeIfAbsent(store, k -> new LongAdder()).increment();
+                }
             }
+
+            return pagesToRetry;
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 87a06040b1a78..ac4f163ce5c60 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -1185,9 +1185,7 @@ private CacheDataStore init0(boolean checkExists) throws IgniteCheckedException
 
             IgniteCacheDatabaseSharedManager dbMgr = ctx.database();
 
-            dbMgr.checkpointReadLock();
-
-            if (init.compareAndSet(false, true)) {
+                dbMgr.checkpointReadLock();if (init.compareAndSet(false, true)) {
                 try {
                     Metas metas = getOrAllocatePartitionMetas();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 10d0a3f4cb047..f40365d7a6543 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -177,6 +177,9 @@ public class PageMemoryImpl implements PageMemoryEx {
     /** Number of random pages that will be picked for eviction. */
     public static final int RANDOM_PAGES_EVICT_NUM = 5;
 
+    /** Try again tag. */
+    public static final int TRY_AGAIN_TAG = -1;
+
     /** Tracking io. */
     private static final TrackingPageIO trackingIO = TrackingPageIO.VERSIONS.latest();
 
@@ -377,9 +380,9 @@ private void initWriteThrottle() {
         if (throttlingPlc == ThrottlingPolicy.SPEED_BASED)
             writeThrottle = new PagesWriteSpeedBasedThrottle(this, cpProgressProvider, stateChecker, log);
         else if (throttlingPlc == ThrottlingPolicy.TARGET_RATIO_BASED)
-            writeThrottle = new PagesWriteThrottle(this, cpProgressProvider, stateChecker, false);
+            writeThrottle = new PagesWriteThrottle(this, cpProgressProvider, stateChecker, false, log);
         else if (throttlingPlc == ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY)
-            writeThrottle = new PagesWriteThrottle(this, null, stateChecker, true);
+            writeThrottle = new PagesWriteThrottle(this, null, stateChecker, true, log);
     }
 
     /** {@inheritDoc} */
@@ -1118,7 +1121,7 @@ private boolean isThrottlingEnabled() {
             }
         }
         else
-            return copyPageForCheckpoint(absPtr, fullId, outBuf, pageSingleAcquire, tracker) ? tag : null;
+            return copyPageForCheckpoint(absPtr, fullId, outBuf, pageSingleAcquire, tracker) ? tag : TRY_AGAIN_TAG;
     }
 
     /**
@@ -1128,6 +1131,8 @@ private boolean isThrottlingEnabled() {
      * @param pageSingleAcquire Page is acquired only once. We don't pin the page second time (until page will not be
      * copied) in case checkpoint temporary buffer is used.
      * @param tracker Checkpoint statistics tracker.
+     *
+     * @return False if someone else holds lock on page.
      */
     private boolean copyPageForCheckpoint(
         long absPtr,
@@ -1139,7 +1144,10 @@ private boolean copyPageForCheckpoint(
         assert absPtr != 0;
         assert PageHeader.isAcquired(absPtr);
 
-        rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
+        boolean locked = rwLock.tryWriteLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
+
+        if (!locked)
+            return false;
 
         try {
             long tmpRelPtr = PageHeader.tempBufferPointer(absPtr);
@@ -2345,6 +2353,8 @@ private int partGeneration(int grpId, int partId) {
 
             Integer tag = partGenerationMap.get(new GroupPartitionId(grpId, partId));
 
+            assert tag == null || tag >= 0 : "Negative tag=" + tag;
+
             return tag == null ? INIT_PART_GENERATION : tag;
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
index 68fa52993cc2b..2dd81275d108e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteSpeedBasedThrottle.java
@@ -24,6 +24,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
  * Throttles threads that generate dirty pages during ongoing checkpoint.
@@ -113,10 +114,12 @@ public class PagesWriteSpeedBasedThrottle implements PagesWriteThrottlePolicy {
      * @param stateChecker Checkpoint lock state provider.
      * @param log Logger.
      */
-    public PagesWriteSpeedBasedThrottle(PageMemoryImpl pageMemory,
-        CheckpointWriteProgressSupplier cpProgress,
-        CheckpointLockStateChecker stateChecker,
-        IgniteLogger log) {
+    public PagesWriteSpeedBasedThrottle(
+            PageMemoryImpl pageMemory,
+            CheckpointWriteProgressSupplier cpProgress,
+            CheckpointLockStateChecker stateChecker,
+            IgniteLogger log
+    ) {
         this.pageMemory = pageMemory;
         this.cpProgress = cpProgress;
         totalPages = pageMemory.totalPages();
@@ -229,6 +232,11 @@ public PagesWriteSpeedBasedThrottle(PageMemoryImpl pageMemory,
      * @param throttleParkTimeNs the maximum number of nanoseconds to wait
      */
     protected void doPark(long throttleParkTimeNs) {
+        if (throttleParkTimeNs > LOGGING_THRESHOLD) {
+            U.warn(log, "Parking thread=" + Thread.currentThread().getName()
+                + " for timeout(ms)=" + (throttleParkTimeNs / 1_000_000));
+        }
+
         LockSupport.parkNanos(throttleParkTimeNs);
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
index 166cdcd6c47bb..d5f4bd5929aef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
@@ -18,8 +18,10 @@
 
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.LockSupport;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
  * Throttles threads that generate dirty pages during ongoing checkpoint.
@@ -50,21 +52,27 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
     /** Counter for checkpoint buffer usage ratio throttling (we need a separate one due to IGNITE-7751). */
     private final AtomicInteger inCheckpointBackoffCntr = new AtomicInteger(0);
 
+    /** Logger. */
+    private IgniteLogger log;
+
     /**
      * @param pageMemory Page memory.
      * @param cpProgress Database manager.
      * @param stateChecker checkpoint lock state checker.
      * @param throttleOnlyPagesInCheckpoint If true, throttle will only protect from checkpoint buffer overflow.
+     * @param log Logger.
      */
     public PagesWriteThrottle(PageMemoryImpl pageMemory,
         CheckpointWriteProgressSupplier cpProgress,
         CheckpointLockStateChecker stateChecker,
-        boolean throttleOnlyPagesInCheckpoint
+        boolean throttleOnlyPagesInCheckpoint,
+        IgniteLogger log
     ) {
         this.pageMemory = pageMemory;
         this.cpProgress = cpProgress;
         this.stateChecker = stateChecker;
         this.throttleOnlyPagesInCheckpoint = throttleOnlyPagesInCheckpoint;
+        this.log = log;
 
         if (!throttleOnlyPagesInCheckpoint)
             assert cpProgress != null : "cpProgress must be not null if ratio based throttling mode is used";
@@ -111,7 +119,14 @@ public PagesWriteThrottle(PageMemoryImpl pageMemory,
         if (shouldThrottle) {
             int throttleLevel = cntr.getAndIncrement();
 
-            LockSupport.parkNanos((long)(STARTING_THROTTLE_NANOS * Math.pow(BACKOFF_RATIO, throttleLevel)));
+            long throttleParkTimeNs = (long) (STARTING_THROTTLE_NANOS * Math.pow(BACKOFF_RATIO, throttleLevel));
+
+            if (throttleParkTimeNs > LOGGING_THRESHOLD) {
+                U.warn(log, "Parking thread=" + Thread.currentThread().getName()
+                    + " for timeout(ms)=" + (throttleParkTimeNs / 1_000_000));
+            }
+
+            LockSupport.parkNanos(throttleParkTimeNs);
         }
         else
             cntr.set(0);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottlePolicy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottlePolicy.java
index adeaa3df85ede..53a8017aea311 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottlePolicy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottlePolicy.java
@@ -17,10 +17,15 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.pagemem;
 
+import java.util.concurrent.TimeUnit;
+
 /**
  * Throttling policy, encapsulates logic of delaying write operations.
  */
 public interface PagesWriteThrottlePolicy {
+    /** Max park time. */
+    public long LOGGING_THRESHOLD = TimeUnit.SECONDS.toNanos(10);
+
     /**
      * Callback to apply throttling delay.
      * @param isPageInCheckpoint flag indicating if current page is in scope of current checkpoint.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
new file mode 100644
index 0000000000000..2b5a65dcf914f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
@@ -0,0 +1,358 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.file.OpenOption;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.LockSupport;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+/**
+ *
+ */
+public class CheckpointBufferDeadlockTest extends GridCommonAbstractTest {
+    /** Ip finder. */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** Max size. */
+    private static final int MAX_SIZE = 500 * 1024 * 1024;
+
+    /** CP buffer size. */
+    private static final int CP_BUF_SIZE = 20 * 1024 * 1024;
+
+    /** Slow checkpoint enabled. */
+    private static final AtomicBoolean slowCheckpointEnabled = new AtomicBoolean(false);
+
+    /** Checkpoint park nanos. */
+    private static final int CHECKPOINT_PARK_NANOS = 50_000_000;
+
+    /** Entry byte chunk size. */
+    private static final int ENTRY_BYTE_CHUNK_SIZE = 900;
+
+    /** Pages touched under CP lock. */
+    private static final int PAGES_TOUCHED_UNDER_CP_LOCK = 20;
+
+    /** Slop load flag. */
+    private static final AtomicBoolean stop = new AtomicBoolean(false);
+
+    /** Checkpoint threads. */
+    private int checkpointThreads;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setFileIOFactory(new SlowCheckpointFileIOFactory())
+                .setCheckpointThreads(checkpointThreads)
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setMaxSize(MAX_SIZE)
+                        .setCheckpointPageBufferSize(CP_BUF_SIZE)
+                )
+        );
+
+        cfg.setFailureHandler(new StopNodeFailureHandler());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stop.set(false);
+
+        slowCheckpointEnabled.set(false);
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stop.set(true);
+
+        slowCheckpointEnabled.set(false);
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     *
+     */
+    public void testFourCheckpointThreads() throws Exception {
+        checkpointThreads = 4;
+
+        runDeadlockScenario();
+    }
+
+    /**
+     *
+     */
+    public void testOneCheckpointThread() throws Exception {
+        checkpointThreads = 1;
+
+        runDeadlockScenario();
+    }
+
+    /**
+     *
+     */
+    private void runDeadlockScenario() throws Exception {
+        IgniteEx ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig.context().cache().context().database();
+
+        FilePageStoreManager pageStoreMgr = (FilePageStoreManager)ig.context().cache().context().pageStore();
+
+        IgniteCache<Object, Object> singlePartCache = ig.getOrCreateCache(new CacheConfiguration<>()
+            .setName("single-part")
+            .setAffinity(new RendezvousAffinityFunction(false, 1)));
+
+        db.enableCheckpoints(false).get();
+
+        Thread.sleep(1_000);
+
+        try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(singlePartCache.getName())) {
+            int entries = MAX_SIZE / ENTRY_BYTE_CHUNK_SIZE / 4;
+
+            for (int i = 0; i < entries; i++)
+                streamer.addData(i, new byte[ENTRY_BYTE_CHUNK_SIZE]);
+
+            streamer.flush();
+        }
+
+        slowCheckpointEnabled.set(true);
+        log.info(">>> Slow checkpoints enabled");
+
+        db.enableCheckpoints(true).get();
+
+        AtomicBoolean fail = new AtomicBoolean(false);
+
+        GridTestUtils.runMultiThreadedAsync(new Runnable() {
+            @Override public void run() {
+                int loops = 0;
+
+                while (!stop.get()) {
+                    if (loops % 10 == 0 && loops > 0 && loops < 500 || loops % 500 == 0 && loops >= 500)
+                        log.info("Successfully completed " + loops + " loops");
+
+                    db.checkpointReadLock();
+
+                    try {
+                        Set<FullPageId> pickedPagesSet = new HashSet<>();
+
+                        PageStore store = pageStoreMgr.getStore(CU.cacheId("single-part"), 0);
+
+                        int pages = store.pages();
+
+                        DataRegion region = db.dataRegion(DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME);
+
+                        PageMemoryImpl pageMem = (PageMemoryImpl)region.pageMemory();
+
+                        while (pickedPagesSet.size() < PAGES_TOUCHED_UNDER_CP_LOCK) {
+                            int pageIdx = ThreadLocalRandom.current().nextInt(
+                                PAGES_TOUCHED_UNDER_CP_LOCK, pages - PAGES_TOUCHED_UNDER_CP_LOCK);
+
+                            long pageId = PageIdUtils.pageId(0, PageIdAllocator.FLAG_DATA, pageIdx);
+
+                            pickedPagesSet.add(new FullPageId(pageId, CU.cacheId("single-part")));
+                        }
+
+                        List<FullPageId> pickedPages = new ArrayList<>(pickedPagesSet);
+
+                        assertEquals(PAGES_TOUCHED_UNDER_CP_LOCK, pickedPages.size());
+
+                        // Sort to avoid deadlocks on pages rw-locks.
+                        pickedPages.sort(new Comparator<FullPageId>() {
+                            @Override public int compare(FullPageId o1, FullPageId o2) {
+                                int cmp = Long.compare(o1.groupId(), o2.groupId());
+                                if (cmp != 0)
+                                    return cmp;
+
+                                return Long.compare(PageIdUtils.effectivePageId(o1.pageId()),
+                                    PageIdUtils.effectivePageId(o2.pageId()));
+                            }
+                        });
+
+                        List<Long> readLockedPages = new ArrayList<>();
+
+                        // Read lock many pages at once intentionally.
+                        for (int i = 0; i < PAGES_TOUCHED_UNDER_CP_LOCK / 2; i++) {
+                            FullPageId fpid = pickedPages.get(i);
+
+                            long page = pageMem.acquirePage(fpid.groupId(), fpid.pageId());
+
+                            long abs = pageMem.readLock(fpid.groupId(), fpid.pageId(), page);
+
+                            assertFalse(fpid.toString(), abs == 0);
+
+                            readLockedPages.add(page);
+                        }
+
+                        // Emulate writes to trigger throttling.
+                        for (int i = PAGES_TOUCHED_UNDER_CP_LOCK / 2; i < PAGES_TOUCHED_UNDER_CP_LOCK; i++) {
+                            FullPageId fpid = pickedPages.get(i);
+
+                            long page = pageMem.acquirePage(fpid.groupId(), fpid.pageId());
+
+                            long abs = pageMem.writeLock(fpid.groupId(), fpid.pageId(), page);
+
+                            assertFalse(fpid.toString(), abs == 0);
+
+                            pageMem.writeUnlock(fpid.groupId(), fpid.pageId(), page, null, true);
+
+                            pageMem.releasePage(fpid.groupId(), fpid.pageId(), page);
+                        }
+
+                        for (int i = 0; i < PAGES_TOUCHED_UNDER_CP_LOCK / 2; i++) {
+                            FullPageId fpid = pickedPages.get(i);
+
+                            pageMem.readUnlock(fpid.groupId(), fpid.pageId(), readLockedPages.get(i));
+
+                            pageMem.releasePage(fpid.groupId(), fpid.pageId(), readLockedPages.get(i));
+                        }
+                    }
+                    catch (Throwable e) {
+                        log.error("Error in loader thread", e);
+
+                        fail.set(true);
+                    }
+                    finally {
+                        db.checkpointReadUnlock();
+                    }
+
+                    loops++;
+                }
+
+            }
+        }, 10, "load-runner");
+
+        Thread.sleep(10_000); // Await for the start of throttling.
+
+        slowCheckpointEnabled.set(false);
+        log.info(">>> Slow checkpoints disabled");
+
+        assertFalse(fail.get());
+
+        forceCheckpoint(); // Previous checkpoint should eventually finish.
+    }
+
+    /**
+     * Create File I/O that emulates poor checkpoint write speed.
+     */
+    private static class SlowCheckpointFileIOFactory implements FileIOFactory {
+        /** Serial version uid. */
+        private static final long serialVersionUID = 0L;
+
+        /** Delegate factory. */
+        private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            return create(file, CREATE, READ, WRITE);
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... openOption) throws IOException {
+            final FileIO delegate = delegateFactory.create(file, openOption);
+
+            return new FileIODecorator(delegate) {
+                @Override public int write(ByteBuffer srcBuf) throws IOException {
+                    parkIfNeeded();
+
+                    return delegate.write(srcBuf);
+                }
+
+                @Override public int write(ByteBuffer srcBuf, long position) throws IOException {
+                    parkIfNeeded();
+
+                    return delegate.write(srcBuf, position);
+                }
+
+                @Override public int write(byte[] buf, int off, int len) throws IOException {
+                    parkIfNeeded();
+
+                    return delegate.write(buf, off, len);
+                }
+
+                /**
+                 * Parks current checkpoint thread if slow mode is enabled.
+                 */
+                private void parkIfNeeded() {
+                    if (slowCheckpointEnabled.get() && Thread.currentThread().getName().contains("checkpoint"))
+                        LockSupport.parkNanos(CHECKPOINT_PARK_NANOS);
+                }
+
+                /** {@inheritDoc} */
+                @Override public MappedByteBuffer map(int sizeBytes) throws IOException {
+                    return delegate.map(sizeBytes);
+                }
+            };
+        }
+    }
+
+
+}

From 692d8871fc7707fe16aebd6acc340637b9e122f1 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Thu, 12 Jul 2018 12:17:56 +0300
Subject: [PATCH 246/543] IGNITE-8863 Tx rollback can cause remote tx hang. -
 Fixes #4262.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 6440e0c)
---
 .../distributed/dht/GridDhtTxFinishRequest.java   |  6 ++++++
 .../distributed/dht/GridDhtTxPrepareRequest.java  | 15 ++++++++++-----
 .../distributed/near/GridNearTxFinishFuture.java  |  5 +----
 .../distributed/near/GridNearTxFinishRequest.java |  6 ++++++
 .../near/GridNearTxPrepareRequest.java            |  6 ++++++
 .../cache/transactions/IgniteTxHandler.java       |  6 +++---
 6 files changed, 32 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
index 90f36876556fb..823b5fee5ed9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
@@ -473,6 +474,11 @@ public void needReturnValue(boolean retVal) {
         return 27;
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return U.safeAbs(version().hashCode());
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridDhtTxFinishRequest.class, this, super.toString());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
index 12819bf67215a..88da7b011ea2c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
@@ -380,11 +380,6 @@ public boolean skipCompletedVersion() {
         }
     }
 
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return S.toString(GridDhtTxPrepareRequest.class, this, "super", super.toString());
-    }
-
     /** {@inheritDoc} */
     @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
         writer.setBuffer(buf);
@@ -612,4 +607,14 @@ public boolean skipCompletedVersion() {
     @Override public byte fieldsCount() {
         return 33;
     }
+
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return U.safeAbs(version().hashCode());
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(GridDhtTxPrepareRequest.class, this, "super", super.toString());
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index da8c6c8a9a570..ede8a4ec39484 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -757,9 +757,6 @@ private void finish(int miniId, GridDistributedTxMapping m, boolean commit, bool
         if (m.explicitLock())
             syncMode = FULL_SYNC;
 
-        // Version to be added in completed versions on primary node.
-        GridCacheVersion completedVer = !commit && useCompletedVer ? tx.xidVersion() : null;
-
         GridNearTxFinishRequest req = new GridNearTxFinishRequest(
             futId,
             tx.xidVersion(),
@@ -772,7 +769,7 @@ private void finish(int miniId, GridDistributedTxMapping m, boolean commit, bool
             m.explicitLock(),
             tx.storeEnabled(),
             tx.topologyVersion(),
-            completedVer, // Reuse 'baseVersion' to do not add new fields in message.
+            null,
             null,
             null,
             tx.size(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java
index dc322634fcb17..00c29e5f7b5c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxFinishRequest;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
 import org.apache.ignite.plugin.extensions.communication.MessageWriter;
@@ -216,6 +217,11 @@ public void miniId(int miniId) {
         return 22;
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return U.safeAbs(version().hashCode());
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return GridToStringBuilder.toString(GridNearTxFinishRequest.class, this, "super", super.toString());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
index 063eb27beb709..a6706093b11cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
 import org.apache.ignite.plugin.extensions.communication.MessageWriter;
@@ -415,6 +416,11 @@ private boolean isFlag(int mask) {
         return 26;
     }
 
+    /** {@inheritDoc} */
+    @Override public int partition() {
+        return U.safeAbs(version().hashCode());
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         StringBuilder flags = new StringBuilder();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 6bfa1e8c61979..8f2dbc05bab31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -797,9 +797,9 @@ else if (txFinishMsgLog.isDebugEnabled()) {
         if (locTx != null)
             req.txState(locTx.txState());
 
-        // 'baseVersion' message field is re-used for version to be added in completed versions.
-        if (!req.commit() && req.baseVersion() != null)
-            ctx.tm().addRolledbackTx(null, req.baseVersion());
+        // Always add near version to rollback history to prevent races with rollbacks.
+        if (!req.commit())
+            ctx.tm().addRolledbackTx(null, req.version());
 
         // Transaction on local cache only.
         if (locTx != null && !locTx.nearLocallyMapped() && !locTx.colocatedLocallyMapped())

From 8ca5d55ff43db466fba0fddea747fd69bf6fce13 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Wed, 11 Jul 2018 19:08:56 +0300
Subject: [PATCH 247/543] IGNITE-8905 Incorrect assertion in
 GridDhtPartitionsExchangeFuture - Fixes #4288.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry-picked from commit#6ad291d2285726858e67b6ee9b28a14c134247cf)
---
 .../GridCachePartitionExchangeManager.java      | 17 ++++++++++++-----
 .../GridDhtPartitionsExchangeFuture.java        |  8 ++++----
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 715c2907b2506..2bdda1906ca21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -1563,12 +1563,19 @@ else if (!grp.isLocal())
                 if (log.isDebugEnabled())
                     log.debug("Notifying exchange future about single message: " + exchFut);
 
-                if (msg.client() && !exchFut.isDone()) {
-                    if (exchFut.initialVersion().compareTo(readyAffinityVersion()) <= 0) {
+                if (msg.client()) {
+                    AffinityTopologyVersion initVer = exchFut.initialVersion();
+                    AffinityTopologyVersion readyVer = readyAffinityVersion();
+
+                    if (initVer.compareTo(readyVer) <= 0 && !exchFut.exchangeDone()) {
                         U.warn(log, "Client node tries to connect but its exchange " +
-                            "info is cleaned up from exchange history." +
-                            " Consider increasing 'IGNITE_EXCHANGE_HISTORY_SIZE' property " +
-                            "or start clients in  smaller batches."
+                            "info is cleaned up from exchange history. " +
+                            "Consider increasing 'IGNITE_EXCHANGE_HISTORY_SIZE' property " +
+                            "or start clients in  smaller batches. " +
+                            "Current settings and versions: " +
+                            "[IGNITE_EXCHANGE_HISTORY_SIZE=" + EXCHANGE_HISTORY_SIZE + ", " +
+                            "initVer=" + initVer + ", " +
+                            "readyVer=" + readyVer + "]."
                         );
 
                         exchFut.forceClientReconnect(node, msg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 1f03ff01fad59..23bfe8fad0a5f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -3189,11 +3189,11 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti
                                 if (!F.isEmpty(msg.getErrorsMap())) {
                                     Exception e = msg.getErrorsMap().get(cctx.localNodeId());
 
-                                    assert e != null : msg.getErrorsMap();
+                                    if (e instanceof IgniteNeedReconnectException) {
+                                        onDone(e);
 
-                                    onDone(e);
-
-                                    return;
+                                        return;
+                                    }
                                 }
 
                                 AffinityTopologyVersion resVer = msg.resultTopologyVersion() != null ? msg.resultTopologyVersion() : initialVersion();

From 0f9da8bd157cdb40c8ea010efa3c297182a7d56c Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Wed, 27 Jun 2018 11:30:54 +0300
Subject: [PATCH 248/543] IGNITE-8768 Fixed javadoc

(cherry picked from commit 67a2aac011d12a46cbd7d16a30f4f48b60f77075)
---
 .../cache/distributed/dht/GridDhtPartitionsReservation.java     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsReservation.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsReservation.java
index 97bd16b61bb9f..845d3edd73936 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsReservation.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsReservation.java
@@ -238,7 +238,7 @@ private void unregister() {
     }
 
     /**
-     * Must be checked in {@link GridDhtLocalPartition#tryClear()}.
+     * Must be checked in {@link GridDhtLocalPartition#tryClear(EvictionContext)}.
      * If returns {@code true} this reservation object becomes invalid and partitions
      * can be evicted or at least cleared.
      * Also this means that after returning {@code true} here method {@link #reserve()} can not

From c048b74898450f7bee1c4c9cef475bec17e7d82c Mon Sep 17 00:00:00 2001
From: Vitaliy Biryukov <BiryukovVitaliy92@gmail.com>
Date: Wed, 4 Jul 2018 19:00:53 +0300
Subject: [PATCH 249/543] IGNITE-8182 Fix for
 ZookeeperDiscoverySpiTest#testRandomTopologyChanges_RestartZk fails on TC. -
 Fixes #4192.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
(cherry picked from commit b8653002f5f298173a81ce8c0a829b2affd679a6)
---
 .../zk/ZookeeperDiscoverySpiTestSuite1.java   | 22 ++++++++++++
 .../internal/ZookeeperDiscoverySpiTest.java   | 34 ++++++++++++++-----
 2 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
index 860488b1a926f..7096cf9e1494d 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
@@ -18,14 +18,36 @@
 package org.apache.ignite.spi.discovery.zk;
 
 import junit.framework.TestSuite;
+import org.apache.curator.test.ByteCodeRewrite;
 import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClientTest;
 import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoverySpiSaslSuccessfulAuthTest;
 import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoverySpiTest;
+import org.apache.zookeeper.jmx.MBeanRegistry;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.quorum.LearnerZooKeeperServer;
 
 /**
  *
  */
 public class ZookeeperDiscoverySpiTestSuite1 extends TestSuite {
+    /**
+     * During test suite processing GC can unload some classes whose bytecode has been rewritten here
+     * {@link ByteCodeRewrite}. And the next time these classes will be loaded without bytecode rewriting.
+     *
+     * This workaround prevents unloading of these classes.
+     *
+     * @see <a href="https://github.com/Netflix/curator/issues/121">Issue link.</a>.
+     */
+    @SuppressWarnings("unused")
+    private static final Class[] WORKAROUND;
+
+    static {
+        ByteCodeRewrite.apply();
+
+        // GC will not unload this classes.
+        WORKAROUND = new Class[] {ZooKeeperServer.class, LearnerZooKeeperServer.class, MBeanRegistry.class};
+    }
+
     /**
      * @return Test suite.
      * @throws Exception Thrown in case of the failure.
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index fbacebfd5dff7..e902f42b8135f 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -50,6 +50,9 @@
 import javax.management.JMX;
 import javax.management.MBeanServer;
 import javax.management.ObjectName;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
 import org.apache.curator.test.TestingCluster;
 import org.apache.curator.test.TestingZooKeeperServer;
 import org.apache.ignite.Ignite;
@@ -427,6 +430,8 @@ private void clearAckEveryEventSystemProperty() {
             zkCluster = ZookeeperDiscoverySpiAbstractTestSuite.createTestingCluster(ZK_SRVS);
 
             zkCluster.start();
+
+            waitForZkClusterReady(zkCluster);
         }
 
         reset();
@@ -455,6 +460,22 @@ private void clearAckEveryEventSystemProperty() {
         }
     }
 
+
+    /**
+     * Wait for Zookeeper testing cluster ready for communications.
+     *
+     * @param zkCluster Zk cluster.
+     */
+    private static void waitForZkClusterReady(TestingCluster zkCluster) throws InterruptedException {
+        try (CuratorFramework curator = CuratorFrameworkFactory
+            .newClient(zkCluster.getConnectString(), new RetryNTimes(10, 1_000))) {
+            curator.start();
+
+            assertTrue("Failed to wait for Zookeeper testing cluster ready.",
+                curator.blockUntilConnected(30, SECONDS));
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -1919,13 +1940,11 @@ public void testTopologyChangeMultithreaded() throws Exception {
      * @throws Exception If failed.
      */
     public void testTopologyChangeMultithreaded_RestartZk() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-8184");
-
         try {
             topologyChangeWithRestarts(true, false);
         }
         finally {
-            zkCluster.stop();
+            zkCluster.close();
 
             zkCluster = null;
         }
@@ -1935,13 +1954,11 @@ public void testTopologyChangeMultithreaded_RestartZk() throws Exception {
      * @throws Exception If failed.
      */
     public void testTopologyChangeMultithreaded_RestartZk_CloseClients() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-8184");
-
         try {
             topologyChangeWithRestarts(true, true);
         }
         finally {
-            zkCluster.stop();
+            zkCluster.close();
 
             zkCluster = null;
         }
@@ -2134,8 +2151,6 @@ public void testRandomTopologyChanges_RestartZk() throws Exception {
      * @throws Exception If failed.
      */
     public void testRandomTopologyChanges_CloseClients() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-8182");
-
         randomTopologyChanges(false, true);
     }
 
@@ -4350,7 +4365,7 @@ private IgniteInternalFuture<?> startRestartZkServers(final long stopTime, final
                 ThreadLocalRandom rnd = ThreadLocalRandom.current();
 
                 while (!stop.get() && System.currentTimeMillis() < stopTime) {
-                    U.sleep(rnd.nextLong(2500) + 2500);
+                    U.sleep(rnd.nextLong(2500));
 
                     int idx = rnd.nextInt(ZK_SRVS);
 
@@ -4358,6 +4373,7 @@ private IgniteInternalFuture<?> startRestartZkServers(final long stopTime, final
 
                     zkCluster.getServers().get(idx).restart();
 
+                    waitForZkClusterReady(zkCluster);
                 }
 
                 return null;

From 430049a4939e9e5b2a414c65f74effd3a15d251f Mon Sep 17 00:00:00 2001
From: AMedvedev <amedvedev@gridgain.com>
Date: Fri, 13 Jul 2018 13:51:25 +0300
Subject: [PATCH 250/543] IGNITE-8957 testFailGetLock() constantly fails. Last
 entry checkpoint history can be emp - Fixes #4334.

Signed-off-by: Ivan Rakov <irakov@apache.org>
---
 .../GridCacheDatabaseSharedManager.java       | 32 +++++++--
 .../checkpoint/CheckpointHistory.java         | 67 +++++++++++--------
 .../checkpoint/IgniteMassLoadSandboxTest.java |  2 +-
 3 files changed, 66 insertions(+), 35 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 7280f96384d3e..e6778f4e86e39 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -3196,7 +3196,9 @@ private void doCheckpoint() {
 
                 if (chp.hasDelta() || destroyedPartitionsCnt > 0) {
                     if (printCheckpointStats) {
-                        if (log.isInfoEnabled())
+                        if (log.isInfoEnabled()) {
+                            String walSegsCoveredMsg = prepareWalSegsCoveredMsg(chp.walSegsCoveredRange);
+
                             log.info(String.format("Checkpoint finished [cpId=%s, pages=%d, markPos=%s, " +
                                     "walSegmentsCleared=%d, walSegmentsCovered=%s, markDuration=%dms, pagesWrite=%dms, fsync=%dms, " +
                                     "total=%dms]",
@@ -3204,11 +3206,12 @@ private void doCheckpoint() {
                                 chp.pagesSize,
                                 chp.cpEntry != null ? chp.cpEntry.checkpointMark() : "",
                                 chp.walFilesDeleted,
-                                chp.walSegmentsCovered,
+                                walSegsCoveredMsg,
                                 tracker.markDuration(),
                                 tracker.pagesWriteDuration(),
                                 tracker.fsyncDuration(),
                                 tracker.totalDuration()));
+                        }
                     }
 
                     persStoreMetrics.onCheckpoint(
@@ -3239,6 +3242,23 @@ private void doCheckpoint() {
             }
         }
 
+        /** */
+        private String prepareWalSegsCoveredMsg(IgniteBiTuple<Long, Long> walRange) {
+            String res;
+
+            long startIdx = walRange.get1();
+            long endIdx = walRange.get2();
+
+            if (endIdx < 0 || endIdx < startIdx)
+                res = "[]";
+            else if (endIdx == startIdx)
+                res = "[" + endIdx + "]";
+            else
+                res = "[" + startIdx + " - " + endIdx + "]";
+
+            return res;
+        }
+
         /**
          * Processes all evicted partitions scheduled for destroy.
          *
@@ -3918,7 +3938,7 @@ public static class Checkpoint {
         private int walFilesDeleted;
 
         /** WAL segments fully covered by this checkpoint. */
-        private List<Long> walSegmentsCovered;
+        private IgniteBiTuple<Long, Long> walSegsCoveredRange;
 
         /** */
         private final int pagesSize;
@@ -3955,10 +3975,10 @@ public void walFilesDeleted(int walFilesDeleted) {
         }
 
         /**
-         * @param walSegmentsCovered WAL segments fully covered by this checkpoint.
+         * @param walSegsCoveredRange WAL segments fully covered by this checkpoint.
          */
-        public void walSegmentsCovered(final List<Long> walSegmentsCovered) {
-            this.walSegmentsCovered = walSegmentsCovered;
+        public void walSegsCoveredRange(final IgniteBiTuple<Long, Long> walSegsCoveredRange) {
+            this.walSegsCoveredRange = walSegsCoveredRange;
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
index 95f150b2f79f8..3fb845754fdcc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointHistory.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
@@ -198,33 +199,9 @@ public List<CheckpointEntry> onWalTruncated(WALPointer ptr) {
      * @return List of checkpoints removed from history.
      */
     public List<CheckpointEntry> onCheckpointFinished(GridCacheDatabaseSharedManager.Checkpoint chp, boolean truncateWal) {
-        List<CheckpointEntry> removed = new ArrayList<>();
-
-        final Map.Entry<Long, CheckpointEntry> lastEntry = histMap.lastEntry();
-
-        assert lastEntry != null;
+        List<CheckpointEntry> rmv = new ArrayList<>();
 
-        final Map.Entry<Long, CheckpointEntry> previousEntry = histMap.lowerEntry(lastEntry.getKey());
-
-        final WALPointer lastWALPointer = lastEntry.getValue().checkpointMark();
-
-        long lastIdx = 0;
-
-        long prevIdx = 0;
-
-        final ArrayList<Long> walSegmentsCovered = new ArrayList<>();
-
-        if (lastWALPointer instanceof FileWALPointer) {
-            lastIdx = ((FileWALPointer)lastWALPointer).index();
-
-            if (previousEntry != null)
-                prevIdx = ((FileWALPointer)previousEntry.getValue().checkpointMark()).index();
-        }
-
-        for (long walCovered = prevIdx; walCovered < lastIdx; walCovered++)
-            walSegmentsCovered.add(walCovered);
-
-        chp.walSegmentsCovered(walSegmentsCovered);
+        chp.walSegsCoveredRange(calculateWalSegmentsCovered());
 
         int deleted = 0;
 
@@ -245,12 +222,46 @@ public List<CheckpointEntry> onCheckpointFinished(GridCacheDatabaseSharedManager
 
             histMap.remove(entry.getKey());
 
-            removed.add(cpEntry);
+            rmv.add(cpEntry);
         }
 
         chp.walFilesDeleted(deleted);
 
-        return removed;
+        return rmv;
+    }
+
+    /**
+     * Calculates indexes of WAL segments covered by last checkpoint.
+     *
+     * @return list of indexes or empty list if there are no checkpoints.
+     */
+    private IgniteBiTuple<Long, Long> calculateWalSegmentsCovered() {
+        IgniteBiTuple<Long, Long> tup = new IgniteBiTuple<>(-1L, -1L);
+
+        Map.Entry<Long, CheckpointEntry> lastEntry = histMap.lastEntry();
+
+        if (lastEntry == null)
+            return tup;
+
+        Map.Entry<Long, CheckpointEntry> previousEntry = histMap.lowerEntry(lastEntry.getKey());
+
+        WALPointer lastWALPointer = lastEntry.getValue().checkpointMark();
+
+        long lastIdx = 0;
+
+        long prevIdx = 0;
+
+        if (lastWALPointer instanceof FileWALPointer) {
+            lastIdx = ((FileWALPointer)lastWALPointer).index();
+
+            if (previousEntry != null)
+                prevIdx = ((FileWALPointer)previousEntry.getValue().checkpointMark()).index();
+        }
+
+        tup.set1(prevIdx);
+        tup.set2(lastIdx - 1);
+
+        return tup;
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
index 7fb277ca9f11e..79dc48e995127 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteMassLoadSandboxTest.java
@@ -377,7 +377,7 @@ public void testCoveredWalLogged() throws Exception {
                 final String coveredMatcherGrp = coveredMatcher.group(1);
 
                 final long[] covered = coveredMatcherGrp.length() > 0 ?
-                    Arrays.stream(coveredMatcherGrp.split(",")).mapToLong(e -> Integer.valueOf(e.trim())).toArray() :
+                    Arrays.stream(coveredMatcherGrp.split(" - ")).mapToLong(e -> Integer.valueOf(e.trim())).toArray() :
                     new long[0];
 
                 assertEquals(nextCovered, covered[0]);

From 4f7eb97c7d59c7dc7ecb64b3def20cfbd1c9431f Mon Sep 17 00:00:00 2001
From: ezagumennov <ezagumennov@gridgain.com>
Date: Fri, 13 Jul 2018 11:37:19 +0300
Subject: [PATCH 251/543] IGNITE-8738 Improved coordinator change information -
 Fixes #4198.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 324e610564637d243155368908964976a771e383)
---
 .../ignite/internal/managers/discovery/DiscoCache.java | 10 ++++++++++
 .../managers/discovery/GridDiscoveryManager.java       |  7 +++++++
 2 files changed, 17 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index 0bb01f3f2c3ec..73f6d2333f1cb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -309,6 +309,16 @@ public boolean baselineNode(ClusterNode node) {
         return null;
     }
 
+    /**
+     * @return Oldest server node.
+     */
+    @Nullable public ClusterNode oldestServerNode(){
+        if (srvNodes.size() > 0)
+            return srvNodes.get(0);
+
+        return null;
+    }
+
     /**
      * @param nodeId Node ID.
      * @return {@code True} if node is in alives list.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 4122fd62363d7..38ce9bdebc6a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -61,6 +61,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.failure.RestartProcessFailureHandler;
@@ -1589,6 +1590,12 @@ private void topologySnapshotMessage(IgniteClosure<String, Void> clo, long topVe
 
         clo.apply(summary);
 
+        ClusterNode currCrd = discoCache.oldestServerNode();
+
+        if ((evtType == EventType.EVT_NODE_FAILED || evtType == EventType.EVT_NODE_LEFT) &&
+                currCrd != null && currCrd.order() > evtNode.order())
+            clo.apply("Coordinator changed [prev=" + evtNode + ", cur=" + currCrd + "]");
+
         DiscoveryDataClusterState state = discoCache.state();
 
         clo.apply("  ^-- Node [id=" + discoCache.localNode().id().toString().toUpperCase() + ", clusterState="

From 8a87ecfca2aaac247aa9ca176155f5e41f6c4c5d Mon Sep 17 00:00:00 2001
From: Andrey Gura <agura@apache.org>
Date: Thu, 5 Jul 2018 19:40:26 +0300
Subject: [PATCH 252/543] IGNITE-8938 Failure handling for file-decompressor
 thread added

---
 .../wal/FileWriteAheadLogManager.java         | 114 +++++++++------
 .../FsyncModeFileWriteAheadLogManager.java    | 130 ++++++++++-------
 .../worker/WorkersControlMXBeanImpl.java      |   7 +-
 .../failure/SystemWorkersTerminationTest.java | 132 ++++++++++++++++++
 .../testsuites/IgniteBasicTestSuite.java      |   2 +
 5 files changed, 291 insertions(+), 94 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersTerminationTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 2bb6cd9ce7ef6..7cfacb2e7d84a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -443,10 +443,11 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
             if (dsCfg.isWalCompactionEnabled()) {
                 compressor = new FileCompressor();
 
-                decompressor = new FileDecompressor(log);
+                if (decompressor == null) {  // Preventing of two file-decompressor thread instantiations.
+                    decompressor = new FileDecompressor(log);
 
-                if (!cctx.kernalContext().clientNode())
                     new IgniteThread(decompressor).start();
+                }
             }
 
             if (mode != WALMode.NONE) {
@@ -1664,8 +1665,11 @@ private synchronized boolean locked(long absIdx) {
                     }
                 }
             }
-            catch (InterruptedException ignore) {
+            catch (InterruptedException t) {
                 Thread.currentThread().interrupt();
+
+                if (!stopped)
+                    err = t;
             }
             catch (Throwable t) {
                 err = t;
@@ -1675,9 +1679,9 @@ private synchronized boolean locked(long absIdx) {
                     err = new IllegalStateException("Worker " + name() + " is terminated unexpectedly");
 
                 if (err instanceof OutOfMemoryError)
-                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                    failureProcessor.process(new FailureContext(CRITICAL_ERROR, err));
                 else if (err != null)
-                    cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+                    failureProcessor.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
             }
         }
 
@@ -2138,61 +2142,81 @@ private class FileDecompressor extends GridWorker {
          * @param log Logger.
          */
         FileDecompressor(IgniteLogger log) {
-            super(cctx.igniteInstanceName(), "wal-file-decompressor%" + cctx.igniteInstanceName(), log);
+            super(cctx.igniteInstanceName(), "wal-file-decompressor%" + cctx.igniteInstanceName(), log,
+                cctx.kernalContext().workersRegistry());
         }
 
         /** {@inheritDoc} */
         @Override protected void body() {
-            while (!isCancelled()) {
-                long segmentToDecompress = -1L;
+            Throwable err = null;
 
-                try {
-                    segmentToDecompress = segmentsQueue.take();
+            try {
+                while (!isCancelled()) {
+                    long segmentToDecompress = -1L;
 
-                    if (isCancelled())
-                        break;
+                    try {
+                        segmentToDecompress = segmentsQueue.take();
 
-                    File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".zip");
-                    File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".tmp");
-                    File unzip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress));
+                        if (isCancelled())
+                            break;
 
-                    try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
-                         FileIO io = ioFactory.create(unzipTmp)) {
-                        zis.getNextEntry();
+                        File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".zip");
+                        File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".tmp");
+                        File unzip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress));
 
-                        while (io.writeFully(arr, 0, zis.read(arr)) > 0)
-                            ;
-                    }
+                        try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
+                             FileIO io = ioFactory.create(unzipTmp)) {
+                            zis.getNextEntry();
 
-                    try {
-                        Files.move(unzipTmp.toPath(), unzip.toPath());
-                    }
-                    catch (FileAlreadyExistsException e) {
-                        U.error(log, "Can't rename temporary unzipped segment: raw segment is already present " +
-                            "[tmp=" + unzipTmp + ", raw=" + unzip + "]", e);
+                            while (io.writeFully(arr, 0, zis.read(arr)) > 0)
+                                ;
+                        }
 
-                        if (!unzipTmp.delete())
-                            U.error(log, "Can't delete temporary unzipped segment [tmp=" + unzipTmp + "]");
-                    }
+                        try {
+                            Files.move(unzipTmp.toPath(), unzip.toPath());
+                        }
+                        catch (FileAlreadyExistsException e) {
+                            U.error(log, "Can't rename temporary unzipped segment: raw segment is already present " +
+                                "[tmp=" + unzipTmp + ", raw=" + unzip + "]", e);
 
-                    synchronized (this) {
-                        decompressionFutures.remove(segmentToDecompress).onDone();
-                    }
-                }
-                catch (InterruptedException ignore) {
-                    Thread.currentThread().interrupt();
-                }
-                catch (Throwable t) {
-                    if (!isCancelled && segmentToDecompress != -1L) {
-                        IgniteCheckedException e = new IgniteCheckedException("Error during WAL segment " +
-                            "decompression [segmentIdx=" + segmentToDecompress + "]", t);
+                            if (!unzipTmp.delete())
+                                U.error(log, "Can't delete temporary unzipped segment [tmp=" + unzipTmp + "]");
+                        }
 
                         synchronized (this) {
-                            decompressionFutures.remove(segmentToDecompress).onDone(e);
+                            decompressionFutures.remove(segmentToDecompress).onDone();
+                        }
+                    }
+                    catch (IOException ex) {
+                        if (!isCancelled && segmentToDecompress != -1L) {
+                            IgniteCheckedException e = new IgniteCheckedException("Error during WAL segment " +
+                                "decompression [segmentIdx=" + segmentToDecompress + "]", ex);
+
+                            synchronized (this) {
+                                decompressionFutures.remove(segmentToDecompress).onDone(e);
+                            }
                         }
                     }
                 }
             }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+
+                if (!isCancelled)
+                    err = e;
+            }
+            catch (Throwable t) {
+                err = t;
+            }
+            finally {
+                if (err == null && !isCancelled)
+                    err = new IllegalStateException("Worker " + name() + " is terminated unexpectedly");
+
+                if (err instanceof OutOfMemoryError)
+                    failureProcessor.process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    failureProcessor.process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+            }
         }
 
         /**
@@ -2218,10 +2242,8 @@ synchronized IgniteInternalFuture<Void> decompressFile(long idx) {
             return res;
         }
 
-        /**
-         * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
-         */
-        private void shutdown() throws IgniteInterruptedCheckedException {
+        /** */
+        private void shutdown() {
             synchronized (this) {
                 U.cancel(this);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index a75dd3100bf3f..f1a60a4c64dae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -380,7 +380,11 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
             if (dsCfg.isWalCompactionEnabled()) {
                 compressor = new FileCompressor();
 
-                decompressor = new FileDecompressor(log);
+                if (decompressor == null) {  // Preventing of two file-decompressor thread instantiations.
+                    decompressor = new FileDecompressor(log);
+
+                    new IgniteThread(decompressor).start();
+                }
             }
 
             if (mode != WALMode.NONE) {
@@ -452,15 +456,14 @@ private void checkWalConfiguration() throws IgniteCheckedException {
         start0();
 
         if (!cctx.kernalContext().clientNode()) {
-            assert archiver != null;
+            if (isArchiverEnabled()) {
+                assert archiver != null;
 
-            new IgniteThread(archiver).start();
+                new IgniteThread(archiver).start();
+            }
 
             if (compressor != null)
                 compressor.start();
-
-            if (decompressor != null)
-                new IgniteThread(decompressor).start();
         }
     }
 
@@ -549,6 +552,18 @@ private void scheduleNextInactivityPeriodElapsedCheck() {
         cctx.time().addTimeoutObject(nextAutoArchiveTimeoutObj);
     }
 
+    /**
+     * Archiver can be not created, all files will be written to WAL folder, using absolute segment index.
+     *
+     * @return flag indicating if archiver is disabled.
+     */
+    private boolean isArchiverEnabled() {
+        if (walArchiveDir != null && walWorkDir != null)
+            return !walArchiveDir.equals(walWorkDir);
+
+        return !new File(dsCfg.getWalArchivePath()).equals(new File(dsCfg.getWalPath()));
+    }
+
     /**
      *  Collect wal segment files from low pointer (include) to high pointer (not include) and reserve low pointer.
      *
@@ -1494,8 +1509,11 @@ private synchronized void release(long absIdx) {
                     }
                 }
             }
-            catch (InterruptedException ignore) {
+            catch (InterruptedException t) {
                 Thread.currentThread().interrupt();
+
+                if (!stopped)
+                    err = t;
             }
             catch (Throwable t) {
                 err = t;
@@ -1945,61 +1963,81 @@ private class FileDecompressor extends GridWorker {
          * @param log Logger.
          */
         FileDecompressor(IgniteLogger log) {
-            super(cctx.igniteInstanceName(), "wal-file-decompressor%" + cctx.igniteInstanceName(), log);
+            super(cctx.igniteInstanceName(), "wal-file-decompressor%" + cctx.igniteInstanceName(), log,
+                cctx.kernalContext().workersRegistry());
         }
 
         /** {@inheritDoc} */
         @Override protected void body() {
-            while (!isCancelled()) {
-                long segmentToDecompress = -1L;
+            Throwable err = null;
 
-                try {
-                    segmentToDecompress = segmentsQueue.take();
+            try {
+                while (!isCancelled()) {
+                    long segmentToDecompress = -1L;
 
-                    if (isCancelled())
-                        break;
+                    try {
+                        segmentToDecompress = segmentsQueue.take();
 
-                    File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".zip");
-                    File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".tmp");
-                    File unzip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress));
+                        if (isCancelled())
+                            break;
 
-                    try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
-                         FileIO io = ioFactory.create(unzipTmp)) {
-                        zis.getNextEntry();
+                        File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".zip");
+                        File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".tmp");
+                        File unzip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress));
 
-                        while (io.writeFully(arr, 0, zis.read(arr)) > 0)
-                            ;
-                    }
+                        try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
+                             FileIO io = ioFactory.create(unzipTmp)) {
+                            zis.getNextEntry();
 
-                    try {
-                        Files.move(unzipTmp.toPath(), unzip.toPath());
-                    }
-                    catch (FileAlreadyExistsException e) {
-                        U.error(log, "Can't rename temporary unzipped segment: raw segment is already present " +
-                            "[tmp=" + unzipTmp + ", raw=" + unzip + ']', e);
+                            while (io.writeFully(arr, 0, zis.read(arr)) > 0)
+                                ;
+                        }
 
-                        if (!unzipTmp.delete())
-                            U.error(log, "Can't delete temporary unzipped segment [tmp=" + unzipTmp + ']');
-                    }
+                        try {
+                            Files.move(unzipTmp.toPath(), unzip.toPath());
+                        }
+                        catch (FileAlreadyExistsException e) {
+                            U.error(log, "Can't rename temporary unzipped segment: raw segment is already present " +
+                                "[tmp=" + unzipTmp + ", raw=" + unzip + ']', e);
 
-                    synchronized (this) {
-                        decompressionFutures.remove(segmentToDecompress).onDone();
-                    }
-                }
-                catch (InterruptedException ignore) {
-                    Thread.currentThread().interrupt();
-                }
-                catch (Throwable t) {
-                    if (!isCancelled && segmentToDecompress != -1L) {
-                        IgniteCheckedException e = new IgniteCheckedException("Error during WAL segment " +
-                            "decompression [segmentIdx=" + segmentToDecompress + ']', t);
+                            if (!unzipTmp.delete())
+                                U.error(log, "Can't delete temporary unzipped segment [tmp=" + unzipTmp + ']');
+                        }
 
                         synchronized (this) {
-                            decompressionFutures.remove(segmentToDecompress).onDone(e);
+                            decompressionFutures.remove(segmentToDecompress).onDone();
+                        }
+                    }
+                    catch (IOException ex) {
+                        if (!isCancelled && segmentToDecompress != -1L) {
+                            IgniteCheckedException e = new IgniteCheckedException("Error during WAL segment " +
+                                "decompression [segmentIdx=" + segmentToDecompress + ']', ex);
+
+                            synchronized (this) {
+                                decompressionFutures.remove(segmentToDecompress).onDone(e);
+                            }
                         }
                     }
                 }
             }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+
+                if (!isCancelled)
+                    err = e;
+            }
+            catch (Throwable t) {
+                err = t;
+            }
+            finally {
+                if (err == null && !isCancelled)
+                    err = new IllegalStateException("Worker " + name() + " is terminated unexpectedly");
+
+                if (err instanceof OutOfMemoryError)
+                    cctx.kernalContext().failure().process(new FailureContext(CRITICAL_ERROR, err));
+                else if (err != null)
+                    cctx.kernalContext().failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, err));
+            }
         }
 
         /**
@@ -2025,9 +2063,7 @@ synchronized IgniteInternalFuture<Void> decompressFile(long idx) {
             return res;
         }
 
-        /**
-         * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
-         */
+        /** */
         private void shutdown() {
             synchronized (this) {
                 U.cancel(this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
index 65f872c0162ca..1f082b53ea4dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/worker/WorkersControlMXBeanImpl.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.worker;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import org.apache.ignite.internal.util.worker.GridWorker;
@@ -41,7 +42,11 @@ public WorkersControlMXBeanImpl(WorkersRegistry registry) {
 
     /** {@inheritDoc} */
     @Override public List<String> getWorkerNames() {
-        return new ArrayList<>(workerRegistry.names());
+        List<String> names = new ArrayList<>(workerRegistry.names());
+
+        Collections.sort(names);
+
+        return names;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersTerminationTest.java b/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersTerminationTest.java
new file mode 100644
index 0000000000000..0df870dbac041
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/failure/SystemWorkersTerminationTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.failure;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.internal.worker.WorkersRegistry;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests system critical workers termination.
+ */
+public class SystemWorkersTerminationTest extends GridCommonAbstractTest {
+    /** Handler latch. */
+    private static volatile CountDownLatch hndLatch;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setFailureHandler(new TestFailureHandler());
+
+        DataRegionConfiguration drCfg = new DataRegionConfiguration();
+        drCfg.setPersistenceEnabled(true);
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+        dsCfg.setDefaultDataRegionConfiguration(drCfg);
+        dsCfg.setWalCompactionEnabled(true);
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        deleteWorkFiles();
+
+        startGrid(0);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        deleteWorkFiles();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTermination() throws Exception {
+        Ignite ignite = ignite(0);
+
+        ignite.cluster().active(true);
+
+        WorkersRegistry registry = ((IgniteKernal)ignite).context().workersRegistry();
+
+        Collection<String> threadNames = new ArrayList<>(registry.names());
+
+        int cnt = 0;
+
+        for (String threadName : threadNames) {
+            log.info("Worker termination: " + threadName);
+
+            hndLatch = new CountDownLatch(1);
+
+            GridWorker w = registry.worker(threadName);
+
+            Thread t = w.runner();
+
+            t.interrupt();
+
+            assertTrue(hndLatch.await(3, TimeUnit.SECONDS));
+
+            log.info("Worker is terminated: " + threadName);
+
+            cnt++;
+        }
+
+        assertEquals(threadNames.size(), cnt);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    private void deleteWorkFiles() throws Exception {
+        cleanPersistenceDir();
+
+        U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "snapshot", false));
+    }
+
+    /**
+     * Test failure handler.
+     */
+    private class TestFailureHandler implements FailureHandler {
+        /** {@inheritDoc} */
+        @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
+            hndLatch.countDown();
+
+            return false;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index eaaff3f24369b..576bd5a64a242 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.failure.OomFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest;
+import org.apache.ignite.failure.SystemWorkersTerminationTest;
 import org.apache.ignite.internal.ClassSetTest;
 import org.apache.ignite.internal.ClusterGroupHostsSelfTest;
 import org.apache.ignite.internal.ClusterGroupSelfTest;
@@ -208,6 +209,7 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(IoomFailureHandlerTest.class);
         suite.addTestSuite(OomFailureHandlerTest.class);
         suite.addTestSuite(AccountTransferTransactionTest.class);
+        suite.addTestSuite(SystemWorkersTerminationTest.class);
 
         suite.addTestSuite(CacheRebalanceConfigValidationTest.class);
 

From 18584d7d158bc285072fcf82ffeb5583a7fbbada Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Mon, 16 Jul 2018 13:14:00 +0300
Subject: [PATCH 253/543] IGNITE-8995 Proper handlig of exceptions from scan
 query filter and transformer

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../cache/query/GridCacheQueryManager.java    |  88 ++++++---
 .../query/CacheScanQueryFailoverTest.java     | 174 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite.java        |   2 +
 3 files changed, 242 insertions(+), 22 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/CacheScanQueryFailoverTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index f3105116c47b5..8f0edb73b3231 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -804,6 +804,7 @@ private GridCloseableIterator scanIterator(final GridCacheQueryAdapter<?> qry, I
         boolean locNode)
         throws IgniteCheckedException {
         final IgniteBiPredicate<K, V> keyValFilter = qry.scanFilter();
+        final InternalScanFilter<K,V> intFilter = keyValFilter != null ? new InternalScanFilter<>(keyValFilter) : null;
 
         try {
             injectResources(keyValFilter);
@@ -813,7 +814,9 @@ private GridCloseableIterator scanIterator(final GridCacheQueryAdapter<?> qry, I
             if (part != null && (part < 0 || part >= cctx.affinity().partitions()))
                 return new GridEmptyCloseableIterator() {
                     @Override public void close() throws IgniteCheckedException {
-                        closeScanFilter(keyValFilter);
+                        if (intFilter != null)
+                            intFilter.close();
+
                         super.close();
                     }
                 };
@@ -852,22 +855,13 @@ private GridCloseableIterator scanIterator(final GridCacheQueryAdapter<?> qry, I
             return new ScanQueryIterator(it, qry, topVer, locPart, keyValFilter, transformer, locNode, cctx, log);
         }
         catch (IgniteCheckedException | RuntimeException e) {
-            closeScanFilter(keyValFilter);
+            if (intFilter != null)
+                intFilter.close();
 
             throw e;
         }
     }
 
-    /**
-     * Closes a filter if it is closeable.
-     *
-     * @param f Filter.
-     */
-    private static void closeScanFilter(Object f) {
-        if (f instanceof PlatformCacheEntryFilter)
-            ((PlatformCacheEntryFilter)f).onClose();
-    }
-
     /**
      * @param o Object to inject resources to.
      * @throws IgniteCheckedException If failure occurred while injecting resources.
@@ -1378,7 +1372,8 @@ protected GridCloseableIterator scanQueryLocal(final GridCacheQueryAdapter qry,
 
         final String namex = cctx.name();
 
-        final IgniteBiPredicate<K, V> scanFilter = qry.scanFilter();
+        final InternalScanFilter<K, V> intFilter = qry.scanFilter() != null ?
+                new InternalScanFilter<>(qry.scanFilter()) : null;
 
         try {
             assert qry.type() == SCAN;
@@ -1399,7 +1394,7 @@ protected GridCloseableIterator scanQueryLocal(final GridCacheQueryAdapter qry,
                     namex,
                     null,
                     null,
-                    scanFilter,
+                    intFilter != null ? intFilter.scanFilter() : null,
                     null,
                     null,
                     subjId,
@@ -1417,7 +1412,8 @@ protected GridCloseableIterator scanQueryLocal(final GridCacheQueryAdapter qry,
             return it;
         }
         catch (Exception e) {
-            closeScanFilter(scanFilter);
+            if (intFilter != null)
+                intFilter.close();
 
             if (updateStatistics)
                 cctx.queries().collectMetrics(GridCacheQueryType.SCAN, namex, startTime,
@@ -2850,7 +2846,7 @@ private static final class ScanQueryIterator<K, V> extends GridCloseableIterator
         private final GridDhtLocalPartition locPart;
 
         /** */
-        private final IgniteBiPredicate<K, V> scanFilter;
+        private final InternalScanFilter<K, V> intScanFilter;
 
         /** */
         private final boolean statsEnabled;
@@ -2930,7 +2926,7 @@ private static final class ScanQueryIterator<K, V> extends GridCloseableIterator
             this.it = it;
             this.topVer = topVer;
             this.locPart = locPart;
-            this.scanFilter = scanFilter;
+            this.intScanFilter = scanFilter != null ? new InternalScanFilter<>(scanFilter) : null;
             this.cctx = cctx;
             this.log = log;
             this.locNode = locNode;
@@ -2998,7 +2994,8 @@ private static final class ScanQueryIterator<K, V> extends GridCloseableIterator
             if (locPart != null)
                 locPart.release();
 
-            closeScanFilter(scanFilter);
+            if (intScanFilter != null)
+                intScanFilter.close();
         }
 
         /**
@@ -3086,7 +3083,7 @@ private void advance() {
                         metrics.addGetTimeNanos(System.nanoTime() - start);
                     }
 
-                    if (scanFilter == null || scanFilter.apply(key0, val0)) {
+                    if (intScanFilter == null || intScanFilter.apply(key0, val0)) {
                         if (readEvt) {
                             cctx.gridEvents().record(new CacheQueryReadEvent<>(
                                 cctx.localNode(),
@@ -3096,7 +3093,7 @@ private void advance() {
                                 cacheName,
                                 null,
                                 null,
-                                scanFilter,
+                                intScanFilter != null ? intScanFilter.scanFilter() : null,
                                 null,
                                 null,
                                 subjId,
@@ -3107,8 +3104,14 @@ private void advance() {
                                 null));
                         }
 
-                        if (transform != null)
-                            next0 = transform.apply(new CacheQueryEntry<>(key0, val0));
+                        if (transform != null) {
+                            try {
+                                next0 = transform.apply(new CacheQueryEntry<>(key0, val0));
+                            }
+                            catch (Throwable e) {
+                                throw new IgniteException(e);
+                            }
+                        }
                         else
                             next0 = !locNode ? new GridCacheQueryResponseEntry<>(key0, val0):
                                 new CacheQueryEntry<>(key0, val0);
@@ -3125,4 +3128,45 @@ private void advance() {
             }
         }
     }
+
+    /**
+     * Wrap scan filter in order to catch unhandled errors.
+     */
+    private static class InternalScanFilter<K, V> implements IgniteBiPredicate<K, V> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private final IgniteBiPredicate<K, V> scanFilter;
+
+        /**
+         * @param scanFilter User scan filter.
+         */
+        InternalScanFilter(IgniteBiPredicate<K, V> scanFilter) {
+            this.scanFilter = scanFilter;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(K k, V v){
+            try {
+                return scanFilter == null || scanFilter.apply(k, v);
+            }
+            catch (Throwable e) {
+                throw new IgniteException(e);
+            }
+        }
+
+        /** */
+        void close() {
+            if (scanFilter instanceof PlatformCacheEntryFilter)
+                ((PlatformCacheEntryFilter)scanFilter).onClose();
+        }
+
+        /**
+         * @return Wrapped scan filter.
+         */
+        IgniteBiPredicate<K, V> scanFilter() {
+            return scanFilter;
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/CacheScanQueryFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/CacheScanQueryFailoverTest.java
new file mode 100644
index 0000000000000..0633138f0c245
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/CacheScanQueryFailoverTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.query;
+
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteBinary;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheMode.LOCAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
+/**
+ * ScanQuery failover test. Tests scenario where user supplied closures throw unhandled errors.
+ */
+public class CacheScanQueryFailoverTest extends GridCommonAbstractTest {
+    /** */
+    private static final String LOCAL_CACHE_NAME = "local";
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean isMultiJvm() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean isRemoteJvm(String igniteInstanceName) {
+        if(igniteInstanceName.equals("client") || igniteInstanceName.equals("server"))
+            return false;
+        else
+            return super.isRemoteJvm(igniteInstanceName);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(name);
+
+        if (name.equals("client"))
+            cfg.setClientMode(true);
+
+        cfg.setFailureHandler(new StopNodeOrHaltFailureHandler());
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testScanQueryWithFailedClosures() throws Exception {
+        Ignite srv = startGrids(4);
+        Ignite client = startGrid("client");
+
+        CacheConfiguration cfg = new CacheConfiguration(DEFAULT_CACHE_NAME).setCacheMode(PARTITIONED);
+
+        // Test query from client node.
+        queryCachesWithFailedPredicates(client, cfg);
+
+        // Test query from server node.
+        queryCachesWithFailedPredicates(srv, cfg);
+
+        assertEquals(client.cluster().nodes().size(), 5);
+    };
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testScanQueryOverLocalCacheWithFailedClosures() throws Exception {
+        Ignite srv = startGrids(4);
+
+        queryCachesWithFailedPredicates(srv, new CacheConfiguration(LOCAL_CACHE_NAME).setCacheMode(LOCAL));
+
+        assertEquals(srv.cluster().nodes().size(), 4);
+    };
+
+    /**
+     * @param ignite Ignite instance.
+     * @param configs Cache configurations.
+     */
+    private void queryCachesWithFailedPredicates(Ignite ignite, CacheConfiguration... configs) {
+        if (configs == null)
+            return;
+
+        for (CacheConfiguration cfg: configs) {
+            IgniteCache cache = ignite.getOrCreateCache(cfg);
+
+            populateCache(ignite, cache.getName());
+
+            // Check that exception propagates to client from filter failure.
+            GridTestUtils.assertThrowsAnyCause(log, () -> {
+                try (QueryCursor<Cache.Entry<Integer, BinaryObject>> cursor =
+                             cache.withKeepBinary().query(new ScanQuery<>(filter))) {
+                    for (Cache.Entry<Integer, BinaryObject> entry : cursor)
+                        log.info("Entry " + entry.toString());
+                }
+
+                return null;
+            }, Error.class, "Poison pill");
+
+            // Check that exception propagates to client from transformer failure.
+            GridTestUtils.assertThrowsAnyCause(log, () -> {
+                try (QueryCursor<Cache.Entry<Integer, BinaryObject>> cursor =
+                             cache.withKeepBinary().query(new ScanQuery<>(), transformer)) {
+                    for (Cache.Entry<Integer, BinaryObject> entry : cursor)
+                        log.info("Entry " + entry.toString());
+                }
+
+                return null;
+            }, Error.class, "Poison pill");
+        }
+    }
+
+    /**
+     * @param ignite Ignite instance.
+     * @param cacheName Cache name.
+     */
+    private void populateCache(Ignite ignite, String cacheName) {
+        IgniteBinary binary = ignite.binary();
+
+        try (IgniteDataStreamer<Object, Object> streamer = ignite.dataStreamer(cacheName)) {
+            for (int i = 0; i < 1_000; i++)
+                streamer.addData(i, binary.builder("type_name").setField("f_" + i, "v_" + i).build());
+        }
+    }
+
+    /** Failed filter. */
+    private static IgniteBiPredicate<Integer, BinaryObject> filter = (key, value) -> {
+            throw new Error("Poison pill");
+        };
+
+    /** Failed entry transformer. */
+    private static IgniteClosure<Cache.Entry<Integer, BinaryObject>, Cache.Entry<Integer, BinaryObject>> transformer =
+            integerBinaryObjectEntry -> {
+                throw new Error("Poison pill");
+        };
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 8a88602768177..1ebe654e2ee6b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -136,6 +136,7 @@
 import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalQueryCancelOrTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.local.IgniteCacheLocalQuerySelfTest;
+import org.apache.ignite.internal.processors.cache.query.CacheScanQueryFailoverTest;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryTransformerSelfTest;
 import org.apache.ignite.internal.processors.cache.query.IgniteCacheQueryCacheDestroySelfTest;
 import org.apache.ignite.internal.processors.cache.query.IndexingSpiQuerySelfTest;
@@ -272,6 +273,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheQueryH2IndexingLeakTest.class);
         suite.addTestSuite(IgniteCacheQueryNoRebalanceSelfTest.class);
         suite.addTestSuite(GridCacheQueryTransformerSelfTest.class);
+        suite.addTestSuite(CacheScanQueryFailoverTest.class);
         suite.addTestSuite(IgniteCachePrimitiveFieldsQuerySelfTest.class);
 
         suite.addTestSuite(IgniteCacheJoinQueryWithAffinityKeyTest.class);

From 1505b240c03e9fb626a6abf0009f37b0f2c105d5 Mon Sep 17 00:00:00 2001
From: Slava Koptilin <slava.koptilin@gmail.com>
Date: Mon, 16 Jul 2018 16:40:56 +0300
Subject: [PATCH 254/543] IGNITE-1094 Fixed hanging during cache initialization

Signed-off-by: Andrey Gura <agura@apache.org>
---
 .../apache/ignite/internal/IgniteKernal.java  |   5 +
 .../ignite/internal/IgniteNodeAttributes.java |   3 +
 .../managers/discovery/DiscoCache.java        |  20 +
 .../cache/CacheAffinitySharedManager.java     | 148 +++-
 .../processors/cache/ClusterCachesInfo.java   |  25 +
 .../DynamicCacheChangeFailureMessage.java     | 151 ++++
 .../processors/cache/ExchangeActions.java     |  19 +-
 .../GridCachePartitionExchangeManager.java    |   8 +
 .../processors/cache/GridCacheProcessor.java  | 169 ++--
 .../GridDhtPartitionsExchangeFuture.java      | 281 ++++++-
 ...niteAbstractDynamicCacheStartFailTest.java | 775 ++++++++++++++++++
 ...amicCacheStartCoordinatorFailoverTest.java | 262 ++++++
 .../IgniteDynamicCacheStartFailTest.java      |  46 ++
 ...amicCacheStartFailWithPersistenceTest.java |  91 ++
 .../testsuites/IgniteCacheTestSuite4.java     |   6 +
 ...ueryAfterDynamicCacheStartFailureTest.java |  69 ++
 .../IgniteCacheWithIndexingTestSuite.java     |   3 +
 17 files changed, 1920 insertions(+), 161 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartCoordinatorFailoverTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheQueryAfterDynamicCacheStartFailureTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 343c621b73e0c..3c78927dc6c12 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -242,6 +242,7 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DATA_STORAGE_CONFIG;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DATA_STREAMER_POOL_SIZE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DEPLOYMENT_MODE;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DYNAMIC_CACHE_START_ROLLBACK_SUPPORTED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IPS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JIT_NAME;
@@ -1675,6 +1676,10 @@ private void fillNodeAttributes(boolean notifyEnabled) throws IgniteCheckedExcep
         if (cfg.getConnectorConfiguration() != null)
             add(ATTR_REST_PORT_RANGE, cfg.getConnectorConfiguration().getPortRange());
 
+        // Whether rollback of dynamic cache start is supported or not.
+        // This property is added because of backward compatibility.
+        add(ATTR_DYNAMIC_CACHE_START_ROLLBACK_SUPPORTED, Boolean.TRUE);
+
         // Save data storage configuration.
         addDataStorageConfigurationAttributes();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
index 663a6f9ad244e..ed16a7715b5b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteNodeAttributes.java
@@ -202,6 +202,9 @@ public final class IgniteNodeAttributes {
     /** Rebalance thread pool size. */
     public static final String ATTR_REBALANCE_POOL_SIZE = ATTR_PREFIX + ".rebalance.pool.size";
 
+    /** Internal attribute name constant. */
+    public static final String ATTR_DYNAMIC_CACHE_START_ROLLBACK_SUPPORTED = ATTR_PREFIX + ".dynamic.cache.start.rollback.supported";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
index 73f6d2333f1cb..8cdcbf367330d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/DiscoCache.java
@@ -423,6 +423,26 @@ else if (cmp > 0)
         return -(low + 1);
     }
 
+    /**
+     *
+     * Returns {@code True} if all nodes has the given attribute and its value equals to {@code expVal}.
+     *
+     * @param <T> Attribute Type.
+     * @param name Attribute name.
+     * @param expVal Expected value.
+     * @return {@code True} if all the given nodes has the given attribute and its value equals to {@code expVal}.
+     */
+    public <T> boolean checkAttribute(String name, T expVal) {
+        for (ClusterNode node : allNodes) {
+            T attr = node.attribute(name);
+
+            if (attr == null || !expVal.equals(attr))
+                return false;
+        }
+
+        return true;
+    }
+
     /**
      * @param nodes Cluster nodes.
      * @return Empty collection if nodes list is {@code null}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 08eb43f5291c8..2871e8294c179 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -730,6 +730,28 @@ public void stopCacheGroupOnReconnect(CacheGroupContext grpCtx) {
         caches.registeredGrps.remove(grpCtx.groupId());
     }
 
+    /**
+     * Called during the rollback of the exchange partitions procedure
+     * in order to stop the given cache even if it's not fully initialized (e.g. failed on cache init stage).
+     *
+     * @param fut Exchange future.
+     * @param crd Coordinator flag.
+     * @param exchActions Cache change requests.
+     */
+    public void forceCloseCaches(
+        GridDhtPartitionsExchangeFuture fut,
+        boolean crd,
+        final ExchangeActions exchActions
+    ) {
+        assert exchActions != null && !exchActions.empty() && exchActions.cacheStartRequests().isEmpty(): exchActions;
+
+        caches.updateCachesInfo(exchActions);
+
+        processCacheStopRequests(fut, crd, exchActions, true);
+
+        cctx.cache().forceCloseCaches(exchActions);
+    }
+
     /**
      * Called on exchange initiated for cache start/stop request.
      *
@@ -745,13 +767,70 @@ public void onCacheChangeRequest(
     ) throws IgniteCheckedException {
         assert exchActions != null && !exchActions.empty() : exchActions;
 
-        final ExchangeDiscoveryEvents evts = fut.context().events();
-
         caches.updateCachesInfo(exchActions);
 
         // Affinity did not change for existing caches.
         onCustomMessageNoAffinityChange(fut, crd, exchActions);
 
+        processCacheStartRequests(fut, crd, exchActions);
+
+        Set<Integer> stoppedGrps = processCacheStopRequests(fut, crd, exchActions, false);
+
+        if (stoppedGrps != null) {
+            AffinityTopologyVersion notifyTopVer = null;
+
+            synchronized (mux) {
+                if (waitInfo != null) {
+                    for (Integer grpId : stoppedGrps) {
+                        boolean rmv = waitInfo.waitGrps.remove(grpId) != null;
+
+                        if (rmv) {
+                            notifyTopVer = waitInfo.topVer;
+
+                            waitInfo.assignments.remove(grpId);
+                        }
+                    }
+                }
+            }
+
+            if (notifyTopVer != null) {
+                final AffinityTopologyVersion topVer = notifyTopVer;
+
+                cctx.kernalContext().closure().runLocalSafe(new Runnable() {
+                    @Override public void run() {
+                        onCacheGroupStopped(topVer);
+                    }
+                });
+            }
+        }
+
+        ClientCacheChangeDiscoveryMessage msg = clientCacheChanges.get();
+
+        if (msg != null) {
+            msg.checkCachesExist(caches.registeredCaches.keySet());
+
+            if (msg.empty())
+                clientCacheChanges.remove();
+        }
+    }
+
+    /**
+     * Process cache start requests.
+     *
+     * @param fut Exchange future.
+     * @param crd Coordinator flag.
+     * @param exchActions Cache change requests.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void processCacheStartRequests(
+        GridDhtPartitionsExchangeFuture fut,
+        boolean crd,
+        final ExchangeActions exchActions
+    ) throws IgniteCheckedException {
+        assert exchActions != null && !exchActions.empty() : exchActions;
+
+        final ExchangeDiscoveryEvents evts = fut.context().events();
+
         for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) {
             DynamicCacheDescriptor cacheDesc = action.descriptor();
 
@@ -830,6 +909,24 @@ public void onCacheChangeRequest(
                 }
             }
         }
+    }
+
+    /**
+     * Process cache stop requests.
+     *
+     * @param fut Exchange future.
+     * @param crd Coordinator flag.
+     * @param exchActions Cache change requests.
+     * @param forceClose
+     * @return Set of cache groups to be stopped.
+     */
+    private Set<Integer> processCacheStopRequests(
+        GridDhtPartitionsExchangeFuture fut,
+        boolean crd,
+        final ExchangeActions exchActions,
+        boolean forceClose
+    ) {
+        assert exchActions != null && !exchActions.empty() : exchActions;
 
         for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests())
             cctx.cache().blockGateway(action.request().cacheName(), true, action.request().restart());
@@ -844,54 +941,21 @@ public void onCacheChangeRequest(
                 if (data.descriptor().config().getCacheMode() != LOCAL) {
                     CacheGroupHolder cacheGrp = grpHolders.remove(data.descriptor().groupId());
 
-                    assert cacheGrp != null : data.descriptor();
-
-                    if (stoppedGrps == null)
-                        stoppedGrps = new HashSet<>();
-
-                    stoppedGrps.add(cacheGrp.groupId());
-
-                    cctx.io().removeHandler(true, cacheGrp.groupId(), GridDhtAffinityAssignmentResponse.class);
-                }
-            }
-        }
-
-        if (stoppedGrps != null) {
-            AffinityTopologyVersion notifyTopVer = null;
+                    assert cacheGrp != null || forceClose : data.descriptor();
 
-            synchronized (mux) {
-                if (waitInfo != null) {
-                    for (Integer grpId : stoppedGrps) {
-                        boolean rmv = waitInfo.waitGrps.remove(grpId) != null;
+                    if (cacheGrp != null) {
+                        if (stoppedGrps == null)
+                            stoppedGrps = new HashSet<>();
 
-                        if (rmv) {
-                            notifyTopVer = waitInfo.topVer;
+                        stoppedGrps.add(cacheGrp.groupId());
 
-                            waitInfo.assignments.remove(grpId);
-                        }
+                        cctx.io().removeHandler(true, cacheGrp.groupId(), GridDhtAffinityAssignmentResponse.class);
                     }
                 }
             }
-
-            if (notifyTopVer != null) {
-                final AffinityTopologyVersion topVer = notifyTopVer;
-
-                cctx.kernalContext().closure().runLocalSafe(new Runnable() {
-                    @Override public void run() {
-                        onCacheGroupStopped(topVer);
-                    }
-                });
-            }
         }
 
-        ClientCacheChangeDiscoveryMessage msg = clientCacheChanges.get();
-
-        if (msg != null) {
-            msg.checkCachesExist(caches.registeredCaches.keySet());
-
-            if (msg.empty())
-                clientCacheChanges.remove();
-        }
+        return stoppedGrps;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 9ff84e7d510cd..97fae16339fd0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -395,6 +395,31 @@ public void onClientCacheChange(ClientCacheChangeDiscoveryMessage msg, ClusterNo
         }
     }
 
+    /**
+     * Creates exchanges actions. Forms a list of caches and cache groups to be stopped
+     * due to dynamic cache start failure.
+     *
+     * @param failMsg Dynamic change request fail message.
+     * @param topVer Topology version.
+     */
+    public void onCacheChangeRequested(DynamicCacheChangeFailureMessage failMsg, AffinityTopologyVersion topVer) {
+        ExchangeActions exchangeActions = new ExchangeActions();
+
+        List<DynamicCacheChangeRequest> requests = new ArrayList<>(failMsg.cacheNames().size());
+
+        for (String cacheName : failMsg.cacheNames()) {
+            DynamicCacheDescriptor cacheDescr = registeredCaches.get(cacheName);
+
+            assert cacheDescr != null : "Dynamic cache descriptor is missing [cacheName=" + cacheName + "]";
+
+            requests.add(DynamicCacheChangeRequest.stopRequest(ctx, cacheName, cacheDescr.sql(), true));
+        }
+
+        processCacheChangeRequests(exchangeActions, requests, topVer,false);
+
+        failMsg.exchangeActions(exchangeActions);
+    }
+
     /**
      * @param batch Cache change request.
      * @param topVer Topology version.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
new file mode 100644
index 0000000000000..d0cb08da4a1e1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/DynamicCacheChangeFailureMessage.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collection;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * This class represents discovery message that is used to provide information about dynamic cache start failure.
+ */
+public class DynamicCacheChangeFailureMessage implements DiscoveryCustomMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Cache names. */
+    @GridToStringInclude
+    private Collection<String> cacheNames;
+
+    /** Custom message ID. */
+    private IgniteUuid id;
+
+    /** */
+    private GridDhtPartitionExchangeId exchId;
+
+    /** */
+    @GridToStringInclude
+    private IgniteCheckedException cause;
+
+    /** Cache updates to be executed on exchange. */
+    private transient ExchangeActions exchangeActions;
+
+    /**
+     * Creates new DynamicCacheChangeFailureMessage instance.
+     *
+     * @param locNode Local node.
+     * @param exchId Exchange Id.
+     * @param cause Cache start error.
+     * @param cacheNames Cache names.
+     */
+    public DynamicCacheChangeFailureMessage(
+        ClusterNode locNode,
+        GridDhtPartitionExchangeId exchId,
+        IgniteCheckedException cause,
+        Collection<String> cacheNames)
+    {
+        assert exchId != null;
+        assert cause != null;
+        assert !F.isEmpty(cacheNames) : cacheNames;
+
+        this.id = IgniteUuid.fromUuid(locNode.id());
+        this.exchId = exchId;
+        this.cause = cause;
+        this.cacheNames = cacheNames;
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteUuid id() {
+        return id;
+    }
+
+    /**
+     * @return Collection of failed caches.
+     */
+    public Collection<String> cacheNames() {
+        return cacheNames;
+    }
+
+    /**
+     * @return Cache start error.
+     */
+    public IgniteCheckedException error() {
+        return cause;
+    }
+
+    /**
+     * @return Cache updates to be executed on exchange.
+     */
+    public ExchangeActions exchangeActions() {
+        return exchangeActions;
+    }
+
+    /**
+     * @param exchangeActions Cache updates to be executed on exchange.
+     */
+    public void exchangeActions(ExchangeActions exchangeActions) {
+        assert exchangeActions != null && !exchangeActions.empty() : exchangeActions;
+
+        this.exchangeActions = exchangeActions;
+    }
+
+    /**
+     * @return Exchange version.
+     */
+    @Nullable public GridDhtPartitionExchangeId exchangeId() {
+        return exchId;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public DiscoveryCustomMessage ackMessage() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isMutable() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean stopProcess() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public DiscoCache createDiscoCache(
+        GridDiscoveryManager mgr,
+        AffinityTopologyVersion topVer,
+        DiscoCache discoCache) {
+        return mgr.createDiscoCacheOnCacheChange(topVer, discoCache);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DynamicCacheChangeFailureMessage.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
index c289b6e31e7f4..6431d0f6c21a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeActions.java
@@ -99,17 +99,18 @@ public Collection<CacheActionData> cacheStartRequests() {
     /**
      * @return Stop cache requests.
      */
-    Collection<CacheActionData> cacheStopRequests() {
+    public Collection<CacheActionData> cacheStopRequests() {
         return cachesToStop != null ? cachesToStop.values() : Collections.<CacheActionData>emptyList();
     }
 
     /**
      * @param ctx Context.
+     * @param err Error if any.
      */
-    public void completeRequestFutures(GridCacheSharedContext ctx) {
-        completeRequestFutures(cachesToStart, ctx);
-        completeRequestFutures(cachesToStop, ctx);
-        completeRequestFutures(cachesToResetLostParts, ctx);
+    public void completeRequestFutures(GridCacheSharedContext ctx, Throwable err) {
+        completeRequestFutures(cachesToStart, ctx, err);
+        completeRequestFutures(cachesToStop, ctx, err);
+        completeRequestFutures(cachesToResetLostParts, ctx, err);
     }
 
     /**
@@ -130,10 +131,14 @@ public boolean systemCachesStarting() {
      * @param map Actions map.
      * @param ctx Context.
      */
-    private void completeRequestFutures(Map<String, CacheActionData> map, GridCacheSharedContext ctx) {
+    private void completeRequestFutures(
+        Map<String, CacheActionData> map,
+        GridCacheSharedContext ctx,
+        @Nullable Throwable err
+    ) {
         if (map != null) {
             for (CacheActionData req : map.values())
-                ctx.cache().completeCacheStartFuture(req.req, true, null);
+                ctx.cache().completeCacheStartFuture(req.req, (err == null), err);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 2bdda1906ca21..d3fddabef7936 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -494,6 +494,14 @@ else if (msg.exchangeId().topologyVersion().topologyVersion() >= cctx.discovery(
                     exchangeFuture(msg.exchangeId(), null, null, null, null)
                         .onAffinityChangeMessage(evt.eventNode(), msg);
             }
+            else if (customMsg instanceof DynamicCacheChangeFailureMessage) {
+                DynamicCacheChangeFailureMessage msg = (DynamicCacheChangeFailureMessage) customMsg;
+
+                if (msg.exchangeId().topologyVersion().topologyVersion() >=
+                    affinityTopologyVersion(cctx.discovery().localJoinEvent()).topologyVersion())
+                    exchangeFuture(msg.exchangeId(), null, null, null, null)
+                        .onDynamicCacheChangeFail(evt.eventNode(), msg);
+            }
             else if (customMsg instanceof SnapshotDiscoveryMessage
                 && ((SnapshotDiscoveryMessage) customMsg).needExchange()) {
                 exchId = exchangeId(n.id(), affinityTopologyVersion(evt), evt);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index a0e61b5fb88e5..d392c6819b66c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2098,7 +2098,6 @@ void blockGateway(String cacheName, boolean stop, boolean restart) {
             }
             else
                 proxy.closeProxy();
-
         }
     }
 
@@ -2138,6 +2137,8 @@ private void stopGateway(DynamicCacheChangeRequest req) {
      * @return Stopped cache context.
      */
     private GridCacheContext<?, ?> prepareCacheStop(String cacheName, boolean destroy) {
+        assert sharedCtx.database().checkpointLockIsHeldByThread();
+
         GridCacheAdapter<?, ?> cache = caches.remove(cacheName);
 
         if (cache != null) {
@@ -2243,7 +2244,14 @@ private void closeCache(GridCacheContext cctx, boolean destroy) {
 
             cctx.gate().onStopped();
 
-            prepareCacheStop(cctx.name(), destroy);
+            sharedCtx.database().checkpointReadLock();
+
+            try {
+                prepareCacheStop(cctx.name(), destroy);
+            }
+            finally {
+                sharedCtx.database().checkpointReadUnlock();
+            }
 
             if (!cctx.group().hasCaches())
                 stopCacheGroup(cctx.group().groupId());
@@ -2251,101 +2259,119 @@ private void closeCache(GridCacheContext cctx, boolean destroy) {
     }
 
     /**
-     * Callback invoked when first exchange future for dynamic cache is completed.
+     * Called during the rollback of the exchange partitions procedure
+     * in order to stop the given cache even if it's not fully initialized (e.g. failed on cache init stage).
      *
-     * @param cacheStartVer Started caches version to create proxy for.
-     * @param exchActions Change requests.
-     * @param err Error.
+     * @param exchActions Stop requests.
      */
-    @SuppressWarnings("unchecked")
-    public void onExchangeDone(
-        AffinityTopologyVersion cacheStartVer,
-        @Nullable ExchangeActions exchActions,
-        @Nullable Throwable err
-    ) {
-        initCacheProxies(cacheStartVer, err);
-
-        if (exchActions == null)
-            return;
-
-        if (exchActions.systemCachesStarting() && exchActions.stateChangeRequest() == null) {
-            ctx.dataStructures().restoreStructuresState(ctx);
+    void forceCloseCaches(ExchangeActions exchActions) {
+        assert exchActions != null && !exchActions.cacheStopRequests().isEmpty();
 
-            ctx.service().updateUtilityCache();
-        }
+        processCacheStopRequestOnExchangeDone(exchActions);
+    }
 
-        if (err == null) {
-            // Force checkpoint if there is any cache stop request
-            if (exchActions.cacheStopRequests().size() > 0) {
-                try {
-                    sharedCtx.database().waitForCheckpoint("caches stop");
-                }
-                catch (IgniteCheckedException e) {
-                    U.error(log, "Failed to wait for checkpoint finish during cache stop.", e);
-                }
+    /**
+     * @param exchActions Change requests.
+     */
+    private void processCacheStopRequestOnExchangeDone(ExchangeActions exchActions) {
+        // Force checkpoint if there is any cache stop request
+        if (exchActions.cacheStopRequests().size() > 0) {
+            try {
+                sharedCtx.database().waitForCheckpoint("caches stop");
             }
+            catch (IgniteCheckedException e) {
+                U.error(log, "Failed to wait for checkpoint finish during cache stop.", e);
+            }
+        }
 
-            for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) {
-                CacheGroupContext gctx = cacheGrps.get(action.descriptor().groupId());
-
-                // Cancel all operations blocking gateway
-                if (gctx != null) {
-                    final String msg = "Failed to wait for topology update, cache group is stopping.";
-
-                    // If snapshot operation in progress we must throw CacheStoppedException
-                    // for correct cache proxy restart. For more details see
-                    // IgniteCacheProxy.cacheException()
-                    gctx.affinity().cancelFutures(new CacheStoppedException(msg));
-                }
-
-                stopGateway(action.request());
+        for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) {
+            CacheGroupContext gctx = cacheGrps.get(action.descriptor().groupId());
 
-                sharedCtx.database().checkpointReadLock();
+            // Cancel all operations blocking gateway
+            if (gctx != null) {
+                final String msg = "Failed to wait for topology update, cache group is stopping.";
 
-                try {
-                    prepareCacheStop(action.request().cacheName(), action.request().destroy());
-                }
-                finally {
-                    sharedCtx.database().checkpointReadUnlock();
-                }
+                // If snapshot operation in progress we must throw CacheStoppedException
+                // for correct cache proxy restart. For more details see
+                // IgniteCacheProxy.cacheException()
+                gctx.affinity().cancelFutures(new CacheStoppedException(msg));
             }
 
+            stopGateway(action.request());
+
             sharedCtx.database().checkpointReadLock();
 
             try {
-                // Do not invoke checkpoint listeners for groups are going to be destroyed to prevent metadata corruption.
-                for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) {
-                    Integer groupId = action.descriptor().groupId();
-                    CacheGroupContext grp = cacheGrps.get(groupId);
-
-                    if (grp != null && grp.persistenceEnabled() && sharedCtx.database() instanceof GridCacheDatabaseSharedManager) {
-                        GridCacheDatabaseSharedManager mngr = (GridCacheDatabaseSharedManager) sharedCtx.database();
-                        mngr.removeCheckpointListener((DbCheckpointListener) grp.offheap());
-                    }
-                }
+                prepareCacheStop(action.request().cacheName(), action.request().destroy());
             }
             finally {
                 sharedCtx.database().checkpointReadUnlock();
             }
+        }
 
-            List<IgniteBiTuple<CacheGroupContext, Boolean>> stoppedGroups = new ArrayList<>();
+        sharedCtx.database().checkpointReadLock();
 
+        try {
+            // Do not invoke checkpoint listeners for groups are going to be destroyed to prevent metadata corruption.
             for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) {
                 Integer groupId = action.descriptor().groupId();
+                CacheGroupContext grp = cacheGrps.get(groupId);
 
-                if (cacheGrps.containsKey(groupId)) {
-                    stoppedGroups.add(F.t(cacheGrps.get(groupId), action.destroy()));
-
-                    stopCacheGroup(groupId);
+                if (grp != null && grp.persistenceEnabled() && sharedCtx.database() instanceof GridCacheDatabaseSharedManager) {
+                    GridCacheDatabaseSharedManager mngr = (GridCacheDatabaseSharedManager) sharedCtx.database();
+                    mngr.removeCheckpointListener((DbCheckpointListener) grp.offheap());
                 }
             }
+        }
+        finally {
+            sharedCtx.database().checkpointReadUnlock();
+        }
+
+        List<IgniteBiTuple<CacheGroupContext, Boolean>> stoppedGroups = new ArrayList<>();
+
+        for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) {
+            Integer groupId = action.descriptor().groupId();
+
+            if (cacheGrps.containsKey(groupId)) {
+                stoppedGroups.add(F.t(cacheGrps.get(groupId), action.destroy()));
+
+                stopCacheGroup(groupId);
+            }
+        }
+
+        if (!sharedCtx.kernalContext().clientNode())
+            sharedCtx.database().onCacheGroupsStopped(stoppedGroups);
+
+        if (exchActions.deactivate())
+            sharedCtx.deactivate();
+    }
+
+    /**
+     * Callback invoked when first exchange future for dynamic cache is completed.
+     *
+     * @param cacheStartVer Started caches version to create proxy for.
+     * @param exchActions Change requests.
+     * @param err Error.
+     */
+    @SuppressWarnings("unchecked")
+    public void onExchangeDone(
+        AffinityTopologyVersion cacheStartVer,
+        @Nullable ExchangeActions exchActions,
+        @Nullable Throwable err
+    ) {
+        initCacheProxies(cacheStartVer, err);
 
-            if (!sharedCtx.kernalContext().clientNode())
-                sharedCtx.database().onCacheGroupsStopped(stoppedGroups);
+        if (exchActions == null)
+            return;
 
-            if (exchActions.deactivate())
-                sharedCtx.deactivate();
+        if (exchActions.systemCachesStarting() && exchActions.stateChangeRequest() == null) {
+            ctx.dataStructures().restoreStructuresState(ctx);
+
+            ctx.service().updateUtilityCache();
         }
+
+        if (err == null)
+            processCacheStopRequestOnExchangeDone(exchActions);
     }
 
     /**
@@ -3476,6 +3502,9 @@ else if (msg0 instanceof WalStateFinishMessage)
         if (msg instanceof DynamicCacheChangeBatch)
             return cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer);
 
+        if (msg instanceof DynamicCacheChangeFailureMessage)
+            cachesInfo.onCacheChangeRequested((DynamicCacheChangeFailureMessage) msg, topVer);
+
         if (msg instanceof ClientCacheChangeDiscoveryMessage)
             cachesInfo.onClientCacheChange((ClientCacheChangeDiscoveryMessage)msg, node);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 23bfe8fad0a5f..50a6f979d25ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -59,7 +59,6 @@
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord;
-import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.CacheAffinityChangeMessage;
@@ -67,6 +66,7 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
 import org.apache.ignite.internal.processors.cache.CachePartitionExchangeWorkerTask;
 import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch;
+import org.apache.ignite.internal.processors.cache.DynamicCacheChangeFailureMessage;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.ExchangeActions;
 import org.apache.ignite.internal.processors.cache.ExchangeContext;
@@ -78,13 +78,13 @@
 import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
 import org.apache.ignite.internal.processors.cache.StateChangeRequest;
 import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsStateValidator;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -116,6 +116,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DYNAMIC_CACHE_START_ROLLBACK_SUPPORTED;
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
 import static org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents.serverJoinEvent;
@@ -138,6 +139,9 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     /** */
     private static final IgniteProductVersion FORCE_AFF_REASSIGNMENT_SINCE = IgniteProductVersion.fromString("2.4.3");
 
+    /** */
+    private static final String DISTRIBUTED_LATCH_ID = "exchange";
+
     /** */
     @GridToStringExclude
     private final Object mux = new Object();
@@ -245,11 +249,14 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
      */
     private boolean forceAffReassignment;
 
-    /** Change global state exception. */
-    private Exception changeGlobalStateE;
+    /** Exception that was thrown during init phase on local node. */
+    private Exception exchangeLocE;
 
-    /** Change global state exceptions. */
-    private final Map<UUID, Exception> changeGlobalStateExceptions = new ConcurrentHashMap<>();
+    /** Exchange exceptions from all participating nodes. */
+    private final Map<UUID, Exception> exchangeGlobalExceptions = new ConcurrentHashMap<>();
+
+    /** Used to track the fact that {@link DynamicCacheChangeFailureMessage} was sent. */
+    private volatile boolean cacheChangeFailureMsgSent;
 
     /** */
     private ConcurrentMap<UUID, GridDhtPartitionsSingleMessage> msgs = new ConcurrentHashMap<>();
@@ -499,6 +506,15 @@ private boolean stateChangeExchange() {
         return exchActions != null && exchActions.stateChangeRequest() != null;
     }
 
+    /**
+     * @return {@code True} if this exchange was triggered by DynamicCacheChangeBatch message
+     * in order to start cache(s).
+     */
+    private boolean dynamicCacheStartExchange() {
+        return exchActions != null && !exchActions.cacheStartRequests().isEmpty()
+            && exchActions.cacheStopRequests().isEmpty();
+    }
+
     /**
      * @return {@code True} if activate cluster exchange.
      */
@@ -765,7 +781,7 @@ else if (msg instanceof WalStateAbstractMessage)
             if (reconnectOnError(e))
                 onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
             else {
-                U.error(log, "Failed to reinitialize local partitions (preloading will be stopped): " + exchId, e);
+                U.error(log, "Failed to reinitialize local partitions (rebalancing will be stopped): " + exchId, e);
 
                 onDone(e);
             }
@@ -898,7 +914,7 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
         DiscoveryDataClusterState state = cctx.kernalContext().state().clusterState();
 
         if (state.transitionError() != null)
-            changeGlobalStateE = state.transitionError();
+            exchangeLocE = state.transitionError();
 
         if (req.activeChanged()) {
             if (req.activate()) {
@@ -938,11 +954,11 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
                         ", client=" + cctx.kernalContext().clientNode() +
                         ", topVer=" + initialVersion() + "]", e);
 
-                    changeGlobalStateE = e;
+                    exchangeLocE = e;
 
                     if (crd) {
                         synchronized (mux) {
-                            changeGlobalStateExceptions.put(cctx.localNodeId(), e);
+                            exchangeGlobalExceptions.put(cctx.localNodeId(), e);
                         }
                     }
                 }
@@ -973,7 +989,7 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
                         ", client=" + cctx.kernalContext().clientNode() +
                         ", topVer=" + initialVersion() + "]", e);
 
-                    changeGlobalStateE = e;
+                    exchangeLocE = e;
                 }
             }
         }
@@ -998,7 +1014,7 @@ assert firstEventCache().minimumNodeVersion()
                     ", client=" + cctx.kernalContext().clientNode() +
                     ", topVer=" + initialVersion() + "]", e);
 
-                changeGlobalStateE = e;
+                exchangeLocE = e;
             }
         }
 
@@ -1015,7 +1031,21 @@ private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedExcep
 
         assert !exchActions.clientOnlyExchange() : exchActions;
 
-        cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
+        try {
+            cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
+        }
+        catch (Exception e) {
+            if (reconnectOnError(e) || !isRollbackSupported())
+                // This exception will be handled by init() method.
+                throw e;
+
+            U.error(log, "Failed to initialize cache(s) (will try to rollback). " + exchId, e);
+
+            exchangeLocE = new IgniteCheckedException(
+                "Failed to initialize exchange locally [locNodeId=" + cctx.localNodeId() + "]", e);
+
+            exchangeGlobalExceptions.put(cctx.localNodeId(), exchangeLocE);
+        }
 
         return cctx.kernalContext().clientNode() ? ExchangeType.CLIENT : ExchangeType.ALL;
     }
@@ -1265,8 +1295,9 @@ private void changeWalModeIfNeeded() {
     private void waitPartitionRelease(boolean distributed) throws IgniteCheckedException {
         Latch releaseLatch = null;
 
+        // Wait for other nodes only on first phase.
         if (distributed)
-            releaseLatch = cctx.exchange().latch().getOrCreate("exchange", initialVersion());
+            releaseLatch = cctx.exchange().latch().getOrCreate(DISTRIBUTED_LATCH_ID, initialVersion());
 
         IgniteInternalFuture<?> partReleaseFut = cctx.partitionReleaseFuture(initialVersion());
 
@@ -1476,9 +1507,9 @@ private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException
                 resetLostPartitions(caches);
         }
 
-        if (cctx.kernalContext().clientNode()) {
+        if (cctx.kernalContext().clientNode() || (dynamicCacheStartExchange() && exchangeLocE != null)) {
             msg = new GridDhtPartitionsSingleMessage(exchangeId(),
-                true,
+                cctx.kernalContext().clientNode(),
                 cctx.versions().last(),
                 true);
         }
@@ -1495,8 +1526,8 @@ private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException
                 msg.partitionHistoryCounters(partHistReserved0);
         }
 
-        if (stateChangeExchange() && changeGlobalStateE != null)
-            msg.setError(changeGlobalStateE);
+        if ((stateChangeExchange() || dynamicCacheStartExchange()) && exchangeLocE != null)
+            msg.setError(exchangeLocE);
         else if (localJoinExchange())
             msg.cacheGroupsAffinityRequest(exchCtx.groupsAffinityRequestOnJoin());
 
@@ -1529,8 +1560,8 @@ private GridDhtPartitionsFullMessage createPartitionsMessage(boolean compress,
             partHistSuppliers,
             partsToReload);
 
-        if (stateChangeExchange() && !F.isEmpty(changeGlobalStateExceptions))
-            m.setErrorsMap(changeGlobalStateExceptions);
+        if (stateChangeExchange() && !F.isEmpty(exchangeGlobalExceptions))
+            m.setErrorsMap(exchangeGlobalExceptions);
 
         return m;
     }
@@ -1729,7 +1760,7 @@ public void finishMerged() {
         cctx.kernalContext().authentication().onActivate();
 
         if (exchActions != null && err == null)
-            exchActions.completeRequestFutures(cctx);
+            exchActions.completeRequestFutures(cctx, null);
 
         if (stateChangeExchange() && err == null)
             cctx.kernalContext().state().onStateChangeExchangeDone(exchActions.stateChangeRequest());
@@ -1842,15 +1873,15 @@ public void cleanUp() {
         pendingSingleMsgs.clear();
         fullMsgs.clear();
         msgs.clear();
-        changeGlobalStateExceptions.clear();
         crd = null;
         partReleaseFut = null;
-        changeGlobalStateE = null;
         exchActions = null;
         mergedJoinExchMsgs = null;
         pendingJoinMsg = null;
         exchCtx = null;
         newCrdFut = null;
+        exchangeLocE = null;
+        exchangeGlobalExceptions.clear();
     }
 
     /**
@@ -2055,13 +2086,13 @@ private void processMergedMessage(final ClusterNode node, final GridDhtPartition
     public void forceClientReconnect(ClusterNode node, GridDhtPartitionsSingleMessage msg) {
         Exception e = new IgniteNeedReconnectException(node, null);
 
-        changeGlobalStateExceptions.put(node.id(), e);
+        exchangeGlobalExceptions.put(node.id(), e);
 
         onDone(null, e);
 
         GridDhtPartitionsFullMessage fullMsg = createPartitionsMessage(true, false);
 
-        fullMsg.setErrorsMap(changeGlobalStateExceptions);
+        fullMsg.setErrorsMap(exchangeGlobalExceptions);
 
         FinishState finishState0 = new FinishState(cctx.localNodeId(),
             initialVersion(),
@@ -2152,6 +2183,11 @@ public void waitAndReplyToNode(final UUID nodeId, final GridDhtPartitionsSingleM
                 if (cctx.kernalContext().isStopping())
                     return;
 
+                // DynamicCacheChangeFailureMessage was sent.
+                // Thus, there is no need to create and send GridDhtPartitionsFullMessage.
+                if (cacheChangeFailureMsgSent)
+                    return;
+
                 FinishState finishState0;
 
                 synchronized (mux) {
@@ -2220,8 +2256,8 @@ private void processSingleMessage(UUID nodeId, GridDhtPartitionsSingleMessage ms
 
                         pendingSingleUpdates++;
 
-                        if (stateChangeExchange() && msg.getError() != null)
-                            changeGlobalStateExceptions.put(nodeId, msg.getError());
+                        if ((stateChangeExchange() || dynamicCacheStartExchange()) && msg.getError() != null)
+                            exchangeGlobalExceptions.put(nodeId, msg.getError());
 
                         allReceived = remaining.isEmpty();
 
@@ -2253,7 +2289,10 @@ private void processSingleMessage(UUID nodeId, GridDhtPartitionsSingleMessage ms
         }
 
         if (finishState0 != null) {
-            sendAllPartitionsToNode(finishState0, msg, nodeId);
+            // DynamicCacheChangeFailureMessage was sent.
+            // Thus, there is no need to create and send GridDhtPartitionsFullMessage.
+            if (!cacheChangeFailureMsgSent)
+                sendAllPartitionsToNode(finishState0, msg, nodeId);
 
             return;
         }
@@ -2556,6 +2595,69 @@ private void resetLostPartitions(Collection<String> cacheNames) {
         }
     }
 
+    /**
+     * Creates an IgniteCheckedException that is used as root cause of the exchange initialization failure.
+     * This method aggregates all the exceptions provided from all participating nodes.
+     *
+     * @param globalExceptions collection exceptions from all participating nodes.
+     * @return exception that represents a cause of the exchange initialization failure.
+     */
+    private IgniteCheckedException createExchangeException(Map<UUID, Exception> globalExceptions) {
+        IgniteCheckedException ex = new IgniteCheckedException("Failed to complete exchange process.");
+
+        for (Map.Entry<UUID, Exception> entry : globalExceptions.entrySet())
+            if (ex != entry.getValue())
+                ex.addSuppressed(entry.getValue());
+
+        return ex;
+    }
+
+    /**
+     * @return {@code true} if the given {@code discoEvt} supports the rollback procedure.
+     */
+    private boolean isRollbackSupported() {
+        if (!firstEvtDiscoCache.checkAttribute(ATTR_DYNAMIC_CACHE_START_ROLLBACK_SUPPORTED, Boolean.TRUE))
+            return false;
+
+        // Currently the rollback process is supported for dynamically started caches only.
+        return firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && dynamicCacheStartExchange();
+    }
+
+    /**
+     * Sends {@link DynamicCacheChangeFailureMessage} to all participated nodes
+     * that represents a cause of exchange failure.
+     */
+    private void sendExchangeFailureMessage() {
+        assert crd != null && crd.isLocal();
+
+        try {
+            IgniteCheckedException err = createExchangeException(exchangeGlobalExceptions);
+
+            List<String> cacheNames = new ArrayList<>(exchActions.cacheStartRequests().size());
+
+            for (ExchangeActions.CacheActionData actionData : exchActions.cacheStartRequests())
+                cacheNames.add(actionData.request().cacheName());
+
+            DynamicCacheChangeFailureMessage msg = new DynamicCacheChangeFailureMessage(
+                cctx.localNode(), exchId, err, cacheNames);
+
+            if (log.isDebugEnabled())
+                log.debug("Dynamic cache change failed (send message to all participating nodes): " + msg);
+
+            cacheChangeFailureMsgSent = true;
+
+            cctx.discovery().sendCustomEvent(msg);
+
+            return;
+        }
+        catch (IgniteCheckedException  e) {
+            if (reconnectOnError(e))
+                onDone(new IgniteNeedReconnectException(cctx.localNode(), e));
+            else
+                onDone(e);
+        }
+    }
+
     /**
      * @param sndResNodes Additional nodes to send finish message to.
      */
@@ -2567,8 +2669,16 @@ private void onAllReceived(@Nullable Collection<ClusterNode> sndResNodes) {
 
             if (!exchCtx.mergeExchanges() && !crd.equals(events().discoveryCache().serverNodes().get(0))) {
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
-                    if (!grp.isLocal())
+                    if (grp.isLocal())
+                        continue;
+
+                    // It is possible affinity is not initialized.
+                    // For example, dynamic cache start failed.
+                    if (grp.affinity().lastVersion().topologyVersion() > 0)
                         grp.topology().beforeExchange(this, !centralizedAff && !forceAffReassignment, false);
+                    else
+                        assert exchangeLocE != null :
+                            "Affinity is not calculated for the cache group [groupName=" + grp.name() + "]";
                 }
             }
 
@@ -2603,6 +2713,12 @@ private void onAllReceived(@Nullable Collection<ClusterNode> sndResNodes) {
      */
     private void finishExchangeOnCoordinator(@Nullable Collection<ClusterNode> sndResNodes) {
         try {
+            if (!F.isEmpty(exchangeGlobalExceptions) && dynamicCacheStartExchange() && isRollbackSupported()) {
+                sendExchangeFailureMessage();
+
+                return;
+            }
+
             AffinityTopologyVersion resTopVer = exchCtx.events().topologyVersion();
 
             if (log.isInfoEnabled()) {
@@ -2819,12 +2935,12 @@ else if (forceAffReassignment)
 
                 boolean stateChangeErr = false;
 
-                if (!F.isEmpty(changeGlobalStateExceptions)) {
+                if (!F.isEmpty(exchangeGlobalExceptions)) {
                     stateChangeErr = true;
 
                     err = new IgniteCheckedException("Cluster state change failed.");
 
-                    cctx.kernalContext().state().onStateChangeError(changeGlobalStateExceptions, req);
+                    cctx.kernalContext().state().onStateChangeError(exchangeGlobalExceptions, req);
                 }
                 else {
                     boolean hasMoving = !partsToReload.isEmpty();
@@ -3091,15 +3207,27 @@ private void processSinglePartitionRequest(ClusterNode node, GridDhtPartitionsSi
             try {
                 assert msg.restoreExchangeId() != null : msg;
 
-                GridDhtPartitionsSingleMessage res = cctx.exchange().createPartitionsSingleMessage(
-                    msg.restoreExchangeId(),
-                    cctx.kernalContext().clientNode(),
-                    true,
-                    node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0,
-                    exchActions);
+                GridDhtPartitionsSingleMessage res;
 
-                if (localJoinExchange() && finishState0 == null)
-                    res.cacheGroupsAffinityRequest(exchCtx.groupsAffinityRequestOnJoin());
+                if (dynamicCacheStartExchange() && exchangeLocE != null) {
+                    res = new GridDhtPartitionsSingleMessage(msg.restoreExchangeId(),
+                        cctx.kernalContext().clientNode(),
+                        cctx.versions().last(),
+                        true);
+
+                    res.setError(exchangeLocE);
+                }
+                else {
+                    res = cctx.exchange().createPartitionsSingleMessage(
+                        msg.restoreExchangeId(),
+                        cctx.kernalContext().clientNode(),
+                        true,
+                        node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0,
+                        exchActions);
+
+                    if (localJoinExchange() && finishState0 == null)
+                        res.cacheGroupsAffinityRequest(exchCtx.groupsAffinityRequestOnJoin());
+                }
 
                 res.restoreState(true);
 
@@ -3264,6 +3392,21 @@ else if (localJoinExchange() && !exchCtx.fetchAffinityOnJoin())
             else if (forceAffReassignment)
                 cctx.affinity().applyAffinityFromFullMessage(this, msg);
 
+            if (dynamicCacheStartExchange() && !F.isEmpty(exchangeGlobalExceptions)) {
+                assert cctx.localNode().isClient();
+
+                // TODO: https://issues.apache.org/jira/browse/IGNITE-8796
+                // The current exchange has been successfully completed on all server nodes,
+                // but has failed on that client node for some reason.
+                // It looks like that we need to rollback dynamically started caches on the client node,
+                // complete DynamicCacheStartFutures (if they are registered) with the cause of that failure
+                // and complete current exchange without errors.
+
+                onDone(exchangeLocE);
+
+                return;
+            }
+
             updatePartitionFullMap(resTopVer, msg);
 
             IgniteCheckedException err = null;
@@ -3350,6 +3493,57 @@ private void updatePartitionSingleMap(UUID nodeId, GridDhtPartitionsSingleMessag
         }
     }
 
+    /**
+     * Cache change failure message callback, processed from the discovery thread.
+     *
+     * @param node Message sender node.
+     * @param msg Failure message.
+     */
+    public void onDynamicCacheChangeFail(final ClusterNode node, final DynamicCacheChangeFailureMessage msg) {
+        assert exchId.equals(msg.exchangeId()) : msg;
+        assert firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT && dynamicCacheStartExchange();
+
+        final ExchangeActions actions = exchangeActions();
+
+        onDiscoveryEvent(new IgniteRunnable() {
+            @Override public void run() {
+                // The rollbackExchange() method has to wait for checkpoint.
+                // That operation is time consumed, and therefore it should be executed outside the discovery thread.
+                cctx.kernalContext().getSystemExecutorService().submit(new Runnable() {
+                    @Override public void run() {
+                        if (isDone() || !enterBusy())
+                            return;
+
+                        try {
+                            assert msg.error() != null: msg;
+
+                            // Try to revert all the changes that were done during initialization phase
+                            cctx.affinity().forceCloseCaches(GridDhtPartitionsExchangeFuture.this,
+                                crd.isLocal(), msg.exchangeActions());
+
+                            synchronized (mux) {
+                                finishState = new FinishState(crd.id(), initialVersion(), null);
+
+                                state = ExchangeLocalState.DONE;
+                            }
+
+                            if (actions != null)
+                                actions.completeRequestFutures(cctx, msg.error());
+
+                            onDone(exchId.topologyVersion());
+                        }
+                        catch (Throwable e) {
+                            onDone(e);
+                        }
+                        finally {
+                            leaveBusy();
+                        }
+                    }
+                });
+            }
+        });
+    }
+
     /**
      * Affinity change message callback, processed from the same thread as {@link #onNodeLeft}.
      *
@@ -3563,8 +3757,8 @@ public void onNodeLeft(final ClusterNode node) {
                         }
 
                         if (crd0.isLocal()) {
-                            if (stateChangeExchange() && changeGlobalStateE != null)
-                                changeGlobalStateExceptions.put(crd0.id(), changeGlobalStateE);
+                            if (stateChangeExchange() && exchangeLocE != null)
+                                exchangeGlobalExceptions.put(crd0.id(), exchangeLocE);
 
                             if (crdChanged) {
                                 if (log.isInfoEnabled()) {
@@ -3739,6 +3933,9 @@ private void onBecomeCoordinator(InitNewCoordinatorFuture newCrdFut) {
                     if (!msg.client()) {
                         msgs.put(e.getKey().id(), e.getValue());
 
+                        if (dynamicCacheStartExchange() && msg.getError() != null)
+                            exchangeGlobalExceptions.put(e.getKey().id(), msg.getError());
+
                         updatePartitionSingleMap(e.getKey().id(), msg);
                     }
                 }
@@ -4006,7 +4203,7 @@ private enum ExchangeLocalState {
         CLIENT,
 
         /**
-         * Previous coordinator failed before echange finished and
+         * Previous coordinator failed before exchange finished and
          * local performs initialization to become new coordinator.
          */
         BECOME_CRD,
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java
new file mode 100644
index 0000000000000..9b9e5d70d7079
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java
@@ -0,0 +1,775 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.CacheException;
+import javax.cache.configuration.Factory;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.AffinityFunctionContext;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cache.store.CacheStore;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Tests the recovery after a dynamic cache start failure.
+ */
+public abstract class IgniteAbstractDynamicCacheStartFailTest extends GridCacheAbstractSelfTest {
+    /** */
+    private static final String DYNAMIC_CACHE_NAME = "TestDynamicCache";
+
+    /** */
+    private static final String CLIENT_GRID_NAME = "client";
+
+    /** */
+    protected static final String EXISTING_CACHE_NAME = "existing-cache";;
+
+    /** */
+    private static final int PARTITION_COUNT = 16;
+
+    /** Coordinator node index. */
+    private int crdIdx = 0;
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return 3;
+    }
+
+    /**
+     * Returns {@code true} if persistence is enabled.
+     *
+     * @return {@code true} if persistence is enabled.
+     */
+    protected boolean persistenceEnabled() {
+        return false;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnServerFailedOnClient() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnClient";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-server-1");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, clientName));
+
+        try {
+            IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+        }
+        catch (CacheException e) {
+            fail("Exception should not be thrown.");
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnServerFailedOnServer() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnServerFailedOnServer";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-server-2");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(0)));
+
+        try {
+            IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+
+            fail("Expected exception was not thrown.");
+        }
+        catch (CacheException e) {
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testBrokenAffinityFunStartOnClientFailOnServer() throws Exception {
+        final String clientName = CLIENT_GRID_NAME + "testBrokenAffinityFunStartOnClientFailOnServer";
+
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite client = startGrid(clientName, clientCfg);
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName(DYNAMIC_CACHE_NAME + "-client-2");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(0)));
+
+        try {
+            IgniteCache cache = client.getOrCreateCache(cfg);
+
+            fail("Expected exception was not thrown.");
+        }
+        catch (CacheException e) {
+        }
+
+        stopGrid(clientName);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on all nodes.
+     */
+    public void testBrokenAffinityFunOnAllNodes() {
+        final boolean failOnAllNodes = true;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunOnInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 1;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on non-initiator node.
+     */
+    public void testBrokenAffinityFunOnNonInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 2;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on coordinator node.
+     */
+    public void testBrokenAffinityFunOnCoordinatorDiffInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = (crdIdx + 1) % gridCount();
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Test cache start with broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunOnCoordinator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = crdIdx;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Tests cache start with node filter and broken affinity function that throws an exception on initiator node.
+     */
+    public void testBrokenAffinityFunWithNodeFilter() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 0;
+        final int numOfCaches = 1;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenAffinityFun(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, true),
+            initiator);
+    }
+
+    /**
+     * Tests cache start with broken cache store that throws an exception on all nodes.
+     */
+    public void testBrokenCacheStoreOnAllNodes() {
+        final boolean failOnAllNodes = true;
+        final int unluckyNode = 0;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 0;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Tests cache start with broken cache store that throws an exception on initiator node.
+     */
+    public void testBrokenCacheStoreOnInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 1;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     * Tests cache start with broken cache store that throws an exception on non-initiator node.
+     */
+    public void testBrokenCacheStoreOnNonInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = 2;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     *  Tests cache start with broken cache store that throws an exception on initiator node.
+     */
+    public void testBrokenCacheStoreOnCoordinatorDiffInitiator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = (crdIdx + 1) % gridCount();
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     *  Tests cache start with broken cache store that throws an exception on coordinator node.
+     */
+    public void testBrokenCacheStoreFunOnCoordinator() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = crdIdx;
+        final int unluckyCfg = 1;
+        final int numOfCaches = 3;
+        final int initiator = crdIdx;
+
+        testDynamicCacheStart(
+            createCacheConfigsWithBrokenCacheStore(
+                failOnAllNodes, unluckyNode, unluckyCfg, numOfCaches, false),
+            initiator);
+    }
+
+    /**
+     *  Tests multiple creation of cache with broken affinity function.
+     */
+    public void testCreateCacheMultipleTimes() {
+        final boolean failOnAllNodes = false;
+        final int unluckyNode = 1;
+        final int unluckyCfg = 0;
+        final int numOfAttempts = 15;
+
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
+            failOnAllNodes, unluckyNode, unluckyCfg, 1, false).get(0);
+
+        for (int i = 0; i < numOfAttempts; ++i) {
+            try {
+                IgniteCache cache = ignite(0).getOrCreateCache(cfg);
+
+                fail("Expected exception was not thrown");
+            }
+            catch (CacheException e) {
+            }
+        }
+    }
+
+    /**
+     * Tests that a cache with the same name can be started after failure if cache configuration is corrected.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testCacheStartAfterFailure() throws Exception {
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
+            false, 1, 0, 1, false).get(0);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                grid(0).getOrCreateCache(cfg);
+                return null;
+            }
+        }, CacheException.class, null);
+
+        // Correct the cache configuration. Default constructor creates a good affinity function.
+        cfg.setAffinity(new BrokenAffinityFunction());
+
+        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
+
+        checkCacheOperations(cache);
+    }
+
+    /**
+     * Tests that other cache (existed before the failed start) is still operable after the failure.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testExistingCacheAfterFailure() throws Exception {
+        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
+
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
+            false, 1, 0, 1, false).get(0);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                grid(0).getOrCreateCache(cfg);
+                return null;
+            }
+        }, CacheException.class, null);
+
+        checkCacheOperations(cache);
+    }
+
+    /**
+     * Tests that other cache works as expected after the failure and further topology changes.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testTopologyChangesAfterFailure() throws Exception {
+        final String clientName = "testTopologyChangesAfterFailure";
+
+        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
+
+        checkCacheOperations(cache);
+
+        CacheConfiguration cfg = createCacheConfigsWithBrokenAffinityFun(
+            false, 0, 0, 1, false).get(0);
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                grid(0).getOrCreateCache(cfg);
+                return null;
+            }
+        }, CacheException.class, null);
+
+        awaitPartitionMapExchange();
+
+        checkCacheOperations(cache);
+
+        // Start a new server node and check cache operations.
+        Ignite serverNode = startGrid(gridCount() + 1);
+
+        if (persistenceEnabled()) {
+            List<BaselineNode> baseline = new ArrayList<>(grid(0).cluster().currentBaselineTopology());
+
+            baseline.add(serverNode.cluster().localNode());
+
+            grid(0).cluster().setBaselineTopology(baseline);
+        }
+
+        awaitPartitionMapExchange();
+
+        checkCacheOperations(serverNode.cache(EXISTING_CACHE_NAME));
+
+        // Start a new client node and check cache operations.
+        IgniteConfiguration clientCfg = getConfiguration(clientName);
+
+        clientCfg.setClientMode(true);
+
+        Ignite clientNode = startGrid(clientName, clientCfg);
+
+        checkCacheOperations(clientNode.cache(EXISTING_CACHE_NAME));
+    }
+
+    public void testConcurrentClientNodeJoins() throws Exception {
+        final int clientCnt = 3;
+        final int numberOfAttempts = 5;
+
+        IgniteCache<Integer, Value> cache = grid(0).getOrCreateCache(createCacheConfiguration(EXISTING_CACHE_NAME));
+
+        final AtomicInteger attemptCnt = new AtomicInteger();
+        final CountDownLatch stopLatch = new CountDownLatch(clientCnt);
+
+        IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                String clientName = Thread.currentThread().getName();
+
+                try {
+                    for (int i = 0; i < numberOfAttempts; ++i) {
+                        int uniqueCnt = attemptCnt.getAndIncrement();
+
+                        IgniteConfiguration clientCfg = getConfiguration(clientName + uniqueCnt);
+
+                        clientCfg.setClientMode(true);
+
+                        final Ignite clientNode = startGrid(clientName, clientCfg);
+
+                        CacheConfiguration cfg = new CacheConfiguration();
+
+                        cfg.setName(clientName + uniqueCnt);
+
+                        String instanceName = getTestIgniteInstanceName(uniqueCnt % gridCount());
+
+                        cfg.setAffinity(new BrokenAffinityFunction(false, instanceName));
+
+                        GridTestUtils.assertThrows(log, new Callable<Object>() {
+                            @Override public Object call() throws Exception {
+                                clientNode.getOrCreateCache(cfg);
+                                return null;
+                            }
+                        }, CacheException.class, null);
+
+                        stopGrid(clientName, true);
+                    }
+                }
+                catch (Exception e) {
+                    fail("Unexpected exception: " + e.getMessage());
+                }
+                finally {
+                    stopLatch.countDown();
+                }
+
+                return null;
+            }
+        }, clientCnt, "start-client-thread");
+
+        stopLatch.await();
+
+        assertEquals(numberOfAttempts * clientCnt, attemptCnt.get());
+
+        checkCacheOperations(cache);
+    }
+
+    protected void testDynamicCacheStart(final Collection<CacheConfiguration> cfgs, final int initiatorId) {
+        assert initiatorId < gridCount();
+
+        GridTestUtils.assertThrows(log, new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                grid(initiatorId).getOrCreateCaches(cfgs);
+                return null;
+            }
+        }, CacheException.class, null);
+
+        for (CacheConfiguration cfg: cfgs) {
+            IgniteCache cache = grid(initiatorId).cache(cfg.getName());
+
+            assertNull(cache);
+        }
+    }
+
+    /**
+     * Creates new cache configuration with the given name.
+     *
+     * @param cacheName Cache name.
+     * @return New cache configuration.
+     */
+    protected CacheConfiguration createCacheConfiguration(String cacheName) {
+        CacheConfiguration cfg = new CacheConfiguration()
+            .setName(cacheName)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+            .setAffinity(new BrokenAffinityFunction());
+
+        return cfg;
+    }
+
+    /**
+     * Create list of cache configurations.
+     *
+     * @param failOnAllNodes {@code true} if affinity function should be broken on all nodes.
+     * @param unluckyNode Node, where exception is raised.
+     * @param unluckyCfg Unlucky cache configuration number.
+     * @param cacheNum Number of caches.
+     * @param useFilter {@code true} if NodeFilter should be used.
+     *
+     * @return List of cache configurations.
+     */
+    protected List<CacheConfiguration> createCacheConfigsWithBrokenAffinityFun(
+        boolean failOnAllNodes,
+        int unluckyNode,
+        final int unluckyCfg,
+        int cacheNum,
+        boolean useFilter
+    ) {
+        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
+
+        final UUID uuid = ignite(unluckyNode).cluster().localNode().id();
+
+        List<CacheConfiguration> cfgs = new ArrayList<>();
+
+        for (int i = 0; i < cacheNum; ++i) {
+            CacheConfiguration cfg = createCacheConfiguration(DYNAMIC_CACHE_NAME + "-" + i);
+
+            if (i == unluckyCfg)
+                cfg.setAffinity(new BrokenAffinityFunction(failOnAllNodes, getTestIgniteInstanceName(unluckyNode)));
+
+            if (useFilter)
+                cfg.setNodeFilter(new NodeFilter(uuid));
+
+            cfgs.add(cfg);
+        }
+
+        return cfgs;
+    }
+
+    /**
+     * Create list of cache configurations.
+     *
+     * @param failOnAllNodes {@code true} if cache store should be broken on all nodes.
+     * @param unluckyNode Node, where exception is raised.
+     * @param unluckyCfg Unlucky cache configuration number.
+     * @param cacheNum Number of caches.
+     * @param useFilter {@code true} if NodeFilter should be used.
+     *
+     * @return List of cache configurations.
+     */
+    protected List<CacheConfiguration> createCacheConfigsWithBrokenCacheStore(
+        boolean failOnAllNodes,
+        int unluckyNode,
+        int unluckyCfg,
+        int cacheNum,
+        boolean useFilter
+    ) {
+        assert unluckyCfg >= 0 && unluckyCfg < cacheNum;
+
+        final UUID uuid = ignite(unluckyNode).cluster().localNode().id();
+
+        List<CacheConfiguration> cfgs = new ArrayList<>();
+
+        for (int i = 0; i < cacheNum; ++i) {
+            CacheConfiguration cfg = new CacheConfiguration();
+
+            cfg.setName(DYNAMIC_CACHE_NAME + "-" + i);
+
+            if (i == unluckyCfg)
+                cfg.setCacheStoreFactory(new BrokenStoreFactory(failOnAllNodes, getTestIgniteInstanceName(unluckyNode)));
+
+            if (useFilter)
+                cfg.setNodeFilter(new NodeFilter(uuid));
+
+            cfgs.add(cfg);
+        }
+
+        return cfgs;
+    }
+
+    /**
+     * Test the basic cache operations.
+     *
+     * @param cache Cache.
+     * @throws Exception If test failed.
+     */
+    protected void checkCacheOperations(IgniteCache<Integer, Value> cache) throws Exception {
+        int cnt = 1000;
+
+        // Check cache operations.
+        for (int i = 0; i < cnt; ++i)
+            cache.put(i, new Value(i));
+
+        for (int i = 0; i < cnt; ++i) {
+            Value v = cache.get(i);
+
+            assertNotNull(v);
+            assertEquals(i, v.getValue());
+        }
+
+        // Check Data Streamer functionality.
+        try (IgniteDataStreamer<Integer, Value> streamer = grid(0).dataStreamer(cache.getName())) {
+            for (int i = 0; i < 10_000; ++i)
+                streamer.addData(i, new Value(i));
+        }
+    }
+
+    /**
+     *
+     */
+    public static class Value {
+        @QuerySqlField
+        private final int fieldVal;
+
+        public Value(int fieldVal) {
+            this.fieldVal = fieldVal;
+        }
+
+        public int getValue() {
+            return fieldVal;
+        }
+    }
+
+    /**
+     * Filter specifying on which node the cache should be started.
+     */
+    public static class NodeFilter implements IgnitePredicate<ClusterNode> {
+        /** Cache should be created node with certain UUID. */
+        public UUID uuid;
+
+        /**
+         * @param uuid node ID.
+         */
+        public NodeFilter(UUID uuid) {
+            this.uuid = uuid;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode clusterNode) {
+            return clusterNode.id().equals(uuid);
+        }
+    }
+
+    /**
+     * Affinity function that throws an exception when affinity nodes are calculated on the given node.
+     */
+    public static class BrokenAffinityFunction extends RendezvousAffinityFunction {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Exception should arise on all nodes. */
+        private boolean eOnAllNodes = false;
+
+        /** Exception should arise on node with certain name. */
+        private String gridName;
+
+        /**
+         * Constructs a good affinity function.
+         */
+        public BrokenAffinityFunction() {
+            super(false, PARTITION_COUNT);
+            // No-op.
+        }
+
+        /**
+         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
+         * @param gridName Exception should arise on node with certain name.
+         */
+        public BrokenAffinityFunction(boolean eOnAllNodes, String gridName) {
+            super(false, PARTITION_COUNT);
+
+            this.eOnAllNodes = eOnAllNodes;
+            this.gridName = gridName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext affCtx) {
+            if (eOnAllNodes || ignite.name().equals(gridName))
+                throw new IllegalStateException("Simulated exception [locNodeId="
+                    + ignite.cluster().localNode().id() + "]");
+            else
+                return super.assignPartitions(affCtx);
+        }
+    }
+
+    /**
+     * Factory that throws an exception is got created.
+     */
+    public static class BrokenStoreFactory implements Factory<CacheStore<Integer, String>> {
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Exception should arise on all nodes. */
+        boolean eOnAllNodes = true;
+
+        /** Exception should arise on node with certain name. */
+        public static String gridName;
+
+        /**
+         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
+         * @param gridName Exception should arise on node with certain name.
+         */
+        public BrokenStoreFactory(boolean eOnAllNodes, String gridName) {
+            this.eOnAllNodes = eOnAllNodes;
+
+            this.gridName = gridName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public CacheStore<Integer, String> create() {
+            if (eOnAllNodes || ignite.name().equals(gridName))
+                throw new IllegalStateException("Simulated exception [locNodeId="
+                    + ignite.cluster().localNode().id() + "]");
+            else
+                return null;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartCoordinatorFailoverTest.java
new file mode 100644
index 0000000000000..36e1879a1c96b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartCoordinatorFailoverTest.java
@@ -0,0 +1,262 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.affinity.AffinityFunctionContext;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridJobExecuteResponse;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryCustomEventMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+public class IgniteDynamicCacheStartCoordinatorFailoverTest extends GridCommonAbstractTest {
+    /** Default IP finder for single-JVM cloud grid. */
+    private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** Latch which blocks DynamicCacheChangeFailureMessage until main thread has sent node fail signal. */
+    private static volatile CountDownLatch latch;
+
+    /** */
+    private static final String COORDINATOR_ATTRIBUTE = "coordinator";
+
+    /** Client mode flag. */
+    private Boolean appendCustomAttribute;
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        latch = new CountDownLatch(1);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+        discoSpi.setIpFinder(ipFinder);
+
+        cfg.setDiscoverySpi(discoSpi);
+
+        TcpCommunicationSpi commSpi = new CustomCommunicationSpi();
+        commSpi.setLocalPort(GridTestUtils.getNextCommPort(getClass()));
+
+        cfg.setCommunicationSpi(commSpi);
+
+        cfg.setFailureDetectionTimeout(15_000);
+
+        if (appendCustomAttribute) {
+            Map<String, Object> attrs = new HashMap<>();
+
+            attrs.put(COORDINATOR_ATTRIBUTE, Boolean.TRUE);
+
+            cfg.setUserAttributes(attrs);
+        }
+
+        return cfg;
+    }
+
+    /**
+     * Tests coordinator failover during cache start failure.
+     *
+     * @throws Exception If test failed.
+     */
+    public void testCoordinatorFailure() throws Exception {
+        // Start coordinator node.
+        appendCustomAttribute = true;
+
+        Ignite g = startGrid(0);
+
+        appendCustomAttribute = false;
+
+        Ignite g1 = startGrid(1);
+        Ignite g2 = startGrid(2);
+
+        awaitPartitionMapExchange();
+
+        CacheConfiguration cfg = new CacheConfiguration();
+
+        cfg.setName("test-coordinator-failover");
+
+        cfg.setAffinity(new BrokenAffinityFunction(false, getTestIgniteInstanceName(2)));
+
+        GridTestUtils.runAsync(new Callable<Object>() {
+            @Override public Object call() throws Exception {
+                GridTestUtils.assertThrows(log, new Callable<Object>() {
+                    @Override public Object call() throws Exception {
+                        g1.getOrCreateCache(cfg);
+                        return null;
+                    }
+                }, CacheException.class, null);
+
+                return null;
+            }
+        }, "cache-starter-thread");
+
+        latch.await();
+
+        stopGrid(0, true);
+
+        awaitPartitionMapExchange();
+
+        // Correct the cache configuration.
+        cfg.setAffinity(new RendezvousAffinityFunction());
+
+        IgniteCache cache = g1.getOrCreateCache(cfg);
+
+        checkCacheOperations(g1, cache);
+    }
+
+    /**
+     * Test the basic cache operations.
+     *
+     * @param cache Cache.
+     * @throws Exception If test failed.
+     */
+    protected void checkCacheOperations(Ignite ignite, IgniteCache cache) throws Exception {
+        int cnt = 1000;
+
+        // Check base cache operations.
+        for (int i = 0; i < cnt; ++i)
+            cache.put(i, i);
+
+        for (int i = 0; i < cnt; ++i) {
+            Integer v = (Integer) cache.get(i);
+
+            assertNotNull(v);
+            assertEquals(i, v.intValue());
+        }
+
+        // Check Data Streamer capabilities.
+        try (IgniteDataStreamer streamer = ignite.dataStreamer(cache.getName())) {
+            for (int i = 0; i < 10_000; ++i)
+                streamer.addData(i, i);
+        }
+    }
+
+    /**
+     * Communication SPI which could optionally block outgoing messages.
+     */
+    private static class CustomCommunicationSpi extends TcpCommunicationSpi {
+        /**
+         * Send message optionally either blocking it or throwing an exception if it is of
+         * {@link GridJobExecuteResponse} type.
+         *
+         * @param node Destination node.
+         * @param msg Message to be sent.
+         * @param ackClosure Ack closure.
+         * @throws org.apache.ignite.spi.IgniteSpiException If failed.
+         */
+        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackClosure)
+            throws IgniteSpiException {
+
+            if (msg instanceof GridIoMessage) {
+                GridIoMessage msg0 = (GridIoMessage)msg;
+
+                if (msg0.message() instanceof GridDhtPartitionsSingleMessage) {
+                    Boolean attr = (Boolean) node.attributes().get(COORDINATOR_ATTRIBUTE);
+
+                    GridDhtPartitionsSingleMessage singleMsg = (GridDhtPartitionsSingleMessage) msg0.message();
+
+                    Exception err = singleMsg.getError();
+
+                    if (Boolean.TRUE.equals(attr) && err != null) {
+                        // skip message
+                        latch.countDown();
+
+                        return;
+                    }
+                }
+            }
+
+            super.sendMessage(node, msg, ackClosure);
+        }
+    }
+
+    /**
+     * Affinity function that throws an exception when affinity nodes are calculated on the given node.
+     */
+    public static class BrokenAffinityFunction extends RendezvousAffinityFunction {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        @IgniteInstanceResource
+        private Ignite ignite;
+
+        /** Exception should arise on all nodes. */
+        private boolean eOnAllNodes = false;
+
+        /** Exception should arise on node with certain name. */
+        private String gridName;
+
+        /**
+         * Default constructor.
+         */
+        public BrokenAffinityFunction() {
+            // No-op.
+        }
+
+        /**
+         * @param eOnAllNodes {@code True} if exception should be thrown on all nodes.
+         * @param gridName Exception should arise on node with certain name.
+         */
+        public BrokenAffinityFunction(boolean eOnAllNodes, String gridName) {
+            this.eOnAllNodes = eOnAllNodes;
+            this.gridName = gridName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext affCtx) {
+            if (eOnAllNodes || ignite.name().equals(gridName))
+                throw new IllegalStateException("Simulated exception [locNodeId="
+                    + ignite.cluster().localNode().id() + "]");
+            else
+                return super.assignPartitions(affCtx);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailTest.java
new file mode 100644
index 0000000000000..888a45805e091
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailTest.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ * Tests the recovery after a dynamic cache start failure.
+ */
+public class IgniteDynamicCacheStartFailTest extends IgniteAbstractDynamicCacheStartFailTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrids(gridCount());
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        super.afterTestsStopped();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java
new file mode 100644
index 0000000000000..3b7bf52cf90dd
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+
+/**
+ * Tests the recovery after a dynamic cache start failure, with enabled persistence.
+ */
+public class IgniteDynamicCacheStartFailWithPersistenceTest extends IgniteAbstractDynamicCacheStartFailTest {
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 5 * 60 * 1000;
+    }
+
+    protected boolean persistenceEnabled() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(256L * 1024 * 1024)
+                    .setPersistenceEnabled(true))
+            .setWalMode(WALMode.LOG_ONLY));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        cleanPersistenceDir();
+
+        startGrids(gridCount());
+
+        grid(0).cluster().active(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        super.afterTestsStopped();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    protected void checkCacheOperations(IgniteCache<Integer, Value> cache) throws Exception {
+        super.checkCacheOperations(cache);
+
+        // Disable write-ahead log.
+        grid(0).cluster().disableWal(cache.getName());
+
+        try (IgniteDataStreamer<Integer, Value> streamer = grid(0).dataStreamer(cache.getName())) {
+            for (int i = 10_000; i < 15_000; ++i)
+                streamer.addData(i, new Value(i));
+        }
+
+        grid(0).cluster().enableWal(cache.getName());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 7a4d4be2e79f3..b973c9125bb21 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -78,6 +78,9 @@
 import org.apache.ignite.internal.processors.cache.IgniteClientCacheInitializationFailTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheFilterTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheMultinodeTest;
+import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartCoordinatorFailoverTest;
+import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailTest;
+import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailWithPersistenceTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartNoExchangeTimeoutTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartStopConcurrentTest;
@@ -225,6 +228,9 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteDynamicCacheStartSelfTest.class);
         suite.addTestSuite(IgniteDynamicCacheMultinodeTest.class);
+        suite.addTestSuite(IgniteDynamicCacheStartFailTest.class);
+        suite.addTestSuite(IgniteDynamicCacheStartFailWithPersistenceTest.class);
+        suite.addTestSuite(IgniteDynamicCacheStartCoordinatorFailoverTest.class);
         suite.addTestSuite(IgniteDynamicCacheWithConfigStartSelfTest.class);
         suite.addTestSuite(IgniteCacheDynamicStopSelfTest.class);
         suite.addTestSuite(IgniteDynamicCacheStartStopConcurrentTest.class);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheQueryAfterDynamicCacheStartFailureTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheQueryAfterDynamicCacheStartFailureTest.java
new file mode 100644
index 0000000000000..f28833ebe568d
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheQueryAfterDynamicCacheStartFailureTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.List;
+import javax.cache.Cache;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+public class CacheQueryAfterDynamicCacheStartFailureTest extends IgniteAbstractDynamicCacheStartFailTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrids(gridCount());
+
+        awaitPartitionMapExchange();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        super.afterTestsStopped();
+    }
+
+    /** {@inheritDoc} */
+    protected CacheConfiguration createCacheConfiguration(String cacheName) {
+        CacheConfiguration cfg = new CacheConfiguration()
+            .setName(cacheName)
+            .setIndexedTypes(Integer.class, Value.class);
+
+        return cfg;
+    }
+
+    protected void checkCacheOperations(IgniteCache<Integer, Value> cache) throws Exception {
+        super.checkCacheOperations(cache);
+
+        // Check SQL API.
+        String sql = "fieldVal >= ? and fieldVal <= ?";
+        List<Cache.Entry<Integer, Value>> res = cache.query(
+            new SqlQuery<Integer, Value>(Value.class, sql).setArgs(1, 100)).getAll();
+
+        assertNotNull(res);
+        assertEquals(100, res.size());
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index c89673626e83a..c8cf92d758fc2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.internal.processors.cache.CacheConfigurationP2PTest;
 import org.apache.ignite.internal.processors.cache.CacheIndexStreamerTest;
 import org.apache.ignite.internal.processors.cache.CacheOperationsWithExpirationTest;
+import org.apache.ignite.internal.processors.cache.CacheQueryAfterDynamicCacheStartFailureTest;
 import org.apache.ignite.internal.processors.cache.CacheQueryFilterExpiredTest;
 import org.apache.ignite.internal.processors.cache.CacheRandomOperationsMultithreadedTest;
 import org.apache.ignite.internal.processors.cache.ClientReconnectAfterClusterRestartTest;
@@ -78,6 +79,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(ClientReconnectAfterClusterRestartTest.class);
 
+        suite.addTestSuite(CacheQueryAfterDynamicCacheStartFailureTest.class);
+
         suite.addTestSuite(IgniteCacheGroupsSqlTest.class);
 
         suite.addTestSuite(IgniteDataStreamerTest.class);

From 54a33672ab1c673fff7145610c8aedce9539abef Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Mon, 16 Jul 2018 17:27:49 +0300
Subject: [PATCH 255/543] IGNITE-8897 Node with longer BaselineHistory joining
 the cluster causes cluster stopping. - Fixes #4357.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>

(cherry-picked from commit#e7c3566d11d6276b18a8c54e7b7f9bf3f9b3126a)
---
 .../cluster/DiscoveryDataClusterState.java    |  11 +-
 .../cluster/GridClusterStateProcessor.java    |  28 +-
 ...aselineAffinityTopologyActivationTest.java | 289 +++++++++++++++---
 3 files changed, 281 insertions(+), 47 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
index b022754c6670a..d626c9f9b2dd1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
@@ -32,9 +32,9 @@
  * baseline topology.
  * <p>
  * This object also captures a transitional cluster state, when one or more fields are changing. In this case,
- * a {@code transitionReqId} field is set to a non-null value and {@code prevState} captures previous cluster state.
+ * a {@code transitionReqId} field is set to a non-null value and {@code previousBaselineTopology} captures previous cluster state.
  * A joining node catching the cluster in an intermediate state will observe {@code transitionReqId} field to be
- * non-null, however the {@code prevState} will not be sent to the joining node.
+ * non-null, however the {@code previousBaselineTopology} will not be sent to the joining node.
  *
  * TODO https://issues.apache.org/jira/browse/IGNITE-7640 This class must be immutable, transitionRes must be set by calling finish().
  */
@@ -200,6 +200,13 @@ public boolean active() {
         return baselineTopology;
     }
 
+    /**
+     * @return Previous Baseline topology.
+     */
+    @Nullable public BaselineTopology previousBaselineTopology() {
+        return prevState != null ? prevState.baselineTopology() : null;
+    }
+
     /**
      * @return Nodes participating in state change exchange.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index d4d0eb1a76e95..da0bbf6867c8c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -947,8 +947,24 @@ private IgniteInternalFuture<?> changeGlobalState0(final boolean activate,
             }
         }
 
+        if (globalState.transition() && globalState.previousBaselineTopology() == null) {
+            //case when cluster is activating for the first time and other node with existing baseline topology
+            //tries to join
+
+            String msg = "Node with set up BaselineTopology is not allowed " +
+                "to join cluster in the process of first activation: " + node.consistentId();
+
+            return new IgniteNodeValidationResult(node.id(), msg, msg);
+        }
+
+        BaselineTopology clusterBlt;
+
+        if (globalState.transition())
+            clusterBlt = globalState.previousBaselineTopology();
+        else
+            clusterBlt = globalState.baselineTopology();
+
         BaselineTopology joiningNodeBlt = joiningNodeState.baselineTopology();
-        BaselineTopology clusterBlt = globalState.baselineTopology();
 
         String recommendation = " Consider cleaning persistent storage of the node and adding it to the cluster again.";
 
@@ -968,7 +984,7 @@ private IgniteInternalFuture<?> changeGlobalState0(final boolean activate,
             if (!clusterBlt.isCompatibleWith(joiningNodeBlt)) {
                 String msg = "BaselineTopology of joining node ("
                     + node.consistentId()
-                    + " ) is not compatible with BaselineTopology in the cluster."
+                    + ") is not compatible with BaselineTopology in the cluster."
                     + " Branching history of cluster BlT (" + clusterBlt.branchingHistory()
                     + ") doesn't contain branching point hash of joining node BlT ("
                     + joiningNodeBlt.branchingPointHash()
@@ -1174,9 +1190,13 @@ private void onFinalActivate(final StateChangeRequest req) {
     }
 
     /** {@inheritDoc} */
-    @Override public void onBaselineTopologyChanged(BaselineTopology blt, BaselineTopologyHistoryItem prevBltHistItem) throws IgniteCheckedException {
+    @Override public void onBaselineTopologyChanged
+    (
+        BaselineTopology blt,
+        BaselineTopologyHistoryItem prevBltHistItem
+    ) throws IgniteCheckedException {
         if (compatibilityMode) {
-            if (log.isDebugEnabled())
+            if (log.isInfoEnabled())
                 log.info("BaselineTopology won't be stored as this node is running in compatibility mode");
 
             return;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
index 6b6d666cd021d..92342c0c5174f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteBaselineAffinityTopologyActivationTest.java
@@ -21,13 +21,14 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.BaselineNode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -36,18 +37,26 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.DetachedClusterNode;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
 import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.processors.cluster.BaselineTopologyHistory;
 import org.apache.ignite.internal.processors.cluster.BaselineTopologyHistoryItem;
 import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.junit.Assert;
 
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+
 /**
  *
  */
@@ -58,6 +67,9 @@ public class IgniteBaselineAffinityTopologyActivationTest extends GridCommonAbst
     /** Entries count to add to cache. */
     private static final int ENTRIES_COUNT = 100;
 
+    /** */
+    private static final String CACHE_NAME = "dfltCache";
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -73,6 +85,16 @@ public class IgniteBaselineAffinityTopologyActivationTest extends GridCommonAbst
             ).setWalMode(WALMode.LOG_ONLY)
         );
 
+        cfg.setCommunicationSpi(new SingleMessageInterceptorCommunicationSpi());
+
+        cfg.setCacheConfiguration(new CacheConfiguration<Integer, Integer>()
+            .setName(CACHE_NAME)
+            .setCacheMode(PARTITIONED)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+            .setBackups(1)
+            .setAffinity(new RendezvousAffinityFunction(32, null))
+        );
+
         return cfg;
     }
 
@@ -98,12 +120,12 @@ public class IgniteBaselineAffinityTopologyActivationTest extends GridCommonAbst
     public void testAutoActivationWithCompatibleOldNode() throws Exception {
         startGridWithConsistentId("A");
         startGridWithConsistentId("B");
-        startGridWithConsistentId("C").active(true);
+        startGridWithConsistentId("C").cluster().active(true);
 
         stopAllGrids(false);
 
         startGridWithConsistentId("A");
-        startGridWithConsistentId("B").active(true);
+        startGridWithConsistentId("B").cluster().active(true);
 
         {
             IgniteEx nodeA = grid("A");
@@ -133,7 +155,7 @@ public void testAutoActivationWithCompatibleOldNode() throws Exception {
         boolean active = GridTestUtils.waitForCondition(
             new GridAbsPredicate() {
                 @Override public boolean apply() {
-                    return nodeC.active();
+                    return nodeC.cluster().active();
                 }
             },
             10_000
@@ -149,7 +171,7 @@ public void testAutoActivationWithCompatibleOldNode() throws Exception {
     public void testBltChangeTopVerRemoveOnlineNodeFails() throws Exception {
         Ignite ignite = startGridWithConsistentId("A");
 
-        ignite.active(true);
+        ignite.cluster().active(true);
 
         long singleNodeTopVer = ignite.cluster().topologyVersion();
 
@@ -181,7 +203,7 @@ public void testOnlineNodesCannotBeRemovedFromBaselineTopology() throws Exceptio
         Ignite nodeB = startGridWithConsistentId("B");
         Ignite nodeC = startGridWithConsistentId("OnlineConsID");
 
-        nodeC.active(true);
+        nodeC.cluster().active(true);
 
         boolean expectedExceptionIsThrown = false;
 
@@ -253,16 +275,16 @@ public void testNodeFailsToJoinWithIncompatiblePreviousBaselineTopology() throws
     public void testIncompatibleBltNodeIsProhibitedToJoinCluster() throws Exception {
         startGridWithConsistentId("A");
         startGridWithConsistentId("B");
-        startGridWithConsistentId("C").active(true);
+        startGridWithConsistentId("C").cluster().active(true);
 
         stopAllGrids(false);
 
         startGridWithConsistentId("A");
-        startGridWithConsistentId("B").active(true);
+        startGridWithConsistentId("B").cluster().active(true);
 
         stopAllGrids(false);
 
-        startGridWithConsistentId("C").active(true);
+        startGridWithConsistentId("C").cluster().active(true);
 
         stopAllGrids(false);
 
@@ -326,7 +348,7 @@ public void testNodeWithOldBltIsAllowedToJoinCluster() throws Exception {
         Ignite nodeB = startGridWithConsistentId("B");
         Ignite nodeC = startGridWithConsistentId("C");
 
-        nodeC.active(true);
+        nodeC.cluster().active(true);
         verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeA, nodeB, nodeC});
 
         stopAllGrids(false);
@@ -334,7 +356,7 @@ public void testNodeWithOldBltIsAllowedToJoinCluster() throws Exception {
         nodeA = startGridWithConsistentId("A");
         nodeB = startGridWithConsistentId("B");
 
-        nodeB.active(true);
+        nodeB.cluster().active(true);
 
         verifyBaselineTopologyOnNodes(verifier2, new Ignite[] {nodeA, nodeB});
 
@@ -347,6 +369,161 @@ public void testNodeWithOldBltIsAllowedToJoinCluster() throws Exception {
         verifyBaselineTopologyOnNodes(verifier2, new Ignite[] {nodeA, nodeB, nodeC});
     }
 
+    /**
+     *
+     * Test verifies that restart node from baseline when PME and BLT change processes
+     * are taking place in the cluster simultaneously doesn't lead to shut down of alive cluster nodes.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNodeJoinsDuringPartitionMapExchange() throws Exception {
+        startGridWithConsistentId("A");
+        startGridWithConsistentId("B");
+        startGridWithConsistentId("C");
+
+        IgniteEx grid = grid("B");
+
+        grid.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = grid.getOrCreateCache(CACHE_NAME);
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, i * 2);
+
+        awaitPartitionMapExchange();
+
+        final long topVer = grid.cluster().topologyVersion() + 1;
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        SingleMessageInterceptorCommunicationSpi commSpi = (SingleMessageInterceptorCommunicationSpi) grid
+            .configuration().getCommunicationSpi();
+
+        commSpi.blockMsgsWithLatch(latch);
+
+        try {
+            GridTestUtils.runAsync(
+                () -> startGridWithConsistentId("D")
+            ).get(20_000);
+        }
+        catch (Exception ignored) {
+            // timeout exception is expected here
+        }
+
+        try {
+            GridTestUtils.runAsync(
+                () -> grid.cluster().setBaselineTopology(topVer)
+            ).get(10_000);
+        }
+        catch (Exception ignored) {
+            // timeout exception is expected here
+        }
+
+        IgniteInternalFuture restartFut = GridTestUtils.runAsync(
+            () -> {
+                try {
+                    stopGrid("C", true);
+                    startGridWithConsistentId("C");
+                }
+                catch (Exception ignored) {
+                    //ignored
+                }
+            }
+        );
+
+        latch.countDown();
+
+        restartFut.get();
+
+        awaitPartitionMapExchange();
+
+        long expActivationHash = (long)"A".hashCode() + "B".hashCode() + "C".hashCode();
+
+        checkBaselineTopologyOnNode(grid("A"), 1, 1, 1, expActivationHash);
+        checkBaselineTopologyOnNode(grid("B"), 1, 1, 1, expActivationHash);
+        checkBaselineTopologyOnNode(grid("C"), 1, 1, 1, expActivationHash);
+        checkBaselineTopologyOnNode(grid("D"), 1, 1, 1, expActivationHash);
+    }
+
+    /**
+     * @param ig Ignite.
+     * @param expBltId Expected BaselineTopology ID.
+     * @param expBltHistSize Expected Baseline history size.
+     * @param expBranchingHistSize Expected branching history size.
+     * @param expActivationHash Expected activation hash.
+     */
+    private void checkBaselineTopologyOnNode(
+        Ignite ig,
+        int expBltId,
+        int expBltHistSize,
+        int expBranchingHistSize,
+        long expActivationHash) {
+        BaselineTopology blt = getBaselineTopology(ig);
+        BaselineTopologyHistory bltHist = getBaselineTopologyHistory(ig);
+
+        assertNotNull(bltHist);
+        assertEquals(expBltId, blt.id());
+
+        assertEquals(expBltHistSize, bltHist.history().size());
+        BaselineTopologyHistoryItem histItem = bltHist.history().get(0);
+
+        assertEquals(expBranchingHistSize, histItem.branchingHistory().size());
+        assertEquals(expActivationHash, (long)histItem.branchingHistory().get(0));
+    }
+
+    /**
+     * Test verifies that node with set up BaselineTopology is not allowed to join the cluster
+     * in the process of on-going first activation.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNodeWithBltIsNotAllowedToJoinClusterDuringFirstActivation() throws Exception {
+        Ignite nodeC = startGridWithConsistentId("C");
+
+        nodeC.cluster().active(true);
+
+        stopGrid("C", false);
+
+        Ignite nodeA = startGridWithConsistentId("A");
+        Ignite nodeB = startGridWithConsistentId("B");
+
+        final CountDownLatch latch = new CountDownLatch(1);
+
+        SingleMessageInterceptorCommunicationSpi commSpi = (SingleMessageInterceptorCommunicationSpi) nodeB
+            .configuration().getCommunicationSpi();
+
+        commSpi.blockMsgsWithLatch(latch);
+
+        GridTestUtils.runAsync(
+            () -> {
+                try {
+                    nodeA.cluster().active(true);
+                }
+                catch (Exception e) {
+                    log.warning("Exception during activation", e);
+                }
+            });
+
+        try {
+            startGridWithConsistentId("C");
+        }
+        catch (Exception e) {
+            Throwable cause = e.getCause();
+
+            while (!(cause instanceof IgniteSpiException))
+                cause = cause.getCause();
+
+            assertNotNull(cause);
+
+            String msg = cause.getMessage();
+            assertNotNull(msg);
+            assertTrue(msg.startsWith("Node with set up BaselineTopology is not allowed " +
+                "to join cluster in the process of first activation:"));
+        }
+
+        latch.countDown();
+    }
+
     /**
      * Verifies that when new node outside of baseline topology joins active cluster with BLT already set
      * it receives BLT from the cluster and stores it locally.
@@ -356,7 +533,7 @@ public void testNewNodeJoinsToActiveCluster() throws Exception {
         Ignite nodeB = startGridWithConsistentId("B");
         Ignite nodeC = startGridWithConsistentId("C");
 
-        nodeC.active(true);
+        nodeC.cluster().active(true);
 
         BaselineTopologyVerifier verifier1 = new BaselineTopologyVerifier() {
             @Override public void verify(BaselineTopology blt) {
@@ -376,7 +553,7 @@ public void testNewNodeJoinsToActiveCluster() throws Exception {
 
         nodeD = startGridWithConsistentId("D");
 
-        assertFalse(nodeD.active());
+        assertFalse(nodeD.cluster().active());
 
         verifyBaselineTopologyOnNodes(verifier1, new Ignite[] {nodeD});
     }
@@ -403,7 +580,7 @@ public void testRemoveNodeFromBaselineTopology() throws Exception {
         startGridWithConsistentId("B");
         Ignite nodeC = startGridWithConsistentId("C");
 
-        nodeC.active(true);
+        nodeC.cluster().active(true);
 
         stopGrid("B", false);
 
@@ -457,7 +634,7 @@ public void testAddNodeToBaselineTopology() throws Exception {
         Ignite nodeB = startGridWithConsistentId("B");
         Ignite nodeC = startGridWithConsistentId("C");
 
-        nodeC.active(true);
+        nodeC.cluster().active(true);
 
         IgniteEx nodeD = (IgniteEx) startGridWithConsistentId("D");
 
@@ -480,7 +657,7 @@ public void testRemoveBaselineTopology() throws Exception {
         Ignite nodeB = startGridWithConsistentId("B");
         Ignite nodeC = startGridWithConsistentId("C");
 
-        nodeA.active(true);
+        nodeA.cluster().active(true);
 
         nodeA.cluster().setBaselineTopology(null);
 
@@ -541,11 +718,11 @@ public void testActivationHashIsNotUpdatedOnMultipleActivationRequests() throws
 
         Ignite nodeA = startGridWithConsistentId("A");
 
-        nodeA.active(true);
+        nodeA.cluster().active(true);
 
         Ignite nodeB = startGridWithConsistentId("B");
 
-        nodeA.active(true);
+        nodeA.cluster().active(true);
 
         verifyBaselineTopologyOnNodes(verifier, new Ignite[] {nodeA, nodeB});
     }
@@ -557,7 +734,7 @@ public void testActivationHashIsNotUpdatedOnMultipleActivationRequests() throws
     public void testAutoActivationWithBaselineTopologyPreset() throws Exception {
         Ignite ig = startGridWithConsistentId("A");
 
-        ig.active(true);
+        ig.cluster().active(true);
 
         ig.cluster().setBaselineTopology(Arrays.asList(new BaselineNode[] {
             createBaselineNodeWithConsId("A"), createBaselineNodeWithConsId("B"), createBaselineNodeWithConsId("C")}));
@@ -573,7 +750,7 @@ public void testAutoActivationWithBaselineTopologyPreset() throws Exception {
         boolean activated = GridTestUtils.waitForCondition(
             new GridAbsPredicate() {
                @Override public boolean apply() {
-                   return ig1.active();
+                   return ig1.cluster().active();
                }
             },
             10_000
@@ -599,7 +776,7 @@ public void testAutoActivationSimple() throws Exception {
 
         IgniteEx srv = grid(0);
 
-        srv.active(true);
+        srv.cluster().active(true);
 
         createAndFillCache(srv);
 
@@ -614,7 +791,7 @@ public void testAutoActivationSimple() throws Exception {
         boolean clusterActive = GridTestUtils.waitForCondition(
             new GridAbsPredicate() {
                 @Override public boolean apply() {
-                    return ig.active();
+                    return ig.cluster().active();
                 }
             },
             10_000);
@@ -632,21 +809,21 @@ public void testNoAutoActivationOnJoinNewNodeToInactiveCluster() throws Exceptio
 
         IgniteEx srv = grid(0);
 
-        srv.active(true);
+        srv.cluster().active(true);
 
         awaitPartitionMapExchange();
 
-        assertTrue(srv.active());
+        assertTrue(srv.cluster().active());
 
-        srv.active(false);
+        srv.cluster().active(false);
 
-        assertFalse(srv.active());
+        assertFalse(srv.cluster().active());
 
         startGrid(2);
 
         Thread.sleep(3_000);
 
-        assertFalse(srv.active());
+        assertFalse(srv.cluster().active());
     }
 
     /**
@@ -657,13 +834,13 @@ public void testBaselineTopologyRemainsTheSameOnClusterDeactivation() throws Exc
 
         IgniteEx srv = grid(0);
 
-        srv.active(true);
+        srv.cluster().active(true);
 
         awaitPartitionMapExchange();
 
-        assertTrue(srv.active());
+        assertTrue(srv.cluster().active());
 
-        srv.active(false);
+        srv.cluster().active(false);
 
         BaselineTopology blt = getBaselineTopology(srv);
 
@@ -704,7 +881,7 @@ public void testBaselineHistorySyncWithNewNode() throws Exception {
         startGridWithConsistentId("B");
         startGridWithConsistentId("C");
 
-        nodeA.active(true);
+        nodeA.cluster().active(true);
 
         stopGrid("C", false);
 
@@ -755,7 +932,7 @@ public void testBaselineHistorySyncWithOldNodeWithCompatibleHistory() throws Exc
         startGridWithConsistentId("B");
         startGridWithConsistentId("C");
 
-        nodeA.active(true);
+        nodeA.cluster().active(true);
 
         stopGrid("C", false);
 
@@ -783,11 +960,11 @@ public void testBaselineNotDeletedOnDeactivation() throws Exception {
         startGridWithConsistentId("B");
         startGridWithConsistentId("C");
 
-        nodeA.active(true);
+        nodeA.cluster().active(true);
 
         assertNotNull(nodeA.cluster().currentBaselineTopology());
 
-        nodeA.active(false);
+        nodeA.cluster().active(false);
 
         stopAllGrids();
 
@@ -800,7 +977,7 @@ public void testBaselineNotDeletedOnDeactivation() throws Exception {
         boolean clusterActive = GridTestUtils.waitForCondition(
             new GridAbsPredicate() {
                 @Override public boolean apply() {
-                    return ig.active();
+                    return ig.cluster().active();
                 }
             },
             10_000);
@@ -822,7 +999,7 @@ public void testNodeWithBltIsProhibitedToJoinNewCluster() throws Exception {
 
         Ignite nodeC = startGridWithConsistentId("C");
 
-        nodeC.active(true);
+        nodeC.cluster().active(true);
 
         stopGrid("C", false);
 
@@ -869,14 +1046,14 @@ public void _testBaselineTopologyHistoryIsDeletedOnBaselineDelete() throws Excep
         startGridWithConsistentId("B");
         startGridWithConsistentId("C");
 
-        nodeA.active(true);
+        nodeA.cluster().active(true);
 
         stopAllGrids(false);
 
         nodeA = startGridWithConsistentId("A");
         startGridWithConsistentId("B");
 
-        nodeA.active(true);
+        nodeA.cluster().active(true);
 
         nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
 
@@ -884,7 +1061,7 @@ public void _testBaselineTopologyHistoryIsDeletedOnBaselineDelete() throws Excep
 
         nodeA = startGridWithConsistentId("A");
 
-        nodeA.active(true);
+        nodeA.cluster().active(true);
 
         nodeA.cluster().setBaselineTopology(baselineNodes(nodeA.cluster().forServers().nodes()));
 
@@ -894,7 +1071,7 @@ public void _testBaselineTopologyHistoryIsDeletedOnBaselineDelete() throws Excep
 
         boolean activated = GridTestUtils.waitForCondition(new GridAbsPredicate() {
             @Override public boolean apply() {
-                return node.active();
+                return node.cluster().active();
             }
         }, 10_000);
 
@@ -959,12 +1136,42 @@ private void createAndFillCache(Ignite srv) {
     private CacheConfiguration cacheConfiguration() {
         return new CacheConfiguration()
             .setName(DEFAULT_CACHE_NAME)
-            .setCacheMode(CacheMode.PARTITIONED)
+            .setCacheMode(PARTITIONED)
             .setAtomicityMode(CacheAtomicityMode.ATOMIC)
             .setBackups(2)
             .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
     }
 
+    /**
+     * TcpCommunicationSpi aimed to delay {@link GridDhtPartitionsSingleMessage} to emulate PME hanging.
+     */
+    private static class SingleMessageInterceptorCommunicationSpi extends TcpCommunicationSpi {
+        /** */
+        private volatile CountDownLatch singleMsgSndLatch;
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(
+            ClusterNode node,
+            Message msg,
+            IgniteInClosure<IgniteException> ackC
+        ) throws IgniteSpiException {
+            if (((GridIoMessage) msg).message() instanceof GridDhtPartitionsSingleMessage) {
+                try {
+                    if (singleMsgSndLatch != null)
+                        singleMsgSndLatch.await();
+                }
+                catch (Exception ignored) { }
+            }
+
+            super.sendMessage(node, msg, ackC);
+        }
+
+        /** */
+        void blockMsgsWithLatch(CountDownLatch latch) {
+            singleMsgSndLatch = latch;
+        }
+    }
+
     /** */
     private static final class TestValue {
         /** */

From 43cf9b1897bd0f01f7f0e93fda6a24d7efd2115d Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Mon, 16 Jul 2018 17:18:21 +0300
Subject: [PATCH 256/543] IGNITE-8820 Add ability to accept changing
 txTimeoutOnPartitionMapExchange while waiting for pending transactions. -
 Fixes #4217.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 09ce06c)
---
 .../GridCachePartitionExchangeManager.java    |  14 +-
 .../GridDhtPartitionsExchangeFuture.java      |  37 +++--
 .../optimized/OptimizedMarshallerTest.java    |   6 -
 ...etTxTimeoutOnPartitionMapExchangeTest.java | 146 ++++++++++++++++++
 4 files changed, 171 insertions(+), 32 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index d3fddabef7936..644c26e5dc664 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -46,7 +46,6 @@
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
@@ -2493,17 +2492,13 @@ else if (task instanceof ForceRebalanceExchangeTask) {
 
                             int dumpCnt = 0;
 
-                            IgniteConfiguration cfg = cctx.gridConfig();
-
-                            long rollbackTimeout = cfg.getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange();
-
                             final long dumpTimeout = 2 * cctx.gridConfig().getNetworkTimeout();
 
                             long nextDumpTime = 0;
 
                             while (true) {
                                 try {
-                                    resVer = exchFut.get(rollbackTimeout > 0 ? rollbackTimeout : dumpTimeout);
+                                    resVer = exchFut.get(dumpTimeout);
 
                                     break;
                                 }
@@ -2512,7 +2507,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                         U.warn(diagnosticLog, "Failed to wait for partition map exchange [" +
                                             "topVer=" + exchFut.initialVersion() +
                                             ", node=" + cctx.localNodeId() + "]. " +
-                                            (rollbackTimeout == 0 ? "Consider changing TransactionConfiguration.txTimeoutOnPartitionMapSynchronization to non default value to avoid this message. " : "") +
                                             "Dumping pending objects that might be the cause: ");
 
                                         try {
@@ -2524,12 +2518,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
 
                                         nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, dumpTimeout);
                                     }
-
-                                    if (rollbackTimeout > 0) {
-                                        rollbackTimeout = 0; // Try automatic rollback only once.
-
-                                        cctx.tm().rollbackOnTopologyChange(exchFut.initialVersion());
-                                    }
                                 }
                                 catch (Exception e) {
                                     if (exchFut.reconnectOnError(e))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 50a6f979d25ac..c204c8ad1f736 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1175,10 +1175,11 @@ private void distributedExchange() throws IgniteCheckedException {
             distributed = false;
 
         // On first phase we wait for finishing all local tx updates, atomic updates and lock releases on all nodes.
-        waitPartitionRelease(distributed);
+        waitPartitionRelease(distributed, true);
 
         // Second phase is needed to wait for finishing all tx updates from primary to backup nodes remaining after first phase.
-        waitPartitionRelease(false);
+        if (distributed)
+            waitPartitionRelease(false, false);
 
         boolean topChanged = firstDiscoEvt.type() != EVT_DISCOVERY_CUSTOM_EVT || affChangeMsg != null;
 
@@ -1289,10 +1290,12 @@ private void changeWalModeIfNeeded() {
      * {@link GridCacheSharedContext#partitionReleaseFuture(AffinityTopologyVersion)} javadoc.
      *
      * @param distributed If {@code true} then node should wait for partition release completion on all other nodes.
+     * @param doRollback If {@code true} tries to rollback transactions which lock partitions. Avoids unnecessary calls
+     *      of {@link org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager#rollbackOnTopologyChange}
      *
      * @throws IgniteCheckedException If failed.
      */
-    private void waitPartitionRelease(boolean distributed) throws IgniteCheckedException {
+    private void waitPartitionRelease(boolean distributed, boolean doRollback) throws IgniteCheckedException {
         Latch releaseLatch = null;
 
         // Wait for other nodes only on first phase.
@@ -1312,34 +1315,37 @@ private void waitPartitionRelease(boolean distributed) throws IgniteCheckedExcep
 
         int dumpCnt = 0;
 
-        long waitStart = U.currentTimeMillis();
-
         long nextDumpTime = 0;
 
         IgniteConfiguration cfg = cctx.gridConfig();
 
-        boolean rollbackEnabled = cfg.getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange() > 0;
+        long waitStart = U.currentTimeMillis();
 
         long waitTimeout = 2 * cfg.getNetworkTimeout();
 
+        boolean txRolledBack = !doRollback;
+
         while (true) {
+            // Read txTimeoutOnPME from configuration after every iteration.
+            long curTimeout = cfg.getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange();
+
             try {
-                partReleaseFut.get(rollbackEnabled ?
-                    cfg.getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange() :
-                    waitTimeout, TimeUnit.MILLISECONDS);
+                // This avoids unnessesary waiting for rollback.
+                partReleaseFut.get(curTimeout > 0 && !txRolledBack ?
+                        Math.min(curTimeout, waitTimeout) : waitTimeout, TimeUnit.MILLISECONDS);
 
                 break;
             }
             catch (IgniteFutureTimeoutCheckedException ignored) {
                 // Print pending transactions and locks that might have led to hang.
                 if (nextDumpTime <= U.currentTimeMillis()) {
-                    dumpPendingObjects(partReleaseFut);
+                    dumpPendingObjects(partReleaseFut, curTimeout <= 0 && !txRolledBack);
 
                     nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, waitTimeout);
                 }
 
-                if (rollbackEnabled) {
-                    rollbackEnabled = false;
+                if (!txRolledBack && curTimeout > 0 && U.currentTimeMillis() - waitStart >= curTimeout) {
+                    txRolledBack = true;
 
                     cctx.tm().rollbackOnTopologyChange(initialVersion());
                 }
@@ -1448,12 +1454,17 @@ private void onLeft() {
 
     /**
      * @param partReleaseFut Partition release future.
+     * @param txTimeoutNotifyFlag If {@code true} print transaction rollback timeout on PME notification.
      */
-    private void dumpPendingObjects(IgniteInternalFuture<?> partReleaseFut) {
+    private void dumpPendingObjects(IgniteInternalFuture<?> partReleaseFut, boolean txTimeoutNotifyFlag) {
         U.warn(cctx.kernalContext().cluster().diagnosticLog(),
             "Failed to wait for partition release future [topVer=" + initialVersion() +
             ", node=" + cctx.localNodeId() + "]");
 
+        if (txTimeoutNotifyFlag)
+            U.warn(cctx.kernalContext().cluster().diagnosticLog(), "Consider changing TransactionConfiguration." +
+                    "txTimeoutOnPartitionMapExchange to non default value to avoid this message.");
+
         U.warn(log, "Partition release future: " + partReleaseFut);
 
         U.warn(cctx.kernalContext().cluster().diagnosticLog(),
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
index 79496ae1163c0..a7e29c41a16ee 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/marshaller/optimized/OptimizedMarshallerTest.java
@@ -413,12 +413,6 @@ public void _testAllocationOverflow() {
             return null;
         });
 
-        allocationOverflowCheck(() -> {
-            marshaller().marshal(new int[1<<30]);
-            marshaller().marshal(new int[1<<30]);
-            return null;
-        });
-
         allocationOverflowCheck(() -> {
             marshaller().marshal(new float[1<<29]);
             marshaller().marshal(new float[1<<29]);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java
index 3152349a30025..7033529c6810f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java
@@ -18,15 +18,25 @@
 package org.apache.ignite.internal.processors.cache;
 
 import java.lang.management.ManagementFactory;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import javax.management.MBeanServer;
 import javax.management.MBeanServerInvocationHandler;
 import javax.management.ObjectName;
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.TransactionConfiguration;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.TransactionsMXBeanImpl;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -36,6 +46,13 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionRollbackException;
+import org.apache.ignite.transactions.TransactionTimeoutException;
+
+import static org.apache.ignite.internal.util.typedef.X.hasCause;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
 /**
  *
@@ -119,6 +136,135 @@ public void testClusterSetTxTimeoutOnPartitionMapExchange() throws Exception {
         assertTxTimeoutOnPartitionMapExchange(expTimeout2);
     }
 
+    /**
+     * Tests applying new txTimeoutOnPartitionMapExchange while an exchange future runs.
+     *
+     * @throws Exception If fails.
+     */
+    public void testSetTxTimeoutDuringPartitionMapExchange() throws Exception {
+        IgniteEx ig = (IgniteEx)startGrids(2);
+
+        final long longTimeout = 600_000L;
+        final long shortTimeout = 5_000L;
+
+        TransactionsMXBean mxBean = txMXBean(0);
+
+        // Case 1: set very long txTimeoutOnPME, transaction should be rolled back.
+        mxBean.setTxTimeoutOnPartitionMapExchange(longTimeout);
+        assertTxTimeoutOnPartitionMapExchange(longTimeout);
+
+        AtomicReference<Exception> txEx = new AtomicReference<>();
+
+        IgniteInternalFuture<Long> fut = startDeadlock(ig, txEx, 0);
+
+        startGridAsync(2);
+
+        waitForExchangeStarted(ig);
+
+        mxBean.setTxTimeoutOnPartitionMapExchange(shortTimeout);
+
+        awaitPartitionMapExchange();
+
+        fut.get();
+
+        assertTrue("Transaction should be rolled back", hasCause(txEx.get(), TransactionRollbackException.class));
+
+        // Case 2: txTimeoutOnPME will be set to 0 after starting of PME, transaction should be cancelled on timeout.
+        mxBean.setTxTimeoutOnPartitionMapExchange(longTimeout);
+        assertTxTimeoutOnPartitionMapExchange(longTimeout);
+
+        fut = startDeadlock(ig, txEx, 10000L);
+
+        startGridAsync(3);
+
+        waitForExchangeStarted(ig);
+
+        mxBean.setTxTimeoutOnPartitionMapExchange(0);
+
+        fut.get();
+
+        assertTrue("Transaction should be canceled on timeout", hasCause(txEx.get(), TransactionTimeoutException.class));
+    }
+
+    /**
+     * Start test deadlock
+     *
+     * @param ig Ig.
+     * @param txEx Atomic reference to transaction exception.
+     * @param timeout Transaction timeout.
+     */
+    private IgniteInternalFuture<Long> startDeadlock(Ignite ig, AtomicReference<Exception> txEx, long timeout) {
+        IgniteCache<Object, Object> cache = ig.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+                .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL));
+
+        AtomicInteger thCnt = new AtomicInteger();
+
+        CyclicBarrier barrier = new CyclicBarrier(2);
+
+        return GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+            @Override public Void call() {
+                int thNum = thCnt.incrementAndGet();
+
+                try (Transaction tx = ig.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, timeout, 0)) {
+                    cache.put(thNum, 1);
+
+                    barrier.await();
+
+                    cache.put(thNum % 2 + 1, 1);
+
+                    tx.commit();
+                }
+                catch (Exception e) {
+                    txEx.set(e);
+                }
+
+                return null;
+            }
+        }, 2, "tx-thread");
+    }
+
+    /**
+     * Starts grid asynchronously and returns just before grid starting.
+     * Avoids blocking on PME.
+     *
+     * @param idx Test grid index.
+     * @throws Exception If fails.
+     */
+    private void startGridAsync(int idx) throws Exception {
+        GridTestUtils.runAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    startGrid(idx);
+                }
+                catch (Exception e) {
+                    // no-op.
+                }
+            }
+        });
+    }
+
+    /**
+     * Waits for srarting PME on grid.
+     *
+     * @param ig Ignite grid.
+     * @throws IgniteCheckedException If fails.
+     */
+    private void waitForExchangeStarted(IgniteEx ig) throws IgniteCheckedException {
+        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                for (GridDhtPartitionsExchangeFuture fut: ig.context().cache().context().exchange().exchangeFutures()) {
+                    if (!fut.isDone())
+                        return true;
+                }
+
+                return false;
+            }
+        }, WAIT_CONDITION_TIMEOUT);
+
+        // Additional waiting to ensure that code really start waiting for partition release.
+        U.sleep(5_000L);
+    }
+
     /**
      *
      */

From bac613b50c06dec633ff415ba3b8927730a228be Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Fri, 6 Jul 2018 19:09:57 +0300
Subject: [PATCH 257/543] IGNITE-8827 Disable WAL during apply updates on
 recovery

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit 8adff24)
---
 .../pagemem/store/IgnitePageStoreManager.java |   5 +
 .../processors/cache/WalStateManager.java     | 225 ++++++++++++++-
 .../GridCacheDatabaseSharedManager.java       | 105 +++----
 .../IgniteCacheDatabaseSharedManager.java     |   7 +
 .../file/FilePageStoreManager.java            |  37 ++-
 .../persistence/metastorage/MetaStorage.java  |   8 +-
 .../persistence/tree/util/PageHandler.java    |   2 +
 .../wal/FileWriteAheadLogManager.java         |  20 +-
 .../FsyncModeFileWriteAheadLogManager.java    |  15 +-
 ...IgniteNodeStoppedDuringDisableWALTest.java | 261 ++++++++++++++++++
 .../pagemem/NoOpPageStoreManager.java         |   5 +
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 12 files changed, 613 insertions(+), 80 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index 7dba8aee94c5e..5475bef76ebaa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -228,4 +228,9 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cac
      * @param cacheConfiguration Cache configuration of cache which should be cleanup.
      */
     public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws IgniteCheckedException;
+
+    /**
+     * Cleanup persistent space for all caches.
+     */
+    public void cleanupPersistentSpace() throws IgniteCheckedException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
index 3729ec610a3b2..b35b41e97bd46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -17,6 +17,16 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
@@ -28,9 +38,15 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointFuture;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage;
 import org.apache.ignite.internal.util.GridBoundedConcurrentLinkedHashSet;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -42,22 +58,12 @@
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.lang.IgniteUuid;
-import org.apache.ignite.thread.OomExceptionHandler;
 import org.apache.ignite.thread.IgniteThread;
+import org.apache.ignite.thread.OomExceptionHandler;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
 import static org.apache.ignite.internal.GridTopic.TOPIC_WAL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
@@ -113,6 +119,9 @@ public class WalStateManager extends GridCacheSharedManagerAdapter {
     /** Holder for groups with temporary disabled WAL. */
     private volatile TemporaryDisabledWal tmpDisabledWal;
 
+    /** */
+    private volatile WALDisableContext walDisableContext;
+
     /**
      * Constructor.
      *
@@ -157,6 +166,15 @@ public WalStateManager(GridKernalContext kernalCtx) {
     @Override protected void start0() throws IgniteCheckedException {
         if (srv)
             cctx.kernalContext().io().addMessageListener(TOPIC_WAL, ioLsnr);
+
+        walDisableContext = new WALDisableContext(
+            cctx.cache().context().database(),
+            cctx.pageStore(),
+            log
+        );
+
+        cctx.kernalContext().internalSubscriptionProcessor().registerMetastorageListener(walDisableContext);
+
     }
 
     /** {@inheritDoc} */
@@ -1026,6 +1044,40 @@ private WalStateResult awaitCheckpoint(CheckpointFuture cpFut, WalStateProposeMe
         return res;
     }
 
+    /**
+     * Checks WAL disabled for cache group.
+     *
+     * @param grpId Group id.
+     * @return {@code True} if WAL disable for group. {@code False} If not.
+     */
+    public boolean isDisabled(int grpId) {
+        CacheGroupContext ctx = cctx.cache().cacheGroup(grpId);
+
+        return ctx != null && !ctx.walEnabled();
+    }
+
+    /**
+     * @return WAL disable context.
+     */
+    public WALDisableContext walDisableContext(){
+        return walDisableContext;
+    }
+
+    /**
+     * None record will be logged in closure call.
+     *
+     * @param cls Closure to execute out of WAL scope.
+     * @throws IgniteCheckedException If operation failed.
+     */
+    public void runWithOutWAL(IgniteRunnable cls) throws IgniteCheckedException {
+        WALDisableContext ctx = walDisableContext;
+
+        if (ctx == null)
+            throw new IgniteCheckedException("Disable WAL context is not initialized.");
+
+        ctx.execute(cls);
+    }
+
     /**
      * WAL state change worker.
      */
@@ -1080,4 +1132,153 @@ public TemporaryDisabledWal(
             this.topVer = topVer;
         }
     }
+
+    /**
+     *
+     */
+    public static class WALDisableContext implements MetastorageLifecycleListener{
+        /** */
+        public static final String WAL_DISABLED = "wal-disabled";
+
+        /** */
+        private final IgniteLogger log;
+
+        /** */
+        private final IgniteCacheDatabaseSharedManager dbMgr;
+
+        /** */
+        private volatile ReadWriteMetastorage metaStorage;
+
+        /** */
+        private final IgnitePageStoreManager pageStoreMgr;
+
+        /** */
+        private volatile boolean resetWalFlag;
+
+        /** */
+        private volatile boolean disableWal;
+
+        /**
+         * @param dbMgr  Database manager.
+         * @param pageStoreMgr Page store manager.
+         * @param log
+         *
+         */
+        public WALDisableContext(
+            IgniteCacheDatabaseSharedManager dbMgr,
+            IgnitePageStoreManager pageStoreMgr,
+            @Nullable IgniteLogger log
+        ) {
+            this.dbMgr = dbMgr;
+            this.pageStoreMgr = pageStoreMgr;
+            this.log = log;
+        }
+
+        /**
+         * @param cls Closure to execute with disabled WAL.
+         * @throws IgniteCheckedException If execution failed.
+         */
+        public void execute(IgniteRunnable cls) throws IgniteCheckedException {
+            if (cls == null)
+                throw new IgniteCheckedException("Task to execute is not specified.");
+
+            if (metaStorage == null)
+                throw new IgniteCheckedException("Meta storage is not ready.");
+
+            writeMetaStoreDisableWALFlag();
+
+            dbMgr.waitForCheckpoint("Checkpoint before apply updates on recovery.");
+
+            disableWAL(true);
+
+            try {
+                cls.run();
+            }
+            catch (IgniteException e) {
+                throw new IgniteCheckedException(e);
+            }
+            finally {
+                disableWAL(false);
+
+                dbMgr.waitForCheckpoint("Checkpoint after apply updates on recovery.");
+
+                removeMetaStoreDisableWALFlag();
+            }
+        }
+
+        /**
+         * @throws IgniteCheckedException If write meta store flag failed.
+         */
+        protected void writeMetaStoreDisableWALFlag() throws IgniteCheckedException {
+            dbMgr.checkpointReadLock();
+
+            try {
+                metaStorage.write(WAL_DISABLED, Boolean.TRUE);
+            }
+            finally {
+                dbMgr.checkpointReadUnlock();
+            }
+        }
+
+        /**
+         * @throws IgniteCheckedException If remove meta store flag failed.
+         */
+        protected void removeMetaStoreDisableWALFlag() throws IgniteCheckedException {
+            dbMgr.checkpointReadLock();
+
+            try {
+                metaStorage.remove(WAL_DISABLED);
+            }
+            finally {
+                dbMgr.checkpointReadUnlock();
+            }
+        }
+
+        /**
+         * @param disable Flag wal disable.
+         */
+        protected void disableWAL(boolean disable) throws IgniteCheckedException {
+            dbMgr.checkpointReadLock();
+
+            try {
+                disableWal = disable;
+
+                if (log != null)
+                    log.info("WAL logging " + (disable ? "disabled" : "enabled"));
+            }
+            finally {
+                dbMgr.checkpointReadUnlock();
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onReadyForRead(ReadOnlyMetastorage ms) throws IgniteCheckedException {
+            Boolean disabled = (Boolean)ms.read(WAL_DISABLED);
+
+            // Node crash when WAL was disabled.
+            if (disabled != null && disabled){
+                resetWalFlag = true;
+
+                pageStoreMgr.cleanupPersistentSpace();
+
+                dbMgr.cleanupTempCheckpointDirectory();
+
+                dbMgr.cleanupCheckpointDirectory();
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onReadyForReadWrite(ReadWriteMetastorage ms) throws IgniteCheckedException {
+            // On new node start WAL always enabled. Remove flag from metastore.
+            if (resetWalFlag)
+                ms.remove(WAL_DISABLED);
+
+            metaStorage = ms;
+        }
+
+        /** {@inheritDoc} */
+        public boolean check() {
+            return disableWal;
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index e6778f4e86e39..9a60d14722d68 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -215,7 +215,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     private static final boolean ASSERTION_ENABLED = GridCacheDatabaseSharedManager.class.desiredAssertionStatus();
 
     /** Checkpoint file name pattern. */
-    private static final Pattern CP_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-(.*)-(START|END)\\.bin");
+    public static final Pattern CP_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-(.*)-(START|END)\\.bin");
 
     /** Checkpoint file temporary suffix. This is needed to safe writing checkpoint markers through temporary file and renaming. */
     public static final String FILE_TMP_SUFFIX = ".tmp";
@@ -349,7 +349,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
 
     /** File I/O factory for writing checkpoint markers. */
     private final FileIOFactory ioFactory;
-
     /**
      * @param ctx Kernal context.
      */
@@ -492,7 +491,7 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
     /**
      * Cleanup checkpoint directory from all temporary files {@link #FILE_TMP_SUFFIX}.
      */
-    private void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
+    public void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
         try {
             try (DirectoryStream<Path> files = Files.newDirectoryStream(
                 cpDir.toPath(),
@@ -825,8 +824,11 @@ private void unRegistrateMetricsMBean() {
 
             cctx.pageStore().initializeForMetastorage();
 
-            metaStorage = new MetaStorage(cctx, dataRegionMap.get(METASTORE_DATA_REGION_NAME),
-                (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME));
+            metaStorage = new MetaStorage(
+                cctx,
+                dataRegionMap.get(METASTORE_DATA_REGION_NAME),
+                (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME)
+            );
 
             WALPointer restore = restoreMemory(status);
 
@@ -837,7 +839,6 @@ private void unRegistrateMetricsMBean() {
 
             // First, bring memory to the last consistent checkpoint state if needed.
             // This method should return a pointer to the last valid record in the WAL.
-
             cctx.wal().resumeLogging(restore);
 
             WALPointer ptr = cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
@@ -2128,62 +2129,70 @@ public void applyUpdatesOnRecovery(
         IgnitePredicate<DataEntry> entryPredicate,
         Map<T2<Integer, Integer>, T2<Integer, Long>> partStates
     ) throws IgniteCheckedException {
-        if (it != null) {
-            while (it.hasNextX()) {
-                IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
+        cctx.walState().runWithOutWAL(() -> {
+            if (it != null) {
+                while (it.hasNext()) {
+                    IgniteBiTuple<WALPointer, WALRecord> next = it.next();
 
-                WALRecord rec = next.get2();
+                    WALRecord rec = next.get2();
 
-                if (!recPredicate.apply(next))
-                    break;
+                    if (!recPredicate.apply(next))
+                        break;
 
-                switch (rec.type()) {
-                    case DATA_RECORD:
-                        checkpointReadLock();
+                    switch (rec.type()) {
+                        case DATA_RECORD:
+                            checkpointReadLock();
 
-                        try {
-                            DataRecord dataRec = (DataRecord)rec;
+                            try {
+                                DataRecord dataRec = (DataRecord)rec;
 
-                            for (DataEntry dataEntry : dataRec.writeEntries()) {
-                                if (entryPredicate.apply(dataEntry)) {
-                                    checkpointReadLock();
+                                for (DataEntry dataEntry : dataRec.writeEntries()) {
+                                    if (entryPredicate.apply(dataEntry)) {
+                                        checkpointReadLock();
 
-                                    try {
-                                        int cacheId = dataEntry.cacheId();
+                                        try {
+                                            int cacheId = dataEntry.cacheId();
 
-                                        GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+                                            GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
-                                        if (cacheCtx != null)
-                                            applyUpdate(cacheCtx, dataEntry);
-                                        else if (log != null)
-                                            log.warning("Cache (cacheId=" + cacheId + ") is not started, can't apply updates.");
-                                    }
-                                    finally {
-                                        checkpointReadUnlock();
+                                            if (cacheCtx != null)
+                                                applyUpdate(cacheCtx, dataEntry);
+                                            else if (log != null)
+                                                log.warning("Cache (cacheId=" + cacheId + ") is not started, can't apply updates.");
+                                        }
+                                        finally {
+                                            checkpointReadUnlock();
+                                        }
                                     }
                                 }
                             }
-                        }
-                        finally {
-                            checkpointReadUnlock();
-                        }
+                            catch (IgniteCheckedException e) {
+                                throw new IgniteException(e);
+                            }
+                            finally {
+                                checkpointReadUnlock();
+                            }
 
-                        break;
+                            break;
 
-                    default:
-                        // Skip other records.
+                        default:
+                            // Skip other records.
+                    }
                 }
             }
-        }
 
-        checkpointReadLock();
+            checkpointReadLock();
 
-        try {
-            restorePartitionStates(partStates, null);
-        }
-        finally {
-            checkpointReadUnlock();
-        }
+            try {
+                restorePartitionStates(partStates, null);
+            }
+            catch (IgniteCheckedException e) {
+                throw new IgniteException(e);
+            }
+            finally {
+                checkpointReadUnlock();
+            }
+        });
     }
 
     /**
@@ -2315,8 +2324,10 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
      * @param onlyForGroups If not {@code null} restore states only for specified cache groups.
      * @throws IgniteCheckedException If failed to restore partition states.
      */
-    private void restorePartitionStates(Map<T2<Integer, Integer>, T2<Integer, Long>> partStates,
-                                       @Nullable Set<Integer> onlyForGroups) throws IgniteCheckedException {
+    private void restorePartitionStates(
+        Map<T2<Integer, Integer>, T2<Integer, Long>> partStates,
+        @Nullable Set<Integer> onlyForGroups
+    ) throws IgniteCheckedException {
         for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
             if (grp.isLocal() || !grp.affinityNode()) {
                 // Local cache has no partitions and its states.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 06f1c5762a983..f03d37fa83a6e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -739,6 +739,13 @@ public void cleanupCheckpointDirectory() throws IgniteCheckedException {
         // No-op.
     }
 
+    /**
+     * No-op for non-persistent storage.
+     */
+    public void cleanupTempCheckpointDirectory() throws IgniteCheckedException{
+        // No-op.
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 4dd275ddfe249..e611303d3c618 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -66,6 +66,9 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import static java.nio.file.Files.delete;
+import static java.nio.file.Files.newDirectoryStream;
+
 /**
  * File page store manager.
  */
@@ -97,6 +100,9 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
     /** */
     public static final String DFLT_STORE_DIR = "db";
 
+    /** */
+    public static final String META_STORAGE_NAME = "metastorage";
+
     /** Marshaller. */
     private static final Marshaller marshaller = new JdkMarshaller();
 
@@ -160,21 +166,41 @@ public FilePageStoreManager(GridKernalContext ctx) {
     }
 
     /** {@inheritDoc} */
-    public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws IgniteCheckedException {
+    @Override public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws IgniteCheckedException {
         try {
             File cacheWorkDir = cacheWorkDir(cacheConfiguration);
 
             if(!cacheWorkDir.exists())
                 return;
 
-            try (DirectoryStream<Path> files = Files.newDirectoryStream(cacheWorkDir.toPath(),
+            try (DirectoryStream<Path> files = newDirectoryStream(cacheWorkDir.toPath(),
                 new DirectoryStream.Filter<Path>() {
                     @Override public boolean accept(Path entry) throws IOException {
                         return entry.toFile().getName().endsWith(FILE_SUFFIX);
                     }
                 })) {
                 for (Path path : files)
-                    Files.delete(path);
+                    delete(path);
+            }
+        }
+        catch (IOException e) {
+            throw new IgniteCheckedException("Failed to cleanup persistent directory: ", e);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cleanupPersistentSpace() throws IgniteCheckedException {
+        try {
+            try (DirectoryStream<Path> files = newDirectoryStream(
+                storeWorkDir.toPath(), entry -> {
+                    String name = entry.toFile().getName();
+
+                    return !name.equals(META_STORAGE_NAME) &&
+                        (name.startsWith(CACHE_DIR_PREFIX) || name.startsWith(CACHE_GRP_DIR_PREFIX));
+                }
+            )) {
+                for (Path path : files)
+                    U.delete(path);
             }
         }
         catch (IOException e) {
@@ -261,8 +287,7 @@ public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws
 
         if (!idxCacheStores.containsKey(grpId)) {
             CacheStoreHolder holder = initDir(
-                new File(storeWorkDir,
-                    "metastorage"),
+                new File(storeWorkDir, META_STORAGE_NAME),
                 grpId,
                 1,
                 delta -> {/* No-op */} );
@@ -840,7 +865,7 @@ private void removeCacheGroupConfigurationData(CacheGroupContext ctx) throws Ign
                 }
             };
 
-            try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(cacheGrpDir.toPath(), cacheCfgFileFilter)) {
+            try (DirectoryStream<Path> dirStream = newDirectoryStream(cacheGrpDir.toPath(), cacheCfgFileFilter)) {
                 for(Path path: dirStream)
                     Files.deleteIfExists(path);
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index e2e26e8b9372b..14bd450471401 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -115,8 +115,12 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     private final Marshaller marshaller = new JdkMarshaller();
 
     /** */
-    public MetaStorage(GridCacheSharedContext cctx, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
-        boolean readOnly) {
+    public MetaStorage(
+        GridCacheSharedContext cctx,
+        DataRegion dataRegion,
+        DataRegionMetricsImpl regionMetrics,
+        boolean readOnly
+    ) {
         wal = cctx.wal();
         this.dataRegion = dataRegion;
         this.regionMetrics = regionMetrics;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
index 8f854cffeea9d..a52038ab66cc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
@@ -23,6 +23,8 @@
 import org.apache.ignite.internal.pagemem.PageSupport;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedManager;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.util.GridUnsafe;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 7cfacb2e7d84a..d56fed764beec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -68,7 +68,6 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.events.EventType;
 import org.apache.ignite.events.WalSegmentArchivedEvent;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.failure.FailureType;
@@ -85,9 +84,9 @@
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
 import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
+import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
 import org.apache.ignite.internal.processors.cache.persistence.DataStorageMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
@@ -176,7 +175,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     public static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal");
 
     /** */
-    private static final Pattern WAL_TEMP_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal\\.tmp");
+    public static final Pattern WAL_TEMP_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal\\.tmp");
 
     /** WAL segment file filter, see {@link #WAL_NAME_PATTERN} */
     public static final FileFilter WAL_SEGMENT_FILE_FILTER = new FileFilter() {
@@ -317,6 +316,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     /** Current log segment handle */
     private volatile FileWriteHandle currHnd;
 
+    /** */
+    private volatile WALDisableContext walDisableContext;
+
     /**
      * Positive (non-0) value indicates WAL can be archived even if not complete<br>
      * See {@link DataStorageConfiguration#setWalAutoArchiveAfterInactivity(long)}<br>
@@ -450,6 +452,8 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
                 }
             }
 
+            walDisableContext = cctx.walState().walDisableContext();
+
             if (mode != WALMode.NONE) {
                 if (log.isInfoEnabled())
                     log.info("Started write-ahead log manager [mode=" + mode + ']');
@@ -729,8 +733,10 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
 
         FileWriteHandle currWrHandle = currentHandle();
 
+        WALDisableContext isDisable = walDisableContext;
+
         // Logging was not resumed yet.
-        if (currWrHandle == null)
+        if (currWrHandle == null || (isDisable != null && isDisable.check()))
             return null;
 
         // Need to calculate record size first.
@@ -996,9 +1002,7 @@ private boolean segmentReservedOrLocked(long absIdx) {
 
     /** {@inheritDoc} */
     @Override public boolean disabled(int grpId) {
-        CacheGroupContext ctx = cctx.cache().cacheGroup(grpId);
-
-        return ctx != null && !ctx.walEnabled();
+        return cctx.walState().isDisabled(grpId);
     }
 
     /**
@@ -1357,7 +1361,7 @@ private void checkOrPrepareFiles() throws StorageException {
     }
 
     /** {@inheritDoc} */
-    public void cleanupWalDirectories() throws IgniteCheckedException {
+    @Override public void cleanupWalDirectories() throws IgniteCheckedException {
         try {
             try (DirectoryStream<Path> files = Files.newDirectoryStream(walWorkDir.toPath())) {
                 for (Path path : files)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index f1a60a4c64dae..2f64bd7584e3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -78,9 +78,9 @@
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
 import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
+import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
 import org.apache.ignite.internal.processors.cache.persistence.DataStorageMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
@@ -268,6 +268,9 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
     /** Current log segment handle */
     private volatile FileWriteHandle currentHnd;
 
+    /** */
+    private volatile WALDisableContext walDisableContext;
+
     /**
      * Positive (non-0) value indicates WAL can be archived even if not complete<br>
      * See {@link DataStorageConfiguration#setWalAutoArchiveAfterInactivity(long)}<br>
@@ -387,6 +390,8 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
                 }
             }
 
+            walDisableContext = cctx.walState().walDisableContext();
+
             if (mode != WALMode.NONE) {
                 if (log.isInfoEnabled())
                     log.info("Started write-ahead log manager [mode=" + mode + ']');
@@ -652,8 +657,10 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
 
         FileWriteHandle currWrHandle = currentHandle();
 
+        WALDisableContext isDisable = walDisableContext;
+
         // Logging was not resumed yet.
-        if (currWrHandle == null)
+        if (currWrHandle == null || (isDisable != null && isDisable.check()))
             return null;
 
         // Need to calculate record size first.
@@ -906,9 +913,7 @@ private boolean hasIndex(long absIdx) {
 
     /** {@inheritDoc} */
     @Override public boolean disabled(int grpId) {
-        CacheGroupContext ctx = cctx.cache().cacheGroup(grpId);
-
-        return ctx != null && !ctx.walEnabled();
+        return cctx.walState().isDisabled(grpId);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
new file mode 100644
index 0000000000000..80198e8af14ee
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashMap;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+import static java.nio.file.FileVisitResult.CONTINUE;
+import static java.nio.file.Files.walkFileTree;
+import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.CP_FILE_NAME_PATTERN;
+import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.FILE_TMP_SUFFIX;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.META_STORAGE_NAME;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.WAL_NAME_PATTERN;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.WAL_TEMP_NAME_PATTERN;
+import static org.apache.ignite.testframework.GridTestUtils.setFieldValue;
+
+/***
+ *
+ */
+public class IgniteNodeStoppedDuringDisableWALTest extends GridCommonAbstractTest {
+    /** */
+    public static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(name);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        cfg.setAutoActivationEnabled(false);
+
+        cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void test() throws Exception {
+        for (NodeStopPoint nodeStopPoint : NodeStopPoint.values()) {
+            testStopNodeWithDisableWAL(nodeStopPoint);
+
+            stopAllGrids();
+
+            cleanPersistenceDir();
+        }
+    }
+
+    /**
+     * @param nodeStopPoint Stop point.
+     * @throws Exception If failed.
+     */
+    private void testStopNodeWithDisableWAL(NodeStopPoint nodeStopPoint) throws Exception {
+        log.info("Start test crash " + nodeStopPoint);
+
+        IgniteEx ig0 = startGrid(0);
+
+        GridCacheSharedContext<Object, Object> sharedContext = ig0.context().cache().context();
+
+        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)sharedContext.database();
+        IgniteWriteAheadLogManager WALmgr = sharedContext.wal();
+
+        WALDisableContext walDisableContext = new WALDisableContext(dbMgr, sharedContext.pageStore(), log) {
+            @Override protected void writeMetaStoreDisableWALFlag() throws IgniteCheckedException {
+                if (nodeStopPoint == NodeStopPoint.BEFORE_WRITE_KEY_TO_META_STORE)
+                    failNode(nodeStopPoint);
+
+                super.writeMetaStoreDisableWALFlag();
+
+                if (nodeStopPoint == NodeStopPoint.AFTER_WRITE_KEY_TO_META_STORE)
+                    failNode(nodeStopPoint);
+            }
+
+            @Override protected void removeMetaStoreDisableWALFlag() throws IgniteCheckedException {
+                if (nodeStopPoint == NodeStopPoint.AFTER_CHECKPOINT_AFTER_ENABLE_WAL)
+                    failNode(nodeStopPoint);
+
+                super.removeMetaStoreDisableWALFlag();
+
+                if (nodeStopPoint == NodeStopPoint.AFTER_REMOVE_KEY_TO_META_STORE)
+                    failNode(nodeStopPoint);
+            }
+
+            @Override protected void disableWAL(boolean disable) throws IgniteCheckedException {
+                if (disable) {
+                    if (nodeStopPoint == NodeStopPoint.AFTER_CHECKPOINT_BEFORE_DISABLE_WAL)
+                        failNode(nodeStopPoint);
+
+                    super.disableWAL(disable);
+
+                    if (nodeStopPoint == NodeStopPoint.AFTER_DISABLE_WAL)
+                        failNode(nodeStopPoint);
+
+                }
+                else {
+                    super.disableWAL(disable);
+
+                    if (nodeStopPoint == NodeStopPoint.AFTER_ENABLE_WAL)
+                        failNode(nodeStopPoint);
+                }
+            }
+        };
+
+        setFieldValue(sharedContext.walState(), "walDisableContext", walDisableContext);
+
+        setFieldValue(WALmgr, "walDisableContext", walDisableContext);
+
+        ig0.context().internalSubscriptionProcessor().registerMetastorageListener(walDisableContext);
+
+        ig0.cluster().active(true);
+
+        try (IgniteDataStreamer<Integer, Integer> st = ig0.dataStreamer(DEFAULT_CACHE_NAME)) {
+            st.allowOverwrite(true);
+
+            for (int i = 0; i < 10_000; i++)
+                st.addData(i, -i);
+        }
+
+        boolean fail = false;
+
+        try (WALIterator it = sharedContext.wal().replay(null)) {
+            dbMgr.applyUpdatesOnRecovery(it, (tup) -> true, (entry) -> true, new HashMap<>());
+        }
+        catch (IgniteCheckedException e) {
+            if (nodeStopPoint.needCleanUp)
+                fail = true;
+        }
+
+        Assert.assertEquals(nodeStopPoint.needCleanUp, fail);
+
+        Ignite ig1 = startGrid(0);
+
+        String msg = nodeStopPoint.toString();
+
+        if (nodeStopPoint.needCleanUp) {
+            PdsFoldersResolver foldersResolver = ((IgniteEx)ig1).context().pdsFolderResolver();
+
+            File root = foldersResolver.resolveFolders().persistentStoreRootPath();
+
+            walkFileTree(root.toPath(), new SimpleFileVisitor<Path>() {
+                @Override public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
+                    String name = path.toFile().getName();
+
+                    String filePath = path.toString();
+
+                    if (path.toFile().getParentFile().getName().equals(META_STORAGE_NAME))
+                        return CONTINUE;
+
+                    if (WAL_NAME_PATTERN.matcher(name).matches() || WAL_TEMP_NAME_PATTERN.matcher(name).matches())
+                        return CONTINUE;
+
+                    boolean failed = false;
+
+                    if (name.endsWith(FILE_TMP_SUFFIX))
+                        failed = true;
+
+                    if (CP_FILE_NAME_PATTERN.matcher(name).matches())
+                        failed = true;
+
+                    if (name.startsWith(PART_FILE_PREFIX))
+                        failed = true;
+
+                    if (name.startsWith(INDEX_FILE_NAME))
+                        failed = true;
+
+                    if (failed)
+                        fail(msg + " " + filePath);
+
+                    return CONTINUE;
+                }
+            });
+        }
+    }
+
+    /**
+     * @param nodeStopPoint Stop point.
+     * @throws IgniteCheckedException Always throws exception.
+     */
+    private void failNode(NodeStopPoint nodeStopPoint) throws IgniteCheckedException {
+        stopGrid(0, true);
+
+        throw new IgniteCheckedException(nodeStopPoint.toString());
+    }
+
+    /**
+     * Crash point.
+     */
+    private enum NodeStopPoint {
+        BEFORE_WRITE_KEY_TO_META_STORE(false),
+        AFTER_WRITE_KEY_TO_META_STORE(true),
+        AFTER_CHECKPOINT_BEFORE_DISABLE_WAL(true),
+        AFTER_DISABLE_WAL(true),
+        AFTER_ENABLE_WAL(true),
+        AFTER_CHECKPOINT_AFTER_ENABLE_WAL(true),
+        AFTER_REMOVE_KEY_TO_META_STORE(false);
+
+        /** Clean up flag. */
+        private final boolean needCleanUp;
+
+        NodeStopPoint(boolean up) {
+            needCleanUp = up;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index ba236af929365..c61b3c0a39a2d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -221,4 +221,9 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
     @Override public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws IgniteCheckedException {
         // No-op.
     }
+
+    /** {@inheritDoc} */
+    @Override public void cleanupPersistentSpace() throws IgniteCheckedException {
+        // No-op.
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 9779ca58801e8..b6e5082b42ab0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsUnusedWalSegmentsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.filename.IgniteUidAsConsistentIdMigrationTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteNodeStoppedDuringDisableWALTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundWithMmapBufferSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncSelfTest;
@@ -180,5 +181,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgniteWalIteratorSwitchSegmentTest.class);
 
         suite.addTestSuite(IgniteWalIteratorExceptionDuringReadTest.class);
+
+        suite.addTestSuite(IgniteNodeStoppedDuringDisableWALTest.class);
     }
 }

From b18f5a7158cd7d04d8d4477e77854871bd4371e3 Mon Sep 17 00:00:00 2001
From: ezagumennov <ezagumennov@gridgain.com>
Date: Mon, 16 Jul 2018 17:15:05 +0300
Subject: [PATCH 258/543] IGNITE-8745 Add ability to monitor TCP discovery ring
 information - Fixes #4256.

Signed-off-by: Ivan Rakov <irakov@apache.org>
---
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 17 ++++
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 15 ++++
 .../spi/discovery/tcp/TcpDiscoveryImpl.java   | 14 ++++
 .../spi/discovery/tcp/TcpDiscoverySpi.java    | 11 +++
 .../discovery/tcp/TcpDiscoverySpiMBean.java   | 14 ++++
 .../tcp/TcpDiscoverySpiMBeanTest.java         | 77 +++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |  2 +
 7 files changed, 150 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBeanTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index dc62bf3331fb2..934b1da6e3880 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -26,8 +26,10 @@
 import java.net.SocketTimeoutException;
 import java.util.ArrayDeque;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -230,6 +232,21 @@ class ClientImpl extends TcpDiscoveryImpl {
         U.quietAndInfo(log, b.toString());
     }
 
+    /** {@inheritDoc} */
+    @Override public void dumpRingStructure(IgniteLogger log) {
+        ClusterNode[] serverNodes = getRemoteNodes().stream()
+                .filter(node -> !node.isClient())
+                .sorted(Comparator.comparingLong(ClusterNode::order))
+                .toArray(ClusterNode[]::new);
+
+        U.quietAndInfo(log, Arrays.toString(serverNodes));
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCurrentTopologyVersion() {
+        return topVer;
+    }
+
     /** {@inheritDoc} */
     @Override public String getSpiState() {
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index bb76895bd204c..5ce7f20bec33b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -1849,6 +1849,16 @@ TcpDiscoveryNodesRing ring() {
         U.quietAndInfo(log, b.toString());
     }
 
+    /** {@inheritDoc} */
+    @Override public void dumpRingStructure(IgniteLogger log) {
+        U.quietAndInfo(log, ring.toString());
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getCurrentTopologyVersion() {
+        return ring.topologyVersion();
+    }
+
     /**
      * @param msg Message.
      * @return {@code True} if recordable in debug mode.
@@ -5606,6 +5616,11 @@ private void checkConnection() {
                 lastTimeConnCheckMsgSent = U.currentTimeMillis();
             }
         }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return String.format("%s, nextNode=[%s]", super.toString(), next);
+        }
     }
 
     /** Thread that executes {@link TcpServer}'s code. */
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
index 00d83dd3ee154..c9c48c2cfb10d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryImpl.java
@@ -231,6 +231,20 @@ protected void onMessageExchanged() {
      */
     public abstract void failNode(UUID nodeId, @Nullable String warning);
 
+    /**
+     * Dumps ring structure to logger.
+     *
+     * @param log Logger.
+     */
+    public abstract void dumpRingStructure(IgniteLogger log);
+
+    /**
+     * Get current topology version.
+     *
+     * @return Current topology version.
+     */
+    public abstract long getCurrentTopologyVersion();
+
     /**
      * @param igniteInstanceName Ignite instance name.
      * @throws IgniteSpiException If failed.
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 801f2b6565521..73f53bf430982 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -2503,8 +2503,19 @@ private class TcpDiscoverySpiMBeanImpl extends IgniteSpiMBeanAdapter implements
             return TcpDiscoverySpi.this.getCoordinatorSinceTimestamp();
         }
 
+        /** {@inheritDoc} */
         @Override public void checkRingLatency(int maxHops) {
             TcpDiscoverySpi.this.impl.checkRingLatency(maxHops);
         }
+
+        /** {@inheritDoc} */
+        @Override public long getCurrentTopologyVersion() {
+            return impl.getCurrentTopologyVersion();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void dumpRingStructure() {
+            impl.dumpRingStructure(log);
+        }
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
index cb0fd362c4a54..37dd45e8f7f83 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBean.java
@@ -281,4 +281,18 @@ public interface TcpDiscoverySpiMBean extends IgniteSpiManagementMBean, Discover
         }
     )
     public void checkRingLatency(int maxHops);
+
+    /**
+     * Current topology version.
+     *
+     * @return current topVer.
+     */
+    @MXBeanDescription("Get current topology version.")
+    public long getCurrentTopologyVersion();
+
+    /**
+     * Dumps ring structure to log.
+     */
+    @MXBeanDescription("Dumps ring structure to log.")
+    public void dumpRingStructure();
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBeanTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBeanTest.java
new file mode 100644
index 0000000000000..350b05ccbf083
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpiMBeanTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridStringLogger;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import javax.management.JMX;
+import javax.management.MBeanServer;
+import javax.management.ObjectName;
+import java.lang.management.ManagementFactory;
+
+/**
+ * Tests TcpDiscoverySpiMBean.
+ */
+public class TcpDiscoverySpiMBeanTest extends GridCommonAbstractTest {
+    /** */
+    private GridStringLogger strLog = new GridStringLogger();
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(final String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+        TcpDiscoverySpi tcpSpi = new TcpDiscoverySpi();
+        cfg.setDiscoverySpi(tcpSpi);
+        cfg.setGridLogger(strLog);
+
+        return cfg;
+    }
+
+    /**
+     * Tests TcpDiscoverySpiMBean#getCurrentTopologyVersion() and TcpDiscoverySpiMBean#dumpRingStructure().
+     *
+     * @throws Exception if fails.
+     */
+    public void testMBean() throws Exception {
+        startGrids(3);
+
+        MBeanServer srv = ManagementFactory.getPlatformMBeanServer();
+
+        try {
+            for (int i = 0; i < 3; i++) {
+                IgniteEx grid = grid(i);
+
+                ObjectName spiName = U.makeMBeanName(grid.context().igniteInstanceName(), "SPIs",
+                        TcpDiscoverySpi.class.getSimpleName());
+
+                TcpDiscoverySpiMBean bean = JMX.newMBeanProxy(srv, spiName, TcpDiscoverySpiMBean.class);
+
+                assertNotNull(bean);
+                assertEquals(grid.cluster().topologyVersion(), bean.getCurrentTopologyVersion());
+
+                bean.dumpRingStructure();
+                assertTrue(strLog.toString().contains("TcpDiscoveryNodesRing"));
+            }
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index ef582a5df7565..984dcf696fe97 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySnapshotHistoryTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiConfigSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiFailureTimeoutSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiMBeanTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiReconnectDelayTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiStartStopSelfTest;
@@ -79,6 +80,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(TcpDiscoverySelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySpiSelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySpiFailureTimeoutSelfTest.class));
+        suite.addTest(new TestSuite(TcpDiscoverySpiMBeanTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySpiStartStopSelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySpiConfigSelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoveryMarshallerCheckSelfTest.class));

From 489acccb2408ffb1ef103e94040a1c268fba2d31 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Tue, 17 Jul 2018 16:48:43 +0300
Subject: [PATCH 259/543] IGNITE-8684 Fixed infinite loop of partition
 single/full messages when partition state does not change - Fixes #4287.

(cherry picked from commit dd47fab)
---
 .../dht/GridDhtPartitionTopologyImpl.java     |   6 +-
 ...RebalanceScheduleResendPartitionsTest.java | 288 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 3 files changed, 296 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteRebalanceScheduleResendPartitionsTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index c79121987f688..78012dfe314c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -1509,7 +1509,11 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
 
                 GridDhtPartitionMap nodeMap = partMap.get(ctx.localNodeId());
 
-                if (nodeMap != null && grp.persistenceEnabled() && readyTopVer.initialized()) {
+                // Only in real exchange occurred.
+                if (exchangeVer != null &&
+                    nodeMap != null &&
+                    grp.persistenceEnabled() &&
+                    readyTopVer.initialized()) {
                     for (Map.Entry<Integer, GridDhtPartitionState> e : nodeMap.entrySet()) {
                         int p = e.getKey();
                         GridDhtPartitionState state = e.getValue();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteRebalanceScheduleResendPartitionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteRebalanceScheduleResendPartitionsTest.java
new file mode 100644
index 0000000000000..274b16d5e2225
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteRebalanceScheduleResendPartitionsTest.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.typedef.T3;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.resources.LoggerResource;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+
+/**
+ *
+ */
+public class IgniteRebalanceScheduleResendPartitionsTest extends GridCommonAbstractTest {
+    /** */
+    public static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(name);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setConsistentId(name);
+
+        cfg.setAutoActivationEnabled(false);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        cfg.setCacheConfiguration(
+            new CacheConfiguration(DEFAULT_CACHE_NAME)
+                .setAffinity(
+                    new RendezvousAffinityFunction(false, 32))
+                .setBackups(1)
+        );
+
+        cfg.setCommunicationSpi(new BlockTcpCommunicationSpi());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * https://issues.apache.org/jira/browse/IGNITE-8684
+     *
+     * @throws Exception If failed.
+     */
+    public void test() throws Exception {
+        Ignite ig0 = startGrids(3);
+
+        ig0.cluster().active(true);
+
+        int entries = 100_000;
+
+        try (IgniteDataStreamer<Integer, Integer> st = ig0.dataStreamer(DEFAULT_CACHE_NAME)) {
+            st.allowOverwrite(true);
+
+            for (int i = 0; i < entries; i++)
+                st.addData(i, -i);
+        }
+
+        IgniteEx ig3 = startGrid(3);
+
+        AtomicInteger cnt = new AtomicInteger();
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        runAsync(() -> {
+            try {
+                latch.await();
+
+                // Sleep should be less that full awaitPartitionMapExchange().
+                Thread.sleep(super.getPartitionMapExchangeTimeout());
+
+                log.info("Await completed, continue rebalance.");
+
+                unwrapSPI(ig3).resume();
+            }
+            catch (InterruptedException ignored) {
+                // No-op.
+            }
+        });
+
+        // Compare previous single message with current.
+        MessageComparator prevMessageComparator = new MessageComparator();
+
+        // Pause rebalance and count of single map messages.
+        unwrapSPI(ig3).pause(GridDhtPartitionDemandMessage.class, (msg) -> {
+            System.out.println("Send partition single message:" + msg);
+
+            // Continue after baseline changed exchange occurred.
+            if (msg.exchangeId() != null)
+                latch.countDown();
+
+            if (prevMessageComparator.prevEquals(msg))
+                cnt.incrementAndGet();
+        });
+
+        ig3.cluster().setBaselineTopology(ig3.context().discovery().topologyVersion());
+
+        awaitPartitionMapExchange();
+
+        // We should not send equals single map on schedule partition state.
+        Assert.assertEquals(0, cnt.get());
+
+        IgniteCache<Integer, Integer> cache = ig3.cache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < entries; i++) {
+            Integer val = cache.get(i);
+
+            Assert.assertEquals(-i, (int)val);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getPartitionMapExchangeTimeout() {
+        return super.getPartitionMapExchangeTimeout() * 2;
+    }
+
+    /**
+     * @param ig Ignite.
+     * @return BlockTcpCommunicationSpi.
+     */
+    private BlockTcpCommunicationSpi unwrapSPI(IgniteEx ig) {
+        return ((BlockTcpCommunicationSpi)ig.configuration().getCommunicationSpi());
+    }
+
+    /**
+     * Compare previous single message with current.
+     */
+    private static class MessageComparator {
+        /** */
+        private GridDhtPartitionsSingleMessage prev;
+
+        /**
+         * @param msg Partition single message.
+         */
+        private synchronized boolean prevEquals(GridDhtPartitionsSingleMessage msg) {
+            if (msg.exchangeId() != null)
+                return false;
+
+            if (prev == null) {
+                prev = msg;
+
+                return false;
+            }
+
+            AtomicBoolean prevEquals = new AtomicBoolean(true);
+
+            prev.partitions().forEach((k0, v0) -> {
+                GridDhtPartitionMap val1 = msg.partitions().get(k0);
+
+                if (val1 == null)
+                    prevEquals.set(false);
+
+                boolean equals = v0.map().equals(val1.map());
+
+                prevEquals.set(prevEquals.get() && equals);
+            });
+
+            prev = msg;
+
+            return prevEquals.get();
+        }
+    }
+
+    /**
+     *
+     */
+    protected static class BlockTcpCommunicationSpi extends TcpCommunicationSpi {
+        private volatile IgniteInClosure<GridDhtPartitionsSingleMessage> cls;
+
+        /** */
+        private volatile Class msgCls;
+
+        /** */
+        private final Queue<T3<ClusterNode, Message, IgniteInClosure>> queue = new ConcurrentLinkedQueue<>();
+
+        /** */
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC)
+            throws IgniteSpiException {
+            Class msgCls0 = msgCls;
+
+            if (msgCls0 != null && ((GridIoMessage)msg).message().getClass().equals(msgCls0)) {
+                queue.add(new T3<>(node, msg, ackC));
+
+                log.info("Block message: " + msg);
+
+                return;
+            }
+
+            if (((GridIoMessage)msg).message().getClass().equals(GridDhtPartitionsSingleMessage.class)) {
+                if (cls != null)
+                    cls.apply((GridDhtPartitionsSingleMessage)((GridIoMessage)msg).message());
+            }
+
+            super.sendMessage(node, msg, ackC);
+        }
+
+        /**
+         * @param msgCls Message class.
+         */
+        private synchronized void pause(Class msgCls, IgniteInClosure<GridDhtPartitionsSingleMessage> cls) {
+            this.msgCls = msgCls;
+            this.cls = cls;
+        }
+
+        /**
+         *
+         */
+        private synchronized void resume() {
+            msgCls = null;
+
+            for (T3<ClusterNode, Message, IgniteInClosure> msg : queue)
+                super.sendMessage(msg.get1(), msg.get2(), msg.get3());
+
+            queue.clear();
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index b6e5082b42ab0..4643abbc50bb9 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteRebalanceScheduleResendPartitionsTest;
 import org.apache.ignite.internal.processors.cache.persistence.LocalWalModeChangeDuringRebalancingSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAbsentEvictionNodeOutOfBaselineTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.ClientAffinityAssignmentWithBaselineTest;
@@ -183,5 +184,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgniteWalIteratorExceptionDuringReadTest.class);
 
         suite.addTestSuite(IgniteNodeStoppedDuringDisableWALTest.class);
+
+        suite.addTestSuite(IgniteRebalanceScheduleResendPartitionsTest.class);
     }
 }

From 0ae822cba971687256f7b8843e5ff61b7f6dc936 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Tue, 17 Jul 2018 16:52:22 +0300
Subject: [PATCH 260/543] IGNITE-8975 Invalid initialization of compressed
 archived WAL segment when WAL compression is switched off. - Fixes #4345.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 46db052)
---
 .../wal/FileWriteAheadLogManager.java         | 11 +++++++---
 .../FsyncModeFileWriteAheadLogManager.java    | 15 ++++++++-----
 .../persistence/db/wal/WalCompactionTest.java | 21 +++++++++++++++++++
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index d56fed764beec..fcc565b6b22b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -3048,7 +3048,9 @@ private RecordsIterator(
             @NotNull AbstractFileDescriptor desc,
             @Nullable FileWALPointer start
         ) throws IgniteCheckedException, FileNotFoundException {
-            if (decompressor != null && !desc.file().exists()) {
+            AbstractFileDescriptor currDesc = desc;
+
+            if (!desc.file().exists()) {
                 FileDescriptor zipFile = new FileDescriptor(
                     new File(walArchiveDir, FileDescriptor.fileName(desc.idx()) + ".zip"));
 
@@ -3057,10 +3059,13 @@ private RecordsIterator(
                         "[segmentIdx=" + desc.idx() + "]");
                 }
 
-                decompressor.decompressFile(desc.idx()).get();
+                if (decompressor != null)
+                    decompressor.decompressFile(desc.idx()).get();
+                else
+                    currDesc = zipFile;
             }
 
-            return (ReadFileHandle) super.initReadHandle(desc, start);
+            return (ReadFileHandle) super.initReadHandle(currDesc, start);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 2f64bd7584e3e..7521f73190c4e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -3175,19 +3175,24 @@ private RecordsIterator(
             @NotNull AbstractFileDescriptor desc,
             @Nullable FileWALPointer start
         ) throws IgniteCheckedException, FileNotFoundException {
-            if (decompressor != null && !desc.file().exists()) {
+            AbstractFileDescriptor currDesc = desc;
+
+            if (!desc.file().exists()) {
                 FileDescriptor zipFile = new FileDescriptor(
-                    new File(walArchiveDir, FileDescriptor.fileName(desc.idx()) + ".zip"));
+                        new File(walArchiveDir, FileDescriptor.fileName(desc.idx()) + ".zip"));
 
                 if (!zipFile.file.exists()) {
                     throw new FileNotFoundException("Both compressed and raw segment files are missing in archive " +
-                        "[segmentIdx=" + desc.idx() + "]");
+                            "[segmentIdx=" + desc.idx() + "]");
                 }
 
-                decompressor.decompressFile(desc.idx()).get();
+                if (decompressor != null)
+                    decompressor.decompressFile(desc.idx()).get();
+                else
+                    currDesc = zipFile;
             }
 
-            return (ReadFileHandle) super.initReadHandle(desc, start);
+            return (ReadFileHandle) super.initReadHandle(currDesc, start);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
index fe0169f70d908..39513b8c7d6c7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
@@ -112,9 +112,28 @@ public class WalCompactionTest extends GridCommonAbstractTest {
     }
 
     /**
+     * Tests applying updates from compacted WAL archive.
+     *
      * @throws Exception If failed.
      */
     public void testApplyingUpdatesFromCompactedWal() throws Exception {
+        testApplyingUpdatesFromCompactedWal(false);
+    }
+
+    /**
+     * Tests applying updates from compacted WAL archive when compressor is disabled.
+     *
+     * @throws Exception If failed.
+     */
+    public void testApplyingUpdatesFromCompactedWalWhenCompressorDisabled() throws Exception {
+        testApplyingUpdatesFromCompactedWal(true);
+    }
+
+    /**
+     * @param switchOffCompressor Switch off compressor after restart.
+     * @throws Exception If failed.
+     */
+    private void testApplyingUpdatesFromCompactedWal(boolean switchOffCompressor) throws Exception {
         IgniteEx ig = (IgniteEx)startGrids(3);
         ig.active(true);
 
@@ -172,6 +191,8 @@ public void testApplyingUpdatesFromCompactedWal() throws Exception {
         for (File f : lfsFiles)
             f.delete();
 
+        compactionEnabled = !switchOffCompressor;
+
         ig = (IgniteEx)startGrids(3);
         ig.active(true);
 

From b5ce4f44fd767c178daa9b05590368b0e9e648cd Mon Sep 17 00:00:00 2001
From: Eduard Shangareev <eduard.shangareev@gmail.com>
Date: Thu, 12 Jul 2018 19:26:48 +0300
Subject: [PATCH 261/543] IGNITE-8955 Fix of test after Checkpoint can't get
 write lock if massive eviction on node start started

(cherry picked from commit 584a88d)
Signed-off-by: EdShangGG <eshangareev@gridgain.com>
---
 ...PdsCheckpointSimulationWithRealCpDisabledTest.java | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
index 08ce6d78d8555..0fbb2556d1caf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
@@ -897,7 +897,16 @@ private IgniteBiTuple<Map<FullPageId, Integer>, WALPointer> runCheckpointing(
                 for (FullPageId fullId : pageIds) {
                     long cpStart = System.nanoTime();
 
-                    Integer tag = mem.getForCheckpoint(fullId, tmpBuf, null);
+                    Integer tag;
+
+                    while (true) {
+                        tag = mem.getForCheckpoint(fullId, tmpBuf, null);
+
+                        if (tag != null && tag == PageMemoryImpl.TRY_AGAIN_TAG)
+                            continue;
+
+                        break;
+                    }
 
                     if (tag == null)
                         continue;

From 4e3d700056eb83403a25ccff4eed19d540e5e54d Mon Sep 17 00:00:00 2001
From: DmitriyGovorukhin <dmitriy.govorukhin@gmail.com>
Date: Wed, 18 Jul 2018 16:16:58 +0300
Subject: [PATCH 262/543] IGNITE-8929 Do not disable WAL if node does not have
 MOVING partitions. Fixes #4372

(cherry picked from commit 2b22933)
---
 .../processors/cache/CacheGroupContext.java   |  18 +-
 .../processors/cache/WalStateManager.java     |  35 ++--
 ...ngeDuringRebalanceOnNonNodeAssignTest.java | 169 ++++++++++++++++++
 ...alModeChangeDuringRebalancingSelfTest.java |  72 ++++++--
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 5 files changed, 261 insertions(+), 36 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 1ebe253fce3c4..d545bcc97d29c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -1056,18 +1056,28 @@ public boolean globalWalEnabled() {
      * @param enabled Global WAL enabled flag.
      */
     public void globalWalEnabled(boolean enabled) {
-        persistGlobalWalState(enabled);
+        if (globalWalEnabled != enabled) {
+            log.info("Global WAL state for group=" + cacheOrGroupName() +
+                " changed from " + globalWalEnabled + " to " + enabled);
 
-        this.globalWalEnabled = enabled;
+            persistGlobalWalState(enabled);
+
+            globalWalEnabled = enabled;
+        }
     }
 
     /**
      * @param enabled Local WAL enabled flag.
      */
     public void localWalEnabled(boolean enabled) {
-        persistLocalWalState(enabled);
+        if (localWalEnabled != enabled){
+            log.info("Local WAL state for group=" + cacheOrGroupName() +
+                " changed from " + localWalEnabled + " to " + enabled);
 
-        this.localWalEnabled = enabled;
+            persistLocalWalState(enabled);
+
+            localWalEnabled = enabled;
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
index b35b41e97bd46..4d12eb33bf06f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -42,7 +42,6 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointFuture;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
@@ -66,6 +65,7 @@
 
 import static org.apache.ignite.internal.GridTopic.TOPIC_WAL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
 
 /**
@@ -379,6 +379,7 @@ public void changeLocalStatesOnExchangeDone(AffinityTopologyVersion topVer) {
                 continue;
 
             boolean hasOwning = false;
+            boolean hasMoving = false;
 
             int parts = 0;
 
@@ -394,18 +395,23 @@ public void changeLocalStatesOnExchangeDone(AffinityTopologyVersion topVer) {
 
                         break;
                     }
+
+                    parts++;
                 }
 
-                parts++;
+                if (locPart.state() == MOVING)
+                    hasMoving = true;
             }
 
-            log.info("Prepare change WAL state, grp=" + grp.cacheOrGroupName() +
-                ", grpId=" + grp.groupId() + ", hasOwning=" + hasOwning +
-                ", WALState=" + grp.walEnabled() + ", parts=" + parts);
+            if (log.isDebugEnabled())
+                log.debug("Prepare change WAL state, grp=" + grp.cacheOrGroupName() +
+                    ", grpId=" + grp.groupId() + ", hasOwning=" + hasOwning + ", hasMoving=" + hasMoving +
+                    ", WALState=" + grp.walEnabled() + ", parts=" + parts);
 
-            if (hasOwning && !grp.localWalEnabled())
+            if (hasOwning && !grp.localWalEnabled()) {
                 grpsToEnableWal.add(grp.groupId());
-            else if (!hasOwning && grp.localWalEnabled()) {
+            }
+            else if (hasMoving && !hasOwning && grp.localWalEnabled()) {
                 grpsToDisableWal.add(grp.groupId());
 
                 grpsWithWalDisabled.add(grp.groupId());
@@ -421,7 +427,7 @@ else if (!grp.localWalEnabled())
 
         try {
             if (hasNonEmptyOwning && !grpsToEnableWal.isEmpty())
-                triggerCheckpoint(0).finishFuture().get();
+                triggerCheckpoint("wal-local-state-change-" + topVer).finishFuture().get();
         }
         catch (IgniteCheckedException ex) {
             throw new IgniteException(ex);
@@ -465,7 +471,7 @@ public void onGroupRebalanceFinished(int grpId, AffinityTopologyVersion topVer)
                 tmpDisabledWal = null;
             }
 
-            CheckpointFuture cpFut = triggerCheckpoint(0);
+            CheckpointFuture cpFut = triggerCheckpoint("wal-local-state-changed-rebalance-finished-" + topVer);
 
             assert cpFut != null;
 
@@ -618,7 +624,7 @@ public void onProposeExchange(WalStateProposeMessage msg) {
                         res = new WalStateResult(msg, false);
                     else {
                         // Initiate a checkpoint.
-                        CheckpointFuture cpFut = triggerCheckpoint(msg.groupId());
+                        CheckpointFuture cpFut = triggerCheckpoint("wal-state-change-grp-" + msg.groupId());
 
                         if (cpFut != null) {
                             try {
@@ -1009,11 +1015,11 @@ private void addResult(WalStateResult res) {
     /**
      * Force checkpoint.
      *
-     * @param grpId Group ID.
+     * @param msg Message.
      * @return Checkpoint future or {@code null} if failed to get checkpointer.
      */
-    @Nullable private CheckpointFuture triggerCheckpoint(int grpId) {
-        return cctx.database().forceCheckpoint("wal-state-change-grp-" + grpId);
+    @Nullable private CheckpointFuture triggerCheckpoint(String msg) {
+        return cctx.database().forceCheckpoint(msg);
     }
 
     /**
@@ -1126,7 +1132,8 @@ private static class TemporaryDisabledWal {
         /** */
         public TemporaryDisabledWal(
             Set<Integer> disabledGrps,
-            AffinityTopologyVersion topVer) {
+            AffinityTopologyVersion topVer
+        ) {
             this.disabledGrps = Collections.unmodifiableSet(disabledGrps);
             this.remainingGrps = new HashSet<>(disabledGrps);
             this.topVer = topVer;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest.java
new file mode 100644
index 0000000000000..675ed310d8063
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static java.lang.String.valueOf;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISABLE_WAL_DURING_REBALANCING;
+import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD;
+import static org.apache.ignite.internal.processors.cache.GridCacheUtils.cacheId;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ *
+ */
+public class LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest extends GridCommonAbstractTest {
+
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private final int NODES = 3;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(name);
+
+        cfg.setConsistentId(name);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setWalPath(walPath(name))
+                .setWalArchivePath(walArchivePath(name))
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        cfg.setCacheConfiguration(
+            new CacheConfiguration(DEFAULT_CACHE_NAME)
+                .setAffinity(
+                    new RendezvousAffinityFunction(false, 3))
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+
+        System.setProperty(IGNITE_DISABLE_WAL_DURING_REBALANCING, "true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        System.clearProperty(IGNITE_DISABLE_WAL_DURING_REBALANCING);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void test() throws Exception {
+        Ignite ig = startGrids(NODES);
+
+        ig.cluster().active(true);
+
+        int entries = 100_000;
+
+        try (IgniteDataStreamer<Integer, Integer> st = ig.dataStreamer(DEFAULT_CACHE_NAME)) {
+            st.allowOverwrite(true);
+
+            for (int i = 0; i < entries; i++)
+                st.addData(i, -i);
+        }
+
+        IgniteEx ig4 = startGrid(NODES);
+
+        ig4.cluster().setBaselineTopology(ig4.context().discovery().topologyVersion());
+
+        IgniteWalIteratorFactory iteratorFactory = new IgniteWalIteratorFactory(log);
+
+        String name = ig4.name();
+
+        try (WALIterator it = iteratorFactory.iterator(walPath(name), walArchivePath(name))) {
+            while (it.hasNext()) {
+                IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+                WALRecord rec = tup.get2();
+
+                if (rec.type() == METASTORE_DATA_RECORD) {
+                    MetastoreDataRecord metastoreDataRecord = (MetastoreDataRecord)rec;
+
+                    String key = metastoreDataRecord.key();
+
+                    if (key.startsWith("grp-wal-") &&
+                        key.contains(valueOf(cacheId(DEFAULT_CACHE_NAME))) &&
+                        metastoreDataRecord.value() != null)
+                        fail("WAL was disabled but should not.");
+                }
+            }
+        }
+    }
+
+    /**
+     *
+     * @param nodeName Node name.
+     * @return Path to WAL work directory.
+     * @throws IgniteCheckedException If failed.
+     */
+    private String walPath(String nodeName) throws IgniteCheckedException {
+        String workDir = U.defaultWorkDirectory();
+
+        return workDir + "/" + DFLT_STORE_DIR + "/" + nodeName + "/wal";
+    }
+
+    /**
+     *
+     * @param nodeName Node name.
+     * @return Path to WAL archive directory.
+     * @throws IgniteCheckedException If failed.
+     */
+    private String walArchivePath(String nodeName) throws IgniteCheckedException {
+        String workDir = U.defaultWorkDirectory();
+
+        return workDir + "/" + DFLT_STORE_DIR + "/" + nodeName + "/walArchive";
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
index d6c244367cc10..9ec27ce82a886 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -26,6 +26,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheMode;
@@ -50,10 +51,11 @@
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
-import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.junit.Assert;
 
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
 /**
  *
  */
@@ -184,6 +186,13 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr
         disableWalDuringRebalancing = true;
     }
 
+    /**
+     * @return Count of entries to be processed within test.
+     */
+    protected int getKeysCount() {
+        return 10_000;
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -210,17 +219,19 @@ private void doTestSimple() throws Exception {
 
         IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
 
-        for (int k = 0; k < 10_000; k++)
+        int keysCnt = getKeysCount();
+
+        for (int k = 0; k < keysCnt; k++)
             cache.put(k, k);
 
         IgniteEx newIgnite = startGrid(3);
 
-        final CheckpointHistory cpHistory =
+        final CheckpointHistory cpHist =
             ((GridCacheDatabaseSharedManager)newIgnite.context().cache().context().database()).checkpointHistory();
 
-        GridTestUtils.waitForCondition(new GridAbsPredicate() {
+        waitForCondition(new GridAbsPredicate() {
             @Override public boolean apply() {
-                return !cpHistory.checkpoints().isEmpty();
+                return !cpHist.checkpoints().isEmpty();
             }
         }, 10_000);
 
@@ -228,7 +239,9 @@ private void doTestSimple() throws Exception {
 
         long newIgniteStartedTimestamp = System.currentTimeMillis();
 
-        ignite.cluster().setBaselineTopology(4);
+        newIgnite.cluster().setBaselineTopology(4);
+
+        awaitExchange(newIgnite);
 
         CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
 
@@ -249,14 +262,14 @@ private void doTestSimple() throws Exception {
 
         long rebalanceFinishedTimestamp = System.currentTimeMillis();
 
-        for (Integer k = 0; k < 1000; k++)
+        for (Integer k = 0; k < keysCnt; k++)
             assertEquals("k=" + k, k, cache.get(k));
 
         int checkpointsBeforeNodeStarted = 0;
         int checkpointsBeforeRebalance = 0;
         int checkpointsAfterRebalance = 0;
 
-        for (Long timestamp : cpHistory.checkpoints()) {
+        for (Long timestamp : cpHist.checkpoints()) {
             if (timestamp < newIgniteStartedTimestamp)
                 checkpointsBeforeNodeStarted++;
             else if (timestamp >= newIgniteStartedTimestamp && timestamp < rebalanceStartedTimestamp)
@@ -280,12 +293,14 @@ public void testLocalAndGlobalWalStateInterdependence() throws Exception {
 
         IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
 
-        for (int k = 0; k < 10_000; k++)
+        for (int k = 0; k < getKeysCount(); k++)
             cache.put(k, k);
 
         IgniteEx newIgnite = startGrid(3);
 
-        ignite.cluster().setBaselineTopology(ignite.cluster().nodes());
+        newIgnite.cluster().setBaselineTopology(ignite.cluster().nodes());
+
+        awaitExchange(newIgnite);
 
         CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
 
@@ -312,7 +327,7 @@ public void testLocalAndGlobalWalStateInterdependence() throws Exception {
      */
     public void testWithExchangesMerge() throws Exception {
         final int nodeCnt = 5;
-        final int keyCnt = 10_000;
+        final int keyCnt = getKeysCount();
 
         Ignite ignite = startGrids(nodeCnt);
 
@@ -357,7 +372,7 @@ public void testWithExchangesMerge() throws Exception {
             IgniteCache<Integer, Integer> cache0 = grid(nodeIdx).cache(REPL_CACHE);
 
             for (int k = 0; k < keyCnt; k++)
-                Assert.assertEquals("nodeIdx=" + nodeIdx + ", key=" + k, (Integer) (2 * k), cache0.get(k));
+                Assert.assertEquals("nodeIdx=" + nodeIdx + ", key=" + k, (Integer)(2 * k), cache0.get(k));
         }
     }
 
@@ -385,7 +400,7 @@ private void doTestParallelExchange(AtomicReference<CountDownLatch> latchRef) th
 
         IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
 
-        for (int k = 0; k < 10_000; k++)
+        for (int k = 0; k < getKeysCount(); k++)
             cache.put(k, k);
 
         IgniteEx newIgnite = startGrid(3);
@@ -398,6 +413,9 @@ private void doTestParallelExchange(AtomicReference<CountDownLatch> latchRef) th
 
         ignite.cluster().setBaselineTopology(ignite.cluster().nodes());
 
+        // Await fully exchange complete.
+        awaitExchange(newIgnite);
+
         for (Ignite g : G.allGrids())
             g.cache(DEFAULT_CACHE_NAME).rebalance();
 
@@ -417,7 +435,7 @@ private void doTestParallelExchange(AtomicReference<CountDownLatch> latchRef) th
 
         awaitPartitionMapExchange();
 
-        assertTrue(grpCtx.walEnabled());
+        assertTrue(waitForCondition(grpCtx::walEnabled, 2_000));
     }
 
     /**
@@ -430,12 +448,17 @@ public void testDataClearedAfterRestartWithDisabledWal() throws Exception {
 
         IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
 
-        for (int k = 0; k < 10_000; k++)
+        int keysCnt = getKeysCount();
+
+        for (int k = 0; k < keysCnt; k++)
             cache.put(k, k);
 
         IgniteEx newIgnite = startGrid(1);
 
-        ignite.cluster().setBaselineTopology(2);
+        newIgnite.cluster().setBaselineTopology(2);
+
+        // Await fully exchange complete.
+        awaitExchange(newIgnite);
 
         CacheGroupContext grpCtx = newIgnite.cachex(DEFAULT_CACHE_NAME).context().group();
 
@@ -452,7 +475,7 @@ public void testDataClearedAfterRestartWithDisabledWal() throws Exception {
 
         cache = newIgnite.cache(DEFAULT_CACHE_NAME);
 
-        for (int k = 0; k < 10_000; k++)
+        for (int k = 0; k < keysCnt; k++)
             assertFalse("k=" + k +", v=" + cache.get(k), cache.containsKey(k));
     }
 
@@ -466,7 +489,9 @@ public void testWalNotDisabledAfterShrinkingBaselineTopology() throws Exception
 
         IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
 
-        for (int k = 0; k < 10_000; k++)
+        int keysCnt = getKeysCount();
+
+        for (int k = 0; k < keysCnt; k++)
             cache.put(k, k);
 
         for (Ignite g : G.allGrids()) {
@@ -479,6 +504,9 @@ public void testWalNotDisabledAfterShrinkingBaselineTopology() throws Exception
 
         ignite.cluster().setBaselineTopology(5);
 
+        // Await fully exchange complete.
+        awaitExchange((IgniteEx)ignite);
+
         for (Ignite g : G.allGrids()) {
             CacheGroupContext grpCtx = ((IgniteEx)g).cachex(DEFAULT_CACHE_NAME).context().group();
 
@@ -496,6 +524,14 @@ public void testWalNotDisabledAfterShrinkingBaselineTopology() throws Exception
         }
     }
 
+    /**
+     *
+     * @param ig Ignite.
+     */
+    private void awaitExchange(IgniteEx ig) throws IgniteCheckedException {
+        ig.context().cache().context().exchange().lastTopologyFuture().get();
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 4643abbc50bb9..a777797d6f07b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteRebalanceScheduleResendPartitionsTest;
+import org.apache.ignite.internal.processors.cache.persistence.LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest;
 import org.apache.ignite.internal.processors.cache.persistence.LocalWalModeChangeDuringRebalancingSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAbsentEvictionNodeOutOfBaselineTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.ClientAffinityAssignmentWithBaselineTest;
@@ -114,6 +115,8 @@ private static void addRealPageStoreTestsLongRunning(TestSuite suite) {
 
         suite.addTestSuite(LocalWalModeChangeDuringRebalancingSelfTest.class);
 
+        suite.addTestSuite(LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest.class);
+
         suite.addTestSuite(IgniteWalFlushFsyncSelfTest.class);
 
         suite.addTestSuite(IgniteWalFlushFsyncWithDedicatedWorkerSelfTest.class);

From 4ac50fdc42b3df41bcf2054a0e8f7264929fd2e1 Mon Sep 17 00:00:00 2001
From: Eduard Shangareev <eshangareev@gridgain.com>
Date: Fri, 20 Jul 2018 17:22:52 +0300
Subject: [PATCH 263/543] IGNITE-9039 Fixed non-releasing pinned page on
 throttling fallback - Fixes #4390.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 0aaaab0)
---
 .../persistence/pagemem/PageMemoryImpl.java   |  8 +++-
 .../db/CheckpointBufferDeadlockTest.java      | 41 ++++++++++++----
 .../testframework/GridStringLogger.java       | 18 ++++++-
 .../testsuites/IgniteCacheTestSuite7.java     | 48 +++++++++++++++++++
 4 files changed, 104 insertions(+), 11 deletions(-)
 create mode 100755 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index f40365d7a6543..109c26fca1861 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -1146,8 +1146,14 @@ private boolean copyPageForCheckpoint(
 
         boolean locked = rwLock.tryWriteLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
 
-        if (!locked)
+        if (!locked) {
+            // We release the page only once here because this page will be copied sometime later and
+            // will be released properly then.
+            if (!pageSingleAcquire)
+                PageHeader.releasePage(absPtr);
+
             return false;
+        }
 
         try {
             long tmpRelPtr = PageHeader.tempBufferPointer(absPtr);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
index 2b5a65dcf914f..3afafe691f462 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -54,8 +55,10 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridStringLogger;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.logger.GridTestLog4jLogger;
 
 import static java.nio.file.StandardOpenOption.CREATE;
 import static java.nio.file.StandardOpenOption.READ;
@@ -92,6 +95,9 @@ public class CheckpointBufferDeadlockTest extends GridCommonAbstractTest {
     /** Checkpoint threads. */
     private int checkpointThreads;
 
+    /** String logger. */
+    private GridStringLogger strLog;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -112,6 +118,10 @@ public class CheckpointBufferDeadlockTest extends GridCommonAbstractTest {
 
         cfg.setFailureHandler(new StopNodeFailureHandler());
 
+        strLog = new GridStringLogger(false, new GridTestLog4jLogger());
+
+        cfg.setGridLogger(strLog);
+
         return cfg;
     }
 
@@ -167,9 +177,13 @@ private void runDeadlockScenario() throws Exception {
 
         FilePageStoreManager pageStoreMgr = (FilePageStoreManager)ig.context().cache().context().pageStore();
 
-        IgniteCache<Object, Object> singlePartCache = ig.getOrCreateCache(new CacheConfiguration<>()
-            .setName("single-part")
-            .setAffinity(new RendezvousAffinityFunction(false, 1)));
+        final String cacheName = "single-part";
+
+        CacheConfiguration<Object, Object> cacheCfg = new CacheConfiguration<>()
+                .setName(cacheName)
+                .setAffinity(new RendezvousAffinityFunction(false, 1));
+
+        IgniteCache<Object, Object> singlePartCache = ig.getOrCreateCache(cacheCfg);
 
         db.enableCheckpoints(false).get();
 
@@ -191,7 +205,7 @@ private void runDeadlockScenario() throws Exception {
 
         AtomicBoolean fail = new AtomicBoolean(false);
 
-        GridTestUtils.runMultiThreadedAsync(new Runnable() {
+        IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Runnable() {
             @Override public void run() {
                 int loops = 0;
 
@@ -204,7 +218,7 @@ private void runDeadlockScenario() throws Exception {
                     try {
                         Set<FullPageId> pickedPagesSet = new HashSet<>();
 
-                        PageStore store = pageStoreMgr.getStore(CU.cacheId("single-part"), 0);
+                        PageStore store = pageStoreMgr.getStore(CU.cacheId(cacheName), 0);
 
                         int pages = store.pages();
 
@@ -218,7 +232,7 @@ private void runDeadlockScenario() throws Exception {
 
                             long pageId = PageIdUtils.pageId(0, PageIdAllocator.FLAG_DATA, pageIdx);
 
-                            pickedPagesSet.add(new FullPageId(pageId, CU.cacheId("single-part")));
+                            pickedPagesSet.add(new FullPageId(pageId, CU.cacheId(cacheName)));
                         }
 
                         List<FullPageId> pickedPages = new ArrayList<>(pickedPagesSet);
@@ -233,7 +247,7 @@ private void runDeadlockScenario() throws Exception {
                                     return cmp;
 
                                 return Long.compare(PageIdUtils.effectivePageId(o1.pageId()),
-                                    PageIdUtils.effectivePageId(o2.pageId()));
+                                        PageIdUtils.effectivePageId(o2.pageId()));
                             }
                         });
 
@@ -253,7 +267,7 @@ private void runDeadlockScenario() throws Exception {
                         }
 
                         // Emulate writes to trigger throttling.
-                        for (int i = PAGES_TOUCHED_UNDER_CP_LOCK / 2; i < PAGES_TOUCHED_UNDER_CP_LOCK; i++) {
+                        for (int i = PAGES_TOUCHED_UNDER_CP_LOCK / 2; i < PAGES_TOUCHED_UNDER_CP_LOCK && !stop.get(); i++) {
                             FullPageId fpid = pickedPages.get(i);
 
                             long page = pageMem.acquirePage(fpid.groupId(), fpid.pageId());
@@ -298,6 +312,17 @@ private void runDeadlockScenario() throws Exception {
         assertFalse(fail.get());
 
         forceCheckpoint(); // Previous checkpoint should eventually finish.
+
+        stop.set(true);
+
+        fut.get();
+
+        db.enableCheckpoints(true).get();
+
+        //check that there is no problem with pinned pages
+        ig.destroyCache(cacheName);
+
+        assertFalse(strLog.toString().contains("AssertionError"));
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java
index 16bdaf7acfca1..9056dd65c1349 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridStringLogger.java
@@ -17,8 +17,11 @@
 
 package org.apache.ignite.testframework;
 
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -131,7 +134,18 @@ private synchronized void log(String msg) {
         log(msg);
 
         if (e != null)
-            log(e.toString());
+            logThrowable(e);
+    }
+
+    /**
+     * @param e Exception.
+     */
+    private void logThrowable(@NotNull Throwable e) {
+        StringWriter writer = new StringWriter();
+
+        e.printStackTrace(new PrintWriter(writer));
+
+        log(writer.toString());
     }
 
     /** {@inheritDoc} */
@@ -144,7 +158,7 @@ private synchronized void log(String msg) {
         log(msg);
 
         if (e != null)
-            log(e.toString());
+            logThrowable(e);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
new file mode 100755
index 0000000000000..dbd455acf39a5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import java.util.Set;
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.db.CheckpointBufferDeadlockTest;
+
+/**
+ * Test suite.
+ */
+public class IgniteCacheTestSuite7 extends TestSuite {
+    /**
+     * @return IgniteCache test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        return suite(null);
+    }
+
+    /**
+     * @param ignoredTests Tests to ignore.
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
+        TestSuite suite = new TestSuite("IgniteCache Test Suite part 7");
+
+        suite.addTestSuite(CheckpointBufferDeadlockTest.class);
+
+        return suite;
+    }
+}

From afeaeea37a687c7329ae98a1a1925abf7727a812 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Mon, 23 Jul 2018 11:56:21 +0300
Subject: [PATCH 264/543] IGNITE-8892 Fixed OOME when scan query is used for a
 big partition - Fixes #4391.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit a164296)
---
 .../processors/cache/GridCacheAdapter.java    |  1 -
 .../processors/cache/query/CacheQuery.java    | 14 +----
 .../GridCacheDistributedQueryManager.java     |  1 -
 .../cache/query/GridCacheQueryAdapter.java    | 20 -------
 .../query/GridCacheQueryFutureAdapter.java    |  9 +--
 .../service/GridServiceProcessor.java         |  2 -
 .../cache/CacheIteratorScanQueryTest.java     | 55 ++++++++++++++++++-
 7 files changed, 56 insertions(+), 46 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 567cc6e71083f..51c695a9af60f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3943,7 +3943,6 @@ private Iterator<Cache.Entry<K, V>> igniteIterator(boolean keepBinary,
         final CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         final GridCloseableIterator<Map.Entry<K, V>> iter = ctx0.queries().createScanQuery(p, null, keepBinary)
-            .keepAll(false)
             .executeScanQuery();
 
         return ctx.itHolder().iterator(iter, new CacheIteratorConverter<Cache.Entry<K, V>, Map.Entry<K, V>>() {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQuery.java
index 0cd01fb75e0ff..d8eb7cad94512 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQuery.java
@@ -205,15 +205,6 @@ public interface CacheQuery<T> {
      */
     public CacheQuery<T> timeout(long timeout);
 
-    /**
-     * Sets whether or not to keep all query results local. If not - only the current page
-     * is kept locally. Default value is {@code true}.
-     *
-     * @param keepAll Keep results or not.
-     * @return {@code this} query instance for chaining.
-     */
-    public CacheQuery<T> keepAll(boolean keepAll);
-
     /**
      * Sets whether or not to include backup entries into query result. This flag
      * is {@code false} by default.
@@ -245,10 +236,7 @@ public interface CacheQuery<T> {
      * Executes the query and returns the query future. Caller may decide to iterate
      * over the returned future directly in which case the iterator may block until
      * the next value will become available, or wait for the whole query to finish
-     * by calling any of the {@code 'get(..)'} methods on the returned future. If
-     * {@link #keepAll(boolean)} flag is set to {@code false}, then {@code 'get(..)'}
-     * methods will only return the last page received, otherwise all pages will be
-     * accumulated and returned to user as a collection.
+     * by calling any of the {@code 'get(..)'} methods on the returned future.
      * <p>
      * Note that if the passed in grid projection is a local node, then query
      * will be executed locally without distribution to other nodes.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
index b9c24a36686b4..aac1659c0fd24 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
@@ -267,7 +267,6 @@ protected void removeQueryFuture(long reqId) {
                 log,
                 req.pageSize(),
                 0,
-                false,
                 req.includeBackups(),
                 false,
                 null,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
index b5fdd23fc23b3..51fdd5866ea82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
@@ -109,9 +109,6 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
     /** */
     private volatile long timeout;
 
-    /** */
-    private volatile boolean keepAll = true;
-
     /** */
     private volatile boolean incBackups;
 
@@ -205,7 +202,6 @@ public GridCacheQueryAdapter(GridCacheContext<?, ?> cctx,
      * @param log Logger.
      * @param pageSize Page size.
      * @param timeout Timeout.
-     * @param keepAll Keep all flag.
      * @param incBackups Include backups flag.
      * @param dedup Enable dedup flag.
      * @param prj Grid projection.
@@ -223,7 +219,6 @@ public GridCacheQueryAdapter(GridCacheContext<?, ?> cctx,
         IgniteLogger log,
         int pageSize,
         long timeout,
-        boolean keepAll,
         boolean incBackups,
         boolean dedup,
         ClusterGroup prj,
@@ -240,7 +235,6 @@ public GridCacheQueryAdapter(GridCacheContext<?, ?> cctx,
         this.log = log;
         this.pageSize = pageSize;
         this.timeout = timeout;
-        this.keepAll = keepAll;
         this.incBackups = incBackups;
         this.dedup = dedup;
         this.prj = prj;
@@ -351,20 +345,6 @@ public long timeout() {
         return timeout;
     }
 
-    /** {@inheritDoc} */
-    @Override public CacheQuery<T> keepAll(boolean keepAll) {
-        this.keepAll = keepAll;
-
-        return this;
-    }
-
-    /**
-     * @return Keep all flag.
-     */
-    public boolean keepAll() {
-        return keepAll;
-    }
-
     /** {@inheritDoc} */
     @Override public CacheQuery<T> includeBackups(boolean incBackups) {
         this.incBackups = incBackups;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java
index c418ca2c50588..9a5dd26052fa6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java
@@ -75,9 +75,6 @@ public abstract class GridCacheQueryFutureAdapter<K, V, R> extends GridFutureAda
     /** */
     private final Queue<Collection<R>> queue = new LinkedList<>();
 
-    /** */
-    private final Collection<Object> allCol = new LinkedList<>();
-
     /** */
     private final AtomicInteger cnt = new AtomicInteger();
 
@@ -403,11 +400,8 @@ public void onPage(@Nullable UUID nodeId, @Nullable Collection<?> data, @Nullabl
                 synchronized (this) {
                     enqueue(data);
 
-                    if (qry.query().keepAll())
-                        allCol.addAll(maskNulls((Collection<Object>)data));
-
                     if (onPage(nodeId, finished)) {
-                        onDone((Collection<R>)(qry.query().keepAll() ? unmaskNulls(allCol) : data));
+                        onDone(/* data */);
 
                         clear();
                     }
@@ -580,7 +574,6 @@ void clear() {
     public void printMemoryStats() {
         X.println(">>> Query future memory statistics.");
         X.println(">>>  queueSize: " + queue.size());
-        X.println(">>>  allCollSize: " + allCol.size());
         X.println(">>>  keysSize: " + keys.size());
         X.println(">>>  cnt: " + cnt);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index ff701f4b56d45..cb87045bd79b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1517,8 +1517,6 @@ private Iterator<Cache.Entry<Object, Object>> serviceEntries(IgniteBiPredicate<O
 
             CacheQuery<Map.Entry<Object, Object>> qry = qryMgr.createScanQuery(p, null, false);
 
-            qry.keepAll(false);
-
             DiscoveryDataClusterState clusterState = ctx.state().clusterState();
 
             if ((clusterState.baselineTopology() != null
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheIteratorScanQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheIteratorScanQueryTest.java
index 015147100968c..05a85cab0ca0d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheIteratorScanQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheIteratorScanQueryTest.java
@@ -19,12 +19,19 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import javax.cache.Cache;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
 
@@ -43,6 +50,11 @@ public CacheIteratorScanQueryTest() {
         super(false);
     }
 
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
@@ -98,6 +110,47 @@ public void testScanQuery() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testQueryGetAllClientSide() throws Exception {
+        Ignite server = startGrid(0);
+
+        IgniteCache<Integer, Integer> cache = server.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        client = true;
+
+        Ignite client = startGrid(1);
+
+        IgniteCache<Integer, Integer> cliCache = client.cache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < 100_000; i++)
+            cache.put(i, i);
+
+        ScanQuery<Integer, Integer> qry = new ScanQuery<>();
+
+        qry.setPageSize(100);
+
+        try (QueryCursor<Cache.Entry<Integer, Integer>> cur = cliCache.query(qry)) {
+            List<Cache.Entry<Integer, Integer>> res = cur.getAll();
+
+            assertEquals(100_000, res.size());
+
+            Collections.sort(res, (e1, e2) -> {
+                    return e1.getKey().compareTo(e2.getKey());
+            });
+
+            int exp = 0;
+
+            for (Cache.Entry<Integer, Integer> e : res) {
+                assertEquals(exp, e.getKey().intValue());
+                assertEquals(exp, e.getValue().intValue());
+
+                exp++;
+            }
+        }
+    }
+
     /**
      * Return always false.
      */
@@ -107,4 +160,4 @@ public static class AlwaysFalseCacheFilter implements IgnitePredicate<ClusterNod
             return false;
         }
     }
-}
\ No newline at end of file
+}

From 47d299c4961143f27d1f8aaa0d8c8e26b8ade7ed Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Mon, 23 Jul 2018 11:56:21 +0300
Subject: [PATCH 265/543] IGNITE-8892 Fixed OOME when scan query is used for a
 big partition - Fixes #4391.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit a164296)
---
 .../processors/cache/GridCacheAdapter.java    |  1 -
 .../processors/cache/query/CacheQuery.java    | 14 +----
 .../GridCacheDistributedQueryManager.java     |  1 -
 .../cache/query/GridCacheQueryAdapter.java    | 20 -------
 .../query/GridCacheQueryFutureAdapter.java    |  9 +--
 .../service/GridServiceProcessor.java         |  2 -
 .../cache/CacheIteratorScanQueryTest.java     | 55 ++++++++++++++++++-
 7 files changed, 56 insertions(+), 46 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 567cc6e71083f..51c695a9af60f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -3943,7 +3943,6 @@ private Iterator<Cache.Entry<K, V>> igniteIterator(boolean keepBinary,
         final CacheOperationContext opCtx = ctx.operationContextPerCall();
 
         final GridCloseableIterator<Map.Entry<K, V>> iter = ctx0.queries().createScanQuery(p, null, keepBinary)
-            .keepAll(false)
             .executeScanQuery();
 
         return ctx.itHolder().iterator(iter, new CacheIteratorConverter<Cache.Entry<K, V>, Map.Entry<K, V>>() {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQuery.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQuery.java
index 0cd01fb75e0ff..d8eb7cad94512 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQuery.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/CacheQuery.java
@@ -205,15 +205,6 @@ public interface CacheQuery<T> {
      */
     public CacheQuery<T> timeout(long timeout);
 
-    /**
-     * Sets whether or not to keep all query results local. If not - only the current page
-     * is kept locally. Default value is {@code true}.
-     *
-     * @param keepAll Keep results or not.
-     * @return {@code this} query instance for chaining.
-     */
-    public CacheQuery<T> keepAll(boolean keepAll);
-
     /**
      * Sets whether or not to include backup entries into query result. This flag
      * is {@code false} by default.
@@ -245,10 +236,7 @@ public interface CacheQuery<T> {
      * Executes the query and returns the query future. Caller may decide to iterate
      * over the returned future directly in which case the iterator may block until
      * the next value will become available, or wait for the whole query to finish
-     * by calling any of the {@code 'get(..)'} methods on the returned future. If
-     * {@link #keepAll(boolean)} flag is set to {@code false}, then {@code 'get(..)'}
-     * methods will only return the last page received, otherwise all pages will be
-     * accumulated and returned to user as a collection.
+     * by calling any of the {@code 'get(..)'} methods on the returned future.
      * <p>
      * Note that if the passed in grid projection is a local node, then query
      * will be executed locally without distribution to other nodes.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
index b9c24a36686b4..aac1659c0fd24 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
@@ -267,7 +267,6 @@ protected void removeQueryFuture(long reqId) {
                 log,
                 req.pageSize(),
                 0,
-                false,
                 req.includeBackups(),
                 false,
                 null,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
index b5fdd23fc23b3..51fdd5866ea82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
@@ -109,9 +109,6 @@ public class GridCacheQueryAdapter<T> implements CacheQuery<T> {
     /** */
     private volatile long timeout;
 
-    /** */
-    private volatile boolean keepAll = true;
-
     /** */
     private volatile boolean incBackups;
 
@@ -205,7 +202,6 @@ public GridCacheQueryAdapter(GridCacheContext<?, ?> cctx,
      * @param log Logger.
      * @param pageSize Page size.
      * @param timeout Timeout.
-     * @param keepAll Keep all flag.
      * @param incBackups Include backups flag.
      * @param dedup Enable dedup flag.
      * @param prj Grid projection.
@@ -223,7 +219,6 @@ public GridCacheQueryAdapter(GridCacheContext<?, ?> cctx,
         IgniteLogger log,
         int pageSize,
         long timeout,
-        boolean keepAll,
         boolean incBackups,
         boolean dedup,
         ClusterGroup prj,
@@ -240,7 +235,6 @@ public GridCacheQueryAdapter(GridCacheContext<?, ?> cctx,
         this.log = log;
         this.pageSize = pageSize;
         this.timeout = timeout;
-        this.keepAll = keepAll;
         this.incBackups = incBackups;
         this.dedup = dedup;
         this.prj = prj;
@@ -351,20 +345,6 @@ public long timeout() {
         return timeout;
     }
 
-    /** {@inheritDoc} */
-    @Override public CacheQuery<T> keepAll(boolean keepAll) {
-        this.keepAll = keepAll;
-
-        return this;
-    }
-
-    /**
-     * @return Keep all flag.
-     */
-    public boolean keepAll() {
-        return keepAll;
-    }
-
     /** {@inheritDoc} */
     @Override public CacheQuery<T> includeBackups(boolean incBackups) {
         this.incBackups = incBackups;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java
index c418ca2c50588..9a5dd26052fa6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryFutureAdapter.java
@@ -75,9 +75,6 @@ public abstract class GridCacheQueryFutureAdapter<K, V, R> extends GridFutureAda
     /** */
     private final Queue<Collection<R>> queue = new LinkedList<>();
 
-    /** */
-    private final Collection<Object> allCol = new LinkedList<>();
-
     /** */
     private final AtomicInteger cnt = new AtomicInteger();
 
@@ -403,11 +400,8 @@ public void onPage(@Nullable UUID nodeId, @Nullable Collection<?> data, @Nullabl
                 synchronized (this) {
                     enqueue(data);
 
-                    if (qry.query().keepAll())
-                        allCol.addAll(maskNulls((Collection<Object>)data));
-
                     if (onPage(nodeId, finished)) {
-                        onDone((Collection<R>)(qry.query().keepAll() ? unmaskNulls(allCol) : data));
+                        onDone(/* data */);
 
                         clear();
                     }
@@ -580,7 +574,6 @@ void clear() {
     public void printMemoryStats() {
         X.println(">>> Query future memory statistics.");
         X.println(">>>  queueSize: " + queue.size());
-        X.println(">>>  allCollSize: " + allCol.size());
         X.println(">>>  keysSize: " + keys.size());
         X.println(">>>  cnt: " + cnt);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index ff701f4b56d45..cb87045bd79b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1517,8 +1517,6 @@ private Iterator<Cache.Entry<Object, Object>> serviceEntries(IgniteBiPredicate<O
 
             CacheQuery<Map.Entry<Object, Object>> qry = qryMgr.createScanQuery(p, null, false);
 
-            qry.keepAll(false);
-
             DiscoveryDataClusterState clusterState = ctx.state().clusterState();
 
             if ((clusterState.baselineTopology() != null
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheIteratorScanQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheIteratorScanQueryTest.java
index 015147100968c..05a85cab0ca0d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheIteratorScanQueryTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheIteratorScanQueryTest.java
@@ -19,12 +19,19 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.ScanQuery;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import javax.cache.Cache;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
 
@@ -43,6 +50,11 @@ public CacheIteratorScanQueryTest() {
         super(false);
     }
 
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
@@ -98,6 +110,47 @@ public void testScanQuery() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testQueryGetAllClientSide() throws Exception {
+        Ignite server = startGrid(0);
+
+        IgniteCache<Integer, Integer> cache = server.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        client = true;
+
+        Ignite client = startGrid(1);
+
+        IgniteCache<Integer, Integer> cliCache = client.cache(DEFAULT_CACHE_NAME);
+
+        for (int i = 0; i < 100_000; i++)
+            cache.put(i, i);
+
+        ScanQuery<Integer, Integer> qry = new ScanQuery<>();
+
+        qry.setPageSize(100);
+
+        try (QueryCursor<Cache.Entry<Integer, Integer>> cur = cliCache.query(qry)) {
+            List<Cache.Entry<Integer, Integer>> res = cur.getAll();
+
+            assertEquals(100_000, res.size());
+
+            Collections.sort(res, (e1, e2) -> {
+                    return e1.getKey().compareTo(e2.getKey());
+            });
+
+            int exp = 0;
+
+            for (Cache.Entry<Integer, Integer> e : res) {
+                assertEquals(exp, e.getKey().intValue());
+                assertEquals(exp, e.getValue().intValue());
+
+                exp++;
+            }
+        }
+    }
+
     /**
      * Return always false.
      */
@@ -107,4 +160,4 @@ public static class AlwaysFalseCacheFilter implements IgnitePredicate<ClusterNod
             return false;
         }
     }
-}
\ No newline at end of file
+}

From e2695cd960b7f0cbbba545ef60645ea964f3d7fa Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Mon, 23 Jul 2018 11:25:49 +0300
Subject: [PATCH 266/543] IGNITE-9042 Fixed partial tranasaction state wheh
 transaction is timed out - Fixes #4397.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 33f485a)
---
 .../dht/GridDhtTxPrepareFuture.java           |   3 -
 ...thSmallTimeoutAndContentionOneKeyTest.java | 247 ++++++++++++++++++
 .../junits/common/GridCommonAbstractTest.java |  61 +++--
 .../testsuites/IgniteCacheTestSuite6.java     |   2 +
 4 files changed, 291 insertions(+), 22 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index 622774dd58f6f..a6e349fc7024a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -1292,9 +1292,6 @@ private void sendPrepareRequests() {
             if (F.isEmpty(dhtWrites) && F.isEmpty(nearWrites))
                 continue;
 
-            if (tx.remainingTime() == -1)
-                return;
-
             MiniFuture fut = new MiniFuture(n.id(), ++miniId, dhtMapping, nearMapping);
 
             add(fut); // Append new future.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java
new file mode 100644
index 0000000000000..c81714497095f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteTransactions;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.testframework.GridTestUtils.runMultiThreadedAsync;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
+
+/**
+ *
+ */
+public class TxWithSmallTimeoutAndContentionOneKeyTest extends GridCommonAbstractTest {
+    /** */
+    public static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int TIME_TO_EXECUTE = 30 * 1000;
+
+    /** */
+    private boolean client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(name);
+
+        cfg.setConsistentId("NODE_" + name.substring(name.length() - 1));
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        cfg.setCacheConfiguration(
+            new CacheConfiguration(DEFAULT_CACHE_NAME)
+                .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+                .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+                .setBackups(3)
+        );
+
+        if (client){
+            cfg.setConsistentId("Client");
+
+            cfg.setClientMode(client);
+        }
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * @return Random transaction type.
+     */
+    protected TransactionConcurrency transactionConcurrency() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+
+        return random.nextBoolean() ? OPTIMISTIC : PESSIMISTIC;
+    }
+
+    /**
+     * @return Random transaction isolation level.
+     */
+    protected TransactionIsolation transactionIsolation(){
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+
+        switch (random.nextInt(3)) {
+            case 0:
+                return READ_COMMITTED;
+            case 1:
+                return REPEATABLE_READ;
+            case 2:
+                return SERIALIZABLE;
+            default:
+                throw new UnsupportedOperationException();
+        }
+    }
+
+    /**
+     * @return Random timeout.
+     */
+    protected long randomTimeOut() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+
+        return random.nextLong(5, 20);
+    }
+
+    /**
+     * https://issues.apache.org/jira/browse/IGNITE-9042
+     *
+     * @throws Exception If failed.
+     */
+    public void test() throws Exception {
+        startGrids(4);
+
+        client = true;
+
+        IgniteEx igClient = startGrid(4);
+
+        igClient.cluster().active(true);
+
+        AtomicBoolean stop = new AtomicBoolean(false);
+
+        IgniteCache<Integer, Long> cache = igClient.cache(DEFAULT_CACHE_NAME);
+
+        int threads = 1;
+
+        int keyId = 0;
+
+        CountDownLatch finishLatch = new CountDownLatch(threads);
+
+        AtomicLong cnt = new AtomicLong();
+
+        IgniteInternalFuture<Long> f = runMultiThreadedAsync(() -> {
+            IgniteTransactions txMgr = igClient.transactions();
+
+            while (!stop.get()) {
+                long newVal = cnt.getAndIncrement();
+
+                TransactionConcurrency concurrency = transactionConcurrency();
+
+                TransactionIsolation transactionIsolation = transactionIsolation();
+
+                try (Transaction tx = txMgr.txStart(concurrency, transactionIsolation, randomTimeOut(), 1)) {
+                    cache.put(keyId, newVal);
+
+                    tx.commit();
+                }
+                catch (Throwable e) {
+                  // Ignore.
+                }
+            }
+
+            finishLatch.countDown();
+
+        }, threads, "tx-runner");
+
+        runAsync(() -> {
+            try {
+                Thread.sleep(TIME_TO_EXECUTE);
+            }
+            catch (InterruptedException ignore) {
+                // Ignore.
+            }
+
+            stop.set(true);
+        });
+
+        finishLatch.await();
+
+        f.get();
+
+        Map<PartitionKey, List<PartitionHashRecord>> idleVerifyResult = idleVerify(igClient, DEFAULT_CACHE_NAME);
+
+        log.info("Current counter value:" + cnt.get());
+
+        Long val = cache.get(keyId);
+
+        log.info("Last commited value:" + val);
+
+        if (!F.isEmpty(idleVerifyResult)) {
+            SB sb = new SB();
+
+            sb.a("\n");
+
+            buildConflicts("Conflicts:\n", sb, idleVerifyResult);
+
+            System.out.println(sb);
+
+            fail();
+        }
+    }
+
+    /**
+     * @param msg Header message.
+     * @param conflicts Conflicts map.
+     * @param sb String builder.
+     */
+    private void buildConflicts(String msg, SB sb, Map<PartitionKey, List<PartitionHashRecord>> conflicts) {
+        sb.a(msg);
+
+        for (Map.Entry<PartitionKey, List<PartitionHashRecord>> entry : conflicts.entrySet()) {
+            sb.a(entry.getKey()).a("\n");
+
+            for (PartitionHashRecord rec : entry.getValue())
+                sb.a("\t").a(rec).a("\n");
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 71abb95c47ac3..38df0f488196d 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -96,6 +96,10 @@
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.testframework.GridTestNode;
@@ -786,25 +790,6 @@ protected void awaitPartitionMapExchange(
         log.info("awaitPartitionMapExchange finished");
     }
 
-    /**
-     * Compares checksums between primary and backup partitions of specified caches.
-     * Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
-     * concurrently updated.
-     *
-     * @param ig Ignite instance.
-     * @param cacheNames Cache names (if null, all user caches will be verified).
-     * @throws IgniteCheckedException If checksum conflict has been found.
-     */
-    protected void verifyBackupPartitions(Ignite ig, Set<String> cacheNames) throws IgniteCheckedException {
-        Map<PartitionKey, List<PartitionHashRecord>> conflicts = ig.compute().execute(
-            new VerifyBackupPartitionsTask(), cacheNames);
-
-        if (!conflicts.isEmpty()) {
-            throw new IgniteCheckedException("Partition checksums are different for backups " +
-                "of the following partitions: " + conflicts.keySet());
-        }
-    }
-
     /**
      * @param top Topology.
      * @param topVer Version to wait for.
@@ -1890,4 +1875,42 @@ protected void forceCheckpoint(Collection<Ignite> nodes) throws IgniteCheckedExc
             dbMgr.waitForCheckpoint("test");
         }
     }
+
+    /**
+     * Compares checksums between primary and backup partitions of specified caches.
+     * Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
+     * concurrently updated.
+     *
+     * @param ig Ignite instance.
+     * @param caches Cache names (if null, all user caches will be verified).
+     * @return Conflicts result.
+     * @throws IgniteException If none caches or node found.
+     */
+    protected Map<PartitionKey, List<PartitionHashRecord>> idleVerify(Ignite ig, String... caches) {
+        IgniteEx ig0 = (IgniteEx)ig;
+
+        Set<String> cacheNames = new HashSet<>();
+
+        if (F.isEmpty(caches))
+            cacheNames.addAll(ig0.cacheNames());
+        else
+            Collections.addAll(cacheNames, caches);
+
+        if (cacheNames.isEmpty())
+            throw new IgniteException("None cache for checking.");
+
+        ClusterNode node = !ig0.localNode().isClient() ? ig0.localNode() : ig0.cluster().forServers().forRandom().node();
+
+        if (node == null)
+            throw new IgniteException("None server node for verification.");
+
+        VisorIdleVerifyTaskArg taskArg = new VisorIdleVerifyTaskArg(cacheNames);
+
+        VisorIdleVerifyTaskResult res = ig.compute().execute(
+            VisorIdleVerifyTask.class.getName(),
+            new VisorTaskArgument<>(node.id(), taskArg, false)
+        );
+
+        return res.getConflicts();
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 66c1c488cf447..77285bed1963e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -54,6 +54,7 @@
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTopologyChangeTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxWithSmallTimeoutAndContentionOneKeyTest;
 
 /**
  * Test suite.
@@ -78,6 +79,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TxRollbackOnTimeoutTest.class);
         suite.addTestSuite(TxRollbackOnTimeoutNoDeadlockDetectionTest.class);
         suite.addTestSuite(TxRollbackOnTimeoutNearCacheTest.class);
+        suite.addTestSuite(TxWithSmallTimeoutAndContentionOneKeyTest.class);
         suite.addTestSuite(IgniteCacheThreadLocalTxTest.class);
         suite.addTestSuite(TxRollbackAsyncTest.class);
         suite.addTestSuite(TxRollbackAsyncNearCacheTest.class);

From 702c31859251738b187e8db2a8155264f39b8b52 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Mon, 23 Jul 2018 15:29:08 +0300
Subject: [PATCH 267/543] IGNITE-8820 Fix rollback logic when tx is initiated
 from client. - Fixes #4399.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 5794eb0)
---
 .../GridCachePartitionExchangeManager.java    | 27 +++++++++++++++++--
 ...etTxTimeoutOnPartitionMapExchangeTest.java | 22 ++++++++++++++-
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 644c26e5dc664..1f28da2ae09bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -36,6 +36,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
@@ -46,6 +47,7 @@
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
@@ -2492,13 +2494,25 @@ else if (task instanceof ForceRebalanceExchangeTask) {
 
                             int dumpCnt = 0;
 
-                            final long dumpTimeout = 2 * cctx.gridConfig().getNetworkTimeout();
+                            long waitStart = U.currentTimeMillis();
+
+                            // Call rollback logic only for client node, for server nodes
+                            // rollback logic is in GridDhtPartitionsExchangeFuture.
+                            boolean txRolledBack = !cctx.localNode().isClient();
+
+                            IgniteConfiguration cfg = cctx.gridConfig();
+
+                            final long dumpTimeout = 2 * cfg.getNetworkTimeout();
 
                             long nextDumpTime = 0;
 
                             while (true) {
+                                // Read txTimeoutOnPME from configuration after every iteration.
+                                long curTimeout = cfg.getTransactionConfiguration().getTxTimeoutOnPartitionMapExchange();
+
                                 try {
-                                    resVer = exchFut.get(dumpTimeout);
+                                    resVer = exchFut.get(curTimeout > 0 && !txRolledBack ?
+                                            Math.min(curTimeout, dumpTimeout) : dumpTimeout, TimeUnit.MILLISECONDS);
 
                                     break;
                                 }
@@ -2507,6 +2521,9 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                         U.warn(diagnosticLog, "Failed to wait for partition map exchange [" +
                                             "topVer=" + exchFut.initialVersion() +
                                             ", node=" + cctx.localNodeId() + "]. " +
+                                            (curTimeout <= 0 && !txRolledBack ? "Consider changing " +
+                                                    "TransactionConfiguration.txTimeoutOnPartitionMapSynchronization" +
+                                                    " to non default value to avoid this message. " : "") +
                                             "Dumping pending objects that might be the cause: ");
 
                                         try {
@@ -2518,6 +2535,12 @@ else if (task instanceof ForceRebalanceExchangeTask) {
 
                                         nextDumpTime = U.currentTimeMillis() + nextDumpTimeout(dumpCnt++, dumpTimeout);
                                     }
+
+                                    if (!txRolledBack && curTimeout > 0 && U.currentTimeMillis() - waitStart >= curTimeout) {
+                                        txRolledBack = true; // Try automatic rollback only once.
+
+                                        cctx.tm().rollbackOnTopologyChange(exchFut.initialVersion());
+                                    }
                                 }
                                 catch (Exception e) {
                                     if (exchFut.reconnectOnError(e))
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java
index 7033529c6810f..c9ae34fcc18cd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/SetTxTimeoutOnPartitionMapExchangeTest.java
@@ -142,8 +142,28 @@ public void testClusterSetTxTimeoutOnPartitionMapExchange() throws Exception {
      * @throws Exception If fails.
      */
     public void testSetTxTimeoutDuringPartitionMapExchange() throws Exception {
-        IgniteEx ig = (IgniteEx)startGrids(2);
+        IgniteEx ig = (IgniteEx) startGrids(2);
 
+        checkSetTxTimeoutDuringPartitionMapExchange(ig);
+    }
+
+    /**
+     * Tests applying new txTimeoutOnPartitionMapExchange while an exchange future runs on client node.
+     *
+     * @throws Exception If fails.
+     */
+    public void testSetTxTimeoutOnClientDuringPartitionMapExchange() throws Exception {
+        IgniteEx ig = (IgniteEx) startGrids(2);
+        IgniteEx client = startGrid(getConfiguration("client").setClientMode(true));
+
+        checkSetTxTimeoutDuringPartitionMapExchange(client);
+    }
+
+    /**
+     * @param ig Ignite instance where deadlock tx will start.
+     * @throws Exception If fails.
+     */
+    private void checkSetTxTimeoutDuringPartitionMapExchange(IgniteEx ig) throws Exception {
         final long longTimeout = 600_000L;
         final long shortTimeout = 5_000L;
 

From 32777e77797f4cf3260d42d4ca408336247d4e55 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Mon, 23 Jul 2018 18:01:37 +0300
Subject: [PATCH 268/543] IGNITE-9049 Fixed write of SWITCH_SEGMENT_RECORD at
 the end of a segment file - Fixes #4401.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 713a428)
---
 .../cache/persistence/wal/FileWriteAheadLogManager.java         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index fcc565b6b22b3..b952353cbc660 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -2627,7 +2627,7 @@ public void writeHeader() {
 
                         ByteBuffer buf = seg.buffer();
 
-                        if (buf == null || (stop.get() && rec.type() != SWITCH_SEGMENT_RECORD))
+                        if (buf == null)
                             return null; // Can not write to this segment, need to switch to the next one.
 
                         ptr = new FileWALPointer(idx, pos, rec.size());

From bbf8f83afe3b07e807912d9bab78873edf698d4d Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Tue, 24 Jul 2018 17:38:34 +0300
Subject: [PATCH 269/543] IGNITE-8892 Fixed CacheQueryFuture usage in
 DataStructures processor - Fixes #4415.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 281a400)
---
 .../processors/datastructures/GridCacheSetImpl.java   |  6 ++++--
 .../GridCacheFullTextQueryMultithreadedSelfTest.java  | 11 ++++++++---
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
index c27770f8dda85..0e3e102679e9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheSetImpl.java
@@ -168,11 +168,13 @@ public boolean checkHeader() throws IgniteCheckedException {
 
             qry.projection(ctx.grid().cluster().forNodes(nodes));
 
-            Iterable<Integer> col = (Iterable<Integer>)qry.execute(new SumReducer()).get();
+            CacheQueryFuture<Integer> qryFut = qry.execute(new SumReducer());
 
             int sum = 0;
 
-            for (Integer val : col)
+            Integer val;
+
+            while((val = qryFut.next()) != null)
                 sum += val;
 
             return sum;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQueryMultithreadedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQueryMultithreadedSelfTest.java
index e12e24657e2ea..acb3d7375e359 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQueryMultithreadedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheFullTextQueryMultithreadedSelfTest.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.Callable;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -26,6 +25,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.cache.query.CacheQuery;
+import org.apache.ignite.internal.processors.cache.query.CacheQueryFuture;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
@@ -105,12 +105,17 @@ public void testH2Text() throws Exception {
                     int cnt = 0;
 
                     while (!stop.get()) {
-                        Collection<Map.Entry<Integer, H2TextValue>> res = qry.execute().get();
+                        CacheQueryFuture<Map.Entry<Integer, H2TextValue>> qryFut = qry.execute();
+
+                        int size = 0;
+
+                        while(qryFut.next() != null)
+                            size++;
 
                         cnt++;
 
                         if (cnt % logFreq == 0) {
-                            X.println("Result set: " + res.size());
+                            X.println("Result set: " + size);
                             X.println("Executed queries: " + cnt);
                         }
                     }

From 86c52700a007368a351ac3595a8e38bb02923080 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Wed, 25 Jul 2018 16:26:12 +0300
Subject: [PATCH 270/543] IGNITE-9040 StopNodeFailureHandler is not able to
 stop node correctly on node segmentation

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry-picked from commit#469aaba59c0539507972f4725642b2f2f81c08a0)
---
 .../ignite/internal/GridKernalContext.java    |   7 +
 .../internal/GridKernalContextImpl.java       |  14 +-
 .../apache/ignite/internal/IgnitionEx.java    |  13 +-
 .../reader/StandaloneGridKernalContext.java   |   5 +
 .../internal/ZookeeperDiscoverySpiTest.java   | 139 +++++++++++++++++-
 5 files changed, 164 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 505c3d6a4b0b1..051978c0fc728 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -444,6 +444,13 @@ public interface GridKernalContext extends Iterable<GridComponent> {
      */
     public boolean invalid();
 
+    /**
+     * Checks whether this node detected its segmentation from the rest of the grid.
+     *
+     * @return {@code True} if this node has segmented, {@code false} otherwise.
+     */
+    public boolean segmented();
+
     /**
      * Gets failure processor.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 2f4ecbc7ddfab..2be64e5d1330c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.managers.checkpoint.GridCheckpointManager;
 import org.apache.ignite.internal.managers.collision.GridCollisionManager;
 import org.apache.ignite.internal.managers.communication.GridIoManager;
@@ -1125,7 +1126,18 @@ void disconnected(boolean disconnected) {
     @Override public boolean invalid() {
         FailureProcessor failureProc = failure();
 
-        return failureProc != null && failureProc.failureContext() != null;
+        return failureProc != null
+            && failureProc.failureContext() != null
+            && failureProc.failureContext().type() != FailureType.SEGMENTATION;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean segmented() {
+        FailureProcessor failureProc = failure();
+
+        return failureProc != null
+            && failureProc.failureContext() != null
+            && failureProc.failureContext().type() == FailureType.SEGMENTATION;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 4d1724d5afa97..c58c677151ed4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -2604,15 +2604,12 @@ private synchronized void stop0(boolean cancel) {
                     throw e;
             }
             finally {
-                if (!grid0.context().invalid())
+                if (grid0.context().segmented())
+                    state = STOPPED_ON_SEGMENTATION;
+                else if (grid0.context().invalid())
+                    state = STOPPED_ON_FAILURE;
+                else
                     state = STOPPED;
-                else {
-                    FailureContext failure = grid0.context().failure().failureContext();
-
-                    state = failure.type() == FailureType.SEGMENTATION ?
-                        STOPPED_ON_SEGMENTATION :
-                        STOPPED_ON_FAILURE;
-                }
 
                 grid = null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index 795d46004f727..b9ab76a3c7b05 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -471,6 +471,11 @@ protected IgniteConfiguration prepareIgniteConfiguration() {
         return false;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean segmented() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public FailureProcessor failure() {
         return null;
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index e902f42b8135f..ba0fa27d7628b 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -60,6 +60,8 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteState;
+import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
@@ -128,6 +130,7 @@
 import org.apache.ignite.spi.discovery.zk.ZookeeperDiscoverySpiTestSuite2;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZKUtil;
 import org.apache.zookeeper.ZkTestClientCnxnSocketNIO;
@@ -147,6 +150,9 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_CREDENTIALS;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2;
 import static org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl.IGNITE_ZOOKEEPER_DISCOVERY_SPI_ACK_THRESHOLD;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
 import static org.apache.zookeeper.ZooKeeper.ZOOKEEPER_CLIENT_CNXN_SOCKET;
 
 /**
@@ -181,6 +187,12 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
     /** */
     private boolean testSockNio;
 
+    /** */
+    private CacheAtomicityMode atomicityMode;
+
+    /** */
+    private int backups = -1;
+
     /** */
     private boolean testCommSpi;
 
@@ -278,11 +290,7 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
 
         cfg.setDiscoverySpi(zkSpi);
 
-        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
-
-        ccfg.setWriteSynchronizationMode(FULL_SYNC);
-
-        cfg.setCacheConfiguration(ccfg);
+        cfg.setCacheConfiguration(getCacheConfiguration());
 
         Boolean clientMode = clientThreadLoc.get();
 
@@ -362,6 +370,21 @@ public class ZookeeperDiscoverySpiTest extends GridCommonAbstractTest {
         return cfg;
     }
 
+    /** */
+    private CacheConfiguration getCacheConfiguration() {
+        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+        ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+        if (atomicityMode != null)
+            ccfg.setAtomicityMode(atomicityMode);
+
+        if (backups > 0)
+            ccfg.setBackups(backups);
+
+        return ccfg;
+    }
+
     /**
      * @param clientMode Client mode flag for started nodes.
      */
@@ -1072,6 +1095,112 @@ private void registerTestEventListeners(Ignite node,
         );
     }
 
+    /**
+     * Verifies correct handling of SEGMENTATION event with STOP segmentation policy: node is stopped successfully,
+     * all its threads are shut down.
+     *
+     * @throws Exception If failed.
+     *
+     * @see <a href="https://issues.apache.org/jira/browse/IGNITE-9040">IGNITE-9040</a> ticket for more context of the test.
+     */
+    public void testStopNodeOnSegmentaion() throws Exception {
+        try {
+            System.setProperty("IGNITE_WAL_LOG_TX_RECORDS", "true");
+
+            sesTimeout = 2000;
+            testSockNio = true;
+            persistence = true;
+            atomicityMode = CacheAtomicityMode.TRANSACTIONAL;
+            backups = 2;
+
+            final Ignite node0 = startGrid(0);
+
+            sesTimeout = 10_000;
+            testSockNio = false;
+
+            startGrid(1);
+
+            node0.cluster().active(true);
+
+            clientMode(true);
+
+            final IgniteEx client = startGrid(2);
+
+            //first transaction
+            client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 0, 0);
+            client.cache(DEFAULT_CACHE_NAME).put(0, 0);
+
+            //second transaction to create a deadlock with the first one
+            // and guarantee transaction futures will be presented on segmented node
+            // (erroneous write to WAL on segmented node stop happens
+            // on completing transaction with NodeStoppingException)
+            GridTestUtils.runAsync(new Runnable() {
+                @Override public void run() {
+                    Transaction tx2 = client.transactions().txStart(OPTIMISTIC, READ_COMMITTED, 0, 0);
+                    client.cache(DEFAULT_CACHE_NAME).put(0, 0);
+                    tx2.commit();
+                }
+            });
+
+            //next block simulates Ignite node segmentation by closing socket of ZooKeeper client
+            {
+                final CountDownLatch l = new CountDownLatch(1);
+
+                node0.events().localListen(new IgnitePredicate<Event>() {
+                    @Override public boolean apply(Event evt) {
+                        l.countDown();
+
+                        return false;
+                    }
+                }, EventType.EVT_NODE_SEGMENTED);
+
+                ZkTestClientCnxnSocketNIO c0 = ZkTestClientCnxnSocketNIO.forNode(node0);
+
+                c0.closeSocket(true);
+
+                for (int i = 0; i < 10; i++) {
+                    Thread.sleep(1_000);
+
+                    if (l.getCount() == 0)
+                        break;
+                }
+
+                info("Allow connect");
+
+                c0.allowConnect();
+
+                assertTrue(l.await(10, TimeUnit.SECONDS));
+            }
+
+            waitForNodeStop(node0.name());
+
+            checkStoppedNodeThreads(node0.name());
+        }
+        finally {
+            System.clearProperty("IGNITE_WAL_LOG_TX_RECORDS");
+        }
+    }
+
+    /** */
+    private void checkStoppedNodeThreads(String nodeName) {
+        Set<Thread> threads = Thread.getAllStackTraces().keySet();
+
+        for (Thread t : threads) {
+            if (t.getName().contains(nodeName))
+                throw new AssertionError("Thread from stopped node has been found: " + t.getName());
+        }
+    }
+
+    /** */
+    private void waitForNodeStop(String name) throws Exception {
+        while (true) {
+            if (IgnitionEx.state(name).equals(IgniteState.STARTED))
+                Thread.sleep(2000);
+            else
+                break;
+        }
+    }
+
     /**
      * @throws Exception If failed.
      */

From e1c3cc9d30baeaff4e653751775ab39a347b753b Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Tue, 24 Jul 2018 17:48:57 +0300
Subject: [PATCH 271/543] IGNITE-8791 Fixed missed update counter in WAL data
 record for backup transaction - Fixes #4264.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 13e2a31)
---
 .../pagemem/wal/record/DataEntry.java         |  12 ++
 .../pagemem/wal/record/LazyDataEntry.java     |   3 +
 .../GridDistributedTxRemoteAdapter.java       |  44 +++--
 .../serializer/RecordDataV1Serializer.java    |   4 +
 ...sAtomicCacheHistoricalRebalancingTest.java |   5 +
 ...IgnitePdsCacheRebalancingAbstractTest.java | 179 ++++++++++++------
 .../IgnitePdsTxCacheRebalancingTest.java      |   1 -
 .../IgnitePdsTxHistoricalRebalancingTest.java |  64 +++++++
 modules/indexing/pom.xml                      |   7 +
 .../IgnitePdsWithIndexingCoreTestSuite.java   |   9 +-
 10 files changed, 257 insertions(+), 71 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxHistoricalRebalancingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
index 3511affe5d531..d13a68ae518cc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataEntry.java
@@ -157,6 +157,18 @@ public long partitionCounter() {
         return partCnt;
     }
 
+    /**
+     * Sets partition update counter to entry.
+     *
+     * @param partCnt Partition update counter.
+     * @return {@code this} for chaining.
+     */
+    public DataEntry partitionCounter(long partCnt) {
+        this.partCnt = partCnt;
+
+        return this;
+    }
+
     /**
      * @return Expire time.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyDataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyDataEntry.java
index 0ad87d7bf9c8e..6b56da5b7b3e5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyDataEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/LazyDataEntry.java
@@ -96,6 +96,9 @@ public LazyDataEntry(
                 IgniteCacheObjectProcessor co = cctx.kernalContext().cacheObjects();
 
                 key = co.toKeyCacheObject(cacheCtx.cacheObjectContext(), keyType, keyBytes);
+
+                if (key.partition() == -1)
+                    key.partition(partId);
             }
 
             return key;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index 5e3111c1250cf..1b9b3a8ad28fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -27,6 +27,7 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.InvalidEnvironmentException;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -63,6 +64,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringBuilder;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -485,7 +487,8 @@ private void commitIfLocked() throws IgniteCheckedException {
                     try {
                         Collection<IgniteTxEntry> entries = near() || cctx.snapshot().needTxReadLogging() ? allEntries() : writeEntries();
 
-                        List<DataEntry> dataEntries = null;
+                        // Data entry to write to WAL and associated with it TxEntry.
+                        List<T2<DataEntry, IgniteTxEntry>> dataEntries = null;
 
                         batchStoreCommit(writeMap().values());
 
@@ -571,17 +574,20 @@ else if (conflictCtx.isMerge()) {
                                                     dataEntries = new ArrayList<>(entries.size());
 
                                                 dataEntries.add(
-                                                    new DataEntry(
-                                                        cacheCtx.cacheId(),
-                                                        txEntry.key(),
-                                                        val,
-                                                        op,
-                                                        nearXidVersion(),
-                                                        writeVersion(),
-                                                        0,
-                                                        txEntry.key().partition(),
-                                                        txEntry.updateCounter()
-                                                    )
+                                                        new T2<>(
+                                                                new DataEntry(
+                                                                        cacheCtx.cacheId(),
+                                                                        txEntry.key(),
+                                                                        val,
+                                                                        op,
+                                                                        nearXidVersion(),
+                                                                        writeVersion(),
+                                                                        0,
+                                                                        txEntry.key().partition(),
+                                                                        txEntry.updateCounter()
+                                                                ),
+                                                                txEntry
+                                                        )
                                                 );
                                             }
 
@@ -630,6 +636,8 @@ else if (conflictCtx.isMerge()) {
                                                         dhtVer,
                                                         txEntry.updateCounter());
 
+                                                    txEntry.updateCounter(updRes.updatePartitionCounter());
+
                                                     if (updRes.loggedPointer() != null)
                                                         ptr = updRes.loggedPointer();
 
@@ -665,6 +673,8 @@ else if (op == DELETE) {
                                                     dhtVer,
                                                     txEntry.updateCounter());
 
+                                                txEntry.updateCounter(updRes.updatePartitionCounter());
+
                                                 if (updRes.loggedPointer() != null)
                                                     ptr = updRes.loggedPointer();
 
@@ -757,8 +767,14 @@ else if (op == READ) {
                                 }
                             }
 
-                            if (!near() && !F.isEmpty(dataEntries) && cctx.wal() != null)
-                                cctx.wal().log(new DataRecord(dataEntries));
+                            if (!near() && !F.isEmpty(dataEntries) && cctx.wal() != null) {
+                                // Set new update counters for data entries received from persisted tx entries.
+                                List<DataEntry> entriesWithCounters = dataEntries.stream()
+                                        .map(tuple -> tuple.get1().partitionCounter(tuple.get2().updateCounter()))
+                                        .collect(Collectors.toList());
+
+                                cctx.wal().log(new DataRecord(entriesWithCounters));
+                            }
 
                             if (ptr != null && !cctx.tm().logTxRecords())
                                 cctx.wal().flush(ptr, false);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
index e8d116bcbee2e..a477a131c59c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
@@ -1481,6 +1481,10 @@ DataEntry readDataEntry(ByteBufferBackedDataInput in) throws IOException, Ignite
             CacheObjectContext coCtx = cacheCtx.cacheObjectContext();
 
             KeyCacheObject key = co.toKeyCacheObject(coCtx, keyType, keyBytes);
+
+            if (key.partition() == -1)
+                key.partition(partId);
+
             CacheObject val = valBytes != null ? co.toCacheObject(coCtx, valType, valBytes) : null;
 
             return new DataEntry(
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
index f06494b965235..cce9a4084aef5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheHistoricalRebalancingTest.java
@@ -34,6 +34,11 @@ public class IgnitePdsAtomicCacheHistoricalRebalancingTest extends IgnitePdsAtom
         return cfg;
     }
 
+    /** {@inheritDoc */
+    @Override protected long checkpointFrequency() {
+        return 15 * 1000;
+    }
+
     /** {@inheritDoc */
     @Override protected void beforeTest() throws Exception {
         // Use rebalance from WAL if possible.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
index 347412df79965..368c6094cf7db 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -24,16 +24,19 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Random;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import com.google.common.collect.Lists;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.PartitionLossPolicy;
@@ -137,8 +140,7 @@ public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAb
 
         DataStorageConfiguration dsCfg = new DataStorageConfiguration()
             .setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4)
-            .setPageSize(1024)
-            .setCheckpointFrequency(10 * 1000)
+            .setCheckpointFrequency(checkpointFrequency())
             .setWalMode(WALMode.LOG_ONLY)
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
                 .setName("dfltDataRegion")
@@ -167,6 +169,13 @@ private static CacheConfiguration[] asArray(List<CacheConfiguration> cacheCfgs)
         return res;
     }
 
+    /**
+     * @return Checkpoint frequency;
+     */
+    protected long checkpointFrequency() {
+        return DataStorageConfiguration.DFLT_CHECKPOINT_FREQ;
+    }
+
     /** {@inheritDoc} */
     @Override protected long getTestTimeout() {
         return 20 * 60 * 1000;
@@ -315,14 +324,17 @@ public void testRebalancingOnRestartAfterCheckpoint() throws Exception {
      * @throws Exception If failed.
      */
     public void testTopologyChangesWithConstantLoad() throws Exception {
-        final long timeOut = U.currentTimeMillis() + 10 * 60 * 1000;
+        final long timeOut = U.currentTimeMillis() + 5 * 60 * 1000;
 
         final int entriesCnt = 10_000;
         final int maxNodesCnt = 4;
-        final int topChanges = 50;
+        final int topChanges = 25;
+        final boolean allowRemoves = true;
 
+        final AtomicLong orderCounter = new AtomicLong();
         final AtomicBoolean stop = new AtomicBoolean();
         final AtomicBoolean suspend = new AtomicBoolean();
+        final AtomicBoolean suspended = new AtomicBoolean();
 
         final ConcurrentMap<Integer, TestValue> map = new ConcurrentHashMap<>();
 
@@ -333,45 +345,64 @@ public void testTopologyChangesWithConstantLoad() throws Exception {
         IgniteCache<Integer, TestValue> cache = ignite.cache(INDEXED_CACHE);
 
         for (int i = 0; i < entriesCnt; i++) {
-            cache.put(i, new TestValue(i, i));
-            map.put(i, new TestValue(i, i));
+            long order = orderCounter.get();
+
+            cache.put(i, new TestValue(order, i, i));
+            map.put(i, new TestValue(order, i, i));
+
+            orderCounter.incrementAndGet();
         }
 
         final AtomicInteger nodesCnt = new AtomicInteger(4);
 
         IgniteInternalFuture fut = runMultiThreadedAsync(new Callable<Void>() {
+            /**
+             * @param chance Chance of remove operation in percents.
+             * @return {@code true} if it should be remove operation.
+             */
+            private boolean removeOp(int chance) {
+                return ThreadLocalRandom.current().nextInt(100) + 1 <= chance;
+            }
+
             @Override public Void call() throws Exception {
+                Random rnd = ThreadLocalRandom.current();
+
                 while (true) {
                     if (stop.get())
                         return null;
 
                     if (suspend.get()) {
+                        suspended.set(true);
+
                         U.sleep(10);
 
                         continue;
                     }
 
-                    int k = ThreadLocalRandom.current().nextInt(entriesCnt);
-                    int v1 = ThreadLocalRandom.current().nextInt();
-                    int v2 = ThreadLocalRandom.current().nextInt();
+                    int k = rnd.nextInt(entriesCnt);
+                    long order = orderCounter.get();
 
-                    int n = nodesCnt.get();
+                    int v1 = 0, v2 = 0;
+                    boolean remove = false;
 
-                    if (n <= 0)
-                        continue;
+                    if (removeOp(allowRemoves ? 20 : 0))
+                        remove = true;
+                    else {
+                        v1 = rnd.nextInt();
+                        v2 = rnd.nextInt();
+                    }
+
+                    int nodes = nodesCnt.get();
 
                     Ignite ignite;
 
                     try {
-                        ignite = grid(ThreadLocalRandom.current().nextInt(n));
+                        ignite = grid(rnd.nextInt(nodes));
                     }
                     catch (Exception ignored) {
                         continue;
                     }
 
-                    if (ignite == null)
-                        continue;
-
                     Transaction tx = null;
                     boolean success = true;
 
@@ -379,7 +410,12 @@ public void testTopologyChangesWithConstantLoad() throws Exception {
                         tx = ignite.transactions().txStart();
 
                     try {
-                        ignite.cache(INDEXED_CACHE).put(k, new TestValue(v1, v2));
+                        IgniteCache<Object, Object> cache = ignite.cache(INDEXED_CACHE);
+
+                        if (remove)
+                            cache.remove(k);
+                        else
+                            cache.put(k, new TestValue(order, v1, v2));
                     }
                     catch (Exception ignored) {
                         success = false;
@@ -395,13 +431,19 @@ public void testTopologyChangesWithConstantLoad() throws Exception {
                         }
                     }
 
-                    if (success)
-                        map.put(k, new TestValue(v1, v2));
+                    if (success) {
+                        map.put(k, new TestValue(order, v1, v2, remove));
+
+                        orderCounter.incrementAndGet();
+                    }
                 }
             }
         }, 1, "load-runner");
 
-        boolean[] changes = new boolean[] {false, false, true, true};
+        // "False" means stop last started node, "True" - start new node.
+        List<Boolean> predefinedChanges = Lists.newArrayList(false, false, true, true);
+
+        List<Boolean> topChangesHistory = new ArrayList<>();
 
         try {
             for (int it = 0; it < topChanges; it++) {
@@ -410,32 +452,62 @@ public void testTopologyChangesWithConstantLoad() throws Exception {
 
                 U.sleep(3_000);
 
-                boolean add;
+                boolean addNode;
 
-                if (it < changes.length)
-                    add = changes[it];
+                if (it < predefinedChanges.size())
+                    addNode = predefinedChanges.get(it);
                 else if (nodesCnt.get() <= maxNodesCnt / 2)
-                    add = true;
+                    addNode = true;
                 else if (nodesCnt.get() >= maxNodesCnt)
-                    add = false;
+                    addNode = false;
                 else // More chance that node will be added
-                    add = ThreadLocalRandom.current().nextInt(3) <= 1;
+                    addNode = ThreadLocalRandom.current().nextInt(3) <= 1;
 
-                if (add)
+                if (addNode)
                     startGrid(nodesCnt.getAndIncrement());
                 else
                     stopGrid(nodesCnt.decrementAndGet());
 
+                topChangesHistory.add(addNode);
+
                 awaitPartitionMapExchange();
 
+                if (fut.error() != null)
+                    break;
+
+                // Suspend loader and wait for last operation completion.
                 suspend.set(true);
+                GridTestUtils.waitForCondition(suspended::get, 5_000);
+
+                // Fix last successful cache operation to skip operations that can be performed during check.
+                long maxOrder = orderCounter.get();
 
-                U.sleep(200);
+                for (Map.Entry<Integer, TestValue> entry : map.entrySet()) {
+                    final String assertMsg = "Iteration: " + it + ". Changes: " + Objects.toString(topChangesHistory)
+                            + ". Key: " + Integer.toString(entry.getKey());
 
-                for (Map.Entry<Integer, TestValue> entry : map.entrySet())
-                    assertEquals(it + " " + Integer.toString(entry.getKey()), entry.getValue(), cache.get(entry.getKey()));
+                    TestValue expected = entry.getValue();
 
+                    if (expected.order < maxOrder)
+                        continue;
+
+                    TestValue actual = cache.get(entry.getKey());
+
+                    if (expected.removed) {
+                        assertNull(assertMsg + " should be removed.", actual);
+
+                        continue;
+                    }
+
+                    if (entry.getValue().order < maxOrder)
+                        continue;
+
+                    assertEquals(assertMsg, expected, actual);
+                }
+
+                // Resume progress for loader.
                 suspend.set(false);
+                suspended.set(false);
             }
         }
         finally {
@@ -443,11 +515,6 @@ else if (nodesCnt.get() >= maxNodesCnt)
         }
 
         fut.get();
-
-        awaitPartitionMapExchange();
-
-        for (Map.Entry<Integer, TestValue> entry : map.entrySet())
-            assertEquals(Integer.toString(entry.getKey()), entry.getValue(), cache.get(entry.getKey()));
     }
 
     /**
@@ -596,46 +663,52 @@ public void testPartitionCounterConsistencyOnUnstableTopology() throws Exception
      *
      */
     private static class TestValue implements Serializable {
+        /** Operation order. */
+        private final long order;
+
         /** V 1. */
         private final int v1;
+
         /** V 2. */
         private final int v2;
 
-        /**
-         * @param v1 V 1.
-         * @param v2 V 2.
-         */
-        private TestValue(int v1, int v2) {
+        /** Flag indicates that value has removed. */
+        private final boolean removed;
+
+        private TestValue(long order, int v1, int v2) {
+            this(order, v1, v2, false);
+        }
+
+        private TestValue(long order, int v1, int v2, boolean removed) {
+            this.order = order;
             this.v1 = v1;
             this.v2 = v2;
+            this.removed = removed;
         }
 
         /** {@inheritDoc} */
         @Override public boolean equals(Object o) {
-            if (this == o)
-                return true;
-            if (o == null || getClass() != o.getClass())
-                return false;
+            if (this == o) return true;
 
-            TestValue val = (TestValue)o;
+            if (o == null || getClass() != o.getClass()) return false;
 
-            return v1 == val.v1 && v2 == val.v2;
+            TestValue testValue = (TestValue) o;
 
+            return order == testValue.order &&
+                v1 == testValue.v1 &&
+                v2 == testValue.v2;
         }
 
         /** {@inheritDoc} */
         @Override public int hashCode() {
-            int res = v1;
-
-            res = 31 * res + v2;
-
-            return res;
+            return Objects.hash(order, v1, v2);
         }
 
         /** {@inheritDoc} */
         @Override public String toString() {
             return "TestValue{" +
-                "v1=" + v1 +
+                "order=" + order +
+                ", v1=" + v1 +
                 ", v2=" + v2 +
                 '}';
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheRebalancingTest.java
index c641ea4857dbc..3b324c37436fd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheRebalancingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheRebalancingTest.java
@@ -36,7 +36,6 @@ public class IgnitePdsTxCacheRebalancingTest extends IgnitePdsCacheRebalancingAb
         ccfg.setCacheMode(CacheMode.PARTITIONED);
         ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
         ccfg.setBackups(1);
-        ccfg.setRebalanceDelay(10_000);
         ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
         ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxHistoricalRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxHistoricalRebalancingTest.java
new file mode 100644
index 0000000000000..8236bd3c3ae05
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxHistoricalRebalancingTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRebalanceTest;
+
+/**
+ *
+ */
+public class IgnitePdsTxHistoricalRebalancingTest extends IgnitePdsTxCacheRebalancingTest {
+    /** {@inheritDoc */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCommunicationSpi(new IgniteWalRebalanceTest.WalRebalanceCheckingCommunicationSpi());
+
+        return cfg;
+    }
+
+    /** {@inheritDoc */
+    @Override protected long checkpointFrequency() {
+        return 15 * 1000;
+    }
+
+    /** {@inheritDoc */
+    @Override protected void beforeTest() throws Exception {
+        // Use rebalance from WAL if possible.
+        System.setProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc */
+    @Override protected void afterTest() throws Exception {
+        boolean walRebalanceInvoked = !IgniteWalRebalanceTest.WalRebalanceCheckingCommunicationSpi.allRebalances()
+            .isEmpty();
+
+        IgniteWalRebalanceTest.WalRebalanceCheckingCommunicationSpi.cleanup();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
+
+        super.afterTest();
+
+        if (!walRebalanceInvoked)
+            throw new AssertionError("WAL rebalance hasn't been invoked.");
+    }
+}
diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml
index d8f5f25a93593..29d7a3b911ee5 100644
--- a/modules/indexing/pom.xml
+++ b/modules/indexing/pom.xml
@@ -119,6 +119,13 @@
             <version>${spring.version}</version>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index 571576010995b..c47766b02d84a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -24,6 +24,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCorruptedIndexTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsMarshallerMappingRestoreOnNodeStartTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheRebalancingTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxHistoricalRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreCacheGroupsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionTest;
@@ -60,11 +61,13 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteWalRecoveryTest.class);
         suite.addTestSuite(IgniteWalRecoveryWithCompactionTest.class);
         suite.addTestSuite(IgnitePdsNoActualWalHistoryTest.class);
-        suite.addTestSuite(IgnitePdsAtomicCacheRebalancingTest.class);
-        suite.addTestSuite(IgnitePdsTxCacheRebalancingTest.class);
+        suite.addTestSuite(IgniteWalRebalanceTest.class);
 
+        suite.addTestSuite(IgnitePdsAtomicCacheRebalancingTest.class);
         suite.addTestSuite(IgnitePdsAtomicCacheHistoricalRebalancingTest.class);
-        suite.addTestSuite(IgniteWalRebalanceTest.class);
+
+        suite.addTestSuite(IgnitePdsTxCacheRebalancingTest.class);
+        suite.addTestSuite(IgnitePdsTxHistoricalRebalancingTest.class);
 
         suite.addTestSuite(IgniteWalRecoveryPPCTest.class);
         suite.addTestSuite(IgnitePdsDiskErrorsRecoveringTest.class);

From 2ea5b376d895fe41cc882c4383b5d3d2793ba684 Mon Sep 17 00:00:00 2001
From: devozerov <ppozerov@gmail.com>
Date: Tue, 31 Jul 2018 12:53:51 +0300
Subject: [PATCH 272/543] IGNITE-9114: SQL: fail query after some timeout if it
 cannot be mapped to topology. This closes #4453.

---
 .../apache/ignite/IgniteSystemProperties.java |   3 +
 .../query/h2/opt/GridH2IndexBase.java         |  31 ++--
 .../h2/twostep/GridMapQueryExecutor.java      |  75 ++++++++--
 .../h2/twostep/GridReduceQueryExecutor.java   | 136 ++++++++++++++----
 ...tedPartitionQueryNodeRestartsSelfTest.java |  21 ++-
 ...eryNodeRestartDistributedJoinSelfTest.java |  10 ++
 6 files changed, 230 insertions(+), 46 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 3f47d76f26584..68f95fe760082 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -468,6 +468,9 @@ public final class IgniteSystemProperties {
     /** Force all SQL queries to be processed lazily regardless of what clients request. */
     public static final String IGNITE_SQL_FORCE_LAZY_RESULT_SET = "IGNITE_SQL_FORCE_LAZY_RESULT_SET";
 
+    /** SQL retry timeout. */
+    public static final String IGNITE_SQL_RETRY_TIMEOUT = "IGNITE_SQL_RETRY_TIMEOUT";
+
     /** Maximum size for affinity assignment history. */
     public static final String IGNITE_AFFINITY_HISTORY_SIZE = "IGNITE_AFFINITY_HISTORY_SIZE";
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
index 1c8921348f61c..77bd69a4aeb48 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java
@@ -361,7 +361,7 @@ private void send(Collection<ClusterNode> nodes, Message msg) {
             locNodeHnd,
             GridIoPolicy.IDX_POOL,
             false))
-            throw new GridH2RetryException("Failed to send message to nodes: " + nodes + ".");
+            throw retryException("Failed to send message to nodes: " + nodes);
     }
 
     /**
@@ -554,14 +554,15 @@ private List<SegmentKey> broadcastSegments(GridH2QueryContext qctx, GridCacheCon
                     ClusterNode node = ctx.discovery().node(nodeId);
 
                     if (node == null)
-                        throw new GridH2RetryException("Failed to find node.");
+                        throw retryException("Failed to get node by ID during broadcast [nodeId=" + nodeId + ']');
 
                     nodes.add(node);
                 }
             }
 
             if (F.isEmpty(nodes))
-                throw new GridH2RetryException("Failed to collect affinity nodes.");
+                throw retryException("Failed to collect affinity nodes during broadcast [" +
+                    "cacheName=" + cctx.name() + ']');
         }
 
         int segmentsCount = segmentsCount();
@@ -615,7 +616,7 @@ private SegmentKey rangeSegment(GridCacheContext<?, ?> cctx, GridH2QueryContext
             node = cctx.affinity().primaryByKey(affKeyObj, qctx.topologyVersion());
 
             if (node == null) // Node was not found, probably topology changed and we need to retry the whole query.
-                throw new GridH2RetryException("Failed to find node.");
+                throw retryException("Failed to get primary node by key for range segment.");
         }
 
         return new SegmentKey(node, segmentForPartition(partition));
@@ -1317,10 +1318,10 @@ private GridH2IndexRangeResponse awaitForResponse() {
 
             for (int attempt = 0;; attempt++) {
                 if (qctx.isCleared())
-                    throw new GridH2RetryException("Query is cancelled.");
+                    throw retryException("Query is cancelled.");
 
                 if (kernalContext().isStopping())
-                    throw new GridH2RetryException("Stopping node.");
+                    throw retryException("Local node is stopping.");
 
                 GridH2IndexRangeResponse res;
 
@@ -1328,7 +1329,7 @@ private GridH2IndexRangeResponse awaitForResponse() {
                     res = respQueue.poll(500, TimeUnit.MILLISECONDS);
                 }
                 catch (InterruptedException ignored) {
-                    throw new GridH2RetryException("Interrupted.");
+                    throw retryException("Interrupted while waiting for reply.");
                 }
 
                 if (res != null) {
@@ -1355,10 +1356,10 @@ private GridH2IndexRangeResponse awaitForResponse() {
 
                         case STATUS_NOT_FOUND:
                             if (req == null || req.bounds() == null) // We have already received the first response.
-                                throw new GridH2RetryException("Failure on remote node.");
+                                throw retryException("Failure on remote node.");
 
                             if (U.currentTimeMillis() - start > 30_000)
-                                throw new GridH2RetryException("Timeout.");
+                                throw retryException("Timeout reached.");
 
                             try {
                                 U.sleep(20 * attempt);
@@ -1381,7 +1382,7 @@ private GridH2IndexRangeResponse awaitForResponse() {
                 }
 
                 if (!kernalContext().discovery().alive(node))
-                    throw new GridH2RetryException("Node left: " + node);
+                    throw retryException("Node has left topology: " + node.id());
             }
         }
 
@@ -1584,6 +1585,16 @@ public void refreshColumnIds() {
             columnIds[pos] = columns[pos].getColumnId();
     }
 
+    /**
+     * Create retry exception for distributed join.
+     *
+     * @param msg Message.
+     * @return Exception.
+     */
+    private GridH2RetryException retryException(String msg) {
+        return new GridH2RetryException(msg);
+    }
+
     /**
      *
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 930ada22f523e..216a259da73e4 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -53,6 +53,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsReservation;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
 import org.apache.ignite.internal.processors.cache.query.CacheQueryType;
@@ -293,6 +294,8 @@ private GridDhtLocalPartition partition(GridCacheContext<?, ?> cctx, int p) {
      * @param topVer Topology version.
      * @param explicitParts Explicit partitions list.
      * @param reserved Reserved list.
+     * @param nodeId Node ID.
+     * @param reqId Request ID.
      * @return {@code true} If all the needed partitions successfully reserved.
      * @throws IgniteCheckedException If failed.
      */
@@ -300,7 +303,9 @@ private boolean reservePartitions(
         @Nullable List<Integer> cacheIds,
         AffinityTopologyVersion topVer,
         final int[] explicitParts,
-        List<GridReservable> reserved
+        List<GridReservable> reserved,
+        UUID nodeId,
+        long reqId
     ) throws IgniteCheckedException {
         assert topVer != null;
 
@@ -312,8 +317,14 @@ private boolean reservePartitions(
         for (int i = 0; i < cacheIds.size(); i++) {
             GridCacheContext<?, ?> cctx = ctx.cache().context().cacheContext(cacheIds.get(i));
 
-            if (cctx == null) // Cache was not found, probably was not deployed yet.
+            // Cache was not found, probably was not deployed yet.
+            if (cctx == null) {
+                logRetry("Failed to reserve partitions for query (cache is not found on local node) [" +
+                    "rmtNodeId=" + nodeId + ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" +
+                    cacheIds.get(i) + "]");
+
                 return false;
+            }
 
             if (cctx.isLocal() || !cctx.rebalanceEnabled())
                 continue;
@@ -325,8 +336,13 @@ private boolean reservePartitions(
 
             if (explicitParts == null && r != null) { // Try to reserve group partition if any and no explicits.
                 if (r != MapReplicatedReservation.INSTANCE) {
-                    if (!r.reserve())
+                    if (!r.reserve()) {
+                        logRetry("Failed to reserve partitions for query (group reservation failed) [" +
+                            "rmtNodeId=" + nodeId + ", reqId=" + reqId + ", affTopVer=" + topVer +
+                            ", cacheId=" + cacheIds.get(i) + ", cacheName=" + cctx.name() + "]");
+
                         return false; // We need explicit partitions here -> retry.
+                    }
 
                     reserved.add(r);
                 }
@@ -340,8 +356,17 @@ private boolean reservePartitions(
                             GridDhtLocalPartition part = partition(cctx, p);
 
                             // We don't need to reserve partitions because they will not be evicted in replicated caches.
-                            if (part == null || part.state() != OWNING)
+                            GridDhtPartitionState partState = part != null ? part.state() : null;
+
+                            if (partState != OWNING) {
+                                logRetry("Failed to reserve partitions for query (partition of " +
+                                    "REPLICATED cache is not in OWNING state) [rmtNodeId=" + nodeId +
+                                    ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
+                                    ", cacheName=" + cctx.name() + ", part=" + p + ", partFound=" + (part != null) +
+                                    ", partState=" + partState + "]");
+
                                 return false;
+                            }
                         }
 
                         // Mark that we checked this replicated cache.
@@ -355,14 +380,31 @@ private boolean reservePartitions(
                     for (int partId : partIds) {
                         GridDhtLocalPartition part = partition(cctx, partId);
 
-                        if (part == null || part.state() != OWNING || !part.reserve())
+                        GridDhtPartitionState partState = part != null ? part.state() : null;
+
+                        if (partState != OWNING || !part.reserve()) {
+                            logRetry("Failed to reserve partitions for query (partition of " +
+                                "PARTITIONED cache cannot be reserved) [rmtNodeId=" + nodeId + ", reqId=" + reqId +
+                                ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
+                                ", cacheName=" + cctx.name() + ", part=" + partId + ", partFound=" + (part != null) +
+                                ", partState=" + partState + "]");
+
                             return false;
+                        }
 
                         reserved.add(part);
 
                         // Double check that we are still in owning state and partition contents are not cleared.
-                        if (part.state() != OWNING)
+                        partState = part.state();
+
+                        if (part.state() != OWNING) {
+                            logRetry("Failed to reserve partitions for query (partition of " +
+                                "PARTITIONED cache is not in OWNING state after reservation) [rmtNodeId=" + nodeId +
+                                ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
+                                ", cacheName=" + cctx.name() + ", part=" + partId + ", partState=" + partState + "]");
+
                             return false;
+                        }
                     }
 
                     if (explicitParts == null) {
@@ -387,6 +429,15 @@ private boolean reservePartitions(
         return true;
     }
 
+    /**
+     * Load failed partition reservation.
+     *
+     * @param msg Message.
+     */
+    private void logRetry(String msg) {
+        log.info(msg);
+    }
+
     /**
      * @param ints Integers.
      * @return Collection wrapper.
@@ -622,7 +673,7 @@ private void onQueryRequest0(
         try {
             if (topVer != null) {
                 // Reserve primary for topology version or explicit partitions.
-                if (!reservePartitions(cacheIds, topVer, parts, reserved)) {
+                if (!reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId)) {
                     // Unregister lazy worker because re-try may never reach this node again.
                     if (lazy)
                         stopAndUnregisterCurrentLazyWorker();
@@ -739,8 +790,14 @@ private void onQueryRequest0(
             if (lazy)
                 stopAndUnregisterCurrentLazyWorker();
 
-            if (X.hasCause(e, GridH2RetryException.class))
+            GridH2RetryException retryErr = X.cause(e, GridH2RetryException.class);
+
+            if (retryErr != null) {
+                logRetry("Failed to execute non-collocated query (will retry) [nodeId=" + node.id() +
+                    ", reqId=" + reqId + ", errMsg=" + retryErr.getMessage() + ']');
+
                 sendRetry(node, reqId, segmentId);
+            }
             else {
                 U.error(log, "Failed to execute local query.", e);
 
@@ -788,7 +845,7 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th
 
         List<GridReservable> reserved = new ArrayList<>();
 
-        if (!reservePartitions(cacheIds, topVer, parts, reserved)) {
+        if (!reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId)) {
             U.error(log, "Failed to reserve partitions for DML request. [localNodeId=" + ctx.localNodeId() +
                 ", nodeId=" + node.id() + ", reqId=" + req.requestId() + ", cacheIds=" + cacheIds +
                 ", topVer=" + topVer + ", parts=" + Arrays.toString(parts) + ']');
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index cd76bc1977bc3..d778fccc30c1d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -45,6 +45,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.query.QueryCancelledException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.DiscoveryEvent;
@@ -101,6 +102,7 @@
 import org.jetbrains.annotations.Nullable;
 
 import static java.util.Collections.singletonList;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT;
 import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
 import static org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery.EMPTY_PARAMS;
 import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
@@ -111,6 +113,9 @@
  * Reduce query executor.
  */
 public class GridReduceQueryExecutor {
+    /** Fail query after 10 seconds of unsuccessful attempts to reserve partitions. */
+    public static final long DFLT_RETRY_TIMEOUT = 10_000L;
+
     /** */
     private static final String MERGE_INDEX_UNSORTED = "merge_scan";
 
@@ -441,15 +446,25 @@ private Map<ClusterNode, IntArray> stableDataNodesMap(AffinityTopologyVersion to
         return mapping;
     }
 
+    /**
+     * Load failed partition reservation.
+     *
+     * @param msg Message.
+     */
+    private void logRetry(String msg) {
+        log.info(msg);
+    }
+
     /**
      * @param isReplicatedOnly If we must only have replicated caches.
      * @param topVer Topology version.
      * @param cacheIds Participating cache IDs.
      * @param parts Partitions.
+     * @param qryId Query ID.
      * @return Data nodes or {@code null} if repartitioning started and we need to retry.
      */
     private Map<ClusterNode, IntArray> stableDataNodes(boolean isReplicatedOnly, AffinityTopologyVersion topVer,
-        List<Integer> cacheIds, int[] parts) {
+        List<Integer> cacheIds, int[] parts, long qryId) {
         GridCacheContext<?, ?> cctx = cacheContext(cacheIds.get(0));
 
         // If the first cache is not partitioned, find it (if it's present) and move it to index 0.
@@ -507,8 +522,15 @@ private Map<ClusterNode, IntArray> stableDataNodes(boolean isReplicatedOnly, Aff
                 disjoint = !extraNodes.equals(nodes);
 
             if (disjoint) {
-                if (isPreloadingActive(cacheIds))
+                if (isPreloadingActive(cacheIds)) {
+                    logRetry("Failed to calculate nodes for SQL query (got disjoint node map during rebalance) " +
+                        "[qryId=" + qryId + ", affTopVer=" + topVer + ", cacheIds=" + cacheIds +
+                        ", parts=" + (parts == null ? "[]" : Arrays.toString(parts)) +
+                        ", replicatedOnly=" + isReplicatedOnly + ", lastCache=" + extraCctx.name() +
+                        ", lastCacheId=" + extraCctx.cacheId() + ']');
+
                     return null; // Retry.
+                }
                 else
                     throw new CacheException("Caches have distinct sets of data nodes [cache1=" + cctx.name() +
                         ", cache2=" + extraCacheName + "]");
@@ -546,8 +568,14 @@ public Iterator<List<?>> query(
 
         final boolean isReplicatedOnly = qry.isReplicatedOnly();
 
-        // Fail if all caches are replicated and explicit partitions are set.
+        long retryTimeout = retryTimeout();
+
+        final long startTime = U.currentTimeMillis();
+
         for (int attempt = 0;; attempt++) {
+            if (attempt > 0 && retryTimeout > 0 && (U.currentTimeMillis() - startTime > retryTimeout))
+                throw new CacheException("Failed to map SQL query to topology.");
+
             if (attempt != 0) {
                 try {
                     Thread.sleep(attempt * 10); // Wait for exchange.
@@ -559,7 +587,7 @@ public Iterator<List<?>> query(
                 }
             }
 
-            final long qryReqId = qryIdGen.incrementAndGet();
+            long qryReqId = qryIdGen.incrementAndGet();
 
             final ReduceQueryRun r = new ReduceQueryRun(qryReqId, qry.originalSql(), schemaName,
                 h2.connectionForSchema(schemaName), qry.mapQueries().size(), qry.pageSize(),
@@ -602,7 +630,8 @@ public Iterator<List<?>> query(
             if (qry.isLocal())
                 nodes = singletonList(ctx.discovery().localNode());
             else {
-                NodesForPartitionsResult nodesParts = nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly);
+                NodesForPartitionsResult nodesParts =
+                    nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly, qryReqId);
 
                 nodes = nodesParts.nodes();
                 partsMap = nodesParts.partitionsMap();
@@ -704,9 +733,11 @@ public Iterator<List<?>> query(
 
                 final boolean distributedJoins = qry.distributedJoins();
 
+                final long qryReqId0 = qryReqId;
+
                 cancel.set(new Runnable() {
                     @Override public void run() {
-                        send(finalNodes, new GridQueryCancelRequest(qryReqId), null, false);
+                        send(finalNodes, new GridQueryCancelRequest(qryReqId0), null, false);
                     }
                 });
 
@@ -885,10 +916,10 @@ public UpdateResult update(
     ) {
         AffinityTopologyVersion topVer = h2.readyTopologyVersion();
 
-        NodesForPartitionsResult nodesParts = nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly);
-
         final long reqId = qryIdGen.incrementAndGet();
 
+        NodesForPartitionsResult nodesParts = nodesForPartitions(cacheIds, topVer, parts, isReplicatedOnly, reqId);
+
         final GridRunningQueryInfo qryInfo = new GridRunningQueryInfo(reqId, selectQry, GridCacheQueryType.SQL_FIELDS,
             schemaName, U.currentTimeMillis(), cancel, false);
 
@@ -1128,9 +1159,10 @@ private GridThreadLocalTable fakeTable(Connection c, int idx) {
      * Calculates data nodes for replicated caches on unstable topology.
      *
      * @param cacheIds Cache IDs.
+     * @param qryId Query ID.
      * @return Collection of all data nodes owning all the caches or {@code null} for retry.
      */
-    private Collection<ClusterNode> replicatedUnstableDataNodes(List<Integer> cacheIds) {
+    private Collection<ClusterNode> replicatedUnstableDataNodes(List<Integer> cacheIds, long qryId) {
         int i = 0;
 
         GridCacheContext<?, ?> cctx = cacheContext(cacheIds.get(i++));
@@ -1145,7 +1177,7 @@ private Collection<ClusterNode> replicatedUnstableDataNodes(List<Integer> cacheI
             assert cctx.isReplicated(): "all the extra caches must be replicated here";
         }
 
-        Set<ClusterNode> nodes = replicatedUnstableDataNodes(cctx);
+        Set<ClusterNode> nodes = replicatedUnstableDataNodes(cctx, qryId);
 
         if (F.isEmpty(nodes))
             return null; // Retry.
@@ -1161,15 +1193,20 @@ private Collection<ClusterNode> replicatedUnstableDataNodes(List<Integer> cacheI
                     "with tables in partitioned caches [replicatedCache=" + cctx.name() + ", " +
                     "partitionedCache=" + extraCctx.name() + "]");
 
-            Set<ClusterNode> extraOwners = replicatedUnstableDataNodes(extraCctx);
+            Set<ClusterNode> extraOwners = replicatedUnstableDataNodes(extraCctx, qryId);
 
             if (F.isEmpty(extraOwners))
                 return null; // Retry.
 
             nodes.retainAll(extraOwners);
 
-            if (nodes.isEmpty())
+            if (nodes.isEmpty()) {
+                logRetry("Failed to calculate nodes for SQL query (got disjoint node map for REPLICATED caches " +
+                    "during rebalance) [qryId=" + qryId + ", cacheIds=" + cacheIds +
+                    ", lastCache=" + extraCctx.name() + ", lastCacheId=" + extraCctx.cacheId() + ']');
+
                 return null; // Retry.
+            }
         }
 
         return nodes;
@@ -1190,9 +1227,10 @@ private Collection<ClusterNode> dataNodes(int grpId, AffinityTopologyVersion top
      * Collects all the nodes owning all the partitions for the given replicated cache.
      *
      * @param cctx Cache context.
+     * @param qryId Query ID.
      * @return Owning nodes or {@code null} if we can't find owners for some partitions.
      */
-    private Set<ClusterNode> replicatedUnstableDataNodes(GridCacheContext<?,?> cctx) {
+    private Set<ClusterNode> replicatedUnstableDataNodes(GridCacheContext<?,?> cctx, long qryId) {
         assert cctx.isReplicated() : cctx.name() + " must be replicated";
 
         String cacheName = cctx.name();
@@ -1206,13 +1244,23 @@ private Set<ClusterNode> replicatedUnstableDataNodes(GridCacheContext<?,?> cctx)
         for (int p = 0, parts = cctx.affinity().partitions(); p < parts; p++) {
             List<ClusterNode> owners = cctx.topology().owners(p);
 
-            if (F.isEmpty(owners))
+            if (F.isEmpty(owners)) {
+                logRetry("Failed to calculate nodes for SQL query (partition of a REPLICATED cache has no owners) [" +
+                    "qryId=" + qryId + ", cacheName=" + cctx.name() + ", cacheId=" + cctx.cacheId() +
+                    ", part=" + p + ']');
+
                 return null; // Retry.
+            }
 
             dataNodes.retainAll(owners);
 
-            if (dataNodes.isEmpty())
+            if (dataNodes.isEmpty()) {
+                logRetry("Failed to calculate nodes for SQL query (partitions of a REPLICATED has no common owners) [" +
+                    "qryId=" + qryId + ", cacheName=" + cctx.name() + ", cacheId=" + cctx.cacheId() +
+                    ", lastPart=" + p + ']');
+
                 return null; // Retry.
+            }
         }
 
         return dataNodes;
@@ -1222,10 +1270,11 @@ private Set<ClusterNode> replicatedUnstableDataNodes(GridCacheContext<?,?> cctx)
      * Calculates partition mapping for partitioned cache on unstable topology.
      *
      * @param cacheIds Cache IDs.
+     * @param qryId Query ID.
      * @return Partition mapping or {@code null} if we can't calculate it due to repartitioning and we need to retry.
      */
     @SuppressWarnings("unchecked")
-    private Map<ClusterNode, IntArray> partitionedUnstableDataNodes(List<Integer> cacheIds) {
+    private Map<ClusterNode, IntArray> partitionedUnstableDataNodes(List<Integer> cacheIds, long qryId) {
         // If the main cache is replicated, just replace it with the first partitioned.
         GridCacheContext<?,?> cctx = findFirstPartitioned(cacheIds);
 
@@ -1260,8 +1309,14 @@ private Map<ClusterNode, IntArray> partitionedUnstableDataNodes(List<Integer> ca
 
                     continue;
                 }
-                else if (!F.isEmpty(dataNodes(cctx.groupId(), NONE)))
+                else if (!F.isEmpty(dataNodes(cctx.groupId(), NONE))) {
+                    logRetry("Failed to calculate nodes for SQL query (partition has no owners, but corresponding " +
+                        "cache group has data nodes) [qryId=" + qryId + ", cacheIds=" + cacheIds +
+                        ", cacheName=" + cctx.name() + ", cacheId=" + cctx.cacheId() + ", part=" + p +
+                        ", cacheGroupId=" + cctx.groupId() + ']');
+
                     return null; // Retry.
+                }
 
                 throw new CacheException("Failed to find data nodes [cache=" + cctx.name() + ", part=" + p + "]");
             }
@@ -1289,8 +1344,15 @@ else if (!F.isEmpty(dataNodes(cctx.groupId(), NONE)))
                         continue; // Skip unmapped partitions.
 
                     if (F.isEmpty(owners)) {
-                        if (!F.isEmpty(dataNodes(extraCctx.groupId(), NONE)))
+                        if (!F.isEmpty(dataNodes(extraCctx.groupId(), NONE))) {
+                            logRetry("Failed to calculate nodes for SQL query (partition has no owners, but " +
+                                "corresponding cache group has data nodes) [qryId=" + qryId +
+                                ", cacheIds=" + cacheIds + ", cacheName=" + extraCctx.name() +
+                                ", cacheId=" + extraCctx.cacheId() + ", part=" + p +
+                                ", cacheGroupId=" + extraCctx.groupId() + ']');
+
                             return null; // Retry.
+                        }
 
                         throw new CacheException("Failed to find data nodes [cache=" + extraCctx.name() +
                             ", part=" + p + "]");
@@ -1301,8 +1363,14 @@ else if (!F.isEmpty(dataNodes(cctx.groupId(), NONE)))
                     else {
                         partLocs[p].retainAll(owners); // Intersection of owners.
 
-                        if (partLocs[p].isEmpty())
+                        if (partLocs[p].isEmpty()) {
+                            logRetry("Failed to calculate nodes for SQL query (caches have no common data nodes for " +
+                                "partition) [qryId=" + qryId + ", cacheIds=" + cacheIds +
+                                ", lastCacheName=" + extraCctx.name() + ", lastCacheId=" + extraCctx.cacheId() +
+                                ", part=" + p + ']');
+
                             return null; // Intersection is empty -> retry.
+                        }
                     }
                 }
             }
@@ -1314,19 +1382,29 @@ else if (!F.isEmpty(dataNodes(cctx.groupId(), NONE)))
                 if (!extraCctx.isReplicated())
                     continue;
 
-                Set<ClusterNode> dataNodes = replicatedUnstableDataNodes(extraCctx);
+                Set<ClusterNode> dataNodes = replicatedUnstableDataNodes(extraCctx, qryId);
 
                 if (F.isEmpty(dataNodes))
                     return null; // Retry.
 
+                int part = 0;
+
                 for (Set<ClusterNode> partLoc : partLocs) {
                     if (partLoc == UNMAPPED_PARTS)
                         continue; // Skip unmapped partition.
 
                     partLoc.retainAll(dataNodes);
 
-                    if (partLoc.isEmpty())
+                    if (partLoc.isEmpty()) {
+                        logRetry("Failed to calculate nodes for SQL query (caches have no common data nodes for " +
+                            "partition) [qryId=" + qryId + ", cacheIds=" + cacheIds +
+                            ", lastReplicatedCacheName=" + extraCctx.name() +
+                            ", lastReplicatedCacheId=" + extraCctx.cacheId() + ", part=" + part + ']');
+
                         return null; // Retry.
+                    }
+
+                    part++;
                 }
             }
         }
@@ -1475,19 +1553,20 @@ private static Map<UUID, int[]> convert(Map<ClusterNode, IntArray> m) {
      * @param topVer Topology version.
      * @param parts Partitions array.
      * @param isReplicatedOnly Allow only replicated caches.
+     * @param qryId Query ID.
      * @return Result.
      */
     private NodesForPartitionsResult nodesForPartitions(List<Integer> cacheIds, AffinityTopologyVersion topVer,
-        int[] parts, boolean isReplicatedOnly) {
+        int[] parts, boolean isReplicatedOnly, long qryId) {
         Collection<ClusterNode> nodes = null;
         Map<ClusterNode, IntArray> partsMap = null;
         Map<ClusterNode, IntArray> qryMap = null;
 
         if (isPreloadingActive(cacheIds)) {
             if (isReplicatedOnly)
-                nodes = replicatedUnstableDataNodes(cacheIds);
+                nodes = replicatedUnstableDataNodes(cacheIds, qryId);
             else {
-                partsMap = partitionedUnstableDataNodes(cacheIds);
+                partsMap = partitionedUnstableDataNodes(cacheIds, qryId);
 
                 if (partsMap != null) {
                     qryMap = narrowForQuery(partsMap, parts);
@@ -1497,7 +1576,7 @@ private NodesForPartitionsResult nodesForPartitions(List<Integer> cacheIds, Affi
             }
         }
         else {
-            qryMap = stableDataNodes(isReplicatedOnly, topVer, cacheIds, parts);
+            qryMap = stableDataNodes(isReplicatedOnly, topVer, cacheIds, parts, qryId);
 
             if (qryMap != null)
                 nodes = qryMap.keySet();
@@ -1676,6 +1755,13 @@ private Map<ClusterNode, IntArray> narrowForQuery(Map<ClusterNode, IntArray> par
         return cp.isEmpty() ? null : cp;
     }
 
+    /**
+     * @return Query retry timeout.
+     */
+    private static long retryTimeout() {
+        return IgniteSystemProperties.getLong(IGNITE_SQL_RETRY_TIMEOUT, DFLT_RETRY_TIMEOUT);
+    }
+
     /** */
     private static class ExplicitPartitionsSpecializer implements IgniteBiClosure<ClusterNode, Message, Message> {
         /** Partitions map. */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java
index 852541023ffe4..806fbbf37f439 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest.java
@@ -24,14 +24,31 @@
 import java.util.concurrent.atomic.AtomicIntegerArray;
 
 import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor;
 
 /**
  * Tests distributed queries over set of partitions on unstable topology.
  */
-public class IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest
-        extends IgniteCacheDistributedPartitionQueryAbstractSelfTest {
+public class IgniteCacheDistributedPartitionQueryNodeRestartsSelfTest extends
+    IgniteCacheDistributedPartitionQueryAbstractSelfTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT, Long.toString(1000_000L));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT,
+            Long.toString(GridReduceQueryExecutor.DFLT_RETRY_TIMEOUT));
+
+        super.afterTestsStopped();
+    }
+
     /**
      * Tests join query within region on unstable topology.
      */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartDistributedJoinSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartDistributedJoinSelfTest.java
index 4f2007818a689..bad53030d26dd 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartDistributedJoinSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheQueryNodeRestartDistributedJoinSelfTest.java
@@ -19,8 +19,10 @@
 
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor;
 import org.apache.ignite.internal.util.GridRandom;
 import org.apache.ignite.internal.util.typedef.CAX;
 import org.apache.ignite.internal.util.typedef.X;
@@ -41,6 +43,8 @@ public class IgniteCacheQueryNodeRestartDistributedJoinSelfTest extends IgniteCa
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT, Long.toString(1000_000L));
+
         super.beforeTestsStarted();
 
         if (totalNodes > GRID_CNT) {
@@ -51,6 +55,12 @@ public class IgniteCacheQueryNodeRestartDistributedJoinSelfTest extends IgniteCa
             totalNodes = GRID_CNT;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT,
+            Long.toString(GridReduceQueryExecutor.DFLT_RETRY_TIMEOUT));
+    }
+
     /**
      * @throws Exception If failed.
      */

From 5b230b7015148e483136c229c9e6081bb9c68023 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Fri, 20 Apr 2018 18:41:06 +0300
Subject: [PATCH 273/543] IGNITE-8134 Subscribe to system cache events on nodes
 outside BLT

Signed-off-by: Andrey Gura <agura@apache.org>
(cherry picked from commit c82277e)
---
 .../cluster/DiscoveryDataClusterState.java    |  11 +-
 .../service/GridServiceProcessor.java         |  12 +-
 .../ServiceDeploymentOutsideBaselineTest.java | 280 ++++++++++++++++++
 .../testsuites/IgniteKernalSelfTestSuite.java |   2 +
 4 files changed, 301 insertions(+), 4 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOutsideBaselineTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
index d626c9f9b2dd1..71357ad52a67e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
@@ -32,9 +32,9 @@
  * baseline topology.
  * <p>
  * This object also captures a transitional cluster state, when one or more fields are changing. In this case,
- * a {@code transitionReqId} field is set to a non-null value and {@code previousBaselineTopology} captures previous cluster state.
+ * a {@code transitionReqId} field is set to a non-null value and {@code prevState} captures previous cluster state.
  * A joining node catching the cluster in an intermediate state will observe {@code transitionReqId} field to be
- * non-null, however the {@code previousBaselineTopology} will not be sent to the joining node.
+ * non-null, however the {@code prevState} will not be sent to the joining node.
  *
  * TODO https://issues.apache.org/jira/browse/IGNITE-7640 This class must be immutable, transitionRes must be set by calling finish().
  */
@@ -200,6 +200,13 @@ public boolean active() {
         return baselineTopology;
     }
 
+    /**
+     * @return {@code True} if baseline topology is set in the cluster. {@code False} otherwise.
+     */
+    public boolean hasBaselineTopology() {
+        return baselineTopology != null;
+    }
+
     /**
      * @return Previous Baseline topology.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index cb87045bd79b0..3149857b492fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -246,9 +246,17 @@ private void onKernalStart0() throws IgniteCheckedException {
             if (ctx.deploy().enabled())
                 ctx.cache().context().deploy().ignoreOwnership(true);
 
-            if (!ctx.clientNode() && serviceCache.context().affinityNode()) {
+            if (!ctx.clientNode()) {
+                DiscoveryDataClusterState clusterState = ctx.state().clusterState();
+
+                boolean isLocLsnr = !clusterState.hasBaselineTopology() ||
+                    CU.baselineNode(ctx.cluster().get().localNode(), clusterState);
+
+                // Register query listener and run it for local entries, if data is available locally.
+                // It is also invoked on rebalancing.
+                // Otherwise remote listener is registered.
                 serviceCache.context().continuousQueries().executeInternalQuery(
-                    new ServiceEntriesListener(), null, true, true, false
+                    new ServiceEntriesListener(), null, isLocLsnr, true, false
                 );
             }
             else {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOutsideBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOutsideBaselineTest.java
new file mode 100644
index 0000000000000..1a5b630900599
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/ServiceDeploymentOutsideBaselineTest.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.service;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCluster;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/** */
+public class ServiceDeploymentOutsideBaselineTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final String SERVICE_NAME = "test-service";
+
+    /** */
+    private boolean persistence;
+
+    /** */
+    private ServiceConfiguration srvcCfg;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi discoverySpi = new TcpDiscoverySpi();
+        discoverySpi.setIpFinder(IP_FINDER);
+        cfg.setDiscoverySpi(discoverySpi);
+
+        if (persistence) {
+            cfg.setDataStorageConfiguration(
+                new DataStorageConfiguration()
+                    .setDefaultDataRegionConfiguration(
+                        new DataRegionConfiguration()
+                            .setPersistenceEnabled(true)
+                            .setMaxSize(10 * 1024 * 1024)
+                    ).setWalMode(WALMode.LOG_ONLY)
+            );
+        }
+
+        if (srvcCfg != null)
+            cfg.setServiceConfiguration(srvcCfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        persistence = false;
+        srvcCfg = null;
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOutsideBaseline() throws Exception {
+        checkDeploymentFromOutsideNode(true, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOutsideBaselineNoPersistence() throws Exception {
+        checkDeploymentFromOutsideNode(false, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOutsideBaselineStatic() throws Exception {
+        checkDeploymentFromOutsideNode(true, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployOutsideBaselineStaticNoPersistence() throws Exception {
+        checkDeploymentFromOutsideNode(false, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployFromNodeAddedToBlt() throws Exception {
+        checkDeployWithNodeAddedToBlt(true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployToNodeAddedToBlt() throws Exception {
+        checkDeployWithNodeAddedToBlt(false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployFromNodeRemovedFromBlt() throws Exception {
+        checkDeployFromNodeRemovedFromBlt(true, false);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployFromNodeRemovedFromBltStatic() throws Exception {
+        checkDeployFromNodeRemovedFromBlt(true, true);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeployToNodeRemovedFromBlt() throws Exception {
+        checkDeployFromNodeRemovedFromBlt(false, false);
+    }
+
+    /**
+     * @param persistence If {@code true}, then persistence will be enabled.
+     * @param staticDeploy If {@code true}, then static deployment will be used instead of a dynamic one.
+     * @throws Exception If failed.
+     */
+    private void checkDeploymentFromOutsideNode(boolean persistence, boolean staticDeploy) throws Exception {
+        this.persistence = persistence;
+
+        Ignite insideNode = startGrid(0);
+
+        if (persistence)
+            insideNode.cluster().active(true);
+        else {
+            IgniteCluster cluster = insideNode.cluster();
+
+            cluster.setBaselineTopology(cluster.topologyVersion());
+        }
+
+        CountDownLatch exeLatch = new CountDownLatch(1);
+
+        DummyService.exeLatch(SERVICE_NAME, exeLatch);
+
+        deployServiceFromNewNode(staticDeploy);
+
+        assertTrue(exeLatch.await(10, TimeUnit.SECONDS));
+    }
+
+    /**
+     * @param from If {@code true}, then added node will be an initiator of deployment.
+     * Otherwise deployment <b>to</b> this node will be tested.
+     * @throws Exception If failed.
+     */
+    private void checkDeployWithNodeAddedToBlt(boolean from) throws Exception {
+        persistence = true;
+
+        Ignite insideNode = startGrid(0);
+
+        IgniteCluster cluster = insideNode.cluster();
+
+        cluster.active(true);
+
+        Ignite outsideNode = startGrid(1);
+
+        cluster.setBaselineTopology(cluster.topologyVersion());
+
+        CountDownLatch exeLatch = new CountDownLatch(from ? 1 : 2);
+
+        DummyService.exeLatch(SERVICE_NAME, exeLatch);
+
+        if (from) {
+            IgniteFuture<Void> depFut = outsideNode.services().deployClusterSingletonAsync(SERVICE_NAME, new DummyService());
+
+            depFut.get(10, TimeUnit.SECONDS);
+        }
+        else {
+            IgniteFuture<Void> depFut = outsideNode.services().deployNodeSingletonAsync(SERVICE_NAME, new DummyService());
+
+            depFut.get(10, TimeUnit.SECONDS);
+        }
+
+        assertTrue(exeLatch.await(10, TimeUnit.SECONDS));
+    }
+
+    /**
+     * @param from If {@code true}, then added node will be an initiator of deployment.
+     * Otherwise deployment <b>to</b> this node will be tested.
+     * @param staticDeploy If {@code true}, then static deployment will be used instead of a dynamic one.
+     * @throws Exception If failed.
+     */
+    private void checkDeployFromNodeRemovedFromBlt(boolean from, boolean staticDeploy) throws Exception {
+        persistence = true;
+
+        Ignite insideNode = startGrid(0);
+        startGrid(1);
+
+        IgniteCluster cluster = insideNode.cluster();
+
+        cluster.active(true);
+
+        stopGrid(1);
+
+        cluster.setBaselineTopology(cluster.topologyVersion());
+
+        CountDownLatch exeLatch = new CountDownLatch(from ? 1 : 2);
+
+        DummyService.exeLatch(SERVICE_NAME, exeLatch);
+
+        if (from)
+            deployServiceFromNewNode(staticDeploy);
+        else {
+            startGrid(1);
+
+            IgniteFuture<Void> depFut = insideNode.services().deployNodeSingletonAsync(SERVICE_NAME, new DummyService());
+
+            depFut.get(10, TimeUnit.SECONDS);
+        }
+
+        assertTrue(exeLatch.await(10, TimeUnit.SECONDS));
+    }
+
+    /**
+     * @param staticDeploy If {@code true}, then static deployment will be used instead of a dynamic one.
+     * @throws Exception If node failed to start.
+     */
+    private void deployServiceFromNewNode(boolean staticDeploy) throws Exception {
+        if (staticDeploy) {
+            srvcCfg = getClusterSingletonServiceConfiguration();
+
+            startGrid(1);
+        }
+        else {
+            Ignite node = startGrid(1);
+
+            IgniteFuture<Void> depFut = node.services().deployClusterSingletonAsync(SERVICE_NAME, new DummyService());
+
+            depFut.get(10, TimeUnit.SECONDS);
+        }
+    }
+
+    /**
+     * @return Test service configuration.
+     */
+    private ServiceConfiguration getClusterSingletonServiceConfiguration() {
+        ServiceConfiguration srvcCfg = new ServiceConfiguration();
+        srvcCfg.setName(SERVICE_NAME);
+        srvcCfg.setService(new DummyService());
+        srvcCfg.setTotalCount(1);
+
+        return srvcCfg;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 5306aebd7611a..48f10a9b2c22a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -69,6 +69,7 @@
 import org.apache.ignite.internal.processors.service.IgniteServiceDynamicCachesSelfTest;
 import org.apache.ignite.internal.processors.service.IgniteServiceProxyTimeoutInitializedTest;
 import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
+import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest;
 import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
 import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
 import org.apache.ignite.services.ServiceThreadPoolSelfTest;
@@ -148,6 +149,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(ServiceThreadPoolSelfTest.class);
         suite.addTestSuite(GridServiceProcessorBatchDeploySelfTest.class);
         suite.addTestSuite(GridServiceDeploymentCompoundFutureSelfTest.class);
+        suite.addTestSuite(ServiceDeploymentOutsideBaselineTest.class);
 
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingJdkMarshallerTest.class);

From 2528435ec946c479f6bf4eb7eaeabf092a0536a3 Mon Sep 17 00:00:00 2001
From: devozerov <ppozerov@gmail.com>
Date: Tue, 31 Jul 2018 17:15:49 +0300
Subject: [PATCH 274/543] IGNITE-9114: SQL: use query time in retry timeout
 calculation. This closes #4460.

---
 .../query/h2/twostep/GridReduceQueryExecutor.java      | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index d778fccc30c1d..39b2bbcfc09ad 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -114,7 +114,7 @@
  */
 public class GridReduceQueryExecutor {
     /** Fail query after 10 seconds of unsuccessful attempts to reserve partitions. */
-    public static final long DFLT_RETRY_TIMEOUT = 10_000L;
+    public static final long DFLT_RETRY_TIMEOUT = 30_000L;
 
     /** */
     private static final String MERGE_INDEX_UNSORTED = "merge_scan";
@@ -568,7 +568,7 @@ public Iterator<List<?>> query(
 
         final boolean isReplicatedOnly = qry.isReplicatedOnly();
 
-        long retryTimeout = retryTimeout();
+        long retryTimeout = retryTimeout(timeoutMillis);
 
         final long startTime = U.currentTimeMillis();
 
@@ -1756,9 +1756,13 @@ private Map<ClusterNode, IntArray> narrowForQuery(Map<ClusterNode, IntArray> par
     }
 
     /**
+     * @param qryTimeout Query timeout.
      * @return Query retry timeout.
      */
-    private static long retryTimeout() {
+    private static long retryTimeout(long qryTimeout) {
+        if (qryTimeout > 0)
+            return qryTimeout;
+
         return IgniteSystemProperties.getLong(IGNITE_SQL_RETRY_TIMEOUT, DFLT_RETRY_TIMEOUT);
     }
 

From c78a7f215699f65983edfcc0c9014d9e3108f381 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Wed, 1 Aug 2018 12:23:03 +0300
Subject: [PATCH 275/543] IGNITE-8757 idle_verify utility doesn't show both
 update counter and hash conflicts

(cherry picked from commit 9131e4d)
---
 .../internal/commandline/CommandHandler.java  |  95 +++++
 .../cache/verify/IdleVerifyResultV2.java      | 113 ++++++
 .../cache/verify/PartitionHashRecordV2.java   | 168 +++++++++
 .../cache/verify/PartitionKeyV2.java          | 127 +++++++
 .../verify/VerifyBackupPartitionsTask.java    |  16 +-
 .../verify/VerifyBackupPartitionsTaskV2.java  | 347 ++++++++++++++++++
 .../visor/verify/VisorIdleVerifyTask.java     |   2 +
 .../visor/verify/VisorIdleVerifyTaskV2.java   |  94 +++++
 .../resources/META-INF/classnames.properties  |   1 +
 .../core/src/main/resources/ignite.properties |   2 +-
 .../junits/common/GridCommonAbstractTest.java |  17 +
 .../ignite/util/GridCommandHandlerTest.java   | 140 ++++++-
 12 files changed, 1107 insertions(+), 15 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKeyV2.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskV2.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 69811e1cfc8d2..ca14e080cadb5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -34,6 +34,7 @@
 import java.util.stream.Collectors;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.client.GridClient;
 import org.apache.ignite.internal.client.GridClientAuthenticationException;
 import org.apache.ignite.internal.client.GridClientClosedException;
@@ -51,8 +52,12 @@
 import org.apache.ignite.internal.commandline.cache.CacheCommand;
 import org.apache.ignite.internal.processors.cache.verify.CacheInfo;
 import org.apache.ignite.internal.processors.cache.verify.ContentionInfo;
+import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
 import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2;
+import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -82,6 +87,7 @@
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskV2;
 import org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult;
 import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskResult;
@@ -89,6 +95,7 @@
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskResult;
 import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
 
@@ -710,10 +717,39 @@ private void cacheView(GridClient client, CacheArguments cacheArgs) throws GridC
     }
 
     /**
+     * Executes appropriate version of idle_verify check. Old version will be used if there are old nodes in the cluster.
+     *
      * @param client Client.
      * @param cacheArgs Cache args.
      */
     private void cacheIdleVerify(GridClient client, CacheArguments cacheArgs) throws GridClientException {
+        Collection<GridClientNode> nodes = client.compute().nodes(GridClientNode::connectable);
+
+        boolean idleVerifyV2 = true;
+
+        for (GridClientNode node : nodes) {
+            String nodeVerStr = node.attribute(IgniteNodeAttributes.ATTR_BUILD_VER);
+
+            IgniteProductVersion nodeVer = IgniteProductVersion.fromString(nodeVerStr);
+
+            if (nodeVer.compareTo(VerifyBackupPartitionsTaskV2.V2_SINCE_VER) < 0) {
+                idleVerifyV2 = false;
+
+                break;
+            }
+        }
+
+        if (idleVerifyV2)
+            cacheIdleVerifyV2(client, cacheArgs);
+        else
+            legacyCacheIdleVerify(client, cacheArgs);
+    }
+
+    /**
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     */
+    private void legacyCacheIdleVerify(GridClient client, CacheArguments cacheArgs) throws GridClientException {
         VisorIdleVerifyTaskResult res = executeTask(
             client, VisorIdleVerifyTask.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
 
@@ -735,6 +771,65 @@ private void cacheIdleVerify(GridClient client, CacheArguments cacheArgs) throws
         }
     }
 
+    /**
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     */
+    private void cacheIdleVerifyV2(GridClient client, CacheArguments cacheArgs) throws GridClientException {
+        IdleVerifyResultV2 res = executeTask(
+            client, VisorIdleVerifyTaskV2.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
+
+        if (!res.hasConflicts()) {
+            log("idle_verify check has finished, no conflicts have been found.");
+            nl();
+        }
+        else {
+            int cntrConflictsSize = res.counterConflicts().size();
+            int hashConflictsSize = res.hashConflicts().size();
+
+            log("idle_verify check has finished, found " + (cntrConflictsSize + hashConflictsSize) +
+                " conflict partitions: [counterConflicts=" + cntrConflictsSize + ", hashConflicts=" +
+                hashConflictsSize + "]");
+            nl();
+
+            if (!F.isEmpty(res.counterConflicts())) {
+                log("Update counter conflicts:");
+
+                for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : res.counterConflicts().entrySet()) {
+                    log("Conflict partition: " + entry.getKey());
+
+                    log("Partition instances: " + entry.getValue());
+                }
+
+                nl();
+            }
+
+            if (!F.isEmpty(res.hashConflicts())) {
+                log("Hash conflicts:");
+
+                for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : res.hashConflicts().entrySet()) {
+                    log("Conflict partition: " + entry.getKey());
+
+                    log("Partition instances: " + entry.getValue());
+                }
+
+                nl();
+            }
+        }
+
+        if (!F.isEmpty(res.movingPartitions())) {
+            log("Verification was skipped for " + res.movingPartitions().size() + " MOVING partitions:");
+
+            for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : res.movingPartitions().entrySet()) {
+                log("Rebalancing partition: " + entry.getKey());
+
+                log("Partition instances: " + entry.getValue());
+            }
+
+            nl();
+        }
+    }
+
     /**
      * Change baseline.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
new file mode 100644
index 0000000000000..d5815cd2b02e1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
@@ -0,0 +1,113 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Encapsulates result of {@link VerifyBackupPartitionsTaskV2}.
+ */
+public class IdleVerifyResultV2 extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Counter conflicts. */
+    private Map<PartitionKeyV2, List<PartitionHashRecordV2>> cntrConflicts;
+
+    /** Hash conflicts. */
+    private Map<PartitionKeyV2, List<PartitionHashRecordV2>> hashConflicts;
+
+    /** Moving partitions. */
+    private Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions;
+
+    /**
+     * @param cntrConflicts Counter conflicts.
+     * @param hashConflicts Hash conflicts.
+     * @param movingPartitions Moving partitions.
+     */
+    public IdleVerifyResultV2(
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> cntrConflicts,
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> hashConflicts,
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions
+    ) {
+        this.cntrConflicts = cntrConflicts;
+        this.hashConflicts = hashConflicts;
+        this.movingPartitions = movingPartitions;
+    }
+
+    /**
+     * Default constructor for Externalizable.
+     */
+    public IdleVerifyResultV2() {
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, cntrConflicts);
+        U.writeMap(out, hashConflicts);
+        U.writeMap(out, movingPartitions);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer,
+        ObjectInput in) throws IOException, ClassNotFoundException {
+        cntrConflicts = U.readMap(in);
+        hashConflicts = U.readMap(in);
+        movingPartitions = U.readMap(in);
+    }
+
+    /**
+     * @return Counter conflicts.
+     */
+    public Map<PartitionKeyV2, List<PartitionHashRecordV2>> counterConflicts() {
+        return cntrConflicts;
+    }
+
+    /**
+     * @return Hash conflicts.
+     */
+    public Map<PartitionKeyV2, List<PartitionHashRecordV2>> hashConflicts() {
+        return hashConflicts;
+    }
+
+    /**
+     * @return Moving partitions.
+     */
+    public Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions() {
+        return movingPartitions;
+    }
+
+    /**
+     * @return <code>true</code> if any conflicts were discovered during idle_verify check.
+     */
+    public boolean hasConflicts() {
+        return !F.isEmpty(hashConflicts()) || !F.isEmpty(counterConflicts());
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IdleVerifyResultV2.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
new file mode 100644
index 0000000000000..c0f8121057f10
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionHashRecordV2.java
@@ -0,0 +1,168 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Record containing partition checksum, primary flag and consistent ID of owner.
+ */
+public class PartitionHashRecordV2 extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Moving partition size. */
+    public static final long MOVING_PARTITION_SIZE = Long.MIN_VALUE;
+
+    /** Partition key. */
+    @GridToStringExclude
+    private PartitionKeyV2 partKey;
+
+    /** Is primary flag. */
+    private boolean isPrimary;
+
+    /** Consistent id. */
+    @GridToStringInclude
+    private Object consistentId;
+
+    /** Partition hash. */
+    @GridToStringExclude
+    private int partHash;
+
+    /** Update counter. */
+    private long updateCntr;
+
+    /** Size. */
+    @GridToStringExclude
+    private long size;
+
+    /**
+     * @param partKey Partition key.
+     * @param isPrimary Is primary.
+     * @param consistentId Consistent id.
+     * @param partHash Partition hash.
+     * @param updateCntr Update counter.
+     * @param size Size.
+     */
+    public PartitionHashRecordV2(PartitionKeyV2 partKey, boolean isPrimary,
+        Object consistentId, int partHash, long updateCntr, long size) {
+        this.partKey = partKey;
+        this.isPrimary = isPrimary;
+        this.consistentId = consistentId;
+        this.partHash = partHash;
+        this.updateCntr = updateCntr;
+        this.size = size;
+    }
+
+    /**
+     * Default constructor for Externalizable.
+     */
+    public PartitionHashRecordV2() {
+    }
+
+    /**
+     * @return Partition key.
+     */
+    public PartitionKeyV2 partitionKey() {
+        return partKey;
+    }
+
+    /**
+     * @return Is primary.
+     */
+    public boolean isPrimary() {
+        return isPrimary;
+    }
+
+    /**
+     * @return Consistent id.
+     */
+    public Object consistentId() {
+        return consistentId;
+    }
+
+    /**
+     * @return Partition hash.
+     */
+    public int partitionHash() {
+        return partHash;
+    }
+
+    /**
+     * @return Update counter.
+     */
+    public long updateCounter() {
+        return updateCntr;
+    }
+
+    /**
+     * @return Size.
+     */
+    public long size() {
+        return size;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        out.writeObject(partKey);
+        out.writeBoolean(isPrimary);
+        out.writeObject(consistentId);
+        out.writeInt(partHash);
+        out.writeLong(updateCntr);
+        out.writeLong(size);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        partKey = (PartitionKeyV2)in.readObject();
+        isPrimary = in.readBoolean();
+        consistentId = in.readObject();
+        partHash = in.readInt();
+        updateCntr = in.readLong();
+        size = in.readLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return size == MOVING_PARTITION_SIZE ?
+            S.toString(PartitionHashRecordV2.class, this, "state", "MOVING") :
+            S.toString(PartitionHashRecordV2.class, this, "size", size, "partHash", partHash);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        PartitionHashRecordV2 record = (PartitionHashRecordV2)o;
+
+        return consistentId.equals(record.consistentId);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        return consistentId.hashCode();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKeyV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKeyV2.java
new file mode 100644
index 0000000000000..1332b0a0e3057
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/PartitionKeyV2.java
@@ -0,0 +1,127 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Partition key - pair of cache group ID and partition ID.
+ */
+public class PartitionKeyV2 extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Group id. */
+    private int grpId;
+
+    /** Group name. Optional field, used only for output. */
+    private volatile String grpName;
+
+    /** Partition id. */
+    private int partId;
+
+    /**
+     * @param grpId Group id.
+     * @param partId Partition id.
+     * @param grpName Group name.
+     */
+    public PartitionKeyV2(int grpId, int partId, String grpName) {
+        this.grpId = grpId;
+        this.partId = partId;
+        this.grpName = grpName;
+    }
+
+    /**
+     * Default constructor for Externalizable.
+     */
+    public PartitionKeyV2() {
+    }
+
+    /**
+     * @return Group id.
+     */
+    public int groupId() {
+        return grpId;
+    }
+
+    /**
+     * @return Partition id.
+     */
+    public int partitionId() {
+        return partId;
+    }
+
+    /**
+     * @return Group name.
+     */
+    public String groupName() {
+        return grpName;
+    }
+
+    /**
+     * @param grpName Group name.
+     */
+    public void groupName(String grpName) {
+        this.grpName = grpName;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        out.writeInt(grpId);
+        U.writeString(out, grpName);
+        out.writeInt(partId);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        grpId = in.readInt();
+        grpName = U.readString(in);
+        partId = in.readInt();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        PartitionKeyV2 key = (PartitionKeyV2)o;
+
+        return grpId == key.grpId && partId == key.partId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        int res = grpId;
+
+        res = 31 * res + partId;
+
+        return res;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(PartitionKeyV2.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
index b884cb01ac934..c99681864de0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
@@ -67,8 +67,11 @@
  * <br>
  * Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
  * concurrently updated.
+ *
+ * @deprecated Legacy version of {@link VerifyBackupPartitionsTaskV2}.
  */
 @GridInternal
+@Deprecated
 public class VerifyBackupPartitionsTask extends ComputeTaskAdapter<Set<String>,
     Map<PartitionKey, List<PartitionHashRecord>>> {
     /** */
@@ -154,9 +157,10 @@ public class VerifyBackupPartitionsTask extends ComputeTaskAdapter<Set<String>,
     }
 
     /**
-     *
+     * Legacy version of {@link VerifyBackupPartitionsTaskV2} internal job, kept for compatibility.
      */
-    public static class VerifyBackupPartitionsJob extends ComputeJobAdapter {
+    @Deprecated
+    private static class VerifyBackupPartitionsJob extends ComputeJobAdapter {
         /** */
         private static final long serialVersionUID = 0L;
 
@@ -177,7 +181,7 @@ public static class VerifyBackupPartitionsJob extends ComputeJobAdapter {
         /**
          * @param names Names.
          */
-        private VerifyBackupPartitionsJob(Set<String> names) {
+        public VerifyBackupPartitionsJob(Set<String> names) {
             cacheNames = names;
         }
 
@@ -244,7 +248,7 @@ private VerifyBackupPartitionsJob(Set<String> names) {
                 Future<Map<PartitionKey, PartitionHashRecord>> fut = partHashCalcFutures.get(i);
 
                 try {
-                    Map<PartitionKey, PartitionHashRecord> partHash = fut.get(10, TimeUnit.SECONDS);
+                    Map<PartitionKey, PartitionHashRecord> partHash = fut.get(100, TimeUnit.MILLISECONDS);
 
                     res.putAll(partHash);
 
@@ -261,7 +265,7 @@ else if (e.getCause() instanceof IgniteException)
                     else
                         throw new IgniteException(e.getCause());
                 }
-                catch (TimeoutException e) {
+                catch (TimeoutException ignored) {
                     if (U.currentTimeMillis() - lastProgressLogTs > 3 * 60 * 1000L) {
                         lastProgressLogTs = U.currentTimeMillis();
 
@@ -289,7 +293,6 @@ private Future<Map<PartitionKey, PartitionHashRecord>> calculatePartitionHashAsy
             });
         }
 
-
         /**
          * @param grpCtx Group context.
          * @param part Local partition.
@@ -355,5 +358,4 @@ private Map<PartitionKey, PartitionHashRecord> calculatePartitionHash(
             return Collections.singletonMap(partKey, partRec);
         }
     }
-
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
new file mode 100644
index 0000000000000..826393ac09602
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
@@ -0,0 +1,347 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteInterruptedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeTaskAdapter;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.lang.GridIterator;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task for comparing update counters and checksums between primary and backup partitions of specified caches.
+ * <br>
+ * Argument: Set of cache names, 'null' will trigger verification for all caches.
+ * <br>
+ * Result: {@link IdleVerifyResultV2} with conflict partitions.
+ * <br>
+ * Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
+ * concurrently updated.
+ */
+@GridInternal
+public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVerifyTaskArg, IdleVerifyResultV2> {
+    /** First version of Ignite that is capable of executing Idle Verify V2. */
+    public static final IgniteProductVersion V2_SINCE_VER = IgniteProductVersion.fromString("2.5.3");
+
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
+        List<ClusterNode> subgrid, VisorIdleVerifyTaskArg arg) throws IgniteException {
+        Map<ComputeJob, ClusterNode> jobs = new HashMap<>();
+
+        for (ClusterNode node : subgrid)
+            jobs.put(new VerifyBackupPartitionsJobV2(arg), node);
+
+        return jobs;
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public IdleVerifyResultV2 reduce(List<ComputeJobResult> results)
+        throws IgniteException {
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes = new HashMap<>();
+
+        for (ComputeJobResult res : results) {
+            Map<PartitionKeyV2, PartitionHashRecordV2> nodeHashes = res.getData();
+
+            for (Map.Entry<PartitionKeyV2, PartitionHashRecordV2> e : nodeHashes.entrySet()) {
+                List<PartitionHashRecordV2> records = clusterHashes.computeIfAbsent(e.getKey(), k -> new ArrayList<>());
+
+                records.add(e.getValue());
+            }
+        }
+
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> hashConflicts = new HashMap<>();
+
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> updateCntrConflicts = new HashMap<>();
+
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingParts = new HashMap<>();
+
+        for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> e : clusterHashes.entrySet()) {
+            Integer partHash = null;
+            Long updateCntr = null;
+
+            for (PartitionHashRecordV2 record : e.getValue()) {
+                if (record.size() == PartitionHashRecordV2.MOVING_PARTITION_SIZE) {
+                    List<PartitionHashRecordV2> records = movingParts.computeIfAbsent(
+                        e.getKey(), k -> new ArrayList<>());
+
+                    records.add(record);
+
+                    continue;
+                }
+
+                if (partHash == null) {
+                    partHash = record.partitionHash();
+
+                    updateCntr = record.updateCounter();
+                }
+                else {
+                    if (record.updateCounter() != updateCntr)
+                        updateCntrConflicts.putIfAbsent(e.getKey(), e.getValue());
+
+                    if (record.partitionHash() != partHash)
+                        hashConflicts.putIfAbsent(e.getKey(), e.getValue());
+                }
+            }
+        }
+
+        return new IdleVerifyResultV2(updateCntrConflicts, hashConflicts, movingParts);
+    }
+
+    /**
+     * Job that collects update counters and hashes of local partitions.
+     */
+    private static class VerifyBackupPartitionsJobV2 extends ComputeJobAdapter {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** Ignite instance. */
+        @IgniteInstanceResource
+        private IgniteEx ignite;
+
+        /** Injected logger. */
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** Idle verify arguments. */
+        private VisorIdleVerifyTaskArg arg;
+
+        /** Counter of processed partitions. */
+        private final AtomicInteger completionCntr = new AtomicInteger(0);
+
+        /**
+         * @param arg Argument.
+         */
+        public VerifyBackupPartitionsJobV2(VisorIdleVerifyTaskArg arg) {
+            this.arg = arg;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Map<PartitionKeyV2, PartitionHashRecordV2> execute() throws IgniteException {
+            Set<Integer> grpIds = new HashSet<>();
+
+            Set<String> missingCaches = new HashSet<>();
+
+            if (arg.getCaches() != null) {
+                for (String cacheName : arg.getCaches()) {
+                    DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName);
+
+                    if (desc == null) {
+                        missingCaches.add(cacheName);
+
+                        continue;
+                    }
+
+                    grpIds.add(desc.groupId());
+                }
+
+                if (!missingCaches.isEmpty()) {
+                    StringBuilder strBuilder = new StringBuilder("The following caches do not exist: ");
+
+                    for (String name : missingCaches)
+                        strBuilder.append(name).append(", ");
+
+                    strBuilder.delete(strBuilder.length() - 2, strBuilder.length());
+
+                    throw new IgniteException(strBuilder.toString());
+                }
+            }
+            else {
+                Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
+
+                for (CacheGroupContext grp : groups) {
+                    if (!grp.systemCache() && !grp.isLocal())
+                        grpIds.add(grp.groupId());
+                }
+            }
+
+            List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> partHashCalcFutures = new ArrayList<>();
+
+            completionCntr.set(0);
+
+            for (Integer grpId : grpIds) {
+                CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
+
+                if (grpCtx == null)
+                    continue;
+
+                List<GridDhtLocalPartition> parts = grpCtx.topology().localPartitions();
+
+                for (GridDhtLocalPartition part : parts)
+                    partHashCalcFutures.add(calculatePartitionHashAsync(grpCtx, part));
+            }
+
+            Map<PartitionKeyV2, PartitionHashRecordV2> res = new HashMap<>();
+
+            long lastProgressLogTs = U.currentTimeMillis();
+
+            for (int i = 0; i < partHashCalcFutures.size(); ) {
+                Future<Map<PartitionKeyV2, PartitionHashRecordV2>> fut = partHashCalcFutures.get(i);
+
+                try {
+                    Map<PartitionKeyV2, PartitionHashRecordV2> partHash = fut.get(100, TimeUnit.MILLISECONDS);
+
+                    res.putAll(partHash);
+
+                    i++;
+                }
+                catch (InterruptedException | ExecutionException e) {
+                    for (int j = i + 1; j < partHashCalcFutures.size(); j++)
+                        partHashCalcFutures.get(j).cancel(false);
+
+                    if (e instanceof InterruptedException)
+                        throw new IgniteInterruptedException((InterruptedException)e);
+                    else if (e.getCause() instanceof IgniteException)
+                        throw (IgniteException)e.getCause();
+                    else
+                        throw new IgniteException(e.getCause());
+                }
+                catch (TimeoutException ignored) {
+                    if (U.currentTimeMillis() - lastProgressLogTs > 3 * 60 * 1000L) {
+                        lastProgressLogTs = U.currentTimeMillis();
+
+                        log.warning("idle_verify is still running, processed " + completionCntr.get() + " of " +
+                            partHashCalcFutures.size() + " local partitions");
+                    }
+                }
+            }
+
+            return res;
+        }
+
+        /**
+         * @param grpCtx Group context.
+         * @param part Local partition.
+         */
+        private Future<Map<PartitionKeyV2, PartitionHashRecordV2>> calculatePartitionHashAsync(
+            final CacheGroupContext grpCtx,
+            final GridDhtLocalPartition part
+        ) {
+            return ForkJoinPool.commonPool().submit(new Callable<Map<PartitionKeyV2, PartitionHashRecordV2>>() {
+                @Override public Map<PartitionKeyV2, PartitionHashRecordV2> call() throws Exception {
+                    return calculatePartitionHash(grpCtx, part);
+                }
+            });
+        }
+
+
+        /**
+         * @param grpCtx Group context.
+         * @param part Local partition.
+         */
+        private Map<PartitionKeyV2, PartitionHashRecordV2> calculatePartitionHash(
+            CacheGroupContext grpCtx,
+            GridDhtLocalPartition part
+        ) {
+            if (!part.reserve())
+                return Collections.emptyMap();
+
+            int partHash = 0;
+            long partSize;
+            long updateCntrBefore = part.updateCounter();
+
+            PartitionKeyV2 partKey = new PartitionKeyV2(grpCtx.groupId(), part.id(), grpCtx.cacheOrGroupName());
+
+            Object consId = ignite.context().discovery().localNode().consistentId();
+
+            boolean isPrimary = part.primary(grpCtx.topology().readyTopologyVersion());
+
+            try {
+                if (part.state() == GridDhtPartitionState.MOVING) {
+                    PartitionHashRecordV2 movingHashRecord = new PartitionHashRecordV2(partKey, isPrimary, consId,
+                        partHash, updateCntrBefore, PartitionHashRecordV2.MOVING_PARTITION_SIZE);
+
+                    return Collections.singletonMap(partKey, movingHashRecord);
+                }
+                else if (part.state() != GridDhtPartitionState.OWNING)
+                    return Collections.emptyMap();
+
+                partSize = part.dataStore().fullSize();
+
+                GridIterator<CacheDataRow> it = grpCtx.offheap().partitionIterator(part.id());
+
+                while (it.hasNextX()) {
+                    CacheDataRow row = it.nextX();
+
+                    partHash += row.key().hashCode();
+
+                    partHash += Arrays.hashCode(row.value().valueBytes(grpCtx.cacheObjectContext()));
+                }
+
+                long updateCntrAfter = part.updateCounter();
+
+                if (updateCntrBefore != updateCntrAfter) {
+                    throw new IgniteException("Cluster is not idle: update counter of partition [grpId=" +
+                        grpCtx.groupId() + ", partId=" + part.id() + "] changed during hash calculation [before=" +
+                        updateCntrBefore + ", after=" + updateCntrAfter + "]");
+                }
+            }
+            catch (IgniteCheckedException e) {
+                U.error(log, "Can't calculate partition hash [grpId=" + grpCtx.groupId() +
+                    ", partId=" + part.id() + "]", e);
+
+                return Collections.emptyMap();
+            }
+            finally {
+                part.release();
+            }
+
+            PartitionHashRecordV2 partRec = new PartitionHashRecordV2(
+                partKey, isPrimary, consId, partHash, updateCntrBefore, partSize);
+
+            completionCntr.incrementAndGet();
+
+            return Collections.singletonMap(partKey, partRec);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTask.java
index 05f2621310366..e67c681acea45 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTask.java
@@ -37,6 +37,7 @@
  * Task to verify checksums of backup partitions.
  */
 @GridInternal
+@Deprecated
 public class VisorIdleVerifyTask extends VisorOneNodeTask<VisorIdleVerifyTaskArg, VisorIdleVerifyTaskResult> {
     /** */
     private static final long serialVersionUID = 0L;
@@ -49,6 +50,7 @@ public class VisorIdleVerifyTask extends VisorOneNodeTask<VisorIdleVerifyTaskArg
     /**
      *
      */
+    @Deprecated
     private static class VisorIdleVerifyJob extends VisorJob<VisorIdleVerifyTaskArg, VisorIdleVerifyTaskResult> {
         /** */
         private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskV2.java
new file mode 100644
index 0000000000000..b9250ef28c543
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskV2.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobContext;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
+import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.resources.JobContextResource;
+
+/**
+ * Task to verify checksums of backup partitions.
+ */
+@GridInternal
+public class VisorIdleVerifyTaskV2 extends VisorOneNodeTask<VisorIdleVerifyTaskArg, IdleVerifyResultV2> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<VisorIdleVerifyTaskArg, IdleVerifyResultV2> job(VisorIdleVerifyTaskArg arg) {
+        return new VisorIdleVerifyJobV2(arg, debug);
+    }
+
+    /**
+     *
+     */
+    private static class VisorIdleVerifyJobV2 extends VisorJob<VisorIdleVerifyTaskArg, IdleVerifyResultV2> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        private ComputeTaskFuture<IdleVerifyResultV2> fut;
+
+        /** Auto-inject job context. */
+        @JobContextResource
+        protected transient ComputeJobContext jobCtx;
+
+        /**
+         * @param arg Argument.
+         * @param debug Debug.
+         */
+        private VisorIdleVerifyJobV2(VisorIdleVerifyTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected IdleVerifyResultV2 run(VisorIdleVerifyTaskArg arg) throws IgniteException {
+            if (fut == null) {
+                fut = ignite.compute().executeAsync(VerifyBackupPartitionsTaskV2.class, arg);
+
+                if (!fut.isDone()) {
+                    jobCtx.holdcc();
+
+                    fut.listen(new IgniteInClosure<IgniteFuture<IdleVerifyResultV2>>() {
+                        @Override public void apply(IgniteFuture<IdleVerifyResultV2> f) {
+                            jobCtx.callcc();
+                        }
+                    });
+
+                    return null;
+                }
+            }
+
+            return fut.get();
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorIdleVerifyJobV2.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index f1aec976e4c53..6c91de43700cb 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -1138,6 +1138,7 @@ org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysT
 org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysTask$CollectPartitionEntryHashesJob
 org.apache.ignite.internal.processors.cache.verify.ContentionClosure
 org.apache.ignite.internal.processors.cache.verify.ContentionInfo
+org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2
 org.apache.ignite.internal.processors.cache.verify.PartitionEntryHashRecord
 org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord
 org.apache.ignite.internal.processors.cache.verify.PartitionKey
diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index e70660b1c6a64..785f962378054 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=2.5.0-SNAPSHOT
+ignite.version=2.6.0-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 38df0f488196d..29ad91d167576 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -790,6 +790,23 @@ protected void awaitPartitionMapExchange(
         log.info("awaitPartitionMapExchange finished");
     }
 
+    /**
+     * Compares checksums between primary and backup partitions of specified caches.
+     * Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
+     * concurrently updated.
+     *
+     * @param ig Ignite instance.
+     * @param cacheNames Cache names (if null, all user caches will be verified).
+     * @throws IgniteCheckedException If checksum conflict has been found.
+     */
+    protected void verifyBackupPartitions(Ignite ig, Set<String> cacheNames) throws IgniteCheckedException {
+        Map<PartitionKey, List<PartitionHashRecord>> conflicts = ig.compute().execute(
+            new VerifyBackupPartitionsTask(), cacheNames);
+
+        if (!conflicts.isEmpty())
+            throw new IgniteCheckedException("Conflict partitions: " + conflicts.keySet());
+    }
+
     /**
      * @param top Topology.
      * @param topVer Version to wait for.
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 1432171485a00..55c10345b12c4 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -17,6 +17,9 @@
 
 package org.apache.ignite.util;
 
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.PrintStream;
@@ -33,9 +36,6 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.LongAdder;
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.MutableEntry;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteAtomicSequence;
 import org.apache.ignite.IgniteCache;
@@ -59,18 +59,17 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheFuture;
-import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
+import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
-import org.apache.ignite.internal.processors.cache.GridCacheOperation;
-import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
@@ -883,6 +882,77 @@ public void testCacheIdleVerify() throws Exception {
         assertTrue(testOut.toString().contains("conflict partitions"));
     }
 
+    /**
+     * Tests that both update counter and hash conflicts are detected.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCacheIdleVerifyTwoConflictTypes() throws Exception {
+        IgniteEx ignite = (IgniteEx)startGrids(2);
+
+        ignite.cluster().active(true);
+
+        int parts = 32;
+
+        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, parts))
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME));
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, i);
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify"));
+
+        assertTrue(testOut.toString().contains("no conflicts have been found"));
+
+        GridCacheContext<Object, Object> cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context();
+
+        corruptDataEntry(cacheCtx, 1, true, false);
+
+        corruptDataEntry(cacheCtx, 1 + parts / 2, false, true);
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify"));
+
+        assertTrue(testOut.toString().contains("found 2 conflict partitions"));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCacheIdleVerifyMovingParts() throws Exception {
+        IgniteEx ignite = (IgniteEx)startGrids(2);
+
+        ignite.cluster().active(true);
+
+        int parts = 32;
+
+        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, parts))
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME)
+            .setRebalanceDelay(10_000));
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, i);
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify"));
+
+        assertTrue(testOut.toString().contains("no conflicts have been found"));
+
+        startGrid(2);
+
+        resetBaselineTopology();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify"));
+
+        assertTrue(testOut.toString().contains("MOVING partitions"));
+    }
+
     /**
      *
      */
@@ -1232,4 +1302,60 @@ private static class IncrementClosure implements EntryProcessor<Long, Long, Void
             return null;
         }
     }
+
+    /**
+     * Corrupts data entry.
+     *
+     * @param ctx Context.
+     * @param key Key.
+     * @param breakCntr Break counter.
+     * @param breakData Break data.
+     */
+    private void corruptDataEntry(
+        GridCacheContext<Object, Object> ctx,
+        int key,
+        boolean breakCntr,
+        boolean breakData
+    ) {
+        int partId = ctx.affinity().partition(key);
+
+        try {
+            long updateCntr = ctx.topology().localPartition(partId).updateCounter();
+
+            Object valToPut = ctx.cache().keepBinary().get(key);
+
+            if (breakCntr)
+                updateCntr++;
+
+            if (breakData)
+                valToPut = valToPut.toString() + " broken";
+
+            // Create data entry
+            DataEntry dataEntry = new DataEntry(
+                ctx.cacheId(),
+                new KeyCacheObjectImpl(key, null, partId),
+                new CacheObjectImpl(valToPut, null),
+                GridCacheOperation.UPDATE,
+                new GridCacheVersion(),
+                new GridCacheVersion(),
+                0L,
+                partId,
+                updateCntr
+            );
+
+            GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ctx.shared().database();
+
+            db.checkpointReadLock();
+
+            try {
+                U.invoke(GridCacheDatabaseSharedManager.class, db, "applyUpdate", ctx, dataEntry);
+            }
+            finally {
+                db.checkpointReadUnlock();
+            }
+        }
+        catch (IgniteCheckedException e) {
+            e.printStackTrace();
+        }
+    }
 }

From 66369583096b04d71c3f82e1ebfd1922ae6b0b6a Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Tue, 31 Jul 2018 16:13:27 +0300
Subject: [PATCH 276/543] IGNITE-8973 Calculate partition hash and print into
 standard output

Signed-off-by: Andrey Gura <agura@apache.org>

(cherry picked from commit 8309cef)
---
 .../internal/commandline/CommandHandler.java  | 162 ++++++------
 .../commandline/cache/CacheArguments.java     |  34 +++
 .../cache/verify/IdleVerifyDumpResult.java    |  73 ++++++
 .../cache/verify/IdleVerifyResultV2.java      |  85 +++++--
 .../VerifyBackupPartitionsDumpTask.java       | 230 ++++++++++++++++++
 .../visor/verify/VisorIdleVerifyDumpTask.java |  37 +++
 .../verify/VisorIdleVerifyDumpTaskArg.java    |  73 ++++++
 .../visor/verify/VisorIdleVerifyJob.java      |  83 +++++++
 .../resources/META-INF/classnames.properties  |   2 +
 .../ignite/util/GridCommandHandlerTest.java   | 165 ++++++++++++-
 10 files changed, 837 insertions(+), 107 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyDumpResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyJob.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index ca14e080cadb5..7b5ce44332a7a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -34,6 +34,7 @@
 import java.util.stream.Collectors;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeTask;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.client.GridClient;
 import org.apache.ignite.internal.client.GridClientAuthenticationException;
@@ -54,9 +55,7 @@
 import org.apache.ignite.internal.processors.cache.verify.ContentionInfo;
 import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
 import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
-import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
-import org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2;
 import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
@@ -84,6 +83,8 @@
 import org.apache.ignite.internal.visor.verify.VisorContentionTask;
 import org.apache.ignite.internal.visor.verify.VisorContentionTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorContentionTaskResult;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTask;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult;
@@ -154,8 +155,15 @@ public class CommandHandler {
     /** */
     protected static final String CMD_PING_TIMEOUT = "--ping-timeout";
 
+    /** */
+    private static final String CMD_DUMP = "--dump";
+
+    /** */
+    private static final String CMD_SKIP_ZEROS = "--skipZeros";
+
     /** List of optional auxiliary commands. */
     private static final Set<String> AUX_COMMANDS = new HashSet<>();
+
     static {
         AUX_COMMANDS.add(CMD_HELP);
         AUX_COMMANDS.add(CMD_HOST);
@@ -237,7 +245,7 @@ public class CommandHandler {
     private static final String TX_ORDER = "order";
 
     /** */
-    public static final String CMD_TX_ORDER_START_TIME="START_TIME";
+    public static final String CMD_TX_ORDER_START_TIME = "START_TIME";
 
     /** */
     private static final String TX_SERVERS = "servers";
@@ -463,14 +471,14 @@ private void state(GridClient client) throws Throwable {
     }
 
     /**
-     *
      * @param client Client.
      * @param taskCls Task class.
      * @param taskArgs Task arguments.
      * @return Task result.
      * @throws GridClientException If failed to execute task.
      */
-    private <R> R executeTask(GridClient client, Class<?> taskCls, Object taskArgs) throws GridClientException {
+    private <R> R executeTask(GridClient client, Class<? extends ComputeTask<?, R>> taskCls,
+        Object taskArgs) throws GridClientException {
         return executeTaskByNameOnNode(client, taskCls.getName(), taskArgs, null);
     }
 
@@ -595,7 +603,7 @@ private void printCacheHelp() {
 
         usage("  Show information about caches, groups or sequences that match a regex:", CACHE, " list regexPattern [groups|seq] [nodeId]");
         usage("  Show hot keys that are point of contention for multiple transactions:", CACHE, " contention minQueueSize [nodeId] [maxPrint]");
-        usage("  Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [cache1,...,cacheN]");
+        usage("  Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [--dump] [--skipZeros] [cache1,...,cacheN]");
         usage("  Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId] [checkFirst|checkThrough]");
 
         log("  If [nodeId] is not specified, contention and validate_indexes commands will be broadcasted to all server nodes.");
@@ -717,7 +725,8 @@ private void cacheView(GridClient client, CacheArguments cacheArgs) throws GridC
     }
 
     /**
-     * Executes appropriate version of idle_verify check. Old version will be used if there are old nodes in the cluster.
+     * Executes appropriate version of idle_verify check. Old version will be used if there are old nodes in the
+     * cluster.
      *
      * @param client Client.
      * @param cacheArgs Cache args.
@@ -739,7 +748,9 @@ private void cacheIdleVerify(GridClient client, CacheArguments cacheArgs) throws
             }
         }
 
-        if (idleVerifyV2)
+        if (cacheArgs.dump())
+            cacheIdleVerifyDump(client, cacheArgs);
+        else if (idleVerifyV2)
             cacheIdleVerifyV2(client, cacheArgs);
         else
             legacyCacheIdleVerify(client, cacheArgs);
@@ -760,7 +771,7 @@ private void legacyCacheIdleVerify(GridClient client, CacheArguments cacheArgs)
             nl();
         }
         else {
-            log ("idle_verify check has finished, found " + conflicts.size() + " conflict partitions.");
+            log("idle_verify check has finished, found " + conflicts.size() + " conflict partitions.");
             nl();
 
             for (Map.Entry<PartitionKey, List<PartitionHashRecord>> entry : conflicts.entrySet()) {
@@ -771,6 +782,20 @@ private void legacyCacheIdleVerify(GridClient client, CacheArguments cacheArgs)
         }
     }
 
+    /**
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     */
+    private void cacheIdleVerifyDump(GridClient client, CacheArguments cacheArgs) throws GridClientException {
+        String path = executeTask(
+            client,
+            VisorIdleVerifyDumpTask.class,
+            new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros())
+        );
+
+        log("VisorIdleVerifyDumpTask successfully written output to '" + path + "'");
+    }
+
     /**
      * @param client Client.
      * @param cacheArgs Cache args.
@@ -779,62 +804,15 @@ private void cacheIdleVerifyV2(GridClient client, CacheArguments cacheArgs) thro
         IdleVerifyResultV2 res = executeTask(
             client, VisorIdleVerifyTaskV2.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
 
-        if (!res.hasConflicts()) {
-            log("idle_verify check has finished, no conflicts have been found.");
-            nl();
-        }
-        else {
-            int cntrConflictsSize = res.counterConflicts().size();
-            int hashConflictsSize = res.hashConflicts().size();
-
-            log("idle_verify check has finished, found " + (cntrConflictsSize + hashConflictsSize) +
-                " conflict partitions: [counterConflicts=" + cntrConflictsSize + ", hashConflicts=" +
-                hashConflictsSize + "]");
-            nl();
-
-            if (!F.isEmpty(res.counterConflicts())) {
-                log("Update counter conflicts:");
-
-                for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : res.counterConflicts().entrySet()) {
-                    log("Conflict partition: " + entry.getKey());
-
-                    log("Partition instances: " + entry.getValue());
-                }
-
-                nl();
-            }
-
-            if (!F.isEmpty(res.hashConflicts())) {
-                log("Hash conflicts:");
-
-                for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : res.hashConflicts().entrySet()) {
-                    log("Conflict partition: " + entry.getKey());
-
-                    log("Partition instances: " + entry.getValue());
-                }
-
-                nl();
-            }
-        }
-
-        if (!F.isEmpty(res.movingPartitions())) {
-            log("Verification was skipped for " + res.movingPartitions().size() + " MOVING partitions:");
-
-            for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : res.movingPartitions().entrySet()) {
-                log("Rebalancing partition: " + entry.getKey());
-
-                log("Partition instances: " + entry.getValue());
-            }
-
-            nl();
-        }
+        res.print(System.out::print);
     }
 
     /**
      * Change baseline.
      *
      * @param client Client.
-     * @param baselineAct Baseline action to execute.  @throws GridClientException If failed to execute baseline action.
+     * @param baselineAct Baseline action to execute.  @throws GridClientException If failed to execute baseline
+     * action.
      * @param baselineArgs Baseline action arguments.
      * @throws Throwable If failed to execute baseline action.
      */
@@ -928,7 +906,7 @@ private void baselinePrint0(VisorBaselineTaskResult res) {
         else {
             log("Baseline nodes:");
 
-            for(VisorBaselineNode node : baseline.values()) {
+            for (VisorBaselineNode node : baseline.values()) {
                 log("    ConsistentID=" + node.getConsistentId() + ", STATE=" +
                     (srvs.containsKey(node.getConsistentId()) ? "ONLINE" : "OFFLINE"));
             }
@@ -950,7 +928,7 @@ private void baselinePrint0(VisorBaselineTaskResult res) {
             else {
                 log("Other nodes:");
 
-                for(VisorBaselineNode node : others)
+                for (VisorBaselineNode node : others)
                     log("    ConsistentID=" + node.getConsistentId());
 
                 log("Number of other nodes: " + others.size());
@@ -1123,7 +1101,7 @@ else if (arg.getOperation() == VisorTxOperation.KILL)
      * @throws Throwable If failed to execute wal action.
      */
     private void wal(GridClient client, String walAct, String walArgs) throws Throwable {
-        switch (walAct){
+        switch (walAct) {
             case WAL_DELETE:
                 deleteUnusedWalSegments(client, walArgs);
 
@@ -1145,7 +1123,7 @@ private void wal(GridClient client, String walAct, String walArgs) throws Throwa
      */
     private void deleteUnusedWalSegments(GridClient client, String walArgs) throws Throwable {
         VisorWalTaskResult res = executeTask(client, VisorWalTask.class,
-                walArg(VisorWalTaskOperation.DELETE_UNUSED_WAL_SEGMENTS, walArgs));
+            walArg(VisorWalTaskOperation.DELETE_UNUSED_WAL_SEGMENTS, walArgs));
         printDeleteWalSegments0(res);
     }
 
@@ -1157,7 +1135,7 @@ private void deleteUnusedWalSegments(GridClient client, String walArgs) throws T
      */
     private void printUnusedWalSegments(GridClient client, String walArgs) throws Throwable {
         VisorWalTaskResult res = executeTask(client, VisorWalTask.class,
-                walArg(VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS, walArgs));
+            walArg(VisorWalTaskOperation.PRINT_UNUSED_WAL_SEGMENTS, walArgs));
         printUnusedWalSegments0(res);
     }
 
@@ -1168,7 +1146,7 @@ private void printUnusedWalSegments(GridClient client, String walArgs) throws Th
      * @param s Argument from command line.
      * @return Task argument.
      */
-    private VisorWalTaskArg walArg(VisorWalTaskOperation op, String s){
+    private VisorWalTaskArg walArg(VisorWalTaskOperation op, String s) {
         List<String> consistentIds = null;
 
         if (!F.isEmpty(s)) {
@@ -1202,23 +1180,23 @@ private void printUnusedWalSegments0(VisorWalTaskResult taskRes) {
         Map<String, Exception> failRes = taskRes.exceptions();
         Map<String, VisorClusterNode> nodesInfo = taskRes.getNodesInfo();
 
-        for(Map.Entry<String, Collection<String>> entry: res.entrySet()) {
+        for (Map.Entry<String, Collection<String>> entry : res.entrySet()) {
             VisorClusterNode node = nodesInfo.get(entry.getKey());
 
             log("Node=" + node.getConsistentId());
-            log("     addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+            log("     addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
 
-            for(String fileName: entry.getValue())
+            for (String fileName : entry.getValue())
                 log("   " + fileName);
 
             nl();
         }
 
-        for(Map.Entry<String, Exception> entry: failRes.entrySet()) {
+        for (Map.Entry<String, Exception> entry : failRes.entrySet()) {
             VisorClusterNode node = nodesInfo.get(entry.getKey());
 
             log("Node=" + node.getConsistentId());
-            log("     addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+            log("     addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
             log("   failed with error: " + entry.getValue().getMessage());
             nl();
         }
@@ -1237,19 +1215,19 @@ private void printDeleteWalSegments0(VisorWalTaskResult taskRes) {
         Map<String, Exception> errors = taskRes.exceptions();
         Map<String, VisorClusterNode> nodesInfo = taskRes.getNodesInfo();
 
-        for(Map.Entry<String, Collection<String>> entry: res.entrySet()) {
+        for (Map.Entry<String, Collection<String>> entry : res.entrySet()) {
             VisorClusterNode node = nodesInfo.get(entry.getKey());
 
             log("Node=" + node.getConsistentId());
-            log("     addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+            log("     addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
             nl();
         }
 
-        for(Map.Entry<String, Exception> entry: errors.entrySet()) {
+        for (Map.Entry<String, Exception> entry : errors.entrySet()) {
             VisorClusterNode node = nodesInfo.get(entry.getKey());
 
             log("Node=" + node.getConsistentId());
-            log("     addresses " + U.addressesAsString(node.getAddresses(),node.getHostNames()));
+            log("     addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
             log("   failed with error: " + entry.getValue().getMessage());
             nl();
         }
@@ -1284,7 +1262,7 @@ private boolean isConnectionError(Throwable e) {
     private void usage(String desc, Command cmd, String... args) {
         log(desc);
         log("    control.sh [--host HOST_OR_IP] [--port PORT] [--user USER] [--password PASSWORD] " +
-                " [--ping-interval PING_INTERVAL] [--ping-timeout PING_TIMEOUT] " + cmd.text() + String.join("", args));
+            " [--ping-interval PING_INTERVAL] [--ping-timeout PING_TIMEOUT] " + cmd.text() + String.join("", args));
         nl();
     }
 
@@ -1419,8 +1397,8 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         if (WAL_PRINT.equals(walAct) || WAL_DELETE.equals(walAct))
                             walArgs = (str = peekNextArg()) != null && !isCommandOrOption(str)
-                                    ? nextArg("Unexpected argument for " + WAL.text() + ": " + walAct)
-                                    : "";
+                                ? nextArg("Unexpected argument for " + WAL.text() + ": " + walAct)
+                                : "";
                         else
                             throw new IllegalArgumentException("Unexpected action " + walAct + " for " + WAL.text());
 
@@ -1499,7 +1477,7 @@ Arguments parseAndValidate(List<String> rawArgs) {
             throw new IllegalArgumentException("Both user and password should be specified");
 
         return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
-                pingTimeout, pingInterval, autoConfirmation);
+            pingTimeout, pingInterval, autoConfirmation);
     }
 
     /**
@@ -1529,9 +1507,18 @@ private CacheArguments parseAndValidateCacheArgs() {
                 break;
 
             case IDLE_VERIFY:
-                if (hasNextCacheArg())
-                    parseCacheNames(nextArg(""), cacheArgs);
+                int idleVerifyArgsCnt = 3;
+
+                while (hasNextCacheArg() && idleVerifyArgsCnt-- > 0) {
+                    String nextArg = nextArg("");
 
+                    if (CMD_DUMP.equals(nextArg))
+                        cacheArgs.dump(true);
+                    else if (CMD_SKIP_ZEROS.equals(nextArg))
+                        cacheArgs.skipZeros(true);
+                    else
+                        parseCacheNames(nextArg, cacheArgs);
+                }
                 break;
 
             case CONTENTION:
@@ -1709,7 +1696,7 @@ private VisorTxTaskArg parseTransactionArguments() {
                 case TX_LIMIT:
                     nextArg("");
 
-                    limit = (int) nextLongArg(TX_LIMIT);
+                    limit = (int)nextLongArg(TX_LIMIT);
                     break;
 
                 case TX_ORDER:
@@ -1746,7 +1733,7 @@ private VisorTxTaskArg parseTransactionArguments() {
                 case TX_SIZE:
                     nextArg("");
 
-                    size = (int) nextLongArg(TX_SIZE);
+                    size = (int)nextLongArg(TX_SIZE);
                     break;
 
                 case TX_LABEL:
@@ -1807,9 +1794,9 @@ private long nextLongArg(String lb) {
     }
 
     /**
-     *  Check if raw arg is command or option.
+     * Check if raw arg is command or option.
      *
-     *  @return {@code true} If raw arg is command, overwise {@code false}.
+     * @return {@code true} If raw arg is command, overwise {@code false}.
      */
     private boolean isCommandOrOption(String raw) {
         return raw != null && raw.contains("--");
@@ -1843,11 +1830,11 @@ public int execute(List<String> rawArgs) {
                     "[minSize SIZE] [label PATTERN_REGEX] [servers|clients] " +
                     "[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE|", CMD_TX_ORDER_START_TIME, "] [kill] [" + CMD_AUTO_CONFIRMATION + "]");
 
-                if(enableExperimental) {
+                if (enableExperimental) {
                     usage("  Print absolute paths of unused archived wal segments on each node:", WAL,
-                            " print [consistentId1,consistentId2,....,consistentIdN]");
+                        " print [consistentId1,consistentId2,....,consistentIdN]");
                     usage("  Delete unused archived wal segments on each node:", WAL,
-                            " delete [consistentId1,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
+                        " delete [consistentId1,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
                 }
 
                 log("  View caches information in a cluster. For more details type:");
@@ -1962,6 +1949,7 @@ public static void main(String[] args) {
 
     /**
      * Used for tests.
+     *
      * @return Last operation result;
      */
     @SuppressWarnings("unchecked")
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
index 1411b2a2352ce..353a63ce5269c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
@@ -55,6 +55,12 @@ public class CacheArguments {
     /** Cache view command. */
     private @Nullable VisorViewCacheCmd cacheCmd;
 
+    /** Calculate partition hash and print into standard output. */
+    private boolean dump;
+
+    /** Skip zeros partitions. */
+    private boolean skipZeros;
+
     /**
      * @return Command.
      */
@@ -194,4 +200,32 @@ public int checkThrough() {
     public void checkThrough(int checkThrough) {
         this.checkThrough = checkThrough;
     }
+
+    /**
+     * @return Calculate partition hash and print into standard output.
+     */
+    public boolean dump() {
+        return dump;
+    }
+
+    /**
+     * @param dump Calculate partition hash and print into standard output.
+     */
+    public void dump(boolean dump) {
+        this.dump = dump;
+    }
+
+    /**
+     * @return Skip zeros partitions(size == 0) in result.
+     */
+    public boolean isSkipZeros() {
+        return skipZeros;
+    }
+
+    /**
+     * @param skipZeros Skip zeros partitions.
+     */
+    public void skipZeros(boolean skipZeros) {
+        this.skipZeros = skipZeros;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyDumpResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyDumpResult.java
new file mode 100644
index 0000000000000..6a744050ad8da
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyDumpResult.java
@@ -0,0 +1,73 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*      http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Encapsulates result of {@link VerifyBackupPartitionsDumpTask}.
+ */
+public class IdleVerifyDumpResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Cluster hashes. */
+    private Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes;
+
+    /**
+     * @param clusterHashes Cluster hashes.
+     */
+    public IdleVerifyDumpResult(Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes) {
+        this.clusterHashes = clusterHashes;
+    }
+
+    /**
+     * Default constructor for Externalizable.
+     */
+    public IdleVerifyDumpResult() {
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, clusterHashes);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer,
+        ObjectInput in) throws IOException, ClassNotFoundException {
+        clusterHashes = U.readLinkedMap(in);
+    }
+
+    /**
+     * @return Cluster hashes.
+     */
+    public Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes() {
+        return clusterHashes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IdleVerifyDumpResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
index d5815cd2b02e1..c31d6627ed94a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
@@ -1,19 +1,19 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*      http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package org.apache.ignite.internal.processors.cache.verify;
 
 import java.io.IOException;
@@ -21,6 +21,7 @@
 import java.io.ObjectOutput;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Consumer;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -106,6 +107,60 @@ public boolean hasConflicts() {
         return !F.isEmpty(hashConflicts()) || !F.isEmpty(counterConflicts());
     }
 
+    /**
+     * Print formatted result to given printer.
+     *
+     * @param printer Consumer for handle formatted result.
+     */
+    public void print(Consumer<String> printer) {
+        if (!hasConflicts())
+            printer.accept("idle_verify check has finished, no conflicts have been found.\n");
+        else {
+            int cntrConflictsSize = counterConflicts().size();
+            int hashConflictsSize = hashConflicts().size();
+
+            printer.accept("idle_verify check has finished, found " + (cntrConflictsSize + hashConflictsSize) +
+                " conflict partitions: [counterConflicts=" + cntrConflictsSize + ", hashConflicts=" +
+                hashConflictsSize + "]\n");
+
+            if (!F.isEmpty(counterConflicts())) {
+                printer.accept("Update counter conflicts:\n");
+
+                for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : counterConflicts().entrySet()) {
+                    printer.accept("Conflict partition: " + entry.getKey() + "\n");
+
+                    printer.accept("Partition instances: " + entry.getValue() + "\n");
+                }
+
+                printer.accept("\n");
+            }
+
+            if (!F.isEmpty(hashConflicts())) {
+                printer.accept("Hash conflicts:\n");
+
+                for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : hashConflicts().entrySet()) {
+                    printer.accept("Conflict partition: " + entry.getKey() + "\n");
+
+                    printer.accept("Partition instances: " + entry.getValue() + "\n");
+                }
+
+                printer.accept("\n");
+            }
+        }
+
+        if (!F.isEmpty(movingPartitions())) {
+            printer.accept("Verification was skipped for " + movingPartitions().size() + " MOVING partitions:\n");
+
+            for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : movingPartitions().entrySet()) {
+                printer.accept("Rebalancing partition: " + entry.getKey() + "\n");
+
+                printer.accept("Partition instances: " + entry.getValue() + "\n");
+            }
+
+            printer.accept("\n");
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IdleVerifyResultV2.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
new file mode 100644
index 0000000000000..7bbd9f19b9b15
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
@@ -0,0 +1,230 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.verify;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJob;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeTaskAdapter;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task for collection checksums primary and backup partitions of specified caches. <br> Argument: Set of cache names,
+ * 'null' will trigger verification for all caches. <br> Result: {@link IdleVerifyDumpResult} with all found partitions.
+ * <br> Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
+ * concurrently updated.
+ */
+@GridInternal
+public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter<VisorIdleVerifyTaskArg, String> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Time formatter for dump file name. */
+    private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH-mm-ss_SSS");
+
+    /** Visible for testing. */
+    public static final String IDLE_DUMP_FILE_PREMIX = "idle-dump-";
+
+    /** Delegate for map execution */
+    private final VerifyBackupPartitionsTaskV2 delegate = new VerifyBackupPartitionsTaskV2();
+
+    /** */
+    private VisorIdleVerifyDumpTaskArg taskArg;
+
+    @IgniteInstanceResource
+    private Ignite ignite;
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
+        List<ClusterNode> subgrid, VisorIdleVerifyTaskArg arg) throws IgniteException {
+        if (arg instanceof VisorIdleVerifyDumpTaskArg)
+            taskArg = (VisorIdleVerifyDumpTaskArg)arg;
+
+        return delegate.map(subgrid, arg);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public String reduce(List<ComputeJobResult> results)
+        throws IgniteException {
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes = new TreeMap<>(buildPartitionKeyComparator());
+
+        for (ComputeJobResult res : results) {
+            Map<PartitionKeyV2, PartitionHashRecordV2> nodeHashes = res.getData();
+
+            for (Map.Entry<PartitionKeyV2, PartitionHashRecordV2> e : nodeHashes.entrySet()) {
+                clusterHashes
+                    .computeIfAbsent(e.getKey(), k -> new ArrayList<>())
+                    .add(e.getValue());
+            }
+        }
+
+        Comparator<PartitionHashRecordV2> recordComp = buildRecordComparator().reversed();
+
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> partitions = new LinkedHashMap<>();
+
+        int skippedRecords = 0;
+
+        for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : clusterHashes.entrySet()) {
+            if (needToAdd(entry.getValue())) {
+                entry.getValue().sort(recordComp);
+
+                partitions.put(entry.getKey(), entry.getValue());
+            }
+            else
+                skippedRecords++;
+        }
+
+        return writeHashes(partitions, delegate.reduce(results), skippedRecords);
+    }
+
+    /**
+     * Checking conditions for adding given record to result.
+     *
+     * @param records records to check.
+     * @return {@code true} if this records should be add to result and {@code false} otherwise.
+     */
+    private boolean needToAdd(List<PartitionHashRecordV2> records) {
+        if (records.isEmpty() || (taskArg != null && !taskArg.isSkipZeros()))
+            return true;
+
+        PartitionHashRecordV2 record = records.get(0);
+
+        if (record.updateCounter() != 0 || record.size() != 0)
+            return true;
+
+        int firstHash = record.partitionHash();
+
+        for (int i = 1; i < records.size(); i++) {
+            record = records.get(i);
+
+            if (record.partitionHash() != firstHash || record.updateCounter() != 0 || record.size() != 0)
+                return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * @param partitions Dump result.
+     * @return Path where results are written.
+     * @throws IgniteException If failed to write the file.
+     */
+    private String writeHashes(
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> partitions,
+        IdleVerifyResultV2 conflictRes,
+        int skippedRecords
+    ) throws IgniteException {
+        File workDir = ignite.configuration().getWorkDirectory() == null
+            ? new File("/tmp")
+            : new File(ignite.configuration().getWorkDirectory());
+
+        File out = new File(workDir, IDLE_DUMP_FILE_PREMIX + LocalDateTime.now().format(TIME_FORMATTER) + ".txt");
+
+        ignite.log().info("IdleVerifyDumpTask will write output to " + out.getAbsolutePath());
+
+        try (BufferedWriter writer = new BufferedWriter(new FileWriter(out))) {
+            try {
+
+                writer.write("idle_verify check has finished, found " + partitions.size() + " partitions\n");
+
+                if (skippedRecords > 0)
+                    writer.write(skippedRecords + " partitions was skipped\n");
+
+                if (!F.isEmpty(partitions)) {
+                    writer.write("Cluster partitions:\n");
+
+                    for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : partitions.entrySet()) {
+                        writer.write("Partition: " + entry.getKey() + "\n");
+
+                        writer.write("Partition instances: " + entry.getValue() + "\n");
+                    }
+
+                    writer.write("\n\n-----------------------------------\n\n");
+
+                    conflictRes.print(str -> {
+                        try {
+                            writer.write(str);
+                        }
+                        catch (IOException e) {
+                            throw new IgniteException("Failed to write partitions conflict.", e);
+                        }
+                    });
+                }
+            }
+            finally {
+                writer.flush();
+            }
+
+            ignite.log().info("IdleVerifyDumpTask successfully written dump to '" + out.getAbsolutePath() + "'");
+        }
+        catch (IOException | IgniteException e) {
+            ignite.log().error("Failed to write dump file: " + out.getAbsolutePath(), e);
+
+            throw new IgniteException(e);
+        }
+
+        return out.getAbsolutePath();
+    }
+
+    /**
+     * @return Comparator for {@link PartitionHashRecordV2}.
+     */
+    @NotNull private Comparator<PartitionHashRecordV2> buildRecordComparator() {
+        return (o1, o2) -> {
+            int compare = Boolean.compare(o1.isPrimary(), o2.isPrimary());
+
+            if (compare != 0)
+                return compare;
+
+            return o1.consistentId().toString().compareTo(o2.consistentId().toString());
+        };
+    }
+
+    /**
+     * @return Comparator for {@link PartitionKeyV2}.
+     */
+    @NotNull private Comparator<PartitionKeyV2> buildPartitionKeyComparator() {
+        return (o1, o2) -> {
+            int compare = Integer.compare(o1.groupId(), o2.groupId());
+
+            if (compare != 0)
+                return compare;
+
+            return Integer.compare(o1.partitionId(), o2.partitionId());
+        };
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTask.java
new file mode 100644
index 0000000000000..8f34c68224871
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTask.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsDumpTask;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+
+/**
+ * Task to verify checksums of backup partitions and return all collected information.
+ */
+@GridInternal
+public class VisorIdleVerifyDumpTask extends VisorOneNodeTask<VisorIdleVerifyTaskArg, String> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<VisorIdleVerifyTaskArg, String> job(VisorIdleVerifyTaskArg arg) {
+        return new VisorIdleVerifyJob<>(arg, debug, VerifyBackupPartitionsDumpTask.class);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
new file mode 100644
index 0000000000000..6316c2443a73d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Set;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * Arguments for {@link VisorIdleVerifyDumpTask}.
+ */
+public class VisorIdleVerifyDumpTaskArg extends VisorIdleVerifyTaskArg {
+    /** */
+    private static final long serialVersionUID = 0L;
+    /** */
+    private boolean skipZeros;
+
+    /**
+     * Default constructor.
+     */
+    public VisorIdleVerifyDumpTaskArg() {
+    }
+
+    /**
+     * @param caches Caches.
+     * @param skipZeros Skip zeros partitions.
+     */
+    public VisorIdleVerifyDumpTaskArg(Set<String> caches, boolean skipZeros) {
+        super(caches);
+        this.skipZeros = skipZeros;
+    }
+
+    /**
+     * @return Skip zeros partitions.
+     */
+    public boolean isSkipZeros() {
+        return skipZeros;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        super.writeExternalData(out);
+        out.writeBoolean(skipZeros);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        super.readExternalData(protoVer, in);
+        skipZeros = in.readBoolean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorIdleVerifyDumpTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyJob.java
new file mode 100644
index 0000000000000..a8dc697f07713
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyJob.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobContext;
+import org.apache.ignite.compute.ComputeTask;
+import org.apache.ignite.compute.ComputeTaskFuture;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.resources.JobContextResource;
+
+/**
+ *
+ */
+class VisorIdleVerifyJob<ResultT> extends VisorJob<VisorIdleVerifyTaskArg, ResultT> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private ComputeTaskFuture<ResultT> fut;
+
+    /** Auto-inject job context. */
+    @JobContextResource
+    protected transient ComputeJobContext jobCtx;
+
+    /** Task class for execution */
+    private final Class<? extends ComputeTask<VisorIdleVerifyTaskArg, ResultT>> taskCls;
+
+    /**
+     * @param arg Argument.
+     * @param debug Debug.
+     * @param taskCls Task class for execution.
+     */
+    VisorIdleVerifyJob(VisorIdleVerifyTaskArg arg, boolean debug,
+        Class<? extends ComputeTask<VisorIdleVerifyTaskArg, ResultT>> taskCls) {
+        super(arg, debug);
+        this.taskCls = taskCls;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected ResultT run(VisorIdleVerifyTaskArg arg) throws IgniteException {
+        if (fut == null) {
+            fut = ignite.compute().executeAsync(taskCls, arg);
+
+            if (!fut.isDone()) {
+                jobCtx.holdcc();
+
+                fut.listen(new IgniteInClosure<IgniteFuture<ResultT>>() {
+                    @Override public void apply(IgniteFuture<ResultT> f) {
+                        jobCtx.callcc();
+                    }
+                });
+
+                return null;
+            }
+        }
+
+        return fut.get();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorIdleVerifyJob.class, this);
+    }
+}
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 6c91de43700cb..a94048f90fb13 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -1139,6 +1139,7 @@ org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysT
 org.apache.ignite.internal.processors.cache.verify.ContentionClosure
 org.apache.ignite.internal.processors.cache.verify.ContentionInfo
 org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2
+org.apache.ignite.internal.processors.cache.verify.IdleVerifyDumpResult
 org.apache.ignite.internal.processors.cache.verify.PartitionEntryHashRecord
 org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord
 org.apache.ignite.internal.processors.cache.verify.PartitionKey
@@ -2160,6 +2161,7 @@ org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask
 org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask$VisorIdleVerifyJob
 org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask$VisorIdleVerifyJob$1
 org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg
 org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult
 org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult
 org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskArg
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 55c10345b12c4..3dcb8beccbfa4 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -23,6 +23,10 @@
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.PrintStream;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -36,6 +40,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.LongAdder;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteAtomicSequence;
 import org.apache.ignite.IgniteCache;
@@ -88,12 +94,16 @@
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionRollbackException;
 import org.apache.ignite.transactions.TransactionTimeoutException;
+import org.jetbrains.annotations.NotNull;
 
+import static java.nio.file.Files.delete;
+import static java.nio.file.Files.newDirectoryStream;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
+import static org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsDumpTask.IDLE_DUMP_FILE_PREMIX;
 import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
 import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
@@ -138,6 +148,15 @@ protected File folder(String folder) throws IgniteCheckedException {
 
         cleanPersistenceDir();
 
+        //delete idle-verify dump files.
+        try (DirectoryStream<Path> files = newDirectoryStream(
+            Paths.get(U.defaultWorkDirectory()),
+            entry -> entry.toFile().getName().startsWith(IDLE_DUMP_FILE_PREMIX)
+        )) {
+            for (Path path : files)
+                delete(path);
+        }
+
         System.clearProperty(IGNITE_ENABLE_EXPERIMENTAL_COMMAND);
 
         System.setOut(sysOut);
@@ -504,14 +523,14 @@ else if (entry.getKey().equals(node2)) {
         validate(h, map -> {
             VisorTxTaskResult res = map.get(grid(0).localNode());
 
-            assertTrue(res.getInfos().get(0).getSize() >=  res.getInfos().get(1).getSize());
+            assertTrue(res.getInfos().get(0).getSize() >= res.getInfos().get(1).getSize());
         }, "--tx", "order", "SIZE");
 
         // test order by duration.
         validate(h, map -> {
             VisorTxTaskResult res = map.get(grid(0).localNode());
 
-            assertTrue(res.getInfos().get(0).getDuration() >=  res.getInfos().get(1).getDuration());
+            assertTrue(res.getInfos().get(0).getDuration() >= res.getInfos().get(1).getDuration());
         }, "--tx", "order", "DURATION");
 
         // test order by start_time.
@@ -584,7 +603,7 @@ public void testKillHangingLocalTransactions() throws Exception {
 
         GridNearTxLocal clientTx = null;
 
-        try(Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 2000, 1)) {
+        try (Transaction tx = client.transactions().txStart(PESSIMISTIC, READ_COMMITTED, 2000, 1)) {
             clientTx = ((TransactionProxyImpl)tx).tx();
 
             client.cache(DEFAULT_CACHE_NAME).put(0L, 0L);
@@ -715,7 +734,7 @@ public void testKillHangingRemoteTransactions() throws Exception {
         primSpi.waitForBlocked(clients.length);
 
         // Unblock only finish messages from clients from 2 to 4.
-        primSpi.stopBlock(true, new IgnitePredicate<T2<ClusterNode,GridIoMessage>>() {
+        primSpi.stopBlock(true, new IgnitePredicate<T2<ClusterNode, GridIoMessage>>() {
             @Override public boolean apply(T2<ClusterNode, GridIoMessage> objects) {
                 GridIoMessage iom = objects.get2();
 
@@ -919,6 +938,141 @@ public void testCacheIdleVerifyTwoConflictTypes() throws Exception {
         assertTrue(testOut.toString().contains("found 2 conflict partitions"));
     }
 
+    /**
+     * Tests that idle verify print partitions info.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCacheIdleVerifyDump() throws Exception {
+        IgniteEx ignite = (IgniteEx)startGrids(3);
+
+        ignite.cluster().active(true);
+
+        int parts = 32;
+
+        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, parts))
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME));
+
+        ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, parts))
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME + "other"));
+
+        injectTestSystemOut();
+
+        int keysCount = 20;//less than parts number for ability to check skipZeros flag.
+
+        for (int i = 0; i < keysCount; i++)
+            cache.put(i, i);
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME));
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--skipZeros", DEFAULT_CACHE_NAME));
+
+        Matcher fileNameMatcher = dumpFileNameMatcher();
+
+        if (fileNameMatcher.find()) {
+            String dumpWithZeros = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+            assertTrue(dumpWithZeros.contains("idle_verify check has finished, found " + parts + " partitions"));
+            assertTrue(dumpWithZeros.contains("Partition: PartitionKeyV2 [grpId=1544803905, grpName=default, partId=0]"));
+            assertTrue(dumpWithZeros.contains("updateCntr=0, size=0, partHash=0"));
+            assertTrue(dumpWithZeros.contains("no conflicts have been found"));
+
+            assertSort(parts, dumpWithZeros);
+        }
+
+        if (fileNameMatcher.find()) {
+            String dumpWithoutZeros = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+            assertTrue(dumpWithoutZeros.contains("idle_verify check has finished, found " + keysCount + " partitions"));
+            assertTrue(dumpWithoutZeros.contains((parts - keysCount) + " partitions was skipped"));
+            assertTrue(dumpWithoutZeros.contains("Partition: PartitionKeyV2 [grpId=1544803905, grpName=default, partId="));
+
+            assertFalse(dumpWithoutZeros.contains("updateCntr=0, size=0, partHash=0"));
+
+            assertTrue(dumpWithoutZeros.contains("no conflicts have been found"));
+
+            assertSort(keysCount, dumpWithoutZeros);
+        }
+        else
+            fail("Should be found both files");
+    }
+
+    /**
+     * Checking sorting of partitions.
+     *
+     * @param expectedPartsCount Expected parts count.
+     * @param output Output.
+     */
+    private void assertSort(int expectedPartsCount, String output) {
+        Pattern partIdPattern = Pattern.compile(".*partId=([0-9]*)");
+        Pattern primaryPattern = Pattern.compile("Partition instances: \\[PartitionHashRecordV2 \\[isPrimary=true");
+
+        Matcher partIdMatcher = partIdPattern.matcher(output);
+        Matcher primaryMatcher = primaryPattern.matcher(output);
+
+        int i = 0;
+
+        while (partIdMatcher.find()) {
+            assertEquals(i++, Integer.parseInt(partIdMatcher.group(1)));
+            assertTrue(primaryMatcher.find());//primary node should be first in every line
+        }
+
+        assertEquals(expectedPartsCount, i);
+    }
+
+    /**
+     * Tests that idle verify print partitions info.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCacheIdleVerifyDumpForCorruptedData() throws Exception {
+        IgniteEx ignite = (IgniteEx)startGrids(3);
+
+        ignite.cluster().active(true);
+
+        int parts = 32;
+
+        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, parts))
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME));
+
+        injectTestSystemOut();
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, i);
+
+        GridCacheContext<Object, Object> cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context();
+
+        corruptDataEntry(cacheCtx, 0, true, false);
+
+        corruptDataEntry(cacheCtx, 0 + parts / 2, false, true);
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump"));
+
+        Matcher fileNameMatcher = dumpFileNameMatcher();
+
+        if (fileNameMatcher.find()) {
+            String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+            assertTrue(dumpWithConflicts.contains("found 2 conflict partitions: [counterConflicts=1, hashConflicts=1]"));
+        }else
+            fail("Should be found dump with conflicts");
+    }
+
+    /**
+     * @return Build matcher for dump file name.
+     */
+    @NotNull private Matcher dumpFileNameMatcher() {
+        Pattern fileNamePattern = Pattern.compile(".*VisorIdleVerifyDumpTask successfully written output to '(.*)'");
+
+        return fileNamePattern.matcher(testOut.toString());
+    }
+
     /**
      * @throws Exception If failed.
      */
@@ -1296,7 +1450,8 @@ private void checkFutures() {
     /** */
     private static class IncrementClosure implements EntryProcessor<Long, Long, Void> {
         /** {@inheritDoc} */
-        @Override public Void process(MutableEntry<Long, Long> entry, Object... arguments) throws EntryProcessorException {
+        @Override public Void process(MutableEntry<Long, Long> entry,
+            Object... arguments) throws EntryProcessorException {
             entry.setValue(entry.exists() ? entry.getValue() + 1 : 0);
 
             return null;

From e92788d392d4c60f76c4fd0f38c52669f4e3e772 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Wed, 1 Aug 2018 14:05:58 +0300
Subject: [PATCH 277/543] IGNITE-8939 Catch and proper propagate transaction
 string reprsentation - Fixes #4454.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
(cherry picked from commit 458480c5b0520aa8e28935361a37ab49e1e65ff6)
---
 .../cache/transactions/IgniteTxHandler.java   | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 8f2dbc05bab31..5b82333ac8a80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -23,6 +23,8 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -954,7 +956,23 @@ private IgniteInternalFuture<IgniteInternalTx> finishDhtLocal(UUID nodeId,
             }
         }
         catch (Throwable e) {
-            U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e);
+            try {
+                U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e);
+            }
+            catch (Throwable e0) {
+                ClusterNode node0 = ctx.discovery().node(nodeId);
+
+                U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" +
+                        CU.txString(tx) + ']', e);
+
+                U.error(log, "Failed to log message due to an error: ", e0);
+
+                if (node0 != null && (!node0.isClient() || node0.isLocal())) {
+                    ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+                    throw e;
+                }
+            }
 
             if (tx != null) {
                 tx.commitError(e);

From 60155cb55ca35790da7f80ae167dd7fb3f3793fd Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Wed, 1 Aug 2018 13:23:32 +0300
Subject: [PATCH 278/543] IGNITE-9154: Fixed conflict version passed to events
 for ATOMIC cache. This closes #4463.

(cherry picked from commit 3ea3a56)
---
 .../internal/processors/cache/GridCacheMapEntry.java  | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 9f3686aad47ec..ed3fa34ad3b49 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -1854,7 +1854,7 @@ else if (ttl != CU.TTL_ZERO)
                     key,
                     evtNodeId,
                     null,
-                    newVer,
+                    updateVer,
                     EVT_CACHE_OBJECT_READ,
                     evtOld, evtOld != null,
                     evtOld, evtOld != null,
@@ -1881,7 +1881,7 @@ else if (ttl != CU.TTL_ZERO)
                         key,
                         evtNodeId,
                         null,
-                        newVer,
+                        updateVer,
                         EVT_CACHE_OBJECT_PUT,
                         updateVal,
                         true,
@@ -1898,7 +1898,7 @@ else if (ttl != CU.TTL_ZERO)
 
                 clearReaders();
 
-                drReplicate(drType, null, newVer, topVer);
+                drReplicate(drType, null, updateVer, topVer);
 
                 recordNodeId(affNodeId, topVer);
 
@@ -1909,7 +1909,8 @@ else if (ttl != CU.TTL_ZERO)
                     cctx.events().addEvent(partition(),
                         key,
                         evtNodeId,
-                        null, newVer,
+                        null,
+                        updateVer,
                         EVT_CACHE_OBJECT_REMOVED,
                         null, false,
                         evtOld, evtOld != null,
@@ -4882,7 +4883,7 @@ else if (newSysTtl == CU.TTL_ZERO) {
                 }
             }
             else {
-                newSysTtl = newTtl = conflictCtx.ttl();
+               newSysTtl = newTtl = conflictCtx.ttl();
                 newSysExpireTime = newExpireTime = conflictCtx.expireTime();
             }
 

From 158f36de551e860176ac7f3bcf92e9751039a35c Mon Sep 17 00:00:00 2001
From: ilantukh <ilantukh@gridgain.com>
Date: Wed, 1 Aug 2018 16:31:15 +0300
Subject: [PATCH 279/543] IGNITE-8900 SqlFieldsQuery provides incorrect result
 when item size exceeds page size.

(cherry picked from commit f1ecbbc)
---
 .../delta/DataPageInsertFragmentRecord.java   |   2 +-
 .../cache/IgniteCacheOffheapManagerImpl.java  |   3 +
 .../freelist/AbstractFreeList.java            |   2 +-
 .../freelist/CacheFreeListImpl.java           |  11 ++
 .../tree/io/AbstractDataPageIO.java           |  23 ++-
 .../processors/cache/BigEntryQueryTest.java   | 150 ++++++++++++++++++
 .../IgniteBinaryCacheQueryTestSuite.java      |   2 +
 7 files changed, 177 insertions(+), 16 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BigEntryQueryTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
index 5324d5605d72d..2b02bb5748fdb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
@@ -57,7 +57,7 @@ public DataPageInsertFragmentRecord(
     @Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
         AbstractDataPageIO io = PageIO.getPageIO(pageAddr);
 
-        io.addRowFragment(pageAddr, payload, lastLink, pageMem.pageSize());
+        io.addRowFragment(PageIO.getPageId(pageAddr), pageAddr, payload, lastLink, pageMem.pageSize());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 5c78eb5b4b4a4..07f99312a56d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -1329,6 +1329,9 @@ private boolean canUpdateOldRow(GridCacheContext cctx, @Nullable CacheDataRow ol
 
                 assert oldRow == null || oldRow.cacheId() == cacheId : oldRow;
 
+                if (key.partition() == -1)
+                    key.partition(partId);
+
                 DataRow dataRow = new DataRow(key, val, ver, partId, expireTime, cacheId);
 
                 CacheObjectContext coCtx = cctx.cacheObjectContext();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
index 5f5948dab1f00..ada09e4112f1e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/AbstractFreeList.java
@@ -236,7 +236,7 @@ private int addRowFragment(
             // Read last link before the fragment write, because it will be updated there.
             long lastLink = row.link();
 
-            int payloadSize = io.addRowFragment(pageMem, pageAddr, row, written, rowSize, pageSize());
+            int payloadSize = io.addRowFragment(pageMem, pageId, pageAddr, row, written, rowSize, pageSize());
 
             assert payloadSize > 0 : payloadSize;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
index dc0c92e8cdbfb..4aa9d88843f59 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/CacheFreeListImpl.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.cache.persistence.freelist;
 
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
@@ -26,6 +27,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
  * FreeList implementation for cache.
@@ -52,6 +54,15 @@ public CacheFreeListImpl(int cacheId, String name, DataRegionMetricsImpl regionM
         return DataPageIO.VERSIONS;
     }
 
+    /** {@inheritDoc} */
+    @Override public void insertDataRow(CacheDataRow row) throws IgniteCheckedException {
+        super.insertDataRow(row);
+
+        assert row.key().partition() == PageIdUtils.partId(row.link()) :
+            "Constructed a link with invalid partition ID [partId=" + row.key().partition() +
+            ", link=" + U.hexLong(row.link()) + ']';
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return "FreeList [name=" + name + ']';
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
index ca63f27bf72f0..50b577978e4ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/AbstractDataPageIO.java
@@ -907,6 +907,7 @@ private int getDataOffsetForWrite(long pageAddr, int fullEntrySize, int directCn
      * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
      *
      * @param pageMem Page memory.
+     * @param pageId Page ID to use to construct a link.
      * @param pageAddr Page address.
      * @param row Data row.
      * @param written Number of bytes of row size that was already written.
@@ -917,18 +918,20 @@ private int getDataOffsetForWrite(long pageAddr, int fullEntrySize, int directCn
      */
     public int addRowFragment(
         PageMemory pageMem,
+        long pageId,
         long pageAddr,
         T row,
         int written,
         int rowSize,
         int pageSize
     ) throws IgniteCheckedException {
-        return addRowFragment(pageMem, pageAddr, written, rowSize, row.link(), row, null, pageSize);
+        return addRowFragment(pageMem, pageId, pageAddr, written, rowSize, row.link(), row, null, pageSize);
     }
 
     /**
      * Adds this payload as a fragment to this data page.
      *
+     * @param pageId Page ID to use to construct a link.
      * @param pageAddr Page address.
      * @param payload Payload bytes.
      * @param lastLink Link to the previous written fragment (link to the tail).
@@ -936,18 +939,20 @@ public int addRowFragment(
      * @throws IgniteCheckedException If failed.
      */
     public void addRowFragment(
+        long pageId,
         long pageAddr,
         byte[] payload,
         long lastLink,
         int pageSize
     ) throws IgniteCheckedException {
-        addRowFragment(null, pageAddr, 0, 0, lastLink, null, payload, pageSize);
+        addRowFragment(null, pageId, pageAddr, 0, 0, lastLink, null, payload, pageSize);
     }
 
     /**
      * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
      *
      * @param pageMem Page memory.
+     * @param pageId Page ID to use to construct a link.
      * @param pageAddr Page address.
      * @param written Number of bytes of row size that was already written.
      * @param rowSize Row size.
@@ -960,6 +965,7 @@ public void addRowFragment(
      */
     private int addRowFragment(
         PageMemory pageMem,
+        long pageId,
         long pageAddr,
         int written,
         int rowSize,
@@ -1004,22 +1010,11 @@ private int addRowFragment(
         int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
 
         if (row != null)
-            setLink(row, pageAddr, itemId);
+            setLinkByPageId(row, pageId, itemId);
 
         return payloadSize;
     }
 
-    /**
-     * @param row Row to set link to.
-     * @param pageAddr Page address.
-     * @param itemId Item ID.
-     */
-    private void setLink(T row, long pageAddr, int itemId) {
-        long pageId = getPageId(pageAddr);
-
-        setLinkByPageId(row, pageId, itemId);
-    }
-
     /**
      * @param row Row to set link to.
      * @param pageId Page ID.
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BigEntryQueryTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BigEntryQueryTest.java
new file mode 100644
index 0000000000000..3b5562a3035b8
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BigEntryQueryTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.LongStream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * This is a specific test for IGNITE-8900.
+ */
+public class BigEntryQueryTest extends GridCommonAbstractTest {
+    /** */
+    public static final String CACHE = "cache";
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 60_000;
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testBigEntriesSelect() throws Exception {
+        startGrids(2);
+
+        Random random = new Random(1);
+
+        Ignition.setClientMode(true);
+
+        Ignite client = startGrid(2);
+
+        int ctr = 0;
+
+        long time0 = System.currentTimeMillis();
+
+        while ((System.currentTimeMillis() - time0) < 30_000) {
+            String cacheName = CACHE + ctr++;
+
+            IgniteCache<Long, Value> cache = client.getOrCreateCache(new CacheConfiguration<Long, Value>(cacheName)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setBackups(1)
+                .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+                .setIndexedTypes(Long.class, Value.class));
+
+            cache.putAll(LongStream.range(610026643276160000L, 610026643276170000L).boxed()
+                .collect(Collectors.toMap(Function.identity(),
+                    t -> Value.of(new byte[(random.nextInt(16)) * 1000]))));
+
+            for (int i = 0; i < 10; i++) {
+                long start = 610026643276160000L;
+                long end = start + random.nextInt(10);
+
+                int expectedResultCount = (int)(end - start + 1);
+
+                String sql = String.format(
+                    "SELECT _KEY " +
+                        "FROM %s " +
+                        "WHERE _KEY >= %d AND _KEY <= %d", Value.class.getSimpleName().toLowerCase(),
+                    start, end);
+
+                Set<Long> keySet = new HashSet<>();
+                for (long l = start; l < end + 1; l++)
+                    keySet.add(l);
+
+                List<Long> resultKeys;
+                try (FieldsQueryCursor<List<?>> results = cache.query(new SqlFieldsQuery(sql))) {
+                    resultKeys = new ArrayList<>();
+                    results.forEach(objects -> resultKeys.add((Long)objects.get(0)));
+                    Collections.sort(resultKeys);
+                }
+
+                assertEquals(expectedResultCount, resultKeys.size());
+            }
+            cache.destroy();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration() throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration();
+
+        cfg.setIncludeEventTypes(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
+
+        DataStorageConfiguration sCfg = new DataStorageConfiguration();
+
+        sCfg.setPageSize(1024 * 8);
+
+        cfg.setDataStorageConfiguration(sCfg);
+
+        return cfg;
+    }
+
+    /**
+     * Class containing value to be placed into the cache.
+     */
+    public static class Value implements Serializable {
+        /** */
+        final byte[] data;
+
+        /**
+         * @param data Data.
+         */
+        public Value(final byte[] data) {
+            this.data = data;
+        }
+
+        /**
+         * @param bytes Bytes.
+         * @return Value.
+         */
+        public static Value of(final byte[] bytes) {
+            return new Value(bytes);
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
index 8164fe05518f0..b44ff2dbc3517 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.internal.processors.cache.BinarySerializationQueryWithReflectiveSerializerSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryObjectsScanSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryObjectsScanWithEventsSelfTest;
+import org.apache.ignite.internal.processors.cache.BigEntryQueryTest;
 
 /**
  * Cache query suite with binary marshaller.
@@ -39,6 +40,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(BinarySerializationQueryWithReflectiveSerializerSelfTest.class);
         suite.addTestSuite(IgniteCacheBinaryObjectsScanSelfTest.class);
         suite.addTestSuite(IgniteCacheBinaryObjectsScanWithEventsSelfTest.class);
+        suite.addTestSuite(BigEntryQueryTest.class);
 
         //Should be adjusted. Not ready to be used with BinaryMarshaller.
         //suite.addTestSuite(GridCacheBinarySwapScanQuerySelfTest.class);

From ef331f3ba58da49746f34546f1972cf65c083b3d Mon Sep 17 00:00:00 2001
From: Maxim Muzafarov <maxmuzaf@gmail.com>
Date: Wed, 1 Aug 2018 18:39:54 +0300
Subject: [PATCH 280/543] IGNITE-7165 Re-balancing is cancelled if client node
 joins

Signed-off-by: Anton Vinogradov <av@apache.org>
(cherry picked from commit 137dd06aaee9cc84104e6b4bf48306b050341e3a)
---
 .../GridCachePartitionExchangeManager.java    |  68 +++++---
 .../processors/cache/GridCachePreloader.java  |  21 ++-
 .../cache/GridCachePreloaderAdapter.java      |   6 +
 .../preloader/GridDhtPartitionDemander.java   |  55 ++++---
 .../preloader/GridDhtPartitionSupplier.java   |  26 ++-
 .../dht/preloader/GridDhtPreloader.java       |  60 ++++++-
 .../GridDhtPreloaderAssignments.java          |   6 +-
 .../ClusterBaselineNodesMetricsSelfTest.java  |   1 -
 .../cache/CacheValidatorMetricsTest.java      |   4 +-
 .../dht/GridCacheDhtPreloadSelfTest.java      |  68 +-------
 .../atomic/IgniteCacheAtomicProtocolTest.java |   1 +
 .../GridCacheRebalancingAsyncSelfTest.java    |   7 +-
 .../GridCacheRebalancingCancelTest.java       | 106 +++++++++++++
 ...CacheRebalancingPartitionCountersTest.java |   3 +-
 .../GridCacheRebalancingSyncSelfTest.java     | 149 +++++++-----------
 ...entAffinityAssignmentWithBaselineTest.java |   4 +-
 ...owHistoricalRebalanceSmallHistoryTest.java |   5 +-
 ...lushMultiNodeFailoverAbstractSelfTest.java |   2 +-
 .../GridMarshallerMappingConsistencyTest.java |   3 +-
 .../junits/common/GridCommonAbstractTest.java | 108 ++++++++++---
 .../testsuites/IgniteCacheTestSuite3.java     |   9 +-
 21 files changed, 439 insertions(+), 273 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 1f28da2ae09bb..952557eaebe9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -65,7 +65,6 @@
 import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
@@ -87,6 +86,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionsToReloadMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.RebalanceReassignExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
@@ -178,6 +178,14 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     private final ConcurrentMap<AffinityTopologyVersion, AffinityReadyFuture> readyFuts =
         new ConcurrentSkipListMap<>();
 
+    /**
+     * Latest started rebalance topology version but possibly not finished yet. Value {@code NONE}
+     * means that previous rebalance is undefined and the new one should be initiated.
+     *
+     * Should not be used to determine latest rebalanced topology.
+     */
+    private volatile AffinityTopologyVersion rebTopVer = AffinityTopologyVersion.NONE;
+
     /** */
     private final AtomicReference<AffinityTopologyVersion> readyTopVer =
         new AtomicReference<>(AffinityTopologyVersion.NONE);
@@ -829,6 +837,13 @@ public AffinityTopologyVersion readyAffinityVersion() {
         return readyTopVer.get();
     }
 
+    /**
+     * @return Latest rebalance topology version or {@code NONE} if there is no info.
+     */
+    public AffinityTopologyVersion rebalanceTopologyVersion() {
+        return rebTopVer;
+    }
+
     /**
      * @return Last initialized topology future.
      */
@@ -2569,6 +2584,9 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (grp.isLocal())
                                     continue;
 
+                                if (grp.preloader().rebalanceRequired(rebTopVer, exchFut))
+                                    rebTopVer = AffinityTopologyVersion.NONE;
+
                                 changed |= grp.topology().afterExchange(exchFut);
                             }
 
@@ -2576,7 +2594,11 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 refreshPartitions();
                         }
 
-                        if (!cctx.kernalContext().clientNode()) {
+                        // Schedule rebalance if force rebalance or force reassign occurs.
+                        if (exchFut == null)
+                            rebTopVer = AffinityTopologyVersion.NONE;
+
+                        if (!cctx.kernalContext().clientNode() && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
                             assignsMap = new HashMap<>();
 
                             IgniteCacheSnapshotManager snp = cctx.snapshot();
@@ -2593,6 +2615,9 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                     assigns = grp.preloader().generateAssignments(exchId, exchFut);
 
                                 assignsMap.put(grp.groupId(), assigns);
+
+                                if (resVer == null)
+                                    resVer = grp.topology().readyTopologyVersion();
                             }
                         }
                     }
@@ -2601,7 +2626,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         busy = false;
                     }
 
-                    if (assignsMap != null) {
+                    if (assignsMap != null && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
                         int size = assignsMap.size();
 
                         NavigableMap<Integer, List<Integer>> orderMap = new TreeMap<>();
@@ -2639,11 +2664,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (assigns != null)
                                     assignsCancelled |= assigns.cancelled();
 
-                                // Cancels previous rebalance future (in case it's not done yet).
-                                // Sends previous rebalance stopped event (if necessary).
-                                // Creates new rebalance future.
-                                // Sends current rebalance started event (if necessary).
-                                // Finishes cache sync future (on empty assignments).
                                 Runnable cur = grp.preloader().addAssignments(assigns,
                                     forcePreload,
                                     cnt,
@@ -2661,7 +2681,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         if (forcedRebFut != null)
                             forcedRebFut.markInitialized();
 
-                        if (assignsCancelled) { // Pending exchange.
+                        if (assignsCancelled || hasPendingExchange()) {
                             U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
                                 "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
@@ -2669,25 +2689,31 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         else if (r != null) {
                             Collections.reverse(rebList);
 
-                            U.log(log, "Rebalancing scheduled [order=" + rebList + "]");
+                            U.log(log, "Rebalancing scheduled [order=" + rebList +
+                                ", top=" + resVer + ", force=" + (exchFut == null) +
+                                ", evt=" + exchId.discoveryEventName() +
+                                ", node=" + exchId.nodeId() + ']');
 
-                            if (!hasPendingExchange()) {
-                                U.log(log, "Rebalancing started " +
-                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
-                                    ", node=" + exchId.nodeId() + ']');
+                            rebTopVer = resVer;
 
-                                r.run(); // Starts rebalancing routine.
-                            }
-                            else
-                                U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
-                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
-                                    ", node=" + exchId.nodeId() + ']');
+                            // Start rebalancing cache groups chain. Each group will be rebalanced
+                            // sequentially one by one e.g.:
+                            // ignite-sys-cache -> cacheGroupR1 -> cacheGroupP2 -> cacheGroupR3
+                            r.run();
                         }
                         else
                             U.log(log, "Skipping rebalancing (nothing scheduled) " +
-                                "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
+                                "[top=" + resVer + ", force=" + (exchFut == null) +
+                                ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
                     }
+                    else
+                        U.log(log, "Skipping rebalancing (no affinity changes) " +
+                            "[top=" + resVer +
+                            ", rebTopVer=" + rebTopVer +
+                            ", evt=" + exchId.discoveryEventName() +
+                            ", evtNode=" + exchId.nodeId() +
+                            ", client=" + cctx.kernalContext().clientNode() + ']');
                 }
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index 5fa7a821c69ef..d629e94db7e84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -24,6 +24,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.ForceRebalanceExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
@@ -62,10 +63,17 @@ public interface GridCachePreloader {
      */
     public void onInitialExchangeComplete(@Nullable Throwable err);
 
+    /**
+     * @param rebTopVer Previous rebalance topology version or {@code NONE} if there is no info.
+     * @param exchFut Completed exchange future.
+     * @return {@code True} if rebalance should be started (previous will be interrupted).
+     */
+    public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer, GridDhtPartitionsExchangeFuture exchFut);
+
     /**
      * @param exchId Exchange ID.
-     * @param exchFut Exchange future.
-     * @return Assignments or {@code null} if detected that there are pending exchanges.
+     * @param exchFut Completed exchange future. Can be {@code null} if forced or reassigned generation occurs.
+     * @return Partition assignments which will be requested from supplier nodes.
      */
     @Nullable public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      @Nullable GridDhtPartitionsExchangeFuture exchFut);
@@ -74,10 +82,10 @@ public interface GridCachePreloader {
      * Adds assignments to preloader.
      *
      * @param assignments Assignments to add.
-     * @param forcePreload Force preload flag.
-     * @param rebalanceId Rebalance id.
-     * @param next Runnable responsible for cache rebalancing start.
-     * @param forcedRebFut Rebalance future.
+     * @param forcePreload {@code True} if preload requested by {@link ForceRebalanceExchangeTask}.
+     * @param rebalanceId Rebalance id created by exchange thread.
+     * @param next Runnable responsible for cache rebalancing chain.
+     * @param forcedRebFut External future for forced rebalance.
      * @return Rebalancing runnable.
      */
     public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
@@ -114,7 +122,6 @@ public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
      * Future result is {@code false} in case rebalancing cancelled or finished with missed partitions and will be
      * restarted at current or pending topology.
      *
-     * Note that topology change creates new futures and finishes previous.
      */
     public IgniteInternalFuture<Boolean> rebalanceFuture();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index af916791daa3e..c5e4a817d0418 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -151,6 +151,12 @@ public GridCachePreloaderAdapter(CacheGroupContext grp) {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
+        GridDhtPartitionsExchangeFuture exchFut) {
+        return true;
+    }
+
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      GridDhtPartitionsExchangeFuture exchFut) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 1eeebaef3a79e..54d3c93ed2f61 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -235,12 +235,12 @@ else if (log.isDebugEnabled())
 
     /**
      * @param fut Future.
-     * @return {@code True} if topology changed.
+     * @return {@code True} if rebalance topology version changed by exchange thread or force
+     * reassing exchange occurs, see {@link RebalanceReassignExchangeTask} for details.
      */
     private boolean topologyChanged(RebalanceFuture fut) {
-        return
-            !grp.affinity().lastVersion().equals(fut.topologyVersion()) || // Topology already changed.
-                fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
+        return !ctx.exchange().rebalanceTopologyVersion().equals(fut.topVer) ||
+            fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
     }
 
     /**
@@ -253,14 +253,21 @@ void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
     }
 
     /**
-     * Initiates new rebalance process from given {@code assignments}.
-     * If previous rebalance is not finished method cancels it.
-     * In case of delayed rebalance method schedules new with configured delay.
+     * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
+     */
+    Collection<UUID> remainingNodes() {
+        return rebalanceFut.remainingNodes();
+    }
+
+    /**
+     * This method initiates new rebalance process from given {@code assignments} by creating new rebalance
+     * future based on them. Cancels previous rebalance future and sends rebalance started event.
+     * In case of delayed rebalance method schedules the new one with configured delay based on {@code lastExchangeFut}.
      *
-     * @param assignments Assignments.
-     * @param force {@code True} if dummy reassign.
-     * @param rebalanceId Rebalance id.
-     * @param next Runnable responsible for cache rebalancing start.
+     * @param assignments Assignments to process.
+     * @param force {@code True} if preload request by {@link ForceRebalanceExchangeTask}.
+     * @param rebalanceId Rebalance id generated from exchange thread.
+     * @param next Runnable responsible for cache rebalancing chain.
      * @param forcedRebFut External future for forced rebalance.
      * @return Rebalancing runnable.
      */
@@ -440,17 +447,7 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
             if (fut.isDone())
                 return;
 
-            // Must add all remaining node before send first request, for avoid race between add remaining node and
-            // processing response, see checkIsDone(boolean).
-            for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assignments.entrySet()) {
-                UUID nodeId = e.getKey().id();
-
-                IgniteDhtDemandedPartitionsMap parts = e.getValue().partitions();
-
-                assert parts != null : "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + nodeId + "]";
-
-                fut.remaining.put(nodeId, new T2<>(U.currentTimeMillis(), parts));
-            }
+            fut.remaining.forEach((key, value) -> value.set1(U.currentTimeMillis()));
         }
 
         final CacheConfiguration cfg = grp.config();
@@ -979,6 +976,13 @@ public static class RebalanceFuture extends GridFutureAdapter<Boolean> {
             exchId = assignments.exchangeId();
             topVer = assignments.topologyVersion();
 
+            assignments.forEach((k, v) -> {
+                assert v.partitions() != null :
+                    "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + k.id() + "]";
+
+                remaining.put(k.id(), new T2<>(U.currentTimeMillis(), v.partitions()));
+            });
+
             this.grp = grp;
             this.log = log;
             this.rebalanceId = rebalanceId;
@@ -1217,6 +1221,13 @@ private void checkIsDone(boolean cancelled) {
             }
         }
 
+        /**
+         * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
+         */
+        private synchronized Collection<UUID> remainingNodes() {
+            return remaining.keySet();
+        }
+
         /**
          *
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 4946d7e3f0eed..ea7f4c9b4aacd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -17,12 +17,14 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
@@ -118,19 +120,20 @@ private static void clearContext(
     }
 
     /**
-     * Handles new topology version and clears supply context map of outdated contexts.
-     *
-     * @param topVer Topology version.
+     * Handle topology change and clear supply context map of outdated contexts.
      */
-    @SuppressWarnings("ConstantConditions")
-    void onTopologyChanged(AffinityTopologyVersion topVer) {
+    void onTopologyChanged() {
         synchronized (scMap) {
             Iterator<T3<UUID, Integer, AffinityTopologyVersion>> it = scMap.keySet().iterator();
 
+            Collection<UUID> aliveNodes = grp.shared().discovery().aliveServerNodes().stream()
+                .map(ClusterNode::id)
+                .collect(Collectors.toList());
+
             while (it.hasNext()) {
                 T3<UUID, Integer, AffinityTopologyVersion> t = it.next();
 
-                if (topVer.compareTo(t.get3()) > 0) { // Clear all obsolete contexts.
+                if (!aliveNodes.contains(t.get1())) { // Clear all obsolete contexts.
                     clearContext(scMap.get(t), log);
 
                     it.remove();
@@ -171,17 +174,6 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
         AffinityTopologyVersion curTop = grp.affinity().lastVersion();
         AffinityTopologyVersion demTop = d.topologyVersion();
 
-        if (curTop.compareTo(demTop) > 0) {
-            if (log.isDebugEnabled())
-                log.debug("Demand request outdated [grp=" + grp.cacheOrGroupName()
-                        + ", currentTopVer=" + curTop
-                        + ", demandTopVer=" + demTop
-                        + ", from=" + nodeId
-                        + ", topicId=" + topicId + "]");
-
-            return;
-        }
-
         T3<UUID, Integer, AffinityTopologyVersion> contextId = new T3<>(nodeId, topicId, demTop);
 
         if (d.rebalanceId() < 0) { // Demand node requested context cleanup.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 77f48661d588b..7cf55a35536fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -24,6 +24,7 @@
 import java.util.UUID;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -39,7 +40,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -160,11 +160,67 @@ private IgniteCheckedException stopError() {
 
     /** {@inheritDoc} */
     @Override public void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
-        supplier.onTopologyChanged(lastFut.initialVersion());
+        supplier.onTopologyChanged();
 
         demander.onTopologyChanged(lastFut);
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
+        GridDhtPartitionsExchangeFuture exchFut) {
+        if (ctx.kernalContext().clientNode() || rebTopVer.equals(AffinityTopologyVersion.NONE))
+            return false; // No-op.
+
+        if (exchFut.localJoinExchange())
+            return true; // Required, can have outdated updSeq partition counter if node reconnects.
+
+        if (!grp.affinity().cachedVersions().contains(rebTopVer)) {
+            assert rebTopVer.compareTo(grp.localStartVersion()) <= 0 :
+                "Empty hisroty allowed only for newly started cache group [rebTopVer=" + rebTopVer +
+                    ", localStartTopVer=" + grp.localStartVersion() + ']';
+
+            return true; // Required, since no history info available.
+        }
+
+        final IgniteInternalFuture<Boolean> rebFut = rebalanceFuture();
+
+        if (rebFut.isDone() && !rebFut.result())
+            return true; // Required, previous rebalance cancelled.
+
+        final AffinityTopologyVersion exchTopVer = exchFut.context().events().topologyVersion();
+
+        Collection<UUID> aliveNodes = ctx.discovery().aliveServerNodes().stream()
+            .map(ClusterNode::id)
+            .collect(Collectors.toList());
+
+        return assignmentsChanged(rebTopVer, exchTopVer) ||
+            !aliveNodes.containsAll(demander.remainingNodes()); // Some of nodes left before rabalance compelete.
+    }
+
+    /**
+     * @param oldTopVer Previous topology version.
+     * @param newTopVer New topology version to check result.
+     * @return {@code True} if affinity assignments changed between two versions for local node.
+     */
+    private boolean assignmentsChanged(AffinityTopologyVersion oldTopVer, AffinityTopologyVersion newTopVer) {
+        final AffinityAssignment aff = grp.affinity().readyAffinity(newTopVer);
+
+        // We should get affinity assignments based on previous rebalance to calculate difference.
+        // Whole history size described by IGNITE_AFFINITY_HISTORY_SIZE constant.
+        final AffinityAssignment prevAff = grp.affinity().cachedVersions().contains(oldTopVer) ?
+            grp.affinity().cachedAffinity(oldTopVer) : null;
+
+        if (prevAff == null)
+            return false;
+
+        boolean assignsChanged = false;
+
+        for (int p = 0; !assignsChanged && p < grp.affinity().partitions(); p++)
+            assignsChanged |= aff.get(p).contains(ctx.localNode()) != prevAff.get(p).contains(ctx.localNode());
+
+        return assignsChanged;
+    }
+
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId, GridDhtPartitionsExchangeFuture exchFut) {
         assert exchFut == null || exchFut.isDone();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
index 41dd076c3f745..6e847bb0e1852 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
@@ -20,7 +20,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
@@ -73,9 +73,9 @@ GridDhtPartitionExchangeId exchangeId() {
     }
 
     /**
-     * @return Topology version.
+     * @return Topology version based on last {@link GridDhtPartitionTopologyImpl#readyTopVer}.
      */
-    AffinityTopologyVersion topologyVersion() {
+    public AffinityTopologyVersion topologyVersion() {
         return topVer;
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
index 565317720d2a9..46b09ac3b1819 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
@@ -149,7 +149,6 @@ public void testBaselineNodes() throws Exception {
     private void resetBlt() throws Exception {
         resetBaselineTopology();
 
-        waitForRebalancing();
         awaitPartitionMapExchange();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
index 5c601a1bd3bbf..0b441dd94c0e8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
@@ -103,14 +103,14 @@ public void testCacheValidatorMetrics() throws Exception {
 
         startGrid(2);
 
-        waitForRebalancing();
+        awaitPartitionMapExchange();
 
         assertCacheStatus(CACHE_NAME_1, true, true);
         assertCacheStatus(CACHE_NAME_2, true, true);
 
         stopGrid(1);
 
-        waitForRebalancing();
+        awaitPartitionMapExchange();
 
         // Invalid for writing due to invalid topology.
         assertCacheStatus(CACHE_NAME_1, true, false);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
index 83eff893b8894..23ba4b31b4e0a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
@@ -22,29 +22,23 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.events.CacheRebalancingEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -58,9 +52,6 @@
 import static org.apache.ignite.configuration.CacheConfiguration.DFLT_REBALANCE_BATCH_SIZE;
 import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS;
 import static org.apache.ignite.events.EventType.EVTS_CACHE_REBALANCE;
-import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
-import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
-import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
@@ -141,15 +132,6 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) {
         return cacheCfg;
     }
 
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-//        resetLog4j(Level.DEBUG, true,
-//            // Categories.
-//            GridDhtPreloader.class.getPackage().getName(),
-//            GridDhtPartitionTopologyImpl.class.getName(),
-//            GridDhtLocalPartition.class.getName());
-    }
-
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         backups = DFLT_BACKUPS;
@@ -227,11 +209,6 @@ public void testActivePartitionTransferAsyncRandomCoordinator() throws Exception
      */
     private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
-//        resetLog4j(Level.DEBUG, true,
-//            // Categories.
-//            GridDhtPreloader.class.getPackage().getName(),
-//            GridDhtPartitionTopologyImpl.class.getName(),
-//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -270,8 +247,6 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
             info(">>> Finished checking nodes [keyCnt=" + keyCnt + ", nodeCnt=" + nodeCnt + ", grids=" +
                 U.grids2names(ignites) + ']');
 
-            Collection<IgniteFuture<?>> futs = new LinkedList<>();
-
             Ignite last = F.last(ignites);
 
             for (Iterator<Ignite> it = ignites.iterator(); it.hasNext(); ) {
@@ -285,21 +260,8 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
 
                 checkActiveState(ignites);
 
-                final UUID nodeId = g.cluster().localNode().id();
-
                 it.remove();
 
-                futs.add(waitForLocalEvent(last.events(), new P1<Event>() {
-                    @Override public boolean apply(Event e) {
-                        CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
-
-                        ClusterNode node = evt.discoveryNode();
-
-                        return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
-                            (evt.discoveryEventType() == EVT_NODE_LEFT || evt.discoveryEventType() == EVT_NODE_FAILED);
-                    }
-                }, EVT_CACHE_REBALANCE_STOPPED));
-
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
@@ -312,14 +274,6 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
                 awaitPartitionMapExchange(); // Need wait, otherwise test logic is broken if EVT_NODE_FAILED exchanges are merged.
             }
 
-            info("Waiting for preload futures: " + F.view(futs, new IgnitePredicate<IgniteFuture<?>>() {
-                @Override public boolean apply(IgniteFuture<?> fut) {
-                    return !fut.isDone();
-                }
-            }));
-
-            X.waitAll(futs);
-
             info("Finished waiting for preload futures.");
 
             assert last != null;
@@ -499,11 +453,6 @@ private void stopGrids(Iterable<Ignite> grids) {
      */
     private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
-//        resetLog4j(Level.DEBUG, true,
-//            // Categories.
-//            GridDhtPreloader.class.getPackage().getName(),
-//            GridDhtPartitionTopologyImpl.class.getName(),
-//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -555,28 +504,13 @@ private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuf
 
                 it.remove();
 
-                Collection<IgniteFuture<?>> futs = new LinkedList<>();
-
-                for (Ignite gg : ignites)
-                    futs.add(waitForLocalEvent(gg.events(), new P1<Event>() {
-                        @Override public boolean apply(Event e) {
-                            CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
-
-                            ClusterNode node = evt.discoveryNode();
-
-                            return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
-                                (evt.discoveryEventType() == EVT_NODE_LEFT ||
-                                    evt.discoveryEventType() == EVT_NODE_FAILED);
-                        }
-                    }, EVT_CACHE_REBALANCE_STOPPED));
-
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
 
                 info(">>> Waiting for preload futures [leftNode=" + g.name() + ", remaining=" + U.grids2names(ignites) + ']');
 
-                X.waitAll(futs);
+                awaitPartitionMapExchange();
 
                 info("After grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java
index 0bfb4fbd06471..14c85717ae857 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java
@@ -806,6 +806,7 @@ private void readerUpdateDhtFails(boolean updateNearEnabled,
 
         startServers(2);
 
+        // Waiting for minor topology changing because of late affinity assignment.
         awaitPartitionMapExchange();
 
         Ignite srv0 = ignite(0);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
index 4ebcd5df35271..0a8698a597cd8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
@@ -17,10 +17,11 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
 
-import org.apache.ignite.Ignite;
+import java.util.Collections;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
@@ -43,7 +44,7 @@ public class GridCacheRebalancingAsyncSelfTest extends GridCacheRebalancingSyncS
      * @throws Exception Exception.
      */
     public void testNodeFailedAtRebalancing() throws Exception {
-        Ignite ignite = startGrid(0);
+        IgniteEx ignite = startGrid(0);
 
         generateData(ignite, 0, 0);
 
@@ -60,7 +61,7 @@ public void testNodeFailedAtRebalancing() throws Exception {
 
         ((TestTcpDiscoverySpi)grid(1).configuration().getDiscoverySpi()).simulateNodeFailure();
 
-        waitForRebalancing(0, 3);
+        awaitPartitionMapExchange(false, false, Collections.singletonList(ignite.localNode()));
 
         checkSupplyContextMapIsEmpty();
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
new file mode 100644
index 0000000000000..3965290480a3d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
@@ -0,0 +1,106 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test cases for checking cancellation rebalancing process if some events occurs.
+ */
+public class GridCacheRebalancingCancelTest extends GridCommonAbstractTest {
+    /** */
+    private static final String DHT_PARTITIONED_CACHE = "cacheP";
+
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration dfltCfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)dfltCfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        dfltCfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        return dfltCfg;
+    }
+
+    /**
+     * Test rebalance not cancelled when client node join to cluster.
+     *
+     * @throws Exception Exception.
+     */
+    public void testClientNodeJoinAtRebalancing() throws Exception {
+        final IgniteEx ignite0 = startGrid(0);
+
+        IgniteCache<Integer, Integer> cache = ignite0.createCache(
+            new CacheConfiguration<Integer, Integer>(DHT_PARTITIONED_CACHE)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setRebalanceMode(CacheRebalanceMode.ASYNC)
+                .setBackups(1)
+                .setRebalanceOrder(2)
+                .setAffinity(new RendezvousAffinityFunction(false)));
+
+        for (int i = 0; i < 2048; i++)
+            cache.put(i, i);
+
+        TestRecordingCommunicationSpi.spi(ignite0)
+            .blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+                @Override public boolean apply(ClusterNode node, Message msg) {
+                    return (msg instanceof GridDhtPartitionSupplyMessage)
+                        && ((GridCacheGroupIdMessage)msg).groupId() == groupIdForCache(ignite0, DHT_PARTITIONED_CACHE);
+                }
+            });
+
+        final IgniteEx ignite1 = startGrid(1);
+
+        TestRecordingCommunicationSpi.spi(ignite0).waitForBlocked();
+
+        GridDhtPartitionDemander.RebalanceFuture fut = (GridDhtPartitionDemander.RebalanceFuture)ignite1.context().
+            cache().internalCache(DHT_PARTITIONED_CACHE).preloader().rebalanceFuture();
+
+        String igniteClntName = getTestIgniteInstanceName(2);
+
+        startGrid(igniteClntName, optimize(getConfiguration(igniteClntName).setClientMode(true)));
+
+        // Resend delayed rebalance messages.
+        TestRecordingCommunicationSpi.spi(ignite0).stopBlock(true);
+
+        awaitPartitionMapExchange();
+
+        // Previous rebalance future should not be cancelled.
+        assertTrue(fut.result());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
index 1280e87f9f091..cb414eda13bf5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
@@ -141,7 +141,8 @@ else if ("index.bin".equals(f.getName())) {
         assertTrue(primaryRemoved);
 
         ignite.cluster().active(true);
-        waitForRebalancing();
+
+        awaitPartitionMapExchange();
 
         List<String> issues = new ArrayList<>();
         HashMap<Integer, Long> partMap = new HashMap<>();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
index ed51cf392a992..a027a41cfb463 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
@@ -21,10 +21,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cluster.ClusterNode;
@@ -42,11 +41,13 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -68,6 +69,9 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private static final int TEST_SIZE = 100_000;
 
+    /** */
+    private static final long TOPOLOGY_STILLNESS_TIME = 30_000L;
+
     /** partitioned cache name. */
     protected static final String CACHE_NAME_DHT_PARTITIONED = "cacheP";
 
@@ -89,11 +93,11 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private volatile boolean concurrentStartFinished3;
 
-    /** */
-    private volatile boolean record;
-
-    /** */
-    private final ConcurrentHashMap<Class, AtomicInteger> map = new ConcurrentHashMap<>();
+    /**
+     * Time in milliseconds of last received {@link GridDhtPartitionsSingleMessage}
+     * or {@link GridDhtPartitionsFullMessage} using {@link CollectingCommunicationSpi}.
+     */
+    private static volatile long lastPartMsgTime;
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -102,7 +106,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setIpFinder(ipFinder);
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setForceServerMode(true);
 
-        TcpCommunicationSpi commSpi = new CountingCommunicationSpi();
+        TcpCommunicationSpi commSpi = new CollectingCommunicationSpi();
 
         commSpi.setLocalPort(GridTestUtils.getNextCommPort(getClass()));
         commSpi.setTcpNoDelay(true);
@@ -232,47 +236,35 @@ public void testSimpleRebalancing() throws Exception {
 
         startGrid(1);
 
-        int waitMinorVer = ignite.configuration().isLateAffinityAssignment() ? 1 : 0;
-
-        waitForRebalancing(0, 2, waitMinorVer);
-        waitForRebalancing(1, 2, waitMinorVer);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         stopGrid(0);
 
-        waitForRebalancing(1, 3);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         startGrid(2);
 
-        waitForRebalancing(1, 4, waitMinorVer);
-        waitForRebalancing(2, 4, waitMinorVer);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         stopGrid(2);
 
-        waitForRebalancing(1, 5);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         long spend = (System.currentTimeMillis() - start) / 1000;
 
@@ -331,13 +323,10 @@ public void testLoadRebalancing() throws Exception {
 
         startGrid(4);
 
-        waitForRebalancing(3, 6);
-        waitForRebalancing(4, 6);
+        awaitPartitionMapExchange(true, true, null);
 
         concurrentStartFinished = true;
 
-        awaitPartitionMapExchange(true, true, null);
-
         checkSupplyContextMapIsEmpty();
 
         t1.join();
@@ -442,27 +431,29 @@ protected void checkPartitionMapExchangeFinished() {
     }
 
     /**
+     * Method checks for {@link GridDhtPartitionsSingleMessage} or {@link GridDhtPartitionsFullMessage}
+     * not received within {@code TOPOLOGY_STILLNESS_TIME} bound.
+     *
      * @throws Exception If failed.
      */
-    protected void checkPartitionMapMessagesAbsent() throws Exception {
-        map.clear();
-
-        record = true;
-
-        log.info("Checking GridDhtPartitions*Message absent (it will take 30 SECONDS) ... ");
-
-        U.sleep(30_000);
-
-        record = false;
-
-        AtomicInteger iF = map.get(GridDhtPartitionsFullMessage.class);
-        AtomicInteger iS = map.get(GridDhtPartitionsSingleMessage.class);
-
-        Integer fullMap = iF != null ? iF.get() : null;
-        Integer singleMap = iS != null ? iS.get() : null;
-
-        assertTrue("Unexpected full map messages: " + fullMap, fullMap == null || fullMap.equals(1)); // 1 message can be sent right after all checks passed.
-        assertNull("Unexpected single map messages", singleMap);
+    protected void awaitPartitionMessagesAbsent() throws Exception {
+        log.info("Checking GridDhtPartitions*Message absent (it will take up to " +
+            TOPOLOGY_STILLNESS_TIME + " ms) ... ");
+
+        // Start waiting new messages from current point of time.
+        lastPartMsgTime = U.currentTimeMillis();
+
+        assertTrue("Should not have partition Single or Full messages within bound " +
+                TOPOLOGY_STILLNESS_TIME + " ms.",
+            GridTestUtils.waitForCondition(
+                new GridAbsPredicateX() {
+                    @Override public boolean applyx() {
+                        return lastPartMsgTime + TOPOLOGY_STILLNESS_TIME < U.currentTimeMillis();
+                    }
+                },
+                2 * TOPOLOGY_STILLNESS_TIME // 30 sec to gain stable topology and 30 sec of silence.
+            )
+        );
     }
 
     /** {@inheritDoc} */
@@ -495,11 +486,7 @@ public void testComplexRebalancing() throws Exception {
                     while (!concurrentStartFinished2)
                         U.sleep(10);
 
-                    waitForRebalancing(0, 5, 0);
-                    waitForRebalancing(1, 5, 0);
-                    waitForRebalancing(2, 5, 0);
-                    waitForRebalancing(3, 5, 0);
-                    waitForRebalancing(4, 5, 0);
+                    awaitPartitionMapExchange();
 
                     //New cache should start rebalancing.
                     CacheConfiguration<Integer, Integer> cacheRCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
@@ -552,12 +539,6 @@ public void testComplexRebalancing() throws Exception {
         t2.join();
         t3.join();
 
-        waitForRebalancing(0, 5, 1);
-        waitForRebalancing(1, 5, 1);
-        waitForRebalancing(2, 5, 1);
-        waitForRebalancing(3, 5, 1);
-        waitForRebalancing(4, 5, 1);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
@@ -577,35 +558,23 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(1);
 
-        waitForRebalancing(0, 6);
-        waitForRebalancing(2, 6);
-        waitForRebalancing(3, 6);
-        waitForRebalancing(4, 6);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(0);
 
-        waitForRebalancing(2, 7);
-        waitForRebalancing(3, 7);
-        waitForRebalancing(4, 7);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(2);
 
-        waitForRebalancing(3, 8);
-        waitForRebalancing(4, 8);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         checkSupplyContextMapIsEmpty();
 
@@ -613,7 +582,7 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(3);
 
-        waitForRebalancing(4, 9);
+        awaitPartitionMapExchange();
 
         checkSupplyContextMapIsEmpty();
 
@@ -634,36 +603,26 @@ public void testComplexRebalancing() throws Exception {
     /**
      *
      */
-    private class CountingCommunicationSpi extends TcpCommunicationSpi {
+    private static class CollectingCommunicationSpi extends TcpCommunicationSpi {
+        /** */
+        @LoggerResource
+        private IgniteLogger log;
+
         /** {@inheritDoc} */
         @Override public void sendMessage(final ClusterNode node, final Message msg,
             final IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
             final Object msg0 = ((GridIoMessage)msg).message();
 
-            recordMessage(msg0);
+            if (msg0 instanceof GridDhtPartitionsSingleMessage ||
+                msg0 instanceof GridDhtPartitionsFullMessage) {
+                lastPartMsgTime = U.currentTimeMillis();
 
-            super.sendMessage(node, msg, ackC);
-        }
-
-        /**
-         * @param msg Message.
-         */
-        private void recordMessage(Object msg) {
-            if (record) {
-                Class id = msg.getClass();
-
-                AtomicInteger ai = map.get(id);
-
-                if (ai == null) {
-                    ai = new AtomicInteger();
-
-                    AtomicInteger oldAi = map.putIfAbsent(id, ai);
-
-                    (oldAi != null ? oldAi : ai).incrementAndGet();
-                }
-                else
-                    ai.incrementAndGet();
+                log.info("Last seen time of GridDhtPartitionsSingleMessage or GridDhtPartitionsFullMessage updated " +
+                    "[lastPartMsgTime=" + lastPartMsgTime +
+                    ", node=" + node.id() + ']');
             }
+
+            super.sendMessage(node, msg, ackC);
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
index 15ec41557dfba..ce84156e56dca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
@@ -383,7 +383,7 @@ public void testRejoinWithCleanLfs() throws Exception {
         startGrid("flaky");
 
         System.out.println("### Starting rebalancing after flaky node join");
-        waitForRebalancing();
+        awaitPartitionMapExchange();
         System.out.println("### Rebalancing is finished after flaky node join");
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
@@ -689,7 +689,7 @@ private void tryChangeBaselineDown(
         ig0.cluster().setBaselineTopology(fullBlt.subList(0, newBaselineSize));
 
         System.out.println("### Starting rebalancing after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
-        waitForRebalancing();
+        awaitPartitionMapExchange();
         System.out.println("### Rebalancing is finished after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
index 8f2e7389f8bb3..3500c8de7f7ed 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
@@ -157,7 +157,8 @@ public void testReservation() throws Exception {
 
         SUPPLY_MESSAGE_LATCH.get().countDown();
 
-        waitForRebalancing(); // Partition is OWNING on grid(0) and grid(1)
+        // Partition is OWNING on grid(0) and grid(1)
+        awaitPartitionMapExchange();
 
         for (int i = 0; i < 2; i++) {
             for (int j = 0; i < 500; i++)
@@ -178,7 +179,7 @@ public void testReservation() throws Exception {
 
         startGrid(0);
 
-        waitForRebalancing();
+        awaitPartitionMapExchange();
 
         assertEquals(2, grid(1).context().discovery().aliveServerNodes().size());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index d585a82e66a2f..0ed39ce003f4e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -192,7 +192,7 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
 
                     grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
 
-                    waitForRebalancing();
+                    awaitPartitionMapExchange();
                 }
                 catch (Throwable expected) {
                     // There can be any exception. Do nothing.
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index 78f3c03eda818..9de2702d3790e 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -120,7 +120,8 @@ public void testMappingsPersistedOnJoin() throws Exception {
         c1.put(k, new DummyObject(k));
 
         startGrid(2);
-        waitForRebalancing();
+
+        awaitPartitionMapExchange();
 
         stopAllGrids();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 29ad91d167576..f8f1c9c2f0f04 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteEvents;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.CachePeekMode;
@@ -61,6 +62,7 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityFunctionContextImpl;
@@ -102,6 +104,9 @@
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.testframework.GridTestNode;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.GridAbstractTest;
@@ -681,32 +686,34 @@ protected void awaitPartitionMapExchange(
 
                                 if (affNodesCnt != ownerNodesCnt || !affNodes.containsAll(owners) ||
                                     (waitEvicts && loc != null && loc.state() != GridDhtPartitionState.OWNING)) {
+                                    if (i % 50 == 0)
+                                        LT.warn(log(), "Waiting for topology map update [" +
+                                            "igniteInstanceName=" + g.name() +
+                                            ", cache=" + cfg.getName() +
+                                            ", cacheId=" + dht.context().cacheId() +
+                                            ", topVer=" + top.readyTopologyVersion() +
+                                            ", p=" + p +
+                                            ", affNodesCnt=" + affNodesCnt +
+                                            ", ownersCnt=" + ownerNodesCnt +
+                                            ", affNodes=" + F.nodeIds(affNodes) +
+                                            ", owners=" + F.nodeIds(owners) +
+                                            ", topFut=" + topFut +
+                                            ", locNode=" + g.cluster().localNode() + ']');
+                                }
+                                else
+                                    match = true;
+                            }
+                            else {
+                                if (i % 50 == 0)
                                     LT.warn(log(), "Waiting for topology map update [" +
                                         "igniteInstanceName=" + g.name() +
                                         ", cache=" + cfg.getName() +
                                         ", cacheId=" + dht.context().cacheId() +
                                         ", topVer=" + top.readyTopologyVersion() +
+                                        ", started=" + dht.context().started() +
                                         ", p=" + p +
-                                        ", affNodesCnt=" + affNodesCnt +
-                                        ", ownersCnt=" + ownerNodesCnt +
-                                        ", affNodes=" + F.nodeIds(affNodes) +
-                                        ", owners=" + F.nodeIds(owners) +
-                                        ", topFut=" + topFut +
+                                        ", readVer=" + readyVer +
                                         ", locNode=" + g.cluster().localNode() + ']');
-                                }
-                                else
-                                    match = true;
-                            }
-                            else {
-                                LT.warn(log(), "Waiting for topology map update [" +
-                                    "igniteInstanceName=" + g.name() +
-                                    ", cache=" + cfg.getName() +
-                                    ", cacheId=" + dht.context().cacheId() +
-                                    ", topVer=" + top.readyTopologyVersion() +
-                                    ", started=" + dht.context().started() +
-                                    ", p=" + p +
-                                    ", readVer=" + readyVer +
-                                    ", locNode=" + g.cluster().localNode() + ']');
                             }
 
                             if (!match) {
@@ -868,7 +875,7 @@ protected void printPartitionState(String cacheName, int firstParts) {
                     .append(" res=").append(f.isDone() ? f.get() : "N/A")
                     .append(" topVer=")
                     .append((U.hasField(f, "topVer") ?
-                        String.valueOf(U.field(f, "topVer")) : "[unknown] may be it is finished future"))
+                        String.valueOf(U.<Object>field(f, "topVer")) : "[unknown] may be it is finished future"))
                     .append("\n");
 
                 Map<UUID, T2<Long, Collection<Integer>>> remaining = U.field(f, "remaining");
@@ -952,6 +959,61 @@ protected void printPartitionState(String cacheName, int firstParts) {
         log.info("dump partitions state for <" + cacheName + ">:\n" + sb.toString());
     }
 
+    /**
+     * Use method for manual rebalaincing cache on all nodes. Note that using
+     * <pre name="code" class="java">
+     *   for (int i = 0; i < G.allGrids(); i++)
+     *     grid(i).cache(CACHE_NAME).rebalance().get();
+     * </pre>
+     * for rebalancing cache will lead to flaky test cases.
+     *
+     * @param ignite Ignite server instance for getting {@code compute} facade over all cluster nodes.
+     * @param cacheName Cache name for manual rebalancing on cluster. Usually used when used when
+     * {@link CacheConfiguration#getRebalanceDelay()} configuration parameter set to {@code -1} value.
+     * @throws IgniteCheckedException If fails.
+     */
+    protected void manualCacheRebalancing(Ignite ignite,
+        final String cacheName) throws IgniteCheckedException {
+        if (ignite.configuration().isClientMode())
+            return;
+
+        IgniteFuture<Void> fut =
+            ignite.compute().withTimeout(5_000).broadcastAsync(new IgniteRunnable() {
+                /** */
+                @LoggerResource
+                IgniteLogger log;
+
+                /** */
+                @IgniteInstanceResource
+                private Ignite ignite;
+
+                /** {@inheritDoc} */
+                @Override public void run() {
+                    IgniteCache<?, ?> cache = ignite.cache(cacheName);
+
+                    assertNotNull(cache);
+
+                    while (!(Boolean)cache.rebalance().get()) {
+                        try {
+                            U.sleep(100);
+                        }
+                        catch (IgniteInterruptedCheckedException e) {
+                            throw new IgniteException(e);
+                        }
+                    }
+
+                    if (log.isInfoEnabled())
+                        log.info("Manual rebalance finished [node=" + ignite.name() + ", cache=" + cacheName + "]");
+                }
+            });
+
+        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return fut.isDone();
+            }
+        }, 5_000));
+    }
+
     /**
      * @param id Node id.
      * @param major Major ver.
@@ -1000,13 +1062,13 @@ protected void waitForRebalancing(IgniteEx ignite, AffinityTopologyVersion top)
 
             for (GridCacheAdapter c : ignite.context().cache().internalCaches()) {
                 GridDhtPartitionDemander.RebalanceFuture fut =
-                    (GridDhtPartitionDemander.RebalanceFuture)c.preloader().rebalanceFuture();
+                        (GridDhtPartitionDemander.RebalanceFuture)c.preloader().rebalanceFuture();
 
                 if (fut.topologyVersion() == null || fut.topologyVersion().compareTo(top) < 0) {
                     finished = false;
 
                     log.info("Unexpected future version, will retry [futVer=" + fut.topologyVersion() +
-                        ", expVer=" + top + ']');
+                            ", expVer=" + top + ']');
 
                     U.sleep(100);
 
@@ -1667,6 +1729,8 @@ protected static <T> T doInTransaction(Ignite ignite,
      *
      */
     protected void cleanPersistenceDir() throws Exception {
+        assertTrue("Grids are not stopped", F.isEmpty(G.allGrids()));
+
         U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "cp", false));
         U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
         U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", false));
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index b66bf5b5026fa..e386beb227acf 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -53,11 +53,10 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingAsyncSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingCancelTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheDaemonNodeReplicatedSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicGetAndTransformStoreSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedBasicApiTest;
@@ -84,6 +83,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.replicated.preloader.GridCacheReplicatedPreloadStartStopEventsSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheDaemonNodeLocalSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheLocalByteArrayValuesSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
 
 /**
  * Test suite.
@@ -94,6 +94,8 @@ public class IgniteCacheTestSuite3 extends TestSuite {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite() throws Exception {
+//        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = new TestSuite("IgniteCache Test Suite part 3");
 
         suite.addTestSuite(IgniteCacheGroupsTest.class);
@@ -151,8 +153,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
         suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
         suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
-        suite.addTestSuite(GridCacheRebalancingPartitionCountersTest.class);
-        suite.addTestSuite(GridCacheRebalancingWithAsyncClearingTest.class);
+        suite.addTestSuite(GridCacheRebalancingCancelTest.class);
 
         // Test for byte array value special case.
         suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);

From cfe95a1b1762501dd9ac48a1a24228ac0bc3b5c3 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 2 Aug 2018 15:51:17 +0300
Subject: [PATCH 281/543] IGNITE-9111 Do not wait for deactivation.

Signed-off-by: agura <agura@apache.org>

(cherry picked from commit 8811a14)
---
 .../cluster/DiscoveryDataClusterState.java    |  7 ---
 .../cluster/GridClusterStateProcessor.java    |  2 +-
 .../IgniteClusterActivateDeactivateTest.java  | 48 +++++++++++++++++++
 3 files changed, 49 insertions(+), 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
index 71357ad52a67e..b6b301e5fae6a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
@@ -172,13 +172,6 @@ public boolean transition() {
         return transitionReqId != null;
     }
 
-    /**
-     * @return {@code True} if cluster active state change is in progress, {@code false} otherwise.
-     */
-    public boolean activeStateChanging() {
-        return transition() && (prevState == null || (prevState.active != active));
-    }
-
     /**
      * @return State change exchange version.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index da0bbf6867c8c..8d2620f531056 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -179,7 +179,7 @@ public boolean compatibilityMode() {
 
         assert globalState != null;
 
-        if (globalState.transition() && globalState.activeStateChanging()) {
+        if (globalState.transition() && globalState.active()) {
             Boolean transitionRes = globalState.transitionResult();
 
             if (transitionRes != null)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 838e56deebdd3..501732cfab64f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -42,6 +42,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -53,6 +54,7 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
@@ -1173,6 +1175,52 @@ private void stateChangeFailover3(boolean activate) throws Exception {
         checkCaches1(6);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClusterStateNotWaitForDeactivation() throws Exception {
+        testSpi = true;
+
+        final int nodes = 2;
+
+        IgniteEx crd = (IgniteEx) startGrids(nodes);
+
+        crd.cluster().active(true);
+
+        AffinityTopologyVersion curTopVer = crd.context().discovery().topologyVersionEx();
+
+        AffinityTopologyVersion deactivationTopVer = new AffinityTopologyVersion(
+            curTopVer.topologyVersion(),
+            curTopVer.minorTopologyVersion() + 1
+        );
+
+        for (int gridIdx = 0; gridIdx < nodes; gridIdx++) {
+            TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(grid(gridIdx));
+
+            blockExchangeSingleMessage(spi, deactivationTopVer);
+        }
+
+        IgniteInternalFuture deactivationFut = GridTestUtils.runAsync(() -> crd.cluster().active(false));
+
+        // Wait for deactivation start.
+        GridTestUtils.waitForCondition(() -> {
+            DiscoveryDataClusterState clusterState = crd.context().state().clusterState();
+
+            return clusterState.transition() && !clusterState.active();
+        }, getTestTimeout());
+
+        // Check that deactivation transition wait is not happened.
+        Assert.assertFalse(crd.context().state().publicApiActiveState(true));
+
+        for (int gridIdx = 0; gridIdx < nodes; gridIdx++) {
+            TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(grid(gridIdx));
+
+            spi.stopBlock();
+        }
+
+        deactivationFut.get();
+    }
+
     /**
      * @param exp If {@code true} there should be recorded messages.
      */

From 2e43db33e09e1ed456ccaf3e9361e9b2f07f99a7 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 2 Aug 2018 16:18:44 +0300
Subject: [PATCH 282/543] IGNITE-9129 P2P class deployment with ZK discovery
 fixed.

Signed-off-by: agura <agura@apache.org>

(cherry picked from commit 0e9bb1c)
---
 .../continuous/GridContinuousProcessor.java   | 72 ++++++++++---------
 .../GridP2PContinuousDeploymentSelfTest.java  | 70 +++++++++++++++++-
 .../zk/ZookeeperDiscoverySpiTestSuite2.java   |  3 +
 3 files changed, 109 insertions(+), 36 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
index 2d48b7d9b16fb..6723ea42624c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousProcessor.java
@@ -544,7 +544,7 @@ private Map<UUID, LocalRoutineInfo> copyLocalInfos(Map<UUID, LocalRoutineInfo> l
      * @param routineInfo Routine info.
      */
     private void startDiscoveryDataRoutine(ContinuousRoutineInfo routineInfo) {
-        IgnitePredicate<ClusterNode> nodeFilter = null;
+        IgnitePredicate<ClusterNode> nodeFilter;
 
         try {
             if (routineInfo.nodeFilter != null) {
@@ -552,6 +552,8 @@ private void startDiscoveryDataRoutine(ContinuousRoutineInfo routineInfo) {
 
                 ctx.resource().injectGeneric(nodeFilter);
             }
+            else
+                nodeFilter = null;
         }
         catch (IgniteCheckedException e) {
             U.error(log, "Failed to unmarshal continuous routine filter, ignore routine [" +
@@ -561,45 +563,47 @@ private void startDiscoveryDataRoutine(ContinuousRoutineInfo routineInfo) {
             return;
         }
 
-        if (nodeFilter == null || nodeFilter.apply(ctx.discovery().localNode())) {
-            GridContinuousHandler hnd;
+        ctx.discovery().localJoinFuture().listen(f -> ctx.closure().runLocalSafe(() -> {
+            if (nodeFilter == null || nodeFilter.apply(ctx.discovery().localNode())) {
+                GridContinuousHandler hnd;
 
-            try {
-                hnd = U.unmarshal(marsh, routineInfo.hnd, U.resolveClassLoader(ctx.config()));
+                try {
+                    hnd = U.unmarshal(marsh, routineInfo.hnd, U.resolveClassLoader(ctx.config()));
 
-                if (ctx.config().isPeerClassLoadingEnabled())
-                    hnd.p2pUnmarshal(routineInfo.srcNodeId, ctx);
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to unmarshal continuous routine handler, ignore routine [" +
-                    "routineId=" + routineInfo.routineId +
-                    ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+                    if (ctx.config().isPeerClassLoadingEnabled())
+                        hnd.p2pUnmarshal(routineInfo.srcNodeId, ctx);
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to unmarshal continuous routine handler, ignore routine [" +
+                        "routineId=" + routineInfo.routineId +
+                        ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
 
-                return;
-            }
+                    return;
+                }
 
-            try {
-                registerHandler(routineInfo.srcNodeId,
-                    routineInfo.routineId,
-                    hnd,
-                    routineInfo.bufSize,
-                    routineInfo.interval,
-                    routineInfo.autoUnsubscribe,
-                    false);
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to register continuous routine handler, ignore routine [" +
-                    "routineId=" + routineInfo.routineId +
-                    ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+                try {
+                    registerHandler(routineInfo.srcNodeId,
+                        routineInfo.routineId,
+                        hnd,
+                        routineInfo.bufSize,
+                        routineInfo.interval,
+                        routineInfo.autoUnsubscribe,
+                        false);
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to register continuous routine handler, ignore routine [" +
+                        "routineId=" + routineInfo.routineId +
+                        ", srcNodeId=" + routineInfo.srcNodeId + ']', e);
+                }
             }
-        }
-        else {
-            if (log.isDebugEnabled()) {
-                log.debug("Do not register continuous routine, rejected by node filter [" +
-                    "routineId=" + routineInfo.routineId +
-                    ", srcNodeId=" + routineInfo.srcNodeId + ']');
+            else {
+                if (log.isDebugEnabled()) {
+                    log.debug("Do not register continuous routine, rejected by node filter [" +
+                        "routineId=" + routineInfo.routineId +
+                        ", srcNodeId=" + routineInfo.srcNodeId + ']');
+                }
             }
-        }
+        }));
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PContinuousDeploymentSelfTest.java b/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PContinuousDeploymentSelfTest.java
index 0c021b6684f6f..7b5bf1ffbd86d 100644
--- a/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PContinuousDeploymentSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/p2p/GridP2PContinuousDeploymentSelfTest.java
@@ -17,9 +17,14 @@
 
 package org.apache.ignite.p2p;
 
+import java.util.UUID;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -49,6 +54,12 @@ public class GridP2PContinuousDeploymentSelfTest extends GridCommonAbstractTest
     /** Second test task name. */
     private static final String TEST_TASK_2 = "org.apache.ignite.tests.p2p.GridP2PContinuousDeploymentTask2";
 
+    /** Test predicate. */
+    private static final String TEST_PREDICATE = "org.apache.ignite.tests.p2p.GridEventConsumeFilter";
+
+    /** Client mode. */
+    private boolean clientMode;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -66,6 +77,11 @@ public class GridP2PContinuousDeploymentSelfTest extends GridCommonAbstractTest
 
         cfg.setDiscoverySpi(disco);
 
+        cfg.setPeerClassLoadingEnabled(true);
+
+        if (clientMode)
+            cfg.setClientMode(true);
+
         return cfg;
     }
 
@@ -85,8 +101,13 @@ protected CacheConfiguration cacheConfiguration() throws Exception {
     }
 
     /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        startGridsMultiThreaded(GRID_CNT);
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
     }
 
     /** {@inheritDoc} */
@@ -99,6 +120,8 @@ protected CacheConfiguration cacheConfiguration() throws Exception {
      */
     @SuppressWarnings("unchecked")
     public void testDeployment() throws Exception {
+        startGridsMultiThreaded(GRID_CNT);
+
         Ignite ignite = startGrid(IGNITE_INSTANCE_NAME);
 
         Class cls = getExternalClassLoader().loadClass(TEST_TASK_1);
@@ -115,4 +138,47 @@ public void testDeployment() throws Exception {
 
         stopGrid(IGNITE_INSTANCE_NAME);
     }
+
+    /**
+     * Tests that server node joins correctly to existing cluster if it has deployed user class with enabled P2P.
+     *
+     * @throws Exception If failed.
+     */
+    public void testServerJoinWithP2PClassDeployedInCluster() throws Exception {
+        startGrids(GRID_CNT);
+
+        ClassLoader extLdr = getExternalClassLoader();
+
+        clientMode = true;
+
+        Ignite client = startGrid(2);
+
+        Class<?> cls = extLdr.loadClass(TEST_PREDICATE);
+
+        client.events().remoteListen(
+            new IgniteBiPredicate<UUID, Event>() {
+                @Override public boolean apply(UUID uuid, Event event) {
+                    return true;
+                }
+            },
+            (IgnitePredicate<Event>) cls.newInstance(),
+            EventType.EVT_CACHE_OBJECT_PUT
+        );
+
+        clientMode = false;
+
+        Ignite srv = startGrid(3);
+
+        srv.events().remoteListen(
+            new IgniteBiPredicate<UUID, Event>() {
+                @Override public boolean apply(UUID uuid, Event event) {
+                    return true;
+                }
+            },
+            (IgnitePredicate<Event>) cls.newInstance(),
+            EventType.EVT_CACHE_OBJECT_PUT
+        );
+
+        awaitPartitionMapExchange();
+    }
 }
\ No newline at end of file
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
index ddb003bec5366..012366f70abe6 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.internal.processors.cache.multijvm.GridCacheAtomicMultiJvmFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.multijvm.GridCachePartitionedMultiJvmFullApiSelfTest;
 import org.apache.ignite.internal.processors.continuous.GridEventConsumeSelfTest;
+import org.apache.ignite.p2p.GridP2PContinuousDeploymentSelfTest;
 import org.apache.ignite.util.GridCommandHandlerTest;
 
 /**
@@ -91,6 +92,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(GridCommandHandlerTest.class);
 
+        suite.addTestSuite(GridP2PContinuousDeploymentSelfTest.class);
+
         return suite;
     }
 }

From d1ae06b8a2adbb1890b7e0605708dddb52cd0d29 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 2 Aug 2018 18:57:22 +0300
Subject: [PATCH 283/543] IGNITE-8869
 PartitionsExchangeOnDiscoveryHistoryOverflowTest hangs on TeamCity. - Fixes
 #4277.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
(cherry picked from commit b718e4438e469ad0341f1bf9a605c2fe3ab5970c)
---
 .../preloader/latch/ExchangeLatchManager.java | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index 25424b8822ba7..e111a6edc7c5f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -30,7 +30,9 @@
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.GridTopic;
@@ -42,6 +44,7 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteProductVersion;
@@ -228,6 +231,28 @@ public Latch getOrCreate(String id, AffinityTopologyVersion topVer) {
         }
     }
 
+    /**
+     * Gets alive server nodes from disco cache for provided AffinityTopologyVersion.
+     *
+     * @param topVer Topology version.
+     * @return Collection of nodes with at least one cache configured.
+     */
+    private Collection<ClusterNode> aliveNodesForTopologyVer(AffinityTopologyVersion topVer) {
+        if (topVer == AffinityTopologyVersion.NONE)
+            return discovery.aliveServerNodes();
+        else {
+            Collection<ClusterNode> histNodes = discovery.topology(topVer.topologyVersion());
+
+            if (histNodes != null)
+                return histNodes.stream().filter(n -> !CU.clientNode(n) && !n.isDaemon() && discovery.alive(n))
+                        .collect(Collectors.toList());
+            else
+                throw new IgniteException("Topology " + topVer + " not found in discovery history "
+                        + "; consider increasing IGNITE_DISCOVERY_HISTORY_SIZE property. Current value is "
+                        + IgniteSystemProperties.getInteger(IgniteSystemProperties.IGNITE_DISCOVERY_HISTORY_SIZE, -1));
+        }
+    }
+
     /**
      * @param topVer Latch topology version.
      * @return Collection of alive server nodes with latch functionality.

From c9ebbb2c2b8f1a1ebaac21586fa56ea6f57a7bb0 Mon Sep 17 00:00:00 2001
From: Vitaliy Biryukov <BiryukovVitaliy92@gmail.com>
Date: Mon, 30 Jul 2018 20:33:04 +0300
Subject: [PATCH 284/543] IGNITE-9127 ZooKeeper (Discovery) 1 suite hang, JMX
 logging disabled for ZK - Fixes #4450.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
(cherry picked from commit 774ed773a6786cbcd4e6ca7f12290211f4f52201)
---
 .../spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java       | 1 +
 .../spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java    | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
index 7096cf9e1494d..fa602441b1168 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite1.java
@@ -54,6 +54,7 @@ public class ZookeeperDiscoverySpiTestSuite1 extends TestSuite {
      */
     public static TestSuite suite() throws Exception {
         System.setProperty("zookeeper.forceSync", "false");
+        System.setProperty("zookeeper.jmx.log4j.disable", "true");
 
         TestSuite suite = new TestSuite("ZookeeperDiscoverySpi Test Suite");
 
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index ba0fa27d7628b..035ca3b3d2d85 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -503,7 +503,7 @@ private static void waitForZkClusterReady(TestingCluster zkCluster) throws Inter
      * @throws Exception If failed.
      */
     private void checkInternalStructuresCleanup() throws Exception {
-        for (Ignite node : G.allGrids()) {
+        for (Ignite node : IgnitionEx.allGridsx()) {
             final AtomicReference<?> res = GridTestUtils.getFieldValue(spi(node), "impl", "commErrProcFut");
 
             GridTestUtils.waitForCondition(new GridAbsPredicate() {

From c191c75e67be4819b3c5b243dfdfb58f2e8240e2 Mon Sep 17 00:00:00 2001
From: Sergey Chugunov <sergey.chugunov@gmail.com>
Date: Sat, 28 Apr 2018 16:52:44 +0300
Subject: [PATCH 285/543] IGNITE-8252 NPE is replaced with IgniteException with
 meaningful message - Fixes #3908.

Signed-off-by: dspavlov <dpavlov@gridgain.com>
(cherry picked from commit 5135f82f3f53eddef9ab91da2e647e14488b2d5a)
---
 .../dht/preloader/latch/ExchangeLatchManager.java        | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index e111a6edc7c5f..084d3a31027ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.internal.GridTopic;
 import org.apache.ignite.internal.managers.communication.GridIoManager;
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
@@ -258,9 +259,7 @@ private Collection<ClusterNode> aliveNodesForTopologyVer(AffinityTopologyVersion
      * @return Collection of alive server nodes with latch functionality.
      */
     private Collection<ClusterNode> getLatchParticipants(AffinityTopologyVersion topVer) {
-        Collection<ClusterNode> aliveNodes = topVer == AffinityTopologyVersion.NONE
-                ? discovery.aliveServerNodes()
-                : discovery.discoCache(topVer).aliveServerNodes();
+        Collection<ClusterNode> aliveNodes = aliveNodesForTopologyVer(topVer);
 
         return aliveNodes
                 .stream()
@@ -273,9 +272,7 @@ private Collection<ClusterNode> getLatchParticipants(AffinityTopologyVersion top
      * @return Oldest alive server node with latch functionality.
      */
     @Nullable private ClusterNode getLatchCoordinator(AffinityTopologyVersion topVer) {
-        Collection<ClusterNode> aliveNodes = topVer == AffinityTopologyVersion.NONE
-                ? discovery.aliveServerNodes()
-                : discovery.discoCache(topVer).aliveServerNodes();
+        Collection<ClusterNode> aliveNodes = aliveNodesForTopologyVer(topVer);
 
         return aliveNodes
                 .stream()

From 63b4c4d2a479b957b6aa5cb00ac31c749e40cdfe Mon Sep 17 00:00:00 2001
From: Anton Vinogradov <av@apache.org>
Date: Mon, 23 Jul 2018 13:18:16 +0300
Subject: [PATCH 286/543] IGNITE-8783 Failover tests periodically cause hanging
 of the whole Data Structures suite on TC

Signed-off-by: Anton Vinogradov <av@apache.org>
(cherry picked from commit 6582b084943ac63c36b062c8ae66b43883ef6c19)
---
 .../preloader/latch/ExchangeLatchManager.java | 216 ++++++++++--------
 1 file changed, 119 insertions(+), 97 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index 084d3a31027ab..57305fbbe4c0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -16,11 +16,11 @@
  */
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch;
 
-import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
+import java.util.Comparator;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
@@ -80,15 +80,15 @@ public class ExchangeLatchManager {
     private volatile ClusterNode crd;
 
     /** Pending acks collection. */
-    private final ConcurrentMap<T2<String, AffinityTopologyVersion>, Set<UUID>> pendingAcks = new ConcurrentHashMap<>();
+    private final ConcurrentMap<CompletableLatchUid, Set<UUID>> pendingAcks = new ConcurrentHashMap<>();
 
     /** Server latches collection. */
     @GridToStringInclude
-    private final ConcurrentMap<T2<String, AffinityTopologyVersion>, ServerLatch> serverLatches = new ConcurrentHashMap<>();
+    private final ConcurrentMap<CompletableLatchUid, ServerLatch> serverLatches = new ConcurrentHashMap<>();
 
     /** Client latches collection. */
     @GridToStringInclude
-    private final ConcurrentMap<T2<String, AffinityTopologyVersion>, ClientLatch> clientLatches = new ConcurrentHashMap<>();
+    private final ConcurrentMap<CompletableLatchUid, ClientLatch> clientLatches = new ConcurrentHashMap<>();
 
     /** Lock. */
     private final ReentrantLock lock = new ReentrantLock();
@@ -130,37 +130,30 @@ public ExchangeLatchManager(GridKernalContext ctx) {
      * Creates server latch with given {@code id} and {@code topVer}.
      * Adds corresponding pending acks to it.
      *
-     * @param id Latch id.
-     * @param topVer Latch topology version.
+     * @param latchUid Latch uid.
      * @param participants Participant nodes.
      * @return Server latch instance.
      */
-    private Latch createServerLatch(String id, AffinityTopologyVersion topVer, Collection<ClusterNode> participants) {
-        final T2<String, AffinityTopologyVersion> latchId = new T2<>(id, topVer);
+    private Latch createServerLatch(CompletableLatchUid latchUid, Collection<ClusterNode> participants) {
+        assert !serverLatches.containsKey(latchUid);
 
-        if (serverLatches.containsKey(latchId))
-            return serverLatches.get(latchId);
+        ServerLatch latch = new ServerLatch(latchUid, participants);
 
-        ServerLatch latch = new ServerLatch(id, topVer, participants);
-
-        serverLatches.put(latchId, latch);
+        serverLatches.put(latchUid, latch);
 
         if (log.isDebugEnabled())
-            log.debug("Server latch is created [latch=" + latchId + ", participantsSize=" + participants.size() + "]");
+            log.debug("Server latch is created [latch=" + latchUid + ", participantsSize=" + participants.size() + "]");
 
-        if (pendingAcks.containsKey(latchId)) {
-            Set<UUID> acks = pendingAcks.get(latchId);
+        if (pendingAcks.containsKey(latchUid)) {
+            Set<UUID> acks = pendingAcks.get(latchUid);
 
             for (UUID node : acks)
                 if (latch.hasParticipant(node) && !latch.hasAck(node))
                     latch.ack(node);
 
-            pendingAcks.remove(latchId);
+            pendingAcks.remove(latchUid);
         }
 
-        if (latch.isCompleted())
-            serverLatches.remove(latchId);
-
         return latch;
     }
 
@@ -168,32 +161,23 @@ private Latch createServerLatch(String id, AffinityTopologyVersion topVer, Colle
      * Creates client latch.
      * If there is final ack corresponds to given {@code id} and {@code topVer}, latch will be completed immediately.
      *
-     * @param id Latch id.
-     * @param topVer Latch topology version.
+     * @param latchUid Latch uid.
      * @param coordinator Coordinator node.
      * @param participants Participant nodes.
      * @return Client latch instance.
      */
-    private Latch createClientLatch(String id, AffinityTopologyVersion topVer, ClusterNode coordinator, Collection<ClusterNode> participants) {
-        final T2<String, AffinityTopologyVersion> latchId = new T2<>(id, topVer);
-
-        if (clientLatches.containsKey(latchId))
-            return clientLatches.get(latchId);
+    private Latch createClientLatch(CompletableLatchUid latchUid, ClusterNode coordinator, Collection<ClusterNode> participants) {
+        assert !serverLatches.containsKey(latchUid);
+        assert !clientLatches.containsKey(latchUid);
 
-        ClientLatch latch = new ClientLatch(id, topVer, coordinator, participants);
+        ClientLatch latch = new ClientLatch(latchUid, coordinator, participants);
 
         if (log.isDebugEnabled())
-            log.debug("Client latch is created [latch=" + latchId
-                    + ", crd=" + coordinator
-                    + ", participantsSize=" + participants.size() + "]");
+            log.debug("Client latch is created [latch=" + latchUid
+                + ", crd=" + coordinator
+                + ", participantsSize=" + participants.size() + "]");
 
-        // There is final ack for created latch.
-        if (pendingAcks.containsKey(latchId)) {
-            latch.complete();
-            pendingAcks.remove(latchId);
-        }
-        else
-            clientLatches.put(latchId, latch);
+        clientLatches.put(latchUid, latch);
 
         return latch;
     }
@@ -212,20 +196,24 @@ public Latch getOrCreate(String id, AffinityTopologyVersion topVer) {
         lock.lock();
 
         try {
-            ClusterNode coordinator = getLatchCoordinator(topVer);
+            final CompletableLatchUid latchUid = new CompletableLatchUid(id, topVer);
 
-            if (coordinator == null) {
-                ClientLatch latch = new ClientLatch(id, AffinityTopologyVersion.NONE, null, Collections.emptyList());
-                latch.complete();
+            CompletableLatch latch = clientLatches.containsKey(latchUid) ?
+                clientLatches.get(latchUid) : serverLatches.get(latchUid);
 
+            if (latch != null)
                 return latch;
-            }
+
+            ClusterNode coordinator = getLatchCoordinator(topVer);
+
+            if (coordinator == null)
+                return null;
 
             Collection<ClusterNode> participants = getLatchParticipants(topVer);
 
             return coordinator.isLocal()
-                ? createServerLatch(id, topVer, participants)
-                : createClientLatch(id, topVer, coordinator, participants);
+                ? createServerLatch(latchUid, participants)
+                : createClientLatch(latchUid, coordinator, participants);
         }
         finally {
             lock.unlock();
@@ -275,10 +263,11 @@ private Collection<ClusterNode> getLatchParticipants(AffinityTopologyVersion top
         Collection<ClusterNode> aliveNodes = aliveNodesForTopologyVer(topVer);
 
         return aliveNodes
-                .stream()
-                .filter(node -> node.version().compareTo(VERSION_SINCE) >= 0)
-                .findFirst()
-                .orElse(null);
+            .stream()
+            .filter(node -> node.version().compareTo(VERSION_SINCE) >= 0)
+            .sorted(Comparator.comparing(ClusterNode::order))
+            .findFirst()
+            .orElse(null);
     }
 
     /**
@@ -300,39 +289,36 @@ private void processAck(UUID from, LatchAckMessage message) {
             if (coordinator == null)
                 return;
 
-            T2<String, AffinityTopologyVersion> latchId = new T2<>(message.latchId(), message.topVer());
+            CompletableLatchUid latchUid = new CompletableLatchUid(message.latchId(), message.topVer());
 
             if (message.isFinal()) {
                 if (log.isDebugEnabled())
-                    log.debug("Process final ack [latch=" + latchId + ", from=" + from + "]");
+                    log.debug("Process final ack [latch=" + latchUid + ", from=" + from + "]");
+
+                assert serverLatches.containsKey(latchUid) || clientLatches.containsKey(latchUid);
+
+                if (clientLatches.containsKey(latchUid)) {
+                    ClientLatch latch = clientLatches.remove(latchUid);
 
-                if (clientLatches.containsKey(latchId)) {
-                    ClientLatch latch = clientLatches.remove(latchId);
                     latch.complete();
                 }
-                else if (!coordinator.isLocal()) {
-                    pendingAcks.computeIfAbsent(latchId, (id) -> new GridConcurrentHashSet<>());
-                    pendingAcks.get(latchId).add(from);
-                }
-                else if (coordinator.isLocal())
-                    serverLatches.remove(latchId);
-            } else {
+
+                serverLatches.remove(latchUid);
+            }
+            else {
                 if (log.isDebugEnabled())
-                    log.debug("Process ack [latch=" + latchId + ", from=" + from + "]");
+                    log.debug("Process ack [latch=" + latchUid + ", from=" + from + "]");
 
-                if (serverLatches.containsKey(latchId)) {
-                    ServerLatch latch = serverLatches.get(latchId);
+                if (serverLatches.containsKey(latchUid)) {
+                    ServerLatch latch = serverLatches.get(latchUid);
 
-                    if (latch.hasParticipant(from) && !latch.hasAck(from)) {
+                    if (latch.hasParticipant(from) && !latch.hasAck(from))
                         latch.ack(from);
-
-                        if (latch.isCompleted())
-                            serverLatches.remove(latchId);
-                    }
                 }
                 else {
-                    pendingAcks.computeIfAbsent(latchId, (id) -> new GridConcurrentHashSet<>());
-                    pendingAcks.get(latchId).add(from);
+                    pendingAcks.computeIfAbsent(latchUid, (id) -> new GridConcurrentHashSet<>());
+
+                    pendingAcks.get(latchUid).add(from);
                 }
             }
         }
@@ -349,17 +335,18 @@ private void becomeNewCoordinator() {
         if (log.isInfoEnabled())
             log.info("Become new coordinator " + crd.id());
 
-        List<T2<String, AffinityTopologyVersion>> latchesToRestore = new ArrayList<>();
+        Set<CompletableLatchUid> latchesToRestore = new HashSet<>();
+
         latchesToRestore.addAll(pendingAcks.keySet());
         latchesToRestore.addAll(clientLatches.keySet());
 
-        for (T2<String, AffinityTopologyVersion> latchId : latchesToRestore) {
-            String id = latchId.get1();
-            AffinityTopologyVersion topVer = latchId.get2();
+        for (CompletableLatchUid latchUid : latchesToRestore) {
+            String id = latchUid.id;
+            AffinityTopologyVersion topVer = latchUid.topVer;
             Collection<ClusterNode> participants = getLatchParticipants(topVer);
 
             if (!participants.isEmpty())
-                createServerLatch(id, topVer, participants);
+                createServerLatch(latchUid, participants);
         }
     }
 
@@ -389,12 +376,12 @@ private void processNodeLeft(ClusterNode left) {
                 return;
 
             // Clear pending acks.
-            for (Map.Entry<T2<String, AffinityTopologyVersion>, Set<UUID>> ackEntry : pendingAcks.entrySet())
+            for (Map.Entry<CompletableLatchUid, Set<UUID>> ackEntry : pendingAcks.entrySet())
                 if (ackEntry.getValue().contains(left.id()))
                     pendingAcks.get(ackEntry.getKey()).remove(left.id());
 
             // Change coordinator for client latches.
-            for (Map.Entry<T2<String, AffinityTopologyVersion>, ClientLatch> latchEntry : clientLatches.entrySet()) {
+            for (Map.Entry<CompletableLatchUid, ClientLatch> latchEntry : clientLatches.entrySet()) {
                 ClientLatch latch = latchEntry.getValue();
                 if (latch.hasCoordinator(left.id())) {
                     // Change coordinator for latch and re-send ack if necessary.
@@ -404,7 +391,7 @@ private void processNodeLeft(ClusterNode left) {
                         /* If new coordinator is not able to take control on the latch,
                            it means that all other latch participants are left from topology
                            and there is no reason to track such latch. */
-                        AffinityTopologyVersion topVer = latchEntry.getKey().get2();
+                        AffinityTopologyVersion topVer = latchEntry.getKey().topVer;
 
                         assert getLatchParticipants(topVer).isEmpty();
 
@@ -415,7 +402,7 @@ private void processNodeLeft(ClusterNode left) {
             }
 
             // Add acknowledgements from left node.
-            for (Map.Entry<T2<String, AffinityTopologyVersion>, ServerLatch> latchEntry : serverLatches.entrySet()) {
+            for (Map.Entry<CompletableLatchUid, ServerLatch> latchEntry : serverLatches.entrySet()) {
                 ServerLatch latch = latchEntry.getValue();
 
                 if (latch.hasParticipant(left.id()) && !latch.hasAck(left.id())) {
@@ -423,9 +410,6 @@ private void processNodeLeft(ClusterNode left) {
                         log.debug("Process node left [latch=" + latchEntry.getKey() + ", left=" + left.id() + "]");
 
                     latch.ack(left.id());
-
-                    if (latch.isCompleted())
-                        serverLatches.remove(latchEntry.getKey());
                 }
             }
 
@@ -458,12 +442,11 @@ class ServerLatch extends CompletableLatch {
         /**
          * Constructor.
          *
-         * @param id Latch id.
-         * @param topVer Latch topology version.
+         * @param latchUid Latch uid.
          * @param participants Participant nodes.
          */
-        ServerLatch(String id, AffinityTopologyVersion topVer, Collection<ClusterNode> participants) {
-            super(id, topVer, participants);
+        ServerLatch(CompletableLatchUid latchUid, Collection<ClusterNode> participants) {
+            super(latchUid, participants);
             this.permits = new AtomicInteger(participants.size());
 
             // Send final acks when latch is completed.
@@ -558,13 +541,12 @@ class ClientLatch extends CompletableLatch {
         /**
          * Constructor.
          *
-         * @param id Latch id.
-         * @param topVer Latch topology version.
+         * @param latchUid Latch uid.
          * @param coordinator Coordinator node.
          * @param participants Participant nodes.
          */
-        ClientLatch(String id, AffinityTopologyVersion topVer, ClusterNode coordinator, Collection<ClusterNode> participants) {
-            super(id, topVer, participants);
+        ClientLatch(CompletableLatchUid latchUid, ClusterNode coordinator, Collection<ClusterNode> participants) {
+            super(latchUid, participants);
 
             this.coordinator = coordinator;
         }
@@ -658,13 +640,12 @@ private abstract static class CompletableLatch implements Latch {
         /**
          * Constructor.
          *
-         * @param id Latch id.
-         * @param topVer Latch topology version.
+         * @param latchUid Latch uid.
          * @param participants Participant nodes.
          */
-        CompletableLatch(String id, AffinityTopologyVersion topVer, Collection<ClusterNode> participants) {
-            this.id = id;
-            this.topVer = topVer;
+        CompletableLatch(CompletableLatchUid latchUid, Collection<ClusterNode> participants) {
+            this.id = latchUid.id;
+            this.topVer = latchUid.topVer;
             this.participants = participants.stream().map(ClusterNode::id).collect(Collectors.toSet());
         }
 
@@ -724,6 +705,47 @@ String latchId() {
         }
     }
 
+    /**
+     * Latch id + topology
+     */
+    private static class CompletableLatchUid {
+        /** Id. */
+        private String id;
+
+        /** Topology version. */
+        private AffinityTopologyVersion topVer;
+
+        /**
+         * @param id Id.
+         * @param topVer Topology version.
+         */
+        private CompletableLatchUid(String id, AffinityTopologyVersion topVer) {
+            this.id = id;
+            this.topVer = topVer;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+            CompletableLatchUid uid = (CompletableLatchUid)o;
+            return Objects.equals(id, uid.id) &&
+                Objects.equals(topVer, uid.topVer);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return Objects.hash(id, topVer);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "CompletableLatchUid{" + "id='" + id + '\'' + ", topVer=" + topVer + '}';
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(ExchangeLatchManager.class, this);

From aa4a9b09dfd25a628cef62a92decd6c583141025 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Mon, 23 Jul 2018 16:48:18 +0300
Subject: [PATCH 287/543] IGNITE-8998 Send reconnect exception without
 partitions - Fixes #4358.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 78e0bb7efbc53e969c4c4918b6c6272c7b98dc36)
---
 .../GridCachePartitionExchangeManager.java    | 56 ++++++++++++-------
 .../GridDhtPartitionsExchangeFuture.java      | 18 +++---
 .../Authentication1kUsersNodeRestartTest.java |  1 +
 ...uthenticationConfigurationClusterTest.java |  1 +
 .../AuthenticationOnNotActiveClusterTest.java |  1 +
 ...AuthenticationProcessorNPEOnStartTest.java |  1 +
 ...uthenticationProcessorNodeRestartTest.java |  1 +
 .../AuthenticationProcessorSelfTest.java      |  1 +
 .../IgniteCacheClientReconnectTest.java       |  2 +
 9 files changed, 55 insertions(+), 27 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 952557eaebe9c..824aa6718b2ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -30,6 +30,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
+import java.util.Objects;
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
@@ -186,10 +187,6 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
      */
     private volatile AffinityTopologyVersion rebTopVer = AffinityTopologyVersion.NONE;
 
-    /** */
-    private final AtomicReference<AffinityTopologyVersion> readyTopVer =
-        new AtomicReference<>(AffinityTopologyVersion.NONE);
-
     /** */
     private GridFutureAdapter<?> reconnectExchangeFut;
 
@@ -767,7 +764,7 @@ public static Object rebalanceTopic(int idx) {
         // Do not allow any activity in exchange manager after stop.
         busyLock.writeLock().lock();
 
-        exchFuts = null;
+        exchFuts.clear();
     }
 
     /**
@@ -834,7 +831,7 @@ public GridClientPartitionTopology clearClientTopology(int grpId) {
      * @return Topology version of latest completed partition exchange.
      */
     public AffinityTopologyVersion readyAffinityVersion() {
-        return readyTopVer.get();
+        return exchFuts.readyTopVer();
     }
 
     /**
@@ -891,7 +888,7 @@ public void lastFinishedFuture(GridDhtTopologyFuture fut) {
             return lastInitializedFut0;
         }
 
-        AffinityTopologyVersion topVer = readyTopVer.get();
+        AffinityTopologyVersion topVer = exchFuts.readyTopVer();
 
         if (topVer.compareTo(ver) >= 0) {
             if (log.isDebugEnabled())
@@ -906,7 +903,7 @@ public void lastFinishedFuture(GridDhtTopologyFuture fut) {
         if (log.isDebugEnabled())
             log.debug("Created topology ready future [ver=" + ver + ", fut=" + fut + ']');
 
-        topVer = readyTopVer.get();
+        topVer = exchFuts.readyTopVer();
 
         if (topVer.compareTo(ver) >= 0) {
             if (log.isDebugEnabled())
@@ -1415,15 +1412,7 @@ public void onExchangeDone(AffinityTopologyVersion topVer, AffinityTopologyVersi
             log.debug("Exchange done [topVer=" + topVer + ", err=" + err + ']');
 
         if (err == null) {
-            while (true) {
-                AffinityTopologyVersion readyVer = readyTopVer.get();
-
-                if (readyVer.compareTo(topVer) >= 0)
-                    break;
-
-                if (readyTopVer.compareAndSet(readyVer, topVer))
-                    break;
-            }
+            exchFuts.readyTopVer(topVer);
 
             for (Map.Entry<AffinityTopologyVersion, AffinityReadyFuture> entry : readyFuts.entrySet()) {
                 if (entry.getKey().compareTo(topVer) <= 0) {
@@ -1591,7 +1580,7 @@ else if (!grp.isLocal())
                     AffinityTopologyVersion initVer = exchFut.initialVersion();
                     AffinityTopologyVersion readyVer = readyAffinityVersion();
 
-                    if (initVer.compareTo(readyVer) <= 0 && !exchFut.exchangeDone()) {
+                    if (initVer.compareTo(readyVer) < 0 && !exchFut.isDone()) {
                         U.warn(log, "Client node tries to connect but its exchange " +
                             "info is cleaned up from exchange history. " +
                             "Consider increasing 'IGNITE_EXCHANGE_HISTORY_SIZE' property " +
@@ -1652,7 +1641,7 @@ public ExchangeLatchManager latch() {
     public void dumpDebugInfo(@Nullable GridDhtPartitionsExchangeFuture exchFut) throws Exception {
         AffinityTopologyVersion exchTopVer = exchFut != null ? exchFut.initialVersion() : null;
 
-        U.warn(diagnosticLog, "Ready affinity version: " + readyTopVer.get());
+        U.warn(diagnosticLog, "Ready affinity version: " + exchFuts.readyTopVer());
 
         U.warn(diagnosticLog, "Last exchange future: " + lastInitializedFut);
 
@@ -2807,6 +2796,10 @@ private static class ExchangeFutureSet extends GridListSet<GridDhtPartitionsExch
         /** */
         private final int histSize;
 
+        /** */
+        private final AtomicReference<AffinityTopologyVersion> readyTopVer =
+            new AtomicReference<>(AffinityTopologyVersion.NONE);
+
         /**
          * Creates ordered, not strict list set.
          *
@@ -2843,7 +2836,7 @@ private ExchangeFutureSet(int histSize) {
             while (size() > histSize) {
                 GridDhtPartitionsExchangeFuture last = last();
 
-                if (!last.isDone())
+                if (!last.isDone() || Objects.equals(last.initialVersion(), readyTopVer()))
                     break;
 
                 removeLast();
@@ -2853,6 +2846,29 @@ private ExchangeFutureSet(int histSize) {
             return cur == null ? fut : cur;
         }
 
+        /**
+         * @return Ready top version.
+         */
+        public AffinityTopologyVersion readyTopVer() {
+            return readyTopVer.get();
+        }
+
+        /**
+         * @param readyTopVersion Ready top version.
+         * @return {@code true} if version was set and {@code false} otherwise.
+         */
+        public boolean readyTopVer(AffinityTopologyVersion readyTopVersion) {
+            while (true) {
+                AffinityTopologyVersion readyVer = readyTopVer.get();
+
+                if (readyVer.compareTo(readyTopVersion) >= 0)
+                    return false;
+
+                if (readyTopVer.compareAndSet(readyVer, readyTopVersion))
+                    return true;
+            }
+        }
+
         /** {@inheritDoc} */
         @Nullable @Override public synchronized GridDhtPartitionsExchangeFuture removex(
             GridDhtPartitionsExchangeFuture val) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index c204c8ad1f736..c111706fa8a5c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -2095,21 +2095,25 @@ private void processMergedMessage(final ClusterNode node, final GridDhtPartition
      * @param msg Single message received from the client which didn't find original ExchangeFuture.
      */
     public void forceClientReconnect(ClusterNode node, GridDhtPartitionsSingleMessage msg) {
-        Exception e = new IgniteNeedReconnectException(node, null);
+        Exception reconnectException = new IgniteNeedReconnectException(node, null);
 
-        exchangeGlobalExceptions.put(node.id(), e);
+        exchangeGlobalExceptions.put(node.id(), reconnectException);
 
-        onDone(null, e);
+        onDone(null, reconnectException);
 
         GridDhtPartitionsFullMessage fullMsg = createPartitionsMessage(true, false);
 
         fullMsg.setErrorsMap(exchangeGlobalExceptions);
 
-        FinishState finishState0 = new FinishState(cctx.localNodeId(),
-            initialVersion(),
-            fullMsg);
+        try {
+            cctx.io().send(node, fullMsg, SYSTEM_POOL);
 
-        sendAllPartitionsToNode(finishState0, msg, node.id());
+            if (log.isDebugEnabled())
+                log.debug("Full message for reconnect client was sent to node: " + node + ", fullMsg: " + fullMsg);
+        }
+        catch (IgniteCheckedException e) {
+            U.error(log, "Failed to send reconnect client message [node=" + node + ']', e);
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/Authentication1kUsersNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/Authentication1kUsersNodeRestartTest.java
index 384cbe8be1b83..0ca621e3bc2ad 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/Authentication1kUsersNodeRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/Authentication1kUsersNodeRestartTest.java
@@ -49,6 +49,7 @@ public class Authentication1kUsersNodeRestartTest extends GridCommonAbstractTest
 
         cfg.setDataStorageConfiguration(new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(200L * 1024 * 1024)
                 .setPersistenceEnabled(true)));
 
         return cfg;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationConfigurationClusterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationConfigurationClusterTest.java
index 7d946dcb198e1..fb61d60ad656a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationConfigurationClusterTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationConfigurationClusterTest.java
@@ -59,6 +59,7 @@ private IgniteConfiguration configuration(int idx, boolean authEnabled, boolean
 
         cfg.setDataStorageConfiguration(new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(200L * 1024 * 1024)
                 .setPersistenceEnabled(true)));
 
         return cfg;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationOnNotActiveClusterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationOnNotActiveClusterTest.java
index 7405d7a622d9e..638c378bb1837 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationOnNotActiveClusterTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationOnNotActiveClusterTest.java
@@ -56,6 +56,7 @@ public class AuthenticationOnNotActiveClusterTest extends GridCommonAbstractTest
 
         cfg.setDataStorageConfiguration(new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(200L * 1024 * 1024)
                 .setPersistenceEnabled(true)));
 
         return cfg;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNPEOnStartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNPEOnStartTest.java
index 9748db772747a..661c875e61916 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNPEOnStartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNPEOnStartTest.java
@@ -49,6 +49,7 @@ public class AuthenticationProcessorNPEOnStartTest extends GridCommonAbstractTes
 
         cfg.setDataStorageConfiguration(new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(200L * 1024 * 1024)
                 .setPersistenceEnabled(true)));
 
         return cfg;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNodeRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNodeRestartTest.java
index 632a8b9060b44..7496cfe9bdde6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNodeRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorNodeRestartTest.java
@@ -71,6 +71,7 @@ public class AuthenticationProcessorNodeRestartTest extends GridCommonAbstractTe
 
         cfg.setDataStorageConfiguration(new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(200L * 1024 * 1024)
                 .setPersistenceEnabled(true)));
 
         return cfg;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorSelfTest.java
index 71d86f2125cc4..6c79c7f380411 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/authentication/AuthenticationProcessorSelfTest.java
@@ -85,6 +85,7 @@ private static String randomString(int len) {
 
         cfg.setDataStorageConfiguration(new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(200L * 1024 * 1024)
                 .setPersistenceEnabled(true)));
 
         return cfg;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java
index 4beb31a9d92e6..a0796a3e22ec7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java
@@ -141,6 +141,8 @@ public void testClientReconnectOnExchangeHistoryExhaustion() throws Exception {
 
             waitForTopology(SRV_CNT + CLIENTS_CNT);
 
+            awaitPartitionMapExchange();
+
             verifyPartitionToNodeMappings();
 
             verifyAffinityTopologyVersions();

From 7e89cf89088f11d282f79c331897d17d7db00ed2 Mon Sep 17 00:00:00 2001
From: EdShangGG <eshangareev@gridgain.com>
Date: Tue, 17 Jul 2018 17:19:36 +0300
Subject: [PATCH 288/543] IGNITE-9012 Fixed exchange await logic in
 GridServiceProcessor - Fixes #4367.

(cherry picked from commit 66e547a9eebf3e8354135b3300619754294a805d)
---
 .../dht/GridDhtTopologyFuture.java            |  2 +-
 .../service/GridServiceProcessor.java         | 35 ++++++++++---------
 2 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
index 0bcc4a80c9882..761cb45547365 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
@@ -47,7 +47,7 @@ public interface GridDhtTopologyFuture extends IgniteInternalFuture<AffinityTopo
 
     /**
      * Gets result topology version of this future. Result version can differ from initial exchange version
-     * if excanges for multiple discovery events are merged, in this case result version is version of last
+     * if exchanges for multiple discovery events are merged, in this case result version is version of last
      * discovery event.
      * <p>
      * This method should be called only for finished topology future
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 3149857b492fa..1c83118b6f7a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1741,39 +1741,40 @@ private class TopologyListener implements DiscoveryEventListener {
          * @param initTopVer listening-in topology version.
          * @return {@code True} if current event is not last and should be skipped.
          */
-        private boolean skipExchange(AffinityTopologyVersion initTopVer) {
+        private boolean skipExchange(final AffinityTopologyVersion initTopVer) {
             AffinityTopologyVersion pendingTopVer = null;
-            AffinityTopologyVersion newTopVer = currTopVer;
+            AffinityTopologyVersion newTopVer;
 
-            if (!initTopVer.equals(newTopVer))
+            if (!initTopVer.equals(newTopVer = currTopVer))
                 pendingTopVer = newTopVer;
             else {
-                GridDhtTopologyFuture fut = ctx.cache().context().exchange().lastTopologyFuture();
+                IgniteInternalFuture<?> affReadyFut = ctx.cache().context().exchange().affinityReadyFuture(initTopVer);
 
-                if (!fut.isDone() && !fut.isCancelled()) {
+                if (affReadyFut != null) {
                     try {
-                        fut.get();
+                        affReadyFut.get();
                     }
                     catch (IgniteCheckedException e) {
-                        throw U.convertException(e);
+                        U.error(log, "Failed to wait for affinity ready future " +
+                            "(the assignment will be recalculated anyway)", e);
                     }
                 }
 
-                AffinityTopologyVersion lastTopVer;
-
                 // If exchange already moved forward - skip current version.
-                if (fut.exchangeDone() && newTopVer.compareTo(lastTopVer = fut.topologyVersion()) < 0)
-                    pendingTopVer = lastTopVer;
+                if (!initTopVer.equals(newTopVer = currTopVer))
+                    pendingTopVer = newTopVer;
             }
 
-            if (pendingTopVer != null && log.isInfoEnabled()) {
+            boolean skipExchange = pendingTopVer != null;
+
+            if (skipExchange && log.isInfoEnabled()) {
                 log.info("Service processor detected a topology change during " +
                     "assignments calculation (will abort current iteration and " +
                     "re-calculate on the newer version): " +
                     "[topVer=" + initTopVer + ", newTopVer=" + pendingTopVer + ']');
             }
 
-            return pendingTopVer != null;
+            return skipExchange;
         }
 
         /** {@inheritDoc} */
@@ -1864,11 +1865,11 @@ else if (msg instanceof DynamicCacheChangeBatch) {
                         // Clean up zombie assignments.
                         IgniteInternalCache<Object, Object> cache = serviceCache();
 
-                        // If topology changed again, let next event handle it.
-                        if (skipExchange(topVer))
-                            return;
-
                         while (it.hasNext()) {
+                            // If topology changed again, let next event handle it.
+                            if (skipExchange(topVer))
+                                return;
+
                             Cache.Entry<Object, Object> e = it.next();
 
                             if (cache.context().affinity().primaryByKey(ctx.grid().localNode(), e.getKey(), topVer)) {

From 52cf2809a759071d440719f968ab0d0040fdf23e Mon Sep 17 00:00:00 2001
From: EdShangGG <eshangareev@gridgain.com>
Date: Tue, 17 Jul 2018 18:04:38 +0300
Subject: [PATCH 289/543] IGNITE-9013 Fail cache future when local node is
 stopping - Fixes #4369.

(cherry picked from commit 85b2002796fb601d7e7ce7d7320943f9323c2bdd)
---
 .../processors/cache/GridCacheAdapter.java    | 14 ++++---
 .../dht/GridDhtTxPrepareFuture.java           | 22 +++--------
 .../IgniteServiceReassignmentTest.java        | 38 +++++++++++++++++++
 3 files changed, 52 insertions(+), 22 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 51c695a9af60f..5121753207c89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -2893,12 +2893,14 @@ protected V getAndRemove0(final K key) throws IgniteCheckedException {
             @Override public V op(GridNearTxLocal tx) throws IgniteCheckedException {
                 K key0 = keepBinary ? (K)ctx.toCacheKeyObject(key) : key;
 
-                V ret = tx.removeAllAsync(ctx,
-                    null,
-                    Collections.singletonList(key0),
-                    /*retval*/true,
-                    null,
-                    /*singleRmv*/false).get().value();
+                IgniteInternalFuture<GridCacheReturn> fut = tx.removeAllAsync(ctx,
+                        null,
+                        Collections.singletonList(key0),
+                        /*retval*/true,
+                        null,
+                        /*singleRmv*/false);
+
+                V ret = fut.get().value();
 
                 if (ctx.config().getInterceptor() != null) {
                     K key = keepBinary ? (K)ctx.unwrapBinaryIfNeeded(key0, true, false) : key0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index a6e349fc7024a..d02b8510c7165 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -1382,23 +1382,13 @@ private void sendPrepareRequests() {
                 fut.onNodeLeft();
             }
             catch (IgniteCheckedException e) {
-                if (!cctx.kernalContext().isStopping()) {
-                    if (msgLog.isDebugEnabled()) {
-                        msgLog.debug("DHT prepare fut, failed to send request dht [txId=" + tx.nearXidVersion() +
-                            ", dhtTxId=" + tx.xidVersion() +
-                            ", node=" + n.id() + ']');
-                    }
-
-                    fut.onResult(e);
-                }
-                else {
-                    if (msgLog.isDebugEnabled()) {
-                        msgLog.debug("DHT prepare fut, failed to send request dht, ignore [txId=" + tx.nearXidVersion() +
-                            ", dhtTxId=" + tx.xidVersion() +
-                            ", node=" + n.id() +
-                            ", err=" + e + ']');
-                    }
+                if (msgLog.isDebugEnabled()) {
+                    msgLog.debug("DHT prepare fut, failed to send request dht [txId=" + tx.nearXidVersion() +
+                        ", dhtTxId=" + tx.xidVersion() +
+                        ", node=" + n.id() + ']');
                 }
+
+                fut.onResult(e);
             }
         }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java
index 865f1213e3ccc..e74b27de99caa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/service/IgniteServiceReassignmentTest.java
@@ -243,6 +243,44 @@ public void testZombieAssignmentsCleanup() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNodeStopWhileThereAreCacheActivitiesInServiceProcessor() throws Exception {
+        final int nodesCnt = 2;
+        final int maxSvc = 1024;
+
+        startGridsMultiThreaded(nodesCnt);
+
+        IgniteEx ignite = grid(0);
+
+        IgniteInternalCache<GridServiceAssignmentsKey, Object> sysCache = ignite.utilityCache();
+
+        // Adding some assignments without deployments.
+        for (int i = 0; i < maxSvc; i++) {
+            String name = "svc-" + i;
+
+            ServiceConfiguration svcCfg = new ServiceConfiguration();
+
+            svcCfg.setName(name);
+
+            GridServiceAssignmentsKey key = new GridServiceAssignmentsKey(name);
+
+            UUID nodeId = grid(i % nodesCnt).localNode().id();
+
+            sysCache.put(key, new GridServiceAssignments(svcCfg, nodeId, ignite.cluster().topologyVersion()));
+        }
+
+        // Simulate exchange with merge.
+        GridTestUtils.runAsync(() -> startGrid(nodesCnt));
+        GridTestUtils.runAsync(() -> startGrid(nodesCnt + 1));
+        startGrid(nodesCnt + 2);
+
+        Thread.sleep((int)(1000 * ThreadLocalRandom.current().nextDouble()));
+
+        stopAllGrids();
+    }
+
     /**
      * @param node Node.
      * @throws Exception If failed.

From 92dbb2197488b5c0c61182586d49254263b3a49b Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Fri, 10 Aug 2018 17:47:13 +0300
Subject: [PATCH 290/543] IGNITE-9231 improvement throttle implementation,
 unpark threads on cpBuf condition. - Fixes #4506.

Signed-off-by: Ivan Rakov <irakov@apache.org>
---
 .../apache/ignite/IgniteSystemProperties.java | 12 +++++++++
 .../pagemem/PagesWriteThrottle.java           | 26 ++++++++++++++++---
 .../pagemem/PagesWriteThrottlePolicy.java     |  7 ++++-
 3 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 68f95fe760082..3c8b51ac5972e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -903,6 +903,18 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_DUMP_THREADS_ON_FAILURE = "IGNITE_DUMP_THREADS_ON_FAILURE";
 
+    /**
+     * Throttling timeout in millis which avoid excessive PendingTree access on unwind if there is nothing to clean yet.
+     *
+     * Default is 500 ms.
+     */
+    public static final String IGNITE_UNWIND_THROTTLING_TIMEOUT = "IGNITE_UNWIND_THROTTLING_TIMEOUT";
+
+    /**
+     * Threshold for throttling operations logging.
+     */
+    public static final String IGNITE_THROTTLE_LOG_THRESHOLD = "IGNITE_THROTTLE_LOG_THRESHOLD";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
index d5f4bd5929aef..2828c4348af8d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
@@ -16,13 +16,18 @@
 */
 package org.apache.ignite.internal.processors.cache.persistence.pagemem;
 
+import java.util.Collection;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.LockSupport;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointWriteProgressSupplier;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_STARVATION_CHECK_INTERVAL;
+
 /**
  * Throttles threads that generate dirty pages during ongoing checkpoint.
  * Designed to avoid zero dropdowns that can happen if checkpoint buffer is overflowed.
@@ -46,6 +51,9 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
     /** Backoff ratio. Each next park will be this times longer. */
     private static final double BACKOFF_RATIO = 1.05;
 
+    /** Checkpoint buffer fullfill upper bound. */
+    private static final float CP_BUF_FILL_THRESHOLD = 2f / 3;
+
     /** Counter for dirty pages ratio throttling. */
     private final AtomicInteger notInCheckpointBackoffCntr = new AtomicInteger(0);
 
@@ -55,6 +63,9 @@ public class PagesWriteThrottle implements PagesWriteThrottlePolicy {
     /** Logger. */
     private IgniteLogger log;
 
+    /** Currently parking threads. */
+    private final Collection<Thread> parkThrds = new ConcurrentLinkedQueue<>();
+
     /**
      * @param pageMemory Page memory.
      * @param cpProgress Database manager.
@@ -85,7 +96,7 @@ public PagesWriteThrottle(PageMemoryImpl pageMemory,
         boolean shouldThrottle = false;
 
         if (isPageInCheckpoint) {
-            int checkpointBufLimit = pageMemory.checkpointBufferPagesSize() * 2 / 3;
+            int checkpointBufLimit = (int)(pageMemory.checkpointBufferPagesSize() * CP_BUF_FILL_THRESHOLD);
 
             shouldThrottle = pageMemory.checkpointBufferPagesCount() > checkpointBufLimit;
         }
@@ -126,10 +137,19 @@ public PagesWriteThrottle(PageMemoryImpl pageMemory,
                     + " for timeout(ms)=" + (throttleParkTimeNs / 1_000_000));
             }
 
+            if (isPageInCheckpoint)
+                parkThrds.add(Thread.currentThread());
+
             LockSupport.parkNanos(throttleParkTimeNs);
         }
-        else
-            cntr.set(0);
+        else {
+            int oldCntr = cntr.getAndSet(0);
+
+            if (isPageInCheckpoint && oldCntr != 0) {
+                parkThrds.forEach(LockSupport::unpark);
+                parkThrds.clear();
+            }
+        }
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottlePolicy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottlePolicy.java
index 53a8017aea311..e6aab794761eb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottlePolicy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottlePolicy.java
@@ -17,14 +17,19 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.pagemem;
 
+import org.apache.ignite.IgniteSystemProperties;
+
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_THROTTLE_LOG_THRESHOLD;
+
 /**
  * Throttling policy, encapsulates logic of delaying write operations.
  */
 public interface PagesWriteThrottlePolicy {
     /** Max park time. */
-    public long LOGGING_THRESHOLD = TimeUnit.SECONDS.toNanos(10);
+    public long LOGGING_THRESHOLD = TimeUnit.SECONDS.toNanos(IgniteSystemProperties.getInteger
+            (IGNITE_THROTTLE_LOG_THRESHOLD, 10));
 
     /**
      * Callback to apply throttling delay.

From 0e66d270a41caeedee20a93a3ad4aea95f074dce Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Mon, 13 Aug 2018 11:59:27 +0300
Subject: [PATCH 291/543] IGNITE-9244 Partition eviction should not take all
 threads in system pool

(cherry picked from commit 2d63040)
Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
---
 .../apache/ignite/IgniteSystemProperties.java |   5 +
 .../processors/cache/CacheGroupContext.java   |  21 +-
 .../processors/cache/GridCacheProcessor.java  |   3 +
 .../cache/GridCacheSharedContext.java         |  48 +-
 .../dht/GridDhtLocalPartition.java            |   3 +-
 .../dht/GridDhtPartitionsEvictor.java         | 310 ----------
 .../dht/PartitionsEvictManager.java           | 566 ++++++++++++++++++
 .../wal/reader/IgniteWalIteratorFactory.java  |   2 +-
 .../IgniteWalIteratorSwitchSegmentTest.java   |   2 +
 .../pagemem/BPlusTreePageMemoryImplTest.java  |   1 +
 .../BPlusTreeReuseListPageMemoryImplTest.java |   1 +
 .../IndexStoragePageMemoryImplTest.java       |   1 +
 .../pagemem/PageMemoryImplNoLoadTest.java     |   1 +
 .../pagemem/PageMemoryImplTest.java           |   1 +
 .../hashmap/GridCacheTestContext.java         |   2 +
 15 files changed, 633 insertions(+), 334 deletions(-)
 delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 3c8b51ac5972e..c390e196cce9f 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -915,6 +915,11 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_THROTTLE_LOG_THRESHOLD = "IGNITE_THROTTLE_LOG_THRESHOLD";
 
+    /**
+     * Number of concurrent operation for evict partitions.
+     */
+    public static final String IGNITE_EVICTION_PERMITS = "IGNITE_EVICTION_PERMITS";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index d545bcc97d29c..225c53a9198b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -127,9 +127,6 @@ public class CacheGroupContext {
     /** */
     private GridCachePreloader preldr;
 
-    /** Partition evictor. */
-    private GridDhtPartitionsEvictor evictor;
-
     /** */
     private final DataRegion dataRegion;
 
@@ -259,13 +256,6 @@ public GridCachePreloader preloader() {
         return preldr;
     }
 
-    /**
-     * @return Partitions evictor.
-     */
-    public GridDhtPartitionsEvictor evictor() {
-        return evictor;
-    }
-
     /**
      * @return IO policy for the given cache group.
      */
@@ -320,7 +310,7 @@ private void addCacheContext(GridCacheContext cctx) {
             drEnabled = true;
 
         this.caches = caches;
-   }
+    }
 
     /**
      * @param cctx Cache context.
@@ -379,8 +369,8 @@ public GridCacheContext singleCacheContext() {
         List<GridCacheContext> caches = this.caches;
 
         assert !sharedGroup() && caches.size() == 1 :
-            "stopping=" +  ctx.kernalContext().isStopping() + ", groupName=" + ccfg.getGroupName() +
-            ", caches=" + caches;
+            "stopping=" + ctx.kernalContext().isStopping() + ", groupName=" + ccfg.getGroupName() +
+                ", caches=" + caches;
 
         return caches.get(0);
     }
@@ -441,6 +431,7 @@ public void addRebalanceEvent(int part, int type, ClusterNode discoNode, int dis
             }
         }
     }
+
     /**
      * Adds partition unload event.
      *
@@ -730,7 +721,7 @@ void stopGroup() {
         IgniteCheckedException err =
             new IgniteCheckedException("Failed to wait for topology update, cache (or node) is stopping.");
 
-        evictor.stop();
+        ctx.evict().onCacheGroupStopped(this);
 
         aff.cancelFutures(err);
 
@@ -906,8 +897,6 @@ public void start() throws IgniteCheckedException {
         else
             preldr = new GridCachePreloaderAdapter(this);
 
-        evictor = new GridDhtPartitionsEvictor(this);
-
         if (persistenceEnabled()) {
             try {
                 offheapMgr = new GridCacheOffheapManager();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index d392c6819b66c..d10ad86167712 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -81,6 +81,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionsEvictManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
@@ -2481,6 +2482,7 @@ private GridCacheSharedContext createSharedContext(GridKernalContext kernalCtx,
         GridCacheIoManager ioMgr = new GridCacheIoManager();
         CacheAffinitySharedManager topMgr = new CacheAffinitySharedManager();
         GridCacheSharedTtlCleanupManager ttl = new GridCacheSharedTtlCleanupManager();
+        PartitionsEvictManager evict = new PartitionsEvictManager();
 
         CacheJtaManagerAdapter jta = JTA.createOptional();
 
@@ -2499,6 +2501,7 @@ private GridCacheSharedContext createSharedContext(GridKernalContext kernalCtx,
             topMgr,
             ioMgr,
             ttl,
+            evict,
             jta,
             storeSesLsnrs
         );
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index b1955085e29db..550c5e746d51b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -45,6 +45,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionsEvictManager;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.jta.CacheJtaManagerAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
@@ -126,6 +127,9 @@ public class GridCacheSharedContext<K, V> {
     /** Ttl cleanup manager. */
     private GridCacheSharedTtlCleanupManager ttlMgr;
 
+    /** */
+    private PartitionsEvictManager evictMgr;
+
     /** Cache contexts map. */
     private ConcurrentHashMap<Integer, GridCacheContext<K, V>> ctxMap;
 
@@ -199,13 +203,30 @@ public GridCacheSharedContext(
         CacheAffinitySharedManager<K, V> affMgr,
         GridCacheIoManager ioMgr,
         GridCacheSharedTtlCleanupManager ttlMgr,
+        PartitionsEvictManager evictMgr,
         CacheJtaManagerAdapter jtaMgr,
         Collection<CacheStoreSessionListener> storeSesLsnrs
     ) {
         this.kernalCtx = kernalCtx;
 
-        setManagers(mgrs, txMgr, jtaMgr, verMgr, mvccMgr, pageStoreMgr, walMgr, walStateMgr, dbMgr, snpMgr, depMgr,
-            exchMgr, affMgr, ioMgr, ttlMgr);
+        setManagers(
+            mgrs,
+            txMgr,
+            jtaMgr,
+            verMgr,
+            mvccMgr,
+            pageStoreMgr,
+            walMgr,
+            walStateMgr,
+            dbMgr,
+            snpMgr,
+            depMgr,
+            exchMgr,
+            affMgr,
+            ioMgr,
+            ttlMgr,
+            evictMgr
+        );
 
         this.storeSesLsnrs = storeSesLsnrs;
 
@@ -364,7 +385,9 @@ void onDisconnected(IgniteFuture<?> reconnectFut) throws IgniteCheckedException
     void onReconnected(boolean active) throws IgniteCheckedException {
         List<GridCacheSharedManager<K, V>> mgrs = new LinkedList<>();
 
-        setManagers(mgrs, txMgr,
+        setManagers(
+            mgrs,
+            txMgr,
             jtaMgr,
             verMgr,
             mvccMgr,
@@ -377,7 +400,9 @@ void onReconnected(boolean active) throws IgniteCheckedException {
             new GridCachePartitionExchangeManager<K, V>(),
             affMgr,
             ioMgr,
-            ttlMgr);
+            ttlMgr,
+            evictMgr
+        );
 
         this.mgrs = mgrs;
 
@@ -419,7 +444,8 @@ private boolean restartOnDisconnect(GridCacheSharedManager<?, ?> mgr) {
      * @param ttlMgr Ttl cleanup manager.
      */
     @SuppressWarnings("unchecked")
-    private void setManagers(List<GridCacheSharedManager<K, V>> mgrs,
+    private void setManagers(
+        List<GridCacheSharedManager<K, V>> mgrs,
         IgniteTxManager txMgr,
         CacheJtaManagerAdapter jtaMgr,
         GridCacheVersionManager verMgr,
@@ -433,7 +459,9 @@ private void setManagers(List<GridCacheSharedManager<K, V>> mgrs,
         GridCachePartitionExchangeManager<K, V> exchMgr,
         CacheAffinitySharedManager affMgr,
         GridCacheIoManager ioMgr,
-        GridCacheSharedTtlCleanupManager ttlMgr) {
+        GridCacheSharedTtlCleanupManager ttlMgr,
+        PartitionsEvictManager evictMgr
+    ) {
         this.mvccMgr = add(mgrs, mvccMgr);
         this.verMgr = add(mgrs, verMgr);
         this.txMgr = add(mgrs, txMgr);
@@ -448,6 +476,7 @@ private void setManagers(List<GridCacheSharedManager<K, V>> mgrs,
         this.affMgr = add(mgrs, affMgr);
         this.ioMgr = add(mgrs, ioMgr);
         this.ttlMgr = add(mgrs, ttlMgr);
+        this.evictMgr = add(mgrs, evictMgr);
     }
 
     /**
@@ -779,6 +808,13 @@ public GridTimeoutProcessor time() {
         return kernalCtx.timeout();
     }
 
+    /**
+     * @return Partition evict manager.
+     */
+    public PartitionsEvictManager evict() {
+        return evictMgr;
+    }
+
     /**
      * @return Node ID.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index ae36dd4d9e115..a8adf52d04aeb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -688,7 +688,7 @@ && getReservations(state) == 0 && !groupReserved()) {
             }
         }
 
-        grp.evictor().evictPartitionAsync(this);
+        ctx.evict().evictPartitionAsync(grp,this);
     }
 
     /**
@@ -700,6 +700,7 @@ public void clearAsync() {
             return;
 
         clear = true;
+
         clearAsync0(false);
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java
deleted file mode 100644
index 72063974b6d13..0000000000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsEvictor.java
+++ /dev/null
@@ -1,310 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal.processors.cache.distributed.dht;
-
-import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.function.Function;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.util.GridConcurrentHashSet;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.typedef.internal.LT;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- * Class that serves asynchronous part eviction process.
- * Only one partition from group can be evicted at the moment.
- */
-public class GridDhtPartitionsEvictor {
-    /** Default eviction progress show frequency. */
-    private static final int DEFAULT_SHOW_EVICTION_PROGRESS_FREQ_MS = 2 * 60 * 1000; // 2 Minutes.
-
-    /** Eviction progress frequency property name. */
-    private static final String SHOW_EVICTION_PROGRESS_FREQ = "SHOW_EVICTION_PROGRESS_FREQ";
-
-    /** */
-    private final GridCacheSharedContext<?, ?> ctx;
-
-    /** */
-    private final CacheGroupContext grp;
-
-    /** */
-    private final IgniteLogger log;
-
-    /** Lock object. */
-    private final Object mux = new Object();
-
-    /** Queue contains partitions scheduled for eviction. */
-    private final DeduplicationQueue<Integer, GridDhtLocalPartition> evictionQueue = new DeduplicationQueue<>(GridDhtLocalPartition::id);
-
-    /**
-     * Flag indicates that eviction process is running at the moment.
-     * This is needed to schedule partition eviction if there are no currently running self-scheduling eviction tasks.
-     * Guarded by {@link #mux}.
-     */
-    private boolean evictionRunning;
-
-    /** Flag indicates that eviction process has stopped. */
-    private volatile boolean stop;
-
-    /** Future for currently running partition eviction task. */
-    private volatile GridFutureAdapter<Boolean> evictionFut;
-
-    /** Eviction progress frequency in ms. */
-    private final long evictionProgressFreqMs = IgniteSystemProperties.getLong(SHOW_EVICTION_PROGRESS_FREQ,
-        DEFAULT_SHOW_EVICTION_PROGRESS_FREQ_MS);
-
-    /** Next time of show eviction progress. */
-    private long nextShowProgressTime;
-
-    /**
-     * Constructor.
-     *
-     * @param grp Cache group context.
-     */
-    public GridDhtPartitionsEvictor(CacheGroupContext grp) {
-        assert grp != null;
-
-        this.grp = grp;
-        this.ctx = grp.shared();
-
-        this.log = ctx.logger(getClass());
-    }
-
-    /**
-     * Adds partition to eviction queue and starts eviction process.
-     *
-     * @param part Partition to evict.
-     */
-    public void evictPartitionAsync(GridDhtLocalPartition part) {
-        if (stop)
-            return;
-
-        boolean added = evictionQueue.offer(part);
-
-        if (!added)
-            return;
-
-        synchronized (mux) {
-            if (!evictionRunning) {
-                nextShowProgressTime = U.currentTimeMillis() + evictionProgressFreqMs;
-
-                scheduleNextPartitionEviction();
-            }
-        }
-    }
-
-    /**
-     * Stops eviction process.
-     * Method awaits last offered partition eviction.
-     */
-    public void stop() {
-        stop = true;
-
-        synchronized (mux) {
-            // Wait for last offered partition eviction completion.
-            IgniteInternalFuture<Boolean> evictionFut0 = evictionFut;
-
-            if (evictionFut0 != null) {
-                try {
-                    evictionFut0.get();
-                }
-                catch (IgniteCheckedException e) {
-                    if (log.isDebugEnabled())
-                        log.warning("Failed to await partition eviction during stopping", e);
-                }
-            }
-        }
-    }
-
-    /**
-     * Gets next partition from the queue and schedules it for eviction.
-     */
-    private void scheduleNextPartitionEviction() {
-        if (stop)
-            return;
-
-        synchronized (mux) {
-            GridDhtLocalPartition next = evictionQueue.poll();
-
-            if (next != null) {
-                showProgress();
-
-                evictionFut = new GridFutureAdapter<>();
-
-                ctx.kernalContext().closure().callLocalSafe(new PartitionEvictionTask(next, () -> stop), true);
-            }
-            else
-                evictionRunning = false;
-        }
-    }
-
-    /**
-     * Shows progress of eviction.
-     */
-    private void showProgress() {
-        if (U.currentTimeMillis() >= nextShowProgressTime) {
-            int size = evictionQueue.size() + 1; // Queue size plus current partition.
-
-            if (log.isInfoEnabled())
-                log.info("Eviction in progress [grp=" + grp.cacheOrGroupName()
-                    + ", remainingPartsCnt=" + size + "]");
-
-            nextShowProgressTime = U.currentTimeMillis() + evictionProgressFreqMs;
-        }
-    }
-
-    /**
-     * Task for self-scheduled partition eviction / clearing.
-     */
-    private class PartitionEvictionTask implements Callable<Boolean> {
-        /** Partition to evict. */
-        private final GridDhtLocalPartition part;
-
-        /** Eviction context. */
-        private final EvictionContext evictionCtx;
-
-        /**
-         * @param part Partition.
-         * @param evictionCtx Eviction context.
-         */
-        public PartitionEvictionTask(GridDhtLocalPartition part, EvictionContext evictionCtx) {
-            this.part = part;
-            this.evictionCtx = evictionCtx;
-        }
-
-        /** {@inheritDoc} */
-        @Override public Boolean call() throws Exception {
-            if (stop) {
-                evictionFut.onDone();
-
-                return false;
-            }
-
-            try {
-                boolean success = part.tryClear(evictionCtx);
-
-                if (success) {
-                    if (part.state() == GridDhtPartitionState.EVICTED && part.markForDestroy())
-                        part.destroy();
-                }
-                else // Re-offer partition if clear was unsuccessful due to partition reservation.
-                    evictionQueue.offer(part);
-
-                // Complete eviction future before schedule new to prevent deadlock with
-                // simultaneous eviction stopping and scheduling new eviction.
-                evictionFut.onDone();
-
-                scheduleNextPartitionEviction();
-
-                return true;
-            }
-            catch (Throwable ex) {
-                evictionFut.onDone(ex);
-
-                if (ctx.kernalContext().isStopping()) {
-                    LT.warn(log, ex, "Partition eviction failed (current node is stopping).",
-                        false,
-                        true);
-                }
-                else
-                    LT.error(log, ex, "Partition eviction failed, this can cause grid hang.");
-            }
-
-            return false;
-        }
-    }
-
-    /**
-     * Thread-safe blocking queue with items deduplication.
-     *
-     * @param <K> Key type of item used for deduplication.
-     * @param <V> Queue item type.
-     */
-    private static class DeduplicationQueue<K, V> {
-        /** Queue. */
-        private final Queue<V> queue;
-
-        /** Unique items set. */
-        private final Set<K> uniqueItems;
-
-        /** Key mapping function. */
-        private final Function<V, K> keyMappingFunction;
-
-        /**
-         * Constructor.
-         *
-         * @param keyExtractor Function to extract a key from a queue item.
-         *                     This key is used for deduplication if some item has offered twice.
-         */
-        public DeduplicationQueue(Function<V, K> keyExtractor) {
-            keyMappingFunction = keyExtractor;
-            queue = new LinkedBlockingQueue<>();
-            uniqueItems = new GridConcurrentHashSet<>();
-        }
-
-        /**
-         * Offers item to the queue.
-         *
-         * @param item Item.
-         * @return {@code true} if item has been successfully offered to the queue,
-         *         {@code false} if item was rejected because already exists in the queue.
-         */
-        public boolean offer(V item) {
-            K key = keyMappingFunction.apply(item);
-
-            if (uniqueItems.add(key)) {
-                queue.offer(item);
-
-                return true;
-            }
-
-            return false;
-        }
-
-        /**
-         * Polls next item from queue.
-         *
-         * @return Next item or {@code null} if queue is empty.
-         */
-        public V poll() {
-            V item = queue.poll();
-
-            if (item != null) {
-                K key = keyMappingFunction.apply(item);
-
-                uniqueItems.remove(key);
-            }
-
-            return item;
-        }
-
-        /**
-         * @return Size of queue.
-         */
-        public int size() {
-            return queue.size();
-        }
-    }
-}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java
new file mode 100644
index 0000000000000..f76310d439ee9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java
@@ -0,0 +1,566 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.internal.LT;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_EVICTION_PERMITS;
+import static org.apache.ignite.IgniteSystemProperties.getInteger;
+import static org.apache.ignite.IgniteSystemProperties.getLong;
+
+/**
+ * Class that serves asynchronous part eviction process.
+ * Multiple partition from group can be evicted at the same time.
+ */
+public class PartitionsEvictManager extends GridCacheSharedManagerAdapter {
+
+    /** Default eviction progress show frequency. */
+    private static final int DEFAULT_SHOW_EVICTION_PROGRESS_FREQ_MS = 2 * 60 * 1000; // 2 Minutes.
+
+    /** Eviction progress frequency property name. */
+    private static final String SHOW_EVICTION_PROGRESS_FREQ = "SHOW_EVICTION_PROGRESS_FREQ";
+
+    /** Eviction thread pool policy. */
+    private static final byte EVICT_POOL_PLC = GridIoPolicy.SYSTEM_POOL;
+
+    /** Eviction progress frequency in ms. */
+    private final long evictionProgressFreqMs = getLong(SHOW_EVICTION_PROGRESS_FREQ, DEFAULT_SHOW_EVICTION_PROGRESS_FREQ_MS);
+
+    /** */
+    private final int confPermits = getInteger(IGNITE_EVICTION_PERMITS, -1);
+
+    /** Next time of show eviction progress. */
+    private long nextShowProgressTime;
+
+    private final Map<Integer, GroupEvictionContext> evictionGroupsMap = new ConcurrentHashMap<>();
+
+    /** Flag indicates that eviction process has stopped. */
+    private volatile boolean stop;
+
+    /** Check stop eviction context. */
+    private final EvictionContext sharedEvictionContext = () -> stop;
+
+    /** Number of maximum concurrent operations. */
+    private volatile int threads;
+
+    /** How many eviction task may execute concurrent. */
+    private volatile int permits;
+
+    /** Bucket queue for load balance partitions to the threads via count of partition size.
+     *  Is not thread-safe.
+     *  All method should be called under mux synchronization.
+     */
+    private volatile BucketQueue evictionQueue;
+
+    /** Lock object. */
+    private final Object mux = new Object();
+
+    /**
+     * Stops eviction process for group.
+     *
+     * Method awaits last offered partition eviction.
+     *
+     * @param grp Group context.
+     */
+    public void onCacheGroupStopped(CacheGroupContext  grp){
+        GroupEvictionContext groupEvictionContext = evictionGroupsMap.remove(grp.groupId());
+
+        if (groupEvictionContext != null){
+            groupEvictionContext.stop();
+
+            groupEvictionContext.awaitFinishAll();
+        }
+    }
+
+    /**
+     * Adds partition to eviction queue and starts eviction process if permit available.
+     *
+     * @param grp Group context.
+     * @param part Partition to evict.
+     */
+    public void evictPartitionAsync(CacheGroupContext grp, GridDhtLocalPartition part) {
+        // Check node stop.
+        if (sharedEvictionContext.shouldStop())
+            return;
+
+        GroupEvictionContext groupEvictionContext = evictionGroupsMap.computeIfAbsent(
+            grp.groupId(), (k) -> new GroupEvictionContext(grp));
+
+        PartitionEvictionTask evictionTask = groupEvictionContext.createEvictPartitionTask(part);
+
+        if (evictionTask == null)
+            return;
+
+        int bucket;
+
+        synchronized (mux) {
+            bucket = evictionQueue.offer(evictionTask);
+        }
+
+        scheduleNextPartitionEviction(bucket);
+    }
+
+    /**
+     * Gets next partition from the queue and schedules it for eviction.
+     *
+     * @param bucket Bucket.
+     */
+    private void scheduleNextPartitionEviction(int bucket) {
+        // Check node stop.
+        if (sharedEvictionContext.shouldStop())
+            return;
+
+        synchronized (mux) {
+            // Check that we have permits for next operation.
+            if (permits > 0) {
+                // If queue is empty not need to do.
+                if (evictionQueue.isEmpty())
+                    return;
+
+                // Get task until we have permits.
+                while (permits >= 0) {
+                    // Get task from bucket.
+                    PartitionEvictionTask evictionTask = evictionQueue.poll(bucket);
+
+                    // If bucket empty try get from another.
+                    if (evictionTask == null) {
+                        // Until queue have tasks.
+                        while (!evictionQueue.isEmpty()) {
+                            // Get task from any other bucket.
+                            evictionTask = evictionQueue.pollAny();
+
+                            // Stop iteration if we found task.
+                            if (evictionTask != null)
+                                break;
+                        }
+
+                        // If task not found no need to do some.
+                        if (evictionTask == null)
+                            return;
+                    }
+
+                    // Print current eviction progress.
+                    showProgress();
+
+                    GroupEvictionContext groupEvictionContext = evictionTask.groupEvictionContext;
+
+                    // Check that group or node stopping.
+                    if (groupEvictionContext.shouldStop())
+                        continue;
+
+                    // Get permit for this task.
+                    permits--;
+
+                    // Register task future, may need if group or node will be stopped.
+                    groupEvictionContext.taskScheduled(evictionTask);
+
+                    evictionTask.finishFut.listen(f -> {
+                        synchronized (mux) {
+                            // Return permit after task completed.
+                            permits++;
+                        }
+
+                        // Re-schedule new one task form same bucket.
+                        scheduleNextPartitionEviction(bucket);
+                    });
+
+                    // Submit task to executor.
+                     cctx.kernalContext()
+                        .closure()
+                        .runLocalSafe(evictionTask, EVICT_POOL_PLC);
+                }
+            }
+        }
+    }
+
+    /**
+     * Shows progress of eviction.
+     */
+    private void showProgress() {
+        if (U.currentTimeMillis() >= nextShowProgressTime) {
+            int size = evictionQueue.size() + 1; // Queue size plus current partition.
+
+            if (log.isInfoEnabled())
+                log.info("Eviction in progress [permits=" + permits+
+                    ", threads=" + threads +
+                    ", groups=" + evictionGroupsMap.keySet().size() +
+                    ", remainingPartsToEvict=" + size + "]");
+
+            evictionGroupsMap.values().forEach(GroupEvictionContext::showProgress);
+
+            nextShowProgressTime = U.currentTimeMillis() + evictionProgressFreqMs;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void start0() throws IgniteCheckedException {
+        super.start0();
+
+        // If property is not setup, calculate permits as parts of sys pool.
+        if (confPermits == -1) {
+            int sysPoolSize = cctx.kernalContext().config().getSystemThreadPoolSize();
+
+            threads = permits = sysPoolSize / 4;
+        }
+        else
+            threads = permits = confPermits;
+
+        // Avoid 0 permits if sys pool size less that 4.
+        if (threads == 0)
+            threads = permits = 1;
+
+        log.info("Evict partition permits=" + permits);
+
+        evictionQueue = new BucketQueue(threads);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void stop0(boolean cancel) {
+        super.stop0(cancel);
+
+        stop = true;
+
+        Collection<GroupEvictionContext> evictionGrps = evictionGroupsMap.values();
+
+        evictionGrps.forEach(GroupEvictionContext::stop);
+
+        evictionGrps.forEach(GroupEvictionContext::awaitFinishAll);
+    }
+
+    /**
+     *
+     */
+    private class GroupEvictionContext implements EvictionContext {
+        /** */
+        private final CacheGroupContext grp;
+
+        /** Deduplicate set partition ids. */
+        private final Set<Integer> partIds = new GridConcurrentHashSet<>();
+
+        /** Future for currently running partition eviction task. */
+        private final Map<Integer, IgniteInternalFuture<?>> partsEvictFutures = new ConcurrentHashMap<>();
+
+        /** Flag indicates that eviction process has stopped for this group. */
+        private volatile boolean stop;
+
+        /** Total partition to evict. */
+        private AtomicInteger totalTasks = new AtomicInteger();
+
+        /** Total partition evict in progress. */
+        private int taskInProgress;
+
+        /**
+         * @param grp Group context.
+         */
+        private GroupEvictionContext(CacheGroupContext grp) {
+            this.grp = grp;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean shouldStop() {
+            return stop || sharedEvictionContext.shouldStop();
+        }
+
+        /**
+         *
+         * @param part Grid local partition.
+         */
+        private PartitionEvictionTask createEvictPartitionTask(GridDhtLocalPartition part){
+            if (shouldStop() || !partIds.add(part.id()))
+                return null;
+
+            totalTasks.incrementAndGet();
+
+            return new PartitionEvictionTask(part, this);
+        }
+
+        /**
+         *
+         * @param task Partition eviction task.
+         */
+        private synchronized void taskScheduled(PartitionEvictionTask task) {
+            if (shouldStop())
+                return;
+
+            taskInProgress++;
+
+            GridFutureAdapter<?> fut = task.finishFut;
+
+            int partId = task.part.id();
+
+            partsEvictFutures.put(partId, fut);
+
+            fut.listen(f -> {
+                synchronized (this) {
+                    taskInProgress--;
+
+                    partsEvictFutures.remove(partId, f);
+
+                    if (totalTasks.decrementAndGet() == 0)
+                        evictionGroupsMap.remove(grp.groupId());
+                }
+            });
+        }
+
+        /**
+         * Stop eviction for group.
+         */
+        private void stop() {
+            stop = true;
+        }
+
+        /**
+         * Await evict finish.
+         */
+        private void awaitFinishAll(){
+            partsEvictFutures.forEach(this::awaitFinish);
+
+            evictionGroupsMap.remove(grp.groupId());
+        }
+
+        /**
+         * Await evict finish partition.
+         */
+        private void awaitFinish(Integer part, IgniteInternalFuture<?> fut) {
+            // Wait for last offered partition eviction completion
+            try {
+                log.info("Await partition evict, grpName=" + grp.cacheOrGroupName() +
+                    ", grpId=" + grp.groupId() + ", partId=" + part);
+
+                fut.get();
+            }
+            catch (IgniteCheckedException e) {
+                if (log.isDebugEnabled())
+                    log.warning("Failed to await partition eviction during stopping.", e);
+            }
+        }
+
+        /**
+         * Shows progress group of eviction.
+         */
+        private void showProgress() {
+            if (log.isInfoEnabled())
+                log.info("Group eviction in progress [grpName=" + grp.cacheOrGroupName()+
+                    ", grpId=" + grp.groupId() +
+                    ", remainingPartsToEvict=" + (totalTasks.get() - taskInProgress) +
+                    ", partsEvictInProgress=" + taskInProgress +
+                    ", totalParts= " + grp.topology().localPartitions().size() + "]");
+        }
+    }
+
+    /**
+     * Task for self-scheduled partition eviction / clearing.
+     */
+    private class PartitionEvictionTask implements Runnable {
+        /** Partition to evict. */
+        private final GridDhtLocalPartition part;
+
+        private final long size;
+
+        /** Eviction context. */
+        private final GroupEvictionContext groupEvictionContext;
+
+        /** */
+        private final GridFutureAdapter<?> finishFut = new GridFutureAdapter<>();
+
+        /**
+         * @param part Partition.
+         */
+        private PartitionEvictionTask(
+            GridDhtLocalPartition part,
+            GroupEvictionContext groupEvictionContext
+        ) {
+            this.part = part;
+            this.groupEvictionContext = groupEvictionContext;
+
+            size = part.fullSize();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            if (groupEvictionContext.shouldStop())
+                return;
+
+            try {
+                boolean success = part.tryClear(groupEvictionContext);
+
+                if (success) {
+                    if (part.state() == GridDhtPartitionState.EVICTED && part.markForDestroy())
+                        part.destroy();
+                }
+                else // Re-offer partition if clear was unsuccessful due to partition reservation.
+                    evictionQueue.offer(this);
+
+                // Complete eviction future before schedule new to prevent deadlock with
+                // simultaneous eviction stopping and scheduling new eviction.
+                finishFut.onDone();
+            }
+            catch (Throwable ex) {
+                finishFut.onDone(ex);
+
+                if (cctx.kernalContext().isStopping()) {
+                    LT.warn(log, ex, "Partition eviction failed (current node is stopping).",
+                        false,
+                        true);
+                }
+                else{
+                    LT.error(log, ex, "Partition eviction failed, this can cause grid hang.");
+                }
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    private class BucketQueue {
+        /** Queues contains partitions scheduled for eviction. */
+        private final Queue<PartitionEvictionTask>[] buckets;
+
+        /** */
+        private final long[] bucketSizes;
+
+        /**
+         * @param buckets Number of buckets.
+         */
+        BucketQueue(int buckets) {
+            this.buckets = new Queue[buckets];
+
+            for (int i = 0; i < buckets; i++)
+                this.buckets[i] = createEvictPartitionQueue();
+
+            bucketSizes = new long[buckets];
+        }
+
+        /**
+         * Poll eviction task from queue for specific bucket.
+         *
+         * @param bucket Bucket index.
+         * @return Partition evict task, or {@code null} if bucket queue is empty.
+         */
+        PartitionEvictionTask poll(int bucket) {
+            PartitionEvictionTask task = buckets[bucket].poll();
+
+            if (task != null)
+                bucketSizes[bucket] -= task.size;
+
+            return task;
+        }
+
+        /**
+         * Poll eviction task from queue (bucket is not specific).
+         *
+         * @return Partition evict task.
+         */
+        PartitionEvictionTask pollAny() {
+            for (int bucket = 0; bucket < bucketSizes.length; bucket++){
+                if (!buckets[bucket].isEmpty())
+                    return poll(bucket);
+            }
+
+            return null;
+        }
+
+        /**
+         * Offer task to queue.
+         *
+         * @return Bucket index.
+         */
+        int offer(PartitionEvictionTask task) {
+            int bucket = calculateBucket();
+
+            buckets[bucket].offer(task);
+
+            bucketSizes[bucket] += task.size;
+
+            return bucket;
+        }
+
+
+        /**
+         * @return {@code True} if queue is empty, {@code} False if not empty.
+         */
+        boolean isEmpty(){
+            return size() == 0;
+        }
+
+        /**
+         * @return Queue size.
+         */
+        int size(){
+            int size = 0;
+
+            for (Queue<PartitionEvictionTask> queue : buckets) {
+                size += queue.size();
+            }
+
+            return size;
+        }
+
+        /***
+         * @return Bucket index.
+         */
+        private int calculateBucket() {
+            int min = 0;
+
+            for (int bucket = min; bucket < bucketSizes.length; bucket++) {
+                if (bucketSizes[min] > bucketSizes[bucket])
+                    min = bucket;
+            }
+
+            return min;
+        }
+
+        /**
+         * 0 - PRIORITY QUEUE (compare by partition size).
+         * default (any other values) - FIFO.
+         */
+        private static final byte QUEUE_TYPE = 1;
+
+        /**
+         *
+         * @return Queue for evict partitions.
+         */
+        private Queue<PartitionEvictionTask> createEvictPartitionQueue() {
+            switch (QUEUE_TYPE) {
+                case 1:
+                    return new PriorityBlockingQueue<>(
+                        1000, Comparator.comparingLong(p -> p.part.fullSize()));
+                default:
+                    return new LinkedBlockingQueue<>();
+            }
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index 2bfc22d79ac54..42a3ff4ed6cc7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -327,7 +327,7 @@ private FileDescriptor readFileDescriptor(File file, FileIOFactory ioFactory) {
             kernalCtx, null, null, null,
             null, null, null, dbMgr, null,
             null, null, null, null,
-            null, null, null
+            null, null,null, null
         );
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
index 00ed6f166dcb0..9dbef5dcfa181 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
@@ -142,6 +142,7 @@ private void checkInvariantSwitchSegmentSize(int serVer) throws Exception {
                 null,
                 null,
                 null,
+                null,
 
                 null)
         ).createSerializer(serVer);
@@ -370,6 +371,7 @@ else if (walMgrClass.equals(FsyncModeFileWriteAheadLogManager.class)) {
             new GridCacheIoManager(),
             null,
             null,
+            null,
             null
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
index 373720456d149..7719b43feef92 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
@@ -60,6 +60,7 @@ public class BPlusTreePageMemoryImplTest extends BPlusTreeSelfTest {
             null,
             null,
             null,
+            null,
             null
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
index 0c786adc46408..2bafb08ee534b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
@@ -60,6 +60,7 @@ public class BPlusTreeReuseListPageMemoryImplTest extends BPlusTreeReuseSelfTest
             null,
             null,
             null,
+            null,
             null
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
index 9087b1c1d10e1..43fbb6e4a0e51 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IndexStoragePageMemoryImplTest.java
@@ -75,6 +75,7 @@ public class IndexStoragePageMemoryImplTest extends IndexStorageSelfTest {
             null,
             null,
             null,
+            null,
             null
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
index 34fd93b035c9c..52aff0ca16acd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
@@ -65,6 +65,7 @@ public class PageMemoryImplNoLoadTest extends PageMemoryNoLoadSelfTest {
             null,
             null,
             null,
+            null,
             null
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
index 3697c4cc7b499..000131a86b1ec 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -297,6 +297,7 @@ private PageMemoryImpl createPageMemory(PageMemoryImpl.ThrottlingPolicy throttli
             null,
             null,
             null,
+            null,
             null
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
index 393395393c15c..64d29e6e0927e 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheTtlManager;
 import org.apache.ignite.internal.processors.cache.WalStateManager;
 import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
+import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionsEvictManager;
 import org.apache.ignite.internal.processors.cache.dr.GridOsCacheDrManager;
 import org.apache.ignite.internal.processors.cache.jta.CacheNoopJtaManager;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
@@ -78,6 +79,7 @@ public GridCacheTestContext(GridTestKernalContext ctx) throws Exception {
                 new CacheAffinitySharedManager<K, V>(),
                 new GridCacheIoManager(),
                 new GridCacheSharedTtlCleanupManager(),
+                new PartitionsEvictManager(),
                 new CacheNoopJtaManager(),
                 null
             ),

From fcef6b826ef1eef13c14e752e52c1d05e49ac508 Mon Sep 17 00:00:00 2001
From: Alexey Kukushkin <AlexeyKukushkin@yahoo.com>
Date: Thu, 26 Apr 2018 19:31:43 +0300
Subject: [PATCH 292/543] IGNITE-8237 Ignite blocks on SecurityException in
 exchange-worker due to unauthorised on-heap cache configuration. - Fixes
 #3818.

Signed-off-by: dpavlov <dpavlov@apache.org>
(cherry picked from commit 54cb262438bc83af3c4e864a7e5897b36fcd8c73)
---
 .../processors/cache/GridCacheProcessor.java  | 52 ++++++++++++-------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index d392c6819b66c..93e384a7cf486 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -123,6 +123,7 @@
 import org.apache.ignite.internal.processors.query.schema.SchemaNodeLeaveExchangeWorkerTask;
 import org.apache.ignite.internal.processors.query.schema.message.SchemaAbstractDiscoveryMessage;
 import org.apache.ignite.internal.processors.query.schema.message.SchemaProposeDiscoveryMessage;
+import org.apache.ignite.internal.processors.security.SecurityContext;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
 import org.apache.ignite.internal.util.F0;
@@ -1179,9 +1180,6 @@ private void startCache(GridCacheAdapter<?, ?> cache, QuerySchema schema) throws
 
         CacheConfiguration cfg = cacheCtx.config();
 
-        if (cacheCtx.userCache())
-            authorizeCacheCreate(cacheCtx.name(), cfg);
-
         // Intentionally compare Boolean references using '!=' below to check if the flag has been explicitly set.
         if (cfg.isStoreKeepBinary() && cfg.isStoreKeepBinary() != CacheConfiguration.DFLT_STORE_KEEP_BINARY
             && !(ctx.config().getMarshaller() instanceof BinaryMarshaller))
@@ -2546,6 +2544,23 @@ private GridCacheSharedContext createSharedContext(GridKernalContext kernalCtx,
             StringBuilder errorMessage = new StringBuilder();
 
             for (CacheJoinNodeDiscoveryData.CacheInfo cacheInfo : nodeData.caches().values()) {
+                try {
+                    byte[] secCtxBytes = node.attribute(IgniteNodeAttributes.ATTR_SECURITY_SUBJECT_V2);
+
+                    if (secCtxBytes != null) {
+                        SecurityContext secCtx = U.unmarshal(marsh, secCtxBytes, U.resolveClassLoader(ctx.config()));
+
+                        if (secCtx != null && cacheInfo.cacheType() == CacheType.USER)
+                            authorizeCacheCreate(cacheInfo.cacheData().config(), secCtx);
+                    }
+                }
+                catch (SecurityException | IgniteCheckedException ex) {
+                    if (errorMessage.length() > 0)
+                        errorMessage.append("\n");
+
+                    errorMessage.append(ex.getMessage());
+                }
+
                 DynamicCacheDescriptor localDesc = cacheDescriptor(cacheInfo.cacheData().config().getName());
 
                 if (localDesc == null)
@@ -3409,30 +3424,29 @@ private Collection<DynamicCacheStartFuture> initiateCacheChanges(
     }
 
     /**
-     * Authorize dynamic cache management.
+     * Authorize creating cache.
+     */
+    private void authorizeCacheCreate(CacheConfiguration cfg, SecurityContext secCtx) {
+        ctx.security().authorize(null, SecurityPermission.CACHE_CREATE, secCtx);
+
+        if (cfg != null && cfg.isOnheapCacheEnabled() &&
+            IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_DISABLE_ONHEAP_CACHE))
+            throw new SecurityException("Authorization failed for enabling on-heap cache.");
+    }
+
+    /**
+     * Authorize dynamic cache management for this node.
      */
     private void authorizeCacheChange(DynamicCacheChangeRequest req) {
+        // Null security context means authorize this node.
         if (req.cacheType() == null || req.cacheType() == CacheType.USER) {
             if (req.stop())
-                ctx.security().authorize(req.cacheName(), SecurityPermission.CACHE_DESTROY, null);
+                ctx.security().authorize(null, SecurityPermission.CACHE_DESTROY, null);
             else
-                authorizeCacheCreate(req.cacheName(), req.startCacheConfiguration());
+                authorizeCacheCreate(req.startCacheConfiguration(), null);
         }
     }
 
-    /**
-     * Authorize start/create cache operation.
-     */
-    private void authorizeCacheCreate(String cacheName, CacheConfiguration cacheCfg) {
-        ctx.security().authorize(cacheName, SecurityPermission.CACHE_CREATE, null);
-
-        if (cacheCfg != null && cacheCfg.isOnheapCacheEnabled() &&
-            System.getProperty(IgniteSystemProperties.IGNITE_DISABLE_ONHEAP_CACHE, "false")
-                .toUpperCase().equals("TRUE")
-            )
-            throw new SecurityException("Authorization failed for enabling on-heap cache.");
-    }
-
     /**
      * @return Non null exception if node is stopping or disconnected.
      */

From 0a12c50b4755380ab1b25dceee7420d465a6bfec Mon Sep 17 00:00:00 2001
From: dpavlov <dpavlov@apache.org>
Date: Thu, 26 Apr 2018 19:38:05 +0300
Subject: [PATCH 293/543] IGNITE-8237 Javadoc for method parameters added.

(cherry picked from commit ebe55e3ff84232f67a2885354e3e26426a6b16cb)
---
 .../ignite/internal/processors/cache/GridCacheProcessor.java   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 93e384a7cf486..e788763a1f07a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3425,6 +3425,8 @@ private Collection<DynamicCacheStartFuture> initiateCacheChanges(
 
     /**
      * Authorize creating cache.
+     * @param cfg Cache configuration.
+     * @param secCtx Optional security context.
      */
     private void authorizeCacheCreate(CacheConfiguration cfg, SecurityContext secCtx) {
         ctx.security().authorize(null, SecurityPermission.CACHE_CREATE, secCtx);
@@ -3436,6 +3438,7 @@ private void authorizeCacheCreate(CacheConfiguration cfg, SecurityContext secCtx
 
     /**
      * Authorize dynamic cache management for this node.
+     * @param req start/stop cache request.
      */
     private void authorizeCacheChange(DynamicCacheChangeRequest req) {
         // Null security context means authorize this node.

From 35cc55405823482e51929d42b74c5e9030bb74e9 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Fri, 10 Aug 2018 16:21:25 +0300
Subject: [PATCH 294/543] IGNITE-8724 Fixed misleading U.warn implementation -
 Fixes #4145.

---
 .../RendezvousAffinityFunction.java           |  4 +-
 .../ignite/internal/GridDiagnostic.java       | 22 +++----
 .../apache/ignite/internal/IgniteKernal.java  |  7 +--
 .../apache/ignite/internal/IgnitionEx.java    |  3 +-
 .../impl/GridTcpRouterNioListenerAdapter.java |  3 +-
 .../checkpoint/GridCheckpointManager.java     | 11 ++--
 .../deployment/GridDeploymentManager.java     |  7 +--
 .../discovery/GridDiscoveryManager.java       |  3 +-
 .../cache/GridCacheEvictionManager.java       |  3 +-
 .../processors/cache/GridCacheProcessor.java  | 12 ++--
 .../protocols/tcp/GridTcpRestProtocol.java    | 10 ++--
 .../ignite/internal/util/GridLogThrottle.java | 60 ++++++++-----------
 .../ignite/internal/util/IgniteUtils.java     | 23 ++++---
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 17 ++----
 .../processors/query/h2/IgniteH2Indexing.java |  6 +-
 .../zk/internal/ZookeeperDiscoveryImpl.java   |  9 +--
 16 files changed, 77 insertions(+), 123 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
index 0e9afb79f2a9d..4f6be40306176 100644
--- a/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
+++ b/modules/core/src/main/java/org/apache/ignite/cache/affinity/rendezvous/RendezvousAffinityFunction.java
@@ -388,9 +388,7 @@ else if ((backupFilter != null && backupFilter.apply(primary, node))
 
             if (!exclNeighborsWarn) {
                 LT.warn(log, "Affinity function excludeNeighbors property is ignored " +
-                        "because topology has no enough nodes to assign backups.",
-                    "Affinity function excludeNeighbors property is ignored " +
-                        "because topology has no enough nodes to assign backups.");
+                    "because topology has no enough nodes to assign backups.");
 
                 exclNeighborsWarn = true;
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridDiagnostic.java b/modules/core/src/main/java/org/apache/ignite/internal/GridDiagnostic.java
index 4a33b9d089635..ac623ddf59007 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridDiagnostic.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridDiagnostic.java
@@ -61,14 +61,12 @@ static void runBackgroundCheck(String igniteInstanceName, Executor exec, IgniteL
 
                         if (!locHost.isReachable(REACH_TIMEOUT)) {
                             U.warn(log, "Default local host is unreachable. This may lead to delays on " +
-                                "grid network operations. Check your OS network setting to correct it.",
-                                "Default local host is unreachable.");
+                                "grid network operations. Check your OS network setting to correct it.");
                         }
                     }
                     catch (IOException ignore) {
                         U.warn(log, "Failed to perform network diagnostics. It is usually caused by serious " +
-                            "network configuration problem. Check your OS network setting to correct it.",
-                            "Failed to perform network diagnostics.");
+                            "network configuration problem. Check your OS network setting to correct it.");
                     }
                 }
             });
@@ -80,14 +78,12 @@ static void runBackgroundCheck(String igniteInstanceName, Executor exec, IgniteL
 
                         if (locHost.isLoopbackAddress()) {
                             U.warn(log, "Default local host is a loopback address. This can be a sign of " +
-                                "potential network configuration problem.",
-                                "Default local host is a loopback address.");
+                                "potential network configuration problem.");
                         }
                     }
                     catch (IOException ignore) {
                         U.warn(log, "Failed to perform network diagnostics. It is usually caused by serious " +
-                            "network configuration problem. Check your OS network setting to correct it.",
-                            "Failed to perform network diagnostics.");
+                            "network configuration problem. Check your OS network setting to correct it.");
                     }
                 }
             });
@@ -98,8 +94,7 @@ static void runBackgroundCheck(String igniteInstanceName, Executor exec, IgniteL
                     if (!U.isSufficientlyTestedOs()) {
                         U.warn(log, "This operating system has been tested less rigorously: " + U.osString() +
                             ". Our team will appreciate the feedback if you experience any problems running " +
-                            "ignite in this environment.",
-                            "This OS is tested less rigorously: " + U.osString());
+                            "ignite in this environment.");
                     }
                 }
             });
@@ -109,8 +104,7 @@ static void runBackgroundCheck(String igniteInstanceName, Executor exec, IgniteL
                     // Fix for GG-1075.
                     if (F.isEmpty(U.allLocalMACs()))
                         U.warn(log, "No live network interfaces detected. If IP-multicast discovery is used - " +
-                            "make sure to add 127.0.0.1 as a local address.",
-                            "No live network interfaces. Add 127.0.0.1 as a local address.");
+                            "make sure to add 127.0.0.1 as a local address.");
                 }
             });
 
@@ -131,7 +125,7 @@ static void runBackgroundCheck(String igniteInstanceName, Executor exec, IgniteL
 
                         U.warn(log, "JMX remote management is enabled but JMX port is either not set or invalid. " +
                             "Check system property 'com.sun.management.jmxremote.port' to make sure it specifies " +
-                            "valid TCP/IP port.", "JMX remote port is invalid - JMX management is off.");
+                            "valid TCP/IP port.");
                     }
                 }
             });
@@ -156,4 +150,4 @@ static void runBackgroundCheck(String igniteInstanceName, Executor exec, IgniteL
                 "Failed to start background network diagnostics.", e);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 3c78927dc6c12..4979e610b9046 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1589,8 +1589,7 @@ private void fillNodeAttributes(boolean notifyEnabled) throws IgniteCheckedExcep
         // Warn about loopback.
         if (ips.isEmpty() && macs.isEmpty())
             U.warn(log, "Ignite is starting on loopback address... Only nodes on the same physical " +
-                    "computer can participate in topology.",
-                "Ignite is starting on loopback address...");
+                "computer can participate in topology.");
 
         // Stick in network context into attributes.
         add(ATTR_IPS, (ips.isEmpty() ? "" : ips));
@@ -2612,9 +2611,7 @@ private void ackP2pConfiguration() {
             U.warn(
                 log,
                 "Peer class loading is enabled (disable it in production for performance and " +
-                    "deployment consistency reasons)",
-                "Peer class loading is enabled (disable it for better performance)"
-            );
+                    "deployment consistency reasons)");
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index c58c677151ed4..6aa9547e35946 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -2259,8 +2259,7 @@ private IgniteConfiguration initializeConfiguration(IgniteConfiguration cfg)
                         "(only recent 1.6 and 1.7 versions HotSpot VMs are supported). " +
                         "To enable fast marshalling upgrade to recent 1.6 or 1.7 HotSpot VM release. " +
                         "Switching to standard JDK marshalling - " +
-                        "object serialization performance will be significantly slower.",
-                        "To enable fast marshalling upgrade to recent 1.6 or 1.7 HotSpot VM release.");
+                        "object serialization performance will be significantly slower.");
 
                     marsh = new JdkMarshaller();
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
index 75aa6f29ccc3f..9345b754a8390 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/router/impl/GridTcpRouterNioListenerAdapter.java
@@ -150,8 +150,7 @@ public GridTcpRouterNioListenerAdapter(IgniteLogger log, GridRouterClientImpl cl
 
                 U.warn(
                     log,
-                    "Message forwarding was interrupted (will ignore last message): " + e.getMessage(),
-                    "Message forwarding was interrupted.");
+                    "Message forwarding was interrupted (will ignore last message): " + e.getMessage());
             }
         }
         else if (msg instanceof GridClientHandshakeRequest) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
index f0b19f3741270..6337d533f192c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointManager.java
@@ -188,8 +188,7 @@ public boolean storeCheckpoint(GridTaskSessionInternal ses,
                         U.warn(log, S.toString("Checkpoint will not be saved due to session invalidation",
                             "key", key, true,
                             "val", state, true,
-                            "ses", ses, false),
-                            "Checkpoint will not be saved due to session invalidation.");
+                            "ses", ses, false));
 
                         break;
                     }
@@ -198,8 +197,7 @@ public boolean storeCheckpoint(GridTaskSessionInternal ses,
                         U.warn(log, S.toString("Checkpoint will not be saved due to session timeout",
                             "key", key, true,
                             "val", state, true,
-                            "ses", ses, false),
-                            "Checkpoint will not be saved due to session timeout.");
+                            "ses", ses, false));
 
                         break;
                     }
@@ -224,8 +222,7 @@ public boolean storeCheckpoint(GridTaskSessionInternal ses,
                             U.warn(log, S.toString("Checkpoint will not be saved due to session invalidation",
                                 "key", key, true,
                                 "val", state, true,
-                                "ses", ses, false),
-                                "Checkpoint will not be saved due to session invalidation.");
+                                "ses", ses, false));
 
                             keyMap.remove(ses.getId(), keys);
 
@@ -508,4 +505,4 @@ private class CheckpointRequestListener implements GridMessageListener {
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentManager.java
index cea178604be50..01d8604ceaffd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentManager.java
@@ -470,7 +470,7 @@ else if (locDep != null) {
                             "in some other mode). Either change IgniteConfiguration.getDeploymentMode() property to " +
                             "SHARED or CONTINUOUS or remove class from local classpath and any of " +
                             "the local GAR deployments that may have it [cls=" + meta.className() + ", depMode=" +
-                            locDep.deployMode() + ']', "Failed to deploy class in SHARED or CONTINUOUS mode.");
+                            locDep.deployMode() + ']');
 
                         return null;
                     }
@@ -478,8 +478,7 @@ else if (locDep != null) {
                     if (!locDep.userVersion().equals(meta.userVersion())) {
                         U.warn(log, "Failed to deploy class in SHARED or CONTINUOUS mode for given user version " +
                             "(class is locally deployed for a different user version) [cls=" + meta.className() +
-                            ", localVer=" + locDep.userVersion() + ", otherVer=" + meta.userVersion() + ']',
-                            "Failed to deploy class in SHARED or CONTINUOUS mode.");
+                            ", localVer=" + locDep.userVersion() + ", otherVer=" + meta.userVersion() + ']');
 
                         return null;
                     }
@@ -664,4 +663,4 @@ private LocalDeployment(DeploymentMode depMode, ClassLoader clsLdr, IgniteUuid c
             return S.toString(LocalDeployment.class, this, super.toString());
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 38ce9bdebc6a7..82abdf5213f9e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -1221,8 +1221,7 @@ private void checkAttributes(Iterable<ClusterNode> nodes) throws IgniteCheckedEx
                         "(all nodes in topology should have identical value) " +
                         "[locPreferIpV4=" + locPreferIpV4 + ", rmtPreferIpV4=" + rmtPreferIpV4 +
                         ", locId8=" + U.id8(locNode.id()) + ", rmtId8=" + U.id8(n.id()) +
-                        ", rmtAddrs=" + U.addressesAsString(n) + ", rmtNode=" + U.toShortString(n) + "]",
-                        "Local and remote 'java.net.preferIPv4Stack' system properties do not match.");
+                        ", rmtAddrs=" + U.addressesAsString(n) + ", rmtNode=" + U.toShortString(n) + "]");
 
                 ipV4Warned = true;
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
index 2c9dec7f5d25b..5d11ca1bd80e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEvictionManager.java
@@ -233,8 +233,7 @@ private void warnFirstEvict() {
         }
 
         U.warn(log, "Evictions started (cache may have reached its capacity)." +
-                " You may wish to increase 'maxSize' on eviction policy being used for cache: " + cctx.name(),
-            "Evictions started (cache may have reached its capacity): " + cctx.name());
+            " You may wish to increase 'maxSize' on eviction policy being used for cache: " + cctx.name());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index e788763a1f07a..a4078f7652179 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -509,18 +509,15 @@ private void validate(IgniteConfiguration c,
 
         if (delay != 0) {
             if (cc.getCacheMode() != PARTITIONED)
-                U.warn(log, "Rebalance delay is supported only for partitioned caches (will ignore): " + (cc.getName()),
-                    "Will ignore rebalance delay for cache: " + U.maskName(cc.getName()));
+                U.warn(log, "Rebalance delay is supported only for partitioned caches (will ignore): " + (cc.getName()));
             else if (cc.getRebalanceMode() == SYNC) {
                 if (delay < 0) {
                     U.warn(log, "Ignoring SYNC rebalance mode with manual rebalance start (node will not wait for " +
-                            "rebalancing to be finished): " + U.maskName(cc.getName()),
-                        "Node will not wait for rebalance in SYNC mode: " + U.maskName(cc.getName()));
+                        "rebalancing to be finished): " + U.maskName(cc.getName()));
                 }
                 else {
                     U.warn(log, "Using SYNC rebalance mode with rebalance delay (node will wait until rebalancing is " +
-                            "initiated for " + delay + "ms) for cache: " + U.maskName(cc.getName()),
-                        "Node will wait until rebalancing is initiated for " + delay + "ms for cache: " + U.maskName(cc.getName()));
+                        "initiated for " + delay + "ms) for cache: " + U.maskName(cc.getName()));
                 }
             }
         }
@@ -683,8 +680,7 @@ private void cleanup(CacheConfiguration cfg, @Nullable Object rsrc, boolean near
         if (!F.isEmpty(ctx.config().getCacheConfiguration())) {
             if (depMode != CONTINUOUS && depMode != SHARED)
                 U.warn(log, "Deployment mode for cache is not CONTINUOUS or SHARED " +
-                        "(it is recommended that you change deployment mode and restart): " + depMode,
-                    "Deployment mode for cache is not CONTINUOUS or SHARED.");
+                    "(it is recommended that you change deployment mode and restart): " + depMode);
         }
 
         initializeInternalCacheNames();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
index 0049dbc5d72f3..febbc5882e081 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestProtocol.java
@@ -126,14 +126,12 @@ else if (depFactory != null)
                 "[firstPort=" + cfg.getPort() + ", lastPort=" + lastPort + ", host=" + host + ']');
         }
         catch (SSLException e) {
-            U.warn(log, "Failed to start " + name() + " protocol on port " + port + ": " + e.getMessage(),
-                "Failed to start " + name() + " protocol on port " + port + ". Check if SSL context factory is " +
-                    "properly configured.");
+            U.warn(log, "Failed to start " + name() + " protocol on port " + port + ". Check if SSL context factory " +
+                "is properly configured: " + e.getMessage());
         }
         catch (IOException e) {
-            U.warn(log, "Failed to start " + name() + " protocol on port " + port + ": " + e.getMessage(),
-                "Failed to start " + name() + " protocol on port " + port + ". " +
-                    "Check restTcpHost configuration property.");
+            U.warn(log, "Failed to start " + name() + " protocol on port " + port + ". " +
+                "Check restTcpHost configuration property: " + e.getMessage());
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java
index 59ddd1c88a63b..394e4dc4a9985 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLogThrottle.java
@@ -72,7 +72,7 @@ public static long throttleTimeout() {
     public static void error(@Nullable IgniteLogger log, @Nullable Throwable e, String msg) {
         assert !F.isEmpty(msg);
 
-        log(log, e, msg, null, LogLevel.ERROR, false, false);
+        log(log, e, msg, LogLevel.ERROR, false, false);
     }
 
     /**
@@ -86,7 +86,7 @@ public static void error(@Nullable IgniteLogger log, @Nullable Throwable e, Stri
     public static void error(@Nullable IgniteLogger log, @Nullable Throwable e, String msg, boolean byMsg) {
         assert !F.isEmpty(msg);
 
-        log(log, e, msg, null, LogLevel.ERROR, false, byMsg);
+        log(log, e, msg, LogLevel.ERROR, false, byMsg);
     }
 
     /**
@@ -98,7 +98,7 @@ public static void error(@Nullable IgniteLogger log, @Nullable Throwable e, Stri
     public static void warn(@Nullable IgniteLogger log, String msg) {
         assert !F.isEmpty(msg);
 
-        log(log, null, msg, null, LogLevel.WARN, false, false);
+        log(log, null, msg, LogLevel.WARN, false, false);
     }
 
     /**
@@ -113,7 +113,7 @@ public static void warn(@Nullable IgniteLogger log, String msg) {
     public static void warn(@Nullable IgniteLogger log, @Nullable Throwable e, String msg, boolean quite, boolean byMsg) {
         assert !F.isEmpty(msg);
 
-        log(log, e, msg, null, LogLevel.WARN, quite, byMsg);
+        log(log, e, msg, LogLevel.WARN, quite, byMsg);
     }
 
 
@@ -127,20 +127,7 @@ public static void warn(@Nullable IgniteLogger log, @Nullable Throwable e, Strin
     public static void warn(@Nullable IgniteLogger log, String msg, boolean quiet) {
         assert !F.isEmpty(msg);
 
-        log(log, null, msg, null, LogLevel.WARN, quiet, false);
-    }
-
-    /**
-     * Logs warning if needed.
-     *
-     * @param log Logger.
-     * @param longMsg Long message (or just message).
-     * @param shortMsg Short message for quiet logging.
-     */
-    public static void warn(@Nullable IgniteLogger log, String longMsg, @Nullable String shortMsg) {
-        assert !F.isEmpty(longMsg);
-
-        log(log, null, longMsg, shortMsg, LogLevel.WARN, false, false);
+        log(log, null, msg, LogLevel.WARN, quiet, false);
     }
 
     /**
@@ -153,7 +140,7 @@ public static void warn(@Nullable IgniteLogger log, String longMsg, @Nullable St
     public static void info(@Nullable IgniteLogger log, String msg, boolean quiet) {
         assert !F.isEmpty(msg);
 
-        log(log, null, msg, null, LogLevel.INFO, quiet, false);
+        log(log, null, msg, LogLevel.INFO, quiet, false);
     }
 
     /**
@@ -181,13 +168,17 @@ public static void clear() {
      * @param log Logger.
      * @param e Error (optional).
      * @param longMsg Long message (or just message).
-     * @param shortMsg Short message for quiet logging.
      * @param level Level where messages should appear.
      * @param byMsg Errors group by message, not by tuple(error, msg).
      */
     @SuppressWarnings({"RedundantTypeArguments"})
-    private static void log(@Nullable IgniteLogger log, @Nullable Throwable e, String longMsg,
-        @Nullable String shortMsg, LogLevel level, boolean quiet, boolean byMsg) {
+    private static void log(@Nullable IgniteLogger log,
+        @Nullable Throwable e,
+        String longMsg,
+        LogLevel level,
+        boolean quiet,
+        boolean byMsg
+    ) {
         assert !F.isEmpty(longMsg);
 
         IgniteBiTuple<Class<? extends Throwable>, String> tup =
@@ -201,7 +192,7 @@ private static void log(@Nullable IgniteLogger log, @Nullable Throwable e, Strin
 
             if (loggedTs == null || loggedTs < curTs - throttleTimeout) {
                 if (replace(tup, loggedTs, curTs)) {
-                    level.doLog(log, longMsg, shortMsg, e, quiet);
+                    level.doLog(log, longMsg, e, quiet);
 
                     break;
                 }
@@ -242,32 +233,32 @@ protected GridLogThrottle() {
     private enum LogLevel {
         /** Error level. */
         ERROR {
-            @Override public void doLog(IgniteLogger log, String longMsg, String shortMsg, Throwable e, boolean quiet) {
+            @Override public void doLog(IgniteLogger log, String msg, Throwable e, boolean quiet) {
                 if (e != null)
-                    U.error(log, longMsg, e);
+                    U.error(log, msg, e);
                 else
-                    U.error(log, longMsg);
+                    U.error(log, msg);
             }
         },
 
         /** Warn level. */
         WARN {
-            @Override public void doLog(IgniteLogger log, String longMsg, String shortMsg, Throwable e, boolean quiet) {
+            @Override public void doLog(IgniteLogger log, String msg, Throwable e, boolean quiet) {
                 if (quiet)
-                    U.quietAndWarn(log, longMsg, F.isEmpty(shortMsg) ? longMsg : shortMsg);
+                    U.quietAndWarn(log, msg);
                 else
-                    U.warn(log, longMsg, F.isEmpty(shortMsg) ? longMsg : shortMsg);
+                    U.warn(log, msg);
             }
         },
 
         /** Info level. */
         INFO {
-            @Override public void doLog(IgniteLogger log, String longMsg, String shortMsg, Throwable e, boolean quiet) {
+            @Override public void doLog(IgniteLogger log, String msg, Throwable e, boolean quiet) {
                 if (quiet)
-                    U.quietAndInfo(log, longMsg);
+                    U.quietAndInfo(log, msg);
                 else {
                     if (log.isInfoEnabled())
-                        log.info(longMsg);
+                        log.info(msg);
                 }
             }
         };
@@ -276,10 +267,9 @@ private enum LogLevel {
          * Performs logging operation.
          *
          * @param log Logger to use.
-         * @param longMsg Long message.
-         * @param shortMsg Short message.
+         * @param msg Long message.
          * @param e Exception to attach to log.
          */
-        public abstract void doLog(IgniteLogger log, String longMsg, String shortMsg, Throwable e, boolean quiet);
+        public abstract void doLog(IgniteLogger log, String msg, Throwable e, boolean quiet);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index a76f20c21512b..8bfb33fa20c43 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -4210,7 +4210,7 @@ public static void warn(@Nullable IgniteLogger log, Object msg) {
 
         String s = msg.toString();
 
-        warn(log, s, s);
+        warn(log, s, null);
     }
 
     /**
@@ -4266,18 +4266,23 @@ public static void error(@Nullable IgniteLogger log, Object msg) {
      * or in QUIET mode it will add {@code (wrn)} prefix to the message.
      *
      * @param log Optional logger to use when QUIET mode is not enabled.
-     * @param longMsg Message to log using normal logger.
-     * @param shortMsg Message to log using quiet logger.
+     * @param msg Message to log using normal logger.
+     * @param e Optional exception.
      */
-    public static void warn(@Nullable IgniteLogger log, Object longMsg, Object shortMsg) {
-        assert longMsg != null;
-        assert shortMsg != null;
+    public static void warn(@Nullable IgniteLogger log, Object msg, @Nullable Throwable e) {
+        assert msg != null;
 
         if (log != null)
-            log.warning(compact(longMsg.toString()));
-        else
+            log.warning(compact(msg.toString()), e);
+        else {
             X.println("[" + SHORT_DATE_FMT.format(new java.util.Date()) + "] (wrn) " +
-                compact(shortMsg.toString()));
+                    compact(msg.toString()));
+
+            if (e != null)
+                e.printStackTrace(System.err);
+            else
+                X.printerrln();
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 5ce7f20bec33b..e4ea0be621e5e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -3583,9 +3583,7 @@ else if (log.isDebugEnabled())
                         if (subj == null) {
                             // Node has not pass authentication.
                             LT.warn(log, "Authentication failed [nodeId=" + node.id() +
-                                    ", addrs=" + U.addressesAsString(node) + ']',
-                                "Authentication failed [nodeId=" + U.id8(node.id()) + ", addrs=" +
-                                    U.addressesAsString(node) + ']');
+                                ", addrs=" + U.addressesAsString(node) + ']');
 
                             // Always output in debug.
                             if (log.isDebugEnabled())
@@ -3614,10 +3612,7 @@ else if (log.isDebugEnabled())
                             if (!(subj instanceof Serializable)) {
                                 // Node has not pass authentication.
                                 LT.warn(log, "Authentication subject is not Serializable [nodeId=" + node.id() +
-                                        ", addrs=" + U.addressesAsString(node) + ']',
-                                    "Authentication subject is not Serializable [nodeId=" + U.id8(node.id()) +
-                                        ", addrs=" +
-                                        U.addressesAsString(node) + ']');
+                                    ", addrs=" + U.addressesAsString(node) + ']');
 
                                 authFailedMsg = "Authentication subject is not serializable";
                             }
@@ -4213,9 +4208,7 @@ else if (!locNodeId.equals(node.id()) && ring.node(node.id()) != null) {
                             if (!permissionsEqual(coordSubj.subject().permissions(), subj.subject().permissions())) {
                                 // Node has not pass authentication.
                                 LT.warn(log, "Authentication failed [nodeId=" + node.id() +
-                                        ", addrs=" + U.addressesAsString(node) + ']',
-                                    "Authentication failed [nodeId=" + U.id8(node.id()) + ", addrs=" +
-                                        U.addressesAsString(node) + ']');
+                                    ", addrs=" + U.addressesAsString(node) + ']');
 
                                 // Always output in debug.
                                 if (log.isDebugEnabled())
@@ -4318,9 +4311,7 @@ else if (!locNodeId.equals(node.id()) && ring.node(node.id()) != null) {
                                         LT.warn(log,
                                             "Failed to authenticate local node " +
                                                 "(local authentication result is different from rest of topology) " +
-                                                "[nodeId=" + node.id() + ", addrs=" + U.addressesAsString(node) + ']',
-                                            "Authentication failed [nodeId=" + U.id8(node.id()) +
-                                                ", addrs=" + U.addressesAsString(node) + ']');
+                                                "[nodeId=" + node.id() + ", addrs=" + U.addressesAsString(node) + ']');
 
                                         joinRes.set(authFail);
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index b113a8b065b95..920a99e59f72b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -1211,19 +1211,17 @@ private ResultSet executeSqlQueryWithTimer(PreparedStatement stmt, Connection co
             long longQryExecTimeout = ctx.config().getLongQueryWarningTimeout();
 
             if (time > longQryExecTimeout) {
-                String msg = "Query execution is too long (" + time + " ms): " + sql;
-
                 ResultSet plan = executeSqlQuery(conn, preparedStatementWithParams(conn, "EXPLAIN " + sql,
                     params, false), 0, null);
 
                 plan.next();
 
                 // Add SQL explain result message into log.
-                String longMsg = "Query execution is too long [time=" + time + " ms, sql='" + sql + '\'' +
+                String msg = "Query execution is too long [time=" + time + " ms, sql='" + sql + '\'' +
                     ", plan=" + U.nl() + plan.getString(1) + U.nl() + ", parameters=" +
                     (params == null ? "[]" : Arrays.deepToString(params.toArray())) + "]";
 
-                LT.warn(log, longMsg, msg);
+                LT.warn(log, msg);
             }
 
             return rs;
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 43d6aeb9f3a6a..f480933b1a108 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -2022,9 +2022,7 @@ private ZkNodeValidateResult authenticateNode(ZookeeperClusterNode node) {
 
         if (subj == null) {
             U.warn(log, "Authentication failed [nodeId=" + node.id() +
-                    ", addrs=" + U.addressesAsString(node) + ']',
-                "Authentication failed [nodeId=" + U.id8(node.id()) + ", addrs=" +
-                    U.addressesAsString(node) + ']');
+                ", addrs=" + U.addressesAsString(node) + ']');
 
             // Note: exception message test is checked in tests.
             return new ZkNodeValidateResult("Authentication failed");
@@ -2032,10 +2030,7 @@ private ZkNodeValidateResult authenticateNode(ZookeeperClusterNode node) {
 
         if (!(subj instanceof Serializable)) {
             U.warn(log, "Authentication subject is not Serializable [nodeId=" + node.id() +
-                    ", addrs=" + U.addressesAsString(node) + ']',
-                "Authentication subject is not Serializable [nodeId=" + U.id8(node.id()) +
-                    ", addrs=" +
-                    U.addressesAsString(node) + ']');
+                ", addrs=" + U.addressesAsString(node) + ']');
 
             return new ZkNodeValidateResult("Authentication subject is not serializable");
         }

From 8ee5db8f253a0092a2200593fa28887878cd9a15 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Fri, 10 Aug 2018 15:32:19 +0300
Subject: [PATCH 295/543] IGNITE-9050 WAL iterator should throw an exception if
 segment tail is reached inside archive directory - Fixes #4429.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit dbf5574)
---
 .../wal/AbstractWalRecordsIterator.java       |  17 +-
 .../wal/reader/IgniteWalIteratorFactory.java  |  15 +-
 .../wal/serializer/RecordV2Serializer.java    |  19 +-
 ...ReachedDuringIterationOverArchiveTest.java | 245 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 5 files changed, 279 insertions(+), 20 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index 01b093399fbc1..ac68ea9a182aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -162,6 +162,13 @@ protected void advance() throws IgniteCheckedException {
                 }
             }
             catch (WalSegmentTailReachedException e) {
+                AbstractReadFileHandle currWalSegment = this.currWalSegment;
+
+                if (!currWalSegment.workDir())
+                    throw new IgniteCheckedException(
+                        "WAL tail reached in archive directory, " +
+                            "WAL segment file is corrupted.", e);
+
                 log.warning(e.getMessage());
 
                 curRec = null;
@@ -197,7 +204,8 @@ protected void advance() throws IgniteCheckedException {
      * @throws IgniteCheckedException if reading failed
      */
     protected abstract AbstractReadFileHandle advanceSegment(
-        @Nullable final AbstractReadFileHandle curWalSegment) throws IgniteCheckedException;
+        @Nullable final AbstractReadFileHandle curWalSegment
+    ) throws IgniteCheckedException;
 
     /**
      * Switches to new record.
@@ -222,8 +230,11 @@ private IgniteBiTuple<WALPointer, WALRecord> advanceRecord(
             return new IgniteBiTuple<>((WALPointer)actualFilePtr, postProcessRecord(rec));
         }
         catch (IOException | IgniteCheckedException e) {
-            if (e instanceof WalSegmentTailReachedException)
-                throw (WalSegmentTailReachedException)e;
+            if (e instanceof WalSegmentTailReachedException) {
+                throw new WalSegmentTailReachedException(
+                    "WAL segment tail reached. [idx=" + hnd.idx() +
+                        ", isWorkDir=" + hnd.workDir() + ", serVer=" + hnd.ser() + "]", e);
+            }
 
             if (!(e instanceof SegmentEofException) && !(e instanceof EOFException)) {
                 IgniteCheckedException e0 = handleRecordException(e, actualFilePtr);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index 42a3ff4ed6cc7..aae9775207612 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -133,10 +133,7 @@ public WALIterator iterator(
         return new StandaloneWalRecordsIterator(log,
             prepareSharedCtx(iteratorParametersBuilder),
             iteratorParametersBuilder.ioFactory,
-            resolveWalFiles(
-                iteratorParametersBuilder.filesOrDirs,
-                iteratorParametersBuilder
-            ),
+            resolveWalFiles(iteratorParametersBuilder),
             iteratorParametersBuilder.filter,
             iteratorParametersBuilder.keepBinary,
             iteratorParametersBuilder.bufferSize
@@ -182,10 +179,7 @@ public List<T2<Long, Long>> hasGaps(
 
         List<T2<Long, Long>> gaps = new ArrayList<>();
 
-        List<FileDescriptor> descriptors = resolveWalFiles(
-            iteratorParametersBuilder.filesOrDirs,
-            iteratorParametersBuilder
-        );
+        List<FileDescriptor> descriptors = resolveWalFiles(iteratorParametersBuilder);
 
         Iterator<FileDescriptor> it = descriptors.iterator();
 
@@ -217,10 +211,11 @@ public List<T2<Long, Long>> hasGaps(
      * @param iteratorParametersBuilder IteratorParametersBuilder.
      * @return list of file descriptors with checked header records, having correct file index is set
      */
-    private List<FileDescriptor> resolveWalFiles(
-        File[] filesOrDirs,
+    public List<FileDescriptor> resolveWalFiles(
         IteratorParametersBuilder iteratorParametersBuilder
     ) {
+        File[] filesOrDirs = iteratorParametersBuilder.filesOrDirs;
+
         if (filesOrDirs == null || filesOrDirs.length == 0)
             return Collections.emptyList();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
index 2c65ebe1d2c96..68e55e0cfeb24 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
@@ -113,10 +113,14 @@ public class RecordV2Serializer implements RecordSerializer {
             if (recType == SWITCH_SEGMENT_RECORD)
                 throw new SegmentEofException("Reached end of segment", null);
 
-            FileWALPointer ptr = readPositionAndCheckPoint(in, expPtr, skipPositionCheck);
+            FileWALPointer ptr = readPositionAndCheckPoint(in, expPtr, skipPositionCheck, recType);
 
-            if (recType == null)
-                throw new IOException("Unknown record type: " + recType);
+            if (recType == null) {
+                FileWALPointer exp = (FileWALPointer)expPtr;
+
+                throw new IOException("Unknown record type: " + recType +
+                    ", expected pointer [idx=" + exp.index() + ", offset=" + exp.fileOffset() + "]");
+            }
 
             if (recordFilter != null && !recordFilter.apply(recType, ptr)) {
                 int toSkip = ptr.length() - REC_TYPE_SIZE - FILE_WAL_POINTER_SIZE - CRC_SIZE;
@@ -241,7 +245,8 @@ public RecordV2Serializer(
     private static FileWALPointer readPositionAndCheckPoint(
         DataInput in,
         WALPointer expPtr,
-        boolean skipPositionCheck
+        boolean skipPositionCheck,
+        WALRecord.RecordType type
     ) throws IgniteCheckedException, IOException {
         long idx = in.readLong();
         int fileOff = in.readInt();
@@ -251,9 +256,9 @@ private static FileWALPointer readPositionAndCheckPoint(
 
         if (!F.eq(idx, p.index()) || (!skipPositionCheck && !F.eq(fileOff, p.fileOffset())))
             throw new WalSegmentTailReachedException(
-                "WAL segment tail is reached. [ " +
-                        "Expected next state: {Index=" + p.index() + ",Offset=" + p.fileOffset() + "}, " +
-                        "Actual state : {Index=" + idx + ",Offset=" + fileOff + "} ]", null);
+                "WAL segment tail reached. [ " +
+                    "Expected next state: {Index=" + p.index() + ",Offset=" + p.fileOffset() + "}, " +
+                    "Actual state : {Index=" + idx + ",Offset=" + fileOff + "} ] recordType=" + type, null);
 
         return new FileWALPointer(idx, fileOff, len);
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
new file mode 100644
index 0000000000000..a7c408180f043
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+import static java.nio.ByteBuffer.allocate;
+import static java.nio.file.StandardOpenOption.WRITE;
+import static java.util.concurrent.ThreadLocalRandom.current;
+
+/**
+ *
+ */
+public class IgniteWALTailIsReachedDuringIterationOverArchiveTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** WAL segment size. */
+    private static final int WAL_SEGMENT_SIZE = 10 * 1024 * 1024;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(name);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setWalSegmentSize(WAL_SEGMENT_SIZE)
+                .setWalSegments(2)
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+
+        Ignite ig = startGrid();
+
+        ig.cluster().active(true);
+
+        try (IgniteDataStreamer<Integer, byte[]> st = ig.dataStreamer(DEFAULT_CACHE_NAME)){
+            st.allowOverwrite(true);
+
+            byte[] payload = new byte[1024];
+
+            // Generate WAL segment files.
+            for (int i = 0; i < 100 * 1024; i++)
+                st.addData(i, payload);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testStandAloneIterator() throws Exception {
+        IgniteEx ig = grid();
+
+        IgniteWriteAheadLogManager wal = ig.context().cache().context().wal();
+
+        File walArchiveDir = U.field(wal, "walArchiveDir");
+
+        IgniteWalIteratorFactory iteratorFactory = new IgniteWalIteratorFactory();
+
+        doTest(wal, iteratorFactory.iterator(walArchiveDir));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testWALManagerIterator() throws Exception {
+        IgniteEx ig = grid();
+
+        IgniteWriteAheadLogManager wal = ig.context().cache().context().wal();
+
+        doTest(wal, wal.replay(null));
+    }
+
+    /**
+     *
+     * @param walMgr WAL manager.
+     * @param it WAL iterator.
+     * @throws IOException If IO exception.
+     * @throws IgniteCheckedException If WAL iterator failed.
+     */
+    private void doTest(IgniteWriteAheadLogManager walMgr, WALIterator it) throws IOException, IgniteCheckedException {
+        File walArchiveDir = U.field(walMgr, "walArchiveDir");
+
+        IgniteWalIteratorFactory iteratorFactory = new IgniteWalIteratorFactory();
+
+        List<FileDescriptor> descs = iteratorFactory.resolveWalFiles(
+            new IteratorParametersBuilder()
+                .filesOrDirs(walArchiveDir)
+        );
+
+        int maxIndex = descs.size() - 1;
+        int minIndex = 1;
+
+        int corruptedIdx = current().nextInt(minIndex, maxIndex);
+
+        log.info("Corrupted segment with idx:" + corruptedIdx);
+
+        FileWALPointer corruptedPtr = corruptedWAlSegmentFile(
+            descs.get(corruptedIdx),
+            new RandomAccessFileIOFactory(),
+            iteratorFactory
+        );
+
+        log.info("Should fail on ptr " + corruptedPtr);
+
+        FileWALPointer lastReadPtr = null;
+
+        boolean exception = false;
+
+        try (WALIterator it0 = it) {
+            while (it0.hasNextX()) {
+                IgniteBiTuple<WALPointer, WALRecord> tup = it0.nextX();
+
+                lastReadPtr = (FileWALPointer)tup.get1();
+            }
+        }
+        catch (IgniteCheckedException e) {
+            if (e.getMessage().contains("WAL tail reached in archive directory, WAL segment file is corrupted."))
+                exception = true;
+        }
+
+        Assert.assertNotNull(lastReadPtr);
+
+        if (!exception) {
+            fail("Last read ptr=" + lastReadPtr + ", corruptedPtr=" + corruptedPtr);
+        }
+    }
+
+    /**
+     *
+     * @param desc WAL segment descriptor.
+     * @param ioFactory IO factory.
+     * @param iteratorFactory Iterator factory.
+     * @return Corrupted position/
+     * @throws IOException If IO exception.
+     * @throws IgniteCheckedException If iterator failed.
+     */
+    private FileWALPointer corruptedWAlSegmentFile(
+        FileDescriptor desc,
+        FileIOFactory ioFactory,
+        IgniteWalIteratorFactory iteratorFactory
+    ) throws IOException, IgniteCheckedException {
+        LinkedList<FileWALPointer> pointers = new LinkedList<>();
+
+        try (WALIterator it = iteratorFactory.iterator(desc.file())) {
+            while (it.hasNext()) {
+                IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+                pointers.add((FileWALPointer)tup.get1());
+            }
+        }
+
+        int pointToCorrupt = current().nextInt(pointers.size());
+
+        FileWALPointer ptr = pointers.get(pointToCorrupt);
+
+        int offset = ptr.fileOffset();
+
+        // 20 pointer size, 8 idx, 4 offset, 4 length.
+        ByteBuffer buf = allocate(20);
+
+        Random r = new Random();
+
+        // Corrupt record pointer.
+        r.nextBytes(buf.array());
+
+        try (FileIO io = ioFactory.create(desc.file(), WRITE)) {
+            io.write(buf, offset + 1);
+
+            io.force(true);
+        }
+
+        return ptr;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index a777797d6f07b..08c6d875d099e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -46,6 +46,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsUnusedWalSegmentsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.filename.IgniteUidAsConsistentIdMigrationTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteNodeStoppedDuringDisableWALTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWALTailIsReachedDuringIterationOverArchiveTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundWithMmapBufferSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncSelfTest;
@@ -189,5 +190,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgniteNodeStoppedDuringDisableWALTest.class);
 
         suite.addTestSuite(IgniteRebalanceScheduleResendPartitionsTest.class);
+
+        suite.addTestSuite(IgniteWALTailIsReachedDuringIterationOverArchiveTest.class);
     }
 }

From 88c1035404a7a5b778e74a45bd443200602f15bd Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Tue, 14 Aug 2018 15:03:21 +0300
Subject: [PATCH 296/543] IGNITE-9260 Fixed failing test (omit check in
 standalone WAL iterator) - Fixes #4533.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 237a99e)
---
 .../wal/AbstractWalRecordsIterator.java       | 23 +++++++++++++++----
 .../reader/StandaloneWalRecordsIterator.java  | 20 ++++++++++++++++
 ...ReachedDuringIterationOverArchiveTest.java |  3 ++-
 3 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index ac68ea9a182aa..27f01022bd110 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -164,10 +164,10 @@ protected void advance() throws IgniteCheckedException {
             catch (WalSegmentTailReachedException e) {
                 AbstractReadFileHandle currWalSegment = this.currWalSegment;
 
-                if (!currWalSegment.workDir())
-                    throw new IgniteCheckedException(
-                        "WAL tail reached in archive directory, " +
-                            "WAL segment file is corrupted.", e);
+                IgniteCheckedException e0 = validateTailReachedException(e, currWalSegment);
+
+                if (e0 != null)
+                    throw e0;
 
                 log.warning(e.getMessage());
 
@@ -178,6 +178,21 @@ protected void advance() throws IgniteCheckedException {
         }
     }
 
+    /**
+     *
+     * @param tailReachedException Tail reached exception.
+     * @param currWalSegment Current WAL segment read handler.
+     * @return If need to throw exception after validation.
+     */
+    protected IgniteCheckedException validateTailReachedException(
+        WalSegmentTailReachedException tailReachedException,
+        AbstractReadFileHandle currWalSegment
+    ) {
+        return !currWalSegment.workDir() ? new IgniteCheckedException(
+            "WAL tail reached in archive directory, " +
+                "WAL segment file is corrupted.", tailReachedException) : null;
+    }
+
     /**
      * Closes and returns WAL segment (if any)
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index 9df446836198d..d92c0e4029cdb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -44,6 +44,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.ReadFileHandle;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
@@ -128,6 +129,25 @@ private void init(List<FileDescriptor> walFiles) {
             log.debug("Initialized WAL cursor [curWalSegmIdx=" + curWalSegmIdx + ']');
     }
 
+    /** {@inheritDoc} */
+    @Override protected IgniteCheckedException validateTailReachedException(
+        WalSegmentTailReachedException tailReachedException,
+        AbstractReadFileHandle currWalSegment
+    ) {
+        FileDescriptor lastWALSegmentDesc = walFileDescriptors.get(walFileDescriptors.size() - 1);
+
+        // Iterator can not be empty.
+        assert lastWALSegmentDesc != null;
+
+        return lastWALSegmentDesc.idx() != currWalSegment.idx() ?
+            new IgniteCheckedException(
+                "WAL tail reached not in the last available segment, " +
+                    "potentially corrupted segment, last available segment idx=" + lastWALSegmentDesc.idx() +
+                    ", path=" + lastWALSegmentDesc.file().getPath() +
+                    ", last read segment idx=" + currWalSegment.idx(), tailReachedException
+            ) : null;
+    }
+
     /** {@inheritDoc} */
     @Override protected AbstractReadFileHandle advanceSegment(
         @Nullable final AbstractReadFileHandle curWalSegment
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
index a7c408180f043..5fd5a1bee8525 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
@@ -185,7 +185,8 @@ private void doTest(IgniteWriteAheadLogManager walMgr, WALIterator it) throws IO
             }
         }
         catch (IgniteCheckedException e) {
-            if (e.getMessage().contains("WAL tail reached in archive directory, WAL segment file is corrupted."))
+            if (e.getMessage().contains("WAL tail reached in archive directory, WAL segment file is corrupted")
+                || e.getMessage().contains("WAL tail reached not in the last available segment"))
                 exception = true;
         }
 

From 80c46addc69d93fd082c37547c6368ee8616ad86 Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Wed, 15 Aug 2018 14:42:42 +0300
Subject: [PATCH 297/543] IGNITE-8761 WAL fsync at rollover should be
 asynchronous in LOG_ONLY and BACKGROUND modes - Fixes #4356.

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit 3e75f9101411f0a6bf72aee1e52b2fc3507792ab)
---
 .../apache/ignite/IgniteSystemProperties.java |  3 +
 .../wal/FileWriteAheadLogManager.java         | 62 ++++++++++++++++++-
 2 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index c390e196cce9f..a67f821e187fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -768,6 +768,9 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_WAL_SERIALIZER_VERSION = "IGNITE_WAL_SERIALIZER_VERSION";
 
+    /** Property for setup Ignite WAL segment sync timeout. */
+    public static final String IGNITE_WAL_SEGMENT_SYNC_TIMEOUT = "IGNITE_WAL_SEGMENT_SYNC_TIMEOUT";
+
     /**
      * If the property is set Ignite will use legacy node comparator (based on node order) inste
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index b952353cbc660..019497aa3030a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -125,6 +125,7 @@
 import static java.nio.file.StandardOpenOption.READ;
 import static java.nio.file.StandardOpenOption.WRITE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SEGMENT_SYNC_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
 import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
 import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED;
@@ -136,12 +137,15 @@
 import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader;
 import static org.apache.ignite.internal.util.IgniteUtils.findField;
 import static org.apache.ignite.internal.util.IgniteUtils.findNonPublicMethod;
+import static org.apache.ignite.internal.util.IgniteUtils.sleep;
 
 /**
  * File WAL manager.
  */
 @SuppressWarnings("IfMayBeConditional")
 public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter implements IgniteWriteAheadLogManager {
+    /** Dfault wal segment sync timeout. */
+    public static final long DFLT_WAL_SEGMENT_SYNC_TIMEOUT = 500L;
     /** {@link MappedByteBuffer#force0(java.io.FileDescriptor, long, long)}. */
     private static final Method force0 = findNonPublicMethod(
         MappedByteBuffer.class, "force0",
@@ -352,6 +356,9 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
      */
     @Nullable private volatile IgniteInClosure<FileIO> createWalFileListener;
 
+    /** Wal segment sync worker. */
+    private WalSegmentSyncer walSegmentSyncWorker;
+
     /**
      * @param ctx Kernal context.
      */
@@ -454,7 +461,10 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
 
             walDisableContext = cctx.walState().walDisableContext();
 
-            if (mode != WALMode.NONE) {
+            if (mode != WALMode.NONE && mode != WALMode.FSYNC) {
+                walSegmentSyncWorker = new WalSegmentSyncer(igCfg.getIgniteInstanceName(),
+                    cctx.kernalContext().log(WalSegmentSyncer.class));
+
                 if (log.isInfoEnabled())
                     log.info("Started write-ahead log manager [mode=" + mode + ']');
             }
@@ -563,6 +573,9 @@ private void checkWalConfiguration() throws IgniteCheckedException {
 
             if (decompressor != null)
                 decompressor.shutdown();
+
+            if (walSegmentSyncWorker != null)
+                walSegmentSyncWorker.shutdown();
         }
         catch (Exception e) {
             U.error(log, "Failed to gracefully close WAL segment: " + this.currHnd.fileIO, e);
@@ -584,6 +597,9 @@ private void checkWalConfiguration() throws IgniteCheckedException {
                 new IgniteThread(archiver).start();
             }
 
+            if (walSegmentSyncWorker != null)
+                new IgniteThread(walSegmentSyncWorker).start();
+
             if (compressor != null)
                 compressor.start();
         }
@@ -3321,7 +3337,7 @@ else if (pos == FILE_FORCE)
                             writeBuffer(seg.position(), seg.buffer());
                         }
                         catch (Throwable e) {
-                            log.error("Exception in WAL writer thread: ", e);
+                            log.error("Exception in WAL writer thread:", e);
 
                             err = e;
                         }
@@ -3505,6 +3521,48 @@ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, Igni
         }
     }
 
+    /**
+     * Syncs WAL segment file.
+     */
+    private class WalSegmentSyncer extends GridWorker {
+        /** Sync timeout. */
+        long syncTimeout;
+
+        /**
+         * @param igniteInstanceName Ignite instance name.
+         * @param log Logger.
+         */
+        public WalSegmentSyncer(String igniteInstanceName, IgniteLogger log) {
+            super(igniteInstanceName, "wal-segment-syncer", log);
+
+            syncTimeout = Math.max(IgniteSystemProperties.getLong(IGNITE_WAL_SEGMENT_SYNC_TIMEOUT,
+                DFLT_WAL_SEGMENT_SYNC_TIMEOUT), 100L);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+            while (!isCancelled()) {
+                sleep(syncTimeout);
+
+                try {
+                    flush(null, true);
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Exception when flushing WAL.", e);
+                }
+            }
+        }
+
+        /** Shutted down the worker. */
+        private void shutdown() {
+            synchronized (this) {
+                U.cancel(this);
+            }
+
+            U.join(this, log);
+        }
+    }
+
     /**
      * Scans provided folder for a WAL segment files
      * @param walFilesDir directory to scan

From fa458b9d715e4f7e01ac2c0363ade40a0106cf29 Mon Sep 17 00:00:00 2001
From: ascherbakoff <alexey.scherbakoff@gmail.com>
Date: Sat, 11 Aug 2018 14:13:26 +0300
Subject: [PATCH 298/543] IGNITE-9246 Optimistic transactions can wait for
 topology future on remap for a long time even if timeout is set.

(cherry picked from commit 5d151063d554f23858373d642e7875b6a4f206f9)
---
 ...OptimisticSerializableTxPrepareFuture.java |  18 +--
 .../GridNearOptimisticTxPrepareFuture.java    |  22 +--
 ...dNearOptimisticTxPrepareFutureAdapter.java | 135 +++++++++++++++---
 .../transactions/TxRollbackAsyncTest.java     |  16 ---
 .../transactions/TxRollbackOnTimeoutTest.java | 122 +++++++++++++++-
 .../TxRollbackOnTopologyChangeTest.java       |  16 ---
 .../junits/common/GridCommonAbstractTest.java |  23 +++
 .../ignite/util/GridCommandHandlerTest.java   |  23 ---
 8 files changed, 270 insertions(+), 105 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index 974de6b9ef30c..38055e71eb625 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -42,11 +42,10 @@
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
-import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
-import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.GridAbsClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.C1;
@@ -914,18 +913,9 @@ void onResult(final GridNearTxPrepareResponse res, boolean updateMapping) {
                                                 parent.remapFut = null;
                                             }
 
-                                            affFut.listen(new CI1<IgniteInternalFuture<?>>() {
-                                                @Override public void apply(IgniteInternalFuture<?> affFut) {
-                                                    try {
-                                                        affFut.get();
-
-                                                        remap(res);
-                                                    }
-                                                    catch (IgniteCheckedException e) {
-                                                        ERR_UPD.compareAndSet(parent, null, e);
-
-                                                        onDone(e);
-                                                    }
+                                            parent.applyWhenReady(affFut, new GridAbsClosureX() {
+                                                @Override public void applyx() throws IgniteCheckedException {
+                                                    remap(res);
                                                 }
                                             });
                                         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 247af8496ed10..820037507eb5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -52,6 +52,7 @@
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.GridAbsClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.C1;
@@ -989,22 +990,11 @@ void onResult(final GridNearTxPrepareResponse res) {
                         IgniteInternalFuture<?> affFut =
                             parent.cctx.exchange().affinityReadyFuture(res.clientRemapVersion());
 
-                        if (affFut != null && !affFut.isDone()) {
-                            affFut.listen(new CI1<IgniteInternalFuture<?>>() {
-                                @Override public void apply(IgniteInternalFuture<?> fut) {
-                                    try {
-                                        fut.get();
-
-                                        remap();
-                                    }
-                                    catch (IgniteCheckedException e) {
-                                        onDone(e);
-                                    }
-                                }
-                            });
-                        }
-                        else
-                            remap();
+                        parent.applyWhenReady(affFut, new GridAbsClosureX() {
+                            @Override public void applyx() throws IgniteCheckedException {
+                                remap();
+                            }
+                        });
                     }
                     else {
                         parent.onPrepareResponse(m, res, m.hasNearCacheEntries());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
index 780674ecb5377..ecc02c23e3a7b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
@@ -18,18 +18,20 @@
 package org.apache.ignite.internal.processors.cache.distributed.near;
 
 import java.util.Collection;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.lang.GridPlainRunnable;
+import org.apache.ignite.internal.util.lang.GridAbsClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -140,23 +142,14 @@ protected final void prepareOnTopology(final boolean remap, @Nullable final Runn
                 c.run();
         }
         else {
-            topFut.listen(new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
-                @Override public void apply(final IgniteInternalFuture<AffinityTopologyVersion> fut) {
-                    cctx.kernalContext().closure().runLocalSafe(new GridPlainRunnable() {
-                        @Override public void run() {
-                            try {
-                                fut.get();
-
-                                prepareOnTopology(remap, c);
-                            }
-                            catch (IgniteCheckedException e) {
-                                onDone(e);
-                            }
-                            finally {
-                                cctx.txContextReset();
-                            }
-                        }
-                    });
+            applyWhenReady(topFut, new GridAbsClosureX() {
+                @Override public void applyx() throws IgniteCheckedException {
+                    try {
+                        prepareOnTopology(remap, c);
+                    }
+                    finally {
+                        cctx.txContextReset();
+                    }
                 }
             });
         }
@@ -168,6 +161,70 @@ protected final void prepareOnTopology(final boolean remap, @Nullable final Runn
      */
     protected abstract void prepare0(boolean remap, boolean topLocked);
 
+    /**
+     * @param fut Future.
+     * @param clo Closure.
+     */
+    protected void applyWhenReady(final IgniteInternalFuture<?> fut, GridAbsClosureX clo) {
+        long remaining = tx.remainingTime();
+
+        if (remaining == -1) {
+            IgniteCheckedException e = tx.timeoutException();
+
+            ERR_UPD.compareAndSet(this, null, e);
+
+            onDone(e);
+
+            return;
+        }
+
+        if (fut == null || fut.isDone()) {
+            try {
+                clo.applyx();
+            }
+            catch (IgniteCheckedException e) {
+                ERR_UPD.compareAndSet(this, null, e);
+
+                onDone(e);
+            }
+        }
+        else {
+            RemapTimeoutObject timeoutObj = null;
+
+            AtomicBoolean state = new AtomicBoolean();
+
+            if (remaining > 0) {
+                timeoutObj = new RemapTimeoutObject(remaining, fut, state);
+
+                cctx.time().addTimeoutObject(timeoutObj);
+            }
+
+            final RemapTimeoutObject finalTimeoutObj = timeoutObj;
+
+            fut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
+                @Override public void apply(IgniteInternalFuture<?> fut) {
+                    if (!state.compareAndSet(false, true))
+                        return;
+
+                    try {
+                        fut.get();
+
+                        clo.applyx();
+                    }
+                    catch (IgniteCheckedException e) {
+                        ERR_UPD.compareAndSet(GridNearOptimisticTxPrepareFutureAdapter.this, null, e);
+
+                        onDone(e);
+                    }
+                    finally {
+                        if (finalTimeoutObj != null)
+                            cctx.time().removeTimeoutObject(finalTimeoutObj);
+                    }
+                }
+            });
+        }
+    }
+
     /**
      * Keys lock future.
      */
@@ -231,4 +288,44 @@ private boolean checkLocks() {
             return S.toString(KeyLockFuture.class, this, super.toString());
         }
     }
+
+    /**
+     *
+     */
+    private class RemapTimeoutObject extends GridTimeoutObjectAdapter {
+        /** */
+        private final IgniteInternalFuture<?> fut;
+
+        /** */
+        private final AtomicBoolean state;
+
+        /**
+         * @param timeout Timeout.
+         * @param fut Future.
+         * @param state State.
+         */
+        RemapTimeoutObject(long timeout, IgniteInternalFuture<?> fut, AtomicBoolean state) {
+            super(timeout);
+
+            this.fut = fut;
+
+            this.state = state;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onTimeout() {
+            if (!fut.isDone() && state.compareAndSet(false, true)) {
+                IgniteCheckedException e = tx.timeoutException();
+
+                ERR_UPD.compareAndSet(GridNearOptimisticTxPrepareFutureAdapter.this, null, e);
+
+                onDone(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(RemapTimeoutObject.class, this);
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
index 3a8be4941c7dc..4eda30c4ae885 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
@@ -893,22 +893,6 @@ private IgniteInternalFuture<?> lockInTx(final Ignite node, final CountDownLatch
         }, 1, "tx-lock-thread");
     }
 
-    /**
-     * Checks if all tx futures are finished.
-     */
-    private void checkFutures() {
-        for (Ignite ignite : G.allGrids()) {
-            IgniteEx ig = (IgniteEx)ignite;
-
-            final Collection<GridCacheFuture<?>> futs = ig.context().cache().context().mvcc().activeFutures();
-
-            for (GridCacheFuture<?> fut : futs)
-                log.info("Waiting for future: " + fut);
-
-            assertTrue("Expecting no active futures: node=" + ig.localNode().id(), futs.isEmpty());
-        }
-    }
-
     /**
      * @param tx Tx to rollback.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
index d1cbfa72613bd..5ad65f50f4a3d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
@@ -19,6 +19,7 @@
 
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.CountDownLatch;
@@ -31,20 +32,26 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareResponse;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -55,14 +62,17 @@
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionDeadlockException;
 import org.apache.ignite.transactions.TransactionIsolation;
-import org.apache.ignite.transactions.TransactionOptimisticException;
 import org.apache.ignite.transactions.TransactionTimeoutException;
 
 import static java.lang.Thread.sleep;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
 import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
 
 /**
  * Tests an ability to eagerly rollback timed out transactions.
@@ -87,6 +97,8 @@ public class TxRollbackOnTimeoutTest extends GridCommonAbstractTest {
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
+        cfg.setConsistentId(igniteInstanceName);
+
         ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
 
         cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
@@ -571,6 +583,114 @@ public void testEnlistManyWrite() throws Exception {
         testEnlistMany(true);
     }
 
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxRemapReadCommitted() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(READ_COMMITTED);
+    }
+
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxRemapRepeatableRead() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(REPEATABLE_READ);
+    }
+
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxRemapSerializable() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(SERIALIZABLE);
+    }
+
+    /**
+     *
+     */
+    private void doTestRollbackOnTimeoutTxRemap(TransactionIsolation isolation) throws Exception {
+        Ignite client = startClient();
+
+        Ignite crd = grid(0);
+
+        assertTrue(crd.cluster().localNode().order() == 1);
+
+        List<Integer> keys = movingKeysAfterJoin(grid(1), CACHE_NAME, 1);
+
+        // Delay exchange finish on client node.
+        TestRecordingCommunicationSpi.spi(crd).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+            @Override public boolean apply(ClusterNode node, Message msg) {
+                return node.equals(client.cluster().localNode()) && msg instanceof GridDhtPartitionsFullMessage;
+            }
+        });
+
+        // Delay prepare until exchange is finished.
+        TestRecordingCommunicationSpi.spi(client).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+            @Override public boolean apply(ClusterNode node, Message msg) {
+                return msg instanceof GridNearTxPrepareRequest;
+            }
+        });
+
+        IgniteInternalFuture fut0 = runAsync(new Runnable() {
+            @Override public void run() {
+                try (Transaction tx = client.transactions().txStart(OPTIMISTIC, isolation, 5000, 1)) {
+                    client.cache(CACHE_NAME).put(keys.get(0), 0);
+
+                    tx.commit();
+
+                    fail();
+                }
+                catch (Exception e) {
+                    assertTrue(X.hasCause(e, TransactionTimeoutException.class));
+                }
+            }
+        });
+
+        IgniteInternalFuture fut = runAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    startGrid(GRID_CNT);
+
+                    TestRecordingCommunicationSpi.spi(crd).waitForBlocked();
+
+                    TestRecordingCommunicationSpi.spi(client).waitForBlocked();
+
+                    // Delivered prepare message will trigger remap.
+                    TestRecordingCommunicationSpi.spi(client).stopBlock();
+                }
+                catch (Exception e) {
+                    fail(e.getMessage());
+                }
+            }
+        });
+
+        // Allow timeout on future wait.
+        try {
+            fut.get(10_000);
+        }
+        catch (IgniteFutureTimeoutCheckedException e) {
+            // Ignored.
+        }
+
+        try {
+            fut0.get(10_000);
+        }
+        catch (IgniteFutureTimeoutCheckedException e) {
+            // Ignored.
+        }
+
+        TestRecordingCommunicationSpi.spi(crd).stopBlock();
+
+        // If using awaitPartitionMapExchange for waiting it some times fail while waiting for owners.
+        IgniteInternalFuture<?> topFut = ((IgniteEx)client).context().cache().context().exchange().
+            affinityReadyFuture(new AffinityTopologyVersion(GRID_CNT + 2, 1));
+
+        assertNotNull(topFut);
+
+        topFut.get(10_000);
+
+        checkFutures();
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java
index 2542cdb5365ef..13c5e41c27cb3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTopologyChangeTest.java
@@ -209,20 +209,4 @@ public void testRollbackOnTopologyChange() throws Exception {
 
         checkFutures();
     }
-
-    /**
-     * Checks if all tx futures are finished.
-     */
-    private void checkFutures() {
-        for (Ignite ignite : G.allGrids()) {
-            IgniteEx ig = (IgniteEx)ignite;
-
-            final Collection<GridCacheFuture<?>> futs = ig.context().cache().context().mvcc().activeFutures();
-
-            for (GridCacheFuture<?> fut : futs)
-                log.info("Waiting for future: " + fut);
-
-            assertTrue("Expecting no active futures: node=" + ig.localNode().id(), futs.isEmpty());
-        }
-    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index f8f1c9c2f0f04..0414a4173d48a 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -1994,4 +1994,27 @@ protected Map<PartitionKey, List<PartitionHashRecord>> idleVerify(Ignite ig, Str
 
         return res.getConflicts();
     }
+
+    /**
+     * Checks if all txs and mvcc futures are finished.
+     */
+    protected void checkFutures() {
+        for (Ignite ignite : G.allGrids()) {
+            IgniteEx ig = (IgniteEx)ignite;
+
+            final Collection<GridCacheFuture<?>> futs = ig.context().cache().context().mvcc().activeFutures();
+
+            for (GridCacheFuture<?> fut : futs)
+                log.info("Waiting for future: " + fut);
+
+            assertTrue("Expecting no active futures: node=" + ig.localNode().id(), futs.isEmpty());
+
+            Collection<IgniteInternalTx> txs = ig.context().cache().context().tm().activeTransactions();
+
+            for (IgniteInternalTx tx : txs)
+                log.info("Waiting for tx: " + tx);
+
+            assertTrue("Expecting no active transactions: node=" + ig.localNode().id(), txs.isEmpty());
+        }
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 3dcb8beccbfa4..2d5a4ddc1396e 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -1424,29 +1424,6 @@ private IgniteInternalFuture<?> startTransactions(CountDownLatch lockLatch,
         }, 4, "tx-thread");
     }
 
-    /**
-     * Checks if all tx futures are finished.
-     */
-    private void checkFutures() {
-        for (Ignite ignite : G.allGrids()) {
-            IgniteEx ig = (IgniteEx)ignite;
-
-            final Collection<GridCacheFuture<?>> futs = ig.context().cache().context().mvcc().activeFutures();
-
-            for (GridCacheFuture<?> fut : futs)
-                log.info("Waiting for future: " + fut);
-
-            assertTrue("Expecting no active futures: node=" + ig.localNode().id(), futs.isEmpty());
-
-            Collection<IgniteInternalTx> txs = ig.context().cache().context().tm().activeTransactions();
-
-            for (IgniteInternalTx tx : txs)
-                log.info("Waiting for tx: " + tx);
-
-            assertTrue("Expecting no active transactions: node=" + ig.localNode().id(), txs.isEmpty());
-        }
-    }
-
     /** */
     private static class IncrementClosure implements EntryProcessor<Long, Long, Void> {
         /** {@inheritDoc} */

From eadc99c7052a10168b4072c457be070d48e25dc8 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Mon, 13 Aug 2018 13:29:29 +0300
Subject: [PATCH 299/543] IGNITE-9147 Race between tx async rollback and lock
 mapping on near node can produce hanging primary tx

(cherry picked from commit a3f9076e475f603a6c3a457b64f721fd0c24a396)
---
 .../internal/TransactionsMXBeanImpl.java      |  20 +-
 .../internal/commandline/CommandHandler.java  |  24 +--
 .../colocated/GridDhtColocatedLockFuture.java | 171 +++++++++++++++---
 .../GridNearPessimisticTxPrepareFuture.java   |   1 -
 .../near/GridNearTxFinishFuture.java          |  12 +-
 .../ignite/internal/visor/tx/VisorTxInfo.java |  33 ++++
 .../ignite/internal/visor/tx/VisorTxTask.java |  18 +-
 .../transactions/TxRollbackOnTimeoutTest.java | 161 +++++++++++++++--
 8 files changed, 350 insertions(+), 90 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
index 210a32008d313..16738de12343d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/TransactionsMXBeanImpl.java
@@ -22,13 +22,10 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
-import java.util.UUID;
 import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.apache.ignite.internal.visor.tx.VisorTxInfo;
 import org.apache.ignite.internal.visor.tx.VisorTxOperation;
@@ -37,7 +34,6 @@
 import org.apache.ignite.internal.visor.tx.VisorTxTask;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
-import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.mxbean.TransactionsMXBean;
 
 /**
@@ -103,21 +99,7 @@ else if ("SIZE".equals(order))
                     w.println(key.toString());
 
                     for (VisorTxInfo info : entry.getValue().getInfos())
-                        w.println("    Tx: [xid=" + info.getXid() +
-                            ", label=" + info.getLabel() +
-                            ", state=" + info.getState() +
-                            ", startTime=" + info.getFormattedStartTime() +
-                            ", duration=" + info.getDuration() / 1000 +
-                            ", isolation=" + info.getIsolation() +
-                            ", concurrency=" + info.getConcurrency() +
-                            ", timeout=" + info.getTimeout() +
-                            ", size=" + info.getSize() +
-                            ", dhtNodes=" + F.transform(info.getPrimaryNodes(), new IgniteClosure<UUID, String>() {
-                            @Override public String apply(UUID id) {
-                                return U.id8(id);
-                            }
-                        }) +
-                            ']');
+                        w.println(info.toUserString());
                 }
 
                 w.flush();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 7b5ce44332a7a..092efffe60862 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -1060,29 +1060,7 @@ else if (arg.getOperation() == VisorTxOperation.KILL)
                     "]");
 
                 for (VisorTxInfo info : entry.getValue().getInfos())
-                    log("    Tx: [xid=" + info.getXid() +
-                        ", label=" + info.getLabel() +
-                        ", state=" + info.getState() +
-                        ", startTime=" + info.getFormattedStartTime() +
-                        ", duration=" + info.getDuration() / 1000 +
-                        ", isolation=" + info.getIsolation() +
-                        ", concurrency=" + info.getConcurrency() +
-                        ", timeout=" + info.getTimeout() +
-                        ", size=" + info.getSize() +
-                        ", dhtNodes=" + (info.getPrimaryNodes() == null ? "N/A" :
-                        F.transform(info.getPrimaryNodes(), new IgniteClosure<UUID, String>() {
-                            @Override public String apply(UUID id) {
-                                return U.id8(id);
-                            }
-                        })) +
-                        ", nearXid=" + info.getNearXid() +
-                        ", parentNodeIds=" + (info.getMasterNodeIds() == null ? "N/A" :
-                        F.transform(info.getMasterNodeIds(), new IgniteClosure<UUID, String>() {
-                            @Override public String apply(UUID id) {
-                                return U.id8(id);
-                            }
-                        })) +
-                        ']');
+                    log(info.toUserString());
             }
         }
         catch (Throwable e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index da0858f61cc5d..55a82cbad4bab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -28,6 +28,7 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
@@ -65,6 +66,7 @@
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.GridAbsClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.C1;
@@ -180,6 +182,9 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
     /** */
     private int miniId;
 
+    /** Used for synchronization between lock cancellation and mapping phase. */
+    private boolean mappingsReady;
+
     /**
      * @param cctx Registry.
      * @param keys Keys to lock.
@@ -547,9 +552,20 @@ private synchronized void onError(Throwable t) {
      * Cancellation has special meaning for lock futures. It's called then lock must be released on rollback.
      */
     @Override public boolean cancel() {
-        if (inTx())
+        if (inTx()) {
             onError(tx.rollbackException());
 
+            synchronized (this) {
+                while (!mappingsReady)
+                    try {
+                        wait();
+                    }
+                    catch (InterruptedException e) {
+                        // Ignore interrupts.
+                    }
+            }
+        }
+
         return onComplete(false, true);
     }
 
@@ -610,6 +626,14 @@ private boolean onComplete(boolean success, boolean distribute) {
             if (timeoutObj != null)
                 cctx.time().removeTimeoutObject(timeoutObj);
 
+            synchronized (this) {
+                if (!mappingsReady) {
+                    mappingsReady = true;
+
+                    notifyAll();
+                }
+            }
+
             return true;
         }
 
@@ -809,16 +833,11 @@ private void mapOnTopology(final boolean remap, @Nullable final Runnable c) {
                 markInitialized();
             }
             else {
-                fut.listen(new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
-                    @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> fut) {
+                applyWhenReady(fut, new GridAbsClosureX() {
+                    @Override public void applyx() throws IgniteCheckedException {
                         try {
-                            fut.get();
-
                             mapOnTopology(remap, c);
                         }
-                        catch (IgniteCheckedException e) {
-                            onDone(e);
-                        }
                         finally {
                             cctx.shared().txContextReset();
                         }
@@ -850,6 +869,13 @@ private void map(Collection<KeyCacheObject> keys, boolean remap, boolean topLock
         catch (IgniteCheckedException ex) {
             onDone(false, ex);
         }
+        finally {
+            synchronized (this) {
+                mappingsReady = true;
+
+                notifyAll();
+            }
+        }
     }
 
     /**
@@ -1114,10 +1140,19 @@ private void proceedMapping0()
             map = mappings.poll();
         }
 
-        // If there are no more mappings to process, complete the future.
+        // If there are no more mappings to process or prepare has timed out, complete the future.
         if (map == null)
             return;
 
+        // Fail fast if the transaction is timed out.
+        if (tx != null && tx.remainingTime() == -1) {
+            GridDhtColocatedLockFuture.this.onDone(false, tx.timeoutException());
+
+            clear();
+
+            return;
+        }
+
         final GridNearLockRequest req = map.request();
         final Collection<KeyCacheObject> mappedKeys = map.distributedKeys();
         final ClusterNode node = map.node();
@@ -1400,6 +1435,62 @@ private ClusterTopologyCheckedException newTopologyException(@Nullable Throwable
         return topEx;
     }
 
+    /**
+     * @param fut Future.
+     * @param clo Closure.
+     */
+    protected void applyWhenReady(final IgniteInternalFuture<?> fut, GridAbsClosureX clo) {
+        long remaining = tx.remainingTime();
+
+        if (remaining == -1) {
+            onDone(tx.timeoutException());
+
+            return;
+        }
+
+        if (fut == null || fut.isDone()) {
+            try {
+                clo.applyx();
+            }
+            catch (IgniteCheckedException e) {
+                onDone(e);
+            }
+        }
+        else {
+            RemapTimeoutObject timeoutObj = null;
+
+            AtomicBoolean state = new AtomicBoolean();
+
+            if (remaining > 0) {
+                timeoutObj = new RemapTimeoutObject(remaining, fut, state);
+
+                cctx.time().addTimeoutObject(timeoutObj);
+            }
+
+            final RemapTimeoutObject finalTimeoutObj = timeoutObj;
+
+            fut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
+                @Override public void apply(IgniteInternalFuture<?> fut) {
+                    if (!state.compareAndSet(false, true))
+                        return;
+
+                    try {
+                        fut.get();
+
+                        clo.applyx();
+                    }
+                    catch (IgniteCheckedException e) {
+                        onDone(e);
+                    }
+                    finally {
+                        if (finalTimeoutObj != null)
+                            cctx.time().removeTimeoutObject(finalTimeoutObj);
+                    }
+                }
+            });
+        }
+    }
+
     /**
      * Lock request timeout object.
      */
@@ -1593,25 +1684,16 @@ void onResult(GridNearLockResponse res) {
                 IgniteInternalFuture<?> affFut =
                     cctx.shared().exchange().affinityReadyFuture(res.clientRemapVersion());
 
-                if (affFut != null && !affFut.isDone()) {
-                    affFut.listen(new CI1<IgniteInternalFuture<?>>() {
-                        @Override public void apply(IgniteInternalFuture<?> fut) {
-                            try {
-                                fut.get();
-
-                                remap();
-                            }
-                            catch (IgniteCheckedException e) {
-                                onDone(e);
-                            }
-                            finally {
-                                cctx.shared().txContextReset();
-                            }
+                applyWhenReady(affFut, new GridAbsClosureX() {
+                    @Override public void applyx() throws IgniteCheckedException {
+                        try {
+                            remap();
                         }
-                    });
-                }
-                else
-                    remap();
+                        finally {
+                            cctx.shared().txContextReset();
+                        }
+                    }
+                });
             }
             else {
                 int i = 0;
@@ -1709,4 +1791,39 @@ private void remap() {
             return S.toString(MiniFuture.class, this, "node", node.id(), "super", super.toString());
         }
     }
+
+    /**
+     *
+     */
+    private class RemapTimeoutObject extends GridTimeoutObjectAdapter {
+        /** */
+        private final IgniteInternalFuture<?> fut;
+
+        /** */
+        private final AtomicBoolean state;
+
+        /**
+         * @param timeout Timeout.
+         * @param fut Future.
+         * @param state State.
+         */
+        RemapTimeoutObject(long timeout, IgniteInternalFuture<?> fut, AtomicBoolean state) {
+            super(timeout);
+
+            this.fut = fut;
+
+            this.state = state;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onTimeout() {
+            if (!fut.isDone() && state.compareAndSet(false, true))
+                onDone(tx.timeoutException());
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(RemapTimeoutObject.class, this);
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index da9bdac22739a..7d1ab8598466a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -38,7 +38,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxMapping;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
-import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.C1;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index ede8a4ec39484..936b51597511b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -412,19 +412,19 @@ public void finish(final boolean commit, final boolean clearThreadMap, final boo
         }
 
         if (!commit && !clearThreadMap)
-            tryRollbackAsync(onTimeout); // Asynchronous rollback.
+            rollbackAsyncSafe(onTimeout); // Asynchronous rollback.
         else
             doFinish(commit, clearThreadMap);
     }
 
     /**
-     * Does async rollback when it's safe.
-     * If current future is not lock future (enlist future) waits until completion and tries again.
-     * Else terminates or waits for lock future depending on rollback mode.
+     * Roll back tx when it's safe.
+     * If current future is not lock future (enlist future) wait until completion and tries again.
+     * Else cancel lock future (onTimeout=false) or wait for completion due to deadlock detection (onTimeout=true).
      *
      * @param onTimeout If {@code true} called from timeout handler.
      */
-    private void tryRollbackAsync(boolean onTimeout) {
+    private void rollbackAsyncSafe(boolean onTimeout) {
         IgniteInternalFuture<?> curFut = tx.tryRollbackAsync();
 
         if (curFut == null) { // Safe to rollback.
@@ -447,7 +447,7 @@ private void tryRollbackAsync(boolean onTimeout) {
                 try {
                     fut.get();
 
-                    tryRollbackAsync(onTimeout);
+                    rollbackAsyncSafe(onTimeout);
                 }
                 catch (IgniteCheckedException e) {
                     doFinish(false, false);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
index 03de5b04a9c1a..6971d7c14060e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
@@ -26,9 +26,11 @@
 import java.util.Collection;
 import java.util.TimeZone;
 import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
@@ -225,6 +227,37 @@ public int getSize() {
 
     }
 
+    /**
+     * Get tx info as user string.
+     *
+     * @return User string.
+     */
+    public String toUserString() {
+        return "    Tx: [xid=" + getXid() +
+            ", label=" + getLabel() +
+            ", state=" + getState() +
+            ", startTime=" + getFormattedStartTime() +
+            ", duration=" + getDuration() / 1000 +
+            ", isolation=" + getIsolation() +
+            ", concurrency=" + getConcurrency() +
+            ", timeout=" + getTimeout() +
+            ", size=" + getSize() +
+            ", dhtNodes=" + (getPrimaryNodes() == null ? "N/A" :
+            F.transform(getPrimaryNodes(), new IgniteClosure<UUID, String>() {
+                @Override public String apply(UUID id) {
+                    return U.id8(id);
+                }
+            })) +
+            ", nearXid=" + getNearXid() +
+            ", parentNodeIds=" + (getMasterNodeIds() == null ? "N/A" :
+            F.transform(getMasterNodeIds(), new IgniteClosure<UUID, String>() {
+                @Override public String apply(UUID id) {
+                    return U.id8(id);
+                }
+            })) +
+            ']';
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(VisorTxInfo.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
index 9919b7d654458..3f35a26711994 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
@@ -183,7 +183,7 @@ private static class VisorTxJob extends VisorJob<VisorTxTaskArg, VisorTxTaskResu
         private static final TxKillClosure NEAR_KILL_CLOSURE = new NearKillClosure();
 
         /** */
-        private static final TxKillClosure LOCAL_KILL_CLOSURE = NEAR_KILL_CLOSURE;
+        private static final TxKillClosure LOCAL_KILL_CLOSURE = new LocalKillClosure();
 
         /** */
         private static final TxKillClosure REMOTE_KILL_CLOSURE = new RemoteKillClosure();
@@ -394,7 +394,7 @@ private interface TxKillClosure extends
         IgniteBiClosure<IgniteInternalTx, IgniteTxManager, IgniteInternalFuture<IgniteInternalTx>> {
     }
 
-    /** Kills near or local tx. */
+    /** Kills near tx. */
     private static class NearKillClosure implements TxKillClosure {
         /** */
         private static final long serialVersionUID = 0L;
@@ -402,7 +402,19 @@ private static class NearKillClosure implements TxKillClosure {
         /** {@inheritDoc} */
         @Override public IgniteInternalFuture<IgniteInternalTx> apply(IgniteInternalTx tx, IgniteTxManager tm) {
             return tx.isRollbackOnly() || tx.state() == COMMITTING || tx.state() == COMMITTED ?
-                new GridFinishedFuture<>() : tx.rollbackAsync();
+                new GridFinishedFuture<>() : ((GridNearTxLocal)tx).rollbackNearTxLocalAsync(false, false);
+        }
+    }
+
+    /** Kills local tx. */
+    private static class LocalKillClosure implements TxKillClosure {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** {@inheritDoc} */
+        @Override public IgniteInternalFuture<IgniteInternalTx> apply(IgniteInternalTx tx, IgniteTxManager tm) {
+            return tx.isRollbackOnly() || tx.state() == COMMITTING || tx.state() == COMMITTED ?
+                new GridFinishedFuture<>() : ((GridDhtTxLocal)tx).rollbackDhtLocalAsync();
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
index 5ad65f50f4a3d..47c00b5f0b929 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
@@ -22,6 +22,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -32,11 +33,12 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -45,14 +47,23 @@
 import org.apache.ignite.internal.processors.cache.GridCacheFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareResponse;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.tx.VisorTxInfo;
+import org.apache.ignite.internal.visor.tx.VisorTxOperation;
+import org.apache.ignite.internal.visor.tx.VisorTxTask;
+import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
+import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
 import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -583,31 +594,158 @@ public void testEnlistManyWrite() throws Exception {
         testEnlistMany(true);
     }
 
+    public void testRollbackOnTopologyLockPessimistic() throws Exception {
+        final Ignite client = startClient();
+
+        Ignite crd = grid(0);
+
+        List<Integer> keys = primaryKeys(grid(1).cache(CACHE_NAME), 1);
+
+        assertTrue(crd.cluster().localNode().order() == 1);
+
+        CountDownLatch txLatch = new CountDownLatch(1);
+        CountDownLatch stopLatch = new CountDownLatch(1);
+        CountDownLatch commitLatch = new CountDownLatch(1);
+
+        // Start tx holding topology.
+        IgniteInternalFuture txFut = runAsync(new Runnable() {
+            @Override public void run() {
+                List<Integer> keys = primaryKeys(grid(0).cache(CACHE_NAME), 1);
+
+                try (Transaction tx = client.transactions().txStart()) {
+                    client.cache(CACHE_NAME).put(keys.get(0), 0);
+
+                    txLatch.countDown();
+
+                    U.awaitQuiet(commitLatch);
+
+                    tx.commit();
+
+                    fail();
+                }
+                catch (Exception e) {
+                    // Expected.
+                }
+            }
+        });
+
+        U.awaitQuiet(txLatch);
+
+        crd.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                runAsync(new Runnable() {
+                    @Override public void run() {
+                        try(Transaction tx = crd.transactions().withLabel("testLbl").txStart()) {
+                            stopLatch.countDown();
+
+                            crd.cache(CACHE_NAME).put(keys.get(0), 0);
+
+                            tx.commit();
+
+                            fail();
+                        }
+                        catch (Exception e) {
+                            // Expected.
+                        }
+                    }
+                });
+
+                return false;
+            }
+        }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
+
+        IgniteInternalFuture restartFut = runAsync(new Runnable() {
+            @Override public void run() {
+                stopGrid(2);
+
+                try {
+                    startGrid(2);
+                }
+                catch (Exception e) {
+                    fail();
+                }
+            }
+        });
+
+        U.awaitQuiet(stopLatch);
+
+        doSleep(500);
+
+        VisorTxTaskArg arg =
+            new VisorTxTaskArg(VisorTxOperation.KILL, null, null, null, null, null, null, null, null, null);
+
+        Map<ClusterNode, VisorTxTaskResult> res = client.compute(client.cluster().forPredicate(F.alwaysTrue())).
+            execute(new VisorTxTask(), new VisorTaskArgument<>(client.cluster().localNode().id(), arg, false));
+
+        int expCnt = 0;
+
+        for (Map.Entry<ClusterNode, VisorTxTaskResult> entry : res.entrySet()) {
+            if (entry.getValue().getInfos().isEmpty())
+                continue;
+
+            for (VisorTxInfo info : entry.getValue().getInfos()) {
+                log.info(info.toUserString());
+
+                expCnt++;
+            }
+        }
+
+        assertEquals("Expecting 2 transactions", 2, expCnt);
+
+        commitLatch.countDown();
+
+        txFut.get();
+        restartFut.get();
+
+        checkFutures();
+    }
+
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxRemapOptimisticReadCommitted() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, READ_COMMITTED);
+    }
+
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxRemapOptimisticRepeatableRead() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, REPEATABLE_READ);
+    }
+
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxRemapOptimisticSerializable() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, SERIALIZABLE);
+    }
+
     /**
      *
      */
-    public void testRollbackOnTimeoutTxRemapReadCommitted() throws Exception {
-        doTestRollbackOnTimeoutTxRemap(READ_COMMITTED);
+    public void testRollbackOnTimeoutTxRemapPessimisticReadCommitted() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, READ_COMMITTED);
     }
 
     /**
      *
      */
-    public void testRollbackOnTimeoutTxRemapRepeatableRead() throws Exception {
-        doTestRollbackOnTimeoutTxRemap(REPEATABLE_READ);
+    public void testRollbackOnTimeoutTxRemapPessimisticRepeatableRead() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, REPEATABLE_READ);
     }
 
     /**
      *
      */
-    public void testRollbackOnTimeoutTxRemapSerializable() throws Exception {
-        doTestRollbackOnTimeoutTxRemap(SERIALIZABLE);
+    public void testRollbackOnTimeoutTxRemapPessimisticSerializable() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, SERIALIZABLE);
     }
 
     /**
      *
      */
-    private void doTestRollbackOnTimeoutTxRemap(TransactionIsolation isolation) throws Exception {
+    private void doTestRollbackOnTimeoutTxRemap(TransactionConcurrency concurrency, TransactionIsolation isolation) throws Exception {
         Ignite client = startClient();
 
         Ignite crd = grid(0);
@@ -626,13 +764,14 @@ private void doTestRollbackOnTimeoutTxRemap(TransactionIsolation isolation) thro
         // Delay prepare until exchange is finished.
         TestRecordingCommunicationSpi.spi(client).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
             @Override public boolean apply(ClusterNode node, Message msg) {
-                return msg instanceof GridNearTxPrepareRequest;
+                return concurrency == PESSIMISTIC ?
+                    msg instanceof GridNearLockRequest : msg instanceof GridNearTxPrepareRequest;
             }
         });
 
         IgniteInternalFuture fut0 = runAsync(new Runnable() {
             @Override public void run() {
-                try (Transaction tx = client.transactions().txStart(OPTIMISTIC, isolation, 5000, 1)) {
+                try (Transaction tx = client.transactions().txStart(concurrency, isolation, 5000, 1)) {
                     client.cache(CACHE_NAME).put(keys.get(0), 0);
 
                     tx.commit();
@@ -654,7 +793,7 @@ private void doTestRollbackOnTimeoutTxRemap(TransactionIsolation isolation) thro
 
                     TestRecordingCommunicationSpi.spi(client).waitForBlocked();
 
-                    // Delivered prepare message will trigger remap.
+                    // Trigger remap.
                     TestRecordingCommunicationSpi.spi(client).stopBlock();
                 }
                 catch (Exception e) {

From 3e18a523817f7175ec3131db26147e352d8a8324 Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Wed, 15 Aug 2018 17:59:32 +0300
Subject: [PATCH 300/543] fix imports (IGNITE-9244)

---
 .../ignite/internal/processors/cache/CacheGroupContext.java      | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 225c53a9198b0..434b8fb5dd9db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -39,7 +39,6 @@
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsEvictor;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;

From b4e29ad9a9dcd3572366ed000395ec3ffc34a79f Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 16 Aug 2018 14:21:19 +0300
Subject: [PATCH 301/543] GG-14091 Add idle verify dump and idle verify v2 to
 security chain.

---
 .../ignite/util/GridCommandHandlerTest.java   | 27 +++++++------------
 1 file changed, 9 insertions(+), 18 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 2d5a4ddc1396e..4f922dbab59c9 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -881,7 +881,7 @@ public void testCacheIdleVerify() throws Exception {
         IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
             .setAffinity(new RendezvousAffinityFunction(false, 32))
             .setBackups(1)
-            .setName("cacheIV"));
+            .setName(DEFAULT_CACHE_NAME));
 
         for (int i = 0; i < 100; i++)
             cache.put(i, i);
@@ -894,7 +894,7 @@ public void testCacheIdleVerify() throws Exception {
 
         HashSet<Integer> clearKeys = new HashSet<>(Arrays.asList(1, 2, 3, 4, 5, 6));
 
-        ((IgniteEx)ignite).context().cache().cache("cacheIV").clearLocallyAll(clearKeys, true, true, true);
+        ((IgniteEx)ignite).context().cache().cache(DEFAULT_CACHE_NAME).clearLocallyAll(clearKeys, true, true, true);
 
         assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify"));
 
@@ -1124,7 +1124,7 @@ public void testCacheContention() throws Exception {
                 .setAffinity(new RendezvousAffinityFunction(false, 32))
                 .setAtomicityMode(TRANSACTIONAL)
                 .setBackups(1)
-                .setName("cacheCont"));
+                .setName(DEFAULT_CACHE_NAME));
 
             final CountDownLatch l = new CountDownLatch(1);
 
@@ -1212,23 +1212,14 @@ public void testCacheGroups() throws Exception {
 
         ignite.cluster().active(true);
 
-        IgniteCache<Object, Object> cache1 = ignite.createCache(new CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, 32))
-            .setBackups(1)
-            .setGroupName("G100")
-            .setName("cacheG1"));
-
-        IgniteCache<Object, Object> cache2 = ignite.createCache(new CacheConfiguration<>()
+        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
             .setAffinity(new RendezvousAffinityFunction(false, 32))
             .setBackups(1)
             .setGroupName("G100")
-            .setName("cacheG2"));
-
-        for (int i = 0; i < 100; i++) {
-            cache1.put(i, i);
+            .setName(DEFAULT_CACHE_NAME));
 
-            cache2.put(i, i);
-        }
+        for (int i = 0; i < 100; i++)
+            cache.put(i, i);
 
         injectTestSystemOut();
 
@@ -1248,7 +1239,7 @@ public void testCacheAffinity() throws Exception {
         IgniteCache<Object, Object> cache1 = ignite.createCache(new CacheConfiguration<>()
             .setAffinity(new RendezvousAffinityFunction(false, 32))
             .setBackups(1)
-            .setName("cacheAf"));
+            .setName(DEFAULT_CACHE_NAME));
 
         for (int i = 0; i < 100; i++)
             cache1.put(i, i);
@@ -1257,7 +1248,7 @@ public void testCacheAffinity() throws Exception {
 
         assertEquals(EXIT_CODE_OK, execute("--cache", "list", ".*"));
 
-        assertTrue(testOut.toString().contains("cacheName=cacheAf"));
+        assertTrue(testOut.toString().contains("cacheName=" + DEFAULT_CACHE_NAME));
         assertTrue(testOut.toString().contains("prim=32"));
         assertTrue(testOut.toString().contains("mapped=32"));
         assertTrue(testOut.toString().contains("affCls=RendezvousAffinityFunction"));

From 64b6504b4273e5a661c1420aea8ecb888f953c30 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Thu, 16 Aug 2018 17:33:52 +0300
Subject: [PATCH 302/543] IGNITE-9196: SQL: Fixed memory lead in mapper. This
 closes #4505.

(cherry picked from commit bfa192ca473c992353c8bae8ba9aa5fa359378b3)
---
 .../query/h2/twostep/MapQueryResult.java      |   2 +-
 .../h2/twostep/CacheQueryMemoryLeakTest.java  | 159 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite2.java       |   5 +-
 3 files changed, 164 insertions(+), 2 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/CacheQueryMemoryLeakTest.java

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryResult.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryResult.java
index 733590c450515..fb928c4ec12ab 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryResult.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/MapQueryResult.java
@@ -239,7 +239,7 @@ synchronized boolean fetchNextPage(List<Value[]> rows, int pageSize) {
             rows.add(res.currentRow());
         }
 
-        return false;
+        return !res.hasNext();
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/CacheQueryMemoryLeakTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/CacheQueryMemoryLeakTest.java
new file mode 100644
index 0000000000000..754504e14a990
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/CacheQueryMemoryLeakTest.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.query.Query;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/** */
+public class CacheQueryMemoryLeakTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration igniteCfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)igniteCfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        if (igniteInstanceName.equals("client"))
+            igniteCfg.setClientMode(true);
+
+        return igniteCfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * Check, that query results are not accumulated, when result set size is a multiple of a {@link Query#pageSize}.
+     *
+     * @throws Exception If failed.
+     */
+    public void testResultIsMultipleOfPage() throws Exception {
+        IgniteEx srv = (IgniteEx)startGrid("server");
+        Ignite client = startGrid("client");
+
+        IgniteCache<Integer, Person> cache = startPeopleCache(client);
+
+        int pages = 3;
+        int pageSize = 1024;
+
+        for (int i = 0; i < pages * pageSize; i++) {
+            Person p = new Person("Person #" + i, 25);
+
+            cache.put(i, p);
+        }
+
+        for (int i = 0; i < 100; i++) {
+            Query<List<?>> qry = new SqlFieldsQuery("select * from people");
+
+            qry.setPageSize(pageSize);
+
+            QueryCursor<List<?>> cursor = cache.query(qry);
+
+            cursor.getAll();
+
+            cursor.close();
+        }
+
+        assertTrue("MapNodeResults is not cleared on the map node.", isMapNodeResultsEmpty(srv));
+    }
+
+    /**
+     * @param node Ignite node.
+     * @return {@code True}, if all MapQueryResults are removed from internal node's structures. {@code False}
+     * otherwise.
+     */
+    private boolean isMapNodeResultsEmpty(IgniteEx node) {
+        IgniteH2Indexing idx = (IgniteH2Indexing)node.context().query().getIndexing();
+
+        GridMapQueryExecutor mapQryExec = idx.mapQueryExecutor();
+
+        Map<UUID, MapNodeResults> qryRess =
+            GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "qryRess");
+
+        for (MapNodeResults nodeRess : qryRess.values()) {
+            Map<MapRequestKey, MapQueryResults> nodeQryRess =
+                GridTestUtils.getFieldValue(nodeRess, MapNodeResults.class, "res");
+
+            if (!nodeQryRess.isEmpty())
+                return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * @param node Ignite instance.
+     * @return Cache.
+     */
+    private static IgniteCache<Integer, Person> startPeopleCache(Ignite node) {
+        CacheConfiguration<Integer, Person> cacheCfg = new CacheConfiguration<>("people");
+
+        QueryEntity qe = new QueryEntity(Integer.class, Person.class);
+
+        qe.setTableName("people");
+
+        cacheCfg.setQueryEntities(Collections.singleton(qe));
+
+        cacheCfg.setSqlSchema("PUBLIC");
+
+        return node.getOrCreateCache(cacheCfg);
+    }
+
+    /** */
+    @SuppressWarnings("unused")
+    public static class Person {
+        /** */
+        @QuerySqlField
+        private String name;
+
+        /** */
+        @QuerySqlField
+        private int age;
+
+        /**
+         * @param name Name.
+         * @param age Age.
+         */
+        public Person(String name, int age) {
+            this.name = name;
+            this.age = age;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index 1b76283f8d685..093423dd50f0d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
-import org.apache.ignite.internal.processors.cache.QueryJoinWithDifferentNodeFiltersTest;
 import org.apache.ignite.internal.processors.cache.CacheScanPartitionQueryFallbackSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheCrossCacheJoinRandomTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheObjectKeyIndexingSelfTest;
@@ -26,6 +25,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheQueryEvictsMultiThreadedSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheQueryMultiThreadedSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheSqlQueryMultiThreadedSelfTest;
+import org.apache.ignite.internal.processors.cache.QueryJoinWithDifferentNodeFiltersTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheClientQueryReplicatedNodeRestartSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheDistributedQueryStopOnCancelOrTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteCacheQueryNodeFailTest;
@@ -50,6 +50,7 @@
 import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlDistributedJoinSelfTest;
 import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlSegmentedIndexMultiNodeSelfTest;
 import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlSegmentedIndexSelfTest;
+import org.apache.ignite.internal.processors.query.h2.twostep.CacheQueryMemoryLeakTest;
 import org.apache.ignite.testframework.IgniteTestSuite;
 
 /**
@@ -107,6 +108,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(QueryJoinWithDifferentNodeFiltersTest.class);
 
+        suite.addTestSuite(CacheQueryMemoryLeakTest.class);
+
         return suite;
     }
 }

From b32f510e0d16adca2232f15ae70af1f15eb39940 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 16 Aug 2018 18:39:49 +0300
Subject: [PATCH 303/543] IGNITE-9227 Fixed missing reply to a single message
 during coordinator failover. Fixes #4518.

(cherry picked from commit 66fcde3)
---
 .../cache/CacheAffinitySharedManager.java     |  17 +-
 .../GridCachePartitionExchangeManager.java    |  34 +++-
 .../preloader/CacheGroupAffinityMessage.java  |  17 +-
 .../GridDhtPartitionsExchangeFuture.java      | 191 +++++++++++-------
 .../GridDhtPartitionsFullMessage.java         |   4 +-
 .../cache/persistence/file/FilePageStore.java |   2 +-
 ...itionsExchangeCoordinatorFailoverTest.java | 155 ++++++++++++++
 .../testsuites/IgniteCacheTestSuite6.java     |   7 +
 8 files changed, 343 insertions(+), 84 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 2871e8294c179..08a07a457f247 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -70,7 +70,6 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteClosure;
-import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
@@ -1383,9 +1382,13 @@ public void onLocalJoin(final GridDhtPartitionsExchangeFuture fut,
 
         final Map<Long, ClusterNode> nodesByOrder = new HashMap<>();
 
-        final Map<Integer, CacheGroupAffinityMessage> joinedNodeAff = msg.joinedNodeAffinity();
+        final Map<Integer, CacheGroupAffinityMessage> receivedAff = msg.joinedNodeAffinity();
 
-        assert F.isEmpty(affReq) || (!F.isEmpty(joinedNodeAff) && joinedNodeAff.size() >= affReq.size()) : msg;
+        assert F.isEmpty(affReq) || (!F.isEmpty(receivedAff) && receivedAff.size() >= affReq.size())
+            : ("Requested and received affinity are different " +
+                "[requestedCnt=" + (affReq != null ? affReq.size() : "none") +
+                ", receivedCnt=" + (receivedAff != null ? receivedAff.size() : "none") +
+                ", msg=" + msg + "]");
 
         forAllCacheGroups(false, new IgniteInClosureX<GridAffinityAssignmentCache>() {
             @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException {
@@ -1398,7 +1401,7 @@ public void onLocalJoin(final GridDhtPartitionsExchangeFuture fut,
                 if (affReq != null && affReq.contains(aff.groupId())) {
                     assert AffinityTopologyVersion.NONE.equals(aff.lastVersion());
 
-                    CacheGroupAffinityMessage affMsg = joinedNodeAff.get(aff.groupId());
+                    CacheGroupAffinityMessage affMsg = receivedAff.get(aff.groupId());
 
                     assert affMsg != null;
 
@@ -1774,8 +1777,10 @@ public boolean onCentralizedAffinityChange(final GridDhtPartitionsExchangeFuture
      * @throws IgniteCheckedException If failed.
      * @return Future completed when caches initialization is done.
      */
-    public IgniteInternalFuture<?> initCoordinatorCaches(final GridDhtPartitionsExchangeFuture fut,
-        final boolean newAff) throws IgniteCheckedException {
+    public IgniteInternalFuture<?> initCoordinatorCaches(
+        final GridDhtPartitionsExchangeFuture fut,
+        final boolean newAff
+    ) throws IgniteCheckedException {
         final List<IgniteInternalFuture<AffinityTopologyVersion>> futs = new ArrayList<>();
 
         final AffinityTopologyVersion topVer = fut.initialVersion();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 824aa6718b2ec..e63c484826594 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -116,6 +116,7 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.thread.IgniteThread;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -330,6 +331,22 @@ private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) {
         cctx.io().addCacheHandler(0, GridDhtPartitionsSingleMessage.class,
             new MessageHandler<GridDhtPartitionsSingleMessage>() {
                 @Override public void onMessage(final ClusterNode node, final GridDhtPartitionsSingleMessage msg) {
+                    GridDhtPartitionExchangeId exchangeId = msg.exchangeId();
+
+                    if (exchangeId != null) {
+                        GridDhtPartitionsExchangeFuture fut = exchangeFuture(exchangeId);
+
+                        boolean fastReplied = fut.fastReplyOnSingleMessage(node, msg);
+
+                        if (fastReplied) {
+                            if (log.isInfoEnabled())
+                                log.info("Fast replied to single message " +
+                                    "[exchId=" + exchangeId + ", nodeId=" + node.id() + "]");
+
+                            return;
+                        }
+                    }
+
                     if (!crdInitFut.isDone() && !msg.restoreState()) {
                         GridDhtPartitionExchangeId exchId = msg.exchangeId();
 
@@ -1362,6 +1379,15 @@ private GridDhtPartitionExchangeId exchangeId(UUID nodeId, AffinityTopologyVersi
         return new GridDhtPartitionExchangeId(nodeId, evt, topVer);
     }
 
+    /**
+     * Gets exchange future by exchange id.
+     *
+     * @param exchId Exchange id.
+     */
+    private GridDhtPartitionsExchangeFuture exchangeFuture(@NotNull GridDhtPartitionExchangeId exchId) {
+        return exchangeFuture(exchId, null, null, null, null);
+    }
+
     /**
      * @param exchId Exchange ID.
      * @param discoEvt Discovery event.
@@ -1370,11 +1396,13 @@ private GridDhtPartitionExchangeId exchangeId(UUID nodeId, AffinityTopologyVersi
      * @param affChangeMsg Affinity change message.
      * @return Exchange future.
      */
-    private GridDhtPartitionsExchangeFuture exchangeFuture(GridDhtPartitionExchangeId exchId,
+    private GridDhtPartitionsExchangeFuture exchangeFuture(
+        @NotNull GridDhtPartitionExchangeId exchId,
         @Nullable DiscoveryEvent discoEvt,
         @Nullable DiscoCache cache,
         @Nullable ExchangeActions exchActions,
-        @Nullable CacheAffinityChangeMessage affChangeMsg) {
+        @Nullable CacheAffinityChangeMessage affChangeMsg
+    ) {
         GridDhtPartitionsExchangeFuture fut;
 
         GridDhtPartitionsExchangeFuture old = exchFuts.addx(
@@ -1571,7 +1599,7 @@ else if (!grp.isLocal())
                     scheduleResendPartitions();
             }
             else {
-                GridDhtPartitionsExchangeFuture exchFut = exchangeFuture(msg.exchangeId(), null, null, null, null);
+                GridDhtPartitionsExchangeFuture exchFut = exchangeFuture(msg.exchangeId());
 
                 if (log.isDebugEnabled())
                     log.debug("Notifying exchange future about single message: " + exchFut);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java
index 8a1ffb4ebb22f..7da4051e27929 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.typedef.F;
@@ -144,7 +145,8 @@ static Map<Integer, CacheGroupAffinityMessage> createAffinityMessages(
         GridCacheSharedContext cctx,
         AffinityTopologyVersion topVer,
         Collection<Integer> affReq,
-        @Nullable Map<Integer, CacheGroupAffinityMessage> cachesAff) {
+        @Nullable Map<Integer, CacheGroupAffinityMessage> cachesAff
+    ) {
         assert !F.isEmpty(affReq) : affReq;
 
         if (cachesAff == null)
@@ -152,7 +154,18 @@ static Map<Integer, CacheGroupAffinityMessage> createAffinityMessages(
 
         for (Integer grpId : affReq) {
             if (!cachesAff.containsKey(grpId)) {
-                GridAffinityAssignmentCache aff = cctx.affinity().affinity(grpId);
+                GridAffinityAssignmentCache aff = cctx.affinity().groupAffinity(grpId);
+
+                // If no coordinator group holder on the node, try fetch affinity from existing cache group.
+                if (aff == null) {
+                    CacheGroupContext grp = cctx.cache().cacheGroup(grpId);
+
+                    assert grp != null : "No cache group holder or cache group to create AffinityMessage"
+                        + ". Requested group id: " + grpId
+                        + ". Topology version: " + topVer;
+
+                    aff = grp.affinity();
+                }
 
                 List<List<ClusterNode>> assign = aff.readyAssignments(topVer);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index c111706fa8a5c..7e9dd1472d1ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -24,8 +24,10 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.Callable;
@@ -37,6 +39,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
+import java.util.stream.Stream;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
@@ -1578,74 +1581,90 @@ private GridDhtPartitionsFullMessage createPartitionsMessage(boolean compress,
     }
 
     /**
-     * @param msg Message to send.
+     * @param fullMsg Message to send.
      * @param nodes Nodes.
      * @param mergedJoinExchMsgs Messages received from merged 'join node' exchanges.
-     * @param joinedNodeAff Affinity if was requested by some nodes.
+     * @param affinityForJoinedNodes Affinity if was requested by some nodes.
      */
     private void sendAllPartitions(
-        GridDhtPartitionsFullMessage msg,
+        GridDhtPartitionsFullMessage fullMsg,
         Collection<ClusterNode> nodes,
         Map<UUID, GridDhtPartitionsSingleMessage> mergedJoinExchMsgs,
-        Map<Integer, CacheGroupAffinityMessage> joinedNodeAff) {
-        boolean singleNode = nodes.size() == 1;
-
-        GridDhtPartitionsFullMessage joinedNodeMsg = null;
-
+        Map<Integer, CacheGroupAffinityMessage> affinityForJoinedNodes
+    ) {
         assert !nodes.contains(cctx.localNode());
 
         if (log.isDebugEnabled()) {
             log.debug("Sending full partition map [nodeIds=" + F.viewReadOnly(nodes, F.node2id()) +
-                ", exchId=" + exchId + ", msg=" + msg + ']');
+                ", exchId=" + exchId + ", msg=" + fullMsg + ']');
         }
 
-        for (ClusterNode node : nodes) {
-            GridDhtPartitionsFullMessage sndMsg = msg;
-
-            if (joinedNodeAff != null) {
-                if (singleNode)
-                    msg.joinedNodeAffinity(joinedNodeAff);
-                else {
-                    GridDhtPartitionsSingleMessage singleMsg = msgs.get(node.id());
+        // Find any single message with affinity request. This request exists only for newly joined nodes.
+        Optional<GridDhtPartitionsSingleMessage> singleMsgWithAffinityReq = nodes.stream()
+            .flatMap(node -> Optional.ofNullable(msgs.get(node.id()))
+                .filter(singleMsg -> singleMsg.cacheGroupsAffinityRequest() != null)
+                .map(Stream::of)
+                .orElse(Stream.empty()))
+            .findAny();
+
+        // Prepare full message for newly joined nodes with affinity request.
+        final GridDhtPartitionsFullMessage fullMsgWithAffinity = singleMsgWithAffinityReq
+            .filter(singleMessage -> affinityForJoinedNodes != null)
+            .map(singleMessage -> fullMsg.copy().joinedNodeAffinity(affinityForJoinedNodes))
+            .orElse(null);
+
+        // Prepare and send full messages for given nodes.
+        nodes.stream()
+            .map(node -> {
+                // No joined nodes, just send a regular full message.
+                if (fullMsgWithAffinity == null)
+                    return new T2<>(node, fullMsg);
+
+                return new T2<>(
+                    node,
+                    // If single message contains affinity request, use special full message for such single messages.
+                    Optional.ofNullable(msgs.get(node.id()))
+                        .filter(singleMsg -> singleMsg.cacheGroupsAffinityRequest() != null)
+                        .map(singleMsg -> fullMsgWithAffinity)
+                        .orElse(fullMsg)
+                );
+            })
+            .map(nodeAndMsg -> {
+                ClusterNode node = nodeAndMsg.get1();
+                GridDhtPartitionsFullMessage fullMsgToSend = nodeAndMsg.get2();
+
+                // If exchange has merged, use merged version of exchange id.
+                GridDhtPartitionExchangeId sndExchId = mergedJoinExchMsgs != null
+                    ? Optional.ofNullable(mergedJoinExchMsgs.get(node.id()))
+                        .map(GridDhtPartitionsAbstractMessage::exchangeId)
+                        .orElse(exchangeId())
+                    : exchangeId();
 
-                    if (singleMsg != null && singleMsg.cacheGroupsAffinityRequest() != null) {
-                        if (joinedNodeMsg == null) {
-                            joinedNodeMsg = msg.copy();
+                if (sndExchId != null && !sndExchId.equals(exchangeId())) {
+                    GridDhtPartitionsFullMessage fullMsgWithUpdatedExchangeId = fullMsgToSend.copy();
 
-                            joinedNodeMsg.joinedNodeAffinity(joinedNodeAff);
-                        }
+                    fullMsgWithUpdatedExchangeId.exchangeId(sndExchId);
 
-                        sndMsg = joinedNodeMsg;
-                    }
+                    return new T2<>(node, fullMsgWithUpdatedExchangeId);
                 }
-            }
-
-            try {
-                GridDhtPartitionExchangeId sndExchId = exchangeId();
 
-                if (mergedJoinExchMsgs != null) {
-                    GridDhtPartitionsSingleMessage mergedMsg = mergedJoinExchMsgs.get(node.id());
+                return new T2<>(node, fullMsgToSend);
+            })
+            .forEach(nodeAndMsg -> {
+                ClusterNode node = nodeAndMsg.get1();
+                GridDhtPartitionsFullMessage fullMsgToSend = nodeAndMsg.get2();
 
-                    if (mergedMsg != null)
-                        sndExchId = mergedMsg.exchangeId();
+                try {
+                    cctx.io().send(node, fullMsgToSend, SYSTEM_POOL);
                 }
-
-                if (sndExchId != null && !sndExchId.equals(exchangeId())) {
-                    sndMsg = sndMsg.copy();
-
-                    sndMsg.exchangeId(sndExchId);
+                catch (ClusterTopologyCheckedException e) {
+                    if (log.isDebugEnabled())
+                        log.debug("Failed to send partitions, node failed: " + node);
                 }
-
-                cctx.io().send(node, sndMsg, SYSTEM_POOL);
-            }
-            catch (ClusterTopologyCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Failed to send partitions, node failed: " + node);
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to send partitions [node=" + node + ']', e);
-            }
-        }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Failed to send partitions [node=" + node + ']', e);
+                }
+            });
     }
 
     /**
@@ -2188,6 +2207,34 @@ public void onReceiveSingleMessage(final ClusterNode node, final GridDhtPartitio
         });
     }
 
+    /**
+     * Tries to fast reply with {@link GridDhtPartitionsFullMessage} on received single message
+     * in case of exchange future has already completed.
+     *
+     * @param node Cluster node which sent single message.
+     * @param msg Single message.
+     * @return {@code true} if fast reply succeed.
+     */
+    public boolean fastReplyOnSingleMessage(final ClusterNode node, final GridDhtPartitionsSingleMessage msg) {
+        GridDhtPartitionsExchangeFuture futToFastReply = this;
+
+        ExchangeLocalState currState;
+
+        synchronized (mux) {
+            currState = state;
+
+            if (currState == ExchangeLocalState.MERGED)
+                futToFastReply = mergedWith;
+        }
+
+        if (currState == ExchangeLocalState.DONE)
+            futToFastReply.processSingleMessage(node.id(), msg);
+        else if (currState == ExchangeLocalState.MERGED)
+            futToFastReply.processMergedMessage(node, msg);
+
+        return currState == ExchangeLocalState.MERGED || currState == ExchangeLocalState.DONE;
+    }
+
     /**
      * @param nodeId Node ID.
      * @param msg Client's message.
@@ -2903,9 +2950,7 @@ else if (forceAffReassignment)
                 synchronized (mux) {
                     srvNodes.remove(cctx.localNode());
 
-                    nodes = U.newHashSet(srvNodes.size());
-
-                    nodes.addAll(srvNodes);
+                    nodes = new LinkedHashSet<>(srvNodes);
 
                     mergedJoinExchMsgs0 = mergedJoinExchMsgs;
 
@@ -3063,41 +3108,45 @@ private void assignPartitionsStates() {
     private void sendAllPartitionsToNode(FinishState finishState, GridDhtPartitionsSingleMessage msg, UUID nodeId) {
         ClusterNode node = cctx.node(nodeId);
 
-        if (node != null) {
-            GridDhtPartitionsFullMessage fullMsg = finishState.msg.copy();
+        if (node == null) {
+            if (log.isDebugEnabled())
+                log.debug("Failed to send partitions, node failed: " + nodeId);
+
+            return;
+        }
 
-            Collection<Integer> affReq = msg.cacheGroupsAffinityRequest();
+        GridDhtPartitionsFullMessage fullMsg = finishState.msg.copy();
 
-            if (affReq != null) {
-                Map<Integer, CacheGroupAffinityMessage> aff = CacheGroupAffinityMessage.createAffinityMessages(
-                    cctx,
-                    finishState.resTopVer,
-                    affReq,
-                    null);
+        Collection<Integer> affReq = msg.cacheGroupsAffinityRequest();
 
-                fullMsg.joinedNodeAffinity(aff);
-            }
+        if (affReq != null) {
+            Map<Integer, CacheGroupAffinityMessage> aff = CacheGroupAffinityMessage.createAffinityMessages(
+                cctx,
+                finishState.resTopVer,
+                affReq,
+                null);
 
-            if (!fullMsg.exchangeId().equals(msg.exchangeId())) {
-                fullMsg = fullMsg.copy();
+            fullMsg.joinedNodeAffinity(aff);
+        }
 
-                fullMsg.exchangeId(msg.exchangeId());
-            }
+        if (!fullMsg.exchangeId().equals(msg.exchangeId())) {
+            fullMsg = fullMsg.copy();
+
+            fullMsg.exchangeId(msg.exchangeId());
+        }
 
             try {
                 cctx.io().send(node, fullMsg, SYSTEM_POOL);
-            }
+}
+
             catch (ClusterTopologyCheckedException e) {
                 if (log.isDebugEnabled())
                     log.debug("Failed to send partitions, node failed: " + node);
             }
             catch (IgniteCheckedException e) {
                 U.error(log, "Failed to send partitions [node=" + node + ']', e);
-            }
-        }
-        else if (log.isDebugEnabled())
-            log.debug("Failed to send partitions, node failed: " + nodeId);
 
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index 59624687b9606..ab45d8be35620 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -218,8 +218,10 @@ public AffinityTopologyVersion resultTopologyVersion() {
     /**
      * @param joinedNodeAff Caches affinity for joining nodes.
      */
-    void joinedNodeAffinity(Map<Integer, CacheGroupAffinityMessage> joinedNodeAff) {
+    GridDhtPartitionsFullMessage joinedNodeAffinity(Map<Integer, CacheGroupAffinityMessage> joinedNodeAff) {
         this.joinedNodeAff = joinedNodeAff;
+
+        return this;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index d2d5506f6cb01..9a1eb30fa58e9 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -165,7 +165,7 @@ private long initFile(FileIO fileIO) throws IOException {
         try {
             ByteBuffer hdr = header(type, dbCfg.getPageSize());
 
-        fileIO.writeFully(hdr);
+            fileIO.writeFully(hdr);
 
             //there is 'super' page in every file
             return headerSize() + dbCfg.getPageSize();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java
new file mode 100644
index 0000000000000..a2adcf77a7bac
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ * Advanced coordinator failure scenarios during PME.
+ */
+public class PartitionsExchangeCoordinatorFailoverTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        IgnitePredicate<ClusterNode> nodeFilter = node -> node.consistentId().equals(igniteInstanceName);
+
+        cfg.setCacheConfiguration(
+            new CacheConfiguration("cache-" + igniteInstanceName)
+                .setBackups(1)
+                .setNodeFilter(nodeFilter)
+                .setAffinity(new RendezvousAffinityFunction(false, 32))
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 60 * 1000L;
+    }
+
+    /**
+     * Tests that new coordinator is able to finish old exchanges in case of in-complete coordinator initialization.
+     */
+    public void testNewCoordinatorCompletedExchange() throws Exception {
+        IgniteEx crd = (IgniteEx) startGrid("crd");
+
+        IgniteEx newCrd = startGrid(1);
+
+        crd.cluster().active(true);
+
+        // 3 node join topology version.
+        AffinityTopologyVersion joinThirdNodeVer = new AffinityTopologyVersion(3, 0);
+
+        // 4 node join topology version.
+        AffinityTopologyVersion joinFourNodeVer = new AffinityTopologyVersion(4, 0);
+
+        // Block FullMessage for newly joined nodes.
+        TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(crd);
+
+        final CountDownLatch sendFullMsgLatch = new CountDownLatch(1);
+
+        // Delay sending full message to newly joined nodes.
+        spi.blockMessages((node, msg) -> {
+            if (msg instanceof GridDhtPartitionsFullMessage && node.order() > 2) {
+                try {
+                    sendFullMsgLatch.await();
+                }
+                catch (Throwable ignored) { }
+
+                return true;
+            }
+
+            return false;
+        });
+
+        IgniteInternalFuture joinTwoNodesFut = GridTestUtils.runAsync(() -> startGridsMultiThreaded(2, 2));
+
+        GridCachePartitionExchangeManager exchangeMgr = newCrd.context().cache().context().exchange();
+
+        // Wait till new coordinator finishes third node join exchange.
+        GridTestUtils.waitForCondition(
+            () -> exchangeMgr.readyAffinityVersion().compareTo(joinThirdNodeVer) >= 0,
+            getTestTimeout()
+        );
+
+        IgniteInternalFuture startLastNodeFut = GridTestUtils.runAsync(() -> startGrid(5));
+
+        // Wait till new coordinator starts third node join exchange.
+        GridTestUtils.waitForCondition(
+            () -> exchangeMgr.lastTopologyFuture().initialVersion().compareTo(joinFourNodeVer) >= 0,
+            getTestTimeout()
+        );
+
+        IgniteInternalFuture stopCrdFut = GridTestUtils.runAsync(() -> stopGrid("crd", true, false));
+
+        // Magic sleep to make sure that coordinator stop process has started.
+        U.sleep(1000);
+
+        // Resume full messages sending to unblock coordinator stopping process.
+        sendFullMsgLatch.countDown();
+
+        // Coordinator stop should succeed.
+        stopCrdFut.get();
+
+        // Nodes join should succeed.
+        joinTwoNodesFut.get();
+
+        startLastNodeFut.get();
+
+        awaitPartitionMapExchange();
+
+        // Check that all caches are operable.
+        for (Ignite grid : G.allGrids()) {
+            IgniteCache cache = grid.cache("cache-" + grid.cluster().localNode().consistentId());
+
+            Assert.assertNotNull(cache);
+
+            cache.put(0, 0);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 77285bed1963e..890954440f909 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.processors.cache.PartitionedAtomicCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.PartitionedTransactionalOptimisticCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.PartitionedTransactionalPessimisticCacheGetsDistributionTest;
+import org.apache.ignite.internal.processors.cache.PartitionsExchangeCoordinatorFailoverTest;
 import org.apache.ignite.internal.processors.cache.ReplicatedAtomicCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalOptimisticCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalPessimisticCacheGetsDistributionTest;
@@ -123,6 +124,12 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteExchangeLatchManagerCoordinatorFailTest.class);
 
+        suite.addTestSuite(PartitionsExchangeCoordinatorFailoverTest.class);
+
+        //suite.addTestSuite(CacheClientsConcurrentStartTest.class);
+        //suite.addTestSuite(GridCacheRebalancingOrderingTest.class);
+        //suite.addTestSuite(IgniteCacheClientMultiNodeUpdateTopologyLockTest.class);
+
         return suite;
     }
 }

From bc1a1c685ec66be5f6360a36f7f842e79b040412 Mon Sep 17 00:00:00 2001
From: Evgenii Zhuravlev <ezhuravlev@gridgain.com>
Date: Fri, 10 Aug 2018 14:23:37 +0300
Subject: [PATCH 304/543] IGNITE-5103 Fixed TcpDiscoverySpi not to ignore
 maxMissedHeartbeats property. Fixes #4446.

(cherry picked from commit 1c840f59016273e0e99c95345c3afde639ef9689)
---
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  40 +++++-
 .../tcp/TcpClientDiscoverySpiSelfTest.java    |  10 +-
 .../TcpDiscoveryClientSuspensionSelfTest.java | 135 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   2 +
 4 files changed, 184 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index e4ea0be621e5e..906b97a7e749c 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6590,6 +6590,9 @@ private class ClientMessageWorker extends MessageWorker<T2<TcpDiscoveryAbstractM
         /** Current client metrics. */
         private volatile ClusterMetrics metrics;
 
+        /** Last metrics update message receive time. */
+        private volatile long lastMetricsUpdateMsgTime;
+
         /** */
         private final AtomicReference<GridFutureAdapter<Boolean>> pingFut = new AtomicReference<>();
 
@@ -6602,10 +6605,12 @@ private class ClientMessageWorker extends MessageWorker<T2<TcpDiscoveryAbstractM
          * @param log Logger.
          */
         private ClientMessageWorker(Socket sock, UUID clientNodeId, IgniteLogger log) {
-            super("tcp-disco-client-message-worker", log, 2000, null);
+            super("tcp-disco-client-message-worker", log, Math.max(spi.metricsUpdateFreq, 10), null);
 
             this.sock = sock;
             this.clientNodeId = clientNodeId;
+
+            lastMetricsUpdateMsgTime = U.currentTimeMillis();
         }
 
         /**
@@ -6626,6 +6631,8 @@ ClusterMetrics metrics() {
          * @param metrics New current client metrics.
          */
         void metrics(ClusterMetrics metrics) {
+            lastMetricsUpdateMsgTime = U.currentTimeMillis();
+
             this.metrics = metrics;
         }
 
@@ -6806,6 +6813,37 @@ public boolean ping(IgniteSpiOperationTimeoutHelper timeoutHelper) throws Interr
 
             U.closeQuiet(sock);
         }
+
+        /** {@inheritDoc} */
+        @Override protected void noMessageLoop() {
+            if (U.currentTimeMillis() - lastMetricsUpdateMsgTime > spi.clientFailureDetectionTimeout()) {
+                TcpDiscoveryNode clientNode = ring.node(clientNodeId);
+
+                if (clientNode != null) {
+                    boolean failedNode;
+
+                    synchronized (mux) {
+                        failedNode = failedNodes.containsKey(clientNode);
+                    }
+
+                    if (!failedNode) {
+                        String msg = "Client node considered as unreachable " +
+                            "and will be dropped from cluster, " +
+                            "because no metrics update messages received in interval: " +
+                            "TcpDiscoverySpi.clientFailureDetectionTimeout() ms. " +
+                            "It may be caused by network problems or long GC pause on client node, try to increase this " +
+                            "parameter. " +
+                            "[nodeId=" + clientNodeId +
+                            ", clientFailureDetectionTimeout=" + spi.clientFailureDetectionTimeout() +
+                            ']';
+
+                        failNode(clientNodeId, msg);
+
+                        U.warn(log, msg);
+                    }
+                }
+            }
+        }
     }
 
     /** */
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index 2d130e1db9052..c85e94e6c3889 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -148,7 +148,7 @@ public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     private boolean longSockTimeouts;
 
     /** */
-    protected long clientFailureDetectionTimeout = 1000;
+    protected long clientFailureDetectionTimeout = 5000;
 
     /** */
     private IgniteInClosure2X<TcpDiscoveryAbstractMessage, Socket> afterWrite;
@@ -263,7 +263,7 @@ protected TcpDiscoverySpi getDiscoverySpi() {
         clientIpFinder = null;
         joinTimeout = TcpDiscoverySpi.DFLT_JOIN_TIMEOUT;
         netTimeout = TcpDiscoverySpi.DFLT_NETWORK_TIMEOUT;
-        clientFailureDetectionTimeout = 1000;
+        clientFailureDetectionTimeout = 5000;
         longSockTimeouts = false;
 
         assert G.allGrids().isEmpty();
@@ -538,6 +538,8 @@ public void testPingFailedClientNode() throws Exception {
 
         ((TestTcpDiscoverySpi)client.configuration().getDiscoverySpi()).resumeAll();
 
+        Thread.sleep(2000);
+
         assert ((IgniteEx)srv1).context().discovery().pingNode(client.cluster().localNode().id());
         assert ((IgniteEx)srv0).context().discovery().pingNode(client.cluster().localNode().id());
     }
@@ -583,6 +585,8 @@ public void testClientReconnectOnRouterSuspend() throws Exception {
      * @throws Exception If failed.
      */
     public void testClientReconnectOnRouterSuspendTopologyChange() throws Exception {
+        clientFailureDetectionTimeout = 20_000;
+
         reconnectAfterSuspend(true);
     }
 
@@ -1266,6 +1270,8 @@ public void testDuplicateId() throws Exception {
     public void testTimeoutWaitingNodeAddedMessage() throws Exception {
         longSockTimeouts = true;
 
+        clientFailureDetectionTimeout = 20_000;
+
         startServerNodes(2);
 
         final CountDownLatch cnt = new CountDownLatch(1);
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
new file mode 100644
index 0000000000000..a519d25d4114f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.Timer;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for missed client metrics update messages.
+ */
+public class TcpDiscoveryClientSuspensionSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(disco);
+
+        cfg.setMetricsUpdateFrequency(100);
+
+        cfg.setClientFailureDetectionTimeout(1000);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        System.setProperty(IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY, "10000");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneServer() throws Exception {
+        doTestClientSuspension(1);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testTwoServers() throws Exception {
+        doTestClientSuspension(2);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testThreeServers() throws Exception {
+        doTestClientSuspension(3);
+    }
+
+    /**
+     * @param serverCnt Servers count.
+     * @throws Exception If failed.
+     */
+    private void doTestClientSuspension(int serverCnt) throws Exception {
+        startGrids(serverCnt);
+
+        Ignition.setClientMode(true);
+
+        Ignite client = startGrid("client");
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(1, grid(i).cluster().forClients().nodes().size());
+
+        Thread.sleep(2000);
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(1, grid(i).cluster().forClients().nodes().size());
+
+        suspendClientMetricsUpdate(client);
+
+        Thread.sleep(2000);
+
+        for (int i = 0; i < serverCnt; i++)
+            assertEquals(0, grid(i).cluster().forClients().nodes().size());
+    }
+
+    /**
+     * @param client Client.
+     */
+    private void suspendClientMetricsUpdate(Ignite client) {
+        assert client.cluster().localNode().isClient();
+
+        ClientImpl impl = U.field(client.configuration().getDiscoverySpi(), "impl");
+
+        Timer timer = U.field(impl, "timer");
+
+        timer.cancel();
+
+        System.out.println("Metrics update message suspended");
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 984dcf696fe97..c6b461a8c1f12 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryClientSuspensionSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMultiThreadedTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeAttributesUpdateOnReconnectTest;
@@ -111,6 +112,7 @@ public static TestSuite suite() throws Exception {
         // Client connect.
         suite.addTest(new TestSuite(IgniteClientConnectTest.class));
         suite.addTest(new TestSuite(IgniteClientReconnectMassiveShutdownTest.class));
+        suite.addTest(new TestSuite(TcpDiscoveryClientSuspensionSelfTest.class));
 
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));

From d8af4076b65302ea31af461cda3fe747aea7c583 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Wed, 15 Aug 2018 20:28:48 +0300
Subject: [PATCH 305/543] IGNITE-8493 GridToStringBuilder arrayToString
 refactoring. - Fixes #4501.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
---
 .../util/tostring/GridToStringBuilder.java    | 1204 +++++++++--------
 .../util/tostring/SBLimitedLength.java        |   19 +
 .../tostring/GridToStringBuilderSelfTest.java |  359 ++++-
 ...teHadoopFileSystemClientBasedOpenTest.java |    2 +-
 4 files changed, 941 insertions(+), 643 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
index dbc33db88c544..1bd759ff27571 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/GridToStringBuilder.java
@@ -17,35 +17,40 @@
 
 package org.apache.ignite.internal.util.tostring;
 
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.Nullable;
-
 import java.io.Externalizable;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Serializable;
+import java.lang.reflect.Array;
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.EventListener;
-import java.util.LinkedList;
+import java.util.IdentityHashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.Queue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
 
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_COLLECTION_LIMIT;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE;
 
 /**
  * Provides auto-generation framework for {@code toString()} output.
  * <p>
+ * In case of recursion, repeatable objects will be shown as "ClassName@hash".
+ * But fields will be printed only for the first entry to prevent recursion.
+ * <p>
  * Default exclusion policy (can be overridden with {@link GridToStringInclude}
  * annotation):
  * <ul>
@@ -79,36 +84,44 @@
  * </ul>
  */
 public class GridToStringBuilder {
+    /** */
+    private static final Object[] EMPTY_ARRAY = new Object[0];
+
     /** */
     private static final Map<String, GridToStringClassDescriptor> classCache = new ConcurrentHashMap<>();
 
     /** {@link IgniteSystemProperties#IGNITE_TO_STRING_INCLUDE_SENSITIVE} */
     public static final boolean INCLUDE_SENSITIVE =
-        IgniteSystemProperties.getBoolean(IGNITE_TO_STRING_INCLUDE_SENSITIVE, true);
+            IgniteSystemProperties.getBoolean(IGNITE_TO_STRING_INCLUDE_SENSITIVE, true);
 
     /** */
     private static final int COLLECTION_LIMIT =
-        IgniteSystemProperties.getInteger(IGNITE_TO_STRING_COLLECTION_LIMIT, 100);
+            IgniteSystemProperties.getInteger(IGNITE_TO_STRING_COLLECTION_LIMIT, 100);
 
-    /** */
-    private static ThreadLocal<Queue<GridToStringThreadLocal>> threadCache = new ThreadLocal<Queue<GridToStringThreadLocal>>() {
-        @Override protected Queue<GridToStringThreadLocal> initialValue() {
-            Queue<GridToStringThreadLocal> queue = new LinkedList<>();
+    /** Every thread has its own string builder. */
+    private static ThreadLocal<SBLimitedLength> threadLocSB = new ThreadLocal<SBLimitedLength>() {
+        @Override protected SBLimitedLength initialValue() {
+            SBLimitedLength sb = new SBLimitedLength(256);
 
-            queue.offer(new GridToStringThreadLocal());
+            sb.initLimit(new SBLengthLimit());
 
-            return queue;
+            return sb;
         }
     };
 
-    /** */
-    private static ThreadLocal<SBLengthLimit> threadCurLen = new ThreadLocal<SBLengthLimit>() {
-        @Override protected SBLengthLimit initialValue() {
-            return new SBLengthLimit();
+    /**
+     * Contains objects currently printing in the string builder.
+     * <p>
+     * Since {@code toString()} methods can be chain-called from the same thread we
+     * have to keep a map of this objects pointed to the position of previous occurrence
+     * and remove/add them in each {@code toString()} apply.
+     */
+    private static ThreadLocal<IdentityHashMap<Object, Integer>> savedObjects = new ThreadLocal<IdentityHashMap<Object, Integer>>() {
+        @Override protected IdentityHashMap<Object, Integer> initialValue() {
+            return new IdentityHashMap<>();
         }
     };
 
-
     /**
      * Produces auto-generated output of string presentation for given object and its declaration class.
      *
@@ -128,18 +141,18 @@ public class GridToStringBuilder {
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0,
-        String name1, Object val1,
-        String name2, Object val2,
-        String name3, Object val3,
-        String name4, Object val4) {
+                                      String name0, Object val0,
+                                      String name1, Object val1,
+                                      String name2, Object val2,
+                                      String name3, Object val3,
+                                      String name4, Object val4) {
         return toString(cls,
-            obj,
-            name0, val0, false,
-            name1, val1, false,
-            name2, val2, false,
-            name3, val3, false,
-            name4, val4, false);
+                obj,
+                name0, val0, false,
+                name1, val1, false,
+                name2, val2, false,
+                name3, val3, false,
+                name4, val4, false);
     }
 
     /**
@@ -163,20 +176,20 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0,
-        String name1, Object val1,
-        String name2, Object val2,
-        String name3, Object val3,
-        String name4, Object val4,
-        String name5, Object val5) {
+                                      String name0, Object val0,
+                                      String name1, Object val1,
+                                      String name2, Object val2,
+                                      String name3, Object val3,
+                                      String name4, Object val4,
+                                      String name5, Object val5) {
         return toString(cls,
-            obj,
-            name0, val0, false,
-            name1, val1, false,
-            name2, val2, false,
-            name3, val3, false,
-            name4, val4, false,
-            name5, val5, false);
+                obj,
+                name0, val0, false,
+                name1, val1, false,
+                name2, val2, false,
+                name3, val3, false,
+                name4, val4, false,
+                name5, val5, false);
     }
 
     /**
@@ -202,22 +215,22 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0,
-        String name1, Object val1,
-        String name2, Object val2,
-        String name3, Object val3,
-        String name4, Object val4,
-        String name5, Object val5,
-        String name6, Object val6) {
+                                      String name0, Object val0,
+                                      String name1, Object val1,
+                                      String name2, Object val2,
+                                      String name3, Object val3,
+                                      String name4, Object val4,
+                                      String name5, Object val5,
+                                      String name6, Object val6) {
         return toString(cls,
-            obj,
-            name0, val0, false,
-            name1, val1, false,
-            name2, val2, false,
-            name3, val3, false,
-            name4, val4, false,
-            name5, val5, false,
-            name6, val6, false);
+                obj,
+                name0, val0, false,
+                name1, val1, false,
+                name2, val2, false,
+                name3, val3, false,
+                name4, val4, false,
+                name5, val5, false,
+                name6, val6, false);
     }
 
     /**
@@ -244,11 +257,11 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0, boolean sens0,
-        String name1, Object val1, boolean sens1,
-        String name2, Object val2, boolean sens2,
-        String name3, Object val3, boolean sens3,
-        String name4, Object val4, boolean sens4) {
+                                      String name0, Object val0, boolean sens0,
+                                      String name1, Object val1, boolean sens1,
+                                      String name2, Object val2, boolean sens2,
+                                      String name3, Object val3, boolean sens3,
+                                      String name4, Object val4, boolean sens4) {
         assert cls != null;
         assert obj != null;
         assert name0 != null;
@@ -257,18 +270,9 @@ public static <T> String toString(Class<T> cls, T obj,
         assert name3 != null;
         assert name4 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] addNames = tmp.getAdditionalNames();
-        Object[] addVals = tmp.getAdditionalValues();
-        boolean[] addSens = tmp.getAdditionalSensitives();
+        Object[] addNames = new Object[5];
+        Object[] addVals = new Object[5];
+        boolean[] addSens = new boolean[5];
 
         addNames[0] = name0;
         addVals[0] = val0;
@@ -286,20 +290,16 @@ public static <T> String toString(Class<T> cls, T obj,
         addVals[4] = val4;
         addSens[4] = sens4;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 5);
+            return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 5);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -330,12 +330,12 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0, boolean sens0,
-        String name1, Object val1, boolean sens1,
-        String name2, Object val2, boolean sens2,
-        String name3, Object val3, boolean sens3,
-        String name4, Object val4, boolean sens4,
-        String name5, Object val5, boolean sens5) {
+                                      String name0, Object val0, boolean sens0,
+                                      String name1, Object val1, boolean sens1,
+                                      String name2, Object val2, boolean sens2,
+                                      String name3, Object val3, boolean sens3,
+                                      String name4, Object val4, boolean sens4,
+                                      String name5, Object val5, boolean sens5) {
         assert cls != null;
         assert obj != null;
         assert name0 != null;
@@ -345,18 +345,9 @@ public static <T> String toString(Class<T> cls, T obj,
         assert name4 != null;
         assert name5 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] addNames = tmp.getAdditionalNames();
-        Object[] addVals = tmp.getAdditionalValues();
-        boolean[] addSens = tmp.getAdditionalSensitives();
+        Object[] addNames = new Object[6];
+        Object[] addVals = new Object[6];
+        boolean[] addSens = new boolean[6];
 
         addNames[0] = name0;
         addVals[0] = val0;
@@ -377,20 +368,16 @@ public static <T> String toString(Class<T> cls, T obj,
         addVals[5] = val5;
         addSens[5] = sens5;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 6);
+            return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 6);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -424,13 +411,13 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0, boolean sens0,
-        String name1, Object val1, boolean sens1,
-        String name2, Object val2, boolean sens2,
-        String name3, Object val3, boolean sens3,
-        String name4, Object val4, boolean sens4,
-        String name5, Object val5, boolean sens5,
-        String name6, Object val6, boolean sens6) {
+                                      String name0, Object val0, boolean sens0,
+                                      String name1, Object val1, boolean sens1,
+                                      String name2, Object val2, boolean sens2,
+                                      String name3, Object val3, boolean sens3,
+                                      String name4, Object val4, boolean sens4,
+                                      String name5, Object val5, boolean sens5,
+                                      String name6, Object val6, boolean sens6) {
         assert cls != null;
         assert obj != null;
         assert name0 != null;
@@ -441,18 +428,9 @@ public static <T> String toString(Class<T> cls, T obj,
         assert name5 != null;
         assert name6 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] addNames = tmp.getAdditionalNames();
-        Object[] addVals = tmp.getAdditionalValues();
-        boolean[] addSens = tmp.getAdditionalSensitives();
+        Object[] addNames = new Object[7];
+        Object[] addVals = new Object[7];
+        boolean[] addSens = new boolean[7];
 
         addNames[0] = name0;
         addVals[0] = val0;
@@ -476,20 +454,16 @@ public static <T> String toString(Class<T> cls, T obj,
         addVals[6] = val6;
         addSens[6] = sens6;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 7);
+            return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 7);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -510,15 +484,15 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0,
-        String name1, Object val1,
-        String name2, Object val2,
-        String name3, Object val3) {
+                                      String name0, Object val0,
+                                      String name1, Object val1,
+                                      String name2, Object val2,
+                                      String name3, Object val3) {
         return toString(cls, obj,
-            name0, val0, false,
-            name1, val1, false,
-            name2, val2, false,
-            name3, val3, false);
+                name0, val0, false,
+                name1, val1, false,
+                name2, val2, false,
+                name3, val3, false);
     }
 
     /**
@@ -542,10 +516,10 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0, boolean sens0,
-        String name1, Object val1, boolean sens1,
-        String name2, Object val2, boolean sens2,
-        String name3, Object val3, boolean sens3) {
+                                      String name0, Object val0, boolean sens0,
+                                      String name1, Object val1, boolean sens1,
+                                      String name2, Object val2, boolean sens2,
+                                      String name3, Object val3, boolean sens3) {
         assert cls != null;
         assert obj != null;
         assert name0 != null;
@@ -553,18 +527,9 @@ public static <T> String toString(Class<T> cls, T obj,
         assert name2 != null;
         assert name3 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] addNames = tmp.getAdditionalNames();
-        Object[] addVals = tmp.getAdditionalValues();
-        boolean[] addSens = tmp.getAdditionalSensitives();
+        Object[] addNames = new Object[4];
+        Object[] addVals = new Object[4];
+        boolean[] addSens = new boolean[4];
 
         addNames[0] = name0;
         addVals[0] = val0;
@@ -579,20 +544,16 @@ public static <T> String toString(Class<T> cls, T obj,
         addVals[3] = val3;
         addSens[3] = sens3;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 4);
+            return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 4);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -611,14 +572,14 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0,
-        String name1, Object val1,
-        String name2, Object val2) {
+                                      String name0, Object val0,
+                                      String name1, Object val1,
+                                      String name2, Object val2) {
         return toString(cls,
-            obj,
-            name0, val0, false,
-            name1, val1, false,
-            name2, val2, false);
+                obj,
+                name0, val0, false,
+                name1, val1, false,
+                name2, val2, false);
     }
 
     /**
@@ -639,27 +600,18 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0, boolean sens0,
-        String name1, Object val1, boolean sens1,
-        String name2, Object val2, boolean sens2) {
+                                      String name0, Object val0, boolean sens0,
+                                      String name1, Object val1, boolean sens1,
+                                      String name2, Object val2, boolean sens2) {
         assert cls != null;
         assert obj != null;
         assert name0 != null;
         assert name1 != null;
         assert name2 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] addNames = tmp.getAdditionalNames();
-        Object[] addVals = tmp.getAdditionalValues();
-        boolean[] addSens = tmp.getAdditionalSensitives();
+        Object[] addNames = new Object[3];
+        Object[] addVals = new Object[3];
+        boolean[] addSens = new boolean[3];
 
         addNames[0] = name0;
         addVals[0] = val0;
@@ -671,20 +623,16 @@ public static <T> String toString(Class<T> cls, T obj,
         addVals[2] = val2;
         addSens[2] = sens2;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 3);
+            return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 3);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -701,8 +649,8 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0,
-        String name1, Object val1) {
+                                      String name0, Object val0,
+                                      String name1, Object val1) {
         return toString(cls, obj, name0, val0, false, name1, val1, false);
     }
 
@@ -721,25 +669,16 @@ public static <T> String toString(Class<T> cls, T obj,
      * @return String presentation of the given object.
      */
     public static <T> String toString(Class<T> cls, T obj,
-        String name0, Object val0, boolean sens0,
-        String name1, Object val1, boolean sens1) {
+                                      String name0, Object val0, boolean sens0,
+                                      String name1, Object val1, boolean sens1) {
         assert cls != null;
         assert obj != null;
         assert name0 != null;
         assert name1 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] addNames = tmp.getAdditionalNames();
-        Object[] addVals = tmp.getAdditionalValues();
-        boolean[] addSens = tmp.getAdditionalSensitives();
+        Object[] addNames = new Object[2];
+        Object[] addVals = new Object[2];
+        boolean[] addSens = new boolean[2];
 
         addNames[0] = name0;
         addVals[0] = val0;
@@ -748,20 +687,16 @@ public static <T> String toString(Class<T> cls, T obj,
         addVals[1] = val1;
         addSens[1] = sens1;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 2);
+            return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 2);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -795,37 +730,24 @@ public static <T> String toString(Class<T> cls, T obj, String name, @Nullable Ob
         assert obj != null;
         assert name != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] addNames = tmp.getAdditionalNames();
-        Object[] addVals = tmp.getAdditionalValues();
-        boolean[] addSens = tmp.getAdditionalSensitives();
+        Object[] addNames = new Object[1];
+        Object[] addVals = new Object[1];
+        boolean[] addSens = new boolean[1];
 
         addNames[0] = name;
         addVals[0] = val;
         addSens[0] = sens;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, addNames, addVals, addSens, 1);
+            return toStringImpl(cls, sb, obj, addNames, addVals, addSens, 1);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -841,30 +763,16 @@ public static <T> String toString(Class<T> cls, T obj) {
         assert cls != null;
         assert obj != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        SBLengthLimit lenLim = threadCurLen.get();
-
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(cls, tmp.getStringBuilder(lenLim), obj, tmp.getAdditionalNames(),
-                tmp.getAdditionalValues(), null, 0);
+            return toStringImpl(cls, sb, obj, EMPTY_ARRAY, EMPTY_ARRAY, null, 0);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -882,68 +790,164 @@ public static <T> String toString(Class<T> cls, T obj, String parent) {
     }
 
     /**
-     * Print value with length limitation
+     * Print value with length limitation.
+     *
      * @param buf buffer to print to.
      * @param val value to print, can be {@code null}.
      */
     private static void toString(SBLimitedLength buf, Object val) {
-        if (val == null)
-            buf.a("null");
-        else
-            toString(buf, val.getClass(), val);
+        toString(buf, null, val);
     }
 
     /**
-     * Print value with length limitation
+     * Print value with length limitation.
+     *
      * @param buf buffer to print to.
-     * @param valClass value class.
-     * @param val value to print
+     * @param cls value class.
+     * @param val value to print.
      */
-    private static void toString(SBLimitedLength buf, Class<?> valClass, Object val) {
-        if (valClass.isArray())
-            buf.a(arrayToString(valClass, val));
-        else {
-            int overflow = 0;
-            char bracket = ' ';
-
-            if (val instanceof Collection && ((Collection)val).size() > COLLECTION_LIMIT) {
-                overflow = ((Collection)val).size() - COLLECTION_LIMIT;
-                bracket = ']';
-                val = F.retain((Collection) val, true, COLLECTION_LIMIT);
-            }
-            else if (val instanceof Map && ((Map)val).size() > COLLECTION_LIMIT) {
-                Map<Object, Object> tmp = U.newHashMap(COLLECTION_LIMIT);
+    @SuppressWarnings({"unchecked"})
+    private static void toString(SBLimitedLength buf, Class<?> cls, Object val) {
+        if (val == null) {
+            buf.a("null");
 
-                overflow = ((Map)val).size() - COLLECTION_LIMIT;
+            return;
+        }
 
-                bracket= '}';
+        if (cls == null)
+            cls = val.getClass();
 
-                int cntr = 0;
+        if (cls.isPrimitive()) {
+            buf.a(val);
 
-                for (Object o : ((Map)val).entrySet()) {
-                    Map.Entry e = (Map.Entry)o;
+            return;
+        }
 
-                    tmp.put(e.getKey(), e.getValue());
+        IdentityHashMap<Object, Integer> svdObjs = savedObjects.get();
 
-                    if (++cntr >= COLLECTION_LIMIT)
-                        break;
-                }
+        if (handleRecursion(buf, val, svdObjs))
+            return;
 
-                val = tmp;
-            }
+        svdObjs.put(val, buf.length());
 
-            buf.a(val);
+        try {
+            if (cls.isArray())
+                addArray(buf, cls, val);
+            else if (val instanceof Collection)
+                addCollection(buf, (Collection) val);
+            else if (val instanceof Map)
+                addMap(buf, (Map<?, ?>) val);
+            else
+                buf.a(val);
+        }
+        finally {
+            svdObjs.remove(val);
+        }
+    }
 
-            if (overflow > 0) {
-                buf.d(buf.length() - 1);
-                buf.a("... and ").a(overflow).a(" more").a(bracket);
-            }
+    /**
+     * Writes array to buffer.
+     *
+     * @param buf String builder buffer.
+     * @param arrType Type of the array.
+     * @param obj Array object.
+     */
+    private static void addArray(SBLimitedLength buf, Class arrType, Object obj) {
+        if (arrType.getComponentType().isPrimitive()) {
+            buf.a(arrayToString(obj));
+
+            return;
         }
+
+        Object[] arr = (Object[]) obj;
+
+        buf.a(arrType.getSimpleName()).a(" [");
+
+        for (int i = 0; i < arr.length; i++) {
+            toString(buf, arr[i]);
+
+            if (i == COLLECTION_LIMIT - 1 || i == arr.length - 1)
+                break;
+
+            buf.a(", ");
+        }
+
+        handleOverflow(buf, arr.length);
+
+        buf.a(']');
+    }
+
+    /**
+     * Writes collection to buffer.
+     *
+     * @param buf String builder buffer.
+     * @param col Collection object.
+     */
+    private static void addCollection(SBLimitedLength buf, Collection col) {
+        buf.a(col.getClass().getSimpleName()).a(" [");
+
+        int cnt = 0;
+
+        for (Object obj : col) {
+            toString(buf, obj);
+
+            if (++cnt == COLLECTION_LIMIT || cnt == col.size())
+                break;
+
+            buf.a(", ");
+        }
+
+        handleOverflow(buf, col.size());
+
+        buf.a(']');
+    }
+
+    /**
+     * Writes map to buffer.
+     *
+     * @param buf String builder buffer.
+     * @param map Map object.
+     */
+    private static <K, V> void addMap(SBLimitedLength buf, Map<K, V> map) {
+        buf.a(map.getClass().getSimpleName()).a(" {");
+
+        int cnt = 0;
+
+        for (Map.Entry<K, V> e : map.entrySet()) {
+            toString(buf, e.getKey());
+
+            buf.a('=');
+
+            toString(buf, e.getValue());
+
+            if (++cnt == COLLECTION_LIMIT || cnt == map.size())
+                break;
+
+            buf.a(", ");
+        }
+
+        handleOverflow(buf, map.size());
+
+        buf.a('}');
+    }
+
+    /**
+     * Writes overflow message to buffer if needed.
+     *
+     * @param buf String builder buffer.
+     * @param size Size to compare with limit.
+     */
+    private static void handleOverflow(SBLimitedLength buf, int size) {
+        int overflow = size - COLLECTION_LIMIT;
+
+        if (overflow > 0)
+            buf.a("... and ").a(overflow).a(" more");
     }
 
     /**
      * Creates an uniformed string presentation for the given object.
      *
+     * @param <T> Type of object.
      * @param cls Class of the object.
      * @param buf String builder buffer.
      * @param obj Object for which to get string presentation.
@@ -952,17 +956,15 @@ else if (val instanceof Map && ((Map)val).size() > COLLECTION_LIMIT) {
      * @param addSens Sensitive flag of values or {@code null} if all values are not sensitive.
      * @param addLen How many additional values will be included.
      * @return String presentation of the given object.
-     * @param <T> Type of object.
      */
-    @SuppressWarnings({"unchecked"})
     private static <T> String toStringImpl(
-        Class<T> cls,
-        SBLimitedLength buf,
-        T obj,
-        Object[] addNames,
-        Object[] addVals,
-        @Nullable boolean[] addSens,
-        int addLen) {
+            Class<T> cls,
+            SBLimitedLength buf,
+            T obj,
+            Object[] addNames,
+            Object[] addVals,
+            @Nullable boolean[] addSens,
+            int addLen) {
         assert cls != null;
         assert buf != null;
         assert obj != null;
@@ -971,13 +973,56 @@ private static <T> String toStringImpl(
         assert addNames.length == addVals.length;
         assert addLen <= addNames.length;
 
+        boolean newStr = buf.length() == 0;
+
+        IdentityHashMap<Object, Integer> svdObjs = savedObjects.get();
+
+        if (newStr)
+            svdObjs.put(obj, buf.length());
+
+        try {
+            String s = toStringImpl0(cls, buf, obj, addNames, addVals, addSens, addLen);
+
+            if (newStr)
+                return s;
+
+            // Called from another GTSB.toString(), so this string is already in the buffer and shouldn't be returned.
+            return "";
+        }
+        finally {
+            if (newStr)
+                svdObjs.remove(obj);
+        }
+    }
+
+    /**
+     * Creates an uniformed string presentation for the given object.
+     *
+     * @param cls Class of the object.
+     * @param buf String builder buffer.
+     * @param obj Object for which to get string presentation.
+     * @param addNames Names of additional values to be included.
+     * @param addVals Additional values to be included.
+     * @param addSens Sensitive flag of values or {@code null} if all values are not sensitive.
+     * @param addLen How many additional values will be included.
+     * @return String presentation of the given object.
+     * @param <T> Type of object.
+     */
+    @SuppressWarnings({"unchecked"})
+    private static <T> String toStringImpl0(
+            Class<T> cls,
+            SBLimitedLength buf,
+            T obj,
+            Object[] addNames,
+            Object[] addVals,
+            @Nullable boolean[] addSens,
+            int addLen
+    ) {
         try {
             GridToStringClassDescriptor cd = getClassDescriptor(cls);
 
             assert cd != null;
 
-            buf.setLength(0);
-
             buf.a(cd.getSimpleClassName()).a(" [");
 
             boolean first = true;
@@ -1009,7 +1054,6 @@ private static <T> String toStringImpl(
         }
         // Specifically catching all exceptions.
         catch (Exception e) {
-
             // Remove entry from cache to avoid potential memory leak
             // in case new class loader got loaded under the same identity hash.
             classCache.remove(cls.getName() + System.identityHashCode(cls.getClassLoader()));
@@ -1032,95 +1076,40 @@ public static String toString(String str, String name, @Nullable Object val) {
     }
 
     /**
-     * @param arrType Type of the array.
-     * @param arr Array object.
+     * Returns limited string representation of array.
+     *
+     * @param arr Array object. Each value is automatically wrapped if it has a primitive type.
      * @return String representation of an array.
      */
-    @SuppressWarnings({"ConstantConditions", "unchecked"})
-    public static <T> String arrayToString(Class arrType, Object arr) {
+    public static String arrayToString(Object arr) {
         if (arr == null)
             return "null";
 
         String res;
-        int more = 0;
 
-        if (arrType.equals(byte[].class)) {
-            byte[] byteArr = (byte[])arr;
-            if (byteArr.length > COLLECTION_LIMIT) {
-                more = byteArr.length - COLLECTION_LIMIT;
-                byteArr = Arrays.copyOf(byteArr, COLLECTION_LIMIT);
-            }
-            res = Arrays.toString(byteArr);
-        }
-        else if (arrType.equals(boolean[].class)) {
-            boolean[] boolArr = (boolean[])arr;
-            if (boolArr.length > COLLECTION_LIMIT) {
-                more = boolArr.length - COLLECTION_LIMIT;
-                boolArr = Arrays.copyOf(boolArr, COLLECTION_LIMIT);
-            }
-            res = Arrays.toString(boolArr);
-        }
-        else if (arrType.equals(short[].class)) {
-            short[] shortArr = (short[])arr;
-            if (shortArr.length > COLLECTION_LIMIT) {
-                more = shortArr.length - COLLECTION_LIMIT;
-                shortArr = Arrays.copyOf(shortArr, COLLECTION_LIMIT);
-            }
-            res = Arrays.toString(shortArr);
-        }
-        else if (arrType.equals(int[].class)) {
-            int[] intArr = (int[])arr;
-            if (intArr.length > COLLECTION_LIMIT) {
-                more = intArr.length - COLLECTION_LIMIT;
-                intArr = Arrays.copyOf(intArr, COLLECTION_LIMIT);
-            }
-            res = Arrays.toString(intArr);
-        }
-        else if (arrType.equals(long[].class)) {
-            long[] longArr = (long[])arr;
-            if (longArr.length > COLLECTION_LIMIT) {
-                more = longArr.length - COLLECTION_LIMIT;
-                longArr = Arrays.copyOf(longArr, COLLECTION_LIMIT);
-            }
-            res = Arrays.toString(longArr);
-        }
-        else if (arrType.equals(float[].class)) {
-            float[] floatArr = (float[])arr;
-            if (floatArr.length > COLLECTION_LIMIT) {
-                more = floatArr.length - COLLECTION_LIMIT;
-                floatArr = Arrays.copyOf(floatArr, COLLECTION_LIMIT);
-            }
-            res = Arrays.toString(floatArr);
-        }
-        else if (arrType.equals(double[].class)) {
-            double[] doubleArr = (double[])arr;
-            if (doubleArr.length > COLLECTION_LIMIT) {
-                more = doubleArr.length - COLLECTION_LIMIT;
-                doubleArr = Arrays.copyOf(doubleArr, COLLECTION_LIMIT);
-            }
-            res = Arrays.toString(doubleArr);
-        }
-        else if (arrType.equals(char[].class)) {
-            char[] charArr = (char[])arr;
-            if (charArr.length > COLLECTION_LIMIT) {
-                more = charArr.length - COLLECTION_LIMIT;
-                charArr = Arrays.copyOf(charArr, COLLECTION_LIMIT);
-            }
-            res = Arrays.toString(charArr);
-        }
-        else {
+        int arrLen;
+
+        if (arr instanceof Object[]) {
             Object[] objArr = (Object[])arr;
-            if (objArr.length > COLLECTION_LIMIT) {
-                more = objArr.length - COLLECTION_LIMIT;
+
+            arrLen = objArr.length;
+
+            if (arrLen > COLLECTION_LIMIT)
                 objArr = Arrays.copyOf(objArr, COLLECTION_LIMIT);
-            }
+
             res = Arrays.toString(objArr);
+        } else {
+            res = toStringWithLimit(arr, COLLECTION_LIMIT);
+
+            arrLen = Array.getLength(arr);
         }
-        if (more > 0) {
+
+        if (arrLen > COLLECTION_LIMIT) {
             StringBuilder resSB = new StringBuilder(res);
 
             resSB.deleteCharAt(resSB.length() - 1);
-            resSB.append("... and ").append(more).append(" more]");
+
+            resSB.append("... and ").append(arrLen - COLLECTION_LIMIT).append(" more]");
 
             res = resSB.toString();
         }
@@ -1128,6 +1117,37 @@ else if (arrType.equals(char[].class)) {
         return res;
     }
 
+    /**
+     * Returns limited string representation of array.
+     *
+     * @param arr Input array. Each value is automatically wrapped if it has a primitive type.
+     * @param limit max array items to string limit.
+     * @return String representation of an array.
+     */
+    private static String toStringWithLimit(Object arr, int limit) {
+        int arrIdxMax = Array.getLength(arr) - 1;
+
+        if (arrIdxMax == -1)
+            return "[]";
+
+        int idxMax = Math.min(arrIdxMax, limit);
+
+        StringBuilder b = new StringBuilder();
+
+        b.append('[');
+
+        for (int i = 0; i <= idxMax; ++i) {
+            b.append(Array.get(arr, i));
+
+            if (i == idxMax)
+                return b.append(']').toString();
+
+            b.append(", ");
+        }
+
+        return b.toString();
+    }
+
     /**
      * Produces uniformed output of string with context properties
      *
@@ -1140,37 +1160,24 @@ else if (arrType.equals(char[].class)) {
     public static String toString(String str, String name, @Nullable Object val, boolean sens) {
         assert name != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] propNames = tmp.getAdditionalNames();
-        Object[] propVals = tmp.getAdditionalValues();
-        boolean[] propSens = tmp.getAdditionalSensitives();
+        Object[] propNames = new Object[1];
+        Object[] propVals = new Object[1];
+        boolean[] propSens = new boolean[1];
 
         propNames[0] = name;
         propVals[0] = val;
         propSens[0] = sens;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 1);
+            return toStringImpl(str, sb, propNames, propVals, propSens, 1);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -1185,7 +1192,7 @@ public static String toString(String str, String name, @Nullable Object val, boo
      * @return String presentation.
      */
     public static String toString(String str, String name0, @Nullable Object val0, String name1,
-        @Nullable Object val1) {
+                                  @Nullable Object val1) {
         return toString(str, name0, val0, false, name1, val1, false);
     }
 
@@ -1202,23 +1209,14 @@ public static String toString(String str, String name0, @Nullable Object val0, S
      * @return String presentation.
      */
     public static String toString(String str,
-        String name0, @Nullable Object val0, boolean sens0,
-        String name1, @Nullable Object val1, boolean sens1) {
+                                  String name0, @Nullable Object val0, boolean sens0,
+                                  String name1, @Nullable Object val1, boolean sens1) {
         assert name0 != null;
         assert name1 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] propNames = tmp.getAdditionalNames();
-        Object[] propVals = tmp.getAdditionalValues();
-        boolean[] propSens = tmp.getAdditionalSensitives();
+        Object[] propNames = new Object[2];
+        Object[] propVals = new Object[2];
+        boolean[] propSens = new boolean[2];
 
         propNames[0] = name0;
         propVals[0] = val0;
@@ -1227,20 +1225,16 @@ public static String toString(String str,
         propVals[1] = val1;
         propSens[1] = sens1;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 2);
+            return toStringImpl(str, sb, propNames, propVals, propSens, 2);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -1260,25 +1254,16 @@ public static String toString(String str,
      * @return String presentation.
      */
     public static String toString(String str,
-        String name0, @Nullable Object val0, boolean sens0,
-        String name1, @Nullable Object val1, boolean sens1,
-        String name2, @Nullable Object val2, boolean sens2) {
+                                  String name0, @Nullable Object val0, boolean sens0,
+                                  String name1, @Nullable Object val1, boolean sens1,
+                                  String name2, @Nullable Object val2, boolean sens2) {
         assert name0 != null;
         assert name1 != null;
         assert name2 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] propNames = tmp.getAdditionalNames();
-        Object[] propVals = tmp.getAdditionalValues();
-        boolean[] propSens = tmp.getAdditionalSensitives();
+        Object[] propNames = new Object[3];
+        Object[] propVals = new Object[3];
+        boolean[] propSens = new boolean[3];
 
         propNames[0] = name0;
         propVals[0] = val0;
@@ -1290,20 +1275,16 @@ public static String toString(String str,
         propVals[2] = val2;
         propSens[2] = sens2;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 3);
+            return toStringImpl(str, sb, propNames, propVals, propSens, 3);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -1326,27 +1307,18 @@ public static String toString(String str,
      * @return String presentation.
      */
     public static String toString(String str,
-        String name0, @Nullable Object val0, boolean sens0,
-        String name1, @Nullable Object val1, boolean sens1,
-        String name2, @Nullable Object val2, boolean sens2,
-        String name3, @Nullable Object val3, boolean sens3) {
+                                  String name0, @Nullable Object val0, boolean sens0,
+                                  String name1, @Nullable Object val1, boolean sens1,
+                                  String name2, @Nullable Object val2, boolean sens2,
+                                  String name3, @Nullable Object val3, boolean sens3) {
         assert name0 != null;
         assert name1 != null;
         assert name2 != null;
         assert name3 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] propNames = tmp.getAdditionalNames();
-        Object[] propVals = tmp.getAdditionalValues();
-        boolean[] propSens = tmp.getAdditionalSensitives();
+        Object[] propNames = new Object[4];
+        Object[] propVals = new Object[4];
+        boolean[] propSens = new boolean[4];
 
         propNames[0] = name0;
         propVals[0] = val0;
@@ -1361,20 +1333,16 @@ public static String toString(String str,
         propVals[3] = val3;
         propSens[3] = sens3;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 4);
+            return toStringImpl(str, sb, propNames, propVals, propSens, 4);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -1400,29 +1368,20 @@ public static String toString(String str,
      * @return String presentation.
      */
     public static String toString(String str,
-        String name0, @Nullable Object val0, boolean sens0,
-        String name1, @Nullable Object val1, boolean sens1,
-        String name2, @Nullable Object val2, boolean sens2,
-        String name3, @Nullable Object val3, boolean sens3,
-        String name4, @Nullable Object val4, boolean sens4) {
+                                  String name0, @Nullable Object val0, boolean sens0,
+                                  String name1, @Nullable Object val1, boolean sens1,
+                                  String name2, @Nullable Object val2, boolean sens2,
+                                  String name3, @Nullable Object val3, boolean sens3,
+                                  String name4, @Nullable Object val4, boolean sens4) {
         assert name0 != null;
         assert name1 != null;
         assert name2 != null;
         assert name3 != null;
         assert name4 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] propNames = tmp.getAdditionalNames();
-        Object[] propVals = tmp.getAdditionalValues();
-        boolean[] propSens = tmp.getAdditionalSensitives();
+        Object[] propNames = new Object[5];
+        Object[] propVals = new Object[5];
+        boolean[] propSens = new boolean[5];
 
         propNames[0] = name0;
         propVals[0] = val0;
@@ -1440,20 +1399,16 @@ public static String toString(String str,
         propVals[4] = val4;
         propSens[4] = sens4;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 5);
+            return toStringImpl(str, sb, propNames, propVals, propSens, 5);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -1482,12 +1437,12 @@ public static String toString(String str,
      * @return String presentation.
      */
     public static String toString(String str,
-        String name0, @Nullable Object val0, boolean sens0,
-        String name1, @Nullable Object val1, boolean sens1,
-        String name2, @Nullable Object val2, boolean sens2,
-        String name3, @Nullable Object val3, boolean sens3,
-        String name4, @Nullable Object val4, boolean sens4,
-        String name5, @Nullable Object val5, boolean sens5) {
+                                  String name0, @Nullable Object val0, boolean sens0,
+                                  String name1, @Nullable Object val1, boolean sens1,
+                                  String name2, @Nullable Object val2, boolean sens2,
+                                  String name3, @Nullable Object val3, boolean sens3,
+                                  String name4, @Nullable Object val4, boolean sens4,
+                                  String name5, @Nullable Object val5, boolean sens5) {
         assert name0 != null;
         assert name1 != null;
         assert name2 != null;
@@ -1495,18 +1450,9 @@ public static String toString(String str,
         assert name4 != null;
         assert name5 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] propNames = tmp.getAdditionalNames();
-        Object[] propVals = tmp.getAdditionalValues();
-        boolean[] propSens = tmp.getAdditionalSensitives();
+        Object[] propNames = new Object[6];
+        Object[] propVals = new Object[6];
+        boolean[] propSens = new boolean[6];
 
         propNames[0] = name0;
         propVals[0] = val0;
@@ -1527,20 +1473,16 @@ public static String toString(String str,
         propVals[5] = val5;
         propSens[5] = sens5;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 6);
+            return toStringImpl(str, sb, propNames, propVals, propSens, 6);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -1572,13 +1514,13 @@ public static String toString(String str,
      * @return String presentation.
      */
     public static String toString(String str,
-        String name0, @Nullable Object val0, boolean sens0,
-        String name1, @Nullable Object val1, boolean sens1,
-        String name2, @Nullable Object val2, boolean sens2,
-        String name3, @Nullable Object val3, boolean sens3,
-        String name4, @Nullable Object val4, boolean sens4,
-        String name5, @Nullable Object val5, boolean sens5,
-        String name6, @Nullable Object val6, boolean sens6) {
+                                  String name0, @Nullable Object val0, boolean sens0,
+                                  String name1, @Nullable Object val1, boolean sens1,
+                                  String name2, @Nullable Object val2, boolean sens2,
+                                  String name3, @Nullable Object val3, boolean sens3,
+                                  String name4, @Nullable Object val4, boolean sens4,
+                                  String name5, @Nullable Object val5, boolean sens5,
+                                  String name6, @Nullable Object val6, boolean sens6) {
         assert name0 != null;
         assert name1 != null;
         assert name2 != null;
@@ -1587,18 +1529,9 @@ public static String toString(String str,
         assert name5 != null;
         assert name6 != null;
 
-        Queue<GridToStringThreadLocal> queue = threadCache.get();
-
-        assert queue != null;
-
-        // Since string() methods can be chain-called from the same thread we
-        // have to keep a list of thread-local objects and remove/add them
-        // in each string() apply.
-        GridToStringThreadLocal tmp = queue.isEmpty() ? new GridToStringThreadLocal() : queue.remove();
-
-        Object[] propNames = tmp.getAdditionalNames();
-        Object[] propVals = tmp.getAdditionalValues();
-        boolean[] propSens = tmp.getAdditionalSensitives();
+        Object[] propNames = new Object[7];
+        Object[] propVals = new Object[7];
+        boolean[] propSens = new boolean[7];
 
         propNames[0] = name0;
         propVals[0] = val0;
@@ -1622,20 +1555,16 @@ public static String toString(String str,
         propVals[6] = val6;
         propSens[6] = sens6;
 
-        SBLengthLimit lenLim = threadCurLen.get();
+        SBLimitedLength sb = threadLocSB.get();
 
-        boolean newStr = false;
+        boolean newStr = sb.length() == 0;
 
         try {
-            newStr = lenLim.length() == 0;
-
-            return toStringImpl(str, tmp.getStringBuilder(lenLim), propNames, propVals, propSens, 7);
+            return toStringImpl(str, sb, propNames, propVals, propSens, 7);
         }
         finally {
-            queue.offer(tmp);
-
             if (newStr)
-                lenLim.reset();
+                sb.reset();
         }
     }
 
@@ -1651,9 +1580,9 @@ public static String toString(String str,
      * @return String presentation of the object.
      */
     private static String toStringImpl(String str, SBLimitedLength buf, Object[] propNames, Object[] propVals,
-        boolean[] propSens, int propCnt) {
+                                       boolean[] propSens, int propCnt) {
 
-        buf.setLength(0);
+        boolean newStr = buf.length() == 0;
 
         if (str != null)
             buf.a(str).a(" ");
@@ -1664,7 +1593,11 @@ private static String toStringImpl(String str, SBLimitedLength buf, Object[] pro
 
         buf.a(']');
 
-        return buf.toString();
+        if (newStr)
+            return buf.toString();
+
+        // Called from another GTSB.toString(), so this string is already in the buffer and shouldn't be returned.
+        return "";
     }
 
     /**
@@ -1678,11 +1611,11 @@ private static String toStringImpl(String str, SBLimitedLength buf, Object[] pro
      * @param addLen How many additional values will be included.
      */
     private static void appendVals(SBLimitedLength buf,
-        boolean first,
-        Object[] addNames,
-        Object[] addVals,
-        boolean[] addSens,
-        int addLen)
+                                   boolean first,
+                                   Object[] addNames,
+                                   Object[] addVals,
+                                   boolean[] addSens,
+                                   int addLen)
     {
         if (addLen > 0) {
             for (int i = 0; i < addLen; i++) {
@@ -1743,31 +1676,31 @@ private static <T> GridToStringClassDescriptor getClassDescriptor(Class<T> cls)
                     add = notSens || INCLUDE_SENSITIVE;
                 }
                 else if (!f.isAnnotationPresent(GridToStringExclude.class) &&
-                    !f.getType().isAnnotationPresent(GridToStringExclude.class)) {
+                        !f.getType().isAnnotationPresent(GridToStringExclude.class)) {
                     if (
                         // Include only private non-static
-                        Modifier.isPrivate(f.getModifiers()) && !Modifier.isStatic(f.getModifiers()) &&
-
-                        // No direct objects & serializable.
-                        Object.class != type &&
-                        Serializable.class != type &&
-                        Externalizable.class != type &&
-
-                        // No arrays.
-                        !type.isArray() &&
-
-                        // Exclude collections, IO, etc.
-                        !EventListener.class.isAssignableFrom(type) &&
-                        !Map.class.isAssignableFrom(type) &&
-                        !Collection.class.isAssignableFrom(type) &&
-                        !InputStream.class.isAssignableFrom(type) &&
-                        !OutputStream.class.isAssignableFrom(type) &&
-                        !Thread.class.isAssignableFrom(type) &&
-                        !Runnable.class.isAssignableFrom(type) &&
-                        !Lock.class.isAssignableFrom(type) &&
-                        !ReadWriteLock.class.isAssignableFrom(type) &&
-                        !Condition.class.isAssignableFrom(type)
-                    )
+                            Modifier.isPrivate(f.getModifiers()) && !Modifier.isStatic(f.getModifiers()) &&
+
+                                    // No direct objects & serializable.
+                                    Object.class != type &&
+                                    Serializable.class != type &&
+                                    Externalizable.class != type &&
+
+                                    // No arrays.
+                                    !type.isArray() &&
+
+                                    // Exclude collections, IO, etc.
+                                    !EventListener.class.isAssignableFrom(type) &&
+                                    !Map.class.isAssignableFrom(type) &&
+                                    !Collection.class.isAssignableFrom(type) &&
+                                    !InputStream.class.isAssignableFrom(type) &&
+                                    !OutputStream.class.isAssignableFrom(type) &&
+                                    !Thread.class.isAssignableFrom(type) &&
+                                    !Runnable.class.isAssignableFrom(type) &&
+                                    !Lock.class.isAssignableFrom(type) &&
+                                    !ReadWriteLock.class.isAssignableFrom(type) &&
+                                    !Condition.class.isAssignableFrom(type)
+                            )
                         add = true;
                 }
 
@@ -1790,4 +1723,101 @@ else if (!f.isAnnotationPresent(GridToStringExclude.class) &&
 
         return cd;
     }
+
+    /**
+     * Returns sorted and compacted string representation of given {@code col}.
+     * Two nearby numbers with difference at most 1 are compacted to one continuous segment.
+     * E.g. collection of [1, 2, 3, 5, 6, 7, 10] will be compacted to [1-3, 5-7, 10].
+     *
+     * @param col Collection of integers.
+     * @return Compacted string representation of given collections.
+     */
+    public static String compact(@NotNull Collection<Integer> col) {
+        if (col.isEmpty())
+            return "[]";
+
+        SB sb = new SB();
+        sb.a('[');
+
+        List<Integer> l = new ArrayList<>(col);
+        Collections.sort(l);
+
+        int left = l.get(0), right = left;
+        for (int i = 1; i < l.size(); i++) {
+            int val = l.get(i);
+
+            if (right == val || right + 1 == val) {
+                right = val;
+                continue;
+            }
+
+            if (left == right)
+                sb.a(left);
+            else
+                sb.a(left).a('-').a(right);
+
+            sb.a(',').a(' ');
+
+            left = right = val;
+        }
+
+        if (left == right)
+            sb.a(left);
+        else
+            sb.a(left).a('-').a(right);
+
+        sb.a(']');
+
+        return sb.toString();
+    }
+
+    /**
+     * Checks that object is already saved.
+     * In positive case this method inserts hash to the saved object entry (if needed) and name@hash for current entry.
+     * Further toString operations are not needed for current object.
+     *
+     * @param buf String builder buffer.
+     * @param obj Object.
+     * @param svdObjs Map with saved objects to handle recursion.
+     * @return {@code True} if object is already saved and name@hash was added to buffer.
+     * {@code False} if it wasn't saved previously and it should be saved.
+     */
+    private static boolean handleRecursion(SBLimitedLength buf, Object obj, IdentityHashMap<Object, Integer> svdObjs) {
+        Integer pos = svdObjs.get(obj);
+
+        if (pos == null)
+            return false;
+
+        String name = obj.getClass().getSimpleName();
+        String hash = '@' + Integer.toHexString(System.identityHashCode(obj));
+        String savedName = name + hash;
+
+        if (!buf.isOverflowed() && buf.impl().indexOf(savedName, pos) != pos) {
+            buf.i(pos + name.length(), hash);
+
+            incValues(svdObjs, obj, hash.length());
+        }
+
+        buf.a(savedName);
+
+        return true;
+    }
+
+    /**
+     * Increment positions of already presented objects afterward given object.
+     *
+     * @param svdObjs Map with objects already presented in the buffer.
+     * @param obj Object.
+     * @param hashLen Length of the object's hash.
+     */
+    private static void incValues(IdentityHashMap<Object, Integer> svdObjs, Object obj, int hashLen) {
+        Integer baseline = svdObjs.get(obj);
+
+        for (IdentityHashMap.Entry<Object, Integer> entry : svdObjs.entrySet()) {
+            Integer pos = entry.getValue();
+
+            if (pos > baseline)
+                entry.setValue(pos + hashLen);
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java
index c47cf40c26ef7..640555acc66aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/tostring/SBLimitedLength.java
@@ -50,6 +50,18 @@ void initLimit(SBLengthLimit lenLimit) {
             tail.reset();
     }
 
+    /**
+     * Resets buffer.
+     */
+    public void reset() {
+        super.setLength(0);
+
+        lenLimit.reset();
+
+        if (tail != null)
+            tail.reset();
+    }
+
     /**
      * @return tail string builder.
      */
@@ -292,4 +304,11 @@ private GridStringBuilder onWrite(int lenBeforeWrite) {
             return res.toString();
         }
     }
+
+    /**
+     * @return {@code True} - if buffer limit is reached.
+     */
+    public boolean isOverflowed() {
+        return lenLimit.overflowed(this);
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java
index 4ac05fb64f398..30ca4bc0fdce6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/tostring/GridToStringBuilderSelfTest.java
@@ -20,19 +20,27 @@
 import java.lang.reflect.Array;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.locks.ReadWriteLock;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_COLLECTION_LIMIT;
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_MAX_LENGTH;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_COLLECTION_LIMIT;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_MAX_LENGTH;
+
 /**
  * Tests for {@link GridToStringBuilder}.
  */
@@ -72,32 +80,164 @@ public void testToStringWithAdditions() throws Exception {
     /**
      * @throws Exception If failed.
      */
-    public void testToStringCheckSimpleRecursionPrevention() throws Exception {
+    public void testToStringCheckSimpleListRecursionPrevention() throws Exception {
         ArrayList<Object> list1 = new ArrayList<>();
         ArrayList<Object> list2 = new ArrayList<>();
 
         list2.add(list1);
         list1.add(list2);
 
-
-        GridToStringBuilder.toString(ArrayList.class, list1);
-        GridToStringBuilder.toString(ArrayList.class, list2);
+        info(GridToStringBuilder.toString(ArrayList.class, list1));
+        info(GridToStringBuilder.toString(ArrayList.class, list2));
     }
 
     /**
      * @throws Exception If failed.
      */
-    public void testToStringCheckAdvancedRecursionPrevention() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-602");
+    public void testToStringCheckSimpleMapRecursionPrevention() throws Exception {
+        HashMap<Object, Object> map1 = new HashMap<>();
+        HashMap<Object, Object> map2 = new HashMap<>();
+
+        map1.put("2", map2);
+        map2.put("1", map1);
+
+        info(GridToStringBuilder.toString(HashMap.class, map1));
+        info(GridToStringBuilder.toString(HashMap.class, map2));
+    }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testToStringCheckListAdvancedRecursionPrevention() throws Exception {
         ArrayList<Object> list1 = new ArrayList<>();
         ArrayList<Object> list2 = new ArrayList<>();
 
         list2.add(list1);
         list1.add(list2);
 
-        GridToStringBuilder.toString(ArrayList.class, list1, "name", list2);
-        GridToStringBuilder.toString(ArrayList.class, list2, "name", list1);
+        info(GridToStringBuilder.toString(ArrayList.class, list1, "name", list2));
+        info(GridToStringBuilder.toString(ArrayList.class, list2, "name", list1));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testToStringCheckMapAdvancedRecursionPrevention() throws Exception {
+        HashMap<Object, Object> map1 = new HashMap<>();
+        HashMap<Object, Object> map2 = new HashMap<>();
+
+        map1.put("2", map2);
+        map2.put("1", map1);
+
+        info(GridToStringBuilder.toString(HashMap.class, map1, "name", map2));
+        info(GridToStringBuilder.toString(HashMap.class, map2, "name", map1));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testToStringCheckObjectRecursionPrevention() throws Exception {
+        Node n1 = new Node();
+        Node n2 = new Node();
+        Node n3 = new Node();
+        Node n4 = new Node();
+
+        n1.name = "n1";
+        n2.name = "n2";
+        n3.name = "n3";
+        n4.name = "n4";
+
+        n1.next = n2;
+        n2.next = n3;
+        n3.next = n4;
+        n4.next = n3;
+
+        n1.nodes = new Node[4];
+        n2.nodes = n1.nodes;
+        n3.nodes = n1.nodes;
+        n4.nodes = n1.nodes;
+
+        n1.nodes[0] = n1;
+        n1.nodes[1] = n2;
+        n1.nodes[2] = n3;
+        n1.nodes[3] = n4;
+
+        String expN1 = n1.toString();
+        String expN2 = n2.toString();
+        String expN3 = n3.toString();
+        String expN4 = n4.toString();
+
+        info(expN1);
+        info(expN2);
+        info(expN3);
+        info(expN4);
+        info(GridToStringBuilder.toString("Test", "Appended vals", n1));
+
+        CyclicBarrier bar = new CyclicBarrier(4);
+
+        IgniteInternalFuture<String> fut1 = GridTestUtils.runAsync(new BarrierCallable(bar, n1, expN1));
+        IgniteInternalFuture<String> fut2 = GridTestUtils.runAsync(new BarrierCallable(bar, n2, expN2));
+        IgniteInternalFuture<String> fut3 = GridTestUtils.runAsync(new BarrierCallable(bar, n3, expN3));
+        IgniteInternalFuture<String> fut4 = GridTestUtils.runAsync(new BarrierCallable(bar, n4, expN4));
+
+        fut1.get(3_000);
+        fut2.get(3_000);
+        fut3.get(3_000);
+        fut4.get(3_000);
+    }
+
+    /**
+     * Test class.
+     */
+    private static class Node {
+        /** */
+        @GridToStringInclude
+        String name;
+
+        /** */
+        @GridToStringInclude
+        Node next;
+
+        /** */
+        @GridToStringInclude
+        Node[] nodes;
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return GridToStringBuilder.toString(Node.class, this);
+        }
+    }
+
+    /**
+     * Test class.
+     */
+    private static class BarrierCallable implements Callable<String> {
+        /** */
+        CyclicBarrier bar;
+
+        /** */
+        Object obj;
+
+        /** Expected value of {@code toString()} method. */
+        String exp;
+
+        /** */
+        private BarrierCallable(CyclicBarrier bar, Object obj, String exp) {
+            this.bar = bar;
+            this.obj = obj;
+            this.exp = exp;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String call() throws Exception {
+            for (int i = 0; i < 10; i++) {
+                bar.await();
+
+                assertEquals(exp, obj.toString());
+            }
+
+            return null;
+        }
     }
 
     /**
@@ -137,17 +277,48 @@ private <T, V> void testArr(V v, int limit) throws Exception {
         Arrays.fill(arrOf, v);
         T[] arr = Arrays.copyOf(arrOf, limit);
 
-        String arrStr = GridToStringBuilder.arrayToString(arr.getClass(), arr);
-        String arrOfStr = GridToStringBuilder.arrayToString(arrOf.getClass(), arrOf);
+        checkArrayOverflow(arrOf, arr, limit);
+    }
+
+    /**
+     * Test array print.
+     *
+     * @throws Exception if failed.
+     */
+    public void testArrLimitWithRecursion() throws Exception {
+        int limit = IgniteSystemProperties.getInteger(IGNITE_TO_STRING_COLLECTION_LIMIT, 100);
+
+        ArrayList[] arrOf = new ArrayList[limit + 1];
+        Arrays.fill(arrOf, new ArrayList());
+        ArrayList[] arr = Arrays.copyOf(arrOf, limit);
+
+        arrOf[0].add(arrOf);
+        arr[0].add(arr);
+
+        checkArrayOverflow(arrOf, arr, limit);
+    }
+
+    /**
+     * @param arrOf Array.
+     * @param arr Array copy.
+     * @param limit Array limit.
+     */
+    private void checkArrayOverflow(Object[] arrOf, Object[] arr, int limit) {
+        String arrStr = GridToStringBuilder.arrayToString(arr);
+        String arrOfStr = GridToStringBuilder.arrayToString(arrOf);
 
         // Simulate overflow
         StringBuilder resultSB = new StringBuilder(arrStr);
-            resultSB.deleteCharAt(resultSB.length()-1);
-            resultSB.append("... and ").append(arrOf.length - limit).append(" more]");
-            arrStr = resultSB.toString();
+        resultSB.deleteCharAt(resultSB.length()-1);
+        resultSB.append("... and ").append(arrOf.length - limit).append(" more]");
+
+        arrStr = resultSB.toString();
+
+        info(arrOfStr);
+        info(arrStr);
 
         assertTrue("Collection limit error in array of type " + arrOf.getClass().getName()
-            + " error, normal arr: <" + arrStr + ">, overflowed arr: <" + arrOfStr + ">", arrStr.equals(arrOfStr));
+                + " error, normal arr: <" + arrStr + ">, overflowed arr: <" + arrOfStr + ">", arrStr.equals(arrOfStr));
     }
 
     /**
@@ -157,89 +328,151 @@ public void testToStringCollectionLimits() throws Exception {
         int limit = IgniteSystemProperties.getInteger(IGNITE_TO_STRING_COLLECTION_LIMIT, 100);
 
         Object vals[] = new Object[] {Byte.MIN_VALUE, Boolean.TRUE, Short.MIN_VALUE, Integer.MIN_VALUE, Long.MIN_VALUE,
-            Float.MIN_VALUE, Double.MIN_VALUE, Character.MIN_VALUE, new TestClass1()};
+                Float.MIN_VALUE, Double.MIN_VALUE, Character.MIN_VALUE, new TestClass1()};
         for (Object val : vals)
             testArr(val, limit);
 
+        int[] intArr1 = new int[0];
+
+        assertEquals("[]", GridToStringBuilder.arrayToString(intArr1));
+        assertEquals("null", GridToStringBuilder.arrayToString(null));
+
+        int[] intArr2 = {1, 2, 3};
+
+        assertEquals("[1, 2, 3]", GridToStringBuilder.arrayToString(intArr2));
+
+        Object[] intArr3 = {2, 3, 4};
+
+        assertEquals("[2, 3, 4]", GridToStringBuilder.arrayToString(intArr3));
+
         byte[] byteArr = new byte[1];
+
         byteArr[0] = 1;
-        assertEquals(Arrays.toString(byteArr), GridToStringBuilder.arrayToString(byteArr.getClass(), byteArr));
+        assertEquals(Arrays.toString(byteArr), GridToStringBuilder.arrayToString(byteArr));
         byteArr = Arrays.copyOf(byteArr, 101);
         assertTrue("Can't find \"... and 1 more\" in overflowed array string!",
-            GridToStringBuilder.arrayToString(byteArr.getClass(), byteArr).contains("... and 1 more"));
+                GridToStringBuilder.arrayToString(byteArr).contains("... and 1 more"));
 
         boolean[] boolArr = new boolean[1];
+
         boolArr[0] = true;
-        assertEquals(Arrays.toString(boolArr), GridToStringBuilder.arrayToString(boolArr.getClass(), boolArr));
+        assertEquals(Arrays.toString(boolArr), GridToStringBuilder.arrayToString(boolArr));
         boolArr = Arrays.copyOf(boolArr, 101);
         assertTrue("Can't find \"... and 1 more\" in overflowed array string!",
-            GridToStringBuilder.arrayToString(boolArr.getClass(), boolArr).contains("... and 1 more"));
+                GridToStringBuilder.arrayToString(boolArr).contains("... and 1 more"));
 
         short[] shortArr = new short[1];
+
         shortArr[0] = 100;
-        assertEquals(Arrays.toString(shortArr), GridToStringBuilder.arrayToString(shortArr.getClass(), shortArr));
+        assertEquals(Arrays.toString(shortArr), GridToStringBuilder.arrayToString(shortArr));
         shortArr = Arrays.copyOf(shortArr, 101);
         assertTrue("Can't find \"... and 1 more\" in overflowed array string!",
-            GridToStringBuilder.arrayToString(shortArr.getClass(), shortArr).contains("... and 1 more"));
+                GridToStringBuilder.arrayToString(shortArr).contains("... and 1 more"));
 
         int[] intArr = new int[1];
+
         intArr[0] = 10000;
-        assertEquals(Arrays.toString(intArr), GridToStringBuilder.arrayToString(intArr.getClass(), intArr));
+        assertEquals(Arrays.toString(intArr), GridToStringBuilder.arrayToString(intArr));
         intArr = Arrays.copyOf(intArr, 101);
         assertTrue("Can't find \"... and 1 more\" in overflowed array string!",
-            GridToStringBuilder.arrayToString(intArr.getClass(), intArr).contains("... and 1 more"));
+                GridToStringBuilder.arrayToString(intArr).contains("... and 1 more"));
 
         long[] longArr = new long[1];
+
         longArr[0] = 10000000;
-        assertEquals(Arrays.toString(longArr), GridToStringBuilder.arrayToString(longArr.getClass(), longArr));
+        assertEquals(Arrays.toString(longArr), GridToStringBuilder.arrayToString(longArr));
         longArr = Arrays.copyOf(longArr, 101);
         assertTrue("Can't find \"... and 1 more\" in overflowed array string!",
-            GridToStringBuilder.arrayToString(longArr.getClass(), longArr).contains("... and 1 more"));
+                GridToStringBuilder.arrayToString(longArr).contains("... and 1 more"));
 
         float[] floatArr = new float[1];
+
         floatArr[0] = 1.f;
-        assertEquals(Arrays.toString(floatArr), GridToStringBuilder.arrayToString(floatArr.getClass(), floatArr));
+        assertEquals(Arrays.toString(floatArr), GridToStringBuilder.arrayToString(floatArr));
         floatArr = Arrays.copyOf(floatArr, 101);
         assertTrue("Can't find \"... and 1 more\" in overflowed array string!",
-            GridToStringBuilder.arrayToString(floatArr.getClass(), floatArr).contains("... and 1 more"));
+                GridToStringBuilder.arrayToString(floatArr).contains("... and 1 more"));
 
         double[] doubleArr = new double[1];
+
         doubleArr[0] = 1.;
-        assertEquals(Arrays.toString(doubleArr), GridToStringBuilder.arrayToString(doubleArr.getClass(), doubleArr));
+        assertEquals(Arrays.toString(doubleArr), GridToStringBuilder.arrayToString(doubleArr));
         doubleArr = Arrays.copyOf(doubleArr, 101);
         assertTrue("Can't find \"... and 1 more\" in overflowed array string!",
-            GridToStringBuilder.arrayToString(doubleArr.getClass(), doubleArr).contains("... and 1 more"));
+                GridToStringBuilder.arrayToString(doubleArr).contains("... and 1 more"));
 
-        char[] charArr = new char[1];
-        charArr[0] = 'a';
-        assertEquals(Arrays.toString(charArr), GridToStringBuilder.arrayToString(charArr.getClass(), charArr));
-        charArr = Arrays.copyOf(charArr, 101);
+        char[] cArr = new char[1];
+
+        cArr[0] = 'a';
+        assertEquals(Arrays.toString(cArr), GridToStringBuilder.arrayToString(cArr));
+        cArr = Arrays.copyOf(cArr, 101);
         assertTrue("Can't find \"... and 1 more\" in overflowed array string!",
-            GridToStringBuilder.arrayToString(charArr.getClass(), charArr).contains("... and 1 more"));
+                GridToStringBuilder.arrayToString(cArr).contains("... and 1 more"));
 
         Map<String, String> strMap = new TreeMap<>();
         List<String> strList = new ArrayList<>(limit+1);
 
+        TestClass1 testCls = new TestClass1();
+
+        testCls.strMap = strMap;
+        testCls.strListIncl = strList;
+
+        for (int i = 0; i < limit; i++) {
+            strMap.put("k" + i, "v");
+            strList.add("e");
+        }
+
+        checkColAndMap(testCls);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testToStringColAndMapLimitWithRecursion() throws Exception {
+        int limit = IgniteSystemProperties.getInteger(IGNITE_TO_STRING_COLLECTION_LIMIT, 100);
+        Map strMap = new TreeMap<>();
+        List strList = new ArrayList<>(limit+1);
+
         TestClass1 testClass = new TestClass1();
         testClass.strMap = strMap;
         testClass.strListIncl = strList;
 
-        for (int i = 0; i < limit; i++) {
+        Map m = new TreeMap();
+        m.put("m", strMap);
+
+        List l = new ArrayList();
+        l.add(strList);
+
+        strMap.put("k0", m);
+        strList.add(l);
+
+        for (int i = 1; i < limit; i++) {
             strMap.put("k" + i, "v");
             strList.add("e");
         }
-        String testClassStr = GridToStringBuilder.toString(TestClass1.class, testClass);
 
-        strMap.put("kz", "v"); // important to add last element in TreeMap here
-        strList.add("e");
+        checkColAndMap(testClass);
+    }
+
+    /**
+     * @param testCls Class with collection and map included in toString().
+     */
+    private void checkColAndMap(TestClass1 testCls) {
+        String testClsStr = GridToStringBuilder.toString(TestClass1.class, testCls);
 
-        String testClassStrOf = GridToStringBuilder.toString(TestClass1.class, testClass);
+        testCls.strMap.put("kz", "v"); // important to add last element in TreeMap here
+        testCls.strListIncl.add("e");
 
-        String testClassStrOfR = testClassStrOf.replaceAll("... and 1 more","");
+        String testClsStrOf = GridToStringBuilder.toString(TestClass1.class, testCls);
 
-        assertTrue("Collection limit error in Map or List, normal: <" + testClassStr + ">, overflowed: <"
-            +"testClassStrOf", testClassStr.length() == testClassStrOfR.length());
+        String testClsStrOfR = testClsStrOf.replaceAll("... and 1 more","");
 
+        info(testClsStr);
+        info(testClsStrOf);
+        info(testClsStrOfR);
+
+        assertTrue("Collection limit error in Map or List, normal: <" + testClsStr + ">, overflowed: <"
+                + testClsStrOf + ">", testClsStr.length() == testClsStrOfR.length());
     }
 
     /**
@@ -248,25 +481,41 @@ public void testToStringCollectionLimits() throws Exception {
     public void testToStringSizeLimits() throws Exception {
         int limit = IgniteSystemProperties.getInteger(IGNITE_TO_STRING_MAX_LENGTH, 10_000);
         int tailLen = limit / 10 * 2;
+
         StringBuilder sb = new StringBuilder(limit + 10);
-        for (int i = 0; i < limit - 100; i++) {
+
+        for (int i = 0; i < limit - 100; i++)
             sb.append('a');
-        }
+
         String actual = GridToStringBuilder.toString(TestClass2.class, new TestClass2(sb.toString()));
-        String expected = "TestClass2 [str=" + sb.toString() + ", nullArr=null]";
-        assertEquals(expected, actual);
+        String exp = "TestClass2 [str=" + sb + ", nullArr=null]";
+
+        assertEquals(exp, actual);
 
-        for (int i = 0; i < 110; i++) {
+        for (int i = 0; i < 110; i++)
             sb.append('b');
-        }
+
         actual = GridToStringBuilder.toString(TestClass2.class, new TestClass2(sb.toString()));
-        expected = "TestClass2 [str=" + sb.toString() + ", nullArr=null]";
-        assertEquals(expected.substring(0, limit - tailLen), actual.substring(0, limit - tailLen));
-        assertEquals(expected.substring(expected.length() - tailLen), actual.substring(actual.length() - tailLen));
+        exp = "TestClass2 [str=" + sb + ", nullArr=null]";
+
+        assertEquals(exp.substring(0, limit - tailLen), actual.substring(0, limit - tailLen));
+        assertEquals(exp.substring(exp.length() - tailLen), actual.substring(actual.length() - tailLen));
+
         assertTrue(actual.contains("... and"));
         assertTrue(actual.contains("skipped ..."));
     }
 
+    /**
+     *
+     */
+    public void testObjectPlusStringToString() {
+        IgniteTxKey k = new IgniteTxKey(new KeyCacheObjectImpl(1, null, 1), 123);
+
+        info(k.toString());
+
+        assertTrue("Wrong string: " + k, k.toString().startsWith("IgniteTxKey ["));
+    }
+
     /**
      * Test class.
      */
@@ -398,8 +647,8 @@ private static class TestClass2{
         /**
          * @param str String.
          */
-        public TestClass2(String str) {
+        TestClass2(String str) {
             this.str = str;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java
index e2b94e4ec3ac7..d56ed094fed0e 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java
@@ -183,7 +183,7 @@ public void testFsOpenMultithreaded() throws Exception {
      */
     private void checkFsOpenWithAllNodesTypes() throws Exception {
         for (int i = 0; i < nodesTypes.length; ++i) {
-            log.info("Begin test case for nodes: " + S.arrayToString(NodeType.class, nodesTypes[i]));
+            log.info("Begin test case for nodes: " + S.arrayToString(nodesTypes[i]));
 
             startNodes(nodesTypes[i]);
 

From cf81547e7fe640eb8b0a72cdb1c32e6c6bfe16d3 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Fri, 17 Aug 2018 15:20:23 +0300
Subject: [PATCH 306/543] IGNITE-9307 Added completing eviction future if node
 was stopped - Fixes #4564.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 654ccf0)
---
 .../cache/distributed/dht/PartitionsEvictManager.java        | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java
index f76310d439ee9..780ca918593f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java
@@ -410,8 +410,11 @@ private PartitionEvictionTask(
 
         /** {@inheritDoc} */
         @Override public void run() {
-            if (groupEvictionContext.shouldStop())
+            if (groupEvictionContext.shouldStop()) {
+                finishFut.onDone();
+
                 return;
+            }
 
             try {
                 boolean success = part.tryClear(groupEvictionContext);

From 618395a35f4cc0406b2f7a73214f1c323d407b6a Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Thu, 9 Aug 2018 16:37:04 +0300
Subject: [PATCH 307/543] IGNITE-8219 Call failure handler when an infinite
 loop is detected in B+ tree. Fixes #3849.

---
 .../jmh/tree/BPlusTreeBenchmark.java          |  2 +-
 .../persistence/GridCacheOffheapManager.java  |  3 +-
 .../cache/persistence/IndexStorageImpl.java   | 13 ++++---
 .../persistence/metastorage/MetaStorage.java  |  7 +++-
 .../metastorage/MetastorageTree.java          | 36 +++++++++++--------
 .../cache/persistence/tree/BPlusTree.java     | 33 +++++++++++++----
 .../persistence/tree/io/BPlusMetaIO.java      |  2 +-
 .../persistence/tree/util/PageHandler.java    | 10 +++---
 .../processors/cache/tree/CacheDataTree.java  |  3 +-
 .../cache/tree/PendingEntriesTree.java        |  8 ++---
 .../database/BPlusTreeSelfTest.java           |  2 +-
 .../database/IndexStorageSelfTest.java        |  3 +-
 .../processors/query/h2/database/H2Tree.java  |  7 ++--
 .../query/h2/database/H2TreeIndex.java        |  3 +-
 .../index/DynamicIndexAbstractSelfTest.java   |  3 ++
 15 files changed, 92 insertions(+), 43 deletions(-)

diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java
index cef00eedd11c5..7ed84cbd8d93e 100644
--- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java
+++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java
@@ -176,7 +176,7 @@ protected static class TestTree extends BPlusTree<Long, Long> {
         TestTree(ReuseList reuseList, int cacheId, PageMemory pageMem, long metaPageId)
             throws IgniteCheckedException {
             super("test", cacheId, pageMem, null, new AtomicLong(), metaPageId, reuseList,
-                new IOVersions<>(new LongInnerIO()), new IOVersions<>(new LongLeafIO()));
+                new IOVersions<>(new LongInnerIO()), new IOVersions<>(new LongLeafIO()), null);
 
             PageIO.registerTest(latestInnerIO(), latestLeafIO());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index ac4f163ce5c60..0a02ca7f0c4ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -117,7 +117,8 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
             PageIdAllocator.FLAG_IDX,
             reuseList,
             metastoreRoot.pageId().pageId(),
-            metastoreRoot.isAllocated());
+            metastoreRoot.isAllocated(),
+            ctx.kernalContext().failure());
 
         ((GridCacheDatabaseSharedManager)ctx.database()).addCheckpointListener(this);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
index 7daef3c9f4f9d..a4266fa83416f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IndexStorageImpl.java
@@ -32,7 +32,9 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Metadata storage.
@@ -78,7 +80,8 @@ public IndexStorageImpl(
         final byte allocSpace,
         final ReuseList reuseList,
         final long rootPageId,
-        final boolean initNew
+        final boolean initNew,
+        final FailureProcessor failureProcessor
     ) {
         try {
             this.pageMem = pageMem;
@@ -88,7 +91,7 @@ public IndexStorageImpl(
             this.reuseList = reuseList;
 
             metaTree = new MetaTree(grpId, allocPartId, allocSpace, pageMem, wal, globalRmvId, rootPageId,
-                reuseList, MetaStoreInnerIO.VERSIONS, MetaStoreLeafIO.VERSIONS, initNew);
+                reuseList, MetaStoreInnerIO.VERSIONS, MetaStoreLeafIO.VERSIONS, initNew, failureProcessor);
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException(e);
@@ -164,6 +167,7 @@ private static class MetaTree extends BPlusTree<IndexItem, IndexItem> {
          * @param reuseList Reuse list.
          * @param innerIos Inner IOs.
          * @param leafIos Leaf IOs.
+         * @param failureProcessor if the tree is corrupted.
          * @throws IgniteCheckedException If failed.
          */
         private MetaTree(
@@ -177,9 +181,10 @@ private MetaTree(
             final ReuseList reuseList,
             final IOVersions<? extends BPlusInnerIO<IndexItem>> innerIos,
             final IOVersions<? extends BPlusLeafIO<IndexItem>> leafIos,
-            final boolean initNew
+            final boolean initNew,
+            @Nullable FailureProcessor failureProcessor
         ) throws IgniteCheckedException {
-            super(treeName("meta", "Meta"), cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList, innerIos, leafIos);
+            super(treeName("meta", "Meta"), cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList, innerIos, leafIos, failureProcessor);
 
             this.allocPartId = allocPartId;
             this.allocSpace = allocSpace;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index 14bd450471401..4a2549b51f8e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -51,6 +51,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.SimpleDataPageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.util.lang.GridCursor;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -114,6 +115,9 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     /** */
     private final Marshaller marshaller = new JdkMarshaller();
 
+    /** */
+    private final FailureProcessor failureProcessor;
+
     /** */
     public MetaStorage(
         GridCacheSharedContext cctx,
@@ -126,6 +130,7 @@ public MetaStorage(
         this.regionMetrics = regionMetrics;
         this.readOnly = readOnly;
         log = cctx.logger(getClass());
+        this.failureProcessor = cctx.kernalContext().failure();
     }
 
     /** */
@@ -145,7 +150,7 @@ public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedExcept
             MetastorageRowStore rowStore = new MetastorageRowStore(freeList, db);
 
             tree = new MetastorageTree(METASTORAGE_CACHE_ID, dataRegion.pageMemory(), wal, rmvId,
-                freeList, rowStore, treeRoot.pageId().pageId(), treeRoot.isAllocated());
+                freeList, rowStore, treeRoot.pageId().pageId(), treeRoot.isAllocated(), failureProcessor);
 
             if (!readOnly)
                 ((GridCacheDatabaseSharedManager)db).addCheckpointListener(this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
index 445522bf24d73..19a145f1019ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
@@ -28,6 +28,8 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
+import org.jetbrains.annotations.Nullable;
 
 /**
  *
@@ -40,12 +42,15 @@ public class MetastorageTree extends BPlusTree<MetastorageSearchRow, Metastorage
     private MetastorageRowStore rowStore;
 
     /**
-     * @param pageMem
-     * @param wal
-     * @param globalRmvId
-     * @param metaPageId
-     * @param reuseList
-     * @throws IgniteCheckedException
+     * @param pageMem Page memory instance.
+     * @param wal WAL manager.
+     * @param globalRmvId Global remove ID.
+     * @param metaPageId Meta page ID.
+     * @param reuseList Reuse list.
+     * @param rowStore Row store.
+     * @param initNew Init new flag, if {@code true}, then new tree will be allocated.
+     * @param failureProcessor To call if the tree is corrupted.
+     * @throws IgniteCheckedException If failed to initialize.
      */
     public MetastorageTree(int cacheId,
         PageMemory pageMem,
@@ -54,9 +59,10 @@ public MetastorageTree(int cacheId,
         ReuseList reuseList,
         MetastorageRowStore rowStore,
         long metaPageId,
-        boolean initNew) throws IgniteCheckedException {
+        boolean initNew,
+        @Nullable FailureProcessor failureProcessor) throws IgniteCheckedException {
         super("Metastorage", cacheId, pageMem, wal,
-            globalRmvId, metaPageId, reuseList, MetastorageInnerIO.VERSIONS, MetastoreLeafIO.VERSIONS);
+            globalRmvId, metaPageId, reuseList, MetastorageInnerIO.VERSIONS, MetastoreLeafIO.VERSIONS, failureProcessor);
 
         this.rowStore = rowStore;
 
@@ -65,7 +71,7 @@ public MetastorageTree(int cacheId,
 
     /** {@inheritDoc} */
     @Override protected int compare(BPlusIO<MetastorageSearchRow> io, long pageAddr, int idx,
-        MetastorageSearchRow row) throws IgniteCheckedException {
+        MetastorageSearchRow row) {
 
         String key = ((DataLinkIO)io).getKey(pageAddr, idx);
 
@@ -132,7 +138,7 @@ public static class MetastorageInnerIO extends BPlusInnerIO<MetastorageSearchRow
 
         /** {@inheritDoc} */
         @Override public void storeByOffset(long pageAddr, int off,
-            MetastorageSearchRow row) throws IgniteCheckedException {
+            MetastorageSearchRow row) {
             assert row.link() != 0;
 
             PageUtils.putLong(pageAddr, off, row.link());
@@ -146,7 +152,7 @@ public static class MetastorageInnerIO extends BPlusInnerIO<MetastorageSearchRow
 
         /** {@inheritDoc} */
         @Override public void store(long dstPageAddr, int dstIdx, BPlusIO<MetastorageSearchRow> srcIo, long srcPageAddr,
-            int srcIdx) throws IgniteCheckedException {
+            int srcIdx) {
             int srcOff = srcIo.offset(srcIdx);
             int dstOff = offset(dstIdx);
 
@@ -162,7 +168,7 @@ public static class MetastorageInnerIO extends BPlusInnerIO<MetastorageSearchRow
 
         /** {@inheritDoc} */
         @Override public MetastorageSearchRow getLookupRow(BPlusTree<MetastorageSearchRow, ?> tree, long pageAddr,
-            int idx) throws IgniteCheckedException {
+            int idx) {
             long link = getLink(pageAddr, idx);
             String key = getKey(pageAddr, idx);
 
@@ -207,7 +213,7 @@ public static class MetastoreLeafIO extends BPlusLeafIO<MetastorageSearchRow> im
 
         /** {@inheritDoc} */
         @Override public void storeByOffset(long pageAddr, int off,
-            MetastorageSearchRow row) throws IgniteCheckedException {
+            MetastorageSearchRow row) {
             assert row.link() != 0;
 
             PageUtils.putLong(pageAddr, off, row.link());
@@ -221,7 +227,7 @@ public static class MetastoreLeafIO extends BPlusLeafIO<MetastorageSearchRow> im
 
         /** {@inheritDoc} */
         @Override public void store(long dstPageAddr, int dstIdx, BPlusIO<MetastorageSearchRow> srcIo, long srcPageAddr,
-            int srcIdx) throws IgniteCheckedException {
+            int srcIdx) {
             int srcOff = srcIo.offset(srcIdx);
             int dstOff = offset(dstIdx);
 
@@ -237,7 +243,7 @@ public static class MetastoreLeafIO extends BPlusLeafIO<MetastorageSearchRow> im
 
         /** {@inheritDoc} */
         @Override public MetastorageSearchRow getLookupRow(BPlusTree<MetastorageSearchRow, ?> tree, long pageAddr,
-            int idx) throws IgniteCheckedException {
+            int idx) {
             long link = getLink(pageAddr, idx);
             String key = getKey(pageAddr, idx);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
index 4d050952b4be6..db26ee725d22c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -28,6 +28,8 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
@@ -53,6 +55,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.util.GridArrays;
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.IgniteTree;
@@ -65,6 +68,7 @@
 import org.apache.ignite.lang.IgniteInClosure;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_BPLUS_TREE_LOCK_RETRIES;
 import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Bool.DONE;
 import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Bool.FALSE;
 import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Bool.READY;
@@ -92,7 +96,7 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
 
     /** */
     private static final int LOCK_RETRIES = IgniteSystemProperties.getInteger(
-        IgniteSystemProperties.IGNITE_BPLUS_TREE_LOCK_RETRIES, IGNITE_BPLUS_TREE_LOCK_RETRIES_DEFAULT);
+        IGNITE_BPLUS_TREE_LOCK_RETRIES, IGNITE_BPLUS_TREE_LOCK_RETRIES_DEFAULT);
 
     /** */
     private final AtomicBoolean destroyed = new AtomicBoolean(false);
@@ -124,6 +128,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
     /** */
     private volatile TreeMetaData treeMeta;
 
+    /** Failure processor. */
+    private final FailureProcessor failureProcessor;
+
     /** */
     private final GridTreePrinter<Long> treePrinter = new GridTreePrinter<Long>() {
         /** */
@@ -720,6 +727,7 @@ private class InitRoot extends PageHandler<Long, Bool> {
      * @param reuseList Reuse list.
      * @param innerIos Inner IO versions.
      * @param leafIos Leaf IO versions.
+     * @param failureProcessor if the tree is corrupted.
      * @throws IgniteCheckedException If failed.
      */
     protected BPlusTree(
@@ -731,9 +739,10 @@ protected BPlusTree(
         long metaPageId,
         ReuseList reuseList,
         IOVersions<? extends BPlusInnerIO<L>> innerIos,
-        IOVersions<? extends BPlusLeafIO<L>> leafIos
+        IOVersions<? extends BPlusLeafIO<L>> leafIos,
+        @Nullable FailureProcessor failureProcessor
     ) throws IgniteCheckedException {
-        this(name, cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList);
+        this(name, cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList, failureProcessor);
         setIos(innerIos, leafIos);
     }
 
@@ -745,6 +754,7 @@ protected BPlusTree(
      * @param globalRmvId Remove ID.
      * @param metaPageId Meta page ID.
      * @param reuseList Reuse list.
+     * @param failureProcessor if the tree is corrupted.
      * @throws IgniteCheckedException If failed.
      */
     protected BPlusTree(
@@ -754,7 +764,8 @@ protected BPlusTree(
         IgniteWriteAheadLogManager wal,
         AtomicLong globalRmvId,
         long metaPageId,
-        ReuseList reuseList
+        ReuseList reuseList,
+        @Nullable FailureProcessor failureProcessor
     ) throws IgniteCheckedException {
         super(cacheId, pageMem, wal);
 
@@ -770,6 +781,7 @@ protected BPlusTree(
         this.name = name;
         this.reuseList = reuseList;
         this.globalRmvId = globalRmvId;
+        this.failureProcessor = failureProcessor;
     }
 
     /**
@@ -2561,8 +2573,17 @@ boolean isFinished() {
          * @throws IgniteCheckedException If the operation can not be retried.
          */
         final void checkLockRetry() throws IgniteCheckedException {
-            if (lockRetriesCnt == 0)
-                throw new IgniteCheckedException("Maximum of retries " + getLockRetries() + " reached.");
+            if (lockRetriesCnt == 0) {
+                IgniteCheckedException e = new IgniteCheckedException("Maximum number of retries " +
+                    getLockRetries() + " reached for " + getClass().getSimpleName() + " operation " +
+                    "(the tree may be corrupted). Increase " + IGNITE_BPLUS_TREE_LOCK_RETRIES + " system property " +
+                    "if you regularly see this message (current value is " + getLockRetries() + ").");
+
+                if (failureProcessor != null)
+                    failureProcessor.process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+                throw e;
+            }
 
             lockRetriesCnt--;
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusMetaIO.java
index afa3c9a9b432e..623951bafc58d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusMetaIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusMetaIO.java
@@ -77,7 +77,7 @@ public void initRoot(long pageAdrr, long rootId, int pageSize) {
      * @return Number of levels in this tree.
      */
     public int getLevelsCount(long pageAddr) {
-        return PageUtils.getByte(pageAddr, LVLS_OFF);
+        return Byte.toUnsignedInt(PageUtils.getByte(pageAddr, LVLS_OFF));
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
index a52038ab66cc3..98c6f1f766cc1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
@@ -151,16 +151,18 @@ public static <X, R> R readPage(
         int intArg,
         R lockFailed
     ) throws IgniteCheckedException {
-        long pageAddr = readLock(pageMem, cacheId, pageId, page, lsnr);
+        long pageAddr = 0L;
 
-        if (pageAddr == 0L)
-            return lockFailed;
         try {
+            if ((pageAddr = readLock(pageMem, cacheId, pageId, page, lsnr)) == 0L)
+                return lockFailed;
+
             PageIO io = PageIO.getPageIO(pageAddr);
             return h.run(cacheId, pageId, page, pageAddr, io, null, arg, intArg);
         }
         finally {
-            readUnlock(pageMem, cacheId, pageId, page, pageAddr, lsnr);
+            if (pageAddr != 0L)
+                readUnlock(pageMem, cacheId, pageId, page, pageAddr, lsnr);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/CacheDataTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/CacheDataTree.java
index f2bfa414e87b2..c6214ebe6aa6a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/CacheDataTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/CacheDataTree.java
@@ -70,7 +70,8 @@ public CacheDataTree(
             metaPageId,
             reuseList,
             grp.sharedGroup() ? CacheIdAwareDataInnerIO.VERSIONS : DataInnerIO.VERSIONS,
-            grp.sharedGroup() ? CacheIdAwareDataLeafIO.VERSIONS : DataLeafIO.VERSIONS);
+            grp.sharedGroup() ? CacheIdAwareDataLeafIO.VERSIONS : DataLeafIO.VERSIONS,
+            grp.shared().kernalContext().failure());
 
         assert rowStore != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/PendingEntriesTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/PendingEntriesTree.java
index 0b1c931e4321a..78a9f56e8b61d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/PendingEntriesTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/PendingEntriesTree.java
@@ -30,7 +30,7 @@
  */
 public class PendingEntriesTree extends BPlusTree<PendingRow, PendingRow> {
     /** */
-    public final static Object WITHOUT_KEY = new Object();
+    public static final Object WITHOUT_KEY = new Object();
 
     /** */
     private final CacheGroupContext grp;
@@ -60,7 +60,8 @@ public PendingEntriesTree(
             metaPageId,
             reuseList,
             grp.sharedGroup() ? CacheIdAwarePendingEntryInnerIO.VERSIONS : PendingEntryInnerIO.VERSIONS,
-            grp.sharedGroup() ? CacheIdAwarePendingEntryLeafIO.VERSIONS : PendingEntryLeafIO.VERSIONS);
+            grp.sharedGroup() ? CacheIdAwarePendingEntryLeafIO.VERSIONS : PendingEntryLeafIO.VERSIONS,
+            grp.shared().kernalContext().failure());
 
         this.grp = grp;
 
@@ -70,8 +71,7 @@ public PendingEntriesTree(
     }
 
     /** {@inheritDoc} */
-    @Override protected int compare(BPlusIO<PendingRow> iox, long pageAddr, int idx, PendingRow row)
-        throws IgniteCheckedException {
+    @Override protected int compare(BPlusIO<PendingRow> iox, long pageAddr, int idx, PendingRow row) {
         PendingRowIO io = (PendingRowIO)iox;
 
         int cmp;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index 83d0ddd278a79..1b18364fd2a96 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -2383,7 +2383,7 @@ protected static class TestTree extends BPlusTree<Long, Long> {
         public TestTree(ReuseList reuseList, boolean canGetRow, int cacheId, PageMemory pageMem, long metaPageId)
             throws IgniteCheckedException {
             super("test", cacheId, pageMem, null, new AtomicLong(), metaPageId, reuseList,
-                new IOVersions<>(new LongInnerIO(canGetRow)), new IOVersions<>(new LongLeafIO()));
+                new IOVersions<>(new LongInnerIO(canGetRow)), new IOVersions<>(new LongLeafIO()), null);
 
             PageIO.registerTest(latestInnerIO(), latestLeafIO());
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
index 04bea541245b8..01222eb2595f3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
@@ -98,7 +98,8 @@ private void metaAllocation() throws Exception {
                 if (metaStore == null) {
                     metaStore = new IndexStorageImpl(mem, null, new AtomicLong(), cacheId,
                         PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX,
-                        null, mem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX), true);
+                        null, mem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX), true,
+                            null);
 
                     storeMap.put(cacheId, metaStore);
                 }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java
index 8da3b05966c1d..424969e6484d2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.processors.query.h2.H2RowCache;
 import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasInnerIO;
 import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasLeafIO;
@@ -81,6 +82,7 @@ public abstract class H2Tree extends BPlusTree<SearchRow, GridH2Row> {
      * @param metaPageId Meta page ID.
      * @param initNew Initialize new index.
      * @param rowCache Row cache.
+     * @param failureProcessor if the tree is corrupted.
      * @throws IgniteCheckedException If failed.
      */
     protected H2Tree(
@@ -96,9 +98,10 @@ protected H2Tree(
         IndexColumn[] cols,
         List<InlineIndexHelper> inlineIdxs,
         int inlineSize,
-        @Nullable H2RowCache rowCache
+        @Nullable H2RowCache rowCache,
+        @Nullable FailureProcessor failureProcessor
     ) throws IgniteCheckedException {
-        super(name, grpId, pageMem, wal, globalRmvId, metaPageId, reuseList);
+        super(name, grpId, pageMem, wal, globalRmvId, metaPageId, reuseList, failureProcessor);
 
         if (!initNew) {
             // Page is ready - read inline size from it.
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
index 2441ff1770e37..393ca3b5352b8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
@@ -139,7 +139,8 @@ public H2TreeIndex(
                         cols,
                         inlineIdxs,
                         computeInlineSize(inlineIdxs, inlineSize),
-                        rowCache) {
+                        rowCache,
+                        cctx.kernalContext().failure()) {
                         @Override public int compareValues(Value v1, Value v2) {
                             return v1 == v2 ? 0 : table.compareTypeSafe(v1, v2);
                         }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractSelfTest.java
index 452ac96ba9e35..1d0f9b816fc32 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractSelfTest.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.util.typedef.T2;
@@ -138,6 +139,8 @@ protected IgniteConfiguration clientConfiguration(int idx) throws Exception {
     protected IgniteConfiguration commonConfiguration(int idx) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(getTestIgniteInstanceName(idx));
 
+        cfg.setFailureHandler(new StopNodeFailureHandler());
+
         cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
 
         cfg.setMarshaller(new BinaryMarshaller());

From 4d286331cd4b749076ef9b915b749c4be187f299 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Fri, 17 Aug 2018 15:45:43 +0300
Subject: [PATCH 308/543] IGNITE-9268 Fixed unreleased page lock when exception
 is thrown from BPlusTree - Fixes #4543.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit e827b60)
---
 .../cache/persistence/tree/BPlusTree.java     | 35 +++++++++++++------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
index db26ee725d22c..38e91bcf949fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -2572,7 +2572,7 @@ boolean isFinished() {
         /**
          * @throws IgniteCheckedException If the operation can not be retried.
          */
-        final void checkLockRetry() throws IgniteCheckedException {
+        void checkLockRetry() throws IgniteCheckedException {
             if (lockRetriesCnt == 0) {
                 IgniteCheckedException e = new IgniteCheckedException("Maximum number of retries " +
                     getLockRetries() + " reached for " + getClass().getSimpleName() + " operation " +
@@ -2661,6 +2661,13 @@ private final class GetCursor extends Get {
      * Put operation.
      */
     private final class Put extends Get {
+        /** Mark of NULL value of page id. It means valid value can't be equal this value. */
+        private static final long NULL_PAGE_ID = 0L;
+        /** Mark of NULL value of page. */
+        private static final long NULL_PAGE = 0L;
+        /** Mark of NULL value of page address. */
+        private static final long NULL_PAGE_ADDRESS = 0L;
+
         /** Right child page ID for split row. */
         long rightId;
 
@@ -2668,9 +2675,8 @@ private final class Put extends Get {
         T oldRow;
 
         /**
-         * This page is kept locked after split until insert to the upper level will not be finished.
-         * It is needed because split row will be "in flight" and if we'll release tail, remove on
-         * split row may fail.
+         * This page is kept locked after split until insert to the upper level will not be finished. It is needed
+         * because split row will be "in flight" and if we'll release tail, remove on split row may fail.
          */
         long tailId;
 
@@ -2731,10 +2737,10 @@ private Put(T row, boolean needOld) {
          * @param tailPageAddr Tail page address
          */
         private void tail(long tailId, long tailPage, long tailPageAddr) {
-            assert (tailId == 0L) == (tailPage == 0L);
-            assert (tailPage == 0L) == (tailPageAddr == 0L);
+            assert (tailId == NULL_PAGE_ID) == (tailPage == NULL_PAGE);
+            assert (tailPage == NULL_PAGE) == (tailPageAddr == NULL_PAGE_ADDRESS);
 
-            if (this.tailPage != 0L)
+            if (this.tailPage != NULL_PAGE)
                 writeUnlockAndClose(this.tailId, this.tailPage, this.tailAddr, null);
 
             this.tailId = tailId;
@@ -2744,7 +2750,7 @@ private void tail(long tailId, long tailPage, long tailPageAddr) {
 
         /** {@inheritDoc} */
         @Override boolean canRelease(long pageId, int lvl) {
-            return pageId != 0L && tailId != pageId;
+            return pageId != NULL_PAGE_ID && tailId != pageId;
         }
 
         /**
@@ -2754,7 +2760,7 @@ private void finish() {
             row = null;
             rightId = 0;
 
-            tail(0L, 0L, 0L);
+            tail(NULL_PAGE_ID, NULL_PAGE, NULL_PAGE_ADDRESS);
         }
 
         /** {@inheritDoc} */
@@ -2825,7 +2831,7 @@ private L insertWithSplit(long pageId, long page, long pageAddr, BPlusIO<L> io,
 
                 long fwdPageAddr = writeLock(fwdId, fwdPage); // Initial write, no need to check for concurrent modification.
 
-                assert fwdPageAddr != 0L;
+                assert fwdPageAddr != NULL_PAGE_ADDRESS;
 
                 // TODO GG-11640 log a correct forward page record.
                 final Boolean fwdPageWalPlc = Boolean.TRUE;
@@ -2873,7 +2879,7 @@ private L insertWithSplit(long pageId, long page, long pageAddr, BPlusIO<L> io,
 
                             long newRootAddr = writeLock(newRootId, newRootPage); // Initial write.
 
-                            assert newRootAddr != 0L;
+                            assert newRootAddr != NULL_PAGE_ADDRESS;
 
                             // Never write full new root page, because it is known to be new.
                             final Boolean newRootPageWalPlc = Boolean.FALSE;
@@ -2990,6 +2996,13 @@ public Result tryReplace(long pageId, long page, long fwdId, int lvl) throws Ign
 
             return write(pageId, page, replace, this, lvl, RETRY);
         }
+
+        /** {@inheritDoc} */
+        @Override void checkLockRetry() throws IgniteCheckedException {
+            //non null tailId means that lock on tail page still hold and we can't fail with exception.
+            if (tailId == NULL_PAGE_ID)
+                super.checkLockRetry();
+        }
     }
 
     /**

From 0c3c34cdda1d9e3044b85fa63403e46b5256366b Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@apache.org>
Date: Fri, 22 Jun 2018 16:58:44 +0700
Subject: [PATCH 309/543] GG-14087 Backport IGNITE-8428 Web Console:
 Implemented connection to secured cluster.

(cherry picked from commit 523a871bfa0c946f6b55755763ca382621f236d3)
---
 modules/web-console/web-agent/README.txt      |   6 +
 .../console/agent/AgentConfiguration.java     | 115 ++++--
 .../ignite/console/agent/AgentLauncher.java   | 334 ++++++++----------
 .../ignite/console/agent/AgentUtils.java      |  24 ++
 .../agent/handlers/AbstractListener.java      |  50 +--
 .../agent/handlers/ClusterListener.java       | 227 ++++++++----
 .../agent/handlers/DatabaseListener.java      |   2 +-
 .../console/agent/handlers/DemoListener.java  | 131 -------
 .../console/agent/handlers/RestListener.java  |  41 ++-
 .../console/agent/rest/RestExecutor.java      | 308 +++++-----------
 .../ignite/console/agent/rest/RestResult.java |  18 +-
 .../ignite/console/demo/AgentClusterDemo.java |   7 +-
 12 files changed, 567 insertions(+), 696 deletions(-)
 delete mode 100644 modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DemoListener.java

diff --git a/modules/web-console/web-agent/README.txt b/modules/web-console/web-agent/README.txt
index cdffe7fd988b8..86f12b884ae1f 100644
--- a/modules/web-console/web-agent/README.txt
+++ b/modules/web-console/web-agent/README.txt
@@ -20,6 +20,8 @@ Configuration file:
     tokens
     server-uri
     node-uri
+    node-login
+    node-password
     driver-folder
 
   Example configuration file:
@@ -47,6 +49,10 @@ Options:
   -n, --node-uri
      Comma-separated list of URIs for connect to Ignite REST server,
      default value: http://localhost:8080
+  -nl, --node-login
+     User name that will be used to connect to secured cluster.
+  -np, --node-password
+     Password that will be used to connect to secured cluster
   -s, --server-uri
      URI for connect to Ignite Web Console via web-socket protocol,
      default value: http://localhost:3000
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java
index 3a3d9508b4266..bb2a8a2a49f13 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentConfiguration.java
@@ -25,6 +25,7 @@
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Properties;
 import org.apache.ignite.internal.util.typedef.F;
@@ -57,10 +58,21 @@ public class AgentConfiguration {
     private String srvUri;
 
     /** */
-    @Parameter(names = {"-n", "--node-uri"}, description = "Comma-separated URIs for connect to Ignite node via REST" +
-        "                        " +
-        "      Default value: " + DFLT_NODE_URI)
-    private String nodeUri;
+    @Parameter(names = {"-n", "--node-uri"},
+        description = "Comma-separated URIs for connect to Ignite node via REST" +
+            "                        " +
+            "      Default value: " + DFLT_NODE_URI)
+    private List<String> nodeURIs;
+
+    /** */
+    @Parameter(names = {"-nl", "--node-login"},
+        description = "User name that will be used to connect to secured cluster")
+    private String nodeLogin;
+
+    /** */
+    @Parameter(names = {"-np", "--node-password"},
+        description = "Password that will be used to connect to secured cluster")
+    private String nodePwd;
 
     /** URI for connect to Ignite demo node REST server */
     private String demoNodeUri;
@@ -116,17 +128,45 @@ public void serverUri(String srvUri) {
     }
 
     /**
-     * @return Node URI.
+     * @return Node URIs.
+     */
+    public List<String> nodeURIs() {
+        return nodeURIs;
+    }
+
+    /**
+     * @param nodeURIs Node URIs.
+     */
+    public void nodeURIs(List<String> nodeURIs) {
+        this.nodeURIs = nodeURIs;
+    }
+
+    /**
+     * @return User name for agent to authenticate on node.
+     */
+    public String nodeLogin() {
+        return nodeLogin;
+    }
+
+    /**
+     * @param nodeLogin User name for agent to authenticate on node.
+     */
+    public void nodeLogin(String nodeLogin) {
+        this.nodeLogin = nodeLogin;
+    }
+
+    /**
+     * @return Agent password to authenticate on node.
      */
-    public String nodeUri() {
-        return nodeUri;
+    public String nodePassword() {
+        return nodePwd;
     }
 
     /**
-     * @param nodeUri Node URI.
+     * @param nodePwd Agent password to authenticate on node.
      */
-    public void nodeUri(String nodeUri) {
-        this.nodeUri = nodeUri;
+    public void nodePassword(String nodePwd) {
+        this.nodePwd = nodePwd;
     }
 
     /**
@@ -208,7 +248,17 @@ public void load(URL cfgUrl) throws IOException {
         val = (String)props.remove("node-uri");
 
         if (val != null)
-            nodeUri(val);
+            nodeURIs(new ArrayList<>(Arrays.asList(val.split(","))));
+
+        val = (String)props.remove("node-login");
+
+        if (val != null)
+            nodeLogin(val);
+
+        val = (String)props.remove("node-password");
+
+        if (val != null)
+            nodePassword(val);
 
         val = (String)props.remove("driver-folder");
 
@@ -217,29 +267,35 @@ public void load(URL cfgUrl) throws IOException {
     }
 
     /**
-     * @param cmd Command.
+     * @param cfg Config to merge with.
      */
-    public void merge(AgentConfiguration cmd) {
+    public void merge(AgentConfiguration cfg) {
         if (tokens == null)
-            tokens(cmd.tokens());
+            tokens(cfg.tokens());
 
         if (srvUri == null)
-            serverUri(cmd.serverUri());
+            serverUri(cfg.serverUri());
 
         if (srvUri == null)
             serverUri(DFLT_SERVER_URI);
 
-        if (nodeUri == null)
-            nodeUri(cmd.nodeUri());
+        if (nodeURIs == null)
+            nodeURIs(cfg.nodeURIs());
+
+        if (nodeURIs == null)
+            nodeURIs(Collections.singletonList(DFLT_NODE_URI));
 
-        if (nodeUri == null)
-            nodeUri(DFLT_NODE_URI);
+        if (nodeLogin == null)
+            nodeLogin(cfg.nodeLogin());
+
+        if (nodePwd == null)
+            nodePassword(cfg.nodePassword());
 
         if (driversFolder == null)
-            driversFolder(cmd.driversFolder());
+            driversFolder(cfg.driversFolder());
 
         if (disableDemo == null)
-            disableDemo(cmd.disableDemo());
+            disableDemo(cfg.disableDemo());
     }
 
     /** {@inheritDoc} */
@@ -247,7 +303,7 @@ public void merge(AgentConfiguration cmd) {
         StringBuilder sb = new StringBuilder();
 
         if (!F.isEmpty(tokens)) {
-            sb.append("User's security tokens        : ");
+            sb.append("User's security tokens          : ");
 
             boolean first = true;
 
@@ -269,9 +325,14 @@ public void merge(AgentConfiguration cmd) {
             sb.append('\n');
         }
 
-        sb.append("URI to Ignite node REST server: ").append(nodeUri == null ? DFLT_NODE_URI : nodeUri).append('\n');
-        sb.append("URI to Ignite Console server  : ").append(srvUri == null ? DFLT_SERVER_URI : srvUri).append('\n');
-        sb.append("Path to agent property file   : ").append(configPath()).append('\n');
+        sb.append("URI to Ignite node REST server  : ")
+            .append(nodeURIs == null ? DFLT_NODE_URI : String.join(", ", nodeURIs)).append('\n');
+
+        if (nodeLogin != null)
+            sb.append("Login to Ignite node REST server: ").append(nodeLogin).append('\n');
+
+        sb.append("URI to Ignite Console server    : ").append(srvUri == null ? DFLT_SERVER_URI : srvUri).append('\n');
+        sb.append("Path to agent property file     : ").append(configPath()).append('\n');
 
         String drvFld = driversFolder();
 
@@ -282,8 +343,8 @@ public void merge(AgentConfiguration cmd) {
                 drvFld = new File(agentHome, "jdbc-drivers").getPath();
         }
 
-        sb.append("Path to JDBC drivers folder   : ").append(drvFld).append('\n');
-        sb.append("Demo mode                     : ").append(disableDemo() ? "disabled" : "enabled");
+        sb.append("Path to JDBC drivers folder     : ").append(drvFld).append('\n');
+        sb.append("Demo mode                       : ").append(disableDemo() ? "disabled" : "enabled");
 
         return sb.toString();
     }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 385ce08a12289..9340417248da1 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -32,7 +32,7 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.net.UnknownHostException;
-import java.security.cert.X509Certificate;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
@@ -43,12 +43,10 @@
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLHandshakeException;
 import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
 import org.apache.ignite.console.agent.handlers.ClusterListener;
-import org.apache.ignite.console.agent.handlers.DemoListener;
-import org.apache.ignite.console.agent.rest.RestExecutor;
 import org.apache.ignite.console.agent.handlers.DatabaseListener;
 import org.apache.ignite.console.agent.handlers.RestListener;
+import org.apache.ignite.console.agent.rest.RestExecutor;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
 import org.json.JSONArray;
@@ -64,6 +62,7 @@
 import static io.socket.client.Socket.EVENT_ERROR;
 import static org.apache.ignite.console.agent.AgentUtils.fromJSON;
 import static org.apache.ignite.console.agent.AgentUtils.toJSON;
+import static org.apache.ignite.console.agent.AgentUtils.trustManager;
 
 /**
  * Ignite Web Agent launcher.
@@ -72,21 +71,6 @@ public class AgentLauncher {
     /** */
     private static final Logger log = LoggerFactory.getLogger(AgentLauncher.class);
 
-    /** */
-    private static final String EVENT_CLUSTER_BROADCAST_START = "cluster:broadcast:start";
-
-    /** */
-    private static final String EVENT_CLUSTER_BROADCAST_STOP = "cluster:broadcast:stop";
-
-    /** */
-    private static final String EVENT_CLUSTER_DISCONNECTED = "cluster:disconnected";
-
-    /** */
-    private static final String EVENT_DEMO_BROADCAST_START = "demo:broadcast:start";
-
-    /** */
-    private static final String EVENT_DEMO_BROADCAST_STOP = "demo:broadcast:stop";
-
     /** */
     private static final String EVENT_SCHEMA_IMPORT_DRIVERS = "schemaImport:drivers";
 
@@ -103,7 +87,7 @@ public class AgentLauncher {
     private static final String EVENT_NODE_REST = "node:rest";
 
     /** */
-    private static final String EVENT_RESET_TOKENS = "agent:reset:token";
+    private static final String EVENT_RESET_TOKEN = "agent:reset:token";
 
     /** */
     private static final String EVENT_LOG_WARNING = "log:warn";
@@ -116,119 +100,79 @@ public class AgentLauncher {
         SLF4JBridgeHandler.install();
     }
 
-    /**
-     * Create a trust manager that trusts all certificates It is not using a particular keyStore
-     */
-    private static TrustManager[] getTrustManagers() {
-        return new TrustManager[] {
-            new X509TrustManager() {
-                /** {@inheritDoc} */
-                @Override public X509Certificate[] getAcceptedIssuers() {
-                    return new X509Certificate[0];
-                }
-
-                /** {@inheritDoc} */
-                @Override public void checkClientTrusted(X509Certificate[] certs, String authType) {
-                }
-
-                /** {@inheritDoc} */
-                @Override public void checkServerTrusted(X509Certificate[] certs, String authType) {
-                }
-            }};
-    }
-
     /**
      * On error listener.
      */
-    private static final Emitter.Listener onError = new Emitter.Listener() {
-        @Override public void call(Object... args) {
-            Throwable e = (Throwable)args[0];
+    private static final Emitter.Listener onError = args -> {
+        Throwable e = (Throwable)args[0];
 
-            ConnectException ce = X.cause(e, ConnectException.class);
+        ConnectException ce = X.cause(e, ConnectException.class);
 
-            if (ce != null)
-                log.error("Failed to establish connection to server (connection refused).");
-            else {
-                Exception ignore = X.cause(e, SSLHandshakeException.class);
+        if (ce != null)
+            log.error("Failed to establish connection to server (connection refused).");
+        else {
+            Exception ignore = X.cause(e, SSLHandshakeException.class);
 
-                if (ignore != null) {
-                    log.error("Failed to establish SSL connection to server, due to errors with SSL handshake.");
-                    log.error("Add to environment variable JVM_OPTS parameter \"-Dtrust.all=true\" to skip certificate validation in case of using self-signed certificate.");
+            if (ignore != null) {
+                log.error("Failed to establish SSL connection to server, due to errors with SSL handshake.");
+                log.error("Add to environment variable JVM_OPTS parameter \"-Dtrust.all=true\" to skip certificate validation in case of using self-signed certificate.");
 
-                    System.exit(1);
-                }
+                System.exit(1);
+            }
 
-                ignore = X.cause(e, UnknownHostException.class);
+            ignore = X.cause(e, UnknownHostException.class);
 
-                if (ignore != null) {
-                    log.error("Failed to establish connection to server, due to errors with DNS or missing proxy settings.");
-                    log.error("Documentation for proxy configuration can be found here: http://apacheignite.readme.io/docs/web-agent#section-proxy-configuration");
+            if (ignore != null) {
+                log.error("Failed to establish connection to server, due to errors with DNS or missing proxy settings.");
+                log.error("Documentation for proxy configuration can be found here: http://apacheignite.readme.io/docs/web-agent#section-proxy-configuration");
 
-                    System.exit(1);
-                }
-
-                ignore = X.cause(e, IOException.class);
+                System.exit(1);
+            }
 
-                if (ignore != null && "404".equals(ignore.getMessage())) {
-                    log.error("Failed to receive response from server (connection refused).");
+            ignore = X.cause(e, IOException.class);
 
-                    return;
-                }
+            if (ignore != null && "404".equals(ignore.getMessage())) {
+                log.error("Failed to receive response from server (connection refused).");
 
-                if (ignore != null && "407".equals(ignore.getMessage())) {
-                    log.error("Failed to establish connection to server, due to proxy requires authentication.");
+                return;
+            }
 
-                    String userName = System.getProperty("https.proxyUsername", System.getProperty("http.proxyUsername"));
+            if (ignore != null && "407".equals(ignore.getMessage())) {
+                log.error("Failed to establish connection to server, due to proxy requires authentication.");
 
-                    if (userName == null || userName.trim().isEmpty())
-                        userName = readLine("Enter proxy user name: ");
-                    else
-                        System.out.println("Read username from system properties: " + userName);
+                String userName = System.getProperty("https.proxyUsername", System.getProperty("http.proxyUsername"));
 
-                    char[] pwd = readPassword("Enter proxy password: ");
+                if (userName == null || userName.trim().isEmpty())
+                    userName = readLine("Enter proxy user name: ");
+                else
+                    System.out.println("Read username from system properties: " + userName);
 
-                    final PasswordAuthentication pwdAuth = new PasswordAuthentication(userName, pwd);
+                char[] pwd = readPassword("Enter proxy password: ");
 
-                    Authenticator.setDefault(new Authenticator() {
-                        @Override protected PasswordAuthentication getPasswordAuthentication() {
-                            return pwdAuth;
-                        }
-                    });
+                final PasswordAuthentication pwdAuth = new PasswordAuthentication(userName, pwd);
 
-                    return;
-                }
+                Authenticator.setDefault(new Authenticator() {
+                    @Override protected PasswordAuthentication getPasswordAuthentication() {
+                        return pwdAuth;
+                    }
+                });
 
-                log.error("Connection error.", e);
+                return;
             }
+
+            log.error("Connection error.", e);
         }
     };
 
     /**
      * On disconnect listener.
      */
-    private static final Emitter.Listener onDisconnect = new Emitter.Listener() {
-        @Override public void call(Object... args) {
-            log.error("Connection closed: {}", args);
-        }
-    };
+    private static final Emitter.Listener onDisconnect = args -> log.error("Connection closed: {}", args);
 
     /**
      * On token reset listener.
      */
-    private static final Emitter.Listener onLogWarning = new Emitter.Listener() {
-        @Override public void call(Object... args) {
-            log.warn(String.valueOf(args[0]));
-        }
-    };
-
-    /**
-     * On demo start request.
-     */
-    private static final Emitter.Listener onDemoStart = new Emitter.Listener() {
-        @Override public void call(Object... args) {
-            log.warn(String.valueOf(args[0]));
-        }
-    };
+    private static final Emitter.Listener onLogWarning = args -> log.warn(String.valueOf(args[0]));
 
     /**
      * @param fmt Format string.
@@ -311,28 +255,26 @@ public static void main(String[] args) throws Exception {
         System.out.println(cfg);
         System.out.println();
 
-        if (cfg.tokens() == null) {
-            String webHost;
+        URI uri;
 
-            try {
-                webHost = new URI(cfg.serverUri()).getHost();
-            }
-            catch (URISyntaxException e) {
-                log.error("Failed to parse Ignite Web Console uri", e);
+        try {
+            uri = new URI(cfg.serverUri());
+        }
+        catch (URISyntaxException e) {
+            log.error("Failed to parse Ignite Web Console uri", e);
 
-                return;
-            }
+            return;
+        }
 
+        if (cfg.tokens() == null) {
             System.out.println("Security token is required to establish connection to the web console.");
-            System.out.println(String.format("It is available on the Profile page: https://%s/profile", webHost));
+            System.out.println(String.format("It is available on the Profile page: https://%s/profile", uri.getHost()));
 
             String tokens = String.valueOf(readPassword("Enter security tokens separated by comma: "));
 
-            cfg.tokens(Arrays.asList(tokens.trim().split(",")));
+            cfg.tokens(new ArrayList<>(Arrays.asList(tokens.trim().split(","))));
         }
 
-        URI uri = URI.create(cfg.serverUri());
-
         // Create proxy authenticator using passed properties.
         switch (uri.getScheme()) {
             case "http":
@@ -352,6 +294,29 @@ public static void main(String[] args) throws Exception {
                 // No-op.
         }
 
+        List<String> nodeURIs = cfg.nodeURIs();
+
+        for (int i = nodeURIs.size() - 1; i >= 0; i--) {
+            String nodeURI = nodeURIs.get(i);
+
+            try {
+                new URI(nodeURI);
+            }
+            catch (URISyntaxException ignored) {
+                log.warn("Failed to parse Ignite node URI: {}.", nodeURI);
+
+                nodeURIs.remove(i);
+            }
+        }
+
+        if (nodeURIs.isEmpty()) {
+            log.error("Failed to find valid URIs for connect to Ignite node via REST. Please check agent settings");
+
+            return;
+        }
+
+        cfg.nodeURIs(nodeURIs);
+
         IO.Options opts = new IO.Options();
 
         opts.path = "/agents";
@@ -361,102 +326,95 @@ public static void main(String[] args) throws Exception {
             SSLContext ctx = SSLContext.getInstance("TLS");
 
             // Create an SSLContext that uses our TrustManager
-            ctx.init(null, getTrustManagers(), null);
+            ctx.init(null, new TrustManager[] {trustManager()}, null);
 
             opts.sslContext = ctx;
         }
 
         final Socket client = IO.socket(uri, opts);
-        final RestExecutor restExecutor = new RestExecutor(cfg.nodeUri());
 
-        try {
-            final ClusterListener clusterLsnr = new ClusterListener(client, restExecutor);
-            final DemoListener demoHnd = new DemoListener(client, restExecutor);
-
-            Emitter.Listener onConnect = new Emitter.Listener() {
-                @Override public void call(Object... args) {
-                    log.info("Connection established.");
-
-                    JSONObject authMsg = new JSONObject();
+        try (RestExecutor restExecutor = new RestExecutor();
+             ClusterListener clusterLsnr = new ClusterListener(cfg, client, restExecutor)) {
+            Emitter.Listener onConnect = connectRes -> {
+                log.info("Connection established.");
 
-                    try {
-                        authMsg.put("tokens", toJSON(cfg.tokens()));
-                        authMsg.put("disableDemo", cfg.disableDemo());
+                JSONObject authMsg = new JSONObject();
 
-                        String clsName = AgentLauncher.class.getSimpleName() + ".class";
+                try {
+                    authMsg.put("tokens", toJSON(cfg.tokens()));
+                    authMsg.put("disableDemo", cfg.disableDemo());
 
-                        String clsPath = AgentLauncher.class.getResource(clsName).toString();
+                    String clsName = AgentLauncher.class.getSimpleName() + ".class";
 
-                        if (clsPath.startsWith("jar")) {
-                            String manifestPath = clsPath.substring(0, clsPath.lastIndexOf('!') + 1) +
-                                "/META-INF/MANIFEST.MF";
+                    String clsPath = AgentLauncher.class.getResource(clsName).toString();
 
-                            Manifest manifest = new Manifest(new URL(manifestPath).openStream());
+                    if (clsPath.startsWith("jar")) {
+                        String manifestPath = clsPath.substring(0, clsPath.lastIndexOf('!') + 1) +
+                            "/META-INF/MANIFEST.MF";
 
-                            Attributes attr = manifest.getMainAttributes();
+                        Manifest manifest = new Manifest(new URL(manifestPath).openStream());
 
-                            authMsg.put("ver", attr.getValue("Implementation-Version"));
-                            authMsg.put("bt", attr.getValue("Build-Time"));
-                        }
+                        Attributes attr = manifest.getMainAttributes();
 
-                        client.emit("agent:auth", authMsg, new Ack() {
-                            @Override public void call(Object... args) {
-                                if (args != null) {
-                                    if (args[0] instanceof String) {
-                                        log.error((String)args[0]);
+                        authMsg.put("ver", attr.getValue("Implementation-Version"));
+                        authMsg.put("bt", attr.getValue("Build-Time"));
+                    }
 
-                                        System.exit(1);
-                                    }
+                    client.emit("agent:auth", authMsg, (Ack) authRes -> {
+                        if (authRes != null) {
+                            if (authRes[0] instanceof String) {
+                                log.error((String)authRes[0]);
 
-                                    if (args[0] == null && args[1] instanceof JSONArray) {
-                                        try {
-                                            List<String> activeTokens = fromJSON(args[1], List.class);
+                                System.exit(1);
+                            }
 
-                                            if (!F.isEmpty(activeTokens)) {
-                                                Collection<String> missedTokens = cfg.tokens();
+                            if (authRes[0] == null && authRes[1] instanceof JSONArray) {
+                                try {
+                                    List<String> activeTokens = fromJSON(authRes[1], List.class);
 
-                                                cfg.tokens(activeTokens);
+                                    if (!F.isEmpty(activeTokens)) {
+                                        Collection<String> missedTokens = cfg.tokens();
 
-                                                missedTokens.removeAll(activeTokens);
+                                        cfg.tokens(activeTokens);
 
-                                                if (!F.isEmpty(missedTokens)) {
-                                                    String tokens = F.concat(missedTokens, ", ");
+                                        missedTokens.removeAll(activeTokens);
 
-                                                    log.warn("Failed to authenticate with token(s): {}. " +
-                                                        "Please reload agent archive or check settings", tokens);
-                                                }
+                                        if (!F.isEmpty(missedTokens)) {
+                                            String tokens = F.concat(missedTokens, ", ");
 
-                                                log.info("Authentication success.");
+                                            log.warn("Failed to authenticate with token(s): {}. " +
+                                                "Please reload agent archive or check settings", tokens);
+                                        }
 
-                                                clusterLsnr.watch();
+                                        log.info("Authentication success.");
 
-                                                return;
-                                            }
-                                        }
-                                        catch (Exception e) {
-                                            log.error("Failed to authenticate agent. Please check agent\'s tokens", e);
+                                        clusterLsnr.watch();
 
-                                            System.exit(1);
-                                        }
+                                        return;
                                     }
                                 }
+                                catch (Exception e) {
+                                    log.error("Failed to authenticate agent. Please check agent\'s tokens", e);
 
-                                log.error("Failed to authenticate agent. Please check agent\'s tokens");
-
-                                System.exit(1);
+                                    System.exit(1);
+                                }
                             }
-                        });
-                    }
-                    catch (JSONException | IOException e) {
-                        log.error("Failed to construct authentication message", e);
+                        }
 
-                        client.close();
-                    }
+                        log.error("Failed to authenticate agent. Please check agent\'s tokens");
+
+                        System.exit(1);
+                    });
+                }
+                catch (JSONException | IOException e) {
+                    log.error("Failed to construct authentication message", e);
+
+                    client.close();
                 }
             };
 
             DatabaseListener dbHnd = new DatabaseListener(cfg);
-            RestListener restHnd = new RestListener(restExecutor);
+            RestListener restHnd = new RestListener(cfg, restExecutor);
 
             final CountDownLatch latch = new CountDownLatch(1);
 
@@ -468,23 +426,17 @@ public static void main(String[] args) throws Exception {
                 .on(EVENT_ERROR, onError)
                 .on(EVENT_DISCONNECT, onDisconnect)
                 .on(EVENT_LOG_WARNING, onLogWarning)
-                .on(EVENT_CLUSTER_BROADCAST_START, clusterLsnr.start())
-                .on(EVENT_CLUSTER_BROADCAST_STOP, clusterLsnr.stop())
-                .on(EVENT_DEMO_BROADCAST_START, demoHnd.start())
-                .on(EVENT_DEMO_BROADCAST_STOP, demoHnd.stop())
-                .on(EVENT_RESET_TOKENS, new Emitter.Listener() {
-                    @Override public void call(Object... args) {
-                        String tok = String.valueOf(args[0]);
+                .on(EVENT_RESET_TOKEN, res -> {
+                    String tok = String.valueOf(res[0]);
 
-                        log.warn("Security token has been reset: {}", tok);
+                    log.warn("Security token has been reset: {}", tok);
 
-                        cfg.tokens().remove(tok);
+                    cfg.tokens().remove(tok);
 
-                        if (cfg.tokens().isEmpty()) {
-                            client.off();
+                    if (cfg.tokens().isEmpty()) {
+                        client.off();
 
-                            latch.countDown();
-                        }
+                        latch.countDown();
                     }
                 })
                 .on(EVENT_SCHEMA_IMPORT_DRIVERS, dbHnd.availableDriversListener())
@@ -498,8 +450,6 @@ public static void main(String[] args) throws Exception {
             latch.await();
         }
         finally {
-            restExecutor.stop();
-
             client.close();
         }
     }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java
index 797951c5fcc99..38edd976850e5 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentUtils.java
@@ -24,7 +24,9 @@
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.security.ProtectionDomain;
+import java.security.cert.X509Certificate;
 import java.util.Arrays;
+import javax.net.ssl.X509TrustManager;
 import org.apache.log4j.Logger;
 import org.json.JSONArray;
 import org.json.JSONObject;
@@ -182,4 +184,26 @@ public static Object toJSON(Object obj) {
     public static <T> T fromJSON(Object obj, Class<T> toValType) throws IllegalArgumentException {
         return MAPPER.convertValue(obj, toValType);
     }
+
+    /**
+     * Create a trust manager that trusts all certificates It is not using a particular keyStore
+     */
+    public static X509TrustManager trustManager() {
+        return new X509TrustManager() {
+            /** {@inheritDoc} */
+            @Override public X509Certificate[] getAcceptedIssuers() {
+                return new X509Certificate[0];
+            }
+
+            /** {@inheritDoc} */
+            @Override public void checkClientTrusted(X509Certificate[] certs, String authType) {
+                // No-op.
+            }
+
+            /** {@inheritDoc} */
+            @Override public void checkServerTrusted(X509Certificate[] certs, String authType) {
+                // No-op.
+            }
+        };
+    }
 }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
index ace208744abd8..33e4c2ba3becb 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
@@ -28,8 +28,10 @@
 import java.util.concurrent.Executors;
 import java.util.zip.GZIPOutputStream;
 import org.apache.commons.codec.binary.Base64OutputStream;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.console.agent.rest.RestResult;
-import org.apache.log4j.Logger;
+import org.apache.ignite.logger.slf4j.Slf4jLogger;
+import org.slf4j.LoggerFactory;
 
 import static org.apache.ignite.console.agent.AgentUtils.removeCallback;
 import static org.apache.ignite.console.agent.AgentUtils.fromJSON;
@@ -40,15 +42,15 @@
  * Base class for web socket handlers.
  */
 abstract class AbstractListener implements Emitter.Listener {
+    /** */
+    final IgniteLogger log = new Slf4jLogger(LoggerFactory.getLogger(AbstractListener.class));
+
     /** UTF8 charset. */
     private static final Charset UTF8 = Charset.forName("UTF-8");
 
     /** */
     private ExecutorService pool;
 
-    /** */
-    final Logger log = Logger.getLogger(this.getClass().getName());
-
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public final void call(Object... args) {
@@ -69,34 +71,32 @@ else if (args.length == 1)
             if (pool == null)
                 pool = newThreadPool();
 
-            pool.submit(new Runnable() {
-                @Override public void run() {
-                    try {
-                        Object res = execute(params);
+            pool.submit(() -> {
+                try {
+                    Object res = execute(params);
 
-                        // TODO IGNITE-6127 Temporary solution until GZip support for socket.io-client-java.
-                        // See: https://github.com/socketio/socket.io-client-java/issues/312
-                        // We can GZip manually for now.
-                        if (res instanceof RestResult) {
-                            RestResult restRes = (RestResult) res;
+                    // TODO IGNITE-6127 Temporary solution until GZip support for socket.io-client-java.
+                    // See: https://github.com/socketio/socket.io-client-java/issues/312
+                    // We can GZip manually for now.
+                    if (res instanceof RestResult) {
+                        RestResult restRes = (RestResult) res;
 
-                            if (restRes.getData() != null) {
-                                ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
-                                Base64OutputStream b64os = new Base64OutputStream(baos, true, 0, null);
-                                GZIPOutputStream gzip = new GZIPOutputStream(b64os);
+                        if (restRes.getData() != null) {
+                            ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
+                            Base64OutputStream b64os = new Base64OutputStream(baos, true, 0, null);
+                            GZIPOutputStream gzip = new GZIPOutputStream(b64os);
 
-                                gzip.write(restRes.getData().getBytes(UTF8));
+                            gzip.write(restRes.getData().getBytes(UTF8));
 
-                                gzip.close();
+                            gzip.close();
 
-                                restRes.zipData(baos.toString());
-                            }
+                            restRes.zipData(baos.toString());
                         }
-
-                        cb.call(null, toJSON(res));
-                    } catch (Exception e) {
-                        cb.call(e, null);
                     }
+
+                    cb.call(null, toJSON(res));
+                } catch (Exception e) {
+                    cb.call(e, null);
                 }
             });
         }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
index 86b9ea5847d3e..0c7560c207da9 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
@@ -17,10 +17,8 @@
 
 package org.apache.ignite.console.agent.handlers;
 
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import io.socket.client.Socket;
-import io.socket.emitter.Emitter;
+import java.io.IOException;
 import java.net.ConnectException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -33,6 +31,11 @@
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
 import org.apache.ignite.IgniteLogger;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.ignite.console.agent.AgentConfiguration;
 import org.apache.ignite.console.agent.rest.RestExecutor;
 import org.apache.ignite.console.agent.rest.RestResult;
 import org.apache.ignite.internal.processors.rest.client.message.GridClientNodeBean;
@@ -51,16 +54,29 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CLIENT_MODE;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IPS;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_SUCCESS;
+import static org.apache.ignite.internal.processors.rest.client.message.GridClientResponse.STATUS_FAILED;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.sortAddresses;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.splitAddresses;
 
 /**
  * API to transfer topology from Ignite cluster available by node-uri.
  */
-public class ClusterListener {
+public class ClusterListener implements AutoCloseable {
     /** */
     private static final IgniteLogger log = new Slf4jLogger(LoggerFactory.getLogger(ClusterListener.class));
 
+    /** */
+    private static final IgniteProductVersion IGNITE_2_1 = IgniteProductVersion.fromString("2.1.0");
+
+    /** */
+    private static final IgniteProductVersion IGNITE_2_3 = IgniteProductVersion.fromString("2.3.0");
+
+    /** Unique Visor key to get events last order. */
+    private static final String EVT_LAST_ORDER_KEY = "WEB_AGENT_" + UUID.randomUUID().toString();
+
+    /** Unique Visor key to get events throttle counter. */
+    private static final String EVT_THROTTLE_CNTR_KEY = "WEB_AGENT_" + UUID.randomUUID().toString();
+
     /** */
     private static final String EVENT_CLUSTER_CONNECTED = "cluster:connected";
 
@@ -82,9 +98,6 @@ public class ClusterListener {
     /** */
     private final WatchTask watchTask = new WatchTask();
 
-    /** */
-    private final BroadcastTask broadcastTask = new BroadcastTask();
-
     /** */
     private static final IgniteClosure<UUID, String> ID2ID8 = new IgniteClosure<UUID, String>() {
         @Override public String apply(UUID nid) {
@@ -97,10 +110,7 @@ public class ClusterListener {
     };
 
     /** */
-    private static final ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
-
-    /** */
-    private ScheduledFuture<?> refreshTask;
+    private AgentConfiguration cfg;
 
     /** */
     private Socket client;
@@ -108,11 +118,18 @@ public class ClusterListener {
     /** */
     private RestExecutor restExecutor;
 
+    /** */
+    private static final ScheduledExecutorService pool = Executors.newScheduledThreadPool(1);
+
+    /** */
+    private ScheduledFuture<?> refreshTask;
+
     /**
      * @param client Client.
      * @param restExecutor Client.
      */
-    public ClusterListener(Socket client, RestExecutor restExecutor) {
+    public ClusterListener(AgentConfiguration cfg, Socket client, RestExecutor restExecutor) {
+        this.cfg = cfg;
         this.client = client;
         this.restExecutor = restExecutor;
     }
@@ -159,32 +176,11 @@ public void watch() {
         refreshTask = pool.scheduleWithFixedDelay(watchTask, 0L, DFLT_TIMEOUT, TimeUnit.MILLISECONDS);
     }
 
-    /**
-     * Start broadcast topology to server-side.
-     */
-    public Emitter.Listener start() {
-        return new Emitter.Listener() {
-            @Override public void call(Object... args) {
-                safeStopRefresh();
-
-                final long timeout = args.length > 1 && args[1] instanceof Long ? (long)args[1] : DFLT_TIMEOUT;
-
-                refreshTask = pool.scheduleWithFixedDelay(broadcastTask, 0L, timeout, TimeUnit.MILLISECONDS);
-            }
-        };
-    }
-
-    /**
-     * Stop broadcast topology to server-side.
-     */
-    public Emitter.Listener stop() {
-        return new Emitter.Listener() {
-            @Override public void call(Object... args) {
-                refreshTask.cancel(true);
+    /** {@inheritDoc} */
+    @Override public void close() {
+        refreshTask.cancel(true);
 
-                watch();
-            }
-        };
+        pool.shutdownNow();
     }
 
     /** */
@@ -210,6 +206,9 @@ private static class TopologySnapshot {
         /** */
         private boolean active;
 
+        /** */
+        private boolean secured;
+
         /**
          * Helper method to get attribute.
          *
@@ -231,6 +230,7 @@ private static <T> T attribute(Map<String, Object> attrs, String name) {
             addrs = U.newHashMap(sz);
             clients = U.newHashMap(sz);
             active = false;
+            secured = false;
 
             for (GridClientNodeBean node : nodes) {
                 UUID nid = node.getNodeId();
@@ -247,7 +247,7 @@ private static <T> T attribute(Map<String, Object> attrs, String name) {
                 clients.put(nid, client);
 
                 Collection<String> nodeAddrs = client
-                    ? splitAddresses((String)attribute(attrs, ATTR_IPS))
+                    ? splitAddresses(attribute(attrs, ATTR_IPS))
                     : node.getTcpAddresses();
 
                 String firstIP = F.first(sortAddresses(nodeAddrs));
@@ -293,6 +293,20 @@ public void setActive(boolean active) {
             this.active = active;
         }
 
+        /**
+         * @return {@code true} If cluster has configured security.
+         */
+        public boolean isSecured() {
+            return secured;
+        }
+
+        /**
+         * @param secured Configured security flag.
+         */
+        public void setSecured(boolean secured) {
+            this.secured = secured;
+        }
+
         /**
          * @return Cluster nodes IDs.
          */
@@ -339,54 +353,114 @@ boolean differentCluster(TopologySnapshot prev) {
 
     /** */
     private class WatchTask implements Runnable {
-        /** {@inheritDoc} */
-        @Override public void run() {
-            try {
-                RestResult res = restExecutor.topology(false, false);
+        /** */
+        private static final String EXPIRED_SES_ERROR_MSG = "Failed to handle request - unknown session token (maybe expired session)";
 
-                switch (res.getStatus()) {
-                    case STATUS_SUCCESS:
-                        List<GridClientNodeBean> nodes = MAPPER.readValue(res.getData(),
-                            new TypeReference<List<GridClientNodeBean>>() {});
+        /** */
+        private String sesTok;
 
-                        TopologySnapshot newTop = new TopologySnapshot(nodes);
+        /**
+         * Execute REST command under agent user.
+         *
+         * @param params Command params.
+         * @return Command result.
+         * @throws IOException If failed to execute.
+         */
+        private RestResult restCommand(Map<String, Object> params) throws IOException {
+            if (!F.isEmpty(sesTok))
+                params.put("sessionToken", sesTok);
+            else if (!F.isEmpty(cfg.nodeLogin()) && !F.isEmpty(cfg.nodePassword())) {
+                params.put("user", cfg.nodeLogin());
+                params.put("password", cfg.nodePassword());
+            }
 
-                        if (newTop.differentCluster(top))
-                            log.info("Connection successfully established to cluster with nodes: " + newTop.nid8());
+            RestResult res = restExecutor.sendRequest(cfg.nodeURIs(), params, null);
 
-                        boolean active = restExecutor.active(newTop.clusterVersion(), F.first(newTop.getNids()));
+            switch (res.getStatus()) {
+                case STATUS_SUCCESS:
+                    sesTok = res.getSessionToken();
 
-                        newTop.setActive(active);
+                    return res;
+                    
+                case STATUS_FAILED:
+                    if (res.getError().startsWith(EXPIRED_SES_ERROR_MSG)) {
+                        sesTok = null;
+                        
+                        params.remove("sessionToken");
 
-                        top = newTop;
+                        return restCommand(params);
+                    }
 
-                        client.emit(EVENT_CLUSTER_TOPOLOGY, toJSON(top));
+                default:
+                    return res;
+            }
+        }
 
-                        break;
+        /**
+         * Collect topology.
+         *
+         * @param full Full.
+         */
+        private RestResult topology(boolean full) throws IOException {
+            Map<String, Object> params = U.newHashMap(3);
 
-                    default:
-                        LT.warn(log, res.getError());
+            params.put("cmd", "top");
+            params.put("attr", true);
+            params.put("mtr", full);
 
-                        clusterDisconnect();
+            return restCommand(params);
+        }
+
+        /**
+         * @param ver Cluster version.
+         * @param nid Node ID.
+         * @return Cluster active state.
+         * @throws IOException If failed to collect cluster active state.
+         */
+        public boolean active(IgniteProductVersion ver, UUID nid) throws IOException {
+            Map<String, Object> params = U.newHashMap(10);
+
+            boolean v23 = ver.compareTo(IGNITE_2_3) >= 0;
+
+            if (v23)
+                params.put("cmd", "currentState");
+            else {
+                params.put("cmd", "exe");
+                params.put("name", "org.apache.ignite.internal.visor.compute.VisorGatewayTask");
+                params.put("p1", nid);
+                params.put("p2", "org.apache.ignite.internal.visor.node.VisorNodeDataCollectorTask");
+                params.put("p3", "org.apache.ignite.internal.visor.node.VisorNodeDataCollectorTaskArg");
+                params.put("p4", false);
+                params.put("p5", EVT_LAST_ORDER_KEY);
+                params.put("p6", EVT_THROTTLE_CNTR_KEY);
+
+                if (ver.compareTo(IGNITE_2_1) >= 0)
+                    params.put("p7", false);
+                else {
+                    params.put("p7", 10);
+                    params.put("p8", false);
                 }
             }
-            catch (ConnectException ignored) {
-                clusterDisconnect();
-            }
-            catch (Exception e) {
-                log.error("WatchTask failed", e);
 
-                clusterDisconnect();
+            RestResult res = restCommand(params);
+
+            switch (res.getStatus()) {
+                case STATUS_SUCCESS:
+                    if (v23)
+                        return Boolean.valueOf(res.getData());
+
+                    return res.getData().contains("\"active\":true");
+
+                default:
+                    throw new IOException(res.getError());
             }
         }
-    }
 
-    /** */
-    private class BroadcastTask implements Runnable {
+
         /** {@inheritDoc} */
         @Override public void run() {
             try {
-                RestResult res = restExecutor.topology(false, true);
+                RestResult res = topology(false);
 
                 switch (res.getStatus()) {
                     case STATUS_SUCCESS:
@@ -395,17 +469,17 @@ private class BroadcastTask implements Runnable {
 
                         TopologySnapshot newTop = new TopologySnapshot(nodes);
 
-                        if (top.differentCluster(newTop)) {
-                            clusterDisconnect();
-
+                        if (newTop.differentCluster(top))
                             log.info("Connection successfully established to cluster with nodes: " + newTop.nid8());
 
-                            watch();
-                        }
+                        boolean active = active(newTop.clusterVersion(), F.first(newTop.getNids()));
+
+                        newTop.setActive(active);
+                        newTop.setSecured(!F.isEmpty(res.getSessionToken()));
 
                         top = newTop;
 
-                        client.emit(EVENT_CLUSTER_TOPOLOGY, res.getData());
+                        client.emit(EVENT_CLUSTER_TOPOLOGY, toJSON(top));
 
                         break;
 
@@ -415,12 +489,13 @@ private class BroadcastTask implements Runnable {
                         clusterDisconnect();
                 }
             }
+            catch (ConnectException ignored) {
+                clusterDisconnect();
+            }
             catch (Exception e) {
-                log.error("BroadcastTask failed", e);
+                log.error("WatchTask failed", e);
 
                 clusterDisconnect();
-
-                watch();
             }
         }
     }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java
index 9da118d00d33e..b6bd623995c78 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DatabaseListener.java
@@ -125,7 +125,7 @@ public class DatabaseListener {
 
     /** */
     private final AbstractListener availableDriversLsnr = new AbstractListener() {
-        @Override public Object execute(Map<String, Object> args) throws Exception {
+        @Override public Object execute(Map<String, Object> args) {
             if (driversFolder == null) {
                 log.info("JDBC drivers folder not specified, returning empty list");
 
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DemoListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DemoListener.java
deleted file mode 100644
index ce420326c857b..0000000000000
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/DemoListener.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.console.agent.handlers;
-
-import io.socket.client.Ack;
-import io.socket.client.Socket;
-import io.socket.emitter.Emitter;
-import java.io.IOException;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ScheduledFuture;
-import java.util.concurrent.TimeUnit;
-import org.apache.ignite.console.agent.rest.RestExecutor;
-import org.apache.ignite.console.agent.rest.RestResult;
-import org.apache.ignite.console.demo.AgentClusterDemo;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.ignite.console.agent.AgentUtils.safeCallback;
-import static org.apache.ignite.console.agent.AgentUtils.toJSON;
-
-/**
- * API to retranslate topology from Ignite demo cluster.
- */
-public class DemoListener {
-    /** */
-    private static final String EVENT_DEMO_TOPOLOGY = "demo:topology";
-
-    /** Default timeout. */
-    private static final long DFLT_TIMEOUT = 3000L;
-
-    /** */
-    private static final Logger log = LoggerFactory.getLogger(DemoListener.class);
-
-    /** */
-    private static final ScheduledExecutorService pool = Executors.newScheduledThreadPool(2);
-
-    /** */
-    private ScheduledFuture<?> refreshTask;
-
-    /** */
-    private Socket client;
-
-    /** */
-    private RestExecutor restExecutor;
-
-    /**
-     * @param client Client.
-     * @param restExecutor Client.
-     */
-    public DemoListener(Socket client, RestExecutor restExecutor) {
-        this.client = client;
-        this.restExecutor = restExecutor;
-    }
-
-    /**
-     * Start broadcast topology to server-side.
-     */
-    public Emitter.Listener start() {
-        return new Emitter.Listener() {
-            @Override public void call(final Object... args) {
-                final Ack demoStartCb = safeCallback(args);
-
-                final long timeout = args.length > 1  && args[1] instanceof Long ? (long)args[1] : DFLT_TIMEOUT;
-
-                if (refreshTask != null)
-                    refreshTask.cancel(true);
-
-                final CountDownLatch latch = AgentClusterDemo.tryStart();
-
-                pool.schedule(new Runnable() {
-                    @Override public void run() {
-                        try {
-                            U.await(latch);
-
-                            demoStartCb.call();
-
-                            refreshTask = pool.scheduleWithFixedDelay(new Runnable() {
-                                @Override public void run() {
-                                    try {
-                                        RestResult top = restExecutor.topology(true, true);
-
-                                        client.emit(EVENT_DEMO_TOPOLOGY, toJSON(top));
-                                    }
-                                    catch (IOException e) {
-                                        log.info("Lost connection to the demo cluster", e);
-
-                                        stop().call(); // TODO WTF????
-                                    }
-                                }
-                            }, 0L, timeout, TimeUnit.MILLISECONDS);
-                        }
-                        catch (Exception e) {
-                            demoStartCb.call(e);
-                        }
-                    }
-                }, 0, TimeUnit.MILLISECONDS);
-            }
-        };
-    }
-
-    /**
-     * Stop broadcast topology to server-side.
-     */
-    public Emitter.Listener stop() {
-        return new Emitter.Listener() {
-            @Override public void call(Object... args) {
-                refreshTask.cancel(true);
-
-                AgentClusterDemo.stop();
-            }
-        };
-    }
-}
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
index 8855060807e3b..24c2097531c6a 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
@@ -20,19 +20,28 @@
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import org.apache.ignite.console.agent.AgentConfiguration;
 import org.apache.ignite.console.agent.rest.RestExecutor;
+import org.apache.ignite.console.agent.rest.RestResult;
+import org.apache.ignite.console.demo.AgentClusterDemo;
+import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
  * API to translate REST requests to Ignite cluster.
  */
 public class RestListener extends AbstractListener {
+    /** */
+    private final AgentConfiguration cfg;
+
     /** */
     private final RestExecutor restExecutor;
 
     /**
-     * @param restExecutor Config.
+     * @param cfg Config.
+     * @param restExecutor Executor.
      */
-    public RestListener(RestExecutor restExecutor) {
+    public RestListener(AgentConfiguration cfg, RestExecutor restExecutor) {
+        this.cfg = cfg;
         this.restExecutor = restExecutor;
     }
 
@@ -42,15 +51,10 @@ public RestListener(RestExecutor restExecutor) {
     }
 
     /** {@inheritDoc} */
-    @Override public Object execute(Map<String, Object> args) throws Exception {
+    @Override public Object execute(Map<String, Object> args) {
         if (log.isDebugEnabled())
             log.debug("Start parse REST command args: " + args);
 
-        String path = null;
-
-        if (args.containsKey("uri"))
-            path = args.get("uri").toString();
-
         Map<String, Object> params = null;
 
         if (args.containsKey("params"))
@@ -66,11 +70,24 @@ public RestListener(RestExecutor restExecutor) {
         if (args.containsKey("headers"))
             headers = (Map<String, Object>)args.get("headers");
 
-        String body = null;
+        try {
+            if (demo) {
+                if (AgentClusterDemo.getDemoUrl() == null) {
+                    AgentClusterDemo.tryStart().await();
+
+                    if (AgentClusterDemo.getDemoUrl() == null)
+                        return RestResult.fail(404, "Failed to send request because of embedded node for demo mode is not started yet.");
+                }
+
+                return restExecutor.sendRequest(AgentClusterDemo.getDemoUrl(), params, headers);
+            }
 
-        if (args.containsKey("body"))
-            body = args.get("body").toString();
+            return restExecutor.sendRequest(this.cfg.nodeURIs(), params, headers);
+        }
+        catch (Exception e) {
+            U.error(log, "Failed to execute REST command with parameters: " + params, e);
 
-        return restExecutor.execute(demo, path, params, headers, body);
+            return RestResult.fail(404, e.getMessage());
+        }
     }
 }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java
index f2892a6934f81..bb06c32579b86 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java
@@ -20,14 +20,11 @@
 import java.io.IOException;
 import java.io.StringWriter;
 import java.net.ConnectException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
 import java.util.concurrent.TimeUnit;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
 import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.core.JsonParser;
@@ -39,24 +36,20 @@
 import okhttp3.Dispatcher;
 import okhttp3.FormBody;
 import okhttp3.HttpUrl;
-import okhttp3.MediaType;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
-import okhttp3.RequestBody;
 import okhttp3.Response;
 import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.console.demo.AgentClusterDemo;
 import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper;
-import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.logger.slf4j.Slf4jLogger;
 import org.slf4j.LoggerFactory;
 
 import static com.fasterxml.jackson.core.JsonToken.END_ARRAY;
 import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
 import static com.fasterxml.jackson.core.JsonToken.START_ARRAY;
+import static org.apache.ignite.console.agent.AgentUtils.trustManager;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_AUTH_FAILED;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_FAILED;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_SUCCESS;
@@ -64,19 +57,7 @@
 /**
  * API to translate REST requests to Ignite cluster.
  */
-public class RestExecutor {
-    /** */
-    private static final IgniteProductVersion IGNITE_2_1 = IgniteProductVersion.fromString("2.1.0");
-
-    /** */
-    private static final IgniteProductVersion IGNITE_2_3 = IgniteProductVersion.fromString("2.3.0");
-
-    /** Unique Visor key to get events last order. */
-    private static final String EVT_LAST_ORDER_KEY = "WEB_AGENT_" + UUID.randomUUID().toString();
-
-    /** Unique Visor key to get events throttle counter. */
-    private static final String EVT_THROTTLE_CNTR_KEY = "WEB_AGENT_" + UUID.randomUUID().toString();
-
+public class RestExecutor implements AutoCloseable {
     /** */
     private static final IgniteLogger log = new Slf4jLogger(LoggerFactory.getLogger(RestExecutor.class));
 
@@ -86,33 +67,45 @@ public class RestExecutor {
     /** */
     private final OkHttpClient httpClient;
 
-    /** Node URLs. */
-    private Set<String> nodeUrls = new LinkedHashSet<>();
-
-    /** Latest alive node URL. */
-    private volatile String latestNodeUrl;
+    /** Index of alive node URI. */
+    private Map<List<String>, Integer> startIdxs = U.newHashMap(2);
 
     /**
      * Default constructor.
      */
-    public RestExecutor(String nodeUrl) {
-        Collections.addAll(nodeUrls, nodeUrl.split(","));
-
+    public RestExecutor() {
         Dispatcher dispatcher = new Dispatcher();
         
         dispatcher.setMaxRequests(Integer.MAX_VALUE);
         dispatcher.setMaxRequestsPerHost(Integer.MAX_VALUE);
 
-        httpClient = new OkHttpClient.Builder()
+        OkHttpClient.Builder builder = new OkHttpClient.Builder()
             .readTimeout(0, TimeUnit.MILLISECONDS)
-            .dispatcher(dispatcher)
-            .build();
+            .dispatcher(dispatcher);
+
+        // Workaround for use self-signed certificate
+        if (Boolean.getBoolean("trust.all")) {
+            try {
+                SSLContext ctx = SSLContext.getInstance("TLS");
+
+                // Create an SSLContext that uses our TrustManager
+                ctx.init(null, new TrustManager[] {trustManager()}, null);
+
+                builder.sslSocketFactory(ctx.getSocketFactory(), trustManager());
+
+                builder.hostnameVerifier((hostname, session) -> true);
+            } catch (Exception ignored) {
+                LT.warn(log, "Failed to initialize the Trust Manager for \"-Dtrust.all\" option to skip certificate validation.");
+            }
+        }
+
+        httpClient = builder.build();
     }
 
     /**
      * Stop HTTP client.
      */
-    public void stop() {
+    @Override public void close() {
         if (httpClient != null) {
             httpClient.dispatcher().executorService().shutdown();
 
@@ -121,28 +114,37 @@ public void stop() {
     }
 
     /** */
-    private RestResult sendRequest0(String nodeUrl, boolean demo, String path, Map<String, Object> params,
-        Map<String, Object> headers, String body) throws IOException {
-        if (demo && AgentClusterDemo.getDemoUrl() == null) {
-            try {
-                AgentClusterDemo.tryStart().await();
-            }
-            catch (InterruptedException ignore) {
-                throw new IllegalStateException("Failed to send request because of embedded node for demo mode is not started yet.");
+    private RestResult parseResponse(Response res) throws IOException {
+        if (res.isSuccessful()) {
+            RestResponseHolder holder = MAPPER.readValue(res.body().byteStream(), RestResponseHolder.class);
+
+            int status = holder.getSuccessStatus();
+
+            switch (status) {
+                case STATUS_SUCCESS:
+                    return RestResult.success(holder.getResponse(), holder.getSessionToken());
+
+                default:
+                    return RestResult.fail(status, holder.getError());
             }
         }
 
-        String url = demo ? AgentClusterDemo.getDemoUrl() : nodeUrl;
+        if (res.code() == 401)
+            return RestResult.fail(STATUS_AUTH_FAILED, "Failed to authenticate in cluster. " +
+                "Please check agent\'s login and password or node port.");
 
-        HttpUrl httpUrl = HttpUrl.parse(url);
+        if (res.code() == 404)
+            return RestResult.fail(STATUS_FAILED, "Failed connect to cluster.");
 
-        if (httpUrl == null)
-            throw new IllegalStateException("Failed to send request because of node URL is invalid: " + url);
+        return RestResult.fail(STATUS_FAILED, "Failed to execute REST command: " + res.message());
+    }
 
-        HttpUrl.Builder urlBuilder = httpUrl.newBuilder();
+    /** */
+    private RestResult sendRequest(String url, Map<String, Object> params, Map<String, Object> headers) throws IOException {
+        HttpUrl httpUrl = HttpUrl.parse(url);
 
-        if (path != null)
-            urlBuilder.addPathSegment(path);
+        HttpUrl.Builder urlBuilder = httpUrl.newBuilder()
+            .addPathSegment("ignite");
 
         final Request.Builder reqBuilder = new Request.Builder();
 
@@ -152,199 +154,51 @@ private RestResult sendRequest0(String nodeUrl, boolean demo, String path, Map<S
                     reqBuilder.addHeader(entry.getKey(), entry.getValue().toString());
         }
 
-        if (body != null) {
-            MediaType contentType = MediaType.parse("text/plain");
-
-            reqBuilder.post(RequestBody.create(contentType, body));
-        }
-        else {
-            FormBody.Builder formBody = new FormBody.Builder();
+        FormBody.Builder bodyParams = new FormBody.Builder();
 
-            if (params != null) {
-                for (Map.Entry<String, Object> entry : params.entrySet()) {
-                    if (entry.getValue() != null)
-                        formBody.add(entry.getKey(), entry.getValue().toString());
-                }
+        if (params != null) {
+            for (Map.Entry<String, Object> entry : params.entrySet()) {
+                if (entry.getValue() != null)
+                    bodyParams.add(entry.getKey(), entry.getValue().toString());
             }
-
-            reqBuilder.post(formBody.build());
         }
 
-        reqBuilder.url(urlBuilder.build());
+        reqBuilder.url(urlBuilder.build())
+            .post(bodyParams.build());
 
         try (Response resp = httpClient.newCall(reqBuilder.build()).execute()) {
-            if (resp.isSuccessful()) {
-                RestResponseHolder res = MAPPER.readValue(resp.body().byteStream(), RestResponseHolder.class);
-
-                int status = res.getSuccessStatus();
-
-                switch (status) {
-                    case STATUS_SUCCESS:
-                        return RestResult.success(res.getResponse());
-
-                    default:
-                        return RestResult.fail(status, res.getError());
-                }
-            }
-
-            if (resp.code() == 401)
-                return RestResult.fail(STATUS_AUTH_FAILED, "Failed to authenticate in cluster. " +
-                    "Please check agent\'s login and password or node port.");
-
-            if (resp.code() == 404)
-                return RestResult.fail(STATUS_FAILED, "Failed connect to cluster.");
-
-            return RestResult.fail(STATUS_FAILED, "Failed to execute REST command: " + resp.message());
+            return parseResponse(resp);
         }
     }
 
-    /**
-     * Send request to cluster.
-     *
-     * @param demo {@code true} If demo mode.
-     * @param path Request path.
-     * @param params Request params.
-     * @param headers Request headers.
-     * @param body Request body.
-     * @return Request result.
-     * @throws IOException If failed to process request.
-     */
-    private RestResult sendRequest(boolean demo, String path, Map<String, Object> params,
-        Map<String, Object> headers, String body) throws IOException {
-        String url = latestNodeUrl;
-
-        try {
-            if (F.isEmpty(url)) {
-                Iterator<String> it = nodeUrls.iterator();
-
-                while (it.hasNext()) {
-                    String nodeUrl = it.next();
-
-                    try {
-                        RestResult res = sendRequest0(nodeUrl, demo, path, params, headers, body);
+    /** */
+    public RestResult sendRequest(List<String> nodeURIs, Map<String, Object> params, Map<String, Object> headers) throws IOException {
+        Integer startIdx = startIdxs.getOrDefault(nodeURIs, 0);
 
-                        log.info("Connected to cluster [url=" + nodeUrl + "]");
+        for (int i = 0;  i < nodeURIs.size(); i++) {
+            Integer currIdx = (startIdx + i) % nodeURIs.size();
 
-                        latestNodeUrl = nodeUrl;
+            String nodeUrl = nodeURIs.get(currIdx);
 
-                        return res;
-                    }
-                    catch (ConnectException ignored) {
-                        String msg = "Failed connect to cluster [url=" + nodeUrl + ", parameters=" + params + "]";
+            try {
+                RestResult res = sendRequest(nodeUrl, params, headers);
 
-                        LT.warn(log, msg);
+                LT.info(log, "Connected to cluster [url=" + nodeUrl + "]");
 
-                        if (!it.hasNext())
-                            throw new ConnectException(msg);
-                    }
-                }
+                startIdxs.put(nodeURIs, currIdx);
 
-                throw new ConnectException("Failed connect to cluster [urls=" + nodeUrls + ", parameters=" + params + "]");
+                return res;
             }
-            else {
-                try {
-                    return sendRequest0(url, demo, path, params, headers, body);
-                }
-                catch (ConnectException e) {
-                    latestNodeUrl = null;
-
-                    if (nodeUrls.size() > 1)
-                        return sendRequest(demo, path, params, headers, body);
-
-                    throw e;
-                }
-            }        }
-        catch (ConnectException ce) {
-            LT.warn(log, "Failed connect to cluster. " +
-                "Please ensure that nodes have [ignite-rest-http] module in classpath " +
-                "(was copied from libs/optional to libs folder).");
-
-            throw ce;
-        }
-    }
-
-    /**
-     * @param demo Is demo node request.
-     * @param path Path segment.
-     * @param params Params.
-     * @param headers Headers.
-     * @param body Body.
-     */
-    public RestResult execute(boolean demo, String path, Map<String, Object> params,
-        Map<String, Object> headers, String body) {
-        if (log.isDebugEnabled())
-            log.debug("Start execute REST command [uri=/" + (path == null ? "" : path) +
-                ", parameters=" + params + "]");
-
-        try {
-            return sendRequest(demo, path, params, headers, body);
-        }
-        catch (Exception e) {
-            U.error(log, "Failed to execute REST command [uri=/" + (path == null ? "" : path) +
-                ", parameters=" + params + "]", e);
-
-            return RestResult.fail(404, e.getMessage());
-        }
-    }
-
-    /**
-     * @param demo {@code true} in case of demo mode.
-     * @param full Flag indicating whether to collect metrics or not.
-     * @throws IOException If failed to collect topology.
-     */
-    public RestResult topology(boolean demo, boolean full) throws IOException {
-        Map<String, Object> params = new HashMap<>(3);
-
-        params.put("cmd", "top");
-        params.put("attr", true);
-        params.put("mtr", full);
-
-        return sendRequest(demo, "ignite", params, null, null);
-    }
-
-    /**
-     * @param ver Cluster version.
-     * @param nid Node ID.
-     * @return Cluster active state.
-     * @throws IOException If failed to collect cluster active state.
-     */
-    public boolean active(IgniteProductVersion ver, UUID nid) throws IOException {
-        Map<String, Object> params = new HashMap<>();
-
-        boolean v23 = ver.compareTo(IGNITE_2_3) >= 0;
-
-        if (v23)
-            params.put("cmd", "currentState");
-        else {
-            params.put("cmd", "exe");
-            params.put("name", "org.apache.ignite.internal.visor.compute.VisorGatewayTask");
-            params.put("p1", nid);
-            params.put("p2", "org.apache.ignite.internal.visor.node.VisorNodeDataCollectorTask");
-            params.put("p3", "org.apache.ignite.internal.visor.node.VisorNodeDataCollectorTaskArg");
-            params.put("p4", false);
-            params.put("p5", EVT_LAST_ORDER_KEY);
-            params.put("p6", EVT_THROTTLE_CNTR_KEY);
-
-            if (ver.compareTo(IGNITE_2_1) >= 0)
-                params.put("p7", false);
-            else {
-                params.put("p7", 10);
-                params.put("p8", false);
+            catch (ConnectException ignored) {
+                // No-op.
             }
         }
 
-        RestResult res = sendRequest(false, "ignite", params, null, null);
-
-        switch (res.getStatus()) {
-            case STATUS_SUCCESS:
-                if (v23)
-                    return Boolean.valueOf(res.getData());
+        LT.warn(log, "Failed connect to cluster. " +
+            "Please ensure that nodes have [ignite-rest-http] module in classpath " +
+            "(was copied from libs/optional to libs folder).");
 
-                return res.getData().contains("\"active\":true");
-
-            default:
-                throw new IOException(res.getError());
-        }
+        throw new ConnectException("Failed connect to cluster [urls=" + nodeURIs + ", parameters=" + params + "]");
     }
 
     /**
@@ -361,7 +215,7 @@ private static class RestResponseHolder {
         private String res;
 
         /** Session token string representation. */
-        private String sesTokStr;
+        private String sesTok;
 
         /**
          * @return {@code True} if this request was successful.
@@ -410,14 +264,14 @@ public void setResponse(String res) {
          * @return String representation of session token.
          */
         public String getSessionToken() {
-            return sesTokStr;
+            return sesTok;
         }
 
         /**
          * @param sesTokStr String representation of session token.
          */
         public void setSessionToken(String sesTokStr) {
-            this.sesTokStr = sesTokStr;
+            this.sesTok = sesTokStr;
         }
     }
 
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestResult.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestResult.java
index 962ffb6709764..fc8b4e9349e48 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestResult.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestResult.java
@@ -30,6 +30,9 @@ public class RestResult {
     /** The field contains result of command. */
     private String data;
 
+    /** Session token string representation. */
+    private String sesTok;
+
     /** Flag of zipped data. */
     private boolean zipped;
 
@@ -57,8 +60,12 @@ public static RestResult fail(int status, String error) {
      * @param data The field contains result of command.
      * @return Request result.
      */
-    public static RestResult success(String data) {
-        return new RestResult(0, null, data);
+    public static RestResult success(String data, String sesTok) {
+        RestResult res = new RestResult(0, null, data);
+
+        res.sesTok = sesTok;
+
+        return res;
     }
 
     /**
@@ -82,6 +89,13 @@ public String getData() {
         return data;
     }
 
+    /**
+     * @return String representation of session token.
+     */
+    public String getSessionToken() {
+        return sesTok;
+    }
+
     /**
      * @param data Set zipped data.
      */
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
index 1fd286c3a5247..2555ee15f2dab 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
@@ -21,6 +21,7 @@
 import java.nio.file.Paths;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -100,7 +101,7 @@ public class AgentClusterDemo {
     private static CountDownLatch initLatch = new CountDownLatch(1);
 
     /** */
-    private static volatile String demoUrl;
+    private static volatile List<String> demoUrl;
 
     /**
      * Configure node.
@@ -204,7 +205,7 @@ private static void deployServices(IgniteServices services) {
     }
 
     /** */
-    public static String getDemoUrl() {
+    public static List<String> getDemoUrl() {
         return demoUrl;
     }
 
@@ -269,7 +270,7 @@ public static CountDownLatch tryStart() {
 
                             log.info("DEMO: Started embedded node for demo purpose [TCP binary port={}, Jetty REST port={}]", port, jettyPort);
 
-                            demoUrl = String.format("http://%s:%d", jettyHost, jettyPort);
+                            demoUrl = Collections.singletonList(String.format("http://%s:%d", jettyHost, jettyPort));
 
                             initLatch.countDown();
                         }

From 13c36fef466f76393c6e6307638ed4a2ff4fd081 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 22 Jun 2018 17:00:11 +0700
Subject: [PATCH 310/543] GG-14087 Backport IGNITE-8428 Web Console:
 Implemented connection to secured cluster.

(cherry picked from commit c8be8c2b8e8a474a90735d6e9eaebc67c1c8e3de)
---
 .../rest/AbstractRestProcessorSelfTest.java   |   4 +-
 .../JettyRestProcessorAbstractSelfTest.java   | 141 +++------------
 .../JettyRestProcessorCommonSelfTest.java     | 171 ++++++++++++++++++
 .../JettyRestProcessorSignedSelfTest.java     |   6 +-
 .../JettyRestProcessorUnsignedSelfTest.java   |  11 +-
 .../apache/ignite/IgniteSystemProperties.java |  13 +-
 .../processors/rest/GridRestProcessor.java    |  94 ++++++----
 .../visor/compute/VisorGatewayTask.java       |  11 +-
 .../security/SecurityBasicPermissionSet.java  |   4 +-
 9 files changed, 280 insertions(+), 175 deletions(-)
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/AbstractRestProcessorSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/AbstractRestProcessorSelfTest.java
index 712b71a0c4cc7..3738915ad8638 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/AbstractRestProcessorSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/AbstractRestProcessorSelfTest.java
@@ -29,7 +29,7 @@
 /**
  * Abstract class for REST protocols tests.
  */
-abstract class AbstractRestProcessorSelfTest extends GridCommonAbstractTest {
+public abstract class AbstractRestProcessorSelfTest extends GridCommonAbstractTest {
     /** IP finder. */
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
@@ -60,7 +60,7 @@ abstract class AbstractRestProcessorSelfTest extends GridCommonAbstractTest {
     @Override protected void afterTest() throws Exception {
         jcache().clear();
 
-        assertTrue(jcache().localSize() == 0);
+        assertEquals(0, jcache().localSize());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
index 0285f3af530ed..d5797ff3b9948 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
@@ -18,13 +18,8 @@
 package org.apache.ignite.internal.processors.rest;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.LineNumberReader;
 import java.io.Serializable;
 import java.io.UnsupportedEncodingException;
-import java.net.URL;
-import java.net.URLConnection;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.sql.Date;
@@ -34,7 +29,6 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Iterator;
-import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
@@ -42,7 +36,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
 
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
@@ -61,7 +54,6 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheSqlIndexMetadata;
 import org.apache.ignite.internal.processors.cache.query.GridCacheSqlMetadata;
 import org.apache.ignite.internal.processors.rest.handlers.GridRestCommandHandler;
-import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
@@ -148,7 +140,6 @@
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.testframework.GridTestUtils;
 
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_JETTY_PORT;
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_ASYNC;
@@ -163,120 +154,22 @@
  * Tests for Jetty REST protocol.
  */
 @SuppressWarnings("unchecked")
-public abstract class JettyRestProcessorAbstractSelfTest extends AbstractRestProcessorSelfTest {
-    /** Grid count. */
-    private static final int GRID_CNT = 3;
-
+public abstract class JettyRestProcessorAbstractSelfTest extends JettyRestProcessorCommonSelfTest {
     /** Used to sent request charset. */
     private static final String CHARSET = StandardCharsets.UTF_8.name();
 
-    /** JSON to java mapper. */
-    private static final ObjectMapper JSON_MAPPER = new GridJettyObjectMapper();
-
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
-        System.setProperty(IGNITE_JETTY_PORT, Integer.toString(restPort()));
-
         super.beforeTestsStarted();
 
         initCache();
     }
 
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        super.afterTestsStopped();
-
-        System.clearProperty(IGNITE_JETTY_PORT);
-    }
-
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         grid(0).cache(DEFAULT_CACHE_NAME).removeAll();
     }
 
-    /** {@inheritDoc} */
-    @Override protected int gridCount() {
-        return GRID_CNT;
-    }
-
-    /**
-     * @return Port to use for rest. Needs to be changed over time because Jetty has some delay before port unbind.
-     */
-    protected abstract int restPort();
-
-    /**
-     * @return Test URL
-     */
-    protected String restUrl() {
-        return "http://" + LOC_HOST + ":" + restPort() + "/ignite?";
-    }
-
-    /**
-     * @return Security enabled flag. Should be the same with {@code ctx.security().enabled()}.
-     */
-    protected boolean securityEnabled() {
-        return false;
-    }
-
-    /**
-     * Execute REST command and return result.
-     *
-     * @param params Command parameters.
-     * @return Returned content.
-     * @throws Exception If failed.
-     */
-    protected String content(Map<String, String> params) throws Exception {
-        SB sb = new SB(restUrl());
-
-        for (Map.Entry<String, String> e : params.entrySet())
-            sb.a(e.getKey()).a('=').a(e.getValue()).a('&');
-
-        URL url = new URL(sb.toString());
-
-        URLConnection conn = url.openConnection();
-
-        String signature = signature();
-
-        if (signature != null)
-            conn.setRequestProperty("X-Signature", signature);
-
-        InputStream in = conn.getInputStream();
-
-        StringBuilder buf = new StringBuilder(256);
-
-        try (LineNumberReader rdr = new LineNumberReader(new InputStreamReader(in, "UTF-8"))) {
-            for (String line = rdr.readLine(); line != null; line = rdr.readLine())
-                buf.append(line);
-        }
-
-        return buf.toString();
-    }
-
-    /**
-     * @param cacheName Optional cache name.
-     * @param cmd REST command.
-     * @param params Command parameters.
-     * @return Returned content.
-     * @throws Exception If failed.
-     */
-    protected String content(String cacheName, GridRestCommand cmd, String... params) throws Exception {
-        Map<String, String> paramsMap = new LinkedHashMap<>();
-
-        if (cacheName != null)
-            paramsMap.put("cacheName", cacheName);
-
-        paramsMap.put("cmd", cmd.key());
-
-        if (params != null) {
-            assertEquals(0, params.length % 2);
-
-            for (int i = 0; i < params.length; i += 2)
-                paramsMap.put(params[i], params[i + 1]);
-        }
-
-        return content(paramsMap);
-    }
-
     /**
      * @param content Content to check.
      * @param err Error message.
@@ -483,6 +376,7 @@ public void testGetBinaryObjects() throws Exception {
 
         JsonNode json = assertResponseSucceeded(ret, false);
         assertEquals(ref1.name, json.get("name").asText());
+        assertEquals(ref1.ref.toString(), json.get("ref").toString());
 
         ref2.ref(ref1);
 
@@ -1519,7 +1413,7 @@ public void testTopology() throws Exception {
 
         JsonNode res = jsonResponse(ret);
 
-        assertEquals(GRID_CNT, res.size());
+        assertEquals(gridCount(), res.size());
 
         for (JsonNode node : res) {
             assertTrue(node.get("attributes").isNull());
@@ -2425,10 +2319,10 @@ public void testTypedPut() throws Exception {
         putTypedValue("timestamp", "2018-03-18%2001:01:01", "error", STATUS_FAILED);
         putTypedValue("timestamp", "error", "error", STATUS_FAILED);
 
-        IgniteCache<Timestamp, Timestamp> cTimestamp = typedCache();
+        IgniteCache<Timestamp, Timestamp> cTs = typedCache();
 
-        assertEquals(Timestamp.valueOf("2017-01-01 02:02:02"), cTimestamp.get(Timestamp.valueOf("2018-02-18 01:01:01")));
-        assertEquals(Timestamp.valueOf("2018-05-05 05:05:05"), cTimestamp.get(Timestamp.valueOf("2018-01-01 01:01:01")));
+        assertEquals(Timestamp.valueOf("2017-01-01 02:02:02"), cTs.get(Timestamp.valueOf("2018-02-18 01:01:01")));
+        assertEquals(Timestamp.valueOf("2018-05-05 05:05:05"), cTs.get(Timestamp.valueOf("2018-01-01 01:01:01")));
 
         // Test UUID type.
         UUID k1 = UUID.fromString("121f5ae8-148d-11e8-b642-0ed5f89f718b");
@@ -2624,19 +2518,13 @@ public void testTypedGet() throws Exception {
         getTypedValue("int", "888", PARTITIONED.toString());
     }
 
-    /**
-     * @return Signature.
-     * @throws Exception If failed.
-     */
-    protected abstract String signature() throws Exception;
-
     /**
      * @return True if any query cursor is available.
      */
     private boolean queryCursorFound() {
         boolean found = false;
 
-        for (int i = 0; i < GRID_CNT; ++i) {
+        for (int i = 0; i < gridCount(); ++i) {
             Map<GridRestCommand, GridRestCommandHandler> handlers =
                 GridTestUtils.getFieldValue(grid(i).context().rest(), "handlers");
 
@@ -2781,6 +2669,19 @@ public CircularRef ref() {
         public void ref(CircularRef ref) {
             this.ref = ref;
         }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            SB sb = new SB();
+
+            sb.a('{')
+                .a('"').a("id").a('"').a(':').a(id).a(',')
+                .a('"').a("name").a('"').a(':').a('"').a(name).a('"').a(',')
+                .a('"').a("ref").a('"').a(':').a(ref)
+                .a('}');
+
+            return sb.toString();
+        }
     }
 
     /**
@@ -3083,7 +2984,7 @@ private static String concat(Object[] vals, String delim) {
 
         DataRegionConfiguration drCfg = new DataRegionConfiguration();
         drCfg.setName("testDataRegion");
-        drCfg.setMaxSize(100 * 1024 * 1024);
+        drCfg.setMaxSize(100L * 1024 * 1024);
 
         dsCfg.setDefaultDataRegionConfiguration(drCfg);
 
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java
new file mode 100644
index 0000000000000..2076d490ad72f
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.rest;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_JETTY_PORT;
+
+/**
+ * Base class for testing Jetty REST protocol.
+ */
+public abstract class JettyRestProcessorCommonSelfTest extends AbstractRestProcessorSelfTest {
+    /** Grid count. */
+    private static final int GRID_CNT = 3;
+
+    /** REST port. */
+    private static final int DFLT_REST_PORT = 8091;
+
+    /** JSON to java mapper. */
+    protected static final ObjectMapper JSON_MAPPER = new GridJettyObjectMapper();
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        System.setProperty(IGNITE_JETTY_PORT, Integer.toString(restPort()));
+
+        super.beforeTestsStarted();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        System.clearProperty(IGNITE_JETTY_PORT);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected int gridCount() {
+        return GRID_CNT;
+    }
+
+    /**
+     * @return Port to use for rest. Needs to be changed over time because Jetty has some delay before port unbind.
+     */
+    protected int restPort() {
+        return DFLT_REST_PORT;
+    }
+
+    /**
+     * @return Test URL
+     */
+    protected String restUrl() {
+        return "http://" + LOC_HOST + ":" + restPort() + "/ignite?";
+    }
+
+    /**
+     * @return Security enabled flag. Should be the same with {@code ctx.security().enabled()}.
+     */
+    protected boolean securityEnabled() {
+        return false;
+    }
+
+    /**
+     * @return Signature.
+     * @throws Exception If failed.
+     */
+    protected abstract String signature() throws Exception;
+
+    /**
+     * Execute REST command and return result.
+     *
+     * @param params Command parameters.
+     * @return Returned content.
+     * @throws Exception If failed.
+     */
+    protected String content(Map<String, String> params) throws Exception {
+        SB sb = new SB(restUrl());
+
+        for (Map.Entry<String, String> e : params.entrySet())
+            sb.a(e.getKey()).a('=').a(e.getValue()).a('&');
+
+        URL url = new URL(sb.toString());
+
+        URLConnection conn = url.openConnection();
+
+        String signature = signature();
+
+        if (signature != null)
+            conn.setRequestProperty("X-Signature", signature);
+
+        InputStream in = conn.getInputStream();
+
+        StringBuilder buf = new StringBuilder(256);
+
+        try (LineNumberReader rdr = new LineNumberReader(new InputStreamReader(in, "UTF-8"))) {
+            for (String line = rdr.readLine(); line != null; line = rdr.readLine())
+                buf.append(line);
+        }
+
+        return buf.toString();
+    }
+
+    /**
+     * @param cacheName Optional cache name.
+     * @param cmd REST command.
+     * @param params Command parameters.
+     * @return Returned content.
+     * @throws Exception If failed.
+     */
+    protected String content(String cacheName, GridRestCommand cmd, String... params) throws Exception {
+        Map<String, String> paramsMap = new LinkedHashMap<>();
+
+        if (cacheName != null)
+            paramsMap.put("cacheName", cacheName);
+
+        paramsMap.put("cmd", cmd.key());
+
+        if (params != null) {
+            assertEquals(0, params.length % 2);
+
+            for (int i = 0; i < params.length; i += 2)
+                paramsMap.put(params[i], params[i + 1]);
+        }
+
+        return content(paramsMap);
+    }
+
+    /**
+     * @param json JSON content.
+     * @param field Field name in JSON object.
+     * @return Field value.
+     * @throws IOException If failed.
+     */
+    protected String jsonField(String json, String field) throws IOException {
+        assertNotNull(json);
+        assertFalse(json.isEmpty());
+
+        JsonNode node = JSON_MAPPER.readTree(json);
+
+        JsonNode fld = node.get(field);
+
+        assertNotNull(fld);
+
+        return fld.asText();
+    }
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorSignedSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorSignedSelfTest.java
index 00e4c68a9e951..3be99b489eef3 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorSignedSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorSignedSelfTest.java
@@ -82,11 +82,11 @@ public void testUnauthorized() throws Exception {
     @Override protected String signature() throws Exception {
         long ts = U.currentTimeMillis();
 
-        String s = ts + ":" + REST_SECRET_KEY;
-
         try {
             MessageDigest md = MessageDigest.getInstance("SHA-1");
 
+            String s = ts + ":" + REST_SECRET_KEY;
+
             md.update(s.getBytes());
 
             String hash = Base64.getEncoder().encodeToString(md.digest());
@@ -97,4 +97,4 @@ public void testUnauthorized() throws Exception {
             throw new Exception("Failed to create authentication signature.", e);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorUnsignedSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorUnsignedSelfTest.java
index 988cedf38fe42..c7ff0d27ae7db 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorUnsignedSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorUnsignedSelfTest.java
@@ -18,16 +18,11 @@
 package org.apache.ignite.internal.processors.rest;
 
 /**
- *
+ * Unsigned REST tests.
  */
 public class JettyRestProcessorUnsignedSelfTest extends JettyRestProcessorAbstractSelfTest {
     /** {@inheritDoc} */
-    @Override protected int restPort() {
-        return 8091;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected String signature() throws Exception {
+    @Override protected String signature() {
         return null;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index a67f821e187fc..aaf8305ec6f3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -117,9 +117,12 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_JETTY_LOG_NO_OVERRIDE = "IGNITE_JETTY_LOG_NO_OVERRIDE";
 
-    /** This property allow rewriting default ({@code 30}) rest session expire time (in seconds). */
+    /** This property allow rewriting default ({@code 30}) REST session expire time (in seconds). */
     public static final String IGNITE_REST_SESSION_TIMEOUT = "IGNITE_REST_SESSION_TIMEOUT";
 
+    /** This property allow rewriting default ({@code 300}) REST session security token expire time (in seconds). */
+    public static final String IGNITE_REST_SECURITY_TOKEN_TIMEOUT = "IGNITE_REST_SECURITY_TOKEN_TIMEOUT";
+
     /**
      * This property allows to override maximum count of task results stored on one node
      * in REST processor.
@@ -996,7 +999,7 @@ public static boolean getBoolean(String name, boolean dflt) {
      * The result is transformed to {@code int} using {@code Integer.parseInt()} method.
      *
      * @param name Name of the system property or environment variable.
-     * @param dflt Default value
+     * @param dflt Default value.
      * @return Integer value of the system property or environment variable.
      *         Returns default value in case neither system property
      *         nor environment variable with given name is found.
@@ -1024,7 +1027,7 @@ public static int getInteger(String name, int dflt) {
      * The result is transformed to {@code float} using {@code Float.parseFloat()} method.
      *
      * @param name Name of the system property or environment variable.
-     * @param dflt Default value
+     * @param dflt Default value.
      * @return Float value of the system property or environment variable.
      *         Returns default value in case neither system property
      *         nor environment variable with given name is found.
@@ -1052,7 +1055,7 @@ public static float getFloat(String name, float dflt) {
      * The result is transformed to {@code long} using {@code Long.parseLong()} method.
      *
      * @param name Name of the system property or environment variable.
-     * @param dflt Default value
+     * @param dflt Default value.
      * @return Integer value of the system property or environment variable.
      *         Returns default value in case neither system property
      *         nor environment variable with given name is found.
@@ -1080,7 +1083,7 @@ public static long getLong(String name, long dflt) {
      * The result is transformed to {@code double} using {@code Double.parseDouble()} method.
      *
      * @param name Name of the system property or environment variable.
-     * @param dflt Default value
+     * @param dflt Default value.
      * @return Integer value of the system property or environment variable.
      *         Returns default value in case neither system property
      *         nor environment variable with given name is found.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
index da5e5c205525a..d7a30f95787e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
@@ -72,6 +72,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.internal.util.worker.GridWorkerFuture;
+import org.apache.ignite.internal.visor.compute.VisorGatewayTask;
 import org.apache.ignite.internal.visor.util.VisorClusterGroupEmptyException;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteInClosure;
@@ -81,6 +82,8 @@
 import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.thread.IgniteThread;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_REST_SECURITY_TOKEN_TIMEOUT;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_REST_SESSION_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_REST_START_ON_CLIENT;
 import static org.apache.ignite.internal.processors.rest.GridRestCommand.AUTHENTICATE;
 import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_AUTH_FAILED;
@@ -99,8 +102,14 @@ public class GridRestProcessor extends GridProcessorAdapter {
     /** Delay between sessions timeout checks. */
     private static final int SES_TIMEOUT_CHECK_DELAY = 1_000;
 
-    /** Default session timout. */
-    private static final int DEFAULT_SES_TIMEOUT = 30_000;
+    /** Default session timeout, in seconds. */
+    private static final int DFLT_SES_TIMEOUT = 30;
+
+    /** The default interval used to invalidate sessions, in seconds. */
+    private static final int DFLT_SES_TOKEN_INVALIDATE_INTERVAL = 5 * 60;
+
+    /** Index of task name wrapped by VisorGatewayTask */
+    private static final int WRAPPED_TASK_IDX = 1;
 
     /** Protocols. */
     private final Collection<GridRestProtocol> protos = new ArrayList<>();
@@ -140,6 +149,9 @@ public class GridRestProcessor extends GridProcessorAdapter {
     /** Session time to live. */
     private final long sesTtl;
 
+    /** Interval to invalidate session tokens. */
+    private final long sesTokTtl;
+
     /**
      * @param req Request.
      * @return Future.
@@ -252,8 +264,8 @@ private IgniteInternalFuture<GridRestResponse> handleRequest(final GridRestReque
                 SecurityContext secCtx0 = ses.secCtx;
 
                 try {
-                    if (secCtx0 == null)
-                        ses.secCtx = secCtx0 = authenticate(req);
+                    if (secCtx0 == null || ses.isTokenExpired(sesTokTtl))
+                        ses.secCtx = secCtx0 = authenticate(req, ses);
 
                     authorize(req, secCtx0);
                 }
@@ -426,7 +438,7 @@ private Session session(final GridRestRequest req) throws IgniteCheckedException
                 UUID sesId = clientId2SesId.get(clientId);
 
                 if (sesId == null || !sesId.equals(U.bytesToUuid(sesTok, 0)))
-                    throw new IgniteCheckedException("Failed to handle request - unsupported case (misamatched " +
+                    throw new IgniteCheckedException("Failed to handle request - unsupported case (mismatched " +
                         "clientId and session token) [clientId=" + clientId + ", sesTok=" +
                         U.byteArray2HexString(sesTok) + "]");
 
@@ -452,22 +464,8 @@ private Session session(final GridRestRequest req) throws IgniteCheckedException
     public GridRestProcessor(GridKernalContext ctx) {
         super(ctx);
 
-        long sesExpTime0;
-        String sesExpTime = null;
-
-        try {
-            sesExpTime = System.getProperty(IgniteSystemProperties.IGNITE_REST_SESSION_TIMEOUT);
-
-            sesExpTime0 = sesExpTime != null ? Long.valueOf(sesExpTime) * 1000 : DEFAULT_SES_TIMEOUT;
-        }
-        catch (NumberFormatException ignore) {
-            U.warn(log, "Failed parsing IGNITE_REST_SESSION_TIMEOUT system variable [IGNITE_REST_SESSION_TIMEOUT="
-                + sesExpTime + "]");
-
-            sesExpTime0 = DEFAULT_SES_TIMEOUT;
-        }
-
-        sesTtl = sesExpTime0;
+        sesTtl = IgniteSystemProperties.getLong(IGNITE_REST_SESSION_TIMEOUT, DFLT_SES_TIMEOUT) * 1000;
+        sesTokTtl = IgniteSystemProperties.getLong(IGNITE_REST_SECURITY_TOKEN_TIMEOUT, DFLT_SES_TOKEN_INVALIDATE_INTERVAL) * 100;
 
         sesTimeoutCheckerThread = new IgniteThread(ctx.igniteInstanceName(), "session-timeout-worker",
             new GridWorker(ctx.igniteInstanceName(), "session-timeout-worker", log) {
@@ -479,9 +477,8 @@ public GridRestProcessor(GridKernalContext ctx) {
                             Session ses = e.getValue();
 
                             if (ses.isTimedOut(sesTtl)) {
-                                sesId2Ses.remove(ses.sesId, ses);
-
                                 clientId2SesId.remove(ses.clientId, ses.sesId);
+                                sesId2Ses.remove(ses.sesId, ses);
                             }
                         }
                     }
@@ -607,7 +604,7 @@ private void interceptRequest(GridRestRequest req) {
             return;
 
         if (req instanceof GridRestCacheRequest) {
-            GridRestCacheRequest req0 = (GridRestCacheRequest) req;
+            GridRestCacheRequest req0 = (GridRestCacheRequest)req;
 
             req0.key(interceptor.onReceive(req0.key()));
             req0.value(interceptor.onReceive(req0.value()));
@@ -625,7 +622,7 @@ private void interceptRequest(GridRestRequest req) {
             }
         }
         else if (req instanceof GridRestTaskRequest) {
-            GridRestTaskRequest req0 = (GridRestTaskRequest) req;
+            GridRestTaskRequest req0 = (GridRestTaskRequest)req;
 
             List<Object> oldParams = req0.params();
 
@@ -731,7 +728,7 @@ private SecurityCredentials credentials(GridRestRequest req) {
         Object creds = req.credentials();
 
         if (creds instanceof SecurityCredentials)
-            return  (SecurityCredentials)creds;
+            return (SecurityCredentials)creds;
 
         if (creds instanceof String) {
             String credStr = (String)creds;
@@ -757,7 +754,7 @@ private SecurityCredentials credentials(GridRestRequest req) {
      * @return Authentication subject context.
      * @throws IgniteCheckedException If authentication failed.
      */
-    private SecurityContext authenticate(GridRestRequest req) throws IgniteCheckedException {
+    private SecurityContext authenticate(GridRestRequest req, Session ses) throws IgniteCheckedException {
         assert req.clientId() != null;
 
         AuthenticationContext authCtx = new AuthenticationContext();
@@ -766,10 +763,24 @@ private SecurityContext authenticate(GridRestRequest req) throws IgniteCheckedEx
         authCtx.subjectId(req.clientId());
         authCtx.nodeAttributes(Collections.<String, Object>emptyMap());
         authCtx.address(req.address());
-        authCtx.credentials(credentials(req));
+
+        SecurityCredentials creds = credentials(req);
+
+        if (creds.getLogin() == null) {
+            SecurityCredentials sesCreds = ses.creds;
+
+            if (sesCreds != null)
+                creds = ses.creds;
+        }
+        else
+            ses.creds = creds;
+
+        authCtx.credentials(creds);
 
         SecurityContext subjCtx = ctx.security().authenticate(authCtx);
 
+        ses.lastInvalidateTime.set(U.currentTimeMillis());
+
         if (subjCtx == null) {
             if (req.credentials() == null)
                 throw new IgniteCheckedException("Failed to authenticate remote client (secure session SPI not set?): " + req);
@@ -839,7 +850,13 @@ private void authorize(GridRestRequest req, SecurityContext sCtx) throws Securit
             case EXE:
             case RESULT:
                 perm = SecurityPermission.TASK_EXECUTE;
-                name = ((GridRestTaskRequest)req).taskName();
+
+                GridRestTaskRequest taskReq = (GridRestTaskRequest)req;
+                name = taskReq.taskName();
+
+                // We should extract task name wrapped by VisorGatewayTask.
+                if (VisorGatewayTask.class.getName().equals(name))
+                    name = (String)taskReq.params().get(WRAPPED_TASK_IDX);
 
                 break;
 
@@ -880,7 +897,6 @@ private void authorize(GridRestRequest req, SecurityContext sCtx) throws Securit
     }
 
     /**
-     *
      * @return Whether or not REST is enabled.
      */
     private boolean isRestEnabled() {
@@ -972,7 +988,7 @@ private void startProtocol(GridRestProtocol proto) throws IgniteCheckedException
      * Session.
      */
     private static class Session {
-        /** Expiration flag. It's a final state of lastToucnTime. */
+        /** Expiration flag. It's a final state of lastTouchTime. */
         private static final Long TIMEDOUT_FLAG = 0L;
 
         /** Client id. */
@@ -987,12 +1003,18 @@ private static class Session {
          */
         private final AtomicLong lastTouchTime = new AtomicLong(U.currentTimeMillis());
 
+        /** Time when session token was invalidated last time. */
+        private final AtomicLong lastInvalidateTime = new AtomicLong(U.currentTimeMillis());
+
         /** Security context. */
         private volatile SecurityContext secCtx;
 
         /** Authorization context. */
         private volatile AuthorizationContext authCtx;
 
+        /** Credentials that can be used for security token invalidation.*/
+        private volatile SecurityCredentials creds;
+
         /**
          * @param clientId Client ID.
          * @param sesId session ID.
@@ -1047,6 +1069,16 @@ boolean isTimedOut(long sesTimeout) {
             return U.currentTimeMillis() - time0 > sesTimeout && lastTouchTime.compareAndSet(time0, TIMEDOUT_FLAG);
         }
 
+        /**
+         * Checks if session token should be invalidated.
+         *
+         * @param sesTokTtl Session token expire time.
+         * @return {@code true} if session token should be invalidated.
+         */
+        boolean isTokenExpired(long sesTokTtl) {
+            return U.currentTimeMillis() - lastInvalidateTime.get() > sesTokTtl;
+        }
+
         /**
          * Checks whether session at expired state (EXPIRATION_FLAG) or not, if not then tries to update last touch time.
          *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index ee0a54b51491b..e213bf4de3c53 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -185,8 +185,9 @@ private static class VisorGatewayJob extends ComputeJobAdapter {
 
                 Class<?> valCls = Class.forName(valClsName);
 
-                return new IgniteBiTuple<>(toObject(keyCls, (String)argument(startIdx + 2)),
-                    toObject(valCls, (String)argument(startIdx + 3)));
+                return new IgniteBiTuple<>(
+                    toObject(keyCls, argument(startIdx + 2)),
+                    toObject(valCls, argument(startIdx + 3)));
             }
 
             if (cls == Map.class) {
@@ -229,8 +230,10 @@ private static class VisorGatewayJob extends ComputeJobAdapter {
                 Class<?> v2Cls = Class.forName(v2ClsName);
                 Class<?> v3Cls = Class.forName(v3ClsName);
 
-                return new GridTuple3<>(toObject(v1Cls, (String)argument(startIdx + 3)), toObject(v2Cls,
-                    (String)argument(startIdx + 4)), toObject(v3Cls, (String)argument(startIdx + 5)));
+                return new GridTuple3<>(
+                    toObject(v1Cls, argument(startIdx + 3)),
+                    toObject(v2Cls, argument(startIdx + 4)),
+                    toObject(v3Cls, argument(startIdx + 5)));
             }
 
             return toObject(cls, arg);
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
index 370eadd801f83..fa7c74882a038 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityBasicPermissionSet.java
@@ -36,8 +36,8 @@
 import static org.apache.ignite.internal.processors.security.SecurityUtils.serializeVersion;
 
 /**
- * Simple implementation of {@link SecurityPermissionSet} interface. Provides
- * convenient way to specify permission set in the XML configuration.
+ * Simple implementation of {@link SecurityPermissionSet} interface.
+ * Provides convenient way to specify permission set in the XML configuration.
  */
 public class SecurityBasicPermissionSet implements SecurityPermissionSet {
     /** Serial version uid. */

From 0d88f7852a2ef725ea6bab13464b24dd78172d38 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 11 Jul 2018 20:25:52 +0700
Subject: [PATCH 311/543] GG-14087 Backport IGNITE-8428 Web Console: Added
 support for task arguments with private constructors.

(cherry picked from commit d7723dcc4efd668efc47ecbcd50661b096e3501a)
---
 .../processors/rest/JettyRestProcessorAbstractSelfTest.java     | 2 +-
 .../apache/ignite/internal/visor/compute/VisorGatewayTask.java  | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
index d5797ff3b9948..5bd8ea20fbc5d 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
@@ -2818,7 +2818,7 @@ public VisorGatewayArgument(Class cls) {
          * @return This helper for chaining method calls.
          */
         public VisorGatewayArgument forNode(ClusterNode node) {
-            put("p1", node.id().toString());
+            put("p1", node != null ? node.id().toString() :  null);
 
             return this;
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index e213bf4de3c53..d14463ff90f6c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -393,6 +393,7 @@ else if (isBuildInObject(argCls))
                                     }
                                 }
 
+                                ctor.setAccessible(true);
                                 jobArgs = ctor.newInstance(initArgs);
 
                                 break;

From 0b4ee13eb69c35ca7829849d6506eadd9c359a37 Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Wed, 30 May 2018 16:55:46 +0700
Subject: [PATCH 312/543] GG-14116 Backport IGNITE-8568 Added support for
 "Collocated" query mode.

(cherry picked from commit da907693316674d22ab5a44162531f48f111f980)
---
 .../internal/visor/query/VisorQueryTask.java  |  1 +
 .../visor/query/VisorQueryTaskArg.java        | 69 +++++++++++++++++--
 2 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java
index 2e322763dc703..b5af1b04b5151 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTask.java
@@ -79,6 +79,7 @@ private VisorQueryJob(VisorQueryTaskArg arg, boolean debug) {
                 qry.setPageSize(arg.getPageSize());
                 qry.setLocal(arg.isLocal());
                 qry.setDistributedJoins(arg.isDistributedJoins());
+                qry.setCollocated(arg.isCollocated());
                 qry.setEnforceJoinOrder(arg.isEnforceJoinOrder());
                 qry.setReplicatedOnly(arg.isReplicatedOnly());
                 qry.setLazy(arg.getLazy());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTaskArg.java
index e9428809d1128..5220b02632bb1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/query/VisorQueryTaskArg.java
@@ -55,6 +55,9 @@ public class VisorQueryTaskArg extends VisorDataTransferObject {
     /** Lazy query execution flag */
     private boolean lazy;
 
+    /** Collocation flag. */
+    private boolean collocated;
+
     /**
      * Default constructor.
      */
@@ -71,9 +74,16 @@ public VisorQueryTaskArg() {
      * @param loc Flag whether to execute query locally.
      * @param pageSize Result batch size.
      */
-    public VisorQueryTaskArg(String cacheName, String qryTxt, boolean distributedJoins,
-        boolean enforceJoinOrder, boolean replicatedOnly, boolean loc, int pageSize) {
-        this(cacheName, qryTxt, distributedJoins, enforceJoinOrder, replicatedOnly, loc, pageSize, false);
+    public VisorQueryTaskArg(
+        String cacheName,
+        String qryTxt,
+        boolean distributedJoins,
+        boolean enforceJoinOrder,
+        boolean replicatedOnly,
+        boolean loc,
+        int pageSize
+    ) {
+        this(cacheName, qryTxt, distributedJoins, enforceJoinOrder, replicatedOnly, loc, pageSize, false, false);
     }
 
     /**
@@ -86,8 +96,41 @@ public VisorQueryTaskArg(String cacheName, String qryTxt, boolean distributedJoi
      * @param pageSize Result batch size.
      * @param lazy Lazy query execution flag.
      */
-    public VisorQueryTaskArg(String cacheName, String qryTxt, boolean distributedJoins,
-        boolean enforceJoinOrder, boolean replicatedOnly, boolean loc, int pageSize, boolean lazy) {
+    public VisorQueryTaskArg(
+        String cacheName,
+        String qryTxt,
+        boolean distributedJoins,
+        boolean enforceJoinOrder,
+        boolean replicatedOnly,
+        boolean loc,
+        int pageSize,
+        boolean lazy
+    ) {
+        this(cacheName, qryTxt, distributedJoins, enforceJoinOrder, replicatedOnly, loc, pageSize, lazy, false);
+    }
+
+    /**
+     * @param cacheName Cache name for query.
+     * @param qryTxt Query text.
+     * @param distributedJoins If {@code true} then distributed joins enabled.
+     * @param enforceJoinOrder If {@code true} then enforce join order.
+     * @param replicatedOnly {@code true} then query contains only replicated tables.
+     * @param loc Flag whether to execute query locally.
+     * @param pageSize Result batch size.
+     * @param lazy Lazy query execution flag.
+     * @param collocated Collocation flag.
+     */
+    public VisorQueryTaskArg(
+        String cacheName,
+        String qryTxt,
+        boolean distributedJoins,
+        boolean enforceJoinOrder,
+        boolean replicatedOnly,
+        boolean loc,
+        int pageSize,
+        boolean lazy,
+        boolean collocated
+    ) {
         this.cacheName = cacheName;
         this.qryTxt = qryTxt;
         this.distributedJoins = distributedJoins;
@@ -96,6 +139,7 @@ public VisorQueryTaskArg(String cacheName, String qryTxt, boolean distributedJoi
         this.loc = loc;
         this.pageSize = pageSize;
         this.lazy = lazy;
+        this.collocated = collocated;
     }
 
     /**
@@ -156,9 +200,18 @@ public boolean getLazy() {
         return lazy;
     }
 
+    /**
+     * Flag indicating if this query is collocated.
+     *
+     * @return {@code true} If the query is collocated.
+     */
+    public boolean isCollocated() {
+        return collocated;
+    }
+
     /** {@inheritDoc} */
     @Override public byte getProtocolVersion() {
-        return V2;
+        return V3;
     }
 
     /** {@inheritDoc} */
@@ -170,6 +223,7 @@ public boolean getLazy() {
         out.writeBoolean(loc);
         out.writeInt(pageSize);
         out.writeBoolean(lazy);
+        out.writeBoolean(collocated);
     }
 
     /** {@inheritDoc} */
@@ -183,6 +237,9 @@ public boolean getLazy() {
 
         if (protoVer > V1)
             lazy = in.readBoolean();
+
+        if (protoVer > V2)
+            collocated = in.readBoolean();
     }
 
     /** {@inheritDoc} */

From 72aa4cbab4484104617e422825895ef533b5f766 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 3 Aug 2018 16:31:41 +0700
Subject: [PATCH 313/543] GG-14117 Backport IGNITE-9179 Baseline: Added
 separate task for collecting info about baseline.

(cherry picked from commit ff7372b7128df3b57ee02b735cc3010645697b7a)
---
 .../visor/baseline/VisorBaselineTask.java     |  2 +-
 .../visor/baseline/VisorBaselineViewTask.java | 73 +++++++++++++++++++
 2 files changed, 74 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
index 721b4b3d7519d..3c00452c83a48 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineTask.java
@@ -34,7 +34,7 @@
 import org.jetbrains.annotations.Nullable;
 
 /**
- * Task that will collect baseline topology information.
+ * Task that will collect information about baseline topology and can change its state.
  */
 @GridInternal
 public class VisorBaselineTask extends VisorOneNodeTask<VisorBaselineTaskArg, VisorBaselineTaskResult> {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java
new file mode 100644
index 0000000000000..472b9070d0d90
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/baseline/VisorBaselineViewTask.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.baseline;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Task that will collect information about baseline topology.
+ */
+@GridInternal
+public class VisorBaselineViewTask extends VisorOneNodeTask<Void, VisorBaselineTaskResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorBaselineViewJob job(Void arg) {
+        return new VisorBaselineViewJob(arg, debug);
+    }
+
+    /**
+     * Job that will collect baseline topology information.
+     */
+    private static class VisorBaselineViewJob extends VisorJob<Void, VisorBaselineTaskResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param arg Formal job argument.
+         * @param debug Debug flag.
+         */
+        private VisorBaselineViewJob(Void arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorBaselineTaskResult run(@Nullable Void arg) throws IgniteException {
+            IgniteClusterEx cluster = ignite.cluster();
+
+            return new VisorBaselineTaskResult(
+                ignite.cluster().active(),
+                cluster.topologyVersion(),
+                cluster.currentBaselineTopology(),
+                cluster.forServers().nodes()
+            );
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorBaselineViewJob.class, this);
+        }
+    }
+}

From ff57a8673ffe7cb88542539b82053e5c7f0ec2bf Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Tue, 21 Aug 2018 14:58:58 +0700
Subject: [PATCH 314/543] GG-13195 IGNITE-9337  Added optional group name to
 VisorNodeDataCollectorTask.  Implemented VisorCacheNamesCollectorTask to
 collect all cache names and deployment IDs.  Minor code cleanup.

(cherry picked from commit 1ad03f070b14f6f0c24ff0fde25b6c0cb8e5bcca)
---
 .../VisorCacheConfigurationCollectorTask.java |  1 -
 .../cache/VisorCacheLostPartitionsTask.java   |  5 +-
 .../visor/cache/VisorCacheModifyTask.java     | 14 +--
 .../visor/cache/VisorCacheModifyTaskArg.java  | 14 +--
 .../cache/VisorCacheModifyTaskResult.java     | 28 +++---
 .../cache/VisorCacheNamesCollectorTask.java   | 90 +++++++++++++++++++
 .../VisorCacheNamesCollectorTaskResult.java   | 88 ++++++++++++++++++
 .../visor/cache/VisorCachePartitionsTask.java |  2 +-
 .../VisorCacheResetLostPartitionsTask.java    |  2 +-
 .../internal/visor/file/VisorFileBlock.java   |  2 +-
 .../visor/node/VisorNodeDataCollectorJob.java | 19 ++--
 .../node/VisorNodeDataCollectorTask.java      |  2 +-
 .../node/VisorNodeDataCollectorTaskArg.java   | 47 +++++++++-
 .../visor/service/VisorServiceTask.java       |  2 +-
 14 files changed, 272 insertions(+), 44 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNamesCollectorTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNamesCollectorTaskResult.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java
index 154e39e6572dc..fd224a893eb39 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java
@@ -20,7 +20,6 @@
 import java.util.Map;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
-import org.apache.ignite.lang.IgniteUuid;
 
 /**
  * Task that collect cache metrics from all nodes.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java
index 24b406915a02b..52db55291b262 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java
@@ -66,8 +66,9 @@ private VisorCacheLostPartitionsJob(VisorCacheLostPartitionsTaskArg arg, boolean
                 IgniteInternalCache cache = ignite.cachex(cacheName);
 
                 if (cache != null) {
-                    GridDhtPartitionTopology topology = cache.context().topology();
-                    List<Integer> lostPartitions = new ArrayList<>(topology.lostPartitions());
+                    GridDhtPartitionTopology top = cache.context().topology();
+
+                    List<Integer> lostPartitions = new ArrayList<>(top.lostPartitions());
 
                     if (!lostPartitions.isEmpty())
                         res.put(cacheName, lostPartitions);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTask.java
index d6b1ff76f0d85..62e7ac6449cf3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTask.java
@@ -41,7 +41,7 @@ public class VisorCacheModifyTask extends VisorOneNodeTask<VisorCacheModifyTaskA
     }
 
     /**
-     * Job that clear specified caches.
+     * Job that modify value in specified cache.
      */
     private static class VisorCacheModifyJob extends VisorJob<VisorCacheModifyTaskArg, VisorCacheModifyTaskResult> {
         /** */
@@ -88,18 +88,18 @@ private VisorCacheModifyJob(VisorCacheModifyTaskArg arg, boolean debug) {
                         VisorQueryUtils.convertValue(old));
 
                 case GET:
-                    Object value = cache.get(key);
+                    Object val = cache.get(key);
 
-                    return new VisorCacheModifyTaskResult(nid, VisorTaskUtils.compactClass(value),
-                        VisorQueryUtils.convertValue(value));
+                    return new VisorCacheModifyTaskResult(nid, VisorTaskUtils.compactClass(val),
+                        VisorQueryUtils.convertValue(val));
 
                 case REMOVE:
-                    Object removed = cache.get(key);
+                    Object rmv = cache.get(key);
 
                     cache.remove(key);
 
-                    return new VisorCacheModifyTaskResult(nid, VisorTaskUtils.compactClass(removed),
-                        VisorQueryUtils.convertValue(removed));
+                    return new VisorCacheModifyTaskResult(nid, VisorTaskUtils.compactClass(rmv),
+                        VisorQueryUtils.convertValue(rmv));
             }
 
             return new VisorCacheModifyTaskResult(nid, null, null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskArg.java
index 706aab70626bc..bef73ed5fa508 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskArg.java
@@ -41,7 +41,7 @@ public class VisorCacheModifyTaskArg extends VisorDataTransferObject {
     private Object key;
 
     /** Specified value. */
-    private Object value;
+    private Object val;
 
     /**
      * Default constructor.
@@ -54,13 +54,13 @@ public VisorCacheModifyTaskArg() {
      * @param cacheName Cache name.
      * @param mode Modification mode.
      * @param key Specified key.
-     * @param value Specified value.
+     * @param val Specified value.
      */
-    public VisorCacheModifyTaskArg(String cacheName, VisorModifyCacheMode mode, Object key, Object value) {
+    public VisorCacheModifyTaskArg(String cacheName, VisorModifyCacheMode mode, Object key, Object val) {
         this.cacheName = cacheName;
         this.mode = mode;
         this.key = key;
-        this.value = value;
+        this.val = val;
     }
 
     /**
@@ -88,7 +88,7 @@ public Object getKey() {
      * @return Specified value.
      */
     public Object getValue() {
-        return value;
+        return val;
     }
 
     /** {@inheritDoc} */
@@ -96,7 +96,7 @@ public Object getValue() {
         U.writeString(out, cacheName);
         U.writeEnum(out, mode);
         out.writeObject(key);
-        out.writeObject(value);
+        out.writeObject(val);
     }
 
     /** {@inheritDoc} */
@@ -104,7 +104,7 @@ public Object getValue() {
         cacheName = U.readString(in);
         mode = VisorModifyCacheMode.fromOrdinal(in.readByte());
         key = in.readObject();
-        value = in.readObject();
+        val = in.readObject();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskResult.java
index ce09bb2a32972..8d0152af57631 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheModifyTaskResult.java
@@ -36,10 +36,10 @@ public class VisorCacheModifyTaskResult extends VisorDataTransferObject {
     private UUID affinityNode;
 
     /** Result type name. */
-    private String resultType;
+    private String resType;
 
     /** Value for specified key or number of modified rows. */
-    private Object result;
+    private Object res;
 
     /**
      * Default constructor.
@@ -50,13 +50,13 @@ public VisorCacheModifyTaskResult() {
 
     /**
      * @param affinityNode Node ID where modified data contained.
-     * @param resultType Result type name.
-     * @param result Value for specified key or number of modified rows.
+     * @param resType Result type name.
+     * @param res Value for specified key or number of modified rows.
      */
-    public VisorCacheModifyTaskResult(UUID affinityNode, String resultType, Object result) {
+    public VisorCacheModifyTaskResult(UUID affinityNode, String resType, Object res) {
         this.affinityNode = affinityNode;
-        this.resultType = resultType;
-        this.result = result;
+        this.resType = resType;
+        this.res = res;
     }
 
     /**
@@ -70,28 +70,28 @@ public UUID getAffinityNode() {
      * @return Result type name.
      */
     public String getResultType() {
-        return resultType;
+        return resType;
     }
 
     /**
-     * @return Value for specified key or number of modified rows..
+     * @return Value for specified key or number of modified rows.
      */
     public Object getResult() {
-        return result;
+        return res;
     }
 
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeUuid(out, affinityNode);
-        U.writeString(out, resultType);
-        out.writeObject(result);
+        U.writeString(out, resType);
+        out.writeObject(res);
     }
 
     /** {@inheritDoc} */
     @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
         affinityNode = U.readUuid(in);
-        resultType = U.readString(in);
-        result = in.readObject();
+        resType = U.readString(in);
+        res = in.readObject();
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNamesCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNamesCollectorTask.java
new file mode 100644
index 0000000000000..7d934d14339e0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNamesCollectorTask.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.cache;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.apache.ignite.lang.IgniteUuid;
+
+/**
+ * Task that collect cache names and deployment IDs.
+ */
+@GridInternal
+public class VisorCacheNamesCollectorTask extends VisorOneNodeTask<Void, VisorCacheNamesCollectorTaskResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCacheNamesCollectorJob job(Void arg) {
+        return new VisorCacheNamesCollectorJob(arg, debug);
+    }
+
+    /**
+     * Job that collect cache names and deployment IDs.
+     */
+    private static class VisorCacheNamesCollectorJob extends VisorJob<Void, VisorCacheNamesCollectorTaskResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job.
+         *
+         * @param arg Task argument.
+         * @param debug Debug flag.
+         */
+        private VisorCacheNamesCollectorJob(Void arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorCacheNamesCollectorTaskResult run(Void arg) {
+            GridCacheProcessor cacheProc = ignite.context().cache();
+
+            Map<String, IgniteUuid> caches = new HashMap<>();
+            Set<String> groups = new HashSet<>();
+
+            for (Map.Entry<String, DynamicCacheDescriptor> item : cacheProc.cacheDescriptors().entrySet()) {
+                DynamicCacheDescriptor cd = item.getValue();
+
+                caches.put(item.getKey(), cd.deploymentId());
+
+                String grp = cd.groupDescriptor().groupName();
+
+                if (!F.isEmpty(grp))
+                    groups.add(grp);
+            }
+
+            return new VisorCacheNamesCollectorTaskResult(caches, groups);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorCacheNamesCollectorJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNamesCollectorTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNamesCollectorTaskResult.java
new file mode 100644
index 0000000000000..a2e0908f3e9ff
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheNamesCollectorTaskResult.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.cache;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Map;
+import java.util.Set;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.apache.ignite.lang.IgniteUuid;
+
+/**
+ * Result for {@link VisorCacheNamesCollectorTask}.
+ */
+public class VisorCacheNamesCollectorTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Cache names and deployment IDs. */
+    private Map<String, IgniteUuid> caches;
+
+    /** Cache groups. */
+    private Set<String> groups;
+
+    /**
+     * Default constructor.
+     */
+    public VisorCacheNamesCollectorTaskResult() {
+        // No-op.
+    }
+
+    /**
+     * @param caches Cache names and deployment IDs.
+     */
+    public VisorCacheNamesCollectorTaskResult(Map<String, IgniteUuid> caches, Set<String> groups) {
+        this.caches = caches;
+        this.groups = groups;
+    }
+
+    /**
+     * @return Value for specified key or number of modified rows..
+     */
+    public Map<String, IgniteUuid> getCaches() {
+        return caches;
+    }
+
+    /**
+     * @return Value for specified key or number of modified rows..
+     */
+    public Set<String> getGroups() {
+        return groups;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, caches);
+        U.writeCollection(out, groups);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        caches = U.readMap(in);
+        groups = U.readSet(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorCacheNamesCollectorTaskResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCachePartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCachePartitionsTask.java
index af65de0a4c605..76ace1770f2b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCachePartitionsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCachePartitionsTask.java
@@ -63,7 +63,7 @@ public class VisorCachePartitionsTask extends VisorMultiNodeTask<VisorCacheParti
             if (res.getException() != null)
                 throw res.getException();
 
-            parts.put(res.getNode().id(), (VisorCachePartitions)res.getData());
+            parts.put(res.getNode().id(), res.getData());
         }
 
         return parts;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java
index eb48cd2efd458..2ad88ee75b7b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheResetLostPartitionsTask.java
@@ -43,7 +43,7 @@ private static class VisorCacheResetLostPartitionsJob extends VisorJob<VisorCach
         private static final long serialVersionUID = 0L;
 
         /**
-         * @param arg Object with list cache names to reset lost partitons.
+         * @param arg Object with list cache names to reset lost partitions.
          * @param debug Debug flag.
          */
         private VisorCacheResetLostPartitionsJob(VisorCacheResetLostPartitionsTaskArg arg, boolean debug) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlock.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlock.java
index 23540b55fed6b..79d0e9315fa87 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlock.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/file/VisorFileBlock.java
@@ -53,7 +53,7 @@ public class VisorFileBlock extends VisorDataTransferObject {
      * Default constructor.
      */
     public VisorFileBlock() {
-
+        // No-op.
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
index 14b928164978c..578300e813936 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
@@ -20,8 +20,9 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.concurrent.ConcurrentMap;
-import org.apache.ignite.IgniteFileSystem;
+
 import org.apache.ignite.DataRegionMetrics;
+import org.apache.ignite.IgniteFileSystem;
 import org.apache.ignite.cache.CacheMetrics;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.FileSystemConfiguration;
@@ -32,6 +33,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
 import org.apache.ignite.internal.processors.igfs.IgfsProcessorAdapter;
 import org.apache.ignite.internal.util.ipc.IpcServerEndpoint;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorJob;
@@ -159,7 +161,7 @@ protected void memoryMetrics(VisorNodeDataCollectorJobResult res) {
             List<VisorMemoryMetrics> memoryMetrics = res.getMemoryMetrics();
 
             // TODO: Should be really fixed in IGNITE-7111.
-            if (ignite.active()) {
+            if (ignite.cluster().active()) {
                 for (DataRegionMetrics m : ignite.dataRegionMetrics())
                     memoryMetrics.add(new VisorMemoryMetrics(m));
             }
@@ -181,17 +183,21 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto
 
             GridCacheProcessor cacheProc = ignite.context().cache();
 
-            List<VisorCache> resCaches = res.getCaches();
+            String cacheGrpToCollect = arg.getCacheGroup();
 
             int partitions = 0;
             double total = 0;
             double ready = 0;
 
+            List<VisorCache> resCaches = res.getCaches();
+
             for (String cacheName : cacheProc.cacheNames()) {
                 if (proxyCache(cacheName))
                     continue;
 
-                if (arg.getSystemCaches() || !(isSystemCache(cacheName) || isIgfsCache(cfg, cacheName))) {
+                boolean sysCache = isSystemCache(cacheName);
+
+                if (arg.getSystemCaches() || !(sysCache || isIgfsCache(cfg, cacheName))) {
                     long start0 = U.currentTimeMillis();
 
                     try {
@@ -213,7 +219,10 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto
                         total += partTotal;
                         ready += partReady;
 
-                        resCaches.add(new VisorCache(ignite, ca, arg.isCollectCacheMetrics()));
+                        String cacheGrp = ca.configuration().getGroupName();
+
+                        if (F.eq(cacheGrpToCollect, cacheGrp))
+                            resCaches.add(new VisorCache(ignite, ca, arg.isCollectCacheMetrics()));
                     }
                     catch(IllegalStateException | IllegalArgumentException e) {
                         if (debug && ignite.log() != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java
index fffc3bfcdbd7b..067c57ba18033 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTask.java
@@ -72,7 +72,7 @@ protected VisorNodeDataCollectorTaskResult reduce(VisorNodeDataCollectorTaskResu
             }
         }
 
-        taskRes.setActive(ignite.active());
+        taskRes.setActive(ignite.cluster().active());
 
         return taskRes;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskArg.java
index 1876d06c1b11d..e6f80273fcb74 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskArg.java
@@ -46,6 +46,9 @@ public class VisorNodeDataCollectorTaskArg extends VisorDataTransferObject {
     /** If {@code false} then cache metrics will not be collected. */
     private boolean collectCacheMetrics;
 
+    /** Optional cache group, if provided, then caches only from that group will be collected. */
+    private String cacheGrp;
+
     /**
      * Default constructor.
      */
@@ -61,19 +64,40 @@ public VisorNodeDataCollectorTaskArg() {
      * @param evtThrottleCntrKey Event throttle counter key, unique for Visor instance.
      * @param sysCaches If {@code true} then collect information about system caches.
      * @param collectCacheMetrics If {@code false} then cache metrics will not be collected.
+     * @param cacheGrp Optional cache group, if provided, then caches only from that group will be collected.
      */
     public VisorNodeDataCollectorTaskArg(
         boolean taskMonitoringEnabled,
         String evtOrderKey,
         String evtThrottleCntrKey,
         boolean sysCaches,
-        boolean collectCacheMetrics
+        boolean collectCacheMetrics,
+        String cacheGrp
     ) {
         this.taskMonitoringEnabled = taskMonitoringEnabled;
         this.evtOrderKey = evtOrderKey;
         this.evtThrottleCntrKey = evtThrottleCntrKey;
         this.sysCaches = sysCaches;
         this.collectCacheMetrics = collectCacheMetrics;
+        this.cacheGrp = cacheGrp;
+    }
+    /**
+     * Create task arguments with given parameters.
+     *
+     * @param taskMonitoringEnabled If {@code true} then Visor should collect information about tasks.
+     * @param evtOrderKey Event order key, unique for Visor instance.
+     * @param evtThrottleCntrKey Event throttle counter key, unique for Visor instance.
+     * @param sysCaches If {@code true} then collect information about system caches.
+     * @param collectCacheMetrics If {@code false} then cache metrics will not be collected.
+     */
+    public VisorNodeDataCollectorTaskArg(
+        boolean taskMonitoringEnabled,
+        String evtOrderKey,
+        String evtThrottleCntrKey,
+        boolean sysCaches,
+        boolean collectCacheMetrics
+    ) {
+        this(taskMonitoringEnabled, evtOrderKey, evtThrottleCntrKey, sysCaches, collectCacheMetrics, null);
     }
 
     /**
@@ -90,7 +114,7 @@ public VisorNodeDataCollectorTaskArg(
         String evtThrottleCntrKey,
         boolean sysCaches
     ) {
-        this(taskMonitoringEnabled, evtOrderKey, evtThrottleCntrKey, sysCaches, true);
+        this(taskMonitoringEnabled, evtOrderKey, evtThrottleCntrKey, sysCaches, true, null);
     }
 
     /**
@@ -163,9 +187,23 @@ public void setCollectCacheMetrics(boolean collectCacheMetrics) {
         this.collectCacheMetrics = collectCacheMetrics;
     }
 
+    /**
+     * @return Optional cache group, if provided, then caches only from that group will be collected.
+     */
+    public String getCacheGroup() {
+        return cacheGrp;
+    }
+
+    /**
+     * @param cacheGrp Optional cache group, if provided, then caches only from that group will be collected.
+     */
+    public void setCollectCacheMetrics(String cacheGrp) {
+        this.cacheGrp = cacheGrp;
+    }
+
     /** {@inheritDoc} */
     @Override public byte getProtocolVersion() {
-        return V2;
+        return V3;
     }
 
     /** {@inheritDoc} */
@@ -175,6 +213,7 @@ public void setCollectCacheMetrics(boolean collectCacheMetrics) {
         U.writeString(out, evtThrottleCntrKey);
         out.writeBoolean(sysCaches);
         out.writeBoolean(collectCacheMetrics);
+        U.writeString(out, cacheGrp);
     }
 
     /** {@inheritDoc} */
@@ -185,6 +224,8 @@ public void setCollectCacheMetrics(boolean collectCacheMetrics) {
         sysCaches = in.readBoolean();
 
         collectCacheMetrics = protoVer < V2 || in.readBoolean();
+
+        cacheGrp = protoVer < V3 ? null : U.readString(in);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java
index f2489bc62c1bd..e2a1fb7024720 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/service/VisorServiceTask.java
@@ -59,7 +59,7 @@ protected VisorServiceJob(Void arg, boolean debug) {
         @Override protected Collection<VisorServiceDescriptor> run(final Void arg) {
             Collection<VisorServiceDescriptor> res = new ArrayList<>();
 
-            if (ignite.active()) {
+            if (ignite.cluster().active()) {
                 Collection<ServiceDescriptor> services = ignite.services().serviceDescriptors();
 
                 for (ServiceDescriptor srvc : services)

From 59eae045eb92ee56ede34a37c7017fe623f04c7d Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 22 Aug 2018 11:19:59 +0700
Subject: [PATCH 315/543] GG-14081 Squashed commit with latest Web Console.

---
 modules/web-console/.dockerignore             |     8 +-
 modules/web-console/.gitignore                |     3 +-
 modules/web-console/DEVNOTES.txt              |    98 +-
 modules/web-console/assembly/README.txt       |    11 +-
 .../backend/agent_dists/README.txt            |     5 +-
 .../web-console/backend/app/agentSocket.js    |   117 +-
 .../web-console/backend/app/agentsHandler.js  |     4 +-
 .../backend/app/browsersHandler.js            |    38 +-
 modules/web-console/backend/app/mongo.js      |     5 +-
 modules/web-console/backend/app/schemas.js    |     3 +-
 .../ignite_modules/migrations/README.txt      |     4 -
 modules/web-console/backend/index.js          |   122 +-
 modules/web-console/backend/injector.js       |     3 +-
 modules/web-console/backend/launch-tools.js   |   109 +
 .../web-console/backend/middlewares/api.js    |    12 +-
 .../1516948939797-migrate-configs.js          |   121 +-
 .../backend/migrations/migration-utils.js     |    38 +-
 modules/web-console/backend/package.json      |    44 +-
 modules/web-console/backend/routes/admin.js   |     4 +-
 .../web-console/backend/services/clusters.js  |    15 +-
 .../web-console/backend/services/downloads.js |     2 +
 .../backend/services/notifications.js         |     4 +-
 modules/web-console/backend/test/app/db.js    |     6 +-
 .../web-console/backend/test/app/httpAgent.js |     2 +-
 .../web-console/backend/test/data/caches.json |    60 +-
 .../backend/test/data/clusters.json           |     6 +-
 .../backend/test/data/domains.json            |    21 +-
 modules/web-console/backend/test/index.js     |     4 -
 modules/web-console/backend/test/injector.js  |    41 +-
 .../backend/test/routes/clusters.js           |    13 +-
 .../web-console/backend/test/routes/public.js |     4 +-
 .../backend/test/unit/AuthService.test.js     |    11 -
 .../backend/test/unit/CacheService.test.js    |     4 +-
 .../backend/test/unit/ClusterService.test.js  |    22 +-
 .../backend/test/unit/Utils.test.js           |    48 +
 .../docker/compose/backend/.dockerignore      |     3 -
 .../docker/compose/backend/Dockerfile         |    15 +-
 .../docker/compose/backend/build.sh           |    57 -
 .../docker/compose/docker-compose.yml         |    70 +-
 .../docker/compose/frontend/.dockerignore     |     3 -
 .../docker/compose/frontend/Dockerfile        |    23 +-
 .../docker/compose/frontend/DockerfileBuild   |    30 -
 .../docker/compose/frontend/build.sh          |    59 -
 .../compose/frontend/nginx/web-console.conf   |     5 -
 .../docker/standalone/.dockerignore           |     7 -
 .../web-console/docker/standalone/Dockerfile  |    93 +-
 .../web-console/docker/standalone/build.sh    |    59 -
 .../{entrypoint.sh => docker-entrypoint.sh}   |     2 +-
 .../docker/standalone/nginx/web-console.conf  |     2 +-
 modules/web-console/e2e/docker-compose.yml    |     4 +-
 modules/web-console/e2e/testcafe/Dockerfile   |    10 +-
 .../e2e/testcafe/components/FormField.js      |     3 +
 .../e2e/testcafe/components/Table.js          |    23 +-
 .../testcafe/components/pageConfiguration.js  |     2 +-
 .../testcafe/{ => environment}/envtools.js    |    10 +-
 .../e2e/testcafe/environment/launch-env.js    |    26 +
 .../e2e/testcafe/fixtures/admin-panel.js      |     2 +-
 .../testcafe/fixtures/auth/forgot-password.js |    25 +-
 .../e2e/testcafe/fixtures/auth/logout.js      |     7 +-
 .../fixtures/auth/signup-validation-local.js  |    52 +
 .../e2e/testcafe/fixtures/auth/signup.js      |    29 +-
 .../testcafe/fixtures/configuration/basic.js  |     9 +-
 .../clusterFormChangeDetection.js             |    56 +
 .../configuration/newClusterWithCache.js      |    45 +
 .../fixtures/configuration/overview.js        |     2 +-
 .../e2e/testcafe/fixtures/menu-smoke.js       |     4 +-
 .../fixtures/queries/notebooks-list.js        |     2 +-
 .../fixtures/user-profile/credentials.js      |    11 +-
 .../testcafe/fixtures/user-profile/profile.js |     2 +-
 modules/web-console/e2e/testcafe/helpers.js   |     4 +-
 modules/web-console/e2e/testcafe/index.js     |    38 +
 modules/web-console/e2e/testcafe/package.json |    22 +-
 .../PageConfigurationAdvancedCluster.js       |     8 +-
 .../page-models/PageConfigurationBasic.js     |    23 +-
 .../page-models/PageConfigurationOverview.js  |    10 +-
 .../e2e/testcafe/page-models/PageSignIn.js    |    61 +-
 .../pageConfigurationAdvancedIGFS.js          |     4 +-
 .../pageConfigurationAdvancedModels.js        |     6 +-
 .../page-models/pageForgotPassword.js         |    24 +
 .../e2e/testcafe/page-models/pageProfile.js   |     4 +-
 .../e2e/testcafe/page-models/pageSignup.js    |    48 +
 modules/web-console/e2e/testcafe/roles.js     |    10 +-
 .../e2e/testcafe/testcafe-runner.js           |    62 +
 modules/web-console/e2e/testcafe/testcafe.js  |    86 -
 modules/web-console/e2e/testenv/Dockerfile    |    71 +-
 modules/web-console/e2e/testenv/entrypoint.sh |    21 -
 .../web-console/e2e/testenv/nginx/nginx.conf  |    58 +-
 modules/web-console/frontend/.gitignore       |     7 +-
 modules/web-console/frontend/app/app.js       |    44 +-
 .../components/bs-select-menu/controller.js   |     5 +-
 .../app/components/bs-select-menu/index.js    |     2 +
 .../components/bs-select-menu/index.spec.js   |    67 +
 .../app/components/bs-select-menu/style.scss  |    20 +-
 .../components/bs-select-menu/template.pug    |     5 +-
 .../transcludeToBody.directive.js             |    50 +
 .../cluster-security-icon/component.js}       |    13 +-
 .../components/cluster-security-icon/index.js |    24 +
 .../cluster-security-icon/template.pug        |    30 +
 .../components/cluster-selector/controller.js |    47 +-
 .../components/cluster-selector/style.scss    |    11 +-
 .../components/cluster-selector/template.pug  |    51 +-
 .../connected-clusters-badge/controller.js    |    51 +
 .../index.js                                  |     6 +-
 .../connected-clusters-badge/style.scss       |    43 +
 .../connected-clusters-badge/template.pug}    |    13 +-
 .../components/cell-logout/index.js}          |    31 +-
 .../components/cell-logout/template.pug       |    23 +
 .../components/cell-status/index.js           |    26 +
 .../components/cell-status/style.scss         |    46 +
 .../components/cell-status/template.pug       |    18 +
 .../components/list/column-defs.js            |    59 +
 .../components/list/controller.js             |    59 +
 .../components/list/index.js                  |    29 +
 .../components/list/style.scss                |    49 +
 .../components/list/template.tpl.pug          |    19 +
 .../connected-clusters-dialog/controller.js   |    24 +
 .../connected-clusters-dialog/index.js        |    35 +
 .../connected-clusters-dialog/service.js}     |    36 +-
 .../connected-clusters-dialog/style.scss      |    23 +
 .../template.tpl.pug                          |    34 +
 .../copyInputValueButton.directive.js         |    86 +
 .../app/components/form-field/index.js        |    26 +
 .../showValidationError.directive.js          |    51 +
 .../style.scss                                |    25 +-
 .../grid-column-selector/controller.js        |     8 +-
 .../app/components/grid-export/template.pug   |     4 +-
 .../list-editable-cols/cols.directive.js      |     3 +-
 .../list-editable-cols/cols.style.scss        |    10 +-
 .../list-editable-cols/cols.template.pug      |     1 +
 .../directives.js                             |    14 +-
 .../list-editable-transclude/directive.js     |    14 +-
 .../components/list-editable/controller.js    |     6 +-
 .../list-of-registered-users/column-defs.js   |     2 +-
 .../list-of-registered-users/style.scss       |    11 +
 .../list-of-registered-users/template.tpl.pug |     2 +-
 .../components/cache-edit-form/controller.js  |    11 +-
 .../cache-edit-form/template.tpl.pug          |     6 +-
 .../cluster-edit-form/controller.js           |    19 +-
 .../cluster-edit-form/template.tpl.pug        |     8 +-
 .../cluster-edit-form/templates/binary.pug    |     4 +-
 .../templates/cache-key-cfg.pug               |     2 +-
 .../templates/checkpoint.pug                  |     2 +-
 .../templates/client-connector.pug            |    10 +-
 .../templates/data-storage.pug                |   147 +-
 .../cluster-edit-form/templates/discovery.pug |     6 +-
 .../cluster-edit-form/templates/failover.pug  |    15 +-
 .../templates/general/discovery/multicast.pug |     2 +-
 .../cluster-edit-form/templates/igfs.pug      |    34 -
 .../templates/load-balancing.pug              |     7 +-
 .../cluster-edit-form/templates/memory.pug    |     4 +-
 .../cluster-edit-form/templates/odbc.pug      |    14 +-
 .../cluster-edit-form/templates/service.pug   |    26 +-
 .../templates/sql-connector.pug               |    10 +-
 .../cluster-edit-form/templates/swap.pug      |     2 +-
 .../components/igfs-edit-form/controller.js   |    12 +-
 .../igfs-edit-form/template.tpl.pug           |     7 +-
 .../components/model-edit-form/controller.js  |    22 +-
 .../model-edit-form/template.tpl.pug          |     7 +-
 .../controller.js                             |     4 +-
 .../controller.js                             |     4 +-
 .../controller.js                             |     4 +-
 .../controller.js                             |     4 +-
 .../page-configure-advanced/style.scss        |     1 +
 .../page-configure-basic/controller.js        |    17 +-
 .../page-configure-basic/style.scss           |    31 +-
 .../page-configure-basic/template.pug         |    15 +-
 .../pco-grid-column-categories/directive.js   |     4 +-
 .../page-configure-overview/controller.js     |     5 +
 .../page-configure-overview/style.scss        |     2 +-
 .../page-configure-overview/template.pug      |    14 +-
 .../components/fakeUICanExit.js               |    27 +-
 .../components/fakeUICanExit.spec.js          |    32 +
 .../components/formUICanExitGuard.js          |    12 +-
 .../modal-import-models/component.js          |    23 +-
 .../tables-action-cell/component.js           |    14 +-
 .../modal-preview-project/controller.js       |    10 +-
 .../pc-form-field-size/controller.js          |     8 +-
 .../components/pc-items-table/controller.js   |    26 +-
 .../components/pc-items-table/template.pug    |     2 +-
 .../components/pc-split-button/component.js   |    27 +
 .../components/pc-split-button/controller.js  |    45 +
 .../components/pc-split-button/index.js       |    23 +
 .../components/pc-split-button/template.pug   |    28 +
 .../pc-ui-grid-filters/directive.js           |     4 +-
 .../components/pc-ui-grid-filters/index.js    |     5 +-
 .../components/pcIsInCollection.js            |     4 +-
 .../page-configure/components/pcValidation.js |    30 +-
 .../app/components/page-configure/index.js    |    18 +-
 .../app/components/page-configure/reducer.js  |    90 +-
 .../page-configure/services/ConfigureState.js |     5 +
 .../app/components/page-configure/states.js   |    94 +-
 .../page-configure/store/actionCreators.js    |     8 +-
 .../page-configure/store/effects.js           |   180 +-
 .../page-configure/store/effects.spec.js      |   135 +
 .../page-configure/store/selectors.js         |    51 +-
 .../app/components/page-configure/style.scss  |    21 +
 .../components/page-configure/template.pug    |    10 +-
 .../transitionHooks/errorState.js             |     5 +-
 .../page-forgot-password/component.js         |    30 +
 .../page-forgot-password/controller.js        |    71 +
 .../components/page-forgot-password/index.js  |    28 +
 .../components/page-forgot-password/run.js    |    48 +
 .../page-forgot-password/style.scss           |    53 +
 .../page-forgot-password/template.pug         |    49 +
 .../components/page-forgot-password/types.ts  |    24 +
 .../app/components/page-landing/template.pug  |    24 +-
 .../page-password-changed/style.scss          |     2 +-
 .../app/components/page-profile/controller.js |     2 +-
 .../app/components/page-profile/index.js      |    12 +-
 .../app/components/page-profile/style.scss    |     3 +
 .../app/components/page-profile/template.pug  |    38 +-
 .../components/queries-notebook/controller.js |    81 +-
 .../components/queries-notebook/style.scss    |    51 +-
 .../queries-notebook/template.tpl.pug         |   181 +-
 .../queries-notebooks-list/controller.js      |    12 +-
 .../queries-notebooks-list/style.scss         |     4 +
 .../queries-notebooks-list/template.tpl.pug   |     4 +-
 .../app/components/page-queries/index.js      |    35 +-
 .../components/page-queries/style.scss}       |     8 +-
 .../app/components/page-signin/component.js   |    26 +
 .../app/components/page-signin/controller.js  |   119 +-
 .../app/components/page-signin/index.js       |    38 +-
 .../app/components/page-signin/run.js         |    56 +
 .../app/components/page-signin/style.scss     |    48 +-
 .../app/components/page-signin/template.pug   |   194 +-
 .../app/components/page-signin/types.ts       |    26 +
 .../app/components/page-signup/component.js   |    28 +
 .../app/components/page-signup/controller.js  |    73 +
 .../app/components/page-signup/index.js       |    28 +
 .../app/components/page-signup/run.js         |    35 +
 .../app/components/page-signup/style.scss     |    60 +
 .../app/components/page-signup/template.pug   |   121 +
 .../app/components/page-signup/types.ts       |    36 +
 .../panel-collapsible/controller.js           |    19 +-
 .../panel-collapsible/index.spec.js           |     1 -
 .../components/panel-collapsible/style.scss   |     3 +-
 .../components/password-visibility/index.js   |    26 +
 .../password-visibility/index.spec.js         |    65 +
 .../password-visibility/root.directive.js     |    49 +
 .../components/password-visibility/style.scss |    54 +
 .../toggle-button.component.js                |    49 +
 .../app/components/progress-line/component.js |    28 +
 .../components/progress-line/controller.js    |    60 +
 .../app/components/progress-line/index.js     |    23 +
 .../components/progress-line/index.spec.js    |    69 +
 .../app/components/progress-line/style.scss   |    82 +
 .../template.pug                              |     4 +-
 .../ui-grid-column-resizer/directive.js       |    29 +
 .../ui-grid-column-resizer/index.js           |    24 +
 .../components/ui-grid-filters/directive.js   |     3 +-
 .../app/components/ui-grid-filters/index.js   |     4 +-
 .../components/ui-grid-hovering/style.scss    |     6 +
 .../components/user-notifications/service.js  |     7 +-
 .../components/user-notifications/style.scss  |     2 +-
 .../web-console-footer/template.pug           |     2 +-
 .../components/web-console-header/style.scss  |    32 +-
 .../web-console-header/template.pug           |     4 +-
 .../controllers/reset-password.controller.js  |    50 -
 .../frontend/app/data/getting-started.json    |    30 +-
 modules/web-console/frontend/app/data/i18n.js |     2 +-
 .../app/directives/match.directive.js         |    32 +-
 .../app/directives/match.directive.spec.js    |    84 +
 .../helpers/jade/form/form-field-feedback.pug |     2 +-
 .../helpers/jade/form/form-field-password.pug |     2 +-
 .../app/helpers/jade/form/form-field-text.pug |     2 +-
 .../frontend/app/modules/ace.module.js        |    18 +-
 .../app/modules/agent/AgentManager.service.js |   321 +-
 .../app/modules/agent/AgentModal.service.js   |     5 +
 .../app/modules/agent/agent.module.js         |     8 +-
 .../components/cluster-login/component.js     |    40 +
 .../agent/components/cluster-login/index.js   |    26 +
 .../agent/components/cluster-login/service.js |    66 +
 .../components/cluster-login/template.pug     |    57 +
 .../app/modules/agent/types/Cluster.js        |    37 +
 .../app/modules/agent/types/ClusterSecrets.js |    61 +
 .../agent/types/ClusterSecretsManager.js      |    70 +
 .../app/modules/branding/header-logo.pug      |     2 +-
 .../modules/branding/powered-by-apache.pug    |     2 +-
 .../generator/ConfigurationGenerator.js       |     9 +-
 .../generator/Docker.service.spec.js          |     2 +-
 .../configuration/generator/Readme.service.js |     3 +-
 .../field/bs-select-placeholder.directive.js  |     4 +-
 .../form/validator/unique.directive.js        |     6 +-
 .../modules/nodes/nodes-dialog.controller.js  |     6 +-
 .../app/modules/nodes/nodes-dialog.scss       |    17 +-
 .../app/modules/nodes/nodes-dialog.tpl.pug    |    28 +-
 .../app/modules/states/admin.state.js         |    12 +-
 .../app/modules/states/settings.state.js      |    33 +
 .../frontend/app/modules/user/Auth.service.js |    10 +-
 .../frontend/app/primitives/badge/index.scss  |     2 +-
 .../frontend/app/primitives/btn/index.scss    |     5 +
 .../app/primitives/datepicker/index.scss      |     6 +-
 .../app/primitives/form-field/checkbox.pug    |     2 +-
 .../app/primitives/form-field/dropdown.pug    |     2 +
 .../app/primitives/form-field/index.scss      |   100 +-
 .../app/primitives/form-field/number.pug      |     5 +-
 .../app/primitives/form-field/password.pug    |    12 +-
 .../frontend/app/primitives/panel/index.scss  |     2 +-
 .../app/primitives/switcher/index.scss        |     2 +-
 .../app/primitives/timepicker/index.scss      |     6 +-
 .../app/primitives/ui-grid-header/index.scss  |     7 +-
 .../primitives/ui-grid-settings/index.scss    |     6 +-
 .../app/primitives/ui-grid/index.scss         |    12 +
 .../frontend/app/services/Caches.js           |    10 +-
 .../frontend/app/services/Clusters.js         |    42 +-
 .../app/services/ErrorParser.service.js       |    89 +
 .../app/services/FormUtils.service.js         |    19 +-
 .../frontend/app/services/IGFSs.js            |     8 +-
 .../app/services/LegacyUtils.service.js       |     2 +-
 .../frontend/app/services/Messages.service.js |    38 +-
 .../frontend/app/services/Models.js           |    17 +-
 .../frontend/app/services/exceptionHandler.js |     4 +-
 .../web-console/frontend/app/types/index.ts   |    33 +
 .../frontend/app/utils/SimpleWorkerPool.js    |     2 +
 modules/web-console/frontend/app/vendor.js    |     1 -
 .../frontend/ignite_modules/README.txt        |     6 -
 modules/web-console/frontend/index.js         |    22 +
 .../web-console/frontend/package-lock.json    | 14512 ----------------
 modules/web-console/frontend/package.json     |   153 +-
 .../frontend/public/images/cache.png          |   Bin 24791 -> 15087 bytes
 .../frontend/public/images/cluster-quick.png  |   Bin 0 -> 16407 bytes
 .../frontend/public/images/cluster.png        |   Bin 29376 -> 24083 bytes
 .../frontend/public/images/domains.png        |   Bin 22131 -> 17899 bytes
 .../icons/{alert.svg => alert.icon.svg}       |     0
 .../{attention.svg => attention.icon.svg}     |     0
 .../icons/{check.svg => check.icon.svg}       |     0
 .../public/images/icons/checkmark.icon.svg    |     3 +
 .../public/images/icons/checkmark.svg         |     3 -
 .../icons/{clock.svg => clock.icon.svg}       |     0
 .../icons/{collapse.svg => collapse.icon.svg} |     0
 ...lusters.svg => connectedClusters.icon.svg} |     0
 .../public/images/icons/copy.icon.svg         |     3 +
 .../icons/{cross.svg => cross.icon.svg}       |     0
 .../images/icons/{csv.svg => csv.icon.svg}    |     0
 .../icons/{download.svg => download.icon.svg} |     0
 .../{exclamation.svg => exclamation.icon.svg} |     0
 .../public/images/icons/exit.icon.svg         |     3 +
 .../icons/{expand.svg => expand.icon.svg}     |     0
 .../public/images/icons/eyeClosed.icon.svg    |     6 +
 .../public/images/icons/eyeOpened.icon.svg    |     7 +
 .../icons/{filter.svg => filter.icon.svg}     |     0
 .../images/icons/{gear.svg => gear.icon.svg}  |     0
 .../images/icons/{home.svg => home.icon.svg}  |     0
 .../frontend/public/images/icons/index.js     |    51 +-
 .../images/icons/{info.svg => info.icon.svg}  |     0
 .../public/images/icons/lockClosed.icon.svg   |     3 +
 .../public/images/icons/lockOpened.icon.svg   |     3 +
 .../icons/{manual.svg => manual.icon.svg}     |     0
 .../images/icons/{plus.svg => plus.icon.svg}  |     0
 .../icons/{refresh.svg => refresh.icon.svg}   |     0
 .../icons/{search.svg => search.icon.svg}     |     0
 .../images/icons/{sort.svg => sort.icon.svg}  |     0
 .../{structure.svg => structure.icon.svg}     |     0
 .../frontend/public/images/igfs.png           |   Bin 14139 -> 14683 bytes
 .../public/images/main-screenshot.png         |   Bin 86830 -> 0 bytes
 .../frontend/public/images/multicluster.png   |   Bin 21921 -> 31178 bytes
 .../public/images/page-landing-carousel-1.png |   Bin 84434 -> 20468 bytes
 .../public/images/page-landing-carousel-2.png |   Bin 106670 -> 34564 bytes
 .../public/images/page-landing-carousel-3.png |   Bin 70876 -> 27808 bytes
 .../frontend/public/images/pb-ignite@2x.png   |   Bin 8558 -> 0 bytes
 .../frontend/public/images/preview.png        |   Bin 0 -> 29829 bytes
 .../frontend/public/images/query-chart.png    |   Bin 17142 -> 0 bytes
 .../frontend/public/images/query-metadata.png |   Bin 39361 -> 0 bytes
 .../frontend/public/images/query-table.png    |   Bin 28065 -> 19943 bytes
 .../frontend/public/images/summary.png        |   Bin 33650 -> 0 bytes
 .../stylesheets/_bootstrap-variables.scss     |     2 +-
 .../frontend/public/stylesheets/style.scss    |   129 +-
 .../frontend/test/check-doc-links/Dockerfile  |     2 +-
 .../web-console/frontend/test/ci/Dockerfile   |    15 +-
 .../frontend/test/karma.conf.babel.js         |    10 +-
 modules/web-console/frontend/tsconfig.json    |     5 +-
 .../web-console/frontend/views/403.tpl.pug    |     2 +-
 .../web-console/frontend/views/404.tpl.pug    |     2 +-
 modules/web-console/frontend/views/base.pug   |     4 +-
 .../frontend/views/includes/header-left.pug   |     2 +-
 .../frontend/views/includes/header-right.pug  |     4 +-
 modules/web-console/frontend/views/index.pug  |     2 +-
 .../frontend/views/sql/cache-metadata.tpl.pug |     3 +-
 .../frontend/views/sql/chart-settings.tpl.pug |     2 +-
 .../frontend/views/sql/paragraph-rate.tpl.pug |     2 +-
 .../views/templates/agent-download.tpl.pug    |    17 +-
 .../frontend/views/templates/confirm.tpl.pug  |     4 +-
 .../frontend/views/templates/dropdown.tpl.pug |     2 +-
 .../frontend/webpack/webpack.common.js        |    68 +-
 .../frontend/webpack/webpack.dev.babel.js     |    41 +-
 .../frontend/webpack/webpack.prod.babel.js    |    23 +-
 modules/web-console/pom.xml                   |   565 +-
 .../ignite/console/agent/AgentLauncher.java   |     6 +-
 .../agent/handlers/ClusterListener.java       |     7 +
 .../ignite/console/demo/AgentClusterDemo.java |     5 +-
 parent/pom.xml                                |    24 +-
 391 files changed, 6904 insertions(+), 17599 deletions(-)
 delete mode 100644 modules/web-console/backend/ignite_modules/migrations/README.txt
 create mode 100644 modules/web-console/backend/launch-tools.js
 create mode 100644 modules/web-console/backend/test/unit/Utils.test.js
 delete mode 100644 modules/web-console/docker/compose/backend/.dockerignore
 delete mode 100755 modules/web-console/docker/compose/backend/build.sh
 delete mode 100644 modules/web-console/docker/compose/frontend/.dockerignore
 delete mode 100644 modules/web-console/docker/compose/frontend/DockerfileBuild
 delete mode 100755 modules/web-console/docker/compose/frontend/build.sh
 delete mode 100644 modules/web-console/docker/standalone/.dockerignore
 delete mode 100755 modules/web-console/docker/standalone/build.sh
 rename modules/web-console/docker/standalone/{entrypoint.sh => docker-entrypoint.sh} (98%)
 mode change 100755 => 100644
 rename modules/web-console/e2e/testcafe/{ => environment}/envtools.js (95%)
 create mode 100644 modules/web-console/e2e/testcafe/environment/launch-env.js
 create mode 100644 modules/web-console/e2e/testcafe/fixtures/auth/signup-validation-local.js
 create mode 100644 modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js
 create mode 100644 modules/web-console/e2e/testcafe/fixtures/configuration/newClusterWithCache.js
 create mode 100644 modules/web-console/e2e/testcafe/index.js
 create mode 100644 modules/web-console/e2e/testcafe/page-models/pageForgotPassword.js
 create mode 100644 modules/web-console/e2e/testcafe/page-models/pageSignup.js
 create mode 100644 modules/web-console/e2e/testcafe/testcafe-runner.js
 delete mode 100644 modules/web-console/e2e/testcafe/testcafe.js
 delete mode 100644 modules/web-console/e2e/testenv/entrypoint.sh
 create mode 100644 modules/web-console/frontend/app/components/bs-select-menu/index.spec.js
 create mode 100644 modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js
 rename modules/{core/src/main/java/org/apache/ignite/internal/worker/package-info.java => web-console/frontend/app/components/cluster-security-icon/component.js} (84%)
 create mode 100644 modules/web-console/frontend/app/components/cluster-security-icon/index.js
 create mode 100644 modules/web-console/frontend/app/components/cluster-security-icon/template.pug
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-badge/controller.js
 rename modules/web-console/frontend/app/components/{connected-clusters => connected-clusters-badge}/index.js (85%)
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-badge/style.scss
 rename modules/web-console/frontend/{views/base2.pug => app/components/connected-clusters-badge/template.pug} (76%)
 rename modules/web-console/frontend/app/components/{connected-clusters/controller.js => connected-clusters-dialog/components/cell-logout/index.js} (67%)
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/template.pug
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/index.js
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/style.scss
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/template.pug
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/column-defs.js
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/index.js
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/style.scss
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/template.tpl.pug
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/controller.js
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/index.js
 rename modules/{core/src/test/java/org/apache/ignite/loadtests/offheap/unsafe/GridUnsafeMapPerformanceTest.java => web-console/frontend/app/components/connected-clusters-dialog/service.js} (58%)
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/style.scss
 create mode 100644 modules/web-console/frontend/app/components/connected-clusters-dialog/template.tpl.pug
 create mode 100644 modules/web-console/frontend/app/components/form-field/copyInputValueButton.directive.js
 create mode 100644 modules/web-console/frontend/app/components/form-field/index.js
 create mode 100644 modules/web-console/frontend/app/components/form-field/showValidationError.directive.js
 rename modules/web-console/frontend/app/components/{connected-clusters => form-field}/style.scss (70%)
 delete mode 100644 modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/igfs.pug
 create mode 100644 modules/web-console/frontend/app/components/page-configure/components/fakeUICanExit.spec.js
 create mode 100644 modules/web-console/frontend/app/components/page-configure/components/pc-split-button/component.js
 create mode 100644 modules/web-console/frontend/app/components/page-configure/components/pc-split-button/controller.js
 create mode 100644 modules/web-console/frontend/app/components/page-configure/components/pc-split-button/index.js
 create mode 100644 modules/web-console/frontend/app/components/page-configure/components/pc-split-button/template.pug
 create mode 100644 modules/web-console/frontend/app/components/page-configure/store/effects.spec.js
 create mode 100644 modules/web-console/frontend/app/components/page-forgot-password/component.js
 create mode 100644 modules/web-console/frontend/app/components/page-forgot-password/controller.js
 create mode 100644 modules/web-console/frontend/app/components/page-forgot-password/index.js
 create mode 100644 modules/web-console/frontend/app/components/page-forgot-password/run.js
 create mode 100644 modules/web-console/frontend/app/components/page-forgot-password/style.scss
 create mode 100644 modules/web-console/frontend/app/components/page-forgot-password/template.pug
 create mode 100644 modules/web-console/frontend/app/components/page-forgot-password/types.ts
 rename modules/web-console/frontend/{ignite_modules/index.js => app/components/page-queries/style.scss} (89%)
 create mode 100644 modules/web-console/frontend/app/components/page-signin/component.js
 create mode 100644 modules/web-console/frontend/app/components/page-signin/run.js
 create mode 100644 modules/web-console/frontend/app/components/page-signin/types.ts
 create mode 100644 modules/web-console/frontend/app/components/page-signup/component.js
 create mode 100644 modules/web-console/frontend/app/components/page-signup/controller.js
 create mode 100644 modules/web-console/frontend/app/components/page-signup/index.js
 create mode 100644 modules/web-console/frontend/app/components/page-signup/run.js
 create mode 100644 modules/web-console/frontend/app/components/page-signup/style.scss
 create mode 100644 modules/web-console/frontend/app/components/page-signup/template.pug
 create mode 100644 modules/web-console/frontend/app/components/page-signup/types.ts
 create mode 100644 modules/web-console/frontend/app/components/password-visibility/index.js
 create mode 100644 modules/web-console/frontend/app/components/password-visibility/index.spec.js
 create mode 100644 modules/web-console/frontend/app/components/password-visibility/root.directive.js
 create mode 100644 modules/web-console/frontend/app/components/password-visibility/style.scss
 create mode 100644 modules/web-console/frontend/app/components/password-visibility/toggle-button.component.js
 create mode 100644 modules/web-console/frontend/app/components/progress-line/component.js
 create mode 100644 modules/web-console/frontend/app/components/progress-line/controller.js
 create mode 100644 modules/web-console/frontend/app/components/progress-line/index.js
 create mode 100644 modules/web-console/frontend/app/components/progress-line/index.spec.js
 create mode 100644 modules/web-console/frontend/app/components/progress-line/style.scss
 rename modules/web-console/frontend/app/components/{connected-clusters => progress-line}/template.pug (90%)
 create mode 100644 modules/web-console/frontend/app/components/ui-grid-column-resizer/directive.js
 create mode 100644 modules/web-console/frontend/app/components/ui-grid-column-resizer/index.js
 delete mode 100644 modules/web-console/frontend/app/controllers/reset-password.controller.js
 create mode 100644 modules/web-console/frontend/app/directives/match.directive.spec.js
 create mode 100644 modules/web-console/frontend/app/modules/agent/components/cluster-login/component.js
 create mode 100644 modules/web-console/frontend/app/modules/agent/components/cluster-login/index.js
 create mode 100644 modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js
 create mode 100644 modules/web-console/frontend/app/modules/agent/components/cluster-login/template.pug
 create mode 100644 modules/web-console/frontend/app/modules/agent/types/Cluster.js
 create mode 100644 modules/web-console/frontend/app/modules/agent/types/ClusterSecrets.js
 create mode 100644 modules/web-console/frontend/app/modules/agent/types/ClusterSecretsManager.js
 create mode 100644 modules/web-console/frontend/app/modules/states/settings.state.js
 create mode 100644 modules/web-console/frontend/app/services/ErrorParser.service.js
 create mode 100644 modules/web-console/frontend/app/types/index.ts
 delete mode 100644 modules/web-console/frontend/ignite_modules/README.txt
 create mode 100644 modules/web-console/frontend/index.js
 delete mode 100644 modules/web-console/frontend/package-lock.json
 create mode 100644 modules/web-console/frontend/public/images/cluster-quick.png
 rename modules/web-console/frontend/public/images/icons/{alert.svg => alert.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{attention.svg => attention.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{check.svg => check.icon.svg} (100%)
 create mode 100644 modules/web-console/frontend/public/images/icons/checkmark.icon.svg
 delete mode 100644 modules/web-console/frontend/public/images/icons/checkmark.svg
 rename modules/web-console/frontend/public/images/icons/{clock.svg => clock.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{collapse.svg => collapse.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{connectedClusters.svg => connectedClusters.icon.svg} (100%)
 create mode 100644 modules/web-console/frontend/public/images/icons/copy.icon.svg
 rename modules/web-console/frontend/public/images/icons/{cross.svg => cross.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{csv.svg => csv.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{download.svg => download.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{exclamation.svg => exclamation.icon.svg} (100%)
 create mode 100644 modules/web-console/frontend/public/images/icons/exit.icon.svg
 rename modules/web-console/frontend/public/images/icons/{expand.svg => expand.icon.svg} (100%)
 create mode 100644 modules/web-console/frontend/public/images/icons/eyeClosed.icon.svg
 create mode 100644 modules/web-console/frontend/public/images/icons/eyeOpened.icon.svg
 rename modules/web-console/frontend/public/images/icons/{filter.svg => filter.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{gear.svg => gear.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{home.svg => home.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{info.svg => info.icon.svg} (100%)
 create mode 100644 modules/web-console/frontend/public/images/icons/lockClosed.icon.svg
 create mode 100644 modules/web-console/frontend/public/images/icons/lockOpened.icon.svg
 rename modules/web-console/frontend/public/images/icons/{manual.svg => manual.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{plus.svg => plus.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{refresh.svg => refresh.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{search.svg => search.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{sort.svg => sort.icon.svg} (100%)
 rename modules/web-console/frontend/public/images/icons/{structure.svg => structure.icon.svg} (100%)
 delete mode 100644 modules/web-console/frontend/public/images/main-screenshot.png
 delete mode 100644 modules/web-console/frontend/public/images/pb-ignite@2x.png
 create mode 100644 modules/web-console/frontend/public/images/preview.png
 delete mode 100644 modules/web-console/frontend/public/images/query-chart.png
 delete mode 100644 modules/web-console/frontend/public/images/query-metadata.png
 delete mode 100644 modules/web-console/frontend/public/images/summary.png

diff --git a/modules/web-console/.dockerignore b/modules/web-console/.dockerignore
index bcac98f1755d1..1958528b3f073 100644
--- a/modules/web-console/.dockerignore
+++ b/modules/web-console/.dockerignore
@@ -1,4 +1,10 @@
 .git
 *Dockerfile*
 *docker-compose*
-*/node_modules*
\ No newline at end of file
+**/build/
+**/node_modules/
+**/target/
+!web-agent/target/*.zip
+**/backend/agent_dists/*.zip
+**/backend/config/*.json
+**/backend/test
\ No newline at end of file
diff --git a/modules/web-console/.gitignore b/modules/web-console/.gitignore
index 8648ae5323960..e99ad51aa95a3 100644
--- a/modules/web-console/.gitignore
+++ b/modules/web-console/.gitignore
@@ -1,5 +1,4 @@
 .npmrc
-package-lock.json
 build/
 node_modules/
-data/
+package-lock.json
diff --git a/modules/web-console/DEVNOTES.txt b/modules/web-console/DEVNOTES.txt
index f7206118f40ed..ba61150b41bd9 100644
--- a/modules/web-console/DEVNOTES.txt
+++ b/modules/web-console/DEVNOTES.txt
@@ -1,14 +1,14 @@
 Ignite Web Console Build Instructions
 =====================================
-1. Install locally MongoDB (version >=3.2.x) follow instructions from site http://docs.mongodb.org/manual/installation.
-2. Install locally NodeJS (version >=6.5.x) using installer from site https://nodejs.org/en/download/current for your OS.
-3. Change directory to '/modules/web-console/backend' and
+1. Install MongoDB (version >=3.2.0 <=3.4.15) using instructions from http://docs.mongodb.org/manual/installation.
+2. Install Node.js (version >=8.0.0) using installer from https://nodejs.org/en/download/current for your OS.
+3. Change directory to 'modules/web-console/backend' and
  run "npm install --no-optional" for download backend dependencies.
-4. Change directory to '/modules/web-console/frontend' and
+4. Change directory to 'modules/web-console/frontend' and
  run "npm install --no-optional" for download frontend dependencies.
 5. Build ignite-web-agent module follow instructions from 'modules/web-console/web-agent/README.txt'.
-6. Copy ignite-web-agent-<version>.zip from '/modules/web-console/web-agent/target'
- to '/modules/web-console/backend/agent_dists' folder.
+6. Copy ignite-web-agent-<version>.zip from 'modules/web-console/web-agent/target'
+ to 'modules/web-console/backend/agent_dists' folder.
 
 Steps 1 - 4 should be executed once.
 
@@ -17,10 +17,10 @@ Ignite Web Console Run In Development Mode
 1. Configure MongoDB to run as service or in terminal change dir to $MONGO_INSTALL_DIR/server/3.2/bin
   and start MongoDB by executing "mongod".
 
-2. In new terminal change directory to '/modules/web-console/backend'.
+2. In new terminal change directory to 'modules/web-console/backend'.
    If needed run "npm install --no-optional" (if dependencies changed) and run "npm start" to start backend.
 
-3. In new terminal change directory to '/modules/web-console/frontend'.
+3. In new terminal change directory to 'modules/web-console/frontend'.
   If needed run "npm install --no-optional" (if dependencies changed) and start webpack in development mode "npm run dev".
 
 4. In browser open: http://localhost:9000
@@ -35,19 +35,69 @@ How to migrate model:
 Ignite Web Console Direct-Install Maven Build Instructions
 ==========================================================
 To build direct-install archive from sources run following command in Ignite project root folder:
-"mvn clean package -pl :ignite-web-agent,:ignite-web-console -am -P web-console -DskipTests=true -DskipClientDocs -Dmaven.javadoc.skip=true"
+"mvn clean package -pl :ignite-web-console -am -P web-console,direct-install -DskipTests=true -DskipClientDocs -Dmaven.javadoc.skip=true"
 
-Assembled archive can be found here: `/modules/web-console/target/ignite-web-console-direct-install-*.zip`.
+Assembled archive can be found here: `modules/web-console/target/ignite-web-console-direct-install-*.zip`.
+
+
+Ignite Web Console Docker Images Build Instructions
+===================================================
+Install Docker (version >=17.05) using instructions from https://www.docker.com/community-edition.
+
+To build docker images from sources run following command in Ignite project root folder:
+
+"mvn clean package -pl :ignite-web-console -am -P web-console,docker-image -DskipTests=true -DskipClientDocs -Dmaven.javadoc.skip=true"
+
+Prepared image can be listed with `docker images` command.
+
+
+Ignite Web Console Standalone Docker Image Build Manual Instructions
+====================================================================
+Install Docker (version >=17.05) using instructions from https://www.docker.com/community-edition.
+
+1. Build Apache Ignite Web Agent archive as described in `modules/web-console/web-agent/README.txt`.
+2. Goto Web Console's module directory: `cd modules/web-console`
+3. Build docker image:
+
+"docker build . -t apacheignite/web-console-standalone[:<version>] -f docker/standalone/Dockerfile"
+
+Prepared image can be listed in `docker images` command output.
+
+
+Ignite Web Console Backend Docker Image Build Manual Instructions
+====================================================================
+Install Docker (version >=17.05) using instructions from https://www.docker.com/community-edition.
+
+1. Build Apache Ignite Web Agent archive as described in `modules/web-console/web-agent/README.txt`.
+2. Goto Web Console's module directory: `cd modules/web-console`
+3. Build docker image:
+
+"docker build . -t apacheignite/web-console-backend[:<version>] -f docker/compose/backend/Dockerfile"
+
+Prepared image can be listed in `docker images` command output.
+
+
+Ignite Web Console Frontend Docker Image Build Manual Instructions
+====================================================================
+Install Docker (version >=17.05) using instructions from https://www.docker.com/community-edition.
+
+1. Build Apache Ignite Web Agent archive as described in `modules/web-console/web-agent/README.txt`.
+2. Goto Web Console's module directory: `cd modules/web-console`
+3. Build docker image:
+
+"docker build . -t apacheignite/web-console-frontend[:<version>] -f docker/compose/frontend/Dockerfile"
+
+Prepared image can be listed in `docker images` command output.
 
 
 End-to-end tests
 ================
-E2E tests are performed with TestCafe framework - https://testcafe.devexpress.com/.
+E2E tests are performed with TestCafe framework - https://testcafe.devexpress.com
 
 To launch tests on your local machine you will need:
 1. Install and launch MongoDB.
 2. Optionally install Chromium (https://www.chromium.org/getting-involved/download-chromium or https://chromium.woolyss.com).
-   You may use any other browser, just set 'BROWSERS' constant in 'modules\web-console\e2e\testcafe.js'.
+   You may use any other browser, just set 'BROWSERS' constant in 'modules/web-console/e2e/testcafe/index.js'.
 3. In new terminal change directory to 'modules/web-console/e2e/testcafe' folder and execute: "npm install".
 4. To start test environment and tests execute: "npm run test".
 
@@ -57,7 +107,7 @@ To perform it do the following:
 1. Ensure that MongoDB is up and running and all dependencies for backend and frontend are installed.
 2. Open directory "modules/web-console/e2e/testcafe" in terminal. Install dependencies for E2E testing with "npm install" command.
 3. Execute command "npm run env". This will start backend and frontend environment.
-4. Open another terminal window and run command "node testcafe.js" in the same directory. This will run only tests without launching environment.
+4. Open another terminal window and run command "node index.js" in the same directory. This will run only tests without launching environment.
 
 Please refer to TestCafe documentation at https://devexpress.github.io/testcafe/documentation/test-api/test-code-structure.html#skipping-tests
  upon how to specify which particular test should be run or skipped.
@@ -65,9 +115,27 @@ Please refer to TestCafe documentation at https://devexpress.github.io/testcafe/
 You can modify the following params with environment variables:
 - DB_URL - connection string to test MongoDB. Default: mongodb://localhost/console-e2e
 - APP_URL - URL for test environment applications. Default: http://localhost:9001
-- TEAMCITY - Whether to use TeamCity reporter. Default: false (native Testcafe "spec" reporter is used)
+- REPORTER - Which "TestCafe" reporter to use. Set to 'teamcity' to use Teamcity reporter. Default: "spec" (native Testcafe reporter)
 
 You can run tests in docker:
 1. Install docker and docker-compose.
-2. Execute in terminal: "docker-compose up --abort-on-container-exit" in directory "modules/web-console/e2e".
+2. Execute in terminal: "docker-compose up --build --abort-on-container-exit" in directory "modules/web-console/e2e".
 3. If you need to cleanup docker container then execute "docker-compose down".
+
+
+Frontend unit tests
+===================
+Unit tests are performed with Mocha framework - https://mochajs.org
+
+To launch tests on your local machine you will need:
+1. In new terminal change directory to 'modules/web-console/frontend' folder and execute: "npm install".
+2. To start test environment and tests execute: "npm run test".
+
+
+Backend unit tests
+==================
+Unit tests are performed with Mocha framework - https://mochajs.org
+
+To launch tests on your local machine you will need:
+1. In new terminal change directory to 'modules/web-console/backend' folder and execute: "npm install".
+2. To start test environment and tests execute: "npm run test".
diff --git a/modules/web-console/assembly/README.txt b/modules/web-console/assembly/README.txt
index 540aa9fb38cc9..9699a48496d03 100644
--- a/modules/web-console/assembly/README.txt
+++ b/modules/web-console/assembly/README.txt
@@ -1,6 +1,6 @@
 Requirements
 -------------------------------------
-1. JDK 7 bit for your platform, or newer.
+1. JDK 8 bit for your platform, or newer.
 2. Supported browsers: Chrome, Firefox, Safari, Edge.
 3. Ignite cluster should be started with `ignite-rest-http` module in classpath. For this copy `ignite-rest-http` folder from `libs\optional` to `libs` folder.
 
@@ -8,16 +8,17 @@ Requirements
 How to run
 -------------------------------------
 1. Unpack ignite-web-console-x.x.x.zip to some folder.
-2. Start ignite-web-console-xxx executable for you platform:
+2. Change work directory to folder where Web Console was unpacked.
+3. Start ignite-web-console-xxx executable for you platform:
 	For Linux: ignite-web-console-linux
 	For MacOS: ignite-web-console-macos
 	For Windows: ignite-web-console-win.exe
 
 Note: on Linux and Mac OS X `root` permission is required to bind to 80 port, but you may always start Web Console on another port if you don't have such permission.
 
-3. Open URL `localhost` in browser.
-4. Login with user `admin@admin` and password `admin`.
-5. Start web agent from folder `web agent`. For Web Agent settings see `web-agent\README.txt`.
+4. Open URL `localhost` in browser.
+5. Login with user `admin@admin` and password `admin`.
+6. Start web agent from folder `web agent`. For Web Agent settings see `web-agent\README.txt`.
 Cluster URL should be specified in `web-agent\default.properties` in `node-uri` parameter.
 
 Technical details
diff --git a/modules/web-console/backend/agent_dists/README.txt b/modules/web-console/backend/agent_dists/README.txt
index d51bdf973fb1f..e0a67c6cea5af 100644
--- a/modules/web-console/backend/agent_dists/README.txt
+++ b/modules/web-console/backend/agent_dists/README.txt
@@ -1,7 +1,6 @@
 Ignite Web Console
 ======================================
 
-This is default folder for agent distributives.
-
-Also, you could specify custom folder in `serve/config/settings.json`
+This is a default directory for agent distributions.
 
+A custom directory can be set in `backend/config/settings.json`.
diff --git a/modules/web-console/backend/app/agentSocket.js b/modules/web-console/backend/app/agentSocket.js
index 39d880fc16609..aff62c4b3f304 100644
--- a/modules/web-console/backend/app/agentSocket.js
+++ b/modules/web-console/backend/app/agentSocket.js
@@ -28,56 +28,6 @@ module.exports = {
     implements: 'agent-socket'
 };
 
-/**
- * Helper class to contract REST command.
- */
-class Command {
-    /**
-     * @param {Boolean} demo Is need run command on demo node.
-     * @param {String} name Command name.
-     */
-    constructor(demo, name) {
-        this.demo = demo;
-
-        /**
-         * Command name.
-         * @type {String}
-         */
-        this._name = name;
-
-        /**
-         * Command parameters.
-         * @type {Array.<Object.<String, String>>}
-         */
-        this._params = [];
-
-        this._paramsLastIdx = 1;
-    }
-
-    /**
-     * Add parameter to command.
-     * @param {Object} value Parameter value.
-     * @returns {Command}
-     */
-    addParam(value) {
-        this._params.push({key: `p${this._paramsLastIdx++}`, value});
-
-        return this;
-    }
-
-    /**
-     * Add parameter to command.
-     * @param {String} key Parameter key.
-     * @param {Object} value Parameter value.
-     * @returns {Command}
-     */
-    addNamedParam(key, value) {
-        this._params.push({key, value});
-
-        return this;
-    }
-}
-
 /**
  * @returns {AgentSocket}
  */
@@ -107,7 +57,7 @@ module.exports.factory = function() {
          * Send event to agent.
          *
          * @this {AgentSocket}
-         * @param {String} event Command name.
+         * @param {String} event Event name.
          * @param {Array.<Object>} args - Transmitted arguments.
          * @param {Function} [callback] on finish
          */
@@ -178,71 +128,6 @@ module.exports.factory = function() {
         attachToDemoCluster(browserSocket) {
             this.demo.browserSockets.push(...browserSocket);
         }
-
-        startCollectTopology(timeout) {
-            return this.emitEvent('start:collect:topology', timeout);
-        }
-
-        stopCollectTopology(demo) {
-            return this.emitEvent('stop:collect:topology', demo);
-        }
-
-        /**
-         * Execute REST request on node.
-         *
-         * @param {Boolean} demo Is need run command on demo node.
-         * @param {String} cmd REST command.
-         * @param {Array.<String>} args - REST command arguments.
-         * @return {Promise}
-         */
-        restCommand(demo, cmd, ...args) {
-            const params = {cmd};
-
-            _.forEach(args, (arg, idx) => {
-                params[`p${idx + 1}`] = args[idx];
-            });
-
-            return this.emitEvent('node:rest', {uri: 'ignite', demo, params})
-                .then(this.restResultParse);
-        }
-
-        gatewayCommand(demo, nids, taskCls, argCls, ...args) {
-            const cmd = new Command(demo, 'exe')
-                .addNamedParam('name', 'org.apache.ignite.internal.visor.compute.VisorGatewayTask')
-                .addParam(nids)
-                .addParam(taskCls)
-                .addParam(argCls);
-
-            _.forEach(args, (arg) => cmd.addParam(arg));
-
-            return this.restCommand(cmd);
-        }
-
-        /**
-         * @param {Boolean} demo Is need run command on demo node.
-         * @param {Boolean} attr Get attributes, if this parameter has value true. Default value: true.
-         * @param {Boolean} mtr Get metrics, if this parameter has value true. Default value: false.
-         * @returns {Promise}
-         */
-        topology(demo, attr, mtr) {
-            const cmd = new Command(demo, 'top')
-                .addNamedParam('attr', attr !== false)
-                .addNamedParam('mtr', !!mtr);
-
-            return this.restCommand(cmd);
-        }
-
-        /**
-         * @param {Boolean} demo Is need run command on demo node.
-         * @param {String} cacheName Cache name.
-         * @returns {Promise}
-         */
-        metadata(demo, cacheName) {
-            const cmd = new Command(demo, 'metadata')
-                .addNamedParam('cacheName', cacheName);
-
-            return this.restCommand(cmd);
-        }
     }
 
     return AgentSocket;
diff --git a/modules/web-console/backend/app/agentsHandler.js b/modules/web-console/backend/app/agentsHandler.js
index eb3a1e0cd416a..4af81b6699fca 100644
--- a/modules/web-console/backend/app/agentsHandler.js
+++ b/modules/web-console/backend/app/agentsHandler.js
@@ -93,6 +93,7 @@ module.exports.factory = function(settings, mongo, AgentSocket) {
             this.clients = top.clients;
             this.clusterVersion = top.clusterVersion;
             this.active = top.active;
+            this.secured = top.secured;
         }
 
         isSameCluster(top) {
@@ -106,6 +107,7 @@ module.exports.factory = function(settings, mongo, AgentSocket) {
             this.clients = top.clients;
             this.clusterVersion = top.clusterVersion;
             this.active = top.active;
+            this.secured = top.secured;
         }
 
         same(top) {
@@ -428,7 +430,7 @@ module.exports.factory = function(settings, mongo, AgentSocket) {
 
             const sockets = this._agentSockets[token];
 
-            _.forEach(sockets, (socket) => socket._emit('agent:reset:token', token));
+            _.forEach(sockets, (socket) => socket._sendToAgent('agent:reset:token', token));
         }
     }
 
diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js
index a8ca7067b6f79..c3c2ea42dfccc 100644
--- a/modules/web-console/backend/app/browsersHandler.js
+++ b/modules/web-console/backend/app/browsersHandler.js
@@ -185,22 +185,14 @@ module.exports = {
             /**
              * @param {Promise.<AgentSocket>} agent
              * @param {Boolean} demo
+             * @param {{sessionId: String}|{'login': String, 'password': String}} credentials
              * @param {Object.<String, String>} params
              * @return {Promise.<T>}
              */
-            executeOnNode(agent, demo, params) {
+            executeOnNode(agent, demo, credentials, params) {
                 return agent
-                    .then((agentSock) => agentSock.emitEvent('node:rest', {uri: 'ignite', demo, params}))
-                    .then((res) => {
-                        if (res.status === 0) {
-                            if (res.zipped)
-                                return res;
-
-                            return JSON.parse(res.data);
-                        }
-
-                        throw new Error(res.error);
-                    });
+                    .then((agentSock) => agentSock.emitEvent('node:rest',
+                        {uri: 'ignite', demo, params: _.merge({}, credentials, params)}));
             }
 
             registerVisorTask(taskId, taskCls, ...argCls) {
@@ -212,13 +204,13 @@ module.exports = {
 
             nodeListeners(sock) {
                 // Return command result from grid to browser.
-                sock.on('node:rest', (clusterId, params, cb) => {
+                sock.on('node:rest', ({clusterId, params, credentials}, cb) => {
                     const demo = sock.request._query.IgniteDemoMode === 'true';
                     const token = sock.request.user.token;
 
                     const agent = this._agentHnd.agent(token, demo, clusterId);
 
-                    this.executeOnNode(agent, demo, params)
+                    this.executeOnNode(agent, demo, credentials, params)
                         .then((data) => cb(null, data))
                         .catch((err) => cb(this.errorTransformer(err)));
                 });
@@ -241,38 +233,34 @@ module.exports = {
                 this.registerVisorTask('toggleClusterState', internalVisor('misc.VisorChangeGridActiveStateTask'), internalVisor('misc.VisorChangeGridActiveStateTaskArg'));
 
                 // Return command result from grid to browser.
-                sock.on('node:visor', (clusterId, taskId, nids, ...args) => {
+                sock.on('node:visor', ({clusterId, params = {}, credentials} = {}, cb) => {
                     const demo = sock.request._query.IgniteDemoMode === 'true';
                     const token = sock.request.user.token;
 
-                    const cb = _.last(args);
-                    args = _.dropRight(args);
+                    const {taskId, nids, args = []} = params;
 
                     const desc = this._visorTasks.get(taskId);
 
                     if (_.isNil(desc))
                         return cb(this.errorTransformer(new errors.IllegalArgumentException(`Failed to find Visor task for id: ${taskId}`)));
 
-                    const params = {
+                    const exeParams = {
                         cmd: 'exe',
                         name: 'org.apache.ignite.internal.visor.compute.VisorGatewayTask',
                         p1: nids,
                         p2: desc.taskCls
                     };
 
-                    _.forEach(_.concat(desc.argCls, args), (param, idx) => { params[`p${idx + 3}`] = param; });
+                    _.forEach(_.concat(desc.argCls, args), (param, idx) => { exeParams[`p${idx + 3}`] = param; });
 
                     const agent = this._agentHnd.agent(token, demo, clusterId);
 
-                    this.executeOnNode(agent, demo, params)
+                    this.executeOnNode(agent, demo, credentials, exeParams)
                         .then((data) => {
-                            if (data.zipped)
-                                return cb(null, data);
-
-                            if (data.finished)
+                            if (data.finished && !data.zipped)
                                 return cb(null, data.result);
 
-                            cb(this.errorTransformer(data.error));
+                            return cb(null, data);
                         })
                         .catch((err) => cb(this.errorTransformer(err)));
                 });
diff --git a/modules/web-console/backend/app/mongo.js b/modules/web-console/backend/app/mongo.js
index 61f148409b678..6a843dbdd15e7 100644
--- a/modules/web-console/backend/app/mongo.js
+++ b/modules/web-console/backend/app/mongo.js
@@ -18,6 +18,8 @@
 'use strict';
 
 const _ = require('lodash');
+const {MongodHelper} = require('mongodb-prebuilt');
+const {MongoDBDownload} = require('mongodb-download');
 
 // Fire me up!
 
@@ -64,9 +66,6 @@ module.exports.factory = function(settings, mongoose, schemas) {
         .catch((err) => {
             console.log('Failed to connect to local MongoDB, will try to download and start embedded MongoDB', err);
 
-            const {MongodHelper} = require('mongodb-prebuilt');
-            const {MongoDBDownload} = require('mongodb-download');
-
             const helper = new MongodHelper(['--port', '27017', '--dbpath', `${process.cwd()}/user_data`]);
 
             helper.mongoBin.mongoDBPrebuilt.mongoDBDownload = new MongoDBDownload({
diff --git a/modules/web-console/backend/app/schemas.js b/modules/web-console/backend/app/schemas.js
index 3f37487827e06..5898ed88cb247 100644
--- a/modules/web-console/backend/app/schemas.js
+++ b/modules/web-console/backend/app/schemas.js
@@ -1138,7 +1138,8 @@ module.exports.factory = function(mongoose) {
             qryType: String,
             nonCollocatedJoins: {type: Boolean, default: false},
             enforceJoinOrder: {type: Boolean, default: false},
-            lazy: {type: Boolean, default: false}
+            lazy: {type: Boolean, default: false},
+            collocated: Boolean
         }]
     });
 
diff --git a/modules/web-console/backend/ignite_modules/migrations/README.txt b/modules/web-console/backend/ignite_modules/migrations/README.txt
deleted file mode 100644
index daeae364d0abb..0000000000000
--- a/modules/web-console/backend/ignite_modules/migrations/README.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-Ignite Web Console
-======================================
-
-This folder contains scripts for modules model migration.
diff --git a/modules/web-console/backend/index.js b/modules/web-console/backend/index.js
index ad334bcf698ca..4697de91eaffd 100644
--- a/modules/web-console/backend/index.js
+++ b/modules/web-console/backend/index.js
@@ -17,138 +17,22 @@
 
 'use strict';
 
-const fs = require('fs');
 const path = require('path');
 
 const appPath = require('app-module-path');
 appPath.addPath(__dirname);
 appPath.addPath(path.join(__dirname, 'node_modules'));
 
-const _ = require('lodash');
-const getos = require('getos');
-const http = require('http');
-const https = require('https');
-const MigrateMongoose = require('migrate-mongoose');
+const { migrate, init } = require('./launch-tools');
 
-
-const packaged = __dirname.startsWith('/snapshot/') || __dirname.startsWith('C:\\snapshot\\');
-
-const igniteModules = !packaged && process.env.IGNITE_MODULES ?
-    path.join(path.normalize(process.env.IGNITE_MODULES), 'backend') : path.join(__dirname, 'ignite_modules');
-
-let injector;
-
-try {
-    const igniteModulesInjector = path.resolve(path.join(igniteModules, 'injector.js'));
-
-    fs.accessSync(igniteModulesInjector, fs.F_OK);
-
-    process.env.NODE_PATH = path.join(__dirname, 'node_modules');
-
-    injector = require(igniteModulesInjector);
-}
-catch (ignore) {
-    injector = require(path.join(__dirname, 'injector'));
-}
-
-/**
- * Event listener for HTTP server "error" event.
- */
-const _onError = (addr, error) => {
-    if (error.syscall !== 'listen')
-        throw error;
-
-    // Handle specific listen errors with friendly messages.
-    switch (error.code) {
-        case 'EACCES':
-            console.error(`Requires elevated privileges for bind to ${addr}`);
-            process.exit(1);
-
-            break;
-        case 'EADDRINUSE':
-            console.error(`${addr} is already in use`);
-            process.exit(1);
-
-            break;
-        default:
-            throw error;
-    }
-};
-
-/**
- * @param settings
- * @param {ApiServer} apiSrv
- * @param {AgentsHandler} agentsHnd
- * @param {BrowsersHandler} browsersHnd
- */
-const init = ([settings, apiSrv, agentsHnd, browsersHnd]) => {
-    // Start rest server.
-    const srv = settings.server.SSLOptions ? https.createServer(settings.server.SSLOptions) : http.createServer();
-
-    srv.listen(settings.server.port, settings.server.host);
-
-    const addr = `${settings.server.host}:${settings.server.port}`;
-
-    srv.on('error', _onError.bind(null, addr));
-    srv.on('listening', () => console.log(`Start listening on ${addr}`));
-
-    apiSrv.attach(srv);
-
-    agentsHnd.attach(srv, browsersHnd);
-    browsersHnd.attach(srv, agentsHnd);
-
-    // Used for automated test.
-    if (process.send)
-        process.send('running');
-};
-
-/**
- * Run mongo model migration.
- *
- * @param dbConnectionUri Mongo connection url.
- * @param group Migrations group.
- * @param migrationsPath Migrations path.
- * @param collectionName Name of collection where migrations write info about applied scripts.
- */
-const migrate = (dbConnectionUri, group, migrationsPath, collectionName) => {
-    const migrator = new MigrateMongoose({
-        migrationsPath,
-        dbConnectionUri,
-        collectionName,
-        autosync: true
-    });
-
-    console.log(`Running ${group} migrations...`);
-
-    return migrator.run('up')
-        .then(() => console.log(`All ${group} migrations finished successfully.`))
-        .catch((err) => {
-            const msg = _.get(err, 'message');
-
-            if (_.startsWith(msg, 'There are no migrations to run') || _.startsWith(msg, 'There are no pending migrations.')) {
-                console.log(`There are no ${group} migrations to run.`);
-
-                return;
-            }
-
-            throw err;
-        });
-};
-
-getos(function(e, os) {
-    if (e)
-        return console.log(e);
-
-    console.log('Your OS is: ' + JSON.stringify(os));
-});
+const injector = require('./injector');
 
 injector.log.info = () => {};
 injector.log.debug = () => {};
 
 Promise.all([injector('settings'), injector('mongo')])
     .then(([{mongoUrl}]) => {
-        return migrate(mongoUrl, 'Ignite', path.join(__dirname, 'migrations'))
-            .then(() => migrate(mongoUrl, 'Ignite Modules', path.join(igniteModules, 'migrations'), 'migrationsModules'));
+        return migrate(mongoUrl, 'Ignite', path.join(__dirname, 'migrations'));
     })
     .then(() => Promise.all([injector('settings'), injector('api-server'), injector('agents-handler'), injector('browsers-handler')]))
     .then(init)
diff --git a/modules/web-console/backend/injector.js b/modules/web-console/backend/injector.js
index 754967fa0a318..c30609a4690b8 100644
--- a/modules/web-console/backend/injector.js
+++ b/modules/web-console/backend/injector.js
@@ -24,7 +24,6 @@ module.exports = fireUp.newInjector({
         './errors/**/*.js',
         './middlewares/**/*.js',
         './routes/**/*.js',
-        './services/**/*.js',
-        './ignite_modules/**/*.js'
+        './services/**/*.js'
     ]
 });
diff --git a/modules/web-console/backend/launch-tools.js b/modules/web-console/backend/launch-tools.js
new file mode 100644
index 0000000000000..f1f3b2f4e835c
--- /dev/null
+++ b/modules/web-console/backend/launch-tools.js
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const _ = require('lodash');
+const http = require('http');
+const https = require('https');
+const MigrateMongoose = require('migrate-mongoose');
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+const _onError = (addr, error) => {
+    if (error.syscall !== 'listen')
+        throw error;
+
+    // Handle specific listen errors with friendly messages.
+    switch (error.code) {
+        case 'EACCES':
+            console.error(`Requires elevated privileges for bind to ${addr}`);
+            process.exit(1);
+
+            break;
+        case 'EADDRINUSE':
+            console.error(`${addr} is already in use`);
+            process.exit(1);
+
+            break;
+        default:
+            throw error;
+    }
+};
+
+/**
+ * @param settings
+ * @param {ApiServer} apiSrv
+ * @param {AgentsHandler} agentsHnd
+ * @param {BrowsersHandler} browsersHnd
+ */
+const init = ([settings, apiSrv, agentsHnd, browsersHnd]) => {
+    // Start rest server.
+    const srv = settings.server.SSLOptions ? https.createServer(settings.server.SSLOptions) : http.createServer();
+
+    srv.listen(settings.server.port, settings.server.host);
+
+    const addr = `${settings.server.host}:${settings.server.port}`;
+
+    srv.on('error', _onError.bind(null, addr));
+    srv.on('listening', () => console.log(`Start listening on ${addr}`));
+
+    apiSrv.attach(srv);
+
+    agentsHnd.attach(srv, browsersHnd);
+    browsersHnd.attach(srv, agentsHnd);
+
+    // Used for automated test.
+    if (process.send)
+        process.send('running');
+};
+
+/**
+ * Run mongo model migration.
+ *
+ * @param dbConnectionUri Mongo connection url.
+ * @param group Migrations group.
+ * @param migrationsPath Migrations path.
+ * @param collectionName Name of collection where migrations write info about applied scripts.
+ */
+const migrate = (dbConnectionUri, group, migrationsPath, collectionName) => {
+    const migrator = new MigrateMongoose({
+        migrationsPath,
+        dbConnectionUri,
+        collectionName,
+        autosync: true
+    });
+
+    console.log(`Running ${group} migrations...`);
+
+    return migrator.run('up')
+        .then(() => console.log(`All ${group} migrations finished successfully.`))
+        .catch((err) => {
+            const msg = _.get(err, 'message');
+
+            if (_.startsWith(msg, 'There are no migrations to run') || _.startsWith(msg, 'There are no pending migrations.')) {
+                console.log(`There are no ${group} migrations to run.`);
+
+                return;
+            }
+
+            throw err;
+        });
+};
+
+module.exports = { migrate, init };
diff --git a/modules/web-console/backend/middlewares/api.js b/modules/web-console/backend/middlewares/api.js
index e901ec4d3481c..27e130d4e8e38 100644
--- a/modules/web-console/backend/middlewares/api.js
+++ b/modules/web-console/backend/middlewares/api.js
@@ -19,12 +19,13 @@
 
 // Fire me up!
 
+const _ = require('lodash');
+
 module.exports = {
-    implements: 'middlewares:api',
-    inject: ['require(lodash)']
+    implements: 'middlewares:api'
 };
 
-module.exports.factory = (_) => {
+module.exports.factory = () => {
     return (req, res, next) => {
         // Set headers to avoid API caching in browser (esp. IE)
         res.header('Cache-Control', 'must-revalidate');
@@ -38,11 +39,16 @@ module.exports.factory = (_) => {
 
                 res.status(err.httpCode || err.code || 500).send(err.message);
             },
+
             ok(data) {
                 if (_.isNil(data))
                     return res.sendStatus(404);
 
                 res.status(200).json(data);
+            },
+
+            done() {
+                res.sendStatus(200);
             }
         };
 
diff --git a/modules/web-console/backend/migrations/1516948939797-migrate-configs.js b/modules/web-console/backend/migrations/1516948939797-migrate-configs.js
index cfaf0f2e085ca..adeccf0376240 100644
--- a/modules/web-console/backend/migrations/1516948939797-migrate-configs.js
+++ b/modules/web-console/backend/migrations/1516948939797-migrate-configs.js
@@ -26,6 +26,12 @@ const getCacheForMigration = require('./migration-utils').getCacheForMigration;
 const _debug = false;
 const DUPLICATE_KEY_ERROR = 11000;
 
+let dup = 1;
+
+function makeDup(name) {
+    return name + `_dup_${dup++}`;
+}
+
 function linkCacheToCluster(clustersModel, cluster, cachesModel, cache, domainsModel) {
     return clustersModel.update({_id: cluster._id}, {$addToSet: {caches: cache._id}}).exec()
         .then(() => cachesModel.update({_id: cache._id}, {clusters: [cluster._id]}).exec())
@@ -55,15 +61,20 @@ function cloneCache(clustersModel, cachesModel, domainsModel, cache) {
             delete cache._id;
 
             const newCache = _.clone(cache);
+            const domainIds = newCache.domains;
 
             newCache.clusters = [cluster];
+            newCache.domains = [];
 
             return clustersModel.update({_id: {$in: newCache.clusters}}, {$pull: {caches: cacheId}}, {multi: true}).exec()
                 .then(() => cachesModel.create(newCache))
                 .catch((err) => {
                     if (err.code === DUPLICATE_KEY_ERROR) {
-                        log(`Failed to clone cache, will change cache name and retry [cache=${newCache.name}]`);
-                        newCache.name += '_dup';
+                        const retryWith = makeDup(newCache.name);
+
+                        error(`Failed to clone cache, will change cache name and retry [cache=${newCache.name}, retryWith=${retryWith}]`);
+
+                        newCache.name = retryWith;
 
                         return cachesModel.create(newCache);
                     }
@@ -73,8 +84,6 @@ function cloneCache(clustersModel, cachesModel, domainsModel, cache) {
                 .then((clone) => clustersModel.update({_id: {$in: newCache.clusters}}, {$addToSet: {caches: clone._id}}, {multi: true}).exec()
                     .then(() => clone))
                 .then((clone) => {
-                    const domainIds = newCache.domains;
-
                     if (_.isEmpty(domainIds))
                         return Promise.resolve();
 
@@ -91,14 +100,20 @@ function cloneCache(clustersModel, cachesModel, domainsModel, cache) {
                                 return domainsModel.create(newDomain)
                                     .catch((err) => {
                                         if (err.code === DUPLICATE_KEY_ERROR) {
-                                            log(`Failed to clone domain, will change type name and retry [cache=${newCache.name}, valueType=${newDomain.valueType}]`);
-                                            newDomain.valueType += '_dup';
+                                            const retryWith = makeDup(newDomain.valueType);
+
+                                            error(`Failed to clone domain, will change type name and retry [cache=${newCache.name}, valueType=${newDomain.valueType}, retryWith=${retryWith}]`);
+
+                                            newDomain.valueType = retryWith;
 
                                             return domainsModel.create(newDomain);
                                         }
                                     })
-                                    .then((createdDomain) => clustersModel.update({_id: cluster}, {$addToSet: {models: createdDomain._id}}).exec())
-                                    .catch((err) => error('Failed to clone domain', err));
+                                    .then((createdDomain) => {
+                                        return clustersModel.update({_id: cluster}, {$addToSet: {models: createdDomain._id}}).exec()
+                                            .then(() => cachesModel.update({_id: clone.id}, {$addToSet: {domains: createdDomain._id}}));
+                                    })
+                                    .catch((err) => error('Failed to clone domain during cache clone', err));
                             })
                             .catch((err) => error(`Failed to duplicate domain model[domain=${domainId}], cache=${clone.name}]`, err));
                     }), Promise.resolve());
@@ -112,9 +127,9 @@ function cloneCache(clustersModel, cachesModel, domainsModel, cache) {
 }
 
 function migrateCache(clustersModel, cachesModel, domainsModel, cache) {
-    const len = _.size(cache.clusters);
+    const clustersCnt = _.size(cache.clusters);
 
-    if (len < 1) {
+    if (clustersCnt < 1) {
         if (_debug)
             log(`Found cache not linked to cluster [cache=${cache.name}]`);
 
@@ -122,9 +137,9 @@ function migrateCache(clustersModel, cachesModel, domainsModel, cache) {
             .then((clusterLostFound) => linkCacheToCluster(clustersModel, clusterLostFound, cachesModel, cache, domainsModel));
     }
 
-    if (len > 1) {
+    if (clustersCnt > 1) {
         if (_debug)
-            log(`Found cache linked to many clusters [cache=${cache.name}, cnt=${len}]`);
+            log(`Found cache linked to many clusters [cache=${cache.name}, clustersCnt=${clustersCnt}]`);
 
         return cloneCache(clustersModel, cachesModel, domainsModel, cache);
     }
@@ -136,10 +151,10 @@ function migrateCache(clustersModel, cachesModel, domainsModel, cache) {
 function migrateCaches(clustersModel, cachesModel, domainsModel) {
     return cachesModel.find({}).lean().exec()
         .then((caches) => {
-            const sz = _.size(caches);
+            const cachesCnt = _.size(caches);
 
-            if (sz > 0) {
-                log(`Caches to migrate: ${sz}`);
+            if (cachesCnt > 0) {
+                log(`Caches to migrate: ${cachesCnt}`);
 
                 return _.reduce(caches, (start, cache) => start.then(() => migrateCache(clustersModel, cachesModel, domainsModel, cache)), Promise.resolve())
                     .then(() => log('Caches migration finished.'));
@@ -181,9 +196,9 @@ function cloneIgfs(clustersModel, igfsModel, igfs) {
 }
 
 function migrateIgfs(clustersModel, igfsModel, igfs) {
-    const len = _.size(igfs.clusters);
+    const clustersCnt = _.size(igfs.clusters);
 
-    if (len < 1) {
+    if (clustersCnt < 1) {
         if (_debug)
             log(`Found IGFS not linked to cluster [IGFS=${igfs.name}]`);
 
@@ -191,9 +206,9 @@ function migrateIgfs(clustersModel, igfsModel, igfs) {
             .then((clusterLostFound) => linkIgfsToCluster(clustersModel, clusterLostFound, igfsModel, igfs));
     }
 
-    if (len > 1) {
+    if (clustersCnt > 1) {
         if (_debug)
-            log(`Found IGFS linked to many clusters [IGFS=${igfs.name}, cnt=${len}]`);
+            log(`Found IGFS linked to many clusters [IGFS=${igfs.name}, clustersCnt=${clustersCnt}]`);
 
         return cloneIgfs(clustersModel, igfsModel, igfs);
     }
@@ -205,10 +220,10 @@ function migrateIgfs(clustersModel, igfsModel, igfs) {
 function migrateIgfss(clustersModel, igfsModel) {
     return igfsModel.find({}).lean().exec()
         .then((igfss) => {
-            const sz = _.size(igfss);
+            const igfsCnt = _.size(igfss);
 
-            if (sz > 0) {
-                log(`IGFS to migrate: ${sz}`);
+            if (igfsCnt > 0) {
+                log(`IGFS to migrate: ${igfsCnt}`);
 
                 return _.reduce(igfss, (start, igfs) => start.then(() => migrateIgfs(clustersModel, igfsModel, igfs)), Promise.resolve())
                     .then(() => log('IGFS migration finished.'));
@@ -228,11 +243,13 @@ function linkDomainToCluster(clustersModel, cluster, domainsModel, domain) {
 function linkDomainToCache(cachesModel, cache, domainsModel, domain) {
     return cachesModel.update({_id: cache._id}, {$addToSet: {domains: domain._id}}).exec()
         .then(() => domainsModel.update({_id: domain._id}, {caches: [cache._id]}).exec())
-        .catch((err) => error(`Failed link domain model to cache[cache=${cache.name}, domain=${domain._id}]`, err));
+        .catch((err) => error(`Failed link domain model to cache [cache=${cache.name}, domain=${domain._id}]`, err));
 }
 
 function migrateDomain(clustersModel, cachesModel, domainsModel, domain) {
-    if (_.isEmpty(domain.caches)) {
+    const cachesCnt = _.size(domain.caches);
+
+    if (cachesCnt < 1) {
         if (_debug)
             log(`Found domain model not linked to cache [domain=${domain._id}]`);
 
@@ -244,16 +261,51 @@ function migrateDomain(clustersModel, cachesModel, domainsModel, domain) {
     }
 
     if (_.isEmpty(domain.clusters)) {
-        return cachesModel.findOne({_id: {$in: domain.caches}}).lean().exec()
-            .then((cache) => {
-                if (cache) {
-                    const clusterId = cache.clusters[0];
+        const cachesCnt = _.size(domain.caches);
 
-                    return domainsModel.update({_id: domain._id}, {clusters: [clusterId]}).exec()
-                        .then(() => clustersModel.update({_id: clusterId}, {$addToSet: {models: domain._id}}).exec());
+        if (_debug)
+            log(`Found domain model without cluster: [domain=${domain._id}, cachesCnt=${cachesCnt}]`);
+
+        const grpByClusters = {};
+
+        return cachesModel.find({_id: {$in: domain.caches}}).lean().exec()
+            .then((caches) => {
+                if (caches) {
+                    _.forEach(caches, (cache) => {
+                        const c = _.get(grpByClusters, cache.clusters[0]);
+
+                        if (c)
+                            c.push(cache._id);
+                        else
+                            grpByClusters[cache.clusters[0]] = [cache._id];
+                    });
+
+                    return _.reduce(_.keys(grpByClusters), (start, cluster, idx) => start.then(() => {
+                        const domainId = domain._id;
+
+                        const clusterCaches = grpByClusters[cluster];
+
+                        if (idx > 0) {
+                            delete domain._id;
+                            domain.caches = clusterCaches;
+
+                            return domainsModel.create(domain)
+                                .then((clonedDomain) => {
+                                    return cachesModel.update({_id: {$in: clusterCaches}}, {$addToSet: {domains: clonedDomain._id}}).exec()
+                                        .then(() => clonedDomain);
+                                })
+                                .then((clonedDomain) => linkDomainToCluster(clustersModel, {_id: cluster, name: `stub${idx}`}, domainsModel, clonedDomain))
+                                .then(() => {
+                                    return cachesModel.update({_id: {$in: clusterCaches}}, {$pull: {domains: domainId}}, {multi: true}).exec();
+                                });
+                        }
+
+                        return domainsModel.update(domainsModel.update({_id: domainId}, {caches: clusterCaches}).exec())
+                            .then(() => linkDomainToCluster(clustersModel, {_id: cluster, name: `stub${idx}`}, domainsModel, domain));
+                    }), Promise.resolve());
                 }
 
-                log(`Found broken domain: [domain=${domain._id}, caches=${domain.caches}]`);
+                error(`Found domain with orphaned caches: [domain=${domain._id}, caches=${domain.caches}]`);
 
                 return Promise.resolve();
             })
@@ -267,10 +319,10 @@ function migrateDomain(clustersModel, cachesModel, domainsModel, domain) {
 function migrateDomains(clustersModel, cachesModel, domainsModel) {
     return domainsModel.find({}).lean().exec()
         .then((domains) => {
-            const sz = _.size(domains);
+            const domainsCnt = _.size(domains);
 
-            if (sz > 0) {
-                log(`Domain models to migrate: ${sz}`);
+            if (domainsCnt > 0) {
+                log(`Domain models to migrate: ${domainsCnt}`);
 
                 return _.reduce(domains, (start, domain) => start.then(() => migrateDomain(clustersModel, cachesModel, domainsModel, domain)), Promise.resolve())
                     .then(() => log('Domain models migration finished.'));
@@ -335,6 +387,7 @@ exports.up = function up(done) {
         .then(() => migrateCaches(clustersModel, cachesModel, domainsModel))
         .then(() => migrateIgfss(clustersModel, igfsModel))
         .then(() => migrateDomains(clustersModel, cachesModel, domainsModel))
+        .then(() => log(`Duplicates counter: ${dup}`))
         .then(() => done())
         .catch(done);
 };
diff --git a/modules/web-console/backend/migrations/migration-utils.js b/modules/web-console/backend/migrations/migration-utils.js
index 0397247846ab4..8b7dd47869056 100644
--- a/modules/web-console/backend/migrations/migration-utils.js
+++ b/modules/web-console/backend/migrations/migration-utils.js
@@ -20,7 +20,7 @@ function log(msg) {
 }
 
 function error(msg, err) {
-    console.log(`[${new Date().toISOString()}] [ERROR] ${msg}. Error: ${err}`);
+    console.log(`[${new Date().toISOString()}] [ERROR] ${msg}.` + (err ? ` Error: ${err}` : ''));
 }
 
 function recreateIndex0(done, model, oldIdxName, oldIdx, newIdx) {
@@ -52,19 +52,11 @@ function recreateIndex(done, model, oldIdxName, oldIdx, newIdx) {
 
 const LOST_AND_FOUND = 'LOST_AND_FOUND';
 
-let _clusterLostAndFound = null;
-
 function getClusterForMigration(clustersModel, space) {
-    if (_clusterLostAndFound)
-        return Promise.resolve(_clusterLostAndFound);
-
-    return clustersModel.findOne({name: LOST_AND_FOUND}).lean().exec()
+    return clustersModel.findOne({space, name: LOST_AND_FOUND}).lean().exec()
         .then((cluster) => {
-            if (cluster) {
-                _clusterLostAndFound = cluster;
-
+            if (cluster)
                 return cluster;
-            }
 
             return clustersModel.create({
                 space,
@@ -82,28 +74,15 @@ function getClusterForMigration(clustersModel, space) {
                     Multicast: {addresses: ['127.0.0.1:47500..47510']},
                     Vm: {addresses: ['127.0.0.1:47500..47510']}
                 }
-            })
-                .then((cluster) => {
-                    _clusterLostAndFound = cluster;
-
-                    return cluster;
-                });
+            });
         });
 }
 
-let _cacheLostAndFound = null;
-
 function getCacheForMigration(clustersModel, cachesModel, space) {
-    if (_cacheLostAndFound)
-        return Promise.resolve(_cacheLostAndFound);
-
-    return cachesModel.findOne({name: LOST_AND_FOUND})
+    return cachesModel.findOne({space, name: LOST_AND_FOUND})
         .then((cache) => {
-            if (cache) {
-                _cacheLostAndFound = cache;
-
+            if (cache)
                 return cache;
-            }
 
             return getClusterForMigration(clustersModel, space)
                 .then((cluster) => {
@@ -131,11 +110,6 @@ function getCacheForMigration(clustersModel, cachesModel, space) {
                 .then((cache) => {
                     return clustersModel.update({_id: cache.clusters[0]}, {$addToSet: {caches: cache._id}}).exec()
                         .then(() => cache);
-                })
-                .then((cache) => {
-                    _cacheLostAndFound = cache;
-
-                    return cache;
                 });
         });
 }
diff --git a/modules/web-console/backend/package.json b/modules/web-console/backend/package.json
index 0aa56c91aa0e6..896b5fa8cd935 100644
--- a/modules/web-console/backend/package.json
+++ b/modules/web-console/backend/package.json
@@ -1,31 +1,24 @@
 {
   "name": "ignite-web-console",
-  "version": "1.0.0",
+  "version": "2.5.0",
   "description": "Interactive Web console for configuration, executing SQL queries and monitoring of Apache Ignite Cluster",
   "private": true,
+  "main": "index.js",
   "scripts": {
     "ci-test": "cross-env NODE_ENV=test MOCHA_REPORTER=mocha-teamcity-reporter node ./test/index.js",
     "test": "cross-env NODE_ENV=test CONFIG_PATH='./test/config/settings.json' node ./test/index.js",
     "eslint": "eslint --env node --format node_modules/eslint-friendly-formatter ./ -- --eff-by-issue",
     "start": "node ./index.js",
-    "build": "pkg . --out-path build",
-    "mongodb-download": "./node_modules/.bin/mongodb-download"
+    "build": "pkg . --out-path build"
   },
-  "author": "",
-  "contributors": [
-    {
-      "name": "",
-      "email": ""
-    }
-  ],
   "license": "Apache-2.0",
   "keywords": [
     "Apache Ignite Web console"
   ],
   "homepage": "https://ignite.apache.org/",
   "engines": {
-    "npm": "^3.x.x",
-    "node": "^6.5.x"
+    "npm": ">=5.x.x",
+    "node": ">=8.x.x <10.x.x"
   },
   "os": [
     "darwin",
@@ -35,49 +28,52 @@
   "bin": "index.js",
   "pkg": {
     "assets": [
+      "app/*",
+      "errors/*",
+      "middlewares/*",
+      "migrations/*",
+      "routes/*",
+      "services/*",
+      "node_modules/getos/logic/*",
+      "node_modules/mongodb-download/node_modules/getos/logic/*"
+    ],
+    "scripts": [
       "app/*.js",
       "errors/*.js",
-      "ignite_modules/*",
-      "injector.js",
       "middlewares/*.js",
-      "migrations/*",
-      "node_modules/getos/logic/*.js",
+      "migrations/*.js",
       "routes/*.js",
-      "routes/**/*.json",
       "services/*.js"
     ]
   },
   "dependencies": {
     "app-module-path": "2.2.0",
     "body-parser": "1.17.2",
-    "common-tags": "1.4.0",
     "connect-mongo": "1.3.2",
     "cookie-parser": "1.4.3",
     "express": "4.15.3",
     "express-session": "1.15.4",
     "fire-up": "1.0.0",
     "glob": "7.1.2",
-    "getos": "3.1.0",
     "jszip": "3.1.3",
-    "lodash": "4.17.4",
+    "lodash": "4.17.10",
     "migrate-mongoose": "3.2.2",
     "mongodb-prebuilt": "6.3.3",
     "mongoose": "4.11.4",
     "morgan": "1.8.2",
-    "nexmo": "2.0.2",
     "nconf": "0.8.4",
     "nodemailer": "4.0.1",
     "passport": "0.3.2",
     "passport-local": "1.0.0",
     "passport-local-mongoose": "4.0.0",
     "passport.socketio": "3.7.0",
+    "pkg": "4.3.1",
     "socket.io": "1.7.3",
-    "uuid": "3.1.0",
-    "pkg": "4.2.4"
+    "uuid": "3.1.0"
   },
   "devDependencies": {
     "chai": "4.1.0",
-    "cross-env": "5.0.1",
+    "cross-env": "5.1.6",
     "eslint": "4.3.0",
     "eslint-friendly-formatter": "3.0.0",
     "mocha": "3.4.2",
diff --git a/modules/web-console/backend/routes/admin.js b/modules/web-console/backend/routes/admin.js
index 2c34cdb7d675e..4a38723e2f349 100644
--- a/modules/web-console/backend/routes/admin.js
+++ b/modules/web-console/backend/routes/admin.js
@@ -79,10 +79,10 @@ module.exports.factory = function(settings, mongo, spacesService, mailsService,
                 .catch(res.api.error);
         });
 
-        // Revert to your identity.
+        // Update notifications.
         router.put('/notifications', (req, res) => {
             notificationsService.merge(req.user._id, req.body.message, req.body.isShown)
-                .then(res.api.ok)
+                .then(res.api.done)
                 .catch(res.api.error);
         });
 
diff --git a/modules/web-console/backend/services/clusters.js b/modules/web-console/backend/services/clusters.js
index 0cc2b9f4a3321..470b9a3aeaeae 100644
--- a/modules/web-console/backend/services/clusters.js
+++ b/modules/web-console/backend/services/clusters.js
@@ -94,8 +94,11 @@ module.exports.factory = (mongo, spacesService, cachesService, modelsService, ig
      * @returns {Promise.<RemoveResult>} - that resolves results of remove operation.
      */
     const removeAllBySpaces = (spaceIds) => {
-        return mongo.Cache.update({space: {$in: spaceIds}}, {clusters: []}, {multi: true}).exec()
-            .then(() => mongo.Igfs.update({space: {$in: spaceIds}}, {clusters: []}, {multi: true}).exec())
+        return Promise.all([
+            mongo.DomainModel.remove({space: {$in: spaceIds}}).exec(),
+            mongo.Cache.remove({space: {$in: spaceIds}}).exec(),
+            mongo.Igfs.remove({space: {$in: spaceIds}}).exec()
+        ])
             .then(() => mongo.Cluster.remove({space: {$in: spaceIds}}).exec());
     };
 
@@ -245,14 +248,18 @@ module.exports.factory = (mongo, spacesService, cachesService, modelsService, ig
             return Promise.all(_.map(ids, (id) => {
                 return mongo.Cluster.findByIdAndRemove(id).exec()
                     .then((cluster) => {
+                        if (_.isNil(cluster))
+                            return 0;
+
                         return Promise.all([
                             mongo.DomainModel.remove({_id: {$in: cluster.models}}).exec(),
                             mongo.Cache.remove({_id: {$in: cluster.caches}}).exec(),
                             mongo.Igfs.remove({_id: {$in: cluster.igfss}}).exec()
-                        ]);
+                        ])
+                            .then(() => 1);
                     });
             }))
-                .then(() => ({rowsAffected: ids.length}));
+                .then((res) => ({rowsAffected: _.sum(res)}));
         }
 
         /**
diff --git a/modules/web-console/backend/services/downloads.js b/modules/web-console/backend/services/downloads.js
index 6a03fe29a597a..d7f616a5ae0e4 100644
--- a/modules/web-console/backend/services/downloads.js
+++ b/modules/web-console/backend/services/downloads.js
@@ -64,6 +64,8 @@ module.exports.factory = (settings, agentsHnd, errors) => {
                             prop.push(`server-uri=${host}`);
                             prop.push('#Uncomment following options if needed:');
                             prop.push('#node-uri=http://localhost:8080');
+                            prop.push('#node-login=ignite');
+                            prop.push('#node-password=ignite');
                             prop.push('#driver-folder=./jdbc-drivers');
 
                             zip.file(`${folder}/default.properties`, prop.join('\n'));
diff --git a/modules/web-console/backend/services/notifications.js b/modules/web-console/backend/services/notifications.js
index af70aeeb7aeca..4dacf510c7093 100644
--- a/modules/web-console/backend/services/notifications.js
+++ b/modules/web-console/backend/services/notifications.js
@@ -42,9 +42,7 @@ module.exports.factory = (mongo, browsersHnd) => {
          */
         static merge(owner, message, isShown = false, date = new Date()) {
             return mongo.Notifications.create({owner, message, date, isShown})
-                .then(({message, date, isShown}) => {
-                    browsersHnd.updateNotification({message, date, isShown});
-                });
+                .then(({message, date, isShown}) => browsersHnd.updateNotification({message, date, isShown}));
         }
     }
 
diff --git a/modules/web-console/backend/test/app/db.js b/modules/web-console/backend/test/app/db.js
index e07f887ab6e96..68a75f35ade4b 100644
--- a/modules/web-console/backend/test/app/db.js
+++ b/modules/web-console/backend/test/app/db.js
@@ -19,6 +19,8 @@
 
 // Fire me up!
 
+const _ = require('lodash');
+
 const testAccounts = require('../data/accounts.json');
 const testClusters = require('../data/clusters.json');
 const testCaches = require('../data/caches.json');
@@ -28,10 +30,10 @@ const testSpaces = require('../data/spaces.json');
 
 module.exports = {
     implements: 'dbHelper',
-    inject: ['require(lodash)', 'mongo', 'mongoose']
+    inject: ['mongo', 'mongoose']
 };
 
-module.exports.factory = (_, mongo, mongoose) => {
+module.exports.factory = (mongo, mongoose) => {
     const prepareUserSpaces = () => Promise.all([mongo.Account.create(testAccounts), mongo.Space.create(testSpaces)]);
     const prepareClusters = () => mongo.Cluster.create(testClusters);
     const prepareDomains = () => mongo.DomainModel.create(testDomains);
diff --git a/modules/web-console/backend/test/app/httpAgent.js b/modules/web-console/backend/test/app/httpAgent.js
index 76b191e6fb1bb..2b660faacb8dd 100644
--- a/modules/web-console/backend/test/app/httpAgent.js
+++ b/modules/web-console/backend/test/app/httpAgent.js
@@ -35,7 +35,7 @@ module.exports.factory = (apiSrv, http, request) => {
 
             return new Promise((resolve, reject) => {
                 authAgentInstance = request.agent(express);
-                authAgentInstance.post('/signin')
+                authAgentInstance.post('/api/v1/signin')
                     .send({email, password})
                     .end((err, res) => {
                         if (res.status === 401 || err)
diff --git a/modules/web-console/backend/test/data/caches.json b/modules/web-console/backend/test/data/caches.json
index 697d4148562ef..dd14b4e1eaae7 100644
--- a/modules/web-console/backend/test/data/caches.json
+++ b/modules/web-console/backend/test/data/caches.json
@@ -16,7 +16,7 @@
       }
     },
     "domains": ["000000000000000000000001", "000000000000000000000002", "000000000000000000000003", "000000000000000000000004", "000000000000000000000005"],
-    "clusters": ["000000000000000000000001", "000000000000000000000002"]
+    "clusters": ["000000000000000000000001"]
   },
   {
     "_id" : "000000000000000000000002",
@@ -35,10 +35,48 @@
       }
     },
     "domains": ["000000000000000000000001", "000000000000000000000002", "000000000000000000000003", "000000000000000000000004", "000000000000000000000005"],
-    "clusters": ["000000000000000000000001", "000000000000000000000002"]
+    "clusters": ["000000000000000000000001"]
   },
   {
-    "_id" : "000000000000000000000003",
+    "_id" : "000000000000000000000021",
+    "space": "000000000000000000000001",
+    "name": "CarCache",
+    "cacheMode": "PARTITIONED",
+    "atomicityMode": "ATOMIC",
+    "readThrough": true,
+    "writeThrough": true,
+    "sqlFunctionClasses": [],
+    "cacheStoreFactory": {
+      "kind": "CacheJdbcPojoStoreFactory",
+      "CacheJdbcPojoStoreFactory": {
+        "dataSourceBean": "dsH2",
+        "dialect": "H2"
+      }
+    },
+    "domains": ["000000000000000000000023", "000000000000000000000024", "000000000000000000000025"],
+    "clusters": ["000000000000000000000020"]
+  },
+  {
+    "_id" : "000000000000000000000022",
+    "space": "000000000000000000000001",
+    "name": "ParkingCache",
+    "cacheMode": "PARTITIONED",
+    "atomicityMode": "ATOMIC",
+    "readThrough": true,
+    "writeThrough": true,
+    "sqlFunctionClasses": [],
+    "cacheStoreFactory": {
+      "kind": "CacheJdbcPojoStoreFactory",
+      "CacheJdbcPojoStoreFactory": {
+        "dataSourceBean": "dsH2",
+        "dialect": "H2"
+      }
+    },
+    "domains": ["000000000000000000000023", "000000000000000000000024", "000000000000000000000025"],
+    "clusters": ["000000000000000000000020"]
+  },
+  {
+    "_id" : "000000000000000000000023",
     "space": "000000000000000000000001",
     "name": "CountryCache",
     "cacheMode": "PARTITIONED",
@@ -53,11 +91,11 @@
         "dialect": "H2"
       }
     },
-    "domains": ["000000000000000000000001", "000000000000000000000002", "000000000000000000000003", "000000000000000000000004", "000000000000000000000005"],
-    "clusters": ["000000000000000000000002"]
+    "domains": ["000000000000000000000023", "000000000000000000000024", "000000000000000000000025"],
+    "clusters": ["000000000000000000000020"]
   },
   {
-    "_id" : "000000000000000000000004",
+    "_id" : "000000000000000000000024",
     "space": "000000000000000000000001",
     "name": "DepartmentCache",
     "cacheMode": "PARTITIONED",
@@ -72,11 +110,11 @@
         "dialect": "H2"
       }
     },
-    "domains": ["000000000000000000000001", "000000000000000000000002", "000000000000000000000003", "000000000000000000000004", "000000000000000000000005"],
-    "clusters": ["000000000000000000000002"]
+    "domains": ["000000000000000000000023", "000000000000000000000024", "000000000000000000000025"],
+    "clusters": ["000000000000000000000020"]
   },
   {
-    "_id" : "000000000000000000000005",
+    "_id" : "000000000000000000000025",
     "space": "000000000000000000000001",
     "name": "EmployeeCache",
     "cacheMode": "PARTITIONED",
@@ -91,7 +129,7 @@
         "dialect": "H2"
       }
     },
-    "domains": ["000000000000000000000001", "000000000000000000000002", "000000000000000000000003", "000000000000000000000004", "000000000000000000000005"],
-    "clusters": ["000000000000000000000002"]
+    "domains": ["000000000000000000000023", "000000000000000000000024", "000000000000000000000025"],
+    "clusters": ["000000000000000000000020"]
   }
 ]
diff --git a/modules/web-console/backend/test/data/clusters.json b/modules/web-console/backend/test/data/clusters.json
index 8e16e76dfb1b0..55e05ce307abd 100644
--- a/modules/web-console/backend/test/data/clusters.json
+++ b/modules/web-console/backend/test/data/clusters.json
@@ -11,6 +11,7 @@
     },
     "igfss": ["000000000000000000000001"],
     "caches": ["000000000000000000000001", "000000000000000000000002"],
+    "models": ["000000000000000000000001", "000000000000000000000002"],
     "binaryConfiguration": {
       "compactFooter": true,
       "typeConfigurations": []
@@ -26,7 +27,7 @@
     }
   },
   {
-    "_id" : "000000000000000000000002",
+    "_id" : "000000000000000000000020",
     "space": "000000000000000000000001",
     "name": "cluster-caches",
     "connector": {
@@ -36,7 +37,8 @@
       "tcpNoDelay": true
     },
     "igfss": [],
-    "caches": ["000000000000000000000001", "000000000000000000000002", "000000000000000000000003", "000000000000000000000004", "000000000000000000000005"],
+    "caches": ["000000000000000000000021", "000000000000000000000022", "000000000000000000000023", "000000000000000000000024", "000000000000000000000025"],
+    "models": ["000000000000000000000023", "000000000000000000000024", "000000000000000000000025"],
     "binaryConfiguration": {
       "compactFooter": true,
       "typeConfigurations": []
diff --git a/modules/web-console/backend/test/data/domains.json b/modules/web-console/backend/test/data/domains.json
index e2662db589c1b..1585e98f82af3 100644
--- a/modules/web-console/backend/test/data/domains.json
+++ b/modules/web-console/backend/test/data/domains.json
@@ -41,7 +41,8 @@
         "javaFieldType": "int"
       }
     ],
-    "caches": []
+    "caches": [],
+    "clusters": ["000000000000000000000001"]
   },
   {
     "_id" : "000000000000000000000002",
@@ -85,10 +86,11 @@
         "javaFieldType": "int"
       }
     ],
-    "caches": []
+    "caches": [],
+    "clusters": ["000000000000000000000001"]
   },
   {
-    "_id" : "000000000000000000000003",
+    "_id" : "000000000000000000000023",
     "space": "000000000000000000000001",
     "keyType": "Integer",
     "valueType": "model.Employee",
@@ -224,10 +226,11 @@
         "javaFieldType": "int"
       }
     ],
-    "caches": []
+    "caches": [],
+    "clusters": ["000000000000000000000002"]
   },
   {
-    "_id" : "000000000000000000000004",
+    "_id" : "000000000000000000000024",
     "space": "000000000000000000000001",
     "keyType": "Integer",
     "valueType": "model.Country",
@@ -268,10 +271,11 @@
         "javaFieldType": "int"
       }
     ],
-    "caches": []
+    "caches": [],
+    "clusters": ["000000000000000000000002"]
   },
   {
-    "_id" : "000000000000000000000005",
+    "_id" : "000000000000000000000025",
     "space": "000000000000000000000001",
     "keyType": "Integer",
     "valueType": "model.Car",
@@ -312,6 +316,7 @@
         "javaFieldType": "int"
       }
     ],
-    "caches": []
+    "caches": [],
+    "clusters": ["000000000000000000000002"]
   }
 ]
diff --git a/modules/web-console/backend/test/index.js b/modules/web-console/backend/test/index.js
index 4519219c795c0..258a876618716 100644
--- a/modules/web-console/backend/test/index.js
+++ b/modules/web-console/backend/test/index.js
@@ -17,14 +17,10 @@
 
 const Mocha = require('mocha');
 const glob = require('glob');
-const path = require('path');
 
 const mocha = new Mocha({ui: 'tdd', reporter: process.env.MOCHA_REPORTER || 'spec'});
 const testPath = ['./test/unit/**/*.js', './test/routes/**/*.js'];
 
-if (process.env.IGNITE_MODULES)
-    testPath.push(path.join(process.env.IGNITE_MODULES, 'backend', 'test', 'unit', '**', '*.js'));
-
 testPath
     .map((mask) => glob.sync(mask))
     .reduce((acc, items) => acc.concat(items), [])
diff --git a/modules/web-console/backend/test/injector.js b/modules/web-console/backend/test/injector.js
index bdeca91a82dd9..a7c7d59578228 100644
--- a/modules/web-console/backend/test/injector.js
+++ b/modules/web-console/backend/test/injector.js
@@ -15,34 +15,19 @@
  * limitations under the License.
  */
 
-const fs = require('fs');
 const path = require('path');
 const fireUp = require('fire-up');
 
-const igniteModules = process.env.IGNITE_MODULES || './ignite_modules';
-
-let injector;
-
-try {
-    const igniteModulesInjector = path.resolve(path.join(igniteModules, 'backend', 'test', 'injector.js'));
-
-    fs.accessSync(igniteModulesInjector, fs.F_OK);
-
-    injector = require(igniteModulesInjector);
-} catch (ignore) {
-    injector = fireUp.newInjector({
-        basePath: path.join(__dirname, '../'),
-        modules: [
-            './app/**/*.js',
-            './config/**/*.js',
-            './errors/**/*.js',
-            './middlewares/**/*.js',
-            './routes/**/*.js',
-            './services/**/*.js',
-            './test/app/*.js'
-        ],
-        use: ['mongoose:mock']
-    });
-}
-
-module.exports = injector;
+module.exports = fireUp.newInjector({
+    basePath: path.join(__dirname, '../'),
+    modules: [
+        './app/**/*.js',
+        './config/**/*.js',
+        './errors/**/*.js',
+        './middlewares/**/*.js',
+        './routes/**/*.js',
+        './services/**/*.js',
+        './test/app/*.js'
+    ],
+    use: ['mongoose:mock']
+});
diff --git a/modules/web-console/backend/test/routes/clusters.js b/modules/web-console/backend/test/routes/clusters.js
index b9f6565356048..c682ec5234eac 100644
--- a/modules/web-console/backend/test/routes/clusters.js
+++ b/modules/web-console/backend/test/routes/clusters.js
@@ -17,7 +17,6 @@
 
 const assert = require('chai').assert;
 const injector = require('../injector');
-const mongoose = require('mongoose');
 
 let agentFactory;
 let db;
@@ -36,16 +35,16 @@ suite('routes.clusters', () => {
     });
 
     test('Save cluster model', (done) => {
-        const newCluster = Object.assign({}, db.mocks.clusters[0], {name: 'newClusterName'});
+        const cluster = Object.assign({}, db.mocks.clusters[0], {name: 'newClusterName'});
 
         agentFactory.authAgent(db.mocks.accounts[0])
             .then((agent) => {
-                agent.post('/configuration/clusters/save')
-                    .send(newCluster)
+                agent.put('/api/v1/configuration/clusters')
+                    .send({cluster})
                     .expect(200)
                     .expect((res) => {
                         assert.isNotNull(res.body);
-                        assert.isTrue(mongoose.Types.ObjectId.isValid(res.body));
+                        assert.equal(res.body.rowsAffected, 1);
                     })
                     .end(done);
             })
@@ -55,7 +54,7 @@ suite('routes.clusters', () => {
     test('Remove cluster model', (done) => {
         agentFactory.authAgent(db.mocks.accounts[0])
             .then((agent) => {
-                agent.post('/configuration/clusters/remove')
+                agent.post('/api/v1/configuration/clusters/remove')
                     .send({_id: db.mocks.clusters[0]._id})
                     .expect(200)
                     .expect((res) => {
@@ -70,7 +69,7 @@ suite('routes.clusters', () => {
     test('Remove all clusters', (done) => {
         agentFactory.authAgent(db.mocks.accounts[0])
             .then((agent) => {
-                agent.post('/configuration/clusters/remove/all')
+                agent.post('/api/v1/configuration/clusters/remove/all')
                     .expect(200)
                     .expect((res) => {
                         assert.isNotNull(res.body);
diff --git a/modules/web-console/backend/test/routes/public.js b/modules/web-console/backend/test/routes/public.js
index 3c573c5fa3e32..fe7793c41c1a1 100644
--- a/modules/web-console/backend/test/routes/public.js
+++ b/modules/web-console/backend/test/routes/public.js
@@ -41,7 +41,7 @@ suite('routes.public', () => {
 
         agentFactory.guestAgent()
             .then((agent) => {
-                agent.post('/signin')
+                agent.post('/api/v1/signin')
                     .send({email: user.email, password: user.password})
                     .expect(200)
                     .expect((res) => {
@@ -58,7 +58,7 @@ suite('routes.public', () => {
 
         agentFactory.guestAgent()
             .then((agent) => {
-                agent.post('/signin')
+                agent.post('/api/v1/signin')
                     .send({email: user.email, password: 'notvalidpassword'})
                     .expect(401)
                     .end(done);
diff --git a/modules/web-console/backend/test/unit/AuthService.test.js b/modules/web-console/backend/test/unit/AuthService.test.js
index 5ce473d272013..e33b6bd094a05 100644
--- a/modules/web-console/backend/test/unit/AuthService.test.js
+++ b/modules/web-console/backend/test/unit/AuthService.test.js
@@ -37,17 +37,6 @@ suite('AuthServiceTestsSuite', () => {
 
     setup(() => db.init());
 
-    test('Check token generator', () => {
-        const tokenLength = 16;
-        const token1 = authService.generateResetToken(tokenLength);
-        const token2 = authService.generateResetToken(tokenLength);
-
-        assert.equal(token1.length, tokenLength);
-        assert.equal(token2.length, tokenLength);
-        assert.notEqual(token1, token2);
-    });
-
-
     test('Reset password token for non existing user', (done) => {
         authService.resetPasswordToken('non-exisitng@email.ee')
             .catch((err) => {
diff --git a/modules/web-console/backend/test/unit/CacheService.test.js b/modules/web-console/backend/test/unit/CacheService.test.js
index 52936a0743d2b..419b9f7ae739c 100644
--- a/modules/web-console/backend/test/unit/CacheService.test.js
+++ b/modules/web-console/backend/test/unit/CacheService.test.js
@@ -127,7 +127,7 @@ suite('CacheServiceTestsSuite', () => {
     test('Get all caches by space', (done) => {
         cachesService.listBySpaces(testSpaces[0]._id)
             .then((caches) =>
-                assert.equal(caches.length, 5)
+                assert.equal(caches.length, 7)
             )
             .then(done)
             .catch(done);
@@ -136,7 +136,7 @@ suite('CacheServiceTestsSuite', () => {
     test('Remove all caches in space', (done) => {
         cachesService.removeAll(testAccounts[0]._id, false)
             .then(({rowsAffected}) =>
-                assert.equal(rowsAffected, 5)
+                assert.equal(rowsAffected, 7)
             )
             .then(done)
             .catch(done);
diff --git a/modules/web-console/backend/test/unit/ClusterService.test.js b/modules/web-console/backend/test/unit/ClusterService.test.js
index 66c7cf1c6453d..93edfc1f464d2 100644
--- a/modules/web-console/backend/test/unit/ClusterService.test.js
+++ b/modules/web-console/backend/test/unit/ClusterService.test.js
@@ -153,17 +153,17 @@ suite('ClusterServiceTestsSuite', () => {
             .then((clusters) => {
                 assert.equal(clusters.length, 2);
 
-                assert.equal(clusters[0].name, 'cluster-caches');
+                assert.equal(clusters[0].name, 'cluster-igfs');
                 assert.isNotNull(clusters[0].discovery);
-                assert.equal(clusters[0].cachesCount, 5);
-                assert.equal(clusters[0].modelsCount, 5);
-                assert.equal(clusters[0].igfsCount, 0);
+                assert.equal(clusters[0].cachesCount, 2);
+                assert.equal(clusters[0].modelsCount, 2);
+                assert.equal(clusters[0].igfsCount, 1);
 
-                assert.equal(clusters[1].name, 'cluster-igfs');
+                assert.equal(clusters[1].name, 'cluster-caches');
                 assert.isNotNull(clusters[1].discovery);
-                assert.equal(clusters[1].cachesCount, 2);
-                assert.equal(clusters[1].modelsCount, 5);
-                assert.equal(clusters[1].igfsCount, 1);
+                assert.equal(clusters[1].cachesCount, 5);
+                assert.equal(clusters[1].modelsCount, 3);
+                assert.equal(clusters[1].igfsCount, 0);
             })
             .then(done)
             .catch(done);
@@ -179,7 +179,7 @@ suite('ClusterServiceTestsSuite', () => {
             .then((output) => {
                 assert.isNotNull(output);
 
-                assert.equal(output.n, 1);
+                assert.equal(output.rowsAffected, 1);
             })
             .then(() => clusterService.get(testAccounts[0]._id, false, cluster._id))
             .then((savedCluster) => {
@@ -277,9 +277,7 @@ suite('ClusterServiceTestsSuite', () => {
             })
             .then(() => cacheService.get(testAccounts[0]._id, false, _.head(testClusters).caches[1]))
             .then((c2) => {
-                assert.isNotNull(c2);
-                assert.equal(c2.cacheMode, 'PARTITIONED');
-                assert.isTrue(c2.readThrough);
+                assert.isNull(c2);
             })
             .then(done)
             .catch(done);
diff --git a/modules/web-console/backend/test/unit/Utils.test.js b/modules/web-console/backend/test/unit/Utils.test.js
new file mode 100644
index 0000000000000..10aa0d02ad5f0
--- /dev/null
+++ b/modules/web-console/backend/test/unit/Utils.test.js
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const assert = require('chai').assert;
+const injector = require('../injector');
+
+let utils;
+let errors;
+let db;
+
+suite('UtilsTestsSuite', () => {
+    suiteSetup(() => {
+        return Promise.all([injector('services/utils'),
+            injector('errors'),
+            injector('dbHelper')])
+            .then(([_utils, _errors, _db]) => {
+                utils = _utils;
+                errors = _errors;
+                db = _db;
+            });
+    });
+
+    setup(() => db.init());
+
+    test('Check token generator', () => {
+        const tokenLength = 16;
+        const token1 = utils.randomString(tokenLength);
+        const token2 = utils.randomString(tokenLength);
+
+        assert.equal(token1.length, tokenLength);
+        assert.equal(token2.length, tokenLength);
+        assert.notEqual(token1, token2);
+    });
+});
diff --git a/modules/web-console/docker/compose/backend/.dockerignore b/modules/web-console/docker/compose/backend/.dockerignore
deleted file mode 100644
index 05df665e2b560..0000000000000
--- a/modules/web-console/docker/compose/backend/.dockerignore
+++ /dev/null
@@ -1,3 +0,0 @@
-build/config/*.json
-build/node_modules
-build/test
diff --git a/modules/web-console/docker/compose/backend/Dockerfile b/modules/web-console/docker/compose/backend/Dockerfile
index 6ae4c93ad36a6..de2652ce6eb4f 100644
--- a/modules/web-console/docker/compose/backend/Dockerfile
+++ b/modules/web-console/docker/compose/backend/Dockerfile
@@ -15,15 +15,20 @@
 # limitations under the License.
 #
 
-FROM node:8
+FROM node:8-slim
 
-RUN mkdir -p /opt/web-console-backend
+ENV NPM_CONFIG_LOGLEVEL error
 
-WORKDIR /opt/web-console-backend
+WORKDIR /opt/web-console
 
-COPY build .
+# Install node modules for frontend and backend modules.
+COPY backend/package*.json backend/
+RUN (cd backend && npm install --no-optional --production)
 
-RUN npm install --only=production --no-optional
+# Copy source.
+COPY backend backend
+
+COPY web-agent/target/ignite-web-agent-*.zip backend/agent_dists
 
 EXPOSE 3000
 
diff --git a/modules/web-console/docker/compose/backend/build.sh b/modules/web-console/docker/compose/backend/build.sh
deleted file mode 100755
index a10b70641bd50..0000000000000
--- a/modules/web-console/docker/compose/backend/build.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-if [ -z "$IGNITE_HOME" ]; then
-    echo "Ignite source folder is not found or IGNITE_HOME environment variable is not valid."
-
-    exit 1
-fi
-
-WORK_DIR=`cd "$(dirname "$0")"; pwd`
-
-BUILD_DIR="$WORK_DIR/build"
-
-IGNITE_WEB_CONSOLE_BACKEND_DIR="$IGNITE_HOME/modules/web-console/backend"
-DOCKER_IMAGE_NAME="apacheignite/web-console-backend"
-
-echo "Receiving version..."
-VERSION=`cd $IGNITE_HOME && mvn org.apache.maven.plugins:maven-help-plugin:evaluate -Dexpression=project.version| grep -Ev '(^\[|Download\w+:)'`
-RELEASE_VERSION=${VERSION%-SNAPSHOT}
-
-echo "Building $DOCKER_IMAGE_NAME:$RELEASE_VERSION"
-echo "Step 1. Prepare build temp paths."
-cd $WORK_DIR
-rm -Rf $BUILD_DIR
-docker rmi -f $DOCKER_IMAGE_NAME:$RELEASE_VERSION
-
-echo "Step 2. Build ignite web agent."
-cd $IGNITE_HOME
-mvn versions:set -DnewVersion=$RELEASE_VERSION -DgenerateBackupPoms=false -Pweb-console -DartifactId='*'
-mvn clean package -pl :ignite-web-agent -am -P web-console -DskipTests=true
-mvn versions:set -DnewVersion=$VERSION -DgenerateBackupPoms=false -Pweb-console -DartifactId='*'
-
-echo "Step 3. Copy sources."
-cd $WORK_DIR
-cp -r $IGNITE_WEB_CONSOLE_BACKEND_DIR/. $BUILD_DIR
-cp $IGNITE_HOME/modules/web-console/web-agent/target/ignite-web-agent*.zip $BUILD_DIR/agent_dists/.
-
-echo "Step 4. Build docker image."
-docker build -f=./Dockerfile -t $DOCKER_IMAGE_NAME:$RELEASE_VERSION -t $DOCKER_IMAGE_NAME:latest .
-
-echo "Step 5. Cleanup."
-rm -Rf $BUILD_DIR
diff --git a/modules/web-console/docker/compose/docker-compose.yml b/modules/web-console/docker/compose/docker-compose.yml
index 89ffe98b399b8..109bfca85ab2d 100644
--- a/modules/web-console/docker/compose/docker-compose.yml
+++ b/modules/web-console/docker/compose/docker-compose.yml
@@ -15,40 +15,42 @@
 # limitations under the License.
 #
 
-mongodb:
-  image: mongo:latest
-  volumes:
-    # External volume for persisting data. (HOST_PATH:CONTAINER_PATH).
-    - ./data/mongo:/data/db
+version: 1
 
-backend:
-  image: apacheignite/web-console-backend
-  links:
-    # Link mongodb container as with mongodb hostname.
-    - mongodb:mongodb
-  # Restart on crash.
-  restart: always  
-  environment:
-    # Port for serving frontend API
-    - server_port=3000
-    # Cookie session secret
-    - server_sessionSecret=CHANGE ME
-    # URL for mongodb connection
-    - mongodb_url=mongodb://mongodb/console
-    # Mail connection settings. Leave empty if no needed. See also settings, https://github.com/nodemailer/nodemailer
-    - mail_service=
-    - mail_sign=
-    - mail_greeting=
-    - mail_from=
-    - mail_auth_user=
-    - mail_auth_pass=
+services:
+  mongodb:
+    image: mongo:3.4
+    container_name: 'mongodb'
+    volumes:
+      # External volume for persisting data. (HOST_PATH:CONTAINER_PATH).
+      - ./data/mongo:/data/db
 
-frontend:
-  image: apacheignite/web-console-frontend
-  links:
-    # Link backend container to proxy backend requests throught nginx container.
-    - backend:backend
+  backend:
+    image: apacheignite/web-console-backend
+    depends_on:
+      - mongodb
+    # Restart on crash.
+    restart: always
+    environment:
+      # Port for serving frontend API
+      - server_port=3000
+      # Cookie session secret
+      - server_sessionSecret=CHANGE ME
+      # URL for mongodb connection
+      - mongodb_url=mongodb://mongodb/console
+      # Mail connection settings. Leave empty if no needed. See also settings, https://github.com/nodemailer/nodemailer
+      - mail_service=
+      - mail_sign=
+      - mail_greeting=
+      - mail_from=
+      - mail_auth_user=
+      - mail_auth_pass=
 
-  ports:
-    # Proxy HTTP nginx port (HOST_PORT:DOCKER_PORT)
-    - 80:80
+  frontend:
+    image: apacheignite/web-console-frontend
+    depends_on:
+      - mongodb
+      - testenv
+    ports:
+      # Proxy HTTP nginx port (HOST_PORT:DOCKER_PORT)
+      - 80:80
diff --git a/modules/web-console/docker/compose/frontend/.dockerignore b/modules/web-console/docker/compose/frontend/.dockerignore
deleted file mode 100644
index caf0e8e6e2df1..0000000000000
--- a/modules/web-console/docker/compose/frontend/.dockerignore
+++ /dev/null
@@ -1,3 +0,0 @@
-src/build
-src/ignite_modules_temp
-src/node_modules
diff --git a/modules/web-console/docker/compose/frontend/Dockerfile b/modules/web-console/docker/compose/frontend/Dockerfile
index 1b578d17cc828..07cbc467e6f46 100644
--- a/modules/web-console/docker/compose/frontend/Dockerfile
+++ b/modules/web-console/docker/compose/frontend/Dockerfile
@@ -15,16 +15,29 @@
 # limitations under the License.
 #
 
-FROM nginx
+FROM node:8-slim as frontend-build
 
-RUN mkdir -p /data/www
+ENV NPM_CONFIG_LOGLEVEL error
+
+WORKDIR /opt/web-console
+
+# Install node modules for frontend.
+COPY frontend/package*.json frontend/
+RUN (cd frontend && npm install --no-optional)
+
+# Copy source.
+COPY frontend frontend
+
+RUN (cd frontend && npm run build)
+
+FROM nginx:1-alpine
 
 WORKDIR /data/www
 
-COPY ./build .
+COPY --from=frontend-build /opt/web-console/frontend/build .
 
-COPY nginx/nginx.conf /etc/nginx/nginx.conf
-COPY nginx/web-console.conf /etc/nginx/web-console.conf
+COPY docker/compose/frontend/nginx/nginx.conf /etc/nginx/nginx.conf
+COPY docker/compose/frontend/nginx/web-console.conf /etc/nginx/web-console.conf
 
 VOLUME /etc/nginx
 VOLUME /data/www
diff --git a/modules/web-console/docker/compose/frontend/DockerfileBuild b/modules/web-console/docker/compose/frontend/DockerfileBuild
deleted file mode 100644
index 94f52446a53c5..0000000000000
--- a/modules/web-console/docker/compose/frontend/DockerfileBuild
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-FROM node:8
-
-RUN mkdir -p /opt/web-console-frontend
-
-WORKDIR /opt/web-console-frontend
-
-COPY src .
-
-RUN npm install --no-optional --prod
-
-VOLUME /opt/web-console-frontend/build
-
-CMD ["npm", "run", "build"]
diff --git a/modules/web-console/docker/compose/frontend/build.sh b/modules/web-console/docker/compose/frontend/build.sh
deleted file mode 100755
index c807a86125c6b..0000000000000
--- a/modules/web-console/docker/compose/frontend/build.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-if [ -z "$IGNITE_HOME" ]; then
-    echo "Ignite source folder is not found or IGNITE_HOME environment variable is not valid."
-
-    exit 1
-fi
-
-WORK_DIR=`cd "$(dirname "$0")"; pwd`
-
-SOURCE_DIR=$WORK_DIR/src
-BUILD_DIR=$WORK_DIR/build
-
-DOCKER_BUILD_CONTAINER=web-console-frontend-builder
-DOCKER_BUILD_IMAGE_NAME=apacheignite/$DOCKER_BUILD_CONTAINER
-DOCKER_IMAGE_NAME=apacheignite/web-console-frontend
-
-echo "Receiving version..."
-VERSION=`cd $IGNITE_HOME && mvn org.apache.maven.plugins:maven-help-plugin:evaluate -Dexpression=project.version| grep -Ev '(^\[|Download\w+:)'`
-RELEASE_VERSION=${VERSION%-SNAPSHOT}
-
-echo "Building $DOCKER_IMAGE_NAME:$RELEASE_VERSION"
-echo "Step 1. Build frontend SPA"
-cd $WORK_DIR
-
-rm -Rf $SOURCE_DIR
-rm -Rf $BUILD_DIR
-mkdir -p $SOURCE_DIR
-mkdir -p $BUILD_DIR
-
-cp -r $IGNITE_HOME/modules/web-console/frontend/. $SOURCE_DIR
-
-docker build -f=./DockerfileBuild -t $DOCKER_BUILD_IMAGE_NAME:latest .
-docker run -it -v $BUILD_DIR:/opt/web-console-frontend/build --name $DOCKER_BUILD_CONTAINER $DOCKER_BUILD_IMAGE_NAME
-
-echo "Step 2. Build NGINX container with SPA and proxy configuration"
-docker build -f=./Dockerfile -t $DOCKER_IMAGE_NAME:$RELEASE_VERSION -t $DOCKER_IMAGE_NAME:latest .
-
-echo "Step 3. Cleanup"
-docker rm -f $DOCKER_BUILD_CONTAINER
-docker rmi -f $DOCKER_BUILD_IMAGE_NAME
-rm -r $SOURCE_DIR
-rm -r $BUILD_DIR
diff --git a/modules/web-console/docker/compose/frontend/nginx/web-console.conf b/modules/web-console/docker/compose/frontend/nginx/web-console.conf
index 3f5157dab7b08..4fbf204814a6e 100644
--- a/modules/web-console/docker/compose/frontend/nginx/web-console.conf
+++ b/modules/web-console/docker/compose/frontend/nginx/web-console.conf
@@ -24,17 +24,12 @@ server {
   server_name _;
 
   set $ignite_console_dir /data/www;
-  set $maintenance $ignite_console_dir/maintenance.file;
 
   root $ignite_console_dir;
 
   error_page 500 502 503 504 /50x.html;
 
   location / {
-    if (-f $maintenance) {
-      return 503;
-    }
-
     try_files $uri /index.html = 404;
   }
 
diff --git a/modules/web-console/docker/standalone/.dockerignore b/modules/web-console/docker/standalone/.dockerignore
deleted file mode 100644
index c59189ec92ce3..0000000000000
--- a/modules/web-console/docker/standalone/.dockerignore
+++ /dev/null
@@ -1,7 +0,0 @@
-build/frontend/build
-build/frontend/node_modules
-build/frontend/ignite_modules_temp
-build/frontend/test
-build/backend/config/*.json
-build/backend/node_modules
-build/backend/test
diff --git a/modules/web-console/docker/standalone/Dockerfile b/modules/web-console/docker/standalone/Dockerfile
index bfb79016e2897..9b007349b6a62 100644
--- a/modules/web-console/docker/standalone/Dockerfile
+++ b/modules/web-console/docker/standalone/Dockerfile
@@ -15,75 +15,62 @@
 # limitations under the License.
 #
 
-FROM ubuntu:14.04
-
-ENV NPM_CONFIG_LOGLEVEL info
-ENV NODE_VERSION 8.11.1
-
-# Before package list update.
-RUN set -ex  && \
-      for key in \
-        9554F04D7259F04124DE6B476D5A82AC7E37093B \
-        94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
-        FD3A5288F042B6850C66B31F09FE44734EB7990E \
-        71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
-        DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
-        B9AE9905FFD7803F25714661B63B535A4C206CA9 \
-        C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
-        56730D5401028683275BD23C23EFEFE93C4CFFFE \
-      ; do \
-        gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" || \
-        gpg --keyserver pgp.mit.edu --recv-keys "$key" || \
-        gpg --keyserver keyserver.pgp.com --recv-keys "$key" ; \
-    done
-
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 && \
-   echo "deb http://repo.mongodb.org/apt/ubuntu trusty/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
+FROM node:8-slim as frontend-build
+
+ENV NPM_CONFIG_LOGLEVEL error
+
+WORKDIR /opt/web-console
+
+# Install node modules for frontend.
+COPY frontend/package*.json frontend/
+RUN (cd frontend && npm install --no-optional)
+
+# Copy source.
+COPY frontend frontend
+
+RUN (cd frontend && npm run build)
+
+FROM node:8-slim
+
+ENV NPM_CONFIG_LOGLEVEL error
 
 # Update package list & install.
-RUN apt-get update && \
-    apt-get install -y nginx-light mongodb-org-server curl xz-utils git
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 0C49F3730359A14518585931BC711F9BA15703C6 \
+    && echo "deb http://repo.mongodb.org/apt/debian jessie/mongodb-org/3.4 main" | tee /etc/apt/sources.list.d/mongodb-org-3.4.list
 
-# Install Node JS.
-RUN curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz"  && \
-  curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" && \
-  gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc && \
-  grep " node-v$NODE_VERSION-linux-x64.tar.xz\$" SHASUMS256.txt | sha256sum -c - && \
-  tar -xJf "node-v$NODE_VERSION-linux-x64.tar.xz" -C /usr/local --strip-components=1 && \
-  rm "node-v$NODE_VERSION-linux-x64.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt
+# Update package list & install.
+RUN apt-get update \
+    && apt-get install -y nginx-light mongodb-org-server dos2unix \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 # Install global node packages.
 RUN npm install -g pm2
 
-# Install frontend & backend apps.
-RUN mkdir -p /opt/web-console
-
-# Copy source.
 WORKDIR /opt/web-console
-COPY build .
 
-# Install node modules.
-RUN cd /opt/web-console/frontend && npm install --no-optional --prod && npm run build
-RUN cd /opt/web-console/backend && npm install --only=production --no-optional
+COPY docker/standalone/docker-entrypoint.sh docker-entrypoint.sh
 
-# Returns to base path.
-WORKDIR /opt/web-console
+RUN chmod +x docker-entrypoint.sh \
+    && dos2unix docker-entrypoint.sh
 
 # Copy nginx config.
-COPY ./nginx/nginx.conf /etc/nginx/nginx.conf
-COPY ./nginx/web-console.conf /etc/nginx/web-console.conf
+COPY docker/standalone/nginx/* /etc/nginx/
 
-# Setup entrypoint.
-COPY ./entrypoint.sh .
-RUN chmod 755 /opt/web-console/entrypoint.sh
+# Install node modules for frontend and backend modules.
+COPY backend/package*.json backend/
+RUN (cd backend && npm install --no-optional --production)
 
-# Clean up.
-RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+# Copy source.
+COPY backend backend
+
+COPY web-agent/target/ignite-web-agent-*.zip backend/agent_dists
+
+COPY --from=frontend-build /opt/web-console/frontend/build static
 
 VOLUME ["/etc/nginx"]
-VOLUME ["/var/lib/mongodb"]
+VOLUME ["/data/db"]
 VOLUME ["/opt/web-console/serve/agent_dists"]
 
 EXPOSE 80
-
-ENTRYPOINT ["/opt/web-console/entrypoint.sh"]
+ENTRYPOINT ["/opt/web-console/docker-entrypoint.sh"]
diff --git a/modules/web-console/docker/standalone/build.sh b/modules/web-console/docker/standalone/build.sh
deleted file mode 100755
index c32dc403ba92d..0000000000000
--- a/modules/web-console/docker/standalone/build.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-if [ -z "$IGNITE_HOME" ]; then
-    echo "Ignite source folder is not found or IGNITE_HOME environment variable is not valid."
-
-    exit 1
-fi
-
-WORK_DIR=`cd "$(dirname "$0")"; pwd`
-
-BUILD_DIR="$WORK_DIR/build"
-
-IGNITE_WEB_CONSOLE_DIR="$IGNITE_HOME/modules/web-console"
-DOCKER_IMAGE_NAME="apacheignite/web-console-standalone"
-
-echo "Receiving version..."
-VERSION=`cd $IGNITE_HOME && mvn org.apache.maven.plugins:maven-help-plugin:evaluate -Dexpression=project.version| grep -Ev '(^\[|Download\w+:)'`
-RELEASE_VERSION=${VERSION%-SNAPSHOT}
-
-echo "Building $DOCKER_IMAGE_NAME:$RELEASE_VERSION"
-echo "Step 1. Prepare build temp paths."
-cd $WORK_DIR
-rm -Rf $BUILD_DIR
-docker rmi -f $DOCKER_IMAGE_NAME:$RELEASE_VERSION
-mkdir -p $BUILD_DIR/frontend $BUILD_DIR/backend
-
-echo "Step 2. Build ignite web agent."
-cd $IGNITE_HOME
-mvn versions:set -DnewVersion=$RELEASE_VERSION -DgenerateBackupPoms=false -Pweb-console -DartifactId='*'
-mvn clean package -pl :ignite-web-agent -am -P web-console -DskipTests=true
-mvn versions:set -DnewVersion=$VERSION -DgenerateBackupPoms=false -Pweb-console -DartifactId='*'
-
-echo "Step 3. Copy sources."
-cd $WORK_DIR
-cp -r $IGNITE_WEB_CONSOLE_DIR/frontend/. $BUILD_DIR/frontend
-cp -r $IGNITE_WEB_CONSOLE_DIR/backend/. $BUILD_DIR/backend
-cp $IGNITE_HOME/modules/web-console/web-agent/target/ignite-web-agent*.zip $BUILD_DIR/backend/agent_dists/.
-
-echo "Step 4. Build docker image."
-docker build -f=./Dockerfile -t $DOCKER_IMAGE_NAME:$RELEASE_VERSION -t $DOCKER_IMAGE_NAME:latest .
-
-echo "Step 5. Cleanup."
-rm -Rf $BUILD_DIR
diff --git a/modules/web-console/docker/standalone/entrypoint.sh b/modules/web-console/docker/standalone/docker-entrypoint.sh
old mode 100755
new mode 100644
similarity index 98%
rename from modules/web-console/docker/standalone/entrypoint.sh
rename to modules/web-console/docker/standalone/docker-entrypoint.sh
index 3882f06a2ed5e..6757de6b299ce
--- a/modules/web-console/docker/standalone/entrypoint.sh
+++ b/modules/web-console/docker/standalone/docker-entrypoint.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
diff --git a/modules/web-console/docker/standalone/nginx/web-console.conf b/modules/web-console/docker/standalone/nginx/web-console.conf
index 6e36eed30220d..caf171e5e22cf 100644
--- a/modules/web-console/docker/standalone/nginx/web-console.conf
+++ b/modules/web-console/docker/standalone/nginx/web-console.conf
@@ -23,7 +23,7 @@ server {
   listen 80;
   server_name _;
 
-  set $ignite_console_dir /opt/web-console/frontend/build;
+  set $ignite_console_dir /opt/web-console/static;
 
   root $ignite_console_dir;
 
diff --git a/modules/web-console/e2e/docker-compose.yml b/modules/web-console/e2e/docker-compose.yml
index 83054ccaf46ee..0159dfb327981 100644
--- a/modules/web-console/e2e/docker-compose.yml
+++ b/modules/web-console/e2e/docker-compose.yml
@@ -35,8 +35,8 @@ services:
     environment:
       - DB_URL=mongodb://mongodb:27017/console-e2e
       - APP_URL=http://testenv:9001/
-      - TEAMCITY=true
+      - REPORTER=teamcity
       - QUARANTINE_MODE=true
     depends_on:
       - mongodb
-      - testenv
\ No newline at end of file
+      - testenv
diff --git a/modules/web-console/e2e/testcafe/Dockerfile b/modules/web-console/e2e/testcafe/Dockerfile
index d890d10fec447..d207eec02e168 100644
--- a/modules/web-console/e2e/testcafe/Dockerfile
+++ b/modules/web-console/e2e/testcafe/Dockerfile
@@ -19,16 +19,14 @@ FROM testcafe/testcafe:latest
 
 USER 0
 
-RUN mkdir -p /opt/testcafe/tests
-
 WORKDIR /opt/testcafe/tests
 
-COPY . /opt/testcafe/tests
+COPY . .
 
-ENV NPM_CONFIG_LOGLEVEL warn
+ENV NPM_CONFIG_LOGLEVEL error
 
-RUN npm install --production && \
+RUN npm install --no-optional --production && \
  npm cache verify --force && \
  rm -rf /tmp/*
 
-ENTRYPOINT ["node", "./testcafe.js"]
\ No newline at end of file
+ENTRYPOINT ["node", "./index.js"]
\ No newline at end of file
diff --git a/modules/web-console/e2e/testcafe/components/FormField.js b/modules/web-console/e2e/testcafe/components/FormField.js
index 5bd70f56d5d99..71d951c74a7db 100644
--- a/modules/web-console/e2e/testcafe/components/FormField.js
+++ b/modules/web-console/e2e/testcafe/components/FormField.js
@@ -24,6 +24,9 @@ export class FormField {
     static CONTROL_SELECTOR = '[ng-model]';
     static ERRORS_SELECTOR = '.ignite-form-field__errors';
 
+    /** @type {ReturnType<Selector>} */
+    _selector;
+
     constructor({id = '', label = '', model = ''} = {}) {
         if (!id && !label && !model) throw new Error('ID, label or model are required');
         if (id)
diff --git a/modules/web-console/e2e/testcafe/components/Table.js b/modules/web-console/e2e/testcafe/components/Table.js
index e690599a3711f..415f98db8818d 100644
--- a/modules/web-console/e2e/testcafe/components/Table.js
+++ b/modules/web-console/e2e/testcafe/components/Table.js
@@ -15,31 +15,34 @@
  * limitations under the License.
  */
 
-import {Selector, t} from 'testcafe'
+import {Selector, t} from 'testcafe';
 
 const findCell = Selector((table, rowIndex, columnLabel) => {
     table = table();
 
     const columnIndex = [].constructor.from(
         table.querySelectorAll('.ui-grid-header-cell:not(.ui-grid-header-span)'),
-        e => e.textContent
-    ).findIndex(t => t.includes(columnLabel));
+        (e) => e.textContent
+    ).findIndex((t) => t.includes(columnLabel));
 
-    const row = table.querySelector(`.ui-grid-render-container:not(.left) .ui-grid-viewport .ui-grid-row:nth-of-type(${rowIndex+1})`);
+    const row = table.querySelector(`.ui-grid-render-container:not(.left) .ui-grid-viewport .ui-grid-row:nth-of-type(${rowIndex + 1})`);
     const cell = row.querySelector(`.ui-grid-cell:nth-of-type(${columnIndex})`);
+
     return cell;
 });
 
 export class Table {
+    /** @param {ReturnType<Selector>} selector */
     constructor(selector) {
         this._selector = selector;
         this.title = this._selector.find('.panel-title');
         this.actionsButton = this._selector.find('.btn-ignite').withText('Actions');
-        this.allItemsCheckbox = this._selector.find('[role="checkbox button"]')
+        this.allItemsCheckbox = this._selector.find('[role="checkbox button"]');
     }
 
+    /** @param {string} label */
     async performAction(label) {
-        await t.hover(this.actionsButton).click(Selector('.dropdown-menu a').withText(label))
+        await t.hover(this.actionsButton).click(Selector('.dropdown-menu a').withText(label));
     }
 
     /**
@@ -47,10 +50,14 @@ export class Table {
      * @param {number} index Index of row, starting with 1
      */
     async toggleRowSelection(index) {
-        await t.click(this._selector.find(`.ui-grid-pinned-container .ui-grid-row:nth-of-type(${index}) .ui-grid-selection-row-header-buttons`))
+        await t.click(this._selector.find(`.ui-grid-pinned-container .ui-grid-row:nth-of-type(${index}) .ui-grid-selection-row-header-buttons`));
     }
 
+    /**
+     * @param {number} rowIndex
+     * @param {string} columnLabel
+     */
     findCell(rowIndex, columnLabel) {
-        return Selector(findCell(this._selector, rowIndex, columnLabel))
+        return Selector(findCell(this._selector, rowIndex, columnLabel));
     }
 }
diff --git a/modules/web-console/e2e/testcafe/components/pageConfiguration.js b/modules/web-console/e2e/testcafe/components/pageConfiguration.js
index c3642085d567f..033412b819014 100644
--- a/modules/web-console/e2e/testcafe/components/pageConfiguration.js
+++ b/modules/web-console/e2e/testcafe/components/pageConfiguration.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import {Selector} from 'testcafe'
+import {Selector} from 'testcafe';
 
 export const basicNavButton = Selector('.tabs.tabs--blue a[ui-sref="base.configuration.edit.basic"]');
 export const advancedNavButton = Selector('.tabs.tabs--blue a[ui-sref="base.configuration.edit.advanced.cluster"]');
diff --git a/modules/web-console/e2e/testcafe/envtools.js b/modules/web-console/e2e/testcafe/environment/envtools.js
similarity index 95%
rename from modules/web-console/e2e/testcafe/envtools.js
rename to modules/web-console/e2e/testcafe/environment/envtools.js
index e6b6d6cfade71..c86a34a254e71 100644
--- a/modules/web-console/e2e/testcafe/envtools.js
+++ b/modules/web-console/e2e/testcafe/environment/envtools.js
@@ -144,7 +144,7 @@ const exec = (command, onResolveString, cwd, env) => {
     });
 };
 
-const startEnv = () => {
+const startEnv = (webConsoleRootDirectoryPath = '../../') => {
     return new Promise(async(resolve) => {
         const command = `${process.platform === 'win32' ? 'npm.cmd' : 'npm'} start`;
 
@@ -153,8 +153,8 @@ const startEnv = () => {
         if (process.env.APP_URL)
             port = parseInt(url.parse(process.env.APP_URL).port, 10) || 80;
 
-        const backendInstanceLaunch = exec(command, 'Start listening', '../../backend', {server_port: 3001, mongodb_url: mongoUrl});
-        const frontendInstanceLaunch = exec(command, 'Compiled successfully', '../../frontend', {BACKEND_PORT: 3001, PORT: port});
+        const backendInstanceLaunch = exec(command, 'Start listening', `${webConsoleRootDirectoryPath}backend`, {server_port: 3001, mongodb_url: mongoUrl}); // Todo: refactor cwd for backend when it's linked
+        const frontendInstanceLaunch = exec(command, 'Compiled successfully', `${webConsoleRootDirectoryPath}frontend`, {BACKEND_PORT: 3001, PORT: port});
 
         console.log('Building backend in progress...');
         await backendInstanceLaunch;
@@ -185,6 +185,10 @@ if (stop) {
 }
 
 
+/**
+ * @param {string} targetUrl
+ * @returns {string}
+ */
 const resolveUrl = (targetUrl) => {
     return url.resolve(process.env.APP_URL || 'http://localhost:9001', targetUrl);
 };
diff --git a/modules/web-console/e2e/testcafe/environment/launch-env.js b/modules/web-console/e2e/testcafe/environment/launch-env.js
new file mode 100644
index 0000000000000..73b4baee86c59
--- /dev/null
+++ b/modules/web-console/e2e/testcafe/environment/launch-env.js
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { startEnv, dropTestDB } = require('./envtools');
+
+startEnv();
+
+process.on('SIGINT', async() => {
+    await dropTestDB();
+
+    process.exit(0);
+});
diff --git a/modules/web-console/e2e/testcafe/fixtures/admin-panel.js b/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
index 9eea360f31731..0851d3f8c4615 100644
--- a/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
+++ b/modules/web-console/e2e/testcafe/fixtures/admin-panel.js
@@ -17,7 +17,7 @@
 
 import { Selector } from 'testcafe';
 import { AngularJSSelector } from 'testcafe-angular-selectors';
-import { dropTestDB, insertTestUser, resolveUrl } from '../envtools';
+import { dropTestDB, insertTestUser, resolveUrl } from '../environment/envtools';
 import { createRegularUser } from '../roles';
 
 const regularUser = createRegularUser();
diff --git a/modules/web-console/e2e/testcafe/fixtures/auth/forgot-password.js b/modules/web-console/e2e/testcafe/fixtures/auth/forgot-password.js
index 6b151edae2487..7a9f79681e1eb 100644
--- a/modules/web-console/e2e/testcafe/fixtures/auth/forgot-password.js
+++ b/modules/web-console/e2e/testcafe/fixtures/auth/forgot-password.js
@@ -15,14 +15,13 @@
  * limitations under the License.
  */
 
-import { dropTestDB, resolveUrl, insertTestUser } from '../../envtools';
-import {PageSignIn} from '../../page-models/PageSignIn';
+import { dropTestDB, resolveUrl, insertTestUser } from '../../environment/envtools';
+import {PageSignIn} from '../../page-models/pageSignin';
 import {errorNotification} from '../../components/notifications';
-
-const page = new PageSignIn();
+import {pageForgotPassword as page} from '../../page-models/pageForgotPassword';
 
 fixture('Password reset')
-    .page(resolveUrl('/signin'))
+    .page(resolveUrl('/forgot-password'))
     .before(async() => {
         await dropTestDB();
         await insertTestUser();
@@ -33,26 +32,22 @@ fixture('Password reset')
 
 test('Incorrect email', async(t) => {
     await t
-        .click(page.showForgotPasswordButton)
-        .typeText(page.forgotPassword.email.control, 'aa')
-        .expect(page.forgotPassword.email.getError('email').exists).ok('Marks field as invalid')
-        .expect(page.remindPasswordButton.getAttribute('disabled')).ok('Disables submit button');
+        .typeText(page.email.control, 'aa')
+        .expect(page.email.getError('email').exists).ok('Marks field as invalid');
 });
 
 test('Unknown email', async(t) => {
     await t
-        .click(page.showForgotPasswordButton)
-        .typeText(page.forgotPassword.email.control, 'nonexisting@mail.com', {replace: true})
+        .typeText(page.email.control, 'nonexisting@mail.com', {replace: true})
         .click(page.remindPasswordButton)
         .expect(errorNotification.withText('Account with that email address does not exists!').exists).ok('Shows global error notification')
-        .expect(page.forgotPassword.email.getError('server').exists).ok('Marks input as server-invalid');
+        .expect(page.email.getError('server').exists).ok('Marks input as server-invalid');
 });
 
 // TODO: IGNITE-8028 Implement this test as unit test.
 test.skip('Successful reset', async(t) => {
     await t
-        .click(page.showForgotPasswordButton)
-        .typeText(page.forgotPassword.email.control, 'a@a', {replace: true})
+        .typeText(page.email.control, 'a@a', {replace: true})
         .click(page.remindPasswordButton)
-        .expect(page.forgotPassword.email.getError('server').exists).notOk('No errors happen');
+        .expect(page.email.getError('server').exists).notOk('No errors happen');
 });
diff --git a/modules/web-console/e2e/testcafe/fixtures/auth/logout.js b/modules/web-console/e2e/testcafe/fixtures/auth/logout.js
index 196c03f017ac0..de495effe5df9 100644
--- a/modules/web-console/e2e/testcafe/fixtures/auth/logout.js
+++ b/modules/web-console/e2e/testcafe/fixtures/auth/logout.js
@@ -15,13 +15,12 @@
  * limitations under the License.
  */
 
-import {dropTestDB, resolveUrl, insertTestUser} from 'envtools';
+import {dropTestDB, resolveUrl, insertTestUser} from '../../environment/envtools';
 import {createRegularUser} from '../../roles';
 import {userMenu} from '../../components/userMenu';
-import {PageSignIn} from '../../page-models/PageSignIn';
+import {pageSignin} from '../../page-models/pageSignin';
 
 const user = createRegularUser();
-const pageSignIn = new PageSignIn();
 
 fixture('Logout')
     .before(async() => {
@@ -35,5 +34,5 @@ fixture('Logout')
 test('Successful logout', async(t) => {
     await t.useRole(user).navigateTo(resolveUrl('/settings/profile'));
     await userMenu.clickOption('Log out');
-    await t.expect(pageSignIn._selector.exists).ok('Goes to sign in page after logout');
+    await t.expect(pageSignin.selector.exists).ok('Goes to sign in page after logout');
 });
diff --git a/modules/web-console/e2e/testcafe/fixtures/auth/signup-validation-local.js b/modules/web-console/e2e/testcafe/fixtures/auth/signup-validation-local.js
new file mode 100644
index 0000000000000..d26ce3335a26a
--- /dev/null
+++ b/modules/web-console/e2e/testcafe/fixtures/auth/signup-validation-local.js
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {resolveUrl} from '../../environment/envtools';
+import {pageSignup as page} from '../../page-models/pageSignup';
+
+fixture('Signup validation local').page(resolveUrl('/signup'));
+
+test('Most fields have validation', async(t) => {
+    await page.fillSignupForm({
+        email: 'foobar',
+        password: '1',
+        passwordConfirm: '2',
+        firstName: '  ',
+        lastName: '   ',
+        company: '   ',
+        country: 'Brazil'
+    });
+
+    await t
+        .expect(page.email.getError('email').exists).ok()
+        .expect(page.passwordConfirm.getError('mismatch').exists).ok()
+        .expect(page.firstName.getError('required').exists).ok()
+        .expect(page.lastName.getError('required').exists).ok()
+        .expect(page.company.getError('required').exists).ok();
+});
+
+test('Errors on submit', async(t) => {
+    await t
+        .typeText(page.email.control, 'email@example.com')
+        .click(page.signupButton)
+        .expect(page.password.control.focused).ok()
+        .expect(page.password.getError('required').exists).ok()
+        .typeText(page.password.control, 'Foo')
+        .click(page.signupButton)
+        .expect(page.passwordConfirm.control.focused).ok()
+        .expect(page.passwordConfirm.getError('required').exists).ok();
+});
diff --git a/modules/web-console/e2e/testcafe/fixtures/auth/signup.js b/modules/web-console/e2e/testcafe/fixtures/auth/signup.js
index 065bff96b2d84..44b199f91556e 100644
--- a/modules/web-console/e2e/testcafe/fixtures/auth/signup.js
+++ b/modules/web-console/e2e/testcafe/fixtures/auth/signup.js
@@ -15,15 +15,13 @@
  * limitations under the License.
  */
 
-import {dropTestDB, resolveUrl, insertTestUser} from '../../envtools';
-import {PageSignIn} from '../../page-models/PageSignIn';
+import {dropTestDB, resolveUrl, insertTestUser} from '../../environment/envtools';
+import {pageSignup as page} from '../../page-models/pageSignup';
 import {errorNotification} from '../../components/notifications';
 import {userMenu} from '../../components/userMenu';
 
-const page = new PageSignIn();
-
 fixture('Signup')
-    .page(resolveUrl('/signin'))
+    .page(resolveUrl('/signup'))
     .before(async() => {
         await dropTestDB();
         await insertTestUser();
@@ -33,8 +31,6 @@ fixture('Signup')
     });
 
 test('Local validation', async(t) => {
-    const isButtonDisabled = page.signupButton.hasAttribute('disabled');
-    await t.expect(isButtonDisabled).ok('Button disabled by default');
     await page.fillSignupForm({
         email: 'foobar',
         password: '1',
@@ -45,20 +41,9 @@ test('Local validation', async(t) => {
         country: 'Brazil'
     });
     await t
-        .expect(page.signup.email.getError('email').exists).ok()
-        .expect(page.signup.passwordConfirm.getError('mismatch').exists).ok()
-        .expect(page.signup.firstName.getError('required').exists).ok()
-        .expect(isButtonDisabled).ok('Button disabled with invalid fields');
-    await page.fillSignupForm({
-        email: 'foobar@bar.baz',
-        password: '1',
-        passwordConfirm: '1',
-        firstName: 'John',
-        lastName: 'Doe',
-        company: 'FooBar',
-        country: 'Brazil'
-    });
-    await t.expect(isButtonDisabled).notOk('Button enabled with valid fields');
+        .expect(page.email.getError('email').exists).ok()
+        .expect(page.passwordConfirm.getError('mismatch').exists).ok()
+        .expect(page.firstName.getError('required').exists).ok();
 });
 test('Server validation', async(t) => {
     await page.fillSignupForm({
@@ -73,7 +58,7 @@ test('Server validation', async(t) => {
     await t
         .click(page.signupButton)
         .expect(errorNotification.withText('A user with the given username is already registered').exists).ok('Shows global error')
-        .expect(page.signup.email.getError('server').exists).ok('Marks email input as server-invalid');
+        .expect(page.email.getError('server').exists).ok('Marks email input as server-invalid');
 });
 test('Successful signup', async(t) => {
     await page.fillSignupForm({
diff --git a/modules/web-console/e2e/testcafe/fixtures/configuration/basic.js b/modules/web-console/e2e/testcafe/fixtures/configuration/basic.js
index 090fd0a6f7fe6..b431e7bfaf01d 100644
--- a/modules/web-console/e2e/testcafe/fixtures/configuration/basic.js
+++ b/modules/web-console/e2e/testcafe/fixtures/configuration/basic.js
@@ -15,8 +15,7 @@
  * limitations under the License.
  */
 
-import {Selector, Role} from 'testcafe';
-import {dropTestDB, insertTestUser, resolveUrl} from '../../envtools';
+import {dropTestDB, insertTestUser, resolveUrl} from '../../environment/envtools';
 import {createRegularUser} from '../../roles';
 import {PageConfigurationBasic} from '../../page-models/PageConfigurationBasic';
 import {successNotification} from '../../components/notifications';
@@ -24,7 +23,7 @@ import {successNotification} from '../../components/notifications';
 const regularUser = createRegularUser();
 
 fixture('Basic configuration')
-    .before(async(t) => {
+    .before(async() => {
         await dropTestDB();
         await insertTestUser();
     })
@@ -62,7 +61,6 @@ test('Basic editing', async(t) => {
 
     await t
         .expect(page.buttonPreviewProject.visible).notOk('Preview project button is hidden for new cluster configs')
-        .expect(page.buttonDownloadProject.visible).notOk('Download project button is hidden for new cluster configs')
         .typeText(page.clusterNameInput.control, clusterName, {replace: true});
     await page.cachesList.addItem();
     await page.cachesList.addItem();
@@ -79,8 +77,7 @@ test('Basic editing', async(t) => {
     await t.expect(cache1.getItemViewColumn(1).textContent).eql(localMode, 'Can edit cache mode');
     await t.expect(cache1.getItemViewColumn(2).textContent).eql(atomic, 'Can edit cache atomicity');
 
-    // TODO IGNITE-8094: restore to save method call.
-    await page.saveWithoutDownload();
+    await page.save();
     await t
         .expect(successNotification.visible).ok('Shows success notifications')
         .expect(successNotification.textContent).contains(`Cluster "${clusterName}" saved.`, 'Success notification has correct text', {timeout: 500});
diff --git a/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js b/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js
new file mode 100644
index 0000000000000..a862172a8e282
--- /dev/null
+++ b/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {dropTestDB, insertTestUser, resolveUrl} from '../../environment/envtools';
+import {createRegularUser} from '../../roles';
+import {PageConfigurationOverview} from '../../page-models/PageConfigurationOverview';
+import {PageConfigurationAdvancedCluster} from '../../page-models/PageConfigurationAdvancedCluster';
+import {advancedNavButton} from '../../components/pageConfiguration';
+import {pageAdvancedConfiguration} from '../../components/pageAdvancedConfiguration';
+import {confirmation} from '../../components/confirmation';
+import {scrollIntoView} from '../../helpers';
+
+const regularUser = createRegularUser();
+
+fixture('Cluster configuration form change detection')
+    .before(async() => {
+        await dropTestDB();
+        await insertTestUser();
+    })
+    .beforeEach(async(t) => {
+        await t.useRole(regularUser);
+    })
+    .after(dropTestDB);
+
+test('New cluster change detection', async(t) => {
+    const overview = new PageConfigurationOverview();
+    const advanced = new PageConfigurationAdvancedCluster();
+
+    await t
+        .navigateTo(resolveUrl(`/configuration/overview`))
+        .click(overview.createClusterConfigButton)
+        .click(advancedNavButton)
+        .click(advanced.sections.connectorConfiguration.panel.heading);
+
+    await scrollIntoView.with({dependencies: {el: advanced.sections.connectorConfiguration.panel.heading}})();
+
+    await t
+        .click(advanced.sections.connectorConfiguration.inputs.enable.control)
+        .click(advanced.saveButton)
+        .click(pageAdvancedConfiguration.cachesNavButton)
+        .expect(confirmation.body.exists).notOk(`Doesn't show changes confiramtion after saving new cluster`);
+});
diff --git a/modules/web-console/e2e/testcafe/fixtures/configuration/newClusterWithCache.js b/modules/web-console/e2e/testcafe/fixtures/configuration/newClusterWithCache.js
new file mode 100644
index 0000000000000..d6976f5ba0902
--- /dev/null
+++ b/modules/web-console/e2e/testcafe/fixtures/configuration/newClusterWithCache.js
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {dropTestDB, insertTestUser, resolveUrl} from '../../environment/envtools';
+import {createRegularUser} from '../../roles';
+import {PageConfigurationOverview} from '../../page-models/PageConfigurationOverview';
+import {PageConfigurationAdvancedCluster} from '../../page-models/PageConfigurationAdvancedCluster';
+import {configureNavButton} from '../../components/topNavigation';
+
+const regularUser = createRegularUser();
+
+fixture('New cluster with cache')
+    .before(async() => {
+        await dropTestDB();
+        await insertTestUser();
+    })
+    .beforeEach(async(t) => {
+        await t.useRole(regularUser);
+    })
+    .after(dropTestDB);
+
+test(`New cluster name doesn't disappear`, async(t) => {
+    const overview = new PageConfigurationOverview();
+    const advanced = new PageConfigurationAdvancedCluster();
+
+    await t
+        .navigateTo(resolveUrl(`/configuration/new/advanced/caches/new`))
+        .click(advanced.saveButton)
+        .click(configureNavButton)
+        .expect(overview.clustersTable.findCell(0, 'Name').textContent).contains('Cluster');
+});
diff --git a/modules/web-console/e2e/testcafe/fixtures/configuration/overview.js b/modules/web-console/e2e/testcafe/fixtures/configuration/overview.js
index f0495bd8ce3e9..8d7093a8dee19 100644
--- a/modules/web-console/e2e/testcafe/fixtures/configuration/overview.js
+++ b/modules/web-console/e2e/testcafe/fixtures/configuration/overview.js
@@ -17,7 +17,7 @@
 
 import {Selector} from 'testcafe';
 import {getLocationPathname} from '../../helpers';
-import {dropTestDB, insertTestUser, resolveUrl} from '../../envtools';
+import {dropTestDB, insertTestUser, resolveUrl} from '../../environment/envtools';
 import {createRegularUser} from '../../roles';
 import {PageConfigurationOverview} from '../../page-models/PageConfigurationOverview';
 import {PageConfigurationBasic} from '../../page-models/PageConfigurationBasic';
diff --git a/modules/web-console/e2e/testcafe/fixtures/menu-smoke.js b/modules/web-console/e2e/testcafe/fixtures/menu-smoke.js
index fdd0edb49723a..9b3853a39b93a 100644
--- a/modules/web-console/e2e/testcafe/fixtures/menu-smoke.js
+++ b/modules/web-console/e2e/testcafe/fixtures/menu-smoke.js
@@ -16,7 +16,7 @@
  */
 
 import { Selector } from 'testcafe';
-import { dropTestDB, insertTestUser, resolveUrl } from '../envtools';
+import { dropTestDB, insertTestUser, resolveUrl } from '../environment/envtools';
 import { createRegularUser } from '../roles';
 import { queriesNavButton, configureNavButton } from '../components/topNavigation';
 
@@ -35,7 +35,7 @@ fixture('Checking Ingite main menu')
         await dropTestDB();
     });
 
-test('Ingite main menu smoke test', async(t) => {
+test('Ignite main menu smoke test', async(t) => {
     await t
         .click(configureNavButton)
         .expect(Selector('title').innerText)
diff --git a/modules/web-console/e2e/testcafe/fixtures/queries/notebooks-list.js b/modules/web-console/e2e/testcafe/fixtures/queries/notebooks-list.js
index 0d6581a9382bf..6037b28b2a2d5 100644
--- a/modules/web-console/e2e/testcafe/fixtures/queries/notebooks-list.js
+++ b/modules/web-console/e2e/testcafe/fixtures/queries/notebooks-list.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 import { Selector } from 'testcafe';
-import { dropTestDB, insertTestUser, resolveUrl } from '../../envtools';
+import { dropTestDB, insertTestUser, resolveUrl } from '../../environment/envtools';
 import { createRegularUser } from '../../roles';
 import { PageQueriesNotebooksList } from '../../page-models/PageQueries';
 
diff --git a/modules/web-console/e2e/testcafe/fixtures/user-profile/credentials.js b/modules/web-console/e2e/testcafe/fixtures/user-profile/credentials.js
index 42c76cff0ad74..ea18aafca9114 100644
--- a/modules/web-console/e2e/testcafe/fixtures/user-profile/credentials.js
+++ b/modules/web-console/e2e/testcafe/fixtures/user-profile/credentials.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import { dropTestDB, insertTestUser, resolveUrl } from '../../envtools';
+import { dropTestDB, insertTestUser, resolveUrl } from '../../environment/envtools';
 import { createRegularUser } from '../../roles';
 import {pageProfile} from '../../page-models/pageProfile';
 import {confirmation} from '../../components/confirmation';
@@ -39,13 +39,16 @@ fixture('Checking user credentials change')
 test('Testing secure token change', async(t) => {
     await t.click(pageProfile.securityToken.panel.heading);
 
-    const currentToken = await pageProfile.securityToken.value.innerText;
+    const currentToken = await pageProfile.securityToken.value.control.value;
 
     await t
         .click(pageProfile.securityToken.generateTokenButton)
-        .expect(confirmation.body.innerText).contains('Are you sure you want to change security token?')
+        .expect(confirmation.body.innerText).contains(
+`Are you sure you want to change security token?
+If you change the token you will need to restart the agent.`
+        )
         .click(confirmation.confirmButton)
-        .expect(pageProfile.securityToken.value.innerText).notEql(currentToken);
+        .expect(pageProfile.securityToken.value.control.value).notEql(currentToken);
 });
 
 test('Testing password change', async(t) => {
diff --git a/modules/web-console/e2e/testcafe/fixtures/user-profile/profile.js b/modules/web-console/e2e/testcafe/fixtures/user-profile/profile.js
index 1e9f0c3b7ff97..d6030d97136de 100644
--- a/modules/web-console/e2e/testcafe/fixtures/user-profile/profile.js
+++ b/modules/web-console/e2e/testcafe/fixtures/user-profile/profile.js
@@ -16,7 +16,7 @@
  */
 
 import { Selector } from 'testcafe';
-import { dropTestDB, insertTestUser, resolveUrl } from '../../envtools';
+import { dropTestDB, insertTestUser, resolveUrl } from '../../environment/envtools';
 import { createRegularUser } from '../../roles';
 import {pageProfile} from '../../page-models/pageProfile';
 
diff --git a/modules/web-console/e2e/testcafe/helpers.js b/modules/web-console/e2e/testcafe/helpers.js
index fc68f936223e5..908bd6e988196 100644
--- a/modules/web-console/e2e/testcafe/helpers.js
+++ b/modules/web-console/e2e/testcafe/helpers.js
@@ -34,4 +34,6 @@ const getLocationPathname = ClientFunction(() => Promise.resolve(location.pathna
  */
 const isVisible = (node) => !!node.getBoundingClientRect().width;
 
-module.exports = { mouseenterTrigger, getLocationPathname, isVisible };
+const scrollIntoView = ClientFunction(() => el().scrollIntoView());
+
+module.exports = { mouseenterTrigger, getLocationPathname, isVisible, scrollIntoView };
diff --git a/modules/web-console/e2e/testcafe/index.js b/modules/web-console/e2e/testcafe/index.js
new file mode 100644
index 0000000000000..837f2df0ece6b
--- /dev/null
+++ b/modules/web-console/e2e/testcafe/index.js
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const glob = require('glob');
+const argv = require('minimist')(process.argv.slice(2));
+const { startTestcafe } = require('./testcafe-runner');
+
+const enableEnvironment = argv.env;
+
+// See all supported browsers at http://devexpress.github.io/testcafe/documentation/using-testcafe/common-concepts/browsers/browser-support.html#locally-installed-browsers
+const BROWSERS = ['chromium:headless --no-sandbox']; // For example: ['chrome', 'firefox'];
+
+const FIXTURES_PATHS = glob.sync('./fixtures/**/*.js');
+
+const testcafeRunnerConfig = {
+    browsers: BROWSERS,
+    enableEnvironment: enableEnvironment || false,
+    reporter: process.env.REPORTER || 'spec',
+    fixturesPathsArray: FIXTURES_PATHS
+};
+
+startTestcafe(testcafeRunnerConfig).then(() => {
+    process.exit(0);
+});
diff --git a/modules/web-console/e2e/testcafe/package.json b/modules/web-console/e2e/testcafe/package.json
index 2501102ed4425..2da992cb2d18b 100644
--- a/modules/web-console/e2e/testcafe/package.json
+++ b/modules/web-console/e2e/testcafe/package.json
@@ -1,27 +1,21 @@
 {
   "name": "ignite-web-console-e2e-tests",
-  "version": "1.0.0",
+  "version": "2.5.0",
   "description": "E2E tests for Apache Ignite Web console",
   "private": true,
+  "main": "index.js",
   "scripts": {
-    "env": "node envtools.js start",
-    "test": "node testcafe.js --env=true"
+    "env": "node environment/launch-env.js",
+    "test": "node index.js --env=true"
   },
-  "author": "",
-  "contributors": [
-    {
-      "name": "",
-      "email": ""
-    }
-  ],
   "license": "Apache-2.0",
   "keywords": [
     "Apache Ignite Web console"
   ],
   "homepage": "https://ignite.apache.org/",
   "engines": {
-    "npm": "3.x.x",
-    "node": "4.x.x"
+    "npm": ">=5.x.x",
+    "node": ">=8.x.x <10.x.x"
   },
   "os": [
     "darwin",
@@ -32,14 +26,14 @@
     "app-module-path": "2.2.0",
     "cross-env": "5.1.1",
     "glob": "7.1.2",
-    "lodash": "4.17.5",
+    "lodash": "4.17.10",
     "minimist": "1.2.0",
     "mongodb": "2.2.33",
     "node-cmd": "3.0.0",
     "objectid": "3.2.1",
     "path": "0.12.7",
     "sinon": "2.3.8",
-    "testcafe": "^0.19.0",
+    "testcafe": "0.20.5",
     "testcafe-angular-selectors": "0.3.0",
     "testcafe-reporter-teamcity": "1.0.9",
     "type-detect": "4.0.3",
diff --git a/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js b/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js
index 0f62707a70ce8..1f1f5590f34b4 100644
--- a/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js
+++ b/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js
@@ -15,14 +15,14 @@
  * limitations under the License.
  */
 
-import {Selector, t} from 'testcafe'
+import {Selector, t} from 'testcafe';
 
 export class PageConfigurationAdvancedCluster {
     constructor() {
-        this._selector = Selector('page-configure-advanced-cluster')
-        this.saveButton = Selector('.pc-form-actions-panel .btn-ignite').withText('Save')
+        this.saveButton = Selector('.pc-form-actions-panel .btn-ignite').withText('Save');
     }
+
     async save() {
-        await t.click(this.saveButton)
+        await t.click(this.saveButton);
     }
 }
diff --git a/modules/web-console/e2e/testcafe/page-models/PageConfigurationBasic.js b/modules/web-console/e2e/testcafe/page-models/PageConfigurationBasic.js
index 38610bc42b953..bef6606d71512 100644
--- a/modules/web-console/e2e/testcafe/page-models/PageConfigurationBasic.js
+++ b/modules/web-console/e2e/testcafe/page-models/PageConfigurationBasic.js
@@ -15,13 +15,13 @@
  * limitations under the License.
  */
 
-import {Selector, t} from 'testcafe'
-import {FormField} from '../components/FormField'
-import {ListEditable} from '../components/ListEditable'
+import {Selector, t} from 'testcafe';
+import {FormField} from '../components/FormField';
+import {ListEditable} from '../components/ListEditable';
 
 class VersionPicker {
     constructor() {
-        this._selector = Selector('version-picker')
+        this._selector = Selector('version-picker');
     }
     /**
      * @param {string} label Version label
@@ -29,24 +29,23 @@ class VersionPicker {
     pickVersion(label) {
         return t
             .hover(this._selector)
-            .click(this._selector.find('[role="menuitem"]').withText(label))
+            .click(this._selector.find('[role="menuitem"]').withText(label));
     }
 }
 
 export class PageConfigurationBasic {
-    static SAVE_CHANGES_AND_DOWNLOAD_LABEL = 'Save changes and download project';
-    static SAVE_CHANGES_LABEL = 'Save changes';
+    static SAVE_CHANGES_AND_DOWNLOAD_LABEL = 'Save and Download';
+    static SAVE_CHANGES_LABEL = 'Save';
 
     constructor() {
         this._selector = Selector('page-configure-basic');
-        this.versionPicker = new VersionPicker;
+        this.versionPicker = new VersionPicker();
         this.totalOffheapSizeInput = Selector('pc-form-field-size#memory');
         this.mainFormAction = Selector('.pc-form-actions-panel .btn-ignite-group .btn-ignite:nth-of-type(1)');
         this.contextFormActionsButton = Selector('.pc-form-actions-panel .btn-ignite-group .btn-ignite:nth-of-type(2)');
         this.contextSaveButton = Selector('a[role=menuitem]').withText(new RegExp(`^${PageConfigurationBasic.SAVE_CHANGES_LABEL}$`));
         this.contextSaveAndDownloadButton = Selector('a[role=menuitem]').withText(PageConfigurationBasic.SAVE_CHANGES_AND_DOWNLOAD_LABEL);
         this.buttonPreviewProject = Selector('button-preview-project');
-        this.buttonDownloadProject = Selector('button-download-project');
         this.clusterNameInput = new FormField({id: 'clusterNameInput'});
         this.clusterDiscoveryInput = new FormField({id: 'discoveryInput'});
         this.cachesList = new ListEditable(Selector('.pcb-caches-list'), {
@@ -55,14 +54,14 @@ export class PageConfigurationBasic {
             atomicityMode: {id: 'atomicityModeInput'},
             backups: {id: 'backupsInput'}
         });
-        this.pageHeader = Selector('.pc-page-header')
+        this.pageHeader = Selector('.header-with-selector h1');
     }
 
     async save() {
-        await t.click(this.mainFormAction)
+        await t.click(this.mainFormAction);
     }
 
     async saveWithoutDownload() {
-        return await t.click(this.contextFormActionsButton).click(this.contextSaveButton)
+        return await t.click(this.contextFormActionsButton).click(this.contextSaveButton);
     }
 }
diff --git a/modules/web-console/e2e/testcafe/page-models/PageConfigurationOverview.js b/modules/web-console/e2e/testcafe/page-models/PageConfigurationOverview.js
index 34a648642e7ab..652b50fd4ad5e 100644
--- a/modules/web-console/e2e/testcafe/page-models/PageConfigurationOverview.js
+++ b/modules/web-console/e2e/testcafe/page-models/PageConfigurationOverview.js
@@ -15,17 +15,17 @@
  * limitations under the License.
  */
 
-import {Selector, t} from 'testcafe'
-import {Table} from '../components/Table'
-import {confirmation} from '../components/confirmation'
-import {successNotification} from '../components/notifications'
+import {Selector, t} from 'testcafe';
+import {Table} from '../components/Table';
+import {confirmation} from '../components/confirmation';
+import {successNotification} from '../components/notifications';
 
 export class PageConfigurationOverview {
     constructor() {
         this.createClusterConfigButton = Selector('.btn-ignite').withText('Create Cluster Configuration');
         this.importFromDBButton = Selector('.btn-ignite').withText('Import from Database');
         this.clustersTable = new Table(Selector('pc-items-table'));
-        this.pageHeader = Selector('.pc-page-header')
+        this.pageHeader = Selector('.pc-page-header');
     }
     async removeAllItems() {
         await t.click(this.clustersTable.allItemsCheckbox);
diff --git a/modules/web-console/e2e/testcafe/page-models/PageSignIn.js b/modules/web-console/e2e/testcafe/page-models/PageSignIn.js
index 1712cd69f7703..57031b6f3a0d0 100644
--- a/modules/web-console/e2e/testcafe/page-models/PageSignIn.js
+++ b/modules/web-console/e2e/testcafe/page-models/PageSignIn.js
@@ -16,62 +16,17 @@
  */
 
 import {Selector, t} from 'testcafe';
-import { resolveUrl } from '../envtools';
-import {AngularJSSelector} from 'testcafe-angular-selectors';
 import {CustomFormField} from '../components/FormField';
 
-export class PageSignIn {
-    constructor() {
-        this._selector = Selector('page-sign-in');
-        this.signin = {
-            email: new CustomFormField({model: '$ctrl.data.signin.email'}),
-            password: new CustomFormField({model: '$ctrl.data.signin.password'})
-        };
-        this.forgotPassword = {
-            email: new CustomFormField({model: '$ctrl.data.remindPassword.email'})
-        };
-        this.signup = {
-            email: new CustomFormField({model: '$ctrl.data.signup.email'}),
-            password: new CustomFormField({model: '$ctrl.data.signup.password'}),
-            passwordConfirm: new CustomFormField({model: 'confirm'}),
-            firstName: new CustomFormField({model: '$ctrl.data.signup.firstName'}),
-            lastName: new CustomFormField({model: '$ctrl.data.signup.lastName'}),
-            company: new CustomFormField({model: '$ctrl.data.signup.company'}),
-            country: new CustomFormField({model: '$ctrl.data.signup.country'})
-        };
-        this.signinButton = Selector('#signin_submit');
-        this.signupButton = Selector('#signup_submit');
-        this.showForgotPasswordButton = Selector('#forgot_show');
-        this.remindPasswordButton = Selector('#forgot_submit');
-    }
-
-    async open() {
-        await t.navigateTo(resolveUrl('/signin'));
-    }
-
+export const pageSignin = {
+    email: new CustomFormField({model: '$ctrl.data.email'}),
+    password: new CustomFormField({model: '$ctrl.data.password'}),
+    signinButton: Selector('button').withText('Sign In'),
+    selector: Selector('page-signin'),
     async login(email, password) {
         return await t
-            .typeText(this.signin.email.control, email)
-            .typeText(this.signin.password.control, password)
+            .typeText(this.email.control, email)
+            .typeText(this.password.control, password)
             .click(this.signinButton);
     }
-
-    async fillSignupForm({
-        email,
-        password,
-        passwordConfirm,
-        firstName,
-        lastName,
-        company,
-        country
-    }) {
-        await t
-            .typeText(this.signup.email.control, email, {replace: true})
-            .typeText(this.signup.password.control, password, {replace: true})
-            .typeText(this.signup.passwordConfirm.control, passwordConfirm, {replace: true})
-            .typeText(this.signup.firstName.control, firstName, {replace: true})
-            .typeText(this.signup.lastName.control, lastName, {replace: true})
-            .typeText(this.signup.company.control, company, {replace: true});
-        await this.signup.country.selectOption(country);
-    }
-}
+};
diff --git a/modules/web-console/e2e/testcafe/page-models/pageConfigurationAdvancedIGFS.js b/modules/web-console/e2e/testcafe/page-models/pageConfigurationAdvancedIGFS.js
index f3ac35cd7d8ca..8225f7edbe078 100644
--- a/modules/web-console/e2e/testcafe/page-models/pageConfigurationAdvancedIGFS.js
+++ b/modules/web-console/e2e/testcafe/page-models/pageConfigurationAdvancedIGFS.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import {Selector} from 'testcafe'
-import {isVisible} from '../helpers'
+import {Selector} from 'testcafe';
+import {isVisible} from '../helpers';
 
 export const createIGFSButton = Selector('pc-items-table footer-slot .link-success').filter(isVisible);
diff --git a/modules/web-console/e2e/testcafe/page-models/pageConfigurationAdvancedModels.js b/modules/web-console/e2e/testcafe/page-models/pageConfigurationAdvancedModels.js
index 196ac3c18a23c..c9b64b98c43f9 100644
--- a/modules/web-console/e2e/testcafe/page-models/pageConfigurationAdvancedModels.js
+++ b/modules/web-console/e2e/testcafe/page-models/pageConfigurationAdvancedModels.js
@@ -15,9 +15,9 @@
  * limitations under the License.
  */
 
-import {Selector} from 'testcafe'
-import {FormField} from '../components/FormField'
-import {isVisible} from '../helpers'
+import {Selector} from 'testcafe';
+import {FormField} from '../components/FormField';
+import {isVisible} from '../helpers';
 
 export const createModelButton = Selector('pc-items-table footer-slot .link-success').filter(isVisible);
 export const general = {
diff --git a/modules/web-console/e2e/testcafe/page-models/pageForgotPassword.js b/modules/web-console/e2e/testcafe/page-models/pageForgotPassword.js
new file mode 100644
index 0000000000000..3b4160a24ef2c
--- /dev/null
+++ b/modules/web-console/e2e/testcafe/page-models/pageForgotPassword.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Selector} from 'testcafe';
+import {CustomFormField} from '../components/FormField';
+
+export const pageForgotPassword = {
+    email: new CustomFormField({model: '$ctrl.data.email'}),
+    remindPasswordButton: Selector('button').withText('Send it to me')
+};
diff --git a/modules/web-console/e2e/testcafe/page-models/pageProfile.js b/modules/web-console/e2e/testcafe/page-models/pageProfile.js
index a2eb1dba6f117..8a81f362e180b 100644
--- a/modules/web-console/e2e/testcafe/page-models/pageProfile.js
+++ b/modules/web-console/e2e/testcafe/page-models/pageProfile.js
@@ -28,8 +28,8 @@ export const pageProfile = {
     company: new CustomFormField({id: 'companyInput'}),
     securityToken: {
         panel: new PanelCollapsible('security token'),
-        generateTokenButton: Selector('i').withAttribute('ng-click', '$ctrl.generateToken()'),
-        value: Selector('#current-security-token')
+        generateTokenButton: Selector('a').withText('Generate Random Security Token?'),
+        value: new CustomFormField({id: 'securityTokenInput'})
     },
     password: {
         panel: new PanelCollapsible('password'),
diff --git a/modules/web-console/e2e/testcafe/page-models/pageSignup.js b/modules/web-console/e2e/testcafe/page-models/pageSignup.js
new file mode 100644
index 0000000000000..c380abed41907
--- /dev/null
+++ b/modules/web-console/e2e/testcafe/page-models/pageSignup.js
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Selector, t} from 'testcafe';
+import {CustomFormField} from '../components/FormField';
+
+export const pageSignup = {
+    email: new CustomFormField({model: '$ctrl.data.email'}),
+    password: new CustomFormField({model: '$ctrl.data.password'}),
+    passwordConfirm: new CustomFormField({model: 'confirm'}),
+    firstName: new CustomFormField({model: '$ctrl.data.firstName'}),
+    lastName: new CustomFormField({model: '$ctrl.data.lastName'}),
+    company: new CustomFormField({model: '$ctrl.data.company'}),
+    country: new CustomFormField({model: '$ctrl.data.country'}),
+    signupButton: Selector('button').withText('Sign Up'),
+    async fillSignupForm({
+        email,
+        password,
+        passwordConfirm,
+        firstName,
+        lastName,
+        company,
+        country
+    }) {
+        await t
+            .typeText(this.email.control, email, {replace: true})
+            .typeText(this.password.control, password, {replace: true})
+            .typeText(this.passwordConfirm.control, passwordConfirm, {replace: true})
+            .typeText(this.firstName.control, firstName, {replace: true})
+            .typeText(this.lastName.control, lastName, {replace: true})
+            .typeText(this.company.control, company, {replace: true});
+        await this.country.selectOption(country);
+    }
+};
diff --git a/modules/web-console/e2e/testcafe/roles.js b/modules/web-console/e2e/testcafe/roles.js
index 5f584b234aad1..ae5a6a52567f4 100644
--- a/modules/web-console/e2e/testcafe/roles.js
+++ b/modules/web-console/e2e/testcafe/roles.js
@@ -15,18 +15,16 @@
  * limitations under the License.
  */
 
-const { Role, t } = require('testcafe');
-import { resolveUrl } from './envtools';
-const { PageSignIn } = require('./page-models/PageSignIn');
+import { Role, t } from 'testcafe';
+import { resolveUrl } from './environment/envtools';
+import {pageSignin as page} from './page-models/pageSignin';
 
 export const createRegularUser = () => {
-    return new Role(resolveUrl('/signin'), async() => {
+    return Role(resolveUrl('/signin'), async() => {
         await t.eval(() => window.localStorage.clear());
 
         // Disable "Getting started" modal.
         await t.eval(() => window.localStorage.showGettingStarted = 'false');
-
-        const page = new PageSignIn();
         await page.login('a@a', 'a');
     });
 };
diff --git a/modules/web-console/e2e/testcafe/testcafe-runner.js b/modules/web-console/e2e/testcafe/testcafe-runner.js
new file mode 100644
index 0000000000000..681ee2ed0e758
--- /dev/null
+++ b/modules/web-console/e2e/testcafe/testcafe-runner.js
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { startEnv, dropTestDB, insertTestUser } = require('./environment/envtools');
+
+const createTestCafe = require('testcafe');
+
+let testcafe = null;
+
+const startTestcafe = (config) => {
+    return createTestCafe('localhost', 1337, 1338)
+        .then(async(tc) => {
+            try {
+                if (config.enableEnvironment)
+                    await startEnv();
+
+                await dropTestDB();
+                await insertTestUser();
+
+                testcafe = tc;
+
+                const runner = testcafe.createRunner();
+
+                console.log('Start E2E testing!');
+
+                return runner
+                    .src(config.fixturesPathsArray)
+                    .browsers(config.browsers)
+                    .reporter(config.reporter)
+                    .run({ skipJsErrors: true });
+            } catch (err) {
+                console.log(err);
+
+                process.exit(1);
+            }
+        })
+        .then(async(failedCount) => {
+            console.log('Cleaning after tests...');
+
+            testcafe.close();
+
+            await dropTestDB();
+
+            console.log('Tests failed: ' + failedCount);
+        });
+};
+
+module.exports = { startTestcafe };
diff --git a/modules/web-console/e2e/testcafe/testcafe.js b/modules/web-console/e2e/testcafe/testcafe.js
deleted file mode 100644
index 3d6cc027e713e..0000000000000
--- a/modules/web-console/e2e/testcafe/testcafe.js
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-const glob = require('glob');
-const path = require('path');
-
-require('app-module-path').addPath(path.join(__dirname, 'node_modules'));
-require('app-module-path').addPath(__dirname);
-
-const argv = require('minimist')(process.argv.slice(2));
-const envEnabled = argv.env;
-
-const { startEnv, dropTestDB } = require('./envtools');
-
-const createTestCafe = require('testcafe');
-
-// See all supported browsers at http://devexpress.github.io/testcafe/documentation/using-testcafe/common-concepts/browsers/browser-support.html#locally-installed-browsers
-const BROWSERS = ['chromium:headless --no-sandbox']; // For example: ['chrome', 'firefox'];
-
-let testcafe = null;
-
-const resolveFixturesPaths = () => {
-    let fixturesPaths = glob.sync(' ./fixtures/**/*.js');
-
-    if (process.env.IGNITE_MODULES) {
-        const igniteModulesTestcafe = path.join(process.env.IGNITE_MODULES, 'e2e/testcafe');
-        const additionalFixturesPaths = glob.sync(path.join(igniteModulesTestcafe, 'fixtures', '**/*.js'));
-        const relativePaths = new Set(additionalFixturesPaths.map((fixturePath) => path.relative(igniteModulesTestcafe, fixturePath)));
-
-        fixturesPaths = fixturesPaths.filter((fixturePath) => !relativePaths.has(path.relative(process.cwd(), fixturePath))).concat(additionalFixturesPaths);
-    }
-
-    return fixturesPaths;
-};
-
-createTestCafe('localhost', 1337, 1338)
-    .then(async(tc) => {
-        try {
-            if (envEnabled)
-                await startEnv();
-
-            await dropTestDB();
-
-            testcafe = tc;
-
-            const runner = testcafe.createRunner();
-            const reporter = process.env.TEAMCITY ? 'teamcity' : 'spec';
-
-            console.log('Start E2E testing!');
-
-            return runner
-                .src(resolveFixturesPaths())
-                .browsers(BROWSERS)
-                .reporter(reporter)
-                .run({ skipJsErrors: true, quarantineMode: process.env.QUARANTINE_MODE || false });
-        } catch (err) {
-            console.log(err);
-
-            process.exit(1);
-        }
-    })
-    .then(async(failedCount) => {
-        console.log('Cleaning after tests...');
-
-        testcafe.close();
-
-        if (envEnabled)
-            await dropTestDB();
-
-        console.log('Tests failed: ' + failedCount);
-
-        process.exit(0);
-    });
diff --git a/modules/web-console/e2e/testenv/Dockerfile b/modules/web-console/e2e/testenv/Dockerfile
index 9192e1f1b152a..8cb01fe3c38e4 100644
--- a/modules/web-console/e2e/testenv/Dockerfile
+++ b/modules/web-console/e2e/testenv/Dockerfile
@@ -15,72 +15,37 @@
 # limitations under the License.
 #
 
-FROM ubuntu:14.04
+FROM node:8-alpine
 
-ENV NPM_CONFIG_LOGLEVEL info
-ENV NODE_VERSION 8.11.1
-
-# Before package list update.
-RUN set -ex  && \
-      for key in \
-        9554F04D7259F04124DE6B476D5A82AC7E37093B \
-        94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
-        FD3A5288F042B6850C66B31F09FE44734EB7990E \
-        71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
-        DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
-        B9AE9905FFD7803F25714661B63B535A4C206CA9 \
-        C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
-        56730D5401028683275BD23C23EFEFE93C4CFFFE \
-      ; do \
-        gpg --keyserver ha.pool.sks-keyservers.net --recv-keys "$key" || \
-        gpg --keyserver pgp.mit.edu --recv-keys "$key" || \
-        gpg --keyserver keyserver.pgp.com --recv-keys "$key" ; \
-    done
+ENV NPM_CONFIG_LOGLEVEL error
 
 # Update package list & install.
-RUN apt-get update && \
-    apt-get install -y nginx-light curl xz-utils git dos2unix
-
-# Install Node JS.
-RUN curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.xz"  && \
-  curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" && \
-  gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc && \
-  grep " node-v$NODE_VERSION-linux-x64.tar.xz\$" SHASUMS256.txt | sha256sum -c - && \
-  tar -xJf "node-v$NODE_VERSION-linux-x64.tar.xz" -C /usr/local --strip-components=1 && \
-  rm "node-v$NODE_VERSION-linux-x64.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt
-
-ENV NPM_CONFIG_LOGLEVEL warn
+RUN apk add --no-cache nginx 
 
 # Install global node packages.
 RUN npm install -g pm2
 
-# Install frontend & backend apps.
-RUN mkdir -p /opt/web-console
+# Copy nginx config.
+COPY e2e/testenv/nginx/nginx.conf /etc/nginx/nginx.conf
+COPY e2e/testenv/nginx/web-console.conf /etc/nginx/web-console.conf
 
-# Copy source.
 WORKDIR /opt/web-console
-COPY frontend ./frontend
-COPY backend ./backend
 
+# Install node modules for frontend and backend modules.
+COPY backend/package*.json backend/
+RUN (cd backend && npm install --no-optional --production)
 
-# Install node modules.
-RUN cd /opt/web-console/frontend && npm install --no-optional --prod && npm run build
-RUN cd /opt/web-console/backend && npm install --only=production --no-optional
+COPY frontend/package*.json frontend/
+RUN (cd frontend && npm install --no-optional)
 
-# Returns to base path.
-WORKDIR /opt/web-console
-
-# Copy nginx config.
-COPY ./e2e/testenv/nginx/nginx.conf /etc/nginx/nginx.conf
-COPY ./e2e/testenv/nginx/web-console.conf /etc/nginx/web-console.conf
-
-# Setup entrypoint.
-COPY ./e2e/testenv/entrypoint.sh .
-RUN chmod 755 /opt/web-console/entrypoint.sh && dos2unix /opt/web-console/entrypoint.sh
+# Copy source.
+COPY backend backend
+COPY frontend frontend
 
-# Clean up.
-RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN (cd frontend && npm run build)
 
 EXPOSE 9001
 
-ENTRYPOINT ["/opt/web-console/entrypoint.sh"]
+WORKDIR /opt/web-console/backend
+
+CMD nginx && pm2-runtime index.js -n web-console-backend
diff --git a/modules/web-console/e2e/testenv/entrypoint.sh b/modules/web-console/e2e/testenv/entrypoint.sh
deleted file mode 100644
index f6107a4fe7603..0000000000000
--- a/modules/web-console/e2e/testenv/entrypoint.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-service nginx start
-
-cd /opt/web-console/backend && pm2 start ./index.js --no-daemon
diff --git a/modules/web-console/e2e/testenv/nginx/nginx.conf b/modules/web-console/e2e/testenv/nginx/nginx.conf
index dbc79d7d77bf8..169b3342cb10a 100644
--- a/modules/web-console/e2e/testenv/nginx/nginx.conf
+++ b/modules/web-console/e2e/testenv/nginx/nginx.conf
@@ -15,41 +15,41 @@
 # limitations under the License.
 #
 
-user  www-data;
-worker_processes  1;
+user nginx;
+worker_processes 1;
 
 error_log  /var/log/nginx/error.log  warn;
 pid        /var/run/nginx.pid;
 
 events {
-    worker_connections  128;
+  worker_connections  128;
 }
 
 http {
-    server_tokens off;
-    sendfile            on;
-    tcp_nopush          on;
-
-    keepalive_timeout   60;
-    tcp_nodelay         on;
-
-    client_max_body_size 100m;
-
-    #access log
-    log_format main '$http_host $remote_addr - $remote_user [$time_local] '
-    '"$request" $status $bytes_sent '
-    '"$http_referer" "$http_user_agent" '
-    '"$gzip_ratio"';
-
-    include /etc/nginx/mime.types;
-    default_type  application/octet-stream;
-    gzip on;
-    gzip_disable "msie6";
-    gzip_types text/plain text/css text/xml text/javascript application/json application/x-javascript application/xml application/xml+rss application/javascript;
-    gzip_vary on;
-    gzip_comp_level 5;
-
-    access_log  /var/log/nginx/access.log  main;
-    #conf.d
-    include web-console.conf ;
+  server_tokens off;
+  sendfile            on;
+  tcp_nopush          on;
+
+  keepalive_timeout   60;
+  tcp_nodelay         on;
+
+  client_max_body_size 100m;
+
+  #access log
+  log_format main '$http_host $remote_addr - $remote_user [$time_local] '
+  '"$request" $status $bytes_sent '
+  '"$http_referer" "$http_user_agent" '
+  '"$gzip_ratio"';
+
+  include /etc/nginx/mime.types;
+  default_type  application/octet-stream;
+  gzip on;
+  gzip_disable "msie6";
+  gzip_types text/plain text/css text/xml text/javascript application/json application/x-javascript application/xml application/xml+rss application/javascript;
+  gzip_vary on;
+  gzip_comp_level 5;
+
+  access_log  /var/log/nginx/access.log  main;
+  #conf.d
+  include web-console.conf ;
 }
diff --git a/modules/web-console/frontend/.gitignore b/modules/web-console/frontend/.gitignore
index 60d202902383d..7fe0d3f8e6a9e 100644
--- a/modules/web-console/frontend/.gitignore
+++ b/modules/web-console/frontend/.gitignore
@@ -1,8 +1,5 @@
-*.idea
-*.log
 *.log.*
 .npmrc
-build/*
-node_modules
 public/stylesheets/*.css
-
+build/
+node_modules/
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 692acc54f3d64..b9d72ab4878e2 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -29,6 +29,7 @@ import './modules/demo/Demo.module';
 import './modules/states/logout.state';
 import './modules/states/admin.state';
 import './modules/states/errors.state';
+import './modules/states/settings.state';
 
 // ignite:modules
 import './core';
@@ -85,6 +86,7 @@ import SqlTypes from './services/SqlTypes.service';
 import LegacyTable from './services/LegacyTable.service';
 import LegacyUtils from './services/LegacyUtils.service';
 import Messages from './services/Messages.service';
+import ErrorParser from './services/ErrorParser.service';
 import ModelNormalizer from './services/ModelNormalizer.service.js';
 import UnsavedChangesGuard from './services/UnsavedChangesGuard.service';
 import Caches from './services/Caches';
@@ -105,9 +107,6 @@ import hasPojo from './filters/hasPojo.filter';
 import uiGridSubcategories from './filters/uiGridSubcategories.filter';
 import id8 from './filters/id8.filter';
 
-// Controllers
-import resetPassword from './controllers/reset-password.controller';
-
 // Components
 import igniteListOfRegisteredUsers from './components/list-of-registered-users';
 import IgniteActivitiesUserDialog from './components/activities-user-dialog';
@@ -131,30 +130,34 @@ import bsSelectMenu from './components/bs-select-menu';
 import protectFromBsSelectRender from './components/protect-from-bs-select-render';
 import uiGridHovering from './components/ui-grid-hovering';
 import uiGridFilters from './components/ui-grid-filters';
+import uiGridColumnResizer from './components/ui-grid-column-resizer';
 import listEditable from './components/list-editable';
 import breadcrumbs from './components/breadcrumbs';
 import panelCollapsible from './components/panel-collapsible';
 import clusterSelector from './components/cluster-selector';
-import connectedClusters from './components/connected-clusters';
-import pageSignIn from './components/page-signin';
+import connectedClusters from './components/connected-clusters-badge';
+import connectedClustersDialog from './components/connected-clusters-dialog';
 import pageLanding from './components/page-landing';
+import passwordVisibility from './components/password-visibility';
+import progressLine from './components/progress-line';
+import formField from './components/form-field';
 
 import pageProfile from './components/page-profile';
 import pagePasswordChanged from './components/page-password-changed';
 import pagePasswordReset from './components/page-password-reset';
+import pageSignup from './components/page-signup';
+import pageSignin from './components/page-signin';
+import pageForgotPassword from './components/page-forgot-password';
 
 import igniteServices from './services';
 
 import uiAceJava from './directives/ui-ace-java';
 import uiAceSpring from './directives/ui-ace-spring';
 
-// Inject external modules.
-import IgniteModules from 'IgniteModules/index';
-
 import baseTemplate from 'views/base.pug';
 import * as icons from '../public/images/icons';
 
-angular.module('ignite-console', [
+export default angular.module('ignite-console', [
     // Optional AngularJS modules.
     'ngAnimate',
     'ngSanitize',
@@ -164,7 +167,6 @@ angular.module('ignite-console', [
     'dndLists',
     'gridster',
     'mgcrea.ngStrap',
-    'ngRetina',
     'nvd3',
     'pascalprecht.translate',
     'smart-table',
@@ -193,6 +195,7 @@ angular.module('ignite-console', [
     'ignite-console.states.logout',
     'ignite-console.states.admin',
     'ignite-console.states.errors',
+    'ignite-console.states.settings',
     // Common modules.
     'ignite-console.dialog',
     'ignite-console.navbar',
@@ -221,6 +224,7 @@ angular.module('ignite-console', [
     bsSelectMenu.name,
     uiGridHovering.name,
     uiGridFilters.name,
+    uiGridColumnResizer.name,
     protectFromBsSelectRender.name,
     AngularStrapTooltip.name,
     AngularStrapSelect.name,
@@ -229,18 +233,22 @@ angular.module('ignite-console', [
     clusterSelector.name,
     servicesModule.name,
     connectedClusters.name,
+    connectedClustersDialog.name,
     igniteListOfRegisteredUsers.name,
     pageProfile.name,
     exposeInput.name,
-    pageSignIn.name,
     pageLanding.name,
     pagePasswordChanged.name,
     pagePasswordReset.name,
+    pageSignup.name,
+    pageSignin.name,
+    pageForgotPassword.name,
     uiAceJava.name,
     uiAceSpring.name,
     breadcrumbs.name,
-    // Ignite modules.
-    IgniteModules.name
+    passwordVisibility.name,
+    progressLine.name,
+    formField.name
 ])
 .service($exceptionHandler.name, $exceptionHandler)
 // Directives.
@@ -250,7 +258,7 @@ angular.module('ignite-console', [
 .directive(...igniteCopyToClipboard)
 .directive(...igniteHideOnStateChange)
 .directive(...igniteInformation)
-.directive(...igniteMatch)
+.directive('igniteMatch', igniteMatch)
 .directive(...igniteOnClickFocus)
 .directive(...igniteOnEnter)
 .directive(...igniteOnEnterFocusMove)
@@ -278,6 +286,7 @@ angular.module('ignite-console', [
 .service(...Focus)
 .service(...InetAddress)
 .service(...Messages)
+.service('IgniteErrorParser', ErrorParser)
 .service(...ModelNormalizer)
 .service(...LegacyTable)
 .service(...FormUtils)
@@ -288,8 +297,6 @@ angular.module('ignite-console', [
 .service(CSV.name, CSV)
 .service('IGFSs', IGFSs)
 .service('Models', Models)
-// Controllers.
-.controller(...resetPassword)
 // Filters.
 .filter('byName', byName)
 .filter('defaultName', defaultName)
@@ -308,11 +315,6 @@ angular.module('ignite-console', [
             url: '',
             abstract: true,
             template: baseTemplate
-        })
-        .state('base.settings', {
-            url: '/settings',
-            abstract: true,
-            template: '<ui-view></ui-view>'
         });
 
     $urlRouterProvider.otherwise('/404');
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/controller.js b/modules/web-console/frontend/app/components/bs-select-menu/controller.js
index f8c0171eccd92..70346ed147177 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/controller.js
+++ b/modules/web-console/frontend/app/components/bs-select-menu/controller.js
@@ -18,8 +18,11 @@
 export default class {
     static $inject = ['$scope'];
 
+    /**
+     * @param {ng.IScope} $scope
+     */
     constructor($scope) {
-        Object.assign(this, {$scope});
+        this.$scope = $scope;
     }
 
     areAllSelected() {
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/index.js b/modules/web-console/frontend/app/components/bs-select-menu/index.js
index e64e1fad27cb5..a9bafb250b465 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/index.js
+++ b/modules/web-console/frontend/app/components/bs-select-menu/index.js
@@ -18,7 +18,9 @@
 import angular from 'angular';
 
 import directive from './directive';
+import {directive as transcludeToBody} from './transcludeToBody.directive';
 
 export default angular
     .module('ignite-console.bs-select-menu', [])
+    .directive('bssmTranscludeToBody', transcludeToBody)
     .directive('bsSelectMenu', directive);
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/index.spec.js b/modules/web-console/frontend/app/components/bs-select-menu/index.spec.js
new file mode 100644
index 0000000000000..9b9fa3968c9cd
--- /dev/null
+++ b/modules/web-console/frontend/app/components/bs-select-menu/index.spec.js
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import 'mocha';
+import {assert} from 'chai';
+import angular from 'angular';
+import componentModule from './index.js';
+
+suite('bs-select-menu', () => {
+    /** @type {ng.IScope} */
+    let $scope;
+    /** @type {ng.ICompileService} */
+    let $compile;
+
+    setup(() => {
+        angular.module('test', [componentModule.name]);
+        angular.mock.module('test');
+        angular.mock.inject((_$rootScope_, _$compile_) => {
+            $compile = _$compile_;
+            $scope = _$rootScope_.$new();
+        });
+    });
+
+    test('Create/destroy', () => {
+        $scope.$matches = [];
+        $scope.show = false;
+        const el = angular.element(`
+            <div ng-if='show'>
+                <bs-select-menu></bs-select-menu>
+            </div>
+        `);
+
+        const overlay = () => document.body.querySelector('.bssm-click-overlay');
+
+        $compile(el)($scope);
+        $scope.$digest();
+        assert.notOk(overlay(), 'No overlay on init');
+
+        $scope.show = true;
+        $scope.$isShown = true;
+        $scope.$digest();
+        assert.ok(overlay(), 'Adds overlay to body on show');
+
+        $scope.show = false;
+        $scope.$digest();
+        assert.notOk(overlay(), 'Removes overlay when element is removed from DOM');
+
+        $scope.show = true;
+        $scope.$isShown = false;
+        $scope.$digest();
+        assert.notOk(overlay(), 'Removes overlay menu is closed');
+    });
+});
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/style.scss b/modules/web-console/frontend/app/components/bs-select-menu/style.scss
index 4c071f697e105..bfa0063340732 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/style.scss
+++ b/modules/web-console/frontend/app/components/bs-select-menu/style.scss
@@ -69,21 +69,12 @@
             }
         }
 
-        &:last-child > .bssm-item-button {
+        &:last-of-type > .bssm-item-button {
             border-bottom: none;
             padding-bottom: 10px;
         }
     }
 
-    .bssm-click-overlay {
-        position: fixed;
-        top: 0;
-        right: 0;
-        bottom: 0;
-        left: 0;
-        z-index: -1;
-    }
-    
     [class*='bssm-multiple'] {
         .bssm-active-indicator {
             display: initial;
@@ -100,3 +91,12 @@
         }
     }
 }
+
+.bssm-click-overlay {
+    position: fixed;
+    top: 0;
+    right: 0;
+    bottom: 0;
+    left: 0;
+    z-index: 1999;
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/template.pug b/modules/web-console/frontend/app/components/bs-select-menu/template.pug
index c8c6eaac09099..4ffa144ae5ce2 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/template.pug
+++ b/modules/web-console/frontend/app/components/bs-select-menu/template.pug
@@ -34,7 +34,7 @@ ul.bs-select-menu(
             type='button'
             role='menuitem'
             tabindex='-1'
-            ng-click='$select($index, $event)'
+            ng-click='$select($index, $event); $event.stopPropagation();'
             ng-class=`{ 'bssm-item-button__active': $isActive($index) }`
             data-placement='right auto'
             title='{{ ::match.label }}'
@@ -43,4 +43,5 @@ ul.bs-select-menu(
                 ng-src='{{ $isActive($index) ? "/images/checkbox-active.svg" : "/images/checkbox.svg" }}'
             )
             span.bssm-item-text(ng-bind-html='match.label')
-    .bssm-click-overlay(ng-click='$hide()')
+    bssm-transclude-to-body(ng-if='$isShown')
+        .bssm-click-overlay(ng-click='$hide()')
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js b/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js
new file mode 100644
index 0000000000000..b415719e87aab
--- /dev/null
+++ b/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class Controller {
+    static $inject = ['$transclude', '$document'];
+
+    /**
+     * @param {ng.ITranscludeFunction} $transclude
+     * @param {JQLite} $document
+     */
+    constructor($transclude, $document) {
+        this.$transclude = $transclude;
+        this.$document = $document;
+    }
+
+    $postLink() {
+        this.$transclude((clone) => {
+            this.clone = clone;
+            this.$document.find('body').append(clone);
+        });
+    }
+
+    $onDestroy() {
+        this.clone.remove();
+        this.clone = this.$document = null;
+    }
+}
+
+export function directive() {
+    return {
+        restrict: 'E',
+        transclude: true,
+        controller: Controller,
+        scope: {}
+    };
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java b/modules/web-console/frontend/app/components/cluster-security-icon/component.js
similarity index 84%
rename from modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
rename to modules/web-console/frontend/app/components/cluster-security-icon/component.js
index 03ca62116b7f7..abbafe338ad28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/worker/package-info.java
+++ b/modules/web-console/frontend/app/components/cluster-security-icon/component.js
@@ -15,8 +15,11 @@
  * limitations under the License.
  */
 
-/**
- * <!-- Package description. -->
- * System worker registry and control MBean implementation.
- */
-package org.apache.ignite.internal.worker;
\ No newline at end of file
+import template from './template.pug';
+
+export default {
+    bindings: {
+        secured: '<'
+    },
+    template
+};
diff --git a/modules/web-console/frontend/app/components/cluster-security-icon/index.js b/modules/web-console/frontend/app/components/cluster-security-icon/index.js
new file mode 100644
index 0000000000000..a7e91396264d1
--- /dev/null
+++ b/modules/web-console/frontend/app/components/cluster-security-icon/index.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import component from './component';
+
+export default angular
+    .module('ignite-console.cluster-security-icon', [])
+    .component('clusterSecurityIcon', component);
diff --git a/modules/web-console/frontend/app/components/cluster-security-icon/template.pug b/modules/web-console/frontend/app/components/cluster-security-icon/template.pug
new file mode 100644
index 0000000000000..a4d8f8a9fbf4f
--- /dev/null
+++ b/modules/web-console/frontend/app/components/cluster-security-icon/template.pug
@@ -0,0 +1,30 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+svg(
+    ng-if='$ctrl.secured'
+    ignite-icon='lockClosed'
+    bs-tooltip=''
+    data-title='Security Cluster'
+    data-placement='top'
+)
+svg(
+    ng-if='!$ctrl.secured'
+    ignite-icon='lockOpened'
+    bs-tooltip=''
+    data-title='Non Security Cluster'
+    data-placement='top'
+)
diff --git a/modules/web-console/frontend/app/components/cluster-selector/controller.js b/modules/web-console/frontend/app/components/cluster-selector/controller.js
index d47432b3c24c1..fb64b9f229eed 100644
--- a/modules/web-console/frontend/app/components/cluster-selector/controller.js
+++ b/modules/web-console/frontend/app/components/cluster-selector/controller.js
@@ -17,41 +17,72 @@
 
 import _ from 'lodash';
 
+import { BehaviorSubject } from 'rxjs/BehaviorSubject';
+import 'rxjs/add/operator/combineLatest';
+
 export default class {
-    static $inject = ['$scope', 'AgentManager', 'IgniteConfirm'];
+    static $inject = ['AgentManager', 'IgniteConfirm', 'IgniteVersion', 'IgniteMessages'];
 
-    constructor($scope, agentMgr, Confirm) {
-        Object.assign(this, { $scope, agentMgr, Confirm });
+    /**
+     * @param agentMgr Agent manager.
+     * @param Confirm Confirmation service.
+     * @param Version Version check service.
+     * @param Messages Messages service.
+     */
+    constructor(agentMgr, Confirm, Version, Messages) {
+        this.agentMgr = agentMgr;
+        this.Confirm = Confirm;
+        this.Version = Version;
+        this.Messages = Messages;
 
         this.clusters = [];
         this.isDemo = agentMgr.isDemoMode();
+        this._inProgressSubject = new BehaviorSubject(false);
     }
 
     $onInit() {
+        if (this.isDemo)
+            return;
+
+        this.inProgress$ = this._inProgressSubject.asObservable();
+
         this.clusters$ = this.agentMgr.connectionSbj
-            .do(({ cluster, clusters }) => {
-                this.cluster = cluster;
+            .combineLatest(this.inProgress$)
+            .do(([sbj, inProgress]) => this.inProgress = inProgress)
+            .filter(([sbj, inProgress]) => !inProgress)
+            .do(([{cluster, clusters}]) => {
+                this.cluster = cluster ? {...cluster} : null;
                 this.clusters = _.orderBy(clusters, ['name'], ['asc']);
             })
             .subscribe(() => {});
     }
 
     $onDestroy() {
-        this.clusters$.unsubscribe();
+        if (!this.isDemo)
+            this.clusters$.unsubscribe();
     }
 
     change() {
         this.agentMgr.switchCluster(this.cluster);
     }
 
+    isChangeStateAvailable() {
+        return !this.isDemo && this.cluster && this.Version.since(this.cluster.clusterVersion, '2.0.0');
+    }
+
     toggle($event) {
         $event.preventDefault();
 
         const toggleClusterState = () => {
-            this.inProgress = true;
+            this._inProgressSubject.next(true);
 
             return this.agentMgr.toggleClusterState()
-                .finally(() => this.inProgress = false);
+                .then(() => this._inProgressSubject.next(false))
+                .catch((err) => {
+                    this._inProgressSubject.next(false);
+
+                    this.Messages.showError('Failed to toggle cluster state: ', err);
+                });
         };
 
         if (this.cluster.active) {
diff --git a/modules/web-console/frontend/app/components/cluster-selector/style.scss b/modules/web-console/frontend/app/components/cluster-selector/style.scss
index 966be9991002c..069dd641d5a6f 100644
--- a/modules/web-console/frontend/app/components/cluster-selector/style.scss
+++ b/modules/web-console/frontend/app/components/cluster-selector/style.scss
@@ -25,9 +25,12 @@ cluster-selector {
     align-items: center;
     justify-content: space-between;
 
-	& > .btn-ignite {
+	.btn-ignite.btn-ignite--primary,
+    .btn-ignite.btn-ignite--success {
         border-radius: 9px;
         min-height: 0;
+
+        color: white;
         font-size: 12px;
         font-weight: bold;
         line-height: 17px;
@@ -60,6 +63,12 @@ cluster-selector {
         color: $ignite-brand-success;
     }
 
+    [ignite-icon='lockClosed'], [ignite-icon='lockOpened'] {
+        margin-right: 5px;
+        height: 10px;
+        width: 8px;
+    }
+
     .bs-select-menu {
         color: $text-color;
     }
diff --git a/modules/web-console/frontend/app/components/cluster-selector/template.pug b/modules/web-console/frontend/app/components/cluster-selector/template.pug
index c97a69809de2d..f1e80db67f2c4 100644
--- a/modules/web-console/frontend/app/components/cluster-selector/template.pug
+++ b/modules/web-console/frontend/app/components/cluster-selector/template.pug
@@ -29,40 +29,49 @@ button.btn-ignite.btn-ignite--primary(
 button.btn-ignite.btn-ignite--primary(
     data-ng-if='!$ctrl.isDemo && $ctrl.clusters.length == 1'
 )
+    cluster-security-icon(secured='$ctrl.cluster.secured')
     | {{ $ctrl.cluster.name }}
 
-div.btn-ignite.btn-ignite--primary(
-    data-ng-if='!$ctrl.isDemo && $ctrl.clusters.length > 1'
-
-    data-ng-model='$ctrl.cluster'
-
-    bs-select=''
-    bs-options='item as item.name for item in $ctrl.clusters'
-    data-trigger='hover focus'
-    data-container='self'
-
-    data-ng-change='$ctrl.change()'
-
-    protect-from-bs-select-render
-)
-    span(ng-if='!$ctrl.cluster') No clusters available
-    span(ng-if='$ctrl.cluster') {{ $ctrl.cluster.name }}
-        span.icon-right.fa.fa-caret-down
+span(data-ng-if='!$ctrl.isDemo && $ctrl.clusters.length > 1')
+    div.btn-ignite.btn-ignite--primary(
+        ng-model='$ctrl.cluster'
+
+        bs-dropdown=''
+        data-trigger='click'
+        data-container='body'
+
+        tabindex='0'
+        aria-haspopup='true'
+        aria-expanded='false'
+    )
+        span(ng-if='!$ctrl.cluster') No clusters available
+        
+        span(ng-if='$ctrl.cluster')
+            cluster-security-icon(secured='$ctrl.cluster.secured')
+            | {{ $ctrl.cluster.name }}
+            span.icon-right.fa.fa-caret-down
+
+    ul.bs-select-menu.dropdown-menu(role='menu')
+        li(ng-repeat='item in $ctrl.clusters')
+            button.btn-ignite.bssm-item-button(ng-click='$ctrl.cluster = item; $ctrl.change()')
+                span.icon-left
+                    svg(ignite-icon='{{ item.secured ? "lockClosed" : "lockOpened" }}')
+                | {{ item.name }}
 
 svg(
     ng-if='!$ctrl.isDemo'
     ignite-icon='info'
     bs-tooltip=''
     data-title='Multi-Cluster Support<br/>\
-            <a href="https://apacheignite-tools.readme.io/docs/multi-cluster-support" target="_blank">More info</a>'
+        <a href="https://apacheignite-tools.readme.io/docs/multi-cluster-support" target="_blank">More info</a>'
     data-placement='bottom'
 )
 
-.cluster-selector--state(ng-if='!$ctrl.isDemo && $ctrl.cluster')
+.cluster-selector--state(ng-if='$ctrl.isChangeStateAvailable()')
     | Cluster {{ $ctrl.cluster.active ? 'active' : 'inactive' }}
 
 +switcher()(
-    ng-if='!$ctrl.isDemo && $ctrl.cluster'
+    ng-if='$ctrl.isChangeStateAvailable()'
     ng-click='$ctrl.toggle($event)'
     ng-checked='$ctrl.cluster.active'
     ng-disabled='$ctrl.inProgress'
@@ -71,5 +80,5 @@ svg(
     is-in-progress='{{ $ctrl.inProgress }}'
 )
 
-div(ng-if='$ctrl.inProgress')
+div(ng-if='$ctrl.inProgress && $ctrl.isChangeStateAvailable()')
     | {{ !$ctrl.cluster.active ? 'Activating...' : 'Deactivating...' }}
diff --git a/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js b/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js
new file mode 100644
index 0000000000000..64b23cf01a895
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import AgentManager from 'app/modules/agent/AgentManager.service';
+
+export default class {
+    static $inject = [AgentManager.name, 'ConnectedClustersDialog'];
+
+    /** @type {Number} */
+    connectedClusters = 0;
+
+    /**
+     * @param {AgentManager} agentMgr
+     * @param connectedClustersDialog
+     */
+    constructor(agentMgr, connectedClustersDialog) {
+        this.agentMgr = agentMgr;
+        this.connectedClustersDialog = connectedClustersDialog;
+    }
+
+    show() {
+        this.connectedClustersDialog.show({
+            clusters: this.clusters
+        });
+    }
+
+    $onInit() {
+        this.connectedClusters$ = this.agentMgr.connectionSbj
+            .do(({ clusters }) => this.connectedClusters = clusters.length)
+            .do(({ clusters }) => this.clusters = clusters)
+            .subscribe();
+    }
+
+    $onDestroy() {
+        this.connectedClusters$.unsubscribe();
+    }
+}
diff --git a/modules/web-console/frontend/app/components/connected-clusters/index.js b/modules/web-console/frontend/app/components/connected-clusters-badge/index.js
similarity index 85%
rename from modules/web-console/frontend/app/components/connected-clusters/index.js
rename to modules/web-console/frontend/app/components/connected-clusters-badge/index.js
index fd8dd115e276a..8d5aa073dc52f 100644
--- a/modules/web-console/frontend/app/components/connected-clusters/index.js
+++ b/modules/web-console/frontend/app/components/connected-clusters-badge/index.js
@@ -21,8 +21,12 @@ import './style.scss';
 import template from './template.pug';
 import controller from './controller';
 
+import connectedClustersDialog from '../connected-clusters-dialog';
+
 export default angular
-    .module('ignite-console.connected-clusters', [])
+    .module('ignite-console.connected-clusters-badge', [
+        connectedClustersDialog.name
+    ])
     .component('connectedClusters', {
         template,
         controller
diff --git a/modules/web-console/frontend/app/components/connected-clusters-badge/style.scss b/modules/web-console/frontend/app/components/connected-clusters-badge/style.scss
new file mode 100644
index 0000000000000..3b2d202527ed7
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-badge/style.scss
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+connected-clusters {
+    @import "./../../../public/stylesheets/variables.scss";
+
+    div {
+        position: absolute;
+        top: 0;
+        right: 30px;
+
+        display: flex;
+        align-items: center;
+        padding: 3px 10px;
+
+        cursor: pointer;
+        color: white;
+        font-size: 12px;
+        line-height: 12px;
+
+        background-color: $text-color;
+
+        border-radius: 0 0 4px 4px;
+
+        [ignite-icon] {
+            margin-right: 6px;
+        }
+    }
+}
diff --git a/modules/web-console/frontend/views/base2.pug b/modules/web-console/frontend/app/components/connected-clusters-badge/template.pug
similarity index 76%
rename from modules/web-console/frontend/views/base2.pug
rename to modules/web-console/frontend/app/components/connected-clusters-badge/template.pug
index 22028d5f37868..49224e5c778c5 100644
--- a/modules/web-console/frontend/views/base2.pug
+++ b/modules/web-console/frontend/app/components/connected-clusters-badge/template.pug
@@ -14,13 +14,6 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-web-console-header
-    web-console-header-left
-        include ./includes/header-left
-    web-console-header-right
-        include ./includes/header-right
-
-.container.body-container.flex-full-height
-    div(ui-view='').flex-full-height
-
-web-console-footer
\ No newline at end of file
+div(ng-click='$ctrl.show()')
+    svg(ignite-icon='connectedClusters')
+    | Connected clusters: {{ $ctrl.connectedClusters }}
diff --git a/modules/web-console/frontend/app/components/connected-clusters/controller.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/index.js
similarity index 67%
rename from modules/web-console/frontend/app/components/connected-clusters/controller.js
rename to modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/index.js
index 377948ee67eff..26f2b12d4f8bb 100644
--- a/modules/web-console/frontend/app/components/connected-clusters/controller.js
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/index.js
@@ -15,22 +15,29 @@
  * limitations under the License.
  */
 
-export default class {
-    static $inject = ['AgentManager'];
+import template from './template.pug';
 
-    constructor(agentMgr) {
-        Object.assign(this, { agentMgr });
+import AgentManager from 'app/modules/agent/AgentManager.service';
 
-        this.connectedClusters = 0;
-    }
+class controller {
+    static $inject = [AgentManager.name];
 
-    $onInit() {
-        this.connectedClusters$ = this.agentMgr.connectionSbj
-            .do(({ clusters }) => this.connectedClusters = clusters.length)
-            .subscribe();
+    /**
+     * @param {AgentManager} agentMgr
+     */
+    constructor(agentMgr) {
+        this.agentMgr = agentMgr;
     }
 
-    $onDestroy() {
-        this.connectedClusters$.unsubscribe();
+    logout() {
+        this.agentMgr.clustersSecrets.reset(this.clusterId);
     }
 }
+
+export default {
+    controller,
+    template,
+    bindings: {
+        clusterId: '<'
+    }
+};
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/template.pug b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/template.pug
new file mode 100644
index 0000000000000..1f29439a5f4fb
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/template.pug
@@ -0,0 +1,23 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+button.btn-ignite.btn-ignite--link-dashed-secondary(
+    ng-click='$event.stopPropagation(); $ctrl.logout()'
+    bs-tooltip=''
+    data-title='Click here to logout from cluster.'
+    data-placement='top'
+)
+    svg(ignite-icon='exit')
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/index.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/index.js
new file mode 100644
index 0000000000000..480fe54002ec1
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/index.js
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './style.scss';
+import template from './template.pug';
+
+export default {
+    template,
+    bindings: {
+        status: '<'
+    }
+};
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/style.scss b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/style.scss
new file mode 100644
index 0000000000000..2f0c98149d1bc
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/style.scss
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+connected-clusters-cell-status {
+    $color-active: #417505;
+    $color-not-active: #ee2b27;
+
+    display: block;
+
+    div {
+        display: flex;
+
+        &:before {
+            content: '●';
+
+            position: relative;
+            height: 16px;
+            margin-right: 10px;
+            
+            font-size: 24px;
+            line-height: 19px;
+        }
+    }
+
+    .#{&}-active {
+        color: $color-active;
+    }
+
+    .#{&}-not-active {
+        color: $color-not-active;
+    }
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/template.pug b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/template.pug
new file mode 100644
index 0000000000000..e4f21b6c4b39a
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-status/template.pug
@@ -0,0 +1,18 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+div(ng-class='["connected-clusters-cell-status-"+($ctrl.status ? "active" : "not-active")]')
+    i {{ $ctrl.status ? 'Active' : 'Not Active' }}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/column-defs.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/column-defs.js
new file mode 100644
index 0000000000000..ca5dc6b23f04e
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/column-defs.js
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default [
+    {
+        name: 'name',
+        displayName: 'Cluster',
+        field: 'name',
+        cellTemplate: `
+            <div class='ui-grid-cell-contents'>
+                <cluster-security-icon
+                    secured='row.entity.secured'
+                ></cluster-security-icon>
+                {{ COL_FIELD }}
+            </div>
+        `,
+        width: 240,
+        minWidth: 240
+    },
+    {
+        name: 'nids',
+        displayName: 'Number of Nodes',
+        field: 'nids.length',
+        cellClass: 'ui-grid-number-cell',
+        width: 160,
+        minWidth: 160
+    },
+    {
+        name: 'status',
+        displayName: 'Status',
+        field: 'active',
+        cellTemplate: `
+            <div class='ui-grid-cell-contents ui-grid-cell--status'>
+                <connected-clusters-cell-status
+                    status='COL_FIELD'
+                ></connected-clusters-cell-status>
+                <connected-clusters-cell-logout
+                    ng-if='row.entity.secured && grid.appScope.$ctrl.agentMgr.hasCredentials(row.entity.id)'
+                    cluster-id='row.entity.id'
+                ></connected-clusters-cell-logout>
+            </div>
+        `,
+        minWidth: 140
+    }
+];
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js
new file mode 100644
index 0000000000000..c4b0f93a5f752
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import _ from 'lodash';
+
+import columnDefs from './column-defs';
+
+export default class SnapshotsListCachesCtrl {
+    static $inject = ['$scope', 'AgentManager'];
+
+    /**
+     * @param {ng.IScope} $scope
+     */
+    constructor($scope, agentMgr) {
+        this.$scope = $scope;
+        this.agentMgr = agentMgr;
+    }
+
+    $onInit() {
+        this.gridOptions = {
+            data: _.orderBy(this.data, ['name'], ['asc']),
+            columnDefs,
+            columnVirtualizationThreshold: 30,
+            rowHeight: 46,
+            enableColumnMenus: false,
+            enableFullRowSelection: true,
+            enableFiltering: false,
+            selectionRowHeaderWidth: 52,
+            onRegisterApi: (api) => {
+                this.gridApi = api;
+
+                this.$scope.$watch(() => this.gridApi.grid.getVisibleRows().length, (rows) => this.adjustHeight(rows));
+            }
+        };
+    }
+
+    adjustHeight(rows) {
+        // One row for grid-no-data.
+        const height = Math.max(1, Math.min(rows, 6)) * 48 + 49 + (rows ? 8 : 0);
+
+        this.gridApi.grid.element.css('height', height + 'px');
+
+        this.gridApi.core.handleWindowResize();
+    }
+}
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/index.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/index.js
new file mode 100644
index 0000000000000..6bcb3e79cfaee
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/index.js
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './style.scss';
+import templateUrl from './template.tpl.pug';
+import controller from './controller';
+
+export default {
+    controller,
+    templateUrl,
+    bindings: {
+        gridApi: '=?',
+        data: '<?options'
+    }
+};
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/style.scss b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/style.scss
new file mode 100644
index 0000000000000..180f65ea5bfa2
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/style.scss
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+connected-clusters-list {
+    position: relative;
+    display: block;
+
+    .ui-grid-cell--status {
+        display: flex;
+        justify-content: space-between;
+    }
+
+    .ui-grid-header-cell:last-child .ui-grid-column-resizer.right {
+        border-right: none;
+    }
+
+    .ui-grid-viewport {
+        overflow: auto !important;
+    }
+
+    connected-clusters-cell-logout {
+        margin-top: -8px;
+        margin-right: -10px;
+    }
+
+    grid-no-data {
+        position: absolute;
+        top: 50%;
+
+        width: 100%;
+        padding: 0 20px;
+
+        text-align: center;
+    }
+}
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/template.tpl.pug b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/template.tpl.pug
new file mode 100644
index 0000000000000..db7c259b11686
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/template.tpl.pug
@@ -0,0 +1,19 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+.grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-hovering ui-grid-filters)
+grid-no-data(grid-api='$ctrl.gridApi')
+    | You have no connected clusters.
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/controller.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/controller.js
new file mode 100644
index 0000000000000..79975dbd15948
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/controller.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class {
+    static $inject = ['clusters'];
+
+    constructor(clusters) {
+        this.clusters = clusters;
+    }
+}
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/index.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/index.js
new file mode 100644
index 0000000000000..f4abf27c1c9b5
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/index.js
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import clusterSecurityIcon from '../cluster-security-icon';
+
+import ConnectedClustersDialog from './service';
+
+import connectedClustersList from './components/list';
+import connectedClustersCellStatus from './components/cell-status';
+import connectedClustersCellLogout from './components/cell-logout';
+
+export default angular
+    .module('ignite-console.connected-clusters-dialog', [
+        clusterSecurityIcon.name
+    ])
+    .service('ConnectedClustersDialog', ConnectedClustersDialog)
+    .component('connectedClustersList', connectedClustersList)
+    .component('connectedClustersCellStatus', connectedClustersCellStatus)
+    .component('connectedClustersCellLogout', connectedClustersCellLogout);
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/offheap/unsafe/GridUnsafeMapPerformanceTest.java b/modules/web-console/frontend/app/components/connected-clusters-dialog/service.js
similarity index 58%
rename from modules/core/src/test/java/org/apache/ignite/loadtests/offheap/unsafe/GridUnsafeMapPerformanceTest.java
rename to modules/web-console/frontend/app/components/connected-clusters-dialog/service.js
index af691b49a4365..243c330ce50cd 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/offheap/unsafe/GridUnsafeMapPerformanceTest.java
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/service.js
@@ -15,18 +15,30 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.loadtests.offheap.unsafe;
+import './style.scss';
+import controller from './controller';
+import templateUrl from './template.tpl.pug';
 
-import org.apache.ignite.internal.util.offheap.GridOffHeapMap;
-import org.apache.ignite.internal.util.offheap.GridOffHeapMapFactory;
-import org.apache.ignite.internal.util.offheap.GridOffHeapMapPerformanceAbstractTest;
+export default class {
+    static $inject = ['$modal'];
 
-/**
- * Unsafe map test.
- */
-public class GridUnsafeMapPerformanceTest extends GridOffHeapMapPerformanceAbstractTest {
-    /** {@inheritDoc} */
-    @Override protected GridOffHeapMap newMap() {
-        return GridOffHeapMapFactory.unsafeMap(concurrency, load, initCap, mem, lruStripes, evictClo);
+    /**
+     * @param {mgcrea.ngStrap.modal.IModalService} $modal
+     */
+    constructor($modal) {
+        this.$modal = $modal;
+    }
+
+    show({ clusters }) {
+        const modal = this.$modal({
+            templateUrl,
+            resolve: {
+                clusters: () => clusters
+            },
+            controller,
+            controllerAs: '$ctrl'
+        });
+
+        return modal.$promise;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/style.scss b/modules/web-console/frontend/app/components/connected-clusters-dialog/style.scss
new file mode 100644
index 0000000000000..4f9dd87069cf7
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/style.scss
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.connected-clusters-dialog {
+    .btn-ignite.btn-ignite--success {
+        padding-left: 23px;
+        padding-right: 23px;
+    }
+}
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/template.tpl.pug b/modules/web-console/frontend/app/components/connected-clusters-dialog/template.tpl.pug
new file mode 100644
index 0000000000000..dd7b4fcb87939
--- /dev/null
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/template.tpl.pug
@@ -0,0 +1,34 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins
+
+.modal.modal--ignite.theme--ignite.connected-clusters-dialog(tabindex='-1' role='dialog')
+    .modal-dialog.modal-dialog--adjust-height
+        form.modal-content(name='$ctrl.form' novalidate)
+            .modal-header
+                h4.modal-title
+                    | Connected Clusters
+                button.close(type='button' aria-label='Close' ng-click='$hide()')
+                     svg(ignite-icon="cross")
+            .modal-body.modal-body-with-scroll
+                .panel--ignite
+                    ul.tabs.tabs--blue
+                    connected-clusters-list(data-options='$ctrl.clusters')
+
+            .modal-footer
+                button.btn-ignite.btn-ignite--success(type='button' ng-click='$hide()') Ok
+                
diff --git a/modules/web-console/frontend/app/components/form-field/copyInputValueButton.directive.js b/modules/web-console/frontend/app/components/form-field/copyInputValueButton.directive.js
new file mode 100644
index 0000000000000..06500e44a5c4a
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-field/copyInputValueButton.directive.js
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const template = `
+    <svg
+        class='copy-input-value-button'
+        ignite-icon='copy'
+        ignite-copy-to-clipboard='{{ $ctrl.value }}'
+        bs-tooltip=''
+        data-title='{{::$ctrl.title}}'
+        ng-show='$ctrl.value'
+    ></svg>
+`;
+
+class CopyInputValueButtonController {
+    /** @type {ng.INgModelController} */
+    ngModel;
+
+    /**
+     * Tooltip title
+     * @type {string}
+     */
+    title;
+
+    static $inject = ['$element', '$compile', '$scope'];
+
+    /**
+     * @param {JQLite} $element
+     * @param {ng.ICompileService} $compile
+     * @param {ng.IScope} $scope
+     */
+    constructor($element, $compile, $scope) {
+        this.$element = $element;
+        this.$compile = $compile;
+        this.$scope = $scope;
+    }
+
+    $postLink() {
+        this.buttonScope = this.$scope.$new(true);
+        this.buttonScope.$ctrl = this;
+        this.$compile(template)(this.buttonScope, (clone) => {
+            this.$element[0].parentElement.appendChild(clone[0]);
+            this.buttonElement = clone;
+        });
+    }
+
+    $onDestroy() {
+        this.buttonScope.$ctrl = null;
+        this.buttonScope.$destroy();
+        this.buttonElement.remove();
+        this.buttonElement = this.$element = this.ngModel = null;
+    }
+
+    get value() {
+        return this.ngModel
+            ? this.ngModel.$modelValue
+            : void 0;
+    }
+}
+
+export function directive() {
+    return {
+        scope: false,
+        bindToController: {
+            title: '@copyInputValueButton'
+        },
+        controller: CopyInputValueButtonController,
+        require: {
+            ngModel: 'ngModel'
+        }
+    };
+}
diff --git a/modules/web-console/frontend/app/components/form-field/index.js b/modules/web-console/frontend/app/components/form-field/index.js
new file mode 100644
index 0000000000000..077ace0a01a1c
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-field/index.js
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import './style.scss';
+import {directive as showValidationError} from './showValidationError.directive';
+import {directive as copyInputValue} from './copyInputValueButton.directive';
+
+export default angular
+    .module('ignite-console.form-field', [])
+    .directive('ngModel', showValidationError)
+    .directive('copyInputValueButton', copyInputValue);
diff --git a/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js b/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js
new file mode 100644
index 0000000000000..8720c56a173a7
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-field/showValidationError.directive.js
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Brings user attention to invalid form fields.
+ * Use IgniteFormUtils.triggerValidation to trigger the event.
+ */
+export function directive($timeout) {
+    return {
+        require: ['ngModel', '?^^bsCollapseTarget', '?^^igniteFormField', '?^^panelCollapsible'],
+        link(scope, el, attr, [ngModel, bsCollapseTarget, igniteFormField, panelCollapsible]) {
+            const off = scope.$on('$showValidationError', (e, target) => {
+                if (target !== ngModel)
+                    return;
+
+                ngModel.$setTouched();
+
+                bsCollapseTarget && bsCollapseTarget.open();
+                panelCollapsible && panelCollapsible.open();
+
+                $timeout(() => {
+                    if (el[0].scrollIntoViewIfNeeded)
+                        el[0].scrollIntoViewIfNeeded();
+                    else
+                        el[0].scrollIntoView();
+
+                    if (!attr.bsSelect)
+                        $timeout(() => el[0].focus());
+
+                    igniteFormField && igniteFormField.notifyAboutError();
+                });
+            });
+        }
+    };
+}
+
+directive.$inject = ['$timeout'];
diff --git a/modules/web-console/frontend/app/components/connected-clusters/style.scss b/modules/web-console/frontend/app/components/form-field/style.scss
similarity index 70%
rename from modules/web-console/frontend/app/components/connected-clusters/style.scss
rename to modules/web-console/frontend/app/components/form-field/style.scss
index b8af8d6202f36..2cf767f11c898 100644
--- a/modules/web-console/frontend/app/components/connected-clusters/style.scss
+++ b/modules/web-console/frontend/app/components/form-field/style.scss
@@ -15,26 +15,15 @@
  * limitations under the License.
  */
 
-connected-clusters {
-    @import "./../../../public/stylesheets/variables.scss";
-
+.copy-input-value-button {
     position: absolute;
-    top: 0;
-    right: 0;
-
-    display: flex;
-    align-items: center;
-    padding: 3px 10px;
-
-    color: white;
-    font-size: 12px;
-    line-height: 12px;
-
-    background-color: $text-color;
+    top: 31px;
+    right: 10px;
 
-    border-radius: 0 0 4px 4px;
+    &:hover {
+        @import 'public/stylesheets/variables';
 
-    [ignite-icon] {
-        margin-right: 6px;
+        color: $ignite-brand-success;
+        cursor: pointer;
     }
 }
diff --git a/modules/web-console/frontend/app/components/grid-column-selector/controller.js b/modules/web-console/frontend/app/components/grid-column-selector/controller.js
index 1f048613cf327..2f5554ed6eb7f 100644
--- a/modules/web-console/frontend/app/components/grid-column-selector/controller.js
+++ b/modules/web-console/frontend/app/components/grid-column-selector/controller.js
@@ -77,9 +77,13 @@ export default class GridColumnSelectorController {
     }
 
     findScrollToNext(columns, prevColumns) {
-        if (!prevColumns) return;
+        if (!prevColumns)
+            return;
+
         const diff = difference(columns, prevColumns);
-        if (diff.length === 1 && columns.includes(diff[0])) return diff[0];
+
+        if (diff.length === 1 && columns.includes(diff[0]))
+            return diff[0];
     }
 
     setSelectedColumns() {
diff --git a/modules/web-console/frontend/app/components/grid-export/template.pug b/modules/web-console/frontend/app/components/grid-export/template.pug
index 99780ee8461e3..fac10df42a839 100644
--- a/modules/web-console/frontend/app/components/grid-export/template.pug
+++ b/modules/web-console/frontend/app/components/grid-export/template.pug
@@ -14,5 +14,5 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-button.btn-ignite.btn-ignite--primary-outline(ng-click='$ctrl.export()' bs-tooltip='' data-title='Export filtered rows to CSV' data-placement='top')
-    svg(ignite-icon='csv')
+button.btn-ignite.btn-ignite--link-dashed-secondary(ng-click='$ctrl.export()' bs-tooltip='' data-title='Export filtered rows to CSV' data-placement='top')
+    svg(ignite-icon='download')
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.directive.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.directive.js
index b38cb7af1055f..e1664027331d2 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.directive.js
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.directive.js
@@ -70,7 +70,8 @@ export default function listEditableCols() {
         },
         bindToController: {
             colDefs: '<listEditableCols',
-            rowClass: '@?listEditableColsRowClass'
+            rowClass: '@?listEditableColsRowClass',
+            ngDisabled: '<?'
         }
     };
 }
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.style.scss b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.style.scss
index 12c9ba65b8c3b..5df29cc540b4e 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.style.scss
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.style.scss
@@ -21,6 +21,7 @@
 
     margin-left: 10px;
     margin-right: $remove-column-width;
+    transition: 0.2s opacity;
 
     &__multiple-cols {
         margin-left: $index-column-width;
@@ -33,7 +34,7 @@
     }
 
     [ignite-icon='info'] {
-        @import 'public/stylesheets/variables';
+        @import '../../../../../public/stylesheets/variables';
 
         margin-left: 5px;
         color: $ignite-brand-success;
@@ -43,9 +44,14 @@
         .ignite-form-field__label {
             display: none;
         }
-        
+
         .le-row-item-view:nth-last-child(2) {
             display: none;
         }
     }
+
+    &[disabled] {
+        opacity: 0.5;
+        cursor: default;
+    }
 }
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug
index 6541c92e86434..d060f45802e40 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug
@@ -16,6 +16,7 @@
 
 .list-editable-cols__header(
     ng-class='::[$ctrl.rowClass, {"list-editable-cols__header__multiple-cols": $ctrl.colDefs.length > 1}]'
+    ng-disabled='$ctrl.ngDisabled'
 )
     .list-editable-cols__header-cell(ng-repeat='col in ::$ctrl.colDefs' ng-class='::col.cellClass')
         span.ignite-form-field__label
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/directives.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/directives.js
index 4b6e785db6a2a..f408647926c89 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/directives.js
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/directives.js
@@ -30,7 +30,9 @@ export function ngModel() {
          * @param {ng.INgModelController} ngModel
          */
         link(scope, el, attr, {ngModel, list}) {
-            if (!list) return;
+            if (!list)
+                return;
+
             ngModel.$viewChangeListeners.push(() => {
                 el[0].dispatchEvent(new CustomEvent(CUSTOM_EVENT_TYPE, {bubbles: true, cancelable: true}));
             });
@@ -54,15 +56,21 @@ export function listEditableTransclude() {
          * @param {ListEditableController} list
          */
         link(scope, el, attr, {list, transclude}) {
-            if (attr.listEditableTransclude !== 'itemEdit') return;
-            if (!list) return;
+            if (attr.listEditableTransclude !== 'itemEdit')
+                return;
+
+            if (!list)
+                return;
+
             let listener = (e) => {
                 e.stopPropagation();
                 scope.$evalAsync(() => {
                     if (scope.form.$valid) list.save(scope.item, transclude.$index);
                 });
             };
+
             el[0].addEventListener(CUSTOM_EVENT_TYPE, listener);
+
             scope.$on('$destroy', () => {
                 el[0].removeEventListener(CUSTOM_EVENT_TYPE, listener);
                 listener = null;
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/directive.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/directive.js
index 6391eb1125564..36750aea5be40 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/directive.js
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/directive.js
@@ -24,7 +24,7 @@ import {default as ListEditable} from '../../controller';
  * User can provide an alias for $item by setting item-name attribute on transclusion slot element.
  */
 export class ListEditableTransclude {
-    /** 
+    /**
      * Transcluded slot name.
      *
      * @type {string}
@@ -63,7 +63,9 @@ export class ListEditableTransclude {
                 [itemName]: {
                     get: () => {
                         // Scope might get destroyed
-                        if (!this.$scope) return;
+                        if (!this.$scope)
+                            return;
+
                         return this.$scope.item;
                     },
                     set: (value) => {
@@ -80,7 +82,9 @@ export class ListEditableTransclude {
                 $form: {
                     get: () => {
                         // Scope might get destroyed
-                        if (!this.$scope) return;
+                        if (!this.$scope)
+                            return;
+
                         return this.$scope.form;
                     },
                     // Allows to delete property later
@@ -98,7 +102,9 @@ export class ListEditableTransclude {
      * @returns {number}
      */
     get $index() {
-        if (!this.$scope) return;
+        if (!this.$scope)
+            return;
+
         return this.$scope.$index;
     }
 
diff --git a/modules/web-console/frontend/app/components/list-editable/controller.js b/modules/web-console/frontend/app/components/list-editable/controller.js
index 586743ec41a64..d1b890044bdf3 100644
--- a/modules/web-console/frontend/app/components/list-editable/controller.js
+++ b/modules/web-console/frontend/app/components/list-editable/controller.js
@@ -102,10 +102,12 @@ export default class {
         // ng-model-option, then it will always save. Be careful and pay extra attention to validation
         // when doing so, it's an easy way to miss invalid values this way.
 
-        // Dont close if form is invalid and allowInvalid is turned off (which is default value)
-        if (!form.$valid && !this.ngModel.$options.getOption('allowInvalid')) return;
+        // Don't close if form is invalid and allowInvalid is turned off (which is default value)
+        if (!form.$valid && !this.ngModel.$options.getOption('allowInvalid'))
+            return;
 
         delete this._cache[idx];
+
         this.save(data, idx);
     }
 }
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js
index f53de699bfed0..419a0636f23aa 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js
@@ -17,7 +17,7 @@
 
 const ICON_SORT = '<span ui-grid-one-bind-id-grid="col.uid + \'-sortdir-text\'" ui-grid-visible="col.sort.direction" aria-label="Sort Descending"><i ng-class="{ \'ui-grid-icon-up-dir\': col.sort.direction == asc, \'ui-grid-icon-down-dir\': col.sort.direction == desc, \'ui-grid-icon-blank\': !col.sort.direction }" title="" aria-hidden="true"></i></span>';
 
-const USER_TEMPLATE = '<div class="ui-grid-cell-contents">' +
+const USER_TEMPLATE = '<div class="ui-grid-cell-contents user-cell">' +
     '<i class="pull-left" ng-class="row.entity.admin ? \'icon-admin\' : \'icon-user\'"></i>&nbsp;<label bs-tooltip data-title="{{ COL_FIELD }}">{{ COL_FIELD }}</label></div>';
 
 const CLUSTER_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='icon-cluster'></i>${ICON_SORT}</div>`;
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/style.scss b/modules/web-console/frontend/app/components/list-of-registered-users/style.scss
index b0a4c4d4d8376..5a0c71310de34 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/style.scss
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/style.scss
@@ -18,6 +18,17 @@
 ignite-list-of-registered-users {
     display: block;
 
+    .user-cell {
+        display: flex;
+        overflow-wrap: normal;
+        white-space: normal;
+
+        label {
+            overflow: hidden;
+            text-overflow: ellipsis;
+        }
+    }
+
     .ui-grid-settings--heading {
         display: flex;
     }
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug b/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug
index 98185f3bcb3cc..6656c90a01144 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug
@@ -34,7 +34,7 @@ ul.tabs.tabs--blue
             span.badge.badge--blue {{ $ctrl.countries.length }}
 
 .panel--ignite
-    .panel-heading.ui-grid-settings
+    .panel-heading.ui-grid-settings.ui-grid-ignite__panel
         .panel-title
             +ignite-form-field-bsdropdown({
                 label: 'Actions',
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/controller.js
index f1bba1f61a079..e044a7012d589 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/controller.js
@@ -28,6 +28,8 @@ export default class CacheEditFormController {
      * @type {Array<string>}
      */
     igfsIDs;
+    /** @type {ng.ICompiledExpression} */
+    onSave;
 
     static $inject = ['IgniteConfirm', 'IgniteVersion', '$scope', 'Caches', 'IgniteFormUtils'];
     constructor(IgniteConfirm, IgniteVersion, $scope, Caches, IgniteFormUtils) {
@@ -66,6 +68,11 @@ export default class CacheEditFormController {
 
         // TODO: Do we really need this?
         this.$scope.ui = this.IgniteFormUtils.formUI();
+
+        this.formActions = [
+            {text: 'Save', icon: 'checkmark', click: () => this.save()},
+            {text: 'Save and Download', icon: 'download', click: () => this.save(true)}
+        ];
     }
     $onDestroy() {
         this.subscription.unsubscribe();
@@ -90,10 +97,10 @@ export default class CacheEditFormController {
     getValuesToCompare() {
         return [this.cache, this.clonedCache].map(this.Caches.normalize);
     }
-    save() {
+    save(download) {
         if (this.$scope.ui.inputForm.$invalid)
             return this.IgniteFormUtils.triggerValidation(this.$scope.ui.inputForm, this.$scope);
-        this.onSave({$event: cloneDeep(this.clonedCache)});
+        this.onSave({$event: {cache: cloneDeep(this.clonedCache), download}});
     }
     reset = (forReal) => forReal ? this.clonedCache = cloneDeep(this.cache) : void 0;
     confirmAndReset() {
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/template.tpl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/template.tpl.pug
index a8ae2f2a93e2e..7305a39290219 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/template.tpl.pug
@@ -18,7 +18,6 @@ form(
     name='ui.inputForm'
     id='cache'
     novalidate
-    ng-submit='$ctrl.save($ctrl.clonedCache)'
 )
     include ./templates/general
     include ./templates/memory
@@ -40,7 +39,4 @@ form(
         ng-click='$ctrl.confirmAndReset()'
     )
         | Cancel
-    button.btn-ignite.btn-ignite--success(
-        form='cache'
-        type='submit'
-    ) Save
+    pc-split-button(actions=`::$ctrl.formActions`)
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
index 0207729821379..881e5430378b7 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/controller.js
@@ -25,10 +25,16 @@ export default class ClusterEditFormController {
     caches;
     /** @type {ig.menu<string>} */
     cachesMenu;
+    /** @type {ng.ICompiledExpression} */
+    onSave;
 
     static $inject = ['IgniteLegacyUtils', 'IgniteEventGroups', 'IgniteConfirm', 'IgniteVersion', '$scope', 'Clusters', 'IgniteFormUtils'];
+    /**
+     * @param {import('app/services/Clusters').default} Clusters
+     */
     constructor(IgniteLegacyUtils, IgniteEventGroups, IgniteConfirm, IgniteVersion, $scope, Clusters, IgniteFormUtils) {
-        Object.assign(this, {IgniteLegacyUtils, IgniteEventGroups, IgniteConfirm, IgniteVersion, $scope, Clusters, IgniteFormUtils});
+        Object.assign(this, {IgniteLegacyUtils, IgniteEventGroups, IgniteConfirm, IgniteVersion, $scope, IgniteFormUtils});
+        this.Clusters = Clusters;
     }
 
     $onDestroy() {
@@ -38,8 +44,6 @@ export default class ClusterEditFormController {
     $onInit() {
         this.available = this.IgniteVersion.available.bind(this.IgniteVersion);
 
-        let __original_value;
-
         const rebuildDropdowns = () => {
             this.eventStorage = [
                 {value: 'Memory', label: 'Memory'},
@@ -89,6 +93,11 @@ export default class ClusterEditFormController {
 
         this.$scope.ui = this.IgniteFormUtils.formUI();
         this.$scope.ui.loadedPanels = ['checkpoint', 'serviceConfiguration', 'odbcConfiguration'];
+
+        this.formActions = [
+            {text: 'Save', icon: 'checkmark', click: () => this.save()},
+            {text: 'Save and Download', icon: 'download', click: () => this.save(true)}
+        ];
     }
 
     $onChanges(changes) {
@@ -120,10 +129,10 @@ export default class ClusterEditFormController {
         return [this.cluster, this.clonedCluster].map(this.Clusters.normalize);
     }
 
-    save() {
+    save(download) {
         if (this.$scope.ui.inputForm.$invalid)
             return this.IgniteFormUtils.triggerValidation(this.$scope.ui.inputForm, this.$scope);
-        this.onSave({$event: cloneDeep(this.clonedCluster)});
+        this.onSave({$event: {cluster: cloneDeep(this.clonedCluster), download}});
     }
 
     reset = () => this.clonedCluster = cloneDeep(this.cluster);
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug
index c2bfd68a57668..d5cb90930f8c2 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/template.tpl.pug
@@ -16,7 +16,7 @@
 
 include /app/helpers/jade/mixins
 
-form(id='cluster' name='ui.inputForm' novalidate ng-submit='$ctrl.save()')
+form(id='cluster' name='ui.inputForm' novalidate)
     .panel-group
         include ./templates/general
 
@@ -72,7 +72,6 @@ form(id='cluster' name='ui.inputForm' novalidate ng-submit='$ctrl.save()')
 
 .pc-form-actions-panel(n_g-show='$ctrl.$scope.selectedItem')
     button-preview-project(cluster='$ctrl.cluster' ng-hide='$ctrl.isNew')
-    button-download-project(cluster='$ctrl.cluster' ng-hide='$ctrl.isNew')
 
     .pc-form-actions-panel__right-after
 
@@ -81,7 +80,4 @@ form(id='cluster' name='ui.inputForm' novalidate ng-submit='$ctrl.save()')
         ng-click='$ctrl.confirmAndReset()'
     )
         | Cancel
-    button.btn-ignite.btn-ignite--success(
-        form='cluster'
-        type='submit'
-    ) Save
\ No newline at end of file
+    pc-split-button(actions=`::$ctrl.formActions`)
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/binary.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/binary.pug
index 6c1b2465f9570..17fe4cd66f03c 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/binary.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/binary.pug
@@ -36,11 +36,11 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                 +java-class('Serializer:', model + '.serializer', '"serializer"', 'true', 'false', 'Class with custom serialization logic for binary objects')
             .pc-form-grid-col-60
                 .ignite-form-field
-                    +ignite-form-field__label('Type configurations:', '"typeСonfigurations"')
+                    +ignite-form-field__label('Type configurations:', '"typeConfigurations"')
                         +tooltip(`Configuration properties for binary types`)
                     .ignite-form-field__control
                         -var items = model + '.typeConfigurations'
-                        list-editable(ng-model=items name='typeСonfigurations')
+                        list-editable.pc-list-editable-with-form-grid(ng-model=items name='typeConfigurations')
                             list-editable-item-edit.pc-form-grid-row
                                 - form = '$parent.form'
                                 .pc-form-grid-col-60
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/cache-key-cfg.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/cache-key-cfg.pug
index abc8ff10c1fe0..0b34ce44f5252 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/cache-key-cfg.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/cache-key-cfg.pug
@@ -30,7 +30,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     +ignite-form-field__label('Cache key configuration:', '"cacheKeyConfiguration"')
                     .ignite-form-field__control
                         -let items = model
-                        list-editable(ng-model=items name='cacheKeyConfiguration')
+                        list-editable.pc-list-editable-with-form-grid(ng-model=items name='cacheKeyConfiguration')
                             list-editable-item-edit.pc-form-grid-row
                                 - form = '$parent.form'
                                 .pc-form-grid-col-60
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint.pug
index b00c98cba705d..7d56f14e7b209 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint.pug
@@ -32,7 +32,7 @@ panel-collapsible(ng-form=form)
                 .ignite-form-field
                     +ignite-form-field__label('Checkpoint SPI configurations:', '"checkpointSPIConfigurations"')
                     .ignite-form-field__control
-                        list-editable(ng-model=model name='checkpointSPIConfigurations')
+                        list-editable.pc-list-editable-with-form-grid(ng-model=model name='checkpointSPIConfigurations')
                             list-editable-item-edit(item-name='$checkpointSPI').pc-form-grid-row
                                 .pc-form-grid-col-60
                                     +dropdown-required('Checkpoint SPI:', '$checkpointSPI.kind', '"checkpointKind"', 'true', 'true', 'Choose checkpoint configuration variant', '[\
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/client-connector.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/client-connector.pug
index d27fa14d6aacf..620137b9d923c 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/client-connector.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/client-connector.pug
@@ -29,17 +29,17 @@ panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.lo
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
                 +checkbox('Enabled', connectionEnabled, '"ClientConnectorEnabled"', 'Flag indicating whether to configure client connector configuration')
-            .pc-form-grid-col-40
+            .pc-form-grid-col-60
                 +text-enabled('Host:', `${connectionModel}.host`, '"ClientConnectorHost"', connectionEnabled, 'false', 'localhost')
-            .pc-form-grid-col-20
+            .pc-form-grid-col-30
                 +number('Port:', `${connectionModel}.port`, '"ClientConnectorPort"', connectionEnabled, '10800', '1025')
-            .pc-form-grid-col-20
+            .pc-form-grid-col-30
                 +number('Port range:', `${connectionModel}.portRange`, '"ClientConnectorPortRange"', connectionEnabled, '100', '0')
-            .pc-form-grid-col-20
+            .pc-form-grid-col-30
                 +number('Socket send buffer size:', `${connectionModel}.socketSendBufferSize`, '"ClientConnectorSocketSendBufferSize"', connectionEnabled, '0', '0',
                     'Socket send buffer size<br/>\
                     When set to <b>0</b>, operation system default will be used')
-            .pc-form-grid-col-20
+            .pc-form-grid-col-30
                 +number('Socket receive buffer size:', `${connectionModel}.socketReceiveBufferSize`, '"ClientConnectorSocketReceiveBufferSize"', connectionEnabled, '0', '0',
                     'Socket receive buffer size<br/>\
                     When set to <b>0</b>, operation system default will be used')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug
index c9fea569a4f5b..ea27c3c632bdb 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug
@@ -59,25 +59,25 @@ mixin data-region-form({modelAt, namePlaceholder, dataRegionsAt})
             min=`{{ $ctrl.Clusters.dataRegion.maxSize.min(${modelAt}) }}`
         )
 
-    .pc-form-grid-col-60
-        +text('Swap file path:', `${modelAt}.swapFilePath`, '"swapFilePath"', 'false', 'Input swap file path', 'An optional path to a memory mapped file for this data region')
+    .pc-form-grid-col-60(ng-if=`!${modelAt}.persistenceEnabled || ${modelAt}.swapPath`)
+        +text('Swap file path:', `${modelAt}.swapPath`, '"swapPath"', 'false', 'Input swap file path', 'An optional path to a memory mapped file for this data region')
         
     .pc-form-grid-col-60
         +number('Checkpoint page buffer:', `${modelAt}.checkpointPageBufferSize`, '"checkpointPageBufferSize"', 'true', '0', '0', 'Amount of memory allocated for a checkpoint temporary buffer in bytes')
 
     .pc-form-grid-col-60
         +dropdown('Eviction mode:', `${modelAt}.pageEvictionMode`, '"pageEvictionMode"', 'true', 'DISABLED',
-        '[\
-            {value: "DISABLED", label: "DISABLED"},\
-            {value: "RANDOM_LRU", label: "RANDOM_LRU"},\
-            {value: "RANDOM_2_LRU", label: "RANDOM_2_LRU"}\
-        ]',
-        `An algorithm for memory pages eviction
-        <ul>
-            <li>DISABLED - Eviction is disabled</li>
-            <li>RANDOM_LRU - Once a memory region defined by a data region is configured, an off-heap array is allocated to track last usage timestamp for every individual data page</li>
-            <li>RANDOM_2_LRU - Differs from Random - LRU only in a way that two latest access timestamps are stored for every data page</li>
-        </ul>`)
+            '[\
+                {value: "DISABLED", label: "DISABLED"},\
+                {value: "RANDOM_LRU", label: "RANDOM_LRU"},\
+                {value: "RANDOM_2_LRU", label: "RANDOM_2_LRU"}\
+            ]',
+            `An algorithm for memory pages eviction
+            <ul>
+                <li>DISABLED - Eviction is disabled</li>
+                <li>RANDOM_LRU - Once a memory region defined by a data region is configured, an off-heap array is allocated to track last usage timestamp for every individual data page</li>
+                <li>RANDOM_2_LRU - Differs from Random - LRU only in a way that two latest access timestamps are stored for every data page</li>
+            </ul>`)
 
     .pc-form-grid-col-30
         +sane-ignite-form-field-number({
@@ -105,35 +105,35 @@ mixin data-region-form({modelAt, namePlaceholder, dataRegionsAt})
     .pc-form-grid-col-30
         +sane-ignite-form-field-number({
             label: 'Metrics sub interval count:',
-            model: `${modelAt}.subIntervals`,
-            name: '"subIntervals"',
-            placeholder: '{{ ::$ctrl.Clusters.dataRegion.subIntervals.default }}',
-            min: '{{ ::$ctrl.Clusters.dataRegion.subIntervals.min }}',
-            step: '{{ ::$ctrl.Clusters.dataRegion.subIntervals.step }}',
+            model: `${modelAt}.metricsSubIntervalCount`,
+            name: '"metricsSubIntervalCount"',
+            placeholder: '{{ ::$ctrl.Clusters.dataRegion.metricsSubIntervalCount.default }}',
+            min: '{{ ::$ctrl.Clusters.dataRegion.metricsSubIntervalCount.min }}',
+            step: '{{ ::$ctrl.Clusters.dataRegion.metricsSubIntervalCount.step }}',
             tip: 'A number of sub-intervals the whole rate time interval will be split into to calculate allocation and eviction rates'
         })
 
     .pc-form-grid-col-30
         pc-form-field-size(
-            ng-model=`${modelAt}.rateTimeInterval`
+            ng-model=`${modelAt}.metricsRateTimeInterval`
             ng-model-options='{allowInvalid: true}'
-            name='rateTimeInterval'
+            name='metricsRateTimeInterval'
             size-type='seconds'
             label='Metrics rate time interval:'
-            placeholder='{{ $ctrl.Clusters.dataRegion.rateTimeInterval.default / _rateTimeIntervalScale.value }}'
-            min=`{{ ::$ctrl.Clusters.dataRegion.rateTimeInterval.min }}`
+            placeholder='{{ $ctrl.Clusters.dataRegion.metricsRateTimeInterval.default / _metricsRateTimeIntervalScale.value }}'
+            min=`{{ ::$ctrl.Clusters.dataRegion.metricsRateTimeInterval.min }}`
             tip='Time interval for allocation rate and eviction rate monitoring purposes'
-            on-scale-change='_rateTimeIntervalScale = $event'
+            on-scale-change='_metricsRateTimeIntervalScale = $event'
             size-scale-label='s'
         )
             
     .pc-form-grid-col-60
         +checkbox('Metrics enabled', `${modelAt}.metricsEnabled`, '"MemoryPolicyMetricsEnabled"',
-        'Whether memory metrics are enabled by default on node startup')
+            'Whether memory metrics are enabled by default on node startup')
 
-    .pc-form-grid-col-60
+    .pc-form-grid-col-60(ng-if=`!${modelAt}.swapPath`)
         +checkbox('Persistence enabled', `${modelAt}.persistenceEnabled`, '"RegionPersistenceEnabled" + $index',
-        'Enable Ignite Native Persistence')
+            'Enable Ignite Native Persistence')
 
 panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Data storage configuration
@@ -152,7 +152,7 @@ panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.lo
                 })
             .pc-form-grid-col-30
                 +number('Concurrency level:', model + '.concurrencyLevel', '"DataStorageConfigurationConcurrencyLevel"',
-                'true', 'availableProcessors', '2', 'The number of concurrent segments in Ignite internal page mapping tables')
+                    'true', 'availableProcessors', '2', 'The number of concurrent segments in Ignite internal page mapping tables')
             .pc-form-grid-col-60.pc-form-group__text-title
                 span System region
             .pc-form-group.pc-form-grid-row
@@ -188,7 +188,10 @@ panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.lo
                 .ignite-form-field
                     .ignite-form-field__label Data region configurations
                     .ignite-form-field__control
-                        list-editable(name='dataRegionConfigurations' ng-model=dataRegionConfigurations)
+                        list-editable.pc-list-editable-with-form-grid(
+                            name='dataRegionConfigurations'
+                            ng-model=dataRegionConfigurations
+                        )
                             list-editable-item-edit.pc-form-grid-row
                                 - form = '$parent.form'
                                 +data-region-form({
@@ -206,38 +209,38 @@ panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.lo
 
             .pc-form-grid-col-60
                 +text-enabled('Storage path:', `${model}.storagePath`, '"DataStoragePath"', 'true', 'false', 'db',
-                'Directory where index and partition files are stored')
+                    'Directory where index and partition files are stored')
             .pc-form-grid-col-60
                 +number('Checkpoint frequency:', `${model}.checkpointFrequency`, '"DataStorageCheckpointFrequency"', 'true', '180000', '1',
-                'Frequency which is a minimal interval when the dirty pages will be written to the Persistent Store')
+                    'Frequency which is a minimal interval when the dirty pages will be written to the Persistent Store')
             .pc-form-grid-col-20
                 +number('Checkpoint threads:', `${model}.checkpointThreads`, '"DataStorageCheckpointThreads"', 'true', '4', '1', 'A number of threads to use for the checkpoint purposes')
             .pc-form-grid-col-20
                 +dropdown('Checkpoint write order:', `${model}.checkpointWriteOrder`, '"DataStorageCheckpointWriteOrder"', 'true', 'SEQUENTIAL',
-                '[\
-                    {value: "RANDOM", label: "RANDOM"},\
-                    {value: "SEQUENTIAL", label: "SEQUENTIAL"}\
-                ]',
-                'Order of writing pages to disk storage during checkpoint.\
-                <ul>\
-                    <li>RANDOM - Pages are written in order provided by checkpoint pages collection iterator</li>\
-                    <li>SEQUENTIAL - All checkpoint pages are collected into single list and sorted by page index</li>\
-                </ul>')
+                    '[\
+                        {value: "RANDOM", label: "RANDOM"},\
+                        {value: "SEQUENTIAL", label: "SEQUENTIAL"}\
+                    ]',
+                    'Order of writing pages to disk storage during checkpoint.\
+                    <ul>\
+                        <li>RANDOM - Pages are written in order provided by checkpoint pages collection iterator</li>\
+                        <li>SEQUENTIAL - All checkpoint pages are collected into single list and sorted by page index</li>\
+                    </ul>')
             .pc-form-grid-col-20
                 +dropdown('WAL mode:', `${model}.walMode`, '"DataStorageWalMode"', 'true', 'DEFAULT',
-                '[\
-                    {value: "DEFAULT", label: "DEFAULT"},\
-                    {value: "LOG_ONLY", label: "LOG_ONLY"},\
-                    {value: "BACKGROUND", label: "BACKGROUND"},\
-                    {value: "NONE", label: "NONE"}\
-                ]',
-                'Type define behavior wal fsync.\
-                <ul>\
-                    <li>DEFAULT - full-sync disk writes</li>\
-                    <li>LOG_ONLY - flushes application buffers</li>\
-                    <li>BACKGROUND - does not force application&#39;s buffer flush</li>\
-                    <li>NONE - WAL is disabled</li>\
-                </ul>')
+                    '[\
+                        {value: "DEFAULT", label: "DEFAULT"},\
+                        {value: "LOG_ONLY", label: "LOG_ONLY"},\
+                        {value: "BACKGROUND", label: "BACKGROUND"},\
+                        {value: "NONE", label: "NONE"}\
+                    ]',
+                    'Type define behavior wal fsync.\
+                    <ul>\
+                        <li>DEFAULT - full-sync disk writes</li>\
+                        <li>LOG_ONLY - flushes application buffers</li>\
+                        <li>BACKGROUND - does not force application&#39;s buffer flush</li>\
+                        <li>NONE - WAL is disabled</li>\
+                    </ul>')
             .pc-form-grid-col-60
                 +text-enabled('WAL path:', `${model}.walPath`, '"DataStorageWalPath"', 'true', 'false', 'db/wal', 'A path to the directory where WAL is stored')
             .pc-form-grid-col-60
@@ -250,52 +253,52 @@ panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.lo
                 +number('WAL history size:', `${model}.walHistorySize`, '"DataStorageWalHistorySize"', 'true', '20', '1', 'A total number of checkpoints to keep in the WAL history')
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.4.0")')
                 +number('WAL buffer size:', `${model}.walBufferSize`, '"DataStorageWalBufferSize"', 'true', 'WAL segment size / 4', '1',
-                'Size of WAL buffer')
+                    'Size of WAL buffer')
             .pc-form-grid-col-30
                 +number('WAL flush frequency:', `${model}.walFlushFrequency`, '"DataStorageWalFlushFrequency"', 'true', '2000', '1',
-                'How often will be fsync, in milliseconds. In background mode, exist thread which do fsync by timeout')
+                    'How often will be fsync, in milliseconds. In background mode, exist thread which do fsync by timeout')
             .pc-form-grid-col-30
                 +number('WAL fsync delay:', `${model}.walFsyncDelayNanos`, '"DataStorageWalFsyncDelay"', 'true', '1000', '1', 'WAL fsync delay, in nanoseconds')
             .pc-form-grid-col-60
                 +number('WAL record iterator buffer size:', `${model}.walRecordIteratorBufferSize`, '"DataStorageWalRecordIteratorBufferSize"', 'true', '67108864', '1',
-                'How many bytes iterator read from disk(for one reading), during go ahead WAL')
+                    'How many bytes iterator read from disk(for one reading), during go ahead WAL')
             .pc-form-grid-col-30
                 +number('Lock wait time:', `${model}.lockWaitTime`, '"DataStorageLockWaitTime"', 'true', '10000', '1',
-                'Time out in milliseconds, while wait and try get file lock for start persist manager')
+                    'Time out in milliseconds, while wait and try get file lock for start persist manager')
             .pc-form-grid-col-30
                 +number('WAL thread local buffer size:', `${model}.walThreadLocalBufferSize`, '"DataStorageWalThreadLocalBufferSize"', 'true', '131072', '1',
-                'Define size thread local buffer. Each thread which write to WAL have thread local buffer for serialize recode before write in WAL')
+                    'Define size thread local buffer. Each thread which write to WAL have thread local buffer for serialize recode before write in WAL')
             .pc-form-grid-col-30
                 +number('Metrics sub interval count:', `${model}.metricsSubIntervalCount`, '"DataStorageMetricsSubIntervalCount"', 'true', '5', '1',
-                'Number of sub - intervals the whole rate time interval will be split into to calculate rate - based metrics')
+                    'Number of sub - intervals the whole rate time interval will be split into to calculate rate - based metrics')
             .pc-form-grid-col-30
                 +number('Metrics rate time interval:', `${model}.metricsRateTimeInterval`, '"DataStorageMetricsRateTimeInterval"', 'true', '60000', '1000',
-                'The length of the time interval for rate - based metrics. This interval defines a window over which hits will be tracked')
+                    'The length of the time interval for rate - based metrics. This interval defines a window over which hits will be tracked')
             .pc-form-grid-col-30
                 +dropdown('File IO factory:', `${model}.fileIOFactory`, '"DataStorageFileIOFactory"', 'true', 'Default',
-                '[\
-                    {value: "RANDOM", label: "RANDOM"},\
-                    {value: "ASYNC", label: "ASYNC"},\
-                    {value: null, label: "Default"},\
-                ]',
-                'Order of writing pages to disk storage during checkpoint.\
-                <ul>\
-                    <li>RANDOM - Pages are written in order provided by checkpoint pages collection iterator</li>\
-                    <li>SEQUENTIAL - All checkpoint pages are collected into single list and sorted by page index</li>\
-                </ul>')
+                    '[\
+                        {value: "RANDOM", label: "RANDOM"},\
+                        {value: "ASYNC", label: "ASYNC"},\
+                        {value: null, label: "Default"},\
+                    ]',
+                    'Order of writing pages to disk storage during checkpoint.\
+                    <ul>\
+                        <li>RANDOM - Pages are written in order provided by checkpoint pages collection iterator</li>\
+                        <li>SEQUENTIAL - All checkpoint pages are collected into single list and sorted by page index</li>\
+                    </ul>')
             .pc-form-grid-col-30
                 +number('WAL auto archive after inactivity:', `${model}.walAutoArchiveAfterInactivity`, '"DataStorageWalAutoArchiveAfterInactivity"', 'true', '-1', '-1',
-                'Time in millis to run auto archiving segment after last record logging')
+                    'Time in millis to run auto archiving segment after last record logging')
             .pc-form-grid-col-60
                 +checkbox-enabled('Metrics enabled', `${model}.metricsEnabled`, '"DataStorageMetricsEnabled"', 'true', 'Flag indicating whether persistence metrics collection is enabled')
             .pc-form-grid-col-60
                 +checkbox-enabled('Always write full pages', `${model}.alwaysWriteFullPages`, '"DataStorageAlwaysWriteFullPages"', 'true', 'Flag indicating whether always write full pages')
             .pc-form-grid-col-60
                 +checkbox('Write throttling enabled', `${model}.writeThrottlingEnabled`, '"DataStorageWriteThrottlingEnabled"',
-                'Throttle threads that generate dirty pages too fast during ongoing checkpoint')
+                    'Throttle threads that generate dirty pages too fast during ongoing checkpoint')
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.4.0")')
                 +checkbox('Enable WAL compaction', `${model}.walCompactionEnabled`, '"DataStorageWalCompactionEnabled"',
-                'If true, system filters and compresses WAL archive in background')
+                    'If true, system filters and compresses WAL archive in background')
 
         .pca-form-column-6
             +preview-xml-java(model, 'clusterDataStorageConfiguration')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/discovery.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/discovery.pug
index 35bd5e81d5254..d0a9102bbc1c6 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/discovery.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/discovery.pug
@@ -22,8 +22,8 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Discovery
     panel-description
-        | TCP/IP discovery configuration. 
-        | #[a.link-success(href="https://apacheignite.readme.io/docs/cluster-config" target="_blank") More info]
+        | TCP/IP discovery configuration.
+        | #[a.link-success(href="https://apacheignite.readme.io/docs/cluster-discovery" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-20
@@ -63,7 +63,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                 +number('Thread priority:', `${model}.threadPriority`, '"threadPriority"', 'true', '10', '1', 'Thread priority for all threads started by SPI')
 
             //- Removed in ignite 2.0
-            .pc-form-grid-col-60(ng-if='$ctrl.available(["1.0.0", "2.0.0"])')
+            .pc-form-grid-col-60(ng-if-start='$ctrl.available(["1.0.0", "2.0.0"])')
                 +number('Heartbeat frequency:', `${model}.heartbeatFrequency`, '"heartbeatFrequency"', 'true', '2000', '1', 'Heartbeat messages issuing frequency')
             .pc-form-grid-col-30
                 +number('Max heartbeats miss w/o init:', `${model}.maxMissedHeartbeats`, '"maxMissedHeartbeats"', 'true', '1', '1',
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug
index 2e92f83caefbc..85c441e8fd37d 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug
@@ -36,7 +36,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                 +number('Client failure detection timeout:', model + '.clientFailureDetectionTimeout', '"clientFailureDetectionTimeout"', 'true',
                     '30000', '1', 'Failure detection timeout is used to determine how long the communication or discovery SPIs should wait before considering a remote connection failed')
 
-            .pc-form-grid-col-60(ng-init='failoverSpiTbl={type: "failoverSpi", model: "failoverSpi", focusId: "kind", ui: "failover-table"}')
+            .pc-form-grid-col-60
                 mixin clusters-failover-spi
                     .ignite-form-field
                         +ignite-form-field__label('Failover SPI configurations:', '"failoverSpi"')
@@ -44,10 +44,9 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                         .ignite-form-field__control
                             -let items = failoverSpi
 
-                            list-editable(ng-model=items name='failoverSpi')
-                                list-editable-item-edit
-                                    - form = '$parent.form'
-                                    .settings-row
+                            list-editable.pc-list-editable-with-form-grid(ng-model=items name='failoverSpi')
+                                list-editable-item-edit.pc-form-grid-row
+                                    .pc-form-grid-col-60
                                         +sane-ignite-form-field-dropdown({
                                             required: true,
                                             label: 'Failover SPI:',
@@ -66,13 +65,13 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                                             </ul>`
                                         })
 
-                                    .settings-row(ng-show='$item.kind === "JobStealing"')
+                                    .pc-form-grid-col-60(ng-show='$item.kind === "JobStealing"')
                                         +number('Maximum failover attempts:', '$item.JobStealing.maximumFailoverAttempts', '"jsMaximumFailoverAttempts"', 'true', '5', '0',
                                             'Maximum number of attempts to execute a failed job on another node')
-                                    .settings-row(ng-show='$item.kind === "Always"')
+                                    .pc-form-grid-col-60(ng-show='$item.kind === "Always"')
                                         +number('Maximum failover attempts:', '$item.Always.maximumFailoverAttempts', '"alwaysMaximumFailoverAttempts"', 'true', '5', '0',
                                             'Maximum number of attempts to execute a failed job on another node')
-                                    .settings-row(ng-show=failoverCustom)
+                                    .pc-form-grid-col-60(ng-show=failoverCustom)
                                         +java-class('SPI implementation', '$item.Custom.class', '"failoverSpiClass"', 'true', failoverCustom,
                                             'Custom FailoverSpi implementation class name.', failoverCustom)
 
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast.pug
index 639a374c75960..2d7aa4bd6b578 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast.pug
@@ -32,7 +32,7 @@ mixin discovery-multicast(modelAt = '$ctrl.clonedCluster')
             +number('Attempts count:', `${model}.addressRequestAttempts`, '"addressRequestAttempts"', 'true', '2', '0',
                 'Number of attempts to send multicast address request<br/>\
                 IP finder re - sends request only in case if no reply for previous request is received')
-        .pc-form-grid-col-20
+        .pc-form-grid-col-20.pc-form-grid-col-free
             +text-ip-address('Local address:', `${model}.localAddress`, '"localAddress"', 'true', '0.0.0.0',
                 'Local host address used by this IP finder<br/>\
                 If provided address is non - loopback then multicast socket is bound to this interface<br/>\
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/igfs.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/igfs.pug
deleted file mode 100644
index c1216a27e7f1b..0000000000000
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/igfs.pug
+++ /dev/null
@@ -1,34 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-include /app/helpers/jade/mixins
-
--var form = 'igfs'
--var model = '$ctrl.clonedCluster'
-
-panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
-    panel-title IGFS
-    panel-description 
-        | IGFS (Ignite In-Memory File System) configurations assigned to cluster. 
-        | #[a.link-success(href="https://apacheignite-fs.readme.io/docs/in-memory-file-system" target="_blank") More info]
-    panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
-        .pca-form-column-6
-            .settings-row
-                +dropdown-multiple('<span>IGFS:</span><a ui-sref="base.configuration.edit.advanced.igfs({linkId: linkId()})"> (add)</a>',
-                    `${model}.igfss`, '"igfss"', true, 'Choose IGFS', 'No IGFS configured', 'igfss',
-                    'Select IGFS to start in cluster or add a new IGFS')
-        .pca-form-column-6
-            +preview-xml-java(model, 'igfss', 'igfss')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/load-balancing.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/load-balancing.pug
index ff817e198b9b7..20ea1f0ed85fe 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/load-balancing.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/load-balancing.pug
@@ -36,7 +36,10 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     .ignite-form-field__control
                         -let items = loadBalancingSpi
 
-                        list-editable(ng-model=items name='loadBalancingConfigurations')
+                        list-editable.pc-list-editable-with-legacy-settings-rows(
+                            ng-model=items
+                            name='loadBalancingConfigurations'
+                        )
                             list-editable-item-edit
                                 - form = '$parent.form'
                                 .settings-row
@@ -86,7 +89,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                                             +checkbox('Use processors', '$item.Adaptive.loadProbe.CPU.useProcessors', '"loadBalancingAdaptiveCPUUseProcessors"', "divide each node's CPU load by the number of processors on that node")
                                         .details-row
                                             +number-min-max-step('Processor coefficient:', '$item.Adaptive.loadProbe.CPU.processorCoefficient',
-                                                '"loadBalancingAdaptiveCPUProcessorCoefficient"', 'true', '1', '0.001', '1', '0.05', 'Coefficient of every CPU')
+                                                '"loadBalancingAdaptiveCPUProcessorCoefficient"', 'true', '1', '0.001', '1', '0.001', 'Coefficient of every CPU')
                                     .panel-details(ng-show='$item.Adaptive.loadProbe.kind === "ProcessingTime"')
                                         .details-row
                                             +checkbox('Use average', '$item.Adaptive.loadProbe.ProcessingTime.useAverage', '"loadBalancingAdaptiveJobUseAverage"', 'Use average execution time vs. current')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/memory.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/memory.pug
index 831adea592711..181ae4561b30e 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/memory.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/memory.pug
@@ -98,7 +98,7 @@ panel-collapsible(
                             .ignite-form-field__control
                                 -let items = memoryPolicies
 
-                                list-editable(ng-model=items name='memoryPolicies')
+                                list-editable.pc-list-editable-with-form-grid(ng-model=items name='memoryPolicies')
                                     list-editable-item-edit.pc-form-grid-row
                                         - form = '$parent.form'
                                         .pc-form-grid-col-60
@@ -158,7 +158,7 @@ panel-collapsible(
                                             </ul>')
                                         .pc-form-grid-col-30
                                             +number-min-max-step('Eviction threshold:', '$item.evictionThreshold', '"MemoryPolicyEvictionThreshold"',
-                                            'true', '0.9', '0.5', '0.999', '0.05', 'A threshold for memory pages eviction initiation')
+                                            'true', '0.9', '0.5', '0.999', '0.001', 'A threshold for memory pages eviction initiation')
                                         .pc-form-grid-col-30
                                             +sane-ignite-form-field-number({
                                                 label: 'Empty pages pool size:',
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/odbc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/odbc.pug
index 74b1f02c316c2..481a9aa984c1d 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/odbc.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/odbc.pug
@@ -30,8 +30,8 @@ panel-collapsible(
         | ODBC server configuration. 
         | #[a.link-success(href="https://apacheignite.readme.io/docs/odbc-driver" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`$ctrl.available(["1.0.0", "2.1.0"]) && ui.isPanelLoaded('${form}')`)
-        .pca-form-column-6
-            .settings-row
+        .pca-form-column-6.pc-form-grid-row
+            .pc-form-grid-col-60
                 +sane-form-field-checkbox({
                     label: 'Enabled',
                     model: enabled,
@@ -44,7 +44,7 @@ panel-collapsible(
                     ui-validate-watch='$ctrl.Clusters.odbc.odbcEnabled.correctMarshallerWatch("$ctrl.clonedCluster")'
                 )
                     +form-field-feedback(null, 'correctMarshaller', 'ODBC can only be used with BinaryMarshaller')
-            .settings-row
+            .pc-form-grid-col-60
                 +text-ip-address-with-port-range('ODBC endpoint address:', `${model}.endpointAddress`, '"endpointAddress"', enabled, '0.0.0.0:10800..10810',
                     'ODBC endpoint address. <br/>\
                     The following address formats are permitted:\
@@ -53,17 +53,17 @@ panel-collapsible(
                         <li>hostname:port - will use provided hostname and port</li>\
                         <li>hostname:port_from..port_to - will use provided hostname and port range</li>\
                     </ul>')
-            .settings-row
+            .pc-form-grid-col-30
                 +number('Send buffer size:', `${model}.socketSendBufferSize`, '"ODBCSocketSendBufferSize"', enabled, '0', '0',
                     'Socket send buffer size.<br/>\
                     When set to <b>0</b>, operation system default will be used')
-            .settings-row
+            .pc-form-grid-col-30
                 +number('Socket receive buffer size:', `${model}.socketReceiveBufferSize`, '"ODBCSocketReceiveBufferSize"', enabled, '0', '0',
                     'Socket receive buffer size.<br/>\
                     When set to <b>0</b>, operation system default will be used')
-            .settings-row
+            .pc-form-grid-col-30
                 +number('Maximum open cursors', `${model}.maxOpenCursors`, '"maxOpenCursors"', enabled, '128', '1', 'Maximum number of opened cursors per connection')
-            .settings-row
+            .pc-form-grid-col-30
                 +number('Pool size:', `${model}.threadPoolSize`, '"ODBCThreadPoolSize"', enabled, 'max(8, availableProcessors)', '1',
                     'Size of thread pool that is in charge of processing ODBC tasks')
         .pca-form-column-6
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug
index a244602835dd0..b37067bb6b6eb 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug
@@ -32,15 +32,9 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     .ignite-form-field__control
                         -let items = model
 
-                        list-editable(ng-model=items name='serviceConfigurations')
-                            list-editable-item-edit
-                                - form = '$parent.form'
-                    
-                                -var nodeFilter = '$item.nodeFilter';
-                                -var nodeFilterKind = nodeFilter + '.kind';
-                                -var customFilter = nodeFilterKind + ' === "Custom"'
-
-                                .settings-row
+                        list-editable.pc-list-editable-with-form-grid(ng-model=items name='serviceConfigurations')
+                            list-editable-item-edit.pc-form-grid-row
+                                .pc-form-grid-col-60
                                     +sane-ignite-form-field-text({
                                         label: 'Name:',
                                         model: '$item.name',
@@ -56,23 +50,23 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                                         ng-model-options='{allowInvalid: true}'
                                     )
                                         +form-field-feedback('"serviceName', 'uniqueName', 'Service with that name is already configured')
-                                .settings-row
+                                .pc-form-grid-col-60
                                     +java-class('Service class', '$item.service', '"serviceService"', 'true', 'true', 'Service implementation class name')
-                                .settings-row
+                                .pc-form-grid-col-60
                                     +number('Max per node count:', '$item.maxPerNodeCount', '"ServiceMaxPerNodeCount"', 'true', 'Unlimited', '0',
                                         'Maximum number of deployed service instances on each node.<br/>' +
                                         'Zero for unlimited')
-                                .settings-row
+                                .pc-form-grid-col-60
                                     +number('Total count:', '$item.totalCount', '"serviceTotalCount"', 'true', 'Unlimited', '0',
                                         'Total number of deployed service instances in the cluster.<br/>' +
                                         'Zero for unlimited')
-                                .settings-row
+                                .pc-form-grid-col-60
                                     +dropdown-required-empty('Cache:', '$item.cache', '"serviceCache"', 'true', 'false',
                                         'Choose cache', 'No caches configured for current cluster', '$ctrl.cachesMenu', 'Cache name used for key-to-node affinity calculation')(
                                         pc-is-in-collection='$ctrl.clonedCluster.caches'
-                                    ).settings-row
-                                        +form-field-feedback(form, 'isInCollection', `Cluster doesn't have such a cache`)
-                                .settings-row
+                                    )
+                                        +form-field-feedback(_, 'isInCollection', `Cluster doesn't have such a cache`)
+                                .pc-form-grid-col-60
                                     +text('Affinity key:', '$item.affinityKey', '"serviceAffinityKey"', 'false', 'Input affinity key',
                                         'Affinity key used for key-to-node affinity calculation')
 
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/sql-connector.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/sql-connector.pug
index 2e61fc2439cee..708aa0d8a3b17 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/sql-connector.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/sql-connector.pug
@@ -32,17 +32,17 @@ panel-collapsible(
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
                 +checkbox('Enabled', connectionEnabled, '"SqlConnectorEnabled"', 'Flag indicating whether to configure SQL connector configuration')
-            .pc-form-grid-col-40
+            .pc-form-grid-col-60
                 +text-enabled('Host:', `${connectionModel}.host`, '"SqlConnectorHost"', connectionEnabled, 'false', 'localhost')
-            .pc-form-grid-col-20
+            .pc-form-grid-col-30
                 +number('Port:', `${connectionModel}.port`, '"SqlConnectorPort"', connectionEnabled, '10800', '1025')
-            .pc-form-grid-col-20
+            .pc-form-grid-col-30
                 +number('Port range:', `${connectionModel}.portRange`, '"SqlConnectorPortRange"', connectionEnabled, '100', '0')
-            .pc-form-grid-col-20
+            .pc-form-grid-col-30
                 +number('Socket send buffer size:', `${connectionModel}.socketSendBufferSize`, '"SqlConnectorSocketSendBufferSize"', connectionEnabled, '0', '0',
                     'Socket send buffer size.<br/>\
                     When set to <b>0</b>, operation system default will be used')
-            .pc-form-grid-col-20
+            .pc-form-grid-col-30
                 +number('Socket receive buffer size:', `${connectionModel}.socketReceiveBufferSize`, '"SqlConnectorSocketReceiveBufferSize"', connectionEnabled, '0', '0',
                     'Socket receive buffer size.<br/>\
                     When set to <b>0</b>, operation system default will be used')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/swap.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/swap.pug
index ef73ab91f757d..d314296356f19 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/swap.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/swap.pug
@@ -59,7 +59,7 @@ panel-collapsible(
                     )
                         +form-field-feedback('"readStripesNumber"', 'powerOfTwo', 'Read stripe size must be positive and power of two')
                 .pc-form-grid-col-30
-                    +number-min-max-step('Maximum sparsity:', `${fileSwapModel}.maximumSparsity`, '"maximumSparsity"', 'true', '0.5', '0', '0.999', '0.05',
+                    +number-min-max-step('Maximum sparsity:', `${fileSwapModel}.maximumSparsity`, '"maximumSparsity"', 'true', '0.5', '0', '0.999', '0.001',
                         'This property defines maximum acceptable wasted file space to whole file size ratio<br/>\
                         When this ratio becomes higher than specified number compacting thread starts working')
                 .pc-form-grid-col-30
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/controller.js
index 68120020b90f6..b787ce813c83f 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/controller.js
@@ -19,6 +19,9 @@ import cloneDeep from 'lodash/cloneDeep';
 import get from 'lodash/get';
 
 export default class IgfsEditFormController {
+    /** @type {ng.ICompiledExpression} */
+    onSave;
+
     static $inject = ['IgniteConfirm', 'IgniteVersion', '$scope', 'IGFSs', 'IgniteFormUtils'];
     constructor( IgniteConfirm, IgniteVersion, $scope, IGFSs, IgniteFormUtils) {
         Object.assign(this, { IgniteConfirm, IgniteVersion, $scope, IGFSs, IgniteFormUtils});
@@ -28,6 +31,11 @@ export default class IgfsEditFormController {
 
         this.$scope.ui = this.IgniteFormUtils.formUI();
         this.$scope.ui.loadedPanels = ['general', 'secondaryFileSystem', 'misc'];
+
+        this.formActions = [
+            {text: 'Save', icon: 'checkmark', click: () => this.save()},
+            {text: 'Save and Download', icon: 'download', click: () => this.save(true)}
+        ];
     }
 
     $onChanges(changes) {
@@ -44,10 +52,10 @@ export default class IgfsEditFormController {
     getValuesToCompare() {
         return [this.igfs, this.$scope.backupItem].map(this.IGFSs.normalize);
     }
-    save() {
+    save(download) {
         if (this.$scope.ui.inputForm.$invalid)
             return this.IgniteFormUtils.triggerValidation(this.$scope.ui.inputForm, this.$scope);
-        this.onSave({$event: cloneDeep(this.$scope.backupItem)});
+        this.onSave({$event: {igfs: cloneDeep(this.$scope.backupItem), download}});
     }
     reset = (forReal) => forReal ? this.$scope.backupItem = cloneDeep(this.igfs) : void 0;
     confirmAndReset() {
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/template.tpl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/template.tpl.pug
index f505e589562df..a85a33c68563c 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/template.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-form(id='igfs' name='ui.inputForm' novalidate ng-submit='$ctrl.save()')
+form(id='igfs' name='ui.inputForm' novalidate)
     include ./templates/general
 
     include ./templates/secondary
@@ -32,7 +32,4 @@ form(id='igfs' name='ui.inputForm' novalidate ng-submit='$ctrl.save()')
         ng-click='$ctrl.confirmAndReset()'
     )
         | Cancel
-    button.btn-ignite.btn-ignite--success(
-        form='igfs'
-        type='submit'
-    ) Save
\ No newline at end of file
+    pc-split-button(actions=`::$ctrl.formActions`)
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js
index 20384d5e21c0b..b7d4ebe3b513d 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js
@@ -28,6 +28,9 @@ import {Confirm} from 'app/services/Confirm.service';
 export default class ModelEditFormController {
     /** @type {ig.config.model.DomainModel} */
     model;
+    /** @type {ng.ICompiledExpression} */
+    onSave;
+
     static $inject = ['ModalImportModels', 'IgniteErrorPopover', 'IgniteLegacyUtils', Confirm.name, 'ConfigChangesGuard', IgniteVersion.name, '$scope', Models.name, 'IgniteFormUtils'];
     /**
      * @param {ModalImportModels} ModalImportModels
@@ -54,6 +57,11 @@ export default class ModelEditFormController {
         this.$scope.javaBuiltInClasses = this.LegacyUtils.javaBuiltInClasses;
         this.$scope.supportedJdbcTypes = this.LegacyUtils.mkOptions(this.LegacyUtils.SUPPORTED_JDBC_TYPES);
         this.$scope.supportedJavaTypes = this.LegacyUtils.mkOptions(this.LegacyUtils.javaBuiltInTypes);
+
+        this.formActions = [
+            {text: 'Save', icon: 'checkmark', click: () => this.save()},
+            {text: 'Save and Download', icon: 'download', click: () => this.save(true)}
+        ];
     }
 
     /**
@@ -163,22 +171,30 @@ export default class ModelEditFormController {
         if ('caches' in changes)
             this.cachesMenu = (changes.caches.currentValue || []).map((c) => ({label: c.name, value: c._id}));
     }
+
     /**
      * @param {ig.config.model.DomainModel} model
      */
     onQueryFieldsChange(model) {
         this.$scope.backupItem = this.Models.removeInvalidFields(model);
     }
+
     getValuesToCompare() {
         return [this.model, this.$scope.backupItem].map(this.Models.normalize);
     }
-    save() {
+
+    save(download) {
         if (this.$scope.ui.inputForm.$invalid)
             return this.IgniteFormUtils.triggerValidation(this.$scope.ui.inputForm, this.$scope);
-        if (!this.validate(this.$scope.backupItem)) return;
-        this.onSave({$event: cloneDeep(this.$scope.backupItem)});
+
+        if (!this.validate(this.$scope.backupItem))
+            return;
+
+        this.onSave({$event: {model: cloneDeep(this.$scope.backupItem), download}});
     }
+
     reset = (forReal) => forReal ? this.$scope.backupItem = cloneDeep(this.model) : void 0;
+
     confirmAndReset() {
         return this.Confirm.confirm('Are you sure you want to undo all changes for current model?').then(() => true)
         .then(this.reset)
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/template.tpl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/template.tpl.pug
index 78ae7693dcadb..4751861a7dad0 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/template.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-form(id='model' name='ui.inputForm' novalidate ng-submit='$ctrl.save()')
+form(id='model' name='ui.inputForm' novalidate)
     include ./templates/general
     include ./templates/query
     include ./templates/store
@@ -26,7 +26,4 @@ form(id='model' name='ui.inputForm' novalidate ng-submit='$ctrl.save()')
         ng-click='$ctrl.confirmAndReset()'
     )
         | Cancel
-    button.btn-ignite.btn-ignite--success(
-        form='model'
-        type='submit'
-    ) Save
+    pc-split-button(actions=`::$ctrl.formActions`)
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js
index 60244e57f2a9c..ecb7a15a3efad 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js
@@ -168,7 +168,7 @@ export default class Controller {
         this.$state.go('base.configuration.edit.advanced.caches.cache', {cacheID});
     }
 
-    save(cache) {
-        this.ConfigureState.dispatchAction(advancedSaveCache(cache));
+    save({cache, download}) {
+        this.ConfigureState.dispatchAction(advancedSaveCache(cache, download));
     }
 }
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js
index ad2eed4cbe57f..2b05940eb4ae2 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js
@@ -45,7 +45,7 @@ export default class PageConfigureAdvancedCluster {
         this.isBlocked$ = clusterID$;
     }
 
-    save(cluster) {
-        this.ConfigureState.dispatchAction(advancedSaveCluster(cluster));
+    save({cluster, download}) {
+        this.ConfigureState.dispatchAction(advancedSaveCluster(cluster, download));
     }
 }
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js
index 09d139dd7bc7a..dd0e5daac5c45 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js
@@ -128,8 +128,8 @@ export default class PageConfigureAdvancedIGFS {
     edit(igfsID) {
         this.$state.go('base.configuration.edit.advanced.igfs.igfs', {igfsID});
     }
-    save(igfs) {
-        this.ConfigureState.dispatchAction(advancedSaveIGFS(igfs));
+    save({igfs, download}) {
+        this.ConfigureState.dispatchAction(advancedSaveIGFS(igfs, download));
     }
     remove(itemIDs) {
         this.ConfigureState.dispatchAction(
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js
index 7771735631fe9..d2a07e879f731 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js
@@ -156,8 +156,8 @@ export default class PageConfigureAdvancedModels {
         this.$state.go('base.configuration.edit.advanced.models.model', {modelID});
     }
 
-    save(model) {
-        this.ConfigureState.dispatchAction(advancedSaveModel(model));
+    save({model, download}) {
+        this.ConfigureState.dispatchAction(advancedSaveModel(model, download));
     }
 
     /**
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/style.scss b/modules/web-console/frontend/app/components/page-configure-advanced/style.scss
index 2bc5e84ca2346..9480486c9e790 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/style.scss
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/style.scss
@@ -43,6 +43,7 @@ page-configure-advanced {
         background-color: #f9f9f9;
         list-style: none;
         border-bottom: 1px solid #dddddd !important;
+        position: -webkit-sticky;
         position: sticky;
 
         .pca-menu-link {
diff --git a/modules/web-console/frontend/app/components/page-configure-basic/controller.js b/modules/web-console/frontend/app/components/page-configure-basic/controller.js
index e764ac6dfef4d..105765a3a423d 100644
--- a/modules/web-console/frontend/app/components/page-configure-basic/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-basic/controller.js
@@ -81,8 +81,11 @@ export default class PageConfigureBasicController {
     }
 
     _uiCanExit($transition$) {
-        if ($transition$.options().custom.justIDUpdate) return true;
+        if ($transition$.options().custom.justIDUpdate)
+            return true;
+
         $transition$.onSuccess({}, () => this.reset());
+
         return Observable.forkJoin(
             this.ConfigureState.state$.pluck('edit', 'changes').take(1),
             this.clusterID$.switchMap((id) => this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterShortCaches(id))).take(1),
@@ -136,12 +139,12 @@ export default class PageConfigureBasicController {
 
         this.formActionsMenu = [
             {
-                text: 'Save changes and download project',
+                text: 'Save and Download',
                 click: () => this.save(true),
                 icon: 'download'
             },
             {
-                text: 'Save changes',
+                text: 'Save',
                 click: () => this.save(),
                 icon: 'checkmark'
             }
@@ -178,7 +181,9 @@ export default class PageConfigureBasicController {
     }
 
     save(download = false) {
-        if (this.form.$invalid) return this.IgniteFormUtils.triggerValidation(this.form, this.$scope);
+        if (this.form.$invalid)
+            return this.IgniteFormUtils.triggerValidation(this.form, this.$scope);
+
         this.ConfigureState.dispatchAction((download ? basicSaveAndDownload : basicSave)(cloneDeep(this.clonedCluster)));
     }
 
@@ -189,7 +194,7 @@ export default class PageConfigureBasicController {
 
     confirmAndReset() {
         return this.Confirm.confirm('Are you sure you want to undo all changes for current cluster?')
-        .then(() => this.reset())
-        .catch(() => {});
+            .then(() => this.reset())
+            .catch(() => {});
     }
 }
diff --git a/modules/web-console/frontend/app/components/page-configure-basic/style.scss b/modules/web-console/frontend/app/components/page-configure-basic/style.scss
index 64d1f2fdb915a..7e0d8baac1b01 100644
--- a/modules/web-console/frontend/app/components/page-configure-basic/style.scss
+++ b/modules/web-console/frontend/app/components/page-configure-basic/style.scss
@@ -46,31 +46,6 @@ page-configure-basic {
         }
     }
 
-    .pcb-buttons-group {
-        display: flex;
-        flex-direction: row;
-
-        &>*+* {
-            margin-left: 10px;
-        }
-
-        .link-primary + .link-primary {
-            margin-left: 40px;
-        }
-    }
-
-    .pcb-select-existing-cache {
-        position: relative;
-
-        button {
-            background: none;
-            border: none;
-            padding: 0;
-            margin: 0;
-            outline: none !important;
-        }
-    }
-
     .pcb-section-notification {
         font-size: 14px;
         color: #757575;
@@ -155,6 +130,9 @@ page-configure-basic {
             &>.pc-form-grid-col-120 {
                 flex: 0 0 100%;
             }
+            &>.pc-form-grid-col-free {
+                flex: 1;
+            }
         }
         @media(max-width: 992px) {
             &>.pc-form-grid-col-10 {
@@ -177,6 +155,9 @@ page-configure-basic {
             &>.pc-form-grid-col-120 {
                 flex: 0 0 100%;
             }
+            &>.pc-form-grid-col-free {
+                flex: 1;
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure-basic/template.pug b/modules/web-console/frontend/app/components/page-configure-basic/template.pug
index 3bcb216dd2861..e85b3d8e56965 100644
--- a/modules/web-console/frontend/app/components/page-configure-basic/template.pug
+++ b/modules/web-console/frontend/app/components/page-configure-basic/template.pug
@@ -171,7 +171,6 @@ form(novalidate name=form)
         
     .pc-form-actions-panel
         button-preview-project(ng-hide='$ctrl.isNew$|async:this' cluster=model)
-        button-download-project(ng-hide='$ctrl.isNew$|async:this' cluster=model)
 
         .pc-form-actions-panel__right-after
         button.btn-ignite.btn-ignite--link-success(
@@ -179,16 +178,4 @@ form(novalidate name=form)
             ng-click='$ctrl.confirmAndReset()'
         )
             | Cancel
-        .btn-ignite-group
-            button.btn-ignite.btn-ignite--success(
-                ng-click='::$ctrl.formActionsMenu[0].click()'
-                type='button'
-            )
-                svg(ignite-icon='{{ ::$ctrl.formActionsMenu[0].icon }}').icon-left
-                | {{ ::$ctrl.formActionsMenu[0].text }}
-            button.btn-ignite.btn-ignite--success(
-                bs-dropdown='$ctrl.formActionsMenu'
-                data-placement='top-right'
-                type='button'
-            )
-                span.icon.fa.fa-caret-up
\ No newline at end of file
+        pc-split-button(actions=`::$ctrl.formActionsMenu`)
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure-overview/components/pco-grid-column-categories/directive.js b/modules/web-console/frontend/app/components/page-configure-overview/components/pco-grid-column-categories/directive.js
index 27d00dce2d7c1..ff03e7f9e7f89 100644
--- a/modules/web-console/frontend/app/components/page-configure-overview/components/pco-grid-column-categories/directive.js
+++ b/modules/web-console/frontend/app/components/page-configure-overview/components/pco-grid-column-categories/directive.js
@@ -38,7 +38,9 @@ export default function directive(uiGridConstants) {
         require: '^uiGrid',
         link: {
             pre(scope, el, attr, grid) {
-                if (!grid.grid.options.enableColumnCategories) return;
+                if (!grid.grid.options.enableColumnCategories)
+                    return;
+
                 grid.grid.api.core.registerColumnsProcessor((cp) => {
                     const oldCategories = grid.grid.options.categories;
                     const newCategories = uniqBy(cp.filter(notSelectionColumn).map(({colDef: cd}) => {
diff --git a/modules/web-console/frontend/app/components/page-configure-overview/controller.js b/modules/web-console/frontend/app/components/page-configure-overview/controller.js
index 6a24f964cf977..db21d9afe23a2 100644
--- a/modules/web-console/frontend/app/components/page-configure-overview/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-overview/controller.js
@@ -70,6 +70,9 @@ export default class PageConfigureOverviewController {
     /** @param {Array<ig.config.cluster.ShortCluster>} clusters */
     removeClusters(clusters) {
         this.ConfigureState.dispatchAction(confirmClustersRemoval(clusters.map((c) => c._id)));
+
+        // TODO: Implement storing selected rows in store to share this data between other components.
+        this.selectedRows$.next([]);
     }
 
     /** @param {ig.config.cluster.ShortCluster} cluster */
@@ -137,6 +140,8 @@ export default class PageConfigureOverviewController {
         /** @type {Subject<Array<ig.config.cluster.ShortCluster>>} */
         this.selectedRows$ = new Subject();
 
+        this.selectedRowsIDs$ = this.selectedRows$.map((selectedClusters) => selectedClusters.map((cluster) => cluster._id));
+
         this.actions$ = this.selectedRows$.map((selectedClusters) => [
             {
                 action: 'Edit',
diff --git a/modules/web-console/frontend/app/components/page-configure-overview/style.scss b/modules/web-console/frontend/app/components/page-configure-overview/style.scss
index e198fa4abe8ca..e461d06e692f6 100644
--- a/modules/web-console/frontend/app/components/page-configure-overview/style.scss
+++ b/modules/web-console/frontend/app/components/page-configure-overview/style.scss
@@ -27,7 +27,7 @@ page-configure-overview {
         flex-direction: row;
 
         &>* {
-            margin-left: 10px;
+            margin-left: 20px;
         }
     }
 }
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure-overview/template.pug b/modules/web-console/frontend/app/components/page-configure-overview/template.pug
index 753ee0648b659..be8e99976ceff 100644
--- a/modules/web-console/frontend/app/components/page-configure-overview/template.pug
+++ b/modules/web-console/frontend/app/components/page-configure-overview/template.pug
@@ -14,10 +14,11 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-h1.pc-page-header Configuration
-
-.pco-relative-root
-    .pco-table-context-buttons
+header.header-with-selector
+    div
+        h1 Configuration
+        version-picker
+    div
         a.btn-ignite.btn-ignite--primary(
             type='button'
             ui-sref='^.edit({clusterID: "new"})'
@@ -25,6 +26,8 @@ h1.pc-page-header Configuration
             svg.icon-left(ignite-icon='plus')
             | Create Cluster Configuration
         button-import-models(cluster-id='::"new"')
+
+.pco-relative-root
     pc-items-table(
         table-title='::"My Cluster Configurations"'
         column-defs='$ctrl.clustersColumnDefs'
@@ -32,9 +35,10 @@ h1.pc-page-header Configuration
         on-action='$ctrl.onClustersAction($event)'
         max-rows-to-show='10'
         one-way-selection='::false'
+        selected-row-id='$ctrl.selectedRowsIDs$|async:this'
         on-selection-change='$ctrl.selectedRows$.next($event)'
         actions-menu='$ctrl.actions$|async:this'
     )
         footer-slot(ng-hide='($ctrl.shortClusters$|async:this).length' style='font-style: italic')
             | You have no cluster configurations.
-            a.link-success(ui-sref='base.configuration.edit.basic({clusterID: "new"})')  Create one?
\ No newline at end of file
+            a.link-success(ui-sref='base.configuration.edit.basic({clusterID: "new"})') Create one?
diff --git a/modules/web-console/frontend/app/components/page-configure/components/fakeUICanExit.js b/modules/web-console/frontend/app/components/page-configure/components/fakeUICanExit.js
index c0837edbb6036..246562f1bc198 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/fakeUICanExit.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/fakeUICanExit.js
@@ -15,25 +15,42 @@
  * limitations under the License.
  */
 
-class FakeUiCanExitController {
+export class FakeUiCanExitController {
     static $inject = ['$element', '$transitions'];
     static CALLBACK_NAME = 'uiCanExit';
+
+    /** @type {string} Name of state to listen exit from */
+    fromState;
+
+    /**
+     * @param {JQLite} $element
+     * @param {import('@uirouter/angularjs').TransitionService} $transitions
+     */
     constructor($element, $transitions) {
-        Object.assign(this, {$element, $transitions});
+        this.$element = $element;
+        this.$transitions = $transitions;
     }
+
     $onInit() {
         const data = this.$element.data();
         const {CALLBACK_NAME} = this.constructor;
+
         const controllerWithCallback = Object.keys(data)
             .map((key) => data[key])
             .find((controller) => controller[CALLBACK_NAME]);
-        if (!controllerWithCallback) return;
-        const off = this.$transitions.onBefore({from: this.fromState}, (...args) => {
+
+        if (!controllerWithCallback)
+            return;
+
+        this.off = this.$transitions.onBefore({from: this.fromState}, (...args) => {
             return controllerWithCallback[CALLBACK_NAME](...args);
         });
     }
+
     $onDestroy() {
-        if (this.off) this.off();
+        if (this.off)
+            this.off();
+
         this.$element = null;
     }
 }
diff --git a/modules/web-console/frontend/app/components/page-configure/components/fakeUICanExit.spec.js b/modules/web-console/frontend/app/components/page-configure/components/fakeUICanExit.spec.js
new file mode 100644
index 0000000000000..7d21ebafb14e8
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-configure/components/fakeUICanExit.spec.js
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {spy} from 'sinon';
+import {assert} from 'chai';
+import {FakeUiCanExitController} from './fakeUICanExit';
+
+suite('Page configuration fakeUIcanExit directive', () => {
+    test('It unsubscribes from state events when destroyed', () => {
+        const $element = {data: () => [{uiCanExit: () => {}}]};
+        const off = spy();
+        const $transitions = {onBefore: () => off};
+        const i = new FakeUiCanExitController($element, $transitions);
+        i.$onInit();
+        i.$onDestroy();
+        assert.ok(off.calledOnce, 'Calls off when destroyed');
+    });
+});
diff --git a/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js b/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js
index 0225c5fc003e3..c96216143d7cc 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js
@@ -19,6 +19,7 @@ import {default as ConfigChangesGuard} from '../services/ConfigChangesGuard';
 
 class FormUICanExitGuardController {
     static $inject = ['$element', ConfigChangesGuard.name];
+
     /**
      * @param {JQLite} $element
      * @param {ConfigChangesGuard} ConfigChangesGuard
@@ -27,23 +28,30 @@ class FormUICanExitGuardController {
         this.$element = $element;
         this.ConfigChangesGuard = ConfigChangesGuard;
     }
+
     $onDestroy() {
         this.$element = null;
     }
+
     $onInit() {
         const data = this.$element.data();
         const controller = Object.keys(data)
             .map((key) => data[key])
             .find(this._itQuacks);
 
-        if (!controller) return;
+        if (!controller)
+            return;
 
         controller.uiCanExit = ($transition$) => {
-            if ($transition$.options().custom.justIDUpdate) return true;
+            if ($transition$.options().custom.justIDUpdate)
+                return true;
+
             $transition$.onSuccess({}, controller.reset);
+
             return this.ConfigChangesGuard.guard(...controller.getValuesToCompare());
         };
     }
+
     _itQuacks(controller) {
         return controller.reset instanceof Function &&
             controller.getValuesToCompare instanceof Function &&
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
index 813c998f0e00e..1d4ba7c48dbd3 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
@@ -123,11 +123,12 @@ export class ModalImportModels {
         this.ActivitiesData = ActivitiesData;
         Object.assign(this, {Confirm, Focus, Messages, Loading, FormUtils, LegacyUtils});
     }
+
     loadData() {
         return Observable.of(this.clusterID)
-        .switchMap((id = 'new') => {
-            return this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterToEdit(id, defaultNames.importedCluster));
-        })
+            .switchMap((id = 'new') => {
+                return this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterToEdit(id, defaultNames.importedCluster));
+            })
         .switchMap((cluster) => {
             return (!(cluster.caches || []).length && !(cluster.models || []).length)
                 ? Observable.of({
@@ -154,7 +155,9 @@ export class ModalImportModels {
         .take(1);
     }
     saveBatch(batch) {
-        if (!batch.length) return;
+        if (!batch.length)
+            return;
+
         this.Loading.start('importDomainFromDb');
         this.ConfigureState.dispatchAction({
             type: 'ADVANCED_SAVE_COMPLETE_CONFIGURATION',
@@ -212,8 +215,12 @@ export class ModalImportModels {
         return this.visibleTables = rows.map((r) => r.entity);
     }
     onCacheSelect(cacheID) {
-        if (cacheID < 0) return;
-        if (this.loadedCaches[cacheID]) return;
+        if (cacheID < 0)
+            return;
+
+        if (this.loadedCaches[cacheID])
+            return;
+
         return this.onCacheSelectSubcription = Observable.merge(
             Observable.timer(0, 1).take(1)
                 .do(() => this.ConfigureState.dispatchAction({type: 'LOAD_CACHE', cacheID})),
@@ -290,10 +297,10 @@ export class ModalImportModels {
         }).subscribe();
 
         // New
-
         this.loadedCaches = {
             ...CACHE_TEMPLATES.reduce((a, c) => ({...a, [c.value]: c.cache}), {})
         };
+
         this.actions = [
             {value: 'connect', label: this.$root.IgniteDemoMode ? 'Description' : 'Connection'},
             {value: 'schemas', label: 'Schemas'},
@@ -302,7 +309,6 @@ export class ModalImportModels {
         ];
 
         // Legacy
-
         $scope.ui.invalidKeyFieldsTooltip = 'Found key types without configured key fields<br/>' +
             'It may be a result of import tables from database without primary keys<br/>' +
             'Key field for such key types should be configured manually';
@@ -591,7 +597,6 @@ export class ModalImportModels {
 
                 if ($scope._curDbTable.actionWatch) {
                     $scope._curDbTable.actionWatch();
-
                     $scope._curDbTable.actionWatch = null;
                 }
             }
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js
index 072a4975bfa5f..7fde5baad212f 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js
@@ -22,23 +22,33 @@ const IMPORT_DM_NEW_CACHE = 1;
 
 export class TablesActionCell {
     static $inject = ['$element'];
+
     constructor($element) {
         Object.assign(this, {$element});
     }
+
     onClick(e) {
         e.stopPropagation();
     }
+
     $postLink() {
         this.$element.on('click', this.onClick);
     }
+
     $onDestroy() {
         this.$element.off('click', this.onClick);
         this.$element = null;
     }
+
     tableActionView(table) {
-        if (!this.caches) return;
+        if (!this.caches)
+            return;
+
         const cache = this.caches.find((c) => c.value === table.cacheOrTemplate);
-        if (!cache) return;
+
+        if (!cache)
+            return;
+
         const cacheName = cache.label;
 
         if (table.action === IMPORT_DM_NEW_CACHE)
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/controller.js b/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/controller.js
index e67ed25b1cdf4..1480fc11727ad 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/controller.js
@@ -47,9 +47,15 @@ export default class ModalPreviewProjectController {
 
     showPreview(node) {
         this.fileText = '';
-        if (!node) return;
+
+        if (!node)
+            return;
+
         this.fileExt = node.file.name.split('.').reverse()[0].toLowerCase();
-        if (node.file.dir) return;
+
+        if (node.file.dir)
+            return;
+
         node.file.async('string').then((text) => {
             this.fileText = text;
             this.$scope.$applyAsync();
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js b/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js
index 3253fe407f57d..0d751e849d299 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js
@@ -115,12 +115,16 @@ export default class PCFormFieldSizeController {
     }
 
     _defaultLabel() {
-        if (!this.sizesMenu) return;
+        if (!this.sizesMenu)
+            return;
+
         return this.sizesMenu[1].label;
     }
 
     chooseSizeScale(label = this._defaultLabel()) {
-        if (!label) return;
+        if (!label)
+            return;
+
         return this.sizesMenu.find((option) => option.label.toLowerCase() === label.toLowerCase());
     }
 
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/controller.js b/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/controller.js
index f4d1f47df4474..c97e50f08907d 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/controller.js
@@ -44,23 +44,28 @@ export default class ItemsTableController {
             },
             onRegisterApi: (api) => {
                 this.gridAPI = api;
+
                 api.selection.on.rowSelectionChanged(this.$scope, (row, e) => {
                     this.onRowsSelectionChange([row], e);
                 });
+
                 api.selection.on.rowSelectionChangedBatch(this.$scope, (rows, e) => {
                     this.onRowsSelectionChange(rows, e);
                 });
+
                 api.core.on.rowsVisibleChanged(this.$scope, () => {
                     const visibleRows = api.core.getVisibleRows();
                     if (this.onVisibleRowsChange) this.onVisibleRowsChange({$event: visibleRows});
                     this.adjustHeight(api, visibleRows.length);
                     this.showFilterNotification = this.grid.data.length && visibleRows.length === 0;
                 });
+
                 if (this.onFilterChanged) {
                     api.core.on.filterChanged(this.$scope, () => {
                         this.onFilterChanged();
                     });
                 }
+
                 this.$timeout(() => {
                     if (this.selectedRowId) this.applyIncomingSelection(this.selectedRowId);
                 });
@@ -73,10 +78,16 @@ export default class ItemsTableController {
     oneWaySelection = false;
 
     onRowsSelectionChange = debounce((rows, e = {}) => {
-        if (e.ignore) return;
+        if (e.ignore)
+            return;
+
         const selected = this.gridAPI.selection.legacyGetSelectedRows();
-        if (this.oneWaySelection) rows.forEach((r) => r.isSelected = false);
-        if (this.onSelectionChange) this.onSelectionChange({$event: selected});
+
+        if (this.oneWaySelection)
+            rows.forEach((r) => r.isSelected = false);
+
+        if (this.onSelectionChange)
+            this.onSelectionChange({$event: selected});
     });
 
     makeActionsMenu(incomingActionsMenu = []) {
@@ -85,17 +96,22 @@ export default class ItemsTableController {
 
     $onChanges(changes) {
         const hasChanged = (binding) => binding in changes && changes[binding].currentValue !== changes[binding].previousValue;
+
         if (hasChanged('items') && this.grid) {
             this.grid.data = changes.items.currentValue;
             this.gridAPI.grid.modifyRows(this.grid.data);
             this.adjustHeight(this.gridAPI, this.grid.data.length);
+
             // Without property existence check non-set selectedRowId binding might cause
             // unwanted behavior, like unchecking rows during any items change, even if
             // nothing really changed.
-            if ('selectedRowId' in this) this.applyIncomingSelection(this.selectedRowId);
+            if ('selectedRowId' in this)
+                this.applyIncomingSelection(this.selectedRowId);
         }
+
         if (hasChanged('selectedRowId') && this.grid && this.grid.data)
             this.applyIncomingSelection(changes.selectedRowId.currentValue);
+
         if ('incomingActionsMenu' in changes)
             this.actionsMenu = this.makeActionsMenu(changes.incomingActionsMenu.currentValue);
     }
@@ -103,9 +119,11 @@ export default class ItemsTableController {
     applyIncomingSelection(selected = []) {
         this.gridAPI.selection.clearSelectedRows({ignore: true});
         const rows = this.grid.data.filter((r) => selected.includes(r[this.rowIdentityKey]));
+
         rows.forEach((r) => {
             this.gridAPI.selection.selectRow(r, {ignore: true});
         });
+
         if (rows.length === 1) {
             this.$timeout(() => {
                 this.gridAPI.grid.scrollToIfNecessary(this.gridAPI.grid.getRow(rows[0]), null);
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/template.pug b/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/template.pug
index 0a8475c4740be..0dbb760d2d408 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/template.pug
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/template.pug
@@ -17,7 +17,7 @@
 include /app/helpers/jade/mixins
 
 .panel--ignite
-    .panel-heading.ui-grid-settings(ng-if='!$ctrl.hideHeader')
+    .panel-heading.ui-grid-settings.ui-grid-ignite__panel(ng-if='!$ctrl.hideHeader')
         .panel-title
             .pc-items-table__table-name.ng-animate-disabled(
                 ng-hide='$ctrl.gridAPI.selection.getSelectedCount()'
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/component.js b/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/component.js
new file mode 100644
index 0000000000000..c891f1cad36d6
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/component.js
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+import controller from './controller';
+
+export default {
+    controller,
+    template,
+    bindings: {
+        actions: '<'
+    }
+};
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/controller.js b/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/controller.js
new file mode 100644
index 0000000000000..8aa9495cd6491
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/controller.js
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @typedef {{icon: string, text: string, click: ng.ICompiledExpression}} ActionMenuItem
+ */
+
+/**
+ * @typedef {Array<ActionMenuItem>} ActionsMenu
+ */
+
+/**
+ * Groups multiple buttons into a single button with all but first buttons in a dropdown
+ */
+export default class SplitButton {
+    /** @type {ActionsMenu} */
+    actions = [];
+
+    static $inject = ['$element'];
+
+    /**
+     * @param {JQLite} $element Component root element
+     */
+    constructor($element, $transclude) {
+        this.$element = $element;
+    }
+
+    $onInit() {
+        this.$element[0].classList.add('btn-ignite-group');
+    }
+}
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/index.js b/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/index.js
new file mode 100644
index 0000000000000..4b4b816f9ff7d
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/index.js
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import component from './component';
+
+export default angular
+    .module('ignite-console.page-configure.pc-split-button', [])
+    .component('pcSplitButton', component);
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/template.pug b/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/template.pug
new file mode 100644
index 0000000000000..d44d543180a59
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-split-button/template.pug
@@ -0,0 +1,28 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+button.btn-ignite.btn-ignite--success(
+    ng-click='$ctrl.actions[0].click()'
+    type='button'
+)
+    svg(ignite-icon='{{ ::$ctrl.actions[0].icon }}').icon-left
+    | {{ ::$ctrl.actions[0].text }}
+button.btn-ignite.btn-ignite--success(
+    bs-dropdown='$ctrl.actions'
+    data-placement='top-right'
+    type='button'
+)
+    span.icon.fa.fa-caret-down
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/directive.js b/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/directive.js
index ccbdb8ddae59c..3d4617c69c709 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/directive.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/directive.js
@@ -23,7 +23,9 @@ export default function pcUiGridFilters(uiGridConstants) {
         require: 'uiGrid',
         link: {
             pre(scope, el, attr, grid) {
-                if (!grid.grid.options.enableFiltering) return;
+                if (!grid.grid.options.enableFiltering)
+                    return;
+
                 grid.grid.options.columnDefs.filter((cd) => cd.multiselectFilterOptions).forEach((cd) => {
                     cd.headerCellTemplate = template;
                     cd.filter = {
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/index.js b/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/index.js
index 4ec96e2726417..a34a78a352188 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/index.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-ui-grid-filters/index.js
@@ -27,9 +27,12 @@ export default angular
             instance.$referenceElement = el;
             instance.destroy = flow(instance.destroy, () => instance.$referenceElement = null);
             instance.$applyPlacement = flow(instance.$applyPlacement, () => {
-                if (!instance.$element) return;
+                if (!instance.$element)
+                    return;
+
                 const refWidth = instance.$referenceElement[0].getBoundingClientRect().width;
                 const elWidth = instance.$element[0].getBoundingClientRect().width;
+
                 if (refWidth > elWidth) {
                     instance.$element.css({
                         width: refWidth,
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pcIsInCollection.js b/modules/web-console/frontend/app/components/page-configure/components/pcIsInCollection.js
index d2a9ae8d63564..50ea4a46bcea7 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pcIsInCollection.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pcIsInCollection.js
@@ -18,7 +18,9 @@
 class Controller {
     $onInit() {
         this.ngModel.$validators.isInCollection = (item) => {
-            if (!item || !this.items) return true;
+            if (!item || !this.items)
+                return true;
+
             return this.items.includes(item);
         };
     }
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js b/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js
index 5ddd4eff24375..1702c35ae9ee5 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/pcValidation.js
@@ -56,7 +56,9 @@ export default angular.module('ignite-console.page-configure.validation', [])
 
             $onInit() {
                 this.ngModel.$validators.notInCollection = (item) => {
-                    if (!this.items) return true;
+                    if (!this.items)
+                        return true;
+
                     return !this.items.includes(item);
                 };
             }
@@ -87,7 +89,9 @@ export default angular.module('ignite-console.page-configure.validation', [])
 
             $onInit() {
                 this.ngModel.$validators.inCollection = (item) => {
-                    if (!this.items) return false;
+                    if (!this.items)
+                        return false;
+
                     const items = this.pluck ? this.items.map((i) => i[this.pluck]) : this.items;
                     return Array.isArray(item)
                         ? item.every((i) => items.includes(i))
@@ -146,28 +150,6 @@ export default angular.module('ignite-console.page-configure.validation', [])
             }]
         };
     })
-    .directive('ngModel', ['$timeout', function($timeout) {
-        return {
-            require: ['ngModel', '?^^bsCollapseTarget', '?^^igniteFormField', '?^^panelCollapsible'],
-            link(scope, el, attr, [ngModel, bsCollapseTarget, igniteFormField, panelCollapsible]) {
-                const off = scope.$on('$showValidationError', (e, target) => {
-                    if (target !== ngModel) return;
-                    ngModel.$setTouched();
-                    bsCollapseTarget && bsCollapseTarget.open();
-                    panelCollapsible && panelCollapsible.open();
-                    $timeout(() => {
-                        if (el[0].scrollIntoViewIfNeeded)
-                            el[0].scrollIntoViewIfNeeded();
-                        else
-                            el[0].scrollIntoView();
-
-                        if (!attr.bsSelect) $timeout(() => el[0].focus());
-                        igniteFormField && igniteFormField.notifyAboutError();
-                    });
-                });
-            }
-        };
-    }])
     .directive('igniteFormField', function() {
         return {
             restrict: 'C',
diff --git a/modules/web-console/frontend/app/components/page-configure/index.js b/modules/web-console/frontend/app/components/page-configure/index.js
index 34b8cfe3f0f6f..9beea7a57ad0d 100644
--- a/modules/web-console/frontend/app/components/page-configure/index.js
+++ b/modules/web-console/frontend/app/components/page-configure/index.js
@@ -46,6 +46,7 @@ import buttonImportModels from './components/button-import-models';
 import buttonDownloadProject from './components/button-download-project';
 import buttonPreviewProject from './components/button-preview-project';
 import previewPanel from './components/preview-panel';
+import pcSplitButton from './components/pc-split-button';
 
 import {errorState} from './transitionHooks/errorState';
 import {default as ActivitiesData} from 'app/core/activities/Activities.data';
@@ -60,6 +61,7 @@ Observable.prototype.debug = function(l) {
 
 import {
     editReducer2,
+    shortObjectsReducer,
     reducer,
     editReducer,
     loadingReducer,
@@ -106,7 +108,8 @@ export default angular
         buttonImportModels.name,
         buttonDownloadProject.name,
         buttonPreviewProject.name,
-        previewPanel.name
+        previewPanel.name,
+        pcSplitButton.name
     ])
     .config(registerStates)
     .config(['DefaultStateProvider', (DefaultState) => {
@@ -122,16 +125,18 @@ export default angular
             });
 
             ConfigureState.actions$
-            .filter((e) => e.type !== 'DISPATCH')
-            .withLatestFrom(ConfigureState.state$.skip(1))
-            .subscribe(([action, state]) => devTools.send(action, state));
+                .filter((e) => e.type !== 'DISPATCH')
+                .withLatestFrom(ConfigureState.state$.skip(1))
+                .subscribe(([action, state]) => devTools.send(action, state));
 
             ConfigureState.addReducer(reduxDevtoolsReducer);
         }
+
         ConfigureState.addReducer(refsReducer({
             models: {at: 'domains', store: 'caches'},
             caches: {at: 'caches', store: 'models'}
         }));
+
         ConfigureState.addReducer((state, action) => Object.assign({}, state, {
             clusterConfiguration: editReducer(state.clusterConfiguration, action),
             configurationLoading: loadingReducer(state.configurationLoading, action),
@@ -146,14 +151,19 @@ export default angular
             shortIgfss: mapCacheReducerFactory(shortIGFSsActionTypes)(state.shortIgfss, action),
             edit: editReducer2(state.edit, action)
         }));
+
+        ConfigureState.addReducer(shortObjectsReducer);
+
         ConfigureState.addReducer((state, action) => {
             switch (action.type) {
                 case 'APPLY_ACTIONS_UNDO':
                     return action.state;
+
                 default:
                     return state;
             }
         });
+
         const la = ConfigureState.actions$.scan((acc, action) => [...acc, action], []);
 
         ConfigureState.actions$
diff --git a/modules/web-console/frontend/app/components/page-configure/reducer.js b/modules/web-console/frontend/app/components/page-configure/reducer.js
index f959b0a573e21..0f24bfac970e6 100644
--- a/modules/web-console/frontend/app/components/page-configure/reducer.js
+++ b/modules/web-console/frontend/app/components/page-configure/reducer.js
@@ -16,6 +16,7 @@
  */
 
 import difference from 'lodash/difference';
+import capitalize from 'lodash/capitalize';
 
 export const LOAD_LIST = Symbol('LOAD_LIST');
 export const ADD_CLUSTER = Symbol('ADD_CLUSTER');
@@ -33,6 +34,7 @@ import {
 } from './store/actionTypes';
 
 const defaults = {clusters: new Map(), caches: new Map(), spaces: new Map()};
+
 const mapByID = (items) => {
     return Array.isArray(items) ? new Map(items.map((item) => [item._id, item])) : new Map(items);
 };
@@ -48,21 +50,25 @@ export const reducer = (state = defaults, action) => {
                 plugins: mapByID(action.list.plugins)
             };
         }
+
         case ADD_CLUSTER: {
             return Object.assign({}, state, {
                 clusters: new Map([...state.clusters.entries(), [action.cluster._id, action.cluster]])
             });
         }
+
         case ADD_CLUSTERS: {
             return Object.assign({}, state, {
                 clusters: new Map([...state.clusters.entries(), ...action.clusters.map((c) => [c._id, c])])
             });
         }
+
         case REMOVE_CLUSTERS: {
             return Object.assign({}, state, {
                 clusters: new Map([...state.clusters.entries()].filter(([id, value]) => !action.clusterIDs.includes(id)))
             });
         }
+
         case UPDATE_CLUSTER: {
             const id = action._id || action.cluster._id;
             return Object.assign({}, state, {
@@ -74,31 +80,38 @@ export const reducer = (state = defaults, action) => {
                 }))
             });
         }
+
         case UPSERT_CLUSTERS: {
             return action.clusters.reduce((state, cluster) => reducer(state, {
                 type: state.clusters.has(cluster._id) ? UPDATE_CLUSTER : ADD_CLUSTER,
                 cluster
             }), state);
         }
+
         case ADD_CACHE: {
             return Object.assign({}, state, {
                 caches: new Map([...state.caches.entries(), [action.cache._id, action.cache]])
             });
         }
+
         case UPDATE_CACHE: {
             const id = action.cache._id;
+
             return Object.assign({}, state, {
                 caches: new Map(state.caches).set(id, Object.assign({}, state.caches.get(id), action.cache))
             });
         }
+
         case UPSERT_CACHES: {
             return action.caches.reduce((state, cache) => reducer(state, {
                 type: state.caches.has(cache._id) ? UPDATE_CACHE : ADD_CACHE,
                 cache
             }), state);
         }
+
         case REMOVE_CACHE:
             return state;
+
         default:
             return state;
     }
@@ -119,34 +132,40 @@ export const editReducer = (state = {originalCluster: null}, action) => {
                 ...state,
                 originalCluster: action.cluster
             };
+
         case RECEIVE_CACHE_EDIT: {
             return {
                 ...state,
                 originalCache: action.cache
             };
         }
+
         case RECEIVE_IGFSS_EDIT:
             return {
                 ...state,
                 originalIGFSs: action.igfss
             };
+
         case RECEIVE_IGFS_EDIT: {
             return {
                 ...state,
                 originalIGFS: action.igfs
             };
         }
+
         case RECEIVE_MODELS_EDIT:
             return {
                 ...state,
                 originalModels: action.models
             };
+
         case RECEIVE_MODEL_EDIT: {
             return {
                 ...state,
                 originalModel: action.model
             };
         }
+
         default:
             return state;
     }
@@ -160,8 +179,10 @@ export const loadingReducer = (state = loadingDefaults, action) => {
     switch (action.type) {
         case SHOW_CONFIG_LOADING:
             return {...state, isLoading: true, loadingText: action.loadingText};
+
         case HIDE_CONFIG_LOADING:
             return {...state, isLoading: false};
+
         default:
             return state;
     }
@@ -171,12 +192,16 @@ export const setStoreReducerFactory = (actionTypes) => (state = new Set(), actio
     switch (action.type) {
         case actionTypes.SET:
             return new Set(action.items.map((i) => i._id));
+
         case actionTypes.RESET:
             return new Set();
+
         case actionTypes.UPSERT:
             return action.items.reduce((acc, item) => {acc.add(item._id); return acc;}, new Set(state));
+
         case actionTypes.REMOVE:
             return action.items.reduce((acc, item) => {acc.delete(item); return acc;}, new Set(state));
+
         default:
             return state;
     }
@@ -186,14 +211,22 @@ export const mapStoreReducerFactory = (actionTypes) => (state = new Map(), actio
     switch (action.type) {
         case actionTypes.SET:
             return new Map(action.items.map((i) => [i._id, i]));
+
         case actionTypes.RESET:
             return new Map();
+
         case actionTypes.UPSERT:
-            if (!action.items.length) return state;
+            if (!action.items.length)
+                return state;
+
             return action.items.reduce((acc, item) => {acc.set(item._id, item); return acc;}, new Map(state));
+
         case actionTypes.REMOVE:
-            if (!action.ids.length) return state;
+            if (!action.ids.length)
+                return state;
+
             return action.ids.reduce((acc, id) => {acc.delete(id); return acc;}, new Map(state));
+
         default:
             return state;
     }
@@ -201,6 +234,7 @@ export const mapStoreReducerFactory = (actionTypes) => (state = new Map(), actio
 
 export const mapCacheReducerFactory = (actionTypes) => {
     const mapStoreReducer = mapStoreReducerFactory(actionTypes);
+
     return (state = {value: mapStoreReducer(), pristine: true}, action) => {
         switch (action.type) {
             case actionTypes.SET:
@@ -210,11 +244,13 @@ export const mapCacheReducerFactory = (actionTypes) => {
                     value: mapStoreReducer(state.value, action),
                     pristine: false
                 };
+
             case actionTypes.RESET:
                 return {
                     value: mapStoreReducer(state.value, action),
                     pristine: true
                 };
+
             default:
                 return state;
         }
@@ -248,26 +284,31 @@ export const shortIGFSsActionTypes = mapStoreActionTypesFactory('SHORT_IGFSS');
 export const itemsEditReducerFactory = (actionTypes) => {
     const setStoreReducer = setStoreReducerFactory(actionTypes);
     const mapStoreReducer = mapStoreReducerFactory(actionTypes);
+
     return (state = {ids: setStoreReducer(), changedItems: mapStoreReducer()}, action) => {
         switch (action.type) {
             case actionTypes.SET:
                 return action.state;
+
             case actionTypes.LOAD:
                 return {
                     ...state,
                     ids: setStoreReducer(state.ids, {...action, type: actionTypes.UPSERT})
                 };
+
             case actionTypes.RESET:
             case actionTypes.UPSERT:
                 return {
                     ids: setStoreReducer(state.ids, action),
                     changedItems: mapStoreReducer(state.changedItems, action)
                 };
+
             case actionTypes.REMOVE:
                 return {
                     ids: setStoreReducer(state.ids, {type: action.type, items: action.ids}),
                     changedItems: mapStoreReducer(state.changedItems, action)
                 };
+
             default:
                 return state;
         }
@@ -278,6 +319,7 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
     switch (action.type) {
         case 'SET_EDIT':
             return action.state;
+
         case 'EDIT_CLUSTER': {
             return {
                 ...state,
@@ -293,6 +335,7 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
                 }
             };
         }
+
         case 'RESET_EDIT_CHANGES': {
             return {
                 ...state,
@@ -308,6 +351,7 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
                 }
             };
         }
+
         case 'UPSERT_CLUSTER': {
             return {
                 ...state,
@@ -317,6 +361,7 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
                 }
             };
         }
+
         case 'UPSERT_CLUSTER_ITEM': {
             const {itemType, item} = action;
             return {
@@ -330,8 +375,10 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
                 }
             };
         }
+
         case REMOVE_CLUSTER_ITEMS_CONFIRMED: {
             const {itemType, itemIDs} = action;
+
             return {
                 ...state,
                 changes: {
@@ -343,9 +390,11 @@ export const editReducer2 = (state = editReducer2.getDefaults(), action) => {
                 }
             };
         }
+
         default: return state;
     }
 };
+
 editReducer2.getDefaults = () => ({
     changes: ['caches', 'models', 'igfss'].reduce((a, t) => ({...a, [t]: {ids: [], changedItems: []}}), {cluster: null})
 });
@@ -356,15 +405,19 @@ export const refsReducer = (refs) => (state, action) => {
             const newCluster = action.changedItems.cluster;
             const oldCluster = state.clusters.get(newCluster._id) || {};
             const val = Object.keys(refs).reduce((state, ref) => {
-                if (!state || !state[refs[ref].store].size) return state;
+                if (!state || !state[refs[ref].store].size)
+                    return state;
 
                 const addedSources = new Set(difference(newCluster[ref], oldCluster[ref] || []));
                 const removedSources = new Set(difference(oldCluster[ref] || [], newCluster[ref]));
                 const changedSources = new Map(action.changedItems[ref].map((m) => [m._id, m]));
 
                 const targets = new Map();
+
                 const maybeTarget = (id) => {
-                    if (!targets.has(id)) targets.set(id, {[refs[ref].at]: {add: new Set(), remove: new Set()}});
+                    if (!targets.has(id))
+                        targets.set(id, {[refs[ref].at]: {add: new Set(), remove: new Set()}});
+
                     return targets.get(id);
                 };
 
@@ -373,11 +426,13 @@ export const refsReducer = (refs) => (state, action) => {
                     .filter((sourceID) => removedSources.has(sourceID))
                     .forEach((sourceID) => maybeTarget(target._id)[refs[ref].at].remove.add(sourceID));
                 });
+
                 [...addedSources.values()].forEach((sourceID) => {
                     (changedSources.get(sourceID)[refs[ref].store] || []).forEach((targetID) => {
                         maybeTarget(targetID)[refs[ref].at].add.add(sourceID);
                     });
                 });
+
                 action.changedItems[ref].filter((s) => !addedSources.has(s._id)).forEach((source) => {
                     const newSource = source;
                     const oldSource = state[ref].get(source._id);
@@ -412,8 +467,35 @@ export const refsReducer = (refs) => (state, action) => {
                     }
                     : state;
             }, state);
+
             return val;
         }
+
+        default:
+            return state;
+    }
+};
+
+export const shortObjectsReducer = (state, action) => {
+    switch (action.type) {
+        case REMOVE_CLUSTER_ITEMS_CONFIRMED: {
+            const {itemType, itemIDs} = action;
+
+            const target = 'short' + capitalize(itemType);
+
+            const oldItems = state[target];
+
+            const newItems = {
+                value: itemIDs.reduce((acc, id) => {acc.delete(id); return acc;}, oldItems.value),
+                pristine: oldItems.pristine
+            };
+
+            return {
+                ...state,
+                [target]: newItems
+            };
+        }
+
         default:
             return state;
     }
diff --git a/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js b/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js
index ea7f527469f3c..27e227d8dce9b 100644
--- a/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js
+++ b/modules/web-console/frontend/app/components/page-configure/services/ConfigureState.js
@@ -32,15 +32,19 @@ export default class ConfigureState {
                 return this._combinedReducer(state, action);
             } catch (e) {
                 console.error(e);
+
                 return state;
             }
         };
+
         this.actions$.scan(reducer, {}).do((v) => this.state$.next(v)).subscribe();
     }
 
     addReducer(combineFn) {
         const old = this._combinedReducer;
+
         this._combinedReducer = (state, action) => combineFn(old(state, action), action);
+
         return this;
     }
 
@@ -49,6 +53,7 @@ export default class ConfigureState {
             return action((a) => this.actions$.next(a), () => this.state$.getValue());
 
         this.actions$.next(action);
+
         return action;
     }
 }
diff --git a/modules/web-console/frontend/app/components/page-configure/states.js b/modules/web-console/frontend/app/components/page-configure/states.js
index a75e851174ffe..ed43f48c47bbc 100644
--- a/modules/web-console/frontend/app/components/page-configure/states.js
+++ b/modules/web-console/frontend/app/components/page-configure/states.js
@@ -15,7 +15,6 @@
  * limitations under the License.
  */
 
-import base2 from 'views/base2.pug';
 import pageConfigureAdvancedClusterComponent from '../page-configure-advanced/components/page-configure-advanced-cluster/component';
 import pageConfigureAdvancedModelsComponent from '../page-configure-advanced/components/page-configure-advanced-models/component';
 import pageConfigureAdvancedCachesComponent from '../page-configure-advanced/components/page-configure-advanced-caches/component';
@@ -26,13 +25,15 @@ import {Observable} from 'rxjs/Observable';
 const idRegex = `new|[a-z0-9]+`;
 
 const shortCachesResolve = ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
-    if ($transition$.params().clusterID === 'new') return Promise.resolve();
+    if ($transition$.params().clusterID === 'new')
+        return Promise.resolve();
+
     return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
-    .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
-    .switchMap((cluster) => {
-        return etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id});
-    })
-    .toPromise();
+        .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+        .switchMap((cluster) => {
+            return etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id});
+        })
+        .toPromise();
 }];
 
 function registerStates($stateProvider) {
@@ -43,11 +44,6 @@ function registerStates($stateProvider) {
         permission: 'configuration',
         url: '/configuration',
         onEnter: ['ConfigureState', (ConfigureState) => ConfigureState.dispatchAction({type: 'PRELOAD_STATE', state: {}})],
-        views: {
-            '@': {
-                template: base2
-            }
-        },
         resolve: {
             _shortClusters: ['ConfigEffects', ({etp}) => {
                 return etp('LOAD_USER_CLUSTERS');
@@ -140,17 +136,19 @@ function registerStates($stateProvider) {
         component: pageConfigureAdvancedCachesComponent.name,
         resolve: {
             _shortCachesAndModels: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
-                if ($transition$.params().clusterID === 'new') return Promise.resolve();
+                if ($transition$.params().clusterID === 'new')
+                    return Promise.resolve();
+
                 return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
-                .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
-                .map((cluster) => {
-                    return Promise.all([
-                        etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
-                        etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id}),
-                        etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
-                    ]);
-                })
-                .toPromise();
+                    .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+                    .map((cluster) => {
+                        return Promise.all([
+                            etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
+                            etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id}),
+                            etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
+                        ]);
+                    })
+                    .toPromise();
             }]
         },
         resolvePolicy: {
@@ -166,7 +164,9 @@ function registerStates($stateProvider) {
         resolve: {
             _cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
                 const {clusterID, cacheID} = $transition$.params();
-                if (cacheID === 'new') return Promise.resolve();
+                if (cacheID === 'new')
+                    return Promise.resolve();
+
                 return etp('LOAD_CACHE', {cacheID});
             }]
         },
@@ -186,16 +186,18 @@ function registerStates($stateProvider) {
         permission: 'configuration',
         resolve: {
             _shortCachesAndModels: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
-                if ($transition$.params().clusterID === 'new') return Promise.resolve();
+                if ($transition$.params().clusterID === 'new')
+                    return Promise.resolve();
+
                 return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
-                .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
-                .map((cluster) => {
-                    return Promise.all([
-                        etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
-                        etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id})
-                    ]);
-                })
-                .toPromise();
+                    .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+                    .map((cluster) => {
+                        return Promise.all([
+                            etp('LOAD_SHORT_CACHES', {ids: cluster.caches, clusterID: cluster._id}),
+                            etp('LOAD_SHORT_MODELS', {ids: cluster.models, clusterID: cluster._id})
+                        ]);
+                    })
+                    .toPromise();
             }]
         },
         resolvePolicy: {
@@ -210,7 +212,10 @@ function registerStates($stateProvider) {
         resolve: {
             _cache: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
                 const {clusterID, modelID} = $transition$.params();
-                if (modelID === 'new') return Promise.resolve();
+
+                if (modelID === 'new')
+                    return Promise.resolve();
+
                 return etp('LOAD_MODEL', {modelID});
             }]
         },
@@ -228,15 +233,17 @@ function registerStates($stateProvider) {
         permission: 'configuration',
         resolve: {
             _shortIGFSs: ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
-                if ($transition$.params().clusterID === 'new') return Promise.resolve();
+                if ($transition$.params().clusterID === 'new')
+                    return Promise.resolve();
+
                 return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
-                .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
-                .map((cluster) => {
-                    return Promise.all([
-                        etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
-                    ]);
-                })
-                .toPromise();
+                    .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
+                    .map((cluster) => {
+                        return Promise.all([
+                            etp('LOAD_SHORT_IGFSS', {ids: cluster.igfss, clusterID: cluster._id})
+                        ]);
+                    })
+                    .toPromise();
             }]
         },
         resolvePolicy: {
@@ -252,7 +259,10 @@ function registerStates($stateProvider) {
         resolve: {
             _igfs: ['ConfigEffects', '$transition$', ({etp}, $transition$) => {
                 const {clusterID, igfsID} = $transition$.params();
-                if (igfsID === 'new') return Promise.resolve();
+
+                if (igfsID === 'new')
+                    return Promise.resolve();
+
                 return etp('LOAD_IGFS', {igfsID});
             }]
         },
diff --git a/modules/web-console/frontend/app/components/page-configure/store/actionCreators.js b/modules/web-console/frontend/app/components/page-configure/store/actionCreators.js
index c9114267b2cd0..d6b174909d1bc 100644
--- a/modules/web-console/frontend/app/components/page-configure/store/actionCreators.js
+++ b/modules/web-console/frontend/app/components/page-configure/store/actionCreators.js
@@ -153,10 +153,10 @@ export const completeConfiguration = (configuration) => ({
     configuration
 });
 
-export const advancedSaveCluster = (cluster) => ({type: ADVANCED_SAVE_CLUSTER, cluster});
-export const advancedSaveCache = (cache) => ({type: ADVANCED_SAVE_CACHE, cache});
-export const advancedSaveIGFS = (igfs) => ({type: ADVANCED_SAVE_IGFS, igfs});
-export const advancedSaveModel = (model) => ({type: ADVANCED_SAVE_MODEL, model});
+export const advancedSaveCluster = (cluster, download = false) => ({type: ADVANCED_SAVE_CLUSTER, cluster, download});
+export const advancedSaveCache = (cache, download = false) => ({type: ADVANCED_SAVE_CACHE, cache, download});
+export const advancedSaveIGFS = (igfs, download = false) => ({type: ADVANCED_SAVE_IGFS, igfs, download});
+export const advancedSaveModel = (model, download = false) => ({type: ADVANCED_SAVE_MODEL, model, download});
 
 export const basicSave = (cluster) => ({type: BASIC_SAVE, cluster});
 export const basicSaveAndDownload = (cluster) => ({type: BASIC_SAVE_AND_DOWNLOAD, cluster});
diff --git a/modules/web-console/frontend/app/components/page-configure/store/effects.js b/modules/web-console/frontend/app/components/page-configure/store/effects.js
index 3ab216aaa5566..9647461776522 100644
--- a/modules/web-console/frontend/app/components/page-configure/store/effects.js
+++ b/modules/web-console/frontend/app/components/page-configure/store/effects.js
@@ -18,49 +18,58 @@
 import {Observable} from 'rxjs/Observable';
 import 'rxjs/add/operator/ignoreElements';
 import 'rxjs/add/operator/let';
+import 'rxjs/add/operator/exhaustMap';
+import 'rxjs/add/operator/switchMap';
+import 'rxjs/add/operator/map';
+import 'rxjs/add/operator/pluck';
+import 'rxjs/add/operator/withLatestFrom';
+import 'rxjs/add/operator/merge';
+import 'rxjs/add/operator/take';
+import 'rxjs/add/operator/catch';
 import 'rxjs/add/operator/zip';
 import {merge} from 'rxjs/observable/merge';
 import {empty} from 'rxjs/observable/empty';
 import {of} from 'rxjs/observable/of';
+import {from} from 'rxjs/observable/from';
 import {fromPromise} from 'rxjs/observable/fromPromise';
-import {uniqueName} from 'app/utils/uniqueName';
 import uniq from 'lodash/uniq';
+import {uniqueName} from 'app/utils/uniqueName';
+import {defaultNames} from '../defaultNames';
 
 import {
-    clustersActionTypes,
     cachesActionTypes,
-    shortClustersActionTypes,
+    clustersActionTypes,
+    igfssActionTypes,
+    modelsActionTypes,
     shortCachesActionTypes,
-    shortModelsActionTypes,
+    shortClustersActionTypes,
     shortIGFSsActionTypes,
-    modelsActionTypes,
-    igfssActionTypes
-} from './../reducer';
+    shortModelsActionTypes
+} from '../reducer';
 
 import {
-    REMOVE_CLUSTER_ITEMS,
-    REMOVE_CLUSTER_ITEMS_CONFIRMED,
-    CONFIRM_CLUSTERS_REMOVAL,
-    CONFIRM_CLUSTERS_REMOVAL_OK,
-    COMPLETE_CONFIGURATION,
-    ADVANCED_SAVE_CLUSTER,
     ADVANCED_SAVE_CACHE,
+    ADVANCED_SAVE_CLUSTER,
+    ADVANCED_SAVE_COMPLETE_CONFIGURATION,
     ADVANCED_SAVE_IGFS,
     ADVANCED_SAVE_MODEL,
     BASIC_SAVE,
     BASIC_SAVE_AND_DOWNLOAD,
-    BASIC_SAVE_OK
+    BASIC_SAVE_OK,
+    COMPLETE_CONFIGURATION,
+    CONFIRM_CLUSTERS_REMOVAL,
+    CONFIRM_CLUSTERS_REMOVAL_OK,
+    REMOVE_CLUSTER_ITEMS,
+    REMOVE_CLUSTER_ITEMS_CONFIRMED
 } from './actionTypes';
 
 import {
-    removeClusterItemsConfirmed,
     advancedSaveCompleteConfiguration,
-    confirmClustersRemoval,
-    confirmClustersRemovalOK,
-    completeConfiguration,
-    basicSave,
+    basicSaveErr,
     basicSaveOK,
-    basicSaveErr
+    completeConfiguration,
+    confirmClustersRemovalOK,
+    removeClusterItemsConfirmed
 } from './actionCreators';
 
 import ConfigureState from 'app/components/page-configure/services/ConfigureState';
@@ -141,7 +150,7 @@ export default class ConfigEffects {
             ].filter((v) => v)));
 
         this.saveCompleteConfigurationEffect$ = this.ConfigureState.actions$
-            .let(ofType('ADVANCED_SAVE_COMPLETE_CONFIGURATION'))
+            .let(ofType(ADVANCED_SAVE_COMPLETE_CONFIGURATION))
             .switchMap((action) => {
                 const actions = [
                     {
@@ -232,10 +241,17 @@ export default class ConfigEffects {
 
         this.loadAndEditClusterEffect$ = ConfigureState.actions$
             .let(ofType('LOAD_AND_EDIT_CLUSTER'))
-            .exhaustMap((a) => {
+            .withLatestFrom(this.ConfigureState.state$.let(this.ConfigSelectors.selectShortClustersValue()))
+            .exhaustMap(([a, shortClusters]) => {
                 if (a.clusterID === 'new') {
                     return of(
-                        {type: 'EDIT_CLUSTER', cluster: this.Clusters.getBlankCluster()},
+                        {
+                            type: 'EDIT_CLUSTER',
+                            cluster: {
+                                ...this.Clusters.getBlankCluster(),
+                                name: uniqueName(defaultNames.cluster, shortClusters)
+                            }
+                        },
                         {type: 'LOAD_AND_EDIT_CLUSTER_OK'}
                     );
                 }
@@ -247,18 +263,18 @@ export default class ConfigEffects {
                                 {type: 'LOAD_AND_EDIT_CLUSTER_OK'}
                             );
                         }
-                        return fromPromise(this.Clusters.getCluster(a.clusterID))
-                        .switchMap(({data}) => of(
-                            {type: clustersActionTypes.UPSERT, items: [data]},
-                            {type: 'EDIT_CLUSTER', cluster: data},
-                            {type: 'LOAD_AND_EDIT_CLUSTER_OK'}
-                        ))
-                        .catch((error) => of({
-                            type: 'LOAD_AND_EDIT_CLUSTER_ERR',
-                            error: {
-                                message: `Failed to load cluster: ${error.data}.`
-                            }
-                        }));
+                        return from(this.Clusters.getCluster(a.clusterID))
+                            .switchMap(({data}) => of(
+                                {type: clustersActionTypes.UPSERT, items: [data]},
+                                {type: 'EDIT_CLUSTER', cluster: data},
+                                {type: 'LOAD_AND_EDIT_CLUSTER_OK'}
+                            ))
+                            .catch((error) => of({
+                                type: 'LOAD_AND_EDIT_CLUSTER_ERR',
+                                error: {
+                                    message: `Failed to load cluster: ${error.data}.`
+                                }
+                            }));
                     });
             });
 
@@ -267,12 +283,14 @@ export default class ConfigEffects {
             .exhaustMap((a) => {
                 return this.ConfigureState.state$.let(this.ConfigSelectors.selectCache(a.cacheID)).take(1)
                     .switchMap((cache) => {
-                        if (cache) return of({type: `${a.type}_OK`, cache});
+                        if (cache)
+                            return of({type: `${a.type}_OK`, cache});
+
                         return fromPromise(this.Caches.getCache(a.cacheID))
-                        .switchMap(({data}) => of(
-                            {type: 'CACHE', cache: data},
-                            {type: `${a.type}_OK`, cache: data}
-                        ));
+                            .switchMap(({data}) => of(
+                                {type: 'CACHE', cache: data},
+                                {type: `${a.type}_OK`, cache: data}
+                            ));
                     })
                     .catch((error) => of({
                         type: `${a.type}_ERR`,
@@ -289,7 +307,9 @@ export default class ConfigEffects {
         this.loadShortCachesEffect$ = ConfigureState.actions$
             .let(ofType('LOAD_SHORT_CACHES'))
             .exhaustMap((a) => {
-                if (!(a.ids || []).length) return of({type: `${a.type}_OK`});
+                if (!(a.ids || []).length)
+                    return of({type: `${a.type}_OK`});
+
                 return this.ConfigureState.state$.let(this.ConfigSelectors.selectShortCaches()).take(1)
                     .switchMap((items) => {
                         if (!items.pristine && a.ids && a.ids.every((_id) => items.value.has(_id)))
@@ -315,12 +335,14 @@ export default class ConfigEffects {
             .exhaustMap((a) => {
                 return this.ConfigureState.state$.let(this.ConfigSelectors.selectIGFS(a.igfsID)).take(1)
                     .switchMap((igfs) => {
-                        if (igfs) return of({type: `${a.type}_OK`, igfs});
+                        if (igfs)
+                            return of({type: `${a.type}_OK`, igfs});
+
                         return fromPromise(this.IGFSs.getIGFS(a.igfsID))
-                        .switchMap(({data}) => of(
-                            {type: 'IGFS', igfs: data},
-                            {type: `${a.type}_OK`, igfs: data}
-                        ));
+                            .switchMap(({data}) => of(
+                                {type: 'IGFS', igfs: data},
+                                {type: `${a.type}_OK`, igfs: data}
+                            ));
                     })
                     .catch((error) => of({
                         type: `${a.type}_ERR`,
@@ -368,12 +390,14 @@ export default class ConfigEffects {
             .exhaustMap((a) => {
                 return this.ConfigureState.state$.let(this.ConfigSelectors.selectModel(a.modelID)).take(1)
                     .switchMap((model) => {
-                        if (model) return of({type: `${a.type}_OK`, model});
+                        if (model)
+                            return of({type: `${a.type}_OK`, model});
+
                         return fromPromise(this.Models.getModel(a.modelID))
-                        .switchMap(({data}) => of(
-                            {type: 'MODEL', model: data},
-                            {type: `${a.type}_OK`, model: data}
-                        ));
+                            .switchMap(({data}) => of(
+                                {type: 'MODEL', model: data},
+                                {type: `${a.type}_OK`, model: data}
+                            ));
                     })
                     .catch((error) => of({
                         type: `${a.type}_ERR`,
@@ -427,9 +451,21 @@ export default class ConfigEffects {
             .do((a) => this.configurationDownload.downloadClusterConfiguration(a.changedItems.cluster))
             .ignoreElements();
 
+        this.advancedDownloadAfterSaveEffect$ = merge(
+            this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_CLUSTER)),
+            this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_CACHE)),
+            this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_IGFS)),
+            this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_MODEL)),
+        )
+            .filter((a) => a.download)
+            .zip(this.ConfigureState.actions$.let(ofType('ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK')))
+            .pluck('1')
+            .do((a) => this.configurationDownload.downloadClusterConfiguration(a.changedItems.cluster))
+            .ignoreElements();
+
         this.advancedSaveRedirectEffect$ = this.ConfigureState.actions$
             .let(ofType('ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK'))
-            .withLatestFrom(this.ConfigureState.actions$.let(ofType('ADVANCED_SAVE_COMPLETE_CONFIGURATION')))
+            .withLatestFrom(this.ConfigureState.actions$.let(ofType(ADVANCED_SAVE_COMPLETE_CONFIGURATION)))
             .pluck('1', 'changedItems')
             .map((req) => {
                 const firstChangedItem = Object.keys(req).filter((k) => k !== 'cluster')
@@ -442,39 +478,48 @@ export default class ConfigEffects {
                 const go = (state, params = {}) => this.$state.go(
                     state, {...params, clusterID: cluster._id}, {location: 'replace', custom: {justIDUpdate: true}}
                 );
+
                 switch (type) {
                     case 'models': {
                         const state = 'base.configuration.edit.advanced.models.model';
                         this.IgniteMessages.showInfo(`Model "${value.valueType}" saved`);
-                        if (
-                            this.$state.is(state) && this.$state.params.modelID !== value._id
-                        ) return go(state, {modelID: value._id});
+
+                        if (this.$state.is(state) && this.$state.params.modelID !== value._id)
+                            return go(state, {modelID: value._id});
+
                         break;
                     }
+
                     case 'caches': {
                         const state = 'base.configuration.edit.advanced.caches.cache';
                         this.IgniteMessages.showInfo(`Cache "${value.name}" saved`);
-                        if (
-                            this.$state.is(state) && this.$state.params.cacheID !== value._id
-                        ) return go(state, {cacheID: value._id});
+
+                        if (this.$state.is(state) && this.$state.params.cacheID !== value._id)
+                            return go(state, {cacheID: value._id});
+
                         break;
                     }
+
                     case 'igfss': {
                         const state = 'base.configuration.edit.advanced.igfs.igfs';
                         this.IgniteMessages.showInfo(`IGFS "${value.name}" saved`);
-                        if (
-                            this.$state.is(state) && this.$state.params.igfsID !== value._id
-                        ) return go(state, {igfsID: value._id});
+
+                        if (this.$state.is(state) && this.$state.params.igfsID !== value._id)
+                            return go(state, {igfsID: value._id});
+
                         break;
                     }
+
                     case 'cluster': {
                         const state = 'base.configuration.edit.advanced.cluster';
                         this.IgniteMessages.showInfo(`Cluster "${value.name}" saved`);
-                        if (
-                            this.$state.is(state) && this.$state.params.clusterID !== value._id
-                        ) return go(state);
+
+                        if (this.$state.is(state) && this.$state.params.clusterID !== value._id)
+                            return go(state);
+
                         break;
                     }
+
                     default: break;
                 }
             })
@@ -485,7 +530,7 @@ export default class ConfigEffects {
             .exhaustMap((a) => {
                 return a.confirm
                     // TODO: list items to remove in confirmation
-                    ? fromPromise(this.Confirm.confirm('Are you sure want to remove these items?'))
+                    ? fromPromise(this.Confirm.confirm('Are you sure you want to remove these items?'))
                         .mapTo(a)
                         .catch(() => empty())
                     : of(a);
@@ -510,7 +555,7 @@ export default class ConfigEffects {
             .switchMap((ids) => this.ConfigureState.state$.let(this.ConfigSelectors.selectClusterNames(ids)).take(1))
             .exhaustMap((names) => {
                 return fromPromise(this.Confirm.confirm(`
-                    <p>Are you sure want to remove these clusters?</p>
+                    <p>Are you sure you want to remove these clusters?</p>
                     <ul>${names.map((name) => `<li>${name}</li>`).join('')}</ul>
                 `))
                 .map(confirmClustersRemovalOK)
@@ -577,7 +622,7 @@ export default class ConfigEffects {
         )
             .withLatestFrom(this.ConfigureState.state$.pluck('edit'))
             .map(([action, edit]) => ({
-                type: 'ADVANCED_SAVE_COMPLETE_CONFIGURATION',
+                type: ADVANCED_SAVE_COMPLETE_CONFIGURATION,
                 changedItems: _applyChangedIDs(edit, action)
             }));
 
@@ -644,6 +689,7 @@ export default class ConfigEffects {
         const err = `${action.type}_ERR`;
 
         setTimeout(() => this.ConfigureState.dispatchAction(action));
+
         return this.ConfigureState.actions$
             .filter((a) => a.type === ok || a.type === err)
             .take(1)
diff --git a/modules/web-console/frontend/app/components/page-configure/store/effects.spec.js b/modules/web-console/frontend/app/components/page-configure/store/effects.spec.js
new file mode 100644
index 0000000000000..10e066330a795
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-configure/store/effects.spec.js
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {assert} from 'chai';
+import {of} from 'rxjs/observable/of';
+import {_throw} from 'rxjs/observable/throw';
+import {default as Effects} from './effects';
+import {default as Selectors} from './selectors';
+import {TestScheduler} from 'rxjs/testing/TestScheduler';
+
+const makeMocks = (target, mocks) => new Map(target.$inject.map((provider) => {
+    return (provider in mocks) ? [provider, mocks[provider]] : [provider, {}];
+}));
+
+suite('Configuration store effects', () => {
+    suite('Load and edit cluster', () => {
+        const actionValues = {
+            a: {type: 'LOAD_AND_EDIT_CLUSTER', clusterID: 'new'},
+            b: {type: 'LOAD_AND_EDIT_CLUSTER', clusterID: '1'},
+            c: {type: 'LOAD_AND_EDIT_CLUSTER', clusterID: '2'}
+        };
+
+        const stateValues = {
+            A: {
+                shortClusters: {value: new Map()},
+                clusters: new Map()
+            },
+            B: {
+                shortClusters: {value: new Map([['1', {id: '1', name: 'Cluster'}]])},
+                clusters: new Map([['1', {id: '1', name: 'Cluster'}]])
+            }
+        };
+
+        const setup = ({actionMarbles, stateMarbles, mocks}) => {
+            const testScheduler = new TestScheduler((actual, expected) => assert.deepEqual(actual, expected));
+            const mocksMap = makeMocks(Effects, {
+                ...mocks,
+                ConfigureState: (() => {
+                    const actions$ = testScheduler.createHotObservable(actionMarbles, actionValues);
+                    const state$ = testScheduler.createHotObservable(stateMarbles, stateValues);
+                    return {actions$, state$};
+                })()
+            });
+
+            const effects = new Effects(...mocksMap.values());
+
+            return {testScheduler, effects};
+        };
+
+        const mocks = {
+            Clusters: {
+                getBlankCluster: () => ({id: 'foo'}),
+                getCluster: (id) => of({data: {id}})
+            },
+            ConfigSelectors: new Selectors()
+        };
+
+        test('New cluster', () => {
+            const {testScheduler, effects} = setup({
+                actionMarbles: '-a',
+                stateMarbles: 'B-',
+                mocks
+            });
+
+            testScheduler.expectObservable(effects.loadAndEditClusterEffect$).toBe('-(ab)', {
+                a: {type: 'EDIT_CLUSTER', cluster: {id: 'foo', name: 'Cluster1'}},
+                b: {type: 'LOAD_AND_EDIT_CLUSTER_OK'}
+            });
+
+            testScheduler.flush();
+        });
+
+        test('Cached cluster', () => {
+            const {testScheduler, effects} = setup({
+                actionMarbles: '-b',
+                stateMarbles: 'AB',
+                mocks
+            });
+
+            testScheduler.expectObservable(effects.loadAndEditClusterEffect$).toBe('-(ab)', {
+                a: {type: 'EDIT_CLUSTER', cluster: {id: '1', name: 'Cluster'}},
+                b: {type: 'LOAD_AND_EDIT_CLUSTER_OK'}
+            });
+
+            testScheduler.flush();
+        });
+
+        test('Cluster from server, success', () => {
+            const {testScheduler, effects} = setup({
+                actionMarbles: '-c',
+                stateMarbles: 'AB',
+                mocks
+            });
+
+            testScheduler.expectObservable(effects.loadAndEditClusterEffect$).toBe('-(abc)', {
+                a: {type: 'UPSERT_CLUSTERS', items: [{id: '2'}]},
+                b: {type: 'EDIT_CLUSTER', cluster: {id: '2'}},
+                c: {type: 'LOAD_AND_EDIT_CLUSTER_OK'}
+            });
+
+            testScheduler.flush();
+        });
+
+        test('Cluster from server, error', () => {
+            const {testScheduler, effects} = setup({
+                actionMarbles: '-c',
+                stateMarbles: 'AB',
+                mocks: {
+                    ...mocks,
+                    Clusters: {getCluster: () => _throw({data: 'Error'})}
+                }
+            });
+
+            testScheduler.expectObservable(effects.loadAndEditClusterEffect$).toBe('-a', {
+                a: {type: 'LOAD_AND_EDIT_CLUSTER_ERR', error: {message: `Failed to load cluster: Error.`}}
+            });
+
+            testScheduler.flush();
+        });
+    });
+});
diff --git a/modules/web-console/frontend/app/components/page-configure/store/selectors.js b/modules/web-console/frontend/app/components/page-configure/store/selectors.js
index 0f60214f9bb95..c9df6436ca676 100644
--- a/modules/web-console/frontend/app/components/page-configure/store/selectors.js
+++ b/modules/web-console/frontend/app/components/page-configure/store/selectors.js
@@ -19,6 +19,7 @@ import {uniqueName} from 'app/utils/uniqueName';
 import {of} from 'rxjs/observable/of';
 import {empty} from 'rxjs/observable/empty';
 import {combineLatest} from 'rxjs/observable/combineLatest';
+import 'rxjs/add/operator/filter';
 import 'rxjs/add/operator/mergeMap';
 import 'rxjs/add/observable/combineLatest';
 import {Observable} from 'rxjs/Observable';
@@ -30,23 +31,37 @@ import {default as IGFSs} from 'app/services/IGFSs';
 import {default as Models} from 'app/services/Models';
 
 const isDefined = (s) => s.filter((v) => v);
+
 const selectItems = (path) => (s) => s.filter((s) => s).pluck(path).filter((v) => v);
+
 const selectValues = (s) => s.map((v) => v && [...v.value.values()]);
+
 export const selectMapItem = (mapPath, key) => (s) => s.pluck(mapPath).map((v) => v && v.get(key));
+
 const selectMapItems = (mapPath, keys) => (s) => s.pluck(mapPath).map((v) => v && keys.map((key) => v.get(key)));
+
 const selectItemToEdit = ({items, itemFactory, defaultName = '', itemID}) => (s) => s.switchMap((item) => {
-    if (item) return of(Object.assign(itemFactory(), item));
-    if (itemID === 'new') return items.take(1).map((items) => Object.assign(itemFactory(), {name: uniqueName(defaultName, items)}));
-    if (!itemID) return of(null);
+    if (item)
+        return of(Object.assign(itemFactory(), item));
+
+    if (itemID === 'new')
+        return items.take(1).map((items) => Object.assign(itemFactory(), {name: uniqueName(defaultName, items)}));
+
+    if (!itemID)
+        return of(null);
+
     return empty();
 });
+
 const currentShortItems = ({changesKey, shortKey}) => (state$) => {
     return Observable.combineLatest(
         state$.pluck('edit', 'changes', changesKey).let(isDefined).distinctUntilChanged(),
         state$.pluck(shortKey, 'value').let(isDefined).distinctUntilChanged()
     )
         .map(([{ids = [], changedItems}, shortItems]) => {
-            if (!ids.length || !shortItems) return [];
+            if (!ids.length || !shortItems)
+                return [];
+
             return ids.map((id) => changedItems.find(({_id}) => _id === id) || shortItems.get(id));
         })
         .map((v) => v.filter((v) => v));
@@ -58,6 +73,7 @@ const selectNames = (itemIDs, nameAt = 'name') => (items) => items
 
 export default class ConfigSelectors {
     static $inject = ['Caches', 'Clusters', 'IGFSs', 'Models'];
+
     /**
      * @param {Caches} Caches
      * @param {Clusters} Clusters
@@ -91,15 +107,25 @@ export default class ConfigSelectors {
             .let(this.selectShortClusters())
             .let(selectNames(clusterIDs));
     }
+
     selectCluster = (id) => selectMapItem('clusters', id);
+
     selectShortClusters = () => selectItems('shortClusters');
+
     selectCache = (id) => selectMapItem('caches', id);
+
     selectIGFS = (id) => selectMapItem('igfss', id);
+
     selectShortCaches = () => selectItems('shortCaches');
+
     selectShortCachesValue = () => (state$) => state$.let(this.selectShortCaches()).let(selectValues);
+
     selectShortIGFSs = () => selectItems('shortIgfss');
+
     selectShortIGFSsValue = () => (state$) => state$.let(this.selectShortIGFSs()).let(selectValues);
+
     selectShortModelsValue = () => (state$) => state$.let(this.selectShortModels()).let(selectValues);
+
     selectCacheToEdit = (cacheID) => (state$) => state$
         .let(this.selectCache(cacheID))
         .distinctUntilChanged()
@@ -109,6 +135,7 @@ export default class ConfigSelectors {
             defaultName: defaultNames.cache,
             itemID: cacheID
         }));
+
     selectIGFSToEdit = (itemID) => (state$) => state$
         .let(this.selectIGFS(itemID))
         .distinctUntilChanged()
@@ -118,6 +145,7 @@ export default class ConfigSelectors {
             defaultName: defaultNames.igfs,
             itemID
         }));
+
     selectModelToEdit = (itemID) => (state$) => state$
         .let(this.selectModel(itemID))
         .distinctUntilChanged()
@@ -126,6 +154,7 @@ export default class ConfigSelectors {
             itemFactory: () => this.Models.getBlankModel(),
             itemID
         }));
+
     selectClusterToEdit = (clusterID, defaultName = defaultNames.cluster) => (state$) => state$
         .let(this.selectCluster(clusterID))
         .distinctUntilChanged()
@@ -135,23 +164,33 @@ export default class ConfigSelectors {
             defaultName,
             itemID: clusterID
         }));
+
     selectCurrentShortCaches = currentShortItems({changesKey: 'caches', shortKey: 'shortCaches'});
+
     selectCurrentShortIGFSs = currentShortItems({changesKey: 'igfss', shortKey: 'shortIgfss'});
+
     selectCurrentShortModels = currentShortItems({changesKey: 'models', shortKey: 'shortModels'});
+
     selectClusterShortCaches = (clusterID) => (state$) => {
-        if (clusterID === 'new') return of([]);
+        if (clusterID === 'new')
+            return of([]);
+
         return combineLatest(
             state$.let(this.selectCluster(clusterID)).pluck('caches'),
             state$.let(this.selectShortCaches()).pluck('value'),
             (ids, items) => ids.map((id) => items.get(id))
         );
     };
+
     selectCompleteClusterConfiguration = ({clusterID, isDemo}) => (state$) => {
         const hasValues = (array) => !array.some((v) => !v);
         return state$.let(this.selectCluster(clusterID))
         .exhaustMap((cluster) => {
-            if (!cluster) return of({__isComplete: false});
+            if (!cluster)
+                return of({__isComplete: false});
+
             const withSpace = (array) => array.map((c) => ({...c, space: cluster.space}));
+
             return Observable.forkJoin(
                 state$.let(selectMapItems('caches', cluster.caches || [])).take(1),
                 state$.let(selectMapItems('models', cluster.models || [])).take(1),
diff --git a/modules/web-console/frontend/app/components/page-configure/style.scss b/modules/web-console/frontend/app/components/page-configure/style.scss
index 94a3ebcfe62e7..36ae7524da51e 100644
--- a/modules/web-console/frontend/app/components/page-configure/style.scss
+++ b/modules/web-console/frontend/app/components/page-configure/style.scss
@@ -34,6 +34,7 @@ page-configure {
         flex-direction: row;
         padding: 10px 20px 10px 30px;
         box-shadow: 0 0px 4px 0 rgba(0, 0, 0, 0.2), 0px 3px 4px -1px rgba(0, 0, 0, 0.2);
+        position: -webkit-sticky;
         position: sticky;
         bottom: 0px;
         // margin: 20px -30px -30px;
@@ -186,6 +187,22 @@ list-editable .pc-form-group__text-title {
     }
 }
 
+// Add this class to list-editable when a pc-form-grid is used inside list-editable-item-edit
+.pc-list-editable-with-form-grid > .le-body > .le-row {
+    & > .le-row-index,
+    & > .le-row-cross {
+        margin-top: 28px;
+    }
+}
+
+// Add this class to list-editable when legacy settings-row classes are used inside list-editable-item-edit
+.pc-list-editable-with-legacy-settings-rows > .le-body > .le-row {
+    & > .le-row-index,
+    & > .le-row-cross {
+        margin-top: 18px;
+    }
+}
+
 .pc-form-grid-row {
     &>.pc-form-grid-col-10 {
         flex-basis: calc(100% / 6);
@@ -207,6 +224,10 @@ list-editable .pc-form-group__text-title {
         flex-basis: calc(100% / 1);
     }
 
+    &>.pc-form-grid-col-free {
+        flex: 1;
+    }
+
     @media(max-width: 992px) {
         &>.pc-form-grid-col-10 {
             flex-basis: calc(25%);
diff --git a/modules/web-console/frontend/app/components/page-configure/template.pug b/modules/web-console/frontend/app/components/page-configure/template.pug
index c56320b10ca5e..bd299bec740cf 100644
--- a/modules/web-console/frontend/app/components/page-configure/template.pug
+++ b/modules/web-console/frontend/app/components/page-configure/template.pug
@@ -14,11 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-h1.pc-page-header
-    span.pc-page-header-title
-        | {{ $ctrl.clusterName$|async:this }}
+header.header-with-selector
+    div
+        h1 {{ $ctrl.clusterName$|async:this }}
         version-picker
-    button-import-models(cluster-id='$ctrl.clusterID$|async:this')
+    div
+        button-import-models(cluster-id='$ctrl.clusterID$|async:this')
+
 div.pc-content-container
     ul.tabs.tabs--blue
         li(role='presentation' ui-sref-active='active')
diff --git a/modules/web-console/frontend/app/components/page-configure/transitionHooks/errorState.js b/modules/web-console/frontend/app/components/page-configure/transitionHooks/errorState.js
index f36a9318ea2a2..fd9ca9bbce028 100644
--- a/modules/web-console/frontend/app/components/page-configure/transitionHooks/errorState.js
+++ b/modules/web-console/frontend/app/components/page-configure/transitionHooks/errorState.js
@@ -44,9 +44,12 @@ const getResolvePromises = ($transition) => $transition.getResolveTokens()
  */
 export const errorState = ($uiRouter) => {
     $uiRouter.transitionService.onError(match, ($transition) => {
-        if ($transition.error().type !== RejectType.ERROR) return;
+        if ($transition.error().type !== RejectType.ERROR)
+            return;
+
         go($transition);
     });
+
     $uiRouter.transitionService.onStart(match, ($transition) => {
         Promise.all(getResolvePromises($transition)).catch((e) => go($transition));
     });
diff --git a/modules/web-console/frontend/app/components/page-forgot-password/component.js b/modules/web-console/frontend/app/components/page-forgot-password/component.js
new file mode 100644
index 0000000000000..8c70e73786a53
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-forgot-password/component.js
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+import controller from './controller';
+
+import './style.scss';
+
+/** @type {ng.IComponentOptions} */
+export default {
+    controller,
+    template,
+    bindings: {
+        email: '<?'
+    }
+};
diff --git a/modules/web-console/frontend/app/components/page-forgot-password/controller.js b/modules/web-console/frontend/app/components/page-forgot-password/controller.js
new file mode 100644
index 0000000000000..7d06773c58a72
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-forgot-password/controller.js
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class PageForgotPassword {
+    /** @type {string} Optional email to populate form with */
+    email;
+    /** @type {import('./types').IForgotPasswordFormController} */
+    form;
+    /** @type {import('./types').IForgotPasswordData} */
+    data = {email: null};
+    /** @type {string} */
+    serverError = null;
+
+    static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils'];
+
+    /**
+     * @param {import('app/modules/user/Auth.service').default} Auth
+     */
+    constructor(Auth, IgniteMessages, IgniteFormUtils) {
+        this.Auth = Auth;
+        this.IgniteMessages = IgniteMessages;
+        this.IgniteFormUtils = IgniteFormUtils;
+    }
+    /** @param {import('./types').IForgotPasswordFormController} form */
+    canSubmitForm(form) {
+        return form.$error.server ? true : !form.$invalid;
+    }
+    $postLink() {
+        this.form.email.$validators.server = () => !this.serverError;
+    }
+
+    /** @param {string} error */
+    setServerError(error) {
+        this.serverError = error;
+        this.form.email.$validate();
+    }
+
+    /**
+     * @param {{email: ng.IChangesObject<string>}} changes
+     */
+    $onChanges(changes) {
+        if ('email' in changes) this.data.email = changes.email.currentValue;
+    }
+    remindPassword() {
+        this.IgniteFormUtils.triggerValidation(this.form);
+
+        this.setServerError(null);
+
+        if (!this.canSubmitForm(this.form))
+            return;
+
+        return this.Auth.remindPassword(this.data.email).catch((res) => {
+            this.IgniteMessages.showError(null, res.data);
+            this.setServerError(res.data);
+        });
+    }
+}
diff --git a/modules/web-console/frontend/app/components/page-forgot-password/index.js b/modules/web-console/frontend/app/components/page-forgot-password/index.js
new file mode 100644
index 0000000000000..dec8fd9f82cad
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-forgot-password/index.js
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import component from './component';
+import {registerState} from './run';
+
+export default angular
+    .module('ignite-console.page-forgot-password', [
+        'ui.router',
+        'ignite-console.user'
+    ])
+    .component('pageForgotPassword', component)
+    .run(registerState);
diff --git a/modules/web-console/frontend/app/components/page-forgot-password/run.js b/modules/web-console/frontend/app/components/page-forgot-password/run.js
new file mode 100644
index 0000000000000..c7ee6a64a1d3f
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-forgot-password/run.js
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @param {import("@uirouter/angularjs").UIRouter} $uiRouter
+ */
+export function registerState($uiRouter) {
+    /** @type {import("app/types").IIgniteNg1StateDeclaration} */
+    const state = {
+        name: 'forgotPassword',
+        url: '/forgot-password',
+        component: 'pageForgotPassword',
+        unsaved: true,
+        tfMetaTags: {
+            title: 'Forgot Password'
+        },
+        resolve: [
+            {
+                token: 'email',
+                deps: ['$uiRouter'],
+                /**
+                 * @param {import('@uirouter/angularjs').UIRouter} $uiRouter
+                 */
+                resolveFn($uiRouter) {
+                    return $uiRouter.stateService.transition.targetState().params().email;
+                }
+            }
+        ]
+    };
+
+    $uiRouter.stateRegistry.register(state);
+}
+
+registerState.$inject = ['$uiRouter'];
diff --git a/modules/web-console/frontend/app/components/page-forgot-password/style.scss b/modules/web-console/frontend/app/components/page-forgot-password/style.scss
new file mode 100644
index 0000000000000..86425c189806e
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-forgot-password/style.scss
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+page-forgot-password {
+    display: flex;
+    flex-direction: column;
+    flex: 1 0 auto;
+
+    section {
+        margin-left: auto;
+        margin-right: auto;
+        width: 530px;
+
+        h3 {
+            font-size: 38px;
+            font-weight: 300;
+            margin: 30px 0 30px;
+        }
+
+        p {
+            margin-bottom: 20px;
+        }
+
+        .form-field {
+            margin: 10px 0;
+        }
+
+        .form-footer {
+            padding: 15px 0;
+            text-align: right;
+            display: flex;
+            align-items: center;
+
+            .btn-ignite {
+                margin-left: auto;
+            }
+        }
+    }
+}
diff --git a/modules/web-console/frontend/app/components/page-forgot-password/template.pug b/modules/web-console/frontend/app/components/page-forgot-password/template.pug
new file mode 100644
index 0000000000000..82e7898599873
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-forgot-password/template.pug
@@ -0,0 +1,49 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins
+
+web-console-header
+    web-console-header-left
+        ignite-header-title
+
+.container--responsive.body-container
+    section
+        - const form = '$ctrl.form'
+        h3 Forgot password?
+        p Enter the email address for your account & we'll email you a link to reset your password.
+        form(name=form novalidate ng-submit='$ctrl.remindPassword()')
+            +form-field__email({
+                label: 'Email:',
+                model: '$ctrl.data.email',
+                name: '"email"',
+                placeholder: 'Input email',
+                required: true
+            })(
+                ng-model-options='{allowInvalid: true}'
+                autocomplete='email'
+                ignite-auto-focus
+                tabindex='0'
+            )
+                +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`})
+            footer.form-footer
+                a(ui-sref='signin') Back to sign in
+                button.btn-ignite.btn-ignite--primary(
+                    tabindex='1'
+                    type='submit'
+                ) Send it to me
+
+web-console-footer
diff --git a/modules/web-console/frontend/app/components/page-forgot-password/types.ts b/modules/web-console/frontend/app/components/page-forgot-password/types.ts
new file mode 100644
index 0000000000000..cbcff81109b20
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-forgot-password/types.ts
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface IForgotPasswordData {
+    email: string
+}
+
+export interface IForgotPasswordFormController extends ng.IFormController {
+    email: ng.INgModelController
+}
diff --git a/modules/web-console/frontend/app/components/page-landing/template.pug b/modules/web-console/frontend/app/components/page-landing/template.pug
index a1ce4f71d614b..0fc1f9c5ffb01 100644
--- a/modules/web-console/frontend/app/components/page-landing/template.pug
+++ b/modules/web-console/frontend/app/components/page-landing/template.pug
@@ -25,42 +25,42 @@ section.intro-container-wrapper
         .col-lg-6.col-md-6.col-sm-6.col-xs-12.intro-content
             h1  Web Console
             h2 An Interactive Configuration Wizard and Management Tool for Apache™ Ignite®
-            p It provides an interactive configuration wizard which helps you create and download configuration files and code snippets for your Apache Ignite projects. Additionally, the tool allows you to automatically load SQL metadata from any RDBMS, run SQL queries on your in-memory cache as well as view execution plans, in-memory schema, and streaming charts.
-            a#signup_show.btn.btn-lg.btn-primary.btn-custom(ui-sref='signin') Try Now
+            p It provides an interactive configuration wizard which helps you create and download configuration files and code snippets for your Apache Ignite projects. Additionally, the tool allows you to automatically load SQL metadata from any RDBMS, run SQL queries on your in-memory cache, and view execution plans, in-memory schema, and streaming charts.
+            a#signup_show.btn.btn-lg.btn-primary.btn-custom(ui-sref='signup') Sign Up
         .col-lg-6.col-md-6.col-sm-6.col-xs-12
             ui-carousel(slides='$ctrl.images' autoplay='true' slides-to-show='1' arrows='false')
                 carousel-item
                     img(ng-src='{{item}}')
 section.features-container-wrapper
     .container.features-container
-        .section-title  Web Console allows you to:
+        .section-title The Web Console allows you to:
         .row
             .col-lg-6.col-md-6.col-sm-6.col-xs-12.feature
                 .col-lg-2.col-md-2.col-sm-2.col-xs-2
                     i.fa.fa-sitemap
                 .col-lg-9.col-md-9.col-sm-9.col-xs-9
-                    h3 Configure Apache Ignite Clusters and Caches
-                    p The Web Console configuration wizard takes you through a step-by-step process to define all of your required configuration parameters. The system then generates a ready-to-use project with all of the required config files for you.
+                    h3 Configure Apache Ignite clusters and caches
+                    p The Web Console configuration wizard takes you through a step-by-step process that helps you define all the required configuration parameters. The system then generates a ready-to-use project with all the required config files.
             .col-lg-6.col-md-6.col-sm-6.col-xs-12.feature
                 .col-lg-2.col-md-2.col-sm-2.col-xs-2
                     i.fa.fa-search
                 .col-lg-9.col-md-9.col-sm-9.col-xs-9
-                    h3 Run Free Form SQL Queries on #[br] Apache Ignite Caches
-                    p By connecting The Web Console to your Apache Ignite cluster, you can execute SQL queries on your in-memory cache. You can also view the execution plan, in-memory schema, and streaming charts for your cluster.
+                    h3 Run free-form SQL queries on #[br] Apache Ignite caches
+                    p By connecting the Web Console to your Apache Ignite cluster, you can execute SQL queries on your in-memory cache. You can also view the execution plan, in-memory schema, and streaming charts for your cluster.
         .row
             .col-lg-6.col-md-6.col-sm-6.col-xs-12.feature
                 .col-lg-2.col-md-2.col-sm-2.col-xs-2
                     i.fa.fa-database
                 .col-lg-9.col-md-9.col-sm-9.col-xs-9
-                    h3 Import Database Schemas from #[br] Virtually Any RDBMS
-                    p To speed the creation of your configuration files, The Web Console allows you to automatically import the database schema from any current RDBMS.  and Apache Ignite support virtually any RDBMS including Oracle, SAP, MySQL, PostgreSQL and many more.
+                    h3 Import database schemas from #[br] virtually any RDBMS
+                    p To speed the creation of your configuration files, the Web Console allows you to automatically import the database schema from virtually any RDBMS including Oracle, SAP, MySQL, PostgreSQL, and many more.
             .col-lg-6.col-md-6.col-sm-6.col-xs-12.feature
                 .col-lg-2.col-md-2.col-sm-2.col-xs-2
                     i.fa.fa-gears
                 .col-lg-9.col-md-9.col-sm-9.col-xs-9
-                    h3 Manage The Web Console users
-                    p With The Web Console you can have accounts with different roles.
+                    h3 Manage the Web Console users
+                    p The Web Console allows you to have accounts with different roles.
         .align-center.text-center
-            a.btn.btn-lg.btn-primary.btn-custom(ui-sref='signin') Get Started
+            a.btn.btn-lg.btn-primary.btn-custom(ui-sref='signup') Get Started
 
 web-console-footer
diff --git a/modules/web-console/frontend/app/components/page-password-changed/style.scss b/modules/web-console/frontend/app/components/page-password-changed/style.scss
index c4f3d83c9fb50..eeb44ec451eab 100644
--- a/modules/web-console/frontend/app/components/page-password-changed/style.scss
+++ b/modules/web-console/frontend/app/components/page-password-changed/style.scss
@@ -39,4 +39,4 @@ page-password-changed {
         font-size: 16px;
         text-align: center;
     }
-}    
+}
diff --git a/modules/web-console/frontend/app/components/page-profile/controller.js b/modules/web-console/frontend/app/components/page-profile/controller.js
index c67a603d6a72b..2c786f029b1ee 100644
--- a/modules/web-console/frontend/app/components/page-profile/controller.js
+++ b/modules/web-console/frontend/app/components/page-profile/controller.js
@@ -40,7 +40,7 @@ export default class PageProfileController {
     }
 
     generateToken() {
-        this.Confirm.confirm('Are you sure you want to change security token?')
+        this.Confirm.confirm('Are you sure you want to change security token?<br>If you change the token you will need to restart the agent.')
             .then(() => this.ui.user.token = this.LegacyUtils.randomString(20));
     }
 
diff --git a/modules/web-console/frontend/app/components/page-profile/index.js b/modules/web-console/frontend/app/components/page-profile/index.js
index faa3c210823de..d8b1a53db7fa6 100644
--- a/modules/web-console/frontend/app/components/page-profile/index.js
+++ b/modules/web-console/frontend/app/components/page-profile/index.js
@@ -16,10 +16,7 @@
  */
 
 import angular from 'angular';
-
 import component from './component';
-import template from 'views/base2.pug';
-
 import './style.scss';
 
 export default angular
@@ -30,14 +27,7 @@ export default angular
         // set up the states
         $stateProvider.state('base.settings.profile', {
             url: '/profile',
-            views: {
-                '@': {
-                    template
-                },
-                '@base.settings.profile': {
-                    component: 'pageProfile'
-                }
-            },
+            component: 'pageProfile',
             permission: 'profile',
             tfMetaTags: {
                 title: 'User profile'
diff --git a/modules/web-console/frontend/app/components/page-profile/style.scss b/modules/web-console/frontend/app/components/page-profile/style.scss
index 8c9387f6dce18..30dd9437a42f3 100644
--- a/modules/web-console/frontend/app/components/page-profile/style.scss
+++ b/modules/web-console/frontend/app/components/page-profile/style.scss
@@ -16,6 +16,9 @@
  */
 
 page-profile {
+    max-width: 800px;
+    display: block;
+
     panel-collapsible {
         width: 100%;
     }
diff --git a/modules/web-console/frontend/app/components/page-profile/template.pug b/modules/web-console/frontend/app/components/page-profile/template.pug
index eafde6bd7e9ec..2ec6c90c78386 100644
--- a/modules/web-console/frontend/app/components/page-profile/template.pug
+++ b/modules/web-console/frontend/app/components/page-profile/template.pug
@@ -24,7 +24,7 @@ div
     -var form = '$ctrl.form'
     form.theme--ignite(name='$ctrl.form' novalidate)
         .row
-            .col-25
+            .col-50
                 +form-field__text({
                     label: 'First name:',
                     model: '$ctrl.ui.user.firstName',
@@ -35,7 +35,7 @@ div
                 ignite-auto-focus
                     ignite-on-enter-focus-move='lastNameInput'
                 )
-            .col-25
+            .col-50
                 +form-field__text({
                     label: 'Last name:',
                     model: '$ctrl.ui.user.lastName',
@@ -46,7 +46,7 @@ div
                     ignite-on-enter-focus-move='emailInput'
                 )
         .row
-            .col-50
+            .col-100
                 +form-field__email({
                     label: 'Email:',
                     model: '$ctrl.ui.user.email',
@@ -57,7 +57,7 @@ div
                     ignite-on-enter-focus-move='phoneInput'
                 )
         .row
-            .col-25
+            .col-50
                 +form-field__phone({
                     label: 'Phone:',
                     model: '$ctrl.ui.user.phone',
@@ -67,7 +67,7 @@ div
                 })(
                     ignite-on-enter-focus-move='companyInput'
                 )
-            .col-25
+            .col-50
                 +form-field__dropdown({
                     label: 'Country:',
                     model: '$ctrl.ui.user.country',
@@ -77,7 +77,7 @@ div
                     options: '$ctrl.ui.countries'
                 })
         .row
-            .col-50
+            .col-100
                 +form-field__text({
                     label: 'Company:',
                     model: '$ctrl.ui.user.company',
@@ -89,7 +89,7 @@ div
                 )
 
         .row#security-token-section
-            .col-50
+            .col-100
                 panel-collapsible(
                     opened='$ctrl.ui.expandedToken'
                     on-open='$ctrl.ui.expandedToken = true'
@@ -99,17 +99,23 @@ div
                         | {{ $panel.opened ? 'Cancel security token changing...' : 'Show security token...' }}
                     panel-content
                         .row
-                            .col-25
-                                label.required Security token:
-                            .col-75
-                                label#current-security-token {{$ctrl.ui.user.token || 'No security token. Regenerate please.'}}
-                                label
-                                    i.tipLabel.fa.fa-refresh(ng-click='$ctrl.generateToken()' bs-tooltip='' data-title='Generate random security token')
-                                    i.tipLabel.fa.fa-clipboard(ignite-copy-to-clipboard='{{$ctrl.ui.user.token}}' bs-tooltip='' data-title='Copy security token to clipboard')
-                                    i.tipLabel.icon-help(ng-if=lines bs-tooltip='' data-title='The security token is used for authorization of web agent')
+                            .col-50
+                                +form-field__text({
+                                    label: 'Security Token:',
+                                    model: '$ctrl.ui.user.token',
+                                    tip: 'The security token is used for authentication of Web agent',
+                                    name: '"securityToken"',
+                                    placeholder: 'No security token. Regenerate please.'
+                                })(
+                                    autocomplete='security-token'
+                                    ng-disabled='::true'
+                                    copy-input-value-button='Copy security token to clipboard'
+                                )
+                            .col-50
+                                a(ng-click='$ctrl.generateToken()') Generate Random Security Token?
 
         .row
-            .col-50
+            .col-100
                 panel-collapsible(
                     opened='$ctrl.ui.expandedPassword'
                     on-open='$ctrl.ui.expandedToken = false'
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
index fa7460f98859d..c887f4db27507 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
@@ -41,6 +41,8 @@ const ROW_IDX = {value: -2, type: 'java.lang.Integer', label: 'ROW_IDX'};
 
 const NON_COLLOCATED_JOINS_SINCE = '1.7.0';
 
+const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], '2.5.2'];
+
 const ENFORCE_JOIN_SINCE = [['1.7.9', '1.8.0'], ['1.8.4', '1.9.0'], '1.9.1'];
 
 const LAZY_QUERY_SINCE = [['2.1.4-p1', '2.2.0'], '2.2.1'];
@@ -64,7 +66,7 @@ const _fullColName = (col) => {
 let paragraphId = 0;
 
 class Paragraph {
-    constructor($animate, $timeout, JavaTypes, paragraph) {
+    constructor($animate, $timeout, JavaTypes, errorParser, paragraph) {
         const self = this;
 
         self.id = 'paragraph-' + paragraphId++;
@@ -144,14 +146,14 @@ class Paragraph {
 
         this.setError = (err) => {
             this.error.root = err;
-            this.error.message = err.message;
+            this.error.message = errorParser.extractMessage(err);
 
             let cause = err;
 
             while (nonNil(cause)) {
                 if (nonEmpty(cause.className) &&
                     _.includes(['SQLException', 'JdbcSQLException', 'QueryCancelledException'], JavaTypes.shortClassName(cause.className))) {
-                    this.error.message = cause.message || cause.className;
+                    this.error.message = errorParser.extractMessage(cause.message || cause.className);
 
                     break;
                 }
@@ -249,16 +251,16 @@ class Paragraph {
 
 // Controller for SQL notebook screen.
 export class NotebookCtrl {
-    static $inject = ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'AgentManager', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData', 'JavaTypes', 'IgniteCopyToClipboard', CSV.name];
+    static $inject = ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'AgentManager', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData', 'JavaTypes', 'IgniteCopyToClipboard', CSV.name, 'IgniteErrorParser'];
 
     /**
      * @param {CSV} CSV
      */
-    constructor($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMgr, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes, IgniteCopyToClipboard, CSV) {
+    constructor($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMgr, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes, IgniteCopyToClipboard, CSV, errorParser) {
         const $ctrl = this;
 
         this.CSV = CSV;
-        Object.assign(this, { $root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMgr, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes });
+        Object.assign(this, { $root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMgr, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes, errorParser });
 
         // Define template urls.
         $ctrl.paragraphRateTemplateUrl = paragraphRateTemplateUrl;
@@ -591,7 +593,7 @@ export class NotebookCtrl {
                         showControls: true,
                         legend: {
                             vers: 'furious',
-                            margin: {right: -25}
+                            margin: {right: -15}
                         }
                     }
                 };
@@ -657,9 +659,7 @@ export class NotebookCtrl {
                             donutRatio: 0.35,
                             legend: {
                                 vers: 'furious',
-                                margin: {
-                                    right: -25
-                                }
+                                margin: {right: -15}
                             }
                         },
                         title: {
@@ -699,9 +699,7 @@ export class NotebookCtrl {
                         useInteractiveGuideline: true,
                         legend: {
                             vers: 'furious',
-                            margin: {
-                                right: -25
-                            }
+                            margin: {right: -15}
                         }
                     }
                 };
@@ -743,7 +741,7 @@ export class NotebookCtrl {
                         style,
                         legend: {
                             vers: 'furious',
-                            margin: {right: -25}
+                            margin: {right: -15}
                         }
                     }
                 };
@@ -936,7 +934,7 @@ export class NotebookCtrl {
 
         const _startWatch = () => {
             const awaitClusters$ = fromPromise(
-                agentMgr.startClusterWatch('Back to Configuration', 'default-state'));
+                agentMgr.startClusterWatch('Leave Queries', 'default-state'));
 
             const finishLoading$ = defer(() => {
                 if (!$root.IgniteDemoMode)
@@ -959,6 +957,10 @@ export class NotebookCtrl {
                 .subscribe();
         };
 
+        const _newParagraph = (paragraph) => {
+            return new Paragraph($animate, $timeout, JavaTypes, errorParser, paragraph);
+        };
+
         Notebook.find($state.params.noteId)
             .then((notebook) => {
                 $scope.notebook = _.cloneDeep(notebook);
@@ -971,8 +973,7 @@ export class NotebookCtrl {
                 if (!$scope.notebook.paragraphs)
                     $scope.notebook.paragraphs = [];
 
-                $scope.notebook.paragraphs = _.map($scope.notebook.paragraphs,
-                    (paragraph) => new Paragraph($animate, $timeout, JavaTypes, paragraph));
+                $scope.notebook.paragraphs = _.map($scope.notebook.paragraphs, (p) => _newParagraph(p));
 
                 if (_.isEmpty($scope.notebook.paragraphs))
                     $scope.addQuery();
@@ -1044,7 +1045,7 @@ export class NotebookCtrl {
 
             ActivitiesData.post({ action: '/queries/add/query' });
 
-            const paragraph = new Paragraph($animate, $timeout, JavaTypes, {
+            const paragraph = _newParagraph({
                 name: 'Query' + (sz === 0 ? '' : sz),
                 query: '',
                 pageSize: $scope.pageSizes[1],
@@ -1073,7 +1074,7 @@ export class NotebookCtrl {
 
             ActivitiesData.post({ action: '/queries/add/scan' });
 
-            const paragraph = new Paragraph($animate, $timeout, JavaTypes, {
+            const paragraph = _newParagraph({
                 name: 'Scan' + (sz === 0 ? '' : sz),
                 query: '',
                 pageSize: $scope.pageSizes[1],
@@ -1406,7 +1407,7 @@ export class NotebookCtrl {
                 .then(() => _closeOldQuery(paragraph))
                 .then(() => args.localNid || _chooseNode(args.cacheName, false))
                 .then((nid) => agentMgr.querySql(nid, args.cacheName, args.query, args.nonCollocatedJoins,
-                    args.enforceJoinOrder, false, !!args.localNid, args.pageSize, args.lazy))
+                    args.enforceJoinOrder, false, !!args.localNid, args.pageSize, args.lazy, args.collocated))
                 .then((res) => _processQueryResult(paragraph, false, res))
                 .catch((err) => paragraph.setError(err));
         };
@@ -1439,6 +1440,15 @@ export class NotebookCtrl {
             return false;
         };
 
+        $scope.collocatedJoinsAvailable = (paragraph) => {
+            const cache = _.find($scope.caches, {name: paragraph.cacheName});
+
+            if (cache)
+                return !!_.find(cache.nodes, (node) => Version.since(node.version, ...COLLOCATED_QUERY_SINCE));
+
+            return false;
+        };
+
         $scope.enforceJoinOrderAvailable = (paragraph) => {
             const cache = _.find($scope.caches, {name: paragraph.cacheName});
 
@@ -1474,11 +1484,13 @@ export class NotebookCtrl {
             const nonCollocatedJoins = !!paragraph.nonCollocatedJoins;
             const enforceJoinOrder = !!paragraph.enforceJoinOrder;
             const lazy = !!paragraph.lazy;
+            const collocated = !!paragraph.collocated;
 
             $scope.queryAvailable(paragraph) && _chooseNode(paragraph.cacheName, local)
                 .then((nid) => {
-                    Notebook.save($scope.notebook)
-                        .catch(Messages.showError);
+                    // If we are executing only selected part of query then Notebook shouldn't be saved.
+                    if (!paragraph.partialQuery)
+                        Notebook.save($scope.notebook).catch(Messages.showError);
 
                     paragraph.localQueryMode = local;
                     paragraph.prevQuery = paragraph.queryArgs ? paragraph.queryArgs.query : paragraph.query;
@@ -1487,23 +1499,26 @@ export class NotebookCtrl {
 
                     return _closeOldQuery(paragraph)
                         .then(() => {
+                            const query = paragraph.partialQuery || paragraph.query;
+
                             const args = paragraph.queryArgs = {
                                 type: 'QUERY',
                                 cacheName: $scope.cacheNameForSql(paragraph),
-                                query: paragraph.query,
+                                query,
                                 pageSize: paragraph.pageSize,
                                 maxPages: paragraph.maxPages,
                                 nonCollocatedJoins,
                                 enforceJoinOrder,
                                 localNid: local ? nid : null,
-                                lazy
+                                lazy,
+                                collocated
                             };
 
-                            const qry = args.maxPages ? addLimit(args.query, args.pageSize * args.maxPages) : paragraph.query;
-
                             ActivitiesData.post({ action: '/queries/execute' });
 
-                            return agentMgr.querySql(nid, args.cacheName, qry, nonCollocatedJoins, enforceJoinOrder, false, local, args.pageSize, lazy);
+                            const qry = args.maxPages ? addLimit(args.query, args.pageSize * args.maxPages) : query;
+
+                            return agentMgr.querySql(nid, args.cacheName, qry, nonCollocatedJoins, enforceJoinOrder, false, local, args.pageSize, lazy, collocated);
                         })
                         .then((res) => {
                             _processQueryResult(paragraph, true, res);
@@ -1534,6 +1549,10 @@ export class NotebookCtrl {
         };
 
         $scope.explain = (paragraph) => {
+            const nonCollocatedJoins = !!paragraph.nonCollocatedJoins;
+            const enforceJoinOrder = !!paragraph.enforceJoinOrder;
+            const collocated = !!paragraph.collocated;
+
             if (!$scope.queryAvailable(paragraph))
                 return;
 
@@ -1556,7 +1575,7 @@ export class NotebookCtrl {
 
                     ActivitiesData.post({ action: '/queries/explain' });
 
-                    return agentMgr.querySql(nid, args.cacheName, args.query, false, !!paragraph.enforceJoinOrder, false, false, args.pageSize, false);
+                    return agentMgr.querySql(nid, args.cacheName, args.query, nonCollocatedJoins, enforceJoinOrder, false, false, args.pageSize, false, collocated);
                 })
                 .then((res) => _processQueryResult(paragraph, true, res))
                 .catch((err) => {
@@ -1750,7 +1769,7 @@ export class NotebookCtrl {
             return Promise.resolve(args.localNid || _chooseNode(args.cacheName, false))
                 .then((nid) => args.type === 'SCAN'
                     ? agentMgr.queryScanGetAll(nid, args.cacheName, args.query, !!args.regEx, !!args.caseSensitive, !!args.near, !!args.localNid)
-                    : agentMgr.querySqlGetAll(nid, args.cacheName, args.query, !!args.nonCollocatedJoins, !!args.enforceJoinOrder, false, !!args.localNid, !!args.lazy))
+                    : agentMgr.querySqlGetAll(nid, args.cacheName, args.query, !!args.nonCollocatedJoins, !!args.enforceJoinOrder, false, !!args.localNid, !!args.lazy, !!args.collocated))
                 .then((res) => _export(exportFileName(paragraph, true), paragraph.gridOptions.columnDefs, res.columns, res.rows))
                 .catch(Messages.showError)
                 .then(() => {
@@ -1921,9 +1940,7 @@ export class NotebookCtrl {
 
                 const addToTrace = (item) => {
                     if (nonNil(item)) {
-                        const clsName = _.isEmpty(item.className) ? '' : '[' + JavaTypes.shortClassName(item.className) + '] ';
-
-                        scope.content.push((scope.content.length > 0 ? tab : '') + clsName + (item.message || ''));
+                        scope.content.push((scope.content.length > 0 ? tab : '') + errorParser.extractFullMessage(item));
 
                         addToTrace(item.cause);
 
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss
index 718899b1bc95b..a5fd50ab46124 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss
@@ -18,9 +18,22 @@
 queries-notebook {
     // TODO: Delete this breadcrumbs styles after template refactoring to new design.
     .notebooks-top {
-        margin-bottom: 20px;
         display: flex;
         align-items: center;
+        margin: 40px 0 30px;
+        flex-direction: row;
+
+        h1 {
+            margin: 0 !important;
+            display: flex;
+            align-items: center;
+            align-content: center;
+
+            label {
+                font-size: 24px;
+                margin-right: 10px;
+            }
+        }
 
         breadcrumbs {
             padding-left: 0;
@@ -36,24 +49,6 @@ queries-notebook {
         }
     }
 
-    .docs-content > header {
-        margin: 0;
-        margin-bottom: 30px;
-
-        display: flex;
-        flex-direction: row;
-        align-items: center;
-
-        h1 {
-            margin: 0;
-            margin-right: 8px;
-        }
-    }
-
-    .affix + .block-information {
-        margin-top: 90px;
-    }
-
     .block-information {
         padding: 18px 10px 18px 36px;
 
@@ -108,4 +103,22 @@ queries-notebook {
             margin: 0;
         }
     }
+
+    .notebook-top-buttons {
+        display: flex;
+        align-items: center;
+        margin-left: auto;
+
+        button:first-of-type {
+            margin-right: 20px;
+        }
+
+        a {
+            color: rgb(57, 57, 57);
+        }
+    }
+
+    .btn.btn-default.select-toggle.tipLabel {
+        padding-right: 25px;
+    }
 }
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
index a163125be0ce7..713f83e540f35 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
@@ -48,38 +48,10 @@ mixin chart-settings
         .col-xs-2
             +result-toolbar
 
-mixin notebook-rename
-    .notebooks-top
-        breadcrumbs
-            a.link-success(ui-sref='^.') Notebooks
-            span(ui-sref='.' ui-sref-active) Notebook '{{notebook.name}}'
-        cluster-selector
-
-    .docs-header.notebook-header
-        h1.col-sm-6(ng-hide='notebook.edit')
-            label(style='max-width: calc(100% - 60px)') {{notebook.name}}
-            .btn-group(ng-if='!demo')
-                +btn-toolbar('fa-pencil', 'notebook.edit = true;notebook.editName = notebook.name', 'Rename notebook')
-                +btn-toolbar('fa-trash', 'removeNotebook(notebook)', '{{$ctrl.isDemo ? "Demo notebook cannot be deleted" : "Remove notebook"}}')
-        h1.col-sm-6(ng-show='notebook.edit')
-            i.btn.fa.fa-floppy-o(ng-show='notebook.editName' ng-click='renameNotebook(notebook.editName)' bs-tooltip data-title='Save notebook name' data-trigger='hover')
-            .input-tip
-                input.form-control(ng-model='notebook.editName' required ignite-on-enter='renameNotebook(notebook.editName)' ignite-on-escape='notebook.edit = false;')
-        h1.pull-right
-            a.dropdown-toggle(style='margin-right: 20px' data-toggle='dropdown' bs-dropdown='scrollParagraphs' data-placement='bottom-right') Scroll to query
-                span.caret
-            button.btn.btn-default(style='margin-top: 2px' ng-click='addQuery()' ignite-on-click-focus=focusId)
-                i.fa.fa-fw.fa-plus
-                | Add query
-
-            button.btn.btn-default(style='margin-top: 2px' ng-click='addScan()' ignite-on-click-focus=focusId)
-                i.fa.fa-fw.fa-plus
-                | Add scan
-
 mixin notebook-error
     h2 Failed to load notebook
-    label.col-sm-12 Notebook not accessible any more. Go back to configuration or open to another notebook.
-    button.h3.btn.btn-primary(ui-sref='default-state') Back to configuration
+    label.col-sm-12 Notebook not accessible any more. Go back to notebooks list.
+    button.h3.btn.btn-primary(ui-sref='default-state') Leave Notebook
 
 mixin paragraph-rename
     .col-sm-6(ng-hide='paragraph.edit')
@@ -117,6 +89,14 @@ mixin query-settings
                     <b>NOTE</b>: In some cases it may consume more heap memory or may take a long time than collocated joins.' data-trigger='hover')
                     input(type='checkbox' ng-model='paragraph.nonCollocatedJoins')
                     span Allow non-collocated joins
+            .row(ng-if='collocatedJoinsAvailable(paragraph)')
+                label.tipLabel(bs-tooltip data-placement='bottom' data-title='Used For Optimization Purposes Of Queries With GROUP BY Statements.<br/>\
+                    <b>NOTE:</b> Whenever Ignite executes a distributed query, it sends sub-queries to individual cluster members.<br/>\
+                    If you know in advance that the elements of your query selection are collocated together on the same node\
+                    and you group by collocated key (primary or affinity key), then Ignite can make significant performance and\
+                    network optimizations by grouping data on remote nodes.' data-trigger='hover')
+                    input(type='checkbox' ng-model='paragraph.collocated')
+                    span Collocated Query
             .row(ng-if='enforceJoinOrderAvailable(paragraph)')
                 label.tipLabel(bs-tooltip data-placement='bottom' data-title='Enforce join order of tables in the query.<br/>\
                     If <b>set</b>, then query optimizer will not reorder tables within join.<br/>\
@@ -144,7 +124,6 @@ mixin query-actions
             i.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.executionInProgress(true)')
             span.tipLabelExecute Execute on selected node
 
-
     a.btn.btn-default(ng-disabled='!queryAvailable(paragraph)' ng-click='explain(paragraph)' data-placement='bottom' bs-tooltip='' data-title='{{queryTooltip(paragraph, "explain query")}}') Explain
 
 mixin table-result-heading-query
@@ -237,7 +216,7 @@ mixin table-result-heading-scan
                         span.caret
 
 mixin table-result-body
-    .grid(ui-grid='paragraph.gridOptions' ui-grid-resize-columns ui-grid-exporter)
+    .grid(ui-grid='paragraph.gridOptions' ui-grid-resize-columns ui-grid-exporter ui-grid-hovering)
 
 mixin chart-result
     div(ng-hide='paragraph.scanExplain()')
@@ -279,9 +258,9 @@ mixin paragraph-scan
                     span.tipLabelExecute Scan
 
             button.btn.btn-primary(ng-disabled='!scanAvailable(paragraph)' ng-click='scan(paragraph, true)')
-                    i.fa.fa-fw.fa-play(ng-hide='paragraph.checkScanInProgress(true)')
-                    i.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.checkScanInProgress(true)')
-                    span.tipLabelExecute Scan on selected node
+                i.fa.fa-fw.fa-play(ng-hide='paragraph.checkScanInProgress(true)')
+                i.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.checkScanInProgress(true)')
+                span.tipLabelExecute Scan on selected node
 
         .col-sm-12.sql-result(ng-if='paragraph.queryExecuted() && !paragraph.scanningInProgress' ng-switch='paragraph.resultType()')
             .error(ng-switch-when='error') Error: {{paragraph.error.message}}
@@ -304,16 +283,16 @@ mixin paragraph-scan
 mixin paragraph-query
     .row.panel-heading(bs-collapse-toggle)
         +paragraph-rename
-    .panel-collapse(role='tabpanel' bs-collapse-target)
+    .panel-collapse.ng-animate-disabled(role='tabpanel' bs-collapse-target)
         .col-sm-12
             .col-xs-8.col-sm-9(style='border-right: 1px solid #eee')
                 .sql-editor(ignite-ace='{onLoad: aceInit(paragraph), theme: "chrome", mode: "sql", require: ["ace/ext/language_tools"],' +
                 'advanced: {enableSnippets: false, enableBasicAutocompletion: true, enableLiveAutocompletion: true}}'
-                ng-model='paragraph.query')
+                ng-model='paragraph.query' on-selection-change='paragraph.partialQuery = $event')
             .col-xs-4.col-sm-3
                 div(ng-show='caches.length > 0' style='padding: 5px 10px' st-table='displayedCaches' st-safe-src='caches')
                     lable.labelField.labelFormField Caches:
-                    i.fa.fa-database.tipField(title='Click to show cache types metadata dialog' bs-popover data-template-url='{{ $ctrl.cacheMetadataTemplateUrl }}' data-placement='bottom' data-trigger='click' data-container='#{{ paragraph.id }}')
+                    i.fa.fa-database.tipField(title='Click to show cache types metadata dialog' bs-popover data-template-url='{{ $ctrl.cacheMetadataTemplateUrl }}' data-placement='bottom-right' data-trigger='click' data-container='#{{ paragraph.id }}')
                     .input-tip
                         input.form-control(type='text' st-search='label' placeholder='Filter caches...')
                     table.links
@@ -361,54 +340,78 @@ mixin paragraph-query
                     i.fa.fa-chevron-circle-right
                     a Next
 
-.row
-    .docs-content
-        .row(ng-if='notebook' bs-affix style='margin-bottom: 20px;')
-            +notebook-rename
-
-        -var example = `CREATE TABLE Person(ID INTEGER PRIMARY KEY, NAME VARCHAR(100));\nINSERT INTO Person(ID, NAME) VALUES (1, 'Ed'), (2, 'Ann'), (3, 'Emma');\nSELECT * FROM Person;`;
-
-        ignite-information(
-            data-title='With query notebook you can'
-            style='margin-bottom: 30px'
-            ng-init=`example = "${example}"`
-        )
-            ul
-                li Create any number of queries
-                li Execute and explain SQL queries
-                li Execute scan queries
-                li View data in tabular form and as charts
-                li
-                    a(href='https://apacheignite-sql.readme.io/docs/sql-reference-overview' target='_blank') More info
-            .example
-                .group
-                    .group-legend
-                        label Examples:
-                    .group-content
-                        .sql-editor(ignite-ace='{\
-                            onLoad: aceInit({}),\
-                            theme: "chrome",\
-                            mode: "sql",\
-                            require: ["ace/ext/language_tools"],\
-                            showGutter: false,\
-                            advanced: {\
-                                enableSnippets: false,\
-                                enableBasicAutocompletion: true,\
-                                enableLiveAutocompletion: true\
-                            }}'
-                            ng-model='example'
-                            readonly='true'
-                        )
-
-        div(ng-if='notebookLoadFailed' style='text-align: center')
-            +notebook-error
-
-        div(ng-if='notebook' ignite-loading='sqlLoading' ignite-loading-text='{{ loadingText }}' ignite-loading-position='top')
-            .docs-body.paragraphs
-                .panel-group(bs-collapse ng-model='notebook.expandedParagraphs' data-allow-multiple='true' data-start-collapsed='false')
-
-                    .panel-paragraph(ng-repeat='paragraph in notebook.paragraphs' id='{{paragraph.id}}' ng-form='form_{{paragraph.id}}')
-                        .panel.panel-default(ng-if='paragraph.qryType === "scan"')
-                            +paragraph-scan
-                        .panel.panel-default(ng-if='paragraph.qryType === "query"')
-                            +paragraph-query
+div(ng-if='notebook')
+    .notebooks-top
+        h1(ng-hide='notebook.edit')
+            label {{notebook.name}}
+            .btn-group(ng-if='!demo')
+                +btn-toolbar('fa-pencil', 'notebook.edit = true;notebook.editName = notebook.name', 'Rename notebook')
+
+        h1(ng-show='notebook.edit')
+            input.form-control(ng-model='notebook.editName' required ignite-on-enter='renameNotebook(notebook.editName)' ignite-on-escape='notebook.edit = false;')
+            i.btn.fa.fa-floppy-o(ng-show='notebook.editName' ng-click='renameNotebook(notebook.editName)' bs-tooltip data-title='Save notebook name' data-trigger='hover')
+
+        cluster-selector
+
+        .notebook-top-buttons
+            a.dropdown-toggle(style='margin-right: 20px' data-toggle='dropdown' bs-dropdown='scrollParagraphs' data-placement='bottom-left') Scroll to query
+                span.caret
+            button.btn-ignite.btn-ignite--primary(ng-click='addQuery()' ignite-on-click-focus=focusId)
+                svg.icon-left(ignite-icon='plus')
+                | Add query
+
+            button.btn-ignite.btn-ignite--primary(ng-click='addScan()' ignite-on-click-focus=focusId)
+                svg.icon-left(ignite-icon='plus')
+                | Add scan
+
+div
+    breadcrumbs
+        a.link-success(ui-sref='base.sql.tabs') Notebooks
+        span(ui-sref='.' ui-sref-active) Notebook '{{notebook.name}}'
+
+    -var example = `CREATE TABLE Person(ID INTEGER PRIMARY KEY, NAME VARCHAR(100));\nINSERT INTO Person(ID, NAME) VALUES (1, 'Ed'), (2, 'Ann'), (3, 'Emma');\nSELECT * FROM Person;`;
+
+    ignite-information(
+        data-title='With query notebook you can'
+        style='margin-bottom: 30px'
+        ng-init=`example = "${example}"`
+    )
+        ul
+            li Create any number of queries
+            li Execute and explain SQL queries
+            li Execute scan queries
+            li View data in tabular form and as charts
+            li
+                a(href='https://apacheignite-sql.readme.io/docs/sql-reference-overview' target='_blank') More info
+        .example
+            .group
+                .group-legend
+                    label Examples:
+                .group-content
+                    .sql-editor(ignite-ace='{\
+                        onLoad: aceInit({}),\
+                        theme: "chrome",\
+                        mode: "sql",\
+                        require: ["ace/ext/language_tools"],\
+                        showGutter: false,\
+                        advanced: {\
+                           enableSnippets: false,\
+                           enableBasicAutocompletion: true,\
+                           enableLiveAutocompletion: true\
+                        }}'
+                        ng-model='example'
+                        readonly='true'
+                    )
+
+    div(ng-if='notebookLoadFailed' style='text-align: center')
+        +notebook-error
+
+    div(ng-if='notebook' ignite-loading='sqlLoading' ignite-loading-text='{{ loadingText }}' ignite-loading-position='top')
+        .docs-body.paragraphs
+            .panel-group(bs-collapse ng-model='notebook.expandedParagraphs' data-allow-multiple='true' data-start-collapsed='false')
+
+                .panel-paragraph(ng-repeat='paragraph in notebook.paragraphs' id='{{paragraph.id}}' ng-form='form_{{paragraph.id}}')
+                    .panel.panel-default(ng-if='paragraph.qryType === "scan"')
+                        +paragraph-scan
+                    .panel.panel-default(ng-if='paragraph.qryType === "query"')
+                        +paragraph-query
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js
index 7c06f2a5c1fe6..2e5ac39efd998 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/controller.js
@@ -27,20 +27,20 @@ export class NotebooksListCtrl {
 
         this.rowsToShow = 8;
 
-        const notebookNameTemplate = `<div class="ui-grid-cell-contents notebook-name"><a ui-sref="base.sql.tabs.notebook({ noteId: row.entity._id })">{{ row.entity.name }}</a></div>`;
+        const notebookNameTemplate = `<div class="ui-grid-cell-contents notebook-name"><a ui-sref="base.sql.notebook({ noteId: row.entity._id })">{{ row.entity.name }}</a></div>`;
         const sqlQueryTemplate = `<div class="ui-grid-cell-contents">{{row.entity.sqlQueriesParagraphsLength}}</div>`;
         const scanQueryTemplate = `<div class="ui-grid-cell-contents">{{row.entity.scanQueriesPsaragraphsLength}}</div>`;
 
         const categories = [
             { name: 'Name', visible: true, enableHiding: false },
-            { name: 'SQL Query', visible: true, enableHiding: false },
-            { name: 'Scan Query', visible: true, enableHiding: false }
+            { name: 'SQL Queries', visible: true, enableHiding: false },
+            { name: 'Scan Queries', visible: true, enableHiding: false }
         ];
 
         const columnDefs = [
             { name: 'name', displayName: 'Notebook name', categoryDisplayName: 'Name', field: 'name', cellTemplate: notebookNameTemplate, pinnedLeft: true, filter: { placeholder: 'Filter by Name...' } },
-            { name: 'sqlQueryNum', displayName: 'SQL Query', categoryDisplayName: 'SQL Query', field: 'sqlQueriesParagraphsLength', cellTemplate: sqlQueryTemplate, enableSorting: true, type: 'number', minWidth: 150, width: 150, enableFiltering: false },
-            { name: 'scanQueryNum', displayName: 'Scan Query', categoryDisplayName: 'Scan Query', field: 'scanQueriesParagraphsLength', cellTemplate: scanQueryTemplate, enableSorting: true, type: 'number', minWidth: 150, width: 150, enableFiltering: false }
+            { name: 'sqlQueryNum', displayName: 'SQL Queries', categoryDisplayName: 'SQL Queries', field: 'sqlQueriesParagraphsLength', cellTemplate: sqlQueryTemplate, enableSorting: true, type: 'number', minWidth: 150, width: '10%', enableFiltering: false },
+            { name: 'scanQueryNum', displayName: 'Scan Queries', categoryDisplayName: 'Scan Queries', field: 'scanQueriesParagraphsLength', cellTemplate: scanQueryTemplate, enableSorting: true, type: 'number', minWidth: 150, width: '10%', enableFiltering: false }
         ];
 
         this.gridOptions = {
@@ -103,6 +103,8 @@ export class NotebooksListCtrl {
             this.IgniteMessages.showError(err);
 
         } finally {
+            this.$scope.$applyAsync();
+
             await this.IgniteLoading.finish('notebooksLoading');
         }
     }
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/style.scss b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/style.scss
index 0b96b88d86e2a..6cdbd397689d7 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/style.scss
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/style.scss
@@ -27,6 +27,10 @@
     background: white;
   }
 
+  .ui-grid-render-container-left:before {
+    display: none;
+  }
+
   .notebook-name {
     a {
       color: #0067b9;
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/template.tpl.pug
index 75b5e99e3a4d6..614a6a6d067bd 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/template.tpl.pug
@@ -27,7 +27,7 @@ page-queries-slot(slot-name="'queriesButtons'" ng-if="!$root.IgniteDemoMode")
 
 .queries-notebooks-list
     .panel--ignite
-        .panel-heading.ui-grid-settings
+        .panel-heading.ui-grid-settings.ui-grid-ignite__panel
             .panel-title
                 div(ng-if="!$root.IgniteDemoMode")
                     +ignite-form-field-bsdropdown({
@@ -45,7 +45,7 @@ page-queries-slot(slot-name="'queriesButtons'" ng-if="!$root.IgniteDemoMode")
 
 
         .panel-collapse(ignite-loading='notebooksLoading' ignite-loading-text='Loading notebooks...')
-            .grid.ui-grid--ignite#queriesNotebooksList(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-pinning ui-grid-hovering)
+            .grid.ui-grid--ignite#queriesNotebooksList(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-hovering)
             grid-no-data(grid-api='$ctrl.gridApi')
                 | You have no notebooks.
                 a.link-success(ng-click='$ctrl.createNotebook()') Create one?
diff --git a/modules/web-console/frontend/app/components/page-queries/index.js b/modules/web-console/frontend/app/components/page-queries/index.js
index 5728d8d03894b..9434dc19556a5 100644
--- a/modules/web-console/frontend/app/components/page-queries/index.js
+++ b/modules/web-console/frontend/app/components/page-queries/index.js
@@ -15,16 +15,14 @@
  * limitations under the License.
  */
 
+import './style.scss';
+
 import angular from 'angular';
 
 import queriesNotebooksList from './components/queries-notebooks-list';
 import queriesNotebook from './components/queries-notebook';
 import pageQueriesCmp from './component';
 
-import template from 'views/base2.pug';
-// This template is deprecated for notebooks view
-import legacyTemplate from 'views/base.pug';
-
 import Notebook from './notebook.service';
 
 export default angular.module('ignite-console.sql', [
@@ -49,11 +47,9 @@ export default angular.module('ignite-console.sql', [
             }
 
             $postLink() {
-                this.$timeout(() => {
-                    this.$transclude((clone) => {
-                        this.pageQueries[this.slotName].empty();
-                        clone.appendTo(this.pageQueries[this.slotName]);
-                    });
+                this.$transclude((clone) => {
+                    this.pageQueries[this.slotName].empty();
+                    clone.appendTo(this.pageQueries[this.slotName]);
                 });
             }
         },
@@ -64,15 +60,7 @@ export default angular.module('ignite-console.sql', [
         // set up the states
         $stateProvider
             .state('base.sql', {
-                abstract: true,
-                views: {
-                    '@': {
-                        template
-                    },
-                    '@base.sql': {
-                        template: '<ui-view></ui-view>'
-                    }
-                }
+                abstract: true
             })
             .state('base.sql.tabs', {
                 url: '/queries',
@@ -88,16 +76,9 @@ export default angular.module('ignite-console.sql', [
                     title: 'Notebooks'
                 }
             })
-            .state('base.sql.tabs.notebook', {
+            .state('base.sql.notebook', {
                 url: '/notebook/{noteId}',
-                views: {
-                    '@': {
-                        template: legacyTemplate
-                    },
-                    '@base.sql.tabs.notebook': {
-                        component: 'queriesNotebook'
-                    }
-                },
+                component: 'queriesNotebook',
                 permission: 'query',
                 tfMetaTags: {
                     title: 'Query notebook'
diff --git a/modules/web-console/frontend/ignite_modules/index.js b/modules/web-console/frontend/app/components/page-queries/style.scss
similarity index 89%
rename from modules/web-console/frontend/ignite_modules/index.js
rename to modules/web-console/frontend/app/components/page-queries/style.scss
index 21ccf6ae1eeb6..8f13c2e563ae8 100644
--- a/modules/web-console/frontend/ignite_modules/index.js
+++ b/modules/web-console/frontend/app/components/page-queries/style.scss
@@ -15,8 +15,6 @@
  * limitations under the License.
  */
 
-import angular from 'angular';
-
-export default angular
-.module('ignite-console.modules', [
-]);
+button.select-toggle.btn-chart-column-agg-fx::after {
+	right: 0;
+}
diff --git a/modules/web-console/frontend/app/components/page-signin/component.js b/modules/web-console/frontend/app/components/page-signin/component.js
new file mode 100644
index 0000000000000..968ff396e298d
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-signin/component.js
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+import controller from './controller';
+import './style.scss';
+
+/** @type {ng.IComponentOptions} */
+export default {
+    controller,
+    template
+};
diff --git a/modules/web-console/frontend/app/components/page-signin/controller.js b/modules/web-console/frontend/app/components/page-signin/controller.js
index fb576203b4255..f2d28e886c07e 100644
--- a/modules/web-console/frontend/app/components/page-signin/controller.js
+++ b/modules/web-console/frontend/app/components/page-signin/controller.js
@@ -15,107 +15,56 @@
  * limitations under the License.
  */
 
-// eslint-disable-next-line
-import {default as AuthService} from 'app/modules/user/Auth.service';
-
 export default class {
-    /** @type {ng.IFormController} */
-    form_signup;
-    /** @type {ng.IFormController} */
-    form_signin;
-    /** @type {ng.IFormController} */
-    form_forgot;
+    /** @type {import('./types').ISiginData} */
+    data = {
+        email: null,
+        password: null
+    };
+    /** @type {import('./types').ISigninFormController} */
+    form;
+    /** @type {string} */
+    serverError = null;
 
-    static $inject = ['IgniteFocus', 'IgniteCountries', 'Auth', 'IgniteMessages'];
+    static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils'];
 
     /**
-     * @param {AuthService} Auth
+     * @param {import('app/modules/user/Auth.service').default} Auth
      */
-    constructor(Focus, Countries, Auth, IgniteMessages) {
+    constructor(Auth, IgniteMessages, IgniteFormUtils) {
         this.Auth = Auth;
         this.IgniteMessages = IgniteMessages;
-        this.countries = Countries.getAll();
-
-        Focus.move('user_email');
-    }
-    $onInit() {
-        const data = this.data = {
-            signup: {
-                email: null,
-                password: null,
-                firstName: null,
-                lastName: null,
-                company: null,
-                country: null
-            },
-            signin: {
-                /** @type {string} */
-                email: null,
-                /** @type {string} */
-                password: null
-            },
-            remindPassword: {
-                /** 
-                 * Explicitly mirrors signin email so user won't have to type it twice
-                 * @type {string}
-                 */
-                get email() {
-                    return data.signin.email;
-                },
-                set email(value) {
-                    data.signin.email = value;
-                }
-            }
-        };
-        /** @type {('signin'|'remindPassword')} */
-        this.activeForm = 'signin';
-        /** @type {{signup: string, signin: string, remindPassword: string}} */
-        this.serverErrors = {
-            signup: null,
-            signin: null,
-            remindPassword: null
-        };
-    }
-    get isSigninOpened() {
-        return this.activeForm === 'signin';
-    }
-    get isRemindPasswordOpened() {
-        return this.activeForm === 'remindPassword';
+        this.IgniteFormUtils = IgniteFormUtils;
     }
-    /** Toggles between signin and remind password forms */
-    toggleActiveForm() {
-        this.activeForm = this.activeForm === 'signin' ? 'remindPassword' : 'signin';
-    }
-    /** @param {ng.IFormController} form */
+
+    /** @param {import('./types').ISigninFormController} form */
     canSubmitForm(form) {
         return form.$error.server ? true : !form.$invalid;
     }
+
     $postLink() {
-        this.form_signup.signupEmail.$validators.server = () => !this.serverErrors.signup;
-        this.form_signin.signinEmail.$validators.server = () => !this.serverErrors.signin;
-        this.form_signin.signinPassword.$validators.server = () => !this.serverErrors.signin;
-        this.form_forgot.forgotEmail.$validators.server = () => !this.serverErrors.remindPassword;
+        this.form.email.$validators.server = () => !this.serverError;
+        this.form.password.$validators.server = () => !this.serverError;
     }
-    signup() {
-        return this.Auth.signnup(this.data.signup).catch((res) => {
-            this.IgniteMessages.showError(null, res.data);
-            this.serverErrors.signup = res.data;
-            this.form_signup.signupEmail.$validate();
-        });
+
+    /** @param {string} error */
+    setServerError(error) {
+        this.serverError = error;
+        this.form.email.$validate();
+        this.form.password.$validate();
     }
+
     signin() {
-        return this.Auth.signin(this.data.signin.email, this.data.signin.password).catch((res) => {
-            this.IgniteMessages.showError(null, res.data);
-            this.serverErrors.signin = res.data;
-            this.form_signin.signinEmail.$validate();
-            this.form_signin.signinPassword.$validate();
-        });
-    }
-    remindPassword() {
-        return this.Auth.remindPassword(this.data.remindPassword.email).catch((res) => {
+        this.IgniteFormUtils.triggerValidation(this.form);
+
+        this.setServerError(null);
+
+        if (!this.canSubmitForm(this.form))
+            return;
+
+        return this.Auth.signin(this.data.email, this.data.password).catch((res) => {
             this.IgniteMessages.showError(null, res.data);
-            this.serverErrors.remindPassword = res.data;
-            this.form_forgot.forgotEmail.$validate();
+            this.setServerError(res.data);
         });
     }
 }
diff --git a/modules/web-console/frontend/app/components/page-signin/index.js b/modules/web-console/frontend/app/components/page-signin/index.js
index 6be374f27b3cc..ee027a69ddd99 100644
--- a/modules/web-console/frontend/app/components/page-signin/index.js
+++ b/modules/web-console/frontend/app/components/page-signin/index.js
@@ -16,41 +16,13 @@
  */
 
 import angular from 'angular';
-
-import template from './template.pug';
-import controller from './controller';
-import './style.scss';
+import component from './component';
+import {registerState} from './run';
 
 export default angular
-    .module('ignite-console.sign-in', [
+    .module('ignite-console.page-signin', [
         'ui.router',
         'ignite-console.user'
     ])
-    .component('pageSignIn', {
-        controller,
-        template
-    })
-    .config(['$stateProvider', function($stateProvider) {
-        // set up the states
-        $stateProvider
-        .state('signin', {
-            url: '/signin',
-            template: '<page-sign-in></page-sign-in>',
-            redirectTo: (trans) => {
-                return trans.injector().get('User').read()
-                    .then(() => {
-                        try {
-                            const {name, params} = JSON.parse(localStorage.getItem('lastStateChangeSuccess'));
-
-                            const restored = trans.router.stateService.target(name, params);
-
-                            return restored.valid() ? restored : 'default-state';
-                        } catch (ignored) {
-                            return 'default-state';
-                        }
-                    })
-                    .catch(() => true);
-            },
-            unsaved: true
-        });
-    }]);
+    .component('pageSignin', component)
+    .run(registerState);
diff --git a/modules/web-console/frontend/app/components/page-signin/run.js b/modules/web-console/frontend/app/components/page-signin/run.js
new file mode 100644
index 0000000000000..457e488912234
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-signin/run.js
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @param {import("@uirouter/angularjs").UIRouter} $uiRouter
+ */
+export function registerState($uiRouter) {
+    /** @type {import("app/types").IIgniteNg1StateDeclaration} */
+    const state = {
+        url: '/signin',
+        name: 'signin',
+        component: 'pageSignin',
+        unsaved: true,
+        redirectTo: (trans) => {
+            const skipStates = new Set(['signup', 'forgotPassword', 'landing']);
+
+            if (skipStates.has(trans.from().name))
+                return;
+
+            return trans.injector().get('User').read()
+                .then(() => {
+                    try {
+                        const {name, params} = JSON.parse(localStorage.getItem('lastStateChangeSuccess'));
+
+                        const restored = trans.router.stateService.target(name, params);
+
+                        return restored.valid() ? restored : 'default-state';
+                    } catch (ignored) {
+                        return 'default-state';
+                    }
+                })
+                .catch(() => true);
+        },
+        tfMetaTags: {
+            title: 'Sign In'
+        }
+    };
+
+    $uiRouter.stateRegistry.register(state);
+}
+
+registerState.$inject = ['$uiRouter'];
diff --git a/modules/web-console/frontend/app/components/page-signin/style.scss b/modules/web-console/frontend/app/components/page-signin/style.scss
index 8ea143af440cd..51da101770161 100644
--- a/modules/web-console/frontend/app/components/page-signin/style.scss
+++ b/modules/web-console/frontend/app/components/page-signin/style.scss
@@ -15,34 +15,40 @@
  * limitations under the License.
  */
 
-page-sign-in {
+page-signin {
     display: flex;
     flex-direction: column;
     flex: 1 0 auto;
 
-    font-family: Roboto;
-
-    h3 {
-        font-size: 38px;
-        font-weight: 300;
-        margin: 30px 0 60px;
-    }
-
     section {
-        flex-grow: 1;
-        padding: 25px 0 60px;
+        margin-left: auto;
+        margin-right: auto;
+        width: 530px;
 
-        background-color: #ffffff;
-        color: #444444;
-    }
+        h3 {
+            font-size: 38px;
+            font-weight: 300;
+            margin: 30px 0 30px;
+        }
+
+        .form-field {
+            margin: 10px 0;
+        }
 
-    .ps-grid {
-        display: grid;
-        grid-gap: 10px;
-        grid-template-columns: 1fr 1fr;
+        .form-footer {
+            padding: 15px 0;
+            text-align: right;
+            display: flex;
+            align-items: center;
 
-        .ps-grid-full-width {
-            grid-column: 1 / 3;
+            .btn-ignite {
+                margin-left: auto;
+            }
         }
     }
-}
\ No newline at end of file
+
+    .page-signin__no-account-message {
+        text-align: center;
+        margin: 20px 0;
+    } 
+}
diff --git a/modules/web-console/frontend/app/components/page-signin/template.pug b/modules/web-console/frontend/app/components/page-signin/template.pug
index 58d85715dc0a5..b31a9bdb69798 100644
--- a/modules/web-console/frontend/app/components/page-signin/template.pug
+++ b/modules/web-console/frontend/app/components/page-signin/template.pug
@@ -20,164 +20,40 @@ web-console-header
     web-console-header-left
         ignite-header-title
 
-section
-    .container
-        .row
-            .col-lg-6.col-md-6.col-sm-6.col-xs-12
-                .row
-                    .col-xs-12.col-md-11
-                        -var form = '$ctrl.form_signup'
-                        h3 Don't Have An Account?
-                        form.ps-grid(name=form novalidate)
-                            .ps-grid-full-width
-                                +form-field__email({
-                                    label: 'Email:',
-                                    model: '$ctrl.data.signup.email',
-                                    name: '"signupEmail"',
-                                    placeholder: 'Input email',
-                                    required: true
-                                })(
-                                    ignite-on-enter-focus-move='passwordInput'
-                                    ng-model-options='{allowInvalid: true}'
-                                )
-                                    +form-field__error({error: 'server', message: `{{$ctrl.serverErrors.signup}}`})
-                            div
-                                +form-field__password({
-                                    label: 'Password:',
-                                    model: '$ctrl.data.signup.password',
-                                    name: '"password"',
-                                    placeholder: 'Input password',
-                                    required: true
-                                })(
-                                    ignite-on-enter-focus-move='confirmInput'
-                                )
-                            div
-                                +form-field__password({
-                                    label: 'Confirm:',
-                                    model: 'confirm',
-                                    name: '"confirm"',
-                                    placeholder: 'Confirm password',
-                                    required: true
-                                })(
-                                    ignite-on-enter-focus-move='firstNameInput'
-                                    ignite-match='$ctrl.data.signup.password'
-                                )
-                            div
-                                +form-field__text({
-                                    label: 'First name:',
-                                    model: '$ctrl.data.signup.firstName',
-                                    name: '"firstName"',
-                                    placeholder: 'Input first name',
-                                    required: true
-                                })(
-                                    ignite-on-enter-focus-move='lastNameInput'
-                                )
-                            div
-                                +form-field__text({
-                                    label: 'Last name:',
-                                    model: '$ctrl.data.signup.lastName',
-                                    name: '"lastName"',
-                                    placeholder: 'Input last name',
-                                    required: true
-                                })(
-                                    ignite-on-enter-focus-move='companyInput'
-                                )
-                            .ps-grid-full-width
-                                +form-field__dropdown({
-                                    label: 'Country:',
-                                    model: '$ctrl.data.signup.country',
-                                    name: '"country"',
-                                    required: true,
-                                    placeholder: 'Choose your country',
-                                    options: '$ctrl.countries'
-                                })(
-                                    ignite-on-enter-focus-move='signup_submit'
-                                )
-                            .ps-grid-full-width
-                                +form-field__text({
-                                    label: 'Company:',
-                                    model: '$ctrl.data.signup.company',
-                                    name: '"company"',
-                                    placeholder: 'Input company name',
-                                    required: true
-                                })(
-                                    ignite-on-enter-focus-move='countryInput'
-                                )
-                            .login-footer.ps-grid-full-width
-                                button#signup_submit.btn-ignite.btn-ignite--primary(
-                                    ng-click='$ctrl.signup()'
-                                    ng-disabled=`!$ctrl.canSubmitForm(${form})`
-                                ) Sign Up
-
-            .col-lg-6.col-md-6.col-sm-6.col-xs-12
-                .row
-                    .col-xs-12.col-md-11
-                        -var form = '$ctrl.form_signin'
-                        form.row(ng-show='$ctrl.isSigninOpened' name=form novalidate)
-                            .settings-row
-                                h3 Sign In
-                            .settings-row
-                                +form-field__email({
-                                    label: 'Email:',
-                                    model: '$ctrl.data.signin.email',
-                                    name: '"signinEmail"',
-                                    placeholder: 'Input email',
-                                    required: true
-                                })(
-                                    ignite-auto-focus
-                                    ignite-on-enter-focus-move='singinPasswordInput'
-                                    ng-model-options='{allowInvalid: true}'
-                                )
-                                    +form-field__error({error: 'server', message: `{{$ctrl.serverErrors.signin}}`})
-                            .settings-row
-                                +form-field__password({
-                                    label: 'Password:',
-                                    model: '$ctrl.data.signin.password',
-                                    name: '"signinPassword"',
-                                    placeholder: 'Input password',
-                                    required: true
-                                })(
-                                    ignite-on-enter='$ctrl.signin($ctrl.data.signin.email, $ctrl.data.signin.password)'
-                                    ng-model-options='{allowInvalid: true}'
-                                )
-                                    +form-field__error({error: 'server', message: `{{$ctrl.serverErrors.signin}}`})
-                            .login-footer
-                                a.labelField#forgot_show(
-                                    ng-click='$ctrl.toggleActiveForm()'
-                                    ignite-on-click-focus='forgot_email'
-                                ) Forgot password?
-                                button#signin_submit.btn-ignite.btn-ignite--primary(
-                                    ng-click='$ctrl.signin()'
-                                    ng-disabled=`!$ctrl.canSubmitForm(${form})`
-                                ) Sign In
-
-                        - var form = '$ctrl.form_forgot'
-                        form.row(ng-show='$ctrl.isRemindPasswordOpened' name=form novalidate)
-                            .settings-row
-                                h3 Forgot password?
-                            .settings-row
-                                p.col-xs-12.col-md-11 That's ok! Simply enter your email below and a reset password link will be sent to you via email. You can then follow that link and select a new password.
-                            .settings-row
-                                +form-field__email({
-                                    label: 'Email:',
-                                    model: '$ctrl.data.remindPassword.email',
-                                    name: '"forgotEmail"',
-                                    placeholder: 'Input email',
-                                    required: true
-                                })(
-                                    ignite-auto-focus
-                                    ng-model-options='{allowInvalid: true}'
-                                    ignite-on-enter='$ctrl.remindPassword()'
-                                )
-                                    +form-field__error({error: 'server', message: `{{$ctrl.serverErrors.remindPassword}}`})
-                            .login-footer
-                                a.labelField#forgot_signin(
-                                    ng-click='$ctrl.toggleActiveForm()'
-                                    ignite-on-click-focus='signin_email'
-                                ) Sign In
-                                button#forgot_submit.btn-ignite.btn-ignite--primary(
-                                    ng-click='$ctrl.remindPassword()'
-                                    ng-disabled=`!$ctrl.canSubmitForm(${form})`
-                                ) Send it to me
+.container--responsive.body-container
+    section
+        -var form = '$ctrl.form'
+        h3 Sign In
+        form(name=form novalidate ng-submit='$ctrl.signin()')
+            +form-field__email({
+                label: 'Email:',
+                model: '$ctrl.data.email',
+                name: '"email"',
+                placeholder: 'Input email',
+                required: true
+            })(
+                ng-model-options='{allowInvalid: true}'
+                autocomplete='email'
+                ignite-auto-focus
+            )
+                +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`})
+            +form-field__password({
+                label: 'Password:',
+                model: '$ctrl.data.password',
+                name: '"password"',
+                placeholder: 'Input password',
+                required: true
+            })(
+                ng-model-options='{allowInvalid: true}'
+                autocomplete='current-password'
+            )
+                +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`})
+            footer.form-footer
+                a(ui-sref='forgotPassword({email: $ctrl.data.email})') Forgot password?
+                button.btn-ignite.btn-ignite--primary(
+                    type='submit'
+                ) Sign In
+        footer.page-signin__no-account-message
+            | Don't have an account? #[a(ui-sref='signup') Get started]
 
 web-console-footer
diff --git a/modules/web-console/frontend/app/components/page-signin/types.ts b/modules/web-console/frontend/app/components/page-signin/types.ts
new file mode 100644
index 0000000000000..96cca75a5e41d
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-signin/types.ts
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface ISiginData {
+    email: string,
+    password: string
+}
+
+export interface ISigninFormController extends ng.IFormController {
+    email: ng.INgModelController,
+    password: ng.INgModelController
+}
diff --git a/modules/web-console/frontend/app/components/page-signup/component.js b/modules/web-console/frontend/app/components/page-signup/component.js
new file mode 100644
index 0000000000000..789a68187e828
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-signup/component.js
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import template from './template.pug';
+import controller from './controller';
+import './style.scss';
+
+/** @type {ng.IComponentOptions} */
+export default {
+    controller,
+    template
+};
diff --git a/modules/web-console/frontend/app/components/page-signup/controller.js b/modules/web-console/frontend/app/components/page-signup/controller.js
new file mode 100644
index 0000000000000..57f404a558c74
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-signup/controller.js
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class PageSignup {
+    /** @type {import('./types').ISignupFormController} */
+    form;
+    /** @type {import('./types').ISignupData} */
+    data = {
+        email: null,
+        password: null,
+        firstName: null,
+        lastName: null,
+        company: null,
+        country: null
+    };
+    /** @type {string} */
+    serverError = null;
+
+    static $inject = ['IgniteCountries', 'Auth', 'IgniteMessages', 'IgniteFormUtils'];
+
+    /**
+     * @param {import('app/modules/user/Auth.service').default} Auth
+     */
+    constructor(Countries, Auth, IgniteMessages, IgniteFormUtils) {
+        this.Auth = Auth;
+        this.IgniteMessages = IgniteMessages;
+        this.countries = Countries.getAll();
+        this.IgniteFormUtils = IgniteFormUtils;
+    }
+
+    /** @param {import('./types').ISignupFormController} form */
+    canSubmitForm(form) {
+        return form.$error.server ? true : !form.$invalid;
+    }
+
+    $postLink() {
+        this.form.email.$validators.server = () => !this.serverError;
+    }
+
+    /** @param {string} error */
+    setServerError(error) {
+        this.serverError = error;
+        this.form.email.$validate();
+    }
+
+    signup() {
+        this.IgniteFormUtils.triggerValidation(this.form);
+
+        this.setServerError(null);
+
+        if (!this.canSubmitForm(this.form))
+            return;
+
+        return this.Auth.signnup(this.data).catch((res) => {
+            this.IgniteMessages.showError(null, res.data);
+            this.setServerError(res.data);
+        });
+    }
+}
diff --git a/modules/web-console/frontend/app/components/page-signup/index.js b/modules/web-console/frontend/app/components/page-signup/index.js
new file mode 100644
index 0000000000000..4efadb5f47af5
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-signup/index.js
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import component from './component';
+import {registerState} from './run';
+
+export default angular
+    .module('ignite-console.page-signup', [
+        'ui.router',
+        'ignite-console.user'
+    ])
+    .component('pageSignup', component)
+    .run(registerState);
diff --git a/modules/web-console/frontend/app/components/page-signup/run.js b/modules/web-console/frontend/app/components/page-signup/run.js
new file mode 100644
index 0000000000000..1d04fa257714f
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-signup/run.js
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @param {import("@uirouter/angularjs").UIRouter} $uiRouter
+ */
+export function registerState($uiRouter) {
+    /** @type {import("app/types").IIgniteNg1StateDeclaration} */
+    const state = {
+        name: 'signup',
+        url: '/signup',
+        component: 'pageSignup',
+        unsaved: true,
+        tfMetaTags: {
+            title: 'Sign Up'
+        }
+    };
+    $uiRouter.stateRegistry.register(state);
+}
+
+registerState.$inject = ['$uiRouter'];
diff --git a/modules/web-console/frontend/app/components/page-signup/style.scss b/modules/web-console/frontend/app/components/page-signup/style.scss
new file mode 100644
index 0000000000000..93167ce6856da
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-signup/style.scss
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+page-signup {
+    display: flex;
+    flex-direction: column;
+    flex: 1 0 auto;
+
+    section {
+        h3 {
+            font-size: 38px;
+            font-weight: 300;
+            margin: 30px 0 30px;
+        }
+
+        margin-left: auto;
+        margin-right: auto;
+        width: 530px;
+
+        form footer {
+            padding: 15px 0;
+            text-align: right;
+            display: flex;
+            align-items: center;
+
+            .btn-ignite {
+                margin-left: auto;
+            }
+        }
+
+        form {
+            display: grid;
+            grid-gap: 10px;
+            grid-template-columns: 1fr 1fr;
+
+            .full-width {
+                grid-column: 1 / 3;
+            }
+        }
+    }
+
+    .page-signup__has-account-message {
+        text-align: center;
+        margin: 20px 0;
+    } 
+}
diff --git a/modules/web-console/frontend/app/components/page-signup/template.pug b/modules/web-console/frontend/app/components/page-signup/template.pug
new file mode 100644
index 0000000000000..11bb50a15c733
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-signup/template.pug
@@ -0,0 +1,121 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins
+
+web-console-header
+    web-console-header-left
+        ignite-header-title
+
+.container--responsive.body-container
+    section
+        -var form = '$ctrl.form'
+        h3 Don't Have An Account?
+        form(name=form novalidate ng-submit='$ctrl.signup()')
+            .full-width
+                +form-field__email({
+                    label: 'Email:',
+                    model: '$ctrl.data.email',
+                    name: '"email"',
+                    placeholder: 'Input email',
+                    required: true
+                })(
+                    ng-model-options='{allowInvalid: true}'
+                    autocomplete='email'
+                    ignite-auto-focus
+                )
+                    +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`})
+            div
+                +form-field__password({
+                    label: 'Password:',
+                    model: '$ctrl.data.password',
+                    name: '"password"',
+                    placeholder: 'Input password',
+                    required: true
+                })(
+                    autocomplete='new-password'
+                )
+            div
+                +form-field__password({
+                    label: 'Confirm:',
+                    model: 'confirm',
+                    name: '"confirm"',
+                    placeholder: 'Confirm password',
+                    required: true
+                })(
+                    ignite-match='$ctrl.data.password'
+                    autocomplete='off'
+                )
+            div
+                +form-field__text({
+                    label: 'First name:',
+                    model: '$ctrl.data.firstName',
+                    name: '"firstName"',
+                    placeholder: 'Input first name',
+                    required: true
+                })(
+                    autocomplete='given-name'
+                )
+            div
+                +form-field__text({
+                    label: 'Last name:',
+                    model: '$ctrl.data.lastName',
+                    name: '"lastName"',
+                    placeholder: 'Input last name',
+                    required: true
+                })(
+                    autocomplete='family-name'
+                )
+            div
+                +form-field__phone({
+                    label: 'Phone:',
+                    model: '$ctrl.data.phone',
+                    name: '"phone"',
+                    placeholder: 'Input phone (ex.: +15417543010)',
+                    optional: true
+                })(
+                    autocomplete='tel'
+                )
+            div
+                +form-field__dropdown({
+                    label: 'Country:',
+                    model: '$ctrl.data.country',
+                    name: '"country"',
+                    required: true,
+                    placeholder: 'Choose your country',
+                    options: '$ctrl.countries'
+                })(
+                    autocomplete='country'
+                )
+            .full-width
+                +form-field__text({
+                    label: 'Company:',
+                    model: '$ctrl.data.company',
+                    name: '"company"',
+                    placeholder: 'Input company name',
+                    required: true
+                })(
+                    ignite-on-enter-focus-move='countryInput'
+                    autocomplete='organization'
+                )
+            footer.full-width.form-footer
+                button.btn-ignite.btn-ignite--primary(
+                    type='submit'
+                ) Sign Up
+        footer.page-signup__has-account-message
+            | Already have an account? #[a(ui-sref='signin') Sign in here]
+
+web-console-footer
diff --git a/modules/web-console/frontend/app/components/page-signup/types.ts b/modules/web-console/frontend/app/components/page-signup/types.ts
new file mode 100644
index 0000000000000..49e8d5d15072b
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-signup/types.ts
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface ISignupData {
+    email: string,
+    password: string,
+    firstName: string,
+    lastName: string,
+    phone?: string,
+    company: string,
+    country: string
+}
+
+export interface ISignupFormController extends ng.IFormController {
+    email: ng.INgModelController,
+    password: ng.INgModelController,
+    firstName: ng.INgModelController,
+    lastName: ng.INgModelController,
+    phone: ng.INgModelController,
+    company: ng.INgModelController,
+    country: ng.INgModelController
+}
diff --git a/modules/web-console/frontend/app/components/panel-collapsible/controller.js b/modules/web-console/frontend/app/components/panel-collapsible/controller.js
index 0b8b4dafeb9b9..65ec06abd65c5 100644
--- a/modules/web-console/frontend/app/components/panel-collapsible/controller.js
+++ b/modules/web-console/frontend/app/components/panel-collapsible/controller.js
@@ -33,20 +33,31 @@ export default class PanelCollapsible {
     constructor($transclude) {
         this.$transclude = $transclude;
     }
+
     toggle() {
         if (this.opened)
             this.close();
         else
             this.open();
     }
+
     open() {
-        if (this.disabled) return;
+        if (this.disabled)
+            return;
+
         this.opened = true;
-        if (this.onOpen && this.opened) this.onOpen({});
+
+        if (this.onOpen && this.opened)
+            this.onOpen({});
     }
+
     close() {
-        if (this.disabled) return;
+        if (this.disabled)
+            return;
+
         this.opened = false;
-        if (this.onClose && !this.opened) this.onClose({});
+
+        if (this.onClose && !this.opened)
+            this.onClose({});
     }
 }
diff --git a/modules/web-console/frontend/app/components/panel-collapsible/index.spec.js b/modules/web-console/frontend/app/components/panel-collapsible/index.spec.js
index 79722464c1a07..b5c00cd5a3933 100644
--- a/modules/web-console/frontend/app/components/panel-collapsible/index.spec.js
+++ b/modules/web-console/frontend/app/components/panel-collapsible/index.spec.js
@@ -18,7 +18,6 @@
 import 'mocha';
 import {assert} from 'chai';
 import angular from 'angular';
-import 'angular-mocks';
 import {spy} from 'sinon';
 import componentModule from './index.js';
 
diff --git a/modules/web-console/frontend/app/components/panel-collapsible/style.scss b/modules/web-console/frontend/app/components/panel-collapsible/style.scss
index 73eba25ca51df..f421862035db2 100644
--- a/modules/web-console/frontend/app/components/panel-collapsible/style.scss
+++ b/modules/web-console/frontend/app/components/panel-collapsible/style.scss
@@ -59,6 +59,7 @@ panel-collapsible {
 
     .#{&}__actions {
         margin-left: auto;
+        flex: 0 0 auto;
 
         & > panel-actions {
             display: flex;
@@ -76,4 +77,4 @@ panel-collapsible {
         padding: 15px 20px;
         contain: content;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/web-console/frontend/app/components/password-visibility/index.js b/modules/web-console/frontend/app/components/password-visibility/index.js
new file mode 100644
index 0000000000000..d735869d03354
--- /dev/null
+++ b/modules/web-console/frontend/app/components/password-visibility/index.js
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import './style.scss';
+import {directive as visibilityRoot} from './root.directive';
+import {component as toggleButton} from './toggle-button.component';
+
+export default angular
+    .module('ignite-console.passwordVisibility', [])
+    .directive('passwordVisibilityRoot', visibilityRoot)
+    .component('passwordVisibilityToggleButton', toggleButton);
diff --git a/modules/web-console/frontend/app/components/password-visibility/index.spec.js b/modules/web-console/frontend/app/components/password-visibility/index.spec.js
new file mode 100644
index 0000000000000..f887092b5fcb6
--- /dev/null
+++ b/modules/web-console/frontend/app/components/password-visibility/index.spec.js
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import 'mocha';
+import {assert} from 'chai';
+import angular from 'angular';
+import module from './index';
+
+const PASSWORD_VISIBLE_CLASS = 'password-visibility__password-visible';
+
+suite('password-visibility', () => {
+    /** @type {ng.IScope} */
+    let $scope;
+    /** @type {ng.ICompileService} */
+    let $compile;
+
+    angular.module('test', [module.name]);
+
+    setup(() => {
+        angular.module('test', [module.name]);
+        angular.mock.module('test');
+        angular.mock.inject((_$rootScope_, _$compile_) => {
+            $compile = _$compile_;
+            $scope = _$rootScope_.$new();
+        });
+    });
+
+    test('Visibility toggle', () => {
+        const el = angular.element(`
+            <div password-visibility-root on-password-visibility-toggle='visible = $event'>
+                <password-visibility-toggle-button></password-visibility-toggle-button>
+            </div>
+        `);
+        $compile(el)($scope);
+        const toggleButton = el.find('password-visibility-toggle-button').children()[0];
+        $scope.$digest();
+
+        assert.isFalse(el.hasClass(PASSWORD_VISIBLE_CLASS), 'Password is hidden by default');
+
+        toggleButton.click();
+        $scope.$digest();
+
+        assert.isTrue(el.hasClass(PASSWORD_VISIBLE_CLASS), 'Password is visible after click on toggle button');
+        assert.equal(true, $scope.visible, 'Event emits current visibility value');
+
+        toggleButton.click();
+        $scope.$digest();
+
+        assert.isFalse(el.hasClass(PASSWORD_VISIBLE_CLASS), 'Password is hidden again after two clicks on button');
+    });
+});
diff --git a/modules/web-console/frontend/app/components/password-visibility/root.directive.js b/modules/web-console/frontend/app/components/password-visibility/root.directive.js
new file mode 100644
index 0000000000000..a041398a187e7
--- /dev/null
+++ b/modules/web-console/frontend/app/components/password-visibility/root.directive.js
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const PASSWORD_VISIBLE_CLASS = 'password-visibility__password-visible';
+
+export class PasswordVisibilityRoot {
+    /** @type {ng.ICompiledExpression} */
+    onPasswordVisibilityToggle;
+
+    isVisible = false;
+    static $inject = ['$element'];
+
+    /**
+     * @param {JQLite} $element
+     */
+    constructor($element) {
+        this.$element = $element;
+    }
+    toggleVisibility() {
+        this.isVisible = !this.isVisible;
+        this.$element.toggleClass(PASSWORD_VISIBLE_CLASS, this.isVisible);
+        if (this.onPasswordVisibilityToggle) this.onPasswordVisibilityToggle({$event: this.isVisible});
+    }
+}
+
+export function directive() {
+    return {
+        restrict: 'A',
+        scope: false,
+        controller: PasswordVisibilityRoot,
+        bindToController: {
+            onPasswordVisibilityToggle: '&?'
+        }
+    };
+}
diff --git a/modules/web-console/frontend/app/components/password-visibility/style.scss b/modules/web-console/frontend/app/components/password-visibility/style.scss
new file mode 100644
index 0000000000000..67457eb1676fc
--- /dev/null
+++ b/modules/web-console/frontend/app/components/password-visibility/style.scss
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+[password-visibility-root] {
+    &:not(.password-visibility__password-visible) {
+        .password-visibility__icon-visible,
+        .password-visibility__password-visible {
+            display: none;
+        }        
+    }
+
+    &.password-visibility__password-visible {
+        .password-visibility__icon-hidden,
+        .password-visibility__password-hidden {
+            display: none;
+        }
+    }
+}
+
+password-visibility-toggle-button {
+    display: inline-block;
+    width: 36px;
+
+    button {
+        display: inline-flex;
+        align-items: center;
+        justify-content: center;
+        width: 100%;
+        height: 100%;
+        background: none;
+        border: none;
+        outline: none;
+        padding: 0 !important;
+        margin: 0 !important;
+
+        &:focus {
+            color: #0067b9;
+        }
+    }
+}
diff --git a/modules/web-console/frontend/app/components/password-visibility/toggle-button.component.js b/modules/web-console/frontend/app/components/password-visibility/toggle-button.component.js
new file mode 100644
index 0000000000000..2e16201f018d5
--- /dev/null
+++ b/modules/web-console/frontend/app/components/password-visibility/toggle-button.component.js
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {PasswordVisibilityRoot} from './root.directive';
+
+class Controller {
+    /** @type {PasswordVisibilityRoot} */
+    visibilityRoot;
+
+    toggleVisibility() {
+        this.visibilityRoot.toggleVisibility();
+    }
+    get isVisible() {
+        return this.visibilityRoot.isVisible;
+    }
+}
+
+export const component = {
+    template: `
+        <button
+            type='button'
+            ng-click='$ctrl.toggleVisibility()'
+            bs-tooltip=''
+            data-title='{{ $ctrl.isVisible ? "Hide password" : "Show password" }}'
+            data-placement='top'
+        >
+            <svg ignite-icon='eyeOpened' class='password-visibility__icon-visible'></svg>
+            <svg ignite-icon='eyeClosed' class='password-visibility__icon-hidden'></svg>
+        </button>
+    `,
+    require: {
+        visibilityRoot: '^passwordVisibilityRoot'
+    },
+    controller: Controller
+};
diff --git a/modules/web-console/frontend/app/components/progress-line/component.js b/modules/web-console/frontend/app/components/progress-line/component.js
new file mode 100644
index 0000000000000..f998ec809f023
--- /dev/null
+++ b/modules/web-console/frontend/app/components/progress-line/component.js
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './style.scss';
+import controller from './controller';
+import template from './template.pug';
+
+export default {
+    controller,
+    template,
+    bindings: {
+        value: '<?'
+    }
+};
diff --git a/modules/web-console/frontend/app/components/progress-line/controller.js b/modules/web-console/frontend/app/components/progress-line/controller.js
new file mode 100644
index 0000000000000..f383d594ebed4
--- /dev/null
+++ b/modules/web-console/frontend/app/components/progress-line/controller.js
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const INDETERMINATE_CLASS = 'progress-line__indeterminate';
+const COMPLETE_CLASS = 'progress-line__complete';
+
+/**
+ * @typedef {-1} IndeterminateValue
+ */
+
+/**
+ * @typedef {1} CompleteValue
+ */
+
+/**
+ * @typedef {IndeterminateValue|CompleteValue} ProgressLineValue
+ */
+
+export default class ProgressLine {
+    /** @type {ProgressLineValue} */
+    value;
+
+    static $inject = ['$element'];
+
+    /**
+     * @param {JQLite} $element
+     */
+    constructor($element) {
+        this.$element = $element;
+    }
+
+    /**
+     * @param {{value: ng.IChangesObject<ProgressLineValue>}} changes
+     */
+    $onChanges(changes) {
+        if (changes.value.currentValue === -1) {
+            this.$element[0].classList.remove(COMPLETE_CLASS);
+            this.$element[0].classList.add(INDETERMINATE_CLASS);
+            return;
+        }
+        if (typeof changes.value.currentValue === 'number') {
+            if (changes.value.currentValue === 1) this.$element[0].classList.add(COMPLETE_CLASS);
+            this.$element[0].classList.remove(INDETERMINATE_CLASS);
+        }
+    }
+}
diff --git a/modules/web-console/frontend/app/components/progress-line/index.js b/modules/web-console/frontend/app/components/progress-line/index.js
new file mode 100644
index 0000000000000..a91b97bb259f3
--- /dev/null
+++ b/modules/web-console/frontend/app/components/progress-line/index.js
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import component from './component';
+
+export default angular
+    .module('ignite-console.progress-line', [])
+    .component('progressLine', component);
diff --git a/modules/web-console/frontend/app/components/progress-line/index.spec.js b/modules/web-console/frontend/app/components/progress-line/index.spec.js
new file mode 100644
index 0000000000000..f5725e938a3b5
--- /dev/null
+++ b/modules/web-console/frontend/app/components/progress-line/index.spec.js
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import 'mocha';
+import {assert} from 'chai';
+import angular from 'angular';
+import module from './index';
+
+const INDETERMINATE_CLASS = 'progress-line__indeterminate';
+const COMPLETE_CLASS = 'progress-line__complete';
+
+suite('progress-line', () => {
+    let $scope;
+    let $compile;
+
+    setup(() => {
+        angular.module('test', [module.name]);
+        angular.mock.module('test');
+        angular.mock.inject((_$rootScope_, _$compile_) => {
+            $compile = _$compile_;
+            $scope = _$rootScope_.$new();
+        });
+    });
+
+    test('Progress states', () => {
+        $scope.progress = -1;
+        const el = angular.element(`<progress-line value='progress'></progress-line>`);
+
+        $compile(el)($scope);
+        $scope.$digest();
+
+        assert.isTrue(
+            el[0].classList.contains(INDETERMINATE_CLASS),
+            'Adds indeterminate class for indeterminate state'
+        );
+
+        assert.isFalse(
+            el[0].classList.contains(COMPLETE_CLASS),
+            'Does not have complete class when in indeterminate state'
+        );
+
+        $scope.progress = 1;
+        $scope.$digest();
+
+        assert.isFalse(
+            el[0].classList.contains(INDETERMINATE_CLASS),
+            'Does not has indeterminate class when in finished state'
+        );
+
+        assert.isTrue(
+            el[0].classList.contains(COMPLETE_CLASS),
+            'Adds complete class when in finished state'
+        );
+    });
+});
diff --git a/modules/web-console/frontend/app/components/progress-line/style.scss b/modules/web-console/frontend/app/components/progress-line/style.scss
new file mode 100644
index 0000000000000..3b7f4efe70b2a
--- /dev/null
+++ b/modules/web-console/frontend/app/components/progress-line/style.scss
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+progress-line {
+    @import 'public/stylesheets/variables';
+
+    --background-color: transparent;
+    --foreground-color: #{$ignite-brand-primary};
+
+    height: 1px;
+    position: relative;
+    display: block;
+    overflow: hidden;
+
+    @keyframes progress-line-indeterminate {
+        0% {
+            left: -33%;
+            width: 33%;
+        }
+        100% {
+            left: 100%;
+            width: 33%;
+        }
+    }
+
+    @keyframes progress-line-indeterminate-to-complete {
+        0% {
+            opacity: 0;
+        }
+        100% {
+            opacity: 1;
+        }
+    }
+
+    .progress-line__background {
+        position: absolute;
+        top: 0;
+        left: 0;
+        right: 0;
+        bottom: 0;
+        display: block;
+        background: var(--background-color);
+    }
+
+    .progress-line__foreground {
+        position: absolute;
+        top: 0;
+        bottom: 0;
+        content: "";
+        display: block;
+        background: var(--foreground-color);
+    }
+
+    &.progress-line__complete .progress-line__foreground {
+        animation-name: progress-line-indeterminate-to-complete;
+        animation-iteration-count: 1;
+        animation-duration: 0.2s;
+        left: 0;
+        right: 0;
+        width: 100%;
+    }
+
+    &.progress-line__indeterminate .progress-line__foreground {
+        animation-name: progress-line-indeterminate;
+        animation-iteration-count: infinite;
+        animation-duration: 2s;
+    }
+}
diff --git a/modules/web-console/frontend/app/components/connected-clusters/template.pug b/modules/web-console/frontend/app/components/progress-line/template.pug
similarity index 90%
rename from modules/web-console/frontend/app/components/connected-clusters/template.pug
rename to modules/web-console/frontend/app/components/progress-line/template.pug
index f238eac04de3c..b48beae29bf8c 100644
--- a/modules/web-console/frontend/app/components/connected-clusters/template.pug
+++ b/modules/web-console/frontend/app/components/progress-line/template.pug
@@ -14,5 +14,5 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-svg(ignite-icon='connectedClusters')
-| Connected clusters: {{ $ctrl.connectedClusters }}
+.progress-line__background
+.progress-line__foreground
diff --git a/modules/web-console/frontend/app/components/ui-grid-column-resizer/directive.js b/modules/web-console/frontend/app/components/ui-grid-column-resizer/directive.js
new file mode 100644
index 0000000000000..6ba2a788a9d63
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid-column-resizer/directive.js
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default function() {
+    return {
+        priority: -200,
+        restrict: 'A',
+        require: '?^uiGrid',
+        link($scope, $element) {
+            $element.on('dblclick', function($event) {
+                $event.stopImmediatePropagation();
+            });
+        }
+    };
+}
diff --git a/modules/web-console/frontend/app/components/ui-grid-column-resizer/index.js b/modules/web-console/frontend/app/components/ui-grid-column-resizer/index.js
new file mode 100644
index 0000000000000..9edf1ef711f37
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid-column-resizer/index.js
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import uiGridColumnResizer from './directive';
+
+export default angular
+    .module('ignite-console.ui-grid-column-resizer', [])
+    .directive('uiGridColumnResizer', uiGridColumnResizer);
diff --git a/modules/web-console/frontend/app/components/ui-grid-filters/directive.js b/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
index 6602c11d9f074..c9b84af4219dc 100644
--- a/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
+++ b/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
@@ -23,7 +23,8 @@ export default function uiGridFilters(uiGridConstants) {
         require: 'uiGrid',
         link: {
             pre(scope, el, attr, gridApi) {
-                if (!gridApi.grid.options.enableFiltering) return;
+                if (!gridApi.grid.options.enableFiltering)
+                    return;
 
                 const applyMultiselectFilter = (cd) => {
                     cd.headerCellTemplate = template;
diff --git a/modules/web-console/frontend/app/components/ui-grid-filters/index.js b/modules/web-console/frontend/app/components/ui-grid-filters/index.js
index 0f05b779f2318..2c60d8961284b 100644
--- a/modules/web-console/frontend/app/components/ui-grid-filters/index.js
+++ b/modules/web-console/frontend/app/components/ui-grid-filters/index.js
@@ -27,7 +27,9 @@ export default angular
             instance.$referenceElement = el;
             instance.destroy = flow(instance.destroy, () => instance.$referenceElement = null);
             instance.$applyPlacement = flow(instance.$applyPlacement, () => {
-                if (!instance.$element) return;
+                if (!instance.$element)
+                    return;
+
                 const refWidth = instance.$referenceElement[0].getBoundingClientRect().width;
                 const elWidth = instance.$element[0].getBoundingClientRect().width;
                 if (refWidth > elWidth) {
diff --git a/modules/web-console/frontend/app/components/ui-grid-hovering/style.scss b/modules/web-console/frontend/app/components/ui-grid-hovering/style.scss
index 6c7597a459f78..d660045fd6715 100644
--- a/modules/web-console/frontend/app/components/ui-grid-hovering/style.scss
+++ b/modules/web-console/frontend/app/components/ui-grid-hovering/style.scss
@@ -20,3 +20,9 @@
         background: #ededed;
     }
 }
+
+.grid[ui-grid-hovering='ui-grid-hovering'] {
+    .ui-grid-row.ui-grid-row-hovered > [ui-grid-row] > .ui-grid-cell {
+        background: #ededed;
+    }
+}
diff --git a/modules/web-console/frontend/app/components/user-notifications/service.js b/modules/web-console/frontend/app/components/user-notifications/service.js
index 2fc8064f132df..d008482d66d08 100644
--- a/modules/web-console/frontend/app/components/user-notifications/service.js
+++ b/modules/web-console/frontend/app/components/user-notifications/service.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
 import controller from './controller';
 import templateUrl from './template.tpl.pug';
 import {CancellationError} from 'app/errors/CancellationError';
@@ -22,6 +24,9 @@ import {CancellationError} from 'app/errors/CancellationError';
 export default class UserNotificationsService {
     static $inject = ['$http', '$modal', '$q', 'IgniteMessages'];
 
+    /** @type {ng.IQService} */
+    $q;
+
     constructor($http, $modal, $q, Messages) {
         Object.assign(this, {$http, $modal, $q, Messages});
 
@@ -56,7 +61,7 @@ export default class UserNotificationsService {
             .finally(modalHide)
             .then(({ message, isShown }) => {
                 this.$http.put('/api/v1/admin/notifications', { message, isShown })
-                    .catch((err) => this.Messages.showError(err));
+                    .catch(this.Messages.showError);
             });
     }
 }
diff --git a/modules/web-console/frontend/app/components/user-notifications/style.scss b/modules/web-console/frontend/app/components/user-notifications/style.scss
index 748a963d6f0ce..e4fa39eda5e51 100644
--- a/modules/web-console/frontend/app/components/user-notifications/style.scss
+++ b/modules/web-console/frontend/app/components/user-notifications/style.scss
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-@import 'public/stylesheets/variables';
+@import '../../../public/stylesheets/variables';
 
 $disabled-color: #c5c5c5;
 
diff --git a/modules/web-console/frontend/app/components/web-console-footer/template.pug b/modules/web-console/frontend/app/components/web-console-footer/template.pug
index e7eecf51be784..aa2812af64904 100644
--- a/modules/web-console/frontend/app/components/web-console-footer/template.pug
+++ b/modules/web-console/frontend/app/components/web-console-footer/template.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-.container.wcf-content
+.container--responsive.wcf-content
     ignite-footer
     web-console-footer-links(ng-if='$parent.user')
     ignite-powered-by-apache
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/web-console-header/style.scss b/modules/web-console/frontend/app/components/web-console-header/style.scss
index 71766c95b4ea3..e92fdd5b66843 100644
--- a/modules/web-console/frontend/app/components/web-console-header/style.scss
+++ b/modules/web-console/frontend/app/components/web-console-header/style.scss
@@ -38,7 +38,6 @@ web-console-header {
         height: 20px;
         position: absolute;
         overflow: hidden;
-        box-shadow: 0 4px 4px -2px rgba(0, 0, 0, 0.3);
         bottom: -4px;
     }
 
@@ -65,8 +64,9 @@ web-console-header {
         flex-direction: row;
         flex-wrap: wrap;
         align-items: center;
-        padding: 18.5px 0;
+        padding: 18px 30px;
         position: relative;
+        height: 81px;
     }
 
     .wch-logo {
@@ -102,7 +102,7 @@ web-console-header {
         &:hover {
             color: $color-hover;
         }
-        
+
         &:focus, :focus,
         &.active, .active {
             color: $color-active;
@@ -120,6 +120,28 @@ web-console-header {
         &:not(:last-child) {
             margin-right: $nav-item-margin;
         }
+
+        .dropdown-menu {
+            a {
+                &:hover, &:focus {
+                    color: inherit;
+                }
+            }
+            li.active a {
+                color: inherit;
+            }
+        }
+
+        .user-name-wrapper {
+            display: flex;
+            align-items: center;
+
+           & > span {
+               max-width: 180px;
+               overflow: hidden;
+               text-overflow: ellipsis;
+           }
+        }
     }
 
     .wch-demo-toggle {
@@ -170,6 +192,10 @@ web-console-header {
             }
         }
     }
+
+    .wch-additional-nav-items {
+        display: flex;
+    }
 }
 
 @keyframes pulse {
diff --git a/modules/web-console/frontend/app/components/web-console-header/template.pug b/modules/web-console/frontend/app/components/web-console-header/template.pug
index a05e5f818b415..801b742e949d1 100644
--- a/modules/web-console/frontend/app/components/web-console-header/template.pug
+++ b/modules/web-console/frontend/app/components/web-console-header/template.pug
@@ -20,10 +20,10 @@
     | You are currently viewing user #[strong {{$root.user.firstName}} {{$root.user.lastName}}] as administrator. #[a(ng-click='$root.revertIdentity()') Revert to your identity?]
 
 .wch-notification.wch-notification--demo(ng-if='$root.IgniteDemoMode')
-    .container(ng-controller='demoController')
+    div(ng-controller='demoController')
         | You are now in #[b Demo Mode]. #[a(ng-click='closeDemo();') Close Demo?]
 
-.wch-content.container
+.wch-content
     connected-clusters(ng-if='$ctrl.$rootScope.user && !$ctrl.$rootScope.IgniteDemoMode && $ctrl.isConnectedClustersVisible && !$root.user.becomeUsed')
 
     a(ui-sref='landing')
diff --git a/modules/web-console/frontend/app/controllers/reset-password.controller.js b/modules/web-console/frontend/app/controllers/reset-password.controller.js
deleted file mode 100644
index f84a8762af49c..0000000000000
--- a/modules/web-console/frontend/app/controllers/reset-password.controller.js
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Controller for password reset.
-export default ['resetPassword', [
-    '$scope', '$modal', '$http', '$state', 'IgniteMessages', 'IgniteFocus',
-    ($scope, $modal, $http, $state, Messages, Focus) => {
-        if ($state.params.token) {
-            $http.post('/api/v1/password/validate/token', {token: $state.params.token})
-                .then(({data}) => {
-                    $scope.email = data.email;
-                    $scope.token = data.token;
-                    $scope.error = data.error;
-
-                    if ($scope.token && !$scope.error)
-                        Focus.move('user_password');
-                });
-        }
-
-        // Try to reset user password for provided token.
-        $scope.resetPassword = (reset_info) => {
-            $http.post('/api/v1/password/reset', reset_info)
-                .then(() => {
-                    $state.go('signin');
-
-                    Messages.showInfo('Password successfully changed');
-                })
-                .catch(({data, state}) => {
-                    if (state === 503)
-                        $state.go('signin');
-
-                    Messages.showError(data);
-                });
-        };
-    }
-]];
diff --git a/modules/web-console/frontend/app/data/getting-started.json b/modules/web-console/frontend/app/data/getting-started.json
index caed92045928f..1802b554aaa82 100644
--- a/modules/web-console/frontend/app/data/getting-started.json
+++ b/modules/web-console/frontend/app/data/getting-started.json
@@ -17,6 +17,19 @@
             "</div>"
         ]
     },
+    {
+        "title": "Quick cluster configuration",
+        "message": [
+            "<div class='col-xs-7'>",
+            " <img src='/images/cluster-quick.png' width='100%' />",
+            "</div>",
+            "<div class='col-xs-5'>",
+            " <ul>",
+            "  <li>Quick configuration of cluster and it's caches</li>",
+            " </ul>",
+            "</div>"
+        ]
+    },
     {
         "title": "Clusters",
         "message": [
@@ -26,7 +39,6 @@
             "<div class='col-xs-5'>",
             " <ul>",
             "  <li>Configure cluster properties</li>",
-            "  <li>Associate cluster with caches</li>",
             " </ul>",
             "</div>"
         ]
@@ -54,7 +66,6 @@
             "<div class='col-xs-5'>",
             " <ul>",
             "  <li>Configure memory settings</li>",
-            "  <li>Configure persistence</li>",
             " </ul>",
             "</div>"
         ]
@@ -68,7 +79,20 @@
             "<div class='col-xs-5'>",
             " <ul>",
             "  <li>Configure IGFS properties</li>",
-            "  <li>Associate IGFS with clusters</li>",
+            " </ul>",
+            "</div>"
+        ]
+    },
+    {
+        "title": "Preview configuration result",
+        "message": [
+            "<div class='col-xs-7'>",
+            " <img src='/images/preview.png' width='100%' />",
+            "</div>",
+            "<div class='col-xs-5'>",
+            " <ul>",
+            "  <li>Preview configured project files</li>",
+            "  <li>Download configured project files</li>",
             " </ul>",
             "</div>"
         ]
diff --git a/modules/web-console/frontend/app/data/i18n.js b/modules/web-console/frontend/app/data/i18n.js
index d992153623071..e21b8c5c338c8 100644
--- a/modules/web-console/frontend/app/data/i18n.js
+++ b/modules/web-console/frontend/app/data/i18n.js
@@ -21,7 +21,7 @@ export default {
     'base.configuration.overview': 'Cluster configurations',
     '/configuration/overview': 'Cluster configurations',
     'base.configuration.edit.basic': 'Basic cluster configuration edit',
-    '/configuration/new': 'Сluster configuration create',
+    '/configuration/new': 'Cluster configuration create',
     '/configuration/new/basic': 'Basic cluster configuration create',
     '/configuration/new/advanced/cluster': 'Advanced cluster configuration create',
     'base.configuration.edit.advanced.cluster': 'Advanced cluster configuration edit',
diff --git a/modules/web-console/frontend/app/directives/match.directive.js b/modules/web-console/frontend/app/directives/match.directive.js
index 3a45f6da8daae..940ca086de4db 100644
--- a/modules/web-console/frontend/app/directives/match.directive.js
+++ b/modules/web-console/frontend/app/directives/match.directive.js
@@ -16,12 +16,32 @@
  */
 
 // Directive to enable validation to match specified value.
-export default ['igniteMatch', ['$parse', ($parse) => {
+export default function() {
     return {
-        require: 'ngModel',
-        link(scope, elem, attrs, ctrl) {
-            scope.$watch(() => $parse(attrs.igniteMatch)(scope) === ctrl.$modelValue,
-                (currentValue) => ctrl.$setValidity('mismatch', currentValue));
+        require: {
+            ngModel: 'ngModel'
+        },
+        scope: false,
+        bindToController: {
+            igniteMatch: '<'
+        },
+        controller: class {
+            /** @type {ng.INgModelController} */
+            ngModel;
+            /** @type {string} */
+            igniteMatch;
+
+            $postLink() {
+                this.ngModel.$overrideModelOptions({allowInvalid: true});
+                this.ngModel.$validators.mismatch = (value) => value === this.igniteMatch;
+            }
+
+            /**
+             * @param {{igniteMatch: ng.IChangesObject<string>}} changes
+             */
+            $onChanges(changes) {
+                if ('igniteMatch' in changes) this.ngModel.$validate();
+            }
         }
     };
-}]];
+}
diff --git a/modules/web-console/frontend/app/directives/match.directive.spec.js b/modules/web-console/frontend/app/directives/match.directive.spec.js
new file mode 100644
index 0000000000000..c274585ff6851
--- /dev/null
+++ b/modules/web-console/frontend/app/directives/match.directive.spec.js
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import 'mocha';
+import {assert} from 'chai';
+import angular from 'angular';
+import directive from './match.directive';
+
+/**
+ * @param {HTMLInputElement} el
+ * @returns {ng.INgModelController}
+ */
+const ngModel = (el) => angular.element(el).data().$ngModelController;
+
+suite('ignite-match', () => {
+    /** @type {ng.IScope} */
+    let $scope;
+    /** @type {ng.ICompileService} */
+    let $compile;
+
+    setup(() => {
+        angular.module('test', []).directive('igniteMatch', directive);
+        angular.mock.module('test');
+        angular.mock.inject((_$rootScope_, _$compile_) => {
+            $compile = _$compile_;
+            $scope = _$rootScope_.$new();
+        });
+    });
+
+    test('Matching', () => {
+        const el = angular.element(`
+            <input type="password" ng-model="data.master"/>
+            <input type="password" ng-model="data.slave" ignite-match="data.master"/>
+        `);
+
+        const setValue = (el, value) => {
+            ngModel(el).$setViewValue(value, 'input');
+            $scope.$digest();
+        };
+
+        $scope.data = {};
+        $compile(el)($scope);
+        $scope.$digest();
+
+        // const [master, , slave] = el;
+        // For some reason, this code not work after Babel, replaced with 'old' syntax.
+        const master = el[0];
+        const slave = el[2];
+
+        setValue(slave, '123');
+        $scope.$digest();
+
+        assert.isTrue(
+            slave.classList.contains('ng-invalid-mismatch'),
+            `Invalidates if slave input changes value and it doesn't match master value`
+        );
+        assert.equal(
+            $scope.data.slave,
+            '123',
+            'Allows invalid value into model'
+        );
+
+        setValue(master, '123');
+
+        assert.isFalse(
+            slave.classList.contains('ng-invalid-mismatch'),
+            `Runs validation on master value change`
+        );
+    });
+});
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
index 2fa0a3caa2b29..dcdcf0e179f28 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
@@ -15,4 +15,4 @@
     limitations under the License.
 
 mixin form-field-feedback(name, error, message)
-    div(ng-message=error) #{message}
\ No newline at end of file
+    div(ng-message=error) #{message}
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
index 6de29c8bd9dd6..3e3597476eff8 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
@@ -44,4 +44,4 @@ mixin ignite-form-field-password(label, model, name, disabled, required, placeho
             if block
                 block
 
-            +form-field-feedback(name, 'required', `${errLbl} could not be empty!`)
\ No newline at end of file
+            +form-field-feedback(name, 'required', `${errLbl} could not be empty!`)
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
index d1c6491894e03..3d28e178e7464 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
+++ b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
@@ -50,4 +50,4 @@ mixin ignite-form-field-text(lbl, model, name, disabled, required, placeholder,
 mixin sane-ignite-form-field-text({label, model, name, disabled, required, placeholder, tip})
     +ignite-form-field-text(label, model, name, disabled, required, placeholder, tip)&attributes(attributes)
         if block
-            block
\ No newline at end of file
+            block
diff --git a/modules/web-console/frontend/app/modules/ace.module.js b/modules/web-console/frontend/app/modules/ace.module.js
index 6a6e70a45f83e..44e51caf0a015 100644
--- a/modules/web-console/frontend/app/modules/ace.module.js
+++ b/modules/web-console/frontend/app/modules/ace.module.js
@@ -134,8 +134,12 @@ angular
 
         return {
             restrict: 'EA',
-            require: ['?ngModel', '?^form'],
-            link: (scope, elm, attrs, [ngModel, form]) => {
+            require: ['?ngModel', '?^form', 'igniteAce'],
+            bindToController: {
+                onSelectionChange: '&?'
+            },
+            controller() {},
+            link: (scope, elm, attrs, [ngModel, form, igniteAce]) => {
                 /**
                  * Corresponds the igniteAceConfig ACE configuration.
                  *
@@ -165,6 +169,8 @@ angular
                  */
                 const session = acee.getSession();
 
+                const selection = session.getSelection();
+
                 /**
                  * Reference to a change listener created by the listener factory.
                  *
@@ -223,6 +229,14 @@ angular
                     ngModel.$render = () => session.setValue(ngModel.$viewValue);
 
                     acee.on('change', () => ngModel.$setViewValue(acee.getValue()));
+
+                    selection.on('changeSelection', () => {
+                        if (igniteAce.onSelectionChange) {
+                            const aceSelection = selection.isEmpty() ? null : acee.session.getTextRange(acee.getSelectionRange());
+
+                            igniteAce.onSelectionChange({$event: aceSelection});
+                        }
+                    });
                 }
 
                 // Listen for option updates.
diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
index b73b2c15c6866..a1c0ff966b0b5 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
@@ -19,11 +19,17 @@ import _ from 'lodash';
 import {nonEmpty, nonNil} from 'app/utils/lodashMixins';
 
 import { BehaviorSubject } from 'rxjs/BehaviorSubject';
+import 'rxjs/add/operator/first';
 
+import AgentModal from './AgentModal.service';
+// @ts-ignore
 import Worker from './decompress.worker';
 import SimpleWorkerPool from '../../utils/SimpleWorkerPool';
 import maskNull from 'app/core/utils/maskNull';
 
+import {ClusterSecretsManager} from './types/ClusterSecretsManager';
+import ClusterLoginService from './components/cluster-login/service';
+
 const State = {
     DISCONNECTED: 'DISCONNECTED',
     AGENT_DISCONNECTED: 'AGENT_DISCONNECTED',
@@ -31,7 +37,22 @@ const State = {
     CONNECTED: 'CONNECTED'
 };
 
+const IGNITE_2_0 = '2.0.0';
 const LAZY_QUERY_SINCE = [['2.1.4-p1', '2.2.0'], '2.2.1'];
+const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], '2.5.2'];
+
+// Error codes from o.a.i.internal.processors.restGridRestResponse.java
+
+const SuccessStatus = {
+    /** Command succeeded. */
+    STATUS_SUCCESS: 0,
+    /** Command failed. */
+    STATUS_FAILED: 1,
+    /** Authentication failure. */
+    AUTH_FAILED: 2,
+    /** Security check failed. */
+    SECURITY_CHECK_FAILED: 3
+};
 
 class ConnectionState {
     constructor(cluster) {
@@ -61,7 +82,7 @@ class ConnectionState {
         if (_.isNil(this.cluster))
             this.cluster = _.head(clusters);
 
-        if (nonNil(this.cluster))
+        if (this.cluster)
             this.cluster.connected = !!_.find(clusters, {id: this.cluster.id});
 
         if (count === 0)
@@ -93,30 +114,50 @@ class ConnectionState {
     }
 }
 
-export default class IgniteAgentManager {
-    static $inject = ['$rootScope', '$q', '$transitions', 'igniteSocketFactory', 'AgentModal', 'UserNotifications', 'IgniteVersion' ];
+export default class AgentManager {
+    static $inject = ['$rootScope', '$q', '$transitions', 'igniteSocketFactory', AgentModal.name, 'UserNotifications', 'IgniteVersion', ClusterLoginService.name];
 
-    constructor($root, $q, $transitions, socketFactory, AgentModal, UserNotifications, Version) {
-        Object.assign(this, {$root, $q, $transitions, socketFactory, AgentModal, UserNotifications, Version});
+    /** @type {ng.IScope} */
+    $root;
 
-        this.promises = new Set();
+    /** @type {ng.IQService} */
+    $q;
 
-        this.pool = new SimpleWorkerPool('decompressor', Worker, 4);
+    /** @type {AgentModal} */
+    agentModal;
 
-        this.socket = null; // Connection to backend.
+    /** @type {ClusterLoginService} */
+    ClusterLoginSrv;
 
-        let cluster;
+    /** @type {String} */
+    clusterVersion = '2.4.0';
 
-        try {
-            cluster = JSON.parse(localStorage.cluster);
+    connectionSbj = new BehaviorSubject(new ConnectionState(AgentManager.restoreActiveCluster()));
 
-            localStorage.removeItem('cluster');
+    /** @type {ClusterSecretsManager} */
+    clustersSecrets = new ClusterSecretsManager();
+
+    pool = new SimpleWorkerPool('decompressor', Worker, 4);
+
+    /** @type {Set<ng.IDifferend>} */
+    promises = new Set();
+
+    socket = null;
+
+    static restoreActiveCluster() {
+        try {
+            return JSON.parse(localStorage.cluster);
         }
         catch (ignore) {
-            // No-op.
+            return null;
         }
+        finally {
+            localStorage.removeItem('cluster');
+        }
+    }
 
-        this.connectionSbj = new BehaviorSubject(new ConnectionState(cluster));
+    constructor($root, $q, $transitions, socketFactory, agentModal, UserNotifications, Version, ClusterLoginSrv) {
+        Object.assign(this, {$root, $q, $transitions, socketFactory, agentModal, UserNotifications, Version, ClusterLoginSrv});
 
         let prevCluster;
 
@@ -124,8 +165,6 @@ export default class IgniteAgentManager {
             .distinctUntilChanged(({ cluster }) => prevCluster === cluster)
             .do(({ cluster }) => prevCluster = cluster);
 
-        this.clusterVersion = '2.4.0';
-
         if (!this.isDemoMode()) {
             this.connectionSbj.subscribe({
                 next: ({cluster}) => {
@@ -144,41 +183,39 @@ export default class IgniteAgentManager {
         return this.$root.IgniteDemoMode;
     }
 
-    available(sinceVersion) {
-        return this.Version.since(this.clusterVersion, sinceVersion);
+    available(...sinceVersion) {
+        return this.Version.since(this.clusterVersion, ...sinceVersion);
     }
 
     connect() {
-        const self = this;
-
-        if (nonNil(self.socket))
+        if (nonNil(this.socket))
             return;
 
-        self.socket = self.socketFactory();
+        this.socket = this.socketFactory();
 
         const onDisconnect = () => {
-            const conn = self.connectionSbj.getValue();
+            const conn = this.connectionSbj.getValue();
 
             conn.disconnect();
 
-            self.connectionSbj.next(conn);
+            this.connectionSbj.next(conn);
         };
 
-        self.socket.on('connect_error', onDisconnect);
+        this.socket.on('connect_error', onDisconnect);
 
-        self.socket.on('disconnect', onDisconnect);
+        this.socket.on('disconnect', onDisconnect);
 
-        self.socket.on('agents:stat', ({clusters, count}) => {
-            const conn = self.connectionSbj.getValue();
+        this.socket.on('agents:stat', ({clusters, count}) => {
+            const conn = this.connectionSbj.getValue();
 
-            conn.update(self.isDemoMode(), count, clusters);
+            conn.update(this.isDemoMode(), count, clusters);
 
-            self.connectionSbj.next(conn);
+            this.connectionSbj.next(conn);
         });
 
-        self.socket.on('cluster:changed', (cluster) => this.updateCluster(cluster));
+        this.socket.on('cluster:changed', (cluster) => this.updateCluster(cluster));
 
-        self.socket.on('user:notifications', (notification) => this.UserNotifications.notification = notification);
+        this.socket.on('user:notifications', (notification) => this.UserNotifications.notification = notification);
     }
 
     saveToStorage(cluster = this.connectionSbj.getValue().cluster) {
@@ -216,7 +253,7 @@ export default class IgniteAgentManager {
 
     /**
      * @param states
-     * @returns {Promise}
+     * @returns {ng.IPromise}
      */
     awaitConnectionState(...states) {
         const defer = this.$q.defer();
@@ -249,33 +286,31 @@ export default class IgniteAgentManager {
     /**
      * @param {String} backText
      * @param {String} [backState]
-     * @returns {Promise}
+     * @returns {ng.IPromise}
      */
     startAgentWatch(backText, backState) {
-        const self = this;
-
-        self.backText = backText;
-        self.backState = backState;
+        this.backText = backText;
+        this.backState = backState;
 
-        const conn = self.connectionSbj.getValue();
+        const conn = this.connectionSbj.getValue();
 
         conn.useConnectedCluster();
 
-        self.connectionSbj.next(conn);
+        this.connectionSbj.next(conn);
 
         this.modalSubscription && this.modalSubscription.unsubscribe();
 
-        self.modalSubscription = this.connectionSbj.subscribe({
+        this.modalSubscription = this.connectionSbj.subscribe({
             next: ({state}) => {
                 switch (state) {
                     case State.CONNECTED:
                     case State.CLUSTER_DISCONNECTED:
-                        this.AgentModal.hide();
+                        this.agentModal.hide();
 
                         break;
 
                     case State.AGENT_DISCONNECTED:
-                        this.AgentModal.agentDisconnected(self.backText, self.backState);
+                        this.agentModal.agentDisconnected(this.backText, this.backState);
 
                         break;
 
@@ -285,43 +320,41 @@ export default class IgniteAgentManager {
             }
         });
 
-        return self.awaitAgent();
+        return this.awaitAgent();
     }
 
     /**
      * @param {String} backText
      * @param {String} [backState]
-     * @returns {Promise}
+     * @returns {ng.IPromise}
      */
     startClusterWatch(backText, backState) {
-        const self = this;
+        this.backText = backText;
+        this.backState = backState;
 
-        self.backText = backText;
-        self.backState = backState;
-
-        const conn = self.connectionSbj.getValue();
+        const conn = this.connectionSbj.getValue();
 
         conn.useConnectedCluster();
 
-        self.connectionSbj.next(conn);
+        this.connectionSbj.next(conn);
 
         this.modalSubscription && this.modalSubscription.unsubscribe();
 
-        self.modalSubscription = this.connectionSbj.subscribe({
+        this.modalSubscription = this.connectionSbj.subscribe({
             next: ({state}) => {
                 switch (state) {
                     case State.CONNECTED:
-                        this.AgentModal.hide();
+                        this.agentModal.hide();
 
                         break;
 
                     case State.AGENT_DISCONNECTED:
-                        this.AgentModal.agentDisconnected(self.backText, self.backState);
+                        this.agentModal.agentDisconnected(this.backText, this.backState);
 
                         break;
 
                     case State.CLUSTER_DISCONNECTED:
-                        self.AgentModal.clusterDisconnected(self.backText, self.backState);
+                        this.agentModal.clusterDisconnected(this.backText, this.backState);
 
                         break;
 
@@ -331,9 +364,9 @@ export default class IgniteAgentManager {
             }
         });
 
-        self.$transitions.onExit({}, () => self.stopWatch());
+        this.$transitions.onExit({}, () => this.stopWatch());
 
-        return self.awaitCluster();
+        return this.awaitCluster();
     }
 
     stopWatch() {
@@ -345,11 +378,11 @@ export default class IgniteAgentManager {
     /**
      *
      * @param {String} event
-     * @param {Object} [args]
-     * @returns {Promise}
+     * @param {Object} [payload]
+     * @returns {ng.IPromise}
      * @private
      */
-    _emit(event, ...args) {
+    _sendToAgent(event, payload = {}) {
         if (!this.socket)
             return this.$q.reject('Failed to connect to server');
 
@@ -363,7 +396,7 @@ export default class IgniteAgentManager {
 
         this.socket.on('disconnect', onDisconnect);
 
-        args.push((err, res) => {
+        this.socket.emit(event, payload, (err, res) => {
             this.socket.removeListener('disconnect', onDisconnect);
 
             if (err)
@@ -372,76 +405,129 @@ export default class IgniteAgentManager {
             latch.resolve(res);
         });
 
-        this.socket.emit(event, ...args);
-
         return latch.promise;
     }
 
     drivers() {
-        return this._emit('schemaImport:drivers');
+        return this._sendToAgent('schemaImport:drivers');
     }
 
     /**
-     * @param {Object} jdbcDriverJar
-     * @param {Object} jdbcDriverClass
-     * @param {Object} jdbcUrl
-     * @param {Object} user
-     * @param {Object} password
-     * @returns {Promise}
+     * @param {{jdbcDriverJar: String, jdbcDriverClass: String, jdbcUrl: String, user: String, password: String}}
+     * @returns {ng.IPromise}
      */
     schemas({jdbcDriverJar, jdbcDriverClass, jdbcUrl, user, password}) {
         const info = {user, password};
 
-        return this._emit('schemaImport:schemas', {jdbcDriverJar, jdbcDriverClass, jdbcUrl, info});
+        return this._sendToAgent('schemaImport:schemas', {jdbcDriverJar, jdbcDriverClass, jdbcUrl, info});
     }
 
     /**
-     * @param {Object} jdbcDriverJar
-     * @param {Object} jdbcDriverClass
-     * @param {Object} jdbcUrl
-     * @param {Object} user
-     * @param {Object} password
-     * @param {Object} schemas
-     * @param {Object} tablesOnly
-     * @returns {Promise} Promise on list of tables (see org.apache.ignite.schema.parser.DbTable java class)
+     * @param {{jdbcDriverJar: String, jdbcDriverClass: String, jdbcUrl: String, user: String, password: String, schemas: String, tablesOnly: String}}
+     * @returns {ng.IPromise} Promise on list of tables (see org.apache.ignite.schema.parser.DbTable java class)
      */
     tables({jdbcDriverJar, jdbcDriverClass, jdbcUrl, user, password, schemas, tablesOnly}) {
         const info = {user, password};
 
-        return this._emit('schemaImport:metadata', {jdbcDriverJar, jdbcDriverClass, jdbcUrl, info, schemas, tablesOnly});
+        return this._sendToAgent('schemaImport:metadata', {jdbcDriverJar, jdbcDriverClass, jdbcUrl, info, schemas, tablesOnly});
     }
 
     /**
-     *
+     * @param {Object} cluster
+     * @param {Object} credentials
      * @param {String} event
-     * @param {Object} [args]
-     * @returns {Promise}
+     * @param {Object} params
+     * @returns {ng.IPromise}
      * @private
      */
-    _rest(event, ...args) {
-        return this._emit(event, _.get(this.connectionSbj.getValue(), 'cluster.id'), ...args)
-            .then((data) => {
-                if (data.zipped)
-                    return this.pool.postMessage(data.data);
+    _executeOnActiveCluster(cluster, credentials, event, params) {
+        return this._sendToAgent(event, {clusterId: cluster.id, params, credentials})
+            .then((res) => {
+                const {status = SuccessStatus.STATUS_SUCCESS} = res;
 
-                return data;
+                switch (status) {
+                    case SuccessStatus.STATUS_SUCCESS:
+                        if (cluster.secured)
+                            this.clustersSecrets.get(cluster.id).sessionToken = res.sessionToken;
+
+                        if (res.zipped)
+                            return this.pool.postMessage(res.data);
+
+                        return res;
+
+                    case SuccessStatus.STATUS_FAILED:
+                        if (res.error.startsWith('Failed to handle request - unknown session token (maybe expired session)')) {
+                            this.clustersSecrets.get(cluster.id).resetSessionToken();
+
+                            return this._executeOnCluster(event, params);
+                        }
+
+                        throw new Error(res.error);
+
+                    case SuccessStatus.AUTH_FAILED:
+                        this.clustersSecrets.get(cluster.id).resetCredentials();
+
+                        throw new Error('Cluster authentication failed. Incorrect user and/or password.');
+
+                    case SuccessStatus.SECURITY_CHECK_FAILED:
+                        throw new Error('Access denied. You are not authorized to access this functionality.');
+
+                    default:
+                        throw new Error('Illegal status in node response');
+                }
             });
     }
 
+    /**
+     * @param {String} event
+     * @param {Object} params
+     * @returns {Promise}
+     * @private
+     */
+    _executeOnCluster(event, params) {
+        if (this.isDemoMode())
+            return Promise.resolve(this._executeOnActiveCluster({}, {}, event, params));
+
+        return this.connectionSbj.first().toPromise()
+            .then(({cluster}) => {
+                if (_.isNil(cluster))
+                    throw new Error('Failed to execute request on cluster.');
+
+                if (cluster.secured) {
+                    return Promise.resolve(this.clustersSecrets.get(cluster.id))
+                        .then((secrets) => {
+                            if (secrets.hasCredentials())
+                                return secrets;
+
+                            return this.ClusterLoginSrv.askCredentials(secrets)
+                                .then((secrets) => {
+                                    this.clustersSecrets.put(cluster.id, secrets);
+
+                                    return secrets;
+                                });
+                        })
+                        .then((secrets) => ({cluster, credentials: secrets.getCredentials()}));
+                }
+
+                return {cluster, credentials: {}};
+            })
+            .then(({cluster, credentials}) => this._executeOnActiveCluster(cluster, credentials, event, params));
+    }
+
     /**
      * @param {Boolean} [attr]
      * @param {Boolean} [mtr]
      * @returns {Promise}
      */
     topology(attr = false, mtr = false) {
-        return this._rest('node:rest', {cmd: 'top', attr, mtr});
+        return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr});
     }
 
     /**
      * @returns {Promise}
      */
     metadata() {
-        return this._rest('node:rest', {cmd: 'metadata'})
+        return this._executeOnCluster('node:rest', {cmd: 'metadata'})
             .then((caches) => {
                 let types = [];
 
@@ -503,7 +589,7 @@ export default class IgniteAgentManager {
 
                     columns = _.sortBy(columns, 'name');
 
-                    if (!_.isEmpty(indexes)) {
+                    if (nonEmpty(indexes)) {
                         columns = columns.concat({
                             type: 'indexes',
                             name: 'Indexes',
@@ -544,7 +630,7 @@ export default class IgniteAgentManager {
 
         nids = _.isArray(nids) ? nids.join(';') : maskNull(nids);
 
-        return this._rest('node:visor', taskId, nids, ...args);
+        return this._executeOnCluster('node:visor', {taskId, nids, args});
     }
 
     /**
@@ -555,17 +641,21 @@ export default class IgniteAgentManager {
      * @param {Boolean} enforceJoinOrder Flag whether enforce join order is enabled.
      * @param {Boolean} replicatedOnly Flag whether query contains only replicated tables.
      * @param {Boolean} local Flag whether to execute query locally.
-     * @param {int} pageSz
-     * @param {Boolean} lazy query flag.
+     * @param {Number} pageSz
+     * @param {Boolean} [lazy] query flag.
+     * @param {Boolean} [collocated] Collocated query.
      * @returns {Promise}
      */
-    querySql(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz, lazy) {
-        if (this.available('2.0.0')) {
-            const task = this.available(...LAZY_QUERY_SINCE) ?
-                this.visorTask('querySqlX2', nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz, lazy) :
-                this.visorTask('querySqlX2', nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz);
+    querySql(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz, lazy = false, collocated = false) {
+        if (this.available(IGNITE_2_0)) {
+            let args = [cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz];
+
+            if (this.available(...COLLOCATED_QUERY_SINCE))
+                args = [...args, lazy, collocated];
+            else if (this.available(...LAZY_QUERY_SINCE))
+                args = [...args, lazy];
 
-            return task.then(({error, result}) => {
+            return this.visorTask('querySqlX2', nid, ...args).then(({error, result}) => {
                 if (_.isEmpty(error))
                     return result;
 
@@ -595,12 +685,12 @@ export default class IgniteAgentManager {
 
     /**
      * @param {String} nid Node id.
-     * @param {int} queryId
-     * @param {int} pageSize
+     * @param {Number} queryId
+     * @param {Number} pageSize
      * @returns {Promise}
      */
     queryNextPage(nid, queryId, pageSize) {
-        if (this.available('2.0.0'))
+        if (this.available(IGNITE_2_0))
             return this.visorTask('queryFetchX2', nid, queryId, pageSize);
 
         return this.visorTask('queryFetch', nid, queryId, pageSize);
@@ -615,9 +705,10 @@ export default class IgniteAgentManager {
      * @param {Boolean} replicatedOnly Flag whether query contains only replicated tables.
      * @param {Boolean} local Flag whether to execute query locally.
      * @param {Boolean} lazy query flag.
+     * @param {Boolean} collocated Collocated query.
      * @returns {Promise}
      */
-    querySqlGetAll(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, lazy) {
+    querySqlGetAll(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, lazy, collocated) {
         // Page size for query.
         const pageSz = 1024;
 
@@ -635,17 +726,17 @@ export default class IgniteAgentManager {
                 });
         };
 
-        return this.querySql(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz, lazy)
+        return this.querySql(nid, cacheName, query, nonCollocatedJoins, enforceJoinOrder, replicatedOnly, local, pageSz, lazy, collocated)
             .then(fetchResult);
     }
 
     /**
      * @param {String} nid Node id.
-     * @param {int} [queryId]
+     * @param {Number} [queryId]
      * @returns {Promise}
      */
     queryClose(nid, queryId) {
-        if (this.available('2.0.0')) {
+        if (this.available(IGNITE_2_0)) {
             return this.visorTask('queryCloseX2', nid, 'java.util.Map', 'java.util.UUID', 'java.util.Collection',
                 nid + '=' + queryId);
         }
@@ -661,11 +752,11 @@ export default class IgniteAgentManager {
      * @param {Boolean} caseSensitive Case sensitive filtration.
      * @param {Boolean} near Scan near cache.
      * @param {Boolean} local Flag whether to execute query locally.
-     * @param {int} pageSize Page size.
+     * @param {Number} pageSize Page size.
      * @returns {Promise}
      */
     queryScan(nid, cacheName, filter, regEx, caseSensitive, near, local, pageSize) {
-        if (this.available('2.0.0')) {
+        if (this.available(IGNITE_2_0)) {
             return this.visorTask('queryScanX2', nid, cacheName, filter, regEx, caseSensitive, near, local, pageSize)
                 .then(({error, result}) => {
                     if (_.isEmpty(error))
@@ -725,10 +816,14 @@ export default class IgniteAgentManager {
      * @returns {Promise}
      */
     toggleClusterState() {
-        const state = this.connectionSbj.getValue();
-        const active = !state.cluster.active;
+        const { cluster } = this.connectionSbj.getValue();
+        const active = !cluster.active;
 
         return this.visorTask('toggleClusterState', null, active)
-            .then(() => state.updateCluster(Object.assign(state.cluster, { active })));
+            .then(() => this.updateCluster({ ...cluster, active }));
+    }
+
+    hasCredentials(clusterId) {
+        return this.clustersSecrets.get(clusterId).hasCredentials();
     }
 }
diff --git a/modules/web-console/frontend/app/modules/agent/AgentModal.service.js b/modules/web-console/frontend/app/modules/agent/AgentModal.service.js
index 52abf6d022486..15a08a2287784 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentModal.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentModal.service.js
@@ -23,6 +23,7 @@ export default class AgentModal {
     constructor($root, $state, $modal, Messages) {
         const self = this;
 
+        this.$root = $root;
         self.$state = $state;
         self.Messages = Messages;
 
@@ -86,4 +87,8 @@ export default class AgentModal {
 
         self.modal.$promise.then(self.modal.show);
     }
+
+    get securityToken() {
+        return this.$root.user.becameToken || this.$root.user.token;
+    }
 }
diff --git a/modules/web-console/frontend/app/modules/agent/agent.module.js b/modules/web-console/frontend/app/modules/agent/agent.module.js
index 63257c5516fd2..1812af088e619 100644
--- a/modules/web-console/frontend/app/modules/agent/agent.module.js
+++ b/modules/web-console/frontend/app/modules/agent/agent.module.js
@@ -20,9 +20,11 @@ import angular from 'angular';
 import AgentModal from './AgentModal.service';
 import AgentManager from './AgentManager.service';
 
+import clusterLogin from './components/cluster-login';
+
 angular
     .module('ignite-console.agent', [
-
+        clusterLogin.name
     ])
-    .service('AgentModal', AgentModal)
-    .service('AgentManager', AgentManager);
+    .service(AgentModal.name, AgentModal)
+    .service(AgentManager.name, AgentManager);
diff --git a/modules/web-console/frontend/app/modules/agent/components/cluster-login/component.js b/modules/web-console/frontend/app/modules/agent/components/cluster-login/component.js
new file mode 100644
index 0000000000000..a8311ed76ee55
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/agent/components/cluster-login/component.js
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+import {ClusterSecrets} from '../../types/ClusterSecrets';
+
+export const component = {
+    name: 'clusterLogin',
+    bindings: {
+        secrets: '=',
+        onLogin: '&',
+        onHide: '&'
+    },
+    controller: class {
+        /** @type {ClusterSecrets} */
+        secrets;
+
+        login() {
+            if (this.form.$invalid)
+                return;
+
+            this.onLogin();
+        }
+    },
+    template
+};
diff --git a/modules/web-console/frontend/app/modules/agent/components/cluster-login/index.js b/modules/web-console/frontend/app/modules/agent/components/cluster-login/index.js
new file mode 100644
index 0000000000000..24cce4ff123f8
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/agent/components/cluster-login/index.js
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import {component} from './component';
+import service from './service';
+
+export default angular
+    .module('ignite-console.agent.cluster-login', [
+    ])
+    .service(service.name, service)
+    .component(component.name, component);
diff --git a/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js b/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js
new file mode 100644
index 0000000000000..a066bcf60ded6
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import _ from 'lodash';
+
+import {ClusterSecrets} from '../../types/ClusterSecrets';
+import {CancellationError} from 'app/errors/CancellationError';
+
+export default class ClusterLoginService {
+    static $inject = ['$modal', '$q'];
+
+    constructor($modal, $q) {
+        this.$modal = $modal;
+        this.$q = $q;
+    }
+
+    /**
+     * @param {ClusterSecrets} baseSecrets
+     * @return {ng.IDifferend<ClusterSecrets>}
+     */
+    askCredentials(baseSecrets) {
+        const deferred = this.$q.defer();
+
+        const modal = this.$modal({
+            template: `
+                <cluster-login
+                    secrets='$ctrl.secrets'
+                    on-login='$ctrl.onLogin()'
+                    on-hide='$ctrl.onHide()'
+                ></cluster-login>
+            `,
+            controller: [function() {
+                this.secrets = _.clone(baseSecrets);
+
+                this.onLogin = () => {
+                    deferred.resolve(this.secrets);
+                };
+
+                this.onHide = () => {
+                    deferred.reject(new CancellationError());
+                };
+            }],
+            controllerAs: '$ctrl',
+            backdrop: 'static',
+            show: true
+        });
+
+        return modal.$promise
+            .then(() => deferred.promise)
+            .finally(modal.hide);
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/agent/components/cluster-login/template.pug b/modules/web-console/frontend/app/modules/agent/components/cluster-login/template.pug
new file mode 100644
index 0000000000000..c6bc474947393
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/agent/components/cluster-login/template.pug
@@ -0,0 +1,57 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins
+
+.modal.modal--ignite.theme--ignite(tabindex='-1' role='dialog')
+    .modal-dialog
+       -var form = '$ctrl.form'
+
+       form.modal-content(name=form novalidate ng-submit='$ctrl.login()')
+            .modal-header
+                h4.modal-title
+                    i.icon-confirm
+                    span Cluster Authentication
+                button.close(type='button' aria-label='Close' ng-click='$ctrl.onHide()')
+                    svg(ignite-icon="cross")
+            .modal-body
+                p Enter your credentials to access the cluster.
+                .row
+                    .col-50
+                        +form-field__text({
+                            label: 'User:',
+                            model: '$ctrl.secrets.user',
+                            name: '"user"',
+                            placeholder: 'Enter user',
+                            required: true
+                        })(
+                            ng-model-options='{allowInvalid: true}'
+                            autocomplete='node-user'
+                            ignite-auto-focus
+                        )
+                    .col-50
+                        +form-field__password({
+                            label: 'Password:',
+                            model: '$ctrl.secrets.password',
+                            name: '"password"',
+                            placeholder: 'Enter password',
+                            required: true
+                        })(
+                            autocomplete='node-password'
+                        )
+            .modal-footer
+                button#btn-cancel.btn-ignite.btn-ignite--link-success(type='button' ng-click='$ctrl.onHide()') Cancel
+                button#btn-login.btn-ignite.btn-ignite--success Login
diff --git a/modules/web-console/frontend/app/modules/agent/types/Cluster.js b/modules/web-console/frontend/app/modules/agent/types/Cluster.js
new file mode 100644
index 0000000000000..dd054050e714f
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/agent/types/Cluster.js
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class Cluster {
+    /** @type {String} */
+    id;
+
+    /** @type {String} */
+    name;
+
+    /** @type {Boolean} */
+    connected = true;
+
+    /** @type {Boolean} */
+    secured;
+
+    constructor({id, name, secured = false}) {
+        this.id = id;
+        this.name = name;
+        this.secured = secured;
+    }
+}
+
diff --git a/modules/web-console/frontend/app/modules/agent/types/ClusterSecrets.js b/modules/web-console/frontend/app/modules/agent/types/ClusterSecrets.js
new file mode 100644
index 0000000000000..edff351097625
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/agent/types/ClusterSecrets.js
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {nonEmpty} from 'app/utils/lodashMixins';
+
+export class ClusterSecrets {
+    /** @type {String} */
+    user;
+
+    /** @type {String} */
+    password;
+
+    /** @type {String} */
+    sessionToken;
+
+    constructor() {
+        this.user = 'ignite';
+    }
+
+    hasCredentials() {
+        return nonEmpty(this.user) && nonEmpty(this.password);
+    }
+
+    resetCredentials() {
+        this.resetSessionToken();
+
+        this.password = null;
+    }
+
+    resetSessionToken() {
+        this.sessionToken = null;
+    }
+
+    /**
+     * @return {{sessionToken: String}|{'user': String, 'password': String}}
+     */
+    getCredentials() {
+        const { sessionToken } = this;
+
+        if (sessionToken)
+            return { sessionToken };
+
+        const { user, password } = this;
+
+        return { user, password };
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/agent/types/ClusterSecretsManager.js b/modules/web-console/frontend/app/modules/agent/types/ClusterSecretsManager.js
new file mode 100644
index 0000000000000..f5bd5ac80aa8b
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/agent/types/ClusterSecretsManager.js
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {ClusterSecrets} from './ClusterSecrets';
+
+export class ClusterSecretsManager {
+    /** @type {Map<String, ClusterSecrets>} */
+    memoryCache = new Map();
+
+    /**
+     * @param {String} clusterId
+     * @private
+     */
+    _has(clusterId) {
+        return this.memoryCache.has(clusterId);
+    }
+
+    /**
+     * @param {String} clusterId
+     * @private
+     */
+    _get(clusterId) {
+        return this.memoryCache.get(clusterId);
+    }
+
+    /**
+     * @param {String} clusterId
+     */
+    get(clusterId) {
+        if (this._has(clusterId))
+            return this._get(clusterId);
+
+        const secrets = new ClusterSecrets();
+
+        this.put(clusterId, secrets);
+
+        return secrets;
+    }
+
+    /**
+     * @param {String} clusterId
+     * @param {ClusterSecrets} secrets
+     */
+    put(clusterId, secrets) {
+        this.memoryCache.set(clusterId, secrets);
+    }
+
+    /**
+     * @param {String} clusterId
+     */
+    reset(clusterId) {
+        const secrets = this._get(clusterId);
+
+        secrets && secrets.resetCredentials();
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/branding/header-logo.pug b/modules/web-console/frontend/app/modules/branding/header-logo.pug
index b58f6706e48bf..f0453d39da8ab 100644
--- a/modules/web-console/frontend/app/modules/branding/header-logo.pug
+++ b/modules/web-console/frontend/app/modules/branding/header-logo.pug
@@ -15,4 +15,4 @@
     limitations under the License.
 
 a(ui-sref='signin')
-    img.navbar-brand(ng-src='{{logo.url}}' height='40')
+    img.navbar-brand(ng-src='{{logo.url}}')
diff --git a/modules/web-console/frontend/app/modules/branding/powered-by-apache.pug b/modules/web-console/frontend/app/modules/branding/powered-by-apache.pug
index af9aadfdb1e29..6031235968c8a 100644
--- a/modules/web-console/frontend/app/modules/branding/powered-by-apache.pug
+++ b/modules/web-console/frontend/app/modules/branding/powered-by-apache.pug
@@ -15,4 +15,4 @@
     limitations under the License.
 
 a(ng-if='poweredBy.show' href='//ignite.apache.org' target='_blank')
-    img(ng-src='/images/pb-ignite.png' height='65')
+    img(ng-src='/images/pb-ignite.png')
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index c5f82d3b80841..c3efea5db8438 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -1394,7 +1394,8 @@ export default class IgniteConfigurationGenerator {
                     .longProperty('rateTimeInterval');
             }
 
-            if (plcBean.isEmpty()) return;
+            if (plcBean.isEmpty())
+                return;
 
             policies.push(plcBean);
         });
@@ -1423,8 +1424,10 @@ export default class IgniteConfigurationGenerator {
             .intProperty('metricsSubIntervalCount')
             .longProperty('metricsRateTimeInterval')
             .longProperty('checkpointPageBufferSize')
-            .boolProperty('metricsEnabled')
-            .boolProperty('persistenceEnabled');
+            .boolProperty('metricsEnabled');
+
+        if (!plcBean.valueOf('swapPath'))
+            plcBean.boolProperty('persistenceEnabled');
 
         return plcBean;
     }
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.spec.js b/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.spec.js
index becc35988dc30..0ecada79d3b5c 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.spec.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Docker.service.spec.js
@@ -19,7 +19,7 @@ import DockerGenerator from './Docker.service';
 import {assert} from 'chai';
 import {outdent} from 'outdent/lib';
 
-suite.only('Dockerfile generator', () => {
+suite('Dockerfile generator', () => {
     const generator = new DockerGenerator();
 
     test('Target 2.0', () => {
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/Readme.service.js b/modules/web-console/frontend/app/modules/configuration/generator/Readme.service.js
index 0aa34ee2878b9..d0709128f8b36 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/Readme.service.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/Readme.service.js
@@ -49,13 +49,12 @@ export default class IgniteReadmeGenerator {
         sb.emptyLine();
 
         sb.append('Project structure:');
-        sb.append('    /config - this folder contains client and server XML configurations.');
         sb.append('    /jdbc-drivers - this folder should contains proprietary JDBC drivers.');
         sb.append('    /src - this folder contains generated java code.');
         sb.append('    /src/main/java/config - this folder contains generated java classes with cluster configuration from code.');
         sb.append('    /src/main/java/startup - this folder contains generated java classes with server and client nodes startup code.');
         sb.append('    /src/main/java/[model] - this optional folder will be named as package name for your POJO classes and contain generated POJO files.');
-        sb.append('    /src/main/resources - this optional folder contains generated secret.properties file with security sensitive information if any.');
+        sb.append('    /src/main/resources - this folder contains generated configurations in XML format and secret.properties file with security sensitive information if any.');
         sb.append('    Dockerfile - sample Docker file. With this file you could package Ignite deployment with all the dependencies into a standard container.');
         sb.append('    pom.xml - generated Maven project description, could be used to open generated project in IDE or build with Maven.');
         sb.append('    README.txt - this file.');
diff --git a/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js b/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js
index 56aa82a3fad4c..cecb49c54a770 100644
--- a/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js
+++ b/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js
@@ -24,7 +24,9 @@ export default ['bsSelect', [() => {
         const $render = ngModel.$render;
 
         ngModel.$render = () => {
-            if (scope.$destroyed) return;
+            if (scope.$destroyed)
+                return;
+
             scope.$applyAsync(() => {
                 $render();
                 const value = ngModel.$viewValue;
diff --git a/modules/web-console/frontend/app/modules/form/validator/unique.directive.js b/modules/web-console/frontend/app/modules/form/validator/unique.directive.js
index d32c565301c08..318b8b6c828a0 100644
--- a/modules/web-console/frontend/app/modules/form/validator/unique.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/unique.directive.js
@@ -42,12 +42,14 @@ class Controller {
 
             if (!this.skip) {
                 // Return true in case if array not exist, array empty.
-                if (!this.items || !this.items.length) return true;
+                if (!this.items || !this.items.length)
+                    return true;
 
                 const idx = this.items.findIndex(matches);
 
                 // In case of new element check all items.
-                if (isNew) return idx < 0;
+                if (isNew)
+                    return idx < 0;
 
                 // Case for new component list editable.
                 const $index = this.listEditableTransclude
diff --git a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js
index fbe6203ea5396..2296aedccd374 100644
--- a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js
+++ b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js
@@ -18,9 +18,9 @@
 const NID_TEMPLATE = '<div class="ui-grid-cell-contents" title="{{ COL_FIELD }}">{{ COL_FIELD | limitTo:8 }}</div>';
 
 const COLUMNS_DEFS = [
-    {displayName: 'Node ID8', field: 'nid', headerTooltip: 'Node ID8', cellTemplate: NID_TEMPLATE, minWidth: 85, width: 85, pinnedLeft: true},
-    {displayName: 'Node IP', field: 'ip', headerTooltip: 'Primary IP address of node', minWidth: 75, width: 120},
-    {displayName: 'Grid name', field: 'gridName', headerTooltip: 'Name of node grid cluster', minWidth: 75, width: 120},
+    {displayName: 'Node ID8', field: 'nid', headerTooltip: 'Node ID8', cellTemplate: NID_TEMPLATE, minWidth: 85, width: 145, pinnedLeft: true},
+    {displayName: 'Node IP', field: 'ip', headerTooltip: 'Primary IP address of node', minWidth: 100, width: 150},
+    {displayName: 'Grid name', field: 'gridName', headerTooltip: 'Name of node grid cluster', minWidth: 110, width: 150},
     {displayName: 'Version', field: 'version', headerTooltip: 'Node version', minWidth: 75, width: 140},
     {displayName: 'OS information', field: 'os', headerTooltip: 'OS information for node\'s host', minWidth: 125}
 ];
diff --git a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.scss b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.scss
index 8145dbc4a3f98..cbb90006a0d77 100644
--- a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.scss
+++ b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.scss
@@ -16,22 +16,17 @@
  */
 
 .ignite-nodes-dialog {
-    label {
-        font-size: 18px;
-        margin-right: 20px;
-    }
-
     .ui-grid-pinned-container.ui-grid-pinned-container-left .ui-grid-cell:last-child,
     .ui-grid-pinned-container.ui-grid-pinned-container-left .ui-grid-header-cell:last-child,
     .ui-grid-header-cell:last-child .ui-grid-column-resizer.right {
-      //border-right: none;
+      border-right: none;
     }
 
-    .nodes-grid {
-        height: 320px;
+    .modal-dialog {
+        width: 900px;
     }
-    .panel-body_collapse {
-        padding: 0;
-        margin: 0;
+
+    button.pull-left.btn-ignite {
+        background: none;
     }
 }
diff --git a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
index d9ea68cdff2a4..22f3b1d057c9e 100644
--- a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
+++ b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
@@ -14,22 +14,28 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-.modal.ignite-nodes-dialog(tabindex='-1' role='dialog')
-    .modal-dialog
-        .modal-content
+.modal.modal--ignite.theme--ignite.ignite-nodes-dialog(tabindex='-1' role='dialog')
+    .modal-dialog.modal-dialog--adjust-height
+        form.modal-content
             .modal-header
-                button.close(ng-click='$cancel()' aria-hidden='true') &times;
                 h4.modal-title Select Node
+                button.close(type='button' aria-label='Close' ng-click='$cancel()')
+                     svg(ignite-icon="cross")
             .modal-body.modal-body-with-scroll
                 p Choose node to execute query for cache: #[strong {{ $ctrl.options.target }}]
 
-                .panel.panel-default.nodes-grid
-                    .panel-heading
-                        label Cache Nodes: {{ $ctrl.nodes.length }}
+                ul.tabs.tabs--blue
+                    li.active(role='presentation')
+                        a
+                            span Cache Nodes
+                            span.badge.badge--blue {{ $ctrl.data.length }}
 
-                    .panel-body.panel-body_collapse
-                        .grid(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-pinning)
+                .panel--ignite
+                    .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-pinning ui-grid-hovering)
 
             .modal-footer
-                button.btn.btn-primary(id='confirm-btn-confirm' ng-click='$ok($ctrl.selected)' ng-disabled='$ctrl.selected.length === 0') Select node
-                button.btn.btn-default(id='confirm-btn-close' ng-click='$cancel()') Cancel
+                button.pull-left.btn-ignite(disabled)
+                    grid-item-selected(class='pull-left' grid-api='$ctrl.gridApi')
+
+                button.btn-ignite.btn-ignite--link-success(id='confirm-btn-close' ng-click='$cancel()') Cancel
+                button.btn-ignite.btn-ignite--success(id='confirm-btn-confirm' ng-click='$ok($ctrl.selected)' ng-disabled='$ctrl.selected.length === 0') Select node
diff --git a/modules/web-console/frontend/app/modules/states/admin.state.js b/modules/web-console/frontend/app/modules/states/admin.state.js
index 208cd2c861c3b..f45421aedcfbf 100644
--- a/modules/web-console/frontend/app/modules/states/admin.state.js
+++ b/modules/web-console/frontend/app/modules/states/admin.state.js
@@ -17,8 +17,6 @@
 
 import angular from 'angular';
 
-import template from 'views/base2.pug';
-
 angular
 .module('ignite-console.states.admin', [
     'ui.router'
@@ -28,15 +26,7 @@ angular
     $stateProvider
     .state('base.settings.admin', {
         url: '/admin',
-        views: {
-            '@': {
-                template
-            },
-            '@base.settings.admin': {
-                template: '<page-admin></page-admin>'
-            }
-        },
-        // templateUrl,
+        component: 'pageAdmin',
         permission: 'admin_page',
         tfMetaTags: {
             title: 'Admin panel'
diff --git a/modules/web-console/frontend/app/modules/states/settings.state.js b/modules/web-console/frontend/app/modules/states/settings.state.js
new file mode 100644
index 0000000000000..1651376cdd015
--- /dev/null
+++ b/modules/web-console/frontend/app/modules/states/settings.state.js
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import angular from 'angular';
+
+angular
+    .module('ignite-console.states.settings', [
+        'ui.router'
+    ])
+    .config(['$stateProvider', function($stateProvider) {
+        // Set up the states.
+        $stateProvider
+            .state('base.settings', {
+                url: '/settings',
+                abstract: true,
+                template: '<ui-view></ui-view>'
+            });
+    }]);
diff --git a/modules/web-console/frontend/app/modules/user/Auth.service.js b/modules/web-console/frontend/app/modules/user/Auth.service.js
index 5102c20d5d025..744943ad05c98 100644
--- a/modules/web-console/frontend/app/modules/user/Auth.service.js
+++ b/modules/web-console/frontend/app/modules/user/Auth.service.js
@@ -48,14 +48,14 @@ export default class AuthService {
         return this._auth('signup', userInfo);
     }
     /**
-     * @param {string} email    
-     * @param {string} password 
+     * @param {string} email
+     * @param {string} password
      */
     signin(email, password) {
         return this._auth('signin', {email, password});
     }
     /**
-     * @param {string} email    
+     * @param {string} email
      */
     remindPassword(email) {
         return this._auth('password/forgot', {email}).then(() => this.$state.go('password.send'));
@@ -65,8 +65,8 @@ export default class AuthService {
     /**
      * Performs the REST API call.
      * @private
-     * @param {('signin'|'signup'|'password/forgot')} action   
-     * @param {{email:string,password:string}|SignupUserInfo|{email:string}} userInfo 
+     * @param {('signin'|'signup'|'password/forgot')} action
+     * @param {{email:string,password:string}|SignupUserInfo|{email:string}} userInfo
      */
     _auth(action, userInfo) {
         return this.$http.post('/api/v1/' + action, userInfo)
diff --git a/modules/web-console/frontend/app/primitives/badge/index.scss b/modules/web-console/frontend/app/primitives/badge/index.scss
index 79082e0bcb770..96d6b54c246a6 100644
--- a/modules/web-console/frontend/app/primitives/badge/index.scss
+++ b/modules/web-console/frontend/app/primitives/badge/index.scss
@@ -22,7 +22,7 @@
   min-width: 26px;
   height: 18px;
 
-  padding: 2px 9px;
+  padding: 3px 9px;
 
   border-radius: 9px;
 
diff --git a/modules/web-console/frontend/app/primitives/btn/index.scss b/modules/web-console/frontend/app/primitives/btn/index.scss
index 061d4119f9be9..2d9e9c43428f8 100644
--- a/modules/web-console/frontend/app/primitives/btn/index.scss
+++ b/modules/web-console/frontend/app/primitives/btn/index.scss
@@ -90,6 +90,11 @@ $btn-content-padding-with-border: 9px 11px;
         -webkit-pointer-events: none;
         pointer-events: none;
     }
+
+    [ignite-icon='plus'] {
+        height: 12px;
+        width: 12px;
+    }
 }
 
 .btn-ignite--primary {
diff --git a/modules/web-console/frontend/app/primitives/datepicker/index.scss b/modules/web-console/frontend/app/primitives/datepicker/index.scss
index 16707044cbf44..b0c13e6ea6b70 100644
--- a/modules/web-console/frontend/app/primitives/datepicker/index.scss
+++ b/modules/web-console/frontend/app/primitives/datepicker/index.scss
@@ -57,6 +57,8 @@
     }
 
     .ignite-form-field__control {
+        @import "./../../../public/stylesheets/variables.scss";
+
         width: auto;
 
         input {
@@ -72,13 +74,13 @@
             font-size: inherit;
             line-height: $height;
             text-align: left;
-            text-shadow: 0 0 0 #ee2b27;
+            text-shadow: 0 0 0 $ignite-brand-success;
 
             border: none;
             box-shadow: none;
 
             &:hover, &:focus {
-                text-shadow: 0 0 0 #a8110f;
+                text-shadow: 0 0 0 change-color($ignite-brand-success, $lightness: 26%);
             }
         }
     }
diff --git a/modules/web-console/frontend/app/primitives/form-field/checkbox.pug b/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
index 09596d46287e2..b498cbd51de6a 100644
--- a/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
@@ -15,7 +15,7 @@
     limitations under the License.
 
 mixin form-field__checkbox({ label, model, name, disabled, required, tip })
-    .form-field.form-field__checkbox
+    .form-field.form-field__checkbox(id=`{{ ${name} }}Field`)
         +form-field__label({ label, name, required })
             +form-field__tooltip({ title: tip, options: tipOpts })
 
diff --git a/modules/web-console/frontend/app/primitives/form-field/dropdown.pug b/modules/web-console/frontend/app/primitives/form-field/dropdown.pug
index cb058295c0077..de83bf9229ce3 100644
--- a/modules/web-console/frontend/app/primitives/form-field/dropdown.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/dropdown.pug
@@ -15,8 +15,10 @@
     limitations under the License.
 
 mixin form-field__dropdown({ label, model, name, disabled, required, multiple, placeholder, placeholderEmpty, options, tip })
+    -var errLbl = label.substring(0, label.length - 1)
     mixin __form-field__input()
         button.select-toggle(
+            type='button'
             id=`{{ ${name} }}Input`
             name=`{{ ${name} }}`
 
diff --git a/modules/web-console/frontend/app/primitives/form-field/index.scss b/modules/web-console/frontend/app/primitives/form-field/index.scss
index 1035adecfc6c9..070dfe722649b 100644
--- a/modules/web-console/frontend/app/primitives/form-field/index.scss
+++ b/modules/web-console/frontend/app/primitives/form-field/index.scss
@@ -130,7 +130,7 @@
     }
    .ignite-form-field__errors {
         color: $ignite-brand-primary;
-        padding: 5px 10px 0px;
+        padding: 5px 10px 0;
         line-height: 14px;
         font-size: 12px;
         clear: both;
@@ -215,6 +215,7 @@
             box-sizing: border-box;
             width: 100%;
             max-width: initial;
+            min-width: 0;
             height: 36px;
             padding: 9px 10px;
             margin-right: 0;
@@ -241,7 +242,16 @@
             }
         }
 
+        &--postfix::after {
+            content: attr(data-postfix);
+            display: inline-flex;
+            place-self: center;
+            margin-left: 10px;
+        }
+
         // Added right offset to appearance of input for invalid password
+        & > input[type='email'].ng-invalid.ng-touched,
+        & > input[type='text'].ng-invalid.ng-touched,
         & > input[type='password'].ng-invalid.ng-touched {
             padding-right: 36px;
         }
@@ -259,6 +269,8 @@
         right: 0;
         bottom: 0;
 
+        display: flex;
+
         [ng-message] {
             // TODO: remove after replace all fields to new
             overflow: visible !important;
@@ -266,6 +278,12 @@
         }
     }
 
+    &__control--postfix + &__errors::after {
+        content: attr(data-postfix);
+        margin-left: 10px;
+        visibility: hidden;
+    }
+
     &__error {
         z-index: 2;
         position: relative;
@@ -307,6 +325,7 @@
 }
 
 .form-field__checkbox {
+    $errorSize: 16px;
     display: flex;
 
     .form-field {
@@ -319,12 +338,91 @@
             width: auto;
             margin-right: 10px;
             padding: 3px 0;
+            flex: 0 0 auto;
 
             input {
                 width: auto;
                 height: auto;
                 margin: 0;
+                border-radius: 0;
+            }
+        }
+
+        &__errors {
+            position: static;
+            right: initial;
+            bottom: initial;
+            order: 3;
+            margin-left: 5px;
+
+            .form-field__error {
+                width: $errorSize;
+                height: $errorSize;
+
+                div {
+                    // Required to correctly position error popover
+                    top: -10px;
+                    height: 36px;
+                    
+                    width: $errorSize;                    
+                }
+
+                [ignite-icon] {
+                    width: $errorSize;
+                    top: 0;
+                }
             }
         }
     }
 }
+
+.form-field__password {
+    // Validation error notification will overlap with visibility button if it's not moved more to the left
+    input[type='password'].ng-invalid.ng-touched,
+    input[type='password'].ng-invalid.ng-touched + input {
+        padding-right: 62px;
+    }
+
+    // Extra space for visibility button
+    input {
+        padding-right: 36px;
+    }
+
+    // Distance between error notification and visibility button
+    .form-field__errors {
+        right: 26px;
+    }
+
+    password-visibility-toggle-button {
+        position: absolute;
+        right: 0;
+        height: 36px;
+    }
+}
+
+.form-fieldset {
+    padding: 10px;
+
+    border: 1px solid hsla(0,0%,77%,.5);
+    border-radius: 4px;
+
+    legend {
+        width: auto;
+        margin: 0 -5px;
+        padding: 0 5px;
+
+        border: 0;
+
+        color: #393939;
+        font-size: 14px;
+        line-height: 1.42857;
+    }
+
+    legend + * {
+        margin-top: 0 !important;
+    }
+
+    & > *:last-child {
+        margin-bottom: 0 !important;
+    }
+}
diff --git a/modules/web-console/frontend/app/primitives/form-field/number.pug b/modules/web-console/frontend/app/primitives/form-field/number.pug
index e5f14548f969f..11f8e22bc2ee8 100644
--- a/modules/web-console/frontend/app/primitives/form-field/number.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/number.pug
@@ -14,14 +14,14 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin form-field__number({ label, model, name, disabled, required, placeholder, tip, min, max, step })
+mixin form-field__number({ label, model, name, disabled, required, placeholder, tip, min, max, step, postfix })
     -var errLbl = label.substring(0, label.length - 1)
 
     .form-field
         +form-field__label({ label, name, required })
             +form-field__tooltip({ title: tip, options: tipOpts })
 
-        .form-field__control
+        .form-field__control(class=postfix && 'form-field__control--postfix' data-postfix=postfix)
             - attributes.type = 'number'
             - attributes.min = min ? min : '0'
             - attributes.max = max ? max : '{{ Number.MAX_VALUE }}'
@@ -29,6 +29,7 @@ mixin form-field__number({ label, model, name, disabled, required, placeholder,
             +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes)
 
         .form-field__errors(
+            data-postfix=postfix
             ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}`
         )
             if block
diff --git a/modules/web-console/frontend/app/primitives/form-field/password.pug b/modules/web-console/frontend/app/primitives/form-field/password.pug
index ba38cce8e833d..40e1aa90b87d7 100644
--- a/modules/web-console/frontend/app/primitives/form-field/password.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/password.pug
@@ -17,13 +17,23 @@
 mixin form-field__password({ label, model, name, disabled, required, placeholder, tip })
     -var errLbl = label.substring(0, label.length - 1)
 
-    .form-field
+    .form-field.form-field__password(
+        password-visibility-root
+        on-password-visibility-toggle=`${form}[${name}].$setTouched()`
+    )
         +form-field__label({ label, name, required })
             +form-field__tooltip({ title: tip, options: tipOpts })
 
         .form-field__control
             - attributes.type='password'
+            - attributes.class = 'password-visibility__password-hidden'
             +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes)
+            - attributes.class = 'password-visibility__password-visible'
+            - attributes.type='text'
+            - attributes.autocomplete = 'off'
+            +form-field__input({ name: name + `+"Text"`, model, disabled, required, placeholder })(attributes=attributes)
+
+            password-visibility-toggle-button
 
         .form-field__errors(
             ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}`
diff --git a/modules/web-console/frontend/app/primitives/panel/index.scss b/modules/web-console/frontend/app/primitives/panel/index.scss
index e16210ce5b2b2..5dab39f31fb01 100644
--- a/modules/web-console/frontend/app/primitives/panel/index.scss
+++ b/modules/web-console/frontend/app/primitives/panel/index.scss
@@ -54,7 +54,7 @@
             padding: 22px 20px;
 
             background-color: initial;
-            border-bottom: 1px solid $ignite-brand-primary;
+            border-bottom: 1px solid $table-border-color;
 
             &:hover {
                 text-decoration: none;
diff --git a/modules/web-console/frontend/app/primitives/switcher/index.scss b/modules/web-console/frontend/app/primitives/switcher/index.scss
index fb2fd1b0d5cd8..9430f71510f69 100644
--- a/modules/web-console/frontend/app/primitives/switcher/index.scss
+++ b/modules/web-console/frontend/app/primitives/switcher/index.scss
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-@import 'public/stylesheets/variables';
+@import '../../../public/stylesheets/variables';
 
 label.switcher--ignite {
     $width: 34px;
diff --git a/modules/web-console/frontend/app/primitives/timepicker/index.scss b/modules/web-console/frontend/app/primitives/timepicker/index.scss
index 361e9491382c8..5be534e10f2dd 100644
--- a/modules/web-console/frontend/app/primitives/timepicker/index.scss
+++ b/modules/web-console/frontend/app/primitives/timepicker/index.scss
@@ -46,6 +46,8 @@
     }
 
     .ignite-form-field__control {
+        @import "./../../../public/stylesheets/variables.scss";
+
         width: auto;
 
         input {
@@ -61,14 +63,14 @@
             font-size: inherit;
             line-height: $height;
             text-align: left;
-            text-shadow: 0 0 0 #ee2b27;
+            text-shadow: 0 0 0 $ignite-brand-success;
 
             border: none;
             box-shadow: none;
             background: none;
 
             &:hover, &:focus {
-                text-shadow: 0 0 0 #a8110f;
+                text-shadow: 0 0 0 change-color($ignite-brand-success, $lightness: 26%);
             }
         }
     }
diff --git a/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss b/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss
index 1ff27b29c05dd..7c2efc3196471 100644
--- a/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss
+++ b/modules/web-console/frontend/app/primitives/ui-grid-header/index.scss
@@ -20,6 +20,10 @@
 .ui-grid-header--subcategories {
     border-color: $table-border-color;
 
+    .ui-grid-header-canvas {
+        background-color: #f5f5f5;
+    }
+
     .ui-grid-row:nth-child(even) .ui-grid-cell.cell-total {
         background-color: rgba(102,175,233,.6);
     }
@@ -115,7 +119,8 @@
     }
 }
 
-.ui-grid[ui-grid-selection][ui-grid-grouping] {
+.ui-grid[ui-grid-selection][ui-grid-grouping],
+.ui-grid[ui-grid-selection][ui-grid-tree-view] {
     .ui-grid-pinned-container-left {
         .ui-grid-header--subcategories {
             .ui-grid-header-span {
diff --git a/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss b/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss
index 56bab220a1e56..0a0d4a39d7e3a 100644
--- a/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss
+++ b/modules/web-console/frontend/app/primitives/ui-grid-settings/index.scss
@@ -194,6 +194,8 @@
             }
 
             .ignite-form-field__control {
+                @import "./../../../public/stylesheets/variables.scss";
+
                 width: auto;
 
                 .input-tip {
@@ -213,13 +215,13 @@
                         font-size: inherit;
                         line-height: $height;
                         text-align: left;
-                        text-shadow: 0 0 0 #ee2b27;
+                        text-shadow: 0 0 0 $ignite-brand-success;
 
                         border: none;
                         box-shadow: none;
 
                         &:hover, &:focus {
-                            text-shadow: 0 0 0 #a8110f;
+                            text-shadow: 0 0 0 change-color($ignite-brand-success, $lightness: 26%);
                         }
 
                         button {
diff --git a/modules/web-console/frontend/app/primitives/ui-grid/index.scss b/modules/web-console/frontend/app/primitives/ui-grid/index.scss
index 2a5c5872a5917..c450dadaad201 100644
--- a/modules/web-console/frontend/app/primitives/ui-grid/index.scss
+++ b/modules/web-console/frontend/app/primitives/ui-grid/index.scss
@@ -17,6 +17,14 @@
 
 @import '../../../public/stylesheets/variables';
 
+// Use this class to control grid header height
+.ui-grid-ignite__panel {
+    $panel-height: 64px;
+    $title-height: 36px;
+    padding-top: ($panel-height - $title-height) / 2 !important;
+    padding-bottom: ($panel-height - $title-height) / 2 !important;
+}
+
 .ui-grid.ui-grid--ignite {
     $height: 46px;
 
@@ -147,6 +155,10 @@
     }
 
     .ui-grid-header--subcategories {
+        .ui-grid-header-canvas {
+            background-color: white;
+        }
+
         .ui-grid-header-span.ui-grid-header-cell {
             background: initial;
 
diff --git a/modules/web-console/frontend/app/services/Caches.js b/modules/web-console/frontend/app/services/Caches.js
index add63f85a63f7..7df4bb6ec13d8 100644
--- a/modules/web-console/frontend/app/services/Caches.js
+++ b/modules/web-console/frontend/app/services/Caches.js
@@ -165,7 +165,10 @@ export default class Caches {
         maxMemorySize: {
             min: (evictionPolicy) => {
                 const policy = evictionPolicy[evictionPolicy.kind];
-                if (!policy) return true;
+
+                if (!policy)
+                    return true;
+
                 const maxSize = policy.maxSize === null || policy.maxSize === void 0
                     ? this.evictionPolicy.maxSize.default
                     : policy.maxSize;
@@ -177,7 +180,10 @@ export default class Caches {
         maxSize: {
             min: (evictionPolicy) => {
                 const policy = evictionPolicy[evictionPolicy.kind];
-                if (!policy) return true;
+
+                if (!policy)
+                    return true;
+
                 const maxMemorySize = policy.maxMemorySize === null || policy.maxMemorySize === void 0
                     ? this.evictionPolicy.maxMemorySize.default
                     : policy.maxMemorySize;
diff --git a/modules/web-console/frontend/app/services/Clusters.js b/modules/web-console/frontend/app/services/Clusters.js
index 4e057fcf8652a..0228a77b4d2cd 100644
--- a/modules/web-console/frontend/app/services/Clusters.js
+++ b/modules/web-console/frontend/app/services/Clusters.js
@@ -57,7 +57,9 @@ export default class Clusters {
             messageQueueLimit = this.messageQueueLimit.default,
             ackSendThreshold = this.ackSendThreshold.default
         ) => {
-            if (currentValue === this.unacknowledgedMessagesBufferSize.default) return currentValue;
+            if (currentValue === this.unacknowledgedMessagesBufferSize.default)
+                return currentValue;
+
             const {validRatio} = this.unacknowledgedMessagesBufferSize;
             return Math.max(messageQueueLimit * validRatio, ackSendThreshold * validRatio);
         },
@@ -186,7 +188,11 @@ export default class Clusters {
                 tcpNoDelay: true
             },
             clientConnectorConfiguration: {
-                tcpNoDelay: true
+                tcpNoDelay: true,
+                jdbcEnabled: true,
+                odbcEnabled: true,
+                thinClientEnabled: true,
+                useIgniteSslContextFactory: true
             },
             space: void 0,
             discovery: {
@@ -253,7 +259,9 @@ export default class Clusters {
         maxSize: {
             default: '0.2 * totalMemoryAvailable',
             min: (dataRegion) => {
-                if (!dataRegion) return;
+                if (!dataRegion)
+                    return;
+
                 return dataRegion.initialSize || this.dataRegion.initialSize.default;
             }
         },
@@ -267,20 +275,23 @@ export default class Clusters {
             default: 100,
             min: 11,
             max: (cluster, dataRegion) => {
-                if (!cluster || !dataRegion || !dataRegion.maxSize) return;
+                if (!cluster || !dataRegion || !dataRegion.maxSize)
+                    return;
+
                 const perThreadLimit = 10; // Took from Ignite
                 const maxSize = dataRegion.maxSize;
                 const pageSize = cluster.dataStorageConfiguration.pageSize || this.dataStorageConfiguration.pageSize.default;
                 const maxPoolSize = Math.floor(maxSize / pageSize / perThreadLimit);
+
                 return maxPoolSize;
             }
         },
-        subIntervals: {
+        metricsSubIntervalCount: {
             default: 5,
             min: 1,
             step: 1
         },
-        rateTimeInterval: {
+        metricsRateTimeInterval: {
             min: 1000,
             default: 60000,
             step: 1000
@@ -293,7 +304,10 @@ export default class Clusters {
 
     addDataRegionConfiguration(cluster) {
         const dataRegionConfigurations = get(cluster, 'dataStorageConfiguration.dataRegionConfigurations');
-        if (!dataRegionConfigurations) return;
+
+        if (!dataRegionConfigurations)
+            return;
+
         return dataRegionConfigurations.push(Object.assign(this.makeBlankDataRegionConfiguration(), {
             name: uniqueName('New data region', dataRegionConfigurations.concat(cluster.dataStorageConfiguration.defaultDataRegionConfiguration))
         }));
@@ -318,7 +332,10 @@ export default class Clusters {
             defaultMemoryPolicyExists: (name, items = []) => {
                 const def = this.memoryPolicy.name.default;
                 const normalizedName = (name || def);
-                if (normalizedName === def) return true;
+
+                if (normalizedName === def)
+                    return true;
+
                 return items.some((policy) => (policy.name || def) === normalizedName);
             },
             uniqueMemoryPolicyName: (a, items = []) => {
@@ -330,7 +347,9 @@ export default class Clusters {
             default: 100,
             min: 11,
             max: (cluster, memoryPolicy) => {
-                if (!memoryPolicy || !memoryPolicy.maxSize) return;
+                if (!memoryPolicy || !memoryPolicy.maxSize)
+                    return;
+
                 const perThreadLimit = 10; // Took from Ignite
                 const maxSize = memoryPolicy.maxSize;
                 const pageSize = cluster.memoryConfiguration.pageSize || this.memoryConfiguration.pageSize.default;
@@ -406,7 +425,10 @@ export default class Clusters {
 
     addMemoryPolicy(cluster) {
         const memoryPolicies = get(cluster, 'memoryConfiguration.memoryPolicies');
-        if (!memoryPolicies) return;
+
+        if (!memoryPolicies)
+            return;
+
         return memoryPolicies.push(Object.assign(this.makeBlankMemoryPolicy(), {
             // Blank name for default policy if there are not other policies
             name: memoryPolicies.length ? uniqueName('New memory policy', memoryPolicies) : ''
diff --git a/modules/web-console/frontend/app/services/ErrorParser.service.js b/modules/web-console/frontend/app/services/ErrorParser.service.js
new file mode 100644
index 0000000000000..9cda8ed1d60b2
--- /dev/null
+++ b/modules/web-console/frontend/app/services/ErrorParser.service.js
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import isEmpty from 'lodash/isEmpty';
+import {nonEmpty} from 'app/utils/lodashMixins';
+
+export default class {
+    static $inject = ['JavaTypes'];
+
+    /**
+     * @param JavaTypes service.
+     */
+    constructor(JavaTypes) {
+        this.JavaTypes = JavaTypes;
+    }
+
+    extractMessage(err, prefix) {
+        prefix = prefix || '';
+
+        if (err) {
+            if (err.hasOwnProperty('data'))
+                err = err.data;
+
+            if (err.hasOwnProperty('message')) {
+                let msg = err.message;
+
+                const traceIndex = msg.indexOf(', trace=');
+
+                if (traceIndex > 0)
+                    msg = msg.substring(0, traceIndex);
+
+                const lastIdx = msg.lastIndexOf(' err=');
+                let msgEndIdx = msg.indexOf(']', lastIdx);
+
+                if (lastIdx > 0 && msgEndIdx > 0) {
+                    let startIdx = msg.indexOf('[', lastIdx);
+
+                    while (startIdx > 0) {
+                        const tmpIdx = msg.indexOf(']', msgEndIdx + 1);
+
+                        if (tmpIdx > 0)
+                            msgEndIdx = tmpIdx;
+
+                        startIdx = msg.indexOf('[', startIdx + 1);
+                    }
+                }
+
+                return prefix + (lastIdx >= 0 ? msg.substring(lastIdx + 5, msgEndIdx > 0 ? msgEndIdx : traceIndex) : msg);
+            }
+
+            if (nonEmpty(err.className)) {
+                if (isEmpty(prefix))
+                    prefix = 'Internal cluster error: ';
+
+                return prefix + err.className;
+            }
+
+            return prefix + err;
+        }
+
+        return prefix + 'Internal error.';
+    }
+
+    extractFullMessage(err) {
+        const clsName = _.isEmpty(err.className) ? '' : '[' + this.JavaTypes.shortClassName(err.className) + '] ';
+
+        let msg = err.message || '';
+        const traceIndex = msg.indexOf(', trace=');
+
+        if (traceIndex > 0)
+            msg = msg.substring(0, traceIndex);
+
+        return clsName + (msg);
+    }
+}
diff --git a/modules/web-console/frontend/app/services/FormUtils.service.js b/modules/web-console/frontend/app/services/FormUtils.service.js
index da1d73700d59b..2c81c5740da92 100644
--- a/modules/web-console/frontend/app/services/FormUtils.service.js
+++ b/modules/web-console/frontend/app/services/FormUtils.service.js
@@ -16,7 +16,7 @@
  */
 import _ from 'lodash';
 
-export default ['IgniteFormUtils', ['$window', 'IgniteFocus', ($window, Focus) => {
+export default ['IgniteFormUtils', ['$window', 'IgniteFocus', '$rootScope', ($window, Focus, $rootScope) => {
     function ensureActivePanel(ui, pnl, focusId) {
         if (ui && ui.loadPanel) {
             const collapses = $('[bs-collapse-target]');
@@ -326,19 +326,26 @@ export default ['IgniteFormUtils', ['$window', 'IgniteFocus', ($window, Focus) =
     }
 
     // TODO: move somewhere else
-    function triggerValidation(form, $scope) {
+    function triggerValidation(form) {
         const fe = (m) => Object.keys(m.$error)[0];
         const em = (e) => (m) => {
-            if (!e) return;
+            if (!e)
+                return;
+
             const walk = (m) => {
-                if (!m.$error[e]) return;
-                if (m.$error[e] === true) return m;
+                if (!m.$error[e])
+                    return;
+
+                if (m.$error[e] === true)
+                    return m;
+
                 return walk(m.$error[e][0]);
             };
+
             return walk(m);
         };
 
-        $scope.$broadcast('$showValidationError', em(fe(form))(form));
+        $rootScope.$broadcast('$showValidationError', em(fe(form))(form));
     }
 
     return {
diff --git a/modules/web-console/frontend/app/services/IGFSs.js b/modules/web-console/frontend/app/services/IGFSs.js
index 87dfd17eb8866..de9e43d253b36 100644
--- a/modules/web-console/frontend/app/services/IGFSs.js
+++ b/modules/web-console/frontend/app/services/IGFSs.js
@@ -64,11 +64,15 @@ export default class IGFSs {
 
     secondaryFileSystemEnabled = {
         requiredWhenIGFSProxyMode: (igfs) => {
-            if (get(igfs, 'defaultMode') === 'PROXY') return get(igfs, 'secondaryFileSystemEnabled') === true;
+            if (get(igfs, 'defaultMode') === 'PROXY')
+                return get(igfs, 'secondaryFileSystemEnabled') === true;
+
             return true;
         },
         requiredWhenPathModeProxyMode: (igfs) => {
-            if (get(igfs, 'pathModes', []).some((pm) => pm.mode === 'PROXY')) return get(igfs, 'secondaryFileSystemEnabled') === true;
+            if (get(igfs, 'pathModes', []).some((pm) => pm.mode === 'PROXY'))
+                return get(igfs, 'secondaryFileSystemEnabled') === true;
+
             return true;
         }
     };
diff --git a/modules/web-console/frontend/app/services/LegacyUtils.service.js b/modules/web-console/frontend/app/services/LegacyUtils.service.js
index 8f283c03a0567..a16934354f431 100644
--- a/modules/web-console/frontend/app/services/LegacyUtils.service.js
+++ b/modules/web-console/frontend/app/services/LegacyUtils.service.js
@@ -183,7 +183,7 @@ export default ['IgniteLegacyUtils', ['IgniteErrorPopover', (ErrorPopover) => {
         'volatile',
         'while'
     ];
-    /*eslint-enable */
+    /* eslint-enable */
 
     const VALID_JAVA_IDENTIFIER = new RegExp('^[a-zA-Z_$][a-zA-Z\\d_$]*$');
 
diff --git a/modules/web-console/frontend/app/services/Messages.service.js b/modules/web-console/frontend/app/services/Messages.service.js
index 620d3728eb7fb..1337e24c1e0f6 100644
--- a/modules/web-console/frontend/app/services/Messages.service.js
+++ b/modules/web-console/frontend/app/services/Messages.service.js
@@ -16,40 +16,14 @@
  */
 
 import {CancellationError} from 'app/errors/CancellationError';
-import isEmpty from 'lodash/isEmpty';
-import {nonEmpty} from 'app/utils/lodashMixins';
 
 // Service to show various information and error messages.
-export default ['IgniteMessages', ['$alert', ($alert) => {
+export default ['IgniteMessages', ['$alert', 'IgniteErrorParser', ($alert, errorParser) => {
     // Common instance of alert modal.
     let msgModal;
 
     const errorMessage = (prefix, err) => {
-        prefix = prefix || '';
-
-        if (err) {
-            if (err.hasOwnProperty('data'))
-                err = err.data;
-
-            if (err.hasOwnProperty('message')) {
-                const msg = err.message;
-
-                const errIndex = msg.indexOf(' err=');
-
-                return prefix + (errIndex >= 0 ? msg.substring(errIndex + 5, msg.length - 1) : msg);
-            }
-
-            if (nonEmpty(err.className)) {
-                if (isEmpty(prefix))
-                    prefix = 'Internal cluster error: ';
-
-                return prefix + err.className;
-            }
-
-            return prefix + err;
-        }
-
-        return prefix + 'Internal error.';
+        return errorParser.extractMessage(err, prefix);
     };
 
     const hideAlert = () => {
@@ -73,16 +47,16 @@ export default ['IgniteMessages', ['$alert', ($alert) => {
     return {
         errorMessage,
         hideAlert,
-        showError(message, err) {
+        showError(message, err, duration = 10) {
             if (message instanceof CancellationError)
                 return false;
 
-            _showMessage(message, err, 'danger', 10);
+            _showMessage(message, err, 'danger', duration);
 
             return false;
         },
-        showInfo(message) {
-            _showMessage(message, null, 'success', 3);
+        showInfo(message, duration = 5) {
+            _showMessage(message, null, 'success', duration);
         }
     };
 }]];
diff --git a/modules/web-console/frontend/app/services/Models.js b/modules/web-console/frontend/app/services/Models.js
index 3b714c4599154..4b360be5ca148 100644
--- a/modules/web-console/frontend/app/services/Models.js
+++ b/modules/web-console/frontend/app/services/Models.js
@@ -85,14 +85,19 @@ export default class Models {
      * @param {ig.config.model.DomainModel} model
      */
     addIndex(model) {
-        if (!model) return;
-        if (!model.indexes) model.indexes = [];
+        if (!model)
+            return;
+
+        if (!model.indexes)
+            model.indexes = [];
+
         model.indexes.push({
             _id: ObjectID.generate(),
             name: '',
             indexType: 'SORTED',
             fields: []
         });
+
         return model.indexes[model.indexes.length - 1];
     }
 
@@ -142,7 +147,9 @@ export default class Models {
          */
         indexFieldsHaveUniqueNames: ($value = []) => {
             return $value.every((index) => {
-                if (!index.fields) return true;
+                if (!index.fields)
+                    return true;
+
                 const uniqueNames = new Set(index.fields.map((ec) => ec.name));
                 return uniqueNames.size === index.fields.length;
             });
@@ -156,7 +163,9 @@ export default class Models {
      * @returns {ig.config.model.DomainModel}
      */
     removeInvalidFields(model) {
-        if (!model) return model;
+        if (!model)
+            return model;
+
         const fieldNames = new Set((model.fields || []).map((f) => f.name));
         return {
             ...model,
diff --git a/modules/web-console/frontend/app/services/exceptionHandler.js b/modules/web-console/frontend/app/services/exceptionHandler.js
index 0d9cf3d9aa702..82a88e15be481 100644
--- a/modules/web-console/frontend/app/services/exceptionHandler.js
+++ b/modules/web-console/frontend/app/services/exceptionHandler.js
@@ -19,7 +19,9 @@ import {CancellationError} from 'app/errors/CancellationError';
 
 export function $exceptionHandler($log) {
     return function(exception, cause) {
-        if (exception instanceof CancellationError) return;
+        if (exception instanceof CancellationError)
+            return;
+
         $log.error(exception, cause);
     };
 }
diff --git a/modules/web-console/frontend/app/types/index.ts b/modules/web-console/frontend/app/types/index.ts
new file mode 100644
index 0000000000000..333c00be875d2
--- /dev/null
+++ b/modules/web-console/frontend/app/types/index.ts
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {Ng1StateDeclaration} from '@uirouter/angularjs';
+
+interface ITfMetatagsConfig {
+    title: string
+}
+
+export interface IIgniteNg1StateDeclaration extends Ng1StateDeclaration {
+    /**
+     * Whether to store state as last visited in local storage or not.
+     * true - will be saved
+     * false (default) - won't be saved
+     * @type {boolean}
+     */
+    unsaved?: boolean,
+    tfMetaTags: ITfMetatagsConfig
+}
diff --git a/modules/web-console/frontend/app/utils/SimpleWorkerPool.js b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js
index 495a4d20d8e5b..8e7aab74e849c 100644
--- a/modules/web-console/frontend/app/utils/SimpleWorkerPool.js
+++ b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
 import {Observable} from 'rxjs/Observable';
 import {Subject} from 'rxjs/Subject';
 import 'rxjs/add/observable/race';
diff --git a/modules/web-console/frontend/app/vendor.js b/modules/web-console/frontend/app/vendor.js
index 58b1edefaadc0..84dea92ee812c 100644
--- a/modules/web-console/frontend/app/vendor.js
+++ b/modules/web-console/frontend/app/vendor.js
@@ -24,7 +24,6 @@ import 'angular-sanitize';
 import 'angular-strap';
 import 'angular-strap/dist/angular-strap.tpl';
 import 'angular-socket-io';
-import 'angular-retina';
 
 import 'angular-messages';
 import '@uirouter/angularjs';
diff --git a/modules/web-console/frontend/ignite_modules/README.txt b/modules/web-console/frontend/ignite_modules/README.txt
deleted file mode 100644
index 365abc77d0a04..0000000000000
--- a/modules/web-console/frontend/ignite_modules/README.txt
+++ /dev/null
@@ -1,6 +0,0 @@
-Ignite Web Console Modules
-======================================
-
-If you are are planning to create or use custom modules you need to copy them in this folder before build.
-
-This is default folder for user modules.
diff --git a/modules/web-console/frontend/index.js b/modules/web-console/frontend/index.js
new file mode 100644
index 0000000000000..c18afc389ce90
--- /dev/null
+++ b/modules/web-console/frontend/index.js
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+
+import igniteConsole from './app/app';
+
+angular.bootstrap(document, [igniteConsole.name], {strictDi: true});
diff --git a/modules/web-console/frontend/package-lock.json b/modules/web-console/frontend/package-lock.json
deleted file mode 100644
index 1fec909e3860e..0000000000000
--- a/modules/web-console/frontend/package-lock.json
+++ /dev/null
@@ -1,14512 +0,0 @@
-{
-  "name": "ignite-web-console",
-  "version": "1.0.0",
-  "lockfileVersion": 1,
-  "requires": true,
-  "dependencies": {
-    "@mrmlnc/readdir-enhanced": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/@mrmlnc/readdir-enhanced/-/readdir-enhanced-2.2.1.tgz",
-      "integrity": "sha512-bPHp6Ji8b41szTOcaP63VlnbbO5Ny6dwAATtY6JTjh5N2OLrb5Qk/Th5cRkRQhkWCt+EJsYrNB0MiL+Gpn6e3g==",
-      "dev": true,
-      "requires": {
-        "call-me-maybe": "1.0.1",
-        "glob-to-regexp": "0.3.0"
-      }
-    },
-    "@posthtml/esm": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/@posthtml/esm/-/esm-1.0.0.tgz",
-      "integrity": "sha512-dEVG+ITnvqKGa4v040tP+n8LOKOqr94qjLva7bE5pnfm2KHJwsKz69J4KMxgWLznbpBJzy8vQfCayEk3vLZnZQ=="
-    },
-    "@sindresorhus/is": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.7.0.tgz",
-      "integrity": "sha512-ONhaKPIufzzrlNbqtWFFd+jlnemX6lJAgq9ZeiZtS7I1PIf/la7CW4m83rTXRnVnsMbW2k56pGYu7AUFJD9Pow=="
-    },
-    "@types/angular": {
-      "version": "1.6.43",
-      "resolved": "https://registry.npmjs.org/@types/angular/-/angular-1.6.43.tgz",
-      "integrity": "sha512-3GrHCRZS62ruJjHMtOx3WYsS0I8i0FRcIqOwqIfWXnlR9g2FebEhUNdMk3LZIvfhZ08xe+S1x2iwP1t9vKCHag==",
-      "dev": true
-    },
-    "@types/angular-animate": {
-      "version": "1.5.9",
-      "resolved": "https://registry.npmjs.org/@types/angular-animate/-/angular-animate-1.5.9.tgz",
-      "integrity": "sha512-kRUrLBKCBNQQUMGf4OIEe3MchzWVVyLFvRDkQ4f3aUc+FAUabRmQeATRY8CZpSD7Lcw3efKIKzWsmm7Aenfy5A==",
-      "dev": true,
-      "requires": {
-        "@types/angular": "1.6.43"
-      }
-    },
-    "@types/angular-mocks": {
-      "version": "1.5.11",
-      "resolved": "https://registry.npmjs.org/@types/angular-mocks/-/angular-mocks-1.5.11.tgz",
-      "integrity": "sha512-C8ipXVKQvw+w64kH97Npa3a7uZB7ZL9Kr4+sOe33oYIyxeg09M8bzAWCIYCmPRRV0px6ozFTZeSVjBXDikz2zw==",
-      "dev": true,
-      "requires": {
-        "@types/angular": "1.6.43"
-      }
-    },
-    "@types/angular-strap": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/@types/angular-strap/-/angular-strap-2.3.1.tgz",
-      "integrity": "sha512-PIIQbwgbxHRHeZ5sfGwjKG9PL2P6zh7KVt+V4lkHlNZSQctxRmu2fd0wfAurnwTnC0l+6SdgL+KEIghe7GOjYw==",
-      "dev": true,
-      "requires": {
-        "@types/angular": "1.6.43"
-      }
-    },
-    "@types/babel-types": {
-      "version": "7.0.1",
-      "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.1.tgz",
-      "integrity": "sha512-EkcOk09rjhivbovP8WreGRbXW20YRfe/qdgXOGq3it3u3aAOWDRNsQhL/XPAWFF7zhZZ+uR+nT+3b+TCkIap1w=="
-    },
-    "@types/babylon": {
-      "version": "6.16.2",
-      "resolved": "https://registry.npmjs.org/@types/babylon/-/babylon-6.16.2.tgz",
-      "integrity": "sha512-+Jty46mPaWe1VAyZbfvgJM4BAdklLWxrT5tc/RjvCgLrtk6gzRY6AOnoWFv4p6hVxhJshDdr2hGVn56alBp97Q==",
-      "requires": {
-        "@types/babel-types": "7.0.1"
-      }
-    },
-    "@types/chai": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.1.2.tgz",
-      "integrity": "sha512-D8uQwKYUw2KESkorZ27ykzXgvkDJYXVEihGklgfp5I4HUP8D6IxtcdLTMB1emjQiWzV7WZ5ihm1cxIzVwjoleQ==",
-      "dev": true
-    },
-    "@types/jquery": {
-      "version": "3.3.1",
-      "resolved": "https://registry.npmjs.org/@types/jquery/-/jquery-3.3.1.tgz",
-      "integrity": "sha512-N3h+rzN518yl2xKrW0o6KKdNmWZ+OwG6SoM5TBEQFF0tTv5wXPEsoOuYQ2Kt3/89XbcSZUJLdjiT/2c3BR/ApQ==",
-      "dev": true
-    },
-    "@types/lodash": {
-      "version": "4.14.107",
-      "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.107.tgz",
-      "integrity": "sha512-afvjfP2rl3yvtv2qrCRN23zIQcDinF+munMJCoHEw2BXF22QJogTlVfNPTACQ6ieDyA6VnyKT4WLuN/wK368ng==",
-      "dev": true
-    },
-    "@types/mocha": {
-      "version": "2.2.48",
-      "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-2.2.48.tgz",
-      "integrity": "sha512-nlK/iyETgafGli8Zh9zJVCTicvU3iajSkRwOh3Hhiva598CMqNJ4NcVCGMTGKpGpTYj/9R8RLzS9NAykSSCqGw==",
-      "dev": true
-    },
-    "@types/node": {
-      "version": "9.6.5",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-9.6.5.tgz",
-      "integrity": "sha512-NOLEgsT6UiDTjnWG5Hd2Mg25LRyz/oe8ql3wbjzgSFeRzRROhPmtlsvIrei4B46UjERF0td9SZ1ZXPLOdcrBHg==",
-      "dev": true
-    },
-    "@types/sinon": {
-      "version": "4.3.1",
-      "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-4.3.1.tgz",
-      "integrity": "sha512-DK4YtH30I67k4klURIBS4VAe1aBISfS9lgNlHFkibSmKem2tLQc5VkKoJreT3dCJAd+xRyCS8bx1o97iq3yUVg==",
-      "dev": true
-    },
-    "@types/tapable": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@types/tapable/-/tapable-1.0.2.tgz",
-      "integrity": "sha512-42zEJkBpNfMEAvWR5WlwtTH22oDzcMjFsL9gDGExwF8X8WvAiw7Vwop7hPw03QT8TKfec83LwbHj6SvpqM4ELQ==",
-      "dev": true
-    },
-    "@types/uglify-js": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/@types/uglify-js/-/uglify-js-3.0.2.tgz",
-      "integrity": "sha512-o8hU2+4xsyGC27Vujoklvxl88Ew5zmJuTBYMX1Uro2rYUt4HEFJKL6fuq8aGykvS+ssIsIzerWWP2DRxonownQ==",
-      "dev": true,
-      "requires": {
-        "source-map": "0.6.1"
-      },
-      "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        }
-      }
-    },
-    "@types/ui-grid": {
-      "version": "0.0.38",
-      "resolved": "https://registry.npmjs.org/@types/ui-grid/-/ui-grid-0.0.38.tgz",
-      "integrity": "sha512-xOe0ySy+PlaBf1lD9VQY9KRT3zpegDb80ivTj9lzwaQA4/mnA4tk9aFJQu4eKdKlivczA91WzFR53SyPCyTQvg==",
-      "dev": true,
-      "requires": {
-        "@types/angular": "1.6.43",
-        "@types/jquery": "3.3.1"
-      }
-    },
-    "@types/webpack": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/@types/webpack/-/webpack-4.1.3.tgz",
-      "integrity": "sha512-NoGVTCumOsyFfuy3934f3ktiJi+wcXHJFxT47tby3iCpuo6M/WjFA9VqT5bYO+FE46i3R0N00RpJX75HxHKDaQ==",
-      "dev": true,
-      "requires": {
-        "@types/node": "9.6.5",
-        "@types/tapable": "1.0.2",
-        "@types/uglify-js": "3.0.2",
-        "source-map": "0.6.1"
-      },
-      "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        }
-      }
-    },
-    "@types/webpack-merge": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/@types/webpack-merge/-/webpack-merge-4.1.3.tgz",
-      "integrity": "sha512-VdmNuYIvIouYlCI73NLKOE1pOVAxv5m5eupvTemojZz9dqghoQXmeEveI6CqeuWpCH6x6FLp6+tXM2sls20/MA==",
-      "dev": true,
-      "requires": {
-        "@types/webpack": "4.1.3"
-      }
-    },
-    "@uirouter/angularjs": {
-      "version": "1.0.15",
-      "resolved": "https://registry.npmjs.org/@uirouter/angularjs/-/angularjs-1.0.15.tgz",
-      "integrity": "sha512-qV+fz+OV5WRNNCXfeVO7nEcSSNESXOxLC0lXM9sv+IwTW6gyiynZ2wHP7fP2ETbr20sPxtbFC+kMVLzyiw/yIg==",
-      "requires": {
-        "@uirouter/core": "5.0.17"
-      }
-    },
-    "@uirouter/core": {
-      "version": "5.0.17",
-      "resolved": "https://registry.npmjs.org/@uirouter/core/-/core-5.0.17.tgz",
-      "integrity": "sha512-aJOSpaRbctGw24Mh74sonLwCyskl7KzFz7M0jRDqrd+eHZK6s/xxi4ZSNuGHRy6kF4x7195buQSJEo7u82t+rA=="
-    },
-    "@uirouter/rx": {
-      "version": "0.4.5",
-      "resolved": "https://registry.npmjs.org/@uirouter/rx/-/rx-0.4.5.tgz",
-      "integrity": "sha512-lfYfNE1n9At5Sl7if73XT6iqHKBz0ZQhVbvQ6WZ8zjnBDGEjvIK03IgIYI/YudJmQd1t5my/WSt5+Bygbhw/Mw==",
-      "requires": {
-        "rollup": "0.41.6",
-        "rollup-plugin-commonjs": "8.4.1",
-        "rollup-plugin-node-resolve": "3.3.0",
-        "rollup-plugin-progress": "0.2.1",
-        "rollup-plugin-sourcemaps": "0.4.2",
-        "rollup-plugin-uglify": "1.0.2",
-        "rollup-plugin-visualizer": "0.2.1"
-      }
-    },
-    "@uirouter/visualizer": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/@uirouter/visualizer/-/visualizer-4.0.2.tgz",
-      "integrity": "sha512-95T0g9HHAjEa+sqwzfSbF6HxBG3shp2oTeGvqYk3VcLEHzrgNopEKJojd+3GNcVznQ+MUAaX4EDHXrzaHKJT6Q==",
-      "requires": {
-        "d3-hierarchy": "1.1.6",
-        "d3-interpolate": "1.1.6",
-        "preact": "7.2.1"
-      }
-    },
-    "JSONStream": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.2.tgz",
-      "integrity": "sha1-wQI3G27Dp887hHygDCC7D85Mbeo=",
-      "dev": true,
-      "requires": {
-        "jsonparse": "1.3.1",
-        "through": "2.3.8"
-      }
-    },
-    "abbrev": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
-      "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q=="
-    },
-    "accepts": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.5.tgz",
-      "integrity": "sha1-63d99gEXI6OxTopywIBcjoZ0a9I=",
-      "requires": {
-        "mime-types": "2.1.18",
-        "negotiator": "0.6.1"
-      }
-    },
-    "acorn": {
-      "version": "5.5.3",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.5.3.tgz",
-      "integrity": "sha512-jd5MkIUlbbmb07nXH0DT3y7rDVtkzDi4XZOUVWAer8ajmF/DTSSbl5oNFyDOl/OXA33Bl79+ypHhl2pN20VeOQ=="
-    },
-    "acorn-dynamic-import": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/acorn-dynamic-import/-/acorn-dynamic-import-3.0.0.tgz",
-      "integrity": "sha512-zVWV8Z8lislJoOKKqdNMOB+s6+XV5WERty8MnKBeFgwA+19XJjJHs2RP5dzM57FftIs+jQnRToLiWazKr6sSWg==",
-      "requires": {
-        "acorn": "5.5.3"
-      }
-    },
-    "acorn-globals": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-3.1.0.tgz",
-      "integrity": "sha1-/YJw9x+7SZawBPqIDuXUZXOnMb8=",
-      "requires": {
-        "acorn": "4.0.13"
-      },
-      "dependencies": {
-        "acorn": {
-          "version": "4.0.13",
-          "resolved": "https://registry.npmjs.org/acorn/-/acorn-4.0.13.tgz",
-          "integrity": "sha1-EFSVrlNh1pe9GVyCUZLhrX8lN4c="
-        }
-      }
-    },
-    "acorn-jsx": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-3.0.1.tgz",
-      "integrity": "sha1-r9+UiPsezvyDSPb7IvRk4ypYs2s=",
-      "requires": {
-        "acorn": "3.3.0"
-      },
-      "dependencies": {
-        "acorn": {
-          "version": "3.3.0",
-          "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz",
-          "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo="
-        }
-      }
-    },
-    "acorn-node": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/acorn-node/-/acorn-node-1.3.0.tgz",
-      "integrity": "sha512-efP54n3d1aLfjL2UMdaXa6DsswwzJeI5rqhbFvXMrKiJ6eJFpf+7R0zN7t8IC+XKn2YOAFAv6xbBNgHUkoHWLw==",
-      "dev": true,
-      "requires": {
-        "acorn": "5.5.3",
-        "xtend": "4.0.1"
-      }
-    },
-    "adjust-sourcemap-loader": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/adjust-sourcemap-loader/-/adjust-sourcemap-loader-1.2.0.tgz",
-      "integrity": "sha512-958oaHHVEXMvsY7v7cC5gEkNIcoaAVIhZ4mBReYVZJOTP9IgKmzLjIOhTtzpLMu+qriXvLsVjJ155EeInp45IQ==",
-      "requires": {
-        "assert": "1.4.1",
-        "camelcase": "1.2.1",
-        "loader-utils": "1.1.0",
-        "lodash.assign": "4.2.0",
-        "lodash.defaults": "3.1.2",
-        "object-path": "0.9.2",
-        "regex-parser": "2.2.9"
-      },
-      "dependencies": {
-        "lodash.defaults": {
-          "version": "3.1.2",
-          "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-3.1.2.tgz",
-          "integrity": "sha1-xzCLGNv4vJNy1wGnNJPGEZK9Liw=",
-          "requires": {
-            "lodash.assign": "3.2.0",
-            "lodash.restparam": "3.6.1"
-          },
-          "dependencies": {
-            "lodash.assign": {
-              "version": "3.2.0",
-              "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-3.2.0.tgz",
-              "integrity": "sha1-POnwI0tLIiPilrj6CsH+6OvKZPo=",
-              "requires": {
-                "lodash._baseassign": "3.2.0",
-                "lodash._createassigner": "3.1.1",
-                "lodash.keys": "3.1.2"
-              }
-            }
-          }
-        }
-      }
-    },
-    "after": {
-      "version": "0.8.2",
-      "resolved": "https://registry.npmjs.org/after/-/after-0.8.2.tgz",
-      "integrity": "sha1-/ts5T58OAqqXaOcCvaI7UF+ufh8="
-    },
-    "ajv": {
-      "version": "5.5.2",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
-      "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
-      "requires": {
-        "co": "4.6.0",
-        "fast-deep-equal": "1.1.0",
-        "fast-json-stable-stringify": "2.0.0",
-        "json-schema-traverse": "0.3.1"
-      }
-    },
-    "ajv-keywords": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.1.0.tgz",
-      "integrity": "sha1-rCsnk5xUPpXSwG5/f1wnvkqlQ74="
-    },
-    "align-text": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/align-text/-/align-text-0.1.4.tgz",
-      "integrity": "sha1-DNkKVhCT810KmSVsIrcGlDP60Rc=",
-      "requires": {
-        "kind-of": "3.2.2",
-        "longest": "1.0.1",
-        "repeat-string": "1.6.1"
-      }
-    },
-    "alphanum-sort": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/alphanum-sort/-/alphanum-sort-1.0.2.tgz",
-      "integrity": "sha1-l6ERlkmyEa0zaR2fn0hqjsn74KM="
-    },
-    "amdefine": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz",
-      "integrity": "sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU="
-    },
-    "angular": {
-      "version": "1.6.6",
-      "resolved": "https://registry.npmjs.org/angular/-/angular-1.6.6.tgz",
-      "integrity": "sha1-/Vo8+0N844LYVO4BEgeXl4Uny2Q="
-    },
-    "angular-acl": {
-      "version": "0.1.10",
-      "resolved": "https://registry.npmjs.org/angular-acl/-/angular-acl-0.1.10.tgz",
-      "integrity": "sha1-4UI97jgmLXowieJm9tk9qxBv5Os="
-    },
-    "angular-animate": {
-      "version": "1.6.6",
-      "resolved": "https://registry.npmjs.org/angular-animate/-/angular-animate-1.6.6.tgz",
-      "integrity": "sha1-aSVkexQaBA0kG/ElBA8aFQ/NinA="
-    },
-    "angular-aria": {
-      "version": "1.6.6",
-      "resolved": "https://registry.npmjs.org/angular-aria/-/angular-aria-1.6.6.tgz",
-      "integrity": "sha1-WN10jglWS8hAn3Ob3lezX77ltqU="
-    },
-    "angular-cookies": {
-      "version": "1.6.6",
-      "resolved": "https://registry.npmjs.org/angular-cookies/-/angular-cookies-1.6.6.tgz",
-      "integrity": "sha1-MRZC2v28T/fNaSILiSW4g1n7oUg="
-    },
-    "angular-drag-and-drop-lists": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/angular-drag-and-drop-lists/-/angular-drag-and-drop-lists-1.4.0.tgz",
-      "integrity": "sha1-KREEyTXhCkL4RZaW7TAGlQnXm/w="
-    },
-    "angular-gridster": {
-      "version": "0.13.14",
-      "resolved": "https://registry.npmjs.org/angular-gridster/-/angular-gridster-0.13.14.tgz",
-      "integrity": "sha1-er6/Y9fJ++xFOLnMpFfg2oBdQgQ="
-    },
-    "angular-messages": {
-      "version": "1.6.9",
-      "resolved": "https://registry.npmjs.org/angular-messages/-/angular-messages-1.6.9.tgz",
-      "integrity": "sha512-/2xvG6vDC+Us8h0baSa1siDKwPj5R2A7LldxxhK2339HInc09bq9shMVCUy9zqnuvwnDUJ/DSgkSaBoSHSZrqg=="
-    },
-    "angular-mocks": {
-      "version": "1.6.10",
-      "resolved": "https://registry.npmjs.org/angular-mocks/-/angular-mocks-1.6.10.tgz",
-      "integrity": "sha512-1865/NmqHNogibNoglY1MGBjx882iu2hI46BBhYDWyz0C4TDM5ER8H8SnYwQKUUG4RXMDsJizszEQ2BEoYKV9w==",
-      "dev": true
-    },
-    "angular-motion": {
-      "version": "0.4.4",
-      "resolved": "https://registry.npmjs.org/angular-motion/-/angular-motion-0.4.4.tgz",
-      "integrity": "sha1-/EozZDOD697cI5ZEaLYMJqMC8jg="
-    },
-    "angular-nvd3": {
-      "version": "1.0.9",
-      "resolved": "https://registry.npmjs.org/angular-nvd3/-/angular-nvd3-1.0.9.tgz",
-      "integrity": "sha1-jYpxSH2eWhIuil0PXNJqsOSRpvU=",
-      "requires": {
-        "angular": "1.6.6",
-        "d3": "3.5.17",
-        "nvd3": "1.8.6"
-      }
-    },
-    "angular-retina": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/angular-retina/-/angular-retina-0.4.0.tgz",
-      "integrity": "sha1-JcxPXAf+rgHku3/8KBZL2JiAcmE="
-    },
-    "angular-sanitize": {
-      "version": "1.6.6",
-      "resolved": "https://registry.npmjs.org/angular-sanitize/-/angular-sanitize-1.6.6.tgz",
-      "integrity": "sha1-D9BloZkxUX++zmZZbTJdcrbgYEE="
-    },
-    "angular-smart-table": {
-      "version": "2.1.8",
-      "resolved": "https://registry.npmjs.org/angular-smart-table/-/angular-smart-table-2.1.8.tgz",
-      "integrity": "sha1-Uh8ypGXnM04HPFwa4JPFnCuPylA="
-    },
-    "angular-socket-io": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/angular-socket-io/-/angular-socket-io-0.7.0.tgz",
-      "integrity": "sha1-eKjWCqVxlKAzC8MTfU2hCGmLl88="
-    },
-    "angular-strap": {
-      "version": "2.3.12",
-      "resolved": "https://registry.npmjs.org/angular-strap/-/angular-strap-2.3.12.tgz",
-      "integrity": "sha1-+uIWVc13B5Zxv5GKt+1iJ3gwYvo="
-    },
-    "angular-translate": {
-      "version": "2.16.0",
-      "resolved": "https://registry.npmjs.org/angular-translate/-/angular-translate-2.16.0.tgz",
-      "integrity": "sha512-kMPOh6lQMF/dji0iq+G/hM8HodxfHV/1VqnvaoklMZXHQOdBP1ucGDuWrWIte/dZ/tbW7MbnE1vOmZY152nwVw==",
-      "requires": {
-        "angular": "1.6.6"
-      }
-    },
-    "angular-tree-control": {
-      "version": "0.2.28",
-      "resolved": "https://registry.npmjs.org/angular-tree-control/-/angular-tree-control-0.2.28.tgz",
-      "integrity": "sha1-bPWNWQ7o4FA7uoma5SmuS4dBCDs="
-    },
-    "angular-ui-carousel": {
-      "version": "0.1.10",
-      "resolved": "https://registry.npmjs.org/angular-ui-carousel/-/angular-ui-carousel-0.1.10.tgz",
-      "integrity": "sha1-ClsQZgGLQOcAuMbuNLDJf8MhlSs="
-    },
-    "angular-ui-grid": {
-      "version": "4.4.6",
-      "resolved": "https://registry.npmjs.org/angular-ui-grid/-/angular-ui-grid-4.4.6.tgz",
-      "integrity": "sha512-0d14HDY4XeqFHI508thxeufiR0AlFoZQ8ihk0x8TRCQc+b9CCk1/F63W2zihirxF0cdOAqBCY2pVSM7vfZvXBQ==",
-      "requires": {
-        "angular": "1.6.6"
-      }
-    },
-    "angular-ui-validate": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/angular-ui-validate/-/angular-ui-validate-1.2.3.tgz",
-      "integrity": "sha1-vrFB9kQJv926ZcEvdYG4k931kyE="
-    },
-    "angular1-async-filter": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/angular1-async-filter/-/angular1-async-filter-1.1.0.tgz",
-      "integrity": "sha1-0CtjYqwbH5ZsXQ7Y8P0ri0DJP3g="
-    },
-    "ansi-escapes": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz",
-      "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw=="
-    },
-    "ansi-html": {
-      "version": "0.0.7",
-      "resolved": "https://registry.npmjs.org/ansi-html/-/ansi-html-0.0.7.tgz",
-      "integrity": "sha1-gTWEAhliqenm/QOflA0S9WynhZ4="
-    },
-    "ansi-regex": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
-      "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
-    },
-    "ansi-styles": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
-      "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4="
-    },
-    "any-observable": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/any-observable/-/any-observable-0.2.0.tgz",
-      "integrity": "sha1-xnhwBYADV5AJCD9UrAq6+1wz0kI="
-    },
-    "anymatch": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz",
-      "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==",
-      "requires": {
-        "micromatch": "3.1.10",
-        "normalize-path": "2.1.1"
-      },
-      "dependencies": {
-        "arr-diff": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
-          "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA="
-        },
-        "array-unique": {
-          "version": "0.3.2",
-          "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
-          "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg="
-        },
-        "braces": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
-          "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
-          "requires": {
-            "arr-flatten": "1.1.0",
-            "array-unique": "0.3.2",
-            "extend-shallow": "2.0.1",
-            "fill-range": "4.0.0",
-            "isobject": "3.0.1",
-            "repeat-element": "1.1.2",
-            "snapdragon": "0.8.2",
-            "snapdragon-node": "2.1.1",
-            "split-string": "3.1.0",
-            "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            }
-          }
-        },
-        "define-property": {
-          "version": "2.0.2",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
-          "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
-          "requires": {
-            "is-descriptor": "1.0.2",
-            "isobject": "3.0.1"
-          }
-        },
-        "expand-brackets": {
-          "version": "2.1.4",
-          "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
-          "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=",
-          "requires": {
-            "debug": "2.6.9",
-            "define-property": "0.2.5",
-            "extend-shallow": "2.0.1",
-            "posix-character-classes": "0.1.1",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "define-property": {
-              "version": "0.2.5",
-              "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-              "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
-              "requires": {
-                "is-descriptor": "0.1.6"
-              }
-            },
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            },
-            "is-accessor-descriptor": {
-              "version": "0.1.6",
-              "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-              "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
-              "requires": {
-                "kind-of": "3.2.2"
-              },
-              "dependencies": {
-                "kind-of": {
-                  "version": "3.2.2",
-                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-                  "requires": {
-                    "is-buffer": "1.1.6"
-                  }
-                }
-              }
-            },
-            "is-data-descriptor": {
-              "version": "0.1.4",
-              "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-              "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
-              "requires": {
-                "kind-of": "3.2.2"
-              },
-              "dependencies": {
-                "kind-of": {
-                  "version": "3.2.2",
-                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-                  "requires": {
-                    "is-buffer": "1.1.6"
-                  }
-                }
-              }
-            },
-            "is-descriptor": {
-              "version": "0.1.6",
-              "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-              "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-              "requires": {
-                "is-accessor-descriptor": "0.1.6",
-                "is-data-descriptor": "0.1.4",
-                "kind-of": "5.1.0"
-              }
-            },
-            "kind-of": {
-              "version": "5.1.0",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-              "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
-            }
-          }
-        },
-        "extend-shallow": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
-          "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
-          "requires": {
-            "assign-symbols": "1.0.0",
-            "is-extendable": "1.0.1"
-          },
-          "dependencies": {
-            "is-extendable": {
-              "version": "1.0.1",
-              "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
-              "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
-              "requires": {
-                "is-plain-object": "2.0.4"
-              }
-            }
-          }
-        },
-        "extglob": {
-          "version": "2.0.4",
-          "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz",
-          "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==",
-          "requires": {
-            "array-unique": "0.3.2",
-            "define-property": "1.0.0",
-            "expand-brackets": "2.1.4",
-            "extend-shallow": "2.0.1",
-            "fragment-cache": "0.2.1",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "define-property": {
-              "version": "1.0.0",
-              "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-              "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
-              "requires": {
-                "is-descriptor": "1.0.2"
-              }
-            },
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            }
-          }
-        },
-        "fill-range": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
-          "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=",
-          "requires": {
-            "extend-shallow": "2.0.1",
-            "is-number": "3.0.0",
-            "repeat-string": "1.6.1",
-            "to-regex-range": "2.1.1"
-          },
-          "dependencies": {
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            }
-          }
-        },
-        "is-accessor-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-data-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-descriptor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
-          "requires": {
-            "is-accessor-descriptor": "1.0.0",
-            "is-data-descriptor": "1.0.0",
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-number": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-          "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        },
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
-        },
-        "micromatch": {
-          "version": "3.1.10",
-          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
-          "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==",
-          "requires": {
-            "arr-diff": "4.0.0",
-            "array-unique": "0.3.2",
-            "braces": "2.3.2",
-            "define-property": "2.0.2",
-            "extend-shallow": "3.0.2",
-            "extglob": "2.0.4",
-            "fragment-cache": "0.2.1",
-            "kind-of": "6.0.2",
-            "nanomatch": "1.2.9",
-            "object.pick": "1.3.0",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          }
-        }
-      }
-    },
-    "app-root-path": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-2.0.1.tgz",
-      "integrity": "sha1-zWLc+OT9WkF+/GZNLlsQZTxlG0Y=",
-      "dev": true
-    },
-    "aproba": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
-      "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
-    },
-    "are-we-there-yet": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.4.tgz",
-      "integrity": "sha1-u13KOCu5TwXhUZQ3PRb9O6HKEQ0=",
-      "requires": {
-        "delegates": "1.0.0",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "argparse": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
-      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
-      "requires": {
-        "sprintf-js": "1.0.3"
-      }
-    },
-    "arr-diff": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-2.0.0.tgz",
-      "integrity": "sha1-jzuCf5Vai9ZpaX5KQlasPOrjVs8=",
-      "requires": {
-        "arr-flatten": "1.1.0"
-      }
-    },
-    "arr-flatten": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz",
-      "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg=="
-    },
-    "arr-union": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz",
-      "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ="
-    },
-    "array-differ": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-1.0.0.tgz",
-      "integrity": "sha1-7/UuN1gknTO+QCuLuOVkuytdQDE="
-    },
-    "array-filter": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/array-filter/-/array-filter-0.0.1.tgz",
-      "integrity": "sha1-fajPLiZijtcygDWB/SH2fKzS7uw=",
-      "dev": true
-    },
-    "array-find-index": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz",
-      "integrity": "sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E="
-    },
-    "array-flatten": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.1.tgz",
-      "integrity": "sha1-Qmu52oQJDBg42BLIFQryCoMx4pY="
-    },
-    "array-includes": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.0.3.tgz",
-      "integrity": "sha1-GEtI9i2S10UrsxsyMWXH+L0CJm0=",
-      "requires": {
-        "define-properties": "1.1.2",
-        "es-abstract": "1.11.0"
-      }
-    },
-    "array-map": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/array-map/-/array-map-0.0.0.tgz",
-      "integrity": "sha1-iKK6tz0c97zVwbEYoAP2b2ZfpmI=",
-      "dev": true
-    },
-    "array-reduce": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/array-reduce/-/array-reduce-0.0.0.tgz",
-      "integrity": "sha1-FziZ0//Rx9k4PkR5Ul2+J4yrXys=",
-      "dev": true
-    },
-    "array-slice": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/array-slice/-/array-slice-0.2.3.tgz",
-      "integrity": "sha1-3Tz7gO15c6dRF82sabC5nshhhvU=",
-      "dev": true
-    },
-    "array-union": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz",
-      "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=",
-      "requires": {
-        "array-uniq": "1.0.3"
-      }
-    },
-    "array-uniq": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz",
-      "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY="
-    },
-    "array-unique": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.2.1.tgz",
-      "integrity": "sha1-odl8yvy8JiXMcPrc6zalDFiwGlM="
-    },
-    "arraybuffer.slice": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.6.tgz",
-      "integrity": "sha1-8zshWfBTKj8xB6JywMz70a0peco="
-    },
-    "arrify": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz",
-      "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0="
-    },
-    "asap": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz",
-      "integrity": "sha1-5QNHYR1+aQlDIIu9r+vLwvuGbUY="
-    },
-    "asn1": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz",
-      "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y="
-    },
-    "asn1.js": {
-      "version": "4.10.1",
-      "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-4.10.1.tgz",
-      "integrity": "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==",
-      "requires": {
-        "bn.js": "4.11.8",
-        "inherits": "2.0.3",
-        "minimalistic-assert": "1.0.1"
-      }
-    },
-    "assert": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/assert/-/assert-1.4.1.tgz",
-      "integrity": "sha1-mZEtWRg2tab1s0XA8H7vwI/GXZE=",
-      "requires": {
-        "util": "0.10.3"
-      }
-    },
-    "assert-plus": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz",
-      "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ="
-    },
-    "assertion-error": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz",
-      "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
-      "dev": true
-    },
-    "assign-symbols": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz",
-      "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c="
-    },
-    "ast-types": {
-      "version": "0.11.3",
-      "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.11.3.tgz",
-      "integrity": "sha512-XA5o5dsNw8MhyW0Q7MWXJWc4oOzZKbdsEJq45h7c8q/d9DwWZ5F2ugUc1PuMLPGsUnphCt/cNDHu8JeBbxf1qA=="
-    },
-    "astw": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/astw/-/astw-2.2.0.tgz",
-      "integrity": "sha1-e9QXhNMkk5h66yOba04cV6hzuRc=",
-      "dev": true,
-      "requires": {
-        "acorn": "4.0.13"
-      },
-      "dependencies": {
-        "acorn": {
-          "version": "4.0.13",
-          "resolved": "https://registry.npmjs.org/acorn/-/acorn-4.0.13.tgz",
-          "integrity": "sha1-EFSVrlNh1pe9GVyCUZLhrX8lN4c=",
-          "dev": true
-        }
-      }
-    },
-    "async": {
-      "version": "2.6.0",
-      "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz",
-      "integrity": "sha512-xAfGg1/NTLBBKlHFmnd7PlmUW9KhVQIUuSrYem9xzFUZy13ScvtyGGejaae9iAVRiRq9+Cx7DPFaAAhCpyxyPw==",
-      "requires": {
-        "lodash": "4.17.5"
-      }
-    },
-    "async-each": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.1.tgz",
-      "integrity": "sha1-GdOGodntxufByF04iu28xW0zYC0="
-    },
-    "async-foreach": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/async-foreach/-/async-foreach-0.1.3.tgz",
-      "integrity": "sha1-NhIfhFwFeBct5Bmpfb6x0W7DRUI="
-    },
-    "async-limiter": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.0.tgz",
-      "integrity": "sha512-jp/uFnooOiO+L211eZOoSyzpOITMXx1rBITauYykG3BRYPu8h0UcxsPNB04RR5vo4Tyz3+ay17tR6JVf9qzYWg==",
-      "dev": true
-    },
-    "asynckit": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
-    },
-    "atob": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.0.tgz",
-      "integrity": "sha512-SuiKH8vbsOyCALjA/+EINmt/Kdl+TQPrtFgW7XZZcwtryFu9e5kQoX3bjCW6mIvGH1fbeAZZuvwGR5IlBRznGw=="
-    },
-    "autoprefixer": {
-      "version": "6.7.7",
-      "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-6.7.7.tgz",
-      "integrity": "sha1-Hb0cg1ZY41zj+ZhAmdsAWFx4IBQ=",
-      "requires": {
-        "browserslist": "1.7.7",
-        "caniuse-db": "1.0.30000830",
-        "normalize-range": "0.1.2",
-        "num2fraction": "1.2.2",
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0"
-      }
-    },
-    "aws-sign2": {
-      "version": "0.6.0",
-      "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz",
-      "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8="
-    },
-    "aws4": {
-      "version": "1.7.0",
-      "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.7.0.tgz",
-      "integrity": "sha512-32NDda82rhwD9/JBCCkB+MRYDp0oSvlo2IL6rQWA10PQi7tDUM3eqMSltXmY+Oyl/7N3P3qNtAlv7X0d9bI28w=="
-    },
-    "babel-code-frame": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz",
-      "integrity": "sha1-Y/1D99weO7fONZR9uP42mj9Yx0s=",
-      "requires": {
-        "chalk": "1.1.3",
-        "esutils": "2.0.2",
-        "js-tokens": "3.0.2"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        }
-      }
-    },
-    "babel-core": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-core/-/babel-core-6.26.0.tgz",
-      "integrity": "sha1-rzL3izGm/O8RnIew/Y2XU/A6C7g=",
-      "requires": {
-        "babel-code-frame": "6.26.0",
-        "babel-generator": "6.26.1",
-        "babel-helpers": "6.24.1",
-        "babel-messages": "6.23.0",
-        "babel-register": "6.26.0",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0",
-        "babylon": "6.18.0",
-        "convert-source-map": "1.5.1",
-        "debug": "2.6.9",
-        "json5": "0.5.1",
-        "lodash": "4.17.5",
-        "minimatch": "3.0.4",
-        "path-is-absolute": "1.0.1",
-        "private": "0.1.8",
-        "slash": "1.0.0",
-        "source-map": "0.5.7"
-      }
-    },
-    "babel-eslint": {
-      "version": "7.2.3",
-      "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-7.2.3.tgz",
-      "integrity": "sha1-sv4tgBJkcPXBlELcdXJTqJdxCCc=",
-      "requires": {
-        "babel-code-frame": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0",
-        "babylon": "6.18.0"
-      }
-    },
-    "babel-generator": {
-      "version": "6.26.1",
-      "resolved": "https://registry.npmjs.org/babel-generator/-/babel-generator-6.26.1.tgz",
-      "integrity": "sha512-HyfwY6ApZj7BYTcJURpM5tznulaBvyio7/0d4zFOeMPUmfxkCjHocCuoLa2SAGzBI8AREcH3eP3758F672DppA==",
-      "requires": {
-        "babel-messages": "6.23.0",
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0",
-        "detect-indent": "4.0.0",
-        "jsesc": "1.3.0",
-        "lodash": "4.17.5",
-        "source-map": "0.5.7",
-        "trim-right": "1.0.1"
-      }
-    },
-    "babel-helper-bindify-decorators": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-bindify-decorators/-/babel-helper-bindify-decorators-6.24.1.tgz",
-      "integrity": "sha1-FMGeXxQte0fxmlJDHlKxzLxAozA=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helper-builder-binary-assignment-operator-visitor": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-builder-binary-assignment-operator-visitor/-/babel-helper-builder-binary-assignment-operator-visitor-6.24.1.tgz",
-      "integrity": "sha1-zORReto1b0IgvK6KAsKzRvmlZmQ=",
-      "requires": {
-        "babel-helper-explode-assignable-expression": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helper-call-delegate": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-call-delegate/-/babel-helper-call-delegate-6.24.1.tgz",
-      "integrity": "sha1-7Oaqzdx25Bw0YfiL/Fdb0Nqi340=",
-      "requires": {
-        "babel-helper-hoist-variables": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helper-define-map": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-helper-define-map/-/babel-helper-define-map-6.26.0.tgz",
-      "integrity": "sha1-pfVtq0GiX5fstJjH66ypgZ+Vvl8=",
-      "requires": {
-        "babel-helper-function-name": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0",
-        "lodash": "4.17.5"
-      }
-    },
-    "babel-helper-explode-assignable-expression": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-explode-assignable-expression/-/babel-helper-explode-assignable-expression-6.24.1.tgz",
-      "integrity": "sha1-8luCz33BBDPFX3BZLVdGQArCLKo=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helper-explode-class": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-explode-class/-/babel-helper-explode-class-6.24.1.tgz",
-      "integrity": "sha1-fcKjkQ3uAHBW4eMdZAztPVTqqes=",
-      "requires": {
-        "babel-helper-bindify-decorators": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helper-function-name": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-function-name/-/babel-helper-function-name-6.24.1.tgz",
-      "integrity": "sha1-00dbjAPtmCQqJbSDUasYOZ01gKk=",
-      "requires": {
-        "babel-helper-get-function-arity": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helper-get-function-arity": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-get-function-arity/-/babel-helper-get-function-arity-6.24.1.tgz",
-      "integrity": "sha1-j3eCqpNAfEHTqlCQj4mwMbG2hT0=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helper-hoist-variables": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-hoist-variables/-/babel-helper-hoist-variables-6.24.1.tgz",
-      "integrity": "sha1-HssnaJydJVE+rbyZFKc/VAi+enY=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helper-optimise-call-expression": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-optimise-call-expression/-/babel-helper-optimise-call-expression-6.24.1.tgz",
-      "integrity": "sha1-96E0J7qfc/j0+pk8VKl4gtEkQlc=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helper-regex": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-helper-regex/-/babel-helper-regex-6.26.0.tgz",
-      "integrity": "sha1-MlxZ+QL4LyS3T6zu0DY5VPZJXnI=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0",
-        "lodash": "4.17.5"
-      }
-    },
-    "babel-helper-remap-async-to-generator": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-remap-async-to-generator/-/babel-helper-remap-async-to-generator-6.24.1.tgz",
-      "integrity": "sha1-XsWBgnrXI/7N04HxySg5BnbkVRs=",
-      "requires": {
-        "babel-helper-function-name": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helper-replace-supers": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helper-replace-supers/-/babel-helper-replace-supers-6.24.1.tgz",
-      "integrity": "sha1-v22/5Dk40XNpohPKiov3S2qQqxo=",
-      "requires": {
-        "babel-helper-optimise-call-expression": "6.24.1",
-        "babel-messages": "6.23.0",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-helpers": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-helpers/-/babel-helpers-6.24.1.tgz",
-      "integrity": "sha1-NHHenK7DiOXIUOWX5Yom3fN2ArI=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0"
-      }
-    },
-    "babel-loader": {
-      "version": "7.1.4",
-      "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-7.1.4.tgz",
-      "integrity": "sha512-/hbyEvPzBJuGpk9o80R0ZyTej6heEOr59GoEUtn8qFKbnx4cJm9FWES6J/iv644sYgrtVw9JJQkjaLW/bqb5gw==",
-      "requires": {
-        "find-cache-dir": "1.0.0",
-        "loader-utils": "1.1.0",
-        "mkdirp": "0.5.1"
-      }
-    },
-    "babel-messages": {
-      "version": "6.23.0",
-      "resolved": "https://registry.npmjs.org/babel-messages/-/babel-messages-6.23.0.tgz",
-      "integrity": "sha1-8830cDhYA1sqKVHG7F7fbGLyYw4=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-add-module-exports": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-add-module-exports/-/babel-plugin-add-module-exports-0.2.1.tgz",
-      "integrity": "sha1-mumh9KjcZ/DN7E9K7aHkOl/2XiU="
-    },
-    "babel-plugin-check-es2015-constants": {
-      "version": "6.22.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-check-es2015-constants/-/babel-plugin-check-es2015-constants-6.22.0.tgz",
-      "integrity": "sha1-NRV7EBQm/S/9PaP3XH0ekYNbv4o=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-syntax-async-functions": {
-      "version": "6.13.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-async-functions/-/babel-plugin-syntax-async-functions-6.13.0.tgz",
-      "integrity": "sha1-ytnK0RkbWtY0vzCuCHI5HgZHvpU="
-    },
-    "babel-plugin-syntax-async-generators": {
-      "version": "6.13.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-async-generators/-/babel-plugin-syntax-async-generators-6.13.0.tgz",
-      "integrity": "sha1-a8lj67FuzLrmuStZbrfzXDQqi5o="
-    },
-    "babel-plugin-syntax-class-constructor-call": {
-      "version": "6.18.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-class-constructor-call/-/babel-plugin-syntax-class-constructor-call-6.18.0.tgz",
-      "integrity": "sha1-nLnTn+Q8hgC+yBRkVt3L1OGnZBY="
-    },
-    "babel-plugin-syntax-class-properties": {
-      "version": "6.13.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-class-properties/-/babel-plugin-syntax-class-properties-6.13.0.tgz",
-      "integrity": "sha1-1+sjt5oxf4VDlixQW4J8fWysJ94="
-    },
-    "babel-plugin-syntax-decorators": {
-      "version": "6.13.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-decorators/-/babel-plugin-syntax-decorators-6.13.0.tgz",
-      "integrity": "sha1-MSVjtNvePMgGzuPkFszurd0RrAs="
-    },
-    "babel-plugin-syntax-dynamic-import": {
-      "version": "6.18.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-dynamic-import/-/babel-plugin-syntax-dynamic-import-6.18.0.tgz",
-      "integrity": "sha1-jWomIpyDdFqZgqRBBRVyyqF5sdo="
-    },
-    "babel-plugin-syntax-exponentiation-operator": {
-      "version": "6.13.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-exponentiation-operator/-/babel-plugin-syntax-exponentiation-operator-6.13.0.tgz",
-      "integrity": "sha1-nufoM3KQ2pUoggGmpX9BcDF4MN4="
-    },
-    "babel-plugin-syntax-export-extensions": {
-      "version": "6.13.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-export-extensions/-/babel-plugin-syntax-export-extensions-6.13.0.tgz",
-      "integrity": "sha1-cKFITw+QiaToStRLrDU8lbmxJyE="
-    },
-    "babel-plugin-syntax-flow": {
-      "version": "6.18.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-flow/-/babel-plugin-syntax-flow-6.18.0.tgz",
-      "integrity": "sha1-TDqyCiryaqIM0lmVw5jE63AxDI0="
-    },
-    "babel-plugin-syntax-object-rest-spread": {
-      "version": "6.13.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-object-rest-spread/-/babel-plugin-syntax-object-rest-spread-6.13.0.tgz",
-      "integrity": "sha1-/WU28rzhODb/o6VFjEkDpZe7O/U="
-    },
-    "babel-plugin-syntax-trailing-function-commas": {
-      "version": "6.22.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-syntax-trailing-function-commas/-/babel-plugin-syntax-trailing-function-commas-6.22.0.tgz",
-      "integrity": "sha1-ugNgk3+NBuQBgKQ/4NVhb/9TLPM="
-    },
-    "babel-plugin-transform-async-generator-functions": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-async-generator-functions/-/babel-plugin-transform-async-generator-functions-6.24.1.tgz",
-      "integrity": "sha1-8FiQAUX9PpkHpt3yjaWfIVJYpds=",
-      "requires": {
-        "babel-helper-remap-async-to-generator": "6.24.1",
-        "babel-plugin-syntax-async-generators": "6.13.0",
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-async-to-generator": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-async-to-generator/-/babel-plugin-transform-async-to-generator-6.24.1.tgz",
-      "integrity": "sha1-ZTbjeK/2yx1VF6wOQOs+n8jQh2E=",
-      "requires": {
-        "babel-helper-remap-async-to-generator": "6.24.1",
-        "babel-plugin-syntax-async-functions": "6.13.0",
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-class-constructor-call": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-class-constructor-call/-/babel-plugin-transform-class-constructor-call-6.24.1.tgz",
-      "integrity": "sha1-gNwoVQWsBn3LjWxl4vbxGrd2Xvk=",
-      "requires": {
-        "babel-plugin-syntax-class-constructor-call": "6.18.0",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-class-properties": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-class-properties/-/babel-plugin-transform-class-properties-6.24.1.tgz",
-      "integrity": "sha1-anl2PqYdM9NvN7YRqp3vgagbRqw=",
-      "requires": {
-        "babel-helper-function-name": "6.24.1",
-        "babel-plugin-syntax-class-properties": "6.13.0",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-decorators": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-decorators/-/babel-plugin-transform-decorators-6.24.1.tgz",
-      "integrity": "sha1-eIAT2PjGtSIr33s0Q5Df13Vp4k0=",
-      "requires": {
-        "babel-helper-explode-class": "6.24.1",
-        "babel-plugin-syntax-decorators": "6.13.0",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-arrow-functions": {
-      "version": "6.22.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-arrow-functions/-/babel-plugin-transform-es2015-arrow-functions-6.22.0.tgz",
-      "integrity": "sha1-RSaSy3EdX3ncf4XkQM5BufJE0iE=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-block-scoped-functions": {
-      "version": "6.22.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-block-scoped-functions/-/babel-plugin-transform-es2015-block-scoped-functions-6.22.0.tgz",
-      "integrity": "sha1-u8UbSflk1wy42OC5ToICRs46YUE=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-block-scoping": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-block-scoping/-/babel-plugin-transform-es2015-block-scoping-6.26.0.tgz",
-      "integrity": "sha1-1w9SmcEwjQXBL0Y4E7CgnnOxiV8=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0",
-        "lodash": "4.17.5"
-      }
-    },
-    "babel-plugin-transform-es2015-classes": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-classes/-/babel-plugin-transform-es2015-classes-6.24.1.tgz",
-      "integrity": "sha1-WkxYpQyclGHlZLSyo7+ryXolhNs=",
-      "requires": {
-        "babel-helper-define-map": "6.26.0",
-        "babel-helper-function-name": "6.24.1",
-        "babel-helper-optimise-call-expression": "6.24.1",
-        "babel-helper-replace-supers": "6.24.1",
-        "babel-messages": "6.23.0",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-computed-properties": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-computed-properties/-/babel-plugin-transform-es2015-computed-properties-6.24.1.tgz",
-      "integrity": "sha1-b+Ko0WiV1WNPTNmZttNICjCBWbM=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-destructuring": {
-      "version": "6.23.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-destructuring/-/babel-plugin-transform-es2015-destructuring-6.23.0.tgz",
-      "integrity": "sha1-mXux8auWf2gtKwh2/jWNYOdlxW0=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-duplicate-keys": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-duplicate-keys/-/babel-plugin-transform-es2015-duplicate-keys-6.24.1.tgz",
-      "integrity": "sha1-c+s9MQypaePvnskcU3QabxV2Qj4=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-for-of": {
-      "version": "6.23.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-for-of/-/babel-plugin-transform-es2015-for-of-6.23.0.tgz",
-      "integrity": "sha1-9HyVsrYT3x0+zC/bdXNiPHUkhpE=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-function-name": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-function-name/-/babel-plugin-transform-es2015-function-name-6.24.1.tgz",
-      "integrity": "sha1-g0yJhTvDaxrw86TF26qU/Y6sqos=",
-      "requires": {
-        "babel-helper-function-name": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-literals": {
-      "version": "6.22.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-literals/-/babel-plugin-transform-es2015-literals-6.22.0.tgz",
-      "integrity": "sha1-T1SgLWzWbPkVKAAZox0xklN3yi4=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-modules-amd": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-modules-amd/-/babel-plugin-transform-es2015-modules-amd-6.24.1.tgz",
-      "integrity": "sha1-Oz5UAXI5hC1tGcMBHEvS8AoA0VQ=",
-      "requires": {
-        "babel-plugin-transform-es2015-modules-commonjs": "6.26.0",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-modules-commonjs": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-modules-commonjs/-/babel-plugin-transform-es2015-modules-commonjs-6.26.0.tgz",
-      "integrity": "sha1-DYOUApt9xqvhqX7xgeAHWN0uXYo=",
-      "requires": {
-        "babel-plugin-transform-strict-mode": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-modules-systemjs": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-modules-systemjs/-/babel-plugin-transform-es2015-modules-systemjs-6.24.1.tgz",
-      "integrity": "sha1-/4mhQrkRmpBhlfXxBuzzBdlAfSM=",
-      "requires": {
-        "babel-helper-hoist-variables": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-modules-umd": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-modules-umd/-/babel-plugin-transform-es2015-modules-umd-6.24.1.tgz",
-      "integrity": "sha1-rJl+YoXNGO1hdq22B9YCNErThGg=",
-      "requires": {
-        "babel-plugin-transform-es2015-modules-amd": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-object-super": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-object-super/-/babel-plugin-transform-es2015-object-super-6.24.1.tgz",
-      "integrity": "sha1-JM72muIcuDp/hgPa0CH1cusnj40=",
-      "requires": {
-        "babel-helper-replace-supers": "6.24.1",
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-parameters": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-parameters/-/babel-plugin-transform-es2015-parameters-6.24.1.tgz",
-      "integrity": "sha1-V6w1GrScrxSpfNE7CfZv3wpiXys=",
-      "requires": {
-        "babel-helper-call-delegate": "6.24.1",
-        "babel-helper-get-function-arity": "6.24.1",
-        "babel-runtime": "6.26.0",
-        "babel-template": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-shorthand-properties": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-shorthand-properties/-/babel-plugin-transform-es2015-shorthand-properties-6.24.1.tgz",
-      "integrity": "sha1-JPh11nIch2YbvZmkYi5R8U3jiqA=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-spread": {
-      "version": "6.22.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-spread/-/babel-plugin-transform-es2015-spread-6.22.0.tgz",
-      "integrity": "sha1-1taKmfia7cRTbIGlQujdnxdG+NE=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-sticky-regex": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-sticky-regex/-/babel-plugin-transform-es2015-sticky-regex-6.24.1.tgz",
-      "integrity": "sha1-AMHNsaynERLN8M9hJsLta0V8zbw=",
-      "requires": {
-        "babel-helper-regex": "6.26.0",
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-template-literals": {
-      "version": "6.22.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-template-literals/-/babel-plugin-transform-es2015-template-literals-6.22.0.tgz",
-      "integrity": "sha1-qEs0UPfp+PH2g51taH2oS7EjbY0=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-typeof-symbol": {
-      "version": "6.23.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-typeof-symbol/-/babel-plugin-transform-es2015-typeof-symbol-6.23.0.tgz",
-      "integrity": "sha1-3sCfHN3/lLUqxz1QXITfWdzOs3I=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-es2015-unicode-regex": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-es2015-unicode-regex/-/babel-plugin-transform-es2015-unicode-regex-6.24.1.tgz",
-      "integrity": "sha1-04sS9C6nMj9yk4fxinxa4frrNek=",
-      "requires": {
-        "babel-helper-regex": "6.26.0",
-        "babel-runtime": "6.26.0",
-        "regexpu-core": "2.0.0"
-      }
-    },
-    "babel-plugin-transform-exponentiation-operator": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-exponentiation-operator/-/babel-plugin-transform-exponentiation-operator-6.24.1.tgz",
-      "integrity": "sha1-KrDJx/MJj6SJB3cruBP+QejeOg4=",
-      "requires": {
-        "babel-helper-builder-binary-assignment-operator-visitor": "6.24.1",
-        "babel-plugin-syntax-exponentiation-operator": "6.13.0",
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-export-extensions": {
-      "version": "6.22.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-export-extensions/-/babel-plugin-transform-export-extensions-6.22.0.tgz",
-      "integrity": "sha1-U3OLR+deghhYnuqUbLvTkQm75lM=",
-      "requires": {
-        "babel-plugin-syntax-export-extensions": "6.13.0",
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-flow-strip-types": {
-      "version": "6.22.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-flow-strip-types/-/babel-plugin-transform-flow-strip-types-6.22.0.tgz",
-      "integrity": "sha1-hMtnKTXUNxT9wyvOhFaNh0Qc988=",
-      "requires": {
-        "babel-plugin-syntax-flow": "6.18.0",
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-object-rest-spread": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-object-rest-spread/-/babel-plugin-transform-object-rest-spread-6.26.0.tgz",
-      "integrity": "sha1-DzZpLVD+9rfi1LOsFHgTepY7ewY=",
-      "requires": {
-        "babel-plugin-syntax-object-rest-spread": "6.13.0",
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-regenerator": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-regenerator/-/babel-plugin-transform-regenerator-6.26.0.tgz",
-      "integrity": "sha1-4HA2lvveJ/Cj78rPi03KL3s6jy8=",
-      "requires": {
-        "regenerator-transform": "0.10.1"
-      }
-    },
-    "babel-plugin-transform-runtime": {
-      "version": "6.23.0",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-runtime/-/babel-plugin-transform-runtime-6.23.0.tgz",
-      "integrity": "sha1-iEkNRGUC6puOfvsP4J7E2ZR5se4=",
-      "requires": {
-        "babel-runtime": "6.26.0"
-      }
-    },
-    "babel-plugin-transform-strict-mode": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-plugin-transform-strict-mode/-/babel-plugin-transform-strict-mode-6.24.1.tgz",
-      "integrity": "sha1-1fr3qleKZbvlkc9e2uBKDGcCB1g=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0"
-      }
-    },
-    "babel-polyfill": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-polyfill/-/babel-polyfill-6.26.0.tgz",
-      "integrity": "sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "core-js": "2.5.5",
-        "regenerator-runtime": "0.10.5"
-      },
-      "dependencies": {
-        "regenerator-runtime": {
-          "version": "0.10.5",
-          "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz",
-          "integrity": "sha1-M2w+/BIgrc7dosn6tntaeVWjNlg="
-        }
-      }
-    },
-    "babel-preset-es2015": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-preset-es2015/-/babel-preset-es2015-6.24.1.tgz",
-      "integrity": "sha1-1EBQ1rwsn+6nAqrzjXJ6AhBTiTk=",
-      "requires": {
-        "babel-plugin-check-es2015-constants": "6.22.0",
-        "babel-plugin-transform-es2015-arrow-functions": "6.22.0",
-        "babel-plugin-transform-es2015-block-scoped-functions": "6.22.0",
-        "babel-plugin-transform-es2015-block-scoping": "6.26.0",
-        "babel-plugin-transform-es2015-classes": "6.24.1",
-        "babel-plugin-transform-es2015-computed-properties": "6.24.1",
-        "babel-plugin-transform-es2015-destructuring": "6.23.0",
-        "babel-plugin-transform-es2015-duplicate-keys": "6.24.1",
-        "babel-plugin-transform-es2015-for-of": "6.23.0",
-        "babel-plugin-transform-es2015-function-name": "6.24.1",
-        "babel-plugin-transform-es2015-literals": "6.22.0",
-        "babel-plugin-transform-es2015-modules-amd": "6.24.1",
-        "babel-plugin-transform-es2015-modules-commonjs": "6.26.0",
-        "babel-plugin-transform-es2015-modules-systemjs": "6.24.1",
-        "babel-plugin-transform-es2015-modules-umd": "6.24.1",
-        "babel-plugin-transform-es2015-object-super": "6.24.1",
-        "babel-plugin-transform-es2015-parameters": "6.24.1",
-        "babel-plugin-transform-es2015-shorthand-properties": "6.24.1",
-        "babel-plugin-transform-es2015-spread": "6.22.0",
-        "babel-plugin-transform-es2015-sticky-regex": "6.24.1",
-        "babel-plugin-transform-es2015-template-literals": "6.22.0",
-        "babel-plugin-transform-es2015-typeof-symbol": "6.23.0",
-        "babel-plugin-transform-es2015-unicode-regex": "6.24.1",
-        "babel-plugin-transform-regenerator": "6.26.0"
-      }
-    },
-    "babel-preset-stage-1": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-preset-stage-1/-/babel-preset-stage-1-6.24.1.tgz",
-      "integrity": "sha1-dpLNfc1oSZB+auSgqFWJz7niv7A=",
-      "requires": {
-        "babel-plugin-transform-class-constructor-call": "6.24.1",
-        "babel-plugin-transform-export-extensions": "6.22.0",
-        "babel-preset-stage-2": "6.24.1"
-      }
-    },
-    "babel-preset-stage-2": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-preset-stage-2/-/babel-preset-stage-2-6.24.1.tgz",
-      "integrity": "sha1-2eKWD7PXEYfw5k7sYrwHdnIZvcE=",
-      "requires": {
-        "babel-plugin-syntax-dynamic-import": "6.18.0",
-        "babel-plugin-transform-class-properties": "6.24.1",
-        "babel-plugin-transform-decorators": "6.24.1",
-        "babel-preset-stage-3": "6.24.1"
-      }
-    },
-    "babel-preset-stage-3": {
-      "version": "6.24.1",
-      "resolved": "https://registry.npmjs.org/babel-preset-stage-3/-/babel-preset-stage-3-6.24.1.tgz",
-      "integrity": "sha1-g2raCp56f6N8sTj7kyb4eTSkg5U=",
-      "requires": {
-        "babel-plugin-syntax-trailing-function-commas": "6.22.0",
-        "babel-plugin-transform-async-generator-functions": "6.24.1",
-        "babel-plugin-transform-async-to-generator": "6.24.1",
-        "babel-plugin-transform-exponentiation-operator": "6.24.1",
-        "babel-plugin-transform-object-rest-spread": "6.26.0"
-      }
-    },
-    "babel-register": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-register/-/babel-register-6.26.0.tgz",
-      "integrity": "sha1-btAhFz4vy0htestFxgCahW9kcHE=",
-      "requires": {
-        "babel-core": "6.26.0",
-        "babel-runtime": "6.26.0",
-        "core-js": "2.5.5",
-        "home-or-tmp": "2.0.0",
-        "lodash": "4.17.5",
-        "mkdirp": "0.5.1",
-        "source-map-support": "0.4.18"
-      }
-    },
-    "babel-runtime": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-6.26.0.tgz",
-      "integrity": "sha1-llxwWGaOgrVde/4E/yM3vItWR/4=",
-      "requires": {
-        "core-js": "2.5.5",
-        "regenerator-runtime": "0.11.1"
-      }
-    },
-    "babel-template": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-template/-/babel-template-6.26.0.tgz",
-      "integrity": "sha1-3gPi0WOWsGn0bdn/+FIfsaDjXgI=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-traverse": "6.26.0",
-        "babel-types": "6.26.0",
-        "babylon": "6.18.0",
-        "lodash": "4.17.5"
-      }
-    },
-    "babel-traverse": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-traverse/-/babel-traverse-6.26.0.tgz",
-      "integrity": "sha1-RqnL1+3MYsjlwGTi0tjQ9ANXZu4=",
-      "requires": {
-        "babel-code-frame": "6.26.0",
-        "babel-messages": "6.23.0",
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0",
-        "babylon": "6.18.0",
-        "debug": "2.6.9",
-        "globals": "9.18.0",
-        "invariant": "2.2.4",
-        "lodash": "4.17.5"
-      }
-    },
-    "babel-types": {
-      "version": "6.26.0",
-      "resolved": "https://registry.npmjs.org/babel-types/-/babel-types-6.26.0.tgz",
-      "integrity": "sha1-o7Bz+Uq0nrb6Vc1lInozQ4BjJJc=",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "esutils": "2.0.2",
-        "lodash": "4.17.5",
-        "to-fast-properties": "1.0.3"
-      }
-    },
-    "babylon": {
-      "version": "6.18.0",
-      "resolved": "https://registry.npmjs.org/babylon/-/babylon-6.18.0.tgz",
-      "integrity": "sha512-q/UEjfGJ2Cm3oKV71DJz9d25TPnq5rhBVL2Q4fA5wcC3jcrdn7+SssEybFIxwAvvP+YCsCYNKughoF33GxgycQ=="
-    },
-    "backo2": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz",
-      "integrity": "sha1-MasayLEpNjRj41s+u2n038+6eUc="
-    },
-    "balanced-match": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
-      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
-    },
-    "base": {
-      "version": "0.11.2",
-      "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz",
-      "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==",
-      "requires": {
-        "cache-base": "1.0.1",
-        "class-utils": "0.3.6",
-        "component-emitter": "1.2.1",
-        "define-property": "1.0.0",
-        "isobject": "3.0.1",
-        "mixin-deep": "1.3.1",
-        "pascalcase": "0.1.1"
-      },
-      "dependencies": {
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        }
-      }
-    },
-    "base64-arraybuffer": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-0.1.5.tgz",
-      "integrity": "sha1-c5JncZI7Whl0etZmqlzUv5xunOg="
-    },
-    "base64-js": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz",
-      "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw=="
-    },
-    "base64id": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/base64id/-/base64id-1.0.0.tgz",
-      "integrity": "sha1-R2iMuZu2gE8OBtPnY7HDLlfY5rY=",
-      "dev": true
-    },
-    "batch": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz",
-      "integrity": "sha1-3DQxT05nkxgJP8dgJyUl+UvyXBY="
-    },
-    "better-assert": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/better-assert/-/better-assert-1.0.2.tgz",
-      "integrity": "sha1-QIZrnhueC1W0gYlDEeaPr/rrxSI=",
-      "requires": {
-        "callsite": "1.0.0"
-      }
-    },
-    "big.js": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/big.js/-/big.js-3.2.0.tgz",
-      "integrity": "sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q=="
-    },
-    "bignumber.js": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-4.1.0.tgz",
-      "integrity": "sha512-eJzYkFYy9L4JzXsbymsFn3p54D+llV27oTQ+ziJG7WFRheJcNZilgVXMG0LoZtlQSKBsJdWtLFqOD0u+U0jZKA=="
-    },
-    "binary-extensions": {
-      "version": "1.11.0",
-      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.11.0.tgz",
-      "integrity": "sha1-RqoXUftqL5PuXmibsQh9SxTGwgU="
-    },
-    "binaryextensions": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/binaryextensions/-/binaryextensions-2.1.1.tgz",
-      "integrity": "sha512-XBaoWE9RW8pPdPQNibZsW2zh8TW6gcarXp1FZPwT8Uop8ScSNldJEWf2k9l3HeTqdrEwsOsFcq74RiJECW34yA=="
-    },
-    "blob": {
-      "version": "0.0.4",
-      "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.4.tgz",
-      "integrity": "sha1-vPEwUspURj8w+fx+lbmkdjCpSSE="
-    },
-    "block-stream": {
-      "version": "0.0.9",
-      "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz",
-      "integrity": "sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo=",
-      "requires": {
-        "inherits": "2.0.3"
-      }
-    },
-    "bluebird": {
-      "version": "3.5.1",
-      "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz",
-      "integrity": "sha512-MKiLiV+I1AA596t9w1sQJ8jkiSr5+ZKi0WKrYGUn6d1Fx+Ij4tIj+m2WMQSGczs5jZVxV339chE8iwk6F64wjA=="
-    },
-    "bn.js": {
-      "version": "4.11.8",
-      "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.8.tgz",
-      "integrity": "sha512-ItfYfPLkWHUjckQCk8xC+LwxgK8NYcXywGigJgSwOP8Y2iyWT4f2vsZnoOXTTbo+o5yXmIUJ4gn5538SO5S3gA=="
-    },
-    "body-parser": {
-      "version": "1.18.2",
-      "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz",
-      "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=",
-      "requires": {
-        "bytes": "3.0.0",
-        "content-type": "1.0.4",
-        "debug": "2.6.9",
-        "depd": "1.1.2",
-        "http-errors": "1.6.3",
-        "iconv-lite": "0.4.19",
-        "on-finished": "2.3.0",
-        "qs": "6.5.1",
-        "raw-body": "2.3.2",
-        "type-is": "1.6.16"
-      },
-      "dependencies": {
-        "iconv-lite": {
-          "version": "0.4.19",
-          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
-          "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
-        },
-        "qs": {
-          "version": "6.5.1",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz",
-          "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A=="
-        }
-      }
-    },
-    "bonjour": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/bonjour/-/bonjour-3.5.0.tgz",
-      "integrity": "sha1-jokKGD2O6aI5OzhExpGkK897yfU=",
-      "requires": {
-        "array-flatten": "2.1.1",
-        "deep-equal": "1.0.1",
-        "dns-equal": "1.0.0",
-        "dns-txt": "2.0.2",
-        "multicast-dns": "6.2.3",
-        "multicast-dns-service-types": "1.1.0"
-      }
-    },
-    "boolbase": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
-      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
-    },
-    "boom": {
-      "version": "2.10.1",
-      "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz",
-      "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=",
-      "requires": {
-        "hoek": "2.16.3"
-      }
-    },
-    "bootstrap-sass": {
-      "version": "3.3.7",
-      "resolved": "https://registry.npmjs.org/bootstrap-sass/-/bootstrap-sass-3.3.7.tgz",
-      "integrity": "sha1-ZZbHq0D2Y3OTMjqwvIDQZPxjBJg="
-    },
-    "brace": {
-      "version": "0.10.0",
-      "resolved": "https://registry.npmjs.org/brace/-/brace-0.10.0.tgz",
-      "integrity": "sha1-7e9OubCSi6HuX3F//BV3SabdXXY=",
-      "requires": {
-        "w3c-blob": "0.0.1"
-      }
-    },
-    "brace-expansion": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
-      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
-      "requires": {
-        "balanced-match": "1.0.0",
-        "concat-map": "0.0.1"
-      }
-    },
-    "braces": {
-      "version": "1.8.5",
-      "resolved": "https://registry.npmjs.org/braces/-/braces-1.8.5.tgz",
-      "integrity": "sha1-uneWLhLf+WnWt2cR6RS3N4V79qc=",
-      "requires": {
-        "expand-range": "1.8.2",
-        "preserve": "0.2.0",
-        "repeat-element": "1.1.2"
-      }
-    },
-    "brorand": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz",
-      "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8="
-    },
-    "browser-pack": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/browser-pack/-/browser-pack-6.1.0.tgz",
-      "integrity": "sha512-erYug8XoqzU3IfcU8fUgyHqyOXqIE4tUTTQ+7mqUjQlvnXkOO6OlT9c/ZoJVHYoAaqGxr09CN53G7XIsO4KtWA==",
-      "dev": true,
-      "requires": {
-        "JSONStream": "1.3.2",
-        "combine-source-map": "0.8.0",
-        "defined": "1.0.0",
-        "safe-buffer": "5.1.1",
-        "through2": "2.0.3",
-        "umd": "3.0.3"
-      }
-    },
-    "browser-resolve": {
-      "version": "1.11.2",
-      "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-1.11.2.tgz",
-      "integrity": "sha1-j/CbCixCFxihBRwmCzLkj0QpOM4=",
-      "dev": true,
-      "requires": {
-        "resolve": "1.1.7"
-      },
-      "dependencies": {
-        "resolve": {
-          "version": "1.1.7",
-          "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz",
-          "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=",
-          "dev": true
-        }
-      }
-    },
-    "browser-stdout": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.0.tgz",
-      "integrity": "sha1-81HTKWnTL6XXpVZxVCY9korjvR8=",
-      "dev": true
-    },
-    "browser-update": {
-      "version": "2.1.9",
-      "resolved": "https://registry.npmjs.org/browser-update/-/browser-update-2.1.9.tgz",
-      "integrity": "sha1-IAcFl7w9Qp9mrHzM0cgd8ZCanvo="
-    },
-    "browserify": {
-      "version": "14.5.0",
-      "resolved": "https://registry.npmjs.org/browserify/-/browserify-14.5.0.tgz",
-      "integrity": "sha512-gKfOsNQv/toWz+60nSPfYzuwSEdzvV2WdxrVPUbPD/qui44rAkB3t3muNtmmGYHqrG56FGwX9SUEQmzNLAeS7g==",
-      "dev": true,
-      "requires": {
-        "JSONStream": "1.3.2",
-        "assert": "1.4.1",
-        "browser-pack": "6.1.0",
-        "browser-resolve": "1.11.2",
-        "browserify-zlib": "0.2.0",
-        "buffer": "5.1.0",
-        "cached-path-relative": "1.0.1",
-        "concat-stream": "1.5.2",
-        "console-browserify": "1.1.0",
-        "constants-browserify": "1.0.0",
-        "crypto-browserify": "3.12.0",
-        "defined": "1.0.0",
-        "deps-sort": "2.0.0",
-        "domain-browser": "1.1.7",
-        "duplexer2": "0.1.4",
-        "events": "1.1.1",
-        "glob": "7.1.2",
-        "has": "1.0.1",
-        "htmlescape": "1.1.1",
-        "https-browserify": "1.0.0",
-        "inherits": "2.0.3",
-        "insert-module-globals": "7.0.6",
-        "labeled-stream-splicer": "2.0.1",
-        "module-deps": "4.1.1",
-        "os-browserify": "0.3.0",
-        "parents": "1.0.1",
-        "path-browserify": "0.0.0",
-        "process": "0.11.10",
-        "punycode": "1.4.1",
-        "querystring-es3": "0.2.1",
-        "read-only-stream": "2.0.0",
-        "readable-stream": "2.3.6",
-        "resolve": "1.7.1",
-        "shasum": "1.0.2",
-        "shell-quote": "1.6.1",
-        "stream-browserify": "2.0.1",
-        "stream-http": "2.8.1",
-        "string_decoder": "1.0.3",
-        "subarg": "1.0.0",
-        "syntax-error": "1.4.0",
-        "through2": "2.0.3",
-        "timers-browserify": "1.4.2",
-        "tty-browserify": "0.0.0",
-        "url": "0.11.0",
-        "util": "0.10.3",
-        "vm-browserify": "0.0.4",
-        "xtend": "4.0.1"
-      },
-      "dependencies": {
-        "buffer": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.1.0.tgz",
-          "integrity": "sha512-YkIRgwsZwJWTnyQrsBTWefizHh+8GYj3kbL1BTiAQ/9pwpino0G7B2gp5tx/FUBqUlvtxV85KNR3mwfAtv15Yw==",
-          "dev": true,
-          "requires": {
-            "base64-js": "1.3.0",
-            "ieee754": "1.1.11"
-          }
-        },
-        "concat-stream": {
-          "version": "1.5.2",
-          "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz",
-          "integrity": "sha1-cIl4Yk2FavQaWnQd790mHadSwmY=",
-          "dev": true,
-          "requires": {
-            "inherits": "2.0.3",
-            "readable-stream": "2.0.6",
-            "typedarray": "0.0.6"
-          },
-          "dependencies": {
-            "readable-stream": {
-              "version": "2.0.6",
-              "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz",
-              "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=",
-              "dev": true,
-              "requires": {
-                "core-util-is": "1.0.2",
-                "inherits": "2.0.3",
-                "isarray": "1.0.0",
-                "process-nextick-args": "1.0.7",
-                "string_decoder": "0.10.31",
-                "util-deprecate": "1.0.2"
-              }
-            },
-            "string_decoder": {
-              "version": "0.10.31",
-              "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
-              "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=",
-              "dev": true
-            }
-          }
-        },
-        "domain-browser": {
-          "version": "1.1.7",
-          "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.1.7.tgz",
-          "integrity": "sha1-hnqksJP6oF8d4IwG9NeyH9+GmLw=",
-          "dev": true
-        },
-        "process-nextick-args": {
-          "version": "1.0.7",
-          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
-          "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=",
-          "dev": true
-        },
-        "punycode": {
-          "version": "1.4.1",
-          "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
-          "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=",
-          "dev": true
-        },
-        "string_decoder": {
-          "version": "1.0.3",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
-          "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==",
-          "dev": true,
-          "requires": {
-            "safe-buffer": "5.1.1"
-          }
-        },
-        "timers-browserify": {
-          "version": "1.4.2",
-          "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-1.4.2.tgz",
-          "integrity": "sha1-ycWLV1voQHN1y14kYtrO50NZ9B0=",
-          "dev": true,
-          "requires": {
-            "process": "0.11.10"
-          }
-        }
-      }
-    },
-    "browserify-aes": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz",
-      "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==",
-      "requires": {
-        "buffer-xor": "1.0.3",
-        "cipher-base": "1.0.4",
-        "create-hash": "1.2.0",
-        "evp_bytestokey": "1.0.3",
-        "inherits": "2.0.3",
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "browserify-cipher": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz",
-      "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==",
-      "requires": {
-        "browserify-aes": "1.2.0",
-        "browserify-des": "1.0.1",
-        "evp_bytestokey": "1.0.3"
-      }
-    },
-    "browserify-des": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.1.tgz",
-      "integrity": "sha512-zy0Cobe3hhgpiOM32Tj7KQ3Vl91m0njwsjzZQK1L+JDf11dzP9qIvjreVinsvXrgfjhStXwUWAEpB9D7Gwmayw==",
-      "requires": {
-        "cipher-base": "1.0.4",
-        "des.js": "1.0.0",
-        "inherits": "2.0.3"
-      }
-    },
-    "browserify-rsa": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
-      "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
-      "requires": {
-        "bn.js": "4.11.8",
-        "randombytes": "2.0.6"
-      }
-    },
-    "browserify-sign": {
-      "version": "4.0.4",
-      "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.0.4.tgz",
-      "integrity": "sha1-qk62jl17ZYuqa/alfmMMvXqT0pg=",
-      "requires": {
-        "bn.js": "4.11.8",
-        "browserify-rsa": "4.0.1",
-        "create-hash": "1.2.0",
-        "create-hmac": "1.1.7",
-        "elliptic": "6.4.0",
-        "inherits": "2.0.3",
-        "parse-asn1": "5.1.1"
-      }
-    },
-    "browserify-zlib": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz",
-      "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==",
-      "requires": {
-        "pako": "1.0.6"
-      }
-    },
-    "browserslist": {
-      "version": "1.7.7",
-      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-1.7.7.tgz",
-      "integrity": "sha1-C9dnBCWL6CmyOYu1Dkti0aFmsLk=",
-      "requires": {
-        "caniuse-db": "1.0.30000830",
-        "electron-to-chromium": "1.3.42"
-      }
-    },
-    "bson-objectid": {
-      "version": "1.1.5",
-      "resolved": "https://registry.npmjs.org/bson-objectid/-/bson-objectid-1.1.5.tgz",
-      "integrity": "sha1-S54hCpjBxOqp7fY6ygeEyzC3m14="
-    },
-    "buffer": {
-      "version": "4.9.1",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
-      "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
-      "requires": {
-        "base64-js": "1.3.0",
-        "ieee754": "1.1.11",
-        "isarray": "1.0.0"
-      }
-    },
-    "buffer-from": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.0.0.tgz",
-      "integrity": "sha512-83apNb8KK0Se60UE1+4Ukbe3HbfELJ6UlI4ldtOGs7So4KD26orJM8hIY9lxdzP+UpItH1Yh/Y8GUvNFWFFRxA=="
-    },
-    "buffer-indexof": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/buffer-indexof/-/buffer-indexof-1.1.1.tgz",
-      "integrity": "sha512-4/rOEg86jivtPTeOUUT61jJO1Ya1TrR/OkqCSZDyq84WJh3LuuiphBYJN+fm5xufIk4XAFcEwte/8WzC8If/1g=="
-    },
-    "buffer-xor": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz",
-      "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk="
-    },
-    "builtin-modules": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-2.0.0.tgz",
-      "integrity": "sha512-3U5kUA5VPsRUA3nofm/BXX7GVHKfxz0hOBAPxXrIvHzlDRkQVqEn6yi8QJegxl4LzOHLdvb7XF5dVawa/VVYBg=="
-    },
-    "builtin-status-codes": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz",
-      "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug="
-    },
-    "bytes": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
-      "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
-    },
-    "cacache": {
-      "version": "10.0.4",
-      "resolved": "https://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz",
-      "integrity": "sha512-Dph0MzuH+rTQzGPNT9fAnrPmMmjKfST6trxJeK7NQuHRaVw24VzPRWTmg9MpcwOVQZO0E1FBICUlFeNaKPIfHA==",
-      "requires": {
-        "bluebird": "3.5.1",
-        "chownr": "1.0.1",
-        "glob": "7.1.2",
-        "graceful-fs": "4.1.11",
-        "lru-cache": "4.1.2",
-        "mississippi": "2.0.0",
-        "mkdirp": "0.5.1",
-        "move-concurrently": "1.0.1",
-        "promise-inflight": "1.0.1",
-        "rimraf": "2.6.2",
-        "ssri": "5.3.0",
-        "unique-filename": "1.1.0",
-        "y18n": "4.0.0"
-      }
-    },
-    "cache-base": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz",
-      "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==",
-      "requires": {
-        "collection-visit": "1.0.0",
-        "component-emitter": "1.2.1",
-        "get-value": "2.0.6",
-        "has-value": "1.0.0",
-        "isobject": "3.0.1",
-        "set-value": "2.0.0",
-        "to-object-path": "0.3.0",
-        "union-value": "1.0.0",
-        "unset-value": "1.0.0"
-      },
-      "dependencies": {
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        }
-      }
-    },
-    "cacheable-request": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-2.1.4.tgz",
-      "integrity": "sha1-DYCIAbY0KtM8kd+dC0TcCbkeXD0=",
-      "requires": {
-        "clone-response": "1.0.2",
-        "get-stream": "3.0.0",
-        "http-cache-semantics": "3.8.1",
-        "keyv": "3.0.0",
-        "lowercase-keys": "1.0.0",
-        "normalize-url": "2.0.1",
-        "responselike": "1.0.2"
-      },
-      "dependencies": {
-        "lowercase-keys": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.0.tgz",
-          "integrity": "sha1-TjNms55/VFfjXxMkvfb4jQv8cwY="
-        },
-        "normalize-url": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-2.0.1.tgz",
-          "integrity": "sha512-D6MUW4K/VzoJ4rJ01JFKxDrtY1v9wrgzCX5f2qj/lzH1m/lW6MhUZFKerVsnyjOhOsYzI9Kqqak+10l4LvLpMw==",
-          "requires": {
-            "prepend-http": "2.0.0",
-            "query-string": "5.1.1",
-            "sort-keys": "2.0.0"
-          }
-        },
-        "prepend-http": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz",
-          "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc="
-        },
-        "query-string": {
-          "version": "5.1.1",
-          "resolved": "https://registry.npmjs.org/query-string/-/query-string-5.1.1.tgz",
-          "integrity": "sha512-gjWOsm2SoGlgLEdAGt7a6slVOk9mGiXmPFMqrEhLQ68rhQuBnpfs3+EmlvqKyxnCo9/PPlF+9MtY02S1aFg+Jw==",
-          "requires": {
-            "decode-uri-component": "0.2.0",
-            "object-assign": "4.1.1",
-            "strict-uri-encode": "1.1.0"
-          }
-        },
-        "sort-keys": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-2.0.0.tgz",
-          "integrity": "sha1-ZYU1WEhh7JfXMNbPQYIuH1ZoQSg=",
-          "requires": {
-            "is-plain-obj": "1.1.0"
-          }
-        }
-      }
-    },
-    "cached-path-relative": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/cached-path-relative/-/cached-path-relative-1.0.1.tgz",
-      "integrity": "sha1-0JxLUoAKpMB44t2BqGmqyQ0uVOc=",
-      "dev": true
-    },
-    "call-me-maybe": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.1.tgz",
-      "integrity": "sha1-JtII6onje1y95gJQoV8DHBak1ms=",
-      "dev": true
-    },
-    "caller-path": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-0.1.0.tgz",
-      "integrity": "sha1-lAhe9jWB7NPaqSREqP6U6CV3dR8=",
-      "requires": {
-        "callsites": "0.2.0"
-      }
-    },
-    "callsite": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz",
-      "integrity": "sha1-KAOY5dZkvXQDi28JBRU+borxvCA="
-    },
-    "callsites": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/callsites/-/callsites-0.2.0.tgz",
-      "integrity": "sha1-r6uWJikQp/M8GaV3WCXGnzTjUMo="
-    },
-    "camel-case": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/camel-case/-/camel-case-3.0.0.tgz",
-      "integrity": "sha1-yjw2iKTpzzpM2nd9xNy8cTJJz3M=",
-      "requires": {
-        "no-case": "2.3.2",
-        "upper-case": "1.1.3"
-      }
-    },
-    "camelcase": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz",
-      "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk="
-    },
-    "camelcase-keys": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz",
-      "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=",
-      "requires": {
-        "camelcase": "2.1.1",
-        "map-obj": "1.0.1"
-      },
-      "dependencies": {
-        "camelcase": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz",
-          "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8="
-        }
-      }
-    },
-    "caniuse-api": {
-      "version": "1.6.1",
-      "resolved": "https://registry.npmjs.org/caniuse-api/-/caniuse-api-1.6.1.tgz",
-      "integrity": "sha1-tTTnxzTE+B7F++isoq0kNUuWLGw=",
-      "requires": {
-        "browserslist": "1.7.7",
-        "caniuse-db": "1.0.30000830",
-        "lodash.memoize": "4.1.2",
-        "lodash.uniq": "4.5.0"
-      }
-    },
-    "caniuse-db": {
-      "version": "1.0.30000830",
-      "resolved": "https://registry.npmjs.org/caniuse-db/-/caniuse-db-1.0.30000830.tgz",
-      "integrity": "sha1-bkUlWzRWSf0V/1kHLaHhK7PeLxM="
-    },
-    "caseless": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz",
-      "integrity": "sha1-cVuW6phBWTzDMGeSP17GDr2k99c="
-    },
-    "center-align": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/center-align/-/center-align-0.1.3.tgz",
-      "integrity": "sha1-qg0yYptu6XIgBBHL1EYckHvCt60=",
-      "requires": {
-        "align-text": "0.1.4",
-        "lazy-cache": "1.0.4"
-      }
-    },
-    "chai": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/chai/-/chai-4.1.0.tgz",
-      "integrity": "sha1-MxoDkbVcOvh0CunDt0WLwcOAXm0=",
-      "dev": true,
-      "requires": {
-        "assertion-error": "1.1.0",
-        "check-error": "1.0.2",
-        "deep-eql": "2.0.2",
-        "get-func-name": "2.0.0",
-        "pathval": "1.1.0",
-        "type-detect": "4.0.3"
-      }
-    },
-    "chalk": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.1.0.tgz",
-      "integrity": "sha512-LUHGS/dge4ujbXMJrnihYMcL4AoOweGnw9Tp3kQuqy1Kx5c1qKjqvMJZ6nVJPMWJtKCTN72ZogH3oeSO9g9rXQ==",
-      "dev": true,
-      "requires": {
-        "ansi-styles": "3.2.1",
-        "escape-string-regexp": "1.0.5",
-        "supports-color": "4.5.0"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "dev": true,
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "has-flag": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz",
-          "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=",
-          "dev": true
-        },
-        "supports-color": {
-          "version": "4.5.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.5.0.tgz",
-          "integrity": "sha1-vnoN5ITexcXN34s9WRJQRJEvY1s=",
-          "dev": true,
-          "requires": {
-            "has-flag": "2.0.0"
-          }
-        }
-      }
-    },
-    "character-parser": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/character-parser/-/character-parser-2.2.0.tgz",
-      "integrity": "sha1-x84o821LzZdE5f/CxfzeHHMmH8A=",
-      "requires": {
-        "is-regex": "1.0.4"
-      }
-    },
-    "chardet": {
-      "version": "0.4.2",
-      "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.4.2.tgz",
-      "integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I="
-    },
-    "check-error": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz",
-      "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=",
-      "dev": true
-    },
-    "chokidar": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.0.3.tgz",
-      "integrity": "sha512-zW8iXYZtXMx4kux/nuZVXjkLP+CyIK5Al5FHnj1OgTKGZfp4Oy6/ymtMSKFv3GD8DviEmUPmJg9eFdJ/JzudMg==",
-      "requires": {
-        "anymatch": "2.0.0",
-        "async-each": "1.0.1",
-        "braces": "2.3.2",
-        "glob-parent": "3.1.0",
-        "inherits": "2.0.3",
-        "is-binary-path": "1.0.1",
-        "is-glob": "4.0.0",
-        "normalize-path": "2.1.1",
-        "path-is-absolute": "1.0.1",
-        "readdirp": "2.1.0",
-        "upath": "1.0.4"
-      },
-      "dependencies": {
-        "array-unique": {
-          "version": "0.3.2",
-          "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
-          "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg="
-        },
-        "braces": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
-          "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
-          "requires": {
-            "arr-flatten": "1.1.0",
-            "array-unique": "0.3.2",
-            "extend-shallow": "2.0.1",
-            "fill-range": "4.0.0",
-            "isobject": "3.0.1",
-            "repeat-element": "1.1.2",
-            "snapdragon": "0.8.2",
-            "snapdragon-node": "2.1.1",
-            "split-string": "3.1.0",
-            "to-regex": "3.0.2"
-          }
-        },
-        "fill-range": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
-          "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=",
-          "requires": {
-            "extend-shallow": "2.0.1",
-            "is-number": "3.0.0",
-            "repeat-string": "1.6.1",
-            "to-regex-range": "2.1.1"
-          }
-        },
-        "glob-parent": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz",
-          "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=",
-          "requires": {
-            "is-glob": "3.1.0",
-            "path-dirname": "1.0.2"
-          },
-          "dependencies": {
-            "is-glob": {
-              "version": "3.1.0",
-              "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz",
-              "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=",
-              "requires": {
-                "is-extglob": "2.1.1"
-              }
-            }
-          }
-        },
-        "is-extglob": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
-          "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI="
-        },
-        "is-glob": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz",
-          "integrity": "sha1-lSHHaEXMJhCoUgPd8ICpWML/q8A=",
-          "requires": {
-            "is-extglob": "2.1.1"
-          }
-        },
-        "is-number": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-          "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
-          "requires": {
-            "kind-of": "3.2.2"
-          }
-        },
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        }
-      }
-    },
-    "chownr": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.0.1.tgz",
-      "integrity": "sha1-4qdQQqlVGQi+vSW4Uj1fl2nXkYE="
-    },
-    "chrome-trace-event": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-0.1.3.tgz",
-      "integrity": "sha512-sjndyZHrrWiu4RY7AkHgjn80GfAM2ZSzUkZLV/Js59Ldmh6JDThf0SUmOHU53rFu2rVxxfCzJ30Ukcfch3Gb/A=="
-    },
-    "cipher-base": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz",
-      "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==",
-      "requires": {
-        "inherits": "2.0.3",
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "circular-json": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz",
-      "integrity": "sha512-UZK3NBx2Mca+b5LsG7bY183pHWt5Y1xts4P3Pz7ENTwGVnJOUWbRb3ocjvX7hx9tq/yTAdclXm9sZ38gNuem4A=="
-    },
-    "clap": {
-      "version": "1.2.3",
-      "resolved": "https://registry.npmjs.org/clap/-/clap-1.2.3.tgz",
-      "integrity": "sha512-4CoL/A3hf90V3VIEjeuhSvlGFEHKzOz+Wfc2IVZc+FaUgU0ZQafJTP49fvnULipOPcAfqhyI2duwQyns6xqjYA==",
-      "requires": {
-        "chalk": "1.1.3"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        }
-      }
-    },
-    "class-utils": {
-      "version": "0.3.6",
-      "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz",
-      "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==",
-      "requires": {
-        "arr-union": "3.1.0",
-        "define-property": "0.2.5",
-        "isobject": "3.0.1",
-        "static-extend": "0.1.2"
-      },
-      "dependencies": {
-        "define-property": {
-          "version": "0.2.5",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
-          "requires": {
-            "is-descriptor": "0.1.6"
-          }
-        },
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        }
-      }
-    },
-    "clean-css": {
-      "version": "4.1.11",
-      "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.1.11.tgz",
-      "integrity": "sha1-Ls3xRaujj1R0DybO/Q/z4D4SXWo=",
-      "requires": {
-        "source-map": "0.5.7"
-      }
-    },
-    "cli-cursor": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz",
-      "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=",
-      "requires": {
-        "restore-cursor": "2.0.0"
-      }
-    },
-    "cli-spinners": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-0.1.2.tgz",
-      "integrity": "sha1-u3ZNiOGF+54eaiofGXcjGPYF4xw="
-    },
-    "cli-table": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/cli-table/-/cli-table-0.3.1.tgz",
-      "integrity": "sha1-9TsFJmqLGguTSz0IIebi3FkUriM=",
-      "requires": {
-        "colors": "1.0.3"
-      },
-      "dependencies": {
-        "colors": {
-          "version": "1.0.3",
-          "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
-          "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs="
-        }
-      }
-    },
-    "cli-truncate": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-0.2.1.tgz",
-      "integrity": "sha1-nxXPuwcFAFNpIWxiasfQWrkN1XQ=",
-      "requires": {
-        "slice-ansi": "0.0.4",
-        "string-width": "1.0.2"
-      },
-      "dependencies": {
-        "is-fullwidth-code-point": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-          "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
-          "requires": {
-            "number-is-nan": "1.0.1"
-          }
-        },
-        "slice-ansi": {
-          "version": "0.0.4",
-          "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-0.0.4.tgz",
-          "integrity": "sha1-7b+JA/ZvfOL46v1s7tZeJkyDGzU="
-        },
-        "string-width": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
-          "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
-          "requires": {
-            "code-point-at": "1.1.0",
-            "is-fullwidth-code-point": "1.0.0",
-            "strip-ansi": "3.0.1"
-          }
-        }
-      }
-    },
-    "cli-width": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz",
-      "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk="
-    },
-    "cliui": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz",
-      "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=",
-      "requires": {
-        "center-align": "0.1.3",
-        "right-align": "0.1.3",
-        "wordwrap": "0.0.2"
-      }
-    },
-    "clone": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
-      "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4="
-    },
-    "clone-buffer": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/clone-buffer/-/clone-buffer-1.0.0.tgz",
-      "integrity": "sha1-4+JbIHrE5wGvch4staFnksrD3Fg="
-    },
-    "clone-deep": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-2.0.2.tgz",
-      "integrity": "sha512-SZegPTKjCgpQH63E+eN6mVEEPdQBOUzjyJm5Pora4lrwWRFS8I0QAxV/KD6vV/i0WuijHZWQC1fMsPEdxfdVCQ==",
-      "requires": {
-        "for-own": "1.0.0",
-        "is-plain-object": "2.0.4",
-        "kind-of": "6.0.2",
-        "shallow-clone": "1.0.0"
-      },
-      "dependencies": {
-        "for-own": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/for-own/-/for-own-1.0.0.tgz",
-          "integrity": "sha1-xjMy9BXO3EsE2/5wz4NklMU8tEs=",
-          "requires": {
-            "for-in": "1.0.2"
-          }
-        },
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
-        }
-      }
-    },
-    "clone-response": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz",
-      "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=",
-      "requires": {
-        "mimic-response": "1.0.0"
-      }
-    },
-    "clone-stats": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-0.0.1.tgz",
-      "integrity": "sha1-uI+UqCzzi4eR1YBG6kAprYjKmdE="
-    },
-    "cloneable-readable": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/cloneable-readable/-/cloneable-readable-1.1.2.tgz",
-      "integrity": "sha512-Bq6+4t+lbM8vhTs/Bef5c5AdEMtapp/iFb6+s4/Hh9MVTt8OLKH7ZOOZSCT+Ys7hsHvqv0GuMPJ1lnQJVHvxpg==",
-      "requires": {
-        "inherits": "2.0.3",
-        "process-nextick-args": "2.0.0",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "co": {
-      "version": "4.6.0",
-      "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
-      "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
-    },
-    "coa": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/coa/-/coa-1.0.4.tgz",
-      "integrity": "sha1-qe8VNmDWqGqL3sAomlxoTSF0Mv0=",
-      "requires": {
-        "q": "1.5.1"
-      }
-    },
-    "coalescy": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/coalescy/-/coalescy-1.0.0.tgz",
-      "integrity": "sha1-SwZYRrg2NhrabEtKSr9LwcrDG/E="
-    },
-    "code-point-at": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
-      "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c="
-    },
-    "collection-visit": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz",
-      "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=",
-      "requires": {
-        "map-visit": "1.0.0",
-        "object-visit": "1.0.1"
-      }
-    },
-    "color": {
-      "version": "0.11.4",
-      "resolved": "https://registry.npmjs.org/color/-/color-0.11.4.tgz",
-      "integrity": "sha1-bXtcdPtl6EHNSHkq0e1eB7kE12Q=",
-      "requires": {
-        "clone": "1.0.4",
-        "color-convert": "1.9.1",
-        "color-string": "0.3.0"
-      }
-    },
-    "color-convert": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz",
-      "integrity": "sha512-mjGanIiwQJskCC18rPR6OmrZ6fm2Lc7PeGFYwCmy5J34wC6F1PzdGL6xeMfmgicfYcNLGuVFA3WzXtIDCQSZxQ==",
-      "requires": {
-        "color-name": "1.1.3"
-      }
-    },
-    "color-name": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-      "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
-    },
-    "color-string": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/color-string/-/color-string-0.3.0.tgz",
-      "integrity": "sha1-J9RvtnAlxcL6JZk7+/V55HhBuZE=",
-      "requires": {
-        "color-name": "1.1.3"
-      }
-    },
-    "colormin": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/colormin/-/colormin-1.1.2.tgz",
-      "integrity": "sha1-6i90IKcrlogaOKrlnsEkpvcpgTM=",
-      "requires": {
-        "color": "0.11.4",
-        "css-color-names": "0.0.4",
-        "has": "1.0.1"
-      }
-    },
-    "colors": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz",
-      "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM="
-    },
-    "combine-lists": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/combine-lists/-/combine-lists-1.0.1.tgz",
-      "integrity": "sha1-RYwH4J4NkA/Ci3Cj/sLazR0st/Y=",
-      "dev": true,
-      "requires": {
-        "lodash": "4.17.5"
-      }
-    },
-    "combine-source-map": {
-      "version": "0.8.0",
-      "resolved": "https://registry.npmjs.org/combine-source-map/-/combine-source-map-0.8.0.tgz",
-      "integrity": "sha1-pY0N8ELBhvz4IqjoAV9UUNLXmos=",
-      "dev": true,
-      "requires": {
-        "convert-source-map": "1.1.3",
-        "inline-source-map": "0.6.2",
-        "lodash.memoize": "3.0.4",
-        "source-map": "0.5.7"
-      },
-      "dependencies": {
-        "convert-source-map": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.1.3.tgz",
-          "integrity": "sha1-SCnId+n+SbMWHzvzZziI4gRpmGA=",
-          "dev": true
-        },
-        "lodash.memoize": {
-          "version": "3.0.4",
-          "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-3.0.4.tgz",
-          "integrity": "sha1-LcvSwofLwKVcxCMovQxzYVDVPj8=",
-          "dev": true
-        }
-      }
-    },
-    "combined-stream": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz",
-      "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=",
-      "requires": {
-        "delayed-stream": "1.0.0"
-      }
-    },
-    "commander": {
-      "version": "2.13.0",
-      "resolved": "https://registry.npmjs.org/commander/-/commander-2.13.0.tgz",
-      "integrity": "sha512-MVuS359B+YzaWqjCL/c+22gfryv+mCBPHAv3zyVI2GN8EY6IRP8VwtasXn8jyyhvvq84R4ImN1OKRtcbIasjYA=="
-    },
-    "commondir": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
-      "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs="
-    },
-    "component-bind": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/component-bind/-/component-bind-1.0.0.tgz",
-      "integrity": "sha1-AMYIq33Nk4l8AAllGx06jh5zu9E="
-    },
-    "component-emitter": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz",
-      "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY="
-    },
-    "component-inherit": {
-      "version": "0.0.3",
-      "resolved": "https://registry.npmjs.org/component-inherit/-/component-inherit-0.0.3.tgz",
-      "integrity": "sha1-ZF/ErfWLcrZJ1crmUTVhnbJv8UM="
-    },
-    "compressible": {
-      "version": "2.0.13",
-      "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.13.tgz",
-      "integrity": "sha1-DRAgq5JLL9tNYnmHXH1tq6a6p6k=",
-      "requires": {
-        "mime-db": "1.33.0"
-      }
-    },
-    "compression": {
-      "version": "1.7.2",
-      "resolved": "http://registry.npmjs.org/compression/-/compression-1.7.2.tgz",
-      "integrity": "sha1-qv+81qr4VLROuygDU9WtFlH1mmk=",
-      "requires": {
-        "accepts": "1.3.5",
-        "bytes": "3.0.0",
-        "compressible": "2.0.13",
-        "debug": "2.6.9",
-        "on-headers": "1.0.1",
-        "safe-buffer": "5.1.1",
-        "vary": "1.1.2"
-      }
-    },
-    "concat-map": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
-    },
-    "concat-stream": {
-      "version": "1.6.2",
-      "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
-      "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
-      "requires": {
-        "buffer-from": "1.0.0",
-        "inherits": "2.0.3",
-        "readable-stream": "2.3.6",
-        "typedarray": "0.0.6"
-      }
-    },
-    "connect": {
-      "version": "3.6.6",
-      "resolved": "https://registry.npmjs.org/connect/-/connect-3.6.6.tgz",
-      "integrity": "sha1-Ce/2xVr3I24TcTWnJXSFi2eG9SQ=",
-      "dev": true,
-      "requires": {
-        "debug": "2.6.9",
-        "finalhandler": "1.1.0",
-        "parseurl": "1.3.2",
-        "utils-merge": "1.0.1"
-      },
-      "dependencies": {
-        "finalhandler": {
-          "version": "1.1.0",
-          "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.0.tgz",
-          "integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=",
-          "dev": true,
-          "requires": {
-            "debug": "2.6.9",
-            "encodeurl": "1.0.2",
-            "escape-html": "1.0.3",
-            "on-finished": "2.3.0",
-            "parseurl": "1.3.2",
-            "statuses": "1.3.1",
-            "unpipe": "1.0.0"
-          }
-        },
-        "statuses": {
-          "version": "1.3.1",
-          "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz",
-          "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=",
-          "dev": true
-        }
-      }
-    },
-    "connect-history-api-fallback": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/connect-history-api-fallback/-/connect-history-api-fallback-1.5.0.tgz",
-      "integrity": "sha1-sGhzk0vF40T+9hGhlqb6rgruAVo="
-    },
-    "console-browserify": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.1.0.tgz",
-      "integrity": "sha1-8CQcRXMKn8YyOyBtvzjtx0HQuxA=",
-      "requires": {
-        "date-now": "0.1.4"
-      }
-    },
-    "console-control-strings": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
-      "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
-    },
-    "constantinople": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/constantinople/-/constantinople-3.1.2.tgz",
-      "integrity": "sha512-yePcBqEFhLOqSBtwYOGGS1exHo/s1xjekXiinh4itpNQGCu4KA1euPh1fg07N2wMITZXQkBz75Ntdt1ctGZouw==",
-      "requires": {
-        "@types/babel-types": "7.0.1",
-        "@types/babylon": "6.16.2",
-        "babel-types": "6.26.0",
-        "babylon": "6.18.0"
-      }
-    },
-    "constants-browserify": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz",
-      "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U="
-    },
-    "content-disposition": {
-      "version": "0.5.2",
-      "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz",
-      "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ="
-    },
-    "content-type": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
-      "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
-    },
-    "convert-source-map": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.5.1.tgz",
-      "integrity": "sha1-uCeAl7m8IpNl3lxiz1/K7YtVmeU="
-    },
-    "cookie": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz",
-      "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s="
-    },
-    "cookie-signature": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
-      "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
-    },
-    "copy-concurrently": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz",
-      "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==",
-      "requires": {
-        "aproba": "1.2.0",
-        "fs-write-stream-atomic": "1.0.10",
-        "iferr": "0.1.5",
-        "mkdirp": "0.5.1",
-        "rimraf": "2.6.2",
-        "run-queue": "1.0.3"
-      }
-    },
-    "copy-descriptor": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz",
-      "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40="
-    },
-    "copy-webpack-plugin": {
-      "version": "4.5.1",
-      "resolved": "https://registry.npmjs.org/copy-webpack-plugin/-/copy-webpack-plugin-4.5.1.tgz",
-      "integrity": "sha512-OlTo6DYg0XfTKOF8eLf79wcHm4Ut10xU2cRBRPMW/NA5F9VMjZGTfRHWDIYC3s+1kObGYrBLshXWU1K0hILkNQ==",
-      "requires": {
-        "cacache": "10.0.4",
-        "find-cache-dir": "1.0.0",
-        "globby": "7.1.1",
-        "is-glob": "4.0.0",
-        "loader-utils": "1.1.0",
-        "minimatch": "3.0.4",
-        "p-limit": "1.2.0",
-        "serialize-javascript": "1.5.0"
-      },
-      "dependencies": {
-        "globby": {
-          "version": "7.1.1",
-          "resolved": "https://registry.npmjs.org/globby/-/globby-7.1.1.tgz",
-          "integrity": "sha1-+yzP+UAfhgCUXfral0QMypcrhoA=",
-          "requires": {
-            "array-union": "1.0.2",
-            "dir-glob": "2.0.0",
-            "glob": "7.1.2",
-            "ignore": "3.3.7",
-            "pify": "3.0.0",
-            "slash": "1.0.0"
-          }
-        },
-        "is-extglob": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
-          "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI="
-        },
-        "is-glob": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz",
-          "integrity": "sha1-lSHHaEXMJhCoUgPd8ICpWML/q8A=",
-          "requires": {
-            "is-extglob": "2.1.1"
-          }
-        }
-      }
-    },
-    "core-js": {
-      "version": "2.5.5",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.5.5.tgz",
-      "integrity": "sha1-sU3ek2xkDAV5prUMq8wTLdYSfjs="
-    },
-    "core-util-is": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
-      "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
-    },
-    "create-ecdh": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.1.tgz",
-      "integrity": "sha512-iZvCCg8XqHQZ1ioNBTzXS/cQSkqkqcPs8xSX4upNB+DAk9Ht3uzQf2J32uAHNCne8LDmKr29AgZrEs4oIrwLuQ==",
-      "requires": {
-        "bn.js": "4.11.8",
-        "elliptic": "6.4.0"
-      }
-    },
-    "create-hash": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
-      "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==",
-      "requires": {
-        "cipher-base": "1.0.4",
-        "inherits": "2.0.3",
-        "md5.js": "1.3.4",
-        "ripemd160": "2.0.1",
-        "sha.js": "2.4.11"
-      }
-    },
-    "create-hmac": {
-      "version": "1.1.7",
-      "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
-      "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==",
-      "requires": {
-        "cipher-base": "1.0.4",
-        "create-hash": "1.2.0",
-        "inherits": "2.0.3",
-        "ripemd160": "2.0.1",
-        "safe-buffer": "5.1.1",
-        "sha.js": "2.4.11"
-      }
-    },
-    "cross-spawn": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz",
-      "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=",
-      "requires": {
-        "lru-cache": "4.1.2",
-        "shebang-command": "1.2.0",
-        "which": "1.3.0"
-      }
-    },
-    "cryptiles": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz",
-      "integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=",
-      "requires": {
-        "boom": "2.10.1"
-      }
-    },
-    "crypto-browserify": {
-      "version": "3.12.0",
-      "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz",
-      "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==",
-      "requires": {
-        "browserify-cipher": "1.0.1",
-        "browserify-sign": "4.0.4",
-        "create-ecdh": "4.0.1",
-        "create-hash": "1.2.0",
-        "create-hmac": "1.1.7",
-        "diffie-hellman": "5.0.3",
-        "inherits": "2.0.3",
-        "pbkdf2": "3.0.14",
-        "public-encrypt": "4.0.2",
-        "randombytes": "2.0.6",
-        "randomfill": "1.0.4"
-      }
-    },
-    "css": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/css/-/css-2.2.1.tgz",
-      "integrity": "sha1-c6TIHehdtmTU7mdPfUcIXjstVdw=",
-      "requires": {
-        "inherits": "2.0.3",
-        "source-map": "0.1.43",
-        "source-map-resolve": "0.3.1",
-        "urix": "0.1.0"
-      },
-      "dependencies": {
-        "atob": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/atob/-/atob-1.1.3.tgz",
-          "integrity": "sha1-lfE2KbEsOlGl0hWr3OKqnzL4B3M="
-        },
-        "source-map": {
-          "version": "0.1.43",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz",
-          "integrity": "sha1-wkvBRspRfBRx9drL4lcbK3+eM0Y=",
-          "requires": {
-            "amdefine": "1.0.1"
-          }
-        },
-        "source-map-resolve": {
-          "version": "0.3.1",
-          "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.3.1.tgz",
-          "integrity": "sha1-YQ9hIqRFuN1RU1oqcbeD38Ekh2E=",
-          "requires": {
-            "atob": "1.1.3",
-            "resolve-url": "0.2.1",
-            "source-map-url": "0.3.0",
-            "urix": "0.1.0"
-          }
-        },
-        "source-map-url": {
-          "version": "0.3.0",
-          "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.3.0.tgz",
-          "integrity": "sha1-fsrxO1e80J2opAxdJp2zN5nUqvk="
-        }
-      }
-    },
-    "css-color-names": {
-      "version": "0.0.4",
-      "resolved": "https://registry.npmjs.org/css-color-names/-/css-color-names-0.0.4.tgz",
-      "integrity": "sha1-gIrcLnnPhHOAabZGyyDsJ762KeA="
-    },
-    "css-loader": {
-      "version": "0.28.7",
-      "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-0.28.7.tgz",
-      "integrity": "sha512-GxMpax8a/VgcfRrVy0gXD6yLd5ePYbXX/5zGgTVYp4wXtJklS8Z2VaUArJgc//f6/Dzil7BaJObdSv8eKKCPgg==",
-      "requires": {
-        "babel-code-frame": "6.26.0",
-        "css-selector-tokenizer": "0.7.0",
-        "cssnano": "3.10.0",
-        "icss-utils": "2.1.0",
-        "loader-utils": "1.1.0",
-        "lodash.camelcase": "4.3.0",
-        "object-assign": "4.1.1",
-        "postcss": "5.2.18",
-        "postcss-modules-extract-imports": "1.1.0",
-        "postcss-modules-local-by-default": "1.2.0",
-        "postcss-modules-scope": "1.1.0",
-        "postcss-modules-values": "1.3.0",
-        "postcss-value-parser": "3.3.0",
-        "source-list-map": "2.0.0"
-      }
-    },
-    "css-select": {
-      "version": "1.3.0-rc0",
-      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.3.0-rc0.tgz",
-      "integrity": "sha1-b5MZaqrnN2ZuoQNqjLFKj8t6kjE=",
-      "requires": {
-        "boolbase": "1.0.0",
-        "css-what": "2.1.0",
-        "domutils": "1.5.1",
-        "nth-check": "1.0.1"
-      },
-      "dependencies": {
-        "domutils": {
-          "version": "1.5.1",
-          "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
-          "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
-          "requires": {
-            "dom-serializer": "0.1.0",
-            "domelementtype": "1.3.0"
-          }
-        }
-      }
-    },
-    "css-select-base-adapter": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/css-select-base-adapter/-/css-select-base-adapter-0.1.0.tgz",
-      "integrity": "sha1-AQKz0UYw34bD65+p9UVicBBs+ZA="
-    },
-    "css-selector-tokenizer": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/css-selector-tokenizer/-/css-selector-tokenizer-0.7.0.tgz",
-      "integrity": "sha1-5piEdK6MlTR3v15+/s/OzNnPTIY=",
-      "requires": {
-        "cssesc": "0.1.0",
-        "fastparse": "1.1.1",
-        "regexpu-core": "1.0.0"
-      },
-      "dependencies": {
-        "regexpu-core": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-1.0.0.tgz",
-          "integrity": "sha1-hqdj9Y7k18L2sQLkdkBQ3n7ZDGs=",
-          "requires": {
-            "regenerate": "1.3.3",
-            "regjsgen": "0.2.0",
-            "regjsparser": "0.1.5"
-          }
-        }
-      }
-    },
-    "css-tree": {
-      "version": "1.0.0-alpha25",
-      "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha25.tgz",
-      "integrity": "sha512-XC6xLW/JqIGirnZuUWHXCHRaAjje2b3OIB0Vj5RIJo6mIi/AdJo30quQl5LxUl0gkXDIrTrFGbMlcZjyFplz1A==",
-      "requires": {
-        "mdn-data": "1.1.1",
-        "source-map": "0.5.7"
-      }
-    },
-    "css-url-regex": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/css-url-regex/-/css-url-regex-1.1.0.tgz",
-      "integrity": "sha1-g4NCMMyfdMRX3lnuvRVD/uuDt+w="
-    },
-    "css-what": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.0.tgz",
-      "integrity": "sha1-lGfQMsOM+u+58teVASUwYvh/ob0="
-    },
-    "cssesc": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-0.1.0.tgz",
-      "integrity": "sha1-yBSQPkViM3GgR3tAEJqq++6t27Q="
-    },
-    "cssnano": {
-      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/cssnano/-/cssnano-3.10.0.tgz",
-      "integrity": "sha1-Tzj2zqK5sX+gFJDyPx3GjqZcHDg=",
-      "requires": {
-        "autoprefixer": "6.7.7",
-        "decamelize": "1.2.0",
-        "defined": "1.0.0",
-        "has": "1.0.1",
-        "object-assign": "4.1.1",
-        "postcss": "5.2.18",
-        "postcss-calc": "5.3.1",
-        "postcss-colormin": "2.2.2",
-        "postcss-convert-values": "2.6.1",
-        "postcss-discard-comments": "2.0.4",
-        "postcss-discard-duplicates": "2.1.0",
-        "postcss-discard-empty": "2.1.0",
-        "postcss-discard-overridden": "0.1.1",
-        "postcss-discard-unused": "2.2.3",
-        "postcss-filter-plugins": "2.0.2",
-        "postcss-merge-idents": "2.1.7",
-        "postcss-merge-longhand": "2.0.2",
-        "postcss-merge-rules": "2.1.2",
-        "postcss-minify-font-values": "1.0.5",
-        "postcss-minify-gradients": "1.0.5",
-        "postcss-minify-params": "1.2.2",
-        "postcss-minify-selectors": "2.1.1",
-        "postcss-normalize-charset": "1.1.1",
-        "postcss-normalize-url": "3.0.8",
-        "postcss-ordered-values": "2.2.3",
-        "postcss-reduce-idents": "2.4.0",
-        "postcss-reduce-initial": "1.0.1",
-        "postcss-reduce-transforms": "1.0.4",
-        "postcss-svgo": "2.1.6",
-        "postcss-unique-selectors": "2.0.2",
-        "postcss-value-parser": "3.3.0",
-        "postcss-zindex": "2.2.0"
-      }
-    },
-    "csso": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/csso/-/csso-2.3.2.tgz",
-      "integrity": "sha1-3dUsWHAz9J6Utx/FVWnyUuj/X4U=",
-      "requires": {
-        "clap": "1.2.3",
-        "source-map": "0.5.7"
-      }
-    },
-    "currently-unhandled": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz",
-      "integrity": "sha1-mI3zP+qxke95mmE2nddsF635V+o=",
-      "requires": {
-        "array-find-index": "1.0.2"
-      }
-    },
-    "custom-event": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/custom-event/-/custom-event-1.0.1.tgz",
-      "integrity": "sha1-XQKkaFCt8bSjF5RqOSj8y1v9BCU=",
-      "dev": true
-    },
-    "cyclist": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-0.2.2.tgz",
-      "integrity": "sha1-GzN5LhHpFKL9bW7WRHRkRE5fpkA="
-    },
-    "d3": {
-      "version": "3.5.17",
-      "resolved": "https://registry.npmjs.org/d3/-/d3-3.5.17.tgz",
-      "integrity": "sha1-vEZ0gAQ3iyGjYMn8fPUjF5B2L7g="
-    },
-    "d3-color": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-1.1.0.tgz",
-      "integrity": "sha512-IZVcqX5yYFvR2NUBbSfIfbgNcSgAtZ7JbgQWqDXf4CywtN7agvI7Kw6+Q1ETvlHOHWJT55Kyuzt0C3I0GVtRHQ=="
-    },
-    "d3-hierarchy": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/d3-hierarchy/-/d3-hierarchy-1.1.6.tgz",
-      "integrity": "sha512-nn4bhBnwWnMSoZgkBXD7vRyZ0xVUsNMQRKytWYHhP1I4qHw+qzApCTgSQTZqMdf4XXZbTMqA59hFusga+THA/g=="
-    },
-    "d3-interpolate": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-1.1.6.tgz",
-      "integrity": "sha512-mOnv5a+pZzkNIHtw/V6I+w9Lqm9L5bG3OTXPM5A+QO0yyVMQ4W1uZhR+VOJmazaOZXri2ppbiZ5BUNWT0pFM9A==",
-      "requires": {
-        "d3-color": "1.1.0"
-      }
-    },
-    "dargs": {
-      "version": "5.1.0",
-      "resolved": "https://registry.npmjs.org/dargs/-/dargs-5.1.0.tgz",
-      "integrity": "sha1-7H6lDHhWTNNsnV7Bj2Yyn63ieCk="
-    },
-    "dashdash": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
-      "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
-      "requires": {
-        "assert-plus": "1.0.0"
-      },
-      "dependencies": {
-        "assert-plus": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
-        }
-      }
-    },
-    "date-fns": {
-      "version": "1.29.0",
-      "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz",
-      "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw=="
-    },
-    "date-format": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/date-format/-/date-format-1.2.0.tgz",
-      "integrity": "sha1-YV6CjiM90aubua4JUODOzPpuytg=",
-      "dev": true
-    },
-    "date-now": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/date-now/-/date-now-0.1.4.tgz",
-      "integrity": "sha1-6vQ5/U1ISK105cx9vvIAZyueNFs="
-    },
-    "dateformat": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz",
-      "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q=="
-    },
-    "debug": {
-      "version": "2.6.9",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
-      "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
-      "requires": {
-        "ms": "2.0.0"
-      }
-    },
-    "decamelize": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
-      "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA="
-    },
-    "decode-uri-component": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz",
-      "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU="
-    },
-    "decompress-response": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz",
-      "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=",
-      "requires": {
-        "mimic-response": "1.0.0"
-      }
-    },
-    "deep-eql": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-2.0.2.tgz",
-      "integrity": "sha1-sbrAblbwp2d3aG1Qyf63XC7XZ5o=",
-      "dev": true,
-      "requires": {
-        "type-detect": "3.0.0"
-      },
-      "dependencies": {
-        "type-detect": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-3.0.0.tgz",
-          "integrity": "sha1-RtDMhVOrt7E6NSsNbeov1Y8tm1U=",
-          "dev": true
-        }
-      }
-    },
-    "deep-equal": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz",
-      "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU="
-    },
-    "deep-extend": {
-      "version": "0.4.2",
-      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.4.2.tgz",
-      "integrity": "sha1-SLaZwn4zS/ifEIkr5DL25MfTSn8="
-    },
-    "deep-is": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
-      "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ="
-    },
-    "deepmerge": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-1.3.2.tgz",
-      "integrity": "sha1-FmNpFinU2/42T6EqKk8KqGqjoFA="
-    },
-    "define-properties": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.2.tgz",
-      "integrity": "sha1-g6c/L+pWmJj7c3GTyPhzyvbUXJQ=",
-      "requires": {
-        "foreach": "2.0.5",
-        "object-keys": "1.0.11"
-      }
-    },
-    "define-property": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-      "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
-      "requires": {
-        "is-descriptor": "1.0.2"
-      },
-      "dependencies": {
-        "is-accessor-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-data-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-descriptor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
-          "requires": {
-            "is-accessor-descriptor": "1.0.0",
-            "is-data-descriptor": "1.0.0",
-            "kind-of": "6.0.2"
-          }
-        },
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
-        }
-      }
-    },
-    "defined": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz",
-      "integrity": "sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM="
-    },
-    "del": {
-      "version": "2.2.2",
-      "resolved": "https://registry.npmjs.org/del/-/del-2.2.2.tgz",
-      "integrity": "sha1-wSyYHQZ4RshLyvhiz/kw2Qf/0ag=",
-      "requires": {
-        "globby": "5.0.0",
-        "is-path-cwd": "1.0.0",
-        "is-path-in-cwd": "1.0.1",
-        "object-assign": "4.1.1",
-        "pify": "2.3.0",
-        "pinkie-promise": "2.0.1",
-        "rimraf": "2.6.2"
-      },
-      "dependencies": {
-        "globby": {
-          "version": "5.0.0",
-          "resolved": "https://registry.npmjs.org/globby/-/globby-5.0.0.tgz",
-          "integrity": "sha1-69hGZ8oNuzMLmbz8aOrCvFQ3Dg0=",
-          "requires": {
-            "array-union": "1.0.2",
-            "arrify": "1.0.1",
-            "glob": "7.1.2",
-            "object-assign": "4.1.1",
-            "pify": "2.3.0",
-            "pinkie-promise": "2.0.1"
-          }
-        },
-        "pify": {
-          "version": "2.3.0",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw="
-        }
-      }
-    },
-    "delayed-stream": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
-      "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
-    },
-    "delegates": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
-      "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
-    },
-    "depd": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
-      "integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak="
-    },
-    "deps-sort": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/deps-sort/-/deps-sort-2.0.0.tgz",
-      "integrity": "sha1-CRckkC6EZYJg65EHSMzNGvbiH7U=",
-      "dev": true,
-      "requires": {
-        "JSONStream": "1.3.2",
-        "shasum": "1.0.2",
-        "subarg": "1.0.0",
-        "through2": "2.0.3"
-      }
-    },
-    "des.js": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.0.tgz",
-      "integrity": "sha1-wHTS4qpqipoH29YfmhXCzYPsjsw=",
-      "requires": {
-        "inherits": "2.0.3",
-        "minimalistic-assert": "1.0.1"
-      }
-    },
-    "destroy": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
-      "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
-    },
-    "detect-conflict": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/detect-conflict/-/detect-conflict-1.0.1.tgz",
-      "integrity": "sha1-CIZXpmqWHAUBnbfEIwiDsca0F24="
-    },
-    "detect-indent": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-4.0.0.tgz",
-      "integrity": "sha1-920GQ1LN9Docts5hnE7jqUdd4gg=",
-      "requires": {
-        "repeating": "2.0.1"
-      }
-    },
-    "detect-node": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.0.3.tgz",
-      "integrity": "sha1-ogM8CcyOFY03dI+951B4Mr1s4Sc="
-    },
-    "detective": {
-      "version": "4.7.1",
-      "resolved": "https://registry.npmjs.org/detective/-/detective-4.7.1.tgz",
-      "integrity": "sha512-H6PmeeUcZloWtdt4DAkFyzFL94arpHr3NOwwmVILFiy+9Qd4JTxxXrzfyGk/lmct2qVGBwTSwSXagqu2BxmWig==",
-      "dev": true,
-      "requires": {
-        "acorn": "5.5.3",
-        "defined": "1.0.0"
-      }
-    },
-    "di": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/di/-/di-0.0.1.tgz",
-      "integrity": "sha1-gGZJMmzqp8qjMG112YXqJ0i6kTw=",
-      "dev": true
-    },
-    "diff": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz",
-      "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA=="
-    },
-    "diffie-hellman": {
-      "version": "5.0.3",
-      "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
-      "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==",
-      "requires": {
-        "bn.js": "4.11.8",
-        "miller-rabin": "4.0.1",
-        "randombytes": "2.0.6"
-      }
-    },
-    "dir-glob": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-2.0.0.tgz",
-      "integrity": "sha512-37qirFDz8cA5fimp9feo43fSuRo2gHwaIn6dXL8Ber1dGwUosDrGZeCCXq57WnIqE4aQ+u3eQZzsk1yOzhdwag==",
-      "requires": {
-        "arrify": "1.0.1",
-        "path-type": "3.0.0"
-      }
-    },
-    "dns-equal": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/dns-equal/-/dns-equal-1.0.0.tgz",
-      "integrity": "sha1-s55/HabrCnW6nBcySzR1PEfgZU0="
-    },
-    "dns-packet": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-1.3.1.tgz",
-      "integrity": "sha512-0UxfQkMhYAUaZI+xrNZOz/as5KgDU0M/fQ9b6SpkyLbk3GEswDi6PADJVaYJradtRVsRIlF1zLyOodbcTCDzUg==",
-      "requires": {
-        "ip": "1.1.5",
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "dns-txt": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/dns-txt/-/dns-txt-2.0.2.tgz",
-      "integrity": "sha1-uR2Ab10nGI5Ks+fRB9iBocxGQrY=",
-      "requires": {
-        "buffer-indexof": "1.1.1"
-      }
-    },
-    "doctrine": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz",
-      "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==",
-      "requires": {
-        "esutils": "2.0.2"
-      }
-    },
-    "doctypes": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/doctypes/-/doctypes-1.1.0.tgz",
-      "integrity": "sha1-6oCxBqh1OHdOijpKWv4pPeSJ4Kk="
-    },
-    "dom-converter": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.1.4.tgz",
-      "integrity": "sha1-pF71cnuJDJv/5tfIduexnLDhfzs=",
-      "requires": {
-        "utila": "0.3.3"
-      },
-      "dependencies": {
-        "utila": {
-          "version": "0.3.3",
-          "resolved": "https://registry.npmjs.org/utila/-/utila-0.3.3.tgz",
-          "integrity": "sha1-1+jn1+MJEHCSsF+NloiCTWM6QiY="
-        }
-      }
-    },
-    "dom-serialize": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/dom-serialize/-/dom-serialize-2.2.1.tgz",
-      "integrity": "sha1-ViromZ9Evl6jB29UGdzVnrQ6yVs=",
-      "dev": true,
-      "requires": {
-        "custom-event": "1.0.1",
-        "ent": "2.2.0",
-        "extend": "3.0.1",
-        "void-elements": "2.0.1"
-      }
-    },
-    "dom-serializer": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz",
-      "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=",
-      "requires": {
-        "domelementtype": "1.1.3",
-        "entities": "1.1.1"
-      },
-      "dependencies": {
-        "domelementtype": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz",
-          "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs="
-        }
-      }
-    },
-    "domain-browser": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz",
-      "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA=="
-    },
-    "domelementtype": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.0.tgz",
-      "integrity": "sha1-sXrtguirWeUt2cGbF1bg/BhyBMI="
-    },
-    "domhandler": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.1.tgz",
-      "integrity": "sha1-iS5HAAqZvlW783dP/qBWHYh5wlk=",
-      "requires": {
-        "domelementtype": "1.3.0"
-      }
-    },
-    "domready": {
-      "version": "1.0.8",
-      "resolved": "https://registry.npmjs.org/domready/-/domready-1.0.8.tgz",
-      "integrity": "sha1-kfJS5Ze2Wvd+dFriTdAYXV4m1Yw="
-    },
-    "domutils": {
-      "version": "1.7.0",
-      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz",
-      "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==",
-      "requires": {
-        "dom-serializer": "0.1.0",
-        "domelementtype": "1.3.0"
-      }
-    },
-    "duplexer2": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz",
-      "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=",
-      "dev": true,
-      "requires": {
-        "readable-stream": "2.3.6"
-      }
-    },
-    "duplexer3": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz",
-      "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI="
-    },
-    "duplexify": {
-      "version": "3.5.4",
-      "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.5.4.tgz",
-      "integrity": "sha512-JzYSLYMhoVVBe8+mbHQ4KgpvHpm0DZpJuL8PY93Vyv1fW7jYJ90LoXa1di/CVbJM+TgMs91rbDapE/RNIfnJsA==",
-      "requires": {
-        "end-of-stream": "1.4.1",
-        "inherits": "2.0.3",
-        "readable-stream": "2.3.6",
-        "stream-shift": "1.0.0"
-      }
-    },
-    "editions": {
-      "version": "1.3.4",
-      "resolved": "https://registry.npmjs.org/editions/-/editions-1.3.4.tgz",
-      "integrity": "sha512-gzao+mxnYDzIysXKMQi/+M1mjy/rjestjg6OPoYTtI+3Izp23oiGZitsl9lPDPiTGXbcSIk1iJWhliSaglxnUg=="
-    },
-    "ee-first": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
-      "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
-    },
-    "ejs": {
-      "version": "2.5.8",
-      "resolved": "https://registry.npmjs.org/ejs/-/ejs-2.5.8.tgz",
-      "integrity": "sha512-QIDZL54fyV8MDcAsO91BMH1ft2qGGaHIJsJIA/+t+7uvXol1dm413fPcUgUb4k8F/9457rx4/KFE4XfDifrQxQ=="
-    },
-    "electron-to-chromium": {
-      "version": "1.3.42",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.3.42.tgz",
-      "integrity": "sha1-lcM78B0MxAVVauyJn+Yf1NduoPk="
-    },
-    "elegant-spinner": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/elegant-spinner/-/elegant-spinner-1.0.1.tgz",
-      "integrity": "sha1-2wQ1IcldfjA/2PNFvtwzSc+wcp4="
-    },
-    "elliptic": {
-      "version": "6.4.0",
-      "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.4.0.tgz",
-      "integrity": "sha1-ysmvh2LIWDYYcAPI3+GT5eLq5d8=",
-      "requires": {
-        "bn.js": "4.11.8",
-        "brorand": "1.1.0",
-        "hash.js": "1.1.3",
-        "hmac-drbg": "1.0.1",
-        "inherits": "2.0.3",
-        "minimalistic-assert": "1.0.1",
-        "minimalistic-crypto-utils": "1.0.1"
-      }
-    },
-    "emojis-list": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-2.1.0.tgz",
-      "integrity": "sha1-TapNnbAPmBmIDHn6RXrlsJof04k="
-    },
-    "encodeurl": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
-      "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
-    },
-    "encoding": {
-      "version": "0.1.12",
-      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
-      "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=",
-      "dev": true,
-      "requires": {
-        "iconv-lite": "0.4.21"
-      }
-    },
-    "end-of-stream": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.1.tgz",
-      "integrity": "sha512-1MkrZNvWTKCaigbn+W15elq2BB/L22nqrSY5DKlo3X6+vclJm8Bb5djXJBmEX6fS3+zCh/F4VBK5Z2KxJt4s2Q==",
-      "requires": {
-        "once": "1.4.0"
-      }
-    },
-    "engine.io": {
-      "version": "3.1.5",
-      "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-3.1.5.tgz",
-      "integrity": "sha512-D06ivJkYxyRrcEe0bTpNnBQNgP9d3xog+qZlLbui8EsMr/DouQpf5o9FzJnWYHEYE0YsFHllUv2R1dkgYZXHcA==",
-      "dev": true,
-      "requires": {
-        "accepts": "1.3.5",
-        "base64id": "1.0.0",
-        "cookie": "0.3.1",
-        "debug": "3.1.0",
-        "engine.io-parser": "2.1.2",
-        "ws": "3.3.3"
-      },
-      "dependencies": {
-        "arraybuffer.slice": {
-          "version": "0.0.7",
-          "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz",
-          "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==",
-          "dev": true
-        },
-        "debug": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-          "dev": true,
-          "requires": {
-            "ms": "2.0.0"
-          }
-        },
-        "engine.io-parser": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.2.tgz",
-          "integrity": "sha512-dInLFzr80RijZ1rGpx1+56/uFoH7/7InhH3kZt+Ms6hT8tNx3NGW/WNSA/f8As1WkOfkuyb3tnRyuXGxusclMw==",
-          "dev": true,
-          "requires": {
-            "after": "0.8.2",
-            "arraybuffer.slice": "0.0.7",
-            "base64-arraybuffer": "0.1.5",
-            "blob": "0.0.4",
-            "has-binary2": "1.0.2"
-          }
-        },
-        "ultron": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
-          "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==",
-          "dev": true
-        },
-        "ws": {
-          "version": "3.3.3",
-          "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
-          "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
-          "dev": true,
-          "requires": {
-            "async-limiter": "1.0.0",
-            "safe-buffer": "5.1.1",
-            "ultron": "1.1.1"
-          }
-        }
-      }
-    },
-    "engine.io-client": {
-      "version": "1.8.3",
-      "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-1.8.3.tgz",
-      "integrity": "sha1-F5jtk0USRkU9TG9jXXogH+lA1as=",
-      "requires": {
-        "component-emitter": "1.2.1",
-        "component-inherit": "0.0.3",
-        "debug": "2.3.3",
-        "engine.io-parser": "1.3.2",
-        "has-cors": "1.1.0",
-        "indexof": "0.0.1",
-        "parsejson": "0.0.3",
-        "parseqs": "0.0.5",
-        "parseuri": "0.0.5",
-        "ws": "1.1.2",
-        "xmlhttprequest-ssl": "1.5.3",
-        "yeast": "0.1.2"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "2.3.3",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.3.3.tgz",
-          "integrity": "sha1-QMRT5n5uE8kB3ewxeviYbNqe/4w=",
-          "requires": {
-            "ms": "0.7.2"
-          }
-        },
-        "ms": {
-          "version": "0.7.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz",
-          "integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U="
-        }
-      }
-    },
-    "engine.io-parser": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-1.3.2.tgz",
-      "integrity": "sha1-k3sHnwAH0Ik+xW1GyyILjLQ1Igo=",
-      "requires": {
-        "after": "0.8.2",
-        "arraybuffer.slice": "0.0.6",
-        "base64-arraybuffer": "0.1.5",
-        "blob": "0.0.4",
-        "has-binary": "0.1.7",
-        "wtf-8": "1.0.0"
-      }
-    },
-    "enhanced-resolve": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.0.0.tgz",
-      "integrity": "sha512-jox/62b2GofV1qTUQTMPEJSDIGycS43evqYzD/KVtEb9OCoki9cnacUPxCrZa7JfPzZSYOCZhu9O9luaMxAX8g==",
-      "requires": {
-        "graceful-fs": "4.1.11",
-        "memory-fs": "0.4.1",
-        "tapable": "1.0.0"
-      }
-    },
-    "ent": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz",
-      "integrity": "sha1-6WQhkyWiHQX0RGai9obtbOX13R0=",
-      "dev": true
-    },
-    "entities": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.1.tgz",
-      "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA="
-    },
-    "envinfo": {
-      "version": "4.4.2",
-      "resolved": "https://registry.npmjs.org/envinfo/-/envinfo-4.4.2.tgz",
-      "integrity": "sha512-5rfRs+m+6pwoKRCFqpsA5+qsLngFms1aWPrxfKbrObCzQaPc3M3yPloZx+BL9UE3dK58cxw36XVQbFRSCCfGSQ=="
-    },
-    "errno": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz",
-      "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==",
-      "requires": {
-        "prr": "1.0.1"
-      }
-    },
-    "error": {
-      "version": "7.0.2",
-      "resolved": "https://registry.npmjs.org/error/-/error-7.0.2.tgz",
-      "integrity": "sha1-pfdf/02ZJhJt2sDqXcOOaJFTywI=",
-      "requires": {
-        "string-template": "0.2.1",
-        "xtend": "4.0.1"
-      }
-    },
-    "error-ex": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.1.tgz",
-      "integrity": "sha1-+FWobOYa3E6GIcPNoh56dhLDqNw=",
-      "requires": {
-        "is-arrayish": "0.2.1"
-      }
-    },
-    "es-abstract": {
-      "version": "1.11.0",
-      "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.11.0.tgz",
-      "integrity": "sha512-ZnQrE/lXTTQ39ulXZ+J1DTFazV9qBy61x2bY071B+qGco8Z8q1QddsLdt/EF8Ai9hcWH72dWS0kFqXLxOxqslA==",
-      "requires": {
-        "es-to-primitive": "1.1.1",
-        "function-bind": "1.1.1",
-        "has": "1.0.1",
-        "is-callable": "1.1.3",
-        "is-regex": "1.0.4"
-      }
-    },
-    "es-to-primitive": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz",
-      "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=",
-      "requires": {
-        "is-callable": "1.1.3",
-        "is-date-object": "1.0.1",
-        "is-symbol": "1.0.1"
-      }
-    },
-    "es6-promise": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.0.2.tgz",
-      "integrity": "sha1-AQ1YWEI6XxGJeWZfRkhqlcbuK7Y="
-    },
-    "escape-html": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
-      "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
-    },
-    "escape-string-regexp": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
-      "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
-    },
-    "eslint": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-4.3.0.tgz",
-      "integrity": "sha1-/NfJY3a780yF7mftABKimWQrEI8=",
-      "requires": {
-        "ajv": "5.5.2",
-        "babel-code-frame": "6.26.0",
-        "chalk": "1.1.3",
-        "concat-stream": "1.6.2",
-        "cross-spawn": "5.1.0",
-        "debug": "2.6.9",
-        "doctrine": "2.1.0",
-        "eslint-scope": "3.7.1",
-        "espree": "3.5.4",
-        "esquery": "1.0.1",
-        "estraverse": "4.2.0",
-        "esutils": "2.0.2",
-        "file-entry-cache": "2.0.0",
-        "functional-red-black-tree": "1.0.1",
-        "glob": "7.1.2",
-        "globals": "9.18.0",
-        "ignore": "3.3.7",
-        "imurmurhash": "0.1.4",
-        "inquirer": "3.3.0",
-        "is-resolvable": "1.1.0",
-        "js-yaml": "3.11.0",
-        "json-stable-stringify": "1.0.1",
-        "levn": "0.3.0",
-        "lodash": "4.17.5",
-        "minimatch": "3.0.4",
-        "mkdirp": "0.5.1",
-        "natural-compare": "1.4.0",
-        "optionator": "0.8.2",
-        "path-is-inside": "1.0.2",
-        "pluralize": "4.0.0",
-        "progress": "2.0.0",
-        "require-uncached": "1.0.3",
-        "semver": "5.5.0",
-        "strip-json-comments": "2.0.1",
-        "table": "4.0.3",
-        "text-table": "0.2.0"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        },
-        "esprima": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz",
-          "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw=="
-        },
-        "js-yaml": {
-          "version": "3.11.0",
-          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.11.0.tgz",
-          "integrity": "sha512-saJstZWv7oNeOyBh3+Dx1qWzhW0+e6/8eDzo7p5rDFqxntSztloLtuKu+Ejhtq82jsilwOIZYsCz+lIjthg1Hw==",
-          "requires": {
-            "argparse": "1.0.10",
-            "esprima": "4.0.0"
-          }
-        }
-      }
-    },
-    "eslint-friendly-formatter": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/eslint-friendly-formatter/-/eslint-friendly-formatter-3.0.0.tgz",
-      "integrity": "sha1-J4h0Q1psRuwdlPoLH/SU4w7wQpA=",
-      "requires": {
-        "chalk": "1.1.3",
-        "coalescy": "1.0.0",
-        "extend": "3.0.1",
-        "minimist": "1.2.0",
-        "text-table": "0.2.0"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        },
-        "minimist": {
-          "version": "1.2.0",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
-          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
-        }
-      }
-    },
-    "eslint-loader": {
-      "version": "1.9.0",
-      "resolved": "https://registry.npmjs.org/eslint-loader/-/eslint-loader-1.9.0.tgz",
-      "integrity": "sha512-40aN976qSNPyb9ejTqjEthZITpls1SVKtwguahmH1dzGCwQU/vySE+xX33VZmD8csU0ahVNCtFlsPgKqRBiqgg==",
-      "requires": {
-        "loader-fs-cache": "1.0.1",
-        "loader-utils": "1.1.0",
-        "object-assign": "4.1.1",
-        "object-hash": "1.3.0",
-        "rimraf": "2.6.2"
-      }
-    },
-    "eslint-plugin-babel": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/eslint-plugin-babel/-/eslint-plugin-babel-4.1.1.tgz",
-      "integrity": "sha1-7yhchwObZ76zu9In9bDu1Ps3a4c="
-    },
-    "eslint-scope": {
-      "version": "3.7.1",
-      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-3.7.1.tgz",
-      "integrity": "sha1-PWPD7f2gLgbgGkUq2IyqzHzctug=",
-      "requires": {
-        "esrecurse": "4.2.1",
-        "estraverse": "4.2.0"
-      }
-    },
-    "espree": {
-      "version": "3.5.4",
-      "resolved": "https://registry.npmjs.org/espree/-/espree-3.5.4.tgz",
-      "integrity": "sha512-yAcIQxtmMiB/jL32dzEp2enBeidsB7xWPLNiw3IIkpVds1P+h7qF9YwJq1yUNzp2OKXgAprs4F61ih66UsoD1A==",
-      "requires": {
-        "acorn": "5.5.3",
-        "acorn-jsx": "3.0.1"
-      }
-    },
-    "esprima": {
-      "version": "2.7.3",
-      "resolved": "https://registry.npmjs.org/esprima/-/esprima-2.7.3.tgz",
-      "integrity": "sha1-luO3DVd59q1JzQMmc9HDEnZ7pYE="
-    },
-    "esquery": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.0.1.tgz",
-      "integrity": "sha512-SmiyZ5zIWH9VM+SRUReLS5Q8a7GxtRdxEBVZpm98rJM7Sb+A9DVCndXfkeFUd3byderg+EbDkfnevfCwynWaNA==",
-      "requires": {
-        "estraverse": "4.2.0"
-      }
-    },
-    "esrecurse": {
-      "version": "4.2.1",
-      "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz",
-      "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==",
-      "requires": {
-        "estraverse": "4.2.0"
-      }
-    },
-    "estraverse": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz",
-      "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM="
-    },
-    "estree-walker": {
-      "version": "0.5.1",
-      "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.5.1.tgz",
-      "integrity": "sha512-7HgCgz1axW7w5aOvgOQkoR1RMBkllygJrssU3BvymKQ95lxXYv6Pon17fBRDm9qhkvXZGijOULoSF9ShOk/ZLg=="
-    },
-    "esutils": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz",
-      "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs="
-    },
-    "etag": {
-      "version": "1.8.1",
-      "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
-      "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
-    },
-    "eventemitter3": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-1.2.0.tgz",
-      "integrity": "sha1-HIaZHYFq0eUEdQ5zh0Ik7PO+xQg="
-    },
-    "events": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
-      "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ="
-    },
-    "eventsource": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-0.1.6.tgz",
-      "integrity": "sha1-Cs7ehJ7X3RzMMsgRuxG5RNTykjI=",
-      "requires": {
-        "original": "1.0.0"
-      }
-    },
-    "evp_bytestokey": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz",
-      "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==",
-      "requires": {
-        "md5.js": "1.3.4",
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "execa": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/execa/-/execa-0.7.0.tgz",
-      "integrity": "sha1-lEvs00zEHuMqY6n68nrVpl/Fl3c=",
-      "requires": {
-        "cross-spawn": "5.1.0",
-        "get-stream": "3.0.0",
-        "is-stream": "1.1.0",
-        "npm-run-path": "2.0.2",
-        "p-finally": "1.0.0",
-        "signal-exit": "3.0.2",
-        "strip-eof": "1.0.0"
-      }
-    },
-    "exit-hook": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-1.1.1.tgz",
-      "integrity": "sha1-8FyiM7SMBdVP/wd2XfhQfpXAL/g="
-    },
-    "expand-braces": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/expand-braces/-/expand-braces-0.1.2.tgz",
-      "integrity": "sha1-SIsdHSRRyz06axks/AMPRMWFX+o=",
-      "dev": true,
-      "requires": {
-        "array-slice": "0.2.3",
-        "array-unique": "0.2.1",
-        "braces": "0.1.5"
-      },
-      "dependencies": {
-        "braces": {
-          "version": "0.1.5",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-0.1.5.tgz",
-          "integrity": "sha1-wIVxEIUpHYt1/ddOqw+FlygHEeY=",
-          "dev": true,
-          "requires": {
-            "expand-range": "0.1.1"
-          }
-        },
-        "expand-range": {
-          "version": "0.1.1",
-          "resolved": "https://registry.npmjs.org/expand-range/-/expand-range-0.1.1.tgz",
-          "integrity": "sha1-TLjtoJk8pW+k9B/ELzy7TMrf8EQ=",
-          "dev": true,
-          "requires": {
-            "is-number": "0.1.1",
-            "repeat-string": "0.2.2"
-          }
-        },
-        "is-number": {
-          "version": "0.1.1",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-0.1.1.tgz",
-          "integrity": "sha1-aaevEWlj1HIG7JvZtIoUIW8eOAY=",
-          "dev": true
-        },
-        "repeat-string": {
-          "version": "0.2.2",
-          "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-0.2.2.tgz",
-          "integrity": "sha1-x6jTI2BoNiBZp+RlH8aITosftK4=",
-          "dev": true
-        }
-      }
-    },
-    "expand-brackets": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-0.1.5.tgz",
-      "integrity": "sha1-3wcoTjQqgHzXM6xa9yQR5YHRF3s=",
-      "requires": {
-        "is-posix-bracket": "0.1.1"
-      }
-    },
-    "expand-range": {
-      "version": "1.8.2",
-      "resolved": "https://registry.npmjs.org/expand-range/-/expand-range-1.8.2.tgz",
-      "integrity": "sha1-opnv/TNf4nIeuujiV+x5ZE/IUzc=",
-      "requires": {
-        "fill-range": "2.2.3"
-      }
-    },
-    "expand-tilde": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz",
-      "integrity": "sha1-l+gBqgUt8CRU3kawK/YhZCzchQI=",
-      "requires": {
-        "homedir-polyfill": "1.0.1"
-      }
-    },
-    "expose-loader": {
-      "version": "0.7.5",
-      "resolved": "https://registry.npmjs.org/expose-loader/-/expose-loader-0.7.5.tgz",
-      "integrity": "sha512-iPowgKUZkTPX5PznYsmifVj9Bob0w2wTHVkt/eYNPSzyebkUgIedmskf/kcfEIWpiWjg3JRjnW+a17XypySMuw=="
-    },
-    "express": {
-      "version": "4.16.3",
-      "resolved": "https://registry.npmjs.org/express/-/express-4.16.3.tgz",
-      "integrity": "sha1-avilAjUNsyRuzEvs9rWjTSL37VM=",
-      "requires": {
-        "accepts": "1.3.5",
-        "array-flatten": "1.1.1",
-        "body-parser": "1.18.2",
-        "content-disposition": "0.5.2",
-        "content-type": "1.0.4",
-        "cookie": "0.3.1",
-        "cookie-signature": "1.0.6",
-        "debug": "2.6.9",
-        "depd": "1.1.2",
-        "encodeurl": "1.0.2",
-        "escape-html": "1.0.3",
-        "etag": "1.8.1",
-        "finalhandler": "1.1.1",
-        "fresh": "0.5.2",
-        "merge-descriptors": "1.0.1",
-        "methods": "1.1.2",
-        "on-finished": "2.3.0",
-        "parseurl": "1.3.2",
-        "path-to-regexp": "0.1.7",
-        "proxy-addr": "2.0.3",
-        "qs": "6.5.1",
-        "range-parser": "1.2.0",
-        "safe-buffer": "5.1.1",
-        "send": "0.16.2",
-        "serve-static": "1.13.2",
-        "setprototypeof": "1.1.0",
-        "statuses": "1.4.0",
-        "type-is": "1.6.16",
-        "utils-merge": "1.0.1",
-        "vary": "1.1.2"
-      },
-      "dependencies": {
-        "array-flatten": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
-          "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
-        },
-        "qs": {
-          "version": "6.5.1",
-          "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz",
-          "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A=="
-        }
-      }
-    },
-    "extend": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz",
-      "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ="
-    },
-    "extend-shallow": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-      "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-      "requires": {
-        "is-extendable": "0.1.1"
-      }
-    },
-    "external-editor": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-2.2.0.tgz",
-      "integrity": "sha512-bSn6gvGxKt+b7+6TKEv1ZycHleA7aHhRHyAqJyp5pbUFuYYNIzpZnQDk7AsYckyWdEnTeAnay0aCy2aV6iTk9A==",
-      "requires": {
-        "chardet": "0.4.2",
-        "iconv-lite": "0.4.21",
-        "tmp": "0.0.33"
-      }
-    },
-    "extglob": {
-      "version": "0.3.2",
-      "resolved": "https://registry.npmjs.org/extglob/-/extglob-0.3.2.tgz",
-      "integrity": "sha1-Lhj/PS9JqydlzskCPwEdqo2DSaE=",
-      "requires": {
-        "is-extglob": "1.0.0"
-      }
-    },
-    "extract-text-webpack-plugin": {
-      "version": "4.0.0-beta.0",
-      "resolved": "https://registry.npmjs.org/extract-text-webpack-plugin/-/extract-text-webpack-plugin-4.0.0-beta.0.tgz",
-      "integrity": "sha512-Hypkn9jUTnFr0DpekNam53X47tXn3ucY08BQumv7kdGgeVUBLq3DJHJTi6HNxv4jl9W+Skxjz9+RnK0sJyqqjA==",
-      "requires": {
-        "async": "2.6.0",
-        "loader-utils": "1.1.0",
-        "schema-utils": "0.4.5",
-        "webpack-sources": "1.1.0"
-      }
-    },
-    "extsprintf": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
-      "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
-    },
-    "fast-deep-equal": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz",
-      "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ="
-    },
-    "fast-glob": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-2.2.0.tgz",
-      "integrity": "sha512-4F75PTznkNtSKs2pbhtBwRkw8sRwa7LfXx5XaQJOe4IQ6yTjceLDTwM5gj1s80R2t/5WeDC1gVfm3jLE+l39Tw==",
-      "dev": true,
-      "requires": {
-        "@mrmlnc/readdir-enhanced": "2.2.1",
-        "glob-parent": "3.1.0",
-        "is-glob": "4.0.0",
-        "merge2": "1.2.1",
-        "micromatch": "3.1.10"
-      },
-      "dependencies": {
-        "arr-diff": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
-          "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=",
-          "dev": true
-        },
-        "array-unique": {
-          "version": "0.3.2",
-          "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
-          "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=",
-          "dev": true
-        },
-        "braces": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
-          "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
-          "dev": true,
-          "requires": {
-            "arr-flatten": "1.1.0",
-            "array-unique": "0.3.2",
-            "extend-shallow": "2.0.1",
-            "fill-range": "4.0.0",
-            "isobject": "3.0.1",
-            "repeat-element": "1.1.2",
-            "snapdragon": "0.8.2",
-            "snapdragon-node": "2.1.1",
-            "split-string": "3.1.0",
-            "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "dev": true,
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            }
-          }
-        },
-        "define-property": {
-          "version": "2.0.2",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
-          "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
-          "dev": true,
-          "requires": {
-            "is-descriptor": "1.0.2",
-            "isobject": "3.0.1"
-          }
-        },
-        "expand-brackets": {
-          "version": "2.1.4",
-          "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
-          "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=",
-          "dev": true,
-          "requires": {
-            "debug": "2.6.9",
-            "define-property": "0.2.5",
-            "extend-shallow": "2.0.1",
-            "posix-character-classes": "0.1.1",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "define-property": {
-              "version": "0.2.5",
-              "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-              "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
-              "dev": true,
-              "requires": {
-                "is-descriptor": "0.1.6"
-              }
-            },
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "dev": true,
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            },
-            "is-accessor-descriptor": {
-              "version": "0.1.6",
-              "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-              "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
-              "dev": true,
-              "requires": {
-                "kind-of": "3.2.2"
-              },
-              "dependencies": {
-                "kind-of": {
-                  "version": "3.2.2",
-                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-                  "dev": true,
-                  "requires": {
-                    "is-buffer": "1.1.6"
-                  }
-                }
-              }
-            },
-            "is-data-descriptor": {
-              "version": "0.1.4",
-              "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-              "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
-              "dev": true,
-              "requires": {
-                "kind-of": "3.2.2"
-              },
-              "dependencies": {
-                "kind-of": {
-                  "version": "3.2.2",
-                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-                  "dev": true,
-                  "requires": {
-                    "is-buffer": "1.1.6"
-                  }
-                }
-              }
-            },
-            "is-descriptor": {
-              "version": "0.1.6",
-              "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-              "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-              "dev": true,
-              "requires": {
-                "is-accessor-descriptor": "0.1.6",
-                "is-data-descriptor": "0.1.4",
-                "kind-of": "5.1.0"
-              }
-            },
-            "kind-of": {
-              "version": "5.1.0",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-              "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==",
-              "dev": true
-            }
-          }
-        },
-        "extend-shallow": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
-          "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
-          "dev": true,
-          "requires": {
-            "assign-symbols": "1.0.0",
-            "is-extendable": "1.0.1"
-          },
-          "dependencies": {
-            "is-extendable": {
-              "version": "1.0.1",
-              "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
-              "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
-              "dev": true,
-              "requires": {
-                "is-plain-object": "2.0.4"
-              }
-            }
-          }
-        },
-        "extglob": {
-          "version": "2.0.4",
-          "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz",
-          "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==",
-          "dev": true,
-          "requires": {
-            "array-unique": "0.3.2",
-            "define-property": "1.0.0",
-            "expand-brackets": "2.1.4",
-            "extend-shallow": "2.0.1",
-            "fragment-cache": "0.2.1",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "define-property": {
-              "version": "1.0.0",
-              "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-              "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
-              "dev": true,
-              "requires": {
-                "is-descriptor": "1.0.2"
-              }
-            },
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "dev": true,
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            }
-          }
-        },
-        "fill-range": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
-          "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=",
-          "dev": true,
-          "requires": {
-            "extend-shallow": "2.0.1",
-            "is-number": "3.0.0",
-            "repeat-string": "1.6.1",
-            "to-regex-range": "2.1.1"
-          },
-          "dependencies": {
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "dev": true,
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            }
-          }
-        },
-        "glob-parent": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz",
-          "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=",
-          "dev": true,
-          "requires": {
-            "is-glob": "3.1.0",
-            "path-dirname": "1.0.2"
-          },
-          "dependencies": {
-            "is-glob": {
-              "version": "3.1.0",
-              "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz",
-              "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=",
-              "dev": true,
-              "requires": {
-                "is-extglob": "2.1.1"
-              }
-            }
-          }
-        },
-        "is-accessor-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
-          "dev": true,
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-data-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
-          "dev": true,
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-descriptor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
-          "dev": true,
-          "requires": {
-            "is-accessor-descriptor": "1.0.0",
-            "is-data-descriptor": "1.0.0",
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-extglob": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
-          "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=",
-          "dev": true
-        },
-        "is-glob": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.0.tgz",
-          "integrity": "sha1-lSHHaEXMJhCoUgPd8ICpWML/q8A=",
-          "dev": true,
-          "requires": {
-            "is-extglob": "2.1.1"
-          }
-        },
-        "is-number": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-          "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
-          "dev": true,
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "dev": true,
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=",
-          "dev": true
-        },
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==",
-          "dev": true
-        },
-        "micromatch": {
-          "version": "3.1.10",
-          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
-          "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==",
-          "dev": true,
-          "requires": {
-            "arr-diff": "4.0.0",
-            "array-unique": "0.3.2",
-            "braces": "2.3.2",
-            "define-property": "2.0.2",
-            "extend-shallow": "3.0.2",
-            "extglob": "2.0.4",
-            "fragment-cache": "0.2.1",
-            "kind-of": "6.0.2",
-            "nanomatch": "1.2.9",
-            "object.pick": "1.3.0",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          }
-        }
-      }
-    },
-    "fast-json-stable-stringify": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
-      "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
-    },
-    "fast-levenshtein": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
-      "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc="
-    },
-    "fastparse": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/fastparse/-/fastparse-1.1.1.tgz",
-      "integrity": "sha1-0eJkOzipTXWDtHkGDmxK/8lAcfg="
-    },
-    "faye-websocket": {
-      "version": "0.10.0",
-      "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.10.0.tgz",
-      "integrity": "sha1-TkkvjQTftviQA1B/btvy1QHnxvQ=",
-      "requires": {
-        "websocket-driver": "0.7.0"
-      }
-    },
-    "figures": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
-      "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=",
-      "requires": {
-        "escape-string-regexp": "1.0.5"
-      }
-    },
-    "file-entry-cache": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-2.0.0.tgz",
-      "integrity": "sha1-w5KZDD5oR4PYOLjISkXYoEhFg2E=",
-      "requires": {
-        "flat-cache": "1.3.0",
-        "object-assign": "4.1.1"
-      }
-    },
-    "file-loader": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/file-loader/-/file-loader-1.1.11.tgz",
-      "integrity": "sha512-TGR4HU7HUsGg6GCOPJnFk06RhWgEWFLAGWiT6rcD+GRC2keU3s9RGJ+b3Z6/U73jwwNb2gKLJ7YCrp+jvU4ALg==",
-      "requires": {
-        "loader-utils": "1.1.0",
-        "schema-utils": "0.4.5"
-      }
-    },
-    "file-saver": {
-      "version": "1.3.3",
-      "resolved": "https://registry.npmjs.org/file-saver/-/file-saver-1.3.3.tgz",
-      "integrity": "sha1-zdTETTqiZOrC9o7BZbx5HDSvEjI="
-    },
-    "filename-regex": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.1.tgz",
-      "integrity": "sha1-wcS5vuPglyXdsQa3XB4wH+LxiyY="
-    },
-    "fill-range": {
-      "version": "2.2.3",
-      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-2.2.3.tgz",
-      "integrity": "sha1-ULd9/X5Gm8dJJHCWNpn+eoSFpyM=",
-      "requires": {
-        "is-number": "2.1.0",
-        "isobject": "2.1.0",
-        "randomatic": "1.1.7",
-        "repeat-element": "1.1.2",
-        "repeat-string": "1.6.1"
-      }
-    },
-    "finalhandler": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz",
-      "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==",
-      "requires": {
-        "debug": "2.6.9",
-        "encodeurl": "1.0.2",
-        "escape-html": "1.0.3",
-        "on-finished": "2.3.0",
-        "parseurl": "1.3.2",
-        "statuses": "1.4.0",
-        "unpipe": "1.0.0"
-      }
-    },
-    "find-cache-dir": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-1.0.0.tgz",
-      "integrity": "sha1-kojj6ePMN0hxfTnq3hfPcfww7m8=",
-      "requires": {
-        "commondir": "1.0.1",
-        "make-dir": "1.2.0",
-        "pkg-dir": "2.0.0"
-      }
-    },
-    "find-up": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz",
-      "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=",
-      "requires": {
-        "locate-path": "2.0.0"
-      }
-    },
-    "first-chunk-stream": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/first-chunk-stream/-/first-chunk-stream-2.0.0.tgz",
-      "integrity": "sha1-G97NuOCDwGZLkZRVgVd6Q6nzHXA=",
-      "requires": {
-        "readable-stream": "2.3.6"
-      }
-    },
-    "flat-cache": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-1.3.0.tgz",
-      "integrity": "sha1-0wMLMrOBVPTjt+nHCfSQ9++XxIE=",
-      "requires": {
-        "circular-json": "0.3.3",
-        "del": "2.2.2",
-        "graceful-fs": "4.1.11",
-        "write": "0.2.1"
-      }
-    },
-    "flatten": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/flatten/-/flatten-1.0.2.tgz",
-      "integrity": "sha1-2uRqnXj74lKSJYzB54CkHZXAN4I="
-    },
-    "flow-parser": {
-      "version": "0.70.0",
-      "resolved": "https://registry.npmjs.org/flow-parser/-/flow-parser-0.70.0.tgz",
-      "integrity": "sha512-gGdyVUZWswG5jcINrVDHd3RY4nJptBTAx9mR9thGsrGGmAUR7omgJXQSpR+fXrLtxSTAea3HpAZNU/yzRJc2Cg=="
-    },
-    "flush-write-stream": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.0.3.tgz",
-      "integrity": "sha512-calZMC10u0FMUqoiunI2AiGIIUtUIvifNwkHhNupZH4cbNnW1Itkoh/Nf5HFYmDrwWPjrUxpkZT0KhuCq0jmGw==",
-      "requires": {
-        "inherits": "2.0.3",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "font-awesome": {
-      "version": "4.7.0",
-      "resolved": "https://registry.npmjs.org/font-awesome/-/font-awesome-4.7.0.tgz",
-      "integrity": "sha1-j6jPBBGhoxr9B7BtKQK7n8gVoTM="
-    },
-    "for-in": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
-      "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA="
-    },
-    "for-own": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz",
-      "integrity": "sha1-UmXGgaTylNq78XyVCbZ2OqhFEM4=",
-      "requires": {
-        "for-in": "1.0.2"
-      }
-    },
-    "foreach": {
-      "version": "2.0.5",
-      "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz",
-      "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k="
-    },
-    "forever-agent": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
-      "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
-    },
-    "form-data": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.4.tgz",
-      "integrity": "sha1-M8GDrPGTJ27KqYFDpp6Uv+4XUNE=",
-      "requires": {
-        "asynckit": "0.4.0",
-        "combined-stream": "1.0.6",
-        "mime-types": "2.1.18"
-      }
-    },
-    "formatio": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/formatio/-/formatio-1.2.0.tgz",
-      "integrity": "sha1-87IWfZBoxGmKjVH092CjmlTYGOs=",
-      "dev": true,
-      "requires": {
-        "samsam": "1.3.0"
-      }
-    },
-    "forwarded": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
-      "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
-    },
-    "fragment-cache": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz",
-      "integrity": "sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk=",
-      "requires": {
-        "map-cache": "0.2.2"
-      }
-    },
-    "fresh": {
-      "version": "0.5.2",
-      "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
-      "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
-    },
-    "from2": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz",
-      "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=",
-      "requires": {
-        "inherits": "2.0.3",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "fs-access": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/fs-access/-/fs-access-1.0.1.tgz",
-      "integrity": "sha1-1qh/JiJxzv6+wwxVNAf7mV2od3o=",
-      "dev": true,
-      "requires": {
-        "null-check": "1.0.0"
-      }
-    },
-    "fs-write-stream-atomic": {
-      "version": "1.0.10",
-      "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz",
-      "integrity": "sha1-tH31NJPvkR33VzHnCp3tAYnbQMk=",
-      "requires": {
-        "graceful-fs": "4.1.11",
-        "iferr": "0.1.5",
-        "imurmurhash": "0.1.4",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "fs.realpath": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
-    },
-    "fstream": {
-      "version": "1.0.11",
-      "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.11.tgz",
-      "integrity": "sha1-XB+x8RdHcRTwYyoOtLcbPLD9MXE=",
-      "requires": {
-        "graceful-fs": "4.1.11",
-        "inherits": "2.0.3",
-        "mkdirp": "0.5.1",
-        "rimraf": "2.6.2"
-      }
-    },
-    "function-bind": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
-      "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
-    },
-    "functional-red-black-tree": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
-      "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc="
-    },
-    "gauge": {
-      "version": "2.7.4",
-      "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
-      "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
-      "requires": {
-        "aproba": "1.2.0",
-        "console-control-strings": "1.1.0",
-        "has-unicode": "2.0.1",
-        "object-assign": "4.1.1",
-        "signal-exit": "3.0.2",
-        "string-width": "1.0.2",
-        "strip-ansi": "3.0.1",
-        "wide-align": "1.1.2"
-      },
-      "dependencies": {
-        "is-fullwidth-code-point": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-          "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
-          "requires": {
-            "number-is-nan": "1.0.1"
-          }
-        },
-        "string-width": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
-          "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
-          "requires": {
-            "code-point-at": "1.1.0",
-            "is-fullwidth-code-point": "1.0.0",
-            "strip-ansi": "3.0.1"
-          }
-        }
-      }
-    },
-    "gaze": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/gaze/-/gaze-1.1.2.tgz",
-      "integrity": "sha1-hHIkZ3rbiHDWeSV+0ziP22HkAQU=",
-      "requires": {
-        "globule": "1.2.0"
-      }
-    },
-    "generate-function": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz",
-      "integrity": "sha1-aFj+fAlpt9TpCTM3ZHrHn2DfvnQ="
-    },
-    "generate-object-property": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz",
-      "integrity": "sha1-nA4cQDCM6AT0eDYYuTf6iPmdUNA=",
-      "requires": {
-        "is-property": "1.0.2"
-      }
-    },
-    "get-caller-file": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz",
-      "integrity": "sha1-9wLmMSfn4jHBYKgMFVSstw1QR+U="
-    },
-    "get-func-name": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz",
-      "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=",
-      "dev": true
-    },
-    "get-stdin": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz",
-      "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4="
-    },
-    "get-stream": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
-      "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ="
-    },
-    "get-value": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz",
-      "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg="
-    },
-    "getpass": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
-      "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
-      "requires": {
-        "assert-plus": "1.0.0"
-      },
-      "dependencies": {
-        "assert-plus": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
-        }
-      }
-    },
-    "gh-got": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/gh-got/-/gh-got-6.0.0.tgz",
-      "integrity": "sha512-F/mS+fsWQMo1zfgG9MD8KWvTWPPzzhuVwY++fhQ5Ggd+0P+CAMHtzMZhNxG+TqGfHDChJKsbh6otfMGqO2AKBw==",
-      "requires": {
-        "got": "7.1.0",
-        "is-plain-obj": "1.1.0"
-      },
-      "dependencies": {
-        "got": {
-          "version": "7.1.0",
-          "resolved": "https://registry.npmjs.org/got/-/got-7.1.0.tgz",
-          "integrity": "sha512-Y5WMo7xKKq1muPsxD+KmrR8DH5auG7fBdDVueZwETwV6VytKyU9OX/ddpq2/1hp1vIPvVb4T81dKQz3BivkNLw==",
-          "requires": {
-            "decompress-response": "3.3.0",
-            "duplexer3": "0.1.4",
-            "get-stream": "3.0.0",
-            "is-plain-obj": "1.1.0",
-            "is-retry-allowed": "1.1.0",
-            "is-stream": "1.1.0",
-            "isurl": "1.0.0",
-            "lowercase-keys": "1.0.1",
-            "p-cancelable": "0.3.0",
-            "p-timeout": "1.2.1",
-            "safe-buffer": "5.1.1",
-            "timed-out": "4.0.1",
-            "url-parse-lax": "1.0.0",
-            "url-to-options": "1.0.1"
-          }
-        },
-        "p-cancelable": {
-          "version": "0.3.0",
-          "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.3.0.tgz",
-          "integrity": "sha512-RVbZPLso8+jFeq1MfNvgXtCRED2raz/dKpacfTNxsx6pLEpEomM7gah6VeHSYV3+vo0OAi4MkArtQcWWXuQoyw=="
-        },
-        "p-timeout": {
-          "version": "1.2.1",
-          "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-1.2.1.tgz",
-          "integrity": "sha1-XrOzU7f86Z8QGhA4iAuwVOu+o4Y=",
-          "requires": {
-            "p-finally": "1.0.0"
-          }
-        },
-        "url-parse-lax": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz",
-          "integrity": "sha1-evjzA2Rem9eaJy56FKxovAYJ2nM=",
-          "requires": {
-            "prepend-http": "1.0.4"
-          }
-        }
-      }
-    },
-    "github-username": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/github-username/-/github-username-4.1.0.tgz",
-      "integrity": "sha1-y+KABBiDIG2kISrp5LXxacML9Bc=",
-      "requires": {
-        "gh-got": "6.0.0"
-      }
-    },
-    "glob": {
-      "version": "7.1.2",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz",
-      "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==",
-      "requires": {
-        "fs.realpath": "1.0.0",
-        "inflight": "1.0.6",
-        "inherits": "2.0.3",
-        "minimatch": "3.0.4",
-        "once": "1.4.0",
-        "path-is-absolute": "1.0.1"
-      }
-    },
-    "glob-all": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/glob-all/-/glob-all-3.1.0.tgz",
-      "integrity": "sha1-iRPd+17hrHgSZWJBsD1SF8ZLAqs=",
-      "requires": {
-        "glob": "7.1.2",
-        "yargs": "1.2.6"
-      },
-      "dependencies": {
-        "minimist": {
-          "version": "0.1.0",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.1.0.tgz",
-          "integrity": "sha1-md9lelJXTCHJBXSX33QnkLK0wN4="
-        },
-        "yargs": {
-          "version": "1.2.6",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-1.2.6.tgz",
-          "integrity": "sha1-nHtKgv1dWVsr8Xq23MQxNUMv40s=",
-          "requires": {
-            "minimist": "0.1.0"
-          }
-        }
-      }
-    },
-    "glob-base": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/glob-base/-/glob-base-0.3.0.tgz",
-      "integrity": "sha1-27Fk9iIbHAscz4Kuoyi0l98Oo8Q=",
-      "requires": {
-        "glob-parent": "2.0.0",
-        "is-glob": "2.0.1"
-      }
-    },
-    "glob-parent": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-2.0.0.tgz",
-      "integrity": "sha1-gTg9ctsFT8zPUzbaqQLxgvbtuyg=",
-      "requires": {
-        "is-glob": "2.0.1"
-      }
-    },
-    "glob-to-regexp": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.3.0.tgz",
-      "integrity": "sha1-jFoUlNIGbFcMw7/kSWF1rMTVAqs=",
-      "dev": true
-    },
-    "global-modules": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-1.0.0.tgz",
-      "integrity": "sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==",
-      "requires": {
-        "global-prefix": "1.0.2",
-        "is-windows": "1.0.2",
-        "resolve-dir": "1.0.1"
-      }
-    },
-    "global-prefix": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz",
-      "integrity": "sha1-2/dDxsFJklk8ZVVoy2btMsASLr4=",
-      "requires": {
-        "expand-tilde": "2.0.2",
-        "homedir-polyfill": "1.0.1",
-        "ini": "1.3.5",
-        "is-windows": "1.0.2",
-        "which": "1.3.0"
-      }
-    },
-    "globals": {
-      "version": "9.18.0",
-      "resolved": "https://registry.npmjs.org/globals/-/globals-9.18.0.tgz",
-      "integrity": "sha512-S0nG3CLEQiY/ILxqtztTWH/3iRRdyBLw6KMDxnKMchrtbj2OFmehVh0WUCfW3DUrIgx/qFrJPICrq4Z4sTR9UQ=="
-    },
-    "globby": {
-      "version": "8.0.1",
-      "resolved": "https://registry.npmjs.org/globby/-/globby-8.0.1.tgz",
-      "integrity": "sha512-oMrYrJERnKBLXNLVTqhm3vPEdJ/b2ZE28xN4YARiix1NOIOBPEpOUnm844K1iu/BkphCaf2WNFwMszv8Soi1pw==",
-      "dev": true,
-      "requires": {
-        "array-union": "1.0.2",
-        "dir-glob": "2.0.0",
-        "fast-glob": "2.2.0",
-        "glob": "7.1.2",
-        "ignore": "3.3.7",
-        "pify": "3.0.0",
-        "slash": "1.0.0"
-      }
-    },
-    "globule": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/globule/-/globule-1.2.0.tgz",
-      "integrity": "sha1-HcScaCLdnoovoAuiopUAboZkvQk=",
-      "requires": {
-        "glob": "7.1.2",
-        "lodash": "4.17.5",
-        "minimatch": "3.0.4"
-      }
-    },
-    "got": {
-      "version": "8.3.0",
-      "resolved": "https://registry.npmjs.org/got/-/got-8.3.0.tgz",
-      "integrity": "sha512-kBNy/S2CGwrYgDSec5KTWGKUvupwkkTVAjIsVFF2shXO13xpZdFP4d4kxa//CLX2tN/rV0aYwK8vY6UKWGn2vQ==",
-      "requires": {
-        "@sindresorhus/is": "0.7.0",
-        "cacheable-request": "2.1.4",
-        "decompress-response": "3.3.0",
-        "duplexer3": "0.1.4",
-        "get-stream": "3.0.0",
-        "into-stream": "3.1.0",
-        "is-retry-allowed": "1.1.0",
-        "isurl": "1.0.0",
-        "lowercase-keys": "1.0.1",
-        "mimic-response": "1.0.0",
-        "p-cancelable": "0.4.1",
-        "p-timeout": "2.0.1",
-        "pify": "3.0.0",
-        "safe-buffer": "5.1.1",
-        "timed-out": "4.0.1",
-        "url-parse-lax": "3.0.0",
-        "url-to-options": "1.0.1"
-      }
-    },
-    "graceful-fs": {
-      "version": "4.1.11",
-      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz",
-      "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg="
-    },
-    "graceful-readlink": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz",
-      "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=",
-      "dev": true
-    },
-    "grouped-queue": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/grouped-queue/-/grouped-queue-0.3.3.tgz",
-      "integrity": "sha1-wWfSpTGcWg4JZO9qJbfC34mWyFw=",
-      "requires": {
-        "lodash": "4.17.5"
-      }
-    },
-    "growl": {
-      "version": "1.9.2",
-      "resolved": "https://registry.npmjs.org/growl/-/growl-1.9.2.tgz",
-      "integrity": "sha1-Dqd0NxXbjY3ixe3hd14bRayFwC8=",
-      "dev": true
-    },
-    "handle-thing": {
-      "version": "1.2.5",
-      "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz",
-      "integrity": "sha1-/Xqtcmvxpf0W38KbL3pmAdJxOcQ="
-    },
-    "har-validator": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz",
-      "integrity": "sha1-zcvAgYgmWtEZtqWnyKtw7s+10n0=",
-      "requires": {
-        "chalk": "1.1.3",
-        "commander": "2.13.0",
-        "is-my-json-valid": "2.17.2",
-        "pinkie-promise": "2.0.1"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        }
-      }
-    },
-    "has": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/has/-/has-1.0.1.tgz",
-      "integrity": "sha1-hGFzP1OLCDfJNh45qauelwTcLyg=",
-      "requires": {
-        "function-bind": "1.1.1"
-      }
-    },
-    "has-ansi": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz",
-      "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=",
-      "requires": {
-        "ansi-regex": "2.1.1"
-      }
-    },
-    "has-binary": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/has-binary/-/has-binary-0.1.7.tgz",
-      "integrity": "sha1-aOYesWIQyVRaClzOBqhzkS/h5ow=",
-      "requires": {
-        "isarray": "0.0.1"
-      },
-      "dependencies": {
-        "isarray": {
-          "version": "0.0.1",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
-          "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
-        }
-      }
-    },
-    "has-binary2": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/has-binary2/-/has-binary2-1.0.2.tgz",
-      "integrity": "sha1-6D26SfC5vk0CbSc2U1DZ8D9Uvpg=",
-      "dev": true,
-      "requires": {
-        "isarray": "2.0.1"
-      },
-      "dependencies": {
-        "isarray": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.1.tgz",
-          "integrity": "sha1-o32U7ZzaLVmGXJ92/llu4fM4dB4=",
-          "dev": true
-        }
-      }
-    },
-    "has-color": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/has-color/-/has-color-0.1.7.tgz",
-      "integrity": "sha1-ZxRKUmDDT8PMpnfQQdr1L+e3iy8="
-    },
-    "has-cors": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/has-cors/-/has-cors-1.1.0.tgz",
-      "integrity": "sha1-XkdHk/fqmEPRu5nCPu9J/xJv/zk="
-    },
-    "has-flag": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-1.0.0.tgz",
-      "integrity": "sha1-nZ55MWXOAXoA8AQYxD+UKnsdEfo="
-    },
-    "has-symbol-support-x": {
-      "version": "1.4.2",
-      "resolved": "https://registry.npmjs.org/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz",
-      "integrity": "sha512-3ToOva++HaW+eCpgqZrCfN51IPB+7bJNVT6CUATzueB5Heb8o6Nam0V3HG5dlDvZU1Gn5QLcbahiKw/XVk5JJw=="
-    },
-    "has-symbols": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz",
-      "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q="
-    },
-    "has-to-string-tag-x": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/has-to-string-tag-x/-/has-to-string-tag-x-1.4.1.tgz",
-      "integrity": "sha512-vdbKfmw+3LoOYVr+mtxHaX5a96+0f3DljYd8JOqvOLsf5mw2Otda2qCDT9qRqLAhrjyQ0h7ual5nOiASpsGNFw==",
-      "requires": {
-        "has-symbol-support-x": "1.4.2"
-      }
-    },
-    "has-unicode": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
-      "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
-    },
-    "has-value": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz",
-      "integrity": "sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc=",
-      "requires": {
-        "get-value": "2.0.6",
-        "has-values": "1.0.0",
-        "isobject": "3.0.1"
-      },
-      "dependencies": {
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        }
-      }
-    },
-    "has-values": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz",
-      "integrity": "sha1-lbC2P+whRmGab+V/51Yo1aOe/k8=",
-      "requires": {
-        "is-number": "3.0.0",
-        "kind-of": "4.0.0"
-      },
-      "dependencies": {
-        "is-number": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-          "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "kind-of": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz",
-          "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=",
-          "requires": {
-            "is-buffer": "1.1.6"
-          }
-        }
-      }
-    },
-    "hash-base": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz",
-      "integrity": "sha1-X8hoaEfs1zSZQDMZprCj8/auSRg=",
-      "requires": {
-        "inherits": "2.0.3",
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "hash.js": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.3.tgz",
-      "integrity": "sha512-/UETyP0W22QILqS+6HowevwhEFJ3MBJnwTf75Qob9Wz9t0DPuisL8kW8YZMK62dHAKE1c1p+gY1TtOLY+USEHA==",
-      "requires": {
-        "inherits": "2.0.3",
-        "minimalistic-assert": "1.0.1"
-      }
-    },
-    "hawk": {
-      "version": "3.1.3",
-      "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz",
-      "integrity": "sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ=",
-      "requires": {
-        "boom": "2.10.1",
-        "cryptiles": "2.0.5",
-        "hoek": "2.16.3",
-        "sntp": "1.0.9"
-      }
-    },
-    "he": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz",
-      "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0="
-    },
-    "hmac-drbg": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz",
-      "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=",
-      "requires": {
-        "hash.js": "1.1.3",
-        "minimalistic-assert": "1.0.1",
-        "minimalistic-crypto-utils": "1.0.1"
-      }
-    },
-    "hoek": {
-      "version": "2.16.3",
-      "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz",
-      "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0="
-    },
-    "home-or-tmp": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/home-or-tmp/-/home-or-tmp-2.0.0.tgz",
-      "integrity": "sha1-42w/LSyufXRqhX440Y1fMqeILbg=",
-      "requires": {
-        "os-homedir": "1.0.2",
-        "os-tmpdir": "1.0.2"
-      }
-    },
-    "homedir-polyfill": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.1.tgz",
-      "integrity": "sha1-TCu8inWJmP7r9e1oWA921GdotLw=",
-      "requires": {
-        "parse-passwd": "1.0.0"
-      }
-    },
-    "hosted-git-info": {
-      "version": "2.6.0",
-      "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.6.0.tgz",
-      "integrity": "sha512-lIbgIIQA3lz5XaB6vxakj6sDHADJiZadYEJB+FgA+C4nubM1NwcuvUr9EJPmnH1skZqpqUzWborWo8EIUi0Sdw=="
-    },
-    "hpack.js": {
-      "version": "2.1.6",
-      "resolved": "https://registry.npmjs.org/hpack.js/-/hpack.js-2.1.6.tgz",
-      "integrity": "sha1-h3dMCUnlE/QuhFdbPEVoH63ioLI=",
-      "requires": {
-        "inherits": "2.0.3",
-        "obuf": "1.1.2",
-        "readable-stream": "2.3.6",
-        "wbuf": "1.7.3"
-      }
-    },
-    "html-comment-regex": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/html-comment-regex/-/html-comment-regex-1.1.1.tgz",
-      "integrity": "sha1-ZouTd26q5V696POtRkswekljYl4="
-    },
-    "html-entities": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-1.2.1.tgz",
-      "integrity": "sha1-DfKTUfByEWNRXfueVUPl9u7VFi8="
-    },
-    "html-loader": {
-      "version": "1.0.0-alpha.0",
-      "resolved": "https://registry.npmjs.org/html-loader/-/html-loader-1.0.0-alpha.0.tgz",
-      "integrity": "sha512-KcuaIRWTU0kFjOJCs32a3JsGNCWkeOak0/F/uvJNp3x/N4McXdqHpcK64cYTozK7QLPKKtUqb9h7wR9K9rYRkg==",
-      "requires": {
-        "@posthtml/esm": "1.0.0",
-        "htmlnano": "0.1.8",
-        "loader-utils": "1.1.0",
-        "posthtml": "0.11.3",
-        "schema-utils": "0.4.5"
-      }
-    },
-    "html-minifier": {
-      "version": "3.5.15",
-      "resolved": "https://registry.npmjs.org/html-minifier/-/html-minifier-3.5.15.tgz",
-      "integrity": "sha512-OZa4rfb6tZOZ3Z8Xf0jKxXkiDcFWldQePGYFDcgKqES2sXeWaEv9y6QQvWUtX3ySI3feApQi5uCsHLINQ6NoAw==",
-      "requires": {
-        "camel-case": "3.0.0",
-        "clean-css": "4.1.11",
-        "commander": "2.15.1",
-        "he": "1.1.1",
-        "param-case": "2.1.1",
-        "relateurl": "0.2.7",
-        "uglify-js": "3.3.21"
-      },
-      "dependencies": {
-        "commander": {
-          "version": "2.15.1",
-          "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz",
-          "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag=="
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        },
-        "uglify-js": {
-          "version": "3.3.21",
-          "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.3.21.tgz",
-          "integrity": "sha512-uy82472lH8tshK3jS3c5IFb5MmNKd/5qyBd0ih8sM42L3jWvxnE339U9gZU1zufnLVs98Stib9twq8dLm2XYCA==",
-          "requires": {
-            "commander": "2.15.1",
-            "source-map": "0.6.1"
-          }
-        }
-      }
-    },
-    "html-webpack-plugin": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz",
-      "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=",
-      "requires": {
-        "html-minifier": "3.5.15",
-        "loader-utils": "0.2.17",
-        "lodash": "4.17.5",
-        "pretty-error": "2.1.1",
-        "tapable": "1.0.0",
-        "toposort": "1.0.6",
-        "util.promisify": "1.0.0"
-      },
-      "dependencies": {
-        "loader-utils": {
-          "version": "0.2.17",
-          "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-0.2.17.tgz",
-          "integrity": "sha1-+G5jdNQyBabmxg6RlvF8Apm/s0g=",
-          "requires": {
-            "big.js": "3.2.0",
-            "emojis-list": "2.1.0",
-            "json5": "0.5.1",
-            "object-assign": "4.1.1"
-          }
-        }
-      }
-    },
-    "htmlescape": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/htmlescape/-/htmlescape-1.1.1.tgz",
-      "integrity": "sha1-OgPtwiFLyjtmQko+eVk0lQnLA1E=",
-      "dev": true
-    },
-    "htmlnano": {
-      "version": "0.1.8",
-      "resolved": "https://registry.npmjs.org/htmlnano/-/htmlnano-0.1.8.tgz",
-      "integrity": "sha512-wwmDRJn5OQ9BqFYy5vWaufUQTKj7Ct6xTv+od7QNNJzJM7K3yqR4lJ8SHSOTcBahlXMO5EzddUdsS+fmdGvXpw==",
-      "requires": {
-        "cssnano": "3.10.0",
-        "object-assign": "4.1.1",
-        "posthtml": "0.11.3",
-        "posthtml-render": "1.1.3",
-        "svgo": "1.0.5",
-        "uglify-es": "3.3.9"
-      },
-      "dependencies": {
-        "coa": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/coa/-/coa-2.0.1.tgz",
-          "integrity": "sha512-5wfTTO8E2/ja4jFSxePXlG5nRu5bBtL/r1HCIpJW/lzT6yDtKl0u0Z4o/Vpz32IpKmBn7HerheEZQgA9N2DarQ==",
-          "requires": {
-            "q": "1.5.1"
-          }
-        },
-        "csso": {
-          "version": "3.5.0",
-          "resolved": "https://registry.npmjs.org/csso/-/csso-3.5.0.tgz",
-          "integrity": "sha512-WtJjFP3ZsSdWhiZr4/k1B9uHPgYjFYnDxfbaJxk1hz5PDLIJ5BCRWkJqaztZ0DbP8d2ZIVwUPIJb2YmCwkPaMw==",
-          "requires": {
-            "css-tree": "1.0.0-alpha.27"
-          },
-          "dependencies": {
-            "css-tree": {
-              "version": "1.0.0-alpha.27",
-              "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-1.0.0-alpha.27.tgz",
-              "integrity": "sha512-BAYp9FyN4jLXjfvRpTDchBllDptqlK9I7OsagXCG9Am5C+5jc8eRZHgqb9x500W2OKS14MMlpQc/nmh/aA7TEQ==",
-              "requires": {
-                "mdn-data": "1.1.1",
-                "source-map": "0.5.7"
-              }
-            }
-          }
-        },
-        "esprima": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz",
-          "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw=="
-        },
-        "js-yaml": {
-          "version": "3.10.0",
-          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.10.0.tgz",
-          "integrity": "sha512-O2v52ffjLa9VeM43J4XocZE//WT9N0IiwDa3KSHH7Tu8CtH+1qM8SIZvnsTh6v+4yFy5KUY3BHUVwjpfAWsjIA==",
-          "requires": {
-            "argparse": "1.0.10",
-            "esprima": "4.0.0"
-          }
-        },
-        "svgo": {
-          "version": "1.0.5",
-          "resolved": "https://registry.npmjs.org/svgo/-/svgo-1.0.5.tgz",
-          "integrity": "sha512-nYrifviB77aNKDNKKyuay3M9aYiK6Hv5gJVDdjj2ZXTQmI8WZc8+UPLR5IpVlktJfSu3co/4XcWgrgI6seGBPg==",
-          "requires": {
-            "coa": "2.0.1",
-            "colors": "1.1.2",
-            "css-select": "1.3.0-rc0",
-            "css-select-base-adapter": "0.1.0",
-            "css-tree": "1.0.0-alpha25",
-            "css-url-regex": "1.1.0",
-            "csso": "3.5.0",
-            "js-yaml": "3.10.0",
-            "mkdirp": "0.5.1",
-            "object.values": "1.0.4",
-            "sax": "1.2.4",
-            "stable": "0.1.7",
-            "unquote": "1.1.1",
-            "util.promisify": "1.0.0"
-          }
-        },
-        "uglify-es": {
-          "version": "3.3.9",
-          "resolved": "https://registry.npmjs.org/uglify-es/-/uglify-es-3.3.9.tgz",
-          "integrity": "sha512-r+MU0rfv4L/0eeW3xZrd16t4NZfK8Ld4SWVglYBb7ez5uXFWHuVRs6xCTrf1yirs9a4j4Y27nn7SRfO6v67XsQ==",
-          "requires": {
-            "commander": "2.13.0",
-            "source-map": "0.6.1"
-          },
-          "dependencies": {
-            "source-map": {
-              "version": "0.6.1",
-              "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-              "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-            }
-          }
-        }
-      }
-    },
-    "htmlparser2": {
-      "version": "3.9.2",
-      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz",
-      "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=",
-      "requires": {
-        "domelementtype": "1.3.0",
-        "domhandler": "2.4.1",
-        "domutils": "1.7.0",
-        "entities": "1.1.1",
-        "inherits": "2.0.3",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "http-cache-semantics": {
-      "version": "3.8.1",
-      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz",
-      "integrity": "sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w=="
-    },
-    "http-deceiver": {
-      "version": "1.2.7",
-      "resolved": "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz",
-      "integrity": "sha1-+nFolEq5pRnTN8sL7HKE3D5yPYc="
-    },
-    "http-errors": {
-      "version": "1.6.3",
-      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz",
-      "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=",
-      "requires": {
-        "depd": "1.1.2",
-        "inherits": "2.0.3",
-        "setprototypeof": "1.1.0",
-        "statuses": "1.4.0"
-      }
-    },
-    "http-parser-js": {
-      "version": "0.4.11",
-      "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.4.11.tgz",
-      "integrity": "sha512-QCR5O2AjjMW8Mo4HyI1ctFcv+O99j/0g367V3YoVnrNw5hkDvAWZD0lWGcc+F4yN3V55USPCVix4efb75HxFfA=="
-    },
-    "http-proxy": {
-      "version": "1.16.2",
-      "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.16.2.tgz",
-      "integrity": "sha1-Bt/ykpUr9k2+hHH6nfcwZtTzd0I=",
-      "requires": {
-        "eventemitter3": "1.2.0",
-        "requires-port": "1.0.0"
-      }
-    },
-    "http-proxy-middleware": {
-      "version": "0.17.4",
-      "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.17.4.tgz",
-      "integrity": "sha1-ZC6ISIUdZvCdTxJJEoRtuutBuDM=",
-      "requires": {
-        "http-proxy": "1.16.2",
-        "is-glob": "3.1.0",
-        "lodash": "4.17.5",
-        "micromatch": "2.3.11"
-      },
-      "dependencies": {
-        "is-extglob": {
-          "version": "2.1.1",
-          "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
-          "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI="
-        },
-        "is-glob": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz",
-          "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=",
-          "requires": {
-            "is-extglob": "2.1.1"
-          }
-        }
-      }
-    },
-    "http-signature": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz",
-      "integrity": "sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8=",
-      "requires": {
-        "assert-plus": "0.2.0",
-        "jsprim": "1.4.1",
-        "sshpk": "1.14.1"
-      }
-    },
-    "https-browserify": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz",
-      "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM="
-    },
-    "iconv-lite": {
-      "version": "0.4.21",
-      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.21.tgz",
-      "integrity": "sha512-En5V9za5mBt2oUA03WGD3TwDv0MKAruqsuxstbMUZaj9W9k/m1CV/9py3l0L5kw9Bln8fdHQmzHSYtvpvTLpKw==",
-      "requires": {
-        "safer-buffer": "2.1.2"
-      }
-    },
-    "icss-replace-symbols": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/icss-replace-symbols/-/icss-replace-symbols-1.1.0.tgz",
-      "integrity": "sha1-Bupvg2ead0njhs/h/oEq5dsiPe0="
-    },
-    "icss-utils": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-2.1.0.tgz",
-      "integrity": "sha1-g/Cg7DeL8yRheLbCrZE28TWxyWI=",
-      "requires": {
-        "postcss": "6.0.21"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "postcss": {
-          "version": "6.0.21",
-          "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.21.tgz",
-          "integrity": "sha512-y/bKfbQz2Nn/QBC08bwvYUxEFOVGfPIUOTsJ2CK5inzlXW9SdYR1x4pEsG9blRAF/PX+wRNdOah+gx/hv4q7dw==",
-          "requires": {
-            "chalk": "2.4.0",
-            "source-map": "0.6.1",
-            "supports-color": "5.4.0"
-          }
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "ieee754": {
-      "version": "1.1.11",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.11.tgz",
-      "integrity": "sha512-VhDzCKN7K8ufStx/CLj5/PDTMgph+qwN5Pkd5i0sGnVwk56zJ0lkT8Qzi1xqWLS0Wp29DgDtNeS7v8/wMoZeHg=="
-    },
-    "iferr": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz",
-      "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE="
-    },
-    "ignore": {
-      "version": "3.3.7",
-      "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.7.tgz",
-      "integrity": "sha512-YGG3ejvBNHRqu0559EOxxNFihD0AjpvHlC/pdGKd3X3ofe+CoJkYazwNJYTNebqpPKN+VVQbh4ZFn1DivMNuHA=="
-    },
-    "ignore-loader": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/ignore-loader/-/ignore-loader-0.1.2.tgz",
-      "integrity": "sha1-2B8kA3bQuk8Nd4lyw60lh0EXpGM=",
-      "dev": true
-    },
-    "image-size": {
-      "version": "0.5.5",
-      "resolved": "https://registry.npmjs.org/image-size/-/image-size-0.5.5.tgz",
-      "integrity": "sha1-Cd/Uq50g4p6xw+gLiZA3jfnjy5w="
-    },
-    "immediate": {
-      "version": "3.0.6",
-      "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz",
-      "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps="
-    },
-    "import-local": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/import-local/-/import-local-1.0.0.tgz",
-      "integrity": "sha512-vAaZHieK9qjGo58agRBg+bhHX3hoTZU/Oa3GESWLz7t1U62fk63aHuDJJEteXoDeTCcPmUT+z38gkHPZkkmpmQ==",
-      "requires": {
-        "pkg-dir": "2.0.0",
-        "resolve-cwd": "2.0.0"
-      }
-    },
-    "imurmurhash": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
-      "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o="
-    },
-    "in-publish": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/in-publish/-/in-publish-2.0.0.tgz",
-      "integrity": "sha1-4g/146KvwmkDILbcVSaCqcf631E="
-    },
-    "indent-string": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz",
-      "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=",
-      "requires": {
-        "repeating": "2.0.1"
-      }
-    },
-    "indexes-of": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/indexes-of/-/indexes-of-1.0.1.tgz",
-      "integrity": "sha1-8w9xbI4r00bHtn0985FVZqfAVgc="
-    },
-    "indexof": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/indexof/-/indexof-0.0.1.tgz",
-      "integrity": "sha1-gtwzbSMrkGIXnQWrMpOmYFn9Q10="
-    },
-    "inflight": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-      "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
-      "requires": {
-        "once": "1.4.0",
-        "wrappy": "1.0.2"
-      }
-    },
-    "inherits": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
-      "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
-    },
-    "ini": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
-      "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
-    },
-    "inline-source-map": {
-      "version": "0.6.2",
-      "resolved": "https://registry.npmjs.org/inline-source-map/-/inline-source-map-0.6.2.tgz",
-      "integrity": "sha1-+Tk0ccGKedFyT4Y/o4tYY3Ct4qU=",
-      "dev": true,
-      "requires": {
-        "source-map": "0.5.7"
-      }
-    },
-    "inquirer": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-3.3.0.tgz",
-      "integrity": "sha512-h+xtnyk4EwKvFWHrUYsWErEVR+igKtLdchu+o0Z1RL7VU/jVMFbYir2bp6bAj8efFNxWqHX0dIss6fJQ+/+qeQ==",
-      "requires": {
-        "ansi-escapes": "3.1.0",
-        "chalk": "2.4.0",
-        "cli-cursor": "2.1.0",
-        "cli-width": "2.2.0",
-        "external-editor": "2.2.0",
-        "figures": "2.0.0",
-        "lodash": "4.17.5",
-        "mute-stream": "0.0.7",
-        "run-async": "2.3.0",
-        "rx-lite": "4.0.8",
-        "rx-lite-aggregates": "4.0.8",
-        "string-width": "2.1.1",
-        "strip-ansi": "4.0.0",
-        "through": "2.3.8"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg="
-        },
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "strip-ansi": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
-          "requires": {
-            "ansi-regex": "3.0.0"
-          }
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "insert-module-globals": {
-      "version": "7.0.6",
-      "resolved": "https://registry.npmjs.org/insert-module-globals/-/insert-module-globals-7.0.6.tgz",
-      "integrity": "sha512-R3sidKJr3SsggqQQ5cEwQb3pWG8RNx0UnpyeiOSR6jorRIeAOzH2gkTWnNdMnyRiVbjrG047K7UCtlMkQ1Mo9w==",
-      "dev": true,
-      "requires": {
-        "JSONStream": "1.3.2",
-        "combine-source-map": "0.8.0",
-        "concat-stream": "1.6.2",
-        "is-buffer": "1.1.6",
-        "lexical-scope": "1.2.0",
-        "path-is-absolute": "1.0.1",
-        "process": "0.11.10",
-        "through2": "2.0.3",
-        "xtend": "4.0.1"
-      }
-    },
-    "internal-ip": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/internal-ip/-/internal-ip-1.2.0.tgz",
-      "integrity": "sha1-rp+/k7mEh4eF1QqN4bNWlWBYz1w=",
-      "requires": {
-        "meow": "3.7.0"
-      }
-    },
-    "interpret": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.1.0.tgz",
-      "integrity": "sha1-ftGxQQxqDg94z5XTuEQMY/eLhhQ="
-    },
-    "into-stream": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-3.1.0.tgz",
-      "integrity": "sha1-lvsKk2wSur1v8XUqF9BWFqvQlMY=",
-      "requires": {
-        "from2": "2.3.0",
-        "p-is-promise": "1.1.0"
-      }
-    },
-    "invariant": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz",
-      "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==",
-      "requires": {
-        "loose-envify": "1.3.1"
-      }
-    },
-    "invert-kv": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",
-      "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY="
-    },
-    "ip": {
-      "version": "1.1.5",
-      "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
-      "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
-    },
-    "ipaddr.js": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.6.0.tgz",
-      "integrity": "sha1-4/o1e3c9phnybpXwSdBVxyeW+Gs="
-    },
-    "is-absolute-url": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-2.1.0.tgz",
-      "integrity": "sha1-UFMN+4T8yap9vnhS6Do3uTufKqY="
-    },
-    "is-accessor-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-      "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
-      "requires": {
-        "kind-of": "3.2.2"
-      }
-    },
-    "is-arrayish": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
-      "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0="
-    },
-    "is-binary-path": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz",
-      "integrity": "sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=",
-      "requires": {
-        "binary-extensions": "1.11.0"
-      }
-    },
-    "is-buffer": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
-      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w=="
-    },
-    "is-builtin-module": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz",
-      "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=",
-      "requires": {
-        "builtin-modules": "1.1.1"
-      },
-      "dependencies": {
-        "builtin-modules": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz",
-          "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8="
-        }
-      }
-    },
-    "is-callable": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.3.tgz",
-      "integrity": "sha1-hut1OSgF3cM69xySoO7fdO52BLI="
-    },
-    "is-data-descriptor": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-      "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
-      "requires": {
-        "kind-of": "3.2.2"
-      }
-    },
-    "is-date-object": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz",
-      "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY="
-    },
-    "is-descriptor": {
-      "version": "0.1.6",
-      "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-      "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-      "requires": {
-        "is-accessor-descriptor": "0.1.6",
-        "is-data-descriptor": "0.1.4",
-        "kind-of": "5.1.0"
-      },
-      "dependencies": {
-        "kind-of": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
-        }
-      }
-    },
-    "is-dotfile": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/is-dotfile/-/is-dotfile-1.0.3.tgz",
-      "integrity": "sha1-pqLzL/0t+wT1yiXs0Pa4PPeYoeE="
-    },
-    "is-equal-shallow": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/is-equal-shallow/-/is-equal-shallow-0.1.3.tgz",
-      "integrity": "sha1-IjgJj8Ih3gvPpdnqxMRdY4qhxTQ=",
-      "requires": {
-        "is-primitive": "2.0.0"
-      }
-    },
-    "is-expression": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/is-expression/-/is-expression-3.0.0.tgz",
-      "integrity": "sha1-Oayqa+f9HzRx3ELHQW5hwkMXrJ8=",
-      "requires": {
-        "acorn": "4.0.13",
-        "object-assign": "4.1.1"
-      },
-      "dependencies": {
-        "acorn": {
-          "version": "4.0.13",
-          "resolved": "https://registry.npmjs.org/acorn/-/acorn-4.0.13.tgz",
-          "integrity": "sha1-EFSVrlNh1pe9GVyCUZLhrX8lN4c="
-        }
-      }
-    },
-    "is-extendable": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
-      "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik="
-    },
-    "is-extglob": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz",
-      "integrity": "sha1-rEaBd8SUNAWgkvyPKXYMb/xiBsA="
-    },
-    "is-finite": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.0.2.tgz",
-      "integrity": "sha1-zGZ3aVYCvlUO8R6LSqYwU0K20Ko=",
-      "requires": {
-        "number-is-nan": "1.0.1"
-      }
-    },
-    "is-fullwidth-code-point": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
-      "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8="
-    },
-    "is-glob": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz",
-      "integrity": "sha1-0Jb5JqPe1WAPP9/ZEZjLCIjC2GM=",
-      "requires": {
-        "is-extglob": "1.0.0"
-      }
-    },
-    "is-module": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-module/-/is-module-1.0.0.tgz",
-      "integrity": "sha1-Mlj7afeMFNW4FdZkM2tM/7ZEFZE="
-    },
-    "is-my-ip-valid": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-my-ip-valid/-/is-my-ip-valid-1.0.0.tgz",
-      "integrity": "sha512-gmh/eWXROncUzRnIa1Ubrt5b8ep/MGSnfAUI3aRp+sqTCs1tv1Isl8d8F6JmkN3dXKc3ehZMrtiPN9eL03NuaQ=="
-    },
-    "is-my-json-valid": {
-      "version": "2.17.2",
-      "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.17.2.tgz",
-      "integrity": "sha512-IBhBslgngMQN8DDSppmgDv7RNrlFotuuDsKcrCP3+HbFaVivIBU7u9oiiErw8sH4ynx3+gOGQ3q2otkgiSi6kg==",
-      "requires": {
-        "generate-function": "2.0.0",
-        "generate-object-property": "1.2.0",
-        "is-my-ip-valid": "1.0.0",
-        "jsonpointer": "4.0.1",
-        "xtend": "4.0.1"
-      }
-    },
-    "is-number": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/is-number/-/is-number-2.1.0.tgz",
-      "integrity": "sha1-Afy7s5NGOlSPL0ZszhbezknbkI8=",
-      "requires": {
-        "kind-of": "3.2.2"
-      }
-    },
-    "is-object": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.1.tgz",
-      "integrity": "sha1-iVJojF7C/9awPsyF52ngKQMINHA="
-    },
-    "is-observable": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/is-observable/-/is-observable-0.2.0.tgz",
-      "integrity": "sha1-s2ExHYPG5dcmyr9eJQsCNxBvWuI=",
-      "requires": {
-        "symbol-observable": "0.2.4"
-      },
-      "dependencies": {
-        "symbol-observable": {
-          "version": "0.2.4",
-          "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-0.2.4.tgz",
-          "integrity": "sha1-lag9smGG1q9+ehjb2XYKL4bQj0A="
-        }
-      }
-    },
-    "is-odd": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/is-odd/-/is-odd-2.0.0.tgz",
-      "integrity": "sha512-OTiixgpZAT1M4NHgS5IguFp/Vz2VI3U7Goh4/HA1adtwyLtSBrxYlcSYkhpAE07s4fKEcjrFxyvtQBND4vFQyQ==",
-      "requires": {
-        "is-number": "4.0.0"
-      },
-      "dependencies": {
-        "is-number": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-4.0.0.tgz",
-          "integrity": "sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ=="
-        }
-      }
-    },
-    "is-path-cwd": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz",
-      "integrity": "sha1-0iXsIxMuie3Tj9p2dHLmLmXxEG0="
-    },
-    "is-path-in-cwd": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz",
-      "integrity": "sha512-FjV1RTW48E7CWM7eE/J2NJvAEEVektecDBVBE5Hh3nM1Jd0kvhHtX68Pr3xsDf857xt3Y4AkwVULK1Vku62aaQ==",
-      "requires": {
-        "is-path-inside": "1.0.1"
-      }
-    },
-    "is-path-inside": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz",
-      "integrity": "sha1-jvW33lBDej/cprToZe96pVy0gDY=",
-      "requires": {
-        "path-is-inside": "1.0.2"
-      }
-    },
-    "is-plain-obj": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz",
-      "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4="
-    },
-    "is-plain-object": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
-      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
-      "requires": {
-        "isobject": "3.0.1"
-      },
-      "dependencies": {
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        }
-      }
-    },
-    "is-posix-bracket": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz",
-      "integrity": "sha1-MzTceXdDaOkvAW5vvAqI9c1ua8Q="
-    },
-    "is-primitive": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/is-primitive/-/is-primitive-2.0.0.tgz",
-      "integrity": "sha1-IHurkWOEmcB7Kt8kCkGochADRXU="
-    },
-    "is-promise": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz",
-      "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o="
-    },
-    "is-property": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz",
-      "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ="
-    },
-    "is-regex": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz",
-      "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=",
-      "requires": {
-        "has": "1.0.1"
-      }
-    },
-    "is-resolvable": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-resolvable/-/is-resolvable-1.1.0.tgz",
-      "integrity": "sha512-qgDYXFSR5WvEfuS5dMj6oTMEbrrSaM0CrFk2Yiq/gXnBvD9pMa2jGXxyhGLfvhZpuMZe18CJpFxAt3CRs42NMg=="
-    },
-    "is-retry-allowed": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.1.0.tgz",
-      "integrity": "sha1-EaBgVotnM5REAz0BJaYaINVk+zQ="
-    },
-    "is-scoped": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-scoped/-/is-scoped-1.0.0.tgz",
-      "integrity": "sha1-RJypgpnnEwOCViieyytUDcQ3yzA=",
-      "requires": {
-        "scoped-regex": "1.0.0"
-      }
-    },
-    "is-stream": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz",
-      "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ="
-    },
-    "is-svg": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/is-svg/-/is-svg-2.1.0.tgz",
-      "integrity": "sha1-z2EJDaDZ77yrhyLeum8DIgjbsOk=",
-      "requires": {
-        "html-comment-regex": "1.1.1"
-      }
-    },
-    "is-symbol": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz",
-      "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI="
-    },
-    "is-typedarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
-      "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
-    },
-    "is-utf8": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz",
-      "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI="
-    },
-    "is-windows": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz",
-      "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA=="
-    },
-    "is-wsl": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz",
-      "integrity": "sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0="
-    },
-    "isarray": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-      "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
-    },
-    "isbinaryfile": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/isbinaryfile/-/isbinaryfile-3.0.2.tgz",
-      "integrity": "sha1-Sj6XTsDLqQBNP8bN5yCeppNopiE=",
-      "dev": true
-    },
-    "isexe": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-      "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA="
-    },
-    "isobject": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz",
-      "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=",
-      "requires": {
-        "isarray": "1.0.0"
-      }
-    },
-    "isstream": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
-      "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
-    },
-    "istextorbinary": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/istextorbinary/-/istextorbinary-2.2.1.tgz",
-      "integrity": "sha512-TS+hoFl8Z5FAFMK38nhBkdLt44CclNRgDHWeMgsV8ko3nDlr/9UI2Sf839sW7enijf8oKsZYXRvM8g0it9Zmcw==",
-      "requires": {
-        "binaryextensions": "2.1.1",
-        "editions": "1.3.4",
-        "textextensions": "2.2.0"
-      }
-    },
-    "isurl": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/isurl/-/isurl-1.0.0.tgz",
-      "integrity": "sha512-1P/yWsxPlDtn7QeRD+ULKQPaIaN6yF368GZ2vDfv0AL0NwpStafjWCDDdn0k8wgFMWpVAqG7oJhxHnlud42i9w==",
-      "requires": {
-        "has-to-string-tag-x": "1.4.1",
-        "is-object": "1.0.1"
-      }
-    },
-    "jasmine-core": {
-      "version": "2.6.4",
-      "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-2.6.4.tgz",
-      "integrity": "sha1-3skmzQqfoof7bbXHVfpIfnTOysU=",
-      "dev": true
-    },
-    "jquery": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.2.1.tgz",
-      "integrity": "sha1-XE2d5lKvbNCncBVKYxu6ErAVx4c="
-    },
-    "js-base64": {
-      "version": "2.4.3",
-      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-2.4.3.tgz",
-      "integrity": "sha512-H7ErYLM34CvDMto3GbD6xD0JLUGYXR3QTcH6B/tr4Hi/QpSThnCsIp+Sy5FRTw3B0d6py4HcNkW7nO/wdtGWEw=="
-    },
-    "js-stringify": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/js-stringify/-/js-stringify-1.0.2.tgz",
-      "integrity": "sha1-Fzb939lyTyijaCrcYjCufk6Weds="
-    },
-    "js-tokens": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz",
-      "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls="
-    },
-    "js-yaml": {
-      "version": "3.7.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz",
-      "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=",
-      "requires": {
-        "argparse": "1.0.10",
-        "esprima": "2.7.3"
-      }
-    },
-    "jscodeshift": {
-      "version": "0.5.0",
-      "resolved": "https://registry.npmjs.org/jscodeshift/-/jscodeshift-0.5.0.tgz",
-      "integrity": "sha512-JAcQINNMFpdzzpKJN8k5xXjF3XDuckB1/48uScSzcnNyK199iWEc9AxKL9OoX5144M2w5zEx9Qs4/E/eBZZUlw==",
-      "requires": {
-        "babel-plugin-transform-flow-strip-types": "6.22.0",
-        "babel-preset-es2015": "6.24.1",
-        "babel-preset-stage-1": "6.24.1",
-        "babel-register": "6.26.0",
-        "babylon": "7.0.0-beta.44",
-        "colors": "1.1.2",
-        "flow-parser": "0.70.0",
-        "lodash": "4.17.5",
-        "micromatch": "2.3.11",
-        "neo-async": "2.5.1",
-        "node-dir": "0.1.8",
-        "nomnom": "1.8.1",
-        "recast": "0.14.7",
-        "temp": "0.8.3",
-        "write-file-atomic": "1.3.4"
-      },
-      "dependencies": {
-        "babylon": {
-          "version": "7.0.0-beta.44",
-          "resolved": "https://registry.npmjs.org/babylon/-/babylon-7.0.0-beta.44.tgz",
-          "integrity": "sha512-5Hlm13BJVAioCHpImtFqNOF2H3ieTOHd0fmFGMxOJ9jgeFqeAwsv3u5P5cR7CSeFrkgHsT19DgFJkHV0/Mcd8g=="
-        }
-      }
-    },
-    "jsesc": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-1.3.0.tgz",
-      "integrity": "sha1-RsP+yMGJKxKwgz25vHYiF226s0s="
-    },
-    "json-bigint": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.2.3.tgz",
-      "integrity": "sha1-EY1/b/HThlnxn5TPc+ZKdaP5iKg=",
-      "requires": {
-        "bignumber.js": "4.1.0"
-      }
-    },
-    "json-buffer": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz",
-      "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg="
-    },
-    "json-loader": {
-      "version": "0.5.7",
-      "resolved": "https://registry.npmjs.org/json-loader/-/json-loader-0.5.7.tgz",
-      "integrity": "sha512-QLPs8Dj7lnf3e3QYS1zkCo+4ZwqOiF9d/nZnYozTISxXWCfNs9yuky5rJw4/W34s7POaNlbZmQGaB5NiXCbP4w=="
-    },
-    "json-parse-better-errors": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz",
-      "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw=="
-    },
-    "json-schema": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
-      "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
-    },
-    "json-schema-traverse": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz",
-      "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A="
-    },
-    "json-stable-stringify": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz",
-      "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=",
-      "requires": {
-        "jsonify": "0.0.0"
-      }
-    },
-    "json-stringify-safe": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
-      "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
-    },
-    "json3": {
-      "version": "3.3.2",
-      "resolved": "https://registry.npmjs.org/json3/-/json3-3.3.2.tgz",
-      "integrity": "sha1-PAQ0dD35Pi9cQq7nsZvLSDV19OE="
-    },
-    "json5": {
-      "version": "0.5.1",
-      "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz",
-      "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE="
-    },
-    "jsondiffpatch": {
-      "version": "0.2.5",
-      "resolved": "https://registry.npmjs.org/jsondiffpatch/-/jsondiffpatch-0.2.5.tgz",
-      "integrity": "sha1-UDYdmVz4yGE36NVYnyD6UiDbNRE=",
-      "requires": {
-        "chalk": "0.5.1"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "0.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz",
-          "integrity": "sha1-DY6UaWej2BQ/k+JOKYUl/BsiNfk="
-        },
-        "ansi-styles": {
-          "version": "1.1.0",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.1.0.tgz",
-          "integrity": "sha1-6uy/Zs1waIJ2Cy9GkVgrj1XXp94="
-        },
-        "chalk": {
-          "version": "0.5.1",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz",
-          "integrity": "sha1-Zjs6ZItotV0EaQ1JFnqoN4WPIXQ=",
-          "requires": {
-            "ansi-styles": "1.1.0",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "0.1.0",
-            "strip-ansi": "0.3.0",
-            "supports-color": "0.2.0"
-          }
-        },
-        "has-ansi": {
-          "version": "0.1.0",
-          "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-0.1.0.tgz",
-          "integrity": "sha1-hPJlqujA5qiKEtcCKJS3VoiUxi4=",
-          "requires": {
-            "ansi-regex": "0.2.1"
-          }
-        },
-        "strip-ansi": {
-          "version": "0.3.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz",
-          "integrity": "sha1-JfSOoiynkYfzF0pNuHWTR7sSYiA=",
-          "requires": {
-            "ansi-regex": "0.2.1"
-          }
-        },
-        "supports-color": {
-          "version": "0.2.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-0.2.0.tgz",
-          "integrity": "sha1-2S3iaU6z9nMjlz1649i1W0wiGQo="
-        }
-      }
-    },
-    "jsonify": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz",
-      "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM="
-    },
-    "jsonparse": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz",
-      "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=",
-      "dev": true
-    },
-    "jsonpointer": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.0.1.tgz",
-      "integrity": "sha1-T9kss04OnbPInIYi7PUfm5eMbLk="
-    },
-    "jsprim": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
-      "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
-      "requires": {
-        "assert-plus": "1.0.0",
-        "extsprintf": "1.3.0",
-        "json-schema": "0.2.3",
-        "verror": "1.10.0"
-      },
-      "dependencies": {
-        "assert-plus": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
-        }
-      }
-    },
-    "jstransformer": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/jstransformer/-/jstransformer-1.0.0.tgz",
-      "integrity": "sha1-7Yvwkh4vPx7U1cGkT2hwntJHIsM=",
-      "requires": {
-        "is-promise": "2.1.0",
-        "promise": "7.3.1"
-      }
-    },
-    "jszip": {
-      "version": "3.1.5",
-      "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.1.5.tgz",
-      "integrity": "sha512-5W8NUaFRFRqTOL7ZDDrx5qWHJyBXy6velVudIzQUSoqAAYqzSh2Z7/m0Rf1QbmQJccegD0r+YZxBjzqoBiEeJQ==",
-      "requires": {
-        "core-js": "2.3.0",
-        "es6-promise": "3.0.2",
-        "lie": "3.1.1",
-        "pako": "1.0.6",
-        "readable-stream": "2.0.6"
-      },
-      "dependencies": {
-        "core-js": {
-          "version": "2.3.0",
-          "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.3.0.tgz",
-          "integrity": "sha1-+rg/uwstjchfpjbEudNMdUIMbWU="
-        },
-        "process-nextick-args": {
-          "version": "1.0.7",
-          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
-          "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M="
-        },
-        "readable-stream": {
-          "version": "2.0.6",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz",
-          "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=",
-          "requires": {
-            "core-util-is": "1.0.2",
-            "inherits": "2.0.3",
-            "isarray": "1.0.0",
-            "process-nextick-args": "1.0.7",
-            "string_decoder": "0.10.31",
-            "util-deprecate": "1.0.2"
-          }
-        },
-        "string_decoder": {
-          "version": "0.10.31",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
-          "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
-        }
-      }
-    },
-    "karma": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/karma/-/karma-2.0.0.tgz",
-      "integrity": "sha512-K9Kjp8CldLyL9ANSUctDyxC7zH3hpqXj/K09qVf06K3T/kXaHtFZ5tQciK7OzQu68FLvI89Na510kqQ2LCbpIw==",
-      "dev": true,
-      "requires": {
-        "bluebird": "3.5.1",
-        "body-parser": "1.18.2",
-        "browserify": "14.5.0",
-        "chokidar": "1.7.0",
-        "colors": "1.1.2",
-        "combine-lists": "1.0.1",
-        "connect": "3.6.6",
-        "core-js": "2.5.5",
-        "di": "0.0.1",
-        "dom-serialize": "2.2.1",
-        "expand-braces": "0.1.2",
-        "glob": "7.1.2",
-        "graceful-fs": "4.1.11",
-        "http-proxy": "1.16.2",
-        "isbinaryfile": "3.0.2",
-        "lodash": "4.17.5",
-        "log4js": "2.5.3",
-        "mime": "1.4.1",
-        "minimatch": "3.0.4",
-        "optimist": "0.6.1",
-        "qjobs": "1.2.0",
-        "range-parser": "1.2.0",
-        "rimraf": "2.6.2",
-        "safe-buffer": "5.1.1",
-        "socket.io": "2.0.4",
-        "source-map": "0.6.1",
-        "tmp": "0.0.33",
-        "useragent": "2.3.0"
-      },
-      "dependencies": {
-        "anymatch": {
-          "version": "1.3.2",
-          "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-1.3.2.tgz",
-          "integrity": "sha512-0XNayC8lTHQ2OI8aljNCN3sSx6hsr/1+rlcDAotXJR7C1oZZHCNsfpbKwMjRA3Uqb5tF1Rae2oloTr4xpq+WjA==",
-          "dev": true,
-          "requires": {
-            "micromatch": "2.3.11",
-            "normalize-path": "2.1.1"
-          }
-        },
-        "chokidar": {
-          "version": "1.7.0",
-          "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-1.7.0.tgz",
-          "integrity": "sha1-eY5ol3gVHIB2tLNg5e3SjNortGg=",
-          "dev": true,
-          "requires": {
-            "anymatch": "1.3.2",
-            "async-each": "1.0.1",
-            "glob-parent": "2.0.0",
-            "inherits": "2.0.3",
-            "is-binary-path": "1.0.1",
-            "is-glob": "2.0.1",
-            "path-is-absolute": "1.0.1",
-            "readdirp": "2.1.0"
-          }
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
-          "dev": true
-        }
-      }
-    },
-    "karma-babel-preprocessor": {
-      "version": "6.0.1",
-      "resolved": "https://registry.npmjs.org/karma-babel-preprocessor/-/karma-babel-preprocessor-6.0.1.tgz",
-      "integrity": "sha1-euHT5klQ2+EfQht0BAqwj7WmbCE=",
-      "dev": true,
-      "requires": {
-        "babel-core": "6.26.0"
-      }
-    },
-    "karma-chrome-launcher": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/karma-chrome-launcher/-/karma-chrome-launcher-2.2.0.tgz",
-      "integrity": "sha512-uf/ZVpAabDBPvdPdveyk1EPgbnloPvFFGgmRhYLTDH7gEB4nZdSBk8yTU47w1g/drLSx5uMOkjKk7IWKfWg/+w==",
-      "dev": true,
-      "requires": {
-        "fs-access": "1.0.1",
-        "which": "1.3.0"
-      }
-    },
-    "karma-mocha": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/karma-mocha/-/karma-mocha-1.3.0.tgz",
-      "integrity": "sha1-7qrH/8DiAetjxGdEDStpx883eL8=",
-      "dev": true,
-      "requires": {
-        "minimist": "1.2.0"
-      },
-      "dependencies": {
-        "minimist": {
-          "version": "1.2.0",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
-          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
-          "dev": true
-        }
-      }
-    },
-    "karma-mocha-reporter": {
-      "version": "2.2.3",
-      "resolved": "https://registry.npmjs.org/karma-mocha-reporter/-/karma-mocha-reporter-2.2.3.tgz",
-      "integrity": "sha1-BP3aRaHZaXpzhxx0ciI8WBcBqyA=",
-      "dev": true,
-      "requires": {
-        "chalk": "1.1.3"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "dev": true,
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        }
-      }
-    },
-    "karma-teamcity-reporter": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/karma-teamcity-reporter/-/karma-teamcity-reporter-1.0.0.tgz",
-      "integrity": "sha1-0zwSF3U8FBiX9kVg8l8PN76QUjM=",
-      "dev": true
-    },
-    "karma-webpack": {
-      "version": "4.0.0-beta.0",
-      "resolved": "https://registry.npmjs.org/karma-webpack/-/karma-webpack-4.0.0-beta.0.tgz",
-      "integrity": "sha512-3mBfzOSnWdlMNtIIFpZ0/fGbXCq6dko0HOnwU7nntpNu7tTcY7/JbaWV8bxvmIre+yNUPIglq7p3EuwXj35BmA==",
-      "dev": true,
-      "requires": {
-        "async": "2.6.0",
-        "babel-runtime": "6.26.0",
-        "loader-utils": "1.1.0",
-        "lodash": "4.17.5",
-        "source-map": "0.5.7",
-        "webpack-dev-middleware": "3.0.1"
-      }
-    },
-    "keyv": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.0.0.tgz",
-      "integrity": "sha512-eguHnq22OE3uVoSYG0LVWNP+4ppamWr9+zWBe1bsNcovIMy6huUJFPgy4mGwCd/rnl3vOLGW1MTlu4c57CT1xA==",
-      "requires": {
-        "json-buffer": "3.0.0"
-      }
-    },
-    "killable": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/killable/-/killable-1.0.0.tgz",
-      "integrity": "sha1-2ouEvUfeU5WHj5XWTQLyRJ/gXms="
-    },
-    "kind-of": {
-      "version": "3.2.2",
-      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-      "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-      "requires": {
-        "is-buffer": "1.1.6"
-      }
-    },
-    "labeled-stream-splicer": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/labeled-stream-splicer/-/labeled-stream-splicer-2.0.1.tgz",
-      "integrity": "sha512-MC94mHZRvJ3LfykJlTUipBqenZz1pacOZEMhhQ8dMGcDHs0SBE5GbsavUXV7YtP3icBW17W0Zy1I0lfASmo9Pg==",
-      "dev": true,
-      "requires": {
-        "inherits": "2.0.3",
-        "isarray": "2.0.4",
-        "stream-splicer": "2.0.0"
-      },
-      "dependencies": {
-        "isarray": {
-          "version": "2.0.4",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.4.tgz",
-          "integrity": "sha512-GMxXOiUirWg1xTKRipM0Ek07rX+ubx4nNVElTJdNLYmNO/2YrDkgJGw9CljXn+r4EWiDQg/8lsRdHyg2PJuUaA==",
-          "dev": true
-        }
-      }
-    },
-    "lazy-cache": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz",
-      "integrity": "sha1-odePw6UEdMuAhF07O24dpJpEbo4="
-    },
-    "lcid": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz",
-      "integrity": "sha1-MIrMr6C8SDo4Z7S28rlQYlHRuDU=",
-      "requires": {
-        "invert-kv": "1.0.0"
-      }
-    },
-    "levn": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
-      "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=",
-      "requires": {
-        "prelude-ls": "1.1.2",
-        "type-check": "0.3.2"
-      }
-    },
-    "lexical-scope": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/lexical-scope/-/lexical-scope-1.2.0.tgz",
-      "integrity": "sha1-/Ope3HBKSzqHls3KQZw6CvryLfQ=",
-      "dev": true,
-      "requires": {
-        "astw": "2.2.0"
-      }
-    },
-    "lie": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/lie/-/lie-3.1.1.tgz",
-      "integrity": "sha1-mkNrLMd0bKWd56QfpGmz77dr2H4=",
-      "requires": {
-        "immediate": "3.0.6"
-      }
-    },
-    "listr": {
-      "version": "0.13.0",
-      "resolved": "https://registry.npmjs.org/listr/-/listr-0.13.0.tgz",
-      "integrity": "sha1-ILsLowuuZg7oTMBQPfS+PVYjiH0=",
-      "requires": {
-        "chalk": "1.1.3",
-        "cli-truncate": "0.2.1",
-        "figures": "1.7.0",
-        "indent-string": "2.1.0",
-        "is-observable": "0.2.0",
-        "is-promise": "2.1.0",
-        "is-stream": "1.1.0",
-        "listr-silent-renderer": "1.1.1",
-        "listr-update-renderer": "0.4.0",
-        "listr-verbose-renderer": "0.4.1",
-        "log-symbols": "1.0.2",
-        "log-update": "1.0.2",
-        "ora": "0.2.3",
-        "p-map": "1.2.0",
-        "rxjs": "5.4.2",
-        "stream-to-observable": "0.2.0",
-        "strip-ansi": "3.0.1"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        },
-        "figures": {
-          "version": "1.7.0",
-          "resolved": "https://registry.npmjs.org/figures/-/figures-1.7.0.tgz",
-          "integrity": "sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4=",
-          "requires": {
-            "escape-string-regexp": "1.0.5",
-            "object-assign": "4.1.1"
-          }
-        },
-        "log-symbols": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-1.0.2.tgz",
-          "integrity": "sha1-N2/3tY6jCGoPCfrMdGF+ylAeGhg=",
-          "requires": {
-            "chalk": "1.1.3"
-          }
-        }
-      }
-    },
-    "listr-silent-renderer": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/listr-silent-renderer/-/listr-silent-renderer-1.1.1.tgz",
-      "integrity": "sha1-kktaN1cVN3C/Go4/v3S4u/P5JC4="
-    },
-    "listr-update-renderer": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/listr-update-renderer/-/listr-update-renderer-0.4.0.tgz",
-      "integrity": "sha1-NE2YDaLKLosUW6MFkI8yrj9MyKc=",
-      "requires": {
-        "chalk": "1.1.3",
-        "cli-truncate": "0.2.1",
-        "elegant-spinner": "1.0.1",
-        "figures": "1.7.0",
-        "indent-string": "3.2.0",
-        "log-symbols": "1.0.2",
-        "log-update": "1.0.2",
-        "strip-ansi": "3.0.1"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        },
-        "figures": {
-          "version": "1.7.0",
-          "resolved": "https://registry.npmjs.org/figures/-/figures-1.7.0.tgz",
-          "integrity": "sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4=",
-          "requires": {
-            "escape-string-regexp": "1.0.5",
-            "object-assign": "4.1.1"
-          }
-        },
-        "indent-string": {
-          "version": "3.2.0",
-          "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz",
-          "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok="
-        },
-        "log-symbols": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-1.0.2.tgz",
-          "integrity": "sha1-N2/3tY6jCGoPCfrMdGF+ylAeGhg=",
-          "requires": {
-            "chalk": "1.1.3"
-          }
-        }
-      }
-    },
-    "listr-verbose-renderer": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/listr-verbose-renderer/-/listr-verbose-renderer-0.4.1.tgz",
-      "integrity": "sha1-ggb0z21S3cWCfl/RSYng6WWTOjU=",
-      "requires": {
-        "chalk": "1.1.3",
-        "cli-cursor": "1.0.2",
-        "date-fns": "1.29.0",
-        "figures": "1.7.0"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        },
-        "cli-cursor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz",
-          "integrity": "sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc=",
-          "requires": {
-            "restore-cursor": "1.0.1"
-          }
-        },
-        "figures": {
-          "version": "1.7.0",
-          "resolved": "https://registry.npmjs.org/figures/-/figures-1.7.0.tgz",
-          "integrity": "sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4=",
-          "requires": {
-            "escape-string-regexp": "1.0.5",
-            "object-assign": "4.1.1"
-          }
-        },
-        "onetime": {
-          "version": "1.1.0",
-          "resolved": "http://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz",
-          "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k="
-        },
-        "restore-cursor": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz",
-          "integrity": "sha1-NGYfRohjJ/7SmRR5FSJS35LapUE=",
-          "requires": {
-            "exit-hook": "1.1.1",
-            "onetime": "1.1.0"
-          }
-        }
-      }
-    },
-    "load-json-file": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz",
-      "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=",
-      "requires": {
-        "graceful-fs": "4.1.11",
-        "parse-json": "2.2.0",
-        "pify": "2.3.0",
-        "pinkie-promise": "2.0.1",
-        "strip-bom": "2.0.0"
-      },
-      "dependencies": {
-        "pify": {
-          "version": "2.3.0",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw="
-        }
-      }
-    },
-    "loader-fs-cache": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/loader-fs-cache/-/loader-fs-cache-1.0.1.tgz",
-      "integrity": "sha1-VuC/CL2XCLJqdltoUJhAyN7J/bw=",
-      "requires": {
-        "find-cache-dir": "0.1.1",
-        "mkdirp": "0.5.1"
-      },
-      "dependencies": {
-        "find-cache-dir": {
-          "version": "0.1.1",
-          "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-0.1.1.tgz",
-          "integrity": "sha1-yN765XyKUqinhPnjHFfHQumToLk=",
-          "requires": {
-            "commondir": "1.0.1",
-            "mkdirp": "0.5.1",
-            "pkg-dir": "1.0.0"
-          }
-        },
-        "find-up": {
-          "version": "1.1.2",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz",
-          "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=",
-          "requires": {
-            "path-exists": "2.1.0",
-            "pinkie-promise": "2.0.1"
-          }
-        },
-        "path-exists": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz",
-          "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=",
-          "requires": {
-            "pinkie-promise": "2.0.1"
-          }
-        },
-        "pkg-dir": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-1.0.0.tgz",
-          "integrity": "sha1-ektQio1bstYp1EcFb/TpyTFM89Q=",
-          "requires": {
-            "find-up": "1.1.2"
-          }
-        }
-      }
-    },
-    "loader-runner": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.3.0.tgz",
-      "integrity": "sha1-9IKuqC1UPgeSFwDVpG7yb9rGuKI="
-    },
-    "loader-utils": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.1.0.tgz",
-      "integrity": "sha1-yYrvSIvM7aL/teLeZG1qdUQp9c0=",
-      "requires": {
-        "big.js": "3.2.0",
-        "emojis-list": "2.1.0",
-        "json5": "0.5.1"
-      }
-    },
-    "locate-path": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz",
-      "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=",
-      "requires": {
-        "p-locate": "2.0.0",
-        "path-exists": "3.0.0"
-      }
-    },
-    "lodash": {
-      "version": "4.17.5",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz",
-      "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw=="
-    },
-    "lodash._baseassign": {
-      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/lodash._baseassign/-/lodash._baseassign-3.2.0.tgz",
-      "integrity": "sha1-jDigmVAPIVrQnlnxci/QxSv+Ck4=",
-      "requires": {
-        "lodash._basecopy": "3.0.1",
-        "lodash.keys": "3.1.2"
-      }
-    },
-    "lodash._basecopy": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/lodash._basecopy/-/lodash._basecopy-3.0.1.tgz",
-      "integrity": "sha1-jaDmqHbPNEwK2KVIghEd08XHyjY="
-    },
-    "lodash._basecreate": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/lodash._basecreate/-/lodash._basecreate-3.0.3.tgz",
-      "integrity": "sha1-G8ZhYU2qf8MRt9A78WgGoCE8+CE=",
-      "dev": true
-    },
-    "lodash._bindcallback": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz",
-      "integrity": "sha1-5THCdkTPi1epnhftlbNcdIeJOS4="
-    },
-    "lodash._createassigner": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/lodash._createassigner/-/lodash._createassigner-3.1.1.tgz",
-      "integrity": "sha1-g4pbri/aymOsIt7o4Z+k5taXCxE=",
-      "requires": {
-        "lodash._bindcallback": "3.0.1",
-        "lodash._isiterateecall": "3.0.9",
-        "lodash.restparam": "3.6.1"
-      }
-    },
-    "lodash._getnative": {
-      "version": "3.9.1",
-      "resolved": "https://registry.npmjs.org/lodash._getnative/-/lodash._getnative-3.9.1.tgz",
-      "integrity": "sha1-VwvH3t5G1hzc3mh9ZdPuy6o6r/U="
-    },
-    "lodash._isiterateecall": {
-      "version": "3.0.9",
-      "resolved": "https://registry.npmjs.org/lodash._isiterateecall/-/lodash._isiterateecall-3.0.9.tgz",
-      "integrity": "sha1-UgOte6Ql+uhCRg5pbbnPPmqsBXw="
-    },
-    "lodash.assign": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz",
-      "integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc="
-    },
-    "lodash.camelcase": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
-      "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY="
-    },
-    "lodash.clonedeep": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz",
-      "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8="
-    },
-    "lodash.create": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/lodash.create/-/lodash.create-3.1.1.tgz",
-      "integrity": "sha1-1/KEnw29p+BGgruM1yqwIkYd6+c=",
-      "dev": true,
-      "requires": {
-        "lodash._baseassign": "3.2.0",
-        "lodash._basecreate": "3.0.3",
-        "lodash._isiterateecall": "3.0.9"
-      }
-    },
-    "lodash.defaults": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz",
-      "integrity": "sha1-0JF4cW/+pN3p5ft7N/bwgCJ0WAw="
-    },
-    "lodash.isarguments": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz",
-      "integrity": "sha1-L1c9hcaiQon/AGY7SRwdM4/zRYo="
-    },
-    "lodash.isarray": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/lodash.isarray/-/lodash.isarray-3.0.4.tgz",
-      "integrity": "sha1-eeTriMNqgSKvhvhEqpvNhRtfu1U="
-    },
-    "lodash.keys": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/lodash.keys/-/lodash.keys-3.1.2.tgz",
-      "integrity": "sha1-TbwEcrFWvlCgsoaFXRvQsMZWCYo=",
-      "requires": {
-        "lodash._getnative": "3.9.1",
-        "lodash.isarguments": "3.1.0",
-        "lodash.isarray": "3.0.4"
-      }
-    },
-    "lodash.memoize": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
-      "integrity": "sha1-vMbEmkKihA7Zl/Mj6tpezRguC/4="
-    },
-    "lodash.mergewith": {
-      "version": "4.6.1",
-      "resolved": "https://registry.npmjs.org/lodash.mergewith/-/lodash.mergewith-4.6.1.tgz",
-      "integrity": "sha512-eWw5r+PYICtEBgrBE5hhlT6aAa75f411bgDz/ZL2KZqYV03USvucsxcHUIlGTDTECs1eunpI7HOV7U+WLDvNdQ=="
-    },
-    "lodash.restparam": {
-      "version": "3.6.1",
-      "resolved": "https://registry.npmjs.org/lodash.restparam/-/lodash.restparam-3.6.1.tgz",
-      "integrity": "sha1-k2pOMJ7zMKdkXtQUWYbIWuWyCAU="
-    },
-    "lodash.tail": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/lodash.tail/-/lodash.tail-4.1.1.tgz",
-      "integrity": "sha1-0jM6NtnncXyK0vfKyv7HwytERmQ="
-    },
-    "lodash.uniq": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz",
-      "integrity": "sha1-0CJTc662Uq3BvILklFM5qEJ1R3M="
-    },
-    "log-symbols": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz",
-      "integrity": "sha512-VeIAFslyIerEJLXHziedo2basKbMKtTw3vfn5IzG0XTjhAVEJyNHnL2p7vc+wBDSdQuUpNw3M2u6xb9QsAY5Eg==",
-      "requires": {
-        "chalk": "2.4.0"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "log-update": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/log-update/-/log-update-1.0.2.tgz",
-      "integrity": "sha1-GZKfZMQJPS0ucHWh2tivWcKWuNE=",
-      "requires": {
-        "ansi-escapes": "1.4.0",
-        "cli-cursor": "1.0.2"
-      },
-      "dependencies": {
-        "ansi-escapes": {
-          "version": "1.4.0",
-          "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-1.4.0.tgz",
-          "integrity": "sha1-06ioOzGapneTZisT52HHkRQiMG4="
-        },
-        "cli-cursor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz",
-          "integrity": "sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc=",
-          "requires": {
-            "restore-cursor": "1.0.1"
-          }
-        },
-        "onetime": {
-          "version": "1.1.0",
-          "resolved": "http://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz",
-          "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k="
-        },
-        "restore-cursor": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz",
-          "integrity": "sha1-NGYfRohjJ/7SmRR5FSJS35LapUE=",
-          "requires": {
-            "exit-hook": "1.1.1",
-            "onetime": "1.1.0"
-          }
-        }
-      }
-    },
-    "log4js": {
-      "version": "2.5.3",
-      "resolved": "https://registry.npmjs.org/log4js/-/log4js-2.5.3.tgz",
-      "integrity": "sha512-YL/qpTxYtK0iWWbuKCrevDZz5lh+OjyHHD+mICqpjnYGKdNRBvPeh/1uYjkKUemT1CSO4wwLOwphWMpKAnD9kw==",
-      "dev": true,
-      "requires": {
-        "circular-json": "0.5.3",
-        "date-format": "1.2.0",
-        "debug": "3.1.0",
-        "semver": "5.5.0",
-        "streamroller": "0.7.0"
-      },
-      "dependencies": {
-        "circular-json": {
-          "version": "0.5.3",
-          "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.5.3.tgz",
-          "integrity": "sha512-YlxLOimeIoQGHnMe3kbf8qIV2Bj7uXLbljMPRguNT49GmSAzooNfS9EJ91rSJKbLBOOzM5agvtx0WyechZN/Hw==",
-          "dev": true
-        },
-        "debug": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-          "dev": true,
-          "requires": {
-            "ms": "2.0.0"
-          }
-        }
-      }
-    },
-    "loglevel": {
-      "version": "1.6.1",
-      "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.1.tgz",
-      "integrity": "sha1-4PyVEztu8nbNyIh82vJKpvFW+Po="
-    },
-    "loglevelnext": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/loglevelnext/-/loglevelnext-1.0.4.tgz",
-      "integrity": "sha512-V3N6LAJAiGwa/zjtvmgs2tyeiCJ23bGNhxXN8R+v7k6TNlSlTz40mIyZYdmO762eBnEFymn0Mhha+WuAhnwMBg=="
-    },
-    "lolex": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/lolex/-/lolex-1.6.0.tgz",
-      "integrity": "sha1-OpoCg0UqR9dDnnJzG54H1zhuSfY=",
-      "dev": true
-    },
-    "longest": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz",
-      "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc="
-    },
-    "loose-envify": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.3.1.tgz",
-      "integrity": "sha1-0aitM/qc4OcT1l/dCsi3SNR4yEg=",
-      "requires": {
-        "js-tokens": "3.0.2"
-      }
-    },
-    "loud-rejection": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz",
-      "integrity": "sha1-W0b4AUft7leIcPCG0Eghz5mOVR8=",
-      "requires": {
-        "currently-unhandled": "0.4.1",
-        "signal-exit": "3.0.2"
-      }
-    },
-    "lower-case": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-1.1.4.tgz",
-      "integrity": "sha1-miyr0bno4K6ZOkv31YdcOcQujqw="
-    },
-    "lowercase-keys": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
-      "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA=="
-    },
-    "lru-cache": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.2.tgz",
-      "integrity": "sha512-wgeVXhrDwAWnIF/yZARsFnMBtdFXOg1b8RIrhilp+0iDYN4mdQcNZElDZ0e4B64BhaxeQ5zN7PMyvu7we1kPeQ==",
-      "requires": {
-        "pseudomap": "1.0.2",
-        "yallist": "2.1.2"
-      }
-    },
-    "macaddress": {
-      "version": "0.2.8",
-      "resolved": "https://registry.npmjs.org/macaddress/-/macaddress-0.2.8.tgz",
-      "integrity": "sha1-WQTcU3w57G2+/q6QIycTX6hRHxI="
-    },
-    "magic-string": {
-      "version": "0.22.5",
-      "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz",
-      "integrity": "sha512-oreip9rJZkzvA8Qzk9HFs8fZGF/u7H/gtrE8EN6RjKJ9kh2HlC+yQ2QezifqTZfGyiuAV0dRv5a+y/8gBb1m9w==",
-      "requires": {
-        "vlq": "0.2.3"
-      }
-    },
-    "make-dir": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.2.0.tgz",
-      "integrity": "sha512-aNUAa4UMg/UougV25bbrU4ZaaKNjJ/3/xnvg/twpmKROPdKZPZ9wGgI0opdZzO8q/zUFawoUuixuOv33eZ61Iw==",
-      "requires": {
-        "pify": "3.0.0"
-      }
-    },
-    "map-cache": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz",
-      "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8="
-    },
-    "map-obj": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz",
-      "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0="
-    },
-    "map-visit": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz",
-      "integrity": "sha1-7Nyo8TFE5mDxtb1B8S80edmN+48=",
-      "requires": {
-        "object-visit": "1.0.1"
-      }
-    },
-    "math-expression-evaluator": {
-      "version": "1.2.17",
-      "resolved": "https://registry.npmjs.org/math-expression-evaluator/-/math-expression-evaluator-1.2.17.tgz",
-      "integrity": "sha1-3oGf282E3M2PrlnGrreWFbnSZqw="
-    },
-    "md5.js": {
-      "version": "1.3.4",
-      "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.4.tgz",
-      "integrity": "sha1-6b296UogpawYsENA/Fdk1bCdkB0=",
-      "requires": {
-        "hash-base": "3.0.4",
-        "inherits": "2.0.3"
-      }
-    },
-    "mdn-data": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-1.1.1.tgz",
-      "integrity": "sha512-2J5JENcb4yD5AzBI4ilTakiq2P9gHSsi4LOygnMu/bkchgTiA63AjsHAhDc+3U36AJHRfcz30Qv6Tb7i/Qsiew=="
-    },
-    "media-typer": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
-      "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
-    },
-    "mem": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/mem/-/mem-1.1.0.tgz",
-      "integrity": "sha1-Xt1StIXKHZAP5kiVUFOZoN+kX3Y=",
-      "requires": {
-        "mimic-fn": "1.2.0"
-      }
-    },
-    "mem-fs": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/mem-fs/-/mem-fs-1.1.3.tgz",
-      "integrity": "sha1-uK6NLj/Lb10/kWXBLUVRoGXZicw=",
-      "requires": {
-        "through2": "2.0.3",
-        "vinyl": "1.2.0",
-        "vinyl-file": "2.0.0"
-      }
-    },
-    "mem-fs-editor": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/mem-fs-editor/-/mem-fs-editor-3.0.2.tgz",
-      "integrity": "sha1-3Qpuryu4prN3QAZ6pUnrUwEFr58=",
-      "requires": {
-        "commondir": "1.0.1",
-        "deep-extend": "0.4.2",
-        "ejs": "2.5.8",
-        "glob": "7.1.2",
-        "globby": "6.1.0",
-        "mkdirp": "0.5.1",
-        "multimatch": "2.1.0",
-        "rimraf": "2.6.2",
-        "through2": "2.0.3",
-        "vinyl": "2.1.0"
-      },
-      "dependencies": {
-        "clone": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
-          "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18="
-        },
-        "clone-stats": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/clone-stats/-/clone-stats-1.0.0.tgz",
-          "integrity": "sha1-s3gt/4u1R04Yuba/D9/ngvh3doA="
-        },
-        "globby": {
-          "version": "6.1.0",
-          "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz",
-          "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=",
-          "requires": {
-            "array-union": "1.0.2",
-            "glob": "7.1.2",
-            "object-assign": "4.1.1",
-            "pify": "2.3.0",
-            "pinkie-promise": "2.0.1"
-          }
-        },
-        "pify": {
-          "version": "2.3.0",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw="
-        },
-        "replace-ext": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/replace-ext/-/replace-ext-1.0.0.tgz",
-          "integrity": "sha1-3mMSg3P8v3w8z6TeWkgMRaZ5WOs="
-        },
-        "vinyl": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-2.1.0.tgz",
-          "integrity": "sha1-Ah+cLPlR1rk5lDyJ617lrdT9kkw=",
-          "requires": {
-            "clone": "2.1.2",
-            "clone-buffer": "1.0.0",
-            "clone-stats": "1.0.0",
-            "cloneable-readable": "1.1.2",
-            "remove-trailing-separator": "1.1.0",
-            "replace-ext": "1.0.0"
-          }
-        }
-      }
-    },
-    "memory-fs": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.4.1.tgz",
-      "integrity": "sha1-OpoguEYlI+RHz7x+i7gO1me/xVI=",
-      "requires": {
-        "errno": "0.1.7",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "meow": {
-      "version": "3.7.0",
-      "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz",
-      "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=",
-      "requires": {
-        "camelcase-keys": "2.1.0",
-        "decamelize": "1.2.0",
-        "loud-rejection": "1.6.0",
-        "map-obj": "1.0.1",
-        "minimist": "1.2.0",
-        "normalize-package-data": "2.4.0",
-        "object-assign": "4.1.1",
-        "read-pkg-up": "1.0.1",
-        "redent": "1.0.0",
-        "trim-newlines": "1.0.0"
-      },
-      "dependencies": {
-        "minimist": {
-          "version": "1.2.0",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
-          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
-        }
-      }
-    },
-    "merge-descriptors": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
-      "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
-    },
-    "merge-options": {
-      "version": "0.0.64",
-      "resolved": "https://registry.npmjs.org/merge-options/-/merge-options-0.0.64.tgz",
-      "integrity": "sha1-y+BPWUppheryf3+PCyo6z2+dVi0=",
-      "requires": {
-        "is-plain-obj": "1.1.0"
-      }
-    },
-    "merge2": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.2.1.tgz",
-      "integrity": "sha512-wUqcG5pxrAcaFI1lkqkMnk3Q7nUxV/NWfpAFSeWUwG9TRODnBDCUHa75mi3o3vLWQ5N4CQERWCauSlP0I3ZqUg==",
-      "dev": true
-    },
-    "methods": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
-      "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
-    },
-    "micromatch": {
-      "version": "2.3.11",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-2.3.11.tgz",
-      "integrity": "sha1-hmd8l9FyCzY0MdBNDRUpO9OMFWU=",
-      "requires": {
-        "arr-diff": "2.0.0",
-        "array-unique": "0.2.1",
-        "braces": "1.8.5",
-        "expand-brackets": "0.1.5",
-        "extglob": "0.3.2",
-        "filename-regex": "2.0.1",
-        "is-extglob": "1.0.0",
-        "is-glob": "2.0.1",
-        "kind-of": "3.2.2",
-        "normalize-path": "2.1.1",
-        "object.omit": "2.0.1",
-        "parse-glob": "3.0.4",
-        "regex-cache": "0.4.4"
-      }
-    },
-    "miller-rabin": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz",
-      "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==",
-      "requires": {
-        "bn.js": "4.11.8",
-        "brorand": "1.1.0"
-      }
-    },
-    "mime": {
-      "version": "1.4.1",
-      "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz",
-      "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ=="
-    },
-    "mime-db": {
-      "version": "1.33.0",
-      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz",
-      "integrity": "sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ=="
-    },
-    "mime-types": {
-      "version": "2.1.18",
-      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz",
-      "integrity": "sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==",
-      "requires": {
-        "mime-db": "1.33.0"
-      }
-    },
-    "mimic-fn": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz",
-      "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ=="
-    },
-    "mimic-response": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.0.tgz",
-      "integrity": "sha1-3z02Uqc/3ta5sLJBRub9BSNTRY4="
-    },
-    "minimalistic-assert": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz",
-      "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A=="
-    },
-    "minimalistic-crypto-utils": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz",
-      "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo="
-    },
-    "minimatch": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
-      "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
-      "requires": {
-        "brace-expansion": "1.1.11"
-      }
-    },
-    "minimist": {
-      "version": "0.0.8",
-      "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
-      "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
-    },
-    "mississippi": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-2.0.0.tgz",
-      "integrity": "sha512-zHo8v+otD1J10j/tC+VNoGK9keCuByhKovAvdn74dmxJl9+mWHnx6EMsDN4lgRoMI/eYo2nchAxniIbUPb5onw==",
-      "requires": {
-        "concat-stream": "1.6.2",
-        "duplexify": "3.5.4",
-        "end-of-stream": "1.4.1",
-        "flush-write-stream": "1.0.3",
-        "from2": "2.3.0",
-        "parallel-transform": "1.1.0",
-        "pump": "2.0.1",
-        "pumpify": "1.4.0",
-        "stream-each": "1.2.2",
-        "through2": "2.0.3"
-      }
-    },
-    "mitt": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/mitt/-/mitt-1.1.2.tgz",
-      "integrity": "sha1-OA5hSA1qYVtmDwertg1R4KTkvtY="
-    },
-    "mixin-deep": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz",
-      "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==",
-      "requires": {
-        "for-in": "1.0.2",
-        "is-extendable": "1.0.1"
-      },
-      "dependencies": {
-        "is-extendable": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
-          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
-          "requires": {
-            "is-plain-object": "2.0.4"
-          }
-        }
-      }
-    },
-    "mixin-object": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz",
-      "integrity": "sha1-T7lJRB2rGCVA8f4DW6YOGUel5X4=",
-      "requires": {
-        "for-in": "0.1.8",
-        "is-extendable": "0.1.1"
-      },
-      "dependencies": {
-        "for-in": {
-          "version": "0.1.8",
-          "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz",
-          "integrity": "sha1-2Hc5COMSVhCZUrH9ubP6hn0ndeE="
-        }
-      }
-    },
-    "mkdirp": {
-      "version": "0.5.1",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
-      "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
-      "requires": {
-        "minimist": "0.0.8"
-      }
-    },
-    "mocha": {
-      "version": "3.4.2",
-      "resolved": "https://registry.npmjs.org/mocha/-/mocha-3.4.2.tgz",
-      "integrity": "sha1-0O9NMyEm2/GNDWQMmzgt1IvpdZQ=",
-      "dev": true,
-      "requires": {
-        "browser-stdout": "1.3.0",
-        "commander": "2.9.0",
-        "debug": "2.6.0",
-        "diff": "3.2.0",
-        "escape-string-regexp": "1.0.5",
-        "glob": "7.1.1",
-        "growl": "1.9.2",
-        "json3": "3.3.2",
-        "lodash.create": "3.1.1",
-        "mkdirp": "0.5.1",
-        "supports-color": "3.1.2"
-      },
-      "dependencies": {
-        "commander": {
-          "version": "2.9.0",
-          "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz",
-          "integrity": "sha1-nJkJQXbhIkDLItbFFGCYQA/g99Q=",
-          "dev": true,
-          "requires": {
-            "graceful-readlink": "1.0.1"
-          }
-        },
-        "debug": {
-          "version": "2.6.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.0.tgz",
-          "integrity": "sha1-vFlryr52F/Edn6FTYe3tVgi4SZs=",
-          "dev": true,
-          "requires": {
-            "ms": "0.7.2"
-          }
-        },
-        "diff": {
-          "version": "3.2.0",
-          "resolved": "https://registry.npmjs.org/diff/-/diff-3.2.0.tgz",
-          "integrity": "sha1-yc45Okt8vQsFinJck98pkCeGj/k=",
-          "dev": true
-        },
-        "glob": {
-          "version": "7.1.1",
-          "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.1.tgz",
-          "integrity": "sha1-gFIR3wT6rxxjo2ADBs31reULLsg=",
-          "dev": true,
-          "requires": {
-            "fs.realpath": "1.0.0",
-            "inflight": "1.0.6",
-            "inherits": "2.0.3",
-            "minimatch": "3.0.4",
-            "once": "1.4.0",
-            "path-is-absolute": "1.0.1"
-          }
-        },
-        "ms": {
-          "version": "0.7.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz",
-          "integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U=",
-          "dev": true
-        },
-        "supports-color": {
-          "version": "3.1.2",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.1.2.tgz",
-          "integrity": "sha1-cqJiiU2dQIuVbKBf83su2KbiotU=",
-          "dev": true,
-          "requires": {
-            "has-flag": "1.0.0"
-          }
-        }
-      }
-    },
-    "mocha-teamcity-reporter": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/mocha-teamcity-reporter/-/mocha-teamcity-reporter-1.1.1.tgz",
-      "integrity": "sha1-aW5ns9PTv3Iiw2CPNSNTf4VBFDk=",
-      "dev": true,
-      "requires": {
-        "mocha": "3.4.2"
-      }
-    },
-    "module-deps": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/module-deps/-/module-deps-4.1.1.tgz",
-      "integrity": "sha1-IyFYM/HaE/1gbMuAh7RIUty4If0=",
-      "dev": true,
-      "requires": {
-        "JSONStream": "1.3.2",
-        "browser-resolve": "1.11.2",
-        "cached-path-relative": "1.0.1",
-        "concat-stream": "1.5.2",
-        "defined": "1.0.0",
-        "detective": "4.7.1",
-        "duplexer2": "0.1.4",
-        "inherits": "2.0.3",
-        "parents": "1.0.1",
-        "readable-stream": "2.3.6",
-        "resolve": "1.7.1",
-        "stream-combiner2": "1.1.1",
-        "subarg": "1.0.0",
-        "through2": "2.0.3",
-        "xtend": "4.0.1"
-      },
-      "dependencies": {
-        "concat-stream": {
-          "version": "1.5.2",
-          "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz",
-          "integrity": "sha1-cIl4Yk2FavQaWnQd790mHadSwmY=",
-          "dev": true,
-          "requires": {
-            "inherits": "2.0.3",
-            "readable-stream": "2.0.6",
-            "typedarray": "0.0.6"
-          },
-          "dependencies": {
-            "readable-stream": {
-              "version": "2.0.6",
-              "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz",
-              "integrity": "sha1-j5A0HmilPMySh4jaz80Rs265t44=",
-              "dev": true,
-              "requires": {
-                "core-util-is": "1.0.2",
-                "inherits": "2.0.3",
-                "isarray": "1.0.0",
-                "process-nextick-args": "1.0.7",
-                "string_decoder": "0.10.31",
-                "util-deprecate": "1.0.2"
-              }
-            }
-          }
-        },
-        "process-nextick-args": {
-          "version": "1.0.7",
-          "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
-          "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=",
-          "dev": true
-        },
-        "string_decoder": {
-          "version": "0.10.31",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
-          "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=",
-          "dev": true
-        }
-      }
-    },
-    "move-concurrently": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz",
-      "integrity": "sha1-viwAX9oy4LKa8fBdfEszIUxwH5I=",
-      "requires": {
-        "aproba": "1.2.0",
-        "copy-concurrently": "1.0.5",
-        "fs-write-stream-atomic": "1.0.10",
-        "mkdirp": "0.5.1",
-        "rimraf": "2.6.2",
-        "run-queue": "1.0.3"
-      }
-    },
-    "ms": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-      "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
-    },
-    "multicast-dns": {
-      "version": "6.2.3",
-      "resolved": "https://registry.npmjs.org/multicast-dns/-/multicast-dns-6.2.3.tgz",
-      "integrity": "sha512-ji6J5enbMyGRHIAkAOu3WdV8nggqviKCEKtXcOqfphZZtQrmHKycfynJ2V7eVPUA4NhJ6V7Wf4TmGbTwKE9B6g==",
-      "requires": {
-        "dns-packet": "1.3.1",
-        "thunky": "1.0.2"
-      }
-    },
-    "multicast-dns-service-types": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz",
-      "integrity": "sha1-iZ8R2WhuXgXLkbNdXw5jt3PPyQE="
-    },
-    "multimatch": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/multimatch/-/multimatch-2.1.0.tgz",
-      "integrity": "sha1-nHkGoi+0wCkZ4vX3UWG0zb1LKis=",
-      "requires": {
-        "array-differ": "1.0.0",
-        "array-union": "1.0.2",
-        "arrify": "1.0.1",
-        "minimatch": "3.0.4"
-      }
-    },
-    "mute-stream": {
-      "version": "0.0.7",
-      "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz",
-      "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s="
-    },
-    "nan": {
-      "version": "2.10.0",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz",
-      "integrity": "sha512-bAdJv7fBLhWC+/Bls0Oza+mvTaNQtP+1RyhhhvD95pgUJz6XM5IzgmxOkItJ9tkoCiplvAnXI1tNmmUD/eScyA=="
-    },
-    "nanomatch": {
-      "version": "1.2.9",
-      "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.9.tgz",
-      "integrity": "sha512-n8R9bS8yQ6eSXaV6jHUpKzD8gLsin02w1HSFiegwrs9E098Ylhw5jdyKPaYqvHknHaSCKTPp7C8dGCQ0q9koXA==",
-      "requires": {
-        "arr-diff": "4.0.0",
-        "array-unique": "0.3.2",
-        "define-property": "2.0.2",
-        "extend-shallow": "3.0.2",
-        "fragment-cache": "0.2.1",
-        "is-odd": "2.0.0",
-        "is-windows": "1.0.2",
-        "kind-of": "6.0.2",
-        "object.pick": "1.3.0",
-        "regex-not": "1.0.2",
-        "snapdragon": "0.8.2",
-        "to-regex": "3.0.2"
-      },
-      "dependencies": {
-        "arr-diff": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
-          "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA="
-        },
-        "array-unique": {
-          "version": "0.3.2",
-          "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
-          "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg="
-        },
-        "define-property": {
-          "version": "2.0.2",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
-          "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
-          "requires": {
-            "is-descriptor": "1.0.2",
-            "isobject": "3.0.1"
-          }
-        },
-        "extend-shallow": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
-          "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
-          "requires": {
-            "assign-symbols": "1.0.0",
-            "is-extendable": "1.0.1"
-          }
-        },
-        "is-accessor-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-data-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-descriptor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
-          "requires": {
-            "is-accessor-descriptor": "1.0.0",
-            "is-data-descriptor": "1.0.0",
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-extendable": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
-          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
-          "requires": {
-            "is-plain-object": "2.0.4"
-          }
-        },
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        },
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
-        }
-      }
-    },
-    "native-promise-only": {
-      "version": "0.8.1",
-      "resolved": "https://registry.npmjs.org/native-promise-only/-/native-promise-only-0.8.1.tgz",
-      "integrity": "sha1-IKMYwwy0X3H+et+/eyHJnBRy7xE=",
-      "dev": true
-    },
-    "natural-compare": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
-      "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc="
-    },
-    "natural-compare-lite": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/natural-compare-lite/-/natural-compare-lite-1.4.0.tgz",
-      "integrity": "sha1-F7CVgZiJef3a/gIB6TG6kzyWy7Q="
-    },
-    "negotiator": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
-      "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk="
-    },
-    "neo-async": {
-      "version": "2.5.1",
-      "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.5.1.tgz",
-      "integrity": "sha512-3KL3fvuRkZ7s4IFOMfztb7zJp3QaVWnBeGoJlgB38XnCRPj/0tLzzLG5IB8NYOHbJ8g8UGrgZv44GLDk6CxTxA=="
-    },
-    "nice-try": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.4.tgz",
-      "integrity": "sha512-2NpiFHqC87y/zFke0fC0spBXL3bBsoh/p5H1EFhshxjCR5+0g2d6BiXbUFz9v1sAcxsk2htp2eQnNIci2dIYcA=="
-    },
-    "no-case": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/no-case/-/no-case-2.3.2.tgz",
-      "integrity": "sha512-rmTZ9kz+f3rCvK2TD1Ue/oZlns7OGoIWP4fc3llxxRXlOkHKoWPPWJOfFYpITabSow43QJbRIoHQXtt10VldyQ==",
-      "requires": {
-        "lower-case": "1.1.4"
-      }
-    },
-    "node-dir": {
-      "version": "0.1.8",
-      "resolved": "https://registry.npmjs.org/node-dir/-/node-dir-0.1.8.tgz",
-      "integrity": "sha1-VfuN62mQcHB/tn+RpGDwRIKUx30="
-    },
-    "node-fetch": {
-      "version": "1.7.3",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz",
-      "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==",
-      "dev": true,
-      "requires": {
-        "encoding": "0.1.12",
-        "is-stream": "1.1.0"
-      }
-    },
-    "node-forge": {
-      "version": "0.7.1",
-      "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.1.tgz",
-      "integrity": "sha1-naYR6giYL0uUIGs760zJZl8gwwA="
-    },
-    "node-gyp": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-3.6.2.tgz",
-      "integrity": "sha1-m/vlRWIoYoSDjnUOrAUpWFP6HGA=",
-      "requires": {
-        "fstream": "1.0.11",
-        "glob": "7.1.2",
-        "graceful-fs": "4.1.11",
-        "minimatch": "3.0.4",
-        "mkdirp": "0.5.1",
-        "nopt": "3.0.6",
-        "npmlog": "4.1.2",
-        "osenv": "0.1.5",
-        "request": "2.79.0",
-        "rimraf": "2.6.2",
-        "semver": "5.3.0",
-        "tar": "2.2.1",
-        "which": "1.3.0"
-      },
-      "dependencies": {
-        "semver": {
-          "version": "5.3.0",
-          "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz",
-          "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8="
-        }
-      }
-    },
-    "node-libs-browser": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.1.0.tgz",
-      "integrity": "sha512-5AzFzdoIMb89hBGMZglEegffzgRg+ZFoUmisQ8HI4j1KDdpx13J0taNp2y9xPbur6W61gepGDDotGBVQ7mfUCg==",
-      "requires": {
-        "assert": "1.4.1",
-        "browserify-zlib": "0.2.0",
-        "buffer": "4.9.1",
-        "console-browserify": "1.1.0",
-        "constants-browserify": "1.0.0",
-        "crypto-browserify": "3.12.0",
-        "domain-browser": "1.2.0",
-        "events": "1.1.1",
-        "https-browserify": "1.0.0",
-        "os-browserify": "0.3.0",
-        "path-browserify": "0.0.0",
-        "process": "0.11.10",
-        "punycode": "1.4.1",
-        "querystring-es3": "0.2.1",
-        "readable-stream": "2.3.6",
-        "stream-browserify": "2.0.1",
-        "stream-http": "2.8.1",
-        "string_decoder": "1.1.1",
-        "timers-browserify": "2.0.9",
-        "tty-browserify": "0.0.0",
-        "url": "0.11.0",
-        "util": "0.10.3",
-        "vm-browserify": "0.0.4"
-      },
-      "dependencies": {
-        "punycode": {
-          "version": "1.4.1",
-          "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
-          "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
-        }
-      }
-    },
-    "node-sass": {
-      "version": "4.8.3",
-      "resolved": "https://registry.npmjs.org/node-sass/-/node-sass-4.8.3.tgz",
-      "integrity": "sha512-tfFWhUsCk/Y19zarDcPo5xpj+IW3qCfOjVdHtYeG6S1CKbQOh1zqylnQK6cV3z9k80yxAnFX9Y+a9+XysDhhfg==",
-      "requires": {
-        "async-foreach": "0.1.3",
-        "chalk": "1.1.3",
-        "cross-spawn": "3.0.1",
-        "gaze": "1.1.2",
-        "get-stdin": "4.0.1",
-        "glob": "7.1.2",
-        "in-publish": "2.0.0",
-        "lodash.assign": "4.2.0",
-        "lodash.clonedeep": "4.5.0",
-        "lodash.mergewith": "4.6.1",
-        "meow": "3.7.0",
-        "mkdirp": "0.5.1",
-        "nan": "2.10.0",
-        "node-gyp": "3.6.2",
-        "npmlog": "4.1.2",
-        "request": "2.79.0",
-        "sass-graph": "2.2.4",
-        "stdout-stream": "1.4.0",
-        "true-case-path": "1.0.2"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        },
-        "cross-spawn": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-3.0.1.tgz",
-          "integrity": "sha1-ElYDfsufDF9549bvE14wdwGEuYI=",
-          "requires": {
-            "lru-cache": "4.1.2",
-            "which": "1.3.0"
-          }
-        }
-      }
-    },
-    "nomnom": {
-      "version": "1.8.1",
-      "resolved": "https://registry.npmjs.org/nomnom/-/nomnom-1.8.1.tgz",
-      "integrity": "sha1-IVH3Ikcrp55Qp2/BJbuMjy5Nwqc=",
-      "requires": {
-        "chalk": "0.4.0",
-        "underscore": "1.6.0"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-1.0.0.tgz",
-          "integrity": "sha1-yxAt8cVvUSPquLZ817mAJ6AnkXg="
-        },
-        "chalk": {
-          "version": "0.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-0.4.0.tgz",
-          "integrity": "sha1-UZmj3c0MHv4jvAjBsCewYXbgxk8=",
-          "requires": {
-            "ansi-styles": "1.0.0",
-            "has-color": "0.1.7",
-            "strip-ansi": "0.1.1"
-          }
-        },
-        "strip-ansi": {
-          "version": "0.1.1",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.1.1.tgz",
-          "integrity": "sha1-OeipjQRNFQZgq+SmgIrPcLt7yZE="
-        }
-      }
-    },
-    "nopt": {
-      "version": "3.0.6",
-      "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz",
-      "integrity": "sha1-xkZdvwirzU2zWTF/eaxopkayj/k=",
-      "requires": {
-        "abbrev": "1.1.1"
-      }
-    },
-    "normalize-package-data": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
-      "integrity": "sha512-9jjUFbTPfEy3R/ad/2oNbKtW9Hgovl5O1FvFWKkKblNXoN/Oou6+9+KKohPK13Yc3/TyunyWhJp6gvRNR/PPAw==",
-      "requires": {
-        "hosted-git-info": "2.6.0",
-        "is-builtin-module": "1.0.0",
-        "semver": "5.5.0",
-        "validate-npm-package-license": "3.0.3"
-      }
-    },
-    "normalize-path": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz",
-      "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=",
-      "requires": {
-        "remove-trailing-separator": "1.1.0"
-      }
-    },
-    "normalize-range": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/normalize-range/-/normalize-range-0.1.2.tgz",
-      "integrity": "sha1-LRDAa9/TEuqXd2laTShDlFa3WUI="
-    },
-    "normalize-url": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-1.9.1.tgz",
-      "integrity": "sha1-LMDWazHqIwNkWENuNiDYWVTGbDw=",
-      "requires": {
-        "object-assign": "4.1.1",
-        "prepend-http": "1.0.4",
-        "query-string": "4.3.4",
-        "sort-keys": "1.1.2"
-      }
-    },
-    "npm-run-path": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz",
-      "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=",
-      "requires": {
-        "path-key": "2.0.1"
-      }
-    },
-    "npmlog": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
-      "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
-      "requires": {
-        "are-we-there-yet": "1.1.4",
-        "console-control-strings": "1.1.0",
-        "gauge": "2.7.4",
-        "set-blocking": "2.0.0"
-      }
-    },
-    "nth-check": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz",
-      "integrity": "sha1-mSms32KPwsQQmN6rgqxYDPFJquQ=",
-      "requires": {
-        "boolbase": "1.0.0"
-      }
-    },
-    "null-check": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/null-check/-/null-check-1.0.0.tgz",
-      "integrity": "sha1-l33/1xdgErnsMNKjnbXPcqBDnt0=",
-      "dev": true
-    },
-    "num2fraction": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/num2fraction/-/num2fraction-1.2.2.tgz",
-      "integrity": "sha1-b2gragJ6Tp3fpFZM0lidHU5mnt4="
-    },
-    "number-is-nan": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
-      "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0="
-    },
-    "nvd3": {
-      "version": "1.8.6",
-      "resolved": "https://registry.npmjs.org/nvd3/-/nvd3-1.8.6.tgz",
-      "integrity": "sha1-LT66dL8zNjtRAevx0JPFmlOuc8Q="
-    },
-    "oauth-sign": {
-      "version": "0.8.2",
-      "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
-      "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
-    },
-    "object-assign": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
-      "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
-    },
-    "object-component": {
-      "version": "0.0.3",
-      "resolved": "https://registry.npmjs.org/object-component/-/object-component-0.0.3.tgz",
-      "integrity": "sha1-8MaapQ78lbhmwYb0AKM3acsvEpE="
-    },
-    "object-copy": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz",
-      "integrity": "sha1-fn2Fi3gb18mRpBupde04EnVOmYw=",
-      "requires": {
-        "copy-descriptor": "0.1.1",
-        "define-property": "0.2.5",
-        "kind-of": "3.2.2"
-      },
-      "dependencies": {
-        "define-property": {
-          "version": "0.2.5",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
-          "requires": {
-            "is-descriptor": "0.1.6"
-          }
-        }
-      }
-    },
-    "object-hash": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-1.3.0.tgz",
-      "integrity": "sha512-05KzQ70lSeGSrZJQXE5wNDiTkBJDlUT/myi6RX9dVIvz7a7Qh4oH93BQdiPMn27nldYvVQCKMUaM83AfizZlsQ=="
-    },
-    "object-keys": {
-      "version": "1.0.11",
-      "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.11.tgz",
-      "integrity": "sha1-xUYBd4rVYPEULODgG8yotW0TQm0="
-    },
-    "object-path": {
-      "version": "0.9.2",
-      "resolved": "https://registry.npmjs.org/object-path/-/object-path-0.9.2.tgz",
-      "integrity": "sha1-D9mnT8X60a45aLWGvaXGMr1sBaU="
-    },
-    "object-visit": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz",
-      "integrity": "sha1-95xEk68MU3e1n+OdOV5BBC3QRbs=",
-      "requires": {
-        "isobject": "3.0.1"
-      },
-      "dependencies": {
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        }
-      }
-    },
-    "object.assign": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz",
-      "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==",
-      "requires": {
-        "define-properties": "1.1.2",
-        "function-bind": "1.1.1",
-        "has-symbols": "1.0.0",
-        "object-keys": "1.0.11"
-      }
-    },
-    "object.getownpropertydescriptors": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz",
-      "integrity": "sha1-h1jIRvW0B62rDyNuCYbxSwUcqhY=",
-      "requires": {
-        "define-properties": "1.1.2",
-        "es-abstract": "1.11.0"
-      }
-    },
-    "object.omit": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/object.omit/-/object.omit-2.0.1.tgz",
-      "integrity": "sha1-Gpx0SCnznbuFjHbKNXmuKlTr0fo=",
-      "requires": {
-        "for-own": "0.1.5",
-        "is-extendable": "0.1.1"
-      }
-    },
-    "object.pick": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz",
-      "integrity": "sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c=",
-      "requires": {
-        "isobject": "3.0.1"
-      },
-      "dependencies": {
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        }
-      }
-    },
-    "object.values": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.0.4.tgz",
-      "integrity": "sha1-5STaCbT2b/Bd9FdUbscqyZ8TBpo=",
-      "requires": {
-        "define-properties": "1.1.2",
-        "es-abstract": "1.11.0",
-        "function-bind": "1.1.1",
-        "has": "1.0.1"
-      }
-    },
-    "obuf": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz",
-      "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg=="
-    },
-    "on-finished": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
-      "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
-      "requires": {
-        "ee-first": "1.1.1"
-      }
-    },
-    "on-headers": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.1.tgz",
-      "integrity": "sha1-ko9dD0cNSTQmUepnlLCFfBAGk/c="
-    },
-    "once": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-      "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
-      "requires": {
-        "wrappy": "1.0.2"
-      }
-    },
-    "onetime": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz",
-      "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=",
-      "requires": {
-        "mimic-fn": "1.2.0"
-      }
-    },
-    "opn": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/opn/-/opn-5.3.0.tgz",
-      "integrity": "sha512-bYJHo/LOmoTd+pfiYhfZDnf9zekVJrY+cnS2a5F2x+w5ppvTqObojTP7WiFG+kVZs9Inw+qQ/lw7TroWwhdd2g==",
-      "requires": {
-        "is-wsl": "1.1.0"
-      }
-    },
-    "optimist": {
-      "version": "0.6.1",
-      "resolved": "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz",
-      "integrity": "sha1-2j6nRob6IaGaERwybpDrFaAZZoY=",
-      "dev": true,
-      "requires": {
-        "minimist": "0.0.8",
-        "wordwrap": "0.0.2"
-      }
-    },
-    "optionator": {
-      "version": "0.8.2",
-      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz",
-      "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=",
-      "requires": {
-        "deep-is": "0.1.3",
-        "fast-levenshtein": "2.0.6",
-        "levn": "0.3.0",
-        "prelude-ls": "1.1.2",
-        "type-check": "0.3.2",
-        "wordwrap": "1.0.0"
-      },
-      "dependencies": {
-        "wordwrap": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
-          "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus="
-        }
-      }
-    },
-    "options": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/options/-/options-0.0.6.tgz",
-      "integrity": "sha1-7CLTEoBrtT5zF3Pnza788cZDEo8="
-    },
-    "ora": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/ora/-/ora-0.2.3.tgz",
-      "integrity": "sha1-N1J9Igrc1Tw5tzVx11QVbV22V6Q=",
-      "requires": {
-        "chalk": "1.1.3",
-        "cli-cursor": "1.0.2",
-        "cli-spinners": "0.1.2",
-        "object-assign": "4.1.1"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        },
-        "cli-cursor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz",
-          "integrity": "sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc=",
-          "requires": {
-            "restore-cursor": "1.0.1"
-          }
-        },
-        "onetime": {
-          "version": "1.1.0",
-          "resolved": "http://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz",
-          "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k="
-        },
-        "restore-cursor": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz",
-          "integrity": "sha1-NGYfRohjJ/7SmRR5FSJS35LapUE=",
-          "requires": {
-            "exit-hook": "1.1.1",
-            "onetime": "1.1.0"
-          }
-        }
-      }
-    },
-    "original": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/original/-/original-1.0.0.tgz",
-      "integrity": "sha1-kUf5P6FpbQS+YeAb1QuurKZWvTs=",
-      "requires": {
-        "url-parse": "1.0.5"
-      },
-      "dependencies": {
-        "url-parse": {
-          "version": "1.0.5",
-          "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.0.5.tgz",
-          "integrity": "sha1-CFSGBCKv3P7+tsllxmLUgAFpkns=",
-          "requires": {
-            "querystringify": "0.0.4",
-            "requires-port": "1.0.0"
-          }
-        }
-      }
-    },
-    "os-browserify": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz",
-      "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc="
-    },
-    "os-homedir": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
-      "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M="
-    },
-    "os-locale": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz",
-      "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=",
-      "requires": {
-        "lcid": "1.0.0"
-      }
-    },
-    "os-tmpdir": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
-      "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ="
-    },
-    "osenv": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz",
-      "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==",
-      "requires": {
-        "os-homedir": "1.0.2",
-        "os-tmpdir": "1.0.2"
-      }
-    },
-    "outdent": {
-      "version": "0.5.0",
-      "resolved": "https://registry.npmjs.org/outdent/-/outdent-0.5.0.tgz",
-      "integrity": "sha512-/jHxFIzoMXdqPzTaCpFzAAWhpkSjZPF4Vsn6jAfNpmbH/ymsmd7Qc6VE9BGn0L6YMj6uwpQLxCECpus4ukKS9Q=="
-    },
-    "p-cancelable": {
-      "version": "0.4.1",
-      "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz",
-      "integrity": "sha512-HNa1A8LvB1kie7cERyy21VNeHb2CWJJYqyyC2o3klWFfMGlFmWv2Z7sFgZH8ZiaYL95ydToKTFVXgMV/Os0bBQ=="
-    },
-    "p-each-series": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-each-series/-/p-each-series-1.0.0.tgz",
-      "integrity": "sha1-kw89Et0fUOdDRFeiLNbwSsatf3E=",
-      "requires": {
-        "p-reduce": "1.0.0"
-      }
-    },
-    "p-finally": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
-      "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4="
-    },
-    "p-is-promise": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz",
-      "integrity": "sha1-nJRWmJ6fZYgBewQ01WCXZ1w9oF4="
-    },
-    "p-lazy": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-lazy/-/p-lazy-1.0.0.tgz",
-      "integrity": "sha1-7FPIAvLuOsKPFmzILQsrAt4nqDU="
-    },
-    "p-limit": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.2.0.tgz",
-      "integrity": "sha512-Y/OtIaXtUPr4/YpMv1pCL5L5ed0rumAaAeBSj12F+bSlMdys7i8oQF/GUJmfpTS/QoaRrS/k6pma29haJpsMng==",
-      "requires": {
-        "p-try": "1.0.0"
-      }
-    },
-    "p-locate": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz",
-      "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=",
-      "requires": {
-        "p-limit": "1.2.0"
-      }
-    },
-    "p-map": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/p-map/-/p-map-1.2.0.tgz",
-      "integrity": "sha512-r6zKACMNhjPJMTl8KcFH4li//gkrXWfbD6feV8l6doRHlzljFWGJ2AP6iKaCJXyZmAUMOPtvbW7EXkbWO/pLEA=="
-    },
-    "p-reduce": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-reduce/-/p-reduce-1.0.0.tgz",
-      "integrity": "sha1-GMKw3ZNqRpClKfgjH1ig/bakffo="
-    },
-    "p-timeout": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-2.0.1.tgz",
-      "integrity": "sha512-88em58dDVB/KzPEx1X0N3LwFfYZPyDc4B6eF38M1rk9VTZMbxXXgjugz8mmwpS9Ox4BDZ+t6t3QP5+/gazweIA==",
-      "requires": {
-        "p-finally": "1.0.0"
-      }
-    },
-    "p-try": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz",
-      "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M="
-    },
-    "pako": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.6.tgz",
-      "integrity": "sha512-lQe48YPsMJAig+yngZ87Lus+NF+3mtu7DVOBu6b/gHO1YpKwIj5AWjZ/TOS7i46HD/UixzWb1zeWDZfGZ3iYcg=="
-    },
-    "parallel-transform": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.1.0.tgz",
-      "integrity": "sha1-1BDwZbBdojCB/NEPKIVMKb2jOwY=",
-      "requires": {
-        "cyclist": "0.2.2",
-        "inherits": "2.0.3",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "param-case": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/param-case/-/param-case-2.1.1.tgz",
-      "integrity": "sha1-35T9jPZTHs915r75oIWPvHK+Ikc=",
-      "requires": {
-        "no-case": "2.3.2"
-      }
-    },
-    "parents": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/parents/-/parents-1.0.1.tgz",
-      "integrity": "sha1-/t1NK/GTp3dF/nHjcdc8MwfZx1E=",
-      "dev": true,
-      "requires": {
-        "path-platform": "0.11.15"
-      }
-    },
-    "parse-asn1": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz",
-      "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==",
-      "requires": {
-        "asn1.js": "4.10.1",
-        "browserify-aes": "1.2.0",
-        "create-hash": "1.2.0",
-        "evp_bytestokey": "1.0.3",
-        "pbkdf2": "3.0.14"
-      }
-    },
-    "parse-glob": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/parse-glob/-/parse-glob-3.0.4.tgz",
-      "integrity": "sha1-ssN2z7EfNVE7rdFz7wu246OIORw=",
-      "requires": {
-        "glob-base": "0.3.0",
-        "is-dotfile": "1.0.3",
-        "is-extglob": "1.0.0",
-        "is-glob": "2.0.1"
-      }
-    },
-    "parse-json": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz",
-      "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=",
-      "requires": {
-        "error-ex": "1.3.1"
-      }
-    },
-    "parse-passwd": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz",
-      "integrity": "sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY="
-    },
-    "parsejson": {
-      "version": "0.0.3",
-      "resolved": "https://registry.npmjs.org/parsejson/-/parsejson-0.0.3.tgz",
-      "integrity": "sha1-q343WfIJ7OmUN5c/fQ8fZK4OZKs=",
-      "requires": {
-        "better-assert": "1.0.2"
-      }
-    },
-    "parseqs": {
-      "version": "0.0.5",
-      "resolved": "https://registry.npmjs.org/parseqs/-/parseqs-0.0.5.tgz",
-      "integrity": "sha1-1SCKNzjkZ2bikbouoXNoSSGouJ0=",
-      "requires": {
-        "better-assert": "1.0.2"
-      }
-    },
-    "parseuri": {
-      "version": "0.0.5",
-      "resolved": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz",
-      "integrity": "sha1-gCBKUNTbt3m/3G6+J3jZDkvOMgo=",
-      "requires": {
-        "better-assert": "1.0.2"
-      }
-    },
-    "parseurl": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz",
-      "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M="
-    },
-    "pascalcase": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz",
-      "integrity": "sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ="
-    },
-    "path-browserify": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz",
-      "integrity": "sha1-oLhwcpquIUAFt9UDLsLLuw+0RRo="
-    },
-    "path-dirname": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/path-dirname/-/path-dirname-1.0.2.tgz",
-      "integrity": "sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA="
-    },
-    "path-exists": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz",
-      "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU="
-    },
-    "path-is-absolute": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
-    },
-    "path-is-inside": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz",
-      "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM="
-    },
-    "path-key": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz",
-      "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A="
-    },
-    "path-parse": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.5.tgz",
-      "integrity": "sha1-PBrfhx6pzWyUMbbqK9dKD/BVxME="
-    },
-    "path-platform": {
-      "version": "0.11.15",
-      "resolved": "https://registry.npmjs.org/path-platform/-/path-platform-0.11.15.tgz",
-      "integrity": "sha1-6GQhf3TDaFDwhSt43Hv31KVyG/I=",
-      "dev": true
-    },
-    "path-to-regexp": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
-      "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
-    },
-    "path-type": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz",
-      "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==",
-      "requires": {
-        "pify": "3.0.0"
-      }
-    },
-    "pathval": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz",
-      "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=",
-      "dev": true
-    },
-    "pbkdf2": {
-      "version": "3.0.14",
-      "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.14.tgz",
-      "integrity": "sha512-gjsZW9O34fm0R7PaLHRJmLLVfSoesxztjPjE9o6R+qtVJij90ltg1joIovN9GKrRW3t1PzhDDG3UMEMFfZ+1wA==",
-      "requires": {
-        "create-hash": "1.2.0",
-        "create-hmac": "1.1.7",
-        "ripemd160": "2.0.1",
-        "safe-buffer": "5.1.1",
-        "sha.js": "2.4.11"
-      }
-    },
-    "pify": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz",
-      "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY="
-    },
-    "pinkie": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz",
-      "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA="
-    },
-    "pinkie-promise": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz",
-      "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=",
-      "requires": {
-        "pinkie": "2.0.4"
-      }
-    },
-    "pkg-dir": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz",
-      "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=",
-      "requires": {
-        "find-up": "2.1.0"
-      }
-    },
-    "pluralize": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-4.0.0.tgz",
-      "integrity": "sha1-WbcIwcAZCi9pLxx2GMRGsFL9F2I="
-    },
-    "portfinder": {
-      "version": "1.0.13",
-      "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.13.tgz",
-      "integrity": "sha1-uzLs2HwnEErm7kS1o8y/Drsa7ek=",
-      "requires": {
-        "async": "1.5.2",
-        "debug": "2.6.9",
-        "mkdirp": "0.5.1"
-      },
-      "dependencies": {
-        "async": {
-          "version": "1.5.2",
-          "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
-          "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo="
-        }
-      }
-    },
-    "posix-character-classes": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz",
-      "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs="
-    },
-    "postcss": {
-      "version": "5.2.18",
-      "resolved": "https://registry.npmjs.org/postcss/-/postcss-5.2.18.tgz",
-      "integrity": "sha512-zrUjRRe1bpXKsX1qAJNJjqZViErVuyEkMTRrwu4ud4sbTtIBRmtaYDrHmcGgmrbsW3MHfmtIf+vJumgQn+PrXg==",
-      "requires": {
-        "chalk": "1.1.3",
-        "js-base64": "2.4.3",
-        "source-map": "0.5.7",
-        "supports-color": "3.2.3"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          },
-          "dependencies": {
-            "supports-color": {
-              "version": "2.0.0",
-              "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
-              "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
-            }
-          }
-        },
-        "supports-color": {
-          "version": "3.2.3",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-3.2.3.tgz",
-          "integrity": "sha1-ZawFBLOVQXHYpklGsq48u4pfVPY=",
-          "requires": {
-            "has-flag": "1.0.0"
-          }
-        }
-      }
-    },
-    "postcss-calc": {
-      "version": "5.3.1",
-      "resolved": "https://registry.npmjs.org/postcss-calc/-/postcss-calc-5.3.1.tgz",
-      "integrity": "sha1-d7rnypKK2FcW4v2kLyYb98HWW14=",
-      "requires": {
-        "postcss": "5.2.18",
-        "postcss-message-helpers": "2.0.0",
-        "reduce-css-calc": "1.3.0"
-      }
-    },
-    "postcss-colormin": {
-      "version": "2.2.2",
-      "resolved": "https://registry.npmjs.org/postcss-colormin/-/postcss-colormin-2.2.2.tgz",
-      "integrity": "sha1-ZjFBfV8OkJo9fsJrJMio0eT5bks=",
-      "requires": {
-        "colormin": "1.1.2",
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0"
-      }
-    },
-    "postcss-convert-values": {
-      "version": "2.6.1",
-      "resolved": "https://registry.npmjs.org/postcss-convert-values/-/postcss-convert-values-2.6.1.tgz",
-      "integrity": "sha1-u9hZPFwf0uPRwyK7kl3K6Nrk1i0=",
-      "requires": {
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0"
-      }
-    },
-    "postcss-discard-comments": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/postcss-discard-comments/-/postcss-discard-comments-2.0.4.tgz",
-      "integrity": "sha1-vv6J+v1bPazlzM5Rt2uBUUvgDj0=",
-      "requires": {
-        "postcss": "5.2.18"
-      }
-    },
-    "postcss-discard-duplicates": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/postcss-discard-duplicates/-/postcss-discard-duplicates-2.1.0.tgz",
-      "integrity": "sha1-uavye4isGIFYpesSq8riAmO5GTI=",
-      "requires": {
-        "postcss": "5.2.18"
-      }
-    },
-    "postcss-discard-empty": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/postcss-discard-empty/-/postcss-discard-empty-2.1.0.tgz",
-      "integrity": "sha1-0rS9nVztXr2Nyt52QMfXzX9PkrU=",
-      "requires": {
-        "postcss": "5.2.18"
-      }
-    },
-    "postcss-discard-overridden": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/postcss-discard-overridden/-/postcss-discard-overridden-0.1.1.tgz",
-      "integrity": "sha1-ix6vVU9ob7KIzYdMVWZ7CqNmjVg=",
-      "requires": {
-        "postcss": "5.2.18"
-      }
-    },
-    "postcss-discard-unused": {
-      "version": "2.2.3",
-      "resolved": "https://registry.npmjs.org/postcss-discard-unused/-/postcss-discard-unused-2.2.3.tgz",
-      "integrity": "sha1-vOMLLMWR/8Y0Mitfs0ZLbZNPRDM=",
-      "requires": {
-        "postcss": "5.2.18",
-        "uniqs": "2.0.0"
-      }
-    },
-    "postcss-filter-plugins": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/postcss-filter-plugins/-/postcss-filter-plugins-2.0.2.tgz",
-      "integrity": "sha1-bYWGJTTXNaxCDkqFgG4fXUKG2Ew=",
-      "requires": {
-        "postcss": "5.2.18",
-        "uniqid": "4.1.1"
-      }
-    },
-    "postcss-merge-idents": {
-      "version": "2.1.7",
-      "resolved": "https://registry.npmjs.org/postcss-merge-idents/-/postcss-merge-idents-2.1.7.tgz",
-      "integrity": "sha1-TFUwMTwI4dWzu/PSu8dH4njuonA=",
-      "requires": {
-        "has": "1.0.1",
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0"
-      }
-    },
-    "postcss-merge-longhand": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/postcss-merge-longhand/-/postcss-merge-longhand-2.0.2.tgz",
-      "integrity": "sha1-I9kM0Sewp3mUkVMyc5A0oaTz1lg=",
-      "requires": {
-        "postcss": "5.2.18"
-      }
-    },
-    "postcss-merge-rules": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/postcss-merge-rules/-/postcss-merge-rules-2.1.2.tgz",
-      "integrity": "sha1-0d9d+qexrMO+VT8OnhDofGG19yE=",
-      "requires": {
-        "browserslist": "1.7.7",
-        "caniuse-api": "1.6.1",
-        "postcss": "5.2.18",
-        "postcss-selector-parser": "2.2.3",
-        "vendors": "1.0.1"
-      }
-    },
-    "postcss-message-helpers": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/postcss-message-helpers/-/postcss-message-helpers-2.0.0.tgz",
-      "integrity": "sha1-pPL0+rbk/gAvCu0ABHjN9S+bpg4="
-    },
-    "postcss-minify-font-values": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/postcss-minify-font-values/-/postcss-minify-font-values-1.0.5.tgz",
-      "integrity": "sha1-S1jttWZB66fIR0qzUmyv17vey2k=",
-      "requires": {
-        "object-assign": "4.1.1",
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0"
-      }
-    },
-    "postcss-minify-gradients": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/postcss-minify-gradients/-/postcss-minify-gradients-1.0.5.tgz",
-      "integrity": "sha1-Xb2hE3NwP4PPtKPqOIHY11/15uE=",
-      "requires": {
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0"
-      }
-    },
-    "postcss-minify-params": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/postcss-minify-params/-/postcss-minify-params-1.2.2.tgz",
-      "integrity": "sha1-rSzgcTc7lDs9kwo/pZo1jCjW8fM=",
-      "requires": {
-        "alphanum-sort": "1.0.2",
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0",
-        "uniqs": "2.0.0"
-      }
-    },
-    "postcss-minify-selectors": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/postcss-minify-selectors/-/postcss-minify-selectors-2.1.1.tgz",
-      "integrity": "sha1-ssapjAByz5G5MtGkllCBFDEXNb8=",
-      "requires": {
-        "alphanum-sort": "1.0.2",
-        "has": "1.0.1",
-        "postcss": "5.2.18",
-        "postcss-selector-parser": "2.2.3"
-      }
-    },
-    "postcss-modules-extract-imports": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-1.1.0.tgz",
-      "integrity": "sha1-thTJcgvmgW6u41+zpfqh26agXds=",
-      "requires": {
-        "postcss": "6.0.21"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "postcss": {
-          "version": "6.0.21",
-          "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.21.tgz",
-          "integrity": "sha512-y/bKfbQz2Nn/QBC08bwvYUxEFOVGfPIUOTsJ2CK5inzlXW9SdYR1x4pEsG9blRAF/PX+wRNdOah+gx/hv4q7dw==",
-          "requires": {
-            "chalk": "2.4.0",
-            "source-map": "0.6.1",
-            "supports-color": "5.4.0"
-          }
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "postcss-modules-local-by-default": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-1.2.0.tgz",
-      "integrity": "sha1-99gMOYxaOT+nlkRmvRlQCn1hwGk=",
-      "requires": {
-        "css-selector-tokenizer": "0.7.0",
-        "postcss": "6.0.21"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "postcss": {
-          "version": "6.0.21",
-          "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.21.tgz",
-          "integrity": "sha512-y/bKfbQz2Nn/QBC08bwvYUxEFOVGfPIUOTsJ2CK5inzlXW9SdYR1x4pEsG9blRAF/PX+wRNdOah+gx/hv4q7dw==",
-          "requires": {
-            "chalk": "2.4.0",
-            "source-map": "0.6.1",
-            "supports-color": "5.4.0"
-          }
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "postcss-modules-scope": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-1.1.0.tgz",
-      "integrity": "sha1-1upkmUx5+XtipytCb75gVqGUu5A=",
-      "requires": {
-        "css-selector-tokenizer": "0.7.0",
-        "postcss": "6.0.21"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "postcss": {
-          "version": "6.0.21",
-          "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.21.tgz",
-          "integrity": "sha512-y/bKfbQz2Nn/QBC08bwvYUxEFOVGfPIUOTsJ2CK5inzlXW9SdYR1x4pEsG9blRAF/PX+wRNdOah+gx/hv4q7dw==",
-          "requires": {
-            "chalk": "2.4.0",
-            "source-map": "0.6.1",
-            "supports-color": "5.4.0"
-          }
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "postcss-modules-values": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-1.3.0.tgz",
-      "integrity": "sha1-7P+p1+GSUYOJ9CrQ6D9yrsRW6iA=",
-      "requires": {
-        "icss-replace-symbols": "1.1.0",
-        "postcss": "6.0.21"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "postcss": {
-          "version": "6.0.21",
-          "resolved": "https://registry.npmjs.org/postcss/-/postcss-6.0.21.tgz",
-          "integrity": "sha512-y/bKfbQz2Nn/QBC08bwvYUxEFOVGfPIUOTsJ2CK5inzlXW9SdYR1x4pEsG9blRAF/PX+wRNdOah+gx/hv4q7dw==",
-          "requires": {
-            "chalk": "2.4.0",
-            "source-map": "0.6.1",
-            "supports-color": "5.4.0"
-          }
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "postcss-normalize-charset": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/postcss-normalize-charset/-/postcss-normalize-charset-1.1.1.tgz",
-      "integrity": "sha1-757nEhLX/nWceO0WL2HtYrXLk/E=",
-      "requires": {
-        "postcss": "5.2.18"
-      }
-    },
-    "postcss-normalize-url": {
-      "version": "3.0.8",
-      "resolved": "https://registry.npmjs.org/postcss-normalize-url/-/postcss-normalize-url-3.0.8.tgz",
-      "integrity": "sha1-EI90s/L82viRov+j6kWSJ5/HgiI=",
-      "requires": {
-        "is-absolute-url": "2.1.0",
-        "normalize-url": "1.9.1",
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0"
-      }
-    },
-    "postcss-ordered-values": {
-      "version": "2.2.3",
-      "resolved": "https://registry.npmjs.org/postcss-ordered-values/-/postcss-ordered-values-2.2.3.tgz",
-      "integrity": "sha1-7sbCpntsQSqNsgQud/6NpD+VwR0=",
-      "requires": {
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0"
-      }
-    },
-    "postcss-prefix-selector": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/postcss-prefix-selector/-/postcss-prefix-selector-1.6.0.tgz",
-      "integrity": "sha1-tJWUnWOcYxRxRWSDJoUyFvPBCQA=",
-      "requires": {
-        "postcss": "5.2.18"
-      }
-    },
-    "postcss-reduce-idents": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/postcss-reduce-idents/-/postcss-reduce-idents-2.4.0.tgz",
-      "integrity": "sha1-wsbSDMlYKE9qv75j92Cb9AkFmtM=",
-      "requires": {
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0"
-      }
-    },
-    "postcss-reduce-initial": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/postcss-reduce-initial/-/postcss-reduce-initial-1.0.1.tgz",
-      "integrity": "sha1-aPgGlfBF0IJjqHmtJA343WT2ROo=",
-      "requires": {
-        "postcss": "5.2.18"
-      }
-    },
-    "postcss-reduce-transforms": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/postcss-reduce-transforms/-/postcss-reduce-transforms-1.0.4.tgz",
-      "integrity": "sha1-/3b02CEkN7McKYpC0uFEQCV3GuE=",
-      "requires": {
-        "has": "1.0.1",
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0"
-      }
-    },
-    "postcss-selector-parser": {
-      "version": "2.2.3",
-      "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-2.2.3.tgz",
-      "integrity": "sha1-+UN3iGBsPJrO4W/+jYsWKX8nu5A=",
-      "requires": {
-        "flatten": "1.0.2",
-        "indexes-of": "1.0.1",
-        "uniq": "1.0.1"
-      }
-    },
-    "postcss-svgo": {
-      "version": "2.1.6",
-      "resolved": "https://registry.npmjs.org/postcss-svgo/-/postcss-svgo-2.1.6.tgz",
-      "integrity": "sha1-tt8YqmE7Zm4TPwittSGcJoSsEI0=",
-      "requires": {
-        "is-svg": "2.1.0",
-        "postcss": "5.2.18",
-        "postcss-value-parser": "3.3.0",
-        "svgo": "0.7.2"
-      }
-    },
-    "postcss-unique-selectors": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/postcss-unique-selectors/-/postcss-unique-selectors-2.0.2.tgz",
-      "integrity": "sha1-mB1X0p3csz57Hf4f1DuGSfkzyh0=",
-      "requires": {
-        "alphanum-sort": "1.0.2",
-        "postcss": "5.2.18",
-        "uniqs": "2.0.0"
-      }
-    },
-    "postcss-value-parser": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.0.tgz",
-      "integrity": "sha1-h/OPnxj3dKSrTIojL1xc6IcqnRU="
-    },
-    "postcss-zindex": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/postcss-zindex/-/postcss-zindex-2.2.0.tgz",
-      "integrity": "sha1-0hCd3AVbka9n/EyzsCWUZjnSryI=",
-      "requires": {
-        "has": "1.0.1",
-        "postcss": "5.2.18",
-        "uniqs": "2.0.0"
-      }
-    },
-    "posthtml": {
-      "version": "0.11.3",
-      "resolved": "https://registry.npmjs.org/posthtml/-/posthtml-0.11.3.tgz",
-      "integrity": "sha512-quMHnDckt2DQ9lRi6bYLnuyBDnVzK+McHa8+ar4kTdYbWEo/92hREOu3h70ZirudOOp/my2b3r0m5YtxY52yrA==",
-      "requires": {
-        "object-assign": "4.1.1",
-        "posthtml-parser": "0.3.3",
-        "posthtml-render": "1.1.3"
-      }
-    },
-    "posthtml-parser": {
-      "version": "0.3.3",
-      "resolved": "https://registry.npmjs.org/posthtml-parser/-/posthtml-parser-0.3.3.tgz",
-      "integrity": "sha512-H/Z/yXGwl49A7hYQLV1iQ3h87NE0aZ/PMZhFwhw3lKeCAN+Ti4idrHvVvh4/GX10I7u77aQw+QB4vV5/Lzvv5A==",
-      "requires": {
-        "htmlparser2": "3.9.2",
-        "isobject": "2.1.0",
-        "object-assign": "4.1.1"
-      }
-    },
-    "posthtml-rename-id": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/posthtml-rename-id/-/posthtml-rename-id-1.0.4.tgz",
-      "integrity": "sha512-bxsGN02JGqcihc9eztWu8Qlj2P/If9sY0ckYmEL+6hqrWRvwJw4RvnXSnlKmjS4yDBcT4cSpJdMy+xsSuHDvZw==",
-      "requires": {
-        "escape-string-regexp": "1.0.5"
-      }
-    },
-    "posthtml-render": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/posthtml-render/-/posthtml-render-1.1.3.tgz",
-      "integrity": "sha512-aYvEMUxvKAv7D+ob2qlosyEL5qzsxCvauN/gjkRxr7fsW3+R2CJ1L3YHxx9kAjBJehmSwbNkSKzREdVfz1NWew=="
-    },
-    "posthtml-svg-mode": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/posthtml-svg-mode/-/posthtml-svg-mode-1.0.2.tgz",
-      "integrity": "sha512-yH4w0CULTg6v3YW5hVUY2z14R+11XWvtxMRqk30FDgxg0JWv+BhS9FLtKNIu858/1mrO1NcIC4heb6IezLAfHw==",
-      "requires": {
-        "merge-options": "0.0.64",
-        "posthtml": "0.9.2",
-        "posthtml-parser": "0.2.1",
-        "posthtml-render": "1.1.3"
-      },
-      "dependencies": {
-        "posthtml": {
-          "version": "0.9.2",
-          "resolved": "https://registry.npmjs.org/posthtml/-/posthtml-0.9.2.tgz",
-          "integrity": "sha1-9MBtufZ7Yf0XxOJW5+PZUVv3Jv0=",
-          "requires": {
-            "posthtml-parser": "0.2.1",
-            "posthtml-render": "1.1.3"
-          }
-        },
-        "posthtml-parser": {
-          "version": "0.2.1",
-          "resolved": "https://registry.npmjs.org/posthtml-parser/-/posthtml-parser-0.2.1.tgz",
-          "integrity": "sha1-NdUw3jhnQMK6JP8usvrznM3ycd0=",
-          "requires": {
-            "htmlparser2": "3.9.2",
-            "isobject": "2.1.0"
-          }
-        }
-      }
-    },
-    "preact": {
-      "version": "7.2.1",
-      "resolved": "https://registry.npmjs.org/preact/-/preact-7.2.1.tgz",
-      "integrity": "sha1-FZ4YkvYUmF5J6wqW/W5ti9+LvMU="
-    },
-    "prelude-ls": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
-      "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ="
-    },
-    "prepend-http": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz",
-      "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw="
-    },
-    "preserve": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/preserve/-/preserve-0.2.0.tgz",
-      "integrity": "sha1-gV7R9uvGWSb4ZbMQwHE7yzMVzks="
-    },
-    "prettier": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.12.1.tgz",
-      "integrity": "sha1-wa0g6APndJ+vkFpAnSNn4Gu+cyU="
-    },
-    "pretty-bytes": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-4.0.2.tgz",
-      "integrity": "sha1-sr+C5zUNZcbDOqlaqlpPYyf2HNk="
-    },
-    "pretty-error": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/pretty-error/-/pretty-error-2.1.1.tgz",
-      "integrity": "sha1-X0+HyPkeWuPzuoerTPXgOxoX8aM=",
-      "requires": {
-        "renderkid": "2.0.1",
-        "utila": "0.4.0"
-      }
-    },
-    "private": {
-      "version": "0.1.8",
-      "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz",
-      "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg=="
-    },
-    "process": {
-      "version": "0.11.10",
-      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
-      "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI="
-    },
-    "process-nextick-args": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
-      "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
-    },
-    "progress": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz",
-      "integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8="
-    },
-    "progress-bar-webpack-plugin": {
-      "version": "1.11.0",
-      "resolved": "https://registry.npmjs.org/progress-bar-webpack-plugin/-/progress-bar-webpack-plugin-1.11.0.tgz",
-      "integrity": "sha512-XT6r8strD6toU0ZVip25baJINo7uE4BD4H8d4vhOV4GIK5PvNNky8GYJ2wMmVoYP8eo/sSmtNWn0Vw7zWDDE3A==",
-      "requires": {
-        "chalk": "1.1.3",
-        "object.assign": "4.1.0",
-        "progress": "1.1.8"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        },
-        "progress": {
-          "version": "1.1.8",
-          "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz",
-          "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74="
-        }
-      }
-    },
-    "promise": {
-      "version": "7.3.1",
-      "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
-      "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==",
-      "requires": {
-        "asap": "2.0.6"
-      }
-    },
-    "promise-inflight": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
-      "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM="
-    },
-    "proxy-addr": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.3.tgz",
-      "integrity": "sha512-jQTChiCJteusULxjBp8+jftSQE5Obdl3k4cnmLA6WXtK6XFuWRnvVL7aCiBqaLPM8c4ph0S4tKna8XvmIwEnXQ==",
-      "requires": {
-        "forwarded": "0.1.2",
-        "ipaddr.js": "1.6.0"
-      }
-    },
-    "prr": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz",
-      "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY="
-    },
-    "pseudomap": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
-      "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM="
-    },
-    "public-encrypt": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.2.tgz",
-      "integrity": "sha512-4kJ5Esocg8X3h8YgJsKAuoesBgB7mqH3eowiDzMUPKiRDDE7E/BqqZD1hnTByIaAFiwAw246YEltSq7tdrOH0Q==",
-      "requires": {
-        "bn.js": "4.11.8",
-        "browserify-rsa": "4.0.1",
-        "create-hash": "1.2.0",
-        "parse-asn1": "5.1.1",
-        "randombytes": "2.0.6"
-      }
-    },
-    "pug": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/pug/-/pug-2.0.3.tgz",
-      "integrity": "sha1-ccuoJTfJWl6rftBGluQiH1Oqh44=",
-      "requires": {
-        "pug-code-gen": "2.0.1",
-        "pug-filters": "3.1.0",
-        "pug-lexer": "4.0.0",
-        "pug-linker": "3.0.5",
-        "pug-load": "2.0.11",
-        "pug-parser": "5.0.0",
-        "pug-runtime": "2.0.4",
-        "pug-strip-comments": "1.0.3"
-      }
-    },
-    "pug-attrs": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/pug-attrs/-/pug-attrs-2.0.3.tgz",
-      "integrity": "sha1-owlflw5kFR972tlX7vVftdeQXRU=",
-      "requires": {
-        "constantinople": "3.1.2",
-        "js-stringify": "1.0.2",
-        "pug-runtime": "2.0.4"
-      }
-    },
-    "pug-code-gen": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/pug-code-gen/-/pug-code-gen-2.0.1.tgz",
-      "integrity": "sha1-CVHsgyJddNjPxHan+Zolm199BQw=",
-      "requires": {
-        "constantinople": "3.1.2",
-        "doctypes": "1.1.0",
-        "js-stringify": "1.0.2",
-        "pug-attrs": "2.0.3",
-        "pug-error": "1.3.2",
-        "pug-runtime": "2.0.4",
-        "void-elements": "2.0.1",
-        "with": "5.1.1"
-      }
-    },
-    "pug-error": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/pug-error/-/pug-error-1.3.2.tgz",
-      "integrity": "sha1-U659nSm7A89WRJOgJhCfVMR/XyY="
-    },
-    "pug-filters": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/pug-filters/-/pug-filters-3.1.0.tgz",
-      "integrity": "sha1-JxZVVbwEwjbkqisDZiRt+gIbYm4=",
-      "requires": {
-        "clean-css": "4.1.11",
-        "constantinople": "3.1.2",
-        "jstransformer": "1.0.0",
-        "pug-error": "1.3.2",
-        "pug-walk": "1.1.7",
-        "resolve": "1.7.1",
-        "uglify-js": "2.8.29"
-      }
-    },
-    "pug-html-loader": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/pug-html-loader/-/pug-html-loader-1.1.0.tgz",
-      "integrity": "sha1-w5aGShAgQfZvYqmgGTBdWTMZydI=",
-      "requires": {
-        "loader-utils": "0.2.17",
-        "pug": "2.0.3"
-      },
-      "dependencies": {
-        "loader-utils": {
-          "version": "0.2.17",
-          "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-0.2.17.tgz",
-          "integrity": "sha1-+G5jdNQyBabmxg6RlvF8Apm/s0g=",
-          "requires": {
-            "big.js": "3.2.0",
-            "emojis-list": "2.1.0",
-            "json5": "0.5.1",
-            "object-assign": "4.1.1"
-          }
-        }
-      }
-    },
-    "pug-lexer": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/pug-lexer/-/pug-lexer-4.0.0.tgz",
-      "integrity": "sha1-IQwYRX7y4XYCQnQMXmR715TOwng=",
-      "requires": {
-        "character-parser": "2.2.0",
-        "is-expression": "3.0.0",
-        "pug-error": "1.3.2"
-      }
-    },
-    "pug-linker": {
-      "version": "3.0.5",
-      "resolved": "https://registry.npmjs.org/pug-linker/-/pug-linker-3.0.5.tgz",
-      "integrity": "sha1-npp65ABWgtAn3uuWsAD4juuDoC8=",
-      "requires": {
-        "pug-error": "1.3.2",
-        "pug-walk": "1.1.7"
-      }
-    },
-    "pug-load": {
-      "version": "2.0.11",
-      "resolved": "https://registry.npmjs.org/pug-load/-/pug-load-2.0.11.tgz",
-      "integrity": "sha1-5kjlftET/iwfRdV4WOorrWvAFSc=",
-      "requires": {
-        "object-assign": "4.1.1",
-        "pug-walk": "1.1.7"
-      }
-    },
-    "pug-loader": {
-      "version": "2.4.0",
-      "resolved": "https://registry.npmjs.org/pug-loader/-/pug-loader-2.4.0.tgz",
-      "integrity": "sha512-cD4bU2wmkZ1EEVyu0IfKOsh1F26KPva5oglO1Doc3knx8VpBIXmFHw16k9sITYIjQMCnRv1vb4vfQgy7VdR6eg==",
-      "requires": {
-        "loader-utils": "1.1.0",
-        "pug-walk": "1.1.7",
-        "resolve": "1.7.1"
-      }
-    },
-    "pug-parser": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/pug-parser/-/pug-parser-5.0.0.tgz",
-      "integrity": "sha1-45Stmz/KkxI5QK/4hcBuRKt+aOQ=",
-      "requires": {
-        "pug-error": "1.3.2",
-        "token-stream": "0.0.1"
-      }
-    },
-    "pug-runtime": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/pug-runtime/-/pug-runtime-2.0.4.tgz",
-      "integrity": "sha1-4XjhvaaKsujArPybztLFT9iM61g="
-    },
-    "pug-strip-comments": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/pug-strip-comments/-/pug-strip-comments-1.0.3.tgz",
-      "integrity": "sha1-8VWVkiBu3G+FMQ2s9K+0igJa9Z8=",
-      "requires": {
-        "pug-error": "1.3.2"
-      }
-    },
-    "pug-walk": {
-      "version": "1.1.7",
-      "resolved": "https://registry.npmjs.org/pug-walk/-/pug-walk-1.1.7.tgz",
-      "integrity": "sha1-wA1cUSi6xYBr7BXSt+fNq+QlMfM="
-    },
-    "pump": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz",
-      "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==",
-      "requires": {
-        "end-of-stream": "1.4.1",
-        "once": "1.4.0"
-      }
-    },
-    "pumpify": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.4.0.tgz",
-      "integrity": "sha512-2kmNR9ry+Pf45opRVirpNuIFotsxUGLaYqxIwuR77AYrYRMuFCz9eryHBS52L360O+NcR383CL4QYlMKPq4zYA==",
-      "requires": {
-        "duplexify": "3.5.4",
-        "inherits": "2.0.3",
-        "pump": "2.0.1"
-      }
-    },
-    "punycode": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.0.tgz",
-      "integrity": "sha1-X4Y+3Im5bbCQdLrXlHvwkFbKTn0="
-    },
-    "q": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz",
-      "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc="
-    },
-    "qjobs": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/qjobs/-/qjobs-1.2.0.tgz",
-      "integrity": "sha512-8YOJEHtxpySA3fFDyCRxA+UUV+fA+rTWnuWvylOK/NCjhY+b4ocCtmu8TtsWb+mYeU+GCHf/S66KZF/AsteKHg==",
-      "dev": true
-    },
-    "qs": {
-      "version": "6.3.2",
-      "resolved": "https://registry.npmjs.org/qs/-/qs-6.3.2.tgz",
-      "integrity": "sha1-51vV9uJoEioqDgvaYwslUMFmUCw="
-    },
-    "query-string": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/query-string/-/query-string-4.3.4.tgz",
-      "integrity": "sha1-u7aTucqRXCMlFbIosaArYJBD2+s=",
-      "requires": {
-        "object-assign": "4.1.1",
-        "strict-uri-encode": "1.1.0"
-      }
-    },
-    "querystring": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
-      "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
-    },
-    "querystring-es3": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz",
-      "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM="
-    },
-    "querystringify": {
-      "version": "0.0.4",
-      "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-0.0.4.tgz",
-      "integrity": "sha1-DPf4T5Rj/wrlHExLFC2VvjdyTZw="
-    },
-    "randomatic": {
-      "version": "1.1.7",
-      "resolved": "https://registry.npmjs.org/randomatic/-/randomatic-1.1.7.tgz",
-      "integrity": "sha512-D5JUjPyJbaJDkuAazpVnSfVkLlpeO3wDlPROTMLGKG1zMFNFRgrciKo1ltz/AzNTkqE0HzDx655QOL51N06how==",
-      "requires": {
-        "is-number": "3.0.0",
-        "kind-of": "4.0.0"
-      },
-      "dependencies": {
-        "is-number": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-          "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "kind-of": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz",
-          "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=",
-          "requires": {
-            "is-buffer": "1.1.6"
-          }
-        }
-      }
-    },
-    "randombytes": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.0.6.tgz",
-      "integrity": "sha512-CIQ5OFxf4Jou6uOKe9t1AOgqpeU5fd70A8NPdHSGeYXqXsPe6peOwI0cUl88RWZ6sP1vPMV3avd/R6cZ5/sP1A==",
-      "requires": {
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "randomfill": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz",
-      "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==",
-      "requires": {
-        "randombytes": "2.0.6",
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "range-parser": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz",
-      "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4="
-    },
-    "raw-body": {
-      "version": "2.3.2",
-      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz",
-      "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=",
-      "requires": {
-        "bytes": "3.0.0",
-        "http-errors": "1.6.2",
-        "iconv-lite": "0.4.19",
-        "unpipe": "1.0.0"
-      },
-      "dependencies": {
-        "depd": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz",
-          "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k="
-        },
-        "http-errors": {
-          "version": "1.6.2",
-          "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz",
-          "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=",
-          "requires": {
-            "depd": "1.1.1",
-            "inherits": "2.0.3",
-            "setprototypeof": "1.0.3",
-            "statuses": "1.4.0"
-          }
-        },
-        "iconv-lite": {
-          "version": "0.4.19",
-          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
-          "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
-        },
-        "setprototypeof": {
-          "version": "1.0.3",
-          "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz",
-          "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ="
-        }
-      }
-    },
-    "read-chunk": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-2.1.0.tgz",
-      "integrity": "sha1-agTAkoAF7Z1C4aasVgDhnLx/9lU=",
-      "requires": {
-        "pify": "3.0.0",
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "read-only-stream": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/read-only-stream/-/read-only-stream-2.0.0.tgz",
-      "integrity": "sha1-JyT9aoET1zdkrCiNQ4YnDB2/F/A=",
-      "dev": true,
-      "requires": {
-        "readable-stream": "2.3.6"
-      }
-    },
-    "read-pkg": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz",
-      "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=",
-      "requires": {
-        "load-json-file": "1.1.0",
-        "normalize-package-data": "2.4.0",
-        "path-type": "1.1.0"
-      },
-      "dependencies": {
-        "path-type": {
-          "version": "1.1.0",
-          "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz",
-          "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=",
-          "requires": {
-            "graceful-fs": "4.1.11",
-            "pify": "2.3.0",
-            "pinkie-promise": "2.0.1"
-          }
-        },
-        "pify": {
-          "version": "2.3.0",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw="
-        }
-      }
-    },
-    "read-pkg-up": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz",
-      "integrity": "sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=",
-      "requires": {
-        "find-up": "1.1.2",
-        "read-pkg": "1.1.0"
-      },
-      "dependencies": {
-        "find-up": {
-          "version": "1.1.2",
-          "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz",
-          "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=",
-          "requires": {
-            "path-exists": "2.1.0",
-            "pinkie-promise": "2.0.1"
-          }
-        },
-        "path-exists": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz",
-          "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=",
-          "requires": {
-            "pinkie-promise": "2.0.1"
-          }
-        }
-      }
-    },
-    "readable-stream": {
-      "version": "2.3.6",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
-      "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
-      "requires": {
-        "core-util-is": "1.0.2",
-        "inherits": "2.0.3",
-        "isarray": "1.0.0",
-        "process-nextick-args": "2.0.0",
-        "safe-buffer": "5.1.1",
-        "string_decoder": "1.1.1",
-        "util-deprecate": "1.0.2"
-      }
-    },
-    "readdirp": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.1.0.tgz",
-      "integrity": "sha1-TtCtBg3zBzMAxIRANz9y0cxkLXg=",
-      "requires": {
-        "graceful-fs": "4.1.11",
-        "minimatch": "3.0.4",
-        "readable-stream": "2.3.6",
-        "set-immediate-shim": "1.0.1"
-      }
-    },
-    "recast": {
-      "version": "0.14.7",
-      "resolved": "https://registry.npmjs.org/recast/-/recast-0.14.7.tgz",
-      "integrity": "sha512-/nwm9pkrcWagN40JeJhkPaRxiHXBRkXyRh/hgU088Z/v+qCy+zIHHY6bC6o7NaKAxPqtE6nD8zBH1LfU0/Wx6A==",
-      "requires": {
-        "ast-types": "0.11.3",
-        "esprima": "4.0.0",
-        "private": "0.1.8",
-        "source-map": "0.6.1"
-      },
-      "dependencies": {
-        "esprima": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz",
-          "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw=="
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        }
-      }
-    },
-    "rechoir": {
-      "version": "0.6.2",
-      "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz",
-      "integrity": "sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q=",
-      "requires": {
-        "resolve": "1.7.1"
-      }
-    },
-    "redent": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz",
-      "integrity": "sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94=",
-      "requires": {
-        "indent-string": "2.1.0",
-        "strip-indent": "1.0.1"
-      }
-    },
-    "reduce-css-calc": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/reduce-css-calc/-/reduce-css-calc-1.3.0.tgz",
-      "integrity": "sha1-dHyRTgSWFKTJz7umKYca0dKSdxY=",
-      "requires": {
-        "balanced-match": "0.4.2",
-        "math-expression-evaluator": "1.2.17",
-        "reduce-function-call": "1.0.2"
-      },
-      "dependencies": {
-        "balanced-match": {
-          "version": "0.4.2",
-          "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz",
-          "integrity": "sha1-yz8+PHMtwPAe5wtAPzAuYddwmDg="
-        }
-      }
-    },
-    "reduce-function-call": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/reduce-function-call/-/reduce-function-call-1.0.2.tgz",
-      "integrity": "sha1-WiAL+S4ON3UXUv5FsKszD9S2vpk=",
-      "requires": {
-        "balanced-match": "0.4.2"
-      },
-      "dependencies": {
-        "balanced-match": {
-          "version": "0.4.2",
-          "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz",
-          "integrity": "sha1-yz8+PHMtwPAe5wtAPzAuYddwmDg="
-        }
-      }
-    },
-    "regenerate": {
-      "version": "1.3.3",
-      "resolved": "https://registry.npmjs.org/regenerate/-/regenerate-1.3.3.tgz",
-      "integrity": "sha512-jVpo1GadrDAK59t/0jRx5VxYWQEDkkEKi6+HjE3joFVLfDOh9Xrdh0dF1eSq+BI/SwvTQ44gSscJ8N5zYL61sg=="
-    },
-    "regenerator-runtime": {
-      "version": "0.11.1",
-      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.1.tgz",
-      "integrity": "sha512-MguG95oij0fC3QV3URf4V2SDYGJhJnJGqvIIgdECeODCT98wSWDAJ94SSuVpYQUoTcGUIL6L4yNB7j1DFFHSBg=="
-    },
-    "regenerator-transform": {
-      "version": "0.10.1",
-      "resolved": "https://registry.npmjs.org/regenerator-transform/-/regenerator-transform-0.10.1.tgz",
-      "integrity": "sha512-PJepbvDbuK1xgIgnau7Y90cwaAmO/LCLMI2mPvaXq2heGMR3aWW5/BQvYrhJ8jgmQjXewXvBjzfqKcVOmhjZ6Q==",
-      "requires": {
-        "babel-runtime": "6.26.0",
-        "babel-types": "6.26.0",
-        "private": "0.1.8"
-      }
-    },
-    "regex-cache": {
-      "version": "0.4.4",
-      "resolved": "https://registry.npmjs.org/regex-cache/-/regex-cache-0.4.4.tgz",
-      "integrity": "sha512-nVIZwtCjkC9YgvWkpM55B5rBhBYRZhAaJbgcFYXXsHnbZ9UZI9nnVWYZpBlCqv9ho2eZryPnWrZGsOdPwVWXWQ==",
-      "requires": {
-        "is-equal-shallow": "0.1.3"
-      }
-    },
-    "regex-not": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",
-      "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==",
-      "requires": {
-        "extend-shallow": "3.0.2",
-        "safe-regex": "1.1.0"
-      },
-      "dependencies": {
-        "extend-shallow": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
-          "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
-          "requires": {
-            "assign-symbols": "1.0.0",
-            "is-extendable": "1.0.1"
-          }
-        },
-        "is-extendable": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
-          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
-          "requires": {
-            "is-plain-object": "2.0.4"
-          }
-        }
-      }
-    },
-    "regex-parser": {
-      "version": "2.2.9",
-      "resolved": "https://registry.npmjs.org/regex-parser/-/regex-parser-2.2.9.tgz",
-      "integrity": "sha512-VncXxOF6uFlYog5prG2j+e2UGJeam5MfNiJnB/qEgo4KTnMm2XrELCg4rNZ6IlaEUZnGlb8aB6lXowCRQtTkkA=="
-    },
-    "regexpu-core": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-2.0.0.tgz",
-      "integrity": "sha1-SdA4g3uNz4v6W5pCE5k45uoq4kA=",
-      "requires": {
-        "regenerate": "1.3.3",
-        "regjsgen": "0.2.0",
-        "regjsparser": "0.1.5"
-      }
-    },
-    "regjsgen": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/regjsgen/-/regjsgen-0.2.0.tgz",
-      "integrity": "sha1-bAFq3qxVT3WCP+N6wFuS1aTtsfc="
-    },
-    "regjsparser": {
-      "version": "0.1.5",
-      "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz",
-      "integrity": "sha1-fuj4Tcb6eS0/0K4ijSS9lJ6tIFw=",
-      "requires": {
-        "jsesc": "0.5.0"
-      },
-      "dependencies": {
-        "jsesc": {
-          "version": "0.5.0",
-          "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz",
-          "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0="
-        }
-      }
-    },
-    "relateurl": {
-      "version": "0.2.7",
-      "resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz",
-      "integrity": "sha1-VNvzd+UUQKypCkzSdGANP/LYiKk="
-    },
-    "remove-trailing-separator": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz",
-      "integrity": "sha1-wkvOKig62tW8P1jg1IJJuSN52O8="
-    },
-    "renderkid": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/renderkid/-/renderkid-2.0.1.tgz",
-      "integrity": "sha1-iYyr/Ivt5Le5ETWj/9Mj5YwNsxk=",
-      "requires": {
-        "css-select": "1.2.0",
-        "dom-converter": "0.1.4",
-        "htmlparser2": "3.3.0",
-        "strip-ansi": "3.0.1",
-        "utila": "0.3.3"
-      },
-      "dependencies": {
-        "css-select": {
-          "version": "1.2.0",
-          "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
-          "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
-          "requires": {
-            "boolbase": "1.0.0",
-            "css-what": "2.1.0",
-            "domutils": "1.5.1",
-            "nth-check": "1.0.1"
-          }
-        },
-        "domhandler": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.1.0.tgz",
-          "integrity": "sha1-0mRvXlf2w7qxHPbLBdPArPdBJZQ=",
-          "requires": {
-            "domelementtype": "1.3.0"
-          }
-        },
-        "domutils": {
-          "version": "1.5.1",
-          "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
-          "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
-          "requires": {
-            "dom-serializer": "0.1.0",
-            "domelementtype": "1.3.0"
-          }
-        },
-        "htmlparser2": {
-          "version": "3.3.0",
-          "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz",
-          "integrity": "sha1-zHDQWln2VC5D8OaFyYLhTJJKnv4=",
-          "requires": {
-            "domelementtype": "1.3.0",
-            "domhandler": "2.1.0",
-            "domutils": "1.1.6",
-            "readable-stream": "1.0.34"
-          },
-          "dependencies": {
-            "domutils": {
-              "version": "1.1.6",
-              "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.1.6.tgz",
-              "integrity": "sha1-vdw94Jm5ou+sxRxiPyj0FuzFdIU=",
-              "requires": {
-                "domelementtype": "1.3.0"
-              }
-            }
-          }
-        },
-        "isarray": {
-          "version": "0.0.1",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
-          "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
-        },
-        "readable-stream": {
-          "version": "1.0.34",
-          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
-          "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
-          "requires": {
-            "core-util-is": "1.0.2",
-            "inherits": "2.0.3",
-            "isarray": "0.0.1",
-            "string_decoder": "0.10.31"
-          }
-        },
-        "string_decoder": {
-          "version": "0.10.31",
-          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
-          "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
-        },
-        "utila": {
-          "version": "0.3.3",
-          "resolved": "https://registry.npmjs.org/utila/-/utila-0.3.3.tgz",
-          "integrity": "sha1-1+jn1+MJEHCSsF+NloiCTWM6QiY="
-        }
-      }
-    },
-    "repeat-element": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.2.tgz",
-      "integrity": "sha1-7wiaF40Ug7quTZPrmLT55OEdmQo="
-    },
-    "repeat-string": {
-      "version": "1.6.1",
-      "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz",
-      "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc="
-    },
-    "repeating": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz",
-      "integrity": "sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo=",
-      "requires": {
-        "is-finite": "1.0.2"
-      }
-    },
-    "replace-ext": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/replace-ext/-/replace-ext-0.0.1.tgz",
-      "integrity": "sha1-KbvZIHinOfC8zitO5B6DeVNSKSQ="
-    },
-    "request": {
-      "version": "2.79.0",
-      "resolved": "https://registry.npmjs.org/request/-/request-2.79.0.tgz",
-      "integrity": "sha1-Tf5b9r6LjNw3/Pk+BLZVd3InEN4=",
-      "requires": {
-        "aws-sign2": "0.6.0",
-        "aws4": "1.7.0",
-        "caseless": "0.11.0",
-        "combined-stream": "1.0.6",
-        "extend": "3.0.1",
-        "forever-agent": "0.6.1",
-        "form-data": "2.1.4",
-        "har-validator": "2.0.6",
-        "hawk": "3.1.3",
-        "http-signature": "1.1.1",
-        "is-typedarray": "1.0.0",
-        "isstream": "0.1.2",
-        "json-stringify-safe": "5.0.1",
-        "mime-types": "2.1.18",
-        "oauth-sign": "0.8.2",
-        "qs": "6.3.2",
-        "stringstream": "0.0.5",
-        "tough-cookie": "2.3.4",
-        "tunnel-agent": "0.4.3",
-        "uuid": "3.2.1"
-      }
-    },
-    "require-directory": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
-      "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
-    },
-    "require-main-filename": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz",
-      "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE="
-    },
-    "require-uncached": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz",
-      "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=",
-      "requires": {
-        "caller-path": "0.1.0",
-        "resolve-from": "1.0.1"
-      }
-    },
-    "requires-port": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
-      "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8="
-    },
-    "resolve": {
-      "version": "1.7.1",
-      "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.7.1.tgz",
-      "integrity": "sha512-c7rwLofp8g1U+h1KNyHL/jicrKg1Ek4q+Lr33AL65uZTinUZHe30D5HlyN5V9NW0JX1D5dXQ4jqW5l7Sy/kGfw==",
-      "requires": {
-        "path-parse": "1.0.5"
-      }
-    },
-    "resolve-cwd": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-2.0.0.tgz",
-      "integrity": "sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=",
-      "requires": {
-        "resolve-from": "3.0.0"
-      },
-      "dependencies": {
-        "resolve-from": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz",
-          "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g="
-        }
-      }
-    },
-    "resolve-dir": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/resolve-dir/-/resolve-dir-1.0.1.tgz",
-      "integrity": "sha1-eaQGRMNivoLybv/nOcm7U4IEb0M=",
-      "requires": {
-        "expand-tilde": "2.0.2",
-        "global-modules": "1.0.0"
-      }
-    },
-    "resolve-from": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-1.0.1.tgz",
-      "integrity": "sha1-Jsv+k10a7uq7Kbw/5a6wHpPUQiY="
-    },
-    "resolve-url": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz",
-      "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo="
-    },
-    "resolve-url-loader": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/resolve-url-loader/-/resolve-url-loader-2.1.0.tgz",
-      "integrity": "sha1-J8lcwWpDU5I/29wtuvXu8iIyxHc=",
-      "requires": {
-        "adjust-sourcemap-loader": "1.2.0",
-        "camelcase": "4.1.0",
-        "convert-source-map": "1.5.1",
-        "loader-utils": "1.1.0",
-        "lodash.defaults": "4.2.0",
-        "rework": "1.0.1",
-        "rework-visit": "1.0.0",
-        "source-map": "0.5.7",
-        "urix": "0.1.0"
-      },
-      "dependencies": {
-        "camelcase": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
-          "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0="
-        }
-      }
-    },
-    "responselike": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz",
-      "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=",
-      "requires": {
-        "lowercase-keys": "1.0.1"
-      }
-    },
-    "restore-cursor": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz",
-      "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=",
-      "requires": {
-        "onetime": "2.0.1",
-        "signal-exit": "3.0.2"
-      }
-    },
-    "ret": {
-      "version": "0.1.15",
-      "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz",
-      "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg=="
-    },
-    "rework": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/rework/-/rework-1.0.1.tgz",
-      "integrity": "sha1-MIBqhBNCtUUQqkEQhQzUhTQUSqc=",
-      "requires": {
-        "convert-source-map": "0.3.5",
-        "css": "2.2.1"
-      },
-      "dependencies": {
-        "convert-source-map": {
-          "version": "0.3.5",
-          "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-0.3.5.tgz",
-          "integrity": "sha1-8dgClQr33SYxof6+BZZVDIarMZA="
-        }
-      }
-    },
-    "rework-visit": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/rework-visit/-/rework-visit-1.0.0.tgz",
-      "integrity": "sha1-mUWygD8hni96ygCtuLyfZA+ELJo="
-    },
-    "right-align": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz",
-      "integrity": "sha1-YTObci/mo1FWiSENJOFMlhSGE+8=",
-      "requires": {
-        "align-text": "0.1.4"
-      }
-    },
-    "rimraf": {
-      "version": "2.6.2",
-      "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz",
-      "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==",
-      "requires": {
-        "glob": "7.1.2"
-      }
-    },
-    "ripemd160": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.1.tgz",
-      "integrity": "sha1-D0WEKVxTo2KK9+bXmsohzlfRxuc=",
-      "requires": {
-        "hash-base": "2.0.2",
-        "inherits": "2.0.3"
-      },
-      "dependencies": {
-        "hash-base": {
-          "version": "2.0.2",
-          "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-2.0.2.tgz",
-          "integrity": "sha1-ZuodhW206KVHDK32/OI65SRO8uE=",
-          "requires": {
-            "inherits": "2.0.3"
-          }
-        }
-      }
-    },
-    "roboto-font": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/roboto-font/-/roboto-font-0.1.0.tgz",
-      "integrity": "sha1-w+4Z2Cygh7x0JCPA+ZdDhFVeGf0="
-    },
-    "rollup": {
-      "version": "0.41.6",
-      "resolved": "https://registry.npmjs.org/rollup/-/rollup-0.41.6.tgz",
-      "integrity": "sha1-4NBUl4d6OYwQTYFtJzOnGKepTio=",
-      "requires": {
-        "source-map-support": "0.4.18"
-      }
-    },
-    "rollup-plugin-commonjs": {
-      "version": "8.4.1",
-      "resolved": "https://registry.npmjs.org/rollup-plugin-commonjs/-/rollup-plugin-commonjs-8.4.1.tgz",
-      "integrity": "sha512-mg+WuD+jlwoo8bJtW3Mvx7Tz6TsIdMsdhuvCnDMoyjh0oxsVgsjB/N0X984RJCWwc5IIiqNVJhXeeITcc73++A==",
-      "requires": {
-        "acorn": "5.5.3",
-        "estree-walker": "0.5.1",
-        "magic-string": "0.22.5",
-        "resolve": "1.7.1",
-        "rollup-pluginutils": "2.0.1"
-      }
-    },
-    "rollup-plugin-node-resolve": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/rollup-plugin-node-resolve/-/rollup-plugin-node-resolve-3.3.0.tgz",
-      "integrity": "sha512-9zHGr3oUJq6G+X0oRMYlzid9fXicBdiydhwGChdyeNRGPcN/majtegApRKHLR5drboUvEWU+QeUmGTyEZQs3WA==",
-      "requires": {
-        "builtin-modules": "2.0.0",
-        "is-module": "1.0.0",
-        "resolve": "1.7.1"
-      }
-    },
-    "rollup-plugin-progress": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/rollup-plugin-progress/-/rollup-plugin-progress-0.2.1.tgz",
-      "integrity": "sha1-48kyoDFjdNqvN/RxyG1K37nK7lc=",
-      "requires": {
-        "chalk": "1.1.3",
-        "rollup-pluginutils": "1.5.2"
-      },
-      "dependencies": {
-        "chalk": {
-          "version": "1.1.3",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
-          "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
-          "requires": {
-            "ansi-styles": "2.2.1",
-            "escape-string-regexp": "1.0.5",
-            "has-ansi": "2.0.0",
-            "strip-ansi": "3.0.1",
-            "supports-color": "2.0.0"
-          }
-        },
-        "estree-walker": {
-          "version": "0.2.1",
-          "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.2.1.tgz",
-          "integrity": "sha1-va/oCVOD2EFNXcLs9MkXO225QS4="
-        },
-        "rollup-pluginutils": {
-          "version": "1.5.2",
-          "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-1.5.2.tgz",
-          "integrity": "sha1-HhVud4+UtyVb+hs9AXi+j1xVJAg=",
-          "requires": {
-            "estree-walker": "0.2.1",
-            "minimatch": "3.0.4"
-          }
-        }
-      }
-    },
-    "rollup-plugin-sourcemaps": {
-      "version": "0.4.2",
-      "resolved": "https://registry.npmjs.org/rollup-plugin-sourcemaps/-/rollup-plugin-sourcemaps-0.4.2.tgz",
-      "integrity": "sha1-YhJaqUCHqt97g+9N+vYptHMTXoc=",
-      "requires": {
-        "rollup-pluginutils": "2.0.1",
-        "source-map-resolve": "0.5.1"
-      }
-    },
-    "rollup-plugin-uglify": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/rollup-plugin-uglify/-/rollup-plugin-uglify-1.0.2.tgz",
-      "integrity": "sha1-1KpvXfE1Iurhuhd4DHxMcJYDg1k=",
-      "requires": {
-        "uglify-js": "2.8.29"
-      }
-    },
-    "rollup-plugin-visualizer": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/rollup-plugin-visualizer/-/rollup-plugin-visualizer-0.2.1.tgz",
-      "integrity": "sha1-+M4USiPz9B1ryTAGYRm0fxQV1Ks="
-    },
-    "rollup-pluginutils": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/rollup-pluginutils/-/rollup-pluginutils-2.0.1.tgz",
-      "integrity": "sha1-fslbNXP2VDpGpkYb2afFRFJdD8A=",
-      "requires": {
-        "estree-walker": "0.3.1",
-        "micromatch": "2.3.11"
-      },
-      "dependencies": {
-        "estree-walker": {
-          "version": "0.3.1",
-          "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-0.3.1.tgz",
-          "integrity": "sha1-5rGlHPcpJSTnI3wxLl/mZgwc4ao="
-        }
-      }
-    },
-    "run-async": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz",
-      "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=",
-      "requires": {
-        "is-promise": "2.1.0"
-      }
-    },
-    "run-queue": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz",
-      "integrity": "sha1-6Eg5bwV9Ij8kOGkkYY4laUFh7Ec=",
-      "requires": {
-        "aproba": "1.2.0"
-      }
-    },
-    "rx-lite": {
-      "version": "4.0.8",
-      "resolved": "https://registry.npmjs.org/rx-lite/-/rx-lite-4.0.8.tgz",
-      "integrity": "sha1-Cx4Rr4vESDbwSmQH6S2kJGe3lEQ="
-    },
-    "rx-lite-aggregates": {
-      "version": "4.0.8",
-      "resolved": "https://registry.npmjs.org/rx-lite-aggregates/-/rx-lite-aggregates-4.0.8.tgz",
-      "integrity": "sha1-dTuHqJoRyVRnxKwWJsTvxOBcZ74=",
-      "requires": {
-        "rx-lite": "4.0.8"
-      }
-    },
-    "rxjs": {
-      "version": "5.4.2",
-      "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-5.4.2.tgz",
-      "integrity": "sha1-KjI2/L8D31e64G/Wly/ZnlwI/Pc=",
-      "requires": {
-        "symbol-observable": "1.2.0"
-      }
-    },
-    "safe-buffer": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
-      "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
-    },
-    "safe-regex": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz",
-      "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=",
-      "requires": {
-        "ret": "0.1.15"
-      }
-    },
-    "safer-buffer": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
-      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
-    },
-    "samsam": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/samsam/-/samsam-1.3.0.tgz",
-      "integrity": "sha512-1HwIYD/8UlOtFS3QO3w7ey+SdSDFE4HRNLZoZRYVQefrOY3l17epswImeB1ijgJFQJodIaHcwkp3r/myBjFVbg==",
-      "dev": true
-    },
-    "sass-graph": {
-      "version": "2.2.4",
-      "resolved": "https://registry.npmjs.org/sass-graph/-/sass-graph-2.2.4.tgz",
-      "integrity": "sha1-E/vWPNHK8JCLn9k0dq1DpR0eC0k=",
-      "requires": {
-        "glob": "7.1.2",
-        "lodash": "4.17.5",
-        "scss-tokenizer": "0.2.3",
-        "yargs": "7.1.0"
-      },
-      "dependencies": {
-        "camelcase": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz",
-          "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo="
-        },
-        "cliui": {
-          "version": "3.2.0",
-          "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz",
-          "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=",
-          "requires": {
-            "string-width": "1.0.2",
-            "strip-ansi": "3.0.1",
-            "wrap-ansi": "2.1.0"
-          }
-        },
-        "is-fullwidth-code-point": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-          "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
-          "requires": {
-            "number-is-nan": "1.0.1"
-          }
-        },
-        "string-width": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
-          "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
-          "requires": {
-            "code-point-at": "1.1.0",
-            "is-fullwidth-code-point": "1.0.0",
-            "strip-ansi": "3.0.1"
-          }
-        },
-        "y18n": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz",
-          "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE="
-        },
-        "yargs": {
-          "version": "7.1.0",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-7.1.0.tgz",
-          "integrity": "sha1-a6MY6xaWFyf10oT46gA+jWFU0Mg=",
-          "requires": {
-            "camelcase": "3.0.0",
-            "cliui": "3.2.0",
-            "decamelize": "1.2.0",
-            "get-caller-file": "1.0.2",
-            "os-locale": "1.4.0",
-            "read-pkg-up": "1.0.1",
-            "require-directory": "2.1.1",
-            "require-main-filename": "1.0.1",
-            "set-blocking": "2.0.0",
-            "string-width": "1.0.2",
-            "which-module": "1.0.0",
-            "y18n": "3.2.1",
-            "yargs-parser": "5.0.0"
-          }
-        }
-      }
-    },
-    "sass-loader": {
-      "version": "6.0.7",
-      "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-6.0.7.tgz",
-      "integrity": "sha512-JoiyD00Yo1o61OJsoP2s2kb19L1/Y2p3QFcCdWdF6oomBGKVYuZyqHWemRBfQ2uGYsk+CH3eCguXNfpjzlcpaA==",
-      "requires": {
-        "clone-deep": "2.0.2",
-        "loader-utils": "1.1.0",
-        "lodash.tail": "4.1.1",
-        "neo-async": "2.5.1",
-        "pify": "3.0.0"
-      }
-    },
-    "sax": {
-      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
-      "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
-    },
-    "schema-utils": {
-      "version": "0.4.5",
-      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.5.tgz",
-      "integrity": "sha512-yYrjb9TX2k/J1Y5UNy3KYdZq10xhYcF8nMpAW6o3hy6Q8WSIEf9lJHG/ePnOBfziPM3fvQwfOwa13U/Fh8qTfA==",
-      "requires": {
-        "ajv": "6.4.0",
-        "ajv-keywords": "3.1.0"
-      },
-      "dependencies": {
-        "ajv": {
-          "version": "6.4.0",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.4.0.tgz",
-          "integrity": "sha1-06/3jpJ3VJdx2vAWTP9ISCt1T8Y=",
-          "requires": {
-            "fast-deep-equal": "1.1.0",
-            "fast-json-stable-stringify": "2.0.0",
-            "json-schema-traverse": "0.3.1",
-            "uri-js": "3.0.2"
-          }
-        }
-      }
-    },
-    "scoped-regex": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/scoped-regex/-/scoped-regex-1.0.0.tgz",
-      "integrity": "sha1-o0a7Gs1CB65wvXwMfKnlZra63bg="
-    },
-    "scss-tokenizer": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz",
-      "integrity": "sha1-jrBtualyMzOCTT9VMGQRSYR85dE=",
-      "requires": {
-        "js-base64": "2.4.3",
-        "source-map": "0.4.4"
-      },
-      "dependencies": {
-        "source-map": {
-          "version": "0.4.4",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz",
-          "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=",
-          "requires": {
-            "amdefine": "1.0.1"
-          }
-        }
-      }
-    },
-    "select-hose": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/select-hose/-/select-hose-2.0.0.tgz",
-      "integrity": "sha1-Yl2GWPhlr0Psliv8N2o3NZpJlMo="
-    },
-    "selfsigned": {
-      "version": "1.10.2",
-      "resolved": "https://registry.npmjs.org/selfsigned/-/selfsigned-1.10.2.tgz",
-      "integrity": "sha1-tESVgNmZKbZbEKSDiTAaZZIIh1g=",
-      "requires": {
-        "node-forge": "0.7.1"
-      }
-    },
-    "semver": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz",
-      "integrity": "sha512-4SJ3dm0WAwWy/NVeioZh5AntkdJoWKxHxcmyP622fOkgHa4z3R0TdBJICINyaSDE6uNwVc8gZr+ZinwZAH4xIA=="
-    },
-    "send": {
-      "version": "0.16.2",
-      "resolved": "https://registry.npmjs.org/send/-/send-0.16.2.tgz",
-      "integrity": "sha512-E64YFPUssFHEFBvpbbjr44NCLtI1AohxQ8ZSiJjQLskAdKuriYEP6VyGEsRDH8ScozGpkaX1BGvhanqCwkcEZw==",
-      "requires": {
-        "debug": "2.6.9",
-        "depd": "1.1.2",
-        "destroy": "1.0.4",
-        "encodeurl": "1.0.2",
-        "escape-html": "1.0.3",
-        "etag": "1.8.1",
-        "fresh": "0.5.2",
-        "http-errors": "1.6.3",
-        "mime": "1.4.1",
-        "ms": "2.0.0",
-        "on-finished": "2.3.0",
-        "range-parser": "1.2.0",
-        "statuses": "1.4.0"
-      }
-    },
-    "serialize-javascript": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-1.5.0.tgz",
-      "integrity": "sha512-Ga8c8NjAAp46Br4+0oZ2WxJCwIzwP60Gq1YPgU+39PiTVxyed/iKE/zyZI6+UlVYH5Q4PaQdHhcegIFPZTUfoQ=="
-    },
-    "serve-index": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/serve-index/-/serve-index-1.9.1.tgz",
-      "integrity": "sha1-03aNabHn2C5c4FD/9bRTvqEqkjk=",
-      "requires": {
-        "accepts": "1.3.5",
-        "batch": "0.6.1",
-        "debug": "2.6.9",
-        "escape-html": "1.0.3",
-        "http-errors": "1.6.3",
-        "mime-types": "2.1.18",
-        "parseurl": "1.3.2"
-      }
-    },
-    "serve-static": {
-      "version": "1.13.2",
-      "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.2.tgz",
-      "integrity": "sha512-p/tdJrO4U387R9oMjb1oj7qSMaMfmOyd4j9hOFoxZe2baQszgHcSWjuya/CiT5kgZZKRudHNOA0pYXOl8rQ5nw==",
-      "requires": {
-        "encodeurl": "1.0.2",
-        "escape-html": "1.0.3",
-        "parseurl": "1.3.2",
-        "send": "0.16.2"
-      }
-    },
-    "set-blocking": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
-      "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc="
-    },
-    "set-immediate-shim": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz",
-      "integrity": "sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E="
-    },
-    "set-value": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
-      "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==",
-      "requires": {
-        "extend-shallow": "2.0.1",
-        "is-extendable": "0.1.1",
-        "is-plain-object": "2.0.4",
-        "split-string": "3.1.0"
-      }
-    },
-    "setimmediate": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz",
-      "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU="
-    },
-    "setprototypeof": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz",
-      "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ=="
-    },
-    "sha.js": {
-      "version": "2.4.11",
-      "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz",
-      "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==",
-      "requires": {
-        "inherits": "2.0.3",
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "shallow-clone": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-1.0.0.tgz",
-      "integrity": "sha512-oeXreoKR/SyNJtRJMAKPDSvd28OqEwG4eR/xc856cRGBII7gX9lvAqDxusPm0846z/w/hWYjI1NpKwJ00NHzRA==",
-      "requires": {
-        "is-extendable": "0.1.1",
-        "kind-of": "5.1.0",
-        "mixin-object": "2.0.1"
-      },
-      "dependencies": {
-        "kind-of": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
-        }
-      }
-    },
-    "shasum": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/shasum/-/shasum-1.0.2.tgz",
-      "integrity": "sha1-5wEjENj0F/TetXEhUOVni4euVl8=",
-      "dev": true,
-      "requires": {
-        "json-stable-stringify": "0.0.1",
-        "sha.js": "2.4.11"
-      },
-      "dependencies": {
-        "json-stable-stringify": {
-          "version": "0.0.1",
-          "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-0.0.1.tgz",
-          "integrity": "sha1-YRwj6BTbN1Un34URk9tZ3Sryf0U=",
-          "dev": true,
-          "requires": {
-            "jsonify": "0.0.0"
-          }
-        }
-      }
-    },
-    "shebang-command": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
-      "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=",
-      "requires": {
-        "shebang-regex": "1.0.0"
-      }
-    },
-    "shebang-regex": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
-      "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM="
-    },
-    "shell-quote": {
-      "version": "1.6.1",
-      "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.6.1.tgz",
-      "integrity": "sha1-9HgZSczkAmlxJ0MOo7PFR29IF2c=",
-      "dev": true,
-      "requires": {
-        "array-filter": "0.0.1",
-        "array-map": "0.0.0",
-        "array-reduce": "0.0.0",
-        "jsonify": "0.0.0"
-      }
-    },
-    "shelljs": {
-      "version": "0.8.1",
-      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.1.tgz",
-      "integrity": "sha512-YA/iYtZpzFe5HyWVGrb02FjPxc4EMCfpoU/Phg9fQoyMC72u9598OUBrsU8IrtwAKG0tO8IYaqbaLIw+k3IRGA==",
-      "requires": {
-        "glob": "7.1.2",
-        "interpret": "1.1.0",
-        "rechoir": "0.6.2"
-      }
-    },
-    "signal-exit": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
-      "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0="
-    },
-    "sinon": {
-      "version": "2.3.8",
-      "resolved": "https://registry.npmjs.org/sinon/-/sinon-2.3.8.tgz",
-      "integrity": "sha1-Md4G/tj7o6Zx5XbdltClhjeW8lw=",
-      "dev": true,
-      "requires": {
-        "diff": "3.5.0",
-        "formatio": "1.2.0",
-        "lolex": "1.6.0",
-        "native-promise-only": "0.8.1",
-        "path-to-regexp": "1.7.0",
-        "samsam": "1.3.0",
-        "text-encoding": "0.6.4",
-        "type-detect": "4.0.3"
-      },
-      "dependencies": {
-        "isarray": {
-          "version": "0.0.1",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
-          "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=",
-          "dev": true
-        },
-        "path-to-regexp": {
-          "version": "1.7.0",
-          "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz",
-          "integrity": "sha1-Wf3g9DW62suhA6hOnTvGTpa5k30=",
-          "dev": true,
-          "requires": {
-            "isarray": "0.0.1"
-          }
-        }
-      }
-    },
-    "slash": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz",
-      "integrity": "sha1-xB8vbDn8FtHNF61LXYlhFK5HDVU="
-    },
-    "slice-ansi": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-1.0.0.tgz",
-      "integrity": "sha512-POqxBK6Lb3q6s047D/XsDVNPnF9Dl8JSaqe9h9lURl0OdNqy/ujDrOiIHtsqXMGbWWTIomRzAMaTyawAU//Reg==",
-      "requires": {
-        "is-fullwidth-code-point": "2.0.0"
-      }
-    },
-    "slide": {
-      "version": "1.1.6",
-      "resolved": "https://registry.npmjs.org/slide/-/slide-1.1.6.tgz",
-      "integrity": "sha1-VusCfWW00tzmyy4tMsTUr8nh1wc="
-    },
-    "snapdragon": {
-      "version": "0.8.2",
-      "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz",
-      "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==",
-      "requires": {
-        "base": "0.11.2",
-        "debug": "2.6.9",
-        "define-property": "0.2.5",
-        "extend-shallow": "2.0.1",
-        "map-cache": "0.2.2",
-        "source-map": "0.5.7",
-        "source-map-resolve": "0.5.1",
-        "use": "3.1.0"
-      },
-      "dependencies": {
-        "define-property": {
-          "version": "0.2.5",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
-          "requires": {
-            "is-descriptor": "0.1.6"
-          }
-        }
-      }
-    },
-    "snapdragon-node": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz",
-      "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==",
-      "requires": {
-        "define-property": "1.0.0",
-        "isobject": "3.0.1",
-        "snapdragon-util": "3.0.1"
-      },
-      "dependencies": {
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        }
-      }
-    },
-    "snapdragon-util": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz",
-      "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==",
-      "requires": {
-        "kind-of": "3.2.2"
-      }
-    },
-    "sntp": {
-      "version": "1.0.9",
-      "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz",
-      "integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=",
-      "requires": {
-        "hoek": "2.16.3"
-      }
-    },
-    "socket.io": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-2.0.4.tgz",
-      "integrity": "sha1-waRZDO/4fs8TxyZS8Eb3FrKeYBQ=",
-      "dev": true,
-      "requires": {
-        "debug": "2.6.9",
-        "engine.io": "3.1.5",
-        "socket.io-adapter": "1.1.1",
-        "socket.io-client": "2.0.4",
-        "socket.io-parser": "3.1.3"
-      },
-      "dependencies": {
-        "arraybuffer.slice": {
-          "version": "0.0.7",
-          "resolved": "https://registry.npmjs.org/arraybuffer.slice/-/arraybuffer.slice-0.0.7.tgz",
-          "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==",
-          "dev": true
-        },
-        "engine.io-client": {
-          "version": "3.1.6",
-          "resolved": "https://registry.npmjs.org/engine.io-client/-/engine.io-client-3.1.6.tgz",
-          "integrity": "sha512-hnuHsFluXnsKOndS4Hv6SvUrgdYx1pk2NqfaDMW+GWdgfU3+/V25Cj7I8a0x92idSpa5PIhJRKxPvp9mnoLsfg==",
-          "dev": true,
-          "requires": {
-            "component-emitter": "1.2.1",
-            "component-inherit": "0.0.3",
-            "debug": "3.1.0",
-            "engine.io-parser": "2.1.2",
-            "has-cors": "1.1.0",
-            "indexof": "0.0.1",
-            "parseqs": "0.0.5",
-            "parseuri": "0.0.5",
-            "ws": "3.3.3",
-            "xmlhttprequest-ssl": "1.5.5",
-            "yeast": "0.1.2"
-          },
-          "dependencies": {
-            "debug": {
-              "version": "3.1.0",
-              "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-              "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-              "dev": true,
-              "requires": {
-                "ms": "2.0.0"
-              }
-            }
-          }
-        },
-        "engine.io-parser": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-2.1.2.tgz",
-          "integrity": "sha512-dInLFzr80RijZ1rGpx1+56/uFoH7/7InhH3kZt+Ms6hT8tNx3NGW/WNSA/f8As1WkOfkuyb3tnRyuXGxusclMw==",
-          "dev": true,
-          "requires": {
-            "after": "0.8.2",
-            "arraybuffer.slice": "0.0.7",
-            "base64-arraybuffer": "0.1.5",
-            "blob": "0.0.4",
-            "has-binary2": "1.0.2"
-          }
-        },
-        "isarray": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.1.tgz",
-          "integrity": "sha1-o32U7ZzaLVmGXJ92/llu4fM4dB4=",
-          "dev": true
-        },
-        "socket.io-client": {
-          "version": "2.0.4",
-          "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-2.0.4.tgz",
-          "integrity": "sha1-CRilUkBtxeVAs4Dc2Xr8SmQzL44=",
-          "dev": true,
-          "requires": {
-            "backo2": "1.0.2",
-            "base64-arraybuffer": "0.1.5",
-            "component-bind": "1.0.0",
-            "component-emitter": "1.2.1",
-            "debug": "2.6.9",
-            "engine.io-client": "3.1.6",
-            "has-cors": "1.1.0",
-            "indexof": "0.0.1",
-            "object-component": "0.0.3",
-            "parseqs": "0.0.5",
-            "parseuri": "0.0.5",
-            "socket.io-parser": "3.1.3",
-            "to-array": "0.1.4"
-          }
-        },
-        "socket.io-parser": {
-          "version": "3.1.3",
-          "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-3.1.3.tgz",
-          "integrity": "sha512-g0a2HPqLguqAczs3dMECuA1RgoGFPyvDqcbaDEdCWY9g59kdUAz3YRmaJBNKXflrHNwB7Q12Gkf/0CZXfdHR7g==",
-          "dev": true,
-          "requires": {
-            "component-emitter": "1.2.1",
-            "debug": "3.1.0",
-            "has-binary2": "1.0.2",
-            "isarray": "2.0.1"
-          },
-          "dependencies": {
-            "debug": {
-              "version": "3.1.0",
-              "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-              "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-              "dev": true,
-              "requires": {
-                "ms": "2.0.0"
-              }
-            }
-          }
-        },
-        "ultron": {
-          "version": "1.1.1",
-          "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz",
-          "integrity": "sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==",
-          "dev": true
-        },
-        "ws": {
-          "version": "3.3.3",
-          "resolved": "https://registry.npmjs.org/ws/-/ws-3.3.3.tgz",
-          "integrity": "sha512-nnWLa/NwZSt4KQJu51MYlCcSQ5g7INpOrOMt4XV8j4dqTXdmlUmSHQ8/oLC069ckre0fRsgfvsKwbTdtKLCDkA==",
-          "dev": true,
-          "requires": {
-            "async-limiter": "1.0.0",
-            "safe-buffer": "5.1.1",
-            "ultron": "1.1.1"
-          }
-        },
-        "xmlhttprequest-ssl": {
-          "version": "1.5.5",
-          "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz",
-          "integrity": "sha1-wodrBhaKrcQOV9l+gRkayPQ5iz4=",
-          "dev": true
-        }
-      }
-    },
-    "socket.io-adapter": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-1.1.1.tgz",
-      "integrity": "sha1-KoBeihTWNyEk3ZFZrUUC+MsH8Gs=",
-      "dev": true
-    },
-    "socket.io-client": {
-      "version": "1.7.3",
-      "resolved": "https://registry.npmjs.org/socket.io-client/-/socket.io-client-1.7.3.tgz",
-      "integrity": "sha1-sw6GqhDV7zVGYBwJzeR2Xjgdo3c=",
-      "requires": {
-        "backo2": "1.0.2",
-        "component-bind": "1.0.0",
-        "component-emitter": "1.2.1",
-        "debug": "2.3.3",
-        "engine.io-client": "1.8.3",
-        "has-binary": "0.1.7",
-        "indexof": "0.0.1",
-        "object-component": "0.0.3",
-        "parseuri": "0.0.5",
-        "socket.io-parser": "2.3.1",
-        "to-array": "0.1.4"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "2.3.3",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.3.3.tgz",
-          "integrity": "sha1-QMRT5n5uE8kB3ewxeviYbNqe/4w=",
-          "requires": {
-            "ms": "0.7.2"
-          }
-        },
-        "ms": {
-          "version": "0.7.2",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz",
-          "integrity": "sha1-riXPJRKziFodldfwN4aNhDESR2U="
-        }
-      }
-    },
-    "socket.io-parser": {
-      "version": "2.3.1",
-      "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-2.3.1.tgz",
-      "integrity": "sha1-3VMgJRA85Clpcya+/WQAX8/ltKA=",
-      "requires": {
-        "component-emitter": "1.1.2",
-        "debug": "2.2.0",
-        "isarray": "0.0.1",
-        "json3": "3.3.2"
-      },
-      "dependencies": {
-        "component-emitter": {
-          "version": "1.1.2",
-          "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.1.2.tgz",
-          "integrity": "sha1-KWWU8nU9qmOZbSrwjRWpURbJrsM="
-        },
-        "debug": {
-          "version": "2.2.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
-          "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=",
-          "requires": {
-            "ms": "0.7.1"
-          }
-        },
-        "isarray": {
-          "version": "0.0.1",
-          "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
-          "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
-        },
-        "ms": {
-          "version": "0.7.1",
-          "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz",
-          "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg="
-        }
-      }
-    },
-    "sockjs": {
-      "version": "0.3.19",
-      "resolved": "https://registry.npmjs.org/sockjs/-/sockjs-0.3.19.tgz",
-      "integrity": "sha512-V48klKZl8T6MzatbLlzzRNhMepEys9Y4oGFpypBFFn1gLI/QQ9HtLLyWJNbPlwGLelOVOEijUbTTJeLLI59jLw==",
-      "requires": {
-        "faye-websocket": "0.10.0",
-        "uuid": "3.2.1"
-      }
-    },
-    "sockjs-client": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/sockjs-client/-/sockjs-client-1.1.4.tgz",
-      "integrity": "sha1-W6vjhrd15M8U51IJEUUmVAFsixI=",
-      "requires": {
-        "debug": "2.6.9",
-        "eventsource": "0.1.6",
-        "faye-websocket": "0.11.1",
-        "inherits": "2.0.3",
-        "json3": "3.3.2",
-        "url-parse": "1.3.0"
-      },
-      "dependencies": {
-        "faye-websocket": {
-          "version": "0.11.1",
-          "resolved": "https://registry.npmjs.org/faye-websocket/-/faye-websocket-0.11.1.tgz",
-          "integrity": "sha1-8O/hjE9W5PQK/H4Gxxn9XuYYjzg=",
-          "requires": {
-            "websocket-driver": "0.7.0"
-          }
-        }
-      }
-    },
-    "sort-keys": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz",
-      "integrity": "sha1-RBttTTRnmPG05J6JIK37oOVD+a0=",
-      "requires": {
-        "is-plain-obj": "1.1.0"
-      }
-    },
-    "source-list-map": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.0.tgz",
-      "integrity": "sha512-I2UmuJSRr/T8jisiROLU3A3ltr+swpniSmNPI4Ml3ZCX6tVnDsuZzK7F2hl5jTqbZBWCEKlj5HRQiPExXLgE8A=="
-    },
-    "source-map": {
-      "version": "0.5.7",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
-      "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w="
-    },
-    "source-map-resolve": {
-      "version": "0.5.1",
-      "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.1.tgz",
-      "integrity": "sha512-0KW2wvzfxm8NCTb30z0LMNyPqWCdDGE2viwzUaucqJdkTRXtZiSY3I+2A6nVAjmdOy0I4gU8DwnVVGsk9jvP2A==",
-      "requires": {
-        "atob": "2.1.0",
-        "decode-uri-component": "0.2.0",
-        "resolve-url": "0.2.1",
-        "source-map-url": "0.4.0",
-        "urix": "0.1.0"
-      }
-    },
-    "source-map-support": {
-      "version": "0.4.18",
-      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.4.18.tgz",
-      "integrity": "sha512-try0/JqxPLF9nOjvSta7tVondkP5dwgyLDjVoyMDlmjugT2lRZ1OfsrYTkCd2hkDnJTKRbO/Rl3orm8vlsUzbA==",
-      "requires": {
-        "source-map": "0.5.7"
-      }
-    },
-    "source-map-url": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz",
-      "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM="
-    },
-    "spdx-correct": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.0.0.tgz",
-      "integrity": "sha512-N19o9z5cEyc8yQQPukRCZ9EUmb4HUpnrmaL/fxS2pBo2jbfcFRVuFZ/oFC+vZz0MNNk0h80iMn5/S6qGZOL5+g==",
-      "requires": {
-        "spdx-expression-parse": "3.0.0",
-        "spdx-license-ids": "3.0.0"
-      }
-    },
-    "spdx-exceptions": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.1.0.tgz",
-      "integrity": "sha512-4K1NsmrlCU1JJgUrtgEeTVyfx8VaYea9J9LvARxhbHtVtohPs/gFGG5yy49beySjlIMhhXZ4QqujIZEfS4l6Cg=="
-    },
-    "spdx-expression-parse": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz",
-      "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==",
-      "requires": {
-        "spdx-exceptions": "2.1.0",
-        "spdx-license-ids": "3.0.0"
-      }
-    },
-    "spdx-license-ids": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.0.tgz",
-      "integrity": "sha512-2+EPwgbnmOIl8HjGBXXMd9NAu02vLjOO1nWw4kmeRDFyHn+M/ETfHxQUK0oXg8ctgVnl9t3rosNVsZ1jG61nDA=="
-    },
-    "spdy": {
-      "version": "3.4.7",
-      "resolved": "https://registry.npmjs.org/spdy/-/spdy-3.4.7.tgz",
-      "integrity": "sha1-Qv9B7OXMD5mjpsKKq7c/XDsDrLw=",
-      "requires": {
-        "debug": "2.6.9",
-        "handle-thing": "1.2.5",
-        "http-deceiver": "1.2.7",
-        "safe-buffer": "5.1.1",
-        "select-hose": "2.0.0",
-        "spdy-transport": "2.1.0"
-      }
-    },
-    "spdy-transport": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-2.1.0.tgz",
-      "integrity": "sha512-bpUeGpZcmZ692rrTiqf9/2EUakI6/kXX1Rpe0ib/DyOzbiexVfXkw6GnvI9hVGvIwVaUhkaBojjCZwLNRGQg1g==",
-      "requires": {
-        "debug": "2.6.9",
-        "detect-node": "2.0.3",
-        "hpack.js": "2.1.6",
-        "obuf": "1.1.2",
-        "readable-stream": "2.3.6",
-        "safe-buffer": "5.1.1",
-        "wbuf": "1.7.3"
-      }
-    },
-    "split-string": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz",
-      "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==",
-      "requires": {
-        "extend-shallow": "3.0.2"
-      },
-      "dependencies": {
-        "extend-shallow": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
-          "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
-          "requires": {
-            "assign-symbols": "1.0.0",
-            "is-extendable": "1.0.1"
-          }
-        },
-        "is-extendable": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
-          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
-          "requires": {
-            "is-plain-object": "2.0.4"
-          }
-        }
-      }
-    },
-    "sprintf-js": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
-      "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw="
-    },
-    "sshpk": {
-      "version": "1.14.1",
-      "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.1.tgz",
-      "integrity": "sha1-Ew9Zde3a2WPx1W+SuaxsUfqfg+s=",
-      "requires": {
-        "asn1": "0.2.3",
-        "assert-plus": "1.0.0",
-        "dashdash": "1.14.1",
-        "getpass": "0.1.7"
-      },
-      "dependencies": {
-        "assert-plus": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
-        }
-      }
-    },
-    "ssri": {
-      "version": "5.3.0",
-      "resolved": "https://registry.npmjs.org/ssri/-/ssri-5.3.0.tgz",
-      "integrity": "sha512-XRSIPqLij52MtgoQavH/x/dU1qVKtWUAAZeOHsR9c2Ddi4XerFy3mc1alf+dLJKl9EUIm/Ht+EowFkTUOA6GAQ==",
-      "requires": {
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "stable": {
-      "version": "0.1.7",
-      "resolved": "https://registry.npmjs.org/stable/-/stable-0.1.7.tgz",
-      "integrity": "sha512-LmxBix+nUtyihSBpxXAhRakYEy49fan2suysdS1fUZcKjI+krXmH8DCZJ3yfngfrOnIFNU8O73EgNTzO2jI53w=="
-    },
-    "static-extend": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz",
-      "integrity": "sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY=",
-      "requires": {
-        "define-property": "0.2.5",
-        "object-copy": "0.1.0"
-      },
-      "dependencies": {
-        "define-property": {
-          "version": "0.2.5",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-          "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
-          "requires": {
-            "is-descriptor": "0.1.6"
-          }
-        }
-      }
-    },
-    "statuses": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.4.0.tgz",
-      "integrity": "sha512-zhSCtt8v2NDrRlPQpCNtw/heZLtfUDqxBM1udqikb/Hbk52LK4nQSwr10u77iopCW5LsyHpuXS0GnEc48mLeew=="
-    },
-    "stdout-stream": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/stdout-stream/-/stdout-stream-1.4.0.tgz",
-      "integrity": "sha1-osfIWH5U2UJ+qe2zrD8s1SLfN4s=",
-      "requires": {
-        "readable-stream": "2.3.6"
-      }
-    },
-    "stream-browserify": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz",
-      "integrity": "sha1-ZiZu5fm9uZQKTkUUyvtDu3Hlyds=",
-      "requires": {
-        "inherits": "2.0.3",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "stream-combiner2": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/stream-combiner2/-/stream-combiner2-1.1.1.tgz",
-      "integrity": "sha1-+02KFCDqNidk4hrUeAOXvry0HL4=",
-      "dev": true,
-      "requires": {
-        "duplexer2": "0.1.4",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "stream-each": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/stream-each/-/stream-each-1.2.2.tgz",
-      "integrity": "sha512-mc1dbFhGBxvTM3bIWmAAINbqiuAk9TATcfIQC8P+/+HJefgaiTlMn2dHvkX8qlI12KeYKSQ1Ua9RrIqrn1VPoA==",
-      "requires": {
-        "end-of-stream": "1.4.1",
-        "stream-shift": "1.0.0"
-      }
-    },
-    "stream-http": {
-      "version": "2.8.1",
-      "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-2.8.1.tgz",
-      "integrity": "sha512-cQ0jo17BLca2r0GfRdZKYAGLU6JRoIWxqSOakUMuKOT6MOK7AAlE856L33QuDmAy/eeOrhLee3dZKX0Uadu93A==",
-      "requires": {
-        "builtin-status-codes": "3.0.0",
-        "inherits": "2.0.3",
-        "readable-stream": "2.3.6",
-        "to-arraybuffer": "1.0.1",
-        "xtend": "4.0.1"
-      }
-    },
-    "stream-shift": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.0.tgz",
-      "integrity": "sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI="
-    },
-    "stream-splicer": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/stream-splicer/-/stream-splicer-2.0.0.tgz",
-      "integrity": "sha1-G2O+Q4oTPktnHMGTUZdgAXWRDYM=",
-      "dev": true,
-      "requires": {
-        "inherits": "2.0.3",
-        "readable-stream": "2.3.6"
-      }
-    },
-    "stream-to-observable": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/stream-to-observable/-/stream-to-observable-0.2.0.tgz",
-      "integrity": "sha1-WdbqOT2HwsDdrBCqDVYbxrpvDhA=",
-      "requires": {
-        "any-observable": "0.2.0"
-      }
-    },
-    "streamroller": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-0.7.0.tgz",
-      "integrity": "sha512-WREzfy0r0zUqp3lGO096wRuUp7ho1X6uo/7DJfTlEi0Iv/4gT7YHqXDjKC2ioVGBZtE8QzsQD9nx1nIuoZ57jQ==",
-      "dev": true,
-      "requires": {
-        "date-format": "1.2.0",
-        "debug": "3.1.0",
-        "mkdirp": "0.5.1",
-        "readable-stream": "2.3.6"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-          "dev": true,
-          "requires": {
-            "ms": "2.0.0"
-          }
-        }
-      }
-    },
-    "strict-uri-encode": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz",
-      "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM="
-    },
-    "string-template": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/string-template/-/string-template-0.2.1.tgz",
-      "integrity": "sha1-QpMuWYo1LQH8IuwzZ9nYTuxsmt0="
-    },
-    "string-width": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
-      "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
-      "requires": {
-        "is-fullwidth-code-point": "2.0.0",
-        "strip-ansi": "4.0.0"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg="
-        },
-        "strip-ansi": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
-          "requires": {
-            "ansi-regex": "3.0.0"
-          }
-        }
-      }
-    },
-    "string_decoder": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
-      "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
-      "requires": {
-        "safe-buffer": "5.1.1"
-      }
-    },
-    "stringstream": {
-      "version": "0.0.5",
-      "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz",
-      "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg="
-    },
-    "strip-ansi": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
-      "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
-      "requires": {
-        "ansi-regex": "2.1.1"
-      }
-    },
-    "strip-bom": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz",
-      "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=",
-      "requires": {
-        "is-utf8": "0.2.1"
-      }
-    },
-    "strip-bom-stream": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/strip-bom-stream/-/strip-bom-stream-2.0.0.tgz",
-      "integrity": "sha1-+H217yYT9paKpUWr/h7HKLaoKco=",
-      "requires": {
-        "first-chunk-stream": "2.0.0",
-        "strip-bom": "2.0.0"
-      }
-    },
-    "strip-eof": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
-      "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8="
-    },
-    "strip-indent": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz",
-      "integrity": "sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI=",
-      "requires": {
-        "get-stdin": "4.0.1"
-      }
-    },
-    "strip-json-comments": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
-      "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo="
-    },
-    "style-loader": {
-      "version": "0.19.0",
-      "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-0.19.0.tgz",
-      "integrity": "sha512-9mx9sC9nX1dgP96MZOODpGC6l1RzQBITI2D5WJhu+wnbrSYVKLGuy14XJSLVQih/0GFrPpjelt+s//VcZQ2Evw==",
-      "requires": {
-        "loader-utils": "1.1.0",
-        "schema-utils": "0.3.0"
-      },
-      "dependencies": {
-        "schema-utils": {
-          "version": "0.3.0",
-          "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.3.0.tgz",
-          "integrity": "sha1-9YdyIs4+kx7a4DnxfrNxbnE3+M8=",
-          "requires": {
-            "ajv": "5.5.2"
-          }
-        }
-      }
-    },
-    "subarg": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/subarg/-/subarg-1.0.0.tgz",
-      "integrity": "sha1-9izxdYHplrSPyWVpn1TAauJouNI=",
-      "dev": true,
-      "requires": {
-        "minimist": "1.2.0"
-      },
-      "dependencies": {
-        "minimist": {
-          "version": "1.2.0",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
-          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
-          "dev": true
-        }
-      }
-    },
-    "supports-color": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
-      "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc="
-    },
-    "svg-baker": {
-      "version": "1.2.17",
-      "resolved": "https://registry.npmjs.org/svg-baker/-/svg-baker-1.2.17.tgz",
-      "integrity": "sha512-Tjx9jgFoNpWQPLFgxEXKvCOjbo+8xoAxR9beUcdeR4k+5RFISfteWu6Y6OR7FPEXVBBrQzoZQM5/gSYTVfKxiA==",
-      "requires": {
-        "bluebird": "3.5.1",
-        "clone": "2.1.2",
-        "he": "1.1.1",
-        "image-size": "0.5.5",
-        "loader-utils": "1.1.0",
-        "merge-options": "0.0.64",
-        "micromatch": "3.1.0",
-        "postcss": "5.2.18",
-        "postcss-prefix-selector": "1.6.0",
-        "posthtml-rename-id": "1.0.4",
-        "posthtml-svg-mode": "1.0.2",
-        "query-string": "4.3.4",
-        "traverse": "0.6.6"
-      },
-      "dependencies": {
-        "arr-diff": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
-          "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA="
-        },
-        "array-unique": {
-          "version": "0.3.2",
-          "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
-          "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg="
-        },
-        "braces": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
-          "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
-          "requires": {
-            "arr-flatten": "1.1.0",
-            "array-unique": "0.3.2",
-            "extend-shallow": "2.0.1",
-            "fill-range": "4.0.0",
-            "isobject": "3.0.1",
-            "repeat-element": "1.1.2",
-            "snapdragon": "0.8.2",
-            "snapdragon-node": "2.1.1",
-            "split-string": "3.1.0",
-            "to-regex": "3.0.2"
-          }
-        },
-        "clone": {
-          "version": "2.1.2",
-          "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
-          "integrity": "sha1-G39Ln1kfHo+DZwQBYANFoCiHQ18="
-        },
-        "expand-brackets": {
-          "version": "2.1.4",
-          "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
-          "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=",
-          "requires": {
-            "debug": "2.6.9",
-            "define-property": "0.2.5",
-            "extend-shallow": "2.0.1",
-            "posix-character-classes": "0.1.1",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "define-property": {
-              "version": "0.2.5",
-              "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-              "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
-              "requires": {
-                "is-descriptor": "0.1.6"
-              }
-            }
-          }
-        },
-        "extglob": {
-          "version": "2.0.4",
-          "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz",
-          "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==",
-          "requires": {
-            "array-unique": "0.3.2",
-            "define-property": "1.0.0",
-            "expand-brackets": "2.1.4",
-            "extend-shallow": "2.0.1",
-            "fragment-cache": "0.2.1",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          }
-        },
-        "fill-range": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
-          "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=",
-          "requires": {
-            "extend-shallow": "2.0.1",
-            "is-number": "3.0.0",
-            "repeat-string": "1.6.1",
-            "to-regex-range": "2.1.1"
-          }
-        },
-        "is-number": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-          "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        },
-        "kind-of": {
-          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-          "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
-        },
-        "micromatch": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.0.tgz",
-          "integrity": "sha512-3StSelAE+hnRvMs8IdVW7Uhk8CVed5tp+kLLGlBP6WiRAXS21GPGu/Nat4WNPXj2Eoc24B02SaeoyozPMfj0/g==",
-          "requires": {
-            "arr-diff": "4.0.0",
-            "array-unique": "0.3.2",
-            "braces": "2.3.2",
-            "define-property": "1.0.0",
-            "extend-shallow": "2.0.1",
-            "extglob": "2.0.4",
-            "fragment-cache": "0.2.1",
-            "kind-of": "5.1.0",
-            "nanomatch": "1.2.9",
-            "object.pick": "1.3.0",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          }
-        }
-      }
-    },
-    "svg-baker-runtime": {
-      "version": "1.3.5",
-      "resolved": "https://registry.npmjs.org/svg-baker-runtime/-/svg-baker-runtime-1.3.5.tgz",
-      "integrity": "sha512-BKxJT/Zz9M+K043zXbZf7CA3c10NKWByxobAukO30VLv71OvmpagjG32Z0UIay6ctMaOUmywOKHuceiSDqwUOA==",
-      "requires": {
-        "deepmerge": "1.3.2",
-        "mitt": "1.1.2",
-        "svg-baker": "1.2.17"
-      }
-    },
-    "svg-sprite-loader": {
-      "version": "3.0.7",
-      "resolved": "https://registry.npmjs.org/svg-sprite-loader/-/svg-sprite-loader-3.0.7.tgz",
-      "integrity": "sha1-vwKyAa9q93m1w4RcxxNRy21PqDA=",
-      "requires": {
-        "bluebird": "3.5.1",
-        "deepmerge": "1.3.2",
-        "domready": "1.0.8",
-        "escape-string-regexp": "1.0.5",
-        "loader-utils": "1.1.0",
-        "svg-baker": "1.2.17",
-        "svg-baker-runtime": "1.3.5",
-        "url-slug": "2.0.0"
-      }
-    },
-    "svgo": {
-      "version": "0.7.2",
-      "resolved": "https://registry.npmjs.org/svgo/-/svgo-0.7.2.tgz",
-      "integrity": "sha1-n1dyQTlSE1xv779Ar+ak+qiLS7U=",
-      "requires": {
-        "coa": "1.0.4",
-        "colors": "1.1.2",
-        "csso": "2.3.2",
-        "js-yaml": "3.7.0",
-        "mkdirp": "0.5.1",
-        "sax": "1.2.4",
-        "whet.extend": "0.9.9"
-      }
-    },
-    "symbol-observable": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz",
-      "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ=="
-    },
-    "syntax-error": {
-      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/syntax-error/-/syntax-error-1.4.0.tgz",
-      "integrity": "sha512-YPPlu67mdnHGTup2A8ff7BC2Pjq0e0Yp/IyTFN03zWO0RcK07uLcbi7C2KpGR2FvWbaB0+bfE27a+sBKebSo7w==",
-      "dev": true,
-      "requires": {
-        "acorn-node": "1.3.0"
-      }
-    },
-    "table": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/table/-/table-4.0.3.tgz",
-      "integrity": "sha512-S7rnFITmBH1EnyKcvxBh1LjYeQMmnZtCXSEbHcH6S0NoKit24ZuFO/T1vDcLdYsLQkM188PVVhQmzKIuThNkKg==",
-      "requires": {
-        "ajv": "6.4.0",
-        "ajv-keywords": "3.1.0",
-        "chalk": "2.4.0",
-        "lodash": "4.17.5",
-        "slice-ansi": "1.0.0",
-        "string-width": "2.1.1"
-      },
-      "dependencies": {
-        "ajv": {
-          "version": "6.4.0",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.4.0.tgz",
-          "integrity": "sha1-06/3jpJ3VJdx2vAWTP9ISCt1T8Y=",
-          "requires": {
-            "fast-deep-equal": "1.1.0",
-            "fast-json-stable-stringify": "2.0.0",
-            "json-schema-traverse": "0.3.1",
-            "uri-js": "3.0.2"
-          }
-        },
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "tapable": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.0.0.tgz",
-      "integrity": "sha512-dQRhbNQkRnaqauC7WqSJ21EEksgT0fYZX2lqXzGkpo8JNig9zGZTYoMGvyI2nWmXlE2VSVXVDu7wLVGu/mQEsg=="
-    },
-    "tar": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz",
-      "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=",
-      "requires": {
-        "block-stream": "0.0.9",
-        "fstream": "1.0.11",
-        "inherits": "2.0.3"
-      }
-    },
-    "teamcity-service-messages": {
-      "version": "0.1.9",
-      "resolved": "https://registry.npmjs.org/teamcity-service-messages/-/teamcity-service-messages-0.1.9.tgz",
-      "integrity": "sha512-agmBUllpL8n02cG/6s16St5yXYEdynkyyGDWM5qsFq9sKEkc+gBAJgcgJQCVsqlxbZZUToRwTI1hLataRjCGcw==",
-      "dev": true
-    },
-    "temp": {
-      "version": "0.8.3",
-      "resolved": "https://registry.npmjs.org/temp/-/temp-0.8.3.tgz",
-      "integrity": "sha1-4Ma8TSa5AxJEEOT+2BEDAU38H1k=",
-      "requires": {
-        "os-tmpdir": "1.0.2",
-        "rimraf": "2.2.8"
-      },
-      "dependencies": {
-        "rimraf": {
-          "version": "2.2.8",
-          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.2.8.tgz",
-          "integrity": "sha1-5Dm+Kq7jJzIZUnMPmaiSnk/FBYI="
-        }
-      }
-    },
-    "text-encoding": {
-      "version": "0.6.4",
-      "resolved": "https://registry.npmjs.org/text-encoding/-/text-encoding-0.6.4.tgz",
-      "integrity": "sha1-45mpgiV6J22uQou5KEXLcb3CbRk=",
-      "dev": true
-    },
-    "text-table": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
-      "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ="
-    },
-    "textextensions": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/textextensions/-/textextensions-2.2.0.tgz",
-      "integrity": "sha512-j5EMxnryTvKxwH2Cq+Pb43tsf6sdEgw6Pdwxk83mPaq0ToeFJt6WE4J3s5BqY7vmjlLgkgXvhtXUxo80FyBhCA=="
-    },
-    "tf-metatags": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/tf-metatags/-/tf-metatags-2.0.0.tgz",
-      "integrity": "sha1-NNHSE4hM4iKkil0F/9Nf7mrT6Yg="
-    },
-    "through": {
-      "version": "2.3.8",
-      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
-      "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
-    },
-    "through2": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.3.tgz",
-      "integrity": "sha1-AARWmzfHx0ujnEPzzteNGtlBQL4=",
-      "requires": {
-        "readable-stream": "2.3.6",
-        "xtend": "4.0.1"
-      }
-    },
-    "thunky": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.0.2.tgz",
-      "integrity": "sha1-qGLgGOP7HqLsP85dVWBc9X8kc3E="
-    },
-    "timed-out": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz",
-      "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8="
-    },
-    "timers-browserify": {
-      "version": "2.0.9",
-      "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.9.tgz",
-      "integrity": "sha512-2DhyvVpCWwY7gk8UmKhYvgHQl9XTlO0Dg0/2UZcLgPnpulhdm2aGIlFy5rU5igmOCA51w6jPHqLRA4UH1YmhcA==",
-      "requires": {
-        "setimmediate": "1.0.5"
-      }
-    },
-    "tmp": {
-      "version": "0.0.33",
-      "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz",
-      "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==",
-      "requires": {
-        "os-tmpdir": "1.0.2"
-      }
-    },
-    "to-array": {
-      "version": "0.1.4",
-      "resolved": "https://registry.npmjs.org/to-array/-/to-array-0.1.4.tgz",
-      "integrity": "sha1-F+bBH3PdTz10zaek/zI46a2b+JA="
-    },
-    "to-arraybuffer": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz",
-      "integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M="
-    },
-    "to-fast-properties": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz",
-      "integrity": "sha1-uDVx+k2MJbguIxsG46MFXeTKGkc="
-    },
-    "to-object-path": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz",
-      "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=",
-      "requires": {
-        "kind-of": "3.2.2"
-      }
-    },
-    "to-regex": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz",
-      "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==",
-      "requires": {
-        "define-property": "2.0.2",
-        "extend-shallow": "3.0.2",
-        "regex-not": "1.0.2",
-        "safe-regex": "1.1.0"
-      },
-      "dependencies": {
-        "define-property": {
-          "version": "2.0.2",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
-          "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
-          "requires": {
-            "is-descriptor": "1.0.2",
-            "isobject": "3.0.1"
-          }
-        },
-        "extend-shallow": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
-          "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
-          "requires": {
-            "assign-symbols": "1.0.0",
-            "is-extendable": "1.0.1"
-          }
-        },
-        "is-accessor-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-data-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-descriptor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
-          "requires": {
-            "is-accessor-descriptor": "1.0.0",
-            "is-data-descriptor": "1.0.0",
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-extendable": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
-          "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
-          "requires": {
-            "is-plain-object": "2.0.4"
-          }
-        },
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        },
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
-        }
-      }
-    },
-    "to-regex-range": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz",
-      "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=",
-      "requires": {
-        "is-number": "3.0.0",
-        "repeat-string": "1.6.1"
-      },
-      "dependencies": {
-        "is-number": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-          "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
-          "requires": {
-            "kind-of": "3.2.2"
-          }
-        }
-      }
-    },
-    "token-stream": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/token-stream/-/token-stream-0.0.1.tgz",
-      "integrity": "sha1-zu78cXp2xDFvEm0LnbqlXX598Bo="
-    },
-    "toposort": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/toposort/-/toposort-1.0.6.tgz",
-      "integrity": "sha1-wxdI5V0hDv/AD9zcfW5o19e7nOw="
-    },
-    "tough-cookie": {
-      "version": "2.3.4",
-      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.4.tgz",
-      "integrity": "sha512-TZ6TTfI5NtZnuyy/Kecv+CnoROnyXn2DN97LontgQpCwsX2XyLYCC0ENhYkehSOwAp8rTQKc/NUIF7BkQ5rKLA==",
-      "requires": {
-        "punycode": "1.4.1"
-      },
-      "dependencies": {
-        "punycode": {
-          "version": "1.4.1",
-          "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
-          "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
-        }
-      }
-    },
-    "traverse": {
-      "version": "0.6.6",
-      "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz",
-      "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc="
-    },
-    "trim-newlines": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",
-      "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM="
-    },
-    "trim-right": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/trim-right/-/trim-right-1.0.1.tgz",
-      "integrity": "sha1-yy4SAwZ+DI3h9hQJS5/kVwTqYAM="
-    },
-    "true-case-path": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/true-case-path/-/true-case-path-1.0.2.tgz",
-      "integrity": "sha1-fskRMJJHZsf1c74wIMNPj9/QDWI=",
-      "requires": {
-        "glob": "6.0.4"
-      },
-      "dependencies": {
-        "glob": {
-          "version": "6.0.4",
-          "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
-          "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=",
-          "requires": {
-            "inflight": "1.0.6",
-            "inherits": "2.0.3",
-            "minimatch": "3.0.4",
-            "once": "1.4.0",
-            "path-is-absolute": "1.0.1"
-          }
-        }
-      }
-    },
-    "tty-browserify": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz",
-      "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY="
-    },
-    "tunnel-agent": {
-      "version": "0.4.3",
-      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.3.tgz",
-      "integrity": "sha1-Y3PbdpCf5XDgjXNYM2Xtgop07us="
-    },
-    "type-check": {
-      "version": "0.3.2",
-      "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
-      "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=",
-      "requires": {
-        "prelude-ls": "1.1.2"
-      }
-    },
-    "type-detect": {
-      "version": "4.0.3",
-      "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.3.tgz",
-      "integrity": "sha1-Dj8mcLRAmbC0bChNE2p+9Jx0wuo=",
-      "dev": true
-    },
-    "type-is": {
-      "version": "1.6.16",
-      "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz",
-      "integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==",
-      "requires": {
-        "media-typer": "0.3.0",
-        "mime-types": "2.1.18"
-      }
-    },
-    "typedarray": {
-      "version": "0.0.6",
-      "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
-      "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
-    },
-    "uglify-js": {
-      "version": "2.8.29",
-      "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-2.8.29.tgz",
-      "integrity": "sha1-KcVzMUgFe7Th913zW3qcty5qWd0=",
-      "requires": {
-        "source-map": "0.5.7",
-        "yargs": "3.10.0"
-      },
-      "dependencies": {
-        "yargs": {
-          "version": "3.10.0",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz",
-          "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=",
-          "requires": {
-            "camelcase": "1.2.1",
-            "cliui": "2.1.0",
-            "decamelize": "1.2.0",
-            "window-size": "0.1.0"
-          }
-        }
-      }
-    },
-    "uglifyjs-webpack-plugin": {
-      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/uglifyjs-webpack-plugin/-/uglifyjs-webpack-plugin-1.2.4.tgz",
-      "integrity": "sha512-z0IbjpW8b3O/OVn+TTZN4pI29RN1zktFBXLIzzfZ+++cUtZ1ERSlLWgpE/5OERuEUs1ijVQnpYAkSlpoVmQmSQ==",
-      "requires": {
-        "cacache": "10.0.4",
-        "find-cache-dir": "1.0.0",
-        "schema-utils": "0.4.5",
-        "serialize-javascript": "1.5.0",
-        "source-map": "0.6.1",
-        "uglify-es": "3.3.9",
-        "webpack-sources": "1.1.0",
-        "worker-farm": "1.6.0"
-      },
-      "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        },
-        "uglify-es": {
-          "version": "3.3.9",
-          "resolved": "https://registry.npmjs.org/uglify-es/-/uglify-es-3.3.9.tgz",
-          "integrity": "sha512-r+MU0rfv4L/0eeW3xZrd16t4NZfK8Ld4SWVglYBb7ez5uXFWHuVRs6xCTrf1yirs9a4j4Y27nn7SRfO6v67XsQ==",
-          "requires": {
-            "commander": "2.13.0",
-            "source-map": "0.6.1"
-          }
-        }
-      }
-    },
-    "ultron": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.0.2.tgz",
-      "integrity": "sha1-rOEWq1V80Zc4ak6I9GhTeMiy5Po="
-    },
-    "umd": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/umd/-/umd-3.0.3.tgz",
-      "integrity": "sha512-4IcGSufhFshvLNcMCV80UnQVlZ5pMOC8mvNPForqwA4+lzYQuetTESLDQkeLmihq8bRcnpbQa48Wb8Lh16/xow==",
-      "dev": true
-    },
-    "underscore": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz",
-      "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag="
-    },
-    "unidecode": {
-      "version": "0.1.8",
-      "resolved": "https://registry.npmjs.org/unidecode/-/unidecode-0.1.8.tgz",
-      "integrity": "sha1-77swFTi8RSRqmsjFWdcvAVMFBT4="
-    },
-    "union-value": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
-      "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=",
-      "requires": {
-        "arr-union": "3.1.0",
-        "get-value": "2.0.6",
-        "is-extendable": "0.1.1",
-        "set-value": "0.4.3"
-      },
-      "dependencies": {
-        "set-value": {
-          "version": "0.4.3",
-          "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
-          "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
-          "requires": {
-            "extend-shallow": "2.0.1",
-            "is-extendable": "0.1.1",
-            "is-plain-object": "2.0.4",
-            "to-object-path": "0.3.0"
-          }
-        }
-      }
-    },
-    "uniq": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz",
-      "integrity": "sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8="
-    },
-    "uniqid": {
-      "version": "4.1.1",
-      "resolved": "https://registry.npmjs.org/uniqid/-/uniqid-4.1.1.tgz",
-      "integrity": "sha1-iSIN32t1GuUrX3JISGNShZa7hME=",
-      "requires": {
-        "macaddress": "0.2.8"
-      }
-    },
-    "uniqs": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/uniqs/-/uniqs-2.0.0.tgz",
-      "integrity": "sha1-/+3ks2slKQaW5uFl1KWe25mOawI="
-    },
-    "unique-filename": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.0.tgz",
-      "integrity": "sha1-0F8v5AMlYIcfMOk8vnNe6iAVFPM=",
-      "requires": {
-        "unique-slug": "2.0.0"
-      }
-    },
-    "unique-slug": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.0.tgz",
-      "integrity": "sha1-22Z258fMBimHj/GWCXx4hVrp9Ks=",
-      "requires": {
-        "imurmurhash": "0.1.4"
-      }
-    },
-    "unpipe": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
-      "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
-    },
-    "unquote": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/unquote/-/unquote-1.1.1.tgz",
-      "integrity": "sha1-j97XMk7G6IoP+LkF58CYzcCG1UQ="
-    },
-    "unset-value": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz",
-      "integrity": "sha1-g3aHP30jNRef+x5vw6jtDfyKtVk=",
-      "requires": {
-        "has-value": "0.3.1",
-        "isobject": "3.0.1"
-      },
-      "dependencies": {
-        "has-value": {
-          "version": "0.3.1",
-          "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz",
-          "integrity": "sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8=",
-          "requires": {
-            "get-value": "2.0.6",
-            "has-values": "0.1.4",
-            "isobject": "2.1.0"
-          },
-          "dependencies": {
-            "isobject": {
-              "version": "2.1.0",
-              "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz",
-              "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=",
-              "requires": {
-                "isarray": "1.0.0"
-              }
-            }
-          }
-        },
-        "has-values": {
-          "version": "0.1.4",
-          "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz",
-          "integrity": "sha1-bWHeldkd/Km5oCCJrThL/49it3E="
-        },
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        }
-      }
-    },
-    "untildify": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/untildify/-/untildify-3.0.2.tgz",
-      "integrity": "sha1-fx8wIFWz/qDz6B3HjrNnZstl4/E="
-    },
-    "upath": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/upath/-/upath-1.0.4.tgz",
-      "integrity": "sha512-d4SJySNBXDaQp+DPrziv3xGS6w3d2Xt69FijJr86zMPBy23JEloMCEOUBBzuN7xCtjLCnmB9tI/z7SBCahHBOw=="
-    },
-    "upper-case": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/upper-case/-/upper-case-1.1.3.tgz",
-      "integrity": "sha1-9rRQHC7EzdJrp4vnIilh3ndiFZg="
-    },
-    "uri-js": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-3.0.2.tgz",
-      "integrity": "sha1-+QuFhQf4HepNz7s8TD2/orVX+qo=",
-      "requires": {
-        "punycode": "2.1.0"
-      }
-    },
-    "urix": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz",
-      "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI="
-    },
-    "url": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz",
-      "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=",
-      "requires": {
-        "punycode": "1.3.2",
-        "querystring": "0.2.0"
-      },
-      "dependencies": {
-        "punycode": {
-          "version": "1.3.2",
-          "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
-          "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
-        }
-      }
-    },
-    "url-join": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.0.tgz",
-      "integrity": "sha1-TTNA6AfTdzvamZH4MFrNzCpmXSo="
-    },
-    "url-parse": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.3.0.tgz",
-      "integrity": "sha512-zPvPA3T7P6M+0iNsgX+iAcAz4GshKrowtQBHHc/28tVsBc8jK7VRCNX+2GEcoE6zDB6XqXhcyiUWPVZY6C70Cg==",
-      "requires": {
-        "querystringify": "1.0.0",
-        "requires-port": "1.0.0"
-      },
-      "dependencies": {
-        "querystringify": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-1.0.0.tgz",
-          "integrity": "sha1-YoYkIRLFtxL6ZU5SZlK/ahP/Bcs="
-        }
-      }
-    },
-    "url-parse-lax": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz",
-      "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=",
-      "requires": {
-        "prepend-http": "2.0.0"
-      },
-      "dependencies": {
-        "prepend-http": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz",
-          "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc="
-        }
-      }
-    },
-    "url-slug": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/url-slug/-/url-slug-2.0.0.tgz",
-      "integrity": "sha1-p4nVrtSZXA2VrzM3etHVxo1NcCc=",
-      "requires": {
-        "unidecode": "0.1.8"
-      }
-    },
-    "url-to-options": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/url-to-options/-/url-to-options-1.0.1.tgz",
-      "integrity": "sha1-FQWgOiiaSMvXpDTvuu7FBV9WM6k="
-    },
-    "use": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/use/-/use-3.1.0.tgz",
-      "integrity": "sha512-6UJEQM/L+mzC3ZJNM56Q4DFGLX/evKGRg15UJHGB9X5j5Z3AFbgZvjUh2yq/UJUY4U5dh7Fal++XbNg1uzpRAw==",
-      "requires": {
-        "kind-of": "6.0.2"
-      },
-      "dependencies": {
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
-        }
-      }
-    },
-    "useragent": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/useragent/-/useragent-2.3.0.tgz",
-      "integrity": "sha512-4AoH4pxuSvHCjqLO04sU6U/uE65BYza8l/KKBS0b0hnUPWi+cQ2BpeTEwejCSx9SPV5/U03nniDTrWx5NrmKdw==",
-      "dev": true,
-      "requires": {
-        "lru-cache": "4.1.2",
-        "tmp": "0.0.33"
-      }
-    },
-    "util": {
-      "version": "0.10.3",
-      "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz",
-      "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=",
-      "requires": {
-        "inherits": "2.0.1"
-      },
-      "dependencies": {
-        "inherits": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz",
-          "integrity": "sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE="
-        }
-      }
-    },
-    "util-deprecate": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
-      "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
-    },
-    "util.promisify": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/util.promisify/-/util.promisify-1.0.0.tgz",
-      "integrity": "sha512-i+6qA2MPhvoKLuxnJNpXAGhg7HphQOSUq2LKMZD0m15EiskXUkMvKdF4Uui0WYeCUGea+o2cw/ZuwehtfsrNkA==",
-      "requires": {
-        "define-properties": "1.1.2",
-        "object.getownpropertydescriptors": "2.0.3"
-      }
-    },
-    "utila": {
-      "version": "0.4.0",
-      "resolved": "https://registry.npmjs.org/utila/-/utila-0.4.0.tgz",
-      "integrity": "sha1-ihagXURWV6Oupe7MWxKk+lN5dyw="
-    },
-    "utils-merge": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
-      "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
-    },
-    "uuid": {
-      "version": "3.2.1",
-      "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz",
-      "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA=="
-    },
-    "v8-compile-cache": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-1.1.2.tgz",
-      "integrity": "sha512-ejdrifsIydN1XDH7EuR2hn8ZrkRKUYF7tUcBjBy/lhrCvs2K+zRlbW9UHc0IQ9RsYFZJFqJrieoIHfkCa0DBRA=="
-    },
-    "validate-npm-package-license": {
-      "version": "3.0.3",
-      "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.3.tgz",
-      "integrity": "sha512-63ZOUnL4SIXj4L0NixR3L1lcjO38crAbgrTpl28t8jjrfuiOBL5Iygm+60qPs/KsZGzPNg6Smnc/oY16QTjF0g==",
-      "requires": {
-        "spdx-correct": "3.0.0",
-        "spdx-expression-parse": "3.0.0"
-      }
-    },
-    "vary": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
-      "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
-    },
-    "vendors": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/vendors/-/vendors-1.0.1.tgz",
-      "integrity": "sha1-N61zyO5Bf7PVgOeFMSMH0nSEfyI="
-    },
-    "verror": {
-      "version": "1.10.0",
-      "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
-      "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
-      "requires": {
-        "assert-plus": "1.0.0",
-        "core-util-is": "1.0.2",
-        "extsprintf": "1.3.0"
-      },
-      "dependencies": {
-        "assert-plus": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
-        }
-      }
-    },
-    "vinyl": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/vinyl/-/vinyl-1.2.0.tgz",
-      "integrity": "sha1-XIgDbPVl5d8FVYv8kR+GVt8hiIQ=",
-      "requires": {
-        "clone": "1.0.4",
-        "clone-stats": "0.0.1",
-        "replace-ext": "0.0.1"
-      }
-    },
-    "vinyl-file": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/vinyl-file/-/vinyl-file-2.0.0.tgz",
-      "integrity": "sha1-p+v1/779obfRjRQPyweyI++2dRo=",
-      "requires": {
-        "graceful-fs": "4.1.11",
-        "pify": "2.3.0",
-        "pinkie-promise": "2.0.1",
-        "strip-bom": "2.0.0",
-        "strip-bom-stream": "2.0.0",
-        "vinyl": "1.2.0"
-      },
-      "dependencies": {
-        "pify": {
-          "version": "2.3.0",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw="
-        }
-      }
-    },
-    "vlq": {
-      "version": "0.2.3",
-      "resolved": "https://registry.npmjs.org/vlq/-/vlq-0.2.3.tgz",
-      "integrity": "sha512-DRibZL6DsNhIgYQ+wNdWDL2SL3bKPlVrRiBqV5yuMm++op8W4kGFtaQfCs4KEJn0wBZcHVHJ3eoywX8983k1ow=="
-    },
-    "vm-browserify": {
-      "version": "0.0.4",
-      "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz",
-      "integrity": "sha1-XX6kW7755Kb/ZflUOOCofDV9WnM=",
-      "requires": {
-        "indexof": "0.0.1"
-      }
-    },
-    "void-elements": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/void-elements/-/void-elements-2.0.1.tgz",
-      "integrity": "sha1-wGavtYK7HLQSjWDqkjkulNXp2+w="
-    },
-    "w3c-blob": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/w3c-blob/-/w3c-blob-0.0.1.tgz",
-      "integrity": "sha1-sM01KhpQ9RVWNCD/1YYflQ8dhbg="
-    },
-    "watchpack": {
-      "version": "1.5.0",
-      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.5.0.tgz",
-      "integrity": "sha512-RSlipNQB1u48cq0wH/BNfCu1tD/cJ8ydFIkNYhp9o+3d+8unClkIovpW5qpFPgmL9OE48wfAnlZydXByWP82AA==",
-      "requires": {
-        "chokidar": "2.0.3",
-        "graceful-fs": "4.1.11",
-        "neo-async": "2.5.1"
-      }
-    },
-    "wbuf": {
-      "version": "1.7.3",
-      "resolved": "https://registry.npmjs.org/wbuf/-/wbuf-1.7.3.tgz",
-      "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==",
-      "requires": {
-        "minimalistic-assert": "1.0.1"
-      }
-    },
-    "webpack": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.5.0.tgz",
-      "integrity": "sha512-6GrZsvQJnG7o7mjbfjp6s5CyMfdopjt1A/X8LcYwceis9ySjqBX6Lusso2wNZ06utHj2ZvfL6L3f7hfgVeJP6g==",
-      "requires": {
-        "acorn": "5.5.3",
-        "acorn-dynamic-import": "3.0.0",
-        "ajv": "6.4.0",
-        "ajv-keywords": "3.1.0",
-        "chrome-trace-event": "0.1.3",
-        "enhanced-resolve": "4.0.0",
-        "eslint-scope": "3.7.1",
-        "loader-runner": "2.3.0",
-        "loader-utils": "1.1.0",
-        "memory-fs": "0.4.1",
-        "micromatch": "3.1.10",
-        "mkdirp": "0.5.1",
-        "neo-async": "2.5.1",
-        "node-libs-browser": "2.1.0",
-        "schema-utils": "0.4.5",
-        "tapable": "1.0.0",
-        "uglifyjs-webpack-plugin": "1.2.4",
-        "watchpack": "1.5.0",
-        "webpack-sources": "1.1.0"
-      },
-      "dependencies": {
-        "ajv": {
-          "version": "6.4.0",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.4.0.tgz",
-          "integrity": "sha1-06/3jpJ3VJdx2vAWTP9ISCt1T8Y=",
-          "requires": {
-            "fast-deep-equal": "1.1.0",
-            "fast-json-stable-stringify": "2.0.0",
-            "json-schema-traverse": "0.3.1",
-            "uri-js": "3.0.2"
-          }
-        },
-        "arr-diff": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz",
-          "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA="
-        },
-        "array-unique": {
-          "version": "0.3.2",
-          "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz",
-          "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg="
-        },
-        "braces": {
-          "version": "2.3.2",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz",
-          "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==",
-          "requires": {
-            "arr-flatten": "1.1.0",
-            "array-unique": "0.3.2",
-            "extend-shallow": "2.0.1",
-            "fill-range": "4.0.0",
-            "isobject": "3.0.1",
-            "repeat-element": "1.1.2",
-            "snapdragon": "0.8.2",
-            "snapdragon-node": "2.1.1",
-            "split-string": "3.1.0",
-            "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            }
-          }
-        },
-        "define-property": {
-          "version": "2.0.2",
-          "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz",
-          "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==",
-          "requires": {
-            "is-descriptor": "1.0.2",
-            "isobject": "3.0.1"
-          }
-        },
-        "expand-brackets": {
-          "version": "2.1.4",
-          "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz",
-          "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=",
-          "requires": {
-            "debug": "2.6.9",
-            "define-property": "0.2.5",
-            "extend-shallow": "2.0.1",
-            "posix-character-classes": "0.1.1",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "define-property": {
-              "version": "0.2.5",
-              "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz",
-              "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=",
-              "requires": {
-                "is-descriptor": "0.1.6"
-              }
-            },
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            },
-            "is-accessor-descriptor": {
-              "version": "0.1.6",
-              "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz",
-              "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=",
-              "requires": {
-                "kind-of": "3.2.2"
-              },
-              "dependencies": {
-                "kind-of": {
-                  "version": "3.2.2",
-                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-                  "requires": {
-                    "is-buffer": "1.1.6"
-                  }
-                }
-              }
-            },
-            "is-data-descriptor": {
-              "version": "0.1.4",
-              "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz",
-              "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=",
-              "requires": {
-                "kind-of": "3.2.2"
-              },
-              "dependencies": {
-                "kind-of": {
-                  "version": "3.2.2",
-                  "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-                  "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-                  "requires": {
-                    "is-buffer": "1.1.6"
-                  }
-                }
-              }
-            },
-            "is-descriptor": {
-              "version": "0.1.6",
-              "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz",
-              "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==",
-              "requires": {
-                "is-accessor-descriptor": "0.1.6",
-                "is-data-descriptor": "0.1.4",
-                "kind-of": "5.1.0"
-              }
-            },
-            "kind-of": {
-              "version": "5.1.0",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz",
-              "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw=="
-            }
-          }
-        },
-        "extend-shallow": {
-          "version": "3.0.2",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz",
-          "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=",
-          "requires": {
-            "assign-symbols": "1.0.0",
-            "is-extendable": "1.0.1"
-          },
-          "dependencies": {
-            "is-extendable": {
-              "version": "1.0.1",
-              "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz",
-              "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==",
-              "requires": {
-                "is-plain-object": "2.0.4"
-              }
-            }
-          }
-        },
-        "extglob": {
-          "version": "2.0.4",
-          "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz",
-          "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==",
-          "requires": {
-            "array-unique": "0.3.2",
-            "define-property": "1.0.0",
-            "expand-brackets": "2.1.4",
-            "extend-shallow": "2.0.1",
-            "fragment-cache": "0.2.1",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          },
-          "dependencies": {
-            "define-property": {
-              "version": "1.0.0",
-              "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz",
-              "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=",
-              "requires": {
-                "is-descriptor": "1.0.2"
-              }
-            },
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            }
-          }
-        },
-        "fill-range": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz",
-          "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=",
-          "requires": {
-            "extend-shallow": "2.0.1",
-            "is-number": "3.0.0",
-            "repeat-string": "1.6.1",
-            "to-regex-range": "2.1.1"
-          },
-          "dependencies": {
-            "extend-shallow": {
-              "version": "2.0.1",
-              "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-              "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-              "requires": {
-                "is-extendable": "0.1.1"
-              }
-            }
-          }
-        },
-        "is-accessor-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz",
-          "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==",
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-data-descriptor": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz",
-          "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==",
-          "requires": {
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-descriptor": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz",
-          "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==",
-          "requires": {
-            "is-accessor-descriptor": "1.0.0",
-            "is-data-descriptor": "1.0.0",
-            "kind-of": "6.0.2"
-          }
-        },
-        "is-number": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz",
-          "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=",
-          "requires": {
-            "kind-of": "3.2.2"
-          },
-          "dependencies": {
-            "kind-of": {
-              "version": "3.2.2",
-              "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-              "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-              "requires": {
-                "is-buffer": "1.1.6"
-              }
-            }
-          }
-        },
-        "isobject": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
-          "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
-        },
-        "kind-of": {
-          "version": "6.0.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz",
-          "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA=="
-        },
-        "micromatch": {
-          "version": "3.1.10",
-          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
-          "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==",
-          "requires": {
-            "arr-diff": "4.0.0",
-            "array-unique": "0.3.2",
-            "braces": "2.3.2",
-            "define-property": "2.0.2",
-            "extend-shallow": "3.0.2",
-            "extglob": "2.0.4",
-            "fragment-cache": "0.2.1",
-            "kind-of": "6.0.2",
-            "nanomatch": "1.2.9",
-            "object.pick": "1.3.0",
-            "regex-not": "1.0.2",
-            "snapdragon": "0.8.2",
-            "to-regex": "3.0.2"
-          }
-        }
-      }
-    },
-    "webpack-addons": {
-      "version": "1.1.5",
-      "resolved": "https://registry.npmjs.org/webpack-addons/-/webpack-addons-1.1.5.tgz",
-      "integrity": "sha512-MGO0nVniCLFAQz1qv22zM02QPjcpAoJdy7ED0i3Zy7SY1IecgXCm460ib7H/Wq7e9oL5VL6S2BxaObxwIcag0g==",
-      "requires": {
-        "jscodeshift": "0.4.1"
-      },
-      "dependencies": {
-        "ast-types": {
-          "version": "0.10.1",
-          "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.10.1.tgz",
-          "integrity": "sha512-UY7+9DPzlJ9VM8eY0b2TUZcZvF+1pO0hzMtAyjBYKhOmnvRlqYNYnWdtsMj0V16CGaMlpL0G1jnLbLo4AyotuQ=="
-        },
-        "async": {
-          "version": "1.5.2",
-          "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
-          "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo="
-        },
-        "esprima": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.0.tgz",
-          "integrity": "sha512-oftTcaMu/EGrEIu904mWteKIv8vMuOgGYo7EhVJJN00R/EED9DCua/xxHRdYnKtcECzVg7xOWhflvJMnqcFZjw=="
-        },
-        "jscodeshift": {
-          "version": "0.4.1",
-          "resolved": "https://registry.npmjs.org/jscodeshift/-/jscodeshift-0.4.1.tgz",
-          "integrity": "sha512-iOX6If+hsw0q99V3n31t4f5VlD1TQZddH08xbT65ZqA7T4Vkx68emrDZMUOLVvCEAJ6NpAk7DECe3fjC/t52AQ==",
-          "requires": {
-            "async": "1.5.2",
-            "babel-plugin-transform-flow-strip-types": "6.22.0",
-            "babel-preset-es2015": "6.24.1",
-            "babel-preset-stage-1": "6.24.1",
-            "babel-register": "6.26.0",
-            "babylon": "6.18.0",
-            "colors": "1.1.2",
-            "flow-parser": "0.70.0",
-            "lodash": "4.17.5",
-            "micromatch": "2.3.11",
-            "node-dir": "0.1.8",
-            "nomnom": "1.8.1",
-            "recast": "0.12.9",
-            "temp": "0.8.3",
-            "write-file-atomic": "1.3.4"
-          }
-        },
-        "recast": {
-          "version": "0.12.9",
-          "resolved": "https://registry.npmjs.org/recast/-/recast-0.12.9.tgz",
-          "integrity": "sha512-y7ANxCWmMW8xLOaiopiRDlyjQ9ajKRENBH+2wjntIbk3A6ZR1+BLQttkmSHMY7Arl+AAZFwJ10grg2T6f1WI8A==",
-          "requires": {
-            "ast-types": "0.10.1",
-            "core-js": "2.5.5",
-            "esprima": "4.0.0",
-            "private": "0.1.8",
-            "source-map": "0.6.1"
-          }
-        },
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        }
-      }
-    },
-    "webpack-cli": {
-      "version": "2.0.14",
-      "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-2.0.14.tgz",
-      "integrity": "sha512-gRoWaxSi2JWiYsn1QgOTb6ENwIeSvN1YExZ+kJ0STsTZK7bWPElW+BBBv1UnTbvcPC3v7E17mK8hlFX8DOYSGw==",
-      "requires": {
-        "chalk": "2.4.0",
-        "cross-spawn": "6.0.5",
-        "diff": "3.5.0",
-        "enhanced-resolve": "4.0.0",
-        "envinfo": "4.4.2",
-        "glob-all": "3.1.0",
-        "global-modules": "1.0.0",
-        "got": "8.3.0",
-        "import-local": "1.0.0",
-        "inquirer": "5.2.0",
-        "interpret": "1.1.0",
-        "jscodeshift": "0.5.0",
-        "listr": "0.13.0",
-        "loader-utils": "1.1.0",
-        "lodash": "4.17.5",
-        "log-symbols": "2.2.0",
-        "mkdirp": "0.5.1",
-        "p-each-series": "1.0.0",
-        "p-lazy": "1.0.0",
-        "prettier": "1.12.1",
-        "supports-color": "5.4.0",
-        "v8-compile-cache": "1.1.2",
-        "webpack-addons": "1.1.5",
-        "yargs": "11.1.0",
-        "yeoman-environment": "2.0.6",
-        "yeoman-generator": "2.0.4"
-      },
-      "dependencies": {
-        "ansi-regex": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg="
-        },
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "camelcase": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
-          "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0="
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "cliui": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/cliui/-/cliui-4.0.0.tgz",
-          "integrity": "sha512-nY3W5Gu2racvdDk//ELReY+dHjb9PlIcVDFXP72nVIhq2Gy3LuVXYwJoPVudwQnv1shtohpgkdCKT2YaKY0CKw==",
-          "requires": {
-            "string-width": "2.1.1",
-            "strip-ansi": "4.0.0",
-            "wrap-ansi": "2.1.0"
-          }
-        },
-        "cross-spawn": {
-          "version": "6.0.5",
-          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz",
-          "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==",
-          "requires": {
-            "nice-try": "1.0.4",
-            "path-key": "2.0.1",
-            "semver": "5.5.0",
-            "shebang-command": "1.2.0",
-            "which": "1.3.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "inquirer": {
-          "version": "5.2.0",
-          "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-5.2.0.tgz",
-          "integrity": "sha512-E9BmnJbAKLPGonz0HeWHtbKf+EeSP93paWO3ZYoUpq/aowXvYGjjCSuashhXPpzbArIjBbji39THkxTz9ZeEUQ==",
-          "requires": {
-            "ansi-escapes": "3.1.0",
-            "chalk": "2.4.0",
-            "cli-cursor": "2.1.0",
-            "cli-width": "2.2.0",
-            "external-editor": "2.2.0",
-            "figures": "2.0.0",
-            "lodash": "4.17.5",
-            "mute-stream": "0.0.7",
-            "run-async": "2.3.0",
-            "rxjs": "5.5.10",
-            "string-width": "2.1.1",
-            "strip-ansi": "4.0.0",
-            "through": "2.3.8"
-          }
-        },
-        "os-locale": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz",
-          "integrity": "sha512-3sslG3zJbEYcaC4YVAvDorjGxc7tv6KVATnLPZONiljsUncvihe9BQoVCEs0RZ1kmf4Hk9OBqlZfJZWI4GanKA==",
-          "requires": {
-            "execa": "0.7.0",
-            "lcid": "1.0.0",
-            "mem": "1.1.0"
-          }
-        },
-        "rxjs": {
-          "version": "5.5.10",
-          "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-5.5.10.tgz",
-          "integrity": "sha512-SRjimIDUHJkon+2hFo7xnvNC4ZEHGzCRwh9P7nzX3zPkCGFEg/tuElrNR7L/rZMagnK2JeH2jQwPRpmyXyLB6A==",
-          "requires": {
-            "symbol-observable": "1.0.1"
-          }
-        },
-        "strip-ansi": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
-          "requires": {
-            "ansi-regex": "3.0.0"
-          }
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        },
-        "symbol-observable": {
-          "version": "1.0.1",
-          "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.0.1.tgz",
-          "integrity": "sha1-g0D8RwLDEi310iKI+IKD9RPT/dQ="
-        },
-        "which-module": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
-          "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho="
-        },
-        "y18n": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz",
-          "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE="
-        },
-        "yargs": {
-          "version": "11.1.0",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-11.1.0.tgz",
-          "integrity": "sha512-NwW69J42EsCSanF8kyn5upxvjp5ds+t3+udGBeTbFnERA+lF541DDpMawzo4z6W/QrzNM18D+BPMiOBibnFV5A==",
-          "requires": {
-            "cliui": "4.0.0",
-            "decamelize": "1.2.0",
-            "find-up": "2.1.0",
-            "get-caller-file": "1.0.2",
-            "os-locale": "2.1.0",
-            "require-directory": "2.1.1",
-            "require-main-filename": "1.0.1",
-            "set-blocking": "2.0.0",
-            "string-width": "2.1.1",
-            "which-module": "2.0.0",
-            "y18n": "3.2.1",
-            "yargs-parser": "9.0.2"
-          }
-        },
-        "yargs-parser": {
-          "version": "9.0.2",
-          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-9.0.2.tgz",
-          "integrity": "sha1-nM9qQ0YP5O1Aqbto9I1DuKaMwHc=",
-          "requires": {
-            "camelcase": "4.1.0"
-          }
-        }
-      }
-    },
-    "webpack-dev-middleware": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/webpack-dev-middleware/-/webpack-dev-middleware-3.0.1.tgz",
-      "integrity": "sha512-JCturcEZNGA0KHEpOJVRTC/VVazTcPfpR9c1Au6NO9a+jxCRchMi87Qe7y3JeOzc0v5eMMKpuGBnPdN52NA+CQ==",
-      "requires": {
-        "loud-rejection": "1.6.0",
-        "memory-fs": "0.4.1",
-        "mime": "2.3.1",
-        "path-is-absolute": "1.0.1",
-        "range-parser": "1.2.0",
-        "url-join": "4.0.0",
-        "webpack-log": "1.2.0"
-      },
-      "dependencies": {
-        "mime": {
-          "version": "2.3.1",
-          "resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz",
-          "integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg=="
-        }
-      }
-    },
-    "webpack-dev-server": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-3.1.1.tgz",
-      "integrity": "sha512-u5lz6REb3+KklgSIytUIOrmWgnpgFmfj/+I+GBXurhEoCsHXpG9twk4NO3bsu72GC9YtxIsiavjfRdhmNt0A/A==",
-      "requires": {
-        "ansi-html": "0.0.7",
-        "array-includes": "3.0.3",
-        "bonjour": "3.5.0",
-        "chokidar": "2.0.3",
-        "compression": "1.7.2",
-        "connect-history-api-fallback": "1.5.0",
-        "debug": "3.1.0",
-        "del": "3.0.0",
-        "express": "4.16.3",
-        "html-entities": "1.2.1",
-        "http-proxy-middleware": "0.17.4",
-        "import-local": "1.0.0",
-        "internal-ip": "1.2.0",
-        "ip": "1.1.5",
-        "killable": "1.0.0",
-        "loglevel": "1.6.1",
-        "opn": "5.3.0",
-        "portfinder": "1.0.13",
-        "selfsigned": "1.10.2",
-        "serve-index": "1.9.1",
-        "sockjs": "0.3.19",
-        "sockjs-client": "1.1.4",
-        "spdy": "3.4.7",
-        "strip-ansi": "3.0.1",
-        "supports-color": "5.4.0",
-        "webpack-dev-middleware": "3.0.1",
-        "webpack-log": "1.2.0",
-        "yargs": "9.0.1"
-      },
-      "dependencies": {
-        "debug": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-          "requires": {
-            "ms": "2.0.0"
-          }
-        },
-        "del": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/del/-/del-3.0.0.tgz",
-          "integrity": "sha1-U+z2mf/LyzljdpGrE7rxYIGXZuU=",
-          "requires": {
-            "globby": "6.1.0",
-            "is-path-cwd": "1.0.0",
-            "is-path-in-cwd": "1.0.1",
-            "p-map": "1.2.0",
-            "pify": "3.0.0",
-            "rimraf": "2.6.2"
-          }
-        },
-        "globby": {
-          "version": "6.1.0",
-          "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz",
-          "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=",
-          "requires": {
-            "array-union": "1.0.2",
-            "glob": "7.1.2",
-            "object-assign": "4.1.1",
-            "pify": "2.3.0",
-            "pinkie-promise": "2.0.1"
-          },
-          "dependencies": {
-            "pify": {
-              "version": "2.3.0",
-              "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-              "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw="
-            }
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "webpack-log": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/webpack-log/-/webpack-log-1.2.0.tgz",
-      "integrity": "sha512-U9AnICnu50HXtiqiDxuli5gLB5PGBo7VvcHx36jRZHwK4vzOYLbImqT4lwWwoMHdQWwEKw736fCHEekokTEKHA==",
-      "requires": {
-        "chalk": "2.4.0",
-        "log-symbols": "2.2.0",
-        "loglevelnext": "1.0.4",
-        "uuid": "3.2.1"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "webpack-merge": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/webpack-merge/-/webpack-merge-4.1.2.tgz",
-      "integrity": "sha512-/0QYwW/H1N/CdXYA2PNPVbsxO3u2Fpz34vs72xm03SRfg6bMNGfMJIQEpQjKRvkG2JvT6oRJFpDtSrwbX8Jzvw==",
-      "requires": {
-        "lodash": "4.17.5"
-      }
-    },
-    "webpack-sources": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.1.0.tgz",
-      "integrity": "sha512-aqYp18kPphgoO5c/+NaUvEeACtZjMESmDChuD3NBciVpah3XpMEU9VAAtIaB1BsfJWWTSdv8Vv1m3T0aRk2dUw==",
-      "requires": {
-        "source-list-map": "2.0.0",
-        "source-map": "0.6.1"
-      },
-      "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        }
-      }
-    },
-    "websocket-driver": {
-      "version": "0.7.0",
-      "resolved": "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.0.tgz",
-      "integrity": "sha1-DK+dLXVdk67gSdS90NP+LMoqJOs=",
-      "requires": {
-        "http-parser-js": "0.4.11",
-        "websocket-extensions": "0.1.3"
-      }
-    },
-    "websocket-extensions": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.3.tgz",
-      "integrity": "sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg=="
-    },
-    "whet.extend": {
-      "version": "0.9.9",
-      "resolved": "https://registry.npmjs.org/whet.extend/-/whet.extend-0.9.9.tgz",
-      "integrity": "sha1-+HfVv2SMl+WqVC+twW1qJZucEaE="
-    },
-    "which": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/which/-/which-1.3.0.tgz",
-      "integrity": "sha512-xcJpopdamTuY5duC/KnTTNBraPK54YwpenP4lzxU8H91GudWpFv38u0CKjclE1Wi2EH2EDz5LRcHcKbCIzqGyg==",
-      "requires": {
-        "isexe": "2.0.0"
-      }
-    },
-    "which-module": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz",
-      "integrity": "sha1-u6Y8qGGUiZT/MHc2CJ47lgJsKk8="
-    },
-    "wide-align": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.2.tgz",
-      "integrity": "sha512-ijDLlyQ7s6x1JgCLur53osjm/UXUYD9+0PbYKrBsYisYXzCxN+HC3mYDNy/dWdmf3AwqwU3CXwDCvsNgGK1S0w==",
-      "requires": {
-        "string-width": "1.0.2"
-      },
-      "dependencies": {
-        "is-fullwidth-code-point": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-          "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
-          "requires": {
-            "number-is-nan": "1.0.1"
-          }
-        },
-        "string-width": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
-          "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
-          "requires": {
-            "code-point-at": "1.1.0",
-            "is-fullwidth-code-point": "1.0.0",
-            "strip-ansi": "3.0.1"
-          }
-        }
-      }
-    },
-    "window-size": {
-      "version": "0.1.0",
-      "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.0.tgz",
-      "integrity": "sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0="
-    },
-    "with": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/with/-/with-5.1.1.tgz",
-      "integrity": "sha1-+k2qktrzLE6pTtRTyB8EaGtXXf4=",
-      "requires": {
-        "acorn": "3.3.0",
-        "acorn-globals": "3.1.0"
-      },
-      "dependencies": {
-        "acorn": {
-          "version": "3.3.0",
-          "resolved": "https://registry.npmjs.org/acorn/-/acorn-3.3.0.tgz",
-          "integrity": "sha1-ReN/s56No/JbruP/U2niu18iAXo="
-        }
-      }
-    },
-    "wordwrap": {
-      "version": "0.0.2",
-      "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz",
-      "integrity": "sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8="
-    },
-    "worker-farm": {
-      "version": "1.6.0",
-      "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.6.0.tgz",
-      "integrity": "sha512-6w+3tHbM87WnSWnENBUvA2pxJPLhQUg5LKwUQHq3r+XPhIM+Gh2R5ycbwPCyuGbNg+lPgdcnQUhuC02kJCvffQ==",
-      "requires": {
-        "errno": "0.1.7"
-      }
-    },
-    "worker-loader": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/worker-loader/-/worker-loader-1.1.1.tgz",
-      "integrity": "sha512-qJZLVS/jMCBITDzPo/RuweYSIG8VJP5P67mP/71alGyTZRe1LYJFdwLjLalY3T5ifx0bMDRD3OB6P2p1escvlg==",
-      "requires": {
-        "loader-utils": "1.1.0",
-        "schema-utils": "0.4.5"
-      }
-    },
-    "wrap-ansi": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
-      "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=",
-      "requires": {
-        "string-width": "1.0.2",
-        "strip-ansi": "3.0.1"
-      },
-      "dependencies": {
-        "is-fullwidth-code-point": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-          "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
-          "requires": {
-            "number-is-nan": "1.0.1"
-          }
-        },
-        "string-width": {
-          "version": "1.0.2",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
-          "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
-          "requires": {
-            "code-point-at": "1.1.0",
-            "is-fullwidth-code-point": "1.0.0",
-            "strip-ansi": "3.0.1"
-          }
-        }
-      }
-    },
-    "wrappy": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
-    },
-    "write": {
-      "version": "0.2.1",
-      "resolved": "https://registry.npmjs.org/write/-/write-0.2.1.tgz",
-      "integrity": "sha1-X8A4KOJkzqP+kUVUdvejxWbLB1c=",
-      "requires": {
-        "mkdirp": "0.5.1"
-      }
-    },
-    "write-file-atomic": {
-      "version": "1.3.4",
-      "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-1.3.4.tgz",
-      "integrity": "sha1-+Aek8LHZ6ROuekgRLmzDrxmRtF8=",
-      "requires": {
-        "graceful-fs": "4.1.11",
-        "imurmurhash": "0.1.4",
-        "slide": "1.1.6"
-      }
-    },
-    "ws": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-1.1.2.tgz",
-      "integrity": "sha1-iiRPoFJAHgjJiGz0SoUYnh/UBn8=",
-      "requires": {
-        "options": "0.0.6",
-        "ultron": "1.0.2"
-      }
-    },
-    "wtf-8": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/wtf-8/-/wtf-8-1.0.0.tgz",
-      "integrity": "sha1-OS2LotDxw00e4tYw8V0O+2jhBIo="
-    },
-    "xmlhttprequest-ssl": {
-      "version": "1.5.3",
-      "resolved": "https://registry.npmjs.org/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.3.tgz",
-      "integrity": "sha1-GFqIjATspGw+QHDZn3tJ3jUomS0="
-    },
-    "xtend": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
-      "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68="
-    },
-    "y18n": {
-      "version": "4.0.0",
-      "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
-      "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w=="
-    },
-    "yallist": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
-      "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
-    },
-    "yargs": {
-      "version": "9.0.1",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-9.0.1.tgz",
-      "integrity": "sha1-UqzCP+7Kw0BCB47njAwAf1CF20w=",
-      "requires": {
-        "camelcase": "4.1.0",
-        "cliui": "3.2.0",
-        "decamelize": "1.2.0",
-        "get-caller-file": "1.0.2",
-        "os-locale": "2.1.0",
-        "read-pkg-up": "2.0.0",
-        "require-directory": "2.1.1",
-        "require-main-filename": "1.0.1",
-        "set-blocking": "2.0.0",
-        "string-width": "2.1.1",
-        "which-module": "2.0.0",
-        "y18n": "3.2.1",
-        "yargs-parser": "7.0.0"
-      },
-      "dependencies": {
-        "camelcase": {
-          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz",
-          "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0="
-        },
-        "cliui": {
-          "version": "3.2.0",
-          "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz",
-          "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=",
-          "requires": {
-            "string-width": "1.0.2",
-            "strip-ansi": "3.0.1",
-            "wrap-ansi": "2.1.0"
-          },
-          "dependencies": {
-            "string-width": {
-              "version": "1.0.2",
-              "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
-              "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
-              "requires": {
-                "code-point-at": "1.1.0",
-                "is-fullwidth-code-point": "1.0.0",
-                "strip-ansi": "3.0.1"
-              }
-            }
-          }
-        },
-        "is-fullwidth-code-point": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
-          "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
-          "requires": {
-            "number-is-nan": "1.0.1"
-          }
-        },
-        "load-json-file": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz",
-          "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=",
-          "requires": {
-            "graceful-fs": "4.1.11",
-            "parse-json": "2.2.0",
-            "pify": "2.3.0",
-            "strip-bom": "3.0.0"
-          }
-        },
-        "os-locale": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-2.1.0.tgz",
-          "integrity": "sha512-3sslG3zJbEYcaC4YVAvDorjGxc7tv6KVATnLPZONiljsUncvihe9BQoVCEs0RZ1kmf4Hk9OBqlZfJZWI4GanKA==",
-          "requires": {
-            "execa": "0.7.0",
-            "lcid": "1.0.0",
-            "mem": "1.1.0"
-          }
-        },
-        "path-type": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz",
-          "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=",
-          "requires": {
-            "pify": "2.3.0"
-          }
-        },
-        "pify": {
-          "version": "2.3.0",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw="
-        },
-        "read-pkg": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz",
-          "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=",
-          "requires": {
-            "load-json-file": "2.0.0",
-            "normalize-package-data": "2.4.0",
-            "path-type": "2.0.0"
-          }
-        },
-        "read-pkg-up": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz",
-          "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=",
-          "requires": {
-            "find-up": "2.1.0",
-            "read-pkg": "2.0.0"
-          }
-        },
-        "strip-bom": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM="
-        },
-        "which-module": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz",
-          "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho="
-        },
-        "y18n": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz",
-          "integrity": "sha1-bRX7qITAhnnA136I53WegR4H+kE="
-        },
-        "yargs-parser": {
-          "version": "7.0.0",
-          "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-7.0.0.tgz",
-          "integrity": "sha1-jQrELxbqVd69MyyvTEA4s+P139k=",
-          "requires": {
-            "camelcase": "4.1.0"
-          }
-        }
-      }
-    },
-    "yargs-parser": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-5.0.0.tgz",
-      "integrity": "sha1-J17PDX/+Bcd+ZOfIbkzZS/DhIoo=",
-      "requires": {
-        "camelcase": "3.0.0"
-      },
-      "dependencies": {
-        "camelcase": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz",
-          "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo="
-        }
-      }
-    },
-    "yeast": {
-      "version": "0.1.2",
-      "resolved": "https://registry.npmjs.org/yeast/-/yeast-0.1.2.tgz",
-      "integrity": "sha1-AI4G2AlDIMNy28L47XagymyKxBk="
-    },
-    "yeoman-environment": {
-      "version": "2.0.6",
-      "resolved": "https://registry.npmjs.org/yeoman-environment/-/yeoman-environment-2.0.6.tgz",
-      "integrity": "sha512-jzHBTTy8EPI4ImV8dpUMt+Q5zELkSU5xvGpndHcHudQ4tqN6YgIWaCGmRFl+HDchwRUkcgyjQ+n6/w5zlJBCPg==",
-      "requires": {
-        "chalk": "2.4.0",
-        "debug": "3.1.0",
-        "diff": "3.5.0",
-        "escape-string-regexp": "1.0.5",
-        "globby": "6.1.0",
-        "grouped-queue": "0.3.3",
-        "inquirer": "3.3.0",
-        "is-scoped": "1.0.0",
-        "lodash": "4.17.5",
-        "log-symbols": "2.2.0",
-        "mem-fs": "1.1.3",
-        "text-table": "0.2.0",
-        "untildify": "3.0.2"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "debug": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-          "requires": {
-            "ms": "2.0.0"
-          }
-        },
-        "globby": {
-          "version": "6.1.0",
-          "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz",
-          "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=",
-          "requires": {
-            "array-union": "1.0.2",
-            "glob": "7.1.2",
-            "object-assign": "4.1.1",
-            "pify": "2.3.0",
-            "pinkie-promise": "2.0.1"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "pify": {
-          "version": "2.3.0",
-          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
-          "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    },
-    "yeoman-generator": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/yeoman-generator/-/yeoman-generator-2.0.4.tgz",
-      "integrity": "sha512-Sgvz3MAkOpEIobcpW3rjEl6bOTNnl8SkibP9z7hYKfIGIlw0QDC2k0MAeXvyE2pLqc2M0Duql+6R7/W9GrJojg==",
-      "requires": {
-        "async": "2.6.0",
-        "chalk": "2.4.0",
-        "cli-table": "0.3.1",
-        "cross-spawn": "5.1.0",
-        "dargs": "5.1.0",
-        "dateformat": "3.0.3",
-        "debug": "3.1.0",
-        "detect-conflict": "1.0.1",
-        "error": "7.0.2",
-        "find-up": "2.1.0",
-        "github-username": "4.1.0",
-        "istextorbinary": "2.2.1",
-        "lodash": "4.17.5",
-        "make-dir": "1.2.0",
-        "mem-fs-editor": "3.0.2",
-        "minimist": "1.2.0",
-        "pretty-bytes": "4.0.2",
-        "read-chunk": "2.1.0",
-        "read-pkg-up": "3.0.0",
-        "rimraf": "2.6.2",
-        "run-async": "2.3.0",
-        "shelljs": "0.8.1",
-        "text-table": "0.2.0",
-        "through2": "2.0.3",
-        "yeoman-environment": "2.0.6"
-      },
-      "dependencies": {
-        "ansi-styles": {
-          "version": "3.2.1",
-          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
-          "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
-          "requires": {
-            "color-convert": "1.9.1"
-          }
-        },
-        "chalk": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.0.tgz",
-          "integrity": "sha512-Wr/w0f4o9LuE7K53cD0qmbAMM+2XNLzR29vFn5hqko4sxGlUsyy363NvmyGIyk5tpe9cjTr9SJYbysEyPkRnFw==",
-          "requires": {
-            "ansi-styles": "3.2.1",
-            "escape-string-regexp": "1.0.5",
-            "supports-color": "5.4.0"
-          }
-        },
-        "debug": {
-          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
-          "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
-          "requires": {
-            "ms": "2.0.0"
-          }
-        },
-        "has-flag": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-          "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
-        },
-        "load-json-file": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz",
-          "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=",
-          "requires": {
-            "graceful-fs": "4.1.11",
-            "parse-json": "4.0.0",
-            "pify": "3.0.0",
-            "strip-bom": "3.0.0"
-          }
-        },
-        "minimist": {
-          "version": "1.2.0",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
-          "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
-        },
-        "parse-json": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz",
-          "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=",
-          "requires": {
-            "error-ex": "1.3.1",
-            "json-parse-better-errors": "1.0.2"
-          }
-        },
-        "read-pkg": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz",
-          "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=",
-          "requires": {
-            "load-json-file": "4.0.0",
-            "normalize-package-data": "2.4.0",
-            "path-type": "3.0.0"
-          }
-        },
-        "read-pkg-up": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz",
-          "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=",
-          "requires": {
-            "find-up": "2.1.0",
-            "read-pkg": "3.0.0"
-          }
-        },
-        "strip-bom": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz",
-          "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM="
-        },
-        "supports-color": {
-          "version": "5.4.0",
-          "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz",
-          "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==",
-          "requires": {
-            "has-flag": "3.0.0"
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 87a516f584d78..44bcd6f1bd951 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -1,31 +1,25 @@
 {
   "name": "ignite-web-console",
-  "version": "1.0.0",
+  "version": "2.5.0",
   "description": "Interactive Web console for configuration, executing SQL queries and monitoring of Apache Ignite Cluster",
   "private": true,
+  "main": "index.js",
   "scripts": {
     "start": "webpack-dev-server --config ./webpack/webpack.dev.babel.js",
     "dev": "npm start",
     "build": "webpack --config ./webpack/webpack.prod.babel.js",
     "test": "karma start ./test/karma.conf.js",
     "test-watch": "npm test -- --no-single-run",
-    "eslint": "eslint --format node_modules/eslint-friendly-formatter app/ controllers/ ignite_modules/ -- --eff-by-issue"
+    "eslint": "eslint --format node_modules/eslint-friendly-formatter app/ -- --eff-by-issue"
   },
-  "author": "",
-  "contributors": [
-    {
-      "name": "",
-      "email": ""
-    }
-  ],
   "license": "Apache-2.0",
   "keywords": [
     "Apache Ignite Web console"
   ],
   "homepage": "https://ignite.apache.org/",
   "engines": {
-    "npm": "3.x.x",
-    "node": "6.x.x"
+    "npm": ">=5.0.0",
+    "node": ">=8.0.0 <10.0.0"
   },
   "os": [
     "darwin",
@@ -33,122 +27,125 @@
     "win32"
   ],
   "dependencies": {
-    "@uirouter/angularjs": "^1.0.11",
-    "@uirouter/core": "^5.0.11",
-    "@uirouter/rx": "^0.4.1",
-    "@uirouter/visualizer": "^4.0.2",
-    "angular": "1.6.6",
+    "@uirouter/angularjs": "1.0.18",
+    "@uirouter/core": "5.0.19",
+    "@uirouter/rx": "0.4.1",
+    "@uirouter/visualizer": "4.0.2",
+    "angular": "1.7.2",
     "angular-acl": "0.1.10",
-    "angular-animate": "1.6.6",
+    "angular-animate": "1.7.2",
     "angular-aria": "1.6.6",
-    "angular-cookies": "1.6.6",
+    "angular-cookies": "1.7.2",
     "angular-drag-and-drop-lists": "1.4.0",
     "angular-gridster": "0.13.14",
     "angular-messages": "1.6.9",
     "angular-motion": "0.4.4",
     "angular-nvd3": "1.0.9",
-    "angular-retina": "0.4.0",
-    "angular-sanitize": "1.6.6",
+    "angular-sanitize": "1.7.2",
     "angular-smart-table": "2.1.8",
     "angular-socket-io": "0.7.0",
     "angular-strap": "2.3.12",
     "angular-translate": "2.16.0",
     "angular-tree-control": "0.2.28",
     "angular-ui-carousel": "0.1.10",
-    "angular-ui-grid": "^4.4.3",
-    "angular-ui-validate": "^1.2.3",
-    "angular1-async-filter": "^1.1.0",
-    "babel-core": "6.26.0",
-    "babel-eslint": "7.2.3",
-    "babel-loader": "7.1.4",
-    "babel-plugin-add-module-exports": "0.2.1",
-    "babel-plugin-transform-object-rest-spread": "6.26.0",
-    "babel-plugin-transform-runtime": "6.23.0",
-    "babel-polyfill": "6.26.0",
-    "babel-preset-es2015": "6.24.1",
-    "babel-preset-stage-1": "6.24.1",
-    "babel-runtime": "6.26.0",
-    "bootstrap-sass": "3.3.7",
+    "angular-ui-grid": "4.6.1",
+    "angular-ui-validate": "1.2.3",
+    "angular1-async-filter": "1.1.0",
     "brace": "0.10.0",
     "browser-update": "2.1.9",
     "bson-objectid": "1.1.5",
-    "copy-webpack-plugin": "^4.5.1",
-    "css-loader": "0.28.7",
-    "eslint": "4.3.0",
-    "eslint-friendly-formatter": "3.0.0",
-    "eslint-loader": "1.9.0",
-    "eslint-plugin-babel": "4.1.1",
-    "expose-loader": "0.7.5",
-    "extract-text-webpack-plugin": "^4.0.0-beta.0",
-    "file-loader": "1.1.11",
     "file-saver": "1.3.3",
     "font-awesome": "4.7.0",
-    "html-loader": "1.0.0-alpha.0",
-    "html-webpack-plugin": "3.2.0",
     "jquery": "3.2.1",
     "json-bigint": "0.2.3",
-    "json-loader": "0.5.7",
-    "jsondiffpatch": "^0.2.5",
+    "jsondiffpatch": "0.2.5",
     "jszip": "3.1.5",
-    "lodash": "4.17.5",
-    "natural-compare-lite": "^1.4.0",
-    "node-sass": "4.8.3",
+    "lodash": "4.17.10",
+    "natural-compare-lite": "1.4.0",
     "nvd3": "1.8.6",
-    "outdent": "^0.5.0",
+    "outdent": "0.5.0",
     "pako": "1.0.6",
-    "progress-bar-webpack-plugin": "1.11.0",
-    "pug-html-loader": "1.1.0",
-    "pug-loader": "2.4.0",
-    "resolve-url-loader": "2.1.0",
     "roboto-font": "0.1.0",
     "rxjs": "5.4.2",
-    "sass-loader": "6.0.7",
     "socket.io-client": "1.7.3",
-    "style-loader": "0.19.0",
-    "svg-sprite-loader": "3.0.7",
-    "tf-metatags": "2.0.0",
-    "uglifyjs-webpack-plugin": "1.2.4",
-    "webpack": "4.5.0",
-    "webpack-cli": "2.0.14",
-    "webpack-dev-server": "3.1.1",
-    "webpack-merge": "4.1.2",
-    "worker-loader": "^1.1.1"
+    "tf-metatags": "2.0.0"
   },
   "devDependencies": {
-    "@types/angular": "^1.6.32",
-    "@types/angular-animate": "^1.5.8",
-    "@types/angular-mocks": "^1.5.11",
-    "@types/angular-strap": "^2.3.1",
-    "@types/chai": "^4.1.2",
-    "@types/lodash": "^4.14.77",
-    "@types/mocha": "^2.2.48",
-    "@types/sinon": "^4.0.0",
+    "@types/angular": "1.6.48",
+    "@types/angular-animate": "1.5.10",
+    "@types/angular-mocks": "1.5.12",
+    "@types/angular-strap": "2.3.1",
+    "@types/chai": "4.1.4",
+    "@types/copy-webpack-plugin": "4.4.1",
+    "@types/karma": "1.7.4",
+    "@types/lodash": "4.14.110",
+    "@types/mini-css-extract-plugin": "0.2.0",
+    "@types/mocha": "2.2.48",
+    "@types/sinon": "4.0.0",
+    "@types/socket.io-client": "1.4.32",
     "@types/ui-grid": "0.0.38",
-    "@types/webpack": "^4.1.2",
-    "@types/webpack-merge": "^4.1.3",
-    "angular-mocks": "^1.6.9",
+    "@types/webpack": "4.4.5",
+    "@types/webpack-merge": "4.1.3",
+    "angular-mocks": "1.6.9",
     "app-root-path": "2.0.1",
+    "babel-core": "6.26.0",
+    "babel-eslint": "7.2.3",
+    "babel-loader": "7.1.4",
+    "babel-plugin-add-module-exports": "0.2.1",
+    "babel-plugin-transform-object-rest-spread": "6.26.0",
+    "babel-plugin-transform-runtime": "6.23.0",
+    "babel-polyfill": "6.26.0",
+    "babel-preset-es2015": "6.24.1",
+    "babel-preset-stage-1": "6.24.1",
+    "babel-runtime": "6.26.0",
+    "bootstrap-sass": "3.3.7",
     "chai": "4.1.0",
     "chalk": "2.1.0",
+    "copy-webpack-plugin": "4.5.2",
+    "css-loader": "0.28.7",
+    "eslint": "4.3.0",
+    "eslint-friendly-formatter": "3.0.0",
+    "eslint-loader": "1.9.0",
+    "eslint-plugin-babel": "4.1.1",
+    "expose-loader": "0.7.5",
+    "file-loader": "1.1.11",
     "glob": "7.1.2",
     "globby": "8.0.1",
-    "ignore-loader": "^0.1.2",
+    "html-loader": "1.0.0-alpha.0",
+    "html-webpack-plugin": "3.2.0",
+    "ignore-loader": "0.1.2",
     "jasmine-core": "2.6.4",
+    "json-loader": "0.5.7",
     "karma": "2.0.0",
     "karma-babel-preprocessor": "6.0.1",
-    "karma-chrome-launcher": "^2.2.0",
+    "karma-chrome-launcher": "2.2.0",
     "karma-mocha": "1.3.0",
     "karma-mocha-reporter": "2.2.3",
     "karma-teamcity-reporter": "1.0.0",
     "karma-webpack": "4.0.0-beta.0",
+    "mini-css-extract-plugin": "0.4.1",
     "mocha": "3.4.2",
     "mocha-teamcity-reporter": "1.1.1",
     "node-fetch": "1.7.3",
+    "node-sass": "4.9.0",
     "progress": "2.0.0",
+    "progress-bar-webpack-plugin": "1.11.0",
+    "pug-html-loader": "1.1.0",
+    "pug-loader": "2.4.0",
+    "resolve-url-loader": "2.1.0",
+    "sass-loader": "6.0.7",
     "sinon": "2.3.8",
     "slash": "1.0.0",
+    "style-loader": "0.19.0",
+    "svg-sprite-loader": "3.9.0",
     "teamcity-service-messages": "0.1.9",
     "type-detect": "4.0.3",
+    "uglifyjs-webpack-plugin": "1.2.4",
+    "webpack": "4.12.0",
+    "webpack-cli": "2.0.14",
+    "webpack-dev-server": "3.1.4",
+    "webpack-merge": "4.1.3",
+    "worker-loader": "2.0.0",
     "yargs": "9.0.1"
   }
 }
diff --git a/modules/web-console/frontend/public/images/cache.png b/modules/web-console/frontend/public/images/cache.png
index 3ff310379c05abc75e687ab0e08d9a2026a3b083..2fb3bc827d6fa1308bc035265d7088a657a98c2b 100644
GIT binary patch
literal 15087
zcmbumbzB?o^Di0-MT$EF*WyKjTY(_OY4PGv8lbpqan}OHB}q%6K#K&YNO35T;BLVw
z6sP#*^ZEYnx#!&TI_JL5>*kNm?(EF$%=1j%?|F7Nak|>7ginA^0002t3pMad000vL
z06b#H#e9&Mar~`#IJnZ))K|X0zrVb^yt}*Y?(W{--(Or@ytug7+}vDPSdfvCIXgS^
z^76X5zCJxYY5swJA31b=b9eUlIyg93^Hsfqf`X!Y&1&ncx!sSCA3u(cjyk&i%*vZI
zFl$;_S($<l^}cTy8XCH}xW2w#?Ca~Rb$|Ke$B)6>2s9e~Gr|3@kf@zw$KLM?U8BY_
z`SgTTWNvP5V`HO(gTwC5&d8^fmF|X_nVEWDleDzdfPlb_-SgCpvE$9<Z{NNpWWj4{
zYF_H;swk^``ZDL@;&Oa^92OQ9A0OWmY(1RPk(89Ap`qdH>$}$9`sdG|!oorgIhpbC
zaWgYB6BCnf)l2Q28+DDVrR57<Y5vmjJ>G$R)p`Y0+If{~+102W7z}oMy(=qkRBSH!
z=k~U!2i?&-VrOSJaeWsV8M%ISvpult>fId^6LYxuyXpL<2tLCt^rifb!p+Ue+3&6E
zt=i9@KcAdj8(X%->B&q^PTpK2fA(x$?46G1_scwY8ZXcKvT(aIF;rerKHE@I^-g0A
zeLC=K*WA*^z|g?IY9=OOq^ztge)#I<{LCO{SX$|On@eSjeOZG!e5}Y<R#wAFUgqXc
z_h4-6?ZJ#-<j=&GB}LCB8E^n{>~}@v`?&gz>-m!Tg(FMo!We2yHL%O*!*qD*Y~R|+
zCbFyN@NQ}4_)<l^#b051=WK0l&EfO7_s8Ctz``Gq4!g*WhY!N2_<At<{D)uV=G9Gd
z>Xn!8a%WlY@VWE%ordPMgRb<Ce(96b2dmX_TeTjy^HTvS2zjH}D=XXEPbJDaCd&=e
zbLGg}o}OQMKRdmvUn|Pr@9kx5-GP()%_5o-g8ef5Y(SO!B+07>6W{zI%9pau<n6x{
zh6gWHkMlbZJa%Y=3C8z^YHYT5&y|I{ny9B)D6}k}`bQMl`KD66A5Z$!U;3f`%0KA%
z=#Ou3oX=Y$w}m^icahyC5kv9%Rr#?6yixF`j_Zi%q{`Ozx}1(+utER%{=!^hRC8B|
zD!XHY$BRi306@Iz1z17fZ|--#4uo8RBBmg{)wUQ%){m4poLUbVQLLa#VXcRyV?8aO
z5sI6n5FhiE84UGvAPdJ%?D-lKzt%}vK^+lRNR<(qo1P6hBy&_oM&N9{$uC1gDiQ>+
zLzqBAUL_~>hjL3JzY)AxfR!Rl6u>PE0w7=`Ap}szVs{8W`acq6b%Te7$GrV>KJ#*u
zCe+0^m=gRf+5jBIG2P>vOR@sgNT~KtB^=_@71Kf?bz=x)X%!Jg_vJjls$V(F7h|Ah
zS*h(_X5im;UDz;_F2!b~VnV&^wZdi2v&ZRcDSv-JntImk_)y2ruuBF;RPB8A+R68G
z5C=ewfNhS=+7@hk7X^{o>YkN=W%iY&k^X$ofI4FnTJzL#GXxw`U#2MImj$p!Lz<Mo
zwvlQsrkC`_SqiKUkg_N7njW~1YWRw1I3a@K*cT>bTn~zi^GFllmu#VK67qaf)K5J`
zfZPkZCgze8jCfS1KX<cWmwACpYO4dzvgXaDK$g0xg#;k{jc0ApLs-+W7Z}fBjj#=i
z0B|68Y$9Z%M04ZkNP^<4i&8&@kR#uzZ)i$#y|b{zuDCYc59FlgHcf*Jlb))1hO>Tr
zd$IN<PHvNfTK-?%)0Oie<aIR{-VEvL-v+zmEz`m<)F_M)rrj+Bnf!L(>myXPA?&Tm
z>i0iif(pWiJUSW|`fCFuPzjE(w&%7L3GzL%8jY~Box-f`W&*l^&Yq0?JZf9$DZl6L
zYZg#<Y{~|$8jbICwP2r9ac}9$ubWIrlhMQ)=^f3p@DiV;jc~oKn}&cXMtk|cZ%xpt
zqGL9+W+;oYf{UxAsnUFfeWK{Xd}(&WZc={jAOKl!ti1(%0DI3?Ovw!O@ySY8rE1is
z+>q7T_HgVtga2@_;iEX^HsFJY-AbYt{A;B=&7{Am_D9iZN}yV7XG-$M2ZR;@8UJ?`
znvr1Bx;?*Fo9&Zu@`CRoH&BN+QJs{PGTI6kXsyo3Pu$J0o|KfwKs6d8(EbE7%5gz3
zV>BHW@N>7CtYApJO6llbDx_7ChExyj@%7mpk+Ss&DSt>%EVfN*E5v7!1|By=XPn+i
z86|IKhLSk0B;LO}uu{8Wh$8)cZQf^|v(eW#N5DS3qN+M+JOXr{-*vKQ=!tVVGmaan
zZAy8^6K^WAv!AHN+qCn{15QQj`Vm~a4Cj}ZW29C1NkBC$uVYjbesYr4l5D9zc2x_4
z@1}}%J%naH@uP?-@ax`J14%yg?rXjA!h>ujLlw}qU4MjpY>KTBv%eXN0BY2lSGNpz
zr_IsE)8bzvefp)<N_+is*x<l0eOk>YrC>XgNObA2^Eyh8w>7_M@QbL6cdYrjrJg98
z>UVfA*Sub9dS>vjstRbY0{vOP;xB+bAu!<my}{}qP8jI%k3ZlOdGe%f<z&hCq-MYK
zIyHZ<4%(c}#0eaucYpoR?^yHC<C3T{3NhQTZ(@r%A}~|9&GKl*Bz3PGhaG7;#)9}P
z{xjRzn_AIgMcxGM<XTh`&$3aXT18wLl;KX`JRRUJq5LA6U8+d*bX{%s4fL*=Zx{#Y
z{fa2?+IgnS9=sFfYhsQ%O(o6!WB>AK4Kr2ZuMsxjh9LGlGgzt^0|0nVGtpj$p?OFs
z1yF`{k`u;*Bc1{&;;?l<Tx5iW5M6el460ZS<OfqogB*e*SWzo+*#BpWy>bls&v5|?
zBnvV5RuUyvcajD7{@(?4CxyJ&;GH;|I2D1aE$o%&$v%WtzjNP)z8R&*<dfcHw3?OO
z0t{)x6$F7GVIVtPfeo%8{QqP3W>z#0$r;WCkjFGsqBJDI(#idbqZ7_WdW16(LAl3o
zVb^6?k3ZH01bbHL5wosO(M%aOC_AZlE5dZ~lRlNWC{#_Pf*eF?at&=M_bj<5O|h2k
zd|uhYUM9s$FumJ+?=jGqCq8SF4I$`l$0bkd<l5AU^Rd$T+*ycus39X`LQK&6encER
zZ!W3I5EKR@;%omBrwPJ_6Tl2Y@P4fZdw4YO?qABK#r0a<h@mw9Xg61F(>LCVD~pKM
zccjq9J1sHF2c^UkZ>EgVKQ4GRjAj+2c|$$y-)hnkzHE$d^(2K~ejZEkxHRzHCEZ$4
zP1B}@X7`I~y1>O1hKeJZ6Y6-*qBtX0r+YTok>ilCKNPUMHo+2F$-vR@UdTd6%(#fJ
z$+V4~wr(~OK%jD|?3v&G1iQh}#%X$HQE<&)<gAdMli#l*=zW`s8{>sc9$7)(wa4OP
zSX3>8hs5?hIm|?VdQ@42f?qbG=T?Vn&Ue+Q4I3Lxt&BNn1T*g)ounL6a|>i&RizX6
zAXVP2Ybl;ab7l(Ale&GKRBq8jgfq#zdge5vr?$u<>t2-C8`oW7B?L&x=>>8YI^#&$
zJLVe>P=IqPMi_rGI~J5~K)Pr$=+?+%0H_PkPhUmlCBpxp$psQj9h_NI`=EcjcNm4}
zSD5KZfL~s*NQz6oG^Zue6>I*kT%c}k9qS{=jno<cy6L4D`K$5c`Zp^Bn5yc=(dm~{
zrx1w(?T$X<fL45(#A6i@#s0_HcP<fAVda#RH)pD2xJ|S>h!!m-fhi+>FQP_NThK(U
zYCFYn*_LKyr_a*V&#2mcb)NZy?VXy^HgnSQKO2ze#kLy2IwDsdtZt3v8fY=i)Tb%N
z_9Y}vR~N%f*8Hezb)AGBES6JZtxltw`6V1OqQ5^DUBLnrSf^KL4^Pk&J_dvE8fr>M
zKSPU`N%&Dzk3T!}a4!VYZ@eWp<7D8Ngjds=BXmR*?;F)vflPTORbg)&6OZFJIUbVg
zmeVh*?bX~SFM}{P1TE&H#VECsTj#1h{h^TFjqcKvh$(Bl_m{|RwBKvBhH$T4TR$n#
zs?v175xYO_3i})q$6pkyQLKR})T7KGJ$5j;829h?>ovm}VKAHYU|+wDIcOn*!mfs6
zV=!T=C{$B8ouVqT1#QF0t?>Q>=DS_eO2z}ZZ%<*1ssntyunwz?TVX7GGSnlSROx&c
zpbDaE#$6ExJ3B~RG?{qK1JjG}Xb%aKV5N>NNuh<MNy$E+nlv#-Si>Uxq`-=Zq!k}<
zuOdqAEfjLo%jRv!0~95|Q47<t0A)Cxi^iX`Vsvv+5R!|to3I15ZJL1a1z4w2U~Ie!
zBL?n6TN0W0T=_b5Q9{;~M^&jM_SI)Fc{N-D798Yo;w}*<7TFZ5!s5he#>;>Lb-Lxo
zs}Ln{u$n?4$G@8*Y^=duJU>~1@Cw*uFnX9|i@e3K&Ijo!O<nU<j6iHdxvRRZ?CZc=
zR?2;->bzWI7bAbO_wSK-c48$@^>~%OkGfgMlBO}0J8yg!l<8C&yIQrD*pc)}qg#+D
zn7@!J-Ec4*_7bIn`8s-y%k+8n5XuX+9`%TlP}m{pSQ1D|SYDEM$BI%*o*MN4sWdHn
zBW8Mk6^gHik5JHxEYqEj#$gYxgl@efS5>ZEDRK-u)b@?LM<u@g`2mI!y7huANXc<z
z#k_Ki6-!iT>r{8;JYscceZObS1vF`t=NcrO`pr%`ocA^4ZYoB(vl2d}hEWDle@!8L
z`C63~SZ1-!Lzc8CB|I~KEmf&N<xX=F6(=SyXCd7z&|$G+^^2Nw?gqDyx8XAEC?{B|
z1OpHn#=O4l{rG0$i0lxc15-$WRQ<yrR{!z`**`?X@ek2x{6jP!{~;Q_e~88y)>-|!
zuavOMiYB@}9vQiEu&>MIb@e4<1gbH04w6*v=N8_@I)7J%Lns&P`={;&E=c$V9-{bA
zPQ}WEM(0`kD=_4FA~KYM6iC7bVpfjC?#3llBZXoJDPq3-U!1xxCU(YNNfAPY3f)LO
z+F=1Pt3aN|;}XV_0#jKPF>NBTKgA+LX`mR~tpA+qIIFJ{x6F`Adkw{fVobYdUe*ia
zUvYcYX_EroK95zTHIH9zc+H+rLowQkDl>9*zgD%@@Te_07Ht`p){9-p`OejEI=K=;
ztrr%z>^Q%PWR?G#OMSuft2spU*k10pQtelHqTOFlmYEoqrJ~Q<%-)rl9(<Z^=-NER
zpUptJEDlPpwmQp28aQP`c5BNwu0IDuGU*syV`kLp@*sOpFKVU^l;(aB<;A|P#NDr#
z#m#yH6)cZ|MYl=$<M$z>^XfsKjw(ud(prqQmun$){X}#<HPVVLZ(eQAic9-ctZ|X_
zx!-)T_hfOj&7=t9ONh;k5PTQkV^2tL9T|u*PvIb2rPZ>o>{4jeKm@T~jd<67n-q<o
zmyqA&ZQ^S!$G&Fu=4eA+yPWlz?qtR--u`8~BHWjF0IZOzy^mhYcl03~W(AOiQw8Dd
zplMU*)&<(Brlk;B0egOoxsZ4F(^E(B@$U(iPlQOE^*qF-1%Oo}&`RumC?&@;d^IK)
ztg_+cmYd5eZyEaFj4f!K>P=ixq)K{V)Qp=<(JYLgBYqMM>umId=;_DUzpFgN#;rr<
zIwt&)6Eu&(vb|+|Zt(-3eMe+&BWI}g5v=5?yHPO<;$_CU9~sbPEtUW}XCa)I4M4e>
zf!w5G#O9GapnV^|Ivd;uP7^G24DK?ONPUT;W3j+K>6r35cu(s#X-abuBWlkAUD>Py
zdTI{D>vY47gzxrwVK*y~U@C-~4rsK+um?Nd<2qZq@>m|)YhNH&X7M_iEd-1ZPlhM7
z5*68-Q&)<?+@tLRY)?&z=ahyT^ER-r@*#(nc%AIJq-{j}Gm0NRUF1P<y~5{19iEa8
zSJQL6;mLgZOBQH;XeZZ~b#g-8mVJ>8(N%oZT#wi|HGpKhQY{rrd1qYpg?utsydM5W
zi(n^<cE9;Ep%R%bTxwS${o5)cspb<N55rtUjaMvT!o-kb71vn2E?8Y{N|B_LS|>2}
z2VdZJs-J1>x#@&B?m7PN>StBoQ%d0ztaV7;6D7N)i?hpi&&#9MWcytjTqmT|VYp@D
z!flfhCPB<LTknaGmtf+>QzKzr(eXKXr9u(s0(*HGZICJkrnSf%wk59W!OcPeZ2yEa
zI|Fvcf&Zd3Pe=<v-P!-;{R11iyetHpq&Nx}lJUgFKUWHtw4j8HtQ~>)vh=Eq(rpYA
z-6ROHPUZCY_;)`)U&qCIg@t3kG~CJWnxiN@N|)%+;+;-~o5hqM@Rb8~BiDF^sRl})
z6v#8JGqjMKnE^b99_C&$biG|7d{fhdr=buUfRdW4Eh!(rg%K9?;fMFc<zg}dlDS|+
zz?yR2Tqkct5dAc_U~Y+;GN63fsLRH9?!?;HH8><B<j11(I~>52&s6dT(@rJ{Cp8kv
zRp-<KJA0f8Y5%QKY~bWF(gV;f4KrDoQ_A^upy}4_<X=iz3I9QIK)ruczqE_1{bA6-
z+6d&uNf%fC4WhQ5epL^VyHBGSxXOV@4j4R%ZISJh`ZH@k=bS8r3a@t$(I^Dt2n6b{
zmDcl`x&q2oekBmfb?foRsW2odD)H{iQVitBVCw3qKn)Se=M}V{;6FkF8FYkdVe!c-
zN<KZzC;_M8Ow-5asE`_Q&bgv|#B|1|FK~a`p%`2tlzn)86iI;F3prhW4^~~7H?T7-
zU8a+7G@VVpr60l4n_7Y|w{t3g#IQ;lVDBc?#52XpRKa&jIHN&@xpCwS*z1ee&s-1Q
z(|i~;4te_Z(F^5e6oe~-{D@$fr6_UvdE*b=*pK9x96nxP?J*4-W&#+SX=>K+@9o&)
z;q1-*>us{TY|P0=q^Qw?_*omvjuM0V+_qz%Yt8a0&GVc~$|pUICKe?RDURSX!<RhK
z5z@IXHV!VQnZ4t}QVpkC7eyhewGz<C=vz0!NjYb=ujeC-(<trwFK=T+3v*|8_PGiA
z(aOB+&x&6P8<ZBrcq8J<R;$xU3wkYmMz)mW1c}h2UuTn@M5HCxQl_J<NUcHhq#r9e
zZVVI@^c=&fU2Ck;o^sSDm)9`J_{?>U4J&;a3A;C~wJgta)O@2-5OpEhpU~Ed-snP0
zFtz0+E;AkO_iV1P^Qjf$IPg(YgL1{GaRb0&1y;;kJ&H{zVEznXxSm$g+PVK%vM{V(
zWZ&F)<M>fW3-;aFi@!U%c5SoK1~nrpCs#K%&11djqrbsRJ-~fEqFy7q;#pB8t_n|1
z5$QqPTL!%H44Tkdyt{#%#}E$tE*=w4_+>rT+dSg1dI)w~PO+p@bm@_1qmEPzkho`Z
z+JwvFeMa1$nhMqjb2aoMYC@@aUN$caXOQ|t>5uIX3tFqi-fYn1U*hAAZ3Z=$D&)mY
z%!QT8bc7Jb;j8tow{M42$0$w4zx=Yt%)|wmbo$2)r18|k6Sr*>*}N!>RJLV!xWnTG
z1{NuZaxcSJPlge>u9>ka<g3N}AM7M0@DVA#1)@qBz%ViM$&5d*kiRI?%cUOdb>S#I
zGg|101jOsbK?ySO=JnBo>DzxyO=?h&GF3%dh+H2mX|Ca@t0Qhm-=FLXpKSg&><RA*
zlGg@t3t}9po=csaJY)d9@EbjTFn0Ra_#5hgf{;xTA0*)v?G*?ep#T{W?|fAyPP!^_
zX41<^#IA)VA_->%7<w{ErHFTQOP*}6@|b^U;eO7p|F~qCV__@LL3EUdG>NkY3?IJl
zb{7E`t&H|Rp~sWoZ8qUJr_+)O%^Acuwtrtt2oJf*EY67Rk5_n2ukbKZAzb4fXBhkP
zhH)*x{r2KT6l{KJH5ri9SCCZS?A3QL4x;)c>WJt>^Ui&e#Q=_S0{N$b^v?YRtAzRq
z&%Ymi=^M&pxz$rN5<1X46v!V4=a9ntC}uepq<oCa&9$S=s>d6p#NnGr>0X1%janNK
zXEvDPdC?L35oh#sFyIdI#L{$5keJy7Whz9DD1p*)cifj4H$L68t87KdU=VoZ2`NcH
z&h&isH;VaPqw{$rzM+i=!Y5I5nAR)bsXwTyDZF?JsM>XLU5l?~B@#1zdE$~qh{fB`
zz@#1{7=PL2yo1dZwuZ8>2uM*X^Jg>t_4xB@NJR~1errJcy(U4ZS(j|?{bZ$R4hv|a
z%dr*5iHlHW7ux~&kQFE7du?p#4_;(OE+yo&H0e#|axpa5yT_Omhpy7(kU3M_A0JQ|
zC|KCB?ns=mGR&;|H!@u~v_SfF{Ag8V{Sys|D09u4O4rcar+dOmc8j*MOX$|Aw9y>}
z>i_aTxaUG2kTvP6^wzLAO-AvFdh_^RSw;MKG;eRb#0;d#nj-)g%%L^@NnEh-(F7zT
zLP0M*7s7B9i1-v4{6a(|vVV|{-*35vQ)InS&bBqi!=y!DZIM~a)?Z}ZYeKs3!p)Vv
z_Au7k9-bEdlWv&OR+%1xkoiTC+wt+@#Su3wN>)Vf(XjXr)xaxJ&W#8z<4`J*ixTFV
z-9XUFww0DWF5y`f@DGa7cA-RtP0Q9qBoaud%h{-Fd@F2&2OMDgRD2i+GD@VZUWQ~o
z1$b~u;$V%!F_S><<e<Cnw!1wfhtzrFMaifx%i<08XW%L8XoTrS!<1b#QkS^=h*5q<
zx;;y*Mk2oGMo{X78QQM<6Obevm&frDiv!*ZO^QSskh3z=R2JDI!Wrt<pIm<$mLr2*
zI%C$l6ruY=a?egJXXG8;j&L^RA?hzzwe80OpXB5$<TMfQuWB9yUEK=YMH450#bX`3
zbfS@v&KEMoBfrtdECoGvD%Glf-OqI!cO`1&>RO33GAMWCQDbuh{brwkcjyTL%P0$y
zE1r=q{ryCj#Q1u08D5q>Jq7dM$Hm*Q`;<ZFOhdTaV`k$`_>zW4xI2Cq1MqM&*7{B+
zYjFYN23kR-0Z3~#te}K|KCsNDV&0h^u$sPz-?(i?^_ENWGlfl%q_IrtKLf<pk!gM9
zH36acpw=bI>_1#>z0)Dz@|)~%@jY=FnBrxVmniuabtCO?p(INw?Nc-Fd&zWTH9;-q
z3E+h5LBUJ{(kRK&<+j-WQUpI|X=Ngw{qMA1p#ufu{?SqktCGXr^CzKLWmLT|@^U@Z
zN+6*i{f_jLXRI-8=O`x0f%q70#ZmEFkMaqFDK=N6MW%49Utol@E}*x^go!NA{x#6W
z{nMv(UQ~@m!Ad@Suq%W5P~B=M{Qq2GUs)GSL)iJ*s@T;uvN%=^w4}qH`(NIT%98`_
zC;tg1Tfc(%=Nto;a$}!fh2_#-5fuv;Yw<&hF~8+)@wmX8xr>wjFcgF{jRhu%!%gxK
zi7fFyk#KPq$*L?Pa1}_U68;IrK=t2P#w<VEI3PNQcB(~b$+8Q&eL)~yHKsT!#!QY*
zoM&IgMS%6sXg%m}@Rw1C&sl=m=e|U~^RULa{1F#>)}LC>%Azryx+{vJUlnS2HxIJ%
zmpQ1e!YtsXO5o6BRWcpYuVXsz2;tK*e+@r~xs1c^dpc|Cd-z0^ep?V_6Z*nV4M$}!
z`L<F1Gu^OMtZK^fp#712)cH@M+J2IcWYMLx!bd^prihH3{6h1>-^sVxK>^Mz-EdWY
zH8^2yuFv4aAo?LuzGPxOk#)B#O0Y)Gvs#Q|&+~|YfAo|%b0>T=%|zk|tXs0~CkXt-
z7(yZ35$F}nx8lEXlI;JZu^+dIBWAvt%ECuhbGMw!(s#jZI*7Gm7BowFtpGQ{IZBP=
z^ltGP<)sTr?@u*q--}ey7*36&MZMgVl8|nhpu7&-dE2Qvr66Mmdts7KaQViDhJ-i=
z@{;PQQb-OB_CH}{i1w$BIECG4V#R49ij-XHUeaia(|+U<jOVLQPwIQs8TYYu50laV
zK2xKz_d`FJ&Jj=8ZkSJaeO@I`InVf8;JH!)Dj-S~Oi0F3Ud>LWgSh^4CnxdudNEk-
z?DL|GmFaNm&dM<_)jc!gPg7<P5C_aa`orwvc&rlZh0!6dQ|jS)E5_3X8V#Era9+^K
zW(t7hFM2rdh1yvfBqt$g($;V|l^ei!ryvtq_dIb*U{sQMH+DOT#{$cRL#;VnH5<|@
z>)J8E-}}K>67|TnsQ8;D@uSpT^S2HaC0fi%xhl3t#A_@SkAH&Ep`~oXC%PBo|6Vy(
z5cnUno8`FrmN+IZn3&8T-eTpVWei*ye=@+~y2z&h9mFA3vM^^Vix#~4@56YXQR2em
zq*y_7zm6W9_#XCmWnS(3W7>K=q#pb<cXxiN|F3&quWEqUTpuPN9p}4zJ0U>gFK*eA
zLBgfXfAd2!^@vo3+#8(y#ez?+J(GFo38OV3*;1kS$~u+p3#u!zxp0v}na=&)S;+VI
zj}P~pRy>1M2bb*BvYyq)l$iB1aLhfbs%7X?6s~KIfb2k|DRfGNJ4PHGJ-hw95*f{%
zW42j@;*VZBI$=pvI)5@?Hd{1?qgY>;rP4Y0)NPK|Tr=$@<I_JMObs4L{gx;O-cUT?
zK>^-HGtlYjNJW+c-Cgxn?-v-Ax%Z2{qabBMu3VVKZ%zlv4Sy77l)HVl0a4DDaT36t
zn{IZ<*7EQ8OH;ltYlp|Y6ys3(bijp5&7ZHtU|iM)VVRlJT6XNIE^L7ksUDe?BRN!L
z+rq0QE00k^o>+Fm(WnnjMD5i2T_!q9d<wuy9cA0<cMw54+%m%RRMJ32%O3-rck(R~
z1F}@RTa>b_Y<;hWcm=C6RDVm=%2Ar(^jxLTzadqthp+n6doST*XZbx&qawl`vn!6X
z_rrDUp7}KXoGZQ>X|F|-^nS<&bbzUwn-UE{kQaR;?}8kV*HPe<MgSm)r=dY!zok+6
znSVq3Q_pDsx%RR+P<q>=v*@hLUwO2@|CgLa&<g!;DK=A6`QVfb9Z8-m%M+R1NtQ>g
z;mb!#shknFNrpNgYmWE3*hjA)b{{Gy8}jz7hva?t&M142jhd2&&?na>NBcAuENQi%
z)Jf*LP6%0Gw~~d{W9ot%@n1?*<f}0*h(6w%WEJuP$jti^I=gPcxo)<R&Tk?j#Fwnk
zr&qI7EjcBw_~Ac_nl0NEX)2>Yv=d*^r1W&2$(p4->f#6Gualz3a*pVnWhmY4_4!Ki
zlgAkrA=;u3Pn}sApN4n?IDxm(4P;%ryr_C|!SD9{8|}KdC199V!jG#V&~0m=*#!=<
ziDo2IGm4Y(M7oeq3RQKlY4nhP2=m{(u+N$`dx<WHtQh*nn~hnxM)DK*k&JH27lBUR
zSVSy_LN39IY;MXQ)NXn5PxTW=?`iRn1Qf75dDYQb>+z;znL&dKDAC#ZDfNTG-<wFi
z-cjG<C~r%T42*H7=mkg68jH$6L^}k;Sd)z_Z*h0GMs}0@G(j8mTYj8U7-nlB^6wOo
zIdAITccH$MB|YJj+oWVg&8d->r?K2}xpfG_N0-#G%D;nKW&yv-X^%s@agtOVBdjMD
z$m8mdjoE<-afj#)F#7+_Y^xuaeE$|CfzvV)y9~L0mEJDDzTJ>0w$mNqZNZW_rJhSb
zxyJ=vYk5_IRogs3@|*c6tPH!ZlTp|v#Sb^ho^N~jFtrE#R{?$`jrsRI*T2K{;4~?E
zC?-F`{O#12kJAdWrroC+$sUtwDrkTDU|O@nkP+VP>Z?#@ExvxK!F$ee_-~Eqpxd_D
zklVLyO1T6qKc1cOn-mgUeneGFytjAv@$sq7()++^`1#^<z}P4_dcpFiPBA0-Flm~W
z0QUF8r#MpSmcy>sI7-pF={i|O*o~?{cqOv4Y7(cZ1?(B=OK!{dGGfx$=Asv60&7xK
zYsP-iye6)y8I>KQaaN(Lzhd^yf)r^}uep>d@V2=D6;#!`WiCn6`$-h(v84NBNwy?f
zd(T+(NFLNFm7^==`WHD&CDzUdRU$Dz_MyF1e-;>a^7P$26nBF5QOR$Q0*SsgoS<yZ
zCyEb|!216ylAbqC=J@@8l3I{yn@Tgkvcb_LYs@nWDv=U6<Vjq(4vF>gtX+mo>jz}R
z8y|<){$Y8vkM~E2kwbn#&OgdluryMZk?QvF%-}XCXA;U#&u(TnV!;~L#4z?Yf;wp`
z?a5_On-v+YVr+yJ4lOC>7IP6(F-pxAVL(Cqkmd4NI4C!G!>iGJicQJX!u+0OtLjmm
zd)@;-iYD313V-T>$oEP{SINa{B3NFHL&R3(Lwp9nwbc+1_iHuI+$0Y2VYv@+_O=hQ
zlxShpOKy;U>spCd=XCv(Xmw%KZ#HSpIeQDB7H96TT^~4IWp4&j#QcWP{{hmIb9p&s
zL(41nhcU`8b*r;y7l#ZO=)XmV>b;>xPujSTWAd%n9n0LPm$W{fF7%PZ;rTH7X0&^@
z8s1v(8^7rj#FjPwee;cwvL5U6dHuB3kD{8QMkAk(%dTWB=_)0+UR0MJ&@>=@tc-4F
zAR)Z+tM{57z1L!M=a*W4S>s4QgQ&(-s=v~6yfl5|Vj=A$G&i-hV<v|6-h@Ge7q7i$
z3z+1jnr8P6pcLrH4hKh};=j0v?zy^5{y1Fd;q=RCpyG8qzt`UAFCn9KW?Wi3Kgjvu
z`0FGS1@!#3VHT&Z-mVNGs-H|r+B{Pnd~z&(Ejm}EPUBWq+C$L7UTuoast#29Y?}CS
z8Sqo@my<<t&92c@DBT;3LIhoNzA1LL4ldjo!=-0(CYi?*`HcUtrWbBIr0+B*QDuN?
zcx)v-nf+xEGAud7!Tj~_+3T4Xv{7Gu-hm_*Qh|-9?Odl$pscmWgqc{nC)VD?TwDTH
zB2zX9h`@U?9L4w&2le6azc52A-N!eu48~s4J~K^>PDt++`<jU9{w6C64<1)WB8B6X
zC3*G-(t0Jcuj2G;M;Bx(^R38*_sw>#3zbqn8Vx@lwYv9P(7w4>(FmVXV5zOFWXVX{
z_81rHaN&A=6mBX}MxpBGGY;8!X*=J9Qltb{3h9hPOtuFhNs<hMaY6?Tc_@M}1{+_|
zRi%5XeR&`Jyxxh-`B2h~SJvhNK5R8p)4;qm|6<JuX+K_rx=W0vC)GMsh1XBb++A;%
zQn-lb9&Ptue@O@dR>~14_40`ji4P4+KKmeOCk<zLl{s{+1ph{@bY3sm+mkB)`tIzh
zc4#_`*Gwn`-lI)UN%^0p54dlEYo|X)_|ddXZ$e%gL|{=y`lBbu%ci5=`X(AhcxJrk
z-Osd^H$19_S`|AZUQVAC^)*B{g%a)acbj6P`jRs=Jx359X@?@;K|q&`&ao4>$8DHq
zODXleno>A^Uz^^kG$g;~-_#e+T1}`6099tz9RrK6LdT5-=sd(mlgrx7B;z;hi=X|O
zZuN%ujC#UJIZ(htHX;7NM;+d2C(fUNCp0P|+RClD!v<zH+FN>t(!MX?FLF4X+NzSB
zMH^m@LArPxK!Iw&;mr>!wJ~?rf09keanKUU_ldw@8!Vl|PJ(deu~lq1bF~`;NukeE
zm#$$BmTHu0^l{LD2DbP#mhF3dU$+=@5Zp-ipnmmjOZNPNR(<HzLL0lk=J8ud{CqH^
z8z~2fF2Rm2k@)Ynjr@w^v~vdzQcAB(ArQm(?^w&STH~*%)XZ0H^MLdoM@)iADx{yj
z@ne&gkkij2Y?}+!UJIo**V9y9ccYqH-cruyj7jhnZi_~%d}57OIRz@Qy(Cp)3rK>J
zmc>FzfAk__^}CU=7Pb&DPrw%>(VzB|zU4o}3gADKb6@7?ze_S-oSmPO$8MV|4lQa`
zyNiv4#V)ZSwgLf7J0KjZD+1PdvfR6#A){BgClc6)4nem;S(t($Sg06k4uo|bUAZ4n
z(X0qVq6NE9yqzT2k5o~5#14bsrK9$w?040y#@akU?Tv1s{tsw!I$OnM)|;4=yfqKo
ze>I)*G38v{D+pb^cwMoe{&=-WMe(FM>A#HM|HyUTr*C&9kj!}yQD8@~wt`MIsom5S
zAdlZqUc8(-Jr|(?qUMt!m+Ml90PK#c<5!R}PBAg*t8wm*SE{g5sJBR;0;H3aKMfU*
z1TN|1v0%7FlPD7sCMe3*(|?+RTKiSEsfe5y`cyyRe{ZGlnyO{9@0M6swoG^6M@Oai
z38{G<gbWO2%5C4nH7<PnU2)GKZ}MH$UhEdP;mA$V#m&!LE_Q@-CjMU&m!RmRYw4wM
z<<n_Jv{n!Jo*5*^snm@l9|yC381yH^{D$kP35&Ib66=Z%uM?f696qHZUhnh{U~0TF
z`OExsLqS%d#ZcbZg9JLLs8kz|7ZHK$t2vznPJ??JP|5MNji8$EF~&<hjavtQDrvYG
z-yAsP7Ft1g=2;Om((|(aDJcG4WJWJ?q610M_dji^`4PvO^$eJ+$Cx)G)YQ+esF|bB
zNx@Gl`SNjE7Ey;!@vJe9{3ejeQq0ly4XjZ<yBJ*R(lVe>Xc!cgr+Ce&ZoK1H8ldXZ
z^0-*W!-+5(<~`TdrD;*8OQyJf`>M}Iwca)K1D|D6Ve;^m21NWe#G2Cd*G@}>l%rxS
z>#kPApX>+6uCPd-hZ}Ewg<q>$@YA}}&Dgh`7b)O=gKh;j8hpb~NvEfwJ#)h<8iu-L
z5pR?Qs!WI#14$-(&CIN3>Vb6xiT;)4(D4^pSNJ4K^Wu6B=kD&$9ZQn|iTU21g5UuP
zF{={I;$U=<)HkKtH&olhx@6wPImA;D$W4*wp3mHZ2oPrSmc=dr3-WH{`S3&agg~}v
zCn7Um&Xkk_s-Bs1b3TJo3Si8<<c+hnTEcrUbd>!&4Z-`)|9K&@{`cwU{!p5^Hh;9}
z9Bv4(F36o@FUCIZW8Ar`68?j({o}t-Z+mZq7}R^QDa<pv7kRFOFz@pO;EY8x&Am&$
z<e0iaMjM45XotvpdkIwI@f#B>4vF6)K@OPY@`&LkhZv|3;+Fg}5TV{;3*+JJfmWqb
zMA^x>UO^2$nIH7yY3_VlagAQ5t4Bm2!_Sxzjfrn#)*gMKbwXuzR9On|!K@gX7Pj6(
zuAd$kumtx3zJAeOUS>#4u?t*{!S+0`$b3sA=pML-Os%M#B8jA`z=2i{?n>9;8?)^g
zfT_+j#G{^->v#(&5I8S`@%eQb!W0ETHjqJUNW3k?(FziewiW*8>puto^LCyUQ~=#)
zv@HJGh(uOUSmND9{J7qF!Iq4cxLa*qIzmHWkqXfoYPtAIIHw=*MTM+L$RX_8-K_X5
z1mB3h2~@JU_hmxUn-A5lQUiRSzg8kmp2)YnX~piqIUKoNJtUXc646XHQ>zcEA3*o_
z(4}ho=R1FR7sc}P)5vT|Hf?m_FZWp88p;xHCdo>=_Wa=yFxprKB@?Z}{s4%%eaKjL
z8wmJ>2ABpWNP;jbv3+>J$KeNDB>SNb!MTv1-!HnrVTMgVao##O=o72DMIX4}fKPd#
z&crR*%@Eg!*gGYNWuM={j$CQwJRxvTE-V%n^^yChczu4BVju)_#$j2GFk)<um>C9v
z2FqgWhDajj@d4Bg9pk5%q{=cd$xy=+%Q}zB_Q%_wHD`(WCvRSQxcfys_cQT16y$el
zA!LoGGQGoUTw;)LQ}46qj9V;EDXI{XV(8-T6eXo$J#~{_$hmm|Mk$KuJlS6VPCw!K
z+~3MLz->3X%8ceC#tnn%#lb(wC<Ud=@6zW)x$9m7-cg!s;46Aoh9<}DFx<LOuavQ&
zOrZ?Z_1N3Lk|9j|Y`DW!$wkWqIjh4YusTJhlZ2#^KTe1b5l&8tn_6pLSkuZY#YxY7
z$L3R0tVxxLE4{;+L6!MCt{C<|#2V?*wkJcMMOyY^zX$|TukOjVS8)b>R;`+&&TeKp
zCgo$5)7M0Vh9dup(6Zhgrb8qjW&`zvjap1qU@a?fy*~=V`%A2@bsZ<(Xv`_eIM?ps
z3Ep@7Z`$sj_|x>P8SgF;-zvXIfN4a!)ON(acyqj#qx$1vdO@i6#Vol;qh1D7V0&~l
z4pMpx`A|t>8eY(o0>$e?+9AmZv_R3Em`dcy(18wQq%A~P5VT5@OF*C{@vm6mCFEMY
zOhpApu)pw;8t?MOBG^>RxJvl_d+fuqtjdg+&n@;bkOm|76oWW_l7R1}JCI9J5a^j!
z?c|=ra~+0gXr{Smc{TnxXmqcn?Al%SI#K5<Xs%peblspQJ8Cnh4SXn90~CmbI=4Zf
zlNd8!qr#=iqM?$2_^rbN%GFxoQTd{2z|5@dx&d4gf1e`kP&U{QQwT9kEeo<^00q{M
zy@h)7;49IU%LGjVkG}?e8jXq0gM%I%wr)e4cw%aK^NBg34(I;v))0P!&_ADmb_PH4
z@%jBGAAZ1SWukqF6WLX6@*WEkQ?WdQ>zycQQU;!7uzdgEP(V7erftXp!q$_3c+XR=
zTLg8EhCsR7%SDJ=WSL|6D_!2rOYT<qfR);%@G9TT*N?Ex@q9k>2hWqB6w7DNxr+_-
zR~!S945eQ00QV_e&fN|sj#8%~&oZXA%FrdoXnR}_o-Kci+j$T|0Ugzn91Mv1y~JK_
zV|(|_78~)rY19dkRILp;3;rwfMn_Rv#+1hNuWatCvXZAJpLyQ5sfyfOxVO!%h)=cB
z9y3;unl`&fbX;t*BhrqkqNwC($GAS*KaJlIM5L0piVNG4C<WsDt$_cxx&2>E`G3R@
z?=(k1ds6P78#HdPQFGpN_}zep1tmRLW{+n%Q`o3opG$4H*T1NtjK)oudYflsPZJ>A
z$D%l3m^~G|j&Qm05R)cPvOW4Q9NDK1^!=5tLi6y>$(H|;JC4?=N0(4)su`{lPTKk-
zPAmd?JPw+<IAv-Z8Z!nnlL{gCzR3rOJIy@hI6DY6)p*@luYSMuM})@iezR$7HB27$
z(8K)$SEdg=+dpuX@LzECAo{S~qd4DwDNGa8G>fD6=!Y&udV0F3K5SoSdRm%UJwFT2
zIZ(L!7s`qb{NiL(anDZ3fX=qFOUroPjYdo?2%tCCjXxa6X_N)x8>|yF5P-MFJw3n%
z_+V!dr&l358Dhye4I#T_MHmP#;&$JV_@@FX0DALB#I@95JaS&d%hkpdLSAS8{vy*l
zf`u;hw^!fv_fT-#y{fq-Md#lJ;~#h4hK(4|K_Rc}-;49Q1Oi}@ag?Sl8w@%|2$=VM
z$;)u34gL3@-XH8*Lt5&R+A_<M;FQim(H2*7QAn(`mZ5=>Q_BY%a?=Bn!7{2SLngLg
z1Ky$Kyu}}g6-CDRJ0|U+pZR(!0n0IBts(f)DsdD~5$=;U%`Oyk?ik1xJxCpJrBMMA
zV5(q6mAke@grA*TuuU8TU}E4-vx)q8G->f?!M_<ctd#d1QkG-_DV;3e=<7<AWQ^8j
z?ZMsmp$@e<%lMYXZK^!ZEsMuL2QN?!MDUm6yfcvYtBpe50idgh2E6aqdTZUd`)=f6
z=tsMg`hL3-BR3?bP7QMdEV%q=V<1I)WC&ea49g=`)G}R&mDa#XkWm$dW#7X23a*Ga
zxkV>z-M=0w_Jq=chcn;+=@^n`na;9CEgu&W2PJ)h_?enOA7fTwd0wa_0Df)#?$xq}
zGb-S7>m9#F`nyZ!pC>jZpS^jhM<pMqZUgS^`E@`c(y7FuBo_@aANWX|@XQl&@SfTB
zqa>e2%?7lLo~k`09#%%6RYR#bZ}oPs?_VtBqWk}XM|((NHJ;UI>`Z8&jW`cys<;X+
zW#W_6dO#!fxvDCr5GbB0*VY)IZ_ZS(-ApuHy6_RG#euue9RF3CnBP^pY&gY%L<v3v
z<xV;Wae1AS{9VE9pZ#21u7dG==uQHE|0hoPjUVYpsNM~-#%IrbRR}Sv!WdS<#J=H$
zF|q9X1$st69lA-OK#;T+J#_#x4<@K|rGWe9DL@JrADxfnjw4_8QpG}1Oi>^GspZ()
zE~(}!p`#XCgeS1*8uP-a%_Hd1v&p3XSm+O;YDLd|Lz?-AGMt)a9Lb$Q$DcpbpmSFa
z<gkg_q=TO7TtH+e1uQe%gbB3@4p34T#qC_q-Ku@ltlE&d&de=;@0N>)X?~=O(6ELm
zxZ-eX7Zyq#3$8z#P+frk0J<yk&e$XL%7>K_<WzY<xj9^j539C}riZ;QhmyIE*Q{}S
z2r5mLw-J)W{LQq--)vZ)49mEMrT=k>2GQTSkt?w+c~eCGOIucupF$v7np`&mE$nDc
zdZqNR|53~vSt5iuLlc)iX$b0%8Y#lElvhT15jCa0yPV-nQ4+Yrq#Z|7r?9`aCb3rE
zm?;q*2Z?5G1d9zA(o_bHqMG-zhO0$T56eZ!KX~%c{_sCQ44DuJrc98e_JBNy2hode
zWLv%OeMKCm8%-b6I4D5q@h}CB0&dSLwlWD@BmwnfJgPpvX1i^-@(3q!?#8_UrKfvi
zMq^oC-Q(xQN*}Imuh>T4k_2N|!h{BtPX7fiRw$wGck92+bm67N0wkuvwf<i*>b?w@
z5Wy6$?xB<Qq}rR-gka^<2Iwpm8NH9YvmS*w6~l9Vu!*8bd1t?<aCAK}#zfbdHC-0f
zfr9??97QKvsm5WNxd0-oHf*9V;^4;_LLjfKj>-NY-IZlY^qvHwthRX<U_?c=WVO!S
zxFfNAot;FA?*xdxe&t?A@{c|uPXYf;?bdPw(p^Jjyo#TXS|Y|ivjk4L5QP$?Y!9EI
zQoTN_zuRNe!+7(AANMOcLo^XzOTZ+~yW#rWl6xvs>O;m=@1P!|&}F3rG0RS5X_YV|
zD)HOnE{2o>m!<&Cfp3OPM~rhvm79yiR}y$mvC5<!-;?!?X_b_o>AK*|UB()6&rT(b
zM?3$BnlfKgeq|qa7b3i#_*AL1s)iBOiYMdL4D?1XL(0*vh<W$2{nLrKsGg2LNv4gS
zN%qCdW^7^vqzRj`&i=GilrKw=Uk&<rlo+s5hy`wvMI3(C7ZaHIZO+tHHRff1dsK&B
zXc&3R;2sOG3JF>~B6i{_iA2_zcpd5S=cDZ!HaLv+HSZ-r7W$fAVMxC`0#er-cJ-Kt
za>%e*hYh49+=v!Fkfdq+Y>7#`ht}oS)3@>u-P(XsWVw&d=hm}1ULsQUk200>`rbsO
zo0{WZbpsCO10VX<0`sSCu?T+jRr?KmP<kavRrFA?`q7*w@ksUGeDD8l$0A<ZKw$ri
zx_|RTtG)j6`@<i_lI}nVSIf5SVlNut8{&OFD~N;8!}4V9f9&i2x7?4eOa6bgf&W`|
z4bt3R<Z&NP>60uO54}yc&XX@(ra382X?*_mWus{cIvvRChZP844~ez%#=BP|AgxM}
z1enVt-eK{r_TJ`ql-*mH`@^jnw*OFibLg>-=<(Rq3HpWD!yj1zFO;>x-xRIF|1YG5
B-lPBk

literal 24791
zcmZ5{bzGC*`@VpLw1g7UH4vDLQa~ml4bsg3VIU!>G{}%f83HnxqZ=tfQb|E_bThiU
zyZ4*#PrN?A?;m?TuRYIs&V9~v;=b?ey3R*!EoBN)CemBCZc(VJDC*w2MF72Z3!jUG
z0C%TiMj;#b=UQ7s|2Y<mJw863o}TXR?!LUb`taey!NGy3s3;bDc!<R=FRu&_4~=bN
zv1iLz?1rI%;m8@bT_Ne_bkab*Z}ALUvWER7lY?E&%9Tyn!(u-ST})xImD^Z^<}W#g
z^6kd9d`n4tmDXaX7<uW+nL})gW7%Fj5_{f%I@F6@pZdLzEyz5;Vkbq#)E2SWo4v7v
zrgzxmt*f=N#?f<WQ8DcoRr-a;ViM-qqn5adD=~{78(3`T8FuyCuPo6xQBf%uDbd*!
z-4aKsgOio(<rW|1hV2L3tH;NE-cPZ63n4LIZ&rrhH*Sf_xa7P5T1ki+NJyv}3>I2f
zoHUf|o?)G3>tTwb-=F1+DZy4MTVKeRb=WH(-fTn4*PRSa?d|QiHn;Mu1~0l?O$}bF
zh>Isn##N)wCc~BARP199e@>pPi)tbCV5_O|sPpmYv)%ygWPgrEo2lck7>J>s?bK3<
z<J{M(y;_U;tKGx$++m}bit1UNxB1`RXjNK*=69y0T$`3G<1-hqyPeyu@9PGm%*zXQ
z4L)?*rjCCF6;!Fk`<rOHcV5bxPHIGse9;tjDA>%-&eyk$tF_i`c`aG!J+crW*0Yb@
z>!0rn`03cZ7c;PZHdb{!o|cR`>z=!|Lu~cMHPlob02F&`q8zJj9SpLTvvjJ05-N5&
zLM)9czI&A`X<1dfYikG>T6#t|hzDyWP0NXE3P@E#)XKnqJvDW8^~)!tT}Sf)DG9&E
zUqIg13@jWRw>AgsZ`(9lgjcj~pB!uOIVH91ttJ0?nY>UHpkmsPtC3KmP}`YqS3G_6
zOrz>}{-!E@Vw+1NptxsYxF$%cV)L8&M$yz{B{HvjJl@O7PDf6Rt7s6NlWT5n@O3vg
zeBIyRO?&xD;pdJs?CE3Yl`~hVj_sw4_RY1qzQM({_MfQksoy^;$_CX@+ak`}ERP-a
z!_urj2DlhpCTmZBPHN2!{tA^2v;H;Qu#(3bnLri1FUXyR3XH2hOlOTe%ktY#FH~3C
zeTQ2QciyO&xZS!%M*H`NcPlZO0e6$wT~+fr@f<lFu`qf2a-YMkTkN+~73K83C${S;
z%=JAQ+Of>ZkL`(28F&jit$Bndau3?Q2U$q)*`D>opX+fhuMj*;Q29x?qVT<n;oC1W
z{<WPRF||X-{MsO;n!==sFw~0aSgl=RC1#?r1{*M4dwPuZ4_NKyBIvee{TjZTH@)7C
z5+#1Q@-%IB*Gu;BnuZf-M3k-#7*jD3=2QGmj>ymk81ZZK^DUDwk}m6Zqg8L+&}##b
zY9Db=gbi=$hRJh40d%-aMhu9KlwaYW%W^CN>SnGadlRVh4Z`1b{q5`GNrRO31u73_
z7KnS%f?pb|Lx`+soc3<aQJIxkZkIM+zq_6{dJk6_LLB<U-u;0}fNqOIQhZHH)XkSF
zF5GQVK;gjK(Cv1-a*F29gvq5k9x=N2y!Sy!{nO7y2JHrrj3gF##tKnN!?s{jSh18s
zdxV}Bz1u|<vkD_AG_q_;P)!ZiEA`2QgPG&AgMiG-UaNUTv^yi$I~VBsX~V^`*A$*a
zwnoZ<&f9sDuVL$YyHbheUnoVu0$NWaagCM$s0Z#Z_=cfi%v+BW44yQ@n+CmX1IdmE
z=^BPv9wio$$1b-t#eklVk!TOJLjx~24)8h%{@M;MbC%%G`Z^%Dh*yp>_^g{xo((9b
z(iRA0b9}q|VL+##;(2`9!}!(6^|ve9m4**U717!NX#j1V;2XT@TFv)}MW!gkz~m^Y
zwS5~2fxeBWMHmaN-jSh*{h!t?fZi?_*6;QRj|W6i<=I6C>f;!f4+5*Q-*?{hGqT`D
z=D76Y?bOc;4zu4q^cF76(7g}d$&Ap6mt3m&`tBr8@Ri{JeJ{k1_$zg1o`AT(+F@oQ
zbcKnthk}C~%8se_%gM`zk09Tqaw=Rw#A|eod3@Mb`%RDfj{fgV&Y~skc#N-ScX3g6
zZ{U$crCWMY&SMYs7wMYUWl6J!2nafF{OxNmi&3|D<y=x}Z9sO>YDm1Gmyt+&j;HqS
z7sAw<?)6hoqAL7PZlPuua3U7}_qWX+s7~M+<^Rh5{IK-`{|_Z@V*PP#NY}=7I`r?1
z%q9<&$9<x|mw(T`znA~azQ0BOr{sY)t|-FkIX($#=)0W%nWg`y-2bWZf9LbxlAaq$
zb#-+kW8a4+t1%Q?k?G9UZ0V9dY@E2M`Ny##)$3BBObobNSSiG}R;<CkdxDKRzu&t1
z6xYai=KFl#0{!DoNBn|Hig^?XD>_vtsNS^eCb}Y>VR*Pj!(G6$djdRkDS@({$dWU1
z9W-ow24!NdD1C9moR^Q8Y;N12-R;Z9UZ_SRH0vvEJM}(I@u+`(wT=z-lT|>OGm8BV
zpwC83UWb#C+m`U&*z|$jyU!G6E_d%B>aR6tCI<K#9M|Phft2Z6xSeE}$G($5nf|Jk
z`Q3c#u}M5c4J89em+22MV5PR@vpaZK!N<SQQ@6DpQCD4fFaw#$LUz)HEIe;>rX1*3
zP4w<FJ&NmP=GP<YSh^8J={k(fs@{v3Q_(e;*(QcrIi0w-JU(f;9CIy68RLT5xV(mx
zEfGOCoey<F=OJTM(!V5i@6iEHHzk8*9+7-}Gs0ve#4M_dUc1NBt;iT~UG-Kxz)>3N
zcu02jTtc5a`+Tjqyv4huYm?_t2~E^-eYMTbX0>|7#3R?kH|J~8Mdf4ZuOc~{_AMt%
zdHc9-5-WLbiKyNj3wWfHP<!4{Nh}e*U+r=`mH2c}5c5DDep>VG0so-|TKfwxRB*4U
zjID5f<sqYZi7OmKHU7u~CEIA*xsrmY;!zi#o_Nsa-0fbn+0VkS095$p%Yhp%+|_su
zx&{SM9h6mY3Ui@eB|aAB8cNmcyHwlJMF%ws=~a6ud7%ElvJH2^FYBw-o(53(GNKRb
z1trxRqo{ktYT1(n)2dBnM?(?$CzTeukYD9Zq{{_macQ$t;@~PWJD3WiFY118*Mj;_
zG<BxwFLdg+QOW8TbpZ<xQJVmD6dPsrrceN*rubIB{_ZpMdj@%EuGe(~NEP8>yL}Jf
z<kG)h&3$0@3mxsQ%4}MG<iQaF&mZIQu>ImgjJf=Et~f*cJrJ&wJJEi|G$hPcUZ_!z
z6&y&&x2!L3`1GTlim}78E-N(v6c@$5+Q_8DKlxsHS}s66l~Qbe7Pbm~-{TNUfY@hK
z!g!cB15-a%od%v254B-rT@Fsm^OEXS61>jk<_-B?!xZNS(n!Amq_y=J$PoF4ql76y
zlbks|9{QyRt7_2|bS&%zKL<}DTisN5R_;2Rp4=Q)TRa9vx*7v!3NeE)x3L83PnZOC
z0;j47pwfGVyaP28JXNR1=jZWdq3$mY$6z_m!RC-5&-M4mzM)`kJJn99N(BVrEUn$x
z+vw|^({)eqZ>(o>&*Bo-DouI*hY<@=bW7aLl9B8Zv{qa*%I$ZT(m<oPrvnK%9~7pj
z-_3J1O8$JL$-VgnNMqt`lP$L%n~MA5PUEl6lDoq9tpulOHJ31EEvG@|`RrE>#|Mc*
zz<x<%JZ2H}m{1qIB-dT|wO<n(p!_}k^4%`qbisMl6(4rn9;?NAX0pVk(+G!Oc|m?M
zpO;(y@OMkS!g8{8{<62)d%7j+ew6+5rzdE);b$JFVcWE%^NMZ_Px7m+EMUX$J_ech
z9E>2?S+kC74=eboZHL(($sfeyckh;TKe+CH<}IA%e5Am|8TQ;HmEC*ob`2pPyMi4R
zq+#4$a=y5!%ScAr+Z#_rI)7s)unBnGWK`!YsrtnG9-ymu^lh>Vs>++IKi<46aE)?k
zbK@S6ESF13oCSanZCQh6FMl>Ih6KExE(0vqX*7HnMo|jpukc;W*A)??!>)YW*w)NE
zHv0a+4)mmo>+R>hrj>lCeevhsWP2Ul^7_@z;?}OJNIsL&i~Ay5FG|K_E9W9tiG_Wa
z*Ln1E>zFtbdu)c*+qZ}NKP;E;u>87i8t;VD%4P9w!>Lo5q1jtsm+$a~le%W*XLP=p
za)Noa60a|;yLD@kHtIL01>9JsO3BBrK@~kLm4*ab*Bs)34E5bACL1{)KHSP-hB-v~
zU@ARc+pES0f)?wXb^dfd+JH&`9%}2Jw?>HkhM>i;`$;G(lB;QgK)^-rGIsyT6vagV
z_5f+G%)rk@C3Db)<>rJ&hCR>Os@+Y4mpvTX*uC@)Igj}qB(hmK2C+l1_}g5a&D*6p
z-M{gj=VwiI%G`R9z(-w2u0>6NK<x-qQrvxkt-z>afR9hGKA!2(-i-MIqs6+}^slq5
zOmqwl?r!H=nws0dp-ynO6LGX^VIWBJn8-TnZd*)&FM~)J^?56^5_+Lf@_duYh6>X5
z;_BG--45&H71lVUP+0AY^8E}I%roz$zC{g0Af65|Nz}gSehK_TM-d817%toZmhgV{
z`v4`X?84^w#680M=KSvdp1B$(OObHYvwP#V=$Z;tF!_UyT}`d=5y;nfNcB8mi~EX=
z3oA8re+Bx>B+C7}=tr}h2w*2eX<?>VtnNK|(@)iRM@G@518;M-CB5hw48i7_sEwa*
z>bE9pgtn`hR(9EkbF9(<5a3Wk%7aq%dRc+$dE4KuWv|pQlv3bAvbZ!<fWA+m-VPPQ
zq~TMZFyBb4J9cm433lJrow6+=$OrXl+MgYCAlWQ?&>k>Js{}i6&iz>S1mmuDk~c<w
z6_mpN_H975aViOAl#j#S*(!d*!;TJjsp^Z+=J!2|yCLDL0v&Y@f0}i3TkF4<%HS~`
zHzf`6{k}6EF|Z|^g4DF^)^e3vANCTx(>NL&&No=bHpM|WaBslJ#3{C6huq8RVM_9J
z!Q9VZ57$EkZ5`{iUMRaRs5KPw)0VSYbDwe@-<Ux+t(yt;#6>Qf(gc!beUdXraNDG(
zegi_Qt%}X4usY=1oP*?55XgZ*P3q;={P%B`H^T(j2Md5v$x()<N?&boeNm8t+Q6>8
zzph@3pYgT(oNb1p-XDP^>$;rn#FroqM4^x{go#b|M9xg@G#>gv2kh&2&O^UU+FU;7
z70ALx9YUL=adm1gRPkmNnH9t)<K@!+ZEVK-(0yxn*89*aSE#iJvq%=irm;$_$#A}M
zwj2q5>?Ku|Cd#=6g)KsX0tu$Vpc<h|>rCe@;xw84gWC_$*&*_0Pv_&pwZ6*GjWR3H
zq(*YmTEA^s;61w;?^FQ$?F~IaltsOvXSeNeLtQ7`-B$yWeK~&u1zi?w8aIA29B>?i
zf{o?5FiM78-fIWDyJ?S{B#yH_{vmFB!1aVqo?nmR6U#v-Ei?bqEcv+C+9%^j09e~e
z#%!9nZtewpRe&fbWF4xOlcqOX4gPTA=<P)<_2e?+;@P7-VUwu7gvOh7c;EPF@W(9q
z`|6aT+GwTFusTHQUgU!fowSLAvXE7}gIGcZ;;W!22L5r5Bh_ZAwGw!QkuOR{WxdaZ
zS8u}Pc=*5oULw4<V{LB_=2ejYV`!M=|LdRwuAb3(#p)}H=4`~<cuLody_#BQxhh|6
zYpgqqYH|&W+o^ZnKUL_d)A05TfGXAvc9jNsS>C+DFw;%TuPtr08^3vEW)hX(mEY%l
zzYd-__;Njix->_pi(lD3JglxE)?FIN%yih2M1-bEr5$~IDa5r94+-NOtk)2Kz5C}<
zp{82O-VSUfA@Q^Y_x11qk#>h5-$V1VC2f){XA=Kj^~tq5=T{^)k;Q_ERdzk8^^d$?
zbq~)!dMV?-6zrDQicf!?a94*DN|F(gF=hweGH8S<_IF(obG7pyc5j8(Z#2S5d>Q4}
zOhjEQyMxn_=VVG(QpYnal-j&cFPpGNW#<G<RJlp6jBFl{shTs{<xdD+vNfNNlSjpq
z?SB|#$ITippJ<~Tzxbmq<-KcKLD)?|IO<=;<=IJJR@DKLqi5*Rq2A9NsE<HkCYn*P
znunKoKd4}Bv%Nymfyau5S2p|Ujn8R0Nb$|K23k+sKi6YLl@Hl(XR@y#(R1&~r94|F
z3Do{JE=vDWcc?bFzY{~%Ea3UUjPOt84|afhj4&IcP?)?`qz~%-QdRy?nx_~Q&Uogd
zR@u4tKi-UK3*R0t$&thLyQ9mr%Kgy70XNDapICkm01^%#>GFoi!x6C@X}uS=h(A8m
z5kV~s^;>{s;M5W{=%P}L#23Y;dg<Y1l#`NjZACA=hV!3xy$?`YN-lKr9Jj0H7@lok
zx`!XK3zE{fBqIr%&PRV+7z!(1QEiP4vd^qTzm|U4ffo2y)}#{`y4js5^uA<U%8|?q
z|NY)c;Ngv27<n3Nvpk1$<Wsd&?<XrFxUS37g+RgVCmhMq#Uf?zIGS%&S*izn=O8<S
z@zDg=M$33|TIWG2e70aa|Ao|Q4Qv7tZq5BW{7#vDzkjyXzC@K<n`a{jM*e+7MtZah
zRPQ1PPJYXj{5}hL%wD9@pT}gF_;VT;Su*Obn4^o7G&gtHabc_qO1-{l!<IsKg&5M-
z6<Y>7EE~XT6g@}3pY0MG8}mG+G`?ol9pytOf(z!poJ5!Z>8gNISqf7qqh)5g#A-kN
z8jeFvaJslyCU=)fuGYz9e4JaO^>K|eRb~119G(6abJhE;0#sO7JPkOz(rZv+3{YxM
zY+IKc_o%0@?<T8rKJpJ$Cirg4f^&%;Iq*u04IKl+;wP&gPYw8H?662Rlp~v%fBsCm
z%EL`HRIWNfP6X%}WSLc@)E-~emDsp=l;oR+<bXovp_Tdl-8m8pu?Zu@Xzxvlp%nz<
zu1dOKsQ7ISlidFC=Zy2wK)thjPzMup2TsfH{Thp;m;L5+{D;qgdacDlUJg5Y&lnry
zo%9vy_@2{h)_-=Y%@7Ks{F(hLP+chf@>s9Tklz&Orpm%m;1E8ZBz+T3wH{Zpy6pel
z-hO}Va0W~CTkV{~Ra`rPC|WJEyB~hYpAH9qpGoge$RApNq+2`_knovv<nm8?*WeX4
z@H%B=lfFUO<kRONuVFX0bHZd;k~=&GM0Hf=ofRC2Y?TxjP#I}@1(Tc4Ih=*st|W+|
zRa^`$`cc{LnL@%(uYOg1<C>fN^y+4T!n(yXY4Etm2q^1${qRY^^)&}oaysJqQnA)q
zSZ`N01tTLg(NSTJsZQfiBqdQ%Cb@U4qLO&a?4Ee8Z!kRD!0Wboa$}gFz+`|!I+d@9
zL=wUyL<hJ?iCcvsNVz04T4+{a8HpNZ6mFZjtzUX~F?8^b;*k3X!61b~0STIs>NbPo
zoa%g!D4%EwN|aj@`+ic1ExuKqEn1^oBwYiVzI9dx7}U6?P=brU=HNbC)Lh7bXko0y
zhiIJ8hWzi69#8&_1k_Ipq(PTjD1xV!-M1}_Lx8`2)Uf4}vga1CdnUKK)^$I_MNc!A
z8;`AVfzV5~@8NWC&`3k}wX){Ct`chHg38BdxAA^#b%IWtrKQk>G14SS5h$vyWQv}U
z*ri*6x-;AfT8@3i_7%w@A#W$~GJElepQjZ4vv_Q^Z*nOH`K14_HMuOR?Xao(u_pa9
z4FkEY$B~WQT^fkkIWsT!DAK?(bRRxG{sAG7Yw0T5B=kpEo{<iEP35gnf2&e79Q4OZ
zrv_>AuNNNH4yWK=xZYOF0m09RpdT<an<G@8o$9JS%uJYHm%gWWlJ-2=D|Oq1{l4F7
zNUrvPUO<1J0pz!LK(D>Xbq}y;Pkyg3NKnGymbHQ0coVICG?qnL9XqnV)+gSUsLya?
zK|RWV$s|fa-u>!n0Z68{ly4WrEJXGI)F(A&XHDATm4%LhMEcv8yF%zo?d+|Gze@Ek
z>bl=W=~R~J9F>(jX9GSngpvx$$}>KxYk3DDdU%cx^_3R3lE8YSEDZe^Y@egzHet8H
z46;uUI=uS(A1Jqq+LS@=*49E+AVOl%QQi+iC#@TIMB$!et{zes16>iM-;ow{CCH|N
z_IM)yrb^ZL?bXIYIO8;8;CsK_h(GM+`<qOm+!9~KxAwpDKfMkci5}pS&4!$e!~|Tw
z@hir`RhJ67@u%$}t_OyVW8vaTnNl;O!VsXDiq**CmGTCDnM;7*?}o9L`fOemL&Q?}
zy+Bp&$niRjWw%1MdW-lf82$YppKgOlQQZgD_G8x!^oit%R;B{)ThO95VrG%DdyhmU
z0L-nFu>H#+`_{uFqm<7#M<AN+GB)c8&ackV`KzRo_w(AS#MXXFT-ZM`kSqS46R2L`
z$C3GU(X%`&PRowT@Ze5&Qw85agJX|hQg%v*AJ)=`t~#JeO?G-9d=u*ajT968$hdM=
zQLJ|Ob$HhbpG3f4FhIVewX90fxYMXnO1_ISBE34vA6^EbF&rHv9r3Wmw-qkB#LK0t
zdHJFy;}dZfx#N2V02w9JK(1Th!*9b<;{z@Z1ijpLqZG^aI+Rk)&+m10^fAxneEWu;
z<+^`c!yw4aaCJBpSZ-|HIv*y$3I~1LmovV9PYI=c`bQrR=ep*cM2Y%tLi~kKEmJ<j
zjzPq@9b<Yhb`bf1OLyg~x#_8m{ww&gu-C}ho)eJkO7d&xE*R)W1f|)4xo3307MH!6
z>PqI#?xu%?9ozxBB?j5v+!PQV<8j^SV~Fyn0YSSrvu{IQ!R(A7KO}Dj9l(I)<=Tl<
zjgJ231li6$8B~xe-v^~o<BjrvJ&hygzsB^~s!127ntm67JrMF}`MRgUpv}uLfqkxY
zqu+|pE=$|_47kZCS~MqQvd^;-ZqhJec=9KKor=?&acb{{tm7Ui?W1ivG9$UJ?rwlM
zcX91jxX=%oh>xIy@wC$`xd$Q_YTN|%<jc1CkdO3$^7!jgo{r#d?!u<#aS8{R3@JW)
z!(S5$PcU67PAK1u*2tO5$T~wg)-z1%qo17tAkYEW<Q8{;5as=WE^YrHYFteIPFCrR
zGYgp^Gtvcd{}Z8u!p!pFs!(lz<z8=^!sH|goz7EtZjYe&G%Hn{v^%fDzuZJs_ASWS
zg1$vxxMhZx9QAa@(_zYmELP<v1y^l4;#il~aFCy?-)1^D8*xw?)>0r4k#8Q6!bj2o
z_8itu3imRLP(0X$L&6|LE+_j;m&)*IACqR$hb=e+MpE9NIyWKyZNE*5E)k)e%Ihir
z1&X)~_6~VNB5p>tV3MpFj>Qb}j=6b4=Sc24aqsg__81EQL31j)AgJ^ur?#kV(GPPK
z_&)VX#nO(}-aiq!r)sFmc$Ss`I0&)=E--DI89A{7JF6B^;ag^g1+i<_S@lj6BvDj8
z^GvK0d^w)1N*x%&+y&GjgFXvlBbW4lV&}N_Np8N|d2N(<z55euQtBVdd)<)+gc*It
zO^#s<WQ@0&sdW5fTpq_q(Q?ZWvrOmEao%O6Kl6CJ`PU>QK?pJT19)jY)TMDO7ZSuy
z8*q)3LzIc^g3$NsX?S(p6y##QMHH88ac{MK)4@gVsc*cahlB{A<q*GZs2Y`j$)C}M
ze(E1$5!#|HDvW+@q=up`9nFA6j<|V;>{{9g2yYJc9Y1|{Yvpp`zmaXc_3u~qCx2Gl
zmA_Hpw-zgv>ar(#@k5yc%e)7=Sb6rfsY7-OX<Xhpdt;V}t<RL~uBi~$B&?)I<k#d?
z_aY7H1V_mO6~>P~KD7ze?)j}P%suYiVeCCIzN7VQ6&e@bz?F})hq#uMPW6xj9x?+X
z8>N~Kp6^aFElxannbFoUYkp~CyR^uQR-sQphQ|5~g)QUy_@pr^^0JWBr8~7}1-kWA
z_O4oRhcb+bC*W`5YbbNMvKQ+75T#X^3$)wyGZei|$x$RgBBPMqZ7kg4P9h_eJxNRM
z<Mb#cS5-g}KN^I7&{va=<OsRW30VczTIg?GGwKG6A1T-V9(3cN;Y;9;)#qbV!)`yF
zV8$E@mZ05B=q$ci4@9QjX0g$BY5f*G)|Jc!Yx~t?hs@kc@7`e4JzjWGOtSrp`lDVK
zLS?_KqDX1E$D&5FJ{jayUO$^>;oJYSq8k_^&j@SmzF$|wd|kck&titIgI9XYmy_9~
z9RKM7aA9Zwam0iO;)@bA!Nq}v!(c(x2Du=n>tf4Q_oCJK*TlaP+oGt&qS@ps5MT4#
zIJOEOsL4+fd-cYE#h%$aTvaZIJPWS<R}mI&5;^+lh^kl2I@<;Pu1}g>mj%FsJg;B%
z436n2CNjHU2z|^>P>&k9w`uu$PR)eFt}rVkkLH!|SKs?NaIirO<!y9GSg`F(lX(5r
zARg!!SbY>(!<Gg9Oz%WiJKL3g6uA!aO!hA6S*(e1=5phFz_;7Rf+?uiWDKyaKBw82
zJnJkmUxdEt0;)Xqy{Sc0!*q4!df$FOQc)kk%QQ8)@7`b=AC%qigR1&=)G_>%JcGLY
zTqid#m$iOSaTQuj&lkL*&UXvi78abOd~~HwOnBjy|JcWh&CxhN75L56E}&WM6Y@!=
zCAqnTSt|LRs2_qbJXarUr!bM;rkC955k_{OVXxTL6K_GY*qH8$2%-zH-7(cOKan|V
zN}-?A`Rw<pX`x|ro$@b9vwla5Ez5;ekoXOMWmdfhBN&+`*j5VzTt34Do)r=FJHO=m
zH#(XFi_c)sYQC);G(t7%QzId+lis}r_@jquT#wMH{35Go_Mitq%S?H4p>FfEtzp<k
z<B2hcb<3;9<SAen+tJ3s-DVGQ4w&5RfnY?_0!JF^fE~Y7nkz)_IU~QcosTRhge8fo
z=4D@E{*b8kL(EiHE@|MG2j!1Qwe7ltOJ|owy@wIZB7$=LSa!cJ9R3F&wL4i+`&l_&
zXR3(kDMyw~q1V}h&yAfc9;5g4Ph4@?LWsB!rsIr`nUWK#mD#;uFl??l3}ab%4|MZv
zWVpdJzNQmpk-x$Zb1iLEkXib|Pu1)`UvfW{<_oV9`o`8rHV{M-+9I?6(Yi|!#UO)v
za8bbz=hWq0ksIF~q>N5nR=WKI`&b&+mP8~b+xe&heP6J?$R3^f9BoLb-9u*x#?2xX
z)WbT#n~mE;9uD@tsJP2k?Jqko?t}i?1q2JFwkm{~Rw=;$;&SqXw?TXYrByZwt2T%q
z+A7`OG8Ak=g>2GWsAcZtDnUuI7yz_~>|0*eO|<1x5qp+5(J3c93fX=po{r8kM>mmx
z8M`K`ij)<;V<pslem}nE0})>4F^1NEH3D7|xYGFD#KzsOk%5&c?4w`?M+<OV_xAJx
z2djJtardaqjh6T0czh6_HUH28#K`zK4No{k|Kt0mJkrGk1^~{g5<&#tJj$^8^?-d#
zxlFm|9hY;_xl68pXN;^laI*N3khOkTOv1uz#mY*MU<0?!@9&=m6~BC`2xDSRLWV{n
zP@h4v9!~QE4|!AxKhaHhet4+Ye6}@sr*8aB>DUNWc9H9Km#_JnmrC#UeH#QnOs;w>
zGJJeceR50s`|2yzG3C~~rFvrwJ!NbK7WC%KMN4iu)dG)QcGNx>k~;_uj-||dj=eJ<
zTzUD~sXLe1X<V)0T&G!Y9n-L=n@6{qWz7R#w_V}>{4kka;=z^&Popw=kVyj>sxfH;
z(=R-ojk#oi()hhq^D&+{2Cbe%-i3a$EG?GR<qfd=X~F+FHH)W;bvdo#bf|~euc1Yf
z?kfkMbDqJjtI>?W!#xqSqr167ZPB^3R%1Su{r+ZYf%$ryuNP=GS7T~rtcmdL&6)gc
zrl3&dL(Qcw`a}nF>z5@Al?AJ=Uh&A|6<V4ywp>5Md2+tDM^qvI(E?;Y>+a(%;!C`S
zch2@nNIrE~r(`=W!)3q->5c2OZQbf=t3%r3_fiSVjY3IaLP7O%^{0)Z=x3-`4tJD|
z^VUD#SNT-4Z3gs|h%KXI$=m(5x^h7F*uO+CdSK*-K?%=kG*Yz__7!Vy8Y=ogE6_cs
z`$+UJS>iKhKaSejGr$439^}q0UuRKnc3ot2S)coWy3VE}DRQ-I<{Fi*0hjc?mf1zn
zhQ~vTrylQN4Zih3tm7h9A=P>bJ-kpB>*21S1ob{96Js2@h6k58pfREU7QTM&OP0^M
z&y?!L{jQFvPbeg<TdB3IvIW0dY}tXUsn#hiQ;<3F&xc~?5*}?hN~=)tL{X9%J+Qpj
z3NH-Zi7PR?$D?Z~KKp18kjGXaex$0t+V0C}5<a4#Uc_&f)A4%2S<j9`)8nVpcjOc4
zOKY`P4E(h6_i+|;h0{ECJGSM&{PEW1g2p{}%lenOV^I2EY9lVuREqFZgE0T=Gy~Td
zQLLKvZbywRE)xFb-tujRMxI}tdnN)yQ+SuFJzgvD4Bm6s1}SyUcoNNk>zSO!9g0AZ
z%@0atJ$86mmbMRWJL$AuhRQ1``qun8hwz?Sp(XA-WaU4cj8}vDAbn5;4KD1*pptCz
zM`kbUFPh#)jos^?l$XA7I};_d_|b;L6NyFCU%{tTihGQU{tVD{3@?k?Gt~)BdFvX+
zydA|-!`kG3yiw6$8)VTsEkle<TTASIgk^R=emSL_XWkqkGVK^80xuTH6?e{laiELt
znCD421Q~bU2MqP`)cY_t?pPPHSK;u+2>y(wX2yRFO0KX`OzgTdgy@dne)gWI@q*1T
zv8z_vZ<q_tOz#P3`_ie$=)2ZZy%2T6>5bn$pRD`phH~Ju7i6tHxZKTAwg~1-qPIOc
z2s@l)cE`%oTrJ=AtQVX!GzrYDvH4x?_{nzj>diuZ*<pb@@Zbs;tUp2E#|qRAv1pxu
zZZ0Mh0%mC!u5np>`^_#dniD;|5wz(3*@I9*<Kg+nd5K}6WUPq$O(j;vG}yHM&Q&BF
z!($6JWg2FgEsz}WnY?0|d0_&)^AOkv8~?(^A0ta1miAy3aj0@q7BRW&@dCB~nYpez
zZ6t3O`D?IiYA>zmXkjMgOZ`?-$x;Y*>|@<zKnVW?l}{mF(b=ZYZUBKSQ8XtsRUWl(
zu;irmP{jZP+3kQ`!S>c!+IWFsZ^GsU%HnEDe;Vo5!y6kCV=C(SSe<$<V-)|5yqs<B
zQu)l?)?P{F;L_#o={ob+7$O~McjBz$ylfJ<Omy^!P~Rp(Cl80Vme<3HHy7Awl|IeB
z=dr<RUCt}qEiIT-K@sDS>zAk?D451#0oq%9iB~CM_=s3ERMgneW?^f;kxOf%c>x3A
z?XF8m?da<AyJR|J+KT8fd#HQwtzR)SY@HpIrpFs6<(Y*Ly14LG7QYR7uzvnhU8@&H
zZDSN+W%HQptADgl*pHE~iw@+#J<#i!$#-gF;>Iqvh-t{CA#hVYLoDq}O2CI(Zs@^=
zn=RhMvWC<5Z(p4JQNaLg9xJ?}K3q7>>-*IHIJs>v$cOQkI1}JftruX1uxAjdv4`+d
z>0Xp4bt2?+!@Cinp}OwCRG`Ex4bDD^IJL5in3#K@(2oZFAZ>s29S8Eh6OgWYX93!8
zA#l^jbtnKUGM`j8vU{hHts3k;#>8c~SGC(``=0)Vg;qgievPxS&6ZQ&{Zm`;ZM5;$
zABJA)$v3?>AyKp`F>QO!K(0ubvHF?ofeZNU!inEiqZLouLV_pf?sdR2u5Y=UP(WUX
zbU7R3$wiDBBQwY|FK4F{mYy6GD`^L=b7vKb2W^;5htW^R+bH@Z;<C(5XvnMDzNJO)
z0a1~c7%I<2cOY3axcp%q`Lf5~|G|(IAzXGW1q|K_f3qk7RQF!7v$b7HMN!|f)}byT
zbH=5r*&p$Ebyv1<b6<{?`!v?_#{F39r`FSO9)&ug(m+VqmnG;u#1qNO0e{vI9SN6(
z5-jKyrFxjRnH4UP9r3~iFSsN{<IO@zfZO3+l(Wa>Qv&hb-atSHsp~)WbNeuM!AhAA
z?@w1c!JXXLbYeypZx|TQALHt<_*=*FFkVh1RGKshM=zb@DY?`_R!$KaxSW8MOEv@f
zF{%0<_8IVy7|wo@F_Gi>d^%coRnB=BpYb||qR%E&wgtqVC2x~0?t{8Y?>QheoVM-0
z{rWNCmvgLImA?vGViOKE3=A1>kNlL=wYBTL`401q@MRpxa7{swK%PHtbg5NewgpY6
z>}izQK5nFRhEKV^a{tRHd7E@Sd7~w6m9&g2qkUIT;}a5y?}#eN>1&g5lNJPhrN8a-
z%4&O)6hJ^jHS#lb_k}_RRg<6bJ{7+vq~o9Hw(_lr6Sqw!`ObkT5!HP)3VSAgn@i*K
zyBR{{eu&q{m5lJeG|<1rF8#<|HQLO2rkS)uP)9IBL68`jwY84CVbK9Xz0|!aI+RYX
za^uPt#gJfgaZ{0h7T$U0>v|xc{#H6rLinLZlx=rCT=o&*>c{$%1*0Dnh^7XOy)r93
z9e=~%?2;ZU96N;gZ|cC^Q1e^U3I23RF8v<K@I&u0MU;tLOuTy#eG~da_od&qWwDaC
z{65uC!%R{g3F6Lcb9j`k2BOJA4YSmr*V8MgE%8K*2w)ShJfTVP@?wI#?P0wjpubVQ
zwUgjd`_Cs^v-?9{hmqYI!N2=sD67PtVV>>RI~r0iMTWQK$T3pWV6U=M>?)siM+Pup
zlYCzvGy`ZSets4If;tA{_#F#1lSkqLD_^$o0?X$5aj;XTK(vNkPp>h>mj;R8{icS&
zn-8cKUpV!pvB*C})rD8o+hTmjKoZ0cQ+9a+>hd7=yCraH^at}Z``p$ydV)8x&J8OA
zW^wo1Li=UghE>9jy{uP*YvE-K5{oueznK;vR7KYv<r@1azM8sY=Kq=~OA-AZ392PV
zTnm~J$~mUJL+4vg*v-DZ(+5@a)i)JoI06OimC4F%9c)6CFajiHuQng?(cOs!E3kB>
z{@8-M1!wU*75KqO?u!%8;8@VDa5?Ps(;!wxDqO1Vg5KJ+B*IX^QhawQsSZG@7^=d&
zvKweq!Y@jZ%K6xR>N}7G!<L-3vM9l~3W%zRq)ktBI$nCB2T{muI{9~rf|DN>C1H1%
z^_k1ougw-xB*r6n4<fzh7#=s3X=I)o7efqrSasN)x4;WR2n;4tuLIV>V-fH9iyI1d
z*0D?3q}FA|H6?ZdN2ud#@|#;1@dTLc<iT5vT(t!<x`YfUZV^OfA^E#X#GJpDy9OT&
zUzMZ%nZJ)z@EG=h-<8Aa%RFE&)iIu&;kiGJc7{c|t7tRxE!m#05iBoD7Ltv(^;jH^
zdEbGGOMS1f7`<Z2n;p(Ve#As4Dgk?7&zQ14K*f!(Ubd$1naG<l9Ig@7oj(P#a{55-
zS!Q;ngH<YZEgxrK`G@pfU%tlVI}~o5_C-SxP2nCr2{DHRCVnR&Dcn%?2)hsR290v8
zGVG$6&csg2n#PR|nw7rlB=e`mNLud?-#lO6mpl>bIwD1{rQ@UmK%-Z=Z$qC-W)Y+R
zQhAqdkd<dA!422S6e+(mnM=R+Qu%tElZ4^9-kVN$g;Bx7ac=tT%Qf*I>kr=Hx@~C!
z3#6{UFY7oR0z`6({*o}--*Q=Wf5w8IkrKg?<1AcIcV+;=EY#}*$h$-*EJS&Evns@)
z;Dzp}2IB5MiMLCUk#<7n;t|X7lsOD9Z+3L?AZ!jQWcKo+uj@f4tY1?6I*oea7CQBM
zQt4%SH^aYMan#f$A-2t%U3Ea7<D9=^+rK|7&iy>#fb2qkXx{c8{sl+5c=@7|eN1{1
z9-X)<nYf52WxFe&-cN&0e*CHsicOu<H_H@4!@Zs+|J8qPl4L>JSk^|m*763%IIh>A
zl(yhG&8J7A_ok`qwg#`)1sj`^o=bj3`k2dPQ$ZvorMXP#z0#}R1Ud4s$T%i4t^RHt
z`n2+eEQfaOn!trsL;|Vv{zFE=`TR3RL5Q<;n*Ovl?>GyG!J*Ci9kF9~%635TNbDV=
zh6*U_xc+fzseE}$Vwu6GDc)WQXaXksFFTnw#&DT<J(uS1b|%kJ!=bH29P=?|*4X)z
zOKA?cFESkW6+E4&zJGlDb47mswx%lAb@X`RzB_G-&X5ly_@n=K*1sd+J(NODAU3Ef
zla#|!a%t|91HU^`j4Y0(;kd0)xROQ>kS@?~K2?>$bWt*Mh%$@3Ghav07(=IcuR~_y
zuBq?Q`)T(LM5Qi=c}IxhGDuU5%62!r%!Y?J;+e^^Jsuw3`OZ-pg&L1}E5g2<%16r9
zR*9#t$`4ZYQ8p`U#%To_P7#ADhP^RgtRy@dUD*ilvjzL+7Ok!+0c~8XTNJo`8S{m@
zGqV(iB6P%G$Kx~s<$fm1?*=xACm*Yx%Zdj?{k(`B<btj^!H;eWK^+O82a)!n6gOvO
zw$aI-Ss;TRJeKJiv&iR=-5!{CQnm%qNk1E!({}Jo|I^P4ojCV*&z_Dk<n|Qcjy71#
zyFjjQRhD*{1&KD%Tn$Rh6?(`Xu_ug`N^fN{uvez+h(LhBP#PGEHm^CbaMt!GWMw{*
z?F15EwRJ8p_O|wEM;A`x;ePo+l#5ltE*cx!jMynvOH_Gy!y$Cp#8Ok`LI2)Uci4(2
zK)UkdBE2dLkNd&&;BTIh`njt3D{m<w8={;X7e&hwI;rZ}_O>X85rnuevJ5|1b;IFN
zXqK=MH0AmOZMJw3lDUj6sZQu}K)o4#7G{Ur7^hk8Aby$Sre#gQZ=btVFsk=BLXpL=
z4{3Ns;0B)o<fGmvTz``?8R~<qXrI_*#PoYRot&+62twy4B8b6@aRAO-Y2&FDkOOD&
z8L{1g|5s2P0GfqG8Bdl#R*X*;ML3XmOW%*yb5)lkLOby-X!gI#EKX52b&ZP_$MH5l
zK=&@NtwC?*pwCcMTk~Bx^Vh3TN;@!bc-u4TU>#3Hj-UorE$@OHjl4pQww+HzeJdEp
z@V9oYy~g^YY%tMt{jJn3B*wCCD;?VkLSyAP12ME4$e11K#RrC}v?op&v#UQir>U~C
zis;;`<%D7mpEE&I<FBoB$Vgt^7W|v`2tutJR_3q%_GfBOlQdz68c?p0SgJQ+@%Pp4
zUU`RxhpSh3R~b9us=Xh*jUT-9+NUZx0{O%YrcG<)R+Mk#H;!FD@@s$0vtaEJaoq~8
zE7y`W+&ke19$xo7$EA8a6kG|{#x+LMvFrDMqH1cEbZM;fT+jv}$EApC++@85?#>Yi
z;pK;m;D$Z7+=uGm#ndviMCvDcTwGkI-D145Tdr}qxa~j!584BU%*=gfGK@L28i<%u
zhMf<MNL#y6KHpij_N-Qw11k8SkROe5?)z`2N}bF123xq<A)=DbHXF$^5&K?L6}6hA
zIAJPtJ)F8YWJ8Lw_cq_ng`_A4Q|x7gMpeyqYT#e=+h)ha+AHaAqmVv~g_l42Ui;^u
zMg`damI^9jJ#FvkXkWc*Zye~vm^PZbD*_crEO@$Jh2W%yuuG&8@X!)&{=U9jfN8$(
zpAZvp1Tt$5i5G(#p9F&qs+!I{4x%>1SFN-9ad{4(!aX`|*0n|fDv~evZ0X9__{|`1
za*)S~how4e1oho<U?zOqcL53Ay{-+$wqSiW<=U4JA{sYV!D+d4gz_+uiwkQ}_iuS+
zdOvaDe#TMlEu8kpL_0koz{Hf5fqS13QFcai{D(%sdlG2VtI#^gve&Ai;1it>4q&PZ
z0XhXC@8F0o;afo-;5wRe+2QMmMy4+-L0jP_{}g^k$MX@}zWPRzgw!97Q<IFI?N#3*
z8NJ@KMhn(y9IH7es{1xDxUmYTczvxS|FkxBC{~Earz@fTZNKT#;@0i@Znir3$n`5B
zJ)k>INYw4~jeVMfn(z=bFHCnqE$t(hLG^8aJ^&9VpK07HgAgG;leGmJj-|Y|468C&
z#0BsChp*7SMRVJqx56Jgh9rlxATR~{fe<^P*Kb{Wp0DJzO-y+bLMK2B_;O8txvyxh
zUZZNZzu|lS5T+{?!IEXL!w!{awDByNp!M!qZ*VMD@){IlPDCfWQf=pcTDIasSw6lq
zuN}@`sG|^#tO(midzi^KVyKIyW<15#?r}c=SU8XP^7ki<K0DQqPr15U5q{yyAQW51
z#9}ld>~!;a$aUi`R|gFK-KbCqd>jMnI|7YVR0TWXw&gD4W)DRE#wmgL)rY$WTQ#<@
zj^Bo_{L~9pc0ULM`$@vUH+u=xzQl>~a!#U}$CQ@;3ztPtFNSOpbMSA{tDsi>OceKX
zzD$R}T^4?3<7ht!a`^5&%jRv@InwI1e3i@uovwjC^&5SW>S{f_$*q3%=jQVvst+W+
zSZ#()5Ds0!AD?!A0R8Zh{yhd5(|>_?Yv}t8K)3Bei&iZYsPo^TA#nS1b%F=DV=a1Y
zE@FVJPOuhaE63vxj<!I>zR<_XAC1Z=`NcFt?eiqoHxHaBm4=K`brUi-$sa51{+5ug
zD0s>I5%?s#V_K&9BVxq)j+qhCIa6bc8IUI}K9X|m-kYH7rIxMn%6-$Okgdp1dhD)J
zeYf4IXL+=8dXb|EE=@d#NYv!({Gi*4pdcu`Rbc99)Xz|w;LGgZ5=u`sN+H=Gl~N2V
zRZxCm_B@SYyU5ZsoqS@P6o-)Av#i(tN$@o5{^&4I;mK>YyMSPS$c-Ayy#((BW4$W1
zSoL-?3I6he%h;#)jlgxky9xv{`PGOO(T;lS=Ijer70(z6KY6(4**d7TAg5Fqe`%lf
z2BE1}b?@mUXf;g?M<HJiLMX-{QN{Z#KxyZYZs(Scw>f%hR`GAmN+7vqOpiFBtm{zs
zGpYjkdMfC-B*Gz5I*BQz_5yF(d8DYdxW@;>dvg|+2t|p@is5Sb(CvJzur=f2cl-x*
zc{jiV|8M7W`Az8Q==9wlvGjLVFo`7UC@I5_;tt!(iO_4#&nn;THo33r=H|;XQd7VA
zROxabz=iYVNh(g0DfqfJ<~CDv*G}+oq;YST)n7l3gZuIn&@uyE2_S8&j*b@sbz;pG
z7fk>%K8I%}q$%k+54Omqw7hU5M@J+%jQ>&82yDt=*$jxaY$pmSSbU9a*@V&+RD@M2
z`cU9J_zXLrP=j}U5wdU)PX#rGV{;-1%nAajdJ_bt?tV2b9x9K2?_wZPG&Z!~mBCaK
zOZo6TM_NV5nJoNE1?QUfYD@h4*NnBC37LlXsn6e5CR*DiTx+}jta7;)mnt|~-crUP
zS-Yp8UxR!>tvhHDVGl~nl|nX{AkG;Zo@N;oC+rDnwO@Wg^{s6Sp2zM*+Gncn1?T<>
z&F^#@TYT>h8THy<O#*>pL0)=xSBDBGHb?D0b_#`@dHmF2GAFgOd9>#N;m1mh(v<Z@
zT~_78C|b^JjZ7p?bJKTUU&u|9qA<k%09SQW?s67iYinhk&uQyZ`X8X325V8|pwPUc
z4V(Xd=#Su=dzxpA!QnveqTs`r>M&O(X~0u!ACPnG$MSaUXO>mXZJWJIC~OYed|ZYI
zrgv^Rd(`m}LgWG75<YJN_`R|kd4%?BDFxR<E{B7Fscb*j;`5LisDpwxs(P{@rin_h
z#M9NL7M+YpHh7ARd$Ub>`1d<@W}*^In3L2E4#@d+**_Rpxoenh>~%km<SF}?0>lT>
zwgDTrJWxn?9w_~W@ISqd)A59wUi-5p730ozX;S)2iTUWS4Y|VEz%sFE`|l0_RmcBe
zS;W3Cj;V33U79TC{5v-m&)6g7&Vq*n>K00L6Sg#Na_~nHBeG#N+_lneve)|X=y6nM
zS4T(GGm}S^Ki+IW73O^}W0p($C|Cu|eO{12F$S&$tT?Ubf1of=!>5(qu;Fhb3s7-X
z+$z*|`+>AN>?<HCPBYIoz}KP)IF6#Zf9e&<v9`A!8Cdq!q?{Oxl(9XTvs{`=d+hUy
z)7c_P#QTJAtFbw$Al%zN<vq?NkU!af-u8v9VHJmJSuNq_s`0@eVjSFK4GEJ69{Rys
zw%Xr&oh)!Sd0}@S%Zq_UTN+DO$#hq`aNNT`bU^J$mtizP6**bp_#q+xp#u)ZQpRET
zhO>{{@{w|E3~DH=SJ?ZE=X&&HAI>al?;nA%cg^F!Q{D&s#iSBKWFO5z-~0iee{1Tx
z`aVtOi<<RjR5z|n$TtzuT(txHZV=d{!(~oZp9WAX!*51+2<y8`aEQwxKh6`q`wMDO
zU;K_4v9jI@!(lOmc+^;@Q=Ak)64F-2w#mAqZ(O(b-NMGake%wv`31w&5}1+kJWcS{
zmqtUJvhOxd*cu@ZEa%ZMenP-?-x3%8RR6Dp@($>vJ@P7(SqYBrhCP;xrdRtfLSwfo
zX-Qs^>4@~*_0%<}8q>s3)gFE}{71bZC8ta3zZcvrw39S;S7d4#`w(;USHIEldnNE6
z-kEW{>|2V}^`>tQl92G?6q&OeNYn0slU55S9quQRNZNz5b(PL$Jrz(0dA8A*uYYTy
zXb1?J{<pMaGW<o|*iYPXyW^6``p5M&q8~lL>G^{q9Wd^bLL`q&x%|}v%A0GL*xJ#A
zQBv@e&VLmiv~(Bj<+x)WDpAnA1KYC&)T07s+#wNp0FCbNDwSrlU}TAR!gMGn$5y_!
z^6k{iDN+D0X9E5C<gaCsHihw_FSXK5HbYrwjFZGXx}V2o&nOJ1*rZQxjMKaALTNT@
zxQ$Q$g!7B4`iZ|E7;Bd<$Y_g|E@P|W9wiP0>L|!j10au9cLICDOxdM1A@@X_d$}W?
zHAk(h56e$S^X-HdzlrnEps)}DB%>&%FW6HU9K(J$HM2R{R_wv)_nz@iK_T~?WfYI)
zYUcGNv|{#>gJkVveh}TcB07kj%+-2fhroQ(=FD{Z^b3b41|wUI?#EP=fNMMJx{oo@
zkd0sxLOI@>3t%t+YBK{x2>J48WW&RQ*mdrjSz=VjqcC3uQM0s^yZbuY)QaiOqfRCD
z8tWOm9^<ADnB-d$If_&wVnHW=TLO<Q(`i%S{WGNE#YbH)II10d8%6+H8Zt?vjx7_W
zS8K5xr*p%hNW{Rdnct=j5r-g2Td=d1SI7ms_LSr8hlH6vNZs7KfXMsxTRJpUgmP;Q
z0`t7>VMtA)%?OV2oA7>Xvpwx}`!)O0O5=)5s~_eh51Kb3O6`lRpp<jMGSN683?tbK
z=E><hIC=L$zt>4Nvt!I7bkHtVcKVqKl9Z=8;JZmTZn4zzZ~h3XejD|6VkZ1FT;D>X
zNTf#UazMeK`iHj7lI!<t5zxU~T4?sC-md1q=N}Gm8ldFaw$^I{LP@(moS6KC)qaTm
zyyHdI5|DxwJN3za_CZ6;Yn~y8cw^DqpF#k0)P~C3w^KQEiM%@1UA3r0Snywvt)JXh
zm|@b$bNJKB#7%m_9K#3sY>hjv;%or0Mxa{FxxGzH?~2KePdfNpe6Y-WNU=_gekTdU
z)D1QAZ{{$EUPp34^8Sz5<jNl>wyDm>y=j<ru)=6D=0uFSmcxAIL842XDfgZJKQfi!
z_|GwjFY2j%GPASiQ#2*ID3!wicr*N|zmrV^dC(gN0@|>=Hs~+zrUf{i`|;knISB*n
zB_y=Q^Bydd_J~bd;QNnv)v1Fz5yj>?Br^uxIrji0?%F|jxpRsV7C|@qv56h7%b@R?
z3u-u>$bb$!7t&riJ(n`RHL8ed%{sdO(|b6LUwz>1#b;j^;l|&d?PgtD#1C%<Tp=r|
zFcD8#D^Xkg_I<xz3JAm1+%Grj@$Ais)U}ZH&I54cgVvJ)IX-Z3iqqCLZEl#>KL;`7
z+Wg(WpO_=&^TffKZ(6!cBgr}jgOv$-xpRh$eu>U?H@!;NFdGIO{a!Y*1DrWuhg~sE
zOZ-O)Ec-@mXdSW72_*&R!aZD9rJBDXF}To)X&;avTuX$@JNvu1k0XDa;+b*KKK?vQ
z&=s$sKm!2M?(*1x&z3$S`7~HbyQ1A^ljWRR9+KGX2H3w4Gb`3-JdH*64%mMLZ}T;7
z)FPWAYXnem1A(<y(C>~=CEkxET6u`4)3+`EKoN3m>tyaEs+z97hBZc$>+aMsAkqIS
zd2PXT2N6)#HK^A1eXiRg>@6Z2#hAss@X{ipuBZ>35Y&^4j6gwCd_)xNm10XNppbU!
zjuC~Fw3w|vpmynjjQjcN=?jLxheU!g4IdlN>SOYmE}Kr%01VKUpV1w8!qGGViMR~Y
zzX#TWETSU$sryc)DZos_a_m=|@b=M&R_L8?gi!Zn20eqi(~-Bm0dXAAwtvyKRNy2o
zrML>CKdnFwzXaryUzX%#FKy)`xU9~L{FGfJ37_4T_fjJ1ivsEXJs#yZ9$FH^kp==L
z)m<IGuy1Mj=cGv$aJ+O)<m9GNr#+LaO_O`Pf*K!jCup`R8bc-V@-{;E|8?@6;c#{R
z-g+5sC3>A87%iCGh~A=$FhaD9Iz$i=5+zZiM+t^8LG(mtMhK$UWJV8?32A!oB+76$
zxu55K-s?K&!#SVknmuc;{a<UZHEXT^uk2ESmJuR25Tjeii&!}Xz<8eEyIpA1OuvBm
z_a;~HNFW9JHUM3F)Kha&mp<|;CqnN!Dbp*u(3VXklawrMU$p%GPt!sY5<ZJFUjUZl
zGdW6!H+{&J?As6c>+AwudI@eriJ^g*nhjbXV#q2nCZ-Prcb#%G;W`>P${@;^veO;k
zdFI$%a<L3N)Y&Xa$^Lu6Oz&wRSFtBq_||L>&#rpq{Roc<PK6^Pw@}3_9dc=5Kj4Y{
zyO;Cf@cG#gSuG}VS2I$)+aPaaaLJt8jqaVHWTpaPCa76>9bc_d2Y>tpKEBr06f^mU
zjaq8kquYR#$J!@11xw4KKEMje`kW3u0V^M66T?;zI0<2{4|N7Pw+tJ;#v`BRX!e<R
z9T-H#?Vh^D)tB6BN5|th-cttawR3&73xOey4mQM{?J$SopYZzCw~D1|!4<z_7Aj?x
zYVGL@C8@M{N%2c>#BvaF{h!W=D*;txS|a}tS8_CpJjaHatelNWa7s&J@GrglX!Bmi
z_*X)`A2A|0*MXDC2R|8A_P!eTVZnz@91h`Xb+twJ$tq@1>pw(C+zA}kcB9-6%B<}D
z!T$O_`I||X-GfEk-G_a=+-}#V4q?H1vG(Q@MxA9Gf$zBs83Cq9%-dia=Qs8oc8}5A
zz6s>n;VCh8(mU7?S?ykPw%vy0(9vb*Az}2BklXeJH2UaTu^k3!Ujo=*Np@wZY7;6S
zX?zW*(u=p+*QN;z$V2Vz-%Cv;%2aPT!i5~hJiMC&l_n;xNRE3!DB?qpwPb<Z8(9V!
z^B^@Lxd)tGC|ZyHeIJ8rw9!XjJ3}5Lz6Ru?3~(WtpMKYrbUVTex~91%jQx2)*4mNY
zFVLw1p2F$@ciul=fO?6j*{MqVVj^5ZOwj#mZZ-WOu?qb22$O-3SRGZ*o6`<~VwM%o
z(U_8$BGdN?HwwDFuk)k{P~mQZyT2Ge31~zStRVyb!Uv4)Snz6ZZF4`M*0Bt3e<aLc
zNGUPV5g++P1Q30R428X^(Ne><XVN<o7!A2>PpGYh)taa9vzmsXkFKCc2g_b(5EunW
zrp@4I$Y?l}C;e&|2r|*3o$}l$Ra@XkQ@&<DKH&q?6={~F8pE>(J5qYbg-T}--`BVL
ztvx0%?)SdE9Y<ag=ESEWaX}Dx=xpRTyW<X2mJ7)3Z%TQtbjP3`bK%siUDVlYNn+22
z(v_o*yH6zRt>d0nhNx^kpl8(4irod8>D#3sb}p~7G7hH!292KDRezg)T`N(w=C63u
z!N{82x!%>jiW{qv7e+;`Mb(D!OhTss=IX>(Rajv(eFm}WJybtbr37}bj62ZS;LVk>
z$Nm8Ys6AKa)a8Hp10=Dgg5UjuZ44JUUxR?OP`4BKfx)d}Rozr9*!0S0Ns`X8hgp!c
zQAMg2df`C*_ax=+yIZT&D|6TgMNT7s8Jrc!<sAQ?sI(deqGIh<15g%%Xkh5x6O1sR
zd9t#H=6|3oQjU~@l&L!xGJDwmqzCeNU2u_JHJkeO_obmhtqr0_jzIol(aH1Ig?~`|
z&mcDdq~^`wJQ@xeP}@2xU3fN7{Ff-8q2kXHQsCxpYR~w*Dk*@JYsT;mo-wDF5nB6&
z@XE_I0VBaOM*^l)w{J)KQG3RDU3ClXRCD)?xdMb&WzV~G<G|r7<1V!OeN_x*of^nJ
z`ZxD8&W^y@kJk`6A>MABt;P(4_{dZ+pj3J}htNFP*3#dC9)wBjpxM3+YoLnL2Fvd%
z1#!PM%+9@Q{fjP1B2mYImjxfPxw5XGM?Zc8F7KexSVG#ne7SH#LWTRj@XGTY_Ir7n
z+>^21)4QV7^V#C_;@X>05AN${V&P^FvAsskfJQqjB_-vts}@P5j^?3o&BzV-KBCdg
zo_l}%RaN*TJ%?$=fX}-RvoedGSe`f^<1ghu&+^*%lcbl(b2j;`+BQD7>KpB+(uEoq
z{ZI<|!w-0QiNr6S8iGjzDM!c%fKkf!?INI(Iz?1T$tQlxl8Kv{2kk_Fh#5iv50!|C
zo$Oqu+5!~&hedt?mP^*1xlR4T4==Hd+7x}a7c|w4NwWKy+*8cL&gPRYOA{{y2e3>p
zLKn*D=R1p+X~ab}#!d*2!q%ryuD<~m!J*Ju8|oF?A>@cH>u6UYqlcnTlgyos@Zzee
zlQ+4{Eij4QsG5{ZZ5Q_z4>lHSyi2R@IrwHUS<i_sDO0)BTV~UjN#{JzcnJOBv}Je4
zjoD9E)U@{tl~0s%J3jlnd~e$&S9J>MouYFJ!|c2el(4-2nCkAg9CEYgY((bIi_+37
zhSoMC@?BS@h0^Z4;ATsBS4{m8NKzGB1<BZGY>@~s_wX#O%OR`i>u-`>r=VRKD3VH=
zCujBJX|lS+t(TC1Es}e3oiYZgIFAs=%*N=W`v7cW=;^Fl!Nu~v^L=gx{SvA@8ps;I
zDq@Zf=R$i934*>yb^3S1rAavV>d2{ZsfkGUNQ>^oqCM#Dy%wruF33}6q1WF}nWO0S
z5~|?5TXuw`h~Fp_*8(B~8QycApEB@9Rz}ngjImcT7W&;T9C~X-`~hRJRu{D+-lx=s
z`?U0nLVSnu`hR&qf(4wDdSWqUf-gp~zZ{zG1iT}y^SUioD5U$yjXyYyLAF{djQ^!i
zbo%iHazsPwMjPDqm*-`GU|}1|etLiSk5;B%sYsvsmilz9p>&RS#0f%7$tTavj;Npi
zdCQj4^|wL#YGxYgtyhscto~2Th7c~67VXP=(Z83X0D+>MGyr@QKOW~p^KT#?&HbCB
zab<)2ium8Uurv-!EsAxnvKTA1FJ7S>dM&G)j^r*ej6D8SwX1rZjuHox9S!^T(LI5j
z^6T=gxIdK2!^KME3#R&1Mi1`IilQ|1aZ`Lz+Am?9o!9YuM{P6EZUu6YitI}n#LTxJ
z+?xA!rI@mi=wIRaNCWq6K_nBi-F+87l-|3Ijp-7U7^64)!-k`KG_D>;)SrGFO`1nX
z0twtYKIIk%_(U-OrG8FIGfk=IkP?s+z*fi)ud9{-XcsCsXV_cSy;jMESWll%%+7f)
zWVkVKsXxiLQl$XJWY)zm)wew>go`L3)SINoss@xU+A%7BN2bJi`!O<)5$~LAZBt2C
zIQCEkGHF{ppYJQoM0NP94A4bn2L`)xBS}A;t`#zOJ&-(Vr|D5Q?sd~k<M>IJ?{fDJ
z)We4~N;_h}rpIscdykoP?aG|w{MAPUYum_=V#!-j%z9rzwQ9LZYqtu`7+H7_<;Dxh
z6$u(_s!5HY71#5?&}w3Nqax?C(9C_p$ZV9E{FzHYk|UfdN?RBGudt)HExDFMJma=t
z($0ID4tMma?{epP=KWSvkf7P_(ViL2qo;`r4H-2lSCfgJ22%kwQ9=rRYk%a+R=lNS
zWw7?V%l0uil*2!O3rF5!TmpbHGlzyo_@|IO_E-(Ba!SFQl(G^u7q)OTfaoJC)TZBp
zzDGTE;FC&_2SEHtYx_GuC73%qj1AyE#4okB(E<O&fC}_6z^Z^9Ay-0ILk$(VTlzy~
zx4->At_8~eXes}^=t6V9MCAFpmO=b+Bi(DWLg_&DFazls^_uQ%83Rz8&R=dVJ@_IQ
z3E+?arOYK`YxRYhJKmY7@Cz?#p9&w9s+k`HXpVGI3pc-gK}bV8Pc!`keo5P(%IBb7
z2mT_E|CPHmK!*jN!&dt_=h)!I_aP<lxO6S+Q`!goR|ITRR<4|vZ^uVAfh!TqQQu^x
zg*@1vS8kq{r^y$^L2ZAoA!q@hkVA;E74Di|x%mhOEY@1AK$Euuxm?tBKb4P{V)*9}
zy8i;1D%#{8enMO>T}vg?WC0peAsOx#WL>Su$P=aZWE}75XBnY%FY#*&x=zPbB=qGb
zsrZx+c77?<bDZ5>iap+b{2=f)KfVd<vfJmO;_)Bo6zkt8_6=RF822V`2G&CdIW;7l
ziv(bV)GJBMxyn7-;B?nvt(Y3<e5@*kzsA)zZgAbn({j^YY#x?8pG+v8VaU&l0|(sd
zqAj(_SaRQi{*T~QGn7QI2k-gNGG2Nqf4y4KYQ507F4f3TG|g?BuA%78E_7cU$ng?<
zg)2fD6c_THmKZL^r*@yt_%`rRM5*ZlbXkul0nQ}XU|v6~a?-<~As22nmTP@aq4nO7
ztMj8llgn|_vC>!ABqed?f15s;Zj5FyEX>LqqHw?GpvSDW@o9V|m3`F9$KLm;Nt<xW
zo>Q5_Mxp~`DWL0fM4Y-s|JvG3Q`{>T4E*|VbLl!|mqz*|*66LFEaQMNYxz+=038HH
zpt$~f4DB(3q<iE`=MTg&8S%rhIol+s#n`J<6hE-qM<3GGR<jyf$Dhn{?D@f67xVsp
zgwR#v^PHgUFmq7%DWz0gSqz_68^<-}?F{!xMwKhmhu%FqQ)lf(^|Bj_H4TPcp)ULY
zyNx*NnbxIjyS%p;h}j~*`1(JHvy`HIPHtBWo<kE6uLHUC-~a#6-9APOY&z4WOn%W&
z)KnL1q#gTeiDcyktG$*045Zcz$wG1Czg+IPG%&0gm&hrNG;tUbbUGb~oxU^EYUbE!
zy?}up%S=VWj&?+2MI9^-X39V2N3oPQ58bVtU>D&QllW_7b>=t+dc>i3Xd%P!rfJ>t
z2>Pay8w?mZB*ctdPxK=}ald1wiv8Xxm>UOOgb7SBD?wy&iyAE5y?zN$uPxn~RzM|D
zzAJUv6iOB~r~QcR`Ug?~Va~nD#nNHPmROGV^TqMa5SR-An$`S2$l_0LcrWB5a`ol4
z+u}O*c{0DCi)G#Gv3$sDIC1qtH=HOtlJ-dVM)?!XU8oVRo8@z!=jCG?kk8)J7go@m
z^n;`2nAbssXD9oF1v_aWI>Xv}&5uP)xCBD14|f!eZh{lFaGqqE3x~*y3`$BI{ZZa2
zm->eUsJv5u&UQpVgyL|Q`QD7R5Fpy7Hg0n>!h~ZjK;5yN%OP*d#xJz?EOlo0@!H=p
zn75REx`H^47`=AkJ4In_EU3LmzI4Zs_N{G46{`4BrGT7)DD{|cVlRW?aZ#s{&>QQA
zxw&o62Hr)yHd{XPGx4#1V$d6Qn{Bs3Y}&%jig@*uFCgQ7#Q8aODf@aR$|720=T(^Z
zjd;_a7E71sF@DyAeHeZ-+ux#Xume5>3|I->gx`no7NnJSN*iYSmb@bMcoB5_uA9M6
zj@=IOI=`umDkXTzm{}*EjGw8Xl=Y!k5g;seD&Ke^a#yXAO(vb?R~`G(%HaKXgfs#p
z=$i$p(UGfEX2U3-%S+5HmcPBlC1tEQE>-M$=@EmuzGK(>b|@|Iht<(+`u%~N>`Xm8
z-qOoIU8HNvYcJqfC<(~`)IzlR)upmP%F%W4vjF?vo&8bc7>VZoszE~tun&J9a;Hk<
z?h$4^=HZ|=%_HEv*W1i9*P^4Ey^4T%v$?2GSDVx{U0$2#AOLj;^e{_~cEkq_^K3|+
zJw`jQ6h%u&=B^9UJ{`n&XU7QK2Ong4RFqJA%wQ)_q9cod4TZ9B<*hT4!KCamAdJ26
zxEMHD0Bgb`A~wYlF5;>_|E`qdwb?-O>=dzWtOY<MtTIs?o4}#sp#(rTNvaHzBDBD%
z_EgSMiIo^Qy#0&<+x1)qLBv#R7gfwEi!A`~oon=dXiAE@1%`}YEeDYE@7TImM|+fY
zef4kY&)fmrc<%ic1<M7XZ!D+)Yr#0)>TB`@^kFnnNLO@D1rT@loir~nPREX*e1V+R
ziMZvjr<-n<08GNTH=CMr)vW7WDGCbM{8PEPG|L}6;`0r3+5`SbmCagJ$NOog0M9oR
zOU8fTL0rRMIBR|RY_8(@c_$b?4=f<S*$ZH}M#1OxIJeHugU`#bEtlViaL%m++X9`U
zYGeYNC|85C{KT!sRR8h<2poIzmnz&{2+7w_0ciW(^DC0$tz)}7_TY_@`Zl9H;&xiA
z)FPtfTXO{hj{&mdOCaeT)SpX5#2`%hJR)TDEhW%rg{acy{EGYI2vnu)q4sxnAeg7*
zhB0$T=g$uAcpvu(f1Kxiwij4~?+ZRs)RER^URkBK_x;jg+-i36`Mj3ei$^|QIW{1K
zDh_;954_AE9>C&5e3luX{R0s)`L8mXv6pb3_~bt&cCZ;zvfVz?I}3l}qPU)ZR(R^4
zuz)dSj!Ihwh{Dg#LmA{VHM}qZyhLrnGc#~C%2D;K<M3x47rh+8j1Qqnv@bG`J}<jZ
zg9U@`92Yz{;o3jz9fAVTUpF;oINXtr=S}<yAX~tB4W^rL9>4Ghz!9XordN9Zp9)5c
zh)UP)96Pn4{*))4K}}k~{pH~E8|L1SBHngppPf~i6IKIKWETK__JhXrLHx)O*d1tX
zCBGEwI&WBOJVJf8#Xi0F&o@R|>;0!+{j~o?rU>kpnaD2I|0xTJX`x>Zp;4&kKQ9dF
zqkO0y$&<z8bRM)bT*O7)YAbpDmU_+k3n?Zy6j_}FSU(Rq3txi&+~ThdfL+>(lV_6&
za!Uhpw0#*Vv0F^vLf-<UWv*=h5bp=ZlpsR?DLxVq4M!sz)1UyQBzjY3^hXl2P!Kw_
zT2|0?-HHfwG>usmJ8%I=j_?6>o=woJmjEJ9K2SAK`7Yq?dEWcLZv-UoDFrazPJYwq
zG}_PUqZuWbPAT|hUp)s%Oe(OHnY<BlPP$9_=o`)=5-72)+z{gH=Byh^XGVy1|7ZI}
zz|e8-E%K)V!3=WCkY59b-x><jYBXh)MPqrt0yU%4(j^er_^tTEwDRqWA8^&lHUjRD
zjV9ApW=W?%)PE;GJJAH3BUwRFTK}t$>hxy2I5tyL2uIv_Y?=f)=7bhB2c;CBE8-Q<
z&bN!VR-|ojBLl2a#ZsWGxE_$kKfaW}$C)eYf0(1<Y{ir<JAI<`BG)YW`YJm!X+U!_
zh=p@cmAB7+kqv$FowBP(*QS5WJ)XkkroSuCG`g!CUs7<4#%5m`t0~zP-RV&!@S<eN
z<lk7^!QMElFW;0^R=lOabh-*TEe70B*y_R5{qmd;qo%YsPgRTBx*%CL_8n`g;&j>F
zJ}aeiYb`0Z)gS4mvb~3wChyf?5KEN>owjU#vFNt2mtH)|7DlqDFQv_mMtZ-dzw^yi
zBj;R6ITTl}e)N=7cK7uW;+uAVmCjF%OOjjqb()!e^J3wf#g*2nVxy+1ErhB+gb8Ap
zv+c|re5YS(`SFo`@L>YLM1HiRT=i5olci7C3njKAmYeiK5H#0<EN&0*QSCBr8_=-s
zzyU_8*s^$X0*Rh{Lv}qxC|66zd(kuTTauXAlNi`Ffz&!yi)hnJj^>XlBC$%Z#P%eW
z)9Pbq>Q_e*vmbLG(7bkqgceI?UC)BJ`V9wYOchJ(k9ZZ5caPQ5K@dpvqvCB>mg2Hv
zPD|d}#;;?qEMcjfL9EJ3?PJD;$#}rS6;e*GYbueoZNdK`-+6$Z7g}%zWz_RYSf__d
zJ{zooAo{S8lnG-(f(W@93a$pZrHYEW2u0T5h;ZoLu=4HN9y!Bs?5Kb5m@hC;^%6+X
zKHd5X>_smtM>r@<oRDlGft*Nh9Rh<bawIfVen6%R$U>lkKJkUI0;Z~=_Qb3AS`Ry@
zf<<QxzdIrm-JvSOQP?UW-GF_hQ&)tO2`@dT9?6G>!|-i;+-JiR;ri4z1J?pc(aQ$@
z8oiJ-sUHrBp<}UCNcU-&f0#37oG}IJl>)cFc*V&*kN(!SS%@)u&tj>4=a&y(uK91+
z5Y@|51}wNy_>h>TF0RHJu#-@=3p+Uos;W|5Fr-y+X-&-?6RY@EJ$vn&Mt}u(6f#_r
z#qzoHEOa|+k1=VE)AwgslhA#O7e-#*5oSNqogB-?otnhG-Xq2r`(HaeuNi&Irz5I;
zpnx#3lsD-B?1W(%Sr!LmCvj%_fPLl1Cy%YwIZitvNIEh6fT2>HuM8`wO(O9>XUl(Q
z=8A$@CWTV4;)=yZe=BI@+er#CN-}EXn^wUZjZI>C)$anv#0Ct{%|2{_kEu_62<rd)
z>COt8t4u{nV8}qAIYBXeenS$JNg^-L^s;##%ml{_O!4B(bZ1I$V5-B;oL`?seTDqn
zkM=(&yh<3h-N0ZZ>1Fr~!$aS2J6mK*vNRjcO^m0uJoDpQzBM^8_TI++I=ElayubJn
z_uvSu!)Ge|GWA!L!j&ZKR5U`L3+D#;mrwGnps?NKZNScWg)a#8>3*^K<(0QwneSk!
zV~w>dW>Rc1hur(1=Fg@bYipn5szq@#VtB>E8I8j<!J97{icDnQ1j{$~h0IkLe-Ze|
zUJd&D+l{|aifLHcz12_iFuF}joj|2C-9QcTszbfz=ad5F?f_*=SU^$63^zGBM|-8Q
zeBH~SJ~Omr){JFN>HDi0J(xhW<6B8o;54(4g&QDOr*@e~)-5;SBv6e-48KVg--$QW
zgSil%eg<ORVy|9Df3Si5O%&id*wriIubNm|`4(>ex58qo(x7Po;b48Hp=p0xVmEf}
zcYJ|L7U0!<*gH66ttTtt5=L?F`i1ZHghMZLe^Pq||E0w>j0o`rT$>7VF>k9Je*Nv?
z-wQZ@4YbcM&L+$(gPT=t$w=j4eZw{%9q!-O&OcJaR~kCS{QYVUQ)-hvvPLyRVS|+g
zV?5Z5f}%lYg|^D~?-^B2`TT2U08I2g%qt`0^fs$mICcVbnD&6oWk83wDm70f8_UWm
zUFDWZn+6$pGbR)r!(Dn5_6#%WDWhD!{aQ+#GAL3`cV>Me-r|7u6%I&Aipph$L+Nmk
zxKiaX6pm@@l3dMU$4f<5Zqtx5r}63F!|Tv3QA(51q2i&$!d8eaiD#1g1&DL>ue%1Y
zH?G{&j0L9r#()>olxcs`GmkLm7a$$-yTuflkixJx66p#*&3saF%v`^G?aW?reRp+8
z$)Z18Y(Dlohny*uDEQ+wQlk+*rUe9RF<CLx0W4$0XHsdK?p;7XsOkMtjHn^-&_>tV
zg!lN1LC(P#e)sAX?LaQXJp566KYo$dz{=~YlP22T{|8r2>zYMZs=bHK5=tk^4M8w)
z^D@=-u5I)ZL?}u`nakLeP=dOQjd1lJ!Pov8j>k4)Cj=S9v;*<y2Gw_DrEBvlHt}6~
zPb6>&?+^;1p2Nbl9uP$zWVAV3dTupq4n_hI#DXxW#+o;GRUjqSlCaDoI0t}C=;hBw
z`s1q8wZNX`KA)Lo>Izw`CVz!2%t*<FS2jKwU!j{2{Z&y@$N?ujg?QT8?s?C<r|iij
za{R%&0o{b&rAaknSG{h+B*NJ+OpP2-bEv7#zF3jaHO`3CgYBP^qn+1Q_{-NR055hD
N109oVRay=a{{=cPz+M0V

diff --git a/modules/web-console/frontend/public/images/cluster-quick.png b/modules/web-console/frontend/public/images/cluster-quick.png
new file mode 100644
index 0000000000000000000000000000000000000000..9f3933e0381e9b0cc661e75f776041df95b6cc30
GIT binary patch
literal 16407
zcmajGbyOU|*DXpQA-KB*cS3*(uEQX~3GNPqhv1sv?iM_QyAJN|FnDlx4-g#g{J#6{
zTkEa+-TVIPv#M+AR9Ek__wJhRa1|vPOf)hy1Ox=kFS4Mo2na}E1O!BS6eM_yKEqu(
zya7Q)L0#(g_4WS#{_*Lludi=qW##nr^!Dy{e}8Xtb8~HNO;}j?_4U-n#qIU=?+Lu?
z%huo5*IRh-dRJj3Q6A`;w+W9AUSA(yAID!W2VP%S;<`>=CqiFe=eDcEa;6_ZqN3%i
zFMkJ9nxoxzUtg#1UzJ4UFYlj*FJ5E5iUB>k!|Qj&<N~~0-5*|GRYgVXrVfs)?Yhrj
zMS(6NV)EO!FEbDy)kxS?gY)Y_S#P4dfANx=Pww60OR<0O%fDUYybX7+U}0g?pyB79
zo%@%&#bnO}B}oS_poB+a-^=;zx7aqYgkr_s>z}Ka?Xz2Po2r#my^Ysb`<i3V-!qLa
zk_+%FxNNi#71m8zewcxHx+gvi`X}p(kG#G*N7Rp}yVbz2d?_W+!%a+7Yb;zT&sfZ>
z>d(tw{o_*U%WTYHXIxowU{<J)l%{j9pUlYmnOoY>>r<C|Xr-unw1jD4_Tc8z{n_h!
z(Na@NfV?;=xN)R3?$7?UdfK>f<bZ;GoqO9qxxm3t@0_Rok)H|GiHol{*EcJJCF!mn
zdO%|j{Vz6!(~}cZk+4^Euc_!C9$Y~^n}ufQGiA$V0pGrTa}X7QI*Nr=tPBngi#j&$
z59K;Ho_hu*#<r~~dNmE+y@oa~1^5P7l<m#W&CMogcfG#e9B!Hhmg_qw*WJ7p4(&FD
znSxY|Td!WzyVm%_`s!A%Hs?C7R$Kide@o~mt4SDp_q?PI{gs!nFj9~#nmb?KJIegw
zqA4qWusI)?GaBpqLtIwfRYkmIbz{4&w<SAB$t<wQ*SKbMsj_2OT*fdUpggf?I5?}L
zys0rgvVJhv_u}H*+Pyx=@+(;5OKRQxa2q7vMO(c68n(QKZa*v;I}}_wJX_(_RM3H+
zw6(T2q&4w0zrL&REhE%DD#<(Km#y^4<LT|#s8(2COi_7#c?xs!8JkV>>BiVdW!YJW
zU#aWYq}Cn>2ZzLkf8x1o<uRGxR-PND%M+7A8G;5ZE%hE$?C&Kc?}K`>4Q`SV5QL|{
zfF#sCmQH54FcGK`-g5Ci2}yTNVx&T;yw2W;w&r%@U`Buf8OR92k>vr*c$g_*6?!uE
zso$~y?Ocg?a5X59cFH&$St|GP&*w%Qh8|{w4>)K9ibd#zI{5Yt>BbFp2HALQVISv-
z($H7a(3W5yTMbeImO@c5K5*-{5JU4VB(QPqEMNf*L^{9i-rmfrq1H%&V3Pa%ErtVb
zD=7M#uf`S^ulrArpK|t}_CJJBFp>qT#(tV2!4Pm|UINT~y%PhAN%zliDBeN@!Hj`i
zg2h1SD2rj(daw!U)9R>LfGGfd`l(%ITUKg&>FMd}!mPN@aQTad^K#ju3MC-1n}m`%
zk{685q|O<{h7g&FzsdRbhbRZ3;x@jP!PvFVz81mhdCiFrGgllQl8V3c#Ag-Cj(-(h
z#!<8cQW<~_Gm)QeX06`*Q_QPLVGWDWH)63~x-yJQh3E!eKX>)!4_U+KT|&bLik0Er
zTXqw&zs_gWuiDg;z0<z$`6s-Cn0L=^SZ-6vPPSjSMlIjz%NqxF+b2sHdh=~@vU6p=
zoF(=~z!VL^lvo*Q8qa;^7mi_w9PhK&p!a@H)2fH^;KRv883#w-j|!~cDh@8X!qJHd
zFRp^8p6pt;2iTQeX%%r<qzao}l|TaIJVxf!7?GYu0r8bHT*vG#yQ{yCPLr-#Q|we8
zuk#HsYJA0!KuLiT5Hv4axl|h}Z$u@J!ep@6pAJ(-9BdnYBbJgK`Sv{3KgpT#mVHb3
zsZadjBx)eB#t@<nsJ|T*uFe2z?_^Jlk~^Ln$>&aag>Sbrk%#0W1gfGZN$KHj8X2~i
zO{RLgZ4$)sOY>L?IuKi6HiLipeN+FKPO!BNx!JAxt9w?vXDixzhh$vy-Vrwi@dRog
zCnqx<dNo!04{r*rHE3^C5ICGgRN|7Krkhib3+oi8L4zld1u%3BJo)&f(UXbnm<{~a
zkgf}EFa;Weaw%_u_Er6o23hf=B}4Q3WO)wP0>C@`90oYiJ($TYL`dN&8H*Oo+?gLq
zqR3QE1$3*cBu9Wt4z4L`%%S-UVRAAk9oz5h9<(Af^aZy_KK^=6isWr}rco!b-wlF>
zYY~ULeS0_Fcdr1DR#YXjx3El`)m@Jt1<t?d$?}ZQ=m7M2%vbI{+Jqzt@5xMDi@3lC
zn#DNuY;ye-#hdmwNguE@e-ry`V@Vp#^`L?iO*Nq@$vd~=FJ<m8qLsgU3l`eR#SLu!
z*Ab|nFSbM@@SEX88fmA7-Qx6FzGa1~f^a9-x@Fl)-Wy7_RGyUx%A5N(&XOgSYdz)u
zV1)QTXB91Eg~p&>XP0yHX`Cos#+nWi5p3{>1!Hqz!IdG`nwSDch*chYThEtZes?{X
zm<BliIqvZw^3wwB$u2+TUys%r$Wv=J>sn4tCVfxuN_o50-vKs+vaq!cM~Ov<A#iLR
zKL=Ol?VBUvN2{tONg<snL}+=7{T554eR36ij6hqnMb5id>thVkSD{TWY|VQ^KeIN?
zrd_G;N_u^@cAkZGh*_p$=2;#9Ug65b@k1z9e4$YMNX92JK4@l&&~()f6gC?PN4CFn
z6{D<H{(?##GBK6CEkFiOrzZ>j5wKItat$O3RArhi053HfRlDdfAYe#mU4&*`p<A=Y
zY~5A;An|ZtP2RMeRw&|+i^kdOZAg$nR{M&=0>&owc53P#_)T`8hI&dvG8;$}c~B5a
zMSCSNWCBDbB7LWb!b%&}MGXlpE}FPMbw6KiOD0Ou%bFaW^qf3@nt&ULvuy7|&3xEV
z+vKfCKy4B=>$`_u<$zB>AYGVTP>?i^v@H0oYIqzBr3VEynwk|Ck4%Du27(J>fcHS5
zqXz^OgWx?(%-Mq6nD)?w|9wIdQPuLUdnH*W@n*-gtCJ}DVd^<_#My6=X(G1R<A<yZ
zb5zr!I!5RCE@JeVRDQ?~b6)93+v^kI{q3Y-bY9l8DAQBL*tPMsGKKG$T&z=K&_7nN
zr5G|#NscWo<hiVz4JsI_{HC5G$JDl%lH%KTQ!wx`@b*_HJ0K|NA7>y?@3m;MMf35`
zeKDK2t_kjI>Mz?VHM98qE?oB7QkkB`(Dxa3htljVy7uSUeFHk!B<njss9{8Np~F?3
z)p@cGW>lfNG@0|#bV8CaEBifrb>%<5h?YXE?mKS5`RUJi1Eq6rcVEUn&kvIP;JC7-
zOQ2T_RAaYyLJT2pDhM-8m9fDt<o0jtDaTLilB<-caPJNb6VC|Zoa(&ewbbCl3vmo5
zskaO1iKf2n!qt?O8yynm#Y1#gu)`a0rFFqPdq6GNs9Qo`c9r92rGM}JSc!W3ZoSSp
ztcl*b%U=28<!@IjT7rR={d<REqI!H%>l*A^tpJs<zm^6NOpXsGnP)5QY3R-~>|44%
zLTT2<R&7Z34#Ek&kl@AppC6&uv(GOM=p;sttV!98rP4skexNPZX^!jl$ZE=^y@K{%
z9ewbN$Aa1qqk2B*CmS+_0kHvO0zI$t&cTFF8G?zI!}>^_4ebIuE_pK$fs1p;iZ^2T
z)JR<^qgtC=|HS>=Sz&#&n@?xBT1eYxfjo)LZl4idr$4o2zL!{Z5?&0@S$w{=%9C(*
zisE1A+GudgPNJ&@v%1KRqojJ3fiq^c*#R}`_EuGc92A#tn;PBX9AIsO0X*fB&e89G
z8H`LC!&-PgC)qf1zcB7Omk}NCr++egDgK^Yu44qw1%oiT$|xtaivxAu<D+p>ZpzMU
z50DCp7NILfeJdWk&!b3E_<W_<53FMnDc{e<ut>8UwsK;)BPmnn`|E^qUdRMgY73()
z@97%$ZX1x7`8M(#=+7ZCZo?$wsrsmZI|@ZXCx!L|Tl&rqD0ha5E0oqiucW$31}2pV
zmeP1b76}K3ewzHSZTyBR<*F*p@xWM6X#<=|?U`0}4H|Yjk*M=Mx~PX(Fpa_FWqcq)
zYLcY7A)~2OqdfudDNzlCT35P{K6;U;&f7i3xHW6-6$u5LXJ?_-u>vXs9zFx><cAP`
z43JyN*1(R;7MYy*L|?%v48TC1Ai8Xg@y{_#`rr(9T-_{5V4<y=fATwQNgF0p<3B&G
zQd+;?T^AJ!sWv64fZm$oful?SpKJ^Uv+t|v@AV?h%n^QpQiRaH7+|GKNjD`meEIni
zV9!F3ek6;(wCJ3-HWM?jXV?q^Hx-{LV7{ZI?LjQnrJkMI#VlXZB2itlyCAKwoV&LL
z%1@|kXw*N8gHOhGLaQe+ilpk;3DdXV1JPICFTWgpg{+pbvb&nGAoo)kHv8#zTLi(5
zzLC;EA2O8(W;kha5<BRif-<$(bu2OQZwe(o*!2wr7jgyz@nZmxQ<1xi>KRd(z=R(y
z05?Jv+}O47F<3{e(sJP7RWM0_{7lfzm?ltcrykf#7a+B^cI~2ih$0=vOj)Ld4O#s5
zm$_>9<H>$9btchxEeNbKMJgwQ@rKG2K;G`cPBEChKoymo1Jbs1RKh1iE=4<KpKWJ!
z1mfa`eUU1l6Fdqmr+#b5bi{?0UYCZPO`t?TdI;Rk065`yGRWDvvLrDGPUCOXK9eO)
zTAD>mLk=fqRXh$o_95)-zs=pRysraD&8$E=byJg5XK8bmL%I+hPP6wkF!R5R>q>h;
zuomY?p1LxndeVfgah$uq*X%9~7l%}j#IQ6HS9Ey;@WyG$`tW4#8ixPWG+PF8(H39I
zS<mNd@}|XstO*I9Dv^d2d9X7<wml-cW9~;q@VBw^>@VH68+ZDC`?CCyFqFRhX-nDZ
z-W=gT&J^wfF7Cb*q&GD{o}I8ESz@yFd<-|q&C%4>)awH3!zLa6E!1xz44<G~8kr{Y
zvA4FsnXE&|hQl3!iN9iTDZ1x7VAp)?p+qL`?kMsP9(qpV^zlZ5Kv?yDwlAXNz&>QV
zojxdJz<<!!cImO6<v_VN?AhjI$Y><f4mYI*Hej5#yrpPle_o_z`p0c>Vfqh^ll_`T
z(>woHJHtH_(Sg5ME3LKYSUHRUeK%YV((~(m2t7XhM(8$n3MQkn|1;6t(9mte`A#&*
zEGoK_6H1tX_g{CzMUAEJ-Rf--s+2*gDA5s4%%?Zcs3eZDNB{0tN~2t?beY@QIc>l8
z{Vw{~O2dFE$}RpLbRPg*etXenhpcamY-gcP>Z(^rDq-gjYjJDyz*qEo!G^4MvBpt1
zh=WC@KFu4v2E!BrA+1r=2b1O~xfF0E4Em2Gs?)-i@c$&59X0OzH5>y@6oX<F^<_$j
zM}{F2HHs0lBId3iBotPyaB|P|D8uJS3``X<aGj#KZi%$QoZp5wZK2Tg;7NqYqFlzB
z--&3bF4<eRgbZGJKG&Qc#=NdAp5LF|p0LSaL^I<^(k0M^NE3uXaEEav>CpOMVLR~n
zohcaf9Udcc0cZu_u|Ek}JOvjZR~#c+Him>ul{!S4UM3!sgX6Yj^aj9(2^t|9h9ykT
z>=z)GTusj;iszf-VsL!kNAW4IRL5b_ek(aTk!U02qir*mhDrMlod$|r&pw^6FE3@J
zSsGv-EPph%J2Vk5C)9!n8X^1&Hd|JwSjrfies-khFGncHqF%mz6aPIJFa30sA&)?p
zlIvoAk#)k&%J;T+l=1u^xP`pGOV{jK)E+@{WTDEaQ-Z$~Ij*5cEAxRlEQu{2R=G6s
z1v0Kb<M+VE_3S%(Y{AjkMzmdC)G0XMsYxd}M~}ji%ku@RO_Tq7f}cJfi<T^eA*y-K
zFvwC)3R4f5$?;~IvxW;j)Q)T(4YRe&R@d6HWVx!G?$Fv8obLh&wrome1CD0ad%MdU
zfrqudY-K}j0>Ttcznh8q$jgM!pnf})l_S;#|C+aIU?{vw`9sG03gdN^pbjMzbK(g~
z`q&3M;w`r<nd$p3W?TCMJfn46`sf$u61&GV{w{n5ZCI76mzDvY3vgX&;s3|%NxdxI
z2<+NeQPsex(_(U!dKzSW(=opNq5R7ft>@cOi(q93tXSjt+)EWM)lb}L(Xqogm)**_
z-2b_qrTl>83ANy`k<F=+9!$mE;mKIq-0wU4-s&bHg`*2nwqeB-Q?!YnB(K!xn(B24
zag!y=Ty0CedN3;Fpn8fRi$LAem`XecFj)l|NFA$Q6W<Ta8X~R!u|5J09Ahwu8xEX+
z|8=};l+=T9B8jiuO;OY81f@*7eF%HOha(TAIf#g-rg*j9P<+qK8N$F@xaU6@8H!Nd
z$$lU-tm4zK+HL+TV$zUGCBeTvG9y>w%0YbFb<cMHXBzgnvr=#6!g1(UACfbq7lk%0
zQXO0|N#SW&>ml2|_dc6|$uFkF^J-BT<)0f;Hi=%&u!v!8t(T9f+Z^Im6)XiS^MeK~
zSFy-{Yqud$y<%A@BHFg&s=qK717}!2A6Ump=08TRsHO{*$V#U)vvCVbecnNFR1Z7%
z23MjeJRlo^3!4MW%*;EpFN<vQjzcA;r&WEtgfMij#Oa;m(d?I!1#HW^4BO4;s$hMH
z-*+tAVcWkv#>JpyYBvd|?P5e{O%6ePfIyMUjp%>W39Tbo_N{zShpvQjbScNOa|0KR
zqM3qc+;mB%K1IrB8L*Hm5Km#yN<=vAWEsQr)xlu-EqtN{A&hwV65$El3Zqy^+Yd^3
ze7ZPKlK85goEW{0WbF`poU$=5_{`oi*|ixMJfPA-4Tjo;P(ByBy^SB!Z~%ivb{Frr
z8JIG<MF%}=e)m^CjB0gwILO~|0PLF5io5~qzqtE1S25D!4?GNw&q7G0kX&@iGWwZB
zX9$@prJj=7{?R?Q`^MLMVuzVmCP?zIdxJZCs7Vi&@?|+s=W;x#E8jm;=_-WQJqR?{
zgoI0_x<`%t^boTEe&I9T6Lj8uDlAmQz6tiMIe4xyK;{+wv`zsd0CC@@P6~cJ7;N|c
ziU?<>w{V7{5oMJ8pTYlXfFJwBNXcH+xK!UkO+xp=Ey18&HQT&i%I}cKtvK{tw5B{4
z6KvY3>2G%Us*6xv#Uz>jsTR=sV(Zl(qHX?|?Htkhe^ah%&hv_@Tcu|3O*4kijij_r
z1EjZMc`<*;1{F*dqZsJ5qi|7cInKtDW!cTyiDYPLaRaDno_571uZkC;Y?FfJzM@rP
zDL418S^JMqe#(z!6<kym@}M6-A-K;!FrtS(z&RDpv;Y0^^&@ol$%<;S?nxcTda@I3
zJUgwK5m#0}G0pmY{QOfM49_`oWBY<WK-TOCH~q5(&g=+FoUM?ruESUs5^c09uI^xp
zrSo4iA=C6F$kz~=v#~mQCasSZ{79gsTt;YqC5ct5wZE4<1p<ZqPl|Ohcq=zRTWwE1
z&n!dId;E5r%fX_Pjse{6YVuw6D($$%OsAh<|I}~az&@nI=%&EJ3fka%FKS7Ij_{So
zrysSSdYUL1^5xVe1z1;@#BYC^Cqn@}U!TfTo6pZG6LeQdk5xkDx^h_3GVjMSNTu0P
zJa$xc^8LW$3>nf+IDVAP=dh8Cq;k^1I(5}~Z35ZKarSwvF9{-L6{1Qm^VPZ5cVve(
zu(XH#dy}GZeZVV9RIIA0YRO7MX07^s2Ig<%zBhcDrW8SdCUUxy+olQp)9+6zKp(ff
zVo#-5D$Y;vJbiN~Et9s4`OSr%<Lh(OKV3g*=fWNVT8WOl{_qYV-Pf2eAMhSVQ-AV)
z2iyN$lb3qY$GF@yMX^>D(WESZ@t%IO_lZ+^jw0d|$@}8U;2!KVe6JoS*;^Z|Q`1rL
zQytsf;k|jk4`w7(`c!nOU0W8?Z@*?ckxpMFnC*VAN$H<Ps77rh=Vzx1v|XrGzS(n&
zA;$~-+xU}wK3{+psAj7^4SI?@aOz?bORX}F-+O9@@m7+MkETY1zkCOK%IH6(W8ik)
zeNiWd13boK=$)z={|<1UgFT8iml_RSQncX<J3+J;Em{w=1)eE*t?<E)3M8w}7f7SG
zK1+UNaw!c*9|6W&n&m(Av#@Jn_H1VU?%&uV07^Ix<zR677lTtT23*Mz;3`f6SN1ob
z0J)!LyC4EGN$Jz*>=!c)w@$Ow8>Vb7B*xrn6I9Tj%v+!Bz+90t=V5yCMY*e)q{X~;
z7hga>STE5}<<h3^+V)Lwq}oC-?N3PrGN9-)&%ngHU|ug#oJOzj>|wM9y&>~%qp}DS
zskVc^6pr@?c0{-fQfU;^af(>;wrfKTCK$DCf<t_di)1ESMe|qQ<qpkNL;6q2zuPD}
zBW&-DRh#|m4<NUCz?#OS5bkWINOv7ANzGugE2CW6!d!iC$l`Tr*-L;URsImls7@*+
z6%E5_^o*?~$&HF3zv;R_ZB3d|#j-bE&+BT-)P<aKio`%~W@@e++;OC0<Px{EqhX-n
z8aWmGbZX5*#E<7cpqM20B7mX_Xr#5OEBNpDsN<Z|(A}A~7n{iFo7}LkB`$kEWM(wN
z10rChrHD~66RAW9*2nv-kSr4Yd7rzW7RlBvqN*jD#iqxl8+Uzt$k107_dVkD>6i#u
zR0{%xBlW=_K+6h|`tawY!68ei+x7lIlR)SEL2*<@k^kcKdw32{j5m@qq#8ktwBq1#
z-K2F~_jNBT>(#9}GKnR1?^+o$N+EXEu-9gi?{^EZxRkRFC+UbdX45$gOw(dW_x*-y
zYMt4(KVboJ;knPBf~VtO@+3E&Lk&l}&xnymV5MbOHBZtoTCHcv?ba9&MszWk3Ebns
zk#Nxq_k8epGZZf6(<GAHoP_w#Av8O%HfcB@GmSraNn$8QZP5^eI^!uBLP*jl$P4ys
z6hW>$B-g5j2G784IL98v&TD)i!!7B8+W<JiO#b61;c*Y#Yd#Eg_zn!=z#Z+Y&Ti9c
zdtKS>SyF4gSEJyj_Abt6FCiuL&~HWfF~ytLYI*KT#GvqmI2>FCJeqFwAdl_f{WRZ`
zkJ77{LdUmX#?BO9_SR1Lgs#Riu|lNH|6@(xOVV+}t!aKB1eX+^ZMqv4hQUk^bgudl
zm;PYfP7X3*eG82I@`ZF22h1<Xz<>-gIb08&|28bjJTV(WjV2?3$-&kArfaPc#ZOW<
zq}+bt<Kmwn08Vfob&zGH*81f>L{Tw0s{reZ9<BVc7$L}n83lsr>(;cA6Uy6%+9mK-
zR?RW2M+FDS*3qGf+9hb+)q;x}b0F_jCqu-da=bgF(7aDSriKh?wq>iftl70xZ2H(w
z=}_rLXrqJK<A;xMtFDo0Ee0QA10waK_|r)yX0I2oqbg&!s*nDoWRVnjyHF5su>Qk*
zEim@@)JlX+qj>j?RvPVm$G$0^m5isGj&r}bHsbg!jr?pZe0mo!kj2yoUQ^Eiq;S;x
zQS+WPtsqU(t%^IXT$#W<SqJTd#V?@-cFU08VCCg^-CFY<1bif9!4*Fx*{WA+t=04k
z73707n^a3CA%qjQxZI~#(v{1p<e}1lW>x0{gqk9&&-J5bov716s0hlGEOzfGbZfB3
z9oG%rcuMD4Np)S;@H!l!csLEMT(;(@spOSjH|X`{8_T<mB)Q+<e{X4aL2md{$-z^_
z=}i|SMx5QyCw=vd8C|e!rU8$qeA1tWVLj8WvAyBOqm1K<2&Ywd54RFqfIDkGs*>#E
zJqb~80fMBO?fpMpsd6>m3h(3%u#@%7CT=z?p?_;@JpE%$JFZ0z^bdEd$Wm08?RUOL
zt5jWA(9&BJCHstS@_B-6;0A8FI#be({2dBWaM6zsWFJ@yU&T>^kehgRa)!+>9-?~Y
zKZID)=H{RQ0BR`|plf^`1^-~nk@)B^Kh{u|IGBRYUO<?ijRTPyHDq@Pi}s=9M1X{B
zsuBLobomKDD*~5)4|q%k2dOX|q}Xtf!sC2+iIKwNX?V#+!Q(@C*$u$sYj_#LlBuk|
z+V>T_X2B8~9Z4(dZtYs|d&<2TZf7Sk%XL(~PvVf%KmOy;S(apWsT#QlbgkSfL{pAK
zoGeCJ*Y0CkraVm_wY8+>=5F^U=?rby5#>0UJ?{~WeO+<~V$g7&2GJYl2ZTT+POhMH
z{;8bENa!ypKod0ZoD3_!VNZGczQdGAQ5_MMdpi@Es8rtHch}HNmF$RY?U6iGc#i-r
zxx0RmU{bl!9q3-$7Lp)gTROnFi)GP}ANc3bMoYV8muhVhso@i!oo%Q_;i^!%S~GXC
zclW?FS(N;)gQU@scw3p3x1r>6Qsz1a|CxpK4_czvy?jy3yN3thRgHT*<VJ7}>yKcE
z-5{MoyqL1R$8}B)cGvVDG~E&d*2O-3F)*4^3gmtu379Ar0n6X4cnGIauBDX9e#cfx
z_+GPjJ%%%x&PE^8b1)o{og;FE`Yo`GRqWi=^pT<k<=55Nh3{JZ2m5ST-|;und_QoW
z=6;AT3@VfTJ@>sMa!dcDYA2AY(5-veb~kmCi+RmS;NCF$G3J1h39TG3D&&bsiUyf0
z=}>cMkSHb>TfOy69X@Gt-}+t{Rx|e{mH)$&A=uW<`rN!0ST(x)GyL*{{Mm4InpVX3
zeoH&sip)cmr-4R?p;O9NANcaC{wx@t0MbGoD%5f<MHYaO6FeP>KO};^nNCORA3a2W
zy_b?6&<ykgr$2eNEDP^yx4OJnfj;#Hx4xg4!W{phpAC2zASHX3Jd7s@fm{a<`QMs@
z=PSNEUs!-w>!W?Gr#k{22OD1{z#6ehfkl}PmyrncGJjOcP2HbiPlMk54%g8<0K@?A
zcuUk|HJQtF5$Q*mmdFFuotGleGS`6HH8IEg8D<_G|25h&H}lAw#`;z52`yuvtfH>v
z;K9qsHk5h2>71;Br)r9sIg+MOMF%6kJg;xC{l}B8j|S{FmPJ3YqS+*IRPkvmWU+C7
zEUnJ$CX%abnl1~W&2ve^kGNE0aN=b&>}8u9csI~|B-@?H08P1d8X|Qn^Wkx6t=?1z
z1uf6-IlXg~*jih9UaeS4Yo#~vt!?!Zx;jQ$gYEo!H67Z&k#E%x2x|M0QgMFill=((
z;@!?S&{xbK;jVAqM%v+Y8rBvoEm6~R9^%Rz^luL|oKMTkt}*GN5OrJm1*{xNZxK;i
znOWj=Pmeajst$fJ%jIwsIATK!ckff(^yJ>jzg5K+V2#Zkm<#^ps4qmDg8|tP4Y2sK
zUl#-F48tbGCq?;h%b9i?UfN#&JsTY0x4NB7OOp&ZJxyuWBr)U8LL^gV`%6$_zDKHo
z^q~Bj4<GUgjxO-hB@(`@)g#jcBqS+}Yl6EELy+0tr)}wc?<-(gsgI<IE12{akYo-u
zjo#4n`+!B&Hfr(scdeZf1Be8}1|p$2Hv-AWFrtMKe#%E<SRgy#k#Sn5(I7MdLx`an
zLDGRpy$Ijl8BRJ7kzpcs5rX4hxBv)E$U(^8Sb~s$(*z;oAVS`NQ6O)E#K8z;KrjN$
znGaYkMZYL!Sgu~Esz2%e?Csr$YY_k)!!!!!1`#l$wBiQzw_6H~(1@G9t75$JUGY9a
z`P3sft<;T)T>kW9_&@IW8v2t}aPP;jE4EG{uBP$nOiNWyk$>yQjK*qa{U!=}Gr>jS
zXg=F7%=T1Q?n?MSMIUgMW_zl4g>}Lb@FpAXsBJvPL5k{uq(O|6%B)IuwOj0G9vpOY
zv}#SzT}7Nn`<xtm{+cibF~@!L7t_CrIhI?CN2%%TqSub)%5&z2q6_bT$BG77q8Vio
z7*5cp$W!!|dYNNb;^<~oEqm4{U0U6dG%D?uvMhvK>@_P1eV8o89zNU2xEswRAuVZv
zXbzq3hV6o^cbt+Cpi`kR?(jZxrg&|dfg2YyHUIlMK0swTBSEOb{@}^-VtBfIgwqMC
z2{HM<pbsRI4q#k)=C-8p85uU4db_G|+@I19PccM4J|=1IFr%C@(*tzm7!xPKqaR0I
z%gEYtb-pVZ?9ic9{J~F8G`|7`f1gmoxA!x^b47&7<Vf;Zu*A#SHTf`>BC1`T!xRk5
z3G)6<G1Q6ZUB4a(YY{cJLH{e~%M}yP+WGxt<}RwMO%{C8>oAWu$2$!RI2D`TxpLhn
znByBF(RGd#kqo-fT>&oBwy8^KW_@HlomGoU{C(u%`t_(Sqq3m_dCA_EMCnMRQTVN*
zG^#kc?{ny9s~NNgszKXqu!B0wuXad@f!9>#C7(3^r&OQlOz_!-u2wP1y}_Ok4@xxF
z_nx)bsZHe$r|c**nzc;^OqYOlLB#(TT{vVvLvYCcBg6k2_#S!<DOWz>AWGou+=;(H
z>_J$@HReh$QR>N(Wx>erfq~+{KqQC+HKf}gMjZ#+5C`MPg1=D%P+ahrypS(?>OKO?
z$hy!qh#}(!*~kmoOINf|E?fJJ=~97?VC1J**oF)+5CjZV0>{aMVW^mRsF;Ef30jB*
z6F`<8a1ewX9)z3)qmGCD#wVl4BXjwGI$xNqE18+0z_%7&UF!>0SKWv?wzQ?{!3#C6
zbE>oB^A!_<^E$eXcys&_AXJ9p$EZiNEk;?2il(gyYFwEh#!8Rv<X0Y^hSAxnvk<IK
zW$iK{9wE+u8T+1=4Z1-vgjx8K5YYe-KEszp3I$Uf6JC$FKqM7OFv9<K`Ueek;8LWx
z8c3wmpowCy*ePR&szUH24YobkwoAQ*DZg87v0ZEwjQe}Xw~=##!3F|{e$cj0<@?3_
z0pChtf=bLrp724$RQ_k~#sZ|h@oZ$?pr(S(X0{Y#ck<JUlR@Lc9$ws!+B`MQv;L(U
zG>WBzyU{1V6q1oam1DaHkuCi<5m81VMCvwj?jK7QcGkr1FL(`y4c<oK=&>RPMWlb-
zdL;S;5FOb$0)^~6JbtlP0NVy3>t2t-MzHniEQ3ZJ2P*zvT^-xx$7q!kyDgQNdLWA(
z5+w7<@Gv<?ar+s1Cl+7#+TxFImuysOJ4Felprg8BahO$owhD+J`f>4Pc~>%X4lUa<
z27`XU-XR%iuN54SelhCZG&Yg3OWANmI=jwM(T25NL+?vLBI&B<N^?}l9m41J^!tVJ
zl1czo_2qAK1joHk|8gtEyDs}|?C6GNZkxDjvsN=_)_T%kS5B5s(q{eN27^Ky%P|$N
zH8L)u{XMs-nONA!vy()ecc<2bnS8Q+4|k-^w(l~7$W1@m{BbyT5o>cJnZrEIZ^dg-
zB1KDWnvnUxFY1)`xI^hCTw^JWPH>-;5r56)9^cOZ5IxGbGeheP<!8Cq_mmf1&nLen
zo~ulCCqTIM6onFba|j&YzV*wBnX4aXyu7R=nV`$^^jO1~%uKr)wm=-J8u?1gif_@t
zYIdN}DtNB*Lw5VFgSIohFv(-WrP!5{L758lra)!{BCv-qFvxqBU#7ZdBCVJx?7HJ~
z$F3WR*@3E9euVcJDp1rslA>RFZMvq4u(Ep(`?D+`$)0Mq3ma?m_<KoeCZfsJRyfqt
z5JZ4f&c)~oj*yPIx9Hu!IG*uA@{DH0_KdcZ_wiwz05eNNFQ1m&A#;QkuzwF39`AR<
zCSVD+*m%CpAp_(rAy<(`lK59T_p5(n5I008LmAmS`3qXFH$RvEO&m*++}@JqO)pC9
zQ&*Gey%w_KAR2rMKP6A-Bh@X;F~@SO9jdL8oOM?l9D8Uj7^E@6=Mt3UxfWa!T8HVb
zw2<4>GzD2|_cJMNT^_4e%Od?}4P%;X`ndHlRi&XzT8%Z#SmQS--*QL|q?p$YFi@F4
z5BBxW@vJLYfbjoH0|xi(*^~cZ)zp4(C;6Tm<qT2_9oSByNT*2R0Z7(vUs|sl-p7DZ
zFog%NAU8UT3-rEg?veL7+z>OP1+e4}#d|7DI<52M(MPUn#3+7%p`gZoP9ymi21jrX
z%A2jF{MS;JIsK34&c$E9?|McTVkws2=jV+UVAr!koSuF1zfz?0Yb|`%H3EV+1hTQm
zGf7)`^t9$41mjq>-RE(&E^A#=-4@wGmfilw?%tb<a}I8^6Ip;ib^mhj_1fznHklny
z`L;dw<z)psdOew=QVYH~^o=dq$>dYPEtd_c6TI|a#12DT4>V(<s|5YNu|s@-$Ihzy
zuGf*9qp_)uJ8Shr5GOMM$M5HvyRdv`jY>hw$g&vt^$uM@;ypJhBaNob=mu*RYE@Tc
zgly%Pj7v#ge5&ReW-3>z|KZzJBs+!QM{-((r!I#r+lNC2^UuHUUPSX*fw!d7K2(i+
z=p7m@R2fr#!mZy(?e_(^f~Q5A^ZD4!?uKbQAiR>OiYtc8jcGDQ_xc4Kqr<mTIAjC^
zMz3FMwDX=VBrBR&kjWr(FuWPZod{H#pN!zoP&s#w27^!(aq^4E90VIUV@ARmlnic%
zXyFVR0z-hp-Apl@P&wdk=Kt^Xs|MJKoT{KeWC=$GL08K~vW&}i{>~CKJ}{ZC5r_b-
z7z&xUTYU3%_KU4FLi(ORETB}Ak(lG7lsd2S2O>pK-kxTW5$i$&)h?Y`U9tTn)zqI^
z3`7`o9ecd9)KK<QLjLw#m!l+)f>Eg$p8Tw1Le@yTFv0yO>}i)0gTXdV*Czdd-QIkF
zTXm@${#eGH3PS_jeN&$6z~s0by0f-5n$7D&Wh|*S3Q6G~b})R*<B*=mbsmTZk&fUa
zaIkrRC3r&Xg^kHX6xLl5Jp@00#j5ae_5<*RVTHr^d|TAlYa<AB&#G@inLX~OQ*IFt
zFZg|1vVhZr4x2y10?21+|NbBzeZT8_{jd#i)bDd`O*oa{^b3>Mh`17zMbOo}fUy=V
z??;!hM}xx;Rr8Eyw*mPJJ$GFBlE06`++MoM<WT3NJNaj8Ty}ZKl@(j(c)0Ve|3De1
z;!PD!e$<CJ|H2S3b>ungg6fsEPe|eFNhi$4kJ=2qzJt{wh7wWNmd^5g4YG8QeluR?
zL!N%wT0-|~FvXL=D)pW-jDYg*hZv|(O|MyFTg@mW*?aiV@L1^?uN1dA$<<hsf(8KV
zKn&IF@Fc|R4ukDQuhuf;%DvF(cA`AnFhG&L=4aY2u2M`WquPR}X)<_y9e(i_n|y<Y
z1Fse6ixnt?OVq+YxxPJhwgO!_)-LtYrH_(vG)Vw<d2*%j$wl~5fIG0S9w%G7IwawF
z;ejFgb&101&4w7N8>T2=Ba?Zss%dlzHy?YkO~o>LsK))hhRn-o_`A;fTif&pH}=qZ
z0PlgOGJd>g|4q}zE@$h7#Av&)6sRpM^5hp8d)3s|wCpd|TQ?k|UPODg6Ac+Ol4Yo%
zk>Znf%U}1~saeRoDuYsG>}~SJ!(05)U<y+5cn2mTfUkAmgJ8T53?ve{H=2NZtjr>l
z7gf@CBpaLI)Jq0X3i>A#Ub1}-PSFe{AWzw3)Bf!w#S8E$A$eht;se09p`RopP8x6D
zyfDBk=ae&Oubp51J%U(#{ar&V#M6rK{T-*>%wYqe+d8`=pQN6CTTm5WSLbA7kdjtU
zV_Ykd(TOEkhoO!5?v4QIZt$M{V(!p~_>RC-Ip^IPL+uJi^!gI|hS<h-#f;~p2O)*0
zr&In}$bh3F!mDtVDJ|60P#uM$<PkzB3BCw!7m9RZiP+|US)fj7n9(d*vWZatYGZ$z
z^*aU2%rw?FzrpmRrhkGurEeOgMwyoO-{<*X)Id9$1{B;H5Ri6L!sD;lX@4?X7}sg@
z&7&}&Pxy()xzqVfK+X9TsUI~OZZz>Up<huj5g>1d;LkZvZ{aLK6Am2hm_7jViM`3E
z@dWbR*ne}K4<XV-loeZoDb#q^-x8;lm=+Ly!K^qsSp$j0-qQWN&O;q<I9{6gUZ;dz
z%_szAscXckkOar203h{89l=H!faUlTm;#;AtP!_20(@LWFh>UE#==RF2&<;oW%a)x
zp8RUX@;F$V*%!(Xj1m<S2~I>=$#hscfJ6@Qd`eo_^IwOA+E<G9Unut8s|lzTa_hue
zsa19+H1OP*_DhX0BAt@gkl=h)#lvzCklZ_$TQ&M)FDmu(lZzC&ek(Av;8H}lo(4(Q
zApM{`2w3LCDl?lRiEt=`zhu0YH_puGAVmv}QiY!<bJzR|5wPO)aM}5&L`lP84Lt6I
z@O4h?!1^lk>TDgqvRD-0p-0E8Rjm3bhapVIF#?ubaFw&gmKPX`Obcv%NT%)h*N#yx
z&L!!V%ahw`DfW8fhJD(8;5o7)%NE@VBIa`iAz$Fb;T4KQ)}>4hK%j&(u~@ttWRs7u
z_m#RIVh;pg7%}fvy<Jyd8RSjwmHfCNCjR3!4!xm#<;YYlta^n~xB+$A$cbv%rD&y)
z+XMPh9CaDmf<Z-u9p?of7#;HF0Pc}t@NvE178(i{0IL5mK3o7~{_AviXG-ho<<os^
z<8Fe(fum{AzxAuJ489Ud)z&Xpd!Dky*L+;SKsQ_Lgxt#~w|u0B3=+sQBwa#8Y#~pM
zl9*xEMv|<|EyC9Q+|a3C$w-Q2A4^YG@g-c1QdhAxbqMz&p>h_g+<-InD@@A>qG7i>
zAd^%-tV|Yj<tO(?B(=6mSJwW_$7ycjL%U5ylLPj60ez`8AKCKy#j7a5nlS|cl~BAP
zAyZaVA1$tf4YUbr#unS`7hc;U70ol$Peu1r{sHG|jC@Eaw)~VRICuH-U%%pkZtT-W
zqEw7z%03=;*mUdJE}dr)E|Sp>K72ks*UR)#9SSX~RGMYE6X3d)VB;8eyxY^jvM_M)
zV5fKJ*ipCD>ypx#Bwe-=<gLOT4Qk!^-Wy=GP3`~<(;Gaw#B8M6JIup1Tb6MnURUZM
zCE`4b^Edjw&$>{xiNk;!IuLk!x!BidX(5d@en0Zrp;qIG<$|&E!f(rH0@gLF%x(=%
z4X^@%x6z)aQWeV<`8u-azHJ|Onaf}AZrZhusv8c_<)wEkzjL)VzH3OeA-vX3IN$Q-
zp4{u9P&16T()agl%y(xc{ctQ3S$;{#(ZZiJ=ao0P%A4eSrhRZwrhrvpdK{g76L~ym
zL#GLlDUY38yq|XdP*YF9=YFjl`**=yRTFy2WIeW};`Z{zLfZWgK69-PCWqa`gu80_
zP@Sh!d-;D&GYzTgQ2?MPG=gHOxjo6sY5J6W)^7y1PDQ>vrI{h(C%v$obqYUk6s~J>
zb?F${9BmT)62Ufl*F;C98^vdfRCSBjXzVZA5#g%z9juhS0$s{MiOyX(Zf;ErS6?{n
z_H0gif_cPRW-bNVQPuRTSh|ERa8@cCNnQ=4wrTS()QriL<Y>>s6dD`fwL#5&OiUyh
zpbvm6ud1K%PIlZ|C4=@ZZk4I~gh+`(J}yooxJ$oKCWU1OG7P$LdH(L-<)e(pYh{6>
zv`N`lDAIHzVRz?!Nie~8*jMklcBP{fbCi-2^kkOHs|9S8OicR9#twMNmhWaE%}jcA
zd*uT2zvzF#vx61(eK^<gw0Q!KV5SL`mLszh{n$$+e><wcLR)O<ZDpio_gpUf6Au}C
zeG$?GyM&6A>A%)AC-Q>!!rC`u)hIcbzdDfsSF|dsbF={P<^FZ<CS|^^1?Y8%4NMh4
zWlZ4(yw6$FDA{hW6VAnocE^oz7cpPWvsn6j2cl6vnR;rI87)>@CH76DbHmYz^+`CJ
zLMGDbLsgn7t-^Xt5=mzT+)&vUHW}x|BRbFPe5hHur&n-P=z-|V(K-eHvQ(@Xbq_h7
z^h_r8JabZe=J8zCw5nzJG&b4FhyN-Mb@u(QDIA&K;^lDpRkx}`E}MVV{O%|(ib=af
z3KX6zG26^Aj^V%G%KIwNn9|U8wDnnKmFUs$s2D4{NNYlYXP~h{X6|IIkCaO9bC&k@
z-SLD*lJE#s^q_8nBYVrLbXxp=<+Rs!lt6riqo<Tmilh1j-R%Q>aYta{x>@RC#CCk0
zTr3S0$tReKVowB`#Q1c8JBNX$Y8_(7sa@Dbp((85&pb}a?is=Bp?Q%{D`f^Nz*j*(
z6NZ6s@jik$9ipy-a98m)WGl#$S&t0iMO>$MJ=Bu9J_=z2o1M+#?vb#<jui2ym`dRZ
zydA!S2PZ$W|0_QlEeXy1A?f*q@MC-FGlyHnqy9oAVQtLxmQ1P*5r?j;?P^(Nc^Lss
zNi+<4r^*+NCzlnT%;1^?Ra6fG;gF+-$3Ztkg|>2TDRWL|wS!hqquP|sI}cBwj}v_7
zJnMYt3hWRmSwMgaFc&w7h-L}}h#D@9qZ<UGLt&sNlPFF@dM5iO^gtOF1YTr{tupQo
z=MK;HIJ0z~pKv?j%b@YKx+E;IW#l$TsNA%>Tpm@9{`z9Wj?N*RNpe_+hCCj;ppoaq
zmw;>*Nl~7*BGZA9K{3MDiyY-zOVFb%ndn}t<OY8&u&bfXoFEbb{`$afmJWH+s+qZX
zra;PP^dKQ`VYN5mR;TU**XFu;IV7v%31+c>I1zI{lMi&3(1J&Z4qc9m3TlcQiT`l&
zm&|b7w!;)ZArsr%NKN3MKXxF|(U3`>R{T>mu~#rnr`OdrGjNcy3ruX`Bdv02L5-{6
zuo#Uh&`ZeP(cDo8%A7drmtt!#MJoT}^?cUM{zMQ-6<E$=Az{+#Z$?X8(Q08av+_hi
zxZ}>FrD@?19B7S+W2cO?1~NbT<EHd&stK7R^#h=KeM`<dMp0!d!Z)t#lrLWj1bE6g
z7Ef{Y^g}@hwMOOX%@YjhvS&{GN~-v~i$zL>bW~3&4ds4MJkHV_%;A7W9+WZBfIH#u
zKT%FTYbmV7koqO}5}cxMGk%m+@~0l@w^e;e{x53JLZYdQ>g`#s&%1s)_w-1%4Uh2$
z9cbstcfRMoY8%zWr1{ZzNsKdHU+XHDCoRYrrnV-RMiadnhT;;uwl#>>^AytOqbq7t
zQ!zq1zgoDv2Q_9JgbU0G1?AS08w{b%Q+$tkM4D1z*Z)f*B#`Q_jW%Vh98-M}kfUzB
ztk0G%%q-@1iD2Z)j?MRm$v#J>#>>jk&=%tfFhgOido7bGUMdYWAhcG%<6t}-_8nD9
z@jE<-uJe5_opvT{5O26dHrXu9xABxRdrfMGm_hZSVFN#csD35XInvF)hJDU=Subn4
zEgE%xe{jplGlpm6O+je;oWQswBblXHXq4GSY-B?}?<xrhKteHa`4nQ#=ZPUVb@6!&
z>Ou)<Qej?8`}@12YfvVwK{*8!c%s>@9dbdPnpFDo*!{zS%&QP@3Pf%3#5d)mme1SD
zWqGN|lC^CAO_s?!n|z#%T-&O{Ix@J1nQVt$iNRKCw1Jyw!F^s(fBjhk^Cj7JSWjq~
z;%SV8c4{m2R)m|=k;2DCnLPx-cA)NeZu0!z^@^vbo^2!zRi*YHXVk}99MKaWae${v
zGWyIPSQ|RyO}J*M!Bp-Xy>7fAHj<(;0S0HApgF(snR`F`$3G)Q$+k8MRA%DPo7geS
zDgO*PTXCpM9g4XKcG!GJ=@@&@gG3S1S8+q-RW?Mr!Qk?vu)1Z_I1F>7Dejwx8!eTx
z^;ocTM)UWNa{Y1jUhRIs9cie`zID=R5#7>CKvMQQfzPZ2igk2khWS!uHb-GuEW{Na
zFb%pG@KZbIFslEGY@<1u;Bpf<ZjX6Mc5Fd$y=rO#`srS}rYR>ZrQ|&ZO{rim&8LQl
zut1|c!bd;v==s4gAcveE<)tm4Tp)a?d#{b>82K}x>br>vIle(&FqC9k+@K$a*>M;Y
zxwP-Ek>Z%!9TEF0K$!0<WUbmJ5J~TDRa6?so|5sMvU9N@QDp$xJGQPHJyJcNs*uR}
zMPz%~vKW$E-8<!M16xKv&X6%QLsuV*op++Mte*$)4om6&jk|^YBlRIW9}|?2Qasxi
zULOEy4C2rfSF}x@%MOEV55KSbWlk2w#3h-Qb5ZNA1Qa#<)k(63tTfWg_QwsMz3`p6
z2R(wN*s?n(k5;??eII;g^HnzH>a)G_A&w?o5h(r9HjM+E(rx+b<tMZ+mn-q98?=2p
z_1BrvRLN{&@afx!Q{TJuynakUPJ`lt%ZVqNV*a{8l-ze`dF?~voHi3jS&p|;g*+8C
zGH$)Pgt?bi>^o|M9N6QH9Z?-)GoDnLPe{4-{I>2te$vX?C}BTlDFh}hhij-zbawrb
z>TGVy=c6Ca`;}ldF{8cKnfP|s_oMD?99!;VHJ5p=$plqzVW9m3-;|)i5}5+vrMQ2i
z3p7PbC>b~UXN!)*<XvwZS`0^lUtj8n02?tu|1Y`2AIw$XgnQgdzjL2>eZVb0Kw9#c
zR}Gv&4uCC5_AlyY^5O)|gY--%Rr~WQ-d-9-O5?l|!bO4kKQa*+2Yw6rk4$j<M>+8D
zqN$-MaOIGLYYB|@|In#Ed_IgiOl;$W6h2D89n$|<cEGss@sj?N;s2sd|KCn1G?<U}
z`1B1owj`)i4eS|WTW7{J{1+8FyLE%;uFVIHy$eO?VmOX}QA(+^fAov66Je0OuY&Jj
zqW!_Nix2$#_%M`u@5k8fSEz@!Sp*qI1-2N#zq6?3;3Q=L{bWcP7xprS#V`Fn@hwNh
zv5FzArCI4RQGH`Jx#j+Xu1lvIwZ`cAPc<s$Cpf&~to_1peI0}{@O5;Asrg>ORp`5F
z|EQd!>$?9DVTEkft>&W_>Lq%=SpwKv(7>U;j8X8}d6})<57E5LM+aw3$MW1mKc1WF
zyI>g~afM3-Rki8yO)DqL<{(3^3e(NQa0k<at)`C3S1T)JqGs?@buo2kU8>dUM8!80
zU8J{<P0@U~IQ9by{67sev9VaUfU$&1#i=dx<L`&06bx@7_PTwMbS!u7)sW4~Gn`yT
zkAl4cltf&5yQATxO*xT?=q4bx<q0Y07<>dfCp8u76=+@bH!K1iVvqq}F;KYsKY}!j
zCErfxdWvwMQ$fQrAcEcrW^&j5!9d1HF@Yr-+tWPKtWc)%R6C-RMzgy=zysV`<hBms
zZ5S~$<Q84Kt{pi$Lvh4x4k$ZD|G31;k55o8uygY7eQrhXb033ki!*6h0gfUGf|;C0
zw{q!<iuQlx=0`l?RkJRhqMP<n7K5h&*F|%{9>VZHpCDB8z6hGjiV(hg4ZgqlTGNSE
zDQ_YCjMg%(n1grA{b$Q-M@ik6j7Am?FJ)viSw%u?l0c#)svFIP<+hN>ki}oWHh=!m
zhekGN+kTq*kIr6^C#>$&uuAnVe46QVtsV#lI!*0J`1;4j^4YI5g(HVby)#?n6Eum-
nlG8(9LQ>eyer^UEz3Cy!q9(z9dkFvFAK{CX5~vbr9PmE?^*X~f

literal 0
HcmV?d00001

diff --git a/modules/web-console/frontend/public/images/cluster.png b/modules/web-console/frontend/public/images/cluster.png
index 2d8b8605b93ad01a00cdd0c2aecd1dc2381a3b19..727f0c1eea4af53aed85e0ff190fc81e137ae649 100644
GIT binary patch
literal 24083
zcmaHRWmH^2(<W{~1Hs)wkRTIW2M@sm1a}w`Jh;mMf#AV)aQDGAxMW~(cXxNolK0!)
z@9a6d`={^e+g0^E)lyx3Z-=QW%i&;=V<8|Q;Cz&qR!2ZU1|c9IF=8OUyeWU5OZQT_
zR8`cF`TO_p-QE4;({oQx&(YD*;^Ly8pWo8b()RZD?Ch+p67>G!u1q%R>gu8D>&LaV
zwNL8xm7h|?WUHQ^pH#FO&d%@0CXR~5eqCK%jgOBzJ3HTUeF#s=6&4om>D$@d+<dx!
z7*FkcxVfs;&c3<1ad7GN1^<bRiqzC^p3Z}XM~_M=)aV$s)H<r)9PPHYZ=Ri<?eFh9
zI5>Q@?`-tYUtB(e*01fuH<Oc-vvOxsGNyq*NwcpV_ZO$DJq<H+CthCO!^6Xkjg5~t
z4=W8bJzjP4vT}NQdVPa?W@cuCgM&UkKB1wZjV1+$^W#Baa7IRks<KjE(L59i?daU9
zQc0_^Q*Lc-J)Yg0oSf|J?99r_+U#B`w~*=#cd)Rq*xK6K9_iLMYt7Bgb$4_7`t@sf
zv|F=}alNaymX;=hMRiqG)mqzpi&e?_){3Ed+w$_Vv9a;~!OhXhUG?~0K|#UI;T0R8
zxU{tNa`WuzW}s9mp$%;H{ICFr!v({7+k?Ml_HM?-{aUJ-SYA0FiS63jz8Wt{nyt-^
zOPW|HAM0?e+~`<v_3qb98Ln2#7z%AIuUht~oVB&JU1}@i1b3|+-;5^o+|TBOH7)fX
zKf~6~gn*S9OhHf2N5IH_KC$vYKil#&IFEbdi@<(o4^NY~PfweZ!)c!3qxV+%levpe
zU^VIW$ETo@`PJ3cozcEKNf}s9Wb@6_>e>Z6JNxwXw74{gfi3^R-E*ZPq<ZziJ+Z&j
zx$5b#KG8iUhv(;Z?~1%z$6U!s_WQ7Q+wzE}?T5(d;<CkLV_D0oCmk6ViuiTk&|;uw
zn1n^qQ(ob1yO+PR)>Ciq*+OHecgA|9!@apxo2Qg^%uKso`JKAf_m=aeWc{Zg)uCYJ
z)ve2+f>wN+7T16zXiD&%?_ha+oxOzFqpdh^`LTvo@SQM-%Ms=p+o0=~f0wyp`m^nR
z=pn1TJ2I;~!^6lQq!Ad@u=6l__pSK#>({rMKbxRKs0vB%5fF6eKS~2N+~*G`s&T*)
z_z0a+>rA`;NGu3K^sawV1#a@&8&p!?8ABtT{x;2+3&ULc2#k0WQ~GGo`jKd&8ojpx
zKMktyVIU&Cjq4ORi);Tj0HS6S-VYMXo!tOHVsBxf%YV<i|NpaXuFed<Fq;c^6-NMr
zN|-q5wL{JNrs`xizCjD6z;4m<x+#Ho@|?k@{O3@quis8ya|@wR48{Oif_o-)D|hPx
z6BF5{iP<@tZ`&Ekv%!&Hw&G9lzGh)oVz$y4b+j5Mf=5Qlwpadern@}&O^~UT_}M7G
zihR3^O-Se0OSYo)E&TQu;92InAzf3<DW!UVt6WF+ZxK)49TkxuQ_>u~axUc(0nH#6
zx)kNMN7;s&Lz!DW^qi3efC1A#cFQE5yiZI|$xZ9nP@J!-A8(*aV2S%&-mHn6QFoN1
zVI6wlGF^{Tj8iea3)85#*HG5}=CGIN(V$;!pZm`OX0AZikn2O(S|O>8c`~Jv5=XI;
zSw+E>+2-)Xm@_Jfj`m!^PM8(u&ftue5ct=ai-`$pkH6K{WeZh(IuwCE`1UD%`)PRk
zZBSd^Lz!@kgR0i>3f<ziG~Ks>{X^8w<qr1eg-zyfC5e9`w&X|VaYZ}Mr`(9E-ee|>
zk!>T4EOI`JIZqpyFR2ml>tMDjg-v%feHZ<7btv-brtLwjc{HVz#BMZWjy=z7M4yc0
z6Ap3-O~!PKa&=E+_)&;_{L$yPLTpejbm4Y>p<T@IC8r)ODC_1i^%8VTJ$GJx84Jux
zl*=-5U53D|FT_PnQZ3&7hzL6CNg)+DxL)(hQDV3Rjc4c_ZxBx$1dRkPra78{9c@#H
z!<*EDJ+PHw0l>HXMt-lptri^&lnMj9m*li{L$;l<x62SwL+wPv^XG?w0isR}zo41D
z$wS;vVKb0DG^eMbg6FD;5v~K5KJh6-2pTBN<QH=)Kz=JV_x`GkObDDu^C6->_a4cF
zP#}3oH~_#i`;{iQdaLM4(bL-(umgKq!Ad{sQYHSJ>PRT~T@;My+kBZg4g@9FU4z0D
z_|~G<fugx#n=o%CFAK|Jaqu+u2Vk#)QToG3peK>gg|*J+Gt8EB<zOz?EdanuWd$)~
zO%fV=fR2*YTB@3jky(nuDO$n-lIL+B4ne5<5&Pfe8eVg5#HOEJ4Qky7aJ=4{(<&7+
z@e_#i+%PTqFdOiESVJhV!Oy&Z6=$;>L2aQ&k|SnMWFWj!BD|l;Amz6$%*K0O45fck
zgf!9&lBu=o#dHW$5Dg{Zyn=v@acg~Lji1<GjSO(z1y6stP&ywjOrD%7!2B}NzK8@G
zn}%OsJm2fs<#;M77hol94Oz<ijd(_*$*c>T!Qf<1I)cu97+9k3&hN=OV(0!E|5?6}
z<#R|Z8IuZe3nX=Xf_0#-kS${NN6I{|gbJ3O7}TxWFRghfVXdHdvTyl0^yU5e5r!(%
z&A!#);M%DDvUa|Yk_mM`dI3SZ*icI6kP)uGhV9iV{mj_i2>1vI^s7|*?s2djTYz-p
z#}o7~#J6j9KQ3T^U!csfegIrP(?ga9F2dlkP39wV5wV#9x;%pc#sfhQ<zkhe;MQwo
z2g8ns4fpNuMJt0rWp0a4hc{@-QQYWN)0fyIB4Ei(;3+ni2in1KHI8xlFhJnqU#%0s
zvPp&l&x!%z|F_8h`O?pIn5H(21$!0pg(!p4Zp>IBuOdH38OVXlUxfb?EPzFqB}EWH
z8-To_{uW&g^WEaFmQQBSv>c@J3a`kpB0v7r{S>Ugiy2J(nX%l(;q2l5z2QW-6)#rj
z2|B0S&GJo~Ya`7{n_GV-cLefvgh@00OF>x$!k!3zDK`8?&63{-iJFn3ACh%D)4db|
z#&FVtS0-@MzRr&G!+oYNai6}dMY|`Fp@-ac+y#IPlOjp#v-iQ7ic98j(rvh?lP$VU
z;2_LX#kR_(284alAQG3vgDDb}(Cc@k!-vxmAhiAIkr5#9CSd6z8-w_KMuC8|ATAR%
z&I1{5UK>LqidmGi05Q?><gI9_U~?%yk;O2I%l5=lr$7%M5&T=;6yT_R#?3sBFteCM
z{nV+8i7;dt9O?A>r~uJ6bhrZHd%e29YkH_R9*l(1I9b_~FU9+nk|C@07d{?GZFMaQ
zvq7G_gs%}e+c++L&}l9(V5`reCjHwPyYA0{G5&3P&-w=*qKp{d(!^Mh457q=v->v2
z<3MyTjK&=;kSbHOcO|#Y3!yw!NzErNaUl0j5oudA-1%2G0kV`2=2+V>K(k-0JTZpG
z1AY24e)){|RolJ9yy}(kvQPE{Nhtxo*}0WeAwHwlg(mVY*+=lO>Yh+h3)*Y^Zqm<F
z_C`az8@dy@Vqd0u6urIKrwTMh(P!(|&3Kb|F`M`b9~1~4-MC;j`xgR6Rld=@tO+tw
z!q2N_k1)?IM@bR>p+9U8RvAEIhuXVP4C7?nDZu$*{=(6Sl(xshf|sQUU!4YDabxdR
zFwEWFUOTK5;8ErNPGmtxyvtrz!%z38mYk`li@6=v$&<66EA5gvpNY0vE)zgrD{x!I
z4D!`0F@vj$aa^N>dR%rQ+3V9i^-QYryW|uO(Xc%wI1BXzuFR)*sVEd9PV9OrGNnyo
ze)nD}--wuQtG>7TE6aqOqaJPkYyveU@4rh;1ofVY)yC|ngd&9Otvco2KkEJxaU!(%
z15nY|b0Ex=SxZTL5-D<@_exayg#SKYm7G-RQ>wUfu-$9)>Irg+3^cN|fic{|dar9~
zF-BCG4?+AL#uZdEsqaO)#r?n3gGl{g{vYYs>Q60y;tKFHeGsL%6R4nyrs1?oLp@gz
z*70xmrkwQ7(SajScv#Au@<i^%fJ$c3>EgPEz@-C%#d|5Os<_7e7kXV#kFANOrwdzA
z2IO77LJEqfu}8Y*8xE(k>H?56Zs<{p&>;tEeNOKCJn2jw$LmRs%(X*d(X^JGr7uiy
z_D(B*+)0daP}6DitY3{pU?=Z6Hi<32-C~h|C?Ub)>VWeAAfEZXGvN?{65KzJJ_z(`
z@~L9!9;Tvql&7Hwuj*EVZ-UWW^{BDQzTYYZH5cQCebie|zbf0)={m;-6OMS+Pe4Jv
zF27(EAT18$3J`4Ni8|<-=s@%fp9o$9mKOXjQf2}0#*NhLo91Wt>yghLt=yqYw$~J6
zgu2poS1@wIoZ2$D17m>-W!FK|yG!=B#hwyBBULM?K!ppLgLC(Bf<EL7_qoW4pn&Y^
zl8<qrS1GawfHj?!uSBx(#aMh^K3e{{f7prvh+kVktzu@+eK(OMf14Me<i8L$57`=O
zh^s{j?O_E>Ybl-zYWr#yj{Of%pUY?1yuHmq3Ta$0%e1fAyM{OXho#fZ*z>H|AJ0!E
z0<SSk5r^tF4xJYOs*duIkdI4l(vtwegHpqkpQDV|zPVduTKsD%5llZ1U$=xgqxoDZ
zSoAu$_dP^n`&Fvg==5`SV0+bj)gnLLo+FTMCz-%qaYXr#U!C!xLSM}SLiU7S3di4>
z+=?V!|F3BH&}T<A@p7=^!fxq<oFh1&M9K60C<>;{=o}?O*~1PQp_K=5IimrEz}Wzd
zaWd=-V9bc|E_R?ZniL2>4?rpe{=bNx<wB7kE#Rx2_Cb{YS3}K*^W4_!#+3bMM&r9v
z*^4nuI{+5?MP^4Q0-HW9RR@;BT)Qr{Ml9uo#a9k2wW>k_Sns!Vf?5O10<g(y7C$xZ
zaCgr4P3IK0Z9LPIZdQj1czZlEtAm0VRY5^E<*UIF!k=4LdC)N@$>{XPkkxd4Cr9si
z&qyJ;$PQ=~L?j?2p>m#gTSimuN5tH?@}Vj`6X7}`6W=3%sNJ>}ce(qw7N^u41a1V_
zo<3zPEvd-n4n9#0AhHxBA}twq>>-*(LC{x2!Tx8g0)$O68PyKl>Ov9d>vQ8n2kRq4
z-P@<6-JHKu-f+A=Io09CU6~tnf6PMK?{@b_s<AI932ZH1{TbMZ-UI>SrJ%n*4i$Rk
za_0=J=d(DthIZC@PWL8>aVhqOqfDvdE~R|8W^m<IlemVV%(0vZM|kiQv2c!f_hg`k
zBgZC3jn9u@Q|U6<Yy7kX&~V1&6{8>9xN@37ZBvp(k>@JSs5Wieccl7=Y-V&U1xPiB
zYPV-d{`8V>S7c1FBOt$`2-1}AVCI324IgSDa^0iWeEIOfZnK)J!(D$Qa&V+%u<7ix
z>|zVcm!VN<U9!@@jrwKTM~gZt3yUd6Sfye?X>15z_<ysD(Rw8y&1K;a9e~Cb2Le@l
z;tvn^L<ZCgtXZc~zYo`V(2c7LQMoi}Gc-7XGDdO$F4~`H5cga+9%~IAbkQm(zP^&3
zMp+$%BbnqOHF|}DKg-c>g5R{pT>M6-+?L+Cip$WHu{ws1I81-lmDwd3?zHGdDdWo|
z{@6E?VK5D-f#yPb|73<-j@Ei|n2mNio~T*FrtCd1sNLw6OxM=7uNifJq0Ga@LenXX
zvPlpi)gtwE559>HT3=*9z4S54gW?>eCWdmQuP*hpi_X;r3^4yt|3z?-eSUB6%x4kM
z3JWPGYjF#))N?Ly*}pQBb+t6d$s6Cb3`V49@}AjfQme7`-|qV|>g(!|Cj3g^Rhx*|
zcX6q^R@3b9nKA97GG6IlkGjOU7qpvy&)!}GYHNtw>tWZ>Cw8~Y&)Ws5YSrs@NP|(|
zDaPz8ui3Ab{UDivHkZ{8Wbq-v9=Wa>9fo_@g;%;VaXS=WhHWsug!g+jEBk0atsJJ$
zSre)e^>u_frVU2K4KkL~a$AmBnU*pa)H5ckAGFie={f>?Q5SO?vx<zD-f*zcYp6oJ
ziO#Y)b8-Ux!k#HSPf|Pqyu~R6zp^gCxS~6YTdm&de0RT?c=rx*OXxqi*<rXp6C<%v
z2Fd&|ah8w>=E27M&F<HlA;rm`Xxq(zT!TJBMnqyOK8=3?qZ>~ERJq^JUKt2xU3=8q
ze6$s^megc7s4{*xdJym%3nyZ985i-*(AlFHtfoKO@}{rGHbM^+(ztt16%j2Xw>>Oe
zgVAF%Bloz1ZzPg9Qh$TOcl#t<-PLgQKI;BSuCqH@?`ej`?NH~CG_~208>uf}%$75e
zW;&6iHuwiLVCL0BA~?4Kw^YFKJxcAqC$)#@)F%^5ffV5Eq?0~KV<2VFm9>_6i*idy
zX-OBf9nQ>|o;Ylty0hyp3q-!hDnlMm2`NJJqR0cdq955N45N@f@(*3@TT;I(;t9^7
zJD~3f%;bvzUwrocPAv_6XiD}=8LYR$&hU9S&BqbjvLW1tp^xUVD6yMIABT&`U6g@#
z=12u}FK<l{I9};hh&9)>$goL&_&Tk^=<u5EH%1AMxeM)}RO;*RstS%7YIHQj7iCuJ
z%6O4w05AKrR#lky(yyjYr@(Fw4xMFS*vO%6%*ZU2vtd6m{tGd(LZ+012+1*gM>;ec
zKN39x|Id8f8Rh8Q3s31gf2AxkP}@7cAV82hx<2IJB6f6jy@H!o7le+^wPjkX$T|~`
zRy~d~MeZH0_9e|7R$yR!0s*P?LGqXxsPO!WkRE5*;DE<=WkU%&qh(!@pja2?ab`{;
z0Qe{j|LCc=JdK~`43|9vHAfd%OXu8ooON+1w9%A<`w3Jl{XrU*f}Hhm4?<N==|UVk
z+&?rb*!QTvUYO4GS5dRMh@eJb&yex=|9gB*kOAzjVDa5&Cl__sXG6%pIP@hYRR=Rt
zVRcH=qa$Kuc#M8k?^HG6NE9|H2RJ@ZzECJhC0!K+W><0!2_xVhqt^82lFQb`s%tib
zM;>f5JIR{|^Gg)7twKW$5UB5W+|1I2N^LVg0-BkdCetd&UQ<@t!iQYUT5-qGAS(@l
zA6zDPgca75|AI;4y9tf38;_;JM(n=LNpXA$0QcX#*S|$$Dx+2~<;st9W|97pF00K?
zaymm%9K$v2%DR|1@s@>z$Hh7$Y@O?cnuMwUj>7B{Rx7gcNyQ-?(*B8bN>+K=$r*jc
zdWLN4d)c6a0{k-z`8tajf-NG#RWfLN>v=D(uz50RaN+@^KzR3@q7onx#8)vKfGnhv
zb*5&au{rVz-z5bExCe1wH&3SZLLWfC5$g^VJ*u)PG~>9g@W~<7&aP!R(gxc)Gav1p
zMBX<|#<{o7dDR@?Nq#?yu2YA5@O^c3>e*1YrymY~o9&sHp*wXV0jj$N?&1K1MG8~>
zizT^sN7v6lZ-1EeUHeWtWh$Nbi5~N-Os4t2J<ixPHqMZqX&&0byaqE4vCyl@f<mOi
z<=!ac?qHnAfrkN=o4yy)PjnfA#{I2BLTTFK7_T?cziu7l@_1onR0ld33e>ot+FrJQ
zH=OCdG~RIxN6+ZbVsd4}2*%$3X}h6Jn0Aw!y`54DMf=Slm<l?rNYyI#w(mAN1Eb*V
z()~FE3Q;H*{?OFIA9*E5j5ja)jB{1nN#m^6pE!PD6}E)cB^lUkx-jsSXXs&V1VHju
z2+B~dqc?_kzP9#aLjn)ClgBI6!xc;tbkP;oi|n1257-4>XI%{K!R%18+tTU$@AZb#
znUQl_X^usl$o`5fw03+KJpa1>MEk*VQ#oc1yt8{E`@i5V@Aw1JM0-SXFFd_~pANO~
zv2+st18;ic+xQpKG|2_OE)%TEtHFLdi}#vu+$v#9Yx|fMiVlw*(FO`|QtF7@@}1ic
zG^gjH3rELG?+j<W(lu=eR2?Xpe6t{{9H)Nr<Kb^g_=d-JmVO;go?48iry|kjlhx!5
zg<qB15r=}b<K=D1qM;>Tz413XYgEAEkws+fyS2><QRzRrn8;A_`u=%ScA_`0)l{X~
zQGlu|S<m}`%j)wT`~7BN?hotnxi^p8ur{ZYd~qSK>A4L-g^-GQCi&Tr3(skz%xW5Q
z)lvJ*>Kn7FXWwlM_z)JO&AH+|5bX`P6!Z%ewLh1y&p_*8iR!kilOMC>ggo1j^UWeF
zGu<vLa#P{cRj$s!$X&peWW73PZUS=FjK7XjIXWVwfX^~kgUAB$wesHFOU5<`U~80w
z50!9MsrLr)k!WiL7J<S$apCdi<SB=bH~VCVTCANPWln8Mb@d>e(gl<A>*joG?^rR(
zhBr2Mpuu^YDNKu5pf^k5J~HuBj#-|VS0dsn`;Vm1A?eK9sb1(K7FR`{8`}fE&gq{O
zii|0Mz6J>sylfRqG;O?Y(`}qnQ!_ziKMoE4;dgG)ho*q-W)H2SvSIfdqI0pzNAK;%
zOw{kcK#8R0qHN=ul?c>EV!~m0&viI|$uoJsey5+&?S=}bDkHSSdAjJI!*oO!a66bi
zEQ5j5fXLeq`SHRysuv3Mx%a@`Z8B%-^to;k>wW(QRjGEwt_kKC)omtt`V_QDzx%C+
zWf<Z#6t%8NAZ<dlh{%%U0<}^Of-7dwP?rX(Eh1&9U;UgYd;$y{I<E^?z&S|R^jul&
zv9mBasHvg;%EaLHXKMu>g7S3DpGZ-Xd4s%c4lMUWC;l>NBUYUXT3|y>@J%)xB4$+}
zyG-rSEJwUZEtzQ@uqsJJY~`QgVYE3@oaVz}eQxMbi`6H`6iw^2GONq{nAt?)dIgKK
ze$MsASB`z1&pkaWOzg?wX|apI-(&t!l;X|o6m(ku!e|T31y?Y9#L2;1BWUCC_J62W
z_Wf%kZ~PfvQTKD$FCMS!T?dgvZwe)S6a!k2>(8F3Ds*mm>}LdYpIWz5#Q2G|?`3f-
z;uNoEC!wMnXz&sg&;_t7&F#A?qPkbhh5)|L9nXBi?3wFHx0JRJ7hgA!rWX8Lkm@)}
zBhHhLy{ZuOdH!dG%9w$<r>D@mxfsTmUVLkTIH?}TLlKYttLTcJsPWE?O2k8+aWNFQ
zKw<ZA%s;iK_OVfb^gh9l7qCxh!U44DgLLZxRYX{}$I6EoR01Kxrw*g&C$y@{{j1)&
z8vAqd-I<x(a8iQ83!s9Tu=?U_sedjZ3pn5}ZE;lg`!i&Zdg0z@QT287Ep>dj=7`ol
zKm6NG;piteaQ4|a0!J_MJIy6~8*tnAuV1ZG2Y=pD240Pa+z{Zk$bk~P9?c@U7s1$6
zq;cx2_`uYW10`VDq&Djk2p3U7_MQ842_Xo(MOG2L^@0)a$GLh~67wEtk%Zu={LmQ+
zij>CY@HFl*g`)QQHo??CP)5BEa}-GRoDHwsn$Q)EYil}2z1p6~FS}zNRmpTL4G(^o
z_a!r-b=!oLIhi<(o#pzEMWc`&ZAe>jrZ<Q6mE!g9XzZVH4~M4_qV$*Ju}XAqD|II$
z_Fj)6kfjbnzY!6J&lt-6C#~z@T*0B_N!JEgD0WeFC+I7&YC(p)$EY6C(kkR9^1-G|
z)arN<+Fi_qYqWH;&@2Ql+<X`B_|=r;@a(ombfZ5>b)L)~d@zhMa*bMxIGP@$5IOT)
z6D&k(1G^QqVyv|x)NIg?4fk`LtUo2<Ib_v0&Togl(j0TO(m2b>#$KQH(p`s!R*=mD
z@p3+x?-6{OHK(U!yyjqu;HuonMS=HFp4s%PbG8!~gBT)$a9;pWpmIS<bfBA+g-L2V
zKUj2S_bS;<%c9@+2PC7QcRfXpceZ%ksb<(Gay!R_2;_aU4CiT}RxmY2W*CzV4*~A$
z2V$a5fNsAWW@cCuseFm&X`Q?%1ydRQFOuhwK|E8$>(Hvf>J0+KGi)??J$`P{RPciV
zvxE3@)1w+(%%!$KV||<S+V^xRq7!f(I{NxG`f?|p)qp6MDQ1dd5IAVtMpxAeokro{
zmL=6JGfeQ_dpfRCW`AyA-LxW>?+#ezh_usS>)BlpQ%|+&O#*-4vmw_s6|6|t_$Inz
zJ>0*@(L)82QB)M1W;XE9>1Ix3A`~kZloazW$ZZt)zXu~*P>+~s8TjK|Na*KwGRQP)
znZNyTaEAWMCZ0>+juwEY$VqB@$Q7WnHy3E?6Q6vr^wOrdZgdfG`L_1Yh@Z*w8UlPa
z&@>Hn?ZcrCm&F9<g6!PfX8k9j1aiz@pq<bqd<S+pX>E?>D4Ea9Za)yeScbfG6jd3$
zx=@mXHvDiW6r;3v!O7CCpR<t%U_N?z*xe6@XZk$fBEB9my%!$M%@j;vkbK6eSLK37
zn3VM#N3Hn2;<pTWG(Yk7oNvrw@fG1iX>kGYpb5&qujQTXDk7Z_S5^!*F+n#Q6p+nF
zE7m~Fqn|wIEYc{Ekmle>2#g<<{;B?MufjA=DY*Hth9;LLf3XRf)3PXyE^`*V66>CM
zvpokD61s*(dmXUBTOS@&h$K8}I{fCIxpQzeEb}D$115fhlVjevNRKX6#F5-gwo`~O
zVFm3)5iWNn@=EQq!YT3GvJOT*&ppD_E%3QcX_?PqfkO0$09lr(3q{3T@+HS1z1bSe
zfrBCFX`*W#v&Q(E8PCE9u8z$PzB*(0`Qh(3L!|Ebgf%%)7}>-Tsxfe@Au+S&ji81q
zi{Y64ZJm;dR`?>*?)}cYIh%*hH7RueBR^V;3<ap!=RkDs1d!GgfyrbbKGr#z${a}A
z4P&`0idSmrqKsIK%}#O#ER1k0f`EI4{7%b2T)b45D)x_V&F>nK&p(Gf-r}dp2UjAD
zcz^pWcs8fu>rK9kUHF=tHciFWIdTO4_7b{)<u9k=-2HAL##59h1GIe*Di6^VhZl5L
zC!R4g@=?w7B_vKtu#10Rhc;<{72CEN2}wMr=UXAeP_g!?e~s+_WAt*fQ;Jh1-@h3S
z(2>il3E}tyJ;m^$z;;$O4{k>Ky~xsY(>Hmcj*H>ah;+fkL_`mX4XSp%o}u%xtW$?V
zJ4Z{m=6H4FdTu>tL}S;^qZ`=asb3I4Ir+H>%m$y<^c8L+p3^ZAHsE9NR)(HqparZ4
z8}yhJir}&#=-w&nh2k_p@u`Q}kF$+(IFz}6atWVr$j9{;hy#Q1C+GxRwdsR@z{}_V
z1@=O3(>4ZWKoWF91|&Z4g-{8zR&TOIy1gN+$ld-95ln?f<+toA)(KetBvlH!TmeOA
ze$T94s%P>-)lLQ}Q8m)y(I14Smov~y5DQtO=#*rFsQR1FyPo=X9D;ZF`T75N_uWv`
z3C9v5Vs8@O>rftB0Sh9dx17FN^Q!f*ozu>kQxVi1VGkg!c+h^zIE#!r5+1#}&EBj)
zV7oY6eWf3R9_u1KGEG20`^tMKfZe5!Um@z2_nTVhf9b|X_w-&BpHRm18i72R|8j1N
z%$?qmkyl5B_FIz}8kO*s*k$s5qauiIz;*X2p8w<P;p(B0395+>_L1>nS7d47Iw+IO
zoXU#|#TUnD;f%p%XddI+jDCd6k*U4eXxIBA(XzkhmY#9#85O<kOT0IR)~)Mu-z!kB
za9qHqUZ)w?;yi3ZU)6^cFy%P}0Jum8^_cN6Ok_hE-DsAFXe^{PBOW~MSJ3Ebd<POm
zg{k_6ftM=??p$S>Oz?_CA#{E9x@if}9;=}t)m&^mlZT}bRjf<2`1lVN@w@SAJY~MF
z52R#t&0v+78RffQzP=RM6pD}-F^^ZbmxRsa4~HDB55z;9m?y`!HRbEb#oh}0C!H&z
ze*Wm)x|-)U=DlxXe9)}RR9RMa$+%-dbCNvPA2W4~<jiePPxQboU7+>3s`-j;25Glb
zOeQREX?W^gQ}Q-ImOrT7`|lGTsVadOTL+_q4ZEDPJ*VfhGe>jyTHCG*wW0pS=nDW$
zkoIAFI-nT|`Q>VemYI<``v9QyAhZg8Y;^McECPA*M2hBGLZpFQdJTeL9-|8=gPr$#
z{WN|i62U&x_s4pWQ>Nfr1U--!c>%7AdhRE05z^_BUm)eALu4bbI##-*3F%A=A|^9n
zrjCJJkk9s&s~^qX=mEhjIu&<8=K#hSmAC1s*n=V*;7O57Aqnxm-)otlBGtTGwdR2P
z*NAM;nERmklNNY+ArO@n|1%y3djA8dSU23~dR#Y^icL2Lq%}ie0zq6_>7Zgt{cCGP
zLrC6Z%vAs!2$gj^4XYF5-KT|c?*Q#5#&ELXe(1=oK_!%beeS7xm|Uj+rU9SO%v7RH
z_P;RnCuvNjy&#Nir(CA^<O}GJc>4NU)<=bUOgY)2q=1rSHlgzuw24y$O^RZ19+qZP
zUYWf0=9vb>DKKbk^;B-iD$wrX>B+*nKrQBB9|vPL_sbRb1p%ZNWHcyZP5UCno#d6B
zc@E>D?m4--ADV{c(DLsQ`1^~_kNQPNU#i@#fl@#?jBy@`h*&tC;M=xP)|_38@)RlH
z{d@q=?1mO6)gEY1XVvi}RcHjdh?w^+5mbsiWXf<#d+Oz;d8RVoYKevFQNK9zm;P|<
zX@`rZy4^?R#NczK2eXgXACUzi0a9xW>K+;$I}nldEsO|j-K0PYyPK7!dfB6Yw59J}
zMDJynvO_X+3rIA^eC32bsEAouSq=y`e|$OS(YH6_W1~T=Pw>$Ey(yF5<q1crgnZwV
zlpMRJjZq=_P8Sm|O((jNV~|sY$haQnoUJ{ScTp2FAajmgPMG^2hx`WshJfI}%W}aS
z&Ms>0nZmN0cHYa1YB4xZae^rqo41z0_ehdvWuv6=&&N>_fx*sWR0AA6@gzQc3C?^}
zFr+)hcY3pv*R<(^<nI_I=H*rRUCO$ulfRS1eU?YrSuEbvb`@&Ee0Sl&S6vnMG9Uw>
zmmaK4kp#{chvxc=qG{GaqCd2q6=b_bUU?KShX0P&yJg3244f#e49EPP;A~3gcHgFU
zYh$Sf1Q8&263|xv_FDHa{)0Kbjxx$aL7{}^e@gnA%$TVWB~Lx3rt<>oxehETg7}s8
zUO-dKOGqVB;8l%|=j2$*69<&Tkzy3uFvci<8!nGBndLYo8k^HCWZ+x*FinKqL9KJ0
zi0SXIVt$FpmtRix^R5Af`1`Qfw#5D@rnhQclTnW73q{yC9Nb;(WVE7JuX}&jxIcWo
zsbf~!?7<g<k>zsre)~cq?ii~5{c=;&+ny0G*agxAHbN*CkSz?@GcJ<>Fu22mC@?BB
zeeIpMkRIC{-;9Gu;Lv=5RYL=p1qga^(VCsHLu0EC%(G`Fylmn!CqXh4oq$E&_O?$#
zsNt6qrwPLmWeO7{zi{bdZ+5V>5iiH1h%3h!8l+0+R!b0JRuE$__=YHg({hRMP^pJR
zLC$Q$!T!`I#-kmNCzUcmti?T(Y29&kb$<%{5?&`n2L2vI1C1SSY#x#iW;{H{6zF=u
z7({up-ezCvKHh9f-8pB=i&31&zqfR}pBGY-?-RZ%BiB}H@V6TS0S2QS((M-Cl0^Ay
zi)BPqo>I@a3yB_gsjDx}bks=PT;;>xXM(Z=1gd6B*yrQXO{&12O`*hG`j24G+Eelj
z1DgBO=xmoRDd`8&Yci!2)@+~cQ~Y({kZO$3>eV3IeQCTJ?+}3U0T!E*G9Yj`2p+}p
zJVs{F{e4Qvvp*(j?LA$+H~}rgr>4&YL0>W+4a_6Jk01DWrV>EK{{#(-^_!GL5cf=u
zk_Y~I$};+PEh?-+*4v8r=xZb#cj<uIT%>0gP@phh=TmTszYQEx!%qQ4GJN*v>wwz~
zfzvn1q+caZh)7+U<M?};MeD`@9T*M)sr&zcf`4$~sYeCM8+HU-o_p}|rcuuv^S^jW
z2XW*i8AA#Q!#hu@?*mP(xr%Sl<@jLVDW>sgN<P5WUV1n_o7Uy@L2=)2QUVS)j6$!L
zdlN`VB>gPzvQg~l5u<w>p@y$RlXs?Z;$ayO;!y?uPgGk6x>pI}tzVrwWM<0HCuD+F
z3QZhBquse|nlLy_St!+~L|%P_Oc^Q+VB<+j8EmC|rOvB?dry|j47iiOq}8q7Jz&Rh
z58WPBfxMp{eB<;7zB7j93Plz@D0q@!TMztlBzcXiQ`M*+r>jzFL)LL6e)4S0^z~e1
zJRWBDzoM;1M|c<{y)i4tEI-$VRy)XFogbFyT~`+&yPz(9Q4$f)?JT$gHX?<>7q-{3
zDe=}ie>{7&ZtJ<Ze}IH>P-=;GABob#YoV}Ee4{Sw+BqVQ?{;f3tXnB&rGrclJX6})
zF{NMsm;|C{cUFjC&{5m6wu+?pO4Y8V4M6#9ZIyzL0oB1udDmBW!7o-?bhikOnWf&k
zL&<b|v;{UR`6E1-AGYVeO!7A<Uz81rU@%a(yKv%xPVuR|ciRW8+xaSY@;+hHV*RMa
zL2Ovq4J~*?{6|I7y7j-P;G1O*ik(K%t^;dd2!w{XD|$XkiNtTviQHCIu-5j8uN5VS
z&gR6@J+(=|8L7U0K4J&Z_avQ?+w#eTr|SvyynL1gCFvw3px=pmi1NB0b663ifYg%$
z(Ti0ed$pl3XvjqV2G9oO=YWW{1Oe7B?cw;tA+SSI^7pg(Rn=bf*#f=ZMfyo2&4sap
zv(trI-eWqz-px%ouOp-TzlmAILv4=AuyGX+DsYdn18Z7E5=3K(0hHV>m)miQ4vq-0
zA?HlBeWK<X_GzrsJ+Z4m^Gx0vXCiYUn=jNsBE6_UX)vj-Sx+t!i)rHVsd~!|%PF>m
zAD;X?`Av*YruoTPssQW8Kc(~`2&9RJqkaq6ywGIcZiO#p9pSElCZjlwL^tC;O+-^g
zycDdROU=^{JLQivRiez;`lnVj0E((Edzne+L=FA~IHp!gQX;JW^p<<LuER>}vXuLI
z+7n&_?$dumpBvyPc=Q)#)f$N)LZ>PYJjpFl0r-E@`+1CCvDX&eCMuMizWLG*gqrb1
z+JtgfBVZBFM`y2xy$!jT8uf@kSQJOk(GXZQiS#`NJ$cV4IRo_@2?D5dzudDu+SsR)
z^U_$0tim5NjXz)<9rq3YObmnFZ*Cwg{^#a?E3b+P4N`YNNwr21OYZpn-Bi2H7aH}r
z-d{gb=P6&2yT?e#VOzS~Cd2BXQp5{_LGcxz;y{t*{|;1UVn!NSjc<}C#Z?9^7w~Xt
zevQ@t4T#q?r8YOUsrz^jQ`boX6^~3Owm3EVQ!gSq>>xT6(a2-!bb?V<VILuyss=kc
zZ&y=kvQ8rByuanXst(h*#fmp|S`M=oTvdR+PX4$=^?Ol$WUdu8sPI;JyB=wEjBv?e
zh%c|~Wn>dykAwc!xQSJM&2uF6v!pD~VG8{b(Urx7Y}l{Q>aNzH`A8re`>QAz<rgI*
zhjcyhCtAAqZO%!=tb6L7EM7+zD%bTtEN&`iC!Q(V%1>IEp?i!e8IyBl{Wnr6LMN@b
zC!Ie)+ddHQl{AMs2x1Y9ic7&EQ?E1n934sJSw`*avb)cB*t$~AZd=0yt`lUw0{*}7
z=iey`g8P0nfZX=pN%bd+gr*9c3$-9NJdnvw6_50^@p1m?nN>}gK*bNn;ZwVD3d7zE
zYSlw|tkdSj8B)|Ptn5IaNLf-9zxa|oVrNe9i2ZD{DuK&v)p&q^YB5HQ8KBJ!#19=Y
z4d^~*W_c?zHTUj!CI!yyDu^4yq5w+a&;l8|0dQk*FN{CJ;KPsXnrygqam@a^*y!cn
zvJiB^AH(3;SHs{*Rv(&)Ii<#VxM|ARdmZsl+hF~M;6ucjiu{2Go!@xa=e}9on?V9o
za~a<E-Z-mEa{C|+ySMCK`Q6iSk>9N|@YexaKX<XkcvtZ2?)r>C7m>erB(PPR?>v9(
zvlkH8wFjvbrpA8lzp=@fCr%z=Uo#YkV$h7*4`+yUN8Ete-CS@lpuw!j>qGan`k<cg
zJ-8ALKi62nGUa%tv;AF{6j2*_5UQe&*KXVev+?MiMSKo%kYT3|(hYvfeqIlskPOlb
z+9NgVs-i0(>?V9nWsjeF`G-sEItY7#%SilFC?La@khn}NbkCRj{$23&URCJAW5ki!
z8X6W`da#ImfynU{PijkfW#yl6O`0C>4$>#0_!GltvJeaXEzqK)@McH11F>Hb)lS(a
z1@{GD;Nd)Q3Y~j}6Yc(8z<*h@$<KgcU>6QDF0c`w(>9OLa_aivUp$#+LB1_UuMs!?
zfn)s$9JnTPCe<Mh?hZ_wfhU%W4J<yvl1v`9oBiXH&#Lcf?JRJ8w+Uy(PDcngEIM~A
z#r5pA*FZ@9;I$9Zpck`K+wJ-H_x0|JeUO_$=gn85MC+RGAIK)=*1Mbn;gZyO00%JS
z$%W>3Ts$?_87mCliZqt;;vZW^5ZG51KraoNob+|S+|DG%aujQJM7#im$YhYsz#o^+
zM?*wH^CK{>0Ic;Zyy8w$&Qd?_$hu2LBCde()a;u;o59dsA+PG{depLJy8o#5@jsB^
zW!p<p_vp%^bcmwWN@3|m%d_;LjzzvohH$g*!;;a}*q#CtL_#^p2t*7GkbNQ4zFrr*
z{}VkR{K>{>9x<E!oj_qddL;NY>2;@*%@eE*HV41Xe_s%kSLv)}hww~S0gS;i_HG%`
zPel&{bK8p_Byd|y0`_gNCgv=XAsjdt*HE?hD5t|^d09M2f$Cl*0K;E-1=ouMptSrp
z+R^<jR!0eqPs7dn8HN=BCWZf8`k7$x_7OJgYNp#Gk&EffSnS{Wp;*$sTS6f>IZoG;
zLgobBZazQhj}HO=+a?#vPvhL8su2dYE&BU4SCd>~phxyJ8pL4rwO=Dymo+&BP+;+f
z%B3Uw8@vf6+`aQ1Nx+{<c`>qxzr1+rMJyFJ(2j_X4R0u>e98vp{BC@9g9{^SjI_-7
z0*j|O<Iq<GbTi7Bzn`*I!>Cz8-e0~?YHszmAcZXX_OS`GktaQsO>C-F8cmb=*f9zn
z@cOUT(_8xIZT2@~Q9jH%ZP;3UBzn_4qV>U*XjufW=9$a^0<jhK_C=BIR#}#&9$h&j
z=prbO!oc|%I>^jL_xA%YzLj1vR+i3T{1qS8QG+slM%a6Tmmg?uPq<uRXM~=kO^d;c
z!PmKGZ+L3b`anM*XCH%Gy5Qsb#lI8Qr%GLz6~ykh4kyzJlS=Z>Xa|<z?J`y+V{iec
z!L$kpFQM7Y)YCK%y)~y{8KV2M{9mdGA^Hz2zZiXqH{VJ))FAyV2(c^<9_BRS`v?Dp
z9Z4xTlw}!Z{c12YZE`A15|`4_dr79{RiR7v3j?0Gv*86e5o)6XNPb3GG<mm7?Pvc0
z6H9rUe(c`iwdChYZ#xa52@!cRQ@F*yIev5e^oUMfCK?Iz?XPxVESn<81`_1Z7!@-D
z6Zy{UGk_@)uW{cR?J1r#0NyDve=IUU$h(jpfZ-CEk$94f%TSP$rOn-LAA^J!d4<B0
zUgIc%yjWK|)|=Fd!Vg&SA%@?A{GB0thtrau9rH&TsXmKSysA044j<#8hv$xQX?2H=
zW(~|IZj)pIV^e%h_uo$nW!(|kvRQ0mGJAi^X&6)S6!%Zo+{0fbh`+pn&T0M!HW=4-
z-@<?JcKZm#)+h_w{rFJ{1pZZ7B|0(ZPT8D{K6jA2TuS_)PI>Tf1WzDWZz3o);Ggk`
zdGq4AS4+(vk2ph&&6yMjX$^3KJ~}#wLYInwoQO=pXh_wGLE_hUrK7g+dQGOUp&^XC
zmwfjhPvm4itOfjJbh*6i``;8eC=nI=Kyz@rN@yS7hm65*#dE29W%GxBy$r<xBeoAr
zG(oz>8if8!>f{j~`w``kx!<Km#(_A#edm$K>op^bndmwL5d#wsPD5a>4376O_#*ZN
zpwhS#E>F$a(Z(@2vP033`Ccv-e;Z`jnFWY@G9CTc@*8g0t{X*9_Y$oFfgt`Jpua-j
z-92q+NHvj+WHQ79_0<U|b>tCKU=Kus7{0~wy6X|f_UI;_D`Z`}nguaHVNcAHAJBhP
zv=nlpHmu(UZBWTQDdK6f%f!YA-s;c|?E4C<L?nSxUwPnPy1)HTlia_iIfc-dEdWW~
z)XL)J;&}mLIFkvkZy(hIYCUIM_dw06_%`udJV}rJy00=6FS|wB!TW*srtodhpKktJ
zvhKFz9bn1Bf>_HFEcvMsO+^k^aSA8^L>p(K7l)eu^}B{9%?u4psJ-~Q+$)0jtBCmA
z^&y4(p_(94tPebpOLN9Z?jx!4(GjOSQWmJy(XIA#3z$4YjCgSc{Ra3_w*Zfri<ZB=
zV_mxS;iWXpFU)HUnU;MzT-(^OX2<JtuE&2Na6c6FaS2Jdh9;IP$0VoNA^w&^&?o!N
z%&5!tL^qAEr3ZciH2C)-d`QGT2k=kK;r16~4B6or^p2MJ$uayth)5SA1D&q<77Y+G
zC{3)@CgRj&oct!HC{r(QEgv~*=N}Ah>YzOvI)}A;vO!<`lkx0AG~Y(_lg>w2Tq7$U
zlIVj;1@trx$6l11NBn}z?i9*VDf+-IZl*l~CSrYpYrR}NzGhf2#*~8J&9&Uns3?oI
z`=N)Fd*IX6+UG*rynCIuEyY_N?yu0PSH)X>eMqn$&S@>OzW#vh&EfH+$ahoEAG1=7
z<w9D&+?WBMB7}u|BS18QTsC^*2cX-`wexf+jS2a6R!gaL7pF+_g+mI&))hG5>HvT&
zBd;o}2qF|he!t|@-$kgul-;LX9iroS#VlzS6jn2wY>7~w19*d!>8nWuR_*hl9pXr<
z%FDh&8wlx-I3WhUA=<fCjUp>ADxBZu@bEL%2D}9E<Q^mT%Mda4M#`P9ugUyD_5mtT
zJnWWp+Q=umvfCGce%TM_g#Q8gkm13(vz7(nMIdtH-clBVX!;LJcs9JU<6j|l!Drb9
zoAQef(YmU>vO&`16_=;;!Cp5O@pq%K4BSxptvzrB-70=qLlY05ifZb^cRT+f*&J@F
z+ZcC3%Y|>cWZ>Zx<Zb}lbc=7x6FBlIG9DuPAl53{@=1m4RU*1heKU=6JLKr7m?Lvn
zF)Uk1(mG3&zo2y9$?P}Y``Cc=n1yfnrV5MJ7d6=bE8CnydLY~Qdj=(IcNXR9$|~AZ
zyn-CC$I)VJen&qAA*1vB?=;i)0&3H-C|k6GXHW>xx{>PM@^#IIcBe`_+PG)!l`V_S
zq@=5I$>7IedJ}{3p&UNco0NQbrfCqW9Mx`j3ya4o3$2?Cp73?hkuH}Yd3HIgetozW
zm;@<xWR&!~BxOcyQPGjP*no>3^akoiQw&jS%BVU_P&xr+eDJJ3m?_p-iuRio{tMw7
z5{T+Jndd30pY)~6ucqjfq-d{yITv$fahE^Q+j0a-5an~pskRTOzD}Z(R*M+fO~*_b
zj1W(G;@Q9p_v?Db{!4Vyw9|(<p4dGth7M<2YdSu*g3>H|G|Z4da(DCKJ#)bZQshzI
z8Gatb$8*gAI;qgrLn}|TbuHEex8E^j!_5f?k?yq&Bq^=brzgdw6Ckd>gWy{mzW0=(
zupM7Y@RHa=CB{NSmV*Pt>&w(0fG!p#F|aSkr*FrXg=+z{$Y2Tn<&(&P@YY9c!$~~K
z(f1+ZuL11uD99<S*J1D=();*F*WD2yYPns!W!6djasfoyc1!Dh-2ZnR^ccd;&OOcI
zKoexZItJ>>*h8COQs&_1RpfuA>y6L;!(&9D(-6X|Y=lWBSjXgS(X1AZ$AiBu$c7m1
z!&h(Vd_mzkO$344>QhXtgKs58Kuji;aBMnX@P6vVIxg9VTUrFT&O7Zx0C?=J;pB;o
zb|>9^1R9PQ23$vH_5hGe8~9mIenj~Me43u{qf=}{uv9R|_=D9Ma3BUh6RLhHwvnC=
zczz;-o{9;-O)r4EkJSEth;RZv*aW>73c8sS4A@i*$CFG{xQ%>6jx#>Fyc{&8lIlR&
zTxpN-niyd4Q>>GF`L9dsfQNnAxLkf=8rLG|_|qV_{4G!Q?E59QkkrD!82LViFtmXn
z^`^xN1je@4H~xq<!;;Lb8GJYxhFxokpNqwE#V~YAra|*^VEv=rMy%Q=Hdbdy6?s+s
z2$@gdMk~Pg+=J@k;!-VTkNNQW#W5vd7mdC0cD;mEqUm|EIUA4l`3{PF=)v0CL2crJ
zonM2SA7U^+>D`|LVjuIJ*+V_;HRgYS^px}OBkpO<iwC@8CZf;~*RJ!fMD;tqwu#vs
zwa=a0wt1ZknX^|K)Q>7&&D*q*I?msZQg9jNJ}{%*ymcs_uq^mgIX4&bM|pu)-wDky
zmgF8K$b^kcP|*;)4g0CZ7S$GGVQ=WcsKWnFs4i$p44GJdBxnG;stH8_oDQ|2v<j1<
z<37TPZQztJ!R5(B6k$k5TU;jFqeQ&a7wgFS<s<8Z->hLQHc%`F9?;hNVu!T5cYJr)
zU%9+jjoY9dOu~dx*K+tiiHc9xhDLC&{W`Y5b(Z5jzyNIp!PpfugR~iYRA$RkMWy4B
zT-C|-F<5>C!@2omY%vr2P0BB*><M`cBc_^@mK-i@FaS&GUDE<p@FR`>%6AF6CbU}V
z0xrThyjYGTCDs|xBzpEI)NQa8DUPJmcGdd*Lqj<uh`-P8=)|vOYu8IU38$*bH>9WT
zjt(%KQeOJ!9aUBG;EU*eUg@9r-eMobkv!Dy_WyAD#T!WtrsPfum9>w?rjPA0_ZB8{
zt$>D?J!^uVQ=kqZ^$CHbE2#VHpJFxe2RQn^>s)=zAz(-}3maG@&$JbMa{#pg>ag<A
zY}V!i<I_8THt5l9mDOGK{d6I{1r2+9>Ffajm{Che&{jAv3asUjLcM2GW)FZsFPrYH
zMwXdwlVF70ct$<iL4QN$U<!$SVNj5`n*~>ng$5)OH1a1!Z1XrskDH;|Q)Bh%6Ax?`
z{0hrY?B*!Ud6|-|+%%h$xxmSEpRY%L9Xo*4z9FxrB|ik#q$7h}-ZZbV%~}_emBl*N
zlY=9K`M1ol5xSV*?ZcYkxd6Vxi#N>@TNx{~9cZV^3s2+lO<PK@Z0iw~dkNYJB?Uze
zL3AI>P{zSHih!-q-9|=)VlI}h^&AY8g8*Lpi?zh=?HkZNHIDlJ2%pd{lm`O$ubX8i
z>T8w7uW?iz|0Js_eKPs7c3fzry&!wOALIvx7phi3lIA6oSF$KcR+Ym_h<Bk@uweg)
zagbg$ul(VCfLq(J^Ph)^IkAn&D){aVC=G)+;)ftWwuaen1ihQ6$Dy^lz1>hkXxUrt
zXF7w}x1X{(=X-+#iOvI|R^|nrP)1h0_vsq5=!4ADO3i1$?TisaiLD1HCM};6p%82w
zsZ+4Xo<@2kWgjeZEtW0#**oUyEA3XmAX}U?7vZ1$rQf;&^%@QXg`Dn~;SwhAi0x38
z&fmEvTmu)m@?18+0vyD`S*GG~2(KGGdHo4mr_pO~aP}9_*R1j_#Rks(?Y!{|Q8VZT
zB{XB&#gTnL_!5WsA0@mN@>#uviri>|a>G*ZXj2AK|H)avnQ5|;E=|P45$K(dPu#F>
zi=6OED+<A$H$Z{Zq69po<kv4Qqz6nfdrAzaxFYtnSDXAWPWx>LN?K|!boysa_l_2q
zbi?xuxLS)KcdzKLg&5>3cF^@KhM|Ifbqsd+pvFeSn4rzS+l!tp$bcT7j+5F_`-=9`
zGk1bmJHe^oA0^Z$KfgX7{ACac#xYo*izdj)b3_W__}uT&BLb?Ez5IFsq-ErDIRZbv
z#?XH{iIWtv*^V`}#IW|^(}M5#&XE7uEv0!0{l@%t#nv=E$~{qkMc(xUFWZNw(b-(-
zm?3vW%M5>Y<p1#P3|QcHk=A?-Jam}beJ52vf_ZK&<!l+stcK#An_EQIr47M!Q<U{J
zDo9<wAX9Qf;SdY$&sdqP!!u1v4b3#I_oj}C(uKWf@Tuh;!DHiw@vrYAR``<;2Yj4(
zZ)vk{+w${cGJRrHkUAQ7-&qzPZFbWAM;e8p-ZV~9`v8FJ6n_Kse=Ml*!55x4WGEa>
zUjv1QUy{}Sq@Ueow+!K$0Z~q~oPcA}_vX>7&AOR?eYP5De+#A>gyGxf$U}yw^}4Xv
zeEvrd*BRDiv$R7qp@!ZB5{eiQ3j~o+0-^UNRZ*G}+KW;JAqa*lEp(AUM2a*Kq=^X9
z2?~T}AOebX=>(*Te(}8Ld_SMvduH$5nQNcfGTVD}O<)Qr<ZySA_7QxkIRt5tH%9KR
z5$ZfycYHzfun$c!w2E4qCaHkL2y!^5R{4|-Z>?s$g8$sEJmXeffMF=)jysR@5Ktzg
z&XwQb=2$<ekMlig=-eU(_x=xy{Vpng#&W6gN;kdbfn{j)LXu;CQ!|n7*n8n9ex`%v
zUnJ+4PdIBdf?g5{s#P`m${%|Jx{9Rc3vbl5f4v<k2K}J-JQyfBcahd!zvmK`F-LPd
zrppC#_AmV7G=Ho;4Za#rM1<YF|3^ms`BKssQIq!jgww$7`(Y>FJIrUTJAhm+j&Vu%
zM*8Bx5>uk*?y+iL<P3uRfLA2>O#1Nu^~^gDYN)`{f&JXY%^9Es6!|u6g5InLgEMi(
zu<=4JAme==WWK-1)}T3#>E<f@2>S9RE8{c1b%2<s7kgsz_5(xAP4E%By3p_6NkWil
zPp=uYIM{4qgbv_>)(6z=f<!j4E6-RCC}n+hH7B$K|Drbd!usntK=cMgY(@a!^M{F}
z5);gQuXZ;!nM-yWU1SFpwz_Ds6Z#lUq=1_M)5mln#cH#(SplVJIZQ^Qx872_RIC%Q
zLABU4WDC8rnz+1Ejyxb{anIyq#zXq{D=iH*XS>&ViRR*4PWEk?<=)x;{H8~1QL4G`
zU|Qwjy^nz0QxcH`F<2B&Q%89XSRor^TS}^5X4^^<$kDPFP<^^uC-kTWc~`3g!kvYL
zCq0{HIO@V5C9@a!T*>9uE)Y6+1Nq6Cnw)!HJhh`$nh*g|07tyNSNLav#*$fsu3{KH
z&Lu7&TFy)DcraQt;dHonmRL2t{yS+Y$w2(=SIq8(cOn^DAgbh2H;3M_kBdv-f&4Ew
z%?Q>Q*tQ&+q20u`KQ|nzex`$mEfmny_w+L0Q|O{;v2ay>%h_2(<r8rF?ljzE8&$6&
zL6WfoZSUlbN-Bh5uOO=T9n7!!{^1~g@{?KE<_d5H!ZCVfHzoU3vf<RCCAQ!0?L6s4
zSq~kz4SXsPQ~aJo!GPawwX#S95noeeyt{t1k@)Mjz3;oD8lnWPrGH?EA~#iA-wzsQ
zIYn0S$MJ~2V_t+%ttAmj?t9Vr2D#|sjI2b7)gVr$2+?9$BvHNSa&-rAWF_9B-!fQ7
z)BnslG}$kbV-0Erx=}NF@p4}Yr-{k1{y>7L$xor9a9ecxywoMdc&{P%U~G<F>XOe#
zQPV^_E4%3P)I5XZqxF|@^(2q!#^x{^KggYjm*h)&fFld>?|*mhej{3eMo_oop4IJa
zZbqn7VQE7<GIPpzE5%UU<KD~Er#v)6!XLI;rtnZ%rLUsf)>qm`kjDD@W6ACI4+Gl`
z{PkX6n;S`SyhDB1{^2E=azL9r!f~$uAE}!hLkDM9X6`MB)oz|78H=l|z<)oGx7kPO
zEib)qTm4D737HGUIEiPz1z~#2tHJaGhn16Pr|`jm(hTIh^%#(jyRhO2N^P|>!)Xmg
zP3~L^Tb{}w^Y!g)oiflf8!?Kc!`xBgpKXSRhDXh2Q8rM9#B#d2hDx7Y@w`cL5-`zh
z{&%2Zp7O-G!R+ykk<s%vvr8$)E1xEfg_gzkDzs<a9T7PN=JfH?{twbICk;8N%M;nO
zC4?NC@q6>(1#J65j8PtjQz5LRfOnM1RUdd_GWqeQsXE@w_01BH=^_&J22oZ&0Ti#Z
zTQw5D1q}|YAstiOY6gDDxj40%kG+42ZL#pI1HTx!T`n0D7&ST4flSP~3nsuPEq_I%
zI;MC(6R4BcDj7tAa$Bv;S$@#h7GW{cF-OzWDeyUF<ALyq{nOX8^{phSTvF7QqS?r0
z<AFmcp|w1R@8<^GfwSX>+-H347#iP;=?!qV*Fj|~4kjKQri??;!ZWGT;i=a|zoKu^
z_vi9M?sUvkJgyTkGm3imzMIA!IOJ&hm-Ya6q~~kTAjVcA>QxkDPoVEfN$h{oQq6*<
z1-oy84~lvU<)?w)G46o|L-mKyvxO}I?Ea<wryijmYzL?>Wb{qekp^X)NQFN@hv;T|
zwC_mQE=pD7;3(mt1<N>ur4J7NYEwT^+wCoQv>I8b#~|GgT;R{>zavI8v4LP}bR-uC
zJ%sG1ULnCz+COv%mXP8=wD<%R{bf3pKdv{hOAnH?m-~Q)rSgFnNfK(!pWpsPn$z@g
z5%9%u6p2)J(h=9|7(z%G;pPo?yZ_>&o^B{76Cl7LI@aXg>NgJ+Nfv>1Y}qCdA{_7?
zBMrX$l0T8ZU9E}VG$yhq_1-1-EW7;vGG>f6aHN4=`Rcppq&>H{iPhB&&7_6GWYk&*
zJ|3&u-|c#4$U~(~C6j?OUN%QNR%t#L_UG2R`^px!TR`b{LlH@_68b9jNiD^%8@Uag
z{00e+?8+%4wH;p?EFBx3j|F4h_9_Fi_+wAxm=9ymxTHl{SqJWeG`*g@w;IyT*v_Hz
z^J5ap;y8l7`RSH_Y|_G_)z_&=bkL}9Y-Mpa)WIBoe&DH;A1ih(^ws|7BO}&2jI<tW
zq;fr*R=B|#aaCZ9>xLST?!Iu*2e1NAuCWK`X~RVK{nJz{M5f5qcdu>t2j4PbYdT`9
zyX>iu-L|Z>Xo}&$JCU&(3R+Ko-G9LOcxzU1ka8@*5aJxbbUssHrDMKkDpPx<h0}MB
zzUdy0X9WLqZsRr{F|}5k)9{+Nq@<vt(}&>VA^-!ocIMxO3h*uVT*2jPm4MpwP%d=$
zt9OvzvA>Z-J$py*IPmBj7OOmlKSwV=mOcICbb2@wmlJj`X|G`N#-HOWKh8C^KvL)c
zjW@?6e0*Jg-614%%>Le6yE*m^M<XJI-s8+NzwFfZ0>7d&G=%?Wf8@}Hlg1#WqVqnX
zkaN%h%#Sx9vHQL%`HoM_<sM<AYUeFFT6fN@7@140GU%7UkwkI_ZDS*xlU)`xm|JHj
z#Ha1Q-+%JqRx`B&ltO8VxJTXqh-^A(`Ufk64+MC4PCF_14JT1i=;diUj_RDct+vyB
z{Y9MyJmTACtEFn*lf2(T74_rSZ6-J1nPgx7E=^cKfcysC$LrT7Z1Xi;Bvo=G`+Rk-
z9qN0Sj(h_~!Sg&4!J`!{R-c3(_Ln{N8u}V>l0gc)hZn|3yOMP;d{i>_Z(J6P9Avlh
z@3b5e1KS0-H>GZ-(YoL9APdi|+>t!h^v%{x=jE8=5q^(CsY`px9Y1QNZ{yNTQWpM^
z2~IzK`#AUzda)AfHAMV7$aIA8-y^D9znHQtbJk>~|7kR2dn>|IL-`=|0Icem6Y1TU
zaARoc5%X8V8NCA{>)z~lzrLSe#APABITm$320mNe*w0Z8Q;KsFV|-CRZf7I72~Uwu
z`g7zUPzGK0L3f@cb)|qunPiEGsa)DNMa>xc<*Y1dt8P<In{Zb^$)v{D$DZ;N<r6H*
z$hods?a8efLW`^Q38Byuzu}1o?bTq{x;1Fzfa<g`6D|H0*+5*E(XCnHouCDO(%42J
z8z#>dfPy^WA9~OheCW>#{LYKJF23hf&tF->0uF9fu#5)NE!@YrS|tKHZxfC}hg6bF
zPvcEl?OFT1t7x~!52~l`7eB6d0@Ql)Yfi4#y^YyAneoKG_(qGBNO;_G<|9Jw5_WIO
z<+=^7krm2w5aVY`4|2nh(Tf3cDns#oUnX1$CHzWKiw`BekwnR<?qg=Y;z4>)=&}%v
z(@l)DPwpLLPI=Mc=^IbuStLl6-k^*2_XuET&CUCS;S6TP#nQH@+8;3M9iZNsfe|xk
zHS{g500?7=OwAhBqV)Azd7Md$<D;qMdJRw<;@Jr_t^98d477%7NqGpm0(UPvMJW%A
zOaph~Uba64Yc+r+V&bbQlrZ~u`yPh+>Wud>ZvR5YfpirJH=jqG6&RQm(O7@`T(3K@
zH*j40N(+bf2DsBDWeExD7ji?!zrd{1LY`7=q?>DJNs-uN{nsx4YegL4HUKRA5+@NO
zmy9g%a{_KQ{~Xh{cxCcG4Mb{wo=k5ztr1Qco%_u*jk(6{lj#u;K@nX4D(S#*_U_S)
zd5ie~B{-&?G=kPp4{K{Z{Y5cfksiAA@i~Ufb}2@%gtTs6@Jo(<j$~diIdW0)b|!7j
zCM4Wd1AYVMQ!l}t%kk!BFUDOfBSpVdE!){{j@*3S|6=a|MI084#o!_=_P)ZqT^^_k
z8;SR>z*Db9kID5Jc?>uUzYN)7i>#@W1WblT{Z}MXD^ec9I3YT3yc+rRyK#GwOV8oc
z`ewbKX<Fo*rI%+7#eMc2${D5Lk$+&fU)bo3(r4kC%*V6q9u+WfHa-g3u?Ezff-j}(
z2m>7Gn)BnHZWo<P=VwnGodTvU%OxE_YbXbgX{uZMPF_Q{WC*xhEhRa;3gL%6qeoVt
zN=zdkz_yj&9&W+hUFhEg^7upngXz?M(S=x^%+dRmjv?RC`&WWbZ<d^+_oHd00J-?n
zY3^G?k+0^6=OXLtPe5S3!2VpbuvoPa#8(3rmpanIi&@1j{+jp9wcK#yh2>rY`D{(6
z=Wj1CfcoCcR6d2HrJNfwvo#_31Zx$r=T+~a<4Oe2;7!fvTDy%G=?{M_fBBO)r*sKd
zP-RU|WFfK_WKH@CRyJ$aj%Ub&CGgdE==sGLO<T1Tc&0=wg;me8?g9D(D-<4Q-Nof0
zat#eFzDm}VxLB$jLn$^jOIR2}RUYc@y1v6WjvUiv5fbP*6Bb+Z1x?JnxD>+gz&u+0
z!7;NEQS^@m=Y<RU7+X-ZoD7dGgA131SaBecch|r@R4QE(V@r|JaDJv4!V*Jx6yo*L
z{4(z2bB&7?n8r7&PG5VxG#ek*72Yvo3v)v7(3Oz1bP4AzG_wqJ1x*s#><Q>=e)u{Y
z^R^YsuSFuGH84?H+@=^S4~bqf5#@xdYp9tfgVW)dV%*J3eQRf|Kt-6?&sr{UUY@}R
z-wznv$NYx4JL6ww!xzssv)J-LadDB1j8rQ|;!DFJ6B)Du06#z($^gDy!6z5%`rJqj
zkPhkE4m(}YP&O@y>Aa1@rQt)Lgr>>m-aaxsHs+hPIaWqXL&^GIizqLW4rp`cA#~YD
zoVH4=PfwP_B_YIL7mw+#Sq?3j?iwA!&`SzXU9+Wu^;hXqh&o=wA^Z~cqXKM70;H<i
zuKQ>Dt&K^EYhE9MHH`TURiFzx%btN3|E%wS=6biI1fnh^|6@gV{|&PxIh#_1_gmN|
z)}vR@${_D#Ajd2QH<b)XOXtZg!Y4dZ*7uJ(gK1=fQLtCF{bQ{z@!MV&Dsx;9!mT^Q
zr1D~l@G_zFX3IX^XKjxM=*^`bjH9Is$>_WUIKsOIhTRu>$$*jWrH}la2cU1Zh}w6M
zl9wo*a}~c&{f2fQ>&Np2YZttQ^@JA~==M99%SBq`t{Re1lc5HQv-C5S7DiGoi5Yyb
zCLdiK(0ox&Mrzco)`JftpIDQwon>H1e5n^MLo+0bpUk5=BovzJZ_xE3a1TWk4M4_+
z9J0g44~J2J+y}~MZU|`FZ@_i@BQ5IbUn|JVwCegd@)@`D!p?;2BZ?-@e#Xmulec4(
z!6oZz84clc8AGi@YO%85tP}L*`+mH(9-q=_G=`1jZYgKgIm!+L7oJEP<l56Rq+54V
zi`J5VuRa+^PQ92odsWUz0}?%5yNoq?^4wxx8%(#*p=p5ybb1g@FBivuNp$~N+_|qk
z9<rY)vn9J8qo*img#IhAgk=MQ56WjyN#KgO`kMeDAjee@v})fW%wL)}IC+*Ut~Hh#
zZ+-+_NVy7Od|Pj>!QQn3_^kcqQsm@j;^`|^=&i5lk|Yx7g8<LI@alo9*}u@}g@IR-
zq78VL_Z#%YU)|A_L3G6zPLi;RyR>2QM#g_ZcOO1Sc)tc{kw)GCM)NVf9X<b*yc79F
z^goqN0RR3Vo33i8&(B=(WqZ8XFXz3_xJALQKrZ>lX#OjY0WY)(%r`?T%ln+4Rvn?}
z`UJzT1|n#Gv0fh(ovD26-};}67#GU@Xv`6h_myL>%*DVMu)+oMA0;+Z)nmEa+3I?E
z@RftG1%Ht~!LX|hiyqGeB;|Y*ma*&CJwI>+pWSSB%s~63WtbnJhAZM`$tp>2G~#)c
z=BB5)%<Vhn^_<vmrJR8<wv?MLh2wtfAp$5Hr+;d7$6xMUef5x75_aI6d5V&vL<Y>}
zzi`UFpVVQeom12U?B7~aa@a-jwsbAku#d5$%1*fI2c6xgQU0-hY44w}h+DH*5@a7W
zI<<e;$X`okwG`|n<leTHDM|EtEd3FbO%A%H^+3|2smj!{C7JV})HiW7+Fl7X#~rO>
z&~4{*DD_+9(UXE*4*k0g>hc!?N|%qW1W3aNmwcA`;rY93C)EnY+c+YY$D#xoPr3)Q
zzw!F0*BYawD9S7*#dVqN*oU<!n7L`HvA(blya$0_Q%F5pM`aSAH=-}UNYEZw{N!V2
zYpj&)d1#V#4E6e&Tl)PXVI)vx5v34z+eTKXX$QIhubTT^Xe3^R8F&wS<m>3?*>;8B
zPx8^>`YhEn_0Bf!VzTT{)cAXN3kf}V>S|HdoC3`C);whEj?m%kov!?KBGhiSJ^ezB
zCvMLH{?4a}O-u`UoksWB1NVcYssx}5GC&mExx(=sJ`_>2FRv`PNx2lRR}aE!b&F#f
z{jM%-<4o1Bkt?krI)kcpC*eyE#TLFbcvc$kaOR_D=HU~&&_5Lhq~NZSkod7sewBt6
zE8Dq-kn>15zdVYm_D9HL;7+N?5IjImex|8~g=sv2q?LGvdZcE}nU}%_PKO}>X-)@5
z8JvH9USbmLXY1jod`vSysMwvm%4$wRVmi~gIRb-cSeRkY&BB%55N(3E@%`{Ql%99w
zg;1OZCV1FqMBBZGPmnW*xC>RSebB0+7Vq+y7Y?w0Ss1CZLs{{K?Xa1QrOlz9kh$Nc
z1Ha?@_B{gSkCM8?@*djyd`B}eX|4AizXI$)Q*GZnnKLHvg6XSnRl~sC6(LK_weM7$
zHYV?uLC0|ODEdY9AUi5F{Fa<Zv&uE<KR;LRakYV3G4lC*mF`L>enAc)=MT?46^PJe
zOb`M4-~G&RE)+M~sJoUK;xv&!;$X|6HXCuX7^DN~aMf?CkwnKyyuR||71}FM3w2>r
z=Ih0EN=y=ysYfnT=ufvN|59<)0ax^6yET?CU@SEA59LH%oe%<Abp2GQ^e?yS1r&@V
zeD<O5LQPykzWT2!BHUF&;>x<KL}Js_UkcO}uFOT0R~JpRnvk1lWTE*+jcDOp^({*c
zG_NLD{~;l~*Cq9?y86v-hfSI+S-<FVJkX)ltH)o$pg06qu=2#1g$i}L;P>1%ihi{l
z&BK3xIv}(Z0On31)<9S?S^bnrb6p*p5h3&}Wjq9SpR@S|Ww67b>p@~tB5BSV)-Zv&
z`vu}Pz6aaU+=*uShivCBv`my+qRe-p)!9pDTT<^8A43_MGkoV!hg$aD03FAwY{l2#
z)REqEsN(?7gB|EP6XxIIz=EJu(z6g;kkfY+vFMX3W&rInyh>jUS0QxWHP#m<_17Wm
zq{Z$i?>wp2=w09zi$1Cn`U?t5DdcJ$1%hTQn4XM<l1=|A--q6&yY4mfE=VS48`?->
z1h`6ytzRQ6aw8=JwfTOit%(t??m%DvDp!XV<(oCf09~B*;Q(4f#jX$}LCtL?mVRb2
zs33^&mr<HqGq-`v#5Oc>xmxePpyX(hQswS&*=|*3cqu1%4uP;vweL`^Ypy9<t&7T~
za{nz#L!XK>UAWsfU|r@UB8xoxEX6wnC%Jmca6aP`)n`ADPgO%|eym62`LU0O3b&fU
z{$#DLOoG};PLj7%mG*rxF=vfZSyIvz-3PDyBrBOmvyD+sR$)r}fyKu?&g1@2l2kak
z!ai<RO|3nmo&=w{X7`yg97wwl4$X<$4b@8mh5azFARa9zkF+7K5hB;-n!l<pz;8FY
z>2Ed1g-br?^y2L5)|Y()+1NEXh&qU;WlVvjpdDDEf&|<L#`rd(Oy0HO+Eb1#fzZZM
zfu=S<xFr8b8EW8l+LLwt6SXnq?q3mQi)c2m#9%xp?;GzQF}cZIlcx7YV>aNjQ+_*S
z%4t29E4ypAf$qNcvcp7~I{2|&pLhq>KtX~L;jhOyfUdzPd26&M23b&2q%Q4KlZkLI
zPAnGaSuAl<xLf5tUI0Y({B58oemxJq%N(aA$ngf;i9QJnySD8rYTY<*a&Jrf3uxzz
zynP#9kJcF#(mTE5gYiolLvZue58F_q@$>!(fd!#fzC<{C%!=CXJCzlIfa9n4DyT^h
zUk_NHu!{ro(Kj*-V}boW^KRFlq7NY6gdb|V?`fS&2!9m(F@*DL*j;W|H-mFr-0u3|
z6nK6f^0h50J3!4KBIKr+Rtgn5{yzYESB|<)QRTVvO+Ac1D*L(`(f;^Q420;%T5M(Y
zKj#ETiryZC&;PtNNjp4B(CYKG2ot-L>d8Vt3}UyxUdYfC7XzqH7of@iD3)9Ex7Z%<
z@kja{RMX9Fp7%g>W35vcpK$tT0t^2iN=r06$8oZL0tECPelmZ}4w(N|Skit>nNJ<|
zN<(k%Qa*VDs#_V66o!jTAvo5V;~P7FmU11~IF2IsqN=X4Z?oTBE8T!Uaw5ND%Y01k
zr;=v+y*dQW3d$uxZAIC#BPrx^W{rxV8b&zX`%v`RA(6Oj`q0v!PE?<9qqs8+E0t$i
zfrA!G8DgtV7~5)TV<wYNxA$)@5?V5Bd-wQ8qkO_$#0&bx;;wp;KY?~MPJ|t|Znvcj
z7iPoN3?BKed$SS@-$4G^6P&4*>4#TBVC4A(MV$G$`SyOgDfEt%LV|s5FGSF@&#aS5
zM<x<2;%~XI3$ANNULnt*?PzI1Fz_59;!B}g%Qvzf>+r_2bT!u7XTmXsI&6S+n@Zg2
zk2+1KLfjQ^hH2+;eI%>2@W(DRpB~+#i}pj2ioWTjP5R^J5ks*%;49Oe==62{m!@ut
zU<KV!{wqd2GUGmkb9Jm~hEIRWUK8AakyfViAi1-4y8&_z_1$4O!sr$Xzkupha9_Oq
za*>kCk=N@X)_t^v?f@KD^b(!lOfLTsuE%5>$?Z+>ED<fftX#lu5z$3FKSDkrRJhE+
z$xpG#X)z}8a61T$MD(41*6RNR*nGNbQyoJpo=i>okG^s>V@1V{rh{a{_a?K?YeA~D
zMwG&sgup7n`ii^TqDNIb3C=TD*09(fL3*-`cfx5K7h#hMou6uYnE$2u7+TA~a4}rE
zKO%5aQI9q;FP=@_@vix~->_6IGLPcGR~N;Q7AF644=EguQ>CuwDC)C?$}~sp9~Suv
z&qt|^eMt1X2&QJ74x75Y$Dwa*4#Ck4bu*!SVeT^N=o=i%xoMRy$CHzR=cxnCygKzU
v3DKyBbT`erE|9fhB6NneZ3?0b6tF{D<cJG!fkb=k9bl+yrc<ru{P_O>2?<jG

literal 29376
zcmZVkWmFtZ*9HnBK|&xva2*Ii0)xA2Ah-l~8yteW27)_bU~qQ`4DPOjySuwfu*3bl
z>s#Mi=ltv5RlV!lRokn>loX^e(1_9C;NUQ%ze}jV!6Aa+;1H-#5#J~^tKxZYFSAN=
zs^4B;UvF-1Mn^|iR#x5~+1c4|-bk+}ulKv=e-BSyU)Nq=cXoDP|IIC)zt#&B{5^cV
zzkWR-qg7SaoW6gZ-G6O9dlmauqbxu5N4CjHqP<r<_4Q!6yx^k${#A^>_Vr?8{r>fC
za=b{N=k;{zq`l8?>M@f&VZd7SGBGJzFtOL_@9XQ<{{5?uk58GkK<o7V%fVlec-QMi
zL*nw2f*4d!cKCHCd*%7!b!`e{Y%0Xgp{7uKP;IKg##Lj;zWn=}YU%Y|za0;koTH}3
z<Nbho-uC&+aRxZ$y9SI$e}!AWdcdQ3ufFeTF_J?Vx_fsT6qEdV(H$UF|FYAk=s%)p
zvdqb?f7A50$e{Xuq5izZZSdl?h&OIDqIn~wj)Tup$!G6&*w>@}qCk<!j*lH`9`SEJ
zIm$pIZu<5Nwo@>9St{0+%vEC3dw23hQ!Z+u%2|2KIDSekCU@(()Nd?DrM|#m_>%pL
zrd?uF&E?9~)~IC4L1Eogg5lN3ZE^Y3;OX(@P+pr#lB!-{L(Ar7Sx<%dpTnzlE}LFW
z{SseMjv~3Fx`BJTZ^PHETE=N}SI~mt^{4gC1zDxKA5Kkq#@fwBDj=o0^ZhhafvV9!
zRkyC|O7O0hh8?@q@5Msf$9(w`o7mlKg$UjH+aGj=4(Xk14=bsPTYlUu-}C>vH{Rc$
z9#w4>IB&3iE)PtoRTJZ!g<WeUowoU@rKVNq$qz2oMx3Y=bb@Vb&1z1UZhA`-Wf{uj
zn@(QOk1D6v5~NuA7S{8n@=o|YfAyZ2Cmj7u9cVT+ofs;JbJ0I>8#FL$3@D%BR&)%|
z7MOQsxmw&%3aniUN$;)Mugu8E>saU&2z+qk<xovHO<v6>j~H6*+V_Q&Sh?ObH<*Ss
zY|b>9tA^Du23r0Q&oqkgkh5rNX_{!*DGs*-V+Oiw`da-;yHu45u4upYHNrrnnCa<K
zr`@=NgZr5;Eg`DvzIePOZ;c~L+J8Zm8iqV;tZ`I~0{HBHabOB!4HwnIq6YsF!3CC!
zf-u$2u`sO|)o?L|Hl(vX>F~5Vf8{goW=(!p=b+;t6s@EP%F*;jc`@fY=q_OuMbx{y
z`-<M2>VjEoV3yfMxTQ@7HyzwZG{FNm9o<KW;=gF@LJDUkEe?X?R1X<~+kJb;2podb
zRU-1=%VL8gP(@zntt~xn!}2mAg?mxe^0pEQ-HAE$m`P#7Rz>!mR81!I))xgOY$1O3
z{P&glKp`vx(M-MNU{7u7*H8y@o$o<R49TyS!RKtY$1=)41Xad##XW?ycey4-=<D^1
zQ4&F2%FntC1`B%PxW5~qKaMvhU-liW?DwlfeOpCQa37Lt?xi{Z3=`+{t0WoKOCsO)
zbvcJilPD40zyC&=!+hBI%tkFC)i|{IRe5rDY;e-tmf<1tq%Re5OR~)akBsi3JXUSQ
zRqB&L;b&XeeI2&=7p}HCw@`8?`uNH>CZT0zBPw`$*sXgo)w&f6FVECZ*6DJ*30kuB
zYXXI$|N3tgSS2rtLscH|yayNGjX0u2;joqH%S`!tGtk7;pd<#w3=Bz}g>P<ZLWWKw
zV62aI7EudFNFes5umVeH+AN=Zv+VOYu3JWF-0gtyT#fYNPkq%BNd|gEKotQV+V4Hx
zMW1iSUv(fr8|7hNVA_zj3T8@UT6+-cL}4_J6GN^Y-r`?EwW$GriXwDxs{{54?b}FY
zvp$gN=k4Ra6=Sow3ME6UczGjN3^0Pc5BWvM@|(^^6MDm-b&U(vg~muBFl2C`h{>|>
z2MHd24zi*>$PQFZUHpw#$fA#k>VDo8Tkxfy{~?$nTgB~tsp<mwPg8UDNa^vUwo6Za
zm5Qn~FB5MVyW~0KzRFtnFgmn#1Wz!r4L{ZAZtHrzr-9O>S=Q}x#AZ|V2Myly?01t*
zo!q8`pV1^_PrIQLsSy{lQW9f*0Yw9M+!)_Hod(dKI@=f}6z6izeguElRBJQZY3WbC
zcMjxN%<sLN8zqa0WmgH&3KaUA#4R-~2N@3EZ%)E?C~Sc5AlTH(Rx-Wc{UBmH4;d31
z`F`(9ByuvJ68zHvK^3&&7l8w(MAX5Y%?39n3W8%QLho>T)8Y^Gi07nlTKsDWP5>nl
zzTr)&{~NBAOoSJ)t>Qz5s}1^psr%c9NQv<7AL$Y8-zNTltl|GZ3|Rbo<Z0fmMl3Zm
zF0uqJusp~i4OS-VF!-Q&^p627ca;9Pc~)HN)O#t^CYU@c1z-yvf_s_WyyFNs(eRk~
z-dpAWV`)=J+d+%X=wpQl2u^J=eHODcs|cz<AI;@lAe-38n9dW_h@NF7HFJ|(VskCH
zGgE<?R1QpI9nod6OL^9jbIPy2ZC9<!o;XB(&<L4n2vDQx;4SS|{4IQ#7MJ6=3w3?Q
zOJ}J_)zGpWQvuC6b)NT_&92~~qigkY-YIj&sL%44SuVG8SeM1A1+y1PJ|64N5-G&=
zP31Cz_SpCb5Qrjvqa5v(H9DLQ4gn*o;XZJ@BTO<GH<yjnQ3pj)LJyTx%*`rb_kup!
zels~qr8a7P9P+YtiN|Q*S>I3RrXaGL;k844!Y421HlF25wK+h^(02TbsRBsjATa0r
zcUBoNb`WXiIPo~g!EB^Fz^e+1b=8bj+JW$DplhGD2V(T`+#`BuIX3x`DFm$d@t%|)
z8!sq)G-HTJeGWM7oQYuP6|&TJ)X#1HmtUJ{Su`ARPEr)0cesMdsXsH===Coox9RFM
zL@Y5;ZgL7r5=l)vtZQb^`2pPp$$a)D!7K8fq+56QYcfpWeLK8jrwfGkx2CeAw}^z|
z&pzF{EH&1oS_$7uH)>yHRfnhg<KyF~j<N{i094|g&y;k;5M&1ZZTOrpFm(!6qKdkW
zYmqWUk48!>SR3rBZ5b!^4HvSa9q($Kv|yC^66kxz&F>}G02Tk><s;1bXtq1zr87Er
z=GA_;vNk2h{_BA($}A;Yr+fL%9TolBdHSU4!$>Bf%yAdZKC*NF*{{lg1km!y196up
zc@C=neJ+SvmLnBzIHLI*wDTfjb$u<^-+z@=M_w=Fz;MUYH|Mz>$gVMT!!*XkAvQ09
z8RYPgGE!c035=2i%s?rEg|FDId{Rd6$D`YmLfB2Ly9{6%`uLauaIGp=sax|^aolbZ
zpbo)RLLtA6qx|;U3e66P`-<_?JH>Z;_CZSoLa$edMW7yNw`{lsxcmWVu%+1nvWWIc
zHn^9DeUi^bqG+XQubhHI`psa`=l`<X5gWP6YAa8q2n&n^EpK&s3N0~hZ?dRhk<L)N
z3KVSSeY5b5&=XeE>I<QSq1m_0L-N0F(pYfJrN>DwgDt%=HTzmE{7(%B5_`KlFjh<&
z{l61eURLRzbeLu)zTb`P-E&*nU$@OIC|j+YI)XoyrE)jkPz{{6co-4A+B?Yv<3%-h
z4QUpT374PT``lgXN*X;buB41C%~3qtSf=XeFimzJ&lw-!X}Vrxyd6NEdi-mEC8Y7a
zIgJOwwxiCW_K`I)#zEqdnv%*wv7MzpJg=*i)T{k8l#DpIySNCHlEV(&cFmRoPTk1<
zJ~cdkOxp8webg!Gffgskf`k}w8qCMepvra)87pL`y=*HKe`z8}3W)P0#{^0KVU<u(
zNs=mmorHeA;s@n{LfGJ4ZR+fCGT6n)*&;}UqcZh1k#zew0=Ly4nzsM?U}5cF7^Fls
zYMMDg(Th#tYW_@wI_Rb&Im>&}Rn>>=g~n;RG8<e%`g++6IEEhQQDDKiV=WskPIdvZ
z%=~9&h~#z?lLwf0X--^vsm|SJdn+w<24xinz>#oaMcL*LXwb>iA@WS{jp=UAg}aRV
zV?ZfE2X*eJLgL_PnKbr<n!VFMM3@z%t+u@#+t~yK`w8SEGc3JbGRC0bmNWJdqN*}M
zV<K?%Up9B{5)#O<Da194)6)OqR2ieW=E{|3lvU+N88=mAx`{nkNo#F;#l6}-11V7e
zPqrM)P}zvnWZ_tbw2Vg!>@(pKsn-|6f!;5GtIK7Go&}Z$2Ik-L)zOS;26ulCzVQz8
z(R7DKMiFvmj*7**2pR|3Rom>RU+K?e%m}v{(EosQYM;OMWnVa0^nFH|;#7t;908J_
zo8Ym&2Of-_&+jIXWcpdm1pNerjXW5hY-?5P*M*|tgm7rOS0&&z#i>t2Cth_2Yiy4F
z|M4d2w$yvZy0A=2{$ex{&zR5X?1hG{Mtvl=y*6D)_DD_BF32q?;x>k$-hK-~_q&rN
z|M!lH(Cq6JTUsKS*Pi}r-TgemJf-=<vU;tyEkdu75VKP?x-*I*TITs@7^y8y!dILw
z@SA1F>cim8LAcLK6`<WryWw}HL@&={gq)AWfw!|W#~iz<k7w;%v%PGZSCid}{s|yn
z4WivS(`PXte*+Ig=~w(O6ukmk!Js}U>yjkR+Vg%B3K1-vf7PaerS}k<QRik_kJNf>
z1RAv(HB@^%UiWpT2l~ZIW|Zwh*3ZEfzzdVY!nGojYo`PB(?x3Hm;zHXgwee}+p|h;
zFFVG8tV=BO_DNYMO9|S5-QU}K*aWY?!y1-O8y!>#wCm#gLKMKg(C+jlJs?$8qt(_G
za|_GH66X1?Vf}GrjSMhwV=k_S5*GQvt%4l4kVH>dDxbcB*!1SbF-G8%><Ko^CiABV
z<CWc?rBb)L{*^xwBZ%lkhSQ1KqEhmMngy*ch!*YccVX`PB#{(!+RQqmTd;6;_$!}&
zV})H4TwX-+M~?wE+dija;6G(QXnGDH;5Rwe;)+Z8=nWLjP&sA`fOcZvoLk;zmOe4-
z+<+k!o_@HMP)_(fFLBcy1A?kqu4!|<?sZ9S9sD#9JScUt|0o&hnLGkJ|5ISf95%P!
z>|{3-<n))bL5u64xTegUOK8jqhYL4Su^C%8D=awyB>tm7bDjf3i|ck|8-~N2rY#&2
zvZ<RUr8gM(VR7u|J4Wl6kYBxuX81q_uizxk6jeSG8Maz(4r+(P<kqlU*9-dxbODgO
z^3SBm#jphS;A?C24!w~;15^Y?@w1SE$dF@d9_G~7@V+XkAAbVG69ey2$^m0$mIv|=
zDNQ`#+bFp4cP7GDzNXNkR`Oxm-%U8TW(>$@Nd2z<_+IIF^MkOe1yxGSn7^V89U<K^
z7oJ4Xgg$9}0f>F4MySY-|2|>1zk8t~tN~hJ-LTSMZr`PW4%}AL-mk#bbF(@SUQ;)a
zJiVsU($xUnu|N>=m%#N~UkTD2ZTx>qSs-oaUe%lUJMq1faf)@$a4tvg-oY}G7ebMN
zl}<d`0`#d-Yr<M~g_a9V8N3FW&JBbxE3JKT%$F^!_M*5mvNM$4B~X^NBiM*yRTwdJ
z|8%~8s^$y@EweIqYaU+G?JJ!wMfP9N#4#UUcyQ1a>|X!U7YoJ$)r1!(PaGz)15aSo
zyL1B(Lvc$1;qRtaYY=)|5Mdwk)z3cBIR;474m129mEaovi;Ohzz*&*h0_dTGnwBw@
zBOw1FY?1shz^^k{x&gKG7e>aSg~dyL?hG~zYv5deV36GskQH4=c<;k_wC3weBf-Qr
z_Veto2y8g<WL$O?v)-;Y+HBh}8Z-Bi_<ahj3I0uDiQ9b{?g_pcQ#38xHw6bG1$_8&
zT?<U<<i~mHkG>JOj=l>dC5{KP&XG8I6E1KXLyAE0{{!EX@w?eQC8)|}zdmmv*}V8n
z??0iH=}>2u@IMgn09&`8V@Uycgg3Hj#zxVYRLuphBQT!-369#x->1uuDF2)nxO`rW
z91a|k@j3kuc7#~;`jziH%%-#dwj5U<)$UA+CHs*nwg~)fBP#38SHd+X=I#08G&J&^
zn9JGDa|aJf{?k5kaI}?u)78OyD~fIG8R++KqFO&aghN5V9VzaPiA}bEj4Wg^a%IK{
z+#=B47WgjyF3{J2E0~C2aFdc)MAMYj(z(RdFpWX6e^^vKa_4LX1ZTGz`3DWkr-C?K
zioF<WrDZCn!=B_#NWk)L_E9uU$bu-v1Rs-;SR@T*ZX;@4z@Ui2{lurc=ION>X|=6C
zIbfRf9M?ijDOeR@rIjbb$Zyaz8Y+bsFE>N-28?}pla&yILgB+!m)-|v@TGGs*+slw
z$82FgdZc(xqo$;ORIF>MWGzr7?l~7J)T{)~OPFo~1tJ(K#y$uqh6d(v%$uQ745@I|
zVd_B)U+&;(kyWHf(~nY;<gtK!rHMCE*?um<4nm_SLO6NO@{W{b^N;NfM5i!C+6Xq`
znMf~0#22!d9+j1wXtzH>Pb;0k>G@YG?%6;{t{wdTKouLkmd~aIz+?Dj-5$FO?d@D?
zYSDO<94^x6*<xsZxVsA@Cn;1(Ai%^_V8*Ig3rfA$;JYm>O!$5wal%D;F$0(}MOSAp
zU3MIA`G6vL5UHGhffwuLo_m{35*cfu*v2}EiQ&<No6>o~VlwyVXtJ?Oy?#olIhp{_
zC>w-z5UMOTvUDN{f{Y!LIwFR$f#cvMT<IU>U6K-?p{ZC+cGq<gg4U6B=;>BU>qc*?
z!Xpv}mKSCjWd_Fqf)Ju|;NM~90D>x=y%*V$jfr*+-02vt@yJ@ZeT%W`jk6gI{R0++
z2KGd$*?vcMo^R6dx$HINQ0(M+geeGz_Rlh*o6<Qne?xs)kcXq>l3DZT-05VS<1~fT
zIoBd!${JWen1-OM0M;R+$JKCPf`_no#|es-G%{A5W|XYe&YNco)PY@Qq+Rpk*SMc+
zbKo$NPhLX4I3f1Mjs?o(tL1^>0X>N${uzYbc7*5CPSyk>vDp&4_!nb#pDl?>X*sE@
zhPM_EC-8Np@7B5}3ADk_8czqOx&QzO@~Ov}DzH9Ma^d!a{sV`??(;ePAm(Qx5xeHs
zmjIiMf5gUGqjwMmaF_=ll)Y^Y&N-F>BFI$mscIMkFLCWrIVod?)_J);akGtbF--Ie
zXu=)I1g35*c>}=E_zibK{tH-Hu<C0Vh<r{P0@lKF*YwN*H3|N;*d>pZ3bL^z&g7R*
z&3qwVUXP|}k(wT=kat%uwP_wD+^aY8xZGI_j^BE1Br{EB;`j?EHaGJu{!e<iZ>`fr
zy6#lAz>SE;IM}!3<rON|M)HkYM4RT$cYyLPFcv<j3}99O+O;;-rrAA!OMdx6HtU)V
zszE}}-^xO#xTnb>Rx0L5Y8}(mmiEO`|Lqt&j-xpSVR~m_P2h&-nMqxbs&_SICswYM
zdyP@_wM7!>WMHBl90vFge-@~&j6tColPRuU+I(@$hXGnrdKDi;Kv{VEG5;*Nyr3fR
z);rGQ<~ZH<#%sPp7JBpv)}*#_CwWet*ubtG(Mo1IetH@X@?SbjIlRld5c{~Z1nX~b
zRbN~#p&y&HxL&#3CKzl12#NZ}l8j>e{9|@lOLrSPVggUJpzokpI}Mv6$4cW&PO2}=
zcP_!H1Tbg62`1a$S%pqv^SPifJ`N%#!S<%OYS<VQ$L}wY(^Y)2H1LLqHaGB%=ANLb
zaDt*yliXPa`@|zK-8tucj30G@Z+eJXW3#-Bcg!;)42D>@2TwsGe|#}x)qhM1ptT6J
zG?lcg*W9f*#e~^EihkffY#wB@3g}*$e&Gf?uud`sG&CoM%`x_kQ9PE{#4s8v(LA|#
zxml<*uJpM5aF%Zj`PK+tm<*MxKHD_+OaA*%Lo8CCD&)X?m-5tw)}r+SF=O&I+afQE
zKCvMYz4fF$M%2905JN}}1PP~~uPVQ#)7D~LZUbzZH8z=S;KsYnyp5#?e-gU%5}=OE
z^43e(jy65ghJQeC&<JzkXBdqJnjaMY4qtS?<~yQvF<*Y%s}uZbd~-%W*wLx77IT4I
zp4!z$@W!Fj0s0V1!BJuRDWx5+zJ?r1mZrBw>qV&LdY?D`gs+%BS(pl{X|A2;jt0Iy
zDHWBP%@qBxYOR&VO3`8H)nRm9R=SSV?sIK98J@&-z-d~&P$&BW%w;ruau3L-Md)CI
zrrvk5{U*oX(dKdptNBxjSdUn?kTxuzyrgX4^bP9jy%CU{(&%vScC=9^_`9*L#>5sS
z2^hjziVk-^&h}+y>L)4vUmrx5lUB;Bg_h&&prxT0Ht^1u<?Kk%sZV}AkNvQQc>rm1
zZtg1&5Nr5WQ4Zw<@7@uh1gJSODqgqXw=fvn^8H2D&p<umpk|(QWKCLZwl~{&pJw9a
zDW4k`b*Fl~_k^;Desxq9YyMM8>-vz`a?65$Rn=_#)K2L4o;s<DYl%(~xQC5T#>b@f
zuvSyCx%tP)Xz#4S{7N!fQ$f?ofxOt>FnD*F9cmL2ykSpo1u39ws0H?<leX?Yi+t3h
z{TA~t7ZtC9PD|xo>|B~0RbNX)aOV$=FM$u~JG+nItMN<l9vth}477;YAICUaQdE+8
zb|-T~np|=3(TwbYa598pNcA8+dMta2Z&W<n*p7J!vBdWa7Fim;ao?F@4xM?I^r>x?
z<h{B~T9KSlGMxz_q_)N_67bijo?mA}9@3F#0oFlkVr~07z=VIhnyq1pc(H#5lT&2M
zBd8>GFm7W4jXVcF(7x+3<%QY5Zj7Dh?niQHf6*Pcj9s~<iYAGDqOjBHwU?_(rgw7u
z-m5f>H0i`Rb>jOi_p5F|kK-83-p!Ipyjuxb=kiG5<}M{TaVr#?6Wt|>DcM^1GkBt`
zJVaZvN-{u`YgjzZ5F@%Th|Rra0uGfHHcM1p#im|ARD<3qDiK=^gy_3@#&eCN2O+B2
z3^iMS$@S6?Hb4yVM?dl)g5|Y^fwZ@$juYHZE0vDyz{>Sa1vleJBQ!9n$P`6$*7%9g
z32wA&nOdOzgXnS{TKIr$AshI&F;gu07|gJ1*-CR755`qrJaee&U-jt6WOHW-DaXgu
zoIf-rqED8C{TYL;-!`2nFp})DP7GB62B!aWR3|j=m{XHnLO|}TPmHsvbZ>drVG{$G
zcd;@??x98QaA^>_Q%55OwBcohx$};Mn!eMx7GG?At3%pqZI(AM;KFNwh7Y0xO`@*v
zjUiVY*ot524=&XS#hD4dV;V;~;F_e>5NfHdld^(`q2Gbs*tr&Z3xJT}1hF`zfihAw
z+BO1%j?a|$mL^nvqzf%_T9Mc@Al|m5PcU6k@jie~f>Y1@B{%|fkU_=j=3?0sde(*s
z&ew>x*Npvmr1G_|D8FJM6uY<l>5r%}k$XQ*F_NqNVX9^@?RO2Eg+LH;CNK~u;T2~v
zNwXk?J;(cUY(8)JaRUXRmOIiSiBRY*sS@C)W*E?})2~y}w*SzpOChMq{8%Ht0?mtr
zHRPDQGW=Ep{&RS$Hj&g><Up(fdJWdpZBcCwpOM;`z^sg~sW3e_s&4;f`_X0vD_@%|
zP|l~=FU97*2lRG3gIFa$MK`I}>Guih&xJoWXo!E59#(mTS=GUTZH3Rrq_c-vNOA!-
zmezb#=zAhMDg1Je!j!ijs&10P!21QyC-Llb=Bgv2g!1n*-L?W-2;~S<ER}heV|D+Y
zZ(WP6!+qC>$PP>4+a>wgFuI8o$~j70{w7_htSz7Jew;0CytE%C&mR3;7d<pDIYDrU
z57>lh5Se9u-}xxC-d|F~Ts8DC@!3Q8X%gCTib2K4Tq#nhz<}!HI4!#x{$>VZUsCgz
zAL9l;-(;S)TM&<Zeuze@bK^*{Bw;qjIp@sGUg__uU9TCzK8oC4%6LIF^tJ7}(ZOj8
z&aZucW-G6W|M!`M>Y>)%=!j&f74QhTqrx#W>3Pm-jg$Ahwqr*L5xx4+{9D&t$p?GX
zvc$eg)qLW#%A_c*KPxclPCRlOF<BAKh4z?-t;b`AC*`bm{;2HwOB9QfU<*KH&ig&`
zIi~4i3d?jmHlQp^-5^?CYE^bMhOg|S@!9}e2;qT&2?1X{s;BwULSh$fB2UtCm)Jj5
z&7|;nx$uLK?jem;%v``KCO1MecjSJHi0?$PDN>j?V?qmFcJha9RFj1;{VUeV*b8U6
zVxKBZ@5sEMBad?cZo)B>MIBuThw-tCVjKRIZw;&k;CqeL&}t;Hhg21*FnA%V@Ug!B
z^R;6@QS>megV>B$hWE)t%sPkCvB@#3vlKp^dnjYeLjY%?me%YHhZ%B50-*8+Dzby}
ze#-P6+jTtc+LK$d`z68q^x|=+@qx*4n6{)6p^54rz?UI5iR=2w?8s!pr7P+bKpt#Y
z{9pa#TlI)$(Nmk%Stg~8k28Wf`OWJC4v8aZB;+?3BKMYZUke+)@I{C*M(m<oOpxzz
z$8rMjwUMoUd?|$t<sl=n_R*+Bky~nUyj2^*p-VMlZstSs3>#UX3G`Jo!#2f+bw;J_
z=g_y0hcQ31m&Z;H$}_y>-pcoXK{&6Kf3C+LvENqx^*SKO(8JbfNi9$VL_<14aiI3o
z#Ptx)c@Q;yb65p<lWaeyWK?0FKUisn<A|=YpGnDa2%~cC9HuEqAD9*-+C!4QVVfR-
z;Np@1?-T<E9<hJ>p$bI3{A7$6exh(E9$k&0>r#hx;V3hHFe%zO9c-M<u++-jL7{BW
z7#xV2Nc0P|LthR4!Dt}=P?!Jx27j}sf`=89?5f{gl<u$@)vr?0(JXP32ZsIJ&DF*~
zNH*w0i2*+(0(vmM8Zuw`V(lZ(jz3zI@Yms%Xe|35FZ@@khcI}Qz`)aytJEd5wce)-
zZ-pUG(N5J9o?u$ijhb(4UcV%#A-bZrB27Ee1NWO7G{bas9FIz@u?Koc)7KocI@&7z
z=!m`Zp?Nuvx{(BSJKm}xpE1Q*JE516eMtu2${90YfC*&E-xk>OM(`7h9+ZA-JxnCO
zREaLMl#N{lMy5Lr_Ij1-0ue2#lYvC4@E$dTwf@64;oH=diYQgXCyNvc<~9R3UhEB;
zTocW*|JC^wBJ?KzzdFA~Mnw?erC!M~{``DficCGc#wMh#khM=}2vZEm@D>702Ptwx
z<9K=ZgY8VAOR~Ro?s&C!@vxVh4vU)yA~($zB;Ati{;T~P6FwWlO63E~PU9m%9EN$@
zUpBHqRy7}_Sf;I%;21v=CcYiV9tsive=V@4uNscFu`x-B1M#Ag*i=rAgbNsUe={_9
zZ)s`X${&JDa*r=yPb%<&(3-!tq~1`C%?48jjnOtqoS8`mwIbN^=L!W4?fV!Hh;%u^
zM^#;c!tmkT!MwK)MLP|O2n-<;5TtKP9_|!U@I~RK(p@R6jKma|k``s<JH5P|1ar<d
zy-`dU7Y27t<{GN@EXkP7R|rzXRK|v2bDp`|XlFlhfo9qe?~F*?H?6Qkq5$~vR3NW(
z!!YaxboS3+iLU#8>b6zZNxFI9_b&rF_5QMaCk94UjFE!DO$rrG4v9&$?7A8kzC{-b
z7rq*W>4*GLm2Zm+zl3Yh@e+&CCCos7J~nTNfbM@Qn0~{ac<tEpsTj0n*OvoVpHZKo
z_#`}TQo?+0J<dFY@4UBl(z;q1Y&HC_tV3sR)(0M`YxUJ{$=++`v#es0JO=Vaf^3|<
zq&<X(GC-R*&y_@GvKUgqo!=RqzMxG&?N*~_7FIF!pR*`GNPG;9)83V+91>e)F16-8
z#+HIz-uY@)%WeA;7K5TCZa=Qh`UtNI%<$&(O+AXP2u0S9S*-VcoVqYV-z{*5XZ5hq
z0v6HXrx&I^vQ$PRB=1itj5EXRG>=&MX8#%s#3>N6H9eS2+ahB1p?E-n8-49{;)4`k
z#J@S4!~-C5YnbL9!UHP~{@PaOoe5g#rF+qs!Qy5=2K<w~Yk;%DW!f_iENjE`!|9xo
zGn*?W#|;(r0daMoH1-NUy0<zdn@dc5rS*XOK*7=z`N<-x@DukR^kLKdg#^4o+wjfN
zVi0Gbi*)r_NOcQf`!owo%z*`q#3$RnUk_~jYA7Q2){)SQ{cH$tI``VoCgIh9vG~>c
zHug38MBr@yPn-#dc=J9Jsr7fM{!R0P%1Dq3!VkgWQuVtHTFro_qv<W-BOhVX-RGjV
zQJdlo_F8nffs4)%#Q*IJ-tS6a7`zMveI+ltwzl#@VLM&6+88x3BKn;eW+~*VL<(UD
zA}HYhwFc0Q6fp=TVW|I<|Gj&Ym_nQ9BC-aLi;TM~Up5Mq^kuce?#E{vTIyA8E23}m
zjcQv`ZYg(xDSk?_7{sBV=^Vs{tw3bX>f&$$tRJD^+S^Q>v=0s0+?6UWpKCxG!n@~4
zR7sV<8dcu-^ZYRr{vWi=hOz6-{5gbNw+^<fT79Am-f-mvVeNE3?|C~=0zqZYT6J0W
zY1Sf)KR$PrCxvZ4SHinmFYxmS8b$S=u3^uduaX%j(6b&xiKxsJ3dfpTSr0n}G-slT
za*2FCkp0lsf*yb4?AuzRntY^WLK|o6(?TRGHrFRc?z(5bXGjIIcP~d>9isd)7n=WQ
zhyz3Jy5^Y`*bIgm5wV*`4na^x=|fw)EFnR~AdL~WFVl_X6=e9pZkxrafH|8<XvGah
zf~Gra)^G-_@?V_7PsN+<JZ0tz=lQ>&*?;QK$9GHcR^%z|J#Z?>13EBWznK(vNxT%l
z{9lqzYr<OK3|3E}6~y~ia;*ZqbP(rHIm=@)Mn&%m#C!9o34?e}$?Et_GYE-ew#WF~
zSN}VcHP_K{YbI9zKk%12%n@F#Tb*1bR_|8E$+RH?8RxYq8c9-rioj-C?CN}>prgI2
zzN-iVJE5};wb;ea>cYrf;&iGZ_^k;vf175BCyzig)>n`l==Kyb^o%oZNa40jM(1vD
z&85nBt0wY^LWdWIF3LZ7eW^^q?Y~U*N4~o!*n((85i6VPg#uqYM@C3}|3Q&Vdj+4k
zON@YkLBA@d?YB_e_kS%H)$lv9n7xGtkgwX*nes=;Buk%WdCmOENUz{DI1Cmas~7|l
zZhcwM_0IaHVng%c!M#SH_mhr*ill|p=gQl_Snx!LqycM<97}2+C|0NsDoL5~bWuru
zGylKgWME)w#`7iBC|jouu@;5k11Nmx7q0!OQRT{`(FsZ?5mgckF~vhsFf!KL1fuVZ
zfsRt}+I=5a8*_H!9*iRZ62DO<-i+h%j#JQin#CU#G{r($TQ=M|bggGu*agw1X+A<-
zxywC%#RtA;yz<O$*|?EmboxYE3{u)*H0LZmSKpDQ;v!HpqDV`+>u&Rz9gSt^#E-Td
zCsQ)hUBU(J#{1YXVQDz%t=gV8Uy|fkYU-{MgDODKKsR4Ql$)Y(E!-0XcVS09@d`L=
z)o4pj=94pl-3RUT21VF$V3?NhdVuTGz%hF}M81}h>1P;pjx9`}FzCdfxGM9RlfWxz
z3YvPF?(NL*?*`BlOd)|dcY`sXmvo1sb+$6$A!jZBVAAR7|K5>+=rN^z9>$1u+MKIx
z21s5Tu<ZJw*;K~#6R8;V>A_!f_TSZoQ=rQ)OQO3#LoW}T7g3)NY#c2A0%N}hIhkqr
zk+_A_zFYDadHf|afI;T>;z3|)M5G5%(R9>%S);6#`|QWa&+;_|OynD@6SLWJTAXCi
z=9-RNn+iC|d7%jYnwO9PL#WQsd$wcoz@0kM`6R4+jgj}Hr75XV@ZO{X_DYfmmfkK0
z84D%F0rISII3r6pjIYodB=;KUr_DUwu)K7qu;-;m(uMS-#|ZE04~6${SaBP#g|3Pm
zK+~*nN3d7^|2X7>R@#FK1Q6R%wM@JvK#!{^w<c3pnHJmV9_Qc!MUGWK&;H6p_LIlU
zZQpo&pY49N?_9J7=i=TO1|4=?7F~Bq<Jgm=#_m_3p^FE@cgBR+BhV}3?{BEz85xX1
zx1Z-xg;7cH@+Qisp_jj(;utAv<#OQ8P%w<GwH#Ui&KcLf-=D@P-UvC?TC6Pqkv-|N
z5NAz2vX<XvNf)9bjMbfwqHnZ*wOq85E>L92lP`G!=*_JNu3`!(dJ0z%ua=-0Ds@&E
z;qCBxw&rJpZrz3NJ>Fc_+GfkYdLCH`*zvjV_<sccPxI4p9gB~mftuQ85qE*339Fcc
zY^*QudS2r4@2GyFy2tPmj~+czwgTco9|Mn#c7XpyTBz%J-K}qf!oHa2WyI;TVU&zT
zgh?wvGV_T&vlOm8nNu0=*j(%H%k4|e+D0Xc48GPBU0Ex@B5Xfh1!)dy>@JLjqdZf!
zikb1i=J9zQx#-1=94+8kdo~PcGWBF9BUC8ny6#ZuR?V4^Xr`$CrCu4TGJ3<dA6@z;
zCF5$08|m?L))QM@Y*AoT5GM?o4JiHx2-BpYW|`@3v2p?U?&>pJx9;PdG!EmRNja^(
z#f_t93h;#~^fn7hFt4j5T~;*CNvPDf8{i|uD%;?@VfUV1Ip(M*WJnY9@ctog{M7%G
z&&CJp8o#}=c1@r)<`E|Pp-BYob^V>8RjkmvmE^P@t=XryKmSfrXED9#Of79NU)nK@
z=i}?zEJ;zMy5(B*RthBM735k*{tMKF5~Zw*(|3rT=9oNh_aMMrYzT6z02Pbc+Opfr
z_U~%ta_-8YxPZs6^p#Y~A@O%7zmxP;7iyr|#V%jG$HQJn?x(ch>cQFb#zoWnlf>}A
z?-lqNZc9d$=&OQ4Ga_RkLc)8-f_ML{yBJn?&bw_+#|H=FzGB4i3&OJ{tVZ+9`D+(Q
zI9e@qS(n&zN_zeC74aH6(>x0qWGnNvr|;Zd!DyxQ^;2A_rYaE~A?UV}Dv9Q7D>^;w
zccQfys=jIoIm#r#R}uWy+HbtU*iq0=%u>;c@s(S`Aq_%ARb|7D*=XB1GS-629QDW<
zxucrSYoa^#zWJ}|LWEUhf4!8Whp^q*)EVhjME-xeXtx_v9DU=fI|{9+4qL10U<okI
zZSDSZRMGqIuJ4~Y9BpD~(3l~_TI%|DTZ6VYm@IAbdq1=~5<nT5+4sm^@i7R~=c&mg
z6E+)asxeBK$c~w&hUK;6sb%T@c6pU%bj@73GX;<&sW2r%(~>G7*V?TRD?)~?pZ2_U
zHEeKUEc%`dRuHrqSo<Nv#~HSq1v{45V&A-}r5}QmkIKZWn1LWolP_Ru4b^=N+iaq(
zdz3dRuW3g6)LP(!1G)`~><+7aA2owX3gP(Vz?Qp_thWw~im7(KR29)@*$Mj%I8}9z
zfW#*b*z%i=v7l)(M-*MBamL-_!^nine1S}N{WCX*D0`m`Jj|!U5(me?O5CA(n${AB
zc(wl~e)Skng<LeEw>L=k;U}SBj!bzH;S1uEu#B$67*sBIa@rpJHMnVTwYQ1;dAaqG
z+0Rrj@y#07Cu`cI{GXuPTfr!~yV)Kr9ji{e2A;Q`!u#0_lP-MzBpnP7*)!!W@rtP(
zk&ZrB?-%WQ2IU0*mVdel*BP8!>91mIFFX2&{-vtq72?}Kz`d9SQWEAYH!;dgI`un)
z=PEq%Bb{4{FT8m2$l6-BFm=<^@)CwSjsCFy+>odq!dpLy)OINU@s(eZ5U};Dllj18
znNYE9<9VM@om&AmjEip!p|p@n(HA--el9<_@EQ+QZo2mqc$k58qJ9VCVU)$X<?B&}
zWn*|Vv9A#Juo2JiGgvZiNuo>5=JiZKn-M3F2cZ~q(17lYs42E;oHy-lI9Om7X45ny
zWR=0ad}fo%!D2+Tcvxr;&^Bvg9S@8n|1WO$MY>Q)7I#6;k_J!t9`2qSd?e-os{_Qj
z2YnQn$bi$ex$ALde}KbPTzoncpBk)1pYEr7tAdY)HZHfM-Je#lXkvZZOj(!CPRlSs
z=Q82z`<?Q);9EcJ?SPy17Wt4&+eAi2uv2S?gBDz<_=oy#H%u852KcX#`t8Yq3@Ky(
zqrIgB(+&@wT6-!XNHCjAj|Rz9I}e0&mx_K)(hL?Zk2#D*H_UpUmJS2xwai&a&{P1>
z`lXnO78ZVX)zQb5%FSsnU%H^$>@oH1ozP@AE;PwjcZMgR>?!;uEeuvuM(w>9V(yI(
ztnS?l0E&O>W^QSYIvO;;H=}l08K^|>TN8FveVa>)+5Kn~0oHmHfLbmD$fqeB0ZVtf
ztFPV^bwf!v<H5L-?a`aO;>@nooW*z=`$bnWwdDx;gxv;Ue`G@Z79PUd7x>Qz3Ojis
z{ibLRSQ2Hd$9{Wd7w+-g)w!K|M3=anMQy2m6eTTesNW+mD#~iW>S?v#lzcY<P)(!z
z^%`*YSU_xIYhydaIe6!ka($^!AMbT4eJOO;^C$0?qB;%bhE2&ZQ&L=ka0J5>685#=
zI3)fmR2DRoG+c_Zi`+WgwC@|6nfy?O*@_-U^~kz7cUh|=4wBvzC{m`}2jPDk^>ubW
zI0YT1p_HJv4*x2u2H0lUMRJz?7!6zz`4dQH+UZEH@WY&M9EKH~1EthkE&i}`Ts+`0
zY~J~i@BK=n_W>o%m~&PVrE)7Eij(;L^?(Om=f{7ccsXorCT53nQsFI;TOH8=%e&6r
zrN?Z$Iq7esfk^!GRO+Ea16|XQ=s)g6KX=xY6wDiVyoBxgD*4hK$*ZjgA0D@ku^Hq*
zc5iE|Y=B6^fqEM+XF;DH1&jmdKm!1KU6lpQ<EwFs-}=FV*nPibLyl2?>6=;Q`I!sf
zSMye#ZkuHGXTIT@uzwx(4bu20`kU;(Z}o#ED4(6$hLu&>cUf*dBm(q<zwCnKKpARn
z;G~&;od28i6q2;#d$HF$<ZGBIg}V<L@N?HPi{n0v&<4%#Wy5<C#V03Kt;W3P71Fq7
zL3|X*5oH#4hA?*v#mxV=72l;n!j)rG#Mf*R1aCx4jB!|OPc;DYO~`=m+%3`Qe>~N_
z!XbIGw}=!1BI%V~|C?lb>>5`6&b9gD6$lRU?SVD8>VHgvP50n<+Hl7pK7-OKmv3XO
zr_KL^$|1sRG7r3x%@;7A>V`!(7-PH%y(Yp;HQ&KE?|jZCr7TobX_6U&zYi*cz5&R<
z#4{;Ie}gN+T%9)Lt|PMxs~i(Yyk;vcMz<**`A=AmebB7hLabL|kfA8z@iPmqw7ZtR
zjwoT}k1SkrWX>lAqrU)DjQ;mb-!Zr-R9_UtUG^NkkwUVQVknbzfEoJ?X$X-ocivv)
zp&|}#T2=!`@0Z37cGM9}T9e(+q~`ZXa>@JNb}26^X(uX`vMXR*?O=q8{wt@{=Tyi3
zd?nB|e?QVwTi0Hy8UDT3Nh6bVgpLA&*jCz7>L?zvf2`K^^}mI*R1pjC){=40PaL5f
zC|7w<t41TG)~|ILp2s*XT#E*k6~;&(a_zj3qyTd@T7NDo1_Tc4=2CZ6<cu(;e@SR`
zOD0Z%CIwF>ZStacb?GcdF-h(WKM1r)R6vFiPntNAA9tuHc)?{jr_b}A)pPjDod07N
zmh_b#Mm8rZ#A!_}kcPmr9d~}dsxK_q`QF#|=ogpRzSGi2XD5qDR1Cm-vCfTw@Sp8-
znPo$yYqD&K@J81<?PUe<pLs?rm-QRQ71j2*+lKh!uX^XS#Aw@EDdZI0+km7twX!|}
zRb%3T*H_SB+d|lji_6q-D_;8Ct-jzAxQKT7@U|3ky-fqe^#($?PY-UKI**5~m@C%E
zxT&DOdOxfTmPCq-5kDUv+oM0J5WO}Jp~+`KsStK7@iswXg%Yj9?5r%?F{}TxoRgMe
zUMEvHR6H3R_V>GYR%!|9tZ!{8H*s=7lVie51X2ovgGR3IYO7#Wb7m=Zk#TX*$o7Z7
z?|+6!7MDd`sP(-TK=F)Msi22{ir&nn`1?yyH0`a2MZAJO4VGAf|E7SG4A!rQBQf{_
zw$ER&9sNoo|1v5eceJVhUXwv&{_Cuwwc4{}=L%`>wKZnWHIMA-Zi{)>bsba<c=Q%Z
zDkAO^!epErf{AII#G~T2`B<RcK)JHB<v=|Z>u!lBt#<WGvIsQ|_e}2W&`BxKC`6}Z
zpdM!qG5M|u_>(kl4&6c#Ocip-Vi5eFG9)HV9I-)$s~9Bq35Nk>!)ElknWdwIh{qjz
z*9g2x$H2c5f~vckN-VhAq%`rPaUhr~sLd<#TESNW#3Tp>yxF~IlRHr=^AA<YWvlJ0
zh{9h9jvB2LZo-oKcuVhoE<uBcZYW5H3vs4b{<asN-ROGB*m<Fcmi2r&^Ht4k(BJO%
zVv?PN5PjybhA5_=Y;71^fyD8f(&HYWFMv`kFZR5l`+Y6TVPC>f(`vx>uQV;(T*}5k
zzbacdhc5n&4RcD4X@}tDK?CYxiaEPZX}AbLoPBJjuq@LmnvA5n^y(mHLYdUY(`Uf%
zqO!RRI3=m28}`S}GaaorCqFLwvzqkMWx-G&2S+{3Tv+naJ3$F_5znQ6YhW0yocU_K
zxeyb!s?Q*scX7eNnmow;j!Vz;(n(lVBC+?qCuB#pk9GqzR93@%_n`!6Hx4T+U=wUX
z$KShe+9EPp&PviM(?j7QYl&P%BuW_VpXwo;hkbcp_Up^R_+wSp4kET1zVJQ>z1b9l
zd+Uprw%`>~Rcw+`;GM9acFxcwB9#}BF)nknVNVCfn{N$G;vJos?+;4jfdU3Kd{xP2
za%qYj{@jQK4WXnu-voArMvg3uk}a^l44PBJ{ZZaIi9+^-@Q&X?X*c&7(<3fZEzN|T
zOP(wotleGZ5-(iz4!KVVC`g2R-ObG#!-tA4Z~Gz!vXdZ)^2{cfYC}Fpcix006OR+L
zqeC(AA_LyK<%=r&$!fNi-L5mx+c}bzXz!s0sM)1bQ4^zyVHj~aL*snu?v$s-BGUB_
zbE+JY{e>?k=)yd?lZ5^2I7En#qyowbwnwv#Hd$hjPn;<dx1ijSgCZ!3mshu-{aaKf
zwa=2?!d&h9IVO-!2<H#}(|+ySqxS&+PVpqZ^<x;^$n8(zCOe}?4INpow(1tywKm%4
z;g)~$@ll0xPM6`HS`lnMLrC=8+S7kB5TP2*sXDHuO`L;VKYz20K!zlpLyvXDh}3T3
z!Y5>;#$m$&bx9sHt3fUGSKO@3TcHCD`>QeU=!H!aImypHjtzT`sk*w^sH?}RAgVNe
z#an{fGR%$A{w*4!=U#EPbUKoWC`!VnzHqua;QA(InkoZwHUM4M3;G*2Ok!WNIc1%I
zq-G;xzZxX5LLB^#p7%-3cS3TY%jKq3&-+y-Ck88&fr)+|kZJ=iB`-%VTLpdW#5j#B
z*o1856;m%P4LebE<v^)X*}urfj_U`}Qd0EohsueyJGfsP|A)RH2^4)(3p`hJzq=x*
zx|>^#8Ff(irK3Oc&eEREK9Ty?_S_eJQZfSnyM&*u>0gL>&8hmYHY<f2(ZBVQVR-*;
zPPOCoLA<^{w4!A6o?-Mi`Z_t1`H^;VAb}-<;Jed9^V}hF5f<G8)Hf~q?%7s$&mdgl
z%6?Kt{NFp@qw;Vf;OQX}eQP98s|&_r1d2TlnPG({Eu4HZe}C*s1_hKoQ7I?#CbihW
ztrNKtW<D4zGbbFA!u2O@@>VnRO3Mg6QchmR^u+9yA*<@-pF^1OZ`h8W@M*^u;>UbL
z$B%u*=uO4~vgs2rY{k=On#nleCKoIQBR2ax^n;_`y$1>V7t5tTkN?^ceakzLguSAU
zL4C6NxWbaRY?UADwyOcqq3H!oMon@FWg=BQ$1M&u$VEIsFQz91d);=>{OzQRs{tvM
zdmSA>D<d0t@q#94xE}Yn1yEXtvF4ozd@C<wijG5Q621{id;&O6n5(B7HSfTMPW&Ge
z%KOgM+E+>C-xJL%kcHD+dHK0-^3@25i~C%f+m`vLzg2*^z#0-6Dm88SAGTj|`c&m9
zS;Gk@@Rj58zK5wAb09Ndf51oo6je8$_vMM=YmY@_U(ajD*`t~3nVp;4{J}>D*V~K^
z0oPr9eJ>uTGeM`egZHb(t-#V)X$9X9^rFV1N+P)My1EDXhh1r^Zm0PSzOIV_-c&&q
zZf`)uaW#3^44uTO6h38j3bVY>S{yxh1ma(W+!)aW%~_QY;L*yq{V@?!WZGWL^}h=E
zgU^q^kRApGj7UE1T^RJyZyIWV3r2I5MeAG(#yd=s-l1)IrG;J6(gvt$TGtz1x?g+4
z>=_aiZs-?<j!-^L&<EI-$14v03~5z9L$UqDMB+zI8)#yM`=@@b5C=7$hOXN`FvRct
zhYZq0z=i~NhHSitVc0?~&%!z!o-FZ^v24kX6yGh_n)4WKbP$KX)XJ3(>2cexPS@@O
zQ&P8|I_ylJBnhR`!tlPI-g|kPc&ay1$z*(<8&ma|xDxF`;qNEGpPgg3=o%J}Wq+;u
z8CDfWlB_ozbWXY6Iz3UMjO?RqG(L`*z7(Mj;EQui-;#n~J(LOm0Wp>_3_H-i9mhM{
zQX_b5M?W)8oH0py%-gYT#zRI;3?AbXLPzoUS;@2S$SaVSD(Ons8J+Q(&k~5cbD1$h
zj=I{#e7C^$&?}5qk4`}RxXY<PE=2=>nqsrF_Z_(0fk{$j^IcMI7@je~8pBK2A|V^B
zEQH$TUV!wgLvjR#QlK&C+5{6Ci6A=vGWpP2vk^dnyWUB+fEYckp+|=$vDkzyCK&(a
zH;aT&407i=CSSfP*3WBvtKE?=%I~sw<4wsmjqAWC#lC&D2;T$q7tc`mE@&m^!duZ;
z$|BXQfZbj7<ETQt38MEz^yLO0)P8P6v$a?}rP4;}s>iB*F%I{%69-GP0+&iTqPr_*
zEAIliF5WKl*B>YZaZ!J5Af1Q1$bR{d7h}B;pKHzB8SsJoiQ3BZfVZd>1H)iAbXb&7
zRC(~1VSHXJh4{Gz7TT--k4dU>VMeQ-APMYH0T!yt9$m%+__NV$`V$q8R@ykwK&3nQ
zw||t4ACJfuTT#z~c>2H!y9p7S3<zSKC(B<?2{X<h+^Lf4J769zdV{n4Xru{q7ed8+
z=kYMY{j;AUuuwo^rxcGE_67ajg(}$K#H1$d7lJ@|-QPSaV+i;H{y5;T^BW+z{{s{X
zEeo=n$dP1Iz@=nlr>^W#WwdtWV<3dZf^ZZ>dWSW+RHXJNcZGE0oV=o8_9qghtE;nC
z^N*sb?q?y&=iag8oJe_<aJ6OY#g^KV0Tkz2-O7tC84@tM{cadnni{*1zaRyUbi5eh
zYFDo4v^}CgezvmjHhv`WAdB)l&raP>><F^M<Vlpy8H8a*QTG>ypJBh|KV=ZMqO-7r
zeGWU`dy}H5e|$6<t)I4Ziga;JG3kMLM}h3RS7XSR?;nJllo_`kD8^RHce#9No;}B`
z4IIE54d~l8YL650gs9Nl?8(VhEIUG!DA>NR5BfhiWzTtWA^;D9qwHY3*f~BqMvi{g
zo8@iPD@}i=3Hv<#VdsBpbJ-BtVJ^J<Jt-yA@)Lb~>#8_AAp4fVqmZwQB?eWZ$=V>{
z7s}#s2nh+l-@VtzrQoTEJ~Ls|$&3vl$zQ!=wJy-d8lPEaS=q4CNv?C>^DIo2sB3M_
zH=&!_8&@&Mso~`NGLa-FvYsj&8l|%#-i>Y9Xs=7Q5Ato7wceYkJl~kk0xur1NRNqI
zF3r_O{)7<E>0Sdv#`qu?5bqiv^!gIu*53X@5s6xI)0pSiDF;o+$@BOeTXmpNWM!nb
zCX;ed6M>MV4u#^jujcS5rNG0jxc+V<V3MJ?SFf*D?zKAAfH~M;oEvif*dQ-HHrmN=
z>mh37iIwq0Dq3)sWLX!Z_P%t|tozB6_32mJV9;8>Eis?#$JMW9O$9;YP*P(ecIvx8
zMn6eb9oX<+0`XO}OPxQY+RldQfS7#*abTl1d{y5(SQEKFQPF*qfBgl64RZaZ*6Ca!
zn@gmGm`x1k_*E~2cX;6W?Qd_ll-Dy$-&Vp)^{n4H?H=7jQ+msAzfcqJ7?qbibAUSC
z{GnGr*U{AJ`BBbreSdt7mkR>JdkcXX&L7?w7`}DFIRA%Y>+??}ZMs;{5wjB_>PGHr
zr^cXmm)0%(8O3TlJaDT}T95j=Bbjed!h*;|!N9Hv(!Ix#z55~SvT5zuPx0!gcf90G
zrAFzRo1gTEe{iL<YP<FSY3i-xn*P4{aU=vJ2GR%)L=a)4l#bD$pnw6}=ol%|B{=Cu
zdZZ{RC=#Q)b4W=l-AIfO28`ivzCXY3_woJrwR`G$U-z7I?>*-=+>H`&CCIY`Pz`D&
z+h0^h6;^xLyS7ZE2@$zuXde)r=nTxHx$BRtCXv2);l@95;a3Dzce8Gj)<0Yxmc|rM
z1X#ZQ)PWfO2ZI&NQJzvQkn+JxI*wcfDGDyCQ)d=?axiQMx*v+7PTwPll_6Nauq9I$
z*gc=Y5+6aIU})R9%ESLWk(%8IMa_~qOaxd`6xPgc5+bQ7d|O}bJDXkQWm-Ou{4)>o
z<t6!sJCCsCo_Tq6l@Yf?8lN+JrU-c^1(I`RUr{irqr}eperm8gF7*mxkB=Nh&uJ1*
zWw%o&Yd|~Az-+{Mmfq`k!u>GYIp(7P#?qwkX3eq*ZS9b{*)B7)0qC5jf5<L_IxYFO
z##qmlAHYBFHe)VA_Wi?VARSxUw8V}-WP&8#X6%;FrROgFn;xCF(%&oYlS8j(fADel
zm#(07$n2NUm7n?9a5hsWh%w0`tNh2HA@-I(MAe84<3zlGscd+l@kjG`JECsCh%W%r
z(D#plA$q0_xjfewh&WDioSlnc+5QvO^H(WcFBZ@kgc!BY)Ka&H4X^Yr#1Bpa0>lk%
z-eM`qeedENvt3TPA&SvsxAN0##$0BYNmtQcAId)4DV`aQ<ic*nKpnxxZ(d!SMsNRT
zNlbM!@DpMRhg1&Kaxd>i+iIv9^R0&jqBx|&5<saB$>KHH&h&t`_xlhQoaYLi5_=l<
z`;3gM=s68eN%I5*Y>A*xrjX<-xp4<G=xU<L5oy*{#9%e6U|+RBw~FpTuCRUw9p%%-
zu<iw#yUcQATboEXp0_uyf~NSALRKY~E6{^)V8o;4aa`ou)|(8b-A>tSpP&0eY^j`l
z)zMH)z*9<V`fb($OWe1v&8m77jCATG#HJFpK5U$w)D=1hIQ2awd!F;^5UFo~>dq8x
zo{J&Yr=x?57Xa07iwR>?aVx6zdwfQ3WA7xkr{k<svnor8W53v2eP=V<@3{-1QWMr5
z`g7+b;xjWF(QCUKVW9i%v2*e3Om9KAnt@|^pcfrsoe*zpFBWw0t7w!z?2Q)ByW@Rb
z0uSvs9&sJS1Tbe@c*<>zM-}smet<}cYFu0}EX5L*G+^4M!`5e%YtK?0iW0PWZ^&$!
zc^m5$AsNCA8Nz-kRkV@hIJ)s3)&g=7wVFBi8-O@-obe*QBL<z4FhbS^NkG`#SO#^a
zmebwSkvIuSVClX!iN?&BGftx&{v!&18eEq<2XJBah1}7AJp^|h#D_sm!*qE;&-N*#
z`&eyvDS3sd#vkJ+E84<bAG`7P=Bo@%qhF#GmV8tOQ2PG5bT>=!Nr8%OhqwM1z`*4X
z558kQzMv6)G%c|6h6ZNiB7fH~Uu>5Ez0m+<L%07^O<xg(V7^kBeA&<3iD{|WbMZ?<
z{3eRgmiB{;p5~rvdBifaS<ReUwxZ}%8NW$#FOy8TExTo&C-+sNII=xLAD3tA;l3`;
zScW1flJkeJ5=GcOvfsTen9kq(W?;UkuHB3tZ!gYAw1S$x)tH^O`9h?!Z5k^=_xM)b
zW>@t;73O|<EKJs)*e(I$OBb19w}M%l3SV+CZ%@D|-jgLNR^7^bN&1U;!=JQBk2k>~
zhZbnvi^yVTlq9Cmz3*R3$%tLW22)(sUPAffy)3;7_eW(*vqR&fB%dRj%i*Z5YeH=k
zVF>qoy_A>Q8s~$<nPI>C^iSRqYdbURSZrX$FHo3MH3|WOoQ@yu&WG>3*SNP;S@xoL
zGv?d>|7_DmXTLTDMLqWrOj9G>VEB;k!vVeNAv!l`?CJxV%6N^Q(S%V2US1&k^B1>u
zu8)G;W6y#qHaq`jYn3D`rF=e8IB;9I4Epr&Zrh&BGq3dOS9C9KrVQ%&me_tmnr=4^
zG%l~IgREu?zQLsK@(FANq6TWhTY%#wlP<!+h;WM_TH)>73rX$>AOPDtUdvxM&Ug1a
za+JxMOl(N<S%ISX5xTzpiG~yXT4;PF=rso_qlmMCh^F+O%Wm<L)vP+Zvw*+-6^r|z
zWGiT`kpowR={HMkU#;||DrP+5Wp(VH%Pne3PWh|Z2)8wJBrm2CM6*}wS6O#QLS&VH
zVHFw<NQiuG!|_&i6&;=_aaCR1CoypCvFt3u2i2n>K4={u`(rWP02{Gmk>P8l3q*}c
zu!nwATK4+pTiM7&b#p&@TB5g%Fe}bl&x;Yr4!mK*;lNIeN@XR@3t3ar@z7MhRf%*$
z<O>%UC&}Lg!5O2=c320x+L8lI#!LE}cF<XcJ%2Aul6qJI&_p93P~E3QxTjfO3t}kn
z4@CG%8PYX9%whLuN&k)=OqA8k9+-x15Ca?ZIIZol(Yylm1geE8U~%UN83<U4f3e{0
zw-%H6l-;@^2;!U{rt{^UcTV1Yq?cmc2y%B4ddjOSxzaZbyviXnAs)|{xPRS2jMC^D
zqdjdQnr|WProZK2VYMSGbri{Jk4;jEa;UgLj+Z(WAZMs1#@;mZTdw5xpqt;Tg2t;5
zN3L|y?F7K)J36yZF}LBQ$?F1PMY|*8%Cz>H56%UH55(uC*uyg}@9Xi`_IbUs5NSHp
zZT0+C_PSng9>+SqFuVZRugFx{?@Rs~GrtlgZc4O7m;60u1+_BQ*~cEDa9HLk)g_b@
z;7|;1AGz?Ut+|0h><_Sks;-0b5L>oXlxOZfXTb++eaeHDV0`cOvKL?NGklLV*aX&)
zq4cb55oEJT_{0KEimbY=cIz7e^5yoeZrMxb*JInWB)dh3JEmG}^bg!2f+!wicSw?<
z;m;m2l@%DP4o@S$-$*Y{HSO|orAg38n%_R<@H%!dd@lZlq<O)Qdy27?!GYBp`+TTF
z*2B72odFDAepx+iQnTmi+{VyJdYoks5uN?iK{91LP#G$X<3oH;TdFgb_O7(&(Pq%8
z`oV~@lPys<)@=$qQfa2O5F?Tj^OYPT771BTxs)X$ygVS!rUIx*p1eNRB2$S@Q04c$
zRufxiaagCXL`B9vyw3m^u#mxZX#ir?A4H@QM^6S=)`=zD12cp=Wk`vvf}xk-6hJ3Q
zFIt|MrVMrC4G#SE<$a*C|0$zaH{AH+n>v{ay8)G`V86uupPi6B)*;G{50$(0#LUOg
zShULXds@bN2o;!#LDC$Q2Dno_bXt*zTV)I5k4Vktod`Nyk+}HY$_TgBDM-XueyjJ5
zxCVY;yy*G0g1ye+KyqiT#_}dj_m)q4?zENEiMji)afk24%DTFriJ>Qom{LsLrhU=s
z#)J-wtBN(U9Fq=Yx8o?MV>0LIWx#mC^&_G{U8LfNA8vrF_{aMRzF5_+rsxKE5tgze
zO7lAj>8(kCs;|4ocHeC)j!ElTbGbI7c~>$jAD{L_L*);&JjT)s)ND6D@CqpwOVQRs
zX_%7a4_<i0sJlFJ0FS9&uyM;|EQu~V0~M;0qA?Z^>VC4273W@v|4-fl^3~T8xmWFY
z%lE*72wI+cD4mWYm=UW5oJR9&O`8RivAsIWgUv-l87KMqKx&NlVGOz(v}5Xpy3Uo%
zqP|NHr*7oAfOAic0r%3ie2v>rQ_pEgClp!bcVf#5{Qdr65OeDdWXbN+dgG!AHlh_6
zdP~_1dHnrX5Q0HQFQpBhTdDi4?->kP6F(+E`g*>dxJTUB4=k5v;A?oGZp~2xjb>|R
zL^<3}yam-^XyY=T?nye=$pgiBiq0!NHg$ma%Gdm=*Oe)y^@U7bvV;OR{2_k`Jc?g-
z^8fFq@H)d;>9U}+nO+{E%0RC9RTu30yvm&<VxM1~z6QHvu86G`P4}AC>#<6dKIPWR
zQEwK}6f2mzkd88jm<3lT(_$xV;Tz;lu20*yA9ms03hHORrSA|%aUlx+b-#c7)x9{$
znQmPHb-8i$bG)#e4ZWUF14u=khP!sj)O&?EzOp*8bN3ky)PB4|YjD7-$zoX%_9loe
za=ESmjUrws{VCPBmwc|XB~%$3{#M7qSKD-qLWrSDz9tcMo=xA&H4aYd+Bc3ENsA*^
z8ddu2wHdsy{PWspf48b2yym4f3AiM#cIt8AWX)G^go=@H9u)jixK=e^>V<)|zV!~1
z!I)Gl16V1qm;Ix?+QXtLhP+DVEbZ~)f`ndeEA30bWvk0fMyFC^m&pFSJnmBL!@M7;
zB0ERkXE;0gp0ZBSyVLAXaePPdPH(t32(H7=vopV{L$O-a@(m<jCB;@!P+RUqAN%RR
zhjK&s+?&)~d58=JcF+Uz^M0!-i4CqbVk_gfFpc+c0jLs{ZVc3nPM9;aFv!PPLqqS&
zbA+`{rP6pNSxb?X<x49yyjtTpdL`U2i4xm)fu7nVC4KkN`&b2pN;K+<zW#_3JI-KM
z!T(W+#`wmbj{>H3ayg(rC@23p+;CTLU><)>_8fBsPsl=1rvieXN3CHh=D3;ZRF9K$
zejHvWfl2FDQC8487P!TY4ci06pN82h=+gjNmp?ONELyHa-hGKz_mAw5_);kKlteFD
ztu%5Rou?U1yes)2x`T<uBTn1&c#l>3Ll~9YGy<z*r^Glz_dQRTFs^rUhHm%z{}NLA
z_0-Acw)ifu;U?n5BLC0<nT)_Z3$S6DQQLZYVu#orqG9?;{mw?%@a`hq5ExadnoG{M
zH_>xnG3l4!?k1AfL>6+p{|QEYkWyTicdJees_EqWd^S$5YRrZsL$SYYtnH!aMws>5
zTKfk&f2s$E!>d?D4O`|IgiFDNc69_he51uuUTrjREShDY3DwdEp-=bP`>rpN$PveA
z{Jhk2utDDZQ-`U$y<ye(LGjm?_Z}Y!bAY0~`s40i0jrIW1WGajYq(Je(X@h(o;w}a
ztuiwBo;{sL#0&?18>&w9wW}&UteM^zBSPkQN*9bOJkz#1N?2~|TmuDfCaV0;upcMh
zp4PlDsiS%kgs{a=RQoVJxY)mG>9aGo&pIu3M|N2J262LJ$C8|b`@gCTGeln(pZh~&
zDU2R+wbys;uZ@lgkSi0PJNHHTohr5~oH@w-eCBA}C<GaOP6YF(8ZEeqO(IT>Hi-TS
z-#;idOK02o!zVMK-cBjg;Qg-Pf+;G<yKud@ldxajU1>wWoTZ9!)tdg%j&9)9DEs#^
zz^YKH$UNPX3FmdVWc<!#z;D-dJ-npc`WJ1NpMNRaa)j(~wims>w&iYjy-QojYc?0~
zRMMvRJ^zvH+qRp(Ji0DpbGe501^>vD$Dd>lE{`Qf0z_t32xRkPX*u(=v)8z@oEITY
zRo`@-;&zn^i`*S|Kkw7^VrNRmLYUm%<te)guhl#0Z_@vbIW09|T6mp3j{g<G@$OeG
z24ZCsb?V){2K$6i5U9ZA3m&Vu2YLUcxQD^3oQ%kMJ{q|#UaRR}d{CPc?GmBJqyMr!
z`@3B2;&@~wvkrw;|57unx9#_GkY36%DQ4?Mzgd8F86#f@ejfSZ5xp}jMr!A4MXRD=
z3S3jr7OQnlqBKDTQ`>D@^-5Svmm(5@SKj1g0(B`lys6~7Tw}ao(fmmv7Be4j38Nbo
zp+PzDnX}E8x0-0i24GK^xJI4SFx7n<BW+pVnx3g*DFyt~;~(8a_>3Y+dDy@qLENOp
z4`UF{`LFBb`=1eO$tq(jW7@K!_J^4;)j{R_p!z2ee4@PzcDLd)0{k^MCeJ_#8^QAD
zn%*mc|1qHilX>fs=6kE(ioBzx6>E9XBv3iTDgLHa*!p(;+2dGtN+oHnSg<pqq8yB>
zDL+f_3(=5pm2pdda88XnfAt2EDGib*-|pm2@`GgRck~YgfpmnB1BKff%wS`h%RBs}
zFL$^;P$G6#Kd<cejV{l(O8{M6gu16RYRNqER@L5)#U1Sbfo=nm#qgDIX<LZBq&U8^
zz#na6<SSIrLY(R9khcdZ8MctuCY6ipoC(OG;=+1iXDy7vTW3Wb`n_xu^PGM{LJ!{U
zMTbaHO+G82>MU%1{Lh_8)TEZoefBFW=iQB8vn$lCaPv#YbO`MFgL4IMj?FGU%VGZ-
zOt_`M-=`lO<b%}2oJW8;Y3T#;Pp3H_oG<48+NV*1^%)s?ES&eTHjnl_x6K@Q=SQLX
z236<^ot;2?jnHxbe5elBYyhHtAqmDd_)n5(-`I>E(FVyDsyqZNL^GCJ5NOStBl;Cm
zec#PA{J(*Mvmx1B)wWQDp_R0`OtbD$^zQPv*jtYIP(ubi<P%`;jq<EVL*k8V#4S6T
zl0(@(?v7mP3S0hm;epZK-w;%yWa#dD-OAnWPXkA#)ujsn#_FL^t6A#2dUpudCV@c(
zlk{oa>s2Rz3`+LvEzRp-n(G<Q6MmOF63cD&A~$~z7|o0I`E+TQBZjy=!!sL#U!!Q7
z767L+m-QkmxckJ`d<A&X&1Ppmi09N<OdwpqPC8Z{US!dS?+E-?CtE!)cGDpvm=3F@
ztPD=ByQS%n5u5%C&v?@SQ?M6q02kmxA0SIJqFQe<=wR0V_ayopnGriL_84OmdawYn
z^Xd`=C*ORGQ4b%NkQJ-Lbyexa4zmerWyoKuNf3V`^YDjwf{<f0f8WB_1vxq)cSDE+
z6T)z0G3I|rH<1%oveEZ22l1asqPQPpQo06--wTO0hX%=;7jyCN900D}luto5bcpOy
z)z6<FAoUM*EalDZ+ta<yBB-Lc-%q1I{pv~Ak=O7TX!WEB@jF0T9O~rCn>&_p@na4E
zj5p=E$`QkU(a$~Bd2wBW$osCFG4EM5^ia#dJ9oWh%rGzRs4y;w`kz0)7ygd93MIa&
zjG6-oYPngRMU11Z0v~c1(uWa4oARUm1pvIF(;sp)d7dQVrIua9R*29DUn6(HF91xa
zmI(bun>vjXbZ(huM|p^^cevaSA9y}JE|}ZDc)g$RoLcdO*d<D=-<`oBM^$r%*&ff^
z%jI+3F07VN5_=yKHU1|F2sWN<;bnTMkUF3XpW1s+Ygz=gCCi*FeW>u;v3zyTZjsoA
z?<Z%jjj29<6K@;v=f?vKLm%(cA3CiU7%LEW<fCYWp9}Tw?r3mh9S!p5>Hh@TaOmUS
zJGC8I^!5oBzP8|s5PKr(HzpB-*ke+VD^n}oU~6_Znbx%R^^~)a7-))F=v{vL=VN62
zMe^;1;GXQui`RmBzeBwFA;+g-=Vq}Ru}j0?vAAf&{aY@dcYJAe^!=TQw?Vvmn=~7q
zODlxy@rj4m%422d4TwN|K|uGVYx#Dqow`={O~<|<=KbgR64M9JHTsP8G&d-wLvfd5
zeQ!to!%YLTyr8p9#AYW7tvgD)912hl-0HBrJlovai9!z=rd?gfCon<^UKJ<4-|+s4
zbF1pcxht#o5Ds8UV~5~-wUnZ?OUGG%v4F8J5VJ=;H9P**ydT(Y9bx)?iFuUL<-bom
z_SdhIrCAWnovJBmJkpH%$Vv&33$&BtF-F;inz4P#Sv~0Q7eb{j@hs~E^yM$5GD7q?
zWw++-YF)BmixN-Bi^ZGQZDlrDF|IrV4S?wUDChRRb`Z*WJ+h>|v43GiG;sMs{TD=U
z`*$N~FESC^f(#~m6EluabotencdP6M%QcMqB-#eG99iLAi~2?#ztdj!Y)%&xgC}MR
z&dfqfAv}JI1g>yu$Gy)V+aiU*I@161_kzSO2|@QMiIKMBQpWsYM|zj;jF?Q3WW>+)
zE?$sFU!7|=RXJNbs7=<2)=0d&e_rN~$*59USC~nv)oi7h-4PS$corLAS(qx$_@J~|
z?~*<wecTxGq5)WfP5VZFo$?OdLp<YQrKF6<nBf$A##KptAAuU|^#1m_LvY`pjjfy{
zxz^m;R9nc(=Q}6tU9<7IT}?H=ODaQBp9g(&#vgJyp02Xn5BL=B`9rGv%3`BR5IEDo
zIXR7W<lUfN(MWqTvq%lxsIHQ3?~ho5r)l-)7_v$Iql->wty)i!gWb@KOK?Dl6zPL=
zFNG`3(6VK=HqH$_RAmtLh1=?gSnglN=MT<3WJ`DbA#mJ7nqou(MKCdI-dp$h_zn|t
zJz-rHCY=&L#vwSMh|AwlZs<VkFa!sD+@j<K=ZAU!r3pt-+s^Cf1lWDLi<^rNF)};q
zSwM&4h`A9TB(g`s`;-ZSi~=zYHRe_iF^)by8awaQ#`NQP2XT*JJiAIMV(^Mn|4R!J
z#{Wie^sDVt@}U?_NKMnk#T@2n-vdlao3V4oixRXvL`}H9A>KclwiG!j^)$$jtO2px
zJXH~c{%HU{<@D<>_`j~Y&#U$FOI7lF7lQ0ZWhD4H^}=a0&}q|LVvp@HMgq_MBfq+F
zCpeoAlC&GY;7s>Ol-Ky~FD8a9s)lOa4iqlG$4Ft(P7`G??OB!x9Jo7nr6jz~uW;Wn
z2H_cM^PiI+Q;-#$@q&d1FuXY!q9P9pO9UisWnQ(%gYuObYHQH@Zg6N2&xB-LeAYa3
zpk)3J)H9Wb!oaoNY2nwMzqj{#bwY835#{TLk?VCYK+VZ!&uX>X=fdb|3goYay=T`l
z`C6^}*d`G_)@rrEZz_h>>J3PE^1sX0NH708G&ZB-=w6>RYzPZLq%@+qC56Nt&r)yb
zEubX#L?#n}4xD<u#e2W;tU9&2iq~xOC5P6-<?E2vyNILACf@@w)IK@qKQaFT3~U_<
zl4ZU*h)?7NcHbcP`*XgjM~Sj*SF#ug`k4><A;!>cZvNIHVXTv!LC3#>78>qEjZ!3B
zc!-kv2eQ}KIygjXqwTjDdcx8{`FxqKYV9_&85<%oL@B4_%yOz)*+u_s$-px*4B->S
ziQfQHX{UY`4L5u<;`ZNNso73bwXakd3fx%11@qp+6&oxRK1l(nI!Qm|d1T?t)DyNO
z68mA=iEAwM7U@|MTBCVG#7to*>mPU0e<rXH1EE623>byuQ&Z|EY_d1ReRO=6gCCM+
z#T+!!u9X(<%%>7BK`)O62m72{J3yme9OziZmV-t%m6Wnmr@EIWZzRpM&h5djr?t^v
zX(cR8BHuT)>w7J~<W=aLQ5L;#-2IaRKvO6mq?ya-8QLh4%!Hm7#mBeoSa@;X$gg8(
zGV1AF$*X7S5h~W(=>@-@W5Th%6%t}`3%+w>zojQAa;0~3A+nzG&K(hTW`jfE*}SNv
z!`HE*mdS-R`N5qh`%lQb{O;ivejG=szI+OFdNMl8vPR(P#}fK$f#ef~ufB(-`e+#?
z;6jw)C^Zp=aS1&h3T%XX5y4S%vu5|>L`>zZnPyjnVQbzz%jrIolgawqI62Bkn7JtC
zW}q61a{f|*dorfd4wm+`U*ebW?)*KOrzOu*s)uUMns92rpwGkoW+5eP)=n#e<GzbS
zM=XBA)iq-m!zx_CMy~2+lVQ!)$mcd3)jRRRDBsOH^=hUe)kCR7+OD=d&ng|Xl3wMB
z)$S9n7|wS<*zG;uoeLiqkJo62UFA@xZ*lC>U|V#l+R6t@`K<qmw4U9x!1}1j*66yn
zu=_Lh<@G)}l`QU;m%%hDR>g8hD$?LYR7RqD(BLQ^sp?nv9WsOueXxCVafNT*(LcUW
z=$VSCly++uIX=j@2f(HmH+Q5MhkN+Y4$mJo-$#QkUBULSl@F;N8nCGr;7RTjO{Y-Z
zY^|(~@Xco$n6)_z;ZX6ex%k699kx^-Gl6Qe*l$r}N||P*{qj>MMzA`Chu&gLKf7SM
ztI{tU@CC1biBHlOKk}hJ*`94!sn~U#UFeL6zQ;z*r2*36Zhn{nTmF-#Y^jG$D};**
zQ1TL+=B>SJvmSw%deG*Bb8p)5iGD5VT%k~KO!b>y;TP%aQsOO5W=$DaEMY*%49moD
z=XNe+$Jy!tm&Kzav-2fT>>6=@;IV@MGetdmH*buw1k2Y8&v2S?e0R?pti3k!8bSg-
zHs?M#;gpwInl)ZBD_sy^t_;llAeq`-HPmGfn;{>ZDo4D2aDWVzMu{kXitUCAjN%j5
z>}j1_XR16hbh&_~M^nwa4jChqcHftmzY<<tA?;DWcEwmzoS6;bzduVTz$LBhOBZb8
z4J9+Yb6<?ZR?Uy*KBm9VP*$wE?_nHyK7NQ4lPdW5+YIcnLOAP@Rw$>l=8#bm<G*VR
z-_U8yJT4+@mdIR};`>(ATS+@uLv}qSdh!>oo{O}d<$QP50@6}<O5&G9U8|=%1nRr0
z>$izeZ2`8U_rq~yt$EaoO>3YBaCW+tnZu4Rwe>UCMQJ3<NTu1<2xqA_Q`i-?9wHcw
zpG*}n(X6D<6RAZcr+=I73T!&HY|zHc)_h|sO>>`e$E_~C^jKnSA^xcE!brUJxv8Dw
z5BDBf4D&*#eS!u(H<0RbN^s7?`?gKEjfdHW`62rjy2&&L_PWZa^OiNX@`tMl5rmT;
zFWG6e<uJ{0<fz%ShL-^jYe+7T&<<z5JvKsWed`pOqZOApugB>U+gx~PpCJ668`w|$
zxCAp0nN&w^pYdJX^)8@6DvkiEE1Bq=jL-V=R|q823GZ+5VtZXu`6esT5NB~>?YJ^)
zQr@*kLJIw-|E2<QZOT0VDXH190I+x#7WD~XPd1wt0qR19kMaB;2>?lZeSxfn_=|78
zdruyUUV`dir5B-{WN&JOF1;u|!Yjl|XBXQFdVf2!2IYI#Me~0+ksHMy;!L-14z~cW
z04#_}B0PY&Hg=SFEO(a?t2DINm^1#m<$nVB4Ae`=?(=Z?LG0Vsfn5uh>&FX77^)dK
zuLlP|g`jRXJ}JaLZldQW26f`WcZ1&Mr^&}@h{w`7J}i{u5=QMIZ~<{`%$hp&W&=>o
z!d;*W6*Wu{`*EnzsH}~dBmZcP)$+B@xWL3T;m!+F@j(sqcT9N-5U*AL5oEnSo#Al_
zPu0gpERQak(b;4e=G=R`YFQ-0z$a25Pjm}Q9{U9L@v?Jo*vF3+*od1R_JMm{HnS$4
z0XDx$9irWMQ<AatDXzPObotyNi$Ox<HEzzWNimo$v&+B)#W?e->R=+Ofs}~Ej*`*t
zK|@}_h#s-RiGw~4>cA%AT#SfC?Ec560I?tFfsH^nd2Z(&4P&}Dke}4ZS~#IU3pEST
zrm-vFj7H^5D-){kn7M0!{WL=U8~{V1ibt4H`a#gCc*_Q5cRpN5@fLXjLY88^`FFtq
z&7UNcA3d?5OfywP(ZCkWv_m+9xx)S6bFGaWLzt=f7vB~pba{zbsBjY7CJNsq;JRGx
z(?MMqyQiS%wj(x%E?<g>v7oNK0hiykX%PWnDV&tZT>yY&;%ouVqhrt$Hfw{#x|wm!
z9X%@@Ssyfej4(~IyEyy5d1=6PL2^@?@rd9Spa!R_4$B*e6+GM)aNuX*#NcZa`yv4o
z^5*H(O-!fB>i&C$=BWi{RrtmY861+?e6H1Vdd1x_!yZeiPkWVzBZKuCw@VZT+*iQa
zn{N(i3Q<wP8Vb;2*@*cg*E#Gz8$?t&8-NK53h=4Tdo%cJwek850$YS7S+^}x@z@w&
z`i8LDcoH%)Z7Udi*L=|G?h_(SnKIGt^Y8m5H_OFuoFF}2$Zd(OMdGEO4G4wQ(#yg#
zaz?ddk5Ydr(F25xONuk0ti7&2OEsteKq0<(c*Kk2Ftu~<P{u3TBc%2c;Vdfq7%BK<
zntbN$l49eJJKe(?MFT>wUvKB)!4Iy!nI+N2D^lCa1Gfm?>MzHo;hW+GeocD?h_;~5
zEx?~hQ%d;WeSHoqK$3lGtLMSht!wLk$d^|RaZc>uVmaSyG}tx+FDhDc&&c2B*S;>h
z-PQUibOv#kAp;*9?NHUo4B(fC@RmJ+7NX&yG8JzlKc>ZLowfr=lGZ5_s64;4wg%S0
z09z7$idusm2YkAIW(<4+_t>N&5syH}*iBLD)stZ5MtISX%OazrfqFlFx&Qy+Ml~rB
zqsVzMXvKlX0N>`)(cVRaV!IN8y3uV_x_)Il=QO=L2i_mbP$%`1Xlr8Z{Tq=x?l}AI
z5J@~G_fJIJM+DhC$E0jZzlcFxZE@RPw&%2uxQ)uL#vntgzO}POweq0&2kGy6d7v!$
zPTz%-d?_qJEE^%<-mlK~CswHucPG(**9pofSW7lS<Jkb|Kpu)$4TS0lcokHurG+{P
zOzV<Ctxs>^75*v=jd->MGGbQ7Dp*n$<_zmy+s^eTz?#vA0t3WPtzYbAaku?FnxFGR
z6BRs0@?c8MhSB;$FrMI=p&Y~k8H%B8Ym`K8zOt9&LrcEsCEo6ZDN~CYNM{_~8J5MS
znO^t+w-v~jQ7T@W;!ms-_*c(k=%R7^=Cix0poDjy5rKngmSG`!i!`KetC!6WxLSUo
z;>}WlJg6SEYrGF031|`v|6X1CmrS$&jlik(EXh=*S!!z&6@X2mL;X1Nz}eGh1+OY<
zE^_xomuO-|4H+@J_NT%F3AQ;4qI_7&oy7Xy8TlLFy_5W&G18*lv3p^vc&vTRy`J8<
zaqPplIl$s_wioMUoKy7dYe&;cdLXq--T|bk^<8X4S_?3s4o2Ci&IUb>ybJ_xARYwD
z8+=%$2QBsWR`MEL_B8uK=<T!KwUPmkNzFv#t{+Rue7_*PZX5bRbhV*xj&SWXd@3X<
z4=g^i?Gg07eO^W(?%!dn8B!)kuHQsQ={5ryD%C(9fn1XkM{9Q68NLwqm^pq_VQluK
zglcVKD=q}O!7Td)5&DXc536<GDuXEj#mH4Io`jO6aNyvj!}cZM_N4lMS0r&2Al!pb
zF*c+Q8QI!TFp-FcYs4cc75FJOke9fAZ-){crZ>v5^)hoN$l$hH@ehx;n?f438$BoR
zLkeH8Lx#`N9KPdrNxoF^{#)dGcL>O$KafJ<nUY}?Rs)qWP4thH+rE&Q1mw|FDr+X5
z(b0rr#oafM@<>~FgzUxv!+Wb6wx^uDyb3hr<>2u>a)%79g!j1oWlCKK$6nsqW2S6;
z?$EFkyo`MVnGVW|@w@HyrdFl|et3JdGl3z8!s@7H_~f&YLk3$yIME+27ajHo*(edW
zfu8%6w~+3>w}y@@ptbbJ3I}v!(YPfrEp-Qo!S+*s@_qhCnC3uYwHKxMO&W)d<h^t1
zVS+{Ob-F?3j#xnC(b9N0{M5tDw%6Z+Mom?WhWwu0&*U9>(#{RU+TaL+=tV>OQeI2l
zvsdt>-b7e=KvLxRUQ&2KVg=(LGmr)Q-R_qVULZ4OxOo#p+M#o=<gb&Nt@`pWfpuK3
z%ahOPDe)mKDUUzRA&Z_pRa!@0+G9=3ZDLnskXzypY-w%nJmyWNYHG5a%A!o*-8UP)
zhoc=mXmkVzz9s00AReX6ErR%YZ!tbT>>Q*?)VV2o_iymxdCrGT1(Y2w<B;|^+oN?|
z=D~=cnLTa?XVSWP?DWWixs~Sr$G|g^0N*?7pV^%FcIISOvfeDMW#x#z@qmmA2CzL-
z;=U%?$8`TAt5ev_vVfEIFWtr=SI9&nqy`k%>BGB@Bk=IN%+WrthTco-u&IFU%bfQk
z84`IK<kD+~T3R(uo4Hm0Hu4*BT}x0%fz%9XE;l6~QAdw7CmvNo(}XqCb$^1Svjc1(
zGF%?5Z)W2&dFPC&wMgy)*7d+u24^A#0bIqn$AfG!H4M4}&M2m*twYFIbKuZ!g#Yb=
z*d|@(YHR+8;c!2ag6vKXKY>L@4*j$Jg3I-TPek2WcaAgz{eSc0CZ6~TQDz5Z5U5<B
z^%BLifI0;#M~>@GtCj{Bbs4_4gfQf!#HWM#N9Ssm{4NCnpeglk>9-u~(-F&A&Kp`D
zodaoe0K>ODto{GwsyD{6VEsJR>UBdO96h?4khm3wRUO1)R2G0Y7qri<Dh2c)rG~tR
zr6B?hYip~~Zd;!_18pI5M?9WA{W|Q|QE;Y05-Os)r@vNp{%7UoxXun|^fTj~_70b*
zl*M{znv>?ax_HDtWTfsUzXT=B0&oi2k>I>;O5nys3P@I%l;`Wwj{8_S+&2HfRQo*t
z#wWQ(-&YwaqO49Hb|~+R<uzdt4sNS`{`Ya}>dzO1pkx;`@Te(pN7N~Ucim2~i_qV~
zsS4Ws5}&EtdNHVg4^sI6-;l*_F?U`Fhd(4EKNS4w@I9OKzPgarY1;kVp(-G|=h=A&
zVn1(xWM4VqmFe|oktvAN$HKWFH1MIlJ|+?n9A`&Lr5dsnqCClTVWJuDMWQm3s1)L3
zm7vV9;My2HHif1T&-~tw;HBbW#R|u92rEtcb!^pC`R8<KQSL;4G-$2&82Qq#K@3~^
z8?yjfAp|6IJ>(S`LueO`eT<jTcT!mqr`h7;by5498zFP*V0)mtaD|VynssCB{?AH#
zC}3#gn>zT`4+K5yuTtS`@l0T3CcG$IVgwnWL86|qtMwp&ftq|4xutJJF99V>qkCsQ
zwIvZlrjn;d*$NpJo4I2R%aTvRoRy9tmO>Kz`(XjK&NjRzQ%(guVKX9K9Qo%#=?KS+
zoBsggEuM>~J?9|_CZV{wSp#Y<dITnF+WJ){-tsTd5hnVyEV>eg<2d7)Q|$7#lyV{W
z;rLE|KUDM@klZeIgVHM8vhbf~7W((_W3)h<Rgw<pX4@9wSs#oy$3CVw`w83)NlzJ^
zqPP3Q&h9=iop~Qqk0B2m<9x_>Ly4-5(J2O2dgi&gEtbp^cEBIZ+&RKPN!#_9E(3>X
zFcPne4=CQ+qwTVOq5abi;{JiMa^HJ@?jvGFBHPk8R%l>C0=Q;QlYUWbNh4fLoI*^p
zH8E{1L;Ni=kA3yjvL$*pb+{so<pt|({3GY@BO~!h@o_}rYsFV-8StV{=ze7EJ3o^{
zJo??Z=yd>aj7OmPOb($gwyMXY!A#y_c|jZ%ie5q%6unjYa=YvIM_f$Qk_P8R&nMIv
zEjBNaR>g(=i(5k}q7d>>MUU_f(L-RrD$#F)Z}_yaGu}{XV|UN^KwGIULF5&^zg~pu
zd1o<l%*YeMPY-#TwY|uH_rh%DAkDhGIyo75uHW}*-gArY2b^cKN!Wj1OScQY;X&kM
zq~4a^-~7~DqCoqu+UPA?<2o+)BP^9|MfY;Vwv#L5?!|4HfcJB-sS3g6wKXG>7bFV@
zP2q<WaO`yo*+s3VN+zrV`+ZD`SWLhkTgcX1-9IrfIsZ4jNxTY`E|%d7Vg&cTfqT2R
zuC#f$3=|G($lk`cb<n>a9H`rp_cOU=X!2k!po<X5yv4S<`j&&&J<?=7G^^7z|Ax2B
z81C%WX>xYJXUwOHVUgsu%SCoW0X*XnQv6EMIyS*rKePbTZ$TTPzSxEEOt|<kdH3$U
zwkXUO$yO`Rq~ok3_9tKr4WWFO^5thzAw+w|<NQ%)qr7RSOzl64CYl;$*9pUS?*T=U
z|N8704`wDD6PDNb@RsvA?k~8XwxVVMPP)J48?GBBt_x}iTnyh+UX(YzezZ-0@1A+i
zPbb|%O_^TXf%7ZZBaPGiBjwx@M9>Iww9qmct||0bSE$%054$V@kIs@97)TRgTGve*
zetf(CT<Ii$dir$wq*XU@w{HUL<&Y-3kFha+5Y-xiInYgVgb@;pM95?$bE`1fYXNQh
zm;^#3Q1^XHxg1VydspjheD!7hNTwsix?96?5vlP<ZfIa20&^x7(`K&YapZ$#sq^GY
zH#aya%|$EkOphaRE9$4K9d(@zA8({?Ad7Pd={jxH@dK)u0acSYnM=n;QRCk1?0}#y
zJ4*#?V#i>~LSloqCpFB)E%ZvFs49TygI+V&aBm`?mu$v(&jOBfFy=woSdY_?mwJzC
zUuNNReNXwm4fgp>$vlP;msO@ML|0%jh8EXl0~xwP2lh{(c?Nb<>5GWVIAuy8nMU5h
z2m|+zdfT05VwtGu5>Z_-OtAJfB(`Wd{c{?3@Ey?g0B+o4|H!vcFX}&mE|wqgeI<JS
z@^+U0d>$(?%)px-wc&|PmOS(RI8#0=prEyD#>*=H?r0n#iBZ<jed|?vdCX5mn{F*l
z)gS3+F%co6T7GcysuHp%;OD9^R|vG{xcK~B_Lyf4Ef{^1Rv;Wu4t2os9<{)K8=m)e
zZLjeA_di6c|6qq<G(W_T5B(8+t}*KhDXX14;IQ`aOE_(7p#A+TSJzhE{?TEMmUJj)
z_qN11!k&{XEjKaNHDmicFPylrL!G2EK7Ql_j61Hf%oHk0&sfstp)?xJ6KiNcfSy4&
zcX>m8@OZwD`uIwebKoGO1o1^MW>@P{2k;zIU>L?qC)mKkQW>wY&HL88wt<#|aliN|
zGl}1hV`|{G#LB}YdroW9t>fEqHmU)JwhL(`yvMB|ZHXFCy2Na}!Zw{X8h8Wq5B&+W
zg%HiY7D576Xb!GQ{dsec{L0cX!+_|v#(zFl18nuPhF2tl_|76q5I-@Fsj|4$wd81~
zHxi}Mb|~?mLfofRf9<7@uUq>)$G~f>H2*$|B8Ixu<h~DMhw#u%1$Mro2)i?_!%te}
gstx4`$hA!e{wAHWj|;?qrX+c!@l?G`)gt)+1Jwj73jhEB

diff --git a/modules/web-console/frontend/public/images/domains.png b/modules/web-console/frontend/public/images/domains.png
index 41c047066ee41b8352d246a47da3dd31688ddae5..f22b04d50820dcd519485c24a134ea862cf84b34 100644
GIT binary patch
literal 17899
zcmbrlWl&r}*Di{?I|BrFhk*pQ!QI^@LvVMO;4TRkU~o%-!6gtJ0-3?xL(l-h{qla_
zsk-;nt#i+*x<B^X-Mx12UaM>Er&m`$J6=me5f=-9g@Ay7tE>dlK|nwTBOo9#VIaS@
zm@(hhy;h#I)b!+EUS96*?w+2W1_uXkZf@4r)~>Fu_V)HpPEN$d#ee<!wY0Q!c6R3D
z?JX-Q0g{t@c{xIHXuf}Vy1acjeR=u)@HqMK99+Gan3xzE93mm1GBGjn@^Tpz5U{zq
z84(fD(b*v(Zg6;T(Di*+OGn4p*jPnb<>h7d;qGE*cPBPB_Lg4=EH2)2`gHN{qPMr#
z@xupO$jb7|i@IK0Zf;I^%*4#>?91iO{QUgg%S%~#*~5H6_3BIdkCXVIqQ6iNarM}z
z`)S{dA20i@x4i*h2Dc{TwKb%qGW}EU?pDjImXp)r#-CurBV*My)r83#X=!OB5wpR`
z%>$`E<L58+*=^#WAm`=RNjOTMf8cf-cXaEoiwy1Xld-k6NzoM_3|G=O>+DPkC@d@#
zm$Yg2)K2_$r*EKdE+Ia&ed*@gD=y=Ha(}nmJE#~kXkpj0)$p+pcK32N?ELir3WZ8}
zK%dv=Mn8u&?L6M^uQfC@xVg9{bZm{qcID;e6<dfcUOqmIje`6-atmh7&CUDn9t_6r
zBtA5p4`-KcJvRLMGg#iU|7+%Mzo9TD1Xk$R`0&!wzG>~#)jaoi_xI)FW<^zaj7;ah
zr?c6ukxp1$hCsu4VASV_<3rK-p_kp-l9H08>TnrlUFxEPgqXH(8u=MVFP@&B9br~$
zpG?6ZaX&e69IwHh?v$s;(Xf)C#->I?b-9|8=dG?Kon)A#!nc#@s@+;}?R)9?+Ijxy
z9}4Qt@t-Q!`pW%+;^rX}n$`gzo%i8G4?E{~qdyNW*Loj+#vT2=)embA)0DV9m{4#U
z2oI}%Oi8=<@_oGMlr+s4ha@#*M=x%jTX-hkn^}6dok6Bf@>4sLgS>t1bnavo){m~M
zr84i<8w9@W6bIR~wzZ~O%AKw+6}0^5b*X=<te%-WmPsFTj;-Mms&+C^{=0ZE*j)=R
z9iVn<5w|YOX<hOO8j(yIPIA$2HG@E_Q@mzx7MjN&Lc&;sdbyq2)!izJcw%e9N))8i
z;0Oq^)yg0lJ^$t7nG)O|S|l-BZvuockv<NT62@>#cZW+$%N`mbj399JlYPTAwJLIK
z08c}CI}Tow-8G(75EK-_;fi2sP&Ar<G#Uu>e^42cHDz{UVh;K_yRzpQT2tS8VzwkH
zl-Jbl5!Cow1|qif#6qI4+yyaiz2)E}Tt4cV&+>^e_@gWNy%r4w4)dMj1r4)dZLYDV
zc>b{l90mzI3ner$zgdiHoFU-_=cB(6b=7A7<K;AMl`msb=62hQF>h2@;jwobe)&ue
zvde&GLD^M`uKPvar%c=h3m^T$yPt|`xe>creJBhH-RQYlRV4*UPiWMRMxZe=$haR)
zlRRGR|9mG&(Sd_+Qi|8{Ov~hWgrCwqpBla=v|YT*HE25dH8F{?ZG~8<B&h-eES5w@
ziV}te4f=C_@In`c13cG1z?%If@SP!$dgkGoc;rP@M;?|zHt;7bO$s+pB=F6fd`mJ)
z#rq#EA2~<h*>o@>@^RsZ{n1>BW%y_5ZJScOK$TjrRv3-o6RpmYVpBx>6~o6X4T_xU
zM6)tjh>^6a8^xfaDr6XcSw?5*qs8+RnM-tOX#kn;(7=YMaB)j*8GVyQYYKG9y(I?#
zp;A4)Xt>E`y!HG%V~-s}>M|A1Au5syF%4yL(ZoD~GYY_0Jxi$1|EZdDKbAizl)s!h
z6ijf7+D1exr)@Xlf5UcLuCh#jBp#TSS=(NimCR`X{WG*>AprhG<<d=H>Y>%K=)Kzg
zDaU_;|HT4n1fvN$;kENFEHDbw{*s9T3aUi?<hmn*zT8vk#Jzi?0c7w=<1Iz&YZ7GV
zMb+AnEho?Gb5e_Vo30S7Oh{)_0v7<6M~p-As*vh|2k&BkTwEaNdLj7Pj$?#{j;wGk
zSxRa-;$h2sThX|BUhCUZd}-Fr+EjM?Aq?Q=@)WG%_!nicKVBq{b^6uD0DtlEmfHWg
zG=&pkiCS{mS*fuD$Gyt$l+M5TW*$J+@DlMLHoGA&wez?e0Pm^z$L}}v4|2W8S3Gx1
za#vot*WEIYpEtcz(R5ox<&;b{LUb>Y1t*mP183daCcO)q@qjPWjd%mU(a^o^JbqsK
z%B1#7AC`XjoSg#{JSyoB&g+wAgiRZ=@d=I`52IoMb5eZxpp@Uw)YtLsqX98{tSlZZ
zsOj>oKfAqPs|-HgR@yHsjRu8RLJH>xK2m~<RYUItd*l~6>JAyCz?XrK7C$)-33Fxx
zOYyn}TYQI%_``Nn&UtAve{P9;TiJn0I}XgL?)?$44}?k-9zG>4BLSH4QKA$MjKB6*
zwujJsXAK;A^#@WTWU7YQBM>pdUocU_D0<{?MU_GnGRAK85iwbIWYsm&@6!G_`oYC4
z8B5T~eUp$P6|P>vcAc3+D3dtRi|Rrz!Nh5#N!hQ^Gm>rjP9)&8HY5R$>tJmt;viMu
z7Bps1so<hcn=u68`4=_+lh~5hm42<%C5RBS<YrrPRYN;bnPmkGD?>L!KFZ{A<R)3u
z5U2?W&Pqx^N0Omhu`G+%;!NltymynH32Y)Pk}?8g%G8$bdjY>9hLyy5Qe8>Izz8D$
zj59y#)OdU{7sd|oW~PkHalfp7^E6uwAMg$Jcba<sBp_St0V$NW=9*w7qcK_}Y^#|~
z_8{el+hkQSpymz4u8<QG!9siqB}%8?R94A&eYbDW(qv1Ia40jyneaKW<>$BxE{^wa
z_dH|w471~dN@cwXA^tFAhG8Nk&r5?zn<o{D0!@@}h?N`Da~Hy#*C;wd0Vb&sa?BNv
zCDE5Ay@TCXp`p8`?jQ7(VnqLF*~uz~C5ymX@ueT(wI7G$d9ct<7&_$Dlz4Y1%h#-B
zb~NL>s)E!V)E?ULK<;Qb;)+gCt(QUY5Y(R$0a&h>Xu%tH!XkkyQ(FOstr|UIfX&pU
z#n=H!iuQj6kO1{+l?J;F_&`i?LKi!fVr!%+q(T1X3M#LxPZCs~2x(!zvV-OOi<K7!
zqq9`Jwj&9{_bf7x9cdxjnt=sht_r=Nh#&<{mG6<#x`9~{)F=olgddBa${y8=tH<Ji
zvq{%6X&b_jlY0PgaPMS>#DPHH_Cw3(_Nzu`&H&{wIDSMc!Gi7g_X;z`V>yOQ1V9ZU
ziTMO9;0b>D$iIIl=!xv31keu&w68M;m?4a+Xfq5c*zpVkPLpFULp<V+DQ`q;l29Tm
zy67UZDWj^TG>RAYhgZ>@{!vJp*Lab0>7_%<WHOHvvBBdr`x46-(lbH8-?qw~aVU{2
z{v1O%2FaqDY+>LiTh1xvitbnfC`SS<aO_r_D;Wbi7^qC)`r~6pHsq5R<y%52Fpt~x
z$3HUTqu+{^jq`}`l3LjIhogz`oDZs*r1jE+GVN-R0AwDicA_bO@d{n5t~y)YZfj7n
zvZuZkTvBrB_o~!!M2o>O3Xn+|@akYL0o_3@VWffldDQSm@6g*>5deOssW)oWu%gx5
zU5q5O?q`%Dfc_L*6b%9Y$Iou`R0Gs6kb0Xrnq?(E@aJdMeB4Qi^`=kkE=}2eKi*te
z-{8%C3LhQ`yi#<ZUX2Ddp_)g-ReS^%(b4KX-z0x;85soyZ<Eb5#?#vGQz!3fT!-(d
z3IqisixB{QQYu1zD!gq~XtWF>4@TpD@_nCw9#KJZXE3joUjT6oW0PPKUr&%9*{|*2
zp;%{WNVs)7ZOg;KlLA@UE=3H_P=i*3i7@hdDW_M4-pc(m7w=i+_{WWtpf3fkJ49W@
z!9-Ih)Akm{edCjX4}Q=w;zXwi=xqGjoRU`FWYSoiu<x7Toht*OqMC)sb5n2Yavz*y
zM)p!NfX=<qm56y)BV}gZ%0W}pZ4OniiWQ_Yh3YOydmiSi41n)%%X<#qQ7qG{q`Jl1
zalW-*eHSZm&rmE<y_M+cy~gKA1NQde&-Qt*-(NfyBUiw(Oa9Zle1Q<0wM7N(d#M@6
z2P|1gfF(NftCDc=0iomQ%3n2`QZ&8hAzdMz{OEa~+q$^yZDdH5{m66Y$KJU7{cwyb
zVMF|6f28~)qi~0`Gaif%(9vs+S>BQnq_J%9{y=--#@^Q!j}|#B*_T~ukkyrP>iCKT
z(4xn}C(a#;DMtwJR+#-JdCQIs?D=n4K|*m1Ir$6f-Jm4R3Lg%N48hTNr;%s<z<vZf
ze3FwdU}}OBW&k=UD1I0bZc*wX4ZLLlki?^@0@;XgbHG|m04{j3640nrCLR0@6vY6y
zibs<#?I*%b07WtXA3T^EB3cnafe}g9-hbpwjb`<HwthEAm83UEAxeo0n8Y`_N%m!`
zzfSf=GxFV|v04(lL74a(fM#_4Q>d?L0!)yEmYjq(28NA-)BgVy!+a;DD(X2Bb_|7m
zbj3^;BCV*lpWVTB!^4Xb=xPZVsug8M(OW;sfQfW!pIxPY(4saUM$Z~E6e61mW1K5G
ztv&ap@kMGdY!j`Sq5KpeQxu5z0ILlnJ#XD~``ypC=`<3PS4fN<1-+(kUQ;xGn!Qd#
zlC$1{J>n;>*gRm{M7c^G2E`J=l{`#-l8?W{gU@U_<5bO_{}7=<_15f_aZeWwef@{t
zXEyv-AEfXK?A&kHG*Y2tW}fE*Zp1sqL(Duc0Px<(myM{EPI8MoL+O=tsV^UVo$CJ;
z^VX(V=kT(|qo&am`Tmg-_S|gO=ZXgUV39pl^@rHI=|?DY)d5evSGScOIdQ#mO$T{e
zzjh6n8O0b}k`8Tur*Dk7KF?8LRDS_A5bX8+;zSdOS-ii8e{rK7Ew=x?)GcSE@8Cum
z_)_FzbL(s|UhRzw#&kRIehO*W538X{#>QW^N+j^996}oHxB8uJ?D-?|&I}cdDe=bq
zAr90ppK0DKNl7`dTHUJhWa#34wi8?|<VI}BSv>mwm@T*TI`br`(94(f?q-_yr>RxJ
z%vpf$C+u@SY1N>W>Z@Q0CFkOh3%J5nK*%~RenC%A=3*e&3Mo;!cJXZIg7`to8R+1q
zEF!cuzAg~%2A&VkVRwE=9m)C(m+7jQOM)yU^s@MO?tn_zePMWZElN>EI9NQQU_4HH
zOB8Ur7UduG<iyQ6tatEe!o=6*Z&O%}{wIYw9~hoU7u5TPNfi88hI;ahejYF|ErxP9
z|J5MaIk#03H%)RV!!u<3&!sz4(8Q5KSw@42<9wt-oLxc)EB6UE1M88EXV4v{YKiK;
zHo%jai^i)dlv0j-H~B|q&JR;flfdLUo>+;T-ba&8RC6qFt?3u)BieL>HpvXj+GRBt
z-pT~{OMdz?uZg!W<G-bDsg5uZ6E#gU3Ol;BF1pWC=s(W<F^L2#hVCCjtZ%C5Zlgqv
z0GIn%g}bQTn|mOf{93~{2MjY6yWLB~x(Giyy?$bB$^*)yFg#>_kuL2xh#MN)yOfB%
zaIAO=3h3uyn`h_}?z1D)+VIlf3^@D9o2jp*9$&3jO@!>vXq7qh{&8LvBX_QVvIkt<
zkIA+h3Sk8Dl4U_Fs8J<-uNy%c-?rz^HX<!Yu@zVarTdliUl!EmW1iJ=ceVe-;-oN2
zlf6mZ=fsI^MN_nqovD*DvSMoWfLB4`?B7ipRw{Vf00&XYp<hUlP9(srBu^=tc<2K^
z`GAwNv)LM(iN}$JIs~c?CAD8~9V<7u+XD;_EX-OgqQ0C7PI%?KFK@5sY!UP2yf6+X
zQkA!b&GTCJ26y@YiAh$?%@Kd7^*MqVbx8T%un5MPc6yty|0zzLtG-i#RSoVt1sY!4
ziFo}}RQYhPDC|513Z)gx{Gw6J{Ac~rvn1WKu_t(O#zOLw<;3BAWgurzhEuad2eVKQ
z&hlR1fm*NE)!sU<em*y5gzOAuI64B1M;qQp`pH{{c<FmdQe-%PRTolBgEjua8riqY
zbzeSk_cW(7I;Y3ZezOFlBkxLboG0^%KDPR-*{@>u^s)Xd`%+@#Nn%^ZhvILv#c<!#
zHro=<34W_j9V|waeY?tO+Qfw!V8B;4;nbA;Jk|8$od&7|QT1GE1GLF>3v^MoBVLh$
zOQof^crk+<FReo6h0_hwf1wS+%81FaX=dTBwiv*C@~Y8_vI>oAyp~Au(|)%6g%G*S
z0W8(GK0a7F=-@7uw?5w0#qcT)#qEx1*To766Qi>B80m|Et{9gGP(=DbiFGdZ!Jp!i
zaN(ocXjadDjchv6Zr355w{MBT?d_#PR`-_VneTl~-yR8%T^J_`{pH9>X>B<TZ85K#
zFxB|S9hK8G_IT`R+-oQ;h#vYb_#vS~23c$L@@GLENJSB-<miWxhXd7JK{Oi=yGPGt
zA{aVshfjN~d2lM|3xFy2t1C=DO{QMV;G;2rD)(L8FP+U*Pfp`S^r_@~%Ns>Hy>re-
zvqc($La3+-w&&C)0&C$RtH(P>9Rt~V=2SVx2NyBabMWP^&DH)orAN(t9MMYLETo5x
zx1G)zdK;23Hm0dcnkZ}$E)b&C<Hr0@q)eqom)_LDs1rU^xv@D=GLw#X1;C0`$)n~s
z7jbbp1Gn7$NClIA7$qBcqO0yk$RH<OCd)vyF%=^ED4^eztW8ntF=c#ZrARcWvNfjP
z!$?9$QN>8Vb!FJZGEe{B<CMNQ?ROp%>|z!Zr)}J45{5M$@I||N$_G(+TIk5E-0$au
zje^t=U7k*^PhFqAU14dJSzoZgEJ!Z5yCFBmvc+3EGQ<vjkQ!pJo0d{03n`GIe8rx*
z%=^T&j06qvrkYt7r|9^LG^GtK2!ooC3bZbUW72NO4c@PCn<Y1fXN_7V{oF_tjuYS)
zj)da?23IpRgP>;=p2JqfUtT)WZsh;QM%u62?2@t42)m+1^>5TiDv(=;*X{L~T)=)N
zS)$<%NK>o<i~H3m!1`zyza@RtG8wA>&8Lz;i?wsfpm+~Ilju?M0M%qrb38*s{sXUf
zE@$$gA%w*;@ro5a^X~-Zb4KajI>`u1!S}<4Kt)U3OIvqw@~n*!ERqN!Sm5@gNG6m#
zBTdDZ)rm>3h}h-Gvc1EBVrzO<dP+$6@L@Y47#LZ{2QHI-YhVN5Cm=!t9LPCW#?*7M
z>%ph8-G$)c6<2_sQu$*zY)pf2JySqSyiw~FD!@T3D32gL;G%8Gu_F(K&r#2E!Z5Y~
zH~51*q=noN$e_<n4}fJQ81$1))-=rlQ-mz4O0yNK!8m#GG6OY}@7&}C{pP;sR@H+s
zD2RGW;YhjZF*2?VGvV`FL`FPLoXgvF4u<CwX~(puJi*(!&Fo*b#`0bZji>rB*jv8F
z-EH<<q-AvW%&$cfYQ^$JqW`X0aiC52_KKO1h~YyzJ<%u$lcPA`qd5lfa<@`9!(#pB
zGxCV~`=)W?yw`Q(T4nsb^Xhb7GJ4bam5}mQ5Rmv<(F6vyXeqW%br^n=%%G=9bHd`K
zehP^G3Sto>MO4j*rqWDGwsQPLKnDXObKnLE{N8+rRupy8t>>2~GZXaGJ}UvpNO`jW
zRoW_cyatbU9Oy%;z|RLO3S!yn+Lhjz4^@RO+el}{>8Mb}sB&9Bby8$CM}6*}X(Q}*
z?5A*rIf*<ps6*I|z%jhIu}%xIzV>8-xp~O$;oqBNnVrlalQqCDp9nP$Bz&g#n?m@o
z#&0%rCqE>p#=`QFtLuEwn$uZR-}N+<$&MJz`s+C7_$=6fyOUOV@ZMC>bn+Lcj>d15
z@gS&NrUof6FxB^+eX)g(^vYT2I(-n`=|#sn<UHMSahh<%gE+tTKjC2m%UoVCNu=tH
zkq*KeiP00(%fG`nUUoY0)?QFdgLNPmijMzDl*yz#Awj6#2~X9eX(@K>`5;4FqB{b*
zk}&yDf?b~@sToW38OcOLK$89yLX>2G867ej8w9NF!?%p|@HCQT@UZIR3=naF%1TC8
zTwjeg(^JV?imNMAFx7RR18d~W(G&ysXB$y0k6lU|8l^Q{f}KEXR05Acy`&5ACiRNV
zaLI#Nk9jHA?Mz8-u*0V|a*6{oCfGiA^M>J8?8!G>MKoL-ey~3SBQ;*Ec3<rIAO>ua
z9c3#y)%TDy!~KvgCa>1viAwe6hZqobN^A@;sp6sPn{~(aL-6J3Vc{XG(1aePp?>j$
zRu==)pIZj=AD|aP^hE@U3v5E-+;`xGV8()0fhOFM#|)!x0amc%<+_!m$Z2=S57uoW
zbZ#g9f_G12t~4tM7=zLT&Jov2jvo!!h~iGmHAAM={<{AbYjD%VYsEPUoqF2(e#`Q5
z*w`r-MI;-=1Ozhym1Lt7{&&UPHBkdciiB;x=bKUks4g`sdgYG@jqOcA63fy|-n0^u
zElH(&oL;#<b`##LxiG^~2V2#lb)9n9*77MiQU|(hMQSDqA%K7pNGBhSHi&_%L|B5z
zFN>@L0!t^s!v9yQV~P@Ugm(hMfN$8}Q=|hiaN`I8X$-Q+HqmHhaj<ZT5=3?eAe{nO
zI^n-m?(H(Vrd6IU?Daa)u<(I)p62DBN30vd%L?GW2+!v2z9%0k7U1TulHWR8Uit>}
z20`pb#N#|Y!^;BEIshnHo)v0|bzAfETul45S%a`sPo^F>EeCc?r14oTtj<EC#{_%M
z)5LjwIr^H9^7|z)Amgv({KpesI|<o`)B7pR2i$R<I)ycb%nwc^Ic)|dbCx?Y<fg7&
z!pj#eT|x6RU6-EClAu+0f0L5Q%ETpS{=y?WoR|#}%g;i!Fcl}#pJ5;8)jb6(ZQg@(
zH4Iaiey;?C71ZM=bTkuVo%jL+C(Dj~fSq%W`;EUOmxTOiS|ZqRUc}6PSZ!-7A*Q!j
z882dikiOd0rv>?i`y&CcED*}WF&vpWmS?+4A?16hI?U=&b3k61a@(50`8jGP^BBV-
zTlyJmgvbtM;o;0@kpYNVNDtDI_rrVm85O)td!%)W&)I=0--<j(QU$tnDBt&lL$xfy
zzhB{jf%V6%s0LwLUhWfC@K1*DDd~B|+jzr2i(ekcW%`bJQZwnUkF}Lp_Ch|Ij-+;#
znxDGT=w>`LmZ$^m%-YJXnxL4WGJo0Ne=VJj<cmC1KWLIKV)wjA0}qT_28gZpiqpBr
zRhX7R-aIaFDKluIpIyEn<46H^0Sfz|_^?x_B6e$v+rI2cpdm0AE02DWZmFZ$2YIeH
zN7uY<>fWmrGqm4>^kj$(<8uyTng0bJ+;jIA60QZR*8JO2$A1KUTJw0MClfRQzJ_dd
zqtJZbxcK;IN?G$K&0Wf;MbeXHp%cb<GI-I=9Fq}bgWcFi+Jz3ww^}kDTyMZ+65DoQ
zog3cDqlTGOTfz4QaV)P&jY7?dKgQRUVoiLffJ)kj%iBj4+L6+vyv6)AwiNe{5`iU$
zj!73sp%=VVL;>6EW*~FT7}UGW;xl}wLb8thi8MTsGW-U$0Lepsh4yRDZ4gxi=V2Um
z-;6$#tUw7$q!~jr;ag*%EYQGd{3N24FTA3FUO}Nkk;**uO?#mp`0u`UHcmAi+`><0
zZ@KKuw%pyMkC>U=RF;4epF_4bo$(e<fa&Ngn|Y#RxEpD<#pW?kRTqMW#RTY*QTw@G
z+7sfXy5RCIZimxk*O<xV1EYxtPFcr~b}VlkuwAl!hERA8+W4Q|cQx)&>r5oc>^AWp
z$v>zK-^2-DRp2@f4YI%&o5i&b+D|EKRKKFJD1N4Ub*yglyu<xv8ecP_y;LF_{1@}&
znseZLEtq@c!nvzdc@&dgfv@9X(`lG0oI|ce>&<Pc&`9hiwk2>PU(q4J%WlagRS#H8
zUI?G9?pll$hSjt56;TbLV0)oLR!T+c@!!$FZJx<f8o+p?&D?O$SwXIBaO!Q%uFj48
zj+XyCZfz0R2a=<}O;Rs=A<zrXE$2Wk6*Li5EcJm1E+fO6f<Gw^p-mCVYnKTbl;`4k
zqC&Rj)7_~zy<1bNCDz%nin8!G2rkH{z$q8SM~gvt<c82?5)fOS!mnb(FTY)5<>~S`
z-nVw8fM7m#2j6@Ukj3^z8|)`HL|GzAR7xk&rKi6&y245W=dbnfJ!2p;f=s*KdGu7C
zGkg4{AulOCxBb18QNn~V9*<#_G&R$G_S(@V-2k9mZ2{{xkCH5?onPzSE<xuig^2{s
zv~n}Gw=(^<CliDKsUK>GX@^%Rm@S7<B1Z=+-|pef?QY&vDaA|QT_EVa>D>u>yStFu
z?Z<zkKFSekca>5^<F|(TPr)%*f^KEto%0Zm_L^xE=JqF3scXD@^Jqa`8W)sUGt+n}
z^T3X#`10Q)s)nx0a&Y&f$&;hgg-_rj@6@krNu>S4@^p3NUoE>X=-2>Kie0^r&IeE9
zhnv(V_Pau{`Skhe>MYC4<0dr}cagk}vmb)ZgPXd$I=i}ezX}yxwcSwul8QWck!-#B
zD7w~j<5xPT$xrCWE<uv4OBJ>z@2g_HaPo1H=$nx9SoEpCl6^_?#HY_G!q<HO&2<F^
z{I~tB%ZBq`;u+z>+l@*d*`R^HfyYa=+?&46_(maA@k4GX@(@-z@Rv9FLFqjtx0&>i
zlc%r6lWG3D>K~3s2S<+5KWav_1bORie*OK6D79!{hq32-ekSq}q2W$$9Lwx4PnQu=
zxL-lRrf7q-9Ngy~X)=4orP8czvxlOiMw>`mKt*<fFR8nVviyP0?z&Rp>U6r*Re@O9
zpLZ4^0YJ*ktsU9GC=jqKsFwhmXiMub!guWXvF@$hVqHt(0SE8v3lD)!s)6lbAgduq
zr<$rx4#Ov*@~sV-CD}VYX$rv^cxV8M(|o#Q>fiIVHjx07#3K@(<!o||rw>b!`ifN2
zh4R45QeELceo$O=SG+fBxXJP+7GnGwPzW+LbvYhBzOL1%M05Yoah5!+K-~LTU8Y(k
z7PbgN5f`I`M6#+sx)F++uFgUFY}=)9t?NrC|7`uwi3(n`aIo^Y4rnfTl{&i9cCF9I
zp%U_B`EyHd=#w;*_Rm`Xci+Uarg009+{A%cr5W%YRmwY(`t0vtvuo6#kvaS-!-xu_
zG`f){Gud2O6%6poA_n7_Iy@rA!NVtxDiZSKg`HCQw#fu;gVGk6L3s$h?xMkOOAtha
zC^@H$OCAwE*oslS(#n^!bO3&#^12cHtSq{GqS=s%eE>$7-1=FXgw3<8jg4LejzKyC
zOBX|`m!L0eY$X+lvyXA20A?gRC|MN=7Mf7ITG8uJ%o%D_LWP$Quz*TPM(pdCZp;E;
zZU_4k6*IG|YD+^4u)QB#$?3U7mm}HPhg)^F^69D8{&mcdD{<VXD9DZZ6!1)14|q(`
z733n9lHv@EI<m5XBONHjDCldonx|V!PZ^gRL*PbCek6jTO76arB+}*K8R_Jz3PK}C
z9`I|@p7}nH$^*p*8iPSSXF!IYw)w$mR<XHa-i4W-vq?tNYcLqyDBPgQ8hJxeUmmIJ
zog?H1HV7q1E9GXH-|!>9U^G@+3H&^rv0p*rv4z9Q)Fw9)4gAJGrGO^!Kw%uq9`!c1
zNPvXte4dq)9ZxbEmwrv(Ns1P03ao*ekJC$Uy0Y@fmg#Zh%zjvOVk22}-6E7{YFeEk
z2F7bpcA|J9{Q8VsqN}0x2lCfTHe1*zEBp$L@wdcF2Avdd%6F;dG2OHg#g<#ZZOYz`
z(SUm8jW0xmS&nNz7F%J^aj_#^xx4LQms>SVn{ay24@$4#R0(eZY#1l+W#KdyQ}Iv`
z{z)f6Ph#ofCCZ*d*N+CC)41&O_k06Pa#J!qqFK&)K2%UJL8KTnro5>?@2%?{QQolc
zLd0iEtZ3E71WKcsJLsfoRxy0`)8jmQQqm6r^%An}zg2fZqmU-KyxSDvb^@^;$z=fu
zd7E9_8I=t%EonVT$r-62k0sw^LDL5Mdg8>qh^`w+$jmi{ig;KCDN!tCTR!nkCOcHQ
zB@!Y0SISTpe+rgMuKfdZfZmjw3V?2L`2k7_ZVF85x7G`*D-;j`8=(F^kl;9q-B=;(
z8xuLobyDmbOnN0zg&<RMnqH7bbaxeo<F?pxUunzC-Q@lV&iPgj4q+&=x2>xQ6XY$U
z4&Z7HPL>^kBqOO6aqwTR<x#0BMp2dENf2Ffl`TpvF;sgEy|?@QAHP)dZr?*ohJT;<
zKU#$-c&Wx>d`Ijg+_kzmqYI%-A5+UshqElt)(B|Pjehmc;YN9|Qpx!r?rkbm$)$q8
zc3mAT6M`mV@5TGa;?oq%Rr3nQzo$eY<I#M4xB)ke5d>i{{`=LRp)p@Lc)j}X*n1s<
z3n+Q9J5AB1jxC`JpW;a9wyO1DZp8TPeI)xg`p(b&j3zjdL}cQP^;hUq>jAM@j2^v8
z+_x8G8tYGcg5#q;unZo#H|=D`A7Ia#cV@A59=Q+n-H|O^ypF&R{<-;$mBM+eVwzjR
z;O>b+bXTYu(_4VL4E#K$xZdwoyQ@&^jSxv{<MxCQJ@%P<pHGyCAgnc+;QPFvmgpJJ
zO-MUDhph!ZPS!lh*4Fvs{EV=kPWL!?a9FQgw^3irFs|3)gy*g8STY%|gAP)!UMCWo
zGA^GF1-jL7kfrq8rukpLTeOL&U>!)vGX-Qq7TZiqQ0gmhFW;wD<ZtWdf8W;JP5crJ
z@;luIQuKK|=2j}_u_Q-`n#np6%W68uw92MUKM7O)BlqqoX(va|M(#f_c%SOpP-sq5
z;7!PCZZ^24%$f<@R?P-%9A-X*V4>fLPu?<5&H#VC$V7+j0_tZH<{6|QxQMFkvZ_DD
z`&neekivice!lUhxs(FOOEn%+&@vheE4F5vL)5zs2$$I?4_)|GF1M^0>MkD4{)Xt_
zz}7K``c7jjG{i#dY`24i7C;y6JHFb{EIGZjoDA39EwB^%FB3iZvb6I)z1R0TxDj9Q
zlYjkYx%cU={<QyInFJ8VA~#4IO7uh^$Cv^C3(IOShJ*+{U@6&%$?5)d%29Z5Fc&n=
z`{^4y|K(CH>YDplvy(311ZVOX&{@L))vT$ZGACyGT39kC+rtdrFbOn59&_>o$8z{v
z-u7aa0C3bDqzLnVOkJmzD5sQ+$*|$5f51`4!%_GAU&(S=Cln=`H#=PpUhfI3Z)3qp
z(l{sG4_+lLydVRY^RyCD3cyn~Y%kQn6fyx5eyxOpYl|+v;a$3keMIC-d9&^$?0_&^
zh41y3YaSs|?!Bul(7^WnD*9~1jM&!iH*#WMLb;*oy3MYsaM4SsDK56{dq+oHT#+^9
z&ZA)z1KS#T4K_gH=KwHrR8ZzpImIAs3Sx=vGDp1smGGS?pH%;%{IWi|h_)MI55=<!
zg0HC^`dVl7H)|v~#idtH@@jaA{=nB5G9Ek#CV>$3h$GHF*gruNp7vZk4Sr2YO9(11
zwE4!uD11Cic$2c8VLYZ4RMUo(m3P!07r2&{{?Wm?0lwP1WYh|K&;UE!Tet@3)IbrX
zL=EJtVR+^U)+SR&gnS=xqOj2fY;P`VY{Oa*W>se<6-P$zNlZrPMw)VvEg<>|)i77{
zxR&cjt^UL5I<bAs@*9}I(=!XycL!9<EsjDSxg_^Kn>UtP>`Jqw-xOM(i7+^nc2T7V
z#@N&N<$}a;D6)eI>p3w40N<8a3x9_tlEg(cf11qvHSr)19_VNOLr=>$0Gnc6KVn0V
zWf6&G`QOTpFBt0f;D-_PDizoFV28p~lnp8RS<$y0X-iYEd$PA3xA;)+M8CPwV^(2?
zSsd3D==+)v;MF*-4`Omr(|_hWAKv>eeV%PM7KaPRDljp|DnOG<2*=_|2wiN!AORaN
zXoUiho<sroFE!*v06OBgUR2oAb3p+jJ*SQJJ-@%A;x6{txxY%JtBHQ$h0a45Ni;TE
zr1el=gn?Ztg$?UtAH+l_0b-q4r5tR;d4QloePa9QmU>w0h}Y>U!_A@j{8Cp`l%lcY
z`>)b^i1xCkh0()6LN;rGMXZyco%Vo>gS9TC>Zk_<4gB!hN`$|(;Z;Z~(GH7p(znF^
z=6y3-n%&h{QfCe2#GPyCeC8~LEqf0O-#)j9wpJ!AUc3)a_dfyxCr?0=!PG(<iKu48
zeQV@~7QO!Av!4;p4n^G4VpyhL2X+Mip3<n*Cd_-~QSC54wzIDaFW>C~`MDnK-d%wB
z^J#Qd^(I{z5<Uuc@rdh+q+FbLk_)4|fYr-fdxPoI0-{$hqVUPzz7Xk4$HUa#x*hls
zE)SBo=kzUvTg4m*e8CBa(hRwhH5+TyVBO|?ppphGYgv)~-r<&zvf3G1d+*kZbFzO=
z%m=JV`u84MjN#%?`6v$k1}U?Z2RC3YTEALgTW7Y=Dr;(2Vx7|-O1Ql2BlflN908w)
z(Thd6ga+|(Un%>Xa{=6&{RM7T_7Ap+)_W@ob#9FlG{L@|hZ!`(fWL0N6|jkf#H2#2
zjd0lA`J*`?$2GRpnA987POTNUCkB=)(PZB9uib16F7lLeCevN$DXnc5SS=PcOM`=H
zgjk=Qyg#x9Tdbp}Xru=W{tl^$ZrXt05|R2WB%%ZEl}B2HOJZ8D)xkq`apFGPK`q|j
zif+$~c?=56_V#uZrsH=^mma=*USU+vJf6~oTC2~c#TdkT64{!rk>Bc?ysu<i4Hz{2
zRs|T3MUx*P3%LTYuj{GlF+itRLGme5nI);mYmH6-YwF?ikp<mTi5}`j`Zphos1*Xq
zfT)BXzsP<+M{?1Wb$RpA6YOWTc*(UkOc{%Lf5@G#HgC%M`+2!~eF%+8c``<L;q6Uu
zZ1}CiqvxNapyz$5cY11BIylK-eWVim57eIN;sbxuu*=tMoOQo`f|QX4QLmIiTT1Ri
zHfGozi#Gny$!6E%PF5WR;4!Fus`z~cP&}$}e){I03h{YC{@p!dea>M@n>S^BK1Td{
zAeJ}oN@x;W$edcjo>EdRV@HuUO_2wn$lLy3$?X{7)#hPyT+Z&0BlqROzzg|7m=2o(
zcuJw|6V3Pjgz-gKqV8il53W^j9EAzi5SGg?)Xh;qL(q;5xB0(sZ{$?#-<4$V5`@oR
zX|J>a82cJyZ71gVa&-18)TQ<6hRYmx?a4kF5qP$e_ANFK3%=7)q1q~KbrRwPMAx~Q
zt^^-vm6XdCz&r&$DphE6Yt)HSJ6?;;W88&wasP-_m@oUzn=89APWI10D}Su_%-$eT
zwz&=Gy&2|)cM>=8tF3ezV0mM?GS}Y`R`@BG;WadCq6)Y!sM6LVP*FN8`(pk`o@B7L
zjHvW`2u+X3_TDHii@>xE7=NI!0ld~zCA>9C1Pk;c&0E|@SoZ`Mx^JmYWJJ}%Vw-4q
z4&~kE-%5wE)emtlBNMU$pI+C3p%)40Q!PwDfGVbf`=i6CGG%hUY-Vx3s?+)Wc}e?3
z10<}s-KYBv0PqXJ67Y?s4@|gxKmp5(a%xLy8idV%hTc-1^v99rG2(kIH72VK@D!O|
z&B5^Uuhx5kR)4QV)v*)12JX+2eG>jDXvL8&@j@mEVuvTwNaBM8>>%(-gbtTl{=N_3
zJf~#ErjI{buROnDs6*?{-FXo->Ngjzd*{Xlfw4I(VJ~&ImiKhwC2%Qhb#bqPlU(s=
z@e$bmMvQ?#zIrA(6hn?tk_AE?Tk$To0_}g6?WWNc{LtNVbUo5GVIZXM3yNkDfl0)#
z$r@)DaA>+5&olDx;B?!3J>Nu`XputOe4+}+z?3MAY~Mt9r=;Q$%de_N!Ez5HaX3e;
zLK(v=*e8?_R=<LM{VULOyk51hV4v^`_Wx0rKH!|3^tWYZx}m$tWC`DpFGJSW{@52k
z2K(I*#!dfTm*xC&VzqT!ic_lB(FQAe6Www`{W1AO996q!5OeZ6qTpo9ue4zczn7N&
z@;Afrv5nUS7uU%ycHA_R1K4GuPbkm$N3un?f0#EDf^#pDdpD-I1lrjLVvr*dFA!dc
zw7u$j1u`-#N49gG`TIp9VoC6;fDbZ0pNT6?6$L_yq{Q9hflxRx;^u^+l^~9KK{%WG
zc8u>E3_4SG(%QQN1ZG`JGk6ji%u~FXcM6%LrbVYV)eO^%S2B#~lbGD@&R(iwE^qua
zlL5HG5dkAs8pgY9Lt#Z|QekrTv-`cmR~BR|;em2ZMcQ7T?3U&SJe=fd)^$>s{Z_<<
z0<UP(p!qvefQHzToWBDUY&C`JejsEaHM|<%j^l#9S<hIm*$v}6#6m%nol&0a!kXNF
zg9m|)SG@gfGq=b2_+yel<;$lMdk$jnNZ)nBQGHGf%x1Q<##$HuicAqV%#CZoC!!B7
zTz8%kuql4o_p9rb-tm+G@h(=D2C;GJb{~+#90T7zQjthAr{pHI=A$SFT8_k8|EkC3
zO}M3z?)d7HU7UOF%LR}sHa0(7Go{hpeVEZGOTZ70dj`{8e)l+-0)S6S{ImvhXh`Mk
zsfpqPO0vjh-Clq})kCm217-q~ieY;4F~MNG4y|SGVd1L;5fzihkw$eDQ>wg;8;Qb<
zy}J-R2EC<aSW;fmAG#mWj?0?Ye>a4Z`4Dz)z1%cTb91APiW&aIGfj8Vv(Pd8b&cQU
z-OniT&XrXs!1`<^X4vU7S<oU^>CIl4$E><vR9zV8XYwIk&xI@3@C!bZdstuY%{z%N
zbryTjhM#+>Lxv*t4;bxB93$KI!cxFO+w%ir**VI*pJW*7*6$B&!5Ow-PXa8tOn_V_
zvRo$h|4Xjwlq|`^N+|&K4A39!`)(8fe&7KgaEzNOkB{SFsy8-dfNvZ_qi_w24r@x|
ztAi|VVPQnyCUk388YlS;!LsYvYD+rq%}sQd>Ca1CZ1c(mQ6aVHKlKn3a%6}$5<f5y
z7LVrbv|~DwkcnD^`z;6u9BVA3R@|7{&x7&$1`-pRD(6%=CyJCEiju_!WNrKMTh9j(
zv_Euwz(9BKFcY-U6J(fFuMBqZrsRX7$U*dD43_2zS40paZ|56RFy!*Bz?Ww*4PO=6
zr2x0jq2V(1(@C_kBwuptWMWE~|I4w%4oFGx;p`7;y^Vl%p}a+Gr>O@4IaNNOi6r#S
zL;)f12Mjz{5}&E#_)@STQ8Ww#|1SA`Y%;kQam^a55Uj2s`v0+yz>cmEfL^}B5A)lw
zUD+MY$VyEN{hiU6pL@TAjj22QQC91QmRLS>^MO9WMOLx|^ydVr8M2m<W;zioeP-V0
zyk?s3$TWe8|D9n@z5>pFGR)@xkzsQEcLMrfYPNEiWI+g3a&KV>0$f3f8Or+cBd=Ls
zh=>0z3W$m>8MZos#}ezu6F0vh^)aXA<}4L&3ZcEQ1iRP^q4cT^1ZE{i0Kq$@pp3X-
z9zijKS2PGWJYT+hBPKP-K(BK!CUFprdfTO-J4Ao4F0W4JCJf_tHxMZHBBJQS-v$uW
z_b|)qb|dUxr&IaIs8Q!RAfJDsye9<#TK$^!5%T1>7O+b&!6QSh35k;7hF^L^pZ)>T
z(-GXCC`U(1;tAvC*NI9>)}qcP{1^O=w)DVxhEp;1WAQo7KHgntK|lUAC44V0y40E@
z2)O@!^vXd?zb$w<w+mK&H4U?#lz|w@m|WlV5wCs$jfaP&q@)q`JE|11WlM0x9{+Gd
zgX?$AkMkmr7_kaeHlJZ2?V}9@0|<e}$N5&q6U!18T;MNt*&suyCPvBOb&hRr8%BP-
zdgQbDbm2jPl9I3;C-<JcUC@k6dI9+9;3Iv|x;oVk@ky0_U)`?tC<<*XEY^qvs@8~5
z<*R$jQW9VK*<}SIA!2%*e!8k%nh}dI)Y4_Cb31pHF6sm6Ov5R`5e{M3?oP-uf@%!Q
z57F>Ie&^*NZ7^TRcN^q#7}b8<Ckxbf|48~y??W*$QhFj|5+$GAVKR1~9;J5B=b@O}
zFaxI}nhevChYkom%M{NYDc3RzC%!s<JrgkWbw~XY)B49JJ$iV-E>0(mHW}Q^2P`#a
zkFLlP@l2N#+#HRWCHv&^!GuyJe{R<~b9N7(ur;zq!?6~P8;@|@`5EeJ{L3?r&%P#a
zZlYj5d+D+9mZQ>#7V)j=Z=>#?dmk&jxFkblt3>58myk->kGVqrJN{*@!8)&aV)Y+X
zky;XrGLw&>!+D7;Mg73s97zD9=p;60!<LQn>+?qv=IIXSR5$Y=@N@m6a>^pO3B>!{
zZMV*p$tw8d!cndF(6yN;zk&+|(<mm+|37Y(N2REUB%X%kU*g)uw>D%G=RGXUfH(ip
zOL562Xnx<S4<gb?zcxP0!-o-U3==B?=p~t!7i~LEk*WB(LpB37%f!o>zXn#Z5m_m*
z*L}`%r8TlJ`e_X_Z2V~KxdmbAc83OOjI3biQs#9QdH8fnk`ZfTKq0f}edGe|C>)jC
z%OUfZ<m5cPSVne8tZ4U~DGg}uiMNu#IHm0?c&SeTe!MWd?nC=XIG)B2IkSU36ti2Z
z^%jtq{GK9aFNT{t%Ltt{9{G$*`yYOeX!4@bumqOobn+0#8dm3*{ZdJX`fZ==^@KzD
z2(N7sE)iuvlB$9)@`9e%`OoHM4wsqWT7ik*cvQlLs!1(MT-}l;(3A^8c&r_AKBWPF
zxMylDf~+XA42<YR68V_gt2KQ41B_b9G5$FK1!4FS#krso%)`f^J4b)l=g`@ZT0PfD
zsvq{Zw=Y-}jU{3{AYoW7OZtjD3;AOu7`rKgX_#B!+hdKP;-pw`5tzjozvLRhz|Tz8
zN}XkM`1LHS*O^>e_NN+WlyEqIdV^!}-f;hztSXSYt!593MwT@{k_E1>DwDQP*S;b^
z;rPRD0>HzwtL@d?$pMGX6}@SEm%uc03eLOnw}xNi`|wGs`=>BreHRR*hyuoAVGB*T
z(4${@Gv>Xkx_>C06T;Bbc%Z3w*WX~OC9B_va|m<ejVL;3s+FJVo~Ud{xt6I<`5q_y
z84+2<!#%Xlml2^Vxv{OU9e_#}j|y3>JuvxpeyKh;a;UU(BOmn8C|Xpm9be(OT#70<
zN2_I!h$e|BHoQ$Qxh||fW%3$0xtd)#>^i2${KYGDpYcpJzc(ZMPD8O(JW`wM&R~gu
zQzT61hs(V2egVTfBAD0sa{oApI?Q11c2-*$TgHlm#6Z3L>nSGMJ)>Uh{wajC^CIXR
zl|=P`khnedyijdB2c|=}@98-HiW?nM9+?5{2{>jvcowwWCL?#ZSD~jKJzv_w9HwwC
z-%}T|4UJ|_Q_U@$c?yM62$FN{U0tMd?XmIa9W3U5JA=<&B?Z_&5x8jo<14Epjgk2Y
zCkqeQji;(AP=S89q+eFQ)|cXXSA=Or;niVp)sqp7Z_PwFLb*9X^fi^Nekb^ucD`|C
z_WOuT@d_9&nnG4j^?%={{v=kwOP|n#ZBN%-5TX$&uvFNBL855Izn-xLq!Y@pbN>4J
z*K<R0WU*QQXYrQMgm5_!O>ZDslSd<S^T@?iq|l6sDt5jmGm+Lm<*=1~X^?O$w8O=u
zF3ZUeMB%b+($+C=ej4gSHfbz=8I2YnjV4hd!%!ll2~JZ2!-xS)M9U-VG!O+6S!;?O
zx3g*{;{nd&ibiUO7tRS)7oGKR_bL3po5)mz2C`8!n79U%B>^bNV{f=kFzNIi`B!P^
zjj8w>-unba15@w26)t^yEor4Q79N}>9NZS}FquTY&cB~s8UagZ@ZZ;Ie<-w+&NFt3
z_pFu9(3#*)d3g7e%TX$Y5s&UOZ1e4Z5AeT`&oU%*=2<$ETt4L>Tsq@qO6ev_iEuj!
z1%$C!O`c8O(9DQ>4BY=R@+3b*v=|WqL7HOrdkbPwAFe0@GeDdOR|V+L1jquPDFNBw
zo9SS!(tbvGF(^v0R3;uR6%<9}ay)~G`}#NVkK*TgT&qYo7~(78zUV)Md-4Ae?*0Em
zxL1Q95GOf6wfV4O87L;f5GxX^NO+PkmO{T4$~%F)&nUb&N}tm0&-icT204qq>LzX0
z{hW{COXxilisws;)aJuMNJ(P(m5A_9Y1_4+pg_maj2n~cxK3ZjT7fuDJ-`zs{}6&N
z?D=Mu9ZARYTxg>-m!9sUNF4uv`hpQtl7<s;(gaOfYcE&Obz*Z!Ox4~&>cu{PG(SH?
z*8G&Y+7`!$!KBoqciYOL<_Cl%N6O%xIRGh8kztoy%{u%UZzx@^$Z%pX?oQYXad`gH
zxlFO<gTQ(4x+oB4$4e4nG<}+$Qc`?7Df!hTW<SALG$$7Udb+^x*4_3zzBJ=$G~2u{
zM$PfoBnXQ_GZeX{Y$>$NC<km7@ncS0w9fRUgp-29I;PAyRVD(BLu9ttE&2@ngJh~M
z{S>6JAPNOSO4Uzn9IYtK3i5U8AiGG=Z6Eqe&_>tS5ElSF#qLvPZ$QsYf!FEB=e>fU
z^;&J%Xqb0{?9eat-;!*0X}~Z^w&~zAYcrD4eCPJ&K!qcT+UpU8bo;?*-g-&lPCZh4
za)!0SA{(n090FTYPk|d5Y4rLqyz$uyJE{Nyay%oJM~}-7z39&)-@j|U*%+zL@EgYf
z*eL$k(VR)4#E)O*wq!~-dlvqqBe^mdKjOsNUvy71cVXY<Vc_%|J-%to$MWu~SiUXu
z_Z+E;p`*X=9tKWx<aJb6(TqP%L(VI2Mk|(YU8XX4z+LQfjWx{q{l>~eZxU0D+vNcc
z62sPEZar&vz&T-j!|9-Ed`4531}W4`Sg55E%j8KlUei8Z-`Zb@acwQk)?j;P<sMt<
z&8MS=(4`SpX)paOb2oqaH;bZX$$dt9efjkn%MKkDAOgiWd@E5>;0karGZV|?RK~JP
z{G;*deLD8hS9(Glw_G2XFIttqr%~zgeM>Q>oq+vP!55!FjOAaL#WW~k4f06PIzUUJ
z+7lt{vP%uNd+t}*V|>^CwT=^GUnT3UZ9OPo6t^T~{;s=jMmm40Vz!9@?^8AY&NBts
z-*#)^D6;T`8LT8)fQn+0FEgymV0*%StX$BOmgyL$BpVC8{=^#@9B>xlhXoE{?e$|A
zH$$y$Om87-vToLcjh<p^0l`=I;KH|iXTnJ$Fjw{zW4RS$!uhX;KGc3`*7N<t9NXEt
zu43GzGIsLe!^n7vh6e0UfgznIvo;oSEoWru9|nh2sSR$NDZ2iO$i~=ldgz5}N$Xe7
z9p>nf|E*6+n!`mj>0(a7?hkfxN;BKW2>F2brhivp5+@(WcHz)7C(%Q3kw=<J`y1-Q
zMSE}Oy8j@%iOJ@h#=wIszk!es2jKiApWlt<<5k1SAH~f74v$D?Rs+VsNKoZb6D5}<
zgmCXa`)OH!<-@cFWsu!xH5sGBI_CEU`19<xB!h?&<0z8GMSW5>-npRQT6}03Rf{%b
zrbU;pX^I@CS14lwi#gy_>%Kp}z0nX_{cK$Mm#!TVNH_h~XfCwK)P$U)JKjrsmN+f9
zAm2Q6WlY+Mzxc2CSCpOII+wAI3euk7p?*YLRadbufk;C;drrm4rWN8>x0|zisRzy6
zjtx}`&IR<ywgJliz8!|+UFP01GrAL&dDv~+jO?*zd4ISQhA)iejXk<3IeX+{8pe=-
zuhZfjX<j?AWjVSzc83JrN}Ug{L~UzrZ>_Ixv5xjS$QANkXf;2fr6DPn;9NFxPp}8x
z!37_NkuJ&%n#&nOTR;L0?8WK@i36~m%w$g;nJ!xd+_9u6mesj;aS?VSLJJtnl)+nu
z4jHAd#9Ps$a@&SRZxltx;@`O2ji)um^qD+;FOfi)!qoedsNSNfqgA48VM7W*pFFxo
z;J~{EC_ZiJZr#QVrNv!<1uG|w>HyXzVeysS`6peKpW@bo!$L3wM4Fr1e`ztxZ?zkV
zcve-yS#a*Ky1lu4EE{L_<GxW2k`y~S7^TE$=UxQWkf4wkyHN9R^XyxEvme4>?Qo?r
z`M(0e13vtU+_qxGYQk3yv@vAcHU^-vS-@tNkTu#2sikK$BWS%Fx$oCI^L^0tjE3Z>
zWwh2Gx<mKBm(gq`RYa`Xe3+#fg39$&W=3PNUn^=j+xWH9uxxrp!_O1<Jyn!Q9+^+k
zZXH{O1ZeC-l$^zP_S?n)18uVvN`i(NZjg-0{JBvoX#L_CsO&6X42A|sDptb90GcqI
z*Bmn(G*=0)WqG}r1Why3KvS1fqOr<^T?cLKwo@DSEH2nGXSl?O0%$n6K_i-_g9am_
zQG3YLpMQ=f+WRSoEF);V68t`Bx;v)?BRHesliA+KsCuZ<K?9v&gvamSD6_rKryg=|
zQP8q8+FU8Q34&t|HkE!x6WK>c2}K<6$Of9<<Kr1k1r1}64w@h_CFTxcWXNbfg7#ja
zo5A_nGn#IQj5dyEG)o)Fg1VJwG&an5!$&jIKvS1<Jt5SVI%v{{T{90YHfA*3LS{zG
zqMCR!WY&NNcwjw5LfddOZyTn8b|m-B!sjQy4_eZ_?CaItFIm@d($w(KPu73|G>o<m
z*M1JLb<k+tj0cI@pUUnQX%DR5&I+g-gRZFE$f=3ykM&QZ{m5=|Sl6-9o1xY5hZ<;r
zk0rhIt|PdTaX3mvY5UT9Su>h;MgyUia-7k66<RhH3D7X3q9S633i%n08>yhVXTzAq
zaJ^Q_z4qd98fe1B0GcoaQKez2IG8Wv`%_L;UdRWI5aLQSzDzUIKvS37*Nd%9OP^?<
zE&ejx4$#CI4WQvxX%jGjW}eZ6YT#W*@e0-@{tA|#JqQa>8fbW&cEDW@$i4%b?oJht
zb0Ucwh=C_&^&@V(5j1jP^iq}KnRYY|L-g?AQi@lwR^@<uY>er$cu6vH0p9H9K=`co
z91ZVk3DA%g?L@}sXm%1^M;#@(+!1Q!@nr!`Yy4R%;w^jha-z3qz2Fm|O^H_yP~f1)
z48^S2v1-J}{XyPLMCTHXLG%hM&uOKja4~=;43P&1$68#7@x`%()^!XvR|aE&*fPyb
z15I5{X4fKus|p%uqQ%LsBkpjE_F@KTWOf}@nQ!lN*#@ExOcy_+WfLz*f1h~iXb{Kq
z_X!XwWa_Whe*hZUHu!!BG#h^|Zj=d{sdSm3W$K4O+gBE;qBNPHnM#)lnkiHNnoQ74
znaY$2nyGK?(=iPI01O17vHl%bz$>@}JV23Xutb9*(O`)NMWVqH4T?mAB^nfo21_(3
j5)GDUP$U{G(V*xaaxOg>QxVvF00000NkvXXu0mjfy-5vv

literal 22131
zcmc$Eg<D(Aw{CDRlv1R4DeggwTXFZ`7NAI>IK`nzaSiV75Tq1$3-0dj6!**b{m!}1
zIrks9d7hQbtl4{IWoFI0-&t9|loh40&_AF9001l*X$e&T009gDz*C_jylQG!#PeP|
zP-S`bFE1}IH#axW&(GJ_*H>3pLqkIx9300apI%<>US9tG{rBVLe){?Ky6f$imqQ+I
zWep9DwU-xh(c0;o=a!Sl=j$F0UW?1yqw4(^0S=CrlZK7!*Q+<~uU|Mg1dmDij-Fm_
zFP9Z~xVLUD&N{xE)f_IIzEtRO4P3pfWhy_+1Zv8U`A$9PadIezEPIG@mgk>etI=>s
z2E4pHs${Lt?7tkh^d3{GZdZeCx-VO%u8%FdU-t4HRMlU0mre?|<K_NLg-EL@)lN@O
zJ}xBniKoK8iM^bR-EBe7A&_&{Pqus<xytONmi+mj;~od>ti;;2ts*ou+^(8RPpLjV
z%noxK*2gV8WUoD6&fXs>mON~Csh3_g3FZ}9O!j=O@YGklZ8L1|=x}d73auPHVzj;-
z^!9`O4TwoDc2X4g9o6vZ5!0T!5ToUnwaT6P%k!=4V)W8L)8yp(`iMWM#a87Mww=Kh
ze>XjUGu3cw%iizaxE=TVX6H%$dw!W^ZJRasW0|UypzLvB%WyELNUn}ox45h@hf^)F
zHq0_imZL4t^L#9Jc;oKgTWIj)9J+cv8rfXKUv!@$yZ<M(Rwh0tF3DK#^F~=~Irx_(
zYk5umAE>cmoUwYOqh40;#t+GsagA7$_@k5X>EzV(n#QF($)vT54Tto$`PzV8i&RUi
zuVz_e<9{+iRt<9*rfUId$I^D&<F!S0N%;!B<~gea3#$^k?tj;_1C#4Ed<D*%Pqf3w
zXNHUaG+ZkARQz@N-kV|h*HE%1d8%T1<Dteku5tHrcf7waC5<#LG{9|pp`T7-q;Yt)
z(0|7^prC~-VCeVG%-{tz8&ny*Y^2nZ1?fuRFXFM?@z$yTq48sIxpUEjW~_Vc;qhT@
zW9{hZ==}Wr;r@PmduMrhxkTR;8vs!Al#viscQY_CYIr&8P+Ra-VMCYf2nfC*A|w-b
zM>Lv^)L%gD;*Hc@DI84`qwZryN#sZ%mWaj^>*!#{Np#xV_{d=OK^lEy?IZ4|2o3f)
zqP%^RV*g^CDpkw3l$=@d&q&k-=P!E?I}WYvsye|Ulx^5<e>~b3TbmPa#0231IpdF9
z_S3L_K-PCI<w7!efC_m-fdRbl;R_T9P)7z7VFD81vVZ^)Fra7%?7cp=3dn*kNXC5j
zI{SH2UQg7hO9WsjjTa-4Rv-AyWs=TvqUhKta&tw&ox?bJb8P23815MN{;()riizx2
z5<5iC+V4a`%RCL`jSv?LkJ#VVCoE;q)K>jF)ozIgGe$;6jyGX8<=-Zo2ksnV5a@$1
z$H>fuWK&{hs0y5QsBs&imW@@agES?K=kE@zeFqCRjxgTD0_lYX#rlya!>BoS=e8RX
zo||tVR1!Smjq-yHEWqBBgt<0UBo0#R`KjB^siLcXO!J*gY*k?db#61{(!-&&lJ0Ug
z`t#<=@wK@(MDJ$c=i}hss|dVkV^8~(7KLv|z4=u{G@kIwa)f>Gw12xwkfgvHxoBux
zE3$fozE67-m7ePJ^ngaqyk6}4{DZ5;gT4&R6P$TFGov4LN>p2^Jcca$bwddR>T-j1
zLrEdZFYxL#A6QB|y-Ox|b7q02{>fci(A_Y@WWIxJ!{io_S#cBa?RGW%l?T|jj>)QI
zNcxL;Ab)G3{>UfYg!E(jQ9g!A@iw4ogq8(a#;Oi1Xev~Dei&!{mkRumcb-t$;WD;e
zlu9(rxbh=q2p{95_^En9f~)kBw@G+JX{GHv=wsajoNxLg6NfJC@rh&rQLH+cy>47o
zvZe+oIF+zX0!;*I(U2Bmln<JCwZ>r=$#Xvbp77k-TlM~QZI~!el*-FJ8P+t};jbTw
zvXk*c>Tj^R%-@3>>UlC+mQeiN-`2s|)i;W}y8FXTcKL8;7>wBlK*zPd`WJXU!QY&f
zs+b;qso8=<Lst4>3N_BxTxP?KLnQr9%LIUgx{wp=frrcIT4Bs9@GK!9;fe9z*!w38
z0SpMQVdy&uP=1X$gxBc&PxCc8Ujw!6Dt?{bwu6okDHqUQ@zhf59Q>WcM}R|UKt>T$
zm3fg9!}*v{HM)x^j8ly8-GQnk-COQq4LK6FkdGI=<DF$yPL7Ol)I5I3XLypj^U%d|
z9Qg62;}Mu;!AoC;;|KZS$}5}22yX)v>^tu$nP&G3l3XXT7ux^Fx!3LTuXF#`&n#%g
z|8;kbu_q~vSk!9WyZbFx!jlI;%jr$Rpd)E$NqkPxkuZR*WBnm-=u9j)2k)LZLw|B8
z2buv0M|(5;E+T=uK8XNE(pVTV#O{NuJC;mk`gO<9%u$N!o^rV7YK9P{6gEZ7fN-`y
zl#d-CF(5V6gX`#2g=_le3of73(l5?n+1Pn>h0*T~L*)tXdBB%%BSTR%j^-}h7H>|q
z6fS=K7-tC3@kv)~B;oC+1%KBr&`2W)s2wR4O*Vqo3P0KVFw=sh>X`pflm<>(WN31G
zupLyMxWA*HuF);BC761Wuiw_U>f3Ds7Lmq?3Hl1FnD6&j*!Xv8f>nOx1EWUxuMy@C
z*kA&NG1cj+cIGU1z|%?Ml9qX~;@Q#f;IUaH!FKu2B{8}*39}&Z@G1=1&}=A;s~<9}
z%AjAIV*V|8Vd`0+EkRTvn~dUHu%pg=V;Y}E0ciDmJ@92&r4(rTU^JBiu5~fOs<#rg
z%>;|>3%4b7P+q}v46UnfMvgywn)azBZx-HsRoLxW#r<lKuGH=33|?scQ2>?iNaYO9
zuX4TNP)%kvHGnddzN8$ggPuAyk=d+YPNmdy8xmCLa4!c^a)CX0^y$?6hj`-*>?{Yv
zgAzc~6rN^H{X2cfq@_iV5}TbN+UWpFtI6FgsE}4`$8_^}^WE~T$Xe9R9=ZF&j4!Hy
z*00CEd~);&KiRq1&PFT>`4waz%iDLieCw~Zy&D&K_DuM)SK6yuF@Z<?lAw9K?de0T
zfX1nrb?n)cn{72rk8;Z((c?%`^&JONu7(rUfYXQk&VjH2yWqXLTN+_rwT}MsRcvgY
zzRK~r!bCN4yGWnnSg@=KU(@7SVD6xRFpBqmb$#1s79v-q=y=ikY!xlt^_-DxcZZ=n
zCbhp2-5Gx(|6ayE<ifi0#?pf;wUR$3)M@SB?W~V!3`k{{VpqgblexvE@GJEG%pEL%
zKnMD;zgbw&RvJT`QCyQgljMC1mreQ8kQ3uLVli)cq!Xp2u7xk;KA^ag%43JT6Wj&(
z8IFp8>YJ?c@#hG%%0D7&l1uj8dkL%(>Wc9qTegD{{4c*Hxk^Ini++7vT;K`|v%(*i
zG@&Y)DOpV6p6Guh7u;RpIsLpJQa_2QzDmI1cp`Y3{vf?`(R8|GcSA@S`$6%k>zl%$
ztPK0%XiB!X?T<lsI}MrZyX}H26hQa0X#otysD7Y@mUgtUusa^AIzq>u{HVy4Ypt8@
z<MWIhn_3p+3>^gO9u3j<*&C*ybpz}0wPy=bS#ENS&Lg(<^zbdt86N}*a0;5$^b8^C
zDJR$RFLH3<7zoG)3OQz$$xwc=TLBV-cPbznvv_rYlxFLTfsYJXtJNL0XF?Qze|_3T
zo%It^wDm&kN5>y-8e2Z}3RQV!PH~X+1xf$8X{lAhX?7L$XfVE4&`h$c>6zGoeDA9h
zgO;c1y*(@3Bq!eq#@_R$PLFZ%a#rG@{%JbeSZ@={F8;{a);R{1FsAsdgI`QqOo|pP
z93=rl*7)3G#VNS-GhX{B9mtBEHGK>{*0TvJi8Yy^1{@|~##5e1aHK%IR8?g^Q?%Zt
zmq2<(@IKP#55G`CzF5of#|%1ZC0jJ<EBkQ1tE2$e#b-B=wIcl#!<0E+B0E>&prmKJ
zC76pvL&hD4E<~(P*}IwAe$s^5lkd6Kr=aS)m<0PX)f|D}y-x$HzOa+nL%tFffbmxP
zYr`wS8zcG#<qz32Vh$BP%4k()c7p*PQ0>0Z3Xki`OPH30-w0GwEfFIcY{!+@Js&pQ
zmH`gEl;UVc{`n=Jv`4iL#fh$?P%tV~;Cn>?AtIAAR>Jo~KO-!z9^E0$Wi^t_T@(+p
z^cWDGU@d~wBkgSwDC4INH?Cw^h$R!%6#%DC*<MF2NJ>ho&)1r$+D06awmA*_XP$9b
zg?SPv4-*Pe&T9)4e0Uo$cLPiZ=Ye&YIVF4`&xS;K*)eJQlLwCh+D;$ZwO+I{pnQ#{
zh)Euo0KhTfs!0fWAx$T+ip4|FOUG%)=$%FKo9$D%O@o^ni)PKc*u(DNb1eehP)ea%
z>heN#H=H(ywPm6>W!Ogvj>jnw4PK+ZjreZb$aN&TMBR|&sd)*@xtF0NnniNTTazD=
zQ!0r0%z8spZ(aTFF^l*mt6%V|<x8C#2zHl|$jf8DH|d09cMgNe^n9NR5Rbf#*4bvd
zPWp7q!K0<EB*9>E7kT~`=R;?0ULWroBH;=-PdBq?%;cR3#Z{r%wE7_n{T&2}8SIF=
z8zWKE=lPPy$6h!)Vo5#RvIJf9Xz+5>@iteaz#%=sb8e<UlnXBjqaW@OAB8&Ujn09&
z3Vkn7PTzd`yQ7U@bFKw_6SMYK_XgfRlPTISEf?N}7QG>R8E)ERx2?6&&Kj$qj*n<I
zj^-f#)?_E0AWq$PMAINilbusi(uG4;bFSOq(<2ZXQ68`+24uT$D8VAo|EYegy95JV
zknAwCq90(+_0G||AU)EU0PI#|6ToxbY{PQReecK+4OSS}89it`S^jzi{oUwM{jffe
z^xNxfm5L0uLOl1RO<S7oiET>c{CD-+>ze!hrQ&bx$kctxmcP$jcIemgvHXDb02xk^
zbu5AWdTDOm!hJ8wa$A6Hq|RjzmO7lbUbVXhrycP(nh$HxqQXm0YmxJpQ$}&AkEy<8
zg1&=EgcfLc_-8*cWv4%wPai*Uu!3i_xvA^PM2v`K-r?_g3TtoW_dn-Ast3as9>g*%
z;X}ZjyVT64qtHyKm(6$02(1F)y)MW|@ZV1pK;xcYA))VSYh^I_IHA9reEII?6S38e
zD_d)Eo=8sB+Q@Vtmiml4$hu2_u5g#T4T4t4fI#d@2wXP5AjC5zqh<MS1M_stl!;ZE
z2TUMktq;mjM1hwRlH)0FL+VV}rfJ5u-b@_jO`wr52mq_KeY=K-%BOH(5K&JV{*`sS
zo~*jQ{^PiM&k5=1+zGr91%rCOJD%+HZia)!v8;)eOvrd0^X$mWgN&3EBDyr5^>2t;
zP4iHFZd9+><o57h7Kb})J|;|nT)tHHo!cx<h}^`bG+m!0TjkY+W5s}Ha>hS;#Dn3h
z!*`DQfzM0<@uc9ecQExaa@skTm1_kWW@7$o-5#_jZqYG#@M9cm47|C-ui!ac*|Z%0
zoH3EoKt}8KaGc7MQWDJ|h~Jb~-e=|t$Q@fLbV!Q4kkJ_uvq1ryn5ghliccD>L-CX6
zn>NGxGp9Q~NrQ_|CNnLsrdK`c{5LO3_B&+$;5r-9GqtH>F8C!4O$hCe8{+nF`__E;
zx#;RL)-CiBG&esmi_xe_z76@8tQvW~7VNyR-ETi=Cq0M+UR1gqH=EC0^T4tv)$zyI
ziB<Bbi1m41uiFi?m(FhmW@8gF?v=bF(%ewT@ouI3`hZD}(Z;5mG?&R>F)?-3O>IUd
zjJS)I#S?)c!FW-he-?;Dh4)#C?CdA<BKAunH3K*Z2@7FmDd-WdMpJ1Pc%GG8u*_2L
zNoyTLl+|3gEF<0v^!VNNJ&{^FDyD|Xer){2Ob*Hv#tA{YN;kmy-5+<<68=d%IyvvL
z!bn7=SCBQAG*^;OXbPDRe<Va#;n&sd-?wc!kC@x;5|;tz6e1#73&N!Ydqn_IUB(m@
zH^nirFYN^EUU{drmt~N3UQfdUBwDSgn5^cvgGOnBeHNfcN~xz9x#TlB3hm5#{;QDc
zBQ235k<b2G0V$!SJ}R05`k&WVqPmUR&kWG~$*9$7R*pB49%<Z!R|pC9RBgx$3%IEV
z9^P7y+$!i(@VW5|yI>uv;kEp7a3$0+7a@vUY~`fwdpLJ+l-yv!A*~KJ7w8YzeYbtW
zRy#BX6gxq2aA+oP6c1o;*HplA;a10W@<SAl{4m3=M`T1QZi|I%9fGcq)YLTwM2J3*
zqy?>DFK1_0ho`dOI{P7#yRzEuWBu8iQeZ5pwqYRP(Q$=eA`OBN>Ck8k>0`<K^+rt*
z7q=kd(HAjc!SU0c8P_#S<O_DDl`4arJJWV98(Sv(b+VdC$KD_~1tTYO<c-iwJH=FC
ze7bd;EO?^<6By5_MULJSUy4D2YyI|j`9HSjKB)4v?#f60dlQF6{0;BIpM7{-2ft{@
zfL~;R!%5q0=){L1blh$-Q#f7MkN>&^_#%^W6EGo0LN1=ef)k2p0CM&07D8H<l5TCH
zy3;XlKlAPmK7p4(?q5EC=W!G1&xZ5G_!jtK^{r~yjDLDUU}%pMM3bc*NxI4zNeq9(
ze5vD;qXnf{sPE5coTN{@ySn1uZ@!)=4`wqX^cNIARDo^sc3Hj>@|4+hL8`;w9ansU
z$h*MR4|NM_o@^M{Fxe3q!Se98fZ5dZNOQob0ox<c3a}mG9=cReqS1>u2W(2n22vFv
zh)^r2b6+nOVJ5^ByaC{2sk_VxVmxmUoN62oDueD!bG@jGz!chQSA!G$dHa7R3}IJm
zLir{&#{-2!bRUE!YTxXxe<eA7T1Xf+>|1k4-BMyhUZ=g8xAi}}cTGshbl$GiCIbo)
zDf6O}1L5ZQyE3>(Y@*0%jH?7Q<^ljHHhteoQIP|4*37Kyzf&Qi<8PHf?O4=dUh9Y^
zR>~C4snZ3jI0Ikk0n1_WMPB1v&6c@hcGBOdM0Ep0aO=va=r$0_e!d&JnY$1OFvXPF
zgI@_ezmusxPW}}b?fuyzGE>0k_6btlWK$_$b6h>ip3pV}Jsc{*>nxwX`Le4RP`AIX
z$y_7l-{9(Isp!us=@nPQNY!DYEfrADw8&wc`xEdI4^FxsBGbiuZS=}D$LDLC5^2^8
z%jnjY#iyz-q(4s*nmQt1&y{?KU7O9(RsR9Ia2*yMM^52?|0URBg>tuyi1Gbbd==)x
z<<$JNZyPe?$5Uq1#oyLHK?Hh(57(x(@;--4>*N=T2<l|5^;|G{tt7><TXidvPzFEt
zvxgkvpN&$vdl_XfT`EtuM<mrv2_Tfa04(MQhCC*Gz@AjjZ5bS_mXKQ(az4bSh}7F;
zWX3$27NNaV;J_g{rxR+de{dA=h8B?4r)@KNZujhPHL(aipOEsoyCczYc|Lp5drAVC
z4|_Cl!NRj$l+TgPwSwT0jINMQ(k?eJ0HD~2?8sXxLw;@XxLZABEZNmbsa7HxS7F}s
zPcyb-zTgE6VrtmWoE;ry_uWsUhm2N2aP|y)JAU4Udg)ax&NB$%x&+wktGwN(eGJ$>
z?fjCH=fVTcriiAKU)|*(@9+5HgojmTu<<Xb8Xvs4*Y<Fc#|)F>n&Ei%2o~J?iijI_
zIeP~IogP&kjal}ss?3v~b9B?dBRr4`Zjfx@35;#oPW-@T5Gd>t&a2xT=^Ozvx48#?
zC2CzTVBV~e1WyCk>cTnrD(D86CIG7;%F4`5KCG*!YSBiNM+(EO@P4`l`&BB9tQVXs
zppu$Pv28h~&Eguyuf;Vb;A=T@{Q%t(a3>#35w_UGs@q6T;-Lat>+wpScAYJ5i><}{
z84o^2>jrqdS@+i&P>0nLy)J~`tm=?OILc^_Mt&)2?$wij9=M9#S+j+8)77xC&D&{D
zpBt^if`28~FJUz?-J@ZcXj9Y%gR)Gg-n>`)SK;y78FqZ%H(lT3AY8rXMwJe5jGzf%
zL{LO5`1)lpIb`z&eD4iQk7016&Gha@zBt}yR^$zYl*^@S6p|N?F_}*C_%tNAo-3<N
zO1)fN{0SrPpS`=>go*2BvipCfJB#-+EFeb}NWcE#(Q#mn5>4zG$EXzK-{ZY%b`m{d
zYuUBmhj-5t3fV`wx3Rf@n|Ts#g#8gU)swMBih|ql!5G1(?sl2B0w1BZ?^lbLHlAH%
zqeM04gsO!W?cMd}9O?5?{LOHA^P~fU;$OS%7U%xU+2*d7)%cryPixp}BHF<MHf?PL
zqu*CUo~CsM1ff4#_<bV6?;{l7Na(_?qWHvghT__iD#wfXp}puBiA}VJt9Tgant}Mp
zlon<eppS!?B<uRz&ugf}z_HN!#%rplOjl4W5=ims6WMdu8nGvR6^eJU)YyTQF%c&K
zR8J~T-sVuf!9BxXki~U}2%;hM$>X|96@kQni57oiD@B9Nh(Ptk#jDu!!_PnZnGit`
z>2@-IZ@BK2r7O-Dgn2nde-}Y&7;qH#c<l)t6&g<QkMslmDPXAo3KP};RQ@{v`5LHE
zKxtWL|2Thfv<(v2EDc11GQC~>-5lS-bQ(rWQG;U!dUAw=60N%nn&JC1!G^DczJ4aS
z=0nW?J=sc*2$F<+MSzAgQNdnqGsSWJ3E(-7Tb(raCC#Zt3@9-Sewyf-)h;-Rv-6Fp
zie7X<^Ttl_U>cBr_=*>QXR=>J{@t0#$eOTJN#sM4-}ONpMYAd2v$D^g24tS90}XNq
z9j7rZW97MtszPh@`-XrTVK{eP1zyRhu0JRCV?X}H7z56y3q<MUiF~qV;^1zXzPw#Z
z{7Z8OCQjQGH7C0jgdCqozEy)KE+s|Zk+r6#j%M0CAE%fbb>V7Y3T*VQ?g2!^;9lfS
z@-mGCTYb!N>olaE!D9n=i%1_OYukU;X3aGKm*bwB=^{3b7&Ie__xXs!-e79{+Hs*j
z&P(sXfhTFma6KRWYYs|_J#mpnyA;z#^Tt)h$H<~AVKe_k*mEPkY}Xxcs$7DI@z}bD
zoqdwFIH#n@cl`DX8vf?GnDz@?Q>OYx`H2gwdPrT0bTc(fbz0KZob9DKALsIk8QH^q
zc(S+z$o3|c*5c|Z;I7m?-K`yXeOP7s974co__w6~fJgXeXi8>=gw;|K+uXS&A#-W-
zcs11Mn*x7vW(FX`^Lld~n|m~YEW)LmpsFf@VOeTXj3f4qxb02b+tiq{*1P#)%ikTN
z{>CjG@KyKlu8JSTNzbKPU3huzM|&NXFnBxxM-CH{i`WuXkgrTF=@W~fTk;zDV2H+w
z<(EU9H0{+-D=I14OthWYZYnt+)j8bX&t9g-0TKk56NS6{es&AHAIm6zz2)AWS*$Dq
zH#2YTRT}NO0`g7fC-}l+b_$PThz5wAE^0K%+QK2{nyQIxYvZ3~T3BEAco6nn%ldJ@
z)*&-L>l=K3ZsuH_-q$7&`9cHsp;Sq`+N`B2Tpvv0{VrB(9i_-oHio)wwnS{7y&<m0
ztwIoueoMic=-#g+8>koi;5Cv5CR{pMvLtHXYsjH10eki~oFlkYn;ZH?&#=Sa3KJuY
zY4%Cqfc3HnAjv+VeVb_B;TnHLSu;d_)<I+jN|U7L=&ylIAN2Y}j35bo3s~B)2}Q9F
zsY{71=O190tkpvYO^5Q+P(ia~dpK&{MRs)FCqTx^o0JsqvGBLI;ph5}>}{;25?X*Q
z4OkV|Bgq%DJP@QA{mVuG*0rcS6Kd+}=K7>I+~hw-hX}E0h+DBB%h-7Kum-)AJa9sX
zi9k~UWS_^g{o6LYCe%6Z(d1h)KJDG5S8|=ZkZT8*wvlkyW@xsHlMJ(CSESC`8G?aR
zb#)`G>c{(x*3TYQs}2@<gG<GyzvTr!^z!X+xyoyW(dO00O}<OjHUEGvu)UhtLTDgS
z{X()=V_esnNylx$yPcXws?vy6!01&onY~6R4EGNN<B&}+V++)Y5@Ork`Ywl3nDzah
zsJ-r@blF9a*Z90cWs>-l<42#XDQ^S8A9H7X9=2OQbQMmY@o-knO86uPQ0UY2hHGYn
z4rT=V_}^C8_i%+v{u1VsI$CTKg~chLsQ$ycJk~~n<)Lp5BH0}t7+65(DCcJKodhGn
zM{l24<wUUk9XKm9RFW1Yr+f|j3%}esYs>Fv1oZ1MK1qd&svPI=j71G*#jE!KREPU?
z-NbnAhUodlDUFP2+Nk?AnHDAC{kx|_LL@jd5OP%eYLOCfjUDi8`LY<+LrHd(MS=QN
z{5?qe!B_B)jyAG_D8#SuT+QusTjY}EvPNlOrMHSO?(r)?FZj)D9Uj<c_kDv71*99D
z5g>pdpHc5KS*;kQA+~V-F0=Fh1F%JU2d)hPqD>hz4ip{(GGi5mW}qEV4r#L?oG9ci
z;y(hC#4#fh{YRp~|0@1V`R~9yu)J!J6R{jWQrIJ-MIg+=vyM#^LcQ?-NrWbDdz~Zx
zgVCivEF#=g0+qo-V$rXusTDTn-A;qxtZv<fn*a(}KhXA{XUePn*LUcQ%UMBuJA=wm
zEQ18Zsfp_-k?bT1{6z>~3>g%(Gf7tjxl7$mvo9%GL?%9WWB%qQ*vVh8iG3C9QUc)$
zGRKVm*%JrO{@;KsU{dB;1u*E{fe@vxSpD!kk#i4b;^-ZufH8N)xA$5F`~R8HkwN<e
zL#&n8>fm^k$6%4lY$^iPHjGv^=!{MOx(&ysebg`+j<<ZqVx<F|1$Sq-{3?8b`w!Ft
zkJ7-6aPsn<X`hBZs^R6oyQES9R|Aaj-X#PzVhON+#gt803#lZnm0&p6bZsJF=+<GT
zUQuNxc20nE9Q+FlIDirMfWlDRQ)0<#zEOL<7zyFmnV>f9L<v})#Ih;ayL2oyrrcWy
zKfl7J5ehRxZRuv$INkj=)^lZE6M32&t|{8@iyF1@ZwLi1<(Nq56lF|13nKh^g}=!|
zjF=~tfACKORmzMbh@wgb?Y=LL=6vOe%zveT<umFl8{K=0nm7zr=nPg%0&V*F>x*f+
zb};_0L~be{F-x~ebANmyaq<ZGHfE@jYhT`~39-U}n*Rs-f|zo3W(2oQ99z^_-m(*j
z2lhn#87qE(_8z=rqDs;2?JOJ(3BTNHC#}QG%?16e?Pk7s!26{Ais1uH9#jQvk>6n=
zxujISFE#7{X6JAxLu3Of+8P_0NJ9IyIllx34VaU2z}#p5A!dx!zLBY+-c3d+iU`tj
z@IIuD7N@ISFu-bFk7SjP%R1S25_@g`IIDOcWtdRm$x=n7rUbwBko|Xod3affm-l?u
z3{Z2A;cUa%tO~+qau~9AmbJicq9C3GPD$S)aD>%z3>4}FImpc6t<>%x43RQYxl;R1
z;HJ5<3FPlY#sRL4rI?MF9hcM6D;jO8xNnAqZp#S^$@#w}`zp%N#S=THa)|KqeUDcV
z-k2nvpkLiO3h68$k2{wVo}(Zk$+gSBL9M;cO<p!>(fznFXWCDF{rIuklvX64J%|?W
zKvQnv;M2=PbjYY@Y$deDS{NCR`qmlKP`Wq7K?@jh8{hu3o8~rb^Wc;4oH3n7{GLZj
zE#~+PIwMpbXR5F+05f~yk*vr~z&B|y+3LV6nW9QK*`SY(6%7F24=fW!epuc4WYmXv
zO@Z-B?wJD2v!^@nQM#BFwYw^*iU@F9$L~({m_4t;?qe)z%LirwX30k7CG<gDdj}+s
z$m{wlPE*ehOczy(=dV~&fK&uQ17s}U3LE}|Qz8&sg#WLQf+?<yZ&A-95BpO!OG$SQ
zxqf<Om{57*f7z+z>w9upHEHtzR5sD_JAxz<UnDipXK=QkZqNNpUAVS67X2}+!9#2%
zaWGzepisWbkwXzA0XnCU-Jy}XAa}bB*}vDtJo|Au8@kfyA~GVu%n`IeDHC(ALl=4_
z7@>5>$CX&dIm7Ku#a57SMm@;t@y?Bj=u`0=6rA85g)J=fC@RS;*D`~B`M#<TXbTFW
z_luZ*gXEZB7mF(mad)ynzUVOUo(E`&Q%8zW{vKjn8RB^pMhkePtS;1t$2Xw~-N_uR
zt`T4jO{^{_VP!X!%T2x4d;EqOTH?lR^T>OeYQ4BesC1We8XVs`Pf0z8LGjQB3c=kg
z%o)G&vV?q>{-yXK30xdYBXEZo7lH>K$I6$r@1Cvj>(*U9HYmceu8VV%o5Y6XnthRx
z-@d594jSjKm0$JY$lR6*sB`yr5mnd`nDe<n#nW(=xqt5P)DPS|xuBO~{;3~#74zZ7
zPpKDBP4;}gk!`DuthxHA;zPIa-JWtyGJ-~{ASWG1lkXrn-i1wnKh<l?T1jUn3poi5
zUec*T;FmM;S!a)4D$T@ySR74DI#ZX?J}|?EvCVYTIs9onT@=QKL15U_!TV?qHQnN|
zCH&aWD#6N1Zib?yL5h%YXqQTs3Hc44XFq~N;I4=!Q0S^Z>wk}jia{V;K)lCY1Q~dx
zCu?V!OT<;@czU3BkaPkR3{if_>mMW2%a2v$a)h-#vJ&jez-6MCaq42ox&6LrzV>4S
znDFrVs;a+8f6{NTN)V=;Cq&z7IgM3?y01F4ofhNvgw7b=`go}mR2#8*E&}%(lXL99
zsG4G*G(>dn@6C=5;9<TaPPuvonlkU^xh-vVZ%lsD=_muUY9(uJ#*OMr=fCm-jl-A`
z{B-dj>0_h?<l0yRXfqe6yreox-@*{D8UMkv^nDai5ciW|6!9$pp=BL%zWZ^8T?c)3
zh%j0Hy`aVmR=ci@nmeq+p@0N6x_4v>n&=c>4xp$^W?#4sukN2kYkXL!3Rn6Y_6PhS
zWh)NsU!mA+V+37pX4@+HewZX;+_&5&CEI8-v(q|{()PYIJ%76x>x^cob48Q4Q43#o
zkV4PF75rm7XflcvTBcUF>*47)8Fd?9Nh+IbbN_wTJPOfVOY+B`@3w|>S#<ThVFXSe
zOfUXWp)(M@i5&ku(S!rVL~F9TQ%lIT6BSUQ4aNWcwH=&6f^WV45WTm@2<W{Telw#D
zDWx6;8f+vv{*`HUpBO-2=T@P;3!U)(q5>b46FPyW^?E|4lD!O00_Ij+-R8b$P3ahy
zvO4awlm?Ng49dQzzR$p)VYhM#d|z%F4}RQ~X*Dl$bdVOOvtsc$pRS{ICj6ORg{dN}
z9vlVi|0+!4BQLJV4~s@RF2jdV%DtkY(#JU^_`O4>z*-&NcaXPc$xmf})bv%EPRh(*
z&$yD1eCewgk0Bs5JZstcTHK|kjSPtK%5rd{zeAr0L|#eLFOq%(2i)32dJx1N0`)>U
z^{G<oLj-h+;Iw}$@(H`VMehJu|Lzw5iva_`+JRM=RhUc&|2F^WFd@8l+JXNz7vns9
z=f65)kC~?{o&X9gw%wbA1@p09t3r(M1l;=JjZh-d001MLAh?Ulw~T5Xv+BiO#DF77
zRTqE&zrDz7U7<f<=$U$&X3OZI5bw3h@PEA8pT8)5MUecb#sr&e#lpz#aqF$G*Tq-c
z%@BCEPQ_JsxTYo>-$4C&Zias3b^i(7EJQz(_C)-#FH@b>R?%W#d=0|cdt+ZH;V9wd
zW94M6t!63cWGN&G#We8zg+;41c0Ddz(i0>HHA~*f;S4^!jo4mJ<{_RV=eRIul0@uc
z(>eTmN=%R&A8wp9(-YyyE=CwMK4RDWF88N#l51u(7_$q){;%lvve^)I{UcT25kXTu
zPYNAO3O*XXC0SpuHm}nx_#XE2wm@T;xnC7QhW;{CzWj3CitI}xv$e)9KcppO54P`_
z=^`xT+)<G)4(}ch6%p=g9p(avObe?J)|Z8zmbacYUK|RP;b-vZKd`Y@gwhD5^)l%c
zST65t;|F{mkJ8sNbXzHFH<wQ8gS3vxgCrG0cnUSoZ4LC^UC?cJ-p^+cGGOYOXXuE`
z1yfR@E&i^S|F#%rVur}fkpMPMIuiao?I_l?FE1r$(#ZfBJ6%S_GHIH*nT3?a3>4h3
zln^~(v&$3KGnxC1o(=*(TiN8ePRE01Gte$r+6N&R^A-cN5dF*?SBFw~7p&lR>M$B>
z<FYC>zcS%z3~tN5oKzI;b7~!$wuMrUL0~O*w2TJnrj#p#<bYP3cM9=1ZUwTVKX1_I
zvMJN#9i!PgF|*$J0*f%IDs`{VlYn5NLmZ#I)eXTA^iIzP(2sZ>LuS&l-R?7^Ij9-@
z0^z10<nyANF+rmAaEP}drwfyZvkz>7@#+b4jo7wgp4ZtiBF#P$S38bj>KfyHnH(Z<
zkv1>-zJ&roKQt*p-`mPIC?32cgM9GdQNq`Y@-&-B>Bl>n{`NZXK|vblT;h9~1rrs9
zUuS1HD4na9{Zm3oB@a=R^{}2ZF^c^~JJ%Mj6Fq<H;lcY5NT;@iC7aFsPjK@Jz&F9F
zGvlawD2;n?gi~S6KsjT>I#FF4!LyEdQBo!e+Xn-U*?hRpG5lP*c1p0|sC8DPC+92q
zN@39eBKOA*cn-<eNy@cOIo0emkX9O?%~YWuF@QZ<{OZ6A>PUsEdl~Bj)mZ?vkC2^Z
zt<Jm22i86xy0GkyDzLE|4v@!)`lrxPi$$_Qq|}}*=j<hk&NYS7^ltMi%s*Q74x*yp
z_6Z(Kz03{qDee%8X1)FRNTQFWT~GqMwf&&WGKT@K?eZ~DEn2cB3QE5iB!j6NIvEaX
zlXS7)wUX~}>&uE(ZwYkBb@>^?YBdS_tJ?Z&eZi8+EBfv|i8RP@&!wzs>F5+sOc<QX
z)vfmDskKa)v0D0&<x3{DX|QJ?XxZVa$Ip~V5>=<*J{B^t`&W{;^Al{S$=A9+Tc`I-
z8^1=HLo{|i&d(X-1FQ0#J{96W7KR*O*VzmlK+9T{Jl_+-?zLUqMZqM~du6cLUnWJf
zK&$sv@>kQ<;okSqSfk5rOQLk*=kS>IqrSmW;*KMf^3thVW+S4JdNDCwXoUczSbHnD
zR=)KlWhQQ*?XyvDl6)5WUkBPgLW~{4{n*-YNwcrX2XY`t8uN%L3fTv9;h%qEm@ym?
z=dpu0?hbFOJvi>LmRRHHQ3t8Rk@B;95-xBmY?4E#leh-W*d_Bt(M}DKZ(!<I-(W&Y
zMwMp47Q;04`Ao9Kf|p#XjPA)kOeS@vN!e9loYzE~C9GhgmAWEx{y5=n5cib4P#5HD
z`E>>7ZPSE~Guc6)7>5b@k9k+qyYF8N^Sa)TXNpZ@POYX$Enc&Fj0kF5YTD497#HX(
z`OME{qeJu9{E!*&Q*4=d^v4WOewhwc2rCkSYxtC$aVD)Nr|ZI3g@%ykx5aAUZ|$9~
zbBjm3r_cjCBQ3bUV%6=&56~tTv(5^o9$Zpa1#9rpi@8pJj)QPyDi}^wT;>qI#muZi
z&oDijPe9(h;)e9Zg(O~v6t-7nHe7Gue&r+T6&qbXm!*6z7oDxR{E8HQDOhC(OE*~>
zY)t?Fl-~AP=W=%khOc^|qO<Pu?-J@S%0=S>5!a7p?LZvQZ`^)hWtL159LtK=Zm|l%
zmTAlA*6bCv*=^>z6hB%m_|#2oNpTSpiscT@QhdhlX*b@+_x@YUaxuOo{E{02BPt?1
ztHtc{F@*3aJ&>#=La;vBD4_=#!-ODlm}7?hm|6Ezp^$E=PBxolnNn8MED7WO9mH4Q
zo)v4l{}^XPSRZaY2MqdGV1z-;rvJ+SR^CY|5oJu^hx@;l4-;m<&+%_U>MPjZw5>FH
zd^;KDQVj1Y`e40VqMAU+U-um`xi;A)L~Y_LU_pU@CR>xn*Oa-=pWcNLP8=DJW{tVb
zbOO|FTh>EYWK7;J9B00NGv~q5(fdh1<W~*zspFa3=pS6auCCLxS%gj%F`WNOaQu_p
z_^$-VYsTY$DW%Z<d*r|KAs|p&(I$j3rHNw1H~*GHq=e-c4_-e}jvC$}<uA|^`Cxi5
z31xVPQ18r6DJ9yzkYiD0AlP4d=7keCY{>=o#o*%kh^sc-lrIhihFUCzr;8`ueb#Wh
zOWxUZbKspoNf4sOZI1N~tB~D_f6tSdNj$v#c<!MOPdaNqgqaG)N9&c5d4hSbSAG?Z
zA@T080ZojRSiv+NK*UyNm2$er{H_5UvY<=r0@qLx5@$%)2tKIWN$L1fi@9rsJRB|D
z56nL5sv*#|Ua3oCvb(Q^dxLFkXEO8Fw`A=R*K|{N=ssPCduXoDE0?>MMB`ofQ&1cu
zL;KA;8(S8Tq?+Zy3k;Y?JPLmQM0}XqllTYByK-we=^$R*=;ZnY^1&d%=ZibO0M@VQ
z(#1TgjnTPir0@HaP=s5mods9{M!kAx{KG#hEP925NX5B#qJwB|y<<Uz*&a#GHq~Y@
zOiR$Xm57J(KANSFmI%jb5a)4rS`B!;ldiQBe-x<qoWcSyc(Syy25g#yeFJUE)U=v%
z*4NXmt^V?{#E;Ms=-c?T1ETU_x<{{(Klq9w5fX?205WS^<swURhB+JQU{#%4C7E(e
zCmHchm6A{Vo?D+QW?9D|i6GM>`<85738kye#f%sTn%=JR0j*r!$9l{0Lh`}uyA&k-
z&`Lvzk9CdU38WZDeNk8*1_t$O#}szzlM&!41}e~X(P*|t{+1kR{}Gc2pGi<+d<<Va
zssa3%)XVHB<_|_D)*UCeXHb9;wZb*|$hYn>bNwJ*`&p($Ln)CvudS(^^n*nWZUa(R
zF7=UB5#jeyeG|9A(_D*d{;ga;mHJaVDOFL0FZSAYq>3AhnNF2@$O>~IIxbCu10n48
z&(P&iDUw^X!f@n(Jwx7CKYD4(M<sknVBqC0@LO`LoP-nX$HE6En0&Uw;|YY)CmO8%
zBr<)HW%O%0%CZ1SE|74nXa2X%n<ghn@gwaU9nDL3>fb{rtEF2Y<QflwIWd@n6@*gE
z7$5Q_w)g#$(ze@GE*Z-2naoH1NQi#WN@Ij8H#5d=co?zvQ$vNwGX?DD$biE#v|F|n
zXSmj=7^fxN8F7<evZb}!WRA?dZRsb7nD_;wl$?V~+{?ziQP;<#9e4`e7$35n+t+8+
zF~79RUxMz%iSWU4J}*64;(z&uBTF56e1@WVY?tsxyE~<3f~suG=VPNa`BRAFsd(hr
za@3GdLZs+54$IR?uv#EoMa;B{Z8y84Nockj_mQUCu5$I?26MSuOaTK0{!05MvN9|4
z=tq8_B64kTb_=d_HU$POmL@PS8|TzXT?45~{FzeS`6WoB*ZI*7mH!1AkbX*dCoP48
zyV}P-KUU8PUljUGH+NN?wNaRl>tSk#@s44O7KVzAJl~97X;zGoJH!V+AL9Wb6_#j)
zhBpin&JY-;yDh#8FIOmyMzS-6y!~TA6a&iH90B^jo-deknfSfY@{{20IO4(aVAVoh
zp~oN3u$A{O-74GXvofj4^UKD8NbqpnO+OpUjRd$&LHUfF@%RgPMJRdr=q=OAT=>Kc
z$^gx+=+!sjUl_{uYPkYQT-iC5bH8{9dT@Pck*u3(noLu<_gT7mSQ!~&L1<vKNu0Pr
zn{VZ@+rbKaI&(iAwh&f|k;`=;K)s5(6q28sB?Sk^z9BOSK57~f3=fR;$J>mLK&b%h
z!Sb}?w7P(?Q}GHpqECUd*fDvo6Kj!%)>_h1To9r{8xecvO`YqomzMY1Gt;TA&r`*v
zdG(Er^YE-%U`^6emfDT#@QX|(ZVOKL*c1W8=nK>7dY_n-5?g|&#d?k5#wgK&!8z@v
zKl&#xGv67b7_?QAH$?}5&P(gJK=6`^i(f|U)C1PM{hChiE|?PX^#1Jtu+FmVh4>+Z
z^nGE?&3jxE{^Mfa^x)~@0@6R1gp<Ip_g0%E$KCa_al#nXFGna(eqmbPduL_}^6Bm;
zf3Vfgvso<XT?&k3P&ne=e6z^s9D_dh)$O+GO9i*CC{J?!^Q$o-w(zmGp*AB=CG!ft
z;J+E9?R42DBh07nSfx$EnQ4zJxi#Ta+_Nj6C?VkVI-BDnJ~`0!x!-On4b)jYx&s>P
z2nI)TGQst%l^zQBDg8{c&AeOcs<RRIw<emfnKZ#2+%z2~lCvVhrR)Y`x2CajYeU|k
zBQwGJ8~!4wG9_9icL1;UHj9iT7{L?nAi-w9R`S5%H)wWyUAT2J^lBKzK+%{sH3doI
zc<A<Es#t804sI9hq7yiwzz9y}bhM&NJzCs;HF~bB@jjfed?I1Je6gfL7p>EXh@Q_?
z;QbCbN*gq=&YLlNYOj~IE0a!a*U<lu?VK_RF@Gw_bf>ZsEzBu=mU&`LMB~6z<0cpN
z@sYW+*L7$iH-{vb#uO2z!f1ZM%>v(?JgvQ|>*TB3Qqnx>BHNVS{4<B~NQc!oOy9vo
zD6>Ez*2Q~&u7Xtyty2Qi={WI|g@FSxD<sf-RYyl%_$UmT(a-5FEFW?iz{%aF!Ye2*
zy-DJcQtxjZ5MWI-3+%YwwfHsgU@VqL{u5f}W^ZGoh3p)J<L=&t{A#=cjQun$E58!;
z-q$;tHHF7E#k+05lsHER@;effSPy1ZONaY4Z@E#Tl&=15&JJO<h?qiC9L|zC!blWC
z0|f0F;q{;(&*d%*LAe+GkXnA=bS{6jMdBP(#@K~C3;n=+R_Ib}nPtiXQ8&4<Vv6%v
z=~0dR-67)?FHUQ2FRkdAkf6N_4#mANRQQ7GA)RjjdKZub29Xk~m3*ygBaLan8mz1e
z@7lSld#h@|bF5c)LxoHM!$qn9=QLo~D`VgiP1Ndn5wffZZSY|-b<J%m7!bnU&(pUP
zy4Q|yP3jD4BfFQ+>Tp4@IJLPTND}0n9lCTPrux)HYjp_*=M$D;aq45sOE&gF1Xeec
zY_~JBng+qbY0i63(T&WXJ`|<TtLz5YGs8-(vAKT`m+d=y>GK@OzW*b2bg1eJuGb`d
z8W3a~9fVI&Z>MihZM`RpsUBigoQ<-w!)yAjK3-Rb96QK((^e{dSD{E7&Vq92VV~$L
zr_XcU)wUvc$z89xYLqB;B1O@mM@Zqui_hF8;ZL9>F$!I-w(zqwe*!~AK2n|z{?rOb
zX3uK%K3T-jOwluH6{|ZIRBm<}g?BMUh7pe&x4sp7<_jcb^BG+po^j3s??1@x15^%F
z_fT<W`OidjmE@vvbmS@n)Q~+D`5jOh`+;NBrrmotNWVIOmI*(ye5oXVM66CjPOYfi
z6ugVrg9p~U<T$2k2zSA4-H+;g*GxD8;2kC$4p^-v4nlyG*d0XyBVZz7gW7}JRs7?H
z<WM^?QReu#@d;>IE6IqNLmNUp5@%YTLwCRw@K*rM0@S`+>nE~)*AFxc?SQHj{j6pR
zk$|c~I>vC=u|MTA5ob-i96k*J#e7inWrtRx`h=hrI<UqcHS8D?LWW)kC8RYXOG)TC
zD6gdv@K5;Ra5AT7b9~S)s0c}<g*K*i41$^{un)wEYTItww|u)}aYx~XBzimAXzSr>
z)Q$?pY#hDLtdl4R7b)D09LtbHSM_Vr25L(Zcb@twuS;GMD$vPvsA-ozKUJRs3Ue*p
zWSSxDkh{bCv~?ZLEn8Pf7&L9h9m)iAae+(YgL!IW_5QO@k4Ul0CYXmFDgHzw`Df;x
zWYj1#2ee(k^`m&<-Bj!&jBaM^d7jr}#z5>T-CA>DQ1_Bjh6yz%gmShA-EXR0#!dAD
zA=v&}%3DXIEEaH7`03#NFjdJAbU~9*cX6czyoP;2CrCB&CRqxk`k_%2rp-HGJ;xWA
ze_YEm!|`01v`&4lgS`294Y{hvXnuXLF<tf(EDJ==E=y%h)$2)@3NjBTL_22VG%>TQ
z&<4Fe1|(Dl4HZEN5O!f{By&Ic4)FuZ-@VW?k!UTW>#ag|3cp3aLqrNoaf#axFBL~}
zz0lJ(FNh_R9K&rf1$8$RS92okjh@e5&QCZhWSEe4JW%X;)>R^3FiN;2!3i>MWiTg*
z0}sWF$_M=8>%-jrJIDkf4Cg3!Gf6qtlG%Fy=$ox8B+RSoi(SCSo2(!tJ=;NF>x^lr
zy%Wb8Ml9T{5l@9T2|~$M7)RCpI8I)dYy^8dJuNj}%Lwar9X(f7k}5wIpIfoC?o&}4
zw~|eDO?V}{`IC^dhEDJ6GANMo#X>1z0+f(IB!AB(l{CfA(JR%iNneRfiO_rblVwsM
zPZ#Opgs5V~A`%l$%p7<q>zPSsS#Io*`I|QZkS}@RY|L^wf{C84Y!M04<Apz6luF(2
z1+@{-OhF$8X^lVd1yY+-$qNaWQOAGMhE<G%f4Jjm&b3r<SH7lr5pwnpeohF?2(>TI
zNUrxAKZNWxYHG>0kmIt<PNvTy>M~B!?mQXIny~&2nLFve@bp5b4{o~j-O2a)*m-Q!
z7?dc9{oFkC?CJ2Aqg3BiLO2AeMuZTiJ_Mi5^sW<L^zq$16SCQ5+4+gYxq|?m=+4<W
z4h$&QDwpd=ak;E$Vd}Jxy(?wRRP*At$Y9zY5;#SY<ZLHcWjm8o7*43u`vVgByjYUR
z^wTA3T40lGPGpvl2u6YQ8BbA6_p6LWGT=u98ug)xLH#F0=qH9AcXI|D#GIf;LMPnr
z!Kb8Sou!AEs$H<nOsy&fY6fnDrH(q@P<qi35ROKs4f+x)qW*!*M3zX!gPhjiJ0EBK
zSExo$L$C@*D@LyK`0{GQ*B)Y;9}z13CE|AKPSMvLA=T8D2jUUtwUb&-T@CMpDat<H
znAVNLa?lY6et>6eP39fik=iT<>;k4R9L&_UTBYR6C+;?`HlA>Dy-ydV4vcEJ{_4Jh
z*KDoK1@BDVjX0$%61-8@17RTH3o<S8W6eZ8LKMx+gYmbJ%ANqcgakp81SeM%J#t!C
z6oePg*T>`Cw6K6I?!glqRVzNW`(ClVD~iE?M$0&7p^s5yP4Y7VH@RCBW%v0ezq)5P
zC~pj_aCT07O=$`TbQ<|_^k5B;?KA}h2=&y1=<M0r_-Xk$m6Vcl;+HgLGMT5yck-|^
zS#Zpx8S^qJY6VyzYW$^pfX8r1KJNyBTq*nQ=9{Xr8`}eDNKKEpN+`N&3>dv?Xp0jh
z{_5c@Fr%*sXAIi?Lm0haEady{j2!D#tAalM_j)&8v_0wu{v=3p8jVT=e*@%u_rXA0
zn#d{2B#_hO(Fs;^O_swFcB>;`*4TyV?5Yf5c1S79Zu*xKAjq0DlkXfNUYB>TZgkum
zfn74ZWo=GgsRzm7>Vq+b3O@D)eNfY;!8Gtq>u#8X4@ltk3z+$@`{Gy|Odq{^O(Onf
zEoD0nM?PGp)spuAGTFJ+;6!VKm0dr%>-XcfkDagd#2U1nOv^@}NjWHJSY{c~;L7d2
zu4DcQTSTbDtyql``R512=x!p<2$cx>7=Rbq)Q9g%uQ%2)#S8e#kS#|G%Oi)|ua$|?
z^%A|V<uh%r(yXx%HjfU~htuo`!v@GSMX7ccy-bu@a#k#Fz^=E`Ct9Bmuv0ah_Fcc3
zke_z1rsT0Z>>hu5<NOf0)DKun`^Fly`wrqjUR@a`AzrbSqac`MuEWxJeY}cBfHeI5
z-Wol_I6o`>BQoUcKL%tv>V{A|$ZhvKMLaOOR!MNY(j&^TLSGHER$|Z-u)be^>t4HU
z5qO)8{|U};jHIy5Wz9WYg&PAyHzTrr?l{EiO7{#3_FK}hXPy*4llg{IBSUY@_Ex(i
zT$On{g92U$USpf~<H`OjZNz^r$JKsf>)Wry7trN3hcF+lkf>-OaX~{XQ(af}Q@%5+
z)vXzs1niNpcSA|~4<tJ+>2K~^Sun*WBA$*F)4?_o<0O|Y1#6%0CJYq~o)Z!tPqGvH
zLybhqh3zO^N%dgS#1kRd{=N!Krt~-K4~@|aT_UBP)y(1GPtQ!7I`voEQ}3nwJ;{FO
zV%)o>e1g}5X`^}@y@l1rKlFR)XGPrl=^?|;K02m7>-dktj^1IXrYHzS{Vr+>Fp3XG
zGN^zuAbV+_Ya=6i8TIz*Dqy&d1OA==L=PQ`{*~#1QP>#;TK>|c(}}8W1m^Y4;dMQL
zT>p#cM&Dt!+uZBiLDsCp>sP5W^=f17uH6M}3s2r$vMeZitj!Z@7E<gXM6omfpEA8X
z9Lg_z-;y;MsjM-!?CS{G*9O@_hOvtwOOk!xLr97l#!jdh+YDt1WoO1N85zpHh3t&w
zJAJ;__xI25y8k-wbHC?#&U^0ndY<c?r!)2S@UnQ&I9bOJwgaZK2UZE~yz&ehO`PS<
z%6}an8Z-%&&a6z#@BL!4tNR?%oxkLt5X1UiGHqPQlkVb+_<8%som-<`haqIx5dwKC
zv*_~7Zo!Y*X3jV@6%MTQ!UE1{q~{8|M&psT%*FL9)=c{^8|n^Yj9fpI-#c%(aum5}
z=#pdQx*OuvZ)b*0H|9lz?7&b~_OGd#w$9TZF?;4+<ixHlUal?fhPBa(?uCeZX$H|?
z13E&G`H*7&@dr>rr;*-e<ThbrlB`4Z<ax^Yiq-M8i7%M=;RVKsEibYPL9Y`A+V{ej
zZrkDEIeL527lDDY?(f^5yzhqalvBK*pzjIQ$$22^v+DY-%8iWK7XJNLv%y66s{7sf
zy5?Rq^UhG=m4+PQ^^f+krgAIZ^4{K3ql7^)!90kP?+kd<HpiGhWCWYmXbmMsNH2Wx
z4}OPK30K-y*~@e(k?zeN4LF@5lHXz*V1TR3FFxV|KC|uv-8_se+JU;eKxbKUpQg#a
zTDkE;TpNMZ|NUNT+BQK8!Had2XqU_X-^XvpHVW#S8U8Wg!Q-K7rlg_|8Sp_aUs9VF
zn-0rcdm&Zzzs%=rmsKyd5Kf1b!$N3=7kGdR-iGio_6Lk>-7u0Kq`j2*sj+;w^{Qhp
z2S=$g!Uub|_hY;`=aQZ)eRr#X^?>pp+~_*XWcAqyWc-Bxm)LQwfAfim<-)xe>9FP?
zJurzi|916!D*U)v+Gtx>(+m#WS2W()ta;TvmjU2Nk-CzS!l|pm<v+|8;Ov<7`q5;P
zy$_Nqnz}RN@nA>@Ak91~oB^HsB~%||_5}UBq^e^pJh=B*q5MiKu-_uGo^Z1i;4Z~g
z9QN&vvfUrnt#dR+*zL*jaW8N#Ko1~ck%f<ec~N%8R*Nd>Pd%k0PI1R9Jo>}i2kLmf
zuyCW<kl0a^Wy_4^EN2~p?;xd=JV17_v#%FL)hEw11hSU$nCXM$3n;P(&!)!iWs1#7
z-aDyZezOTuX9!Z?o`I-Zne!{*3tYA@-ybsBI_tpNcShB(c$FJW5Pv_Y*j=#I2!zVt
zVGBTF1GZUR>rPA&rcgd_10<on0F>fg%FAm6>vR%w@+i?3ZBixmC!c7!V{lQOt{KfQ
z0_eijZS@~TrXOi><6+frd%&k@qaWITSX#{-lL^wSy-t$L4>VJpT8qBM$>RTVK7_wj
zQVjS)RqMdU9iWLQruz<D?daEL>$AorMK8_*?t3CtVk+$CP{$%{9miFyXojbm%~)nY
z-7Y0G;dIVT{b3)d-Zb&4t^tP~NlwMP#w{D9lc#eHUhG5tgGHj5FMg{P@VcrwxDT^D
zUvZy_3Mp9`Ln0c!C;S3g9BNA_0>(aVt(J&I#Mcv2XN__N|CxEQjh$Gp!k-p+0Q_bM
zlMnP4#<^EZQZIq2a|6R6?82TbJ2R#C=s#<E;)Z{-*k?6y5EljFmSNU+l5c6^bDv&8
zO|`Wg$v73gpGxm8AN<;`l$?!sNdiMCNdiX>eym>1RqQtM$R;NZ%gT-@Z4q>&xTJ~2
zbNuNpi*s<|siI(NANn2@Yv`JMP0XCPfa2B4KCi0{nJcY2B6|h0C*hOp8oTmiQcqd>
ze#30c`ywaGXQ@E9CQT|9I+=VkF1@M#0@=gEXK78ME227{RXn2=$iw^S-Be&cKh=%B
zBc5D^NUCYZ^_Kg-uwmdWTxyzHOETI{PRjS@GirCs|NGPtN(V^koP4|v!WQPcAYP0~
zt(^Gszs&j#o-At{-EPCRiwY?BYuaGa6(+1dJ17%n#wD42kb%84jfMpJUA*<0;)YHY
z&uLgpyeubU9Zju-NujcYlkWHp%To1v_HuN5VUt$df&K1-f&y+c{Mt+7?Wkv_Js1X_
zz?PPP0oXI}D!R;uW_rlKhT97I+Xw-xiTy#9a~>V|n;q?SG;2Fa_4wZB_16wiC~Fdp
zwPq2MjxRJU>#K*xs%$}dep8(!lQt_q<9_6$<oS%)bfN%Z1j4jNWxBa9&3SB+JkPIO
zpD0ab5@LAJ?o7r13uGSQy^~qw#a=9lNtl6q9$A=P9bJ#>^P|_1%}?fd#xnst?Dir3
z7jpCQYxoAiQYa`77C>+d%hHpq#I95(Ev38&p9BvyNcf%nbbEQROLuWd#w(_<vj-Wb
zUmJnpu`H+$*>1VYy>_QGKz*|bm||u7@oBxEt9YDBU-&dtM?vYVpE()Alt-9evrTcU
z-&Kx>>WJdon-kJX=xf$%0U8a@B%JXCA;wj)BB{FpjERK(7s6h7UauEug=^YsB(0_S
zx@-|u|CUfG>9_gRT{Ot_p{n0~?$3HNqQnm>{wXZ!XNyU0?5b)?szf2}oI4mO1U*&v
zrB|{}T(1WEphdwbGUI!RgWAewXOF9i?nT+OL`p=+HPz2s?GDCfx9l^z1)M8){8}zn
z<;^-Qmu^|!=?77Wa~9-@V0i%O*nk*@@!W|-6`8E=Td}vF(5CZDMAu)=Txdj=X5_rQ
zh@ock!M6jS37Sh{GGD>*cd{%x3%oYfRfU_1+^G1`*ezje@am(-gYut3?y3l@1={cd
zUv3Fw2NDX^%fy{OJ)VCo&d(Dpc{@iJlni^fotB}0vh-!vB()4$g@cW)C&D`P&~rx|
zj3d|vc&;r?4wXk6Re7>6zv8J$h;NnD+1q<888{5{Cr>=-TEU@BAe2!zU|KvEz5(is
z!(G<W&UzuUvB?7eqMw;#@1q8|Y@@bQ0%D?(9p@-=oyYjkZ&)VjDWqz((%G=l9x_yk
z3CV0<N+dCpN2Ul78*uhZ+;@{J5kEdV=~}|?%qySZZorTcbIuM~thRy})W>m3{>~<~
zk0Ao@tmG@alJ|ZrN%7!s8O+<dHilC~{^hoBJm@8pIoLI{IOX(vHkY6OQ=$*#167_|
z5HOcqXbb|shasDGbS5K-;wTv{ZXHKE9@DH7C@XDaYKqtuI>sfir^ZF-fVKen7agga
zv3inzmg)goVzhopuIe7gS8wc!yPI&Q8^d5qF}CY{G}rhGIuhouYmR~*Q{S*aaGr;?
zZZ@OT=VjwrzJ~I3&llAw@g)y6%Jt+7k#}^NvKb8WBfCI^mI_bma_*K*bVmp$)c{CK
z{KpXPy!UMOPlmIzGq+Y7(N}xY6?C;@1FiL`qGk7(jQ9CAn}e@D>88JHcyNq^pXP><
zIW0wEy~9HZQQvj_GqV8~dYQVnOz?Zhfb78m#Pwt9zU{1)lw7DW>Z{E1N6NGd$8c?T
z76Jpu7o*mno}QjkZVg?wq#c~nygYpS&W%n4Iqtx)an^MzrJ!00<&kA+PJLGiM19NT
z5RW`?4*!G!Ht1;Qyx>u!0jr2+wSsF$vfCUwmj^g1Ug_|jH)#mEs`n(Bl@TOxp0}28
zFp8OC`<#X~YpQma`nDBf;w#m|v?fcTn)7VW_4AzS$Z!y+7Z~x};B)ggSTg+VZ|dfG
zU(v)n<oZ)w<Q3N4LGIA8lMqYJ;I<QEwP7il(aK$;TYlj|@+0swa;yH~=0&Hd!Jq1J
z@Y}e^Cj@xG2rlxgU_e%xIxt4D=jISSt8&?<k-YAh;G@`hxUnpeJU7#h`Xp(tLOSC$
zs23o37XutMN^S)<dG$x$r_PMx!Gazg|1@J^6PL@<9O~@by>4|8Bd+aUD-O%*R|NFw
zI-by=#f`KOBxC39^`}-BnxwR2^u$fwF(Vesv+`~C`t%Zzl-STXuLPs@l^eU)4gZ_L
zsI=@zd+5K{8OjX&4*w-DcQ<tu%~SbThH8r{W@4wLe3uUMJ&fA~1q!pW3pMz0Pu~af
z+(a`#1-sSg)#XGTAS)_71WhY=qK1NlzP9zgQ+y}@vQN$hyg^67LvlJRg@rCd3Wl#J
z0>ny{*~Sf{L`sf&rhKwlBEB9)!_0aUmB?cMm3y83p_p};#ZV{x8e?;3RpXk>SzF;X
zPw1$wDefPwLh4_92)M0W#n5}vl{eWiobg+1atRl3y^Vns5eC?K?Gsle*Fe^Hk}HFt
z=y&%5?O3?bB>?_L!{KB#+CRq+X*5VR1!+4V#rgq5{1<Jqx=AXFnOs0NGEGF8BrZ~8
zNE^NtT@3D}p@$}j8&s0mF6{JB#79#0PKW*bgq!M{`7)kAZEq$#Gy1L9r7(-5;hW#W
z>lAT#m2{6x8YB$F5&GjW?o8P{S3Q|#TaSq})1W&)Y3#LlqTbc70ZwN9maHW$7Ff(Z
z;_c(g!iEm=&yH1(R%2`Qt{uFp%PYuqd8uVPe0#mGFuWk39UO?VL?<mtmezIMali2~
zecM%sUnD0YBIGD(Sp1V%0hP}9-~*>WFu3_2n5CokMyD16rky5l%_uDPy}{an5n)xj
z^svWo`h0UAbjKuHOyT<S*tN2{9D$DYaqA>kvcAxZh?}Y>&={mFx?2&F)4lcZk#54P
z&j})8+pvbLKkqkJl;&KJXja#|L}dNM!URkBc7J%JQeB<O$h>k_#G)d3xv@@XQpz18
zH-SC4f3>KGk|e5WWp0`nDN%S|?ok-~C^34=Ny>!$W7FNoIUR8t(5$|QrIG#8ToZQs
z<Mf&>3aQIw^@BCu;5VQfo241tbN14()K}#w!gikoET~T!rh1Fs@GRI|3(3|=CdS7v
z=-XLfGVvxXQleeH>-vYu_rr;y$R(4E*R{!?)ODMd^<~RL1T@(z?%8C^9Ku+b!m!?;
za?;Us9$J#X2-)X@xoOfI=TRYXt+?RbAGk?%nN3dvWR@{=xz{G&2MaCscFX$=*?dLv
zGc8g?zBG-70?x!xXv_8*#c0xQavG8AHtm8wdM&a(!cgC?WhqsFS`k+H@?5)n3cJV%
zbFJu*tm>vWl*@^T4GvAN6e{XTK$j<&mP?uTfGuyyvB@4{TH)=%=W8`B(5jw2*5@Cx
z+y}(J`fxFgB0zlcfg*9=chs)#$-%%36!Zh30PBm-e-$-FGaRpDv=7n7SGH|rt0}1)
zcR*}z`-9k?oUR99?3v7q{SK=#YV8+i>l+Rc-)rXEml*d!4<`>+4y&_TpO+ujm<tME
zn@v919||^SHnZ2@VL8E5_)RQWNIA11&qkn`VPUPz`~e=Gn>6rY2AlS%fxeG4!_*-B
zfrh3geupb{Eo>lt7LwY7E`1#CxxIx=UmxR3rdBy<*`h&7W>CS{-fqe8xgyLGtPe`p
ziD-UERcEW0uk6+Gz}L$_Zg&AALJnKy!*Y-O*Wwg(A~OOODeOV&)fTj%t%u}kZ+9<E
z2f30Ip5+=@<VK8WNOfB^C6##)zc4(PrH0M?Oe={ZQo*YJ?4fm04-dE(u2)X}wm&TT
zxxbt|F=JY+eYdVxXPr98llxp(2T%XT$y0y*+tEHAK$(ALLt~Z9Y<EM3)e5Z4)+B4E
zz*n$_0ov+^8cg@JZ04SBCMthE$~*tCzK4ELc?jD6JYbqKsf29{sB6!AnhBOlHhc-`
ziha3NvFoNTikilv=Jm$y*|%9`w*VJT`CydJ#oR0x=F@X($J<f6f?sR}u%`DnwSD8I
zJ>u`&&@XaL*HWfUcJ@&zz3H~fJUTLeb4KI6WLYrN+c6eEF=11VJW-?go_1EOK8^D!
zIeju0F>G;(36ZBE-A6Znc1h=~Pwc$B#}z~`=j;!$24ijIj8!lO?GY!x8E4ggKWM5g
zf~CnQDLSW8{K^T3$SY}JpRoUf1f>LLL_I@&DkA1TjDki;u%;#fjMX5)3Li5oH3EW6
z%5fO2Zi(Es!Z+=47zM*_GNKG?4)2yUE;v}#f{^FurtL(opflx&U$^;FzYdVqf}-K%
zC<ZbyN-OsN8#=<`$RO0UATkt#BaBQeSKK;Gy+W<VTP!P};$egQdxf*cf`-3875u43
z&>vrz>Ga+-4uS2gPa;Y<{QpAF{?-tpfUXMs^r{efNLw|a4FrMML5Us_2Zz()itlTk
zVLes2TufD<{2rnHZqyI&susB2sZj2TNKI@*(=g_fVuYXKttgeMze|~f>5+6RB=T%?
zdeC?IU1nk4IbRJX8WD1UTkbVM%e@K`Z81rKW9<o1f89<ryTX=*dX)?;$7JCIq3%3l
zsI%L1cWa@<3e~aU$?vWmq~=`uPl$q#BH>ohqmDMlILjK?H_ehBr8|SJHT{i%jH`+t
zf3Lvpy(BDmbL?9ApLwq(7Ut3o&ZP+Z9B;AKyh}S^*hl@E^$5o|;3^%moEe)}ZgQYd
zEkf_w-oeY1%bs)=(yE2$G#IOh7{M2UN)rOS#QqvQUux$BaWG3<g-q5~(H%epk$ZM8
zsDe~mi!_&~1}Y-kKBL?1m9D#c@a|O^^0)m_<JMa+`C%TVD<?1%*~ky>UXF6eIMq#c
z9Tv`Pj63>^h*;<-`;}Bzsu5a_KNO1P*B2?*<uv;??5T&s<)5clS5KCH!vKl&G?2sH
zKXcd;`_{KVtX3jj&@;Ynif4bkRU@}|e!K2JCYYqndD!dERd{XJw;wKBw<xDn#V2$P
z7zW&7M|tfSw$bekG#)w+SzlhF8vrH4m$f!eH&({~pzK0Mt88p?MW-Ph`TfIQRcj}N
zOj3`?*9PGBvBjxIahEy8t30cnA2&AIWx*hD+RJ{Kp4CS@EE{%C2W^F@E7dGuLnkkG
ziK}u2%(T|j1;GaYiTM9%ol@c*tmr?GH2;69|Nk`abId{gB&7QFPtoOnJ>L5FjdiNE
H9Ap0n(7!_A

diff --git a/modules/web-console/frontend/public/images/icons/alert.svg b/modules/web-console/frontend/public/images/icons/alert.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/alert.svg
rename to modules/web-console/frontend/public/images/icons/alert.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/attention.svg b/modules/web-console/frontend/public/images/icons/attention.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/attention.svg
rename to modules/web-console/frontend/public/images/icons/attention.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/check.svg b/modules/web-console/frontend/public/images/icons/check.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/check.svg
rename to modules/web-console/frontend/public/images/icons/check.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/checkmark.icon.svg b/modules/web-console/frontend/public/images/icons/checkmark.icon.svg
new file mode 100644
index 0000000000000..74cacf6f09331
--- /dev/null
+++ b/modules/web-console/frontend/public/images/icons/checkmark.icon.svg
@@ -0,0 +1,3 @@
+<svg version="1.1" viewBox="0 0 14 11" xmlns="http://www.w3.org/2000/svg">
+    <path fill="currentColor" d="m12.877 0.32617l-8.4277 8.4219-3.3184-3.3203-1.1309 1.123 4.4492 4.4492 9.5508-9.5508-1.123-1.123z"/>
+</svg>
diff --git a/modules/web-console/frontend/public/images/icons/checkmark.svg b/modules/web-console/frontend/public/images/icons/checkmark.svg
deleted file mode 100644
index a7896c78b9bde..0000000000000
--- a/modules/web-console/frontend/public/images/icons/checkmark.svg
+++ /dev/null
@@ -1,3 +0,0 @@
-<svg version="1.1" viewBox="0 0 14 11" xmlns="http://www.w3.org/2000/svg">
-<path fill="currentColor" d="m12.877 0.32617l-8.4277 8.4219-3.3184-3.3203-1.1309 1.123 4.4492 4.4492 9.5508-9.5508-1.123-1.123z"/>
-</svg>
diff --git a/modules/web-console/frontend/public/images/icons/clock.svg b/modules/web-console/frontend/public/images/icons/clock.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/clock.svg
rename to modules/web-console/frontend/public/images/icons/clock.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/collapse.svg b/modules/web-console/frontend/public/images/icons/collapse.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/collapse.svg
rename to modules/web-console/frontend/public/images/icons/collapse.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/connectedClusters.svg b/modules/web-console/frontend/public/images/icons/connectedClusters.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/connectedClusters.svg
rename to modules/web-console/frontend/public/images/icons/connectedClusters.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/copy.icon.svg b/modules/web-console/frontend/public/images/icons/copy.icon.svg
new file mode 100644
index 0000000000000..b04d4eaf714a6
--- /dev/null
+++ b/modules/web-console/frontend/public/images/icons/copy.icon.svg
@@ -0,0 +1,3 @@
+<svg version="1.1" viewBox="0 0 16 19" xmlns="http://www.w3.org/2000/svg">
+ <path class="st0" d="m11.8 0h-10.1c-0.9 0-1.7 0.8-1.7 1.7v11.8h1.7v-11.8h10.1v-1.7zm2.5 3.4h-9.3c-0.9 0-1.7 0.8-1.7 1.7v11.8c0 0.9 0.8 1.7 1.7 1.7h9.3c0.9 0 1.7-0.8 1.7-1.7v-11.8c0-1-0.8-1.7-1.7-1.7zm-9.2 1.7h9.3v11.8h-9.3z" fill="currentColor"/>
+</svg>
diff --git a/modules/web-console/frontend/public/images/icons/cross.svg b/modules/web-console/frontend/public/images/icons/cross.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/cross.svg
rename to modules/web-console/frontend/public/images/icons/cross.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/csv.svg b/modules/web-console/frontend/public/images/icons/csv.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/csv.svg
rename to modules/web-console/frontend/public/images/icons/csv.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/download.svg b/modules/web-console/frontend/public/images/icons/download.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/download.svg
rename to modules/web-console/frontend/public/images/icons/download.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/exclamation.svg b/modules/web-console/frontend/public/images/icons/exclamation.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/exclamation.svg
rename to modules/web-console/frontend/public/images/icons/exclamation.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/exit.icon.svg b/modules/web-console/frontend/public/images/icons/exit.icon.svg
new file mode 100644
index 0000000000000..a355dcdecfcb3
--- /dev/null
+++ b/modules/web-console/frontend/public/images/icons/exit.icon.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="16" height="18" viewBox="0 0 16 18">
+    <path fill="currentColor" fill-rule="nonzero" d="M15.723 8.354l-4.17-4.17a.633.633 0 0 0-.893.894l3.134 3.133H5.632a.632.632 0 0 0 0 1.264h8.162l-3.091 3.091a.632.632 0 0 0 .893.893l4.17-4.17a.605.605 0 0 0 .041-.045c.007-.007.011-.015.017-.023l.02-.027c.006-.01.01-.019.017-.028l.015-.025.013-.029c.005-.009.01-.018.013-.027l.01-.028.011-.03.007-.029.008-.031.005-.034.004-.027a.64.64 0 0 0-.004-.152l-.005-.033-.008-.032-.007-.029-.01-.03a.249.249 0 0 0-.01-.028l-.014-.027-.013-.029-.015-.025-.017-.028c-.006-.01-.013-.018-.02-.027l-.017-.023a.61.61 0 0 0-.065-.067.304.304 0 0 0-.019-.022zM1 0h8.614v1.374h-8.24V16.48h8.24v1.373H1a1 1 0 0 1-1-1V1a1 1 0 0 1 1-1z"/>
+</svg>
diff --git a/modules/web-console/frontend/public/images/icons/expand.svg b/modules/web-console/frontend/public/images/icons/expand.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/expand.svg
rename to modules/web-console/frontend/public/images/icons/expand.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/eyeClosed.icon.svg b/modules/web-console/frontend/public/images/icons/eyeClosed.icon.svg
new file mode 100644
index 0000000000000..32bcba84e9bed
--- /dev/null
+++ b/modules/web-console/frontend/public/images/icons/eyeClosed.icon.svg
@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="18" height="8" viewBox="0 0 18 8">
+    <g fill="currentColor" fill-rule="nonzero">
+        <path d="M.562 1.241l.876-.482A8.577 8.577 0 0 0 8.96 5.193 8.55 8.55 0 0 0 16.481.76l.878.48A9.55 9.55 0 0 1 8.96 6.193 9.577 9.577 0 0 1 .562 1.241z"/>
+        <path d="M1.59 4.936c-.155.21-.427.32-.65.157-.222-.163-.198-.456-.043-.667l1.137-1.545c.155-.21.427-.32.65-.157.222.164.198.457.043.667L1.59 4.936zM3.792 6.545c-.136.223-.398.356-.633.212-.236-.144-.237-.439-.1-.661l1-1.637c.136-.223.399-.356.634-.212.236.144.237.439.1.66l-1 1.638zM5.862 7.834c-.254-.106-.3-.397-.2-.637l.741-1.77c.101-.242.34-.413.594-.306.255.107.3.397.2.637l-.74 1.77c-.102.242-.34.413-.595.306zM16.484 5.229l-1.137-1.546c-.155-.21-.179-.503.044-.667.223-.162.494-.053.65.158l1.136 1.545c.155.21.18.503-.043.667-.223.163-.495.053-.65-.157zM14.282 6.838l-1-1.637c-.138-.223-.137-.517.099-.661.236-.144.497-.011.634.212l1 1.636c.137.223.136.518-.1.661-.235.144-.497.012-.633-.211zM11.575 7.574l-.74-1.77c-.101-.24-.056-.53.2-.638.254-.106.492.064.594.306l.74 1.77c.1.24.055.53-.2.637-.254.107-.493-.063-.594-.305z"/>
+    </g>
+</svg>
diff --git a/modules/web-console/frontend/public/images/icons/eyeOpened.icon.svg b/modules/web-console/frontend/public/images/icons/eyeOpened.icon.svg
new file mode 100644
index 0000000000000..74c3f7845d08a
--- /dev/null
+++ b/modules/web-console/frontend/public/images/icons/eyeOpened.icon.svg
@@ -0,0 +1,7 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="18" height="12" viewBox="0 0 18 12">
+    <g fill="currentColor" fill-rule="evenodd" transform="translate(1 1)">
+        <path fill-rule="nonzero" d="M16.428 5.418a9.567 9.567 0 0 1-8.402 4.95A9.566 9.566 0 0 1-.375 5.418a1.008 1.008 0 0 1 0-.97C1.295 1.405 4.505-.533 8.03-.5c2.576.025 4.734 1.039 6.439 2.694 1.219 1.183 2.214 2.756 1.96 3.224zm-1.29-.838a9.278 9.278 0 0 0-1.366-1.669C12.241 1.425 10.32.523 8.021.501 4.869.47 1.997 2.204.5 4.93a8.557 8.557 0 0 0 7.525 4.437c3.1 0 5.905-1.642 7.419-4.241a5.366 5.366 0 0 0-.307-.546z"/>
+        <path fill-rule="nonzero" d="M8 9.517a4.5 4.5 0 1 1 0-9 4.5 4.5 0 0 1 0 9zm0-1a3.5 3.5 0 1 0 0-7 3.5 3.5 0 0 0 0 7z"/>
+        <circle cx="8" cy="5" r="2"/>
+    </g>
+</svg>
diff --git a/modules/web-console/frontend/public/images/icons/filter.svg b/modules/web-console/frontend/public/images/icons/filter.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/filter.svg
rename to modules/web-console/frontend/public/images/icons/filter.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/gear.svg b/modules/web-console/frontend/public/images/icons/gear.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/gear.svg
rename to modules/web-console/frontend/public/images/icons/gear.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/home.svg b/modules/web-console/frontend/public/images/icons/home.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/home.svg
rename to modules/web-console/frontend/public/images/icons/home.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/index.js b/modules/web-console/frontend/public/images/icons/index.js
index da7c3abd282e1..24ffac14e8ab6 100644
--- a/modules/web-console/frontend/public/images/icons/index.js
+++ b/modules/web-console/frontend/public/images/icons/index.js
@@ -15,26 +15,31 @@
  * limitations under the License.
  */
 
-export csv from './csv.svg';
-export cross from './cross.svg';
-export gear from './gear.svg';
-export clock from './clock.svg';
-export manual from './manual.svg';
-export download from './download.svg';
-export filter from './filter.svg';
-export plus from './plus.svg';
-export search from './search.svg';
-export checkmark from './checkmark.svg';
-export sort from './sort.svg';
-export info from './info.svg';
-export connectedClusters from './connectedClusters.svg';
-export check from './check.svg';
-export structure from './structure.svg';
-export alert from './alert.svg';
-export attention from './attention.svg';
-export exclamation from './exclamation.svg';
-export collapse from './collapse.svg';
-export expand from './expand.svg';
-export home from './home.svg';
-export refresh from './refresh.svg';
-
+export alert from './alert.icon.svg';
+export attention from './attention.icon.svg';
+export check from './check.icon.svg';
+export checkmark from './checkmark.icon.svg';
+export clock from './clock.icon.svg';
+export collapse from './collapse.icon.svg';
+export connectedClusters from './connectedClusters.icon.svg';
+export copy from './copy.icon.svg';
+export cross from './cross.icon.svg';
+export csv from './csv.icon.svg';
+export download from './download.icon.svg';
+export exclamation from './exclamation.icon.svg';
+export exit from './exit.icon.svg';
+export expand from './expand.icon.svg';
+export eyeClosed from './eyeClosed.icon.svg';
+export eyeOpened from './eyeOpened.icon.svg';
+export filter from './filter.icon.svg';
+export gear from './gear.icon.svg';
+export home from './home.icon.svg';
+export info from './info.icon.svg';
+export lockClosed from './lockClosed.icon.svg';
+export lockOpened from './lockOpened.icon.svg';
+export manual from './manual.icon.svg';
+export plus from './plus.icon.svg';
+export refresh from './refresh.icon.svg';
+export search from './search.icon.svg';
+export sort from './sort.icon.svg';
+export structure from './structure.icon.svg';
diff --git a/modules/web-console/frontend/public/images/icons/info.svg b/modules/web-console/frontend/public/images/icons/info.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/info.svg
rename to modules/web-console/frontend/public/images/icons/info.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/lockClosed.icon.svg b/modules/web-console/frontend/public/images/icons/lockClosed.icon.svg
new file mode 100644
index 0000000000000..22f81e3dd5b7f
--- /dev/null
+++ b/modules/web-console/frontend/public/images/icons/lockClosed.icon.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="12" height="16" viewBox="0 0 12 16">
+    <path fill="currentColor" fill-rule="nonzero" d="M5.714 10.808v1.483a.286.286 0 0 0 .572 0v-1.483a.858.858 0 1 0-.572 0zm-4-4.522v-2a4.285 4.285 0 1 1 8.572 0v2A1.719 1.719 0 0 1 12 8.006v5.703c0 .956-.77 1.72-1.72 1.72H1.72c-.951 0-1.72-.77-1.72-1.72V8.005c0-.954.767-1.717 1.714-1.72zm1.715 0H8.57v-2A2.574 2.574 0 0 0 6 1.714a2.57 2.57 0 0 0-2.571 2.572v2z"/>
+</svg>
diff --git a/modules/web-console/frontend/public/images/icons/lockOpened.icon.svg b/modules/web-console/frontend/public/images/icons/lockOpened.icon.svg
new file mode 100644
index 0000000000000..bbc22c8be2760
--- /dev/null
+++ b/modules/web-console/frontend/public/images/icons/lockOpened.icon.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="12" height="17" viewBox="0 0 12 17">
+    <path fill="currentColor" fill-rule="nonzero" d="M10.286 4.285v-.428.5c0 .587-.384 1.072-.857 1.072-.477 0-.858-.48-.858-1.072v-.5.428A2.573 2.573 0 0 0 6 1.715a2.57 2.57 0 0 0-2.571 2.57v3.001c0 .096.005.192.015.285H10.286A1.719 1.719 0 0 1 12 9.291v5.704c0 .955-.77 1.72-1.72 1.72H1.72c-.951 0-1.72-.77-1.72-1.72V9.29c0-.954.767-1.717 1.714-1.72V4.285a4.285 4.285 0 1 1 8.572 0zm-4.572 9.292a.286.286 0 0 0 .572 0v-1.483a.857.857 0 1 0-.572 0v1.483z"/>
+</svg>
diff --git a/modules/web-console/frontend/public/images/icons/manual.svg b/modules/web-console/frontend/public/images/icons/manual.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/manual.svg
rename to modules/web-console/frontend/public/images/icons/manual.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/plus.svg b/modules/web-console/frontend/public/images/icons/plus.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/plus.svg
rename to modules/web-console/frontend/public/images/icons/plus.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/refresh.svg b/modules/web-console/frontend/public/images/icons/refresh.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/refresh.svg
rename to modules/web-console/frontend/public/images/icons/refresh.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/search.svg b/modules/web-console/frontend/public/images/icons/search.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/search.svg
rename to modules/web-console/frontend/public/images/icons/search.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/sort.svg b/modules/web-console/frontend/public/images/icons/sort.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/sort.svg
rename to modules/web-console/frontend/public/images/icons/sort.icon.svg
diff --git a/modules/web-console/frontend/public/images/icons/structure.svg b/modules/web-console/frontend/public/images/icons/structure.icon.svg
similarity index 100%
rename from modules/web-console/frontend/public/images/icons/structure.svg
rename to modules/web-console/frontend/public/images/icons/structure.icon.svg
diff --git a/modules/web-console/frontend/public/images/igfs.png b/modules/web-console/frontend/public/images/igfs.png
index b62c27b60cd93babdaf7fb3912e760f0d111aec7..7de1f5217d42b5289a44670be56c9b6fc83d76dc 100644
GIT binary patch
literal 14683
zcmb`ubyOU|*ETq~4eo=x`vggF_uvp9K!OeK5Fij-gKHoG21{@o+}+6xB)B`l-C_B?
z?>YPJclNyJ?C#lro>P6hZ&%e*_dZqKT^+5Zsf3FS!Ug~UxGKu>IsgC)1OPy0#zc9s
zn6liIzZd{o>Us*#&(F8Fw>LL8S65ft+uM(iPbViQzkmOpo15$F>swr0JUl!+xw!|}
z!j_hnJUu<d#l`<@pO=nrUtC<gxPD3<IDLLb+&@p|e%gF`x!Ka~^YdAM+VgHhO#N20
zAJo^^PhLjm?oV!6a7f?jvwuv(jliqoh3ED4^~>9b%*@Q^=Mzo+#?a8vjkEiw<AK1y
zz~|@PKY#x;T|PfQ{e8GSmXvt=d_D1a_}9qDDD6|SykA#y+lIJ|2Sh?V%0C0*(q(67
z_pn@=6CR+fqSl${`|saB;}327PtUVY&lSs0IXOALA*m^b;)Nyi@$vCFh`XYqqT0Gz
zZ{P6e>y5;YoqwH4U&2Nob}N_4Lcb=##U&JUB&7xiM*>o+w6wIo>WbG*A4Nw;OGueT
zPuzbgnk_3Yn>v2X{Qmd&Z^hQmrG#Ewd6@6*Euv=U`T1zeFK1$GY^*#j3>NDb>Fc{T
z*t}l%H7UsVu-+Z!uDjMU+uGjN+}v2LnLTv(IFRn|-+ARwu^U}JE3I7fNkKffZ$mt0
zV6r^TrfA`EsUsk)d?YLQi;rhaW^v0G^EN-(l>Xxa_)$-H=U{NtL}A?Lnz`AT+0Gc}
z!C=*aoxAD6u{W-D_db3Xdz(aw>+YWh-en@b7WViDK)ZUk#Z|*rGmP)gn@vqkJN`W{
zE}z_gjfVzAbgo_0I4hJ$r93YF-5tqKJA4*~os$Gj<@m;b{P-bh=CQJ7d3S2~=6-vn
zr_ja4<wjQFWNjLOK=3+^s5<rC^?c?H?fIG1fM_iG99!@(IJni(cYJU?p3s%_DRQmI
za&mfK*RQ_Axw6r$u*6d8@po)zUi9$TUUKCatTIL0)cwvH;%=fYr|U4ZeAH9i`H+`y
z?v)y3tG_$4W*u2|t81V)cr}?d5S!AX3yCah>6Nl9O!BmtO*clw=)TvKTWC!!3V83j
z@RT`xwfA>ZJ9_AJ`4HAMTAkUv|Gf6S^4DWcEx%|PIC+7Avv6gnF`qw1scs{WD>6dB
z6Tchu2mlC^smROdeVRX5Scz5tKt1SY-n<xF#lxWxr|disJ^{;iL}<k%v%hAOi!6+L
zL&?Hmo$9(asA@0Ql((43KtA##m2~Rs{P1}7_$>2t(DBB7X5EgsB@fGwsU0tYx;Jlj
ze6Kz(|FF#w5{d<^=E^|;ykY2xikP@v;c&ntCKRcO03;1!00YQnQIwg&&{t(40J|_a
zz!VdgK8^qsgabt?L{!P1Pn>f$rz~t;%q+@Sj|r~4R*c6O@A{JFgVz)|Cn?-BbB9S^
z!3;xby<vM6E!mT84H%Ie_ZQS&mv!1PC3^kUD<rD)XEqYKgUio9b>6?uV&MbGSFzD0
z-s0Q><X4H(kVJQov76S?t!z5(Av`_Xy){7D?AU;0KA1_zrlgmemoWyP>5F+y^wxEd
z?+xucvjyr;HlrffV_vzDRXVcS@8qVw?CiTED|6u<qO#aQN_`-I@2Sm3)I?LLCG%(b
z?`9M`3v0fR;~($SQt%e!;5n{iD=bEnx24M=>FgiNaqVD}D=VYccBtdkiDF(RhHuaP
z6RR>)X;-JqnrO!6Qc@NW0Uvt^8B(v%v@0!0_A&m`lq*v;yQ(+m4V+J;9#AZb?U=Q^
zlE}K2FSff(K%2Q^7P7pQp}hPku_Kax#AGcev~<J}bm1<0Znizk=RUg^;qAZvVWLW6
zNljHws8RRa^EV_}dGR2)VC8HHegL+$PATxS*650RwP9%2O}MQSeLMZ02$V(3sf1iO
z!~HXnA+SCHeZoxywydTUz`fp~g!A@Fd(4qY7*_9{^*xM?hh~)8jDG%$s*j{6kw3e1
zR=+omL-sHgVm$WWz7!r0XyCx4Jk!G2(etgmFid#GShjf32+|VmGM#^}lBrT{6~+yR
zm-o<~tA&U81lmBCobh6&Yi9Wqlt1O}Z||J?lN3Nz-W6DV?UVH{iB-gb%GfYx#2X{V
z<feWk{~aXi{bkF?irhXzhIr}CFx=oESx~BjEgViSlnmWd^Dl;$>}W&cgPEv1k&B^o
z1u0NOR|}rdOal*F@XIw%qF~z@w>!AHoH>V}24uwlp>iUyKQ-dN-0h?~aWxtR&Y1lo
z3<8#TlyqZeiwY#gXtNpvHFKL&ek7;4C%jy^4pn&MzFDedN(=2_AsUA3nTYTmN)zIa
zdp>Zyjef60Da%ENGuH1Q3rVCLhVS~W{vI8yDo`<QN!JyI{S)IIOUZ0w^P<hkrM9`}
za6U}Gw@slQFh;<~z;qYxOMissF-7jav{g|JTuEBRM3CW?zKK<rI@l3)RD^yz$G<Nq
zbc7VcsP9?skshreu+9ln>H~gqUb}*H+;NA|L<L^Y<kooi#Ie9&RN2rmIJ|{3xvRhB
zw%pQJ6$0j`BvGo*U<XI<xw7q$n{0F0YC`S;z?}_sJ6_;#+ny4JkV`gUxrPpbT(36h
zAU$~K>$?lOy&RklTlBf>T)C8~Rdyt+WgqXOS@aL|nEAgJgB2LUU~+`=o&W`HBnE+R
zI5k_I*rIVq?X(?7u*sup`0~PW%jcZ3a7`P2ZrHJ3OZx-$x2`7gL$|LIN6(|k#I7@h
z$?&`+I6W3F2n+ZBzoPfmUM9LU026r@0~3cS=Y$UnN#xm8Wgi;14VWIsD$n)_vOZo!
z;cnfbw3riG0gQGuqZ@fgqsxFH%phh&UJ@ixK2$MZh6ooFjjr<l*0^drvn-btvo0F+
zoct6`?ellZzSXXdnvgthtC&eej<~^<u$hO^B!RR5qrE|$y*A$`7U)M2Qm98kx=p4N
zY?O+X_5}v!<d7XHKTx=e{DS(Zd0pOGA7i%f+wm*Xqrv|YU+Q7vcIh$h1Kh~y<9S(L
zT=rnNkPG#LR=oNUfGqu4JU2LfU|1zIDzp8quJgVG5QH=OHfTTree>%Nk5)fvcH2`e
z^tf;qXZQ`<5kl0IZ<*jNgy|A=lf!LVA$7T>hKr`VworHQSpxpO1ZW%_&N2c2>QVbw
zltMO}@W;8qhz5pqe?g832R_?I?d`kM!h2f&c{7?!Fx`MRx$SaATm);5$v7z<SqHYU
z&e7}Y$cL8q7{PP{AMDGw6FYyseyodWix(J;4|v*M*g;;o87{kx?FPDsM(h83+wqyF
z-0%+u(MzSv@ObCfsB5WYjg`LJ&OtF_k?#+)$xLJ%JR0kFV8&k?##-O!a4#HIUQwII
zhiyAb4)L`K-a0^|txG+1-^QiUZ3DhGc?9>v46TFT4vPL870IaqKt4h_Ub{7wOGC`D
zbw1;9b4!tel%pfk>u>>gEMJkzO%lL~uh>j}(I-}}r`LLJ`)%%+*~9en!^NNV?8K0>
zys;`zZPA09{u(x6M6}Ab*d%~u3=iLw;~0AICIERjL`Z!rl$9WpTLq`r-Vn?<kB)qM
zT*u!t&Egl5#(Y232Gnk8%|EoN9_nH7n-<YFaCI(L&zQsBv7dUBiZpqFtNxHqpdJ3Q
zD{~V7I;o`@-wppR)w_6iyWBCY*-=3BsA8hzp=)A&(kxS@g_>8RPiTtG&3#%5Au%q|
zDH)4s`!0utRh*IlCM^_lXQ&HakzinASo1seE8wAm@HSB~Lal#P;u;FSHFN%V($joZ
z8{nC|x$qq#-Z2RC2tGlYAEE)#L*X}+<38+hZHu8-A)Dn762(OA;ma6YAc3q5|Bvs2
zf#a`~U0Wv4EB|~{V6p|*I9jgvEb1L5|MR5ofiLG&DZPEK|F5OkZz?nhPE}5796}kV
zaY<rqZTnFPuF9F!UM~a<{qe_Wy~7K3Y+lbnMa8I^+&|3^J}pePVhv}#TFm~Dd`1X6
zEz#<6!wYTv{_wj1#kF)oHKmYHE|~6&R*4y06U3Fp{R2__k!F<7ek!anc>B$bVu)c&
zf;4p8C`cp;%2H4uVAQ3k3$Z+PA;1|K4mrKq<cCQs0gOwHOaIX51H8Ju+K~t@f6z<r
z0u5onUk)Sma9$?CH!W5lqJ4x=Ig(<8@jNsVx#rhqnyRJVhZtnJKN~-n)sW(j>ehN=
zMfwKg!F#CeLf{0GZQ$GyB7FoLQ`Q3z=gTHmaN37YpuA}<xGEB^XciM;WA-h*m<i4f
zaTk;ARkMb81x#c7CIn$K<-78$>sfq+u4T`@i(~14uhz86oGFff=z-^v;9D^h!IrEj
zq5V0C2o|a~QDkN52~}cLY2mOYdO0;vE(mLUF?Qc538G-){nY>lfe2k8eOn_BXC-qa
z`N%mx2=CjwZve)XKqw3lMuEL-a<z`mGv#VaLLzN^@9L(#t1_TTk{dzr_hqPu%G-~2
zOK;u8O81RCWchb8P;#==H@N?VfEnP5(?I(RMQLK67V+64Q&p&4yVGpg;)fzzn4r5N
z@#$AGTqmn>tUAM@0iV?%xR&8rmq!OeD_=GBH-seXW|C1RlngC!)D{MiR^w0WghmHP
z*unx2z77!W{C+(@n!_nXm+Vbr?yeS{Fm46e6q_H*#*+yt?vJ$Rv^~9Jhlu;e3oGCj
zR|GnObm%9;r0XI*E7g=)fVe=D0Mm1eU|AH8Atl-}M5SeRxHnA!G}`1boJ-$c5Jn8y
zl=0RkpPsC_h3)SSRx*W0T*k@DX}}qMXb7`P457PJAe1;Sns>85ukF5%qFH>n4Lv)S
z+4#+EZ@T`XqEZ~sv|u;BxoBjZ(cIK8`15bB_JYs>fQS!?UuMYpQCyH|gCgZaSwJMw
z=xShgBHVO{7Bh$&k*5qc%9lxkRLh4lA}pfO74o}@aAV{*W-BZ~0E-v*=NlB-pra+i
z;cbF?Zm3KW+yV<XCJbHe#T{%U8_EdnCIXRQ;ikj$lp)n&=nAr-mrO8}&o+qFey=cE
z==-aGwDlh-g4S<@;=U(;-J_&s(dF30elIBIZ&+3A;X(34I+l-K@E^8u+YO1C$r)PQ
zVxb0f^(4^v0XQ&47hOn_HAKJxPD+&}req9fP1b=h$`!)pb+9$8qDZooRiV+Qd=AEb
zCL3V|{T$(}?kvQ6PRuAB|4bvpWQm(KH4K6*`>G^(NwbC$<jR6M|5SS>IRV7q98>dv
z<_G*SxVZy%-f@8UGPb6$JSV7$Rj7afwRz>vAhrc(a?paZx9bHW<?m^kC4_IQmKwSU
zsM>!NeUuZ{fi&`ppu3xGi|@VZnndEE`V?6C9&&hf83Oz@=Tksxd2V1)hdeSI$gG~j
z1zQjfHN7I>o;#HF@H_!V!q=~$thlAia2xa~U*YVPtx-p*M8R*et=3FSwl5h0aQzwy
zcXWZ=u)^+Z*|#YTes5fFD1VzXm-9D9VV`cwdoTTz`}*GE7-&o^)+_o%mx7Rwr98pu
zQ=^O|l;4#($+P$o3YOsup%1|z<YS4XPG*<*wEMvhkCg%wmsT7VM}v<vqj*X8T`fT}
z8omUqn)?=+<)<}=Q?4&q^W#UMx!Kb3hh*H24e=`UtHx)aWnb6h$n;3$XK;FaqkKWp
zhL4<@(lzp<3S(uTSRdI4hcAB<%6-90VlVN{kN#c>d?F%y7Sl9Z9AtC3l?3t8NIh-w
z%z2uCKQFBSyMw|D-?O`D|LQ+9*-TGVjtcFj8Iv%E$%iNq;~+gV;yOG96*RPgFsK7n
zTbo(lTX$UI;l$c)5<gb$@bSqA2aGS=H!5r#KDszZML05Q=Y%$;DYuZ~YgKkrEGh=V
zAV&`&yz}e-zGoL=lZbC^{V^2`zO|k^s*05f-uQN;V3gUT3RCG;8`Q?^8cmQQvq(Q>
z%c35OWzHH$zxjbF`hjibpKoEZG$-2jIi1v(dm3);;w(fy;&P7f?oc?><{_GuNt+2_
z8vQxYPKJEEZZlXYXp<;Oeg%A2%O9BrAzn2@>|!4tx-hbSgG6DZ3Z|xFK6@c_&|co5
zeC0zV8gp8{vaVWE28+?-$g~<YdS|jlP4x*mI8^fwTM^@ibbf7-s?PK=CB}8QweBxE
zer>Eyd;j4?ApiXno-TxC{DOF&YQZPBQAgj<6~AI?ZB61k<hY*LQ}30jUzOblf&5Z7
z!{dJsL`<3a@^zUum7Xz?v~2TFXzGN7jN7@QvmN(sFZ0?Fv2Bs4cv}XimQKJ)jp-74
zSeXwC^#|0cR0x?+@c1)+KGuD<OFXA7;{;clNYA@1^;PI61F^=_n(=~yk?kigc;m$R
z4AlbTm2gA&Qe2$NjDnvP3UP6jv;JRsdA;!rZ4T)^bcuDgOxt~<F9Y@5^LM;CxEj*=
z(3De!KT-JKO6P&LiFj1Hrx)Y)`uiHYQ{caYy=LLXrSAjXKnAU*AEE{%Yatc(CC~O<
z+jzo!j!igR2yWU>al*#1`Rg>Cg>5|VD>u|#!S?wb28otc%Xay5J=ZIA)a*0rF7n?}
zk9Hdt$2;#o-+Ek?RdtM9nW`{NqQN1;#`a}3$ue$eC^^S#Q)U$m^KevVV<8ReZ)vXb
zs77E*&Yb0_YF!P$ry<f`A$DNH3&Bc$`Oz@#f9cD1cZ{`JGIVzpVUHsdSk9I#S>a;p
zQCE-Vg}=A)0_ybMWM4!@zDYYD31<Lt=EQtrnLoN)T)x`(B0HRGT%a-!*^5ob`u&Pg
ze<@yvxx(I_5T$RX!KJ%C0wPE3nqQP{MHM|oKAsu#Q$$`)dpt^f6Pql~{CmV6zad3_
zlIJ*>4MeZlW6{S%*mbN_<x2HJ$3Qy7p)*H+Iq_XpB?}0DkFY!cwD<9c5JmoDb?keh
zan0yrwVq(BsA=dYhBun3No%7&ZT(wUjj0`Xk(BarHM9<*=Ix{q08b`mD&5OsHsY#2
z7RxMgFj1U&n^EC^dhSrt(E*WkBk*#WP_8?#TXDl6ehZfI<~CLbMRIpB57u~lU)V(L
zxSuDCG<WG>?YuDH8E4SEX(?KwC(Q8B?5hDoX%R0dV#2!$YmbWyh01f4*r%n(vnzx}
zl%`v_p3EUNk<Bg}kvEE%Nzr8m`Ox?qn(md(6JEyhSeE8M*h=SvCrVUtWN6*dVh8x1
zl8QTUJi=@Cf4zNgg=i@)4L}WHqRwDz3LR$4TE(3H;T$`>UXEdoY1xvXNNCg5yAV~l
zu1+Wxjunr(I8MTxMV-3A8>>Q`5h|$8PmADX|6_wWQjV_f>Km`EE{ULa&KR^I=4Oo^
z-*#)JNl=D;Nthh&)eJ=bY?RM4AQ1u)utR<OnXuTkap4J(AJ;C#OoJ|$p=k%){bk%4
zT~NX>iQ3C1)#O`f@|RJvWp$)$MvDshI27{!yJF=le|7S+$ms3bce#~dH+jUiF*pNq
zC7OSxj2Kv?{Ma-#y!!e8uyR%!sCP(_FVW^(_>HwLh6`@oVo76uDNoviTbQj_H<&7L
z9h{GqZkrORAf&eNN(Xl89dQF`O~@8)`cN=a+<o0+1a@9$7oLn`JSI9D#&@A*K=e=&
z@hmxhUxx+fw;<OT>qKTr6l2a2Ie8PJ=|&GQ8z@O^_`X`6Nc?zO$`U#iy8s3vZj6d=
zCBLs-(&j#_ZW9v4jl^(;7{|cildg9iXr84%pf(8{Eq8T9P7xmvV>MpfUx^OaA#Fbx
z6xGnx`kg>>`9&t<23c>!ZZ;!D62L2V%(j>B2~#0U=#2fCf7^)#77qko8AS{CX1)c#
z9g-B>cNKf7-z?<=A-8^~Mgi3agx(1SZVVKi4V3!;-zL-qYhml8pS}WVl&1%A^8Yyy
zkoa$JSiKRr<&`S<Yok!u!ge!DsKSOY?$HqdT;HofokV0z*rB+QLN_b^F)9Ll9g4Y!
z_u7u^(70}pq=KqKKHEv1Ijh9>$>#@y($uvgT`ojh<&3uYB)5(Pkm%$uS4L~0@zMSl
z-Jp9Qc@Fw@l#Oxsvx)7{mRK>~?P>n)&Q_{ncVsoo8alMESAJA=ME2^$h}`KzPG)vO
z#p_VD$f;}mtnO6itZ=MiE`-4xg)dP)%^gxx(d%fNNKHYEu8sG7DM9l7jGcQ2ObJS0
zf%LG<A>V`&Ej{<;B42hVKr?fyB@3Av8op&9Qer|ol&4htVOfhThFOJUm-&JyDMGA^
z=2GXI*-kwZ=~;t0pwiMfN43jzIa{}n3t_60e&~)b&!vU6`k9dq(~4#OfImlmBbVdY
zAYe~RUeLgUw;mT!XcaKGU{>afLm-P}D`PATPA}yfL}g%^L(>#K&KClOOB8;h-%Q^J
zlm6v!a8<UxwpZ_RXU`s233}$wj3cd-_fXU1M?oAkYyFEhvSFE57buqRb)#iKOBIM6
zVu3{b3ZOT}w30_MGGm1*A2ftOXkAs0x+evah#7wRsSVc7pp-W><9WDAr+?=yLD$Of
zXTivm$3-lnAll?<t9RO$gX$?ivdqH-I^^s21nE=1GtFQ!A@Gn|1G(wK^Y7_zn^}3=
zDWb^m!&6@yfh(C7<lFJ`iSk48<lGSgSnn$F-@gUFq2Wk`$h=E`#}A#JkVtyonN)?%
zefT^BDV84{4%<(M5RjcO|9vB`))%tgMlz<F(*eQzwi`Pz6;n&i1x$}W5|utGI$>@y
z2UC(wb<cX!EQzGYSGKWzvwU`rGDsKobdENtM0zzGXt_6(Pjeqko@$UBnh}4_U01U|
zYlfU;iMYd<yJI`6dA^Rdxg=G``uLgp8u1;i3Yp|1jFRY|Qv^3%J}+rV;9fCA*16;e
za|i_isp`eUt1Ms0B_@<G%kx0Sv3B{kpZL2XZ9#_s(feu{h!r;dPt|udwRa!C1xLq;
z-q=9r`{n0@=+99zD+fzbF7wjV1E+(ogHmcYpsRmvV#<}=E??8R`G92UuNn2wBb>ke
zc*QQHlUa9)Ey!R!G`d^Hfu5CGOaxVt!;_*b++?(V9Jz8kS)enn$G*elz9nOm`@@JL
zyl_uT-&#|K#!u0|1Q7wlGMvRLB_desvr)2rQ1iw3sd;i5u4CcJMO5{A#AchzUZn6-
zOP{$wAnw|vR{?bbbS;o_aDsZPS_*7rJx6f|FSPFcUKbxqxC@8VzoF@#aWmI9=}Cmo
zr}#;#$}D<`v(WLu_}!H_az!ZS?QTCo3FY3%0LOvAao7Jl@?lo=z~kQ<P-G#l@S6!9
zR8KZiY8n60dmO&JaC9B&$FT4MiRZY~k^>nX=cpGvt<qwaN;5VcqBKdA_OqZzH9`e$
zM&bw$VBC;AFm6{YLeYvbLh*uHj!BSEj>#_`O86}bN+>Z1k5a#Qf(G1*zrTGzlixaB
zhW=Pteo72_aP#-a%dh>k1D~Ml__)82TBL3FumXp5Rqp==l$QzNbp3r5rMjL7e-MK!
z*XOE@eGg&d3s@rM!#|uN>AJnxzUxNje?-l97v%>-k9h=~KTaz_#TqK0<^gUHkrlxA
zCBTd=pkpcM7d0NxNf2o@yi+#Y3Q&tGO|<F9joxA&JcitfHEWk>7i@3-6RQ>rTNi8f
zo^7=lQ$tfm4-X?^n&zVu)x3C9b#=OEucqP3%p|Lgodzn8C&rJL2Yx!eGZNXnlc4&y
zBKSMmQqd!8UGPa9Fzo2#Hl#u*ZwVj$iWR)tgHek$>33STJ{v@qo|5-ZFK9)swdD~x
zLect+&k(lF36pFnZC;s+@Rma7#lW2J?_`eWmR0*65*Jsp80L+3GMzNCxoQjh0T)EF
zpgsy~XwUchZck|3U-DQ(S=&EOz?p&~r*;9ay$>B6PpssT`*9G_5TK~$5!Np>-&eZC
zM9BU*H{aDOqNmq~AgwnT+P(kJV)B+W%WH2R+~*8gc(0Unmr^7Bv+9wu7Hhh)s@lEb
z>Gqq8k*;0aHDUQ4WkZML2_{(@IzB1!W850_)KXAS^<kCmzlWXL{R28)GyfvafN=P^
zL3VuEEBxbkS;JQm+u>#(r)pTthGfE67o?0EoqR;2PKG$s;+M7|?^6!x$Z}m_M=t?J
zgcmC^_XKyui<Oe(;1ZS-n!}N;o1Hrq0QU-jg_L8VlyT5+!BEQh{{fHk;EY9j)U(<B
z?8<;o@l*~-Ks_93*vTK%paV4e2*qcLEIxUPrQyGfDm|^3%IjOzlyQzm?JS99I5$jK
zrln+2P=r)OIx?ipwXR7^ct8TDY-m7Q7q=n@T|7;O7yNSCK{j+mBvuRSYm3od0zfo2
zcpnMV&%Jn?o*(R|0EsTGB5#NXi1eyfGOu#fS5{V>y}{^%@4kZm4ao^tZmx_$%gd4N
z)xje1ESH6IS=*1>LVm7Q!(%$gaOuw^syd90Oc5YXS`>CNHTXj6_IZpqBOZ3eZrRl*
zd@<enm0U#rmijBLD7HKij`5v}jS|tD=~Fv|7VUrp*3dG=&|hlpKKQcM_BrLDRU0F$
z!Ja-kgS>ZZNYRS~03D&6r)#2=tpZkn@Tb}2OnHJTxd5mIoO1p>@F-iF@g6vE5I)xR
zzc8@Y=eaz(*Nu8m3h|I^;6EU>A%oCBNm|`DA|Q{n$Sa7#!)p}Vw|AASt}~&N`5B5Q
z38Wimv0uZJnA|+F@7z2G8NWKu-X#CB&4nUI91k7tf`?f`1o^>h6j?x^Mw3rix+P?0
z5aSm%IaOB4dD804Pd&|@M4aLELE)tr&0)*?>nz%!QtJa`<R%eo*c)+g{@A@qFFP!1
zY=|B|O&Z_39z7udW{jo)AzlQB`9)}6zX%NTM~M6m1t{r1I`pDH$-AL)4l5YYd*LX-
zEjc*{r$&FYUyt4k$l4K~tsoUkOq#4nY6aZ4e9$c(6!P+8BuH5)x<x&@wl^~tkBEn<
ziIVtbp=%`(0D_i%sfj|MC!I>EB1<(J(cns=Gp`QN6X|Nfwx~tYwqeE`ueJ}m%ZY6D
zyqwQFO3Rx;TIxB3dt>;9^(6d!izwO{!XE$d`iSU6Jq#Ib)dX!;9$EsB|CU4#qry-+
zf@_7ooA5k$fi^3QUlE=~X6==3fSiuP2D2m1!;*vHco=Epm0q9KQYo=y_tlYFozqsy
zT$P8jJHu`05|BHb*f%J$)+EE1kUA0ybl;UZQ32xwSx+g)CBmi~0YI|OFtT=3_l}Jv
zY9Z>eASfwlon31XZUEq;;jkpCKmY+pRkpDFB2DNI!9VAvy1$i_b~I^EY&%zgzP;HD
zsRCHCz3?N4A1$=u5xt;_V$(X|uhWq6HVKscg!*FAaGNF(K$f%aKE=;h^NoNcebS^Q
z5dv$)WH_uy_u9-2^YGIQVetlcFT71=sggktwm(7r0kTp72cFw4@%9K2zN;cArF~jH
z$8i`*u7giD)t}N<<i(^6(llCVr#}?Hn@Tcwr@{tuhqVk&BXj?IfKQ?IBCUL$POZ3n
zjS8{26*P`?Wu8t63qeIHWK{DuZ{`el=dV&;HBKtO96IaUtsv^_xwwvc(z%6UeD~gJ
z2(N&@#)psOiKeGe{UcGXrxB98BTk@(U<B*`8L;={&@>ra3XlNUmm<p*sDU2AN-dWX
z48_V|hjlDa`Gfd5BWVFQO9Qdpa9&GDDlho*qnBbk(rSWIfi_0wtB2$&ig^}b++@Tw
z$2HPD8igc%{v~>w=Yvp$;-2h29{|`AWEhETk)91aOrQW20eSLUXjdf@s49aMe6J=i
z-GPTh5sF`kvzCN{=<o}ye`Cpc;1ryuLf`kn*#jVrTZDimfDLXvJ6D$rdxSTVT)_o2
z4J7wHb8Fq9ne6dp>i@6GZ9IQQiF4!G3KqEGjD1pU{R-6{f&WR^DX!cH75}cXnL%nh
z`Yd_Xq}Un;)t-lAZ-t84YgX<20gT6b$MD0{C_rvi1>!&r(f<oI|DDS>V+54{J2QiD
zD+@@E0Qf)VQ#syJ;32V=k3L>=B6pzURF)j?i?9p|6UxbuDA*o$qE`$DDx<J6<YD?K
zzagpptJM`GFIcYl`e!7UhM*t*wO@e0$ikm_x(H0EV?Ocu$0>pj4#i&n{1pq|Yd#Hy
zq0?aD3PNR+WJA><@R#q4#2`;*@PC_YW8j~=rEB)1YS6XpX#^JIwH&{SCRn!6LdU#l
z2PyBH0zOM1DRf9WTP8Ts4IWamyT2Le=wH0@09a21WY<)BI7_FA?X(QkmP?mdM~@<t
zwDzI>Vg=s>!}FM+GOXZTWk@0x?rS2Dm27By82Y#Wn7gH-(Adejw0La*{WgwBBnuz$
z^JZyn0sWdx<R@GXg7E17t%?6b%l{vmVJ_a|=*1PBF&i@qajgA<0pZ^G*cB|Jj_j6a
zY2Tuuy(HW=^Exc_bMobwC8z|-U_mO^ki6<9by;TF;t9vtF`m%|tACyz5l+#(mXrca
zqqX>6*;8XY0t0=|dlF8;mG@l?6DRoa>${_Xo#peR1dk&-+gD}E;~~Oj@0Ma>kmvJl
z>t!Tbl^=-P4Qco)8Oxp=`y22x#MyU!4C?MWIk6!n^VK^2qPTJ2RNvMwiNibi-h4uA
z??vUJIT<`VC?ILPF=H%WvL58El+3N)E)H(!>|s-$!8cY!$bB@ZYw7pYhxb{*Cp+I*
zO1E5VdUb1{;C>z_fKXCJohld2{fH$qnItpmBk`128sk7m+G438AN-;BYHc3ZZu#-N
zU%zlSPDF6BukfaX>ikbBLJ={^V}#WrN~j?oWcOFi+!>FC_G`DB-Z(SlJmzjlL!S*N
zY+cO$wT?Re!C-B<WxK1*hW6AiDLndzVySGeMI3IEla^@dGLVoywFtbZUM)5jb84x0
zW-oB1e_$X-AnXcqXjzOfjtsi#*iXAozgc~nj?}xJuPajWOLQ0*u#q4aTxr|9$lO!L
zf4y$3JgtvA-%j-t(^#cq;I3=~)W^>Eo79lV&+Pu)mwZJep7xGU!A(f@Rhh3L6nlLt
zO}RB1LXc|y$2<;m7vDtFTjgg`Tn-Fi$fcYVUV#Iv46&5;U;9mMkGXEc2}4KXxlyIJ
z$704#gyY2<K2<#a#9$tYEBk;sm(p2tzlXo)tU>A5CpED=pV$i6_h;>n122%JHgT<I
z7D%0Pf62ge0-Np_ox6j}?k<!ayt=Qg>1)OI#rFM4QOaUDv4zZ?B|I`%tvZ@7hPsA&
z3+OVW=30*GPneERIOSAMb&)j?<mO5c)!*i<Pr>;?>26=3O@wKBnv}316)VeuDI5^@
z165xC_pVV90l*;$SBiLM`XZFfP2RoJsnoThpAf?q7<9l;PgzF8Trp;}yhAX3oo*x;
za|aAQ(CLsP9QPQL#XxJ?B<R{h`A12vNka*aG5emO!v#oRHI~8;I586!RMck{HYE_H
zaFN8>z8*xLSc>x4FwF3JTLo!k%Y{XMZ_(Vrh5}zTqXT1}T;HDsq1=9OpQ*+#$!b{S
zA{yDZUn$ii9M!w{%epU38QAoOq)AS3Trljr^M0vjhMQ8X@ErSMWb*YD=N*!NZD;B5
z-`>wXT4zBF)!eyVt@a;}l$DU2^^Y{F%vsBMmQr8mKU#*OVVga;eGb99Y~p&iB~Su8
zQPla-E;~233UXZ~0-k-&8_=vcntaU2Htqk{cF(tS=l(C*ZOyJ+zua)kG}J?${O2+D
z7<td?4vt48Qdy33PD7K3I97gFrPuhN5tS=W3+;uWKy%8&D=kCOw;*EK*9x)46!e!t
zhTp*!;G^1JCEN+x?&a`ccYhzaY>(OOIl6ICZ;I1Dc^q*7JG9K1q3sw^g34AuW8T0g
zen3!5buVlDF+)ho1t4(iGt-^PBU$63rrO!k&bQ4Ms!rq4A-dSOMZR^NnHd=BNT?u0
zj>x-eDX955jZ%TXJvO}PIY6)~>#K({EpJgjD)Hy{zS$^~-lqgbjD53|j7brFD(?*F
zr4_sj>NG*UTyIUTZw?zK>qsz`B8Y-S?x0^oKAew}o^#-%S=zL)jDGj8ipZLe6<Rqn
zIe*M5Exc)4j63>Vkk5BrJePs+m*X`bh{#bVNMBuS0&wxSZzC&#v9b;-M;zTD{2hfl
z8YXSai)$KGyoDAP0c8doDsH=@PWrg77YLw?{$wrtu~;xrK~{!?iX&voYQbNSN|*X3
zH@E#SGQ3s<X5I0|C`e%I5BCOfJY+F-fEh4<f?XugpO~?bZP^(#CRkS}tQpTrfv`t+
z)NfAref8v0!4rs!M+siMFZx^#KWomcG7Vu-EDB>iEXcMj++giKMYn7x-rjAwAh)z|
zD)|F^MN%t(1T8S%*yGO|44~j0FO&p;fTcO@X6oPD7mS=8T(FhEJ25moX@6pPEg-}^
zCjd;CjEjw0(MTS&J%2{bou70rE`WijW|R`v*|7-B(YzjP#?Q$-?@bR05Lf``2G$%t
zDrxdV0@%QxvKc{*1lr(~JQ@88Ti`5bQYlXLY_n#k&J%8MgeCfT!0xV&ZaFHhEJ5Fv
zGQ8HrQpGfKtb~J0qMngp9$9;8$zAwLxrS2)=Nkq@bo3)!)_(Ad9N$v_dEVIx8*SC4
z2NgeIo{Eb+qb|^b-AnHXM_l0{>HxQ+&vX<OZsn8pvMn{D*p~(kiv#c6qH4%g+5zk|
ztK@*;!z^FdSZNE7@Yl0OVl$q8z%jb;q_Pi^tD6WW28PutglTp4umM3LTO>FVyXpw|
zNA;@fH>qOcS+4)0R%F`}oo4rK;E)cOY~0!fqP!L6RVZ51D=lIR|3oAMw=!>;cuJ<Q
z>NR&n<73Pd!;$~!EPgqZGA&L7X6g@6RV(uG`D31$`uePzy(4zO*g6D_7GS7hkt?%d
zXQB7TN@NeGp^q2eCQRxx>(%A)e!@LEW^?CaTh&U?V`j{Xg}lGk=ezz;)c@kvGH**9
zG5X3kEL8y>ybwY<G!$in7ZPEy+4*?oBb!%LPiy{^4Z;GsYqadV$7`@zGK2L+0zTOg
zFU($g@2SQ^9L?fT<I;Sc;n{iNub<RoYy)=*N7y1l0gyMj8cuMr6Z*Xt489GSlqD?i
zb!l<Co{y1C?8l!p!$K8^K6F!`iVO#I(Hmw+-lEs+K;D|O6%LtejHsyjiU#}I_u0vm
zaAV02@FqJ7@N17MKjHc*c6U6UgR>EY(RsZlPH2SrgO*7amGnlmCOQw#-tQFs*Kc1!
zEu5|9r|y0(rZL={2!$4%YU8=ltEA+m%O*I%5$t8*yvoOBvi?B^`G0w{F+qfCkqkIV
zd20|xh$zq-=O*MOIz#gU?v%QR%>kf5Et2jeBwt1ntQW#`;jqB?2<mviZFJtMh5`M&
zc!1s!g5&`4AF4uF(H6L1`rT>-U>Ilr1Sb)g>*qH?Sz3<5F0_!WjI8B8AvzPv>bH3%
zjE<5-;AV+}{shu-2lLy%+B#U0*78$SQ)Hcrc??X%SVZ6tyjA;T#L_8C3|j$_AzxKE
zD5d}St*|CGTg17y%YOrA-7Bn3nD^#=P!s0O$NJr6fb%0$m%5K^0QFSmPgI1(!mm1Q
zKd)BRcSY23L98JIggi-1^I4ei$#Xh>8^?-PdJ5PHv%Hmj_9ovbGn~JaRvtzG2}I}F
z#Z&gX))}uYQ-URiE5j4zT>a;w<~`(7I+~dOi<ft~#^;I;n^CNJ&t~$mXX6CcoqIR^
zf7v+(w3AN18GbS4m`pd!T^16C{u|2UkTSeEMr7y@{&Xp&_!M~HhzULkB_PEz&U@Y5
z5@BexUourjh+qfk2!4B7+U;6$v8^Z!?Oo2h<q9oM{SM>M@0Bb^du8hkNtoWBxIVX3
zh^WdB_H{j~^h4)aID)C8YIqNtz;3%Wr8m8hvEDZ0-g7#1xFNMaP6lJbtKWA|Ggm7W
zNDyzu!cJ?E36o|~b)~VgUo^2=Nw;C)CQN3dSmq^*QwQ)+YWquBnS>=`*!sH)f<azc
zZh5D)xE0=QZzx=7`zkqLvZUYaTd+wbiicmK3yMC(p*I-J`dH+rQ{5|AJeAYHfSKiv
zG4A=P|DL~BVvOc=k4FYRr1UM8`d}&9SV=gB!I1%o=hD#k31>rw>Of<V7Pl8u10|=?
z$3x^Q9LD^<@}SmDz|9;Oh~f`hs787Dr@*1^tc(R2?95PENqJLQP)lsy{d(O8RZ=+i
zz273$Lz`NMfH5UGx<=E?`iQZ7$yLoP;uU7-Su#-F5IIiLWi&s4vgA>T_z;0vBZ*Q}
zu`{Zw6vvqZ=y+L%d{KbIt>5a_o|~1bYg6L&H?>nWzl_$Js{IT)9ckaykUZY;`Y|fJ
z=?(epyk27H)>>fWRQUXiR%p`5m&(Bn*3bxShBRC(*dWN%DY$>MNFG}>M7q~PhmVa^
ztG>J}<1Gc)8HCDnjl?OSqjIFhl6J~ZOpE)jZl5<?X2YF1V!P!#3EeLgbeRoVNyac`
z>6aDsfJO*whyB_EKjeUKtx}D*Dzx*LDita&vGlpej|oFE6DxcplU-`8;PmEF;()o<
zqn7aZ%4D;$6kuLD&1!yHSi9FIM%Do4+k(*?>eAP$=G82F?&`@hC>p3BLmDjb*0+w#
zs?UUZZm|x#>5o^E2)D=NzoT{;`Nthsjr^Z=pC=VPHCA?1Q^!uVrdPsTCP~h;kvZw6
z=o<(QxtPOFD#|DUJhq!_k<{z2!&hOSe<-MTep>K&Wt|TU+z%>t{NiS)A&^_xPH>G+
zHy`}wH}08Ms{RxK%4`oeMC9#wvF^-UywdvuiFNbDZ<9%7Wqr-si?8$EtrQiiE;84P
zAbw6wu{ZGJ%&#0wA>jf5z+U?2O1?bw4pebK6na%OItz%K2v-$s#0*M>R4aqo5f(`h
zt^96AM4o)8LcUBidaQgXQJiICyc2aqFaR@0S>>5PPPltk;g{o#&6cMhI_B6#byw}n
zh8}fD_5XzVY%r!Ey+LAQTzdTg+4K0sgOUT<f>tGEj|+h#$o-|?Gm^@3m4l00LxAbC
zbh5+2;n%<he~ih+!vk%HW46SiYcq#bonXD9K7$}D6Gv%bXg9fRC@B`M&HsRb^w^2K
zoQb@CCm)XSoB$EQ^q7*^0_ty%Itxi$u|K;`s^iC!r?;O2s}H^jb=3;J?3JC_L=J2o
zmL`VE5P`r%AZDn{%cJi9hhq0k<y@FBAsW&r0-4kvgC%CC2B{h;65OR(dq;v-@FD`s
zMEgk#Z5%=td9?~BWxSYv=@Jbt3hinPd|cX8gIyVUG_>vzd9*an-FFHKg!q1^>*{7?
z#3HoZ261Cbdq>$zKYsKEj&)@4C>Vhw1bXrX)Hz<@D<N6j2i2t9sOgrslXElJz+Ubs
z$oQmB*7>@NNLA>2labL;A1Sp=H}N~0-$%CZvH5{Q_=v*Kz<deuTC2TCeKw?j4Q+k_
z5myuu-x?t&#0bq*v(dCzvi$-%GQhnr@M+=CD+98GaVeVK2SUW^({_h<&GpSUa(5&D
z?5;o|-dh89f>eUe8A+lvo1d4VVv2GnY+|3^1%F2N*C)L9GR+?Q#L>wu^hFtbyVSqo
zBYY4kJ5;oK5Jo-xiD9-1K%^eA_03of3R%Kfb(w6a`LMjiJ-qbw_D6njUM7T%oX5Rm
z<!tz~ZSSiB$n>)2pOd?@?+|wd+@QxJ`6G+RiKZ_`(Q#_5Z&#z8ypc=&(0AMZ&Ho&i
z6iBvRe;QUzbWyYuDxEVcT?J8Ghj&Q|9#ugLNe;0r|3qlFf&clR_DvUzkfvnu5JunY
zR+mkOU~tb_GWg99@0}10liN=*e<2=_2eok+5KT<qDVV(3shR%bb{)MqeMgh2L8!0w
zV5Fx?_tW!JMi`}9y+-Y)F=Q{8yVZ4Xg_a(yp)4Z^{>J!+zrwqm@D%&LC25**$!T*k
z#|ypNVndXebgbc;0aZ|a>rWJ#SE{Z4$RhjoygNp+dq;fPDklGuKPU~T(b#cref{PP
zWEtUq=f&z}Ih5;;>pe^=Vb5%@V>@Z$g~;#avW%kz%N5oou#AKZtNw#fI85>5rV(HR
ze~iZ1oHfBT9bDFP-fuQs=`i5tW5;m)LDcRQzi<B$AGh3PLC!xG5_MHl@AWOFAipa9
zf?$xjL~RU#2l$Ft9zcoSUht~>j)O0Hd*}iw$HZC?B}50n8N(!A495ZGAXk!m4Od*4
z`ef_*7U^W<0NCtR2r(DESAQJP{5;ipO;=TLG&gylM6{Q`R@^=vvJ>lB4SA*4$|Ht2
zdE03RFN`{hwfi+>>b%%LuB6>>OD(MlI|@Ym?P;YG(*~lj>E4FOx6Zbnx0O^EL@TAg
z&g#0Aq<P}as+Z(4_DY>r-Q{qJRf{u6V2TGZHRp5f)7VItGBdxUq=^0<A>C1O8!@aV
zt}&z{l8?(ly#G5!csBa%VlO`$mzHdgUJ$<ceOjRayk4BDtrzESRG5?PT|=*}&;cPO
z*)vqcc<|tSVdf#eIJTJY*j(H>v*5*Zk5cjx9r4SS{+7n6^sOgY&&U(jn=wqU?$gb*
z5xa;+N)58k|8-uwjEIRgcr1W0(sX!$-WUf%GPeigg*}b+KKxZ6-b?I~!Swu&67epP
zPtFr@L0A5kr1Ev~Ptpu;V{_nMV97e8l{pk5pi4O(f+wZri4f!vOD`|f_7+alf(Yq_
z+OSt{dGczeg-CE}eg%5KKIT{CsxFCUPY`PqdRWPl^Dhc->Gx5O$9urU5hv(b)oQr~
zi0$NGA?xrQyq#^xnHTBvn?byx#-<X1&QlYm0tf+Nt_5in>(nu1VO$K6x8=MTAyq~i
zvkh0LHC4HA5!uggo`JV8#+@0^?1;mbBi)R-sSS{A2RXvz-OJ)VHM9{HtpA>qR}k{A
zZy%eEcL&gYt(=xiEps40!3*CbV@R57Hl4IhSxEJ*HiEmsMFp1wy2P0z#4t}!<lfZ2
znR}}~^GJD~u*HuQWN`|Xs>uJrm~-=(0siM#+@M~H4+x54k{1}jj2+-F`VT0+C?LoG
elEK;=s`egr%O5$*A76f&1gI!z%2&!-g!~_{E)0wS

literal 14139
zcmaKT1yEc~*X9rc1cJLI1a}(<9^4_gyA3W0?hxDwHn_XP;O;s|aF^ijuFL!GSG)V~
z{<XJmSDn*+s&7~K>F1p4`*f&+oCFFIArb%pK#`IZRRREDK>)xz@(-|YHMOfEd2dR%
zf~<<z>+35U8{084#qI6w$jHd`_4UikGY1>T>+Ai_&DGiK>+_q%?TkDZN6*1+a&pSu
zoBDcDv;QK@#(sT!!p1IqbG^sGDSx-uHFNvS#;*N(w^^pgb$WBtclG+bn}0RxtIf{#
z`nsQ+oVNb@der&Dq-wWf^=7jg^t1Q;^|~{8>0$Qp^}g3)Jy%IcsP=hl@n#kBuwU;W
z@Kq^uO+luXjmKv7@?rb__2z0p`qx}OZ@j9iN{bmsV@C4K%&eg7kA|j3tJcHgPn`La
zXWtz9A7)2i@Ae<AUq?Iy6`k{z>JkS+RIcVS*yX~(Db<dm9S2SQigM$=Vr*<8?qwFg
zuO%twljYcSYhPcNW98Xr4lWH6$Bs$lqaAgwF8^|Bm*n&vyq=7l)gE0Rjt`oPJuUw>
z=U~fIW|57X9bVZxGVGQI1s+aRSbf(h%RBEANxGW8RWCm&`XPL4%>$C-71CV#HFeLf
zoN=1DJRc%?4y3q9OqBEQI>>9kX(_34RZK1FJC-ly6xH$26Z4HsJ!ZFSoxHo*dTGgW
zEbp3Zwri~8D-4@B_0&||oNAlyDf8(&1SRc4eVceJoAjI_FQ8ioYoj&OM<>2Rj@cav
z#tNst+?EkFsgQ-Cl2k~T`D3{}NM`8uY){E#El#?5zc=evlcsrMHG5*Mv2m}?P<J%l
zNLkB0xOzfYtNtFWeBhZsTInVIO{PG5tWdQsIq2s^Lfg1nLP35(U`FO+l}Cx+R2p5v
zu~4wOGU$X~MHW~=Mpc=W8ZtZIf8}L;Xc+u@cj@W>i%xXZRG{iyf~}1|+fTD$d@cu4
zv@itOo*lm2*w{EaI(&S5JU>6*-r2stzh7Bd+3((01OV8rq(p^OT=k3$8tv~VQ5Jm_
zS&+oL@}cVZ_!assNCs<Bx{DurxS}4j&?JKL{w!1O8w%1=(ZY36P~_FJ2RRn0;la{|
z773e|%rjAb@%wF*h$~H`fQ(^LIO)ej^P#!kUpZJ(={0TDBXK7RDF~T%`^l?QB=c`t
z`*P~->W@l*VD5eCX=`E(jNoQ#<yhN$z;*k_6G}ki9Wy<wAPC?)3{W5d(7^&!;C+W7
zqVk`X5dn1cp3ayy(-+Hi-PxD+AVASkUs_zNw(5&<2~ByLw~t8j7Z)hWXAXL{Q!8~q
z;Cb8}^b`H%krahwO*vmTMqPJ`;`TR}(Xx@6-4$J1>kg-|$3?WxY41M`xpQ$kM+%yf
zIr9zFWHo@pS(@SQ8uD_W8$@(iG$VMq_MuvIRTm+(=UymBOEc_1M%F^WraVGq=wN<%
zLfVM|(}3t5zP%ln7}ti`tdA<F{li2-4>>TrNn^0jR*TS)T7EZg=I+Tvbg?M%RACyp
zg$9a^HM1CpwFz}QQt<~5dot6v6KuC@qo7jiT1)s2hqTslc~Ft(<yDkei^exrm{H|x
z&{Xs@qL}J1RMO(eZB4T~G&Z_Jfkz}U9G^z)HFZRkrj-^3g>^D?wiw_r{b}o|1wqwW
zt3$}&cPon+WTwZgLj(W9O@tOMSU=Rk&mCSj?@`CR4Y0*jXGejZu<mKx!@CAjc7s4Y
z97`CV$x!_bhAwwsyT4L8-xyRQ4%(@a)g#Zrj*c}PDw6F;>2U^yHc(FLlrV@+sn(l;
z02k4}e*FUBBC%l~*BYhORp4Kzxq3V$=q)@nR#}&i5ZBXP@@0j(zy@z<n(DFwI~o;1
z^@m{?Srg_}Ofk70FQqO5pZ8-F`}?{H;WVZMq-o1@C^?c6G=s{AyEo2n=fYT@Al-cS
z@kaHJCkVcSG>bEG?aVm^n&fmJ@x<KiSF|)e<}0jaJ>yfE!&Az6?}gfckEdI(0fUe2
zr45sOsw(wzw&P7=)}*w>pI{aJL*>Ozh4flV5bkdjj<)1IuN^kH%hM9DQ=D;7)017I
zX1&_JYIh{S(;VTS4z(js3Acu0sqJw0uXrB9)yDIk9WK$rY%YEtL#v-nHnx*YzS~zh
zF$o>?xYbIldiR?ItGw0w?tK2c<wIuL(CMxfd}ktgPe0jh9uROw3?9_Dy)FU*_RN}A
zkFFcIV6sYw0e}RUEFj>`^NWz*#5d3XpW?rpz;~^=>xX~60d|C+BEX;$RS_F0izPzO
zmeAF&(cf^m)4B)IPXo?>#t4%k2OoI@_k9<mdA)|7JuH(1YZzHky+b!jgU{2?2>Ev^
z@N}>VFmL78FyO5Sf_(oiC-45P5XvVuJ1LB8yr>)Y&W6kh<)>fUXL5!uCXgqz!FZav
zU`#+4*u-X1W)3QcKMRBWL*%#@O~!&X^>KZYxe3M>lb<s$7xXeP%0<RF)md@tt~(e#
z0CHjPP4kR@eMUycr0leIjt?zXvT*r|#bP`Y%nEV<AnqD>TB{9y8SiymXUEcFG6rk|
z_wR?FB5^T?pM=GSpFC+V@9$e0gT6Dhc#md)w%$Ej%@KsWSJ$WnE_cF6kiw#|n6dV$
z*|Y5d>)covOcr-xMqJsNNOMVuDFxqY<f_{HQ~0)01;~9dfZYc6h=1I_zidn#-Nd|-
zGl5m2bv`h!*J<Jqby#qElu4|N()n>8`z>bSXcz?FYJlB1m&gKhiNTea&)Vl_uL*=z
zV#0dA17v}v3CL&SWrB8ATW1JP%1LFU+anLa`TB(-@2wxqkS0SbbdB1)L0f)@Wr={J
zv}biJ_0Rnsio`+5A#5?aA=xk@f&l^V*=4}*_aPU07z)j*<xboQlb=<lakU%ZmTN3=
z^64zgomF8DS}G(o_KQEbFQ3p#+vp-caeJJ~b_??M3XRFKHjSeVrH`b0FqvKaZMx+h
z4@HYNKjHg8ma`~iVtm!Hv%Kr4?2LH7WHS6tPtEaaDKF1~Sic^4Net`5ml<EXJ{R9L
zxK}Evx$1z_tvP+k2T~IkLdRHu`_WI^5~8<7WBe1%k^L7?Ie;i^tTL}-;jmvXNG<9E
z_sbKv_!!hT<(5tfmlf%6`NRi>95ZxT(Ym6NbCREZAzDS}IIo{df;~~S*V1ZUu}i#{
z*Cw7CxYthaoUVInH!5{>l-P3^4xl(8z~`_TalwFZuVhkDkljiO5IOoPp-}v&j;N;a
zjYyGqYh+1iL-_Tc#(W5A0n=Y9@B1*Zgn{QoDkb=z^tfgGqh!svN8g&i5bgm32(z>{
zvoA|mfHJqg^CJ{)a#|!s6~9Q9u}YIbYZ+V}ZgTztwp#AGGk_Z)(Vv5ZDSj-DKYyID
zdi#~jX77LkAS2%|ovNwj$9th6G`!x{YJAuCr++G}Ur}+f%?tFv^%@NeNDtRME8563
zzn}RF$v|Ii^iIW5pEl6kWg?Xj7Tu<CQRC0&jK{6<-EP;PB9U8LLuf#4?B|e1P>c-B
zl@m2S9NPhdT<f?K#8NScHLhJt<+yVc1#{|&X!Y0Ck1U33Cz0g>-s{2|aDLG#NL0&}
zU@vihIMN`LSz$vF@Ac{Kc;GR7CRf&ngQP#fwFen1p^A~1MrNtL#dP$v^&=huraq^?
zu*EpZ#Il8=TC;VN<if0vsQ9eJR8#ZC_m9TP?e{d8m2+2&G1adod*42Y<&ALevg7y<
z$M@M>WbkR&jhwl+4K$V8to~v<Im?cyv#Xy3(UXcK+J)*9Mo{uxzrVbQuYRINrcpGM
zd$bLtrty+mfm!gLUO_;Mb}%YzL7fYDlw7khc_#w8`9XRJf-1inbjE9L$a|jkrC=fJ
zV|pB=W9qtHmE5v;Zzn==3p;1qkiRS-2t4tY0zqK#15{$^AoBHlJBDuBtn-d0K@PoV
zD+75jEK8jz6Y!4*xMw<N?QARn$nwdVKKJNu#j%fiuM$mmqSAsogsz0em!f24X7H}%
zPo^!JF0*!pekQqV@)XK%McX_X{(9l=C+u~HH6zQ@Blu<t1%kf<PZTz14hgiSeJuI2
zL=#~XA%i^cp6>TpawF<;eeAy|Qb8vryUV}C<J3!ByYZ=KFKq4`rYMG^5!;%bo3VO5
zXJoi!%*)C{U@UZ{JfBpnauWlV@9r#JjBKA&ajSUH(tww3r~^txj@(LVB24ND7!3hf
z8(ziEl}koFb4c;!OR?z+qq3g`^U5xM<+7ICX6J@m*f&M_XDOPIZ=M=iRN>UU5|`+~
zhgPezfPR|c9XP@a<1&jm+SD&PX>6<EtFso#5UfT?#cMJlAMtJlvE(3^<HG>sz<_le
zW0vpV33M0!JhDAZ&JsHt`}`phFQ@U`<F%gJ9DmXi_wo5sA%XwiP)U2#z(}o5kFR!j
z;;Z)JSN+L$uhK{W_Ss61@OxOU(*9;TmwLTo7Dwl#U-<wkeQ1O-m<q-B5ivfQ4tb2u
zN->aNPj8>OCq7j7Oa<xD-Dg9168h(OxwYL_TV37OKSE!res6ewejZ*y7ZWt}fl2a4
z8UAcLtlr(hRaF2Gd&&U!yV5&?HcR%jX~4j0oya<=C`B*mbjcTu<l2Mpd0{hN$U&aX
z7B*0FsKhi@+nIFuNdERoqrqyf&KjUrMJ}spmQZ&wTTUW6{JBvF)xvhVuaH39!n0gd
zvpWw)bQs;pd$^NZr*6`RwLlg-16SLzWeqn)55De_{9|Kn-~ER&eFe3aADrgo7JXbS
zhJ8h^vnUVsPj7RnegQZ9gYuQK4f&30ye$bn)f<wyACm}r>=;wH>V-FOpLF@ME@3fq
znu8~*DT{mooF1ppijhn%rYq?-IKV%2Kdzf4$H#W=7Ol{Q%8AT{#cS4!8Z*bie^8j!
zG(=T^w>C^ry#NY%W}@6JWfaPOIDu|}+r|O<IrQsSRvZ#K8jZ6f<dX(7Po7m<-bZ@s
z4L29L9|wM1i+mimT#RyfsL2<;hD}Z(fYh&u&P0>@)*-dU6;5utu-@m+D;;9TD+*?Y
zS?t|f<DtAeg)(%LTCNzw4eu>Ix7ps+reZvNjPEnaV$>9-mpBa-Zz=OA+X5v4ty+3!
zxMuql%lFOo!97BPW!r}y4AGUKPCfPG`nefZwm1ZItBCKY185sz)iZxFWujE_4-FGl
z+3WP+M^j)U^A&GI5_^PZJ>ZBbn4%ckDzr07r^=r(j{hiaoHLd0FwmVcW6<B{sEm7?
z<Ve4C97Y!Fo%&|)t#h1qei!$oJHGbr5Af=i#w5COBhX`&bE3)roYElA507inOp6H4
z(fI2%Tnrb)bVY<mc`gB`HTo`rRMWWEDBopYUV^#9Fe;{9&r<bAp%j%5d_3tsk^6F1
z@|{|+@xe1f<ZMNaYIf34<-M{fffjeHEWhvY0vc`;It|wGM{f}t64<a2mz`LuJil(L
zxXdQ02#>*V*W~y^z8u|~p8ZO-KLVOUD!Wys6Ep;>4KSj~L6i5kHSR(f3{8l%-+!Ys
zE58UtyDqRXRr^lRJd%19|2!8)Uunh?7X(B4lm3^b<eMcl72$8Wo7<Ew9+gkOMci{%
zz`++e&oi!O(puS1rhraH8e)4#F{e$}5LY8tsRqrqtPkMj0Bj1$ki`28Ay??G|2}Q$
zXqMtl4+Y=;yAOOP-Hap;g4SYAh5q<x_avx+)_Wm?`y8eC*f77-e_!_@Vs3!Q-P)Vb
z+fxpqMCloc`{SQsBOxSQb)=>O0nA?mwG}^ns@^_idDtX1=EA-Zl9>Jx{Pxulhe5;N
zssHI<3$NAC{Q)YD%0BFEV2n{iqSmTRhpAP5FNnx^1p4I~LI6l_eq_;_$qIQ2Fu|mF
z3#6thl?~Hw2y%pVL*7-caTF#3WhyPX!!)&NJyCQuf9QyXGZD|xiIMKTf@khK%S;tS
zy@etF47N**UAu<l$kiW32UgHOLgi!FpuD8?@2W4%Z_%6d_wyebnjAI109S&Tn+3G(
z9M6bmx$?G<L=3R`Q>tFd%gKQCu(#xf)Q!kDjEUbmofh{kBsFx8zr-LU2jOKOmDlFp
z7AT0z^vzd^lT8Gsabxm9ip}B+Xoc7tNjM8=$%rvW!8~Mo^#@?p+@s*PO3DQ^3zOJq
z1yxZ^&WJ@5fwWCh)Fin7SPI7k%&^yxFs}YG&ia1SLmS9tDs*Ff5wKN5@@jS(;hn<M
z!}5}`$>q3PJ3osK?-cLMQ;YhpkHsiTs7Sj?V}y)U?N@~DFHIs$_yCrHGC^YU1sKVs
zFUAle=*WQ}R_sup>`wj16TspNw?{{%M7Xbve8l9Z_QGEUg>CCQiMw8zmkGhgpcZ<)
zw)*xapSpInPYC6ht6K|bgS9mzonzkSYSxVO?bSXw<67a>Z}WOIV5N3l0C?2wHvQUa
zF;PR}@x}SZxbjMk_@-fl=QzRi8B}%-zLsXI*0T^NxCHKiUL`Ak0mTWhE+LMK%QB`E
z;tTZ82zJk|Z;eZ>p+wE(&7YPNvoxLM@$$CpHY%m!S=vFpjf>0{g(ElVWD+DHOk(_~
z`$GyGFJ0c2xpyQ%q9f3!V0sdYe)=knVq<Dhe>nBu4wF#y5+!?%;z}7U@vIL%wMsk4
z5)yO$38oH$3sT8HofAt)Ki;P#tiHV#Uf3z#QZ-7lzR2itRTaM=#&CUr?rNDF(_t0*
zr34@avi^vq7A-Gfo!@&21X1JW_z5kx^t$)(OgqEDNM$Vz@4Kl?e3LeW7vm8%Xc*u5
z9`c_9p%y143%0|@u-bR%E(N2<Usj|Q_(xE_|2~ZN5WqaWws&*5A7kFb`{2{AiU3Yf
zU9F*jDbv|~rHUu;xTE&LAkpr1sZ-oRo}#6#G7umBlTS5zBIiiA)XE^A-bbr+qxN@_
zOk!#9dSIAV(I;MN-qor|;A6hdu?(SI4ad#$Xi~r(Ro*G6Kg<kOymiF&zJ#EvYUNCI
zo7s3JQM3%*6VKy)+f2t!I-n)@=T%sX@t5;I(`LnXumF+qtoKYWsSuoYeaNK@3G;@Y
z>7K>?mN=G@c)xgL{KV{cx;*4It4ZYcF|44;y{qFh-@E&xEtn4@W0ap?6p?Ky+1<s0
z^~9H!x^Tyss!kSHqr%L}NkWeI{lCq^NoE4OW)xcjfdbhhFGb__g!H{oyD&#11{(&7
zI~a@JKxbtVu2ZdsGEfjOR4y!>(dlZzsZC${#N;@GDKh@sTK{|DWD;6(WX-WxD!)hS
z7}<zgq&W^=L?)0h0+w;4tkVx7kP_nDZ`ijq95>FHSw<0_e;~QZF3uzQFR_Fpl1Cv1
zjG4>H^k@m*Y*mhG#%?8selFe-A4D}r=>|y;oP+Esy(i4llt?fJeLE3LoGgSL&Z0t0
z{+)wNE$(}98wNAjTZF(mZ0*R5_3n0*SpmMjrF>y@hXRDsB}#TqiaxAh3~h_lZW|dJ
zpC;7}3XvuNSrsN)tU;CHf7y5vxYSaBRt*?%J&3M^2>OoJZ?0g?J$Qd#kvsI|(Hc_>
z77>sK5jrx%LYx!SaLGVd1aF!DuaBzhC7q>NC!p#2C&`8AhUw6l(?Y$1m&puPzZ6!?
zosspPtIbvRbnG}R<M@}a&DqG7e}?L*@8f~7TZbYMz}>&Y6QXVI9mvYj6X$=A79$-{
zKFYTcMcpy`G=FL4ZSATC(a(Q6g`NNI!t%@|H;MMu*m^h7_Ug=EiTAqzyaWSAthXzF
z++fS%X<}f;3Ueu(>^Izgt?f{htI8$AQ;gjsK?O%zhps}O9pA@$y`94+y#|+jHVxUu
zac`BF=@$z8mZt2rPjG5$KREE{#MN1|XYPZVbnMz!0!Xql$E<K?DAu;`VHm<&&8Rl^
zfxjsP{2JlNp5iLq<kb(nlXq%JR4M8*-*2`Omnh~0;;%xLxdCG7iuA`l=};G!Vf%7-
zbuBPLug?PSW+KNVdz65`6Oc2`B=-G8FR#bfQp=WM%kbs_+Z`=FQ3opkoUVtXVskS*
zoH;GnUw8#+07*h=USuIdQ3D+h{EiXZy+bei(ZMM8iTknDF1A^#p(SvHgZJ_o5SKiX
z3RdE8J>_$|@cy)5xQ%YGV1W#bz)DnUQQ3E@ef6fbD=maC@~mKLb2<&)icf|SDQ`2z
zH(O3D+QYTQH_%+se7ZO7YdAF)ZD8>Qnf-i`x!O|q1pzuX0zd$;=IEAf3L^v$01e}|
z=j<a}_~;9gP^%7ImQkLd!|(-h_<w4JVepUu0H6Rs6*52r9>4$#=mY``2>wyuYTvp7
zIBI0~Aaz1w6QFp@GW=!>l?0v4`YAFJC;8K9^s#LK4O1Kydp651vQ(J)m@Ltg0>>x-
zLr*)&{4H!J<N+26VDNt%OBe8H9R<pyhdyEASXJ!GC2_L>Axoa}mp`FljPrKK()~Lt
zb6qz=#F@NH>kofFBX7exbws*BFQ9tDgzxBg-exV>Zfay|Z{-*xfIf*?ko+m$xQ+EZ
zC;vW9J6AH-m9rDLp`e}a-V;}h_xRTd^}E&(jsup%gZ0%<?q_o<VY`j->-rJ%XlCq-
z<atypTr4BT{s5CM(Y^|_&5tlVFdj+7800sx#ZS(BL>dv11j65RmqV+PJ<C5O1>YaH
zMfT4eIYN|{A`44-$E4K!KzgX;L|S@?=o0U5hUjzr2S~yi1^tZ=ytSc4-}uGzK2eY}
z3_Vg%Ach)0H7SU$enUiyylo|P`Hf<Z_<5l}=%Q-`{*L^clBx(*<yZCr+F#*Qj3j}%
zq!Rh>yCJK8(J=_DUP?f8MAT$FON{&qm6%>Tm$C(%3ACSQn@%sw9zt6izM*-egHdeh
z&_2V9PAj!B7@N7vX>cShN8OLtma*}J4H8|KQ8B!D$;>(o$Vxv#$NHYSfyaV$50J9g
z&kSr_=V~$$6TbGNQlSl1vZ;-atLHdU?%zs-c~tk5PM)(lpVrqncXu6+GbgOwz3Fnk
zi`FDt)R!G!<Xw9mQeqHka6<3yl%ew}>#Kf0gFt+Az@&%}(w*55Avz3sQza?Z>}nfp
zO$FEa^)zX_!kfno^Fhu1<0@-8AE$hputIr;zLOAkLCh-&k*)NdrV^J%5lh$AIcmM_
zu%8A6{;ikv*E*~h$aO!gleq@(_mH7?Hh*KdWE`%^L4y)a8fuA^-Y;2I*34E4{zl2M
zrG*S^4+*pa++)BI&|<*G{!XdKw!Llqd!6lvRK1#`DYuvLQ<4*|Q*COs6Qpn|#?5HN
zJE{ea#B$2*+*tx}T}@P7vGgN@@rjBwC27eBh~cjp3V))e>QEHpCJy;TR?!2ObzjIX
z1U2XU>VnaYa<7$;REc}xd6!YUQi~&jE)h!JJAL0&CL$+Eigc3f2e;`24U!cC3LmQp
zGT}Or5ypd1ofP>KY`#7?B=^OpS9t%HwWF3$6);%n+op&4sa7MKR{60_PTRTFXshh@
zAS?!@+$Hj7WSiks9xDci4Rz+Y+59Ir&ZN>svO2n*%O#_o&7TxbvC8^ROq&i<@n6M%
zu4H``KRh^0@U%$IrMtgtXAso=yP|SmdiM2az6^B7tlmBu+UjcPPCpK6<|<Yd@FPL9
z0~Kh0J`jP%6<^gTtdx+{okIOWP3SVC%wQ)8xxWsz1g3hD#mGs|L7F-1e}2o1V#J@E
zh*1Yeo>qe*FT4oa6U0x0FXjf^ufxty9vg4T$tacMXHOgHF0L6e19)zG!iu(-`ISCD
zjn6^kGOfdYT^DaPbR0Z?aZe6S-T^Z;3-$*{>nw+)#-@PE&_B|2;Ir}1ZypR7`fAWN
z1;mFBWgNiD`Cofy^&ojgz$2<8c>9&i*b>g<ik7>O<Z3-`jtj3QlqLINIm8*Zo@W|o
zY2TFt_Hey5f6o`7GQd_vsIbm)B$-qpuO~8BA$I~xlb65Oh&jVnDHAuP8pRyJbwbEX
z=S5HQi$iqf4U?Y@5bOZZRg8Zi@E;@u!~8eE2>7J7SM{!{%w+ol?lyK@^TrDP44Swz
z8RJ3wbvMj}`3-jA5|$&hss>E~&J;8;Ewt(60BF@I!p$?FY6RYZz!&t-P&9%*wHhWb
zV{|xi3U_=S^R0$L>NO~7Ej%B2V!+<ERIi{vr^I=Roj^I4Au!ui+Q~(!#S(rU6q0RH
z^0eV*GRlq@U>|%RvG0-!N-LXo-rPTkGLB6!NamoATY5Zg4+Q1K>Vi>_e#yW`4&aFz
zDG3HqHwA3QclParqq`@5D=WP-Om=an6wGsPxYDD1@L6LdTvpL)xu(>w6?xQ2o}!LV
zCi-jHSxW~ga3SYo5rswSA^o-bm@%t6=9n2a-3J^Cr4lNvI`TgR{u$8)6QsG8Qudn*
zHtodsYHM&fW=EGe(#C0%lU<!k-4R|(@Y39!r&XZTKINWKFd(2XuF(semn$b$S<}^x
z8vaW6N#l{#*(<0+Y5NSeKp0ADVimS$BF{~^-hC&R-=O^HCC|PUf90$}b(tL<P6+Je
zxVvAu4FC~pi^H;EL}9#YN!Ion^WF;E8!Q<bd~pv2XH>*-boC4tg@cGYn?9KtFF>^T
zJURox1kc0qnQ0~jJ3xLKWenPAjxG>-%#L?AP!Ij0W!kM}TYxB6_0#@SAiZbFhpiQN
z4e_-XDHEv<{Tha(^^S26_&ltdkn|u6hdcusQoFHc=G_JPfH*mRG7>-suJa+45bvN*
zgw7S~fmLC%^Fj771f=%**ZwB%yxjQ@Fi>-FVhDvU&)exi2##@x9}bkKNI4CikxF8J
zW1Al5lUXh)<FP(c<y=ia{xT7kgJz!PLe@|_4(v(kmJF0wb!gxgq}_y!2P`VB3zk$9
z@Lb86{CQH!(!smF!01DAQo+~b)iV>#&;4VlA$BRF{dymdEtvU+WVZStrkP;;V8d->
zUCJg~LBpXx1!GM*_cj+aDc=@036Z-(R6p#P2yJF^rLv><K}1Gg`_Q2B(y`=5fp@*#
zsVVj&9GAq6uk@Q!=~lFvXry9)4YdSTL1#7iEjjRt5Z|Vmi_IhKP$bDM$%&qD%F^cI
z;ysdRsET~^^n&j+(zUSQ1%$Um0JDeSEfw%xaY`9lVb?a{1n~B~?GW4{JpcnQB<%Z^
zUP@T~sXLMZf%Dl*j290D06*jxz}e)b?OO17q;m?=D{>J4@&y?XXuxa0gMa-i{-c0l
z-V_6Zf5p)q)T%fR7hjGwM+eyX%uifgFn}N-06u!ltF>kJreoL>UwE8V+b2~@HivgV
zBvfJUjf@Qp401o=!{r)a1Lo_Dac8P&P}hT%Lw+#orEGU$V?YsLD^_M<I7hLkKHyql
z7gQtVA!5LV|CUGpFh7WEQHR^xpKLU|^33lswoPb=Z&;YLvU~Q*x=F!#yKMD0(*rB`
zh5C(p@&A9;8FvfaEcP4#f*zUG!^RKDfc5eP#DbvdZ*1~IW;@f%5^#tDap2oX2|l)~
z%C922oWD`^5j%s3QfOxC+%CDlvTFq7dF#Ax-HMY)E?j4GxJh1N_lmk)`n6;FMlh=d
z#e2^(TijJD-)sX|3cU*uAIo=WC?=ZotaA@j8@AW;CXy(KRe}N<e6|g!gC8R@C2CeB
zA8e9JTa^0;rtd~B)je|OR+r$_7m@Q%Fn3zn7s;cS?7Nj$O2;y`1m(~s>+IPm*fwpB
z1s(kXcj%K}HdQ705zd$DWKW?d{(USlae7`fw`qhPHWPCqnJzty<0H`wU-clsmA#*h
zZ*&#OA5_=F=H=|YyOWm+Ic;N5eJk3D0zS_LQJ++2PMT{k*=;&?W)G&J(Ju-6KRyM%
zeI<!r#9vo3E=ET4FqAP7l2IT=K9>Ns35^xcvpbUeNfdZf$>9Q_eg3>uag)fQXl}(j
zFrUC7f+!#T9B6al-?+fqTu7qLeQc7D-W5k+!jf`i3j_hIjl~Ips7MXqZ#R$62#LVw
zvX<jq9#8TKFVIeM5VRnoS&>v=>zFN7t^qZuUafJ`lxAjolL>4MNoh}noEJgZT<glf
zxtk6ad$=%FMG+_jyNQoLDUmqHbUp(erDq^1S!Sui<V?jlZJb=_%9nzjR&X5?twVld
zc3F#Cw<`70wb&gMsN7-LG7)$Ire2v{{o<P<1`_EncH3>2Rh~Y=57iQ_PwtO})R}bY
z6*^_?uS|EXsx@gf7$2n@X-jL_OD(id;~e50XLC$=SZT8Lh*%yGH3|XvIVs50IB8aH
zL9y{k95*pyI8hRA+<f0{P~942T|pT~HKD56jF<?=&BRhGf>xX-Lqj}tIu>l$7<F+X
z++}EMCvFq^C&K&p8heDAgrUXN`>_dsMzd}~(L4%fdajqn1S2BNp8#kJlqPnsx5hoh
zy={yLEfLCWQc0P{m>KM<PUsES+#aL5UgS=g{7f}ZT{Gf=&V%{AZ;P_uS_uTnE|P2E
zA57~IF#q*ul*_yG=;y0p8=POu-{zV*E*i_T+DC!G&tx~d&&{USn$y-+f2`HI>1%JN
zefYA%ol@{34%7N%xJO+#&AcrAzyL7dj4#fXhXpV`K9c_bH{ZXlKF{~jOe<L^{{n$d
zvuEJ1zYle#fSb{?{74F<b^fC&hAQnJHBwCYMc7}(J|v1`s#PV1y!$=_OXf;pr1ar?
zf<_i5x=<0tJER;xqw3mSi9j{MFTMbT59@`nWakgN%N{jELsE+eLgvzIX!z=8E~dWE
z=c}!nVx<Q71<u#<10Z@z<2PwR5vM%BFv)VjY(g48=ziH5%!;ht&+<>PCg`aGA^%&z
znzX+B=QcXyVFuoWI2*rUFm#_HSgbX7YKsK-pIaM=>Yh0ps>^ET&W?0j7kl06dt7Qq
zJYIyJLRhR9y361*+8X9!8HJCI{+8`5V7pMb5<zw3M?V=Fx3%Vi&>~nn?l@*G6cvpS
z6em8jSSLT`z6${6B1u(Nmkx72)HM>NH`G|qvnkv*eFX=`21b^SIJ=eT*TNxd<0^cy
z=aWmhex@e-Hntf((hQ#u8#~5pWq_M3E+ZFiK=mie!WXBNDg94)rZ)RXFX<Kcy(#kR
z-l#vb9*2B4K-s<W-4~SPKb@w7`q7||ATiojoB2B0YOJpx0RFQw0DXZKL5G%Lm@Zm-
zJLEK&T(XLF1^;=6Q{J?1jMnx;$<QgI{dG%EB|IBI57<r7Pj}{fikKs{fyq>N|1&ny
zEtDmAm}0c;638GnF+9Ie_Uk8EgKbi#T4n%kO>J}b4@EH2d)75R<`nYox*3$6!mrPo
z(Wmovkv)^Gf@QA5d?5p9306=lcTX!E#>rDjyWTOnZ##0f7=Ka&tK4#eF52rZ$worU
zMT~(J;W|ijO1-TPJ}U~C=OR7Ivb}hQeTWWInkuz<T^a=*3bioE+Tud+YI&x^Az#iQ
zS1;w{%N5d0j!j@3B;^(5ht0EiIVJ-Gq-)uN{S2IrbWn5f5JXTm<uX{h+Vpon5r@lJ
zig=}2#8td80qOx!KD*Oc#`4L`=TJN?wLV&~ceQ^hX9U_j!6th-W&FSiA|ria-;jfp
z4xVBDXT;OfiGlkeM`r$~XTy1jg&?+eO_Tei8vHO;u>z}7xmspQs+u%>vzgU|Uf@W~
zsbh#rx3hYFn0QZnM4Cp}<YsF9ApMIDFo&})LNujP4R1@sFJu4m<82$evrz@UpnSDv
zSNpS@nZ9Yh=fKPXf4R9Z`Jvxg%2n?Y;%O<9mIX%t^dJ3Zb1eLNWK%*0a(f>zsSFWm
z=)I&CIiiwm4q$p)L%jSlOa`h`2ttZR0msB{Nhm|lkSG?4l$(7gSA#60gQnISXUUQR
zRSyaj{}}wfYbJ)i`$HXUQ#I0EtAb-=LeN97DQn!H)D#Ph5Fy!dNBBwh%n036rv}5t
z4@G(`eM12&=4tmE+)_fp8@(~c*&ZHh{!;1iW44M~A=av43WRO-3=l#Ij%gw=ijG+X
z*`)~%iFIZ~%-@57q<G}C$nOd}&R6hr718x&@kh%Yg{vd@pzzAO0xT%aL`w|o1}?qO
zt<T$0TpaOH@1JLk9TU_>xtEY-J35;gh~+oqGV09lj963nIOQ%hNi+2`XhHP%^;$wn
z4pC#v7Js!0(8bjZRE6;7S$FKJBI@P5W3mr|9XoLp2pMvpP!viEz=q@~<t1{=`;r`r
zNnZywFtQL@-L7^5Yb^-|GxT#u8FQuErNK>0^|9k;#aE)ynw@q6@wDl!w9A#)=g&7(
z-&BS&&!N)%IIG1`{B_xjHcS@k7avJ1q3A6%sp4DnZRtvB<F$T*RNRP$lYB{QJiV<K
zmb^#2Qc&rap10py)yPO@yKc?t-3-!lxP7xfPK0=RE_Kodl>>uuYzCtp<ZX+z!+&!@
zf^lLM8mln+l=n3{%;oW4x=X;k4DWC+oeVRjQv-LemG`ka9O!iS8V;#4CP0=5qI>B^
zRB|X9VOn;``9I<VNbNoYoM{6V;N1L}U<|DBQ$MVto@sFfk13L#tFbE>v2QwEuW2za
zewA1JXb!QAwmPS9<Vk>EQ$$+I@35FB75Y=+bHrSR|3cL<W!oNhxRa<j>MZGf?gU)H
zXfs?Kdl0)6d!A-_dIdqgFkAFBbRqYECrEFT=uW~KCO3)D-_#YlSRt}!A2Lk~i8m;|
zA@BY#VEkVZTJ;~`E_T)lQ|o3K0R@{gBKJ5xK~ag*004hMbY$X!*Tt*%&}t+f-jm@b
zm@o8C?fH!`f)`<tZ*;Xt=l@Wu5KztE;u1Rif<PU2dsoi?6~QX#6Pf>;V0BXTwU`BY
zy%Fv3?-v&T;sHu5OohlVEM11|Ke_={&beRm=Ka3J*ApQ+lfTD6NB>F_`^)iLh%bDX
zvv%fs$l>AiIhP_k_EOX1tL=8v)27GjWv5~TU{C+)=4v?tS+HZbdipz{%*QkG9bnB9
z=3iQv3jDu(up;CvNNmUvj(VTzR=b$W%+t%{@41x{Cb%rf^Fv7hzmwyhm(1cQHHmcB
zWlar<B6&rMjnv8LcymWu5@9l07Vxr^lp}XXRidvk-_kZ<PUY)$>k-?>$$Fpg`^!Y~
zY}=aG`{R;tKr}&+ln9Wz#o6?nAR$@%uPCd#tH`Ug*kw<Q4z;R)*MwC5CeNGEp+!uz
z-qaYwW@^RJ!e+#26tZ#i84o#v#9{urmk-xZw%sXi09`l%+2m&v(=BRx-s&=a#)WfS
zzf(sYYdf~Hvtl)hjFrJdVnBkbsCe>MDmtvx^DM%7q?Os92_6_TNA%=rAC5R2NdXBn
zrYg8RtE=s5swnfXAms$gZAA&Za^&@1jZ`-y9p!<&rZT)HKGu>G{nH{QSEKih&+`PL
zRG_tyVYRS{p3!sO{uTs@lIp3}gV$AW!15OFJGf%8{<TC(hfpDq%*qyQYl5@007QSX
z%U0QD>|~&+=8=^c@e}i_8lgcC-M2+!>muXY0j{qQcHi`;5IK#BQkefN$s23$e=iU4
zjkl^mAP53CBl>Rd7pDTQ_zsu=83IS0U5Egd5AVA=4Ie7M%U2PXo;7S%q!9ri-KW}N
zL;#A)d3=2Qd2<BuE*g9|fb+U9{F^fx{(e?Y-6Uf8hiCqOJnPN>Ep_)aN{_H9hbwCL
z&u1nWs!mfXDSsG|34Gl5<un)!p)~7_512x%MhpQ3VYnI?e&`?YVFGp4CHEm@N6OWI
zbhEns9vAU4aYN=$GN&BOfaKZq*v+&oFZ`XoTN&ra9}{_hy|H&Yk28G&ZQTh70L!ZZ
zZ+`;;e^9+l&;aY4`v1^&--wL=(02djG(sk>q?3373YsxIvC($|;iDwrB<_hafvq<J
z;&B-f_Uk^SOWIMa7iCyxKBU+r=#|mo#VsCnjGUOkci5<dc5+%PNw<fNUftpKwSu7`
z)knLT+oD&VTLF-OLQD919Xe)RRSIPCVYtO;vn*n*XyuS>OP(P4Oc2kEt=-@t>U7Wk
zs=L8pxj>1_`#ry2-#$lrXp;@JfU)yJgTa^p4pr;E=wAwP&t>>nPHx<X0;yJfW5MUQ
zef`1R1M{wk;T%Pgo_H!bQlgm-iyB4g{xEqye$o4NOwx2-r{L+@JzlFgGdgFOsWqzq
z_Nbd_MtS7p{?=b!V?N`IvdL|?>}-lM>1^{h9#m+Ra`KU=OYu#Dz%b4|Gs2vm=Ha-&
z0q2HCt%la>2ZtX3IwA#86&IlLRQj-a6=V09rHtN4P&;Zd>*sPNu`B&@Y-apPVe2rH
zVx@R_sMiod6KV6%P8y-fGymoZej8iAw{Z%VSn`F$wcm2dkZx+)BZtySxWMOU)}Ahy
zBJ1P;d$y%3x~VedLI~^4l?1gith6ZwOCe+-KErEK%7Q|7$+E$Pg4$usy`VHy;IKkg
zG68darKlsTbUG%Q-J&|4mv`G2Mn&9eB_SP8biV?!Jn=bnXqx4_i#kVMDQn>dLFjK{
zA}l@7tN=Yx*um)fin$BufeIo++&K*3?XNrelsgRBQKc&W?e(;Ms#q)w63|^RmMT>H
z!=!Yp&Y|>a?({@_1<q$+kk1F-5Op;Q<HrN*p{Te_j0YW2`7G7)r%m*4ZIzIiS{Kuw
zcAzGHnc$=7hmWiK2&wX@R~X?82qfziYn3CVIm=l?(|(Hd``Y}9nkT`HDt*U4TtJYm
z0d`p@ugRoqi8i=binU^fGwOTa!b$2Dm9-1`i{tS4_o7R49=MF+Mh@`;S8^A4(LF#b
z{a(GnpU+?izSnDaN1hua)pWZi^&PK8{)oY|$)B#IpL~3rW)w7i;#c6DRj-uy=V4vx
zQ0{_MV?vf0wVNtyqKPeHK|U;H47*oxdjiP2#q5;KYTH_mzn$g8uX0U!xH2VGLW{O_
z%#_o(a{ZG*WMKlI_qNhNc5fY?u<WLyT@b1Ho^JU}xW&vxoq}rSzn;0RT=cwg-(TL3
zcoA-Wh7kni&T!6sZwo)d0H~;0;F~Ob3wfgu!y)`n4)K2>(!XuPR1ZYJVm8q~RC~mq
zb})i_cFybT?*aJl{S4Mwp63dvHKJ7SmVT;_p#U`h-ZJ)~0Akgxk&%(DYhm9GZ6e+Q
zcz+1K-EQ$^8GI=oJx9TX`#%8bP5hzT<hc)T)A@#8x$BeL^vK9_mAY_ME(n$1trdv}
zJkCD7VG>%BY{4f?Nmq6$+7H#y!y&?i*eo`XQ1s$_tq&<+wN6ZQQU!Ck198}d_hNex
zKO~h8N#&^(@0ChkqjpWnjLT>JOg5kRJ6Wzc(q|G^n{I_&H2~{-mx(bwhT`x5iD2??
zpQ!+Q&vk3!FutdrZ!`3t>jeM7r5x6r*Lzu@Xg=%98_}9R&a}LJN8kK7qYmhDlkpF5
z&Eld!G8+~gt{l^mNL~){ElI}&I4j092-Kn8AZikPbl>3|J8TO}(%y`)l;4MYtuw^p
zpbNprnPEk%rQZnf@3J5fIs0=CJT=%JW4H9-)DgVdhbG^vw5QZ7P6+c>^{(kR2c2?)
zp4c1MA0}&&8Z^pmXs?~6&`#zoYIyVM!$)SN&^&-m&d%r0mb|<P8(#n_96VyRSYZbv
z1wG!N?(|I5hFK?l`RlLlV&%@fL2)<qo3r)BNNg<yy38YG%H(4@vOLkeQZ2L{COin?
z@cY9+(~6d&&w5v%T|2n7>X}mIcw#;}0Nx-+U37ZStl(mRF1$alNTL~7G58fyjTLJ^
z!O{U_8h+*7k2WdNYRA}FVtu5`q^FWGw*7vhu_iSL#B^g%@{L++!cW%SK+0d%$bfzG
zuE5?eHQQ5kmlPv1=f(-T;ZTG<7#XXJ!-cKU-|cnOHB$TI9?V4Qk$RPdkJEG)2=gln
zay-C>kza2B^%pszX~_vwpqN2~c4rWWml1<wWA#BwSR6hfoq8OiuNfq5gQ^7bCCJ4~
z9?o=SzNDYJ?Cy^9xfxZaOMPkwUdHNc8jEY+a~PZGJvNLIXAzJaTyJU#0Goe$UT;>k
zIt+PvD=Y+hxZjigbO*geRGS~&vGs#)bQ>!Z#aDNe{o5a4G=?e<o`IF<4O9u&`n9%p
ze~J^DSj}R&>8bfkK-A>}GRBjIC!CYU@>Gc@>bT1Zo}CqruSWyHauA;z5!{tbmjig5
zN!v&@v;*9P?s9Irqdo-s`+REB=U;7mHIQqg!0@5b0g^*!<+&EOKdgiMLE9X2&yAEb
zD4)$cKAcpruLsNwIYd}#n{*7L=X14)sr`*J{A#><G`W;OWoD;WfhIuNmVnVlpRPU0
zdQ4P%Um<Rp(?MTuf4b2jU<l{ScE5W!{!jRQbNRP~@!!nke|i6ZF4MfNQgkpPz@0pp
zPZscltGfpdpx^gn`(Rs!(;W8Sv?GxTcK(+!A>bcSpvb$!3(hJrcx<7ovxD_rksL<#
uwL?IC|F+f9gSO?<yJ6}n|0@FgzZ$}nxtl&P?2z-HCMhvF(F$P$zyATid04do

diff --git a/modules/web-console/frontend/public/images/main-screenshot.png b/modules/web-console/frontend/public/images/main-screenshot.png
deleted file mode 100644
index 5e1b9e9642de61b15e0ee2e0362d3e52188489bf..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 86830
zcmeFZWl&sA*Df3&NN`AS4Fm}?XwU$`g9Hoi?hb=X5<FON_uw+PyM*BGPS6<!_d(7i
zxu5rWe!Nxh`SaCRr_R|`Q%pB|H*2q6z1Fp^UQL*if;1L7Df*KqPq1X(OQ<|~g0u<$
zV4|VGd*)Orn&3YWTvVjRo>YvI@538UEkr+vK6z3Vjd5p$3~xVke6Qv5<Oz27?+*e2
zqwJ?APagASBt%s`^bb}rbXCo|QI4hYT0aG+mIa2XW~Y@QUk1f%sJ;)9=ge;n`o3>W
zY(*NBEG3sJ6-#dwXl!p8L{hYIul}_~j4)m-I6g2utSFlk&F9E_Cjy&&`GwT?-A4%b
zuCT>udY9);Te`U!V(8!NCu3@9w8Y=b^eN`%KVPMKsTKa|l=vSz0ow-eWO|vlWBv|V
z^zwc?z+-=>oYBt|SGHpjQFAqal#b@lWlikALY?W4FqfJ69n-APPx>sTX9oG7KZ>;r
zBI%VxW23x_S0SyKIxvz3M*K76#@)r2M$+z~sQePHe<Y9CWh3g9qY*YnWB4<4?$Qp3
zpx5kZ4`WS@VUT(0s$4+o@4(ZJseZUg8?Xf4_=(AprFd!a5*;@|EOI_23fjMq7IbcX
z5T$T%Hg(vWlZsFiFt!n1cZ&CGBQS%te@s$8?$G6a4&jE(s%0qG-QD@TqN97s%q*8u
zUt6n632c9S@Mh7jMPpD%?>za{pRjE+Th=8_<@?=sDo+}x)_Mlr`|7Z-+H5HPqC&sz
ztNlvzMk%OWWj5~Y_Iy9j)_tQJ*>=8CbV1=z&R?(z8}HAx!k=mz)@X|gIKlx?r7c+Z
z)E!na(Se?pn#{zB>LjWTb&ZZ<A9XyoH$UYAf?w$S-Ixuh^FObzuRp%HP}>=JC*=R&
zW43xWCIT9c>-hZhVsLOU-l@r;Y4y8IT)Zb|yxi_smd4J&{UCJfY`}NF{vS;UN>5>K
zo>Fea8K_X^SC_5D_ovz`(~El#boYT(p~xsGB_@m10HkJ?6+dusA-iGc6SH>mxsf<J
zu(Qz0Tarp({)Y42MhKNU<+ig-upPgmJ%d~iaB1*j`|V!~Xm;Tqpa%5R`i<>1gZD-#
zoh51>qF?_~i^*t8SQYEH<5U`|EGrdmcUTiPd)OICOTa&>vzcRp?0spl7t(7?{N*fk
zQ^brn#~|)kV?GkEa-y6krB*_y_;;!!b!s@ZI+~s-mwdmX{I0NKcq_-53T)AMwlIfc
zBbQog>PaO3*dnd9hD(x(ipBIuep{%FVnvc`_C7HHbESH)$tQlQ+Pc|i+CVYUW9VBs
z#)-3DQ|7XQuxzu_id!?#b<0O@zg3sM6f`8Q)$DgL&tcY%39A-(`_HUVegK3HG41Db
zRDq|lMx?Q&!`bGTepM4)#D=O3dZ?N1H|<w|;+6E3KZv%{kbK+Drb;Cf>-!bEAxuXn
z`Ruwm*7LQ1Kyz1g&|f3Rv~gCO8oDH_P}gtvWR{(?Mrt1LcF5BLWx@xGj|FNuSJZQL
zXjNl199-zSf$4iH%f9$=ZPhy$$)!1xJYdDpLEgkHfir7L?s5M}sJ!x~joJ_F4lJv-
zsrdqGpx|9;Yf?A5OEvf;pvGJ6s$IWfzt-NSLQ_s=N}Uu?CZ_+9F$p@5k)9sUv(XIQ
ze%La!^w#RrbHgqaA*Si6TP`HuasvPS#&*Sg#g?UUmF}!NI(6~npV{CA?E`1sGz5XY
zpYAYZUpL_YFcWmd`;jKXFRy2r%p$|4br~a|CRp98rH;oty^#8<r|iJ?1Ij5f24Rtl
z!#BEd=adQ|4@pjTNd4SVl^%-IP3x`rhb^AiL;h_)tc2zv&r{%ll71RKu=(r-9SW_#
zEj#*ys>QgSE4KHVC*@d2jYh=p*H->Gwe`j03?aJIL`|uuagwVU?HB8&u_69xN%MGR
ztX9{cTiwf^1{VGbR!<wwg_P7N%ze&xbM;$VC>H$hoynR+?sEjHr+8NECbnEi(bowE
zfLo9$Hkra(C*TpymbcK?vnV0Q_BFIERLQ)`9I=z_EJvpuG^mTvtuUapW<S_PdUpN7
zV&!DsBzmK&OFy~!73DrMSnT(T$kiF&60j*%1MM*_s4u@VEa9xM-4()XbW~2M_-Sgz
zC~w|g6rnMvmByl0L7YqFwf8c=!I{L^OlfRzP$g-ZT2L)6M>FvY+~{si30(LM67`3m
z%z&h`u`CgTZKnb7cC|or)N+>JH=gr!xObQnbd7_69oPm|=efAJW?nbHk!&QtVfiYI
zklJwr&~9==JpHJ24$@F|9)8BUDI&JiQCvD3$6Ok+4hF-F@cD7dZ5MME))u=|^`uC@
zODXn6%#eBvX9iio+>o53jJ>xyD(N8{tt1z0jXI^J_`kKkf=SYwmb$6@eDJIMVa1Hn
zflyW1flqt2p4E<n8EGK`%G?yDmx`A3VvR+#{K__p)l|WUgq^W;{*PJ(4wK(y)G8Rn
zfK4|v+{|YelG4&iI?~vu$t41SCY*cETi6SP_GM1~2y%WEJ+Gd-n=|HehOB6?=xm|c
zuJ<L=(BsY*0dXK}`N0?OtLj!YD{#c^$3}?*_r+Z^jcKdNI7+?CGRUWDx;27gF4ei}
zp?1H%I@@<YGppsmEPtjaZQ^_}6oXaxCi_R!ysIgh;|Nk)0=|>ynq&U+hKJ~4IR{4A
z%MTQ#sX5@q;F6G`k}dZYwTF7k>d1N)8IY!sZpUJCRK<7gteo8nEOF>ORvkhTC~uQP
zV+JJ?yd!JOJ-GN5b?PO-Eibz#E~}JCsQJPIql(4Gsf@1I5b7=!^z&I%G>yNx=?xe%
z5Z-$y$@udck?2S9%ra_6Xt?x)L~(WZRU_jLfn@juQdc9*k278?cSqn<xbZ!a)4u=U
z_uR%!zbI%YGKU|BR*P=SNbafH(&H8KDRTvvrx5z{H8DSts;n6iE22;hpjlip@Ab<n
z91eLOj3myAj>#qZdnH5#EfE}$^vE<nQ=IoY$E}s6Td#>?MY<FANeyL>33DyQJ8=3E
zCq}f;CBoo{koyapFPQ-?Wjn%m{H>I}FLq_8&H9CvX4ld#Nz8@b;6{(Eg(I%CTl6J4
z(CV-FJskm^XbP-gAeCzVK^&3dahRLUQS;VhKg?hbS{Zs)7=VVz@ZwfGR_kMp15LZF
zpD}ZS;!_N}10~>F(-{YGXg8J}8{E(@ayDHCcLm)JlAYfYplU(&vvQ)giVg<>nJ8Xs
z(L3~DOd-31-mcS;9Lv`XvJz-noQQLrx19&!wCrom=p%CKV!xQbI5xc^kKEK~mp;>S
zGS1Mf^*G>`rDyBhK675^vGiR>lkX3kklxKnAm*j=YtU6%c*IxY0}$tD6y|Nu(puiz
z`^O`9d`Gzz5I7~|hJ+j9pX15HfXe22D=$8ko?+)Lw6e0V5ui?WQ2nTLa>#ziKf09v
z#6SHGX>S&f0;`qE{7SHsAjFYLIzyREL*=9^BB`=q%ZlaETCg3>MS7x;Z6p6;AB;X}
z;URP`?V5!j%-cp`m@~>+Sa82})Ys8_$w0o=+tIM9`Cx3y#J%|DcNi&(v5XbfZ3;@k
zoF4E7H6Vbn76iTE@qDnJb#u1Zr7l%yt0+pX+<3`hVW)+uIudB2#34U~@3~?(#Dq;S
zmvAM`@N42gH_WCVCA+VMt*6ySFd67e3?-D#_B7|_KA{;lxT2O{#gNq51q=$<72f2M
zxMgpJ);}UmT|&oo2`}qy!_1ABRb_4x^lol_<BiDW*G?tLPJGJK`*Wb)E`9CxKP;1_
zzM7B<KOq!aM34+oeo>DzvLlfVedm<!-$F$Q8h-Bo4d2o32zpT8hL!|w60aAjmt3!5
zVGCsSAjMWEY~0P+<rrGSE1adLx$v^8L(=zx86*pI6CC2K<c%_ee%zbhhe9=UH!kEI
zmQ9NF9=!JC935Pq9iV0XR*;jGm#A){y@-%<#<<(hS>rhkOLK3KK%o~M##)>H2R)Hq
zUMfv-hl4LU_|OtnqRGf&9-(}VAiB*TK-M=19<Ge?7lQUSzn;czyjz&ax{wtT%t`TM
zbx%W$$MIU&ZjKLscrDg}c7+7%EQ=J7vs|qx7f{!*o6vO-dA}4gIpm(#<jbHd=#NLr
zuWKK&6Sc|bKl7-(q2P`!-SGh#g|EB0=n3-8Vorvd`}*K(Q+<&8Djp$9co)#RiVAY*
zXy=QTRsMiqT!@j4x6zshsYwPy(0e8_f6u*r5<MIw^E!Myd|u&y<lrgCH5E6oxD>DE
zp5>%g+-!ID$!aif+Gn!{ta)D~XX)^sKyC*fqqU%I^++UI`I9zy`ey7Mt{Xcg5gO8o
z)eq&*nQG%Jr2TVE2lE&Y3dvfXprOrs^;Ti^Gbo#BwQQJUSneS$_Q_}Y&Dl{UOVJ5}
ziGG&|t+MzhgT(Hbu2<b<1WQifug&Sdg(HnDAnM(|??PT!g)L6S)y_<-(Z6&t^m1iO
zg(HLv6V3g(rQaZj2_QFVHqb~8lF|EOWrDR?-Q09=U0#p0(rgZGej=@yJN>RuJ4wTv
z_Hl%Q7amKelNKc72S;ZXS7+F%c4A8}sE$F90K%~T3E{eKc8J?7VO6Kl)f%mE_AaKI
za{+%!l(?g0L-T6;@Gg?i)rvboaQ<<%O46MA2Urz2(n4I_2==)x^l^)B+2#1dTcTsS
z2PLBCzhT9%aX744C|;Z9u#4X7i^lAJ(~(qx>P%9k)7zKbmt<4QO2H;n06lBG`DrBk
z2^9G?$L(xq1gvRReOB|MQ;FaP?DuN?s5Me;3L<TTW5C^V=kVgF%n=6`o8<@%&r&is
z$$Jsl;CTKt>fl|KK4_;g{Yz}yX7l0wl^39?os1qQ)MDjAr~-^UIT@Zb%~5X7y7l;m
zaR&A|`+Q6!0k)(zw5oM?!XFV!8wt9o4k%@4-Yyfpi$Pbrk$ZTzNo0U%Pf7e8Wi7E<
zyn#7s5E$$R(C7e#M(qKXe2!q)#Xu_(C7BVJOmxQA-GI%yoTXk&3cHn)I}Z(ND9+Yv
zo8OR9*<|83v<hK%ElKo8$hR?3zWY=&$>n7wU&h&T^K*k;Fva#*-R#zs%w@q-$4>R&
znI4LocmrZw1Dg%YOLezy^%xKcB;XeCcJ&gvlmuywq}#ms4zWxgX{5?O%j^#-XHa&1
zp!v#qwXs>dt8Bazp7@JWRh^5BlZ4(4<_Mj+Qodf2wDf~f674?CK6hwmKbyRi(9hEL
z922cMxXST#5dwtowxqUFCh2Rb0!$kYAaoC*W&L$m_>SBD{+oj@t48k<z?0Us6zDbC
zA<Be2NJLlG24^$1Pj%Bz@d0L~kY&K-*v7@{6<iUBXJ8*iU-7aFAJA4%eZ&LXC`1=;
z>!{+=dKE7kiy$kj5$Cuo>LbmktnP_M_B+VbsR)4YEf<F3u79w9Td7@(UUtRR<kvhP
zc%-%9-WAqQJAR<^l~;qRkLm`QFR6~gMp)=+P5X-S9HP%}@L{d_fh`nL2dQ|kuG7JD
zLuYyD_ijJ_Lu>=TjqrptNrxI|=+fe1$wzI@=|R!Ns3FlPRCt0W=YZqs&FRU~ta2k0
zNB=rnfYVTrpWmqK3-X}XsRt?+scMA@??U48`2Fy{d96g6RaVYS8h}yfh6Qlt{UV70
zAY~PeTYGz@TRQ=CHb13aw447zco~vIa?gk!q9|EB;F>H6DgPLqx)dp%cH#2*C@IJC
zI<K+%T!TAHEobihGSl*6V5oDH*AeJwpsV-5EU%xXQJOiU6Ri`jE-Ivao7ixg3X<0e
zn_T^6IIE?%QO^;OcSP!Q(=G_`r3~E7JneQ7wK=A34x^LC>!7{&`c&_j7KBrvX>ueu
z)&(gpzTVT&81~C=XjYwBB&gx>dvf)KEunttP*EtF*FCvwrOkWAnq#_>swVT}N!KZn
z(^w331mz(%Xn<;6;)uFTYdtDzQRt8H(oU4gEA^*gY9Rtj^XA^K7O*ghy{1S)(9_Tc
zq}<=gCD3hf77ZGfD7urH4OFqPwx{TLLl<faC;hhHHg$a6dV3Fsp6!ll?2rfmNRsC&
z^h!rxD;^rCD<;i=A<n}S6BC%5{*wK)PJZVw_Ave^w^49w$2XDs)~MaJ2kezQ4Wk>r
z5HEY|cfYe&e}WGc61vvH1E>WZuWaO+OCfZ6O*0Fx=xQ6Tw*nv;jc*_`HM27l%&|G@
z>O!C}%+Vjlj6WeYa1z2#_gnsQ9Udj+DDDH1+PO{Y_2U!nnhAx+nPsj`1I?PZm>_Si
zz~4S=V#Edx22=#ho^LE3EX!;QZ_>+&CoClm@N5tqy#~mQ%Mk+9XOp_xZoccZsHI>Q
zr=+Bycq=EV&3#l+i9hycHk}7)Vzo1!dYF%<YuSw_ictTYy!$#ceGv(V;IBpfZ?=<W
z_@}4!!u+X{E-8O(*Je3-LST)jJJ^QTmdX2TLw9UDlolGUHsuO`Qqq+`daXBY>ZT0#
zuy(aWL4dbLe^9vWYcDH_@AXONX)Ji4_m}S~AVIpp4ti|coWlT}dU=vLD|rXWzNq;v
zr!{ais%v3W=nw}+&W(@AASW9l^hmm<upn>SRk7Co6VLu1V!)pO&O22o2A;--M#uN-
zJlo)Ov@qLWYew_f%L!pZ)?C<6$;i-a8XYs;=l6d3mr%04CGNclqxNH_+XxnLilZB-
z7H9xO3>^zuPZ!DN=%`bGJ}f!^UZylO%{3gMb6g8Bk3JPAl5VpQtJ4!W#lxh5UqQPR
zKV%p7JKBX7S()I1EP`eGY`P~M|2&r45k<dba|@-BYu^^Cw-LWI1~?%9!2Qd-ozHdQ
zpP<kP+Iydk@FtY(5dn0ScC81Z7PeUXGQGgBFD3rDkEn;*{?#evN;MW=055!s;gf7Y
z7QpxC8(s`vRpftt<p}+40`Q9j@87@wKc({xGg7brUzSJ06q5z-?<CQ1(x5o~Q6>!y
ztugMuI&;m}e9o|z{_A~ZY64JkIRB$LJ+1oh=d9dr&#nHm`Ecj?pUs<7v;S;ffYtt^
zsjjN}Z?g2&$c*$q(r_aF`=);j_TNhT|2lu`>_4XOe^Q7wT<NL>QB%<PqzJ4S<#|VH
zEKiN$@h%}^s*Y7Eis~y`CJym3!|07TSdVrK&tIco$<d@)CDxsf9(Mv6Zo9n+bgz3S
z8WOcfowS{Aq&F<U5o$I#_}9Q(8EkIpHj%aX(krpa8Yrbs9gqiE!YfAixE$G~JR{$Y
zDGZh^F`3-gBv%Wd?g<bc`SMYW5`q!;3$c=`Z3~G>z9fl|hG#A|{WDyM^`xk-i!lYF
zL6q1c0lyi-(%~$7017`tb#~-dJ@Ltopa*arT6UZfk^%CjJs0fsytMEbi%S^$>p4=Z
z3&?uMY`bjQk2a49xX#nBu~$>KBCzSf)v<`CuUMoEul?|T+HqV4%p^bHZ~=3wwvC2#
z9a__#o~Mw7xCg%`m@NJh>+9`k*N+`f4x==)aPE!32R>69>U}yQpwqLJ6Eb1Er<9CI
z{2f63dn$65^26%!h&Fh1gB?3>Q`~Wdwb%mX=m{&Sns7Hh+V)8h@{}zj;z5h>nC@>F
zbTc?fG>|Md7iXWwn`N1!fiRpZw)o?A6*9#=0o(k4sQ=uhLMxP$wabmtSNxMye!nu5
z)wCM!*Et*$X+`p1itA{!rMEuXzuP;IIVR`BMe1AUKOqSGK~FpO4tsd%GPODi#$eT%
zz;Y0Q+4VhU5sDI(9{BBN#!r=!pOBxij!n8cqyP&2t#Up8b=lB1%lpRa)3-v%9S6OG
zp3Nq%Ay*hLHJg|*Qd=`TK8&WdNP^;cS6r}rZI&>B_v(GlD;%xu&pkKm?4DhPT(NTq
zIU!X~tG%G}e81F+`7qXKGe@-(`T9)Vc$W=D!JhS}$}#iv{+z8n{~3PQ(yb=d(PF*^
zpz})%t2Mi>V8vYAivjwBI}QxwJE6eeet)k3&Uk0KzkPNZY~biDI@S`e6%0H?Njki&
zJX*ycFFJdL$A}{>|K-XSkLHm%*Ic71svPZlfX1)`F^dQY9c7+MUx`j&Q^5c1D>Axb
zhNE+HN7H){jnx;Q#6vC4)>;dFb!8KcU0ye34zBgRT2xb<3;lHPD8|S`{>^T6WPAg`
z#?o&g8q48>==)xUSEZ1dGy@!3*#p?hNVU5=$4Wl!qIJ>}4bI9#Gpg$l2Qlr>L6<<y
z7u3HxQ3Bi71HM6lr7qd=fn$D>uP5FFt7nmm!lr*&UL^|Ty?(x{nz^K%5HUohpD6zs
z_THam^)h_1q1$2pu|njnP+wgoYry9}xn5CJswu9ByE<zjbhSJfW`ET?AFXb3Aq~py
z(7zm6sv~~sWK2~KTc1gWz7LIolu3$pTMzpT%LlFi(R^$Q^<`=@?Z_&(+fy_%Dm51D
zq2?#oIa!lSO*=^mG~5u`5W(z6+om<iV@Q6+ueY8t$%$vy6N0x1J(~Q|Iy82IEW}yR
z#i+*UK8~;bb!Wje7DpS-9jw#gT!K~G=SstO>s+sF72b8XzFgEP3*)LfWA&G~FZ#S-
z!d1lsP=E;tuSJcSUwuvqb%6EcRX%>Hb}qME)lq@rKFku*N$B`(mW~z)e%K;o#7vE3
zu5z{}W#O?CZ84jQH>=}uq+F!nD<GGHne-br8Bxm5NNxw^X6Q;&*l#yFr1CWk@>eK`
za;mrUAsf_DGFR+AE%UXZzxxs0H-Exa#ZHf~lvkM7qwasrwh&Do{3q1qYX8P?F8hSq
zJ~pvc0Nr3st_Uxemr|}XrStYi(aJ8n*&W0)K14yR47D{w`v)P%o_U5KcMQ&O)PCdc
z_t5LnU)xMBYqhOy21ao#35Rb9T@XV$wdYV)dok++4En_Uy@67`-~bJxj<3p(mBD>_
zgq2Eir%uKgx$V!ZIxcXZj!_`%toU>-auK_re2rlith#7fTzRIap2p(_JfEZ$%nnn0
zSEcgUfswkjh7$*CW*oA%;AUARM~%CbWR!UU<JS)~7wXR$;Zy`RwO=&Si1{0BS^;k_
zzwI^-GJ)y%=TAX%-?r;WneRe6)m@00ANRvPA7&oAx`$jr$i5kEV0K*184m|31=?3+
zK~n!LGEjDy+pHUgpG+5F{^AYexp(h7$Z5;1Nh5xVOXZ;sQhB|dV0NBgzau&q-}zo1
z#mHN;##e0$=-+PFdIpp0=5Wus9h2=?1xidV1_)OKO7pxX+w}^e2cy1dx2o3YhMzN|
z8Cg%9uID}untU)l+H1&dIwD;L%Cp)<tv!>|E3VOY_M0QpF)U^g32sS*F;RUXQ=2lg
zy0Bz)$XT(h6{$rj-)dA@EKfA168z#OinOIG<S%{3BgrTi-ruKHqg=`PEZ^Av3^sk$
zmY$`|BG$3v93$WhQw3JMc}-Tu;xAC(z0@rq7Yt<U2*wgY5L)Rsj=$F3@@>F>aN&8`
z-%Co1Xpw6t@xpC3=Lgd7fHz@465xUCsYSBLv;xqZC|Ai&v=6i%VdKBREpd>-uAW|d
za{HFr+pi<z)Dax0{mc)9X2GV=6^}0db5Rv~x@PU(zQsCdzsjAgcti_0m};c&whxnG
z;QpAj$8F<6*r$V&S6f7vibEc$t~Y~>uBF{3{%qpf4i$YZ;>BD*MJ<d*7T<k+@y3mv
zgcoP0Vi2XI!SQ4w<eP~4($ChW(#X>dCP8nNlnSCek#4=ykWb*?ysGiiZ%VX1yUkBR
z8tlX#ImC^aMUc#eIKXxL#IRfIG7nz?bAVQ4$|LTXzpU7)*2%mTV<u@6m0+CxSmEbC
zW}q<jDxgto39J)5mlnIEmRl^^MFLIMN?QUxx*!2eU;dJi+TO5T(Jp1ade))!9M<_g
za$edR5r91MBR?;p)IW*Rmoi9nEN^dKl4Ir>eKq(FSY3OI7@r17()~oQ{PfZ()6%IS
zd2~6t)cd*e_k$V?`QV`hOk!g6@Ch75Ns*kr5lf)C(Df$k3yRy18SGm$vhA8-Ot*O4
zKaZOTkAAt2I)a_@e1mNDIac(1Dn{u|dURfD-sB}O)G?69D<Ia2C{R3H5U%q63Bw)V
z=t+77-Bjel6w^yK9@eeqv*Q=r!|!=NCR<FZP`oT2Tjo1hX=wl(O`z4eG@zJ*5<d~?
zDbYiMD$D~j7>0WiJD2lP>woF_PF$8xY)&?Y+6*B%Zu0{D+V~<%Kl(WK84?q{^tmr>
zB(7J?Xn&K+Y|g9S5JgWiMBn};p@q}%@~3OrwIkvI)-37EKN5lAv1~Ob1S7s;P$l-8
zx*nyV3%ZPKv0V(y^hejqy16L&`{0i#-d|>^?;GoV>lct_v8Y(Ez@I7<7OTpp8d?M#
zQC%SwLY^Vfg*K%ZYq{nb4mnaiAd9;Eg$o&X3UY&J4ER=>J_TU(<E{bk6Fc?R2*T=~
z`VXK@6<2ZBALfh^Yd&baJ{7FvP2Pm3Onz?Yr{8G9mooJGz;q@PA3@^XavGyu&33dU
z8KS?z*L3Kc5uQyw)b-ay{MA&-S@aqt*j#RQ)2lY#hE#kIu|6aP;vDYkl|ZD&H<jB#
zJ@ZX^QOQQrk24-|RVc<-A=0%7;Nw+slqEfbjvyU=3d8dH5QR}rB@H+ed*}8%!(i!(
zH0S=r7^Xm85%R?{E`(CO-@dydOIiCB;*R*QaUR=ei<cF0W4i)LwHvTQR3i=-?n+^V
zInzVv26A|UoFU~U!lM1{`=ziYAWdMubauX;+`7L=-uIxS`g5O5Hto>-@<ezn`9P&Q
zzhk~1a-zfh4F39d8`pf57lPaNcf89pR9&a_KnJXHN1aacS4YEQRGuK;-;+gCq72{6
zr*Vuz^C9q+$R549d-K#UHsbC5Boq!CZLvkyt+xTo$7CW;=P<5@(>dtht-`beJ~#v$
z4mo%kwNfLc5e6CW5>zdkxUj3cX#&&came%9<~$}jhjp@o5mzA06Vx_yEok?>j`=>)
zNy@U8v$pyntP6{tUokf8+Ac_WitNT9-3KwxlFyRoAqg31dbQ6;nsxH!Ha(mlTChQ_
zP5HA|srDSMMFMWm{d5AEA0H_5S(~$ik_so|JXgv?c0$pb?5G0wEfs5fi5NY&Naw0`
zy1!>J47Ub%fBzuI%{-^Iq*VLurS9!gk1<74CUG#15}b{M9fpM6`(9jso(evZi(^<(
z0iKtP*1^%{-m%t;`aJ_2PeELP$>*64P{ylYie0OWWh}{`HWdEJDN7>FB_$;<^y3<;
z#?v)JI+OCg6`bq;!R)e~qx><*bj<bj{qwv;M$K|T=sPq;44}(?=BmX_qwtE$O8fE#
zy}2sQlL^=JX-KAk!;;OhzALe#CxH{nqL@b8TO6GouU{*#?8qSQ!D{oT{Rza^lPDZC
znJyp?1*ooM)co$NDn+OwviaWer#IXkKTT>Ssqi9jhVVzejxikJ2sz-dCS2VU4ht-V
z*VWWglmID$l&|c%3)%>x@>5M34`oT!+xMceWYa7?S?c#JbNws4jc;Ka#HH#cpn;j!
zagc5O&5<6rO#leMxFSeKWd5qiMTT?#9!<OaK_a6>zNuZfrcaPcVQ!ZAHD82TP42hF
zDVz-i-*_qzx}SDPQ;l1<=f3r|(9gdGLaJmRY9(}s%3x2+u6<ZnQ+S?^>V0O(7ivVC
z)63v6T@=)W-GB#Q>T*_nI;*V<)#0QNr>sKun_4>*>Xl5w*=PvAsC37LZOBL9_glW)
z^jKxv9j{u`eCpqd$OTh#qJnR<?lu^oWb!cx38sj;sJHS&zPW`xHL07Q!$rD!1|lG-
z&HHt%zs`7mWKAC>d6fR)1?<5`vRu5;^m5SFsf>5-xn?-8$%I0&*aIF5%3#Spk+N6d
z%T{tH`vezVEQk?Cm9YnkSZ2x>cizI6sH4d@6V_#kZ=|ZHD(OxMu7?F8gpd9*XLvJ`
zaR9tW<{=pm<S~E@3MGcra!0T#Q%=Y8_=BZ@(i#Ii+Pbo-3hTN%YVC^_vJulLw&u`S
z5Ml0m`BP|Hm&~(`^2AouID~*{k{RY}C_7KNei#<&^di=zUj3ekEh2pc6{2^W!qV?J
z7FuMEx%oi_!L5BiY`pE$Mf}XQZTe)$9Yyi^ZdSIZgG`(1M~0X#Z;{;iK;B1K064vQ
zLYMK=k6}c@$ZRRJKk+}~tLU9VflSY+-Mg`_%gTDCQWTrhZ!wj`4|vtkM%FuF{qScv
zmP9K9ru)}*6oN|52j1}+1H~?{ZwwN{!koi77X+IRlFRG!*Z*X((~iWwM~117fPmeD
z57riT1SvUV%XJ<W^Q_BpX@6u61yclEXUN9|)avO}S<U&8KY$kft^hiGE$98m93~uF
z2t2EQ>%gpSiF^O1wxJP4GHdPsD)Xu``Y#6Of1&{XAEg3!@c%ve_5Zo*9KLfSBQuT{
zJ^xE@Oq;TY^2&P7pCmGfxo;OwO~-H9d1E1=R!F1@_*%U08T^cNvdE(!_vh{gxK$Qx
z84WFerjcJEUd}p^wzTCum7P|zmqEqMNcyziMKNt<eO}Gm<)1C6{!pawm$Fp*EP6-y
zo6TtRzG4#ZhCLs5gD@v2eai#1y)h>wtYrVlP&6?#--1U{{`rG<Y!~6*qC!;)loKyw
z^C?u5AAGc|ud(Ksj|jgT^|Z<y$LH_#w4P|o-{8A^*db(r6&7meK9J7l@6}HyHbOLz
zM&-0Z<}#f$W-ro-zhE&CUy(+}Vv<K4ecBJ-%~o7iCkH6)LUac@gXh{d0BbG&fSyDH
zhgNs~w0jQ6!$h*Vh|rUe9@Nc+WK5{(o4Ic)eL}_zu_pQfSM1Q!@e!?UKeVFD=GOzv
z4R(DHE{h}1>uA-aKKi`Pp+Av3H?4<S-vFJn^6?#~t@omsPEP9+SkHacj5)n={ZZfQ
zyy5)eG&9HCE0cUQv!qP@NNCF?jeP~H8_H-;fh}~?*-tB^vy+=I=!el!1yob0oVLE5
z_zSDW%(knJl}f#F2GkMrTH62Zb{iZ#pVO8$VkchHqWfvK*$HkEVj@HpmJB_C^atlD
zs-UZY9AZD$LT8B&joE(KGzNcf1o9ns%J=>V^X+)^f@I_6e0(vW^s;$^+RlzzUvL#?
zBMs|7#}>AjHsF{}6asbPNM7B?v?qV_hlN`~M>)}P^oY5-;MqhHDm1j&b;muVMT$55
z{BU+@4CS_P{iZg2<Sg~r|EZR|?%+%e+qr_TySVVpuzc@gw0vhbou6a<IAAuZ-i4xC
zN;pV)rBCcR7vAa}J3}X>ofhLNn&mei!{@h+IPXxCyB16kIop>%LzMZC*y|6g`_8Rs
z4VM-%%`?1=dmSx+IpYHa_quL0bRT@V{N?&LS|s@V-ziKxV#xqZ*f%(wu)=)ZJxuvB
z!I?qvn9x&eP+IG~CXq>j5CFJlk_B~XlagNaYl6C73(@GS*<j3L6ZXGym;Gzz5pa9m
zYrtD;VMgH6kv2NKt%axsB$Vv3S(UyTsyw7B1$-00>I)1@syyhEL3_3ahwFf@#Q~Cf
z_u?VYP*1Tu*6q0?E{9@2qEOrY_?lG4_odx#KiieJPw%}Ek%`ci#s?_x#}0ap^hSaY
zFv){G3`%`K0*0Tw{cU9?)FsE{<ixs;g)4cL=}S9;nXE9l<W90x*+3i3%2CWnL-Zvg
zufn>#Cyc7>Vl)}+g>pXEw3V;c{}El)fVdYD9)_Hr<%Mo1vhdxGR=1LB=-~$7e9I=b
z46xHQ*vQjU8^RP~0R6a34kBKX3+Ndt=sfFV{H~`=G)3{!;)wpBUm@_;qub>>ajQX3
za4Wa%a%ugEdYx)O-AlsgbLp(M0UAOg>WdC7>pIt*A2xgHk==sTgDr>sulQL=b9l;e
zHHu{9{;B>2KM(L|WA3CQDd3u_+#W=GyKRMo@g|U9-+kJR2WdcL=bcLus~0w9_&QEE
zM~)%IEX8=KTiM<xdpC1FpSOCT=Kw5cKUt|+WMHg)967ndaKM^0LAUL&yk;+zRIpEn
zATWrl+Oeeh^3MH2^pR4>>AGL_`*$qqtV8e}X+;ir#&KnSJ};W(`CJ_XTOm?e3*vy*
zg$wGFKMR#9TC}y@Qi5@*5pO^*V|`vme(9nY_Z97S$lDjXKgEC4yH_2!Kucn^L^%`N
z&Pj70N}>63aT#B0xn}pQbZ%BRghJ#$L3cov`+;sNUmm44>a@sWoM??EppM22Jm!}n
zHItb-iW(M}9X1s81);n}HOwe2<Ub0t4y8v4u(&NG8wp>wy~Q}+w}_5c>=3KgpxW~y
zDcUD3k*S;e5c1$*y~SUq)AEdjGdU^r+ruw7F)8D%%zsH09#sXY+>oH;)<FMOr<Zv5
z$IB}F8sc^!ZwuJCWPM4n2w88t>hD$u)b8PQ@WQAs3|5cNJrR_(?%7uiB+AyDYa2?c
z$-<A1U;Yd`R<6HFuX{dww%YSl&#I^$l|B^cO+%*%^AXOnWZ&rtBBntSS>dtrp50jf
z`LL%|yx3HZR!vd0{i(Ik12rNUMj4#tOBEN~)N_^vG32piDVuo%WYG=I7FQt*-YdzM
zd!r}*rued%Lp<wKo#y<TXD+_I7lQN8AF?tKw3&6f52*Y+EGphyj-3R<*r`|)AYmIw
zbwhtZ$=)nCGZPcj$NqotM$zXgi9d{FXz0(-f1{94(f>a?i?o!Ll~V*1rcG-x=S!m|
zGKv_Dv;UT_PoG!*ck&j=sDEH%HnD?sG^=x66`Qy+G&kFR`qv}eDby7hYaNddEvG-(
zALe!{xj$0m2`b;R2>xADMf+CxqPoHME91>T8zA71XNH%vOqni*4&nMi7vBKC{zY^}
zPS(F~Jf56Xc|k-}>hiZYl(bM#I22OyeTaG#|DWAA>w{F)8CQ}Y|EQU?hMu5z`_H~M
zJPA*Jlk%_g1g;E*|2JYhrd@b#>fm2}U<Led8{0pI;U~?+%q(tY^_rU$3t-TWX=7sp
zl=<b%3S-a}YNViBgGw)u0O@VV55f>EImRi<4;H&5GUQwZ_XVtN;N{$>@BcXVi*V7_
z&k1)&?C5K$k+E(5B`!}}P$$Yh;0;bHlyC)7=1tJJVh?-Qn!G`7F8^m~kP-%XAT_5F
zN0ipIzaxQ{{?4bcr)?o~rMX$=Xq@{($Ko1EJMib!>9Y@<<+#&`KhM^nWSfb<h-P5x
zz6qN!HhrRfaEoU5Sdj7Id-UJeE;6b*uKTRJa!yn66L+<TsGudP(|MJzc3)dKP>%Jm
zor<tsZromjTss2%rJA^E1-6!_si)6ON91G%q1lQ)42Kt6_a5VpM*BS@YIu11A*?wn
z=7kx#)W%B=a=}O&@V^ckd=_m*&=tHCt{7cH<?Dz@<Zv-k4v*d4v7?e$;uQ%<Il5;I
z&kXGTP^er2C{-V@(oN-hs<A7PchFdI>y78=gDSh@k>|KW)M(wyu77%QHZG4RzGd{U
zO<g2-kklEF(bXaktL8x(-(0b>oRZ1_?Ve6j{#^S$XRh1-onqMF0gd&RAjfu9hw&~;
zpo3JSBRSG0jHqw^-m={TW06uY>ass7=i=1Kl3rt;ETYta%meTztk5*G!pC2zqka5Q
zXY#Hly8)}dD^YDW+>{GilI;5VCL*ctr?BqQ`A4k$O%c(vfvd%dDwlEd>-)u$BYM^e
z>vj3s<8c7f5>PWShU}I93#F18#YaSq8PCqE#v)ZGw#NWbCn-e0D0z;$KT$wZ*@K^o
zPUM5X0ky$!%flk}(_gZ6OCn$IQew9KY`^VFY4NxT0T`rdI<kmyKFZCqWl3~RCe@y8
z3EWFj`&%Ja=&JMlP3)RY*hA+^!)fFzh@yXu8(*(1Y7x|uUf^kWym^<wS>`!5VL71~
z;e&7BTX*^xmYrEnVfRq<T)r-^#BGzfpvyFs_!c;k7`66g@)r$vG~7z{80y;EuW&(Z
zAJzS-K6lS-^2r~H)nppCSNMZ?+-(YHND=b;^OX4%R`i|l%vx4Men2F3RSbH7R^iIr
zP~Z3g;hsZ6P~H}UF1a9L--Mu2Ff8Fdckk6;uEj6#{CiyvXF-p@h+C(NS|8*mxvzi1
z-ikA<ZN0mk-eM%sh~vorjIN`&#(KQ$qag4gSUtc%s)fXu7{)$4?a2!Pk0#JO#Sq%f
zozZZryxXMK9^K1jVb*D*&^V&}+QQS@b@1AZa2#e<i$CS=`IG9ELf&z?Bx79u98OzA
zqI6W=eGFgn+gAl_M?H1PU!eox<HIJD@){t~p^Yn~;w9zS*SjKUaf<bwDu=w<v!uA3
z8o7LGf`1|AZ%+zRe1swAbJ{L0Eju6E0oT_09{nd1(Rt}_3YDD;-WPMOnJ+5TDkkLI
znh{%~`}aIz=WQjs)!y{()<Sod-lEkg`_^(@sUOg-5*0W^1q9hp)mBYBO8%l5^n))Y
zy6V|ZZm3HkrMMu;-e1Rlh>N+2mA$Z{4%VESu%2GB1e9i2fTXMX#T%)YEaAr=sLzZ?
z+FC5KrV|S=@`tjdA2c1z8k;#K1-W3|7uy0O=MHB(e59;zO~W3o;z!B9$l(Zmy?b5b
z<a8vHx0V_CH)gBD%V`+m)Ca}2F0Hmt)Z=GQRt?N7%*OV&^;Rx~0RW<&;G-Pn-IcN(
zo=nRY7Y6P2?NU1`Km9<+wIt&uI3#@H4Unmj;fx_ZzqseI1T0p87zeOyxqhPEAJ}br
z9PlnJoKZ0TQ!ELs5+6^h-Cr(ZFRGhbKkM-D%TDzoaFNYR=-XO)Zp5pTM`26#6``Mx
zm)U4P7bhlZUlWwL6KyQI$1sMy==nUAAe#;;xxSOsZCS=H=-Fk0_AF5eqD?tnh0{qg
z>fF)+G?I!R-5xJSH&a1x3)El3H<Q(M>e{wWB`pDjIZbUuEQx(N_c6aL^IC+KP9(>D
zG=|>&JwG#z_IojP^q8v3t#=CRmgd?uv`Wq;c*2x;K%fopFlL2ci?dlz(u6H@Q0#gr
z0~fgKP%p8<ojf5j`S-P|weB1hcH>dJkSq2QuwaFjpk0U!UnNqXb-$xz2tISX<6~GU
zNaSm`Uv#eF#g$TK_m+LKuxnQJbhd$Fo@yB*>Wx5r)I#87-%*~DyS?mJ$2a^4Nj8lJ
zUQU~(gDyl-`krk>H;Fj>E6$3?f8vWo0}>pmTZbi5j)nD8G+qX6fEQ0M&6;Bkhm?aI
zoZO&FwU_!^aY7T?85#-dilYJhj9Q$&`1vkU>yzS(Fk6GnDt#SvVfGj0Ql&;qpJplB
zt+Q7`Y<gzw3h&6ww!an~NtM0^d&LMh?$IrIM0alU3&!V<6nN$&q@L@%eXA`POA$un
z#6r}OD8d8xk()};@?p-roQV~TyPOb5vrXF<GmahS>XcJP#tc$h!l+T>^&I}r^=@P>
zR_wm#iOofStA{w5;GDs1_kv;*s>edL8Dp7;|G{G8oe<#BMrhoeT-!s9A{)fDj~L#t
zIqr3%%k4gkap@**bgIs>$U+vWeoK+1HVq`#_CMuJz5jq!<J<yYzSDAo#U|jlM0V|(
zh;dys=giR7?fc%o=I~$R=C?hTITaQ>$7abR#MDs+3Ux@_MDaHxc2aqEccjT4LS=yG
z50})bHvrL6CmePEm-U{+gpW}H#YalqJu-GMXHOBQCTNF&6xq(-s<P&Q--x$)W~Zk?
zO-qYJR!%Nw!qO35!=$OBQz~blHF?11=%mU%_`ri*FmYyR)VSVK9Xr*??eeq&O3($|
zM_A53f1zNiyE|sHgk9^v)-v!%FS)lr;Pq(Ka{eQ<z7s4iLSLnYqQgTTrt3>9qy`=@
zH@Y?L&>VQqxN435Ign``3$M5N^mQs%na>Vy&=ld9cqV%aCk3mw`)8}A#6dRdd`Y65
z8#58^EWfX@*GGNnZkED|fyY%>J*tPF_IM1#{?x50_ZZ_k80IG_Y@XgWD7D^I3iN+V
z?CjiUPvgbfnOUib_=)y!sL~h{Pk0}eTierkxx&Uc1m8{YUya<s0gBeZmW>3CMQckA
zr4}Xrb+~o968>Zn`QM118{9X#_z%2|_}|F}z{*ZWFVWcuavLQwL@)9*z4b9bB~!l_
zf;&5W$lNq105XJ7zORwPgx!3Y@2OwLYx5oBs}7$H$ZmKtH`H2H#2H?ivpaJ&r3zG_
zy%LXa1oYOONaZ#%{T5@;1eYOKiuEc)LIRMCu;#>h{MBevby)93ct8hHiOn)fkW45V
znz)a)@c1mcdd5<o+){ymOT)5V*W%UsFTq!sb2U<)tndR^6BJ=rcnrDDq$9q)a_BMm
zTtss9+quLAm91-dqbr|=sV1FnZF4xsjBFXdWYz~XHHTHbIK&A!I(^aUhnUEA{v{!-
zIXf&))JMuyV4d<yBpY9eiv#FRNDi;{^>at8&QlAD>86VV^9o5V8p>403Oo@O|0N`K
z@BxbZTS6}3w`lX8wA|xZiC|lJoVC`~q_Kb1XE9mj{qY42;c#cfla)$byevAJZPoT>
zLcIMgzvsd1cs}C;NTSM+Dkc`RDSgcDT!INrDq#fdg1+}_mAF+RFG_WY6^(N0pHAn1
zh79NfdDT-(cVoQiiJF2)feQCF!*j-TsbbKZ>l$J#cPY3s_uv7Ax3=cBCedd2F47mg
z*J4*)wYupV0%VixcPXmmFE-xVj;cnS8GP-HD_ed4{pVWb&$@l%UIgK@yY*>xL6`s;
z+tZOf47kira<SJU%19e^zcDq|C$%XurNf7OH2us#pA&+S9Ve>mNH_7T(H-^~W>MzR
zF$oo87UK>DQTwXbWhltht|b^ON1-b9y06+fkd%vTX5@e^m^gv9=i<ML)@21EbF6PK
zO4g#sx^Gheo6k)<^8?V%!r!4Y8At~)xSOmLI=z=2lQiNmw6;Pe8he(^*Im_Z!AziR
zLP(>0_^YN+`L0j4*H%&TGT%74zV34#OhTRt>S{u!2mB@L+MnH0-<Q4~39pd>sBcs1
zh5U?e7qxDG%jhzHe%jefwEV3_-quC;5x5sMgy6?4wlN`-;;9GjH>q@N2@P%^!V{2(
z4NR$#C7qeOD@gRp-n`v8#@0d3f+eWMGtH(f<>8)4#-5T+4B|&^I!p!bE*ola7?LX4
z&77B93az5I6n>qX{BU&CyCCrtCjC-ReaUYHp*X8T{Gg%uX9v?7v-G*Q8zMa>)7n<{
zq>ILdK5+>;z#sSCk*-~4NTVbsZ4H7Jy?eJl)w0u}&FjlbT9&GRb)}{p8$Hd+p_%k8
zP~H<7ng_RPWc^<Z-KB-vH@fZ$bsxQ${DrN(g<bo9;gMrC*sgd(B0lYD?UlCk-&mc5
z-f&aOFl@J%z{=n`aEx$bC@=0!K8I4^akqj!nr0Q{23Om<<wtm4aw;dcRwb%Q*@Wg7
zol5IIrZOC+P!25q>O+}7Mcn~>4?1*{SvWz9heSS;+SCG4D$^#U0KP=8uNzwMUB9=3
z8M~=+=jPETVvsbg0VEBTUbz)!gji`XGzuM}ot38WUGIxsvxQ36cP$y1(fU=1bhj%=
zS9yAC0tq*ERCrX{F#|g8$V+W<;DjXR*(V(wJ9LhtjRJXZOh{C!U%k=;J<C_34Ncp*
z0d#eW5#?mUkjnL-P?6#aRh;VWwQuY<Z#fCIa@s0l9AyCbJq5=0=c1-WI$c(d27L8Q
z65xvW!x_(mrgfMG31DH^(}ys8!v(EYVL2|-!)S@)^!Kj@#5un6kto;f(wr_$vDds<
za>d_$BSO%kF)DeSv;dY>D>~Ni{2>F=Z=8?EI1Zx09rMs~E7L%$q4eFL@7YOt(aVZ1
zQCViMAE47j2nq68BfK`*C`5H?{w_pLb@gs7b90g36Bp8Z2La_IVCil_j_mR>>>*Eh
zk>++r(xX*R7M`iE>YPZWYImy$b)BYO8dPGV`ZG(xGJQ9c4{H)bbyvHi;k#zCztJfl
ztE`We3~IBd(Zi<Io=(nmPj<cAF&d`yw-&n7hEgb=&B%)ajgGF**7iy9zOKBdOJIK?
zI_tXw+rgx9HB+VvtFwBq6UImCT;cdSCGz-K!+h4GVcfE>`q&+m1&qS&*ck0ay5@{>
zOdmnl)NkoYhiDVSjR<C?<#E07vUVLzs&tZ`9YHZf3`9Iytb3v&OB*<$yGumYcN_<B
zC)m3dXsp>yYe|r+@pD%b5XDSBbvQ@eP?GKS99=O(`f7he^Kn7Sc9|GGI%@G|KMR>w
zw%({d>yzV2t0)@^wdo8~pz!OcJ6)RY+_-rBku>Eyah62;F(QXhB>F&5KhrijK-$$5
zEt6e@Z8e%HT~^fmh+0A#G?{q?ccTPb1yA=68rfo0SA=?)TZ#7e^L^PeUl=I~N;gr`
z5Qe?ew8d^{+H7UR?5u6t2#rccVOXdT)P6cidfKj#0qXNo@!=-q=%2Z=n{f^tJ{Rxw
zak4}84lR{w4OTR%+bu4qZQX)U6+FBcZ=z~t)=%U06t3=TW%!~I29eSI2$`9@D3uAl
zSJj`|6f$5E<~>7NXArSr7LT{7xc0y_j-LG$^a0Kn^}>N~L#%DDe{87FNs#Mi>1Mc3
z(*bj7c}YVX$#c5)TMa{FY=$)@`N|}NP#Un3vt0<oa*N5;LrMF43tB4+;boKywZgE}
zg7ZMe@1Z!KZ$iV4n+%S6ER2z(HEI!P9wBT_$FXjgSZ>ERg+#XJHf64Ui2Bh(F}9`;
zZZ@B3y!~5M%;sx5sOSausgkgke|U$%lbq=AY@VM5K2S2e)b<7XUzPl=^m`o&E_@M}
zpYMj0SRVXdX%P98szz)Hg~^s9q*YCA<UfW7qJ=!qgcd3gZwBwTE3nTAs=F|-c>oD%
z`}Tjf_bm!8z&cbbP|s7JWUYGbw&ju0dkq<8ME3U7$!|6c+_+R1y5-0vIa45|UY&||
zdf<3qRt0#nvW=w)QK_#YB#)PhJwlRBXQ;k4XNNyTkL*Mpr*I%p_ADD(j2>>nBVP$T
zry2Vi?H9)8k4pXnRn_`2sfCr&Ln~3mP9oHshOj+`G1n8BhW#ho;LfIv_kp@NoqMoW
z)4AjoT$NO7kscmh<zn7y6bqR=&RwbzAN{C_Ly9qGU-oqpt#`+Tg`$xc-hE>qWh0IW
zsm8j3|J#kP_Tlf=C7{vRzX|{nTbc51#u8X8L@6RerNb|LsW=?TnB^=Q&%e>W=8GvF
zo-je@x?eA04{AzPb3Uts?l>q2t3+Fg?B7%Jj{lhNEjw-1d5PUT`7rs>Nh!FX-X6?9
zJ~fEF0^R;Og0lAfy$}r*bp5jEvljyWOq;-z><BRatfE$9END@$>%*X=bu5pnz=eCU
zD;XP^H^Vl^)%iY?U#Bg3nOBpgT<uj^Z#Py;Or~C_asFTfKY3O>-KxH0nI!B%ho36Z
z4zo2RgM7moM7|K2kW)3E5mSENPn}X)_q^2$9Q3=shABj}RiA*yYEJZou7Ir-MXln8
z<(uInFFtqjebkVMsf2*W=^iBE&2XoUOQiSZc4RG4_v<Iuq6fk%{D%b$(BN%cqU5aN
zc;QbGn!{LOy`RM_<kLw4vr~lkGGSQV`q)yNBtVLNd75sqT*qT;cMnCJ?t_ykzQ=w1
zvLU4;gTi`(du|FnDO74p`zQ1Q1e=|MhJkh<IiPoL^I)IVwpXn<Q9xZNtI?F2U}B2h
zu#GZO<S9}Lg{}QZgV>un5g0?Ipa8n^b}%2gk}!mq#7FL{a@kXMr&XIN%zn$>3rATB
z!52CkZ^Dhzv>9IWQZ>Y1k)=iwiS^vAPs-fc-E-o1eQqbdD!c3A#&3bR7DnC*Q}G;Y
zH=_AfsLL9l#yC9fc28f$0ohW)52~oL!jr0mu;-^O5pi7PI##=?#c?0r%qW+ypC@Z-
zr++yYnbw*a?A_P>$k%cy3HDnMR#%i^!nSP))NvrQWb)-0l=3kpch>Qww%p*x+sar!
zZzM8r6QWRfFFj1uTyog={+0%1zYjZ^UF4~7K(AMT4{ogo&oWbDFL`v&Bs6GG&;-w1
zXn{zf4c2Yqg;pTKbX1dOnJsOQTQc(^IUEk-mE5RjineJqUWan4g6&O{;1E@Z27d@{
zA^OE7eiU2bp9&+0nO5w{tE1I@MvVEi8<_SR&a}BZ){tt{%ldDRm7ZRE<vHMR;rY4K
za#}K(Pv^KOX58{op}Q<J3^<oXKa<{*7x54|d{QH3XW{)3$-WNh9lX@OAX2ndIAcVr
z;F{9h4>S`w`lW<sHmFRcNI~5c2rARoS;A*J`A%b+cIo_iXsjjXyGC2>^`8B~LkrO@
zNfF0am*V%gdT-~B83rF64Ht+M&k2_FL0~_`;*ee!J|P6&1-Fm=%P4NQQ)1v8hu)-a
zfTcldRD9lR@HgxaI?Q;83e;zWOFokZ;z<>YCENS^Dd1>P6kUN(I=}Ygi*RL_PNf)8
z{#Rf<%QP1?)k23=zctvS)GzE8n4YRspd#2}LF;}&arKLwTccXv|6=Veyy9rvK0zFU
zH%@Q}Awclp7CgaSg1fuBL*tM@AZU=_?(PyCf;%+cIHd80>EwC7cfOt7nX`M&)*nz^
zU3JTK|Ljt}@UU(8rE-&R!g}dQKeZsZ=G_!BsoGbiMQYOw0gda&VD-%o!uv<wQ5zZ8
zqwV5v8hwggwU+~El~{bHfw+ob+L$9o4he5^aS=?@CMzXU&}A$1|5A87Oo46gOt8(p
zyU=uC-ZSOL6Y=@4E+spEt6hX9v2V9~iIrygB*QYFr=ZUJvM2C_OGPPoHW`X3smpFN
zPq^RxYD9{l!SE&>;6S6FQrB!Q1M>ZB7!r$lWD+~9nsP<Ub$+u74|H=)7q`+~QDND=
zvu5@O&frQEG3G_6zCmvIA~Y))E-@#zVB4kt#adZq{4cn2BU6TzMIzceOT-`X^e++N
zC($)!twv;we*{oPt7tELmTo%*X<&fPddA&<YErD>P?vxpq`*>ojll(!!Zu15XX@S8
zvc4L;o-CR_D$i_1JR&E%V}rs!l;7gnwV&7jdbXf>r<KglU7T~3`=!gWgs+lRySbvb
zQ{HZqTXsdva7)8u9u{$=PKa`GUywaM#x)nI=zTW#D&wUrg~R1&ndt^YViz0fh*rMb
z$Hk5ffJ4IUW8eD**}0)lSMPzjpNYJge@M?oB)?>l<CV{N6^JP-bF8cKp+x&vIz>YD
zZmlSaoR`p{3lx<W6^DusIHubyT=;08CVT1ZbRt0iRtNhkM6N=hsZ=x`F<UtRb9BAB
zw%6I5?x{~({j{S8eJv%v(&a+I<gRq+E>e>gtm~fFxZK)po5mLc3I*Q3`=R=<_82Sq
zs&LKDs4*la5?_9x&6!=G@X^<u5m}|oqYNJ_zvIV!s`}$vh4$8aPhoX|GgW6+>;Qk|
z%&nKJI}KLa`WsquZini?Rjv{sw6*s^GcBaxc?7^*ztE-9oTfYh6rv(r-}m9CVj5-D
zd_ypqbT8!>?y3q*?)x~BrqP1<C-lf469oCFIZa^b5*;{Pc{v)3xdlK|E(<E8t%-gJ
zw|r)z3#BLrf3jNdLw2}xUd*EDT!eb^s#1EE3Ain~shobv^lx<WUpiK+m7GTZ>ax=K
zb(zg$netnyF>(f~*|6ar%Z3N=Cpd~4Y4{VJB|d;>oS2J9lA1{X0R!sK!RkcuD@I}K
zR{YJQ&xoz!+%O_MTWfbMAH@-Zd8DREMnW$x-y9WSgQ&MYgV-X$nIL!i=Wz#MlQp#3
zM-RR>*%hVm9)<S0yrt)Sk6n)}{j!Ozf3{>9Vku9!5>eDUQBo;&57JX0*e{JNSu{+m
z?Ft{w!)>smvAB)$?*xRWVYS&(^fJ}@t>WMhbf!Ecjo!%m`*+UtCYm*iD0Xl_+wvRx
z?<3zJ-N)?fe{-YTj6R~VsvL?tqIrXvG@XQoksj^5Z9xs1dDB1YTeh%SSUJEn3-Na+
z;l`lMl2>WoAABHcd4t=j)>o~IWF&WEeXi`ornR<R49y5+q{JwU@kbq4yUi`io`@9J
zo2l#P=jf<^801gN;qEFyTcKMIj;Bn!4$I|HfgQGA!9d}%QztjVjcU%%i8kL^tYBwT
zb~*uy!)dNr->4yk1QOZhr4LVve)JAUz*E9yqlGbehraQehqllU_t!}GtpZvU0s=OU
zqF>q|WCdWYFK^|0gF`_Wr};KTt`+@9pGC>Zg@Rr|X)}B6%ruKjmpY2th1CQ)mkeu~
zrHW^D$W^$LAHZxfUe#2q<YOvn0k}~s*r0w5SGT}bR59jxy8e#K{h`w5w_%YAp}LdT
zv^UZA9zUoT3NSLWvGJ~$*H<>Q$E1`ohVl*_oCsRpJ<GGYTyY<8YLly-&n&N~X}l6w
zJ9rQ~2Q%{G9_8>S^ft1dV_-yX^l_th%I<W`?pLYC=e7-d7X)MgB`<jMl>Iu_{3LKi
zo?kDc9KCrk2>xD_Wi%s87tc++kD;*jE&$M5)LP8UjVs5Cgyh{L7B<;i2O@{Qo=9o^
zz<IVCslgnwxwe8n(DlRUrg1HiyRPl`wRHEd-SZ=j7+3Ev3wdJh{)XceTf~6M%oSGf
zL*qlSS9n&|rm!c!X2+)XQzSV)s#n)(WG7GI8={BQ8=A?vhc<n-eF>UMYj4FppT3eZ
z#)+bm9ByaA;mPI8-2kU{mK<)_mM17tmzS5v>HZk6uBjOT<Cd-7z0~XUBb}dbI(&Lx
z3Zlwm=Y9t$REjW}^Rj0Ub=UBFg)qs+k5+cEg%((QkB6+&%i7l=Q*9t42Rn0T;xl*R
zwop?u!Li!N*e$!mcM<w%^mCbTY>)G0HaYk8C$WN<XRVK)Jl}r4JlSkgFE4t+Rn8<r
zenyUZbG(C8HLdhrP;oPZ4V08_7z=U;&<^Mb%@HpMAP0u8U>NiE;)ukb1jl^itA@%I
ztWp*DS-&Hc`FcBuDQW(b=@>McUQ5AEv_D6Quu}J_#5q$zvO18Uzo^+dfU<w1p9ADb
zdklI0>ve`U)JV2xf4ja^J%!JHcE<#j9V`}o3NHahi2zMzI?qY{zRca*Wp2&4kXW25
z>EUc?)ZX;uh0jb)3XyWuhBlbj@EJppLe{$gz4SBL0ovp{;im&7KwZZ9nDsc&(-nuW
z_5dx8jx1ihR441yal$WH(ghT&mZbxi9n7Qt;<e3AbRd7OTRPeUs{&_%g=K-2%&{Fw
z?;m1E{Vl)|%@PkF#EgD;a&Mw(1*a!adSwf9AAe#;J&H$E<kF-r4(w~Jc*Pk;55n3}
z<>ADrFj~+#+kdn4Z7GxeRW_oXjS6C+ieN^Ro0=fs!Sm8cyv#YdJsG&OZh?AJ<%vM%
zpDE0nD8A&ZY(;yiyVPU5_g4NuUqWYUDHQ1EO7D?_e4L$`ka&TOBIrbGV+Gsp^KX@r
z2E(b4KI!z1{E{D=hM{$SKPzkGeQ&K-tTNt|ttZuXWd?U+wXoaUhF@PMXnmxa$YD`8
z$ob)m*2~;z{u;Kx-(#5KI=&AdGOal`bpF}S)M>TyvvE$u$+sD!d~8lN7bI?$BikF?
z?A1>-PGqR9u*D}rP?uf5O|02!Pts5Jb4mS`L)_3Q-63BElDb-&Cm5Sp$<eD#aDAFa
zkLeLgjkoP#>bd-4XOCHwt@|=SrrBFDOpA&vZDXqn21x*mMgCxqa!Y(9#K9@LoGG7a
z`ud44DV3-<f4p!4`8OW&z>r?pW5UuNFhPT=nF(XR%?ikeB^?|DRIgaJ4vdh?9H5&Z
zFkD8ufCe2W-hbd~n4)5r{U1v+_uKXV($N0@C~EmXiVsx%UQ?iA!=j&X7(a*>O@tAW
z>*-G^VJfb@o%5D#TDZH~PZ@$GdNBVJE~(xHdy~sc1mzFZ2pi;Vk~IROv%k<!bgiA9
z<e%cav-H?6QmoILH}tY3+<gD@r%*0XlG_3qLsA)Ns0ypSS5X7<*QPq^_~jL53By2-
z@~`LzX|In6c8D^1ZIt1}i(0!?9kT`*|Kg$3>o}tu*Fh-)si%JbCRk#{*!PotIBhgS
zipDiq9L9o&E!lpB#N_rPkxs+2zp(v>P>j8IXm5pW;#Y;`I2Z&7!@-x!Bf>9Zs&zU^
zcr7poV@XjBpTo6p>^-t>Bw*tic45Ip74QT_QZ#%56bF0ZcQHk|jtuT%?#s(PbnN6}
zr(b_=P)1bSw}3;^3Fsbd0r$iMbPo<Vx8qpiQ+lFaZo>n-qX>Xo=QRo8e`Gk^<gjv1
z^K;y-5?+!cI4G)0F-Ri(^+s)hU#zMPMQnc^Se^pkmG3UXB*dTNS3VsR8}BPRA`B6Q
z$QEB*BILX7iYEL_pSB(bEB;;v{x`WIU|=P^z(4n1bAS~)<+lf^x0jyX#(r=@Okcxt
z6eNWZs-3r`-`*hS@f4L}zz$?Rwjso0N%4ynhv2bN0y@SEvYkF;{2X_Jm1N7g=NEiY
z?yA4I=J>@wim390(M&mJUZVy_sns~yqhHF+?h4Sie9&pbRbv>T!WuNnG^D0&G|#IB
z`G{`~OG*~cfl?sXW@U1}^+qH?wcSS=`$eq4{?jjnBVde%quY4399OWoWPq)a;Xg0f
zlig!pvkKjreT*2u=+D9Y8jiJMb}8Z!H+<8MEexA$c3g)QZzC<ZSd9`b>LS!Mg`(+)
zX_g^7NKk9d_|wG?aVz<8<4STln$KUA27W>Jocxwi4RGp*?q;<m?T5qes&UKD-NQ<1
zQ@oxM-%DoM;iT4xIMs3RR%b8k=<3R8H`GaNFyy@VtkZjCAK!DTW^s;YfZS!U+>xtX
zF6c|+!Rr}3>FRJmdtFtYYLD7=BhetLooq(uph#^px!ALfOvTs4+<z#{q3v0Ki_I2B
zH=DKALn>!_Sbox!De8QA^GtIl9(+mq&l11{Q+=eMWo}FIxnsU}Q{IrJpXCL9Z(|{*
z>?5B$9C2wH8;ETPKi6p`U%x5kVlFYA+=cN#n4coY;Ni7@*M|z!k$fB-G(K$N4G?-G
zf)~7>N-lYGScc1aFbEx=S-7Y5ia|uk3z=%FU<J0(yfJy1>Bhzr$V;RcCkg&eZI^jk
zFs2GiQVG*3>wj`TBBbnB-9Rf4A&ow`0BLj1_uRyJX)An+m!g~1QvDpY*xdrG``uR>
zPPv&iAD7R8bYaR+>%?=0;Ip`fSQhWuQ^3!B_BUr`rpK`fa%jiH83TXiTTzO=1*?OF
zEKu_Dh-sVCv6sqEQ^feS!e~$2I@mpX>d^phRj?UJus71PNBjJl6Okc+?+wV?(5v}t
zLMJwz1n#>93~U>*ro1`|yWfW8=AL{WL<vQ%1C&41$rlaK-0TxT+l(LhOR<Nz*@=$`
zpE67ICx4;yfM{--j>+aW6plCXYXD@Vz0ay<ADc0^pj3B)<$?{Ce`(9S*O41eWI+l;
zHYTPFIAKio{S=7->{3`|(r!G+H8Fbzjt?x6;VX0I(Ryag3M1}wOAzhlo{}cI^3pOF
zwordvcXidk%<f??$+zedix3*Twn-~372p-#c?jJ*-`-8Z>A%&#gOY&D(&VaPe0e;p
zTrX+&pFOy^1jM`c&q0oR$hOQ_<Xm|Oi>X8bfjjhDzybIBMgUuTsp8LHx2b)&o+Ic=
zle+N-#a8BM2UV<&A5Ljx8QfD)I+!4uCRvc9RJz~DY<!I)l8-|@X%~-}lF)9lYG}$&
zo~>GeziWYt+5k-KAh6s$p&qtjSZm9A_(b#Kn}1fR-9i?$>{w+DTXT-*u`x9Ci&t-~
zdI8x;X)3l!R1TYkU+#&&@J$zLqJbTD)wjA$V@Kqx#=Q|EoyxEkbRLgO6}Il9WP*HU
zqz4n2zFVv0%XKDoU4*OZ+ia!NwiZHi6mQezRVPUql%5I=R=m`K+`^~rlEXxQk?MDU
z6`1Lq0V@?+Yg-3Qq0V@!S93Z(4yzJ+m@;~$2ORn%(wCmJjg9V-o=fYkvV~3;UI<@5
zh>uy8&<docwmp*-bdx(I95lTk-qP(-@HA%n9ACO!dfIUxEzDqilXS^O`%uPz4#DVi
zK&+~;X?4BC!hWip_%)63`>s~Dbwkh3TQ-X(YT;~FPRL;HeSX$IOT60<*Ee3r^$hSW
zh1_+ffvRc<2xL-kuPdQ{f?^)h)i5FhdXl(pS&}#h&#7cpnLC_CSY~@m3-W2{&+7Y`
zVTpI$Va=Nsn(EznfLEE^{BYOg)T-%>nX#W#V>RuglX9-Ot}FDTfsQUtH`V-TBsR!g
zHKi?Rw}Ku;o7(sM+VvbgjxwS$JD;`5nR&zx8?K!ww<1s_Kk(MngbpM`Ew*ms25{L&
z&Li)iU&Y_lc+EIk@s3e(QP$yAy(4DKIqMsapCA#rh3}X+M0a+%(JXk+yMd(d<1sW4
z`eN~MJkWXW(jj>-t7q~fsAVIn-|;3P=xRfLs<e*Z1<#Y#IcHr!kQTRvnd(Gn&^Ygl
zpCV^OnO1`a&8H@Z3Za)PN(@TdXE(!@53mso1g5`+!F(2%O{`W>s3y69+slgkmfW`R
z*6)qzYBd%Y0tJRgF(q7)c`@=+bDUx_L%9c@h8zJOWhFuLYl}B09Dwgs-RFm5X<Vy{
zFSerNae8(g`9JXgrno-Yh%_x)+2VLMx?L$UwRPcX6?!6|R1FYhA3}saYz6n`T#SG9
zI$rg}9h&eE6W!2B<0=)b=U%cU>qq8d=ELzqI(fRG0Btv!+yvBg&AdJUZ$(BS+E2;O
zkFp;jq(0$%-*Myga8O&|U!i23M4|bqE5vZ*K!Wv8PL4_+$iJVg_nRcrWVE<XTW1>K
zOSsfa<ZTV+Vd=*Zer->3r}P>GF{t+pD`)45Kh?z~K2DGC&wMb!pM6(lrq5+CVrmId
z%vIRP`;_`C%EUlxena!eC0N~yIThj5#;{KpMDvdTx%i_=loukC5M@mGbF{>sM)&OC
z3xj<B&1)xO?Qf$F{o((;T$SdK|F>G5DFZ8*>HX_q{8LF{ov89)4@w#Bew7E*mzkhn
z{abN@3npk=75<xKqK06fj!T%7QHC=qBO`v+PmWJOyQfrFw7FTJyKuFmJ<5UVe<?Y6
z;ptxz5Wo?6&UvjM-<M4je1C6>f`MT`2HUWk@C9w@Xj^CWhu<ID(s48!iel=1K`*En
zWS^YFHH}-%X+GY-Pvv<oQB;!{U1`DXBeIjp7JzX+l1-?vcdddrDi<$yertTd5aJ>_
zd4cwHj_V>j8xEmnxDYPq+mG_Z_b^^TlKH+*U?-P`z;R7XD-MN)tu^oYx~C^L;lvcm
zOWUOju(*-nh1g%MO^ap0P5^l;e+0M1@YVs^!9feyJ;oD0$IHHOZ6<qtcTGmIfu=GY
zDYDrQbc9YW;dSNF4zw0&X;AL6AJ1o&?^kL=3O>v6c?u!!{|@}kQD@ZP2D_nj5bTCO
zgT}_>Zyz3*e=J_AC@$!um~2E32(xv?MRf>t%ZHiOVPBtT69U&a0wOwez&6MwKz~!A
z2uAw789*mD#!Gj=#A_qPE~n#LCCoi99HJpE5<233nxkmFw8${62jhE7liziRBQ=$`
zVzDvj8&w<V9i@ja&HQ*!>DC6G)?`d3ECq3y8`b6sZ7Zhii_{!^Tl_dp8V8vsMJ@Cr
zIEl4t6RX`?w@fSFU*#k?4X=?BQ-PymlsaO2>uKFTdZ&-iI8Q&zbwJh2TMfd>G9;Xx
zgBB+xOakGlwxVkfd<lf!!7X4%Pjheg*Lvn=QXXDbH9t0HGBlF1V<Tq-4se?XuUfoA
z=dGHi)J;aa=xs3Ul<Wx;S%$0?<!_1x=Bewb2KyOiNA<)DZ>60wPKb3Gr**n*K+>_j
zb@#t-{Ng)So|qOtW}cduTw<JG=t!s<pw>BCNSnoZFgAY7JsQVE@tqVqzPPm1n6GRO
zpg6EMJiPnd?|aaJ7TI5`9q8;wWwW7R-l2u>@Q9}VIdX^FXJIhiZ09hf?TK?2LSMW<
z0h@YjOn=5M;@7u2AyffVg=%+iijIztUle3JO9$?w69x-ng+2RnO|7WLrkc?>T)Vwm
zY!8_<kGdPe4-Qn4cFGIz_9mYXuubTOtW-0f>Ni`4ZO2uaMW4Y<dg2)E)|vcXO5t<b
zgnRU5BumccZ%3HP3;qbmsIhf@SD1Fv4ekT?ej4sxW=Umddc<!RoYj=yHy#QZ2HFct
z`HnG5Q}08EwJ&l;RN8;{%QWBj-dIs~;_M9%TM>Hk%4YkLral){Dx)r7cVLVQ5>JEb
z%(P#fg_h_eL?epgUpyLXldnV)`47lWR>`!PR^>I+hNE3p#SJ#_z*}WyM)26St6Wc5
z=?ND+&}>>J+u=Pt9AMOwd*uAQ``9e7WuGj3gS-rc=(08A{9qgFPuZB8V-~1;X`28M
z`?=p%ZF|C8Ekgb_#4Wy6`IZTL_~wD`a)$B1o@OJ`Db_GE6tK18WOQcn_O464{blvb
z5}zXbA=C_`Yrl7m_8YP0N(Q~rL4+SPQ*D1~(rq@V|JG{C*>D%M=%kE1nk;_yV}0Vo
z<t45(X`GBk<U3cAz#@G2K_5RzE8#{)GDLg->aKc?AK$UKiiY6{YfR`E*d$6k>LF#)
zzvLwU^rmHmO7Bm>bp2-;!<-6t%8H(8SWwl~ZzsoCe9GanJHyMjexaSThxJUHX-p$G
zCfH^&a#hLq+c^Z-B~W=zx{rP|Z^3@FgE^^k{ZjxInO%B<%$H0O89%8I?3sYgZo3A5
zt)vvz)LP%z4lV4Sn<r|_8t>-_jPmS+v^$SAW~bdIqr4C;D6`#Au7}VWjn!E#)v?IS
zf9kdCmQU-v_I%eC^-e221M9b&Sd8zdyq1r;_1F3+Gl8w@P@E+kn5f%-eMF!T#HYI`
z97kjtt(;!Y0{xo)5R>TON%bk(pZ59P2&*XOmq?fwUVCHK{pBZVl<!iy?z^(T*Ha%N
zKq<ti)b7xDva^i31U|O->V^iQGcU@<KvL8dV(6ANa|W(<_frXDnmR#fv*R4d<&g1W
zz4+|Qc9<6XZYAHVH=|e8j{2BmzZ*RR22m@mBa=$oa7Tm<S$0k&#$IyWD&2&w(x18v
z)=jq1LYzun>=_j(eFS<{42mooodUf{*C|E>zrYInIn=+i%Ll?_Sg+^Y+PRZ85mJ~S
z+<L^l(opRg0sW?)SxRf&Wbe<bY|`xGX<n50Q~`_?Hv4$ZVwEd`4$CH)qQ-Vb8D$z>
z7Sl{vP!}i=Rk@Y}r1OguH?^QmO-7%?Qh0SxEhG#juVNgx!D?*zM%zSs9d@B#D{o*4
z<OWHv)lg0+l^=`CM=B|z7k(m-n6Pv#gm-!84a~+gUzZcJc0fs4i}Ztr^I5GJZCPhS
z!YdHVCo_AgTJ>1g)c|K2fz(IrQ!55%{_Hj3{!~UARYkL;glXD+^MDV+EON!q%Iib1
zH$#S^`>mz>Qmv&syV+GlibXLeT(}}ngAaV1^2aJ2jpQ8??9>+;vA+$0bGl!WW|jdH
z!WFwSus1Z#^2etP%h5PqTIg$nH5cHO2}_q1E+D2p5*_~8`dyKSeVj09dU@8>4-5b*
z;gBnr_8C%PY6e!y*i%Lf44Tzr2U3o+orUh{Xc3g(8rWayhiCV|to>LgzazUN=fVja
zByFGBj5@tZKk*Ug?qp3jKiVO|^!UUJ1@@@gF*nz$9S!-7iEWY-KghbAyt>-T9{EM9
zNwTwM!+8hDVmuGelrYbHY<xA&ga~ponrJS6<W`Fi#spbL(D12Lf&X9<Ywu8;=@>=6
z#II%K;no<TSRfrlgadEi0vL-f=Nf>;HPv->ebI!>$wstPyKY(TVC<oZ;l-*Ax>Pq0
zHqG{0BGKsX`P>+mF>K4tFat(=F=<Xt0l_@oj#blAI=igkeA3qJtuQ=9NH1i@u3=b{
zyskocES`a`V~`5(RJsoJQ{Gz$Lx(<**p&ocbm@JaR;4<$biIC&mF0z209#1OTA$U`
zqlvcPEp0WJ&Z@K-tL59;27HaeN0#UQua9SzdLQ}YI7c6*d={^)+FrkWohs7Xu-4Ev
z*|MjO@>zE!KcmOU56A97-H)m^(V2wA$&lAwT1tlxg1G7PENt>f$tQ$OE>PJ7tpJ1I
zv~*rSl?mu8!q<KtaQEuLoEO&Hx|<x(7A@kvW>IKHJXL~8w`XtvAJvA$8!YB)Kd&rR
zot49`s`=G8CIT7pl!%xza8=ZdadNyxZnvSHxSm2d>3A2zlcVofe>ss;fZ~F)2AG6}
zw>G%2nMD+(gWvLt`l6LBkcS*$WSbn_8^sL?=_LW$hy(*zg7#!7IXk1JGFu4_P=>ZF
z7Mm=oT;9F3Qe-f7pN;#00&yZ9YR9q7$BDA<uYZlBBe-3CR_;;L`L$0<d;OL)`WGuB
zA!0O+)FwE_7IVIw_vjGe(UKv0pmibosef*~%;`iFEM6~p7$~mtsESni(}=}GIrF;X
zXAw@mw{MGHSiqg`$ETbrJ*HpRZmbg(AFC*uiCPRxcnq1Yb?Q&J^h)yb`M$_~d=n5L
zP)YfV@~B~>Q|6FLe`@vA;RHQ}y!R_h$@0xf2%qEVwvnK*5#f#F8Qe`#M-pTj{gqkN
z7fn^_^~gDGSmtIs)q?GG2LrZX^6pEsfvWE4@?4T>nJ3yAc*N|>!Ir=(qJ__mr)_&~
zOdve9vQwdoP_$=I?xr3kI%dVJF`TrpJGSjYe^fV1;ehMjBXzc@Lfd6YzZQqRGs4c+
z&u!?%zT0;tUgv>76Sx8<KYrixkZMvINCHZ*HG<qCg@wtOLM_Q^0_I%aCTNF4$aZgO
zk%UgduJ1kq(bJ<znZG2mb<F})z8KAqQUT8RX!+g9QE-kUd+>>=TZY<hg}j0qC0Ouv
zz1VbKOdj8y$7-)n@}i_l%h1l|8$|aJQV5Ax|7wF9EyC*Cv|N8LZfn_GHfjdhhi8j0
zc|Ryf!sW#-9dR1khjs>DfiD|6*2fJUuFbmLboSb>MzLphmt}ZLI((x2<p0Pni-`LD
zJU6@f(irfm+?UG!sj%|<S+J$?wX|+>`xI&_sz@B@3Tn*xUjKD{Ixg1C*3uiwC4Ay6
zeU<t-6c)`WXxZe-Wy^q7qvz{NpUUY7wJ#4-_k9rAlIuAc-@Hu~B?1b9sMT&~CQ2^@
z@-l`IA}oz~$Ysn6t7}DqPZ0G~EfA*67YZ-;0zTAiN9s97q%#pM$RvY>Kf=W!xvkv`
z@`wOL8I$WkJ3y-$IypqC(Umt-_b#?s6UEf!9XtIaA!cIMj~64?SN6`gkR(~(Q)76J
zpByE%4Ez@YSz9cXbsaQx7h_#mDW(}ckX<FfGKAeil+K{9+c@p|jNGa%lXziZG>5u+
zh6w5BN*$IT3iE`Q>279j!x36Mtb!<=D=#iCmZ#{VOG!z|t*<9|8HbX}W*E`nzI`yw
z?WxO-kUtZw=#cib_wIK^3D(9B`Bwo15yf<O)owUqw;1v4TErShk#AyOD1^zJOW$f7
z$T5l16EX9iN`r^}2x6?&<OKP!uY)(N7_<)q0m2zTqvtp3r)RejE)SxqA&Uv7HG(Yo
zceQR^DfBQIWY1sC?6{|RD_5pypn<pDh5@6+yVn1lCArbYX~J-@Wrmb^OiZ8zZO@RO
z4?y2DndfF*`+Wb<_Jmuva=Y8cYuI7RZ3RX({e`8qBhDQR(z;+aVLcmL)P!y)q!wM;
zKwpyH#@$j32WgsLqFETv)qDkcRF8WoG=G#t*F=WDJ254Ul;8eJ0bdaQz62Z%yEEfB
z=+C)N0H-K%BW^d|Ztaz^-nfSz;;5tvvXDes&O(62fTXNGbLxF@cy!hs>bq@{{w1GY
zh=@A}YzqO95{7#D|2Zr(ME&G-@9x)P=+2EcvDY!7)Z?(rNb4KsCBQ$9Nus0(mP9;Z
zv8AWOmg}4dYw<<X3|9!!b!6a!oMI#p6B}zJl8q@281G)&xtj}TQS|$`tc`?bxjbZx
zlgECg-}eVp^P!1^&hCav1{n&GIaK@tKrS4rkNESXrCqT_Uw1}YPh$45PhM!BXV4L`
zeM5UFEw%b%eO3uIfo1a!$aM9mb|GSKBW|jgfAnGknMbwIyw*HV-$Zr=c$zk8YU#8y
z4SbM&lnyf{miKFPfq1ww@HRIf*UoTCl~<Pe?Tl<-x4K8_?^5*H0}0Ff(VJAiDoJ8>
zj}c|}FxPmuwF}R&+64?zRo3S7wR&d>)!cq{q|nUX79Du#v-y)$9QXnT(3j$VY{QOz
zvr23bT0ZNY@*^NKjI|RRoD454v}wwA{krpA(R|Bxqn=q2G2(T+M)IJbk@&2%9VS?}
zjptxjyJs&zjd~l-gX~~yZA)kAMOK0r${&6SmIs9n^XH88zgXg^_*aDG!UI@JxbOLx
z^==GZF<}XQfbqk#M3#o_zH?{KvUNtU`ER#oH+dPoDRm^)=z-H%JA#qy%|Ad7toqi3
zVbwEBc4pDaHUU){gg?exKCHfV@d;la$Es=7`pUd7e$u_n*T~kN#$n2YwYyyFc^acL
zno4(IEkn(M{Z}5g%6A)4E-^CRX#E%dhEa9KxQ=i<6e-!`Ob;~3B!|JJ%oV(uvkA;d
z$GmvVO}mwO3vN_J=uI`8H6b&L)L}z*{r7}1xH~I2nnfVMo+zP;=xoO>KVXeAJp_T^
z4e#n}(<xXkPT|COT(N;wf5Svf$Gles9)Mfuzce!ut<0lRpolk;cebJDu?KDiGLyxt
zuC0p0<MTHT3D>pwg`i7AzE;d><RfdRveGYwOg;*p$9AQa%HOOR1o^NsBBYo@H|Ke%
zqDDcb9aQ`PZX)l+P9qYc(=oLo$9yWy?}lB3i7dY9mFnf!<Pme$U^Ds>?dd8i&ByPG
z>pXG0d|+0`qfDl%{Prvi4!a%?=Q%ZcSOQ{`n?OX=O^4x6Z0Gn9!?F=?f9~M@cwlih
zOa)OC&r^;6t>HHHc;v@;b;!^PS&ehPhxX|a9O}j{8ihr(pU{KWoklOVraP-hQ%GDW
zA5e|)!(3ZJnmMn2wMUq~uDgZ4c|Q%i5eh91=^F@-r?{ckh1Vex=cKhIASrvsIc?q1
zi4L8QVpAlLOgzF(xyUw=C}voV$k@>InP=T9VFsXv*Jv!O<bJ8h4<nEwX=oOCHuUu%
z>PfHpi7>jrKrNAtaQ&;uXjk>EPxf|~*T&Py!@!D585S7(|HmZ;#+K1qD)Ulp!PWEp
zquG$WYm7fI*w@AU);qBMLYtQGXr5BCGq}_|J%h!rYXQY$-^JNzMQlfN)$t=MC)YrJ
z35A`2jU`|ji`3U7Y5gT|B`bDNRcwB&7lFtEEU6$nL9>0!uV-^Sr$Kh%oW5ob&$ZUE
za+Z9xq@EUK+c8i4U?|tKL~eEeTkLGYVxel%Wx>W&&E>^@yyYgjsGy`<BrL33tI+B6
z48;`?Vv1i{_1&KPZT-lva}b>&?X_pOMZwDW8v0lOg1Dhm*xm}-EovBzhZ4GKeDv0e
zq-4aH)G?!)Hvi&x@zJyN2AALsJ)AVE1o7`r0XO|Lw0yWnoG|2W38Ni@l^Yorv3`DY
zBhwWCO_t&JpONKsIyj0yH@tkQ*CpSbi{Rd=XuxS=Ko-?gYg4@M7Unqq3*+denLXB!
zn^H~rhEc2^3(D~Qk78ud**MvD!<;O$J#+HPjTchj*Id>~iQY0OyX8?_*6uOE<hHQ3
z_jq4nINg?(yMhk)oTz3jMKA5y<;o8IkWqYXT(IekCq2fCR&doR9f$v+$PURPt^Ic!
zw}H|ljm#>hgt}Ka!?t@y1#Q<Y`{f^{v+q|&c;vRLCHNb|7a2ClppXYF{Gs%qa^__S
z#N3~s6iRd)dU`TT4r^p$X_A<$xGj%hag8w;kKqCuJSsi)qrDllCF8M2#@28b2`Ae(
zC>InVsTqQ-dqa=%dTlk;z4ePr1><cdEaMi+IzJ;61Q5kupe&^9+rBorVW9R~W<Rkl
z_afKg#Px-e3_yHpKj<s#w1DR<@!5+E`8*zF&X)CYEzgBD-yE@tA43-Ct6n%miGa2~
z7duCjT}sdK^mCc{e*h;J^<Ic6)v>TlyC?F<1Ham-VWr^~^v#DDsdKM2@8*l8WG4N-
zz~lGTUJAP+5-`<K;Rg9D^#gSc%F&<I65U%;B#hS;9YnTP+25aacz*NBB<wAao#pst
zae&D(Yb@l$!KcWJ0NVLGWoWV%gjeN><=o(v2><)?F6}~H!(y;wl1o6Z$tcxTHaP*%
zRQH`uWmRX6PfkK<e8yk>zyjBdfx(Dy_f=QC*yH%BbU8aA$26*n-7>LAsQv6WcgtPM
z5DhuR%{gx3D5^*ryCf2kSgFz|v>C^$?#qwy#jU%Zb7Q`D#&glQ^2ZvnYt!HW(Rdl3
zvVb(AMs%7`M;R?QNr>a{XZqVM)DvD67)s{&^))}dZChM-w*lfN^tZ?{UwKZyEAGv`
zJURG+hv!T<NdLd^E2<MM-SI$8qsxcXP3Q$H`C=RCRhz@m*ur+druau9kcE;=$T;(@
zp2Ne1<mSc~KxT1(_1oe1^4;<U0&LT#52k@`tF&K_P(`ya0SH9<M*W}dLEauTE|T%L
zw5<rkAa(&v!&pRSzkb=uG%`=Pm`GdZTbqATC7je1b-qRy^lBxfuOzpdU-VI(Hm~SO
zd^~6@0+K)N65p0;_;+Gb6`_>aBAkOSo`4?`CxxjR1ua2wULE<5{~g!Lruh%jbaWQo
z53JitdsNv?$879C>AhcZ%Tz6z`Qo~{e1aP=_Dj)L(VSZ6>$~+IS@R`eKUrr%r5^YB
z{l!M8+O6)O@dPSwNB}<wik|ijeLv7%bG=Hb&{MKE*@)XJ>WW1?vYmCFdB3gKwsOE-
z1>&q8C{os8!2D+MnX=HaW{{afqaq`APM7<fpvrlH=foK)us{&oTS1}EM!tt(#nbpo
zS3+v>-K;=6le6|ig)qW`V_#*n=T<f#p0`tVgD(`Xn$5Gg`85nD2Y<AB&|tT?uaxTf
zBG=Eq3|u(Ip}Fm&Xb_N59poK8;B)V&wg0(YRBf_u)?PD2R5c8h5yJ4hjX>K6kLWmi
zxqm_^w+yKCuVMh=3b2p}{+4KX;`U2mM#20=eYzQUm2{DVwDbEgVMqYhJT3lNC}>=)
z@_kOrqTlyNm^hQt<84~(;=EdYpexg32b0W~Si+fJXY1b#i}dqr<Yq7oW#j#-b$4e1
zLpjdbkZ0?HvJ*k%by`nb)gxn9dRJ7(qcgQb<?Rlf2y=f6wCK9%`Q<Z_{aR#uu1VlZ
z)7F<LdjK&f{+9*9Zr#U@708{f6CZgjq3^8HC+;ByN%b69d6GpaWN;Z8YArC&0(`Hk
zzbuz(MrVMu((hX6<l{`Mhq-}RAq0pu&u|q*Y?9>a(h9qSd0q^v8Wjww;&!i!dd54&
zt4_0oaG6rOlt*ii730snRFI%)=noxTJgD++mj6PJMLp9{XD0{=5Su;p3ja&u(||`u
zwKF!qH3-9*O-4Hn&Z6-Y4Kt8f;%tspqcrYmjWj-M91b6YFTs~=^wCG+5KO}Y&-|->
z>p7&(V`ZW;?XsO2+duO=ONzoue1pz92_zXBx9f^v>}dRbXDWb}$H5SD@?qyHtw+<(
z`plmmHSo=kmWUnWjaHJhjsqMeUC3hgMX>F%ae?O8o#l<msG{y|EG3++j^Hh;FU&^J
zGv&LpR8j_>)S~x(z^xm^e3}ua!NK?B-(K3+OY-4^NLy#t;k*{vMbp)_1(L_R1_I$`
zW&rV=?=zNqp}Xc@p2ZEdHvBjIrf{mxvUk%OVW(%u945`&*i;Lr{LI^lXynhuYHem`
z`|ZXX!<ZHo_FMdeWQ&Z~Rjf0qt)?us*)NgLC_%u12Xp8xLHl*B@G_HQ1v1K*XpqwL
zVJtS-mhwDpaSsPVRq#oVSc6gqYrk&!r~x5HjE3*LL^N|(Y{cDMhR&KFMA0e6k6p7m
zCf<&t<Skw&1KpxKOLS3S9Gd?0miU>~JN}@;PEGJH;%jTJYv~nVD>Gl?)`7Pit;b;!
zMkfG-u=q5a9&FRHUXwCSbls{qH4X;lBEmV_9oDKh@GPuSJ7RWBinpEMOBj0$B1r7+
zORNPxi}3TM=R%p2TFsGFb1sLHpW7}Bl)$cNWnC=>H+j9>cJK>g5(6!=nzoi^TSHKO
zlJAdLty0#8pe#pW$H(y+2*G5TST7EW4ZOiEBEJ^-*0EKEx@*&I*GI314#oo(wg}3~
z^J71>;S#VCamn!Fv^U;ripQb*W9-77qe>3o{Fr`&8DkNLlxcmO{DN1fqfr2-drGXk
zGiPo=dCeIgFh_@S@qZ&6HGaodo*j!xqKx&lZU?P>xdKdeMeBYCB&qxcYdkGfz%X<V
zW2#0!ZURJdsShp6K?uygm_L_-B^Kmhk4PZ|_o6DU|M%3~hg&7=<!j~8*eT;FN%?ky
z_~GqPz|$PInO;rA)$<ZlFvpo2(L4X+Y5lg+-rjZ6YfDKhJj3@`tY^D%j8LpF*&j?5
zmFd+av)~+!2gJjT=-ay|uakS96@o*p;IA&V{g3NT)fz?gEAVNwA)$U0?rPAx(jB)}
zuT#vh1g!)WOj(Upybdb5{z8^HmEmbYBt$&Jfe$cQRtGmJf5kxRWj08zrS(zZi-VJ&
zURaje%M0E|BvqJ~2Ym4M^UMn?Nj|<55A2Q;h@l%rgEAE};9+968*;V=i62IYS5oQ;
zVqIqZVN=odsX+^KMs#K(73`+};eJS^@2>pRr$Vv<RBn$zjq;;4U~X&~h}fC;<d#n^
z<LIVlcvfMva_<1)wkk6}GSF?YKwNMSa@V?;YPAoyrNs#RIcMuSeppvaRJG%9aX)LV
zad~#h@S^Xmp-<wPwmiEPa6BLmwOs^V<erOs36kK9vZRpey?`g74t-=98U3uWx6N*O
z9=GtiiX|>Lb$*xvKc_3~oOqB<TQf^E;KlsJM74$u5ZDzea=lpd{04m|A!TE*$0v;W
zSn9mu%Cem`Fs1-|J~2mj*XP&G3f1zJ&dHyVLIo4q(Uv(yvi**r-~@D8p$sBN&|I&`
z?+}3@$(nk=<JdKzJ^|4A39#s|`FO=8LO`=k(Oqkvw_^K0L2WuqCGu-yR3Z^B%s!z3
zb{A}uD^)<-zO?n%2AMaF^pc<yvH73(oN#n9D5jB>)-h<oY9dSFlN;B;+bI%WXW_<a
z<nCM=t!1eV9B^DsZvDvzh<r}(F>N4|=`hGzuI&k8%pt_A9$4Wxm(3R1UZbr!Juf=p
zDO|=%ar}S5t>G@#kr~#D`<z0NpDqF`KYWmr5wqcZoB0f1r}+FnC)TVWH4|^E8-GH0
zRwJ&&h^n`UMSKFze$MsIM@yuPwTyT!bMlj674qWEli6DVE;u>RtpA%i$Wh^wAc1RG
z*ZNUuOR0wj*lT=_w{D%}{5StGlm>&JQ81AMrkFiq{&0=fUvyt3hiyZ6aXGemaPz{W
zB9Kf3$J*&b`$k`3xpe6=(Z>7$e!Sq9i|ufAa>e=xODf+Jgxu4di~C1ib?BcjLJa#N
zX6t>ZF`8j?Mqw0*Ss~5lt{i=ynJ!e$<|O+u-Mq8_&v(QY!tJ>ACc@&`^`>YHHb}?E
z$JHhS=(`wMFqXkRHB@3FvoCq>-Vl4>+?x7rr$`v4&L_3R@SSHf>dF3>*9pG8&sh+!
zac`>iCE68Fn?z3`Qdr7jb+vo{l%+(q%kV3L@+cJt-?{$)_altBH0=3xu(n)|j}c#O
zCkklqv>I4;+`Vr(_%RwMy-b5nW6EG|;*m4;o9->=&8Z9U@?2@Ty;QMiv>Q}|4Y-i?
zRSklJzA!8EF|d94%KSYs>cuKZ@3~ZQp%_kQ7({Ns9@<bY^6(!xJt)`mV5$zsE+MCP
zeOAq>UFj>|a&PVe%`1lN-f5bNTN#aTuciXT)}?P-Q<Jqj`ZA%J^=4+ng-Bh6v|G^(
z(%#l(+01efSrpqqeG=u9yYeR=mNNoX^M|vdcH=7M;<>;aTN85(*cz=KJ~8w%T)@zE
zj^nohF*Q^hrQFw}?Db*{JEI7d&I3N~jXhHYwo6CUGDC`cbwlJguY{R0yIPmfJv)@o
zfw(yrZFXCH{irm|wCsnfm?qa{#1h0>m?xx>eH@XAPN@uf?xLUOAY2BiT~ZCo+kZYX
zK5T)WpGWa$VClVwlvP_jy}SM6>vj~>UOPkL<v8UMCfMRQ0(3YbWWH=BhyunQ-(vSa
z3BXP8qWjd_`S#7Yu;7^~S<xmQdzrpmPwGGzt#;dzDR{<gUiQGpw`fkjq%7=ed^vm>
z;~0gaGo+S87uxuC4b0eC5Y1Nmmq=K>KBg%*a7C+R^Ah(S$v#apBa2ipG`C7M{4fY!
z;sRu-a6EauzcGMK+({L{4!W|_r}7(3zRz<%(CKd<qe4~@xM$tF;Q)CPP+x9PeA$VR
zQV}id^!r20qF@q^R~R*h#}e^+#lzy_i)QVfi^>cy*Eac9F(0mfx^tW4fVA^Kv*xPq
z`*OuznG`5SvugX*9txRg3nrg3`!cCO)pQlVR17?sE;QtIGlSl?$bN(mNVb=GaKo3e
z2s#mEKUXIQ1&{#)iyNjWGJUsURx_uYXJ|h7T<y?rK5l#JV>Q%f2I6XXo-^mg%}#W9
znRbRxLn4#p_ZA14_%gA^9k9MqTwbVgbkrMUGiiynV9O8kr$iuE5Q!FY9^a)<>dw0<
ztx7jg+#h1EpQP#U2OjN2H!DuSMwYD(X$zn^%C=G<p6+q$b`HJ#DdD<f;=zM^p3-9=
znStram~nE<`%QabywvelE<}|@f4WcqaeOJVVLEbl(uK0tW6(G!m`ZYtm14m6smegq
zVf>IP$!G9od}C`4PC3W<N3DMQSSgtcukE?nbJ8=X1l5N2&CKS`6tz#0e@4*NpAm$E
z;%v&%5T>uh>-BJNyu2!<O-U4HZL;c{h-mkz<Oo;g%jZ^KmI+~QxcHRP$z>1~RFKyA
zS;gut3N<???)(Ol(PDSN0j~qHOS=}RL2Q|AqS2JL+U^7PIvK-PEb@pTig~mbImPyJ
z)+QKd^vqk|pp@B3EBAw&zYacx`4k(HC>(Qen2?fxu{15HJ)4^nx*3k!^*NU5!V3J-
zkao<EIT`l)d;W&)Igv4|%))?_rKh|%rM9sBE27isubAH;FZzrAMfKap{-XLEi!8(_
zV|PgSfSBQ0`w~mI>l-g(z@%XGfo78S&44oIPcKwc6r91IQ~pjQTmGgUFDJBix8H>x
zjpoqS9~qAYNc&<JK?PB*D)c)zFCN(fL%J1paj92EasC7QYlPYO^5Kk#f6cmrP#f<Z
zU>?=e{yx+hC-y|#uTwl`pf&kM_j%6sSw0y>Bq^7KDR7E#-)cl$s~{r2zuMBT=3YJU
zw(C{=X%o!YuqdEMoTBmf>^3^xIaz6q!ST!WmEVF{?z3B<Gj?5~TG`Are74x=ruVgQ
ztS42hG23AG*tm>j+u--g1wAagjthqZ2A8kG`Rxna^ZFs4VfO}FL;ZkC1JFfDu2V%3
zdF4aKwmxn?^eql$q$jrR)Vwl&@Pl7JH#h*=9Ep4y1bH-1D_+pc4|NUvsT8ji*g-{T
zb03?*$87)eCL*?<>v{g`=Y=On6_#00a}*H#zct;&ZRZ&03C{g1*+23ty7Rg42GMxo
zc&lf@#7-GsLYu8k6CsbRNY~bhJa|p=S2<3Yy5=!Tn!e0GR$-tu{Xe=h8`(+%YRi3_
z^4mEw#7r_{G`8vQn|4el`Jheq-V~U=?dkJBy0dglENp*1Ow;wro&x$Ot5|;WB7EFU
zVSK|+k<|S$+@w(lRZ$u<uxK7lg@2t9{|ejw-y~jJ+DreRy$QsB6l$>RAC;UP`n*4G
zuwy)No_3bGjdl2S#XG*h*Uy9PimYk3Bk#Oc9V;hjT800oV9pD1WUY&!XY@-Vj@qKo
zDPq^O!@>FzdEsj?y0~+|>(6+mfCqPUmly?O?l*9`YWhYcP1s2sYp2vtyYa8Qu-qa(
zx;AG1X{M1^7zN8?+;PB=HgXe~8pfL0$c*UBnnl)VNsj7=`TpErQGGq7K@02wG_aDv
z6gsC$3pA%cU+fq#p}-XFGoy(33=iEUh5PiSn;xha!ZN>%HXx4Gr*6L4f0`Nti*sQt
zbP%Zi%`MvpNn>8Nu^dr&>;{Ed`tT1?*piFa;xO!;&Mhsa_4`Yh1u4A)H!9r2$#))u
zRTDH^xkt-LgVLsy-bcqr9?+q^mSuAW`;85kC&~K~(W&R8t~&L9)Vi+%Mk^fYmw?Z&
z0C$DhX}6`hxU3;pX&fZ^m?|c+2~qr+ei9r#og|0TfDb}t6IIgpg+D$BSt8~%5gRX8
zcRGPjkcx(c*;(G{9W(ra&=U>vD&^f!4{ZE-1>Bo49-be#Tqd$*K3bQw{%xjJY;6?f
zCCvPfv^^~A%%zWhS@ad{^fDsloCc|ACw3>zMK@4cQprWx@}+8Qf%$gzZ{0ruJE8dk
zS`2}csz{Cq?fa7<x-$0eccjyY=V<F_T?SxT@gH}hlE(5@v_g9keW#mv<$|B%dQH^Q
zZq9qlwfy_93TRLJ-uQXHBO9o&o4Q$oyTz(&Ge|LL6EzY~!$B8~d+(QioUGXR<)-hL
z)S;T+6*q}WYODJ~{Cfprb{Yp6e&)!{OG7Ug-g*C*F^J&Cy_?qV{hR=wtfgV9cSN+)
z!f2zuujx*GIgXwJ(&o8KqFqzt-2-}nk;LL!3PFe~n_7n+sX$n(vx_fX&p6)1=*g4+
zq+_Z5NiF<e%@GT`v=?!tz>)Vq*!8ko*BJrTrjp&2uise&QUJS^uBJUL_z)<oo`|g6
zu|S1|dk4QZPz@5&^3OjAnOvLYHK(EqO`J*lu!-IRcLHf?h@7p%|B<&gOjfNKzLwW*
zh}=eV6uJm<=<rL!xeS`gB_K$+4%QtK|HmP*ckbg$_<E5!1F>t!^wjpqX?YL}U#7Ji
zy{P74qvpd%Q|QJ+E;xb22G?;Ht!seh{Gi;9(TpxYlX`wPrkjEup-F(aYsh%8T$@0I
z5#jz89I*O*5DzBxf?erMnhI4~-}q>b(gpeZYE;n{TztIgcq-2&EDlH{q2DK9q}zWZ
zvXC=+X13-OLPJL%x5)6K`t!g2>?mbzmpaM8?8ze*o@(+m^peNvar2fbp~6eLUo?4}
zzJpS<?r{Dh8>^~u;KLH5Z{&g&<~}hoja9ir2ensBwO+F&FT4MrFaa&y@F#bj&p8Q>
z*z3gwIp02q^~U>iKq1!WUV54huwK^-*%2Df0-zhqGTV|<_uv@_F6&aacMz5n<BO86
zvbwO<S+kv9atBt-Eg1C&BW->Ea65HNKi{H^oi&u^4Y@LF5C2+Kq(n<G_LIm%i;U1&
zV@5P(o$Vbi{RZRw7Mb3rujBA6q2e}Jv0BKJhdKwtCYergrJL*9mBTJ{SH9oLLuT0f
z2DAs@H0@qll&vpC&YuML1x6cvRx%cQR2aiO!|L+a{p6SdSyFbBW1W;cC)t<-mq57R
zb8MTA=Ggq-bhlpsR8}D`_<*jS)KbLDu1PNJ6bHx1y!s`Ae~4Ycw0$r2I-H$=rlFMH
z^zaL)q+10Oa3Ly+{}@F<!fMv1JONwgz|E1*<})tgJHnn7@Gj}^#qN=*QD#jlTQhou
zR)Nwgq^aWk+MC-k;Uw*Yvl!>r_>;$c>O8j-H<#cHH0aXDlI1I3Mv7rl&^OfPH@SaT
zH{gK>jImV$FO3PQjh@>sNcI0>N}d&1`f6(9fEXzzf)<_FQ8>E3kneQ<x%E6=Id(Sf
z{g}QtTzCSKkuxlcX%}}7QP4duU8ncXS%#X9^bpiQT=bewO42DoB=+CW@$fIz#zok3
zmU^5n*BS^Mf+s6t3iZ0KsvWPVC@*b;nTEt9k;b}L6~2Kng<oqr9Ws}FE&}yCF6->=
zMKF)?vN9yrvrrxEMp7FxEY@>JwvaD>lgM2DRObeHZwTnZDcIST{DkLJMV9%k@9zLF
z2EbB1|N0+5CenDSqc?hm^uBZp8Vw>KliO>HXuDT|40(RKGHz@O03GD79<;anw5|^h
z1)4V420)kzJ+7XjwfKR(=`^oie{yX8`P=&@13J3CCIP11ryDrT$5rJ9%?YPN$LQJL
zUIpY#5#w-{nsR@4**s}jVy?lr-j=Qmw8Rpi0{c{6ODYwQ!)h^wL9y4u8Ci6KU)16T
zd`6N3uS1u4B38QrB(!fp@@=LFancGl?XDHNYNtYfYjulutqfur;F8A}48CdUIWnMz
z&inwkwgH&ub?*$_z4ND#*Rte;9rJM;jhpU2Sx}$X2@E)iwG$__!mVrr0*B$1nmHr>
zNhIIyrZ$+A_x0`>qWt+RfO268V>?vRqSFpS-|(DZ<n%P4C1eTXz9fwg0(|AW7?2HP
zf!S_qNywYuh~KX|z0KM2%9m}5!fM~Z3~uW4CI`m9U-UgnoE;>CAyr!Xxg6D!8$oyH
zr!O*ZIq{6C25DPh0Vi5WJqn$Pb$7+~hUbMH54kzyHD{;{yX)J3XQ^b9;s4IPJP-c7
zav)C|Vkt$_p>H^bjujMBDcSlOwmlmDHy167k6}@%`4Anu6UDNUVR)jLP-=s_L5Gyy
z!}Upu%%94Hw$$*C9~bt=-Yp=0-N!bu-qb4>{1js_J4q>N7Q_>_BFs+(kGa+K*58u6
zb_>|4D3tp5*=D_}VX_ZG%7lU5zyX>oCL^0|n7#SGtga)Vf!h4~SLDDLna626f<A)y
zZ3&K}k8Lg#DJp9_IkIA4oHoFfH6Ht0BxXNI>AqT=HtpXY!+(Mj$(DcEk;le#kFb2B
zwbB|O*b~lv-z|r~__c+yAV3HkqY05GK}lvRwzHa3g;=@be|k8bO&KR-EPOas<IQM!
zc_~WnzB;*8hiEhfI61oqDKNVW=dHxT)bxKU7Gqd1Vio@XxHYpH{;n|p-`W-XD!{T1
z9XzWJTlfE|9ZO-QTQT{+{#aP{w9UU>$Rxmj?|gjwT<6IK$Db8Q-rbcf`x$qbJ{7Rn
zsMojm>$L@EQ_bT4An&W9;_A9Bg9LYX4Q|2Rf<th33j}x9;0_@v+#Q0uy9IZ52v%6(
z4qfE`{(Eot7<cqbKlJE6uXRq<slC=-d#<?*uJ>lbY!Q#W*4)libv|5z8D5OqX(d(n
z&kt&eVXDSy`I3B<^;hR{?`<aRTtG+G^Tey*zGtTM`eny@<L%S<8Slj30czm3L7$NR
zb(DpNN1{Rf(>*<xqEF;s^&jEIaIF7skpBPIt!mUvO!5VzNz(KQ2&kk&;$B{SDk>^Q
zR#v}e(Ue8|1_wvt5*RcSNex5#YG1c5j?WzSgGot*JhB1Pm$bSSM890B`Up{kd#dJi
zWe<NwUV1;%8}`-;y^X|H7X1kBrfFl~O2+(Ny~yiE#P<23Sf@jvZWy3!b>R6WvNOi<
z_H4&|*tc%QoovCVrm1tG@`6)f#QDt%VPmxxVk%ctm~j<k|7z(F3IgGhwYyJy)3zr~
zd#9#tzZi55FMt>)-l|yCl)%z0){1Yzl}nNL;?2KR@lteFRarmMyeGKuwtIv9fhOR}
z?fyuY7<<Rj@kF(0@011C8LHiOzl2Ki&V6vb3I#na%=W)*jkrX3z(3==mC@gx6R#|&
zp~USqveNW4uwK+tyW)e-t0gZc`cPQ=)%edY%(JoluGibI<i07CfwxzwV&tYZ_f^7f
zdZtEJV|=lP0JLPO6#a-=`a#>w`(4Qwq~0+pWyb+Biv@0Re5L{wIb1WK?kD#FYPAPj
za8DTUp;Rn=z8DlPTR6epO-l<MDm<@&AlqX6anT(|GA(MVyrxB7s^52727xNd=ZO|+
zAC<vam?(78$~>A)y0@2I*U{g{zmMBKu<u+h-i;cE{vg+;k;NF4$LIWJT-31ez?(io
z?WMg@+~K`N{XNDqFc(7s)(pv2S<+Bb+iy-ts$YUKC&BX*aobIU*zkB7tu;F`j5A{b
z7gYpY-vo&Wup9A0Wv?4s8vljs)N>6>u4NJ^2){e_hyHCLJ`I8wLYAcVOsPVdfK32(
zD<_5O$_8-3xYiN0g-#XaePMqvt2A6IP%f0VJekq?cwXQCLSxKBd3#*4+W*`{cm>#{
zX{ks<7JEN2h;N?fYe%~w|0@}>H~Vn1g4X;96~i^Ibtowf7%EFcdS)%LoBH*d@2hc>
z*H5pF1_jMw!yj9Tlhcagdxh}8&FdD@={^7GIyCe;H=@GyCkur3U4LOedyX4L`?V#a
zvk)qz8=1%ZjE69qeebPiycQN-Gz~L)b~Z5!)U2DnU077}vaTR4*1SjR;TV;qH~Waq
z1y&ZdZHu@wgmS<;fjc{7whBVA(p$0uNj5kb=TK2ZKWSoNl%ZHZ5i_v@g^Rj}<6#Bo
zPo8O39!-`w7~U&eFH_<G;v05)6iosF*!Jd`Y}F&`>5ikg);tX|)eQ`<8JwtW4V%{c
zPhUPnNNP4xblk&A$7|r7h9XchWj{zgua<~XBoo~e)4KrZsVHdebkOljHOX9W$qSCD
z;D5H*5`HRO+23yWI)2Hf4nO0Cz@P4^%TjtF7W+~`a@+S(@Iw4fy+3r)&*eux>hL@y
z+w;O^4&3lHm>e_qAt}jQ7u$Ak1x=Qr4o^JmSw1@D$5^tsi}`cZtE?~BG*Ilw<_c)e
zM=v~px2p1#5qOVKMZ+2B&JFawx{<3x&)mkPOD%B)k$g1>yAEh@Bw*u;%D&$u7jVc5
zxk3br$lcUE*jS03WpFF+&BN!9ZbXhAQFvt4K^}fJ&GveUUsGzRVMg<QUy_Ln0{u~0
zH9U#E+Ub@(`FZuI8@VqO4ZsKPcL-Yvv-JyEUaJwnp`yytVD>f4-_cRA44qeg-}WQ1
zxTHdw$P}ki<W@`Opw(51m4Fiewn8Ek8#eps4r(K6OS1g{oR6saV2b{mVQq1cTYPEs
z^0z}L-3zi%SgZx*D`9$D#OI01KJV0G5&`+Ow}H0@^#gBeo%lkTqt1YnGN_tkN*M^O
zV)#948|kDyG)q0F1jFjlq^doBXnDPK^(~P#yTw&910vB@Vz>(Rd)p^%AC!`nXHb;}
zsoD=1LEUz|U5tk9^*C#ehH{xUg_04Ua+2a=myHUdeCQs6sVaK0mVssmfn?!#@M42o
z9Yt_e*98?T&c+@-{8ziDSAo;*Ps$5dmWdaxk5<;-Ukn+CQ*Yz_Ev|I9MN+EX<s0B4
zUUUX9*9lK}>66oLtfQ&;-wqz(CWubjWuKy-gB}R(<KGZK75MDS#gM4%EV%16i*}m9
z$?Nf99Xo_#VnUy7FY=6WAy8B4N-8Rv!;KBIK6Y3bK)H2?5m7Hkyz;veFHpbV;FEK8
zlejsZGm&K+4wwr<C3lS_#_(>y6}?yavMGO{kil@~e4kY$S|m!rhEpO%5oh=ZHo=QY
zd-=skfCKYnN(#koZf&}DwJ(Tu$xzcria(#eo<(&je#8ir@enMcu)F6U>oj0OeIh_o
zrtx3)UtoB4Ndz<7+QtY;SNR?4FG=ET7UUaODoN;|x$J}m&Q-<IiM@AhrU7jAI@Zyn
zcJS?IIC(z)X6Gr_qW7fzUc~kkt}oCZmg=O2<Fx=M#ZzEw#QD38Nzi>pb6Q__zcH`a
z#|d+oVWegg+k?z?hr0rqI6Al?g;|Xp_WN-KNrZrRr)#%roBz3e?rNN1j=DB<%Vol`
zZ}^qH5}{cuc6a&ydIn<9xuBBWZ^hy!JLanv&V;A}>y7ILVu|;JmZa0)4^P4$D>wfj
zfo)2H!rLj<N6X(!u>6oTw}5Z`Fov39NwG<2Pq(dkxcmidj&!L5D=KKYZJ1qBt&s-a
zc9Hm`&mRG`dhzeVVn7_`!0r1DQG03JYq)*he3*^kU)yPu=(hDMsDUrQsmW=JqPY^(
ztAPoL`x}ZoYwG1$P8bvWPl8Ri5CYM-s1I}5E2_f}Q76xrq-p~cAa_9E`tl^n6&2Ea
z#B<^bZpBqOY<$Tzj*B2JL$|dmfUp*+r5q^-;Zlr#o@@t42|NgK*Y-i=>!=A{?A&oG
z^AbJetrozy5^6X&qszAd>lEl(@5K`8=U;tduAjsG6_Xxkz;^0IJK$<(__;?H_0#=^
zz-|R7w9^-UZGQN<AC+0+!kxvlivR6I@p%Ii7`a?meC_ah>J{wc=5%c0+DjU}N(#{7
zw~42p3jI*gU7EX$QaRBI@tcIpg<8X+sJA;bk%s*k*O*5!AI^Xj&SMMGIz=3lb$#fT
zU#D}Z0v{twCB7Mw<E5xLFs+1qIrIVnxN{mG?A6u<aChl5rIvF^C1`1zBOd)9wsIUb
z4_Gj|J7_)cRy3uTd>{^Zh_TyWCHv1LKgn?;|H^IJxbD`b;K6<@U3>2ps{#VT2Tkyc
zxT>`I_s-$|-gq4&PQkYqRcsMAil$P7McpDtQZ5Y8W0Jc>3H_BzPLiot43I`4_Z(Ic
zkhXZYM%v%N-J1Q>ik%}%AHDi2pC9W_D0A6Jv17Hpqnm0Ld3*lrA<@Qt(Aq8K4Rh1`
zm`zVlbFdsTII7WOlL+~QUtPK6=jV6L&Z?D==LUwyX-dWZg>q5L0ly?ItW(co+7Ed%
zV%uU!9N~wjO{ot5f}zTPz>xX>9xvyb6xcL7{Z>mj$YF@YK5Pf9&#VFfw-4$ap--7_
z(<zIy-(W_yN<k3S_Nj2o6m~n=e}C6SXm4CoiWBJ+9UT>guCAWUS;iHK>{-E8=3e3d
zf(-2qCB>K8fGbHd7!xmHBek8q8JK<^cbfq!vJ+xJ1tv460<!7<9+5_hy<s|W@hd*l
z1#n^U+x7o~EAS?NwPBx#e3cRqXiF@=sECq^>V08hp)4{wwz!W^+mlwali7WBv6-Oo
z8-|%WiUh-4v&&!q@`rL4h{1^a?E^5pA^-2O@Y~9%1%n0qfA_8QKNAxRRrcgQB*=z@
zp^{Pv3KG-N(S79P1m|lMlWp(p><$hmFpR`iezLNn$H2gFczyA`=l@49jKK;5RxlAg
zgr2;-Jmq>TbpV~JBv>!OmzQsqH6J=Q;dBHi9p*UMUkB3Q>znCb{0`|@!p|4<cOryW
z#@nfCB8JfCDgW1*Xg~cwv1ZDaXtjhhEiR%-yWO%7Zz5=?e{*fI+#$Jojfaa4nBJF}
z0H24F96&rl{3)_ff3-X58(7i_J}E;@dkldG=g*)_BJs{Jza3)G;D)iZUm+PEu`yvs
z*QU%kNDTG!y%}BNOAXvj21MtsVdTXaoSbHABF50dbJcwj%VcHILw=t(ROh##qYXGY
z4T8VDW&+;+?2i*aym@(GnBrHSowF`}iu}a<9fi#et^JP#&(!5&i@wbt?TynmtJv@g
z8%C;klD0f5ZLRC<tRql6v?7UP#gsTF@jE@F=wl4shh-v2QA7vLxf(o32ex{{&Tm7P
z@U-Sg5p6oVTE45eX#~E(F)GtH;Hhc;!HwT1Kp3*1O9?h{!)9R1cZ9X?vp$my>JyY9
z&#~l}%#?GN^mBtS>$`V1Zex8KH8a&!1F)RmHoHE4k%Re4O*#6g>0|tJF2)xp{k5Yo
z9qv1HulDdbi*OJ>#L>s2pX+gKBb_f9a!Of5p=+H!IWAFWF^XS#cg{))Andbi+d7;@
zK0N7VY%vyg_5%1sPc>ePrM}g@@TE8BsR@NipJ{73%-~co6y<eVe&F&ICdeC%9Z1@C
zJ!s=5t|+xZzrSQW?kF$&JEv?S=1o#sL$7?MIrbV=%TQ1)X^^{V-G1t^Myu}R>91p@
zimo3j-bdMLafNw$7SN_i;&tqPM432+genk6SP#<l3z&)(E#~%$Zy{_oBN8B%G@b7t
znyNgjynASLsH;kgD!44z<oaCqh%tUK1l8}0Ugs}F&<V59n|v=P!SAg_?a{M!seQ*U
zX_>d!PKoDwa%?QTOmfDll2nlOf}Qo*NisQ}sUyB+ZsG+`ec~m6h2{m~ut2c3)xUSC
zocWl>HJ~Ip>v|~ZZl?_8bLkP8&&H?d7a!uDU#{JM&RwhR?l+bcKffu>6#HNC%@W@|
zJ8PvR9S_v8j>9Fk<m_WalfDcpmc^%i5f7%VouL1MNhClq^%}ODygs3_7USDSxp$)w
zmUYBz@!^*Qzy|(t!`$?@cllJyC2ZB}A)yO;_v>D-s4NtjO%xQ@;ML&jhIsb6w&m@m
zo{z%@YUz#H3PGy71+prWnP_4(yPqhyja}T5Fp{!+slPoBj_(R-QwPf9gn;)4zd?Ip
zim)qD>%(C2gj*wFZ!Wz?#=+I*k@H%8{pxnI;7D%e!CY{s!d28x$S<pZspP-ni*Gt8
z9W`(6z&HGZ9sOv4W!*;9-Eq{{keX-Wg*Kipl=oRwn#%%u6{q``IZmsA;Oxw3ohDmI
z$(4to3NI^Kv>0K3CTD5*)n=J(1RW1PjC@}y64KZX^d}rBtyuSvBNnkqX-biWDNXm9
zxVFn<&!gWAeim!e(5nbD<7YYO6~?y{dFTBX){~aAFnj9xsVyHHLB0KHPV9dT*c;9~
zsQ-C`VaJWrrssQdn`gOfM@7~V1d38gyhoZJd%!SM@D$*+8Om(;qh{TLOHd$6Hwe=v
zSb1=YgyIsmE)=>%8Ta7Xl-uQV@g(TfF7aV?N8vZkS>WU#sK(OOFh{+4TZ>|&x8on)
zEOH<tVTsXPtjZ@rP)82eS#Ci;?jAC&vb1!vxtb6285(NVhJs}J&#RzHW0DIZyJcz&
zxymzUpEI#W9BrnrPB7{(I{D17lMB5DszM1;5rt3thbS+ZoOp)Js6mHHQpcc+&_<52
zv&vav{u!7=9!xiRZ54J4qnItqj&6XWZr*Sn*%QGhD8{L#hMK=q`6fc%#H6HdbZ>`i
zW^e(46G7INGK(zm6^+@&bAc`H+dP(Q+TlBhL3AHYIt(OWqonmcmwCDJgftyMU9^2b
zrSZ%fZi4l@PJmiz5y^*Bt2@<}?0xb*o}0(QUY9GI&u43J>h$<sBB}IyM6b_>zqE9E
zT`rl^r<&GuFXa7+gty|PV@OU9AzG`3W(gASv&hN7lF1eLi*ZM?TLlZ<#}Rl?jUGGW
zn_;&2GKzXT9(PzLYG*``)_aw4!>WhpP~9^R<})ePnK`Vk1GGj^TT#AFkc?!u>iSTg
zKW%f~o%E2$_$CStHXtE!K%KcjKR(c1PUK8`S2ebVYTvJ&p4=EeJ*;pdy>1J}DF1mA
z*MaBT&;O7c!6F?sh@H@p6}8}d!a)*E%H$0MC1x4lAEP=&b7YE9S_00QrI%Teg9iDG
zE;ok1S$4E;3G5&~5o8BRjsXP`1i*dxdZ!wg1BpAklB{1f8lP7%I~5LN$Qgn-PpMuQ
zT_(NCC_|Iq?`S@<R4aI%cDg@BvOFRW`K8sas<BErvy<8ELh^V&(mhV0{%qUHY@&In
z{dgjX8$0G^e6<(?LW&L+O**^~>~95t2W7;D;0X>u6&g+?oR$AQzTfx0Iq3}-F&|eQ
zVdp0r2fdj}X}gP1e%1M)_I_<O3EOet7`}elTuQk~k76RkG5#DR?eSEGl)P7k@U_ls
zGcAFClP@<#(x^1Z+%&28I*mknrCDTc-jTmTojBl7jAKQR(lYoKyCQjA5P2nG|HbTx
zQw^VwxIv}y{>dU@q0@u;fjcsIKhW~Rudg1X(rgZ^4rgxRw6A1Eke!(0;R|nr&j$9P
z0Sex#uHfB@cNUM4t{$P!PBl?)8rFxtH!z>e_}_uKX4FA(@d;`nAp^fMz2=%HEdU!k
zyCZmb=xlMf3&USL(2<P(mxTYnjP+JZsjK5iU^ApBQKJ{|xkEcVJd~D^2`DKkQ4te=
zk4{R;BCJuNr|rim3ogxSayv<_c|)4wS6*)n>=IU(NSkO7j4b@yUz|D$)1UckZU3L3
z^Alv`<W%V{seZ^7^z??I#%E^60svUasj2jw;%aE;=jRuuyTb{)gSmi|lnCpEik!a-
zp}ZCE$4)^>8K0a?`x=X;JfzlOqm4&EFg*C};o(8n@;o6m6~Xa|>2)pM&$(cujX|>l
z6d&iqXch-H|FncK+iC>Tow&Fx2Eh>#5k8yD#J@3>hdvO)%5EGM7t?astu|@F?o102
z5E71D>gecPQ2vXc_}7zfje~bbl9Gh?r@I|e*vtR~6!ez)rlN<pVh_(P;AH)Ej~(QK
z!n=5{P3a|X3JS100YHXwq(_E?g#0s_Qc-c5XTuZZnEOMD4+S7S7^vcIUs4nm5)z`P
zu_`VlHL~Pz(?w1J|B8Wzu3SP%TnC=LIs)6L@(zr*i@yv%yvS~t9r82zHbybPY>DSb
z_#=Z)8?|ACo`~W<4Dd9wJ-xjNyc85};II_YB4>syl)t}!oz<M|csBY}X+nHFc^t*l
zW;ay92Y2wo{`4okcvzQR;q|^@T&}as1E8T}8~(={fo&qaP(CZ+)6-Mta=9L`-^9`~
zWuJ4iUl47g#SA>ZDEMEhc#zQS``F;4TUjWdw#P4Ho8Q2~$OmUI&dyrrPU8anP`EC}
z>U_?1C&h3b5-iGtZuvK`aP{kbiwVd`6PhkcydVNFlvroIa60wDRV;~mj7>&kB`^|{
z5|`t~9B$2tGvY?nTWxBxT#aPP^mEgk8|%`S0+7g(1psgN;hr~}7r^5oe-f-<+CVO#
zN~Z$N8+B1)kSoE8RgVpCXe05fMmmHJt|AE56m)P6H+GI6Uy%82=A-iKi4Yc}r5mc|
zEsycVficA<NwjeujcLbMwA0-iJr*$h8c@`O%rd#pp&r)lkHjXGpKgCCkf}YED9F+e
zP4Q1P8E-JF@@60&OsDR$0}vl~;C4dnxoI`nqI`Z$k1+c>lfY@f4CWK0F~OcDJ*O45
zclVA`O^sS~0Tl(4e2V55YO3x23y~{E`c1f-2Vq{L*`RNjaFeCkjDnp9f|sn;`%sHM
z`=Ly_4P9f#Wc0`GU4L9ffBSp<f|TwIJ6JjQQ1Uqg>sKYe`zaaNzst<OPoiMQA|&o%
zb!Z?Stq<Jf8bG<c4bJ-|Bafb4*$2JC%ad%5(z*nNep<zPi|NF2p#HTlC7y=f;0>>o
zC@;7XjW)J!-uV`gaSw@tl3B6lr(J{2mW_Y}{(WY|-zB__t%~kJi|x>+yVi*1tZwI`
z;vd3#tpkW#B3#P`8N0is3)(-hqr6xb_Ux_EG~zaTTU9LGMQ`p(Th5Tc3)_VNhavJ~
z{0Sq@m_J$lnEr+^wz9S%+9gcmps1}O*A?$yVbT?5&oJ`ow^uZg0OF$wz>Iv$5EhAK
zFETqF79=CJ)v||E){zWX>b3hP*QtJeppV6Cd!2+d-7|M2h>!n_pqNqC$xS~FvFl-p
zlJ)cOHfk19k)Y<TZ|pidV$l96rgM%%J!!PIH_84PCgi+nVtPDjXngGgrO5r}&=HT_
zpwQruiT9-=AdB6re>|O+(h!kZe)>rc-3iT(7&fPfqp(uJ87Y+$R*uWVc<9Mfi+7En
zsgrcSB7$!w6K{438CL91y#f7y$Xu{))c8_D!sdxxjqW!cKCC|pV&$^xmLmu+);2B3
z>ygH2z%X?I$!lxlATzo1gzTo0BqDO<tI0MT_lH~Ez~@(+{?UO!))~*v#UCGtMN4Un
zfFX8T4f{hU>NbKHBa-!^<0whX<wbcHOP$IUFY=ojA1D1`=}EE>gK*DMWE5wn5ui+b
z9VkS#6Ib~lD?J$MDvB882wcB`r)LoHW?&zzi+wjl8+JYWwx=VjoLGvB{td-q?K5$(
zu!&s|{!F=&p|+1(<0hxQ(0H<9NbMj_)9aUz1LWFGuM3D@ox-@f=GR!ZqpZuGh<hD|
z`UX3qRSbnp9pNa>t5yBI_?Tm{MlS*^)l?y82GDb)G0jn-dA!13uMlM1d*e@o!3+bX
z;&hwZP40FNORl`C^8xJ$%vjk}b&E@`3na|wA}ZcewuF9(B<!C2%BIQPNN2;21lzRC
zRsdWyZ94NM)o^=S6OpnL^Wude5Tb}(8AP-XuQ>Me>MKh2N|R|*1*1?WT=f`}$x3Qg
z#ODv8dSbaXfUG{b0FYoJX9yXe6ieGzr!$xyim2@k3KC-z(#o$H<ekpQqAW+AlySz<
zckbahuO#(}kvru{f2M)!4v+bq8hjZ_vQB@4eApT0Zg>PIkSt@QeNsr1Ct;?G78@|?
z1$mr#Tn8^)eWX5P74gKzm!i&UOqEbU)bj^)KuW%wzAkF@9Mul$mg@tAJH0*$ItBuF
zH0${Tqri4#Al(Y#gNzif(NC0fm(OW=^0ldPC`g=wZKolyY2Kf_W2s2INWL^)QK%x8
zw+{3kf%4F>8=r1+oU+`No--vg^+t$@TQo%u{BwZEo>pk%&}t%;jcFruiu7SlyihLf
zK#TQ4-d{&_BNL1yQ{3MM#!`EwZEE#043c>+8MIR9<`kvBP3-wj;)CAX00EefcF<9%
zusu%$@X%)c1lKko%Z<Ha!%CcJ07b6m*30vw#Z>P5P!wYNHz{h+erhlV();82e}uBr
zHP6;1_rF5~perKu0qOES+MY5tWimgU(nmO^ex-^ExD=vrU^IUOurM6}f!Qk9W~_1}
ziCUa_ycbWk&uY{j?vk)Rt16dp6gaO>1d$)j4ognOyBxzF66@bPu|h{Ftg7Q6s6yW?
z!WupC?IwhT^_-gt*<%wQ99ofA8lDAO@C*~W$rTF6TMR|Kc|2@<2Rv<rC>1=#U&?Fc
zI=4fp3>SZ|jTWpdN4sj$JE>!H79h#qNAd<9I*3a{jE_a}#40gXtJ}+^+W7IId=WaC
zbtagOC6-K~no)9`<er?r?G4g>&#NyF9vu>eO#ix@*CENljwQ5SZ!|X#cMiW4X32hL
z6$FHkLL-ZfDX{(>r&0GQOj6b?#|kBFB6a8Yc6@T1gW%2mMz8`VB1dIB<<84*`Bfx{
zTyDig*vOwW-kRq_STz-tEJ-VeWVyLC`j~#_+Y)*$E;4Uc-Z8^t&MRiipB!t+X?RY@
zX!SRI^2Y&~Js8;(3v{n;(wHN4^+dx=NVC4{qaAQp2vfU_{WJbc6AZM6R?4=U0Wt}l
zXwU!(uKn@74ZOy+7G$L=&~P%jJa+N81v5<;XP0Xn^poF*igLCezf{;DT>Uwt`Zc}f
z<X)lzv;?FW_V00ed{){BXU1Px)vvN^L0sK!0Dk^zw}R}7zLMsN9U(U#c@pWF<SYVo
zv^lAkc)tBCy?TUnt6x1l{qs{gb??^^BpbXL;);!@5ZZf{Tr{H0MN&e^tI(D3*Y5qm
zI#imfqcr?Q&RqV6O4sfi^0n#_MS8`4*XoYJ^r6LzV%v3sYs-iiWQM$xgTjL{=$$E8
zH&aki*he3~?*p4zLm<P`h|0aw{^W6z1B+Thq1<WzL%Sy1^%%vIhU_!IPgxj2B)bNF
z(9(hFBES`1^~#cGktA?uNbAX{6*<Uu?nrJ08S^Oa)sDNP<~mTf%F7>_ut9vEE3{Z`
z1}}r3Q`A*o8P5|5F^M$fXS>#hIbr%Zt&|M6f@zK#b;ILf;5*(Ok~$qmf6}O#5tO0b
zAkEUCKjK;KCWdpigQLhwogHlr43IWi&6&WH<&!aY|EYCoQa#iV`-Y<d=3X(IOV#^p
z^Lpga8NiyBMeBQ%Qq(QoG$t(%75WL&R&}89^Z|F@;}o>q!>n4;?p`P%SNMe7JYBXN
z!^6%um-59m-$l;C&b!s{iP&c(Xz`LG=iN>;5`F;8u)rG*@%^=8v`R==O|6^6Ony(6
z7@ZOqULjX?6#me_#Slj4`ko6-F5m8Sn#rlAkcx{Ucp{E@7}Z&{z&+nk(o{{a4i9@Q
z-;*9gvi#i5moy!15F7RB)}%57uF9FeT)zm)YaYHUB^x^ZTsQzhpRY^c>-=LptjG7u
z<?~q!==&%r)?JTHPqoHOWcS99yX_NT(?b%}u@s-thN@lV)6Exm&+Mkb8NI8rLNyF*
zOt44f1f%EtW7t6Jr(G3)2lgX}w^)C$>BhM)+ydSaMEqmbbrW$5r*fiUE?C>^^U-$_
z?sI=QV*9Z#Y?~aR<8hUtUnOk!?ciHfo6rw<xPD6AF{b*SShMG9R4W7U*G116w4*%v
zZB8_I6r-G0XgV4?FYN;AmT%s*{g8@;zPr&===<*JNqh>Mlqm8jcnfjRcri3F++8TT
zjL3HUBsC6mB~6nwR8`6-x+s}+0dn#vJdsP&@dZ#P$vEx}Vei6`xG<4Hssi+lQ`C4y
zl-rSS`Y?DbKiNNonN@Zjy7p8N&f}nA7tu6eA^7T?uU`zoP&Cnhx`7R`M18k@kJsSr
z*Kf3s8eip<1vkFE+3Jby=jJDH(pI(}jI}8vYB!&k?-hqy)t`rNGZ$qQAD|U)6Wd?w
zWjt4zpj_^csWqgDvnHnYT%mQwg|?Bl9@zPGZwt*I7jS1ADCzzSXv?~i$3R`rJ#d?A
z7c~eAyzYGs+97s+y4iu%dXlmYE=L-@9VbnCVTke@H+kNWcRa1)um}u4f3_#9Hbku+
z3($Ivd5<KaeW{!~hB{!upHr&WVFq|$tjken@ic7oxa!?+*#fC|*f-nWKJ4v){EmpP
z-PXJW&HS%H9WgkoZfoC@X4k)!FYb9A-s7r=oI&i$$CW^@NYp!077G+smc?A4Z*MSF
zo&e{?FGCYZh4%^+{5b}Mf@Suh*(bJvLv|Ll1(OWd;#umMRX>IJDX-9+1UjXkWWa7r
zu^QW}2St??rGhodhQ1JN<s%o~RSqu_;2ZskMC_{-x1<!&pahDVZV%p8hC5T$?O7g=
z(-Smboqf!z^Oy?vjSdg66CPsF)o5-=k8UEz&s1v@dK`Kmk5q6L@e-G$zgz%gX$kW7
zbByaxJMykqdW>O>FwgzWyVWhtXfD*A7D3OokTx!>sB8+dt6w#r)qL>r@UrdJT9&9w
zjzcU{Ob?E|K*=>oqMp|-oumldVYbEG-WEbMa|jh*qIEI@tWWtJI~q9G>qn<FU&r{0
z%b7^7fqAKJj5QB*YDacCDDzBVrgzx{($TV#3ET-te($FmJ_S0h)j-g9=B%5aaLdPg
zTym)}OHrH~k~S$ig+7k5HzlP$91!fMuZ%@`W)(Ko?(<A<Lpm<yb245C&gi2i9Q}gQ
zDldx_NpVMSGhq|xcqVOlYSUPQ5}D4Wvy#mUO64Tculeb8$oM0cr5~r{<VjvyCJy&e
z7yIPVC)lx<R8jEHIZUQlf~*Ce2b2(Z-ydhxjI>!5`r+=j*sJ@b3Ne)=fw(#b_bq=O
zI79i%9!+6zSv)84y&KR@M`lgm;K5x(?Q9CeR`DsQd&jN!MC&EMe7V3>i$_9Wfy2Ix
zrwmKhY02*M7i|~2pU?R+evhRKCHQ41(7oIYIk8%IEVn%#B`4dz2ZK@?N|4wOJxA$(
z(;$DCcoq`K!O&Ss-kjc|&3`-*QL99q+iCp}*5zJ($0(f@HV3<#I1%&)6cqTzSTFJ%
znru}PsqE{%aXvXdbn<+l=_7Z!^<kVU^EXg$HJ<zi9*njgs#GGZYR*S7ZrA%_<Vr0h
z$OiY)jlR?>t5U5ro(siC<lYn6WpFS@eU_J{L+suCRE6qub#a(zaBdq-qKzmtlC*Qc
z<SO#KIj_vav=N5jTW)mT2jTC_VP}X!gF-5C6$G1W`%X_Vme%!0Dd^q|2j{H40A(JN
z`BP*y;X-M7MTLRB4LQX5)Gfeh9nQY<m23L8viIhr>tG9PW8h|eCz@S`<Q3$PAF~T0
zgoK1&(>`dsvV1C<gJvZ88SM#0({*COvrVuR**RfXWs4OC%Ck%U=zu~YoEipSj<By1
z^Mlo@g$o#46;{7wY~#xFLd%~q*8Xudl=T9rlxLkC<E^Z{)@%Ab_v;lc*{~DSN0?Ao
zYmtp6imS(~fsaz&tnxjyjU663NNMr$rn3N;43POO{+*2wtUh|1u={JZGlRpgC!WQZ
zD}ES7{HJ1A5xXmfDBp(E*lv}TtocpMul(pn)SKJ!P>aB4559W<iL7E1R*2l1b?!rq
zhZ^ZQM4`LaX}1{GQY~zYy6Yt9IZ><ax8&#N>fv2mfm>`f>~)b>?dR(LW-6BjlAuO&
zSwXSO`&`gOjYr+1vR6fB)lnJNzG!0)R63$JmioxW#s(kWM*+-ret_MVuTWK0J!yCH
zR*wlX9Sa_v(YK;Su9^pDYXNyun3H#c*9YbBugv0VX@-9$dPQ38oR(_C5ndJjzJLk4
zf<#eG{?8VLjM7Lg#>cC%JT!jdWw6WLRBeRRpE+=)kkA!ZO^%livUmQ(Bpcr16`~vm
zE1YSWrh7vx8R)0Em?Kk{-N2sVs3RSMy|h~=#X~!0wOSF_b8o%&M(m5<VIyF_3(ENs
za>aaQXu)D-Y#ruGKip7w$hI88!~2i83!Gu>d2eX9-8nkPsogaXJ_Y3C1aW9y%LL=%
z63W14L`8T*W=x4bAYC4r!RJ9}^SW1SRJa(Wjlm^7T;jDfh8^)ZVSgi7cb6XY0Sy~*
zc^SyDRH9nsaC593931>5EuXDK;m2G<ww2CUmt>=3G5s2UTNaz2R42UFp(eVU9U*Hk
zeb$k<p;7?bwxiIU<K4&4dX5N18lU^BvV0Kkysrf6lv^iwkREto)>VB;V0Y5Q9Zu^|
z(oQDw8pO`P2&LD6G<O|Bvn-8id6_sgu^oH12p$uy#eulZoPQ_FTcgU3CX%sbb8Lja
z+UpmsyHiN((fekzOdYN4<%ThbESiS)n{*Y6Xl~nYBP4|u_X-e@TD6Tv2hUJ153Xzt
zWx@=*nV0h>a0G@~0mvKA-`dC7tIUFCW7gotynTa}#YhzO6E1U`<0~s0kV6ND((vqC
zk?p3bfq1zs)_5AnO-O@u-k_7ckAHaCqJ0GKxS|ac5#rAJ8}`yQTI|!ZojYOgj8_es
zMg_Aw_FB~0quVY>g)+#86VLiJhrfNj5c=E}G2;&t<vY}pDc~+zDtLY}A>?q+iT2_O
z%qf2`ClaaUvv^!^^trw?@Ht|(UBELhSE=3$Yj;R?diT0I_S(AaW5J$Wb~A9-EaZ?}
zs7IMO_+mbFVuDgjM&(gy8ZGG$)3dd|c4u{>da<IobjnV*fjqXp94vQ-Rt`9Njv>&Y
zFnh*2PQ4X=awQYJKNjDbjcecHawY%q@&aQ=X}$??xtDux%sx7&dcm;uwdan9bz%VL
z6x?jz2783$>*|0xY^9EN30YZ$&+-L?;uy-J{ez|cP7Ipk`#LF~DD+`Hz^webbpk?0
zXQ!}Ez14LO?6tHYMA6idvUj&Vo{|(DW$r!(8ks%|xCn1zx5#FxrmC(k7|emH{Kx>%
zB?R7Z*)G#k<;(D)D{XEpz}c6r7)5T)IQgGt_1{axhTng|Xu;`ej$KJ3sy6hN?$BS1
zsy)!pbeKI0$STXXJK)|l#)WilB-y07zaEGNrywXp2sGiwWS6Ho)JZD37@;Ys8BH?{
zulwHWyBy6kZOB+r)-#pJ4sdQ$eFmU%-5SE39i14fehPAsDR@x--T*Y@VZ06K1^k2q
zn!c#m7fJMEvcIvq>|i3f%|r4Dr$I42+V+<5<5Lm-fwoiJYq$a7X!!nF2-~*!)if(5
zI5u;&YGwRw2B)c&4UQ)FYcKev{>6e8#JZFp&QwTI<^GU(P>|a=*^XSYpqT!foi9r)
zCMI^zhIDvAO+kO3e^zsqk=U`)uO7g@J%#HC?4yGYy(A$Nuf%-ISWyE???ixu24vLK
zk|b&0Ab7!ywy%4^S=E((V#iFsXIVBF9sl$>e+)&#OU0u!nW1Im>rO+#XlLfpB1AXq
z?Oraf`~GiM{kI)ZoGN*iSbsQJ?%$mHWlvDSb0gNz3V(#b4)uh<eefgUk+}cR+u*>>
zHZ^gP-#yX(L)ieWbHM>Pm<y&A!4xz&z7}5mz5RFbA$n#l=xcBDdlWEPefO4V0>0FK
zBc#T}g3?0^?l7Vr=GA1w4_v#zjPBouuxTm%PTi3n4zVh{{9=rI!y+u9AQR{*=$AZV
z!1+79)^C|w2EuKU>BSz#lK(u^{+koL><|C({mC)Z6!Ks+`1X|u_+Y{1zYzl9OPSmh
z{0s;9J>O!(^k3eOgcnh6cdGKZZev0A{sh#h9Iwmk#Y(H=L<+wakT~#RzMpW+1VyVM
zaNe=ZF~*^tG=$%Pha|Bkc|g4kLT)h>R)Nz_j18PTZQ}<&ny<2oXYqd1kqb)bnGVSV
zXY_Lsf%$mELvBhoHVkm!;o=MqJ>j$2NMcA%PL2;!ab;yCpo95`!*!<?-aNEId1h$W
z>1p!3s8or!(0hDNK~`pnQ1uu<fyw49Us<bp%vO*U?fd?;*J&o6wLDk?Tb}S%%*Q<c
z*pCtjeU+#~^b_JHck=o36;Z%tC)aaqPmG<fs}g6GY1zEn@ml>vWAfL4;nhDE%lRU!
zGO%(&GJR68pR3MB%$*#oA4jqOGj5T;5fz2Y($m)`F65Ae@+YGB%a2&7f-@I2H7o*d
zJ3X7AbKG`ct?3R+t>SCqy$(%fI!~dA+b4LqXI~}-bzsa(FM+6V4<UcsyERo@hf4w_
z({-N3I(Cs!4XL};a#SKXv28)YI)~~ov#~G1EoJ>v{y*bW%)hmuP#UfEyxYfZ-gXN@
z3bv1t(3x{PIJ1c%kRIT4cPB!CuKUEUG-h09QJQq*R_ejjms4IMEjJ%jffaTOjA5nx
zQ9ZW@qeFs+<g5&6TEI8J7yDQiSN%lqgy5JO9ZujC&a+8#vn6+Re<+r05wZ#D!*BBA
zM#e9Ot+n?HP@kz?ydFbAXM7-E1DbYuW4chEiIU1~J|vQEdJAFeXwDchHjeSXlW)Tc
zoaK|xb{Bir($~aVhyOO?uIT_o`HrKH(?FOnA^>I?0zk0}gDb)-CMMR`4^55)h`4$|
zoff4lzaY{(tH0&YDj*64+wTu-w<It4c2dow+5^gM3(Q^5*UB2Y_G}&4>Uy?s3rt%+
zHu-o#Wi06CG`FF0qnjlM93WhSW*8CGq&}t#MSZH*aNqL1gD_gd@JCV{oEELV>yj&@
zZ20ccg@}T0Z|XQUbKu>C;-+9hAxnB`*n*pL&9>FPLvpH761L%IiOPQ7*4)uh2M4`I
z8@K0rT1rYN`$+3cGQT~!#(Z$wiF0A?`!}u21}|LI(&7HX48US&@RDwAX_@mmg$<l%
zm%uSyoqhF4{_MBb=}&6kb{?v7=_UPOk2wDb-E-@NIqLcVeO(VgKsFtWRo^4gZFVYP
zE4mpC@EB0P8(WV^?w~X>E%`z%LDpes4pa7W5VedySbbgV(bk^BT_@hvwj+y+lx7a$
z+v%$1K0_e(*{%0E5?e9`E#OLUYMvP;(f>wbJ2wk$d5YwDowaAI9?`Mo+~umY1M|pj
zZV(i~B)&qqC~z%Qrm00qnRGI&4bo%V&QU;g?8sZxXw|w~vWFo$Hr7@z871#4p=8#6
zi4r=wMr&KdfzR^Fu8mNki&%_C);$8Zf$OSZ#hrWzyWshgA9hPqw0&@HcQ9j)D&VHu
zb#tjfVZIu6vQf|jTxdjZ^W=~1r(Z!MxFrbcX!b+e*<#<JRq1RHJ)`Gk-xj$-QM6$@
zdZpD$NF-)xi^}q{2+p?*(Bh#*AoaKrxs*z|Bm~_tqM65%W;=&`)^p#0XyhX`a=R4q
z;lxd5f;8zhBK7eqLz#~&f!cTv>)>8ihnfPX+X&^YgWT7rL4no?Cy|liJJqRxB_NC#
zx<slmso#MtL@e%mB<`Wei!l0QJ}TsFT1?BM8;x?XUi@_)$gK>70J%RJB(D!C9rtp1
zZ~(&&Pv%G$(<_;TBm4-fO}`>*@oMF`Ii3-NAd?}<C~{S#fU$O8zCaTb69X7DQI?mN
zFDPjg(q-oX*J}wKBh&6W4|cek?^`8_itpx)p6sV>c#xy|<Bpi=3zm3!d4~icYN#!n
zjo2K<d}+oP8U|XmMLiO^kBWmR+8>GwiKF;|Dr!f}fo7HiPEAB)G?G+#!)X+7Nor`e
z3?D?ED^0!)+ZQ)#Kgz3qSZ|8~jKoVnEx>|DByR~_BApf#(%WXO+8K;Ka`a~i4#wl)
zhL2E}8^=qbRcO39y6lTO+JaB)XA~a&Up#ysg>CLVnN$EKbrzASDoEP3Fg5ED0J*R6
zE<SV((zg5dpJct-7B_xbZ0u7;6TWx=l{K}u;L+hCOCt9u;QVpsyz9hQFkg*8O=}MP
zI0ejYKcnT&=4au0s3O9renm@S=X)|Gb8_~@@Vf`lQ%G0%A-;+i_ADGRd)-=m0NRs}
z-U<oedTf<it~;E+KOHRj`O(5YaNP9|Zp@4Sl?Y*V`QF(AxG!P1bSoyz2GI4qqU~(p
z;AvOn*u^Ro*dfL0p$#{kZvqJN^u3mWvLt1eut?AFKnYC@n!1j8ch?lQWvQ8!^yTdw
zES`<`QjviLsU*435spXmC=UUypZ^N0g(vP7^r^jFd{(R8!1qA1pQ16R?ds~P%BUMs
zzyl=W2S`MeBy^Qp>|H+o*uX~=hLLIJ&Vtk^L8c+EJt_lsGjIF+e)W?{OdLco4=g~M
z8fi7cT*WYa>QuBxa<0Rj`4H`N6eVD)B_Syp%2s4C32!f%5iqgmJc-@iI{Wz8oUR+7
zx;xl?Q?$YAF>K@t%7T3!Hpwb4G_9JwDdN2ATS76H?&_z6t(tp0ce!Y&>A^%&I?04f
z-Mxp?>m-}_Q{F>(AY3Zkz|$Cfx@-!HO&oSnS`#)nO?*j4aZyzvSC_-cQ<<vrqGBxy
zYtSF=^`3wyYS%?1o&9Nrtkf?0e$mf9s6VRj4;$J{`NiRT?MnOyDZ`+Z2HlUxCYjIA
zbLA1^ri9s^lf>awhGNt`ff!RgjJI;o7r0U1JjX%o_}`2v3CNRB$SRS`&pICMV%ZE$
z%v@!DyxggkX;mGWhf6bYstXd7%-S{<%WUknx}SCYusc_4kq)YpNn(>(n+6B*e+D|z
z!6j8-R%+K4s_3*y<Dc)p_(om!1F>5jo+h)jT8@N-QjDTPJ&Ay_G!$SAyCx9iP;&HA
zE9kF{m)a;nIrGkAtTlvB@F>qm1#f&DNkmJBcDz|Kw;_lrHJpcK_X&RWV~zMvo#9`B
zcD<?6P6kmygOy}oZ1JvWuERniP)vMow>kwE&>f#>1BYF^!=OGZkJqtHH`opOoyj$M
zo^vR`CP<CU&pEAClv!!7K3UQz7`6l>Y7Xb#>#8JK_~Kb*^mK2?Pg*K4{7FKK6RbwQ
zO+yV#2^jE6@NTM^MbF{(ML!^=GFjfUn1X;4gh=LA6j7=|*9jk-OuIIEQ83Lvc_09%
zB%`C_-fVrg;N-B@Mx7(%3X-@SRBabvvEvGq2|;ww^22_4+*U3~L0Fa_ow|Cl<neJM
zvU@G;Uf{-h6jPn7*Ok}PKY<_Yq;woQ!wICH?Up)A%MGl6Cpb*+)x2chlGVcF(q|4@
zFDJ;6XqQVCq7;Yj!mXy8#=7YpB@5YJc2x^c_eC$EY}s0CQ6X4+XKo(v@Wl46Xoq5}
z@tI;;ms|Y#f|aM50eQ?8D56j%tU^L<CdTTaF2OTu6AaA#@X%!!t%nuLKyK`yjRPBX
zm@(Phf<p29(i;iY=&;`>@l>(~SNoVM!q0cRkoe47)7N7UH2;813ZE8BUFkcJyZ{c!
zbAgfVe`Ih7IZ)*;1Oey-|DCMyy%3jxQK3cBCx|Xou~rEobK^gqBhMfII7boq^6g)S
z`mW3Q&-2`9%78_lqR&5_3@#P!|EU1T{iHEz-A6~NEVQ`lPu^D97=|1KN>Us)UpeiH
z<32JTQVn$Q@*917n^Ex5-xck_nwS}w54&8fm1A@(TR44`0`^BlK@ZN90&b+@9)Vh1
zzryS^pjaL5MC#%n);bCeXu%#33o#gZi?cAojqthl)YJ}}#LI~dr&fN)%4HM^Q=4!M
zx9<R*(^m5eRCGq<<0HoFRY52nnnj(549cS3{gQg3o61#q4MFxV>l;qcb&!RYs5V(L
z-KWD!iqi>iT;xXGFsT*3GNJI<GcC?Q+}4TduOS|EI8IJ>W9-!$`Ovh4-0zbGSFk%W
z_;1&%KynqId$g`zk_Mf_9Vm#!o)Jg6&Dq2v`D<T1f8!q7OCAvTNtqCPU|w=S8Xd5Y
zU7ut@zF4mUt;Y}PoDK*NsSKmNW$w~O@oKpexqHTapJ7hgGj2zwLkKZoD;#XQ;YRyg
zt5y77Mz|G;{gtCX-FC$2R)a5=SN_`?E&rnLSqP6aKjR7)Y@z>nUrlXujNHrizg?AL
zweKih)IPpQdmA2`oAIVRO@1iLdT9YU>^GDOmZ}sIwi3Xn<;j<ACV*-rg51(WZ}w$3
zM1E_Q)Oow;WYm-lTqw&=4T*g_3>6X$@$yjW$*C!ojY3JOthMRG-4UdI$uI|`bX!>G
za<ZnF>fR^8tzs(`)hD5{|2abhJtxHB#|_wvJw%5SKo1Y^;gtdbrGvVd5R`BX5^6#C
zQ-pRwK+~j76}4oFIOEYK({Lf@clC)d3?Wp}R|EzF9Z|R0X&x*!8<8iT;$6<GI6N6s
zPbOa|SpO{NJx<}aL>uw6AHh!$4qw!wReF!c(!Q0=*LVxs8{$M>-#Eu#2y1@!FQTs-
zhSYgbKQ}M^#nWhk$hL75#vLPGoAy}!<sm7V_l&!2)eRC}l4){&^<YYnu^{@v-Owk3
zG+o2waw92c?TNVVW1>}_<MbS&#VZbTQ*>|A#%pMdADzWwGZgU4+INn(e3j1Lrb@pq
z`y;o_r``dJl^W`+f|;CDSLQTFigh?9MuJ}H*s=n<vuB5z6APE$W<;p~WXyPDR%~p3
zjUXSI<5BVavS)-_YK87+Lk;q4AAA9u4UI_KS(R~0M<@UM&$_{C8y>~O{UWXn1a+^s
z4Se7IWqq50I?@?UP$>=VIjUEr@kx>`zS=PaoPWGgOZzEiUkR>O>LedhqCVOf;9~D`
zbG+vcWG0iWWM1Y_aRQ$go0my6D0i8m{$1}^KXV3ZXUKY{!^w1Co34YF-SPa2p3gTz
z(tejsLDvHw8@@=U2QC<TJ1(-^+ES%&AWmcm@i5O{qqar2Vjt!CVJoLOL5Y&P=X^~L
z@9HeL?r*4E4{~g8DF)O#4ZGf*p7*Sa`6=2x8bb!&diNY{I;P{SbT}|zOS+DdRF=}M
zxV`j-q@{ecR6~RrE?@ZB;{#e4hXM&w-6*$p%*vX7RQ31RwOjMH2S`je`;(kV<tO5g
zS|3CV>uWWiAx!uCvyz*vO{&e>8ixW*NfsSAlrxir8>WgtN&9y9oU$wa$sX{nzNWL+
zD!9Jxh96SvzolGc4p^13*pCR4pni($dQ^fVN#zz}T01^F@;7FkEMG#Ks}-6cn7B1u
z)IV=;2?~w|TBOzstJ&;&Kh_|wfXuAipC)Tj>gs;w8PD)}STDNct)1y#mH9#)-JVKa
zH}-EVg<czbn9?g~G<Y8_MBzR)G7$%q?in8%7Ac))+)jPCeH0R$tKH}*15pi7iD()=
z5r4Of!qGx6=j$A1&9ut)do?aDcZTF<Ko0aY-gsudhiQX!1&JQP^0@TB%hE~g3V_dF
zKr0LDU}-s<Lz-+>XX5zib=um=aHU)tLY~U;D-t1YdW049awEWIyDk#H`n#$O9q3*G
zeff}(`Al*Xues@Zd)+BWh#t>V)e-I-Z#!A;8rvGBaTW_NN0S}nk|&Q|)+(PG7Ji##
z+zDVcv<nJ1Jgz#<cS__OMaCS+5?guS>Tvv9v}s8*WT6-z6se&p*FX#>RaDtv{3=Yy
znsdSR*_AK+OEdL<Njv&4SxZ1`bjp|10Qf;2qZck^cLyq)+@vnMkWMW*6;v*{r0&9s
zw(q_EzqN>=%5Ut29!xV|8lk2e79@EmYKvWN;9P*-<i0=nXNxp3;spkAQtmLkXeN3#
zaS74)t<5PiHCU3hKb0;)e92_2aUPS9tEA%NZbR;5pN3Ot_P`71^}pY2381o(iVe}<
zFyGFr#r4nl{fw|3JkwfrOEFniNjwl+7owhl1%138f_6`OOYd{@o~*zC)rZ2iRMR&<
z{M~sM?=HLh?mIW4m@pZ4-RPH56jGgo?Uo5cf`t&8tp`_Mt(#ZqETmJUpMTJZc-+)W
zqCz&t+u>#!l+M1!+DZ|vQ8k!Gj|WMamg$quD#8z}cS0phq!&V`U#v%c4;usJtZ7A)
z*u1X=ygObd%(e5r{P^;4%XhUx_qeF%qmJ_QUO2{jLA4^7tk#p%=VFNFU``wny~qdc
z`v>~etM0{@F(u`vg|b(z==zhVFYYdf&+DY$JbK;kl*(R*yv@U#uF@Akl7^ze%DPux
zoeZgJSR3(G?Te0#B<6y1uhfa5jax^)co!O^2K^BB=RWFurd1w7E92PlE}H8I2@Z~1
zxKo8CyfqIxcpSBogFr`+Av*+=`7vy?&E%&S{!*yRLR$ycr`_8GdIw1X7pTOe*a6)4
zoz7aUKfX<F5O39U@12{8IPO>N*HUTCnZko>xcxcN-H8mccug;7sk8(i!{N$57=zGH
zG;R(5NgsJgI(eeNJ$85R2FCqOAZcPs+9PKtD%xA>u@$$s!DELiy1@TRc?d$ymA7|Y
z@q7e0Qm8mup1cB_bcs+B5c^5zz?luzoh>IlzT4`1d9s3^37%n)2)Vj{2x;e-)bJ>P
zdWzguJ}%B9+I0hO$-Tm8I`rMKP*m=v3_!7Jjo1H6RtUi$#gcw#*fW=pRa(9NwfA19
z3yXl;G0sHed*7mTHMZ&I#)O)2A!k8}l;YZz$GhfjicN}T)P^OPq*~b(eh7z}+nz>t
z0akOncNeQnqAj@hn~JE$cTZ$7Zq1!suLZ4J4ZQm=q1|I{utJ2sNZ@iN$ZBOvz^#~r
z0DctdpN07A^}ZhWl*QBNH9U5OHkf<g(I}60M1i@ppewI4X)|Te(><DiX2eBY!{&@x
zxAf1*-Tlj+uapWqSw4+)=#N!uWh^gD!Woq$cqwnOON?z014I3DI0UfXKFCXji)gQE
z#SKmFVLm9<qDZw7v!y*DHXM)61nvC5k2@F@FE{{9jJmlfU7k25^ER{6TZuD6g&_xf
zx)L7%aJw%Dp~!Pdg5|IsVXMt!P^cGIvNK(<EM{TEgb%~XI?0-KpE7=vWlJD(B;AM=
zk$5S*Aj_52)<C5`CX3Bd34E>=Pth5!$)0g$rT3PXdT*)^m}3(UIo1DG%RQ$*WQt1J
z`-p6zNF03e&_et50>|Y+clc-syz#xE%vQoZsE)5sI&D`^ZSf5GA~#nnLQn;RzGGq5
z`6)tw-|c(UUxU=&;P^Cs{|;SaA1i9J3i^P<0v7SSS)@6#_*LPn)+`5LU;z`D7Y6sk
zc7rRZqpo~8X1B~v_Q%5qDe2$p`I`2jjf85!8rsCkZFAU(F$`j_;EH55@w`}^k0sft
zzIo;r3reIZtmwf1LE1Y<*Y)-9+D*g8Xv`)}(%810G`1Vtw%yqFj%~BCZQJJA`96Nm
z@0@Yo@$UbVG1kUhbIs3nU$<Xh<usJ!%)<E6)OBc-r2o?eF`a5FYi^3M-Vcujvz+vJ
zRCbop-8n|Ou10$i$cUxT{pfc7z}w0E8gSE)3ZENf{?7ux<=U7^Q+d_~3<JfWxJxcA
zXKHy9A%rL9=Yv$A4QX94!iwCZ;+Xs)i75M^Mw^58(T|VkD|wBs_fH;`vXR)GOg`7J
zlWgHibYgm4;xQhDj@1Ew<)ceX{C#oijnug2n(e_Qhe!9LqOR@irx$x>oS}|3q^qks
z`HYSN)UgWp-TwvWS$Y`(i5X3z3>TylTpyYsmNAxPX2+8S(gHowZSohx^1i+~B3g5m
z+yO4hobU;~K85hhBY(z$9>U+!Lf+Sy3hKsD-y{mQR+~vTNi$I_!Af`v^y2pI!);2V
zQr)NrOC-F^<kv6)WmDVxmg}fD^tktr2<xT#XVD~d8|JVxOH-m8XX8RO6i%+%jJK&8
z>LzS@`hFDOQ1ZL0WnV)VuYhWf>=p!b9R}XCTOOTCpMQDGwRhq1bkw?aUUHXvy;p4@
zNnt5{d5{(5aQqtCOjWgJK~Vo9i#?D2a$f{&jNyo<(<Fay#^IC+P7j9d4gF<DnK27f
z;#n4M`kLBFKHed3)j8)vFy|J0`@x;~42!!EmMeRSaIWjT%<VyNN{i3y?X2DX9a=F*
z+C2thE1u;C(9YVm>sKvg((e>?QUT{fsm2evaGoja=+h(RI`Z6p3rgpgJ`08n3*^Jq
zoZ8D=g0tu39Xx-bH#j%UzUd2v>m&zaFs8H1di5g8lBQPS=lS!Smp-kYOI6bsmJ4uU
z`(ECP+KB@xWsl*{$e8d^ZVnriWi)MT<l>L{9HOVUAan1F=$Y$q0Sz;#nx5QMGWf-*
zJEWJLHqOjvBprdb3^VU%9|t{O7a{D9fB8KQ#6u%dYq!NjMq({~CF8Yxn^dtcDWlxW
z!@OB<4c><8LsmCwIr_6lT6HPOh9z(fVmD!NuCU-v2T9@dOqAnpS8Jjqzuk0&h#y9J
z#*=J|DaPX&{%|LSxUM1DGyem%`&MVN9dsy#Y?zI!OiNbS)#6Cgiz~kD%W#9Yx5(n|
zfZqA{kX#1+nVE+HI)ExDJA$A!Stir`*D&YFl?B0EyEJFJ<=YcNBST_2^g}AMYTvL^
zD5cDMBt0Z2Wllwh(QzN76#G+R$eE@|^5+KubZnw@izwer^|^ZGtqUpHxa4p1EGEmy
z!VMmq&AJ?Scrg7IZBNE?gFAo5H8eP5bi$8PGVPGk?vQ;f`Z({Z5`q^P^mA>;p$-oQ
zwIjlfTHlzzk)S2JGW|Rj`uyD{@A@IVDPlp8;B~89<a}6sT;~$2Po3ya%yXR_W0(~8
zU^doLAYr*|@QQp~=5G`xqgzFkDHW~d4wxi{KBygNzGN*_+eTFXt?f;g>lf%4q>koW
zr(XD@fD|z?QE>?oHGbqNyT><H;0Y*c%f|_*^Pf(p`*c?AWvp|xy$AjdRmi%04zbV<
zf!SJp8%P#JyYC?TxWHr0KbRs>!RJP$H|WG@`)L}L(s51W9EEh11t8?9#qS_>8MY!r
zz00mevu}%MLFvfNm@Vy(z>bAy9<Vey=sv5R<-1~5CICzQ%M*GzIhSGh`ZFF0DNj)9
z>#v6gA-*?Kz%u4`px2G_(KP)y-;Z<Qb&;?F@%aX5rb+(y_9>W8&J_|2w_-_mgskyr
zyPEmPSP*O8<8XA3D+2o@SVr5Kh%oQFS{OW|Fxpg+g1_aUKh7fsT*gX^zflW`7DCxt
zVWX&`r7!PP?@Lkeacel=WUTWioqIS_BOc3K2ZqsDYrh0?CUR=0F08XpbyPi=Wh@gb
z$k=dwN7}E)MSUi|J9WC$_c-@loBy3!-&@OEm1KGxVx<g8sP<Yzh7s)X+Y{$zqKM6!
zD0A&=(-)XA-u=Br8n};HV7@==i|f_~G3F-e>+F4W+m_FlZKv#`eY~UPMV#yI@W>-2
zngL!)wxShsQZqF@(8r(SGg#NHcRdDjGD(3}z5YIPNlgs|4HII5S1%xg!3SU3y!d~z
ziAo*|lRHHH@v-O5P}0UaV06KbKzZ4~eho*`=hL|Eip*`uS~mS8Jd9ZiVthv*#5QT>
z%gsjrOW%z`;8?k#cSF|kV7i!_oCIvE6HVDXnx3jJ9H%mI+Vx4Sc>mW_7xnWq{Ty0)
z4ZZ&2XD8QLTZmQgFWm-G&0S{?N+;7N+*l})9s~E^G_JkbZSm{z8Uz~n8)5pcje3DC
z46_&z=uSGX1-A=!Z)ooCOMbsKT#pY94|ThI!LOXfrgCdFT8(qhpxjhaar>2mwg;Iw
zThIfSQB8Zk#>;NP>DQ(iMd?QB*`|hiatU7GRpNW1meOjFr!Lm=>N6iCbVMwKscM#A
zqXcyBdKNJqJHfskI;qqug@Wha#nL*Lv8<gx;KN(|lxv5Ons6zAJV_(rbK7P)qCRgJ
z*DJaA8JLa^bAj-3MXj~rB0ts?6Rg#8*-0+#_%Fz3>Y<?>1$LY(5kHfH2irRUb*<Mb
z%)Ih}0wwr?E=bZJoH!>S8LIZ$xI}#uWGsLz5q4+z(=4`v_vyW!hY2<{DBUVK#f7Bs
zLWcfD$93HL%!}5t-JdtkctD~+SoZ4d^R0we>3jd@qiICmrTo|n&SDprWDu;|;{ZiG
zd>xH|Ve5s|>jpYZzBk_VLRhokb}h32E<xkGTJ7XUPoY;&dSG;PUi-G+SobK(&&XEe
z8vT_f!Q(K{?%t;nrVwdOhXBHHbjUY2vr44nm!fQk4}7H2a6ujsp~{!7%mPP!`(7K3
z5hl!-@C$}FD5(j)Q4(BT{SGj^_1eC!Oed4HrZE8n{o79#Q5szW<BV~*^j~-5z0Wpt
z8Jc|mVCTG|{@O#e0xrARF`gpO?B(Jiy8e!uqt>t1cgx9`%1Bn`a_05XVJL)_L~Utw
zlGV)>TzcU{IWFdLdt&k5H4vKpe^hy%l>Nv%o<WCn?ib%6U0V?8nZ(XDC7xtwCtNn7
zCa1)h&)mFn2ijLQNN<@fcb^_5`2|I3%R|T0#6V%}&n&`xs3Lw_Q-4Z~LyJ-<qcm#I
z%{O;V=skvlM|sXT<|f%laJEtFCpjulrWsr~_;b&3tvd$FR#iI5wDZSRib;nuvRyB?
zn%%>YlxMxoxwpN%0sKdHweR_7$W3G_JJ@th1U&hKZ=x@w<Fl_n8`;bS+7<<LwV;gj
zGl1QzNK}xo2=C7tTyXV82qj2<lX8#G8L?m8X;iE8Fd#Nn|D83O>1s^CtA!8VSTVt6
zUd|LbJyaruWeE)MU`l&mkd3qIjVwO4R(pD6E&=V4T3cWr2S4)?<d*K`4tlDG$!hWZ
z#JR_|RbgdFzzdzEf2aE#ytYu+g@6?1+H)#$h-HP6)QTFp@yN4LQ9-(`mqV`xn7KQ~
zTsS^~%CwNBtuduU*H*I#JMCcCtB57CoPWT8%m2~rNE(iHHMw1^Tq4q?Egi_U)OoZH
zioxSV(~gV&d9})_xMsAeSU>ovTvw%Pn;xg|q^lvd2M@Pg<NizE!@3nWUO;Ok9o8!*
zR8XeVc1c|ds8(M0_I#Ul@^OUFeTt5T*`?2TuEllss802g`bjDRPJxLA(Hj^_E63cY
z<x|eh$`wdpC55Io)h~^nyJWqm^GgJk3DZNYGaYB32JEtad4nW<dT_svi?5yA4ZEKP
zrDE5us*J5L-fUEAqG=uUg;v_t19IoJY^{wA%R;^b0{!v{4Q~+o_cYQDnXg3!c9cv^
z8kawv-nwS5nKhd%9_Z4q+pOR|=&HAFOvnqFAov3s@U5IO`_mT$uYzuyoQ0rS-J^mg
z^Z5|~yk3G!k8U8#U7jL8ZZR#;*Qgj%2(<brii?MmfBVLViiDI+n1gP&KZfh#ib_vU
zuK;Y*RZ7q*uYFHbs8^C#P^akyOwsq+o(Rp(RkQ$_$KI6VzcWE;=3*_#&S*=^5%;aj
zrE9SX&(`US%*JU}T|LRQl@+Z!^I4XOvuCfZwf`_VOdArbb$F5~mqQt!ugZ-riq0#J
ze5K4dJ}v`<UFy*hNjV&MlmR^WO!U5q-JWX{TNWMxYLlY#*!Ng}&R#Bi9K9mHd)@Y$
z%qp6+k%ASS)tePJKOeZ917g}OOPgyhvdC}8G!;!JG6DYFdQFR;XStfVh2tNZR2&y7
zn!RWg<2}_&d2U>eh!Yh;{IE-q@75F(BnqYrWlHckowF##sEb-PG!(SZ<AnO+LCqAX
zb5;x*<f2_99E{t-Oh?ZM6!n@P_CvXHm#X@O>LzJ~H8TZXibu&4XE_v%a(a0z3m%+!
zs%3Yp4Wq~<pBma*wQqxlg=e#mLjb+K%Ydx5<K6wGgA#LznF^%acMWL;gWz}MahKw`
z(xoDHJgTFdC;)S4bM90tVZXAu+qUdw#gAX7``Shd4f*@xVal2uHlhx~mYGHDHJYs?
zQMmwhIZfA`qtbbN^2AY6MH$bLEk)9Id)e2`<hpcqKrWtMi(5}`p<1JjXx*AY#L#h?
zE580`N6s~o&2Gzxd+CI256`7zQIu2VmZG`ymaNbwCm9DCxmQmffU>6iD^c6~yM`=r
zsJ`#Ru!!O^<57);gw1&eE;)KtR8ZXgt=10s>^Gj>+aKek7BpI}?Z*R(seQI%7C$e)
zdlcrMJ9=8aGRPP_zd6sQ5GgBGUGX&kMk{G{{S{&Q@>~yad0&x(*5(PgeHIs@7&2=y
z-QzhT0*gOtHr)<>1-$Uw%CNrzaf3TO4Cc1*c_miMlKLo*$LDU~%po3{y^KU>ahz)7
z-H)odZTib9cRJ{MfkdT+Bg>$20uxrK%boW!hM5XNI%ox8o7V2r27dO9zWJ{pgO0k+
zL#wC3d1nNbc=%<-Yu#5#P`1PW0Usq;r~dP4wV+Yg)PY9QSsQ@uZ4eDwb%kwq>-h~w
zOIzA|p2_7~P(v0M*dS+ygSM5xfs9&UwNZ!g5XU)z!J~n(RC@WC?J%iVgzTmWi?PtF
z_w%XxH<tt3tU|;2+rn8GS&%{4Gmx@7m-KnBwwd!8ERvLtI0Lhs;UFbii95f6lz8h~
zSYbn8dE&yidy4n)r8~Zge%=dd;A2`3oSUVR@P2#S#LpwM)hvc)X0Yx%mrng_5DiKz
zpM<UJxVXH$D(oBHjs^;%BW;Ana{GV*uVmwyZ2^@%hQ<k^Ax~y!#PE*1<KET!jf)6a
z7Xp5=VZ+H9^pr#Gz6^&P%Lqv6{F$KH=TqB1df3^T=UT>cN{eI7T#Z^y3{SVlMQ7An
zEil|CmVt5dM;<G9aUl?^88-Nt0n#X~>(if6a}o5P1K}=MF~IfZN8yVND^J$I)RYlA
zdq&}wr3<p9Upc1&pkI|dCJHTtR)8u<xE9dgD#DPq`$(7WN=l<y_%@%^d#^26zXo8@
z*!!@fvM`ky>f|o7D#V}RuE}6F<gSC2-jqObG(;rnDmt@px@JEIqmYr$R`}K`z7MGy
z1(A^=%?xlRQtuN_AHOWw$_o#XQ~Z^Tk2%Lq{zovredrkxIo^HBLzuW?ODp$pz5ul@
z;RLF(oO;5pz0b7>o{)ZojejmZ?xUj7-r->TI~21lefk0zw`@z4%CgEdW3+`1zuFAK
zwdkdEIsW9>!MBw+iH(l-sWBQyYkht)lV(TiTvyK7^24+t%!wKKg?MAQBG`ZResNH)
z6tJ^dRwlaq3nX2Zm;<U1ndAxC*Q%G0jO<I5vb)WVID7aJlOCcpv9!r(FeEsaLoZ5>
z+y`wZ`=xMx{R_LIh7C{VvqO7hYGcC@eeJwtkSEfwHK2)7)aPM+d2@5q=agGH`!x{t
zsn)>BhTKKmjJet6dcMhyPPXd<(<8sM&*rhkgjFXC3&>$6A4`V!Yqj(hR}c+Px$oYH
z9<u1?eg*?=W~7|32n*J#^92%**0;T6X>tX;qv}v`i23Y9p64XgBAmmgQ=yRm#`Mj!
z&aNke9x$IRY)>v+|3<_7ysqyv^wsx3L?A;X*uoNN*)ww7YKJad6|0nz6SAtJuGe*v
z19<Q!a@j&vNb)xTGmc7K2~XQUSPTCj*voLRcAxfG`=HJBD02P}hOH2raa);5Li;rl
zu)IF=%&UBGO-)>dAs3FZ1ga`i;p<kb;(|uEt<Rj9jR*z3VCQ28-cjm9J??1Dd#Lt1
zZZ$j!-?*{IZE0?D`hf7|4{Z23;ruS3j6hdM$NOluBv&%|#7Q40G7(|N3y+KxwzZ}I
zOGZ%ZOtzo{B#YQD$+;b#5EbKNd|eg&>clWFyAk?P5TBma(f~8quR%<{p8hP3&LGW&
z)Ej$DS^cdxl_o~sz);{tlR7>tVqmYEtMR5D!f?tTC<oQB+;<WphS5(BxWb*y0U%>E
zwtc4%m{99jVYPtMiS<P+rr+UaXJ@QmfAG&uz$(~bP%RzpHm{Ov!Ppp^kx#|kW>KsL
zwklhNV@8LUF|$lub?PLA<@C0VT?}FP>U!9oTcT|-Vj?l!Alh`wob)6fTu%qlPC|EW
zhA>#aTBbnJ9Cw|hYU|A2P&#_lfCk?PG|vur-;*3juK95%bly_&bl$s@*OnJ_saU4o
zd+edm1Q66oqa3XM<WZUj&p|lpf0(_u?x|#cvpocRG^C3jsI*4V*})I-DhIWCpFlxT
z#~xHYkTQJjYi(r{!dNi}-Na3gZ1Af0P}uLVXBi8rJk8{~w{8&UdbyKuLpmTOGQa-D
zdV+MWu0~>MZydO-%?x@5)_K^H8caXJYaihHV3H895T(QFffis|cGmrT0ovJ2{Dv_N
z*6dXN?Z+tCs6*~d6mXgzA=Js~<X?Xh#^-S)(C#Bj*ZDy($Fom~Ee8if_4f7-l6vR)
zITMn*<eit_;3j-PR?V!Wzo4<eunQk_Hsfe6`c9~zB;}y(e$%WD(xIXU;Q1SA1gwwL
zH7>Y<1&Z-C%B4ATf>J3-_F+^=i`sF9FRSj_{QphE!8$x`{rP`L!;PG$(D3BVJmY1S
zKA|V{r&^1(qne=D+g$aMnziY&bd*3%#e?W=VddVwj7K|WLzDt-|1DnVja6UTowx%}
z)3oUp9tFOat^|Sgb~#;>znr`{FS)3;ed!52+Tb{rm1`S-T-cg~tv~7$p-y)WzQ={s
z8=iv@ec-x~ZMc>9dg?T2VoX;W!s;5+C-6-DeGS(6aX=S(YYBSUc1RVRga^g_C5U6s
zb}>QQl%j!J;(t5fn@eJSbIP8tpqYDrWQIs<1bHM3CU=<aa+=E>!9Zg;on)KEKI_O7
z$dP*je4yWpv_#VR0XL~gn3y3zhSpJ@l)NkN>>w~O@QN<WlMYYnDu7{X-*Z63ZMh4$
zc{qB0FcX!M>SI}V&!q%`L8J7}(7Nl~za0rS+E`LEda)bF!hf$g+`c3tY^}2Vi~K-@
zX|Nfz<N-9niPQmqt7RNtUYCkf#q!MWwyOs$ss@qhG9~$y59h85=X6(@%wPdG=t}Py
zp+kmp4g0Wpt7TD6t55tqKqfNoHV`33ZYz`mn5z(Ag01O@DjVCSGj<gDL3fVApJtB4
z(hHg{#kdy%MCN93z}7_ll*Cz)kG!I)P<%9{c{9IlUMy%J)Nk3$Us_fIYCmDGAnPc!
zv@R@p<dND7p&M}|tx0rxL}a$v?neCNUSG}z>oHl;bT5rY!mnvURuWq@6`hZR>>kO7
zJUY4P#p<zq$SIpCN_)`45@o1PTeN-8&uHY%<9Q03pfYQ}{!-;R+McwgC9%zRvthW6
zuwJk$E2eg7$qs<4>b5TlK6$>8M7DOVr|52p>dZRp)0w~~L!|i=`_Pk%3@H!r_XkrH
z6!Z@s4ml87X|fZ^&<Y3-hn)qkrtu%LL)V=t*IChlnxXF4v-OUOr}g(YQl7oN+gA^d
ztme%+{<^y$eSnAvO@m{}95rWdh75P<5~E%Nx!Q>Edy~Shi=1ZK?pemV0(PrjTt097
zePf3LORvU7cRs+cVJ3gmi&Ck=sb6#VNu;Rxr-0>he7Z)7?d|XS0y1HT7q5=u*7QWc
z8)w>chhhRuBz{Tc+87;vC2w8-sc$n-UOek6!b7)Rcol<ZWhG~!fY-(;q(ABCp|mWr
zrMQ1*F56`3%85JD6ea^D#};|3C`2X!fYP2d>VPQTj~y>EBzKU_SXPXDJK9RKolAdn
zXfbU;ZZ;S##9PXLI+3|e&2ZI=8dCF&Tn4P7I2FH%C4@2UEEE=s!7IKp*c1;vOSS?H
zuT?-5<X;dy#YE8|Fy`aYb*`7<uRGRxGp<ICDicG$A(24oLWuGPKz%DZTq;^M!h!I>
z0X;9~j%p6op9Ut%)PhFu_GOo|SpZ~xY?yaXh_~>KdaLG>b7!=*CuKj?%f)yPx*aZ3
zs0>!hbf^qksQ$l1e%{JP)_k+?5gv}`|14C$nZM0*G+_Al&l~CVIPvIIT*I0#65Jj#
z^IvP>>(2ZhkXOEi8B?4%vcNFMhK-74p<8prUa4*lc*lee+I71;jf1k!nEiHMY7Y#G
z2jBM7NCXVr>#5sG6R8ysi8b_K0|Pr>SVwMsduFeGqa081TIs`8u8zrbW+I@CEx=@Q
z*!I;(w0U#I!slL{h&iw4JgmIM0PQ(Huz{i;;Fs;S6;OUyAy{922yUlUWA-HB=oU+K
zXNK@+&7kf3pE;&<=uqHp4EO@lT3O)_W?2gBBAkUk@XgOU5ZC6HdkXR(HgBeYE6rca
z4KQx<M5c&olY{L2kH6{<e;DqQ<cPQbM-W2QWK<Y^u51|k^=(HmAp9SR2)FETXK^s<
z4Ay|oVEg>69p^IottkTQu+MP;%Da^A9|8TWr-j{$uc#y@I`uXKh#q^NOsU|Mpd~P>
z&h;Gy$zZ^pJW5EL+Xmf!Bf%Crac#N?v<?0<&%sE9x5)()tCYrHTCFkF3Xgf&+X!Y*
z&T0%^Qnz++9g<(I0R}kWl~IPtZv!%e4(g>?OQNdepH77z7V@dq%5t@KbS!|$8_17J
zHP=UIMisSgtr0av)S3=UNZkCQEUj4`3%R{!THn3yB}>Qu$RKDB1#|T0sA<4jYJr-L
zU_+)lLh!EGI7{*P#fcX;Dra+<<E-&9`FHKtq(emw^OC^+oY!S6RX^AS&a78T!B=iV
z|B*o89FQ4@9ViZ9Ky<&Pu60n$zHEIoE?hz6atxw+IdaaD!M?Sf+Qh#AI^qXK(J!^O
z#SWNiNW-H;CCmyt&y5>@RQN-zyJm$o8RJwTw4jypo~$w&cd&i&D7Y8QaGIA6<DH1O
zGS=E}f2zv1cgFz91ShTBI}ymkRO;NXaqR>|qP--#ZrL+|rg0!f8DE~%XqHZBuzX%)
zknU{oW$vygCTiA|dbu;2U-5hD{Y07O4E|=Kr?qsJrd3HF;k!Tn&3h8peEt2?!4#~Q
zq%p**#&YPRo38aUHR4*dcX>HoK}*Pf4E@&oCkI~?8+K{P%!*n*t4Ze5DOt*wTk0u$
z`RF?(i5JFJB>Vi=!2o+6haj$GsAHhJBFb_(VcE>gw=&Z&8sLP!7Q}R?+?@H8XU6K$
zMk#E08uhmIi1|Sh!uA&bD{b^YIlK<dwmR|;p4Gupj0d)qLaCP)X}S|}$;soRB5*mB
zlHJUK_Nt*YED<R$X-wr!7B^D8r%3OJeB=UgpNA1xmDfr1I)+I7W1~xFH$PNDT#aZ$
zD}-S{C(}GJlsOkFb4mkk$63T|swAChN&r-^7r$|$*ntt*=Q)?iM*YP5wjrYWn}NwI
z3fhH!K{6LgBZ6`==PD{>?pt1?Sd;~(EgnX7E_h&?L7Y$RzOoA%*0~}^j!<JQD)U}T
z9t~_;34<-K4XQBH-lK+eMbb^j5gnA^<p=lbPY|z>YTgMsgwo=9)Q;N{GN<je$t)fc
zey}d(6x~?@hh&L(r_0#*?CN~?w3RSABs7&-gbN%e*y0*)Z;SMxi}V)ANg_dll3(!#
zQA<j`=f3F|ULE)(^;4`4#GfxS-e-EpDuj4niY(YUNdmG1a#2%=FuB)pd5`?x4m34Y
zNm<y=GpWHJWLoQ=uio^Ow5y_khyf*rA&@Yi<B7Ge-g-j(2z0^xKPb``AhyAw{e=kL
z`ewUS*0cpplhLc57_{xR7@#BKk0^n1yNA70t2kmE;ILt0dRjhVCJD(453{j;4)cAb
zrhnkwEL0@(2N~izFxp3$1!1}Ud(~6H?l?G=tA1lIRrtvKXjMT0gBk19_18w)G&7jS
zn75J|{T8;0ek)Ee2jL!y!$lceW-217dM{k`XVZLMfiYfAVtWR6C4Bx+sK#lUzTEa$
z*Eavy$c6J(1ZOjxr3m01CVzp04Xj<7hu}mBF3IBwu30UGmtq?a`JCscm86^!LLh3j
zS75a<L@eP+&~#%9aMeMv{6(R>g4lJu6_j)n|7DB{{xivakOfyhb)j40X446$`+4vq
z$GuI1j50AIf`4)tds4*OW{8v_VW-v0O?{Lo$n|YPj#le8)oU)oi?UnPmD5ttL&bu%
zKfL=Dz3oi06)`1gKF;M8Qef**cYJnQQrB(U(U2coemN4Lg*hKZdWz5O@oJaNW=nye
z^pJznBQgKH_GXp39QrHRF*YF;wV3K)>Q{S-u6sX71Y5Xn?fh=&1egZ(e-CfNJA@Jj
zx#>1)W;095Vxq3m3J<&7VeI`NX(%RyV)^{}uBXBmHBiEUfywuAKTG|MS&ycGT;ef=
zf5Vn%Cgqj`;{Aq$QjMa|SN2&9qje7Z-01}&h&fFW9gC4~KN+vmE(Po~Vtqp){6~ui
zntB@oXGWxkU{3QbiaiCV;7z*ZZFjm`Fu>|l#EL6($Typ!wN)?^7W&D%XM}3-O_N0F
z?(3cHNraqtxZcCN)hg#`IJBrCDzl7m0<Ww{C_Ou?5<hW8WF)Ne6LnAWxF9i5eaV<P
z^l2+EV~EYFi**}h-TSyc&Yk<E4Df#@Gr_>X29p^yHCIIzFX#=12BQQOY()orkPlUU
z;!F>5X;M-$K7*m-gAa=q5?S}{TT2;-soCu7aE>75t_?k#+K@41C?k%AJ9p*zpZOPt
zfw<CC@$de?Z}hT-Z)T2<cG=@L<p>urVwl`paWqeN0e#{E@I9+Rn2zH_*0cJK;%E9O
zIdSEMsIX6GwWW!9WrVZF$mpE!;3H~xklvtJ^_U;ItLWg@d+i4Ix}pCrpZ&@{N}9dy
z#D&yR&*Ltz&fj+f_A4|h_)o9)WK*ee%K^d7@hq>}gOCEwElg?3(dLw?I@nsBKX-Su
za|e3q&hekjo;DrU#UKKJDmaoPXd6>Gk{2Nno-LF|M?2`XFt0)$^ODCT1WJo24J+;&
zk5!LRh?Mn#2+Ux%{lz!;W&)&9>@Wfr0m_b9Jf1(QmiW%|bbLK-PSHS0*7JyHZoIDS
zi}m*R_7q#t=2MDKo4dNlrNHPh1JT+z#yrAFU&9&iHT-V0qaq9_8rgigM6%^@>cPvn
z&cdHAm#^-7IDMfTlKw)UB1XtJ>;@2Y97r4H3*y~`UrxTkY9WTr93P677W*BYH+)3s
z7wHfSr#jcy@u}2md7Er^F}zXQAo8~P$gthdR@iDyV$0;o$$$~iHfX-#`pui?SDpEv
zFoRIN{LjC_?GOE5sQbpw{+zA$^@X)Fe{8UF<iv*j8K6jAxkT4@x86JwK5tHm4r!nV
zb}v!4Y0Rc7dY_NGRarl3xtn=m?usyaL=v<*yG_o2goh1)VcmELIpJ2t5~Yc`IVJJn
zqP*e3!Oq1RqhzFUP~g|C59R}H2fhNrF)tItk{!tp|K>ZFJw(C<quEc!SZYCGcR-du
z&rWJ`#(v3-f8v)@tC=LWn7PTp_!KGnVK6NE39g&WI-y>$Liat@m`3uH5mhU|{X#@g
z&=iULnr>zc+@`EwZ!uDz)lpKCBSE*+0D0vkP_JFaf9h(NLR6TOix9C;I?a_cJ&d<m
zPFaqYNXvJnC1e<B+_t55ix;bR6eT0N%NpSD=}r7lUM#oZJXj8`T6Ebp(xUY{a8p?H
zHP~-{MmnjPL%K~x9IR=%%d(5_^5`&m1xPkZl>`0x#7b{fAYQmFkze!4S-$MwclRjP
z#xGetz_ia_nWWp|wsWxjSau8;XxI79sc>X_o%YNxD3j>vh-vs{63Vum&~;X3k7Jke
zsYQ0{eCJ*(n4zJk+QLrfX<w3(L!CAI492KeURw7yg_rZSeCy*qb&&{{__-Gva?O3g
zr8II!roB|8zz5kp8rO#$2;8qu9+Q9jnoUQ1`ukKdq*K>QyI1KDb~##KcV77g5Omg8
zKJ9|@TM(zeJhzJ0a~Um{r=}-ptC8xYlR9mv&V~K+I}p>n2|UneLCb|y&sJ)saeUBF
zJhF2g^Z1ri_<v&L!44?sx*Xx*3zB;*?|A(=%ms}Ls@EhGeup<ok0uz;XQy+D-6K`-
z^g=7fH^=y5Uaov6xN#D3K8=vg{1x$E#;rfLCI4;QGXMU6HEsccNs{f97MM9HC!dxT
zrS#J=i<WF^JvW95LYqHoRho3?w(z1{k!4)<1PWR({}<zy+gINnlxO{(;5BRPOP&Xf
z-*=l}jvfr4EyDH#cXiYY+<6?1k7B#Ujns4ZNer0kcy*%^#P)C6617aFes4UfxzTc1
zduJg)o0e|nDYJVWHLQuP38i}?VEBVZj8lnmJp6&PQ!i3Qa-7L^h0*WsBzod!{5*-~
zUMG^ymAJfg-rpe4*nkb(+v>G!H9;(&28`o2A)0grUBp)-=&#U}z~p<V?1fYqp7bZ#
z<XC+FXlhWkt%NjOQX8I!QHw(2@l~!i65JXCPCGg*k6f3NFQ6uiWZ-=V!m}$A%9-5f
zA>`*Dh|bvHw_*g^Du0FFtgPx>zcTx;E0g=b+*9Cgwj}D5x~J!=IHh2}?A|mqLw2e-
z(Yd*Eo8vinjTff6n>Z0;=#r5bFP42FPe6g}n5EKL{E7c1k-v9yDz-cWK|uUA;b5te
zuy*LEbB9WTkv!8IX!tfsw@3Td;4m>bh0NseiK^%6U`h_>B}luMIA4r;yopBe29_Vb
zyKI}_CeE@vcN)jfaI`B1Zp-TY|D=)xpbrHXW_Aa5v~0)1Z$*hx_ebJB_bO=65@iT{
zL}xX^!pcN+7Al1&ltWRXjI#Pdi!rumznDY@#q+8SrOf+`lE}VU2>Ii+MAQ<c+9*`8
z2wS8?(qXF+O?E-D>?Sls8`BHMdBLRKfm~dp2r0W;|0qWZq`?W&#a1I+d!()Io=96}
z!Y8d~so;h!jO@gaq2nnB&zeV~vMAqrAv1|kNxT2Nc3@0dk;(C&2YZo=saavMq}-~<
za4nCTNH)A|P$6dFbJsi~O95Dwwoe)-Z<-tZ|1G(CSPk<#-&1^8e7vlckF>b`*`}z8
z!R@MG##qWT*8<mz78QBm&Z>d)7(DN~!ODs46gxaWKG;n06Jy0wbl_9d=`u#~<M^94
z`$z|x#k+Z+DF-zX7E7f^vhnc?i04y2scR7UCjMa~IXw>E{ZZ$lfO$_DdMTht8G#)!
zY;S!b_!PbupldZ~U@{=x)1u2lneh&y<e`aYRv!oy;u3b(StQZ-!xY3eY%d_$R5E|Q
z08-1(HRY@Dc0K-=`xA~`)DCSPip~)!*<cNSSRbxcoM$tjqx=S}&6jazmU%!J-dyL7
zVWWDE<LJ!2Rs;OBBy@0_!L(nC)7(6VJo7U(AX>CzXa{eziq-?z9PahIS(eSKeg=N}
zB#$g6$ftnl1E^HBg{24z5ytKF#8dx_{n(#yt5ai-X?`Sr9Vg|AMV}3h8=<ceOD?>X
z$-;-@%1K7WC66-#3~kt<U??KfIq_4M$xrvyJ7njVP)dXYxmR6Iz4)5_Mb7<p#L4Nt
z0Byn4g1xBrn!ik(t~f9AlAYd|&mctDwZGq)k2+gTur*-49?@@GKa>s}%Ah^Z*#*^U
zv9YlzH@CMyMXs$)S9oIqtcKls$aP)WWqJNXOZg#CUq!VlD__A4Fc}~`-p^KZf^n#{
zWW(u!oaNP1pa-<7YL^Rq_^Uo6?}CFWcnY+QE?K1zUhY}sX+#Qcb#Pl}?!ewB8(GsZ
zDVjX*I9}_m6_TBuutnB3@@g=fz=gTD{F)KiZFap%Q<_Vpx~BG3mJA$dc<)Kboab&I
z0SL*@%pgZGnGqbPi#FM#i47#`h0Qp=k>uqKguRx9-g$6$H*Q81;<?}r{35)}LugOm
z38*MY_Z|RuFj29FU}qV(QC}OxNV7EY)sSgvRY$QP7iAcwUCl=UtcM9r96FaKkyzYo
zn@542iNEs}P|=l1Q5Gz@ZY|IH1dsim16hK#@~w^6!r<Kf)G9D@{@`%wY`H93)xOYw
zuD9!U4}2;{0zXv?%mP87EpPLA8I}by*_Fr6I288)(5<XjODKF=D9}{S^2`rTd+3h|
zsQvRM2J=z_pW>m&KoMMX2i0822vzrxl*j`7ac~L*VNuBv1gOO_!Pbe;WAj<zsj{3T
z*i;{gzMcGKuNs6ggE?M8gvs?m=|wE;&k7Ihnc>T28JFLl-|!8CdkTzliR;GQ`{xV<
z-7bGWmU5A1*l7nJVEUW)%qYImzUe@$)Q|8s@V_mqY=k$)!L6^s<L^boCH|c)bH3Q{
zdHF$wjy@7>HYL1@bXUpD5ncvAfNxM((*y*}XoZ;!=twm|Emp#Y*r~AnzH;S~dfr!n
z_jguW!X02M`+H`tVeA>x6o_H$&Y{b2pwp~I@udETK*}sb-jifsX+z=Brt<aB2`LHQ
za|V=v<te_8TU}~Er}$LH>>{KFcjUKsm<YU1$^G`>FC`}*Qva?~WvT_XC7sKh`h#@^
zO|`Nm0e@j}p2iQ9Yc(EUaVU4JuceWCG>n&nv7)5!JL*tRo(~-fRZP8^-}>7D!ZL|m
zz<;t~C9t2B^D+Bh=@HfUk~UW6Nq0)!^N)5X^o*TcUi&(32A-@aKjg|L=s_jQQ<Jhp
z@`SItOjZVw!CBU_scs(EQT;jU7PoRws5F$1-4LsJ<%fDPE&k}g&vNAlWtM5O0JEYC
z#8f`%r;?P5?fBXEW;s&Nt~o&T#RQ;MLH9!AZ-{Txss<B|%?5(fztH(Ii9crVuXQ1^
z?Mm}5e&TD`OXNr`auOUhp~CK#h(PwBRsBn>3c8yZXsF4HSn}xCofx3I$T+%Y#aRFD
zYgg#7VD5!ihwHkJ8I1kkXR=w`IDbErJq`CdQ|)R_|51J5%R+PBdsLjW@(1;-Ts)MO
zhKlm1#j;`JILfk?n=~0?e=2jQ0fKz3d*P={s{iybW%`fFhkY}4_kdt_zX!czAMZ?n
zlVfotVm!Y^5{7f(bWn`!$oS>E2bWuEe5U2jL5#sS6q~F%%Qr%A#X54)pHVAbM1*?8
z+~>_4{a`bu<<7){M}oUk;9~6W*%P~|(|`?=_}r<x^o^~te^CH~L7zrLFnKuSO3#fX
zhZdUKu62VTo=sC=MC5koc)rBRDpGj4mCm#L$GfWeJ}>2OV2u*_A9-LQEIBc;Q2g)T
zS)swfKanQGn(X!?IoqHyF);zdi8K@vH1egU`*D!~vKGB~1F5EY%~g8-*~?>(9Oe#(
zfIKh7>r%nC0U<@Xz~+M$D_lifFOEX3pXu^!4ktynPco`AU+d(V^0&Xvj{PuD4PIpu
znzb1k<IKFj^yEtT>M31k)pHM^J9koyP;rrtFNhm{{$>?s%kJ@1d(`CzITT~vC9%7;
zmEW92Ipiy}2;c(H8puuutWuLFt}d#T__hizu4$-=;450#FBY+5l~GFML;~i*s<&;1
zstfj4&<W*Y+oz|TIFvEb70n=mI~r1gLiw;2iG(A6TEGGwct~64SKds3u%Y$qd9lrx
z_9+P)vj)@CZiXTLE-)Gm0K2n<s`DR0t^~l`8)h-(<nUj#i|s!^$GGoAH|9S~nJY+5
z?Mh@s1S@&E^P_91GyUd0uwkDja3F?~=WF*dim0q)mMiRg10IMD4B26in1gOB*;F3G
zaAW36yZF{BP96^r_+gSo=S35RR*~L2A%q7lRIoI0FQcficr5V({9Aa1ay8VB4}&cV
zzsYm;0@Tbo<%X03tLQ5w{=V(19ig*#<Z_l&4J3#Fp{r^7hj8)7NM?)&bufC9aO-z{
z;a0G)tFFB_m`#s1F7vCV$TaAWXN`E*(Pv5C>(k*4TRD%_pDVKHjkxE*Rc#}?yL)qv
zMGYPl^#Mi4u`<mYB_Y^I<ly4}n>ldtJByp{<9Q-{%uAxq$vpK%qBPX60-~1}Ho>HN
zXL~s3cy<f#hhKsuzl7}nK=Bl(@n)T1F$xg9{z!Y|zz{>|UB*QJ)FDAeYq}taDM<2V
z-p|)Y6LYuVEj1zw-vi>=FGg@MEo9;S1<`RG3J$wOy--A(-&!S({b$7jw3KNc(M}N!
zVV-YZ2`df;NV;pp<WB&4Sxz@lHlX(C<a*Q&tUNxbP4Vv2kD;^L0BgjB6VV`2Z!B~4
zv&d!j$9L%Q_pw8KoaB@=j=!&4pxFvIC(`1-#JV*4S;O)XK1YMz8#RfYV5*pWfgb9G
z7T_$@pQ();q?^HP1N(TooMC;7jc|u+*!#XdRC31qQauC_-2wl5*nFNuH|yG6Wtb=e
zCIXmFY(QgumBMfG>;%{^l*=8#{mr0vp9}X-gWgx4t2(Gh_A~UWxL3gaB|T+LWN%FN
zR$Z%p#1}|Yy~teqC=CaXQM_%P#6#bztqt3bR))=sC)>}hdFnSm;h83P1t^k;pStu%
z#T#8AK=;~!K#$Ua8;s9@`?oElx)jX|T_=X`=@;I|;>zX-$oR2%9uv|g8P11a-DR}p
zgbQc+hVP#{`QOPMo635JM!6dmfq0QBS?*e>Qlj+p)=YkBm3d}84X~N{KLDUdssk1f
zidXXPF$*F?m9`VnJvzf)ZEDLz@n-rdhq?cuylx7PFr+AJYkRgQP@!DoW}lnkuey^u
zxm2dCobB`TD5WwrK6UkGWmu#(_`J!C@eHo8e|ppYN$~t_X0#FR+^q-(oa)HGYYI(C
zVEW}5)7h2XnbL3VS9ZMfmmWQUI;P2meoY3Bh?~+`u3Ax&0v}*>qt1AwX%PNik_VhT
zncMp!7_vR^H+Z7VgubKXrh&$X#`w(#m!R&}?T6T?`gPTM*X`Tiq(1HBOR1sF9dNav
zWN?EmTYyn>My3wQB@B%7f%pA$8uWuG<7*gHZc34A9ieH;tYBX)V?+y69PEfj396JZ
zCrEY&N4J?3tXy2oO7FQt7a&SwbJrS{PCVJ&2o2q+^v(I3(yjJq{KJIptD-|Qo3{r=
zoEZA!SIrIdd`s+(dE8@9o&m{6m<^y0EDo*$sc?#bfZ)?;3X@-?^k#UZq@a)tI~puJ
zd|*IdSOmA`ir`6F;JC7eP|D)7^^?I@tYZpPEa1ad$p}}y?XI|ibi{mc#~yuox7D3=
z?+q>p_V5Yj#RnYo*T9_&y!z8gWtgB*!&*^7-O4+;+6&t!t6$jMyUg&5ry@)Zo6hoN
ze_{?AkT@LYyyN{)P3;Eb_y4lf#j}6F0Ai$vzo>ytbyH5Ne-pt0crh=*w9OZc!Vjq_
zF6@GGtyR#I%!F6|{P_ZGIf(*>MjCbY)XJ<GwlEv_+EMn+)VEi0_#SF%c;^p`FP@FR
ztpMYQSDN{(`2Ui?WP`zLUkrVqR3GLZF5o7@Cn*V|i4+3VHYY~x=}dfG_ppW1qcF^F
ze9-q-?)3Qc){$FgmR&l)nC%6o_L&2*A&y<j)zwv{QU?TxpAzth5BGsr>?%>LnDy4l
zI{9;zj>7Ag^w<<VGjn$gB?!=s*s+_lulYwEv*jzC{^8|71G!GyG@}31Bh2V)yRYC1
zf06<`?*sM<#-mMH3IF=_9T^2>cO8k4Jb5UAio&QnU!Ru@TSEi2$jn)sz_UR+G#Jf7
z*ZG=p%5}WY>3r9t;(BIghSTptd6r3Y2n<~qap-i9n%JI{QxkT=Ed-TZV&2)y6FQ_p
zKH4w~a@Fya);I%IPXldhVhuJg9nyaNE8ok>VP!el61zr#=k}Fo3kE{ZxJXZPz*EiF
znqf@!hNJ9yRTqUCd}xVSl?SQS@HiVH9Ro{(hV1>14uI;vG;=pko51f_AI#rdBvGq@
zxw<#3AEjowO`88W3`9b$o=S<JE<IaVq5jtx7U-j+L2oGOmvsBn2ejG!z}5oJaDn67
zzx}^|oGX@gVqW@}Kyd;snqvcG;KVQtBc}N*M4s@^&p^OAl1FENEdPMG`85-Px|bX#
zCVUy{VIGp(Q#m^8E3tpDK{2Hn^dG>Vp3_fFtCtcoCZsMI^}c&=z~I8iNEs-;8xj^d
z@2dqb%u??FfgPvnjq)NTLnVd;ey(n~<y<o1ZuL40EohZ@X_21t5plbp2Ss_exD2?5
z^G_cD=d=ss?&!-jWCo%E_b=<#+0|r{R;;=AlK8SQ0*x%~SFHfbovl4bcjjwYa+b2F
zhgs0eSs&8+i-qgjQ3J1!2Or=Iwc7Vr@Ak`G)?QYZu|^v?*8MwT^9O$IVq=&zEAIGj
zt3mIS)IG8FdVG8!cMuo81z%B;1ZO4u-z4?udbcM;!Llravp^LJ>(^*I?tl8}pE>m3
z*@!FK+(5WvH8z~M1llNzbpuqFS1%Ps3{(q(*49V+CK^(54n;sOyYiQ9l~#&EP)nO7
z<Vgum$WM5xf3l1Vb!L_uv)eFIn$Cz6MoMj-9$x>b$(ROShsA$f2Sv(^@3}6LJL&gt
zXH$wRL!$Dg;_x4rVJ^)1@6k%4gRq>XHbH_GQpBqw{(WkKYB!jqFx4Ks9y-%$`|!)O
zNJm#AYppM}nT!7|A20Qc1mI3u1u#t|u<FZrBOhU5bcw2SrKZm0w%AC54KT$SED}4h
z0{;~j`EZk{)SO?j?B~dFe2C|Zr^-+%XZn(I=ZyNMQ<{G+?jI+ZK_)7b6kp!Wm;B_D
zQDr66m5a&Pl5~|Qk})s;?Qvds{tA$OQSgTiOgN9Q^BUsl)|4E*(BD-*sEuKTlx=&a
z!*<e6tO+Ni*UMw)cGW1Ot8GT%JGsL|c~p6g`u19@1(dT|!|#g-jE*O%7iLw4>vZVM
z*|h_%A!t`|v9n0sgi~c!E22AHe$pKt*oW&{4`dNwosl`K>E9QLyoX5{$v7{O;G$pT
z>h+YcaB#c&ydEB8+V`%Z3v6)ZPwSg4_vGZn<3YIv@?F0+_NbK~(oDXjF5*Tz*e5ym
zqMq$j^w7>gol{pK?^a#Bx?|fXh6^l1W3ujVgmYv{@z19m-O;JzZmK&WKh??8NUTok
zb4MRpZT((l*2zeUS_+;uTGE|45Iz#dlH+&kwv#1aqQZo|78lR9ei@*@ILz*DND%jd
z>gKWg)UL_)^_GcYB2k0x0$slGZ4Em`xuh`AeBe%n0@g!a({)*&U=OQtiv_z1XDi%C
zE4}j6%z^iv(4xlp?Bl~30&YL!A*RlOvkY`yJv~DFMhVC(K?QoT0iP)+c5-s)2y7PW
z`Yvam&zr3HFRY8G$`0X^dFP1_)m9z{KNQE06~VXU`s-<%W$ao^1L{`Oc6b;1KDF&;
zsoTQ4xtb5OB~Q~8wXhGkFB*J-Pp^RJ34b1(8Rm^BzB>Ty`i*90@^76g<Y6B0HE43c
zrWk?aSa*>)Sty|JwOE#2(bi&9L+*Mquxz`lF)5The+ofm*IhU$802gKKg)2aPKW9=
zF>2+o<2cj4?Xu9Pn89zxc$fU-NpAr+@2aQDrp=_Y##3pP3Z0p(1cl)aE59z6XvD_7
z{1p^g!=_8@%S0^b2B#z1XDsHv*LQOLmwv0sNnXn*Jqx=<?Z1mh$>FHA+rzJRhlexm
zsHmtu%4&g~&oRR6XmOrAdFvI=_vhOYENeeaML(RaeK7xgOF=+DZ1*IUmG5Pl)4G5T
zG$LUC7qG_5Mv~&*zN&K2O>jr~P!R~D1@mBACWb;kZvlm&|LEi5Yy9dB94C<{SGRZk
z>waCE@r$~YAbMF*A5qKpZ}(U-3ZU*ars8|r&sh4Y{OGtUD%gLm{zbhMXFjZd-2fzr
zasTl{ypzPd9AVYqv_Tac%Aq_Y|16W%U3@Lm;$nN*J5skqF&><`?CWr*yu0|&My)dY
zPWocFAA!;IJ<Oh>DzkE1A+S0v%8kW_jZjfF%U1_DOp7Dt!u<2+aOkt_w_$g0H!*}b
za6)o=ps~p|BP6?xBU_%7?iMdJIjo_JN3EFV`=e3sG*Z>MNcfX08rsr3xI^RjJjktk
zt2aTpQ3zVjeQz3O9sn6#Qj-$JP`I<6hA(|0F)By~9qqZIwiaN;45#1uPT0alOx{9e
z1TTCHErn%ctj7ia0dM${6Z4B`YvIVW{KuOk*NgjrH@AK+@MnbW=OuE4$2xp}Dc2*;
zB4kC+;N)><2}DS!wEws}Y@HpSMoE=Q{WeA<F3NPI0ft#J7)wib(m6_Yt^4EEc<^y&
zV^5;W>oB@;ahzxjycM7GI?Y6rwBk&IF=XR=SeUq`Sl39M+T^cZAnW}1_7k$`KHMIc
z+CU*i12*Nq_Q2+N^+@d?;?X<J2#)UcU?qa9ra?r%&&p7asu4>JwS=}}>TDm$6YY-b
zooG6ZVqjj%c?5*h|2o-0gsdnQ-xEkKwJ_3K^!fSSh+N7az~$NbhVM$azE0*U1>WUw
zp#zK8hL??%86qMl(`k}a^zN7mRRu#Z1}Q-jIZ%LTpbAtY&Nae|WbcdGx>@GP{dhg0
zA9<PnYFAI?x2k@~y2l9On0umg--f<5a#{&IOip?u^TZxQlO8WB)Fdj`SCCD1qvRs~
z!X#yH<FFs~@HB4ESe_irm|vdvu}z!+ZWDo^j2#tqJ)Vy)P|2!Imawwo2Cw+!<X>K$
z_OH9UL0wYp{%>Yu#{S@D*_w49dpq@On`t^jEe{N7+;53dnDhLrjiw?u>beoN#A6F6
zzZocGK3u;!a=F_LT1O`X(S8>y6L5zmr?iejh<LcRhZvomydtVuQ$?R6bt>LtSNrh6
z(s;eKzzvE&;c@f6oEoo8bGFZu2SK9_lfeZ1XxJwqx1blOn*4?XYP8&?K>=l{e?sde
z!wN{5`3mwRE?JrG=V!84naYK@36E{Z^X=Xjn%(qCn}KIQTNqeg(SV=h`Q;@<@4xoO
zn!_hPwQ+~sPI=sFKdP>-0e7BOGpu`|ijLF^UOUVgPqY7af6_tkasRlcDG>=s#bY)7
zBN_&sBW_6ScuC!rv*H8B*<;!Hn;Qx(75aX)u`H)!7F0_kMo-x@U=`Y>ii!m0os$La
zM>m@B3p+8Sn@&NQtL8DC#D;@Vh_9t(&w{l4W`4Ai6i+)?Iz%%U`$ZVXc3|-<utKe+
z%Mdt?s>xtpf|2K->Z)&(psjZz9l30Zc&+JeXES?g4D*v}@JXk2=o{C2H8{C@Nn1O6
zga7=-=5vzRNW3w~qfEEa(-3b^m&F62e}SKJUddi^2JA#L{9@my<EHE%y69oYG#5Pt
z*~0zC((Hu#j6<rQ6qk(8j2w#_MX0vT;Bc%>Dv8G>S;~OXL&BlEZV=HH{#TdU>@oab
zO=`0h-zq*np}$YD7*Vf*g$c%UIFyS7Z~fz$LGjaNed{-`tWHuh$#=b3k(R6~P+X$W
z^ro(gHBUv4m-g<9`Q*QmeR*F=eroTk+tS*VDK%o+Sj^z+BAC_mEF4*DoN;FmpAL$x
z98QxZ9(QHR1_&P<(W2_VRR-1{8$Jk;?eYuv$7)W<18=<^uHV((=$^<lH~(sR(kdVf
zKE(iepU^1(B6&YRkHkv~hX(meBvN;N#C3tk6QX`Kjy%@J+6Q+|@5c%}UQd;U!~}yY
zJc*&fe1%0%9Gw<zdaLGGdrQP8&i;{3PqK${D&OPCfld54%DT7!2$yVNu_NO=vJ7@B
zGD<slkY(g^Uvmp>;hCr8gCREF`6jdZbwyCfFpHk~G1kFM9rjeidstSf_i5<D{9xq4
z-I|C>{S3ryoSv&NRT(Y893&~D+sx|UdfBKCQWtF*>sg}$o03l1%wX!>6PCMFK1Z^T
zc9X#NR-ENTT*J1aG~i6ngYk`x2bB$!^5p2=reoJTH|kHiO{%agqmu4`m|1n1t+}Vs
zWkMT8QCYwLfu6)R?7GcUxlYbVJ-9_j_e8QtNg<-hHlG;5!Yvu-@)`w$O{&7Dt>z1B
zmWzNNNv-edyGYC&a#IuLv|go!#eO4tLw<i{b;n>9O<b+V2%ls@)0^_mPhzq`Sduyk
zAmyk;xB7Tsw-naM`M2ZbT(Ipbs0EPL$Be4hIHC6bmJSw6JcQ23H<&8zW)jXJ$ke`<
zNa8PzZ1T+z8f_6j6Nwi%t%zFDfNn5Vv!v9)RK!dv7=`6}%(Lsc*AC#WJs2>u4*B*2
zOl#u%Pk&lol(?Vhs!~5=`%p)-6zL>PkeI48(M}u8ljoWZQ_Ej-HT8?X(}?Djhp>#O
zw*<5BB8hdvB!L+E*&=-w#N>P5=d$DF1(UU8;LLPPxNbjBX7C%?bsxWP+u*@w)bFK`
ztJhmj_+P!fWl)@L*DaW!!Rg?x!JQz%rEv=q+#wL$-625ZA;A(XNaODA?(Xg#+#ROL
z^M3Dn&Y78-e{&Z_QPth_E!W=H+G}k|OR6&?v`&vJ%p-l_ut+!q6b#5ubN6NIe~IC!
z@gTPMx5FT)_*=jC#!Yy(B29_Ru1OIG-Hi}V8aM<8^A_$Kp|C!K?RgSmEb@~YXz}w^
zVc6{Isw&f7Gf55ym128`9^rY&VAa)q*j$5PgXO$wCN|{P*O|n?-d)yjg4&Z`pHzKE
zFEWSA)yV0*wPsQ&96?HD^QpljE-Eq_7x&a3IkIo{z|W04ZyY){`Rzy)vnpVPmb*TK
zsgy{#X@N?j>Yqm?%=nppC`$O1NzC}3OBO+H;ILr8qfF093I}pC(9Q+I#r64$IPwqJ
zLm}h&xLj!C_@pX<@G`A!vQw&a#}yrUn=2am+c}oubqcJfbm=thoqXFvw_m^_XJ(W%
zRJ|DHam4u6V4HbZt_K3HX@s%)SO7Yp1CXr_Nt+dlG9qsb)87EU`UQ#SqOT{PylG--
zc)PFFcG&Eqd3mIBbN{Q=Brw>_z@}rUTJqei>lk8u&(RdLv{k;evEkh?ZO`laJ-MLI
zq{*!*R2bY$8*}RF7RNBWbW(tSOmfZh0O4nc4u?Wr^ayZwIYo76NkcsToHed06qr3p
z?(U2A=D1tP-6bbywQ=RQ#fjENk%zS<CKPC8He3+KXjO&--b+;$4Y#96l-WB5R`V(k
zjsR1zF&F#iv4Z0nA98r51MGx2eAPPZwRa?t6OH)0a0{m7>lYjQ`%dkb-DD?m1Zm;R
zTix5ht#hnI7QOx#ek0!+s|IUj7_)m}1_~PJB3|juV@K+;`GzBVZ2-TehjsZt>`68|
zUOli2JO<K6x=C0)3E@CMEBEi0e3xP2gNN&Cl&^5II{`I5aTSJ#As<-zMM!_@D}%Dp
z2G@48;3#F?oX`;6Th{%v(uX>piysIP-<Gx+`cn~m*0$X^Wtv?KFqZb!E|97jkYQQz
zQ>Q7l^3a0DoG^EWkww~cmFCoaz7)jz3ISYa6jpPulzTx|{AMLwzo3Qm&`xxn=1)}U
z=(%Pu%*Fyc;61FOPnycWG(nq@Exs`~b8jStWf#J2hmR|riXFpnOjhXL+tPmH#B3T$
zIp|oMenvV;&V1cz<~M20aXt6=Z4h|Yt?q2{b6HAsUHjmDwvhE0aPP;+im;xrh_=J7
z$GPLZkX@a6)-OFZZe<(N7&mY>-803R`wcmb9$@_=IZQy!TR;416oSTtcE1@8;<|$O
zZ!O9fM6a{p)+=6@oJ1{#z|PBR^$m9g)WP-*eHswEdTkwX9=!hwy!4gyb7UP06!~O<
zIX1EdUvS!eJV`_nFADGc%Id6A>W}CyPKG=$%}%2wXQs!q)e~l<&b!WA|9Y;JSm;oI
zawTwwJat<9Rbpc!5qXK^4j2w%K4G+OSj-e<2A9ZQGl!KJ9yghPg8zv?1)2<mjp;_A
zN-s<tWKiG1$%vr5XcEgeLc%h<)U0xS*|t|6J@>q7!b|7|Gm1eLj<}Oxr;%*byisE&
zaGk|g8waybo&B607u<nLojM#F{WIudOYFk_x+`x$C;;b=SInme&hKwIKhBloK|9{{
zCJ#P^eW=NQvpS1BkC$ook-%Hn5tC(a-ni$jmGzU;Nf2w^ioDcZg|sH?Pvy;xFRDh8
zO%@QfonH&@pi$ElOS~xwa4cPee9?>EY%2DajZOwQM)njqP0RJ8Aip%5{M{l<5YVk#
zzcpaDPTu;727v)!Q^*OZUcwVC|5d(&PWsw@j#SrW^}(bNfV+#@igfLmB-yEJ<mD(>
z^G@AW4o`lKZyv>iNQ3H3Y<n|@($TN_-OL~3wXdhDDrFvW#7n&o+LO$By7;_5!@N{G
zluZQIZv@g}?okbBl-l5O8#?Wi14cK)RHYma3b6fGzXz_LF!L5vE$`;*pG8&&-+ri5
z)~H%J_mH<|5}CR(=>r_r`bkdZ9&bkc$mI8Cg_JjBvVFiI=;pEC+@&shbVlLW;zuR`
zC0sq4K9ANhH%#K?Q0Q4!RjTjw+3b~vRkfMGuOkOhyuLBDMN063CqK}nIlEVqnrmQ_
zKTeMsGBJ8{q8)yqkw9hG*Bl_B5JqsC(>=sx!T)kIZM%H7w79N32O8Pgl1M_vct{GO
zwo7uLW9Oe2N#>tZD36a@)?P6Q-JrHOj@iP^x^=Q)C9mx>kQ{fv>Ow^oJF(u@RM)?`
zU>q&DS6uywTKWygzI3IzDb!j12d`=Ko=R3kd4Ll+w~t}eoE!L+p1@K1j5v7KUc(8|
zoE7ri%;p<cVW*A_vl1@0>A0^3o;<g|AI93d>o~_4igY1iM?4QTUyHb1Ik~!rle({O
zKNQV(5|onn=zN{qdV}@Ay-c+1ql6Lp>)EX-MEW89`n~+Q4;k&<BRttXZ{NK8Vt)ji
zgdP7%Iwy*6`a$UF#RvR%@M{{tlocE772Ta6{qf^Yw+83cTdUa37PtUbIz1{vZ|lsf
z&JG&@ckpmfm?pos%C;`?>o-xy2&-iZnp-p8l^KF+l1-Fk7PK7Xqn(-x4rUzRw;5X(
zLJ3YKW#_|BKwgR3i-CdXc*@z<s*{k9>gui<^E0w;$1B~b{BFP13zfCCk)52%MW_LS
z1%WWbfp|SV%)+~$B_!Z0Dk>IiX4V|?YVM)kSSm5k@pc<nnLeUH$8F$qn7)DLr^FLd
zPPiH?Y;u~9wDtGP<d4%>=AjpL-6N{C0r;C-{xGBBPIvcfUf23jWYA$Cu$J}~B$D~O
z{op|1zglV|r|e77kgP_g<ccCQIFmWFN=CIXM7X5kFn3Mv8FJgEpYQSeObk2zQpC+8
zLD}-n3xT^(Bo_6Z5fMQ69laDU1^+7+pj!FTybI0n?m<WM8bbE|wL6%F+kHF8<>r2n
zTQ>L#YOO56($C&FnOD1+YWcC-y&J?K9m+@@qdrZz<BRGh%T{FTD9=GE(oEDwyZgzQ
zj}V5UbKNkP^xWU_Y6j6gBRqj8)28{hGF4^<)A|Rx&3N~~$2xN0J382zvrWuP$v00}
z_|%79*uQR(_jya2YHkoxA_m)xAaw=``cXGlZSf#-;6eY7WB8Qb7L6K9^|PC8flNH<
zdtH`9ojl){5~8Z4J5#o$TpyU}{b$KG`;c`Y(B_l){Ffk77pBqQvuMlO9@8Pc-Rv9H
zapvT{rwG|-sYrImPkWhaC>mF}Yx(^nY-$6PM0%!I3}$v77-^k9Fv&{QXt%`d{=f`H
zpGmgPQR}{S3T7JdpDqv?ZdiN#ZhtVw>@k~J=ZJczQV`=PI$uCF0kET>-+J#^a?x8T
z8h~%*^7d5btmK+ZLtn?6<{;IEO|po+#e`Hc>2(4Yaexgi*K2QC<O(T_Dbz|%hn{TK
zWo2CKKp&al%#c6r&>c@YB8+{zyc`k0xBuE?8QLdbVXX+FAji+SvE3Iw7v+?2gwLOO
z^_G@i`h#VVC|ANGcc6ptuy4ZE4qZcLksM@+>}nK*CKowR*AR)9DZNUZAe(3}cb`T#
ztN~+mFI0w|RmL~}JJ*-AWRrG_l)0<^GX+Q!GtfjTAY9<{>V5S6D;k!vV=q$r2|2#U
zkK9Fq1%ehn^j{1(uz3=sDCrIIv-n<!lT6oK=m?Fo2TEWzf0vhbQ|%(d$gh>EDo&~r
zjKZH!_I^8WYM3K}%+yJvDza!DmOCZ%w=-F^`s?>>r*-J3XOKd8WDBA-?2GaBhAiRl
zcvv<V<j1jk_7+C5gA(sMz2cowP`a`ms#%ejl$Zt_lxTSskAj8-bsUbe(X0!btK=r#
zsEm!c5lVlmPIssM>3Na!7E+Iv8watl!jXi-r0>J7E8LYLu3dGeKS~OGSRz^xA~R0B
zSPd0gnM<Plv?DdRc-|^6(vf(+{;2%Ein+fp(%t5+>>?>NzjWK!&ampOz%GRat>Lqd
zYQ3`(#e_x(m8Hy^C$~<tqt3WL_o!V-ZUYVn)zfqIhmD(CB9Vm4GLuyj$alghy?zg)
z?T+(6dQ6_+EI*|bFPnI^RwRDdPKSaOyiay>O3Xd%tJM?dbJQ__L^MLUbGwnv9;<5>
zQnhUB?iyx6c8B5jb3*xQ6R-|5&XffKl{u<j<Sncd6B8#FlS-Y?^4*b4m%TAji+7YZ
z${wUi>FF`)eoq9huC99%;JZAXf-bGe#WFHCD(@IH@15Db#Aqk7Qn90b6t*2@jcL_G
zi5p3$o2Jb?wf7lBNOKS`(i(&NHvO>Me1{i&f(EUMYQ|h8n4h1gV_-1maiL_UVaX1Y
zuj56u9u#%ALhLBMyrJw|B3^E;KCFw$EJoOX6ha%|J(2F!P7vAFNXjyE;X=`ELBLBj
zU+fWdB2u2sI{rU(gJv^h_zrsDW|R7`UwHrF8qH?(E5{c;G=5M%H-Qp`&|_+dlz~b=
zHl_-TfKENUBIP}GinNHMS(J8hMkT>%T=AQjzHdC;nn#@s9zXUnURERYur!0Zv+0i{
zH<*>HzGRG#ZlYK#GT||s=TUfL%R|@&%$297NYxX+1)V9V^y3WjtUombXOq;Ir5a|%
zCChkFgxk_BW;;Rez>5&XRe8s`fxzbT#9VZLfq`E%)z=W!MfUu_Kc}^5*zq;?=Jwas
z=k98LDt*w!s0M|lAB$`}p_VVxh!GvJLLA3{a@l9DndU1%%ZV_k&`}Smf>NEzd)*Q!
zkKW!Mr{icd?zTSR%Xm7y6|Y&lMdv61wq$kntszpjgIh0j5CzH7#a?%_?gE+(xi(l6
ztp>7H|9f5`Js_fsy&UUliKwr8rl<rTac(Wpiibgo|8{VBEo>|CkM!=A6`ZK?6@A}K
z<_eRVw*}qNmBhT8)GkflqwhAhnQsfY+Yv_wext(NpKvDIr8=y|baG?rDpSmLP=rkA
zJ8P45&cGQTEd=xlIxDMMY(&Z!G-NT+!QFqiG~bq4{O^5_;k0V7?7hF)nQt|zDOSml
zfI0#F=pxWEuQ?>ToXV5VC0^y5;}k*=0=>jfK;*r}Rb>^A)OdU$U2|Mirug`a`vG<5
zI&97`W6K<ZAztN^1_eL<V4?TBjek3u)p?joe0GazLF-{62qzk(ij6CsidB5!l>b>O
zE;G*4(QYTKO~Ek+H6JajB3#Y7E{(V*$&N;e+iN~MIMOt#%#Z)5J6ea41MBLjJHGZ@
zxRk`3wB6fa2dor6fEQtg(7g!M`@|=5h1<_elzA{ZB^licq%7`iflNa^Yg|qRF2S^V
z|1%HlCl_>@IQn%+?P%!SyT|K{7#{8%>3I5dlRr{OeRHQZQ#1=alv<UIGh-|48~>X_
ziIfYtP;7gu&2h~C-lB}eQj97!^&Ru!xCaHFs$x5uwe4xbwg@HekFZVzsCpMHt9s$e
zBzm@CZ+&hM;nX_S=d+qi`I)iC1`-x#mD7GD1ODzTrKL5mXVbFW;IeOPVF6k0=;&x<
zi2uU^5KP;jEgh_#`KYh&dp!Y`lg)=LH+iT&6=Aj-Y;T^>`knp#S!waPKbkJoI9`Dk
zpO;KZ#J92vhR8J@*=oiYVS`AO5CKCB?Dgv3eG#R}etj&Q{s!Zil$^}S47{L2Mt(o*
z9TgR|oz9$}nF*t%r4_T}Qz|rDq8F5(zk^(pmX`ML13Gl=J8v!%1A|y7I>}hGmm9bS
z0*Z8Z@@^F@n(_3Gt+$P~-5Pe$%}nShfk1--=c6Sp^fm%0JW<vIzIxjFp_FMgTT<sD
zS2?AoYB@~a#AUII?~u9k5*s?}D=MN#Mn=ZHg^D7NK#by?E8fZ+tI8d+L51uWPCRx5
zOK*bTb*q;IYSHo+DGfgNH7b^TFH<C31UTPOrY8RUnQ#4K?n7f^ALlLkFyaBFeNqcy
zsMw4WB52dsNYcEq{#!#{)eG6voE!8y0MghD?1d00DBR}i&i8q7SA3<-W)vXM-Z(Zc
zE`yFj?$a?P{hH7Tg<0#Pkow(mM{`-Z{M{(FkBxlB0tXRU`bWmgg{*g+8xptG&$hMS
zs;Ka4u&kktrR{P088_`E=%)*tFeP~6Y$iT<`aSL&Br+^&%MCfw_nd)jfzYR{Y8ket
ztfu&piiXF^Vx#$CkhS&wZR8;($~a+V8x|nebz*k57fVSXBkq^v<En%w+k295`c^Mo
zf((MPQ=-?L^(&Y?%~9jvLiQos6tRoLSo{6^FY}(|46;_<c;S)mrTWcrDo*skyWK*E
zy@5%0*D|DnC&ybU%+wRqo|ZX8<L)EqW+EZ$3WLtf?|&m+JGKk<WfObr+T#1BVCW06
zwlBU+uu_L_62S$ll!^J8E3uv*w)sfD(Gw97e*Ts5ZcaqsjU$h7;O1{aUVK+2BIy{c
z(7fBm)5rBJUXLc2gw}Gg*@o!*AuZsOFDnb}HU;AyT+8#Q*3#2(X@H(6sKGj-y+$wF
z+=KL9|HIeD$+RFD(YIIdM-uC$zVL&zKZU2c9wOexj=T)j0}*Czb8h&H#x(qp>0Y(#
z-c4_b;c>(0Nm>13(2+k6MN|YJgQnLO;^rNTjlw`W9mv2FwgA^W!8MDXu0-toznrE@
z>Nk|;d=xhR0is9aANEKDApMlrg`WbF8cisc?%%B^Vb`HZIBerzfYr#h11mmBWmBz)
zsq|S~=t5#Nf?(KZ{!nlH*wQ-|F8N8Mzs!zxHNuTgMdEf8?SXHEps8pNSCa}XsZ;t3
z48Io7asLY#Dm?$no_R@Ea@O}s%})S<NEKp81>=fy&hu6!mVLR<{B+CU(mi(9J5Sz&
z)xF*y<DWX3N*wfjkaLS9QNFiJ2#m|(#(ytOKM-lSq*R#>ZvGHnq8nek{$HfH2kV9X
z7b)s(t6Qw5(G?$D*{&icP&TjhBM4re+mk@mjV}}<%qXH=ftbYn$ZA{k_bYBoh-?l4
zKC9=_vHFyd&<dkN<wKc$Inn#?8Fl3XZ)I*}Ng#00*fje*2>^wQJzt_C-J35^4UKJJ
zJFPs%YLqvIlL51#)AD|G!cK6D3EdeY`8Wg7X<aTDc=T{3P(RJmpmV7%TkR6K9`(Io
z6GxX(60cG_;3+Di4q5f$D|^EgXZPCn(~17O<qjBj3*AmUM@yD(vq;fD?#Z1-)l>K_
zU>7Y}<0mebL7GWX?N&Q&<G_6C)Uk}BqJxr(;`pwl34XUDkz)0%sw(WAogG;efFLq^
zi2Ro*2UFvc(4wLu8jKVP+`x9;X(?|%Q*-mIygcbAMT*RscL^y6E6t7G`m<oU%aI=D
z&7I+71tASt@l}cb^%c!q11`1QGeb+efYoM_fLZ2fljRW3I`Bx>X}Uc7ZnN8(S;kUE
ztoDB6N7F7H;|^{>*^#Er*Dz6Al`kowH*I=s57Nl`{Pg_NBR&tWqo1vVR+Mn5W~bmw
zhuwg9><NoJL{yzLvZ9xQV5H=-cX{|q^9h@L2*w*6JQwPR5_P>2&Zk_G-H_xmU2Y7j
z^w=wf`DmL6Ke0py!Ho1z=!4xtH`OxBy*Q57Ukw*)6wADGI6d!1zcuC#nsiFpbzcUn
zupJvQ9Baf@-~jmV+ML2FFZg3iUNVfh_Ke9Dp)-${V-pe(pm~6|U(>Bdu&RX|Y-v}L
zU;KV*s$K-*=0-<KNvdj1VZ?;+k>m|2(alilIIA9Vmj=c4B|QE)2#=E-<}_;O#G_X^
z|BD#U+w8VS0qfSQ0H5jz#O%hu7!*3=$IA&a@$SR<JN<KLqSxM7M$m*9uo&zpL42A@
zT`~E1cd4qaEoo)im7H8+-bE&<gVr^vQQEcsC^%*Z=2^0iiqoEOdEv_u%%FHyMppJa
zr!6$W?bEzH!X~g-DrFIA<3W*6Dc-AymycT<p@Q^fS8y_U=BJ$S*w_l%6Yc76(d5*G
zTc><wj;T1+z_1#+AxOWHydk(Su`p2t0cI!v-C;}+{0UW{`<3$I_%FPkg{7BQUQi^|
z1K|dOr1`7W$yY21PQ$&kQ~N~9Rf;mzDQ<xrY6~^4S|-`n67Il|Ci)&nGWGssAYTL&
zF+&7iwv#+|Z6T_XkUwpqNsm|envhnMU-nhs3H$e-g8h8zagh|BtF$GQeGd(`aKF25
zb+`sbso2tN2h%lVSYuUl{MYIx96scOkhLNSu=Lh8D3@tQ+BTFZW0SI~cvJ@e75lSk
z=0gDqzyN=wolM)crNK<){})sXEC#(?a8pehVDy>$BcB7RWPwUaafT~p=9N57GYqG<
z29+O(EYPiol3Tsu5H(~KWc>AZq%C}}=i0$GihTn?uz_A@S}ogXnPH8Yj31xjlXjs1
z>}doKxy_!uyVteUW&C>Sfl|BbkJQ4)Xq9h*2cL#k&PWA9X(@#C3+~tnL(lcrDji9i
zc+e$9`v4#p{f3zH*ee}SJwL9oX6__!?n8Zq;=ak;FOeZVe$4t%7E1<&6n2Q5TO53_
zp~3fPXIAp!e>}4Z=LWR7v^`Qjs@FLHC(l%~!1gGqoDMbYc7P>2NY>$lD;{?Vn#7kp
zXPs7@abdN=)g}6_d03rt*Qgwy!yY~9yflx(tVehCs0Ss5S~-WqUO3`;`sLj=c<`Ah
zp}Xy6|A*+|2_Np>9eNz#f`ShXencyK??j*B&68}@+$hW%{~}(CUxZjoF>4@IV#h;r
z_Zy+fy7p<C&odX1HtAG$f8aE+@v`dVqbfR#^BChKi&g#?LbrTg%pu>-LqkAQ3Fn}~
zzn(lWx3G)OWYkdhq$iRH%><N_!vP|cHhYS7=)Nz=O<o)4wyM!i(-MC_&ZJl_t-Z--
zeLg1jA&S)cha?DF-JA;lYXer{k#*-@%D<Q*>bk9aCNk*XT%br6uqdTWM&qO{1U*99
zoLHNqKiJ)LWck7Sg637ZggB`@X`}I=9bVK7ZiatF1C(&cEsov#KB{Xm2)JPOehR_A
ztH@a{R-zyAy(alicH&uovMhOUXZMz63p+D2;KxBo+G5Z)h0xu>;|~#lMPazJwRQqh
z>ARM)X*r2GU4x>zL?kRu6nYU1iM9(ch`uuT_${j`%i`AyNw_i_lNo3Fo0-ZW<po9W
z=54D3{%g+Y<AqQZ43FhvoAH0`R{cK@WIszOZ8U_UFvUCCX!XPIS<DjY$v0+1^a9;W
z&k}8+VBz4ircAn(O!^btV2s4F04h=OQa&jd(g-MLBMO=w%AdhW%iDg5qf%$LOuEqI
z4f#JKG*$3yj(LDfaHriwPm;G7%*U2@5h(ujS_0;VovnR)u}b1?5-lanFKcos=eqHX
zR&6DJJ3r*Fs{m=1hJaJFzV>(o_hG(o?zpi&T@NS$lN_7w7qW|o?7+JWF@?^-yZqCf
zj~nH>nBJ1~HAKO6C*l+h0{k%1&5a_ZSx`yvL&t_9MM*y_j-QBj4k2!kfxiL<>{^(u
zyRD|5v+AxW_+e3R_=n=g@WI-g+@!6^Git*!y~HW&ud4+FX+Jz+Vd18e-<;zNXo9tf
zjDk+Y{o7`2g<02(vFZ(&L2SGE_;xHdM7)N7?&Sbg?8u^w2~^G*jTiOqnDkmI$+zzE
zF6iR&;;9M+7Erw4SjW@kBh<wLX~pnqDrW~1At*gkJ>BH&a1W0o*mC9?mCdw-UK&Bz
zF;oo(yI0!2yu(j~fu#CB+9_<Syz|L=tE){Ao%fF7W;F_rP11Cb#K8X%ysU{wH|<x0
z3$W_OE2H=1PayYj;H|kW?XEYl+-f;ih3<QcH5Xl7T$6_P*(x$qvj42)Q}zwKPPS(<
z26tai%yKCc%s=^JV@rnA1LJl`#!u^^(<kC{&&FlDS*LBc*tfQXXw@i?H;bx~`Gp-P
z{k&HBDKSBQ=iNIa=Vgyyn2cLazWz})fJ?{5$P?RYd<`P0gYi1D`q6Rnqy6Mjjc;ZF
zA5wqRu2LBE#+^r{A<+<R)DOOMQkJEPN|0@XOSnmBK%QQ1%2G`PBSb|PO&|iv?xwv{
z-*%j?P~_*7?U;<}!Ll4*SnCz+_Xr?z2~0uNARRYbaJPxM0imZY#z#-w@mv(cO%5?W
zn|Bjvw?8fH3BN=PB({t@9oZkn$8=r3PO8}Zec^iAMSgoe_^Jjr^;^R2$>TR2Dk`Bl
zS{T6k>e^Vo_zjH!OzG@i0dw`JLGbFO3bJ-l#Nprxx?@l|-2;AvbK$zZnh-zZ{L`u6
zkp4F8=;x;-!Bu4fCEGB-AS^q-bV0f>;))Z(K=tE}jLSres`$?OG`U}7`8Nh;nvPwP
z&#sM@i=II_?dD;XJLO^81W>|g!b2eqP)a9ycclP2IDp3+_7G8c^eI2|6FoU#OeOt8
z(B}D-F~zpv_uE~U+wvC`{Pj~>*^%|DAbs?c3;T&Br7wAU>s+%KBkD6{+=--{*3=1P
z=fE~yT6~C1Zi;$Z5|>0&9w()aFG)dV+iwbV*+q9aU)-;!KJR<!M?Q%%r~c*xZSE1Y
z_j`Q^1aCSg>O>g|E+Ne*q~lL;fa^|HC+lWxUJ>u}#k4q~^~UIl1aBLnDsy}i^gRql
zdW?uI`^+pRK5IORvCL*X!8`Z<w$>JQR3R3S>~Cd8#g<cgK}kOG8IKZh^NCX4K>KT1
zam3mkDBZvYDkF~U-*Z6eyoL>tDaDr>;M7@cg44Pffn$XkZcpTdw3?o9v;g}8Mieay
z4vy)B@t;*N4zgG!*)oeN_V>S8`p~L13UHOeioOv}^ye0yxlatUiYdm)60cQv;V9dA
zb=YaqWhxeo`S`zemy=&IYp>;f(=v%i_2phPd$r1c(%y<5E*CnC#BCikfH@jPq9w(x
zh5YsUxN}|y7azj+n}IXY&6x4(c?@2h)}jX#Q%NuRF!Fe_zLMlmqx|fos{0{u^Jfpf
z`r$1A0`re3T*^HqIq`2%_*?!zqA<qlJFrb8G;Gj&;dPz$RZS@@<!5lcFq75c+HPOB
zJMQ2@sO@8r9y;P4-B<!EC)E4P0=i7a53dO@K?<KnmFjQ#`xJ1o5;oUbrf(vWKXBB-
zw{<$PqniyXh?pbZ*nrxv{D#_oCzs!tM)P~O$`*SVFGk1f&jtDH`tQ;xcJ}c#d>sg|
zkEWOO^e_~e))rt$kz<s^K&<sL>FSMa_m56=;H!t^$Xe~yTWqc0jH9sKOyi@Lw5Vgo
zvf!^Fuh!Y{e!TUd*)rP@S$b_c5Fx`@fnlhV!m5i%w;lS@N|TG{FG$noMv0AyoBUbk
z+l^8Z->w|o*GuQA)rs`8FRx|llM7n)tf*-)DhoAba|xWH49#1m94doRn3^mBz{#l@
z6DeWPwpZ7gwdI!FwOn&%NNhapQgb5Zg{nbZkwFYT3nXxYtflkH#Dk6Gl0UMl?s=fE
zT6HmAWDa(8I!Pd39+jX}0>g8Hf`RbY+eZqni=_fxf#6D&0|R%@LbLqJu|vmtUKf5b
z9<sy^XG-QA^9TaDqVn6(cIoE)scBbE5~D#%qh?bCYa)k$AJ7(WmbOs5ExlW9joSg|
zD$exn=Qq$$$J;nv8Q@+8IW<Eu8?;-56EXMKUJ(~nG#5pNOjmUmUAwww7T6`gi7sl#
zOt0uWN?#Wtj@AfZB=L~>#u}wC6CNix0>6w6-eUJPo)vq4^6PM{ptb8OOX_f^5o9io
zQLxBoA!!0U{lK9g8`qeNhw>v12jRB_f|3eBz0GQFI3I`}eF*Ma1)8I+tI*O8P5e8i
zI(c>I9z~DxGWZ<u@ob$tIeoaaxLrmtc)d;+#IFSCdbX!3>zHtdaG<ygDxy^8CvpwO
zU8a+d_V^xiOSF6$8#Y3I$eEo?O_i3zM@4F*b&@MvVti%P%MrWk$Ja3vY~<+&aXQh&
zhiOWnypP|0gFjIhKv10U54RL)If#l&wF!iI8>dt8K(RWYQbQ>~DNbL72EWTXU=@yK
zf%VVQ;6pKDRJ^<BSmPfjQ56CO_Ahmxd>1>lt}o0+62L?3q@sPp+0)RmY$u4?s~DQJ
z^(iakif>P=b3Z-fgh3y}fL(FPg)gBue_X-TFH#rkQOKFD;M;tvrnsr_=T*hZ=M*^I
zC+?~Z;Tp}UzJyG1Ya{QqR<=!iGFH$Z{}q;33xoMC;n~8RC511-7$8D6pJ)0#J~>5V
zL|o-N@*n<g-o{Gp<hsM@q8QUnZX~K<IY28jYKB!u859#D)Oi`2D%BH62ve`!<dcz}
zb)A}1_mU<mwsC>)(vyMlk(=fH6V0FnM3h>IIc#nW8sozgnE_pbA267Nt7t7o$z^@T
zD@tR0)!vf2o=Rl@<@DF}lad4UD0;`WQ{%$Vte*Lo#3rJ3=M>A72W;9uQI|$`;(*mk
zq1xy2$C@y0tx%{E+c;38-r#l3a<o(rTdf^6F`<%_9MMe;rO|M(zo=im=eE7=76jHV
z|2N*(gMo-b%YWES`!M(RXIr0`0M$)rXO1Z!oDdWhuS-eCQ*2k7!bVd08NoT@V{Wkx
zW%vk!94Dm)Qk{oWI`~YpG@AzJoG16AzM;K(k7W+4!7nl@?Ax>yQ2)wf{B|ex?KtQS
zBr;zv#fT2)MNQSXPoQ$rAsvtpF>(a)dr~3s8wkf7i95x5?BSkl0NtM*Fj1jw+HM8)
z)=r|h-58s^?oP-`LGv5|JU&fIZ#>iQ>oboNVl!$%MdD1{G)c&hTKlYhdv_M1s5BRn
zSw=kmRr$6{CzbgYtc!8jpq3QG!Doc^lOLSxhZJ{@FY2nw*m1H&S$uS(lM#m}>(3y-
z=m}@KHXc+s&OU$9ij_^Odhi{BNV#Z|^Nh6#qvHLZV4uoN8GqTg^dd5jiaczfNfR-^
zS4;=l_x~e2(7c(g3#Y8qVD}pIIw4N`dgBh}qrmf2#WQq`<GJgmS%1rc*~|;8UI@eP
z)+<(bE!V*gke3VJ7yz$n{eK9nvqNa5M@dLt=0uUP(fn?`0+DR6Otic(i;7}LpoH&%
z2jj3tj5YrX)Kt8P-6U*~{=x@l{ug4fN^kKemk*o1K_xPo-yM95`#TIW-=AYch1zjc
zjK(}8S`8)#poM3I|0j{v@zY9(li!>$6*m*RSr*2DGKkX0Nz*l62khgOSyi#v=tHI%
zzU53ybTKvB>GJS(DJz(_mI)4kB4&R;_c1p2z-!fktgQ?F%4d{H;*M5HI1bG!#!^Y!
z0C^!1wL5A%R(hi=`#<s1e^^WfsI&^^Zy+GdbeJ6BN)ReCyX##3Lq{nb8Dv)WOLBf3
z4%@v!n!=uogztAy#n1Tfp|r)GgNr2Dcfa^DR3TAN_G~Necj3=BYv6m0BhO9B5|_QO
zG&adub1&l)M?YxaKwG1Lr1y$g2djR%Xvzjo3gL?ODG#U>EsqGhSa%&Y3&K7Yp4i;Y
zUO3ROC17*&>YCOQ^{=;wHW$8x%#~*e%)S9s3vM*iuVgV7b8&1yiLjHaTg=AnXQ7MJ
zo*wLc4q?Ld=S?R_&8roPwho^6RK3ZGLQDo{L9>M(bJvODvL`8_*?Mhl<<+y=bilh_
z1C!BJA*qX~Rl_AV@SPCsX~*I=uaF>iW44c58Zjs~i{Ikc6BYbpyMf7m`p6~(Lq872
zL6!HZoOVR%S-xO3QqS*}F!#@}?sZRFemAd>RqBNRH*{2}ieTUCoy8Z$b>tHs27^kA
z=$4=S>4rvW+l!!*7jIv|O<5p{M{7k@FbbASG&D8+s^9D>$hIg3WL8Sq+}_&yot+K$
z!arH+NV>Y#jHIe^+g8+Q`$l%mu4->yzvj9FZ!SIGqk62IpNehL=FEOP?3@KOpWfG!
z#D~*3Q@w69(-V0<S&i^m0xb}my=<8;HxWR)$PkZQUtfRLP3~}}1m&oj5WBsUn`pbe
zD!E`bYB(jEqW1PZ3@}#`z@fg8T&+rC*R4usn2I*IP%~A7wg8(V4Lin&ea-k+waw%f
zah5v%FH7mssDl7~+td;?m{CU4m3mRj8)r17n(9v*zvbB&%{oJ&wZJL)3sL){G6ySG
z5vtpQEDc|bWufuv8{qkB(avFKh~Sp^mvk}#KE7e!Hwt$MI5Cd!@CDC61Q4Lf7gP8R
zX#hM?cQG-TAo~fV4HqW;(G8bkg})ILmewn1=h#s2eq~OCJa25tc+D)=Uz3C~Z%_wg
z|G!YtKZNnvWn>+`l3cb`)uGGq3k)Q!#QdMH_@RwP#4Vq;6)wFPHe2p4Xtii+lfsn>
zvV`VJK;N_CK=J}dp#L~_WHSuf#gGf3_b<KopKDs6I(>{yO^wRTq`c)125D>bMw5&~
z#i`q-61hYcS)0M}Zs^%mGymle^`O81P_^(w^>;%vGn(|1h*$m~?L^*Mtm9#*Hq|~d
zQqmpd;9wNJ8wZ?9`Fnr=-{?z;l-00fxPK9t|9U1bpZO6NM{T|Ytt2a~SV!#%(~g0D
zN!M+va~i|7rh~I{+`cyQ--kH&91U_#>l4Ei1o6(#&dw$V*#llmfh_gnkHt^Zu#Xi^
zNo_0|TB4yBA@mO$Xp|eG27N?Ru2nn1>?6=>*WsxDvW-=$R`46cv4H2>-Sl{7N=oP_
zrqJ0?M1jDlabkkgm1eI&rwmL0(*GDdFniNc1zIeL&q>qJ)-BPXkP^_H4y0t4varkE
z=>x4cxX7h>M}wqaG+MqGg3|_Yh%{{>MZRnpln~9&)G!P=w%o!v6+ao#?CO!NO>I9R
zQjTO>C4i1CR=M(ahn~stLUmyZV6?wK{}(|TXo0o4_35%yF5ag2Q~k`=;bQH0u?`P7
z0Yn;HXUzKk@-81kyeZM|b~};AfKS1+i|i@s7KHr3Z01vjYc;FB@)Y#^`0=$>uR(ie
zM1@Y)Hq{Q9U)MyDIlF~DE4ajEo4QB>TY%l%ebHU0Sb|BFx*<}XMb9;TqH?a%f!4U9
zu_M2g{+kzZGSw9o3p*gJ+HN+^r(kgWAYI!mSk#BXKm=Sz+=!=)7)}19M%mJugd!ec
zK3;fcqv1)=do5w~yzWj-;}dhCD}Di4L`mgMC$8o+?zwnbI~^Tj`S~(XHqS0Nd98FG
z*Q7`b{RyuEn{mdvdjyUP!Q;kw<QQ>%LO5~bOL~}lOKqBCX-L@oLc03~5_rnFiz6(R
z&qd6^p*)48)%UUCK@;jf9{STC=0|)Tp(aoeun&q5S;AtsRes~y4^#z@NZplepDY?j
z`PrfS2L<FbIK1!G8|T3;e?phl+kZ>oAY7HYJE;wPB?v`7w{oYxmJjxZSecbm{@W-g
z$-E-@rQr7jjohihkikjziXN9JfiAoG=q<|M$(`8pZsTx}KXH0*_&<fTZ_*fj)gs_9
zzFuoYRgF9+KunIdq&`L$5A<Zy_x6Arvi-bvvvPcr1>wDUnvrY~R`ycX*N^_6bh!K+
zN)n?<_0FwT(UbQw5_9qpEaqOgkxoR4dwEoj!KHEmM|xd5Ns1Gt+oalQyc!IG2&cam
z|5C=ycU;~JB1&p=9YA&eY}b=<E~^)ej3|5>w5y3%7kLR>yMq?hjkN!uO1-Fuqj2>5
zhFw>0XmKF}*C;LOo-&2Ij)<3NxU@<k@M(eeRWgPY#g~#sfu3<6Tegj}c(eLt_;&NS
zOnaKTg>qmctC*Xe<Qla`{t8aov9+Y9jL_Rz#5TQ#(*0}^>lSkOxas`Kt<_D@!`s}>
z0uol#uz~abXKqZEbRfgEUU;5#J~4i9L2e;qNf*@<v;<~)(Lw~d#;rGNYNc2mlcjst
zJesP03Dt~?&kAQ#et`@Bf+QG_fY64uY{==!#xzAB-AMgwbgknP>;)M+46aJ0z8g^!
z+Z@cFq4y8JKfD4U;TdD@2;gi=G+D3t=O435j7dT`ox2^svN$8!k^*HDrp$y$RZ}}L
zokUw^#Vw-Qp^1Xi|L~C)1g=~gPaj|hodf_)thnW|qV^xiK>jZIDso_HAdR-k6JGKf
zk25cfaaLKMR>2jzgthgnKO>^F#+S90?oTEd#^|<U{tnc*J_JE;!PQ0(-lQ*q{!xek
zstS?Bz;gE;%B~B&uY%9~ps`v1wQFvQ=DX}>FJt9{axu#ou{@r#1q0+KJIh~F=Hh@8
zS7coc65zqOpr)k@=Idk0Ov|e6D{1Ph)R?hhUwjA$N_%DMZ`GinBi+u^EIh-LLiV!4
zT)eNu)~i0~o4P}Bgc1-fU^<I`)Zm+`gCc!|OPjWnFiXV4FUXrEBJG!AYYBunY+utd
zCTNlfvj$UPw)8XxfL|Ym-t(eWNo@1??|pFNy`IqZPJxnQ*a}|8-{QZp`yYzXW`7oX
z0C()*L#yT0UP>n?JYrJ+#ZDy#sC?LcRbx{t>EB9Q)J<TBq2oS$73}%Q-K8!ute7OU
zh=LBKp}qi(y9XChYG@GH*eZH%`qMs2<x;6yc;TDm)O9XMH?-C>_9(IW|LH$HAvAd%
zG$L8%;uP%UN;y$5O8L-8a6)KESshXIEO2uL>I&dJmD{2+6URqgLG{m#mmVZ+hr~nd
zg@_`i2At+ICfS|BVT+2HPU7leQ__BrN?A<62j^mJGNlDAmhRuet|+go$&y-!L>1v!
z!xEGu)tPa_nA{1;8)#aPokr;@m(b75h_1lu-h%R8|DP;Xm<<%m<kkOOKG$@Asj2Ow
z^)s6bY$idQC>$b~b!y7t1NknQfSS}43+0RIAWq|NEDfRqz>@-NA<7_@33gkdI~ATC
zTt)wK(PAbSN{&Lrjp@k}&zUkO1Y?%4G2#DIH`HApvsnaRKqoZExBqsY{Uf83s`4wc
zbG4A&@Z#~cE1GaxclYAx+PrzZBbZUjUSJZxQx|b>sv9Y@rmJfKJ#2Tf@Dlj_g7~XP
zB@8CpEEY6pEwxJO?iM(w#dz*30QbLLhr{l5=6kudzS^L<D#}+Q39?xu-vh5kCg`oI
zE4531Cs{tDZ{WfyrKX$eMWL`G(Up;g;St^>o15xO25XT`rHC<?7RcSNg{FOOg{h-@
z94oZy;O*D*z}OSq=_a8}<!Hlcaagn`*P%zL|H+H#_UdOnw3YzcXWOC0-lt7B!sP{r
z{UZ8@`V^8}S|S{g25Nu0(m69B+*m_i{G)ZY^Rd~=QPo+sBrnZj6sS$Pwxl%VZHnHg
zEV2~b;~Y5)m{p>J$ihzv`-Vc(Ty9;r!quSdeyf@huXnr}5T>h<FZM30RMcO%pYBYn
zd-d(7#cfB2KdxVGg;;_iA#Cn2bd7q4;k~H>*h{Y>)4wY|@e_z(eCB$7?*UJD_yu(`
zH;x1F+OsC9yo)gTnK)AIvu(uv(*LCXpN?_np!_4QzQBPO=R7-OC6J@o544xY=8$0e
z^gX)I#fUA53y0;C9)%TWK=7Lhl=E)}Dp^$9;uSr=6H)DX`4AYY3I8yLy?JC%vG77|
z<fb?i6Pn_Y`&Q47->x-+FRbRg`aWRaly%ESm$}*sp{1}i8=CNci3B-PHqWIzEh-q{
zE<T@nBY%A!WKf-)LV!1zc<b?-AwLIhbUD3<OG57%iR);8mj`}SrV+F~_>dD(a<t6W
zHG=vlBGlm_{DRdr7oR}tM+Qe(fSxeqX*tAD#Ey%Fn4jels!2N_L{k?U8rp;|2(E=K
z@c-X)_>(R?1e*PXIAVe++N|g+478&CDWZs|UwTyihY0-TJL3$?`@BlSbc7S^kEgeG
z)+o}YUED@nN%DsT<KSXIrH<49b<`sHk7i2|g`~ZY6MRoLbFZ3)hPyW=tPVzLFR9;w
zfSbV7zW2J&3`xtrzel>yD>v|d{eRQ;1-IBYj{LMS;`Ns`2*Z_1rP>;R*o@n*6s;fc
zd>S(rsZvUbP|y5(*I4@I$U?@rk^!kUk+G~1TwUAhsJ`Q{(Rv(Zv`p9;&tW~IJyh)d
zQC1+V>F)nz*!TbZ2q#~VGD-O=<8*nAs0Jxwi78HyjPx&D<%^4tU;b26J+a*A<^Y|N
zm^W=1mypo0r0bkdohkL{<45x6_uSkGel^+Q;mD?+KXZAkF>!1tkVXC|k(QV5`bHuC
zyY$)rbRIZmIWRc*5`{oV7SnV6&AZrJs;^XmW`BDA1^XtL(5%3y_J7@1Z|L6u$opsj
zZRXBjGWq3{pn=N&T9GjX4h2fk0ARab5|noc_y;@xA1_}YEpN*pcW<Rb0W|13+ubhw
i`itC4aQ+ML<oUIEJplJ$<Batc^pKHI1eS;y2L4|mSCS6^

diff --git a/modules/web-console/frontend/public/images/multicluster.png b/modules/web-console/frontend/public/images/multicluster.png
index dd82495c4d8888d11af2dcb294212378848d9536..ef86c3f01f1d897788b9608b128562722ee75861 100644
GIT binary patch
literal 31178
zcmb4pWmFtZ)FuQ8!95I~;4s)gfZ!f15S$4v4Z$I3f(3VnK?A`F&OmSp1f9X%A!u-S
zTi*Tl&z}9Yefm_NzPIY$=P5Z|ed~sQP<@AoO^J<yf`X^0Agh6bf(D?VJY&Q{dy@Q|
zccFd~pnOo#l6!o7yuH1>y1F_)Ki}WqzvAH+6%l>JV^~~VM6s$Zs_#1BU^zNId4~FI
zU;x?O-F@=#aDR7_O6fB@JG;KNwzISIaKHUX$bNs>@o>Meb9<*QCcggg5I1?<b8^!=
zv-uDhyw1iXD<QqqJH51XC@P_BQLuDB7&>?VAR^=X<5iM}v}9q|?8xx2un2H@_P6|#
zWKjRz!T!qQ<Ms8)xSpi$*P{N1`(b|mH#gf=9k&l|O(!SgwQ}wa67T#ZrF1s;50C0y
z=2AXN*;eMSKHRRhA@3gy6r}d{1}pdO?~Z=;e)#rY%I5d)*4wrGXm#O4sMN!3oK8%m
zj2h7GBY)TaCC5m7b90Nzw&iLzM_4a8KtXtXXJ0RQX8QbERrURk?EFlL67{Izf8|yV
zGVkA-{9He|6j2K=96PYrl0^h5OPS@@yGe7RW1hEK+^$TF#A;hhimDLel>NQ>b^ho8
z->&EXrfB4egRge<yJ&1;GO>Qr)9cIW<RPv?)=9S?HlwvtKu;P=Zqe(&U!$w;o%c9A
zF(z#%_9=g=b+I3Jw<lV@$JwjnVheU>d;F@eNli)J>yI5=+{;$NZag}&V!(-nU~PO)
zKbcA#HTyQ}5&f%t8xq`;x^ees^)~g-db&x^`1YMf(N5}z-^=+XGEHZ>_5)MPT{M)`
z&NH`u@Y%?)h^wJNJ;&Hv3!%4Itbk|TVX@pBo2DBhz{uDruzQc6U(HKfdI3?{6mQ?K
zeA?c9M4PicR2}&4t>9cV|50hSrGJ*Otji6*T5<HZ(v)R6o#^|6yZTVWGqbX`64>X&
zn)#)Fe|l%m{RY08eJJ#sCZ<9tSVURd*jD~1G!f#zROI9dT_$jx3JfXB{bqe1uEOCo
zsOeVR;#80kRX0=WDVequ`KvoMr@h!+6Zx58sVVYO*ygNg9p2hMF}6dYgqVYO5`|3P
zHvRsS*M3=4f8B8?%UXUHpRp8`c&Qk%^Fg)IR^-0a@;#(~>T0&mjJCs)#znMh>2^C)
z-HzY*b36))d$6Lcl$Ph*-gie85-W<|+2jo$7(P5Bx9g5~0OBwv+%)T6;H^pyl&H*@
zK=Mb~0vIKe;;NZLg5)&ug1ZTPe{*uBR+AAwdxKlVOchDFXeP}oMZ!l0@9y$@K?l3)
z*!!~FyySdmm83X(s@$;D+Io?&bnpGJV2guJgo~w2MIW~EPHzrH3SFXe3=`}16%kB8
zlCUF|2bJzMnC2UO0G;VGWpEqnlRnY{+H)beeqQ@0xs&=8GHj!-42CDLJ#B~}vjmQG
z#|=1loA~aVTXHLDnVz)4JgDzHi_P!cH-CB+-r(E@L9M-u+$&&{G-QRZHYUh5t-W7B
z<-r_5;WOr183iH^Ia1jRO1GU26|+g7ZT^+zT%SDHb3{3s_RYFYIrTG62%6sY^L$+M
zP(o|ID7*cxZ~4}j{$}`NEqFsK`J10z^RGHp^ZRw!ekhuwA&HV8ov6)bU~Yu&o+H-A
z@N^>31(*KdWJ@SHV?GhokXYMjb~N6Z)G`%uV1IpU%-?QUpC@oTIC_JVTo>e3XgIH=
z2PS!rkl5!I8s<dKxWiXWn$2bpw(>AS=dZn(TbDg9zxZzJ=P^5vjA_53zmp^VkMfTd
zC#PvMXVZ&@dl=9UrJFlOolE#qVj~d|*Sa6wG|jzs+MK<GBVdJ5qeA`r4V$?)fw6(#
z=S+aR!muvw$mi(34$E4fLs=~Y-FifAGnlgE$!fP2>=kXjFpT$o(Af9@oU|QaqC$Ba
z!ST?fLM`bgiv&~6VaQgiG~6PZNXYiYG-FUS{<tS&GL7LsSik<Sy->e~CQXAK7Jw|K
z2rc2dl?w+V0E!YqawNFTz%os!>ZNgR7d#*%0>BUq4hV!=)~f7;@gTdTq`KVO-lPa$
zcR^%)q)K5*vByr$Z?{Q|QhtK}z>#B<y*qG+x>1%ewC3!}dPYSzXq#TjEHNAr(_Y-U
zWxq8FA8&ltmOYq;1SewpfHR>90@Gvn5VJ<!8US$*sqQVV!_^O&8ocgIX5PQelSXuS
zKK`yia}>W(-QlI=+JHB_@f5h}tt<gk9bbzt2-z#$c6|C&4#2_#&73&YF!R5I8DM0Z
z(7K>a=zkM5Ac}w}MfuYZ525UlIXMhs`82n!tb@w7kN(rTo3fdQe#`mdPKLF*LAb#|
z0%Hc+Sg|P7FfqT;hsiPWayj5)1?chDwmLgIt38RjrN^m4U7tx2`pfZ&8|LfDqX!h$
znJvCnUzkGgi0mN?{a`4ME(ux8C9X|3d>_Q(oQax>v-Fxi%=*(fC%ZjirWQd|2L*ef
zDkyUALi<~g*Y_Bw(BYrKT9v)17m%W!UH&3z3n&g@7h{B5q!KSW)1;~Mumuv6pZ;%H
zSeO$^7dEd9&BYmEs9;Z-{dU-~jebZ=Oz~MRWS`JlUPZpXPm=J}{5rPzhMrZXY_1t7
z7q$7h<O&XR@``gLC~tQ36_zSJ7TnIg|C`&97b@l|qj0aj4tma}lSmIdwGaN<rK~wG
zr>u{+3mi$-NkfW!s*A56H4wZ{5YCf=fGD_qrFJj+L-H(sh`bD$$w_&s<Rl3uw?=X3
z*u%jnGI2+diUVvWF8jYiRT3#0O5jmMZ@<fUbE6`h6w`?;X8Cciydo*vdXmRgYVEX>
z#4vkENI<3P1_u3L4zzf}=GIr}VUL``Z!AE$>)~8-e91s{8-88-!3&I-DbpB}E3+>>
zVL`b7uHenT4O^NeC;Er%e-m*)m~pxPae6cx>~CH+{&RX6nv-v#+V%=z7Em-<`M!p~
z5|-z0G_RZWHDZ3hHrH$S!y{B4fp?xzf8dKn=KSi+JZrK>TQUAY6}5(_u2LOgyb&~j
z0}dBIK6Q&~P$XOsbDm1EQnH$d{x<(qL5S@uj-xs1(0JTjeeQk|5bJC`XYQ1nA-Ln8
zng_%Z?3VK`Hi52ae^Ux;+#Q1FYfnegLEsG6x898Ou$w@h_^SjCg5kEqw`FkfZ!!5H
zzBrNSa9<?<LI!@(G+p_X($~m&WbYd|(86^ZKAWrQ9rVq-xOOqD$w<A8oSDS%m&jY5
z-l_#B<I27}%CKPl3n7+Y)tsh=AqTmr+L{ulp%zdp{rUR-_{SkQ!TUNscBJKB24v5K
z)~%xcGx{CDuLKnS^(r(2Bb=OC1Yy^C%Me;b%g1l8{mb9Jv13CS>zsP64I?}L{${Vg
zyB`#rzT?!GSLezu^03s-?9@B6lwyi*)&cwO0_?i}d6zY0F<aWu@ab-CS<kW@T+I4C
z1z4!OO2EBmL<)clfsbd1VacL)p;P^yJPj=RYSFOZngTS!1N}$@q>TtW3O`E<lcMyB
zmr|bU{Ah|Mu>fc7y-vh^e&dR$oz-%EOA6DfX!pTqKNw0d#p00*z>)Gfk!Nw>m#bmw
zK85y1znOe?y|ooX#e~Hwui!HOOFMpPo%CkzTZ}2-1k2l3vug2&u>D75gMq!+7Rt)c
z1LuIH^trs|hLcD!iEUJ2GGxfGXy^CKPn~?OTP3kHSkPl3wfT`hBtvnbJ9Zl1w~}?>
zi33CB<zA%Rv7It0M|+j_08TX*$AVvOr-7edUDTFX=+u#mHUHVx$zA+?lkyx<BiEl&
z<dhHjv%bkCp<v$XJi5(#u;KP7U$U?#l$<GYT?@Kt;WhJiTSXCzC**uq-cLNsxN23_
zc=KZ)d?L2NvCuyH;b4uUl6r-1cYC3OjCH~j8Pq{${BGKWeI|5vGpg)C@6PJLO+UbF
zR^(S8!R>p+I4064{^NsC{s8B52Q3!|1~Qv*J%P+E%I%JKyfCJ&gQoV2_=8;WP0!)A
z%p9|B$N-gkTYMegWcOAX!F*diIZb=^DV8zqxToLnxHTsd{{73n{pS1TYpcue$N`9Q
zO7pVTiq}YHhUt0=T1;7kQHSA~ex0zyeBUtOT42vGOgFvAXMgve8yYhFQnW#f!;;Uz
zf6X7`?=uQbYn^DMQ9EG3gz{FdcGx0KQ2NzTaGDv)qbG^pPHGU$-sEqaDuUwJ{1-Xg
z6IK8R1Dppwwv|~0%Qe&t#_+v;JUDO_^QGlr?HvGPW3bjoJ#f?H(H!~tM#Ql;#6uZI
zmZ#fW63HF10OOPIhMVD9efoI$RyX#*`9zbtQ*5?$=C{zqudDR;OwnKM0xYPUBcZTs
zu8OKtt;El=wYZNabXhF@kaBm!_G_^X?11<GGE;ePIFXQSU87a}Zx)mSIFE{{xzM6B
zPLT+J^SASW4RqGeGK~y_*qo|;pnjT}4Q4;re{rJ#K)>K$s+LA`&x-gVXWmZO%#X*c
zbjTvt<7pZV-)sFIT<CAl-rCuOnLb#do50-EOlO_&Uv3E`+6Y-;?RPflEdIgx-UIij
zlCIau)fI7=RCEJLOuk9bTYyj>_wsm?QV8r3pStwFY`AS^M;<MYXnx$c9#8r0%&iCU
z?6LZ5I|>drA|EhV<2)VUfW3eSb-wWK=_9dqej5z1j%mU6Nra+LhbaV;v7&0r;ZN(s
zPW;PkfW4oY4RQ|Rs&r|7wW#am+B*SbbDMP<8W+g-q=oI*_|$yfP{&oFgD{XH>s0Cn
zUPCRfb8`L)iYWyAT06BH7EK05wV(B7Uo?oJruNS0N5@Wnc5^4|Ollt6R_gheISFZd
z4kmuGetsiA7*$>7yrXA%n~VAd&yw-u<b9ku6ph$Ci8JwC>Me`jn3_smR4A+Zs|xI4
z(so_2<|Rew@CtUWo`8N&!;QIrVotBLNerxC><(lOwbVrWjPh)*Pi+sK$-XN|ok4$7
zvx#E6=A%HDfY~<^Yu)!gxOOZOFD}BNZfsA$j|5{4VCdU_VBkNkyPK}KHSLMRjPkn>
zms}jVQ^`6N+S##WaF&)C99YO&rkZbyYK!Jzm^lht@LFPphWK;J__Xh3y65jd$Y_J9
zhP-|q`=0PG67ZYr+Su5ve^N5KqtbKHFWs~URU&oN%8*yeXc0;559caq6-JL;rYMv5
zkmd%lQlRwp6$5d`&~BroIqSMdL1kHY#ILMNilQo9!@bX^v}>V-%?HLV_b@r#Lb0%>
zgMo~5in|I|?1SkTTMw1Ax1NwD+POpOX(X(l;oW-=I-^P<*~QrHEnkW^?TaQ5>#Ut2
zrg!gXT+VrrA6~DOA{JuFy6r3oRG+DL!_!2r-|R6SFQxqIxu9>E<#$Bp;@DH=$vdk6
zp!TNR3J)fkl`Y>W>vDt#q-4|A^-9#;stXKrYuL|e;o%e6<Gj;;Md);eTS`@i8)0>c
z<oc}zgFcq=5L~RivuX_$?5k)W<Rm%#iG|S{?UHV~@rG_XFlX6f`(W!5shZAd_&iyU
zIaW{q*cUVja9hzb)HQZ>9gn0>dM@v7J$%u}^lVr8eG5Fz%zH>n^hGr7vyibK0<TVn
zj2l1aLV1M&i?oSOk4lYUbMJ64|39jVw8U?Eb1)1Nu7dW|FklX)7_3={bIYt}sROo+
z2UAI;G+-E!j}9`VvRZP3Bl$!PanPaULi}nyU!QlCjgkopbU+9hv_rkjSR^#!IB|}Z
zU{YFN(ODMpxbbBCFowS7f*pq9fJcz72+1hFZZ0{qx<&mv>kg56s=?|5@*)=Uut$!k
zt;(#dFbRtixx+hY6lk{$DIq$vPO?M_Z<ugU4uw~gEZ@<Du@C;6!5t5|{@ATZm^hjM
z;Fu}gN28-lRfCT_lipsOB}by)Q+BVK2)gv5hhky{_(HAYv0lXHR_ZeT(|z{B?5AYA
zr!pbB)c<oI6L`7Vq4D5;lN@nQycdi~O2~S)ePAW|6jk}3>(1y05<#yA8mNYR5_$O#
z{#0JZPYk3;od9D7ZMXs4iJK<?p?4a83U}ZNGQZIqgl^0{xPhd<f>?7IV|Jt{x?DrB
z+Emh!DN;wk%}l8|<&$Ad(+9)DY_tS&BG#0Yj$;Vh|6d8OHWz=TF6&Hfyv%b5S7yvD
zwjH8%f5~LX-4qos5Tp^x!>T@!6Re;f^jZ|$;EJ_L&-^Ki7?#f^l^;4KLDrChPZVwl
z8NrecaKIt9AN#qyC+@NS%a%{TZ^~Mnz-KQ*uzi_&vu1hLIfO4}(Fs1#;osR0v6`{T
zCoW@S)ac877@6ybJm*lXE{um)_#w2-uBX$~WyD~{JCDL20^URix!sev4?+s<?t*N3
zkQD1C7-ivlGs}h09)1{uuDI&DzHDoBRN={{$z#-;NeIsEJU<eFF#M*3{I&$iN5wp{
zpn&{S!%UwMH=F(A3@<BwXC5wq9BG(akp%rg?{E~A?%@|<NlyfNDdQ9)fvO7eUQv@&
zArT~KV6Tid((+K$oS;_#8WLwFa7|ESB(oJ!qHJBS@*cXH+UJ}wVIq{!<!m-pV>Zke
zkO-92lnqk}1)l|KO;jc%Lz$K9hWt&dXH;kO+}+K1d7||t@SQi6oBeBt`{B~CHv}eW
zlCD+?jfrl!yo>A!O7cKj3G3;VTpXAjU(J9wGu%lCGd<i_%{dfw(&%y(mk0!dab#Z;
z5tjQ>pbn0Y$Hs9bxvzED+_T#h%R+U|&s}Kjq1J-Kr}mTSaNGoy_21RuX^s550u%yb
zf_$i1&bcdWJ3N8ZnhGLNoj!OkoS0k3`4a_Ev6VeU{X^s#iEwpPh^j~d9Zk@g^_Q0~
zLIG&ES{7Yd9(cD&Yqz9jG=~XEA+MLhG_TbB{&g#pZ5<bnET10_KKgy%H{(B)swNuC
zK5zAQ5=JF}*>%R&ldtuWOx|9nifM21S|zOOdWn~G!0e~$V6=4ZwFg@Ti7=p0#tJ$a
z`ipT`LgLGBIO4r;_6TSaia8vaiJnmHFm58n7k-+^gVV!=u5e5bPq1o3%U>iF&!^P(
zxQ<OIkuwB~m_6d@mS(|no*T84wV3`}xG<xR&cU@RO1Rn?k1Ue;g|*lf*g1!V4{muK
z>&(sFM^##5x#*QWK>MuMV#sZseQA!FDtRWbscLKsgDS^COiOo@xtJ<!$SUQ=&z@!8
zmM=L;MyvM@sxwJKfid_LMzD;quvzAX1hBqIWdj`@8#YK02S)+NUu08bOFcKH=O7^`
zGWLvoKJqyyvXFpV0@opU^pJ>`j^(U_&1ZErWct^m#Z2I`qgsJ@ZFh1VjqtrCr)_p8
zB!?HhcBw}Qg=*erUH?JUJItg79;MU{{&I{M>-=*c;d4u+*cRVZRU34wH}y8I^_z($
z8-KvDNwh(@aK39k9LxfBO{OzskQg9EN~?V|v15nb(}cl8W*psz_$>-qqt-5sHhp<I
zgZlM)2$`Wh5QPf>&ZvwV4F;CsjQuCfZ{!sEo-4_ULklUq(s&@z0BPD-eh`KDr==yM
z^}xH~#^Hh17_UK~hxj3IZQ*pmr7-Q*>v2o;a>3(PWQ%*Uc|Vo!Gv99Kt4R5`mdl@O
zRw9EUVh!^Lew{`Tveq?afZ>1tiFY7muS<4-{&a#g|IE6i<~YPQQ_|AkxerwC7MK>O
zfpm4c3>Yp#Yaa)@K4vDW1an-%Z!q8DqQcesJD&YxhkZD)FF9PHNVfqzD7xp9Y@Mdm
z4=*Oq@Vdclfw!6V?^Ra*73W3R{!&WLQ6{|{4??i5eIcG+dS{X_%=gk*-GF<(xCW%i
z`jMlff5*(xx~KS3=#+=RpeXL6b?8W?0!SvSzB~kU@+wrGN>i<e*;F>oNERf8iA=|1
zAz^FwpUyixf~Z0t#Z&{SOR<BsmoMr*s!g%6b8hUJ*A=zPLyKe<hMQbR5vnl9@kQH-
z(Uph|dpU&Ycs*MHqBdP3euEETYSymwrnPK|<_{g=#tq@Ha-&&)hQ@=Z{1ls4_s)SH
zocUiQz%!_FIjDUAr0K*AcqV;`9o7vLlxA`~=~{b3B6a?|fk1Lgruc^DT!ZyM##(U}
z;$|8TAGr^`iYayu`s%MXAx{hxAOGvt>lxJbikSizaM-R9A(D7Crz4VHIZ@*nlygtX
z^Qf|_=Gz5XCez!{;vY-yDB6v^BW;bM&s%s0LLh9fz-0)0VZ0Nv%%OnFFnrR8<UBAm
z#ii}!Z>k~9UVFG2s982G(M|pAQIB<uAp7(;)tcn2{|h4wKSQ<oVl~V6y2oGe+KR=G
zRMV<#i&Ne+O%?A}ev(W2wHgKJAHRvU8T>=4ISEyxP?Y!wBuL$;Q-JnxN#(9os<otl
z;MfZTC;{E8aj+W6#bP>2i5Co|9^HC=-h~QGMPP#VsN33)bB79+Zxhc=s0wHh4gZuj
zNlk<mbouX<`5Tt_9s@DOHJ-B}V6s*|o0j=11i0V5hm>uKoaGR#)b^~_3OOuG7S!rT
z5L7rksei-h&!t@O3v<Ko<Dl4=Y>`(x7G{WaxUR+waj}Kc7(e}OcM@+CL6S3$er*`5
z%p`(NXEl^AQFXS<cfUt9YjB}NyPnJEtG(DrA<{`s3NZXJJv)W=vJ3~Te??R-WRCpc
zXmTUKIWRFmY79l#d8I{{W|F{I#w2yC*LB|}#L6R1h&3rf^~{5k<?lzX=%W%b-GvOh
zHBQI0r_Ct$T0TZr9(u9i7&LwPe7gDWM^xpsETZ^~2lv$ubJerb`Ym-5glHmboG0sR
zJ1Ni&6K`j9(u?;!L~`6Nm+9&BeI<W^Oh~8#NH*!avoN0qn4$IDVw{Z)T?1Q#XQO}h
z2jAh4urtqz5{ywIgV%;At<8<iV`${LuCZ^>I*$hCP%Vg|WD`<gHxNFBN6ClK>HMR!
zWET2LS7v~Rybd_-nD*gE1ibW8Yxz6bIY2z*&RV+Jq;f0EgQr3Su%Q7&{_qdf2piV|
zF_iohYIFH!10nlN2>TcFu}hu8zq2CY78u+d&0OY*xbIIX=xfSZaWwRK`-1lvh$H7;
zh9|u`N{#0g?hb$iF%;!&_ow_HYPOTZ%}hBmmg)CZ?OeC%@P;mJ#`VB8?Pw~MFP*6w
zFV~NU__2S=f(9z+gU83ZC|L8ta3d*9B$MfD3ir^(;fLoC;LrWgbSwn1SvoTEU3T_>
z*M$UUo_5PP;a%RqFT16?SLMHWD*FlDEu*vzQi$HZp<7|01hh`-bTT0*-E8CB9zpb*
z6IT%~QU8#!f5$7RL-Ij+2Y}mK2a6URiCiP;H<g-}EWHg(IpWr8?lQiBe42y$S3z^f
zW!*`2dkh2?8=BR>>2wIeHA~7r;A<jJ?h0UH?N>=`0YI)d&L^X)RTJFzM%c`6R@PM-
zkLn$x$bK;)QooalSAiH<^RTA0BG-g1(bKM@h~{d;SvhLBc6+A)&8M;ONM1QBN8JSO
zVpv6R?bE7s(%M@%MaX9;Mp!lLa2MrFH{tR3#knb_AX$zdQ7kF4Q+Zm_=buxpBz83a
zdP-d|2ocA&M~|xTa_1CJAo(RMK23+1RoZlrBzp&$h#JjfGvZs$5(s>KrJV(7GM2Zc
z8|Vpxe8fsqXRbg}4(WA`Q-%+<mQMlxw2q|j9fW0*UhImspt>NU-gkyYJdq5iCt-C_
zV?fji)U<Pv!09sK(o#hOvD?@g{GV&}!aF6?>ZBPEtO4nnO>q=82ww_EC%I<5R#OBo
zm&MzjW1OjT=;(Me(3$WX`-4F9@f8M3(uX8gs?j%Jt%+C3GqK8y9$u6Sp{1aS`*04p
zqEn^av3~`52SixB+F1IK=<?y`uh(u_*ryQbH1A8ybl$O_WM83{qD+)E3e(R$nX{D;
z>!GzO5BU3rSElv=D~@W<5<yeKVk>`pn5dbKTT2oxDO(;j1AoHIipQUi<?8iBVrDT!
z--#GpRl|GGULXutS#jPCCPB;-c~;dH6V=kZZMjAMVCSXy({&(sKIf(!GvB6pHyMYh
zJk%eih#f<$I$PRLE!QbL>ecC}b+VW@*{EP8n!_YO$M((*GIEQ)SK)~corLd+wP{2n
zVkx;|0l60$M=aalk7z0T3_UbwY}e~y_o)!qkODb~j{9(uAOaFc<S&sFDBsm;3-zIy
zxy};|?QiSuSP9k@Kb^Fzzf-<M28hyE>0<yb%Q?EFT8>{Zp2lZ2tSLqN@fu!vsC3KN
z7X=?{8W@JhSnIf)N@KStJYPR^1F2g+R`6R!x$4f%@s=>`q9XRHZi2l|%Gs1}c&6_H
zFV}5#Au3VZ{T2t-48$!0(?Jlx$yvGgb?_F2H4^K+l&Q(;Qfzs-@^$PrDIVfXeaISL
zu!^cvq~wUx|EbG_R)^0ZbEa;+rV4&eL;yscx%Y!&e;_nQ9lsMv{pD@arNwNA0{Q+!
zZDt-8`9L~&!$hR);@I%Iheqd=)HZ$3NrxyJaL1XSR?7!33a(!LxmfqR_Xe#rR+?>@
znScCfzw+yvIVt>>!uU1(belm7l;>_}I|J%}Hkyxfzp6zUB>#4#uypCLYF9T1lvk}t
zt^SZ^WVyh((6Nr8e!RG2ay=%n5eO;d3T;|5p5wgZ<mRxlfj=C^?)9@kMXygDuOTFe
zHks2jq!#!%qu^Zs%{fE6VQ+|%TAS;Y+}BIb+nR&R3pdK8D&Fa~{k#4YD59PkiO)wW
z6CYwCLX6g3B((p}*sCo{%a>NFUrcOd)s~op*}usZ0q*S19@-2HOVK(7E#GFW+KIEb
za?jmEA4OOQ9-fUX(TIL2daRMZoLjc%&13p$DszM>al3D1S?qgF_2R)REu2wb?E&M`
z^)UaealoTv*T3_{lx`aZYm{OVG?YrDwgDnLz0JGk<+uBj=1oZ2;_hhvwxd>JG)lBN
z^CO#)Wm(^2>t1N<K=y}MO{cbhkn1*HAU9TNyDw5ckB&!mTL7Nbg8i5@%r|eH{Qbvg
zH4;c<M8cyN^y%lnVK6Icg6?-x=|AAJTg%?dFwKz0e~vnlo19dg17&AaEoEmnhd{Dt
z8t=1`l9YJ%bGY%*Gl0{IKfYiY+}Mw6Y5C;Z@f1G1x5$hAhX)s9gd%N4>b1;X4KVP^
zt1>GGBuYlDWy2`<&sL?X#e3;r<ujGKvTVyG2#TGX{zT1Wi1`35khYFg`g&_@fYivo
zhMmVgLYV_VlHw7;uw(|bhANelWXab1!>TAcNfAyNI@pUg&r*1M<%JoPnBupA2M*`F
zt^7-v;UiMc*-x*Bz@%mjzS#|_&N6&CKKHbNe@L}W!$BqtL>(|W`&UI}SCz4VZ`n*|
zNs$OeSJ(!#DO8UfRt<rd5teg|z8~y_t9QZ)Zh?U@9zu*2K}8YKD6AZ5mX{*6n$N3+
zlt#g;uUAb9Baoaq(jgj-5g{5P6Rs{`d#AW>iS(<R692ed*&k4ejz5bj7nfu0k6rl4
z9B_Tv?M;c4=A>GNkno{^`2P|V0}~LN*>(zZP`b}Fedk9sqEcPTC`MDaVu08NN@_EE
z1v7nt?jZw9HZQH*Q^*Jbpwo&P$YLOJ5l+mS3p?lkhH`HWmFLXBwV5@jwMznC9MT9}
z65$jXW<76R8h^^*;8YIVcczWHC6;M>j_Fk$w@htzPW?lI+2~8(@cMF*8fPAi(*bG5
zTsO_MSp}?urguf}E!?=~#O<K(-LN@v>d8vWKv5D7UK{y(<I}h}NVn$V9$d?IZBdq=
z%|qc&uCZ)pRqFRppWne%9MAA)m@JxyLW@6|yqvVw)gyO!RlG!5_Et~VK;${qfUbG@
z(Rj`GJTK5MyAJ)jAT;@3;NOH<hr-4V2AV~%Ng}g-$>Q`{mGW6N7N(2A?CQ2}a!o1k
z@qgJB>N9hePPWa8)W@;r^2BiaRM$E;eIpC5m9~D_6vaud<p5Q8AU5onQVLA4*0e2P
z;>R6e3FLB}&3(>Is0h5sV1(Hb><22m4B+};`U#NMx>}Ms|DNW!Xz%L6|EugUc3@~I
z%|(^q{$}uP^F_b8<1So(&1P+NmG=F@=l#VTN}SdVn|!-9|MX9Ye{}`2pynvU9<D!!
zy8E!+r3;F9P*YrxTHu*O$#MAcO2h7G(x^RX4=#n%Y`5M#P(l-(owC?Hj4aIL=4zbd
z#hKP!ixc_!a`t}2L(D1CT$LYi)fWWlB9xio$Qj_V5qWlYvq;cr>Sxy^=kc=j)|%x}
zlIURN7W|)GQv{fb;CqQh%Dv7Ez%U04<0hR8h7tS4#y=s-J3D}n)|V&EqZLH8_X?ev
zrbK^Akqds(n5W!ZUU?L-?ZJJ64KKC~d)kfv6UudR<y9eX$lU*jk$Flr5|e2U@_$~(
z|8MOrm7&09Y^dSX;oX-M)aQIQq)*)-V8Mlk(jgHBrTgm!YPS2KssByJ^vO&J0`N0>
z?T1vOdn~JdqLu33q=NE$`73qX_6)EG&vBVG{-!aocBCq}GyPwEOyOTl7sdc;6795k
zc<+Kxk7_&h^8pU~-zWcp3h4y`Vp{)oc?O&Plk+i;wcJ}k6GQvMdqiH%oXcPs>lXEf
z=07m#ycI~+-V{RJ_NaO{F^W9&3K?*iSrI+eS4NKY=bQzD?oCMvjnpcDwhR6l(VB_U
zrs<NoMG9UZ5(}fc*K(er(}~rezD<<Cy+*q<y&y_nTbSH`pr9KMg)M`svtIlY<Q?>(
z;#ujNZC|o5jIAHTWi+J$m(~9g?I^+0v!9LrgUTwO@<O4s%9MRfl2EV?@d@G3R+~`}
zRW(?G9A%4Kj+aS_7aLHAabjEg*B3Nm59FU&#lGNqx$$dAbUN8G1@8~-+S~f~k+%k^
z8F22Be7iHR`hED<aeEit2(J2O>X!4~#e;1Wk{x@;{RuGJ?v0s;XzgFTq{*(4f{{ch
zC19&IG^10_sGoT8t(2jy4-LaDS`n2j33ipNtknwDfF-ac?<mM3)=8F3z5DXyH(c-P
zsB9-%uW9(Q1cGmAtTWZu*`n(!K6SFlk7i|&j(F~>7=Ni9@@2CKRmO7IdPwe_EE^f!
ztQ1NW5W#!fKS9gzzhS%DC|lfTe~Tlsd|@hrfry6$^-A<ZD;oDsUoH>aC^_D5@H6gM
z?z;At{us$Pc5u9RZ4q?#o^fSc7BfYt2BpVAn7-J4wCZCqEMycRnLl;GPAAczBKWgY
z)U4t1XHWquQr^dpps&Pb(@eO`|GP1UH(8;QZ>Rr@qcQ5Hz%Fa!6EsgL<rPpsC0bz-
zemS=A@tv|uD!hiOWLki0_+twr(eu37(@g9ACLs)z|BW~Y`R|x3r%Rq~yU=d)<?ZQ@
zlM?DUTK^=rN*k1A1dDvA%U4~(70Xg%M|X6@e84lc5_)w)1IGy-;Gq*Ty%`eyijLka
z0Uq5*t4Dd#&(b4ZWPbYjT6z$gQ`(3}C<o2S$7L>!W9lWU@`#R*a02koT`XvIllvu;
zHj+{Ehl;+(jX5ArGM_!~BtrQQ+fpXUJjOTcl(b5Y=(OD$io9f{2=(klLrZanAppY2
zSF2~wNs0H00U@jZUZ2nym8nn(@k~GkPlH+iDJ(@iSSjfDr%>s4Z|@DZG&(jBQSLz7
zMSt@k-c<@sM?WkHXs!BKjw7Z3l3uehU^E`u8Jr113gsbJx)K+gw{EJ@)XBGt3bPgr
zr)87&a|^Pjo+$W}tuZ1#T_iPc%S}px)|<x64-ACrDjX5Tx%)}aY29br_aGV|1q3?^
zdoCMVrHgGXa-B^+Z{SO7DAEaZ$@-kURwrO!;;;=1sH$fboC8jh+$D4_U7-xeio0+q
z7N+>4V#>yTc<vrYBUn`%`_v`ZVDo((7joJl<U1qa36e?Lu8C}r9c`2n;X8z!k~tnf
z_6RCJ9Uk4J5bZ#S0wMq4a#KLLxGT)>Y>nE{=e^mqa=WwPCe}n)or#%2t15Jx>P-YZ
zX(0c=4f@%gGOMH_Hupy(8(BQ00y*Sx^i9PxR=sG?25IofZr}!luxXzAf*OCVVr$gp
zD(a^Mi&n(gDQY<c8l(UNrEb5<;$$L3v}M!8o6%ye;swP5jdX?AKJctbF6O9_{5%N)
zwu7C!ZzEGo8zGV4LF=?8l1jhH9r<^<#XkbZ_TYX4K&oES+h>2p<PG9l&-%yf-5tA$
z>zi}7Xx9d&lpIkmDT3t{mQ2Gw6#9IU&-gl$mBp#%JY2bAx)vp=8Y?ERp4zTlge$>a
zS~~W-fvSBUmpFo(rI{_y_xr7LERg}r7DDW4LEk7*6R-1gWWSl;K&n%p7=6`<4`t1p
z)y&qbpWN}&8|UIP)}=(&_sxxH|D_xo=ORy85FyR6o0I!;pZI!L{!z;|9b2geX=cgT
zq;huG6#8vhRlYbJlX9<2zDuQ$l#MT~qk=;W52tz;!~W;%6>@e`nnJs}7GI}jYHu@s
zE&PeYr_YTGS7L9+T2ygCqQBhPoTiFGLlfJ)mPhWuf^26Ja$uVKPv%X>5@@&XMUeB|
zio=hoWUXh8G#qYlVI9Q!8#NPP8%JyBgIg8F3Hp0M3CVhQ+t4<dmmI?#`s)qe0-Jm=
ziQXT4>yLprJ)8LX>)Fdv^p~+c<N2fZi|}}O`=#LgDMTE*LjM{wg~s9&m+catZM|{o
z+NX|ctuXgjQ+WN=q3pg_YJM){6Nd}HiF3hW#NKxeQ9zKT0GEp@9?8mHCnlOC@))63
z3(_yL_`oz3XJO`E&;%Y!xPjYS!<tmYD{Avb&F@Zj>($bl*;io`=H8E6oQJ%fz&lVb
zvH_uYg5OlOl*I9AS)oY%d4`;kk#U>xTwSSi!52iGxq_d;;(d!DXJu(Atwftcg>TDO
z&1kAtktu!y0r`~U@uE45J5)aLL!VCS8LUk2i=&rryE-%9CQqXGL0k;@g(~(H>&q2J
z%M#$q(v1@l!~Vy+{#dG3FD5!J5yXfAxu9SAeuV%yQKl44@BSm>oxkE+n1|5(oP8)=
zxPiBG?3&V=c+blNxKM@MABJx6zxXRq+v!-J3gk`+Cy2O?e%a7F%B3f{N9>w=5wJSs
zHLJ{?oMQ5}Ju2fjm`X$rE#;ZFhx^8~E%zKhQv8LWzB1_XOPh4OkVjAH0GvPuQ|MfQ
z$Q4Kf?*BZ>UKn7e?4$mtOPc|hhsP8!n3ZZ4egjUkJcjPz;JI|Ugbj9{FEB9fQ^n{B
zph*du{`?m-GhZ9Q|DR+Wc<2IiR7fX~r%7-=S%#}Ofqi!%<TYpP6x(G#@EM*e;f(_C
zWQu>K>F{z`=CDSvTa1_d!FqC;=bDz7775K_gz_XX8mn+5KcSeVH)M`ad&6ZFDN2Ep
z!fXB=@dVZ`pX+pZ6}px{j%6P8D_t7%sj(SX50Fb?6N(qMWbq#v`5ibmP3jD4%9j=R
zcmsM$c!DQL)@gRc02AdO&gV2}3JHoNU6QSS<3N%N4Jyiw<ghXchvp4ga6rJA+i{Xk
z`PNG(Fe@tqr$bhy_Rky$X_7d0j`Ex1U6gMX>Wnqv=_6FBJf=IYuUP?X2-*)9QaRgR
zAeC1F?&8C(F01az)gvmq0-j8$xaO(KP?wcwvOdVYuX=V1kW;)-PXfdlnbkSNJ`j>M
zwVrDbq5DUU$f#pt1jt&@e-dLvCnB1VAUvB$6S2CCDcUWYL@&);J967va2Zy0lW6xk
zSTt<wY$2T{|7_UPBM)9EQGrJlL~*xuwrUX->O_Tj^!D{p;Is3}aB>7#nOS7ME5_zq
zRuzu~F_;&B!m?xM<U8?+ii&!SP!L!rvw^C|Wyd?W{rI4j=LUl6Vpa%8t~hq~=|!b(
zy%hdO%TSO&B+Fm1P64u#N0Tf9{z+jml3pZ~y<!kR^J+}D01IfTj7|ez*|!aAK@Uc(
zA3tzc8h|2+@H8@CAr&S92J+WbEXj-UZIJlW`bM^z)A8->gwdVvzOsvKck7fI(qwY}
zNZD2HIf}RANjQa^y@Cm=3M49KsMe%MuuR!u*r{+{&H;IW($!IM)m7VX5|>Qe&8}Wb
z!H9C0<MUt2=wX2#=`=#rIb~Si)G#@?9Gkn)A1=S#C3@X7rGiEDcBIX4@N)t1P`0kR
z)+rG#ZmQ7<dH<O>LM+(9`uVhb)Zf=ksdMJCEW+eM(BG)^dj0vWTAW0;;A-n+AGq|y
z-p_h0<iiK#Gt5s=`RI_H=+97)GUeK%tR3seS4i)RAvRQ%B2@fYkU#mhT`6(v&ojh#
zGY{R)a^&mph1Sfrf^PeVju8afW(##0?#P~@ucdZ}u+Oh_x^_(dMe<z)BVnK3`?RHb
z&tQ;Zv@@#BcHC5scu=PA7NzguT!K~DySkm5H$Db`+pEmbh>bxER9}s_e3E@-jO_Uw
z3&gt|3!_dw1>k2WDu)2#ofr9Xx2@_UpCCD?KS@qH8pX|V3GOut7koJr>Bpe{U^MT{
zSl=p3tNgrssP6Yq3Y;m;bi3lSfsW<x4S&Y_|DAzhqhbnApEO<<Zd*a~M|~&mm>y(n
z(>cwT5hA)-FXQoxPZ*J_GC0ebiaOFbJwB+No{HV&GhOw6XNdY}J!+;?0%MHyJlD1n
z4k^0kgMX^mU>37&%>NQUdcmKCP3j~gW%bCn6*^e^DBeDg=5~LkHx=+Rm?z?|M3}SD
z4}bgiOAk($7&aIU-zZy=?O>L8Ls^VPfd$l|d&a7R8uLIwv3eKX^+pd9l_%bIrEYSc
zq}81OU9!jmdaLfC@{9U;z5P6(VWQwwkWs5+`P^*5Kf?i4Yq!STgT}+~w<Z>+e#Qb@
zzCT(-ZxDk2JFrw7V_`o#@dn2iWXE=?L3c#kf~9j7b&7JXu{jMn^DOO(>M~aqcbY9(
zt2ga&uYDBNb9{x=GQ9Xr!h&k&f-yJ@$80Xj&tI4P+>Dz-((-bF7Ag77f{+lqMluN*
zE>7X{`j>%aXjm46Vzc5YjZv_xv~`gJ%jbmc99<E9>m$wIdF5TW4L+4Ol(5(^o>9h=
zL5ViIV(JsR95qg6yHV#v8(ynz8Z11O$+aE6Knwn=nXStdlm*b@up(7c#}a-uKt~(k
zR1;3Z$QI|%oY?AIq3fwaa9oO@=dRBaez9nH#A%fij_D-(wbf<+mQA)6Mt-ZNX!wF~
zwj1w-ETNg=qFTWqGI;>#_Jc#W-l_$U7Tlb=aKA!(8*h@Y23PN>8HQ){?AD>*Nw=uS
zq#Ygy+izc=W8)SnLqGpm=S+bOt_{(`B*w4?9eh+ZfWpnWM<Eph6n~@^tK$o~^lj8u
zRonW6L$#bL?O6=^tE5IhgPWD~FcbVqqaIM+$IFd5fPM%p<$ZS^A?t+r8M;rFBZqY8
zS#~)4XrBU7T#@c(T$kE3aSF+bV*b7R4CGEb$y!=r-+TUZA|qLz1UhTM6+^T9>W{+F
zpI#8*{ljdDN<t+LOObITF}4&*hO`*bTrdyX78?_tLw8wNoZz5IRia0JtKgn5VMR`D
zxns8G=L%jN>!1G1`@GO0s=u+%8&fLQBd(^LvTxna;q-D8JsXjq^}D{;b7Agf28Aad
zKFHdzr&PX&P&Ycl`pJFytn*l!%9OLx<R&9~G%|qU8I+tRrnN*1oLC1MYJ{sTstQG=
zdyZ?f4A&1#^es{&*S}DiripU%EoyRc{}knpB>0ZWN##uw8M~mn1?;RNl|7pkX<wUY
zvv{W+y_ootCdwS?9dIyNb;9_u$BRDdF+>ZIr0dg$%l+6K{PKR$OZaH;)mKfDdBVRR
zT{)E)e|2(OR;BWWT;t12i@Z0qj1z%KnwN~T_Cd^D{4}$Zm8oE$*C%#}gBQNyKB2~c
zBhB1~g#^`EfEh-&G_zz_u@LSW4=he^7((XC^>8e{6hE9)QXBLVp(ulU=lnAJr|pI`
z$tM$EyTsZDy$-V;im{S-S`9R$D_zg)Y^oPm#kG_Q#vu<sewrW{CbhQtyBKK4P)2UI
z%{%0T8zAPm=QL5C%58rl6hPO1=yTs`hn<8_hK#bL7o@6#w2jK|N+fqQdY;VEvNMhP
z(J`bIb!Zg66hlxi${9<7n8;1Ff2fO(^8U*$8+)93@rVwP$9wAt5`16)*1aSYFp4%v
zX~M-@bsf{+P!eU_5%6~EKN679-y?Xw!T(`H2NY_4R3{BNPh=-=x3OgZn=)1a#f%$c
z{*BVH`;;tu%O8K1_J&uOjbQ<=5t66z-g7Hf4^}Qqz-O;&HUb6xM4o9z=>Y2lJ}>y!
z4AghlvbVd)SS|TA9?kecFPN{xxd;62(tq;L$ma+mAmWtS<HaHzr+z2FL%dHpBM8Ft
z$ajnR8H<e5HH0K&HwXBwN!!6{hB>;HpWfsx&ZKi98<a8v74oBMwGBgzI-^%gqU>&7
zlJb>!+31Xi1#I=oSEa`LHpMUwKC)W&!S(54D(G5S1gysC4kGo<^V(-bUj8U9;fYx;
zrwK`~J1F6mEatO*I>klh9FOv5j7W~9wXs$&OoduTgasVFdkrH_{}8cCZyG(BpAeIb
z2g`dKKA9S$agqMID0EyU4SGM^xO~uf>(+onr8VKZy4bpv)$5bF?7QW?m3cVywxPgg
z$d0dYr>)#YKmmcNGk9&6IYX;|c$2?sd^}FQ^l%P}@+M<>F>N?h5;cGHM(AgG9{XCs
zQ5I+`%b331X1@p#dAv9y##e#NHJU%#=xA6ki7HMOGc3J0P!enNxc^-q+p9I`9=$Y&
zndIN0tfS*vKIil|3$Qfv($!1<d6Xurlw$o=Jbmt+*Ii)!A@v>rmZ)%S=(Om3z9)2a
zW01OfpOG|+;UV|&^)28v-|*Fb5qp8lUq|0^b@*ISEpA%4(B|6}3Xm7u-7HUcTW;fC
zru%g}5?7forI~A`3D@|%HIv@mrzd^s;CI~|cEz;xM{W7enZN0pkpvsvc(0?E`I>o7
zq*Go6yMpvO@3<Y8rpjGl-AV{{gQa9Dbf6-cN^-X3ZJhaW|8v<yVzZ!vFOg+v3q5`B
z@mjKWy?_?BLaP|Z>k%56Afj%URi@FlIvz4*KQsAp)*77GcnVL*g1M-o#i!X-LE`yb
zkuqRMj`2tskP3eck~JbMM&kCWCg>!;jrc31)wT9^YXv<k@jjz>iU5`0cNq8auqjS6
z1b@23oJ5|kndKLI96SQ&*3OXo764UDOwJfp?dTbp-SPmBsEtQlFO9D|sjR&DqkP7C
z%Pv>Uoel19gG5I_(tVRM&qY*4y3$pI6r5oCUbXHN!w$~?@}Kn8JxG|(7qEDtjm&hi
zOUwpMRE_4v1T|p3kp3z<uGxu4W!6v9AYO~jvwfh$sdx?0e%i7mi#Ue4STkCt1|HfM
zsk{Y-(8cQZ4EeA|`<LxqS87MSEPo8R7tvsF%D?~AV0=LruP~`+))d9YhdrWJ8rJC2
z_JZ&$E|yXnQBdty&7N`GcYkvtM)?t76{sF5K)A&1lEvw{T}pIWR~vdxxolV*K`>=3
zR%`RYahe^uAtt)<r`pH0C=BZ=E|*7PdU_?vx0U1TT@gox=t;xn1@G|>t%{4iX}^sh
zR9@*7KNHth0x8;*S=t3YS;%F7*Uh4Sj;O<LvXRZ)@N4gkqV;pyx*xNw*VuF?5rw=a
znEl*xyi&FFAX;n!qT`gOt~Xfq3tz!=Z%XY}_rXcyRRuXxpcf5bX<lT<DV!@;OadVe
z4_&3jW0e{cD00spg;)sQQ!M6LA@6<WW!*$(>pp8d*FCAKe%Zu2z@(Sn_z4;DP(AwH
z(!GKj^RiRsd4mJZvOaC&j>&~m+)B}2PnN!TyN%^mORsQW<|XdVw~c9*`Gf6I;imEU
zO5oJ-v~N~l+|Mi1Lpye=`ULmuS(Nv_k{pN(yoQ6ImAoSp>{=3Tvs!-$>bOZWyWk)I
zJ%wqtbiCR8)u(NytGI4s6<S?E%h80n&lrIc!>v}s8`=GX&v|ppTnkw-+NN<8Br8D4
z;rZxy$tg6S%Jz!`{8bqZ0cKiKw+~6Ck5j#t4J~~eHoyO+3o}13|Dc@Q_l>6v_-7u>
zzW=`HTd@KMz6Nbu0O0*0gx(o8F6VCOINP^OUrZN{11R6atAc`E(>N5`qYjr#CjnVd
zx?e))Qm3wgtgN3@8g?LV7f$SJbRd$+z9N+Qamv1pjV*&Z)+jSqWsiU;dNlN=7vVcg
z^J2UR5w(e&ApAnXG{T03V(5<W#$ag6w^LxL_8R=QKK$-r-wxjK!u}<{Yde8_D!hsi
zh*<-?#S{L{aF@;j>h5Tp$t_g#rQ>1F`N%udT%56etR{5<L<m7EM*r^n*{-@iwgP;_
zhY>ybH0`R~vcQ>8wN-KL#F*N@d=d?l3}+auriSF?av(bHwa9q1z%%96r07u_-p)Wa
z^k|6CHY`a!m2}p9zKj)(;ZV(vL~?esUJe<rDS*<yLcP56lnnCvqe&zY2I4g<fpLd8
zqM{YA34d&h@A!4?Qo6Q~_a`1#fV>b4hE*LyiX1?des(Q;hEXYs(P%8~6rv&m_zZd}
z2kp7-;T~OO0&FUuGUio{nm)(n;yQKwbw>fJOzdVKh4XNyVnrJoV%u&l<~g-vkrRlU
z(GXS=-(ZfQ^FP_o$@ZR7JFa_X<G&zPOd0)&f(=RakIaKsGI-07oy4v!11o+_M(NDn
z$!{)YG%z{7pc`p$*zJg2q9thu_H=2cGJYP@_PWIVeo%z_4TVtzSW51s9A@>p{r#iV
zh|(&WU~f?&DW1tMWrs4N)`fO=e>f2PLNnxw$aA;CgCKxIMSy<rOotVr@|W@;C0NeU
zJP6&n><=2H6s_m&X582|PLaH|Sa4(JxUqLDX#5hn5U20p9J=Ah5w8`h)vlsOQSatr
z`|+QNto8x5@^#QqT4zpH(y?ny#T@0QkNGN{Nx{Lgd`6p9aS8Z*n=?dWT;KxY7D?+&
z-#Nzl(eGhX8&$ajBk^4++d>t?)*ptWw#Einck9<_J@hE=+X~2s!YtZ?p#`5mW&=mj
z_c)Kevy%f}CW+r&(|(>@gGzjYS%*{3*=P*EP(|GkCV|EQ?tD(Wc%xk|o6%gk-al*y
z69eR{^A7b?Qnw3J;*72LAFrXS7yZJ!lD%~fml>Q&8Vd*FIC7%-nZZ0^oZJbGnxa`J
zwgL@QjA%H5rp-1H&moBW0sh#nCL%2#tycy6B;&GDas@j<<s@BiYD+qp$eR~Q<5EL7
zFOtj1il7%2Ua)gTUXcpb!LDS+3S0jMor@$=(*m{`B>2x%f}ltRxyQ=UhL(cOrfEHv
z<gp6v(R!Un<y;-9vERAqt~&wS!194A_{i?_OV6OplioXL|AwL5Gp~Qp;!_t1cJ3Fw
zb8CA<(L=+R6q7#6;J5ns@iVQZ`As)LeTQeWJC1ewq!7a%mfl?&zfYXbnbKTIeP6zY
z*RY#oQur`p<0&R@b2mLiRj=+P(dg-oRV>CfAGg5g%jLc+41#T+H!4r$P^u?aCF(>(
zd)fEOT>WNX?ob_lhW<)bJzBf>;+JhberzV|=~e;jy!#{Y9D8fuh7icu663T%0$zo!
zpP~961Ri12g*jO_a#_aZC_hv_xHldKaT}*_Dfh;o$?U7<jcH1neSD$1h}qs!JC38Q
zVHrNGa%U!sP*zqP?4sFDWMkbOuf}|Pm9Y9TQtrG*n*%{eGI6bB7@6U8-86PVi8#^M
zPb#SR;?^(rqwEf9Fh%;IZhcqZp)u+tDN3mdc{kkY=X;XSHebY=cX}!z@3ySOKBR;|
zln7D_d+~QK9_)?Q0h1uC?KgNyv$w!ki5Ye+sv$Pqo!Sgevo*3M>&+JuKhBp1_m47V
zE6%Ry1P4M(2KJKfW46$%vtIh<bUX5htzo3(Zc1{g1wP9QWedKa2`H4YE_Q`)4oZ`0
zC8--lKqkuHAyS%1F4ayg+$8(T$MMBC`wv$RTrNq6WA=+lHJyL3*hv13q=OBUeB`b+
zA+3Y3`);WJufE<gsLk$q0L3*}3zQTqS|mUd97^#(f#MdlNT3w=;uN={30~Z-5VRC6
zp5X43VnvD<E%fsK{xkQ({ctCfiOfE$=h;2a**$w0Me}ji=gf&KI+jZW49U>izrwa=
zDCNZ%X`b!zNs5|ew$Wv`G=@s$%W7+VUHVczM6H1Ah?Xo$mg_09NiJYT<5^eeD&RNb
zz|~>*`(s9kk(HCprrA6Pn>D{x<#YnH8*T;yA3*Jm%nASZ_gC^?#Kd*^z`JkGX=x;A
zqtAT-w>3Aem+O0_E77(IXi1D`M*UG~>t8}izt`3-d*tF*PiF_VNQSgNs3ODM_wvwk
z2N~Sx9EXoj<;O@n`#40<fuPrFO9ldH8@`s(vf%<y9g*xPVnqlZaeGmvQGba`lb146
zuLs}heZK8^==N5m;_Y>mDs%rwv+y&BwqkWpvraa|U#cR%3Q&sSJ_~R|^AAwGDs-q;
zgjr{PMg`bKPIeH$TWHE<pd5WH%MeL;8a6~PO7|t!8~KN7=r+@2l&G7|Ts97zO+s(E
zNliYZEmO2_cf9`iGc!%W!*I|^NIJ|as51TIWv_EXMIJ+jw!MRdf0`|C)Ng(V`w^tl
z5&{--+53#`F0_Z8CIS(N&m}lQ&UDU{sciD~<MNfEXkeyXd)tO%WE7%(P@t!~7+l#n
z8E0H3LwV>HKFAx4`iKt$wkxEuH2#2flZu>1&f)wZ)@_&;F4R<_T%~Yi`bbj>P*Sqv
zl8iGy!$Mc7ez%y&HmNnj9}HaDlR=veg5G2(rj>E=-q&#kkaE$txHPx0G!;4XAC$Fv
z71*~@lBw!dv=wvdD8Y7wvt)oZXJxms%NOz_8Y{2d22$;&1u#d|1wU(Ivi5^$Jpw)Y
zP(=IWQx8^F7FE%@(DQ`uD*diV<IEZG@>>x7ncp_-nUVAKurKKd>KRKpDLpHjHmwn1
z^~3P_dkGq1PUS_7+1o94x9H$Y&5c;23t;bdbkh4BGuYk0aEtb=PHIkoMD_eyT8K^Z
zo6eaS$c%s$f}_A9^yb|kOpGQztaF(0?VZeRq8czK7EeMD<TsCCJQ`D{3br6DdrfZM
z#FeFGj&^wz1Mw3DbpEt=<!%aG*ganvzQKN+cyLLLES~2Mm6bPH%9_8c6H??*^x;CQ
z!(d5aF(loh?4I`AeutkpeDV%59G6xStl`0^uR72_ncMqm_=a9F-SNH;3J^h0o~ZTe
z<o)LfTAjVnMrjNj=usyD7PQa?Jo8dnY-C&%4=&meQiRt{sv|a^G|pXlQ1yMQw?e~M
z?=YKrKXYm0t=H=4{YeO<rtw{ZAGZsW`@6xcl8lQl^?gJV*1Q_!H9vhD=<<7>!LJaA
zL-+UK_!f|4-zT%z&dn5`i)WvxC11R(%m;u)X6pMs&~;w!@4W{XBomxt@-A_znU7kF
zAN5aIO2VT{?d$#k8TGVz^t3oZ$sE%HIx9R<@WN(Ce7N(6Fb5oXIU&T?&M2-nhoN8a
zM4Hqyu&wWxr1U?lqFdUkzClmmzt<G6<a1}U#7IhqVQD~F?hUmML7C1fe`o{&=Dp_@
z1t7eMc=Ku7L)NK}<g*E^GXnAfkNR6tKM?tHteo#Tw7kTWI{_gV&q@JqPD4^nv}{@t
z?Y8@NcVclj$dcwj7j$hMn@k#X7XP@It6qpCxLerfQ0KKU;;Sv+a%nNB5w7wK9=+q`
zV4P`~=(hk*quDG$lOs|g^Fp(!O_%?1ynM6%e;jXH-*u7w)9H-!;hx#$?>JS|nyu6!
zG&o<m%rd<+cV*+`^W8!PiPKOC(&ip=(t0H+VN)3ob7W_H>}9(!2fN{>djGVlEXHun
znjA<?S^4QUhw}c=m(hksn^;=duHmNqsOrj$S(7F&rHU3FU)ehJPvH`Lx>p?hk#-t4
z%Is3y@N`KPm!<J^v8Ms9Cp~Lf1xIKV)9Bk)_HogR_OT{Vo14W!27g~;P-pt*7dolF
zf9nt0<jzOl-VFxQnLbtow(?(2W0EP*FAV#P<Hofn4@~h$xMZ?lbQk5Tb>)Mu6@YY)
zfT?9g66j?`(#0E+Ivp?mAirnug59?1J+byAPf{%?AJnAq(O-ujk?4OQ{~e{!$LQoE
z$xiCE2a^(%OL^R9tvCv&`n&w9!h9fA-B0RznD;BHW(6KAy4)ry9xJ}}A*8zz)ADM+
z%pZC@SQsor{OKDB#=Uz`quD>6WBd*}T4Ul`_)?*+fa&m`aWlPTVe6lfU#&rw^s+#X
zWW<o&O{%Z4@Gs>Z!^Rv0ZIP_pnRZ@Kd1($fzUZt>h$*2;T&PU-56NCMOj<=p3?5~*
z%V)1^WO!DtafsR1K)hBBydMM<0B$ML{7#=AvZ@9~QY&V>Gi$0DD{IIDEQT3NkF%Dr
zm;_;IY5`o(Rh0z<>A8?)MqqWdN<}_^P-RJnr2;SA0|lXjm*eLYXPp^iDve9H&WMN)
z+(_R3FOd7|E_w%61|)#P59z1%g*US<X3Ngs-<F*(gG9{w5gCD9JmYL1QGqgA929=8
ze*tN_Tz8xU#LigYfp525DSXgVu4g*M${X^!&DRa*_Enwc(7N&SEJ2T88s^GwJUBvG
zTjbnzx2ntNMHA~!=vS^_tqX%TYCQUEubcz8P`yuFv_{)cr`cp4ga^@N?<L>KGMAER
zN$!pk6EU^V<c&qOOvGabp_Ri#y7=oYb=4kxO(+xoV8#rRO0<a<fUO3yGCL?#>sw?1
z&o*%NB-3d^qoYvW7aeAEQYlVR8?7J@0sP^ej;W;r;X&Yti+W(L)rcQcp(YQDvd&ub
zbg7qJKPKCZcYRqPbhsVI{`+Bz4Q`%2b75iJOA=+oBZ%7gRv}=58cvOo6#f|~4YztQ
z?N5XcTOCboW569mVvl4*6`N&aGfk$mj&1S4hBUIcT%i<mmDjK;G<FNO#$W#Sn6F>9
zpdxnG1f>7QCb604J(!b=M$lYys}yu|2n<AX(fVq>h#Wka=*_}Lb5WJ56(M_F=&iWS
zNQ9zX_apDK9A{lWRC^dAY6D@W>H_C0L63yZwttJ(jo=c87Bc00<#OBdvhy?)_mI(T
zW{W)Ke6Ux*lfGmnY^`hNy(Mwy<o96TQuSqI+ZrlR6SPV1ce{C2=0xlwnfStxQZBlM
ze_!-;%A4094yd#NPwD#rix4Z;Y@_C~o}M`k_)W#J-Z*KYRgiVN-f%AOyixj<Ifd8a
zMma*{^(+1rybqo1&wGwc#p_8BWXs~GuBTiwvo~xS@#YVYT}Tk<`YKGbDyv9o8mP5m
zW<rHcFUfVrVTs5uyG+EYmbx~Jdrh2B7)giyah9Z86}uiN!FM?^#$&G@D4iTAHK8nS
zdb_S&cw&7Ok4>h2b|)%IXp#;y=pFg$3T0E4BApnuDx!$ANm{&TWEFErWM30^T{i!t
zo%aeSRNVG_AZ#C1#A@fkY)jq9B@!q4q?bN{1k-V~DyN)bQEitQ9H8nFwMk6DZeM#6
zzJsu6%l@O9uY_*PN_myBy`W{0s5$(X$~Ax5MlY(=-zI`{ku4`y&FZSD?<(eT61NDj
zpFT%RUmJLSn*2e2h;b?B$?mB!J@kv+SqbK7%pz9z%!xK@UvE-Cx;I2jLHDQB^L1xJ
z(rb557LJ<p2|?9od9hq6vZ0|aPfc5>m$o=++s9rs9#1ngj<VjVd5^3|P=}C?j}6VK
zje6?8BYUzVnmCgSxg%zut0*KZ8Xgato-xrKx{Aq>F>QKT`cBp{o*1oSg%1a3z#GL&
zFF!)?b~cOX=zb1&XL7pxsE=kb#=5($YK_%nzE+JozR^A8ILUMHwN6u=+w|Jiwtodt
z1n@mnqc^epVeYw0pI%wLL^nKvIBTssV#&+HX0c*X_H%8S^pW4u?jz~KTZNi?`TVA+
z=g&qdr+NMOBzY346{KEJyjNEOa<uy{MIE3w1P``WTHoaK?vmezXo(ayHC>qM$isBI
zwE|SK6QV9>qRGBd7zlV4fUXa~j!b_9*h?+FtGxn%xu&Bdb8#<LG$eE)eNFS_<89Z~
zGP+f)`n15=5KmYuBD2_r{k=h_wG6!49$fS039^7o%Dp+yL4V+r{)-Nb29=CtfJBWb
zLbF;SU}SODSTn+W9_JSg+io<6Yky<-a|gs$rWgU9@5*oPT+q~Jhe%%vy>nGajj~K5
z-G?Q_DcDgVo)?YT$x04?gB>2cnJzq-LrmmzS5^e*Ty1W9g%!?mPWR$(=65ZWF1HvP
zrTvEJoPybNg{uhxs2z7<kux5bEJ;EC8fiDzzWU7l-wfwhO<K^cU%j`zT>s`KA0F{;
z?|XNhf4Y*}`sRqcvodu*R~gOTbrh<lEWv#EQdKj@EiNPXPq#MI8K(mk(K+#X&h>s^
zCJz0&(2b5E8%z!`fO{x^*@b;YB&DJznK5czVL*bQ5{GLEC3Nab|D<v*+_!}a_yoGy
zD3<WMCH)#8ob}Ch`X5v0=x<HO-F`+)bPm1L!^XO>C~D^qhoU<%w2WPLBit!iE<Wt`
zTW$NY!1!%1KC$rc&^zC~Pqw5E2v?&(sNl)0U2J%~-#yjlp$v@HzUXJGOY|rr!0)>{
zZ2J0frABTYm!#z9#j5MpqH~NKD0{B5AoC5aLss$_Tf&pz`O~wkTJsgD-EYn{&Tm;f
zukOZ2J;(P&+Ie{cDCfYw>OKJ51gS;%^aUj!{X_D2e2QqotTsm`DhD+kPcNH#IY7HB
z+!zAc3!lj3?1@I%dcCwQ1Q}J-nY#`{VUxHz>W0WW^4_rfl+G6U5|{=9g2;^E*}!fd
z^~ZU6At;b!Li6H0^NljaZx52ZjJ$(*`&SO8GIuC{O3dR69E*ftjyfRZw+|y6A=KJc
zzo97kPWxeXqCxHsl`#Bd^d%vYOvW3hzTG{%vk^e@mx|90prIY(=f%#C5S~ig%+dci
zRs3sM!r76iz>I$J>_Q&*+6H_BSx>%MYx>{X44-`0Z3pFREsDWzjCvWEI*=LUZkJnd
z?sv+K{q=5UIGpnj#!|5pg!)yxtGwW>!Qfdkj0<$HcO7L@(A^aPz1g`d_upJK#X)w{
zz8}>3i4(5lj4tWSz6B%~^x%aMhNA3A)BfuJ*iY+wipY2*&<Zk-gTUv;j9uj9GgKB!
z_v%h{un|(m0_ZaHrirVLr(!|o<_}xr`e~Hve>D#}KLnIK04U_eLZTXh8yPoy<R#{V
z>L<T$mkaNF9UBSZ&0+4f<FG}`D+8wQxWb1GMJ~bDt+bzi>Yj*a`l4@&zWix^G)LG#
zS2!8N$Q}F5YBg|xb|d*9wE&${7(q8VE;{{_;_S&1m$4jzctn4UcH^9d*-3FSt$NES
zZ+H-MOwdGUPOeOZyu@U_`Kh2tVm-|DZ6Oum?NX5FddF2L@g+abs7Fn?@v-%}BCn>Y
z>a&^9KL~t*-l68&dp$%q)p};E?5`q{uRI&<wPD%xsb+=#^WXq8k8v3C1mi}BTNa9X
zNcFLj?k{EhVUwCoLZu5|64VFE;Bmx3-iMYy%*LV8eMb9Ax-mdy*yIz*CcX>Wm-8sN
zS2vfA*C7yIBr)@>J4G_*u~gaupK)Lang~rD$2F9Lu-8PZXl=Li&PJo2gb9Z<(z%Yl
z<X$1^!ny9IfXz{xu7Y<&+i#uNZ>+LW5!40D0)h;!VwqHegb8@i_Jb=H=ywEPK*2M#
znrOLv<?y@6C1^v>y9Nu6eb99tz_bqk1lNAC(U*nn2e;}guGk@vKEn0kODL-r0*Fo?
z<D%CVv(duf`fdD0C_0;Z%%x3<aPLPJfCxq;+ByD_`r|$9xYKzVTj!zR1~Jv*%nA_m
zLu(NSQfsFEc+J(IlUG%sn#S)qvcBXrxGMNNY*(Z8da!T7&&90j<)CEan%<=uBQMU&
z(mf||jY#1rvoN!ib7@?S8OIjUPPSA`(QLFmS@>FriG_iXIGl`Y^BZ~Xrn@O^Z%&!<
zAF3E7oYg`*u}EBj_^J3Tao>UsTACfEVtHPqopk%dUn-%f&0+i#Of1CM1GMt1J`j<a
zr^eG}s0gG-62N~9JlxebphuN2#Ndf$lIxUC8M|L_=E!-|dS=kc8i)x^x2`geK)HyQ
zTkbGi!|DW{V^E+>m>f619>OOqn=&7p``zEC8}`qwy}r6845knMN|}?oz8e7E7ZpB7
z`t|y7vn&Cs9?Q|jo}GhcbfDD|hNsLg3D8%ywVd4Q=D!A}AV&$-8}yf{u`fz`F-^bT
zs>5(eu1N|f7k%P*)Q+ZNT*t+jh+D$M=6~aXL|<#;K-+2zY{bY5cYWPPcJAO4H$GYg
zeD5P(ER{`c!plN}htqmRg$Ro@UoZ4NBEZ|v6?S5oww&IF+-dT*$LiQcjlX;}u9`G!
zRnWg<uzWQsR>hF=ZDK>ZPghaxdCd~CYev43;14u5$%I%Fs$LP-QJ-Vft_K?z5t1Bl
zTjiCq_hK7ii@VPIH1Q#1xy8x;*|{HNPfg-@LoA-;b>WkOD&`9Fl9P)ez%H5%`*j%;
zvD4F53k(T(gQ?DY)%hape;FIYKf?9yTebIf<TZF<ydlIP5H}7PAefVmPwi}aO}CKQ
zXB&pXUB6U`#4nhGFKd>h5wTP}jo3@dTT9O}O$cOsF2!2o%iXb$2gEc#9qC45ac*F^
zUK!vZl306O`lF{xwZjY0fH&UNu-%6=2`InQAkzSH|1ja#oz|3kbiOElHqUyk;)Tg}
zW#}N7&T?TiA->`|`}!y^?)<5mMb}8!({q4gsbDp=thDxc+YQ{p?mdgC4VP(2v-7W8
zIJLM!0-mzIkN<d-VF9RQDd|sCF<0dhGC24D<Zwxs%Bi$b%<M3rZQI(d+W6iI0;$)B
z`&1_VJBgt!P{MIuCD0W2sC58eT!Zv_7pu73T{wOe0&4IMfD=IRbf*R{?Z)zN2GHn`
z44{wq$SX7bur2GzHvk1yfmzqph8|YKY5~FIjyF9x^@fcdYD32cI1<bIRYa>o*Gz(8
z9;VHds#A&Hz6=q|FW9@DWQT9%KE89Xi1e)xEe+pH!kQGwcUS!Y>s@bh=~(&Gl#*7%
zQ1pK+4DYA}Lw;_Isyly}6u_OUgm5ep5!kBY^Pt=IC+&?t^T*;6_lZ-=Ye5Sq`>Vnu
zjT<Y01G8z^6?2e?&PO`J^H7+bH96~tDYTng;6aPA&ApsWhPaExz@I?^#0kEw#Wi90
zK7=5lM81F?NHW8Qt#(Nz2I9LW9OzaFnpu^SFXct=d2swFjr(u-{mtJ0j{8rHgQvQ~
zpkIiNS<4zM<bL|av?Z(R+j%&b4zIaoFtMUd;UsS4%D0WxoRUwdM=2=Ep^KrHNn`yV
zKZKfUYmsS{n7Yu`mjpF)=h*fIi{MPZ5H68l)z6WZw+Zs8qDn29fD>d5{1PJ!`2w;t
zAzndZ1<`YgqW0Ge3&r)F_4-4A9?1~Phk&U7Wc-1c2(-92oEq0o#mhbE&-$TK#5{j8
zc|Luvaxl~Q1Ay2{NI!mO3+Rb&uRPuO@DEX0^!(KCIc$nK(rQ=DFjF%PoE7Y3Be6sZ
zi~9k7s+`f!S{6oeW0&P0q4Q-pzJk%TWO(niCN#q2O`V-)XH~ED9m*0cZLR^Elz(iX
z9>~|dYI*MYdvga$I<54Cmg1nr*YpjHxj?P?fwpYj#H9RfNIiRRuloKAxY|?Cw&AxM
zaL)?JjmO^QvC8kR(8ZZCLTkjXf%$uFyL8Qy1I_Rf^oN%nBF4=f!+x!tf&n;xFxnqF
zLHbI-AvgCc-udG|Ad1W0^JyYCy7pB=pHqR_w;#iuVi4aq_rsy|Ga-Ukk0__kaz($(
zO<uJQzrBM%qiiC~OhNBUlwRcGneg!#eE=N(3f>vjhTs-Pm=P)wc68mGqo;HGO?Ef`
z2#ME=A!ym+9lM}w0$E5VdwZ~xNuWe3*9Ph6yc@uI%k>ILqO;BZM)ERrpYWy@x0HCQ
z67rYy4bsj}`;QJhLaMHwXG1Hr0+LGg4I3m<2uLM)zCU0A1wNL78(x)HF((TY#0iL5
zAto%%W-Qs!VSlVbsA!dSM(*Bv=KiaxxANe6huk0&l~N>E3E7`%b=%oXEi4(z4zidB
zf>?HDU36`{_LWAw8^sm#B0<-`(vVSOst2im18O%l-qFF`)=S48qT>`u>`X`G@7pDP
zowHn>S&9O+@7_sNw8<r(Z&^fr^|sqIwa1XF<oQD>$P&_>-<knD>MvsZi8e_kzt;AE
zVr)*H&cRn3Gu-eps+f;!Qf%6TFr6KM!pRIv-BwwImY|3DClx`o5b3Bt;?sStiI|#R
zs(%5<;qL9E^#+{%#te3xIb8W{O6_I(Z`22JEn%he20?sX+eh-YWt+L|qwi~cs{Jia
zA*q%T;YGwdfnt>qCg@QX%~WN-WfOuH{O6G76WwrUa5v#FEn(0%p@@cW_K(h9!k~iL
zg_h@8av@Ru<;wM+`LaF8Rlz-zMXkdw6%tU>)YdFb#x2r><j$qCFG1NTvyE?(*H5}E
zl+a?2z&+|spFG)RG%}_v3gR$~#B*o*;D563S05aq$<v@i!dwBWWy80F-myb}EKm5K
zuOq<3(U2(FBLR2Ho@hwG)N}@ffA5vB-$P<pd1?>BNg(Rxl7QzGc#Kjz$zgS)57y(I
zRn2AoV2l#ApF|4Obk%$~(F*4_^Yg&Hes=6rg!;-_%S-_G^ESJ-r2mKWR5s%S@#HXI
zIG2HCPI>LdP@%UsYY6-ipvL%b*UzBi!<fB6-14)tQa97P8!HuS4V@c_`6auEJ4%ko
zaRbeZ=^|c@c%A5k17=@_Ghs07?25*qJ38dT)A^xsO?aQ{Az?GwdxtAD+ee8Iw6rND
z=f9Mt;PN~0zgPL6F4wQc$Q`{Qeq(v`!z`au*WdWBv=<IICwL}N;1mpR$2|Mt;!)c1
znb&)j(!SKeDS<=_s@t5DThoNfb<MXI{QKDd{z42$x3W7}&^7>NItVdj#X;`xdCzno
zAjO6%{Xk17=9YPK?oNWi^0U;jfnL%vLZ1ZB+UKu4*543oWUWT7(yCK8)XV%AhGe$`
zUBC^kAh{?B94cqCwKxK{hqy<UzI1t0Y1=V(w`kGM5kWjv+<#`~q(Z@jdaeA5C<>Cr
zi?d`wGH?o(gllp0YtgO-RdPyxzhZrK0nGyh^L+kLe?-%&%Wu^9zWAxI;0(*D{WI2Q
zrQ)FzmWZ4A9K$9$t+3s_`a$bio$KO-CAuEDNo#lM7XGOv^wCx;z|%Ew&dCZsa`}}k
zifOHV?Qi*ILZIq}V~N~rWpphaCu;~78#A6lM!-<4gxWe$tyV-oJNZ!B77u#URmd2g
zr_7#as6&x}4i`=S(7PKf_OR?8Ir!ycwZSjHg;GB7AB;X(=5@wQ%QX3Dq$kWUdpqzK
z)Hw7KlV^Y91YBDzh+8I5xNA5V?K9|XClqE3Arat~*BN6sg}x$cK$Q1TzhX~Bl@kNE
zt2gLnfXMwqKs&TRjZr(0Obkx_P@^uD)%>#`=&2SZ^qq<L{_T;2nKPh3>=gw^Gqv{V
zBT{D=7k4p{&+7#rSPLopYh{)qYu@lj_6O_*MF<bksxWPRWglbiQ-Sn#le4QUuhYy*
zzxe&Xhr=%g-LFS319!5;$9&|n=P5KZ+Qb#4Ni8Sit$ZZ)G;Qo~QLgTrtNE`Jn4PbG
zt_)^1hoIVa(UOe<Tv^uQMC|DI?$;T=%VwGiL0r69-`k%2rG<Z8^@_uRi#RjGb6138
zPvh%EX4(kMp+;A3zRd8(`h17eQ2*Gydy}Pq(nS2E*LuusBR>5LbC#=+c(e--*M2G4
zyIGA9zv}k-C-a;Rqv`d8sqw~je@A{#3BHn~7SU*8z{7!8+nmL0wtY&S&r8?yOC6DH
zq?d%A1*)ZS)%)CX@z%s&e<@*d5`~%Jo&|0u&7Nz`?q#2J{M{Q(R#-o6Ee^j-CeQer
zkh98kDyldT$o{WEZ>`7t!W1JwhXy&yl*EXeYw7UVng?}wEN|ZGh;|Hpt?vJl-5QX+
z^^B6sUQ}bTj?eE@;NmeoH3Hc{pfP$NwhGO-UON(c1Q$6nKztgTTQ&7=S+VvZi%|nr
zKj}QATl(yN9|lxC{i~y@Z54^pc7a7bzLh<k)W~eASZfnpOp~QsetdUca^49)Ccpkx
za2Ck!me!FJUJeLiB#7RUCac>Nt?Osj@fz32UY#yVOtjINdLj=T0!aU=x1oJLEtn8R
z`6@`!bkC!|_bMwG=-*#N72&ETmhx$=<M$$!tPM*B8tVV}*20;XVy8clk{Jwr)*TKj
z{YiJTs&Oy-{BIwj4HRF5gMiIVf7n`OzP8b8^A{Djbql=rR}z_}G#0TTyoeW)gJ(4J
zDuXJ7dY1AmbC^in^1IJqwT8`X*5}FQ7mRgw9)m6zPAe4a;%$C;70<sCi`JE3wxIUd
zdP!+L7dQ9kd;4#az?IO-%z_oWXC@F9Tn<MO2qI}1p{Gyt<9oK7gx{x5eX1>|#r@%d
z*xfLLHl__L^4#Hr1U11hf8&#<7f&FKa74LjP6<R;e*Pn%Z-gm7Z-RdwuyOwq8NI&f
z9L3aafV3gp_GhsUW~b?UXVKpFm3;jl-9MCjKB8ZmSNf0It&>?fjg|8&Q+wxVmV{wV
z{KGCHOHhVU7~U|Xw5YxGPFU9Fghzn?C=SwR7=zcMCWz)2&P~$QEScKR&KAA1w_B^2
z+v&G(0NIRJG*7Yl&Q~0hWaG&FaPtSvPW+Z_i(v9K4O)z+Vv?*4lc;<c(t%w0^~~Q)
zT{2PwS^xvfMoY%Yz^u9hnhYw!bd?m~h>~1%ZWqbFdr^iP^Y<_B$W694-&KzNtTMUq
zwOgf7Zv*f2h1>ic$ohO{ue#Dh;Xcs$+so_;Rw|=ajqlQLGrJm3iO;JoDaCfY8Ys9C
zvo=hQ>{Ow(SD8m=Oj}!uk*9D#SQ+FoqnDV3RY}%h0VIB}2h}}-f3d4+Pfydj=<wU0
zz?8<G@w*LuJ`s>UkZ3)hG(dQ2Wcy|Cq3l=e*i@#%=-at{M>usKpZvjjh82g;R}IM>
z_J0yb%qOT0^o{aVOdf!PV9*dAeE3`rztgLW8czYWwxko0pqXeBTmFOu=Iy8Gs)_@;
zPNA*+ulKi#(m`VOYIT{spLJ9HF#x)hHTBo|qrU&n1SZBq^WM1ZkS>sNV3gm|M4+xk
zYc>s_BY*V_V(RPC_8~D6ia9k@T;FC*vnJ9>H*1P0cNn}*-lcE2xcENaJ+WBe5R5v0
zP2PJXe%?8S9R6XYCT)?n5D$L5FkEUMg=!OCZu2>-awSJ+<7W&)1C4w=3TJ_Tu1z>$
zA|sA=eng(qwk<$;gL2onNyW%VbFKLfYq4bQ+qaAx#NT=sz{%g#^5v3&@`}lvtjQUy
zthxnNLGK~B4Fumsh@^=0fvA1<=gIO_w}9M(w+GZA?|r0wi`RE+l#b$g5qd|F9{!11
zVZ6{FWiR(!D|t*+6hA2s?jLwx(2$HjGJ@$DkNxMgb&gcwUq1x>SA){Cv}|(zhMMnX
z{;ZJ>lFd6%o!S?E+Fzw<y)BjVxoif^B&}gH^od!^%QQQktXchPb#B4)oP@3VZ$#oZ
z1}Js}yTUx#J8I8#<5bVKw5H+3_MQ>3^Hzc3AEq?E?;a05k!1`NjF_P^+XTDCdI=UN
zSl~wD+TOp&xbs_jcK*x|4jY7jAJXTwmrY`|cs5U*R-pGnr3a_aXrlW4Yk0%(-`wxA
zyGAxtBXmy*D<dGs>FSh9rUFnJmuQ}Bwts*OyC)Ji<!o@uOx~x1oia9ct_=F02g&i_
zmPlVFHwIG+kMGI4+X~bLxCeX*XfX;_{~7S*nDzn%%*K_0(ddd~-Ae6M2G}y%hLTtj
z@lGwtBp^ijy<<J|N*u{rhhl#n5iEC@HPdxaIPF7$X^FG{v_9~pvK$o6LC)h^KD_!Z
z1Fu20wD7|TQhTiem4j~n5BQhxrTsZL#9K(nv*4*L+!Fp%dsu#cZzNwjTm9=y|F}RB
zEa2Ljb|S8)2H&y^9_{~AGlPZ$DL$|mf*yRjxO!SvCD-jO=oYA!kEULO#J=kvH)KBt
z?+Lh1ems?=49}#(*32C)Y7f5_qP=2K4pnIDy84m~QxGI?r;Ui+qP_RzsrM#t#5R24
z5XG+;fd`%_>N!B3eI{bRH=O1FL_HCg-#BaM0~{!O9VNDb10T9IvX$@oSWe87m|;V+
z`+P<}H^{BD=M!5dxWOK|`_6)&Oi``~A~HieN;u-e?Hb4Vi~QLe&6dex$kE@=a4(Gq
z>)br{vrSx-0pN9mJGQm%rLS1-75O7R7QEwy9haxQU^Obzduz+fXEga7kzq{to#T6)
zepx&6ZNwp&jXR#8#tNXGDxk>FOXo?#SXoxxNea)IUUqzO5}uqWf9v;5f!E0H>ayWe
zyn!fNHkZZv*r2C`=Au>`VDxNCLNg&B5t+Hx8^1^?9c}z$;y7()q&$Tc-~;>e%whMD
z(n{PX+9cSrDdAD>TN~r#sE+=WvGZcI43LsgKfOLwaT&|D&|Yl4q0nK^#(^P$UojP4
z#$=&}MpO+yYu9aiSIvgjNb}y-LVoUXkhJJ&1=6SREIoP!T`8d_rq$f8-}5Oy)jy%l
zJ>YV$Fl5pK+N`nh1N|SUmU3+{=^vp1f?Dt{I;U@gZVo=xWRb6PEmE=UcGbyIw-2cJ
zZt!|<=wOoX3sjW)LPW&&vW2jkTw<@f2CG?G1}rKde?`$HPW@0Q;rCzMA5*Bu($j|G
zT`p2L>Lc=L^@h$rt|sUKCGiGV-@Lf`A7?$6PfMG>(V=XB@S860XtOhu@Z4`5+=`%w
zB~S=E=DeV>%BlaMBURw9xm#?7bxU3YxLW0z9n`F+tcK&FU)TMO2f3g*o2IB$Vc(FS
z;ZUU;KKzzF#YofU-rOjicx`RS6u?^mkfwYf;3+7=l&_{v3;>3Uz>u(y%~v?XnlYz3
z=~^aRrq`xO$DIcVPQ8Ai#f-mW+``y!1b4+SfmCcQaR#`<mVOI)L-kpChpXquzKsKd
z4&aZ0U}M$fp;?FLuL>ZfFL0BKo=CHbVo<U{4|t$@O&^R>+YeJya(v<;FG%r~KgecK
zQFmAh=Epgr#1}gCAAs(u;FohRnVWxEt1Z;^8N9XWaZQ)icci|56AYBrdH~3&Km}MZ
zk6rs*f2$joRD$W-G~1)&xKa*9vh&r}M@W&Lg7+wc0THa8XVmzNlZpW39S6mC+Fd*C
zuQIKEkMK$TFKy8`g>H(oe&$`om&QO|R&S{;;zmJBe3#i6)yW|TphuJ1(=@)&{?ykS
zDERn1#LlD%@YI;|6DC#G3RC2|0D0$-A;fP?!0hLJxunf7;QE)Lb1DT|VW`hS_6V%i
z)Wqc8%Vm`jNd#tz3y)D(3>lWEhJ}$5Z-Y+|MA0pjz2S_YiK04s3UD_+r6|A4I)%&b
zrK85@9P=I5$_s<sIf4An&rRM({t0WDkKU$QSiEk2fx*H{hd<O=@(n|<ux)^}sUD>D
z0GVJ|L^#LFcMvY+{~uUh?Qy3T7WGrTet#z{E^21_yjm3s++evAT}WT#AQs8chR!nJ
z;o}n;D_av7?4eQtF)!%9U3&lM6~VJm?TTT8!g3YoHWdNgYJ!Z9)GFnuuo9v$G@eli
zwsIbV=d2<~Kf|a8i(OM2$sME#Jb_>l|5b$Rq&3A^+LZzFn0b~in!3wc8?U{2NTn8@
z$iuNZD?sP9erhv06l?Yw;0d+rqnIqlPOXLGk~brFu!V91#xel<V?FA|4{D_P9Ansh
zTTo+@aI=H_d;}gQ&qs0d+=){oKk70dA9hh0&RR>5gyY)o9@&4Y{=0|Ux!N+8QZu{?
zzNnLgc~+*F3burTHgrv#+v+^u2rgO1Z!Z1bX3tC^^W8|I3z|wTJW0J^m-6n9Pi59^
z@_sx2BIJw_`Kje@H@Kk6OG;l0sNAas47UUhOg8Oryj;NRvDw>{ayMv&_zj3~US|96
zQ+Uiu?-BzTMkYru@zy+q**_y^fC6*Og5<t(7E-8xJq6LzdnRm)e(=(gl{-pWUcVi;
z0EybgEjKA8__L&Cv>flBcaA9*c;wC+MxsG~FmY+PUmz$gW+m@%`^@r(lD4+!_jal^
zn6Fj<M`6i#>f$`cD}n<3U+|3OEn3m~=_$XW+4|5`gBQ0W6wcw2P?CeBSZ(?Lp^nf1
zIN3D5`KWHD@D@R%<`-(wR#lQU@M9Z|RtY41ja>NFWGxiXVfa~Rp&<G%)tV$v<&^h{
z!St?R^^)ej+`Ym*bsdBkVs@*3avlmP96UaPBu3XmLZ3AK77`fALxV(@HnznAbKe!=
z8$WQ`thL6G6L_p_Ti1rxrvomaua<<Sh#Gu8CuJ~RKfg9NUAD~*4A#x|jw66WlYrD}
zqvb@6Gz=|a^bCdQgB1auB_{(Lb#u8IBjiLe&9=W1+|oK7u1Ov?MS+fEtyh<}@lXPm
z3qC*b>(#$yUt5LuYYj?Dl%jx=56!BO6TF>k;8E-NP|Vr=);UQ&B&N!E3>p!XCJP*t
z9!j{Xe|}9(o$EM_pQ}7~?#XoT;_zTZrBY^(GP)66I9cD@O3xc`n(yaUi2F^qKx7UK
zIiXRROkHmXTWO_QKT2AUquQLibuPBmG2YbGVE@xvxg<tcsC(5D5z16CygKpv{-!2J
z=6Pd!#VC18zo@BesDb{PbxG?_t6wi&WABrL;);qhc<Q0g`_j!b>S`Q~L?C}V0GrPP
zzwbq#J=U+vc{R|oLvZ(YFO+~SdCyumquuJj>Lm=>I?O!X|A&k1T(JGb?gp!_NfP*V
zD(AY|BfC@Y&n2wCbg9Jf3vwX^kb4&lkwMZ#coJSeb{sOjYIyONFvb8l(hyz-xX10Q
zJXi;_?a9KAnYoPM`7Sel>uzKnNmyuY?f^FI4pTz34!2%}b4=)|t_pJ9SR?->QWuT|
zCq`lu{pS`Q-dIR%Wy1gWr_4t!IF_bij4?JBex`PfCJ;fQOOQy2%!ecbcyI?p(V9Tz
z5@sG91CF-7M8o0&NaDxY%fUxZxutaehhZ^j8JvJG4#+Gr%~%U8_b+nrXCFHZQjg0j
zv;h@+uk#>@GHbkNwA>K%@T~z7_C&~O=n+|7@J%(O*zkdrgl+^{CJY^@xiEg~e01X}
zg?g?hwW)+8b4++X7dQ6hptnz+=CRrE#Cr-o7y4OsGk;T6NdwuMV<mGh1GN@N>cwWV
zjr~&5R%@cq_sza@_479TSPDIpp#C+#9z)0OL*mlLD#1_!GKmiVY67@GC+f!PUU?72
zN=F-?>H%L{lw3>a9DNbonL_%Ns%oI@6QABA83H3w2Q3lqkY_&`KC92CXF0t1r>U^h
z$$}F-I+R`Ww$eso$5L3ZXPD2J_Q`m-8m%xM%hS1l^yuCw@I?BRD|7SC*HzcwjXU}~
z2&=Ip-1f_dSKzL|(b+5S%eSM^%=x_)78!O&$JOsr?_iSYfzDjGYAuQ6E-~;af_$ki
z*B!4we6|bvsVRy#lF2z{eXr-ouA09XeOMj&<kBL-ScdBK2T9$X*-3Xdz9&3#6xZKe
zCjR6uDbKeg-0f+<@`M4r9bN07HTP$F)Y!fBWf;8pRZ{(^MP<(v+A^<U`*)DtLeXvv
ze8Ihet^X%i>K|46$A%Y=Be_xoxcG%@2?F$b@$Cp;a=#8${8${lBPLz1Fs!?e#D0^Q
zoDnUyu8BMTBJuvfHH~rvX5V!+ZJAv5pFCfCN!mkE;~TJYB_MVi@n#%*?ZDrjF}q7P
zR>R9WChbj0qei5HVH0*xM9S)We>%+jcupLRtyP+Fh)(CNE3Cu}V_83Z8@HKQ0xye+
zLPH#a7VC7AK*Ol$NG~(2ZtcG7QM;POQ6r5o__tJnrRf*HuBo4rQpjQLs{ObKXl^K8
zYGznup8wJ7cKb0N>bFJsgQYoOg`Oc_aym65O?QK!e<NZLTz%}vYc-0oqcJBf$=(m(
zrU?}De3)-G>pZ&(Nhkz4yOvnPT6hyyQ0(f9X`c@pIY{bY+Y+7bi{p0>DUsy}n(_<y
z6Y9q4)P=cGsWY23_k1e0(JRU(uPfL$W<?}OHNfYWp?3@Xf_^{nN8l<x+v-v)uxgP&
z$z>S~l<sOu5I<BHu$GJq>rY^SmFP^TZVgq^y+baV<_>Xs@@+Y%ZDqOfIB{A%`&v-Y
zSl`|S`LevWu=-ZmXB28g8%H;So$EZ0R>AF9-xy|~16+l^j(KX112A4Ys<eAt#5pX4
z=TA)ApEKap@77ljR*d8II|8ti#EY$Z{!SAI8*QW3=ohsmpZ&%hLJu%PAo(lS!N4j5
zT0uMW{`HOnGd=)W-NCq^Qf3c|S!M<1H1?s$3SxMH8;M{#DEk$TwC~ezF3Y^@nqs(I
z@V;iq*Ae2g$_}qX<5={WT#eA$#!zkW)^qOgeeBy`IK4cXYrQ9rurW&g2yQAhbjouq
zuoc5F(5-f6Vvh*l{?4(86@Sa?Hp_s_D((o23&$1(1z9$ichbBXeN_@gC3xKu*-Gix
z8zp8e9)29K@O3dRF$z5MLAp@<<*h{*=4gRH5UW;AC4ByNNz`#gE%g&nh_6Q({HjA{
ze*7)tLRx!^+cqm0bG-cb$+yfv=h)1<<3hMwFHtTI$XaRSW1isrfLpt><WIbpsaeIs
z1AY0U>!2w8yX`KU?H>vMM5(L}QrBOX%cIm%TrdYNZ#Zs~V(Rs7A)B_#t}O2|hHbN+
zBBQkGhI3waQMJljna^&R2?dlR^`Ld2l$~pAm!eyv1?|}Pd_B3ECS~x~cdxAL9bisB
zxa1i|u{^%zmmgrdP}_N>@rRjVx}<*RRE1JsQ8kV-nzqvSZ2~qcTWWz+5q@34)Zo=V
zwR@W3O?NPiz-xv-P4Id#m2XMcRxtaY*L4*9)|b%uMj}ub#f*Y;t%<~sGiHkS+hTSE
zr;NBQ+g5mtQ=3~P(pQcJo7(?!R6j7R7FUYb2rsVv9dbsMO142kacLWIpSh9n%F^T(
zCsFD->usH4TPPrvyPWh#4B`&Ux#-)ZNnGkW&h;k`1EVR(;N}<lf8QqoeW*3{_%(?v
zQkzqY4O+{ESLncBRG)6)PfFWRuFd%Z#WGz7k+$#-2cV?Z{QL7^+L(GrfAf`)vIPpy
zUZOCCxU$4X;3?~NVDmD9&;94~bAVBo#=jn=II9cdN=xNr%2u|;dS*KvztR^2pHSZy
zDEU<~{1+f<>jR?K_*1|u7b{NlDHOYgJ;C`xOZa0D3hvh014}_)-Y&c8Gz~B3CR%iQ
zGq|&*13uC_JFEw$A_8CbN4ZuR*{q#Gsg!<^H^S>4+A=iYT5>X{oTA|AtzTXnk`dgz
z*S@`bK%l0!zE&6I6PTsgN*R#~hQ?XVK;&}czLth-*uUhfiZDB3zsD{-;8Uk1E!-J-
z5;7hFV#5%9e(0K_fLAz*N)@{(in3?H5HTJT@$><64LupHXkgG^!Ae?xrmw8^>mT}U
z>sSNxaqrV;+J+!MKgP*npxo9#u+cLN5i1TV4#eg`0E1Fez2wKlJ#Jeih8F*BF=aM)
z9U@U0Nt@~owMBXl(=}-SurU1)V(!av-FP5g!tk)~;iI+_Gk9|bcD9w7um}5qrS_RS
zHJ}GWxQ6wP=D2<Grh$|0drWQ?a~tG=8xlT@4c%2m#U`;}`f%I#tp39qM%t!;*<xA^
zgJ?ZR*w=69`IKbl_b}{pgm46mbIxCP>^rVZx=?#k*1YMzn#6{ZA<j`RYJ+8mUG7SP
zpbb*cgG|ywAVE6L7_}G5jIV<ZDCuJ0W+8+EHNytR5qwJUXpUZ#cOMf=FQj<8@O`QW
zcxdqPlX}iKV-&M{!C&Nt*6)7*EDui^g2)9aG{<=ZKdtJNwm}PH(?G_?AzBR1KJYEI
zy07w4mcTtKC!n#8C2^fv`E^+)jnhI*EX>Cl{=NG-?)R4sD%AB?mgb?z2SqA*U`zx*
z9DNB+6KyG~34j|o(%Q-5h#(9LU<A&Fk;dCq6tQ$j30MWpA{EJBPRsDT<f4Q`7Qc8>
zx{vI$>8A&oBnymbmA5tW;iXrbSSc<|15f1if6kl|8a6pt**thiuA#{$wG0lpw3o8t
z5k0ENL$&gm@r8tujj5JsIEd}MKNKX%2je{R*9iEi=gueprsS|G0xtc5DTc~em`}&%
zqf=ncjzK8=)61g;!;!>rTWv_sEW~~_dC%v8x?q9vws4$6oSB`uO+y}a{Wj(ng)`Fu
z5bg#i(|j4}ivpMFZ+27-ROispf-U3mysXpj%8c_KTnvOu%RA>m1yB!+k;%)11&XVs
z5JzslD9<+LbSs5zxYZ)%OD$N*XDbEWg@ji87jzWc{F-iY?=rY|7*TQKzzIJ39h?!$
zG<yd5-f#N2E5CmbtqSZPzOPkjZqcs{7&7*69BC0PPFlZ1Rj8-qXgvdf9%(Xs{m-&)
zoe#$-WxLPVoKGK79%rRI9OPp}P*IWy4;%d<)M{!b?}qB$e=45qaqDJeA+R{%FWyn`
z%C8<#YY5l7{;u*R8cv^)ei!_&n;kBMT2sPoW+ptGV`Vw0^bQCuNy+dLu$J~RK4zAf
zv4SZ_q{`ZpJq&1odZDyygec|lsIv9MP(VAs>OC{9tdyY|d-W%!Rd~|4dt9$!sMXUr
z)y<Y(S2fZz)7>tMosZb0v1Hld;(e5Z5X(5ZvCYQ{c-FxG(4pcUGLM)+h@5RA9FGbc
z22G`s2nDf45e-8A(~dPzRzW5;=|Q6Z&t01c|IccJkdUb0|G9<yCeWxi$uWm&vD0ZW
zt7DKV{9_jrfcEUI(ObO-J<}DI@ohYZye(KaH;$P4?=l-0EAAto2l7ny@RD}8;U^Ma
zO$%{unM3epG4)|n0etfHE`YWMr}|>1RQPM3&zrfta<yJ<L$O6ydg#@9C4Xb7=?cRQ
znV8&HrUyZ28{z$81ie7L)DhlG?<RTd-+gmBet9wU&;F$Id};EWvn_z14n3x68Z~yd
z+2Svcx5C(rs2^rY_95KsMsx&>ICCGOo9LRLJtuW|)<`aQklo}tr?IU;G{+`=5!I<^
dpTmMVdjLO)a$x(>e@6+uR#aE0k$W5R{{T8p!s-A3

literal 21921
zcmbq)Ra6{L&@Tkn;IJXMB*0>eyIXK)aa%k<aCdhnNN{IKV9@}<eF+}igS!*_<NrO}
zr~7p8Jj|);I#V^({i~{}KGV@^DsniO?=TS%5O5Uar8N)`kRS*MZ|KmGUPl^MrAl6z
zCp9H4nU|NBySuyV>+9p=<CT?_7X*Z=mVwdHQGR~G!^4A)j*hFVtFyDS*QWvl0=Kuf
zwY9Yu78cewHfCq%o}VwcxVUa#H6Rh~K0QA_{b_1yI(vS8dEPJCd}Y_g`T3qE;xrQH
zPJv91y}qJi67lMSYNBSn{?Y>j1C1_X3SvTFeM3GzWk}`D%hT{!r1BS0;l9Ub5r?{*
zoSdC1E93m7$>$e-;V);6&U2}{Pgh;SW~IUkVeJI?ZI_ScqKeCp&o3|k6s;l!1dX1K
z8)3&!LK^YGrWz}Srq2s$&u8N%re==yr^$2A!jj6*f6HoWYFgV`o#ln6rlxFdY{tjO
zONxtJTwHv*9#VcTN{ia*%8BYYW}6ENM2tPl1r8J%35m)XR$EKXTs#R#`$?<Ys0(Rm
z3h>>p|I`a@SwB9xxj7P)|55|t+`7CI_v${IfwrvQpLGYRMUTQu-P81hU!K|}-CM*3
zBqmli?yuH_4bprn;dTDt=$>739c_Cp*#If-fy1Z$U*&@ogkJ3@Cx=U6$|4ILS@Vq%
zX%0G<3al2PP#sOLQXk`x{#)qGSw;WW<8ppNOte!%*Qs-Ta!S&|NMk!GSBbU6%OSLx
zJJ4TBGG_d`Yi9ZK7*6xPye+}r$x!<WBze=&nMo8%MDR|@6?z?_ls<iyPoDM9fx|TQ
zHzP-@s;Os`hgnag!MCE;@vlwC3qxx&ogq<SW=;X8+`3~2r-x%H!>gw>y1x&Henp!}
zHXI#2)|=~dLYuz%{OP+{+`VwAT8nNSXrt3TaQb+&@m&39_D0uwb7?Xo!X-wLr?ceS
zUzo?5CdD}d!p+#!dE!)$-}kP`L&uObSZ3N37s=z=+FeH0KwH@`H2Ef9@3Qt_xL~~^
zc_}yp+Nqq><<ebiQ6;MDd&=*gssKIJ%<7OZS%OHdEI##yPe1yG3gD2kJIwvv|8Ymc
zuu|8RYuS4Rzr$9}pN4JYQ0u@Q(pDluOsFu1TBo}2CYSH$SJDeVBGK|AAZVB<NK0sW
zEgnrv;B;|dBiGjQCk3Y$0m5Kpp@mTmk-xi9Ug7_Lzs_|1+k#|vzrDG+N?m?4dqppR
z0?pF9sdH_rQ`R;^2(TYD{JsKG)D;vOTxJ8%MVa4`r82u$tUFf&8xQ`WzX>7v5I~JQ
zfu0lMKtK&0qv4z{W6PpJB~ZW~-o&QhRnH7=B|-f{*vorw(T8^v`7h6P(e>e{BZqV9
zCG~Bd@7OO=-=E{FuB)yU;?9Z~N<<kL$jw9#2#r95H^J*Vrf-AnX(0#^q1bPNO~bqs
z5%xJ`-2s16N)Y?(T*-J6G$P;8=SaPY{f!9gEu)d6#zT)-0$+s&()JElmo_wTU&+(i
z18p_R22z3<A{~C$3IbhPd8AScJxW5wsmZKR#Skb6Ly?K#c*}*KKuZO#rTYYjoFfBD
z;HKdJ<Svtngm>7wC6IRwSQc5=;Tjs|b#Z!S!woTdjCj~sRN8R4DXO^O1ax>;_6f(Q
z(tTVWCIzykI8$Vf&E({U5tm9-2N)35_KgH>yA~@uNsHAD2b@fYuCHf&1R2mQi-(T<
zWnDx@j3S>i_vRyfh5*jTET8g@h_n&{LazIiLX7+dQEvYObGA1fFcma~?r0$2KnhB?
zsE#Xu5MhKj{IsR>pmP#A^iW;24E)O}632VG{SCF7!`s8z`K*HGydL@!+afGcEa&1E
zVuaM}7M~e60DXG2n*JRzJkB=PO&sbN;D9tyLPrCKn#DW*{rE(2QmJHA&-z_{A)H0^
zxy;`)$83x5bJcI56Agxdq2NnuGDcR$)XGBQqJ;_F=?P!=k_?e0T?<-W2^B(;J)^me
z_}1(8XyRI?rU+BVODzfE(YCyW++@!5PQ}Ug+o4beR$qP|p2nj#|5kE9=9k~pU6EGH
zAS_3^2FXNz+*J$9nu9Z0TP^yW_UElMs#(V=Dd2STkCyJ7S<2hC><hMi@mJu%YxaL>
ztXQ`ub-`qKNU3obj&bCwm&o<iNi@!`kkULi<G&xKmF<7~RWd<kAO+dy?(e*8W@*;8
zt%iBBF0>vDu;!$?j4!O@Snr#LwJ^FodJ$oNMsx&RGjgpAJl07vEey+It+W*nK%gKI
zBdhr0uH_UoXM+ZB@Ah5TlSB*tsPdrd2Ah~FGSt8dtn(m&6hwjqj{@5FrGA<2#q9qk
zQt(-<BGJK)CQwg+#M~aCc6pUTH6;9S&BtTBWG2?P+3&gWIf6BZ4@q3ebFq*d*D7j`
z@?EZgR0dctN%9kTSUDW?n!XerKC?xQlyo(TVfD|J58$l(IBFh*Gp=L==A$ZDuQQ?=
zk-L$6rjJ`=w@Up2zG+K`xaULjIG=Wjx4_W6WS+am-XCm(Cp$B+R&7qs+9t}RqaDRo
znZAbiE)HWqeqU}%T$UCsPP-Xqtc3Ga6gQ!_Wo}Zj(9<^$SYBKWHh=7Fref_imBO~U
z+@pzgN6wj05(bgFPqCR8+$!yu3mD6ekA2v*Q75a?u$<;WV(puqFr8nSnxK+f`+y4}
zE&G0Fr!kC#mEo5*0XynVzex7K`qiszx6cf04@L2dB!U-s4P#RHH)fn5P@m>KZW;9I
zJ#+({!%?ki;0~76sM5+$Q%M$Bczg1!Js>(Uz_QkGj#cmlQkmaED$_1=-JzbE2mSVP
zO1ZVu-BGeKVO0f$i*IhFvmDd|gRN=mKt3CKUo3;JxKjp9@$Bz~MqwH4KDzanR5oLm
ztt$BN2IaB)gT+qml5OeKC|{`w62clAFEeR9k^}Vn7$p%3LF~}Es)w^tUd>bYDp?N$
zr^5`!nXt+!kBKCE!O*`jF2U74)`S`}Hnu2IVQW#lAe_L1#_-BmGn$OQ24(&&mN^Y=
zTX^da^;gfmn!0;>W~_Flda}kpq49^BV<(1B{KlZ;u7z)x9&zGad6QJ@0zM?)yf{+R
zs2SwI1GHZaiY%wHbIQIf08MSM&Qd47TXf$Ko>!&|i~GhM(W%N#H=y8WY^3(pi<w)?
zYK*aJeu0<G8Ba}8&&3%d);ZumM=ZBr>J|6awa5z&^v?x1<afub@?gWuJfBf&3x1>v
z2LD+>U1Ze)k${iqpV~2FAa{)8lNPSZncjp9Xb;Kfb_H<FaQ{)V*wOGSkseWtVb`RG
z{W#=|bar3#)7Eo!B>%vM@JlNT3015Zh*yhy0De_e`FPLT>7ME2V*hX+p*Xc9*P6Iq
z;-s?Hn@>I(aL^R((0gS#EWo&suI@7lQBq>A*yo?-0MYl1q|I?R)xS|(DRnDd<~qJE
zgErGX$=oOH5!FNc?-D=tV_u7*Z;ls7{gA_$f@tvD$UA0{SS(bkZFT>Kq>m)5NH$;u
z^@GOK3H|e)o#AS9T*f0gdqdgFqD`nU!8Z5DaY}A=R9itG&V`PY)*Ymvi?9sh$^0SL
z-K5({(s^=X2)KK?{yZRG%<f~hhkPls72u5pb3b|=e=ws6KLs8-JfMUr1DpbmXp@(%
z8yrdt*>8z&#FmM%26(*vIGMpSPzTK1C$s{SR#&2Z7EOSwREK0^4ENX`Zio!XQ2M*T
zP2q+hf@x0pg_MsAe9sw(^$Y8evxy*w{N9^Tq_3K$3DrfJ^rZ20P&le+fjYf*T|#iA
zoMd|}=UW#n+%=E(7hFIU#}?4xa*0Ki^R}S>cvK4$?m<qdurs(9j@)8x3~QJq9R$f~
zz(0N2%^;R#eW+WH?U(8YVb2?WkJ4+pFOP|d!O$LtQ2{gDsYTIaA#571F*n?;wILg%
zDPNkfn{JKyEOE?fjhYL8Z0OrN-1HJ=J(*sb;OI)#K_f-n9Ko59!=<|miBDd%jd6S%
zM)quk_2zbG)O$5~_Ppq6MKv}kHm9@$B9CQ=wEP)GR0_b}qsULu5C(Q_YB5Q9E{hO`
zJ7cO7f8jZsNJNN^z7DV+$HFiI!J3Hc?lnz)S-=KO2@(y$Hjka&?X_Tw@seO>LEPjs
z=u_9^kC<fPU%k<r<1nMPxbv-KBe)4uI78|x(jga1Umh~qcvF9T$k;Am+l!N1n^)Ma
z^p*Sz@<6GNixd}L>f*X9c`HAC*B(8jz#d!Kjn%43Z6ut83mUG4?3%xWbLbog(lX<#
zZWmdwh_aE7fVhUY#{14G@>C*R@TEY-oEw4zh*wJcysgxxyBX=sf2&6=GwMW3Q9n-Y
zL9KweW1q-P#dgRz%_<Ttvb;}rT6b@i`sM1J7lR+_x+rl)f5b9r2IvWj?UWUQZ5wu5
zs#|&4zgj?V@^pdI=Be<+%16fXwLc6QhVgwMQI?f1t5QyqZ<Qz2=X<V;B67Amb#*)I
z#1uLvLh37?=>`$mNgmVTwcvdc&D(TF=!P<upop$oDXx)&G8R0Xh|ss#>*3GaGk>o;
zCcWlM3aP6W%i;dCdPSX6gwGPgO-#NVb;(080kIXj<JHIb;0}Fj&j_}={)ijK2y4-V
z2XELUihQ46R<b);s(u+G8ptGKo9q{{4l4B}#NmQv76kgOu5<f*0MOGj(DD2V9$?Ez
zY%hsz>lraP>@(F~&fRt6LEDArUY8TWsmo*I2Vuh<#wa1ldZ-Tz7ayzu-!tu6X169U
zWz=`RXNNj;KsOed`aybfi|l(nx_ZA>qHSd<oFtDw=2d;op0hp0kFFmXd9>f2^4IpH
z5Hh^WrWp&FuXGs0GAYr@WmmKT;I_?U_3ywXGob<nku>M$KS6P(#*W~v#Y7FP@7MC?
zSxdB)iVKqXvA>m>XPkv5Nkm0_!w#X&K1$Anz$%H(`voXB0ROs6Z%1*4rqN;F_#gXO
z<ZlJd-$(cv=hvN5U=j@ubBbboM{;nWUTrbIeTkU{aIZySl$m&~O3OlQ*}i(cvu9WP
zBDNxnFv-jQf^a-Dd@Q-=n_JiCP)`<=kh)knm9xg5H#dlZX^;Xl`@kCYEI;lT15{qm
zT4O?NdjJ7Ca)`|l8@63X5QX`~gkRz3uiicw>Y?i|(U>oEodbyC0rB0AP?0W%zacau
zc4p@_J)Zn_zx@dZ2ybl%HsAWOfxS98HE{|u6cqfB{(g(U7riFK#+54QdHGvMHeO_E
z@;6yM##b6CL@2B6H2i$vmuF(=R(TqUbQlOxDd>C<`49hBOAeR8<;@)BQ#WSQM|DnI
zdZdZk)gp)Yv47>eC(z*qo8QmzMxP<4jxM<o-5d#>DzTEHY5XT__9V&equGRBsTC?N
zwemj|-uA?M9TQ-tC5l(Z^j=z6j!z78<a92#mU^HP-E<O4N~=dXSqn+4Q>S;osNm^l
zpn$<7qvohD0voh&3ry8qYrRow>_Xj$(RGs#dTF~j;pM}wPzWAsa~eem^r$Nqz#JNw
zk5_@cJ{(XCTRdQ5IKcUw!a9%;QUPpGj=uKuS0Peut%&;g?P_ahEc;HJURJEWI8GgK
z6jks$-Ca=o*GY#im~Gt6x~_cQO?rAXlPN*#mU(;cs)8g{vsg2a>yx{tGK#u=B;B|}
z8WwY?$axOcFkgE5>1xNfPF986fQ<#itV)5e(JG)1*#-oLxtf>T$qsu{#=G8(F%a2o
ztoL4wuo@BrX2sekL<zjL8sN^Y8|<o4R#v!bQkOSgIvV=;t2=v4V<h~c&T9BBc7VIj
zkaxu`-UZmwl(M(Tgz?>8P1yeXXPOR1QsootqSP^^&=FHPUR@pXi;gOXY`7L~QCeNW
z4ELb<j>e9wEcDw&pGOBXA2%$|fu&-ma!*NCrOLHKNhUL--NU6fd=Ze|?T9Y1bix_X
z3F<WFIq*sRau9{eG<G3w%&p*E$j)pqV``wH_{pDk*Hln9J&pIn?omJ-2h%tAun|+S
zlR41%Ktm}^W!zS}14P4<8r?Rx`@7%v>i|2m>zrMcoBw;|&FB3hJn6>IRSW)-(E#r_
z_nl8tkXrK14p_UH&sO36*k9O9^+?4LW3j0AY`PLOodU8y+w^$T68B@5d^bEQCL?l|
z-K^}kjapcVt4>CcR15Xti^+mznXS)m<N=y*TmW2h_<+MjK=Y7=ZYiZ+M`t*Uzvi)Z
zu;uKUQ@v&%rlVsoqf|a;gqK|&wOrn)!JIC@9Qq6IHhtkA@3NSQ&clcuVLldEri>L$
z)AgN&q{v!cLOu)Ojf=4I^z|JG{Im4XuZeCjJ)<%1?%{x5!6o|`%(^4~P4s*YT<mXO
zCim*in$8ZNsAI2g){5~14T*)ON;z`_g@5qz_07EQPGu}7eB);E?c}@Zm~k<y3>68m
z{gb^!oZEmmqcxKn;HZB^e~0hf3F)a#4G{Jw9Z@*z-ye~IKvL#VHA5Dxi@=To5PAq+
z*a*UjDXSC&k0|(+GW%Z&3PD&4Zp2QPP^-;A^h28kX`xDV>)%B2A>{ZuuWEX&20E{{
z<1vRSt1VEy&N@ExgGB{4uO#Gc0nvJ<kItO?T)0=j%ar?L+bz%^KL(N4X0vQ#Q6Ezk
z?#=@Y00`>H`K|xer8;sGefvdjV}EnQPgHyCoG0)j;(P&6vA^z*DnFPhEB^R9=}4{|
zP3cU3bh~=hczx2UJW62i%tL4TR1^M_)^B&F=cm-nLhSX-3*h+M-0V;J&af#r^}3TM
z>nUZ)o9K7$O3)dNn)=RQh!#*`;V_vcC&jWB>q)DMzg8;GnqBJ6dS=nb%`yJ^tJ8dF
ziKRk=;j(J#=M5HnFNd1-a}8&sL%&~U{(~E`8HRu-pWjLXAV**f?K$nMv(dE8+3EX)
zDmpGackQi3X_1YfeqD776~oN@Q)aqpBNoMNc9y^CNAB^3pn02`Ba9C$b$vI&Dm;PE
z^q=bU4O^{`e!76fSGelEE>(~I!mFwxS$NM6V~lu|8qVPv|Lp^PYZ{(<6CK9J3|<LH
zEtWmxby}i}Tx#j=!uLO>F6rwWOwBcpNzO7nz(58WRgL(ssY8$D;o$>|cOwIAYTq#t
zuD|ax^H=a?{Y%ya6LTm<)a*!}Gb|<ku{nQBf$5J*v&aC_Apa1f$!?p7tQKk;Oy}lS
zRVU`CXjx>S>7Scbfv-N)4x*&O^vC^q=MJR(V?47`=R*bdjcLsh-~-FgrLc8$iX|hH
zhc9KPT04y<oi|dXAC3aL*r4esUOTU;OuPUl7BOO4+z3NZ)sckYMJ!QTc&P!9-=sfr
zYN6&7z|aan^T_EU2<hF76-S7#6Dt9ED-Nkexgnzp9x)|^{x65Y?2GWfXhTv1IE0K)
z2E9tOZtw)(IIOSr_(rvzjl~li3IF3e6YapJz%WIkDFTZ1p<&{@jCqq_NU*==GIEIy
zn1w64JrALRt26{J>QFyJOL_6u{o}pp=PKlM2{L3O78b?p{esd!qyB`G+ShbYh$kui
zERMMi!a|ZOdsHV%Hf<ngei-VYIHbnzRU?X@t~{7{c`-0n(x>SY*K;WLE%G@@V&PIl
z=4{K%60DYR@~HuN0GQ84gt^>#KPievOw$#Eh|D~#4jj^UK3tOBASbH>G6Fzjqtc}R
zMjc5Lq2z7YJfovCSx2z=RBs162iaK?H9(Kv<><^pEbFPW+&{)*5VW?Ch5M(|m&Gsg
zr|0pn3g)qu9r8+~66+;Sf3rzZpopd0Z3MQLfPfr<P<eaSfLL`vfazSrn_Z?3F6Q3m
zDZ@n<c16fP118L3&^i(`(nuj_9VZ=eTMck#nxA0%x?e|zdguA86E`CWtq0U%0SJRQ
z4}#F)>j@CNjBW}zH88Z!0+s!9uloDInn>dkZ#!%XN0}=!JWIXW$jOX!HyoH0D`?OE
zhW2>gwikuSOef}W{yp}94ck%yw=opkBiVbtaK2a1^MndC|L}{9v2^HDPSMx#!pDHS
zUztwmoo^LfkN4JK6*?VgQgq>p9LQGjhUjb%KU(sCfs6y)7To_zl)7X<DdQ{D31l-c
zG*pn!L%K^mEBScBbq6uxUccV#v;+6QGJ3I=D3k1qpX7b`NctI~^|b+RM20ecgh-MZ
zjKpL_cX*y&XAUk%5kbOtPZzNSZ_7nrJ;OL7LrFn)?7{esrZCmz6vqlhtTXQMLytf<
z;QjQ*pY0!Pof8uj;0y>H_|JR>1n<wx6S)O|;?_BQa&0&0K_Zw2JP`V*<VBeLk~z$t
z!+7wG6%Wei-uP4<{=ktKMTMDB*c3}06=?s??yq{(;8I3#JT7aAnp!9r>C3P&0c6+O
z)$V#MUxSTS)7%(QZ^yJqy!1Hc&GN6yKN`A4@5~eEMDxC?=-*os_s`cKefkTYg)B&M
z)qPCGHLqV=En$7nU1vy?Vq5V&3la-de3|8_Xg<}~Q^f@9qsGAe;s#VJKQ$}|A0y!`
zp%JV9&KPd!vwqkN6WtJ98Il>|Yl51h^|I9@)s$m;@?rl<#?A*fK~R3H6mw?jzSWmh
z8;4cLS%Whnn}<c0zSU{-v_e%4vEg6taFRT6U`oQS2Qsj;q-*cbm|vzCX!`x*2O0{#
z8+utNEkICjwQM(~D`)V(o3lv{s7Dd1R%M!;dB+yzv}-;4TNr*c`z%|cE)!!&=$<dR
z(i~jlsOCE-d3b(xbvcC70DSUxI{8=r)`wHf^O3Hd@0NsZ6TDLd!O|Dlcks9~lG!+Y
z{7Kf%*Z+lv+<;NbjK_*v<vWvux_NSdJirco2#d2WZ(_;qTX8^ZbOXzl@?`gA{-o<C
z90W0!RM#DjBR#`ht?D>s%-)O9nuSF{PD(5W8m!EZ-C|}m!hYVAa@U3F+u2s97gRN}
zYlmn&yo+bhkH(MoPm~Qq>r2r}%HTa0zu){_jP`Qc=4!QQ&|4w#=SgvSJJ)yY_}jzb
z?Y9>Jiopn}wIT<H1{lBGtsm4vBm9^3)Qeo!-^bL87*-h?ahDZ8`lH>aaF>U+Rn^8e
zP?2kmP)_p*BR2zc@x$e@bg?AzM{=^ypQ`>iJ>uE@@<2<l_;h1Z9%t#edcbQ|Ysf|Q
zJ79#KbG@(Cl`n~4en3S!F52kF!i~>}T6`erp+P95l)&RX=5<<ODPeccW;Fw3wBd!B
zoq03>a(83F(Rrv<Fi}nd<ha!;S+0BmK!!{6&jYp5gUrRV?o`#SOnMSc><o8;3S9!B
zFPAa<Bp4&&HB$^;TTIT17OEaWy0i~JtmAr~aWjyRu?Z?d@<0KnS&D!in;GQGzbgJC
z!ZRgl4hKtV-V>-;yH{z2q@5ABa!px@Syv?L*HobbU#1*Q8JSY^oe*FztQo{Y)l8qE
zcn4jyyQy|x#_zrq)a$!%th-W-Bni+4rlFJokM8Y+W!8yw&5qQzfg;OF_8W18)_yA7
ztEM9aVoECabZI}W9JB3h>C=)QXqb9i4tusKMOL>XRRN`0XADaeG50i(n^QHCr5iz(
zZG*IxiIG*`VJAP_)U6)T6$=DQ0BP~Gkn%XbJCm!EVNjsrv)IUJ$5e0PKO+_qO?B;i
zbc?#)?caM{Xgzl75v+JAC~-#0GD_<4;l*aeAFErzc`}tBnmNYqz+N%Nyuuma{C@Y{
zG|uYLE3*$D>m8%Uys9L+9^_i%KdGV0f%BQZSaxTDcw@E%PHt0mDWtgAWQK85L5&Yp
zr2Aftp%lw`M|nmOaPNspxZB42e)Q=QDPxDCm3qEdFUZQ%s6u2>Kw>I86M1H!S$*Jo
z*yYa*C-pKo8JPLa;;b5=I1PxaVU?Qpr&3?cLW#h=fj7~dwGu)xL{#dO#_{nIKFX<N
zF&xUVoni{^j$*UGCK2b9&ipZ;p|s>n#Tpt@WxEt-fDjs~9_kErgE32~GY(p6&<VmT
zHnRm{BCLy5NmM9TOya4MT^f{FwEmKwb~c%JeQbO%eFUvhen2G#>ZeA+HU+5kmoJ3j
z#2?QOm{Ki33(35^y^RWVzJFJ6+&EyjMG4+n3-1zRsZ2y>=OQ%iF?aj2eDjQ4YwVkp
zmEC?g{qoybjnW*bpGpB`-t4fWL8x&7?<7nTL9U-~Ehc1`O&O(HsL;wNVdy$<dNRYv
zl<O)vVaODkf7H?n_ty{7+R2G1MF>VR+-VfY@lhcXmEevky8=3#M5{i0&C+t99yM8o
zU>XCLQcJde!QL!(xTLwx-IJb`NfFh%!M6!L5A1y{;gz|ZKA86?1cIw*p<2-)dGbMl
z8+(gygGLJ!<$Og@zR-XMq-A3M0-7yMLee-VBqV*PmQ|Zj#R6!oOXv5}eD(|Ho6=7D
z;Q%P{(=Z!td(Wo<5dYmy4?zsE4%wdi(bgeTs>(BE7wRW9QXm0<862KMm=Gsms7#<W
zI#-uEE@BsNLopr9|EEVMAM}sq?c~8Ci>({E0bqK?zhST-a<o9n?h&AZYX43$OGXat
zjza&KMx2=eXJi4u@?#-+)=wP#Jwl?+U20`eFFm^;(9v9}Qk!5V6CKBgRy9tj-oQ5w
zy<prZpa7c?$1YMkvS_K<1U!sWM04ls9D`(9&Gw%jv>z5Bo|&Yc4KfDa8}m|xlXY{K
zo<EwSm-J9VLJ3}=SMhLLZq@dOCs6^Gct!6#FG+(Vx@sA`F#L<wP95L+KpjDo2->{v
zvP1JMkm%REz~Yh>oCR9_(^3C(AN)a~gJWWec-a0O0ycsRPj&^CD5|@|0An}}*GH_6
zsQA&QW*P7EDe>txAaU~8S+I1K&xX81fUUG)e@SA!^mH*1Hdjt;B-vn?s1n$PbtL6b
zy~$BI5|<*wE^YGREwIN`%o&4_FO>X`W<=AfvEj<NWO>U*x2@9G1epBOHrB+{lq7nv
z#iK10viR1;#+)T~Lg;6(uAW5u4n|QyBe13CV20e-EWPYER%Ty^O6<mf=JsYaQBD$w
z)VS}yMiMs=Jr#inUb9$;q+id2DCZf5Zpi8@MOv%-Po<IAw62}p5BmnC5{!bJB>`6J
zuVrR7M3BCJhpqbgwwI0R(jR+~HfiuQpbLe`#5l~gBWIf!ZUPE;f0)aYfhu%ocRTrE
zsab4MNTDxoZ#1puh=R1hfnkgb5<Rq<^q!WaE_fhK0s~O{p|`|gJ!<fHNi{3;GlpAI
z)Kw~%Y=v`vfw&x?ay}VG@VojPzZ~d?*hNSSGry&ywPOMca`GZ%ZEV?(T$pj7@Xil^
zP+i3s?{S)Mb5_GeGwB8XtkBYbgd)w0yiG}|t9hew!+fLOf+C5?8z9AG;pU}>{PxSr
zaF4*gz|Ki56&0(iQ^SVv&8-QGRDo-aap33B6mlLIpF?zL@p=HB(QBtCJc8HY(5HVt
zA&!OyB2^xSzkft1$EQ~{7UG>(JQ++dGpHFKYiiHW>Hss~)fl<524;h>^9A{QMm`$N
zkPcmIL*1WVghm3C9h}2*Cmqig-yRBwk6t5XjTqKBq%!@j1Qm?``!;#`!`kk>x!7M?
zd076*_UfNM+@Zi5g|?}bbIyVD4=ZydJBx{4{UYjJ<$X3R6Ws~mvJHrXv}Rk`hCqlD
zZNDVt%YrZi8|BZV%r{CTj}jv%tKD(*P)sMf@=-n3_g}hsZAprdHLW|{Dd?=@Xt?*h
zraBuFIH87|BzaOLQdxi9Fo1DHp)gtCwJ_^BI_IkI#onz3FUMB)KsEW)12@mesN8~E
z3mq*BurAVt6&0ea*LjvpNW_3_b>n@h`e<Q)krReJ@~$$j)6Z5C)Nxy!Jo0Zfk&*td
z5cGBMwAN12j;kjA+&?Z}ec+Ze0CJiTjSv(}7IAL9B7(ju5*<3X^`R*|5ypz`RUGu7
zg(?L}Dh81;hrUnmRs(ERU`LEl%RxA_P+#{i3PD5afY-i1VT96m?ta>ec))ijWKgke
zvMiYBUJ@#0C&Y*d$V6}f7=(`j)tiZ>p?dR4G^G1@jHXHPcms@Ix*f<KVL<R#S;<@s
zM)*J^WYS+lY>!mzsB@xk&2=@8*e;gTvBo=~vnKO0@7DY6!o$+3z;0SOwk(kz1Y340
zSbM-!`o9}J#ee+&)1aA<*3G;FW@vX*<O0Q^?t%qV;t?J;DjlzfaLUEJpf+kUhM5^C
z>pTl)@r^mhKT3PR^82uy)m%AfNXy~jZMP*Ow`U1d6OzjT$_gC3u69@oL}ujb9yT7Z
zgk^GO4XfOM7*l`64J&y#{>(&7*3J1G7S@Oh!{>TIV}8J5<8EE!Q1uSf8xdXFi8(S=
z8wv43dn*st9v8G=3>}$`$$S2u22&t<OFBED%&KVOQ1fjZ4>LhPtkQw-k>#SyXBj2E
ze}5iqUpw-3xDckjq0)YDGm?0N=2T<-7XBb7W2H75J;tygoWzW7`wOIqZa*gL0~jcL
z-Wo}niCVq!Q)P~I153CFYJj!k0;N@WvtB~Tt6|SnBrL%Z(;h1^Iilk#ydG0l9gndi
zK`r~Gj<H{W9)aSON>bH85}-arP+u4-BAvI&+3XfS*;WmME#*ZF+E?C>tyrgZ$a@$2
z0P*fLTGv`8K5{aQ1dUwyJa$Bjw;aaPSZ}k(hp!dhDiG`}v7=84p16$Vf2bQ`2_4)1
z#s3BHqf`!^r_Mg-QixpkY^PX1)9K5n5)qbQL9=53CXcG_L^R>w?-__f;TJ-?iHpeC
z9z$o!aUa>+vgt}IudT6}umWvMerBJ1BCV(|lFM_k4r@8dWSyqy5sE~|IgM7zOhZa#
zTXU<LzX_DLdgF@p3;1OHPj8yfq%)y>$*0G?qK>)P$LT>$+y9(nRz&az`3Yau4>Sl|
zm98c5o62(e3hy-dUFq9XO!(M+A#6-v1AJ&^SKWH>@47b4D-UZV!pS!Tb|I3x(XyX8
zmcfqgJ;l}<vXS5_1-rXuvQMV*k2q2cAK44?<$llXX0T57i<v^5xh>LkcmHBlVLxwG
zz<MY;=;WYl`{3jLZ_QOD8k=@yT7!Gz8+bEZG#~rZ?jv5qkmG=1D0u}ek(th>LR~pz
z6w(0SV@b&i?d}1&#&f1{2H?q$CqI~!<jp!P|C@c?hbr^bi1Mo#&1Shx<ahbFGMaI`
zOI0VZ8dd@db!@bu8N$|-!S^H3-B&!$mw?$=#YMiODIYb}M%D7@7fy;-LtitN9dy7`
z%Gig{^YKg_{Bj;ba{Gg3w6?}_IHP@Z;PR8Z#{rv-Tnq`@Tk8lQONL!FJ^-?C49^4)
zk6<Ei#|J&HQFuZ;zeC~JJ1bnDG>Ww&+FbL){-wRI{Jh4TURVQ}`%VMapn)l<+3+mF
zUG_!76Ojul(!S9WRbu~Gj%{22DraLXTZb7~epb@qT9r8@+Z@_GaT)R8Vp+lvc+~@*
zrgSqZ1i68h48%m35m=%Ln0rItkypq4Bxbp@ZA%;30}O#)JU<vMQ&o?Aq-1GN@a6tK
z0z8O_*8waN(K1(y0?`I0@C$WOsh4_YM#AMZ04h2Y%)SF3+lzZhrFhZS{ommJlR)v9
z4g&tVPu{QciJz5P4V-Xq%&f$JW7qUCIBMjR9%PjD9HeQq;_(E$8sMl5o5hoq>l3>H
z<=ns+B_;wSBK~q=+Mh8uW&W{Yuda%Qa&CT}hl_?vY)_+T^No<j(x+ASOjh{0xxC`n
zcxTes3ZpsSYqRARyujgGmaILn?85Hq2h)J~uzz)wW}uRid{Z`J!#HgjdjaB6)Vu^`
zlcGRPvbjobn9PjOZPGCoeR^kri*Wgb%7o^bcI->x5?=z)?Xx`dp`tsrTe!diXw*Ba
z0(TsK5IZ=;q|Lt_L>czSNL$Z0@W%yc<+*fJDvWNkhdKS6md;i225x-cNYG-Xs%a-w
zzC?p^E0Ddju2OCYPfFKpF@aoaL&=-o_Zv6*(&Eh~j<YMc8rk8iMZB|VWWOp=^nr>5
zxGFBa38u}(%u|V%6SBuQErH8?&^6L=xmnRUj8c79idZo!4o#AKF9_`+HO22u`mJXH
zJbZj(k@YPZ<|5r+e4Xg=F#w`lDZ80drkHcw(thi^|1gon?>)tEs%nN0^UnVLy4(+t
zB6oH0S5PAhVqwnP3Io0x(i<5r`o?eCB98wue9uAbCI_m7=Hx8gVhYT3%D>mJQ|=m7
z%p>j0krz|8)Wz{d4zb^Ml)bT>2*`5%67yYj&xPcxuX(K8v2TYbdm{O7yUwFltg3<7
zE6PR_S)a6%ZL`)60gaL>vSNtL+_OaH4>J|JA3c7+uoWQ(F6yl~s84BfK=%9VNzS_C
zm$cEaZPf2usaZxs0gwNTO5!qQt|F61%lRinwC_nTTLOA=<v$dkzv+)~H2K>e8t>E1
z*?M@uqW^E2YV`BNk=*@TpXQ^hz_vr_-4?eOlBh4zV%x2?fOdgqx5c<_mWxs8TTsiw
zTt1tm1rpuMwK40|oyB{7SB|%f^fW|?8}<4?kX49pmJC9oCzQ?~vlkZJ={-zUZ|x}b
z^8_t5%kZ^$ffnFjm0oMt<%`_^sbK#Vv?GX;73{x)X31^=O8xfA?z~LH(KD1k)Sj?W
z2X-W9#Tn0n4~vQlbNn=QO;Yk2Eig`7s`s5@+oP=${?tx<RoH>>JrdS??j}I`o=f5W
zrR>t)-z5D21IIsi3AD-#EN<BBI$iDmd^GRFFS@SA^x(EK_XS_E=FKXie4nyAM3L~+
zD~0K;xsu>;*WhS?ODvQqAcrPPae(aVO|H~_IAs7>^zQ1@B;+I_7UQrV0kdW0YA$+q
zQ4MYzytyh*Cnx=#8GDn$O}+memUiPyh;h-yU|&h~9rpo!QHwU)wYocr3Aqa|va~33
zRndAHT3{vc`HoWTlfO8Guu+BR0as2FO`FEz6W9RNL90{DO$DLibn;^VB-<e?93-yw
z^B=a|jVgv~kQa24_2Z6H!Luk<e3qIqV_u|#vGD@S8ZZ&bE&RLioVs7-vDUbwAD_%p
zWq4`o4dI<rptb9k)>t0XEqeM5&XKU|eCQP4?{Zb5LbdVk3)heFUl~RvOFuUYM%JtK
z;aoelTTo7}`34?b0<4$LeNShK6AXMu`b$$VDQaGR?`%7LuJW*3s;obaiy>TKpzw04
z+iA)-(PM@x9R4PVGGPa;0a}Dw#7uG3j)ZiI4V-tz>bD<jtVfaWGsKJNp=n0=>2a|@
zmaR<Gu;ONMzh#;}#P>yU=hWP^F4`LohlCzPv^E-n{e>>fJM)(CzAv$AP4Z7_;p*Bf
z*Xz`O0;H9zMUrwy$R@SrTu0ro5CjJq<iA=Ce54+H!$anN6Mo@iuaa-%7hzwCI92TU
zQ!M~o@^VjE9haqNd_ymN<JXXh-@^>g2m><rTFJksO+V)932zG~Z?t(^4Sgpn`;Ifb
zN?Qk}4?#DG7&Zyk0Dwm61hX}=WK;*8DMZ*qy>u(Fe==;yOG8>2{A=&Z({M=uT$4MH
z@xiZ|U>jo+odvXVkCWwv4X=87YFwLiinh=u14)SIoS?j!KIE!4ZZyHXrV<9MfeBLl
z)&jKXLueyL7M*)EDOg@n+Be4+d2-n!%2n46+<8}w1J?iS&u>vO<x~=(K64R1G8%Ud
zL(tOs*@n6ZE(En_N4Dtr0<v;EU%#1pDTCFV_G}j6qjC7<w(Ow<=^0aN2W5mwoY@Dl
z6-v!sbrmMUJ7(#eg!aZax~m`a`Acu$t!JcG?%OjxADm-Z-mYC`CPaO&+BQhYdaCzy
z%Hys7xgVSQU9|1*im1?P+*DC?Vdqn!{jGMksae+Mx8QJz;<jQcOr2`tRqJ|(hWonO
z2C;(T>&^ws^6Ak(Z(l01b2eD>e*by9DAv}pMcGKRtVtSJ&4E@y*~=O^f>wL~B}{uj
ziGw}%a^bE1$MVfsugH)^L}>B@qYrlC*;rU>xTl7{Aad<gm0!Yx-H$YHz7UPs<l`(W
z$?L8aUDt{xPeFh86Aha?aDAEQpKoSNIX8y)HN4zn3_X(^&7jV)abUOql4F0RU5Ffo
z{&%SyFUNqIy4r}U=w|GaY$UD34U)m!{Da{7k+0Dwkc_|<B<O=Il6R5!*_sque87)-
zDV)`GE5<F)4hQ^jce6?-!cpmEp^WZ(GsgXQe`V`^PM?yM@c`RdKrX8gtGH?;#sT&Q
z`V*(WkHf2fo*Te2=f;L-VD9YE4>*d!k_guLIuX?D4B|k3!ytyQWO`-}ND--+e<3|V
z_`F<bRtbfQBqReNCL{zQTq<;oTr}iVB;^4v0rMS4NNZ0>+~>Iu!ja|vB`6~$tRlCh
zu#=A5UEX8%-PI-6fK2?@C{dHwwuQWCB9oI$9rXr+J%D=!(Q1P5LB?LNXS)3~#wAhW
zY;GV$^2z<*uHGlmNvm3M2z?E)YS4^IM{G}MCHYaDA(m$Htf2C}z$MiB+coVG6%`#b
z9<E8o?$g~LUD9n;crXRJmg&DAKjULej=oUuN*-&?)F}j7Fn#zT0VTZ(D)^CgnD5&b
zuD@M_z4i1+mHW7a(oGp($Jzlz{`L`jmbm|iT7``GB%)~+IHzklmiAG(vFs*jgZyhY
zV@$by3TF?#p;2H+?tA65a7;4%O*n6v{JLgg9rh<;X@-CWWZkmEdnAxfElq+OB6Bv!
zXc97(>H0cBlYv5{I3a;o$pa|P(!l{S8PQWC&nQc*j5~z8JIWG(6DB@jyrC75A>4Sz
zof~~=R9n<B8||G30=5u*kW1~y8?ko3HsK{slO!?x{#$in{Rd;I@g5HczUqAs{bd}!
zvCNx#_TC~LihWVHGO=JHB<d4?-L>ItZKEKA%nK8OR@307fA;f8>^Kx}%9}UtMLbhv
z7kJUDothYg3|S-LlzM~|tHa|bUMvi7oAvcN<3bV$PG_x`i{#jr^9tsUWlWEbn<=Pn
z;0(f_Z*qb?6Dw&7DCwAgOviEL{)1H&toPie`Ln4uFgtFjkC%zy-vbER-Kbw=vA5ht
zDj!&YRR4S%U@Y}Bt;L)h8LPnUs4hmetXkI{T}?jd1_Ih=sc9rKHtd+oM}Y42G@9}@
zO^bWq2WJ(5QuPUgAKqUga`@-CEMuEQ2tY34=y3E<D{XiKAxI9B_M054$Gxbyf2^VL
zpEc;uZXv6Y`n23?Jt80E1K0DhhZ#1;K?J}g7oFmLk8r)6PhCXTi=GsYJ&q{pYLmDM
zb}5h{<^>g`XvjZw!A`38)#P^IYsmEw=ogBA6%|~5q01ETC-=L7*A3z6DA~qR<}ZMa
zNdzGGBq!tw?dE&`t8TnEy3RPWPt*2w*r_5^1(rcLK8?*f2S7#P$+%<gi9Qh7wQX~k
zoN{}l?C1mNYhpB{V+0izZr8V->IAy1QR0%d{KmN{;dOyA=WvKJMU^C|J{;(g$`)h^
zz#vKbr!AAv4I15{H_`DEHTiIIRat5H)4U<1dUVj^-D|K+FliUZ1`K$;K^+<Pzl{`6
z2*DTzQB$43ue|ST<Lm~$V+wE_QGKAJ#ZN$Ix$|8xf$;oW(z=*AM%KXYnoeAxi(nJK
zs9UlobnAXYFH`7yH!R`Xl<tN;<6qIdM_FfPF?k+gq#9Wh`oVCU*!@oD&xY~y_Hr`)
zFz8)W3bGx3E=!R!BCbmlYyK}^6fa;Pd)wwQ$>P?HJ4WvAIYjtL95M9vVbSP$1a(t(
zT1J9rtT3xhlZ`|QBCUTUrxQ<DGRK^4YEXRmw&HJg0Q>%%D9Xg<9AqruV)v7atioCq
zE~3X{2h-t4UdAz>wD9CDi&{RU!Vs-L=eT)WijyJl{MXk$Dj>Z5{PaMuM?56qU+m!p
zqkPx^|NNACIHgSaLMbCUAhI8?o2DfOL0U$)I^^LOe4@~?wUkynk_#e`CNP0=%`BH4
z!z-X-=qA*$E3P|MhNLhxM7je#5<y;xt?oouA-<=6U6cB((XIsu&t)XOM+TS{zt%;|
z5TkHnzH>)J>$iCZC$R1>oEP__bXHAJW=P!7L@kViEKbjm6GVm|M$u|+$w7;5b_RJF
z9OjEf=b$Bfr6Xv&J*b4Ww$4Nd;^s}W3gz+o>da7>2#}p+wL)5ennHal?`lT*WUp!+
zrp>z|XoCgz=xS+$hdZ-SQTFhaq`G>nui_{g*i&PWJ9rA22=ulX6hAH1DzC=6)fnbJ
zqne|$lMx`16XH)CCr63Tiy_Ejo3;^Q_}NKn(2FB6eIWPv-9*_X^gM6?(;lp>_dIo9
z&|yI3rKc6Yal=Pcak7#|f%BHguNx4;XeA1piGJkDq*QrQMvV)0Q{bkj_93J=_~RqO
z%3)CmbTN8OoHB3nvnPl!)Rz7lFa1?uG5U#{O_AjxA9$`Duo@JFuwa3YIH0ol8bPKe
zRtr$tV6`7&7t3sN8N-g>Nz@^=*vB>Oucq~+gX%*476Q{-q_pX4{pL;keGD(7&rykT
zHNJIpjKUCKP0G=}83oC5*E+nq<7~nkee8A5Y3%XEY2cVNQ<#C&j~=xTRT|mN7f9O`
z#hCH=CYgqr03B<_mxw<sGW&|x**><oG4tVQA)+s+)<Ilaym^rJO`OeQrU;+R8>}aC
z!zAh!wqOP6pujXMX@ny7O9rn?AKEwXjrvV9K2vB&Qy<lva(K7}0DhnV@FOUOK&W)E
zS|r^#Y}t}v@N0<^g-fpR?K60d@6Mt#l#p*qk@p)&$$GhqX+_Qb&Y<b}$%`F^Ij??Y
z*_!%h(3Ut0iLYQHmf3fBDS*}1!*(QYx)_O+Pc<^8^*T*W(|HFf)PtVSo=QrFxDdoZ
z6=2{<Qo|>Bzt*0x(W__8LBF~)M->{1k}hltAhw~?EJLHuBg@_=I-Kg}RzYvaVvR~H
z83Y*ZY15#o4AA$u1OiZxC|lD$J9p*dbB~O|5J)xEaA2e6wWA!$WuuDPS?MGdUvn!=
zXfkRzyB_&FH-<pQDg)F_ipgD>?tCMRVIDue{wheJe=~LTNmO=wm+2#Ufl=`l`aL#V
zp;9n$cDfZ{xW!ppejMdI(e@_ft?V?pLD)pv`HI59zd4O%>hXsmSHJnDmpN$hSy~Fx
zKmK<(c4*_WZBVcHh9aW0BHW&y+zDx{`=iw#!j0u8lqF<xE7W?O*pumWl6I8BW66^Y
zmUvt%cI1#iBUVJp+l*YYVa6^YqYEt|VwuWWA6H}u&@AT_=GAAk`YfxZ(OHmmdi)^b
z{#^q_n$!N6Hoo#^L`Q@^gXecQ;`Ob19-2|iQMF=d(QL5L>i0Kd<tO=~?5W1mk#x<3
z?W1RGYAkr(n0;HJpMu+o_*AU{@g0*51TV!wE-V($kYrje#1yy5ke=YX!~iF!oOYf+
z>~r~+9Or-k(ox8)YAC(;ddW4q2Z7Krgsp#=q&TYUD&jF6$8%Y@SB6*;N{@FP^9c%F
z=m$1woYn@uM)MY&2pC}4cfGvIwIDgf&5`nc&z4mgO!jR)P@xs}w0p@DI9E5wESmLw
zRwpj>)|Qh|Rjgr7WqxO}L&=}``Tg&I$keW#$EYx<+Uu(hIKLjD8v$~ACC4z?+{fKm
zaT_q*#$j(#x#-wY67~8OBuQim3Gzm-D6k?@g(|Gj=TINpaZUQxv&Ygnq|yclYRfFB
zx3e1slq=98FgpqQo?ZpqwF-Nb<Sal0<BVuBe)~-E{M0hZ7-mVknojHwrPibdGoBXS
zSODT@`&Aj>XR74(wMu1SBqXMid;sZ;jrI>gi4E*rWtYA?Z`YbkaVi<e2<aA(Lm~Q3
zj9{SIYdk+@#wE-!>%OT|N<~MQb3GoF9&hu9b(@06hfxm$E+8tWe|Y?|;AaqG_sefp
zbO>-8?mG~?6<D7n5efm^3Btq1-5f<k_9(mGalz$zyCWe!PGw+h8tfQkucBWr>;`*l
zwkha%+Mi9*NK^yAF}D%zTEb76HBfA5z!BtvG(HK)=MdW;bP7ojw2}gCPP~{K2zRz$
zmTiAa6FXgKA81B6Xjs8S7L_pYFv8<!;`AQ^oe#1w3{l{;Z@rHWHjOa84zWPX(RAk}
zy+G2`s*@%KSgx`Dm;b64{}WjqacgG&`vz+R7MGpz7=M<x`Ll&b#BR>GjZt}XyJwMn
zpBr}!z307Kjux+3vYa&pg377>oGmTm<w_FCXBJP{5HTiA^QR@-T?;(fG;Me*5oq1^
zbU?+w8xEUZ3_kJx8<&|7ey!`?k6H29CnBzxV_aK+xbGR%8m8v_QYp$^o-wC$zx+&2
zf78d(ghfK4anHm+&%H)c1c=UM1+n|4Z<!R&{55CQg(pqMJ_;194!foI7QP6nW_R7m
zqx?+y(w4%rtLD5G8dg#LD{A+sHvyX6UI?z_1Df{rq^$fw60yIU)cA^k=I3Ip?h`IN
zoKKW(BSAU3M>9n3rfoY1L0?&42B;+&_Kh}5b3C@F49g+4cDJ0X`=O~x*kRb7wdC;k
zHoGq%AAgIH(nR`IwJLN(B|;bD2aXajy-GwUd&DpKnu8RN?nv7wS~xDzbivwMhn6sG
z&EZkOZoy%Wegr4Oer+ve8$$g7q1_#E-BBL3uY&p4o?{aCKSAFEjNiiwTUM>_b2@~p
zO=f-#ITU|#UZh?oqyKy$t{Xk!UkfX2^g`p&(U==VhDg2XG1zIjzzhd|;W{^LamLWH
z+60oD0!Y~mn>4;c44)zSxEgp7!3YQ`CDf(ClJ~qXFT~?)+!X>tJ)=|+>|oS=J=9=G
z((8|5djH?#qZ!!N0_h}9M<+AAo{G4|^{O7oX8-FA*DqE!a^kP#vG69$(a#y(Lht~W
zqq)oTpY}h_%y<3FEuwl5#ym~+(7@hoGqq7r>=jeZ)+y6T%D((h<S^o<p@y<1qnCPp
z+vr(XRmCR;Sa%`)6eTy5kzRuUXR`!XdP<5=&XaVldFdNxZ<S2y{(XS0v2~~a>F?4`
zKc|jcK?ksh5))ia`P<`ViVDDfpcVoKSu7&c)<blDTOstmvcVt-WT6jaO4j6n&05UQ
zfo(95!TC3Wv*}3b+?<I_QJlAL4PEl!``bE*65E+7Yph@UFE5E(EF;_+X<S@m)(rba
zYOIFUPxvhCr#3{96c*+Sxe@d4n&O7zzOh!@;VVS+CT%7i6m#Er%^gf9;V?|$j@T+c
z#0>Pn7945(?134GJoX=85NvY`_$F3AIys@Z^v%D^AUzsiIyo5$d?KF;6#lu7pA#7N
ziUX$zx62IBUw&Ov(x{yLVJ+UU1aHZUV>aD>h{Vm!OvlXiO;%%**rIN{`ZfP}23qOV
zPH*)Lp&T(?$gNS=j#>mW!gt!f7Y2Bq`kBMY5XcR5Xx-Yy9Tpma$bCNJ!$@-m6YQ)D
zJqF%h-qlA~wsS)LA(2witKpyg<zuUOGN*qY{hYo#n8o;k)F5MS<ej)I<6pWy(S!=v
z^_x%#xYnrNRy-Ursphx$O9ZvFYY9NY0_+xYNA``>|Cle}WWwD)-xy^cPAWFVAwKAn
z2r#d_XfR9v?qViFPr8GV%BJW`sBcj}E!fZK@O)M(z_R=l@M(zn%D&WZ{>YwZ45V9w
zlgwvb_v7k4LiqwhE#_d^VozDT?-yZI#S#VtZAugQw^_hx27_B^HyMi^VmgeSR`IGY
zY!(Pd_Ta+7z_(SO?0cLcq!{6gQ4C4aQm+Tw<Uf>8-(|cX)paaOp<ry1@%q*T^*NaN
z+&uHQPzx=$ZkE#}E76k^*bA|fAQ!*9!OIp7!obtb@)>f-qRGO$g!0Hv#nJ!W1)<i}
zL*C%{PDGYrXfe66O=o6Yy1}`d#n=;=hP0jv*m?NM=(CqHNfPvXLCiH#Pg17{soB(L
zvpz%SSv26ecd3v*jS<Uy>;HgY0;lywwv|o6Fd4csluh{xtj_y%wV%;Vvk)t03X2OV
zHbDJN<)c;kZZ4W9)n@7kjfiDk96dQ@a-1vR66?qwFd~(mY}u2eV&*MAv6-Y3zfW9z
zlIjR<AD&wuf0zXQPZie{)@0MP=>j61NTe49ApsJk34(}3kswHi&>?grR6*%YkfNZ3
zE+T^T66w7Q2^~Q?getwaQ2gWj9sMVBu-ESFv-dU6%ssn1>)+`!ZE1+!eu#D`B@7ij
zTh?9@?uWDU6(WDk%#fNw1H`PML~Be)^i!1*Ijb-qV<7R$sW6WirGMtBd$~Jkhq{Fx
z>a;TNb_OcuWpv*4-WX+=!F>82-@oKxR4gE6k@$F9`h<h1@241mOZ!F-5v5qC7&t8l
z!gexIx<A14@LPX?f_^AhDLE$Ojk)L&KI^2D(z2kx`h*l>z@~oqgXbmFR9)po!{$_U
zff}1(k2x&SFQ-v_)M{Oas#IExkqv)JypN1KK5t`m%_?lMlz60%Q=rro_3H6yj_C=@
z0EezD(G(H_M&w{jbiyZhHB>orEE*B>ZCcUN*qY!?;lnQ<D<95>h!6P1y_h$<=f4N~
z(D3X?<9(heTIOdQbve!YYt;2Re)ytsxgA}O6j6Oxe9!6TwU0~)1UlH~yKG93c?h#*
zPDjwdlB8TmG6H7*WSuLU+jN>@t`!ZD3wv<YTvYNTrw>X_bOn5ymPS$crm60n#!|qy
zEf{bCVRI$zOZ0!kk#TpCBc~c}y*usbOPY%@ojV-k&<LPi8IsOA7;K&Fb`8-b0r=PS
z>i;)w+5y`ovUh)@H7t@{H!}j^pOBm_Jbo!nPsU>xU%#EDfp&&zpYg4esy`q3z;qYR
zT(m1Rp-1-YZZHh0iOMypE7vAP3|cvIkHaGAVOMyqN#pL+ieBf7@AUe~x*O;IbM<fV
z%VL`33-jms+x~vDtDg~5{ucq6JLgRUv2>gq;vbM5hT$LqTqNHIqzw>n`A<S9Ffkwj
zg&2HUuX@IPB<oYXH}S|Yp|%^LHdnX$U$M^LyD}s84W<i4I`#>Qi;GJy{Poco&g+4;
zkpro4+Wr&5__SweChl$tL+Q@wSY|UF*c9}P;@asciem5SzbUf9KpY-uDLsSV-l+HL
zBYW$UR4>EMHhXhgx#r9z+!&O-&c&Y4UE_96sXARhw^a7qpV$D*bQ|XOiP9Pacb<cs
z+>t3~ry8mQaZfuV5skN`R;dc6T95k7QSU=GhhP1?MfK`1-FpDK{5`<mZv9v4@){@K
zfh^iVT`8k+Z_qZkg|WhY$PfcUvFz|d10EBL%?%kCN|6ORYx4Y}rRck`ns#C>?<R^N
z3B8s6X3===cH9SFPh1QQ$Mu5&9#EtmEi?B^BfC#e>m-UL4W=lc91F8PF(RL?-kd?r
zwk|}*W7%)?gAg7(+K)Z6uk^jVTt$Tt`hJ|xlRK!=At`vaG8DbLSj2-wFZfli!2&hJ
z8dP!=EF&|yT2$1iH}j^@SX%3<sc=2Du>Ud6<5z~9C)E|`$WF%nPeNtw`lI$O5enV2
zqJn3=>|9)4dU^@O^EcOrM=KeFK|Oj^^d;YG*I!oa;8xtG+foJ6u_!x7bH1N5Z+5X8
zQvT+sg6hEbALnu<hpoR-JCW(3*jL-|V&ClGv~FSW8;7+3ze|Y)2<W<Vb&J<xf~WFS
zRR$2>%{+1psqP`gaf_n;xg`qF83kJ>+h;U_^0OOro`nX;A9NbEYrjLw^J@sXfm7l)
zp7ZPNg{EfQdEO;WHa9$zwe27^xBlGa{;X3$QPY{s#T59x-b`HAj=b@aMnJQ(h%k$@
zqvP`hU73*so%)^B7ED1s*ZLKeoae-Er1snCKl$(a|28>_8+zyVPTw%~vKv0wcvwwX
zv5@6620ZA19!&WswN%LDhK+4r!x69|rI)tuhWcl`RbLxCf{-Vl)6*xt?1W_EUl`x<
zFC{xCKLo4RSBD69X)CN7J~F`(maOl0F`tCFgjvsjKKe0$E>8i9a-!F!a9{N1T7%-N
z;JAq~<bX-_={jr_(_XcsC>vxeH4fI9*$MxxRHspZXk3WQ0xpgqjA@T;WO3}#kR?1E
zkp80_Q<a@xRZerqfCNV2LH=+<ESS#X=5XkJ|3KJ9*jvL0+SWqgBS3$YtCY+@=<K#d
zx{gBDL*pckt*PiNT!&r&iWG!fl2w_zU5^}O>b*hvR?~tN?V$35GGEH$dEFh?e!41w
z!a;fQ4OMj6xmJML7*j5Tss4+Egbq4a)BHKse03A9(lR=Q%@`~jc#7&Qo3g>_J?5qD
z)YqbAhXoQBU*lg(`g;~zB=0vm1q`XU0HiCmOo>!a{C#ylRSxNO`798YF1AAU`l>h#
zlsIQ%$}fGkwle3{J(_RjyHp@-)fwwGFbJ&Sic84)lQf#NcTD!8+#PELU<;#058Qg%
z331W5j=a$zBktS?j5<Rl{E%8#r5*+M|M5xZYru1-nBlrjE^2e0$L?R{Pq-g@Ouxq<
z8P^=8&8kt*6%YG6M$#?N%iN4(v~f`B{(YA}Vq9t7jG$;lM{XlauYRzbv+}p?wWnND
zR1kVC!-!NhG*EQJow5pCtfFUigQV|{6RM$nd@_;?;t)H1R^jhsrm*tZNO!eW(<Q31
zvgt<*8-k-(W;h&@1R*59*)~wAXd4V42YpO*M*<VF^9JN9i;mqup`cPy-7{CTn#bQX
zqy%$YqB`zN^~L&zlv+BuibhtHRZ6sBE*PB3)gh#DJqrCBNx0GKtj5UD(G~c~{n(EB
zUMM)4%pDlI{#L;H6UP+q7rtO&>xDy-;?fo}d5=AXuu}hINE<5v5b9v-R>#{^s>1F8
z%w`Z$+mpF?D+`6W#Y%tK+ok>5hEWDuH%<YqQ<5R??u_0AdB|9Q0{s|>T@jIlB?gMF
zmVprc>@H|G`w8YUx*(Q`baGCICKy`IG^r0eDcY-q@4&=}-ipm-JE$XW>W8<YEss=)
zux)4g(c{WkAfGkc1ALMi#74Z0cvtw=>N=h~b$TBw*~Zv5f3MbE+W3U_x?1|}-8?0z
zuXM2<0I|$3`;7pzJ<#6xw*EHL@KS4LXDkR+^1|4*G@zz}j-qw_XJ7E>`+B02QZr$p
z_*%81j8^7TiG9)ot+hrSSvR*s^D)EqH_2?7(>ZM>;^c42BDzl9G??|3w6h3&h^jB8
zy$z)Izl`HEUp=`>Bhf_Byy}L2mx7VA9%M3Tw#6_B#I<AejD&#NWTXz3CSf0AURVO_
zu2Db3wdB&1+HyOlRFZZnk?s|2!BUq4d)ZYo$pBC2%IU_;XSmXJdv=;=0G7*Upz!&W
zIsCwbK}Jztjlr^i3%FyjZ)&(wlrR?^*mWg&vnUiK`N|q^Mp^o>EV~^9Za$u(J-IUh
zEf=V#ySipk^3o2CzqK*(==7eU%aLlD4Q{=3^stexyh!o!i<;7DEghB<dZXM1v)tAV
zq&nJMERfIsbI|B}9iV6om78gm&Y-JEp1qS?4}euSeU6o5=NkbJvZkQ2i`hB4#u};@
zRTc9B(KPWgrM-=yKcw&q+7;!;V?7fd<qyWPXc<#!vr^(4d4k`xRhiL8MN701oQ;5p
zmIK5U9sbOjZx3a-FIo#)4KJI(x#v7<@@s^%>s|&$goi%M4tLu4EeDc(hKvnLuZXKU
z&V!N`@+^<_!7855A(<*n$Mp4l1$33*5^%fzpBt_)RKHi(Q#6xx3clfeIm^beFz_C>
zZubtOwcD>H5&_-)k^3E(E5(((CD)%Jjg?<#Zt?2xBa9%ww*ET)eAW0$K*Vw1R)0c_
z`=MgDD5Sf#sr?;DpbPr>O*JYCY|S}aAO2u@ub=1jd$mpLFuVCE^bH#_W0()84bTWi
z7ffU-)C|_L-oJfAVwp9+B_PADLFv%>qt2hS3jd=YXj9G6kK@7;YFj(1L<T~=HLrnY
zde-$7X-$raVK4lT-mqfe9jkwaMm0YWS<QSty3e24kl5>UkUH+3$9K)l!uV>@`QYO4
z`v8{yXq!ondWuCdpL-EeC)-cT!WEh?XwzxEH~&<fOG#gFs^a2^A0af4#oX0t7<JsK
zby~ipwHjS7`2^=m3!JYoE~f9Rkm}!JUg7ZpsD`W#9=BTdEU<xAS{%3(vM>hzFOlty
zECQAjANh%tiUw68xcJzn52el}Ex94}!HN4~Z|V_ZapRG+=wdj$wcK^FsB=K=`=Lj*
zq>8hGVons1zdi1(Mb0WZawyJXki&`w_KXYm>^C<F(qdMTNNVFQPGXmij#{MC*y}07
zdRP^m`70U~P_$h6-?Lo-8=$g60<!RO6Ma^U27feLHDB~q1dbUWcNN935eDHzz$5YH
z&NmIcjkAX+By64R_j=pFOvGVyDqVj(O#vVI(Hp-ZRfD!h6h<&chRWW+*cpt}6*(A@
z=+$AX^}LQ(3MH<q)Livmk`m7KqA0`%5nHbG86-e$fc8B*9Tz;<yM<~j5=16lz6Lrc
z@p$kWx3Emy_jJK|l-Vs}Xf<XDzVKTgCG8@-Tc14vW`YFnh@?lq%#?DvycjFl!<)>U
zGeyN(bO9*9flL*&`0If3j`oeS;<XU}-hH6vfx<L^Um4BB!WJj@Q^t$1K02b~Kfz99
zyb9OiYFltgi++vc{|K<cHO)pLqIQLA|G00&Lo|``zEvj54#sXMCT3USpG0B*>^zk?
zI9irWmb9YZRilIKKHL{QPZq7n$?Wf%<JGmGHm;2%nzPjQ_R})QI1e2AMQ{Bt!_%?b
zM`zLHr0}SVH+5&HZ|dO6WJa(U5AHgF;Bz$oVA!X<jBMGpotQs9ZlB?z(MS3xCS1T2
zMygr2A`om!G;@~+ofCE*syH&4-GL@zL}an)J%hq~IX&hL3f{NM?~e(oe6=@>dI2g^
zYLO<FL@&Jf$-H1ZCd4xWLDvX5FG9G8^T|*9H#v`P;6$EjhlHni3`_Zb(T4jdFTP2$
zA|Hen<BQp}LD#2QfM0I-3;ez4gqy;#4m-m+;+U{v!0F<*LFeG8al7$&y+0XAl&i0J
zumhn&j)z&NG-||@VH<T~w5B&t9+si{x{?IZQmj(jaZS3sTyW!hFT3wNxLdSu%F;&p
zLS0=G>Tj@pK`x8J-S)ZCX_F3>_DqV?kU!;rPCL*`p`J;?@QY}%KJ6|zUc!8VI>Pv#
ze5B^1TIVzdzm@}e%6vlBUVo_9L{+Gz@&#6v@PMOKliotS5lbE|E@zeE65+r%v|k5r
z=1T_CECi4H5dV7Dnl#8)WX*bNVx<8|JC!lOU(7n9m_Vw0_dZHx(>DLxi#hGX4G(ef
zB`kX=*)z^7<z6TY4qDREqK{>da(l}m&)f}MZ0lsInQJ+~W02$2(?kbd20ESD_3E`Z
ztQ8&-*@lw<=nEi|c8ro;E&V3-$Xyzt{H~`l>eCbP4@&}U*3qD5ldDIA3Vc-X(A^(o
zVJSj>3GT*1X5x~o$c%Kr{@Ew=k=wbVk#XK#)4DjIbgok!E#?2B#W9e0RYb%(e2BLM
z=S@q{pk#9a3-v&)rt!0J6v>cWw!SMrNnNJ)f1bboX8*y^X*<h^c{q%0!0wh0h5DiS
z&D%WK>#5(BDi!Umv*KFgmFklq)-<}u405)S|3!+FQw~62#I9e}2OstllBEP>M@^`=
zATN$OX)un0=$7~fADsP{7H(!9Z|B@DkceA6)v1uJ=M_xt>dn$kcu9Oz$6m*XRh1vy
zU~zv2eoM8%CUl;E7r>Sf&ugeI!z|_*zf=Nv^b4_egW~e9^=mknJo(;u`s(?F&1A*-
z!ror`oEFfXF7X9W`U_M<B`ymp(oUrdLv}+*0W_GzKp9(f#>cecD)+p@VNwy5Ka@s-
z)MDD6y~z#nkVcoiTnOdd=M*p-KnK_+{)b<A7yHW{`3@V@8%=RO*S~XelLo#zdf3$B
z`j-AP(ADIJ7USK0Glk~;?4AQcAix)LR<|mJ%SL*2{n?6g73+3Xo+=!^M!rA!#2|A$
zlc_dyPc*TW6qqSqSPOR4&lImWnL^*ne_(sDR#O$%KRLeM^iWxS0oOKr(!MOO@$I(l
z!ta7y?_{;JTDWUvW?mv<dy0kp{lFbjB%%a-yEHOWV{8Dt?*ZieuxGU9TkLG|Iq~>r
z!NQNszR6|Tz^sSX>t?l9zeX4ut>BpfAf;seA+3~G-zJiKL`?B=F0`+4%e9gv?D&W;
ziHWD7oM~WId*d_%eZDDPLbWF{DF?%>*(oS163Rhl0<kS2Rx7{uHRmnF5_B*MgKp8-
z`FBKEq!tgPuK0t^R#S!uN}{6m%^2}74wN-kpRh}+i7*8@-%#v)#FuD_*93^M)E|Sf
z`xy!0A&sKvWbx>gym-kEYr48hpZ0A5HXL4Bsr7l^d4?{IoATlUyZ_$HNaZN)7gZC}
z0GZv2)CRVR&GQF;Ug8<6B`vR$YOu|EC__nDDm0Dl$vJ%M`y*%d$l`AoRxO}(HG?oN
zk~ug>(>-1mpa_}sJfu(L+r*d%$|`Q=-w%{gLQMapUbU%x%E&XBeVigT!O~Bg*lJ{K
zbbL4)2P!?vqw8PjxoOq*Kf56s&v+0vk4`RvD1hO%@w}J))#SPnyOUR{A&>lbk~)ZX
zWbbm$uiWrWykN%zhNLbg;)3nsZ~abL{1MA&MmoDU*2b@lZ%@;W7$o58tEY?vYs`Le
z<v?Y-N_)Hbm9OHj-lRj?q^{Vq!&S*@y^B^(kf=B@W?|823bsz26+*pmVk=nsCXJav
z-{!38SYR-Ao;^C6LP$XsiDHPRXi;kCCr!lvT^i;mofj4%AIpOBu4)0x=o3lh-eMWa
z^l#EA5}z=cwswvGq7I<3{g|jr^?Mv4I9Pvql*XOd3L&bdNNiO`-=tZXGk5Q#{%vR?
zI2gFY#kF)uUQ%BFRHCBrd-4NVP1g#4@Vi5_4v^E@EuL3VUImGY?^6xIgniB|BSK0X
y)xG0z=p#d06RAqZOAesPqqB?;jizWOiQJ3oDmv#Ro<ER~sHr}GT=K{w<o^J^_Dv4}

diff --git a/modules/web-console/frontend/public/images/page-landing-carousel-1.png b/modules/web-console/frontend/public/images/page-landing-carousel-1.png
index 1ac243c24ec6cea433c81083b3a3ec2e2a48062d..d7498754aebab0f70d9551a86f215fa03571d245 100644
GIT binary patch
literal 20468
zcmXteWmFtYuqAFm5*z{nf(#HoTtXNIC-~s*?iPYuaCf)C2MF%&4#C}Bf(-63n|-^l
z|8#ZLJ$35#X{+uIS5)|pgGG*ofPjD_EhVmufPe%>KtR02K>nu@<0$DzKtM)Nlv9;}
z!{PV$_m7W{S65dr&(F8Fx390SPft&98p^G$t<%#}7z_qSKv-W}^Y`~ZJUkp79i5+_
zUtV6eFg4rT+xvU`cXD#l+QM>fYG!A9$J^Wc9{vxGP*YuPXkZv0pHPrnpslT~2nNHc
z$l=%+Gcz-Ad>pu`(ZkDYTT3gvzb`gA8o|8>4u8%_O@osXWo2c<Ias?oJ3~T3`g(iI
z%gf;*!S(eG_O^ELoozT10~`SruB71Q>Iz?492^`3FfzlZr(Z8G;6j2fKYqf0{DAv-
z!FhS9sHx#=Yw*KE_~;m1OT7eILPS9Hc6JZ{E&+eN9@#iYuxVR7e}I>j!fh<uH!shw
z?-P1=;fe8k7Y{-8i}3v1h=_=p{VRQD5_m>N!~9=(OVj1U3;bpeUR%x0`xU<R1b_Jp
z-=An|YAPx!dg)I;c!uZoZitJC!@D{WOqyh6<q!e~+1c5v#&<T4FW=6Wl9Q8Z0m|^n
zD)@Gfp~!!O=g+}iyNEGk@bw0GjMLj}9!O9S4ts+~MZPVbm{hNR=VamhNc#4)0_T>7
zBOv_}q%Qb#EKZ0AKRH9vu7P_+!%4W|ojV%Yf548ha3cr!FFiV-oV5}N;%5QT;)i3P
zzRh>R(>qS4Ti{X}OZRX#KI;#pK*x$z_{Dgj4iCJ(2iYzG9+5Ge;qGOi)Z#7<SLA_@
zow_w|GQ<wS)1mOT0JwoDW$J`xSm9EA##uu`Rf4ONlM|fQ424Z2*+#AE@2h;-GQ4q)
zH)jTJ8lkT54oCmwTs~wK*S?$UiY#c~Qj~-#?^6|G5BCsr6cC2<p}@-<;pxF$L5^@q
zv()7qQvVt_t~6ZuGu)J7v$vepDpUX*3a@uS=tqFpBEXB67AvE;R4fy#2H*i5K^b-M
zniTobb9meY!t@t-?<a@k3M%h>mw@D{?9l7YKbUgyaA}XJ040!H1Kh7TD<<fy!WSO!
z2Nct=8f)L}r-Lw_g)ra%H!eetY?x{(U9GDE1#h?@Ajnloi;Jkbubieu+V`<zNbE9{
zz~MuE#T)SwO#0<=Cv%Y>Res3&$7%4j_*|$nE}GD^Rs_5E4T)`g8DaiOy0kc|#-@2k
zE;)I{ih)gX8Pm!!{6`|24mW+})z#yuV)<&V&WXZh;8k%tcvoB~nb~d^xlZlX`vUC9
zP4-j8`k5N%j|&xDpX)UvWA{W~FXp4ANXm)7gx?CfN#^MnS+se_>H9rTN{gIeKTU=~
zP6A0n1*!rxmh3NvMs$~e{mT}mPr+_VXe1`~Mmw!P_i2`BEji3{*IoWTiD#n~DG*k9
zU%YLtvHl$I*2`61dX1Up<wX^dN6q>8Lwj^II`z~t8r7}?$ZzPRsV3|_ugw1|`U`J`
z3(i6;c;kZ~b(TV#9_b`7tWRh(>1D<AVfq$R&Ly#3JAt5l4Z}2>{zQo`-2ZlMfgpB!
z14r*mc5leoMR{DcsMsDp16Cl_1ZB=r*H>TLq(@qceFh8XOc*lL^?v;AdBgJu@K^rx
z*;mso5gY`>|7d2#TL)x}BT0>jG<l7Z@T+bAoMGUQ#01x1v2X>k?yMDdf98{RmW?vn
z=>$seBqSf+R>Zk=6HS4GQ*X}=zW&rNC}}4d)sgH-sic%<4eV}EQ6YH>7y5-37K8EJ
z>2}89lIN>hy+CFbT7+z)b5#B<$|xrK8>xKhilLeBd)ft%q+R(m`k)9|z#wV_7|0s?
znK5>iAiO~36Eds!&L>w#ufTzOv9}YXx_g0pId<N-B9rQF6AZ4=&vZ2twa&w4M*i(U
zmP#=~Bb%1W9sUfGLx-er-&%c&i^A0YM&@$y833VH%A??5ZD+wPdt4!~<fH-lP5<Eq
zF*1iU`g{J1FSHwRy+1vFH;~Qsj|0?>6l+(0wp^y%7>i?`!Ok|ZSvYZu`&;zKew-#o
zyl&d+$n{GKpOB6t&`_a4qRE&(pIX3G-M-)9R!&Bh>)st1t+;M;f~oBjUYC_lp29a6
zFq1$C$n3{Vm>_|Zz@3jDUO>D}58ZmN2UKz2L3G9<i@o_P|GC}sS4fm$+x`gc7=fQd
zH-RQLXa?tor`ZJ3$}9oCD%?#hLN?;i$LGw)^|E{G3(E=+^Oxj6Jwq&T$(yDE3Cu()
z_|nd(j|CFyd?N1~qeV8kI@b&`zS{gE7q6T%^hm0}PxG56xlG^VpyWM&Z7%h4Tw0Dh
zZ1lfU#a33zIt_gUv+sHf!?+!TmL|tv97-xTE=d+u25Bo5i`2!HTN3n^e=D~u_9ja5
zD(i<u%eWTB|KdNQM>dQ^jZ>*6iOFEKb*KSe^09g%=Hsbn`^P=gty#E!<>9x^ezuC!
zyM<WkPTxK#v&45un)0(oT_WgHduQNn`r-_)h{kyv2=gd`)k^1tvL*uGLT0=D+kZ6I
zz8IFYD^VDa(YmA{(jFwd5&pKuKTtj+ti=A_bLj~F40})CKAo~TUU*!JFjt;a`V~kR
z_Xo{&Dy~f6`Kwe=e?H(hcddETl>iw}4&iIEDA)cWrmnfG*kGK{u=2sE6gw7~|K^uF
zuVi)tIfkmQ&|}XXhSFH-0GFgvFV)rsp6SsiM%;M(MoQzky`!Bgm)K_ao!`5(&2`@B
zFdbjbi5|YOtAm4si@m*5ThQXJ?a8avb+w8*xR}#VBt4doohwm(=?@LNNtre+A#Opm
zs^%*0$U)>J<Sq;^#iUx3rLz0%?Cjs(FW~sEV0Iz^+SKiYVANb+*uIoEt*NyLMX#F2
z#a|a?tOSZo3J8{N-gh_rejXaC?66jsubZh<$(TOEA<Yxzue@$NN4v)N`#skCbMfn{
zTxNFO_36#x<&L|7re<XcEr5=BM0&((#7<Xnwy&@6@zL4Cy%{(H=zIQC^JkW&RtQt@
z)#&iWBNMInt9Q%~$%$ic(6cc*U8)d%-$+7m#BcWZHPjMFG%2>&dSB?1mpCsF(>`FY
zzqcLtrd%N-1_^6Dtn>GLwUR>pEJX*V*z7pGUyA^F!e$B?nYI#-A}rQp0RXC33x>3a
zT`(jj`GX8l?QpX*I}(cI^UyE#(Y<ij({e>mW~yG-1u+lgoV59wVd(7kbNUUw^dE_x
zhq5ocx9=;x?iV^-xca@kj=L=ERrFFaf=kQf8Whl9FPrgFN6|+DgBp&r%zd1Fj|nQ=
zz}SYV+p^bHY`L>1+t2rX1n~6kZYbA<bl#@Yrb18CcQESF6|Sn42OU!qk!*9nC4I}G
zR`6HntwamDSw0sh#dFvuQ#>N%tM>|_wF#)Fcq>;bT1jHLIafrRi*(66A6a*FhI<ON
zIE~SA;$1Ii-w~1rx#5iU0qED*Q&O~8k15##NF>hlkB&<276IqSdCZ5}C#F~sHVl%r
zM&(N(KX5a!OycMY{96V+ghG0bAC)v{qTK|CWla?mT;E|V_AUPqJEyU4(+#hjq@0Yp
zrJv1mFN9is<aphl9g~Il9?G1lw0^MP5Wt8y83dda22$94D)azJ(N}MGW7AL+B07Kl
z=-6}P?*wI(OG?ShCP87~lM*YI2oWGfcEY+seBw_x?bQsPeldp3v?!9Wx)~a^2#{_n
zG&e`zR%$^?{!=_TA;_+9ZBSS`y~y$T#Y@@oDQB(Aj}u5HKYygMKMU<E{r05(cyo4?
zjC3(#<sqH({)FugY=>@Ww|kK3NLv&%$%MA58mdJ6K?eVfI>e?^F|*eK8#Iu)Wxlc5
zmRWRU^(`-HC~GE4nOGRW&5#x3S~AC-!daX_oMFa&U^Hx!G2jy5kuNT)z&LMNYMx|y
zB;~voXjn+s{GNSnVa?_C%}R^W@$&7M8vEI-^u7XIu^;<Uv%*&k!VLNQHz*qoTn;wA
zdBPd}7wq;W9^_>OS}+Rbr9V4a#60Cz1u2D=ObslWQ#9@TfXHA%bn&gBX5JB7vx8}m
zNpJh#v(DGYU-akN%zcda#li^oJeZD2L6=#U7K)sqsr+(SoO6I<%Y@icr+0-h6Plwk
zw7dNy(n>fn*`Y261mHSF#hZG92+g2`g034of3K7lg{{SR8;Ngr>x<fe25_X;)$KFD
z@F$WaU^lU9+$*$T2o`z|!waR79EJD_3Ir-xIN&A2OMrsG3ef#{JXJbH(r}1HkY+rQ
zO?rxps8NavF-_=qLh1jSs9L^#4w91B3ENIwSu*)?dwlT67oe=PPcUEefgU$K1Q!9M
zR1i@};~f$BbwbfwDo>s~)N%OGTp2qV%lHNHj^PKyRM^bYCtvDtCT38U`<*KNT*MQ~
zX@J|a$O2U=um;ppzjJjP&g)(KjfqC=gdphY!S^dgXHJHtr#D&*9`s1)n-J>Z_kd^N
z2;!eKs*b{|M~V>sN?D!HH-fW9`20ZUX@?f{@Mk&~T??Tf?*%IN-Uhd_|2!6^6SiMT
zb4`5W?7`Um0ddB*y2eh|f+6)#2OB=oC`SZ1r_53>Shl=>0YFgx3J@^(hDCDcD%R&F
z5dZcm3$`X?baeW)68q`s^~lxA+j(vOYEHY!ugnFhZeD?jUhMqigUikN?hsx9)8P?S
zrvS*;&GPv#BTxvhLLnK<O8vK$?gu1SX2)x;0Yvj(LM55)@#i+*0?2J-t-J+DEf$BO
zeYM5w%La;<m&h-wc|n&zmYk!|OR?+$trS$$FrHrNEEC2B34bond8R1|3X*SA6vzbG
zCJIq50$<+ANIRv5hjVx(&=A!9C|u4AkR>eXuAYIOZv9G{vO1VG{$*WJ1%CGpb6*3r
z?f;F|kdX6|rkk|&Jr?6Ce4BvYey9;^6T=YKUiVqNLf4*-C^dc$<KT9LBGdgs;Xb1R
z?Df^wBv!Kh;IA3(*dHB)*OlswWc-E2$gO-Q5wjm5{C_RDY@g=V;;OV^TPi7MlXtsD
zLr;6WSaT~+C~Je(=s!I#!m9u~M4Z+hckqJGh4c1a<%<%L7=ak8zOjM&=p3C}6q$d+
zgB)U{eqJ5&-8Z5as99?^1Z(>SqeGsSHCH!dxuB2nDW34X^83<-@Mr0fMSO{$LgDOg
zlF%=GIig=<U&TKUq=p->Gc*&+cvay5ag>_eV2>3Q`H!Gab7eO3YriR8H1Eolc56J4
z!7V?C;U!p-dV|o)+mQz)NuV||yyfGYT<n-6OUp<pdgkKzfIRTO_SH70!@;`?6#5}&
z!x82uV!av%Mw~nv>4a6~(XbyJKSqEo(ew`4;PNZ!9%xu-GWaERNL01&(O4HkvK{@l
zY4ZqFHU6ljNo>7@Px7iJZ#G+WqGEDy+9<Z|s}g^E0kI6qozP-f`RaXg@v($T*HA}{
zBUMimqTRKUkXUp^5E}&+TwcBGq9zGrPVvsvGVjnXQLkrJYEZH9E^`S0bIzGH?+VeD
zYATl>FQ^$Zr&g#%O8(-3kTU$pE(e!gl>&H{YyP<BEV^pjXz4+6XUCz82_qBG|Au~B
z@wLntE<^V<+>{wQK>tuL;o!L^?oH5{_sXROXnysw4ERs1nSp-k+E!tcb_Ln-`Rx$c
zJ<z#<&K{5<<Xx;6kK@p&(<Spa$BsG>=7VZS`#U#~9lzg>%IkUd`u{k}?)VXF$ApFa
zpwV$|+o2a~r|B^5sZb#^trWFjGJfJIWV``e)C_ZrqOrvMJp?pP4;x}1iE{sV@ZTT`
zW;mZ8H67hU;KPL7Z+$UW*`{w@h4+mD>#rxNu;&Rk@MJQ3*Y!q+iiZzY0qAv8>Xd|0
z9AfF6|IiH}hcI?l4RwIjib1dUQZSMr@u>&#OD*rK#-T5>58_NK0Bs?qmpEAa|3xTy
z<3oC_yz`}E!E=C!s^|221L3^eulw)lzkphN@?VigVLJpJRC1T#&VpCD2)t`Ol!_sy
z&M<I2L^_@v;tfk|=*CJC7BTit$bsRXx}q`u3V<BC*`@@KA<1-aMLwv$yb&Z0@;4*p
zqF}%2!F0O$S>~X^y;au&gCkljy5x#TXq)A17_183f7gq1P@P;1w{-S64fmb>srlbU
zWo45^!_1s>laO9`yG$}x1a2F{-0}6}mf1U5gtM5Yzji7c|0mI)7}!fXU!tw1QrKI<
z28RABLg=bsliKC1z$6anYrzsHtndI4?j<KpmwL(vsTbgtmMn&bHbg0_^Qf1pD<LM~
z$96rX3;dT=S1^BEO}*y%K0fx~(Ot2BffG70S5lwRAGV|@CiS>9m^&M5ua*3yighvj
zD)2jzmY^*!>xdxrdH`yLuq*$c{8PkJ^ZSmGLe7jqw^CHJc{wr7_uTfGCC1qQ;h6nW
z7LDIZOuFsl7pHw@W|*Z<GCR}&iG<~sGbkWZtBmAEF^8$DQ%82IFit`x!=&StMK8^S
z5+CMJ-Y}+VS#iD{v;~bqc5X2@#d)XuC5F3$b$K?*RsF{RomfwEJPDe_g%JyBU#3_z
z^Dk5c?GIUkD5M;naX%;3Et5b<X_*66?fQ}Y$FoLh^$3y?GE6Jf_sLa79qBLsa7Sam
z;fsoXELP#W;O8jweP8l00u`MGpkvK5&(f<F4*>@Iy`+*vmLqcT^v1*zO|vKVXX|Tw
zE^~e{j%?g~Vy}-T@3gS}rm@7tSI|uIYRat;3aRzkRl!hwap@{T&=dXoom_J@Qj{Nm
z4U{h7q3j__P$H=?qU|gf%m_pyfXMCdj{p<sM(0uKIB2tftFQ_M2E#O$s4J4$E#*1b
zTCJ7?TPpD60wC;HJM0*$g*QX=<1i^2g{|gsg}6kRFk~eh6pcnn?krA*pDr$*FhpQL
zSj$ealg!}uO2`&yiC_4cNDV(+%}f~USQl~jR3?yK=)qJ6W#HhowgFk-EE;c3_$Kyg
zSAJ{q=*R_gf9uMvSD}2AzwJoHT1q2Ip^#2Fft@G>UGKd=DKsiP#}kO8y|XGMESY#k
zL01iYZXn-pX+@JP=BtOP_9d1Bc)fC-q+MHjz5d<+{lgn^12ZbgbPhjtC(cG5)O$JG
z00acu9*-runvY`i5-*ya8ZNHZ=+b7d(E2H4Rxa!l9A7Vs^|n(rkZhL|TWvmJgt|yi
zdWmdpMf!?x9JnD=d~>ufU^!bxdUtro#Dp(Vih@x<i?#jx^E+zuo^*Xkz;JFJCded-
zP7+7T@cF+Rl%SYM$%AhsZk6$f?^qg(o9rvovsYhSK4rdQGc2&$`i*%wJXU;1co!3+
zwUnFZBQFy@aaQ#vTEhPW9N+ug_vLXzPseDx2Fh`mdLE&D$7=cGWG-;5#y6v5CxPj>
zXW>=MwL>pN9~E2KJE8Wi^pobI$s--s{&0Y6mcS-GkEqa%f3?!)v+2W*N*L00E(xua
z;r2Z6eR`T-ERnsN23s>f+QHnop^7c7@USm~F8>OS@)Xpz2|VFtE7$0Q(kA4GNhmQI
zq2gcIRom*b(&<$8?VzvdQzlIA^KJ9Rb}wd{`|0ts3Bd5~n&3UV)P?k@FX<mCzufML
zmNqbDCVz`(K6r#Y7Mu$Y20H^irvp)xfuQw<Si~g<SmV~w=QI_8z=5am2ktA^4Dti-
zeF9&*@;_Qo1>V&AcMt{N6P3}3cQrLXV&yk=sStK&hdYKQ`t8?Tt?wdm*glCOZ0~yC
zAHF(0EBLEvox7Too33bSct?l{l_B=3GOs|xI7jII@G$fb@L;8;z&MA!ZF>@44ZQ!}
z{eW=lf_^TAf{1X*GiJTirb*V_{rB2-4Ib+&BWuR?!u|-Zg9N)nL7C6J&5>RyK8248
ze|<^Ms&NfA(Q-{_4&P^jwJ^8Jh60rc|D!h3CfQGfe$qgq(IwL*rla25I1T9u*WsE@
z9~w05e4c>TG#Rj6?X{6Mfki`mKOp=y(7kymCrA?s9VfXn{>lB}dOaf6KkDZ*-DwAR
ztatm1(toEQS7anKp<AvWEVzUt8<|ES+Fx(NjSzHSky36j^T`-hqS;meL~-?LbQa4O
zs;3_rMX=swu6jWiK^>NRc3u<7&XY?w1tx~F@;9Z(Po`dI;O1N83<21{oK@U>CvPd_
zE{|y+>-&Dbn*^uV`O#2nXs+sgTcuYU;mPH_EX3x{o1)o+8Lg@XZ1fN(tH&$&_0o5_
z74r-0E8gwwrXI}ZXTAb1wzAgmYAt_|2Mk@~FkVwkt<w_nz$hPV<sK1>bru0_q>ldY
zzt+`_Ir?aMoqKE^-9C@i<~1O15#G}r0MEE`S$pprtS4%s(REQFR$5Ca<YA`6i@~?a
za*%S->gOdo05dePfuP!edTsIV#0^Ph)xg7Ct&qN9?7xfpgzArQa^dX}hsp1jiB{X<
z-{^7`&$>?tC@$9wA6Ed;Lx5xj3RRv1LAp|5X6uIg7v$cuR_m<ic%!>j_#<WSJ`N(G
z=iB-d^uoCO-t66q#mco3qpWC**8rOpQvWVf5<ry>fZXZA5~J45A$Uy9m_IOmWtcM<
zsfrWYHFmc){*A}LEzp?VmHoxy;rd>ALBXzgk5Kzcr=khzS2$PZM`V0nt~pb4W2F?y
zY4acd5r-+7|EQj^<>h)GK4i%++`LRkZjE8G#m41&=U<F!;;+nnci-S@3o%1wtJQWg
zA_^11`h*#RkN7SLU$E<${4SGMf$YB#BO0^5L}@E=jj?1-nT@y3c|D0n_bh~@nP5W5
z+r(R^Eenh$B?LMXviH^uyPr%t+0`RxVz?p6DvHV9O{BWNSL0o9u~OAXOHxMn**LMd
zn=s@tqQ>Z>Q@ogxb!cfS+>-G8P_N9MZ+x>QfhUTmd~q&&lOm?yZBfx!kJ31Ho`5*C
z_xz^ua19$b<wB(QHSS_NS`I~w4tnkMhX{Yka?36J1nojLd-ln*Vsxj%#tHDGNO~x3
z-AwgHs(}Sg`sQ~b*LY?<VbWcNKVc@F-Uo+#uG>t##@+i1m=I9(-XyUnZG)u6ict!Y
zP1Lr$6Z%yTIlX^(o>xRz1)WvNxcelKy?tQ@g2*kLOj|a1ZrDR-m&-TdYiD=Q_kRu?
zAHIHSTi<toY1G6<^UzPxy4!3=5Nb%zZ+<AXcsjOa{I@?k2*raHg&<c3LS)ZABT)70
zL~yAC=s);XY5T)SBb+G`LCzR?(HrQ|4W{0P{`E+e;1DgwemU56`{2)&+YWn{oSuiE
z6e{7`UXNpRQ-VQo2kBveYJu-(h;VBc*9r8u5QY?V81SC1{humMZ~IkG6!a_k!h_)&
zK)19>zek{n`3+VAiId&Fe|H3<rpr0q+~lD8K0efgdjBp?{So3e4(XPb^9!Zhhk(Z+
z!ew11pqRa9fdjBGh(_&Eb)7`s4}E+ne!@H77wOSk2LAwTaGLF-NN7ee<o9n9^I4OI
zbU1b_>$`pY)0oXLfIGcD?F3*aS;Qa?rV03@VX2V#A(u<o@H0^eA>-v|PH}N|)I@%B
zF-taP8h+FiuKiVR6Eim6yV#Bw*Vs1X%H8HbgxgNMbHS$P)7|xr!_LXNE{ppX;f%PG
zcSyT#h#s>`uS|}9vec;O4w#lMs5{_a+}SGK5LQa2;)11Vx*trAK0Pk7wpk*A^D+sh
z!UJJgL&rFgi36Cp8ij?0bTWw)mG$6ECV3Z9dikG>f3nG%->f{tO*OFEt;+_kqqv=l
zbDiX}kFx{_FzaT6Eth*l%THCq<&KeGp;pOD74hxff9dn+YJ%A5spS+hvVPRG?gjz#
z#Bdtjem%pAgO@NQLAUrJmxRa2;F>3;RRL#F0_H9E0){=SZ%c`5*?@-cISVb9+*bFS
z@#4Y-))^Oc54|c&CBR2M-@`deGgE}_c!WYK22|k>dqi=?=-{i%%RLWJR82(EXS`-=
zV5DFZtkFWj0==VC_29aR*K6`*OcPdRoX`DI5&hK2D(jaJ+Xd(T<o8*Md1s5~9q~(2
ziUPkjPwEdGP_>7Eoc0;?@uPFf2->|f_`Gd|F@lcJ!-2QbjFQJohJ}ncF_)|gANvdD
z!Xu|8Qta^BGT~3mCxYGknB5(#m+R*Q-B+~4yYxQ{XUjaCv1S{rt8XLihgUlH&s*R^
zvkm!^S?LA!jxW)-%BmlX;;y6G+=<wzIs-00|4cHTir*>=o(v5^6TI#Are+Yc2VOFp
z_y5><$55&K@bBb91d}AF`mk5f<c2Yq_R7?+;rAIs;tXvze3tK}1N_@3nkYK7*lKh`
zT`_1p2hh?-U&U;hfEwp$Y^5D?K%3HYc!#bV*IZ@le%G-y-pAKD-~FMP1oU$us#nso
z+We4DK$+iNIB7KRd#x&^;7v)XyU56@?7aI0QnC`1TD(UPk2MDoL1VoIi1I7L`(G-E
zok0T&R-*6!&iUfkZamLUT}?RoZeMNuLR7|l&pzYf;q5~|=hf9ui_zyNRJ=<7N?9|#
zebLZ$+hy!#P5L}Nqzj4L^n5mFN$La3I2NXo{}R$Pq2smK3<vm!LCQ-o_BQhek*Uye
z=r8qQDH&}V(ECD7?}fGR@4vmSwH#yQ^wti&s3V6RNo_gTi5|>Q959xasMhJx898p%
zSF|@9oX?;h5zN`eSfIx+RLY+B$C~PRIM+g#@;LR`T4^Y~4cNQO4av7Q_Ujmp@wo$_
z8K#aMl9xOV!Q)&3&%Z0>$Yn57Y71{9XM;P&6&2?~-adRiJ4Gs~Am3o;To-n;-A6;g
zJJWNzPT%>)EU+x=b$I`R8no2L(Sc^n@Z0qHJ4L3R&vgLmVmWGpehch1`2Igl$7|`7
zN(Ucpuboy1`Bm9H8@;r>+erXw6<g&79L1;ncCv_2`gQ$jB_pcop6$(~$LAx>b8Mzl
z)$FnE>(7t%tM`7RE4VPQiS$>E4g@zZRH?tZC(rMG4`kE~F@)p%_!L)Fv4&73D*etJ
zs?E4abvOWU$B5NNq;ArfIF*ojN^-`-O&(4-%E`k+@HxxhcbmvP5^xA$x&W9xDCVdh
z_L!IWv$QA|C2Ziu#snKT+F2`e8mn_|I9qbMh^TX`E&cAtZ%!b2H}7J0cja6<wbfa4
z=2+Z5JToG@>O>GB<k+L!P1kROQaT;UOHQEpS@|zr19`Z%$UbpBUGdWQE3~0YhB=7T
zu#BCY$$;*iJeXHl|My6JYk&Qd1Qtlj&buc_`<3yObr$rS2ePEuoadAt_3TlHd0WnF
zL3Z!6tRu^Rs+a8Dz_3esZCzod5oiY)sqg47e5HFu<~ld?RQnrCn`n?Gr)e;ujjCZ#
zoNR3mJU;qM>J9*Ba-z2uxU^Oi>!npTLIrG-i8#wz3-W8Zqh=6-<0Z%}6?-SBa=S+e
zOj7Hzl^o14pMJGjOx3|Ut={szOy9q}tc|cUh65ID$}mGwa*QY($_2Sv%^jXo&Z~ba
z{Vi}(YOtPP{Hdg5Ap@S)wbQZENU9X(*0&iH#`zRz$^0enw?BA_sATk~Qfs4gM4g&l
z-<1!W+Rm@io#ew0e=>+mj3>TJ%u8PXH2R~|fP6@`lX6q8xoi+Yzw1ofH5L&SH5M^_
z{8^pM<->=6IY`pR-9|$79_@pQTWf!FMUAIV(EXcAHQrkg&?oKSXnvSwo_>`I;#4_b
z|4O?j-S)-qu7<K$W7cA#jP=p_oJHb1BrDoKq|xT}bp4}vjw{}kD#MlOOsC%&@S%m}
zaF6!zE*)2pCR4}G_hkD!VI$wi#o1+8nrlV261?_!s!!La>~3mhInB?8_Pdmfp)Szu
zhR}&lGowdT<P9}!1%-@Bf&!J&ouNG5EwCb0B+>1%zkEtIBEr?%B?BYH|BwWUpVpn&
z=bn-iiFjQBTq|T0tC@p+xpaZyI_%eQz!l~>FR^9&-x;x<l8RsU!$E?MC9B5zz;Ohp
zm%BMk%!8TVfGiMrR&RcJ>G)DIkN=&I9)!{$4Xp-h%;ONRA{2Pzd6IV4x5I(id4WI^
z)PB}q?J9U8l3-o)Ri?1$+|0Q%`RsU!M<0oLP)&e#ZF1SV`0@2$%~da*Qq6?&=){p5
zPG{*g@tjQ>a2HSnN7A40Q9!P0n)91q;GM}dWX3Y$x%M>myXlgjXKJ!EFi(pIRF)f;
z84pa~bl5Uot9ceuLRiR1WVnO)+HTS1<bsIS!vai7i-(AAOQ}Fug!diGBZ<^*mxbVv
zm>Ji-(yhh(C>2FT+##@t9anZ}u&P&uVs5W-FDeNbQ&w1&1IArKKRlcL?icmZquVB5
zBpvmUcE*M5+fyPl5`|2u(*+%fk<HoXhjy;XA?k#pex{<d+wf{;QnHKDp~Tk$@f>S4
zW2SH@t}lQBo;=DX2wB}sp7xq+r3ZbQAbn3K$$ClG=yR5ijQ%$l`q^viBtq^Fs;rv4
zgk<bbS*T${>FV_<aK9|E)%u+R(Y&!2i9R6G+WZ<n(`@-$x}9Dd%jXFj;I${@<@o)T
zDG4VXY8ng7(y!@I`hn%=qGUMKMRaH6$`zB7j`-)H6^=SeT*pOBu$S_Wg)f}((4wS%
zf~Hk(OR~_KKRyRw@to?wLCP=BK)X1=pZ7T|Fp;5OuDBj6Hb1WlhtTkZie}_fI#X3;
zk|P0xRMEmpVToc<{j-9@1iF--$P!^d`C6e%rY{<m5drL{`J}4&hORlTuKIHt9dk(e
z#&_ylwI(`VyjLiU0$>>+N0iG&P$pPOacRWO3|Fe-`!I92x+?1xcQ?D_PdLNJ0$Y)s
zD^H0WU(!9ME_tpX=<elTBK3^oc<2i#9$>9E%8c((cH5l@f7`Q$X(9xfF7^!*yr^HD
zk>dfw<G2++dGuO7;9f`bWi;?xiFt@}J)!-<m}L0lSKM|@IEj4;bsT1E(#`@l2pdr@
zsdJMIdlcX{l445_s5zs96>>v%2q`0h>rZqN_;YJp=}UZ&7NEQGD`ru7X*vP&AU_We
z^RvyG&7bp%$5!JpCZFUZ@B^>!dJ-7{F69yxL<Lg|Es!R`qEV~v*#_6eP9$fiBA5r$
zvAv=#cfi_;;I*`bY0%K}OQ@6pcqqBqKrN`kZNl$jZql|k{7f%kSmnES6w9+57yXk;
zRKLo3gj&APesnnW#OE41c&qHUM%>ztNK<oN=%cqy0w(>zY~}59*%8Y3+)I!?-ZlYa
z5j;c4g^A`waDK?N_h)gE57pJe&9{wPWP=+WJ!Ui0NKZ893>?K5#z`!pL+zV=(4@ON
z+$Q7XZy(ox!KlBK#xN7UmS!@mrr!V5X(EjgMmI1jEvrAECsW@FM}s5pF=VriCcZGy
z9$-*>zhhp_tD8`EtT&bw*^f{V01rcpx^n|_V9mIF__BwNIn11s^@PiFL@7z)D-#4Z
zvZcA!IuW1o<Ds;sl1uob#E0<!IIJsi2bCpZz}yG`v<bW;BJ=(GENKP1OABpuTk2qF
zfn)<|fCu+f$iwoKJ+t;v&Q2%?@ZTLKv{ASeLBmqp#&%rEA1JN*y_r~Xe6j}%O&gRZ
zNCymMOhrutbYniI!DGa^M6*bzf(}U%>DGKtSjd75kNne6qDKi#PLZ%`S6Q9am^~=X
zoBI9G5nOy7C{6ENW+}GNESRE%wmork3%ZarOmB10ai0@~ckjpazijIDPfq~`aS+PV
zA{Yd~6jwnO!<q?=I9X6ja-FYQ-`kA{<aI1{YLZ7w5|<Nx5=snIULAn1U&CU@SVCnR
zy8>v2PWj{j<QcAMoifnvz&|S_f|o%Y0fR$C{-C5f?B@giDg;C5a1X&;*V@C^DYAKn
zaDawi`<>bKG7t56JMr3UxcoL5fV_x^IAQ{yO=n-pm>Pl0`K512#E1ioj333EmGt6j
zK9$I2TANNthVS`BLP7dP8p3Xh%0}^@`nxDY!hYK*ZCVm$6K+XwcDn8RKE<LBg$`n-
z&h6d9(d3efL!{B<`YL3fVt>byqqGdZHz7nRP^BcsWV0kd*Yy3*a+eIpRfI?Kg<x;c
z5)qwqe&O`j(WohfbvU>;c)LTqJpov}FC}w@6(vL&{*J(p#qZ|f!P{>TM)>HEp$^G%
zD{nA;j^uVOc=g$^r}(AyHyp5XSIl&{cRNq7y~hMIo8>!+ZOEvfxMO|BxeibC>#~|z
znS_CaJSqhB<2*>wF#_?5GkkSNTkav6`AgCmv@TRy8q~wD`0B<}b$1s-r~6I10uK5Y
zy;B4Aww$b+SG=Fb+3$qV>hcTsr+CC<M)WfE#(G4f>OhM*l|JIJ7R*dOAF?AqTp8{D
zhxY7VoqE}J`3ENZp7fQKBlAzO1;`AOx4@I@;3K%u2v}FnA-`A3g6{Fn_U)Yqv)q6?
z!|NmoL!H+1O&$Lj<CsXz!yTnE@LF;&oI>}<wfS`cix11Z?VDLN>7=URmfn{)woR7Z
zqbJTN62r9h0fL^|FS|VLGnE<qKdh5{!%039U@nmT#%bZPoX!_8NBY&;U&%%q%Ko*p
zj-Q34W)^+|dFkp37SH9xMGHn-zhXDcX*ks<BzJM8eXn|!Gdmeirv@K79+LVC5&AR`
z-Ppzs7JjA5%LDi+tT)IFIu8*CvufL{Uc0xx!KQAH-Q`;R0vx0F4He0ywcl`YzI8DC
zOPiKPG`7ler)FcFf=<_R>hM&)o;p9ADvx^DMT&N@@_V<-=o9^>&Q`-G3`<a#L-xy{
z!_1$8qU;9gRqZ4SNyj6R8&-eW9sh#B_>A00e5Y1kAAzFekot0!CO0bU75DF}6S5NL
znh`s=b|TyA`@}6(bG11U0gZZ@EsDq<(E-?`^Ucck_1~&@1(<Q_31_s}s9S3HfjU5%
z4z*?>$9H;8$1`C(bo{n~I$DQ^K&Q9uYFdkVja;GVM5xnKVm-x4G)vIe>L&6~U%dYc
z)@!K0myU}KBrsgOjAc`gqrF%G7oX9dPr+#+f!kWOOMwD89#83Yu}b!Yd6K2rT%XRY
zj9)QH&MDfCoo@k;^!wm6%Uq@mg6wV!Kj$jDo!4;cb&j0EheURH`=hMflc5g)gDOAg
zA|spj%_vT!fDdX<=3wOv(v~u3z#W{Je^!y}Rv6Rwx;~lo{<SB6O@y@WcYIFs#YheV
zH*hcJ(v(~kU!@?+xKy>Bko=RGC^kT8Y3q@lDn!HMk+J>06(d$xE>Ez9uO)FIQ2A)L
zw>W%<VrH3$-a5PlL%JrvG5-el1MQ(-2JGY(H765`ZA7h~fJDU<e14jcXr)Lho*8mw
z1mbAAG=%Kz=Csvr-2WK<e|DJZ8rsj6KbPzWkNNQc$ie-s_t!yta3O%Tcq&JcH44C`
zN6t0%AYf3>_voKt;q@8l%#nPzCzHs0rOaMZbIH9L`;Z7P_jplmwX!}9Ps8_WRV^NV
z`saxLHPCKv!C#PHT%Q<iANJDuHgnL(VF*rLBJ_n1!7`uM$y*)XEzQ!Kf&1Y%&lviI
z)qo}!iKWD@F2C=dvWQ--pU<pHgdRD457iL8uKv>lz3olDx)t94!x@O{v!d-g`EEL`
z5PbO8oH089>;H@WL=*k=*uZsy8r_H#EdcQYZ4YL+7#yx#h9xDUc3HfwXfcR)VJ6_5
z&HLfDp19=ivyWabz)#{T(HoxF#O3MSv>Bfp1YFO)P9(V5v#0v5`1F|F^-tgUPj7vn
ze(TAc3oy7n1LW~|MifIK0W+|BB4O~*#0_27rq!Qm6${UqNf_Of3b&y2b>b{oz^ib}
z&vrA`FM;M;s71x6_YIdSAX1lZtMu!vqI-dPFh3wHoJ;*sbpL*LNcquI{yl!(4KqJ@
z#n&6n11qw~+?TxFm;BmA`aS-afP||;xZiyUP-0M-OKAN<<vva8_9Y0i73mSjdx$CY
zi%4V}V98kW<l!Af;2VJXC19m<vLC!|qQW60A}hqREQIrb8Ijuv*{q=oMSni{I3rFv
zbZO9_!1>gLr7OMq`!j1$geTXxe3E+xBN`SZnO+X_2Asj^7FPl%HtpWO$>`5w!|F1&
zq7I``b0j7cOkOnC604bHowZtY(obc+lNtOv@DAalM@F~vMgfD<zWaNxoVHM=%fnZd
z{U?EModxVgRNsUYlsF-8&Tm!%*?#%ne7Q#tG5JTkF2WGDTt*Ubb<CUeD^(0E4Y_Ty
z^w(9N_sY^e4BM-!xAi?9)$fhjn-h2)6wx>GRrfWIJeM&Eynm|@f0@nnjRZ|xj$V6_
z0(@+vuhYW~MC+1(N&EC*5Y^)>9_kl;+}5ZsqUvEYa<%()Vjfdffp$xr%a{d@SnIO6
z7PLj7^YhCG()6cM-@mI>!;V&CHV(u(D7wtn_uKy{?I3FR?e042W75a17pQDkyu5bB
z1`S4*qL+vp1>@))_$#F4?UM3I6+Vt-h`G>ZW1N<M?B7oEsUZi9<<>e4tqRprBSKiS
zmySa&V_xHI2C(VQ2=F~OLKsU|bYH$!t1++9(|$=&SKIEHX&7cJVD?E#_p~xa(DGvK
z-QhQ5{|TIQLc$ujZ1=%aBpiW`dNy@>?+ivG;CZjL>5N&byVZ1S&G09cgYT0bl9DQ>
zNNmtlroI5{T4%DR3XBfv6<q5ANkZD`Hi%(vTfF2gqX+afUN|gbKN3kbZ0Bqicw6;N
zyE#Yx%nd{jTzPPv`{p{N7N=_<(iodux$%2R&bj#;DRK@)8>W3e<VAXRU9x3m>(%+0
z)$O660V~;#bIoN;0k1C3wcYNk=*l2`yd83uhZ&e}6}u54v?801sq2J;VYEg^N%$d%
zhqf2z^`z||zhgG{q90#QKYD)8BCIRIShlY)C-&ec3X4iZdpHgE_bcQ(U?FvY3M}0t
z`EFQoKjv5&PIEc84H&Fm=F)r7nor!IWufvNLlOeA5(cAf!xF(LS@s-+{G8+W-OO#2
z6n-f8t2Q2aA*`<}al382o2RA|%uff!&I=Q5Ce^mCiBD!!!Jl`^zzhbw*ri&7^8rfx
zedF3!%>|a!M$YYs7{mq(SWO)TDeuqQUciPNp;fcd>oWmpUQ9a#A?cJpG6?MzHvS5e
zjN{+e{6Shs{5UQ~960;sB|nSn_6yJy+Cw;T7q8I~O1J;G6xA6&7VbUZ`ush<GH=aF
zFAZncuI}*DMn>@2ybPE3w6L~C=nn>s80dVtOh4+!cwa1Ir|jm5SQH_zBg%PMik#4b
zwxdF2jU2fT;z6pex<PL^5YS|Z?uVXttF#NlZx-UpX+Y^I!Rfs`jA9+Jhm&#Yr_^k7
zrT+Pj@IrM{Z$9%QKd!iW7H|*^tn2en;hK#+62q0p!txEon_}aGoCNUIZJ&h#kHk>d
zL(&bYlYr)u(hZY8$ZYY+6utm8H@w!plPl|m#%9WZ@*FfdsgiHOng?Hp_|0Y8{9jcl
z6?y-$`)H5YHNPLuV0m)bIZj@U{D$2vZ&ZJjxGqnJZ<D$0&1g)|8V1rrc1+T{ES2qD
zF(v!dwKtahcU>FODp=(CeM0Ao(1#OSu6di`(yCn*_Q@pMM!H%80!-f+7JNVZaF11%
z2=v^W<_}XiP!U!t_EUEE=0o$*@5;n6erHNr2Af9c*>9M{_4r{_?v_TxMTPW2t1@gr
zdL_53=vb_ke7FNOX;~nGEHSa&EqDibu=cAc_uFZo5F8Kf=Pp`ti&O1v0B0n;Bb=S5
zCXC?yBa%0m*IzOd7@|THE0gyU#||MlPh-!QcCN7#<20_JT6ly5&j8dbX$oJQ(;3ok
zV0y@?Q2YJQ1!brgG1@F$r(jj&;;|hyT4d-_l1a}f?%#T+j{=Nsc^H|Ce|LD5uj{`p
zW2BVwe`($!bJw>-N`3qEX=(v2Yw?0S=3#*3d?F3!Vx{$g?3L1a9bjDTw6EKDm4)_g
zlDvnfd^V<`m$?jCU@L^$v9+Vxi$pPL0?;sXqSOC2Wb^<aokNg4nr&6ueQkRutn~7h
zeKjdm(}CDY_J)&v5Xj-X%~_sV_VMT0p?|(N=uE@gM;1nuN%V$O+;^s-&xdH19Gw8Z
zd6bLDe-vNbRI&sqjV{OikH~x#UWxZMKCxawje%lB+Bi;x^T9;$;?05kClfx7r0$K5
zJ7mJ7dwcfMf;s{0y&0ees@WvLhmqV=SilLeA5G5zcO=vYuTBR7j&MvZhBWZF+tM)L
z?pgFRWC`}J62eX}3WYt!JmLbo`ouXQz-!8wNw<IRV>FP7CD=*kXkqCbw}7lnuD`qA
z*YS3$my+z;0?)gb;lR($sOX0R7PCYO<BDy1_>8$<^mHY51BM!}7GK9-JK0)p>BX|o
z5`+=gX|Ep7)mws*VZTYQ!hP|-P+JShcbN$>33e4glurO+pmdl;0@wzVP#APZ<M)+~
zO_&a%{>(lM#la-}M{3-S_+Om_o}Bt8`q3-;OM&#<UsVrOjvocN;)JqNN&XofTqkl+
z?Wl2g;am=M-}9&GjJs2bk&>n2(hO6!5*i}0)KO5KW|>TnO(pqy@~HCKD0OvZgtXXJ
zM2j41<ZYg=q;8z^ffw-0wf!i!A3pcbSEYp?5%McVu=YE%@c7~qy*;;HMywtPlRhFN
z>=04cjEg$IkB`yW@A$vgHg9ELHxN#H*}HuiOzVz117nYoE0>DnbW+6=iY0FEsr^|G
zw!=Q0JvzvzuYR?E-n6zc!3m{WY7#f)jT=Xezdts9T&jfjsMVUr9k!=lZ=*S<g1dZt
z)>0T}34hba@$^-M<tqewNnX_I9HnQhm-I>0E?!{!|1UKC%pfiYP>xaz?a_ukCTB2C
zPeHhJ>mogKDxdGrvzYht{+C7mrzVhsd%#;_#4kaTRU-YQ_cvZ7IfXdH>`0$Q{FJR!
zb5I8UpVt&{`Z+8!2e4*iDOyb?6$E%4O|+^}&m~hQ3&R@Na~07S`Cm@8aJb&nEfN8;
zWzKjb(s7ag!6B+8A6e@>Qt|Qg4^)ua#@M4HGXEH0`~G0I;{B@@9(vQL7eAH#mLF)n
zVD10&#qt}0!tY7EMg07KW$h<E!QTQLj_}z3b?%cNe0K2`ope|WWai|DXk01k#|f?@
zD^$Hhvm~}3F#cwfLmJV+`*r-y5aiSZB=Vhp1COqBSXaFK0^&3c?dDVyevgw`z7Bd$
z8QWY)o7snKmG0%D;6iVhnExZt(>e2R<%7h9S&>I_g7r)G4Em7zK47yy^*%jJ!eA#!
z*53KrBag@)E5S+>u7m7aqU2@{!c)k<53QPv@*{<M5>=(Kdief*{LJy&8@Vg>y2fnd
z{BH?a>!YFeym)7^qFe<d2J`ioe-G^v=NSR{uYfB;Wb8h79y;Ke!I?Qi-TEp$9bky4
zEgM0uxcXb?dScGLQ!6F1T&fnir3aAOM}c-#-dW<Tjg#MP;g;@<auG4tZc)<hV$QZ*
zuCvJq3BzWQ+OB4UFB9Xzv%`H``lxM%@=^SD?8dNrN-B}R!J7>BZ}3YEbQ!;nRQ9Mf
z2qi*^x}CT|66`ny?Arf*D^O?--lf3;|7P?ZOxPuK?PU@Vb#z<m`3y9-%w>0=M_r6l
zi)WsUXZG+^+kFu(drGtO=+yXLQK#w?e!eArFJb7i8ML#O$jqDM1NJj^%J;>6$={`I
zAd%Ym<^xjliU09X_={hy{UXRFZI|=_S_OMk$hq5&yKsrn#}?RKx4iru6S8gJfDN5>
z<iJgx?YkOaGA`hYRHNwOps=t+_@hT0R&B<9TUU*VQdauTlhQ^luxcggglLl3ME0v7
z_@MYO;B4`5%M&QkS#$C`V0ljta}jCvlK(Nh&!z2tFHzIcnw!=zgcw_Cj|i8D=+^x0
zMhIf>p3xqeAhi+$836dtN8EImUAywEI0#e=xO^+HB$j8}8{o*~2Nzl!FW&rAcAW`M
zd?cVEtp(h^FmDP@QjTx*sBFfk9Tan;j}`V@h>syHai`}1q}7@#3XIvRP5(}KmrXZ_
zNJ|EkGfMpO7so8?_TGIuH<*B~TS{J8@Omtg;q1C0Oq$O+VoZL6Z4rKlVSuZeR$rO7
ztro(+KeONv%0&VNe3tpLQ#;xHJocs0E^SFSiq{*Nbgj|wf8v2<re+=yw@XI?<lLRv
zt}k=!n-;xSp^8rH0!`<mfC;8V$z|ci%SvI6H8zM46O)H%z>@%-<FR{2JiF&mPQF@u
zpwU<Q-Ss-R%g<{yi5EO01${^{1(;Exw!H*KHr!)`N1kFPL>*=jqP5%0Yhkn2V#j^y
za$JP#03rWdf_>Z^-6ddlL62Fxm}z^ROVpuhd!`Dt@hTg(3+ll)ukWB_t3Sk+k_^CP
zHXsw$kA0@+S0B4il{>SSbi-Y;s^mGiN{5c*Vrxc;<`#{l(fNHpL7%#)$>tH$28<{=
z11E?PQbL<-i-AVJuQ)LX2d;}kG*pUmH7IlI5RD~FDrBzL3NnaJfEF{?bPc&TbBH?V
z5?BW~=-6PGpq2B!FyF@zJu^bsT6yd7HIo)wu8ds)w_<a?cY3T!M1pbqbDlT;>YbV$
z2M#!@oY3-oF;X&L%C9Rcl8J0oQ)&US^m?)za6@6hM#H~6f$z&8?7zYQ7`qIH`V5|=
z&*eJ&M!51+WL3(of99+vM+>3g_V|q!HS5M+cqq+4HO;7KhA@9VCQZK(Lo-vTY=eVI
zjxI5?H8Tl`3$V&ia+jcHD;)X2L8>y&w!E226rq7>OP({w$${CW&oV|PcOV3j->7o~
zRauESA;1q739h9~X6Q0eEeL~T=um7NQ{yRJmj)=KawSR>QSdz9@8ahC%?n*ed;8Gl
z5US@&_BwV~>oV(V@`ER>xpPwUlO`51$onb*sBeA!$;Y>;>eZ*|Qd-@}z;=7K#o+g&
z=^!O8Y>R=S(vS={mAxw7$Wi;<@y{1WQeVAzr?;-B`qd|-tl=jltpLifRe+0(@O{W^
zG~zif1`X#+6^L@@Y24+po+sA0W0a=SFCSF22o+n~<h#K%iuVsL&M1XTvQ_k`Kyd58
zHjukzfU7Zqob6$d1TquW+CVke9JKFrw)37X%-fh)t^DNMj^?R*!16ftme$Vu&znRa
z@Z@s#3;y$V|J3O>9zr{r0QsDQ?ca+x94|2k<-V~Becp9?Fg^6|<`3E8E+bh8+GSV}
z72zM^;72gysIk@3mZDxoLvSzFs;{F~h!~KRbXhaz&|qBgF#MgOW>yMjI#O|dewGA9
z(@Xb0^qNqBwbLRDza<wJd8z+KLWocs=5I8Q&U2zcjx(9nY;HecyyYTd({33_&#cq&
z?z?Kw|L!AW1iAC_8Cp@FZY?qVauW`T5;16&d6lyJ8jp^P98%Omkhf)ukZVr>r6?yW
zb1OE~2^vku-NigyksM&A{;TrsDQWzzQwdU5TaFnIj#@@pXJtJNo@1xQV?g*TniBDI
zNH`D7@J=o9_fjL}Q;>*O*p#hPaAdHZ#{YyLa1U9yhEjo|MfAD9j$aBr(+L-mvaT^k
z2$u+8UrPVL(u{a2B)pvhG|?L?<Q}#q-7uxU`V^c9;9+dPjM!i72FG*Q(J%HkftW~8
z`?N5I1*~grAhic{7CB5i6<|~X4-%YtHAh+Aj3u=aaID)KH-~){jC2v<r;#zx$yQ^y
zWt#09!D<RZVWJfNq}hnyU>)UR=@v2CJYk-`u%)elxxO2B6?F#MEfbN?4F1uq;QP<$
zXgz3hC9Cc<y$s%H=RFojuj8sW#|Sx_X5$4E9YVg2KCRh8MGiiQ4vg5h;djTm3$w4u
z-}h&6Ct>(H@E|p`J7j3}98}O?^VkbUAjJ$gMb?c!egltGOnPT<-OT`mTa}m5v5KF=
z&-~sB9UjE*p>0Em-oLnSnFwVJATaD!zXL(st#^gL!4C?M437KoUrCojPeI1#1Z}dA
z{%8={>Yeo1{|S5#gYa&JR#&nqLz1%p!~uK(H5&){2#t`>YMLCO6T}F73l|~W!bJS8
zPsa9r;<rQL_>_V8f8zkYz<&Ou%&)K;08wkBPucJ1JHu%H@AD_hA3@Q-2b}NLKFOOt
z{qNUzls|$Z0Kvn2T3_F(v`>^c`V^h-B!xix^w^23Ptk;&_UW;+?$gdbsdWBCiK$Oo
zR-*GKN=$u<d{XKBi4sqrWZEZ6jD3=6pD6M5Y5l<EX_fZrQ4?>Uh)=XnkD7G!32|Zs
zhp0~nO&opNc;NEDK4qJ3L^HdcPoZ|F2vP=u2;9Ix5b8RljQ$Yf1@&X@)7mFReQ=-Z
zh8$tkxACdHTrO{x^DEfkwB3V*$Q`LbaL+i1@J1eJ81iKgegoEcNQS{pZGH}M(*0EX
zJlM5QOs~gm_=ML!J+x1CiI0%B@~QLT<8l_h(qe-_Z2$?;dVNBUT<}_i2qVbB^9E#e
z27bf%1d?G`t6hDbwIMz_e(h>LAMVyCrr%?F%%)G9=es?<|L%Q_9C|mkn;x9LtIfN^
z<*N1`lA-x_qytn(BThCo&`&rcFszr;7Gp`kJLMNyK+V#z2OysWML`*jPGm@e_Usad
z{uRW>`K$|Ff3RDhdOemnaqoPmNuf|rmFk|}e|KI(aJuy0`AZ-#P@UJhH>=N=kPNLC
z<rbhX&jU_*qJVU*R|2Cs`eMja4_U$F$+cX)wvR>%cl^u^PV%Log&_$}`v8}-%Q$@^
zF7^HMo%$5{B>et-r|hRskx%beSA&m!|HdD5z77U{8xmq-70RHLswJ#GS3#N*pK?$h
zgBIk(X%)ERhSAWappJthoNIx3j36ocwfQhspW17m?)7PBpZ4CT_cP!3YwgS0>~omi
zE+oX{ObaCE6)xZLsTuktO|wAIim#5}Hjo>X937lz8AyoXeAa<<_lf)AlcdJbC+%N+
zI$xc)+k;+vaQ=C9*-o%eM?$ewYO=(qk^%a6pHR6P`jnqa0EYHFF!DiF$jeUo8S5NF
zLZIbr0C93f^z8VTvCW_E`=luK2VUPng!C^ywf(QB-H$L|4!f(Xp8s(Oi9meHsBcOf
zSH{}3D4bn??^CJBYk|j>P8jdnE4|{)x1Mxe0Q+sm4J1f})0@@D{-6)>(Ve~P-T1>D
z`_yOH9_iEF1uH*%dQhM8)p3<Tj`{6<B8QnjTf+PB*n|1%10;ia#4yK4AO<?`jDqK+
zPvsoguZv>ffl&@dme6`TeRAzG)~GynL7iT!kOckJ%J==RaR%>d=C96oz5mqf^#m3K
z=I$GTcIgucT!a8y`ZVkVq0=4=0s8GWIrav9NCtsJz;S>?9{8-vVLpR1z(_h0SpZRy
z$j5nw!yKTzA_E{@Vju}nzuy^j`Vc3bZfE!V<0LjeeZqGh2rG5EzTi=_>+`$l2GoX6
zbbY}$cFqPjVHk$Ob-JiZk<}?`CvkvXVB-E~+lnR}LMbK*f|B?73a?+BpSo%??(fLj
z$WI=As-1K{8G@fcDL+Y`<beUBf2tnwQ-M5r_^EaZ`iU9+Q}tx{$^LPXrvv;{H97mc
zg*^EK{8Tj==T8(qfEwT@klR0n2juV*$lc$Cf8aR>KY@aNa#}xadtNknn4jVc;W<xA
zKTWL)`_<bPYzVswEg^o2^TcXDje8Ztk01A76O6*}&om>$$JfS>Q;?_r<R`E7Q?HwO
z-MrrXaiFDJ!^AWue6Vt_pQes9e0=W=fw#b{BYrv^Xgc=(Q`tXtWE6GG6Y4GM28|^W
zKS`e8Cy?f+I8P3KLR~LSp6pqk;3tsoCri>mJ<?B4|FBR+O6MoZ(>Uje;U}<!a=nZF
z#PAcUgrBCPpBR1uxqfPQy{vcG-=UKCchQJn=in!h?I$@fU{3VYFLme7+cXfy;d`C)
zof4}wkU#*bg+yYAdPY%7L>A;FD~r9@;)EqH9YAuF7<+^v0~->9KSFjqu&^LL8F#(V
zls{FCbEW1dHLu>deRad5CwEcfQ1m18fA`ZS*S?O6R(CZ6HTy~N)UE0#YPH8IZHxY&
zexe3YjIjZ5R)G>+v(;l88`>vAxm4q+yr0fpBJHvBM8+An3H;)x(h|5%F__I3?g~9O
zFu;#W_MbQ{TqjfEEQn01b0l^H7w_yRfIsont?VZ>zS&T@IYedr#F<DQ!Y^@I3fB_c
z9Xv*})p9-<5}3}H%hfANhI5f2hT4S6J7s<LvA%)$>%#Na_Ktq?ML_ZdKl4=CPiWjE
zD)$6Wm+zl|GtP5XT*_0*T!>kunk%#y-tUi4Zvw63HxB+XA&qB)#@_sfX&$X=WGJ7#
zv!lfaZ3_ZB`uNFSdl}xvPp9{%GtCS9bf=P^w)bN#>!%DLXIUAZTpq#|NAc7-pu{{p
zGH0(4dOT<x%)5y0-QTZ`plG-VY|P#ppecs^)^qL9i|k$Wr0$>jGf&o*rz(Eh@>JSS
z8Ds_Xb0#FUW=wJvFj-TUDl3U5=JFxlbR2*sjOCejUu=0Y(DBTAcKGtf$p}C1n;0-M
zYID}3nfKPVlh2^fPm(7&wbaD&JgNWClsq;6m7iqRkh5IyQi#@dS;C7APpV5;oU^qX
z%Z`a_F^E6OMbRDO98X8R)~xg3XfSI``W6O?45-T)Z_AUeY<Vh?zx9(8)L!#n`H6F$
z0_2H$*-NE{B5==m3Pcx)PP$2UDP$}ML14(ZZeM_T#uG)e<HqbMLVK&W;E5Qmp+#%t
zr$F%ZO`k9~JO!Hkq<}i_r)EVz4J+)QWby)_NI+7;S$K{o(QWtRs!IxB95V5vH>BAg
zqg(SK>O957Ihjzw6S+5c+9wAH50~}XKN;C#jb3<fDd>tFEFwGg+#UV&;v$~9&5C|<
z?x>H|+`i+K^wXB7HH#sae&WDc8Zj<9chjXTf>?IBDC-SRy3cl2uYWM@cTs2h`sK-e
zi}t3IQGeMok5><$uV%!ki=Pbm>WS~=3bD3%c9t;iCB|;=KY{OqdcWYe171ZxQOg;s
zI2Qd|KZQ)B@cly8G-EQZ=;V`;+K;Pi;nnyF$i4gh=_5o&olb881VlZ2*`?Tg+&>(d
z0JYgbVe>0XQmk<~w)>8~%Ss9F5}=Gf@U$;p@9tLd6HtOw2!%3!O1Td*aSQ+%=Yc4`
zC_IEvmJA>`eKO{*tGd(2RQy(|egA~z!U2$sv8foqP<?$4ctC9t&^b}e`_9hUtuPG2
zP<S~EcCo8L_x`Usi(|0?QQJh$z!WNi1b>bn@u$7r`x@A=!+z#hPg~N{89u=jZ1D*q
zMAN=r6>wE_R}{Q`KC)-0^ShEyDS|r7r?OxVpMLN1jJvKlNH8O8_K?y9P(k+#`Uu>8
z9;|wr#HUhH(0oO4ng$w7R#PTG!Mz`}o12DHeA-^_e5%cff8Z0S9{T%Fx?k{AJq7d0
z>QFkGH5*TS0ba5IPOCo`J{>p6^A5Q?pX!dxoNL%rJP9eDJOMT8ZcI;^Pg^{n?6`5_
z37mLx1k|XvQ9Wfr#m7%}+&J+hq<Bh+P~u7TR0ibHe6o0Ajh-Mp*9uH6A#lZ$E1x9k
zNdxkC1fS||AYhH22)}SKJfDuIQ9Ug_Ek0f26SAIkfw%ay_;iI&c7eAa;dSt7{g3s-
z-Wjbl3`1d<Bt%)uR$JD=1erIz5J6;Rc<B@P3ci58fNyV>QjcA0dol^6%l{ql&Z~WV
z$<nTk1J~!V_P*d9S3G8))(aNveX(7DS#a{$=&XUB7(Ma8XzCOBWcpMv_fLXHmA03$
z!Mu^I{gcp&d^!fI;PlD;eH+&b%nOHCB#=+t1E|FM<m3HP>hl%^&dN~N_51td<8Jr$
z_4M@oe1HG>_I5bjy}W!pJnZ-8c>*^#n@yGHMX_8iwq{K%kL#e7ukBOMCm-*ZI^3>6
z*wm}~$EKaN$?9}yboyv?dOhk?wK`d!nhn*EPl>-i?ch`J)HT&<!l%6Tsr>De==fxW
z^4F(3_!K%dt#(!@wokO)$pN31-18?VpTuDW0;kQ>?>^bxQ-AxE53YB#At#^8C9hAJ
zx^nt{cXbWsjhtY(PbIHUDqEDAL!N3dbHo^3pGYXTKIwE_R2v4>2F!-al3NPH(qd1}
z+Y|G$q@!|m<j1)Fe%w%w_K8NTBwOQ3%0=<->n(J_g?t+C6QS*YfQup>;_pNA?y^Td
zQJ~m;GH7t|2s2ub-Z>>(=seTH>{D1ja0EV)^p|L57{33h+ov=slf`(SLjCav@Id!o
z$#MF9vVk%_F#;9PGe<B!hZZ)jPb8GV`ZOM>P;ajWK9E8%14Zi{eZgydlH-92_5Ny%
zmthBr&NFqlPY4tk`ryy=X)6#Y%;^~1eX>Gj0)YaIKyiFBWI`fP7)GGDJ|R$G=z~Ak
zCj<)9A7`Ib>k|S6dhqv8;?yUJKw)~r-#>L1yiy=gU;xSOQ_m+kHt18NBV2t_tUe)6
zal+p}(SoO}PjY;arzl6rbA6fsd5RIq@`)Bax(RABI>=LmV{r8;wS`K=0JC_&B=Cnc
z<K&a3KFO&;o}!!c`eXwoCPAKJLz;2hkJAo{ksahIvT;1mq&_KDpOB}xpgcMENu?ZN
zk*BD}txv+ePsmeDIA3ous20ZlDWq3^W4w>+^%goH*M`6Qgdd#8$#;8ayW1cP1i{`)
zsPF&AMLAnr3rmUdN#V!LwV^R!J~G*oKorN*EuX&rbfBJ9E7$ubqfY>4?ehp@K)r?>
z8Bw!%y6qEyS^7MuCO&=r2|+!nY}J}i2l)h7td$mh;`oWW)bjOo3Hbyw2C+WzN%o23
zC+bqG`J|AA`tOsJ!`@Tmgi1}b!4pK%1XXJe4{}<MY7M<Dn%Q~VXZ7;)*JC9wxAi2i
zi#zfne7Dc<jp?@`xzA|b^}fBet<4h*r^oXrt8bMzZ&-<8gyi)cx6S=JB-(wy^72md
zE|(uVKQdn2FsFB8ACt|$(f-h9J}CyDP6V)weUg0&e1e-?^Xb4R*!9T)b%IY&kWcpM
z1fQTFp8}tt#;3q1sPf4Hg-=l9Q{WSn=9BDG2%sRJ0-s>tCkIsE6O{OLf={sTQ*=0f
zf)bzfs!XRtA$)?nK3xJTLg)k=J}I&i@&69W@+l69fP%t&lJE%%u2&I0L5WY~6VzX?
zq7Xho+4+{>txPC9-=g599gt7(80?eCC#dv^e1cM+L_R@@PlkMgJ)Z#Z37*N%p&5Mf
d0Zh>w{s5M9*7rwRDS7|^002ovPDHLkV1h09-ckSn

literal 84434
zcmaI7WmsE5*Dg$<6o=wgoECR?io3fONpQE|Qrz8&YjBsMZE<&ZcX$8NKJW8>=f}BD
z{$%gTEMK!`?{!bYloTXU5D5^WprBBsrNmUApr93@px#};e}IfQpK0AfzVMyJwVYM$
zfX;44jsU2yrgp{vQfV6_Gk^-f$kg3o0Kg9g1ruzcrsb?9FUM<QXTxapcMc=S#vVcq
z1;sA}vNtlZ0yvWz1I#RJ1<21@JIP5cOa;g_Ipmq;?L`6R7E&IL096kKH4_gj6CP7?
zAwg1p5HEy*4Zzum6l7y<>%<EZApggg7jpe~n2DV9-xOyn0rLMSr6sRKDr)BlAmw1>
zWH4c2XC~$1VPxTCX6Im`CuL=3;bLOuU}ENEU}oWE=H_K(A^o2ZIYgSHDUer1OyYlJ
zLGA>|&7Ga?d6}5p+}s%5*cj~`&6rqtczBqYS(#W_86YzloZM}ljX(^xP89#;AO>(U
zakQ{^wy?7${maqF*v`dSfE=Rfe@w8kmzV!v#<ouX<0y#Bm_SDMOe~DdOg1)u?fN&{
z$yo*P|EuwTo$aLNZVzBm0XW&YIGRA>0i^gZF~oQOZ$W=2LbTykbhLm3#mHLB&cwwA
zVCyU`CO{7Ph0)Z)l$VW}hr^hK6Tra1#SLI!XEQQpFy`UnVc-U^0fFq?KvParqyO^x
zztMAjm0)A#;S^(LmJnlM5$ERN=J+bYDj~ut!NtujAu91-T4`G+XCqq^z<*_1KxF?9
zt?2)gmRHmfVB~D)sAgwp{ogE5GPiTKb27KHClwX_7j;r<c_R}G+rNV}e*^U&yTt&G
z7OntO2}e5{(tn=jwfMhj!Op{J#Aa;5V9de6!oY6A!ogs~%4)*EZNzNC4m9R4VF3ck
z|A*f6|M1{{G8HlcvBdtr3h{EWaq<{JJjcKcG+||6Vc{}n;O5}qW&i?>S%EwNP9sh>
zR&pkYicEi#=l@8i{|-Si^zY^WE)B@d|1K_oEo4PGLY593<xoD<JMm3vv9D^N#p6sk
zFD&t=KHs@1_>ZK{2kj-`(C~IB9Hr}q3!PuV#=i{=-f+~85fQ;vdGko2^F6&twDRxY
z9eln{7xt)p+e>S2Keo9&y^Tkr+<3U;VQk{!gM2)%mdvu;dsS3aYFB`D{ziDgMtEP|
z(X$BlJuSuma#NSKDlx@3enIh+;%-pldJp|ht3u;JVSKGdX^QGGlQYWWQ=NO7G%k~T
z-X8E)uT-US@KbT=j!ePmCQ)6>p!!S;^HMLZD(*m=M9rcp>ky!kF@B~JZAKug|7TCB
z@9Rm4Zl}N@2eVV>O#NDxsQ>LJOliZf0B!p7rx)})U4}2_4y`LoJz@3Z2JHDrw$5M6
zCV%Da=uQ!!b}=J&O|wTn?rG(9%ROp&cT9BDtd78IKFa0M;vo}X-<5__m1)peu0=KW
zsHRHbD*Wi(M^Ma`tmi+_|Is<I_Q)==Mwm1uw+f6YdE;AT$xrH%`x)6t=#6V*Sm)k@
z*#o>}bz&INqo`991Rj?24g!i^N0R9b+AW*X7&OaGG>Y_p2msY9mGkbJUjdDJ>J><A
zHk|mK0_s24FkViichB_OZi$Wv)OUQ!Mm_mgAE=I+`lOaARc!0_N}3qceo|qaXyy^g
zVza_MwU<f_C4lVWLLU~b3F<`QB?eRe1tOK8a3Tgz{3zdQ(_OuI2CT!N=<z!rmsz3<
zB!N59pI4xnH<5*mS;Ht+r!o^5Y+pSDxJ~6~u;%zV+RTVrWu^9G$c1`PW69Uk@=t?I
z?@zHKUplAA@7_eTR^BpFJ8B6#1D4LlVLCqbA_G#S>M5T~inPI{PhOZjKzejf>LCr?
z!K7mnQ?|W<Iq=~exM<m~rlYL#lH~YEZOO;e(%4XW;WoE{SA1z~_@lYYUE$-=$Ft>r
zTGL$t<snt|t5Djk&5~JjUn{!(Z(Nh6#ADV5KyO=1k)IeuXhK>p%1I;*rZc{PA?z``
zjc<*a-}P}hHi%0QY>HV=sAk)>RDC_;7kn&bS_QFO;vyfz0g0hEG6lYs(XV{7$6aC-
zVjQ!VY70mi))Q(;WUXR8fyO)=Jt`+*y#1Ot&r#2v!=I*(c{B=tYc?{{E@`VZzS4QN
zV_M`7@B+zSeFjS31`W={2GtjyDp<)IXis?4?sUoTKrX9xNiQcQr%w!NC%g-HAo=FC
zcm{*bq)uM1sIqj!T6(b=@p+@=NOvVWpE86l2ahTHdPjVoiJl*`B~{-+Q@mS5Q`SF?
zlNdAz*Z`cgvgJ;)5<iD$ebuYt7C7T0TSDg~Mm|jqCvJdtCYxTqE91g#`9!$*Lb61*
z{)O_%hcB^@AMfry_x?osR=~FCMa}JJ=@cAI+*+3T-Dk}3tM}hY(`v-z+em0!ex&&U
zXG+@Wrc&{WI5AE_QkOWoraL^jbcD2%QX|5Wd&=#3<xDt|#~P;$Ke1iV<soF7|Djz3
z+0<5OghgIRIN8SF$_{{uH%G`LH7m`xX2KHb4ES@j`l4I}wt6(e+-xwzS`+qJMtUhr
zh8-~5CBk|^rItT8d$d`cGB;O-!lorzH@VVm_<&3GX^x*&ZcUfB!jj8qI|Y(#48wW@
zWqa)u$?uxeu9F^4Se(YvAq(UVRDRjh8Z*)urL_~Tk*xKqTJv1B*<h8=K5!|@o0>|{
zC@a>i=CYfmjcF!dPni~VK1f+Ch}Lw}e-g4J;WDe~l9;(<ZS0~>G)u|WS2j4w45Y}E
zd@?v%GMychj+%3(W>CZqFg>KJt)0z2lpBN82xj`mcslch1Gl8m)Nq_O)^uUkS~Vh5
z7VLULnq{q{C3TJWX_>ugp-vvP-3H}XmB%az58Z%Sbd9&|*mC&OfpVm}=z<F$=3(L~
zYkkVP7qy$z0Uw>J;qD;N9Nz@Vy{SAHeE6b4)K@OoK3%kybqGnAg$a`Kr-~MzGg>;8
zya#TAn@pg0B;A!K-k2os>c?EcK;e3+bq|LEMO-=?mv7wGDZ|{fA><wJ|JJlmsI$eT
zfx)3Sv8vcZ35|M`rfMuEl=hnlL~KJP-MSuy1l@_o?xp3aEM?!eXn;6s5EGG24E8By
zw<+0}^85GHu~~rhDmCSrGeH9AENa?QhV|Ma69JPo31olyiwx*g95lQ**9NAxjN8=?
zmOqMFz$W-KEBLm7Cb|>IN{_=r$?~0Rdc6gR(PU(0H}0M`?jG`Y&(Q8iXp~y0)#B3c
zv{I!Lj-7#5tlvLRgX10KDkBbQ=5kBl(se2q^>9N<^{-xB@0Yh0U+IWB%NTqJTIFvm
zt{I<HM%IE4Y&cc-Wc!MI7Qo0FKR7BsD#OP4e96?f26wp^pw&MeTt#rQDL!diIj~a1
z#~UTpYD#3LOUwSDc@q}QV5DMYWknrWbOS<rq^&g3J{_pL%TuTLc-B*)vGtXYIuez)
zgu0BBw%D=8SQJI23Aoiw#nUZ)D4o=Zcj^4Bf}6C!v~Z&Hv~7FaC!pbxuRZN@(6pzt
zvNfY#spV}4Nd779dM;PzZn0#{UVI~V$y!UVuJKH1MLkA5c8Dd<Esk_6Zs9FR5*{92
zU@xPnSQXF1k__e0Tyfr#NcJ^lj`)@DIkWCEpn}!vt^1LMlP{sDWiV-<CF*cOZbf2A
z(bIdn^X92%iI`4aJqDjaUi~sgLnuk0DD75>NC|(%l>{%i#3$0H>crKvFnG59Dea(%
zHeob!s%3Sv@a2>2FeSM&s_QZp&yqq0=+9Tye3TvK!lb06+-5M|4dHakl9b#`?*~2X
z3H7v>gqi*X7ycqwVWC*yVldWMRKw{pj}3{nltunv_a7rea=dj*n>+e5-gK?mjUCfx
zr8+*H4Di__OvE)?hQ1++&SH6M=ZnuAc^zOum6uUO7j&Da-<G&D3m@XO8S8LzO5Zn@
zhpU&dzEEk0)AG@cQ->D$+@F2a$_`-ZNx(O=%}1o6r%&uqKp|iU$BnMl$c0`vmCkyv
zb`$VwFOrK)JYj~0nA8n!m5Qj^0aO`0!nQ3M+mhPd;~l0xds36(?^F)Dr{RB*mPiBJ
zEE2i$P*$QQXwH^exOzCM2a67G16+fk>qK3Eck;U|Bm6mdJJ>en<jGxD^+qkC+J-Y1
zo=P)TYMC}Ur8}jk9P4=%KisGz(Wiamd{tR9)-kc=C0;euu@!Fz*-y1LWuCSS63j-$
z=~|CNmx)1bS%Fst-jFAv^&1ZW1=&==EI#8GgPz0#$Diats!|k5mo`P5Z7nHOf9JMR
zt=xDA8#+RYa^4~Rq^s~I13mR2G3fL_RUq&Sr|o>Gq0b|N`@_QqnPae#r68l{@#);I
zv&7mQ`a|XlDO2kd`a|_GvtU-2@&)%}J!(joQtW)Ut4-UNL0P)v6SBRV-Y9~6X;;j`
zGlUxxptxH7ddfsbx5-Z2Fvs=rz@c6wJ>E6NnW$sM=NZOQF>Pru`>-W-kP`W)d0!}4
zSlE84G2yOY6!|Ewkf4;l?=WzvIRyn`#G1Nh?8q{IN*%U((OK9rqOQxa>#nh4Q&P-8
z{6)J7$;rVkN3$DA7ns#fbBqOr#SVqcatyyMsI|yp!)KYKm<*0lHSf=`T7g=!DOIu*
z0oDqd-EK6z4uGlSGHh2J^x|+%4Ek#*vs_;dvh<A7(As{JQfL`$Rre=H0LfgN_!T0-
zpJU{o=9_)T@r*i1YAW5BCBHy}Basc#Bx3_|_wP!dHxnQ2TZYIL$5pNAp{zCTZA#8T
z?dVYqIv!+(AFGRh9nvM7v7%kvb@=fevMXUIl$4ZQmaOi3<@dDYouCkK#?De5_7{+b
zVK5I<D;BHbi!5a~3DKMGdlh02?MZ5AXmBf@_$1-ukSiAx+l^%jTz=tTV@T$?9``QP
zNd=BPD#L?s%-1^fjWra4jZhIJa-k#3c_+G+`Tb2LD0?tuCc5KSTXyV|BB*f@s2Z#{
z1SQbr6)yCZ+FHdsTTGrfO*sbRI}J@gN;Q6O!Z%7(8^Y8dE=6tM?@4;3P!PIaO#NuN
z4F6N$Go~P!N3EF`>fCaWOn{H1@l{SjtV*YY*2j{kW(I}%_Cj#eArd>Ib`gAL{VMmk
zE$OEN6pHs)CkZ>{F)M}cAde&5)O~}`O>3Kn!67=R@oPKomt6s0Nyh#VTSj_Fx~Y2|
zFEuHKqj|%Bpa~E&VM{lXEd60X=@<Fj3pOf^Qe{93qi$fFIlJzoke%C&;*k+ml#hE#
z4i7%4o?l0KEc|7?*5Ms^K1r--fYvEQUJ2JZx9oP)qPj%S$S5Jv9(k@*vw+7q{1vBd
z^Hqi9UInur$mWs)%A-LqQvsCA*3iq<1SYt0D-o?yNN-}Ev}t1k%qzh;QU_KdedkL=
z8y!?x<}#+-qvoB5(e%WXa1=*k<ZQ>+sFU@boTNoqPy}>|v>8&zrivi6`v<y<i1<RU
zpC0Io^jhH~G11A;v`QK|EOANXo(tUb%T*ywfaadc&s@f4F`k8fDpRWY%XERNM<ZL}
z@;+D!LN#)2oRuOLWuCD~x$x8!MrNk^s%Gpj4W<Y<uPkVLC_jaDimz|J)pss#*sQdu
zKjJmmEX^tjOK;$x6C(0bY={x<2e@BVJMG!B+pS^|&e{nlo41Yp;QB6fZPISrC2_pg
zN%Hb^$JN$&$z3h(wR=bKe0rxjbc|Q@D5m<<Xfs~uiMD;R5&;-W844k+2<1Hl=Z{zD
za(UfvgdmWZ?2jW+2>M?1AWuEbB5BVdAtBOwx&4u*B5!rz#G%*o%j9znj8Rw73D6dL
zCKckcn}B|aJ_;H6;Z!a5%vicNStP#{`a->8QJkeya+U85he8}!8?SOaG}7a!>}%Pf
zp6j$Xs@3XRG2E$HWw7B?YLWO5fK9K+ff6%E6{I&+B;V6uM<Hq3XuA|FkY`w{@%vE~
zA3bT<=qE_)=8yE6Hq~amtVU6FXbyN&IT#(W8v*$I5JR4^$POKCUeva4hoQ9g@@4AY
zx`b6gJ5!}`L={p5?RI9s*<QsqD4gC*Q)L>79?9(d?N-U;8HX;>UST$FXD<Ym7`oi;
z2Hdghs$=?mPc}W_*!RF)x$FHEqzat|6=0`DFkMtI^9;ZFTQY+#x#ly}_V%_K*yIL#
zTI<v_O(c5du?>*_*(%w*x$l-wqwO(Qv=4RP^I92?`5Tl1j*v!imJe^V&@J~xmn!+x
zVN4-06EXfnqfS!=BQaHa>}_x$t7;uJEsX@Q-9v@DLi-_Rc~(4>Qh<^(n&))BIwv<;
ze{DjQs_IpqN&UXwdWCfpP+yg<kv*rbY53Xj@b1tRLmJo6i-p(qG*s8FRo%z;uKg}Y
zGuwdsre1aDu0E<GpZyKTJ=Q!A_oZSbhIAt_KG_oF<1Ba6ym(o<B*pD=Jdgfag#U)G
zBO8l%(L?D<Ev9fq!X5lzF*Wr><Q5$V9e0`U)zCK=E}MT8CwF~UId&Ay_{BLPb$PYB
zi}?P=-CRQRnfNN9K<!9D+M%RgumK}4?D>aGxNN~u#N1X;sXM>22zF1pJi$rRX!$xX
zUZ8hYq=tI52B=+02{wc_2V95tILN@F!}8G?*nd|q^oqEE{R7dxz4h5{8yYDYI>itV
z#|TiOP~ZQ9sMaO?$GC23+owU4G@V*x|9IJrA_omn0kcZTVPwvkLC_{<gKj|#b@nKM
z*o}w=RW0|(>w{SapDBDaju1vE-h@uT^d$j;qKsmf*LC1eix&x+9G%b~x^^g+PcY~Y
zU;rzI`5<$y{s+pHI#0ClASso*VitfSZBZ_t%|oPEDwcWUzA=}OsP)}krH#u4_hAET
zi{!#u32?G#1YHXEZlgD2#8Ilyv#C<6&Qg8*h3HUBfqO#?xz#jvxzTA)fuHTvV)^dZ
zi}|XaL+#6H>lw58IBDUOQ!C5s&PBEPW4J5P;#RLR*TC6n>)vXpcd;+0K7147{z9D@
zHxt1q)hOSc9*u&6;En0B!_vv-@9NK-m1}+%TYXzQ2wM)17R5tetKMd`i;XrRJ}*xQ
zMp~2=`<D1ISZFrg(L{XPoQpAH188gBF&%xx6fso-Lg}CCd`?=b&7Yk%$mw<JrNRjg
zy~23v_Hra+hzISwEW?q#6Vl_ui?fKSV&rk6Njje1Rp>UwTXej=dj5Qi80+-&hk+Y>
zRJxT%*^};5#^<z7<^Qm=RIkmp?9+F`YS>`QgbuATB3dz3nn$D)b2}~yEyL$k+#nd|
zH;KCTQ&c0{@!fVbUJKn7b>BGZTAW`kPo&vpoS#a_*)n-1ue0O_D3|?Fs!PgjDs?cs
z&8*(25@bUl3`e{y%*uJx@Xq>kqUWW7f>|S>mWM!nQr<C$dh&VZG7+3DM!)A^L3Tkm
zi`+0(37(Z15;}IU5>t&dNxm=iO`rjsu&sq9ww$nxB^Z>N+${4r0lrgg6mS-al{;T&
z(&i9wj{eCK@Z2%;Cir2yLdq6>N11q?i#-?=$5a$>cSYZ-BVO)ORxdRA>|?$<)2U8U
zd`rYD_1kfe$i%*8znriB$gf-?3b|^<9XErD?=Ix+sv3)K51D{^E!%rnTP}+mfrk(h
zWiFd}YCd-is;@|Ym@xFJ<kRVUepMUc4u#Z7lDMmdV+^*sjTx|_?0j`s7QB93|BlA&
zyWvG62)=?um~YYT*l#H?Rmn^~4vI`xM+wZq>*`_Yu7ymXk`Kq>p<FDzr}$)A0Qfe(
z{#ym~{Q6{9i>#K+(zNCekWG=^5X%cU=p%B=;D_66FF7;b>WwmY5u<xhU?R9o$SmUa
zDWwyijiU`sjK<%1NphC)Ugp?U@M76QdcGPHQmLL`!DTX_pK2F;Xt00$y@#6o5C<x7
zdo17mdH_~v4Jr>-w;=S0e9pS=zVu`j<?k#GJ<Hg=x(nmXO3tF)f}OPH%@dm@8d?8B
z5bj!elT`ajtZe)%Q6rm)%cSm)KT9q?^5~)px#Ip^RpXeN3prCgCTSpAox0lPOEKRm
z*k;pem4z&FKyH^Xz6?Q7x9VxYB~0KeqqVyacl$73Yz&V-QX+eh)zDPPt48M%$vlZN
zr@EZ^SP%g;hhMH}uvE`$4{}ejpAYr*ENOe}WXHZ_*eB0{g$!FM4|4U)W{PrQXD@wy
z3)pBuLxM)D^cA&7<q1q%&^{Ff#q%`4)L}A3w-v+&c^bJhPo`>z4PQeWX?<*ZWEK_0
zKfph5WQ7~~Q+Q_{OHGCpd-%NoF_CDg$z&v*1J8bNV1K5VtI>7^=P;6f^0J^qs@psh
zc;NEw%_k8jWw1@?$vRz56guE?Qm4Ze{&)TH8KBliK3foF&F87&_+x`89lpiEWhyrB
zUj4K^{{CI?sNqUg6ZKhuJ5pj{s2Dz&zWycKSEIkar$_u!{3cIlqCdV;fXPD|7lwcq
zqYu|t{d`JgDkvmT<mM|*q0@(~@AZjlWQa}pw<d{#0vrtVz5AJhW$tR{Ru#re<33-G
z65?PXOI9+O$^#!|B@MqrwL)<s`1Crt7;G$U#+<_pn>AiD6%?8`AAP+*;gfIPNHp@X
zNg@n@J)%-+u&32!w|F*;`l22Il9^t%W7jKo0Ty-&IC``*l>(?6sL^yiUHv`pcf#xL
zclCC}GaLk6uMcRZz?s2angIdBuGE&?%F?RUa_$pRUm8zFc?}ThIbKe`(L@X6A=yg1
zx9PD51!CF|j^A=0*YBx+^eXj@d*zoFZm?MT(8bvCe(3F$Fx%%bZFv{D#JK`bnUAYo
znCwC<&*J3~rQbH5A@fMh7aaNm6Zfj{Ri3BbLfsr3odqA(?&%aq<wHM!AzpXB@4ZG0
z;8r9eW=EtUGo+WZ?DPC(w}=}Dd04OJ74RAe!{^imeW^lc4-^wMW=EEoO(*evRPGRZ
zyf*>u{xowqt{MEOS<INZNu1Ih_Q?{%T{$a}OWQ|kfBd%1+nt(`;cJ8j1B=(ay`J*~
z^Rj0)EB#C!K$hosU?9N@(EIhl+2ZBrxBXaq1IvZ!+}}T)_68w4d2;rkaP}TkDv6*F
z0YSl;q#hS*237n1nOF`lf{^Im?%-AP^B^5<LH-P*n*Ur$f{I;`NJgA+9xE6O=zO_d
z>e_Q+4x`G!@DD!SL;%BIV4FO!yJ6*uYf$|#AEIP*5+|7gOJlNIr86w?A&hC{p<rWS
z<$-gUi@tur34_lUCCfXWP@qm1|NM9!<3IR??3=y5dGnx(S|mf)Xi&Z#61!pfaMn4O
zRpbfPZNgfKsLmqI;u4=8;#`)kxgPkQ?WAlD<@9(}g<D%7Xi{EGzH{v0cbR-u_xXvy
zd56BkY#Z-9QXl6DLy)SpL(RRiQ@2|8p?fJ3QG#E~ZOc)2!ouEsn|mGmmWLeoLdnsK
zGIQ7NrZo5ly<%t2JW7ghYLuj>v>26!vpk!|?9FCi;P>jXcWwF4r`xa}6BOAVP2rC5
z-}zqrB<aeq%vp;1HK{S*yrte|;n)Gb#p%@PiEw2PSGxn+)r+Vyb{u35%6dBVFW+xZ
zS0cXTQII=`9Ax1=y&vVhDnGwtPn4wBZ^0T$VUSc%C>Vm#2KS?duFiPVj6p&r#WIyI
zIc9J0KCCDsLw>%&vuU|fr^Q@L{zvq4Ds_b`;R7tu+8OBizKj0(X3lVfHP#R-6VqFA
zGvbm$DzIcv5|GPUx|hFOlDtB#Bluu3!BdPtHVsy$;7OF&$@Ll=APSAo!$@m#^(_@8
zxGcCTGARZW3|xX=buCv^A84-O3iCH)2fM%u2|+E;;^g2CMUT#6b7&co#vEQ!<rXF`
z#anT06hg8(BT@V>#d7eSFUGC2pJ~8XimbYDKoPW0!`iMJCdrA3*0^xC=ysq{b8vCM
zOjz*zF((hCH~3!3)inW3<WC#PkCHBiBXuk(BK2axL_L_H9fv!XK=uB+=7+t>Jw@Dl
zb!OH#o8)$TRX;|3kDIeUc_Qv6lM(NueMa~QDk;jM3F@So;;AjUP|_yc4n0g3A5}vR
zr}06HZ8QQe^Re!_3}^zsl^@Iz!L-|Ibnz`XtCg9prs*CxhXYO&L)o-G)T62i$H$0Y
zc}r}~4H2`2E!4;?rj4#VN8G7$bq$>foX_M_nZG_d#*)q<Qn$QXLTR6@)pW33xw&|0
z&TP%rcizORrvgbWautMh9{o-cbiDJT8+tQu?QQ^3wW*fNs9#e!a|l1M>+Fgg^_S_V
zZ>4s47gDn!GUTiX<qYsGvJxz|^$D_Q%#uA=#i@Gc?U^@bsuxD}S>e|DP9cPd1pqWS
zlz@N$)4|KYNfk8*Pjd?#j|nN&m*K}e!x~BM$b`_(6We!>+k2-1p8X$%rmHNDg!z`%
zq^QMphgfu;I4BxRrIf;{Cf0TEv~wbXKyjkVDQ-*LPyLb(+BE6xwv>DT3@#Q$N;QkG
z3A=5j1yr)pt2DYlq8Vvj)NGa-DyrMUU2iijzfMVfq*89+Fhfe`v<p5iT0oCzD1E}$
zzMf`ZYO+ajNy$nh8<N?{b@RxxTrCaK%MxcY&iq33d<{tI`f=UWXoY)-G*L(#QL>=)
zq)sv4`83M6Y4Hqa$~DBPcj|AHnr@!tECx@<rrehgJH}os+@ljO&Qw54XpzyKKwqq?
zKZHt#zLSy{KDdt6H!Fch)g^kDC9q?Mds8P75c*gI8Y&a}VUyC&$|U_!DU1R6lOlJC
zVn6%TC?NqI2b(VagaY5hbw{i!VOWv*AL)@40%sp1GDpsxex*x^E=ug+9qNuGsXDVW
zVvEY<b|+0x+&4C>-ig;9@t*#bY+Q7HCh7`Vk(H&2`C1n%?q|O$j0=ItXvv+=bZG|M
z2e4m!1?#gZvNb9u<G&gol(UWX?vChIJMPfpTVxoeE{fN4(9j7YV@qh&$mExMN|Kk=
zPwO*Qi?FTLlMkJ-;Ar}^yU?;v%jK1dx#Xi8hdt-0{ocN$$&H?FI8pm5qOfJ)Tf$0<
zrL$EjKBU32BB>UYTj_zpVC}!kTGqgOh>OZe7fXwov^;)^<#d}W<ap9@Zrw5-3+>Vk
zcX=hgr$=e#ANSo<o$sksgeZD}=D~9DIp8_4_@cPDK&A|xCt<1@kO!b76BTt`e9Svh
zD^40<BgV%+P>C$T`DuRBlZ|$=XMR#>&@M+aokFiqfSwQpiC5I2qfwJ&li_@3H1!|a
zN5N?7&UZC;kc=Be=d!~fto20<L;WF8Pg;Fhc<rsyB{8m~4}2Lux8|)V9Ox&9G^Gz1
z<UNqo$Y-hA$7RL|u)XIQ7h^Lu_=7Sb2nBgii6BcETSsJfLZb<Hs;}nleHbEX#Z6A_
z6JgTNTK6xqYU$ErYgBihY&P*(dhYUk!6mwCuDPydBhR(+7s((QQv|TqtBj6K?wxdm
zYoz&XvH1fPZPxV?osl~$apM#cwF^;0H8Z@Lo?yYB+wp1HmybUhVg5=23jJWHx#_wl
zGRzY@(xTJN8YHf&J$QK9_93Zaz+?}^wR+_1Vs_DG7Jol_)9Z2H@VNrnBd!P$t<G4o
zj3e?^diF7MLM$$>kVg3(fD1!jN8S8OQKZkjWRJeimlv1mLLIzdX~9#KN|@jhBqd*b
z&Gqh&Uu(K!UN@HVFsXons#;)ox^RjHJ@ln%eg+w(k<0HXd?8jhL9WyabijHa`9*0h
zj|~%cq-paX24X!J>P6kE{E?q}=-#I%cDcX9zdjN+@|kAQg5cr7M(mrtg!1*~EtdMi
z@|O%M7Z$8w((bm2Al$G|*^Jcr6H+0KoyFeu<Xt3=#B(}-)Qk!QDwnCWIt((gl*x>+
zvEHWBygV)nUMr$#N^q4P$0h>RSZBK<2q;xolK06A1e-K@$#Hi{e)+wFxsUz^5t!Nq
zUtzDDQoxfKajyZBTn$o@H<IWkXq-Y@P)>!WDqr^a3)Js2hm<SS1QTFt2Ykl2fb=2g
zD%6=`BQtt-_CTYR=?iY#TiZO{8wh{r1{;-ruLey@X3s{@s*?u2ah|&w{~PDA?p9@w
zNFZ+SRILxP!v8n*upU&394G5>-$jPde-`!+j4o1$C?5P!U|fdz7e-<Z$plRXQ^NCw
z_N5vX-?_0O>gO;b5C8oIS#_^FVfL^9TYe&=2Z6&8%t=dwGWX!kKNS6!Q7SKE)-@gi
zwfDX&M?5#$63br_r};1{HntNCbZKkve`-ObQF%uegOJ|!Q5X{u|I;e9<x7^@W)FxJ
zNjK|1s?&F?sw<iB|6|cL?S6q}mpW>$-<o4f5|N1u1N=K_ml<7gVzyx_KWfi(qTq8G
z=6`9eyAx#CkrHQ$DC-uigVz#fN=9|E!e#wpI-yXp?5+OU+`C+ziN0vm%4^|*_`zQY
z2ii>Uy_^r6l<dyzpTDNc^}uh{FzW;djq7ASzw(k6NZ|=y(b{hcfBQ}UEfnO&+uZOU
zkGK!1vRgOm2zp5x;k=${ulw`$gsAXR<hUT&()_i*D=~tq?kj79pdwCYL+L2;iu@A+
z=u-J2gyK3cp}}7EKM~O5FH!p}RmAJ8Sr&3Kxco~$CQuNpJQ-xK(-E$V_%AHjJ*ogw
zA!-|NfDV^6E>_iD@jKr)m|<Py<N0!Q1v2!T2QI%ROXaDE@Ngv!jiOJic!}8ug_}1X
z%34}@IMr%ZhkId|RK~OA+UN%>woT7j|3daXLJdJZ%|=qx-H6Mv%1|fEGRn`<)6ank
zs?G-n*oHz+fs{~aj9sCH=ygTG?5s3hF7^C;-i0>b;mx;LZ}DAKfd{svWDvRi;cCz1
z_IROMi{5;>nMJ43I;^VWQKpEH+nzL<dlSCxZq<|9e%<f+wRVWHD-xfr+ULb1!~1sO
z{K4xU!tY`)+jsEVW~s3UauFR60G;79%Co=NU^UH*@z0HvDYS3{<ul2}4w!y@#P9VO
z{Q?#G8kI2+<8Ibfg;!s}+DVCt>$xGsf8@DV0>6I$u#sk#6}Ke49?RUa9WQHtJt4WZ
z;<g59Lekmy;+w6fTUGu}P!yyI!q$$r?RWsv8?@osb-pxWspLyU#Y0=qRqCsJ)Au+i
zOl3BLE>g^9oZ|U}`jEz;^e?Gnd;X<Z3x7nM%lF2aL8!E~7i9id_5y&#$(L_?j5%&@
zIpT!8CNJ91WjIiGLiikfwX}uB#UVnzj|as@4$^sv!OK#l3n#8Lq2`3ziBeF??W(+w
za_&lt_4VuzgZ=%&4h}4%{8w_$`{Q3<<pUz$<7xlw-|YFDbi=>9Y-NlV%Fp4*P55Cs
zI(ST7;-O0$k(ZR~&zDGWsJ`M5;#P0jeH%c`k)<Mwm4IoAnpBTccYnE4Uk4lf0+($O
z2Z*6fR2zv}ggx&5uqz1Kzn)S|=}=trz9ZtaU54!BnS5@L9s0+`+S@Dc!omU`i*cab
zTQ8yAdOt->Hv(0L-v<<e{^x#U4%7aRG{2-m^_?fgjV5k8-(D0?=L)1T_1t%oSgV>=
zJ&+*h@yzEcGnb37LH~(kbYVe(sV6g@_19UniXG{fa$8k~Pn5~ef}L2Zf>mh5RaEp@
z{Nzz<#oQWpYmF$POl{S`teXNiKV1o{KxG#DW9?sVB+!(Mqn#M{7De-wF|-DV^8_%4
z-cZQA0-J=jAL4{2+uX0V_e^30aFB2rk$pSu)&wm8d2=yBuhg)p53~RPVBkzzX=i6=
zV2p-_<{ZTDNFpQo1Ri(E8B5HHmWGzznUEyufzW?>BZl;v1)q-DENaN<u(f-M-M7&n
z{+6-S!+<a1pZO}%oCk*K40KeJi-jZ4VM@E253khd=oN)Vao&D%Y)Ziwi3`Cnb)fP;
zKrvt7uLoiapL$&ABMwS(iF^1G+g~*N8ou*qgG&{p<OxXFvA8*$alE%ZTWwbvueVzx
zzB!sp2`CYdxcru(P|Wt67JOm8Rw$by!%`N`SYaQ}s<c(d>2|6|Q$3BOcvjW<%FEci
z4$W>gE0-*Z4WR4?^S?b`#qi_UoK2S`$VmcB{#H#1s;teCMmETXlaIPIgBWr_)T%sD
zH2}+hLqGnL!v5y-FHMn8`i=tCFK4m+wMoTv3$S{=aLk@x#*5UKyW$#7Gu>8y-(PIi
zcs*S9=eOwH&vm|j4mmUr&JxFrP3T4oq6QrlrWC0D#KGWnTXm;mY&#cmKAu;$Jz8ny
z8Wno;G6ez$y`=msu9ki}6+7z`+gQz($3a^_PzuoN!`{WBRh7CqT;C@rWMa_zyA03M
zpS2ls8B|y|22ze4?|jM1hcv^5SkaGX(USv+MD)4d5fnKTQWXZQ%-6XvSx5cIRc^^V
z`filkoT{_Ln7kt`v{b1v-cl%mY*F?HMUUeBRAP+ZYadjd;5!8(L_ivjVZr#I_J`$1
zKbno}<GQiUmZQot(hT@okqPUzDFr^1F@ZZhZX7iW3mS<SVuqI=*MGut@IjDjR(0)9
z9)bQ9hq9J=VpzySd<&c(EmEdYQDDz|R9?qyF-a-<PCges7=<vnkHk0QEK%z&<PM7Y
z#6-)$8$UTrRjjAtJCk-=tCNwp7c%oYXgxXweYumQd5_@RVP1=VNeA>Z&ykbd%NZf)
zWTfQYXyWjYPh&vnehh5I{5nRooQfaJ!j!f7CEQ3}#cXM&6n?9V3Fj?Nv}rXRs2Mof
zf8-aE{jSCB1F#Kd`Nm#~|KR<!=9GVsFTO6BNEcum?-kPWl<+F<^(POpW4K#!%XYs1
zvg&mue{f9H@j+xgUftsUd}9(pA25{6(7cnZyS>I<eg}Dz9>P<`f(s8!o4piMIL0C{
zI-D+1F<<d{I+=$Ql<o#fl9JPvR>{PplH#mC5W|PvBG6&WzP*gu{MzaA7nk*)%Z481
zL%VHy^Ao~Hw474>MOf#z)CW_wjKYq6;#$qEu{P_{q!2N?aYJ?17aB)#@p=rKDzM|e
z?DBx!>~(23Z%C9Ndm?Xtik)Ke3q5T)3qM#o3lwgqJ)x6n*;BpZZZ=uZ0ds+cA^x#*
zT=7}b;S&|o=vB-+J9*eoC`o63)XkLh5(sl1F_nkEFqrg4{#F!vq0Ib=%b-JPwrP>O
zA#ll;&hPFR2qrtiYB!f4hTS@zt1`TQ;&IyT_e-}_QoxR|l^1jr>^!P$9j`M}JQ!B+
z=T(2aJ)s-A8lcLS`Yq{DTAxcJ)gOn+mv8d?{#*RV<g*vQUq^OOYe+Rt7P{-dhra1I
z6dyk5iV@nZp=T`x3Li~L^Y<6LONkP*7k=W^Exa;(VlwddCKz?!b!roOz;5GtA>?<p
zg`@Y_{LsekOBXO8_)W$N8D9{?tikHnNtzMf)#ku)y6d4FpM~~>7+#f&LstLh)uU6M
z>LG>TICI*njs&?_Xm*tlj7KJ-7fsf)q#o+97cN**Vfj>vm!#Uh<l@XhCfxbY>rs{C
z5Y_`t6d{+wBXwMCF7tcHK3;k)uacWvn?$E+ywqs3qjT1FWg06#Q<DL_)c1MVLGOEl
zBnAa=N4(Vs%PZmjVK1AaBflD+Jwq~jA1gK?_<aoDY4iCVB$L=ax$<XRz&ZMTmi3#;
z_Iolk`;|^!X@W$SAz?tfdi8A~sfv<*6=B&I#$L`a=Z)uA{o1-Z_OCuBC`Co+4vZ?c
zN0?&N4}x!mf{#A4N5(>Ad3Sc-uaXJ<6>+E=$DTsEE_c=jUrjfj9~jn;G5y|AA>O<Q
zo^R&=V&V%Nrdn%5H}34&33~M*w;$5FzYELYd_|1(flp-=TPq>*8w1kCU=DS@4<<tS
zw1U)pg8*{!hO1F&S|936`PMUDBjTGCT$E<fZ|?IxILZ^*jA#zZWbIY``2+UW-G$<2
zWijxZoq5xG@7nqPe(S}+<G8)c_;(aT#Xk?<$9fT9Le|$3I=@!dho^j-o#(rFE;ck8
z?3y$FwZdd<vZ~SM^wf3lbs)&W)vWmz$f9W&sXEGyNws=|fpbEm-hr5Cw1t-7FcABj
z`=s27<@h&OjjfQ78uPFbO_th(=@+S@kxDHJdfr_1GOw&q)VzeJM}eHZ3o@4N*({Vq
zGX4qtU7HWVdeF-+q5C<3vFW^DGrWLO&dNLZt#GRP${%G3G*t0<oy}qUtO))P@`LQ}
z*&bNqq$U(%D02%*HYwf}HPEM_BuhUj={Vxe*;2@2X=j?_<9Uo^@UV$YpDff-ZR+7U
zL|p|^(+<N{^1~^doQ&TXuTKHaug|Ch6F&ZS3%_Lpz}jzginq$5Z|+%guRe>`<XGdZ
z30*>+KeJDCd<gr}M2zPobX+ICDqkD4i4!nSTj{+86n<gX>c9Lc651%A>w>z6Si`=F
zGvU<=y_Z4XodBW+ZSaxyF%TmJ#?I3QPW8g<0Cedt-@H)hdwdG5+38;jKAJy>P;=NI
zEd^Kk57}ambp9~E6WM<ed>8%0AfSpAWKZkT_htKZB}n(fousHuW|2$J0>e_?kPp}L
zs>-(Q1N7Tc4TgBGang?Mujx(=V;5qZbCVlSH<N8lyWHkJs0=qp$#Sm;+hx83U1d*=
z=bI;twx{|0)W;=>YxB&gqJwwOjqaSuxYTiv5y@lSC;1Ya`L(m$M*&l@4bB9oKex!r
zZ{XtIP9mC*Ho~-)4(RpB2b*u|B<%-Y6pgjDW9{IBc_P7%vBUOodOre8Wu}JgHl^Pv
zCmupaH#_<U2Bg5<&|;(INU(hcs7J69G|F;CaCW$vOY)lsuQdH+9Ubi`byRdpwzjr`
zq=ny7C5($82;}`{1XCVPQXp1)y|FZjr-+akt}K5X8;^EVy}NStC`f6UMxrY#>eo9m
z$4Td=IMvW2)U?=1rz~^Bvk2FlUbAz<1@ZcFLjNC6FeC!elSiGnt>KM3+~}`OW`AZ4
z;0uR>pm<O9zOB&S+cke|Z0+%G+U39!dP_zA3KZ=iFFGc7>-;EQ*9{{)<=b^PKshgM
z{vx~W9dAD78gcn?y;>a6%VS@<j!a5^`OBq(6$1Y?%X8OaM4Pv<FWIw-N6z*L+IlHf
z{Pi%DguI{2)r@e}@nMx}l`d8UrGjv{Y{V|lS?Z_)JMYIpQLI(tA-@%kSDRlJ{^A*B
zGoMe_CIKM8uE_IQv!~MYGC&FwO`@;)(qR-W!u~3$k96ul_`7Y2$zpx$M4&6qfdGy&
z8K-j9tH|){p=cX%+s8TsFI2OJ`uy|eqJ55CU9UP;G%EVFa9DSN^Hrj`tw;_0=F33T
z`@3@TV=9Ipf12KRRfEFk5T;_Pg+AgNeQN05vzWqJ59aeX6oQL3hi!Z1x1XgE`6!Q(
zxl*pw(Cj>;Mw>Nm_>K{WXo*OK-<uh8z8$goT+ie;&7oxHZblp+X0pVe6%B_nr#_kf
z(HhvFJU2cSM;;hNp^i@G@w;sHq~k=y*%|851zFlL?$s~_3T@cmi3#N{lyjm3DIkrE
zY#sfKMx*6-;#gs!>0?l)M82s#D8gexY|(fk`R$#85Xo;~F4@v_dH4udx^m#62@Hlu
zg$Q5?Z9AOk4vX|8I*y#A#$|6!c!N{RZI|fN#kMB&Ij|)ITSV~FJf<JhDm;F})^cWb
zB6i+Yk)8e7{u-GxwrjZ#a``Qpof~=pD%B+)aS43!xv0wiyKY@%w^Liw)`;_0Rksbk
zni5K1irb2%pdDF9md)WShl}F5&5onW-#M5uk|`z%e}+S16g@zQ!A6IEAQ?(2V8!~A
zp?_4Ff9@QVq7ZThL*P`;9t>4P@hw_lj)z`o2@E?DgXq+il2@_(ucd6LEn;X7*LONx
za0)!X@82yRC3V?ce=pYwi}Z?@N-WoHI?c)CO$vH0qUJ6p*z`-~w9)CN@34nK2%|^L
z75>eGk0QSWrJaXjy@<Querot`XuAYHOGx5+$Rd0l!Lr*zIL-PavJMvdwp$lQM_&N*
z3A#L7;f087^|BSyehGD|UizfAa}&Qj0)0N2RdGBSc7-g-H(|r@d@_QQfnV5+_fPeH
zC*P(mwyfs~*}LCdk3?-i%x-4*oVakg1lsx<Ka9U6pgARnhLa(IvJF-oyv|(DTVx&N
z-d>x*K5o|*X~6(ulJbs8aMIz8`$VO<)3wmW`)SMruiqy2z16$sQ~93{-3}u!vV9|%
z_UQlQJvg0DqKthz+}C7zO_rSy_-=;~%u@~lfk@M)CqjJlw~&`UW4u1+)LAkV)ZJl_
z<uPLY*?IQ=CBOyaEndvB`(b^V&Hs%b!Ryu*tXdP4rO?AurB%4?Fgq|<ZHOgH$Z#o(
z{>MVT<pt}9<T>PNaGf)qqYzW^YHv&kT&&n3hZQZHX(Kro4s9KuGg;?yq(Sq3fVHr)
zsxR==|Jt>h-2%3a|J|w))t?D=)6<6oll784bpD$!S_EeAel$T_asFB>$!BGk7=kKn
z@TXdO7vi-K4qOR1Wg-B|;0CyjaB}xogfYex)3XZ6r0C|9a*~B!iW3mh2RC`8*FXQ#
z4al>S5T1bs@QP|QOjff!*6;fn8l?LfD!wr(Sml1=v0GibzsZK(_q@@!AGgost|IC&
zz=|u&27S3Wp;LNqBK(}P0Hu(Ena!~up;*PO>*l97k@a1fguXkvA8d$*)Y$lIJ2zz$
z9LinrbTM#+wk19HjlSjP3-ZrR+KkyP#*BwWxnZF@FXyv?ud?{N)fkG?EWH>?Iw20~
z!aanISC7vecjc`vsF}PA+ioz~@Nw4<xPBnso|`Ugx@pe@|EWN3f)_uq-%IXkSDrKD
zwU|#|lyL&vA(^?35YKm2cql}9VXuds6jozy*v5jE0`_`S#2&L!1=$1*aLL?Hq0F)m
zm+s_8m<*rfW?3Cwi-Q8C=O+Z)J#OOH2ocJ=u#pa7nqCO+_msImWL6D>bvX!Ad9j<X
z#otF;!<D^#`nF#8t3ChQl08SY-G}q2{R_#};^uxOUEXgA=m?2O5+}0#*fa5jq(e&h
z_$h-7s-`q~)#%aI>(&#fOzlXs6Ft8!PW)&R#9FOC5Ue@o!i=lb-_VFu7u(_32*82v
zU`VY%4>d9?=>tW1SH5ByUGiJ84x`i)x8e)Fso{w{3+E#q2)95u(NEJ|E?j!y3uRfq
z+fz_`KQ~269LZmu|1Lf=0lY%BaTPRu<fbtfSm(!z@xU}Wf@P^rK+5*vh3Tl3Q+9!z
z&?1UJ_a5yIFLB$<+ni{A-FJR|o_W$*wB=s)-S=u1Uull0?^N^wb-SEDb-UEe_W7yW
zb%(m0bvJN5F$2#p?5mzWoK-p-A0IL0X3iiSO;I?gpRP26MQL}vh0tjFP{c!UO1wmw
zhc~`tZK7Mb<G9Q-PNrk{O(}m*;c0fnV}3=s=zfL6Vp9B}<SG6uLwob4egESU)Ek#X
zLF;>SCf<SwNoaB1r09szh+dS8(bSL}s3b}m8t~^JSW<LR<u00IC<N@<XUH*>GgepC
z27_-=c9;lxng;ITh`ix9V!80sJrUB=6MGSiEmEg!fWgF2djza^nL!EfB=s2sIeGLu
zq1q@G)zZuf=M;mVqBraZUh5goFS&kADZ<}8M}OqZt5~_9UFJeoD(9(aeFtM5o{jm{
z3L(oIXPkc9Fp)8EJWle@=J;}Ld(8XA>4ZxL(C?^fIh=rvE)qY(rie@U-|S_=^=2{V
z6(WfOPvCpqHlOED`|cUk#UMN_0<4FyEkHf=BN0P)VsEAMApAN?++8!5|6C7@m<7!G
z=n#6pAV>q-swx+BG@_Au-47bhekH@<`vhLQ&?;bjLpt-~CnA#2_osj}2noBx*^DB=
z-8`}xVLtBFo$wsRs88-yCVb=<#9_KM58f+!3#56wHySz&L*L5?25R6wULQtP*s#Cq
z&LTUXxKr{1Y0;!^6g%O^6q_~=q)9d+VrGAz%Jk(RRSJ(G!=*>M<6*Li%?v*xWWU<@
z*WC!q5jycH3>a5YK?)~vmaL3SV?=a2tTt}_EY(e2i+m<xRj&s*%g)s!dGHmauhmy^
zuXnuo6XtJUYd7m*6EH~Mn(gcFoR}m^bgkk^b0OxlJ@AWig64D#j<H3<8UxyL*w+uY
zPouvwCkG00&b#|>bIUBervMQXG4kPZh)I$Gu}6I}_+8HarrWYZ2y(J?<R>1LUZY7~
z7uEL;I=tcrVr08t6f&rX?B8DQLZ*)AYeZ2o<9n#{oG>27?U#F_2pw)0%w&~ukz_=b
z2WS+qF^M>|#Olp;1;u^3;)cI}=x$h2g;vy{cpsRhn8pX&y#0AS=m6eefrsBu5>N+?
z@W9|M>LxY<Lgw{z%l5%aFgyG6)-HhkJ^e~20$Ej6y4$ADT8Hf*K25W&UOJmVlMc9q
zj%B?5r06$R1brv?A;X2memE`f&LZ4?`=Zdv4|D7TYfr-ckYZ?-82>ElK1o9W4ZDB1
ztIium3_rP{1nO4(?n=X)0pWrL0(vPT6}ga5=N}(a9u?g88gq4YjEvDzqUe2g%^Sgp
zaedMsRkNrvo#P{&6_B6^zji3se`C?k)#C2yLzl!`NMOh)ur#DtJo{{q+JQxrUy1SO
z05|~*A+-lC!7I{A2K<p;Dc+rrAzUx@B#A@6)#heFR;8d<^y%yjS}Y;Z&g$QjCTdh=
zP!!D8L>((hrn>aJ{BXdRy=<@r`Ac7ai<oT9@iGIkBa$D=55H_5m?G~l-Kw1DPbp9g
zkB`s!!X7<j$c`TktG|7n=gG(@lg=>D-QU<275ft&0Ej0+*DmX46_4vxhrDFt9-tNK
zJn1yqG7z=g)MQYP7xJpcbYmqM-5g%Rs#zT15@_hRfmmEmm$g@E5YgQ>!>9%uPT>hg
z^VzY)GXhf&k9{CL@FDQlBUGjw${&iwrR+NY9>Ty_TG+);pWnaIz&DokP3_?a^H6?5
zO{P-qa+TPW7P2l5ry^*iID)k2c<ym~>PkkiuwnRI-&HBXrgNxg3=mFpwe!iJ8A47U
z*Tct!=I*HTx*TG$JSV55P&G}o(zYbiYX>yiEL{@yr*{6%z4s2$&@+YfAurUTc4Vp5
z{L#_tPTF1<*{sciBb{@c;MRXizSF^Ee78Y)Bp+HpmF=a18YNw5S+fr5L~|K{Hem6s
z!phaSRzW6O-00iGnP~%Mt!!am0|t6$enG${CFVkBW7q-YGB*9#VFH0A#tofXTG={E
z_y^>PV_p|`Y>bj^ko-`+9%v(If3||K19D&|)VuQMZu)m(hm3!S$on4EwnCnAGOKvy
zSUG}d^0EYiy@4|{`wGh(E$Mv8nCN+l<RTK^XO=%S8O=Adx`p%qfwgvxE$0<J$C0MV
zNrA#W2)JxBu8s2Sp?`4Ngx?D?FvtggYF-aOD&Pn|O#LS(T{f_*nYJMEH5zwMnr|t3
zzQacMB;CgCV@|USdtDT8N!j{y`owIrgUJE2LfzTl-fdB%(>!-%8CZm7)sZ2XR<5a$
zNEJPNd_=kBFSu9astQQT*U41Vg5>Bwu&zfnPveWo8M6Un(Y(=rtb;&w(+jQ>z?G6*
z#QM4u`j-0An=l5&f35kyFdL4vGQE-|I6#Z-H>7|;xc-Ip7_v%Pt&8v6{{t(QTy4@A
z#L7`j8}Z%#XK1$?QxV9ZD1qMlACM81?*i-fW9=7lOjVt1Dpx$Szsm1cV=E@%H%@g~
z{SPq6t+WHb^@%d!pNy`s0@wl5q5nZ3qd?)fl&U?{Qbm^T#@YW-1C{Wi0AmEYPy&w&
zF}?&%9TM(;@M35&I1IU5`jIq^-@5PLSVS?NRvz-jMK#es;4GvagxFpErKHQXbnL$v
zVaZUx66^(VX`(t)XgwxP0(Hs#B`&#`)5dR-wG!~x>=HU9A-kH`n?PWCQs>gi<%&qQ
zsBg#`5$b0m@5Zo{E=Hk{sYzM+e0f=4t4__g#-*sKan52kXT8T&oV2EN34v%G36~!e
zvIPj?cRAc<H~QgJ9F?l|_X;)iiY&MvR$HPn@|$!221-u<`}@Sl+Tg$sVc6>RU~*}?
z@N1Fa*I8T5HT8D|*b<925>Ma*{;Ym>SGX|j9Xk)r7<rG7)$d;+mvh-EyLtoqG*TAb
zYem2%ZIgQzhm8ODi}rIHJO-uCMjm|pHnHN!8u-)Vo%xz7<h4;u@VZvBf&X#ARmZEV
zN)3zjpPO7@YJZKN>&+jUs(*<qS%Mhb6c_iIz0GXGTO*{${GJi~V(?P#UF_@%7jQdn
zpADR61`mh|ZGU)iJz){qK`ylr-6Nd}8kM(&9QGw?WNW2ILipUNc;P)Mfj8$NYT=#L
z@$>0aMx5ziHWUS0Qc-K*owU}E#r}%53m>lzehqGszsvmpi2COEx|;9nHfU_ywrx9U
zY};m&#!i~XxUp^9w$nIiZ2P@!pYQMe_k8Bg*)wO)o>^<w-d8o5ey89`FH5fr0dJ{r
zO#eeOouPx;ham42X*2kp-PDu$+Vf|rQr-ISMTv-H#XnDX2kN`sDg+}Q@a@yzFFdkP
z<g7I+Dc4{(MK*x~prJbC5~RVAXVm!uRr9sDsd%ezqobo=Gs^zBWk^XK2z631Gf6Y@
z)5fVI%E+3m7hNK}YPMG9Ex!WrhWPf@#ZLFHtEEIx)on_=(9e9Mc8(fN;Oo6cjrI^?
zmQurlTjR3S`kREwNL%^ybTEGW6?uY&XbeR&Rb?D@O|zn11?KN0vuTHsMJz-OE72*G
zPQ}wUt45!oVv)u#HU7PuDdgswf1gK^mOxYTl1WG<O#Qno8ACxskOs?`e+~Y(zDne+
z`m1%Bw+(szuFukVY$hA^RLZR$%9rZ>zgPMx$N6epQ4**+O|i$UlS%&^;+VNGA!Z?n
z@GD?&<+}gRN|6O%C|re{0RyIr2pXnlCT%+<<6kqS3dJ9JQvtf>sHDmSrGnO;wO$=Q
z!G9OG6g6A#!H8_ol|}&d>Z+R%jK0BND~{1afO(6CsD`|>39yTeFaZny*0QT2?-5BP
z1#+t(P=V}(`LK4T)Ax5+q>0!KyfPlfAV`D86}IlhL!^KEFY%s-&Y}=mFz@{ru`mlO
zk(g{SrgriHw&vuL<o{jrm&ET#k!6BQpD<QRSpO3igp|zflu-A%dHokK{!9Y+$b2sR
z+R?Xvssh7fW~dVUJjh+NxoWhCf3b}|a~ysx73!yDUCsZ0bX#{uy$foJ`Y$+PlCR8%
zaVk)kT9nHEkZt@YN^oW@MuOiRf&4l0-}J&mAZuZ20PR^vxAjE-J0~e7$|NCYJ5Su;
zuls6+4n3|p(433^g*ec{aZxo;>!188pZ}+n0QShQkSNr^`@hHq5s%HbMvv`}xk&m}
zji;S?EZ+A=LV`Jn*@i0N_`m42zA{%0Sejzf60>o`INj4iSIoEVJ_*_U9n?qNuhI>{
zvBU)YsLm-OeQu$%!-NAUTv0>Fioh_`;JrMF?39EMcEdm*(4bLn#1o$Q=@r6SM-_K&
zACl1g9+mfIkz+YYJ;I>QuK8XdHuvHf%i|=H!4S_0;he9IB11*5)BVp5(U%~?$8F3J
zPnonI9&*;ZIZs{|hDZY+3R>{OZq!qrn!;rL*$wr{z`Y=QMXAVWIJIc|G1GRt7JU0g
z^}$m!$f4G7y{KxZr)IY%dpWq=iyjOwS}DUwX!2U@(c+lZ=FEleoFACr{Le#1cBs%x
z_knw6CA&tJjd1xM10cpxfj_dO%U-PScg0Mn7|Y{>O;1*xUyru(>UKcc`OsJsobWCx
zP5S~?M9=xB>!s~A@Gc`eLqvc#AT;#dnXi>qUS#)oiN;JTi98-BqCdN)FbDXF8Vsre
zf^QKD?*t*Tl`VU0sLh-X7SMTJ<#@$3G8K73&S=$$)Z7@l;~%1XUT9{gbF}?*xXn=M
z59dJ7ZDp<&LJ15P8R{<QZen+?)S@qM_ut}-uDS_e%|ErR!<W^60!O-h+JXV22?N1K
zt`7=%FK2Hn$B9((N*^vRhwOP>hitCkG#!`Ye0h|g9U3{g-+rVKAHi@fv1{UR8kX0Z
zKApuxgMHk_@Pwdc=Q@?lTIftRllbZx===5~OAISFXz+nf$1P5Onz$aIi#{XcX+3I-
zlB=M1K?sO%GQ#UeffQy1AE=pia{7LE^8Kwv_#44*(4Q(<a5$gwcyjV>wKa;*euVwm
z^{tXTW-Bc*#va1UW#6X}llUi<&wy*je?TqjiJ>#V(%RNyp#jFu(~!B6ms;C%L%;~J
z6@Lxl4Bu_{W8(!Jr7JTMXOat<ul95YH?X=z$Dup)ji+h*duUJu44rm^U<Ao0g;a?M
z{~x6Oa$#^)3-HNf@P3x`Zng}9sdV6S=^PSXoE>QLrVwDS=tOF1Ij(%^lv0R04!Py%
zV7z)ib&%L8j#UC<P@Bu~PuJoCiL214;L>t515q8Bf_@<7j!}bUs_U?nAu6K>nF&iL
z7A)BAXb6q+U}D49huxqIq4v^QAU#F)qMfgYcNBa!VYB$|RRT|xWb43fL&;T*)8Z+1
z*&AHGSTWs=hPpGUBcaec(qd0wL*NJbddXL9)__>w79<R_Rm~<cL$`-Wv00BbT|hHC
zi3Ry#s@^@<TtoF)EI<#=L}oHUuj=8)knrDqXGaQ6&Wt1~{`*hIOxSWTHs59WFmQ<@
zdLIsVy!J}*Lyrwg9r7UM>9dekz7`!ihhs~T8S`V<axK?SrUm>;QuO)x8cOxP^yIQB
z0bE=y!Y*J^5k2wiXc*DB(Kzy)ApA2UE402guV==;o_?{qwCyox7;T1Mn7xJ2feV9Z
z23(rE^d&Y$4R$>@1yqznmht<-8W&acZ`c-CtPY1WWO;w;YCX<bsNvXIS&QwtxR4)A
zu-r5}=)&(U!gvsi5S#aoK$Kd;F#PqnRAaC}4;Cy5qlL$LoWU}4-p&8|zO*UX0+iL@
zFGDr*`RxLh^!m+ZPK?K45O(((qk|Apb4RxkbN8)z*^Z3M8P21;^?f1vPConjzT@k8
zzc}w9w0<FB#36XC|4cLMWjLPV?Qq5<XF#(NfHf7<N8xu7+%^s*3_wd#?;M|_AGV%R
zcp~L_8e=sSU9L-L)ImNx5jE-&g&nC5=9<I#3y5!#U13ps5riFj$tR}_)k8XE4~S-e
zf=A!yu-nz^2NCA5N}*Hc%GVftu6Cia>3R*Ar^po6T&k>wWU^b%7sCPwM^)-~;ae|N
zifL+M{gFNauel=|Pk$2s*+@#rfikct+(Feb<)EqfLsskOYTapSu&qv9+M38(Gd7X7
zyqAD4`0Ht`dxV)@ds59W9hS}GzF^$!F5TqZv)^R8hMIZK2XE=@ICZHGmpka(%GbXK
zB#vF)2Do1GJ)l<NEW_OkLPcCR8BO*I)EGXvy+RX6uYdlU1?eTNq)`j@@cVJ*x(i_^
zp^}Pnd_73uT`%z25}p$m-$t_m1l*t1<#aD;FnhM*q~`EIzuKtVHX?lVD8TqYn3v^V
zNcP!|FQdd#(j6*i>&9SrkgXc(v-a1o0U_oX)F9p35fd~RyUF^sjJeAcJZssxX@)dh
z-~Ezk`*+$u@qUIW7lv$ZvhAPc@-ujd(H*`*&o0l0rD?<SZ}++z0H5li$-C&`>7wiY
ztj%ZAd@9WIsHCV7<=1%Hog}f@jZ)NzxJrho%Gq*%`uV&ICq)23atiSv-tdQFM$vjn
z7=hCJHIp(*Z^=V+_b$dK0fkV-5J5FiF6XbTtIIF5OtiV3+yqcA$fnPm#7a;2VzQRt
zKb!AIQz|5ZtR}?@3iM$H9+q^f-%D0G%@U6C*5qU*VYVdh*{>E96bkwN>;18r8e@t1
zGIa!CE=$Sl`~8}|M3?WIX!`x%;{dsoXt^y$g8*|iqo&Ktd}+g(=nP*jK}oWe*xG3V
zw0w(i7s7pr+|0{-59dSvd2up1_th)j<5>sB<;XM0vUXN@(FPzmHOt3epwk1U32zQ^
zvOTzY|1P%LLz{^Uw1=Aeeca2^A#$?mX5hhi@B@9fO3ZJo&U`T2y+eiYz}TnF4lg3m
z^3Z$Ed-?GORmZa)Zq3bfd+}qkzm&{xYwYDYeAXj4#DSQE1auwX#x;H*NM+`CgUA2u
z#AXARX1o<xWmt)5CFA=6?+(ucg+HUjJvTc;xYv(>%`J;2OhKX=11*NVqa!ILC!wVU
zJ)a1J2VZph{UkAb@~NDM$@KS>2G7-pgDwV6Qr8tLKR|#WIF-+FVvb;EO5meEmq4Ju
zTG=~CHmqGhFEtF#mk5L|gdCab7wv`Y=cLyHNQ%Bj3b3(GS%W>C?#(g(vbKoGpA41$
z9%!gZU8B;8552_ufKCRK7j`6K2C?}Xk@^w6H-9O6(50+gNE%05rR?9T`H&<l!viDF
ztDp$-pk@<b<i4L?+4PHJBmHyVa#)GGyx{z+f{L|jq^Z_KK!b)+Ji$v1p-C6u7UGI&
zN?Ewr)QzF?R&sgVPJv{m09~a%Hn~D(lv19!;o)?S$9|%^!-A^1loFf!`L8fnD&YCT
ziYN5hkn!)){Wgi>8&GY@v=&3lkUJ-<AScvjb|)Ho%+$&(%t~!;h6ZprZ)AJ4c=k##
zb?MiAi0Ta<Y80H_719cQZ@#Y_s_|N|rPO+bGOOx9e$#eWsimarevX&e0w5*lisTXg
z^2KP)<O557W5eiDcti>=FwN~01$9_9kdYEe;}e^AMufw4;~Z8P6&Z@PAyh*a1z)T2
zn&3T!_fq`G<+*z7k{1qHfbe^3#B^MvEMIHJznEHyN>a5-wM#GoL{1T4g;!#sKL&kr
zMlbKr-)uAp<N@uVZ$&^rILphoUB#JNcjkK6<3jfqP3vm(nyIN!z0#kJ_(}vaH{4{o
zh%ox2ibb9$$0CQdK7|S_u!xOu&tqmQV6@Zis?(EUnpm6Fps5&>C~4T!<}6G2)(4*V
zIuudWQ`lo2B!etSS(~WxcepVFy70kk4{^ot97E9gL!FqOnwSrq089@~Oz=P!sLvHd
zfJo){ePW`_&NoF!GDo_v<q?ls&59Y?7}?%9mU5!ySU+3>;}s)+*NGA{)aluL7CWFh
z9(SE#OmI1ZPG1Acv7ljKu6{XB<;Wz{H$NYjF&XxP{gIkpOB5(%@p~NRgzlZhN#F~q
zsiC*Jonb%d{Q;k{5q}i91&Yv6&<h7r1f|*RK*s3t^)e~kxr|`w6Cs$K(gyh}QNOg^
zl<{<>H^%Z-ptj3bUwy<o@B8kV1N`tK`}YvQ6Zj=~!AnDNQ#NiyT_}j%>?sGQvZ3`f
zD!U9KjPtc3IoMaNVFrn_fWR<*5rBANyY~a3`~DY?06NF95xyVv+;`u#PZg5MEAVMc
zQjI>S;mN|U?QBz{pNR?lx}__Px+m{0G0p?VECZWd@Hd_>tV(zS-7w0<tQB#q^ZL(*
zp7CD%?==U_CcBbBv>U^`9MTvfyTF<aaW9vj;?}y}@GyOh_8NoZ{S|_XJ@p_-?fnty
zQb+IQ4?c6egVXXs_drZBASB+s@^$COep1K&QKwQ9QSl_WXDe~3LzKxp>u*D3=Mx;U
zP^6Ku*2MrK9tE+1i`4`lM|9s|#5tYqpA$CpdqTMT8^!2xxKVK6S9KNh-1Bl_UaB@N
zM!$ET)0H|1<Dr;=O5L_<iy2}dC5?Qx+Rz6~aE%2PC)tkX*ysidbi*Gu{RpObZ(x7f
zaeN#D!91x5Z=%2+b|mGT5VLh`55nHFipudoD<C(}R-T_@-<-(27D{NVHnes5JA_&!
zhJ0MUYY0q92|acZEeBp<VPDz`7%Jn|L$r56AFFGi&{dX5SXs6kbQdB?&v$h<U+;Ay
z_q3pLvs?lOIw8$-ez0=9>F%SS7)%<KkjBsyG9W{a3C|?D(LVfGUj8ogNwJwd6~THd
zAIjf)k^;(MDWGu{hREGvPX-%p1@$TsQ8a7lfU#@OV(Yxl|L0wIe@S;-o_}4Mv_1(|
zuHTuk!KyxF3^w?fBgQBFN_4Il!>4cP)%mbJOnru$L|`+p1g}*9gf)M;?hIGKt~s&J
zrcW2+D*5YKAE3UeL@{$B`ES^ZsK~~CVYWE_F15Shs6ER1uYi?dQ5G0#^djR}@DM!{
zpax6S;Mu%pAb+899&15YlG9VPvrN4zX^|9rmMqi}S@vwLY3Ps0?IalACL%J@@sIqu
ztg2Jk(z2vof8`+@n1CZ69l@9h8}A!U&`45b=uI7f&I6y|S#9Rt*$kbEv@>$<lw-5P
zfcY`YDXY)|V$bJ@#ViR0C8mU&jVbr;w26WRDj3GZFyMTh=N=zYy;f0}q~NHwvpOaT
z%FYu&Uv_H<Po5jn&xQU%8>d!*2{&y&6Hf2A@mXImj2-9wZFZ=+T8>;gh_l|RUs!P7
zc31u`_<Ph>P}ZBsj7*f&oW)0zoV}oC%2kN=kKEy1UNK%zk-9^62GGp6Pf2wlphE?r
zHafyPXNbg>SL}{-A$^VBpqI%CcH&mXTa-lKSuC#vy<4V<ioCV84SL7Ae~^fF!hF}x
z!axINYdCmx2#MXMHutjTQ-HzXdkH?3V<_6ct0%lz9+8A<_=c(g9Xn0adGC{a-EkU+
zJwEysX$JrMqp_u<)k-bddHan#tL3a{xD@#{10|&(P&`f?vI*QrQm7+}Ocdur&xW0l
zZKmOiidw1-Puxylo*+)~eL&Nodv*)AC5T|lP0+NuumvVl0$Vzsb0i^yPNQh+5gQtq
z%pn@j#=*7N(cX&np9#U6?0XvhlDe&AgvJJ`wzK0Ga^Hr<s0X0b-v)Oct*eO`)Dv<8
znalL7zRV~c({Y<fp-_*Rjr5GRe}%16gpNGEENtJw@}rcxN2)euMgI1vcRG9+MDo(a
zA%Ak$jYDT())yiuZb6>9V4^*;DORdO`Gv@CHmG@|Ry=J+QlkKRhLnQN;<caB$=gdy
zvLxc$ap*k_&8OdQpHP=jUhI^Epj2(7!V$R@h&{%RsuOI$DzQMh;8@ChhnYnue?uU*
z8L*PJ`3X3DbV&P$>J{7goyZxS=0zOEUB|GYD8hf3Qo0sxGz%lf#XsJkt@W*0NXC>a
zp%!k1Q)CJP<<JF+#6?bC(q;Nx{4YTIMVW$g9tKpK#@Z2Dd}T;~mqhesyN;u8e+dv4
z(&ID!cqa4f?ka1Uz+u>O6ZN&mVO_`P_RWRXZC)b&9dEi=kNUXbG=>O(lijnb&v;O#
zP~A8U6DnzcgV&vbrN5oN$=<ix%n)O>0LxA#_tPhC-V5WciKPu~)#c)|dUv!se|qo}
z7sQrBSgkhSGWQF_Ub!a4lo*>9;Q~|_jabOatM`=??xjZmgFG0*>EIHJZ||V3n8T`V
zoErJJ54oKJONm>%jZgIUOi2O)p9SoOX_|r(_6(n1S5i8j1y6gLCv}4$Ny_3F<Uvsi
z9124DX+hZ`<EG0QE>DCF`fDNtkq>Cc(vnJfAxwVUU<dO{DmTtumU5`1eMSIjS~1lA
zQ3cxupc5TWO#W1KrzX$oN8oUE2D#x~VH71}4d=sxp}`2_aqMZZxY_;s@O@lGET2m_
zmzK_GdLA>vP*o=vvY3G%qOcu!dP9z#-K3l36LA%D6?esWp2;@kj@Yet+LzutI0&tJ
zuq*<ESXXtROW-;Np~|z$a(Lg{+#b$|pppcKo&qru%6yn1PaXU&68O?zhH?H&sH4*_
zJJ-ny%&Ubjvd{PGKD_t%&Ft_))BF>NzdE4m%;X0`otH@3`W!uuM_Hzc0Y)Z`1Kl&~
z2CE$m#*5YreP6#uMJk*H2kbjvzCXIxjBh}`iEo5`2BgoiwmDe~GPd3{74^CuQS0@D
zj^ou9{rHu=uNr_w=eSb>I^#N+rNMF__v^`l$qsLp;Lhzmn2NwBtib!sBEb$pw~eqo
znmbRS%Fg2Q%&FYRnKrZbbjT8KuIBy(fzkB$hvI58SuI=g1PZsBR(3lrx)wG0G$ctS
zKSU(Pd$|3QPxyz9epJTVqy!7mPGjXO-c$!aUcpdo&*;>-XyKaf3`Pv`Jb@>wO^5eq
zfp<t#uZ*0xuaNLc5q`c%AZ8fVj|^Ky3jXRLpT&rm)Xw@VcV5+Ch=SL<OT}`@V2B=W
zSPiVPs(sU!6~b}`gfD3|A=aX}ggqb)8?<e_ALX|OX#A5_cIt-R?=GZ$C_)iAy*YZW
z&z5q!Gjg_)JKu;uD!P4@NRDt4u!8nr8jcVgVQXyT5M71TeaxAWrR<a%we8e|$@2@z
zvLC42R2ekgA>mqP-d3CJ^?{iah~R(p?x>2JJ~wc{QavGgvHaM;@^9V1o9N3m`>qj9
zcU)TFH4b_B4prqoNl2fEhHP+v$zq5i`0cH&jJ3bEo)@~@s(kl}zHfLeeis%&P|~_9
z8$Sqzt_P@H`!+W0@!S{mSR6ZTXs^s~DC9|CmsuEv+b7}1S1UB1sfg=%T>1vo1)@T$
zxIkCvyKp0}JlB^yI1qg8v25-o4~X0?neO+0=Za02DNPBXVqgGED1jI7S75ZDW&T7y
zU1>rPb2^Iu<iiico#DuYkEb10g#Qgsrpp(N!|xtM8ZhD5d-Ue0@@A{i<tlJp(A1=m
zuh8e&S1EK}iRW?JCzf)3IFX|ATPXWv1e4DU7Txa&%E=AAuV}SJTH4VdmNRn!AxnIH
z&Ov9CiDM_{Lq;SIVWTI3K%nm|xr0XR=ft__i|@hCZx)`>!~JJd1Cz<zoBIg(b@m;v
zFg@My{4*EK$f7R;-5AFH529)Hv|AB&5#=;)H1P9?!iSMd{@`W)<FeT~h$OKH7*e87
zECUa7!HQBP6rvH2K5MFi)<l^RZl64fW6R7yY*#^WIKrLLp4cq&`|Y3Kc(&GJX*P)I
z+mK;d2>bO=l#-rz`h9U2`r{EtN_L#hzLdzY#7d7oX(6XrO2*Av4$A`%5P*hqS^CeS
zYGlX~?(43}u<Dn79}j-buh(OaK)AFsE5ndz`}!x73WxxsX7VJ)h<GzChk&<O+lmNl
zn$myt4+J~t$8$E#d9ZX#xpvA~S_^k)lCn?j{8ww|P}+cYAyY#wBVg4mOjZwv3AGs3
zQ*_GSQnAQtvoZW>QYNk6q;zFeUbwEj+$Kup1<a2D=%(VSEa39ih+L5e#{0h~C~vh8
ztCwerRUvRUFC<M?UnyyBdgy2OF4m*KWfqC@R3&ate+k_lM4$~>T^YU%HawcIyP3Ei
zMQ=#32KQO{FDQpam#G_^(bIU{odo{A({*AEeJ#3PtQsth*4{exVU6L>MJlgH2k?Q@
zr{kn!(00#GtSO2A%=6X8T_5mo%JfFQ@Owa*^m8R2%vx8nSp9@cMGppV)0YL4Ol<B~
zgV+YoYO#(-ac3I}=f^OZ*^Y|)O<2K%o0f7^GgQ@@!^l4JoU_7OrbGb5Fx*@$V$8p{
zg68VwCmSURgiAV}e7Z>3@cH3N16ziQ`HU<Rs_~ea$S98eCk5WZey0eG?zq)0v{=w3
z1Q0FgxVkg@t!Sz4dVLY4p50%9q_6vFo~Sjv#d{JWhVX5kU1cKvw>QXDAG`szICkHI
z`A^1EOl(&>yg3SGQv=Xw)P|>6d6R*6^~OAK1pK%Z84dv%fFTh#U<GD0_`7NVs=|&$
zL$I$cPlZRWA<x2fK8AwTr7W@{X(q2-Y(4M7mn%}5?*RhbHfV`9-4IPP*tY02Iz-mt
z>>V3Yt7h&(t|Zr<ztX5a&^dMG#k3%i_U=5;40A`8E^fm3mE)6Tpd<th;eX-lH*Y_#
z`!>I4b=-W4Dm`4itmEtWAq+!zyHN%i@I9i#U$=bO`$bHthLAdon35e;5s@~GQ0z#~
z>mo<Q0_w>1MZlc{=jV?f(?Q*yulFS07pJx??!N&n=96I07*-#1$x+)!>YJw9N-zqs
zSrc`1#EyIWWs??a#6#xD*Sbxh9U~ft+;^+j3>z^0mDhDCLfW!sECWvi7@bSVUa-dj
zY(wA1ZjTs^F#88QlX)Bgd!f>_$Vlf`o7bE^7d$(zkQ_QeWsonKVjW(W^K}{v-h19=
z136CumWGh52UkEC<Y~5TS$9gUTDq62O92B06F>-Z%&V2@qnyEZwdb(RC*s0C7SC>D
zu53A)?m=yVG=s(nk+co*^d<JA(~#MSNi+5Oo~FH)HrV@RU33ULaafZj*3sD4MAHeL
z2<>7gl*)dFbUgohT36QJ*_oaG#U-{N{PP5esg$g!z&C7Ktt~w9xgBD(2Yw4P&I#yU
zZOUbf@j6aj(cJE6_uslUbj|HFl^aV&s>G6YoJZ2y;DXvkihK%Ft-+qT8*?*p)nEaH
zVGi676%KLGk0vp@17es{RS#oveI8eE$uFqs+jS)T@+V*UBHzdz=l8nnU-UaY9O+4T
zo3{D|%Ap&u*$OtlOITo@S6vV5jaRfz?HKcP7+V?%zAX3kIc~ZOockhpfR$*whpR){
z*{@vl65LfDjUDdk&O6aU1~@P|cYe!^0NA=yCpxs~tW~yiRMYdoi;$gr?WtNV*^gZ}
zd)UBF%o>o?OB+|VzPY7|V%KO_G}FIwd;;|O%h#$Rf(cPlpyC*|v%;td`Mk>y4J1Go
zeF!>wprba5F64V>g&z6}7cQHcB@zN(cZH2aA7t3Z0`-$Bspuy3x5oMHK-&Di7zARM
zfe<3twh;%B4kXgxo#$M`ZGr3>{h?3kn9NBXZ=!crk(2&UQBJ4;LdfJV2W!_Vl1B+<
z{>-Q-u|q6<L&XnYMo__kJqR@qbL}>qMLhHVX&FG6Av5WI+A7@VRuo{rJZbKU{I3xz
zC)tAX^udU3K*Q(}Dow(lrjQsd9@Hn*x<XkLx=@SMj+hvK+HU7!D2SP_0g@0B9{K+E
zc}d_PG}*{QM?iVy{kOYqfvo@sOaYXLK~Ll?ia$mmPLg3H>>sHFm^}aBRQPMiYjTGD
zSbPm_xDG`P`N*o<--kNV8E!#W0oH41^}Hk1fWKBGM|n^)DKRbP2joEaUz7eZEFi21
zq*=O#+)|C=e^&iOQYe61IvO*OLk@NP`$~FnUijTr+O~Ko%KsV(T{^AuD+~?%45I#H
z6WDgwZa~D13pZ*5Z}?RFzag^>vhaO3^K?RgrbnmzXF8tNkMSGJZ`6B?dP$JG@}cU~
zExJaA|3*41qOZuBxW{_#jJZ@U`z4P<v9L01`lllJzcFGYz+@He;0{J=W%e8D{CR1R
z%;(rCWlP7*2pH9^8s3sK<Wj+Z&%=Qqr6322j6iIuktb6;<m``$LY(dv@Vf`Pye*U6
zPiy9$*HDx-|GuOHeG8#arcYAQEUjW%BGpIl9r4e<zi7Ut%>V9#UYwg$Tq}zCK-Bkd
zYBOZ$!wlO{*fafArT*)?IfctMtLSHdNyNU?UmGIzcoWSmshL!SgqZd(CpuBa{+kt`
zBd{r9fyIISbE@sGt$-9;cXna4zh8q;2Rh^HkMW($iAI#qWPcv@l96PNN;@JAetQf#
zqc;t8oIsG(-!kSDsmqnOSHa5fFOi2n_KWxDR_Fx0dY)7lgNd&PYrIDtuSO4lY6W^8
z&pvgphocblw?4Pz`o3oQ*p<Y-9No3$YBx}vC0m1uRohd!5v4ZRqK%Z2T}|9n+NVc;
ze4JMnE;AN#B*g<g50m_A?>YHdrwnOPz+*R?p{b09$OjL;;c$q}xY=&7^P4zJN!`Y7
z55%TX)tHA+1P}1j)GaOXbtPuZEKyp3j6PhHJPKL{Py?$*xqHznBxihQPMKaoxVf8d
zN<mAOt2i{k#gghz&-FaKL+~gzmb+EsNlAdbW*fDmP)jGNLdC2`wFm>a;kb3A7T>#1
zk7<JDjYWgveThtcRe|SGgMx8fd`2{j&!qE<-Szh+X?7e$)oZT}A7!DT&q`rSZ(k{#
zHN}@Aoha;Rr{1LHKBcyAaPg`eAaOcdy?_;`9oz-Lg-%4wRrHl#E4rq}bUoUIQbSDc
z=5k%|b~<f2czUaQ#dA0E`#~(WPezcZjeohQ8K1O?AvZCQNC19S436`FyTd1B1%990
z5sKVVVWrS38DJf=2K!RL_y~N_N2rKrzA;@jHPfAzfba8-LZ#gcFB=_thLrb{`xEtj
zv#`alMwE#>P)APOQl<JD=K*203oh}2)rq6&T70v-$S^Iqum%WY#i?Z*_V^Zil$#}+
zJ~WlMMLZrww&5_kC4cSJQ5GMVdFvl7S!CMDNCfc%<KXFsOO;G$iHuH1iIBa6nP@W0
z4vdk-!?aF`lGjVxQoq_mz+}XXtX2Apo#W8D0@op&i#H0X;_g}5KiYyOM&zkkEu@m4
z7Z|x~!<MeAoWK@s$2783{msr+6(tfui{!?PIjhZ(WZZFy+trl08!Usw4g{-$QfM3@
zV3I3|Hma}z4Av5C0t=Q(co^T*fa&yaWBd(##}4FQTp7ON0+-j=oF8H4B&lSfMke&G
zeZ9wrq7ha1=J%1aOZXgIot|`izwwN3YQ(=vI7O_kj<<+hqT!5hnwvb#U$M*m&5(b(
zs<d6$`W|a1F9^Wy<;|C+vc38`C4WqAJ$$LrA&8{9p=?{_A8f`r&y@q`W3LeetjJ^s
z*<UB2jjw}D!Fwf&bt#zb_t@Q5ViW`^(`y&T?o2KXBI|X)FU&Tv455yp_1L+^U=3P_
zYOx%j6qW2I_pK6a8XYEy`>?b!cft83BYdoin<ff`OTpuN8ABrIErba`AQ)9rBdChl
z*ep9BdJphC89k!Zms=VxEepZE8d4vj8A(bvpm+{xN1ZL<cad0o9xb@9R11i7=&Kbr
z9glfmJQRhTFtThMPIZ&L6wkK9QKhSA!<>`zI{`QH{Ucmu$^8WOlgjKq52dkqun|@R
zsNuaAA%?5|c!jIOH-yXW&FMB(d+M^mr{zk#eDV6-JOV${)klSr<j#PZ4?EJOw82+`
zLIc|?+IYX0z}RPYa=R7iug<5^9tFszE=J_har)7Fo80cqPPdr89v@g%vIPAPbLiWw
zY5hWvQF@>FzIcC0R~z59Frj(gd}HOKm5S1=o!ebH_|=ucB;uR@yTYFFeTc#1DvrP2
zX;souV&=+=%*Jif+}H<sa^B<j1aHuLDNaf5`$igWKMH~0*o8ps_7nVPA3jU!kram%
zr{Soh;kS$E?T78Dubxw+g;guje6A><T~G1ZFAyx}LsveI_9E=>7pCZEt9?Hi61Sx7
z1}+5(ncQ!!N$GH=_UI&*>zt?q(F<nX5ACDgF5&RoN>e3@v&;pgfl(>4Ywiq1&h*Ol
zS|jxuEMds-m$50ejc04ZOvu);duxDo`qH2YUqsglg0EgfhP5|aq5R2-CzAo0by1Ab
z;kNGSn8Vi*BVlvMeEAm(e(h_Lst<-l{tkT8)rmOKC(AxZ@hEbbJ$zGTCv*Tob8zc6
z(eypJ?|T8grN64fvMsZ$GkFI_9{x*BAZ=!I;xWfBiQ-Ao2e&xcOvdPrH~Ks)BjOi1
z)t38^bCqyN@?3ZKTyZ)gma#9%x^jKT@hR<3gm~6=#xX6(Q+F27hAq60-L^KmPY5LF
z_@+aPZ$WIQ0lFPj@&Nxa`>sIt#NHm2tlxKV&9`n&J56=k`v>V9x?+-_>Q7&|Z?w+(
z1bW8>*4&Y8W48`+K$%R7ewIyb*U7ycVLOFonAx_}$nf>fpDefOvia&HlBH5Z_KGRA
z5H#LsW|H<cD?_Csa_f1bEj32o*|gC1i*It6)!uLLi=JYRi;%D@9z63jH)--M(lKs3
zwK`(I9=%?3`d)N?S3)tn?A>Fi)_H?Q$vTest$n_D<hVNP$TK0TNo(=mN8c3pBQ5$;
zSxc3|EQF<Z%gxc{fS8<=6y<6PLWe`2&siIL1<E_MlP;O!lzV*J@G;l}uLa=dzJaP*
zcI?_e+oenAMV~nl#n%QIdnR=;YM^~Z6Zqv@-{hC2n^!cr;Z^K@^hC$ivdaCWfuv}5
z)ozOhBj-aXm4MNt^q4HBAjZH0QqGAS(IwlkjZFVZ%U(=mxjLYH#C+Luy^=>r&T4Bl
zx9F!I<aVd}RImmSxKJ}IcPB<~_4*Om$Zat7CHM$x{kRL&XVW{@C_8WTm-{o45v!i5
zsh&!%GDe(A(|+fN*?dpq@ppgB1a2p@)8A|UeHsM5e0`k+QwRub6uWI#HVR`4!q8NP
zR?2YZaMaqrX!Wb(To#_OQP{6%>Z+JevjCY`dS5(mi?Ylsan+);Bri=LSa#C4eVF%*
z`CoQqo|oVa#~EN8f*}l7TL5y_Zr-%zSXbIosvpn0Xcc#MU2*U<emj6Hwb{q>GANZb
z_$uvz$>@g1t<)kQcBR`!n#ruq0SDGPV&xCl{^pLX)$#c~{EAl7v75!4#e0Ah-}_c1
z32#zJpC~G0nEbYVoBh1`0$q#MQs_l1r<Av+=WI>0X0tHH*m(EcWQvwIZVq^e`S9N_
z8E;GkhAiI9-?OfmO_~Ud$9@Dg&f>6>`5x?*>q6rwRnwJfnD>KlHIw*LWmWjZFPHr2
z6&oTa)=rM;fz_mEJ7p9ECQIvQ;UC@<GrH$_W?{jmnq2usR%#KH_?_G6dcFs8?+l)3
zg^zIK>*}E&e4221W8osBS4U739Y_bAEk5i5fKvT+iCrxvP8F;rtf-)ei~ebjF2i>b
zlID|hX<pt}vGVk_;wcsQ{Gi$HqXrTzE0_JWH1fvta)mB-Ro<kYu;+<lRMsd32)OK0
zs!+|FMI;AJ6lPrba!V>{1D+c>F)T~0=GsKT7`w5$Gu6z_OF}GnH`5b=8>tdUhC6EY
z9Hoy?%)95NTMs83grR)q<0{e({0}nq#<t8%E@v#KkKOVU+#Ug##rPlV&~5Noqi8^o
z@Y@Gr4eJXm=4I~KK6IgEtj5dQ!<EX7r(-tnnhMPJ<eC~|^c+cXyPtV}k#dvWx(o#g
z6C$=|_+j=ZU_l<Ja>YP-4uRpZCO0DK1LAx=XQSL#RZ#5VeLnlGN^LJB<qqz<vZ2|d
z<9ij@dSEGRlGX_}-s|4zd01v+*b>=_kcnc=4iKdZt(j6oR4l5a6yvq4tulnUG9gO%
zbUC)3S(Qw9ue<A(y1Hssi%}9?E#$q%GUn6!1naJdnjnc#6y+*9z?yrZQgu3OnSW~s
z{?3Lxo#lKvzA9)v^Myi1>&(1`D+~ZTi0W#eJX?#;YN_)0iKlRTpZCgv;8>+f$|ibL
zCAwG=5CWIbTw8pT4ppv`E-tO14v$czHx<(#(kdlehrE8ruIHtf7-&Euq?$IH=cxLs
z9AAS?eB`69-?|t}#9RsuXk{HZl_wjkqJG$L<~dtyD70Ls)^Lafmqr9bQaCt%1s2SK
zNPH~Wx$y>OKMPSfwXJ8`e{<rVm#Fht#>@2k=Bwb=)Hr1g!G%dM#n$1iPt6W{mxUxS
z79PT@&@EH+8^x^WtL(BlU3Tu8NsrF7#(|ST1Mm{{Ga6f@njO8Jf}pr^kEm6foGr$-
zi@3#;(R7Grg@!ITE&G=`jxJoQw#oEDhr!k1yGMZ*rpG7G5P91cY6e$3Iw#Yd?4l=u
znpIwZ`G(hq72M`K^2EvAJyE;Lw|?fGdgD4CK41IJT8GzLwYyGBBqmx4N%K%@W%&_J
zAw*F@KwDwnR`Q%9?o=1l#KqB1fNhcT2Um?2fXuww%BEqxb*J#BHCz*iVGcm8Y{C6g
ztj4tuLyCCNTxwo8iGKD2@y?-#o7Wub1kaTF9gSQ$=QTq1jd-lO06l#b9{7z)Bz>m)
zb%5z!Kd&1B15B_qfL-O34#g``FZUY|)Ak<mh84}_SOht4?`60+>1diwj-0l_uv|=<
zsR0JV_ux>$wr5LY8L|meivsN%0rrCc<y>7L(E>#DNXo<4DywO36p@U+I<ivBApgLq
z9yAr;XHfi@rw<+v0r*lQ4d#Iq?LQ<8gyc<A%Bka`&tJ=>z)7Ma|A)p>nW^y9nHEWP
z<BIn>b9nv__&H`Bj3{rIwn{@ODW>h0WcM}y#jd>rvZ;%W&Rh!TqV({)e*vh@*PW7i
zQ3@o{TE^<XZIY1(>%Mi%cp4gP|H_doXH1X2CY&Vcv^WGdKj@O9JU|+ZB<_{sTsyqu
zMrJuv`DUHkm{!I6;{$twVa|l`reQH<;U<}iD1cDs*rPvw%CMalSZ_?spYP5R>IjI=
zGhDCo`Pq}LdfW@cJ-GFEMy3d9K!G!gimJdrqq0btn)dxVh?;Us^4uAl0im~q6~c47
zSGWi)UTKV_POpCs2uR2~4mxJWZZN>Mg-g;Vjbkfi``ps{n&(tKER%ARC}QWD*FAf*
zGwU5^fogdh_BBre7PQe2wG8CB(;r5v`lM5nksh^&TuBK*nwedMkYcOzvyEu4dW+vE
zUM7xS&s{2W@|_PNi)MFt(d^+awK7Mwi$Tux8u069C?|*Z!6%lHu<{S0&9z7RIPF1z
z*$6hGHjM#<31T`4<iexWUni3=dD55iZ5dc?)N{8ur7|dNDHZCZ(4rW9bD$~D@bTaw
z3=Iv9T&c}Pi>KNW_ql+0Sm!Wf)Im#gE{SY?g;&keQh}<s+01lQAbTvat72=_4Z|zM
zYDenjADGU74<?RE-jmdwH`EbP7hZ)~IL?Qn$ntO+t0($LNQm+R2$Lc-a^=Z-AA>s?
zs})55<92F1s4*{&Tv~qM4YJL+a8F+SbppuX{<$N_h<M;W|BoOj9#FPSBF~yq77D$}
z<yY9Jk}Uk^Kv>~LUs3j<as%LZ#i$VK{xJ}UlMx==C<58JLd@X*eA(;D#SDIDW5ta)
z`(KA9{mPnD^pb@Lo7Di$sDJBVCbFm&ehm$Sw649vmg4Ww90qqvEP}3-ul#=+0`WHR
zxAa_Bd%Lp0a;pF1dNMq?@l2NG|1l!+yon1tDj9tqGlu`?{J?JKunTFfae%MV$o%*E
z!-4qNS=<4b$-f7^{|1v>?!k@blSBb*_jhnW&WJdJR<iK%Kl)@OtO&@urGDu8{MU<0
zzw!)Ws+c37KW*>E6S?ajaYc`HmB8>*x?1o^!AS1<i8p-nA0?6u@`orIZ$$ZPLjN39
zwr1w_`tX^2Y+v&uU*hLBJY4t-i5oJO)OLdHj4ih|zK>&<R+-aD-97SC!-a^u(<v8}
z^=dfXlOV({KLM<BgJbrn0ey=7-F^_`-RX@E(tOPlwJIc}&mk~7wdI;vr>mXOzb-M4
zRjXuw2-Qj>PPTc>`ee%g=GAj()a|iN+n2%}92>4NuJ$66&HBQ#Lr4WKERyYJd+3Vh
zatfA6J?6|kmS0fdIqDV+jrOh`o#B>(r@g!E$S8D2=aM3lSAwJ+xO#k}&+4;cU3B^A
zy;&<j6`!F4!PL|x=py}vBs9jLtPV-Uk3h!y!zm$d(x$OuQ?yVYtme*Id0U6CuiVy$
z<<^#*L7-Po|9xQ1cFMKIF|OX7C%y5Ul!^qSsYlqOz^Z?-LTuu&`c(zW(fO%M;!ADu
zE{QC`BPm|4*b{WGCx!wU(9`>xhelOI1)pdYz(vr)zEk^qI&Z%{7>toyEP|A(2C|>2
zM5la$K<IIv5)>oL(DTEyYu%q;L(76(q@@LS$6bb^Q7bd%=vk;F@~YdL@)H_t1dn?1
zMa?^~`Sf8B@b)X8W;++5RI<oBb0=KeTe&OX>X+_!mSBP*`Q6JJ9bs+PJombgdh5ls
z<!QjY^+ueaF75W<^_}c`s$=`ZymCLsc`uHLK#s&P7k3<0loy<~BNo*>0>^>vxarhl
z#VK!!>_ie?F%FW4+ob83(c0|==4+qH;witl<bXv~3sru8Yw@pVCp+I$G5m8ylsvDJ
ziQ^Ap*DpqL+#r)H)~BSt>)dWdW=ONuaCFpPhR0|jDNq#1a=4;+bc4l|XE+t`$Kuzi
z6zWOEzronUe<z>VdSQ}gZ3rG7-hEM4HeA=jd;;fPMrgDRT+(C6(=;V`y`~8fEgQwW
z<7+0?S%Pr42~!uH_`o~&mV;_MaD2ZZdX6v~iFKCSk?zEOh3`6dQZ8ALyKTE;&IMT~
zy;vC>C!nZ%49y`m<$Bs1NiT7!6t}j0S+Q22SUXji3Vn}FPS?10Q`x;*;@O^0g`~A7
z84w3e3<!{Xc<6JYkETyLzV$pbwX1Dogm{s49(iSX_k+dn3TW%f8hO#R27t3!m~Q@x
zYrmSoWKYL&oFjQVT&+d_eQIo@TmK`~Z|Wfafw`E`1M*v%swj@D+xA=Y)XIc8)6JLy
zW{VZU9mL!4<B8SS3oT?3$zM)2{3fBP1Gpt>03r-WvltI;n-6?F-ocA?#;bKSFm-0&
z(wZ)zD{=^E9MQlL1e~so?LS~P@V>D8ra?-i(0f~`DSQn+PgHPtC=HZnV=d04P7=H|
zA8687U4!XTCsqX1S-Ss15cp27QrluM^|qPiYqo(;!KB)DdA!CjS7Vi#xO1&nA9H`O
zg5XDJ_x2T?pfhHz6ho1bo(JkGw5CMD2C_nSpK%T<7C&8Xzd-#Hk8(Wc&$MusYLJDz
z0GOf)$87bBLcOZK;wGzz?iX*s=dSO_<xuysF}Sk*mav4mMq;n^b2Oj*?>li~^aIz=
z2J;!p^;o-@lVw74wiMIUI6Ojbbx8^Mg)4iq4i7JP(PHV+jiE;Nnsvd%1`yBHz1|#a
zi8CDcvXKkyUP<ywFs<(MCzS*)m6xh+_M&hAUd{X`)>b=)%)tN2j4T&bQQSVBJ~UU#
z3YeO!WKSzW#gi*sP;=7%J&g3#=`5v}xER~bsVw?!S)M#9Z4};{&aUl2NYCe#y7wC2
z`T@S;a!GR}qa|uvrqQ_!S%EBxb0Kx%<QCQSh{lq%J7;uark*x&%2%ZvH(E?zgV*%S
z6ACVV3=2(L5DT{?w?~3YTYC3lgeXJ`G}_muwUW$+8DlyUnsAX3wEz+l$or-}?W>>g
z_$1nVUn9NbXAIxJ+CuIt@FucJnn@uIvX#Zdt|fL?>Rxu`OqXM|zrmy0)U#5UouyjA
z#WM^hl~#LflZ8bFGVq6@)3vLW?vO@(6KH+-fQbJHZmlrrHh|;@BPg>SGw<b5b84IH
z-dsvS!=1X}R6L1@x<GSBR@VzG+x3_zXD|Q2@qI<BR7u>xLlCrxL*G#5cNw}EYg%`w
zweO*+)_;-H)7R3U98(a-{jd=bj-mQd_`v6&!?~~Y7IAX)HZTRtmm&6CX>1j!E=!=n
z9ETu2K}xc8+(3;<_t?~$)RfsAZRI&A{L-rL;Wv-!rrwg4PQ05^Z@^(!fnv;@UFOFI
zs^6aO^=`K~p4lD7K>l;j1<}H#?hzo7qF+MQ#QXN;;iQPsV)>%K&HAc$$bOEnicdzL
zBbsPa&d6Pu{R;pY4=vq6j8EX}4ts6k{*A0}Hdzag8q0CtL(2YP@5=?h@b0mYXJVhy
z`|~@DZ;EFag#gxgVnZ%xyC=In{A<iGJsu>tTf&*P8=_(zSrr`KceA<`4nJCwp^u3T
zm!eUpCNDH$OYgUDyj}|C@}8{p2ZZq<9aH(yN1PebxL9!gE}a3#*2%S%MtJAnwpCk8
z`3D-~vaatUCPZDj(y-SGnT%uC^Pu3&1dOUY!#JMZdPzWmQWUpk_3_P=;Rbs@arZUq
z4NqrM3EY;T;4ifgZSKuCvC^zv7atxK#t9tu@U#c<3PaU*lGyC3Nq411$g{t_^gM6j
zjUrtgoa1qu0#;s}plrR_)Hi)~I6WaNM%XIWt}ZfE%e9)abk6RmZab~?Yg6x@oqaku
zf=*s;58l&{f5SYAL=IivPOs>AZuaqdIlJ4NX4OXOcj8XYS4bV-Iq!TGUPZ=~ZR4N1
z7L9$*XGj^rAn7tPY(!2d)iHU0rvFwrzmwGV;w*8YF%^fh_%<*^Lhi^pbnLk(PPSL(
z)|LLzR#o)yZ2R)`G{%rx%N<HnRXi9mV9l9<E43++xxaC8j3{6ZBI5TJ+JCDfH(DCg
zOx+pnb$J{AFiSm8X&a(Q&$+86jXNk~_#2ii+l`v&#@BE=zed^i!adfiVK#Cv!Yjlc
zUIoBXhAbX`9W)t!h%SH2oh7jpLWJVDxrkT!b?D{Mw73d;udnS&-#tON@Kc4;%L|Q)
z986a1jmj{-INYa&>6?9*Q~b|AAW`@O@B!Jn&WW82udQIoa=LR5_9dJ2%F({A%g+$4
z-Y+|=aff`9{tcID(rft<$i9H>x0+JjiIV~bC8IJ0d(6^Ffjy|hh4^2n5dBpF>gOR}
zqzx8T54^8LTz@?}bJ99i))=vIGid}IEwbPDuNPX_J?}Y-`@M2@q<fkQ_r6wnS`H)h
zpT!KJ>-j?`QVUw~t`$Y?Oki|v{~)Qd;4H`Gcln{~Ja|1ZAB}$^DHtZ8DG`Gxa6zH>
z=+rqX0I=%EeQdE=Ugys_BkVpSl%@&LOvPPoN?w0anG&$?VzDC^34t?ze+HOPa~VcK
zl3Y$}6)L=?2dCf1=lSmP$&-h_G}ir=B-T6!0nn47e*FqhGvW6`#X3`LHA#Ne{~K)<
zcjCZF<BtLkV|EVJ-jRX#uC9Z?2MK0-g*5Y|l9eezyFoqjZ%<#cJ{mwys5B*=AFVGx
zt|rQDaph%67+D3`Ub!1TnEVBr?6$WI$xOZlk9dVWc4%wf&bhzVEJz#t2p5#0mzZ~@
z$9=#epyf|^=q|_Qa{3NvbC3VA+pFL4vxlC>EqPMe_Bew*>Nr*W#ap$)dsFxe>~mNu
zP2-RA`4VmB-C=3t3;j*UkD;y-^Bwvg@l=rzZoN9Mg%TXko%dpTj?3#N{hM{?WRz^0
zN@|BVM-vWh1l+B#b**MpE#>8!Zv+xQojWdm^2DcJz#GiJPT-CyE{F$H_P!+yN38Ys
zBN1HAT$6&3z-rLx-DEBfF}0kTXl5*nbdWdSs-k%KPA8YIe-lh}%4=0XSqX>GWb(y)
z>(csOlyVHf<w@#32b?^b_UgnGn8m9x;i~i9(c54S{iy0P*UczR1_2E#NA_poiUX}1
zx}NRM;<R#?NheAm7*hu7A$(9>Fc~;)C=$MtNBo`&JXaU)3!yl#D)S1*O7&DrBUXv_
zPq+8?`v70i@an`MdrvvWrZz;L4=A5x8)>p<{hGU<ldN)95`$M^)!SFO+bdOu<~|j$
zq6{QooL@B}i~924wwAsYD5`F4nQuo55H{k?tH8bfUXgx$jnKQ}<ob~vQ;>S|rg~O-
zPll@`tgtXx0<pDPMx?|>nP^=)s-?RbcEB{gDtA84amC|>SvVeAK%;`~idVs7xKSy(
zEO_UmR{COHlo_Xu_Y%2exJv7i_Ynjyw9l}qT9)TZ1luYt8r;m)(_X}lsdSf)xc`#b
z5pO><PDZpY)lr7OJ@Ii8n6w5pMt;5cvI)%T!WADs?SWVtD0N@>K4Q_krygDz+co-e
zXBWjDRC^uy@U->ddAK-k&~^izrsK(=?`AoX&d@%V+bnDM>H`wHa1US6tiq1T*0aOz
z-KJ1!V>YEb8c~4-wrJl66Ud*f9WAI6%FgI?87jNsfY{Y8A180Fm^&@|eJ-f;2bL&K
zC5s`G_Vj@!dINWF`7x}!pqQ7yJ?n$JeW0fOSIzeDE|nq!TQ%@BzDE!e{PUPaR{J#G
z456$Tv%NjGUkfE{HKrNT2OG?(1RS_Mg$TK0pr9VRllP@Ybv4Ij?w$`z!Od{uJ|t1x
zcc-_<*e*;JhXPw{UPB?&Of9j7pqqQK3lb|l&Uh}BJ1qoL&T3O51=zRKgX>R+4}(4e
zd$yfv^>G`ZUHfP9svQunCf5heR5r^gf{F6(%ns8itSSTS#=9zC;PFRR2l7%KAHSTJ
zYmQGRkca6A1$_xp{x#fGBMf%8Kq27CwAyQ+Hnnb`CR{Z9Qxo`i%XCS*V^(DLdj|k2
zlaN8Dscr?&_SCPlF<^ro*T{Y9qz(>?TCn!FduL2%Pxt%DsnzV6UT*@%%W!P%T~pCO
zA#9-&>xxw1h63|tVtUx(vy7kL$XM2g{QVAr=4=&>uBsV!l(a?(yla74({-;*(~KgN
z&&1CLlVP)ayiu~^H{&QQugIa}<az_vD!*Gk?WPH1?|VKK)RTCJ8jZ|RWW(2HGHEbX
zeyCFS2KTC^*byVDqwS<>UB0=bWA{>tBP0bf)>(cbm156EmB68_paZ>NEyL1)+f;&U
zE9<&dGg{{SLb-LO66vwMP=E>9_|Af_YKxm^aJj&{f9!mX-BCq<8qyL?V2=H_J9q4U
zmw*eup%s-o>Y+|h36+Z`B_16Cg%BnC(zjA6&P$Y}395}<YkBpgFPFE39#^KfM{SAS
z^C73DTh9i#i>cDpd>aZtmzLhcEAV*`sW?0>D5|whn~J$LeiU2U8yxEBl_F^s(Z{z?
z2koji&ZpB2V^av>TBm<4)f*YZ-`+a|ZRn_gNu-7-`v$IOl}@txb-NbhBF}hwwPe2I
zG<;3AR37i3=YX$wL>a7%bvKD#x45uh_1w}dh>s?E3%_Zm$z4MDP1YhL?GbNHqxF^X
zKCJMUyCz<34B3w#39>E=U+TkH5gWW=AXu13ePV2HlH$_s3ckkE-#$Q6z_HlkcVw;l
zzdtc``doaMu_w<z2-MroN74vv1O9u@uc?ZlMX6HJBHcUZUztmjXVnA@=};%j4hA%q
z$9VQ0>kBIN<PM}nNahW#q62C}+toGX^f@M1MbP2$1zgFdx(94HMpLecg5*;FkEnNG
z(xmCSw%eSxZQHhO+qOOJ?rBZiwr$(CZF}19dEHOM_vIg`tcc32%)Qq-)(1OcZpoXY
zn9J)W!~FwIob%Un4}4Aserv0|6IUFD;pU51m4eExX(&unTLn$YP+`u;FPrs*ya+~S
z=PKzl9X@_H?d&7RFsDbC*Lh}YS!jx=(V3LX2CDMgz6yb{2RleUGwy;*KVFVKI{eEY
z!sbu2BVkI3l+wn?Uz=!zblWz`R%?4tpPm;>Dj;HP#N$*H<wy#U9!_kY`ZSg8UJ#W5
zYV%_gL2QLkz9O(Lh*8(Uw}kA-jp$R2@+4#vD}t&?7MYQ_X+09#rAm*+;y<f#6)k*M
ze6cgp^xG+`50w*~6*yeC0x_c=uEzjrv)V!U90I}b;(TO(jUBNYooptEOsZW!Mc8-d
zDf31f;?ZIf!V0|XL6}>cl}fmgb{{7km+|G;v{X5FGEqEm1H(pSUI-LU`Fc{z(Z-X#
zDLJ)OM3K~t(C(Nmh|bfxjz!ybHjWlX86ygZOk`Lt^Z;Smk2j+=$Q&Z3)C*D0MinX9
zR}xAaobaXw3C;AG`1o`6(WlW~YaST6VMV<5-^L%f_5;(Q77!nyV3qQD`E_&DjpFWk
zt(#2PF}4Ihz_mZtPfs$_)T1v1+*eL(2E8&kD>X`=HdpNBE}2}ZM!mcqXzO7gwf56E
z0<&f(r4Ee&P7>H^v7}Y@GWH$&C(5}kG@D^UXqdrl!NUeUBi3#P_HMc28W7>JB?;xC
zDqH$$r)=!rbXyhpfQzN--SiFH{PDZ~lzu~s7p%W&Um%^U6_SjkqS(gs)wEtVaq%?b
zL@J<~7VcC!sB_*X^Dl-o8bc5T3tgZ#f*Y@Cr_~&rF%?mpa5a8avjc`FLHp3?$zUhH
z>5|}CJ6*m^ZhPmFrbFiWm)nNpR;Nh*i{w9VT-xjc>K)9f#L<CCg1xQ|xG>$$)rVp&
zR%bqGq=cpqwchP|=x+z2oO7-G3MGVC`hq&QG?nFB8UP)-Hc(*H)8bd`2Xfqo<q}ly
z49B2MZTU+{SD4z|p(}w=Kt`0hzgUKt1-*|tgOQCm5a7z6pF&Lnk-US3=oD1gM8!$0
zKz0dhmfXKj&wD9*oVn?jp>7Z?^rax-u1Gf!r778Ld_b?*)^u><n7XZ{EOR<n)7*Df
zZ1Iw7Z7FvS4W=4NNJ*$@<qpaHV@R!DBh|qcl2+F+s<;4FECfxU(kxApbe~Z4m;JCt
zQQ1yB|AW<s<U;=m-zF*%w8oGgfz?RGYoA;a?U))fTYUgWEX5LK;gUsaN?JHkL+#Ji
zG*?GdjE=`&>l&d~c{nP1!5()Sv(5*g8t(%&>Xx1?EtYT767O0Kv-5j|%>F8`>-LxU
z)3!=~$MIp&?o4g2<)ChSCFM%HzwC)hJYI~|+9PHD2?BC9He~-Kd%!34M5i({o(_hM
z)Xq5r$W_dzOg0>7oG!o8dmY((b)5tV`2EQOkzZ9P84k)>b4!DSnEBj#krl=Ka(o-F
zMLE#~Gr2Uxo<lk8RA(y`j7IGgWRC8h-uYyjQAf?mgo%k^MuVv(>Byv;&eybXptG7`
zU*Vd&lh0bT?yf)nB1d#S22<C%igoVb_u?QoFczP<-T7%*G44B-7)i~9$b0Ph8zOcb
z0f2gVX^c8%u8_KRnH8q%J1J_#9zLv^s7o5XtGK^AcOq=oM$f*Y^_|R?#gC~Aa#L0}
zJ<u)NkY6e{1R(eo2de%c>rYbjKL+n-)-HcF4}1@}C3T9~xROU#5VdG8rguAYaOi2Y
zMidigdl3W-m9~U+J&C&*!`aVHQJmJB*}88XT(_M%SzdeJ`nbzW3}$D1jwCdO7iwh`
z6V?eT&7(1^zQ1KcL9_hLJfZHwY@3NtN@mf(g2-)xSS|_i)82&iH0~Q*f?oL(!j4<=
zy2PZdX3CtpePX#Rnm}zaqSv|H>J6^#6J^1^y578T@};|N^G%d==fZtekrRDT&au1K
zG6B5yxKtqI=HJ9PKwsFt7R7k}dB@SF+3P~EiISxnNv|tWMEjFkpBrPUl~}5oH?$b}
zJ|pRwgrt$>T~i|buyN?1Ubbgi>@sDY=igarweSstKQ%?hWY4tg|IIg<GG#8W_)DYr
z3kVHnqXbrSN)H~#-7PnesA9|ilRh;)E9WnoHrBk&>$&>@yH3|W)t_lknscjkOkx|y
zZP3|;>lH}VpBqe>0D|eYUi5nJ2}%E+20nyUgoG9mO33ZQ22yHqL^K?L48n_A)NDD<
zb_+FyE;PCd<<(&uMyR5+#ceWj(UR4u*))E&yi0ug;Ie?`=J$%-UYa%QP8-U$D49LF
zLKv_?zO`{gg^+z=!~n5I<u9D(hn3jMRO30{MDhUmN0hGNe&pX=UT9sed|Pe^9a@)3
zo|O>w&5yz~p4nmL+fsUk(gB`49Z~`DiN!C^J-Wq8&?$C;Ki9hz#{xhu<b91KFry03
zX)I4U;+bXw1I>I(&y_ZuRV}MFzsyziIYIf38&I&<{$2tCi)<$k_G$&{81wUfVw3eI
zOBwGaxdYKYBO{=Hr*VUoi&OAfQS?s7Ova(!PDC88rf-f72QgLTuTs0U;F&ZfzJ(h@
zM-CmVjS@U+Bp%SvzsR=S@E2k0lJrpbVPZ>rCQ}L{yVu`HF5Dk<IM8%@T)n`brwFTh
zvbvGbwyn^czr}jeet(03f$iPihSRad<1P#0JekeDQIe^<cIYa%Dp-8)Zyknh!r81>
z#Jattu(+OT9W?EafnXu&#lWCZ=eOb=%)a5&e|X(e5cUy7(Pamp)60^%Qj1)=?}bhD
zo7QSiWmz>!(eGuLE9{OZxyfjy1DL(bcbv-@`25Q~nq{gi5GFHB&mGsq0@Nz@#5n4;
z;ts6GuL+A5ep6q54#)c%0*0ylX%S<T*sdMv77THun$Qd)>e~A2i{;+Br*Ax^zdak4
zDiq3wbDy64rgXbt{rr2%<rUOn!W&}J<X@_)ef}K;pSInMWO_M#8wCwgBdS&7b3Aqa
zDRHAQbS+9WPks^k?_BX;W9i1ri%tQ#V$W1TBQkntb)aZ>@6`a1$pd@qP!Y1s(2Mr7
zx&A}BpFcF0_exasuZ)A+#Le@|i?<~A{Bb|0vWu-#zu2J`Y<!Ynzf|r{iBEPm>$!=x
zR<L3dO`G-i@#J2*PF}%9wT=9MlMf*gmW8T-tY1#BpY$B>I*-FPhFdH)-7=Gp$y^F*
zd4;Ha45MbH=)yzqeQi0I#xBQRZg-s_@GpO;9S@&?)pP1!5=(5H4>cq_9JLZvFU_5B
z_8Emko0ez)s#l2-%e>E9%O4FktFzvD&n<m!?OfNJV+7Rc_ZCyLN%=f^n1Jn-57F=6
zUIP+t>gF0{G-~cR&t3BrGD~d#`Qgj-dZjwxD$J2fTU|&QFq+)@SFv@tELY=o2UQq$
zdQngN3nc>3nzsYRaFt-IC4L|DpY4vcJR^Psh<@m;?BzrAWQlK9?;s07L3|@padc#a
zL3d>tVB8uk@0;}@7Oo^CZXxN3Blo8rv;$u<aVglkwN%~qR>KeTKM%{0gG{_$G%*Dr
zbymP6y?EmlEeb4e#hZME9cKA}WQ|436X}ytyTkBS{=xUX<dBkE!>c=j@u>zBMM1vf
zsJMiIfHno+zPDLjXn5G~v}Fg5AHJAgx9SM1t(1{{x3-gqnJF!%uwVhdKA##{`;~&g
z(~^4k1rFa%mayAVKhea$C^C&)%qcqfy`gU+Bg|iuKSq<1CoQpVjGPOatj`$!IrUk2
z8F-%G7vlK2l+#9flg!zdo?5Ww+YwNw)Xy7o`!3IOHC5-MH)6dH=SSO`wBK*oTt#La
z)|e-azK_;nzU(QZ9c)8Me$j6>%JaUX)$wc(wy}Y#Y*hWHV?;*%w^I>kQo$R~WopGF
zew!Q?&qG{Q-|mQmRdFs}(E_naB2!pA30y+m>7UCiNtz&^74vn@X67eD^`8HJFfF>X
zz2nK1S7?n+zgeqpDp-G|rn?JsAuE{Swa~;C*la?Joq2<AIVK{b+zM^B%vE`v_s*NI
z!TG8*m!~H8jOYd9IL6tA+HcM7Y2}E)c9d)H;Cvj|BkwnFH7Ty%y#PhDD?g(zr{s~}
z1tU`*em`jc)^^vR`&*GvS{oRq#~V8E^-s|C8E2s3dM9HQ3oP5AZ@w_Lb#Cb#+Td4k
zJ5Kks%(SB#Cb7{IW*VbDbiIi_q<JH<^R|QHsK&ChyM?EWo<z3(neamxg>_3HleD0`
z;sG=N<{^TZr&*TYtr)?%M7vu+g#BO?1Fy{7nL!%23wf;oJ6SC-JF-?M8pmz`te-<S
z_AO9oB)tOP3<)_(;&<9iL@NG+Ci^e*fC{&^l+jzC5SvQuN7|nCq7rhaHDy!xl*#gz
z(gfq#5&gcZr1W@UV`m6@y}(zMQ&Kv*P}Al6`wJL6xtgw=(m4G6!urNHpEDrxW9?+Z
z8i~dVw{l0;okc?%2}aF=w$BmBwVr(-6%BdAy@A{@XUK&*6O0ZHc1DkHriRB0kW{|Z
zq7bm?_V!Q-JUJ!T&SNSvTi58Uj%Xr=@5B4Zg@QcM6<fhX>}4j-?u~a*G)oG;H%uuk
zEKc9hLSdYofG7T}LVxp_3TQxo2D@@F*F0N$&mxY&_Oq@uF)JaYCW0Y$F<wg98Tm!m
zriXG|{gl(4_S|#347PC7?E0V0i?h4zcb0^?dj6^e%3F;OYdy#FKGiZ4CXY8S3%_5e
zz!J-yW-0Yjk<Tl;ru46Chy3^CR~AjP4U57O#?wooPcCH?C+Z*2Ld%VF?i=4Dqz(;)
zSMt{OacN%ZEH+4=9LlPVe1<qZl+o1y2E)%z8B9BkH|=WBR(JcUtH?>oNhi-q=@|R2
zHGx>8UOAKgHKMJb;1iniL1aWR7BWFVEK3G2qz|lKO=q+eO>3Rm496C;7kMolR=-Z`
zO6gZ(+<n@(Lh@qy=iP{j@8J{vpk_O8>n`q<Q@}4Zi^+*8LMEontss#n>7;Gf4?R;I
zyXbEdB7!c?#l&0t?3{_14W~yw1t&E&EeQom3caJNtE*u7(ne_|*rb=%+e0Rz&FeR&
z9M)Wl@WC6bg2k1tN4wT?R`zFOK)Ph`{k`sD=PS5N;RY>s_@I!vIr;wneuaej+pWC+
z9pTpUD9+P^n%%_DHCbtPaVR(@hy=V3xY!}N8NXa4ENJQjjtJI5lB;ZMtrDF%PgaJF
zl+-9Uj^%(b%_JJN>OTiq{ShMdnzga{D%-gEX=9jt;b!h8JMF`v57**)t8ti@64L|6
zb4D$YNUN2c6SX)U{Oxidt!6KTbpPh|F1P2y6fPTNnpIn}Q9S9#SUKqDDO*{SuMRqC
z%eAuY6K3mwDFjkc8l!7cEi`{lQrTWgp0flJV<y$>Xo&P8sbQ6xt@{=N{+*QON_89t
z$ofAteA>+L*M%PAv?P|;OP^v0Wsk^}AMsC=&`B*_EQMVV`5%?w8$K_{rCqS+Yh_#i
zL+dv1p}y3pF|I}aLN8Q;=EcibvtDQr<CC@(#w)MVX$p&;gTiKWkQWUMPwTK@qiGpQ
z%nyk$FeHt+X!AOJdzGqPN-GWb6|QA;yX;(P+!tg9cH3CsK527^;WNUKX~pYDRf>y^
zQFA++Fuoj_#^p_svdx?s3VZiWX;|y>X5)FG$VaFWyhg{f#I+k|=)b>AZD9INo3LT=
zzwmd>pJ%n652_F<?u8z0sWlsn{VK9bv|kEHAtal~Xf3cmatff*xj<s4H;|A%=5Dgm
z4F6pAaYjkD)Rp&5fy<uHol6^Os<1MBY_R*cY9rgY_UG-^cwok@Dvn1mR&M!F9Clv_
zt!%M^Z5c*u#P;f+2h}D@!kAi6d%5?@bLCO8hzGh(3Z03Ca!Lh;Y+4hhijtb;asiQ1
zPNe(#kloHu4+(vS?n=f4u$Zc&JaVOmJt-4ksBU<NN}~D-_FTRaoK-vnvb@=f_5UyV
znIr`SqY{rV{ymv)TkPSvf+tI}eFF%aJe|3_^th1}#MatD^}Uf+YQlC_g=TZ+>X?=)
zj?px9*ar*V7_AyEq}4+e1VYvAj_7D_56W@l)3byWLqQs=v%;}>&fvsnH)0KX^JV>-
zzPPgG1IRh?)Eijbz{2q`upvv~PR12V4=Wa9E%>pe;}FM9gb?pt-D_&+>auTl3q(8t
z;wlARv|$e0RD(#VvY>-<6%2n|NE83x2L9^8R{0#NzfX8d<d`&slYj!wUV(UDJjYqC
z!1lA2+^wQ<zTpGtw?%?is>^YV3`a*ot2Q%(>D#EkYRdfjVJPCrY^US4-gXSYG|ELL
z#h0Ci?Tn6Bs!2J8&lRg7^@7Qwz=b${P#<O<BTE|SSCzZckoQ%I`zs&<0Px2<Bh7YX
z_uC+oQ9|Bd)+-_G*F?0cIf-|>CJOziR}!I!C?Bm=#-)RSo0!7K*i5eNA5``=9!61b
zRY|TD5}^sswg@s7cCTO#O_W5-<=<$$_Xlhgx!~jgk=~!IgxSO}hc$WMoREhQK$G~K
zj_6qsm*)kSh>ij$H2@8}*+E{G%S&zRJbb;$Pk>5_iqj39_>!a4N^aaih?C&@dR^W}
z2!1m;3?TngAVu>50XX=ej=LGeG@IX>s?~da_5CExGFqx>*D0X}xJC<IpNgwGPhb<Q
zC=rQtG>$kz3*i2zlOoieIyiUz%4GHZ|KwFz_o^Ak=cbk`l=a;I6LvXDMV2-w|DQWg
zdgcElNB`OI0LKKL^x4IGR^!nBbE6#;%L*}11_AU(cyy)%C32aPl9GgceA#(<gaBxl
z)lyl!MY8k%39lUgPk811nR6u{$*BLwGK1TV8W5Fny4N3^lcqwR2%R~7f+UwrtyTcA
zlx}%nR)ax6Oo09;cM5^k<IRQ#cmN|EsdI*gL;<ODBN=QqmTPrH2>kyD-7c1hh>3#$
z(UR`h8^Z&ku<4vm#55W;N1+({c_}S7dR9SG*26c>g&ZG7RQ6xLT5(d@!gcNPl^U^v
zzgb1-N{5)-&pKrzIuhsOUdcCfrQZi_Fxj6}*x7xPkiO_gX9>;zbIi4Ia$YNf^M^a_
z+Vbp1^EhD&AklR^XFBwZ1E3z~yS)Orz7NpdKCdPE-QM{3s5m$>vpIZTfb2OhKx81V
zu{9ti7njp9FNvg-<)s><?Y=|D^C8@9xMXanH^bxIweY&yp~2AS;e1!ro_jp}V9kjV
zExh~Jd83LXcT*-?_@Yx=@vr|>Z5ZjE524hQbf!Z6>+XQ*<Q_{AM`Oe-TOR|kH^lt>
zIf0;%&WAA^1&hhHU*`Y}_-*e`&uKe60sj_DlScw>e+}K5?KJW6uD}~dYuSKXL%<j7
zw>;Bf?Q#6&FZ6#fccZrThD*t80dDw4b}N$w#(-xI42#wC12jQwVf?^qa}P-j<yZ1v
z1!9`r-fS+XGsOnI{;YNztN}jq^6;0dwHCWQeoHla066%@(Jde}5F0_@laavhDgLyT
zW6Mz_a&$EXkL3avm%AyhJ>a%<F=jdLc8Y&WpqGKDuZUv2>S0Q84lJp^0~~hUmlH0E
zDViIPdy$yDXs&+owaU1^LhlGnop41Uafue^dp~qoapk6Y%AhltHazE!yZU^+BgGWp
z=nHZ@2S{rBExZy0(<utLA{exrA^;X;$8EP&v<;7|HKw?`rLgPo&qsyXpyrLc>^K1N
zki}%=7dnIfRI-M?Rsm?cjLscr1-qNeYrVxeD!l#=s@h`%vDqZ6IsQEfGc&4Q1#Idf
zjjt{iWE$&{$q22UlyM4P=3-pMG|$3U?5I~y_$ziSecDiJVUNlC$d_qYI1&n`m||>T
z--h>eS<yBgWJ<Kf-JO9}qZh|HJl?+BX2VUGkLSrgO}tquK=~LFmAnX_a6WkYc!bk}
zl3RL6N<q<RG71lfpFM|`Q2cv#X1UQ~CliOySEgBCAYqB-R|9}hnE;3~fF|q_sfE&P
zr?kdRcQxN+i@o_xSX0+E7cHhh=H5ng+f4-xRI~Zv6b}8?*N4QX!->0bjaSUBfA)|8
zi2?b=95(qEXOLDowa|^c|BnrEz-WIr!S_%T!N}eFHQrt8v7rmiu$|(<`IvVY{QDoa
z{uoR~du$eS`J?=a(KKc=+``v#QFNsLJPiZGk)sc;<TjgcccIjXfS0h0-OUJ<!bzdI
zP%y98d;m`&mu^n?5Syn}%4qbGmY$HBMfsa0BD;Ed{%;tFy-Gi}S&V`9{_&qxYBn{W
zeX&UJm#>4z{<1vu=kuoez((ug%%}sYIE$rP<xF2EhVyHP=l9NE+1WF?t5}tZ^A;lB
zxkm8*iv<&(SHr^ss;X9QK5sgsLEMyL)x3{qzV<40RC40>&x#e_k7x1&cHE#Baw;Ie
zyv+nJc`+=q5akQFTsxFsQ&Uc&hyRmnQmnCOfq!N`Fs~gl*0f9x^pTmgZu0fW0VKG?
zH>)|>=(LZqJU5gpS^}h`Lx}|K?d=qNaWpPCEq^@bPDfyP>kau{pRYrAcYpRAKNjy-
z<j-2|hA(JrXHeZYXoo^p@}vUe`_j;_nf!VofAl(~nl;B|8|KTs+&=wMkLSMcMsVgn
zw-a8kh|7X9;TpqScB^LZID5|Z{vnCZ#j92;64e|4W=|n;DQE+n*xXXU`zJLBr1-2Q
z2&Cj}7rYhi^-H!DD@Hsne!!MJH}QiZaj<PCPlLPMQ;in>kfI(69LzH$k^RYi-7Gab
zF9`5$Ngs~H@cAojL>-=<YEZ_YR|syR<Kp^0pjZV;1yj!zeM~RCe|enzXh(`n#>VS(
zW3;NVHTZPvnKj|3YCIb&8n24OgOk9YWv_EKcd4%0x^uFnl9RQ$j)MPHiW@&#mfR38
z1YdwEwX5xUvbG&Df428155iWXKyT0>M1leA{?g)^a@EsAE4m!kOJz>-bY#cNL9Qhu
zu+xnB(B&@FIr5P8yvfnze^@P?rf;QKCavd({AhJrq7D4th93tkKbrc|S9M+F@^?pv
zhmp|Gj=k#TW21D#bEsoB7fL@wd*1hP29hJJes-QMEv6n(Bu>@eifUy6Wo_R^6CT(W
z^M%s<lPI&(-0`0R1V&^{ffS%c_aGYZI@=v{X780z6OP6b7g{3NUPnG}vOek+t1vb*
z?2$dn4!wDzA8A}G_2_(Qt!qa~V8Abg6FOs9N+(0chUSm9@&u24>qpl2iK$hyU&gZ3
zhXyQ$tI!{c|6azN$$Gg04TW6xE`*fwY*n@0+04G<yu??+3Ba=IPZfVm{*Tp_&F%KU
znVK3hO_IIU)+8`+Ip3hhVT}GY(tn}xDA_nxASUe>bmsSi=le4uI^xTT`N2jv#@iHW
z)##y|e>1(cJ*H4z1%P=hMF*UI_2qfIEEcFk`J~VrE-Qfw$+=qY7Yt+?Nc@<iq%+>5
z=+Ab9p(l%Q)!D;k#<ib+PZ-|J3;(g1pC0}MIOG+5zw8>3u(7@6v0oqQvL>iOVE^PL
zewCDIQqxy6#5m-dyHRsvU1mj;F03>yR>{hTKZe*LSLB%jxP=n}Kc9nR@g!J)SjO*v
zmF7+h{ABk41dG>$FszoM^PMt@c`#NK6RE=+3W*(eQ{ixQx@y-xxADUY=E~=yEIA<P
z2;J#TB-&GCmV#u2>7`We%x)c%zBgECc1uET_E_yC$0CP}&5M!cRKe)+fw7{}rh2;x
z0w+oHC24jKoA-XEl=J%)3nd`4uVfFXq0za_?-|r}p-I01tIz<b#E|9Se{aZ}dZ4&H
zeCO--KLd<*WdYj~cxd8n=V>8V*y$VZ8(7lIwvmVD_cJtQr_Sc7niu5ZVS9<kEvz)O
z4U8&|2|NPAZ0_}o0i)l$t9Uf#KD4{nZTD-A>%74<m#z-4=Te!{mf!1k&J)F;RqH<B
zGN&W!g*_ajXSayeY`N#<b?#c$<!r>7nS}U=M#|Bij%7;OZ^HP7f{9!us<|(G;<`2_
zEjL=jf`W|wd#CzAQ-?$W#kj}$V2X%>1fw$L8u&ZpU90Qzw7dzv6R&iqEY{~6qh-;Y
z&e}{XsubX%cO<H_+hm^|8yHq+$gbXRRgu$RjWJ(e<QEeG&yKmINvy8p40rNCf=y;n
z4Mabc8bCu8*BO<^*<1;`ZCZDW<A0}IuGDx<v+=9oBq13@nYEfqW5N$812D8wI&V5E
zmrQVE&0%Cv@W>O3CBLoED^Ofn{Dp{e6e362spnC(JkMqwh1V4aXk`85IlJ^D*<eRx
zcyF`WE0J@>KV`#w=QCQ&*l-%3RBhIxv4zCFgaBK^1Xih@%ZD(UXGSmx{2_7H9|;{m
zP_wMS*Aogu*BuCe;&eCUw17K-f`Y=w^1y0TtTmkD|86yl?J3MfZiSLaK5KCtc5RyR
z6&ZQn2iK}6GgyJH-W343YF{=GwX0y_xVfjo-$X)}UWmn|IZzP!E5TDc$gzH}VZBb<
zE=Q4}X9nFK^jpXq{$^(;wn~xSMltz`?=sROT&2{X4rKNm%?!EKi6Xs!*;0)~nE6a~
zII%LRY+nrExcN<uKOImGAhlL$H%~O4WScj90br4Q_gSn+#acy8Y&Y^Juw+S~3m*5+
z5)OAWuaA}+r2^H9EsQ6R4p@y&n2Q4KiG46is?Z1|YkuA)_mRDRGc=9vil@g8o~6wZ
z6oAvJqqFO*_6hBJ$vN4U_Sc^K-|DD5kyiIJ_iFQ<9qXoYz4+kCZ}ZrC=;S0%W{P1H
zk145Y?qw*>OFIYTN%U!wdgr)BR0Vrwcs#;kJke=tw6?UBNCuS<r`<ZeeyFk@qfTM?
zLu8{a)&IaRTBkJ@vpJ>~7Ct3P)<nOW(iFeHzaM_nLtHmmGOLu(7azFnGM$8GmRkx)
zKA%~5em)|k|HxWc<Iq(gGZ{iJ#JBxZ5dW$VuhhR*n1q2(iFu0_xPET`$>C?MyV)C6
zD*Wg3i1dl$zTf_hpnSWh%8&WO^K&y{dL8~FQK$hi*J>xG%sy*&lzO<HVUo9O7x$=p
zi*8z-J-%i&>8M&BYquzGE%1ZR(HIy%DA9Oj$i2*CpwRpAp~yoRN@Wl*Md3noe|<Mv
zMqrmBiyHESi=lN6xt?iKqS4wSs?$Wi9Jy^rD(u!m%(E*!{Q?0p?fv8r3|8hX+Q?02
zyVV!etflHz(SfXBVL+^@+IeKbzYS{9DCeS5w|tUcfstyUkq>o9W_}JhsKb_qpVWOD
za96|r;BB(wl97!M!)fTrlLb8gd0Wl9T_jNIebhI$5v?0F*YK0Y`<5=m-RcTrGMLU6
z|M9WHGPT%d$h|jG45+zXPUs(mV03rKeb#0-TukeKl5Kj>>K09KalS*iqcC&8bwSC$
zO(F358gA9{ECUHd=b^#4Cb~Q;<<&KVZS?=na8oVt-VWn!w80yM+#T=4iaAr~3&t|=
z8*6C}$NG65EW%%(F8G2_Q<6EAK%7C~Zd#qi#S*y|ccZjDt#9zV2cSUIk6-ZR7%h>S
zrp4i})`s5QGRK_L(*7-iW$10D#wu|QO@D+BJB8O@ewHnF@bGOck-D5}pVX{}3OSH*
zEedw__*ot03Ot7|*;S_XnDV8&Q<RKtxL(c&XBc@Ej9Ugx6;5R{VYoa3OjH_39y9tS
z-}PC4)w|mY{3#_`Y7F};u8%A)bH!KlSJA4+HhQwIJ5lD-8~?d@W8?W4$R}5cuFx9{
z=QQa)r6^uB-3XE_8}W3R@O2_rVP>~$(j}kIRX~Kd+M62uK|foNwUnieu@N<3T&ZMU
zPI^z&O!S2X{rDRi7%G!JSn)9Kip8`M%TS4;U(LU5-}VQ4gA};bL%ibU$KM;&AA+pt
zVIjRqV_}xyG%)6By(N}pfA#)bC-}{MY;p?~N%s2E;JDg`2hs5Uw83NaPVLqZ@VQ4v
z(kz^R^q<Opeyh+c=nd}rSTqVOS9urw%<j1T)=+W<c$rh5q+z58U4?JEJrGe9QI+Zy
z!FWJmq)}|40C(Op%DkWgLShh*g?N1wZ^BU0kf)dTZx8of8It^;Ka8gz`VTuR>pYDv
zvs>*hwRjuh6ozjJZ)FZq!w3O-g30cDO!h0zvN&w0*Vx!j<g*bJ<bDXFH&ziaNAOp;
zxIg%Q%6&#N;tR!-3&e$!RH^YP=_0{&Jj{(Xd$Q8#S2KHF@b9&+&w+6W&r~p=1C^>`
zZSz4<J}V~E`KVwk_7U3q5Lr8oNC9M;M4Yygd_LGtT=1TuszPZ!5uuzjx5KpoH=->P
zVoQ}Opc5h8njKC&3sWGh(!kE|P_R?uQguh;?Nneg^z^#pq8*_x#knK9JRZc8t<2af
zH6X{Kg=-xjh)ZZe`<L=VlY3Er_WSShETVQA;Urj+%XFJzf||TI-3$R34G<?tl!Qd(
zcnQ(*)m@yG5EIS>RiN2DMz9adc&~3x;{`g8)Z=scAcak*t=qdiq<nRv1w+8^&%P@m
zM5W#essc+2FS|k0V*KE5Sy6luuVVhsiEo~^#FPIKRgrH!*!703;A$yB|6ZV;5}zh4
zwt{paLvY^%WQJiQs-TVQ=!^n|VW(cCYBdrXo^(VNNeTSRJwBac|29h5Ce-0BZqTcG
zR}_3N{UC}0+;HJ`M`22)E;7DdSn#1ii@BvMA~L+m;RB1IUf{58xQ407FRVET7_`Y7
zMO5n%qsGBm3e1f=#4NdKiKvyB{p8#~*Q13*AsixrN=%m&UM^=nf|SKfzdjP9mGE5m
zE}`lp{S6J(!BM-+80%ZU!_N(rMdY5y)4#VoxY!_Gt7BIt8JK5{2Zwl=G4ni9d$UF&
zfn8zK8CYPXe5kogq*^FJNkbrda8Z(RT_ibci2>JR;+brP@PO`$yXX7-eiQPUtSMn(
zXZ9d)o;@AtX`Q7&B`V_6C9Jh|X)_K6FmF#mlczv!m)E-}oChyzSYxhJ$8e|E4?H#i
z+vkCKztrU3vrr1$YO@Bd@qSCkhsp0z8O-LG3HF{QP1HPaB5-|A-gXS8ST6}vO}qrn
zEu*P_?5@P~Xn@LAhY8Y;9e4PT0eS(EPM#2?RS_eK`OzQuql_ubrB^N23S^-9lHWR(
zW?)&`&@Ay38pc+MBN2JABuv##21P|#LmIcTqmMQodzH;m^K)aDF~NtsVT0CTn5rn(
zI;+75lVs9gfg5|!bhLG;NcIs{Mp40%@Xy5LnM1)5zt4?g;zkTo<&o?2iBOTS7?MHJ
z7pCD0DOiQAe#l^D)lnX{gAx1C{gH(3<1EAWk!*L#(qzC#=xesM%J9%<<KzW5XYA@t
zKG1!8HG=1`;pV$?wgj}+-=;V=wDgnJ1DDn?ZR1M$qbVbVE`b3`vP?-(d5-R+{vfxY
zkvdAz^jX52JNhB4PsMU)F;DDU{K<cEZ@kHj3@Hy@&ebi4Ly?F65{9exjyfH{;V1IL
z|NhxSJ^vZpFcUb`Rb>E!d>2gG!9jAz9&Xn{bJWlqIk2idteOwPyVpL$!7=P=!fQmI
z>F4?xWjt>em)yPpC*}T|SsmhF3Y2s}g!_5HNBfEu!{0o3mq<?_5l0}RDk)+*Kei}x
z9w(VC9Sd>WL{*Hx6Vw{+AMvMFekkH32Jz+tZ^0Ey6_qBwGmgn4Q(z~Z88g1e7n}fc
zv{yI>*PSxv8>$)`_y^g?d9p=Sh+LBx)*>&V4W{bt?XGMa+7Ej*B(PYDD7$JIThY;+
zAj(&Elv`Sf5bcRpMa&;U)|J+UXiO@m|J#xUoBbuMLa}r)K!SfE6eJzAObuDfr01w)
z>Q_-l_gwky;MnBsNcuHSJ^OC7gb#a?WSB^KLiLraL!wmZ1RORI^^`$FY~!U(dF`Jl
zBr^Gah~y<HVj4P}k>60PrGa^5G|;085+s8)5{ZtrorBVzB9O^K5fi@>pl=?B`tzYI
z<G2W>`*qZi>wNEZeLwFxBoQn_Z%-8jA7LOUm(3~BXsvr8lJW6=zg!nOx-P@`Hoh`<
z*{Fr2QXyPu<K2Je-L!Qbt3>0nW6PLCdwNYd@~WzV*RsduMe2vcX8MnITx%#+-&X#!
zyizRsd;=wq$^D%!v$~h2M)Rghr5PkUCbH5eu6Mui7lM=90XGNlnLQQ?faR-XQJkei
zka%(EN4P{F+#J|7fR7D8VUGe=e_Y-aqtYll`@H$fe}(tzwkkwJYP=e#GNycvs7Z&5
zMIlejeh(wdMYZ;^>Yg=*mPf39K*_RRhKwX?+J)+*`E{zFCE$muqh8xz)AsAY7@Mjw
z1l3HZI4m<9#3D}oaP3@W?QwLMQHn)Qdaai9urK9#&5a38n=iYtPDD}Du;AfencJKm
zSKL`tY9wX{39DpiBL;GOmP63n1UN3IKk~XOoVfx6%*nOjcX1-8?*p$gp7aHanqpa!
zXpba6S1O{*!LJ3_2yZQB$NeWO(o^K~26Vz>W7>oU*gJ(H)$U^}p48)VS9>0NuMQ^7
zYW4HMqfu-EnP2A<KBM=$;Q`kxo;Ks|U7`#P>E2+4;TXJ6&R^^S7A&!AXme+1tc;NZ
zMPxMh%CTtP9H33oJX7pK?={|zxGgONVYJx&Rx$c_=2XsWcXV}R<~6&~9I&X4E*_wk
zODoPxkZVtCIt`J!wc(waNY*}>J$4_cNf2C!r6bz}+vE(Qlc@{wCLqx__;+`6jcoj$
z<`;3EPj=wE(Z1l6fOP>LBaXnpVTVOCku-O?wSOt(2X(EA<!6Wrx4@dF;7R$ZFtUG@
z-<uEb|0`SPS#b2~nX}0cMLtpAz041(N*xh+eqc}zK@Dpr<{i6wo`{7Y0G}@+1|wp~
zk|tjvyyT7}BX1dNe`bpV?z?Qg0u-73X1J!BN)^HZ%%`j<t9)dPvdJ<l;)~+NaL}p%
zEklvDd=uz92xKFV(6cy8^yxm>@pc&wPNzk9Cr5;{K6OIo6lMoWbpB`x6QWcG6pRhT
zdRgRb9#fao2{lq0So&6R3Fb9~jRghE%lscA5~?W8!!)@SzoP4!5#3nm^!7{K656cY
zD*_1}9h^iQ{%|ZF4<ZIe_|=sy)vdgwBov?q5({VrNZ03a-q3p5^T5N$cEWkkw&Fcv
zH6WKSpud5w+nRIQN?W>e>{E!IqH3beoi86V5Z3D$)*}lnBr~m*;*lSKV0V`cy%C@5
z(;!jBOtuNXYXT@_Q<jd?YoNq@8F1`n`Rd!D?br4clIBFPhwaxjQQMs!Ci4YiZJzhU
z)+<$1V#u$ruM*<o5X#ERcjFYfl5aQvT{2xSSFUfU+Ny!-C0+>iQNqZT;^!XCv~U5#
z0|C{VSiU%kXVbCqd)YPi+xwJ~jgrH>L>z9UTp9&FFun*G2MkIZu4$Ew8H6uGn_C0*
zmOg%0)jOpj$n?Rg6K60^;5z~w7Cle9+1m3o+k^CBC?WxNirD>R3ad)99)!hw>aX*5
zy(?XiBd9Ey0CeU`m6)qDeU)u}2oWM&$Fp!HR2+}WzaUjxJx26gNcho*jd-ZjqsU;u
zIOG?PTz0SdDW+4RpHxXFO+uokAGI+ClmQclBgsDS0A-qVa&2>U+qpd83f1x;Hk*~b
zNrv8}5)1<WfBQBDyw`~4Gw&dm9M|FF0?2CJE{@dXXaQFnkv$~4eJ;}?H+?8jjBQ?h
zo6~(Jz)p^U@kv4WO)#xSHYG~GP^7rk@f=axLIZuW+GM-V3@D3;G?1cE$WK8=y&a(D
z!n=8Ri~<nNf8g=BCDqFSts=d{3-Et_->sUR*Q5ivei#A)ECh|G{=vlr-B7nrhnid-
z`~^|98;E#yGO-se1C-6h`;OcVo3^o)P^gVT^=L@!nZ7X%3`E?wMsz$r2rd^cHy#mP
zet>yx^%{Qp$;;2s(QO>G@jDrFr(k)06YL?4{LC80?vIa;dji@l(wyZi3$AB#L|Hr@
z^3K=+0E{q9>Dk)PX^sy(iFkD1$#kYtr~7pS^mS-&-MuO+D^lEEw*`0qtQGo$Ef414
z3f{oyTGU#qtw^;anQW?p(e(hs{dP;?7&;r2)dChtID&Ygqd+^7W`omSYq4gXDcg*B
z)E!iLutY9in7>`6$mEqAExe>Wu{%RXu)*q2<Pa{}#j7_+Va`pcq{mykXv&A>TdHQ0
zFD!DkpyP+X=lOc+9C3)4Vbh@RIcv{{FTVc<N-%FT{<6-~U|l}wJmpX#2P$AKeZWs?
zeLkHpO3KQHq^7RgVbR~PZ~dJdx65*=Ho<aot~Ou2eEt!5P~AFEMF(|%<JFrq+Cn2u
z<cY=zd}G4nbOe&iU=<P-6^-<xr3G%JZ?;`WmPw(F$86BnL(Tfjl>lhMAW6jH3RzoI
zgMfn8cJ$D(z^W%=b>PFnMkc~O8(_w8G(PO3R0Z*!GxFm<!km=;GXb2Epv1Xjf4V_T
zN-PZDlb|hez-&K_nRh<k5QA{s#61Y`oM%5{#1eDGrjiI4+w*~{`y<w#1NKI_o0U3+
zR8(aZ`f40bjdJ7y%WGePGf7}+b_qfm+)s%bq-0bVH4#0|IHLW}nS0*MkqfGv{uxbB
z(sm*LVbAUyXe5Li_5Z;&ksmc&ptvpU#+WT1>F;+62z-8<LP(**{1X@V(<{&NB`~gZ
z9~hvuDSg|!)3dN!+g#gqm}Z;KGxZbocSkYgo*7(&dVsT)R}T(7g^-F08nB)8Qimn>
z8~&|Qm`r70s5(0^bQ{pGfWu)=lF3P;rNBll(|)ripnlzh5(S^!m9uge6>sUktc~tF
z8+jN>oYWUT<#Lkb%mqJFFxr1lbgW##OQeVsDGG#_+=^BjBsUrN*o@$94xY|}G@Dk(
z<jWpY2jY_%vFh@3C44a--si>Bf1P{PbY_IKzn0i*j6{%r<?OYE7-7&2&7J9vxVTy8
z^dZFOkAoX*(twnzq&*<s8T`O_l9Xcw*0=BK|8JqBf>E=wKIM3q@F2nLsmFd$j9WlY
zs(67-95d?sHfO_<sc&kvUwQ(-#K%%#70SiZW*L}RLKV90%zz$YNTbE9^vuC95;}U=
z=cli!nVH@rKg>rnPBeW562wYH5a{LV{-V~sGJo^d(XW3OO><>xb=iCanhaeV{G=!p
zVSm{@&#yh01d)i}q}Qj+Rm(Im*g&#K=P)Bq<i74&^?*V7w6AU!6}9Xbi$xC;*X!#d
zm&_Ap`l{IWZVRatUtdHuyZ>l*zG=wTlNNdxS6FVCVV;0_O}=e${%?Gf?75KroFawF
zmnPzZOa_IALw2~H6HiY!7+MvNqmURx*QK;bqaka=lZCh%j=s6(sf_zkmBJsYS_2bj
zwEq5i>;89<xUPwk2qY$|xAZtq#K)cj<$%W(-5sM<4`#_51qq{6sNJYuHqvu4By%(?
z1U8{cOP8>+2h609q`r5o4Ic3{Y^WgksYRGcR<`t>r>#fesgd{K6S`mjir)7#nylG#
zL2*@gR(H{AFyCbc$;;%7806_f1xb2$#PY=O3u@)ZrnoMXP8GSruo~Ng`7ztFxh~`i
zkrlfvsjG#Hlk;2T=v=(?dIE4I!>joH!JAAf;qULS{7J5XopLY~F`gBV+>$-)M-6u(
zJ7X+V{}<d$J*QPbl^fJm$9?}`Q4UGp(+g>|=MCmjq1a@s^jV3^oCdu*Zj^*zX5>sC
z&IlF8(e?S)=bKPN(wfro4RDm+(D9llH^-D$jBeJNA5bddA9}Zmh>;oMA!Y#%p`?pl
z22Hqdak$+a6Q)8Rv2ouD#I(g6JnRj7dH$%uQ#nhfGjP`cXI2<xh5gajouRGgfc|PL
zhG^>}|6CVRn!i${o2o!XW&`_ZVD$wEMXi?~8iBihfuT{qrOia2r{#g=^?ql?ZqfgE
zJcePS!+T!bNP0pa!+zy$IY5L3z&LqM>-3aNR_UK*)|p{19jb%knfG&7vV6xU4UHnG
zJht><FN0Z&VI#W4LLe%$E58&Wf{N@z!G$fbHAVBmlO79E<H`h>+y#-4q6XzkNO75x
z0(BxR{P+?Qf*2RR|Bj0df8E<_gCwfY!g=<fuK5gg&Z^^dq6Q(6kB^U6ZUbijQpe@-
zxEWeC2#E3fc#&H4zEZ)R3j73!B@zm~084J21Ii)rMYQ9}sglvF|5j2`PQ?A)Y{>o_
zJ1P!es7Aq$6%P4tY9@H)0**d4(t0^no2Mmg3d1f$%skM%J+O6?Xj>*6xK7Gm>zQcL
z9MgSf&!EpDh4G9-eYBpzBbOHadR|F2qG2oXc`1P-BUyG12g3RN=-tO{=3o03625&x
z{eTnbu0~Td`t<Xh8=*4X6WElEg^E2p2#g3pQ85s=OAtF8;rb_xF$XfMs~nTxXUBa-
z<#i`1_ci((AkltMXWNjN@1y{mU7jCm$c=*VTCS@_f@wwKZ!ls=jRt#?noP8TgE%7J
zdlcOB8M&V3-xte)Dc?U?SiT!Bq^0|c1nfxnTL`j*>(5sRykZSLRTv=`ba}pB{8g-d
z@pq$}eSQ9YBym84IUpTPfdx)@P%<1Cp_`nd&#Q%ecW0D_E3Sr;K2UlPB5SEY@w3t{
zj$4~4r4K=-J97~WBZsO<V(gnOb~K?l{|@5>7#;We2K@9o-IMu`14B920Q?;BIDELm
zdCeImh$lQQ)JSz6b%yi^V4RA=Kq4FoV`ap(B2fH*Yb`JlNV#nHGwdETmr;blRebNY
zCN#xfWyU25h6MVlq9FA_9EInhB=Abc<8&dYazjyLVRCAA%h37RR`9FwvDuy=R}PhU
zV)7WnnQ6fcE(7L_JZFWjH4bbRjh{dB!R-0kriAn(qJv8L?5fq=hk;`)o`?O<Y3)~_
zzc!jcl0$8?dQ2rMAqhlIC&YI&w@7~X7j>b`f(rcT6N6-GcGHE#=@Sii{kqrzCF0ku
zEn*WnrZ2V-6ofUrIDF#6S$iR=yU^HcHeAz~`7ND{pfop-Uw9F#01P5>o}(Rc;tF-2
zYE_7|B8?`a=`SjQAJSO#`2|V@Ma_;NH_8-6690$BmCu6#N}SE-?c4)Gv>9iwJQX?g
zPl;{E4H>mg{iXTbEyE5N%!ZIsBb_bXXMWWxL9E03{2_W~Ba2F=Y#y_cS2fbUqD(`c
zkLj_Zj_Sb0A+Xy7IAM5?M!DzX3U$C3C<r^Is&aa=c3-9`N94|I&|;s50B9~Bg4<sn
zToZzD;Y+dC;=>Mp*e8mDRu_`Gkg<jE+QE%4pr#4XK2z_SVAp9RmEWV4z?b~M{Ked9
z`ngbtV0}>th@6}^Tn|SB&%)@su)M8$>y}GpNPxtUxZx-anfnYtf7Wf=g<h{C1P~N~
z`+T{YI)J5%vZLhb{|UBKy$eQ>*l6TVIbY-hgzb7B1|ifC7j3ZZj>%?+d_lksnJ5gp
zI|gBiTt2Yl7LJ5VrH~5l>IIzmRap1Zr-fQq&MrcT?S3x3I`^0PQ8+Q&AiT34-&a>#
zr&?l~zmRDKKM=7TjS7j)7nfW@%t%KTd5Rq=!+je`mbAtMoZSs6n%`mex5XnJrxz8D
zx=_rHf4=2*ISR378in#%j1<=5=|Ang4>q!0I&s<ycSzw~$nPDP576hlg=EZ*xk($~
zikIK*0RmhGs<|nMHN1kW8({m+%t%30hzU)K7tVnq_=vslP(j9cAqwb=Y;mCPh-H~<
z!C%d*`Bi(sMOnd&$q{{lD2PE3YkE?pv4I<Z^nZK5QtwqW5Owh(7>{Nk!lT4v{&0o(
z4;tceAzq{{Bc9ZPx#JZb`sLey5)Lta5qABcJEliO9MIlLJCvEz5rSaA9n*U6bTIV2
zc-xe{*nyz?Lf{sffGL#mAm@9%SP`F!#`Aw6a{2;aOrV|p;1hFmmu<#|naCPYcG(XY
zNz*<L+7qv{2fZ03;9YiHNFW1yCq?Y2Lj@BQJs>-&KmzkgfapoG>v4kyTmvoGAxd6x
zHo|fu!?}I}l_=zdsj@Bm1)}(rXPp{NPK8cP2DduF$ND7mZo}wSUjM10%i=_Xgq<4>
z(i<5Wk><J%`-K&R&*!bX2>5ZLCtj8*6&Cebt%j};RDZyWUpgbq5jI#^oU7m(krYT@
z+`i-8seUn!WvqZUcfyNbBb&3~QMdYFCd=Sq&sSjhO;Cl=L)j5{{<8CGq$F5xH*=`f
z1^3Lus*4>Pktw<wbS(~N5cG#=oRwt9U5@wbYbPbF!vNz$+9MI2V7EY7FS)JWEh<z1
zH8B8fu|QJ71Rg@NN`He-vIsW%vWHrtAqUN+1NVM{v{uCVoG1QJu8+>E2k__Yftgv3
zbVdol**~FKub_{}ff9Fs*yp<;V;~kHG<`p*LktCoQQ;!W?D9-}f0yv>m1zS1ap?sP
zSZL;z5Og|bjBgkXd=_2O_r)3`4tm(vtUJzM(P9wxi_WonHW2Kt5!4{gxJ(6R(f?gw
z0!XZ2vmb1Y7^g@0gL^T`TmDm_pK2-^oW%eSY+D~L*wBIm{B?}NT|<mia81gPpmRpZ
znTAFaW;!9M+Icb8fv$>4qqKgu&J0Eg^4z5d*vg}h&QnUxC!8@)Q%afpeRBcI#IW^7
zi$lO(J_68r&3AdZSh?Xv-8#KJm{F+$KL!~0C{B%9y+55RV}ioNpZhJg3qk!Nau(C>
zsG@Rae|Ei~!R)L`Em`?1O-jy=fH@B-{2QaF=_omJSw2@xnjd+M1!YZwwD?{zZ6_B<
z4;~2_CqCQGC699+Uh^3RsVTPa?3K_RF7U7PX-(sVI!=uG0gmIFpmoes_Kp{aLRldP
zO8Qul4_vs{iK!(@1kkY_2>l7@Ex$~WHMb)f$ibuvAtOg`qiaql#a`v_gEfC*Wl`mv
zOt{e}CB>s$uvnyh?N+o43kV>7{3m8`AeY|I!blhxzWDy&UBpP4PCsA_STbVlE~0;n
zofKIilxtQPnL8xPAFIR(IDWHZ2@s?UuH`l}f8D&m3z|5U-)Cfl)e2xd0GIQv;G^=U
z*+oEVPN-iDizIm~?=~hT(SOoS|Bq{qDuZ5q=}#Tr2dYG#M>}soAmv<dK!8;;9;Ad7
zXk=4P?^F|5)3iCrg7TR_Jgy06d@`3K?>ZKY9I#<M(%T)l^(rXux#r2ats^M5Yq)Z$
zVSnH6JjDYrp1i62Lpj&#GD=Qt^}wKKWbv0#?LT!?GRO!oaeAp*r5CN_&LTgREx~Ok
zVA&JuDd=OB8=^=-NtRvzR5SH<M%i1-!TV!0BLmmeWQdsrG>_11ul(|TG{2%1X(F@0
z)KkW{*i^V9Y@flu)C3)eg%zR9CRd)5tM#J=<cA*<CMg%bhGF_-Tnqy$;Kxsik$8!N
zN&`adpAm^&t=qR?;6#gkspf?qWaL8rUP3TU5d>wtM&aVV;(Afn^wov}iZm%LyiQ37
z?4hGsBLKW{6hNv+N=hoJG@|7!DPEH%p!znmkNe8w5S0B5qfW&;v`m7dywW_9Z}=58
zM`Nm2<Eq71HS?MgkHV6<Mq*aWIDSfs9}aqLsYCtOEi`snWQnmrD3iWaI9}>hT@m$A
zE_yG5*=pPa8RP`zh`LyC74`fE>$I4;%nl3D{Wd?R1V(YyfSg-Y|H@uEog0l_3?*wH
z1nRU#axu<Y0^RX|cTXuQBB@|bL(1^zuNQi<_Q=VFpR}jI@w|)Q6Fi>+=$iPi&_CdZ
zvyGFd=(Dobt9=LMf8nIU-9Z9!fh)RYLl$-1BP@KV5sAze&#o3@BDUuXRaaOERq5f~
zKf#$QA+JlB^>+|{*~8-c!>_$EDW)rIJ2`GO9-W)-Wwg*B8D$-Y^(i+qS~$M!@3j#f
z6#+RxN~XUO@#)Xl(5&UxArsBPv`zSJC|Mh-bfYNN0j&uTYzzYnvg>{984*5MaaKTz
zXtlC4I<PRF^5v1#2aC|Q0Fk_4OUClqV&8?=5^>6dWU^17Gi74E)TJnHoPF#P2))WN
z{uTV#(T3uE!@3(sBsLPQSvbu9ypY%{&vC^&?Mltgp<)J!$lKke4dYH4Q^k)@ynf>4
z6EUns0xKTZrf){2!&3BrC(|pWr|s^P9$`plL<h1F057nSM7mR=iIbFXX!4%H7X_Ok
z5_2Id@I)LLRwW+-mN5q{jbNR=f*}&TG&SSuYdKb|h!8v`1$OZOLf^o^g7J|9O>&de
z@AZQ^?(7j%Ay$WE=bXd`4v)j;{ic9~q)>47IL>OJ!6Qus)auy##{M5u?-*W5)3psx
zoQabOC*H9$v2E<w*2K2WiEZ1qZ6^~>Y}?7VujhV$eEqYIKDw)`R#$bebplGbyJ-J$
zMSVL3zIVcT8+I{Kd3Z7)5x6gvpywh(Jf@h0Y;26KVF_plyOJSwV8OwDt?jImwcY+;
z&%K9-%%%XhzsA|V``(KsQeBQ_Fkbk!&q4I)`%UPNZ5fY<l<HiG4uyL;BOPLNZo7Op
zFG1=pQyht@S@dsjqHd|enW=AiXC%F;H6S!#8NwTJ6-TR($qeIwW)<S%-7zTT2bsN2
znWL3c{p&ZR{_KLixqwAEdXs>VpbfFRqH%)Pa3|6|0jy_Bwdf%QGQh3@3#<kRMvk1&
z0yURw@?K%5tSaapmmOZjAE5zsP^Z7DSrT$k#tL<bqqCzGTTfc(4XEq&I&!vb2G{j~
zZnmM7ui|I)-_I>@@a32`CocHW!HYQQh6k5@pB}iE9T=OFlVjGhUPKu=RC1wxldZk3
z4u_gjG-lHv-F9ik5^waSqvVdsHEfgM**q8od4IfmSM4s3xta=<JvlX%1aPrz($sx>
zLS4D1w4C7M<8yTvOz?*o@f)QKLlp$7WrFL|Q+=xC`}3>z{^P7Cbswh?LKPm@Q9+I7
zGEf^~F<CGx2`k-411rNIlk;ywX@CivDKwJkQ>%dYEb_~@93d$B`0i(my$v*`Pk8m4
zO-C8vA<8Uy%i=@E?kcOqI9b6as)ZY*-f>7XasjhntuHDU!w=i}ymfKpsx`{7YBFDe
zo&7!fbX_BOH114-S5i-|o5!I1Z``@?0^Ep{qLc@R69tzgwAZUvU8XYOK5t7x8=?{5
zaCfZ5=*@@FK5rv?t}#tIRyfRn&>W(25LXaB?iim<Hln%@f1rlid}<&{G9eQG7JLub
zq81bA5;my7P;j_U!-2u$6#v^eI=!!eC_(Pk9=-QDu$hhh3{0$nbf*U9MkG+cI3l>F
ze{j0*ZttHE4o}zkWf%DEG#GfALMs2%%~3O$(|<(?&uoD+Gv2=4-UglYrP+-;Ze1cz
z4L7#Xl#987o{cDl!L9__+|zVN;qcTU?s1XUMSz*(Lw*`|_shKseZ37?j-X`nmi<7*
zMIzh*A}S>)i^&Zu2=x|bQd@yAd(Db-FhC=(sWB=#*w=RxlTflti63|@rjNndw!7Ur
zaQ<B?%D-lL7j*kptudvWlMXFOl=JlFgzk!$cOmy$by^OOdrDB2KQ<`|3gk3R1?R4;
zjJ~n4(dzgA@<cTIWh`-qwX0U_UcW$=J}=xFRhaafOcD#_DiyWPXR06ocR_BhpG2{y
z=GZ9S%E!k?Q6n8~EIEAUSl|C0H-rE=`iX^y%rPDPo9u0E>k(M=7M>(X5^36&&6~Xu
zg%TkC{n=l+aqAJ)&x8H_Z4j(urP&(S=kvvsYGf0HwIjdiVG~imy)$TnpKZs<rEca~
zHy9V!RE3toM75|w&Cb4@wfs}A<+P&z`zS)Kq#VUOL#!qgNGFH6D$}Nj;|N!3v$?T-
zarhm=wRZ<J65FkE#310d=&x1|0|$SNEVz=J%pExGJUHyh-;O(MV%b)|E%UaN6nDv-
zF6UpW-DiE7l?6RKxQ3%}|Ets>;4ndgI1%(u>SDRwpRbnkUE6`kd;)H6&#cv%QCGvA
z(Fb!DGS4SUEJ><n>v7ID44B@}d_GarA#=ii0vc(b*-iru7-B?9sCDDNWIo$?pWV(Y
z)H8}hRqaN9*%a~^{4Ol<*h@<z+zqZ~jdO8=$MkvqzMMTC5)+D~p^*k_$qrc)>4t|W
z{K>5p`E?=>^<Vwvi+F=Q4{lu8zG9(CbuB6BE?PKb%((yk<&JD(;q9u2;h?4i69ldQ
zGde0UcFW>=sV$Sva`IEj&1K(DqDntd=r`XeX9tFDR8E9-v6P#gfG<m_l>qSk*L$5g
zGXbF^6wRTm=o<<NWau}~Y^adY-^XjU)Ei51RF63V`&1;-p<H?0wK}U&_Iq2xm7b)~
ze9Ks@!o2o7@Ag)@ZQy(K#Ri8IMhY|1Q#5UO>PYxOCx#ocU8PUwl(Hy!RVWD7cZUZA
zdqe?RwVv$fQ0^$fz9XW#xqUggbN#cezY;M5#(Af`o*5^xjEK1|0XWPp-(BU7wVHzZ
z<$=*MKLyfVtU@^hi`LuND@UkJuNBwnx=d-MLw>%lHRTohwD71K<r{E0{zQ_T0g+|!
zVd4CnHJJZ#{)4}`^nhuFRR)b=^;Pvn{K44aHX2><$J^6hP0we);|#AFxxL|X>1H8g
zJ1Ep!=}1W-2SSlYKox{K^96GAh#H<!s23H?w1d(35PsiJ)Q%ZX{n3uSYt5FQ(*>^u
z$ulg4lc|4VTzSJEA<m?hDdrL(YE-+fZx^x@FrmiP-^kFGJRlK0AQ*Vn=h_|w8&Mec
z#xQ($6Y%ABCjJq3#!tj=Zpg_qT9Pkf?GfAAON=6ye|xCixz2jV#I;XS2G-R{c;7%#
zvfBjBTlMvjvm`&M7YVrhIlTUIqh~?H)peFw?sOl0I~Fcq=K#>UqW&t*U)eCLU3>z2
zo_d8W|MnHnLfrGsd`~yjXcdi;x#23DZRoXISz&tRhk$qP-tV(Oq?TECjYW1(@AqGk
zS=HHeV;VOQ4&+^k2PX~k|9*QP!7OkyzPmrmq!N{I?6tCk)-~0QU-P8Y_zueUM~zvD
zcq`*{@}hw}xXrd((X+GiAiLRuNh2fzo-k02Es;o$9)5AH$+FJz2<UXV@zP##U5KZ>
zniA`D!-1BLIPBNVDs^r<d-i=2!6vA2R{qFM`m+>Iwbo{_MP?xRyHo~LdP9l6kSLWs
zk8qahus*PW6r1E3T|G@TJPJ3K-<n!T!le{lH*tB-2L=(He(D_sYN<ky|F{nDVZugY
zd5Vs`TotG`q<^aYdBB*$f<3w31atCZ%kQh6F!%nEzI>f7BaYdcNeEs>KxAc8t%Z%I
z`n?vMvoG)Zb9(@-1Mbil`Yiye6lEJ`gJ$6`sjsjN(xiw54=+#Jd^^-URlmb2X?3JK
zV!NKIAdW(rUCO~Is&Ur~xzG=WP5)Q&-K4BQX~IrqrKq2?$ate~L7@NdHq{Ls1~+o?
zRsC)t2z@I0@%muEqH8V2Z@&u}5D=ixsGS82h?^sqP1ipdPnu8MddM0fyZezisI$ov
zg(A%me-9_ViA45MlB(qEbq<9W!xt#eHD_VuV-3C$7@#mW^W$F?q0k>P@w&YkPPg5D
zLbJtUe2*rO`qs{l*#dQ@njPopT0K-Z?W&a-N})SGpC!e>RG5AOPPnD7xWup9eUN4L
zbh@4b6>VW`=leQih?>Z2@l<VbA!sgFeLtYK?h$egk2f`1q@dn~sJGu|Xm&*R%Ua9j
z-mDS(J9)lUy;jHGWD(23nYT~xpqT7B{h2Asdqs~X2s}CG{E3bkR}%2?rr&A@28-Om
z{6N<QK~7rrmX%QAnZ~opQggWALY72HH49?G<$}^xWC77kq~zpMg;YdD0=i}D(9~+x
zw47X~#>N|Fa^O&R7zRim`8@NIYjp@$-fcfl;rJNU@A9s$S^gcXh%>p;INP8g+YY|N
zcjRJ<oFsNuTCw2spw}RIplIfJxH;~Gs<$1}c(uHtBfUEzad&^9yJK)0&Y#WX>tN{V
zj+x-lSE`n{=SZgY{1{3vxi?5IR@}39&)D|w7!H>}A^pP4TkKURU+>Tu&Tlv`GM?0L
zQIEh3vC~%B3J~d+T5j322<*3qff6&0X|gY)=eXp*EmWq0SK9#nWMTG%b4I4PJpZmh
zPh&t|&rVDX0#N}o`F(W((qzz7Ha7DUR`dlVv|5%8q|<C1ix#8WvOzipA)B%=vcJ)y
z7kKGIiVZp|j3`Hg^F^^D@fLB2CvQcvARjQU5#T~B>GHRApdfT}{K4GQTCp3D`K?zX
zA=cTp2iT1i7@w4!koVIDtLODxEvQ3f{Ts@z>#IkorM!RCn^x%K2N<Wd(i}Ld0j>Y_
zIWRFzL8qMhhIvp0;jRzc>nN)H&SD<1Z_aXVgo3J6GL7!N?^gUcz4udf{5W&i@ZB%l
zc<lAgmhzKbs;h;?qSqEUTdq3sM$N<<A%+^^@qTUGXBf^LmdoY|mrA0>AQ(U)Bt*^v
zQiu}u)FCCjngXQLThZ0CaD_7X0_08tgJe@VeBkQO4S#)D3QCBOEmr7Y=&I?^BCoSu
zh%{?N(erW(Sl$b{?a{kM@rCc~T|c{;UVrGB&VIzDcjnuQ#vK|b0E&RixgBe3{bD7_
zcoqUS0u}_@c?`>|^{%vk9^D~Qn8B`aYnK`46Dy$KkGtFMwP-#=E0N(iIUNWvHmp*T
zlCt;nJESE$%)mhvD5PK(NPG6t!Xc4|i$fUWIX+jq%W@MBRcgm>jl+m)mpl?2%Qk7T
zwkw7^!KorwW;%ScHBcoqN-XQ1URM$#8V}lVX71NzCaL|c^5m&gR4t^}S~aPcw>K&t
zUL?pIRYH|AN{k@t_`&1;e071-uWoO#OewhMPhcDkTNM^Iw%f;3+6#w!FiE|Kk@E97
z`=sV7HX-k7^XwRI+bUyPr9=})1fmamt!5=-xCwH-99>#y4KTxwzLHNjg8KGN{$y#<
z4&83=@Js8&J)C9EbUR@%ItIeiNFuFVIvxj?N_{4|cfG(rQl}S259`8pma^scn?3+_
z*LuRA;csmIf_6uNsnq+&%n2Wi|5Gsv@m<n{9Wx|3dqNddiKRTZ{HM|t=jLA^TzN07
z6CT1Ma^4kA@AOiy0d-8t4So7yL4A7oVmdDPObZz&>Wu^sc;&~RupiD1ax>r7<~(_F
zKEJZOo?`jQd$wN%8Z@n<%k3;MU?hf1!h1VBYq4S9o9OVuIE>FOMir(7QQ}@lO0C62
z(D5CgQLYiY8u;hm%H|x>>~9C@<k;caE|y#Ejc9e+mFxLbKzvSwP4e9#EWJ*Pw}Xk4
zI+F>^M)L(RsdYsHS|n7L_K?xTgQUyj`|W}ipKElV7Y{a&k8nsb;#prZ`kxP&r)}kO
z2-)&CNpF%0GF<1(gPKlO!_hGDy5Fa2>b%Z)x9Lmb$`2@mqJ+c#7rr37X`!u06*sai
z+EnwcXLQtjc7Ne5h=Ndvlf1>T2vOsa{!u0L*=}5G_HWEs3-0-jry~fE%^@pA%#cqU
z+YL!Yqe?icWKnRBf?2tf6x3a@qZ0ART`z`nr7H(HWE@utyG6xDl}<CLBJJghu64)O
z+npIQ-BNg-QFpc)sv_vTG++sQkgu!|UQV}nL%eWauc)Xah>crz7Y>PTVn<ip8pHiq
zafxP|BYt&8#LdVlBY<KS{obqIi3NX52>)u4M;Cb=8`;*ia;5gngQS{D(=Pm4jJ${N
z$Qf*a9A_P0CTdX7kzdXs*xmA&ah^=trXSRuhe?krZxY{;P3>;&ZLO#xt0g!-*I4pw
zkemK1%i$=T$P!pK@#>VDT(+j(1ALb+59YtFunY&k*105STESYeDQ^R8>=y$`-=nDv
zwc@LIsUm7evP~>KFJjpp=kt{tYSX*jjlbfSei_q<QY9x6_-iOX_}*a`+Jm<@x-Xt>
zPT1AEAKCGy?8IaK9vZ1$txYRIneiZ8&(RdBE0_rW_4?NRe2sWt85=W&>od~wCq;kr
zP^VV;B%qF&Pb3AdQ>oQmNFeK~B4#!$8{IsEK%j&E_E=?2JgX|Y+0!FVkz_a41l-}y
z<m{W088m?IGFw0p6Y|I0mbMaAaK9=<+5X;2qJLwAlCFQD@eSqeiW6GbfQK;Zjv?lc
zK*yhv8U)He-P>bjrH`F&y;nWAd{M6T347H@Xw=2Pn>@Idb@+6tMj<GwfZa9!C~ZyK
z=!divJV;Bxxmq;!!sJ>Y6kQ>_B{gKVzj>;Ygcs&wWN_J!PflqdCEca1xK_27W*CiN
zBi3J~%)dmQ5A64%=eu!7r2RX4f9*Z9McE<Sy&r%PNeqiUpUHnEQ@r)CjW=qRshvb}
z7YtR@A~h{TaA~ti?LkQB=+oeP)NzVOs1W|AN9o9I*I{3*{@3s!$Sl{h<^utn@8|P7
z43S^<QIRC6FAQ!TMv_`QP14`~>^x<ivj+$7GaZw^;}?TVJ`Y2u;Z0cNj(_dxZz`N`
z+4?jnpw+Y%C#?JJQ)B|uWc(`hbUTuve~Ga8zM{IhLNkViG5t|_9ES9R(mVhwY&#gc
z+>YA*tx#tuia@}F?RYrZ2V%+J>?g?SE`){`nHY=(YIv2FWP89lpDlL{FKzzPU$eRo
zt=!UPd{z<tV(04%EVzp`Fo2u;R-hpKS{D{{LXy5crS}+jsiGSwSb?J60ipZWil{q1
zljPJ?%G^7UOAeSiy0y(lj?HI}-Mvx`=i0&xTSI|uN{>r*ou3fJP%nVGeqbU>O@ty5
zpKN>Xlzo&ud4wF~Box%=8XQcd9P7EK&IRYPg^Y3=c_M?N7*!1pW`X;3>aenCtT#_Z
z>aAZY%*2EUq=3kKAg<PcF?j3fZC&kM)FXPdB!ByNFc=T+oh<Q>Mu7Kd@n(kNq!qQ7
zP&wA5!L0D&KKacYj4=`YN<@(;#x)k53BkTMHPru#{u=>%##uZsZdQS#rhW4{fXU(8
z3JAioeJWrI5R<Y6P;~jlNJs{s94z=ykUb8V>(9%?k5K2rBvC~|BS<eV0AG2%eqL`J
zp%B_o>9Jqh#{Zq@)iXWT4biIjs|^rEc7pmD3QpkzD0=yhy2YZe%CVUNEAcI9VLubO
zK>@$CVxBMlFT}(CT|(*S)=y}!Gk7a)@Wp&n#|nXiJa*<g^lP{=-zm1-?@L3wWreZ4
z#`R5J=wlh#W97+Xqg4T6QTk+t-sfuDxb4Wv;5&;~_fg;hW^bGpDTH2-Jo^Z@o`y6R
zo*2h><Z)1duSFg*-<(Nf{l9$i5QuQ#5>15!Ma|a;cKdj<XH@#+Is@zGPoGJr=FpqA
z!!rT{ZrgDs{sm4O`LY69YhP$h8M#NqKpf(c+6igM-%7kQrpb6UXIdNB{=WlJ&3omg
zZ0k4-p}kRG+BDS5<f4k$yopsmH%n3P=0N;PW3<+cQ%9V!YYmSl&1Oi|_iemUyN#Cp
zGN(lQXj}+*zp>(-?ypZz5pi*&m0E+aurR-p64N}D@L*Pbnkx`JpSEi7Y`ukn0E9L?
zL&i%QcL+^{b9H^z9muTi?9&el5Q0j|!0Oy^)A_fIvXuyCII}nxzu}t!*7sz+ZJ@~4
zLrXCvn)NcWxbvDWIZ{J(pb?rVVdTE3S@wg93IK#M4ezu9vv3Jm$#sxqXN57ZU1I?I
z@d_3mdi=WR|Kx`5_Gt;g$N%FMm}(O7B4D0X;NA4o1({44PVrP&;@6W65^+f4TCr8w
z3=fjQPpV5>*KaAM2z-ULKH1Vi(V9z>Hv%f{M?$ao0#YvDsK$6RBsJm;f1%k&6gm17
z%@8Z*a`|p_hH_y5MeAFIf~fZF0l|eFXDY-+js9Vv%V8lW04{~MPSUz0&1cBaP#4q3
z1xbz9Qwn>-_IdxMHP797ECqHkp?=y*lE=vSam0o#qz4PsGYkrT5O<0g-z~lgL`?p*
z`IjzHk9hAl`u-}1AKT)d5b070C8n3}QS$2ci7H`Vdlw~gE{0~G|N4;-)@x#cbzg1N
zj~>4SzDj*~Cj9Wki!Wzf>B-HN0=p|3|B#Pd)*WUxNt(u9;0TQhrqPrGYtm_j({7^3
z@NmiGPAEgVx*WTPJ;s%0jHiT(ElK*8H?B+$c*Huv2|*^mWye&P-Pe{bJUPzkO>@fS
ze^BEs13oHnuM`zLiVqsx{t#&nN9mFlSr@!`I*Sr$%D#UGRiutHF&4AA=NlHke?wX!
zZmx#gd6z10c|?=Miv9u)riDrQ0`y42c?fbn{z=}tbRcOoDsbOFki2kuOX^^9EpM$I
z<9z-T&aydFiMy<+fy^4|;l@v8DywBks!XS6DlX>Yagg}L*AOMw!h2$HSER&Hbl5%<
zq`SuWY2r@?i3AUy@SZaqi9-UioLh1Ri8RFuVGIrr1IWq698i2;4pM5;cvAQHVg0#g
zBkgjTqZE_MdXN72lY?;nh1V4-fLnA(PZd|nm^dMP36Vl1sudN)<6uBhsU=yc-`Am*
z{xOSUv!ldGrv!t^FCjxfFk>U4P6q5qy{jb7mqgFZHxa+wmK4om-4NUOMzdb%xL5B!
zk%EVPY}h}N{v+}ATEOv2)}#_pfh4O%qAogk1&|p12Rc5?UV;YcSff02?&E~hy6{;1
zK0YavflBnE3K9$5_-ysp&nAAsC+`EQFnmJ)*#L>rB<$@bcE6?(|EWS5eA3KB8mE{$
zrwnqG`?oNJ)g8OY07J?#`WMc8=*AqJ9D*u_1KYvX1?A~-0WfPzi7U<RnS0usN|=t|
zc9UgTRB&813vC^0oe_b3eG*iIBrH^-U@Jj{5-tk~kwqC3LRm;a5@a=TJ(Hh=Vik#B
z@OZLN#0PPC%Wq@_+`J3Ee*%9{QD{nuwTZxeE&=!%<Ug|e9>OZM+q;$$CIRIR6ii}4
z`to%bzVD(;fYnEHA$}$koRQ&)Hu?UFA7%VwRtLR};ZC%1!5$)+LZwr(Wb+=WVbhPr
ze*E}Hk~6@mZ}4H~QLm|MGn8R4M?z*IZ_B>oojnZbAIF~5OH~x8^*YMzZ6evJ{-5ve
zM7-gAjl7ZBtP!KLht9Y6TdZRVI^<jsVYANkNe;m;#2+6m3omb&4;UHk#yMUdK3UcK
z_iahzr-Vd${LBUWjo(JPq_ZO^AiBFn`%m7s>!s!>SE<)H{i$I<8XgI^>-bzw67}@I
zC~hHOkR&iElV>hLYBmA`A&5mzAh!i<xO%f7-T<*DkzD1%Itn_9U^Ma!E&|fsx-*_c
zasWY!(mra17)6aug057?q>ZAZafz;nb#OmXW^gfuB9BMg9<VshLmrljTyY}<yDeAY
z1~M--3GB8}&K_G*sVZAIs1zLU2ejZ^SmW^yeqm{Y@sXVv9E)-^^{s@K{;y9ji+RV+
zy-wac-tmO*F{$u_P!K-%diJ;qY51fWa1STlBILtWf%07>RHm;VkMO<cV}<(R0xz-g
z<y$GO>%x@UEaP4Qzpp|JR~C99YU1!JP}2Yoji&t9q>fw^La*a8ZOBH+U6C98KIhAx
zics*^$bCBqK4!!kkSH(sqL7pqa%so)TCE9qyvLvB_72#Ac;yXlfCboukDV0=bQFb~
zg4b)Ol;^npGh-%&O%O}|CA&wyT7r#CZ(Ht&(l&;t&ixnenEnAw@0G};vWOx%kNAES
z?${f9J4{9j3HZ~bwCKKH@cgw&FzXnf6JKO+y}5x$GL3R79G}DL)OlL}{vDH5{fz3c
zi3jF#aj8$spwJCPTmev|dBcmy7!6fh(zt6C;UzOvYyqbJfQ5hTgx?85DKT=EQT#G)
zAZ$Yc*IBgjE5MlDn>uL=kn-lDYRUA{LKfe0WxL9m*_AO_vlMlDxg)+`f4N4c<<5C2
zH?c5<LJiEUMr?Mdw)|Udw$<IWzcna?U7IRLoBiX>Zlhqdc-lh%ol11RmIA0Cd<`ga
zN*hIUZoD8Yy3m{8@^|h;5--Ek0c!T**{nV?sTt2X*QD-KK_nlNKA=!>1CWH@!1#ND
z`jZqq1j&VolQMmXfoMbMq#3(r7~rBg)dG}B&cn3&m{do5B_;>@d3O>m`Rxry9Kw)x
zlPQIZ>d#Q^%V{LNgM7EH%^tPeE|tWSnupLlY?#<$kmmOZz4JpNlwpYvgy4&Xt_N{o
zBI#iqCO-tdya<A3`+5UPB+L^s>FK^j1HXB&Ce=KVp_begZahrZCnDXGn^-iDtAZVd
zCZ>u>U1DOjnH~QeiI==tU}i76UFjw>6fbO0f?v`5iF7~=P*fw*`eM30;ohtfE0xhy
z*MUrls4t$?wO*~C<z*rLi|=&t3qG^1=h#*3{eoLTng&Jk-!BpHWb5Ly56FAs{Ix%8
z60)KD%;5n8y=O`i#iZQ4AYbR`D`p^l=EQ0LD4!#g{6uoAkm6M<X^VbWid7Oi^_X?g
z5a&dR$7%<G;Z+F{<J0t>h(pWVxBRRIdR;=%!lzmpr&J>!I8;j8J;X-B=H7tliJ7F5
zzp1`N@W03K+BLMgb#@bFou6q@8|?S|tJeKyj^;W91CzyM{jg5FE5u2^EImpJ3SKNa
z-qLt<km>a(F(Jl>jd(ZJeWeTN)ix-iJ@F2$6q7|_HPPyNHGU#4l=4M!ZOJIEFnE8n
zKW9dPCQPZ~sR~1>EK%rIv(By8bitn46fizk(SM#05OK_P(XtQ_8l!?KepZX$#TChV
zs^xT|kBeFs2?3YbnVcO%8lojK9me3S)$;~O++m{5Sx~xyPdcFgLpK~*M&Xep7CGwQ
zk@Uzsus0e%1kYM12CFs0iP}I0?^$p}$ghtsWV0gg|J#g76`xmFx$@=<0}UoajD@np
z<B$z&0<S;)B+x^5kHAXI_7Gr~ZVb{<<K1JfcRn2oFKpH1FzG8+PQ6+|3hS~EV*Jtf
zoMBv0%tXi~FeCVI{h85wX`E|Gv3>2eJtgPQQ~Ye}A9h^Y{W$lc)qx^tLnXI#K?Xis
zg_ZadfJ0MSdGR{dt^Zxb!Zoh`LZYWfQ{th*|5?BUcFt)!oP_q5iGq^_KzyEs`mTht
zJt!RswE^=UM{ZD9A$ScAbqodut!})N7ph8rZ2I<6B$3xEOqtP=ghW<^&p{&kPid#)
z8VXLyqw70b66hyC8g^Mhq1rN~fvcgmw!UF%l47hhKC5X#O{n#n<+>jmaDjNDLu6Q7
zz}EC8#}&PH{m6@g7KRc!|5QZ^gV@xjzLxUW|MWn#e8gKoC%z&iY?OxNL`NT<+(tik
zdK6XbV=O=e=(FBtIJZl?XMIZ5{Ax=vb|7L4h`T>oUcwm`Ag<wfHz8xZ>soi*KYG4O
zD5<XmDn?+!lgPmR?BtT-NPwq?N}+`bg|{kmWd&d-FZ9kA(S~86LWFL_*>1w;VNiYa
zs)~602G@&^H<y5<N=jU;GZMg~dS^Z|+oIKHi%U{%WcZcLWqq1BGdLb0r|4Hm8a+jp
zF9LTxb}2xi=yJk>=JSQG*dE>(gN|&r=#z#%r^rPgBsLY59>-MT4I`;hbdV6B@vRT_
zyHkzy{sg#azmNW@cw^3}xUAyuDR9ZYxewtUl3me)mt1?ND7NWVfs$h%rzlbY_ehs5
zU5B@16i_O<y5+)q+Zod|0^QtFQ856jzptvdp21%(Eb*r=Og2&QM(pU|OP2uL=DOHA
zO)Oxm7P8-lyhf)t#<X-pMCM<CgJ=9?J1VMIEqNRSfZ#@ae{>s8Ze#5oYbltUYClCU
z&r^=VK^4Z$Z`4>7JPrx)+fwq;fFi6=0hf}<oq;G{CGB!44b}mQ%acE@BBsp8=vdKS
zB%gU^{2@I9YzHOFGS6e0xSuN)cK1nyVC^9p!eKb8@QZvE+?kt{^(_Ez<rn9A*?Acq
zK50A-kf{=A>_xnESC1p`W+A}f(Y*Lad25lc;dlk~73?5WuW~`#qz1FZe-|@NcIy_l
zZ$W~SSz_uDH~J-BqtHRRYh2{bUkUc_MfVYrOTyZc?!!jb+Ld4_kALhC3sl$;6lo2k
z<(q|IJeaq4^46kG;=DdQ4G*5G(~Rie`d)iBxZwFK5t<4jthYJWm4h_slr%c~fL;HN
z+%y%r8vaTJ_T}XzJf9L8i$c=mBZW^S9$V0`TNXsQd$TD}C^L2j+#2UWkLO1+-zFxe
zhBaTTZ?Phe2PmvrvF#roN>M2_dJ(*KgG<*=uGPlG%zS<5N*d{fZjChLRbx}f*cCa^
zK;71xvq2dyj!9Rox?s+P4>_Qv8gbk~5JRm8p}&1s)WuRX`Sb10WSqnVr?*&v8r<`7
ze1V_VC9b1;g;F!o2qcRVZ+Rj;9+IfWBz@IS3wsj#UZFl{_WS!?_&0#0hV>R%&PXMK
z4Ym8K_r<aw!{*gK!8RPeQQ;ljvCk<bQ9zgs3o2PbdLljkdVoYuc=JUOlRK(?-NJ`#
zVJQ!94U)yEkFKP+*Hd<BtC#4`Z`>tWZFNfOx3?q%)ZF?5B@Ki^2}_h!g<`M{NgPYW
zTXbV+UNs#>ZJWut=P(}LfycU{<H-ijKhrAmFT5LBe_cj_-A!STtxCL^WW(`RF2F#H
z4)zCAEQZ0n{yC%B<#Lr)PLuwO>(6TlGi_%r@aYw;I1xUYxjVv=ThiD{-c92icD4(1
zyOa`^1~bpS!`svDDpFHqxmEp?B$Api<4>TAM{y=zn{{ogK2o`~Kaz#aOokW}hbXDo
zg=dp<;M$8uyXgK<5ZXV%I&00m`7|Xph73~!xLaBWDa<Y@^6D2(2VRS)+h(VD&kGNc
z$FK|QE&7pH#fV#X+Iu)qv572nL%(hm(&F!GrB20x#5I%?Zwv}&nDHa#cByFe&b|c}
ztqy9&>77Fev7I%1iOWx!`feGBFr}0O4`s}|f&y1cb_TWaiIb^u`-+ObvhIW#J-?fn
z2fM4aCOgO104RALel%TFg9EJ!^+^h`>C`Q}jArBZA|7cX&m)EsqC7p{AbJgtna7r0
z*+IgNIV0QsR&MvIGki~-pFSkr!0PQZueIpFg36W|;+pikc}!IBzb?&#8x7cTuCuk?
z>n@yC!nrwKlyBJK_a>-mYssXp(?0F_mguGwHX`8XEU#N`ntam}IR5Jc9TI)a^GU*g
zSFspqDt7hx)Jno(Ia9aLsJIK5M~Ux795cU7&$%G8yW@f?{7B<)eOox_!q%-1ZjuKY
z&Irn4FsMxf+{Et0o*ZhLb>dHOxwqM|Ci@!i8hA|KukY!}C`{#Nl}a)n^5f19q6^*g
z>f7S1_6owwIE-JIzwpR?A;y|o)z#au+YApL@93$SNC>KLna&a_-p>V!1e<@KZvfsz
z%^{@Fy0x0~vG-jRS6H&Z=pKf={=9?xxBhXMW_vPRXC0b$yrZ=+ip21+h<H3vw^IYO
zbF+`DDh3uq@+Yt9gu4#3(WSvE?UHeYj)tWu(64VM(}i^5Zs<L953r6cL~%rP6kmYQ
zE|=d52U}jzJPO0i$}*Gv&vXLm64dm(qtF<tr9Km(s;a~M5=Tq(lIX~W!iV{bSkR}B
z*G*8BF6f?-ETpNfxW7j=Q|c}@RaY%-8on{2lZ1V3e-o%L-YB<Sl5N~V8w5HZ6|aGA
zW1Qmj;reiRXr=QBw1UEc8To+4%v!&ZqiOj9*A@}dsFy?4ULWj`(9MSx)o5FUJA1TA
zZlDZ}qtbB_6Ts&gvvrS3C?pqOW*}zXnRc)=#l>o6So9?*pBFh16Gm&r8|Jh@q@BTs
zSL%JQU2!U*(FPmwCA@q%C0?I`{dxy?Z(|8XNcfIZ7|xTyLRXf{@)KLL(a(7VsT8C_
zI0N<y+;N~*Mm!bN7#E-J^3gHwz|EVONOG**bjIriQ}Q6^d424l{u!^9Kt9S>i%c*{
zDB~WZ?ihAXsSTNlnxkdSZElV-FpvvhE^pY-ZJ}Ze4L!<dkxk&bk1v6hcx|<j`ULX?
z?(PsrJ|bHkYtRj3gCW$}QqJ*uVT#7%48Sj>+}+!Y%gO>VQ7ECttp0U!{$#Zv^#*PB
z{T%v44z3r22Up2`G<1Pcrfu@-<vaxh#o4^5_md4|E>sB6ZnBv{gQnR+0@`?us<?Px
z@ssE%26>;?^*#LW6rG4F!SUmZvo-t<Ef>89MO*zeBsc9;%bzFpN0v<zH3|@Yw!qf(
z#TcAF(&9+M?{Bu)sR?(yv-B!fQ+`uO8I=7r+~N*+JsN4NVNOM@ZG9lBY_1{F#Za#0
z7TD>LII;O7?K3H<5OG8cJ-Y=hrx?Vf_iOJ_Szf2ybB(LHRTIcWyKiVr=|sgPcq^;o
z0$ZEr(u&P8ZzjQ#j9>r*@SRSxsofLW*tO4sJNf(xL_ii15&}P6EF&rUeRFpw4B}(Q
zCnOYgx<*RW1@-buj=_SfwY$Q7?OSk{b!k+M>eb=3g(7b)jz@bo7vwctm8v3M#vqcr
zXf(4<`<#+8B{PJwG7D!LzN#UxyObh`xjkLLs3ee^5Q=<Ae9F<pI$opT`#vEMXskAY
z`CtEh=Rn7xN%ofU;BvrX5Ysj6U#cMf3Pd3TCmZEV!QYi#`1xp4zG&g*a*80ELVti8
z>7x@hnZ<q3>DEHYeu7VAc=ma0^yHPl{bta#$tsM2P9a}yt!f%C49vDc+VXzN63JBe
zNT$f~LIa7cg?4x<_>f~-zk|?U>cH{9IND@%xBbyL_I=0XpAA2HY`FX?y?%%Yf>}%8
z-=F?POmYKU`9|^l$F<(5X|)x?Xf3{w`eNXXa_t)It$ugGBfA+P^ws|r_4xX96*EHM
ziJ)7Q5*mmof)LMI?r<<(s?|b|Nv};A$K&$eWao3><<+^XcK&BZ)J$2H=m31W$hx;%
z$Md?M+cv)r%XQ=EKiJrIYl8eQ(}Q|i;E<%sjxA<#t};sf?wG&MTcT9<)^zY6EQVbW
z)JhzLpv(4tTt%bOz@9UX-jf-gESxAlUkTO7O2x1Mp*Bi&I@nG@&-^%aC%3yEm0`z1
z0-lf}BlbstM89Eo@id0<VxlRPqXcRhcnTLh0No)QBvh3Q4km4tuJK(%hJJ1M9a`kK
zU=TR<s{yN)J{A-ut0ol`20V{xo%#+)P|1?jZi&7lWNLcyA2o*+rn>d5gK?IldEJy@
zvdjrw6J~nFJUU0_lYzAaR8{e)|B{UnAA=JE*ND_#6peNWuGl8=q;mCjv@&f~uK3Hc
zU2_m`2y}c>-8wZ;pvw&aXyii^7<FVTIsCFf+C{9a7(Xb0w1Sz~2AVB4_=bjtr2T_l
znUO<Yg4lCwAX*ul?LS(L!tu=R1{0D%iC`3z;N>`fyR6Oq8`HUWFR)$5JF@&AJ>EF~
zUUq2-GT1OLsK=Co%}0C&5}#TM?w-FNYyHLb#FXieN#RlGsfWw()d&sp;w^vj0ylbP
zAFf-1uQgkuC&dk%yBfo~F$B?UNa`KPenb4rXn;!6Tpmn<@YO$>V=1*{9ch^14=las
zeco;;3Ioy$7{3X+IS>9YLm{mpcQPh77&$$7{ylWXp{F0{In?fUoHpmyXhI5-!-)3%
zB0dy5n<EzLZ71~c+p_$D^7#g>jZo@eLctT$4m4U@%{kiblC3kl9?ZaA;)!KvLjoPq
z9SJ;1V)hdWihXpzp}2$u64YD}hLD7owsMf~kYzgT$DKFm?N<7}JGgyz_6NlN-=57A
zo`>s#D^mTCkbldI$F;lOw@tm%!MHEgLrkJk3o4-pw*|hGUe{K>Tj6oK;~*buz_%Qv
zIoh2o!=;-XEHK#aYex>v1(vnODiTIzI18yeAVga|Bgnmz_g6_7H6G7Ftd_(W+==h)
zlwKo}H&+FDIr7otz5ftrqR$iT@=*R~mC19>e#rMlk;s=<`nQqQL#-!LM@wxtkOsi_
zV$AU#jPU*r@*LB~{Ue6_C}c}WM~}Z5JfoWpJJg?sJ<ow2vBAty$BId1+OJUIF24A^
z4BYp_^*Os5zulYY)a9N6Iyqp;@{x#GHk`iiva{}gVhOIJ2@|LL;29CS3ocCa3k&uq
z^JMoRQcKWMxc`t-xoTDVKHfC!eCjVFDHO`z=7w*7YxEd2s>ey?UdP>D$BuGmt9$dy
zY79-}ot=(X5Ssce^CX|DOMeh?577__RH5!`D_3(FtuKO$fEZ3iE@6z2%Y)ZAR|Q(%
zmh?7kv48mx+&a>1hnF#5o7F-eZc5*rHgc*rUFC~pE9)=(<}>3z7L_@lF?>qBISLa0
z<&uH9`4-h5nTL<<JP<$}SX;hL<Z|>4x%EiEY!+3xTwH%~7C7HU=c&58s<2nbO8h+r
z#ocgNMnyn{=Y3oR!JI5%8c)U5jkgO@@~ydd@y6g7XfqtwNgJj@AeO^;i8J#baP|f{
zfg(jO!P4T`qg(DTx#BdBD^EYi?d@$LBP1#+s#-8UEG(=5NS2zcVj+{$kraFhgi%Br
zVYI~13LY^OOoEAd-MkShYByVaf!*B$8vP|aV|-dqD(y7I3hDWTZdYxu>$RC28C#(H
z<_wWJ$X<yg#QPDeZxsL&XtG;Ur2|sO>SJ89p118Y5cQ`=SP}wcswouwE=V5@pB9@M
zt`-Q(Hd|nU>>-#J7y>GV<b9C{qjb13^L-sd7^~x>FvlxwUw%gx>;~bSg0<Yp7Ogam
zs~l_Q5xq(Hhw7h|IP}56TZW1Fd9j*wkD-BC|65}eZU<>a|B{F!5y0LhZ%6#h9%AZ9
zfnaB6Z#0_|0%#WX6@MsGFR7@~V5KIsEE$FlGh<hAHJJU-nQeFy$lPj2xT4pLa);N<
zKlAj?<aCn?v7n_1NgOZ`_t{|}@j|!YxX%to+NIU%%B3dQ^)F(s?VGIc>8Rd0fC!|<
z`l%AfsyI?VJ^(zL=bwP0&A#JQ{c`?hz6NKeSA(X=Ja4~RDHJB}q!S05z7(0Stm(n;
zbh-7tfnQo=Lyx<bpA+iDP`Yz2^^NO)QQh!N=y?Q&EjkynbM#7<@(&&o{5y)!l}b60
z${;wk3o4Dv+}1z%$A1dv5rBvQSZWmyaSu7adyf?DFIP6apjZK+IQ_#uX4iFB7t_Nm
z+wK=JE&M#?v<8QvYEDkInK?)(s4_vu7Mo2EWmp)4IJCM@uN;tEJbqITwmn5Ou@I*6
za5W*n{(I>T4-@-TN$~fGh=M86fpEz%5UZWg!5eqo+O0DbDq-TJ*9WAnLhz6y7CFxS
z?tsV67ovXk;C9oBY<mQ6pQq%M;yDGL{K1*fIKB7sQX%KIJ4fN>aR0(m^6l&wYkTyw
zOCZkS>fpYsuHkZg)w2#qCgx}TA9X;S1nC#Ocn~#5@p!r5->gCe;>`H?c=ZmKDhmxX
zq0!K5=e0VcP_B4j5FjivlE+XFb=Gkx0s{o;L&L?@I=MM3VH=cF8;?}m%F}v&XpBZ2
z-mk5*?UOS;2MTZbAOJ{>oBLvj-W+_gd;G`^vaToAr)u<MhzIauZuD@8%K9pfEn}+C
zON7O(a#PMfPS95$@1`bCXdNMEwS3R*Ai!>9PxyJ_hWIarOA(E&wv1(NSra5k(Z+h{
zvZE>DqbbrCXQ7zzY2<K4Qy)*4{%|@TUZ2(UFnQb`3!jA`6Z%YgA5CXXj3?3H@cX<g
zklY~gb|(63yT5=rDV1FJ9$=yU&4<c^@?43%1hk10f?Or>oI8f6Iv6bkNdCFq`D%0W
z#%;fYzwh<WVqnU9F?I%ZG3IoFYqgo4^cB-i%q8Mwi~c(+Y-NnmDSE7@%Ou<XeWavl
z6JVB7klGj^WOZ<u>X3vUpK{fw3m&q}dopj&b_MkCi_3Z5Av~9JNH9Kq(^##qL1|h4
zvyd!jV}M#6Ax7_rPE`dH|9rq~b<UQ4Y0sqDDgv4(3x7B2D;`~Yi)5pnB-Ju0)J^+-
zDzDr9F^gVl19L6{K%{>dBR%<d(3<>Iw~u4Z(48!Hu|DrTL=FchZD-kt@V!bdBfCo?
zoYJ$iy;YgM?F^@%i;ki|LfFm<d<r+T-h2uyojl3;dvp6erj3gTiu3V`V5a8*B56V|
zaSV+S50>_?Y@|H8#T?>rowlz!Y{iUck;ss+!$9|gM*!C%|9J_<x+B69xe#vMXb)9N
zbVq3EGVWL-=glAKe9H<*hh;j0P=`&fPEsW75eWQ1$?Cz%!iBS1Ov@AYT2}i$kCy;R
zQnc`WJ#mn<`2vQ$lbKD&I5`Y`Pj&s6zju!gFy&Q8y?}{{`53exSGCPCbGBr`YS?{!
zJXJ*RscG8>|7_2t{(L=SP%#U)>BiD^^M^+KIoh&Faqif1eX{3jw^#i8{r!~$!R)8v
zh4;aR3tG6y?bT9CiO0GoXN!(|+<pN$AF%(F6mQrh@n#$q<MD?aPhJS=e`-7Gjb^Ai
z?M}p;oM|2pr=T}FSm~wSWJ0~ut?p<3b{`C3Vt*N4@Sm-{X7=BX`kt77b-W?)<Asd#
zL7^JHA1Qy~6Jzw9YD<E<Wu2rTQq3RM;bp}rrmS1eFkjz5=eA5!Z{^#LJfFoPC4K8Q
zI5>pK;`!~$xmcti;<4vIleYb;K>xlaIp|%DXAWAkK5}T=rT!r)?!1cse&NK098pBl
zaS;SHo>!T$=`tdw9h#a>#>=a7G5(`2doZj$F%kX!{e54{{b%gv|No&^tz9eW+sBjD
zGVxYgCJ#8R+?lW=ZxPGEQJ)n#qSZ#+wH+RJ>^K&o#Z3Gq$k4(I>Fw(ew!_b_*2Q57
z^K2VSh>?nxu{!I@11(LirNjS=QPOx}Wp0VXx>z3|+L*zh6PJo?8OeW3jI)Jj%Vya&
zWkY-;%YU;BIvG!xM<vtoA{CcVTz%~2;(v|rM?*?O-+e42!k)brnX=Fo{_j+Y&w||(
zV`L&BjQ_^-zZ=~+;dudfG3eyxPf0=72X$`x4@FLm(u;z-itGR&KT;Z%?0@aR@RD)7
z0UGFZwB(YAPeqf7LbLe)yNcq-;KI!=pLqp}goC2~|GMw2$n!!g!y5iV>Xl<$EAzj`
zFX#IaO?N=th)E^I|7$UcE^{ikbf@Ouo%W1Z>iU1Z%4FR=QI?qkfc*OY@4VAUx7$pi
zia20RN*T@n^=2QmY*8yJ$fzKw5&d6-2w+GFs!lm6^){HM)XIj3^Az~sFrtn5CjGe7
zx@(QO-2D-~XA>IvKpGboB4%JfTvSx#)}@<91d1#EaIpf;oh{ZE-5ZXPbyaK^1Zd%;
zw=BgQQd(KQGsTyyr0T!sF>n!7<lmE3DZ<f#1oidxi=FzjxYvN81Q9?P?(T{DL<~hG
zP%KqNo8o7cO9y_!n3E>tBGtpGDxDm?xqJK~sK_K)v#@t{MDary5Vt7V3G)zvDI4`f
zC8zS{8Dp=$#RqJGFIfnmGvjgGEn|{SG}D-%(jbi|p`xNvpaXU<$AIQ*k#(n#x6~%u
z)SZDLpfQOOu=OckbxS~3>9y{bm}Hps+`3UBUV!I%tlb1*avI&pg~L++22>P>N1{pv
zP40f9;*USeJ=ED=s>Kn$F;uztij4n@BF-+i=~@Q!Bta$`&CF;M|8iPw|Njd_&<Muu
zX=WSC|G!CWlI~obx8cq)<BFrw5_!rJV-QUdxwYM1G6QPSCM>Ak9{+lMF`6B75CX(y
zdMo1`l>sn1?tuq9ZlKsYG`lB%pY(8E41ZN{3R_o*=xT;3$2btT$IMv1=*Qkle4hQr
zqN|Y&ivI>>9u?i`r0+%LkV->#Q1j;6fHMDY^Ja``Q=!G&XkiK!NF5)k1&{aSzLBCG
zsb6P2&3LtYiit}461;1Vmq4k=?-fgu)~sSU?;oQXPt2}*&dhT^zIGp^J@g%tR8K#t
z-l=4L4oQ+ZFOXPW{(vU2<GCMoQS-*Bl5)>NkYSHUl9VE7R9XnHQoAkyqe8X3k99yu
z=wzjiWGaJw<iJ5?O@Gepp5uw(V3i_gui?s<n5G&3ViHEZ{vYscqWbqeovWSa8IjfF
zn&s47YbWG0m1vG2i6M7xvOmURLaJt;MZ?X)41dE;4&rJ3AawXWe+&Pwxw+Ftc2gbK
zA`!rgoV@jfQqdVwbFw$MRMf+MvY`~94A?zS`Nj8fuf+FyOyY92891;*#KDo;+R9<m
zb)YI1-I^N@9SLY<S@rVS5|(-A&dNjfo)8|;MB8qzw?1Kr?D>+_RZ|}AWzQj#)Cw3N
z0gNS0-4rfZpl9t4AJ|d5H_McnJwV@UG62?3A3Rv3k{`<z+%a}kfs<xV9tYSSjY5@_
zXZWWcb^g#<9L@7cPBedi&sr)^#J<hCKa9IKjl1)@q5nGO!iERpiF+dKkc`KBzdhZ(
z!_WM{!I8AEC~x<An%3cYTrt{kxSMLUT7v5U<dULE;?oMx)DE~s6`RI&gO<E9RA9dJ
zCFmb^Z}li=U~4M}q^_O=N+U3v&y)V=67)lkFHh5GnvhMqsrtG7n#F^ph{ks(ko8Rh
z--pmPhChd~3MNHdcmNvF-Eb9)r`a7>dS2Q*xm3j|IwgPV&iyNEM!nGmD|<gXvg3ms
zx;aeji1zn^D<LB*Z|2>)W-6OA96Vdf{B5Un+edmHDKzECgT<UB&w1JxZ%xk=t8_z^
z$RqZ&FneU4uoV%#7E%MBTgFmyvy(pEl~4S%iY;$nZSs!#m&R~L0{{MB72IFJEe6*n
zY7QD-%|Qed-h&^%O+Xml!SG0IzJS?Tl%>~w1xWQAU2hOBfS1shDI8=41oDDflULLP
z=uw9kMk}1PUjP@)-zm2!aNZ;VW%K3rHk(|!?iVUciW!-iG8M^H$6GBq>7nGlTpC%s
z2me_o>8!i3NJz+cT#qwlBWXP%W%~b)e|;iCfEi+I;c`t_$L<|9#hf4687kdBSB#1a
z=;8l8`K4>n!yefxK=LTN&}uqJ?a=L&K*JrfBlDJp{O=z~T_|qR5;MGqK0%{~UvPLB
z@87E#-rP@i8osta9eSTh@v)FXGP<|E9vRiXSGz4w1bM!RDUoe@*UKU1gW}Kz4kNh>
z+*Im!Bn9F!`r_(F?Zs@l9HzWAd`|EQPo3lOH(k+bd}0aVF%;f187|gIPUd4EU41zP
zdsIQ<KA!h$VHstSB~IR%oj(lSxsr<Y7AgstwxF+%8nytrCI7;!1Q7?XAQe##ZU&uh
zgo>qzAwvTY9Ea!OUg@0Q&acC!N2#gw`Q-(ao)};PIb=tn|C&i4_;Wj#@9M6?PaN2i
zSb4F&Qg8o5>*<n&n+85ZEp)XjwB_Gxbk@*=`*p|aWIAtX8zJv7=dV9)mopgX5d;i1
zV3s0i1=(qXGYP)>%j^lQrGlv34qwKy{?OKIgq^ojJk7(pllvgQm?utOrh~bRlZHdt
z;;K!7ItPNT1^*5~m|?1;^p0)I5EeKJ-vzYBqv+kfr|zRa3_S*Gi^#_)#=Z9gGw1mh
z2`y%cNh?;B8Vw;}M@U3OjD9yQF-JuydWR7cBl7afqr+1f$BS(V)ah(iD}FmHGOtvi
z4REB1N2P`PQk~?o&@W!1e0A2YV@we;njtwIB(AyEXimx|weSB%sDJ)oq)}%GHIYn5
z!pz(dui$zhv&2P0{`6bd_nps2yE`;CO#c0>i19Vt728yMAi4iTJ<59JZGlwx$?W<L
zk7xlM$=ABzIfRb`52&ccYGBtw+F7KE=ktKWN9v#U#~FJ{{{3hxv9Kg=OMBuxX6q0+
z3RU)yAH`sQ08xWyo+M7OBmA-~3wx6dQxw%ut-9vlamuqvFA^sGYtmo6seXNz{JM@g
ztxOnE<E33D3icEbzfOic;WTuqTG*5C?VOpweIFJ2o<9W)0$Ts3rSuFQ4z6ec^u5o?
zyjvy}dc7A!MQQ7N`<=3hCq>5vZR{5=u@GJu4;i)3oao;YIl>S`{E)7~#H;R)WKg27
zJUQul_8P8UGH}F1#accUq5Q5{32Ugg!iLmDfY01<>o7bD=GaILGMEOpYCtO0hp=bn
znV@DMP|XOy(f&QqFn`vvvpm9{6F-q7DB=gn)&^t<ITgQ@^3hGc5KKQshx?`BZ@C3S
zmFiW?dX%+7h7>Ukc`0aR3YQezxjIUplSIqOmp8|3uLGq|&5nfakSa+g(}qb+kuKOQ
z&0}Sibl5!XpJ-CLx&$_+8frz)H1j6^`H$F-;13QC)><CXFZJ;e^#UjilO;%-7H*v?
z?Vvyah=3wHPzZ=A-}m#E33oh}bJVX|fiybfDF}Q63JXbfJ@;7uc^~OCSu!v(#v<Xb
zRk)wV&LpUrPX>wZ)N|H;)W`4UeLu#}yXUnHvWQ(P{z$-T!PG~;B_%?O`ZvixCI0K<
zr0;l%heoW;-42&lB;6q4&1`4K9xpO)ANs3Q;NE9k7wa?TDTICG#J(&*t^iNk^n3sA
zlo|<w+cijxZQ!B)<t6VZRTppi!}g=Ml<ncy*@C;*pn5XF2)kq=O@2kSWCHo*%N{aF
zy{m}6B`H~-E$xQ=t%3q=(<6Vq!QFxy7%JSYjwY#N8MGX7q7nVqL@nXv!YexCfDZHl
zyg$kVfLx$s->i(tzrS}vGRE7UYWb%eoUm!57QFO)vuD0iyMsDCt6P%Q%O;G2aXHez
z%HTL-DsAtUwE`D0I<YcU-jso{bI@zV;2&cB|Fv|E;c>lR7jDxuww=bd-LSE3+jg49
zPGe4N+s?$cZQFRK@9#e!=Q=ZU&6zpRdDhx%?^_}=tmtE>O5O9Pu!u3Kg27i~n`L2t
zMmBsW7%~QWllU!+^RTcelhuUH<%=5%X6A5nKL(zrLM6qM3U5zzHE1n_1SG=6$VMAQ
zDCn6ww;pMXLyWDNBLspDOOfAGr&VnL57>hm<6)GwaD6mG#Za;kN4qlBO5=&M<;d12
z6t?;RXN+W!Bw$a}1bBB@;UxWpArISo3iT%Q_OQkB3V3;+og^=l-4SAi764RNGlJ~c
z!IV*YwcDM)JKX_`>RG3{J}IJ)e1jQ~Vh$S=_{Yn<;lMi6no%3mt>3zLR(RRg*!^^i
zh-R;yk5~ewi182)6#9LVs<s<aGqZXu`?*QsL)3-cp2p60$?K8|HK<LDN2GT<3%3w8
zm%CNX)?3~$_g28l#9Q^Y#o@jzIRoo-s59WRx|DW^2<D6VL>Fn8o&aFvs51iU9;aIU
z&Au(1vcXg-e@!-aP*5-jH>!?SRT+3!qE|)Ww&6g*(0NCEi%Fj?Q^uZOn13_Bs8?h7
z3^SnX$EVZks**WM_HPsMYtp_LY`qUrz`N;&qq=?bw;XF?41*Kp#8{6>XN+CuQ728;
zn1LG-3X0+>HQhD^kr;E>h?Jrm(sVJfjSBF7Tv%Kzes0Nv1E;d;A_AB`JzB~kp;J@}
ziwsv3#sD><#itLf4{ZqsL=Q!vKQ{EU8jX3ZhgO?Yy0+BN5R(>4PzMY~9s<&<3QD>%
z728m|`swd%)>C>>IKCE)TwX!7>N&EY(jUpx4@WdGV(N&%@U1kE%O9$KGvbc8u?Huc
z$y55(YmYLWN4Spj-AHL-*VP?5MtbKB_u^oN{~YRSspbI9K;l?1WC8kUoxUB1@7jhN
z)gQfU%iVBSspX^-T6;xW;&q5ok>`$dCA}(?l3~0RG1o*Jsukvu>Vk)uHv{Bq#3v;U
zfQkOJS#Jcba*UO27u;;)fkBcWbs@xR@{u~f{&1xx)6l3C#hYlOwXL38j@IK1KS&uN
z_k@wSVB31bF`*=}t+K4Q`FN8=6|<TbPI0}+u~S)G9SQ-DJC|lgg`(j67Y!4W5NDXA
zglF&Yu<Wv|`Xs{^#91tYq9DQW``qPb$vRqdrS>HGF#Q&i5$$Yp&eT?MClBt>5X<yz
zq*^4%rQOVF&Cd4HJZvaf?nwFEOZ1l0Z}3K&l3>JD0bJ_9X^5Snc{H3O$dC|KGBu!X
zSeYx{=`AJ2%eNLK3kjJ9t)RCo#lA`>E>yWA6|-e}z?KXMKU(tr-?EVGZM#Qt(~GU~
zPOB26X84jd3N~f2YEF29%EPWIHd@!}y_T1XtP(&mCmD!T<%V<UbiL?(R{Fo<rCCf;
zQnHLfjZ(;<an{UM)#k?~Nt<~bw@WtRKd13#*gAWE?r9dykMwMP4f)gAB2~T=tzO8=
zYdzwc*4O28I8A9du4;Wx)ze4eLfF~Zx4Meh1f5B?l%c{(HA)22B|Heap5ruonS{~t
zB2EGsV?kGU^p`#_Yl^d*KZha;z~wOQbdV~`Fr28Lj9-aGBQ7Q<Iq`nQgUzfL`=5@Z
z+bO^Zq%MvYFAZys%e(lW8{h_9btoP)(Efi-93~Q86iyRu;81FTne{)BSJK;;dS6sN
zWALVqQAB;%OLKUb{})W^KMie>t@^}^KN{yu%>O?{PFfVfV&lP0B&w_{E}`&0hk#ze
zCgd7bhv7d#?q~@dO+9bqv5g(u{oZ?5QHAB89ZVBDcN|%<2g%eB3WxG<e`4ZJGtJW)
zk0gj5Q^d8?RycDDlHJa<{>nO|aT%qe#9Xu)pfUhuBSyfS>_4R*E|ynv^XSm^N{&d+
z5oVuw8hyP;k>EYrZn@`BL7dTUxvRozD3|VDS_;NFlB>Mk23i0<>Hp5Ti10%pK=WHN
zOM)DQlCs_t=QF4Fb5RZ2gF#-Kon!ZHDC#=t#7`aFpAw`l_|q#)>Er6O7^zqw(JGtd
z53;sStkoQFi85r9-Af&9+jhf<xOQZTx!8h2{++-HdfD_7F68*k|D8=gU0#*?pU~5!
z{M;RQHWdGzknkev>oD^Fj1LqlKq(dAe;!P}dug*UTSlMHy4!z#2dN9__2l_`Lp=W-
ztq4kJ{^!n!D6(8DGs&CEaOx0EgRG=~pJ!R&URt7m6V)950XjvXXXfxgE@<C&FIoJ~
z8M@p&vWf~cEfq*)g^4IqRHJv#JTvb9N@2oW{OCpHjfGrKM>LSO4)xu@;Hq+`6Vp?g
zR-WJM%1xIokDpciXUODFRQr%?JHel^3mJ17n`AGhZUL*Vqt8Uf$DlN?5CcP1^`Un*
z*#zAa{c!S^^H$_e=7>SFy#IY1$5rp?Vvj6~wj*r&C-NFAjN?O3W}dxibB{*p8-4%2
zYxf6VtnY7OrTG<`*sPer1H}*19s-l~hR0o9Zgjaa%65VpI|ADf$fi{UFE{F)@)VN7
z)`tDw%5(Sd%n|mH>l>IWPRosBH~n!-4BI6JLJKyo-W%mqlM-qxrRV)bm-XZNng9Ov
zpC)L3Yn4hq;r>p25U-u%<XA0YfEl7A?+Ts{h3It#PA28b!3C<6C1`TMtrz3k>p1bh
z;InL3l$duY#MXREyBAxe!r^~IdF->xkr`4DSrT!+zDLhVgg^c84`n~9^!yS`J-YA?
z&(H;)?v^cc0_G6@a}_=8;b8R&fC}nuoQa;N(b(GzCMc=7saa6w&4im-`S+W~ajOMA
z978yn_OKXDR~&!UfmHH7?RW~C-{5_6CX#-KSb+=l#orot(+Be3)r>rfn7n6X>Bq<*
zi{cxo<5}xK2z*9fJRD}{V-@;97D91ef|+ksm8XMi;AE^5^j=MXKN<6Dd(u6f2dEX%
ztWqyJUCXd_q6TCk`^5{s+m0^v#*tYKJDS%7&gJ<$E^w+&n*7oyX{{IUJ)1C{{&SCo
zQ(tIf|J9E{Qzi+O*54uj;Yhq)A-pJFUWr*tf;29Vijs1x+yC3rxoav`QQnaXPOYGz
z;J@B4_Vmf8dnkdWxHuGTdQ4GKQP3!H(EO$<-8^-APx?qP5+D$G!dw$IY^->hS6{EJ
zL%x9>0sJ#d)mpfA;#PBJ9KC&cDQ94?Eus7C$3gmL@Ydp1klB`t=#%vO-`|M1J>N!d
zD*eNYkteeWc&}*z{^Qf+G$T6)90u2f?z<#Y;1Gf#R#(!1-{jO-<G50Rs5M8ORIj0}
z&QB(s8}1N!E(GQ6x+N`5M(Q13F{pc~o+aI%MEB92t1Z~ipuW_}uFLGzfkehVKh1^|
zXrs-<#Pq9^Cl_-#sjn1U;3~!s_az_v=5eiHj6HZtD{DLhSTH9io9pnxy-@KQ0t#W4
zf4=`>H&1hvjSyd;*Y3d+Ce7m+G^ksiyA8Y5tWX7wOrP%}j`~gv4`e|a1^LQzH$STu
z%T9rt+%$mv)l|?bC2ng=@AY`0LgO0H^8<B{l?c4M(Pq_rCVx0*<V2cCB(xY0=n^`j
z2uBZth^Y4YQYoWcyVIslz5%Nz7rdk!ng-hR-B-!Mg<lr!U>UZ>a|MgJ6LO)|Fu(9)
zd9YY(!%TL7`@L7LJ*i6;`bK3flu?TNg~I%@ViZiC7>TZ;^uo+_cuLm5!Ex?i{(%wC
zX)elv-E_mV^oK&eWuO~Fzm~F%(qr%(tgO~oACXrg$DD1%18l7nH|Z%<K&Uv}3wcrJ
zT9=q4FPbyTzVkZV={0D|bBmHVq8%KvB@CXix~N}Dx($I<_YU%Vs_5Cq-w>W+>`YW*
zhgY571yjQMjQtR)OWMlb313hws3xdH(l5D1aKcT5l%D}Xa%<2`G)Y-(Aft4%uvBbO
zWq#-1wKRfcB4KJuet$C0qZ~TIygR4m0p82qD>y1FB-D%DMqnUw1^t(V8W4bP^{hX_
zzqvdDMyqBTR*XJ4r1zENLPc59$~vnn+HY~2v4Zao8|?V?TC0IDa>reg?}Yydmlm&Y
zelQ;j|Gek1YtBnrDUdp$pFGWR)gs|twDw{>W|4@X)zJoF+~M-_>Vn-d0;KxDOG2(~
zw(02ffo}pD=~X1ubGc9s$(GQwA!4yB>XQd9PDE|LixK#kxFR&(9=%l$D0``Bzi8#a
z+Js(&I@8LTEN*KZ(7*;xVW{#Mq?V4umn&I%aQC)oCxwAlW+5QGW+JG5ZWg5RVJ)UC
z{`KqEQgyy~PP}MM5OkdC4=z<$2PDQtkDnpU5#INfRRcfv=3EEzV14x>s4so8PcI~*
z&h<gZ0+fj+NP!$ZR#alUT;}B5+=D+={sPtrc#)CKaliLffilz#$c8XK#?{X?JIp_s
z`BPfc`PK9UDL(NY&@pr2ANw?`W7u$|%=r!+LG6ngdftNwMZ|M=M00j4ktPCT<P?lS
zWpRC!bSph8e&H6L+rT?3WBrfE*`*^cHVK+mkJCu^Qp)^)_nKT45!rR;ml8VA*X4V%
zqi_aqzoF*+#Np+Bs6Mcm#<y*Q#i1r1E-8ktBYexZe)QzGgj|k&HF)~TN*4cv0yb9)
zfpAunvNw{3Fv>C`7sU7kLYgEvmRpM=Vd7Fr*oN%OGR<7@H@`}XW`}VqjXd9tBms5@
zweAote*XF7KKNt|djI%tU6+15-}u#cMS7?Aygcm{)~g<9Y-*Cnc37H>f{OXeyEyp`
zf~x<>WrXTmScLQS<j&@>u8A5w9c~pl;`N-I*aOtN{3=^uCpU*eeC5BpIF~Mvz6AG}
zHY4csww{}C!}{Bz-EU%1_M@?IbIIPs6?J4J;+bY*-*BTXd{N$cC2-ll#4ws}-CBCc
z&7$A7Z?W|y__eY1TqvA7zO0Z@=F(u8zU7`RLgl@1_HEZuNdm3=jhVJCyr^2`a06tZ
z;6;_!eKQ?>_wjNHcU?cQ{-9zzOn<kQ8|^)2pQ0v@?q@X7lcW7cPavTw_EpyjsfQfi
zR!4(_k?8)2>qe9NWl&#U7Bon9+N=MW)Z}272?EJ*VS6S;PU4FMQM=XctCRM%@_Mt}
z{-Rpr|L-6zF?*Xtyuo{k-AgO}^lE5xQhpwOcQXHWH2d?tw3QscTzmeKAmpk)c#qwk
z2S|v*cCy*+eng;`^wDWVR*>Z5ox#z(<|&$Tf8^49^YKFCzK<_%gRuPiLyAkS3p}HO
zxYXqN8n)~g?7ovRStSW&E75o!&1#>xx^;jfjE>Y#W1lpttecGRR544h#Sf}F-r<FO
z5yhq}6_f@PFD8v=SuheI##7l3>UA<vydefMPU1gb`j+(;!kB$PC>RjwRM>?YGcgmZ
zk>7mkN2}YV??|h(kdV-c_PYJt`b&tgc0z6L-CP#!RH<>lwR#(ZcDVfsJKkkqOfDjS
z+pKnbOz+`wIfh)3kUHp|h}xYtVCS@uYquWb3y+XimD%$4{<=9;h^VvsHo4Y72Z-U}
z#P*RKlfW-@K2$s?Ul;y6pA$`qQux`N@ZfAS|2v*(iAUADH}DXX=_Nn{E|lA!#6w#h
zR%%Ke#Gg7^YT7mv<FAWcrQn}T88CAL|EWMFlg=Im;>j>NAITdU8O=NLl8^+HC>52;
z9)K7ooMfuSH`hTY#5iErYj51qTCA2PUL!VCeFM9y{AlRg_gL2u=}!#cUyv`QY#yV^
zRd0p3{iI0Ee(%We-aK{O`90VmJe6;et$JJlLsDAFcE_}pbPKtqQxP`>yYxvacWB$c
zN2Bs**sRBRKYg&W7oL;Qu|(9*vowGDL%RD;Z;%{bmJ$tQe~~sNq0lxvwA*S+&N49Y
z)i~wgZsCY0>;(ANn7Pf9YTo&MJX3&@g$JNQurVlZu;#jI^Uu^e+vrGty=aRelS;Y*
zB^@*3Dlw5q4>!?dchZ9N%VcgO%GSD4XF0f~#Nl-Gf4qE#+H=qOmh1<oqIv?*4;#uX
zFF==|BXi4a*3}O_2Z7E(t@Yrmy+oM=6EU!WKhVriH_0Itb)8#n9#+E^Q-5)P=?FO@
zaXwV;SJ`j{THU19>C<iG>#ni3@>q0v_w5NROj+)!)1PkaO_d<fJ^_YVKFI3~<OcLS
z_@T)uRx!s+;(3zzI~2K$cUSIscwdQ(!=a@%e+eidD<*e9VX_O<{Mj@nX0>IE*>o$J
zR294(yxiHBV8Ke}jvL#D9fpOX@Ve*jebpAJBL8ds<O9Q^W;2MjoX~>_U*xdWQ(t{<
zcyOqsaUPbNV+1NvAP8hed@EDCdHnbhj3}u>lg0<@XA({E*UGc7xb0l;19tYnrrR{U
z=y6EO95Z$B&n5r<t4BnMSw`snpHnO}6%ow=-?1pAFy(l@<QAn)qvHv%-ZWR!yJ}ta
zgn+|}tkEnk|G8V+!x+)F&`W3f<U2cI8wW3Re(|o)4<67=k{TT5e{i8ld>&Q0sv!GP
zy)J$=9T)U0-xn&7ld8JwkZH5SNJDd4-gZ&<bA<AfDvC5J7EZEG%t$$iNSA}23A={>
zDzHCr&<0p>uk^t8N*k}Z4YDmUFhBg)vHcE#(yvb@zAHtv%=oM(WvF(^)o-JN*dKdt
zXLFDWtoLjmZj0yE7$k|0<@@Q7m3xg0NeYT7<7#rBF-({`6Q{GuIdp47UWie<vo6n;
zInAaAZFg;_K8-8zXP+M?<ZJgBhWd9mn*!Sf$G22eei9`<27$y0FiG8)0)4<Hp(alX
zhZx!UoQG;RVWjOO@dA{)>^d!to8dCvw>K$oMn6b`_@pR8e&Fq;v}sJ=#DPbA^G7G*
za@x7FElCLB#VeuRg!WI_2UXKCJ(*UO2#r%Bn?AcYMn58x_#XG=r+2ltt6hlu#TS9D
zt?0H*ePLbbma?w>8caRo33gmEje;e7vGNvG$urFxH`?9QrY<vur0S6U+x_^-tARZa
zb-riX*E<JW_eVG81k@#V>B9!Ito`^B8mftvZa0RHn8O3>yhgYsOeF&l+;=ud7?Le`
z0)$_Z1ZC+o>P(|>YP2|y3kwTNS3_;&Fpw8dMQGI**s`2nzcJJ}+l^PGnHf>SnQhpr
zR%|MHF;owYsAVy1O?0W=9_@V}lce#Y->_L8>u}+Ml|B0I7XPs3`LS|XJR|P3pbn57
z2$4uJl?zH$14R@p#dhb@n(f!%Mb6t}n>NzNss|^5<pN8RPM-F!9J|+p^!obA<VPHH
zX5BtnwIxt#Nh`~Qvk|A4?O8N&GIw``6HyU&kzVK66A|^-YsFF?cF(Sr;kQNy{caXa
zdlG<_4i4W{zk`fb3MP|5bx<US?Zs0jFrh2|2W(nL_z{jL$C(~{+1b)B$joXJ(LCsG
zo%U2cQ-&X7d&EZLyoH}Nflm&3HCPR3D}ClZ8#=tqt@y|j4Qp?QFn=n2f)6LHf_)cx
zFTAca$SckUzCo`(xo0JPMeLl6qI(}sD}Wk~Vc)rOzg7}S&gy!UZx-y({S`VOSF-F@
z=RPtRsO^j^7lkYNa~eUn<plYFq4W1(C0~|?)x+#bT6!}nTUsPKH?JqGW}YHbRo1{|
z83*G|3j=nhA(7Jsg|zz%+sDal_yi8!{VtFF;%?K-)Z{=>%*H6LpWVU5h6I*#!*=3L
zsY!d4Aqqy?VU{1@`S$QOml)6M;Y`C6S-z1Aec$8xn6q?rf{exe{4cchjc#|WkOEDX
zFCk0SXpoi%gR?FN01OlGg#@BQk+3z<dCFocVd<JPffbgSxHd}-#LX#FR11DFn6p%_
zxkMIzj5z0X@s#$3*z|WeSaE<%VJd?sWU$@OoHK8?bZc7N4Ak|QTu;Y{e^5rV@s*Ib
zVI&AaZJM?(#+@J$6QQP<!m0nI5x8cHJS)}WD|+VnF3d>aJ=s2CsLd`qogR?z4rggJ
zX5^N`Ms=)V*|&?C9gN$p5#CqInhe&L*r<U%DJg-eBtfL`f>gUs&vut^r9g{-kU##8
z&xoHx+RkZ>U@Z&pi$O+rr|U9+U2)U(rpR#c<(uh#t5k)pP?cnXQ-00N;~WRfT{)5*
z8?%NAk|sJZ9?ys3Zw4=c8#LM1`<A_v*A&m$$MzX=MXHP_NusJfzK+y`Or>R>b2UKL
zEk4=m;kIIb(omvpfqq2EVhXu~V5lQYjtfIt{^>EkokCkB2%}NsjwZO}Ok<+|_1MpE
zJhQEnD7zUeGb<Il&9S9b0S;YfhRyI2!12=?cK=7ZLFRG(*dk68L6H;eY|$Q7VjN`K
zy2wd`oa60t#r5cFiL&Z8xSO?<NG!2yqp5uEJWIX|s_0K@Q8h&nFh01ZaVmOVHIRxo
z5`k(v)MCUiPlzcZd_c=}Ub4Zd*yv}OvMWxeqA$8xe9NTxX-)|^?!9IIp;w`g0z1`d
z<9Vp3NEzC#p{;67-4mTE*Fz!A;yOT9Mp)rQYe!L0Jp3c^&yJ!t4#mL|25FG`T;|(5
z-BCxwpqsF?4-xoy5}nqZ$0c}$W<xG${t#$h=&yIWg)p1`Mv91(FY04P>hCYzQrfRy
z+d0oUA~~opouXwlOu2k8vev$x>C{AkeJb*bA|RZNci-OYGOyMf97zQ*(drDpu%*}7
zQ040%_)6Cim<Iad`pKloUMPzGh`i{Jef8SX{e<+ZbVX?}Moo7Wn=-xGcZPPW+kGe0
zxc%@wTgRP!sp-k5993<<=Tjw%F%f8k8sszLG;;>$IXv%4QynWWfcUec6spM<Vxj@x
zyp=YQvbIe7?#Nyl=yWtqQ`rlDW_bOpE4FCQU~q<z(#99^aKn%6@AB#MZHLydULCC3
z6{C<{%lA<`yBaGWs-E@LZH+hRMynzR2Ob8i&1XUj*kprq763no_OApzGHN|!?aWrz
zJfnB4hwm@Q__=$@KA%@pDffYQRf8#zo^j<2s7?hm3s)=2Ue8_u5f_V%dk=!1{LjSr
znZpB^T`k9)h2P~6Y8ZFN8?0%rW=<<@o#8m}Od`2Xh0&GO#GgMTPeH|R$<=<&#FxGq
z6UL^PR}f*Y9RNmbVWMCx)5A`_&KcbFjeaMHr!w2b7K=tNE$A?I>Y4z7)f&2Ii;5eI
zrf{rOJ{wJ5vk5hw?KegM{XtSrl>KrCxcXR1ZeD3KK>_jU>gm>_t^DU6F|-%&b4p+E
z%}1LW;r{&=I-PS2Zl-X`{d?koUD%AP>IhaqRY9rB#JO_xHahbC)JbCaVfM-__4le2
zc4@k3G$IQ%91y&v#Bj!Z+H;gvfuISq;@X}`z1|LU*_Vmblg~2NEKfr%M;@~$9p9Z5
zK|5F&I!eq#oZdltKbw2ipzOD}SybZ#D`^nr$xH#Pp6@dmqQG1A3XsiigZG}Bp=A%H
zp@Ct+<z((S;R!yhM8k3+RuvE<pP%b+!R?b<*AEb#&aEidZU<j&1jpgaj6D80KI3T2
z-AvFuYdx^~SelRdOBQqnXbwJ_v@Ph{3Myw<o=mdyC2j&TiutajG$eP8)O*0U{IDWy
zscIE}JC&U9*@iToT#<~(Umx=j2$}1uBHrpnO#SpQ+PM%PVE4h-$PAfw82E0@nVrL0
z_C+TNGHTu5Rf(L86lu1K&R|%e<|r=uZlyK7V)WL6i?lHgM?1uH9u3BDaK>#X+1mYt
zegQq|$3q3haAa$gzaG5NXw<sI=m{ijG7^t7Cl}2pk$1PbvtPM^-hy+|g~OEDjp0mH
zsHs>;=N;2wUCmOIhLWj+BSgQ0Y+LZ0mCnJUx&UqY*&0zv`%_^U72>6F(ai=$gvkOv
zudffzq4eo6xDAFt6%F|)LATzllf%YbTq+{x^ZuaEo0BdDeaaMo&5h2iNoF{8`&Tk?
z1D~>B18x-CxnDvOMcCe4C`Hs@L3oHSrtYuLK&)_<3iBZBe8!MWtM$kWC+I$&{#s9|
zU`yG@MO9`J)gZAbMx1ofbu=vLpN5n);R!c14d^ynz&f{EUm+nj<0{?WICLILzIC_a
z(5&2(1gd*2g{q~2u}fc~!F|)sxRx=!om?AbEdTB81l!df2A0OHdd#cEnM^#&2g5lF
zCK~jyI69t$r1q$p`u(?trdX5;OdWeZl(h8cgQnE85r&>dBO*lJJ19AND)9Z3)-Q<Z
zk7p`oi%3+|AEC=)V{|KudzwJEu5u;k)TpRivs*I-T4>1v!@VQ0F7^hBAaxve>y#7P
zk6u`YYL8pfc@J$U3Im5UPKVsqi?`cJ0kTeEV#zybw&Fy$rM!ojObDxZCjlD5_umpe
zT}5~8Op$L#LMtK~OPkF8^llGiOsxKt1wLCN)G7g>iDaCS^y_H;^G1w5Z+zC!3kG=W
zB160=mk7Xy`!Aw77$-7V;)7S5hOH1=4j4N_!zA6=huvw3`Asntl}g~jC_F3uV*R`>
z2w}tcM;kTR!{eH>KGMwo6<SUgi6qCXR9_?Xm~ATW54)8j)SGNb1pUv&;mh)6?+=VP
zbcF_K#87ZmVvd|%cb4jZpZileV|fzgaP#2=e6{&Y{N&|El52*F<-v8n($rl|5;KFs
zB3S8Zt<jJX2WV0d=#3d0Kh<b+MNMC{F`R9Mw_Z!qUHDnDM|Ax5jcM_*)CsN3qRo^7
zcG7Oh<6byhIi`osGe05mUO1JvnP{LHiMTZ&ufCkfG<-o`MQ&SzgKu<rz&0*df|yIC
zv{zK&@2$NH-ZB8!w&kzJZdSME+3x|FBOgI0fbTi?!e?6K^iHpi{MZ5>$G3HaaGPEn
zM|oPVb2N-L1a!}@3x<;TWJq&yr-ZGUT?FnyL?oSLD#_zQPJ~Garvq=7y4!54T;8i6
zrY^<z7A>Npsc6lai3@*mszTy+46QrUt66s+(&_iFlgS+?lTwXsHigYEPn%1fzR@1o
z58Uf?k4|okS7z-}Hkia|;YfxJNS8$iV6nbq>$V#4ZmK7Qi=@Yl7p}DtrS+(7_O5+)
z^mXj2A)jmvPP~w~7do+mpGrhQ9RwT#_I~evCx}M!XO2kxm7DLC=hR{{#}3GPf^wUL
zGn?JuO^fnL;13hTQ{T#-I==<XZ#O3H+?DCvwn1CPVi{1-N?&%qR^oismM_n5u*c{+
zs~w$_2$K6wanWShvo)_@o$(?O#D?P+H-Rnv+XL1sD+{QH*;Me(h!5!3-%q?6)Tswl
z=rv0;lZiJ2k<oJ1D&TOO7zGtvi67dAKgqt?Miw3M;il>OHzzy-vFQx~`*j!V9f9kM
zkOA!YMH0UvY!7Gh(-W<$2w(nqv*w5i%`f%|9rX}Mv`OW)$A7nUM<CJ&QVsKL#`M1z
zFuMPEVNF{qgn_MM46<a>%(RRc+HKrbh_<4<CtiDp{q#t#ALE4+qr(XizOTE1eN<{3
z7qgn80{ekwCT*nocp?*dGpsriy2Ce3n%e$UmPA1@k<GZjmbiGT+oYBmz%X@&dWgfT
z%Ea2&qc_WbV`L#baa=*`c2Z6KXIQR9ENT!_T=_Y7t!blJfTkgEz_zJ~#&0*;I?=Nf
zO#wam*;mcFAXHi@-Bjd3lotKFh`G7>dWw^TBh<DR(Z)Pu*6=fH8~{-1>XC70eN##7
zdPe`F(17<NB+ZQMY@-I#Nm)4eTO=ne%ktAC`wg#Pej#O$hb;ZUlZatv%b6<7$kh@j
zSVHZvLaeA}XL3J;f@DttRFY2H5PT##Q+xRx{aRblI8UuT&O5Y}Y@tSAP{y~mVMi_T
z@EBgQ_gCW23cW4>h{cIVF93s3T{QIb-E$ky<j{V3SX$sW1u+0C;v<%f-s=#<^?Xto
z6-jb8p)l|%9Q)T-Ee>0X#Y!S}ej#LT&EDPJ`PU{BYqQx4>307bR%OjARW(NFYm>?=
zAU4l(JB?Dwk&w9cT*^^)t<y5|LoEs4H#=<>#Xs6-<CNsI{L%V(jOSF72BWG7#ypH7
z{nPCh8PtqYPP4<+kW)X{)jJm6-jDYzzF8olk9DnkB_}5ri6P2KNR&9H>+J9ijjMD;
zv`<$X^eK75laZ0JJ!FpPaz&U?@B+`Q$-06=V!cOU+x-OZg$zP@J3#}(EjL1<-oEoZ
zc>yJ*?+u}~x)SNKR+%8R90c5%gc-~wHx^mc1%IrIJ^pAeHzo1A-=u9nL>yA?cTC56
zY3B;udIpsUlW2+mSffRriF84~hy6%Lm-870q>BA%SCs^`rF8NMso!6rz3Jgdz24j_
zZQFh{e;C2Af6s2K#KgC|%vT-p()A8Q7hp5HY8OmI9Jwktn825kz~f01qoP;+yh3UD
zMnLqOr=`wpM6mDyLOPA~U@Qc6M~aHL;da~Fp&Wwe*IWssb`XS_KKH*;t~EgVi5_tg
zriflb@rj00&ft&P>1Gmc6MZBON{>!#RXDITtHgKD9rA$iK}?sr%_&)lj~yzCO!v4H
zP8I_2y9C3#6=6DIv_tySr8+5I&V?1g@s-Den>h$nK6;COn{}h;;wqr|<PC8rH^x}_
zDir>XyoeFdt(tscs<g2Jl(;63v!n*}->mFikpbh;<jgYNUt$ze-QzMeuAN(v535r9
zniaQ~(Z9xN1un|PEFMF2ArXU>g2ErBJI7o6(TeS&TF*Ww+{AIOQ@IvMtymG3$JZ;O
zSn%+kN*=2bm~C;53)&pP%}8MH+Gl`wI;k0MrglHxjg+5n2+^VUoaO4X;r13$DR}{m
zugCVVqoBrG<$!(_?Vk#T-)vH9GX=S<OvzyhUXf4V7g=ZvN^AS0MhmU2m`ek&*B(&X
z$^Gsgh+*nrr@=*kI0qa(FNX3=<0h6Z=FYk&G=@Rsb?M@BDx~OS=Qlxj*%u&ZS4p^~
zLTcq?z9~Y%uo=+vcSpZ-OR{mpd@S_~1PQ{mFeLtH3;D9|5svIepA1<gSQV}5Om(bf
ze4EX53Qd2gM|ol;nqDlZYE|aWUD#S8feHAp*dKQT^;wd(@SVHh1~KCOv5;T}ClQAG
z#}_GOUZN-xra`<)j37#`)=!ZDDpu*|CrdTV;BNEYI76mI*iw6ISpTSKJi6j0<qhac
zL@4sR>>QKHDuKGUXM9H2FC<FQ^H8%nkBj(!U8xIy{@M9PGj~@=^1$)}p$0C3l<O_a
zcz=a>qr`90h37Ss@??n-=q1qv;dJ^A-4DA-*2OG%pwf9783IhzmDoheYZ@6)wGs2<
zMfY3FkG}=QVdCPgff66-$2N7#ANN$6vQSFId?6HorA{1y9queQ=f`mgBo2G14v2Ip
zfzcgxTrIbq00~)_IK!%oJ@6|Ho+h2=t8a;P0vpVro(FMK^ilA<>0+_WqzqpXH(;u(
zT<olKCYI;6ne&3|7eh{qZD)4Y`ciLhek(<pa>-o#JD-0=AVVp|rwp6TPMv$R*bM%_
zE+x=308staYyri972xi(^;dPIbxebYsk(}%veuSsSp}4^kOkl7r3JB6(s90tv)9Wa
z_6o8S5q*sSI*L^ZON-D75zXy!9WrDASspuka3T<@3K$sLomgyo?{0Dym?mDJcG(=T
z7-H^hK$)9Fwh@qt9u^)RXVPyWd$=-Xf&t2l3HzV*%$1`y`8|Kn+1BiAe^NB9AdpV)
zlmNW}k$2wiq~<kg1t=b~=PbXPDM(fn+}kTOe_8qhCK;jk6YBcwNav~<<N93aYMR(c
zV7XD~<9@jdmOw7u57D^)PjL)|Mb|so=uXr<N*C_JbIkDYtCJh!@BKT-FppYn17D6y
z^8Z5>Bo7t)@oKsQ4$s2UM_GUB6(p~n7s885#WscwivI;bN??$aa!nJnGfnC~jXNEn
z<n$84H+F$^Jp3o5W<E_iqGd`hJN*CUg$k)kH5dXRVhRHHVFirGLT|B|-H`~(*C|kI
zqa7A_B9jkA{yQ}PLQ!5`UO2>oB$FYddj<KC(nTf}C|Q(j0CR$~<HJ=Zgg&Q?B|-Er
z3lvuhYS-ob`l-e=BN>AitI<?)o>V0Hx)!~tqPE<K&Jya3^)Kku8U+<dz3U+%E1ogu
znEb=W1XHUfqLLa`=~-Q_6OLFE)L8?H)<B&4=RYW}HUu_OH>l~H#xulKT#A{{fO=@e
zNx_)-q@!pAt00r-#S9dnYuQW4+_e)h4iqfw6crOfW>tE!cj^^2EkzSYY&i;2Q_(W^
z`kZJ6xyt&LR8)(Mo;jWuB+5X#IgJxN3Z*GCcg;Ioa-Erv>=UNfpxqSS67$rwb38lZ
zm%}+a=FIyx!;;<W<?90XW0l#5UApDvcMOfxyJ2toPbc&y-}X=oFZ^GXo3&9Tel|u`
z?#g5Z*6xNhpukw=829NxV;LveXicFxRaXq6g@2s5EYX9u>co5ecPrN&)}X0+BL)ap
zi2-}!^Mp5*cIQLLz-Z63)sX&-7O`^0X!VK(&COcgbYzmX{=IIqG|+s8G`7+XZpGKl
zk;?#E6<=eAXAi_$^T`(NCE62K-GcO)Kg4IgVt17&cr>?EHaNo8`QO!Y5>fmLbXQY_
zEg-PC$&jqUS08lHZ0LdO6+{k1re_aA){=bY8}`5PoEZO(6ElSgW17a-{Xo@{#Jfik
zo^oGThb?!9x7ChdaVVUlpq+?`lPMeW;>B6*CWVu1IOC3$5EO@EGR+%eN;p8@>6g|o
zZV@u2&Y{#(?tM-;TY`Y+Mn$((R!h9l6aBb=tDml592Ch_7(wweW%Ar_v)yDrgI__}
zF6!~or&&&K+OeHZZ?^xW9;cP*tq1Ris9QYwp$@XF$j-DAC!>Ku#=s!MM$8%qVd$Kg
zhQh?n?ZZk3O^t>T$-i-8OcH{mAqneh!yn!YMJ-{Wth3dE-G7;1tl?DTbtzTeXb$V+
z#Xsn*l#n+&uaWY6mqzjVlo%5~`X~rLhGJrPhfhgXcl4C}Br9g1X40#%CbSDq>^a4y
zr1J9fLD{+4K)aMCBzgx22eUg(Yj&;iLUV=2*|^D80n-}eXB2Ch)@gS3y?PzJp5R35
z7Z=26`mgGZCN;iZ1Ucyy`7OK0uYD#Wl^T6BRT%Q+umcVHJe7n5yS44;V;<JfVliRg
zHpOOc=ZU~80X_A{syiLeXD>=pyH%Are)Um_ld=?6=>SEtx{Gvsetbo^a_<A-8~B`r
zF>R@yM9gU~5k{@T%fe#DtYf7JDDeyFSsz$+W`V#QIlwP@35eUyH*BBL+hZ@;@7H@9
zlOt}rCRO7iKR+CATn?uW?+3%QK1pT;ypWx006Sy$Zx=qlvn66`Xh>qWFrw_OKhYPD
zE?b&)_#-xa5M**9&bV$ZTn_VCODLznM_8;E^VrUbg@@sx6-6=-GT0lvK~_IHa*LY@
z)v%f<>d~0VNr=^u{W12_frs<`VlmihQw+wflqM3rs6r$5!tXtH1@|Po!}Mpb<+M||
zHVnaVcS)|Dga$rw(?r*rQd&7hP6^V_xM_=hv-YQbfwk8Qj`k@kO&#{TYglZ?ZKn(T
zsiZ<dUG~YhB&yp*l$4nOSbBUBA3YDF_0bqmB;n@HP!qm0LyDV2wXSVpOGUE=H}!3r
z`QhM|<tkU|cxp@EYSzl+IYB~tQ(MHi$zW}%NpyJ*t3YGy`GV!(oR+NcUB4202123T
z)F!1bx0f^6S*Ap_o6{hQ-+t>iHy8~Q6EY~MLwI9Re^yXbv<pfviHi?mw<V$CFzq`S
z&<z<J0>GGMEQ~qVSgD&lAI_FRbxp1NB5Hbiaw@8@jSjEHJ@CB)>x1rm7whUnq=U_H
zv5H>GCw;C12)^SN8>z+^uL$62xwfh`iBmzufL?U&^-|aJ)`;=_okuf6OSr$$NzLvC
z=F|J+y0$f)NYh+QKG&}K=YnzJTT3dv1dj%P!|%NUOcd9TBHC9I3Q*0sr-K@Hu=w`q
zQHy8cV!>cW!yFHH073>SDqtsp*E0X3#b=4-0QF#PA;dtdvBs|fP`M~V=tCpvyKufF
zqd&NM^nTS*?sb5{Zgx)cI(m6!_sRT?{<!}neWixS3-cluB~6#_cl~!R(x{gB@?~63
z`8z%>=5=S^6B0b5-jucj$+HC;T55Zev!A-!oB_Bjp7L5kxC?4>*ek_jLdKho8J~QI
zJ7E<8=AEB}oy*!nvOiI6eSt9=i&Z~X&r!!z8z-rMRO^Ast<QWx9)-0-$m*!z?ZaKY
zRvwf$Nv)f?*(pbp+h;K}Nuv1wC6~eLLp@NhSD)b6xq0f0>A~T^B^bi9K;`v=)Ao};
zSruP1?s|B14DffNExw9^-VBv6x+|k<ef%g1YiJg6V^FTCedzF;5EPsB?)ahZe!Ek8
zT$?)%3y1A?Niv{F&7ME2{fTo@W_>(%LgCHhavW`q>+ffj#jw|;K<_F|1)MbW{^2hM
z6R!alUYGOl#<D({ZEkP|`fR{>N`{-;TM1cN)e7XeS>tMDYJ3A16k;||*)%eo0>mkI
z(o<Ig*P(Bgc<xRx5Kd_zkD`B%zs4`{eSeV#@TQK9NrQGQ$DjteaU8N})`j&)x={hw
z*j59u_cTa<D_j;Y)<TodXc6<seBdz?b?c<Y9`RE&f4vRbpROXpZ4JmLzNNrUEe$|q
zJ0sS3DQ|c>IEt?wLmWKD&7Xw(@M4D~ZfMn+PG>iq7c1_mS}U+ZMV{8eFjDQ!413WO
zXZUNw1zh{b2*cGCtd+Y{&m(T0PQ1P<j!FILEA3oO7owJ00ya0M=xo7y&L*B7@_kJn
zmqK-9wqvGYR-U5xI0hR|ZSwj8g0*u^q!g?Grg{pQ+&ao9YRIOxuQ>-lYagr!z5C4S
zWx|EpUI?Y>=}ar!lh=B&LMu`5<Ll@%qaGUyTXbWbLU7G;Fe==|=~>duWm>qP`<hJ+
z)lA>&0fY3*qHl?G*gXBiiURn%r~%&8)sp89s^LX9Ik424fM_=(;y&>%kaI~&Ef|C0
z^^+3k@4PqsQ!|D$5+BoqQ21M`L*IJ`_HqaELdzt}i9hxkeQU-WP|2gB8k)iTiXgxc
zF5~gVMf&N&$8oX)LQVI}dsTZF#h?qYtq3+*@0OZ{IeRhMF7;s;$JYH;*!B!-B3A;D
z$@QQr5+7%D1f9m*_ZgCxsTNynb>T4m9JoiFJhfNY^xTInf3T`BE~%4%%ND^&S3=cU
zkDk=Qa?YWstemgeRr=oEL689iqR}LlWKBm+O#Up+H5|EIueVSlm(3WGV(91(ghquz
zM68--!@iq#_B=Ah0d)PDZ<0108B%p_BgQlOjZif%G5zvvooVso85#EBo<LN@V?^w)
zMHCkCx6sMi+RNRjW<v83fLZxd6#Sr-xEf2zYHJjFRsrhgnf_SXx90bZe)7gV4@073
zO#xAL^=lk9$bvt*$g+?@e<p-1JRz1<c`){yCMHF?Q$9raA0c?8aQ@y1vrx+j)>JP=
zH4YI`o2(T8-T&-BkLZYOUr0Y#dq>sh3Q3Q6v_B~F0$uOO;R%xQV|qgQy9s|)Ye>u*
z8^)oCKZ0*DYeLn!C!AtV3P&i|RXRH!{Dy)x*fciiVE*k!US%qCOCLFvz$e_aF8bGz
zz6cmPohWB2{zZ7um4wS3)$biJv5`X+3Afghi+)Uc7mlv0do=#56*a>JVRx|Z=Y0QP
zP*^@A&|QcMO$R5MmW_`y!}d<nU{%4w0rTwzc5;m7huJ+!0USI@_b#c?Bt3B&dy$fN
zsz=9aW)`zvGbTD+-V~Zl*#RBY6~_)j3~_fc6!!GMFqI0--8=2mlZL8~H87<vOUFP*
zLj^~qYJ%_Ss$v!Eb&`?(Y&mNclATNukCPV=h}z7+MQFM%sNP{on!skuKAG9Cb2&yX
ze>pIJOTBTvc2qo`o0sQtH~nkAvMsI4kNgdV#3Gh6_N9Zn#&AKq@l#iFXyLv5d2`h|
zh8g9qXWv;~cNS0T%SS?8oMoSQ=Qb}4x%BQKdzS)dA-Zq~=}u-%#u_fgb(!dJ10sBZ
z-yxPL_<hmrQI*ExGv7>yYkifTq;wVI8%Y3@YI|WJzWLOJ(TC+NaclbP?EA!o2(sMF
z5>_|{mB+HO3V^BfG4~czr3E%#?+0?Yy4Bhd4+if{;<)f-(RRV2La{ZbJcBB7sKb*7
zI+GQgV3swx<r7!FZC3!gTKk`2!-1!M?Ux6h<bD!3!WGH94#)<ZiTjIv@EDqd1O9ug
zOv`)`Dl;%z&ou>GK&KNC5Yct36+?Ez+mp@Wn4nyc+cVNuC?l>E`jKNN{m|86)s5$9
zgCw}D3N({LxU7*U((25$Eq2Ud5LI>%RU#2Rv|;#W&kHLzhL@F9+2&pH1R=LgyuHi7
zz@X?|yLVAUIur*)ol+cJexBg0d}GRpG2<1!={eiixT2Y)uXZ?3ta#d*F_5<K{mu~C
z7Wd>*WxvUD%=<X&W$fpHIC*9<EXS~s(`e|hSI0Li2qM+;x?B#5=sb~fz(w~v=u@dh
zdxk?0e)Qk2k~8lBmf<`;h*rc>RXR*zB7d;6qa+VT_&jp^f;j5FfCgx{A|~&*JNW*2
zM%b{oCf|2YP%9^i-jJVvdpqy^AT1f$exy#ok!D0>WO+-$G-v?eeBR=sw0Xoo-;4lM
zuTZbM<@zvKF|(35;)rX6kD6Z*bw4h~AgxkC*4wP^=L;Q$0IIsOgqKgNj4Ez(n@?9$
z!DsiHv}rt@a|>;D_;=<E;1qRW7TcD#B}BQTo!IGRzBI-a37f>Hxp(YTy8JrVd+~di
z_)dL79kWY_g@qMt#`t^03JnY+Lqu?TV`X}#I(c3-DbrO-hC_$%ys>CZx2S?|-qs6Y
zuC*+^$h)f2BFXU2)0gcZ#NpY*!~whs(7}UnaAVb<JiX9_XmIk?<Bu6!_7`g{pPzmQ
z8?Ub)fTn`|(#YwJt1zqA(0sJN`HnUR79jIl&c}<5r46;H6K~xx1mATD;l7*;T*YpC
zau^#+MA!&B%kPf;bD~3}EhovRp$DAr2JCTQs5zfw#`xyv!O6tYm3rjnnaLsawl+)c
zCqvcE8LmZ2hKU-6_u!*f>XRCM{!jZK<cJj1+JC~bQho-G#EW%cZw}CIXtqYO7C~XO
zH-@9j#hA!Bd^MAq()9fk5Ttp=<<y5NRGeQ>P-8ldqgtgq-s$sNZu1Q~U8kPeBP&el
zb4ys=8IHPX!_3`>uwXaPtvWbV0l>#6U8TdT^L`=)%hg&3-UY0)A(oxuAl_sB-G2<3
zA`-JHrzbutFIcti$_M0K)^T-RViJdH3E|Mw8W=%5VBzq?894#7Tr={xTpz?nHr~$B
zO7wnuQQ|E`5Z=x|<S}A1BkR4LN}y2ZKK}CQeWBnw_!Q?N)+1%N;lJkiwPQFmH~xZC
zGCe+5H)caW@Cnl0Wtxh>_x-!%0;HmfCvwxP?!oA$RSI&+AtD2xuvjt2O2j<v$9Q_=
z?eCcwy~CE=)C~1U-hjW#YtF6$Web-D)Rcf!ewc~v@o$e5xp9wvXVI;au69pJ4;nG?
zXfLC6LTak2aTyubGLVFPFX39WBqV*3*&u)nuRa+<u#9!OH+YjYw!yA&>lQA4%;lGj
ziJ`U&VV7i*u08L$99e+peq$R3ANn-j2%??a>U%8{-ny$!vv%C*uoJgtCNEUXJQ9ID
zSyD+sWybC^*^$XdWE4zxO(jfKwmDeB#KH}jDA#x8M$T-w#aPZ2xqO72ielh%;SaEm
zeEvdH>Iuu5z+luJdSc0_1Uw#DpvgCUe(K&_Vg;{}iXd*-m`ow9&0w=FN<%h|T`D>^
zBMoFTP46j@?+=nz=kco(ihV`5$u;vRJ-H@M*Q<5@ZZu#LzK0%jw(QFq&9P1*#97Fs
zZ<gTr=oISDZI4^?epjw<^T1~jMo^vFJ59(i1!;%J`p{d>mParHgftcx7q3Ps0s(u=
zyhxas{?^tqJ32XOG}~|gMxsn#cV+K2ZOX(Kz`br!(3>WAh%c*k3`tv^D71gL5E-`h
zsO^jH38Ll&wDnclx!pq>%&c#QSjSwcxp*mw-r>(SOImY#A(dZ<9qmc}G0bf(QH1Fo
zq;Om>(6%Kezh*bAv``Hcb&~QU7+VXTHttc`o{{L}Hjif?L$L5;!nH)w9V|9GU}Djf
zkpbW}ag#;cRIsw8#I`i&+B<yo)M$+(4B-I}mZM4Lz27(GhOlqJjC#GV9US||2XQG<
zztN3H)w>MIH@8XP=GJGaWa=N5HktdJ!RLaJ=QuxqPJfC%|FKzZL`FgB1##@uYmLJi
zGW>_eRy(|!*dk<#W@|t+TMSzDsY^|5p|I*yP6?q&(s4Mm7M4B+4yClwaqiM_0JztP
z--=XIlcI`Bz{XTCH7Ax{lVGcFND%qDNm|C5Jqs>l+w*oF78dTEJrq`Qg?f!)%iSV5
zx(01{<)Z!XAD%k2&c#K7D@@f$Jt2mIA85Z+L+`8&mg}`Rs`UJNQIN5gJXpt4tJL2l
zVO~;Jr=MD<9GJ42Fgkyw=4Re8Ec|ubCZ7L=ps<fBnQz4x5`(FzTc5?2RcSa|>FqrL
zWH}%Q!hbJ&UNTMcsf}v4<ES&ZB$Bi$eR?UwyG!)IE4?x;9mwjfU*9{W=4|v~^ZRrs
z!Q`Ej#?B-48He;vD-m|%j?i8nUqi(W95ZmA@G!8zOv<`gD5leDWc2&+E<;x>4-t*P
z(4dp=jl#sl96VdDTW)qBiKmcjOu{}w1V^qrw%3O{^Ej4R&{Je`c*F9f`2gZz9v;7E
zG*oaGkhplPO7NTnkRMQVVj8g(x%u>zxE(5#14PG;enOCWj?E1KCLfrJ1QpGhVYv$6
zOeD9uk=d@b9@Jzbm;&!D7-4Q!@;c6|`=YZT^K!As+WH@KC@RuChlfdw6G@<N+yU<7
z(1|k<)m~Y9B|Jv4S}T7O(Up$<C>XkG10^zy&x7fdwLF=5vCX*>Wm=Trx_r+x4rHT@
zYFFSV!w_$GNaD*I0*qg%>ywS8Q`-`f3B$?k&R@?;Zi#GXSa(QWm<?ZM-yRlHh|t}L
z69l$ZrR3x&XlNopstN?}HTQuwyfZ?oL=r3(DedY8<(njAWExP4K^^nt(kZ{&+S(do
zj#l-S#4@&a{PD}jxx*(WahDl0R%GtES2Fh1+of3vMvC0+pzn2b(+a8S3-Jn%?4J`w
zkZ4Xy-nc_%?mZGzjx0EiOG=A#KD3z%(qX0+aDA^l$BWz?eXU>$j(Lhzz>x(m<<?vu
zMT{{>)-z>;Z-g^8>F;^s2KOhjnl41R4>>55yqmum9n!I93?IA|qJ_^oncJTC0QZ-A
zQhs2n2EQ<~kKs1t#>=uUR6FQEZyM|7T0yX>NKMYIh#Od9hrr-)pN2K%@hA-g=3pd8
zaH=@6;qU9t!D-@$jS~YHAH>h9`w)0M|NJ7A*cGDUy@78yH)@GZ_xCt}BPJ2X+P5b9
zc^u8P?b_R(Xnr3xCpu|x|E95kqeLH5Si><Zo>|`&-Rxg`IyDWh3;lL|PL^pxK#z)2
znoG-?G>rTquOyXe#aVYKk#Eo*(Vu{$k?w|IQ}^vLW)TVb%%95=uPM+SuB1v@{W3f}
zyj+!7Fkj@mor7%l5B{B$IKluZBuvfbC+TE#)w1~;5TCr(``OXT#PJ$%#DZiN0sWP-
z{<9OQ_#vE^-I@zHN1@wq0AkNcfS6LKqk3gIfwhPTPIG>wZWgrl<*S1q;>GLvgYS71
z9{S-Kf6(=AGP%Bl^aYvYnlj#Zev#Z*k}hO@>_t#^x>%|h^B#{sE3jZKL#K>sno3Xt
z@)1rn=g8YkJYlxEzCk9cG2#<wwo^<D$WbAmyP_u9x@s8pBzxaK>s{X|&Ze~E6iwEq
z>bAwtHCRDGD;k+YnHc9nNU|)w4!;)vAz?gyyNZKqm=Pr(<IFt36U@kW$jnfX19Xeq
z5w3{f6>|2g69@ZgrH*T;`SBv4&1Od!$<9!QRqxm46$boN!aeM0SFj+@m83I%J@&k#
sbJXrEBjUC0onNyKHM{wg`~CIHi4Pj$qP27C7tljOR92)?NdM3O0jal_k^lez

diff --git a/modules/web-console/frontend/public/images/page-landing-carousel-2.png b/modules/web-console/frontend/public/images/page-landing-carousel-2.png
index bb48ad6f3d3f45578e4d6bc3ec1b79e814c098ca..375ea0f4e8148fb58fb0854c3c94da713002df7f 100644
GIT binary patch
literal 34564
zcmV(|K+(U6P)<h;3K|Lk000e1NJLTq00K_{00Hp`0{{R3Loukt00093P)t-s`1tt$
z|Ns2^{QCO(@bB>M?(XvO^7r@m_4W1h^YiTM?En4!{~;aR+uQ#D0BB`q<mKh(=jYbe
z*3He$-`?Nh;NkfH|J2aZ$;ioFTwT!D;KtF~s=LYp0Rr#v@{o{`v9PeDp`=evQ2!GT
z|Niy+_VWM3!v8HQac^)XBO~$n_j`GH=k4?O@$LT$2>)ba|Jc~@{r|kUx~{*?7Zn))
z%*?~U!v5{m{qf@d^zHu{7lVL-K0iX4m6wNyiT}8^|2H-@G&S<;=KuZewY0bYYiaHD
z_}}XAoSdEI{r~&o&8w=c|DB%yT3P>4PQ2sw-tqbV?)m@VxBrom^61;@+|x@$Lgn`M
z|E#O0;Q86%=_f2Q<hz^j+r!lE`TybJ{L!obL_yT%@ynZiyx8dt8Zyb??8xZ&HAq@~
zQAGdg#OcPWuG8qhrLO;cd;k68(~n_Rl(^)|yz<hwaF@5xyRnI|(9^D${K%UMDN48R
z{{M%D-l~G5$KirwQD;&@KPw;0^!|^==I-3)Cp=2z&ds8FUj6vimuN~dC{h4xx!H}l
zfTqE<f@5MV7+OCp{^;p9n8^Qha+=fc-^Si)q{@Dbq1XHVr%|4|ppv?fb1hGA!nxda
zKru6OvjkbOp@w-)D>XF`8Doc`aI4tpw$e{+mG!rVNLp-}wb_rEutq^+^ZM(y$k*Ma
z&Iy&w1cJo`P<xMbX2#0rNrR(kcZiMM`iN67g1p?UlZXmJUwk_jMOT5$ajpN-@CIg;
zHExvWvbYO<tgthVtSEd`YkNDh-<q?*V>w0t;Q3Uq+$mp$o1noGEpY!dRK|Wy_OqL?
za!ODZ5Swtd<eq4pERCEFc$HB~v5CRigqWgbiucCf{~I`9b-2r3lK*0Qe9-d;t=zjx
zg!{5>bi?KQoy4z$p8tDRuwO%+Qz>D=?Y*MS|A@8!u-pH1lK7&6zi)yvK8^m9Z@5Uj
z&1f?JO(p+5f38_4|7|q@!~p-Y007Y||H2-K6=VR+oB+jdy?%w!004u(Nkl<Zc-rik
z;m)%l426mQpu7JY{ysRbc>+Z{f?2l2gO1XkRaz>#B})$;+>4xYPHC_0Phgx;AxUVz
zh71cu&%)UFFBq^;z{;%)&mw{`3m4t2!@&8Pa#!%d^k5{mAU%87wk;tGX>+WgshD)}
zT&F>e>By2l1cme->x*zcoo`FeR$41dsO3YeNXb>y9QJxrQ*IB-5~L_Cp)Ln3>Qg7i
z+G&W29pi8zJ{RKN#hGW}AC{U}GGRpE_7Qc&lC=O9MgkwDr;V*mkI$VLxaec^fnyn_
zC)eT-ltxO79;m#0g#?W&HbU)5UViZV;NI3kQv+?^Ra+VC$C)CicsTSS^$6*`iO0s6
z7P4aF2QD<rg0`NX7zvTpme}o2Xm}W^uG%wb-J?w7yKwjV#1-)c(-BSG>U`G0JW~65
z%=^Rn{<O)~s6*`CC||*@R5r;9j1IQWvh`WK7s#!#tq1Or*s?~IuxY3&Q>SszaCv?|
z*=dFholcAZ19Y2Q9`D~e{*}>YSJ=m9ld;7s9L5I1z~}D?EqC`jr>l!Y*O()S#EaSx
z25LZ(G$>F?q{Ij-!%~M)l4IfDUr2yuj)k!_STYP<T*oFqeQXxQ78p+h_vr}&w<tZ=
zk3xO}R!~4}OjG+)RwuvmZ`?XpP@;be`6p++7Oqxuv|GD!s^Y@_#GgP7?4puhuh${z
z_^&D2B48~&!V6Z_V=WdbB-dV!MQw}{iqu%?MYJ=PP92A@&IiVx#iVh8-~+!(kB!sg
zuSDR;q(|^BB#fcOJqqmXdB~r2bNiFfKZN@NQoaO39>M5ugT>aE^&QiIjc{RIhi?2a
zva2fIqt&xh)iZ`_u|ZQy5B{-h#=LFAKoqtOiW08~B03h5PH8&@sHYBD(qt--0$r+q
zmY-Zw&&soOQV1G_i`w~QQ9Ry-qKKCxA?SqAYvJ?;i<-)NL(UiZ)YMo=HlEhmNVMz-
z<%F%G=yXm+O?5GjqLW>ydugW%p>sTZqPtMfrk%WZ{`9!S^dz2bcJ8#Kc8jkP7EsDM
z0F|7mCi~7w;h;RiEuGrLN3!!1<fL*_rl`#P@mA2kV1A6|B#%9uW@fui4$e;1Ku*q@
zKw*l26!Ol0PO*pC$A;gKq0E%7Qu&GRU73<9<Ld@&Ub%<|V1>0QCw+AO<t|9N|8!K^
znK|mipDq8vb!F4j^wdoqe!#yXUV5lFC9X1%k)F83Ri^Q#cmqL6Q(KaXLWxK^SxMiB
z715@4a$tZ_dH}4kphc6`8>428^^EDscyMJAQy5|=_8QKPmqDyqS^Fq%o|v6N&$U-M
zusdDrY};LibRmpI@UCSpicWNlf>w8{07{t3iHdG_+6n9&tvy}bu9$rLHKObavX~K4
zDm`^z!ksk6WUtO-WHU%3N5|M5rhQeBu`Qa>#2O3!jWTKN`qJ^0<z16~q28jL9fcl%
zwnjJ{>g|E1SH!{j3GYz*9O5_Fh^G*oVg|Y}kO^(9eM+H-9ceGQvT7%<G-WzSip8oL
z&J?Q2&l5P!H0PXi@n}i+Rg&3J=mAVerM_30XB9gmIr)i;9sa}og_xY|B=rFCwXcaU
zy5z(r(gfczDOMMcPq$~p<Bp%;bI-59r;Z<yZ-IMob8~Zbb$z{F`$c(heySq9oQkNY
zEE>DmLpD;bXu2S<K<&$*`Oz1p6@16Sm+Q+fAJ2_Ta0kAEhlhu!0A&ZYMVUo;MO`(6
ztg6H`4HUSiCR2#<YZ2ca#qVtZb=f{YZ@2Kgv@dP@;t|!za`_>!>@Lei?Ye8<Xd96b
zRKe^)h}(EM+Tm?^`~C9FxTB6BBr53%Qi7H`qMCX+-Cx<2!wx+^KT%I{lkta~v*(Q(
z1_SW<<R(rc%iqe~lpNDZhlFH_EFo~iNgx#=u_AFo`~{Z&0d{^7dl!BI5(7JX3o5*}
zn{!0qXa~Z;6DP49r(gHzeI55&E=BIWxtl-z^y#Vb5<M1YpFf{*sORE2zxw|DN<4>Q
zP|OLM0|m;H0#5UsAUMr3r~-%>7{kpBxiLvuQmINMiMxnW!zc79%O2iYvx$Jf@XuKT
z?y!krsn-d%J~9DbCKsJ{z(RpfYN7Cau=Mn^O~0dGdGb18?4QmI_#}Y4`Iqg758F@X
zu7Hlum%wN4>iqi!@crF$0YH8ORb~ZJ9rt313^}c&E()hmriE}m6;3j{vfU}9=_ie|
z4W$}<nhKsw=hL!1?U*y_NyCO%fJHVNe_GiVF4c^+_6gkLT5ZFArf%77xwZ-LHbtL?
znGOg#0lRv)5Lga<3WD4?K7o7ArrxI<FfzQ#`$yYn_wGI0KHWbSk6&GW11`_@Ie&h2
z@fEnZI?p)_(|IvVXGpdU@)IPH6mTpOr3+}!6Us$ybXq`Pwv*UWRG`w5IkXAweR_YJ
z!|ksGpOS}n+z<J$ZmzK9`n2sT=aWs@rh;?TvZ~rJ3Hav5cS*TnEsW84qXo7btkWk?
zT#Ge|weB8J&VK;#K-AF1{Jy?X2|hvDjMJUl96zPI(_N{wC?Vf@pZfKwl-w{tk&x|T
zvUBLtKcGF{f7#x9_UzvF)BZ8ve-1u<e*KuAe-A!=KYzyoniuJ;P?bXADU>Jk6ezM%
zryv^m#D$>*I_JbHT2n-jvZ-rl)CfKm24nDP5|-6&`LJeyB`ZQS(}dK7;1+a(lWVS&
z?`kXos~EAQv$!TqE2SB!8Ch9ENQEjXS8&Ig5J7B1h=lIuw)3YLpn_1I_yjl%P)hBZ
z#(JMlX4gKY{hU0FHXWHi4YFMhh@2|#lh}Xx2;nLC^!n`JlZif^bAaY^g636)5J?@u
zCz2w*NZ?Dr&L`db)JX6tb3V<c_K8pC({fpxl9n6q6K>WZPFq!tBo=(q6|(83!KrRQ
z7S`6xrZTmtvc$Osu+-L=;l>7^Fow&ig<hgH(%7zUob5t*`elHspo}L_yVAPV1Er!*
zgE={rz_Ilw0>sHvE}Tz>8)NwX{)g?eXWQ+gJvT32US5Jumv3Hk^RBtL0G%$Ha|3WH
z=7b<q)d+c!Ii9&Q%9U{G6jK=Ee1asL?Pz8xLP%ONgLXm<pKLQH)A_W^DW}0FVwn?#
zDQv+f0DK}YZb_oR9o8~dH^idO`=miQ7_Z$;*FF(xZRI}-YOL)P#_SgKGx8Xqyie`u
z`7W#$lnp%MH9Gi|41O6L_|yD}M4x({xRAjo5bEiN?RNX+g=@Vyy9CZ&zkvF?s|(=j
zTxbJGF(VlW)t-_$RERlAvy9Bd!W6D@B4G@r8PsRlsmfN1G)bE}g?5@qZ6-tKCzJVP
zu@pM^WU<C;EHJ{l_o;#|>vn}*zH0^6%*~(HFr@2<#uaIq#F(s!b{S4O6`f^R5-o&g
zw_&tkFiLn&qEM?wLbju2<9tecpVFYy4Z;&S=#+%XlS@BRY2(l@_HUm)eYAg}HPm0f
zJv)2*+SmE{;_6DB3u>-?f_#}FUP*NDDNU3qNalUg;FHW;=gf9>wStF;u9*|>6E)gz
z6BM22lljzk@KQm(ap`PjAw<}k)pgx=0JG3su?o7F?OHfOt$)7j80&0nwd+s>2CX`_
zayPfW*ws2dAl@S9ZN`syql9<rC*%znUpCi1B^l|7J{^ruarP8+65MdPxPA$=*#k}?
z*M1Ec>ihXQz<u5O1R^H{C3yxAj#JmhDe>Jn_(Vtojf6^0C2am}2x%s1_)Jr5G|xBQ
zr==^f#}eB|uHLYCmUn>VP_dpP62<{t%Ya%8FZ|*TSK>Zw9*s{?r*XckYwPFZK=3IW
z1*oI(DJ*3UvK^>XE~pv4o9lR8OFAq3|D!@^25U^X%;5|z=!E9nl$vS^oo@92_4uKM
zP;hB7cwRkN{#)2`LV{M~Me;u5jg0a>LH4kIeH^Fb_tX6`K1H8;owQL6)f9^EjS1VJ
z9_Xuz<^!ZEc^)xJ!XP6RKM5V9Yk>aSJ8VE+t{>d_uUJNgY8|{9O{1Jq;WsYaWv#=T
zTGE*Dj?vGaqEDLY;FF|7<hT~AYB8YHa@f39#&wQ?Ym*>pATL)>R{tTVo;tAe*g$h2
zyjn_?Qo0|P82l*=@x&E_<MF9acTOkn5}o%6kYNuc9AYI_j}h-0Bg65q(Z6<%-v<}}
zBl+(*xdulL5+wtlvN(mhQGCJ}AHtIapGG3|&3K5_n@KfB|9nCIo%7Hss{cEFuy1xb
z2m&z}bPFNbG$f=OSsy)vXYl@K61Qxj%8&TSb~K(~GX2zWRdm9NZ6a4Q6H4~e4}J1}
ze(u`jQ>18|1^S?!P`-&}Ib5SulTiDQxJ*9fJheo(Vc#2PAt=5SYlZ9<mn$pPD5bPe
zOMUXweCpVzNDpU}3?jniFw&=d?s$z;FrP{X07pPmDHi+0b6k8P3i0WTl0gRu{ND6w
ztxvVgC-cb|0D1k@r>2(qMCOx$fIWgspNeX?PvVoDyBLD`R62l=gxd6J`S&N~6PZtj
z!*uETcczZ^$@{#Y(tKi{?#?J1KtKeZ<wyD?|K6oJ&R_4$r_$j*@yR>9pEyssPv(>H
zV%H3I5(c6u>YEk>UznoC0t-@Nq8m1b1QUYb#szyeP5cyo!Y}I=@G*AkjFifPz8WLt
zo|HmZPwu;OX1)q}`fym*KNU_}OdZF1I(uvtH;+omUY;@>pHJ7wC*W|tK;WNDK4Cq9
zJSLb5I@3G0KVja1!+xB@pMAVsf9fEgKninyd&kP)mGI-a<CY%s2@r=ap9=F1{RxP4
zLB5Wc$*1DH>tQ_sVh%Z%Pb|r&uIo>Itfw=FRkoVUOP)_%^l@OZKGVEh9WQbJ$%iih
zhsK@xa(EK|gn0)H$_WGB^{3K)9KHbL{Zm+Re}_H}EF4;8P|iQWXM#hpaskC^zmVhn
z4pb>nq1cZ*pNV+~-2F`R$zWkViT(IPJR?vW%&hyR^ShoPpLm)iia=dpTcB)zlI`#0
z=fS+zdQG4<Sd~eV{#4yR@kVPM6Q~W$%8t8&A6I|j$77B4R5QM(9ZT;Q$osgr)hQbS
zwSjl{eM|fCozJ|J+%Jfq9<^bsXrJ~aal1|AHJMbC7nAh&MU%iOph#I^An8wXpXqIq
zk635Cy1(8QlNM2bxm-<xWzUc13x73!jt0xy#wxnOHt+;+=XZj9YE?SF+vsFA)frnW
zp#1mYa8AvjGkJ+dSNEf+5kyh9Kc066s~`$|0{@>+_Rcl7r7#S@dr!yIXQejB`lD$b
ztdmApL%XimrALCbm?l*!VL@cXN+gttTf`+nT*49}t}%baC2^M$iF?vCBJo3$T3B4-
z5{u!d@B4brT9?i0m{5IU=k#2hUr*lmJ>U2CD`JJ=aq0a9ndCbs*2p+KuAn4<(z{x%
z9s&7fgL#`>t5j(n$}+3H*T6CpaGGkpD(BNbfH)Z}3_ty;`8x|SDF7Zt5>JOu)d0Lh
z@6hzunEGn04&`_UUo#l&O8fW{8JP4AC1cl*mqU)f6*m0~QyKMV$|dxVQ<0)OWKjjh
z;7s!g`sVl1mh=KBP!>R*QbsB(DCmoKR<+uu96^#|j=vNdo!HFZ?>ftOHP9`*3PK|o
zs|2A+Q$FDjMK#(IEK?G>lIIKaA?cHP5cX+I5wT3BtkCj>xh(isfoO0_mE%a-_wP!&
z8J~o>6o9dDp(dZz;S)0+D}zy)5C9bLHgYh=Q3#;rR4QfiVj&dwryr`ddC*f-7Ec@A
zn}Oesr53Ulq4x=XOMEKnu20^M!pSJ8<rRHm9BL6q0ra8PP@@Jy4aadwpMrgg$)2JI
zsTdb&PfmbuK})a9yVZPuCHxu_1A)>XyEyFmTNbjFR6kt?^^@fI<WvYWf~7%087uh2
zQ+aG^f)+Gn$EnEH26~$Bh-d?QbMzFIss<!6Y~l7GcjU%*-h>`qzCC~8&UL@`(D2ZH
zF2&osKfD%mX01<%szjt>CG{83<$b&F6N)MXChY)^fqd8{_(Vi0!#_9YCnCv-Fg5uF
zMb#w7b<|IJP(e~%QLjM`VwJRvH+B1&xg%FSI5i5s#$Dec;h_evaYvit^?x@A%Mhce
zRKKPFsk^aaR$L)a<pk$YypK|!Y<<U2UjT|pm?|Ms$pC1a8t)K&QqoXW1ej`4o`VF?
zsufQFLZ7ZRfpax?v<b|YP9EZxOfYY&-hXsd*xm${+qk34_n?`3@wI*H-Is1}JJ;mi
z$KBlB#Y5ZT5t7)Vcc&(5!QJo%U}AaQ+ggZuV&xG}D+}(nx<mw@eNJ}S142ylA26Br
zXUZkK-vJoo3XulnG<5hh7>m-Mip~**$PogdS<v3$Q(qZJj>!*4+Gtvoz_7(<)&Zc2
zGuY<CC(b^&(X?QD@aF2^>B?=v%SR`yI<wOKV#Q6>h=cgPm9XG&OQ?Qc_3_~wN8CJc
z`4x1A`*VVXyI4H(RCgCPwZDd`?a^I2@2EC~uC5l>PwvmIadq*#=JsfV4&RYIr1fW_
z49B0O^goq?C7}@U0j{AE)F(elWEyI^z6+o7@ixVHkbva_j!cA5{re_pe(+?$gC*>+
z4_BK)^|t`9PvKw-uB8N@?qHwr{jmr4z<gy1`xBpLM<dNipKz#(u;F+F$1PfqhjnjQ
zbNlPrhE{e#`D4f$g#JwOmvR4-gz?N-WPBhX7EwcuIQ^AUpE#}P6KJ~dsrOetX+Q#Q
z(WeFe7Q8voGW9U_=)=_s*e90w^wf49hP_Dh=@dSF<firP1N<L5!SXplTnooVpDdr_
zyLNepwzhq&r9M6FmuT2E^rKnVuhEIJsXAvMT_}`Ee3CGK=TW#y0e1MRgp3E#C`!gB
zljsN2zYCx8IFjZ}Nh$__1c)biJ9C+N4fbhM{i*MLTKTn4r}616=QIw@=uw?M^m&3w
z=DTK>;8R^=W1}b17QLJFNrw-8Z`hEp+)$oH`A+Eeg-qj<#Pu+6t}}SY0vNr?mpt&{
z16n_kGF%0iykHkT(R#cX2m^`-$w3M;=N$v6ymBqH<^~Jl?avl`>(hoO9X{b})Tehx
z&O?_pSG@rUwSTOAD&)J*AF+Haq<o6tLpJ<Y#)hj}vnW6@C|N#9{(ej5^LK`5Kqm7T
zh)*(^INyRufnC8so<`ldPf8$Z(cc@P0r>V{F-XC@dEVZNmn&|ehrt`mFPvGIicdSK
zPv!%cw{Bc}5Z6rCwU1WZ3a!CDnJ-+vz5LFyE*{!m<qNdq>S$^|Z@_DK4PiyX+2*Yk
zeX8siZH?7Eo{i5w*JW4zlxkNYAW8Fg-StWARH^dqD8+*XUM4jkJVcEtv#Zpd_(Z~%
zURzj=cO&%HG2~{DM&&lY|HOJ0+;eRH`_o}!{VRaQ$0vl&9s`ISUkS78ZQS95?65h9
zZAVw%3$4cW6HHuf^B<qw)kDj5F4xfVT8Qafjowy3;X0S=HT#a#Pi;OHCVE}2#9b6?
zcez?eLH2-EVn7@)KpDN?N$IB>E7sz!v3RhcA-!LvOq^gz_em@Iq~z#DrAaN~SN@K!
z=OoMz<(~qTFAt6a$gFT?S+<#`GL{vOAsvPuvUypQ>}Y%vTQq;AeiGmBGVk|Afre<*
zKqp!}LdS1DNo9CvRTKJ?ogM#i#HT`hO+(J)SJ0~ifyegI`vySIf>!N!T$tDLOSQf+
zi)Vg+@&gH(&fiI2-`BEZY^leqs4^yo1A#H|ya_NxcB_MDNSrcN@cdu|L@1sgg#UO3
zitx5V4lsbXTCMgX26~#Mm-LQ{SrtWk4ysZrKK=RieTN$xd=^MiFu0_okQmfUjTr=g
z1qgCvu>4b6e*u#GewQ)G0%_2ZB~>r^06*T_dO?nV3Q{~tSbw|+){7pZCfMk92;}(3
zNdJ8)l+pN;yl2+;MP+J{vRpIozXn2L`6PXOTzY(>$Tefs1Xay{*F2O3q|M(Eko5Ic
zq*H9t)|dw7L5_b8gxF5vQ*M3VFYKMMZUQk3hEHmx76o;L?&x5GvM|5`3mdSoupwT6
z7wAizc;ZG1C(zmIDT$fx`)*)hwf;E29UJ1Q-`>goC+7P?>ifG8O*L=q=Y;J2ErXCh
zG2a&~`k!2;KC%A}p7W=H!TT2Usq@tNlZ&6P(qrE%sQHs;km5h}*e?Z2{^SM9>l6F$
zz-E4dbNT(_-0z^ypTdI9ocX@&4~D?~ZWq3&`I*o0gr5lyeR65@pQzt~pt)WuLZ9$6
zp-H0}%zjQV-OuTg=KFHq!Q*{P_^9830e8P3{6zOZy;E-isZTEYeK39|=s5p*TsZZ@
zKyr!DDj!eOgQ1C2Xo)Aif4tVGFX%3_*=V#{_4|{_WV6{^E{DTmzyH~8Z@25!k54-u
zce~x`bXu?1<MDX0n9t|u^SP?3Wm)Qe0rAvn^my_C)fP`vEuPS)uJgY>{i^X)d8W%!
z@yF4pt`nnA3vu2x{LIkryDR|WcvI$K1V58_f_9{tS@)~R^R6raT4G52Rr{H89>!+@
zkfKj|{qCaxh}S259tQCQsrsbP_bmXD^hx@e#1kazldRtnPk=r_8us~<^fSMSCos6*
zk&i>JKFNMx^a%{}ry=?TX7@Yv2{QAiho6aG0Qvb7`8ZhSPn>t~GJWE_17m-Dh(1B)
z{SN;L@Sh+LdwrrF3@_Cu>cN0MK??TzB<DrrXF`TP;TM3VKGE+BmimNW@W<ZS$Fxz#
zar~Zp)Phf)o9O9`Z_t<Xl(j`$+n&xAE6P|~Emf>mh2oeKXYpNT#i2o~$=piKG-`Zb
zHeX`Usd2uf8WUqWBTnOg)bx*~Atn?e2@8MvyF1`0*NtMS8+u>ip69tcc+T#_@ArFt
z&(o)>DrTQntMhlN)cHGAWvCMQr)o7{plW&SlbX*|rM~~}KPAjmN_d%j!$;H6))%Pn
zzx(%vvBg*!+14@txHnsYEOTm~QDR2xL>cDnle*sbxGU&3RYttoq5uke-8$2;q?2~e
z)UvaCXUdyr7VgF>Z=VUcB_x`p_(@&=bR5OFw7j-LGBvHk3wkrMI_<F}lLRP9EX~0r
z@5L%_?knNcZdo_)^Ip|_=5Z8b=CzemJF}p-sBTf3v`gekbb|>>&<a}btEp0`$f=dL
z3mW6oX)7k3Wg$;Zi}%I~Iyz7CX3?^{gRp9zeOdwgVQ5yLQB#V}ShLcII?;Gy=leUq
zf+sn8lI@d%r|N&x-yuY(C2RHB8l=!^YZRxd&R#>UES+dPEfPN!Zf38vPv!52fsr6Q
zM+=CSp1mkIhG#CjjasUz&d{KectTZYtgwxVPc)v`rNrY(`&91w0u9nXSX;aCQ$!@T
zZo03w_Ph-U>hw)D=N6%_=UMBH25O0_u0bX8glbftDrujBi`b`f_rpN<KH0``;j8xE
zK*Q7ayt=)g;|A9D5x8<!+aC?ZcshT#%F_wV6T1jc#p0)SnJ2~mzDu%C=%KH<@VmA9
z-{Jnc8#9jm9Ji(U*?#N59zvn}``^Sx?bGt}L}U7BJYlL#;lf50h{DH#CwGL0BM(L8
zc#?^V_(Yz3O-mZzl{<fjJ+E3n*$U<DySPD&9{Vl)*^Lx+$98VuRsuY|feY^@l!o!l
z@I+{hh9+W#5DZnQKm$u|6Kg1%J_=7KU40=g<muYFy1FZh@`M>n(9f2kNYprw9(?`Q
zgF{1uPj_l?g?Ms1y{7qXKIoMynVb{u<TH6kgkLtE*hHwHw@h$i#@~(DN8Nx^s(aMG
zg!rl4^}dY#SL?$Kn8NxO-ubq{{+#vO8wI!5x-0w!qu`0rKiItf$1_BpFu8f-`sVX)
zM3mu?%Qrt*+r05Mbn|`9%^x+u&(nqnYu7hlej_Q$Q`JT*cV)HA)2TIQ*Ij$Y`isvw
zr&_@iW?U?P^jBNA;2N(}{`K<On1P`rk-nh=*AYT(9o%yXT_K)O0Z;sqw0;^&or%xs
zM|C<=Chzg{!U+(sg)iZomm1PJC`;$b;F{l?DI4%i0Z(ut)idgLv;K4(cruGbC8Huy
zL6GF|Dqx=&$FJ7CcOsgiANul9=Ao~<-WQ)={|oochJ`#~&lCMt>-pSO-0lWifBUZU
zt=6vZI*EN}*Oq>$zjo1fIEHsLVC;PLH!JA78;Fu8uuioXTdh~fJk_*W;n%|V+SM{o
zipdP#B&d)dpN}o1D`NH+!!9;sjp$7ucM?($-}J!)6s`<Uq<|;W7(%@!t;wH^=Jlhi
ze97^V9^i>*o1#rgKbxG=W|DrByD6G<^S&l1BDzeInJ<|qX8}(<^mbUwX3Wu8$}yVB
z#=L2}V4t#Jk$@&|$Q6nOy<n9>ArYh^_NmPMef4*})AhsU4M-5gQa^7J5n!I+-g{~x
zPw3vA*7rVGzugKvk$d~CKQzC!)B4#4`|A*ZHSZ5|)`6Fr@8Q0Bk>KCCZF?{J@@?yP
zFJlEyCtbSss@l`nu90~{8@Y??PG7s0t5NtnY++|{8WzlS44Sfr)1J|JZJ${?hFXhQ
zGoBAIl&GOiglsx^@BpE&gX*&L1kA6}67MPifA&mf#F4g7>HG<VkZ0N-fI{F2o)pwE
z%pB&AphJ94Xk_vT)T=v!x^<=S#6veaXJ}q;y5V$7phc_8rV%2J>G1#|MBu4u1S&1b
zu>eie9o{q?TSCH-ZeM(SC)=k|4HSTI`seR6geu@^%i{}p!cHO70_e7bzjHii{pM~2
zFn)4F{Ppky4ao5vw{16E+_~*hWZxCuy8)3WIH)h=32>_88aJLgKYnWFF1zxg=4S2+
zW&9M=&AwDe2m&3_!u+}99e7hF4S(^1n}9Dk<Q>y)uurn<kPVbC!HC{&4~^>&KD~Kp
zGlqKAGV=u82p)vA!!b1Hfat?x`bk|5O`1nMW-w9N6!66R1Kp$Jq<1(V@HCO7d;Mej
zWKN$nr+TKw(BV?-6Ym#$OSnL@caNh{@T!@-KH8MFCr3QdSO9qPCz<Za5&c9;@A3u=
zE^G|JF{QT|15Sl~Qsgt2x?Y8GKC+$TxW;d`(#PhBY5BtXKm!)zapsP%yLL3J(*Qnw
z@z=NQK@|Nv_uAtGJO1o?nCWdCc#%-p@decJ98WS%+}e{-Ay3y{xpDpaB0OpQ+Bu#A
zc6T-q57}%T$-Kofsmod{#CgavnK5=ujae**as;&ojT3CnVwqr#@GAAPvRkl-JPmHf
zxPDVL*|QZMZ?Jy3c``-OtjKJ9JcdS(pbkE-XZf5bo9}^mT;z$_ryCB;@r3NQ6bk0z
zTHP4R<~?jF@sq9z&54x=gV@YGj(UevaW|W?2UFb<zv%ChX{W^)1GZw(kjtA-v}30$
z?Qn$x&Sj3DK$wtqH{bUp$34GwF`mBedX%BC$kRh_hZ_ZPUGGag;d`w=yi7|x?d#w2
zI1P#P`>pS8ka?1DsuOTJ6_v=7ft}A2Zzgf@2{s#YBF97u1&Gs#4rkKNATbcsPNUWw
z=@>&!O!lRblb~t&pT#EfG}H-?g**2QT_QA=l_z<ePC`OC=YWvGfa1PfoYe|Eb<gsI
zCRYt(fhV2FQ#<^tX5l8CFJK9DlsLW<hH(a$+#6UuVhEsa7}UAl0#B_GpA<jEMw|va
z@H80>2J;4|z*EZMas`~F=EqCVXI4Wp2Gjb-c5&ZBJie_DiBJ7I_uk7zc-p~W+#~X|
zGh8R?!;fZoy50K2Q!-CaNj&X+nV!!Rx$t5cC#Ao;?3g^Ig<r5QhYk$~dfYjE3uzrr
z&+ueJqgmsj5zq8EX-(<Dk#wiyQ(kmfj*5+veG+&geS_CimEj4*7sTVbEOG_CE-*dB
zKb!}iJdxoM$v*W?9TEIpM?~c5FdK<v@^Kb$Lugd1{b%v`7=hmM9&f(K>`&Qaskk{Q
z@-$9|%p;^D)pID;4lo(L3E;^IJSpvyGM~Am{5ZaO|4W?&D(HoG{@T#`wRI1J;jI2S
zZg0cU?9*E!Pe2}_X+mgwF`gbL0K+d`?=}?JC+hSoYQlii0-n}$SFXME;){#fC-*%2
z<O#VZAdcPV55(Cqy@T{-QX)?&eLETD{r;55Q`$3~C2`;h!jOgb3H%-W&u!kMfum-y
zzT)FM-ks9>SDD<nH5kw*5SboF`T+9ybv&C#?Lr7Xf)WrELeU=3oAW&^gg^wS`bRwR
z?m&FXq#H)YG2{u59&?U310*i+6d(!OGm)e90lFnm8)#Z@OnY1|%Al}MW#-4py}xxm
ziU`GsJUO1VzIFq`47$BByj#paAzBcYQdNIiKWiXN{5jyM<<G7M7=jU^i}2Li7=|-U
z_s)g!6WwM#{nFKn{P-2&^96;^RK`!f`TkD$6nNkXhIPRaPb=;Y2AnRY6^=O+0#BFH
zPmGb)Jn`mGYavg(+ro(Hf`eG_cbytagAN?fR4D%xGRckLB*4LrPYh>`u3&H~GY&<}
z4dVrO$QGhIRz*U#V9>nEKW1}9-PuW2yDCfeEhV3cj3&7e9OZ@Mq<`Gz@&+UDTcPkl
z{wZi|H+q|*Axpb4=(Qy*Hk%8+To#MMK9#w@F9QR$Ejw;O&R5<R^5ggJvcA{H2#ud!
z#=z5e^{1`@)z7BV2Vi9PLO<jSg!+yR`iJ)KXi)Ox`IFnyhgzQGw%kgn0)KbTwHwzU
z1yA^#YcIR(vTJLIa(w5VH@@o;iX7?%rVwh4<lzWo7C1W1;0ryzJn9)i9oYav4s)Pq
z3doytcS(cDH5r8PkZe6rMUl>-&7>kcu}ywnZkbHdkvFlzQ7aW<4#N%34#uD?{E<LP
z6VCHGla|d|$TYu%`If@o#A`YpKz3_|DLj9~9r61>=BU}6OeSMNAw%w!5}GcM(ms{>
z{$O&)cB>EyHFBTafU*9Ger}+yuAd8kj40qq3X<AB-+)DjcMsg!*3SV?jQvUB$J(sl
zK4sszZFeD0^h2-1lyVyfHm=P1V4RqHxmbThDBOPd>-NP)-8wc3BVl%U+zc0n$72!R
zosId$`s5UNKjF?<^TZk%>DLJq;p^%ta{bWLxI*)Hy5jkKo|g-$Ch!D>d|7mmXy20N
z?}|(au*KGAnuYlm&?4&#%3tqGJ6>snf!e^s*9*P?KX?x`u4?24c3*;i-*r`^sJ+rK
zjkG)oH@^GoZ;v*>kG=4=aAh;LfAN(#z6*aYjPHJZ9%6R-=KV1I`>+q=GI^pHON1|c
z#f5?w8bS$zgH(tA<hpcz7N^kMe9PvcJ)QOS*Y^QV__(hxU|Hokx4Z25;PTc#mA$_&
z#)fNFuc^X#rghEgdIoZq+YTb-t#I*_xf|EjpIj|9ir#qrCAj?cDgcy^<#lC<*G627
z{9u|*2Zsg+`!2x$+4U-{#8VmT$IIWZ2QO$acWnRR2D<p2C906L>a$6SJke(t=B*ix
zE7nwoqM-z{Am0Z1zpQ^+>iY3Yz2BU<sgr1^Wz?Cc6#?qZQ<XTKb{1724PM?iZT}C}
z(}_ftu|BgB`(ZH4ixX81m@9Nnjq==S)zr$;3CGhS{_dFeY31yPp@HWsK_{=Yw@X!)
zCtkr*vIKv3Ec+B&Bz{ut?|XvsMB&mztu!$~ZN&uZT=0V4yjp%yp#B%{5B{$TCt!_C
zWdxZHtmw_WNN+QF5}7IGb{Ih=Q0)dvkox&|$6q)RG*?DRBQDaLx5<1=H3>?BqJm-~
zL9$hv^>eeH4kYZ}W0>7&Ox+LTxC=8C)4#T#7?5<7^}edVQ*~_NVmlrd+9&n%s8yA}
znWns6Mcpr2Rp|&VVxOu6oYej4RF!X^)crG6mCKMaep2%Vs#N==sti@gKB=G2scPBn
zQ?>eeK&qB&e5d+5Rm&Yesh@wRs@(WVoxf96x%{}A&-|Oc^M7d~jN|xBcRrs<WM9;*
z7&Y#qY=cb72@*7;_2q{P*>22Sg|4~jBJGQXnAS*JPelw#XwqGwnPs%=dr{CAP(%fj
z0*e?V_(RkW{R=&J(=wIJrsm+@;GOsQ`8@Z7FW&dLd+xbM(RBEevVO3l>18Y0=b4Ep
zW)*X<2G0>qUO&O8dVFgBln7MJ3KXEGpRFjLXT~s26cuYzVdjP=dHxjD_kC4VOiQI`
zj`<T6(^4s#_WEh!@B1n|DT?m?E_#0gnxX!zL^P%WO{tzw9NRyrP^24`bTHgD<_s#B
zkR+t=G;`y3@XzCSqyKZ-78gAltwb7zk07!$q!j=)S}0|ieoRZed>o$y6r`fx(kZBO
z{T>X)#b%K9mgQP$*$yCy^X8Er;^TkzXGzmf0F*kd2q`?x>G}z&1A^#`I0#t6Lkx4b
zTMtAOm<kXD04SRuk{t>FBHO&)z-|U8IZ9T>;cHb41vyumcQe-eECWQyl&qCw62F8<
z!>II((sZwUg2J{P$7^-1E|e40=`PslVDGkJgj3&k*cmxS)^Ivl20Celr&)TxAnhOg
z2^g~|2m<yp5yKF~68d~d&u9wt$j~d=kfFegvfm7u%?KDxk)8pAxkyWR+M8pRe+5bs
zbES0$B0L4j!X1t-d%PhfIYagFg`GjB6us(8z>+NSwPYcZw))Icofj=6&=!Wvx>~IL
ztr@|ltYcjjwEw6LWX4VU>NYz~a*<k-oUAo1(E=D!3NkDMj~Ruh+485U*W&<AQILe6
zYC-lVAwYQ2*71T^R3`}CdA%6>jBhZ;UV1T)LnS#IFz&q58?YS|)tj&o&_{gD-%V2z
zPiuQh9JO}iMXvpp<H_2JZDpP_M{RxXTDLFWQlCpc(&?zl?O;pW*(0@2X|TGE>+Esa
zw;eT=Nlc;pbbW*;aQNL^51a4LZQWa)?$I^g-_mR|203@7dZ`T*o@V0yF3R=qsE8PS
zK!^xCk<8PI@Z-Coh2fAT95NAqdX1f%#1vkfe_OQRPAfmS5f9bjp{;>B+^}94T+g#0
z`N`q*(~MLXJ!|w=SZk|_+nv>%y}Hetef4r_S7l$DE7-)Azc2@zTmui2`&+H)%l!`}
zLyO<(cQ<CG=CAY7q>mAv68m}zF7~*-Jhb>LNLR%NE|+oM<L-u==e#OV>hW3drzy`5
zrXoUm7Z$LH1;J;Q{fTEkh#vxJyghC;Hg65(y~BYDA&dV^r-|QqmOds*mD7h1ai&3f
zU!&N)TrRwzoTsH-&b8+J_FW#={iCVIfv-0omF(a1X!EIx{?54k^sIrQxcmjXbZp$)
zwyauR)9D;3GcNDh0`9`^Jb{sfzFo7F>)3qhRWPvQNKI9qr}APrF3-~f%JYI5te>KP
zKF;W|AevhQ;Uq)JJT>g&g%IH>iSV?N@Fcdh_^fXTPkgV)GMh$t62qd+uzp2_$jTVj
zK4yk_N-fXTMR@AG*}?A5y}f;xtG_!bf58gR(|Hn4&Q!uv`noP3O%e>pR>aZO?nO1X
zl2m0ni6;aUoUQfAJT-1#wXFQ1?Qnh7p$Ja{l?qSO4p%hmZH(v1jD=8=kMM+sd7^I!
zAyatp(Cv_VrI>LS7wvmsd?Q`^thhTn9Qq^-S^3Vb0SiwqcAt0RA(EGIvu7JM%u{Ce
zb9YsF6I;mHzqheFySnMc=KdGEi<|6yt+wJO=Rm+vPp)^a;Jp%pAYJTkV2ayq^*yX(
zfRC?lHk0ZbQ2#5NbZHEv))h%S-6uRfJ>FDxg7YGUr<r@dAo_Zo@FYq+;Srv~4XQ)$
zq<S2SE~ZWxyz>F$kden26VNagOY**vhC5pWyX&x5`kNjoxbvk2MR<DFoS9hYt}@nh
z99gE(^U_?{#q~J*+?=~5bEltk-y)4%o3k|CnO$=<6DUNgY+d_Fm4zg1;Z?!wI<A9d
zki(<nUa<}j=k~p?upJ@(RO2&V%gw#8P4TDM@~3|u57tUQWHTZOQ%KNgfdaK2K&_Vk
zrjfWYEz|;lNlyVKg;1c;P;$Mqst88qOI?t7AnBwF4M|ThX#jwd(&e7zsD#MLG^AD9
zsmt0vGYLeLeEI}=tNz)q90H(>x6IA?chqEpF^BN05yr);#y9=k!uZCCL%}dfC@AB1
zvphdIw#S13{w_}8X$J1^l=oE>O%Ggu`o8Z{<^4=W|6_hIqGDJ`rD*!#nx60bB8XWj
zI7RPI%KTtObNT*G(GT|SB?mzmio!7VhKMqOnE}HJn6U`=;2zndKp`fSchL~~e~2r%
zIW5E<-DAwqT)!{p%|X8}?AhlNeHfrW6BhP-zN6n4|I4R{e8Q@HqTd%4`h9T}^!vg<
zzb_7AI-gkY@QZ$5);sLEf8zZ(4sgB&?pe_9Yx959SGdG8jsCtMTKzs#{Iq!DREkd7
zN?VEc)}rVY$9aCw{j64{lJ0c3=`WDHOuP^$q{d0})w4@KU63Xh1((PG<N0pu{wXU&
z;l#wal$dtw_st68$pzBg%kt^gJKw@Q;Tupol~kKf^Z68zPnagPu(}uSpU5XnY@j4;
zuIjTfpB~H;I^78Ai5fKB@5_3J4}0ex)6^BmaekK;3MU496;KC4N&*!AC^A|?F)1ZL
z;SU(0HAu@E-p}xUhDX7QFnPp)JY*~gDxl~lgW&@r+jKJmaZXX^B3X<vbN;a{`(uAC
z+3()l+HS&Rvp%-<Tj;s>+;h&w-Oql%-~IKRTQl=yW}nt@cs9>j-(s~-R++>CW%ymg
z->v<8U#q{f>d)~LQuL0IsOQ)xYdmh%dQt{h5Z3RlroVW8fz{txwTgX0p^19FeR_r`
zYdmh%GvSkgC@hH0--(}I^8Fn#mV~mb$XfM;vdR+n$(AWmtKZ+fwEPn=GTSHk##9iX
zZ25*-8_J@>f8rY4c-vZmdT|-zCc+l4&^{3e-0kG!u?dX(-uw6R36;24M5ypB(EoEL
z&=w=t>6!&Dv&r@l?bvr@)h4&DG+KdrQTxO~#FB)&e<yz0`a<~va1^RSJv`iG(Kdvw
z5I>&p1<9`qH9)S9HP+5%5zUTxNRWMjT&?Mph_Qk^OOUA$M`VJbF{!G42*!ivB`M0T
zA?tF2re0P@xn}waUuytmf%U=9@&tBHZq9^fYZEQ=FkBva``I{%>8L&j^2f6Z0z`2A
z!Y7a*y$1GtF!Un033O?=j6iHCdLIukl?GA(>-kK*tqrz^s*J-x=-zmsO8=c<K%kh;
z6x*5M34rNO*fH_f-j*GfGQ2UTG}K{l<P*<f3s$1c>=Rk9eX?LalX!Y<1e>rxfLNJ}
z0jS#6I#BxtmGN^`hoEH9zcNQZotY~w?ico?vzUl+bLWmoiRZ_J63507(3}_cWI{}A
z#QEsJMSmhFIhjf*qVx<rFFct^NJx38a4{aVlgVSdL{N&AsZgaa!!m_Xf|4O@1k4~F
z=|J-YZ}-<BMKPXr1Y-K@`1JX9@re>gC7Q()hN{H#!ri+>Oz)oUd_h5MdFbwO5oOg=
z`}7VQM6D}7?qHet3Aj$4K=Ir>fC4umuyxvw!+iW&sD<7t*+Xcrjw|X&_Bx%b@*VKe
zon@}2yd~&L&hS!7!Z4Z}ZSplJ?a<?~tSa4GdXJ%?CXhV7kc1<uhcm{UeCHK{w4ky2
z;Sb_R8<Mlz6xxmpKDvPvt*`G`ebAJM<ms}7Iy8yei`BxT)%k!tiDxp-Ax{aqOMIvu
zJ|xM>m>$eH29+62Kx(VM0~)8njOm7q3Oon)z7QwR399z#%&4=bLY~?uYTfxK%lSLl
z;<g!iatZ?qqP+|}iDy1@yjon@nsiHAL(-PA$aSP6K{}s8iiVpAHW-x^izF>)Q*w_Y
zPYKc=<TozlL!Rz%Lg#!cU)ynrU;6b3pU^x_#mjH#ldwKsP&8Ztqt%B&KC@H<YTY?)
z@>H+9J^4HF46>yDdP>mJ+61MYm-ZA5D2jrPF>%%nX^p&jDehKs1x9uHMm4q|>I+?X
zOpu0ri7yWDD}A!im>v_{Y3+iljzc1=$797L3)Y=~GLOe!#NWXtsS8l{p|QwQtec@;
z(0sOB`h&Kk;`^T!$Wxr4h~eoAfU3@Obf+R+62OtAr#$7(RPPQO`noKk^E-gT`la~V
z>L$Q;^!10)3!jYYu#CA_%-14MVwJupY)E%FPmhhY(rXDF#e4|)bs@arJpe)bQr0cB
zbyEx%P$l~!_u^-Y@3R{T{3uH@@kByyoNWZqZ06gBm+x%$EhBg40S@GRX64B&Kfd1l
zlLh_V7F7i9IdX8*LkNb-+M^B98gc!Y>LZ|e!g#!kiN`@};OSH&A?+rf#52iWUMl@D
zZD$j~-nt#Xj`b1)PehG89nA;Gohjy5Ay2s~Z0sc)r+MP%(mb^R;<1p}@F7JZrKz`)
z^+9DU7m~pfm1x$!r*8{JrGMmU0Z#$w{;o>BR2j(A2Ye{B^0fL59<{FdPn;LeXX5Rh
z4*{M}Gyu=b(@>At_|A(2M~0Kn8F@-F@nmM7VvgPiV0gL|55Sl&LP2hWURDg0IZt0d
z#C9DtPw1pnm-vLazk6u1Pm;&;;SHBWAZSLO22z_bP@+(zNTxz>_&rAGa0~4dvIS4=
z6V_?Ap|{L}ykXitS$Q&&^rss<+<R5=69vICWXWYK@Puw$W1YCc7vcG;qcx6@BaFX0
z=V#8-sIE9llXEsjd~qRDc|-PtpuM`nUAZ%h;fW&cI{hbq;3;=-Av3WZ8PO&e`wMQg
zj=L*kqqM%WnTa=&zmv3FsB=skE{68xaYuE>XOTtOjR}KGaZD;gqYD}lL2WXgZ*ZVj
z+c6bYGQ-ACwHd`xB^MVGXU?YYpUklGwCd+~>(7rnSSnvYVr8Mh8@2@dcx`%GEojL|
zB9Pm8Hl9HHQXJ$BXH?|M==e$ZkZvTO49e~dI-4T6<*QezGOj_w;|wodb(dCmjxbTr
zf&xqADdQY{KlTY!cIZ{U#R?D)22J%Q)CY~}TGO?@GF4~LeG!ygO!gX{&my_?8D72v
znNXy!!APDRBIMWVCfXYgXY2G$Ah{kSmCBg=J0iJ}k*&MXL|ajv7uu&8ycImqVCBga
zKmF7EILp&No&Q82*vU)k<reA;rn-`dNTjN~&Xywkcf^yB%CPO;f$4;ZccxM(en>%B
zZ2j33ke7!oZrT|~Fd-EdeVQ-9!wgT?Sa0=i0cN3MB9O|G=&&6kz=6YdM1t5eEP6X%
z60x~5Y!RseO@CNyJWzD~$t<8W5qpRrFgBjhxz%468nK1EGAueBTSyARcDQ`W=AS~!
z!#vZ0VAYq-<sXwzcOmEHr>#p>EtDUBfzR(an_OKtQ0A3j`bZeq+%qtHd;s=1s%R+%
z3}P#=A+ss2Z>=H5W`?oY!%RsGji2_VOr@y<+gwnIkr**=VI0WRnw8;?wWgoU^W*RQ
z4f|xd`6@;j{+8(bJG3Rw3cealKFp^qBKf2)yT<Ck{*l;>=KA8NH(@;|gm3p-wf^0L
z)0PTi1mzK_R-D$UXYCW4A7AJEr`JE<cdaW8mO}unAMpM|nTT3P{A7I|#+xGYM*gRK
zI{yi`))zlLo6oeaZ+QuNZfqH_;Z03Y7_ynvdh+Ae_2#cih5a5L9z9WOF+(&SiZ_ra
zWB$pQA*ZbC@7@p{5VpCvZg<Bm8@w|1%$2dBD0ai10?qDTf|rHDDy)8^l3xax)|(%<
zt`B~>B8(0QlYoG_r#CM&t5hgFsYnP76`m}Z9Epz^^6+Vk)a+J>aVlYoq~B*cU(%ls
z<TcCeAFw88CQ++B-`8rNUV?abwA>HiJk;29$xz!+o4~U|GdZrE{IcAs$7Sf8#_7wF
z$#a?>zat|XhT8J3q*tB{D9^Iz*u8!^7{sGrSo}fHJVJW4`M%csI}3~J%t}1j6CGA3
zDy<)h2W4OZ@)Q~wGE@g?Wt!<4NGsO#Y;Ep|80y(9Jn5QB0I=Wd(Z)Z>(I_r0jzMOy
znb>-kr~l;o;MW`FI&QQFMzp^}<<^fV#6vx?G*9u8^Gybx@V#eC^U0kNo89;K3)@6M
z;I)dd0%ESec6qNSpK1Bb;$Nq3o^vt)>ZOY3Yy(D~x@_F-3Ws`z#uJ1|F+O!Bo_x9_
zL&r2d6GMj~ac^FmNbQrNjE6TtgiUd<KhNQ5wfVj;-`_!EaBvi{s2L%p(`(*pLutKN
z(wal_3n$xNYSxF?IfId>Nnxn4r+>OJZ+g>tq0nc%)WFkWjnlMnBn9JfO{h>f9y2*D
ztVlfH1+P`KH3A9<P-d@n*r%8Od|v?bHbmViRY}kB0Yw?$XEqaKQ-Iw>88!fgm8Ru#
zV5nyHpZ@<t^2QXVkqo06N<s4i`{5ukyUbx?YADAQRE7@B*cO&KPZSTc#_kS0LfP4P
zJKFK=ZF#ns8o(dTbevrzvE@-Dz|q}?csu&rQal?1uU~|j{%U}UJm(hg)qEaIoL!tv
zjc_T!lt_Tn+`=7|sSvVxyMll~8N3&~)AYyu!F#`JJi8oozw2J0un`skd;KWEz9_5?
z_gA_`VKr_i`U^I0blDsk;<}9wd1KDlj)0u*?!7PEC6Z=K>6!@wO~X_sg?obiu2CB!
zL-yKG0bBiW4;u>hab2Rq{V52?F$n#~;JHn*I8SSdXnJZ^Ir@5eGFT@<fZ`)a1mEB}
zJiU^9CT#W!ax+cWB$XFxfQs3jI^=cqnq*QY8`<NXrjlki#SA77)X9frvRQf>fFw;S
z%^rs|ulmk`e66YuV!W;aE^Hce(v8K1v>yifookZj0XN_2p-%l_kyh%|sc(Xi_M@^Z
zDVnPd0{~h;rN435Sgpb#<n6x6Q-1X9r(~~&D@j%J@t_{75uBGvXEWt@+@xQ&;8N3i
z8IB>Il*+EGV+w-hd7`|xipZPB6N&-K==lh~;0>@(;NzPd6l&tBc7Rz+B5&4z8FhUi
ztMq77%$*APjZ2Z0*-bXJtz9vL#e4#q`ig)XKl9V1=ae}o3I`AIYw1Ouf}+-4n!!UM
z?e{^Rb4b#%u&vg2elIR**UH+q-WW)!ZOs=C9?~2gNw+0X+IeW>5q(lj&J;od+M~tJ
zH!dVq&BjCJK+R}tlQ^eXo-=z|QeQmLaw%Tjny#)+N*Ju*Qx=suPh?H4ehgl#2v7`B
z6uy~<PsD)Hdg3R<$>Ps{f>7UNUk`Jh8VT0QC(A(oc+4a8=!Z6mfzde&qP$Z{aQ&3W
z?#@dogtFaN9-TAGKQbaBMJJ%>=cw;b@CovCAE4y}v`;io;>ijSt3H9zQ*m01pQ^5=
zU(eo+-AmkRfKW6xaNz0>f)?bHg4$DAP<rYU15ZV*9=jVZrRIDBh4saWR~L3q>dxi)
zPUmm+rz|R#r+=l0(LCWh?d%BupW7#k`nxSo$v)11;z<(BMV>-_J%yi;)@d5XJaaCI
z)HF}6yFENAA_*y-xe`4{^P~`5Kf%{t^}PBJD269N3)-h*mZxL5>N1JpX;uMRuk<2)
zkB7%%i5pEUsT^noNzM=Q`eJk@@Z)?V<k38(Fg!`=^El>oF+2^<hIx3T+m=kK#?lG3
zw*F^45rcUml;OoUJ@&?Sz44RfKffb9F3)t8p!Dc50Cjd&GxGHK0}<icjOvrTuP?AX
z;UeMyJ8V^Zq076-$dgFWl8h8S<tgCC36T7er|c{km8KWX_L2a<Qnwmr0jRzU5>!ul
zI+YH1hALGYNkg7~jwb<;g$ACke!&+w+Nj%XVj2#KC|XvYOwkhnO%naHzWme6pI<=C
zw?o?XW&KC*)BcXA^;7+)^7CXP&7Efz34`;gTGjkcw+cQ1wQOWaHXv%BMxJIxFe>{1
zD4M4aKyc?Xn>6XkX5AriJ@Rz53r1Vxv_Z4|s!NfjvJ(-5=SsX7p2Uv_Vh6{13+tz7
zo^odvJO?i%<u;t?zj(GLq2ba_w>J4;6BOnY^NXZqM`kl?rDgr9V|*)5tO%1VU^7q*
zO2iaDtzn;D%k%6((C_BdrjYmVAcRypIhCb^q`8g6_QZ$uyPbwoAE%KxLJ`~<*|}R$
z`K3rud6ExWRTrUoQdbbtU%{6gaVtCDB<Zgsh3DglwkIVf)Wc~ci(u~3X(Uq_h7msv
z)R5bB3X%7ZiwJEOC!O4~yCBbJ+B3S5J&zna%g>MV1!+!BiwXh$F>Ruk<L?Mg=HP6!
z`4`F-e@rjd1fkacp6XZsd|%^oD6>st!EPEG0WmCj61SXH7`igQz{r%=Bw*+Vt1f~v
z?qNJ`JZ?4snEx4;#mU5zqa(*PVx!}~L|l_N=KAxT16KAt8++8M^9$a@?{9&^=ISP}
zz&LqcNkF|?8F}Jz?KTHGa{ndbJfmmkuOLiD8RxA0YrbDfu)#84QXH_=0@1dmdEziX
zCat>-XB7t4PmHMxU938m8A}|NBs(*fp5f_VW1lQGA%VPx6;tLsl{|Xnxoc|y>isBg
zIbBAihGUt|aaR@;?8NFV1+;cWaoEdR!t8Dj$8x(n2iwr4lIh2xJ7TA81_G3xs9jN9
zt~-xshwly?t^<!l?>XQu*N%rfT)fP95H(S7+TA^|AllP2HeMNv?|a?3tlanG=mn>h
z1;O#kg23&{z`)?(;BCraSJXBK2i$PkGNIhCVZ)ZK91M?~9T6v%CwrugXK!Pj@B30^
z;>l%b`O%}>H#e=^zIX5PUFUCZ_u+0`V*lmaH~sxSz1??t`9a@yJoxBAU#1<GQQGpo
z+xIRnhF4y`-4``ca(VgQ=a+XVf4aTgm&uu62e`Y3<*BqYXh0e?<<CnDjM(eVRl24I
zlm|z-hnGie-r&#iRt5(9JA}Jz*dMqJx$;xDjypJLCOhBqy3&~UwcgFQy3X6cQ{KG?
z%lDorA1psnu6({r-S_Ch+}vH|++0Dy!GnsDgRU+I&rBRY*xh~n%s%Du?tR_+jP{Ag
zSTg&UoX>n#@M3W<!1E4}KQ<xevkkDIo+FC}<tCnZ<-d*dY!BW{)hyS5dU<bRfByb$
zD|L2-EBBW3C%(DayV7;MuO_!IYQL-J#|EArjK@5RQ=jyo_~zc_)9%<vA^+*7qVU1E
z9}UGym$S<^PZ*wXuk=n0j8<or%q07sY7$g-j7{p^x+WOX$$YbGLeh@<`p#z_m`PR*
zFQwbL7tSoC1aRF&xj$zLIP!aoe96o}Z-4<Pzfb#m1Kdk5_hlaVW#2b<56pF!{vta5
zWKXy9$(DoL{Opz8krQY3aKpRzO&s4tg?GCf{T<JaX6coGUzx2bziO^^UQc<!Y5wXL
zt{n3M%mz;7HyZ&+xIX}v0>?-obh?E4O9{%qv|!S}#FJnDZ<$>8#BKhX+jkE>xEYoB
zQ{U~y4N-XYQh&NT@#)<?D|P$tHYQx|Ti)S}{*KYyY0aaL^L9j*e4hEuO{8jhZsqc#
z{ecIIIGh9Lr#`&q&RkXwc{54k@0T`RTx!IJipD$MdaHUzY+Y{9VUjl8s2x5LrT%(8
zzUZwo&DUjnBDG%^`(dBRgr7g=aO8JR?y*mk)*w%eCXgo%C$I0(-2=aD{qzj-q&eeP
zGIww;)p7q5f1aP8pZ&y{<7c|v9RiN;+xP7jo{1-WmM4c-7(bDBo83_NVa*1TO_6I~
zPkv#})asX8&~iz^+9Awynns6`dM~f(ECOmQd*UHim_q)VC(OyeES{LSMsxe#?d5Im
zi9g+auo#6#hxT+?co|pfxQ_ms&*$!qPxu=z?H=T5!hho6?Y;-M<B+Gja}WAb3Lh-`
zhnK(aw{ml(uaRqzd+EhwS@uIsM}|tJdn<`wH4sPHdP`b-rC#?H*OyWppy+Izx>_N4
zJhI1cboiLy4?OYGu0F%lrj@|Cz3zL&bKQPizfbR;nM?IUp2E+ZIkRu#O!x6_KVHD`
z?rzpRu{@!_<GliZ2LNU}IDf!3IQV0_&M>dd=!*z$5dvS$W&vA4ix>_W+p!;?usxUo
zjEN&b^9KsHu`MT(tzu3$bHcTJ=7d7qG2&X;QLISFZZy&Y#xYEP2wfB4PMOd1dqJ6a
zs{C!-juY<euUSs|^z!?WiRbgfomcW>etJ|;@L-(B@ms0G*>(w^`)@3HWb}8ONaTr!
z=-tFNi*j^|6D7-W=<oI%Sh<V-E?oK4-$j-@Zi~pYE$#R$Fg7AMN;LXemK~bIz*x`b
z<cGJGQtkNH2jbLAz4Dnd<jMD#w}B@I<Y`e9azo!}&XXPT)O+yrz9>5$&6Azqr%x37
zlxH@~ZFAUn{7Dr0J09!r!n++!JTWv`=J~#B3pNU+p<FQWR8J=rN!pbZd2=x-&5@QV
zK+>!l%ZJJ^1*yERnejc9Pt*;qZM0(|B{|Zm6sYYPl1|4%$dPQPPMRlv$Pv}4eDUOA
z(7JVLJ%&^#vIz1NlBUYW?D_BPooP%|R}{y&Z^VK(2B`JnXt7Ej4QtY8EdybsK@2$1
zL_%N)n<Wgwu!VI14KQNaWvQ|3B2WYYg{lP<!H6tET%v+dppjT=TdN<6wMqM>NzZ*V
z!|POPr!CZ_{--l{-aeLJ&fU&Er~W{k^Ug^4M#7Glq#$s&^UA&Ig%G&IM4bSh)MiXv
zUhf}O9)>*5_`}Z<cX~1N+cJ|=%VDToDsQE6v{Kxtn}?lF{ktzF-&{909*{cOdUAH6
zT`@DY&VDFkdh%UAX#9rzDu&wD*ltgq*s?Bl26$4*H+E(!Tt9Vk#Ls7bV#92uy<)yH
zJaK5;UI9(!p|*AF=BCs<ZLcm#Sr2`-v*+7$a<d~=7oQ&466zMUzcV1+>PnQ=#SOAg
zJKb(<jS-Dqbhi#^C=&o|_Ep#0>}^F2&Gz|w05J6<?WYIr*V|4H#`w%uy02KPn0*+q
za{Iw4)76&_`dzxPe$}Oiem)(mO^lmNwl@V>fAop%LjP`s0T8j=^&NW4%F1fn?|6zV
zgWJ0$l_aN&4};ZEC=bQuo*&6|*(U=h@c2Hz;(|7>=!WD}bv)M2Ew#;Tyh~?QLoywo
zT~u#H!u2x61tY~L_Q|fd|J)}zUNW$$p_$D+>Xt4~k?lV$s=X4Y6p!SR*qF%Ud-+HJ
zi{kPDLliQ4K*JM)P@5z7v+sHv#h@hV-8^MKJO|~vbZsc^m!!*!stYn8i$iJ&d72K=
zX}n;1aHujYcn&^9bCt7$MD24!<Do%=gGVmGMrBsg92Br;g&LA+*t%vs3+<$6I6O$s
z3Co$|$~6~;NO?rvzl*kl7I)~%jUJg+R_9wCa$(cO-vEM_S_R(mGmI{=vby8tQ)_i6
zsKn}ko2a*6vcfvL7f#CFcN30XZ#C4^n?Zx1Dd^jcM_jt#4#w)jg@-3|E)aG2ZNGHz
z;K5H$Hg$9~HGQ%jDm`0^kJKGJJv->%8SwEi{v&@nsDQGC=0bm*Dt0-9W#`{qm;Weu
z^c;$_xvS!dK7JyCxzW<TF0zoO@W<=kFwMOxi61~rZl`tRXDE);cQCu#xK7b<B`#kK
zZgxNPWr4$jfALW_UsGR&^qeU2Se_Dm3*fHgdAU-T#t2t6PYd>mu!TgU4>+F8Aqw@#
zy5jR$w-OLj+|Pb-ah;+xA<Cu9J+XZWdD7Rng^roEwIO^MT60<fTlyQJgtzJ&S87DI
zSCT8>IJt!TroPfpRcOwHj)ny(4J=;MTd8j#0Ku`>1n}pDy?TbP!BIVd#YVDQ5WhDD
z9>&ebaRpq#KN4gJzsmrgw1Rtzq54?{WcbR^+Q877BZ%V&2-4Jd|4n{#v_-sCy6*S&
z-Caah>*ql)gpuuS5emB$Ft0`-w7?Stm3AR*c9x<<Jw}!#&64;-v}<nX$;f*PJjsQ^
zmK5~${!>tkb!EZS9N{U+W;k;{$CEHd%@f?$CslcM;}iBNMA)sx6a0|oi2M{9c(VQ?
zTPQRyvaV8^m;Xx#VD0%kef?F-Fa1TgwI01u6q&N|U(et1<K$XEE&F<eHJeNkrQLo4
z1-b0G!c?CQ-)?J=1g3&fgv-Fr-0q`)Aoi)x-DD$a7vC;O*<K9xDM7{41EexEQMG08
zf5pHt*Xr~xaFtW@r2q4y@F=AgPdKqN3!&rktm2Cj5^fyqQw7(`CbkOY0Z+;d6l~rN
z?oq2&88G=pDxL;bnXGut=X0Ob(*A$h+2j2S^i5vVdtO9;Y`c^$2--3L(w<n)xqSV*
z$SbQ&O+%rZC(HH^YA7;HKbMz(H9TGFpI&e-v9ym(8%?oJ-`naspTY6euGp`!Pwahp
zSn3#QkH=-!ga_AWr5C*UeKUKrq0BeGrK0%gs_&b9>a7l_c}gNYq3E&BP~ZHTinP+s
z07XM5>wTjAb<_0*#KiT7%EWg?wT)|UWp{f^6KoZ2SjCehULGTQD<(~RzyVy#{|!$s
zvi}n=o!19*FD?J^Wgj1h1~P;h7k5?HM8tj3ERdB-Pn<)+O0l$vhSe@7Y=+sqOqatJ
z)eTa43?}r6TP%mOz&<g+Q`A{8)I%)hI%nc$Q<8Nf`@v1Axm0`@6?d{h4RRahF&KZ!
zGXnb*Nt_Z$Crx@a1~bu8vDj+ZD@rPsN+C@scR-wd55p>$VWVy+r1CP93-w)fvm~xq
zEN$84S`D8ufBNtHzwt!<rGMs9#`o1-g4almt0y4eC~UamHGs@Q-wlp1FmMFBEmjIB
zYzO;^Ec6X1J(A#vys_R2YbZdNS-DX_kyG^OADScVtX0PiZB-HZ;1X^J2wB*z5MYve
z)>bd5q=*)fr;{Vt&emh}0m5*YTxzkxfLeGCWw(*sVWq7erk0E+iZpy_ET9n<SknmE
zL8BL8@@KH7DQh?c=LooSkVX`xaq;4&jRY7A5Xf#7e+nl1bQcz=hSmgJno4>Rj1W@I
zoU0(BF-XM??16mD)PmsaB0<Fjz|-^YU+`3M?I%hn!IX=Z?=j(tT25*46#XqIChyy+
z3J)l$#ChkLYl6gYi8gS>5jHNClvD0VC5L(^8>eKqpsqJjs-<&4wUbkq6O4{k85wnj
zBejAoA&S%pi*%F2whV-lW}Lcg&S8b6vrTyf(v}B2AlSwxOjPgO=7n((WUv%%zUB)q
z2%9|HDazwbmQqXLX!^Q4vvzfW-gaXXH|SkL+uh)g@i1OM{{kFr50vO8q<x}8Yy_s3
zQQABu#?B2M{M=!7a>`_S@^g=&sWtNx)^870K8&4voHJAzdtvDFytxbW6VpRSG=Ke`
zGgiaNevnGiKegm^>R4k*&6)jKvONiklI#04l$P$P&)j;9MoIOM)s#uRCW&&s#v+gU
z@;Ac}!a+`L5+vHPp{jg00^Q!@Pk5>=a7sbxR?TNh&qw@$r&_Z^9%uJvh;sMuNx?Wh
zuo2%Dm-*y&jo9~ev6Mr745g8yVu!oH*FIct`)bnwJ%Pb06Mf+6#gFg%&m97m-yur7
z{dDs&e0Wbze4aOG@HV&^xHi!UX7CRNPoddX#LPYP9<mL2>^j?lG>Yo#Mu^EZ>z91}
zReAyY%YhHS4{u0-{S8-;S|N1^{d|#4-`}FlKETKVAuG+EeId#gqpgK7j2kF)sqyz7
z+uHK!#VmyKdXFNaAGgtP4;xs1*n0|T>-ZiQfA0k7F&pLfS76`B&IlXjKAg6R+G4zg
zrl_EDH%cvmgS|E7upVk^WA`^W17-axzVGr%n<s|jDP(Z^ao(V<!`#H+<8XuzP7Y1@
zvCK3~`NGs(hg<YxrpX;?<V|?WJJVTu?(467w$MMN*j-9GaR7Ej-dU*d9@z$cnjjh*
zK+N~oS?0#pGW{I($GzQ5RSLQdj?x~b7wMl*?@r5x%R)ZAjEHVKOW@wX6TVT1v~~Mp
z?nUp;ET;Igyc36U&J4^z75n3mOv}FA6v)FF3Jp++yQrM?0>Spx);{hWPcMeQ@88yQ
z6uEv8o+{nvzOA&yb~%c<i7Y%l6+CWh%gk0*O}5gc{vzl6fv3zXU!Si33O0XAfn5cJ
zr`iRcqRqCt6j?`&?Xj}jdc6Y8mm`oUa;sI~F?64dy2ZChe)<eWK|dN&ZFOt&#Be-e
z{5ziNyp@d!S)$%Ru~ivP9FH!|h<|`@ZeIVb*Gdvt;;jfuBY{u5-5LPN$8`1TfY($!
z87$`S`%L}m@V_S1;v-w?dC?zuqKSQyfH$Vw%}N$U4fz94v;a7py~a*YK#hc>AJb3p
zlz)h+HM^|lDXzDlg=B9J7I>=D(?iH(><av0Kc~@r8v+Gy3bm`=Q}Nw-s8r9MW`QT~
z69;&ne&l$v>_)+3fN5NPqXcR6bsdtKU6dSu^v3$*k9H@{Fs5!d?%|uBEcSJV`cCAW
zMJ>oKan0-20t;VT1A~ujRXiE~$Ns+9E)^V|ceegHPy*<x|0^NU7s=0!h0<=1PsD!C
z%ya$)em8%eYMQT<1P@Kb#?DVc5#}%izD=?7lRH9@T8=$u_FQdq3jwFfkWW8@Z2x0&
zXkGvCX>(6WzeIym<lb`d{&D@eplw_4BYjyzfpZFtzBcRk%sgKaRZ_Dve^0Au?ELO@
zv!g5u86#8%zt!u~bq*glyI4O~1NTWYt6N~$wRcZZUhnC|UPyf@vuZ4E$iN}BoyNtM
zBRh89p|$HfN)WswR3PwO;|@#(Y}{_L5lCA0_`diL<Kw+GHkOvx)JyOXcTiQRcA>Y+
zy)^sPN3g2kKe>WGim>`x^(XUsDAu~-SxK9xOE5)0d*mdzV^??_Gzh*hTX`XzNxVR`
z9*!MB!50GJ$X38Y@BRyz#_aP-G!hn}`FkiNQz$g5fF`rjypcs}r3;OOtptQ>bMwXu
zhg&f7c!!47t3vbP2!upN<}G(oh^3K#s!^wxx;-9k4!1po6-H!m7`AV0^+y`z*%)<6
zZWbvlEn8iCY@F(yuF*lxJJ)Q3Koi@JU}%pgvlK78)oubk-#3i)>`YAbSLqWEgXNyT
z!+JZMw`fLI`T0A=H5R9)K#$la17RFwQ|^Tm?OB}D>T^G^D+0Q^U18+T@WlyEK+!g&
zzZ+cd9EO<KHPz)|9Au7FHt#ATM3AM?Cw4d4YNg|Ltl1W0ABIt|6S*SEseIQnv{ldD
z_;@;u7yMDU_Ytk@d<!>Tz)PS*JdCCGfAVzM>as)IzkrO3G-P;}UIFi<O!+-Vmfcl4
zTp{bK!0}g2&IWev@~CaeG>;+vPnj*21u#@n+NC_bD@{3^S&-oxDYj8w55p98t=?-{
zE57U95{9Cs-En8KEnQOblrAmrevyFElr74u5h2Q9WkJ|0o)+ZLlkp@uiMx?bJb_j4
zc$cNL^yK{AGWYK||0f$Q|0hUs@9&x`haBXx>=?USM`3PQ0GZ8I{wnMb;I9%dk8t?n
z5J~siaM-b@C{76{)D|VS`FYePGm!~y-lHjqxCk5*M5}o^(S5ui!>$-I<tN*U&L%_D
z4BMpAlyvz%(M-k?)p#!hPe|v<8EuX<FPJ9N(&F=X%NqY_XP{N!W}YYTFYq-rbwg2+
zdv|gEPtrbxIhZU1`uF~H;D99UXNi3hm$$n`l^#IY;x#*kljWyHOyps>nDCTl4t~0$
zcfI>Ko_=T$3YDkV7{x$)9H0XSsr_Y1QuDU;deqYBrQk_tVg4(n3COCWe*yTbEMxm=
z<u>6?gD2Wg$h&m!VFvs&?bof323I8H<0fmaSnz+Uc^CGR2Hz@0rnoFhdYCMGx{;lt
zAol5inx~sWe<T8)gkdDq#g7Feu##0IRP)pis~$r2O{yXbT3^|xf5a0|wD8hIsCz=X
z#EXUw-ZHLl$-FCAxsr9rugN?uNiXn6!QoKfu}R9qu;Gy3#~wcs!^GD_pu|uTS8RAU
zbN}keNN`*dreqG#(W6B?PvBNh2SI&Tn!?EB{>KXrBi{|g^NG}V`2!MWomYG#o7YR?
z9X=yz^W>Vd+Sm>2`+*0fAJX^Jx1|uWfDIErmOJR~hEEcIN<W%64I3%`yXs1=U=hgY
zlTd^V^;v{Z+F~^YcHpMFbb;(u1tAIZ=1KSV(<+U9GF?Xhr{42I@bQ%e`bU-Aakwuw
zYA9n;#Vw`H6~Uu&BWVpARhsSZqb#s&IEN{m1dhGZ6ckh<ls*cT1qy|MjZ7`@R9l2d
zXfhY!{_}TTi^WDtd3Rj#UL$cro;k-83aPD8%45<>Ta@N8&xw8Vn01<&IN~~AIWsd;
z88AOMGZT)Opb4_|5WP1yGcz#-3xT<V{CKAGKBQ6b3|s>1UC5P{>k)Hs$Z6&wJzaSO
zE*@tQCsCwWI4VDYmOz?l_84;{!H4eg>{qIR`VZ#sEE0Es>~q>#lyd|*C_LY_#W>M(
zhZjb^o^P(+Kr^PBLXjxevz<&lA|G&*peb?!3<xMZH^6*Rju+*)#vKll9iGyAaFb0N
zOufj~o4Z}t*isfb0TlAx<hj8b1$%miKRcYXd72(K&Ka6wng-wfBylJvXSVVaH-tZ&
z8~?;~`tix3@wL8lkMrg_rUx$#YWxcv?+=fe!?2M`cGD9_at==Hy9vc?r6hXtp)oAv
zi=M1xr+<A&tFMrW6JqoCRv-_hctC<gPRV}N;uIvx7Z10=eS${iDTtA^^+=NmfIjwa
zPq$Are5y%;?@)rY0@OXtKT}`JKJ8f6`4+^FXFf4dsTJFcAJ62&h;iHMgQC%U{O$Oo
z>I<K%@yyxr-RR?$$YMTb`l0=9JUz}a0P}Z`eUDt4Aj4}%f(OCBAa23GfJ!Yj%T8gD
zTJO(<&xR=^-j%i}y;3p@*E6%@!Id-ZvdPbpTEuA}Wb3}PvF1&+@(NMUr(0`AyXf!x
zx0J%vOZ?WxYO4W3v6;Mn>pikI*6cI-sqr&0CS0kTP&(Tubq5dXyBE&C0K;0$=W#}Q
zF`j}4hlVP_%FazpJ_Hn|p^mc9BmM=I5`asBK(1>A(T}n5g4Gc9nw{nchZ2VdClAAY
zhK7bF+L^gl8o_l*<EfM1Rvua4Nq8u><Pnv>w|%W_>jRiR8GsRN*WS+871t{A#*WeW
z_}*R74gTn-L+}&w%?>@qKJh52qh}(mfy}>SpMa+wF9c7|l`p|!;(5X!u_kNg#wCuk
z`@ZQgbz_KqN^d3pPg6c`d@xagR7!h{ctsnXwB!dMFO63m+=Y<$p#A(r1v7UI5uU)h
zO&pnRr8NYe4J=5OF!@_u0$p0}A=h8CxY1*mKr4BKH<?$-EAWpV%>tfy1HP!#kyyz|
znhu^CC`|EM+&*y)D0O>$Emz;YI6OV);(3Z2f<HRgA%{7CI&LV8fv<FWEBF^kaL{9)
zp()A9OC2n-j(*J2XmMQv>jp}y^vWBW0(XRPFjWtOA4^QEd7iw7yWx}h{=5V+vAa;*
zuMx!T)$c-ipO(3nBp?*OHzLRdcsfNY&@%^~9?u@$nwM4w@9FI)bq5b{dKr0omi9}V
zr^ySm6Gu+YPF7aVOhV)^D2H03xnha|7nZqSAF_F~5MC@CEvk1Qs#q&RKJ&jqWBk`^
zBz(`%xX=9L6n*JeXnt|>=r!AWR07PEWDEBfbf?vtWuHhvu48+I%7Pe_{;6EK=MeL=
zSpv9A-9`Am*#R9ABp^++XiQVO#fv69@%_8i%gPhRs?Vpy1;c-`fARYv{M3sP#<~*4
zKSTH5r8`edM+dC^r<tav4p{!bqJxx8?(cMvO?QO9JlOFG4IaR-ZHpGw$8cypLUwQ=
zi<>^7T`wK%khnHAHPK&oG^v|h4zZq_Q&<94&s&?lQ1E8YotwiDR&3t3EsV6lwry)i
zOps^0!zNEKUL_Y<9rCwOG^RBffmFh}*eAZe11%d*g0;373ghk7e-dc|-Rug*Pns4i
z$}L(944i0_#qUO`&D!Ceuw%72LFXmDOmPDHr^iy0|Ngk>^F2Q9UC{a$0qrqT+b2wG
zH^_CjPnrasJC<d996NsBOk?bweJp5%#h4Gkq&5v0k@Okro`uH_30??O2+`|#&rU|s
zPOA=~Xkiu<njwPft6AC}Q%Ma8?i9a=6J9ViTK-h9H~%Qd)9>$~*@ktV+%|s@C%%hR
zKjH=DiLs^rBEJ-kL7ZjMef;SOo|d@2!^W?hAbty`{4u&M&mwsYEpj&a;g9?%M`LTs
zE`S_ma@MDx2uCFOA&K^s;f5SdH@G2!kclnz(TW^y1N)LBiO7dbrDzmx8h|J}6B=&8
z@#L68@*-~6_k}S}$13?b?j90Ek}(Fr-VI(%&IVS6CCS!^?c9B!0AkmyR`G;=)^Pb1
zP1krKj&V3nGqOgXHKF7*_(9mhME`Z){J!-91n08&#pDU}*Y*iG;>Vx<zI`(0d0Liy
ziYR#W7`QawVR9QKBzt1B`~3uP7(9&P`$<Bw>iy=Wrz0Her7kUDN&5q@hDYVY!tzjV
zg?SnCg{3mE*$YmSS-Est2Pwt!A~t$dSbDw{);m_9^sBV&YL+T5B6CY9E<!k|)G9kh
zl<Ojf%WlA)dZE&#zegF!c^NWQQt^ol_IQhti#&!FW$sn(C;5PI>Zn*;#5z=q#eHE|
zwPamRrU)Uo<6#sWDQ;-=iaH|{*Taw4odgTt6TI-t$CIZ0gf!U{X`S0oi!A6|LY{22
z<3BmxYG%E~-6Xul?Sh!7(lT&Sk(mK2hY*DzFQV_Q>k-~92SjHF?5gE4RQfQaTm&$<
zN#4l5u=Ivj_}7sniH^4t{A2}iG~tch>lLZP#5sWlp3;VQd6Xmwey$m@#n>V>`<&p$
z>9E9x1onQ-$i`fGOzK$8hIAK_qFQ8h%;DD6tgj1Fe9sQ}#b1w!t_7ZO+?fP@h}Dq>
znI2(P5{4y?N8M$6cF6)qd}|Ats^*B0h6hn+``Uf-hePR~Y9JBhVe~Tcq+aBX`0RI>
z{=5A<lcl#$B%h));<wXJ?7Q73Aw2y=vL_Bi%h%^hJ71S}V+wtJqJp%aAYArN<p3R2
zQ-KU~VtWWrDkqd~m1hCvN|Ge5TDSR3K}2**7V?(*t9T;z$s$M5Q<`9K{{h#`fh>)2
z%F75<U-Y_l8R7h6$l+`misJGMS@<Y6X97=05i0KYx_<~EuD)ZoCvH9?_xElXZZ~!(
zw~+fI*Dda{PK0h8g=J2jJ>myZMu!n(QbPz*?ggxMdkO11jeWvu1EuA!qO*N6S$g}l
z+Q`aiEq@jeQIuJRIG!M#rgO^nFS;!4jq3X(gr`R=@D!0`K3r8gKtt+9BvUrl?_b~v
zqRtVXXcAS-;g)$@KJa9eg`$4X({c0Wp3*Yli9D0!yvvi`1D^VK-@1ZGfG1JzG2~5n
z!X8E~JCzp+Pnhs@88L(>>~Kc@)|vDEOyZf*npQY1sD{k(LwlU?#8k<<>a!sXSq5^1
zOpn+hRR{q-U&{K9&(EnbPx$Y)pH?mKWV&U^+T(_+3^B!vskSQE#qo5bpT!W>B;Kl-
z$`3q*INOy4?+9qFJLVrHJb~Y?wRf$mzSHK3j%v6p0QN|dpAfNsdd)r@&++urBR9wm
z{5zhQ$Ww^Pz3BBvo*=89zeVxY1)h%5u7oEPQj#J_mHVf)x%uSxOE5|j-$hYQ;j(5L
zc*;B5$p%YRJV9DVrYb`(s3GO0tnbwMIe8C*f6%{MdY+z$5`fai2HcP<{HfBUoeCj1
zH|;rwZtuTGPV$ys^{l-f@wH3anXv&jx*=gz?)jaWd(=FU{D+V-8j*n_c}nmoG_3dT
zN*Zg~G}<SLZs@EM*RasPyHT*&a%+P1{VOV-3cAtl(&YMv6wDc*WLlJ8hP+(}Pidu%
z^<$?wo&dMokX3Q)6idM)hPRcR?>=7ASXG+LV&w3%d?dfcs~UK^e>CpCd}n0fWp7~y
zVq`8U*0}{Exdj<7Cr>m+Dw;Gv(VBnM^4llfg&pcMXv|a<vC!8SLKZ|HXMY@}S|$j4
zhTkgR?NBT%uS%9U6!(1|Wi#B?Lx!pxdRm#DFf6KXS7k@M<CbdFyvb=7tf4Z3c=b%G
zaM-h)4l4KZI3sN9DYBMv85nVr^FETcQAn~T9)PF5OOrH=BBxz&)V12E%d<Qptq9I2
zk}#0gu|9?C-x*nMaz2dUDp<v(PRzKAQCy>OdJ&{-6pqy26YUJcI5@vEE3V#XxRrh>
z>pP6Gs><U~(s}-FQJzefi6?$mgZEL=yor!@hDD9kSMli)7aZobQz`Pf75OxZn12q%
zzcK$bjn5VTBu-qxpSDOYM;$z@fF~35Vv6EhQR?<M6uS2B47D77R}=g6vdjlRW9gX~
zpcheuBu)E?)+|ln^%wR@+W`uebiU$gskGAA5WRrH<fU=bq_`Ikly3HkcQAOx(-O+i
zWbGR-kk;}tR>?%I-$f_q<B{`$$X^3JS>G80Ph`pYE1q6SEy2_DM<RHh_ysP@$`itT
z@uU#HA7T7_`||!zwHQ>~Yw`Bea@!|IhEReX1&bbCeNQuafj=rcCV2(aGnU}V*CJZX
zKCS#uc)}l#Uv|S3UVU`Y6!?tdl*T>I;b`G2o}L#8__shu`@}o^awAao%d_4bf7o0H
z)A1Ibb(fLKpN^YyT*&fku~9WDi)oL-ZVYEU=j&HqdJ%&JALKuE`Iq)d+hIk;lj&0N
zgjP98o%IoKpYGIMrm@eRx+}<HOJU(X+Tvl+(JRcu!omz7hsi0EG!<PdJhl77fx5zT
z%(}wDW9%!Up1A}>7~Z<HpI+0>(8=4UWyq(9KODbvDgxUV4qV2ug~zVdJz&P`PK7@#
zOld0wyT$A{RnbxR*_(A=*~Qi!3TUc3^k&^>H1$f)O6n#PCNEM@f8EIgp7{1SRCv67
zTKafmz|);`g?E|p&yp|4-T4Z+CZAe0{uOc<_$=&5-B$>6Je^{hYo{clLVwqU&se<W
zRMsm4^-QIKh$rf421>jAWU}P;3Gyla#@_k6whcsa+?FiMmM}<-MS}yG+$DHzX`q`2
zV>)FBC0V+Jb}qF254vdU(lOo(sXdhLT{30MlBLi<ruLucJ3T4S=cv{>axjSRN6{%i
zmOwsy@7_E6^5WUNPd_fEhw&ToDZZ)quV$mmbMk3&dP-liPt$S#>ugNxr~U7w*fxCH
z2og{8`D9Yxx%z)d=n&<761HA$kr3k3@27D*?JJ+?1tb@#PuF4Sv@A>aX-GcRKg959
zv#?Nd7vxjTT{xlcVd^{aY4~gMXZY^izlu*jO|GAiPs7Xj<n_lRf5E5z-^tD4=?n|<
zjUWM~X1RmZCaa*n(|z*RJ>Lo9Q~hT=9=@kk`Vvp3uar;L^UFA%y-n8R<8PnG{fjC2
zbX4EJimB9tPMbp3PxRmkZ>4;a2GqKJ5^6Tx$-}puUCUR$ERBP~o<-QMWz$d9?W*rL
z27AzHb4WfB6j3YZllX-FU?ANmHi{la^_?^{8x8mb{*zC3^5lFnsP9_HC$O}<^La3+
z@3>!(<x@yLb>S0`d}4_*lML2RnNJRU0!!LY$z@RAY45;`qAq*_i|Z%)Pz4@gmQQWc
zDf0>IC$L!G9q?zqL4D`C^YdMh<WmPef%*QEG&W81NjC7{6S&iVqS37XBtCh!`P6|=
zK<YcX6o&or7W>r1d<Sx2m;7`v&!?=RskVf-;S-Sclh5&F_WGxE0}nodf_yTp?{uFc
z_yqKPlJ-+kKCw?N_ymgH-D3Fqf|mH?q&{`w6JV4@X+D|OcMCoR@Cj(7%O}(N4*3KO
z$~zkFkxw4-3FMS_2hx5r&8Md0$$Ym3p8)3*@87A>xW3bmWQ`*D1O%SvR$D)bPsk^b
zw}7I-IG^+eRDgT}cM3ehxIeDXcWO%*K7sl3RZ8SjK>1|oQ~PWHp{h?BPwHWmPp<Bh
zX+9ChXS;P!L43jDo5)j5$jvA5$&B9F4yYi#ot|*qJ5C`vMx&8Bi+n&hy9v!U!6%UG
z{ep(Gs!z7<^*~i&k2IIUFdUFf-oQ~ZiA{v7`6NfwDW4+p$qdT}6=k*K?Q4pBp919*
z086mx($LrAd?H4cPZ4|qLasN2tzAr>PXT-a<?=}yn(zq}$CF){Ptq_5X})`01K{5I
z#5!d@1@H-!$fuSz--YlAl*%W5hcJ8s50y{BqZ$CE`%eyh0&A@AIG}<6K7n%iBpbr;
z39KofvIP`;0%h|_1r&S&<@fKnp(zV;Zv~$kpt3X*+9|CRH(`lqx=$<RlN&}IP&vBY
zND%}<sPG8Kac&(GyYkJ-e98(u^L+B6MiW$y+dpcX5qW6j09vw4ceREA&v&7jPaV$z
z|JXI(m?q9Rjz2G+yKBihS}b8GJqqU7uJp(<ZVG}^nx+u81>!)4rOKp-Q@aH#SdIWT
zjv2a)A~9?c!kpFkXTwOH3W7;ZlR<(b{7DnD7fU4b1^Q-LV&dC9Pb<P%sBPUV;CtcW
z?s<OC?+)(M@AKT#TXR}rxRc=xX(d;Z&Yr>-4Dg5-15a>GYy^x>Yvu2au&vBf$fYP;
z#Nbcx#e$|vvA+BX5~`?_0ExA)i2g)H<xiAMzQhwEDh8fb`@qNhNe*f0zD0_PlpvN3
zs**N^Yg>3#igG<T)GIvUA6c~i6utn3aaw2ol#ryLK$td#S0$&c1BZ&_Pn0Z5e+uyw
zuRlp$K`I!-wxS|?n6xGGc~@&wrBEPT?|^V6sYwE^_3e%PNy-z9k*COv6Y+?a!^Isr
zj1a~p&?}#smXooa87fmiv^*(DVy){9{fUzJ6A>p*==95OBh4ul3MC!K|DGP4{k05Y
zs2InEjuCpbIQj9D=F9uZ6%yo4JH575&kRZKO?g`LGpIN8C$QX~;^YaE$w?_m31sMq
zHq<!<R_rb2V&^HZ^UE?4R;K7>H3R%4y;@l5Nn!Mu4zAF0rjDJmajW5E_;&Macp_CB
z{v_oorsuoxHw^HSW{f<MGGi*s7Zs>wp_?|X8U5kzN#4I-1^`FUxF8_fXCNU5Kn^HW
z1t2OY5;x#>89K%R6sIEeV8K<!m}=egI@}bL4EJM@F$J^eYRCM53FU%BC^Jrk!f8-9
zWy9&0_rtkyjOoajzltM5Jyzt2$j*Fc!=L^ae~OzYDA%$)%d&i;8sftmkFN!Q(gk6%
zLaVisl}4?VgOt4_y-mh_tIgf)_`cHkLE1ix)|guip`poWv_gBDzSL;URW=#ZiVaPN
z*~TrpX7{9h=Pw6k1eRT!t8h%{Da+mTv?{K$&h0gBwrkTAUrkg|7L?h<L79E$PGddE
zG)%QQ($n-9M4UWVaW1W@qG@OPbsblM`c2f+49bFftiTiXL6N3#({i46eU6IHcdcK4
zlE>js7%1F5KAxy;W=)VM{yof-;hFfDBB1WL+vjkK<Dl6o%rk>^-5QP9TOQzxGCflc
z);VhX+wAdH=hNF~1$MIH`RJYIBDTN2lOJs<9p+hK2sC#WIyL;iRuU6C%@*zT_zuqC
zEPqwC&hfl3(#M<kYP?l50~(ERY(E$1MwRytGIgFpPq2a@===rLPvfs_ck+D4jjEbC
zffpX-J8+i9axRP~GCRAZP_uIxPgy&1J^`QQyp^@#Pf@RrOZ^E;9Scto7HCR7V)?z=
zRCW`IJmGnN3%-}zpZKh;;y7byHP3@)w>Q^5EVfnjX+|fupz>C^WlVfbLr5QWd&`c`
z3B7sK=1I~r;3_j%4x3NZo0_{vCsIypdQ5mMtDNTh)2tSE$F-E_!ec|L+jo6mMJMk~
z+gv*~b<N%>T-!d@(UvsJo<%&}zutsS#o$Y;d>)Kp{Ymx#vD#2G=F8RZa(mP4PsGQJ
zlqZZ+YEfc(cJeZwK72Rltxw;|+5O>$KgF^>E}cKc#uKSbElQCU6mA0_u^-FPTzTN$
zkC>;t)84XX#FMGkJkNA_dX%azegw1%y@12sb*PrTUlok&oT?A7y#L3AFI%|5hpsZ3
znRQijre^oN3Czsh&|}k+2gFH01q8oR6=3H9nm?J)$mTgkQFWf4eX2UXx5C|Fqb(U_
zOoPh?^83V77#FB}*6|3C72E0wi0XBF#{A{^b^Nh?-}DKWC@AF#W~c6D`E<Et{*<+Q
z>(;Hiv)1qYNgmhwlXSr#MxGKisS20`GR&-2zT{7(Pkce1=nzk4tPxqOc!@#`&5CCO
z!hG<SBO=nC-~V~)diQrUorifEoORhC;%S^V)sBUDA{)e~3>{$2SV5lj^kA2#jb`eC
znT797b$dPLi@(um{$!IgtuJ_j&pPgKkZZb`nR+3}lkXtn33Xt_S9v0WI3*^(@_hGo
z&ek1YujA*tc-O}<Pw|a+<VPAlRe^Bg%}Ma({E5OIgn813cxu!l)pM=FCEQpu_xk{{
zBitW>bG2xG=0al8IQKf91b-9CV%073bPDry(TM7{(DKT=?N0G5;^{2n$u+XXlc|{<
zy<PI~1y6Z>LgNQo?Zg&RGTy1O3kz93H5o{FVm{lEgMj*k+3=^R=TB09!bl~@HXlzZ
z<oVQeGtU>0%bxEZ3cb2!;TQn_AVX>Jj01p-iMb+P0*Il~!x~{oPGHg7+ch0`jtpyD
z)g(Q*;A-IzPc4+L);&&{_KfxDFi*08_>`ir2t)WiJ`Pn6nP`;sETe0kyMgKq`ih$;
z00=m5fWTBW1FpDEQNe3_)=|xvrrB{o&AQ6)cvr=s!4dU1BbPqkX|pqsJ1K#br>vZ}
zcYm@wXUDD$e~QbW5KS@j1ZCR_d8}*|#D{q=)}MYV=Z?EwKiqV)4{Uq7#arjv|Fs@(
z7w)y`A*32S;v<g0!u!7R{4US=Vmi;!ZN6w7T=4w<N7AhG!L`aRXa5Q82uFtn-*vj;
ziEumPX67ABm(zQKJ37GMw#kkR2$w(Z>~A|VCtf@^%s#F;DO|K3Z5J-vAOShYfccgc
z1laz9c2?*y?HLvePL+-cw=&xMA4#9@GBOp3sVeXaPw#*6ZWhRT_lx&8{0WPSX8t4v
zDrSGeK)v%`p*gceMG{zeeH^bIJg}0m)y<0UImI4eA7-6`zc_#zj+K*mkn<Ho0t@Y5
zxWVaW-19WDDxMXef|(2a)si`u9jZZvPaj+_V1>pi($dMB`TK_)sQP?fAJ3ZSIl1Kt
zYV#k=LyPiGvC%Niv+PI%&+@%`NDp?htT;)P&hq@#)6Vl%`3)Xnh_Vc$82=$DPapF|
z*~;+mTh1In#eoAG^QWMq{{C+10WS`o2uLR2x6L3C63wX;$(3Cmua$!;g_6K+q#;?Y
zB%mSmS{e-ko^hI3lq!LMsK9$j$i7r+RRWI0g$hRmk07LR7Cj7A<5EuqGw&lHN}yJ#
zX!x=pP6yF%oZO&RC!pLYDp{>UJ=FMGq~o0o66D7BcmL1&I1Y@*pQKbvA*b{I{G<Hk
z<-Op3u{1ckr>t;U_+qgXOuUR=%r9++uhtP^P15+K)}`hpo}zjG2`JXNKSeQrdNZCP
zO2@2r$y|-6V?-Zh0Bx^T<dFFLLA=Gl6QxkEKY#j<ox4kM7z6@1E_7VSKp`KMEG+bG
zFR*vn`(I@<ThgRxlZh58I{&nfeJK2xm>?cmKlz?OQyte+XcrqJ`2G3vc8<J56b0RA
zd7iL$b??u_!}E!J@;yOpSM?3~)c2O-3D2X3T8euAl<cXDn)BfIId~QGhdGqe*H3o#
z1eJLLVBIQf8BU9)Wj<lV`ssf_QT2I}HarmyJhpx^F`h2ssmNBIaP7&^Ww`SR#gnu#
zpUm{3c=Jg-Nqh4N4R1dA3#f+Ysg?z5U3sVVSe!K>5|<#PCG&X0``~-(8mY|I&$Iqf
zP)H*x#462`bFGwjm-cAg+}#!E$<&olc?TaZnCAQ&-T_aBek|pi(rU`^ue8&BPa#j1
z@5w4D?=-5{wtc$qGf!GPl~IJJl=&o{3NOJEu9o22Z^y&Sr`Z#I#Zw_gdooSyZqHlZ
zk<oW$hI)_{^kli7T2xZrMSB7V-e4j<i~^oOJe5(jrxtecocwF?WW-a60jjv1-BYwb
zpYY~sD#24jzqK0lcn0a6C3{MmPcQbQ_0X5U*E;L|+B=`u_Q^PoKhPvk5-2wNwscmB
z?hp?#0dZ`$$P|JkV;)wdC0bmSTDrr!OSPK_JGf9UoiZ^_COrsR)bia97R3o3c94pA
z@N6#%&V%|#_<7QFS!}zdn?Hu_`&skkdD67~Zm+%a<XPJ9f9NM~0QEQW6qXs0qA5lc
zMal_uiX>!B<j6*mp+u+x<@|n(`$j0I^Z)<;C0}mD6UQQdGK&d$Jf1><{e;KoN$_4@
z_NhhlFFeI(YIZnBN1~9N5{brQ3QY^q7&Hx9%+zL}ijiiH4wS6X1B!|A1gD5V8kv$x
zPZNlzmBXdwP99y%%4Z+GH_8<p!W;>Uk)}u*F9-LNUr-!bjLB1~`oTf^FXN}#uj`v%
zH?&iz>h=4esuwAdv^MK)x(1zJx2-zK@jB&3Bu`B$mZNB(8YBic2yD|f-;pSY{^ExP
zxC{FDH>^_P+7qiPHBEr=)5@Ez*~KS5($U2{|B5-9CoMihVxX)cWx@)Q7#wW;baS4t
zxOU*aGP=)G_3HwE+7MUj%~<%ft)2aD@f59{HmukmjqKMAKKu32L-pJrjc$G2MS`x(
zwH`cD)U~Huoq|@~GAgYjx%S|rPw$l&t@i6j*#!CYu{YmUDEZlgH$PwcQlhl?Uom0N
z|NK3Wx3M@p^%0P^Ur8gh@x8fn_yDmKRQH=`<HH9}IwnGy!=)XWEIS3XQSSRGd&i)(
zYypF!)R>9Xi0-2(-~fsppC^KU%f}N%cuH7;A|}t8r;TiZ1+*wAl}7V;CU;uTGh(gu
z(C_b^6(6auGnA9lgZryXZ!5KfPJFXfIBk{IHuubxrB~)R_tFckgI8Mb8lb0#iy$s*
z`<{C`If`O7KA#F|c>Fv)cCtgL;_ja8-8p$Xz4FBO)en9~8xMD?Pke_~PWGw?yUCNi
z*^>_1-f?+CnK%HXQjw@Zc#?#Opd$%>rIADdBYkvyw({`=`TKaH;&n(u)K2Slg3@V&
zrYu3M)a$?JnbTv(D!gEx73qjVk%*bOU5<O`OJRFUXEK|+E1gxfvS%hYcjSfdORToJ
zOOAK)4<D|=qqEOH{O}gk;@aasNz~L&!-}7$!*b74wLJA?tMEfhM^Ao<Hb7d>Z=vdk
z#pp|0fSi(zyE_ODPYNYsr1f~hnxJ5b7vZ-VaSX@y$RS@|hVaCZ>ZUxg6pWu{?nX}Q
zM)ItiNn~>v{B##EsucIbUGUTEoXCl^D4!J>su%IGR+$r}l(zXPrquc`z>`$zn0GhJ
z+`@w;j*Oo;iDI`8@(d-a)nfL36Lwnp;OE=)PUG8@VLV}+-F)isz}WuMX>IJmF>gz%
zKC!&KG><DM%kv7l$WweK0UZ|b6fuzsdQy3v1)i`A2s`rn0>}lPxUqR!TTcIlr|sX<
zY^82wj%`yrZQng>7{;k>R_-&+vv#)LFob4;q4=}=49bnVxn(40zoa)_DX6u%=T@FB
zCpSCk<kLIs!uPAhPs!Sm2!0~NEfKRErih<@d}r#X0Dekszbhk@J<?7(xs6sHq=fdP
z2>g#;{1jK;HF%1X6s>aD(Y>E-&rd#{D4wPWPwvr<MMHB<o_O#4?$M<umX8n##q@Zh
zs$bWS>uosPktE>64r14wR$C<Xaf!>4D(@Of%xvyG-P#c|kG8&V9f`?m>*={yiVMqo
zx$41}idt!rjrxmE!1(Dr+{;g_PW?36Pu?ln%F>r+=5Q;s^iF2GRaiSHCl5c8s(Wc^
z<9q$(r7~M9K6Y4?p6{5oT%RXxMn({aCxP%JBbAmhMtz<TJA8HuBM+!+@g(sAAEEs`
z#p9M`iIirM&{nT<TcQXnyeK-|BISmB%$Gr`SU4w=Dkek}EfS%Hh#@jCF3bUR$&*h-
z%FFhi$xzc=%@bZZoLlasrRSHQesHNQRmf!2%F+WnFCQ+=OD``k&#k7mm*?h6sJi3w
zBxPq51W{uKt40(g2_lwgk>zB>GOR12;n&#VAUuWqjS%6<7e7zo7$=GV00{#XNxvf9
z{f!rJAQ!#+E7;M!^?T)wR}625%o2>wr@5LZh{QN8mBTF5jEGV1C(g16aR3Ne%7g`%
zCp8?-k+u+t@R|T}qIuo<$+Bdo^BkOEgka?H<W$3EtNeWse(-L6o*0ga2;4ssDXMr4
zEyr@x3{Rh@kXyz`i16e~pg+D0J;QUV>-s4KJdx!zw<DgP{8$Y#=C{J2E3Oa>Nx#b{
zC^opC=uv)x?tiTOwFw>5GO`T%X`nHjE&ir{@_2#-^AnA|HkNz!cz%+ihE+ggM50*V
ztFQ`M4o!1Q{e(T9u2chr#{J!;%7YIc={=q})Qc5^MDz?S1%NG;wWB)HVkVNTob2u>
zHH!8ZWQ-V2z}Rm9wnQRl`uo)1i@vt~Q@HHCE-5=352w*oTIYqkuAeC827U@%6cmDt
z>!-&S3(4v#B4Id}hT3tnRy!^%9NV>~Ehn4JY}3}Eg&sM!U35a&&UJw&ZGJa)yqiWW
zZvd36C?iUd)y8hF`ROXcjG`b2rzi}I>@uT}y(L8<{66wdwC%xG?c8@d_XWaQL?O6~
zNEJjh(c#cF{lwA?i!T;8$jfLy`2<B;!NyON3%`?8wdmssrD}PV?xs^s5k1ySF91Vo
z0el97`O~`FY?=$l$~m5vvY$q&1~_KFZKmu-qiv%7Mmy1H6zp9YrR+Si8qh${`)Q-G
zj-s$E!b926t3Yo-<<DP4tpoBqT}8Fc2cIgT)a;}2>fDYz(XQgqHT@LA3<dFh*b3`r
zG+6_J8p}_>lb5ymw4W!nX*YA6tTww=*S@=u1E(zHG&^f&n@S($UXqP8Y~*CgGBiL(
zvVPw&c*QP}-!B>M=V*IJPBn_0)mR0d;67;yddSUxe5{no)9T2I@|Ag5d;aM}CL*Ir
z&UO71{H?*G{e(Q8@YsG@c@!Mt=c#6Ob3Lq@X{l=vo@~w5j-lKC<SF2ZL~Xajj*4u;
zm%|g(iAJJPmiHR~2Bo8D5w_%0_EEdt*h%~TIlz-E#`$>l2v#;$Cw?f!Ueiyb%HY<^
z{e)ef#`IGtyI4Seo<gdfmf*osT}5pgxSNxKC+lvPwRFOh=cf?xM9^Vz_Z>xT6P_%c
z@KgqZu)Kd{a8_GRHcaTl$ZErubjmI%iXwV1Gn3Fkk@xFlkCnv&lO;U8oJW&B7dFaI
z=%4kICn$U^KV_;rk$BH1T>qc&LNlg7r8O=5LQ~P25h`ozum;A;$^Iv|8YS8;n^u{I
zHCeT6gO_2zRODkj>#2sJ*=;z&wH2$e4wVl0;c(fYqD8a+5UC0bm{78Mq-5{^;<5Qs
zraEs*!f8nmr+yj$3FlXzd+xcs>=GbpF+oSZNSeTN{%@QrC{9NpKZPy|D!88*++WYc
z@7%f1@V$vb>UO&pIO&mYS4WRnvIb34B7HA(yXHkdLDAanwn>h6iAXz58f_gRHPLPx
zNV3}PoL!PrQK*q>w~hUFTPD421-T&;1!<_%JB#2+DZV%xolR)7i;Ii72^SPQC?FkZ
za?*n+Re=xgKT7*>3g4NTi7Ut#O$p>D2=+4G_=)NJ$)l9;<aUmZUye7)m4cr10o0(M
zFyQ3IF;w<GuzanABq5>@YB*lZkgiM)A&M6XUX%#;R2BE40+t;h!A}*&$SEW&c8Lz;
zr~bNEhg?5h#iQ3Zvgrrz(iXbxnI>9LgMLas_uQ&m^3a)#lcg$3Gk4;5@;G}Z9ye+Q
zLXje6qOqugBrPI1S7y>Q4G%{q4@UcGh@cQjFh3#BPyHIPqx>|YFLYVcdnH3#qmT(r
z1vPT~w7v?-r^)g)hbI$}s9{se7G*V?6IJ(543eNAErQgDj5Wo%lFs>WoV-KO$nldW
zC`fQWal`z?j`mY%uw2_ue)a~QX)34zKZSaQfCRiaKoOp1O!uBVpH0Y8420zJgfK&C
z5m{5nDkzMcyF<lsoE+Rw*u&v+ARydWehLMFbg(&`zRUGA9YKvAKdr9!cv|oAl$s&%
zsJfaB3)uv6c#1m5I}S!vv?dTYK^&eq;E7{djQ&+W1=Lf3(bNDNUh8r_O$9a5Px*C7
zT3IJSkW5Jjf($jGOE3hNoI!b1hlQb9I?@EBDl)*-=Skww73X)Jp?X#D@e>=^Pl4tY
z15DBe)6?`6@4VmvKbh<2A!Y)C5<)DhtI=E}oX|7j9APOY2oYTp1YQtyNuw#6l_-(L
za08}T4qxUcQeV$c?4X~(u-I+fTu*}usHdr*`oV)D0YO3X5O*WR^DIItFEW&@l5;p{
z(3A{n0Dc(Yf?QmVSr(HUf;D`lpIkFRxw)Tu@ZK&xO{|~>{e*Bj--~?h3BZx_<KD_e
zKZS1Qr`xEfNfi_xHub9$qI8U|eYy12ehN8ea>~f(2<9i;^V2QU)1(T@8|jSF!p;5U
zb=<&DWUoO#xg$WxZPU}F3d&1>8Na|~xSssbj_)Tt%ujCTZPU}F3hHt{-CR(AWz|Dv
z{P7cp+kl^LtDYuaP{aLnLqYjkzHa!sxt}~-Z=IedQ&2<wgjtTeFs*}Gd;>a1`-wEi
z@)NqmPq$A`lP;+9?*hvLK;9qzRfeHNIlz1<DF~CVpnh`k2<j(viJxwto+edL1K%j@
zR=wJp?dDjo-*-25pmRJw!TQbobc^*g*@C)Y7+<(PUd}NPtVM2^C2{A%F(wN?a3lT1
zj_^~EggG>r2#=<@H5vREg2ga$6MGakzWVJuPUD%k-WJ~c>a`#4{YiTMljo9eep6n1
zV+*Y;?XuGIt)i0IETgq~vsOY*Gy|)=EYULJIG@D+1pRwPUxSUuApigX07*qoM6N<$
Ef`XX>761SM

literal 106670
zcmaI7WmH^2(=Ch>Ah-p0cXww7clY27Hn@9mcXxLJ!97T@0KpS9!QEZINuK9@zx(5^
zJBziLQ`6m5U0v0(cb_O#Wf@dtLS!f?C{#IFX>}+l7!@ce=!<s<5Q*EF?k(gWfg3>A
zO#^J@=4t9;2_<0xHnSv`b1=2GRJSy>@OB=v6o!I=4F_rJy6Gw@3Yvo*SWW+yVfAuw
zg4BkB5*G7vGBvlebR##jv<5kfP@Z-5QIdl!L@0H5l-QM=q%3VfvOX@B8a~RJ=00}j
z0v42FqU6F}f{+FrEZt1Wy&UWvT?M^FDF3x92>Je3%tlH6uZWwS2<3l_(p6F=mjb(3
zlJl_gvY2ylvy<})uyXLSbMtU8lXJ3j@UgM;u(9*9uyY8q^9yovkpK5X3DM?aVI`<8
z4g9Yz$dd@Ajhma3ARC*frzfi?7c1DsnvFw1K!A;%lZ}&;1yX{=)!Wg{)QiQ@mFj<5
zkhXL+cL6!MfxwRBe_J#)1G~G4P(m#IPZu1Vl$8FrVMo{hItt=4HZM~rHV#&HHV22l
z{rXqh)lJ><|2O0RTH009+sTqm-O?58?qUuZ4=bwwsSNSm|69@Df)Hy2Ra`)jK{2(L
z2AjJ(SUS4NNsCZIeqpr$SqO5m3-Fk6@LIC)@bO!+aC4cOv6u<)39#^6a#>k%^IKW)
zTA2P%JO8)(QUYA;TtE&9AdrWjg9E@Pz|SKg$qAI?1@iIp1EqlfQ&-N>)y>q=-12{P
zgCM&9TUX%!SyxcX#nRLb?4k(<+y74#sM>(tz^*o6CvqvNf1^%LuViWta{Mb~_&ZSl
z>9@3{3&_LL0_XyEAphrSLD2uj3JYFdGk!h+3l;$mb9RUa%=uW%c+3P?I5{}EO?h}N
z*{wLZDF0jE;{WjAe<Bqkf%L@be+&}j<Kh)Cg?NsI-O8Mkg@c37jD??vho8mD%8b)W
zz>?RLmy45<4PqkO-{AQ_g6V%mkO=+z^?y?X^6<aOW$6gXC>KcTgg@HRLP6*D%1KLT
zdMzL4A^PC}o<;(fW~CU5?#j8<#H9fMX$hRhk7th~#-B5EB#s*@3(eScjgOIerJ3m&
z$tmf{dm@y5T;%Fs_p-Wrjva1KZ<7^dy?g!QCr+mReokL+v#j$uew}z^44Xj@vcDf>
zS7c#dnB#5y*{ZGkGQd!nFHDO5-RjFT?f#h){XvOQg>uR9h>bCD79fv((nuhZzMNV^
z`&8x7QC#&!_NlhUWucWdwzV;P9>6@U*3L^X;X+UsquX-y$vsZS)&9?6!I_BC(RxWP
zEY*^MRM#Ex`}vU%N{_aTBNibmCup7Pge~&5wY6?hu#J@}+3QG4LPag0k42XbHmDJ%
zAHq!u34NyboQj8JADyQCdK?D*i7@N0bEMOjj_U5t^q2bd8LInk(gEt3K6J`?z%x{p
z9GObt1B1N2^^Ik2pQD>`5H+~uY5tYOwxDR-wA9dN;$|^ydC|r=NxCtqJ2qE=YHxHu
zCm~6&CdP%JDzIcHTC?HM5;%eDSil2X0^aH6xZ<4f)W3XS(Q!B$kj;r3?y2BiGHAZb
z*IhrOY&mn<tAson8E+-Ui)r7Hia28{b;OZYEOuIKXGh+4VA+~PCsv)EW$NY+wb2k>
zwajn>yD+X(+5WkneO{sDgai2|)-iHKD?__PutgXIpP=2p@4j`A$1M_*tQ{5Z!xj6j
z6JuO{0fSJ|KVxEFl*slMp-Y~#5qE>@eCC8Z`}pF|GZEB3hQph>k|Ke2O$m>p=sYc=
z0I58K{ANmdrl43sMNZT+r80`VjBkeZ;rbq;GN@hlGmqs?PoVWXTZbNiOCmZz>Y6To
zo|Z6Whe1|&A@!4U@K6)QklDPN27`_iWI#{9Fr?P>-u+5k0r)TKxOUueXLzhZ6wgGb
zEypXzm8sfZ7cW(QK;FT?n;5TB%E+5>pc42KHqJJ0Gbp=LJ$Z4&BTDte?Gp<;<jMA5
z<YuFCSDSlmICzLJkNaj@FtciJruD`nvN4ldQ@)gw)M_&K1TZoI&lqmGd-p6{uWKwV
z88EN7*EU>)JM|39cYk0BG<uh}yeorO9Ihxmk%C##-07td+Zrs*s8e|=ONBaJDIhv4
z8kf#w_-mkyLw#2jO!bGgd^vr>=4I)Yn81Omu`)*1VSZFJ&h18;F{AZlNycb?<Oc8b
zv3l|b&*3TR=Ej5lPLFKq{%iZ)&?6z=Y`|fsq5HunoXH-#dH1pX$O@Z~HGpj-5GH$}
z?6O|Y+${5(Y$Xv#n_W1_Z^^N8&%(is6=;B0GEmj=%{KM}^_SL7;E}!Y(uaeM1ty)!
zwsKYQ@-onV=qsCg$AFPIB<4=*D$6ko2YX1ukZWzXff!0g(&lmR@FRJaUx2>x*N2U)
zkTTzJtBfLuKB&(<Rvw6!Fz7Tp2xi-@%66DJ$1UykRFt1EW&5fyPiZtSwI^NkbliRD
zu!>s*a4&h)#z6xov@$G_ETiVrC7ib%=hG{k@U^%lt9Z4`mWKibGr_Ass&3B4s_9)1
z(vBCWmCA|7>X;_ioH%3q`3(4xN-d^kYsXSngA<<?8&1YT!jPZUc+^~}>a}(EQI<a8
zfAh5w*Fb_up|8efNV5_HrU)+CxH9+=PeWqO!zYB6P(iu<23WBe`AtAX+l)<fuxe7&
z{NbDBJSK0hKp*c;w58Dpcj`E|!AM%#l%&o`%crUnQS+Bp<CI{Oq$$s4zI$EN%F2@)
zZTzR$Fat@c-n;qX8@vfbe}5yeyL(?uS%Z3e{kragZ>_W;{R}oEX^NmxX}i)5MX6iY
ziP^$}6*XI{AjjW1&%Rcc2a{{&ei~rFip3e7IfE^g6dIpYn4SdlaWO2jX%YDSLnHUX
zP-8MY?%z}m#S?7A5@`Ic;IK10QA0}!#FYBOtiD{OGY>jKw&OM%wef<k8)Mh(?^2aG
z_&JvN*A+z;^>f#Udv3L%nVd7=+VR?Z=_>dXed&X!QrWAY!;+eMC74n^2Lm@xg2ZU4
z85x5o9)l+?5#KjXoQX6F-P9{7wKo2+C(|w-#O_+crEtb_Av|q&R_3^n)INeMcWl4A
zJAYp0F#%8R5$cHbQJQkd<nv6LwTxOb`A=9^9fH@c@yxjzXL=7hYIP943=^wG(uc?j
zB`Y7*NR!PX9-cHmgb`*{yw<hy6JW|UewNp>D^`vxrTL?&sT<mU&tF<vI^RVzW)q}b
zva)C;RL2N@w=>I%ezR+8QQ)pu4~Yz?z1r55Z!QJE<;)rLhRJSQ;8iuqAZp<TPswQB
zqD!~U?k1;eCdaq^8c6F7{(hGaSreADItE01?KmfuYCl;MPAaV7-L%H>vWZNnD-EmE
zc3#4ioWvRyCNV1?vbJq*npL@^q@<XeA<Wxhw90ekUi7kl^{U`GZ}Ty77}9Cp<XD~j
zxi6dD$*9x(G+!e_<xXTPbn~G0y6A`x@^0sDS+lO>ZW$BPe3^HHSSe*BZdu#h(a)~9
zhCP|e(8%sH#7HfnboHLjiV#p4VgsHk>b$I+Wn^Sj)z%MhzUJ=uzT?6(j$lYMqm^=U
z)R{yUi^!&Nr(35j<siirA{j*qXrFm>iIL(1mv|G%S-WMasp-r)*mdT0_NKbGu=K5L
zhD~hZsYH$LZR}#TfiG5_^jA@fdn-k;)|?j(Dd#UAZSmcIA(-j0mf#iYN~1K?*laC7
z%O1~bemsDmnfG`jd4Hs#dtot@WzZlgFE3x(pyA`=^U3Wsp1TZQBzAvgC$_#w1Ha6r
zc2;hk1RzWtqj#!<nXS)6<Zold?M=u6R$|N7HFRq{m5&5;*tr@4JLSyOXl-+}bF@67
znAWy=T5K0B<V%)&g{9~G@ejNP+`oSK#hd=beHl`mp;K8O<%HHy)Omnr<-aieQrNmz
z$+{x$Frh`JQ98f(I*|4$-(-6O2Ziq529T29r&VX7Z)aZjZYfQM?=D=}g;n#{&Zo**
z=!iK?_xxrhNlBPo<qCW1d50>GaFm$2z3=#BWMtfeY9RjTE&p2b5lNSKr7sKLka1&N
z4dy<Wy6c@TPwnZZ_{FA2!Eqy1;IZg`yegl(zj!)JGSSqmVRZ>NJ<Ra3yB}$wVP(B;
zZT;=8o!4FK-1<AiGUxTxzV+~Fro82seP_rJWqQpPkD9TqDfmXH!P?rg#8O#d<IBh!
zF&AU;0Q@+&Eg+ivk+po2VL2iuAO4f$CF{_lzWQFVd}>;#JkTl}M{-70SC!tsM?G01
zF)0c2^G5<KV>Co$6cp38m7RbEci>tfb#2p^FQ3e8q=^pnSga0M$i>7=JOdR6=@=N0
zh<NS$ujUE{PeptaQ*(#X2H4|5`Hj9}F{{o~Z8dKW+s2?j=Y7<2w*y_q=|qt$82bI_
zgo|r$s%+Il5o(FlkS0RH^9kmf5&MFn<QyHzIp)s@Baf-^fOxshRj;(dyTBbfgXvN<
zq{2Nj90#|`&73a_4^(#q)+#M~DFX-4Rd+IIN-64_DDR<2EFWR|&B9lb-$l$3%>R@u
z!@q<lwS)OykNm)Lg%L=0t{F~@AV0vs!J*c>a6|%b*BhMUnk~DDBDGLY41M>ZJj2>B
zw*jS&ypNpRV&AI_cVY)<Wo1RKa^t~sfJdoTaSYz*^}nR#;bcx1xc}i^rkmOO<k*X$
zN;K1Am$6@)`+krZ=UNheU(a1KszerMEK(MkE>$v$rp|o=&xEFUQ;wg(5=GeDub4r(
z_A2^J{=#srg$qARFYEFZfDo)J<VA!D<Y*C|8LM?ziqSS?;B)SrsWF$Ncc$9D48?Oo
znn}5Ou#Pmjk6`<n2>p}yQJ1>Z5)m$`NvVthP1{;Ped#-}^@Xh9humSp0Bgxdg{gN~
zs?ADPhJ?52ds|T?UpPu!;tT_%o{QT1)@(XhH`H)2-Y<LMX$P9m3wAPXZuJMls{7eU
zvCCpo6XvYH6H;HhrhqQp;{&2<fO6%%Wu+P1t7_*?6PS%wc)>S3*F$g}4jnd`yeaU?
z4F<^H>y5e)kl1csw;!CkK5hD(dk(&6PYJb5HZ^gUmX^bHM#p_~GCkBgk3LfH{-Y2(
zn?1>GL>X0XQ!CHY$Qt&AJq%m<@Kn6aH-S1zq4{c&#ox%aY`Cn~IxdjRv4-XRIEp&E
z_#l36rM{jj2Juj0S`c&colxB;Zifc6NCx<zapbahbXY<H(>*B9=Vk3B7LVHXM$yU&
z+8<Z8iOPa5HF0VbEp|p9_MCMYO^}!ss&I0a8@`^3ddfkUl~w>dOx{(Xt(3=P`-vq6
z0K@L>jXz!9dQx%?(-mHa=H<I95zJgKUGCg(-2wPdVjE`#%jyt&PSOz9<4J@{6V&#m
z^4#H5aq>yq>I>(vm9L?UnrtvV&emgi9rR7-zA263#I-5qEIK`b7m0WsfD#hW;b<h)
zzK%Nqni@pacheiF=PM`mDKi|3Yf<I(c|t`xXQHDUePnO7h72Z-^OT3@gXkkg5v0+2
zU#tK<3C>$n?;k~cZ}-C6lx_@`6ry8l>VzJDnkS~@eWD3u{(5TE1@qLDv2wB2?S1K)
z)9S#nJxcMj)qyEJ=Be0)IZx0XZ8?4xRqRhFAxS}cii!X{6M%%#o6numaNRb&B0PNX
zAX&0rt44cs)iYUyPsqi!QAIf8d&}q7d1>k4Cdkghs?KRvw6?@~nTAP2Tv}E00%Psn
z@FtYTtKkW@a$ZwVe4RhpV-DQ>s);3G28~)dU5$q1DP@6^?aapA7$IdJNbfNnZ0R!`
zoop*VfbH9Rg;X)0|COxqIR3?iV%-s`bO{*|%Wu4~My9V$zXjQ3I_s0@%Ttm6({1kz
zxcXGShgFdVeYw}D%VTt6VsNp}pk|@*@}ipAvq>>v#L;2BX(pLRfqV6<6hh22hTG9P
zDr^Ej@nAUWp!24c`#==kX609z;sM7DP|K70Eq+ugr9z3pBF-JQysXfRWEMy_b$Q|k
zC65+zh5K85`h*WsQfb)Gw?ZmJL^!4Ce34I!b6_m;@r`S>P?avUlMqp#6-H2tw~^1U
zvdfZMo%9r0!M$A9A3fHdt8Q0HE-xgWYf%9aH-kunDv3uRg1Qp!j7VWw?wN>JZLmzD
zlf10_`(r6z;c;iCgBEvjP}+22NqC`2Ux0*n)f_0Huc)NQ!)R*!hnXZ-WYsKz|4-O9
zUUaRq=oM4H#8k7j#*8$DBMo1Q?9sk4W2=mMHsuKH=w0W~cUm}(+@MO|JhiV{WN?1@
z*H&6+T3&Uks&EmMh5M#w?pk_g^Je`-8I*k~0dEsv9ebMXoAv6xBt&LCXhSC}@lY-5
zZ$%ETFnx6zhM`j~8zZ}2<k%-~{pMomY4g*_5ja43ojlU!Q}}zo;@jwJiJdv!PnE`_
zhthZz@M6zF9M-wPR6|OVm{hh3iR#88f8-9TauU|r+w51F;WZ7M8|~^O9NXyV$;uRc
zU?!K8Yycn-!|lmh1Sdsu>PeMKR&I3Fk+PAf2qrCjHcj(S7J1BxwKzLHjnxE~`s;eN
zTTk0ex5IhNu}#6%ZugvH^#lKX3f(p<rSqGUz(`{MYYu+j=U*<rPS?6FwzpJ%pZcCj
z<nW5pt#r8VnO*PC^hOw4(v+*@prst@&6FAF7`m>txU8*tuH!bJRFDNSh8nP4pRDjD
z;=Au*SHuy(cJ&x`JCQ^ffO`SmA6f@@<8s*jAKesv9R-z%1B$=z<z<0OT=!T%aJofe
zgqO`m*QYuMd4Z@rPL^phW<03fb|+u(U=h)ipTxFeon6R~*(SX(+tzt8rxpDPOMr37
z6R(eU-i2uU{G{IZn5)ivB$s@sl||=#QAknkze7-9LR=@xCx13l@g?3B1Tqi%p7kt|
zuYBsX6PZ^)N1}JX%UgbwhLK<M*Ewi=)I=&AFRbrEjsTM#o(f^9;^n-`iejnC-x-z3
zR=BMtL5m-Bx<(0)?a5&>*6Ft;QO$dveg}Y@ljgCh3vmi6#D(zta0jK^Nz+te<VC6*
z%8h&>YZKsz4s*)eP+xXc)Gqw;M}7U`cul}@jb>Ci`%`mQSzro<<q792217%0xQOy2
zef$DSpdLLDF%n%F2f)0n#63D_d~|B4*Z*k;>_8Y}wN6eIe@C6#T!dR4n#5Ea;(0~i
z1q8Hv*Ua--zl!p2jBi9IXe>Fdm456vS&ZV}JN`0Lr@&`nWqFX_HTNx_mh2h*c@LUF
zTbV#aKH%+c>`lI?(ftaqBYy}62_K8j7#oHR%aE<_J{jkOb%B4ppFoZ<h%=qli1F%l
zqpd(u%3IYq72W^!uJ6tCf-L6Zq-|q7LdWq`+3jQ<m2bmG;p04x`*0%7B-1?MiI6}e
zBQr1z!}z)wNakZRdojn$Ajj~*|FB9O?&Wr+LETMFFW~iI?+_36jYCy{;bnI+r;Kf5
zj7lZldxlqoRt)Zomq<BhqEoSXk;+-G|Bn(K^bV4;m0te{VKe~@;FZ8n2dS9+jg)k!
z02N^5_8m?$>y+0k>&&;5WNgcBaY7}TpTiFon5)v^HhZ=v3>qT(VTS-=0B1tjl`Asy
z(mhXASS%-l)9<aot?Cc{#a=AfV8GF9U6b(4h8xVcg2NhnnJRFxlnVT@Ucg0aTd6JL
zYoEe$9;qzb)h*5+{Apd2?BpA*gO@<2BV=KC2}_Nk&_~@$3RTz-p5F-68>e4a_cX-$
zkn4D4nM&u$Ix`Qf#HV)MzRnR<Z4QxGz!$&R9m=yfH9x4g8P()!&n85#JHZ->F^Hm&
ztzBO;D=)j$(1CZ)!k<XSFH2j*FYdlRXuoP3TU9bq3DD@F?G~!FUy~qX(O5vM`z7K8
zH`(ne*BZ2;4aSS1y`UYj)iL)*B+2RLr=mxFXqjX7M@B(bHP&XmdcEILu^_}EP0d)o
zY*Bbu9{Bny(^{t@_5!!zcNOPHUfrdwo}9cRS|gI1f(2_?_=HN**JKo^woVh1eqT<B
zWm(v?zf)wd0;**0SaH3$QV}y>s7~u;rEHArw*Gp@cVb;p8G(Al`*D5MJ5%gTh@f+;
zvLt>Wb8Z9<ffXc%TI5g}m~taG4O4d7OErP*f8BW+>$btLSfOg<sXw71+gQr+K{XR5
zB5-3&Qd-Sk=J5pMj-Jp#hH7Y%!b~r$DOEuy`Q4iLp*yha^{{z`RevI-vrGBpJ7ZVn
z_PLLa?1ZWrNYvx{BrG1^B<TKb9kF#si{^u~pa;pKPPrN>A#VoBFiR&ngE@javLjUV
z8xl1e?!>JXDGhvjazH~H@h73%0o138kHFzi5-^A1p$s1u4`q*Aqo4D>D*uWOOuFTl
z6L&vcMY1m7B0c-=r@)&yr%Xgur@nkDwDYGN)lH5JOKGLmK_+LhuFRFh;b>MjJS6uB
zU~H@<fJ4T_Gp{8B4jp=Z@s?FmxYH|^Uf1k}eezrLmcJ{^|J1cwfx@gxIM_BaId{lS
zS!s}G+`HZXK8A)pStr2+@0qfWD;n4G_3motY)peR*({h@e%>8$MAhhX!`Q*%UNs{X
zdr4b^HtbQ-eTBXa`%655RK)3G7$4DnrNv=M8I8Tje^%t^%d}|P6S2_g`;OR4_3?5y
zRP_u}=YyS$G0SFjM%bN)Q=v12&Ltb7Fe6HElV+UIkzy8NT}5}=X<O^bN|>-{lVVaL
zbrE1KO~m6a(X!-uf9@wabXkiesOv1Y_@q+<F4cGxhM{-Jl}58)*?lQ*@z4TyhlV4c
zE}z3;`C&@SSuV=v>cWOC03Sc2P}48<Gs2%(?myD%+!E|ZUx=n%-d^s+=sj%E_LfSu
zVh(qrFEtitHiAtbuG4o=i$1Q=;-Kz-N-pUnZ=5W0LKnu0=cq6{df57iMYp_l-UG-|
z3Ls-t5RPs_D4Z8W*8U=!!;jGAaw;wnfvL`)MM5KSzyuHy5PP7HWfU^z&hF)(6(r_j
z=G%JHPe>gYGuB3^Df9VMRayHkvk=!*Y6KW3L`R3X*6BGy7RU_n@JPoIZ#ER+PA1;n
zKZ#<6E6d+6m*Y!05%M6fuvBo{|0#h?-11m{lJ;kBn-4hhr>oNN(rb2hHsra)B9U|E
z_gys655|jc?cg`j<K-`ix3iqbm>W5~j;WGye)xj!2b3}rs;}}?DJ7J{uOLQ}7#z=6
z46N5jjX)-Jhu5dyZ%TX9z`+23t3m5A8z@co(f~gzJXFs3&a#X~9jA8Vvsz?SU}M0G
zkN0G?Np^H}%&=6&2NVAX8b&5YH8tblEG}z#l=);D1&h0-N7N&ZlPqh?HWs4|qT!pD
zC6ho3F*fv-HZZ!MjVel*!o$r{n9GXW1!s%d1M>HrI~I?~8@w-CyMU)hh2{dhDwfh=
ztCLRmruS|eg}#zjz;wmCTLUgjg21PnrGC!kMT%4Fx7R1OWUS=#s}T4mX6okAI|v3S
z_HwO3?0k>eq_DRF-W?aN6mIUjjCS#6dojBuw*S=)NI#e^3mi9pyci(^<oyD?w8K6X
z3*qgk?5Mw%3s<p@pWwQzumJ?CD|&oyH*c1V?au3@L~FQTMSRwLtrobVS0n4Ju=SWf
z&X%^=u8MSXqekyOSZO(!FFH0bDzvyCo;)9v7BuTBf8{vH0<35}$;#OUU*mx!69r1e
zD@G{u=r~~#ajPvo6du$Ybm?yVa<Ac3D-R-@S8V#I${H4M9otrnD5(@mPX&7FN$VEm
zJQ|~-i`z=nFEFOZKvhHva=*d@gk}v8@MY~~BxUu;FdSb`m^L{-DSeCgb1o9FbYgXQ
zbxU6M{d6_WbAr!T8Et!z)~7Hu$;!l}5Fq>8X0Mw{4bYgDnyDm#YE)XrCv?bJ;y&(Y
ze)y=-d6<gnb=b`yvKA!k$FM^&1K6RuD0!y>kY;fGbMmW4rd&=*DQyIpE=F?<Gwf~4
z;HbGwo~Oyz%o3(j$gLenjCSI^c`uK8l;bD~Iykso_C4*`8-n$2X||{i<IfAE395ZO
zb2^HBHo`x$?b-;xZNqCqo<m6^Y$6YnKbvurqGzl4v)eu=`f>Xc%aBDiCfU2}W>O`-
z!gX%^L&Wq)3O<|P;>W+b+VpjUHGpvgYElUU+lX}VB(9auH<$I39E1+mEugKW<*<C+
zXj)w>2UyERYT0t;1Thu>ZrfvpRT26PcNx+Q4!$+2%ZG}f%x(HpkIrrIrAi5g*tMa?
zQm{{Rpwf^AJ*L*)rW`lF0ySBM?3%l+%*o~eWolGgO!_aq%Ob^1T6>D;v|S#XsTw7p
zOR`x(%bK|vxiUnpJ=VM$rG{cse>yg_G=6m$?bDx~Bz4QpK;@d;!ZH1{lWS%l`Ru$7
zA<H$IHrXW0vdbTBM%_^@pbc&1>0;bQX9SYxX3Mm-95-`pa3e_4jp50Rkas>H?cMF!
z_uMGRHkqQrJ^s`U{rw>KP`#l+;%81QcVW$QMG?eHCygeXFij<@xD>DComh?CrcWl)
z9c7ipoI_FmHB_-Fe~LUedN0=j(|RHj($nR#c;%`kC&gL2G<YzK)ltDzNOU`>S-@q1
z7iAC-7z23v5zpH*aW)SuW=M5Xzbr#A(o_$rGiV)Gy&S&?m-^O*1*$x%9ng#F>AHiX
zMdIbOCU!7^V42+^&xOCejX_3Ohme!2QGonCv<~W~@BT@`xPmq(<dBN2`0DQzh}c(`
zT}Jb1Xm&J3Io;^X+iMpPNND|`+!@nWdHj$~ibg`a_7dZ2J!zDYt)Ui6_caM@o3!Jx
zAH&qPD7bT`Ppf8mCzPC0NGV1SU?1-C6P+7h^@t{?sBo_6T3uq=F4(Gf=(-IiO?M)c
z%DSP}{0UQMsqNrCuS{&ONxneM*ipa3aJ<jqdA{l%JmdDt3h#q~kB1u-;aWuu4@66D
z@&CDm=kxmmNpeFe3nOc^&xb%4$>L_hq)!6w20#NK7U#TWMLnCovb%nYFjZ1=GV^ez
zSrUqt$@A0iSF{CVbqxdAqF%q@=*0Lu$XE&EWT4dWG`|cG4ty12{ay%b5a}y^?{K}_
z`vvelXDHlH^Z^htd{}_-vrSzYleSLU4BCC6Mb-}UgHlB+_kmuhBobX+8Dt)b{`Odk
zt3RV(BI&UbcG#p5Q_-)gNRcmV$_W1c01rPaUrd)jj~@Q`+*SURpbb@3hsd*L-0+=e
zl1saIR@TRuY~=>HOx@FQnd|X>uDS9+`ot>c0+d6kbb&L8qx$DLZGKtuC<V`!tf*F{
z=rTcD7?d%xdRf{k&QKzC-Fb{tld@T<sGZZ4j0GyMslotZp#H>Kdp2E0G->|O$UB6E
zmiSA0-ONF|9xfbI9>Sph+pvW^N{gVGQVEk3IIPSMjmy}O9Z6TRSdp!pU#;}MR)n7F
zY6aBgkHa{@RN5EMsf9Ag-dNF;dlY1v4RhUH8}|;GE})rhgVv!E4t9Wnd_H#}EK0q;
z&{961)<ku(I!{L+qUI*7lS$bDRXv9IhP*Zi;i6)J2)?>au_tdGOhw5k7;%B2T|sUa
zemr$GdM^`31E6^NNbr|JQi~$8rk^SH(X^vXP5~!{D5Od7Q0%^E>6YDY|BcMln!4BJ
zL)0?r^prQUkk0O2J4g<_RhQkM98KT%`BELu`-_~8FhgJsRhu1r>a8a|1#XNgy!MF%
zLR1+<p;Jm?Cu80$W=Cly1_bBWVhZ`o)MR7Y{wY?Jb7iNys6&C-lq2cUSCI~@rB=!y
z31f)-lx2YamjmK<oloJE!O{o9>-lfvGSulYB{pva)BnWGjaeu;8yy*QP8`IBl!N@u
z!$V(`#);P9YDJ49Ki6%Y4gHhP*+XWrm=wIaz7&HU>8byfDZR^|9VB@tCmx{kuR6Pu
zzAyk=NfXr|4`SViaF7SYreyNp<fK-id5arzY;M7v9PIxnWW^M072BE=eeN(9Ji64;
zqQlD4d80j#3woh{bX<13*K9FeVRO++n(XGP!^mHiieIo*fWTz{Rr1fre|9OR;HfNs
zP)VYNiI@H%Ec{;+l!O_iqc1FQT7~~IX7A^ihcp~i&HnQ+$nAZBQ<b9(HWq5IB1%c4
zgSGl=Xn9s-h3eHZNNvQGvuZYZA&TWc{oy9MD8(3uv249h)?>z7yRmS$ZA!1;6gvNE
zU-&uG01{mg4c|IXu2nr(c;sqQmQbEw5!pAHf0t#6t5de&{p(7#%m?&{b#CU+<-=_c
zK4=QS!1)+{kU0=qp%-=vW5b!(k>#J|^Q)IQ$T4L}o*QbJVTf-itG+q+YRWQjZZ;Dv
zVSh(ElkviWTef_8Z>+RYn<uNjOQjcvG>`f9^Mjb%XIe*wf30Xn(S<F;jcInC61uk;
z0cf0_sNc|O<U28xnU<Us{x<WWywj^1X&bQ&P3y;g=SlbG)jD1OMolnaBFJb(b6||E
zSoDd|X_3{|2ZGl?I3>TXYo8tPBRzkfa%yTS931mP&86?RSc3c0uCtI?9c}ILoB!C)
zrcN_&)lA~P1HU?&=Lqt)<%X%Ag%*D0K3{9--vdLxC5DY@_C#yfLG|XN#}binGGU2b
z?M@C(<q6GHY0wb|K3~WxUdR0~==S2kW7dUA84Eo=wr{puLNf_?C{yIKou_v@{)u_k
z`@{-iFm@iVrp3q#0$&Aq9ake$R743_Gf=)*O%6ar40E2Rs{{r9w*=8KG0x)*6~7au
z(2xj#|B@#2r}`0jR^0OC^}TDq)E)H21r$==X`U1{4&_nrNYHagq8{X$$!g%+OZ)Az
z9f!}gjqt;Eii-deF&1!ZBw++ohB(-B-5aq+yKW#7lgjB~Cu3_yhAeF$npqCAEfq;;
zF$iy4_fbHuI;!tdM*a)|a%F?v*S+`tbnN91Rulv<&i%6IeaK-^{p&W3=}kF`RCxKU
ze86<4!~~&b(Dx28D2@D}X6CvLnB#TW>oe5m6KR=R5&4gkSikdSq^xAL%eJ?-`_B?Y
zAL+?i<x<r%h>)mj78v0%S*T*LkjSe&<z@O9(B*SdQ>n!m7#Qf8m<HwO@<JWgI>-9t
z5|f~)nDF(w|0QUZSZz2?`tbWUI{t4dIy*U1+H?$sgbHmW0F=HRXlD$I#wbLO7gS~w
z`0GOoxY@Y=$t6(rx8tYoTNo!VvEQ8&l&2xm=t)UQ6f9~cfiEr17<pcsQL}fT)76fj
z-QHiCS#OSiVM9)sgLVx);kmiQhTc1AoG&#Ez0uIH2*3AB5`aEFo!=9fwXJiV!*M=R
z4^0Uk)dtx-J?s_ic(53>4gv6usIPps;`lzhA8ETC%mP?7P=x>GRq==gWN&NSuq3&N
z-m)Y2P{*vnOiZ1=%G!pb!CHZWRzXFcPfQuNvg2C2o|O~hY@Pp$J>hXUs~?rGI#i8-
z-HV#t6Hk0|^ExKzY%g+WxpH$Sh=%C?GO&Li`_p0~PvmnFgu0^+L?;#g00~W*k1xTn
zh^1*!qE~y<888ukX`EbKJ3d&N4-XHQo)D6d7%gHoe0GKx*^P9cudKp9SzJ9+*qf8o
zXP@HX?z8FOIpm0-5QD2%HOk2gnx%gDm(raf2!LV1x+J&8?@a3YQt5Omndh+4gYBDF
z^YqLrSNM@{j9Qzr@5;->pr*TFgnii9x8#2BYl*J|8(E>XdU7axgvj{!kUviO_R9^v
zKT9V5I(FIYaiagrzb)`BfWfBI<0QJdnVTVVxwQ`Y8v#B({=L$gSMy0UE>mfymUM<b
zzZ^m3rJg~%QxT5tcvQuij<Gkq`A9tAXtB=H8%GTo5<C9I=f<{!O@F;a20bf&{M+9-
zFBT{aAatl$a;N|twA$gIZFSuicbaGjpZ^RD&R+|j_y%i&?9Bc87L*tG1V}Gq%=1S-
z4{U=$*@enmmRTjyJF=R}jXIdCFy9$Z`|2B)Fak*prmjPq(0n=E#FQcQFnX`sU)E(B
z9fVlCZEGGxQ^Hq@ZfEO8tVbJtV#;DKZp}6`lDkqcig_Cl__u6Mkj0?QF%L)9pw<5C
ze3iz<<pi6K9pWbh)PbJ6^%xHGZ=)`phzXa^Yd?*2gSTs2N9COQjZXm~{L^?`OG@w8
zQ-X^rFK`jTK3M$Jtb^=H?A&ce{1Yi2%|<1KQbK7T)O8>;X$OWPxEK9B37p>@h0_54
zeL2V(9;L4C4T^8gPL}@#f-?>;=$IHO@nc$q70-ALMxc*qsEFW3!fP2D#}k3O)gJjk
zVb~H>=ck=VtOu|FJgN|8SqszRq`+&W;Xe^%@wX={3V*m=wq?8TH)A?svL&f<{cnF|
za#|vrFksT_w;_vte|S$8_(-$MrXS^}Q|og3@vWHcqTictD|5!$^l;$hC+C}GZ0Cs{
zGmGtJS|<pe-N5h@nIH0V6WpmF5s?~<VxL~OaF4>4Eo8?M8&we*HPF4zSaL$B6?zdf
zjro=&)wY;zb%Rt^Q;$IEg|)wieeXx>^aLAlQHrT*5}or%7I@G%@z^yn%JJO$bc&*1
zF%(Q00p{`C@NaXwr!@|IaX&aF;YXDGovd}R8N-&YypIeyqK}~Rt&+c$JN7j6KFWWK
zEaq3G{yv{h=VDgz;~lu~r4>iNZPf*Z*I^mQjde_>+VgB(V#nvWape3fBH8&uu>uV*
z^XogEd=3O`GGDZU*E5<`?nAfS*LmjWMlpn?5MMUqybEFDyZ8ND8f1f(GM+&+75JTK
zhcVl<f;uf3#%6fEh_Rvw?N=1Pah+Mv#1PUEhi*QXV6A7KsdUTUkiB~kxreyT@gAG&
z0%xqLxK)rR+Oca+x!BZBaC3|;n%vl#>>**=IhUDVV;J>0t_|8C9Bfp0sMx!|ElB?=
zjq^orawrP>;^l5b!TPjJA!9tr%eK-t;W^&e=b(&SO=^+h@kgXE@;3pC(Zv2o$XW(K
znAZ$(Va()qy-d1#fZ)>c5X@ZO+5>sMirwkAxL!WG*t06KXtYKdnv#TMeD2SzkK;?M
zUo|oeKcULx2jhzckb%1eu+Nlof<l)Z2e%3!(7uBw`IJP`<R|m`UZRWwQ9^$%>>mfC
zjb2ZWUuM|KK#Nre8ymqF8Ic(oSHmddeElO|f)6~Sj?UkP;5msLJ!9`=L=(0ggYUYN
zK0BvaefU=C#VJ^k$2qIqTUSLH%h$=TVsYWU!%%o+x{2tX0%C!qmK<`pu1N?Epo22;
zLp1b;X5S+rZ(;G9+CA4Re$_4NLUDnjqdV?<t%g8w1+!ZTVyynDvZJaWLhy~fBpzuQ
zRCRv!y}ib)uC6lHc`k!EN6R;Lm8l+BA!JD5%ggngibTn!WS$suuiP_)9+fzvXNGa#
zOQI5l-!5hSN|hkY`S}&H)X$uF*sr&LcHT+wsxxD!T@^+e@v#52eNFtn^I6`|znQ>*
zul5^s;CQ!*=!pMbUJTbrUPRQKsCa!JlE7X4fQ)42>t{0pD`5|!Z*f8>I*M;tPc}Qf
z?p|k&4EG=<3dX&@(42y2jA65j*MjxIW)RrSwL|PGx$b^rkT1%^MesD>K;)UzleXON
zvurwe4!^<tQtIquidbEn6P1;So7lQxlgTq<|H!oBu<yomkP2A`@{)=V*2*^OAE;h`
zQ%Y9CKw8$vJ_B~^onW6@jSH71#dowv>>tPH`YG~)?VTsr?xD{65Bh})`sEQ{&xAh?
zU{^A0B1msTCv33PFYIWgSFKG1=Y+l!+Ohb3df>GCq_w2e0L#`*QEz>Lxrh9A3iEo0
zrN}nb(8SUu9WZwatt^PyzK!>BxW3x&F52#KYC^h~Dwt_VVX^P&LH%RqE`2`Ej0FVe
z7(V<0%;c7=QF{=^TpLn)sPPQ$m3bIeEqk>2eKdewOlu;nJoN#6l>>=i?_E_}0XE{s
z09$9}r6R?91U!PF<jw`0Z*fFAlNZ^Y=gJ~C^ZnpUOUk5TQc^idg_U6$7)1v?PD19C
z)}rQZ_IhJrI(?Ji3^)5V1ocx+tc1)OCO;b4=M{-ZC)PrwfTW+2UYazSnw$=)t~idu
zCw>5%%ltQ#hdZz#e$lZbUa4W>oodG?q3xl7OT}mPg%<VM;Zf|~wR%oFlh|-W)b74u
zulKX3SasI+$^$eJF>M$_=L}LA?B<{3a9r)h{RAj28U)>eJ3rh7-m=XKl7_%4SZ}pH
z<qu4_wvA<ktgc#m*&N7kr$5N=(yxEgJ9~{aSyYStV)pzv;-}d8<R~oS%Mz{N@?%OX
zdhfIkwyp0GHn|Hasm~vJs^^4M(MVrcWqAy()(|;Tux>wt^jVVY9_M!uy=PVX-SI1_
z?xX1U;a3k6itXKL*Rb)8J=gp7%MuLWrjHTrEZHLJ{VDFx`c9-Bx18nAc<p;0A1~dK
z9Erva%Atq=0d_9~6B=-<Jj7mh6hFEWFe)TN<H!)t9@QJ)wE7n$V_%g2D1}4W?Ly(l
z1^b%Ao1nj+i>-7|{N4}S&H+D`TApDNj5z{>(uV~u(-<9-UFD9!6mFU#BTiWlF9<*`
z%v8LKYWoqj1T!s+@N1awbD!g_1^oEL1PWFu&U+1&@7D)&h+nLy(OzQlB%?)b8%yIv
zicX4A=<-Lia%&9^yXkeUsp(H~N(M}2zQ$|GstDjX+bY>NQPjDTjgq6+vv@rYcT^sG
zajd^=(>CK)91Ij?wLj-~olicPJ4vJ?m04EtUp~s5Uy=+C!yokj(TVGy2=t3tyAz34
zpcM%Y<TZg_y3aYk2EqE_Nq@o6($4V;#uU@`3m;=r;y=9wdF^cmnC&PFJH2Xo4U&N;
z6^d7^P!0EKW00a+`E@syUT+-jMO|P50=!|CS3=pJ9%;VkJ>X8|KcJN7&Kgwi#w2+h
z66juC!SEv{)I2$|G_+5r#vJ`9p365t3SOQ(I&$Dc`igDhUT~p5e3L{jZ;k1`RTT<q
zZO`|N4ZTarI=@y9w!E;Dyzc%qWa%y5^mB@$emEoSoM09K31=_P7-8%Afd!XMYa{Yy
z<Bc&l<2ty-NkGxs1mP-IY*@sc?c6;XL)&w}{TGHVn2=?*2YMpj?*cfN-hdQtdV%wY
zLXnRMvU?xkCybh&OF!ukT?r}>g%xZ?G|pmNcT2zz*#~@_@qD$7TGG@IIN!hO7cdVp
z2=jU@Xumi$g3T2Gnm9#2&P)kZSpO<<lP~=K7y_5NoDArTbNh8k?6aUuYJVa3`3H`0
z(QNAJS6~A6scVG7AL(X+0P0TCM;g4^jZZNM9T*ui$RLr<C$?|=&9QC$wDl$YeeM%m
zH=`e$9ph>%-lE%TC6%>Na=AJSHny|kNTbFib-V_1-+pHlPDR{F5%n=HH}<`IIyq^y
zycNo9rbk+g7G`E$4hs=E`W(Yt=FXZgOB|Jfn^X2kytZTJjj5TReI3!ZEWjZU_F=ih
z*sxWOz327w4-Lu^$qU~>Wr|qd7zG2vh_!@}3Ri`fE80UGE9V|IKiJlVO=d6Yt_|w=
z*NPh_6F+9=k;rkMOBda?m?!Z9V<)jY+C)wNXUGn4(hve<P=8T7KT+x5ZRu~H2&N;2
zQpOuN$ejz`3P)C>hFid)C`Sh2=tm#!DqiHZM)FOe2alW~KHkCzHbcjY>3ibO^8nWz
zs1<RTlJZrsqLNXGc;{D&H1tNn(IKvk6ljquq7T$II3|U`_W-;w)xzJov7IhhNwc>Y
z&#lck4;~cE$*J=7{mW!x19$PHne5!~ykC$UZ@Lp(Eu7b=BOMGy!9ooNhUU>+z4(rw
z2FDbQM;+Z}dys;UhbLlnw$zlgi51ec6y@WC*g3wc>oH77$bB#NZ+B@*S}doj<f~z(
zfiI~q2q*m%PF`Yss2Q22eG>Y^c|cLb<xrvgGn8lrbAyR))Z6!0xOso&a{h+E4jDgn
zY-!_Pn`g-6{)Juwg4q16ROc}Y%uEr}SB7VJ^p_NOp{*={;pl8<T3GC1|Kh<$$%otU
zrzbK~@2$Z8!=3)-6;pu&eXm!hQ-VCK`)tT+L!BRsFy>Yh-XVS18bkZ+-SY+59hPjj
z??O>Lg)^{ot=6Bn1PhCf%{PYBpP3T$CXDR&9N!FgPsWrOzf-e#fBC^Xw_$}Xv@DMH
zq3qosZaO0giD1Lqr?bAT-5hW|Z9X2Gv0+1mO0ObLyf}e_%w9B%eRAP!llcf51Kb#A
z+4qJK`Z!T&ht4~hj4|-mN|?^?bB*V<rm~Lv>sO|W3l0^tfk>1e19lSY<ETzXPL<ui
zyO{z9gD07s)u+=p^&6KM=}UN6NxgB=YJaIkr_A{eo{fiAS0|g1f*RFWm-sbkqQgGR
zHq1{qgxTylKOwVk4;(qmL2l+ZXvFY`4@STy^ai)1G=$#G=l4EVtuoY)&;?<C8=PZ6
z+Z7wa^yy~Td8`+6E#ns_M70e8i_0csXjMfNKeq}c?Jw!1z1IA?uETz(a|EiQOwjcD
zwg!I%RdA>&jO@_Yhx(l*_~`X%=-_U^Wrm|aEMZXi$+jI#(7#9*4x*LDrPdH}Gjjfb
zj$7S-;KH|BBHCPG!WYd4cryMiMUfny|DITE3g4%&#Ouocj^_8U4q&IWcsy%$+t&aW
zeAs?N`0>Xd3Qvi_$1#FjNho?h2zAS8)=nwOfU1-wd69KEJ6nop8?8zz=w_oTc<Ej2
zD4?&h^20_f8UTnt=<&Hp#>O~X>&j@z{CE?ZFC}BXsi)_4kX~~!baZC^)zkHSJ}u*q
z+?-9Upm{e&b)ECzzOwA=sa$N}v;Xtsf<gP%#ooDT7m>64*)6-T5q(qF@9J7X6|@A6
z?g+1HW;MqQezx^Oe0;<>Kio<BfPJrA)#QjJ(r###z+3qB_9tb(c6*WW&AM6pxtgsc
z{+;Or_M3yM`{yh4i~bgL(+&(B+88#m?3gCWG|z;Y5~pzn2Cw6Rnv31svx`Bx2D1T%
zJ@e&=nxi^p=iGL*5o>6jJeaxbrQxtEXs3LyuMgt2LJrK`dYhzkcQzF2F&{gW3?mAr
zvDie=VHDaTCziLdKhmvJcdcR3t)ly}7$+O|qFn45L0cD9a_7iu9EOjPT;ox#VNj1m
z;nP1sXVT>(et#srKeYTgC5+SxzX_P2++NzkHtpH<(CyWO^6OKLY{nBWu#d+aW}lqA
z#8ER1gCI%=tthSKUKrM2cvnk$_w7G+VgQc<?vUtxOx)68`uylbH(>ZYVMuf7XX6k%
zJ)oyL<X7!IM-d4GHsmWX7i76b=*4NG)RArFOrd2HRp*_Mg~TPpn8o^DZgXPSG%M(I
zcUGiJ-tq%qy&3lD7qP=ARcYC1)nxkQsAqV=y$@EZ?QqdS_10Oox0eh(LXO$D2(xq|
zv3h5{;RRUspUZDYVhP^9!pk(LfY7=;1oI*LV)U0Y2jN$L0mja_k?yzEuz&+R{+N`R
zJ{g&2#fHPnvHf3|C*MzAv}5_3?bk@lw?q2}1zG?Ik8PghNm!+5{n%jPkdad#yp8S)
zK^M;M&n>dzO>->NeMk-}qC|IVU(i(ig%R0aQi20YM$9EA!YWfSc>(JVf}1~&=NW|*
zj|R`roA-YA>~VCz4QT2L)0Vo6LQ7S3)%41q!JpL>^t;`O^b2j$lmnXD3#DHybf99y
zK0Sm`h(~Ay(}n8#Y_t64pg8`PkC4k)51G~4<(1e`#mF*`;PU+dwQs-Zs(~9&U=%eF
zgMc_2ez&vLn{&(?mqCo(3Rq9+lva8sqJK~$Xpc5(2km#Fnv**i(XQ*XKUeto0e-Md
zhI|3{>LWKJuo=;FnM0AQ^KJ$5Hk{XIgVD{x(_W1Lris*|bNO02EC~9=R4Ft~G0V^B
zPlDj<aW>G-HErK6(|b&|LEl9d05{xqk#(U|>C#jQhd_+jec!BW1efa`rb`VsKlBDQ
zEo{ir1$l)bmXHTI29G7Vj{z@*Qjwm0w;41ii5IrXSTzWx>ULoO?$JV+psw#cWF&}?
zbD$`Fx5@1uEgU8JeKM0nD*aEs?<rF0FUZ~LV_NFKa&ivl%c9fv$sE3|?G%;tVpihR
z@Zw~RTt?dNw1j$i2VEo{<qVnqv!BcKhPPP6PSiy$80Pf9!mQbPFz!kX<GVfCWVDEt
z(1}T1KZ<8zj*?r7j$-$8R4N0WX!C4dG!0|g#OUB=gq#XIyA5~K(gho*YR^6^dD6D=
zV>Iw5W`CdBFMZ1SMcKv!x^m|ag^GclejH_AI16i3wxc#;zKTxe-A%VrFwd$=m3;5V
zw9YCPK-7Lj@o`(Azc3{zRQsJYsmTS3)AzB>Nj}`n+jm(6L8ekt+U#Mps;cNm1>B(I
z=Y4_Lxm-^jLubG_szIbGfFBb8SfJyenn>cL_{>>gq^GB+N*K*AYB~^Vn=~Sap^-mA
z>RTJ@ri=}BB+;wJZOWlnq{~m>g*_8%Hb01W71ZQt|G24^8}R4G>-wO=X?ob&d|aPz
zs&8Aj)Q0t@o!-;I5sGVDUrBU^%I@ZvDCj{9x`NnX=MBwdC`7dv*eqNhJ|LIRMv;r9
zx+XQg*(a*Y4+)!A;ixvtm`l}#FTzRFDYFh0y7`8z703A+%qrV=+t;0C=<qDy^K?2E
z3!~yHU5_((<d|S3gP^lOc+m&l$O_`$ZEQgVU>FTL*fF_<jl^MRHoYe5-^9vO3)IT<
zs@F~~XmqB4ubfXeM|5>s7ForzTgYOs_a)a6>b3qPf^LauYs<RIIQ!}ZRWfGL<DVhd
zC;NS}AHzDeG9bA5T9>De2u5h6&~B#rwxK7S`<@zMG^MhnqO*tCW+aux{Do{tkU-^`
z7rpKaiD*_Iafk0^@cg2Um=_MKNC%(TPYA9xp+S;1t=g8C?=|bO!mt_Dc{2LgY&oHw
z>w9|tZ4E{qLUZ^A>8vB@x52!gnH5FPz6UE0J}otvu(3!|o7woJk&5`hx%cht?M+x-
z*G1OvOk_areNOqobE>U33itkbgy5gd`G?JZYGI5&sOV+O=hnPBPhu03-(Mdi2=fpW
z3j8U?P;OcemG!xvk+EFL)U2k(X6@L<5{7^poQqg`RrMXa*+EW}qgh>m7(Bllj+w!M
zC<rZM%<k5Picif|IK?f&Npx+~RcSYNoRdu>KS~z@Tw<83NtC4-vCA&s`OY;x^!K(n
zWQwJ=$(dm5>t3gd=laVgV3&DC<Lr&uS4AJ-U?}8Q|5UMXS1-4A?h-MqVPx&xr(E^g
zij$Zao#hUEeI~DP+APm;Og4-82Sta+6gdjP6U>dV6%yX8dfZeg14Kk17w~2yq6U!c
ze>KtA_^YcYzZYQlIMxNlsUuQa{v(tAU_y@X?-+XesD{1O6I!V}c6WK}XFP78hP8^5
z9JrU`&6s8)TOmZw0lRX^*}p1{bN0mFaB+&H7qLnfD}#{dzB`O)Mo_xBraJyb_Cc%j
z`a7Z7EAfBL`cOU=U13wF7`kZMEw9)($p4RXQ!*50C^?ag>(r3Mt&~~G;s)U!VH@QC
z<1|2*n+)X1lDdVds_=h+Az8J|+5eH#ZdUQGdFU6={i6>uY1#5A;m78s4qDCsE6UFE
z;<WXijbWeqi*di7J08;b9kG#(Bd=KHpa0V|_n(`rnDT8QSrfL`YW~kW0o9{-Tg22U
z|9;$+{vKk{v;!GU2wi7I^;d!B50vC8-Ta?>|0~3i)GJJVx8)<j*;0i3&otBQI4|TZ
z*^%s0)a0=>kH5+XKYJm6Vvxos^IZ84jd?trLKn*;chIWgm;dwfKm4Z#Fq|<G;^NMe
zRZ`EN+)XP}2WYLr?uT%xI!-Kq142<FIRA<kTTjKNRBxeefw52NdgaaY_lnh3S;+?8
zMWf!%nYjKVJ!l?u0KS2+d^K8Ps4OutS7#ZE1|e74v-D4D<CV;FPm$cl7bmzQwSEd0
zr9<w>xEl<)kQ)y}{X#Bd0?|Z-f9>Vwkrihk`(k2~sH-L&Lzz5ZEX!oE5595PF~AhJ
zH7zwRZ;MNw`twfNyd_EZ;zem=yjSk*N88fTQ%-NoNSUCvi_WDw;YC}~jK?l}fFkjK
z{Y8kx76KQg-+m7#XMb0*-JpYZ<5_u=3XepegzjJ4!jj<8oJY7cjQ@Z`{-4NzZR%IO
zGweUKxJ9MZNPl`mW2T~t>GPc~eW-JbQM=32=`nt&do6d<ChFD>^em`e`B5`2wX%b7
z;&LPUymQS6dKn>~dq_}XrTSBP*vj?yjdKj~U?}D2o2cG2^gnKbWvVV+e!VKJ85;Uo
zqxkYj`3HsEtQpU~){d--Yn==9J_7#?LDdxcY8|E)sPObY3i)9No6zcQLp2?=n8kAt
zB3ga3e7)>o(}$<@y1a4d_x~~V&Vh9{&--xOw2f^yJh9C-XspJzZ6}Rw+qUhT*hyob
zps}s@^l3le_xJC8&fdGTJ2Ut0%v{$V$Xj|QUqV(~y;%-j=I>vTD(0EBWNyf_cd7kV
zvICtTyeq#xP|i5#8!mT|yhO59ShWby%)Z=KA=+U@8Ib)T4v)JTE)y#YZzM5~XWFJ?
zW}e;G0RGWq$vAY`EujSD9`tdyyr&3qYm$Rsa~ujjdag5%JSBqYqyT{Zk=}m_OH~+N
ziP%85Y-v^-l^V}Gbcq%!`i)6U>A-OXI9)C9pWUcv02d<wi4VeQ$_j}9AY(eijHNN#
zmBQa7Hj4!j1}&HrLZpU=l24{~(}=86-X>Zy^RF={x9;dX$wN}&3k#Xjf^$m-O#i7m
z3#0={_(EY@1WEXRUQ9lTCD^J>rT3pD%#p$8u2hoCb$1Xo>W-rRH@l8vd3IQcRLw@X
zK-2X9W$O*$DDmswf6M2HsQYJve{9@wB>-%J{!~+SJSi?ebp8`UPtxz)m5HlWnqIV!
zO~GgR(!h!90q(C0mB;a)jEjK+ZGBWMZaHZu=j-`9NLmL!gFpfLUYD;F2V<7|pUVFF
zk<@ILneP};<PJVyudvT>+NjnN_+LFp2qnCkTVi)F1#x&#+RCZUo!EcU&R(0K@yWE7
zPX7Go%OvGVidX1jfnvPN6Qa?Q|0`P{1;z=G!p1M_YJ@HSEq8#M%?HX`ww~1U{j0VB
zPXbJA$@hcC*~kBtHyGyr;B`YDC@a#XM*C|I8a&u1jm#*dvwHu}BU6!;U;U8c^rGbd
z{YkRi6v;Ld4Q*N>?n{b=>mOynD&#XkLIBl3#SSO_6BI%YTw>#drA!aY%KNwU;ZM6Y
zrx=e6o>MAn`RvV9*-v;!fBoFvPa@_R)ju9hvHO4P2npZ2UqVii2sr)cJNZn(Rbv8M
z(E3u~Pj$_|H7g7Uy8t7IdD6_>0;}S&4Il8Q3SVOWG;nbArst1BL$<9<Pyu6>@RYs=
z)bX(1T5&I&I;vD#z{61SR;SVq{hlHmgggA84ne8?dPWG&&1Vk!^2t}9{*s8c+6w!#
z{-*1A$5z~BF+v8ecJ8aKN8tId9yqXKLAQ3=)nCV*+&Xdf)|+kAKwG8;+76J^IYQR+
z?1*YOoZN7<S<io?FKVbqD|oug`~B0e>c1in(|0Q7X<&;2e*n#Y&jaeH5w-a}C<)R#
z&|S$89C(db=m;3mPFI1+57QpU%Qf$D6Dz&}>j5^?(UFi1<xBcjz2<%sBBpxO`K({c
z;#NSy_6NfGcer6o{m!u90h6<BQx0AF4@_G&*{_!+^B6-MM@ptUK6%@my+sHpr5D3Y
zx@hp-0(k7t*6w#UfXU}OG;nS&Y}X^<WiJqmM*%Amn)w3+y>nUSm#I49<6qS;ptG7p
zb}X6Vn~ni&DYB@Pn@GeDN30LY^#*|XD(a8oRB3~Xu{c4QyfYV@Q%+d!CK%g`T~Xk<
z1ID>Yx7l>c7#M=x;14m}{gRrG!z|oRy7SB%x!J@7FB2VT&TBR}A2RHXsMW3sLsm|k
zNbUxnbhNK9wBD}ZcpNWt7|{3hf5mEFRF&`sBA!Hnk6vpB6KlR*;EIaO{!A=e3wqKL
z8lePtcAbmbt{PT$!~|(@F&^q?Ja^K^reTG~@2`p87f;%0Q)Zhw4m3+a8?TSBPuolk
z4r8hu<u3Rb6>4G6JDpgr7VP~c3NjCLHPSmB`p$e*8W{QYD6WXyD4upRen3T6zWLV|
z%vY@@^Z+NoScvJ+;VP^MJb^#k@W8Q1z%HG}<&xW2R#O66vI!@7msBA_cB3NEGB#@o
zsoW%M?PD+#mk>b$aok5-oibrn>Jyc&COFKuR@A|))lYR3hau&p)Em5DQ%bFK`5_cH
z4Fw|2Hb@8Hi(Ce-TN+a@hF7hjQ;Zx2D)rF`MM|gE=k(c(EfT3*&wub}D0wFnVmMLZ
z4T?CL1cqHK=&fT$ZMh`4P(3-aWj2HOJUoX`7cUEtWI{T3pOjxBlgMTI-NWWVUt6HH
z;#7kP>01#9!_o*{KVx1Khy5T+9QQ`uU%$l&%yh)Cd({ekz-8}Yw)v>V@kDd|xUJ9Q
zaEGeGu!(*An2XEw(rdABnRPys&&_Z&Ss?)BX1ES*6mv%jvVDpN*eWfsY;dl0ebOFJ
z_J1;d38r=3tu`|3iA#c1wgZH79O3YAAN+6_5rtDX1_}(NzfP;sWVB%iH~k<?^b)8Q
zYEjgajE4>(_S9fG`DQ_w_3nDS|EB_?Quz=n)wH4PS69zmiAD>e<W_?m;UpKds&zy(
z4s)X0ekY8J3zv&4Wn9MGnSd<%W50T#+jV2GA>SqzMbG2l2iXEI(RQwO9N3?&xZ_YS
zwuu($NZt?LNzHnHOk<kP6XrJPiC8<bm!nH0BUO)C;?uIo^6hWl45}8}NtiGE6debb
zwnR<XIL|xEXg*TmVt=7!fagt1p2hu9Y@Nx2{fWeb(VMPtbp;t>!xf2Fo~D|t;p=GQ
ziEQ<~y9M8~!>aAgjmbtMgmfr`2JLqAG?(U122{b-RINU;5g!Ng*|U6jdJOyKG;`F~
z+%_z2k4vJ?rvq3X8*c!d*A;<-==J6#Az!awm1lsb=HO;#o3vn-6=$(V$GbXyS3+2R
z7nmlv-%+Yb2w=XL+1txlO%*QdFr;sZNW|aq>c&F_>2QN#<;9p&bZ*_Y>=ZWFHX3z7
zY(chL+eiW>o(P$3+cuIzPTP@h&w-eYT=2nwQfFJQ$dT?Xy{=JjD&eXy#GZ|!eBFmZ
zSjOu#iV>ts0fu2Nm8Taro&_>^#Pr3GvoBz`JH7_4PwSBV1jfC9QdTBt-2Ft~HHa}j
z5w^NPp2P?eIin>$0CTIFVCe2!<9tt0IxX-zp94bsw?A#@?bVK-JGDbaz3mf(AD*C^
z<+T;{CN$yl(_}?_QnwbI>=f(#p%}lt(X$VlrM%OVtfC?;`p{)0WfQ$CUJCon@`mei
zerBb@trc1lcQ$aL8Ecptld)~9+2cj?dSw5c%#ECMbFQ=E1F?7zA@cfj)@Dg@KZkXf
z(F06C<E*znGI?%!`hKYnU7Y8H+Ht!_xXlBj;GO|SB6$DlyF-7ASkH~CW|TpA+Zp|{
zmC`Q^#~jX%SD1jwSm8u`GjL3Cb>V$h8k?HS%bGQ*#?Tb#T48tVk{IbZ^wN0{v=;Wb
zbv>e|QGLKZmeB?SJt~AcNq<+6Go3AdVUkr3+r-bWwxyUrhF0G>6Cr|mHsIf5IYq@9
zc}5JL)SK{YSiaybnA}>Pm;^C%*O@>n6S%w+4|Mv>j?8l>IO0D28(|dIl7t@=Wf;(M
z^kb19ES9T3KT*ZSFdBSm-A$H_PNMIZ`M{*^{QU@_VoS4&z}tEm^7=-VUITsm{l=Ln
z++pDv-f5z1*kk2l6g=(%(PV=Em_B5VL@RCp<mF}~DN&iHa=k?y{N$}h`C`3xaFIp-
zO7X&&Z}<)Wb`TK4=JDr*Xs8VEp=rFkc$6TJlM(+wch$KOBiPQ83hi(lXZl=zGyAjR
z&rxmlz`I1lO;FKi9gQ>^4gNyhQ-WWgEd3XUrHlI^WGsQY`_6q!_@IKiCwl&h{$2aq
zcYxbLYkhShz`F0;$1EHvl+o9e<X}Q?^&oX`+s$EI#6A~$zE-|2gro?<pO6S*iFPhq
z)+^Ormek>+jXi-hJhnGmSm{UIA%bdKRWeE2Y|7+X9lNsOLn!Fmwp(RYiJ-()nz5E#
zR5ecg^M11Igz*R<e^rmL_9?n)Z@Nio*m%HVvhu|CpDhcm%oeT84~mCR@!*Xycopz^
z#h~ym=+jLalE;3qphxF?^ME9XF#>)-Beq82oM7S@bt38%Cb!_{jbPs6MJBCoC!jAU
z0C<hmDO&;#>oOKT70HuUy9q9i3ANxA*d(2N^RVM8w<qi==0sE+T_NG~#-qT1<<hy2
z-fA}cE#T2?MpUTr3R_s%IG8Q}U3V;a3yY^2ak*e1sG=-Z+_n>z`e*zQaSy3GO@*k0
z<OeP?VpgrW8YChk3B1n+WZ4ZX0Vio;Q|`Sv%_aJHx4Q{BbBW<sgZP2tM@$UI*G^dR
zM!l6hL`Eo<KVl5?0lCIbD2@4HFR-?=d4|MXYTJxEx!+t4&K1xwl(UD@e_lzS1i-h6
z%k5ngkMHzk@?h5f_+hJ%hQ^@|1Z~YeZO=&CE_POZ*xMo1u)jc7xlpByG*1ZL5vP8o
zwIiV;f!>cg_6_TtM1>?6m<&zCRLX6REg}Z*D@_}>DbFRLh;rc|hb`cmS7(Z4W%s%d
zH82D38h@wC!31<ukz6U<(U3mSD*nKRuGos$1khxzRl^|fI4I_>{Un*Lb7Z5OqO!+$
zAPCzi(PYX#WxE!FVn4fhGj<B&a^{@c3MTrU61T-|T-sg|RADHh&92+~Q=ZY#h|nSH
z0t3TpHLw@L{rnCfe{qv0KkeG@vdX<n6n>D3Y!@q7weT_Ts<*j6QZuB)#jSVRWx4CP
zX~X_vo(uJRG?ID?G_Mw2?^9HxZ$m9s6VDB^(H##Ay7HdR^<p9i%d9VixpU5Fm^$ND
zj{b#4AyTOp5jd%@X$2gN6iUUua})~%YK9nZyg|sENDs;b4E2DM?l!j(5DX)lzQc7&
z`qPVfw(yp5ix~U$Qa?*{2q!#Dpy$JR5N>Tvn@jCK4Jmr9T%KT^FKdOTF`*IGOAI(}
zbGcB?^!!Mtrk_j#Z{+P6Ok|M<NE|n?b8?(lTv3>(M=XH@yiYYw!*<zs*=D8|Waj60
zj{%F8tRVl13PZA(uMTOJ-LDxGbYx+2zn~g8!qD0306Lx*ix%hPMct|z!R5@>{mdZ9
z07|)lty=jCx9#mAcl?Q3l=@B`;=;%S%tsacT?$GHfdX1c%vDxw%<hnmwE4{}p#8;G
z670oM6`}5P9Yl@cVSx#1)dmuXMr#9#n<SdQr7v>wZPkzP4*I>C)nL$uC4}4cdsjpJ
zVwxl``Qcp8WKHt;-4=f?`w^*Grg8AONh^xJ{pT8??3x++hmm8<NHXc;D^b=cI6192
zB=ZRvTRJ;xn|*%bDkj_sl>J`fMCC{PyMlLUqeBatCaP$=+ySSy!@CERoj1}jZL+7|
zq(jbFvxkOi2GoTz@y@{JU&x7a)#fdD)@W~ZJx~Cs7l9olc)nh0ORkf`{XHEboYn4U
zooFG{eg~zo-qj%!oXlebRE<e!LR*m#_ia?^Fh`un0QMOWy*Kma?j)Px#t#&_n9_PZ
zRlaX)PVFz3bFiwVXo*!LsS~s&h57Zeudc0Y8Bn6O9{MqoddcUkXd_&Lt_6*-a#6Y#
zsiO5}a476_U{M|tx<g0!GLV<T(VtFA;&9$P31|JAB<EeOq)`&YthY*(vX+()BL5oc
z$B17G8KOS|1vJr4d$yS%HNcqimslT{n`DcY6GH99Z=DZIC(MjDv8T;5{76jBL8#~g
zM{vJq-`{C0!UGCj0!-{S`yh6T(4y?W!*z$JMF|0KM0BLn3=B9m&q)h(dgU@0diLTS
zb*a`Ox{e7JXMUVt|AM%f{;6Pk@nl*6`#wZiN!goSQ%?vrO=Mo|NC9`_8+hl7@Bwn=
z(z>z)IE<@@+tilx<kP>^Usm<)t=~n#;tl(5-3D|0RhM-{lMd}nEo3sY86Q>q>3!QM
zitTg6ThQ#&J@GeSlVz0C3uHH*a^OOiH)5Cd3Zn6Y*Yo2#tjjf!bCEDB5RTJil=_a*
zL&rMZEf}?+tg9Dd;T2Qj4g*Ga>)UYSuFK)Nxy{`8*_S4sk*{$C8+o2P(Si)^X|Teh
z{C(b)X@Ol=;zH|IXclUrB4_vfc+P_&iz46S^FPLiFwv>DBF&z8p<KPLl4m{L(t7Lc
zW>F(qH|8EkJBc<0V6^ixq1sBm(TILReQ)=Rm|8|j93`57p2WUy18AC%$biGB95Sf2
z9I6<}k#3^_&~Ovwxvks?98IbixJM@w#rUv4plfO4kSjAEPqz^$<IFvsqsT>080SaF
zwj2b_yuRXJh3LXiY;CUbiyyy$`wkQ?)pD;B1X1YEk<MI8B5e-BeRpuP%=r3XxrmeH
z&-~q?cahF^v0e|tYFQP5yL7C|D9(CLv){@@4in<ZC`TDw?BxfJC*plXaB9jJwrXng
z_pLMWGob3)4Z`bt;gb=Mw|zW8v3?k#;IYGTb~DX5BWh!2$IO%AZRk;YuKYv@dUbyb
zBLf;T3<5}aDYfv5APj|p6zsshL#n>}LG3;WWB5CMjgG~~t+Hn4OJ)_CM1z8&;h#xX
zMVKMb7>g3OpsY&N8vS`kla#M(&eG*xPmDb0mm(^ehz5p5&^3A(Hwux&?w`geMYeC~
zwFY9A$e$VPnFcnVKeV%V1}7kD_}+#nhPs^rqV#(G<Qc2ZSnyolnDSMIibi}4t0a2`
ztv!8y1sE9^6pgg6%}FV52?-3ywsY=(x|V6mJ#UJ-nDUc9SFFUOEGZF{t`1>Zlq3K6
zb162&+^9VeTEr2NdJDsvx)+*R=^b~dU{q@!T<o^TOCs9b4}6+98fi4yAN80xdx&<S
zlNh~uRY--d1>KJjd|jm+M9m1pXo{N978EzajP#B$$mCSlvP7Gg{jbi7PzfY67-75E
z0{J(iJ<!BapKz}{a$x&ih0YSOlvF~&r*qGm5{mjZ&2sZN;(5cc0aYjqx@hEG!xjoN
zuK<&$oqXxFc*BWkWSab7i=@yqYpkF8ZTVOD;b6ao9SRr)SDwCo>2QdHmbCmV?=`9d
zV<i7oW!TWmSXHf18idg$COeI1P6h=2p3W2TtGJ(=@C3*3W2)mZ^ZIp#-M(|x*4a%}
z$^+_aIpt(621h=1<&{ezUY{&$nS5rRc|z!o7L1ck*8U)`2b)h`l=nuNq_ZrjOg!94
zczgW5dDCAo`m3$4Bt|`+&4=#@hv6Nr0l`rc-`tEotK}Ew+}Rp~I)l!Da+bsBYZ1L2
zEsM2nXn;%?CTXs6SQN#;FV53W7R%a!JAxvctGa6*b>YG7D{y?!BnZ}F8Bee=q$2{m
zGBOf^S;+g_uatM2=jkpbCMaCDJ?C;FJ+WC);&JGd#8PHqEcHkHC`~+U==+E!nA5kv
z_;}xUsF8uLA_88M#a@#omT9S9wu@Ume*w67ikBNTpyE%6yI!2U`N*@_n~NcjE6#!T
zx$VC{*m8LG(ss>YmWAkjlf!?yS`>G6d2x;VI!Zfm+a6kKs%7v@enE-8VqKtLe7;*2
zFM;ev`b_=&G)Fv2GT$TyRL8vQNnTg=xcjr&Df}R=mbq%xBwWLGpqjZ;*LN`2z{8my
z`Ss`6fPw4Gww-H%eJxKUyu|r2Z`0;lg1f9jOo%ZiZST^FZ_u@Szin#8z&<;PUwBjG
z1arjc6^<RNJ*?$c0MM)=rfy|24aR09Da|mPgwl;_Z5#v>(R5E^s5#4cSAUAN?nf*A
z=g_uvsQ`iky`N3sSMVVQLp!3!%+Pf!xR4tO1a0=~W}_c>$`<FZB;m#@2|yG$Tci>6
zjxL!Ufn_TKb;WbZpf<zoox;A0*#YA6Mfp%32rpSFA23AlbG<b1{M0nK?h*=&*Gl#q
z?}=m}NZ1vFjuuM<B#lxA#Ok@`en~-y8IU`O-Y%(8&Ahh3#OO)L40N$70uKvrR)=FT
zu_$}Nr1oooU%8}@ulWpBB0vH))Kgz~>%Zg%eB@{3(y!F1xzL(leM8HoN3Ln$`4#?A
zzm88wFc1twibtRnW`narGEo18F&#zxem9m${a*}KK8vN;s!fzn1`McB@HDd5!B)*o
z&7J%kL57d|ybFHEhk`#5P5FPhBi&S{hSUE)oC=OS+tp_v$6Ij&|7Kj^X7Pf>Z;CxL
z+lk5*kN@V0B;zn;zPFw9ei|+p?^B}q6JON*17C}5EQ>)zd*b~seIt*^GBoYPdg-Ir
z^*5S&=<|7tK?}Fu_Wv?Nt@;1rtD78u<GXVJ@YmCTHFgMfAhqBAUm7UA*w)|nf8lf^
z`b+`xnHq9%6u<g!^b`w2hKdKdT+F~xFUa=qP$FDH=QNGkbfo_83B-(U`uC*(RM~$L
zz=84x_+%9ILIlzO#8=_uy;+JmWLUqMyXyTfGsZN(Z1gn|;5=pB`1P+q&a%;Cex{x<
zc3A8vx~nIbVq{f8w32_C_p$JgnN6dOm}=N!0jT4Ehp_RcGf=b$irxREBRr=lz+^>u
z0}Y7)C#)*Oc8yPe8ret3@X5q%^Be?FMJEDk+G~QS&KD)*G`9SKI8pH!V!`=_9lHe-
zG|n);%)bk`;h-FTdu~;l?BRxG%bwHs>r03v>ie$0n4SK8VR2++Bswdrio3@_YMrZG
zi8@`{xZ;X9%uGoMBQ`enKC+u(9IKUmcXzkPs>$9Es~Pg>rO-NUJaBAGvL6UM;1}u8
z0wdkB+uJpq-FLa?HohkXQ0J{TE`wm2CS=nAj(sqA?PgZoYNe-*=5QaV-m0?BhgUbM
z^XjpjPy`!4l9CsFDFW$YIj>_mh|U-1`a0fQ%p0q;RzM*H3F0+lH~O<UoO<I;aNoHY
z+{Y=SRrbNo@(UFFqB%-995xRbd}vw?v<hJ@OB%@~rqRBKcwJHr|KP0dvI1w}ux<MG
z`odwe`3_9euoIxD9!K2(TwT=w^CXohJxmEpT6%+lJ0@U`6EwL$>+oCH0*tT=j*1e|
z(<4?Eh5r=)n7Lf3$qml=D7Wp;bfjr{p`xJVV$f~P_^Vry66Sg1Hu4C&1;^OLt~p6N
z=OIbA?eRqgt!w_Q;J(et7iMd99Q%G5>8O%vj5&CLyH}+ag<I8z1GqEULZwf`+!_|q
z>dwXI)^$#uYSnko{{8M2GeLieFs$0-2;tedSNJy(TKur4&-Smbtp!hMxwc%Q7EOi*
z!a&C278Ow(4*~)=qs6y6XM%G%bWK99#v*o6Q2~=2XSC?TV3tp4u)Y*jPBetX6p6~h
z-GRl$i_|}k^_0H~%Z2{DM#urP9SXQDb#9EUS=2&9<L8g{SuY(|*zd1%Uu7{HPn`EP
z$FyhQ3F!Ph2zTo-I3flBf|{E-h>3}%7i|q;<zO;VeQ0P3y5s0u&jOSv(Do57Uy{ze
z{*bA0yNKJm<a5Nr8>u!v?iZEA=1-H6lYf8Wlj}W+*UswNJ+Q%`7#~qE^SpFi$*^Uw
zH^80c)9!AB1DrhZcE8-+k*EdZWi+4m?)(}{=ZQ2eYpmrbL-co<!93fp0BCQWd8Vi=
zYm79von@P^A?0Lh=F`6fEEIZ-J!bK5)^sDSw?2uR2{-HwAIv7d4J(j)oMUQ!@9NHj
z;Ff1TU$m;1Zskm&5dtodamkjt)f~f(I+gt@<^2u#!NHj!UGd32H#j`p0IWBQ7Hnjh
zXrcB<swDt&a;hx{-(uCe!ujrm)E<7BtE;Ohb&d=SwcfWN84g7Z*%{r{T_;8Q;%Bd{
z(>15o7cmRkMwIzt%9qow#8#qRbeperw;FbGB$?`(;du@hw}eKF-rfi~5p-KUHTo?K
zojjvBw|Kqsx<=%Y-WdrS9s|#r87-ZPG=<$dInHn{LR!B+_(8$5;5)MXFt%c%R<bhM
z@DaF<VKr5tkELlUN(D6!Ew^Ugw8>KrHRT|diRsNI))$|h_)<wO^_^bBmAm<gDp$J{
zlTfTSWZ*v`Zn6bLPZcv-rp*CUvIY(t)Q5)X`gKC{zm4v(R4HF=&Y~OFnZ6h~h3YSB
zkPDarXNy(kapvJTLE87IltXARz!bm9pV=G_4<5wj-`^b5i>Ukz2uctYe(4>uc;AO}
ziSv9WlD-sOZzkjteyU)RfATeNIU0aVosasuKd~3YnO1{iKJ&0$MhyiZWaYZn+e9y0
z;sQArPOc1fW7K>+;=MoAarKLu=UvXI3t`vx@R`IQ%|Tr{sNtrBM$At{;w$v4U6?4W
z;@`V5C9GrViv$<Z(N>ZWw?f?Uu!Vlp#Eos+R44F$wVcT{L&hc$H!#3bO>2y?Y}8>)
zfr5|NPeo?mRJla2nq2v6@KbnI1l5NTADi{nw$cJsk&X}1LU-@!?0f)mlmRC^nb)e^
z3O~{5#r3$8`(TAhEdc=rSc36#xyI9P9U#E^UB=gi3nNECGkR=xU6&`9l^3Mx(a09i
zQpGE^)h%%_*GaO^8rkFT^E=VgwXwiie%6PK)U{m2Y(C%gZvGb@Lp-@HyeLbXYKtkR
zurw4Kq09$M8gCGMr&@CuSA_b^)#0}}W^S>UeaKGL43DXK;WtU+8EV2h?oP&qNUX1E
zF0h}(B$8i8yV~z2+Jx9W5-U9`btDzgv?m@@GE@Tu#>i~0?AcKd8fy)2lW4?7cf-Uc
zbxPLa-*E5n;6IdGJbGckHdgIknKE-)Us4>JSt$j0yxf+xGaKW=<2db&Y(WI0ZkvKu
zqwc-N@e12v<mg|c5CGZER-=iu%R}v8s!M_^#zw2P5ip#$!Vy<o<b10y$b8Kn2IxJr
zJ+Ty3DHPT5oy90@-#-L*G}Vv+glfVP9lON~y~s|u#_ioL+(Fryi}|@A7cDf7Boo2d
zluM!5mx*!LAqzPXAYy}W<$Z4JAdo$#(U?lVu{JRi7!e#5Mp`bd{u!~w*9ciF$RZ=}
zW7-gAoS2A9bmZYS4RAh$n2GOQ-N}8>CN9&$#c=J~?_As|hu<cr2MYq~m1nETzH<8a
zRDIq@7M-b_Nf|`84kwK1BB#eCYqA+fddKS4ptZGw=Q<O}HAq&__A9H()0*{h<3c_d
z{*UeDqr7z;MWJ&dp&eUb{WYlUOn>fIM*Xg5UY#oAtGvanK_kXdG45$kiz@<u<+8A-
zho#&48;2mbTiZ1q4}a*Gd!~`s;pe^Q#tZ~MTzqAMa?Q5HAOg4G-d=$d_o?h}BHmf}
z2%QgmxR%Wppn(Wh##=BHmKhAJzL}?%rYtO%`aKxohrhJlGlz;qx88Be{)6F?YHR?+
z-wELQta(U$-{bRi7-kzFqPO~tpO2y2)sBh2MthRXZq;j58Bk=KUMr0cIn(nTfT=a*
zL2B41naCW}BHt`tj>hTWAE0=mY)Z4#%RJdRx~IX5xq9>Qu7T3}_@KN*68j2G;>PiS
z{aV_ph2b=PXy5zwA!ujE0Eg8~=;`((YA2#w*FKv)t;~`G&A`y+OBIqpUI{LK;JP<=
z8arOhy2<=!Bt_?NeS}!o=wvQGUxU;m$K45AdDpW8$b%K)!0q^oa`i@k{w}wy8btmH
z^)TlIDr~>`qQgXXkSvVCPUSa1f55~2?B0YJK-j?0L%@^nY`WINRY5Z~@X+?*fZdNZ
z48P^`hc&&RaD_y=m$VJ9rwG@u#5nEa`TmL(E84YIq@vLP&LIJdzDvv*J1=7#m9cH?
z(w<M6-0`%CR=*P5j2?W`rjj!2ILOeG<V6%bWGatUPGJp;r+tWcadhcMh@1z-Z432{
z2P5$<*4tI<&<Y9)E}pL@ukJRy2`^slHq0wq0?R5}#ZhIzMR118^`^^rV=2rn&zGav
zjCvnSN=izvmD3#@tf(KIPXkWgb7*<suDe2<9>ZsK1_OXfS_8|Lzbn|z-XTA%9H2$8
ziySbf>viPKG|t;`Dzv?a!>3TQb-SoMwE2a0b^*WL1ay&oUuD0W^JGArX#Lg;vPmSI
zrKAdrMX{xuB%&ts8xNJv7v8ruUokVaZ|%31QEk0J5Lr5_WfC|pYT|yGA-atr(yl+U
z4{>y=8B=|<H62`tq|S)Qn3%m>AA!+SYuq&kuWV422)^rB7Ek<pna+vj?NW0CV;y1$
z`XixN{xH?(9}0|j5PHH7zt9H>?NzHgHpg6EWq!)c1j6I0)EPx6H4+ZjXmWBtcwX4e
z8=xt=k7b|R2nY%0QgrQnX~My79MVI<5pOr9o7AO|0njM)VEYr*lhe|7hVHF>)~Q3U
zcHcc~=Ld;2TB;8i6Ml}xdLo4|{}=*L3I@cqH43(O9UO2P7>0bHeZ7Bju8U<06_^!t
z5v76uFd9{4zlhj692mZKj4lE30snaQnpd7ZtbSzLrS>C6FXfa$zsAA)S&!=H_-yrJ
zdu$w*DU<x~#WiECd(%hIl@6Vn>UZ*bS32d`BD)mwThjt22L)sG_T@6OrHW-PV0NW?
zIh-FmLoTvM3swB3URUH&sWvAK3zv)fQF2XInoLYg^;@+wC}irvwcYNUK9Kd*&@e!;
zK}xC?GVCckI5Nvrw~Yw$hH8%nEYF{+!GU^`+U;<JFK}c6k`9F5rr)fH?l<VS9`14m
z6LNLyKh&nhabL+1d%5DT-@{C^1olM9xokKl=PRzBY@j8=15|YsD>Z~b2SKI`C-fJ+
zR$mw<rPlk8fLYtClQMx7s#BH}!cqL;jFqv5{hGeJb(lJr8n(1jzz;<0bc6wQ2U_7E
z<G${i_|+vjiz^%t2HZPd>rE`TP9!+t<id~_zE`b$`p<WQpc?HZ_|ed;O*z|%F#n-C
z@*b|3dvo)iaDLrx&G_)&(wli8B3jH=IFhrPo^&gH9Ydbt=xC&qwpN|!*F~mAX5#_9
z*9a(`twXeBr!r054iabn5=il2>y^v_);!T$=AcFU+dBHDWRvOQlOM+9ZKd`W^b<s6
zFYknY5eoo>ENnR5-xAx|hcPS^i<$g{cHer><nIy&g+Nq4Hz;&QL(XbQ=p+wid%>8y
zj&v|SPCA<dP|(O*PZf!g2xLc_EEjo27IMipiQUJ9&7l^uGU3SaNsQAQj`y1}(6lkg
zyW%l8Sr4S$2w`P~7qQTPhIksPUoLmz`Ho8T?Y%+LZz>js)m&tvomM0nqfiN;X95_)
zHX{c<&Yt!@`x#`OVqL}Ed%b?Yc5I!g5`=S$3Go@ANOdOcotw9GaU9F@Xf?!FXRieE
zRbxxRVKf?J*6IqQEL>NWa*y=f&Kcb9gyGto3JOFmz*)1HBmN{XE>Oy=%r%u;|GoKT
zTz3Us&F>Zh(*9d8MG#LX@oWe@gkvfI?^gHf#rg`beKl)J7WM}TZmDc6L6d|zs|5F2
z&CqXjYNc~9x(5XpcXS&V*6_t_#@FF$oPD;#{Vvaqz)+Xm&sIE!)Nj4P>{n<4wX&Jf
z%>`)CJf{CwF%Gb#TR+lY{i!?O)}J>GT61N~Wa-0EeM+wEozCNdrHg=vmItDYC3R~|
z9@Ip!$m#W~&{zt<e*)5G4ZpdnEj}2S)z=ZXnQ~{#j&9h0Zw!sRTyC;0OOudBlUb@U
z0MD`F3|y3ohh|jP@2X_~207nT&AoXzSj|5TeP#Ye;*uZXdw|LanWU_g7$AVg`-ICw
z_fbX&XVtq4iPf)~wG&C-Gv-z5K@BE7>1N;_i22*uz;sJPa)Xz?(i6Fztg`Qc`mw0L
z1&40XX+1#2Efm!86p19IxAb<?kDlSHn|ALh{P8r{4XF|Sl}XRD6t`A^W#s-VF1rcu
zhu`q)qjk$dJ!rmpisq|@afEMHV9Il~a}m`E(JXOVq}m~KQ*>z9tAS7P@OA$Fexqx=
z-d^L6S5><?%^wn<Z*bWi+5(Dy)PS)!)&zLGK0r$H*2Z8X+i%dCH|x)nt2NOV?0f?_
z89vKWd7|RbJF!%HE*N{Hj5nt>&-H8xTx;sc@m*O1h~ql>Am{Jvygv6Agcr}hGQ^&)
zEGs=vWggegp-@F`fg15+3)cLTu-fGV*l6EDxB!Nmy~>?Q&!DGH4QU$z34z#Gpu-k$
zZemPl=`mui^Oa2NR9F9U(e^nLf28_?-fn{|0ujf@i3S101Q&6P9UX{msu{~y#$*%^
zM^Xo77pPq#D}`~+wl}Du?i2MJfL^R+7Na&3;??dpZ}JQ0ZCh;WLb4x6l61n1?4J4|
zw>zK|FQt^J2(BN5C*X0@Pq-;i&!5~yl@ZZ5q^hDROJ14S;(8H8-?7^2xZ5CU=M>=4
zRJ<q@W&`yid>dPh4p<}ay9#2g7Np(ttV@yN_v#C*)Hg+!tNBVL_L!E1@hVtZ$HuNT
z=ljmHMNAHM+F)+W+g&Q)pj_paUfluyx3`>x05yD8nQB)Kg^f2RJ7e0(16OXJI52_&
zE7W3!m~`~3HI4@Bq5NctaoZFVvLp9wnZ#U9WxM3uJ#HQklcXp5Xu*%!1iEY5s+DGP
zjEfnJf(pHW6BsTP5Bk73R`pw0Y;6BwjiX}q;L#-d{S4>zX`v`IPCFb-)6YpghmY3>
z_G5RgcE8~_!peJ*DWrAn;a4`5Yp}F~2={xhFOpH(^jqJcVV(wcRWC8&vV^9iM;m$<
zFE5=IRYlqbS+g}QFJhf@ZDvG?>W@`2T|ZZ1gLCc5rWK~6en-0C*gv-TIdbAoW$Ote
z8+t@{u~GAKKN80oZ}DtaIzR-O6fN|qqz0=JH^gfn%S1D8HTGs)fXMAM+eXFttfh%I
zpq)ofD3ry?90rz0&2>N2Dy+syllE&y4k_*)6yu!fyT!ZU1_7iN2q>h4whBjPjw(J4
zH|9@qncaAs$Zb3N%|{-hYk(0dx!)?ny7tqW&C7%4uKGjBgYNP>pgVqnpQ7c*Bk|Ju
zGEzT{wp{?KJi~VSVEQA;bPhT5bTRX|=fvGn>Z<l!#mm}AJT?m~`mSfY0MDDq7mtTE
zrscI~d=SXk{#V>vt=Zzub)_-#L;S=sbQHr1N3eBQGs0vQVV##dla=YN?diTBCi$g^
zhPDwOGu+B{2~WKnbwV4FL#7_aczQoIci;U^93{)xV<xnWM_(Zx`xdX;OwN(9h3H05
zeZd^KW`v#PJ_J35Qw_Uodo5(Olf}P^?=AB4)4tu>qVaw4G<P^YgYQ#Q<xb5R8{>7O
zJGP6Ivq0mR!Fp}i!D=N@j3Z33CU=x0V~1;-<qH+Czs6x@*hhfrEuSF)13)Qmxb_8f
zKAy@dp$_KdjpRP1KrJ^6Y+7p@LI^F8sT;_zRQs;<=yk|eTA*1(!X9yg9WZCU8jpj4
z3pq1e<JLa^xv3$!{Mh3fxn>TE1&T<~zk$j<dL>|u>?M1A{qb6~Li&|TMtRxp4je<F
zL;zjFBf>Dor8Net!p1!>n_D^2J*Ueg8EnIZk<j#V?{qFnHQCwL$L(l_xq6I1(yYp#
zPRDg#Drb5aewJu5igj7~^11PnQ|EI|65so6ZJ)JM$L(9i)s(b{?F?ve${*9t0&>5N
zuM+u3m+_fa-*w7H-1aTG+x{8zJ@afyiadS+xn5M|XH3U%fuyX7)u{FDvy1@4g<AKA
z$+72W5xW<)jLY35HWI;4uyv}V5HYLbx=51CPn3|b_vva<QuNu0l+o?syJP9E5Rl?c
ztGxRVd|-}KBJ$W)Oh;}i;ZBPsrghId&66cHFmVlrf>dg+;rm}@%JhnNR1+o!Ve@hJ
zKYPnW<nE`7Tsl)eYYK5b&dE1B=R@a26ZuP8xGqePMd`J56g1&cQuyF(C2Q0QXF$9A
zzYDoNIt;Nabj)&z=x-&Vc`X-k2v$lSKcszff_;ynK&WY&vKd(lk;91o99K`*$`TVS
zft%p$v$(cfQGrskzUh8+GUKpm>97>4lcbVKUJ1H;5a6LhTJR!@<SMw>gaKX-A<;-{
z3rZ{aYd-dk6sRac)ijbJu2^CwB>)=x1Dn^;^p0=U4e!lFgM${<yV=ZGPP^my=?kp+
zSX5n=*{xGGloJ%&3}z>v<1xPmO(1^+kk3qY_<GM<UF}(oAiSzzjzA!;4hu&dO}})u
znPL)zDxwV)&g~wr>FutIh)m@O3$|`tGbsP6*R7b#%b%3dUD=M%aK7ufwxo~rJ}?Pn
zkLW&h*kTi&Wy#@V6#~+aI(Ry0rm{;TvHF%z!Y1Fy-~<54EHtiJyXaTHXFzZdNpIxw
zLXM{m#7^0Nj0(pSx>}x|J8)!v5df)NSu;KQYTR0eZ(fYj6?w)V`0v%xBU10z(Kp8B
z%n`N_D>afD%b{UM4C@eo;qfm|mxRe!2XqMq*KQ7}^)En?e<2M`p3;#dSQ-WO7&YlJ
z`-cuPib9dV=JdQ*D0*Tc+#FeR>O9Lk=q*Dcy@IReY0*oym5YyA;$C1jKeR7@W=#BY
z5#5SUW<UyviPy^;2JX-1nWLVKhnkGO@iL(tT~(bw#ag@aTJrJ_`s>*&4@oW!M>@zc
zkYDcv>MF}}e~n@_hB)K9LpHxFLQ!S*y(Z+_o(9%6jLnDOii;tXRN~abLzmaOkD|}}
zst18l6H#w^Q6@f;i#7ggMvza96~<MwLjH_VyIH*B5Koo6TRATWUi(mw6_11)Wc0f?
z#&3h|i?SzbwN93LA=EzYRxemFudr3#x4feWXkqRayJ10tvGi&p9b@kI$D@R*@;lvz
zQn~|+B14__9GHm~1_Mp@ONUci>s@hogf7W*X;wR}r}!=IF7)4}Ut;nJ`FDo%3xq=J
z4J{VGmacr2TqtYF5&SLZ_^Sg<&9+?3Wr~o5Dm;?d--pOQ7EL4=`K0xPY4_$x;SFpF
zxkhJ<!ty)vS+H@6=v^;mZzX3;-rtbQX2Ax)ypL{ZE00^Py*4I_HIsGFG^Q!=%2$I-
zmxBuXEU<XIqO`s#v9{d+b09uGpjJTQJ=EzKr!?YTN@aj&bx;j@;jU>gsk9!DC%%P6
zLl+yMyWp0oFL_p1(OWUGZI5sl0v+Z4U<4_P49P8UrU7FKQVXOc!sJ*+KKSNTA_X=)
z%H3bzBpZ^bJJ}yZ6ew9*Z|}HZZmb;qUWS#=<NSTrinvXemv2e?>|UHMC-)n`<0oC%
z<tlN^zV0A?wolu%DgeHvqM=sUX+?}sYAJwJ<dZX)8Ley;@wUjJ&TL*L{&=M?=uTtw
zF234Wo2v|?eDC#r#Wdz_3<|cy;OBMZDL1*MkD;I)QOi31I7#;p9Ut0#<&}T<nsg<?
z(@<~=>Jng^e)AWayVxbIv~f-;6A77(l(3%OiFFupPC(m&j`an|qz_6T;TRDmUovp{
znmCSYI+&|2P^(z}DEEK5i(v2&rN)?eNIe6ix^+$alcg&VOe?t63^=e;vO2BWz`Q-G
zbzH-)lHjAb`n;v{l+dXvB_0K`qsEBa>jd&-){saWZ}>A0ZqD|j1(~!%Pmkeb>fn)a
z?u|{`nsm-v#C^U*1B@t7n^7G?k_nn3w(hfuN47^|ksYI>n|kJ|PehCTrFwnZ6MCKZ
zvtK?fXR2SrJP-qIJmD^CZ;!ea1;id=ZSYj)qGv{e!hauqmT0MOUnqBE&z=%46c~^@
zy3(IiXFPj;YF+}!H@Mxy$Adzc3mk3>vs=o@I&qH`>e21CfFWU^s~vMwJ~*Ay`%N)`
z>+8Bd8*seQkRn+WIdm%6tk@J|t(*@Ukv7tFt6V6@TJm`+Y8Rh-I%9a+(6ku3B$!2;
z2?pN~F)Y;pw__n{{4~M42%!hX7<_#quBR>izA+ju*6E}NT_uhoBWz4_Wid<Le>P@d
zVhzk&>RL=ecV{g_Oxvd7o{jRf0puo{ZhzRrv{)t=iZdDx&0dMPyX;0^3+51rAl9Zj
zfcrz<#^r#tVDt8pYU*^nSOZ7EbANwY<Kkr4)q-XSCih6|Q1{w#Rcy-Gj>m4@<+jrF
z-&%S)=ecI6RKeUs90)=G4Aa&?I<tfs*dZb6C$t)2r+ci>d@Agqba96<{kTE>ar*JS
z%kr<E(n)>U#!%+VXbYdVn`I{b)}wS26Z2WcqV3-G?I}#ZL*1Qyl+xRHckO<8J_k#G
zmYS4tI4yU6rbiPt*|%Iu_@0}O>~d?NI`TMFcN-Up3(hzXYRpT?oX#3irsL}r=FPM)
z%rx5(M6-UNvtxvsz`3W`rnV;I$pcQ}3lYX`p^!83o_S?}h!KjN7sPQzgi8N~1i5E$
zr4zEn8r{bfBn$H&w$337EJ@rN3jLNVJ(%7-Yu2c;z<MPo@GYVl`7KN?(zbaNTwH2T
zCiu0vwKu|}iLdb|%ME3Ag!ZT6>#Nbj4RM8x?XILs_E38nbuI4OO*Eo}{fg6!^%SD|
zccBKw@`#>dbOAH9MB_e=I4G#mBaLW8W-Il*yA)sQWasP9t5%RO1U-1!N1LiJy3EYi
zTsS!iIM>AlLu+ohnJJNkwToB2Ipx552D1$J&yMIBiIq`faknCV`^^4Hm@NnS!$@le
ze1gFF94?E0u<0G6Q+f()U_`FOb4UjyllF_}fI-I>$H3##u&Ai2LWwEC%Sg2HEvetr
z<jCMlD#G0WZA}*BtlVL0tY-SqHJfx3z^#yOQjx*7_MY;NM4G(fyYwLmR-AhA{<~g<
z(x%mBUaipuTgYLaSsIztY9vIp1){s@n`LB<&`^qcPFq+aG#tFpmQ*=EpNooS&sDTQ
z^w7{nGZ8{G@#dxBnKPry+ams0t~bw9)s~<IzE?t!_w#A+c%0&11yQ(_;NX<$rsnpy
zMtwLa64L(SkoF^<*C%)HWRQAreMj$<uU>lXn^Yv}6uJI2!G`^kW;-Qc!UWyz*+5#y
zS+Sn0`#L_u<>qcd#T3_C(4|LGSpWy+6akyO3CC7q9`nkGIXxZSya00h^^AmC4~e?f
zf~NDD?S?j(ivp&oDATQU?2z<R+?rL<H^0yRfq|uJnzm**=7}~Mv);h-!%-{%FO`;*
zT&K|7;Kc~)#{jHqgyy)P%0i&Sx{%yn7@woT*t@TQ-hD4PYQ(QD(0g|nvgdnWemjyR
zI@fLHB{4kX(OeIE?nt+%2crv=Ka)@`?CE0hqY6kxcB<1;eV>McdAXtwwZi_rAdcWK
z@z@>e%)m$eaki+cH;EovmnG!qQx|%IBVj0cM3078{o*5kD48pNYZO2ryq-Lx2s_2V
z^Z<(iXIAY3n_nENJ(J<uh-lBNr`?%jo|=&Ki)OsoS@<9x@x)p+V&v~Hz4OoSuABvu
zv9H>?e#}{M3}nyPJDGlt;MO}?tp{VznkMaT35iEhml)#US%dNpb=V$bExg>^L#G<s
zekV`G`v+<r#pZnR`RV&B+<hOFi;pGw{mTu{`Go$QF;?Oy`Frz0x?7t2fD=nMo8f|+
z7TXyE;>)VuIwLMZOR2nHGs2|WT1l-1jL;c54ZnFd1qjj~`JipNmUqYZmofqzsK&d$
z3#2FUaHrj=`Nt(UCGPK$g*e@rHSTE*!~0F_S0xc{#}tQXU?%p4@t<s!#zfhBDxlcE
zs`X@!e|xT>jFC{AyI7A3b_#DHjVI^|w0B(4^2ZazcUI^S7T_3ezrAa@J?o8g%8h;D
zUaEGcpWuS_4-7ClrR9ps&<TsoU!%+yS|I%vtW`ABMmk*_jA$2=qU<kd{H1D9*q<}m
zW`Xaru5KDqy&4@IG;?6Ph@UL9rZj&a4N<RQr_fRyCH!d*Erdy-k?*1P%%f|!Z%d+m
z^Z*4hT#!3>O_#oohm%KuCt+YmR7xyI`|WjK=zU6;5dxtHU(~A8!qwEKv_G~&!B$kd
zjeH<%F`m%rB>&r-UjC5r(ToH=Z>dglZ8tdfj%vAF&uOBmrA2VoaYt})aDe`Ny~D>e
zlzNO(Ze~)$(zj5mc;qjwQ!X3d6u}zxrL^=mqu?kiA9{P9%dnJh0=bLd&?%s==w$PS
z=Oe#+mCJOhJj>P_L!#ONe!<|ZVw@2>-aO$C<$A)a;v=HFAL2$lG&m(ir>~+C&2Wx-
zoC3P!qxF>Xg(gI@`Ef2_B5K6tf!M}mWxmJI8&xASZSGJ2-SuPn?(-`r;qx^nWo{17
zxp!QPLRZRxwbbsNB^&WdW9~ek+;Cd&fz2Rlc=*yVZdJ!9Kerup#e(m5(i$dBsL~bd
zBI}^bWY5p)Y#Wm<VM}37dZtsMQw?1)yoacK$H-(<FB+ZhN3@;VopGs-%d?6m8%tL(
z%?Fij{fxwmTz8)9&rY8L1oBBWrh?keTHi11(=#}_!sNb4gQOwuyVU8Z-TCA@v4A()
zDK5@?qxDeb>#_8s3Y<%|!;>l6kv|hP(|mk<MB*pyzQ*7g0&qL`v%I~~x5vJXMABNU
z*uT>k;^nHZENa>QeG-sQenwn36~M?AQ&^*`f|wtZsQ7aNMKW4eWd_QidhUx7lv`6n
zbi1$Ks@qwx<#@ud2{{ErcKMV1Bb@|3X_lcB5A-MF;^QNQ1eeQ_uaucPLOnBMWL=)Q
zBXpnUIVm#hB^^x@U4U}IMW8L&?E@9LZLu7Vc89t#cR~UWDcp-aoF~3WB79Ed4&%Uf
zgK+()Zj|ZEtXtm9NO)<}kJowVGDXgw@uT6mmD{`PaVmEDkVNmo0mrX|i>>Jwi0i|h
z1caBr1TpdK%sK1)&=LkoGNlU@k6buTt=gDbMfaes6{)uJWho{;p|o|6juXk&Z<+(m
zG@SwcVN(=Tv($t^ZS?(~UJweTjrs-tSpc4i{Sv40uFi?Hn=E~cEH}V+u9Nh^#c7ZV
z9+>yV+2P*$Xtma=?M;>kUm{v^W7P@WBNGiKK)@CwZ*aaDo`vs1#Oad=2*<|$KoxB;
zGG(HPPjs{`+-1iBC8t4pF~+fB`RZI^7CJ<NlxQ2O(2_e$gfpx9OUmv_`YQqm2H0|6
zON|yPIMfSDHU(YxepT=j%9Z~9fQ2Vmhru|QF$vJ?zw*{pB6J4VE;Y&Zipe1_HPLEg
z96UMnDs)Y|2xt?eu(`I^^=y30k;UU>>Wof@t-CCy1J)3|XcRDe?ouik(D19SXC6Fg
z7)FojZ%L-CNZTI9&_{>6l0oQ>lqh@jP)9*NY!5d}*$J&(d=bv@*xQZPKcJzTEd|da
zufcO6_)a_1xS%q-&7oqdk=827ge^UT_QDb5g#yt8!zQ%S+L}Y|fvwImb>&}=f|lN-
zgXM=V2~nDVx*O@AX*od)Esd^8G7wT(+P2}!0PG~&owglrm*K(J8j|KM5TMm4<;bQp
z4<YMD;IPi@DaA4VMiZe=_t>a4&h~nFR1EZwIWnx4Skg)O79J(Eb03zUr@F_VRdUw!
z*7U}vSTNgEdAR`@Ql#JTjsncjFB-iq1dRq>pjP4v(%PINj)z_erwzr-#XFZztCXHf
z(n4jLY8f6mF!_DW0y(l2;5v$>(3;vqK7|EMMFcJ5<y9x}^8Wc@Q1DZMe1?r(+sJ;+
z)fRtKhWAfNL)RbR7T~utancH)<2mK^sl+{#gj0{k{nsS4E`{U=*C|!GSOSRWq4hL=
zSL|;W!#hGBmwBu%9Ga0n1=aC9aEDX>c@0?bag$JyD*pO&TkySoLX}{aiNBBKw-2gu
z@dc0+!Nq?6^_au451gIO{;Pf9c9Ijp(@r9OhmG=o2gD}dqx%d-xz2PA?_Pf%xmExf
zRMN=&6=lNhVh2@91PrOx0smA^r{Gh|L-}9-94YA-IWP!qa_GLrwYy^DmPLPNBw$;?
zM(v&lpKd}v0ea!7KbHOfX@Q=Okyr!@xys-BU)OZD3djQm2_=#@Eo+9O&-?$L=WrYW
z<0dZ^GCqxL=&lA;rdopc>i?!miwB=HkxDf;lDxP1|3qNotnf7;Fl{dq%=y2O8%4U-
zXMi!|e}|QhE+fLVJh7px0j8<=EB8cbQJ<9pRq!+Z0CnE?m49M^2WgubS+Ft@M6clU
zU!UN5k<EG=EPg&n+cOhxvh07;5A$k18h2sBR!JpllmZjK=7f+0CUYRa=jBYNg9$C?
zSf#r`qx$n-+xVf>b&&D<%U{m~mNAvR*1TL@T^&a+T|0!Gf)A1Eo!$={t+kfCkSF`R
z6_vdkUO|I*68qj0Fk;h+;)$o528je;!2$SXOF)Gr-}?g2aM&CL&ffc!FX@*sVN1;<
z5h!I+`5S)f`*PNQ2Q2b6g<(1&3^PNfKOP1nqp{N=gWPN6?beHDeb*4oL0nNS8I;p{
z<Mw75WwCK-`8@N3YM^~1q4BOwDI-gEwm<H;G48|2ohJ*QY-+YIbKy#5bN|F;L|ZXF
z{<omL8-_HgL#e;BDv4tARx>FdV8#9RB}L>&$g9v{#d0l8T0w<@7RB5=k-16U@$bai
z5?{nS_8dgI5qg)dY>>Pu+W#)Sy}%2YzdX|`_eulhUz)VdcC1)wD-nd$u%3%<?Z~Y;
z{#EFb?c=Uofnwjjh}N<Z-S$6!<n3(+!P%Pp)o)ujyRjS?DiJLHIJ^NxS@7o&Y&jDe
zuSd38iqdcz_^-$Vo{0c31>9eQD%q+3-N;S;tB5|UR#<90_pc-xTg3{KIlcy&U@C`y
zs&643X4RGl9jlsn&b+w)&m2=mO{XpQ?XjvRY*#&@%ws3_#U6P7Pj!$HK#zjp>yY_m
zM$Y~<DB0|4<iU=*duLYvPle?}>Umz0J{&fW3n$RZjOc3o&(P~(6m3|L2VRmg#AfUM
znQq-fm1kyYI#x3iUy!Y-VJS^^y`ci!{|DJXCcjm1HVY_CVCpMLUbp9nMl8x%pyfw~
zc__c{I&n;~%p;b04qoOK`qp3&`-y#A?y8>_z0sgONL7hj#o<~nI_-JYiOfG~R#UwJ
z`Pg`5Lqkocs8A>rXTBCVva<i+%__%vI@=w_rRb3&MGBPElvFlv+{DK98%WukA}U0W
zSHS41w<GH04A5d<r<6@nMETzVk4LM-eY;C@q^!(!LYV6F_ON!zA8bERT)K^3JC~tU
z!0DV0k{2#w<<^}z^+z**^;!H-%ltVK0Wb1UXDuRW!D2S<%Jlv|u}_V-hmf&zBY!U1
zMxM=cu<%wW6bi-JDW0kdRp1}z$v;%V*=TjrEJotG_Uzfi<QHG$u}2@}!Lbi<{{y2K
zJ!UkMUwTnnC%36uK<TtPGkwb<o`3#xQBmi4{lpA9^7gIg)t9I9&-yHFe`#^D=kMUH
z*JtqCnnEnuOL_FpTY2f<R4vcl4jTpe1r*qY43l|n%jc&tH!$(-+1T|uRW}Pt3W_Ky
zae43Cp&ovbpPrl)X5BG_=RRM6OZ+~O`RCE;H9@@oo3|Nu(`#(bvLDktk_v@Fp*Ri2
z(~$<rtS-X1zt!n%bQqVSX3d&4j2SbAjQtthH2h}XnEE>Jz4I=&-F6$>leRN*<Ve=8
zUZZVWWkqT*;K<$0>}Q_hmsKfVL4}jFWnb~wQ%~^apG&aHs$SII)@=(JJ#`wtY|Ft}
z?-KrR+`HV~CQhr^vvw_G{JoFy%lbVSv@RzPp>?9Lm}+W5L7qoiUJfS$bjH%{N-btu
z*A>2QEQ5CJa`CA54h+!w3j<x*{uKnXUSE1lz2EwSEkQa%HP#`$ce&30_*GntzC!xT
z&uws)u;k@Oc=40($SM`1{^#`HNA{rUOc0R}Mu^FR$uIw^%DyVk=`WD|*W>Gx*AZz7
zBr+jb+#_AZysLCwWqDR1?{x<)t2P~aZ(i#v=kp;R2LJn{uflrlpJx@0<2zS%&w)ar
zP@L<Qr^5*<X>~eVBaF)mB4b|$FTVI9ojZ5trI%i!PoF+oSDndhreD8)OnrSS?b@|t
z(&QJ&&&#jEx}a6lG)!#6kRC3UXY3(Y_TboxNM5jj+HpqKrR`>Ck+*V6+w>1j+SQ|5
zk9bj;*P#EPzI2QWL+7%Rxn(0ief<L))}<hgeez6hr;)(G2!h4m`?hap!`5W1ngFWG
z8dBQ*n6r<hwd>fGo~v2FigJr_XyU_4e$D~1Y;N4LW){m__F@j?i}h|PuykbtCogpu
z8`o`Ocdosxf_Ijbkds{^j_D?6?{>B&Ws;MdPkw<-J636DBlf%ka`H-W>B_qrWml!M
zAeX#ix7OX2xobNcH|`+EVbJ`({6&f_Bb6=dHnL-P1|Ge4z1-iLyO-4qK4<cr-&v4m
z73*|m6}+w}lcWvn*t9c4-q_OfdT_Z-SS%4DA4}M?X%m}vXKLb7S4kKZrS1{eyMf(%
zvr9!uxrIpz4{N4K)7tfH-g7|P-<M>M*4XfS={zn9vks6~WY>J!G@&K_bK3IB&djAm
zyRqJMyPOVkGqcIJJIaLV!Rhc?*j$A<WaSo6>?zOFl7c+4tyB=h6$G@-McU4-Y+S#E
zJ=xy-=r7XT#Ru56S>(yKWbHn9*X!hg#eFT-8ViN{lG(6!GwJ#6YK5mlp-`MiPK!8C
zva&xC>yy93xYliV4umC;`SRmWKN1)a$mr3dF&d5N#ASat?IR*1A{qPeSRq^jdF;{0
zxOw=^{4(bkh7KJncC2(9QG*(4CJ^7YH4`^%BeM`9oqL(HtcZ}%Kvpi_MbeI3TC}mS
z_3zEp4(&syXanY?A9&!|w}~I|Cc_L181vc}gf)+0@${z&u^LIgp{3X#l9qkIt+#(h
zR$2ii;ugg>?9CHX9-@iaiO<j6YgN**Z}DuN8uu1E#I;x~!32q?z!ICtoiD#k-;6~}
zm@paF4X^V=@8%fA16sV|51x7aJvzKRl^Z%Xz*v&W4{tujlwWsXlETCsM58;OWa_~4
z*uC_7CO-2q)|hZ?#l_@Za66-RzRr8GSMuy556~e(JT}F9dF`fq+0v#9<0d>#15x=q
z#hYMs7qRxMM;ZHT5fLGJ9Q*cD>~IrVry=)Fm_ql203j$G{QLS-yz<LGWO+owZ8qXA
z9>Nn3+)I<N6#nnA32ZH{OC#F~CJ%j<*ePSUs6!;F3qN7x<WDJaI5m+~)M5zlPQ0BK
zdK01HP8NMLg`o>g6c-dw=nkT3?~%Ota8JTq@>wrr{<Ht%?L}M3ck4uB!9imGI~ae@
z4a5d{&^roQ{OPNVpZ=HjjyUy3B3fO`<cYUZKhW=G*Gu7HXWiG&bN7_x-1x{i?itic
zh+>^oH>9mu$RqzZp4JmzVa(+h5r~Vu|9-^{_y5REW1ryO!7W(u>}XyW=_ss}j<`S@
zo4<INXSO8p$b|8<4-x5hJ6ZbJNWKW{$a61^q=tBReb0gvIiCIN`RMs^%vfKHMcin&
zC7h;19^$oO9ko2#`Ta{wnm&g;MJDv(SVak)xOenehPH_y=g&8|@55DC!@@D7r;~4U
zVXy=;^oduwxntdfd?FPJh2lhVM#OnKk_hAFmZ-C*(3CBIo88X8|Ncv_-n}&8<aWBW
zhIfIeWQ&D3(1ej#78n%B#TQ@9Uw{2owMrJ*;tGhOX@iy|EZ)rC1H}|=Th5Z2jk)E~
zG2E&9lNCGn;Bo9@{x%CC30<ip9}K6PeaU-BE3#3uWq(Foe<A7VPA(WUf~A8x5N33X
zbXrN<n@?23tC{@fD~!Il21{3b#}mJ9*DNgl_tBZ2#24cpWxBBu{hyr5M^D~H;Qrlg
z73Z*O!zMkXr|ji`wWQ2~U?88&z1ujDZ<klghR>hjfloHk>Y4|6|KnE~P!pED_z2%+
zIta8Ek+F6yzZXZ+?}odWbah7(ySCu3*$Y{<K0~uwIJW-9^yF-cA}^w@Q3_N~nT5wu
zP{_LFOVGzO;gKnCGU2vvY+JFCsk8pktTP!uzsk!Wf5JD-hw|z>?=t54UaX(?EKhwh
zTZpMxp1ywsiMC=wnzZNPDfiH!MFh^he{#$J-lU{vJDzxR29s{?O331uxaq4ESUs})
ztc3J^c{J}noGGurz>w%1e)x7OPydl7F5biTPo8G-*YjD{`F7s>@Ez{D>^v5|{wNcE
z_#3-Od-BrHxb>OOh-!BQlRuiygsbY4`s>?_n*1xyQp=KfNrV75W8L@*BgcJ6M7OT=
zy7YW;c()KVUdv>idhs+%$mZ!e+t^<qM_5|f_3Lz22L}*+L36Uc8pkA&j+I@!>A3UK
zcBSd~i(-!nw|FLZrlsL38+hQsS+bM8$p^4|?M1!^0ewj}f4}z(kIyTi`z_;`_TCf*
z#b@y2TTk)&CJzC*|1#^lZ<!fz36H(@9<PoWM%}+>Fl)|tWQgQi91gZ^TF-$z7dKCw
z%GAe)5mLB^@vqHczr&-fPYQ+NthPLz@IJP)t=rD&bdtO$nYdbU6-{<RNXd^swW4d0
zx;Ld7l`KW2i5B%6)1hc7+jBD5{m<Xj42)sufPvI+9>GsrlPTP@g$*GF8uV!46=pgE
zX0yfH?Gh48mzHg5TI?XAK_lA524d2=DJpPt-euSF=>M*#d15oJ9&tYn%w~!<B$rAO
zpEPlkx9cCK>`LVRTSqhI!lu;fcsY|sbRxiH^gayYd`6Q6v%i0fxTmH7izbxaLfm~d
z<#%eeYrz!*T4VNvaqX=GF_}HgUbhd65Oc-4^LX`<r?~g}fpl$^K;i{gb4}4oR&800
zLmY1BPu~(A6-~dZI}7pP!GH9=Q;5}=3$J3-qfgMKSt8vAk6>`50oRUoVr>auFJ8lU
zmg`vh<ZzLf_2_rUy^O!}V!ruwIh)g*#J6ro?^Y$4gQIELE?$UQ9Z5fYM4`t>muqgM
zafplPUV~}bycwx;){y6NQJiO|(@i%sdgK)}Y21zxQ(mNBei~WxSBi7yGHu~z{*Jqq
zg^vuOUP66_j2grJSGQx@%w_C4P{^9E-y^6)Z~kxGC|cF7MVAq;GQLA7xobXRgSa@e
zXk3`>h3sCtnD?K0j=T=Tcwo%KGz&3_#)hY~(IH;O+D+-u`%1#U|4Cd+4mxYP5OK-$
z>(GI#>lHC|{st`_e?Kx<OUHE&l%`|GGSc#_+I_PaO%;TT-fY6;ySKjQK>R4q+`+83
zHgnlcBY5J*Zp77X!QD?zrc2uv<dqcQ3~0#pcTZu}<a;=;RyeiVbfshOL{{!dV_&h?
zFD@ju0mH_>!xiT>r0K;saMSs1sIzq;J8f<?a6zF^oYiQx|Is*%yPU34ONzUiZO*PZ
zPg-0iQNb6N6qhZN`E}E<8?jg{;1_$z;c`$Tx@L7cS^K|6i>Ab!XJF<0IqX}qkK+1Q
z6B`yoV(oSmFI>zwp-zI0VO-ccuF9jhT@J4tDzor;gd{Q(XbLA(t`U{7JvfSvqH<el
zF;x<99;`*VWYmeKR-**(-Xd8JU65FrTPCq27Sq8q_h`O3&JuD%gGt@+FJo`HjRcDe
zo6SmYdI6CZJ-R|C)`U1hB0|cJU!!qz`dn9+dzNfv^bH;Pa!E0cgiGn)(2U1fF*cRV
z>&PGz;lUnFaJmSmO(R&L`H?yBNNZoWmc$2`wC+=nK9c&in$V;uQ{=x(yC3>O1I|zv
z4u?pw(L+(D6$<kB<%v64Q9lTqwC1JlAiKjLA!ps<e%2%|AVHi-c6E6$1SfD&13Pmm
zlutBzy90OkRzw?QxCtRa&9!J+vp$I{i-fqgQJmqSVa>+03=?UVU1NG94STo1{Z%%_
zB1yrf5Nf1-$@sfxutDFJf%n|RrFBer98P>5-#U-XhhX}&@62U2r}NG3y>zv&=PN@4
zZtXl+yr_BDZFX-uWDGEQsph!SbQF6nzH)S-5NckxrRoDpY5q~pA~GZ5X^_y|>nJbo
zgDds|K6~o|EnnT9B0Ppd=Dsn3HQNg)F%`08@ivV8FQ>%rJ%&-7D>l-k-6xL_-OcNn
z2?=y!@*L{y3WY*(0_x`fxZCcCe4hM+WNHFQ6ttCnp3cTl1+pSHn@lus){NC_R?~k#
z|FS@rY5UR%4h*Jw%kpYs-MV$OZr!^2d=#6y;}U67uQk(W&R|)vg_iep(hPt-+P0<h
z_iymVEKvc)4WfNa)%hr@TWngDUa^Am_*-y9W#*u|2&~c+BJOuUd3T=9q%ReU@&qMf
zcL)1(MMW)dsX<nf226TiU{5#2MMlnRbuqVG-w}t+r8zx^2*FUNWfUcgrGRjjiEG&<
z=M1Vv=T2ABa@<@#c{7tggeYlz%{9bm!tda!-RX7-Q6dC_NvB!M^jbQ!Djyjsi$XU}
zk5A{hDbhK~@i@?zdS%-yWu3doNkDigmyLXchI*?wtGFJs16SQn1j*lWU%jK;LMMcP
zsVwuH+CPe1l*>Ff1wvsIxaB#;b~YytPcbE8O{C77k8UASJszjlT`L{^^P?|d*aLmo
z_3`U0Sp7XqvzpPRhM__?x6HD@j@@Y6r3W8=@(t6X*3i6e1N!uj69R5aSvt!1aFwM)
zp2+Darq~+nb-1^P{L>jR8>=r}JVFKOtR;R81@W=F@R$wcF8Gm0MvNr$qTvi5*pKLd
zBH9Ec^Nh_=X2p`voI?sgN=3R$sV0R&af<5aGojn=Xo6&(ZJ`Q`W)p)34Px%xxvczm
zMOi3<y?c^LOH0*+r*ytuzH}Mu*RN+lzy1}hQ5DT{LM#|#Xd4$zP~kpI#@gK0u9mhc
zs#7~^HHji)e-T;dcOqV2o!a(jmI0&1c+`F+(qY^j7Ea98WU`lUEvqo|GgG{ko>T`B
zHL-+-^Xs0>GIyyx$$QzfUv}k*bDN`R9+yLLP%s@YzeeQEU<Ql)=|A8~T15sP6a`(g
zN(iyiwo@0bXnBB7XMBx2uO;{QNbstKBT2XRpG;f`wIgZ~_uJ2G$SBcX9%IRVHf&f*
zX8m{qf&(?nk;mZ@g3m-?pf@csEn_LxTQD^1%e7j1ui>gIu47Q!`a&e>F`IK)k-UrD
zB|2?%p7X#qX702Q;t7y)h<ZU$)SdeioAR|@O-y;aSeLY#%!YA7KnGE~RTOLXZsy;;
z4$UcC=gMIIkGlxa#Z%8H8x}V5Lz;2<;5!-r#2q+)e_!+4O4ovD>ME#fw>g1<?Hf~j
z?M&YKV=vCOSJF*fha7d;Ai$fBjRm^WbnGloM?e6fwG!BsT}X14UAxz&;yo<d;jE%j
zAo4A!CM_biv1jXIvfQO3?{>~-=;+58zag8nU0d0aTAv3V{2zDTJcL05FD9yBKPRE5
zU!hPaj>+=$pK1edi}&aSkI(XSwudT^H|T;3E}-WnJ+;7?PdxF2<`{nKZMSK|_8krf
zix)3u(({wJ>E@eg+O+8*^3!X|Y1uZG$Y>qA>-OfnD1)X8Y|)Kr5mSe7cM3haG}J5_
zT10VaQLx)_l+NoB8&#Y34YT>_lQ#)AJj5NHYhZV~Ds_K($P?*uSu0nXdZHS2V(f*_
z^Ttc#2ramtM8hhkeX<Tu{CQeB>@}K_P^TH8pHAc<a|SIPLP&fwm%XMy&AAy-iC^>J
zUA*_e2i!fPkn8&Oq-6apeq3+k-uFJIvDikQ+kYAow>)nB*0k%;i62&e&#pmN(=N0s
zYo`aRI}f{jru=hF9M9%<kc)|{2liv$k{5XV&Ij3XQ&)=D{>1w~<Z=Jgy=hZZ<fYS0
z<Ib%~`DGE~Cw<D3H(g1KYyXelACF-A*c&-8cnB9-()i-rc^nw}GT&WnhCmbPzfI@=
z#JLA|sL9WteZt0?_2@C6hd8^1>xT5@*LCmm$ZZcZd~i$B7Jtjkg>ELic`*qQ!9)&u
zh>N~?nCBiJMN0qf6wRB>>=X~J#@<4LcpY}wWu6p}S74)2uiJR>?&aM6@|#Sr(~_}6
z&KGjTXHAmR@rdW+vH{fnVKIBs4D{;XMZBa!fEsEtY-nHp*dWqz`&e!s)Pj`7-|$Ik
zI^sf1<h1BP^F5F7!Q^o`FKNb#Uw>rbjsw(<s94HKYoH}Mk*h}!=lW;gVyqC4eOkxx
z{q&h^a71!N?J#QZj-y$v1HAU?b0js2V#|WR*|AFqu`XU~p`AkP&WiULmjk;Ht~O8(
z;|hi1tj6&8<ByNi>2!Ln+sCgY^!=6}*=^@rr!8}rE|-;`&TBrC0!rE#d={yK{CpZW
zZhYEZ@Ydvg)Qhb_fZ23pRu6HV()!e)eFt)LbD1@37K;`wV(Yf8EL*;e&p-cMvr64|
z%WVu9Jh;q9>JX}v$RJp`C6$Jk-^f+XqqQy<oj!n=oQ>=_5Y9C>4WeO)K@(^Cf;5({
zPonmPS8_poIF`t83fwkUE?dPvBr>>bV~W#PkrmvQUKh61{7rCM*|Kmgo>mue#rd)2
z>5<_QLSkrh{(0DvS2BOqDt4xNaM+7+21U~2@<Aj91<<}#eX_S~X6>qfNq2?Ow{JJ1
ziybt&<T6@`iZwj3J-uUtST*lAmTlcYQc5B5eQ)Q6u1yHpx0fwj3TW5=Vw%(l(9W&-
zO6e__x2|TvpR2k4$;Y@jA&d$&uQ#L=Iq7>yDh}n+9^I*7kljE|(nLP*kGzOJJ)05`
z-<)nO<5{zMG3(Z>VSP>zmyCLf3D<WaSi3+Akzr9}{JoeBTZ(9MK}VV;HsX?=t<Y~=
z%Ad<uvvJdQN@6>4-=NEA7U^N<_U*(p>qVR~ncsh3ianqPH;ft2LzmPAIg&E*e7ZIY
z<)7sX*|=^!>q}}f<bTid;AO2vetR&-HsO-?(d=C`hlQ(lkWy%;@91Y4e@%PLQd6XE
zVry;yJ$rYeR!9(y8i{;L`Hffi*Q8g6cEpD0%ik{{rQG2mESxi+RO@+6d}shQJ>Gc@
zed$=V^e;B8-@t~F7?F-=yy+182FBE-g%IT{|NNVco3|3xbs&9X?YP6*(X($mf=j!0
zwW)$cKGkj6k#^={mi+cNtG4YD2RNUnCOylQ^}+~j(12RSIc#3FmQDNf3Fv<t!#l^Z
zrzC=_dUPVrk;$U%d35PDfCeGn_+JIv{$*WGB7LsxLJgDZ!BZ&CZPSqNOLx@!R9sZ`
z{)g(e(^Y+-qezDCu>0tWi;JBum&;Ju0ww?UeI(Mt;*0SleV&|7ht@^sa=Xfw+r>80
zsa0B^Qd3gs*T4U1cRznvxQ6a+8xtNJcxaJR3EzakTQ+TG&b+x=RPmtTAX>I;N$=jh
zNk~Y*<0mo?Dadq&(k?L}K%M@P+Dju-OUHbl-qCx5G5RL1_2|441?!#S9!a~Q>?SR>
z_PNWfPr9;b(QfC#lLc#0qvblg9i7EY`VZq6@^KPRz5hAAYq-Qc*K0z?W%pVXeK9{Z
z2kz2@`GP5W9PTooDQW%EB6LgNnX-wDy=gI6Gx*@KyO_AJHQ&y8l6KLSD$`bWTxYrU
z$+yh6y{<Icxzp~_{ITT4_;2rB@6jx2+I47o<V}}eZYT28qxYqe3X!!n$7!+NsIPQy
z{LX1I8MGJIKmERY=ds$+i#+u<D7<MZ3%FStXGuo&F6*xIi@oH#*PegBX4ses?6~nm
zmfzV?q|trQy>mFKPKOqxQfzN?>G5wL_~(y*5h1T#Zg02C5tUwtdGGnW>!g)RI>{US
z+Zr;>I`3r1{?BUJbvj+BC{GH-xs6ZU9L<cYRSBX)?O&aMs%oF3@saZFf2e(oM{*#4
zhw;imGSBvIJAW}LYlLPknrokf)>k0}s#UC_R(CtAco3?n;B;<R1=k=cVC25COV8sh
z-TI(&dk&c%ZGvJg>b6lc{#es@l9OLZiM#BEYv<I<3ci}O%HwPE_*D`vTE$xSu!?i&
z-DGTA!iTTC&Np9g;QUv{(Im>^9h(nFl#i8{d9=#g8}QTPk<TAhuGeeH@l;5cN9VmJ
z|9vWmAbH#>uG#;bZrM%hIr!db=QJ4)T3`9TdZZti^B{FjKyfn5=6}MQ&(G!;XA{02
z*s|JtS6w=M_r;{I`o2~5oXG8sCiz@>s$8ejgk<Flb{*uhR<%G=C=`mbj|!vheBYIQ
zQKZATv@lhLCa5YjO5JDm99Kspl1{3s_Z(j}rSpx>6W=i%2QkW6OhdER@^GZ*INW-j
zknM5=oINv{Bu69z$GyQF1KJR*b02qbdR?XXP)EA{c&_PK&v``GBSnhS&c39*=sOMM
zxgo>3BsN$RNJo2~qkT3Gwf*rvX9|TvaZ>8Q|Jboo>$VdGq@l8Cs;b-0D~`Oeo}`#^
zI-Cb}+qs<H=I+_nZFe@Km)%(MzG~K*6PqVmSC$Y3R=b;ku;9|U_*^Q}tIjE$HaiZ{
zrU?!<X`>O{ZuN{R6pAxf+?=d#yF-bmQ`T*FETiqtl^~ht0xpmDo(Tng!n<8GpC3~I
zK><1?1`eN7n1TW^>%84_O4}<=ZDps|!l`*qLOGyNoNXMH&y&k_2tS&$Fi7UPg6sxz
zpTwHFN*^CZl>*A+Qhr0{GQPnPm7fd$!3!rxIO)(itTF~GX0sWCiZ!7)?dz>GAi}uM
z=cy`(@v;k%4&x5h9ivbv6vs+_@>aI(%f)8%20E1^AwwFUPwVI)PG;~2S;o7QmN9$1
zlRkYfAud#>ti+1byQ%pf3;bj5w)0VWw4FksP$-U}C5@Zizs}(OwFcUCOvGq&;Bh&L
zt=*osGNgpZh0|>)i~KJ~k{bL>L~>-VORU!)qCY`~mT=0D7*$j}u7Z7ROm@;$j;z!h
z%0iTso(#7~&ZgjhJ~{A1ubt07WQ9U;E^tJar$bSBw4FksP$-V6EsaN%#fhzYanH>c
zmoLrP$d|J=QLpc{bPG-8*B|ENY&U?Ojm#|n;X9V^&Lz11d0g46D>Y3S{5ktiQZZ4S
znodybZgj0#!lI??aYWRo=arYz;J|i%UAqfM_I^@wiU@1ck*j*1ht6QeV$osH%h+Ef
ztoZeN7H`>4bgOP$anbpNSc+Ny+dTf>oPs5`377ThN!`#Kew)3P`u+RR(CkJTs}!fK
zv2i+d+Z`$XK2OSFT%k}Xj!`C6<724T8L;MOusLNf=He1OmPqPHMq}KwkVW%r5L!En
zua~BCYd15Q^FH9mH6?WE(w6OuXY)Z|4esk~WZ{Cv6x2JP3+tL$Id2-v4T*H`7|Yf_
ze`8)~1MY2^#m3)0XIJfu>6PH-$8WzwACkcJp)wx6998Wi>8JPjcCDTB+SO<0!dc7=
zkLDWJKYYC?iPjgM$AM+@c-<7kqnF08e|HMec*C41?WEA2Zy5rxRM{Nvsqhy-`YP0r
zZ~rPs;#FGc`&reH2mfskC%Sz5%YCZb_Ncn;4(0PyzKs%23WY*(P)k{-qo}x;!lFX#
zcKbmB${?rE7Uk!G^woSlJxPcWJ0+HujJoe?F1vL&?@#`SSM5>sy6INhCxqfMT+Y=A
zdg{gr!IiX(MF(=Rc_MMuh^F<Ro4GO}4g20LtZCkp%lozF0&^Pg?A}C5vuIH%)Me;^
zE9l%PfI3AlF?;hWA+Ri%b;UwIrSs>KR9wybaZ#5Bn6|EE-oM-EUOx+$$4G3`j$D*b
zpZ(@&EaEfdzG2usHg=|@duL)$oHNh!k^X%w--lavr28CCTHMv_TPB_k7bNqDl%`Wn
z$e>UhS*y$&{anQvJBQHg$S)|MsHljrurNYGLaJ=<u5~%6)96yh-rJ+%?xsZA)HU$;
z{#cqdt|w<eWc%91e7<lmu`%J~@5?7B&QMms8+9JsrbvjWO_Q2tA&PVu0s_T4?_|y%
zorN%M{+<A0<05dUln7xIj5e#6J%?;roTskhvzcqrnQPJ{K7#megSoWudzQ|4i=W&9
zbh+vVnrX+i5F{GX4H`61_gV3u3Aj^Dg>E~aXgU@jDBoy1)opjGDqww?v#4+0JH;_j
zI_I6NxDJ|v<Yc_;M(^Bo+TW+>oWAq=jOcFF&^hG=ywzr-xVV_8s3-yg0;+qf<phUi
z)86P@+Kf=>geKaz_IH-1nrRW3$?t!zro}ah{4sY6!IzI<Oz#+ec<VV9y7l<#5}hW@
zJ>JQ;UA{@(b!9d#y(f?KVka#qCTUw1rl95o7(7~ch1(KMoW+H+{Z)**v>~RVJPzoC
z$XYZPSEDO<Xh<weW<1YJU;Rz1I@c2IEJhy{L5RB5igO6#@gBf8+OE7VQo`w^3nznj
zv`k^vehw77h^Q4$xJ7ru<`yfPjQDIHK_eYwvr_r}+xZmtyPkoyjmI_e^zf(C<I3K_
z=Wl(;f|5F1cf*bJY7wGMD1CA-Q;%c&3O@Z}6-}-iMwi5(lX{(sgT<3~!gRwSgp=>(
zv$12#d|rPq36sNt!|kC~`xb2bW&vUSMsi>66rO%#7N0j8MboBXtY7jK?{CwSw0aNr
z&P6z#f!GQQvAVnw>+K~a*c{#<Yi>sgMI{d0`al9r`}q2c57`yvWb-B`y&vd64cqS&
z7F%%};<)Ux4!rx-+q{<A9P9Qy)V%6WF0&Ny>jy8eq04y`?6lIRWkYIM(wX()4|KWj
ze$ES2_gZmoSe?qm)A2+YuQ1w9S)PvR<{wS>jZWmZrD!kH-g=EM7Hsy4rLcPR8hkgS
z2er`zkK5zx8uFAysjlu^zAhwu+lq?Tke|lNP5X&&+ltr#ec3)olHRfhx{CSNs@h+h
zr`J<)o2t?%ThdkS?^{>Z@nl0=&ada)%~RT4QQl>y(~ENz>|@E#UsCY@?41RC)z$z1
zpSNSZv7loN7$MzKDj_J?0V;?sHlPR!0=9xhs2HH2Vt^nZA}Nh@Z!lso80)-!@8^Hc
z=iVhD`t|4c`~81+d`z}`Kc5rt6Ytk~XZzb3SX-k$U%CB18q<a7QAXF2{))PB@%!h`
z^T`tE^JG72gd5a3@?L~Sm1$G4-Sg>o9&Oc8YgCny&z{5KRKWxt9`$c2(kSg?rE#6N
zCaTK6ej&LmLh5MH{{f!tP+S)0NxWx?jUwT`XQ)-TA>qc_jC?Yhg2-yrxos%XDW@qi
zgwo@hL1<0k1PgF>|AP-;sV>%~*G#V)?!p&bO$F#t?QY?*7G|P~_fw>)N1ra8sT*pb
z<A53@Gzf>hD|zTPh3Zy?pe{FaUqm9Oa~)`}=s;}4YSaq7nMY%rk#?>KU$YK0N{lDM
z<mBoBeTmU2ffZL(s>(mNr~d6~yUI3Cmy$A-9?SuQ@hZvFUzY|*qfxJ*w?VJ>M;87A
zKVK;k(rOC$arRX1Us=Sv<KLuTEh`6?%x2gducNK=0Z+CK7j-IVJud9x_Gz-1#3U;?
z#0}IXrpauvV$>?Ct-nq%QMc1Zky}A$G+{AGX70s#Z~^bXx{YBUzDrbW7<wt`65LkS
z;TG?iO6w>OTC}2XKAa944v%QpY{9J8sKHfbgnC5=;d4203EpiI2fRtVFLNP^o?<ZQ
zOPPz)<whsyQ5J_r5jSS9_)I_#y91ZTh}B|L6AsI@s))WT23-IUiT2&1euLH@saqr0
zvuIOvT4xmfF==s&G1xqk4rZjqhfk{~I3@z4Nzg*{-RaX|F`LlJeoNaCK_i#LjziR8
zlw%Tg%lwU!4pLJsnVi;O!l2cZLKYbd$1C0`a(U6Ksc8&qUp=-0PVP@9toKAl-`-uc
z?^9#-lmHevW+^FfC|Y$c6pw;l?;ndKb&1MP)+HqpCb?$hn(%qBiTAu3ooLIf&XwGk
zZc&e3E55V&&^Nr2XW#6I*&L)^aP(q6+>T-~_Zt7W)VcBra4u*j`d;Yv_dUq0_s?I|
zmi<RcNcDJ{Rlf*e7W3>@ti&~HNp%T$H6b*Qs#)=j5bLRaLP#Lylb*QR^-I<I=4!QR
zAfE9Ya^kT={lu2Esukpa(c{Gv8b^G%KQ)lYXP`#YR@5kITkI{5PeZN7EyYjCXR=;B
zjT4)QOIy_%sVe_lk|*7`apT5IUM`tedEg}fF8L{uPDzHZekKlXpPGA1Q7;?n4PWa&
zd`l!!g#`s9BqaRD5j-h1ow{*Rm{nuRe-qIK!%Im?;l=UedF$=B$j;8;iYu-Vpy1NU
ziX|{rQg`s#Tc6Tt_yorEZ%KG~1dZ!f!?1oc#)f^UbLvy3ez%U_e*By{tDW4?yAh`s
z&*ahPU*Y39pR-$Ahc?aQ3DvtfyJIDjUKqnWpUvURtw%{{-i8FvLB>sfhhGn*kaT!E
z&iKBxkM?nN(M%p2`wFw>%nsC1pI~j7j6>_Qv1`$%Oqw{64`$Bi_jET6TQ?(4XJg~H
z@A1yE{cKqL39n6jpIzQay0wYtxcGd`%pElD-IW-#hOA9r@c45JseSD=)VAUe%I7-G
zCyz~HbH-toe=vq|AFab4nn?R5u@oo&#?03~!F+XJY6|e<-1RNDO<Y0u8?Pkq*B_Yn
z-aLNavzX~`zQLz^+$2QgG5@Uzj2{0nCwwt<XqSN1p3A1C3s`pcG<(0A%8MWTf}vh>
zTE#~T;Nd0jz-pcwIfnN?n#Ip4kl4N@b&K}!&WE#E`P*tfnEnx#4mSuUruR!}1fVHC
zwv`W_eVU0gKWEvRU4%66Ov4x*%inm6_kK&|?7@v3$_u4+*JgyNb|%um*T>oAvl#v4
z<4jnQL~hyve*8X#mIK<5z4TK)TfCdP-P#i_o=;vdjS*iS<GOyW2y*1Gb@uB#IN@!+
z{_cBDL^h>uqncQfmN5F`MeJGg9q;_Gfrd51`1YeOaMo{6qZkW$TbJ<KsHa39zhT=+
z8_nBZL1d7g;-nQ!nfM|vy*HDew;V-ZzYR^|f-6Y(s>+`~<fYA%yhsEE1zqx4nKWCT
zuNVALkK2;pFZ>LYJ?qyHl%3(Ip#;QQvxE><JD!?xkywn{QgEzMO6n-r&V}0cDdp-7
zyd$M#+A7IYRr%imQ2(-+sIug#s$7P+QBZx>tWVgmc>@lIgD<}Pf>o<lT~@#^Pj();
zj@rc6s#kj0daE_%$#3WI$iQd{j~!?2f~D+??ZmwgUqi;qPZ>S+E82{h%;&E?P0qqs
z8TZRM0ZLBt*%x1u(&AnLcBXN!vYnS+d!3Z<whZagn%d#D7;yJ!`X+>Odc`cpzU!~!
z(^sEV>v&;Fs;WZtOVKnGpW4RwPqt8f$Qyk3@km0p&ftOhTX9J_$f;fY`u$d}==LD*
zym%{V^WR|d@A=fJUz20o*Rpy~x?0b+??2^4NECI$q`g%Mvvzaf@bCP%H=j<UKjfwU
z!F=`HRJNoS<5V0R%FMzUSbj=w656xJu!-O8?i^OHT25xdKt{jt3^jhA$-R%gkFUq0
zeDqR(b_u{S|3DUs$$%;UI9pC>>HP4kOzaiGtQTHnX_gy~uIGgbuX4K6aAtn|4lU9I
zAbEQ^ZnFRxJD2jy78^I-{uo!*Hu-Y{3Q(-KCG*C}(QMXU$&_htGdLlg7oL5Qb;sOv
zz5NNUtsO$GwzqQckZy!}CF4Og{4Ypa$G8{X$3F0BzJLA}lmlzoxo4XibtkttpQG8i
z0l}mITbVoXWoMwr%bwYjnDpaL?w<5MAB?z`bsxUMjGbPLMhEBCEN1KJn%qBhIE|EZ
zByUO~qgXJ@?9GgM;&p62p5?30C)42Y=Zv1Wk^*}nOD8-h`hGR@zxs>;)lPBO)NeRf
z>`|qZRpp;y2LGo&Qd$%gOViegM1~mspy+R^yQ)-`|0T)OUk;L4CJpi1Je2^bDx0Uv
zEBOWacw8P#29xTAV=FGcyal7zpfkxxzGZ;bCor%skTyXhZn(Za&oF=NO;{-Ao1gZx
zrdC(_xii?2XCU60#h#y6QrQ1yhL4!QP){MLY3DH3t%EbgO_o7J$E#aY?Wa6Cc4|xQ
z;6lDRc&N0Fo%#MccCT1Z(T(*8*7&88=4vgN`OXXExe7Uu4MCA{<mEZUcP<LOR<6GH
zNrv3qM)a^L?dGm#L%NNKYddpevk%#}Z$H|$P8J=oQ0vi~iPI>4Lv=O7yBB@6>I@k(
znE|yGOp$$AvgR|gY<3z+iC4J|BvpV?eQ1*OEiR<?z+pT&_*!a<zil06;ZO!M=Jp=q
zt$H+BypogWa_~lZ$cbvn^N&5v9c|2LTaRM#_7N=GpFx|vgRD6b!##t1Bp=QquC|B#
z<ey2-xf6%6DNj8=o?Baml^WIy?1XFIIu7QA@#u$7(l*vWm$;Gay!~l5?M|WBP3>qI
z8^Xr=-Dz7lR#lZqK%^6MnUT7S-x^=VR}bAyQ<K6AcmKxjx6Yyq91<1AsnHAIYQQXD
z;!jDZIFZl|L(UPBT;ViIKg^NMyKrCUz#DxvuTC3BVvvG++a5F~EmrZKd*>38BFx-Y
zPs{#e=Ln9f#fn9HNx!uN2CJS^=W;oireW}tuh8zSonSpxdCvYRhVT!Zouw;DI!siN
zysA`{zYd=M4$qsCPz6=M=`v^~_|QFf-_5DiR1O?ENWXsl=+U$1<pEg5i`%AP@Rr1M
zk#x6lGC37$G(_jqQLSbWP7hf0UV_X<3Qni6VYPt>lLlY>wRG<ggRdZs9jiX%)1405
z#@FP`mftbe?1xtJNLv9uuZJR=3qrVH9o<@o;gaXC;t!kPDmukyv%ch5o{^5tjO<8H
z$I;fP0s}>F6c1VgW4Ga!V9V>qV~wX<yUu*_`$1MF9O9gz2G94ZquNhYMCcH1@Tk8i
zHm6u-CQKUt!EcbD;sP8%N^F9)9>26k<0hhMJeuPT)JOGp8y;ylt(PhbF+`IIcd*G1
zOmtS-U}UERZ5AWZxoPZQzm}XZ3pP)E`drxwgUgP~8iU1CGACv278PU)1l67>ZD0(d
zK5O+FVxHs`L(H92k$8#&V6P1J`NURi5)?vhlmAnfN&|n8HaWx#k5eJUY~sl7{cO}_
zqc`X<x4n)QO)R(^E<C~22@?MjWGH2sJ~a@d6^+Brwl%9cp|hh@tHsdj351#>=r{f?
zvgZE4)CnV*Db`c3p^wqRpjZ1|RjNu=sVY^a9C-R8gK?j?oM&C72;)_1yFXtfJv(>q
z%#4{c$;;29hG4GMqO1L(%erK%L9w-oub#%b!#laH=QRZDG-yvAVa@|@6ZF<J?vaeL
z&{b;l;II|aq-`(8PJD#gdXE4VI`l>p-t{y2^sRmLobf)xd(|VkZV`{Ibf_#=h8~ih
z&e9&@!gXlHTG3$?{q^9tcRRiM6Bh10!JD6bNv~K9o|I&=q!dh6C=iU!QF&}~9ck%!
z%~f>U^d4hh%ER63McRjHI8VSS?eW}x@Kh#k6M#bWGc(V{(JY0ob;P&6GDISM+f=(e
zpZ_(3PAqk==K_*b4TbqO3e)qTQG@_@1)Ngc>USQeova3}cx=ohuCJk20hi8bBrJ0U
zo??$`oLjkshKC9evz#*yC#^A|I0`7vIZLEQd!f63>EC*fi)_(erc*<8lTv{O(z*Rm
zhJ*AX4fR6B{FVF8VJo70@B4Y_!9E0OJgU*S!DPm?Z<PSS3ZC;MMv^Wrj3J2(oiKsU
zI<Gp{ilC3llfmZwcDg_KIs*ouX4ls*^7`0U>AGM#ZDPz9%qUP*s!CO<D*ul1@AkH<
zASSw)1WJ1eX_5p~QtG5io~p{_O*(#K%Q|)IsD|PaIH`uBe_)jB6U^Eh(}Moj-$2^i
z6Zv57N)GMb#0T$<XKD3nv`7pSOx=cEH9D1xDuj+ry3qWm@A-7mDvll9&-`~r@!sx2
zf~>&=S&FfpImWhMe_-_VpRr2Ed6x!V-77F^Qkl1KIVX#)bV)2<$3C@=ceiF$kTM!|
z3Sk<Bw38|PGG!X8R{zd!lh*GB?{VWSx3)Y^7sY|UeJwiCE<TQ&quJa(u-ygp$O>)=
z!b(UZ(2$Io<54u2gQJK}J;9Q%eqzt>Yk1=GO{C?!{GN9LD7oEbaj(1{2hMUJ-|cdW
zK8ejIfMjU)CVu$jEtYQE&60^@*(HG5O^LOr)4U~v!q+l)?zikdc9LJ`y~oot_M!yq
zaf>l|D$SR~7RKg%iLI+;%=j5>-t{}vr%&Tu>s7RAC0&$U;ytIBpNjiT5r8MWL2Kf+
zZD;E1^Vz?59WTw=NM@1$x#);k%o{hcV9{@4uD;`muMeZR9Rj$9(63=@g1`8Pc^h_f
zYVTS;cxwV*9}!^8h#0S@Z2W?5#V$cN(Qj+}zWD5C_)xTUBL6h2COpf`@3&B>ar5gN
zQy4$-E3(X@0*@J^Jr<LgXKj@)RaL1fRpmdZ{6~4)RfwN+0dKp~&jX$os{6G(pd~Z#
z`J>ZTdE5O75LLlT<}wzGRPP8zTfKEB+EmZr=bsj^Z1oQEf?D#tVER4d%;X>0&+*X4
z^lV;7P00|R*pe&i7P5Nb4=i1`gR|bo4D8#Px`_=i+s^Xq$`$N8ZQ#c1yJB<bXxOnE
zwWC4_+IxsKd-idtPIqqY-9^+<5U69PTE`9jyHF?GM8M*d(71Yd_UvHsFKfxGmq?!$
z_3>%r=-;+Jd1sE3AJdvEn?{J|-JIQd2xIH6Tvb0pjLCv_>n7HpXu{)951@LvxKgSI
zkFS_xdyi4O`!zHO5=`8l#qOhpBwl$X4eQsVL47Uj7tG~X0p_mi+X17w8r}PJgtI5e
zb_dg{Q%gd{$}BjO%rT4TSDQFdr<?r;QmEdt8||XBew!~#5NFpcWYt+6{YOsW?q*Ra
zk@e`<tRY8N{m8;)t4Tg*qvuW6(K^b_-qd2^TeYEXHDhVOCDj<$SdD8sG{L!U2|xU@
zf}^^oOddOm{tZLW2=H?5@E&sNbfa&}=+gIm0tg00*QI^DnJtUH;g=l;D6|#eRAT6R
z`vB@StV^URomC6JXGgNabpzW9lom_p?yacZsXcYI$t+p;6F+Y_h$p-~H(c9>>W)-4
zX9d%(Q&U1DO$*O(Al*)4rz>etJDx!;<2bzd8@~T_HCv0LY16Maof_1mQ-?T`)_==_
zpH{N9s22CW@&rBNBderN|KcHkA#XcHb@lg_p0m<<M20Vs7t-?QZ`DuL&wukzWjyg$
z-oKdS>Ce9NcLFnw&1Mr9Xhl~!r(Mb4C7-$7epQCt>|W{NB@P0QxS`8`OH_F>ZjX$C
zBTL%p(_GuX|9{*;ws`$+y0=UqEXeX-om)(e3srt1#XPC;odTbik2ZecXZ`Vr&RY<w
z@v}y?2MVkNHHFB<+ACeifvDu?EpV@|)Cg8e*Zgsiv?Wzjn7)%KBOYgc@5%i5;8lcZ
ze16ZhQmKqT4UIo;O}XgjmBw44P0@~~5*<pO^Y=j;7`tCRrqi8I2mf>B)0&h?!n|IM
z%<7`XzVTit1iOsAR8giINW)S#_LBN9I;Wf8=EjLhO#0ztt}sirn&yJzRLaw{+#Gte
z{wVYWVnmgsp(*L}g~p|(Z7G*Nro!h{<*)Bmu^965^Tqk-A|@uLOad#;$^vo8R2>4u
zGyC`NSI<%@jjqajSXKU;B|w!wyIA7jOMQ2#cmB$If29wXYFl4<lmAhWOmC$_zr^OL
zN^SSg7FEr6fdsP`V#@w!C1fZc4O=qBz`{5$a<)=he@T3w3)fK|OnKFuTo;TGUZGF^
zx+PHXp5Du}2XEmgr90CGwIxKaoR{lOGJBur{C3KZ&A+%SM#EMr(V-$;%IZ{2vOFa~
ztzuO6l6q7Ps01kiI+cIt!jmKW=#OkJrLh{N;@B@Wr>fhxK!=4z4o(>(1z41G;8o4F
zE<INz^eO{TzKR>Q!rvvp%U@-=8a>{Ck-Daef&YVyM?H-7`ck+5D%w|-|5o7izfvxk
z82Ef%P-ZZ$p2OuBK;=*cRplRCxEDwla`9*VwpQm!FD{j8rE;2*i_~$s^;MQg`GTY9
z^TbrHN$f%Unqg{W<G)h=E=F@PdTIjkmHu2SC4mM1!dL`tjIHnCotHJl>%FCZZkL{`
zzeKr<reCRI-v72;jn*I|{rgITUiv|`PK~D~PuY@a`lXTIWqh%~ccm9${DF*R71FuG
zpZ!{qhkUuP6B_jrqeiVSeJ@bAv<H;ZO8h}w%YXMrNys&7?#+_^1SGDMs3_(4dKI*n
z|BOoGD7h4>P9Of8z4%V0ivI$qN}@}m^J~xZ@>bt3_sqp;t=jBr%CCFhe?gLU3AA+;
z#;0a@khaH4*)xjI<Co40x@x5KzBF*^AJK&h{9U8YWx09usN#B6vh81k8kPQj&|kK8
zbQRtUT=W9{@n6I%rpqe9A0-#aKvpgFT=x0DD7$P_=1YNaF6EO1UDg>U>--ZEwDRZU
zWyN`t2IEpaSK;MKK&vEFK~<cms&cs?K^IYzw*70ClY~^2651H6)uCIpGD%QX`5%By
zQrGJ!;K-8Yq@J*0iH}5Y2%%p4Mv#_FYPJWXz(HQOU_c=e)M-|a2%V3dlSfEBRfNW%
zQ5leAHg27jY6%T#7-Qh<!2_Jlw4=yWM0M(sP%8|VIL@{143c-A#-@gt(F+D7z_k1#
zx+aZlnmDoVI1cf-R@${{{Mn<7k=1F^xF%ZPh0?J|Fy`f4QWEDf3;il~gHBD$<kkdJ
zqj^JOf)rAB?4;1C#cc|rUZeU%$zYuVBo&`d;>hU&RV^<i&`MYY^&2%H(hM1?DV$8n
z!DKb!^0+Y8Y(%r#p{fn2HaC?+$?2jEO{oX0JWmXD6G*5Tsir|XKPRO>#LMyByGgYx
zSTsHy7zwUclf;BtnB+T(|G5%=rl_U@0Z6E(Zqlj_ENZGN6=akFR{!_9GRdw|-ps)d
z4s-!YvYMGAkpEJv#@<rY)KDt;Q0psi9-+tT%4F+~!}#J_)3II<ZZTF@_HlOaJ%z(+
z5pyHJi@OLzt+sTDx00T6kmTdJ=uBEQ?M_tP#x$-U?w8^#7YAtI<k-#w<U2qw=qwh8
zSP!8jT+smA$-SJ(*Av^YAq^tkVjZ0zU-ZKg8cp?j;p8SKQ{)iyZ#JU!`n3hdYBi`4
zqZjKn4ZAcOQM9EwI(0E|;=3p;S}*S03{LM8>ryM`Bix8DGL|~Ap%iAG#o^SbCJ~C(
z57u->D?w4g_;PY^dbR2WL&7qH)k;Y4A7`reI_(m+1m4rAdr)sQ_~%A|PrsFirVOA4
zK;3z2yFb0r0|>n;w_R2FOT}9S3@!?ws=(y`SP>w@p2mSEe<id=BHFvc(S5y(w53io
zTj~<N=MZO}J%y%I1lmq|Og|TMN`Rx*)33y`HHqV+lhO7J#aJ&GozI2yheEuM8L2;}
zAw^qKI6Etu!lO2VuWO5M-i<V|dvKjP#EED3<7;O|v$FtiSO`|Fi(*GQF|!iLoVJnN
zV|w&m!Z1XMGyhx;?!!@NR}H1P%Y|Dxe+>Gde9pbSl@yl)iww@X(~G8`8D(ic?zaXL
z(ZfW}cY8=*kxS8OJNYy2pxxa~3AMQ>*tUz*k*OHkg`sJ#$GFKs7PV;bYhOb4C3E&w
z0T?pG-sls8xlb@n1e4YZrtZkgq~OiXWE@d2T`%5^5X^5~25E_QVqfY`v{iGyZKB{`
zS^=KceDgMwzurT$Rvoz__6*CmW)fRx6kq=O5Y?qKkJ~PQtN<l4XJ$+->O@%axm_sE
zV$yT%n8G574%3iz`Xsq-0ZgK!2{P$03b2%uc8m<0LP)rvpT$aq#i&ZxjCvjR^fXT9
z7vZsnQ@?r)CY=Bc4hIEx2a4W=UVvyB*+05^4Z@59^aSF*nN12w3+6Cs+5sN=@e2WJ
zJjjrLjzim7FnKu#%*h-~4xvl8I3oIo(kHPsd)Llo_$xosq1OPSk8b9-EGJE#e2;mL
z^}r|w=DP@Z(kMA(f3=^iA}79e#n94<HzAI&da)>L#M*o_ovb0<vA)$9&)kEY`8AWU
ze(kUhbCCYjHXQAv(H}3wCoTYnT24ISg?JvUgW<b_WFAt`cMCxiWx#y2kj$_mLLLl9
zfBFyyKTSp3AsRz<JFdki$!=>Bz{-kye=4?pVoqvW(bg2}Qqts%p5p7`i8y)==TQMV
z6RZgG;myjzQ9qW5#*vt$mg6G$P_a89OVB{tn+A)Ef>**C7rt04I;jaM62Ml_Un9fz
z$n_w_TxOjLa4!oy`Tj^uE0dR2MHv6*idI`rB<i<AA|Uqs|C!;I7&%|mQ2Xy}BM>$G
zA`?(*Y=0>mri+Mi%Z$mr6^-LdZlooW;_?7Wy(w$?!&qSd1pTXw$p3cAg2BzXuQ!ro
ztx4x^?xVgzhpX!rHWh9Yw{as4Zt9Kwz}L7NUQ5qM6DhuLHA{MKVE55p+;;!9G~Z$8
zOxJd_x-E+0^~LB~@8HA%!H|6dK;1VGo$GUU?uo&7^c0!v(@E@JgZ!hZ_^b`+{eBlR
zU;LSDOAp$PImPl8r!j^n(&b4XYko>0cIwU4+#<kPSTQ>$>_fTO)awKQEXu%kG>h2x
zhSN6q3>!urqs^2-WS;z-v)MMV))Q?E<oB>2U{45nE0QR<x;~*XK~%r?N@{QW0bS=C
z>3mN^Y-#KGso!Q&PH7o%(=GIE_#>+)WYA{p9YojE;Z;MCcrn%QK-XyntQhN~%Nw03
zI+cg_swC|DeW;OqtrxhQcs*(m*^%#kz%lz0YIhpM=tmpl*uR*&dyZu0f%}=(AeArQ
z|A0|*RuLj_PfF8XeE!S`uItnc-=4)hH~9x_A&t4UUnJkop2enJg*-m%7slRHk7Em`
zF>>Z&avXX>^K;nT>_PT@`T(_z9+KC8&3mKXXQRu<$?$0Iy6r)pd+1K;<?rT|(J!;E
zz)bTfA4^v6rpEBed^v6~@mdE?DgDVj%6FfANm94>8Bkk~J=duMgWztr^3KltNq_wz
zzVXI0=;`OE@69F07sQ}zy3yYmLysFq^KgQm6Km!&c*JPNxA=vby~4@0UC4k_@DfdR
z+P(Jxy37>T&R>h6+imn}W5MPWR39>k!fhXtyktK|`o+_6)?Fy~f6j>jdO(H0v2Cfg
zaxFU&o6u&)K+Mn1X5ZZd=x)nm>)hRVqk7ZsxjZ(!QB1<a?a<`s<Ls1$C#6ULm2^@+
zJAv!^*0k+W3%yQ^FL)oWVuLtut;Ai~3|m+V0_oDgRY;-1fV=1fMdnbVTE^nAXA|DE
zAraM#xKfXE=5Rj6@e!C+z2JpzsAAg*V5>J2;mS8)iItwPZd~VVxSc+XK~f^>!lMyD
z&1_IZ7W?c0b-Eh9g+GokUJ~c2e0r7=wOwU{@haf-e}2rWOjk~zR(i#$@^q=)5w|Ne
z|4h0CX$zs(@sFlE6OAe2ro3v=T2ivz{GT)~>hzr2!Ou%KlI6Luy^^Y1*A}v6^^dID
zcS2PaTy9TPnu+aU%N2)Ofz!VMOS}RDHVayxxP4}eTe;m2R-|2l6cI}qoz|`VMcbyo
zhc>>n!>pLQl!IvwsLFNsw@RHfoa5XEAFjJv(m?CN<*=cTY()I)eW?*=^hd7Mcqm-D
zpM&3QA?b@_IBJGcBT9fQu{sPv8VW!EoqhLz!JZfP;OXC$T94L3S1jJMALYax4b^7c
zPN(a&6n%MwGg>|7YBi|VHd+P3ZW(RH=fR<b5pi8D%;H+@4ilF&!G!Wg&1LHi-*G4}
zlBTa-MO`(<*@ZTTsKjd}tWQHCL-Z(aw`v1p6I{yDsTPrSg3u{0?EChUwI_k*%Wk64
zxf2}Um4idIbrS1c@sjn!PWHU>D|^Q85WGc8P@F|oU%S128$Gw=IciWfgUaCa;vJV;
zyno<0JD*(0swb9kQWsCBo7xf_<d2Vc5&14Urw6COMXpW&J8wRFcMGuQvQr}lmb~*A
zPkgbPnM=2FX#W=G#_i>)ho5DAmKSrA?u>oz1`2<EpU2+YPQT~p@Z*bP=~~-}FXIH$
zC%nS?N9MC*$9mqn?|OP#gRqMJ=I;K5(GNXE@||P&`L_*hpKt>^KYWIJXRJlrpapHh
z3)y@4DABi%W$kBU(JcFfX}@I>Z1H;;=NwDo+r1Xr-PMH<yS+@JBKWbC1UeLt8p}<3
z!n>vVPJ<fU*K3!A^9VY5h-=%On|s%0$Es6c(qH2IQq&8a*AoaqBmk#hkRo_bkpo*)
zG~qXeQ7~^CXHxAd@7LSJ*<3G%n066ZyfT}Fm@B6XpGAPkMsWn`6dZ1G4xTy3vFYpC
zHDx`i_DC9aZ$XGR2dAhbvRw_dqD}d(sY6o|dev3q2ztdDF&8-Stv|%+uMUuU){U`u
zU7}55XeuxC;yj!|-tlu3oN=HDGhxzc{!qK2+FNO5z-NC10VJLY*iT8~XvE(>ml(S%
zaG4F<ZYx9>|0BuMg(8esm48r`BPC8s0h^Z1XU@{yVnJzD-MH6o!_>SB_dRkySD3y3
zXHuDxb>q#;;`o^Y!r~iH%}`l-@_!|4)8aV1mbYJA$-T2%)1aD3tl7UeWr$qUu6^I|
z>dR;Npl27V>3!!*pQNtcSH$+E3rK1cMaSm#F!{Xv5ecLLW%0q~O#AX@4jnrun1zwZ
zEBZ6);hTvM*Q+)qr#F4e)H%z@$gzuCes!AkzMavxcSc4ebN;k|NvURr4(m_6#rKcr
znp5eYp2WiMex_H`t~9M_`P=FLKfA9rGQXiXz;yxJQJ<f2c|2xnHHlXln8&HXRO}-^
zC6(X5*nwlULiEyMG>tTfL*J?57E6$o*yw6x>P=Weqp${fWrgH!+DB1ZCNU#dadNMN
zGlrAYJC#k%_PuD-PJ_{xt9q1byaMc46KOcnh1n=B5>BzkTvkG&<HRN5oB%5pqUwZW
zbY4Ua;&bEG)S}*)8h&QC->XV6>|P`FALv1hBA8~ei!(d+<JoD!{<MpX!^Na+KSBGe
ztNRUw#edQc#aSTsw>1=Pq-b|TDz5$s<>bDCIwl>yeaAU?w1|lQZHUpzOO2Vh;9#<9
z+3|*i6ICq)ug!OH!(mBtLohXb`<O9#A)CH7Vl)QRY2xcV8C$>;2a3ouY5DrK$C)px
zupP4r`WLe7;5i2LszIa1jc6I$mV1_e#=~_iI0tq?afr4Hvgz5cH*=;8=hj6vFq@&_
zHGPRTI9R@ZIltAsi5;Wvp`Nb*U*AWWIr0=kK3hTIb63-)X&A)a$CLf9q^`q4pZ33T
z{9q2sb&=>D9x}61NEM*@6Y*pB6#c0+R!-o8CMXiizoe%SU`?#gpoh9rf7D8nmK-J~
zR;&w$066Vi(!RY9gE${jWfLy3t`cLZ7at`)^Y{(QHRAj;3!23iig^wqG%Ofjq1LbZ
z)eBhS$Rc~?VH{_IF@|gLbZbm40faq*=NXj{jFEmDKXXu!0M9Zbh6bJ8E!J8twk(^t
zbcSK>9fMWz{o&cHDsGCLIMX~RV%=&jCX6P(YECHuSJG)ihRP9OQ*KeszhW?6CWO#c
z$a+<!w!2&qqN$JsR7&su*q=*Gr_RNmEx(Hcus)C9+(})SG6t75swOq{ir@1@dx4lw
z+7e^B60x<P;Y&$M`S~i1-xN?UogeQ=Avb9e<Gwh-^Y2ckniP$A6n|j9Qg?aff*vcH
z%APG3iBnWTEmTg;Wey4>CMHZ=mNb`EhXv}Uq|d6+W%(FOzVl-v`2q~V5JFT$7$r@d
z*JoAL7R|lm-os_EVD?@pHc};Zp6Hvwags@Izt8T7YZy7=W`dpP_-xu7nmR3<e*Okb
zM;36?^B)n^>uz2e(g1tPdLDUuG8x5h^6v2F*iY?Z!@*E)@($pV-od)_C4x%8J4Q9m
z45$K2##It~OU)8M#rdloyUWz<8eX2=9g`)*fAg<Q^Ac|=jr8u}{54foF1)Ly8d?)h
zaBU5FE0f3=+MP&gWTVro3B-*8v={~bTtP-+p17K}x4GHaeL2O+`RF^<k-koNiWQ85
zTT}0j>csBL!ntq-Nk<cC_hNNUex8Bv!4?>!jQE--Qg7)Y&aFB{?Yk2Rw|o5(Pt{vV
zWnBboeEwTfkyr-}s$&`3pR4*NQV^8Mj>p#%`|b_Id)yaPYfDL!Me(bcJz9V01$n+$
zGzOfqjhtg7E!Pk<t_dyy!m2&r1n1W~NgCdcmLcNyuQL#HXJ=XuZHV2KMabK;II-vi
zo`H!N<;6vG(jb1MA+q~K2Fa^V(x88PYKb$?5L}(QQ?4Q=BM(pVZdQG%p>?0mM1^WD
zvi>w`z}$lH23&P(HwIrBjkRtoS~qKg#g!zmpq&P7dvMKwp6Ko3LNoA2hDV7u8d@l_
zyKo3_sSj_#HFZKL&d<lK_&pP~S_=uihwxE5A1+ruuCx1iW$IXZby+}!C762p9P-8V
zdZZZHX+v?!&81WGaXRe^U93J)w@c8+EoF*Wr5I2NU}~pRJ;&wQu2hzYizzcn6_Yr;
zw9?znr%`#cItS`rl?mkac(b$F_w63Gjt^J!h$@%9HHmjMsxiE3C#Lc1z%-&AodoUf
znlu{Uf|M6GaXL>Ng}2BRBt7W#D$mi&14Oi`6YmB2jDC==m71yuVjmP2RCghsvuW&I
zag3Vr(pbq&W^y+56D<TctS|a^jO;aT!n-#S;}LtnDu7M%y2P1c33h2HUVen)o{7{F
zwD$y?32j*)t1pkNt%WGXZUMf{e<(~)d9lTSIn3-24P&Zwh8hGdErQ-+PilPS=>7L=
zbgF>UAJRxH<Z~GrjDK-Fg9Z)a(^;S3a63ywLHrSb(r80y*Y#?8_qvMSef!e?y6foK
zxiy9z-}BVtFSGKD*ysXCZJYHnkBs|><I?LReJ|4<A56c2H*@#Y`J_9vYI`PEHtVL1
zVc>xN+;;!d%o7K%WV8j_=JL?T8}Q0aD1MT^n#!x6|H8SHtvveDZ1(K@kw-_p%;uzG
z(H0zBF^79Z9oOGAf>|3*sPC05Kvg9rj)s&KpYgz5H*wvd8=3I!Ci3LwQqJ-9n;-Gz
z+<Ck<bT9*NzMD6f9Kfyhs|Fn>H#6?uLEQe}L%cX|HT#`b!AvTv!?YBqY-HT<AzVN3
zCPsg@PPGq^eSYAD8635XcEpDI@|7`SOq)c%GWlf6EauJrkOyzNo*RZf%C~#7N^i9#
zeO454b6PWxM~B`*|Ld=3?7X#PNr<5<VD-lz@#Akt1Osug{?nKF=(`2HKjt2K_8iRg
zA9vuP%yvmusw+-sf0~UeZW&6S?%nCq`&Qn#uPcquB$HW?!^YX~5)ywi-@G`C9^(6c
z_l{xy!;RRy^gVWny>E&%Q7giN=@RQg?!BTNKk~vIeP};)93Rb{!=#rNa8As{sTE%{
zYx-BDOF$%++0o^5c;J<VIOP0l#oR3Zh#|M!zzuf}<>M_UR6wXZ@(Ytc_>3=RPU7xo
zUS#{BLwxu7Y<8U~R)N3o_-3X*avRrNa|>hMT|~CauZHww?&9MY9uRA@H}{U2!lpF4
z3f8L1#c%OmUkLSXuMHcL*z)!oQcj)b(6`^R<@!(5b(5EwM#j!Oig%o&D6|P}ro`j<
z;1||yJj}WD(`22=p=j?Bl6Iz%?ablW&QxqJ8>jYcA#Hs+`nK`JKYlfh@414oFdxU~
z@8Z~*0^F`llJ*?socKNK%t`i~IEPcJOx*dT?@Pt8GlO&Mc5_l!NKW!mvNmO5)BX|T
z@N({83MUVoqBtj))NMyNC#9(radh!oiqf^1`?sL^Q&&@KKwWg%`5gXUTnJC6lXI|;
zyyGbx*t3`9Me8{Ft+<R|5sWM6G)MMk;NF`{^1c+3cO4+@z-hF$U^LDwj_=Q=FejY@
zn~#Wb6q9%O7-=c_c*^^(R|XqC7wJj6IkGjIU~3%`T6Lvk<C^H*1=y@H460dQ&@mln
zy(_q;_m$K-y_<=XKH?_{64H;eXU7f>Wu9Q;`Yr6)b(Fk(pBh3yYfI<n>7#f$Gn@gp
z4Pj8<E74^nkz%*fx@&h@9Qlct$IakyS{7TD%%=Yvi#R=GAd%i2_HM~!(}n{avpFfs
zKF+b#i#d~gnoOBCOI%cgqifKt$j!+@kJz_LsW!k}aE`s(x3XvVX;QWyWB<-A989yR
zYXAI<Lo7VBik&Bt*|&BPPv1R=Em56Vc4H%K#qLYaWBP$4k`JB4ajcl4!-qMrKNY(S
zj-7Ru!c~P7i@xSGZ9&7fR@|rTDtMM4%bl04dU@s?w&Oc^g44&(VUw%IRlwP!x!Ce^
zII;f(c^P)>r_YjcQUKnd22|^6MEPb5`;rTwZUnku0g7r`3AGBKl%0b+*MU9t3|U2X
zG3H!~PZ#45K;BkREY2Ml_8b@5=xD+k2%Nb)jm%<~N}!9aS84q3k~Uw`JWlWxjZvpw
zB>b9Cu`i`iqa+-I097)EpW^Y0IWEiQ>4IXwi@f8lQsrHS`VBt)^ivisUPMYt3e#uK
zVC|Z<6-29-0#615oJcvy!Q><k?%gXk@nKG!wGo-vf~cZZ44bo@<UOmHJmqIXns%kW
z<1o*Rn#ep&SH_IEgV5zOnDqWaaSQOV_Vs6YW7z@jd}0*W)h%H7L(d2RRfxIhAgP<r
z_@xyZFGrG+`D*_;G$C=^(xxt!s00S}?n~piAhLg%!iecVk#NsbJkmXy8P84PyY1;J
z2=g;_$o+XCQ%BCBR{x<qcW-yre>{m{3s2z_V>z1oD^q5CM)bhj8PY0<@$XJ$>Mp@>
z-N$)u^klXdCUWOZJqRk?#sP7AF={H>5owZt<%Jg~us6OBPd#!AC%>M`Q?vKdyhkgH
z8$RH}rJG1zGnZ$ViCb#h7Gg{V?A-kW)8=oc?|s9#I%qFrCce(HR4*2@zt2XqLh;&V
zOnrO?5xwr@g@^mGYxb+$Fh51Th3!k-z}ob)NI6dW+8=mY&^)94FrMfa$@i~3$-<O8
zRWeg*3KA4a!y1u%J7p4Iui3$gQ|Sa>`#gI;8bxBTkF)7IVw!g(Ru_;CDLR_A>WI--
zNPfWn>QcfR6+oWY#^lGwu%}@!Ucaji`+l0ovK6Z+7B6RJo#OYDRGg}`QNek9jX0E3
z)Zb5j^(rsCH;3l8KFY)0t26V>mza4-05&C$gFnt<&eCkU_vlA$net`fHd1m$e~R}D
zP&<z8A-$RK+^sl&naK0suBF&n%+D`8%)+$VOrA7``q^vgHTi3Ch<Vi;Y*o3){CT~Y
znslVa*hH+~9^%B%pE><S9>$TqX;Q;Lt^nlZ#GtI-PwLWB==)wv^TAPQXK&-s4+pUy
z(PCM7oXiocIQj5Ka`uK{?&-w7@EpdVSc0bQCe7)=z4-*$yN%R3nIXVY4)){8q&=|*
zTe3Kmb|!Ih>K=+^xPtUEq|7@-s7o-8F~4!*k+qzBDUIl6^@-A4N=+>murY^>*SB)^
z{dB^m>-o^toY_(!E{G?{S*jsiN;#(=pg`-O@N)rx2qW&71EefJfIh1ln&pQ$^Y9wZ
zy>$}peVu9kU_<QRtta)(0z#tAxTb94?4!SN@|!aRHLH#(bwBB!XJQSpkvr-)PK{X3
zsdtNr5kSKfqPbvSwOWI?1D#~<)Uh1SSjTVQKg*5Ry~`dM(m`CB3XAmgd38G95AZPa
zo`Ll0eJ#&?w~oP&jo_Xp;T&D?3gcgzL2B?ZUVC6T!$(YK?U5oZlELbX#06^z`}i69
zbnQg%;gh(2>}T9tUz}ad`ZM*1PYB=hF}DpI!2K_M%FPp|bMoGfc#}4>QXCdFQ+{E=
z&SRW7eun+}Xnsps#_FTRnDu&Mq8d^oH<SCe9L8ud`_C#dbr}a%GW_mgJiqe@>kh1E
z{6qJ0<##&>A(yoW^QpP>3@b<9&Fznmr|_B=_;kVBbkf;yTvRHfRqgQ3j9X9Igd|Km
z%@{x0LF#=QNjqaBJM|bLk)c>W*iOol9O6cGqGs1PHBr6vOmiMP#krTaVfUDb`r!~6
zvk!1ay1``&z_chs^xcE~v;AbhyPeGW1%w7gV+_#~J>W`e^bA4w%|6oJ-a_66H@bE)
z=y6iG?HuM*1J<o)DPD4%qE%-hCIqv`jzfSggVu!RPzD8d4W@>*2o4e;?_`m9Q=@|A
z3#se8g*KD|pDKYwjLYLF3pgybI(P*ClG+Ao?<6nu2CX0al&I}4*XF6ReMotmrxI_w
zD#_F3l@ljU;B-0(4h|+GL)`8%&Ry;?i^=L@_rkfnnzRm$*FjNn7~T3j$ulFGGHS$N
z&WwDEQEL>ex8BC+A<ZdT^b#izIeFmqb_7R3*B%WSesB+ur!{A8(pj26H;X%a)W_Jd
z4tmeq_-tpfDMqmv4W)>^M9rf_)T39wD|tV4AAS3^C(=;FwAte3RqGnwjgQ1z+>%b=
z-?C}{K5l9khegId5erS*rY*z1_*mQ^r1-Ns*Mu)+=P4=esfo*>aM#55xW9LEw4Lg3
zVBK>RW!NwtTg$n8J&%3<64%!<<1l*IF-|aOS9$9P_<iMO7N2d%YxgG-8m6INYZvo2
ze#6<}uQKMYd%5Sm2|Oz9J@?%F2G6z?8`Ca8o5w8Z@fJ6?j6v5Xmh{16*#7%5uBvJD
ztFs(ptSwq__ZPDW3NnlJok#C#Ke8${Q?wYVO5>#XT#CIN`kM55csws(R|}mZmZdwN
z;%Is<nuG|yicH*A+~x$PzWf3;-ui@}-X6`nxmsFWe>dX<*o(L7G3&GyY%`>k+@r`_
zlkPIb!(MTt6LhH4gAbn_ORrFcj=p^ic`sA7nNmgW+KQrTqrpfJ#mv~gpQJW-@UWP_
zpt!Dd|LR5dt=osvR~$RRoq7GUH)$WG$GvU7dN5i9b8v3`f$gH-!v=Q1VsX*BVPl>W
z41P@iX4q|B@=Y;Vt2Jljn2EGLVi#*ksynNsPnVQGc{N0J=t$%H8e?;~F`LC6l#U>7
zFZG_fp9UiXwkzTw_vQPj|MV~#XvLjaU>)UtO}U`9f#5lAvG-)|!{_B16u&z>D4jcU
z^`6fDrv#YtDmCfxTbEMtjH+UFiu%os=&>cSoRv(#Y(UzO2nOT1#5(jjs?p+$yGjEL
zs_xyg&z@M?|9FLe+j{+D2z`Dabzan{{gXN!<rylDLmKNkM4Lmdqt7r+X~5bN>5(+%
zRb*$r?-Y>qmix{v-gQ^ZNKmF?cHs`I&wZa9;$dw`dmbEim)|A_HwBsyx=s3qjpMy)
zS9DUU>UQF`6=HexbJjjlUe)cz>2gt2<OFLhgXg9&RHvw3yK4L?(Mgv}{D^ZUxcPN_
zzI33fGWUo*sPj2-+Vd!?(T%AKwyCOeuhW5x2ibRUgrJQ_RpaH_-B=S6xqVn7XQs_z
z?~QK|uNCi#?$z!(l-;}UugIf4PIARu^?Bemu7C8+vKjLDO+B2K6m&>wOHJA?zPD1K
zNCibSsMNiU*qZ|FxcwO$T78~?^gM8h1KQfA6|LvA3CxLq-H0#n#WkhT<YsEn&eEKq
zfqC?)4gqFr^y-55Dvy|Y9eQ!j>ivdCMBm!AbdI&<#EBaDUe8wQx>5sXs({c;Sck@?
zf*yqn1iX~qc99Vnq5@7ofjF`y%UxqKqc>MF7L$^CeK~{izhd)L6(sXtv2lPKZ@iJ@
zq$E;~ALGidSJJs-r^^=ci@nRW!$vddfqrPECxeuK=%rgFSlT_nz*@^0os`RvJ4O;M
z-bi<u1OrWD={KLTQEW-8*qB!*CQx+dxR^5yt>Qw|b!-c7$l!?|s;%TMDZ(eXt+$*i
z%YV3PaEb%OCy=_`C0(q2=#4rW?EaMxXY8O*EGb_|BbrwW#U(>xs2fN8IPL8Hem1)b
zv_yp2_$gT-v5#IouzX5&e+(4WR!AJ^Mm<K2)brcXY1-2y$ln8vLE1HxMa5Q14oj1P
z>SqtKWcEx<Mgg*1R+=@hO}e-^G{66DYAyeT{n?@1^klcdmXo8ZO^hi3_=6&et`|<G
zyV!3V<u_A8aXUD$=ySGadx(nk@bh60)jJt0*?1_JOh)QN1^f4k>R}{zmZqYun%oyP
z+wk1GGjOD*vT@5gzIgjJMr3OEV*GXDu+-o)RIJ+|Hm^K%Jw8n(CU)U9B}LB-L&8W1
z4ID~ZO{xE-9FtM2xq!Wt50lx%k?m{wG<&b0u^uI|HCM#CDHdzX9ThLuka7VCF9=ke
z$FkWUvR+fH8f;!2TMd&wg1)1kWbdqHJT+n^4cm8Nz@2v!F6KyGpjD+J^?kn5bSyF{
zuyn}LNd7C2X1nwN3V;E1pI18jovyM!To-9T0t;vPzddTIpYrqXv0X?!E*QJ*ug;f(
z*M1SrguN0Uak?w^&v}t5UE-8;U7|_*U+Jb7m%RwrCC6b?opRK_T;=wGIL~a(i@)b^
z6$PZc=cmxZBS4lnph^<sFO8R_Nbu&V07Q#Dt_pMIFox6b&WG5MwVVwHGH8-$6~IaY
zNLQt2%Qxr1<NAB&QJoBT*+t9`#x<ph;-&p#sdHKRLq~e-`7V)fPG)%!;77*qGO6B4
ze!Eyd7%U6j;;m$}gr{O#YE*Quufk_?4l4id4`Ax=bNOSg{BxE2S@N#_ud3}zBu`bL
z3jW%G*{^S3Vq#;+I43qh<Hj^-7=M{Dd}KT)Bi7I$f8tkhJJpqEt%n!0`b-jG;+At@
z>3r<XZpYxUQ*3I-sL>N>5o=Oo23ahjSn@aXt^f;p#eSx0bmg)6x2<>@bRa<|>McA>
z0fUGVEf@qi3obbTJbtidHTgNVQ?zxm&u~^e*+tz`2BVCNVZkJCv8usg5$nHwpI3Kk
zd2h;#^sHeZX74aII7;BYTiqn(fq?&pEp0(^a2pJMFrgEDcWWvF2Klx)05cML^4jDl
zxgtz{Xuv4lm<1NeT)K$NqdE9&CVri>kB;}X2vnY{+GUj<Ft&nH<|OT%f;Dz_Eu6u`
z-wJsBy~*@#5JK(4Ls;y%0DzIf(_9z7i%a_;&+a))`7xKF<DMcWt`1k<a5q<+I!^yt
zdpJF^KT)+sk4~>Ak1Gfd82X>tz7n%HN_7NKz3U9(CS)iHYv3P$Y5EHJe*tO>%JCmV
ziBY%&PU7EV`5tn`0aw?eCs<#$YV5^@wCr~uZw>8%Q34#J1*^r3`QQctkdzC0_DNd-
zV|X*}f9-YJh)Gv0Cb9Pof^PX7_BG@2SFYpM13TFC-J87n)NvZjdy5uPri-fk{}+pt
zUP{Sx$>E%sn5Z7iR;#rtJx*1*sM&_O<8{2yxVykzdNd{rLHhrkJ$1Rc?k@G*KmFb1
z_Pg>ke_3r;{s5@TZTBbfyi{Wenzw7$u8hW)9c)p0S~xhFlFXr_wNZ)+1?~`xJ0ymt
z)wRr+Ig7b%9_Ob4;f#IaQ{HLagK<5&67|))yuS7zbM9(I^3tE!>!`tF!@6>9P%!gf
zevi)YjKH{gE~6*!V%WzWXcJwXqRgfIy!{sXx_)QH>Lu*!+*$NnAvmfAk*7BB%cf)+
zcaNu6w`jgy^DVy~Z_j`l>3k#rL6d6+Fral*z$3?lOXDG<dIFJxizof~8jG{-MC!_<
zm?gYILIIc0r%G^qwOb32_PuIP_1T?`*syFNiyYAmHk94Oq>V+BmW^n?bS6J-y_RNu
z8?yVCAK9Vl$&1%!@Y09xQ|JD-crjuhLq2+k`rSU}u6UyWgE>r`zLX}<_2u;Pxx9L+
z4zCCRYjG~~Yoi4yu}f80joMVR7&yN8brzo$&_o}ADoU9Pv{E5dK_FOZ#n_ivLxTDB
z*BPu(BKhEf>#6QYVb1Sc+1F(N5s^{Ux^)=Z!LKvEW*AT0)*IVzAM?_@-NfAc5ly7c
zn7F5=pE=I{gNN}N+^Uqq5Eeu2x-|*b$$)k`!fH0cv~DUhXKmoQLDl%;>s@3Q#j0J1
zw1(nX`!lOHBoiKcnlbYaqpjOa^fs7oG4<H8;72z0ZcE>=EEfN`8q3uqxkI|Emy3w#
zmC_MuJn8XZZq=FG?CmW6ZaeKCXh+J7>Fle26$5*UcApqccK=bl@xs+~+5H{gF7t^w
zRs0@1Ri(mVY%-O(09311t%@jB<ub0%5D^hs#>DmiyZh(gSuRWRRHC-KP>{^hfmEsO
z{<QSzujMAu5fvTE?yu%DX8Tgr=Emo=;b?meL+S*v@Aoi1`1lUmHMjEI&{d3GG@Ak4
zp5ucFx!nEA`#iABL#{TGZudVY7_No8-WpHAsBw&cWEh#6D6Sm)40m=2$I4(vUcHa0
zV;*5)VoTaKx{_O}=~eHQsQ5Mvs6T_5BR{6;TVuKIfw!3C7%Q0NL(EeO(beuohda%F
zBR$onJ&a!0-opL6rt{*X8!>h7LHD*<oXL>(Lve(JMqmm^m-Gfbk+s563@+Reoq2uK
zT|D&WbROPXg9foWy0?kMRL&1xh8>9PI+)j<C}iA+ley<h4H|PKLq~Vummhv&O7QKh
zA2NWBL9Kaq!(j2*0`45$S%4=!O$r5lMl9r1p_U;JKFjbnA-K1M5)$UcVF;pYpPP7M
z>$^;TWH^TIz3ADYK1VZUpPLYDjU+P2;%9&%p@bR&8(C{6I4qI~Lm5+)2H@@nH}d7Q
zP(FV1T}F;rgw~NmP}8eeG4d|zSOg$$e+Qq>wDHCV(|L5|0?e))(uxcyDW@o~x5O9~
zO~T2am^R@cA!2(ui}R@7>t@D`dV<CVMU`4dUD1y>pHE@ZCogc{@^A{%GXyPaAhkR7
zdJp2>+Yj=|<VV;Tl}PLET{xCypg^prE~CWro@e>!`3IQo)nTsHm8YaVgDV6}b$5x~
zyQs@Vu$UWzXwMhhm#-(~^U9>xxNDV}{JaRRA3ccBka|2dc^DJlev_etW>Dmf;?@~Y
z(Y%`Z4>J{1m8w!zs>+`cLi{@cl*VSWc?2lZRR&Q$HT!npN2=!nHct{j$*juqZ;u+W
z)K#jsbGhB(JHPrO?euA`?ce`Dj^K;e@1}dp1i}R4`)?JEm*|EoB#hMfjVV<lW(Z_c
zC!liC_+vbYxpI1>XO=<r*z*M*(kedO#Wn$0ESSuCHTa<~kU`sJw~4tmV=$>Ns5DdC
z62H5|yvw*D`aqkC!-+$q#b7j`*Z3-Uhe=7B*X_XWRzw>X3|d8vlCG2+k5b`Ap{TY}
zYQF{dFP8RPR=)vzWi^sN>bLlDxN)c%D2<r3iXXRW{c%ZD8I@FFNkeJJeufX9#tq{q
za&3c1>@te6)jw|8W=S$n`L5!2QY<b7M$u<!GgR65uyg}g?!QtwYFA(^;(ck9>Jb3Y
zY%yZc_+ucI#M|;p8wjzl3>FJ%zs+LU`<v+a@MK==Tgz``>#Izsigb{sR!)Z<pV5kE
z{Ttl%=5Z$d@F6XwV~*m*Zuj6705!;<^UtLcSOel~x5K5{7Z^<XQu>!BHVuryKQ)Tq
zbysTcy)M<@Uni4k8vOa6q;r9!n~YPVH<{Jsu*&~1aFXjNFF#*BD`R3}E+J9yV3T@p
zuSWGk)ylBn8WUDAUvBA9B-MNxBhnJl<`HX1Cp~`jST%lIIkzp3!dy3mo7FvOu>_;{
zy0I&6|3${*4;G||jbJcZ#CI-Lx+=fYh&l9$Jszwxl=jAfw3Si((fwrrPw~67iIeGK
zE_FTDsHPkq+ya;z#hUPlb>I_w(jcQfdwiwN3mUHnhYUd`V>hYkhZL2!NM}E(cK3)e
z`gDG0iqg6j8)wsUP^>Xjk2GGEoW`vtJ{Eh~<y6fL^!on}#|5q1PCgbwN~T26)FJ4j
zlVgzfPvUQBZz)~t{c-UW_3Y4zi`(BJjq(K|ZA;^B>Ga~1@$tmO83L)IyfUb&jOnJS
z6z@&)CP{N+`A+!M3!Y9&>{Z)K$qzjK^HWA9FV|s@Oo`$Z^KR5$9xudi2df&TYy2T%
zd|p5Q^PknIyvF}*MZ3?EvrQY&V^lVd=MwSPE2!;C{;e#D`ZGZ?tI9uvxGOWy5^n*e
z9MdkF#(>?5POok%jhY9q>>>H9HYg|%95G;c7YNy@xA;F-D%d3X4eX2tU0~q`BC(nc
zsAf?3&U33*4P$jCg7p4HUVe#H2C-#-m)QCU24VDXfD5r;O~A;|EBYg|f2-pwu>lOI
zol0H(OHvKxL9tWvCL<Pod8uldLAtsR1TzVe#`sD_+n@4%zIgvD9^xhM%gpd5^|`YA
zd76?awxWO1zD&}vPWJ~`((HPC(Q&Gsna@Az7v0Y4{K;uH37{tqyaJblbj9d5cvaid
zV6qte0r$#JK*<^~>CdBo#i1w_)RX>Pr2>dsOAOWnYD`H>tzLX?2+U2D#ObmwMOl0K
z?bY8YswO~8t^t=OhwOIkx%v85ocw+sM|KBM=gw<r-%eroD@!?>FK|+wMs$3t3x-`g
z`E~wAPH9|N^E3q2s7sR<Z>8>`ZETpgAFWMKew<)jW&yCXAau}m)H59-dEQAhp^zKq
z!J1*BR)>xxUY9_S%SQ5yZ#kOgK@(}hlHsK|%z+`K9`#3DON&b+tk)gi#8b1^h_k>(
zfmnym0tc---NfMAI<xETk$hheMX!EOb8Dw;=6~=m>raZc-Ss}+Y<`wEKU_vcxD}^Y
z#s-T(Icq2KmdAL!UmaZg*YnPd*&Gvq!drZn8hsz*xxu{%x|C|A;1KV>HkUmf17T((
zn*1#E)f;lhL-*1;()yo!#^<ImWy*9CN4`V92>-r1`ST~dx9t=W?tBz+Im*qmaLoe`
zGoVEbn?ISrS1Aw^W*|2`o7hHIF>J&QRR7C3(ZiM*6PR~Oq1{!Z8Q#~$vQMY*(}7}g
z+74y@9nIM_X9^1rXMzB-{qDV=UI}sPo7p?p@X_2g#NBZRcXe!r$(hHpPiAqz8qbg(
ztysHaG1+zp!J*LtV3{b+%SYR!E4N?Oig3|iZ`x6oFHNG^@S8|5{jq?<n|+r3dy~l5
zn2AelNyBJJ-M^P?uaV%$SQ^%gCHwdZG6gC(>rF&N*C8t0f2nq4XOb_2Ukcvg6_=2R
zuuv?4bI>QicCk}R-VNfSYgWA%<$XpMh<&I%PnAT;ZT+v6syI*o;Rm#GamM>+KL0ay
zS7g3_P1<+4ul-RUq~yq>tHH2`?xSOk(DUt+E&*2mKJa_#@yIc#KuACuYfK!$UhkjM
z@M;1{q76-NWW>k^siX6jf?2igOSb!ezox3nMd+mqC!t0o;$0T<mt_)Cw>eE?Bh)x)
zk)g5pX02xD*j1b=)ban=I}0$Y%I*JuPM<Spx|5+9x*JIWrKOP&umA%@BqRg{6A(p2
z5s;J;MJb1n?rvdVf|%|&9sj-e8DQX2-rx0J@Bg0lJoC)VIeWkTzU#ZzyP{lZc~onY
zFvvns7!x@>dJ8IhZJr*~iAJG;WG%c-vQ-S2A3#KQCdt=z#FVd0wfZrXKAg+VRadb3
zgc6*PME({#F_mMe9HivhxOMD4nTb_kNbz5#;2m9w$WvKlU-O`JL?s-*oa0*IqvzG8
z_rj-48GO|RY@5FaznBU{26za}#(}S&gCA#qNdJ)^a77+QWRyS07XHRf0cf4w{~KSN
zR8yh6g74;@6p&9a0R>l>F+T;R@feez8N`o?W#~8bMFzGD<hL2anY8-UqqR3^2@`<c
ztu<eBExG~`ks;*m{9M@oE6FYP@zCU)W8U;}y#B^WzB_Q~&bui`xAMch`Rq+Lqsoh8
z!PGH~|6(PHc`iZAk#vT+Bj4c7$se$2(K7a*j>A;qtlPVGF(Y4pop--pPkfO}<SD#$
zk>9>w$DR|%iT!j26UNTxZ1(-GaCT07KAH)~p{CE>u~z$s3S$#S#<^4cxY0=EO6Bm@
zsw6?noS(wpl?RBdP@YKPTsyLG1TT#GfxQ32dh`uev19IRh7Wv`)8;^7>_R!eY&$y(
z3ME4;?}$*Y?K#f8IUn-u@>p_;yP;MTacKKDy!Os&lCovk;$wO1%Y|%BG^6(QAwZYM
zvfon)2#um_Ol2x+Z!z-79?llH#z^n&hvDV{7W{tf32$N)!t)6v-pG`+U7{zjP7opr
z;u0uOD5cDi68OnX;>OJzB;824<76XOsgT=DWaSzJIazrpK<4d(%_iQ~I9$TFq^z*p
z?g+e!i4!MI5!cuq6@CeZQS!6fz)Jee;VhQK9rrwAcR09xG#BLOQ=vkIzuf_K`FbkR
zp#f;sD)(vqzsR7Z>n%N{t3=?VR~E+v`5U*@;r!<^P>BjVzc6Z+3qj{};)XH*!Fd!8
zU6a`?Y47#=N9aj(e)u;jL%1M*U$q%2J)twTO9u(eNQGBK5GhG#Nk3OWu74EuU#^e&
zk8`9<>PS6fJeR&Zg;(#^)NEOiz?QWr*C>*ZNFRLs0`Ut@;%KRIG;0}6-u65c&sRiW
zMo*=>Rq@cACd1i~mctqm*f5Cfb^Ezhr3NwHsT|r|NclzGsg`n`b8TzWs$V^F79GYs
zunCp4kLa*hC@t#Lq+Xd&a`wuZ@%7tODi<!<Du&mnLHgWpI2Y<9*-(MEUmS#S^L5^Q
zr#07pT0pY@M|?K9A-jJL<jol`(xir!O`C%09JrF%hl2QG+ZQw~9ZKmMovCU%!w+lD
z&|*MG{2n6j+ckdFsZkD}iwAjX`e!`fv>83>DEMIP?{t2*7n;Mr@Z9s`IUtXwL8O(P
zBR^(aRyn%0El>KDbL?323Lh`r$=cOF2mmOLUk;xmpg}j9`zJBz`5|P5)gi_!i+NMu
zW2;9M2DPb+S79ovx_9CA4RKU%R0YSOkC?R}mU12YQqgdl4@dOoy@kh-YkkQ$v6#gN
z)%5JuklV^URl!C2y!0M*X8*#_GIwn!y-LMUzg7d3xoXm7J(xbdHPHd0_DmtnV*rK=
zI^Ozt7)_pP&45njnf1zh1P>lgb+3QME=W#LwYucZ|B565%pSTC%!J`XVAoAXzTA<r
z;eLdNtJ#xwliDx6gl*FjN;m03C80fD9(IaU1}vK|Qmb`+)W=q_Ra1k~jk?pnUN{wM
z2C#ci6r+aKBRy$9X*Iesyj(Ehq9c-8z#3XFg5+t$#$F`2Nn;|_vWM;~>9@|4$RH;-
z^A$vuD}%R~f6gHzvrtE^+T{`9l-5g4W=1~Y6(TVgrDO96!&@nnyuw8&P-IpS73S|s
zCv!)bkxN>0xl)HlC1&<eNY|Ir)vx@%Pz4Y3y1Q-Y`6z?&+un9=gK;<g{mCu`b^HIF
zoF(F-xaq%#%my=#A~DIS#1~y>a84ct`bbKB5s7xzHZIf+m#$;wnO5wP<tY1n84gZf
zLB<>(<lb83@3o>u@w*0_lAup^u>Bnyd8Gtt|MkH%mrIel$T7&t|6v=O;|_wUkhH)u
z)M*+dFdKhb&hsZicqZRqN1j`VSvdbof7YIG{gd^hAis!0yMYwp@37*}OnIpoqAIg!
zVlU<_`jqeb)DXa-g_|Z1+Kl>)Xp4CDa{-ehpX}TMDpYR6cMH6^xUPt5&8tvee|JR(
zN0dgBdx~pCGRk;Ae5D&rd1N?J$vG5HMcsAgkNk%8USp`_%wff%H`%Uh%*n52P}LJo
zd5@yej4AY(@GKo#bYqfgB8%HEq``M<7~8D|4nqNQ4^I?%iTrl*5;Hzo$B-B=8dRvk
z)!=%FP$u)_S|%Kh=kuc{=vFoma{DsBe|0*3vJ3NPO{m(mGG{HVneokZ^y%&C-*gg3
z^WNl{Vl9*L%0bRin;&cY@Q-@pLW+bo7_IsDd}=WjP-r!gjTpigY*7tp6=bBq@XxP9
zxiy1qzX;x&Je;4t{+P)LEr`)tNiU8?bs=^Oap&`yGkqvszWb7S+b_|jVl!lUS>y#&
z=lMwyy#Lcol&7mOWN0Z4oy>w>rBN2ju%|9y$PgRZN<Gg_9Z0Y0vU~a(9uX<5nRrst
zEa0hm=#9W_03{ErfbQ59;%}xBQ7M>|<TGTHZhj}7l_l*mH`U>^4GP35?4|sST!`{5
z#w>SDPh|IL&RvS3yJie7SjpYr|I9U=>>=;Cfz!W7l9lF6?)okLJi->nO@EDENK6eQ
zSGuESyXjv`;(iv5bnh|3mC2>l5xOR=sa?y56YG8_Lu)}Hh8Hi=5j1oV-S!Wo^4vxQ
zJ}c+em`xlwlU<C)tmM7Yp3c7xp?n8#{B9MHFDB0w&uc*l)>SKw?<*1H9!|!$b2F+1
z=}~CCsnjS2Ph5@&qLH~6?K!GRHOf7q(q1MxBFOHI#4MC!H##ZRaX5V|s#*2rM6wJr
z0YGU8ELV?4l{GFVB_=o*<I8zRP@|$3Dl;VK6j2fjIPF#ngz^-RF^<*<V5cBGk(om}
z(QL$YlDkdf{QN;^!~nF45Hx?B;DwfTs9vKULsuN2Ozm*e^KH^RXoc~~n!bQ3&kN(~
z;X!a<i11sEC8!BY$8=>*|7WRIw;4mn&t}K*Ol&qMsb`N+P`(eH$`*&7@GZxHA^qV}
zzLeqvRcoqNZ$pq?iHC0_Wjw7?K+W5fpK=Kwc^M)+oe$IqN$cZ2{wo?_t3;=hJB;5b
z04xhurSl%4(PT1EWPUQq(JAtBtAoI%gXrI~I=_DQ21x?u@REZ0T1mWklB35D@_enD
zymKp`b2DQ}mgd+=T9K3BwtXm*5YMh&Pf^=9kKBTb#f>wd()4BCC!aB4Pz%DG=@b+Q
z!a<2<DZ+w@@bE#IW_TnYyt_ZVXcXzo`IHPY^K-a<OG$%hpF91|zFL4fCxIhJPLV2r
zaxqH0$$q<>`@HsyIiwq`B|w_2#9hE$lZ$(D+dV8~fn0Gr0Omh3iu;3Py6LZG*|?%c
z%kKgx=|@at`|pBJ#YubrcT<&#V{Yfh>67eF$do)2-Q=c!1zAP=74d4o<iseuYwM;I
zq+^+w6g5Xl(9EVp>^#Z!AB-q{v^ah{%7)I%xTNr-V!ehmdbuM}p}AP&^NU%B6_d_^
zvqm({$M>ep;@xch^t7a<usR*ctCyx}$L_TKs6Hw4c5vy|T}v6!^3EzouolkwKdZ!$
zJZ|MqO4^o-*U&juBR9=*M!hqhupbxk;f3SawAy?2m*}vfEP;v|R;$5@*jN@HPId7j
zQz~mW{Kkoz4XESsX#J`@-%5nG9^d|Pk%WUgSo~HW%6qti^BQxt=-N)>$jP&uJ%5@r
z`(s%&Wh~DI7aI-R1k|QeazE!&=Ou8>zZ<`wy2Pc67dbMg4y)JC;<scc5p|lPxN?jw
zw_HX*a$7Dd*Dr@KwF?+JQn1<!rSEJuE6H{_C3OxexnAi1!|WUdNGULT)qQ7<SW~tD
z07SSTw`8;D+gU93t3dt0yJMKUektRI_v5quH=gX26X#0kf3gvJ4SSn*XMX1WqZ={#
z_+Y+yij6z|V9Ku3?EifyzaQPs3!dxw=#a4H)G7+iIV5<NXWnPulKf%~l*U3SM`|(B
z5x|`qU07LKw;xJh-4<S7{~bqiOZ3T<O`bXho9C08*Z5QsfPHc*myVyN$g3)$#jnZ@
zw}><OQ@w6I>eQ)Ay_yx#8H&ixH5Y>*r?k#xcTssULUMVPI~{kOi5!x*dGY#oA6Nf9
z<dmebBF^pG&c^+hByE!jm_)AtdD$9t@6(-18e7R=-G71BOW6Cy0y2byTpJXCr_#y)
znq9Co4{`&s-+sbpm4<QVjaIJ6+-`Ey|7BfepJ(6fvzSgLlX);-03OTPQnxa#2G+rJ
zbT28JZ=g4?Abw^aYD}(1;sD`Uivvq(70MHbxZG2XrgK?}&YvXSRE}n?qA;F2!O<`N
zK%SGup3_&6d&n`bI*;we9%P=|L)=%1q?|Ue|LHKAjc7}wWAoYj;u7*#y-p)X8fTVo
z;o5>EOkv3!xs*ioyWOd-e~hq@u#+kCHjr@n6q}AGaASmlijALT<iK|1toxLoZ^g0y
zr5PxukLUf-udsg9PO>c`RB&)|=Q@_JT*997NxU=h7^PmD%(C`nQ1%|om+?pFH?$Y|
zy`RBx>Ni$gi{abVZ{YJN0GAub$alsPyW=>TL2q#MnJ$dz(^g8r?NEi$YWQFl4}YI#
z!%p*Th@JDh1W=$U!`2__a%Ia(7JsoA@^<s;$sh5Lj^Wj@BdIJhW#e_;escg{ZrjP=
zx~0h9dj-FWeW>EAz&Efn<C?5saF-su@?0Zqr+;DDtrq;SvL%MAN7%6XCyoc_b9Ug<
z=zpHY24OtXg?buJ9wMdF@~vElRNHpWXF91Ia(Arb*G*&2u0)RQI>NcMjSPRg5^V>J
zq<ig1&aRxslKld(eAkFp0D-G67x3!*RW#7w&D*zY{ye_je~UR|{hnw&-uQ7Qt8U$5
z{i|b8&v>6vV<s@S-+Yp@<C*{Qn|ycl3a=hKfNb!yc<frkzNDM%8~qv;o{wbtvW+ZE
zsYa(sU5Us%#+a{{aK!u+J5HU$c3=fpZe;VxXXgp@^bywPe&l|QC{@Dw(UfkYC{K^p
z@5#ZFO&3nJ@FT<~6w;u3a~CkO6ma$2ISMU0Ze>PNqqG;M9J2uKvMEefqBH6!Fcn}C
z)?Z4xLIAD;inSGi-b)I@pw=j{n2InMwF2)IgaI+cjNFw+@`)mhm(()dl^`5<v{CkZ
zo^jheJ*jHGr0Pc0-aXc=R|?2t<ly#AY`UsrVB4Bf=4Y$fjJKy2PnA=eaE06zJLt}p
zk{FjuuDD0KkzxipT~eO>9<8C%72iXtaJlyvM?NokJ!OfHW$F(}jQ;3tyu?<E@jvdn
zzWT?A>G#)D;u-n9aY96+JkiWf>Pv}YWlo25KYhDx_ex2wK0mI`+x6aVulTnFh7_^~
z{$H{^_x90~zLP$qtI8K;aX5qrkHI7Q?It(fvs`sV`-kIbTZ+nEgte4yB-cyt%HUoh
zhK_S82ms40#YYP6@C<F{SHvj*8C6gOwP*Dgjs^t<Svklm)uHv|>QojOxgk+Ur8b?Z
z>{LkcmZT^n!y=$t1-y(`(GBWLjed41Z<viRs(d&I{|z^hQkp3>_<Ka-HNGk;Q3GXj
zN?H#dqiJ=hb%cZjqw()V?Y3%+xrJzab<#D5zHAF-&wPpkgHWQ?Lb+iRS>^F8)qP!B
z8I@LtZ`r5#<kQv|3N7SnZtop0o*Vxit3wxZ=t2^*`aM`P?^RkwY99?qh35L!Zc4wV
zPYWO@7nP@`gp0Y1>aB*deQ^W}e%miFSOqn^zQXjOeW>7P;diBq3LV~Iw!%iX$%03S
zKf2<s+x_eC-MqOZr>0VrZ$Qy(I9vM+q>{H7tt*PRH>{!B_e<G*r4U{9=h!f77)_$R
zD2xwa*tD5s+I>+8=asI4I37(=`Rngm)<iU9VCZ?Fj0AeUTJ}zBHQxS&gerJ$Xcc-Z
zZ4~5N&}&_1PUWY%Ftd`1oSXuj4ZHKxCp~Cd`7W$fm=al2bPYPq<Aut;Po_Bxsl%5u
zE0b5~kP-<;JU4=$7u6vm+>_Mi{g~5c09s#}#Qz;~Z+gs}$pDoA=KX@0`C>E%4=;iB
z2<Ng)LxaAr)6y$~0GXYv=!T3QZO346-P0)oyD+#~UCNZuAYMs~!{8lEwNT9?=Jx&|
znKHuaH=;#^lW_05F~BsTl?4zguFK-grg9CTrOF8GSA~dPwB7+yx+)RVh_`HVl|O2&
zORuEVdg3WJVHGng$Q4pD^*fh%PZA_kQYTxjcK}q$Aekj2kGq3pJ|S=tm~(D^o*;>=
z<l-NfXxFx#Fjmi^QYaq@Vx1BI%H-QgACgwJ4^#Rzyyttv@y*PMO{T-E1F03L<jSw#
zvntMm7e@A{q9KXZ^S<DhYevG_^kICjmiUU<#N|fL{kVuZn~&oWQH7qbjG#p+FY-@s
zVs>0O6MNQ{5cZ9&%h_QmN7p(!#=SG1z1K6T)P6AU4H`ndvR<Sd|BWx^uO`E{EZtum
zLB}eAlB!>FjhDcPjl}J%nE%r*GKE9B)62tou1=(MpRCABVeR5C*?K-5pK2X=`{f=)
zdJ5prkjjoVi&(Wk3C~i^81h;#Y6R*qUf#*#(@tW1Z?R<g9>VH9!>fWMUCv8!Csc;3
zPCM5&FJb<#2Pvr7hR#Ae^Ofc4*Qqkc7S1KNVNaf}9)`-4&JQbo!y40!{>{pwuotj@
z#X{z9KS6l?_6&PzfMl0&e#25O>dR1Y;2`Rz-Kb$b#3k<*^lTi1QjDu~>=))P{h2GK
zGW2_67|+xUktDubyVvpcx@{y|gL&rVVf3kAx;UtvoBme>r6&oJ=_WVbrkktZru@h8
zoI3h5mDT_E@-$bM6Vu=uhX3{x!y1JAt^2M_y~?*AearQMBbin&@)5d(|E#&hJqF{q
zfB!Ryg5A08{sak(cKNbpe79&J+qP{Jq^wUkxc?weYD6SzW3n^T1c{)KoUClJ({sod
zL`8j6Flnb&Flg-&(r=t$>ZGMyS40ygOJ>T%$xJ<^qGkOu#Lu0=rwg`A^_~BGJX7Yc
zrE=3YgykKl{jd+%e9eL?{Q&#6#<?QQ3ex8JBm4Pc-&JgK9W{Nl=mn{uQd9*(1<BZW
zbP;dNm`hfL#?;c^V(f?y*myoiiZm@QF;R<iee()Njr#$4wU#vVxy7t$<5_rBkf<#Q
z%z6JEUfN@#L9>RKw@qc_ho6xvNM%cA&0y@BThwpf0H2G(#-H&e$4zoPbFT2qybt(l
zX%a0P#c+MyG)6DpOo~-jtTA%^zOiAU(C?4QC{vAwT;RRQAF=D?1*uQR4({T3a;7Ug
zwkehP9m_cxpCR>i|Foe@|Mdv<TDHJ>?0eeHUQ0@maG=F5VcLvOS(XkJO6fVdX(w}z
zrbu-q?_0}jgJ+W#(tsurNqjx!bv9;L$lkS%x87QesZw)VhvqPU+B?iUTkt@Qn48?>
zCO3&xs_P_T-|57StbYw(wUp|@rW2bOT07)#-}ge9hH4%9@>bo*$7mJ*v-TqQ*gQQh
zWWggf&hFfHe~M0@IfKz?#M>-gV{~Q963$FCIk9cq6Wg|JO>Ae9oSb+jwryJz+xEn^
z{qpX;e|zuMUDf?nuUcJIU#+I)=aZqJY+Fs2_QwfQ^0t?sYTjcnKF~?*GpQ*4l{i5#
z!TwP9r9iZi7=Vt$s#6hHiYpw$T86VYg^Ruz-8yTznBmrrGVuFaMZhygi;ezxKP1+(
z@f^;aVVx^m5^lV$8e3`5Pqz4XcwH49p+%adBo0!e(0$XbT;C_`;KiLtS*I_TS!Wq-
zI$`uJQ!K$Cx~yoR?&a1hvt$9FF~UK~RRWWK7oG1NA%DxT8goggZluUlgBNYKiNLza
zkS3d@{Z6*{E%zd6aLJE8L!3T~^tlC~il7v?tJ2p9>F>Z!m`W8f<;h{OvUs7qt{?_!
z@!dYXRX9XRyjYt8_<{MC92fC|<h+IY2OewaHyiHflcFVEAG3YMQoSSP@RuSChfIB0
z=2-XeBZ<0_#}P^wTP*Py3zEUEwZ1+63TEn8m=kVbh5dp^0h7ZD$zTc62D7;{$-NFs
zT9fe{Yz9os{DF0sqfgBZOM`0@=LxcKSrPYihcpL?1_Py@c`v@ejNGRK-8+<XlPC6p
z`9ZAGD<Jb(KBi>$&huOeAC_zMQaor9-6dlD_XBqQzOk(g-4VR#vy0#=?|942fux*V
zf!E#fT(Q!QR1<p0SLjJNyzzWud_`@zSWvX|0X_KvjE@(IFJ^|}_Pp-1Y^pf6k#7OU
zkkYVy^;Vj~@)lbz{TOAov8x@0KKd;Abg^!7-xz(4D|VAdo~q15A;~E@Tr`<8twyjE
zT`py4ZwM@Ci3v8l^mv$lEV(*K*GLMxJa9J~!w>fCDnYq{s(7v$+yv2Q>;ZMt)V(3q
z<$NK=LYi9gg)gnodOkGVX?+s3=pEfLnu%&gz74C2XTYB%*Aq(<xj6_^WgdINCJ#F>
z1Q0#a{VQLnm<T4^|Cwi1y|l|ljUQN*j51L9XH_C-mY6mqJC{RIN<$-QK$;<*e!qFO
z$&SFP3xHp}a(Sy5cS417JcX=ZR1TINzSun0@Y8Jjv24y%pSy4D!R{~d=s}`C81rHj
z#GU<UGYv@d`<)zuMx|6y<>VCq03x29T*aqx6-vadoejM-6?g}lzEhLE5Cu6l?H|D)
z7sV0qwZe|Ar3Lpv;IMxNwpv}uIW{P<JdGmRA0eT)PRMa)*V&6vzx<|n0y|)-%%Ne6
z^VkNPvkRK}q~xs{u5KDoQG?r}p}@LB=xe)VGFaZz%+Q9x8pfxnOy~PW-HeixThqAJ
z8gKVUMfS2}K{}?iz^gAfS{j`|n>q(uy)LSqVv^NU)t8O}n*awLhfv^F9t2haA!y8@
zyo)Zhh#lea{LF0Zer`BDTIFOttxj=>9$dJW=ZHp`$8P3)<b(1>|H+5<$<>YxYZz06
zN7><fY|u@W1J83v;a_AcHQmH>&T@oPoD4Q$ga)XJ<m^UH|3GCAuF9#qV!b@Es`xn4
z6$>spELHB6&PN85k>O!fusD!%P*YYQzB(-+VoxaTi%MqHRH0rnZ!V=*5Dc)YNU<uG
zM1}aQ&{tw~7xl+#Lj-y+sOBQi)#vF)yy?(|x?s!j|4qQ3+M`u~`7*Cw*8MTgl8{_`
zrj1>}KNat5om$zv%jw+JxrM^Etwtmq8UQxsXgJ^eV4IG{3ESMJBMcB<<I|0YA?AKC
z+O6+%fApiXY&}5ZL_=cBIn^Lew!Taw(WcGak#{p9Gvjzl70I^Ir!&&6PG8*Dg*|V;
zE^qe0fN6>`;B*)NR0d0s_67!lU`<LnDkQck3gnKYZ$-)FMxJ4v@K&NH0!HNsl5%n@
z5L3k{^Wzl}%@|;-$d7~T*#qFdEKcq{!3v$G^CeF21ui#jL9p&PJvOW1_h9(^X*Y0V
zEwY!Js7RfVfKHJ0EuqFKiYcT&LD{A=-h9@>;vEftGH($$u7vGbfA~2Dq7vY<*{`zD
z*g2kK&N3N~X5(FH7Ht7DIwN>-6$q=4j&nC9EJHY$Du_^w?xY)ch8^Pe#I{(Si0`b;
zKu=<Gu~aTv(ou3#V9$jbE?L;o>O>h_7>r>Sm6@;Qj`nTB5&Olz+s}}F!cX5dkurpf
zPRjXyLySRYeo3xm_jk7tT9H5LK9ht)2FRcdTxSf#Dd?%*Cr2?b6{BasQE?oh%LxBd
zZRz6KBsqi%)&{Cx4F5yh&b!rbXEe}GXSZh>s@0xf{G1!nAcqvvPD&DbIVDxLtn85i
z+s0sWdq2n{8pTpghyLoCZ=p|K^oN>U&UFNcBHP0->xoQ25dDB^DVh#hLUu3=69(rO
zNQ)o@HmpF--!ybr-oj}DBno|kmiTKnLCjBSz=kPihao=#X3+Ch_#8&%cb0e5<N6{M
z+6vZkEkXf%;F9UsuCZ68X&(030X<(*oO4E`ZW)`rQ7UNf7HwiF9xl8OOXGV4kzwgo
z)Hh<qDmO+Yj&(={_rwbmHpSS)qfR2>2b;{b+h8zhyc7T9een>`#>76kbLeIRRB)6n
z2R8Gf%^<AV?Q3mgXj5XlP!UunS_c20$EO?1*E>t|UvdSLMHf>xz|=V(Y<4KQY|t7=
zV}np)zgQ~ruq%oVJpSy4+xHmqGvA_D#`PD|TY(>4nPSK{*Ke+Tc*;zTzpoptDl_`M
z!TBkEOqgc+?vNJxJbpLPKbMJT0lC=1Q_R_u9HomHp8Y<uny=wu#CuMjUgts1uZh{n
z<!D<4nOwn|jV;;0W!)kB`X`dsEg!ON*;vor<+;wUjHTRCPzv`JBVmbZCOQNZ>HwY8
zuv{j1IK@qpcBr9tY2OV1I@xoxPX#f$yxcJlp!-1ylfSCE#FGuq@ZL#_`iAR=TyVaA
zn0vynS0$P_ZOg$TsKbv6;>=e;CAR{NcydlOnfUrevno#1nyDRq3)8C^#e3h!g!Sa0
zqxU*)R8sh>F`Y?yhlXG*bO$jlmpgGq-28_)b2be;R$At>X;;0}Vpa|*I1<QOo}i$)
z-{`&(%b)7)QSrCQAG2mq3os`K+O}~qUs0rvNXw$nQ;9a5I6{_|Oe%8tB7y`a`&Q|C
zwWU2SI*4y84n`cZ+^?`K2Jm~*EBc?~QCh0-o^rnKzO`PRmNon)j4)eVwUD&G5Hk!6
z>#IoFZC^|~)K}8Qtxps!f+`T#ywfn_v<3)kD<KWb800o=?o6v3C-}lW?b7Ki_z{u+
za^t^7-;xcD+FM#mD;_DUPEdz2oQk&n6*t4!knPgICjX1~C*2yV1-nqd+K(;ewp7*E
za#S@nW9j$`JOc0V6NM+_-`avbhbm}vzL~W-MO>RlU~^T0Mvjc7+6>NHcLq|tuv=Xn
zQks21dRyp)Y>jdR1CN`G$XZnd$hCOUSuzE0H*qOkEgW7CPH$P1)T%#ck5uE~WE|33
z1ZY%$VqvSb=c2CL+e=`%gJ>Szf1}pbgtiOk$(#^Fjl`dy)E`3B$eKe<3KCM=#m=TH
zsK<aS5c=N&Xg#ftb}^QJAMst?Jamg}g$tRE43n_(;etR}M8i;kNdTxx<+mz4SAL<{
z_U+}M!tdr&7A^vAdg(1PwtX|$nKbaMKY~8(37qi4dRAO$0LuKLEJi>5XYcKjP#h_l
z&N6~B;UEo3tak#y4+Li4ps4m+xEWYFOJCn_@RxDN)!2Yxd4<9J>pp=$xMfQ5l^s{H
z*h9?21xxD2`09CIm`y`BW<@-zAXt@YAj0?}cE@yi=iK~0yijy@GqjjYh;Y6@?OEi<
zQAlTh;a8581nEYB)ak97!fRQEMv$lSwLt~}Onx|!g*`syIE?MO>X*r|*UY6ZkJf6H
zhcAV?%E8P`%e$6u;Fk~Af~1jE4Md>GTUk1mhPex{j$?UANRBtefLuy=H3v`BG9H8M
zp3j`&%Q%&KBhr(djwAkDdj4qD?XE{)TD16_H+^!tFc)N6qBB`GPY^dmOs4~Dmu}3o
z(HbiR&?9@cdDpmYS@L~yR5MO1%Fnm#FB7_PvVZW~2$Ocy!;WHE7B$v<(7E1db}d#5
zlOfE@3fo?R#op`O4N-wRLSfGu{HH^nW4m~r?rTQuNjrF{`=>YNH}?b6<Sm|*YKs%y
z^Bz$ucf!$ZDPmR^>mzcu2&Z%Sj3i!D<c&VOC4UZvCpJa703+qQC26L<dJ=3<=ay;Q
zf{Q;p)OZwGV`oGBA3svd`}P$43ia4)VN9?4vd9^8GRH9*<GP^3x@vf-s>$t3M*$i;
zf_{~hn%CzwjM@#nktybeX&Qo9mj$alrWdH$RfU3U3DP4~4#3Sjx{Ig#XNKp+{^&|z
zfXlW8|Jx({_sm&T<r?ZZbAO?<YyGdr9b+23`g@{4`SzJSdW*#N{vsPjMdua9*K1{8
zmdU>Bm1IONZ`QaGH50|d)vLW3WaW1OZ#&+T&dl*Xu~o(6j>GR;G6N5m!qrb-FQT&K
z+8F=>qv(E5M|gJ>lRGa-I@xW$Q!Cfa?Y^))*FlFH1?)zxA0PHV-{Zu0KI*$4&urTM
ziMB8rFj(Gh1bc2^YBX`y<dCgP5$^#h+4+Mlf5T<l%0u$!Aop9_6`2~4horhMrCa)b
z4ZDVKziGh=RhB&_dblJnE#J(jC9mQEskDaQT*4RDXQ_N$H2$hxuMd%QPl1&LTVjcN
zK<CK>$ky50Q)qS<baajY*9UAan*!QJdVu#W&sz>F<lffIUo0fCtF{k-Gxgd>d|wc@
ztK1sIxXXbDY_<M#>EYc}{oHxJU-RQ3y;{P?jPK&d7<ze#WWoH&b@elv21P?Q7$Fq=
z&+FC7Wlqi_K#%TMJ2->}!w9lI5;H@99ej*4?<P9XiMGzskxX-vL6~3L&$u)KK^(;N
zXJHh7ZZXROq<Xc??G`(sM=73R)QqCQbP<jxlMTP<lG2%m&AK@?5MWFLM573#mjOE?
z=Kt!@+IEA6gCFjI*e*{NTg@9q+l7u&-3+|UB#Ok_AUowFw7!dqL0JJXD!-R~*G%v3
zd6yGXBgM)xWSr9v^gt~Z$ItH_Rd|#K=sr1g&mQY#RPnzDFpsx}PR0BMuzLDyO7Idm
zS5n`4p~c{YM;D-WnXz)1mz31i@F=anDSi7I&LRl9HfJb&)RmO)SN;35J)Lsjrf8Ve
zL%B#s%_SB!EM(|vFYMSMKK^lkDHmd^N*C#v(SIv63Evh3M#l_b3kR|47s-b469p;~
zk`w#=e0+h9Imzp{9+j{`ekN>+w6+x}9A3ob>?-Gd5y0tY#RpaUtSWmY(TJ7hbTo6G
z*{+PeCflj-|BZXlsgWYT?e>T%sp<eV?EPS{t&%(4wHxD)-Tu1Vh*o%q{f9#X_xkNw
zF21!f&Zw80^>SBu!Dj|IFaR!VV48obv3VmCMDK!ggKc9yZ|EShxSZSEo>uY(x~wAx
z>pwSvvq3wDlJ{r#aB8kXdT*079r3^BTDFDy01KPk`iz@+y>DeuTF&+^q?BVZ@mamG
z;5{^7h6n5#NsHFVr-V{v(DbAt?rV9v&h;<X;v>qHT>oZ9X0Q46wE0f?&vUa#fzIr$
z33%9M=Kf(Vp0XX<_M5!BSEn*G^Y3~Uda$@W4kmL86U(zdJf(o9<-FY=BzA$@QARBZ
z!VC12LhwC~LkYzbX0hJ7A=4UR+$5>jJ*-ECB<b&8;!SHZr{}GR&Pohd=xHLT0QrEg
ziL$&rF4{D7N8zKF$H=XYE&4O@eclTNI+3V1M>kk(d|5=``N@2OVQ<1X6b(_4EW{p9
zNqO-u;8vnJDkjfc?v1+9<nC>8Ltz=zo0^teQs8c1h=64>hpmv+1t`G<!gY|ImC%`t
zs{Q=OG6)<#(T%>j%-_Jqmtc6&ewqLJnY!ci(yiH;1WR$g4f(aLqq{gnd)s`h&*)pB
z`LR)_mXGT16BiGPw?2Cb##!1{goaf29Ob)BkJbY+^<PeqtmmZnQ!YVXM#w&0ChV*t
z@_>M9@H-G&LcM%6>FCpyDW}23N2CP}OnHmCm|u<Aj9d4wAGD`JgMRqm+B72~7#ol?
zD~D%CH}<`hzpg<kAgdkTt1HxN*uc0xwm!^usodN$1v!Frl-*|inZpE!3#U|?eBF;R
zZ1UFzR_vbJls169XproMPYUn(+;xtXtNrJWt1UAvH0&g1oHq%oI5T0XsJ4#-!&IE@
zK+<E1`jVXpX>z<hZLV5lCPQAAz;ywvqem6SdUe*4-poSxCM-JwUc|IXN9hVmDk3m%
zUEV=JPRGHzTj-H8d!Xt?nJcy?ZC||z4P<Ufz50!*ESCYcXxARDXOj}~s`>@RB!hQ;
zV`(xtF>U=eYdO4}gcxe806by}xCWg?jz1J_Ew!H2y3%ODRvGoTInsuyLGVwKD_tg9
zOLg5?`eo@#5anL`-7cUfu2q_x2Hs9u$xu#O0+0MD?UX08?blal1TA=%_YEpnKNrF`
za`08P1Z`)1th&I~pIEb%aa1dC9`WWQ2L`j%IkfySGrxoKq}8kiR+D1~D&)0@COcUI
zBG(5cIsLI^)W7n?Dkyc?lD29(S1;`aQ>nTSj2K*yk~g!<P}Djpk3ejukmGWzwzhrO
z!Po@crnC!#S*Z_G|B>^jUq>~1%|YbLfzBxPIG~9?_vy8JUKOH(VQaY=X*s%(4v_(I
zGX21oV2uhM`Kl-p$R|s>T(3#jV!DeSlj)Tp+J<RVP`}1Y!}OO{i{zB#=GRxYgX`v)
zIN?}opsGpCpIbh1qId?c>UCL?P9j{jILq`d{Da#CviJsVBosZtfwB|M)HspCEgFkW
zt&wN@$+iICxO6l<a4i2T9tW%B&}7Hi;e#f?jx(Ss>u&H=BOId_T1n^O5woARCi0Yo
zZn%llCRU|V3zCfn^P(n3#7&6*UYPZ{xU?Q?HXyeKcgc_Qi&_hXs25KQmnnGC83y3y
z<L_MkI$EPmu)Qhqz>Vi_0owM>2+_*@w|*BA=p8lEl)^zKiJXgrx?LA<Gd9A><N?_J
zKT`R>Qx(EOVqAPqP|f|A(^^ctmC>4#@^6KYUIHU^C(JgsC8b#w+I^hp7{ChH`s=P2
zE70bBqdYhL#%rH0W_Q;p63JmV+0R9urUX2^ws4doc?*a52kNzEhwtfRaC7{JlO9kV
zm<ej23Z{u}oe`;*hsm|m$5Xi&XPY;4@H$Lg?2G-2_K%bnVF4||BPIuhA8ai%1#A9t
z<T%k0iw4HAdd%s646HOr<Aw{e;}KrE^n;w+eoWVmZ@t0}$Rk0R_J5aEfNKl>PW=J-
zhCVwl*)5(i@d0Qn;(2ph`rt#Jh9s6}6L~UdfQhqmwX7m{Y*_cHZE$je6IY#mM#3*8
z_U%{D)lLu7txF2r0YT(%AqB)>TrkR!`PBuAWLad-4~1PM%jV96F;5F#p5T8bP-pc~
zb6m3RI$0lanf8R)x1>kC=dB#hb+c*BrLAAQ4<!_#NV(w*mm-VaPJIhwPF9ZPyt$XQ
zdt*rd&@a(_+#|v6Y8Q9z>Jlz9qPQCz*VVNW-ttYC(P1!oa%H+330F@Q%dc1)E{JM5
zygxUT*>iw$wUilU=2&7fB-Qv1pVK|*J4D!!&KiZ<9G@nmbYopluo;<Kuo>sX4cmJ-
zp00mSQZWX-{33@E0)Y=g*v!^VgO;dO$6>gcPR4vXKefVmSn4m@qrILV$Y8s5v<=Qa
zC%ksv#%i5okjT%1Hc!hOTIkSHr3lTH!7Z6Jg5bDua`f{Lz|kh>>s;-*#n^K!jz`nq
z>>FhKu{Ve){2P{N=wmPCRZ$9VR57BzOkjImKhNrQ6#5YIPdV)!iUGX`h-sEO)4<{9
zDQ(uQm59YcT?(lM>JV;I;+M0uSwT(G)JWXd@BUmx$MQKp>OpC0@AEHrMjLKT-E$d)
z%!MkBD1QgZp)lAL7Ew{xH{AST9AczWdt(0zL&yVQ(7s71y3trcv3KY2_m?QW-<h&v
z#wr<75HTU$<^2r?!$5oG$~NS{gqbP&Qh^0=3o~h{fnvCvA=-zC&(oIjx+Z+8E#Pyf
za2`fdhY_1;+siX`5F=jri)^}vgfXHo-}EOk5C#8?aeQ$zwR~Gw^Ad0=@`6blM}C==
zBoeMCm0fl4VRaTxq&Be$oy}j;lrf$x<&+st)Dm^di>fVP;Pdqk<+meN)Dpc@(qB{e
z4J{QGg`6r1ShupZJtG@uF6&Uo3p1~)X9>=<sQ%$v{QhV{<0WfE_EO`D&)jNJCGjs9
zK<HLM#I3UJVT@01+9^7OVl*5f6iy$PS9U9G;B^#5&ad16V(uKym?z7E;MK~DV1Di2
z6Q~j5K=m>QGvuK<7?W&qYPJ!(Pv*1X?)<qe<XzHR5lu+(x<7Hc`-bug@YY4>Y01xm
z5J$oS?8nfAEXK=_kiezB1l6(4cAwYtqqZ?CN)FN>66T^p5CWmSeUfeJxTxDt<O11r
zq&6&6?5H9RuN;0QhdO|>71I8gIlGCYb+B_k;xCk!QKe)BFZmp%Q<~d%ArI^R*7DtN
z7L~#j@kQ4?GUfw@7>`wY(HpDa3CYPmFLXHWPa*m>pyU<aOMmb`uX^og(kr}t-(>ls
z`)qi=D7U<;w*-Z~l{HW}r2yldL63Zkzue?(kn^i$h#^()g)0fyMAH>eg%N#659z=7
z)ET2q6R-@Uo*GEyO89NRL*8n70H0(qW+d&Cc<%}N;4*r0bcyWjrr$oKk|ULc29C~d
zvuYFU48|->A?Ho1mixwc(B$X82B9ZtM~G8<69vS=nAzMgvRcewVy(*VtanbwCS~8i
ztn6}jCL8TUGdbNEYlm2V#4j92Lk+sbT?`@`UWqBBHsFud5A#ijFNXTh&Qsj$IJ!Yr
z{aTrj8BLsGSkb{c=&EcWc~sE5hk3JCH=V9eqOL-PtQY8V>#v*BRwX)x{mFd%-Znv0
z!0=m{dvO!_!HBL!p7ND<SL~;wfR8^6?mcra(c<w{_>6!<U763hHPf#D-P8RPfB2@=
zxld6?T~&Os&qXa0P#a`Xk1!c)f6VsK5?`k1#TW4O)|;ueeRTGc>|61Nk`pb<pVOuu
z;?auG55wc|>fXa+3aSOHUYM`r!dn4Y)l}L?$=!YqtCV1LU$7Wz1m|_^56<~E_;8Y!
zPcDT+6TNyX8dw@oYro2)Re|wv$S;mVpVB<^l>BW+`JDY{x2fL7X!0b=(=q7cP)Uh8
z1(I8tk=6awh795LXVr7GskjWQ5Kd8k3W|`3)DJ%%iX@2NVwXhvH^X|$@l=LkvNw%$
zTCwiy@4Bx;tFszksjpu;+eo=!_QJ=QY~N5ihi!(utL8-*e9Y0wHT!*M%)<_EbDL5P
zfQ3FBeBzw(rpkH`N%W9^=%$&jaO8cju$&VO3UBh0im7gOAjBTJG<=|7EutNok9jY^
zXqL|$ks3^A60{w^0a11+2x4i3RyrzNTK#2X1b-7f(VBfQAute-iTL_rrtR=~xX<Ho
ziysy+5e91dB8oWiRZ&IfzEdvAJEjpr1e{~r!qBxEd#@|F3zil)P@jh=8xr6cwWIM`
zEkAUAWIwg`!V%6vDM03Qa^DijhYg47h~8d1xt<Yj#_XRFE}-FI+z%8EMf<Ge#Lh7c
z?kr@at=)cnNjSMBQQ>(*EJ1y*4Et$slSz<NyI1);7YTVb!-nTGNP_%F$`FpCE~P@}
zlhS+(M-G=GM=5S#D5F-Q`6eQ(tYo<!oYC(Fhf+R0+onIH2ZFoV@d>TVrtNCL+4gdh
zZy?5n4?T}GrUa6@h$tmXy>jbvT=&=&SNNA#PWnn`n#?ia)?Ob2NS(2i0HR)RW8LF5
zGbLCWs{A3KEwCO57B%`^67W665y)QVPsYp`JhDPs9Di&KLgEVPQz_SK2*X8Cz;Qd5
zd%;0Q!+Qa?L}1iXUwkWI>+tkal*pZ<s62a4Y4Cmr>sq<Rn)?~<SIQGPMhr#FwCBLe
zhd`rFPPRvO!pKE^5jWhLg@B@1fbHAXM6P<$8XL1<KEUp$|41#vu^VemoqLdsX@ylX
znC=joAn+!xtIt%?`jD+xp(vHA3Om+J^iDO3sYlGOCo=d`qy&>1Ebp|eoIggsg4?zD
zZ&7!=7qAza0coX^XI_TI_G*NUGJ3wuZ(FxE250iNz`hP^H~Q<u1-HacVUg|xQr!rv
zN!qAU>3~s>JFJMVkA!+3T5mtifX;`q`pA2t7hO79KR!VWBNtyIFjo@37_B<4<}18Z
zBK)u(JRRHy(gCjw)uQNhSc&HJ!rkTWpc1q8zF2#WIPDg45pQ=QX~wRDRO$4(+~Bx!
zy@nKAc?BHTbCEWjt*^<{yJNrFYWhWEy6ov!opKMC+AzmQpe5ynq*Z8YWRj{!v1IA>
z@POvfBQJQ;eYLRL(KXiN7|IFnyz>=^6~=+c_<3miviGmef%KTpq2=@8pM9VD3_KqI
z<F5UHCWN=K@Ux<t<Fe@;AcHCb8hGq5Rc%V>W?c-hZSpVSY~Ti87K!3{JVewOYa5Vj
zU8>R@&DQ%qSE@dXsaXr}<qqli+Ap2o8S>f;ProVlBFG*&c{;%eJZ)!(&B1Nf`a40y
zAjWb$<}<H|72`7am!khi{yq>;#_itYz@+`U3Xk4D2nWW*z^`AV+8dp{SNOib;JL1^
zH26GTfyOd;k{$}>_WEjee#zy~UlU4*!ntNv4u$Uwwiw02#J2pr`D9ns?jY{yQO&p0
z8wNQy#nH=FkerdoY@QAuv@vFX*R`04gc;-Y$oPCMXLcTkQWmXt@?_0oNFGlbWDS0X
z|FO{?NiMNnP5b+lR`dl~?3fut&iChoag~y4{2z-^_njPN5=>NZ1I^xe$6hV(*#N#X
zw73vi<N4?l&N&3-Q=DCW@fiq*d^N7+7Sr!R6y+o;ho3AZl#%JoX?a`>-sFp^aWPpA
zAqp0i+Yh?frS)}mPe1T>ifo1KCQSwp?3vAvFlNeGj<l%XBG$yiyMHcAh!cw~1-qtQ
z4PM+5@+QzPo<5qM8c*ylCcA&fE$TrfJflx!c#LLU1W0Y%v8xn*R`X<jW0|aW9EL)%
zODS-Ijh$ffe*bkkz85_pQaSxMlxMKLGDypl1z7<l`EHBs*-jcs&zaNN0lrrpZl?^s
zP14yq*owg}j|tOLOZ`Wp4w)%BAZ=qHs>24G|F5sS`8OgaBh)-fS@Sn`rF3f3)qOEI
z|2MfrJkDS9Mx;hJ2QI$u{FEvHskP4FuwH1^?Yyey0tYASJy5*CD=u~}pX8Mg$wbQ2
zDNSJ{VfTtDc;lRkEEY?Q*{WUWa@KO=5B{A&Ig;->^7luC*j$DPpenR)n|pOuV-6bz
zm=VLtJt6eoEvdF`tO8A=-|-B_f?@1y$$P{1o`p8BnPLbZ$Y3wh9$kAx`L*3~?4!au
zZU(Mury5sp_P)xy$ya1emNEKG(C5Xq$3<W>--+C#4_)5f^qD+x!0G#=dHVqbq6mf$
zY#L24HpW+3<EG;%otKKUFNNvs?|ifHEu4ZyD6{6B`L{EMQJ?Nc=<)9HYUfx+OpqbC
zU~yVfI?4`MEYC4m(79t5KHD1_b@gkZRkDBPjeN#iKcvTJZNXe|dHF_W&xN=y)^06V
zIxG+&At66|P&;02i%nz-NOVBm5mpqUb!5it*IQ2|Kbu55cn%Sbh^?EB>a$Lpa1*CO
zgw|y{QC<ESdCE(BBqL89uH(-Uq2_AXyqIWrJvf|g;1<&PRc8Q3b$5S3x|ljzkYVwg
z<Jazh94kCGV6&|MQNOa8*|F{6daqwv_QeascchF<dt;cV4Z?%MkTR%0YXqP2TDxc=
zSZn7J?~S5OA*<jbxw3uZ7J6gktT&C(y=OKNG*HII=syKKNKGWakQEDw{z&L85%%>v
z-cF48a<8gsmdT()q=js4!=?XbiZZEsW#k{MIJtq^_i!Xxoh7U793Bu3ce;|9p}{z9
zyL7^auoz4J_m8^BM!xH!dVInriTz&(Ki;hF;#!t)z#i{ikBlttGJ=`74aKcb(l2+4
zn>iK0EkXg48$-4!W`JajtnV+m*s)LMoI0eb7uQ0UBf%&^9(0KZvz2q1sUvE-M%mof
zTZ{c<;`pok&YglnS`82~Bmd}6GQ<rZZ@3DV6rY=;Bu;Mlzq8bgsHUA_hq!#2ZI+~b
z{69iwW5njW?QgcS5rT(jG+RY`{}jEjKAovsa>o}A+uVd6vM0ezF7EEnIIewqQykBH
za)!U9Z;ujB{jTZ1s2*{KE%G#xjQNPACIuZZuoUabq-%3c<8K)GbQM0XFZEPoS~xyF
z6ImZs-W&nHa{E7mlj1(cHL8qX9t~D+Oj{1?n(2rXKdpJUua<DVLmnu$bEO9P>lRo<
z9OH4nKts3r4=FIt?}%qyXr=#_ivc?|rNQJ}*wr;2eO8Od;g9|W;)^(5VBYZeqe*is
zP}TrjvO=zSGQQ=hk8r3ZH)YEr;1(DEXr!VNx%o2dS*ZekJKToN`U6S=#eS+V6;WM*
zvGlWr!9_zANx*m3dl`^q*>XfoP+G3*uG7{WF<t`?i}k7D*KO<T{@9z{P`4?&z2^U4
z=v`lV6Ce~HKs3#74`-8NU}ny%$h^$+WjaYJMG7%#X)-A+W>f~9uW=<bz^vaP6(x9e
zshcQO)7gKSF!_3uH&~RP)$6~+(wg>fFuT>>%sv|Jb1swxPok=MNa*z{m0-j3HN|>g
zB1KGU9$jt2rgL6@7IZXG_M*u4N~`HM3ih|SvqnV5S7-vc3*HoA;3`#9kkWZrflrA!
zv<6Ay!!_m4$E4T5PCV{(Mlj?C%bY-A+TId}T-24_-<~RUM!vQFPYEgBb<7*w^63_w
z1P?myqxosfNz!vhxy3HF#|g0Fl`h*yC%@DMp_>e1&{unk*;vB3Y~rPv#|ZrPtY@gK
zaB<E3gZop*a?*v{MJLJHz65DNFwfiMEmA85s}>5eLtb&*kl7Le_VCZLiV-lX%A}7?
z3gGBs_Da<DD0Gs1CMb0HUna1xEJ>Y=(iAnQT+8P2a=Rw+b?|GVjJ0qI#HeJ#luJa+
zM~K<Qz=)!Bkjj++wunVSZ(ct-;%VG)iuh6o8f;LHD<rfGyUVZgTwLJvwb5&9)k+9Z
z#6jxp(||MhId9~{`W%-HJ>pyrb-{jaHtF?=(w3}mH?j0%GGE4ZS_3b%fDhISB_UPf
zKUZdP4Ga?EQ5D%PH6S3S>%}=~a{jPvy+3{v9Ch?xRy&?$^2Obpd`b>Hg}cvWnJxhp
zhu!Ci+?1v-vju(8NSkgEffa98jBr4l)>))(8BTMG8<M@rwUediRqaU^<=E}r<S@p3
zPHe9Z4UldqMm`S`w(X%^LbT%&i(?VCVaxS8Ro|a@&FG0Ii`H+r$^|K36RCK5&J;`G
zzw+u@0e|#(MDW8&Z~br@@+sb^1~2G*%;Xr<-(IG`Ke}2NjlZdvv2}|3bf`C_{B_bO
z!`%6LcH|-C<bl2DnSai|D#Q}k&R6{OqJsgseS86jAj;jI*%PpwZH1RhW!sYaiW>&q
z+0PSFbTViByx=;fkwr|m50B%`{qe<IYh1im*yQkS+h%SChnbdd+RH<QTU0Lc4^b0?
zUl!-xEZxZ^9{Oh+56U`!3hvtP>?XJUsYHw0^?;j`oEu?-bXJ)>B^Ww=VxRrpkpmU%
z*B!Ix=!}ukf-w&Ye9|&bqV#@C2?`CFf@0X6WUeQ4zgqtO^q9{Eh@Wo=TF``NXB4QL
z&eA!|qMUnF^1(g=%XD9R>tPM$B5Uaal37m*XI+KxOa^5Q4f!BB7!^phd^EtsuZ4L^
zjSZmM#G|wi1kKS`Y0C|qDa2cB-mG1`ROJWxhNJjY5m4M2ji!p(dHMW!!L?P%Z?7f&
zns;oe;!WK9={pHgTX}E$f=7)b8E8KXdnSxE9F<lJUC?rWr?d9$vbt?~pXdc>lGta-
zff-by1@yZa%fn<OSQZu2yEt8^m9#&fs6l<ZnV}pB<-ek8No?bKjmNk-3xP313ldH9
zcsTf}aitv_VR)NaMql#-dT>9|q_6Uaeg+fp0GuVpfKNU~O?JzO%>2;><3Y+)<rKqM
z0BN9bBf{tIy-am#RFC1-4HqFNOFCiHSu%YxJrWnq3n&wqiigS|mPE@NZb?{-`dnwR
zV$<$8pEPVTFeU1V&|4tDBzZ#-_sQN7{^n@KL?4J`p<$ya6EGz$Tc%G}KATF)lZe`O
z1dBn>*iT8@ncYyedj3x$m4L+cpj8F|rmrr95&+19_L@Zu`t#V{(e81Skd(r1gMrWO
zhU8nGImBeZhKI>lsO@U=bMq<d(M<;dUf*sD@SFNtpolgzLL?d0z6wm6Voi7-Upm3-
zj;gG4*tJdEaN#E#4WZa^=6XJRv3QEIQoJqxL}5jdqDH9*`E+6lgbK{v4cX3LZsT7D
zgGOW9LkcgyVaF6)2Ke0hemOJq9j<mVY(%S8n!$j{ZC&TwT`uMghf~f|In<o$>~6M7
zgWfYNwdliNXB{m??OEr++baojT)n5<Y3%K9?`4&hSs?~&$PbQ*Y1oU!J_zKtj^)a=
zKHC$3(Xk-5XK*~d|GDVfTV>9A$yka|Z<l_7HQ5HTesD_cFa&WW(RSGrNtIB`n;+xA
z0>L6{RpK}e+PL&yTnYl2qg`zHzxT&2J+=Ia69Yq(zmV*qEqUU0qfffVUs9H@Huc9V
zWKk0>e*Wd_qD(>&k2;I5d6H`H9rmI3cV@)XNPlARHsAea#ba3|Z1kEYGwH>b`y#qj
zT9a|2EJ^d@hCPQLcJ=;4$fxir`ul`hV;E}~@Kvb-@ohy_%tVfye-Jim1iXd)r;>g2
z-p$&#kjV5tZ{L;bmrFAg=Zcv&5<k@t*nqn9prch|^0cDdh_0LCrLeJ9>u!m%_6tEL
z=?xd@9MGV>W|;k4mfyIH3B5FFjbLO+UMz~t<|-llbdL*_{RYuGorsOBX#v<aKX<%%
zl135pOGW1H-k{@th+J*=|8JzvioursqsiD=U)zUWu#v65S@74S8_6TIjRn3(7o2DF
z=^K0XvGEKl061xN$&f$0<8J7CsrLsJiGwLzxwu_wm7*I39@4+kWnM<UsFo`w4$UN3
z=`^s>eCZ3;@CXkfmV!{L+5?On&?Ua(14j7QgOI)b%_ex>xKz0V15ZjkpilDIzC^8l
z&z-m}fWxki5X|7vk|(`i6Fq7JRA|R`dSFy_*zfhiFwRukk@7zp;=LsTQHerWnXp>o
z6KQ#|Trly%bI<^Qh*=TMAvdb?Y6;*nMF7VQ->`du&&~V^3Qlbxwmo8hy92gu>Yb>7
z6}7MThrs;=IDho^Y44;y<8g;(BHe?&Lw|r=T{luIcut(5aKM!CHBcUocur9m9%+^*
zIDokj3w!|bVWZ7-`B-o9qBA97f`2%cuElV9?557gUB}$r+`|3+WrFA82iYP4`AA_j
z;|-uc@rcC0@7khkgwH=k$Z*_v0kJK_U0!C;zJd8lqxb2B)!pf74z65{+;j=0#Hv|b
zjXgJK@y9KW58gq?8<x9lx}>fJG{QJ++D)G)T#w4Zzz$43RYs|`se4ma0DA&VR~Jaj
zKYdIN(?GhGD=Vv(;b2_KWcq7barxa}m`xdBy0%w>(t5!l4}`yEr3(kGr5t*TN_zkl
zH<?nx1grfe$L&^LHXtp-TOyhb8o874{Po!*ws>|Sj(aenid+VC(qa&^GuemxI}GpT
z0qydoBcgDpu(U5s+liI$%6EAzfxveKfx&)Ef1$v6`5Z}}P8VF)#5hhHOFZ|rYaM|1
zV4b4`J!)l3Lify121Ua5>5IdWg2nZrZJn&Gha_L%dctI*kcNYIm8LbOYIyr6S!J7$
z?mJRJ1N&T<MucM#VQ<<;OIDq_E4lu+Et-n-vpuogOm`ARJp+R02%x!9pA>kp_aU<#
zYR^#^h?4od`N#WZPe%HS8YZ2Y4z|aDTKeL+`X*wTB2}~`!h>boI`Tj-yjT8RUct8=
zl;wo&hnHL8A)^n<uEC-~uh`Qyr9ad~nKN5}SIJ*Y1V$KrX<cFC-KNi`VkheywIrS+
zjWsPJ@^<RA1bmB!)F$KC4QOi6)q{nz4H6iR2&@Bw@H&kN$Qh~f?Xz^EQt-g1{^UAm
zsoG$YkwX4X`Rv+IcG^WnWT)8JxbW#S$`+aLjz6^<%UAy_7$aU~wEo^N&tWkvayIA2
zFrc?luP9(33tsyz)E9qYdX@^WiiKx-H$<$eE6l{6machnj@zxFBefCDV(jE3b#-BE
zTO%%CWJ)IvuQAMUT^eL+YKCN1nw2a5OO`&kzp_GSDN{rAIDmMUlgNKhx>3DZ@^Co<
z&49fv66RK+PN_Z*sR@a6Z*NZ+OY?B0h;W%jS9mI*TW{c4t&G|=O^l!oX{Tb~lUu>M
zI{9w(L~?kXWZ^K+@<v<zAi;R}TtopBQ7f&q!G}V<UV~aiA3fH0DDEn=PphfXv;WGi
z+-8bG@9P^0r3;Cx)#^s;=lpf~?Rkc4gG66Ych{D_FEL))sa`Hr3KIwvd&ILc9nk6b
zJ<K({K{#F3ld89Tq;;=u;S?>JNMW&L%Yh=K4f4}lh&b6MSJF*VwP<K&kZ(0DRBrb~
zg;F}F-fqPeUH-A&2x}uvscf8)+>ubO+nA2u0M)u{evF)@Qa2M?i{<uWKkWNG%%-z~
zPEcAMA#6d~y56#OcDCk;Mytt~9hiF>HLP`eGW*%Et28@Dh8wRm+iTQ1(RKr70<HDo
zNW?nw%R{BlweNTC0tzGckYbu;DIt7Q_lc>ZQq4ANaGugAmx=KKY?^Lcg#wj8BzxiZ
z6&`o;Ze;@l@Jzr$#ReH!Mi9TqbGYsLsb*WME00aJ=-_mPJ)x2Uyns%U9%oOp0x#np
z7&1`U@i)ZSG)d+z_YrtO-|S?57S306aq%q#Xa_iGdbr7=MsHt{sU8Q1?sXwJ_<`r_
z;A<Rc9e#Ba8Iu4*k>BGT<tB3ASR<QqK{7M40fU#G5cxayzAJ4vC>sJ(E*oupU(oJe
z6$uR;FPs65T!Opj@uU+X1R905Kg`1uhw9U!!Z&_b@f~dVObuRM^WALDLvOCB$isK_
zc9Y^xcvh8I^$gljR9$PImq6b=E&H~cj~G^6+mW9&LJY$ZIxFB83%Jd;w(&68h%hZK
zR32p-rbbOxwQxT>PtsHMtH{~X+x<$*CqsYJ%Hf;=#^tZCyHx<m71!Ma7{}RX<cjcM
z8kkZlPTyg~p?)Pf??JA*mp>2G?7iJ{i{loGgz+N!ei)8=Mrh9NzH0LqsTX;}unFi%
z?|MqQpJ)3E%xsjU3)BDDbn%?sVSifh;c9Z@!woxA#GSP<S+QOAuQF+L@Wo9Ksm5Qd
zA8uUXKfccq@66Y-RkuE|c8##m)m=04zobQz;0Cp=o@YPvmQ1?=fWkEoA{#&0kO^@m
zQjips1+CywJ^%mDI81WksP@Zxrt8Z}A^qlRU1h^-gA6Zw)ZH8k%S>~BIY|^P8UQHT
z%0DzkHY_Ejy14N7=21IiR5^ze1*oDUI?@RtZ?k$v8`XGDXw?w@TB*Rc`S$~<rPIpG
zNvjP_UMDzCI7gA{UYy^d;6p`Mw*0tJg{tSF-){o|RiE{i9D|24`p+M!n2Hk5g6P0p
zzOMy*Bt=|Gvor5Pd<eR$K(#zu5nELAim>wU5@D=ZtBob6wJKDfQVHi+BA#7=!{M9n
zoau*K8X))Mnz0${SS7>;@9<Lp^MfKDK**&TTDM+`x<Nvb`roC{!t+vsSXxw;Fr~*G
zixmHvDvZ+7tw-BL<kr9>h*7n9RX#UmE;#M~Yba9r<!np2EweA$VqSuZOs#nNBK|*$
z2Iyzh=7n5ksuEf{&Tl>YZ#`@7HOn?}+SR{Qc5I~#-2Y5~)`hn+XS+zy^Dgg1^Y!1M
z@E|!5Ky8X32h9yj<6mta$&EtM<ZQ3(t;w?Cqy_)g;L-DFieEulmL0cHkSpZ>hasH9
zDeDqy*Oo{pFV~BP74O{qk2vAIvY={SAf)o_WMM81mSu;Ig6qz|b8xOOi@J5Qqsz$Q
z52e-dipeA3AtaWx-u)*sO>}73Pf{yuJplVR2J{Q5f0di;-a2n&#jk#jODqaG=bg9x
zcc-=$N$C|pDuVcE&Pw_7C3%zf7AVGySa2n&|9&_A=vP@<EGdu*SViZp>)sq*5+aQ(
zA``9aD29{!#xFOk3jgnpPcz8QD4R}IFw_6ax3Rw*i<X@{b=haw94b5m?&ebhVl0#!
z1phC}r()jY*dcCMf-7-uI?si&w#d@sVYF68fX4@yX|ZR1SJPbl@6ksI<E7(}yXXl=
z8HGY}|EWGrF--y)g2Zr_b=?#)$Gl}$t0D2ekZ(HO1-iXxOR|yc<j}I1OI=l;?N^>P
zqi2**pseTNd+WoQXgI3D%_3||x)_Gt+MKK6kV2lr7@g@XSCK#obDWMmOp@_bqbD%P
zesj|JM@R!sBc+;`kLn@I$4+m#zK7}>s?+bN@}$b4^Zw#?s2f%aI8ZnjT^4bCqYvp4
zs}ni4p2Z)wD@l?oNtUssr(Y%hiZyAE<&*c9tW^C2E@a9zgsOQHMukhsgR9GnM$L3#
zi5P?Uf6qEvhRVfb-PSFlsF3tHRXZ!S*K!4Qk^0kQ!yoeh+23u;{;70LYdI$NGBCAx
z$NzB1c2&=)ebNf6lkxRub}jdR@ObW5vFV}u@pI*EIowrBJF_Pa{9m9c;x%lSG%&R8
z2dg<R6kOI`7A^l!D){tFK9RGX?Ytzh3OvZQkrHeeeE1hNz-`6<MtP9pv;Hw`wD7D5
z1k3{|t<-<v=3<J_wnx2YyI2_YKwQp7-u@p7_wXE=^h~qI)<;#_DgPB06pFBDyQu%w
z4?XoCBmnv+B|s^e1s$gda#MZR`Q`He8Ylv}MDQN9Q|w<RddS{*rGbP{<$o;7vEJBH
zj*eZ=+>+(VUd6xU*-V=4KdS<U6)hCnOuJhP5tWo^{VT#NNx#bM)9Fg!t*>%73;I8E
z<5?p#^){C@B)rrLvV+3!{@Yx|nH>CqRcp_1JV(qG#O{KBc0_nuom;gyFo>BwqV~W0
zhcn)VYE<QtNINQ$BCfUdUvZ<kwb@bG(N*Mn(f>7k#n&~L!*4rsK;`^o0EpJo&W@M;
zGvVVO7)+1&Jybw~d%Isr4PFI2429n9-Z#TQij+eAtKv7SVS$pCJ?K{3S+g^#s)jwe
zMRpX0nHjW31~ET-w~py*4MkZlDF=xrZYdQ5F)>JjU(i*HW!4S<0128Zw5N+Si9>I}
zMNj60Q~=RM&-q{`(fNlRZV6$}CSs{Zlbz?a`;So-SJwuHRAUokX3G(OnST%u{$19R
z7|ybyylD9@CqztcvBAQ#SgLl*)mD!{AJf_l%wYD;7DHiND>pykNHQ(9ptMvPQ(jpJ
zO4ynk`=0|wfLO;t4Fr?ZHDY_J5^@wRM<fQSo9qA$U>k`-abO8VdXutc3x;c_GDX;*
z*XaFvUV`s2L|Hoa6G1Rl_H}zDT+?%SpYO;g%d_z23*6=p8Dpbc_cIz)(aF%pAfNsU
z|3@0Ma03S7vxDU#a@mTo$bEhtbVUilrVQbnr6$W*aA97*Ng-U->4l`~#`Lsd)0vMs
zncu=!uzvEu6~>Tzi>&WqMdCT%Qh86cy&u^zD?ed~=Q;GUcIKv{tR1*`z-PH#(%6Vm
zH0wFlr(!Ieo2joT2VJ0ucO~gx679-=hNdtMQxbGHP^?JB&F|NhwH<3bhQX@W=Ei5K
z=RTUmSY*H12~A#+!Q-y1sw-N^g<7{<PNxXyM~=prS)tEHFl*A&p;HZ6*bpQYQ}zD6
z?w%?YWvb$momQ9keBn~f5=*L75HNC{?yER@Y6wWby>%&9FS8q0CN)P7C(R<AzMk6;
z`<dppMSZ`Fb`+)HA&tw|*r$}L5;Pp#|HGQyKed~xk=X>@U%Yox1Le9|UFm%WbJAf5
ziaGS;2b)o&W_2hgbT6_`PVa-K1ZLXx>(?)P1|^CvKh)+1UbyCJGC6FgonBOhjB@5;
z4Xts|RJ1dq3ppH91nL+N#-1&7g7X;A;E31Q6XH+T#2kf?)f(&qja!Bt*rce4Wl!x2
zH~7L}CtA|(_<QZPi26l4vRMq>v6f$#PhOwrRJhJHEH1MOc=EToZ!Ihum7BJ_DfImK
zO%w<wFYNs@S<>)lh1wxfhd|>76HSx)z0D>P%(p8=nwp2mBqNv3s)~+XVP00M>g+(_
zX=2-NwzHZqMxHB#CiC5F4rg_cHuNLSyOI$%bLKYTLq$rj2^y-3YFc58e+SL$V$$Oy
z=Z(7$9-5YtQ%j78p;eM!=gP{jG($O_i@&>W$F`ksAr`@rZ=m$;z&iD%<(lr`-zqD@
zXDWXAsThQV`Jr|;R#ZPAs4Y)}aizW`8AiH(ZZe;Ec3SLs!bk>Udl9F-Z*5q6?^BUA
z&=Z}={$hHIBU`(_IC+!ZscnUkUXxLLH*R#cBK(y>3lYI}8F%WH0pNbxV<JaH&nzyv
z+-vfjsORhI>e|=eUo!A%$u63HiHeZ)VFltRI26zbuvA>f=ax(bCkqsHVJWO#J7?}2
z=F8V+b8~Z3B`TEiTu*3~&D#Lw*tjdUlnpUdGqb`Fd%}*42>o=Ss%g~bt#~C<h|$Nb
zHmi1*JGJ(kMark^kSkN|Agr;wenOYUt!vPfEC$|p=QlC8wqn!@ACYX2F?HO1sJJkS
zVBL(t2)u9P6|~<Yl&Fr8{IUpx`so@Gw%wqEmsmN9&wf9ikys)H_GsH<&-Jn2HLpJk
z!;^@!4PZD%R&Oy;HM&Raq}d|SLB;52L<B5tTCpCXE21v0?cF0Y6*{hyD%{{Ip~*Xg
zRhg5I*lkA)o^MY)fVbCN*+wI;7XHc@8nseqXnA+D7Gt%q(Gk_sy2l$Ooa@(A(KD`H
zt_k%S<$V-=mu2xMj``q5taXvaz{RM`A<o)P*Cp7kkDy3HR@qWt1D;T|T^gXRIk#$F
z^q#RAyQkvLz=po<v48uE5y(I^@~6CLQE|fSv&?o0U;y0_Ng}K9MJdG2gw3SVFW^7~
ziov5aH2XA_ZPg+9gmufY<u@|#{YWO2WZ8OUQVRwk{08RufV8S=OhR%F+7rIF%}S_d
z?W|%lYh7IS2!~?Rg<k>hh6?D<lSNept3i~X6j31M>p1#`(A`0D5iv;M9eH=Mc$3ui
z3KR;0Dxs|Z^ayi<r-+diY|)h^V7)XlkbtvYud&{r9?r~+a&B&}3$z;7$U)0N7Ad2^
zueS5{h{wmrflkBRWg7i!?6|Dg#IXpwt!4kUKYy2=X?=zvi&%Ci{kK|+&i5OHuwShd
z>FO}bd+k|&Pn%%*D&u*$+wGo2tzLzPuobL2Kox|W3BiMAC5K0{9-XeVX?StgL!$O}
z{W6#zbya_8_Kb~_YZ{{-rN_yyZxcT0k8UlJyv>&Zs+W9JQF{M;c?+XAn3%IA7lrwW
zTTMAr<}VQYLl`Vn+Y$r*qA=k`(*PYUs`^(kf6!49N`G`gou-b7#p9*Wk_-on@0%Ol
z7vhETt6S`3k?EPIQxxF=dl~bgzx;h|v|4r6%yrV%)ntj6%Sl<N$2{?0I#^wV{$hqR
z;vhc3`u|NMPx}+oy)+w5$uU6ES6p?~fRvl-G5m{;?k3y@Nv^3J?b`o2y23Gt%U0EX
zI~dH_R=dZd&ObMMKyu>ubUIrr=67Nb8URO~$1`s@?U>6-YjDY?=yZpM!s(ulW|{`z
zUe(Jr%3>!yi^}o-leXJe3W=oNsn?iazdCgWhcE*Qf3PIT%Mf;M8wSUvTci1z(wt6{
zLNqz!_Egsk0b9^5UM;;MEqf3o_l;7)miVSn!R#tgnxzfi?9?$RjJXAf6<yyL#=_yb
z%;wLaV=cUdcXfM+H-3F#p_s%IjtW8`YjuMj-Dfbn+y6@Ai_uERnu;2I6sQ#HrZMbh
z9>Lu@YC~Z?E?nrBI8r?gKI{beNh@FE@uu*~<P9m7EBQYF-asM0bB3MD34?Pm3z6;(
z9XyMlbOK+nU;*EJ^9}3Pt)sHCl7}CDn122GwJRO#w=<x+63^M^?OFEW2auG6XY7fp
zl~5>-a`tDm-NB66v_Hbk)E<G<THL3mQ?(qav7c<>jegOBg-pJGG6e+%L`O&OcKDWg
zvREukojMhV&CdM~JkX>u@v`ELS6JGAAdmd+N+FyA)YW+EtvX4)_|0E$Wd6c4neyEd
z&hY!0e8&?MC3WS(dtPEtr#QSj7VzLbkCPvY<>b3x=iiwz7|ni`{ref7d+%*NTe^vW
zXbfZf4d?RzyNl~i8%^2L|8n!+J|!tVos`sAN*8>~{G^*1`Ct}*yyZz!;{7a0JemJK
z(T}0)KjV&ne1W~9gwWu#c>LjikmU$a6EtE9Rx<Cu&+vbrZ=th0$oQw<q<5^JnUDOD
zKh%xk)qf3UV##bK|LsNUO6y1(coNS%dpF;__7pFDzZysOIG+3XX)^0p^1+k$@xo8V
zbdA$^<gUA!ea%yR-8TbSTTEsvbN@A&Wo4V0v$aIXs3=BGy_SdmaRq%d>>_``-QvP%
zHnDBGIHwQSlaP=|MsgHeXMV}bbDm_)u)YLlzrk$}|A!yuuAtBj8H0xNt5a^{fyrYj
zo&E*){rMR-Wp>6k<2i0!F`XM9dx_r5-sQ%BP9-|4D_zpeEcs?GiDPeN+^+_+<-J#U
z;+2>9VEG0DLg`q04Ca!XZ{v^WO(4xs%bIup%Du17Va3+1)c72j9f=GdeHITrd@Eh;
z;{Gt&iLd*PXa0UGI~T2HM^O>>tUjFcm)p7J!ZXM)c!d}ah0T;gkOx`(<`X>pV;+u#
zG_sS8%>H}^8MnX4vEA)FegEH?`NINsmek=k_(<qHke-*{!OOoLLG9|<Jb3FPEKKi2
z`lffdchxuC{=qj~-N(!9FW%#^N1tW!vJI5@4W#xT!_`+`&7{kIMXV6qzJt)aCnwf^
z@WBUl0X1pTB;I`UP3F#>%ff{V+a;c?3m3o**9q}dg|)H@&*fKQed$F+MT6udtk1rH
zXVkH(bx|meYAe!_7jERBd1f39zE669K$n|uX3QWzKc9d6;~#pXBO3@gEIKDAhi9I7
zhEq;Cg&8xyW#fho5f1a3E}nMosA6PhZ^rbCA>i^h^wos$c>*|6vKVmu0Cp5q5bAX%
z&)#)0wRQDAI9kw)q4()LbNBDCiL|_;I722gzJ(w2+^bLWx@A1Ce?5mqGe2QsPd5+W
z{5%WR7L#+vH9YpGGug6w3EPNa(!W1t&hwYk@0e4WeB(G4uBzkW6Hj13b`-WDXE46B
zn6ixi{Pu>M8JZB%y9NpQ@i;R$<J^lmC%%xv6E9}s;9UHLOZZ?}5vC=p*)(@0THkZI
z=lCx8>@keF@>+7m=cRZ2mD9SUK}k7ApNG1bT&{WGX>K3WA9Gy}PrvvI|NM17Jb^v0
z3$r{KT>8K(%$qTlXD%Jh#{d0;cb0Az!om{KwG|@Z_0#|I`*>_(Px97pr!v%oyFUDm
znSVN)gv!PI>97A_xi6karp{&6vKidj$Hmw0Pv+HyqFlyY$aAmX!^jL58z<h!t1mpq
zIYPjVxcGXmJ##d!jZ4`OJ)D1j{24DldL@q4U-HsRf8*bEBly>Uzh}{`uekA82hZI0
z9N#a<7dJ{VFF*02DB~2K_-HP3zx$kjos-DcAKzq2si*#oMP1N}S7GaZ3eSHujalD(
zz*Pf0Jpc5kEM8lI*%I+W5enjRTRG*nN4aT;ja}Qek#8N%i(k#+mCH^izGO8Y&sj<S
zMGrFV`|p@L?JbT^u4CpC&#|=BMeLx{c;fX(Ijd708^+(rzh8TWbGkaHSnxe>Uw;SF
zGEU}+ujaAj$7%dAw}`*J^scy;rI@V;?x-(?le{<NJu-9VOcpI##Kegcx$(vu89#n}
z`x1+a7JpIbebA3Qf>pe9@a(_D1H})~G4<;e3dK=xdFoi_(?;>s*3bj3jks$%>h6dX
zPom|PJ{()OZ6!B1ml4B;bIVOPbK!*->c8*0>n=_^?KJxL@2{7|rY)QGF21lO!Qk1&
z;vySPQ!)wG5Rt)|gF;*y>~Yvrx{DuP%&}?o>fRZf*k5<C5dtkJyVAPn;4qu9XhxyN
zJ$(JuB7WN9;<_uw(LXbpnB1O>>^p?y!c}amEXN$znLau3lsN}-)&;k4*{P!$G$0LA
zTm}<Pn!uI0rA*&hOu1WJqnfRJM>K<boxw#TvoI5=KSw!9a?nWXz)>7CY6OLwtHh!J
zMN7XU&zMGzV-xc?FK3r69?y<S5|W1UtI>Ti1T~VfyU`;d7K;qYAOu0c<RmSp54{tj
zF$Qe(>DPz8X;Hh&!teF+yBq$%uZ};4u002F)`=q--np7m>66tY7LXLJwzO{aOo_&w
z-J7vD+`>6yMliBl3bpe;;m7hQI-PJiV|phN6W^K9Cyl}pQ^j{%DzU^T(OVol)-38}
zVrK^Q?@Nl)N>q~go*9qL-<4Z#{v8v>jG|w+ROZiKz_i5`TzbW+4DXUobXFIJ4;)Eq
z#Rj%j6yd=y1g)K*xLzia$JCT;PM>rix8L&sJz^Vt%lx&JrCh+IOD<#h;O_JoaV#TG
zIsqy?LOgq$^p}b2Ey^!0rw85R9HJbv`TgC0X2P+@&?7YtEp0IWdFENZm~;whDM=*s
z7|z8*d(cO#z~>EOiixMkfW9QzED)bbzy1SBHrKIy{T$vYPU6O~;~0^fL2_Cb1`F{_
zK^aBc%H_>^K;3pS`uIZ+J;bhEyC^Oe^{Tv_|NZZOy3_cbciySTU}~2T>(vXMdjY!l
z(7W!08iT58=+g)O`6M23uPGFYBh{pIq*}uEy^X29VGp(_mgKs9`8r9fQ>1u0lBa8t
z-E(qO-Nua@dF-E$>w@XlTW;ky7yX7Vxm|ew{r7qI-FNjsf^wRONJ~P{n#QQCI#&9M
zD0OM1M9$tUtr9gAxC=IjQXeMDQ#8gtzqm*mp@#C34$oe%Kcc>cM6wzdV=O;B^=IZh
zDH>ck7quZkbeBwhID9bV)6>CX!)~`rd4k&)(4B;xox5|w`KR*Ii!<2ty9+UVHI2%6
zC-LWwC&BNbCgPGx%a`At#PFN}41akV>+;UU^~pkd9)CUO_gTwT{}p0w;&MJJ66yCl
zg`O4<(qG;l^fj&<A^F`yKp)Z;#P1Q?2@$o&#;M`72$3cIc1#vxMLrU{q~U2{2wW)O
z!y{ufnJrlDHjyt6F2BfkKxhV=hvI1ubMCYUk%J#KqlRXQBi2%XFt10PTS$Bt=O~@~
z>vgUle?W*vjKrG_LWuY<+5);0w!@al;{V*wuV0Za_p;s#@s`sGhe`al<uK{yUsL$b
z_x$DbQJB)Ya{9%WbIx&N=pJp-^Qt@T+pQRdmJ0a9@q`FYkIf>>T8P)IHTxiu+xbP_
z4dPll?BX~cH@<*|$r_JU+rSr3-pP+EcVSO7Q?h0~m8M+uC<x+weIkF-UrlD-Bfeu4
zB3Kl(ImOS+SMO!k6!BY}lQHNaC8L)R<p=h5C#|nfJn;lOcI=?Au#nB0H{)`-IOB{n
zxZr{d^l0DvWyu;k4C}b@@bnW9WkHX3G3G=(#|%}Ck3yk1s1~KR2&1-aXNnX_Ezw%I
zg7ApDqd_=H%SUufG`TrlbP@H#4?l4K{r7X<z4z(D>4Wz^;IYRaqoSgMgoFfo_2{WL
z#9^VMX%-UF;#stG4YQY*({Dr+H26GOtwzeW<S}>7a&phUnkZZS#HP|k+S5>$a+H)=
z2)8`}3PXDLh0o*Q;+y`D0r7Sr8UnJA33zPec1|GV^Xa}s(i1r3j*wkKocWz;j2L=4
zr@!(bAAI;C3Deh;9NU{~hi3@k<KF8P!4ojkdsr`W2fe|^Q~piaHjUUzPv-n6GoQTh
z5>sBNW0Sj@ai<T$A_SUS2#`oY5ZQ6tTD=QISeOhUvuN|J_}lP`5*el1=c&YHjiu*r
zZsn!{R!VC^dZTL=WgV9jr~CDo_whF5+nt^$c_GkhYs{QI=_W>JCF!CD8C=tArBknT
zA!<Slxadw^IpKW1ojQZX8`iSqjr)1`>CbuOi)T2myS*V@A)sWs_5ZMl|K*HszShbk
zY>5grjNUX_ajkfl%O*WcAafY!9yglqIjNNMF7Ge&(NeH(Ibtv9t)?Q>nd`2)o)fx9
zQBx;GoF=Z7AxhNYXi;wd{dyUc7N#eke3JR|=j-?Q)~#E0;WTdCIPSgoUQ$z2_s`lZ
zM6LPtSK;+3;(}RZpCJ11!?)kW+BpmFZ!RTdv8boMLZN84b!dP0Ibh3Eb7E;<$WR6H
z;d&%?<H<bQEjGPDm^$rqHf`R-tXZ@8+vLA7WYAz<fAbA}M3EFX=bn2mSy@>T;t5?m
z^}G0Y^ndFfo_=)-$Mm|Dp|SPWDRaqso`3jB-mg7|`;Q+^r>GD$;&`s2GF(eG5juVp
zQKJ3bwPFcAIj&vz^{|naU?SF8%#P?D{AGN<-Oa6RxB{YSHW`J$u{)6F2xDxNen~Yv
zdIRb5Xygtb&cM^V^W;-c5|x%h-}A1dpWVk^+DTRzZxBcCL3GO=!1M3@kJR2Hx%QZB
zqRa#7JKD%&Z+=9#^f8>)KVBCIa;QB<CpM!=9~x(kv0?RjbRi&hTtJ+QBQ^$`UDL&b
z-RY3Jk7#EMP9Y}cmKLW|2%Zp8;?rR_;;(H|_pCPT4l_O(s8p0mNCt!yLeZYliHh=t
z;+$g{IcCp@tjdyFqGApB#AZ?Ac)FFw7KOuV7Te0RaNrQv*dRyG#n`c#H6h$Aq{N#^
zjVffDC5OqU?r~7|xT?f?Ixu_kcz*H}v}3N|`WtRWjtkrP()C>N{1^Ot{th9Y`r@?P
zMIETY;gsPA<k(A-o@aSYY!0V5cS92~DDp2YhGv5}Pa|>BPK+j(9$#$L=bwpnPEMQr
zFa9<xPB%6EZ~as}b8AIf=19da@@K;tZ6aW_6Jt*%t#})*=uV6syT@`9tSHCll;X*=
zcg%z~FmK*G{r-?)Iv#oC5xskR!h{Ljci(-aq@?WkJ!f7xAMU+Js7rBe1`UKuF9&h&
zux2&<{eCQ+I^#L{6xC8F6p9Xo@$S3tzDI}wL*!e%6bJHmq+klKkNj-9OrK|};koSf
zcspd!JSl!;w_Hp9TXGtYl$2!p_UXgo#fvE`D`Wfi?JQWhKo0~c#o-l~U%?;#@CUuy
zuF;%b(&~}gg>1I-##`?)XVDrwRTbndoxu~&zQ}u@Zs6oU+`}E0orE*s#vk;s`IWbs
zzcr6CR~^e{yw7tlf5@&X58g^QIX(I@YG@KO=gj8)sXvle;>B3Bf)8GJg!^9K#^9lY
zi7uJP^KX94>diYvOJ7S&r))BkVlin!sRl8|rDEB(mQ~AF614Z^hP(ejKd07wpvX`V
z&6$L0%}f@q+eYbtt9anoL$F0hQN3g}%NMO={rErez!|+o`UY%5C@lZ#f4n;NJ9fHC
z@d+UnmzGO<oQY+he$KoNYuQ>`O;w?&3(+-v{@TlYvvNBnbsCn~PINZz;tR2C+LDbF
z)dtXP8T9I$(9pxD>47GT=1=F1|9--nJdutoK$rA%(h{R_CTF27euvMdeb0u<YU(Ps
z^Z6^!^5PqxU^?z}x>T&^zpuZ>mn$|>RTm;TA&VhhVkuZSgO}d>jCB>o1PvPY#I9r|
zM3J8EV9C10yz$XY))#s(iTyu)`El;~_cC&N5205|5f9yXFP|=6MX8WeRU76p_1o`Q
zwbRGME3PDA{kJ^z!Ix~w&lf^5L|XS;v~@oS5&AjHH*Uon^pTL=l?)*;X$TQ&wfI^4
z&4;`rj=8qdLs`I1ipW>zBqzbmKk&gkaqhczux0H+X1w+k&wjgB2yQQ->|u=Qn}MZr
z3t!LthBeEIp?nL)&VG#Pl_<{bb6);joSP7$THb0te&InTPhX3n&v6V(6hgLz_aJ%S
z`-F&*^*lK_xzRd#FMR*~_x$#^zon?Ch>VO3-D$k(b}b>h`O?zT_I`VlMX_js(6sh?
zAE(nMVCpMg62waj-Ma}fbs2%!*uD0X)?fL8=9HpTO$@A199CMK&Pmgs<SneyTHjV{
zvR#wkS{t@$oSeqqht!(wCTVSYs_;5pJW1UX`2dQX->RuV3NLFao@Do3<gz1GJcaY$
zDKnG7g9eNI8mOtQ(evHAcW<t_dJ>mhb}3G~gFsVjoOHZ4*ywufah#fIX3<xl^2OW*
zEM2mcwRuJOEdlHaNemo1gfyoKV^k_bx;a_;?T36de<5==Sr~i95DK?%X2a%UEL}!3
ze$+7>JFp*))&Jvv-^^q2!nsV}R7>yE&gJ4Uy{Vr4H21x=m{@Cw{8e)Z_8P(H0X=Yp
z`~+nelF3RJLmsn!`jPLuU(GWY4Z+l6$4y8Nq?u|cX8M9P^uG9yOzathPdb773z)Zb
z0moeR2hQp$8cfZEGqj7BUU-4emT$*q_ON_Q5%%F{aza)NS#bfrTe5)pv*u8xd5AG<
zy!Pbl6j@^^-@K7^&XJrJw}prAd4VmKSloFVnBnTiq;Y*PhU&$I(Q0PxYm@oc%&k~7
zFM0Df>W<SRhh$^a91I&Z0*Y7h=7-Z*A<B5+($!>~asd~PA4c_@={)?<f5@}M5L2{;
zrJey?dqyVT|MxYX_~a*I5-dWLtf!#g1kTTO;&G%i;+Wy2Z25|*U;V&>1wZlix?+;X
zp3N1f_9xcvq*I)iwX42n*4$+*nLnGkWu3VE&wt~_k*O?v|52X!dIO0LA&6HmA^XG$
zI2Zhzdp}r0lrzBg74wK4K7nC5DI$LXz5J{;H=jKA6z?zJA%uXJxvRzTdL7T$-f=i`
zx?;;;$JB3rV9`%PT;vYrk^%AL=WS-0cL<k{?~W}d8Ar)BR{roM->=%tmvKY5b^LIK
z3>%KN=trha|C$9tm@M8>LYGr6;<Dp<5oZXs7-79N@w92vCZ<f8!gt?&NB8dCx#NyI
z+AN$}6HihY8OEQ6cJBFjdiJVcmXV2R!Wjh5IS+4oMziTD@uX1vvc*$KYbLFOB%TZ{
zERW2q=J9ysJR`=Y-E!gIO+Ulm!+-kkKu{M?@-rN=pt`D>?40ZlcafzPDK0jyv9osz
z1tcwLKEF?Q4wrG8<!>41Nv><H8&f)mn+WJ#8+EmALKZvGp>n3Z`Y;c@Hj|CrujH$L
z-NeXFHhh5q?&?~qYU{8>#}H+==p&b8dtfk_#qo@~mB;U`qfTt+4Vloa7NVo<(&LDr
z&yBo-(n=E&=Om+)TlKt!#4-zZyNMN3?&gNSy~2oRW^muhUByoR*7Gi8mkd~GFk2hu
zfs<7%(6G!RmW7pnNO!~zHUwUjO_<!rV$gJ-Ur${nbv1s>QBKU#xm|Wg$`yL%<eY3F
zd|v<VWkG`pyQT4U33@$y0y_q!W3|{%ZWF4XLl9pb)zuzxjl{XxWfz|ltD)Wz+v{sM
zo)m?eOxuWGhy$Ojui+O1AuertCN)gb$I0iSuBt{H)2RE}L`OOFG6;nl%EjZtQ&WdK
zD8!>!XO~@c()nMMgFfR<!*wuPtQZ46J>*1qnO-&#U3an=<*@&7`N>a%e*H8(TD^D!
zp}Mjfzqr!2m}nfPATB*YBUTxSTu#v<j$K_*jYpGn2->lWx?$90K5D6~s@2mqSfX$^
zEP5AaFw}aTmrmiuCB^!SLL-qA2<Ul~pYk5Cw9^l1*|KE}960cx-N#$DY!P+JtWTG8
zv`!JFP#mq|NpDXhA|-|6JVlNL4vQm06R~7yMm+7$HsNKV{j@yochHyCoW_p^WWyQ=
zHTZ8uIKf9+*IKq*iH1geTAk5YrSFn%NQvX(zx<P|<d0Yz-IJb~4!lB?$;QkQ6-82X
zRK1h2bYZdDc3ZOKvHapZG_wt7f=zcm-n~vHYmz=QjjdmL#T3`aS6j~VssH8ScRyfF
z!bJYjKTU{+Ap7!OlhxWP&!OQM;i}$vS-pe1HGHm3j<4s-5sSr1L%zh;jT+ogeO?{r
zy_PkqXVT}XY5HM{rxbxg{6xoF8(Yt$7!eJt&Aj*VOhVL{_d4JD^?un1i}V}`;;+&1
zBJyi<>VGle)ncE3ZY8rdx|#HIG&XbYjeoXKe)VEYI#5R^CGM$LQbg%f?$n=CC=?Vo
zzLBVeIHGpvQM|y&?pIMS;u36)(hVMffkxf~^`kiz3WcJr4JmD!(qMlgj2}`l*w)c@
z`!nTATS6EdEh6fG_T8;Cd{lca44?E{MhAWR9A&|<5{}%ip;2x7d4!tTUQYcHB7fok
zxc2Hhs2+L(_dIqD{Zpd#7*qS3@3t;$Hvjvve{+JPJ?lfw7vz4Y*XsE;x8J^wwXgg)
zvwx^f*B{lDrBEn3fC@Rh0%x!1;=!I6IPCw^9wD?Z;bc4ttx*SFk3#tB>KcZvAIgZT
za?;xn@jXZKz<Y1w%<9XaZmAfAaPp}(RnZF4!X(Yod&M#9K!jZe^Z^^F3awE564@Ui
zbTH?`+vn4<t${KdJ%EpbXyZpZK~Y1T>Ag9rOF!`Ya7lj^)y^nddl-d6`UKb(vqkqQ
zlG73GZ{W*ADV*~1@^r^`IbpZ5uqhOZLn*lSTRb)AFs?X41XRF7U9BnyMLXWiG)*5x
zCxc?HTenWPKxx{6KXIhkkxt_Wigbo|L^(-MOV_PrN<1kPiZ(yreh$l1YZj+pg0GW8
z@r!&jlhe5fQIeFDL}Fs%L40a)(B`=ci#a8n6pD`iK%q?@D*NLs-XVORniE2;S)5v&
z3omjQwn6zkDHKQaTbg!|{qi(brV54PAnCHh{5;VWPHO=p!|M!63@H=}g+ig|2rqYs
z(D2+AVYXk2C{jo%QKe8Q6bgmnmn6*gseJb3+oXJ-6bgkxp-}vs>~E-ogT+3BWbVs3
zygA2m>HE}?!xczhj)U&cp-?Ck3dJvxR_3s4Iu%R(=JgT94ZA1#-7|HHA#&nh4$ZVI
z;pL$g=Cx~!Ff;8-Ahp%9bR<pH;&!_mO}U~_C=?3C&#E^J+Fr9T9Fh6ZTA$#z(R!`%
zq`hta-psOafLtleBIm{n3$U;N3PpTxER3389?<vDTFhg3lyAF26*T=G7ESvTq5N=O
zXphf>F{s3oLZMJ74i7%BPmhwX8IIgukiCUd^P(yIy*Y8y(z=#D_ui*ofXa1J_Q>zS
z2$2=k>gUcQV!hly&>+6*r@h}Jn6&Vu@iY#L(loF83{}unG&Q$AQb2WdWFcgxXDUZ`
zg+ifF95$+}tBH<|Rt0e|#oN!w?ob6y#l*hE#gRPXE)WQ)TBlGb6pF)!G&-oZ=*Z=e
zsUGE!txzZw3Wefm)EEamZF*BbsBmAQ(WHEe6bgkxq4=dE=eBIl%GR6!YU{G*mLJHz
zZ4qUW>r4l$aMXNL3WY+UP$<GkJy2R&N>x>rE}+6%A@Z-eWvwmW*Shwk9l3sA-<y6O
zRPm&QlR}|TC=?1wWo4x<o-#5ruv)E+1W0RI;^$n~mi5hT({Ay!ub`RXZIsWGLZMJ7
z6u(@ga7s%{!|8OkC{hlGRyY{PJRr+cWGtx2WeSBtp-?D(+4%i_EEdZl(*=jqNMmY?
zI8y9eUCRm*8MDLd^-^0~ySoh6u3bx1R1{shbWy`o6$*vo7e-7>jOvj&ifC0l?JH1b
zYy6U?Kau6qv}~mRm;95<<)@qyMhdL(GKE5+P$(2fp?GSqU{b2O^|CP-O_&`vj6N^H
zkcQ^>;q`|wm`s=q8j&Tfg@Oe90U7f_tdnA_J`8=(=M(!jY$Mhg_nJ#kh&G>}K!fwX
z!EC`~7TW{^^=XLxePWf#WX7m9h`jI!0<n$XALwvXEy?pU3*m{+gTKY<Xux7K;`jJP
z`83XfLFP3Sz!zw;v(bXx9K_}Js|u=6C^|fzS~ucBht<6QX}Akl@!5}?$m!XKe*L;(
zKa|hzqj(V6e)&ALG~eCPC8UBBt@?qFzgvMdCYQ4>JA;G+NX=+A5inM<>4yc>dM#Mf
zM8%3uCnwHEO<5tOLM)kO8!nVhP|nrkbdVGsi@&6RGO^BR3ZZ#SL?tE@ZIFV;fUB~Y
zav>=6lJ?fm?-LM0Av!6MXw8UOh>6;gZ4@jn!k-a`Bie{1CX-}qfWq<uye_ee#VGy@
zqKW-Y&KMG%(V|iWJCJ}9qSNoCG;a&Rv>Xz}Ln9d4dmD=rcC5%Frc*Lji?LBXtS&Ca
zXR;F?ZQh+PDaeeao0wmd#IaqSRC|;)P@zzCOzKvD%d78w#-i;tn5|CYhn&pB^CvJQ
zvHd+f!OHEt^Z0EP^!qh$zW4y$4q;Q>XhPk9AGgOV+BdCHbZ*i5U=lCKR`E8!YvU$r
zVlwC%6C^)>JCz|DDV?%Nu$w#FHEEZ4YRwWQyXnGRR7ZuSC{)gp`Ab+l;~kci3}ftp
zh$m4gLZ$hv`}0b)q<CEYJy>VDslDbLvd>Cp#~WX;=EFM7eZ)<YWWrkGCs@^q%-ee7
z`1X6&P4g2S3{cZOiiBGya&oQ{ze%HH=1f+-SSW;<Mok|B28+0$YeHCWJD;(mlPUlD
zYZgAWj-nA}VitJ`Rf>vw*9A=IRL7QEKV@f(33I?CL_`2@x({<^A9C*<Pd_2lJb{+4
zqfu6LvjI)07{A<1oSWC{*M*5D#GB1*M4TQ0vE1$P5s>1>YQ>Dd-pXUK;xN`b6MB3B
zFpGVpc}=P!QO8UoCAZgyUs{SxBCV<#W<37|`B(jpdj~{O<?$1Uup$|pv3RE6%f#0-
zUVH41<k~cGY>}4LPOQVmj@Kt~OQ*Z}@~po2T{YDB#qBD@XP{yof4^Ze*T4A^rzL9C
z22`jGg+kH6tWC?O^WaTav1;J8Tz<}2OrBC+d*?as3|V;Z_v49aXL^nvr*p@3KXBW8
zt9Z%&`SMY-b0-THEX3n*h_<8t1>ImU>NEPL^&QB7tQ7qvxhN%;9`R0sH6?7>UWGp)
zRTob|cMZiAE}|2Xa9Fe>Kf?IYXK@N!CSn+M-Xum3PUWJ>OF1yz8^pD0B_(w>#yoQc
z8CDNFeD5%?qLz@c8$Hh*O7-G}*hh|{^Ef+|#UAoMo<Vuxa7J7<5!1SlSnkZ_#H;#Z
zO0fu0A^oO8q#idG|AzNj9(6q9#>G<S2vI$01}nE$<4@bc%zM`meZx4;nUINL=MH{4
zcRGb`7yeb-*yT#2=Tm25y8A6QTrrN(QI#zIa1GS~DS%qB0%?9WP5qFUr*ES?F^*%L
zeze}hxM<u!;>|t^7k|x*AFpCX73B8n%9WQ)ptq%rX;VJr>tHYbblC|6zkZj8=9e*S
z^l@By)?f<1oW{FzHekz0rE>jNR%wY`b;;?R&^MhxNgnG~tY*=sQt}rrVx71y#i0ax
z<ffw=%?wtOT<f{^(dP)9`ZOsCQG|jnj4?5|AAOKB<`mGq>MNFPE#S(}q6wbxD6d_b
zOI3{zW2a*{Gx5*dar^%m`Sx$HSJt6A@DvI~$Dy1L-<!hceQxCa2XCjlqkd`UvTaN$
z*hmfIVa4~n@Wxxr-&9HZz>~Q4#w+L>XREhZt(eY}&pyvjMd=JW<xH-*Y66*dAzEw>
z3JSOL?z0cDdHy1d125;c8_p%eQlEbHmf1Z0=*ujxfMYJcg`3VCfQ-^qy=fu;c~qoV
z?I!)0v0U++OBj%D<Hx7(Vb!3^xaNeOL@IztV~QgwnatEE7OktG&)899#5gc|Dp|j5
z9!1qP;@O)*k5R|b&0rxYUcfW6ve{L#N~BkxqT5r%ij9@@>=!4*)8V&78Jb#_nzJ^w
zWQA&N8>8V!cNjMl6B~uq;XLq8tWbR!xeaa#^GfiS8%VqELe99fuc#JA%w3a-jx$kJ
z=cC}0*(~1VC+D(Jbn2CaEh`O2t_^q3IO6-H6K6LOG#N#D8rGCloZW3`)*33lTEmK$
z^6@+}k{-t;QvUrStg)HoUeS-3D3Om|J?Z)KFmn4x<Hd)sTO!VWsYIoUv<D=Sn4N@E
zvtuwBTFkA%Y+(I6FY(6@R^c4fjbC>)viOIk{9{?Mw8@e;Z#uuZ=SSRWJ-Pkj0mQ9c
z#$WFI4@-S^de{Q&T9e22svwCXPknv)eDUK7%ET?1l4xN0nk~Hi%~}!%^<uDh3(tJE
zknL`v;x;Vc;rHgTR1;-0b2k6?)HH6LGM{20eq@=Ntab|LPvM)zUAbk{@x)vGy6;%X
zB?Q#j7jnai@a9h*s&_3S=GfnH%kTv1JkqZ#gvp-Bi5FhT##t{hYf})LRr%~G6pD_6
zwRk;iRu*yQ>A%`7oFoIUx`T)RdNpaG9o%`(<9r_I&2?A)iuiSJal+#tu+^gx*#0#W
zuX>0rQKxhLB?H<1-hFg^d3JsKY&PLqIEz&kPEJ1gWS0N$Uz{~%aed-zr*ZdR|H{0S
z;hZzRH}5@lKev5eK*+y}e@woc=}E)6_=3~$eElC@dhJW`BsEaBt(Y~%?pE_+wAo0E
zjm2cJ(<viUJoghw%;-YDVPhDQ6~#tzJ~L;{Wo3RvgUKNr%^L!qV(}84k2zGs_SMVT
zR_PMLEQqIc2WyutVcD|ftld(8%P*enwMA@Nzn*Q|^VpG}$Cl0c)O!7m8svbdj<Vun
zN=iy7D=VkQ?bX}8pwB~HZH+!szOJqguP<Uh-6z<m_n63Dwu3%B%h87aAcPgrLZXo$
zK8$YLmf*W-8nfKhR7Bayxc_X9AK8hZygn8Wl`ky76<bEt^+QPOJ%nUm5SO@^A?Z}^
z3lJ0)z+ejD-&Ta%Xv3ZrBWjfq)9b6*R8~pZg#Kg?8B6bgn@wI1_AV);8iRTT2>Bp+
z_$bn4m)MENGWJ+6(Scfa_=6a$UHFHj(e<%0*o3h1?(I}WNHb$`ui-zlDj9h3rQAP0
zmw*r(1Dr`h494OSd0O<*Pjo(U9QXcp9Hwdy1N(R3!5iP^*#)(HFm?!8r5mVl2T2-q
zEEB|=)AwJBvh@bB4<1PWl*Oc<bRNGuuQ%1{F%-PLigi1y=yvQmywYVTFW&POTdw*Y
zFN{v2%<b1LZ-GDvyVc9)>4j|R+n4BGF&G*=%7Y@$j_%nw|LY<ts}sqhvhOTQq;K~)
zT(#c%YpWTF?=pyEP1BfKT*JgHt9X-BVI&j^MF&9(U=+{#WOEY(h1E)ZyzW@8yzm+<
z<Hyn2;G#=W9&=w_z|LzY5}5Wn^gW4(9=n!ADY&*)vddZja_p(CqSGnA;s0*Gg{~s$
zv`yc0=Y}F(vdsKyHZK$o;MG4LPk)Dt>t~DS<LvJUjOfD7t+fm~y*mR>9L0dnU0Jv)
zs6R2!di+JsYJ)D2gLw?%MYyDJCpgS{yW)&VBheB<%gCUpWCN9TzTH-<a7$z~IY>;1
zqN3DCbYe1zRturJovhlj6MJGRX$~XRB}Hs4bkHL?nz$$z#Z`4!91iT!QCK1dR)s{H
zT3u0r$7CngZpH6);Sw*~W+5m;J}<tI5xd2t51^85r|eoZ8-n=c04_tl;X)JVW;%2i
z{phH>CW}#2027u2F+53srTj7yFF2hcPj+E@K>^->&SdpH-?Q_JOUShb2vkHd@X@nK
z9<_;On;q<0G?$8)!3-G?qmPygS>?z&3(jaaYhIqgrq~_~yKgi>A&y)Rp38YXe5~A1
ziSv`Ctn8LXtl2{K+FfiF;_4W)5tq2sn%yUEnh-|2cmr@oVb)}tnsivhY_*e?U}^AM
zYvyvw3R6=@f%tw}PP(`O#Bp6FaxR|02i^b`6{Qpv21(3~Ax_*d6++xtW7FtoH!@!e
zP+y2TpN2u4iyU>R36W+rI;2aJxPgR#6SrVyN{kTqLERi`lle6n^hzq@Vwp7qDWXCL
zh2SySEjTnkl{G$*D=*uFb@WQkptIF1Zq9}qPdd!|%GgvL$6c>a<0R8s9(}inijan_
z-WaIA*~D04$Q0%BPOXpEHJf@^D-?>3gLq!~LS=ZgKod<R+6>vFVX~6a*}_Nvyo-|f
zG|WqWV!PHCtLA0Hf?^EWf7FGOB=t{!<qQ3rPk+W6j2dBWlh&^j&RKHYT^(C%i%G9q
z!Fw+}$W)Jb`7TVR@1R%=Nh7%MhF|m6tM_oz7d;p_>?|%l`{Z_t2<g*W&Z4=qh4>5L
zv&PYH=yCM#6h%TtSJKvRBd=)@$cMGjNlI1%8_Ft3Oz%XL%|O|PVuA((R@t*+4p1ka
z{}t7>kRC^Df|DZgH7>Oi35|LbLil`cTt0)?CPwHt(E!<uRJnvmm5%IYt2jd=QftXJ
z&mety8^R08*K(6+E5+hNE1tBYKs{;RAT{MhLU9&QA__NET(D5PEo`!ahN${sChLo;
zNIRh)-E+E=cic~C3*-$Dq*kb2mr%U^(w&$y`qIZ-z~X1;Qahv@eFli?yv2*zTScL-
zoHDPQIw6>hHWU6pwb<87a907l#&jbiH<6+fUuES*V;OhCP<VVgd5_E?=BB<xmG7kZ
z!8Pm{F_@E1O4rqPsYg_zU4H!4l@$A8NN7zW2E=xWT{+w4<>QaO;>7Dtp$ldF_st)8
zC)kr`CiY>#*jS#Nvy|`GcjbgkE9<_V$_ItjoS7|e->n$7Z)AIZKJMrcv!>_cH*}#+
z+?>WxH3fbj_L2%wX_N7ZJm!_wQ6b7Cq*+Dz8S%NR$S*D>C`8Vt8B6%2Jf3SV7%YU4
zh0KthwN);Pgh(~)w)P2e<+gL_-=9Qt*WxCJCm#z@UREar*zS#}@zhXTVCHx7ElADH
zs8A?65dP>cWM{<i$jp^oH?IHgumY>*e8E>c99(l+DwA)!1NYh2GwIY3#6(pwGp|~_
zM4E}oabhgnEVdr5@8jLRiUs*_?6JM{7jdZ@%ni>%v&n?67{v25R=oHYkGza~@AwUA
zMh|9-U4NO?+!gc~ei6?edpb+zea9ysOySmzyO{FNYe{Hx0C6EgLh%e9F@j`A{mZa3
zCY&oD^3~Vaf_4b`!$KyYi!i+d!5^e*$5v{^%e4&K<BYcG&vj|A(8SBPv=TK*MZBQ<
z@QVDKw1$&5Sc!4guaocj0ns*kg=nqc*Psi``lexz*bFtD7`VBSbidR%PX^yM9-90p
zGk%y$erXlIKkE|CzwBCWzI<$ZUmWzRV76FseYJ`eCr=^I;bo_g#kTt<k`XLp%fG&4
z^A|PPR(?t0TV`;1sE!q4=G0wmxa$*^y;F=ax||i?trcoGK+Td$Ql1<}$!lM-@zG*Y
z@n^E|<vD1^5Uz!#SZ{V>?a`HEAKk^=hreLyr%SPUg+S_@MDBS`U1`@YT*2~3W>NZs
zsDUe{vGmw1(*AV;gAB6UuK7yu3&rrO>&_tW=?}U3_DysSl~Yh<Vd&K(G2877KIc4s
zzvUzDyY*Q*<v1u_yOU!tJCEP=a8euS!YK)MUV3Z_^U_4c-BHYU9{6nDW-hA{<s!B%
zshh^R{V$|s@sCVhTSn10OF2IMSTYjh7(OP0X)nCNg;ULxm)FwgwDY7b4X@8i?#NiO
zr61Hf@z59^McC_G8mMb4%Nx$$OSz}9Mbs>Au$@Jl_$JxLGilNI0v`3WRwxu5hJY=N
z%decoM}K{h(f^F&lSwBMTfB<5o_&DX{cm8BKbmcEaSR-FGJ|Z}d3WUu^4!C5`8E2T
zb}~zDdWZYwj^xhnbv*vJzwmPJ8`&^ChoYilYHFe=tJUai^H5MwLSWZ+N;#g<!+SIS
z!Poh6!x(NImBpNIrm@m-8n^bE$rFEmk%2cn$JOUvMES~TEH5j<C!VPrK7EVroyTy}
zpiV71jE6Ns04o{k>BQ~jV`|a|W@!yR)cPE`GPMTxX)UNHu-nJlY0)qStfcqqL995A
zOj|P;bqi8J2qQzNaY%%)F~eXqV-nALX}Hi65BjKc`!NeO8SuF9Xf~qkCb54Ik8XLQ
zK{SPSf0vdg&ExU-X*j*Mh^NTkO(T#;E|X4G;mH6y65F?LCtX}Pv$=I|eJ$i>dtL!G
z?hsC!30IAq=%iFKQsVa7wz#-hj~*Ty8@qd%C|BL-Oc#{mBK-9I*J5EJ12a-6uJq%`
zNF=Iz0@i>Re?b|(I+Gq4GvsS1(<ld8hNuVyHTb-8=)X|R-ugh7vaidS6^*F<1S>-I
z0W||bDY(UpgNab*SRCTwiyDG=^Db)gUHENL*t*3L6>GyF-YkOEwRrMt#5QJ({s4ZF
zrq(q92ccHo<Bb*r?!sa=?X1w%dtzb&ol~M3q6~;SP*cu^{4y#<xkZbcx=U6f7E$Yi
zLd6HFYRKDBiQjA@Hrgo4RukJq5g+tWRPDwqvX+z-FV3xu61Q|Pbda7P`>SPePZtGS
zODPkz)fyi~PD-qPqXo@&Qamd-=ah2^T>d=&zUCCnJ~#U|BFQ%g7hm3SHn*+0n7K2q
zC%U>u4<M>gC=>^qIPpA;G%!?FR_arB?Rz#3AEooBa_2p-kZ&~-tn!i6V=TAbe**)O
zJpA{;Kk)cBB_!u|B0JH~##yTv_24_)IVPJg{_`mJzxE?3(H`8rPvg$JZeh^&&p7*!
zPf#wNx#K23#-!@G+&|?LR+PqY;>FkU_${Zg;=R9f`&)~3mldzk$@zEvn`;MG^ZLEN
z=Rfm3#8_<vqLaDe_QyDXU>vVsKZ)rhZshIDhcpsT>(;HKbLY<3#mlwkt7Y|^SuEaC
zP4DARWN_D{M%&hv?O^$W<&;O~FnVAPTG2+9tlx&U%OHmI=tQ78kHrh~h|bI+D?5W2
zPXWs|6k$(HCo4Icnv!DN&J;SwTd6GD#m)*J@u?Z4#@K0Ofb!N+URI9R97T-NLcm>z
z2Mf_o8zFD4Xceq_cb<$(6$pi6P>HMHtM_@57Ov2t6i@O~7TA$0p4vl8;;BZ6n{aHb
zrOTEPlMqL*o;~miVX13sf5SLFX)ZN16>ri^8Ekf)zFzJtZ$zWk{JPzqc7}R0rc~v!
zlf1zhT>ov-_C8*l%yI@jj#Ky$kujI#y6`D!4eLUU*2#AXDbVV>Go_D{i~}Vrq(Pjt
z9@|M&P(7-;D7Qew`AYvRLuk)2!|9r$V%ag}G6sE`#LQ>zp--HlnLwINErMd3{2#dV
z*Y|Vg>(jWVmx)@XoD>R0hZ9c?&n!<hHB}x$W-B(apQYjXErSSGS5*@<*|9sV;P((T
zTE%*qZh+c~O56bhPSHNu#6BVM!tVA;-_oG|@@xwF_1*?)sh4BO>~@Q8WvQ#I7R!Tr
zSc7P>Oh1|s6g72q1Uw-O;srn2R^N{LMSJHrHpJ_TK+fr!ot>>~fvO$(ESNW!TB{Y4
zHJ#&59!kt^Ra4E%MawBFcZoL3L2`674nrjcb(%gKM(4r9$kgiCx^@kPt^m$XJ?WJh
zho^8G+ls37F1nDAF>&dgNj255T`ZF)91;R4EhP!3HL{FD1YEUL)Vc^t3zcYZ9F8dL
zRueuUq`b2IY-p?W+=zTx8XU;|!BG7&X%*X-cxp{BDe=^@c=Gvtx(9z)JZ;&sMYqId
zW@f61fe$(cY?fM<&Rj+EzyTyhTleX|qzliowanTQ!}#Hu)cS)MR6!~fibI5a{vU{V
zIueN#PX`UXn<E^?K-0xgC?H<PjfMkCD^gr(`gCZm@48Qi@uq@mU-R8H6;Z!bq&SNd
zXaffhR8NmXii4r!$F}onb_^4;>c7UQ0#qmzii64_@q^w=>}WzbX$L8-`t(9=NN%6v
zsjZU(t4V<s3WY+UP=q;@Bbblq=~>z;td3-YE*Djrg*e;MW?HS_$G2Az=--{}iY=^8
z?!km!@kdq`{B=~y@CGsQL^;}P77nrLhefQkB{IZR!P3$s&O9{>^$MU+C=^GBz&Y&V
z>6ajq_PXfaeBQnJZr)tC0}ocR2TWx0<Cl?c*pJE(bhBpG94cI*xHTEeqTYGiXip$(
zWOqEv7qFw;fXNg@1~H9_&!orTZbXNp%9d<p?x(XU$n#+7)t{l~9z(jtK>nhIY%i%J
zZuqhEi8m3dEoIH3mFzO7GHOIm?5&=wi;q6uOj1k=zS())d_rb~cnVRweKXU)UeB)o
zXYV}Vqpa^g{=R*=yX-y5WQA?m2qKD#TDPT&T4$}*T3c&f)Vk^(*w)t8-M>~*6crh=
zmjVGofRK<KvJ;Za<t}%ZoB!vzBoH>FfM|d3*Xw!R-7|mBxchzX`}>YgEwPb$g8C&e
zepD(E+P;4o{FWwGFWJtHIuDVtAp}R67&vwaSs{J%HnP?1;kHTN5&OF_{Pn)<6TK<a
zuBzgv{b0Up4{sL2rROIf-?QP^OYYZSu)MI5(AX5B<cPzG7cwSJcuffe0znV5PnXqd
zJth9n{_gYy-xq!QwLex}yQ9tj!HB2bUy$2o<d?7ghp+y6ABN4J^5mkLK3Yi3n7_)W
z?1tskyirVhK`jnRM3l^0$f`YdJZlT6x^^kGpO(<RWjA$q&tu{26|~CRR&Qm&)gMr^
z45CxRQ9ZJN)wj&2v0W0g%ePZ>#n<e7YF*$kP5)U%?(|QoTv03MkUiF8%`l~bVLB7B
z@hWuw9&zlYWpgg)UZ2bLdxIDltYPi1O1bKl3JL?U1wI^>c8WK*vN5-nM(h6CKOUQn
zvSK-rr=04nle$7PJ9f2W_j<55m2=0f%en9y9}$@$w7gc$HTReBVS}yfi{Nq4P}WLr
zZVSZ~)<djB9W53%Z)&Ba)<R2@g?5LJ;C}s>^_vj(NY}8n!p!!(HY}b4o$*zj?G)s<
zkeg?wzO#28Cl%@0N(Lo_V>AX~^dH}!1E;Nm2XDEINpsfX8j_9b{g?Uq^|QEbS64nM
zpT|zk_B=Lh*hqfS-u=&$&p~ZT1yzk*>p2_B*}Ns6JuOy5jGsUt_%RX{6%}xdZ)$4V
zS1+#jBnMR;ZE`xD2TXDl`;jJ>%XPpc2W|D9<n&3sCpmDd_Y}w?e$W&s5p$U7xGhir
zi}(iJ&a)qC0~biunL$E-EoCk4zU@pK&)E-NM^oE>P`-3K7dJOhXY!zRET=Ovh9P&I
zOI5+wWc>1GGKXvMU;YK}{A@S7o*7Td<7=rLJc3!TUyPh#%6;WvR*Zj#ogbgYxxXJl
z*netp?Ji`yZv?uQJ(v~YjJxJ6LXIh3wTzmQfiA&BYQIdL*L7K)RL&aiez1_xn<wJD
zB{MK@T#S*><qixQ4N7}C|GM)#*0dM{p`H=L<9X`oiS!RwV=v9)`MZiSj?N+@$x80$
zjcjgI@z9qOVeVG`9hyqRo8trLp&D0>_V?S-o8$^j9(MnG1uuQwPK{lU%kSofhtB5u
z$?<3;vDUsikNaNO!SZ@F6JnvQ&c%#3uHf#m27;q?Y<P1Qk5`tnGtWl9yGHQ02hJi$
z+ODIlihs=cj(NK^1WST0E-8vT9vRQXfx&_E3d>665q&s=(+6^Y-0?(TdRYDTbG%Z}
zk1e_Hk*$-yL|2@H_9r({5)~W}jB8(hj^|(hjJ6a5?Ry7v*I%FHm**x)&rk&)KJfzI
z<~1?y&gs;C`8gl2-$2@L|H7ijZzo(BtpozWj|`no7l^iNX=$Ocv9arqq@1osp*Z6E
zBhBl}I!F3^@ayr;>y7$gYf)c^=j0v#sLETYwfink?%!VKv*I}Z{6b2fcjv@SP~Bc^
zcfHGJOUf|6cq5n2`3bT$(`}Xx=jx#H=>>G27e?E*a#RyD$)ufzwl<Py4Z+mCA5Cx!
znbU(=ySJ4VvmO7mEMl_E6x}wTpd16Sqg8k;U3RCV8#Hwc|J8`@_d~g;=d>GJv8t6k
zaCXvu8`abS+&jiakVYPL3}M8CRL;6Ei~-JOem1k1rFp~Xe{KZ9+1dQ@0X^5v+Qi&x
z*?joMx!m6(ow-KIl6ot@Ikz7@DMoz}30(h*4!2Gpuy!RTC35LqF`U&uh!0-g#tZ+h
zW`-o55^d$&e@_A4S!eO>J<+(9)pGasd6ZZ@-4-hkDqRfcJu;j}3|0K%$`W#anZs0V
z3-3I-hBuX&tXVRGXl%TF$ue$yY#SRsnnbkpyqboM<m$j)B8bd54vVYDOkHU$In#gJ
zC7k4I><{Id_y3MfBa1*c@d-n@?UhLk85+fs7w==vlb=iC=^nh=e%$)!Kj51-g2x_g
z;;DbW%(qX}u$%s9gy~5j5d27y)A-2gfldYA=dfL$El&pvt%JqWkHt}}<b&Vyr++Wy
z_P;M@T14Lsb+V6>%cRGcGn_b|g}iV-kwMXc8{knX(YCm$*EG@a$vT|*QKS{0N1URC
zYQF-l!ryCrRj%8k2o&0tQ^e~%q4XaTM&7+k7&q!_VzLXVYOqOiB)V@mS5BhpRO0gN
zzn6ZkiOBA<cWs4r{HLIerix}}Bz4l}Rio2)wN9alAR$(o|MLvSzc_`-dPTQ2tC_Dl
z{P^7apBKMUg()m1a6U?RGw<eGShaF5l^3YkR_ex-sR|??bMC5SwJC&^4-6$i-tf{m
z-dPcgIXNV-t<C0T(j}7c%1H>w#qrU#RJVIjhS;fcJ8?F)@!&0+q|@@#WcE<&va;Rg
zVOnq>@_fm4l{)2e!ux{<<d9ZFSbQKeyT-bQHDwEUd!vT)M<-CUtA&slb)Z^|T4f|6
zIg62_Vz}karQAQcOGajh-&TP@AoxKEr~gVk9W11JUVlv1rycJ+%44q=Fymi~xGg6h
zxv;grk3n}TuM)S(g!=l?jJk6msUFR~E?Dh#V^0iaz-?EPl~Kl*6PD4jsR^csNgcD6
zvQ0I#UzQrM+W71i>WW*5xhxc8kONJglBg$gnD~yK_=|^Oy(b^L%im`|8gE<(kqS4T
zZEa-6pww>bij#J`7rjx#j=vZ2jwAxM%t|9p-^$7r8xP1Z?l!wgON=B%zyHDZX*DFo
z==sa$dLFnbiKwnrwRAeHxP!uoEG^^pMNK^U$W*2b)sW-f!qN&qodHXm4x`^fOQ(W_
zZfj4%AOm`>eDyr&Gzt{^GJ{LvMp82dX_><3$7@wFbw(C<o)<2Cmb0}F;pH$ji9yG6
z6P@f?Hjnr7JzR0qHDpIg@<<&<a=&nL7raf8a|#3XUB~r0Y}oZCbPL|)jkT@JeC!pj
z%QUbtO2uQ#9Q(dx3ZK+ohlw#G!a;aC2?PSc(FKy6)m!?hR7-!h{b!gx=D0mAPSgSX
zhi-XlS@<lopMQmg116Jh-p(VxeVB*eDeU6|;HGv(E|x_W+P_%J<^~5klfG+MP3ll<
zb`;V4O%rv?${{+3sn5mJ{QLi7QGuJNd&iOT&F3ur*9IDP7qRJ2bJ?yON$S<nSk_e0
zxO+D{OC03<>>NT}m6Y$O#<RJCHgDfLXOAI`hi^!s<mpvB{K0mL^LFy|uf8Js*V|a{
z@L>p*q<|#gyd6!f{#yb0tLzlySJ2>e<7zHr`MNs3UDLpdHM>~9rUY{!)qN;4XAUEK
z<u-16;#;=lmoRV6VnQ!p&+KX!a^axVMh6|8Ei`N`;FB+__`0s1yizmXtPF1Hbo1-0
z7O{FqIqN@K!nElNd1;Fkzq^B?Ep6;9YNFENMOoayyY(F`Utdl~NEjI-!&$krlqS80
z#4xiYde(8<ONH!l9eZe6Z7$*8&)&*YkNk#D*Hv^|nY2v3=0>iso5ze>?qzxIcD{M@
zX^f+$6TQt#aD<9Nw}r-zb}H6>&cB|RLu+k0E9=c*-OJ_`U$bOYJ?lT0&SlAJ3aVPf
zucJU9I4QI`6#O$^!G!ss|EDy)0R!j3tDB+TcJ%((-_gqU)r*<;(R{vMo=f>&JD$D@
z9(OxsTi3IA(E?Vk+QgnF3w@2%x32hx4_<nSSLg3M<nQM9ZKzSro;~}Iy+zc~e3uTM
z-}`fqxax7Vl9!!{Y3F6n|KQ-@W45jOyPXuZs7N<h*s*0RrOps0oIj1T`k4+{SB@Hx
z_ovtE5C0`yWmuF=6#Z1XK|o1~C4^-G3F+=wq#Gp!mXL4>N$HT>UAjv^K<N$%>5!IY
zfu&m!mi$(If6wzicV_OLIcMh1xqPB@KZL}e$UE`%#(L-tCFQ<_@t0D07e>_Ywz(7)
z14zCd+O}>HGkJ882?~i<?(32BMCHe5>$I%&7hiT$%o7g~UF^M%iFuQE@l9tVUqMRr
z*BuN<|F&TQal|&b-{XrWTZ8&H-4d`U_ws{AMi(1sFD}Z2=}(ZYQofva%2*H6zYmPw
zEezTH{JpmOq)Y1y5r>N<9nyO3*k!&`p78od(fXmuY;W@!wL3^1KA)$MK{<kkQ#ob7
z9P@o^`eXVNDn5|D>n4QdINVX?smJN&qFAyax5!oVV=D)GKigS1WGWPtTH4oKkk-+@
z((@_S;^U1K-}#tIm+8va=SN;Tj`|6r62S>;2KD;y+%o((%rd`W>lQ@LP4r~A>{}jM
z2~-d~B$dq61lR6}I92-$YZ4>sNH;iUUn+W<8J1LVJq!8pIa6)~M*42AtqC~9yOmvR
zVSS4L=nXtLabD?pe$+N{dXQvx_5I6tr<Za;EXg(N8EH2^1e}QD@<hGZ)D8VVu)iaE
z7y*cU=^wNjNG2+$@f%l+CUIw)8e72dYsV=d&n7t(wm*;5|335am2pyVpx{LlfW(S8
zV$S;reCsxbYHf8V)$~B^FIOgrZ@Dmg^oWEjVrLm0iYv3;dEctu{_3R7+*DHY+N$zi
z?rYRxhiBP~A%9MM<jAXdn%|T4^&-#B4iU8Ze%@2Ezg<Jwol#mRZ#CpwB>kn=v%S77
zAlT5kMK_3k&&%ga$@!}CI$-rE8=-9ES<4$5-~M`5r2eW^hKVfmQ2Lakpx!(~w^M)R
z!uTcdrg!K326y>x@)4QP1cK<b026;<ic57zNK;2kvLT&L+Rqfx4TV84j;BSVr<gb>
zAll1C3^%MP+JjD5cW#Zte+rA81S~^SdHJQ_#bmsjDvlbVrxmRP?KDQ4X~_?J>;~lq
z8WhC7Oo;um)IH=edwNo{g{8#fi9+c*?g0daG7=w9{62#aM5I-qTPo4hJvKzPG8%?@
z(C++()D@A)vX6Z=#qA*_2<W9E-EVx(qT-PGU6c7%u{9VBPg)bdf^_dE6!Q`&+MS0Z
zXOdS@>zjOuNOXeej6lIPZ`@NdJS=IccFx35-6lDb+(T*&m`L?ib10mCn03}4PP|oj
zY>}4}(R+BOsCQsI(t4j%;P%?4!4<}T{PU%>s=ia87!T_DWEt0J_M@$yEAG+oP3qG?
zpV&Eh4MfE?B4uWV;&o(?<KF6;vOL7dtc$AQPi8>#WkMnbB7Bx4w|7bc`s^u6J)MD5
z+U4Gcyhf~e6AFycy>G6I#LfvArg+PjX2R?H2=(m1&X6DS!f(}ZWx6p6y;4LzF)y>c
zS)tLm#$nSJzJ63&>4Ea$I$(tQ@cKG@q7m|>AD1nbBAnFrEqKV!!3aza^`^kXx*xPU
zbt+4B6}u-j*D~L24R33GW=`<}$7{CraQQY$ZJTN(B){!m)@;vq$%DIeWvz5Iu)%!-
zQ*N%8w4|*kTDP~Oz7mLCRCfpHqxr+(U#0<_C~a_X4QMs2qqTP!$a<z=)i2`etw|@4
z`c%}7lPyoTRAqfl{K;x3Mz&)%g}ugh1Sa!rd!ml}qLnS%^vxZ%*W$@rc7SK4hhcnt
z)zobz(Z_6o*fKkFCAG>}^J&c>x5VGonv`|E{Y|TKAdw@p{LG5|a)>w&Qbak)uFu;4
z@+EMI1v8EJJJY_VZ(7)rMZ_z+UrPvr8D8ir#YTuz53s0BPcJ;@cJ;j}X_+<rG<A_w
z{b34u<?6ra%IGhgd4Rq0tYK2Upb#44X(!XT9eh@@h9j_fO}LXo6;?eBWax^&!V{)3
z^cJ37PL;9N=BG1ksrs>?y+Z_TrgtiHWb>aw5ykSUInt*2tn<<w?9if*FR3`?VEvdH
z6vphZHP{^MQ<*O-T8`!nAjnPr$lhFmp5@8lJ(FU)a@lu++EZ6&tLVOT4xrY33^XIl
zw7SNHwnCdx0N(bm!J`k<bQq$?$fBYN>Ze3VQdbZiVXeCcnRTY#&`nK=^Bt@7ePIbT
zUW-_<Rk{2GteBM|rGwqwJ%5J>N?ziMR8+W(g-r<+*tUon9jQPRf409D&)Ft;pV5U0
zJ4gu>!qPSiIp4#JU5>OUJ2;I?3i<A%N|f}=m=!{T##~ZTz?tKwYN2Zn;WW>nSgh8u
z#i;5h+7f3*aB$Bjs4Fe>kHvUc#c>1tOD3wlxtrgP=p~vP(^3qC-Nvdh+0}tIR4ftU
zis<oQ?yo(H!N^noZtv5&!0PJH1?rh1>F;jSuFwmYOSQ8HHrguEhu)qf=VpA_<yfjf
z^tJZ}$YQi@YNu=cP|YEi?`-{Y=nCnx_cU35#tWZF3wLNGAbnPvt8>Xn>BQ@J^5xP<
zogo;&CW!TA?1L$V+FTVgV#~Q`5UGr;sX7?CJI2&q46?o2*yyo1l8WV2K5c1M)c4jw
z0J;`K``F>h*-B=W<htK?O2xcXJ_Rq!sl3@hQoLjYWpyI4Cet|c<DTnxM!A;jVgH?`
zUz;{XT*sbW_vb9l4ksveI?9RG7VhJ6d#WGF7objY%(>NQx5kltY)$8t$;n809sI$o
z;J#%eB~y9km#t`FR2*l3=cE?RW2DvOHK}oPUTD5%PL1oNm)=YZ&AUTD`pTWa7hbcz
zLmhz!A2SuLN)mqz)7Od{o-RKk<opD&*_(LhXTsQxt^qM|fV+Q>lo{#jX?%-w1}?S6
z?ovM?59t-C3w&$<xho^2E8+Ul6;8ozJt~?Z@)lp+AV9gP=%Ky>0lOhScgK<vd@fi3
z7&VMAqUA2CPnt&73RMXNHUP-rQNj={^o_XFr(ni7TMsesxlXG>j-}4DUQ$v0zY`xo
zpNSa{>WE<Vj+2_wLh&_lX@WnbITVozG!}+Ji@7+n?x`|IH0wHw(Hhsh;#aiAm2wtj
z<;WQ;(P+X-&PvxDGH$A)*6-yDSGcryDB<w@K{;ib4Ew^DUybh1_5p>g`*Nk2%(Xia
zi#sl8)5H{RY1lwbY`#huboh#!5^|s5e&ZM64n36=rkKR#=VW{ArAw3Hm27!Jr!qKb
ztD+`y7;QyhI)73kgcVjy=3J>`KC6_V7R1Yyw0}Y7$H1efKcmj0pHC9YA}UM}vN`1#
zUl3iq%(87HnuA#|*D%x+<J^W41A4}kW(@@Oh;wMaB!k;If+?yD!Uw$UdwUZ=R(GHv
znDsXd?AUMQZv2GIJ2}1`8-(Q?Xkt~za41|g6+gL#zW*h4(WkGa>&b8_g+1jKRm?av
z8CA$e244Rc^^7dR*Vk4eL8a>KXkWDO%>_YX;0o<}WAAhv<vTBSk|7L<2$DJO7#`mh
z!2`p+#Y9#nPd5DjF6tPBs!vaU64;CR>m}^`uhp2PzAyaJ_5O|mcG6$tf{#e!`{Xq>
zHJS3w;{Hpy+UoCD(b3US0fUu_CmCKlO2@V|=#Fp5(b0^(9|pwC3tag<K(l8Vq>mW*
z5n$@BT0RcBoUsp^lwDY+E}ws8#Ul6pQ@rH%JEMq`5Qd)kP0r<-{nehdE=#jNgBGkC
z-;I2Qxg@Hs!=zwG+4e+|2emiV)g@t6+*Dso^ZEQT4#FCbz{6FR%o%A<>UhKn=qyx3
z)mIC961YePJV_}`1#5p0A24kqM^&m4#qVeA#TPl?5_WJa6wj(zDTj=$11yh?lx*4Z
z#AMOP$sPP>kSdSH`=wQJxM#@F@163{J__)?CCqk9lH&i@r2Cx%eFvU3uZO5xEqLJ^
zIsWfuFuDAMcxBEQg`13wp^KAsd;)^qy1Eaq$5#3>yO?f|=QGR@RB`j#eNSx7hl(AG
z-H~VT`z(1?<?knOR8xzHS-6?$!y4e>lCNvGtyMn6bwwt@s%-_?AG+adi#U`)sxKqK
zS~8Pns%{za{RpUzoXuWJ%1oD;6naBfY@Zo(7(|VHBW_zvXzfk88)qQa>OiDaG3A+3
zs6(U>YQJq?vzEe&N6aplISsT(i+^*h0Ahqw_Y>~|UyiM1dDs9rC>RUBzHYFBH_YOY
zJ3Y&t<0$bHIq;N#LC5XSwrh9aze#*6^oi{xbT&jC9FnV-XHZ7s+IoKFACy@@5O(@v
zXfm-JtXwU1CK7WV$<Cbl$HkaLB>Uw(P7{`XR9TZ!D4F;FRElRD2`8p2DJgmNJy-Th
zDSYZ|<@AtUpx}w0I8&yLge?BzbxjMWdaUxnTb&c+ki=={{b2$Iu&$WEVPH6JM42)F
z7U`l3!%)Q7h;4dRauctVEXYklc<Ws)Muu#sp@Z#kAE|I<`V<KHN+T7f%-9%|7o}Do
zP@h0+O1B%9Q^kC%H0n<M6P;3a%wDu!V-k5F9Zf#YP;Qj=!C`88QJ5#}nk}cgR8jW#
z)Q(NvVJ~kTD*p`<V68%nslr1TGN^7R&;PE4?!>+p)~rX7ojUq8#z;@<`9WRM)SMY7
zo8^WsO=NTwE0+;Su3TYfuW)H5&V(64lXKr%h0iBTm?_tfu>qRZc9BGYfO(7cSa<@;
zCQlF!<nkW>Vpl3XJzYghOS;^3SGK|co|HuOxK})v*9D}E6m+*x>%X^WXW`HAXv1c8
zu_h7MB2E2zg!lAB`7OFzgo;=CmZ>R=+zfgG9DOUKvkAjC#Az5#51pKeo-1O$;;3od
zD1RJ1SV5GH7T=Wce8fs;`15R@0&?WjyW&G^Y*|Ira%md?TZ%{UZ@GePDTR36gj+z_
z9%V{L$;#K@5gL%wKML<8la+&!xEN8&KU=TzYCPuPm{{l@D2}!I`X1{St^M<i6D3>s
znZ}b1B5-a&UUVA?DaceH$yXdtU5!n**_Pi#d{fCWB_Q#w<p4EXR$^r(UvP+?h`9L4
z;L!C%J?IfZ9Zr^09t?Xb5vbayrtRA3?1P0M)BZO)K2E}P<KumwTmBr>b1M1C7z@SH
zhN*${J-ydeZjxO}4g&GL%*BHe66`@OXoHWJgmXVOn3O1ZA$b9mL7GAe92^CD+z+g}
zA5m%+j=zWO6M;t50D{(lf{(T;TtgH;TwL|&$zi#=z2EZ2A%zD-fbbo>CxKFrV*6P@
z06gc5SG|BD%em?I-E%=r6Pik=e6SGc1pkes`xe<X0TnroXp9DaK}`{MOD#-ot8>Q_
zS!whlw3m#*3K3Vz#wC^36AD6dlHy+?#A0N9^tkG#Kufn=&l$Z|jihp(zw(FO<Rg?R
z<dv%>qFTZKx-3es)#2Aj>myaCJ;6kUXlN!)UD4K7`)0v?_`6L(buaY1{0n|pF(JwJ
zy_AKh*|MJR&fJL+BoF#_l3BnIhKQk%in6fW_+&tG!%+X)VGk1Xv}a%Xvnd6ed*_OU
zr5zDtR#azeMwVjOKGa^7CX>;K#WY}kex8I(iy#LsT%$=LW+Ap4{i9YiXV5|5U?%Q;
zAY2WZGQXE-p??(`?{|w6^6|=kiNYagFVQNeLw6R4lIl)S%M)?WE@m~(R`B`T4ZKkz
zIXO9d7Z>SGpXwicw6VA`MDLyzyMqn5&Trl_7S+G8T(A4N`;+u!cTkQc0^F<95T!qz
z74$lp!E|n6lVhS#zUagnRAK3&XgXWH`|<rsg4`sV#PumBqL5BcQm*HycUFC)bWsYi
ze3SE9nkD7#>+p7TttBAb#JK(y&FAknIK7?~If+?vwNsyiY2P-`!)sM1NrXQygngS+
zRX3Kr{H_rmI}G4wzhCD!%0(<zfS*#3MWU4=<6hVBV!?i{BVOLgc*#n&Xg@$7H(Z7a
zMkZ<N+|q5F4Xz#iw_^vx<1>d!KYy0u19}Ojq+MS6$jQr-R?48M$QRvLvM8^2^g*p{
z$$g(6YnU#ZxDSan{)+SNs)jX=UXUN%p3eTTi(Q-aYL^j3T7XnvtWR=0lW0mSa*+(l
z1B3-*f2f+CA9&f87_h?tTyFjVd%^HQJ%WFvDf{&bYCIQWebZy9p%V2<rZOXb)Bh{Z
zk{(N~g*)LVP7;&8^ldzL(j%I1BGjU#DrMRbJ}qA+;~3DnBQZu*0N%2z4xAGOIEa@n
zPKJuRGs)5tlN7{xstch|20$L)DyYbKy=Y{7B%rW9xH;OPtGl)~L;SlUTkfgPcIBK~
zOB+<ge7n$bBS9f`ffbSkCgrNYNvalDJugg*5Mw(iNSE~e{Y13u*gXCY=F|)OrgLQ9
z98$1L(hjlvv;zNTg&*q9I2t-<xbNxvD$u9R!Vmrj6uWBJ+_8#E+;(_`C0=FC&x=6<
zK`bm?q%B|%j}fW9Q~Ar%NI_7xV037>9KpGswf;v}R2He5Gnhinna`S+K30~3w+ZDN
zl`R3^I?R#{UG&FMgl9=oYFR&}vV8)7(;hTVKCTe9Q+Z|H2-VimB<_2Zq1IZHiK9rc
z<K_e9FSY6EC7T<IoNaxiX#aRV4oolhFeG2|LT}vWjBsgIBC93axyWL)>D7Jmbbn7S
zIo-)9;u*^~{MH^m_?m+@zm5e}B}(xK2s@AJ*Imn^)MHpB1hMLLc+GQO$aNE4n2`W9
zLz$n2H@ZIXUb>f8NHC$wq8eKmf9!n&doh7k8{_1xqb`snNvqOqN3er4_#o-idX*Hz
z=qZNay&CPQ#diNajS;K;swisQKWB#G4JzOLOC?0IrBB)6JB8PY&of5$m(76(V=7iw
zEJ&m|iyGr7oCnOq!((G>OY!jG&u)rT9(~}=sr4W$Xyb4woWTl(mWmbJT{k~bcxX@X
zZ~H#$w}SL1?8}T6!(cEOD~&mC=3?FiWfL4yMtZ!CJLbIlBTHoPyN#yHpOcU|JA^9%
z{hc>Zt4$GBDZvn3De(trt}2M6B^NK>)MmPwjS4SMzRZcq>pW7-yjc9P55v==<KH0-
zj15KoLz<m|VT)Q%+WtEx#qH1u87~xl`{gWZIrX7x_Mh{WKaNq2ZjU&)EcEjsWVd|x
z53;(7q1%^&r|w32yLJJ8nsufHWyZ<;-b*Ujv*}h+P{0}(7{F*kGt_q&_|O|UGhTCv
z`qn@BG8QubI9&Fu!=HgFAn>cpFz=)PlD~`$Q2!&(c<nDYCOFY(bdXINGjnH<iC48z
zwMoSW3Z=@z1xsXAriZPo8rMfkOW#Q?r0ZirTHM8&={<XgTQjcSG(Mh$(9<{qpB7H1
z_T+}HPIACnK~5y3CI$C2&zmkYh3Cb@*!btzG7)OTVmr+cA>3%Ky7JF@TuJAYwd!n)
zkiVP=%upQ$S*UjoGSjl|i^I2}KZ;s~rwnNOpPurV)+fN>Z{N!|aaiM=TeQd0SpYZV
zyzaWT`Gn3K@4kKDy^d~?WNr=kIk!}|XI3lJFcB2jU*h^}LLy};d>Y9Cn0mk9khgTe
z3II}s)T@VaUR&%RP?u*hPKCc<(J=2jKwy~{%Q53OF}cpWmDX;%Ro<4N%fUIL>yr_$
z^i)|!QSy%8^ISy}ei8eG;KxEuvPkGaIkW}*+y5mpD{qNrg08uUeX^*FR+Nhu9&*wY
zBnkg&HuGka1t`yRQ@LOHHHudV3Yp>?a**fi=0WG|AxAP0Jl3gPVx!Df@Xp1X=MHtC
iN^`>-@r%HFIRB;YYZVf#84mu3`BW9b3ZLa(1^*AmVspL#

diff --git a/modules/web-console/frontend/public/images/page-landing-carousel-3.png b/modules/web-console/frontend/public/images/page-landing-carousel-3.png
index 91946c976f38fa32a7c46bfaa8b71cfebe6b24c3..f8109f6bd6bcdc9ecf93d738fb440619533586a2 100644
GIT binary patch
literal 27808
zcmV(~K+nI4P)<h;3K|Lk000e1NJLTq00K_{00Hp`0{{R3Loukt00093P)t-s|NsB@
z_xJhv`TP6({r&y%@$v2L?e+Eb^YioK;^OA!=KlZy>gwufWoFvh+TGmT-{0TV(9+1r
z$<EKu)YR0=%gX><xc_x?|FW^z*w_F70I{&JVP9a8k&>gJqgq;8PEJq%{`&O)|Ni><
z`u_g_RJLwxZN<dImzI{nz`=rnf&b^{08Fyh)z*)Vjr98b{`2#OhK9Mfx&QC)|Ka2U
ze8sJ-t(lpbR=MDto1C<?wEXt<rlzNSe0=}U&nhV?|JBu=o}M){HUHk-|Lp9$y1M^D
zJR=|?baZvLwzfk;ME}~_jf{+nii-cr$-TY3|28)N{`3Fv&=?vV0h-YN#l-(rQ&CY-
z|1T@5sj5CbJ?8xW{q6YD{QrA*c>hgG!t?$Bi_8CTZAwW<O*1V1{^|bh-2b?^|7T|Z
zU|s+8<^SHf<ns9b)T#gW_-0W)|Gm6ZB@@%~`~R7j|9yM?x{j~x{Qs@1|M}bhrKbO)
zp#S*oA}&7vgM|b&N(&$}0H@gh$(rP>gK}|l)#>vqL{NL&_k?6i{^H61z`_5LlmDBY
zmh1lmL2dscAfMat^TnwD5El~>5y|B8gTmwfvVQoXdk0f`zkyp;RaO6ujh9eGK2Ccv
zKyUvvQGG)~x7_UggIZ0^^Yh}>|Lej4Y`yBanwQJv16`*(CMV<0!~uAoiE3b8G)8Vc
zGvMv=0JY{mx8Ve4gatlZfm1fAZ&EuG5Tt)<GFW2($`$|L_r=!W(42LtQblbr9M|RQ
zYQ^Mlw%28)#|2@P|67xYH!0r1xU$63V~@6`Vl=MM<dC+}^4G?Nnx)uvOAO2Pt+vKo
zg__-uZ}^mL|4d}VYC8YE;J%7?K_^GWr<nDrXih<70b#nuwXiX6im8o+94=`87Bx$u
z#{+c4g{Qy(i?~f@l>c9M<da@bU}<@cno@Iq|F~<LP$w>Ft^cwBE3)0AvAjftuGP1x
z941mOlEwdnuK-%f|Czx5!XE!%FaMCzABLj<%$xwlZ~rO{wGle6003!QNkl<Zc-rim
z%dXQf5QgKF*iICn6tO^DSKK6!*dT#m2NJMhja2ajtaveAk?~~AA9rw_DA56W`X!CW
zo?&9UKaLxn%GGDitVR_srZy%vh=IC66Rc=(nL2e;or_{r{!EmBD5FwjZWNuFv8i4c
z2999N-Cvp>W6Z?qDYMy(ZKg)kbUm4H@I`klw6cBXMA-0=+FY=Fa4_teM^$T#GT1i1
zCC2R{K{Ue|PUv&^z{$l>MH3S&q=<J?T1PS+h5U%Zjp!m#a3;NmlhILx2_Op2WZ-Ig
z)YvdRgic|S6zSM917}ZWdQ4Uvkk=TI*IX*h)7an=lV{HS|A@SzDyK0|Qt^3*bNqM*
zpq|$8va=ipW67P?ztOTNge{+*GKj+0N=I2SSe)ijVa0}&)?)4Nr>uy7pgS?FUf7>?
zKtI%029-6|Cg=&2)#Q#fxtUP~r@}3da?5)$Z6K?TZTY}8>bK!yDM3%zK~52ogO&8$
zWrhF;!rjcTN9T7cKJRpMzhON#H)Cvbb&wewP|l_5l^r{FI4gnm2^%#49N+;B7FPzi
z-)z7ML&C&BFkek33i*>8!9ofZ1t%Sd5=S;ger%ElPRA6+R&ZtquBWFka95h1<Bvl8
z23$o?<2QIQzr%l>;}|;rFU)`0>t(|=qGi2cDLJvZKu>rHl;G&h;k}QRoIYHAFyIX(
z438)};pK)H#Wt5iVh&B5VgxJkqLjc~luvpsQE3i16C>#~Qozsy&rJ`LnI3&73>-G5
zhd~NGaLw@LMuFv@hj`aD|9PkO1Y>-FuVTRlaHCueSN4{p{*7*^TrT5X3rA$7rBsD|
zbSxpLvbL)DYBnL!(wP%NKMVB+3tED@uq7%#?+7Fy4u#Z&PeUgWYRv)K$%Tr7(|aA|
zb&50!PNYB-&`uo!XWD(BQ|Ph5sm3;cdVFDe3S-O8otBh5e6_Hc{KIeJ`b3y9shXNI
zCxz$B7;M&nfv~WT;<BtKj#O*YSyU&8*JQTq2{l%!DSC=hO624ld*(9G(3z6TOl_)4
zdcu2GEJ@CGt+4&g^>7GeXiGQ22j|KuKskRos_dK=zv9^D4;;oe(jL%-qwD&PFLd?+
z3;rZ75fxL`lW_r&Q`^}yr${4ep;P~i0ivIDbs{Fb&lIpaQRcQ&YaoC8-Zy(LQaA*$
z92{G6QE+l%xuC|kBKV3boM_hVUO7u!!F_YOc>P<?<u*`;FN(?U3l4wPg*)5Uo-R-j
ze*(7ITQsE>E-+BlHOi&qhXQsXmqm(6o9w+n=Fvz#^xFf?Z$;>xc~MVToLpQ(x~-i_
zKjNUSgOu~a=xx1m=}fU%{__M(GpadfE}k&uT<A|8bCd~@*jZ=UaDUWpe1@JxPPXZ;
z*;({$f(^Sp&{+me%6C!pc605?%X^oIC&KH2`^4M8E8-6GDe!@K$~@lfb~kU{xbggX
zR|mZ-NtN^jMMpUWJ@vMqLEF?L`fYDEsQR|0WVY?7D`+O04#&QW!`#05@WCbGZs6*`
zBjV%7kKew1dq+Hb_Ut9`f*52ssO{_5S9jjL`S9h-<Hx%et7;UJa70f->{v}icRT9I
z{`gLa{r%_9XusdT-|yddBB#Ept`HR$Ro!|nV`J0OzwKHl)d*Mow(Bp9t}*vMU%g~J
zf{sK;q9jO3w8YC+Sg*obwc9<u)oeW8Z;loyQ(m8-r=X}`?3_Q3(=Zr^?KZJvD^8py
z8KOX8qL)Y)2@B-}f&qz0EjmFbBqYQN60me*U^v0Xci;oCFmnrEfG@$sf^;*(AznKK
zx0Ji`M|7Zlk~V%F_X_mk7blI5WgL9sK7Z^#{rK@Ed#sB${NU541875Wj~##ca?J06
z46rhS`hbIo>%gY(Qv{oy2TcG`rpK@z(qr5x3bHtg6RLM2%$U#DwQ$e7c>aP|6FL~y
z&xOUMv&qKNd@yy%WOb^qOeR;-6nu&YolhE@Po{OMPu+#gE}eYslh1+2{<G<;SJRK|
zF^7&%N5CiDChW^w;LD|Z9Pp*~spp`0hZ(WBhqP9ibR4Y^WDwUrIa;!2*zDA!qPRTL
zwjnIRr**+x<I@@zu&Jzfgf{B(NOb7_2IvY{$79pJ+LW*>7ouz~!%K8k+Siq4Ri<rU
zS{7|_Ke;bj_V!LnC*W6o$|nO(cISO+1+n?G!Y3Hn&BaYR0W4`h<=KPjore$aOkd94
z@HbD6J_APwGoRf%erq^B^cf73z5{Q8l5&L11c|#2JUebG9jN<m$iOFw9q98W8EHb~
zps1mJsA<^wbm^C${&bJeT=(L6u@=r<sLP$R+NTDKaRu9|6je1AyMQUjx(mv&C}E5w
zW64P~CK6x1tSd1}#r;Z{2Uf6_xGZ&1R^XO?6zTm+4;Z4VEGtpn7eygpj9ADgu@q3&
zCo7aa+})idamXW>mK&eC>r<GnoIu(6)3<c$;>jm6GAC$nX3wS%@7#Gf{WyEWW}jN0
zKD~Iu4k4a2Uk-0D7#aEw?m2N4qxOuFxqS>cUL^G&!VSzI3Y~r0q(L4aisA<A+5^|t
zwlG50=2JB}b1@fyqC-lhkd)?>k`i=+jV_0&=_(Wjs{~baPm!DonWj?ET+rb@#(an|
zQ5e@4D+D!8p}-=A9)X7h;{rO(4ruQuiQZc+C)Bb~z|ZGTchmGrF}i&le2Pi0^~vTF
zUY$?he6s1}b3cq3=djM6eSmmseR^^5GoKC_5c++J`e)A~uv63eL@{Cm7v7tM;FIip
z8YSS9htdZ3v`>NkwEwwK>+|W%xtv8obj|ogiZO^&R#mY}75F48)ZlSJuq-Mtsmvjt
zc5#FD71eG50yjAk3M)lOf|bdq3RQTbFexTj;IOD*K%H`&@_n>Vm4eQR_6bHV29%-G
z2{`?LclS!vTxpuyaSHiFdJFkvSuuYC=HiJPpA55GucmkIOs5ZKjJ<n*bObsbJ$k{|
zrBUnC=#Xgw-lr6?Do4nO%t%jf5lOi|oRR>>$biD2o;L}+CO`-!4TV8EyVl40d^&e-
z0)OkE^@%E>MG+}c;1f_GC?U6YX$tP>xlvwFh4ze35`=^CQi<mU$2d`H-}nUDV9-WC
z#4y2Ws7joqI7!S7BU0zni2-GVIw79`?Nc6&@+*@<#>OXi;mcsdihQD-PgW<!vy`>W
zp1zt+r_bIo;N8IyaPUGGmyX-%6PFCgc#nD@R9VwLG$<q3^JtF`n4=pcgE3*?LA&SG
zv3GTVU0mcCYV2l`tqr+GpBj>J+4`i2BxAw}BC_+Tf-a>WDbdHfk`pN;Cu$5s_V%bG
z6)lA#AyGJifutrx8Ev2>SePB<a~u+^Fh9j7n^AK<S)rB!>TZ4Yb~b8?AR1|(aOV@A
z(`m*0X(^r<XIU!sKR=W2X0M+<eK30`b^FEZgM-&EOq(4Jj*rtr9-3hZhhTI)#Li}@
z^9fXmnL|C}lY|V*Jl)xAcJi~sNj}*b^{MeGWYR2aSf#(V<K3BZ2R^C`&@6K?RAVt#
zxybXp+yjJyawsb35^Ar6XK1xgrP>o>PnAkSN3BYDZ4I3ztV+~nS??#Dh<#P7P{3wb
zi@I)iz|+o-*@)8vs??n5Q(ljDlPoE~rx<*4J=)RQv^t-(P7CqGSTgwb5lA)zv_iUm
z0kE*`KLpw~(>{U7E=4Z!0PT~m38s3}`9!G;g&Sln%z`X5w;^CJ3)^>E#}X3%n(^rj
zta@WtTd&yr3#;MWdEiW^*AM*MK?^8V{=8q*m<$E5Q~vVxD8urEeEE21e9H6NW<CzI
zKG|!Uzm!i#CvQO~77aqSbT`*>JNVYhf}Uf39~DBqe%~RH4_m096Ush|WGKUkKn-u$
zVTL+9hF$9-V5GWv?#u?zS>2g?!W3%h4Daq;@ChzK-5#0i<9O+Qy6(E>YM-o5lEtId
zpK=h`b{yE72Z*1f5reX;S)9QSV@Yf~?Y>m+3xhR0G=U#oxOsk~So$8%W>xD|7gR>7
z>({Spqb5p!<H9_*^~q!u);`&gI@Kpv14<a5AWY&>AcGZz^UZWHM<kQ^5|q(nbmuaO
z)IVn(4@2lj!^^|X;p*^eOKr$oCWfY^(2O+Z!;0fLNp^RGFyuxH&p-{<CVaY2T|D{R
zoQZ{L>r;5@`a@+2!yjw=e6p3${)gMJD;eKJe&?pM?-IEh&j!@*<P*_CeT%0oNQ*^9
zRuj(AmKHl^nPBb6jf0)QPi$MT3a~;g&YysjebOe{=~EbJpQ0q%^1qODWG!?5zn6UL
z^qSYo3Po4kt7(0*@f2j)pcxIe{4b;}am$=yb9RNA&MgjDqZXA~amO%PIJ;x5Pf^mK
zEt`zvIDOpvO>90b9PWZF-ukrRIF|k*t9>#$x%TWXOX6sPwrn!<K6Oh=Z(%thpKLtI
ztxsG2KA+t0)2ATY`n2W5eCn;vC-4cdR|}FTj<-H-Hh+^(bUvTLZ0pmOzsV;L_<p%S
zhFPM0%F&ih=C9B10N$S7B}traecEvTa{ffV=TnevecJMsy>pLkDGCGl*Qd6(wlWKD
z=TVl;CPqEFDN}}w+AxzWs}RzeWNb_rvS~dw=CQC18=}TMH_y#0!MuWaGZT-Hcm{t+
z2>$oo+g_d5&{J%6exmo@-t=UD{PKO@{m$trL40!0-l^O^1yY~>Z%F!2LVQxu_(btS
z{Y2vv#jEv`45{Z+3+mH<4{`k@v@>aZq7XmdeKFnAg2pEbt)D23Bt9wX*H7GHI_eX}
zEAdGw)K4v`Pyacj@kx%*dYs~=`8zr77f^mJK7H};=k++vcNDKa-?gIs0*aTO?`S<v
z@#6X?C5=xM^7RG6d<(6|DL;)*;@_*E{4qY!d`BURPl&r7_pm?SV@@xi_zDE^>Dzqg
zNqidVaC8QWuRzd$^2T~!=>~>b4iulk8}Z5W{!{1q48ybmiqAk?KN0oE=P=YK--0+k
zk<{ZEPZ(rAP<#fG`pMnT^k{v-yn)LYj?rJ&;|dW-6$${cObQa2w98OD7s-Iq1R#h{
zWc|#H`Hrsh7@jHML~O(#wgG6Vj~~=CB))~bU3eGypy;r6LHYoB--vD!pfq|&))x@X
z-_2(nJw`G-PkI55%*3+cAPeyyYKhaT*K6bCK@kzkmJ_1=gZega*IW*iMi0UK9Z~%>
z1vMJc;|as_(&bYO%f^B{q=nJaMXTzdjaP?+N6Nc}gh5hV%!DB7Qxo7H&ENInOqv5L
z7$++}`xFB(Tt7h{*E}Lr9ii8&`s#-#w}41hm&iadgbj;VNg^jiDuB{RA^H9t!TNDt
zB*%~p2jgT$J)b}>Yo-mA1Bg^25Ua!hAO?{d58*6pQS(WW(n#^b_k-2-^PK^Hi8_X3
zK8z@2cnX)feZrG~WNk~e2%IQX!;SuOaQeh5(WjySv<y`nsx|u2=e|aTm%o44@)zTi
zD=y80kxyJTdGF6YNd}=ZNQW>f0Emnh7AcDN2hie=9;uu+{Xi4KpTuZIKxs4(#iwuA
zkN>Jqb&Md?2$x!Qf=?1XD-bV&W<kP5VmB~Ylaic1nVM;0g&WcEhsZyAt9Zv{@H%Pt
z%fMIpNAYP2<6=Jan#J&pbopfD<*~3F258439Xqgy1)}j)!}+9dCF{r{W?rsjAmi#r
ztNGSM_?j`BO_jMz$9g*GQmX|A{T|z<M}e<G(0}@Nd*5I8sk4J|QP&mb@kLZ?mrqG(
z$YOtWfEABc1s8?-%S3@Oc=Xr1>Ze2zjN(#VfLs~RStSM%(o^!T+|8_-_|5yatpJK<
z9uNBbZf41Mh0-aygRkQ4^-l))H3mOnTog`$DV$Ia-Y1JX5sM`e0BBe|3FM*^>cYpV
zzVU)jYM6kh0f5AC<v9?La5!frz}h1R@sA1%od%G4Dm(PzOh~<-cP1Cg_XA{CgzYb%
zsk@qs_x!qN`^zd$K|;l-;|&j*1u#2vtgif2Fl77TuxX)T#h@$SUw*t<IbP~2MwJEQ
zyMg$w4?)m>a`y{<`}qzTF`5yWC2jNBCvyOb(i{*8m(d1sS5g2z3wCk)B<7-6WOD!h
zS}rkafq=Qw`vhYSI=T~9?=u~#S_!pDOHD<UGuQ4m+0Merv5<Pb$aHM|!MU3!1;e6E
zS?T7WqNfQ*k0j<C=mbAv%=EFGRZDke-#IrP(#=~T|8QbrL0Q7d*O?{C&8H#lRbry;
zd2s$=yUBK?6Zi}y{U?(3Ps^a!OGaP=v=;K6l}lt1fJmcVJ_UWpCs{08kG?>_)O}IF
zlV4ME{6bCDxD&CrTAkP(mAWYJ%&CNpV|9z{A6p%@N6no+3840IR=U0FUGBQrv#st_
zXZ?848h+60=u_2rSZLci`Ed2libIvtHs)MAK6iQ)4*IlJX?51C52q{M6s!cFfq4Cs
zXZ_6HTz{aALr8fedOu;7IepTiA927D(U4F0>L!Aye5yGJJhN>}0AA%ytbIHY^3R`z
zsd-s|KBZm42(#_pLBS_1tMKhjWvWG)ryKBzA9Tw(!KZEeDwV3zg0hp_x!|=m?M}&M
z)s7EQwWc%O(~ge=pMfMk5zODMU`8(k7_z)jc*=7)q|SU7ihi^MMAEzgpTt};)Vh8V
zd6WtW*tYL2AhPbWtI?;0)k|&{IDJabTj}(vavbFEeOuqB$s2R5R&z`PKH;EW#HXqI
z@DnK|$4~Zg`DDidK159}wOC%S0^fmne_zk`!@&GjST_J*M4hYov;i;pdYmi6*8?Ip
zHQ*Cpk86NKsnIBi>Ze1Mo%n#l`}80$tM)rSZR30z3$RFtPv^#E1pjPMd_|w|)9L6_
z%>obsiulaw)1|8Moh6`C75BSvzQjlK9ntzs59aR}$GnUT{;Vz|Lzr)on$eV@5)ijz
zf9O+@3b?drmPw^btjA3nAV60edv;p+9s8=OnU%TuZ%W2}<x@hbZH;<S&cVrts*1}`
z78G+n>8|Ij>3;R*v<3&=UtYTVbV6zMWmTzd>)babW&JK(oV?MQ?-FY5XB6c(_g^X~
zla->!J_GUkC(rVouu`RP`H&ff=u>C+d~kaf35o2${@5orTI&~shuG-wB0MYxB2teY
zIhK;Mpp))_sk)|M3769_Ts?*K9L}dn=$74j#1@55|F)!*f~<x1t@xQev9B{XP5jBA
z?1xowHPdc2=iGoP=WXj_UntH$M4wWkVDKG#$CQ$+88tTR?vLZaXCSVhJX}vVK&Zt>
zkHqd%YjFDni-if6`aWp|pHxXKhboj6Fsp!!gbT@rGC(Sl-xXsHzSteIFBD_LXdHHl
z@J8w*<NbS2@!W7x6F4mmds6^ukCJcZHaw`|-Tf3S>8~2zDFg8%gC7|#PUFYRZkdY%
zyf@q|4u<?k$rpX794D-wg#CRzSnn%sGt1%N%khybm^MzIVo@Rhc{t0_oI6C~|0K>Q
zKfD(gjaTxaEJs@~#;`m=D)_Uc^}L)~YC8xXu=>_a_`mp<`}_9lI$+s~I;L>o%O1cX
z2{jr+0SW99t~W|TP?oTsMq>*pAbv)p#@}c(N+1AVdny1_MtyjAWS|88Fk=qKm@O|n
z>p2F)e}>ogfAVzyf*vDgE$;<xL@ind`f&!yi}**lMu4s>&X-XBPJANY?@)?5xe>4D
za+qA9wBpfB*NitYNaK_D_ls`NB1LAFqd!oZ79`Jie`bGQX$UJ2Lc>#2gSbEL{r!Cv
zafvKP2Lt^EXOn}VKmO<T_tk`I`-ion>%yBLKaNjyf8S5+oxw`OKomupNUG>oX#XLb
zK(dxXOEyC3rlq@r8{3^=K@|D{RtUKDYo&uS#mtn<<UxWt=l;U$&AdCWZ-D*2;7e!v
zJNEm6hd;637utM3_?GwYK*FEQ_YQyRJx%&M-g^bP{K?F#@V*rA{vBw#<WHydM1Kcj
z`+a#o7@E%)n0a6FOpwdRxxWLk|F@85g5>@J?(d-e{l4%|kvKo`y#t~7r$Nkp3j~~>
z$j8C6o(vZito6iwFbLI?dw%=|dd%mMvMg7t`MfBK^?JG7Za15U;cmCze;p3bA8)TO
zqtWBj`(iP>8;|cNd7dQGEKTk6zSL8{X<AR_dxt;4nR*(!>S>Z3>nTeu`+e~zsP9j+
zpL)uZsrDz6kLx`1ciuNs097KNeEiz%>q!+rSUp`?{%@YD2%(DV&1VkczvuBw1z;iY
zJ!+*||0SR<DsaEA0<aMI?ZeP{CjNw+3fk|BKS78;x%d03)_(}7o5G)<o&L_u`{GZ?
zA;11kpU3eh2=OQPeqZ`K<dA=!i9dlGpZ<>X6B0N-nLJbH1@w2|B=XsJNIs65pP#tD
z1L^!JX1)N`P|)`de*zbgXP!xaht8_LKaoFyRDX9m@5}iKN#^79chES0!k<trdFJEH
z7l3p=&U^t9`F{)dcOcf^nY@7c@o)C7EhdgC44*l(%dU;}hJCO_aCRnCG&-%u3PrO4
zHIu9{W_L)4yTM>sc;Mm&45Y~tR*>662?S7RfPe+5V8Is3B>`G4k#cKoF$SukHH0=$
zttq9x`QSeT+f{~jce<rZv-6$q&YXYFKg0HX`<?&HIdfJ?4`P1iW6JoQ0#A?`KPk_*
zD9P9R<67nU76lJBep2R-D@l1iSb>+=0rSTRzkd^yv=fu}@woE*odP9(Qqm67C#~}Q
zoq`8`e_R>AQ<7In=KXPH{*!|3Sl?F$Wj9F~ix3ufq-Mgm#2`wMO~0r6To1!*V1&qj
zXH$cpW3jYUO7H-feF08+dVo+M7k-GJe9<R@XnrD{T-OZ&Onz*A0!?f##bn3UNX+sQ
zD#-lSf(a#UnRbRe`M$m{F17tSja<sZWLp~&X-P5NAH5C0Cm(>(SHLHCM-c!M3S?mW
z*Y}mLPr#pLuJY<VreMtETb;zg*!9k)?1A>qX?FTM^G@Kc_c3<zi4}IF39F*w!@f{s
zmubUkeP5O=x~_=@KwfeL2XGj-&Et3Y$Qh0+t43l*MMABMd}4*4bA)Q9g?}V1@J`SV
zRz;G6nRI#!KTJy#B!$;ASa5Ngg**WeJmBTzlN7?I*j>rTK80Kq1b^3M2vU}G34d_(
z<62C={KWT<r*9ZtXNnHe2Cfp`_xiqGu~lr|a?SjrMWjs1LeyJR1p=Sm**~yrA@r^D
z7GwKIF_#J{2&VT3TgdnK52P%FhMrH~5HNdtL{R&osMRV;bRQ3mSsh?$!CW}AFLZT)
z!C(fDs7r;&tv;#t#5{M{%<1*7z7n$!<Bc<qzxm=NxQqyVKr8xf4J<kGx`t`74WkNM
z(W>nf<+6)sV$vF`8OgZ<MzOY6pD<O+y|LCimPkRxmrnAPdmYxc`ULb-?_Jy8y*xiL
z*SoipA4{Z}FCl(HU-y=k>;=Oe;2(f7o#DfU0yX6{X3{1r0gekSW3utTXM%h82}(4B
zOU((&jXq&b2(Q<3d?g2Ou}_ksjUKeFgd46~Xplt;Q&%EUG*}T$u<!yK5<WVDeFOjp
z1b`x<4JThKMybzKQFC2c_VFrGG{PbAOmo9<s1UQFjX}aj^rncw0N&2^eSr}zHplVO
z9~>tbg0=VX`pYpV^;~@dR6SRA;l)lp&v!-j=^cMDOtwg~O#bf8s1m*7G$nc;DdG96
zp?$G@R|09Dtn9w<R!N)VI5OG59Cgjg)g-7)xA<;26JL`h@=2-#D24+%84f-lCTzno
zAxhn?zY)qd7o(JGH=9svX3aFnUg}yFnX+1lOxJEsQ4GCya9U??-*crJV|qNe;N}!V
zQ{xAh(v5@qv8m+o!U|jeQqWoz6TSAql7&X?uU@D^t=6t-dqkDWxc1Vtj%?mPer?Hu
z(ag0ARTL;=KqMLcJGuHK_Tg)?lKb`wN|13D+C5~iZ4>!)w-_qa%GKu}V~@bF4L;>o
zBV_u)@d<MB>ANgM4YuW;My3#RBtk8Xa18kLT{=PqZAC}XZ13F@2!gWQ<8Jhcj7S1H
zeTdLgA)G$;%?&<Hc2yyJdsb;lx2<fdpsj4e@+meo7Nh+6^3uBglA>YzU{Q!+F}Y7~
zxQ(%~#~g0^Z3=y5T}k@(T&<&K+SD;zV9mQT(|Ia+E-&1AS@&7zifycXR$n&S(lKJZ
zXxMGIuB*uGKRG{vDmqs>L1Br2fQ<W@GS{aQ>#e6Uj)CugSrrM@{aX%2plpbrg#APs
z+AT*MBA73^{v0QtAhyEjB6mkeZ}91J3YtBEL<e>~#zI?D$Gw5T55g8*i#4w}xr84}
zcwL*)=!kQpPlPHsmj4)$7#*(U-rVTbQ5m+Pl@T2^Th?T%yJM`2(}2;`cogM#lvm9E
zXs?aTn7^%=ZAh=}YGMiC)78Y%GmA$$<Jb&qWOK)DvL$KQUi)2^spH4|juBkYUEI2q
zp3xDRTK72#@l%>P61T>dS0oK<mK!XyWjUoMhbhAE(&2x3@-cow6=xkM)In)y_}dIM
z*gFORFq7}(6Odyl_Y}2>Pgyi(E^;>#?(vC)+P}#C#gOl=mFNKukxxJ~HqDxM97^Z~
zaA4kApF}>rE%52JO8$@vi#8Uej`zjhu+-*;gjxT#48^Hjd@8-wJu$#mwDs?{6*cvZ
zB3j_nqAq1}vBgk~l1t*6JAOnEj~i>hO8`E7*^q<iAWh1Zmv-k>C)a(BfKT~FIq*jg
zx2`0jU+NPoI%}2&G=b6Q!}8as^Ll;()WcjQmuEIuIsKRg_*C;fh7RBV^C@wiPnLi4
zNi>-1I7fU~m{r23qA&p~qi*zx3_hmklJ+39E1c7t4~f-Ez8sN-P(^p~Y}uuVhyh>$
zCGx2lWptDu*tNFXdb(nMbbrJib>9Rd@@Z6Onp{jX9F_9vD`>K#ccu0eO<$QGPHq(W
zl--{w@@coor`MaW*bK#rK1uFp${oLBN^dy~$KvCU+4Q-^)SOeFWb=vonY$yzRuFy}
zhq%pribE;{aM!;>FA8z^3sE2K7Gx}BJ$_0%dp*HekXO81lS4h-WE{HA(!i(A&#A%r
zrq-^b#-VRAthZ6hRPw+1G;yT!FI|Rp#MF3-aq$USIi_G`;7lA*Fh6V@`a~zRch*Q$
zFkimTr}R0YTwH+Mp7)kd@}Ca|9<=tBoe*d^(+DgOV{nYVYLmq$s0Svk6XK`Z-Z348
zS~~x$PgHGhO(KOE#M18cNgc-X99LP%b3CtqmR!I6RM62MGCyH4O%{ciN6=4>%N82j
zq4hg<w|vId9nyW;zHDBxeUsd?&ZnV8o$1yc=CiXav45u9e|9XHtjCQ#T?wYnQ^vu(
zFzX^)n;VvtJBpSa6D^%1##w7f?u4#2EC>D4kkA_2Z>uS$3BPN{4$&v){BgPCcR2Ou
z%D)qkfaSJ^M2MpL%Z&wRxq3N#>aJ!=OE~Dg&1QC22eqEr6h8@l#NQLC=Dy1ockvU7
zIQU+u8N}(O{Rd;PGk%&`ICS1Z!_Clr=hJb@0_+oH>cZ-C!Re}Z7WPe%D&zRROXs4W
zILBf@`#IJQFRJ~}r7Q|JA09C7Da20>fU%0Sj~_bEs!|qKo7&F_XVN}hNE*RrAzyyl
zLN*6m=$U<os}K<okiR~i_xH!iv`oi{DDzPD&?B48<Z}&edafZ+?CS}gHLw!QFD^dy
z{wDH?ckxM|3*ndJI>Tgi^10?Vh{yi~KJg--+CZO-gKeDIVCIUBy3r@FXoc|>F7^qe
zfxHM!kOu6eHb^UM)M_wji6AvWXb~+8mqgPVwI-U-V625tuEwl7NUfy_wMGLkLW{L(
zg4Sp?YS?JOpwSW<0C;E#aD;+{CR$Bk4Y<MO;UeJRcj?#(0_Ds4zI288Y7Ql`Ho3<z
z!5CYU%;u{fhTKR{W!r9K(O`W6y3u5|;rB#Uemt1Jv_Z!XlgoyQdwl9TX*L^1EnuyP
zH<;_omy<4Aj+`_^(gfRbtiPDTbZeu*V7NMk;iTV4^=bVhqUX&?A?kZW$f{Qi@$oN&
zs}7sv;|(9EK7gG0uqylo$i1S87od6YC5S-4{jBfnb^ne<$T>nL&YdLbpmY_95Ee&&
z1*>?R6*8>LOo)Y@@i;mV#Hs`<NvX9eQaFai^Ex*qeL#Z|z2W9&I)D(eV<*2vR{%jG
zfC>%*NJ6yX_05AX5hz1=+qb?imW-!jOncAtjd+~IB)-Rex8Dhu*Oue3uKkUz9}p1z
zyX{=x*SV4YXG)7o&4iOrQh%xQrfW(L|F@(+lp;{(L~Y;tzMHC2TFFv^Z1QH0^jRTL
zlKM-X-HX5`R>@HcMW00Ml=Xdm)F*h0RzI#KWa*P9ffC;peUg>7d3|5t(?iiG=Tt~3
z1v~~w?gh%3_X>(U$;URX?<-%Rq~2Td#JwxwL7-fP9(Gc?cfS7q9hNUpQg2srH@*^<
zxDzNz?%@IlhCq2ge_a0fom_$1l$Ve|>MAYaS)lwgojYJ=kjrTAg!sw(`{P)?KuNRG
z`f$rjcLe}j63G)3dD@=%N$cd3%;#GOc>?7|o;=BqG~o5v#d~-H<@cD)4@v%f74Q0Y
zSiV5HmnT=&11miX6!wci;Xn~6@7lW{`M=Le$Q7vnlP6c+11UWUlz$$G3!o?PwrT!2
zmMu_H?s()F7~~2R_Ew-2d2(R~tnW+sBv1f1!zz-3@&w91PwWQL6T;iGzAyGgpxpGY
z<ucN9(T4>bKmv82?VSgj621tOyZ*IjLU=Cg4}l7(6%4$ePbZ^)hkX(#IN}{8@S`D6
ziabf@KgoMOnD9ZM+z;7G3qBeGrRbCA^)O`ArA&cx+P&cEqajdO5h%~=*~_R(nF8fB
zd(n<Q8Um&0lXN`{wVd^Ru`dGUw0aTeKN<q{4|{j40w4?n0hHYTf3!=aE^@IKp&@Y$
zrAy#h+Isfl$#(O7Nh7y`@?vBkq6SdolbG)d1t_mYx>Exv^aASheP6zQG6^Uz@%C5^
zpxQ-fW}n|d0?I4AKd1o|ngNxc?`slJUf>O~8bGzn_|5hS3Q(LUtbrAPLN}n&^L<SK
ziu1H4ssYsaB<A}<04l^2);#9})sCm=ecy1tuSr00p6(GffEu5~^%Df3f;{0l;&7nG
zCvv}lo!^-PR1_Zw9YwBBvwRZvpZNJ56rhyP9^*jG`sv?i^Z$1{FAlurc@dscTZ-}N
zV(w~VTL|O$<GFf?4_*?Qh_^kxs|2GGDiUuYKFC63O%tIiy{zkpDx!%JZRXH*m1x?e
zof1T;GiJ1#c}Y49*+x6tNEVw_i6U5t*X)ZAp6lIBiI>vz@@%{AF5BJjdG2?=|Npsb
z|GzxGq^(c)v(E>^Tm>p;eIgL}vY}t?-(MCe1io(Q|9pcciLx>NtNdxWSl{<I<SF})
za?~ee2{791RvSXU*7w{)pd|8voQ+=sW*VIc{t<cN+wVRyM3DfKjWG`-d49fSWWM?Y
zVP^Sr3Xm|MPAUtto1E49f5o*r=MV@XLFOtTb5-BF=7-d#zXGgGrfY`?h$U^jC+YVm
zEI%A$azG<LvLeRU1Id3`-!~t9f?@2J4^5*i@F$Zuaxf!T$}>$VeB&QwI*z2)Cp-#v
zT&X@n<g`no?JWhrbqr*ol^7!jl7}k1&=w@QJVDrs%k8V+w+Rn<C;4$+;+sQ(M;1fl
zn*f-OpnsxI{PW;^^ohFH!`X3`*fQRjeCcM$EfBZJ5+8Do$R<|JVqto%PG;89*dA75
zeS+o-W&2A!UI;(?Y7zj*FgPAl)^-S+s|bjHB?fRy@-zQQZD9s4t5&BoOEr(UP)m{$
z*QbNz9b-X3(NzTp0g29{L<X%aJi&lCsP5TLNC3VVcQ#N+)+&X8)oL+B9^(K)5@f$s
z-}krlNf3PsJ1&um${qv278OoJk<GJ1z&>n5a|l&)78w1f$H~h1mTU*3*>(U>2Y)d!
z1SzpT5j(MShq>tl2PUo{O=GIZ#%0yB{j!<EN8HCY!_?f&_VAU~*)fU?`yp_1n-SIs
z3xYz&kXJZ0Q2?eEa!XMBB!f8Vm0F-E-=;^Pc;8tY2y+n3BQ4MrvzVuayIWNDyJa7o
zrdV~$pqPa!ZeA4wl>ObR7}bw}^s_vj`i=r>W>OgCDm369-5<{<f6C&M`#_1Y&Q{v6
zxWp5{jt51x$4d$vH+16ZA@D^coF5(Nph1GREh*#$%Ab4JbAvgz^A#YeEpO}%s|(5>
zgUx52mz2#Px9OrybFO^i`L|Z{T}Q#<$BJhwpBuZXP^TKSHWe9f>wJd`j2{hz=u@l}
zu6i}2pN08EJ*H{l#AydZNd*cPHv|ocXuYLh|FYb8-Egc28<rgMuJ0PR(ZPh9wjc6V
zmK)!$@9|<+qoRv0^S+{uXXlXp|IG+MqbR}9Vi3>{B>zo)-+bnG*?hvLmhOt9J66xJ
z@4H@Fk&2scp3?0-Xcc=!(n3emwl%VrBuz8=;X>a7Kw^Eecd9cn*Joq~Hk9qDEV>$g
zSytsf6V)7e6*+E<YUT$9?+faWcmm=2kGjsZ`DV)MJAJe*Xz-15ApvhBEv6)Wu0e&$
zb`Wd)P{?mdM@T6JisMtAo{PVw*tfRz%m)p}rylQmM`BQ0(0crkOLea+VZq*z#}_x+
zJK1%Yuh}m=A^LZq=27*vqj>15TxhJ5%oTuy^?fDOC*S3|#1+*}$E2kjgI8*<0&Exd
zA~v*tM~D@pojGclT-b*LUWbwwtT{k(eNqGbigtQzfV1388PJ`S4%;jqs|f(}B^rgh
zm(|x9jy$2T>7A}kt$BHISB>M*USq1|iUo@CSuVD|Jsh88PHLuFxB{G#CQu@uAa=~H
z1sGCT;1g%5bkOSSae;fU0QMRU?JVo<J#%Qx%d?M%MLr*&75N0N*ib^?ywZ>I2liX_
zFeJ#IsLegL8LbaW3S2w8UaXqeVXjF~a|gXUQ1nVK?gk05f>zA*DyDu&(u`rK$R~;Q
zNr-p84je^*=5TxEQ$umXQ$^`vflu0X^>t%!a(vR2p5ytHnz!mrY)18k)z59{n|zYf
zbTW!WfpU^k=1)Srleyo7H#SmV^T~qs3G+!tW%zVs_Nop`xO@NV4}^dqi%-SrKHbPH
zKK<qAchcxnrK1Yi+dS6pxB9J3+RbNOV5)yB=9BwE08}pC(;x2yRc8EtD>bl6mtb8p
z-tvSBNUcxgy~QqIx}^3ko&%0Mj(+Bo=D;<DyD3_EI%K|3rx*E@*3>`I*~8`sj}+^=
zqCkvyY&M_z+6Sfk`NT#f)+d>->k(z&7mOVEs52e-s9Aj!?L8C{_{8fIMRD=2+zU)j
z-&z}JJERvzHap(Yv;y-<jCX(c{o~^Of;{yJnku?pRFpI5xFc0_w%ycOK7XTn>*1@E
zsdM%G`riKhiC~0Sk(Nin==2_70n_+=oDCtV??0I?6}_mx=;FZOJbP~n^C_h$ZDLF}
z%jVYyZR;;TpR_P2>Qg6+-kt2N>9IAG^;QJM{0R=xDn&loW^(a)E||qC`CNC1wDrda
zsO<RIHSZH(+vVPhhw;)1&-{&^&<^Z;`pjc>H+e$$PGCNDdAWX_(Yl*1PC0!;C1N1O
ze4I8Q#fBT=%0MzNh|>Gt<){^use9S@IHe++AB?Y}xV2k;`wG8`GIuY(<tN3Pn&PZh
zuEY~2wVZxk#fY;dNJ^|v1NdVj?&YzLT+m{H)p+9l&U>up#iCmwRNOp$!ss{F-EmcM
zbBhY!e?q<o;}bgjyXoWO0E&Y8iM{j3Z5xKdI89lW8)QjPcqe3SLW>Rtg60Y-f(G6?
zX|B<sYZnh5G8Tvr;3e=__!8~X0y%(z4DQsaKrc`#luSxC^+@BH794$y{6`|<PZaSJ
zMNxe7HJ`@AC|Zr@(?OhFizDGl?m_?kzjwU!DEs>A&y0AtkH4<3CpQltypIo~FW;|b
zKYw`M&D)76-S+L{)$h-;CuX+Hnx@HOp2V5yDWfjuFObjQ?c6`UK6r#Hq?D{X2^r;o
z;!~PUEa76^hZ{$|xVMZWfu}o$+&`Xs;%}LybfGOm()JS}^fgKKsZTEXy30JW-R0A2
zn*WH`8s^6)j#JaiUZnOFqTag`mZ&g7vm)jGQty0So7<=TymZBG$^NHsoEha7^G`_-
z79G}LeqRbC@CFkVFQUXkc>+-s-Om}t?jKh@sX6;MZ>FRwBvoM>`<6(<Qt7=6<QsTY
zH4Nq5H0t@~l8puv=5YLY2k-u!9IvvuNv%zM@q&w0FDi&5dKQ`=8pBFf@}y=9;GdB8
z*@&N#qmtwM@rYrMFOY_T&?6e7VRylesvdcjuiyA_J3i@$My=o2<YJhp8qdSf>ev|i
zR6VIK;omCeV|R<9;o;&=;?!C&a)8hyTBB;o4*$*=J<#*P4*A5FZR6=gW(+{Ve1dnY
z9O#d27-q_GL`vp-(i`<y(c=NeKfO@<Gk3?6E})(KA{TAj0ToasPcoAkpOkzqCLotq
ztwOg|8>Kaj&k3lIQG+KnU;3wQJ75AzdMewo2NcyLTs*bpt6^wKAl4P0Ijs48VMoS%
z%0#jF1bNj&kVOX+f;4X6(6^8mdg%Fm=|-w0pA4#*4p^vwLdKIpwXMWcg}u7W?@OUn
zCq9Ar=L<HVFa&!6i=pa_?|%Hae*Py|Wj<+lNaoTVJtRQE_%~&OyC|I1qK30B?yY%e
z)}L8A-x=~LWQ-J2ru5_HCiRQ_T9#Nzwj`@ZOpPP4TE$j#KH;5xB?LmE9OU}$9OP3z
zVsYS1t6|%!_)Sr77F*37pmq^BVWd+&f!B9BJK-d9r#K<j5rfR<n*#8p2UKnGME8QH
zUB>&K8S`lp^_(e62;Cx2m(&bZlDV~;%lW)Lrd_1@2gP;x{^0pAPQ>}noKLYs4g8an
z(<bdZW$GyMg}P4)gj$NK#`{`b-;FLDYU-aha-c#@>iC(u8?{L?Mqi)o_!P8AqvKRt
z$}6bQg7B*P$L-rs-41DVoN6obM9L)_V?ckVEuR9vsZYl$b3Vc2ea{TrPl10*8Xc#s
z_=MDcYU<PR3Zwnh@Tt*$_;lXz>6y!%Pw@P{FB?8Rdm->iJ|Eoh=|6jC1Jq<4$MIio
zVNrCwTsNy-xUsX6E2vR<T0(=MplD3tcDx*mHU#DXVZ-s>FkVy$G~N;gDFP8Bl@~cO
zinjz6i=g4lC1NTG>X|9WQFr!xo(&n=w!m54`TuhIJo`V-^WS!S_Wk|-|NpZecfE}G
z6KZz(lm31>;!k(EEa*><JVM5IcfD@IpYEID>Bjsx@u$09bN44ho~-HhU^4$iWP$l7
zjCW*wM?~hIh*1ACe=_8W_!E&)e=@v9^`F!eBmP8Wk@=_T`Ao8YM`R-2%_4sy{zL>$
zTf)P`!a_n=N@a5LmMt4%HzvfdUAth_s_6B;-rmcdmd(4>pAe5*;~g2_5dpY`$MYQx
z6*)tWuC8{s_|rWm*YC*u_}#471T*o@)$N8cG4wbYsZV!5S-&F+@ijgFG|$O?+MkT^
zH0@7he*7*M@?_MXrg@s_PbTX75`VhOWyYW8Q9Lnj@+WftIME#aX`Wesx{uudL}ZF5
zCc~^hnZ94;_Wfzu!>hrV62CiO<Y3}M0bpL7&p(2>=!uu!dxi-_v*gK+A32Ns$<ph0
zhtI?SN?!LZ7*WKLrW^_Utu*P`U|v3rm$)zot$T8_fP4XUHaywcyZEkJ8|y}MG)sR%
zSz>&bR!<`l`tv8CSKc|?HBesDm9Kd4w-POlxq;SViBX<PcJN@9vU&)tKlN}HJO>Zm
zZ)uNmymvm|0Spu|m~Z%gN2lJ@;mk`h#(SYA5d(E^OufYkreyCD2q;%C>JV?99sLB@
zW+X1qQ;`gyr5LBX(_O)*_TA_UdHTQ9<mJRQ$&>3AB`YM1#q#!z_Vsm%oaV`_KcOt+
zPn@(I1B+qOgk8;@@i^~P1{jDry)kS#o|HY^Jo8MSuXo_abRTdNH1GP`*gB&;ZT;Ez
zfknkR?zwQ`2WJBecp5U~sc@SFXodD{t#UB>A?X*py9-&m<DycpsSKcBEDqL!EJQ3G
zX#u5s=^uziflw%(VCp-4W=4vjaz_dER~HO$Z!`<V8+o9@3$-A37mi0FZSF$h2*VUl
zF1YSOt-5&G`?|UKIx=)J;!hT+?`wi5(Yh-9m`VlR0hMb!((~y<m34xXaZu@>;(3S*
zAR8QQY?g>h1ly{3AnHjP8yby~uam&%B?8Yj2bI(#Cm%wK1?g=JS#fV>O_037xjB~y
z@`j{E{$Bt}UYh}&AFpsZnkYfhK^|oF2-5SpqS`mQu5j_Gwr)2tvBw1-9v;sAp47(U
zX$$l8>m@6zO8?zxg4PbenLp3U;a<doZ@rp2X!T&>R*BtxJlvI+wL923BwFz}R@Ml?
zG6y$2-|9i#%anzUyV=|Gnc!w03F=@kd%g~YVXa%aAGcsw8#<ZHb6quzJVl7d`Q^hc
zK*?@@bFKr}S%0Y~=Rzm;V$XnOPJFz0#L+1dP!-|{K6atTO?hdcqxWQIB>0pMxG9GJ
zFv(L`IE$66RJypV^75X+llk$Tm1X@&UY5@}a<z9~(^&*yESEY`SbgqTYmr1=o6>iX
z;wdwAVfW_^r+Q1$Gh`(xefyd^Vif7w>V|D^^rSAqt7^6<ew{wrn3ON;O8THZOWOSE
z#hf^GS59xOGtHAC<$FM@;woA@UaL#V5Jjc;_S9!oNRv(+(n!xe-I@~zH}r-x>K!Y_
zQh(gqSemD60PxexK53qUKVw9%FY>HvTmpw11PiOUoHS{3T~Ul?s{}yZ<DAr{8OuBw
zkmm2-@e%Zo)h5*(i>hwt(maJs=wg&mJkX*J4i}5pQw$btKvyZ{mUlAIZ5WW~jy$P9
z)`F%~$CJF;WA7&s__4d<py!G1EwudOQY}*@48J}SNS%fMSBSL=VIatbP5}KhPn}$u
z7`rpcQ*`|DWR{cT@<>;2M-%yzW$OEy=E>*c-XJ)322`i$7IoPUE@;*)0mTV?O!3qa
z1Zk;NP_gT@qV+@YDW-Xnx31*+3?6`j^f*wh*$tr=&sud|0yuIs8x*bIs!~1!SqUBQ
z6iIt1o*>Pm4Md~ag-scd)pc6rzY?Il8uv#H(l*A^&MR!@Q(&#|SZ8H%e{z_Y*ScFJ
zZ9ClXq3Gh)A3iU+tnNDZP;q*$yt;d5Pf?7#Hwe$mtLu&_INJA#N^p8-Sw}#5{h`V=
z^#}H+)8SuMK9Z0S-%=qQcb)8%O%5zC$H+uWHrdIecv2xk5y%teEF0aC2ucyf^;L{t
zbE+z(*SIuK0|^PSTD!>+`@+%|&8B#?B0VwXr7a5gKv27DaUZYdsN6STn;CzKby~|(
zy7Ijpk*C`j-#uyxe@c_)yj}&6b}AQ8qNtS!Rl`}x+m@l@>1=?{{?umnr%hKXcjZp;
z6fv0E+?NZ;Q#KT?X=d9tb$nCyK5!bVZP`(qKRk}j2$2C#ysV4PW4$pjSp6wGs_CrA
z69bR@V#-)=z>Iucu=zzmJ-T_tb6{znlGwI_lQH}2dwaVsRn+D}XvrSMt~d~NwH2HQ
zf&w%rTB4FAaN!a_aSazLbF#}<f~@B<h<1Gj00@cLFL*#(uxUeV_<(D03y8!MJlz5O
zrQgKUaLDzkRJ*b$2>+@!)oBo*3dDYX?h~OOcgFT#*Q!RiaG_LZT$+z3fZ|D{=LuV`
zhr5>sn&C<5xRIrFagBCbKI{BxdVFUYe*!rCN@mi^89a#^R!;L&6TsPDf99p7KYUX#
z%@c5T4n?IL((xqieS7J9+toBrIluk9^u3P^dBS+d!FUG|zYne1oBiX}GkC%`75=;F
z;pIMQhk-xMf5CYLoARY+S9mPC-_ksteDlo%_+C=R7B4<iv54kLb0QA;X;Yo!PV<yi
zM)TxTi@AH=37V&U?U02$(f%|Z>BwiQHjS`C#<K>TFo&Xf;`9$|c{-j30(Cse#af<b
z%DvKxB<iR=06!v6bi506<VUK-14<UlNrk~rqvJ^-MgVD^gnFJLCOa9SljG*%U9=1O
zlbbi+%iHme&ySntNmSMj5npfP>UmN(bOd1pYn#DS+Ev`Q!{#L*YovM7l;wiF^-nJH
zgw1>KC=+?2;$0ko0~BDqQ<v%eX*Wca?BJyB#mJcYeIdp;VDKoOID_Y~YY&4#)~n8_
z#n$(}CcrJ*(mY+GH~aTWILjR5i>h?~lmQi#KS`+Z9g{K56Ay~1K%ZmC({%n+P};(b
zjHH5+NJ#O-EF9K?4uJj<o{lFf4H8|S=E<jo`n=TyOy^H(@gGb(N2Y4PLGMplLIejd
zYyl1aqq`GLV*itPy6JeQT#(G7^Kp~@WUL;B1^g-X<O?q@K2%Yc@AFeq-#)?nP}#J#
z3Qz^oy+f@X63Ejh3=K5_I3`OO-IuaQow=f~c5@BIQ$<7lXl+s(sH#sMl+{-E4N2Q&
zbuS~JQCE8#kLHTHQu;ER(3AjKO&YCj+5vfMik@xA$q;2?`hPUPLa-Y!P@a4?s>mQ8
zf8Kuuj_(8lXAjg<mgeafdYjkU_gv%U3Y#hnbeCnBMQ>*o#Z`FL{M?Wu$-{9gV0O0$
zF5uLeruENbtyg$Q-UnH2JIM7sh0|6cnjDE{jmz9i7fjN*jpl)YwFzqj71y_@HccQ;
zm_Q?GBkTPEa4e70hV6ltLV9iWacTRYw9POdFBh&V=nPan(5VuSb1A+y#mC2bCAtqp
zW8$UbDY3kCLE*5Od>p3>aK6CR5%VWs)Ba@oe&4yzkMCsL*x0bosD}>Xd;y!?9e}ej
zpYGw&q29)}Pl9B;7zZeyq0?M|(0zE$Y6x9yv#zc~B7a>1S+(r*IOItzK7WYrp-<v7
z*akqgv(E&8yw<jFupNMIvF#r0=0aXA8%;siRgmcQeEO-_IH)wxvme;7oly(deX!^r
z3-eUj20EZJ*?n=MSGQC1nfPjK$KtS!J@!KPY6bPaArz;jH0{)h_`(CAunl~URbgue
zpnPK2>)upYaF|x6?DreMFb5Fr7as06g0|x4M}d*3)Kcc}7d(PneL}u<JEXDCAJT!L
z(I;K~e!=lPMTjyu8Sw!%D?B_rcs&mLf;*{<DPf$6S1N=3Cb*{OpZw_gCvV>c-qGHX
zbIG6nqwfcY#y|u$9K2(i|1enDqC-6j5C7lSuz5V)=H_N&4@~#MGt~>v^m>ci+Kn5M
zl_4Q6PAEQO*7M{4UHyW7|MPXJ-*VxO>MdNqyBCONvmVU1cUiuQKgabux_-fba{rU-
z?@zzBoeOtX){i~%1i8OpuJ=DNP5Be4?@Kfbo|yP^L*)KUqTAz%Y1W@eeP5#4`V+Z-
zoao>3CsN;+$e38`=ipD|`@uv;v@zDs`1lUB%+>7{f4Ya%_a!nSe%MA=`zg8U#Mj9Y
z#i!~QFwEC4K#}^sL@2k|FnZx`W_gtbRCyI&@8vFZ_GGHQ%FK92>iZIzsV9aU{nPq%
zq`ohaX`Xa2k@~(w7OL+{b42R<5?LzVQ9q=<FOh}n`<l<6Nc~KrS^5*H?@MIK{E5`}
zCA!1@MC$tzSt@@b>vu$R^e01(ZtRizzC?eqcQ(6915p4z^LJ+26a?W$Nr8nXP?|te
zV`xBXB*vshh$~~*uyEtXg^4CU04se2w?2Rm;KEn4Eo#+z9Z`p(_Pf`ccrkj^FJGr+
zD1UPORNddV!r=Lx5?Eb-s;I7ySA6D_`XmynK2`jw>*MO1`}<aX`tSM8^^?Hr`gp}3
ze*UgHzpK#piRS38KOKI3{LQ0F75^5`6ViRYuU{oQeLrpWP4@a`@1=P2)~$Q@Zr{Fs
zJ=<FTaZCA)ls|KSU;FZ3%Zt~^zt!dPd3O24c%1O`>C=~&A3VquN+ilGN;FD5N=3>{
z%2di^%51tpZ+!04f4TqZ%DeYZu0Z*`hm(Z!x0>T!fx`WL|C{IUI0vAux<ksp-3o8O
zNy52ZT#jeZuRhMZ{`7yn|6S%&$B~s!I-iR7>nWh|sr~Q2{*G5Z9o(lM=IMWb|GUa3
zlYPqXhf(!O%YW$8xqYf$-%|C-NS{va{=RyuKIu87^E**qpBPW)r=Y=jpBQC~6~`bg
z3p#bkG4B&W0wtjD6OlyUCspqK>Db--05EbvgYiDKd%fPov*8<XI|o029OFK<R?%b#
zC7|mQcE$~)g*=_NU!|Nr(NzzChXH6X-lx0UQ34izEee?QAjh;%<63yuuazK_u1|P&
z0j)98??Fi6{4VcP^Q?8eK2a|O;QbYljlXUKw1Qv+V&<;vU<&?ypF&8vw($Hr;~*FV
zj931K$;H`GeIlzB0~&eInD`3`!$sepHbVbCIGgEuRnMkc2$;Le$YmF21Ufy048wgQ
z?J;GvZc%01NU0K(N=0SDJ~oB*xbuZi2KuD)X<D1FxAu7JEov7<JeVOAFxsbUYumPv
z_q+W`ZQBoa#2SQd+!xXOs6L6E3jin^QQ!_j=GMGfZLsXc4Lgdbv5$`5n-4bN4Q9*k
z#n#{q`ZWPL=K4g=EXJ55%CgZ$D)vq(l?t`*&DZPk+^1FveVV{BZrCm^;|q1a??fFa
zV7O1CS`<Yd%r>9~Q)upNynn%3uY`YGpVW4`&*cE2HL&IZmw`t~Ji@pGR(B-s+g8})
zy%5-HD&qwU794U6_NisD1d0twg)K9m6jhSaEKoG}NfrBa_sD%(tf0QCtvh_#jpN<M
z@(~J{?$ZPjB@gEy{UNksN4b$->j}S<KH+pNp)qL8dyFrqpyH7eT>u>17k4ASJLBFI
z$jasJWH_ipj`==G$0Af}eEOzOtz-6S5(2<-#m81_QS0<0%d&+XIZ(iGpXM`2$$T26
zK8<QaTu+Yelk`IXm`txlpL~LT<`Zr$z}j%sz37A<_cn=7Q?Pb%ARxzNpPF^XSi{CI
zd=fv?r%vmbed5=;*Kjmg@X6Y_*gd-+UqrW!MW4*~DX4eGbus(s)6TvYw?}a=T|ccR
zkYm^<KJ3o%Zf!KK&8@Xh+`UgC@hJ-J&7eN7b%wjyC!dJ-@IHqeljnE#MN8Ua6c<oB
z4i%1+_}xCqFZyJqJ~dmdyGQF2(8VCyEhUV%+qh5ZVz-UkLUX(`w0_!dqv0a)DHy>x
za_ng52P+Kjg6BbwVV{7{chPRn*xlWBdjVXJ!NLU<UBoDk1}?Y!WU7e_r@(qHz^I%5
z4rcqrsMQ*eZA#IyY)J_siqAe}LVe?tz4s~IP-n7FMHDKYiiLs$aDfCk<50kGpLq6H
zWa*#Gcs2pS^rwsiV^F~0^>O7$NDxQ?82k8xTTxw!bT}lO@sHnE-h@B_W7o$OVfn=-
zLJJi7)Tq}xS)Z!s?@UtkNskp57v3c3o<RYl`wKLDd%U}l9PiqRPu2bc4M+D$j1ay>
zE-pd=v-=Ca{?PlB9`LF@Y58CG7tjKFpPE&l&gDP!DJ2)`&_0=3Kk<*0|9@EZ>9_Vj
z>HC!5;5YNh$olE3z&+wBluqT-an?_^xjrdH_9z9vLe(c7r`0Dk+$SaZ9$CImRiBQt
zesVth)V%Y@`$S+5Dc`56PlnI$^7T{hlfF-Xtxr{-eq;Sq^{MJpIqN6<>{GMq(|P=d
zJ{{Pn!iJUlk^4j)+c3y9eJW>;+$ZyM=m*pHiK+~lLcT!flaW42*i$&a6P>zwsHRV)
z%#r(~o5MU*-zS17QZlD<sWLOp_bJ<8@aOx)#3$G{4_vSaGmN}oY5G*k9Jx=%=g<$P
z?Gr^osE(pf7ZRbG?LXcp;A{_vuOUs!IT(d=)ztK<lsR&rjL)GTOw%XE2~w8uvp$J0
zk9X}q)h9iq3Fe_KO`nc8hu$YB0ou~`i6BcTEXcL%@t^L~U+WW8J{{mw-Y0#Ya-YPP
z$Gd;7PwYhc#7@;G_A`AtCr^5x+U?Q5)hBi$efleX%KP+_>nHggpN`rmY9%z#a<S1z
z_!c?snCg>>Ipn4zrj5gVGBAgcK2eoIB&MI|_(Y`o@(H{$`s6Ib*~}YSu4)DjyJN#w
zs9wudpG?d_rgdl1KFlWra~SE9N<6o1A(1E~RFU(2Ld<<SdY_!oB1BP&yt#I))kpzD
zM**NnzpC&I9m-Jx2mt^Tm4h5WISv7a!a=HR1OVbV^sqkt#vCc=2;e9jhkY>-M`HKJ
zhxzn-b8vJJ0J5(Xg@YVzkv@nH*?*sd6zRhfIWQ!@+$UuzN=f>ukPxwd;oZ6GCoHW`
zGX)8C565e3Hd&vsQLqjjJ_~~ufDAn^xF&jcfGsxwV0ryjFl4w~ucrz<ueZLtSWoI)
zY=Xcg2leT<=710EoK0(^Kpe*Zqt2U%8jXh{lk4b0gyi6$c=6yNVDRks{~|k@E^b=V
zMBJv?Hcwv;g5_cO_1CF0V{kksoF=u{HFev%;OIs@qaq}8d0N&(wwZ9PwaD5CcviPr
zs&>kOjS_l1%rMKc2qK|c;fFf{>(*P~r;#4!AWEo)h8k37fC^218i68s@{@aiKQ&TH
z>?Y3k=(wU|>uhD$B`xYT2B1mhNfGW9<{}t)5_`XFNm|*i2uW6Y@v-!)4JsA)#A=qG
z7WMG6z|J=$DgAi2_v_x=SN3|hq9U5j)50E-_d701-W|DpOuhOTxJhRg8#ib}tWu{;
z%<vd^b<W$+sw%d?PYxjjjXWWWDMAb(s$rP?WFX4U-zYX5Ilr@?R8pvdc*FP+Dz<RQ
znyyw(y#*bXgkaTszVp+B4&zra2#*AZvPmK$JXk9Nslr9kHAy_nPm6j;QFfK~2Zu0Y
zkZu@@<RM$pWIj)edUVPF5{AkKQUtufurC1e64er*%1tiQN};GSq3lt-Az8X${^Sqy
zr&pegK~EQXnl`L>nx{Masjzhi>t!52lC7Z0`o3>S{c;+=6+pd5b-5X7;WdM(B~n$r
z_EX~O3ltiK>m={<XqKNA^^l_K^U~2gy(Ac#4H_C9*9&=?-{VkN01zqyxYj%okTsxQ
z^ArH|F|JfxXr6kcd4eeG6o)1Ar-6hMPiW+cQuHBDzU#R!_x4jkK<Cf+h0qCNGHRTD
zAE`)9rVbDsTkpgZ9(YP}#8q_SiH$tv$iRk0&EXRnGyOEbN08ViV_5koo+`YlkS|{S
zq&tA_fY5xN=J#-{b+t;i7EeO1CZ0MA)iK|Ns2Ti32cAfM?1xeY0+U+v{luwe=8Pvp
z=R=-6&%L*wY%%_%ZRV_gIY=bjE|g3<BUzuuuRYX)15ZZ1gP&jscBB`j+DX!G#|x#P
zt1M45-)?=DtZ32A^wazvT8L7`M0olJ42PQKy3wzQwSjY7GZ9*KYV`43p62%;h3Z7l
zs-#m&15F@RfHtvFi+$Dl&T*3q!GfAXURzMMftAN*vw<)p^Zn%i;fWDsV2lu@hz_!u
zczX9hbMTYAtDkU+LB~#+-C!%sx4;kaft%6>9YdyNxdp(gGeL()1iYd4Ar8_l>+JEu
zi!@*r$0;Tap(#i5Wu~9z_W)-fmmW=tzCjlVOnxY(bWzH6Db(rPTr&!1^EAJQ9*GVE
zZsL*vkqkTuX0$Tu{t&Og4C6zTVjk%6i7aRY)}_pK0DRA8`-wUuI8=j#wNSJ&HBIM#
z?<hL)M9z77y!AW0KiZG5emC@Zg!PtdJ<KA4x3N5xFI&Ge4Z}1pWqMvPoxt-wf8goi
z*6*y~S7$!J`rX*$0eHIF!>lE|jgutK!u2~%(}(N9#w$-`;^{H{M2Yzj`wMiBhgc83
z+Jhl@8-uW=`wOo5$v|&T;Q9CVlLen1bYXwy*Y=orTHIrKv0L{)-R37YE#Ue0oj;kL
z4WAv$=NJ6+r9CE|miAa)?56$Wrp|Rgp|knZ=qLAnegeGp`6tJ7(FH$!ZI6kk<vo@c
zyMDf9I@>Y?P5H=Ap6|W5pXsOP`MYm${*L}lh4G%JGe5bme7>LVdVb>h;A<N+!Jzcb
zg7E04@BIGEA!g`0=<9ic|A42HohF>jo9E-F2b#l{mEbl%(f>1yQY=HCF#6SD9(~Yo
z{e|A=6@-tTXQIhZ6Hhn!$>#rSZ2ypk9{=*#4>yGNDb6Y6HTaZi{&WW0nV;VB#7CaK
z_54Z4TDio4XD-B=GEJVY%$_Em=zE^N^Ze<7-v9Kdv8IePjo)uOAN-yt|GWF?$<tgv
zO_}c}&6D@^)6dRDKN;tKx|OG=pMG}!m7koEC-3Q}AD+v8I{WyDC;YdcqNksJbQb-8
zcm9fB9nXuNe)`e*M}9g9%J-gr`pwQ6{G<&9aQH_lg(4u>5imn!VSzD%t1FSrXk`V9
zg2N_8C>(4;56Fd(KteR(<Y|~R(VP6WYf;8@b3>VnnS4<+O?-*Tm+w6u?R()&*QfCU
zYSyP2Q|v$8p6|vBDEF*SFPp#IC-<yRFPmxlbY}y~{?Gdq`@FV+KQQalA9&7va=V$o
z-6z|^_P)XPf(>`DmvdV}^zmv8A*bxq<U_m&38#5~H-5ln+$ZL5_lenOhUh<W8<#Vu
zqi%mUgJu}79lwEc0mPySq>Ea2B6wtd9l^Uz({z0bcLtb*Rw>+7IDd&e6*-JxGlfhO
z{}I3HlZ~Q=N7~!0x;Mj9pKMmsANoYuhZ_d;U++`YW4*?6w?y<#e)+0g_TmQfif~mH
z)^I;Dl|r-{1r=JKU&lMAZDSeUePX&kg=-y5Lg5^za<(h7{ONatctupi5|96fU-gOp
zxZw4AOvJoH_r^gHR)_A3?N9Ta5hyqP*ZUN8jdFC?C--WK*^W9r#K@<;cfe<rw7VC*
zmV_A;LxS4fwHSk}(?bJQFIEn|z?)A{i#I;mX%f`>F7^06`Fw}nIKt^nE0fNGJj?>c
zZV^D-UvIKS4p|;3COQ7vCoWqck?i_8pePOiwB<CA>GJ7#I9+Jf*=+X^0O$9|@i6Gu
z3AY1y_1Zm8VPc-XPi{tU6hnXaiK2!l8T{<i=mZ19DElx_Ptd1%Tr+>yr~S(<K@cu?
z%s=|Hp<<&NT4k1Ph^eGIq~^~kG9{=g1HkHh2d7#_Fko9RBc=Nkz{(Ec2!!$h`=k5R
zxezauIM>iLf2j*jZiUY^QR^cu$rl}7@}nb#IoV&A$rHn;H*JY6h?3UeIAws7S{no3
zo3yNna+_<&%3Px%Uz{K#%9>KZLvZDVfVzg*mbx5@Vp5*EPlit({pFSW2cJy3P`~lX
z;994N`!u~z=646q(ALp`(=ocRRE;==6gKiPi{aCL$ol@uPFGV4)dlNO1UgGcj?l-l
zeRUPDuWo$WixB%&hoc;Qe4oU0LGH0igXfekvf|gY*5=w;`nZ(kWq+*`mNfoQ&eX-`
z&?mAjv*&WWAn`4QJ<$fzDXYXAN|KlRtWry!&UNBGso82I<I(^RfY`ZmB&roEv`Kvw
zVwjXC?vwFF$L8G$#x|bsD9U0bi2HmRZy5a3KDo9o_qI3Ia|f00eG0_3Hv^_ms{j+w
zeJVQwL4bBtgtN2qvK7E?AQhp`K1G6KpH|L@#8xC`w?4X0z8-K~;!1w7`nT>=U5W#R
zWJzosUkpx5MX5AsmzrarB({EWTu(OpH7M<{&kv;__5r1DjFFTwXiHr4YtJV?$R~^O
z&j!}qdYf$Ov+5JQzv%KZ<Mk6^vRNqGeLmT4`Q-U$eX8Vgk+sTR?qJh8@@WaMsZ4O_
zlLgVT{i<SIuWWr<tHjL47{WmiYq`7!Yh$0Rrumg}HGv7Z*Qbu|Qx~k#1t+%rViss-
zOQm-Ioae=ouP!QW{NNME7Hg7J=yxC%;z|)pT0FV-sf)OnL|@`gw9d)He4?DYhEJ~L
z1gPgR-;Mg@W(dOSv5)&i+`oPr>SVd)(?9Cdr(7-+i3GL|BDRrFdmx$WtKfUUQ}y7J
zYB}36o8Is4hTNcN=-48cQM|fzT5kEYz5-HBJ-$y3jx8MzeG+}(<wIVp<4`G-YGD*u
zwlvp@OLIlTr_&psvV$Jta@+7JofZqck`!xUPQ39cqx+OSZXhG|Nr#1?%?=6^U|ZfK
zZRIt4K8X4Jv+!xuCu2QMEYgIVHW5~uA{-VyQNQxZdTYM3KS`e!V~J~Gi@1eHxID<^
z4lWNi$Qz$lh4(wvBXa2opsv*Ssz5FXk%;QDorPA*doXw}*g*+9!*%=0vh8ySt4`BG
z{_IHgKDtk}xrQPaIXyAtK&vTpEw(26v>IA-rDH7(-&{6)EXC24^>upoL`GlZy4rC=
zO1F=pC?)+*PTzDd8sbIU7$YucFwnTZeqpZj!C6VD)Xp9zVOKZ{Y^{+gVN(^Jg-@eC
zxx9u?E`oB=4w|+Smf>Sm_v{yp=R2Jz=#zQgbIrUNw^^uc7<@l{kMw*>rPfVIcu9a>
zTJr3g!K}P<0hZui&;f&2F%o2y0o)(;-_5RlFi{fB-X4R;_K7TZ&wF3dtY-lByw~z4
z!JhWcS>UkO^EZ+9hrzdijGruYn;E};fkeGGD1w)Y>A}fi&kx|ReIoUZF<3u`beH6e
zk-om&UZmDdQ7<NfNmbMe5KDc2UVWnM#*4!^KGQ*&hLfGK9-pXBuV(ibObprA49_?K
znI3<x)iD)x%Kci1!W(oi<!MaLQ}=1|&+o?jPh+2E_ZLh~*e{Bb!W%v*vfK`1D(q+Z
ze8mh;pjwWRn4G8X(}bVj-F<&&2sP`|18HWtCOK4#ZDLx?RugmxD}gC^qCS}?@6#|)
zL!IoiKD}`6>C?pbKaKAopY>_RwCkt4KF#{{(wU}DcQ-VRd+HWveR|PMw|*Mm@H<*S
zEzbJ%qM4#k#K@=7Bb~E8&6uW7cRvtz%=+}=`Nq!q?6eI8ao95(uh*9AKxL~ZddMX;
zhdSzk9;~XWS`MfZuYlBN=%sxGo*Hj9W}HnzcJqhCc)mdqV6<A1pFXcA((L0WY8X%l
ziv2Wf_Ikb}KeaOql8pT{X!hujn?NZ#kZVC@?59Dq$N6cc0VUJqZTsnY{&>K6`^n+_
zWIf<XcR#7#BK7?7P%#^bx&VPdAh97&QZvuf8=pTO>-pnZ%&d^Kq#D^`%^%lnnm=y&
zNqYE6^b$lNq9RWe3T7+j4(CdMh->^bb~`^oUqSW$G7ckPwu9Yw-!V_rFfc96v74U&
zu!lu?I{8-mP)qOrZhlJHPmI5wC{M6xkPppIC*RBYmw9pb6M*d}nWh))CuWVF9P(s;
z;9~ML5I-T8lb#5#JcaBh=0!a@=Si774a`p`-^;m|dGS3z*?y8S`-xenCr>|x(hv{C
zPsrt^C&%-+eZzZQKLrg4M~40659x`@@qR}ScwT-&*FrHeh?RANyQ?2xH><=JE*$^-
zwBw?1SY3CPJR3a$IQKK7WBLi@^owCCtsRao^!)ML(ekF-Z=wjAF9JnFdzCKHb!R{2
zmY?bwf?<X@t7#$o4~PmoD^?8%2LnY@fEqXzc59r{<esPF_$esL{8E#bMF8DZF@N`6
z&mX_jKeWF~3$^&CUSHw2>i`yDSd?X%wYv#Ef3o+bOhz31lrQ<Is{ekNRP#&q!~b6o
z|0edz`1of6%XCsVsej+!|HR5p^{!qL^7i9YA!au>V{smOlG+4H;(Tx>U_%7k^JEg0
z9N1666Eo-bnl=>2@tj{y5);&Usi^3toDH>#i%xOO+q`t4)3yr3=zOT44=U`TU_lJ@
z$)_q5_Gtf!{Q>*uc4BtX4c7|Y<*Pj>J(o!6$4^d=U^p%qiHDi((~5<l#qSR|5p0aE
ziI@(<410bG4i`(07_7TYYzO^;TAoEL!!VSmmwJT|Ex$HJ%*^@#?(&ms+fM-3LqZ4w
zLK4aZO^Y%R8l)E>>jm77^ai9i$?NvSRf!m*5Q$wjRaKcJjA_Iu*YnVRF)oJXLp6Wx
zF%u51Ry7w!e=oaNPj)8-Tl50=$R>(ZC4hMDPM*FBN_t>FZGGS2_DQYLh~sU;;^Ka^
zF|MFlata_iaNz`OHejUp8M6+UZSG~5e2{0jY~IQ+k0TCR0b_geLco&Kd6=(6teEzB
z8Mwnw22Vel<Z4lFRpnf(gnASCtSXb@a+)IP)|n@fnl6h(UW{AI5@n>WO;d8|?BaTI
z<1Uf(Y*r<5YCJk3q}sb8<Y?+Co+YZS*0PwJ(!PDIfUWr~&-f}L^CBo&3JhyGXY`JL
ztS3MRb{0Uif@lLUi5x9>9^J!F+#m6ijWJ^T$xpcPuF~yR-W6cOPp&rqB4on80FFL~
z!YSig1=^NJEamB#av2VCTy)?c95OElp+POq90XQ_s3}|-BrO$7LpW*sYi}{^=O=kG
zCS#2p^%kV3rIaP{D80@&b#c?_r`;W;b4?~Uy-u@8icO{C&Mj^_x9UxE7#k}}^NdVX
zQtG>lRkfJwe^fi;nATA{&3=y_$!MPd>j?UTb3T3uOn4<VtQhI{`+t#bJ#Fyx$OOO%
zx}G@GZ+S8nP~5}&X`835w}23NeeCZhUxy_8?HUZ8<|dr+V}1RUWU?o`ZIx#bD_RQW
zWT4FC+jcG*;L3ih*&pRXQNcMl2oD;ZHRwXN!5h=*DRY;f;?ky{K-T(XG$S!}LiF{M
zMiz_2MW=Yt^c-oO;*q--n<%B2IyzEEsbaG@>9rEYrU!<dQd(USwN-6ix>fmVHtbx-
z=wMB$*LzC(y5reXP=fEJV68bBhUNXI46R-ST@3o?dfM=l$!3#y01$YJ0J@VW`uK}}
zV&mP{j~l+XmDhO&zP>{d<6QA?j;Wt4I#1Cg!-XTsnve1%S&m;tyJaRB1tuJj%4LQr
z&!v_BcwTE9h8V`Y@-lp={lv@f*3%|W-(&DJ{mv65Yj!#A6o+F%-1KMIEG^xHEP7R?
zHySs!#T6lG<(iZfRVyXZy@aObJeTBFTdIg?l&29PqY0@hQ-byi@Qp~J=44r6zTyF4
zXRwR`{ex`lX@e*I53jGEI3A%LJZ<fe)5nAR2|H}qh|Phw-UFF#<v8>HakmoXSfq!y
zv6%AIpgd*QJS9(Yq?XSHeHdZexg8^)ACf14*F0gIKb8XLjqF?LaE@7V<;5m*))IV$
z(eM4_{KZdh(w~vkn94>wQfFk;n=P7ap89=*Ct7K0RT)jSX^HYA*C|h_`a-35JvDcG
zMXnZ<r^!gBqhhlW<xnGaW%>*QrqBC2!F00nW!p{+$)HH5JqkR=?0=vqou>m9Sjv;m
zkV)czr{PSy{Is=z;>-{1r){1#0#7R4!1;NoTPg5sp4jj}#E}3Twyo@m1t%kv9Xx-|
zI#+9+3U<u8{3!3IgZx~1&ZB67552_ffl22{TDh<zD&prM#Qxq-&Y%4>bCXdYN{U9*
z8RaRbEWkr~qFhXrw0x_imuf8M9>>&6fMQPPsj7I0=1wHktGb%Fy@dWGH5$&E_1x_S
zPo58t1W__sP50Fk1t{FM3z1r!HGvq23c|gn{%Jh{@`g>U$!Y_ei1hWiU>0<zpq+ja
zHvD9MXg^sUaj%}Z!@6ybc)OzKb;ZXTmmpdE$#w>det&f3$LA+dNa9z)!n{Vkk8L3*
z<!KN%A6p8wH~3Zed_D-{@TySoONSN=D70#b7E5yV9>X1e@^1S{=V{vMk8hSl(brG%
zNa^=(T0@nP*r?ZUdgRH?c+?!q7s|-3#BMM3f9jg8GiB6QE{@dLlnz(j?o@2t>)(uy
zmgDvPuHKw}4oIgbV&Ks|EhC%`qL!}9`FapNkH#Or3q|KIb#wpot#RNSF6%*@ddVi5
zB{SlApy7ny@bp7a5A7!qM6VH1fUOR&zj;gZBFudi3M;)YhY>aXllR^*IOAYHVEMOs
zyTi^R;9X`G_&1NRIa79E)*F6NcCjqBKgm0h<X|bDc?Xs<<%R%gcMP7k{X~{-A`d6S
zR!P37=C12<dZraG(&?6rNP0GD$#Q8poLtMYe74lia;2gSsI)ASqBdzE`SL2~(TY<o
zM(Oi-F2}Lz868-cQy0}w4bExj8vK2C?#-De-moZTvzd%fJjnVo`9lHFgJ=Iw?pTv(
zlT5JQ&*Tx95YV6<e%iR<(dUEv3GMp)4gvnwfxZUQfKg#&JdMe;5A<OyEsR0fs9;^O
z*qHq71pD|2azm9GhkV_`gzr0&GNGf_M_(I&*2Vw?`e0)e-FZroSc$gE6VHW8e<1+I
zPUur({d8}|U_C5rRCpcu5A;N{<Le9V;z=*==O=zI-0$xAel2^P5_a~%9)5zvcw8bs
z&4C>+23yUb7bJb9j@6lrkUUX)moXVVX!iHh!|Mr*mb-b{zTp|+@8KuJ@q6IDtK-+#
z0JP63clO=kr!U;v-3r1m3<Y2kkPQ)kz#AVzCV1)lzn9er;|Mi9kal$ub}_p+-SJx)
zJ?*y}fzz0NEM<VR#lx&7z1<&nNUBL9Md^em+r{VUjZ}e867G|=ZQ+yU$!VsdzEAT>
zE57y5K4o|UpqM<x`&2Y)5ueZ>-X~0O9a5wdp3<uepVaw>!6($>34J=7PuN90o)q?f
zUjo*rsGqM5Pgcx)ocD=X(o@Ql#PF1F19`$OE>8kq#tL><Atlw~Jb_PKp2lFGY<OFq
zoIW~S54%mqk}Oa8OFZLMffVJt#q*@V6PKsx`IA*so}5OOCyu9F+VD>3kkZG}bebpL
zC*ge>VU!C`bDq@kX{31a9(2Gqk`#5iwc?4NKMj_rR7F0)C?B4tJh?tyj;}lsJoyhg
zY!gn(r<Uc3_X+Z}exEE)98dm(4x}rF3$9x{PyGA|Cs1iKtly_+d*`y-C=7++`nbMe
zSwI3T&@8ivDijF~tOf}o!48S8@(i=hYxJ$>7^XvRw9p|<qvm%C@xh@c{rc5PZ#MGu
zaXn!MUx=MqDSfV{r|Ev`7f_S@1fxQ`HgXglcgk)zbnM+T3ESe5ymb<K{Jw@;Kd|fd
z^%>H?m(K|uy4W%4dFllv&1RgR*v*Ffwz}czUpVLg5;-}|Xr9LT>4xLmmge`IhH-X~
z^H@RG&hi9@Jc-eMVvi4xH+g$}q?TfU`|Ho;tKHhrJ||y#PczJuFg%U+6Qeg9uoNfs
zo`F2J&E(87aS~6R*+%+_KHh2r^b_)Q^?7{J*5~7XB6~c^Z{R0(_tQ{%tUlx?IOXY^
z_=#S9B44ug`H-K0z#dTTwu1=vU*s|&U8OrDZrV@!?x%t519j@BF+2e|<_X>p@^qV@
z-zK2+ef>1l4sxKU(L6!VQ@ewAa`S#tcRvl}0IC1>ljwN*CVsm5T)tx4&vZW>3rgJm
z^u6==*7}K#_EXOj<{R+D?>vpj|M`hb<4Mw+4d^>hWAefE6Vh}&p4`R8r=J|;Ns{~j
z4;~SiB&Y{C6x0-+$ON8*vbrcikHVjS{P<-7og;UiX2=YF>IHQnKWW9gC`}K;*ND@P
zXRE&e?l>CgP?&2gLPumOPf1;*89kf0EXtUUO;$xwOy`NP%++2q@EXFHlmtMK7MBj0
zxBLV>Ph@UCX->NEW1ghH_JUx^U;E%b2f?#51QdT-2TvZM{~6%NJVn2^&8|5MRJ`3a
zFE0rlm+0lC*}g<{j)>(`@VvREM|(T5RtW^S1OO?3NCV3|ej+_j^ZJR=F6_ilzO5}4
z{4)|+hLoyjL5bx`id%={Af{Z_9t}L~{j{iXUD?)chHx;;F_Td^mSBe~pr8h(H^5Wg
zprU?9bd}Z>cB2|)l~hq3;h~q9X1jt?(=){qcU2O#1FJe+nOd60IG{o<ORS~{R8^&A
zl9gpfCxzRZGfK6?IkBE3Rcl7C7<dii-Jh^$e|ZwZFmRryUQlF$pJw4n|9ryy-2CIo
z^8{9j^p=4q#bW8L74_Hta%~jG%@!1s=rp_=F8%eIkwxI6v%p&8Sh83z)sse%@xf>4
z98bkIK_23)*ll+aEqBeXSk-76L~L7AgUYHc;vp=iG{@bpLU6Xb8jtLXZAs%@Nw+C2
zn<7gaGh(-Gn%_B{628VOwy1hC(q<f}4kZ2TocsB8!xJ3vL;%PzPnd~32|7CbA5Z*f
z=JwOZTiZ{b{AZ3}=_7P5!sNNqVZ`6iO(<kwneznQ<84m}--mD`M1Tl6ge@6gDzH4U
zuq9d8ID!l-+$imcr=;07rTJ=AWkpU=I?*;Rn<UGNRg-ojUeYXjNm5hHX_3eAwp?xN
zwzf^OqFLd+q@igOHBpkUiae`!MHV5b<3s!f!Ali^L-<l-K?FAa#0)ROnPv=?2}}SO
zIhnQ5!2d_+Bg7~3#MV!aR%*TO7vw^Rfdfvk370l-*IC*$)D$*m1oPU*gO45M6Gu_%
zD-v?l8uH|C79PV6RSU9E^i4l8akL)KqGFq)Z}Pm!v#c!gCN5D-b+O7jo}wnEt0qd@
zA}vz7%FC?D^Zcbs3reddDbQYZmX|b7;v&NlS)L-O(|EGXC7J6a=2`*yqMx2KdIEXK
zQ%3+f%M-M%vy;HPV|jwn;jI5Ed=Rx<hDy21rA@j5X|?JJ*VkGz?OA-|fUSXqmU+0x
zafro21)w&>#oa~`n}b*|9#~v+i5+0RyI^nqMo`DoeyR*0ZmMXTMp1<Vib}&1KwU##
z0zev7IS$E@CxEhmA_XKS%j+nLva|p^ROb<tSgIh8vpTEIhzOEAhVdaAZ7zvz7?1@A
zaC`xuo^yHv*?wST?<c^V;ptdVEnXwi@<hkA1D*~Js37odJM_@+&_;R+Kre{3As_{A
zMd-FX*$OarNI-kcLi}tSTn!Lv;&327V?OSffeXM|x-c|5tiSTq7ZHpM667gQAdX{X
zBF(cDK|-d|CPvhvq=^6}l-6yrjtozbrvM2=g&}T>z?Ef|BaT^B;D}eA2m>K-Y9pR6
z5;`C@S}ZJJ$dlzsz|hZa=fI*ZPfUW807McHWQeDeUz?uaPm;4P2>3ymA9$F+)2|i1
zaa`X58HUb=^M!3MoIRe{CUkuT(zb1%bv&uRJc06^(CT>N0f#sJ#7_K_7iC^%S&5oS
zp~#CokN@(N)j3u(B&=>yFvXsyD6fkmYkA6y$09tEV=JPbr;Kf+Wyz3@F$4GQ(9@uO
z$M+fYJjb><%%16b0@m`xkR>UkAkeO~-}RG$WQ?FD`U!;WA`YQ9q_tsx>lmiB=?Sj6
z>2S$f$1(}<uM9$K30++PBLLjIY2I~!hjMsw=EkHr(nVZFRfbfU9d@cX$>JDLR<AXH
zB&xbXj1tSRAGfs_GQ5NEB<&`Pk~m5LN3?HQ0)!LBa5L)plAPNbfZ+}wV<O-S=n3HX
zb<9(m&ie^7x1SE?cv>I*JtA{-Oqi|So=eBmtTC%COjT3}8GJZWF>Nzoidl|;tE6XW
zVvc@Z1t#$Xll*k3hrXQ|`Q~Xxy-Q+QZ@A#qPcyGeqB{P|6(;`hxruUJ4j?VpV&i9c
zc|Fl7JblDZ#(kqhkM7U!r-^fslED0-KxaPR{JEr_dY(Ez!9<=$BR|xc?k75>4f{z7
zGQ-IR{5ji+uj;4c`V^jE&rf4BVJIEw(Mf_Ddr_nOBrfEqiCX!}f|{J8@3%nL#^&RG
z>hDbx)J#S(H-S7|LQws7mUc}+O^52!{Iox>;TiYE%dbuhPZtQCcb@dJf|_57Jx?&*
zPjJH#;-;tnRkO#_bU)qlM?~E8?7wXGd4g$vx@|#`J5QI$!^zWp=jopLc*5Lyy5~Q5
z`lh+#bdfxW^L6rm1k(>Un@7*q@NmyS9{bRPz&rT^m8Z<m+hM%?00000NkvXXu0mjf
D8V_BP

literal 70876
zcmaHSV|1lW({3`cGttDhZQC{{PA0ZBv29xu+qTV#ZEJ$FGtc|H-#Kf2>)b!G_wIC8
zSC_iGuDV0zWyRp2F`<EgfZ!y=g%yE-zQ_Rq0bfIY0kr&yoXQ1!U^t4XJ1W_jIJ)TD
z8v_X#*%%n(OIYcf8Y>#>8@bty7;^&wfdrW=t2?U8NOKz6SkdZ#_MvsPvIUF=0^;U#
zwbeJYFm}W@Fg7)}<{`Xn>mtNAH{u~wW09eku@y2lGZ%NaH&$|&RW@|DFyt^I<m1KX
zcI5;RurhYk$9J`|w07Wh<stk_mlN>)vzd+%|8EyZ3m(FMm{OOK#}~4(H^yh7Wu-A>
zV5Y}s<Dg|=rDtYgpvGsUXJDhFXQ88KrJ-lwq-W=3WWfK=LkO77-pGVgQCRdpa{+$w
z5SlqU+H%s-xwyE{x-ik&*qhQZaBy(Y(KFI9GSUEg&^WkRJL<dASUV8?OF`J!!O-5^
z*3sO?8vm1`zJZOCBM%{f(|?v=Wh*1|U&7W7{}B{GWOT0jwsZ`%^mJBMpX>VD+rd%M
z_<z{=Uwb<!yV)AkDH=Q2IN2Kl^kG8u?_hxJ{{Itwb_8(4DQ9mEP>Q~#u#KUUm9e#>
zgfI^w;6JoR=0==M^c*Y(46Md9ENtw?G|Wu;1~dj7Y#cP~#!MzA%<LvctVa6(()r)U
zvk8bYF><gm(9??wGcbs-bFi}r2r`NavWl{?vx^Fe{ySE}+QCuZ+R*slxy=D{|7R@y
z|932>kiD_Kqm8|?jg95MRv>R?<7nexW@C#lB=lG7_~bJBhUV6vjTE1X`e)t3#`fmU
z#zvy{Hdgq5CCzF6zi?s1Vrb05z{E<!%wYofW6WSoW5CM7NW*Hx#B9W_Z)Cz@XiWH@
z@kalb1pl#AKnq|cw*NB3$;QOWp%0K84ZVpWBMk!sn*j|w3ky4qiHQND35PMOJ}VO=
zAsv84x=-Wzzl`bMCV+*0e*Ld&0Dk<}xs0s=9%T=39b~azszAVqXA;5!%C4(tIuPGc
zM80Lws5u-5qTmxi@!ZksxW^Kz3y7dls3KCK#ZZa>4}1|<^C<`wr<*MZ*nlD-D0uYt
zviWFg8ZmL8NhXQ`J!HadtE*$uGC54YoJhwItWZ@#2?VGH@Zjpj7?d%kO25)|=_9iu
za757&pKb|5h@Iou9}w!aef2QqiNGG~knuR=b&W#p_&QWIJA!5RJkugCs7)15n;a9Y
zHb2%Oo<0TPqm6L6%b1wM>8Xtnjx}ye8imfCo8V!P=sSJvcm=w>6QK95IpDWLW_`&T
zAcTgsqlBFi<p+4KeP4}qQn~PkOd4#yt{hbHC>|0ag7(y&^*vQw-F|gde`;86a8-Or
zqwghPFxJd4@M>-LGOE%4?TFN5-5-oiyu+?>9W+rCFG34%Uk_OWSs!)wBy5TjI)PR5
zcz4Fbd5D_W!DUY)1~D~c7xj?kE8?)G8^s#$R2#r9+b9@4nVLr%w2MvUP6Qw32ZN^;
zI9kAjmpr!ltHa*lc1EQ{T2)e^$>pumOB;<^<sy;2Q&S+DomO^o{`(@-P5m9cx6X{S
z!Ivfzc2jr`M9;+u=ylM(?(XgqVFptxZe*?Fvg!Jx_I9f5a!Qf4Pzfn%aWqw#@I$lQ
zjec&MsS+L91)fng1EqG}Gb22&A{G1e=%L8@Ut($%_MYOYs!h?SU%C0e(T=xzJ2RFB
zUngWFyX@!9)@RDVnf^vqBRJ`;HuK@BB~t}-zx(E|mt>L3Id|&6skX3@|0HXrxoTN-
z6_QDI))7Uy<XmWZ$c?MmWB<&;&O0+AD31de!?t5zyFfn_9BY}_9xK^3ly+~o@C!-t
zH!rOz`f!n>TMV{^^x7~(yvCzwtEU<&qhZpilT}^(;)}LLwqo*qn-kS?su7M}nW4we
zU879CnVv}Y56=vrgZ)nTr8t_$aQpLT_K)$giJ51{<D4ty;T>fP<J$E+T?*sb=jig<
z(Sk9OJ%O+nBJz0JbPOErDUewektr}^RT4{6wKa~TE6?u1QYN}V-KfPzRfwv^nu<Z)
z+9-$m)Y1tdc-p?!QscTk6=t!asp7oo$?I~vZPBw1cZd;A`sS4}1#MT#Yf;J?=~e1r
zRLc0t_9_U9cm$ReN$L}X<6ZZdyo)U_71gN3*&)@7A!8i_Jdku-j7EoPKGVy!U6%^E
zdeSkm@AIPQ=+0$O+Jf}B(Y+Ct_kA3E^sA{6*h)GRID%F#j&33vl4rAu8s84Prz5bV
zOq*74za>LK@N_)b*K2HHADYz(95(al&&_{#nD1iB7HDBptemgrQ1E4<;JQ7czGzjn
zuYaakr6OSUs?F&7b{J2?CT1EY)*|v-Ac_{{Uh<<k!zX&*Tbq`f$26?fpIz2l8$PUN
z7mK2)gf9fiHp8&c$f%-jK;z1rWI<|8Wq9vsx&5MmWX>-9Pj2HV&J@elra8^xUGd9u
zd)3f!VY6*Cf-6Jxo9Ume*@+sv1-EyHvo;GH+N%u1UJuX+brVO9T0;nztPbrc|4-*s
z(1<t^-0j}4?6FUsav%3)snLiacSRpp`!P~)s$M=#T^UuJOSyDDz%tli8f(j_k*pOM
z^#*CV>P(rCKgPDxytyGEBphhnzyGK*V}w@#by+|c*I{~5z;M{(;5d7*ZC^`!$F4jw
zbnk0CyMNMYKep%ZQ%C#Qy~51S*DP6ibRXh2=3bv+JHMgcDp|lIB~@erF0nT|e}Mk^
zBvM^cGE?@MXjKS$lJ1G85U85JR`Rsk(+jR|Al^V2@k(Wnhs9ynUN-rUAb@~??9Rlj
z=kv`-)EuScS(fp@SgC2~xI4B;AGqTCsgr6Uq&MJRhvD=9wmf=V;0_MDGi~}*X!~ky
zTzmQ|3JdTqD%&iurj>N`Ik5NqF=NI(TPyhYynFW!(R_U%(Lqj}!G8AUFKAic)}2SH
ztlpb^js&7Od1n+n0$j=~Tx&mKu_)N)g0DSsy`mOQL~(VY+$?!3(-h|eU#4|yQ6yG)
zgq7r>`smcIlPuK-bIBo=U_*r>tzy9>Vjs+c$e3z&3!5R}f$H)5QL-)0>nSX6*G+JA
zgV-r)pUlY2Hw8d*THDvTOQp*cOL2p&9k?BT+wDOQ2s;Pq@)TglF5rtm9J8a(9BW1;
zdJwVV8Hy0KW&HA2)2M9eeU`w-$v%38v9ACe0bNQHC@{vnNCh9I19#X{3_3Nvv=ORG
z_*|nZSCDYV&-5rXAn983EL^lu;-G7iErKQtvZW!wma@u_$K#%#pQqJr9UL44z_i+7
zL!+37*&Ry8`ERiVzXvYQU^F&rD6u<pdt6~>N-XYF;&6kWrt~`Ovh_2Z2QG6f&lpu8
zsF35F{;}uPfVkWN2G%$w7PVyVD^UfP9zynN$<flrve|k)W=$=n6vJm%vRKy6Q7vCU
zLq&6Qck8X3G5bXdmM7SLWb`^pNrh1QW01T-ZMxn+nec$#f_JwRrI#gPgYfGR87KRR
z>k>m3C*I0)N-xiiA8H3~kU=Dh{~28qA%kKFKX{lM=p*D~-uHqXtQz>zbv8wcs_uPk
z94h*6q!Mmd5-R<NUko|q0?_g&2Og$^i7zedGC`J*!qX|p$*0Yjb#-;4m*4s~Ko#WY
zda7pRz3pO03kof`==URw^2Ehciydmv!T7(o6mxZgA4;Xe?B58$iqUFI1XfO8e+fgK
zfY;@u$yK35wb60W&Eet@&=Jv++FxZl*GZQ_PW2zAB|FX8Hnz)am>DTr@@=@k$=%om
z@rFV5a!kB|rc5&=k-bnkHeVeFI}r-f*S5{G3p%9c1fyDquZ#h$F<%bQERxKish|rg
zKX7#K5V#yaMwFpV#)1WRWkV5x(kX4!-)eO>W=!338SSFnU>j;UHO)~@K^LzRB%BHi
z*3K2#!+Vva<m@PmH|3aw$HeA$!o|g{aWO=|VUNj7RwJ0>1Q%rqh-lQ|Q&KFA97ZZ#
zZdWKuN|uz=T1FaUCW&J#4J8sMMuMr$B}URm!O0+Jwlk~aipZ|Y_VMLYYe}^b8Oktq
z@NF@idBBHBUNhk`{lNeu^2I6gYYVL%W#k7s@^o}lG~2X#ORR%~{q60oiZ!K;jg9VC
z&OFf$UN$zh3RR)eUaKDsnmPP@e0=(e?xrT|h??NjkfW26jpE3C#^i~kYfUyjO-AOD
zKgwohLh&F)BZKDS0whDlY|BvUzS6{AW7)U%3zAC;sP^h7D3gMYicpIu3Zs#$ZYgJp
z`32+%NOX4*6OTLK0*~AT3^D0Bo1iHt3iJpVjM|=QnosEN%NFdc_U^9}eL-p*DugG_
zbal-12L``#4BN|6%xt_9ycby%6%0~5f&<@<)Ev4Ntq)C#7o~Z%FGg=jCCzNWqLt4S
z%=HE5t?hbGLfqLu2fKkU6N$2vOj+Bw(NUx@wI3gu+%|`*Ohpb99UK}8=F<r|E)ao#
zN|a;b;lUZL1W7iCxKXJM-A-KZ^!j77*@=v+Tjg03<)@&d&mh<>_+6tejmN;D+luvg
zB5k{$QGt-^fEz;|C)?akW$-#fnNlB0uy}@;jO69%rTY=mU`v2o23tPubQM0!QY6yQ
z0B!FSrNlY_7~@qPj#l_WfBqYo<r*ay+62zgG^02Fx0Yf&ITKH29D+hP%ttJrmr^<h
z$3n|2t*Wm62`zl?kOQK7={ch*sV9fj3-Ew;3JY!;BOp+d(_7a##Ax{32^-Bby1qIo
z6BE+vaMuXXA$_FMK&qk*SbX~MO377N0=g~k_rYg=yR@8dyM~u`10n>$>X;R(g7V-=
zz=Q}02;WK+#N$0wVTmH_db(&|^WxzB^YRE2+rrr7xp0*O^z&!B={@t6wgb#TcfWSy
zP9;)pWhB_KCT7T#AzS9QXCf~T4%7R+XUx>ncg2)Xk4rGrgK}Wo>sD6Gu9|6tPNvoe
zo{=x1DxIwWUk7gQVFM?GaA%$P)_h5-6oYWV4@CIpek*-+9DY5w`K6c1{8p-<kaX)t
zTk?^qt9+Iq(A`sV4#Ds0iT$dRiFw6oCmH|2g?EyXMib3LNa+s<9;%ot8j1yhklSPX
z?KfKCY<3RUBg7c5own+bDYY~xC@98MYHI57pI759<LWWxU<(3)9xr<%2?_N%Ihrn)
z>-X1>2++_*<1EAHZ8d;<ZziWxt?RJ%j}WBB>nf@)Sfx7E#9vw5-*C}Zt0m^nA|l4r
zD*35ow%S!IG+C_IpPDwk9Y{;{`+{KBeBN6o?2KIMFpBW`^st|VlP7__fs6Cyiiyc?
zp~_fJygVLzo#kz9gCB5qb}x0BZT-5cr>E%&NU_gX@`K}-C`32C-zv3RddU=(a#(5&
ztRBg8+0gw|Ef&g9F)_>J$&rwd=agoL!-na|IHrul3wf4t5VR_ZKtfs!Ch7YKWqAck
zCUtj9Xy?Bz!0o+%2Z6MCv|~3aKtY5E$mG5z7IVU#XyMz0zk+Ggwe!G-r*mJ2+Hart
z!d_@$BY!jW$xgd&l9su)>sQu4phMoI8f2PYC|7f_-rT5nt6OcbaCna=43VjQr9k8(
zCMKq$I^F2-*b(dTr<tiO=^ql#fMj&J6i_f*%GGIu#~~ynTCKNUXs}qa=5fDsy%~u*
zU$t2H!EQ|n12M;3a#M0rtu>)kXYGtWJ^=%Js8B+CW9tG@7T)3w=ZwW_P0b4(G27_H
zm6d_XY=$4u$BGEs1-w;CRUjL3W%@NW8lU%iUrjfNk2eDW*cyw&-pI&*xM|Z3tGC$u
zNuUmG!u{?Hc)J?L&-SQLP1o8O+!B&uTRVrdMX`sBq?7f1^qGA}%NO!hFE|P(jm3gA
zgB%-G&qqZ-J0%hv%ObXxu$Viur!*HP;wb@%2|a%)nX1v(dtL-H#-%|>hy<y$RlH)x
z4Avb)PRqUgs~M}8O9h|$P=R_V`9)J*%Wz<W7Bo^z_jLdS3$F$BqQn>EYDLsnO1TU+
zsUuDCbk%mh+Ueu)qrh!$u^*_jTW5VH1C{*XaF1l$(B*<&D+S)NVp32A5R$uaDQLdk
zmlJPp=Hhy3RpMbiIDXY2DlSi3Jpw|(jn)fcrkFKu@q7x>=ZIp3iULxs7qZldy0*5}
z>R}8K671nqlRw!f78P2ntLb@YI>ajub}bq-3l+`3OJycdv&Rw1j*5m8$PM5Zow$^{
z{dfhbCvR<@tIUIht2ixE9^42oE~q7q#2Q#{R?wWBaNP-=B+9}7Zlj=}(3{G9oFT)5
z0JEEfY;(E9rT-43W^>{wg+@mpNm=6lb$gdg*0(#F_enma-FPl3C=yFlMC6(*C^)!p
za7(~{{)bEx?(Yqc!{4h7LI#)XI<-b4JB3nNXV1hL4b~H@)z$7eoKED*-$soXc~6rn
zw0c5EMn<Y(82l<9E;i<hbhN~Mj;8fvP<-I{X#Z??P{em>G?(~%yjOd?zk<D^ZIzi!
zWU$=(msV${r=RZc7sgQOx^9DI&F@|ek`fc&gl0tY)&YFB?4+sAx(Rq2>Oopy<L^hN
zhHD~XiY&d!k9+VM9%RVqICW0vFWc7-(wPENSWE&ySA){|8~CvjZZD;%=mks1@o`8n
zx3|7rzE@A)Z9B7qi0ymT=5Mv6+&z7BZbZeWtCOi0aOoG6CSxRIGEeU9)PX<3nr59y
zxJ{`vSn?HEUFN|Vl9Wu%9UHwQf!h&=Yx0M-3coC9I3BDZS0N4~XEkNZxw{AH&Tse$
zC79K+Lv(WR+w3SHk|dbaujJ`hvmN*^!87mYEOvK2ct68+R?%`3ei1ht^@taXLjS5j
z*E_Vdw4)D?%ZOg~KE?M38@*)H_x+0N<Mf%I?9pHs9~=Ek38Z>33E~Gt!P6W9-<Q}3
zbOt2N$hocz?}Kp+@d7txCU(%`?oeB_*SD9`s^UDk^OZV?AASA(1y{o%E*l=nk9SJs
zC^=jjKLh)JqYMa1*8hl{g)K4Nej!McEXAa^#kXDV`oM+I4Ty-s=1>$RNuh(X4`#2a
zsj+Wqx5@6bB^3Vi;{_J!YYdpYP8fzvUfQCKI-WuA%%4Z(t+J9@*JZi^fu&4z*0u}m
zZr!EKlI*Gs#|Uu$Ss8*vi8`jb@81$KH7kPPZ1nVUgQH6<1yrAtq#=4`sgHlP9Z62O
zA5i3me~ecMp8Jw1m?4I>=>9tY4uU()1kys5N4IA}&0@aLY0@~xsgn`omXE(j!3&=p
zLUCDQ4(U>|rD}(L3yzm!IQo@wKUJui-WBfZOAs6mbVUkZlieG{mD{p*i2yA5$`y%`
ziHTrt@FMM{vLa<KzS9gGv@nURfDQd3bO|aN8aU0q!ESn(QtQ=kLaJ!h*dNp_V6R%&
zdKOk>s;*B<8#WTrnH$>WlE0O$83VPCivsjcgP}Y@pd&5~cSFfIC@4rti!2_kCp4KM
z7>p{9b3&t@__m09W`JVS<MSzI`3cimt%8L3X>9XyaN?2UY4uuR>FmamCfc0Oe~v2C
z|13?gj<HPD)GFPo^R<Dl%#L1Z7rC7ylXFm-7%_^89iZ^^WCz6{*^uMDnTjn}tFkTP
zNTT-%SO+giN{DggI(XRl?eTc|+vU&0c@z#8y!#Kt%W$$mB3Fvu#vWK0H!?G>r#gBQ
z&T7+}o5|G_ibZEubXj5~$dbf~(6$ze2=HkS(T><$^U~;O1alVV>DA6&r}gC;t1vae
z?|7JcoS1&!dpv)_DcQo&KdNJ*Q#sJg^M>`F1(XCiiFM(xU2%m?QSwfomhPb`*oIUU
z6N@@^it!XaGOA3blYiLTef#-X%bXp>AK;n6tU6a|-N(k%saf<Uh~Lh(ujf>;z#BMe
z0$T`A<SCZ+^uzB;uuo7d!qen;+QO9qI07!{PdUq}Jqw9zUpt^HZO_-YobGPTh93e(
z`Qq`3t=s4rjl|zN>ExRq2^%mq^Q&WHWA`tP2adTraBn804=vMsXT`<$DEde&gj4FZ
zKUf%C;Ts_WlzZJE9e9;cM5HWJq@!?op=(3G@%%wJ3qC*An*a&|x=%_^huMBm>~ub-
z6n;O7kIt115a$1GY-%b*xRSaujIdCp=W};7^OJ;$owz`$ST=^*Gf1n++RrE$77Mz~
zmN1*b21TdIgl6fS?ErN-umB&1-RrIOz8wcC?^=VbxWM-v>*3Cp9juq&>jp1|Hdp@(
zr@$?+6$s8^IAm3}A|B&HiUQ6CJYtsmUg&x1zT-@2L@M{TMugB-+r_Wo8kj-pn(BNK
zB(qYZ!eWR(TLk*99Dd5!1$d~DDLV3Tx~*Vx_Bpb{J@OmLrXPxPMKb=vR?ah_6q=CR
zEy)N0kp{_LX0WxYG+%;-EmrDt40%N44U1$v-g5jNv+%&JK;pA3?qUO4rlRBapBt$Q
z{>&M9bQs9nch6^VcKDu%d-)-XkqJz-Ii2av>(EC2kbBe??O>v<Mv%{JEeR<aZ6~gf
z9ZP3-4f`1$pooNsi1-x)1IKG{YU&##tRQHWs#7dxnumPEQ=9)Kc4x*PC@A3fcc+s@
z;5?)>GQ?~bTQUly%2DliUrG!)Il1fE4tc$T!UF9MM-Cn$9u5{29rc8?H223zoSdm;
zE*GKG1sFKEK*uMS5x06qX#oM?f!JSe<3Z2ffqI>|Ls<LP-M0zTNM>$XTrRkv!L!u)
z3>o0SBwZ!dKu=qX=UhvbI$chuXefrcnaRlr4I1y8T+T;ZAaiC*zcfIgFYk`$C<{*(
z&$Yt4o|Ez@$PZmE);WCMJ)Kn--;lIB+*K%JarwMq2|e80b=qz1E;o3f9xs)M;m8Dk
z9s;&u+eaW&Cni$xT3<RaUI;uJJ@!`Td(7dN&E27AoseHk$CY|rl?hbQ*u1b9z&J(4
zg+g4WV@Wg>S`A5~1s=}lK}z^^;!61o-N7GA*=xIiEupQ>bh39Jhtsj7MBa*-(eo$u
zQoIYU{T|zvWAoQYO_*4hvt?kH<fODl`+=}BcPec#@{^<5YKJ%ymF$@Svp67B!4Sgf
zJl=05za$iRB$9=R2Ck(TDUr$hsWs*xq9hBDO27$t+rK3i2PX#?PnHA+!6=tp-SLC?
z4&OfB20!r7wb8tsP(zMyxMS{!OyN*_>2Cjo{~|u_jOHu_<Dag+d>u|EP*+)9O^)Ba
zX>f#v2iJJ7`(BueiUu4>F!Lhq=~*#fJTIL)#fk|Bs{E1XxV5riA~h#L%;>0D!4BW0
zAg{h<0gjmHdLX^U_w@bQ&#`bIkT^H8RMD=l?#UcA@XtP6N_%)w3}n9_0_<|^6pZWF
z8?c9+(#%6{Z`z6Z1ZDQUi;K!#&MVm0X~<IH9Xv#Q+zxm)uvlayq%A}%++sp+cUP3m
z+peofK)OX>M!VJV7;qBiPllO5aeaSH0P78!;D^9MKv$nhz}pW(51hWEQiN&CS^l+~
zGFsVTZqq1jCo-rKs~Z2YS`i`qRhp2hWWZBFkc50b(Q2(;LPiQ1f#7JV%V$Uyr`YGZ
z&~|0<Se;lgSIX&?l}cIptm7+85S2@Wn+QY-7e_00VnnNdu~2$3+-}k@CQKe4?m|V)
zD@8j82N1X(^oZ);#AT(y!4UH?y)=-#AF{QXEQczXYfy9xX}4!9^k5&cuj}oE6@!~o
z-%c#PQ>?kMYm3bUvcoD3>I<_&Pkk%vs&?1kN$m#xkT!B|K{=2HewS)?=~9*d&R+qf
zkEG9A6xh0tKte*=Gtuf+5)f#*5O|Wa+T-o*y~FA^uP;~i^o5;^#xt+9H0$C5q1J#H
zJORR6Hp9tc@ob}m4WFL>M7xUzT-1q$S_%}}k-j19-qisn8Z{j`DAauF8AHYQym`Iw
zbv<Qx+zs{$&xiobScfqgT0ERoLE(x)(8^N41g*S9(>5#czIOe2P#eGf#x$@|%*k_X
zwVM&#A-EU>PsH@Z&UELB3d`32jC|GvTAuv|8v0`3_wDt)07)OQrR#w_F}CEC_!6dL
zH|y)&G`vY!z>`Rzhz7?>ud7y4ydBa{(n~VpD-hU-AZXfIeOcs6;QqptlYZf-CLEja
zv~J=fr0>nV9e)OgHoLsvf<|=$Eu<rcfEJlLldMKiA@k%7S=>l%b4By1NCoR|3%Ua%
zHsZiAYR;IjYWzOtO}bF=`2fdHrsVPAXSb_cqZ^}qHb!vKK8(HbI%DwQ`2!k#*3y9N
z)-iYTpUWorx<JE((|!v}7zcIp&W&%Vsk6`Y;a+qT2Sg=B4!1X9&@dKY^LZqWA{MXE
zj|_?hhCW`rg1X*WOxADK_F#}K&2PQEgr>FNnsFmWy*z@D>z(n%YyOh^;L;=IDyJU(
zYf()Vo3U+e5X^eC+U;h)hXA>fx^9>7>Bc01oC<E4f=29cV%2QlOf$4wQ^_-u-U>f{
z)KjLgA~59YfUQBlYS0@XSNE&V<eyn}@`j7bWQz_4fohfQ4&K+GZAsD=Ohw(!16j4A
z;3*k4#-@jRzRfk?xpF1T?%)pRT!OkmTedts!^ij$SieuEmN$Gm?83g-auU))pNa{+
zM(=yZ*pn!G1)HEkvHsda+7?neX`;AzGs%f#x;$ynBeK+>bG+(gB4MFFxEF|X=>sfS
zR+0lWUw+GcOAZaH@}3a7Q#O~%j3W-6T!oB4<3k>402PV%WzZ*S?@Xo<KPJ*bBcvt1
z7;PxRIR5IFoMYTM_81pggHEbO?!eOF_cH04{m5Y1d=j&`Y(ZF%{2#e<Tc*C(=wD8j
zq3u#+9*~+Qp{^kz{wS+SX9qDR^c||Vj%CZHd{D`DEYpZ=eeIE_5NGf!2PG4zQLVe`
zZP0x~O}tO1NxsdvQmjF8E}RXhvr2;n7qS+}0(FW14tThUtstn4s}FMHptYGd!ve?;
zXbeZ7jLl}h`(yf*M+}?Ae~ldm`I-l%-J9}T1r&z@R)dy=xX~3;I_h>DWtcF@<qV|`
zr;A{ml!%o3x)DZll6Fl)4(ni?xlJ@vpwOObc<g+&fr>Wj6Gr%8fmH67bA6C6cJk&&
zsmt6=JRe<8AC4HIQ4G7vGx+SH?xn~IS2p>}No_`GIP~847g3CX0rCf$9Ff6Lq~O(*
zQ6%Y`5U#85CujnIH1WZ#f=63N6J#?jDeJr`yp29-ptB^N|5DMxAR#-cH~f|Krw0HN
z0)Rpc%R^#eWNlPB4eO2sD)7%ol*PEs9%3r_2N(6(U!3*_;aX@ilZdl6!$77pTK-qq
z#-yHse}?N?lIc257zq8^dz!iFjqv~(Q?Jtei>&%7$gIf1!t^4`k@Zbw{G>_!hY5gg
z*^QFMtkgI3KP5d6+lY+N$fRNmW)(JgJ7M^@4YYAr-=-R8(fRL?XoNKtNLM>NSilp3
zIq3ZGD;X``s5Z|GURAUAA1<OSO?f1W(0h+K^^f@woz>U=p-taNrE0GV9j%x4+x;s9
zqB9Hs8fxjx$GdRL^ApbBsg%1Bl&XzXUG#@)kA#^|?wMcg;1nsZxCTGz+8Xuk>395S
z!9PPUdYQqMzggPMpM0A+_A)c`Ax_+KiTITE_9!|RFpA2=S*j?}ivjP85wsv?Gp8~c
zde`?)hGF-{kL+_dqYx#!fBYC<e`QPK!668JMyO2brb7o{LLeY5+^rdGCaQE&MXJFY
zAG7TPa+iWRRBTr;e|g}iji5bTvfFRaX{a`;g|u3Zpw$r~o66mLLMmf&*jUUQmai$*
zg@%D?(4<MG979X%3C4mA`uX!qrvQ+g@1|~}<q9AJ8^F$)3EyiM`zfxHpd1A-FGh+m
z$_GsbTPZ?ihVA1!vkSudOZL@9Or6f2V~*<zosJOa3$Z?M^=yc<2A9i?f`WqS>FJB#
zZB!^3rw64<#qfB%(ue`s*?QvQ!+_h%mf6Sq+v#FOoA=wx`noox&-?RXlhvB@;bbOY
zdd$edP47SFXDfAxWYpA_fPMg==I!O)V!kByz8daROc9ie&Zw9=x*heu*9*-Ci^WNW
z_|;<dRfF%3<~Yc?hF-WEG<mF6>!FZIVbSTgPOG|t5b-H<>P@rUikwatKc{xT|52se
zF&Kft<ndsc!#`~mPb$L+IQN|_l>0r9%4R@<rlzGq3qP$pOws7HIs(pXw#tOD-K?sv
zpJuiYd+w@E#qE7S50m<(ZNb>=`NIFG+$|%h4K-L%QQ?-9oE@c7)3fGi(ERS&@>EHn
zyQQV&dd=tc8e(dv`!O&ukl&J71|+qIwgj-hx-!Bx*;jm(^@EgTQl?Z<P*Aws=pZ5_
z{0=gV1c}s7SR$%j_UT$>SmZJ6!|O6&TmH*Ph{Qcg_)Ufexud~8xw8=Z5CKCe(xU#c
z#GyZ5UqvIa_)cJ^Cd7zzg)|-!<MMtA&$p2M_>C=LUjL`cAXu8I)Nap(7jvP^f3*Q6
z8!n*{kc;W>7hbMZtJ-L`+jTWOs4*P6SZ|rD))!jl^?G40X#(`{TLoNaR{;w3-PZMa
zy9eZ*1ertd5E12zWitU6Ho)D<pByym?*1N!+m#-Y*9;k(!+tLq&%4?6ny>&NR_&KM
z@CzL)YokKGsA4HfGWVw{#h?I19uC?jLHIWTI=ZFw8TH$XG!X7{Sa;lO$8g}@E!hSA
z{rY{ufc1Rn!@1v9Ah~G<tlkT>`3T`JWnl)wTZ)Nu2Zn(wc$*DXnRUktB8u0%zx~PP
zc0+1hYq0o+gEJ9qYGNWezTD;0)z!sE9}l<!2LQ5gIn2vRWBcwMxOqJaA%odqa!`!~
z@We1E!w|CByj}pysGb3w7-iB~{h&dwA<}N7m~2M?)e^3-_`}3YgzW4mH=z%N?3&>V
zMi#!nmxaBLYbu4wqwjq=(~{~FaG4w-ozEB(@1Qh)FnR1yjeiS>>lzfR;FDrSdZJow
z`ntO~Wt0ItLO`HrBArd()!4`gh$ApK*u&lZ%Lo|6w`dgz<3XEmXlQ~)@;1|qTW$(D
zoWv&(U<8oD!+h>fZ_kI>!LS%e=;)Ac*Pc*DP;P#n0iXm00FSaoVI06i`i#uRJ+X46
z60tdlmQsRg!r(=e9m;WYqs}uU(;3et0>tk781;rA?k3ohtwsleWltrXsF5BBI2%!d
z7Hd*SC})7QY!bDG{r+g;tfO(QX||X1*>cVAHWxC<pmGq8^R=eSHJdIT&!=icWbz-Q
z2^4+72+<J{0lyC>()sS^l>kVaEgE8ht+k}2B*5eJ*+@yjLuhDet+#sv`QGm0pJ?i>
z)*j;JcrlpG5ci_SN_dVgv!2YK(_m^Fl{VdU-QEA3`(A*oExgI`hU|4M;n-gCw0op*
z(W0WGKWq(S%@s8w|1nY}ISELhPqiiDWH!OpyyrTZM>}5&affA^!ZV>R?DI&TS<7b6
zX??yu?0SDWH7FB0!0o(0U3xrQap?I8_A1XEDSEr8Wdn!9sVwc}J3l{<!1r3MR;8OW
z>pB1~DQgwv<XoZIP^QzyDz5`D3-70Gh;Z<sl{!<g2=wa(H9ejrW7<D2v2;POxOekP
zZ|?BuxBF9Sw-{=QBdvEm&84Rg*m5Q@+IgGpR;M`*5-j@<eBNN1jJAJ%>wQ>`G^|6s
zCon^4x%}eINC+-xf{n0Wp1fetnvHxM`SsGGEE+8=4wfvhTKr?$qAGu?OH>9k$I4>*
zPj>o>tgtIrW6sen8pSp1AV4cVFUQ4#_<ZouMN%{!cwToilbKxlh5+Stz1~P&6A?-v
z+UAQzKRi4f2wiM;eH^6Q^bQV$fBSg<b7{5KX!W}lm0W4snKJRm!`X_(QY9Ft=i_fc
zEbpM=UIw5HZ057V!*-Jy93Ou+KXB{5n6_6R-(TBYERBe^|LF<vJHKU{vUzyurD_?3
zCQI=0s8VGy8mQ^0+YGt7OKmrD>KU{8;S1YVecZsd!JR^g@w8gN_Tn<JUf(<+H8*9#
zmx5O%Hg>sWoxKV_1s(3UO}Vx*$N#gsTvF;_YokQ1s@Br<fXRrzj(68ng<pXt!_QkI
zoOna~`}G9UdOwwMk4sjCZigmhD!1E>9|#1i)vDvccq%Z6Bt&tQMqN>L_0<rb56Y8y
z2L_euujMuun_zgHrz?OwHd-8lU2gUSLQKx?9v^)^-X8Pw^78WY0T=8Zh(_}{@SD|U
zJEEA2B|RUSh!-Huh12_J1ve#|SM$@tvz)1=SKdaKrfs}&b)LsWTnJtr7m6&tUy)sI
zJ6nEzLsP^kD9FC7B(%H~N>!8WZg<0c#k3o2Falog>&K@Oy3nIMeD|{UwCx%?H*eHV
zBmT4d+I$YL*+fGZhkK?ZLj#9L_~iFD3(fO81MqMf*KCFcFtEQQGQA!bJ6?bOS~LA<
ziA~VwRd;=Sz4nJd$ZX4eMYPtdyC}9=Vs1FBeBPZXhKP7FTSN#icJ69Ac#}Cq95dRC
z?6BjicxqhWs8OpbE|m&%@jgl26@$m+aiUS`Yr?R<(3Tsg8I5RjKD){Mf)k&dqFR8O
zmetC2D2K=C_|$*CH#?P2f8QHQ1o6S)%|MI1<c|KKP_l6H6KO9sHa<Q+F)>kJ)ZX4+
zP_Vb7+2hLP@Zmfqonfg;ughSt<@PXt0_eI|73!kDX8T-&SU(mkd|m3DTfJcGix+0?
z+N1r9GtT$leHz3pc!{+*;QZ+8CV<W5iEO&gbn5Qz4uGS0v(eMXxKM5buS%iQ0|f(u
z_4$SFIv1nkk$esz!NJZzIGD9or{{CK+igIwC%}SDCep6g0NcKdj7;DYp3iFy;`Vxr
z!wwg6B@I6Q7T}z5W(aZ~*(7=ond&r59#D{>&T;FnM`v119q)bcm<#~XR+^odHm7Oh
zP2Li_2RNV4=ePZ{+)R6`8_&}g?(d;|OegLg4Obtk+MKLsn^l-T9zghX?GlV-y13V@
zjz;ISe`;K}6~n>&LR~F!i^t8^#pSqpc3~T;vs@qYYID|dPnddKEtAu7Npqx@sOqXM
zbXoF79)hcqiHlzp<ODlC=$b3c74r|-I<Rt&*z`~{{58Ke#liM|93|t!O#nTb{(gh#
zeq>+4YB81$(ki1KX9T~H`KA|L338ZCxAj_hiSu?m@jg2Z?u5DYh)usO4=YTe{^B@A
zuuBSSc@~=WOwRjfGGPM*4M%Qol3F_A{o;N#HnKNOG%v}&^X-;hjUQRI(yNOOW2II^
zS;?)O0PCHjN}qwA5~Ap)n>}EL7ZyI34uO-3UGt3}T}O5%W4^@M0$(^Fw`(cCB+BGa
zI@8wNB)$m6cql)2DyLL0uajqbwJ_ugDDyj-Ye0h)D+R?dV9S2JsMhc6?<C00%>{vg
z-)r(K^gRF@Qk6G1P%<?#D#+Q!j%ElpLnI(26<rP#1%Yha9AEDAYU{4z@9PcZfftvV
zH!kQ{uaq^9w)kGW#GC$Mz{ICxe^*@P?ZoA{cR~Zuyt%-f!x*1NHG4LeHepNWM_kc`
zQrEAyw<fUoS5}|8iz~Qn+9?`OpM+7er(F%EA-IsAtXgO{VJw1WgHj%<ks26WTVtr{
zK5JK+O8Xl}%z{I3ZPJg#?<{EPjL`FznzU>I3B(O*ig9!)X;U9on+fmdJ4;g;9S8g$
z&s<HH%g-uo3)y%R%`v$Z<7-pWZ2JeWp(q|}?yzO28<MgFK>~5i740EQ#hgkrEJ^bz
z#E`~S@750i<pSj&%eAM`*@K8PbTogsgls5BnSS$qUAlkEj`F_N)CnXINz~)-B{Oet
zG~JIe{+9K!Q;Y+M>gYQ>A8VJ<2bk8DIy??dujwjSP1c(C`W<cgrf;sq5KXrX?+&)i
zUwV4U;!$S^F5@)@ZjErcquiYYhTcW-yaH7#g^N66kx9+pnQ}y6VxkdV?sS;lzPy;g
zo1R$Ct*^#GX{zG?0f)m>r&j{xpg&%3<xXG2hqjXGGhAIEwry(&*rDSvm1}Kx`a_Y=
z7h_VRV~-`0$1vwP<FjbE3MueUSFia(gB*CXXKQO<S)x+7P5u<}MGw_p{)ypK>hnV;
zi?X^1jCriKzFne!Ua6N;Zc|lY99Uu1(F}F~-{p?`j#>|T%t-g%Hb;M!NJshrT)HuT
zxWBpRX_Al%Lrq8|iOFdnraUY`-b>m)-EhLgaXt5KmH#%+Y-TXt+ncA2x#BVxAG5(`
z)A=#X5K1HSixB1Idv1NTCJ#}k7av|+QrgxHq@W0t6+@bG3<_LDWj)al-9Ub&VK(H0
zDt;3gCyU<A(u9`zie#bojK%Cw72G!AHywNpO+mXRpX1?+`InCDA39Tdq77$&!a294
zSgBH?h^*1q+oHU6O`O#!^@b{&xX#?(7r5`=UxQaBYooxehEkrqxi+K<DNAWDc-;-5
zUredWw6NaV5$}(qI-h|v*G)T#Qjac}%^qCL_VDjEK<`eS6P!Wd@Qfr*h$*sGW(9Y9
z6})}j_P;>NO$Sz=e~+6T`mreU$gg-@$i|Qtwy2z<$4{{yL7jeFMff{RMeqE9XO8uP
z*Gq5CpE$x=F)V$^S5yR3Skla)xJV%1k4;6nAEAXf%$UdG%fdQke`Yp{tSLPu6tR&g
z^f?kSvpk|m`~~r5!5rae1^MI6tIMDuY`lWn5e7eFA}0d21S9B(q_II(Xp1p5<(`%p
zH58tPL@GJ&FH;mn^YA{tE;nx)?MjJE)UypegRP>LxmGBgzJO``L~)gSq$ujQBT92d
zfj+_Qi(p2D@TTHP=_oC3)04?6D3MGPo2istvLWSQzm>_+ICHB<!!myohbcbt2U1K{
z@QFXz_vcZfyn(Gwf?d`kdNjf<E)y4N?Xz(NPS~0qUneqy@-Xu_1xo%PqfubMg8TUR
zl-Ri1k&ed@UQGbbw^Wa-1Vk;`fP?I0v<i^rM!Uw<3AHza3GMukhDb~nch4}b3eGHC
zZ>)#xOtxx`YwAK5myYMrwg%JlEMf83RU_iC723-KH=oo}x0o*hh`?Yq!fagB&Y*<8
z!*>`#J1x6LfYPm{Iiv3@=}2i+yM>COBg4>ieahUI9HmO9p{i3=T1+Ei`HNDCwUT;6
zgaw_!a$9)8wBskgbDkdVT@Z{(i(Fe{yOqH~ej4b*J2N#Aaf}kB66)yBNl<xv<#vdc
ztBCKJ?24QSM8+~>RN(RJ>j#ytjq%ZWcEQhLwA8|WQO!<^LwUb$eY+UMDt_0vy|4FH
zIy`^1UhMvkq~?SEL9u*ML&YnKO6M*=YfEi0LXF`ke*sI20LEr*Cj>+L>L1h<k%bpM
zsUryNZFYlobccI4jvg>ia*;-?CZx+kce+}P^<sxxJDV?>&%n;UJT?}-$z!(@0Ja7w
z^C_F9)1yp;_J`X7X0_R9_t2QH&|)JeCnqEnY_1;4YX(EGS{uLhkvg#9O#_sth_}BT
z*6u^tTQtDk-bR_NvptTdIr*%QP>)AP>(-ge`S~apprG1ytPl62NiiT>P?`@VSn;Qr
z!_46N<yIg$pTC~>S42eA*R@3t#w8?h4G;-OD;L9SON~f(c|B+Q$PG!MAT{1;Gbzh8
zmQ;klJDtv~1)}I6x*SaLMHQhamo>&OV9ewe4wZ2@e6Yu)6i-G*KYR4l@2bsb%N&Ik
z*r)38-?iKpSN@)oh~8{`OMx%ZvJrllNp}N%fB4fF@4_S=PfO~4IjOW1rVbqvMSdDR
z?lr(zH5i#S{JZ7SN6g^HQChaQFw+Bpj0tRw7jMn7|4%Sd{k^8k@$X`g*5m}Of<7m_
zK_7*+SYq?|D^NPrtgAB(MmW(TXxer=klO)d@ysCBN+GQe;Y`k4yL^t=rgyRVIvDp;
zWTR0AEf%OrF#ckp%SbUstT~3$`xiix25?fwX14`${s0sfc>Q^}KUsjW#)f};y`B0-
zr~3<qhisbcqaJKXsv8i#1vUUl48MM9Nl8qWA1LzF!WN6aFarFcJ+(*E3vRJ&eDc^l
zZwFxx!5`v&cR0#j&be%q@#^zRDmCGmAo#qDRe@)(t_pg+_%7wF#23E^l^5mEGx;y3
zhLcI1z|3oaH#|#VHfU&YkPIZT?C%wr&xhDw-t|H#y?g7w9BuosRK(joEGPP2yh(Sf
zzaxu9wm)qoPxa$joY9!p#i47IB<X+6mzx}ANP7a0DcCj5?ohrPST_~J#h#};+$*L2
z&@cbv3|)4%_Obk3liR!hx>M*io(`|YQS9KuXcveiPuVKlqxdi>G(O9tQ71)>H_y`{
z%>E9OxPona0>R-x28-iG^3r{;FkH<`Ph-BtD};fJjH4<bRAYH;UJy!b6uz&RmhWRw
zOi7n>v80pXd3_?MnDyc4dc7IJfHd0cbboE)Eu>k?E`Q^r(j(=|k{2MEtL&<3#<ZC_
z#GPsy7&JYfut%AK%Grh3a2aeMn{wD!@HEInw%aZ=^DTa+#B}<Z%52Y~gx>E~1%naO
z>IE!R6}LJI)g6!>uIhX#`#ui(0(^dd+4TVc9Cb6!M-!<wJuZ2?J8qVAJ*=K@_63vI
zHhn&LjD`_6Y>ZcGjbj4o2e$y1yS^YeWcAxQIX*D^$gnW?<HF<ss8TGJA9r_ia&ycb
zKNcvx?ZCSeD3s7>H1C>GIg7S#ww2fz8KvvBGD{wGFrfkrT3xP}TT+ZN`#Zc(Y2}ZF
z<a^G`P2)osaj-l_#S13dy$=Yrimm48=~(%$mlyg6tvFvCI_iT0j_O7lFr6Xy-PL7E
z?Pg|{X3bXpB~xw}FC%&N9)6ES=5}vzTwfOcIaZbz4tK{b9Z3V9NPk9;N6?jow%MRv
zuM@sLEPi!3lXlOQrG2!Tu5`-(UNw5T#fbX*>2PmioI1WOIh(+3vDfCdJ$*u_rAW<}
znC77OdaOykR4%<SvTheCablJAVQ)1*tshHlN_S*w#`@h!v)&wIM`9hq?F4nD!JLNa
z`Y@BjW2IOi9TJbEsaztO*PW3-O|I=(nAPTYNhO;nu3OEatX_L&KG|>o6TsbZ6hcls
zSV5vhpiYnvaqZYCqswuj8fR<WcoOS&iy**$9OqtmM3awmz8336|Lf?8kX%=oBb3>V
z=}~&Bar5jVCS*LB9NOS`s6vyqjy}3CNaI?o(`m?8qL`J_{-5V@@$s8(hdA=)g`_-j
z(&FMFk&%Gys!BA!#?8kTlAWEN9??eu)tA=zV*@6wo0NyBWLJtey_)T@1&r#@<+iGE
zDIK%JX`A6*3t10VX(x&qcH9{u_8j*Jq<p7gY>ipWgEddaVkVeLK0CI=%7(Z~WnIVR
zn*3z_n_F}KKpo<CV5@VjH=>4y#%tr&&Q#Ta;dbE-AcFWGAUpfLyOn|L?I)(DBGzN_
z;W6{^n0!5?Q-=){ydK1|2{Bdk8wAbRD3nSDWY!w2))see3KWl*YYYYU^vlZWnK_!G
zD^hrj&zR`iIZtghPiHPCk9PDjtJtdF=5C831_4|Bb#;$9mu|U+1sAg6&d}QR>ZBPn
z{Q+X<>0~#fiup)A&@g)gW?_L{dHY7B5JXh|M0$)E(90uTy(Ow9J5CG5z)Ym*?(tl)
zgMG6=BcT4ho3l3;aJ$Unbb=<ACyi&ZScE0UH|qrWQ5K*3(KJDa?v&I9r_%|u&4xC2
zJuN9|kjob6fj_nNM*B!G0y@2KUWvouWMM(U6+w4tCYOt9sd9B@LIP+otzPFiT@n^H
z_Bud$fNIsy;7UVfAW|I#QFr^!!LhEau4GU?1X$Q#=P`y-MPO^8G^v1`r;~e(i=EwL
zme)h-$vrr9xV$~(iyIB}uYs0I*Tg#24At@uKrKTF!xpie+|<kcDa^yj*_q{XpkCHD
zI_+lsafYBzq+zA9s9Gm*sfp@-<k#x`IkwP#5T<Yr*qnm3b&GQFc!wEa#T)hJ$c&Gz
z9e1-*;b=5*WBd5;gVA_XN;r&Dzt6U}{l?fnvxt%YM6HY~dedUut=g2OmEdgCrjs)M
zgo%<62)NYJ_N{=7FyQ<Ti^tn;w>t>P4<qG8VlW-(kC*@a<l~xrd2*v(g;gwBE&2YO
z5-VRc(*3leDL`l1;m_mcH*7^9o31xjdO});&%8Rz*C^^#?ok|__mpW~2|xym4S)au
zB9)z;onW(KDK>*ae++>T{|VIC<UgPdb%J|o4$=q8bx_B(^G`WaY#B-EU+&rCqe7OJ
zmWqmsCMHFq`B3oi54X4FdR?8aaQL720W2=dxKtp7oXvyD*E-^nlj@zRK7nS8X2egJ
zhg-IUUhTR!YSa6*1pX6q*{w0=GVRBvJS|l0^(1`-{e0zy{s;KlHE-%-R_ZGxXb)LD
zqR7_%zYr8l(0N$Pgyi-$5uzn0A@t2(A|UxD0DVxW?s#+k<sbsH&-p<#S?h$OR{doA
zpT>wGnKg~^!oPC!Qy^NYbPcbc@D%LAc~Ja|zm@b@9u|-&MGV<wNq-WF7(z$Qk^+mS
z9r^^IN@7<h184UC2hOTvYt=?{cm?0I8M5G7w+a4KheF?SRB%J@?xA80x(+R_-#;KA
zpmlPvrgm0_yA@lD@$dR@ksIrP^o>vYlm3|sbIO?%kd|yRx_SFAVg_kTgn0IB6o)dB
zg1*K%)t+Wi+6V(Df=T0@_NmNhvYbXT$QZ5=;&2~xOmC=VRvVNLN#YQMTr*-ny)yPM
zuh+o=kkteMw^-{M#t1!?%0FZ=dP`@8pM*SrcrLzH4&X`Dq6{E`evH-N@C{D77i7v<
znR=)m1qk;WWPUE^S{(#3`h#e~@e6GKE#j2bm+Vk;DAoYP_hd49zu5qtaI;s1j?Ozo
z8es4p)~#eYY<3uLjifbSy=&AYZF=Agh~TYFMeDk{ooJGt-Rpk<L|H{5pkdJ*LZ+4G
zEYj>K5!&7=C-w~BfT7weF*Rno8o45%-6I8TkiUo^cM?#baGY>WH4#U#2ycJ8LOYD+
z@FzyP)-Dp8L#_&W*c7P8E-9tHYI#dtlTb@i95N9nj!j5xJej#qltLHE^`leVI(e&G
zixCQd$wlBD?q`;QR$I}dZ;>1B5u!&{TmAfadU9tsV(t}HG$8!?Rlk>t_!IWzt~4@5
zljx<>N0SS94&D+9*`c??r{L5Ki?g-Ewqt$@Mk*xQKvVSXV%%=5EM}f=-xgofxLdOm
z)<H6uar&lTB(IJCX~(!~h$M;aA!3#vt81t>%WKZ0KaHbQ+mgBtt`zA|J$53prNH(w
zUA?$A$dC(gFO!aUcl$KcmHx32vEI57A9(Sab=x6|<n$1K_Aiw5U;LJw0Y<J5`ZGNp
zix%S35I!^x9I&l%j-;8%;_Yt?BpWiFxIOa6FsP)8lqm-;aX+2R-_=)_CV9%~p#P>J
zbE|KsJMOsGz4~)B;MF<7_{r|CY8t=%@O7_?g%9=PKHpmHE&=7XJd9Rs-Qke?b&DNA
zJ46TmNhu$^$DTG3dWHB}oARIGKgIM%95I&TLjQUIg)2&uG+D8Lenp<X)jv*QSf6P~
zmpCi+Rb&@OrMmGS>R)Yz)e5{A`z-K3<^6r0zOl}9#Id?FGq929Dq;FBX#`)f8Jojp
zggFDp7FW=pIp^a3z+IG|z6qOrP5yF@Jzi)a2k`0mk)3;lPjS<iF;I&oYxNn>CK|X?
zamkT})BpY($&_sxk*D$$FU&wU;>@=Td#9Pb6aG#?yE`+1eW1P#Rm~ws>X&T)7fN>p
zQa0|&B)MLF{3kMWmrRm37_BNf{gf;6|8ERXb^$=+a@$b4l*Ytly5PWnrDIa_|AZk)
zfrV&Vx&;SfzCrk^|HKH8hD<~C@j}H1L^QX*Jg@|u1L6V>K*awF1N{?w;amw0UgG~$
z1>hB8?$(Aew=4&HM~?p>@rfZnnw(oEQv3d{o4$-`G#eFA^z_w&%Nxt$ZxD=-?8&b-
zT0O1VRdZ;#QrvF9!L|QKAw&(Ck%q>F>8_w(8jTD-#ji|=dyfejWkfo{g#B+5BW|W#
z5sP$VK{`OU4LE<R5h8(_pv!r(rQ--OkPXuV%#kos(%k1zX!0qhC%`i|oLY;B87DI=
z7>{6C(r%$N3HO;f^XD+G9|3_-LFK!~L+&V7)VX+Ey@V?(07b7iic?Gl#YHll$BOv1
zJ0ucA9G~q@OCTiZu~3djZL0CDISgN%mF`$t5RKy29OWyqu?$PX)z*RqBl1Oo5r(oK
z-09PfQ~Iagf9>l8CP~ktrG^fzsid144L2h@yjgoTCKVprXMH1xrC(~2_5RZ^l3giD
zN|Im^twuPb_qy2+Rn^6_yTrV@AY9fHA1_Xv8ocDidGUM**g27jFM<Hp;Hs^#{UY=0
zyiBAEM<A7v*8ApozKKlc!^#v^MryQ!B#c2k9D4r5b9$ok8K?%-puxPg-y`UbxiKgi
zNV8%QZLeUjmnT=!Et7g|RJfb!4Ipmi@Yc#aGT%l+$)<8$*cc?vi_1$n-PWY+3ZCVH
zXC<NZ;JJV({qtd?|3lPU2DH(&(b~nG;_mM5?h@SH-QA%SD}@%<;O_2D@gl_?g1Z)X
zIO+4A^L;-v*?Z<5$)4GBt+nF8M@~y$9$sMEY2Q)BSfz#Lz$PcCsNZcxMgJL|i=fJ5
zxAHNXZ{FR8@B0+3T(X7FzB&JQi<=6$#KdWg_V7H00W^p_JO80*?0|<0Ejq0C_xHI{
zl}MCDWScX*&ud6eW+{?r!$WTn(Bi3Bco)SPh8^TA=sHv2-L4z(?negl1YC+uu*<}5
z4Q6XBSb6e`D<J#uo+*;auZ|~exNSezIp}RWb)j3(4NNyBo5sitLu-8xZ`@44fmZh5
z5yEjZ`a(lVt1GWBL7B%MT4XnC`Srzyep#Y!6vSu7fiAM`^ww{-mNy54+a55h@s~It
zK*b2Lk4}{%7cO*yQJlt-cIBp`puafzNt0S+i_k|fmL(?EpUfX$DH>9|?LU$512dnW
zlR+~&3qO%P!}Lc{F&78k&z<PR9V*s!f{}47YH|cjiz-Vq)P1-UpD3w<8QaNR(crUX
z_F{E>EUaW85TAskX6ZC!;*1^tA)Oxb0*M7-o`1&}$AK1i%+RzF3IUC3NLa2-J(1hm
z2KV>65Ecd<Y!we9Cx5FPwyr|STpfc7_IXtYHGTBUL>^V0zisynpqOZby8mZnh?>0P
z$W7L6yJns{*HURogOqKBzMm|B{#-PmVD1LNPqn&3aj`g<)uc0isiZYpY&el2G<6gi
z-)AaEB$$$0rQ#OVZk30l^D8s^s`&ZndR(`ssCZ%7oRA`c$J20jvgbGLMO2sR3L_*^
zjXu!aAq%K8x6(=|lrpo`dF_`@!JJ22s9dMljy8unG?$&xW(>8bTPq7y$x#qNOT`~U
zAs>Dg?jL4W<OwL5t=|!oDTyt#IvX=ObzEo;>X`MV_dv1j?yMr*I;X~!%5yEm`p}&S
z7YR*@1>=D4?cO)qj)@uE@<LFjvRwAFeEl)g_+pD3(yc7RMSBjC2F=OTCY`>NJd7qP
zPvKmSvjx&`p9%)`uyVeNFkf5`oj$0md!QeaqM<KTXoZor0zCVZ0mWc}N@w8HHKvM?
z8<=RPr{kHJn9%VHWIKaZ`Cx^Lpw|gjI9PD<OTQb!PIb6{%|@BH+g;Wo9-7yAu05!c
zEP^>inM_7w@Y9gOvHjok^q<lMn9M1^u@l!lG)7chuC7=$1ixA4eB2Zs->@F~_VB-S
z58R|Wl(4KlT7FeN42R-e&(7!+Ebq+RyT}K+1QT9u`(XNWNd<Z!g713`P1)6{)<AUl
zu9n@G^G9@eEyqzRI)qv0j~h$Cw^f(hR-}%NV6SBZg;+y;&OBKKlUduE69j$h4%}}4
z`#~4f66~@&`$b*4m@`a?{fdOK+`S>S0~1CO<2H)E!nLGo&LyW#iloux91_H6qU)LU
z3+cl{MT0;s1|7M5X8<+7<@Eu+OOrPxUR2T6dPeJp0c~8M_j!#=mmF5F+wpuZkBw9h
zTiF$eRGU<r&az2jza;)Xohzu%ps))e2w0JssJgl~j`QXEv~TZF_TlII{(~S>;*2{m
zQTbB9`tDF8=ZEJUp>CV+?!udBaIfu^$zk(%b2);X9>AjMnayR-%QYH|P|%<52oI6R
zQ%Ch)+(=xf7dM?mP4Z&UdWxm#B@pSLXLiM)mKWU}cyO}4k5n4$vGq;H=@T)9##mry
z@BK0DW%u%6cY`MJQM=W6DlxzN;M^RhH`0HKDJ<-E!yd34Vt+g>tmWq6!DY3{PFGhK
z2-HCU0Pt8Gfe%9}0a{8OuyW0C@A@Aa9}hfT-2J)Jz^~^&e&7`@WbARce?+I0X2!l&
z$(BLfKaq%sCd%94Q1J+v>odSPSxpb{P;<T<XF$O5OESoh18fs#PPZV&t<*z0JO#pl
zQ3-}Dm>?pD#S3WH21N#XRL*`6YCDFTwBl2&!cBam&%mSaKD5E~aepHpX=BefF}PkQ
zE=lLpEL&fuW4>#K?9uSe`N=OHurtONDE<an&Y$SD|9lHrxjrS933?3A+-My_%ALHA
zhUdaZab0mo(DG*qqbtYS^P@i{xP_hE8>QAI8siWUCQ%6uF_T?evF%LNayNI<A!Y(n
zDHRqK27}`M6oPWUlvps9d-@9YW(KsF1O&+?2lGkz*LYhr@-P7({#+;mu=&jUm#zCQ
zq;b7#K*9CD#RA}uAIH{%gq6M-d{}ztHo8{9CHUCy-gA!P2T%F?P#<sX6n{=@Dz8@Q
z>M&o+U+>jnshV>in4fe_l{%<&e@J9Z35=0`8rSM^rbiI;DQbOo+-_~Qlp2cwWyYBZ
zH~1efp>(nmVNIM!$Nk)|F|N@p$9e{3dUKOL^a&=sJIYLaUPH~>J>ATa^fKlb{%%dD
z@toMsiAx-<#eMp4;qexzDb>H>6)ZjS+TfYmFpzX9>qilt+F8DCcYS+lK!5^lX#$E?
z%Rjv=5$kAt=ykZtm$o!g7SS*~^cS$w29stVf=s9P4h8*y*D8+7foFeNHSje`O|0iV
zD#O9Ie!0bJ1E1$xJ6>L1Q;ueRVgiEay(ATelrc`5$*%`c`*2NZNl7==_*Mo?Tw}cf
zzL(m^bGdIK`_%CA0&Sl2O_wbw1VZ)s6Upc=V41@(!MXr4(Mn^;Wp?l4+wDz7wHi$G
zHEb7KTXj}r9j08SpY@Vm97z}j2qD?yvbFH)yrVhp)`PRi@XcbaCbRw88!Gb71>Nz}
zfn~GJ`|9J=en1E@Y|PP|y4mmEtIG&T^^&Mg*G}iOwADzr2MI!#bJ1j6-5;?ly)QXc
zO0^*>Er`Xp{`%1uBrgGTGe{Kz%#m?;SUcLjdzqKPk3lwO8J$z<Awl|`jeh?4g)yR6
zOC`b1E%8`y%@6aH!W1!%I11=Y<S865Gix7@hyLFTb}v{_&-RoE;1N<hpMI4H+u+B=
zi>Soel92V*@28RKRD-;XNwat#+e!MQrTt+n3J{_xq6<Er9UdY&^Q)hK;;oOya2<?{
zthZ(Eo&p1Qmi@f4StujDuPBr_5K`RiJ{I3>Zm9aYWl&u(3#J{EnC{W~%lLh<cF&0%
z1oNp`svb4ETudwG6XgMNbNfagCC1&gNC9sVs?_Q{@TAKl=d;6hr8N&udQ*h@16;+O
zO#vS%jQYI$c|O(?K(61IlK~|zlKbKKnJj71%$Pn79IZ4OIFmubV{ni%;q<aokH<0{
zK5J#@9*0VoN77_1swp4+Sx)Q`X{aEJ4k>jxhnY;sC^ni1mtFdsw4dkb{SkZ3oT$0C
z3K#)*{z3$Xa9ANR_s|Lvy??J}r%Y-K!+ol%FDrAq9DNR&gA|mz>hl8F#+16m7b}SK
z&)zP6(ucK)NX&C`IZVi|?W7xhtY@rN8C?aC+1BEUJh`ndqhYtMt&Bg8y>2F4T{Cmr
z=aI2~G?=Xw8|H5DXyAMiE@>8h_;m7u@?xn5INRQ=IZ!#>dfRe=922*+wDk4$4Giq+
zGcD67odjQu$c}K4<`x!*6F`sGM>kWcX>VrzraGq0zoZa@mwu;Xf99f2P@7Z8LB`K+
zY7TZ}G3qM8HwzOn58Z3ed$mt52d~n(QhUQR6fJJv0CPZUDj9YbF1*=1P3?!XUMUY0
zq5q8IY{CMPx12#$Sz-6xdH7VUKMXJBtTm~yr^kUxl7U=YE%1HP#f_QZ`EvPX=&J2_
zd=p+c_;IN=SeOKFYxC~_t?@wzcD`?>vXfEofw`j8q4rt#sl$?CR;%Oi-;0oX(cssa
z%Sg*Vt9B|Zy6zMnmlF`c(N+&CO;doOb-s4NAN?*{nN0r5Z`L$cmc_Jq)BA+<-xd>d
zd-P483&y8Ho|#1_{VwA=!`8l0ITDf*itbnXjFbgGIj^czdJ+r#PHIsWPHMTVylS4k
zpu%+2^fs5sN&Vf~Wp?ojn;V3SO(pog-*#Ne(Vr<zV)Xp<JWl#9T<!B^HpOpO#OB${
zu4JA>*K2ERh&4EUf%0yvWJ>g-r%u`<_c7bWsN;I*7WgJX7l!`uWM-P|3fs`T825)8
ztz!Jh9T19mLDJ!SZ%w}9HZMjdC<SwCJ6mHDuh?y-<LFIJ`v$M!=xI279vJZ&g`0D}
zJBsHiK+IUV&}!o2;#02M-T<F543T{j!0BHTIaH}&^YH0XqFjB`g-w+>%Zg0%Er3hw
zH!_T)x_*AQH-dV*tk*eCC>eS-iC@dv+5`xccdrZx<&-l}!m%BVukM^z%h-Y@ro4RY
zcgTi<;)*CSn5kbcOkK!*zuQsjt$xkZDMMGE=b+0otINbiQK#(A<3f+Vzno^48WJNw
zD>kboxge<sWCo1QlO?w>(~uh})G2|-)B}xVU{Yq~jqPTAajQ)aL3UThcSF1|bC|-J
zWw5*xpVRtlI+WQ`i<3c~039t@4NB*_TxTXNElm((4R-(i<0=cIut~BY8(PK7cw%9@
z+wTceKMmRry+M;k6hbpsTBip?yt=P%vzIu3{HPyiT?A3N3Ap!<Va-A)jo*$^NEpKU
zbgz-`IQY99IIm6usr;!U5EGr9qqunj0xp)ub$Zt(MTI#9AI7eZMNI<Cug@-11A8qM
zmm{hf)$i1k*ykEww)7m9CjA;n49uH&<MsUx4>z-0{+14T<WvXvO%)T$`S={94qUUF
z)IM5YRq&HafnJZ(=AuBQ%EuSP%g5RDtlrgeT>jN?h1AsOP{ssvwOXzZ`MKS^gPYmf
zoW`*2kVebxS0gvIK|(fH{VoTA0`>z3Ai~RioOVWi6~j8qzShma=h5$#f^miuP9hiC
zr6-FuFYAxxlxwpD&6!e%%K7~PvORps%U-K0o2;)FX`jE9`}_f+rwugtTb#>4jRf7T
ztnZ7)eeU*mmxT?vFcGOYo;g`yJqYeJKc2gQ@!{a2A`y)WG0Q&>-@rvmu<<xxyMhmA
z2v3#)rcD|}iww~j2!ZAevU3f@62EMtZz~bX$dVcc@w6{&!KywL+x}ef=czZ&ltXav
zX4=O}MHodBC+`of!YiEoJ;onNRD^ge*O{Gmdb&`H(oE`;y#0#84|H0h&w9w=Wz+wb
z|MXL_DT_=mTRaJQzNqN|vPs~Zi!k2ZPubnRJFT_dMk@VAsHpirvgPexPPTmJcrJSh
zWD}un(mC+r*OeG3RF>6`QHEr!+xTk25X-$v!W}8iqY;(F3Nd!prz8bfcsnCqAM#iy
zcu$0tMxRpd1Rm?*B*Y~sbg)#tUTbP=S(WC<-kI=Brs8|5VOb33c2<iSBR^I;{1)fu
z0!-emMwhE3EK~vO&r_Q(_%nJBm{xc09Q45C0^}ko@}y&wVtpWvJFOgDv+GC2*0-r|
z`|l!$K^jv%VYx3`Nw8*5rL}lBh&x|itgo&0as*_=Up6Ne*@nNOI(0OEMv?OGuGwq0
z$Jfw2jJ@l|=GrI3pqolTqobvVqL=Y<styl`vd7fg<P;H%D|zMbL}D)f<0OK(${F9O
zDFZeVX1Hqr;$!pVB*Y1kb`p09nzEKz)M-+<@cEwXajO;SOa1R1-5>vs`9;ogUJ%Xs
z`lO=E6_CjRROT?8^#vi_Y%n{LeXE1d-_K7K=&=^-6^u5O9z3|Kk!5;+e~)Dj1JPj-
zogE)o&ys7_+Eua};7&cdt`2=sx%a=kc@A-Y&Jb(`6LE~s0gle}Yd{8(7p)XM4g}k=
z!|XPqZbq%+L!6g`m)hzAYE@o54-Za{>vgS$?o8B!&$dUh<n&Q=iO3W4Y*|SneFmiC
znTJR@Z3!|ov*B2V3ySd8vf?Ofk$lHHh`AK;^WxT3%T=^meu)^8=$Y=qaCg^Bh6a51
z*(tcY%{hI{C%o73Ggy_~!KcnOA<YfnqczwOM#&Q)oWoGGK>ph~?kz?4P#7$hno6f-
z45i33w_rswE|c!3e&u{susM-lK%fK6-(Bgbsja}X%)BQs6pIsmhbNbeL~V9dvntBs
z^VzID#jFkpZ@QTI80x7E71@{*em}Oxu)K{W;IqzKS$>X|J!d0rsrzNR^UV&ez`!ye
zBD4YJ;OaYv^O#CfpV*`9PotjZr6DN7`?Jh$Z?#H2!i}p??Vlz|5L0ngtV>6>?SO^B
zq_&|9xxy#nG-4C^(`GcfU712OD)Eza38?(!5CCTqzY_c3402pzUdV4POrK~lvDo_1
zBO~q;hWGMQ>DuZs|MC;Ycc>OH>D~>=o;u5os01|HY%~*&cpoVlO3GM3=Chy^8hgSv
zN4ck6!+hT>>E3lb_*MBcVS%zp?6nLfJLJmgV_wHWAWOK#;-+T*MTczw=i}L0Aw@Y3
z;WLz^<)|kpLsvRgQC)%^q29;)vRFPYZ`wKx66&q1yIWZJZGf=F5r-y?A}%xdl~oN1
zeL=?g`h>_OgU~`-JzstgBe9AF^x#;|DJ)KDWL_e#LQnqz`wmb2;j++|o=Nji7eYFr
z*Z#f))`(ZH)ZLf+_sNy|a5%-MQI+XpRubMneKQGuOL@r!GxOfJcDJXAw9LG*bWxZ+
zk?A$CoZ8HQCJEFX05+1eGwRf=gRK-3K+ey^0?;A9zJmia0`AZ1?OWx_CIlHb{jFZF
zyQA_{6;B2yuvBC&i<A9c5+iyD;+}~EZci}wcJsO0<k5(wIG&rDgorV$9Vo>=gQkRB
zIZ(;pj*kowfIlUoG2TIsx44xJ9lh7*598k!uMAxuTh{-8IO$|}Nf+}y;$qdB8=5yr
z+-+bRGQX-=La1#&9k0FfRmjjiy!OtFc%gK?VRK05^{`P2XV%*`Ci4TikBW1uDFd%n
zoJ8GFn(nK+MSC8Os|?Eh-2BQ16<m7f3yVpUijvg0mdex@Wni7`JHG8>?qx7<p#KSU
zF<fss<SD{=nCff8i<2WwiBN;o%%d&e=rn)-(Q9NYV0}_?SrTA#5a@FuQlWkG0)8Yh
zFcaPz#j-ur%5J8rpm36vi+{g3_sCQ(!fo=mUT+!&QDDOoi$3btunjYVfAiH#b3U~q
zPm2?@lu(x98;Tp*l5pxKLDcF)M@rM~v4S4GbUF&pn!?-S*{K?XVjdfVNAZfGiT6(3
zz--xZj=;ce@5y02PWPg*MU|izb6$KK^%B+Lc0m{*vb+HIE`F8-j8C2>oM*=wRPRP$
zxfNx(nu}$7!VUE7ksduQ4dXV;Pnq2xtUbP9F9cXFjyw3nkfqoE?gZ=z$T#e3k)oJ6
zy;F_N7w7Q1w#3+rvV{(ncPS!l$+;c~uf<^^!hP#iWj)v^PL{7#K0Cd{`P=`9&%5!y
z6{E8nfu`fK|B%yl6jcV+^GkhcK5MkgNb7xcPXs^Od@I)U${}8S?QVDJY3M~SR07P_
zAxl2<en%YLxW1p8dMF<!SzLOzwRPyF$>B6vdF$(1`sU=YpXx4BKa(5+>NrLGf!Lh7
zAvqi>6w|W2uCG}XLie-iqDh`r+30b@VM&mB{d+<y;e-AI4WhJoE8TPPsD(j2=jQRb
zaeP`OcYv7ve8h@9v4T5m+URJ6(R}%4_>~O}^ZVa-?~hl(v;pHBom%H^mOq3<y6jH#
z322xdxyAVn8XV||7gtgs{>`9A>I{w>eiN@A@oI}?FM;5zS4<P-V6(FtoDZ3ihrp}G
z_q(ngXS?WYB$T1eJa*bvF$Ma<Rj#o&%N0hQ`|$iK&T9Wje~yVkyuV_?$d02-1Kuog
zVr3XkaA{@?6K(#kcPJTWpiZwP=ew{mqR&JNog3Q*J~t{Kmw_`$)tx?7o%}nm`;3g%
zfqQpDx1|*87Cfw(ohP-5l*EQ82QF0bGL25#4=BAvo79QQe4H+e4>QO7&9A4Z$8+$p
ziwKFnbV~AmrP{=u?-H8T^PIsodX3k%<|K)JCwr5s^zqSEA9En>tj^?M@6)v00Ogmz
zg$EmSyzkek)Z?sZN+Q{!<ZT=mkjs;EbKsfKl~M+m-DXdaz5NMm?`tCPDfoU8sxG<!
z;g^R`ZRx`XW@0=V83J|ww+)!&SWP=f^&&ZJH0%@#NCrgFyzk%1)S<<t5;g?GGoWV!
zuAdizFS|W4qPZmRDTmVnbbKDHy{&C*8Wa9RDt3Tfc&*N$=iK0Tu-m*+t5Q4SB9p8R
z*$R8Z?wa8;_0Wa-5bBtXkBoz1P&rSp)IUu0k>zYY1zC70Al)V#-sPK_5S$VS&pk#<
zVC+XjHJO(U=%o9hH%b^-rL$1JD_MzBgUUddncmEpn#<=={57pht<u@i$p`26@4nn1
zY<~)oYBVPQ@Wt8@w&<Ed+GZ`jYbFJ(;*?$dZzm#el40Fx(84MGyD|Kgw>Ts_UYmh0
zjdlJumzEwb$WGH#s{^5?wp!!-xgYDtu}Qszv~JI~qxPqt2)~i9A!iswHZ~pVDjy44
z+#QDY!VKs&k+8iS=FRbPaP!n)63EEvL2X`Dw>zt)T@YfEG)rc=&)Ev{dZv6u_xMs3
z5L5TC<0M2|t6OvHy~`(ExPl}SC+N>_wALLg*SPFTKhm~z$@cO_LqgRdT^E;GSAgR(
z1aHjzeE!Pwxbr7K2DtFpfl?UA^oJ9k9Z#{mrE=EnvFz^LINB-ljKJw+U#WN9jOm0M
zDawm=6l9f}lPpx=!Mmm;8)t3!u{R|s>g8~yo%Zc23LL5f5)IaUWuPwqGp6bBOC3^@
zGGL6VD#2`Mnri?bFmfdJr@@v|N5CWs4Vqh$FhLF@daw!4L#`ofrSkXRsstI6n_-St
z`#s&JxDf1RhGt@Fs;oEvle+9hz^rO~dd|Ra$1LMq5tjK-h=MPjy0veS7%tg*+*W%$
zNWdPp&R@TNG*+X)%D2dP`b5#U;K73Y`LRPuV%RaN<ZE3UTA*w|J@*$&FO%Sx0P{de
z9*j_S``JWwE0Kpuo8I#a{oBWrQZ}gg5;lSuc6RoenHd}$9RKcju%rj<57cHzsi~;}
zzZ*wFM63V{F!=rVL%v}5&3q%`uD<>3K%xr=;*^Gnc;<xFbSD5@5MeJZCMzr63Oh6j
z<TSX<bD(Aq>gw9YkH&A8E!cmp`|i9Tf(RYE&&;ev^k?tzP~ogA!}}I{T+CEn6P-y%
z&Lih8^HBzp6^CfxMm_EqnoR9N`GPfTw3Ni?KHxCkbuw@62cdlF*Hr+$Khk4Y;3}9l
z1eNhUjuSKsbQ2xBmuI`I#X^)pB60`H`3gsb&vRzQIDEyV$KyKkon>vPNu&TzqcW(&
z2zXE+HJ%lfm6F`G_X%d^+iTLrB6dMpd*&*MhsiAU)$6|VsmNU}zTdMX5K+oqW;?|l
zNVn+Gex3x%<ZCYbLF?k{%>`kmZg^3u!o|-`XL*T98CG}vE|JpC?hs?vpsV{jpvkIo
zppfaJL(dun3+WHM!>8wWZq}FPiE}no*cw;zq~-f|^8<gi3^gd^syk!Q6QxXWZx@@z
ztWoD9qm+{aHjKghSC<IN7sEyw6~A)DC_=5t*e`v3h<a_pR;c<?y;c`FB7LDo-<#JR
zf1p9C@WH;o!L9`aEXyf9eW^ZwgM`{A5h)2DF+DywBQ;d8z%^IE)?<ZaT@DC<g=KtQ
z533*`$I!4JM8(_lwVAqt2AFT3TVV^EuNGanQAsvJ_Jw<y@a5*-9}k}eGGi?9G_R^;
zKRItjMR4+NUgdehgqkYQ8wm%7hFQ)y1M+ndkT@J%dc`8<>6g`LzLz8wUHy#<{_9ZZ
zxbl3OmA)93SP|;q!)?9%i%$?%-nYE5P{sB8@aDy_0Z@@fvKyjkPeMXsXLnbNA?3%9
zAK(P>)$jA_#Zmb)<lyX=OfI`buq$ar5o{nbY<J)93kwkac=ZqzWS5XGTxhZbS%ck+
zn}Ln5ze1o=Q+Ix885?ha^{!wolYrm#-rk<2rBrkfRx<(d{^8*R*esaRJvJOeh>3|g
z^NwB905;3bR@!not$x?h>+-#_Ft$31=Ot|e^oQy2`~H1^tG$l!O$n3ZP3s+N1LRme
z#v|&q2fQ4F>8y0tCfbM_!7@p*FDL1Fe#usjv{YRY0tO-wxwM>5fQz@$p3t2DVv~gZ
zB&3YNG6uN2k0@sw+m>Th6S9!amCr7qQn}+>T_?_zq;ml%Qd~V#m01(?mg(YAO_feb
zv1*N25=Mv8BO4TexViafFwv~~O$JZR;|NiT)?-tK-t->o_JF`UQvzB`2{ztRmGq8d
zvB2L2*}=D{*4zZuGh%wH<y61wi=Pc9FqIH9AT0^$s+ni9J%s4FspmLYxs`t_juwo8
zO6ikuuxOJRI`Z%Yf8#aDkVL6t2-hWJYA9qTRRS-w0$S`i6$onlrZ3Ef1>`eq=#y6m
zWhguTBu3>N@Yrbstq$X(suQjDuN$IzKAnH_uBk)MuCq3VK~hE5_KmYz)5=f)$8k=i
zX;Ts1Ofz*t8W_8Klq?`SGBYKt&<w}?5Z>V!X}pDU@7cj;Xt#Y5jQ}W9REX?SGXBbw
zY>kx40K_th)ks_;&G)gnhp*47$2KuJU>F;&cp%<}>4=Tph0L<2d=h6!DU!d!8~1!-
z?7k`8i&8|KVSf6hQAfkzSfv?A$jG=0$GbRDY09R%kTT~@ZFB}$$l=RgFZ%mVsmAcJ
zO8rV|z=#a`JaTO)VltnEM$&M|ske2&uAM$$%9fr+Ta<g*W;8Tm7DqH7IrbUf`v55P
z5!F0iY10fz@6&o);pDQ}uNJbz`mgn&v!ICOI%@PAv(?DC?Kd7m@1JC-+<Bi_&DLw4
zaRupX=1WTLWUTF|)70Cf*zgpUaDTA=N_vuN9RJ{SAkzeoU~uYoR>c^&3Wxk>ri0*;
znR>qB!ylk`Rt%p53N$Bny+2aGuK-0qDA?&?!}?6uE;g@@&^sH<{-}OgM+GlnOSgwW
zP&pc9a#%BC!$XF>QS{vq-~DXxr?~)VxBAC%Z95v=2v3o_@tj0CVr757w_k?j-<m3-
zS_0Z=8!gvkeSbj@sxg53hL!WIPM(ccQAh{vZ{-Vu7xW298ha`p?8a_y|KHDo#!|^+
zm}aq~Tx;IG|EpkOqNQ@If-y>+J?!wmB1%?QygwUsW*t6OICh%<iS+v41|d}96^f};
zXpEsC{uREUwWf7#<TqDs2@BbB{$K4Uwt_u~b13-Ud;pmLnkcSm?GI2@@_<~E5&tV1
zy_oS`sKhC<Q1f304Q8iKhTcj=;5I(|zxw?Sy4s$+<K*?$P~-kri9VU%EL6PHNXnOC
zod2rGfe0lN(z&VO0_>BwYw~Gpx<bY=^Z(mbO$Cyc=Ma7X48BLQRF<i+_YkSj|0<n|
zPvdviAqNy!{eK&di5FYAIcJy<Lvs?09pOa#ubOkZJ4<cpQ380Y8%vzv2w~uVHNkd$
zzdL`VMG$3Q^R|wNbo$?M#WWsl%U%H=xPC@Gs1UMxXYl2Or`TMy6D?|W-W9p0PTz7<
zM~nIIC`^6D-5zCncu2x?T~3Pn@044t2_Lk9!Wzfxog)8R(1%$SVww}8TJ)q#vgWN~
z)d(d=-_PYlnzC&X`{%lBq;I<I$JH1`T09wQ5@sBT3i>K?_a~Dx)`9`SZ~9Q*aIRlo
zZ%j=_Dc+>g?m{}pIy=+E1KT?<OLTjQRgk-^I-IB=#;$Vf-?C4Y*cb&NNfjQ<Yue3c
zUvXW31P_tH$B$CAax@9n9K0qKC@nR{c_de=&#n*bG_ysL;UZ;N=iZdT9q<Ff<^#bw
z<g*{`wLO*Ph6H?!<un(mwF@n^=hjaZi+4A%qg0*(*?uxcQGO%1VIrK!P($S$2?rMV
zsn)hP$j}*PeA<Du8i`4Q-Y1%%u@tH(4i>{uGKA1z@+dKCsy>ANRSkTXONCG@W?NfR
zUo?{0bI#^Z^M@MpzhsPQ``t8fKUOChN?=QwsR3xvGh6#6{go=AzY4T(nTle6QXnF)
z!v)i_3MWJA`Pa+W#rfzXK%{cGsw+EhvzHc^#)KB|=MT3a%+6E%B0BraWNfYRf<b~s
zY<^+x%M%ntFF)N?ckTN;|0W+_1H%pF7{$f*5l5%96dOIG1QK;Tw<=BU^7ne9Xd<qd
z!MMAZbFvvLp-^tOfUfDV7YsFRyx3q)euVX9oNg>UFo<^-)H=n$vQ_BwSRj{MAa8Vh
zob1f>S%;a|rH71PWG*+4#U08znxv4z{?0}8oXYMbo|~Y^+v>vmqnol{Fnk#wm{Il@
zKC$4azf*_RL~$%t&{tY&{=P}0?hNUUOBG{#dOqL5QnKfz4P{!z%66RE@>_<tQ800A
zDK-Lk|2&ndS0)UpJ(PMwlV37kn61T(b4d)l^Rc9aQI#e4^o6WaDAn>$hqOtMvgKYZ
zIXe1#jd#)CCRntSkLZr-pU^Z`?xPVX7C1ylUA}s7qy{;=IqC#Q_&rmL1_Ad}<k&{a
z1mAbDOH__h(h{UQROo6>VJ9CVBm7p!OxO|rb~S?vbzaNC!^KgHVvb_br^?-FV)4<2
z_Y6_1#@n0oO_(rrOZB^Qmr|B)A(yp9+mui0?em5QLJnMh4l|upNf}7dHiP%lWBao^
zvt~buep}7%^v3T6EECG057I_z@Va!PxEIZiVG(x}guvJ^)}=`FpkDLUHGW;?86Zw^
zsI`)8Aka;kmlQp^rO${xtpR0W;O`Yx*&%+u+oB$>x>K<EvzA1(YcXQyQA=>b@@#EQ
z%c=k|T2NpQnaV>!FD_^K+mV4kwHQCDy##tgj<d|zio<SnNoa(PMp&(+<Hux}Wfm4V
zVg1O+@~2D=rjS7IOw6@xHg!{W;`judE7PB7VGIe1G+=2t;@N<VenOYz+sK!-%l^$#
zQ~Hp^R^sdh83b>4GH5oA_zFy30o|`LnvPH<OBfu@d6<KuR^fDTAIcXtl$)Y?slLj_
zW%5d)i4Z~Cr@+QVVyr+1gSw{`{UZ*Kr#9`7q6A7y4g|H=ih^hEg_L&vki>$TZ^tMy
z=b;Su1)pT+!Zi!V+urByf6Tl+FE#Jm#xmK-HF<O|c4t9WV?$2m4}V8xqQsA$)YH7V
z9jIdU(DB~rdkIh-(UH%$)_E0zJQ5Fl8z3^@@2@Wklw0~_F&i4*UIP(Z)ZChwOH9O&
z5`V_X75mYp^XjLYGeH5Nq-GuQz_34>ESC*0fn^tp_X2?t6;6>wRX~#5fY19b<i~GN
zSA&a5ct*#G$bnzm>3*~o!;~%Coh}o?lEwb97~u)5rg*tma#9^QFdiG6i<|_>r$LAF
z{y=QR_^HM1cH6T)Hp9A{LRQg%m=X<GmSFb@EPU$2DSgOsZZlZ2(Lb{$r5BhZFl}||
zUA`I7yrOF~mI~?)IPrPdd?^lw7qE%BHp=Jmeb`F7JRkErTqCt!@_@YS#ux7J%q1Lk
zJY(HQZjE~jBIlo?0MAClb3@HMfj+wnrUTYSZ7Fu!gy=d;oZIvHjq>~<G5U*l4QAX=
zJ{S9dA}NX2yTFYSaYqdmq>b+#ELj5YM+xJ{{^NQ6%mZ)xL`k{6)G-aM<Gpt(fjug6
zo|<iNL-@py68#8-VnM|1iawV+As($i5jU3)5|lZpghsmzY+?<Sw=}_tx?HZUWOf<2
z3;{n9<Qq=?H%uZg!Xk<clcw?Vb%gUfOCSA9EEfraUr|T;x9IY-4cx-iT0d?F-#KAb
z6QoOWBA~NvT%ZnX^6$Cj*YbT*C-~ZL-n@FhDL*;8^&XJzU1r=Igj3&S481O^%G;?s
zzXzRQQ2L;qE@}6dH%{gJeyF~5s*ZPkdWb8w4-A4lYrn0Omk}f!7{bJb?)}{c76Av{
zC!n0-ICn29?y#5277?boHLF@X$#oUBIBTwrvufH!Hj&PHcD;{@mX%|{O7S@09fi*m
zTTKWD9qnZkg_%h09-;ZL&1$t;?zne2kLF0Js_s{i(3=pZO%2N^l`~35|74i3d%j30
z3@w_63K!9pt07i;;$Cf6DdOys*v4e-Z{ZxV%m@~p1sQc@I5an(M+fa-Q-zWxXmn(j
z_GYzjeU`CL$-LG&Z)rT3bUIDEoBF0w@_Jd&m>oZOK`QF<{`=c%ZjZ4)-#gC07~z-O
zl*2o(q|`TMeO;>3YNr{Cor%%&s)}@=c>Lg4(s6t-qq}0t7jiLFn730U_!czb0c03w
zRJnyWm*CgGm@u-X+(%fayH7#x4OvWg=2x#ZNP20RorOy{cR~*F9lJ;4f*;OAoUr8r
zi?&!*U9`#>ScwD_><2k(*H?m@IbQBf9l6gIwKH=ipY5vnI~_*Ha=DDXMw8b{-K$`|
z;hF#WkVLCdj@w+X8ffA>F4z1{Yjh)*XXGPeedDs)ZO7<XInI}AQv>VHLnD(oIJJT-
zo?i+B28+-2*$-7=T`mVf9jSVz4;Q>9xAt2e{>Sqw)gy1sbsG08qN166msJtWDJk_n
zzuG?2eh!uF4Sbv#%S`olWp#5}O8Bwi$2h2NN_vz+=BH!3cR;KFg#tBpZHyCv#X{}F
zJ^dr#v7;`OFYD7RntK1I&b@I3PTM}r^gYWWyK(;&cux|~7Myn;0R89^T0|t1pmFfs
zH849rjHTzBCh@=(juPxtI$HXPV(8V`BsJ-mwlp^{Hp_`MeoN4LLc-Xi-=$M_$KCFM
zHUP9D+?bjZY%{bIJ8@M{xwsfzw>>u(yT>50VelLzdGPxZ?yidBeaT42B2-I0b0o;G
zPk<LxvbHRD8va20D&z~Q*iZ+JVr$r(wO6W7>lxgmWPDudvcC0T&!xU5IaFHK)NamL
z^}mp|Tv=DRG8I-Us1%9E=KogHI}?KvhpCNJELRBZ<o}W>zMPf?7lB1L{)16-N#A?y
zs_DM@tO|RcajxADy|NuA<y~^gllgR6ss&WZ2Nja6@DyN!8t69SJ>3u^>ipEuem|SP
z*Z(RhSW?rz=(Me4(CU<<C;IZzEZUIFjpWn*w~fEDHw%;IwcSrcJnV}@tMiIb$-<w3
z!$#Y(ik^7!hoOC<Jd#KK5wW(LzDbtUCu9cQ;;hh4y}?k<0oEhRplV^c?Jqej?fvpI
zgo*irekK|xpUUa$s?_Y8Za7fPznZcM_>c(rN?BgCht2i!UGToo-4XIs5fQvEEGe{>
zAT2Uw4+;$3=U$(mq$EK(e`&k~+zv_;wz|M83f12}3ddi0s`tpcp2nqBy?(h10A}_Y
z`0_zD>9sam4&YX;U++GUdfvTmE;@~V0zd`DxLxvi_r-|@Es%2^zI1==Atev~`YVHP
z*4FE{f9Bw|@pyk&y;)0;+w3>@T(o`)=gq7!%>29=_hBIN=lmd{kY1*u0lQ(Era45=
zM;LccbLespqb;%LrSrw+?(5I~Kq^ANAJ#Z>aEVH%?ScAB^&vSC0kHAOF4*pf*(lkK
z*3Af(Je-c04iePZk)840^@!iI+4fLO<PUgFYM(*HbFB`%fe%Cd-@`9)UDjNvm@HBC
zLzu2L46yL5h(^Q6O%ilkr;>+6-y5oc?js!_zZUTzaN<Zq?cHMJr5Canq!SB}@|e?Q
zzoEW%%|mZdwTz-|ud!G8<<7q};?(D2nRDggpw%JvXm+QywioT~z{cRqDi1fneP@q5
zw(9w9F9Jo5K(kDIzD&zqkP!R%$YlH&B%@3kZV-m8BhM9*1!&JntF#$Cbo|_H-vYw?
z!xF6J*a^sq!=|(0y|jKEuEk-s^7r(;8K})Rtj6RyEJr#D=ndrdNMoWsMy+%O&~af@
z>7<Et@Akj;r5|JI^IF{uPdNc&&jYIL&yU31wc2rpDki3dr-Qcoh6$dRqBN{9KK>qN
z@);ob|Na9*$?xw!z)G164c|LxgA=oue7>IFsXiAc%zmnMZHNG^gOwM@1u;;^Q68V1
z7=J<_+^EzPgUxlcQaJj#@qN_P#6s`1qgsFqxv80r%XzgV!9n#wj+C&@wbctSGyCh5
z8_<jDR^4rQMph%c^fmY1qzA70w41+bY9T4I*D%5Ba)Q|)jZbsXUE9_1?K$&qy~q?_
zQ}8dILSamDXtEr283fie4D9Yct4KQep@%b$tFbJn$>%GFmzSFM+*Oq0S@UPwKj)58
zD%R90kfvhIx#g9*?bhXynq=kb=^MdO^jtbJdBYOE(0q#CYN~h4l*orHCY)cv!N3e0
zUPoEnl}c#l5HnfybzvAE7`hqY|L{4vW#rFTMhBhpt%Z6dUARHymVy>%l+`6DI7w;A
z>G-Ad8xG`nTT_KsW!E6*k^VY<$tIVN=&mkIjHf$4+s7IWQ6qSIj=#TCG*{h{FPGTj
z&R91H9g4^{iCdBE=bp^qcOP+A^_4sxyG&hU7Z&`S1zOz5srA=2dWDpSD!_oERSCo>
zk0Z9!6`VkA_TV^@{YZvQ)f{q5aO{rVHYT@vkQhFXeq%_PVm}-kA?_yF^%fe3Bu~k!
z;gHOeks3`YL&rTz-}(j;`5?cId+UAeGQY5%H2;gcDMK$d%3w5bBFhIjg7q@-xw?sG
z<5B?+Bh-s=UZ)A9&OZbi0zj#853h1?J?+-}Vz}G#1-W{Nq4su^*oLT$!r$Rl6RtZT
z-V6wMjq-apf}T$`oDbokTgnd6JNWlI#I${R?Jev0$%pAHxM@b}@N5#jO&`owN5j}E
zv=EcT(0ubW#SlyM>&X08>Au2D>38ZOhT|3X6Y7fR4au)V#wY*E>?Ar;3{|&=7<~~1
zzRn=jq9zGS#mfh0bK#)0>OmD2_I=}+<x&7BuFAj4d=dw6AkL1Le(_9CT!<W5hKAPf
zd{WWHoVFavA!fAssJgA-_1@lTxe(+0DCh`R7o1+Lv;T;x9^t}Ivpd44^lYx)HS>B(
zyx2zKSV~4klRs%LA&K~zm>5oTHGk3Aq0P>dOy;4*z^{gIO_p+X^tP=g8akJWs3aj@
zzC#x*L8hyb9*vH`2at$zQ*2UB?(Om$ZVgX$E1|Pb65+lRffg|+xkk1}bFi3SbeQhz
z@7w5E+!``Cd20;5TWs!inRp=nJHs1Rn9HZHOO!7AkV1O9p!fUww;nFe?&&APhDWsY
z(g(|o?ldhuU+hIOHk*i=1BWyu45ZGcrc>I<Lv(du_@uDkt0u+%9yw>$$`vRe0Mmjd
zs4eK`x^ChY7`(2gr>ak?@PT8rd9={#kZJv;C!Ya-9G;>KzU?eiG*IZ{JCpmtXVL1+
z*@ms_R@z1wdOfWvm|tp7Vz}Ut)*a{6i|B;UEotR)Y^u>q6k!cZP4BBq1j@8IGf1eh
z70<>(b^D*2lKu7N95Tiz<)q(QP;Qh1uRz6t0Gi!}sI^0;*XK5Y-z!s$+j~8ga&%~V
zI?B1F_@V}hoLU-o30Ebe&|7~J!!+sc{8}$SypU|bR_6^H>Pei{O5EC9Hz4mp_L4qt
zi-#+9LOd7l6_h09kzoN<a=eUqv1`p-wa=7st&bBn0Uo-X<J<KQ)(bskgTF%;tR=^B
zhR^qo9O3{*Ei3nV$yS~I!eJvzIdgsfKg6n+2ftKZwL>R7Z^N$cj)d+3o|6RgG^#Kf
zV_WgIrE{{2mtDUo`Yb2j0(NzB9r?wU;U3Gr4+Q5<NrPs)_8v9lW8H1q!x^RrbK7*_
zO}lcl602md73mlR^UL57>LbwH^QvsKji-P`MPlyMDOBlMrAL5SI+@intw`sf=ZEh|
zl^`|=_qQz@(pO7Iq5GHI)<4mEF7GZnh~o0!E2vo$VI{20Ll3{I4!7xrmKI0oY3Urm
z!vsV~Nb~^aCO$lJCK^uHv~>6ui-+Q4JDr0BkNz-*tyMaIw4CB#3w?>NRWJ5w;cGhk
zQ55dsrmfez@^l}4dsAh_?eM|MSP~l>hcGVWL)H0SmF>~x2}96L&m(1Rw`<;2clKt6
zaK1{Ye0~>wMr5W`4`~o_DS?M!-TQ7Iizj2avo0k_+NOF~0JC=>ehAx7XpC@pdAq`T
zW`8775e#w`{-q>uD41P;j02->HEFfi2jUo;!wApdhJwKE@QJJ+2+pkg68N!Zw01ms
zCUA($Ztwat(_;8%8>{HEX;z6mNRZHnhk)%%daY)VShmUUOlvxXpW>ONJSw7}7-(y!
zzn{q2IVE)Yo<f!jeB-a*FV!dRbvd<2&>WxhyQz@od@$%Uk)2FzuDZsno58k3q%om_
z5O6Z98T=|@dz^*UjKh@aLv(-OvWuev;NO?DG~`oKS#Mq)j|d8wSwBOnSJu0-IhAHb
z^+5A&Q8!6V$-0JbQF=1aU<VZIQVHl`dS}f;;wMF%WCHz9KjH2Y2?C2JTt%JcVnW>^
zP|Yj1rdOu<9n>??Y;g=J)+PfUmJJxKZCQDIW(+qy2HEWWNpVv^nYDNAukWFD9gL|!
z9q-}DD&VT}zO606$@_q8<DJ^YC~mN*l2nvZH22=YX&c3gYI@1Sw9x*kQ6)=YhwV<$
zH0$OAW*c&0y)H6Gd%Hi1?6Z^);YHiSjT+RXk8R@1L+!CJ?niA*;l<e{((s1hM*r@*
z8%KeXf$Y{}#R#{5_=-z!iU|XXVk#V#?Y7)?F>J^P4G#xe@_HFqLlw;zYXAWSH`8(t
zK=?o`?>syoEdpo2St;YI&LZJN)2oH=kd{bbe^C88Ti^EGsQQ?Xh!7VSU#QYdnLOVu
z=q8R@m!Lc*x*x8q=hTm6tJ*rj-6(F!R7_&k4d}W*FxVw%zd<Y&h}jjQ*LOkdI_1Wn
zWn5IFZf)^GLPc?JQbz-H#?ugYRa-tqRB2?l49K9OG}BaEFHnMIE~Sqro+W(0#ggsr
z%!xwLe^RmaCve@RRs!whVMI4(4mg=?=9xv(`P@H(=maHfP7HaXyaH<5X<fM$LHRXZ
zWBKBdx;_d5v}rA<KZ&`m4F!X-3thoMzkRZkhh<S)EMYPz!JsHpn23}g!PRIkHq`WT
z*gQS&;vpN#;bI*EozhY=^NFReZhYn1-P-6iRF6%*0n5`oIUId6taQt`Ku=|enBmw9
z!o5S%t<zo0-7P{aj6L(1<k<ez;xDdJ%sy}m0f*B9UZEhT?y%G5%3aIp6X<|VnYZT=
z==Oyg%!T2(P}cd1uKtl8?#DY*xKwM#^Ff2(a-V^gr^m>2*^@ex4xKfs!6)hm<(9&K
zpBd>R3^;sc69xU5yH8U)JJtkL)%FEm)KAtnnhS)OmkGETrMG;6F~M&*Y8USFooWIw
z-q~zl#4=4&tsN;T3MoL%?e<YNd+GQt%`vW~ct5NY`dX@t86Rxdnh$^B+=adhIaM)b
zbwf%(trtVGwl>`%)?ywUGPKsaA>B99%3n^SvHG~g{rYOkYA^$ttS|5!v4DEj>f>I$
zSzpgNRCc$e-8!DvqGsIluC%t)y4UQlkG@`iX)utvs_7)JO1166Y~aBlIYB9&d}T4l
z_nYNaKyCbMya%h#4r?n9=_LB0xi;Gwo%^5D5cYQ-p$7mpdO*$U#Z3D@3PUtr&qR5k
z!M?99ZeaT@KZZ*>VS&R#cLm`WjV_p9wu-4!YYYv=$WZ=h8O16Nvd!V14wWF6Cbeac
zf)xjhXO;R&_-p;9ckj;`WaTl$ock)gyu!)0<4EWU1&&etgw)QL$uJJDF5j-}VQJk~
z#-bQgacXCm*Pek>axDhBk|{jWrzwt3jDf6BPAwhuhm6HJd;?&cZ&5n2{e>=%&+NzI
zh-FIe-sm{A!4zBtPDvKdQc(l++AGk1!YbFqlbuf6?m?^Pqs3Vv!<I{s4~G-{6jMd?
zTj8-VovD^nSc4X*Rvi*#%7>pHK{W6=zvz*wOVl!dFW0L#)0J1U03hSZxh7|bz!y|7
zG9J!11qW<aN;)V|LmLcSM@Av+l{Hv7k9{&FNN3jIK8XHwE1)nw|BVlU$nv+hxvDfJ
zwYfd#&aDU84=IQdcB%-}d^Mv&<0z>RZys*8ki^mAeJUs$XX|2{nhpv&i3x6Y&k+zx
z=1!l4em!YXexJ6Zsp{|Oikg1Ad9J40*`LTdWAq2ufb5%VVIa13TG~ei!3!shS_+Vf
zF~15JAyulD%oJ|wY*w&6jupTj{lJmIo6b<0c%z9jTY6*Qmoqg_ypg7v{8UawqCzx~
zl-$9@v~T`dc_5^?X}$<kadg0OUlbX&`qXT6UQSD=<N`2<7o9t<1HIahyO7j2b6ea#
z2Hm7LpH|eozNAwz3$ctv-A@Luz7?;|Xp(6j<S~RxU+#Ycr-4b=u6)(~7u%({0#4i>
z;jZXCkaC)J2_Bat-pv{@iVOgPu*S9}agnmurTMzyX-%z11pzb1M|EPs?J9XFZc`cZ
zv?uj}Ycd)jmtI*CDE-&7eHHBBRKjM`wgz@E8B|x5Hz<9$QCEvA=BYOsl}%5O2<NaI
zJd>N*30@`vJ05!z$K|2wgDK%Wg!PlvyrjMUNCq7e207@_QORUwd~B|Gt6D!@>j;6x
z3ou@XUdX+Vs$-#GeBp!^!yg;$S%pAO@NMTv`8!Ay1eN)y>(=#kN!(W-7cY`Y__}yX
zH&NlYCO4;_%Z^H<#vt$j_TCM1j|>U|n4LN+@<%Au`_`*9ac79V0+$0Y!B+RTc5f(n
zJ+dnj0HwJk2dO*S!R7tZ@S`>@<~612G!2X$HP?W`K;5-BLDdH%hSCe`<6hC+ryEdc
zWyEdaU<2OE(tz{=>qNdy8&u+?Z$hWe1vMNnS-?Z3``CQ(^TRw);J_``U0WW`*Wj-9
z6zkkcnv)2eLGTO+%RKHt!EN}WMAueQ!kF?2l|f$A<@6mLmA!|j*52sj5d5H9`$+jE
z!l|>AopG%C;CUHtd%@7ePO-8*QH3ueR(A0d&$0bCF5kP;XBhBwS<x6|$iUj7u4KW(
zG_q6hE13+DqpG!z>z9y=lTM*~Ihf=jm&mBGGkWqj0vQ26^(>un4@a4R`jcj*1phE*
zPE^cG*2p=Dbyk8_x-)gVX5IhAc-Cve@Fp9clw(Y=mfqIpYyT16Dd=r}*$fdTQctZG
zqq`}@n194CL;84iKd$L~BKZ&ETZ0e~3jzZOt+O#_rVJVC!47UfP<UR^^g@@o%Lo{l
z#fN7-jg_Zpj=z7ug_eUi4E`X2TB2Crt|i~*`m@#l%an~<e@!?sjRPELUrLSOD)^bo
z*KeRyY=p=?9uxZF-}*kFSt%HAdk?JqIEFY9)X9b{B2^$1fE`HC@;l~Wa?P51NL;Mm
z0oyNWo#s(8bPxg86QtL!sjErn2ODp;c46lzh;!}P5qc&eJh}C?&-A_cNy}Sd^f1z;
z%CmK3{f3=Y?RZq7P?JQYlZd{nf$=@vhOEn1D>7Mh00|Xw^h{puYqzV)oW0-Q`jotD
zrcn~`I@qQF+rowquMSF2TxkKJbSVuj*?37$iPx3|KlB0HVW{tIE5eveuIS+UOY6jP
ztRo`>Uur*B*+)kHX{{`aRDu#Z03Vw17S7G;kP<h_e++hFm$IU=Vh4*M+vUOi#xet!
zwj#ca;2(!gmv!ijJXEJwW42bYURl{^9e4erXo2<LSJ5q?FdX9uZnW=k-oNzn*}>#j
z!>P%LA4O-_v4TtGAlPh(bUVecgydgqR<X5|iH?BRu>VJI3(rYB@Tz84_S>NU>nYp>
z8nEC{D9|J^+(cdT{)dslXuf9blJ2HKBxP^)*XjT0c7|OiE#eXHl-Mtr|6$;g^Bgt~
z>7S#6h@by$XhCsJcs89gxGbF(Q}%zCC4pUMWB>=_1Pklw=>JdsYaCfpWMj9@RA~OA
z?8PmdH(-rI{r~v$g8G^;&Hp3M<>w^M3tHdR>A~Gwi2rL1gVC51kVWrfo(!n>=d3k~
z{SR@M>#Zw<lJfSod;C_GSo|E)^hq5hS?X0uy<`eY^<eWzRst(1HJt|#-OC*mpZuXg
z4)^hioGD9(?tLf-m?pwqi!X*Y7}iT|<x_o&^*fz2)8`}M?td}?_upfoE1Y>ovUuDw
z^IA0A*K~r%W`gimJ=C@Ylex69B;*E7d(?@v9jrxrc;CcV0qS=lDz0GBC@q7i;r_o<
z1GVx>*<=d?waHXzO;R37@?Zc&Ha9QZ9f^8*_%-zrb@9VYH<79Igrw`DH_Iq?<llBn
zhTU#pp<#qaZYaHqRln-L6^$T3swBqMapd}X`_0%y@tbko+n9qXehdV@Vw|Uu$Q^gi
zoFC&P3oDWd@YtH9b)az^e0ocULo~C=H0J_p`8vDSeNuqV;A`-If(QG8mdX~noZC-b
zJO78Lw+ySJ>6%7^y9Eou-6dF%;0{59ySux)LvVN3;O-6^*Wm8%uHWQ--gEfJ?73ie
zPxnmKTGdr3{|#-Xj2EzL_iOwx(ZG<(OD?y%$ANZKP<cdo;cIGUJas6a-81_c>Ka+E
zk^s-%u&eZaW&wEd<|mC|1X_&B&Gm7*Qj47QHJ_GaKyMWv7bh2>SgQCN_q%sqP&qKu
zBjhv?rAiQbX5x}1b)Lr);6ugs72Kv8%eBiGb_TB)*)Ljt5ie4nUC4?)th&Ch*R_Vn
zq__H<AHD~?IA(Sobq0GqqkpZmA!N2*&h^JuQ(UefobW!l2^jov4yx4dXazFZfSnSV
zOnQQPdc;X|Y)D{yb5hM+VrxW0Xv4$(12_GCZ~e0Ao8?NE!z7?#w?P#*$(8dSJ8p*r
z$>G!M+D5Hm#K7bH%g6c~ZVI}N1GHJ{?y@oCl^8GsiWO)MWhd2{8vJZ4Ndk5+*MbED
z>5IuWhwHw*gZ%v}tsC_|TkfUVji%nV>9+WG(gdaR9Ny0JIzENA4i?(w`_v_8H?uU0
z`q<}-zxqJFlCrY0oSdAwxw)idcvV%EB8`aE_3P6OU}yE+W?hjcF)J%e*oi>Pbvvoz
zSGk#jBv?;v#_EW|-Ippufssa$a(nei{H30qE$(9!tM^k>ZdXAc!gNWQbZ_U4kv)sa
zHs_D+j!v-qNlA>$cG14N?q6hnfdp}^feF<~QlaoO)PApyw70m>oybb1NF+QswQP|t
zF>|d2^f$JA0#mVnx^%*7{GJQ^d&GT(6$a5scZ<<Wp(4|}`emMPe}@$}v?pOxUHt?H
zBSY{nl%yf2UB3>>>=(diY00jal$C`}O&Q?uK$~EXo6#62QUDsKsHF{aubX+H#-@Il
z1@Lae<=u9o#}igLc#&Y0AM3GqQ{N-_4|D+U(V6ZXhz4j!F1a#agUxI5>oEx`xqt@p
zL!`}Z3RY`6&>rKXv|EZxWPx}T;7x<6;$T#^K<6Of55F*9*2@jS4o0tyLbXwj8tyP2
zYWlg-YQ??rZ;C*yTxlb^D;S8Go_~_L1#Nn|%H{WMaoKqX8u@~|9D~ZDc&qU3DVF>d
z^5|)9n~U|qdqk|wWO$`}cCA-YTt%WzCN`{sIdL*I=3KmK+X$oTU#kS_z%rN8-mH}_
zA0J)a9QBH*=;-7$vLw;`r^lV>g$j-2jEvK>vwt3m2nZ2h^G_EwR!pK_czXn8NZ5Md
zgauYL__|j9?x8V9EYQNLKOYjXdVTcTc+_i&-B$>&V6Lxs8a|60@;00NG8r|QKf%n2
zS-k{nIcHi_4?j~X_e((z79kX;0g3VwdVfeAPGw(K+;-N54It>ApvA-}aLcCDYMP!v
z#7O<jhW5!4sE}4~<D`3p^h~^*Zi@>T>N$k&II!P2!D@KQK0v2^02`N3L#FT?K{37C
zRAity8=O8lo=Tc#h`u`9tziU-c=#rTRuH^lm@gt}1j==Vn@D#;!`w8N)vwuP$7b7v
z2#St;(F3s>uky=lz1O6rG$EHVQ`3~A%)_1?qctH*iqd`3MyR0dnS0M+#`mzpW;0<o
z1PSEqHgJFKN~2Zbg;`<XBb32|35IrbE(HH$^{FFQv|wGcrw2@iAOr**V~ZVkfzJB2
z-LTe2Q`U<Z8298%&VpLDlA~or!gjLSmS5!K!=>cOaQ0SlL+Kc5zDQN}w3xVZ{e|Ie
zl5LiEQTNQ%Vd)V}Cwr7ihbvfvyOUs%sZpXYg@JUPFF}ugMFGW=#q^$r&CX}^@LlJ$
z#guA|+ho=s$06CL&0O=Ku?nSw#y7_9D!!8^o!xXXZ=jl<5*?>*RkEx7dVK!wE9=E~
z%btCsx&sC+9~G3>aj%Q}Y!6YZ>o|(c4_CPtW9hF=mTOn}Ysq(X1?QUI42Ow@^d>^T
z<uAO)Th@+#1`@tX>~^4kuQ+`iE?BUVe#)*fo4G{kda-+tVZST#js*$y{Dxq8VYQsf
z@owX_v1wD&y>FUX@XlVMWG5>de|#{0%}AG(8gW8=EC`5i?P|EMUD5-+-wZeR5T2tF
z#+cMbS6r-A;M0VG7K%?>95+_sW(qQNnDdM*cMj(>-79!g&G|3H&x0Psq+j=6*w9xX
zQPk@NV1D-YAd%{)53!pA<aQTAq9xk#C5zL+_~Cp_ybzz)^9zoHxEL2VcUx;~nPM4W
z+y_KqZ~>dG0F^A@GF~q6{-ctqIHrWnc<tHAg8kKIVAncsBS~h(>V6CXY6*wh>`2{V
z*gjI(BnQV!My5z~f5u=YK|oq70<wwXI~zG0t&YZhJ=D_C(NYVvfSy4`oE%7*$sFFv
z(yS^}o^q`-)fj#C_-^8khfX43nIG554X4E|<LUCxZ9;vdXOTEZE%fg}aa?%j_}SXI
zqTf(zIA*Dwj^w@Z0fBgj>{%PvESAfVU8kES(n+ZuWKXFvaq#M`CB@c8V4q0~iuj$g
z<OB@f8^Jm|;B_nmyt)MH(UjYZ#V$S?jjErx=r*7{e4wwXJnvc_*Y5}+>NbM7zw?h`
z;9IrBL^unsEL9)49IBvD(1-Mg#{)Vum*=Vr<3HLCoTLlgDZ6CllM>Pr6Ju<O>QcdU
zn{D=d)<J~qJ)&2+MBP%JU6Eg#eNgP-BhGcoAaOYh8GK)c2cp&T#ASNLPPXn^E?>M|
ztG<t^G4RA82IDflX&k<xovxuk_-~yR{%k~R|JofVO9UdYF2>{*y{o>U^fj)wBK+fl
z67@#&arEqPg_%cCD+#5*xwsUbT~Q%me$1D{AZ5>tQAW&zWbY6@WA?6ra9s)jLz85_
z6sbbP=brKysi0xQi>!%R-09-Jh@pu^yQ-&4IMOz;>_<!eh=GtltL-Gm?slRt-NDlB
zXQRu`n%ncyFh6xD&1Ni?Q;65Gzc+avy!}$EPt=gBn^F%HJ||ou0<Cq{54su%&Q~xd
z`P^xw39W@%pSP&&|8{#BVm(+V6?M5Dd422{-Wa&)E@a4Lw>m<q9gZO$_heS~ZrciI
zbH62kv%H`7ALT45K>Ch7;r)<odC?P3abH&W;drP)_-?nrK*hQ|<d;x>EIfzctmya6
zWE|9ljX&!|$lk6K>A16KSN^buCeGn<I+H|BE3>E3j)s2j`myL6`)jh~Ya|{|Uf9(x
z?hY1#w2^>^!`4*T=wCeuu`aVob9)wq0>LL}v4i(4hYQb2HIgrk+U+in`V7lLK6*nQ
z7h~LKvv%8$@>^+L!M_PwIB_LvBjN|6-l*3aCQIZq@#{8HIjq*md197HRif$DYQ9a^
zio{<I_P1P6jH}Aabqzf>g|x1(t~(#Xi2WkSy<pI8Psz?^j31^7SD?ZOfJS=WOV+tR
zTlwyCt_e7V)T5=ke-4ABZci&X^UZ<vu)yZ#<`(l1+feSKbd#FI>+)k=%#=oexjniW
zv^ayIUviY16q+c+%$~^lVP&5E-uIx{-eNb;9q}QK=5?X+k@5ZVj@Qcr>6F{E@5pdg
zm)^tpW#qeh!xwh9v4r0~8AKI|%pl$tv(0Kw?1&+IIIWqcYzB^))P{$;tI?0<cqjeY
zlB1g<U=QkPAKOy<{(e>V?a@-3&Bl6_?EAHPQHijP*Y}6a)H=Q_?i#NZ>yC|&<4K(b
z8QgCLVj)-#-#-d>Xm2>(UOwL=Z*D`MpBLn39?t#%tcscfWUTbMF^Gw?#YmKt#pyC3
z*0=>uu=*K`N5x5zIl$J>Ct2?xxUo$);NgG&h&wky&*@7SBxSUP_ea*+bcUY+-^7Le
zoi3D6p!j|*D(8Tzs!?4RP~_AdP8Nx&1QGA(I%vXefBw#9H-C^Ek26(xuF2eDfpR<b
zHpIB5k>im=9#5z0Uc-RAc}(%{+A3xE8Po;GvhJTmpS$u&{{ShAlAqX*`$L>EIsgle
z>6(XKOUF2};8U+hwM{*^qe{g#kua)O(6CC+Oz+}G{nc;3O=q1n^-u7$q2`bc{xKj1
z<@X;L<~{V?n`V@OY*3U0w;$})?{>GS3OF4v{B`?Gmd&QSHGfPO|CT~%YYVz^KtlTc
ze1^rna*JN!v6ihRqrIiAg$DPtR4(K&QgC>y@;N<`KI1mm^)<|~skmfGm5f-f7({Ar
z*tss$HYj$wruj8K)`7`~N1QKOsm{I%o}+>9Hj&fxpp7?EExlT>^(0n&Vews0QmpMd
z6J9`P%5yjEE`Kd{Zk@yBZ}z;@y$?roEQBKp--hkXmCts_RPm%%+luY_J!9J(^r6X6
z5uP+V2JT-n@d2jv)xeWo-xdh>2E{vf`Z*){O37kGD;9@-z7J1t?}>zI=|s857ZdK!
zm49CjFL2IY`Pb%c+h+G8lGiP9%gHdT$3-u+IY06A5&nAFHRf*rxGB4z;9qFyT60+*
z#w8U3Qh#$_izhW$i*q5|iLTFmvk@`ZRTU2k@|qF<P%Ffs*^>=?zW8t(uzNMhgz1Mu
zIaKtyf8P15EWX;w`7XSk0^t!Hyds#+g_G%_$>Z_$HPQd)(dB-vj@Vx@ljb%{J!AHd
zeeS+zn3c}?vd)!}@<znBOK48OIJ;mhiBTuYQ5)DBvhMw2RjA-N<ajhg&dm+P-ai3p
z>VOe2VA}IXegv5sTt-V*mqS6fw(4}*`P*oVEdP7#1qH)L!=Bg%(#`Zl`u)UWw!y;#
zStK@!mTzM`<%Go^y_(ydw`e3JA{?P~n*Ddv-L$=mI3H~er{gyEjn}I3zgrR)Hj!UD
zEiG}M{08cr*U(@3m`yf<E8vFWlDm7`CPAfFzjzDXWpEd1#N7^yUwjDdT)q1Jzqi4^
zd`Hy07P~2xTEEIqmQCaF<ZW>n_(H>d)^Eh?y#J!1PMG?%J|`f?T?N>gfD)yDDKoBp
zo;3`u`m?&CFVLO60^3XuZ$1OB&*is9{iW_x>qJOygWiO={`*(=Ga|S|Bq;0Zdv1<u
z$bFfdRc}oF47fT~4})-1>M*Zcx@{jmL6GXNS#r6Btj(VlW6St6p9@M%_Q~CAMxa8d
zR&z{v7O!z~OA)vi1VX0$LyVws7+tEki+y%VhmKM&qACx_c^;m|yhBPGbZkN=czC^U
z+g<Q2RglBw6L^BF16RBDPUNzqKMle*Xz<rFwtui2vIYz3bmva$J$rcWLMzblyOTuX
zA?<|pWc0+PdXJ@DiR62DpXoYrhpD+)y?s6;g1v!h4$KN;S|LT|)r}Q1+y~=Gcc=Rq
zW-O?Vk;dj%*hR%F@4F9fODyQ|alk<nopm*$<m!NW`hs7}JaC5;)64SJFBdKiZ28gD
z=>FB1=!fS(b{tSVqJ3{S4>b@|uAq!sx|$Bw?l^C(*!flE_3`iw9iFexHP^xX)#<A*
za}T|vH^uON!ygj@{5%qU&%8XJwj}l@=@RYo@7I5{k>q&Yb#C9DJ=8<{l%9WxX741_
z86EBVBtRC^wr6-Z$vijg8a8}e@K$XuZt;*%TTFBu@9z~0?=IBa`0n1-vc0BF`860i
z<U1~FLvoj^lsa2qohET>-e*VZmV}9vcH{Y}-|+rZt3Q%G#+bkB$RgO2@1Pr+3j`;C
zhsm2fZtN1L7hsgia8^hs@9ytf%!1?58JMTSyc$&|1jb(p?oH^-zU3N|!ip2Rvs-d^
z><m^FBQ}4T<zi*E1cl1`VO@G)<sT$)^ST}|77Ck#c+xY0RH!00XrkHCwI-0q{`2zY
zgh>${&gz3>e)VRkF*+~K{&fjfRGL1peO0+iTY9#>xVTvDZz>!Tk_ax8z~OR3c|d@W
z^o(<Yg5(<4^^pc?S90}p;3H)8xlotczP<bG?*L?{v^DO!Z_;HmrjannhwgLpG$)bS
zyyYfZ2%xZH#R7#~-(5cIqaTBlxR|HH(9zyaHy3@AI&B*>`w2lNbIov>QwW<&PrHoe
z*N%vbbK&V~<$r#}xdlRcEUMM%{ccwZRHss$Bdpi2OrdifRVii^Mdn$o@FVXoNPTnm
zvA=I7Ac2%T<bAJ5Y7nS4e>;(#IHPd1k;4Xen{>KCw~v&QS&=7F>3fRrHUzH#n<4Pz
z;-9f=4*8RWxag*X=-KQnl9hqrkNYysED?$iDHD=e)f05$8OfqFky}d2n*}arQTOs9
zF%~dDuCJUED_q|0g^sc4VEYUSiHe#HhL)7E4c3)D(xCjtv{cyttaq*eTC!)NcicSi
zI)V8F+0Jt~)y<-^eu2HCbe{?fWtbJ5`NaP%XCgp`7q!TruQsQU{Ph{(O9TH!zI7qE
z`+@!oPVgQ2n>eXQ_idNyj;mP^SR-k<?jBr9oycV(+Lh4`jp?_N5F*-#EoyqAJ9f=Z
z*w)eRCp+c9U7d&#F_&&}J0&6$lW!RDdHq6;T;CkY`E+hgj7`9Xcmm3QXslEUCP<cE
z(>B~Zs*o#D&}d5NK16>HY<GUue;wfAW6XFwXzY<{jEca*jlSepttKiQ;d+h~od7k1
z_Ds=bm3d4<orTo>N#N38F1qhKUKW(UsYe~M*w@;~#cXf8*RV#g;>E2vs%eqli(*vb
zQ+PX*-5>7J7C)S?FPiLhI??hTFL1S=F48iRa`tjp8eF(fB>0iiy+kKG1-1<TMjV2-
zj4gzPth$droo~uVx0#DA^UF7WoU5Ye<G=Ekn@L53^v588<rILnomM4aJ`*YRug{!e
z@w^>en--JJZ1MD7C_CcIYI$1PerjaaJ|mcU{=H5tsSPv3D@5TQ{I_v)2Zx{|P9z_V
z=A*Gr*(#db?&!GG$NSBFu(vB7nf@#O(N(pdMTd45aOf6y$rcU{4%A!MKp!r|<8q#{
z+7H;s0T#cp41YLxoa?)}=Z-j<ePI0z0~=J<K+7`BTuL;vd&V2BPLoo^puCrRM3gvO
zq(5WY#=E(y=jbJN<u|acm4#pWL(*au&rb)@e^9$~=dF!vBl?SqdhWfUE_Yw!iJio#
z`DvpGm?K-xuahE5w_A=&OOTZK`{qfLkjfdy4F+a3AY8FXpaqEbp^=Fw#R$^vj_URQ
zwC=aS7z(fQA4yxJg^)yO+V0T?uT{TPkL1uFYK!x4QKIc;`4kGj4jZNV4F-#%H%mG&
zn_4k|l3x@<OzbL$49d;u`TU)D(UYL_?f2Ox>CxXWgLH|G7RX2)wCzal0e%Onc5$&}
zeDcKgj?~9U+`crBsT!EV(AhBS0ZVtJ<YXZ~CHxdV$9a@EZVn>Z+Cr%VHCOxd`#3T)
zv+qmaQ2#iD!XLEMLab{+Ky(`GM1PhVUnOPZf#^x^53FU5t!zWFVCOG)8!IL)3~pm8
z%QT$kHI|4zV1qzj6J21a_F=5IK`E9A4^C46i;B`UBYjsc7sz*`7JM%?rc?Q@L`lSw
zL^_J&C|pw|H>F9qV$kXJ(Si_yd^Fz>Z&lT4!G19t6;Y(7J%p|ZHC<I}A1h^#Oepmg
z>qrRgxXx#z!<*E&e#gK}DMpoBb$M-C18T6o)WRz6D<Y^Ms43>S*^Z32B|e*;$z9s-
ztK?3%cq;i9l!|PQk2dA<rrWPr3l(OhM$ygLvUf-GT$=vcn&4w#3CaY>pWU33T!cgB
z-$hI)<S{9eYKDz?nNzg}@fT8}Dx@h;-AypznUxAhMpUvp(%074W2cI=S&k9H?Db5+
z1#I1g%$++w-Gj#`>P=k%=KPT+-@WH{ZT55`HgMM}Pb^}ivJ4u}nzMLD=jVxcY|5+m
ztt=u?Kr=t0(<ILmnR(cE?t?BWI`P`ChNf%IfYe^j_HT|SE%G+P?my^=ya9(5AYewI
zR<%$elh?}~$S)xkjjXP(U+wUCD3VU6qobQr0YYf86+(a=xE0@)Wpp{$m&)B%8t+C%
za7C+-5W=!pZ#tGG{f#3La|^B?t{hv0#l=<p0?^4<?6`H-K*I#^EDkKcGN`F*mz-|*
zAfbB;au#Y4Exa}QLuC~Nm5ltPBIziXjYf16&-@<)=}1*aiQAao!UOA>!*Gf~e!`!Q
zXEB?^*XK;i9^+t{<rd><j3mU+Vw8Ug_b#e=rqG9R{luHmfa}(zm}Nr^e-?Pse}qsH
zbtg5t?~2wvre5q62)&_Vel+|nK8apMxkLQK)yy$<FDLOJSq|1xNW3h@S}hpRLz4^|
zfw-((^fH1H&@_Pda)c)>xw{IB(8%Y-@{-E*(2yApgJ~y*$DJq=#}3n(DT1<I&j6v6
z3E9%+nn*~Ila&g1NZ+RByv=;}W3aJY<wLabK|JHrEo!2-w~@ri`7Y^zV;kGM_AAC<
zh<izyB!RP{#Ng{A`eEbl`(}l2+joxm;)j;Vm0Ya}9gOj|MT)$j<Wyp9t*VfWpWnzR
zNs+!lhhm=%3-=Dq@wqA{zz-!=?yJ9-n;Wg<2D{zuqDWz_x&T>V^Za#{cdgOZG;GI?
zqQR7OtIKZWWAE;k-p9D|GaBjkcZv%|B|@fpC!3iL?st?o^}mYm%Fi?v;R+u5!K4{{
z8jJ2OS0C~>A1Y3Kers0eXy+_xSx+4khO(kZLgP>EKctlr&N>N1;P;85bSxEjL_c58
zY$&*U)&fRH-tzP^Rrnvl=M}l(38O2Gcn)8t1{6;tmCM;?h)@u1ZKv)DzCG=%f9qY_
zfw|F$D#7<Vubvu&p1G*Z>X7?pshZv_FrJ>V@+y;D(?=;P)SHyu#G3WqcY6LFWS*(4
zMP{*0fn1u&;bR`cE3|$zn7vVFM9BG&jCrPkba;QXenE*ccC@5s_d0b|-t>#78@904
z_u7<oB2Mk?Dio*51H?%9aWd1;KX^#I$6%dn^&p*M`SX0X!1~!w+MBom_qnBc!L8%$
ziiPm*G@ZSsNo)Ct@H-`<YJ;(1q+&EdARG4Y_T1>bV5{tVe5Wc|ub~?l=K+^P)WbVi
z6vLbS@g~b9An-~U6;ebV6e;A6?8Tm@tET*OTM|{P&ACD%>*w6|A$UjSE4zqeM`KO)
zghu<Sm$<{nO_#Svao-4IUW~3Vy%rNK1e<x0I=!n4Y+0Sb{BHCg%7&IkuRZ3F^N>7$
z%h03*rjt7}ovKsaHiMU{k7^!RYK)9$v-{)6rp_##s(yhX2hmJri&}liB!+@<w4a6{
z$np@zbSVuZ#({iYdh^Toqz4Vk0_~#tja%XCP?^)>5f!n9ZUk!(jEZy4&^(8Td4vUq
z8g&y>k1ne9k$BU1_aD%n^;_}ESpLw#(g8+7TUCiMc*wC%l0nlogqxWyQ;rxInk1dv
zkjyZRTG$dFQB1}pp4JHOqK}(4DV=$yala4EXFvU-+gN?&r=g;l2>xbw`S-w<I~3``
zy_=2*uN_>g@~!c-oR1Ip%lEvdjjO7S%#(M4_=2wQR$Uwi=?|K;KO2n9?6PolDB!iR
zKE18#o<YvI6IX}m#&=hECo7^=Ke+4#GUk$Y4|(KnflDfpSY7mXH*L9NwM$4fS^9gj
zEz~36tcKgF3ID`{xw!Mi<nwj0nW#`*!MV#}yv~P}q$J_OVE+@tQIo?1-DagL7;+Go
zHoTfmPR-$>e(HI;g7CTYaIwAt#>E9R9F(J;hu7(K<0t8Z_9kdP|3^)nAC@BArWtnb
z<8%Lc#Nis2>y<?Tc0{Mpc>N+mB?BFzdnGMFBJEE<maAR#GS~o7jhJ{FT2mGWhZ_X=
z>)Hm%72ieZbsTE-jo{zlS{Z`NPPX3*w3|I|w4<iW*EGt9nwpY)E0QCE8o0J5yjqp}
zIDCe-8zohth6WyKS*o-<vl1Ah@v5!QL9j5t76j5&a^mMKrWczsnlfc&hi`Ag%n7%s
zN&C^Rp~rVJ?5%aW+*w+DO{a1f`Wc)m3>)e7^Eo=cu~3VY)n6x^AeWm*o2`H(K(e5K
zqHp6`cwTAjWwcUlb}aACY3pTwxzPvjBSn|Rw+N0?5?$Z22A6pX1e$#;PgG^K?*Cde
zp)1_1Y9O9&bi}8J|D`NY6B`%|;tYGxzgQ@gH{E}GJZZ8nDnHu}MmnDPp;~HdzC@<#
zaV8^p%_CAb(Q)GX{3zlS$$*|p+JJ0>RTeGV?8GQfu;9i2rz%}ad^Gs|{(Z|%-dgP{
z*_y22E1v(xS3U#XP&TTJoit&%7MH62A<6^E;|IK{VzC6a;<orQ>#1s^gpfyQjfLM9
zsJe-+XM<<`hQB@Hqm3gi2u<%UX&mkCXG~e$+}waXcx!8GAZ&?MIvpkvKzIfQ2J$y0
zI$i%fR>(g~NKZTY=S+^%*&^GZDNi)yT*6zeg7%ktykKOtRLZ~Jo~+Ml7`N-d`rVOu
z;u&`1)LuKB_NK94I$6)wcPhQs@c0mp*V?ojPV{r2;HbJc!PQ9qxITz=H82c_nOa7x
zAn7Z7E)c?bPFWm`A{QAwoT;6&o~A$cpy6~fj+<+;g0Pv>rJ5`;Zur6Y?zXwP^zyE?
zp2}8QOb*ss@4+?MIvybB6FV$3-t>Ti@Zcoelq<X7#%l9bn}MD{s$kF8#VRk!!|di)
ztH4h!jt7`myDzR5hkW9fapIquiRYn^Uy-;NQ`j=Cj@8QWXBCJ7ps{@)rEGV&49!8+
zMX2g1f@_-(Jhog&?$X~oSG1Ka2_n#Oj~glctp;dmR&(WH@!s)~386$j(LF`NI(Sdb
z%c-ktlrR!<_Rp35V*FzjO{LVcuH}>Q6N5|k$@OQ*Ww<E69)-APV2Uqu(KoP~wQg5z
z?=>V9g&YTsMhW~d3YcAHTlv1IUsJlD-g=1K!eUD%ef#Nb?*jX2ZBHmPyM*V%gLQrK
z3|N{walQHAwY8dM#iULnq?k$ZPr-BSV2_zm3*bo%axQ~8Jc!|((2Bl<484zzVS9K7
zR5M*(RcL~Wih3)xeBGJ0Q%Z=Ud?l&mrRU1X*Bul-@9uT4wdE-ASLo)3Yw8=C!;6#o
zM~udE-f~8<UNPb4%pM6380Re>jU>MgGyMeiyVoKz?@#g7dFJ8-v(JDUr6gp|7fWX&
z+#05}h`~a*t|ulYs<JlqYtM@%wLC42<XnB0LV4rQJXST?*~Gz8X24)G8F+ZsHZ};4
z+OKwovVXWKFeGbeXfSHE)&przVPRo3iN}eSxGCG;c&1^Lc`=}xDd8A=Uf)uE4o*B~
z>{J})x*>3LUk!NUH7ah^Wd|Q?rpmvw+Q`p?Q!7y(9aZ~eA1X~S`nGG@nP-Y3L6dsk
zhH;32A7R`Ip&qhCYb(UJUH4!Pmd+OQGaQf{^Y-K&=*tD7-Pr+c$8Q8bUZ=ChW+=Z6
z7$!~dcbB*DS}6EDyFM~rO}q>ASKNfe4-eqBdKf6F5EP8XlnD`@o}R|IC8wpC8yOjy
znc)OKr+@0~>}(FzwjxNF>qig27~1$4DEL-H;IT8iaeOlYMb+Heo71CDjExbBmLk4_
zi4|^_BH-7a`R0>%rN{=6JM&j_bbq-2hlZxdPj!qhJ}mSrQ)O!<(BGy7&h0g+lvwPx
z`+;EN{P~D6?t)8OKR{cQITS&NW@-89?eS63+@Ry9l}e(ONtWNGE(0djMEn!{IRB$8
z(F1I_C|@%X0SXFwGage4_E3bp)+D2Ys#3xqKRHK;BCzh`w*QGa%<1x4nTKjpe3Bb(
zKqCpMUv}Ssjn05_>dTl^L#uX1BW>;OPxtdMEZ@QZgf(+SV*-hPRaSV6e;OCojoO_V
z{4)^o%Ol9#yY8}CQOX<OyaC@!{Go9Nh=l&BePjhxIW${HD$Oo4FyeZMieegT+_oVI
zTO0jSL&o`;h76Tkp#|AA+wlH7S?q%M@?2Q;hQ8i{u~N!fMnY?Q<<FxZ)sqSkMc;)3
z&7cJyzRS~*b=yhy8BThZZj{7HRB_9~%YSLlcw~ZM=$~B$Qtvz#uNF9;wLjF2ugO5c
z4G3P44@2qJub`h?VQhsw;ql;OsSWv(s;ts0PP!@(LlB;tp9oCwqJtD19G)i6lLU+I
zHT<s53*Mhd5)Bhon4cOP3su1Vq67UiY#^jKWrt<l+eX`WoTTxh&%F#8*jYd70Kdl6
zmg4q<RSy%<!nN|amE{jCt`(Y0L0sH26#$$trK+UpblU=Q4mWjB_P|Hdn-@3Fq4(84
zfz%fiY)!4ByYC4)M!Y|?bFqEbkh)EhxGvV|*6o)t%rB3w^R(I}gZ`aK;aj_4c+8{a
zeZW94X$Z)Fg2Sq{BxIA5lf17zn1KY^06ld}O8MO4(LEYWg5m$pkjUxH=GuNRWCkQg
zePj-8@EuYjos{to-2b#ZK<?Jj#0t*O{oUa6@2HPjH2*wDn7%c}Q&IQ*nm5FnJo_i}
z`gequ((&Mbzqs-*%3M@TF*n5r4&6Ba_c2am`IHAU`K~y{<jOy-SfAlyWq}gSbp)UL
zg~GH}VJ~p)h{40vTEtD5qnBeyh1UPy7Yb=}Z_W7k)gocy&#Zl}Lb~=x{+*28a|Sl7
zki{@LIeM@f_1`E7TB~@j%eW+l3%W8lwRHs+e>+eU`u8Oi!_N?Ry=ZP;{a4g7&lfZY
zXfWAk+WlWHQT<z5_9`@3-AGF5RJ|kZEn1<#xn@jDjyn^_nmsg%zzL|B(Na?3+_tK&
z3Hipy$9IFp!wth;%i}z*W)a)^Wm&FNloYLV*W{?#oSaJ&&hofMWE0)w>qfR*GNcTQ
zD+P>LSbn!Nodt_}1U+hi^`goWnbvi@ydpvVrq~z>Z*{`X9Mw%IqYaXb%T9J>8yOfF
zSX~bPQ-MGqZ`V~A@5C}cKO4{8Yq%)${&$V)IW;pzcSl}IE=}#~Lp_R>ahYLKRL@I@
z+&;kiS5;BW-}<0!b@(QPa0dQgjy;7nS^{uYlG9$vBVI&5eiwDmCqW|_-x(yJa|$Q-
z_L?_0b8vBRtSl|rMSp>*92kIdbG7A5aURRVr)$Yd`LcPeE*Ug~Qnd^rRYnBdEtgVI
z?9aV%J$tfWl?t(>sShVwG{!%^3{YjGrJ40{IW#PjkV14RE&o-8QdZvoac2PuUOO_j
z>+Q-UY?hj6^tzz7+~keeDHYViq?MkqS*goJAebPkTi7loVtu2L``_HWwp5{bteE=0
zDMNek9hG&U%+C7Xszr>~YnwSZ!#JE+8D>k(`2S~7zn4o~>ig{8{~ZZjtjr)yZQW>!
zC(l?>g6rota}>^*Hk~tPbBXu{A_v-=Rj`TZk}@(Az}u!ut1b8p0h=K(D9CO|8E&dZ
zLPFwkR7>yJ*Ure;c>AJ*H`aAs=7*nQ=p`brmNnwVuxB?}huJIzEp54Whx>3GKyKE6
z<i(mHf}M$$E)Jjf*G8+A0y84Wn#RUpIlwNKb`EyTb#EjQ2+w@DyBmNWJIIgb-R|s=
z!sB*E%mZ5i@@^Zf*II$5&MGBE_~GGUXeg{OeN<EwFE6jt@m$Lb5$3$Po}OOH8O^Pu
zn_iX+;YR$EE?B0|`6{QBl+>_u3_7!Kf4{JC+e=hQNl8jdiWs4nVmO>YZbyfg2(F#8
zv-itcQ%}O8v#srB@$v;hb+X&deuL?Bo>VdeC^4B%&0mn>$FF`sU3Y)Jc5-l_Ql;Gy
z7#R4gSSB?!m6)MDoxQHE4yq=x-<h0iJFj?@d~5i@aNRo@KH5+jibxB1Q}3a9f8r~2
z5N^L&(DBh~cby&_EYj)n2K*-lwtzAIcze(h&ovl|T5EHWm5@+tFjpE+V*_6Akp779
z@Y@U(g42+YkllsHSa~;kNu{Nwq2kL876*HK4bErkwp&0LBjCGMzzCEic@(%n+hLSh
za^-(PYbOg;(HMbkx94E_{1172I<8w#uJ!xl>7-D6Zr5as=hu6qKy61tN-E$4?}ux3
zW+wg*GebkNNGKmY-V#K${(%8tcuU2i39hcLR+?-=66}Z&ZKrYsW;`ddco3*f3=MNK
z{UBgS4jvD(ee4|^Y}Q(X07{EN07zB-ibmnVtq8le0szG%%MCh=0-@y9*DQn6VQ_7&
z14tbW1=o|TE?rzyt;SzX!_Wkhyt{*9kRy@rczAe-8Fz0NgVsgao;qPq{hQGu1%EDz
zS^(JlG7muFCDHKv%gMqU+VRvM$NO$Zyjl~@#i~u#_<ztS;LYeY8c%^d&dUN?<v+gA
zkkHWjJDWWLsKmW1KqzQl%<l?wun6Ci_EHf?M*!P1LxSnFSgo+T@{^hSyj~3fKA_uY
zXGqkqx3jXg{eWxT%lpek4J|9{Ehv^0m<~Z50_LH5EqvvWwkAxFhh2#4^z`&VzHi5L
zzL-vnJrs#=G(`E?Mi-&IpAcmB*?I?VI~0!9db`{HSSk@EC1o)ZF1v4&92hGcX_-b7
zLk1lp{5sZkBE$r6zaiA}C?>!s+klkPxgob6&loyEh5pQ9+3dk}z<=7$&M0t%if<}A
z^sAPZ7AE-dRIbo$k+eK4R^S#<Xb1=DP~Uk>jX;$BEy9X@Czg+1?lz{QvhvZ<(IfQ+
z234rElnsCamXG)?1D0>QY*c3OwBeFRfkzZgdACk~++{j2y{-S2{RM8Qu1xh$P?gKY
z`lE+Df{%~S7p6Zv92_Yxz|f)l7rtHgq0BIQd3p{;;Uh!{X7PAF+Ab2j(g_DZBFi`o
z$C2j<L8br~EW!$}7ZeggN|XqeXMH}=EH(S7@2K+~$u8?>>DD6<af))GRxFbV*fj@W
z^n@-3&utR+7y?ij)J-%J&G*hhLj#`_^h0+n<r^?&!tPg32ic@r0Zk%luoi-B{ZzUF
zk3t#9WZ`MHnWiw`#(lvc_+3%IDe$}s*Q$oeTL))p<w#v&zrnTS7;G(}Dw8k>ppq_0
z=`ojndW;a_4`1|6A;K32JNDCJ;#V6IjHnf39<{6Z>7<k-O0^R68<9g-&^8BH;UdCt
zC;oEsQE|W{A=cRqVfCA!*4)GHWYKE6KwVA_k?`l4N?i~8MPYAYFcKkVt#W009O^VP
zFx-Git$I^5_D$L?(H7ittV18u&dQE%lBo~inE~M48Ld`xok5i7F#Bey*PzqYwY8oI
z+aP9$7=mARK{3=MQmW`9c0_`BbHaifm0H{)^UQ~_ykD=Os7byEeu5XpW4HV=2~q~1
zu}#R<W#goSQxk6iMYa?`A`a2~q(G`J7|X4;r&i0%XkuxW-1GLnI`FI0UQCG;D-b8L
z=bVqs3c>72iCvJ*=6ySB+6Xr7Xm8hcKO&b1hc(pQm28|SxsJtpluYG1OfC7GN;6Mv
z2c$3nX54%as}7qj+1}4#dkTBL%R(f8hE@_zAK?=3)xg}mtn#!#_7`su82apujeUVs
z8bO-KJIsdrE+`hTyzS{n)qlSlBD5b4KxA;T!lEaAT5NGR0F1nq+MKPBztduGlfp~L
z%Bo1pWc>(x#SO9zYwe&93KAsm_!akZ-L?A`<<?effn7eRs6svu<oS9YPUQlVmV%tz
zYNPXQuJjkP0sk-%x82dv0px-q7zC*6;%d6SJv%Ml1Cu&0FVC>IXU#QQI6fuC^=f<I
z=3p|L$AcZHV3U)JET}qqXh#)xb8>NV_UKP$a$^$`GSZ%?Qy`o$fCXJF)g1r~0uY+Q
z99b?_gUtx6D;LY)%WWpnYsAF7-LKf>5cUTRWELSRCs_5<R@VaWn18m^^&$Dp%*=oO
z{2}VOytEb9+{c10+~3*)>l1cgXmzp>7NJWE2gi@fxdg773Coe23lCfb5(U9OIRH=o
z3l0uk7^xAA3NdvE7)jAIi;$P(Y>^c-#P~{!)|I6^uU`&!_p$dz6qGVKm`pSQPq#z$
zI*-Rae>d=O#piOQ>Ys12+0gNR+B>Q063uownWtxE#kjLsZ-;{q6d>8C0(@Y{_K=D$
zBikS?5w9Q>V1^A3Z@k~0d+sr$geFI=PW?6dCn==U_(K7k&bkOJCQgtHr|^TS=-wwK
zn}tgJQzKxMVIP@USYVL8&ea%*CGg!Z8c?maxiB^kMdAdkQ)H$R8{I2L3DHn%2@ChH
zwm59@0|A3b_q~;UAt;Qh)t`75{{F=zrXcyEs2ru#C|@U^FFF)WD24@sR^^m`et8k{
zM+Iv}N&*kr-`NS!I#dWGs;Q|tJO+Y}H>-eLQ$s)VL+4k;(-PN7p`uWG{@-z=7aaL`
z@Uup3{v<KZry9(fykm_-e6pFz<r9gP$>qui7Ede`T+gxZZ{@`Swo2DBKM0y-2?7pK
zr@MvG)~Ez~$BPOJ1-z?y)L<tg4n<`qM^aT`#pn`@JlfD(bsU1e0*cG!i3w!Hz%1de
z#1N+(OZTdl*my{0?}U7fef2|0un9EwAcx@r3})4YzX@uDNl_S6f6G<2E@-S64^3cT
zU^rDM!3_L{`TOI>9Jw`JG~kgdZ-9gz&-*2lEj{_XlrN3b$Y~;|5Jx+xr4a6C-A@c^
zwUC{rfX{iH8lgHTI1#0UBDtDAb;yBvZXi+kiI9rMk*BMg%+Zm8mc%^+<M1am1+^2V
zyZy7>HpOt~t9Gr#9&LmUL0Ao$mJ?`U;Q89hhr}y3ToPB`=BO~2NvzUnRnMgQ+wEL?
zC1O1^&pPD9n)Y+-tiAi0E@K{v2n=Fl00a=r?X}e#1O(Wh=@qro_$6WuAl#XLRxZ-0
zc+_6AT5I7Uccnkmq6v3&%4X<t!2X2~upJ2^b37(p{yovKO_sOUR{#Re7fIuuOZ@6N
zTa|gIzi2vt<VgisPX7|Jqf(2(+3Bfh<d+^VQ^96~98ovq+#h1s*gpS=lb(KyR#3Oj
zEA^PQRF?Z56zRWXk(Mg!(^4HuG$>c!QLq2H9i`$x!=Wf1=$cg3?k4_silJpOw~Yx?
zD(&qK*9N~U$f4~|hUL?)^p%!*!&eV{I(*fpe=MtcQ6{edQTF1<er@$!@0{aS=t`+h
z@iqpF77L(7U#M3S;W*<;U%c8vaEv+w6aKNN!2tM}B(>>)q@Paz!El~N+7FojBY9`5
zm5=exP_G=o7y$4cHsL<&=Mz$3TvR>luf1Pc;W{&_n4t0Tf9c^q$Q6#N8X5~F^7&jF
z?8EkYE!9*P)YmsRdt#bS2u^%G>aQl8-@|Q^YbbshRni1q5*Q}{2$!5!Hz_Fz3m<=c
z!=GAKjGT-N0Eb^Rkvte>hW__A@N4Vp&>Hv?;L9;La`dIZrOJdv2;KjFOvz4%pdlI>
zhks4v%`Tq*(N1S-V3e7fNzNCdi091ShzV~6#6$si;s+1U6aJHt=<Pz4j+o2S^Ru!L
z=>M({znhUrWvVPc-+){{S#lQ>tCg(SbH3rniSn*^(PDMI`=VoS@5oG#pwdj6A`Ka*
zoUy2=C}2u;ea6b(5|^+7AYUM>91RWauWmRUb`nCwwKfa`CnqO|mIQL$XHVtlE%JS5
zqo^>_rucYgbEk8m5*#J&82!rzI>Zhp@R)4zipKa_O;rXuNp$-Hql{5qe`*~v<q^H0
z?5lAPafdb<73OnD(o@<bgwY_8C!(;Kc8;Erv+aOXnj3D>Vft~!cB1&6Sg?S--aJ;g
zl;kX}ubMHX5Bop=abTuMJaP&X`|0p=bXa?#B;U*H$D2JRqy}}bn|*1=HpD7ah34kw
zf`S4(9`||x7XwrXc%b2P*y;lCm1fyqS$Q6q=76dEOtqfS`9X*#1IN*Z&&Th0cQ-ey
z<@!<}%n%4aro(Pbeu0Jk#>1oaumxseak9F)8W;#^O3nTm9lb`r<!QOS{l`_W)Y`$>
z?Sz#r*{^&WkW$3OGG-_JuW=l0CGdN=3kclC>T2IjVzR|_O}wDbrhPz)(%ka<29Tfx
zjCE6!%TwYJ;EoKexjh-<mhMv}eXL#*N=g_N>sI8<vpqin;HJ2`a+tv&?}75#+S&pF
z0yMXghe_Uj<%o=6fhO}haGzd(Thvp<o=K*)=IhtFM3zC(-As!S-d$e$x>nO0qynb)
z05Ohrrv5sTpPw(A$rWT}Vq!uCSvLG2$q#pl{`V4HGCni&cfK5zMq_QQAb!83DcCf_
zD%dSFrlr-utY2v86*dx$Jj*SL(<Qd_-#==<${@Xa!}x77GBR>(Ih@qhzj~8A_Uhn8
zAtewo2qn;`_Zk%x73o7zR%D4O6Y92z6HN>dANnNhP|f){PSFYbS`R_@L!#+cMJ5o}
zj*b0XLLRXbE#T#VTJHIqc5ARaNQ!+FY8UE>A4d06MkZKfQ&mMpW#A7HMn4XfFk1@r
zL_|Cn7uSe!AzRObq#kv9lz|;W3?ohVT|V+esPpa-kxqR8(lC-KOj8k__2=$0NGPaW
zS~5<FhJO0d`k&^^)_CVoZBrihg$16@-I@qNbqRfmmz9yRW3`M?V$2s{L}YRim1)}T
zu4rJVp=hML5Ta8yufh!hFerj7>h-2I0OSTXRSDed<`4@7H@CD@s5ii01AARAfMqzE
zL|<xZb+rY&I)NY)h(39ME`QuD=%8H!q!jo5<K?2O??*5SA%kpqH?Y7Os`**Ly^evG
zE+CTBZKVo~5Z(a;1ABXU2!UeI>C{a77FhvMGg~~^lH7a<800@HRRpi6!AQ8khml-8
zJw3pMRLIE<+E2$L_};smEeAQ<CJ6*U_xxtiYQ<nseU@egytg3;e;d~{0iXfEfdJba
zlCTTy>f$R1j_9Z6az0fBFj8Z3S==aqou>ad_D?x*oY$F50$g6nlSDe-530dzb|2uQ
z<tkN!Zh91^T>!=gc>E%?6wmwP4#-$n$M&$<_|b89mB?O`vc!ET3jdlelgmkv+Z3m`
zPitq{mJ9Z4W7vx4QDJfp3IOMZ?g6SNP5WIWq?_}w^V_EkBqL-*#Dr~rS-~24hKsLn
z(c0kSqr4$?y*jT6RlfudR1fL$gu`5?ax#+hYBv)q(*~>bSeQTnjM`-Er>^tZyAl(x
z0cc;0X}NMmHzAM-#O;1drMXx)$s3^sgN)}}3IbuUe~SObDluW+odpcFJMf+i4Gje#
zim<4tpi`wzS5(aG<fJU)LzPlFRwODG@@#J+z=EO2d2hy8n8>DxSHpj<0aXyaq|HVk
zb-6Dt1PM1-6*pv4ax)n%X*-58%XRzsZ+W#<(&!op$_|17g5jd&AWMkSW}_2-0&UNB
z$rS4yZG8zyNL2_};P!syU_kM)Q*ias95bR%7)ESNrb$3ZGhb<>zs0kEx|!t5&&vY}
zFu^&f<N#2F(i(2c-0I~QtR`YbX1ALEvjoseI|jM0&RhDMS9UX5JRaMc)f~+iq+(IN
z)^<J3KR_xIg1J8cC~1krh=L{rhN{0^aEO{hOowf*tDcvj#_=gw2!+(GFz>tHaEE;i
z+)A@ipvxhcI2bZajsMT&#34xHrR|F)YCEzzO~tVr=Z8UeJrz04ClG0hXwj}Db5u4n
zSqM96tDimpAb|E@w~S<<1BSLLfDMOpC9&3Zj}i0T5=`vX$Ruh7`b=ci(}}byQ*@DH
zDmOW@u;OUQr1uGQL2?+aP^1Zc=}hu9Q4)oMb*AXn*j6M4zBp3)LcJj;0)dzW1Psy`
zGGIxQ_NDcjo3K;a;#GRbhuZbg)*)7XX4|K>{V~YRsMx*+TQ8g7L}-vNk743!I1&s;
z8vS(1Bg0VPNK$412u_P*2gk{65~$JE)~<!D?PKd^>x;_B%(Po#g2(gk6L6}ge(g8f
zi3EY<T>*tM9@MaZx}Myo83_CAz|0R7eE~7u-`0*!eb7%=?2`5zS&Uk!mV!+3mVQoS
zf>ug@HULjlbSN(^(sBy(8Vt-*yi~6WZ%P>?%WUMfG_gqv1Lz<bKT#upVd6*LhH!Rv
zp3>^z5-;A(_<_l@k1`vVs0T5VkArBHfL$yZ?dv)<G<0yfRA=82KUc85j!G%$m4Vg>
z8Q-U73*W4+Cg8S}fhQ4TAbE}I#(g<zO6a5;v!_GRZU2o5n;O;?C;_mBS_&;x$71Ek
zSxcd9*2W_7n=5e5WY=~mwrIwLg4QX{URfUjsA@nJ>$ICjR9Jgks7q?}2b27I9MtMe
zxnRLR1uPN&{@BXlVU^VLP1QxBV^9NbHTNpQXUz<3N~oLE3@STDaqYxZ<W(ZbVimFE
zU_&tp9vHIhL{?GtMT!IG!K4WxJE`c=0`7P`Qu4Jb9lrhwIb<_?=98KD?{na4U!8N)
zGPZ;Dp<rMx7Hc3vh#;80SREZN9nBW&T>{TJ;fyvwx3@Mmyh(gr91;Taisp!>278K1
z-u!7bUsILXBv<e9(QM2@HBy*?hQ8J1B1$_^1`p|&tvpny(j~@h2v{Fg7J8S3a#RVm
zF$~bf-HmwLEZ2$1q9oYB51A(*2V_Ep45F0H65fYl7Yqq_jf;=pYb7=UdWf!}xNUOi
z*Z~H5j2xs<BEb(J(rBV-KuYz!5p#3~ot#;4Hx$GhJNo?mOtT$tp*@9h4m&P6%OWG6
zXE1sb=~o#{YLi4vl=!UaX=cHE!Bnll_iSKRR9n0Iy-ZP!0cCdPrQW~(J5I)kQFEx_
zR`kDe<?-=yvz<Z85<oHa5Uanb8UQtyoQ(koy97L-OxNfLARE@uXDZ<zR6E;1QSn48
zj2W3>l2-Z|fDh^HR=qlbO*2qA)NTxGLe&T>%^>9pb*rUXBcP<)JcmvxgNVZnnb7F@
zWY0<2<$9&>illl}YcvWD0pW3b*kezv#b654GH}uL5p@cNd)Qa0)kZTuvT!PiwrOsJ
z=!Zx_K_M!ML=6tVcLw^4zSu2GhJ=7c$N%ecI!1(_|5K$iP!_Z0LB&~-dfM|ZfKLU~
z$0opk7VrSdkz$oVDmyaYIsmmo`4|3>5f9%YI}?jtP9e+u_WevTUnqrLjZxq$C`=Ee
zp_Gtjm&QDZiy-vdrspW_`sj*<qO{FdYDGYxnfhnhtpm(+_%8#H+1XyJy<C!mC~`ca
zG9tpzoCrtg{z)Fz8Y%p@$Iu%|aFWF@`8-H;cSXt_!M{Kmxj>_!jh?tvMnj7f0^VpD
z6)kpT><16&+T=#p@*|%PLOgCFgUFzqMM>#rZxV)Cq!M{psmq6lkb)X8rC`UsYtw!<
zBDlbKxrh3C)<@n&GI+fXpsGD`iHu1v;g6<|jcl`2TbO~%&l!onQmwESDk}rplC)K@
z+(A`kWcVDi(_Vs77c$=MrOT)<Aw1$9vdR4V`r6*!p5x!lUP4-l_Zx%>xftXTSepl4
ziSA)%)f{`x0YKR1>&fm>TUi1qgr47^0d*tfL!4sG&6g?v*Cgt8jeODHfNllOp}QeB
zyCzrjGXn&rzp0!wFK+~`aMQvZIG<f8knO=P7ivHZ?q;>vt(A|=K;Uz?>aH{_LPAbn
zm>2DhTcIhZ#{1%Tax`CoO9#I23Ghqyvp$_IMv*C43~eWv>uJ<P+o>D?mZiMfgw$36
zk$64gQI1inTJN^)#mqt<G=RZA*Zg({_L{?AL^}-Q;qNumz+yB7qYn=SPC*o+<SEJ|
z9LlchiUeRfCZ9SFQ@az!C9aWYxAsAMc3!g*uul<Rb4sORjASTTW6NqIz!aj*Q>S(c
zRzSgDi%59;CFxyX$HJ2yCBKIFYc_msSsP!()oOJrZ#0$JM<O5~K!IKnpa%Asm2mH9
zS^46huQZ{codZQ+X(Lgj#@{h5YIVkYc;_+d$m&6aegVF3eH5cCDSh@WV)oORsO@z=
zP(Wg|D#K+7P&{z}>a@V?OABr&bk_zbnh=fZ^#v;xQ;~1<_xBU&Q+aL5qf|>y(QL0V
zi_y{6Ea(O?Tdr#=sYA)(qwu9s<-h~{o5yzZ(@!bU5yT{fo%azaYAK10PzHRUFn`^^
zyjDhg#U!%&ZLjE06|J2nELsiw3VOWT)04S0FM3uOxCBZb2talu3tKk)b(&uufJKRA
zrnaXI6_y~wE62Ej+((suc8lIt3yLnSgeM(3*7<5(9AxH%q5tRjlnNkHg<N|IW|iP#
z%G-MjgG3|8!bVAXG}!{WNv$3rxB~$AIjAJS^=luN9IMF>{vVC)CIb4j*T3^0_?l*=
zP8G4rUNvJf+`Qk<7{S7d&%d}(UaL2l1HB<>-kcGg0Xp)GWSiK_swAlH|B=FVWA*)g
zpvQBaD=L!E>O+1jiKt9F37z;3w0wI<C*!5Q4W#|#v>AGqC-()N{4IwmRk8Me-tq&|
z2lFP(aTRr2sx?&;9x&M1*#X(kb&YsO_W)5xVsU#x@62Ll0uJ}nG~s)W1tvJo8^p~|
zsL`Mvd%XlcHAb_>+O#Zwsy#EZ#PARX9EwkCD#&S=5D(GcU_-h6MAR~=tjOwap;8rD
z%~V>*>s%mW8a)MQ0_0n>#H~kx=I6*rZ*W5c+uR+Pwv(~?%<$-_fsi*7k&dx5BS&ps
zk-DT;#Pw{tW}!YJ4{aAx&(}dJzLmm)0w7oENUz6j%y}>db#ZB_hB)2&YcaPoz{VYb
z!X3?!f2@LXrd;W)&0fZEKG5PLn@mg{j&&YnsM>s={9CG6X5ZNv-4jy}=qhM7@b*tn
zHQ(~l5C<VFHC=w?CUpE|wNc4<eLUg{9>&SdJHL*P{yd=5Fagvjpbz<s<y6Z=w%8e^
z{-ulzLUcRFEjoT2UuVA}3fh4VQ2*Dv25bBkOyz_l^8g3gu{!a`G}#kqZ`uCW4rjG*
zW6n(<G^;NF$*%p^xDM2npN4Scd+uBN*OiQfp=_G{e=J>PSQTBnmhNt(OB$rR8|jkn
z?(S}oE|Kn%5RjB^1SAD1>5}f|EZ*<%%Y8Wl?%DIiT}$aI29rNPgXVS1{t$7r^(p6=
zZ>-mUXUBXh_c}cD@e=dD{X6?;YaSQ*cylQE^~avAZFx{nWsx8qFn4jV?`r&ySSM~`
zc*1~mwcvbR**^<0UbZO6p2+#gHr~cFcrFQcgCU*8u=8+l?{iibzezv*F*>-gMa!;?
zQHzdNOg7S*<e0AGQ{yGDh=FZ#zr$3wXo`oIm&;*!VRfoN3}qGc%j?5Gpo6<PIf*nK
z{rk7n>T0=p0(X}be4deo-{`pb11<ELmANCuhNGX<-iY-Wbw`e-Y{=Wp3SZ4!h3BxM
zB`ETuqP^}k7J&UV$2H6~%xy0!Aufm2%C>kc2H8B$8=&xQdp>|wh!b#Z<5}Dpm&uN)
z($dnTf<BjEfRc_U0<xY~yc>Wd<AYhXImsbcI8f-_JLz0vyzJD}MzG>(8T#W;%cnEm
zodmwbGQ0qo7qsliWM0=Dzw?oII8Uo-&Z5<&zl1m+EOgxSJbzjcK0{ZyTF*JocCoCq
zNxpn7N>V~V)VpNA+Lx`UC?U?xlugRm?}cge%dEehaYinZ^6}x*BM?{s!2jmv<{T?u
zb)>AK;_s(ZV=1YKb1VZ|l~{mn%k6mh)S@u>bJAe)*X78WGaHgzqf_v+|D0Q?1_bbc
zr=)~LaapNGIVZ5l7P}4a+RWFWfYXzAi(d5g^yK6c>O4zuK}ttC(c8BVJ6VoGVE5DT
zO#oxcJ$M!HbTv=NX%nO-e7hD(r&j8=Hwh7Qd)$7~dAkuyPEPJJjYp4ko}!MzBq0mN
z6;)N$4-eHY@SM{#=)W~?y#4(v0t4$C3onslVyQoW^uQe$z!HnZWkT0aPSc1U$`ugJ
zdr44rND8GaDORo4xm=f?Z6xs3h;ZHj<mk)u1Ir_c%sUnqG>5vtmuFB*;I~!a(I~6P
z52Zi}M_JeC?d`2Y`3uRtx4yo9d~EP+Och%{sE$0D`{8gcg>+}WVqtMHqy#x#EA@!X
zf^`&2Nni>rUjXfCtaIB}1?)6yUJuK+5ywBDh<MMzHb3+WoDq0emKQ+C=e7sPjV<Xm
zQDG2^2>&IYR2hjXHYq6yY;ECk#=bAlPdt@?paM0rz6Xf$LrT7IJE9J1OsIlRBSUIU
zRQ?Yfqy-qvr3Op1h2!ZGc^)uQ1p(N+Cc823gL(u6Ro~t1tq8_8*Mc)R1hG8S)H$n^
z$`pxVbS(erVQ&Y;W<J-(mwcq;X9hHmq<%@zo6{dg*q&(kP{e#PD1^!iTFz@dP=z6R
z!r!fR7iQz$y4~jU)cK+%mE`crqJ$!JV2~=!7nV!PtL&Z{SL$ckc;i0L#+O(fjejz-
z+OuJCtgmM_YcHqe^ZCqJO)it%5tGvSf{H>S=NQi-@Hpgsxq-h2$7w?mU=WjUDqU`X
zE?VJ~Lqg*A{)3$%kV18-Stj8fFDvGLuW-%i#Z6AJ`fHH1)8rt&Dl|w*V6_Q=l7dxM
z+!mP@0c;SW0Q=3t%8K?9chuTs^jaB`xdvbvz}4OBp`)`1=wi@dm%!Y*K^dDood~nI
zw8U*QEAzX{WjiJ%1#7q#(#|q*+AKorc3zn<a+z7HI=~8{wm7H|^Fo7l3Ovg56>4xl
zs0SeEdLxqL({>@ZGFpO(`8^&%Itf7B!R{KdW-*b?3%ujBLnA5gz-PP7VTDgZb#o{I
zPI7x|%kEG8_GW@`g!E_Oz~^LY`MO|Snh&4fOr(tW8U{5c&IKP$<OwJ6`~%AetuSyI
zH8*olkJ^Ks-r&dW&EG(l=hpS@pBdsTpk^UgKI|1GY;0^KUcbUe)HD|a2FvhYtww&y
zxa?oj*{pooVM71Fe#9VZH{56uzz~6(7w{!QNR*E5a`5B5%NMc+L)@8pwULz+lfwNt
zXO3e9H-e5+($R_HE#1q!DUM5L!WZ0*8zB@Du?EYjiJV4n26MB?s@lo27fK|g`$M%w
z&qOyUVz<gr!Er;Wp8*9ZS?h=Ua~T8V2+^~h#N+ph>YoLoVTBbPJ0%&?r+PxMiNshs
zs21)s46UZk_yU7---=rAZ;_Iz4$d2tl}SyevJ6yvdt*T655dMtB@)xD6h<3P4qNWa
zQN<A`$VP8i{DYgCt*tN?H%&yQcmf;+3OSw_pM)0}<{#^f`#g@95mv4*FWsD-MH_D>
z_)o}kP5Iv6Ltw|TL58ip{f4x0HH{_;lL^}`-ix7O^=2en&TCBM#kR(?{>vlXi;j-Y
zw*`6>C$z74$>JtJaseZHeK3b4oYe4BMKn~v(%k&J&y~%>BRGtf$8sWjh{X59aR}y*
zJ5aiQoJBB0tz*-|^#y06!!ZHAtCh^k4hUVy>Tnn|KV)&CsLz5{llm{dwvdXRo*oST
zTwJ*JVw^q3rz|oLPT^KfR2n~j{mRgZl|lJKKG9<Lhppda&0Q^7!2=hDGk>4m`X6dc
ze8Jtt@N`IWa=F4=oo~r}ZkTjQhn99cJct2pUP+|^vYV@g!t+3_zqh3A_DnSXMO3J>
z(HpM%Df0O?w6F+%>R<FGh2>&fnu8u*s_VMIb`^ps!>6g$+(s;X9}_&wkr9lF_1u+^
z=4U<mx6hJaC8nm_4)M2lr#|CZ|53f)GCi*r7iK~ixY>|*dlncNoSq$_;9hCV0jY+#
z^hw2EGSzhG-S_2OcV{lO8+VkFI-VMmB7|jyCrZ9D)*;@(OfFtEnl@V*_xYG6B3#-4
zsRXC(Z;8(^K0hZWBFbJ_qfZ3vB;H#|CGr_U4~Gq`L;>s(vTp(qn9V^6ghL})t}}+}
z#m;O_gXX=3u65a;E}5NGm*7gr@lMHrAxa^Nrhm5Lwf%7QNskgL6dOQk+-AdxpaN4_
zf=as!;772jp3SN$rI}p-fK*B7j*16a1=!_h%T<Q+UVlC~NLn%kkzeQIM-kpcw(FC>
zq-QLMfp^<@RN=2{D6bXTG65iKBg833;hC8C_oj-R7@nV>hw+?cCuOYO)eCm0VUyy&
z!Pxlm-DVEbUJ396r)xb87ULl)Yz1{#<fQKZ6kX2q?R7<EU<m8$i-#hVe`->E?>+kI
zIcOq2ajM@6q@7liMlQ|;T7#;;49rZK*t%}L?$-oWn8<^BnPY-`+yiH(C^t?unP7H#
zd86%+!=O8%LvcyTyRXQO)U>0d{QUN$ke^P}2n9ZCex-nS`^B6Tz{?ZWf1KlKeTm-|
z5JyBjyc)~_v!UAaTRcLUSon-|pHpXMD;5GEfc+ng6OK0yi*f-*kAl6Kd7$^c2|=sg
zs^B9Yr6#o1Ckn`siHNwM1%bw(hZNH*_<{T-PMg`ig>OYf<wk!^FoPIEe?eH27e~-m
zf0g^0llT%I7S?Spx&hzc<1vgr%3^yV$4I?aOR7$;n9T#>9qJ|G>ta3>j)X@Qn+@pJ
zwKmFD%?7Y;>9@J<ue9I5k@~%Pkp2M`H>E#_K%ICNgB+L`)1yW^st(|0pk4bzh+Pik
zCu<Y$O$YaEbd{CSYiz*Dl0X;a=I(AkU$X*~0)4$edT(E;C$(?QaEocS4^EU^@%&4^
zCuK{+IbkB@EeUmtsI*z?>e~1g#)O%PUwW`S<Y<!IyHzFzsqD8=Sb+QRZ$<TsQ}hOB
zg{-MvENGaBh?s%&na;v%;{DdCMtAmVU2*-kr0Qy`rcVwUYD0>bLxlC;3M<t1T9Zsr
z(aJnraD5x<D{MVu5{iZyB-!m3+eI`Fb_@jQ3y6-olS<NB><DNmt0l~dzVD#<Y)g)g
zVOE(cn-D!hq(umY?p_L%g1*!%QQ`i<ora@6+zST62Q(D`|2u$<;cP>a&U{5i2ZJ_Q
zn1|NW(~}1`Y%RY7v3l(5f^9xQo1a>1htHcn(MEJ*H@8a?CJ~o40S-v*^vg7a!!+^m
z1$Z<xY0Bi1RJ9@)GnZt2k-TYQFJE6$evc0mOa6eU26L8JK|+EOXqu!zG$rJJKU=Mf
zr=9PDYZgq#^<9TaD-RY9PJ9Wnk#={5ocIw?6M)w50_j}k=YTX)kQ!uVb+eFY8Md3A
z>0i$3@HdX9<|lY>ILgH5B4HP4TUGGyhyOs!hVlCzy5F7xf_jXXBCQF{`09Q~?gxu;
z2XlB(#hm(97c>V;8DBUGwy;&S--;&|3XDb{%W(Hdc{i>}C!q#B2v+MTSkpE7XI&D7
zL}UCFu}oA5QRoRAN<fkmpU5yIOJ5SPOiqCb#@U4;qK(!RT7r}M+p3m$Ry;z1^}W7F
zAKQupzPlh@`CU0fDmy671Spy(3wVv5pzJ8STNUYjc5rkI+#E#N<7&dVOXI!iW2GNP
z!vq5#g?mZ@QxB5p%oY^Fc>!Z;w-OjYUZF0biCA&^Eo;XFv-AKu>YA~!K$%EB{gq{t
z8p*zc7-9WfbUZ_Oe)W%vjrL2pcRckjRNlL5s^3!#&{@qA^slG;4jzozvwZ-&GrfpL
zBCycu{~#4d2-Gk7-t>|+F|XrBQu?d*gN|;br^k4q&X_E>dDRCAa)3bk6aG4AUSd8M
z<J`_QMbZfreZIRno~c;t=p2~%`0%*>O07HtDoE2px5<_M*YY)xY1C`_75TJ1?i7@O
zub>g29L9%UuGC;q!*mTGseV*&(on_JZ<8SPqs52eroAAywe>0P`gl`5QBuc>*hNb-
z_oX=>0&1`C#`+)b{VwUjoIE->?`bBbZ+}O@#RDT@=IM!>6B(mQ@+(t?EG;>u^{U=i
zbOdB7`*$(n<Q9u@x+5XJMlVL#Ns;EU4f4m2Ta;&d#LI0RP~RPu;I7sDzVeJtiDnaD
z;^Ha&4U|s8;CQq?zb^b{y1&v4Lp?Nz=rA~dXXWr037P0!S~O$*+^v^$x>TI?qPou+
z%(u_%kC4>;mOH!RMd)wCBO=6<fRyKYPRnqvQbS(L@|e>ILrM7Q5+rLC7=cABYD*cO
zGXMxHJHbxyDjX8<#<a0sRO%Aoc$2p;l!o*|muGG>ncoAZ&YPnptp*DP8SRPaplG0<
z46TDbhACjjelu!>qQLx8z6GXxdV=XS!xa>So^qJ*aHlWyY2FqVnn#Pdb2I!r)}6Sp
z*7=LdeUgk06C<M&EG<x99w9x^5`wye&Mik0O;TtS08l#*?%}bHQ4yelhX0+-uhDXu
z85T*QP#FLgz-SaOZa@;G!>lTXNds5^4G9|79MZP)v=?bjAQ9KCL^2katQdTguKR;3
zuw7V@B1cxis07C(uJ5}uHY%vnErywZD$U`zN>r8@PzDG-9?FT>Ml=LcHpy~2MXJd8
zr>7_9L9lO9E=z5{Jnjn5{qkTV3bq1s=cMEw$}-GGe?45gazO;r)d5{;z0GWX#*k_x
zYVo#TQrO&Ax;?n%2)-zhNMXqY)ciq(;guXd4{`~pwaU(?y_Z$&qAw7^vlX#x^yo{j
zMg57T;U)MMhs%<6IVIG0_fc>N-OM}{)j3`~<WylqIsMUMgt2sjsT_8+a$h*HEPGyH
zQ5t^<driyNM!_;w_t#@LUhfrc_&6E=iiZ@AUOpQs8=$q7p|qJ%M3$jYfqD-Ysl4Q2
z!ddzQ&neWA*cpP0YfCTmON45Q#l|OxW0TzTOw0YLU~FvYh^FT6;fu4=g<$tn-A0Gi
zDEm<0t}I67#7YEhugq#E@_mb)eR2-)h3@;)Fl|F_CR;%-;K27#tdvqjILNq=Up=sX
zscx0W$H$3uj2Vp=9zkQ1YfuGYIEc0q>11l$p@EK$Td{$VlI#SxnPn4DO9StyxEu`>
zl<GFb^0!+oI)Z&+`_{bb%{c3x!Pa5!^hO+VUC=7@x^CvAv!qpc@{qLu2U+yCQpjMc
z<wPPpv!UYCee_%wduOSmg>u(3jZdvn3s6&D9<Mxrk2atir30d)j=)v{XjibW=mB;H
zEJNV)eX_edq^WElp}CJ(a@g?`p=bk#kv<XjOO=QRJyM#O*QoRu0rKDz^sw)t?beBs
zp<>~PVbpus48qGl5=lt75#M**wpgD!<$h#u$8lF!I?i-xZSCPP>wWC-;oP_f#PbMF
zZ<G;Og`2FVD8e2|18&LRd3=i5_9)h8SNo`%x)l9Y@u!EJLq=*g5h3AdD{-i6`$NK^
zot!*YU)imrz5mcs$;ElUK%Sf=PUZotVmFIuDSF{?BZ{1|gR!wnfn^;&nxjKmGmRu4
zj5Y<SfU+jSMw!$A+SYsgLFl({>Gvh&aB<Cy&y1E!!f^*&g?4zQtSQL`Qr`7l#h~a2
zHP(h%05B|Kq#@_S&M48Yigx)ef$M*G(^z6q&$LS?``D*EDa+Su=yp8X2n^_=bCxGh
zV>$lcKB5(^Ve|iL`~HQRX+r^JL0Y7a|6y7UIn0+NVr$Kk?BMk#gc-lSkJ@J?p|JcL
z12%)q5rCK>tC{onA^gn&|Lk`qcnS1QY<6W^u0EUT%|B24x&?$;^04?Lrg}dhNk(of
zT~Ii^00Ych`mSz?o2$t=<T$C6q@<o+)6^QO;tL=@+7@PDU{Ka~aA3wF2~`PCGEY-y
zP~!J=tziQul<%1v?C?^jy{viwJi%ljDEiQwLPxyEL`@Bo9hnTIC_tg&vWOhoLmMrL
z?C=w%BhP2M1p9!%g6>M5Maux8dMRI$*IQBen_7j-wpKzk<9Ma70M@)F5Ow|*kn(on
zAMpU37^q6;;z?|Mo6IJxmE)Lk1=>iVnOHavmkAKP?SYh;#TRt2utgu5LYsz_uy*m*
zNwmpAd&(=9za7j_2__fuYgOPQ2QyqcyS?57)?@xL`Ozb<^8Y!1q)aS7AGvj{>CmX-
z(z7G&Nd9-DWE_C@i=F4ke{X0+MSV^WaxKruzL3g9t=cQnDV4TnP)p2^baZKeTy6KZ
z?|&{FBXx6!GQ)B<Ef@a(-HT6^z5XElfMgMPw2)zo5LK!vGq(-PU4{;c-+BM1{&G-1
zubvd})!G(~17GLt+}t234*Zh%8e<+MU6>8G2AK<jV_}vS77?F*uhFu9ti)s~g`9f=
zTs)9L0sJPv4d~|$ZU#z`H)kPbvV4PkSgD&B6y1~8<xE6Rm-~Kb!mK2J`Qr0@*xK0W
zCfExl%*)3I&5h{n;zGZ>c~|^|JPx*h@YDwGj1d5kO<w+IP*4yU%Yv>)``UwgTnhwP
zSd;>06*@H=KNydkbkhk>*~xy=oq&xsOh!;?(H8C0HV6p*&PmW_;p!eg*sEIf6ome4
z!^l{^(QJ{P#}Z~~u2Ep>%y(%@5cxUNcFf&C1r5bJKlEKi+j$k9s-wHGlgxI2K5WsK
zl2Qi3js$wz+O}%?vV!}q6eYK;nH6PbvC4QUBzD@gC>zOwAHNR|?eOq24)%#Nd3&P&
z#Xo4Uizwz(yVorC&Ghw5^-PVZ+?{M4rlw`4rRM^nZH`N4GoioN@p5Zz*Yo}9df)jm
zP@Mv9CW;va85ue!mBN}RD?6JA7grNBUcrqX=2XA40X)E(*w2)Xhaq6067fs|kx6n5
z;`#W<i0C$`(E6Yd0fl)Ea3k6|(a$d~`T(c)UKp4HG|T@@(yTnBX$=%XEW7LJ5%t0Y
z2Zx@YtamGOPWh}&6$?!{m&4dw);Bhmr3^4cSs9jBmnrlL!P>;au_}}rX6f^VCEAit
z6`7e=%cSZI>DwrTcACM>*|zEw$ZSQ)83{NabwWa`DOt2744PV;`zVB2{*EB{*<A<u
z_`!1s3i^Y&8iaceMMOqM!Z`dMV`O7<dAL3T_cwCcm~xDHFanrQzJ6u&_4VD~CIiPe
zaY#HS>Ptz1Zp@vWoD6I{{bf89v;E|CR-awxw*IS5aGR-<4UwlvJG|O|YHDhNuMTkk
zy7O+RhIe*=K>Tu*K48iKT^{JZhIc2cZvSm>Ygi~ld57YGA!r7G%(u6<GK0z6Y<ZzN
zWVoUu09h=V$*^Pc7bMi0miB_v<C4L^j5O?40fEYXciS02A#SaVeNAsiuS9N2%u%9C
z8?8*v^+v0ZD2glae!mQiNxnWlCQD*Stv?rb=p?%~>wRzfBq{gU8yY3*-#ot&w*LCn
z2orZfQNC#Gs{kC_T$E+GsQC1Cb?M%<b84#&%J26K^{VX4l#7f9DC~bO@?Vzq8VaGt
zW7<@#;l?S%e>FF0epX0Ja=4pQ_py=ikcsm4;Mx!}&k!Jb_dPzAJ&!lX<B+U&k)lLi
zt}(#R>-eGwX^nG|5L+aY&Y60H&wU@1*hZFkbk7(P)=ozVdK(&zdywoCwTKaLcXvmw
z?0fia2zVH}fGY`jMlfrzdb9aGkHKpBO7!KPVM<+v6*zGM7(ZAd0-1oq8a~+9&+ieq
z6i#n|M(Gr7jhE*=&(h?ivV3)VuY31CP(Z=}iw}&(u;bpZrIDj^vnMq2kriXdCa%P=
zp7R)sD)e_-i$Jsu`1fEJqmiE(EYcECf+C2zH0uRI&%pXaT%Zhy+e!6CT(BRg!MdU!
z$z6$U1NwjmGEo`ez@h?(Dm~6*)c#j<T2t@=%sP$mR54Y+wj<vC9iSWlG-)Kj#->Wc
zu>rbqAuEGORAS|*DwsQR&Sk(yfbZE3#1O8*Y|eby44^kSz@u!46n{n`<Paf>{4%WI
zJkL|)jFBs)_D1LYM*K7yzsKT5sB$~=Xg9KW)a50>#q(#4Zo=5OQ9dE`=YZ!eJ#jl-
z?LS0<Vz(z7$@5hSZBi&pw)1=E)b;hHVP5VD1{QbPxLMEPj-~0<Y`TrmXHk!g3^P6_
zBz*o~7Mp+BOwTe3qb-|yJ`)`+$tx-z2tAz>6CF-u7hI~#{Y0k<)0f8$wzW6#df)DB
z+FoAO=->WX<Zyw!Apo8BHyicb<H}iaJ?+Pf<~B{Hf)or4j$_WM;{I$UVW>H6YMM$e
zdn((v=NC{+RiWZviwbG4X&izla!a`EUBK(-kD=%hd>#|X$m%%OQAu911Y1A@+WqHG
zD<~}B3xx6V``w;aYE(TwJpt*Su<OnT3F}xq7Kg<~IvG@zd3lzOMw5XUF8jq(x0Y|z
z+iZmCeUYQTfUm7Z=obAHLyX{-zzfssUFJ&<w}o|Zab0ig3b84fB&o=xZ$=EsOJ>y=
z(z!R$51T<rN=izwf=i>L4U5FAfr(zGjz%e0Z$irGB`O(IG<MTLWGe(B2s_`agE_hT
zgoWQzQyAK7U_HP%EfjJy-yDp4c{&LMaR%FOin#Jr=r*C8h-e`^#g;ms?`?_T7j;{l
z$V2vk3@l{i`Tie^#Ee#ro_3uP#G>PJ%UOk5nGm42Pc<zpXt1&4LLWg1f~$qL7y!i(
zqn<oga`UhM%?c2Gpck#5dj9pt`&0JV)RJ_o-Xs3+rC_DqdH_u9Dfib5UdErctIfV}
zB;8+5{dlS=H#N7;Ocp!oQlQK&(i&WRynE-*8+~|4dt5qDsbS%|gK+NU%t20n@Vl>G
zoBaz{-0xaSL((s2aVws#Y`&Kg$zl)JDs}?;38gTucDu}Dc5|c0rx-vwK8Zt3qfH_F
za9|6`>(8!mIun>%))y(!)@1+I{lxfaLyWXj39oNpZsLB>VfFEl>xeP^@xw?ouPyh-
zJR$R^{UI|;`CsvcnGO3L&D9B{P$?BccZr&{rPsu?8x)+V8vZweM~kwC{<5MyZtFz6
zydS??Nv>mHyT*OtY|cEZUYtp3Bs4)*hf9l~7aPx4hes*s0~#7&DIsqJoWompA|fIq
zHRa;6Tpz?|l?l`!jO}GOwVVpGfKzh#T3f5&ppS!3=;mT#kb2a9FxM2$waN`U1JW$5
zwN+~6!->F^7t8Sta6)X{ftOR##^1o^3WS+BI<SSvQo$@~qGBZy@#|zSwtzSob_WzE
zd<IYmRrPD0Vhnc!AoQjWP59M_0G|k+eMF6i*8znm^wbI4h5s--x1heg{Q)@LK#UC=
z3rj8V?cfYeeBng;(B!f`3YfW|k>0*OXM<%VtCQ;HRmc)i1#pRbnBA}hXbO*kFE@et
zO2s|^2;ydMT4!sKM3TQ%wC{D0q&LPh@fcT?OxJA2GsVsC|HAmagP26BFS6?8Sz7sL
zQfQ@ohD1TtTZN}eqfzX|TNZmgrJsQeAD{5tk3{qQO?;o)WXgX0*v}&PGQKoClhv7H
zj)f<kLZjTb_86yYb%(gvI)j-SJyD^X5pb{l6zDj;i&bu+aKE*=9FXvtSXgQ(eyUgU
za>?fWs;Fp)1BOtiD|hu}l5Yc3kUfNm7@j)sg@vCrF+_ZCmad8^%j&0vPOA>PqmcWF
z<eKAhERCUWj+*t6u$d(b=6JX;*1~;1!o`y>=U^iyiUTtZ4J=nZSaJ4PIp8ofz~-JW
zsZ8D?hAq}tkm7%T34tUARbX8n;S@t0kI`%Nh(SbzjMkuq)IS3hDDbh{tRI0+8_+oD
zoyEYHLO{R8O@I0$CwV=y>N7?a?A|o+<#s!K0}M4%s00PfjxGS0K}NB02`s5(B3;*N
z44QWNTgZRZf%VYV1oeZ!Y6mQcp~~)f55v)db+wP6C}?P-PTNcsN`lsr9|$zRFx(*E
zar01So{+;T2+^n(m{JSSFQfkj0<fqK&j-O`z&kV~#0lGa_vjRhj^<n}k+%FP%ds<f
z*9X`qK?a!prv&_%tB;Gvoa`)mm=vx;`cNs<i|)EGzv9^b-qvo1{o0o+{yf;AtdjeM
z5ZWBoACl8fF*|>UiG8UNXX)vFI=<RsM0VqJa`LcL!MR0Yx2DlK)Z2zW_>^%>!|{jQ
zAKsA2cw(FijWWeQEA4%cjJ9nt?0oG}Y1y6b!zDCvfw2+DB`h=*PmlJs1_jGR?iaY}
z44R2~wf7e+j4%3VY*M%4TumF5HBu@@E7|cm9KU~aDP*ZEcpsfB7Ip{OO_DSB2`B0_
zjiJIzFe;#47w6n?lKg_FKA9?l>#YUkI++DU&i5hZ0yN=Ablx7o)tB&zn)bLVH-Mog
z6w-Tof`{^XW66aB9+jSp>lz5KWj264ckQb(q;P~X$r8GcqM~B>?@IMGHIMU5Xk4Wv
zGy0DPAy>H|)uV<5vb}$3NIU?{dKs~>-yqIuS9jOaK(Ci~P-{_ETo_$<3?(Gvv<5M^
zq<*Im!=(TgA64&KMluXK$20I+|2}W<@ObenDJIxK1-@0hd24-?&#~1^D_5sL^-=OV
zqUG)7HIVR=Z=`d|^3z4>rr08yg@R9Ih}I$~fr`z5lkPB^T`0sH#fYzeLtItQO6$h{
zaVYCW8VL^o>ECUrF@<DTO$5uD6g6@G{K2U?g7&vjqgRn8pmKAW#VnKBiZu{2iTjWz
zG(6qy8dB28WfPAg8XRscnwv-Z)EL#~UN*<DUtyfS)Fgs$(v5ZW_lKy0*u~M&{v^4J
z>%3y}=akZ*wKG8hj&Qk>*74;<t^rx}H`=<fKy|-k5<pT(aV1xSohW<~YZ)dM0BqCj
z7Gok|vz`9FyxlO{!8Qt?S;-k0k$qQ|mt}m%T~M?@NTUI;z^Rgy885R8yT`Znv|(ep
zxTOL-n{wiKvH(iVHFel!%N4!_G*V|fda=Hkc(X6ID!CH<My&_y@>d*eL|i{=O<bYi
zC5^kCC|>#WuD}=ICj@jZWd%|@nXh0oWX>DWS2Z_VUD`i|MAmim&uxA#YhV_nmZX=V
z?vVn|*7@04aM>i18x`+J8Qv=>SbEw3#1X(l5*HsYW!^u#VC(3ZP$eb==aWwP&E*?1
zMLnRN0M`@<{MLX>x~=fswt_&6!F4(W4)PU1RRb>xXzBmJFu4c4>F>eLlHqaEG(6EH
z()pG^7B&RoJ(a@lgz8Ih>k9#sW?;~d@v#yi_;`pfVbfETO-UI^(UVD(CTzb9_&@k;
z$6Q_EYe>Q7@F;Tq4yhs{8Xjlz?4RMaqUYh^uB$QPm#m<&E-8&l$~2d^{*8R~cJusM
zPh6-oWMO({V6WV02vW6b+|`wq3m4DMpFu+}x9lA+Ke={eG(0!lgOqeitDLDv){)t%
zJPhGQFT;HIp;|)&iynMvMp9KC8J(5N#M8$%3hCzpCg9K}d2b>CM4#LFf{f;1egRa&
zJ+N06EWpF5U^TG@Jl+Cj{inCwip(}fP3%oyQ5B&ASim$YM9##7A~gQ{H}1A{V`6Jd
zc8xf>SRn1INCT9sEkqCA&)P7QnI*zT0ZZY6ehlnW4*-RGuQHXnPhb{c8K#>rt0;F}
zzO53_4y5w-pyVc21@|S6(e*_er2?%DPM{nyllU}n?*LjCmXRP`*yqwJ5hYb*A~D!;
zGL@(h`ulJr^07$0KA5P$hy<Jr3bcsNAvLLuN`rFVk`=HAL;d}0$*{=yZ%|)9xPk%#
z9w)1U&KnZNK~5gDuh9KB2%lUodqPMXy^*bv6iNyTr$eA^-<imX<tiAhcUJGt*+_gv
z^D`irhG5IF$KZa2|Kn&2rm3ry`UxX=KPi2p+rbf$oH8(qdQ=3Tnk~_~$N|J-TlO27
z!HH6rKTWFC-q9W)lH<y@0;Mrgp+A`@`=e|F)PvO){J(rylaB4;kw2a8cJ&QbF9r=-
zLpmXSiaEuO78btVe||hM$Gr$wy^B;q!Q*qAwCSTs!ov;!wkMsI!5tSkGdLj0y7z-O
z{Z|~7FaNg*^JQHaX6BtgnYhi#Bf>#6N{ul}`NQ(q{tE8bM1K}`JdZ2ol!}C3@{^Qw
zVIn6Zu1pRTzaouO&^ACTrl!$}Q_0CiY)t?uyYuqfSX?F`Y#}L!QH?kr&lP-I|MOSv
zF&=%-TQ0C_f~`bZ|CGQDpCjVb0X$ZKz<rIXz%4-C;GMk#%7}uh*E*TX%E93VjN|}>
znhPBNlnh&v#cAsS^!Q#>!_4pBV=T@AXW}6uDry84V01}0ym+WvL;*qpOuLV%SYB2T
z$>FF*EJX6WPt}>2MRW!l4tvjYPu}fRT_T<8)J#lsL-Wl(_cAq2ZwS^jPfTSX<yxFK
zrAjx*+>CaO3;_@pRV@IS$N|+FqUH9N8@8z%g(5c6?IMOLQhXCW1PIUIQEdU;43Lgg
z^FLsLlx&#=_Tb{js6Y(TU`b`TGLfuT)z2TGXs2Xmt^sCh9-=|CzrkvzEEFcExEnOU
z#&p@>r`H^e+oBCUq0HFw3Al9fg+8Lk<#R$m$V_$rIq!9nLTA<K8`yL!hoTt58C4@g
z5^&o?SDdQW<*r)+NbDr!UMr|aJWV!#JqmtAiJ}IM+{|`c7%>4Ir#Z3(*-l-3eHL%d
zMD~b59lC7t_`N$ehT_#hN)c9Tt0f%VlHH<=`@c$mKPJx;3gMT-e6qJ@0lHjBh<1PE
z67~8bC_37)lUAzR{fuMm)v=%2b5&I<Bx4V2R>&-}8Z2qg>p!oyyQF>L7&)HG8j~@#
zut4IID2#x9&!N837PlRal)rHJyLy{J<?FBPf}Eua{f9$TV*7Za!lwrqh-RzInF7<`
zl9D4^Q_+A{EE?_LIT2s`#pty3{rxXJ;;sUMNMmD%HIh@S9Tdt-ttmAIQ)4e9jGS=}
zdcTfmwUNzCGHk~WBYJQvDsX*0f_qJET8M<5-W*U3EnwJ*zBi-QeENe44M*B+{u6ia
z-6d%3xbslkk(Q}_<hMvzRzN-lq9h(EO#(lnKLhc;n7c}fVWo*c5YL*%T8oHylnQ!3
zz~z3fD|o?%u_Y3Z;gaC6g|myI<_p}ww5QAZ-Dz168u5oNmp+;`r4Tb2{x+H%8HMOR
z=PnT_sum6?4ex2#Q*7-?E>U(vl6~lJW>n$Y;=|~G5XYu>by=v&#-5{sD(G|HwNA|x
zSdlU-IKJ4aM`y}c<_&iVr=7AFZG|Z^;%8?+n_V14xxC^y)x17D8JPVt(MZ9po|aNH
zrRY%M;);~V-E|@AnE5MuaBMuya?qU=6R+>n-JMV5u-F+1TaH8jT&R!AxSr5nwSTYv
z4VxPpo}2qNqTn6R)n3j+*ROLTV*6Uj%z<}i%~s@v``JX31>>nKBYzDU=d%Se7n{|T
zvWq>OQjcOS?CrLWqTZo0F%9S>{18YhDe=oyMU+1nd6G*`ou{M~5~V?eeHV7wm<q=p
zhvI{E9izJ6DJ7R_Jhe%;%|w8MKQime;iIkZ46+mJV@m&xjgby|w<E^kWvr^xsgY4v
z!z4!TG0iis#;X?2W|&!=VxxakRjACF6&S&F+s*oynto+qS>}mSpOLDzM`C`8`w6Dy
zj8{17`)$z>ylRiq<B+};i-rdFy90-yNW{Ee=1NUH7atr4QWwhtd?871m1q~Eb$*4I
z>&h){GX;tg7tIczrmh#szka53?U6WaQ`w9@Hqb5-Jw^2jX6_d|$~yGl9poK5QY<ZN
zmpWbeDH6s;L@{Qmm2^dwa*pwk<Gt`mAxGHbEujfY#I<$75m`889jZL#g*8Q!``Ai{
z$f(u_mcEN{V>9g2+paliN)AJ;(NV?cl$ne3w6R1;nVATve&jP4wi4{@<D7(4sWMBP
zo<f}D(h@4wiF5eKo85x?|8!T)|2C60G1#skhMtinmZYO9q#|tbmm(b=BD%SB@<T%0
z(H5*O)ewAwBg8E+Q^BDJ?8adFstYMA&$srRhBOHyYLqM5Ul@4L>Y2h@wUMs`iIhf|
z<qek*J4^I}+^_Xp9FzY5@o&pDpAHo)#s3V{qQ9c%5z*To)Pi>T($+q<oUa5)4=WD-
z@SI%UH_B&v^#2sWNk_++Qa9#o{~`B;$9+Ha3E=k3yw0ce(_m`<i{Kj6FW}obqL=Q?
zp8ii`tgbwwWVL!T|Mq;Dyhy;g5cy}SGrTLY6W)7#Gz}ju<|c-1d*kT@^}qzMI(-A$
z%MlL{Bm{&yv`KlZ%)efPzNF~>C}vWZsMG%PDDg~FRsFWCc-Q~$Nuh47>C^GgJmexO
zf$*a(kXZ43&DjgKsIq*5>@3y5s_u(}Y-Z$M0IR5{?#)|~wz!{sxHt+spCIVdM*I3g
z=i*fY7UkvSlK+Z0d3Z=g4EgsOsi!%wD1@tCYu<M!)8i{n$oX{tpgJ~AV*fM=kK>+^
zN0Qdf)kPBA;{~rW7`j!m+aN&knjVLYWznvBTg!vNw*KG$dF#)FQEr)P#pkX6I3fDF
zmaNds)BpZ#(wbv{swGguV9lEH|LI>T#{3fpuYcKNr-gXs%>63X-W0XuXD|IPwG5xz
zp;nb{A8Cg=nF!6|zrPIrb>vX1Vf_2Qn^v*qp&{b?sl6KvqCkCka(I}Xl5%nXuh#Wo
z4E%0$JswieCVLODpkL((In#3rDmYH4iW9zr8=@H(du?aWzc~mM%*?jHQF{Q<wgq3-
z*4iLC38)ra@L&0no9rXEiW;h&l;c}+ssVt0p=SjKb*<VqB_&c1zB6(Jp3Bj{gsN@U
zGo?8|!C4Ig153)v0)c`7K(Gly_g1n5B-e${0EKdr0ZWXSxOmm+$}({K0JOZ%JjhMA
z$<Er*@uf~DGJ-j5Dt9F{S&W=~wt}6PZ`D2p0in^HT|c_m=c?QluX5Y_uLJY@BnlQa
zu-AM{sU`LH5%M}0R|f{LRgjAV4EBxSES-soc)?B;a!$@Gz>=Q({{)>8m<vw1POX6v
zBkF;eh$z)&jqr}0k`k&0!JSMYRox2tYjW#~xX?s;R?g;zP6H5p!xn`#rp_XMhb8DE
zBv%m@?gGpOD*;u^pnO3ah>8G_EW`qE@5$+iUPV+>MO9RI0stwvJ=@I5$pM0Cu+3zE
zbJW46({py*@nipePK3u%{=8vP+tu($ux52}>G<A1$K#k&6Vv@|Jm_)g*^;=o*2hFT
zr|fqRe<RzM>ccJkm|$u2Y*eLa%q^zdPgH&xpe4UiQ&k1%T}EQ!8aPPLY$O@nrB|^7
zE-r3AE1_PepEoeEoCcig|M~L={tw1RXkR?3Flq((9c4zCMFN0f7?|OK2--<HkrIDu
zZf<U7rs2^{J+rp8KJm?ECm_ZEh+VI5VnU(X1^_{dq+TwQ^j8e29(tX8mCk@PO{-dm
z)9ZBY<@u)TwiR#@AOi?rxaUj$7yzdWAX5j`-d2b5s6dE1fH5hbnU>dOE2_QS_v6O}
zpoIXpi3A_NyfGV;LQE6{giKD`^sje7)CY)o^WjAD$p&yyA-X{k80Fch2gU)a0w7J?
zaG-40s+zOHZSy+c8Ua~UdYFn-x~pIq0_rmzTZr!wqZ=Pt8L){x7r~SSXf@!h0$TlM
z5Df-k_#4CSySvMwli45COn2ZkjO@amP>I|dnZVJJg07dsEa%x8K|0yTV|}*`9`E0U
z&rm>O2QcJiJvsT3&7pE_*Mmic**lxc{I<WEDH+WRdyCOKzlvlHWAxv0=ZjYaK>1v%
zoLwwy%fz!a>@5@6OeXqVVu$<QBsXA5N=<*KJy?*ftL^WBa$9>dcUAzAkW@nL2Q@H|
z%$JdjDQhlz#FrQex4j+B;O0?^%S4Xs^UQniM-dlLP#50yT>MLjJPkzamHGP?^t;_*
zQd6nk9B*w?-;=ZCMF=V45ae~0@qj!=KzRUA3C6TSyP>qUmId+V@Q*8)v*U^Rd!>6k
zUMJV>M-U(c-j_91Ab%ym!U_#a_Rlzb6YL9CFW``lvTuR8W#&l%?`dV@<TmjgmNnlp
z5)pP&(t0o>s1guLsxWp#%bJ33-ZKJ)EcPo{)=#&0vA1XCgmc(b+)pTh)cYn<bm7&x
z{=@hh!09i4U!LK@-$)+=xEtIyT8R;`CMny4NFP%F%PGnPfqEdP2IdYj7HItfC$Oi>
zA>-iSz#szKMd4#KU-IJ`4G@yDhmG)HV0IF)TZBbK)abRIgJ%sa)S<C=z(|p3vhk(s
z;kX^d|FP2Q{aWqt?RvV_wR%VK4}36w1F<*N({Eiza5>VA@9nAaZ3Uj8pYwFFtQ9)9
z??!(#5RpP3y@gqd9v+B~Cn{mLEO$1w!IZneQ|)LA=@!uec$6XWy)l=+VCcQD+~4t1
zYqfG~S3cL6c+xT@VZ6idmw1&Lx{1FtQmoEuXUkZKg+#>j(AD8u+NS4cBun9sADx+>
zP6dGm_|M-MzC49u{r0_|(X)k|b~=r`_Iz=E>FJE69qkpOtdi1G+=esL+aV5GHTiUg
zh$JzwWiyt0D>v&`)(2Fh4EfJbF!QLsF{@soCMH%}?3d3EAxEVjO6d%z*eUmS!yCOR
z?|v63y}((D{|$V%a6E5lwS;R!Bxj7bH$SkoPG|LSD$3^*W-(4fIUI*B-vKxRWoufJ
z!e=c5@9mLyd0|Bor*PlHcKVoso;|1z0r?#zI|CDPMA@eYzMw)CsbaDmRW?Un%dk=S
z?Vi|jJ4eTplM}-N7Iv*3yg;w_1Nta2J3Ax-Ug(Xp?VgTxsCF(u)Hb{=%DPsaJUz<&
z*AGjZ%%*qN^Kc;;8-?i+gtZNK2LL`q843RkxK1N1f=n$y^Ko1ipo$;PF8;9w-|a^W
zjYBM=qYQ1PWhFhu3)Bt33K0BDq|XY`?;0TBWHy6Rii)W8lzE}C5RnBmn4+f~*x!Jn
zl#Gd$NC4v)5`m~@x8r8@VZSUd*Lx=u=#n?mLG?`Z-%$0vK1AS3!)pS#1HdWBme>({
z^0A~6Fzha8Sqaba$De|IF7x>UE>B=}IzI*I$OL{Z$nCJcQ%$-Ff_$e`gu`Hc&nN~<
zL>g&8vrEWfO7rwR-RDXSl9{J(Fg-ilA+PW+C55E89Jej3|7n3yEkT3TfD4~HW+}@=
z_@(W(&B1)$vp&m#uC}U`+Lx!CVN^leSP3j_m_o_l-*+6vKGi>Jy~l}EYo9Itt2xy#
zJ6p+Ft;2l&L+#!0hzeEdP`p^p<(68b2?3Lyh<L<wr(mNi+;<0s-|2;-Z*312G6Oy{
zEId8Jh<Q9;nB03u^Q_D<+~_kZvzC)yLdRT!w?pi#-^|-&17M!7{v~}@l-c06QEkgs
zPoTdACJ3%{+87mRCmaep5H`r~`5FcW{84cj+Udw%>_j1BLIAq!R7ADb>@IB`I$=pH
zB+u#rB=`_uP0`cEvg?e)%E7Qf%xRO_Gh76gU?A{khlfRm$U0eSK8Yj^fJc(P4==}M
zB6Ack-9A1>#e_NPS-Uy;o6wIM8CtF!^#iddDIp=j;J3^`Ix%@AkQ1u{JP%-nrw!oy
zzN%U&$t=9kc{QWEcn}|VH#Q&-+&w%rE7YSpRX&45g1>x;kwCvAl|gmy-mpTP14Yk?
zvBm~q5DPFUBs@;}7%*$1qK)^S@9Uc_;)Z>>>@k;9zR*7_J;)(r4(ee=t4MuWlid1v
z(xEUlwY3&RQv3*M^H&sE4Dsz19sx;++T;9@uC|L{gx<<PHUVnCAvz8rgz$HJ>4JjA
zz9-s)m`|T1VB83KM&`BZbR1ACE}*r@BY#d5_M5$blz{;@3MQQ>agw)U{7owT)Qg=8
zthP^N_ZFamj(xcIttRU-R+>K1ISvDHLu!rLRk;701q7g5Lm2tpDTY^sgAR{mtGOcf
z(1pGM5I|9<+r|i0nf3Drpp-zb0%rO?1bT8F>?=^~rN)7L>s#~tiA&UH#Nraos&(ee
z-%tAx%t+_qAlh0n;w?y&gnH4iSgjG;Oy?2Z&$#L?DrT<PyU$5^r&PGqLj+7rKxnYg
z8SvCbR0eC>Qy0rZ6<W8w=RwNU<oNd)9JCk@w2P`c0NZpvG))kUph9RD2~ZL^gRU&_
zXN|KHt#;SF&C1KG_3df-#K^RzWXEXON%Ss+nVvn$N)JO2HRBMT_8I0`hAZAiRp}JZ
zEc#y$Ob}#G_#d6&d-(38Gz#1P@erXn$?g^bDnYUcp{M>Aq+RPh)5bHy_hsQV_^gRG
zZSZmz*SO+IriO)$ZiM{l%=)D`^0FQkCJK8VY2M@=)HJI?14>Bne_-#&#i+|Ams%HX
zU^dcF_H-MAI(YQBqPk%2!W$zyZs}mAe!z?b_&&oRqL!Yr_Jm*(^T!0MfyXUj8i^%3
zxtI#^D+8=dN(u}qR0Um#m9tv1|I}ALEnhy@b0;i2`iak!{{8-VNE|2!VILLdm-Fr5
zld~JW8}!!RX<)104B#1>nVFGY_~L8n5D2cJ`}zBCrz!x@41lLt7#LH40@e>KoJ0xw
zjIUZ#0JA*snnqSM+BMq*c~IqfFZXZ*M@d3Q$41L8Fa=c%qbuo#MNHiBzaW~|q5Bgg
zMKBKvS9r@#Nrr(=lU?RA5=tV}A2O=pLSWvNSvi=mMGF^c3j2yA0U|i{)zjHKO7uuL
zSZHajKoW|~`!g&vUf*jwJ%o)4Aoe3q?u|}#>vbkeO~2Q&kQT)QQI^gSGCPZn)}jA^
z+cJ@ZZvEsWLERnnQV^+>{1fF<=u!RN%it8;36ka;5sT9Y)NAM&7tf;fY6;ERnvP*#
zWpaV+Rd20CijNBB+7dcn_@{Zcz6$zG7lxbcVYc8){q>XNc8r-t32e5<2ygdLcEOyh
z5a)X;nlJFQ)74Q4Ai2AZiQN|#DRx+6ic{n?vumj|%!oSbd_qKWTt*K=<bz9)GAt@K
zbGA3e>)*gbmr{{Y?{D{P3A!8`1NdOMsGS{SFPjxhK!{?c0;96aXMAvOEr=M5`ija2
zi-wPf2RGi&C6KST5?O8wW)V<jzOb1i(htu~d|(}f9Y_*0J4I&GOn+8}?RVk_Lb*i#
zuvlm^dU`n$NbAwb-PL$Axy8P}kz5^B+xrCWS2l9z!2Qusi^Rd&#fPJPGgqPZj+B&i
z5;OVr)I(XiUbdt+U|=3!Yj6M~?<>-b$D{+z1(^Y+9}_SM19`%2JGc@A9+0<##0D^}
zU?rt9>9<{`X&D69=(RYBHo>A%YslFoCj*0V;Vm#FzH)yCerJ*_EG&(W=RaAv9hULq
zNO@h%K-7nFErV7KnJVm(Y<k*y{BnfQ+Vz~|rao8(tbTyKt)&jgsIlLIGK&OGjC-Kf
z(nG_PmT=|a6cY%wRX#z>&bGe)aKA<AV-odIxZX7DbAaut>Xxo<)8s{w1fTmYI!a*j
zYJirR{7I*R`@a}$Ru)4lr@gt^*<N*oXzX)gyRLh}9ASB=hZ)L*>GU|AQ6lEgJK^s+
zj4A9_KLCwE#-M6SY2z_zPwjLk8@tR~zo<W=k{@M;Pf2HExn6D%i222|iX;M>+<3Pu
z2E<_yonlBWV%-bQqxq<c_&pP2tGv$FJDTYl2?mB{hDf8$vtPUoPC15kb73r%6y8<P
zxBLTE&OmUQ{77B^ACP$zZzkS?kO1;=u)VY_CcDrI6gC5*&<TLgQ6m78g_hph-@gMC
z-LKE@H^ELf>1u`yKU{bAbo1BL_x^HU^{){;2zIgzc~8=<MQbXM+>GAS9oM}}#ODUT
zjwI&Mf(=hh5t09Dcqa|vc=HU$7M{{C1p~Xa_Az$mtLX9ErZKg=h;caX;IcW{8>Kkg
z>(kSgyfc{q2$QADPnR1fs?K$u<r-D#t(B0n>|^I(<!cLm2aP7mWj$Sttvy=y^BV>}
zepI$>1Q}Rq!8j940;sak0}=AqSw1T2^x^_aIwS=Y0`L1vYbw7<l57m7ppnNDzbzzV
zw)zjW1Uf*b*TDD#5Fo(nQIK<LD?=pPj!1+Tjb;1*Mr<o|G0{_)^BV028XJ%k%mc#f
z?hyqcP{b8fy1<Ne8><VOMX{R__X5NRz#^psq$XZ%h!>74NzBWfcCg&BtfU?HFNgX_
z>KYn^&nKb2Ec4PKdPS;JlV2qU&gVX*CG?L@QoXYs-)j6ixNRCo*ES|KFjQ&Bk-<=J
zZ5}hhvv*D)l!aL>aTXjBg%}g!C<_A-{IE6Qc^tpA?-J~L76Sj$jQiE~hSxIB|N8mm
z<)O>pAa_;Aedm*eH4P=fB6?e)MtAMk2JR$><z!|D(NzNEWMzK8&yVi2oFCdy;SeE&
zMTsNaQ<{V`$*n<dHT4dZB=xfhw}WaHT*+Iq0l}xWvXyAJvk(bMaSE#G(TUP}Rcp^f
zNGAARD9+Lnxh2m%+L78>)9Fp<eB-t50NG5@-7EME2U1DF=JUV~2ROYpiP{c1Bs^Kq
zKRmq%D8wH~1L-wdO!?gpWtep2IK?$$caK!n=(?S;&6rqOvne%kcu7k3W^OFB|Lmu{
zOUg0gSpQT<WM-ZCJW(-l4*^X*h6aTw<O@(0(D;w6yy$gcsv^h-`uo9N)cd2y13tc&
z#OLZ@+oj;kI)il83M>o^47}z>r}i0EX!!7(W+pm1Q8LV%aq>bwQkibp;VDU*`0P?b
zd#$^cE>r&IFfEiNUY1o3&L-1a^!#0r`1<PH1|W@4HqH&o(#na9u;R;YqQmaYs(+WD
zK%H~PC3^4QvdPa~a{b<W`ThOpS(0zLTwGj0S_?mL?xm)t-rjn_D{MK*44gIqS2n1b
z*4EaB{#Q6yx_y@I2lKyuhAO#~+jGjl9(*@`geK{07AXZ2Ic(H`=c;$4Vc3x`m7l!#
ze8~{P7Pn1;k3MEn?6YjuMsbDw<^A+W7lCiSh9Y_^ncR-KU0p9<wHtC>%0GtoBxYr`
zgRsA~wY9s8UG};6^nDsj-n1uMi>TCB*Fq$}J&=EU2!#%aU2!t>8QFXO`kv#PSNk59
z;|w5;p9bHQ@=2+M)YBtqY@=plzY4#LFzRwHAVMqgC$`_&{s+i$IF(mndT_4BZBVt+
zJgefoVxb(E_R<2uuM*}@Ql&>*h_zD2LyP|<`QMZ@nR@>huRzn|kvsg~cVH!iT58qs
zO6&RGK?R$g^B&BO7@N$mc%IazF@I}a)y^M&*dk*wzd1F=Ds_5un0~e!w9u(p<-P|d
zLBH#o+~DgdAk@Imoy;9DaSrf88{C}Z{stgi-RASUAGp*)6~!*Ci{T3b<Xyg~?JLZk
z+5<-Fj_<_QGsi8?rgY+@iZrXLH5m-;ELe|ox0X9u|6GKRZj(6dc!)kzc-`sjvb)qQ
zD*DYxKmR>~ov7S5SS|_S;t_@(HBy&!SAU;}(Dd`?&zXN$fM^AVJ?PytGn!D9yxax!
zN{#l(&wdB>^pz=*$-Y-st&j45k)^Ke;A}CP?5o~%VFveDt@Tdbu#JR|l^;B)WiP#9
zMofA|89%|6L#{OFn;Y+5i{;R6F_69@qE?NIiUzb@BU%5u0*M<dE$ut8;q<GbUn-~2
zz>=*>)$6hM!|Oemr8{t^QHvj4jK?8)1*-`Ynu=oHvgkMJYak}D{Fiu96kG(`s;Ry7
zt@ppT+SIfiTwVJ=&~~@{pS76#mHO_i$6S}<C+Cd1r6azaiMts)u8Wih@Ij~u@8P2s
zaF1K88Jv%HM&0mO_))1)HCLnC{HiAfR5V-W2QWEbUtCl**L1S1ewq;>wAP}>u@Z;c
zX9yQVL_16-`HU!*xe`3Mg^g@jPdq}yuDyg%u3ibUfKuME#hffV7YN=;#qNx^dmtep
zBVRr$<d(BiQknvy9V}qVUW9R_K)DFU^11fHM}ok|W}WKqLZJU0=cBkd)1jekNIHAo
znW#<~IWmY=8QB#QgolGCABY=I%<-96wD55La1v&Htv!z6*PzFb({h7pqaN~-S=Ci7
zWF@GshCKAl!N_>HOA?d4c5zWJY5chNjC@&_zOiBQG2Wo|<>+wqr|}jcoWR$z;$moD
z5WF9PP^)->nNdqHE5m&5NbJSWQTeCA(l>2i{kc+1FpG_XhN68!dc71ks1J-akjRrl
z#r{VBx_`4l6NG$_QZrUD1_-d=r^B^3v%p_c2%J(e4WU^%0P&-!1+$OS;wWWIGKj9&
z+A^!#1b%#wv_D&KF5hC#RF6*1Q1&4+L=N=g&8^Rkjg5d1ev1s-QvoI#Fva-p6-3%_
zABQj&Xsa}6ai*rFMf0wPgoJ=mr{Fz^?l?W*$Z-sRU16ZLH~c#6BT~>f?(Ql*|7xz%
zeS2O{5K>XkXDEK-!5X(qV#`$}C*RZJQLWkF7QL|vb9w)En*`4J22!PJPQT+I!=;~v
za)pI^=$G9$XgGNezb+Ub<f<vFY~X7JO0GQLW8U>prv<=wfB3>dM@E(p#8;errL3P9
z78XDr1S&CKs%66gyptt#CeXOw0(~F%x36GZE7&^)rj9#L+Ha$FT(Nn>IslUn4}*Pc
zWY*zg_PzONQxuYshP|a4bmKIwKa85l5n7M0neW0Nmm~iaWL{x+A}aOm^)ju{u=L^p
z<mH>|u3}gKV`%x>AV?$W3G^nC)}VM7RM5?a{_mTnA28=r{D{e8-bb=?0GOZ^&SH>0
zCMG5>vmQP%adl<T0`g(_3$vjVPIw*G9eEux^Iwn=2?P)-yo;1CJE$H>jx6W=m#|IR
zO?KK65)vOj;`<r(h9LmNxlnJ_x4gQB20@>;pWv$BQlwue7N*g%b`Sci^4qv%#mYTu
z>!0T?%Bd@&^j#W`Z0lt{t3-rsvwTj8n7+n%mhTqql(o33uRSO1zU5okn^2SxeVd3*
zl8gAi0JQ~5`j4Q#{mT6`HuR^w?dTJS<v5Rsj#8OS<qmtmp<!6gW$df};Rjl}m{fig
z{2-Owp=;Ou-yU1`))+-YRd{l%!>Pf>oK^u_8b5w~^XARr8t6Syj$9B1CcFs-3?p>z
zZgoI-hbVzXp^_0JMu^1PMGguJXZS?bAqzoR2n`KIhnJR?!tsX<8;0CHJw5%AM;;L|
z5F6*@<RAh(@W2Bg#dGJ*K?z_43kwTTf|%erE2+8PYpL*bTDPCTM!8o*4iAKELU!}?
z(@%rFAYw4C@$vDwxw#m~X3d(pO-13EqeqWMx{Bq?moI|}HgDcMZrnI5gPb6VU^GuW
z@x<}t$Mf^^A=*HDFq9)lj`Z!@H!v`8%a$#Wk0^is`RAP>Lta0&LA=0GQc_YDELiZ~
zd+&`JHA*B|n>KAigI{{-B{UdO1}YG=ilby?WIXfCGyVJbM}R<W-+lKTR(0&yapcI6
z$kbM@T>0aVKjM7Hj2Y9WO`97xZlDd=3%PU8o;`8ar%s*1L9q>Nclhw((2*NAZd|o$
z)zzz45qHp3L>Yu3^hRQ0BJzxV`}W}{Do1U1GmBg0&{(b7W5=khlrCSQ3m0kFu+pAS
zIRh;Ibm{4IDA^YEnx%D+?qD+QNhE)Nd(<dXNQm~zWkp&l1sSBmAVc}^;TA(1n^LJi
zae}sQlZf!;mrKLC>El*V{D0|mrVi&eBtl`eX%n{4@#ECCopt0$YpVeNvuEktS!&*b
zz(!$R&5<K?Ip^u?e^GX}u0x1zSS0yrBvw`_^+%7=%^PGgO1t+6FMB0YiBZsKoDHMb
z{$y<5UbppI3J5fTAk}JJ>M_bbV;nh3zkeV7?OmEa-8f;ApjPevB>HfszfxiR%NRui
z3+^xvM7l$V4t@Xq_b_HS2n+z$lA4+df5iXZy?Z-np<uws3(ucFkK9zugp2fR{rdGV
zIT$c3r%RVEK|w(<?CsmPBlr#+I1pwmC=iB??O~G!gCQ#`3oiM>3omr<-konE^A$jE
zk3IGn{2j|+yIs3>oj!dU{Vz{GM7R^zRuD;$xIh~Skp~t;itqW=4K>1tyk15<Ps}27
zgcgMR!D&Hq!UC>ezaA12GHB2sw@xJrg{HyAaCKM!tO0y8ZQ3-{gYut${uwL<m;U6F
zPnIrS`s9;O9z1w(_3G8VdiDDJ^Upzz;6m`t`|rOG)^uxys1vq@4W51W*|B5Cq7bc`
zI(2GMQBm8rZD-G(4YfCT@L<s4;>C;6)Z*e|$j!NP=MES!;G1v0S+{N-4hr*yMMgzM
zVPBZpl`B_J4~Ta9^y!FAI0Qn;%$YNh#KpzMA)thZha+HRW@g63#6Ytm;vf$IDT2Sh
z_~MJm$Vkv9R4lT;PMta(IB)=o;g?^2>D5(IkH!VPX))8b1X{a>diAA@blSB;zaq}o
zI!IzVNUqXS`d~UK6}CUWrQ5ZW#*d@$aLUV5Bqiw<e+EaSk3XW)5(@212a=R$vXn9N
zK!NnZG_sYG)kdpU=>HQ>=gv#Y>FQ;=as`1|hh0`Kr`~<(->cQ9Pt)u<bS6vM#qc0C
zYep-UYi<`&=T4NBDb)aad-)amb~EkVrr7>JI((RB&7c8;si=tJ*Z3RDjIY0;+p&!%
zP7n>F{Yko|v2-Arl9N<hw%JhO*I!d%fnr1yefAkmjV_%qNgo$STQ(unp}F&D-hBFK
zf%?@bt4^nAFyE3CSAUa}lVOML+O>m)AUIB(I1#xQJO~L8XbGVZsTC{=rn`Lkau_El
z>be6WJzcnPA?!aQBI2i?euC>QTeb|b6_vooK#Ftb%z<&jhPQ6rikz%pzkYBNNDYuE
z>=s9fjg5s9)zyqblN5Pk0U5dTttM3^6Nn)gCh($OuScSTtXJdB9RB|W7yzTtlkC^+
R9Jv4h002ovPDHLkV1h>RJR1N2

diff --git a/modules/web-console/frontend/public/images/pb-ignite@2x.png b/modules/web-console/frontend/public/images/pb-ignite@2x.png
deleted file mode 100644
index ffcff3817b991d73afc3d169ec4d5e97a13a48d3..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 8558
zcmYj$Wl$V#7wiIyySux)ySqamc#wn;+}&jtcbDL<!7W&@09jmu26uN@E?>QMZ{4ms
z{d6CznfW&}U@Z+rbQBU4004lltR$!Neun-pkl^0AXKnKj00673rKTtUZsFkmmwaae
z0s<l;B6xWCs;a7Y`yU=3AOEj}OG-*QIy(A)wxFQk|7kWhws*a<vhvP3IXPQfTV-Ws
z@4CLe{+$;W7sJBBR##W&=H{lRrn<ViE-o%&Vqz2(6)i0--y3#!cel5<4Gatf1O(pg
z{{H^h*qEN49uNp@Y-}_(Him_TeZR>5{(g6N_jmmte*XMvXJ;oNA@OeCr`{VnIy(L<
z_4f7#1_o+tYrp5;m-zVjyo>+ZY;0`)>v`wjzkk=&)x8hC<AI*_fr#ooX=P>gKJuP{
zUR=Bj7#Ns${dRx<y14W}N&nyM`!?v_-YW$1Xl3*M#NKXhUIqrAz|oHqG7m7AFC87v
zb@kTP){<P@tJ~Ypnb}XNX-^(rPd>hHZ*TA0USD6|pl`UixM*l-SXfxd$jEebbkEPv
z4*&oY6B8vRWol|FIXO8gDe3+FeSCa8H8u6y>+9|9?e+CF0|P^Pdiv$%<;lrObaZr9
zR@VCZdQDADb8|BY1j@_H`}OPB;NW0kVPRxsq=$zG1OjPoZFP2bK07-L2?>#tlk@lY
zpP8A_(9q!I<SZ^OmX?+l6&2;-;i;&oaBy(=^5u(}nc2w5NPBy`x3{;Nn%d~-XirZM
zbYr8Wq~wvE=Vg5S@xw>x@yT;u{!?Poqm#>Xc}4WH&HH@;{M>akbOBKK7w9{R9&?}-
z)%8&5kAeZ{_^&Q#8?@v-odNy%zv_QT{onE7Jrbc%A2cX*eR1mDT$$lRPj}vhvOM(m
zunao$Ed~l@q=Q03p;DMoVRnl5lB<p8JM+B9{|9@8zY+id#Ybg1X+7_i9uq`aT?0Hv
zJ0-3}X(T_*WvU9p%IXit_VZP1e&^!n%Dl)S;gRAtVZjxkV$XW>h4a<>d`>*uQ?baM
z-8jEqzXkF8Rnqe(R`>h0qyE=F<bF2$#H#W0pH-C`murO6c||eFg3bb8dxHvYJkNc0
z=JN=Q6%?Y`g&8jecIo~$@$lHE=haKNCivb50XL!BqMD()@x0UNNv+QzQD=S^{gXMA
z{ujMbXKQ)spVZy=0LD{O+xKsXRCv&lsS><WI+5DL`}Hq|8E0LBYe#BB>Y8M_S})ar
zvp~0U_W8XD<p2xubv&;ZDZ6;wwljZSiMx83_I*QX?Q~x5R|v~+;BHS(?cf1CNWG1d
zOhwcEGd@09*S<MSKS_Y^H;t<K<g3J~)f~t&`O(cqAviO3ZA}WBGX^h)pa0A8?CO9d
zLBF_h%4`bz;i_V70kT4}^~JR+y;WDd^m?wNvWEJ)FDA)jl!(w7Fj#!G1fh?v!>mTq
zbwEx6bADq<G}q}T35^c-0n?pf@@(EnM`2L#W)g4ZLMR_Hz`iFic`e@sT@ZaLx>t4X
zYe`@~9y7*>6@;oB-*EPi*p(M|1BUt7C<~KLr(Cbr=<M_<Ev9x#qMm_fo*4<&fc8n>
zBB>m?2m)1lG84<_xKLm1t2j1ZiD(KotU$4})2OUkAm&r2JgA%fF|8bSQ~9?{=3#ZI
z1od#`T0P7Kf{a9R!g|Ju8mEQy3p&u4R|iGo_Xo7Oe5vLj$HTw6a`}Kb1bJ%iN5EF3
z^Zh0SA1@3Yj|v`U`!@Uq1vCrKOl&;8V=nJ}kBim+c^+x{pMYC_@F*iwVUCJrk^kR`
zK8VB-T_*<(t$RrPLi94hINh(W%s;M3w2eiTbC%Gg#X=a~NJ97BqU~lnFtsYjR0*%X
zMI0RATVCLp7JdsT`i+fiNrL5XQ{;uPU!@{{j*TDsGj2dRZm$?1jlc}L$u1h&e<O($
zEDT3nLL0ND3qsWIl($xVwQdEa2AeVfVddhqBuuAp;~nOtCR>V}6e1F(^1|8+T~e8(
zJs^lEI!)`{RyLchR)@miL`v3{uvo}Ni;?!U3?GLjV_)QW#PfyH?^f1cft5Wh55fbU
zTSjm<>YREsR&T^l!2Y6~f&e;Si!e9is}RXgU0JmsOL+9ne&A7{F(0Z+UhHlWm}yoP
zF#?36OL}^YV<TkCWEB}$05hjSXeSTJ_b~)HjcxOKs}HR;ew<liI!EdcX%0n$mj;bW
zc;ZngYl;lj_@GP)NkN9%jmf&3IPF?-C+Wc_V+5jDLRI?I&9fNPWgrd%kPZmO^g;!V
z82&5b84?`UTuBo&lXq7UB5ck`O#p<SC5*6j=U^ZU`{6jfq);qGsHT%A9~q%xr;vDH
z+$)?*NB!`iX{l7Z?1o5m7D=#)f)kR#QJ^?`P*tA8;1}U%f2EL&M1!K|g}k@w*BZj`
zCtZX_xhD+K0IByk#$jQPj9#CK%Lt&$K~oZ&%ieNi>BsG#d1tX&sqwyGA@3GFP?H_3
z$B(zJ3{Wi<PFdtt=XHcQV>YROV0g__TEKh;D2gqmKiK$-ue=kORe9nJ{z;7|eOP65
zU$K}is(IVuDhg>`t<8G#Gmm$iJJ3{^jB3Rt(U=4bnvu8TB<!ay`t?dThRR<Zz9k!H
z*(PME)qBv%V)ga5W>moWgu!C8p+s_oNI=0kXhAtg&LWu9A1N(f+2;GkTq{+m^NsIy
z5-_2l{Zb0H(i+Y5<j%V%Fa-v~-YX^=zX*pqax8PRxB;uH;>YAcl50c$L}!o{T6<#V
zztmofRG330HQ+q~YNK;BdY2;T1i<}>BLY6O)W1$)Ew4JR<^hjc-1zjIx))&x?yX?w
zQuSaRa|o0J((X3ZTtep~k@A^XmL`xyoF!xU%#GIEN!ueTs+Oj4zXuD>X~+?NW<gX?
zbVKwV*vb!Cot_RE&tie~=_c3XU!~8q<h36`FQp+J$f3yCA}OA_yUDX?g$Xy9{YH_<
z^ZgggRdyeWigON_$w*Kxau4-@>3Gv<ZWqNMUN6%0QC3!V??rnzhSfJt-kM5;e~k!j
zVdax{UNa}naP^%CRtK!LH5#VFhMPStwODpqX0&np^X@V_UZ@!<)s!oZn;M@h<10gv
zDy?eWI5?kNeL1eDv4o0=`NMQ(Ky|~HqL~&x*hLEm{)q_#7ZZM+I#61Fo&1a*Zww6D
z44eEh#z!Ij6b~JQ8$d_6&>wY=8GW%H)TGwN0<|^gNe8&cyX5J{?o8vC_*GLMs}B0R
zmk<XOB)~-$te!ce6#f^}`8A3#6=7z6i*}4~xI=i%CX#L?q|c@1E&aTMjxdg>e~}6c
zddE%AC_A(>N_gg#;ZyaztV)7}312bcaT0gDAQ3u46~>?XV3~cO44JYk3m&}THA8#~
zebbS9Sg6-voK4T!P5#p;Vjaaf6JhR!Y8S%Gn+VS`FZ+@prX%Wxa$psUY(;NntZkme
zO!Dy>y!^#P%X(tc;9>yMPr}Xl9-1)?3e4O>wuQk$@P`RF#XC>t7y3p0x1m#1Q@?%0
z30450F-{s$(W2<oVOal1?@kS4-;Ebmz6!e#(4WklJnk`uZ-cc{LrYZbd!PbM1LkDV
zU&cqKHM9K!&zAGQkJO^rou<q9hvj#{BU07-ME4GwexB-8h8DkdXk=ssNV3EYJxUUy
z%xNb=VqsPaBt;}kn;^oSkEP_l!ZNcdUp*x;wXe0sFrYun=Tg8HT_SWP1Q+<;{|v$g
z(EVb_`0icGmOfp1ctTp-(zYbjVXN@;9BjSaE&L74+pkP`iNzbFmwt)pB?j8JV4X?N
zOChU(SUm=&cY-y(g%zFf_&}!S2WVv-;rj^9H-7>2s5i9&GQJQ+0I|tLblNfRHY2d>
z6#a4+^o9AuEw-kSTStk=itJsNsjW^>JbZ1-F7_6aBc{L!kD7>uy@%1l{y&@}MX<0I
z8C8#}tHUiC2dKerLs!XI{GxK))83w1nPY#EiC1@|YGELaqqN8}Q6>HvIwF$D!tM`(
z!OkmLWcA>3DxelAW^+;m1>Hn!zY`}~zib_Tz81EviMuRBs)F-0)Ymz{6*2Ux6Clt?
zlph^~%tRxNu3*iG4Kt2<FBr{CY_o+b>t^gud|>xRvyR~oGm5K&RTJ|#@Vk2NuUSf9
zZo2MwS#OTPtM4^rnPSh8*FrBC0>6fCe=G5UdgHS7ft|3;PDeTM>byXPVduJ_tkL2h
z5h*Fk_=92TMT(B7TU%(P8kPDv69??Aasc9T3`%)Z&$iE2)w^}9iay|ZC1=3TgE91!
zq%6B1rUjb)eag8G<p^7cJ41f+erjS7W#9JxA(=e0v`kkPK1%2#bXItjncFTtT%Av*
z{<$INKHugxN{;%FPe%bey0h<_O22#dclbaLn=YJ(x87uD$imUUis(CT`u?~FF;ZKo
z%$JgHaLp^mrOM8^e9!v&TA-p|29?yjsbA?dv{HEEt9_TohLPyK;pmuof28R00t%rf
zguIdM{C9R_crH!*x+r9mI*%Oar>;vH4kY1eN^AK1_n*VmB#MvvvmdZ>LZYgXczG4U
zS~4;|p^5O>`{X3JYs@V{6H?E08^N<rrb3>nY?S&8OX0v_>L~%iHn~B<(JnBBYZ(*%
z@s}7JVICS@$(A<9R?c&ONy55rhlc3<ilc<;4ZhQnH|4-9sh@TJXXIn07+;Uq=0ebh
zr-_uaF_I7~WB8AmiAyM_#s|<xmz@Mql#_bU`v5;a@@cw{rT0QkrH;GLf}b)g6D&HH
zLM;VE*3r-yR7Okinf1792q<Q1MN1jj7|$cHIxFq4kPwe8Lva;_pOF>sk2t=w%Gmrc
z;=L+;ityBd7ZBi)I(0=d8QaFL5zNqI>{!f#11c@$j%Ftj#Kvs)A5}epO##>pJG~j#
zj4W~{ffuC=&R_FsBn<5W7B*{9J_Tqm8jz(_FfwFjgE*3tJ9bLM#&_pqt%YgJe<&@|
z$9aN}$+A@k2CK^OwU47A#C9tCm_6s_w(Y5ZNYm2rxMot+!`dAA;D52}BY`xSnd_UF
zl`_z#)1*I@y@o*W^ZQcGN;F6~$8bp+k<1=+Y)B=|4$g2W({JiyZ5uXz%|2H-ApDb&
zuT*5k7+_x}YGpi6=kTL9R;j2Bkj%p^g(-Tv458cl0OQYW=Kba;Khji967Oy1t2Osj
zZy6nTU!#?iAm#kRqlh1&hjh<C@Ezi)F1Mn>^EWP2S%>>CZQQ{y5_b39vLEMF`GTYW
zkY~N1zw*Am=E+e)(e|*`_q7ydMJK(mJRS|PRI8q-FD|a(C=fXu22;4_BSvk?gOJxw
zhVO{#2=zVFPs6;y-<zwG&S-txAx;o0rLVgHP3t+7<uq1nh=OYMX;N2y+5!{f?4VfO
znVUtG7Ft-ln4_W=WMVUKVS&<YFo}=vYYNrlFxzUj4j=AZ3r)R2?4A8KNyqS5we(V|
zrD)*dLJi(ynm+d&D_z<L+|NUBvPPnu-M5y<+qJdza05n1Dn{&S58hLht`qB@3iQ;m
zhTw(RJy@E0VWX7zR}}<#O-zi9#q{KdF)~Bd7W?i-3e28+Z!brTf}l9&t3}oREHud^
zR`K0*icTeth0Fxi>N?9^@NczOnDlWqCyUff1)Zv5pVo@aE)X|)vlJ6P<6H%C-o{#^
zVwr6H7%A{+yzLn=GX9x?)kONeX=%Oes;McW{`YhApN#XTP7X)8MXM}~QXxCVtxx1D
z0avj%%F6k`iAwaLxiR@-?VSWif>*@a+Kuj8$78Dm<!^xSm^scX&)7KjSaJ#P9!D-G
zyC?hl6Mr42WYA5g%{hG~i3=?$x>rri)_QJs7n?AqtuVf>7Zb$6`WU_a1RrNW@BqsA
z?{wkb%(wFP`U}@dYM}g<MM<Nj-NM@9U;V%M;2~f9g7%i6x;tmM0j@B}4`i{~4ANd8
zqj8R+Wa4%=Yg7unaHLvp!TXEL69y-f3CUy}vS9c{;IWrISW+54p{@6tproN-01|Mn
z_wlf1yrQ+HUGpQ!N^$}K?h~l@U&IUtdl+_)>YcBhG2(!1rZEF8k23I4TEM9**DurG
zozIJ`G3mB1{ozco5Aw9OT`LJMe+S=$_^S?ct6dRM$UXIsAQt<ch}@lP;2n>ULykud
zAz0+YaI+Qx@82z(V`(J`pA7y>iH|&{Axkj5o}}@}Sas-Zh^0Y;#1uhb?ruyp`{d<}
zQI4r(nT#L8$xVWV2O=$oL2I$_s&3X`yeA`lMV-O`O8A8kXC~o~A;BxBOapHGEG!*F
z3^*?mRcG8?rR-wY3ddG`7Y~2`<xm=JI0PvAzhKARwP+2@Y!1+AH(Mri5Zu{hNWoai
z2~L=>aL|f)w0i!guab!_k^-5nVTHy~(_tOmqq@2;Nvlzihk9Q&E@YU6ss}zT@cwLW
z?Y)GhV$189xHLgCEBm&8{FNfgM!{$u)|ktUmV<^v0d0)ucY#czPU7^V$R)0EyZiYU
z`OHEjYVH@v%F~S3c?<MnJHd$wYd8wGBzBF~{brR$-o<=9&{h2#)*h3^dW+2j25J|r
zJXx|16JETjRi+}phI-mrXSj)omojnHhOepE^;g!D&8`K(bzkymq8E^fg?oB^r>)BY
zYZV$z1>8UbO8?}vVIc)Fj7qDdatQiFQeLtcgAsv<zCG=E#Pli%t$w5XSsZvt%q6z)
zbjK^%J*jahqBu8%<qP<P9rL9I*A(T<2ZjAy+}nBa(R3LleBDXVQ-z+-2N%mB6dRaj
zXkZ{;LR4usYI7c;tCH|=NP4;ZywOi7@p$=6xlB9kWurrFAa&P>$A^@S*A#;0w3@Lz
zdJ1t5!4W`G=N}U*koz#dVb0_AZ*ONlfK}6RaFWkxYyl0Xc>rhK=QYB~BIbPbz)*hM
zV_q-sG#(F!%D$LMwjIDsL>F$%1f5kX{+sk|f=ySfh=c5Gl}d`xB2y%e_G+4CfQz3?
z2L(Vx7UW>_v~^1v<oLRyD%Hnj<K-l+lxa+jXFW-13$#~hRQsi4$WS#t$tt~+Evlji
zYhyQ<y4c*$b?d6qDQ#(+GB=fKaY0)7vk4iKH}X6Px{C-bHUIgz1$PNg6<)EyekBgX
zpbrMT0ZB(vQc<JUSy^cVK%{DSC>{vz)lIGr&T|OIwVA5QSJl?AmjiQZnuS_DI%3x=
zF-}2GXNdKK46Bx&_ML1<W0;(4M<}zU$Lh1FH4XiT|GIl-J<d)BjfXqs7Co-pB2cCI
zCAy-D8L>PTEAL8ZY<yS-?F^$D)J{>Xa&>qVmFs(6i_?EbM>lduCV7GH1fxAh$=`_a
zdg}6h?HD2FRQBUd<l8qM<iH*Jl6sG}Z>#1iWDhAFw0mHExvhU?)pBmq*r~Ej$*L&y
z1#<c(tn=ewIrrev`m?WzCFS*hF6W@Vw<W<(joBTCU)@d4mgG<Z(&CJvnL<v-302of
z^&H6j;E}QdSnpk<E5F0+Z+M+1`gV&LYkd&cG4IQ<X|dCtVa}o9mka&z5>@)i3S#S~
zk(9j@Fu_EbnF6#E1Lj&s`Sd8cD-a3IwA5xfUjOSg!nx947E|(IthgOhUApN7M^FKT
z+<1{<Y`-+{S(<+rV)l(C_leAR(2GOz?r%u2=cPszExSW59f|r3b(4L!*tKn>+w7BN
z#rXO846jvM+IlmckF=-9s~0pEz3b=@MgJZwqn4lIi#2bN8yL!;pPKUAh3mG!ix4RS
z2{J7}<6^BnDP@(<NHB|e4d^KUYMF|&!sTc%;T$2X#`nP!s85#qn#&jx99L*7&P?^i
zcIrIA)^yS5kUsr^WzdM+cHjV|>%#$!4}qLZ@YFTQh??AvEFn<WTZUPbfWB_;^e21d
zDwKN?%9mm2fbR=$9fx*f=?#wYr*66yvmdE(cJZC_ixbiEZvIRz+kvXOE3jWE1fMxt
z4#Y}96a{-%YOs27r$4wZu5{8;e!K6SzYd^%T&$Rs?cnwi2Dz|dGIK7{U<&Kze^}hJ
zuWZmTMe6P>kM3U|<ah4LZ|Q9BY(1;UFyk}E6U8DHg%}M^9*QXtvvyJ7(e6k~-si3c
zX2^`_dZ<$+OA_zO#8N(S&EQ}AyZFzK2IOq}dMHNdS=*^GNpJ&2xqN!aR0h?@SH<f?
zMEYZI0Xlpxexl5rAv;;xG$&}@-Zs_0#75#P$3A*t^;EmG7A}HY4yHwuf>KsqC=$xW
z<-S+TE-SU6+N@nt#~ArXFqloLO!G-Q;8ozi4h_i)+@i$}b8>OjHY%FRKD4el4G(tg
z$Y+L~C84!$xxFdruIqn|m0w`IS)-LtpKM!X8`Wd@?6-%2;*?7hB<hjP=sjjw&sJNt
zcYyy<q84VFcDTQTnloIAw&J0v#sMXKXK6t?@MN_Kucxv;WnpPXpIa*`<Gu(;n^_VG
zBJ6F$&$peyuOm#mIf}0<c_93mm&Yci;y}(f%UDLBT<1a5B|0ziayT^9<t~zLHys&?
z;oI~<;M4Faf{bIp?dnb_1<baXHey-O5NWBQEpxZZ>G;*p3UTS&F6=B3IJ1A`^zu(E
z)C+K~$@fggqF@H{7a<)UPkLO@KQD<xeR0PW^&OTbxB8fwI_yO?yQ^aPf+XO99~B6(
zIPhvf4mw<e7>D`J#}D#oX1M5bsnLYzw7PL&dyx!(s@N6?%x@DfyZF1W(hG(Etq@4x
zu$RKpJ6b@}6R5-Css(ZBWzo7n__{y6K>8|Vcl5P`)kq>A-2~e%mfAa?stRYgY>%|h
zUVNLJ&8D7wcr^qw=zyfaCLMaeF9{}^XhZaz8c1D7HpA7QGMU*vM!U>jM)37uaM0I>
zBm*o}-3&SQ#@8zC>kb9>!F0gzPUS0RBDTX(ybtv`j7GZD2%Z*Z)UK2HAl4b;06ixR
z%~Y1xS>-<_&FzJ2+(OtCRg6FD^GKrQFSxK{$4oxL*m}at8WnZg)ma7WS}^`*xU!-d
zG!}Ga#Ll`|?N=&XeaRW*{#=MGeXrSbQw0E_Z%q7&P;K)OIdB!u2=yGZNgtEOjvZH-
z`F!GG+*R<!FI`^*9wd*+%kyjZ266~4JdpEA1{M;&=o3XpF%c$@GZyV{hJ*l;3St<g
zVT{^Axt|(yuYukrldYbb2`2pO4ofxVYLVW_qLTOf92w!-KKj~9z)po#X=Ju0ERK&9
z?Px%qO0w-id~b_Pra29&c;x$q5=oTwZwH^&M;i}4@9ZAP`(-M8zg7+f_G-3td+TW4
znzm)IvFoRpT-WemdL@_R4(B2++g~Oj5{XR|F}$YTdP#^n2$*buyjST~1bgQsE%ROJ
z!2GFmf;JR~<|G@6bXgJH@V3(as(oflu6Lu~c9f;k`|KbBO*Pu`N$~xjtk9Sy8vP=!
z5gDFFFf-;q$d#NFhAPQBEN_)U!67fsyl|VOQGlwNhz!Yq#8G!G4sLfGqTD*=AUPUC
zuONfTLUiunQO-8QNF(22w_@7YPZ>TVaqPwz7&)d}&?MWl2Pst$VD3vO{@wRFZCE}W
zKS6EH45lH2l?l@5o5Y3%RvxxU$v%rvo)um=U{J>goGJ3jQ{=zJ+J5bbUFd`K^W-`J
zGsz+z{<*7kI-T-HvLeL^Cj09PJuUaowInwHn;#6M0!@fKg_^*tex58I!s>dBbvcI$
z-%A>94|B9po#?i_Z<3<ki_E`+H)0irN05?hKM-dL4qPZD@qV*Hvt@`DcZbw3hLz5F
z^AN?6C&GrM()MOmyGTT?PJ@%ciUC80`2)hbTM0o;FhtA05r|~?C~;c`O+<Mg)$`BJ
zwDx|nQWsAn9g-5ww5smtC&$~EWg;+Xg7iEddRE(RviWLK-wp*1#E&hxv_S;d?a3lx
zwC*u_D4D4ip3Gho+JQ-(V0>;Iyg0%J+yPla@0>h58myCLXgO)2kCp1m-E3b2fpjqH
zs%co!KUpcRPX(TDQbHmIwP)R!V@1uQBUB@M-q_Oz1ZoJ>)L(PO5&>;KzJ4pL7FXp&
zW*UEczr=}K9B%wU$zP`-yT$t_y)CB}{P~Z8ZfLlL7vh&+Cn)jlso62!A%?G~RQ0J3
zB{)=J%^CDDqWy_?ej9X>3p!SYcqF(UF(~2K)D3Lmz+)UPDX8{z8*3R`JE>Sn>KJwA
zrn9JzwcA&bP*I4y`tUiI7RamdgGnjR+{epb8B){VmsHEkxD2GSF?v5uizf8^`$Shm
z_wBg{7SMDmCP~MauAOq%U=PIz%C~47w&H3`qMjO=A}5x+)enITD*R~@icMOBD8tcI
zJ{m<-h(`Z*-Szlv`#atij#+!zqNRlVK~)l0S-o|&oRoe)X#f3N*09`1Kyczv`_h9g
zV(|BkNCE$IqpcHLRBci#t_ln*x@I@Zz)4Rvfq)UBeATh4<yz!WH+BOp_c+Xp@CLF;
zo^XFM4EsZzBxSqpTCPV!4^9tStIR$b#;&iReP@`;@HPIA%zxIXVa3bqlsT1PBGr%N
zuDkI>cNtv}7RKrfD&nU<$BUcMMk;94!&QSEY6SC}zh`UY1XGn#_3S}K5x<kt2lji{
z(;!KDu<c<KuEY$G>uEN)P^!~XPjRe^QWrP<8A&AY%{CzUh<~&KH#S1qcTR#Y`|_J8
zU!5bAYIE9%>BfCF@yS^D29r02$tKx43I(=?W|Y^J&<@4C(#A_{oQ6pykN_RpQ=oIE
z)`IVxDf5~9h|e4C6qu07z~egN1>O|PA;kiOu0pXGfHns0!g7Jg!Gqc%>>m3YP;GkD
zf#m${Zi7%R>Ru|gbIc98D;`y$Lo%DzF*(9yK*>6eo$k&qEpo6RN2#$M_KP^0w?j**
zNF$+k+*07O1O{Nsd8jeL-*=pRojXfe!dugL6j3s@T4GA???Ttx<+;lFNFD8$e~$Vs
z6xRQE42dg_26eM`7JnGm-b<u(?xh^CsrWoVgAf2i36cqU6vF8@i1^wM^GOCML!^XK
zyUia_KVy~|yXxpt1e!VgGY7a(3D19SJX>eWs-B}jvEYJ?Wqv1jSa)by(n-NQ06@{P
zG$|!hv{UUHPp20(mG&v$UVCQ+R9zK@v$i$ge*Y9i6!+~*PWt4rXPqa}JRQCdIg2GL
z=cgTX-JJfL@Y+Rn7HEBZms-U3&9aCe_2-l1&E{wQ3rq@M3onua%_$a@<-kqs-LtnN
zNy+xNWrJ4ZA3RN|Z_j;SVT4=WUa5X}zNURSf1`w=BFIfD7Kk)l$MP<xC;O75=3D*{
lX*(CYm@5W_2IajunP>A#=Qsox{_khEvb=^|os323{{iPcHWdH>

diff --git a/modules/web-console/frontend/public/images/preview.png b/modules/web-console/frontend/public/images/preview.png
new file mode 100644
index 0000000000000000000000000000000000000000..76275ec0ec2920ff6c968d517f65d521565da8dc
GIT binary patch
literal 29829
zcmZs?WmFtd*CmW3kl+M&4X%y5y99ShfX3ZDNN|UyaS86ySa7#)0>SCv9wZRl$&hEB
zch=1MzJFC`)!loaU8k<pt%}i5SHMIgMMFS9z*JI{)j~i(1|lFJ(Z4}{iBv9okiVSV
zYpCkT{r&qlFfj1*=g-H-$15u<2sX_wE-oi0C&nhGzP`Tx{{F`KPLEGd<*)`lJv}Qc
zD<II?!oosCM3m5*6VH?F&-Iy3w)KY$3vVy)+lSlhn`?P_`41mH$jHc$`tppfeOFdi
z#&%~uKR*|V)1V3vIJ)?)s;bsB(>XIUlhTtbE-rrmaDRJy*VNRc1TxXm(mFgmG|06l
z_vh{F>uYOkV+;{DG&Broj0q$rl}XnBzB%LQ=-AcOwfpB+eSQ7f(dN|D6buF%85yAq
z7Rk%YBO@c*+uMV~;l|EQc6N4FC0;CHQu}9zH8nNS?dc)Si5dMx{4uJMQc|kv=JR`S
z=c=Hhq9Xfpe{|RPmzS6B?(PHx1d))!^78WTd00$L496GQUiirQ?dAOZeC}|$Pkn^B
zxj82%Cm>SMvn=(qlnfXQ-q_qQ1O=)mdeTtS#C2qRX-!!^TrVvx9sR!N8I~L$AHQ|7
zx4pex0a*+S3uAcC7*?OBOQ#~NS-QBl5fKsb`|1=E6EizIJ1s4(exe_bBlS~}yIE$4
zR<aMMqgpai=iumSadB~NZ7n%D*{wRnqc#kQ$1u1iYkXyngeQG^YZ<yRRWsS*R|(d!
zZNy?p4u}T7a-jM)-)oZ}t@5#kSEY!LpC8h>5}N_F^!lcg;`b>#{Lk5KY5l_H?s<Gm
ziLbA>OL1bwXeW(G)_aK@V=<PZp;meph1JbJQ`>X7y^S9%>u6~NXvEEa%pZm(4RbOu
zt{&{jvC;XNiE@MuFtX?c2G~|sR{F;bq!oO(QL`}O)KrLZZv8RhYU04F<NDDrP=|-c
zp)fA1^oL7GH;YW3w}Xp_s@1^qn3I1%fQwFR&$f(Vg;!u;@4|OUy>f3$vxu}&J+}a3
zD_1$|05eGgh4)ebNdtM~s=CR2dqWv&fR>#SD>?hEsj`UE*LWo}=g77d#nf>O=Qi*1
zG#{W+5C`Yv)>&S~Odut-n96mqvI@Cxmj;21dEpQHfDRMetIUFP*JuG|1OzvEC0QvQ
z-=D|1{jona@FN5GwVIgn>d>_Nb%~s<8Kv|4oHE2i1{}{3dJ-H2XmVc3(kxG_y+;l$
z3@emcVH$G`Q1ALV@jKJPp)CDQ0wpiN=9`ef&Ng|{w9hMqF(Lz$5EM4#jl&Qif^8oF
zp$_Z+Cm@L>`(b(2`2imnmkpU3wPV?o{Zkv?8*r$A(ste83Lg_ykx}fwQ^iKDN-|mb
zZY|~+AXicO1KqMww&^QaUrzDTy=1Jlf*ql8fc0hWXnuC17<8jk(7UAUla;{&#hK=+
zn<8oO-%zsz%u_9H*GNzdB|QSC<~{9r`Tg2PxxJF<am2tt9wF%_Iv8_>MzeV`^;rCS
zU0LJz=80{Y8_M9Udi7uP(d-|bRziRq@9>3Fr-XNx!L|!5$+Rgk>IUE#nk+U`&O<9>
zhAelIncl%u{S_K_l5f<zWXuG;?^gL^ew9700M=W7UW8m7s!%Y3mALPy3AXzTuTp#2
z{lPivPSa28MueO)LKgO3?IH8^?M-eH&}n+4QSZn`ykzck9Z7OBmjH6g;<Mi}xxZb+
z3{cEUEET1BVtQhM^7~o9)fAqDd$V6R`(Ab1LjcPSb*9FNY}6bpXUCtSK08XR#8a)u
zDf(9mFgts%M#ovo99D6SwIQ=HiJglzA~Bn>`WdVBs1OSWGxHXkDBF%!x>-QYG9eo>
z=S<P7rWKZ|&1t~=?~k>qQe7B2!%X0DPe9wI&EKa#AYcm*ds8M*@=^#?{N)8(^-ys=
zUvqNot5wqYnx)OJq-#rp<dRk$-iN<NDAxeV;WxpS@F=Lw47<uJ6`gGuQ`R(lTBwA3
zubRyL_>;NeUKWGRlq4E2>|DA<*u1Z&r2R1^h<VoP^CXwsSh0+xsImf2!z)H&KVMT6
z^%=l5SXYT_vMrv;$nBF?e{2e@e%#W9Qt?ouy0-|}`rdc*`ITVmT-g%|1=0+=$6}cs
zzu;y+AbI{t(M@}@f|#P)LUpgmYlfix<Bbg4_B;@$YJL}qaQ|`s%lYtW($UrZK%S#N
zOU7OChsO#eEigr~rig&!4j>smPE@7#&L}~96<ga&!uE$5YSlx(hb3jwcew!XM)lyY
zI9J2_cM0D~<Z7RdR<T=7>5bi1p(%fuS`BCjk|D>$3Ar>LE#TY*)4~Ttb&aZbv#ON6
zpkEp4Jy%_6Q`%K&r(l2Z!gz0A=M3StYU<=|rZJik<9e1>4{>VowF`Va--RN5Itj56
zcwoEZIo8y;l@@A(qONhwtK_!Gl0X0cc2K0om5R>S17CvTgz#CXq={{CbpT=KZk(tQ
zqaYA*<8yysxget8+nyAJkm^@p#6m<E!Z+cMBnYGp7$Mcj{|N*)SPIm}AR`iwQW2P>
zG%N|dwv-edOC@@zcs4_=8wR87+S!mY5cQ33lYX23<wO_#LFn4}GR{e>_}icn>C%$*
z!=q%xC&2bDb{Y)}g`Pb9C!lQUAT*l3CceTm6`Jkxdq09Nh663VY#jxcp!9d~L1;AE
z6$}ZNz#fa1MUO&Or-lqiv#}>MI(8lj_bD_THyw+WF0}zrH1mS=PETG(xF%k402-|v
zm5!@qK%#lj0HA%#O85V8WmKvzdOfO`M_?CWv)TaLuIkoEx8oxe1s#aq`7{lwFgP%h
zuP{^JbM978;8mUK4_~!IFQ;vJk3Ic;6TJT7WGyswGnpVX688ue{1W?#f|k$Re?Ffs
zC9!BlXR7u)w-zI&G;^<-4PcLOB0B-C03mr5WStkH+_B8!C1W@ze@js9qxyo0-B(_<
zuzIP^DsvnCeZGOJuhzDM6uGfs_VbU$qt44S@X4d?Add7axFth~Qge39D~5r(BkOs@
zjhIs2zg)S$li8IoBWF4&LC1^=KoUh-?xsHzM7kgNyt(Q-u63&O_4Z;(_&+&xG?7v)
zKd&78X6+O}`n-{85aIP#$=Fil)Y-ITq)1DqYA?n%F6Kwkk)`^~<KUm*sO=(IB1RkQ
zGIJr9^S^poceb`iZXKz!8XvES8X~X*1y*VRFo^dR*NFTnqRiQAgBa>sw!4*6PO@rK
zbYDPC8eU;7iAQI`aknEa?M7xA?_a^(lVlDknW{hPOp^sDkL*I9A?|aN4{EL5t4|1#
zy$n*zv=)1rH4VORfK5e(zT*&jT4qZ3Giu%pa+I#69GoHTK9BEI2fe?jj*dMwMEr5>
zGSt@g3%|vC1l)P*kblsrW%uCYlS+?-M3cG&R-depSeDsSb}i8&aWe?CLdp(vY~sOM
z-%-|wyDX~RNDnOWSL#?*+T4xmZ&-Pf_;Zl%+_Hcv6dGCDV(CGb^9i;<-Ca#XQvq1>
zf)lPjd|URow$`z%hOtuORP4L$;=p+iv~}?z-HHG8xh@I0rlA<!<iW@HBL;jEY_lW!
zLT|mwj#=<*e88nF(<9wkYQey!9%6GHn@)(_A4W!(?#^QtV;8iY(Qa;I>nsX$Lpp!L
zZ9aR&KT<V1c`;4ehE>gDE8UzAb(BNG?XhwlpWt<klZ*s8>gFc`>1xF;w7sC~g-%!<
z#8I2t=lXJRjob`Asktr5o;vApNk60WA<vpQCC;oGPJ+Suz!DcA)wGNbZ3P(4jXp(<
zch?kYo#Q-D@aNRhdkB#7KWw|cQ!%N3Uk9iae-G2J`;R)08~`l2`Uv^MdC*#Da4)`D
zemPuyHEDWY#dzm($K7*nRRs3@+;!|vDmxE!EUHQ{()KW?Tv(+yVmG>4Yfee%ZO|QO
z;pv#4=F<FWe9F-YT`7A+C3A0C31kryFM@36x-BHOsk-!hx}vK$@9`8@@IYLv{i_KF
zCZqicK4VqylL1O@YLIMv1r)kgVEoY_(Gb)@@vXpMB4<Gs)j`3>N=Kw*K}L&$B4DQ@
zM#um|qM#uIUjhGVHaCgzs67GrNK{W88us7V{{;nixt^>Ou;W7OQ^5Kf+oa5=Y^zy=
z(gLVNR?@LRK(xdNR^<UENQJB8Db7`V8B0KX$2su>eLLO1QDwVy&G*Qh>rCJesy83r
zdTZ3r*c@~1@=HH~*1Vfsne5AP<_Ieej=$t}Kn@P`4WqrS^QldvRyN;W<R4P<!U-II
zN$P;TM3Cc5OWefhSpXKMU@P0+!^M~oIFtuMvvIo|>tcZ}#G1pkHR(Or5`X#hUL?oM
z+jjPh%{q`i!vXFhD?VdOo<h+Se+)hMv$3)Tzc+f_7=tA~u`O^g_otftBl_7&>8v6z
zwwd5k$!%Nb`TDP}%ET+R;h30xJiOr?A_j)*>AhJB5g&4Z$FFGlcL?oswjirT$a<d$
z_V%j2wIMU<xCosLnD3}!?g!RQs5Qy=5;<2#!-E_*^sSCO5q4Nz9s5bI{Rs1@+3qh(
zBmVgLEH;^;{o{u{<Yt2a^bfVA<ibUoxi;b2{Vm4=uIqypT(BPclpHg6@Ov|wBH@go
zyQEt;#D}N5=dd6!!LnIRTihsZgu~m3H`%SEO&$>ga`?QjajQSEn+)@1mrc-4TcWp7
z{B$w&K6u^p&1&RrF_|tw5>Ye$)%b*+fNvAe@5$Kgy*3}p)_8KUPSpBiftS2QmB__v
zv^V$JE*2=gc>dUs;AJ-m^059g?+?I0u>A;ESiF4Kbe!j28yS5R9_{i#bzeW<(})4H
z4UrYeO#0Xr{vI)HF}9+JGuj^SEV}5_{|6_{dViK(p!dzms*T`A6OgRUsjCautD+R1
zP!)(<#(nj6eKep8p5<N2x3u{?FAv<vIwFR@-g5RB%4jR8;Axzhqov@KZ&*GsWe8T@
zf7gY<Ka$jhs{5vYs!6~+vUF^uuOGMmchc$`RumZI6F3vd{yrvth#Vk(Em8DSq1B8G
zjLY}7RJ7H&b^%kBG-f=EGkixn=X+wS^APCEt8&gxU{1bxFl2g(dcs*}I5K;_+;kMn
zP`=Zt9=GxKkgfCb@!eK>R6ycd3Ar$>s;u)y^jRGE?6~vY{+*5;2|M~*md7`t^fjt%
zupV-2EBK-f+><q5ekh*<v+alZ4|Zc>19_vlma2uJYi?P3*pGm}BThqZQNT93xJf{V
zMf0Of6Pa#{BREC7hUa&x&-J8n!B)=OW<&YhZ}Axi*Ju>D40`K-zBL`AveR+jcAFc4
z<Kyy)$AaBtC`2*ydE@tn^GV|Th7NSMsr3+PJDHQvBemN#WTST~ayKhq7x6VE8`Di%
zA>ebCo(AEkD{p8IVIj;B$l;zG=*Q3Ku=LQ@k$rXsm(EwVD=|!cl4V$c-6c%`=GZ%D
z_08#0vW=6NYMrY_#~_ZlWTF;z8a0-l>4zC(ZwAC+Yo~!pzFxg<H-ljvn(3SauZqQ|
z?OJ5=?DQ))g0g{DPd#bA4h5?K(iEG8&2*m2$35J$MSGS+QkWIVs~t|ZZIGeglgOU{
zX>GDTo>@Bd;v3!AU1SqJKtsGmf^MVdP$p1k@((#y<#etlaSdp;Xo9WyvJ%roO!&bM
z%r{)+Y}m{X#`O4eW9SI>ODp2?^R2vr0ist4i+=WoQ{8ip15AERVNb4mip2S<&NQPL
zwkorl%(8C(=F&^Qwwkvlncs-3C2B1IO0@Pioa}fsCwFM-@)Wkn9cp8?wG_UTq{m`2
z8MDH^)1rXcdZdhx#5bZvAIMiFPvgCmgrD@rk%s(a8!^n~z_ge~cOm*qNkVA8D9$v7
z1{>GpJvLjhfe$GnCTzm}<FA(0STh4_H&f<em4gYUu`4#%NaA8&16z}H9jzD&>=}*?
z3A;^n^fwVcLjCAn*Q+3BexNFK52=ATC6Xefo41dS5ow!oDXn=JpctAr<JzMMt3?6v
z(;)mXCc>&=0jn39^Pf9b()H}GuLkmw5T2Q>NlFy}iHkrY{ZTg9W83EG#Vpt@9B*l4
zX1|$zR1-E`%wzRr&&~*vz^4r~lCr05GqCwd!cn?UI&65Ya+;wO%9g2PWA#I+WNEPG
zjS04(xd1&i7dcqd(d%aeTmC$Iai)IcP|3?ZIhS(U&liII$Oa02bNH-EVX%O2(+@em
z6?_b0gKgya_V0e)%MgFyHbhFZuEIhw*)gs8)j2AAG4g2oMo-#)z>Z|NBWkxw5UW4Z
z<ej(nu`ZpS?hg0+&I~9?{*^&SYPj@;7fET;B2cXoE$X7Nab>L$lY$yXx|^%@r8mhT
z3+5ik(2KXPhNMefH7ZxBiNZ8ID1{~1JGSm^B+u=aAecuQwH9x%>7!lq3w<;VVbv83
zC87KqzeLP~Zayt0ue^B5orqjA+{Lz0Q^*9hAA~5I7`-iabc4;DHPjy)UB~2I_&Jec
zY~L<Y6-H5eXf)&v;ILS*qKQ-u#;BrP&4d?DkTblInvMG?h358gQw`hSmUw5**uCO6
zP1K6J3+EH^J?G_C!N;{7yBlvqJz%xN?h|>(`JbImMq}}#B&Vb=Y$7~stJ@~xsS?I0
z0G*vRpAu;XPLu<%6RXRbGK}5|PA*sqjQAt83o#}<-)?VKg*1T0%mN~I+m1K$ob0k>
zpVgj4**o2T_XOU^(6Z;01EEmXJIZ2NBYu6@$$IkXj!FseqUG4aFDOXPh#>_$zOJm2
zs9@Bps0gE<fxMalj8|s%0>>$+jRpGs+9JOzC~5^ay5<Ns(1Yi(`|NmKzl7B`*_)hZ
zlI}%nDyaTNX!<HN`!yF{lB?hPb>mHbftC=3axp?~@jr~V)5!+MO@1$&i&n>N?B~SB
zCGF&r!Xq(L&f}_$3YNql^)&j0_x)K$3slmQt8x?0?q6I7#2J3%f~X-=2n`e5LSELb
zZfwl^zyQg~x+NiAiqRTstB)VYCK|&2pzp;I9LkoJB0Z4Kp5qh>@8ZZ$5&Q5vBJ5ij
zx}(0~hC%-3=)fp@IC}i<7Qwh;o7t)xi&D8Qz=V58SoB)_E80<nUvR6$T`*|Fg-zQD
z!L0MgHO|(9$tP4|Fn(uZ4Z38FM)`0iTJzB_brLl%=vO7<31T0Xa2nTO@Oz(-6;srF
z)6P4g=HvT_mV$aaO;noz;FH4L@aPq`z}YkR--70HAJ8BL&Ga)in^+|nu{|#Vj}5)c
z5G+&%4&Adw$j9(pl~%2_I86Aau5VdZ*@h%bthV+^boxrl?yw@>Ntn{hUo`6^gLl@B
z?qq+w#VyDR5(AtdPyf-Fyg2-pc@}REACDG4&zMl3iE*Trmep}~t^A9vQC0N5Ts#H>
zJF+}0r1DK6RrR({3Kb#_>zv_tk0ys8jB>VO;oVmRl&h5XC|+exe-26i&H;WRN`rS7
z_&I|x**jaj?Dm&^H5JB+IraJV%>zw>{N6X$e7mNg5=R%2-g=@^`OpE(sb(G4fw4Ii
zA}M1tii2Ck?js*4K{dC&xQhp^Wdn!Qv=Sjn>Xedg++I50!#h;+MB(KRBZUaBH9zO7
zdUYnA3_&<L_+2!X2fl!m?0A!saf)qN0@&&_u;wqObxeKvl%vZ^mAASyxNx_0^7_7J
zm|NLb1iuZPBsFXV9PIE@2=*9<A<Q!~{s{jynr2PzosE`i@YA+p0uI^1!hsya!^e3>
z5>N}}C?peIIM}1Wh+V#Ccy-QiMWk<^lm2eZ8+8DEgEkeifpp&y$_Z94jXVS9Ae*=T
zJf6K@bYLuXu%+9^;-Bcm%B`?$Z3W$HX}Ss?<+2BPRD>&qah>{|vV8767AEdyT{mec
z{ysjQ#mX~h3&Fozu>(Gvg6MyJCC;v!x*yR+W(dAf!SZP)jTs&O>7n}vTg|*egAs1^
zbl4O;0}+gBKz4?g%&wwxU#-5)cAWfITggBZPROx4$yrfip8MTu6|GTgV^fd<uJ@Or
z2MKUK9^2r8ayBr?M_`8S^SNc#Jh^#yfR}ZtF}VK=_%wi>_vq1JlL->(qXk$^Vk-`I
zr<c)-1)s6@S~kRU(EwvTV&4u`=+)Y}MYvZu$AHydj6UyzU||(-qolZlF(nT?N9%XK
zpn0XL)rL@|G^<t+QY`*H$WkiT?&E&tw2;^I@8PzBZ{Og-T7#Q>oTZ#}K9T>`Czb}r
zNJ^u%<DMuYeU<#g<ioV2$HcB<Bw4a+;f7!wCtXTGhU?T)+Ol1EWz=HCdBv|@%_A&c
ziQS`<=O(W(DzF}S)-PZ_k^hv+@pW7}jVov8kpla`r|0RKJbOL&``&uf>-+>^1T~|?
zTtJl5GKTXBuJpra>Y3dPh__TG$+-f_mvxC}RK;H58b$aqXf@hjB)`X8on)9hz=&&W
zNVhXtJ&$p%si237^O2g8RpPLqe6^lAO}A_@9snhhxWm?djNp7l0vsS>3|V*SVxyv&
z2PJtJz7{@}@Qt37^P(4XWu|HpJd5e8BknFP9fSI9Ue<WtY(Gzhmc!GJ=BW@K{~#gk
zpbvQH1972NmP9eu`uSl*xUo&bp2KgV9Y}yh`F}2GE@qICq*b%dDXC?%MzrWvzcC*<
zCmt1CIL;wbw-B-0nL)X1?86+Ba9aN=;A?xYS^TVc>zULj^Y~D!UxINX@dyxa2Y4Nn
z<N}!0o$h073eqk5_NO-6Ke(RHq*v~PDfj{o;FZ;&ah6#Jd6Heazp3TfzC>H_DKWnK
z29b9IbI7oh9c{Hc!CeCS_VvSpS-(oGgS1K;bIE5J?7tWEx*z|OTa2!#&-pM3(l^>&
zH8nAMy<i$s%C8e?+Sn3-NQ^GI-ujWQ+rf%Og7ZRmf0T;i0hN{!!UL{Jnts?gMHpV{
zzxf4#9wgu|f<G%{bONj3x?X7zFcKm}OnT@^Vy@w9k$rDERxRM57VnEN2b{n@Qu}(g
zSB66XSF$g<-t6%~jw!asx*@y-D@s=z(ZVQ28cw8Nrfx<1=56_UfaX_gU)|3#2EpD&
z_PZ&4$AL{hE7vMe3u&R`;g!*La%ezJYQ$iCL>sh32#E*&3gZsQ?M_)2G~9)x(Q05a
z#Js%)ib#X>$TF74$7vJlK~21va{(OchJwsw;|Twaz1tE3n&d(nv=(gKh2y@Hs*~tz
zfP~y-NpP3ZT$57jfhJPg*6IZFV)WGDVdiZ#Mxaq;Z4TW8KOvFvJJz8B=PX3g^fw3$
zH&hJAA(aK)voro@PT85<Z~J;`u}dP~1Yw`Ml(sK~0(2&EfU`J<Y@}z5?`?_XpoDH5
zyI6g56sQ}4jX4mE<e~jFK;@3+@r%OL4knd3`%Vf<D1vcH2Bhpc@^uUVL_if@xlTcZ
z_Rb1K;$+uKxQH_3^ML_BCtsRAR(pgX7;1}+V+3PCyYB+EAG$Y($&$P~`dFp<%E<d$
z;37nD*5a?UV;xXJJeH5ysV~e}8I_{pfg0&3#om6g@Zq{&Axmt5ow3p0i=fV4hdw)E
zUDc)j?|Qf}!7CuUQQKY@h`@)ae7><@bv519_V>5q2qqQgQ05+cTp#etW|^ug@wrB%
zOZ7z1w4z})TogAG@;k}Wsp^jmy!+&DG6+Mnn5fe9tC-<joz(dFpXrnDic>=@a>eJ2
zq)G1LK$IH!UQ{oZH_)FdEB4@eU^>I8k)FO|li{9~1AOaJHSznmejoPmX#j3Q0W4k}
z;Yb#4?RecIaoU+<WbS29iK$Lh;r@oH_>5^^+j_(PGC-qTT8ZT2c`(OOJrk>g0NH^D
zyyU^gq`wf|r>?N(rY<a(sia1Xj|*rXl<>Pi>|%&7_sRXBw%+ykY-wBF=25B5vtXQq
z3p=}M8SnBdhN>^yPGzmZMUHh&g`F5@E^RI8A)ur{YNhh}gTB3|dr{;TQ(@B&s11Ap
ztj25JS~)FmLq6#&^t>u74uHbY4=P4@{8!a;r{@uVK@O)p3yXDZg1+dkDRe7b)C7=5
zurFvGz20$Qq3X?luUpMIpT3}-m=l*bC6mggKELGqt1D|N<LJXQn*m?d#H4hVQtkbU
zd}YR#qoYpf6Xjebjw$y$;GT66+oUyWvgY^+_sJ5%nN4J=qC9*@=*{r=Ywy|icA7%(
z9OM+HUeGX+piTTS5gRlE0_*L*LTv2~_gNdd#{P4Jhwx|e^VZ?@iO=nc!y7E@nkd$!
z4EQoR^2y)#(08aQ?u_Cc$k*as*Gra3-SSyV#Y=|l=CC0j-xkEBoHil^f3(WH)o}E&
zAf<(caJw4xu#`mT&$PFQf(C<!(dn3`it0tsIrQV7j0QYc-VS?vR$VC~thzkM19Uq5
zLB~NNQmMn0q-5B6K8~|##ASvJ_BvU45rDMB4aLeibi(q+7y_N&NpE&nu?xM8?t3Nx
z-*6FxAK*7YbS!y8IRav!oyEx59nHu7nXc&wZ`hkd)n%NRa}|>0w8lQd<1b%c6=MTG
zJYH9L4&Odk`|k0yx&HoMAn~LXFZzr|X?lMm<2xrLff+!C;;Oh^S~D91ZcRvIt!v>R
z4P*`gkd>%*rx?+=oE{msjTtMCvrsu)lBcJmOBThm*hn0d#`}}0v^6`wtNF3H=J^C>
zH#Vv?@)n*EBS<yEq#obaNl1vA4ET8T{l8fOA#0aUTAT*C=Z3HOSYLtUp4D|ey}Ua7
zHu#1L8vN+@P)rFm6GKO+4@`p50xja2P6Pc(O5(R;Xn{sT!1vbF&U>!o=Phg`5ji3I
za%cuBr%GC%lG?;Hg1-6(8md5hMe36iX`#WDXL@TA$0}Ky&iJt)k`*?q{XuO%A>acP
zdo2u5H8LWz-|h21qU0t@NPtad!kwTG1Ufa#Wi@&#$}zyCbaN>5)n*sE>WK02sPA1&
zZ^IMvW~criXeZVt+u9>huRl}gUSHBObK{oN_VRmYD799r?@}ENbc-TW@s*M~ye?0%
zod4jBce;NavCm0l`D)kuCa<Qh(%j;#r94>$Zo(m7Yhm3wctU4}_5x3=fGm_6;>w~u
zR&F0@IzWel4IE#D)oTYC=2kVH{qP&*FCFyY&`V%xYKqRP$<}a6_V3@)64%ve{KhCD
z3UcDfy5B}xwS3g*T0Gk`%njPjvgsMsm*8Z7k_A1rH(xFke}qu6NME?oX+~E0;c4!X
z+KS8tvO=v#uf*mUTCDdeuU4KK76xa<&qc2(QrbXGD*C}2?}cNT2KSD&s=9nk3ViHI
zfX3LrYHo1R?@+p(1$3+Hrx)k66MY@4Q|2>9|5L*P9tG;BLPXW_>2O1)CYAEjga4yb
z5U6?dl)Qd3Nm_p0dfH9a?~4&rk$vvFvz=t+2DJ!-9CFyZ<UqLd=DFg3L{~8<DTqHS
z5|*p-fe1obm4}(NF@e=*+$>ez3QLa(v0l!-M7)zNf-JaF`SR+9nM||a(3GXSz`8YI
zkb?uRJ2Fn^=?pAZ!FQaku|k8T58dqNqiJH@s93Z|A+A;Erb<iS3lAz9jqGH3C+CG&
z?Br9n?RG#BQ4sOp&2&}{SpeYXjle>=&GoUKE4`T`zW<5*V&#Ok>0D3BV)pR?JDZ#u
zjsIk!HJ&Cvf8=H&0f;%y8eiz26knzz3bbdBEJ#wLT~C%fla{J}I?bhkwVxT;6h4dK
zJ4BPEh_j4jKK0+zaP9H6rg*CH*Y22$m^RUnAX5Z<@_5yq3Q#+t9S1;wZvGR&+@75Y
z9`ForpD4Sp1r6$L+tM-`W()%b#WM+;$O_?q68d^2FgbHH{nVa?mMB};(}mlS)^Q|k
zkC*~DWDk78+~~7C`Gr_?Y<pDMt>MY#-)-i;Nc&xG&$>jo21%!kPg75KQR{00P-gy+
zd6qwgUHDszDtuJ&zsUaP*e*SV8;Y{e-Qyv3Jsp~iEwUt6iMt7iwRiBlT3Hf+0*sSI
zh<MbTiV<p6Czl<!@?Fu(;B|#6v%?h!mQc&jDs|V_Wo#{P(k2TR06?l@(I{Fr#-V5V
z)1&bA7r~noxRSxcOq%lD6uV8kSJFZu?$|(_MT*y`bb;jQJ4RL>DV-#h>aE#z_%0a$
zhT;V*r2#+VeqIJAJnQ(9>~~Nc%X1a@5C+OAhC93VHFqL8M^SDLu7bMlQ^IA3CeF!n
z`G%qu)&7o;1U<CErv`oA^1u(_JpLVUF@cy}*PY9vy*Sk*j+dxCC*{R%9aJM_SMf)*
z+Pz%nU_h<*tWC8in-%rQP6i}5J`8kCz4(%Y<wYRR?h^BWyY!(;$s*CEu#7m8$h$O1
z?E=rs^t0D$wO&B6d$7ZQ;*iW3vDy20&85XC!amP=%Iy9BlNDAqLb_c7h`r2Sshe$P
zD?Vz3I}1fe2QmB26VcS+tzO3)G_>MgZeIlp{7#(39LRN*uVPg$vi4y3PpN2-*8(j{
zsWp`)(^CR8afr#Arj7@=H)pL#>c@pz^b_Vfxw2&li|6*qk$ATVsQZW?4DAr1IWU-~
z&qUGYbyJtmn%3LT+$ezQY+8ESjBGe3u#SSO>oudw-PAy0yK~CQ6knr*#;lD)6iH&W
z)JWW-x$jEZ2CX2T_0>sk#*BBwhSnG-3rsfxGfP+t6@#;56ndYGVT6PW^P`y!)Kl^O
ztE53s3Go0gUs569=F2cW*CsA5@9`D|?`s~F;4wWF9W1_ml~jO$Rw%}rF)@KjJG?wa
zn~(#_z|cS`E?*SC0N{@LnYmn>qB%0scUz=K@G`NDgM=y7L?umA0UO(l>B`xRb_*Z@
z&3*T`#4UO8kYFmPg>#$ypxlk7qqn1D%`QkT{84-10BK=#^xGzfd=~3#(sORe#0zMn
zKV$0nx}xRanqGjg4)wn-B<JAu!tiJ`0M^$>(bw3v@AOefi_D{wZ*TFJmnp9ZKfk>$
zwR)C)Pddjy)sT-vyl4UazwW74DzI&2>|Dtmsv|>k!!2Kn&xCG|ot)cT;bT&}Z6r@Q
zoSt3r?YsRW^L%It8dR?lHTs8z0tm(l1d2>B|E4j8jTL{>iV6S57u155sPGIrM9@M7
z24W|NfZEHrf1{o51AtI`@}&Q?CxwDf5oi8PN1pt&tSoz5$kv~_L~q8ePRN_ttoc|!
z`KWcR`Sx%x%p9h7(lXW+hwpaThFL=E_OVYZ65z4?mBC$fgvvQUB%jGa0CVXlbK3B3
zqQW6?Z7lObj77B6NEfuaf?U$iJLBw~-#Pj&Nn}dIza<{h+}{(UBU}5ZwP0mjffl@;
zlpesYye1dx$m@_CK4eznC^^r|5+|0Z@X;1kbB{bsMJ0=k<Hkah^Fn?k_mq1>yGmCk
z>y(ljvn2%fx<p5MyRVTqR1<~G2&d&-#Zb;Eyon~dh3$`79KO)pD|dc-?$0aiFG`ma
zsrd1L7o@g(|Ad162ky@coHYDgOfnjhMN-&`_`OBfh!c1=9Bp)0T60M04nS!G-<`B5
z7QL~ITA{uGhK!#566KWF(_$E&K2;&9#_{MGmHbet2O}tc(c}&#2cwb4IYG5pHB#O^
z8nwFD%k7z#C`Q6D-*OkJ@u2~CeA$*Fm9c^98C>P1s)QLLl$9B}(L(FR1200P{^jKk
zy`N$EY1*g;-wNLwN-11~ycm?~qZv!=fV$^2q?qOX@naa1gE;RGa7fHuYb*a|CU9*|
zJxj~v#Mi1@E8@TI6M)8V5Y9?5x*2TbvKhe4al)slH)8Hl^Q0w4&^h*#fE`e+*<opo
zBG8RPJxdtB*5Re#pjCI1VcKOSS4A~qr5nXPW2UfgNt$}!eF$>_uWQMGWWB=^+9{o2
zH>%>y!Fu9(nqH6)qo$|+t9szdsN(#~i)?0X4Pxe_Q`r>$utmU$Wr=-k(+22iYVB^g
zSvT_2Bb>T<(z0|u3r<*@M>}ItEz=c31MG{eBh}2nAK^+ZMCVeI<WGqgis*xc#>h}S
zfBLq$QZ~ru!H{iQ8MAiF$USY_bumfC*>y3Jf77+Koc=N>{+#v5x2;|G>%BR$yNCHO
zVNXqKqA-j6ICjM+^W|&`j4#TWG!(s`EG7hMWtFMg7cTb&WbIm(kajmUTmvY8nUD>l
z9weDlp!E&6R>HHPRU+UCBRQI=OhXxZFM5PUe0u`~C#7^A6v&M*%}ybl<qT<jq&$s+
z2gaKee<rlkwg!e6tYO1One1l-<1jSJauBulkPnaF%Y8ETBkoTfkc=8Nj(be=y9THT
z8}UFlF43WZd?(l<8ai?JEV6M=K)1DX^}CLYtp3r=L-Dy9@)9;0<S)#u2=+Oe%dw#-
zLlz+dVO_HpAoy{*WKBia*g5xJbCO%*gT_8DFouL$)v&e+x{Q%CN|7x&+fh~J?xCS~
zdZmZNiH(9v7T@^~_l;z}>7+I|8F2tK{8uO>*1E8z9}8cN#TpLt&tA|+x>g6lF}pId
z`bK@{gDO1oAww8AWi@L8ZZ+RXD*o^yOrE%QD!qI{RXQ=d4L{-jV`hy+Ww}M+?osfc
zE7=;pmI&FSKJMa4294phEK;9>vLUWMX`?SrzYUC+FJ?3JD&>mM(v;VaH$_>4scu-B
z9E;CI7O^emk5;b2ifTS_ee3D}5n#CXK%C)5b+}UINKMsyV88&(>p4$#Xm&Zr?9JG1
zNzyzOpA-6!WoPxj{U-#r7YAMi8;HxtK~_@!8B7R->0xPuRRHZt+`8|I<=~{s>!XTE
z|2R)9Rnv!THCrQRZn$TPLvH{c55#v&HLN2m8fa8>Vx<0@JS|DwdGzupO<<54iX!ED
zU65aIdYQsYX_Q@am!%A4zK04T578Ti0W4pt<(ivABIYd1Cx09XNQ*r5X=?Np((LiS
zcv;qv?^F)FFrENCRHtFwvOfG=*^UVf)EW@PXYPB9H-^1?W@c?hWtpuMpCtN%y3<sI
zctdDjxN4~JE3UA4a6#id_U{jrohYq);)Cfw0a-#p@dxzWS_Ndze)KTVo#|^yxd}t$
zF84R3UbgNLnx~S%W1?%=7q`UvpqG6>D1!+?;Z}rg^*vP%3@q_E7+k-G&GX`H-cY6t
zXxfbMt$je2T-JLY`*~(`)2AA*8PikrDv6AAc{Zq?@gSX|K2^f$Fq-Gi8$#U;+0&(9
zVdXq_U@jmF@twclrpQPg;5l&`ACwZ_$e;t+>fOQ!uKkFcKrRKjJ|3aH-y>rCx;SeY
z+W8xvw)G2V1yTIILoRE#;T^O&sj*DS&S{Dis35Nw2Y7PMxT}!Mk~0hgoLcJewWrR_
zvJ!bQ1sn|B#$+_KwB0P{E!~aY5^F~$)L%`S^vAexLC9ntMgpRIt@hW>Zl<=k9<~Dn
z&e{5<15JL*^Q~mb@0gsl-aK`qS35qnw%rISdS1SvDxEY*QznZ1ByI=-9=6SF&XhGX
zi}c(qi-nx?X=(F&A`(UY+V+3NDDN7|Jotl>{O=pGfTSd+Id3XTl4pK|*PS;TkHGx*
zOxmNI*lw6!d$`;gozrZTy@9yU?;m4|Aq-c{&u4K+Y2!s~G9WlPfno+k;u=e49z!fp
zY%u99=(opqWzdq}*2I#-p+|>B$r+E$$ViBJ&)Q%?;0r^v-90ec=Dil->AwsVc1*@`
zlWxq`Vst=m2|8QtCVoyB3~6$P)=b*5J~U3y9-*cju1;jFIL+&yZxTnD1C%vdajQ5A
z;N6lW36CmSMjx8$$|mw8Ns`9eQ|ZihD^nV`nVGh;=Q`YjkC9Xv;vf?mdW~-nv{805
zlAd4j=YO`ZWR>6D!ZCY@tZ1*n0rCcm7dj3d@r1f0Arnn*GgIP%&e)Erv{ybtIW`8~
zR&7p-((btn9G_x-^9yu9pzC}O3<L>#VAfrl_0#?oYZ~<=iYEk;oO9Q?oZZFF7R9${
zKLt9Qf*?WN-Q7@%fSaegcab)0r(#9H=G?)!+5A`uNl8pR`doqX{S-V8>b7*Lfc)3T
ziBzv`10C0?jN2oHFQRi*R+|k)gEybnvOX!1P^I$`uL*SB-eqYJ0!N-}FaP4qW{6t6
zZD4}Y7=jETOe)})>JHi!2ErzuDzLpO#Vv|i@jyX~Nu@fg7+O{cHDs>HiXf(nw0b7{
zwI3?UTV1kH8jyoh`%mR$Ws`!43HdzHk<0rdpKmukD-%x*H?Pz05#sAW;v?*=H{2(_
zh$@x!*d^W~2H2Z4DPCoNiI_~%E{S@4oH{rA<<8#;MQPN%Z+XuLG%3iNkiRog?)~ik
zToN3`wi~P?*9a2L8$&I<<c_rLU$A9hvQ&a@X(Cih!8@_V3ImHWG87#Sv>A??(&1xM
zv!?~Q5|!(pelydxQy@05kmn~NI4=5j9RAQYI&z!)3f;BSs3)ew`rw2fU;REX{$!7x
zweYxUs&RXJIy}Sma<&pa<!TpffBjCpfb{8#i3l`H@bgRcZ#fYIpA9TmcWT_B%5acU
zMJcX3fCfnH{vA2TWyQPJ8+3tk+JlF%sBuWW|6U7=h7o7RsOF=CWlkD1YhdeHw={mw
z;*vps(s|2!8^$^{>+P0h=at70y~^+N!WSL%C!A~C>b4*;f!??$@-I-nx2H2s0&CB*
zV5acdUe}xD-_M8Mj)&3nzK>#c*9L8A*APLVlzH>VLh=z^`zYPoYC%PphN@jH@56m~
z(~=u$`3H+p!$(gX1ogkQC(a(`4965bh!{&}*B4U~fB6-2$rWGO!6S2(6z#s`sw$F1
znt0rp`kom5X4=PG>!JFWMxm<a_ABX1hc5oxzsoS;?6Xb(R~C<o_3S|Jt!yj)D!Aa(
z>i2y`k9M9~Rv(VL#Ixw*yOrv1iQ3}d7&cA)`N#s`+~wh=S@8@aAp?pN1Sq-bVjLR=
zZ!;nHI2xG+AevU^)Yj&rBH9}Ov&gMSulMk6zcKNXAcu+Y-QZ*S;TZbtTM)(d1p!@L
z8uEcQ^ncXSDjwMD3?A4AUM_ssQ37dVE}}yPLp2T$gb_BnCvLWy%;p5lW|4#GOQGr}
zITP}ee#Od896n`U?}6tO+qm$HP$%DHPVsj6E*1FKn#wAiY!=k7_NK_YpDVbWmgcu<
z>oOs+hmWNgDHgW+iOy_kkkjC$Y3#CN_72(HXSDKY$gL$3kiYrDj6p&ZL^Ae2y1Iry
zPv0aHyg?EwIj8sqR8=<6U@Ml4LN?wQG)@h!KaQ^@W*k(~^S<$CCCuIN3oGQq^6X5N
zvI0jlN#S@oF01lt2z9upT6#<u!gN38;JfBauxvETI;1t&E1>c5ow8wC5r${Tj{4xJ
z@8QPjlAGS0JL#;4)Gm|yb3{R1QOyOskm@U@ash=Fh<q$23?w|3^#vq9R)7lS+R4s2
z(XZ>&6M+_#9_6njjke2c$qRkSm1JrecL>K6ymsu1oI%m?*Cdv0%cF`&ZNgimZ_*5^
zTltJvK=*xjvYBI1<MUmjn{@GUj6i`OhRFxMK;8gQi|p3}NnPyQeTekulzWe?oM4VL
z5L}rBUHCj6@O@O!aI1Z!HT9Hus&d^cS2VVGwz$}Qvx%y#tmi`shHjLY?RiT}Z4~}{
zRCjCYQY?1{no`{J{ZIf#yY<oP8LzbXi`poc-cSd6pvlzRFVZK6t%KshVkzazq;akl
zwPy!!B+&jnOq$l)bOI~4OX}hZ+0|ZqUX_s#aX#^Qq`(hMZRV(T<)q>)OfL(vs3yD`
zqbM86(BCrmiQJ(8$G3ir7x9pm%iv!&kXq-xPbl}lfRuY*D1FcOR?dS<Y5`R7tQQ*c
zZ@-ZID$elBMAm9uy|GLbs+o~JHt-#c;;Pe=&GPze)5Ht|%}0vXej?UyZ^|fTJm8_X
zvGiY)K2V>#;XM>#*Y@M&p?kWx|E)s((rI_R%uhlscY%gWuxEcnyXe43=0L;qe3<Zw
zEJa&-;w)gk<6K#**G%CcZ_%+=&Jc^aGzphGKrspu|LdVXIdaT~N&N0UQCvu+01#9T
ztn%_=Hg3Pr_}i7Jj`-4<layX_Q&usXm3GHJB|KvOEj3pt-*Y{=x%Hr=8Kq15(90(*
zkq|Dixq{D&$=}-Yg|Fc2xkYh#Bf%$^?OCrG8^wY^Z!*Y-q`;sTO*8F`tMMm;um4ba
zYkw}AFb?$&IA>Wg6l8~gU|NlLH$68yzuDIrOq4ww#F8Q1^iTDaJ9j9<Jy-(m4{y(X
z-t6khK~}#p2JOabJ?nLN;LVh%uCMzNnyVLo=6~~gw)|}QWqW>GlWf2H`0_?j1Xb<Y
z8{g$eU}WgXN8`u!Dsde!x(kZ-$eQT+L1X;~;hGYU4=i(0wu(mul7=%cLWJ4J|F@89
zvzgPd_DJ%Wl_>H^QQ8}ZgV>VA3A&j^=Bc=mb_oN<4~XT#Vz*%ynqWL-B|}XbJr~Zj
z=aFCtPVZIdy@@cTT`qGcZ&G^`I;avqd5Y^tG{2~m$fJb`wGwoRPgY#)`fB_`vFC}>
zmC(km^#6yxop8Q^G3}w@CRP3waDZUEK1-p*Qahh0NyWy}l~*wXyI}IggfCi}gco0=
zcg>bxM%$@%Cg!w61tk)!>#6E+Rl|skz(^J#Ryu?$WT^Xi^mQU{0p-K}j#x&i#|P;8
z!rZTgdmdmSlP0(s%?qrqa*d}fP;8=KC&zcJ7^tWrPiMmUn41cqOi5JpGvl3nGAvhr
zXXQav!2b#;)ypDquCMYfLf?hc@-y9&9L6l~Q_lI_KkHLQ&z8x>5zI2h-YSvYVdz3}
z$|p8GSwAVJG^KR=?aW%57j*4IJWt+Nz;rQdB?3qnP3yJ5UH_MajD>fJ3f`vfR9g`W
z75S{Ctf^)^Ns6OoqMW4r&o%9*2*;UX)x_KTfO>6Rl_(_Ud82gsy-|*-9WV=9WgTga
zPVA8+Lty-YcG;=R3iscIeihZ(_5y{^5>UDw-yOHy915|wOcyqI%yb1b+JxTCEnYIQ
zE8)lA>HgU>HqDsCA#b?z$J_P^K1<A@DF#-mcC|TZrkD8Y$piKBV?XevSD?N6l<52A
zuwAQz_JHpq+S3twQRH?ql`X7rD)12lO8(RGdm$3T$Wi55ofa5$r)m7nRZz3_d@`gK
zqf!<RRwwP<SuI-Utw<4Vp(z*Eer|jKSS>-y!wucRmLgVlD+A0hK21I~X#jbl8bW=t
zf@rIn^`9FPCJftMiQlp~N-3WW(K~TUr5;iw0}5{};UMM<iOWG-C<V0Vk-Q2+K5KQ;
z_U(CHnT6)4etd-bR0pv#t8@=jEL4m$9J_dClk82A_v+Wwy)eoo7?EZ@@q@BVklZ(G
zqJ~dI%4DK=p;$?6dO1tFX_+hoAkEjS7;|PsXP^g*6~CvD?YJ4z!fFahKbir#_t(NL
zZ>x%V83mY|tPFtJ|CT16d@%*Eu&=g`+5*ap4@a;kS3M4L@x6m)zTSAF4?}uSXsvAs
zZct&B!|d$8#m5@N(-xl%{S5LdkW!Oh*xcl%LjvMBHBJqFb@e#Va&bjBR4Wxb4qCZp
zAa^d%ENJz4Kom(He)aIId0R27faXQENa>-`98!is%nidaGs1ndl>-A&XUeFi>+^cZ
zzjOLPUf4*}pP7r58i_X&xPUR0v@gGxy=;h@NI9`uumD&5f{<~}m|I?EO7Jkj4J|BR
zB))l38-l=ij{3HMVbJ&MYW;BVxg1GZO(kvJSW$psEo&VP;x;Q@0Lktb{MI36x|h+t
zKeC$H!AM_M_n8y;wvx6h!)^ErI>PjDUJPK^xa^u1N;|KMm4Req#Rc_v?oFDeFxoU5
z5f_ew1kM;~jY#;)`EphxAxM$ja-9msfYh)m{wg$Z1Le|UmtcarvRd*4T^Vv=`oDp<
zQQbTSYUHs4e<XC{m6Mm!cSzrG$nw6IjwO?6e5YmdEi0ms+7N$bDUW(f`Ilo0CwKLy
zwIWXb?lX-F3S-AP?#a8>HCDoVH%s^1+{9r=r!1i0Ih%}&k?N29Umo$nltXo5^4Z19
zdcenV{nWE<Tm5*?r0GM#2M5^YR*RQDqT#|N+YgwEN2MpF0GmV34G()nOj+~LPsrmp
z@tU<rG6fD@xiuqsR+JC^%gsjC6ql1<f$ZF2S}EWwPKwWNl><a-FH9dw(|`e0$U_|e
zB?Db}{av3r9?sofeJb1sR??etTU3%1^N#`aeG4-V)HjS_8(9Z_Xis6b)(hYkr)jm$
z&5;xp-IO-)bgZmYJWGYw7xhzeqcTD<K}<Um{{e3f0_xAmKk-%pg*vy0I3kzCc8Si$
z=_z|hnFem6u%cO*#pjBbPfF(J5d-4?kBpK#VFo?&If!-Q4;|UztoLLx$CyFH&$}%T
z_6OA9dEF~JM2m7ty6@9Yl>q3SaL>nhvc{}&Y*i1V!8KnN4#Yl!?!il9F&dN@&a3}T
zct(`Dgi8ifPYaSrFB1~p>-q>&pM+%m&k<nn_#<oyut(ttXKoBi)HDRuF99OiMLs4q
z7UUQNg1J|o1g(-9iT)7KdF-4h@IH2>+?M>_<oyGr+<*AYvO2z-&C?n$6nrC-m9uBk
zuOP~cxxC!$o;@w_lkzl`@^pqe>k!?d3HSWwsrtzn?6BG12j(6<4YuM`F8vsS-QZX+
zL8iu^aPTR7DvdI0d^8~$Ib-XbKy=BG<N~{c%E>PuvCj0GdYfVjvrt!0#qg`xhIR72
zcOJ?H2Gh2bX7aEPT|^3cujQ+}Sn`<<dV}+}r3pG>s{VqQUFs_i7Imq>BZ59%^;azo
zxYD6G9wx_Sh@(1-+G7H8xD}K4+duLZ!3pKW)^NbyEzjNdWi1-D-O7Wv@rVdpW9r`|
zIC^*a_>U-c^OS#NIDTLy7>dVkl`=S@`cudKDYYmpcf?_y?)Mbb2@<cTGtsH#!88R~
zlvLDL(g}Kn{ssJgxW?IhcJ%FyY_2660W^CG%>8*uq_?1CfGhDcYlbx=YizA#PLn5r
zo-;?AbqB<{$+d;UNMekzg<mf!ZfQb@-*W(^*<MZ0ZJ5G@t9>I(<m75ZqVMz9oaZls
zQ+JbV+SH#6Yy(#(&8YHk9>n*3#lsXP2iJS!@b<C1dOWGS&~I_TIYZv4KfLEZ1td`}
zhfi;6Es1u*N5a*l6b*ZwP<}B0xgd?G#Cg=f!@M7(NJ~Dga}V199*kwe_Ie{boUlzL
z&%`XyRU1+#(S-Yi(D%Q}0xhOY+9L0wN0x6a+G<M-rEFa-x<2GIl4Ns#+w>q=_WeWn
zL5tscHn`03@{h*8F!0D!%&JN>gIk6&%vHtatD~n&TvGuk$d=5VBXf5wmjS5e^Z{z*
z(;0%e9R4TmDv|1Gb+LRic~5sB;nCFFk8|1AfcdDgg>;>SU04x*a6*#HdSfCF&HN;u
zk<dn*=t&5@_GCYfRDyTMpk|sTzo=A$X8+^|8Jcn1%WP7PUZKlBBs@1FO1){CWaY~C
z#=ro7OL2%#%o<FBj|?i4V5yx7*@*l;>{$_Z(BfZsdS3ndmY7`(C2DDYP@05Cd87IA
zILPtx&DSQ4$WQy!>ZjaH=-5d~SL}*Hd$|;<a46$r-Ig1I3O7~TE=8uHlPaow4Vs)}
zd#e8>=Ci8F%hq+RTCM$)EiXB6p_&T0DQGh4G$Af@vYUqiXc<KM<LAlon>PZ!5`i}Z
zVhK*7!9~aOQv1!*v?Wa*{UsCgYkb^r26i!!Lp%#`qR%$U!-&WK%MW;oW3!6z#fpQ#
z^QqX4l@W9^S0a#Vo4dT}=!Aj!$b3OVUR*32wIZF*Q7@nQv965%DlKVQ8EZmkv_>Og
zHWc^B5xXLtnw&#NGo7xl#Xi-jmZgP-GC{ulc&c>ON$QEW9X+z{^k>*qI?BN2y%wR@
zat3YTRtkH0e3g2ZZaRw&1&g-cM$OO1YVB4(ftISsk{YuhiHfso<j|hrIkjzynRk=Z
zlxr{TM=1Bh)im@bTqH@Pnd5)mSCL75R>gLAw71e-#LN6Jl6qmFO1xjYOr&D78GF>o
zb{ywl^#<@RA;>ESS9oWZyC4s%=m%}c$FYp-^iowXuJ>7&Dcr0eZhc%-dD@O1lfv)^
z2R0Uvn_g}@tjAVUtdi(|HT9NpZGAzvFqGo7g%<ZhaZM=hPH}gK;85J1;(?SB+_g9*
zxD_jw;83hcafjkg-}L`H_r3Q+J|uI_T5Io_-_F@Gb7(Dt-O&kNJu>tL8kiWw1J{TB
zekK!uY799xM939&_9s7QV-!@4`OnT?uH)>N&ejN$Is^c|5s6lrewP`AG6Ys?g*Jh4
zpj2t{?ru-VTl(5V(!z=6FK=mCaI4$$3%>2jiic|{u!n&+KPzPt2>F(N=9rM2X^(rB
z>QD5&h2sE4Op%66DH7~KRH>$r_f;AgPRX*tUcR(MqT<uv(1072{&DV&M!PkzH?U@`
zk$t=)GJk(K2B=q^>b7N92B{DfPKII3U_dWBD>7iU-|_KEtn0JaXQeWPGh#W<cE$kX
zi{TW1gnwT~KsjPn2v8m}l;7WNRC(Nx5n$58h>If>rdQcXipnH1_VYF6^2}JQaw^78
zcbRlSGa{e@W!9bVc0yA_zHis#91&V0)-q?ULUS<PgUiZXFtf)`2eO?>W+gI@OsLM}
ztd@5X8n0H<ms2d8&a#U*Ktw;*>te7KoFbOdml%C6SUnZMQs3u4`{C2el9-#Ah;r?M
z&uTh_%3kU$MdP*5I@O;{)R?=!uU(W4@QcB>%+?}bhOy3D+hD18!jhNPpFFCelgWei
zK+x5|ZM)M>B@mSC(Wr&t?7T-|KJJgu+P;+i)<S2i!HLT?SSn|HHm9hHz)Tvi(PFG6
z?ZmP%$qag|n#Pfo!^6~+!_?{lv+bj<>~hN6Q~Ye<*pMLme_|!>&JCriARie+SF+Qn
zpMl#UimkeDF4AHYGul$`?sPA$Wil#t5dx4X%@p_d=@EsdPMW$A`%kS?)7K8gV0Sdx
zpyH{Nz!wGDh7hz)mXI%2TCEMaI0bV%fB9aubES@*Ke+Wzmna8t6UXb<U?_Zg=^i{4
zCg}FV14ZPixQ%0jcD-o5uTFaiaG)Is3pRmFTZoaxZz0qf?=1_I&l<G}3c!p{p6V?1
zH4LX^e-C2*Pe@R|C}H(y6~wxgR)O=W*SC*_(2r~PD|k6aTn8Tp^0GK0KfjJ0G-qeJ
z@~gJCqW`4qa|j-<bgNDftLq-UDT!vHZ4-L2udsJ82LJ`kM85d(u-|r0=%ss9nJwav
z9VeD2spIUwN|6OCKv+dkiT>pQorw*E#g2_G_ZwW+yP%=QP&bFQ>D;mMlw$BBo%ply
zFG{eMSq+uwNuwO+Q;hHUYFgdyNFzNzTg}<&P<eB$#iSN42=i*=DAutVb^n$m)ZI+T
zEqvA4{U6Iq6g5syPdt^Uuh5JWt883aZoLpVGaG1iLHJi*X^lW)NXe)57EcMKx>w#M
zTax)ASU78SZ2Ej#A0opPu<!^}xRVOdbdyyU=(y4gxrkq6$u(yv!+!9e|1@puTuEhL
z_|orl|K<v_>U!zp{GNl>LH-3S7@NN<+vDVxD)WB0)w%0otmhAekV%4UeAN>E05?as
zZX{X6fbdZ3*Hz==EuEaQsae?=49nK=zXX{)nV#6x`0&wZi^i9m4%A*Xe-#5&9G0@m
zWqJGQ&Pn*>1~^}9mfn(fRm&Oerd*{YwiY#%DxN;~tv5eZ<SU!{^J-hknCZb#`BoKk
z@a%)ZcK`3RjG+f0Y1OvI_gU}C6<GKcxra)-`@GIZ+l#jxHXjogrj`)wXq*TKE`P_7
znrn4K?U3B!`8XwjjuV<|JB?Ha@weJkt8ORi!~R=g-wlxfM>wj?=I9>G_t7w3(V#q|
zPQU>QC|Y&g(`4zRNY4J*qm}xaZG(Ww+Ehoo=4{ujF&*M2X8HW-2yneOiw??vWyU((
zvrlgKf=n34n*$W8xR7<MM&90!Tg93bRAP0^%u%ayjLvSo5r4fJ@)y^sBIG&jN+{va
zpFos1@~6%i6>@&iT2Uj<7h2!0Hbg7eVFne(d}WqZm)pu4`futtH!JGABsv3LZLfbP
z$*gK^F?ycgxk3W~ma6$V@9U`Ori%l`dFz|42_$4zvIR$e+HtUgh`@e?P=J*R`u20s
zRX}!%K7w@Iry9@ATJvfEn}TpmKG)+gs7)&iMsdsNKpJ}Saiprof=hpKR1HiHdQnZ)
zNM)%q4u`{-zSVjjQk8(<siNz#8eh7$UG)g8a{`igDu<W#IE5z3mI6>k;F(USq3cqF
z#B7}DlUi6hVA)#ukV$1~q}wU)<#8jWSYQiorh7H39rMv}iX3sKkgrkaz9C@bNF`Vk
z1F_z99yQJ)QqpMNuc^6d%brLM8u4|&X)RXHB4V*}%Btx?R4Qcnc!-^4TOCj5tV6x{
zZGa+#&uCLN&Ak#zo`J5lW^aghts7_>nHv6AEtnKyjHn`FhNu`Ekyqq|TC7>}BG=kQ
z37`g&aMJ6~us=5=be1bS{a^NA6HCW1NrN1;;-tuxODTGVV9SeC`Ar6pwW0|kQ~@6B
zjFDb<L(bAZ-Dj|-8~Gr#bbZEpV4^v2>-wB^R&2RkC4)!2dv^rP<j!H`^};I5_F5}-
zH5B7})76jv;`zPTcHwx$;v7_I;WMv5!hI5QKa1Srxr=cXu<#J;rC`!tpA#R1Fz*0I
z1Iw|iP^=Y|6(r3dZO-ohA$6#?<yL^z-Q1*(r38NmjctX%+)!D%!PUB}u!AQ&TkvAe
zE-&~SzmGeCw5Dkauw=NR@iUO%Jf@(gLdvFrk0l(|yNxr3^EjwJbW=_>&D&A2N2)-!
z(?{^h@?<(^3Yv3h^=-x%wW9b7WmQd*ICyON{=4!yU?ttnCeN?rzu<jQ;kFRway_q)
zVcn>lX>uZw+Ue!AFnW%*GZ1djB19}g@>?U-`G-@vWI%mMtI1msM?c5^YDb<LF0n>0
z(T%?q_;K5v6L2n}T8C(?Xms8DMO@!VC}`5Z;urj7{QO>NM3%r0b0oFLv^o+Rn=b#`
z-0cNV)13YYntt_(gQfr*du}rx&;^*KhId*qWpk-4iEM-5`DJjOW)=+MsFHa$E)z5`
zv`?OKJ64&{o(lV$aNYM3JhE0&0=o^-S+qzgJtXp)J%p*2uqbI=JuBe^VSb+F)X9WF
z3!LO-xK<m|#w3KWt^Be4#E4cohR;n<@(C~BOKN6!OfF>k#}V4U91jUqg_b%61cI0S
zK&y$z-G*0lm<kFPXPu|iqDlngUa3-qhSolB#=oxLxA=;&IG#%Bdale{dixp`3yO5$
z<uI!(T-}gfi}m#m3V%y|e4u~*`EV;<jci;k6qwtkuN8PB_Q!>wFohH(d-B8+ttK8}
z`agV)B(A?Ns|^slA2*pERv+CS>*eQfcqr>-1*jLkKiyt39o+okcltvt!z|zpQ#}W;
z>V+r%wx1D<*jWMp0dS^5myC*e)CI8juHQf>QJIfD7MSeK_`2(zNa&M+w!8!NMT2@r
z`b|=ra$P0rRUJ~WzuaGdV{2dKyXWro$EKjp<IPiF?_Im0_@L9r1juQh5U2QsWY684
z@Aom{HQ$-V*3_U!9%}LM(nGh+PUKi#8UDTQ&&Lf0$_i))O#|T+8FTihi+gPNF3M&Q
zXZh9ZJBP+3dggfQzq(1?>o-`tJm6%#Qg-0|(0jhi-)s|+>1T>sO`=c^D&CB<CI^n>
z$Aq-V$eZ(zGy2{lao(C&%Wp*dJ3eH<!~Xzu(E(8&16ynsk_z8{?WMmKW4?P-$RO#)
zWp1|)-^<{0e-4`5P+JDpLM@M1v?gpxnQD~{tgli7Jn{qQg7R?zTPdYZM$B4`+*v3f
zx5oqkOVNf#DISe!a3S9pn8VjUfC^}aHhdpNrWWLhfy4CnG5GRGpMCP!!ygOeH_#U0
z?QE@Sv;PL#X&(3sz~=fuwO~ss%q9ExZM}Ro$V_zJ^KEHIQwlIoK831Xua4zh)PthQ
zj-#kc<3S$J-Y}#lu&btV;#2{iE8ffBjrod3qEz{Y6uIzPIs*Nuw_i916hezk{v6!Z
zgQHDoe=R8bi6E23E|L$p{3XCtk0qXZ_0?uB{t05YxflYFGrhw3TacY|f+&+0x960h
z65)2?0*wXgFfpxn;MZ04k;(s|qpFsJGnYb6zgUe=r3RZdg*E2|KCSr_+izSSgalXo
zCx(yOGz5iNjapt4xfWYoHTIs5k3!Z!I47+4gF@G;Jd^Q?-;vnYXl|*nt>*j$<!0;U
z;GZUBqE9jXytkG4;9#XYQmai6*JHH1cLr>n??+`JE`x<FgO*WD`cWq`iFmRy5%&4C
zgt=WK8mjT7gQiLwzbkU_ra$`mjN<`^?ze>XYf$B1!4n7EGQ_I_8ioiUrD2Sc!GKr<
zz*8s=rmgq7gC<t1ROEIG_<FKoK{L$xXLkDBZ|mt0_^<bZWubR#0lV@a^4Wh`3+*K{
z7fE(G^sXJE#D~u4kfTJj$0bAsB_8r80o*)86vBW9e7q6{9%=SzwwDE*jv*}}r7!G`
ztN=-Ax^T_ZE5NZKg9n$Y<^UaSkY6rZiWmE{*JY9T6!pZ1FODm~hgnLX7rTz{pHmKC
zB%hYjCqp}%wkh~5d|%0_k?SjOZlzQ}8FWq^Uo(z3rng*d+beG$9~FcD3Wy5f2%xb)
z>veE<x*#MC-PgE`bvxlc6AsN78(UN+4Ju}*WG7-)N~~zoC4M9Mg)k4+wD@RHL^@hA
zGL-}YM19#XR5tcMfA7KrTK;K)PRC+J@cCFYAt42o7<J8XU?2-U(2OxS2i-!a$ZFvI
zOHE=s=l79ZBd<Q)=_@}MF2W{)#VWwtQl>X<1U7N0CUcNy)Xt3KW0gMU4p8NT9SvqP
zpV>;VJ?p&&z06x2YGnUS54v>@?-E+_^mc&WUhzxxd-(R)n*#_A!MWv7(R+R=NBPe|
z7?`ljb*mX;;&Z0`1B5~4=i?shcH*L)?nQ_t<*XB7=iS-+<L)=~@u8dJvk7__-E_18
zS7aguLFOBKya8XcpcW$IzXglIJ7g~!a};fL(qWU~QOA`Q@HVPyl=#>~ut?AwT&moV
zx7vEMF9y*Ejb3&l&zcH99HI~kI}%NaraR49UQp`aYk>PQ6AJaratAhp?=;9}34#aa
z4Gwk@E@<(gzEX*>N@~7}Jir(w3X7Hv0$#IN$+&c^mS5v;5Jc)d!$(N^b_G_KJ~XB%
zH)L0VLFrY${Eugkv`YNIwpuEfB-`^J&8E=jss(2M%Po6la*)y2s+Yyt?><30EF?q#
zW4rx>Wr7$Qh8mh<hLOeW3pM?_f<s=k2K9iYN`AGCCe2xg!ZSjr3~*zPg5U=%ofEWB
z`hq`bwWfjKOe@!wsEw*xEX~yaTJw*Fk?9}JOb*lPx4(^qfgB8<fU0fFev3@E{-*$k
z#GYV}$4#nP)_0&`Zw|R}$r`i|aGBcgA_;9p?7Sr(X*NnVM;cokMH1}cno%&SbAV>~
zw&9X&?!bExi?qSwV2fi)DLaT+sp<8qQ3mIanM?kaas`zTt4vt6dILm58#L1iIYky@
zIr^Fe#M>B`nkM#cn&NcdMRl}bGcq*VB6O~HCW)(@MPoJO`=mptLVFr2`9mfazf)a;
z3!KGEu9mesVBI;9zFXp7&B&NZdh+lFdHgVVRb(CN-D7X)jj=O)T+H)mbC-0HRvT@$
z+T2D1GVVH^*RZ-vAO>A3lngRg0y9^vs-BejYF2#@%K(qa<7)oJQ&vzNve9)wT;jMV
zy8UbT5pQ^OVl&9c#xQb9N<^Yd<9Z3a$3L3`{MCCUhw-;r+ZPNQNHh?UTR6=aD|nCn
z^T^|RUFBkhuAOI^6-vC`bDMCtW24+vcMMG^c8{pRXH9NOIUzrv4)yl~e*xAbZ{Og_
zk+y$bVN`PR<nxi*W>x=|3c&GW;68zOd^A1S3azaft==uou{iXsCEELhy72If^g{-c
z@ph@ygIPt&Wjyh_;N7RI3}xKY?gzO?51!K1c`UefM-A%GH_K?zbtBu(#EXW<n5D~6
z)Sf5V9rwUs+|n^lND?%o-wH~Vw@0a%{P`C}RJ+=8_wmTtKb+qWa?Ag46yW50YxS1V
z0a#UYNmrmw_cW-VyS`7~O!*&DkdEB{RquX(LE-y{#oIP@+}bb(bBBLFHw*AMfT7;B
z=MB)px9CrNt&4<EeFlP}%0j~8*vV%eC-wm3-$<%eyOKPFp9fn#vLGPyz8-c&bXD_j
zzrDT`s?d!3O*um85dh}ZnDzbNtCm^L+v_`c`DwySE#<M<W)J2I*mKcY;^lPBx6^08
z!8AoNzB<9WDcI=aKWngg(+U!UI$ug=%*)QXm;GkZ;M+8z(J<ehNdhkVh#TLYXjYS^
z!Y*5s5&{zCXX2<7ZU^*hrPUdInzw`MYhHsWB)_(raqIemmu4l%j)YGgCg2vasCbpK
zuaxtasp*?})#dKT3J<ya3#;BDzqT<K)hgT{H;tS5o#IYtbX?40nqfJfHqRxPZyD4v
zPp8QFftU6R!QbhQgl`jgsALNmpBQwTaaYND;PsYL@R!o8B62R5>T+gDS2?p959^P4
zoIwnUA|WeSy17L0KofiL;83+He3xzN`?OHpPS>D(IBaV6{}^MCMwkDL_u99|B2@j?
ze}8ZlkNl6B%03%uriQ1-H=Jy8z29RFew;S?_*kZC^PFuwPBtT#^vuY<Z{A0j9gdgc
zuWlo7KNiQr93LgqA@EQqBFCl|2kbJkcGvspHA!85@#-&w7DoVS@|%gADjIWkc$Kjj
zW8p}Ni&iBLR>EppHa?W4H4I)s%kcrF&e{u3oE&zOu9*X?YvHo*>((d8bZr(X2)3IR
zfVinUalor6NkOEBjEbCD&A(r{y{%+Mn_B;1k_xj6tZD&vX{W%#Js8N(NjkT>o^KtC
z*+U1@ndQFX5`%PK#sR+~c*(XZ7=7V?h>Av!gf(RZh+h%UAelMbYV*OPLI9%Nj}Bol
ze9sq_L)hl7<Zw9wQ<O93o}*!GQ%h&G?lYN*HzRL1@aEsx8G4V#EXRK6CYEfo7kbDr
zHn@vWj~1ZAmO*O@GsqKPCWTA}_)ip$z{9TB!Tx8N*Puz?g89Z!S1o1|>~hD2sr|Nm
zr)hpBay8q)M?W829Qsj%JgXogAkVRl5qJD%1^rRRZwB|Cw_Gj0=vph|Hk0qWcxk==
z7O-bS7Woy(qUWcF67naA_aX1d*8ruW&KmJRHq6D&98Y^@L}ZvGe=k`#zdZbPPYN<J
zkwrOU5_E3kCUY>hJ$}`1@}fIc`H8Zv6VY>pDeJlN)@sh;6{{f<XAI$A*29az(o@s=
z)Y5fWH}x;d2lYi@&Kfpbw9JS>#bI3&#8j^4`WKkFI7^?VUM!5>D%4pspN{!F-VIO#
za`(0H?=QlF`(nEdvxF#Urg}qeDm86zKLcO7-E^FCY(l_b85j~Y#wklSBL(d%_yhqn
zM##}>sxEXZAjS-vI;v|x>+ddu4B%spBWu{$aNk%EMQBoXcZGOM_<R=>oBV^ptQuV4
z`LvZPGQYSx&pC83T@Sj2Tq<V6ypK!D1IXb%6)I-Ti4*4RGJy*5V)F3>)m~@ysGNRh
zC-L!3=18}9eEEW9@+RG<UzC0|EmCP-HAPPK&uho42MNw^g)%U-*59)>S&A9+R9K~y
zl??E}the{%#Ldq^l;P<g@m#sy+0ZRu_=10-n#K1a9gG@s*K>k6<ST<()N^p>(f{Kt
z#^d-KJkWxlyGon~i0X4Rbz)%J(89SPLx!Ch1=6vF2gvc4_8Y;_%LWWplI(C`Ydmzd
z%e2txJA(hH`>HA*ER7%MR3HPZ{jF@Us%XR#-^$W1^?|9!x;L{g_1<F39Y6s3Kr&q_
zVMZ=v;5qPu8yVr8G;cI)poi*BRky``-=IfYu>e1E-~vcx&c<7(M%v8&DypXB6>q!L
z+&oXF)&4x!FM(S#_1PQktCRto;x6Q7Of^BX%&5~qYdimiDt~Hp+vFquSdK$iZs%2-
z(h=wVT^l0rX~*~Y$;6Q?arNsWAB~>;`0m>GEB;EbU-iz+GN>MO+)@5DIv`+tTA+^E
z5j+@E$g@HUC)FIN8P#EF*J(X@*WD3s;`jUA_)kf)!4qk+{~(foG7Ebh<a9DM)`O@6
z@E}XKW_KV8CSuTwioupthlAR3>J1PZpdxli&(kQEhfAQ*Bx^y(;L4a0gtF3635LDN
z;TJn)7+m_J9$*&j&-o^YT6}{M<dA3gD|%xC;L->wIob}~xf~40MJ5)k1W)G#E_DXT
znT5PMev46DWvLtsH1<fPD%o=KyO*r%b$gjJht0R8?3?;Pu_f|iQe&|Y??u(1z#MuC
zjE_L{l9bCRR-=9Pd0O4!jCY%MJ8S*<nRHN4P&mxUr30BELy@@FX_~mUpZ_6#HSDv0
z&{gbH6*ije|HxQ!<DXNPTqtS@f<oGT#&?D9SA@N!lnk~=B{Qx~yrHS}TSvC4o&MrG
z67Wssnfe?4IMTwR23Z681Hhtl@rYyduTG(Tfa&+wMX9hf>A?P>Z<Rn{E)Nc2w?P0@
zV^&N=Brwj3l_fx=-UG=zme9-|ZlqFim9fP(;6W9*6yz_L7m#M9fF#sp6jt<T>R5i<
zP1q5@{6J<)H@?2gY#Ea{MGUet79A+$qK4d1y{?}&MYj36>_=J2NEq3X;ec)XqDKal
zrB*c39bO}f8fx`tz!X}i=6QaW3~=Fv=rr02v9#ADL)z8GcL$@kK8%<YwGVuTHJ_;e
z$CohW&@bqj{i^_Md;0M?7}>qZ`ls*9YTwTRyDMw|^EA*yO-?$dnN=yX5p0sOfMgS0
zj5UFJbu3nA|0Z4X60V8jGYQ6|bogj@7270R_j^;9!kK?3>$qpQ2!{EdWn<%bndfeY
zsV%Q1(P|~QxcypxEB!S{Z`Je5|9Dx6i@Gz{gS0a!8hFY6lz{S2#f6vnwpN+oS1WUJ
zE#wo=Pff@v47naM-!4c)*qOy^p$GaWgsQXY>rUgDnz+y+WVH|AdmMXppP#_*x4o7B
z&(o&DJWsx<M$Qe(*(V`|ohLdAXqnJcx9dYPQP(S_4pEW(#MbuO9q$r%Jk$2mnX=x*
zDfgxhQU8aV&TEgCOfIZB{TV|@r$6JClebSs)|wSWYyZ^uw4A^AMNEVIoiLVji&5_A
z^N*x|jO;!7aVKRv9p-r!NfEKf=?O6Zna6LpK(xm-!9#-nzs~Phn?5D*+Vw5NAGnx>
zXPMmbULdRn1B9q0FTdF~5KYfKy5cHguH})(e*H#j{KrbHKRL<kb}CuHw?o3lft@?!
zQ`OGf=;7s5mVB=`rucbWN|-1vxDjSNlu}Yfqx;f+MDv=Sz&H)p+*fiKgEBrhyhw&j
zEfE8E6hV+*4T_H^Iq-|IuuOxVTBiVC?p>qlmCP8@M!RuLAKdGRfA88FIu*4n@>~H_
zAS$btgAbj%u)BAua&UIKYnCwL4fca+yNDZXX`|btt*v@)3Dw5>LCdJVgT`a&#o$Yk
zAKWiMA_gVMkuuNYc}EqxrW#~&<C@DjtnqMsr-1(Aaj|yb=h3d{icbT@wTS*2>hX!0
zn+QvLD+#ZEiqN6__F9(qqRZ^aEm{#XO=;ylsyQvuVOpP2ngX&73=AMTk1>`oq65Ff
zLNHS$)D4So)=FL<w>X!&h(8!K3!~|Z&Q(~2KwqrZF1e=Vg9p9T2TsRLLXQ3fbob?y
zjaJUc(SUMZReG4~xfJ!Ydx=k33b((F&Pz<u;@gEnL9?jEOV8M5&*uxAyp+`kl68=O
zY98m+EVIwbzYeDR%pC=hJ<0|5{_X!*+u?;u@|Sseo4{V=?B>?TwAGWE>nS`wu)@OY
zVZ>VM@^uQyK+wjt!p$JfxLxNDD)^sW4b9S1icT(m`U{wzX&Cg@D&0-jRDM@ySuYuu
z*#Qqalqaih*H<OmdFVq&BLi<&+ye;nw47>4dlCJ{@61#)=(%S^D0<?!h@6iN`_emy
z{HZO|mrNWytOK<~3<BfLxm>aEuLBJJ>kke}v#b3=r;x5wtw0~@@nHPO@N=4t>vIvh
z1o=yWIT$45$5)d40l|nS-JeR7q>x;eM;e1(b(sVAL-pC^Q$m($#_yk`vTcw98vLMR
z((hSli%f#)me^I1az;Cr=GE^aa~VAsq)f)jd%$~|7&X+jg%<HBcQ`lc^+);8;C8MK
z;$F+8vHvu3c;9~EKx!0e>g;~oJwWh{SRv>D5Y@(aQ46}iPZ(E>yvb4f;aq*n3`vG@
zajx}#6^3HV6{iVsC-V3%ou1*AgidPC`W$JL;DD4kTfUbpl$BJS2|+sC9)7;@-CyQ%
zykNbqqd|5bmZtZ4yBaD=yF0~@`Fq>tS3Nq77noi-PHNI`isr0}SM_s%<PbynpY{5*
zACj8H=So}}>f5iO3@FGvQT~<iojXOxh4Je`98dwzty`@>y5=^k0=!|)qV3}e%CTQQ
zU#+|NebQuG{7tJR$Ng>)J<?BA_Zlg%r%{;$TYpvWA0Fo9n*c^aP(Zb+BQjcU<4LDH
zpM18m&i4t(A;pK+xHr#zpCYacN|gO$!+}s{kJe7bXNyZdubG6Epati~L~!nt4^o}a
zI0otUu;Hw=9&E0Mrr)_LhbxzU1#fm5`Vd^_k+)=?AkuNgNr%nfe@!+^fq5?R=S;Y{
zd*th&PSZl%Ms~7XYIbbFByPV-d`ut0-<v#Krih#BK_-0Pfk?!Cw}K>EFYE<DQ%dhZ
z*mD^?xD(qY0RjGb1klGo;%kwa>FFYgtV6^b^;!u8fB>HuwB<$ZbRu}q!}9;gXjxl4
zK(GWH6ei)xd1%^9yq|wGFA0ByaFc|@tt)<7O-o`}E?*7~JhGAY*=l0u!2^|%fZ!#0
zx&0}_T4$#x!or3TdIs%}0{&NG{Szjw9B@48j4c55qCS)?3KjvG)q7t*+VauCjS6W=
zQ-n;`5ho&Nyj8?n3*dX%t^+#_n{MRvU3N}$<~_AlV>;YsiT}<-{#^a4tLnbMAFNf+
zct#qjBDwR~rJ&)tiaC$&*zIxd)!eZ*8$WBG=-hS3Z~(FzSZRT>(RI<4vg)A*@edSU
z@d|(NSCg8}IOGid6kOt=(5!A4&iIXtZ71*|z1vvVMccpa7p8yPR%hEkuBP_mXe(Y)
zV=r@vvM&&Ma=-yv=cM2T*FU!^nfRPZb0;dNhq3C7)Ll%*93<5i*~wT(<};qb`xdk%
zSAsd{Hu&>wh1^&^S))nox~P$bFf(T*3+A^4`v-##%FLt@OdHFjNjM{2e$)qN^;;3z
z(B??DMb2Gh_SsaW^Bma4!V|(_iLXH&-(BV=jiL7Y!J9w)F^bpM02Twc->Fc677R1{
zjz%*6i^0}(=3EMJN{;Whc%MJwx~QpslfgnnnNM>>vN{{!&pr&@(e+rOIRx)slaizY
zrX7eO8tGx4FZYN|ryXBr)me@eD{H$DLJMRvI`okJ4}?E?osnc4poemBr$Kh`O_gMV
zjMi`G7dLYpq!-gJRi8KYzLYeWa{_GPzHQxC2w<Ff{q~`pw_`2c=ibxL0RP*z{MgU`
zI%z;%uZ80yk}ost0=vX$$hPX#c_2<O8EDX&1RV~zY)@n?ta{MPj%mCl={O@X`L#cu
zo0a-|#9!11%P(&49XA2K^$m}#+B4Tz|4pKb4Cy|Bq`3v25z@z=!Dy>)lRSRvpZB4x
zD$-y;PD7pL>RS>qb(2Den-?cHSwK2sCzxtGeegJW>qsckqsP;T?(^^jA_yI`meMCz
z4(Y|0wDV@B#GiP@-D?&df=R<*eZK+JRik#~bEmW{9`9eEV5_-VdGGizPyU_<cI?y1
z!ntP{XJIyV7Xl)0TT>J45K*oiY;3r_nKa?B%zp?#gw;b<XT6^e9Zug(R_v8vkfe5z
zHSueEd;K?_&gySQBV}nwu{H1&T#YVJr$=Q;KUB8;qzk8Iu8<{|p~G6#J#hG07*b@C
zwM}U@WYdr-G#hXjM2zMkD1RI(PI<X=uIlu(O|?N+nP#jt9A^qq7G%o1o&YRwR9HE>
z`xPrcs3GFl)z*U+V2seTxo>pO9i@NV!%QFV4e<{jLM`d8`^H{r+>2J7M?3>n6a_y4
z4I`5`_}d`dQy*tyfOso_Z6TI=UN(Qxks;+gn7adkglCw2wlsWGW}8kX4?a%Xhp;rM
zj3RogPciiuj6A;(o<p<O*19d9Lo>mNck-Zq1gHUy-k=_d%7F)cGAX;<PJ1<iiw2+x
z95_PagaKd|f6*h+fwl3G;ZurbdD0{PeW``!6-bC#FMj^H^supgx+C>uztvK+Wxu57
z-w8bEVB;A@?XL)};u0N&#gBNt97T_E&dfznlpGr5vP|fn{Y1Z&rx}5F)nR*x9R_^k
z_R;mt><M``9Y?j)RIpXIBiR-|5cHe}xozTw<9SNDY`b0J6?FY-RKm}=VQpeL=hx%8
z^N0Hc$y!ju>JU^Tz6(-kgR;90fbX!XI;YyuQbSH!S~rSAbcOA^0KXf-R}>jQ_tWtd
z7f;lGaQ8KxMy?L^@6k?MyzcjxjWA0UiqignrKMow2al2*#!yb`jo#&h*}VSp(J8zi
z8yOU?Hy7Q6_7|^_)TsYMofN-9y#Wz_HHY|U+>)M;6q8h-Ts^3e1OwbB_;NgdlskT$
zhyo@yO}1atf9(1Am9R)D!*RN^h<7JZIu+R8N%M3xneh7qgQIHXTNAPi&ObBB#kRGE
z5P%-!aivyZvh<BlMD~YR)lM4m)t3o=JQ<@kO<f|GXgyjV2lfEo<2|j%C+bmrR9bu=
zCM&Lsx|UxyBA@gmh5+P#xQBwvzCoPULN?3xQhKCqsvLD98Utdld=@m^425Z<)i!Lh
z*_hL0__t`^yh3o|gQz#AN7Z+q7vTlTHtb!Abh)3R=NH`^&OVa#8a{ZYEdH>5NdVWr
z3`TKj*7~Vjj9GByB=j-XWU`$4{f>@Wai+SelTieh>HN;y)L5-zJbop(YwYI>1hIEm
z*%@hkY%bBCM1p_NoBU(D%x!)*vK?)r{?g8tz-SZ@Yk)XPbG#P_O@>}_<%_MnUeDC%
z8Ofx&amubX1Lz-QP!S$KJ*ee}OPE#<C%z7~RWN*Pv4>gOjKH}~QvS5R^VPU%T$gdn
z{{?hyf6hQjQ(Q$5=JY{xn{ffmfUXN0?^7akk%PN_m@!Vy>Hh~*q;sALUNgHOnk=R8
zBs}Se#0&>sEBup14hosL2eg<}vGt2U8*#q?neb3_4a_5hs1?Yt%#&c6cr1Mk6BQ$9
zhtD*&0GtSEvE2*re>edhyQ5S{te_}}0$F41Fy%97`Qc;tk$KkKm!z6#*j~xcgjhyT
z`k$VHV)b@fED1r>mKg#nOgNKcjNPMOZO}n2&-t>5y8p@BX!LekiIsp_<mRg|g%rdl
zu2;TlcIiCPlKY>YQw$JyZC^G8H5F3B6Gs5_S}H@vV*|hZLQOk9{RbN<#qRC*#>Lw)
z@*nJ%EF08z39LXi6?Y&|o&pY)ape_BlKVW`PdMQzuAnNn{TZ1Be-F@wH*G1NI->H~
zCUVj%lG&~o79@*9VVl|_j+hIFKNOY8Y-Q4cai@OZpCvVx$oztI7{DKVZWiH@!Lq>(
zdSX5&t^aDi@wyb;2kp7JBQ<(4Z^sRyEOi10fqA_8{<PA3_=W9u5og7e0i-+i1p~(#
zd_K_snB;m<3RZXs0%y#{wyrQN<I8tZ(n}fm-uc&J{q+c>-VpQ{1AiQ@L;kK_Fv1Ew
zALydVEf(1CO5yw@a2E-uDndmWghg5K{uUqQYNJ7ax{z)T1f3-6kYn_sB+~5K5=lui
z)!}3q*W~B72n*eyyi`ReStdM1rHubBx2HvU!_L35?5Z?mtyM9@{Mr#s^yY8sChA!<
zLjDFOGz~dEY@7S(;Lz4*)wE*e>fw4?4BP1|`Vi46dAV`f{@`u)qA7r4khOiZoryJI
z^wo(@?@cexk)H0~tM7|z0Hc$-+fth5nait1|KKN$g)z_$tN4-lNd=<5@!WK$_?O>a
z$pfh-S#YHjl|*IwCYrc@&l`FaR<z6_Ag!P_#*8}IXJqAmhcY|`WaWNdZH#xwdi^I=
z*k?XH`pApwkhw}A33@qVP#K=xyP?8oDX0lU>t%TPKywCU>3?~U1oF-@Boy+_84a?}
z<hQ-@qTZ5Vpf*N>X2y8-sTJ~e#}rwkl2W`BR1O_oW-huOqG|Y&`Y)+>^OHm91T^=P
z>STYq;1w-mAK}6FPK0igHwDR3!dB5?@X#$CK5shN?S4aolYY^d=Ue4?`i7wC!it%P
z-5w%3z_elI7kFq1vI@g3{2f#`?R9m=?;lv~wgdVjd89c6ZW>P316UpPK7)KH;Swma
zY(j%xc$gL_nC4`R-`?svNXbf-M5cA<Nj*}uY>GW`37;G6=N@+`3LY2jR8-)tj0dy-
z;<=wRJDW?@tQ4G^t<}cZCS42B#-LA;e9sz@1g9lW=}`dTC&0~%fiVMoIO*REuzSBT
z;M+KLY+2^Lj733_LJ55<hH`h0*}p+RX%y`S1x@H<9Oz>tfW9SyxH&>liq6mp13baM
z1n^`~XWRj_CNb;U&+Jeuw9708qcHW$yhx}*S+DQ}rH}K464qD~Q2AB2STXhWIf=2{
zW2k^jv@bWGvpjnjO1-(3_m+hyH?VyB+6kBuSU#}w(3Yvfm|W|2d0=Bxkd)Y2u#j%b
z6Tu*?^|O{-=bMJtFei7X#iad*5+RK;W_C3v_pm4PcB|4#sp1K*?LJ{8K1=nexPq_6
z$Z8n;-&HVs?c^{*{B&4|onHKcIm8=jrGcuOKXr0b{MF7HjRN>fo&6PzrDu)mrx&}W
z(8h`<4Ep<v=kNtn7TKnG7W*69bkd0THPDGje18U+*s>tkAXfbZMctsT*D@8sTc30-
z2yo&JagFIp8+0xj31De2c?*-~trT-;LxS8cVv~bim`Jz*y&do7Vx?jgyn!wui*79>
z5zq($>T+BbVh4M<DD~qEtR_t-X*WlfIq(;JHoEWxC<Ut}ePEF4S18}bUIQTo<NBda
zEPvts8gu{k;MgyANA{7$e&>}YSJiG(!U;kQO>}d@V%g+^eDxIbgBjKr2%HXBgFPxn
zj*ikqrJN;BSx*7FjzQGymCEc_T@nfGZUb5${0G05_@m8ds3+YuJQH3)Gy%9_pM1DU
znJikUuUwb3R<v+9N)C$hk~UnAa7Bdm1-dsZI$dT_AQ{4x)Mg>p&A)mgxU{W=H0-@+
zkE<?@#-!$BSV}f)MI~t0<iEMrIG_JA5LP3`eUj@NoQWwj98(-K8O9y<!>DoEPqpO*
z^=plONR;TxPP{3CyO{z_r`b|Q>Jo5icZgeQWPC(?{q3xTTk~&y1-?J}&D{z+H>{r5
zP95GrtZpgdWr2vDb3dTkm?W^Yw66TM9z$%^mqkVyItR01R)El34*yp6zB(eRN@M-h
zn*LA31E;f0cO2Ymh3)Tr{qI&)pBg_t3B^4_B^?h$7yRn_GfqaOirKEFehu4>!!NXC
z36uLAX!J|n#U2gYhcEqr@9p*%HoouO-u!6EeGJT)@jee3W~ZR^B)Rj`o<Dx*3EG2n
z{NILoNk?h63)u5F#r_o6E9{VNB}$^?g3#@ldp%KtqN0hodAajY#_ZbMZyFjunTrxD
z+R(d}o>}pkE7whi{zThSdXQn2!gIA5wq;ngG{z5!SjREEwZ3%^qGNXXO{)e<OS+^?
zEI;r-dq`AY6<AV&C`s|XS+~2fRCgt|g4Mq@X)={qbxlhs7oG0^;yg71N;e2a?}3nD
zjg>E$21{rR0-_EaVzmVZzYC^C6D)NL%P{{CqOR_3&6Q00T8OaW@z_Ff^bq3<t!wpD
z=x8AC%gFM;y_-V5tq?hr`pj7rpHPR-xe=f;JZ0P$iJ0Yyrzx@UCMIX{*VWMFK`@sh
zWXOkLLY#d@pQIW#G}y8a$aG*;_I%`US}wr59g=6VqwQUdZ?~uoh;zD0aS0e-yJFuc
z)cCC!?Zfm={=ob1Qxu=YT7~;|LEb9J?Jbp?HRluh8EyeTZ=4DXtU}~Lur`CJtz^c?
z4V=t-**@BsZf1*e6nZf_|Ip`LaG29hPXPMsw;0j1va9s`ajKFrPZMYC8xI0qu4DTd
zhdc&0t(pQCx3?L5X`mA^R8rLG)riof&r0qwpCa%xKr5(;JF?~0KkoDHAFzIeoJ{Im
zp43<Q@FO~<+C^JEw37IWRGtdo3WpJ}!tlyhIZaY+7T9Q#ez+&44aue+VP8p5A~bqq
zZ$*IylBq0dUIZ`rxlMdB6>*$>Apj}bJVf9PJ617b(<1}3gIeT4=+7iY$U7wlX+q%^
z>S+zGRlFGluu!G6&w9P#oV+qcx#Ga4Z%G^x<7~9ZUFX!E!dJPSzGzT-<72QrpD3nH
z*`7i$5ygJ6;B1QRGY*SwDzZ+W$l;U3WjP5BZ81AZ2Wg<p!l5708evWq(*?l=pZ2vH
zKA#<9ZrijX_BW#8Aic5-kXBI3D%0L)1aKDP@aLT>(lagGhT155MzKYVgnm$j%LGSS
z&~EUE-55y4TRb0=nKksH`iBN?3W#Vg6#-<|^T}JTP6W%f)BJhteWtLgNZF`vpqu2<
z%0Blv1Tsioe)U`^4u>{~&22^sWFXtN1yry;+Atjl#Ym8;Yolni-&6h%)YI}kY$AG{
zoE5=3#gUdW@%tVo0h;w99|A5fx#*z*P11}@{lOe-(+1mPRnbS!A_|l^m6Dbr!&iQ$
zYuR6N<*^;EpTfs&Gaif4*hv0a?<XF89}K@4zv`Ni!HWZ{*=9TqfeMVTx{w`IwWAv8
zq3BN?a`i?{O3LrXmeV&a0F2yRu(yLunpkjq4ufdgXcl{^cGB3~7tU#+J=aNz#p-lB
z-6+|;N&CTAnG06$n}qSAIFaLwdn<SeXXin7JR$}av`$+YWX*}IR#4kUlPw2xy1NKn
zn_x)WYNPRwMJNvYQyaK|00Zcuo@4~>aoX1r(=w^{YGwf1woMq}!lQoMadEF8gXpWt
vE;TBmI0A$NjhKKK?x(U4Z=AULLw;&e23Kwtj}zqUMNvM;s>;+znTP*>DY~fb

literal 0
HcmV?d00001

diff --git a/modules/web-console/frontend/public/images/query-chart.png b/modules/web-console/frontend/public/images/query-chart.png
deleted file mode 100644
index 1b7ef41b778b88b3e66cfbe5bd35044af5df3f47..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 17142
zcmch<1yGzpur3OL07-!0uxNnbzF2|=cXv&Y1%kU<2pR~!OK^8z+%4EH?u!Nq?jH1#
zocqps_r6p0Ue&9*wbg&k^mI>mf88@H|34d{sw{)`lJq4K5)zi2tfV>;5()?j=@~5s
z$`hw{St9SL6RoPKDW#L#KQuJ-$jz-9-+6Iy5z#Qq&CN~aQG0oLX`Tt++uv)L++5f>
zHB1{gIRE2ZJa%$*+rPB$T``gVbGdDHw`YFWuWB-{|JTjKL(TZ_@6Cv0B&743Tig7R
z@%7{1M^{0$)8b;{Ngazrzm8V-&!>MMht|!k>|HcZZ#_PaSmh3FZf%Y{KB7EBEgW3k
zKE8}^n?InSD<58eJn!J<wm3UGe!O2V9a_7GTRc8q!>rgI;3<`rRZ|PAkB>)Ac>@;Q
z+#C0gofnVyn?<Q9X+I~Ih-vH|xh*Ef#yPCuf#vYzp-05r+~ei=r_`VD6wt)gW4-)q
zZcs89C**3o%Y5jzYVR?(bNU53D}To3>BgMV8yq1_jFIRMA?(;G{BPdAz}DyGzPIB`
zx8mpGmD-(#nzkO^P5C_}TRD9IUbSdRadST`fvfvxamm!aPF>nqTOF2X6KekC<~De=
z7<!K{!4!&sq_iYek39}O4!3ojiKbA|R0BorFVB_+ckj3it4@ogns?3*sARB#F)P{X
zzb`NHH8~;ck7t5(JX~p7C7Xdi4~_>14}09C>(7oR_pya@=a<*U6@m`<9mcYA?^1^f
zCQnU;O;-{)>M{~IWWGeG@{A0mZb3iZTt2b|G^T>gtEj#_SXn41B93y_!TjaoL8D-u
z@usgj$wNbzYY!DG2cchrIaL#QlZJ&q+DbduO<tT0A#u;t1=)QP(+cVDv&+5IWwtTQ
z82oI6N5y~Ib&{22ZISZh&}-nrF>Q#O`y*2k=u0`RSC7)?ui3#1mYIDv-T~!>yX|cs
z^BtQvR#SfNb@#U|Rg9E<PMp3yzN*@oJv==8wYqkHfB*RS`1|+orKP2-n>&S@K4&DP
z543WUVw#@Drj70wr(m*L<lIjLQ9&$8mBwa`UKE`pfcBqGlE{WmH*@os@hPD*c=C2B
zq$w(R#NVome-UabqP}Ouc($-0s5g^U^+6tX7~$v>Ja;=~dJ-k2x}mS4Db$ht%-QjS
z{NyI<V8G<n=Hil}c5Zw|-s*jzQ<@jIOXq{9V2w}VH}5XaDDU5&C0vB*%t)l|?2E#^
zyEzr}S0joTFnfglaIl$?gtD~&=Vm>9(NdrHRDl>0%PPH&GQV00Rn;{e1*dRf8t`3{
zEWIxPNuFiB!^P}@WGga2Z7Wr#a6{S5;6+AvqMzn~3}z~k%v>nvFHXc0*-%3Hdrk6?
z3l?VwfT3(I?mT0&q7kiWpcD0%@FrxPqKIoX$afp<)s`M&as)62%8F~`2VygugZ?xn
zs|6eMidYW?UWki*wbN-u{U$Coq`UBwNhStri+mVCj5`?(WyKm?*PRjkv)#>P&{qZ8
z#nJSdys*DGC%Q=pqdS=(p9DH@XtK6gBiVlR9D#NeH9jb`u|7@zd`?D`_N|^vFQKgM
zaz@_>I9xzJABWZpe?~MhHSasM3$x9$^OiZ8qy}3l@Wf428dpTyB@eGD&Pdj4))dud
zMvUG;BwuB`QUj?m4^eSqlK@3~c$LP3d^cd?14nuS#18POPVzE;4|#9Y%TK)RBTrZJ
zj<#i$MTjf`vV3Wn9#Km=dY}(4#OSf!3~<qMyY6sN9c2PHMf1!eluQy?wG4|B`a`Es
ztITmka)W6X-^tvuiYayo4e!m~v3sv(__(XV@HLL4ed%zK-?4oz&BCMM<!P-Zqc;Ny
z?6p_mMR%Dl=fO4&X3K8+J6qR}Ke@7!99ov;c`68ucm$njlp0{qu@*4qce4Fjik&9e
zB_pe7EawjFiC^>Xbw$rKc~^njqzyD4yx{HI+uOSB-VM3yx370i06mamy()t<q^fkz
zH%#G}##8aEtZJr_;nv9o<r)sJQzlA(t8Wl`ZJ3v7aR@~S+chb<O{H9nkhBR(TZMmm
zRqs=+!S1%HJ{PvjXjPlVW$syphxTi`Rthq1d+LJo?wje)>rs-Up@rgIid=C8)j9P!
zAd{Zl*Mhs;c|mDBMW|t>VV@(rLQiNZc2|9=d$y)zpYorE=(tWG!z4n#7;}PfNg?&E
zS3X1UyXhCU2QjR=f;Yhl)cn?Lv_9ql48`B>`RaOE0`J$pztC>x;_y3gDZK?cnBWdc
z6#=GVEwTbvIat7`At7uD@n1yn)3|1=>J2mfp2B9z{SeCAY)$!TC)Th)b!SQ6w9=zG
zCY8+IaXZl2XjfmsN}-yP+@%V{%?rsF3v&Cd2hY%BD($*CG<evNVKvvwv3eHhfcWtu
zJ{FW=?s|zUFi6EP=wCStpzUd5eU+E|X7CMQKIxA~lzH-!(xS64T=rXsFay)_&9tr)
z(8j<*oqQ5nV87;T$NoyY8<I({G*lm^cd5EZZe=?pWQM3Bzj1qOuJ;DPv;_>k0s)<D
z`M17K#<0ffif406b8T(`!+g~VdR17}6VOSS-((j7trP3lNIY5LMVg6)-_%p#LExz_
zg=+`$FbXRGc<>4Cmz@9!%P1e6?>PWj_nTW_sLJU=QR9{<$L)`Yjbbt5`d!8der6Qu
z`TGN0txn>wsAqz7eY~B^ozTTqNU!%cfmC_;ydhDUt6BRB*e^_9n%j8N7oHZyD<-it
zw$neCD7c~EUSb)AX;5&ZlE32g^<4XO`QC9&m!n?n(9bKLip|PXUDR^3J+8}|*z5O~
zM-x+*&3rLWW(TZRwlvH{2M78!$1@`>NRaG?$+dRyP*v`@=@zq_o#AC4D$7z9Ec&y`
zde7HIwg!C|gT2S5$(7K&_55$S$teRGO}3$Ur`I_N7QjS!-%xSzs+{}HD4fK<2Zq`<
z#q68bisnr=SCi;Kq~y2Wj&_e*64dUamRI^ZFygtqv&gI1a}PzmniPskK<sT#r+0uN
zghY84d4Q*03sv`#HOS&wS<^B}0A1XVyqL=jL-ovZh`2@9uHkMh+TK<(qmO`>hNy-g
zv)w^`m`k!=!l3fm((6z*T7o|_CY^dAm}BQeyQdhY&nB#<x)^VJ_4Gn-gX9~zb>*<i
z$-ZGXFv<=<+jLIsOhP;U#4{>VZL#$>cdc-kUEhL63~}Rg{Hs8i21MNFsqBw5Vr`%1
zphypH6rrmhzxfPx5cF4p?n(<W4K7K$IBfw>yFBJUf;?^ReVl*kiOKo6i`IiRAEd_l
zQX07j3(3dL7KIZ7MNQKcw^Y><{bhYW-znc`Ao6$oyG;9#${H1<0t)7KU?2>MITxr3
z03&O<W2z=M7}K0@zGp@O|B3aMDFO;ffwG`<Nu}sR9D0B!?#>TuF4s!}@D~mid$rnq
z&C8Cv4<LDv&j*kunrd=8Sa?7ENQF#>wErtJE|Zk-3WZt&1B+UUq#6ez!?k_CnjQi5
zGy}X~QBaENibi4P0|v4Mks3e(QJaY0vUl}CV<~<;FY%42NP=kOwucwd81+ynVW&rX
zOFpKGjFx#2&!B}_lwO8(aJWMdf2wJR!?yZ$=MTH_@7^JbIo2j`RW}2VN`hIK*86pY
zGuE8CErpAeCU(`Ne`Qgg+lUvm2N9)S^OrD|7|Gn~jVblvenJ#Fgy)GLJ@@14_5l4F
z=Zu^c@|l#^zey(!wVJgO<brMX->h&3AG==?wL!}ql<1>7g@mvQ%-ys}#$Q%CVrdhZ
zAO@w@`YLR9yM4sAPpDbNlVUG;j`T|l1X-;!ip)N<sI55gj45Dv(9+@IYZYROUY_-~
zL}f>mmpdI2+rfXCNtK{I>Ze1;0|?McREbV)^N#%LUL+C{Pwbo<FoWpqFHA_ysSWEJ
z!6SB@JLk&v7`##S9!BslxfS$Czs^E0I_;`of<B<AqZJ8Xcsc3Fv~^;syZ`*{Y-CZQ
z>OKWFJVh3?4={TvFrRByA#)#oDkMHrF)T22W5EQ<{eo^bc=5j5gZ^uA=p?62LG~|?
zL;A*rn&z%;n`xfk&9H}MI_rNi#Ei<cnLnpYDHH(F8EpKS8<~#dEbc5L0x6c}S*1{^
z=#s0(_Y4@t5(Tuftyl8z-XxAQakJ*c10VFpOBT(k#5?fD?Ev<QQ}47@U(8uF*j`r}
zb8&n4Rx>AJJ=2m+9uD~V`upklKx_HI3w1Q|Y-XmGke+VPw%N|QUe*v3el`nT_uIoP
z7yV4Pxvh>~B!GAl8iB_;{nwiAX=FRs@5KF)IZ*Ad`oHQfs%ybkM_btLu07nQ8M}!u
zF8hJTTi7Z9rjRuyVvb8>BfliT1Qy+|zE~YX9bKLk$aFy&&*A%WjnD}@KtAU8G&d@k
zXF70kMvGC9bU#0_PO{D2k?3s1!J#s*Vm5DRr7PBV*SAV1yNoxme*f%EuUw(6@I1Ep
zr*(N)<0okb`2l-ZpMs0JU1n;~jqbes`Blq`Xl#6hUWrD*QI9HjmxHiK*ul)b+WMQs
z`ufTwQid-m4+_`J8#GLH(!W^ZfgTNC)tJ^LoZ@h(nao0(5~ri#?3bUN5D}u}p+F$`
zg5YD?8*9TE+i>_dnjEbKh#HWY>DHq6k4jclgXPcFB4WndBiTIu%03XlMho^%kPJ=H
z-h_iRsTzmbWB@houwehO%S&FGW9oxYeHf4@pgFf{p4FVcu8!0$Sp-s#cYhzueo4xu
z;RXv;1`9i`9?t(BQx5~T++GilP1udk@FDQ^y)m(G2hb~GC<p56`H9-eBVIH$Tr^@b
z<x<xtR=oj<E62Z0c2texG8Ee#WWu_6scJ&)m-Lp28t+EP36F7)N%lwcuNE+rEOk1%
z8BV!w?prn}9^7ueI!G-fO^`VP5>8co&t5~a4%|e7U8f|v&E1(~4ILbXXq&{NZ~E4F
z@v;jll~{_-@eWD-l!a@AujL$ZNQ|H~3(ns5%$LcA@@IM%tM+-gQ%C|DBkyX$wC&2#
z3<|#IYZP>3_Kf<QRrKf<+IA}tT%^C<NI*XwD<+|4v7xyj345GkpK~yXFnhx%fJZAX
zW>>ZVe9j#Lq}jkDk{;+C4eeA$cF`H(<axID9)-}C-Kn55wsT8K*E!_ciLrd_EZq$5
z<Ww@|!uoc2DtMn)Sh{3_doJ{)SZHIJYgWIkwRaRnlrTyn7Wgd4tB_n?_YB=TO2+8b
zyN?-C!x}M8Je*+(7)GD4#YjQekF%hdxCRu^w6OHBqKb_T?;OysOg;wp^F2}OkIH5?
z2^HMS0~A2##5xr6qSB`r*X_Fx&ovdNB;yi9E-P3o7t`S!NofGgorFZI$r>L?<o$Xy
zd*7s}e>)uhfl6X=A_@y*{V1Kb?!1}O?0r|Zx_i++`p39?oDl=e#P4W#0InnQz5Uih
zTXa~KHwQ>#3t(m_Em0O#I_b7vP?vV!_`xNLbIKm(dHZJAe}FLY;LCDbbF%V^!P3->
zb=6bpgAp-=9~HSGH%lp#z7)Pw5-FBO%y*BBU_XOWCZ8WZtsj4PpIf#@ghaB0`Y(Lv
z4iF47QnJtP%DA3mjjRRhtdsN#0qsmU+1L#jBg_Z{lF*}FGq~0H9!y!%4gHF|qXN)j
z^+knN0~-!NQV_cNX&Ay71%xweUf{U0J~GXeONk(KUE6Q}5eEbt#>b4j0XhC8Dj;Op
zPZMB?N0xy87z_|1dRf4}58G&&Qf$8r_-f%XRY%JGXZ~r~PF>}O##5#wS9(W}^WlV+
zD+o*fl+k3uKV<F~xBt{bl)NC3yvaq%C|Ldd7{Dmuamw*l^E=d|L2Lz|GCTrKpOG|9
zpGnHp5_UKhnNVU-*H!CXv~kLGp5x72z5u;f$ye~0^FjgTg$%du_v_D93xH}_D6>y+
zinOx#v8QN$nB-yJcO=OIuA^#p%Vg0!P#9^;aUtoP8S;CvakgrCe2Ut$`cBN{(fQ?)
zexibuX_FG%lbVH?16~gn3~5PCv{ZaTq{+7EsqUcnr~IMn?g8De7XGZff<NL>pmYT7
z%aA~+z)k=z&nTYy2}TwZZV}fs7GE?h7^T_OjD1X;`T(zPtPr<Ll;}N2ndkX3Hi220
z<89YS@*52_rUM-#8jI40g_sqtjfW5G^V5CrQwC*<I$xFrrvK<weE5k@bA|EkF(Op4
zT~T#FIT?Db_sT*vn$9wuD5>eQ{yKL>>^1ufO4>P|0sq!vMQSx_xzLP&<--NMr3t|w
z_c(@I$9Hxx(0j?~6U*G8gG`37DMoVPj-1_+$7m<Rct#7M<xlkHXn{emEOt3~O&YaC
zgnJbIxW7uN$m(kX?}@CK+hW=Z{@^S=#No!{5q&x(h-B#Sj~@4hDb`ZeY3ZZ`2|6l|
zy7#}zzF@CxK)-sdbF_o`e>#f(IRCEaU7(9<^r2DK<owJx%p#g99M&>4?VKKLYC!WG
zy>60n^_1jWQ_E-Ycg9kn<Q{r);-Elfi&!3UfqGLc&^u`~aonKeU89FI;ZdVT^FgTt
z*ns*9$EQQ*_Up;8Ex(l(A9FXH$dG9d&*fyx)71};K`iU5FzaJ~y>hS@RAtu)@VXLI
z$g+p<icm4vW<0YZyHv8jQ`{t>Jwe@H;C>cE!$stgFJFFbVMRtR`B6<E03kcs1<X$x
z$zheB1fZMrdd~K<c68`y0bc7^biA2mCf>Ky;Hv@z`QUY6YBMV!s1FW*`gahx@Kyn4
z+lAMgv0>`w>?V1vDOiSZ3Q0^(y3Y#iuCmHH(y&C_tL^ixiDmZUf~YVSZWdmRg0v<L
z!?WsE;Dbay)GySDA<Navs8Ws<5q<uPd;FIYJu^5C(}-2O!U&%n^eh`h>WNzz1K4}U
z?b+p&f6Q{m#_9YDw!%y#-#B5V3YQM0#Z=j<U+p{A$0ujQJQA)_taJt$CNE!1`Bu~S
zv8SvQ?d?cA7JUP*A%ila_P(JTA#=H~5-vQth-&B9vXe!OG$1EC@KDO{`e!CCiG!xb
z^EMQdJ70zF2wc~XJ}`eWeJkhSs_>mN*d0Ufnnqc$*^VdZ!jbDtQAc|7(7zDt!ZtKG
zQ#$k65*GX%i*ETR*a;-Soi9z%3g+MWC0zATHtlHyp$TzIF2GKt=vC(){OP8uO_~F#
z<<R3w{vgP#u-^NYd@n9*O$wL51S8=~m6Y42Y<yNNYsPpXokY*%=OPdhRVJ4+$n9!{
zu`K2g3)Z&>wMW|n`Yc}PYxHS?GEg7PLDbDq_B0apB2`5eQ<r8SlLGbUU@gO>;jvyu
z%)ocbs+;HQD8X5_DE?DPK*|ct*wy8Jxji7GNkKe-Ho-YhUC*hHr0_dEWIQo>jLNh}
zGO0ntT#Xh)pz{%J+%gK^Othi;IcmMC2h6h(xK|%N<Y=T_efy<7DlrbzZjvGLt)6Dz
z)Y9t$@9qR139WqCC*oi*g9F3vVHlq)GDBp;9u#uo2ohdsIJ~;@2yk4Zq{W-`I4xx|
z8!ubeXD3jjkb~2~gyg)rEd}3(v!vwjbRWuuRbYmaU*4b5BQ*)OehYZ`QlHQP+%#kP
zn4YTN2c~$zm+F|8seCQPD}rF0p&K7(&5X~WsAGsR#48_OnEw?}vA-^d3w*f#so!T2
znB=Z=g9=IyX_?qWk*7eXgU#VqOp#9C-~7Q*BIdiLErBYl78qwoPs@XTe)q^14NAY`
zQaB!eUkoxtRKM96-fVqCPCOD*c!w)>qI^?=p)a6?aPqvHwCAXZhT-|!fE*r!390io
z@;n{&Wx?DiY14fPr*Y|98x=e{^ca%g-wcE(uKG;Q91L`5^M$ner50Ko-8m#q1Ml8T
zSM1BkaV=zsHZXll<j_Jx;C%HtjxO{tG9(M~*IuJaSgVvsfqqf)-NL`|TWRJFa)FN;
zf2+Z@GT1ziyh0NNsB!7#8<(DCleEs4@p+`R%`=BtUfr#?v_S<PBA3syqgd8qZ+(_8
zd4EEP^MIh!Z;iM2(W0IRtnZ}RM<e=Q$pm<FfO`}3w__byZm^>S^sJaKKBo;L46Pq2
zu$O;sMmJED&kHi+cR_@FIV!I?E(A@A35XtdUB2N0`j3QsPx*bedRC=zLY=~*B|Z~c
z!Y3+KQ#NgzX@;405I8uA5_0Sl64f8VwcqQgi>=1mG0GV7s)3)qeD{H=iEPqga?8(U
zX*tshT;G3ihQFVQc<#Tslsw4&*b9veJ^q+xOwv@X%jd4<3Cy6^nMtIO4X&`6D_=#@
zt|jH@1ztFk%;IYPYH)02yGjtYn;7+51bAsPQF`Mg<*Nx#H(2qHx!?F!eTf8AMU4Sl
z#-L;)U9am{Gd}<Z<1|8WyyzgS<inbkKv+-Kr(oO?d7O^ec10%T-!S^06BSBq@SZLP
z=^y4lf#Pt;#5XP>BQEyZGS_)%T|WyaBcmeayDfX_V_27p=so)$zqYRs4TPx}bizz(
zJS^QSYo#~kq%19(wkOHJ_Nt)e6|0S%Fd<yOpD>q=NaR<ILUZjOP4GPG72j{(`O#HL
zX6Y3g5r#U<%IY^{5L|utHk4uPlUIUp_YvV_4xhyHDO#iD3iN|6Yi_{Qr{qVo^Pc(k
z89y7k{m)z#=wv5^{P(-he{lFLDbm-OnB6OxdK-5*vCYRgee76~KD`^>J}b4=m9|Pw
zck7lXU<*;G;kFU=j)z}Y*bVjAnXP%(`31(^E!{<PjUMUnww-MGTMnx<&(L;sfSMcH
z*6XN7SyOuV4(z?!l@e@El5S}@&bi_pu$L|WV$-TehYCiFb8s(2=a~wruv8SXJ-kch
z<c8pGa{2d^<hH;s`4^cEye=_&ElwLk*FOVHc8e{1e74xJclkmfxWJY?>M3gg^@cbb
zZSWXS+zLQlyPqkJ(ZuQppe`eMkqrvQ(U->fufZrBaX7{kcMN*8pHyw#!-*5xgdHRz
zk@2fDH|!G!<;y-#PE>(V@f!0XRg7$hG2wiL&<bOqr*+nIQOOIT@87}P-}8PMQtW00
zgqtfo(Cr+qV;sW9|1f(D9Ir+;xzWA*2n{eYNBEBnHuijZHJ@mrN5H>T$|(OflRBcF
z-AgUBy~iyA^o2X*@rB`gkRUr*Z?rEO?R;~B#$zJb7AAiRh_~A7tU&R5$zIKROn)L8
zxL087+A|&Q8(I*TJEV45=$!Z-Mh=Pw$|UDv{;D84cPakYj{5biQ3STO`FSYcmd?iH
zwX6`rX!<>+9U333AP15O``9hJI63LM_L&<MMQx{)_uCOo9H~-o<gBpP2{)c;++?7I
zRw-w*XtnH|A(DlFf|dPXrA(ieL=J!RGc7Fhxa+K$56ML1)tM!nsIG28tBP-p78$Md
zrqzM2#h~i6ey3IeB?SJw4T5>`cI<6#2FP;AwZghT7?C^NsSgK=CIQbie7r*y@X@<a
zHw3JmcrRT1aDZlx5^<!9J+e~*KMAd-vC6G+D~6e}C%j&yHfW;xw@1|9$m#Xqu`k%~
zVu>tLHlc~<fjzKmA`12I(GGXniz+nH3<%Mh1?q!-?n*OcXVu{c-lQ`wJhqR#PgI*1
z4U@>THU9mYy;Vc1Q&vx0ZVu@UT_zO8ykWULnF2;TR-!3gb^|G7^f{UG;)Q%@FrUB$
zxfZ=C7A)aaMMK90k+9J(cVtD~cI=0#o$1&az36^tO>|D)fLEkRUDg-UMMC_ni|H%1
zz2ST;eHuT;B5;$BJxGvWMCn~xH#LW&zVPV1#bzYmSKC3FlZva+hSH@!dNpQWfd!7G
z8NEk0lFBb1dUFC_Y-BE|m;8*rVkFPnPZ^=y$@O&IZ4w^L5TJnP*l)q0ST`cMrQ+Sb
zi9MymPG-X%@_LEc74?IcR;6cvEKD7W?il9a0?c0bm!pIASI3lZMPDdIe*-LK>?Mk2
z53VMBz&Kd>Mwh|2I;*b7^hTXH38yh)25BUphjAR#^{~m<iyoxwKPm2pC@kA07(<fC
z!dfQgLzOB2xn45N;3^?mPAZirUJ}aKL%M-rzViwrtyPC8qL8x`$_tZ}JD-A=RR9*w
z7z#3a6je7jQNiTu>FSGwivkVI>@`g#y0|m(z>?3E`ox?qR@^NK_AARhmHhcb(z&7h
zp}R`KeN1MXO!x$R{acvQM#C|75j1#t$$svVt4JVWik%>WM!X~j^Bn9w!kUs)%$t!b
zz%oR(mh8+nn!sq|_cg+qF~2M!qIq<`^ADyP8b;s%O_*D_T?WYKB$GbuGxKZ4VyoI5
z6}>uv(!Jy|q6tlp-?U<3JbE3S7^3l{L>lOF(-U)9{_o{bcx5m7GoyA9nn;Y#qa53B
zSaX9Ki^F)c4EPz6IoP9L&VA5JDC!^zbJXwU;YpL^l>|^_CSwzbyeTxCFmo-*fS$9o
z#Q>L0>O}$y_q)=Rnb^2K0v8UxEeSw=BZpHo4vCwje99;z)GM_39G4`!8-%~vE|)pW
zG(RCwwE7)6#FV-xaqyz(vdM;D`qg4(q4qa&kUo3zUgD1Ox$D$gfJOD*=**6;@bWqv
zVpW$cP~0jGsPbO9xQ?A%zD2!YQp!5S))fq`G*jXGTvf|c+gxR*Xf^)Y7nM;W19%yR
zl{}c;RL>ANnV)iGQ<0kbrQb7X<d~zx3inpV4XRaglOvPtORtKY?zVPK`-M%W?)57q
zqekkqnO9MTgWsW!uR)efS#t4EN`aGt<L+~R*nZC{T}&c^Er@dQDWNo#w$e)#$7(|$
z<6I}hBK{?m!)T2ff~}CB(%tFTex<aakEA0cwLK>9DetpV3qq~t;^<TmD&;L=dTv@W
zNZr2~BL&b%)YNhLM%-nB;9E>*iR+2B++!^=dWCRH#x0kx&^7aMPF51_=tOW{nK!yb
z2DFv<UKWSvQ`#CCXe<G5e?#D|YfGqMMNK<5Be6Ol2CCj4ljv1njN~&Ok;J9ht+rFB
zRYfl<IYm=^qR02j+b)bzqCf?)UBhbqT1~o?C`2~7?TfPBol)T)mD@Taoa;4N0=#TF
zv#BJF|8=;sl7;}pid|}{Jv%11Y|^n$HH;17wP^O10_<fG1jmB$slWflb?IuaDxEF@
znz{}S|0396KjZ_0<|Zyl0k*jJxnTA0U_)l8d=iPc%!me_u4(^5TN64v=Va4F5zuQ@
zRyn|}P5u?hU}|AmT!ChIaQJ%9C}y;^c1ncHMp%}%<k~`$tu&4_lb{bwdK9GQ69^1l
z{4H?BB+<3sahIK(WgiZ_y*x06!QOP|McuknfiTwEX+`A`4qUyL_+H$u-e#F(DuB3F
zC4}#~o7MeUdcKdkbb2b~x&v73Ojh78;<7VY{o&500(fS=;PlT=zbr>VA&gXom^9~M
zG$32JF2A);q8P#_s@3-#W%m2F%mR~@<wRd(?u5LP*`TNsM%d~6jeiW+2u4sLt3n?m
zSqzuA9sP;v3sD?;4zE$O;7#oxDeS_aWwQ<7q**JbPZWYR>EI2hMEW;Z3~sUxC01Vt
zN6l752;OF<Y5p{Mt=0W<P@oJJ(~w7M*Acu`nK<N!O<qt~Y0>2LD=l0n%_Hvgo|BK|
zrpwQHvm*F?q(w2<LB8j?b}?zA2S@Pus*OL|qjiqR#MU{4;utq{XXc$|nqlyf@ybP7
zNVg_7)lH(r6bcpw5qmweV{1K%M)!kwSj<KPKkXm8!obAw*OC(K)&b;%c&D#ZjqZ;n
zB-s)xQWpI_LTo8`n&n@h{q9%Ji=tDSi19akY#*}WPCuD=M(QRs|Ho6xV9%v+fC9wD
zw+aDFe|}cIx*evPN|Tm0pYFH<1)no1>*P{7&Aoolxp=nl+MGGE^N2*sGLhV&)UpeU
zgUD1A`~EVwvM2n;;jKzMO3l<sobS1qsH2scWz6>1AXo|O(FSFmQLk+K2p_Cb_Mswm
zw9`Sf+nL6=6?LJc3XJMi=KVb>LYs)!z$Eg{L3HZ9t@HO6nfh$8msajo$H!9mOt&pB
z`y%<qR8P8su8jh0SFRf-@FA`*@w!%W7$&L+k)unVF(qD1;f6VG0sC^$18%ctbAH0*
z4w022Cll%5mw_!#B?3}yHJ?A&e%V<P?F+IlA(f&zo1N}HR}7-2tlxJJadJ1ZNCxCv
zN5Ty3>ZL<8;H)_@-)M{oo0eZ(DG^`M97PDr9QPsl5j>2JP^2PZNT?C?I-x2E%2eT?
z+FVsPxFQ!4)XADqi-~KF*l6T6Z=f%C#0g6a_Z8T4Uy)Q|m0mu~uX~LTs(|fF{AY=z
z^)|%bSHkCVS+z#=F5ZI}U(pU6v%$Kj!k|1%R~;NM+xEG&A&SIoOwhi-z5t`yH?wc}
zCc>bn*Kp>V7MjD;>*#x%U;*;eV;CfxE4+LI6W+S-itQ0B$A6v`VQQi=qx_`;GE9D}
zSJGXKB+oJVk@lC=Y!<Xh`kb6;-{`cQdmK+k@^Ov!@r-{50AKxjer#kYw2<*7<<R0s
zV!o+ujtkct#iw7S%*_bwKwsT3INyH!l*l?BCpxn1g7=mc(H)G&v-PJR8yu6@LbWxZ
zHI2ne6I^8~i2=h20tVwGQlo+}s2L^xZ{6*Ql7q6cc*Ka$=#8wcr7D_v-O`6L=7C_%
zL0_Wbax@Q{g+WYMtQ*seCZhOYO1lrmBTI2#-<0etu$Fd4%R~ioC)Z(J*P==^_0#Va
zENci#G!yi(m}*Zs@#L5AUxvP;PGqSj>rn>dS75G!xymd0agH@UuzqgPz)M(n6-mW5
zbGG8q{mIVq<3~VZy6O}bbB{;iaJCEm`7{jeQnIY*@OYW-<1-eOUN=>X)|s`3Rw4~O
z)^5}4`?ZEOb?Oe{NZI6**#)ydR1NJyh9bESrXSzVf8w+hLqr@5uH<&xfqTW0N0f6!
z7C+{e`JL<91y@7+Se8FBPcD;}ebu~|%;4tLeca*Pf>oLFcoMCXx@1p|F<jX^xI_-5
zha-=V{V7qAw2sNzkM$G5zaT<bI{V2T;)e}f`jQR~XVbpc?vV(!xslw;n0tt)hb%+C
z5wd8sn}Wu_RgVlL0uK=appY&#{p4B?-yaSjRqYrH&h$m6h1J2GpXogMhl)S$zUb_Z
z`O?2j7)dcoA!!%%8tLar+X>;k8xJXh?E%w`X1pv|>h?b`Gzj+qc;P}zb_9ujrM<A_
z{c<&n%i3Bf9#zQ?KxM~hF|dpTC4Dmv_>ko=`ZM7bwLm??hH9%R`LTB0o||XQ&x#zv
z-1%?V7PY&-n$u{)T*^o^V9O^g#Y5NZ+K|w<UxdAx<0u|)Z^cZj&hiU>OjB8pI+tL=
zf=zK*ZhyD|3MCWOEo%2CcNEX%>a~;QT=L7N=sdPx41vg)CTWdXMvDU5)~`&gk25n0
zynXXNuk8Oe0#v|hK&{)Ud0RmHMK{Ar2bGlJ0)`nzvN1^}vv#AEjL3J42|uBUp931e
z$(ae3=H_XV5beu1TppWhfJCnE*MiddicNcki91`|pNl1|UDAQ~7QF*D3TnUA%1CO!
zy)1n{5L+)mCyJ1A#arrr*zpx;-69#Z^x>rBPAq8-Qhdp|qu2!eJikAY=8+lI;$reH
zS1x<Xxxc35rr@o>9Ay%PmyvsgatiJy^k)OrHw4Oorzo-<vW-yu02F+$KadM#j6Ayk
zku1sc+hl>8!Z~x&%CSwMPAWHf@j&xX8^=3x;3D`_cm~Nko+~dw5#6gn5%4Ss8+X*7
z>^bN{2e)RKID%W`o;9Nv3T_h`C<QK>5Tc&WhThtrzvmD<+H0eFt#Rf56ZUP;6qGaG
z@&y{8Cfm&B7w97j>0au7y4e>MvB$O8KLCra&G+Ps1UvP`IPiXVFdJ%}&>;3n#k6;O
zT6=yuaSV83dL>mP(y*LoiwUnmsoHo5v1I0-9ll?nEm$z|r_~qSgeeBt1PWcDz>YCX
zulSQz0otme>^4bV<%z3aQL3TGwySsOAjSb-r>7l4Ts8E++c5sO{R2Y`AigbnIkKG}
z$MUU*-7dYtR<E1``Po4&p}uPtP8u&4YP_uKo2ftmFuE_!j#g4GNMRpuOgr4ZNcx59
zTUMijUy*@JTH6UR7Ge-cePY@{=A_HfyWAGE>9_Q+(q50HzFKUESrOh^PW5$~#n~;)
z-FgYf2->b#{c)1v=;zJe5cMW}nKb^wZRJBvG@OkVV{MW&UM!Q^kSrGr@lyGz*!N1w
z)Dbml;qkRSanLpn$w%$3wKFog(Ap8`L(~9*3RQABH9qe*MI)L-wo~JWi)bt0v+!Fo
zm%3v!^t8g(>rKm9Tjbp@Z{Q;(+_~P&dbNy2(d1+s%+HaoXKgSPPy!p<c5kkNs$942
zszCs!r`zE&Csat+LdBzL39mKe+aMQ_M6Bdn(9}0RrzjGle;j6kL9E<2zbL6(;At%V
z1)%rYP_7Ja2=+L;wCibBJUeN5ovq|zgDo5xwQyqSdiGb!y>LFJ&CPfyik)~awx#jV
zZpZhY<Fbr-NwSMv%#m5gnF6Xe08*sbqvyOkJFB(qAq25hyVWQR`t@0K8^Xe&EFzOJ
z#uj_^QE%W;8iOSTf@n3iP}N#id{nplx%#r44Nab^I!JnWVdRR~QX>(0lD1QGEtX<#
zcQGjLu8~})XTHLWr4%!t;JtbeRO}>5kg@#ooJ*m4IzE6p_0o%oqV&&677-gq^IA(4
z>C1sQx+F+ooo*Wh*F1Urs?*k2x4MsQL8_b;&irk*9usVERo~2xFXdC2Bt~)SKg2Dk
zZlq7Jsg4|LS<eQCbROt!xSU8fZ3LTR!KJ05jgpn_1VY>ZHJBpi(DHTW<Ct7SZ*`~Z
zxp&Dc-GpD=ECFJWrRScVEs;xGHTy^nq4l7U%;9E55euqg2SK*=&BhkvnqaxJZaA?T
z8(cHc`Llwltr_#`;xvTXr0jGI=v7&x!BPISc_ogU5$KAGS2_Y_eU~7(wPvla$G*o`
zQ}@@F;T_LIo{xYKmwD<0^L1uJ5w~URk!xdi`WBl+qtJ~%tP-9Oqc*cU`!ENQTq}h2
zHaZKOUX*DoV30`l(}z$}_*2d$hCMw_hFK-jC$x?DJhhWn1HUg!-M_<i38c(#cQ4!D
zd10Y9R|Y47ejDxCgE|_5T=0{TD&ZB|{;t_k#T3U%g^A|NAKCvHoe05*9IKm$-W!>k
zUCpoH>vsz;$r?$k#+-;G?3c6ll$$hs23(RnhqLm1d7!8*Ob9TY-&|)fZ0ltGYN8}?
zL$bjjvtls@B|dBI5BKXgD%_8&>L&?&<r+a`ymx*uj*VJ3m8oz0r?Qx<rJp2PG)cI*
zy{t2Z(&v3<Xy}`wgaB$d>h#BrEzs9XF91P<PIQy?fgrdY^P@RgMGvC5_B%=}^VGw5
zC^OG9F^J!k$rs2{6lA#yHpgE^+JL!CHCVeJO87J~W;V&fW2M)!3Te~8aW!ZZi>f!c
zj6T}OSc7d_KfsBDvUJkOy?f<w&cCA`guU5baKi{<b&+`+yi$Ipv{y}ak294wMbCpD
zl*Q`e(?^aJi^;SK(8};S^dx%u0iPNJYP+fy*wU8?xr@{KRqFXH1zju_v}xyY$%pv~
zI}?I=oJ=o<W1dU_+Mj+IT~m4Le+hY>g7zdcefndauO{lv05F-*MBAudAGil)L~?x?
z#eDjliSz6Oj&Ec0&w&k;$;xhfs(OkVZeg6R2!4G$Ek0RHz2A6BGYQ+PL#Zm4@z-?(
zBYRM3b}HF%@i!!m4p5JNRxA{gC#=_0%Yt$yva^DUA`@_Gb4RcR2vF5@#9k5!e0$dv
zUi!84*V@X-ouD<rYxQRgN-gJ2bU25<U&&<9$sLmEBW{%+woWcI<__GkN0p&xvn!TS
zHgbQa=T~$UU$XXgo{lERa2BZPSKw9_8>bk=Bv?!`v-ysu!|>1KzLe{l(lU*e^>Xot
z_LlWKe-B7nP}KVkp3qV*=Et9$BwS{ZV0oo;uiRex;lvLBN>t;Ua1f=}{@arNKY0<(
zuwU5^ppdCm5qUE0J{B-A&yhW+(+vFU2*Rk6FvTRR(p3w_m>UIcyFB^bdv@@FV+8ik
zOPQsH{q|YrOK^q!UzabL;2%^XY>4cUb$5Y$ygu(7ho!1m=n=G?U|g5=ay}6MG7b70
zWtMH<^>UVOCF8smPqOGSif9cKXik^s`T?O2>JQB?pSIMm`WBL<VhuyRDtQ9i1Z9;-
zUUnm;De2@-Y3nbT8q80Cnpw{opR2qHdei7mS7GFX&hbka)gjlOe3hESS3KB3dzB95
z#bDAGoNH0aNF(9>j<=*zQ@f3N77$^XD+MD9gbvS&b|z@Jv3aH|RGWuq?;TuQN^_mL
zzVgifejDqyBcV$po+Vx3Ur_vqi8X5REJ{zE^gk=F{{aD$`u~@rot?~reR_fAtJ#%k
zFWL6>tl3EQ8TOocFnFo=#?K9uS-oC3^Vbsa|NR2BZz_I#djfe9g(vmCUs~>d>OdKv
zda-4-iuev^On;{#N_WnnU$)TrTU+J44wh{W<TGRrh}g<_$@UFLJnm<|S|joG#^2@~
zuBDm3Lf5IFMX^Awl2Ye+v18xAJ{yOOE#<G-8aYXXsBT26Kxz%jBiqu7e}{qtFzmwR
z(}|S;B$?H@I6coI70-FLWt2uZ@XZm)?yXcAiWf11`kbP;1ugoow;ZT~<aE(Rz&@+h
z&ER}?lq!!f4yB(?<o)R5&%)o93xI-fU+BpMR^eE$u}OWc!3Am?2mlDRqE%6GP4OZ=
zygZi`e-VuIY>_q->v^nfR-y5u^44(cN@u2Biy!{peXmnA#kT)`-RRQh-Q5AJ&6$JM
zv_<kMtB*k{i0uI@DFj+J*x_8a$tp;E+HYkDsrw^ZQAYz$Ot-%;)GH8C<hWk(u&UWN
z-TrlG^fCja@LVSEgb2haq3302K!txfr#jYW@b`=9&5kkeuxU=;fAqd=1v5(kV5*_O
zzhjL52bz0-?yrq^@zwrXEW4e~S+&n;YNgA4GENKU1NH{h#2L#`{;51Y(_Z@C@Z``I
z$i0Hc*qSemcGHXAVS33k@=0;XE1hqY+)JH^Vj%^(Yu76E6avwG%FFMp8^jd%L!<XU
zpu4vJxE3)Ltu7k7E*8pUmM1&riyGPpPlNn_(Q?O~x+`6gn%S;K;)}k8lP<6<z4ztB
zpEv$oH?6b4QohfVdjJ;kyH@TI^|@ck71gag_xNo;yU60FW7OGqAz%7Rw;Hyq%zGt&
zb!#(IdH~k;DAE1>scqs4+^^KD-1DG0&El7<Q^WT(qgx@fPrlh#TqC(p^R5h;iFU18
znQx=H;mvo2D>c&~fL=kFa|<6l@%g^<{5{KT5J;~E3fnc4GV0ugmT8>cLqEf-XYrS{
z1&txoD}N!V8(<NnQq&8p0$!b3f}EeAqJ^s#WUlLk0-^sBsW(Gg@efp>1)oiG5&=wA
z8ZyWjIea)ODJlL0mo%iwOw#h`>3f>rH~hcgr_JGW6OI&!<5K^v13kXXkLMzwOIhiE
zvQqsSUJFS_4lkLWO2o5+b;oYKH9iIiRa8v=?I88HcFYkoZEc7|^{2eXfjGK<d$IgS
zP8ShNxATPF9shjsM!PCpiT#^<Cu2TJ_K+ox;tn%O0BitwWaTtFlCLcFREbc<vhOsP
zZ_cVZz5b582a)Z30X2;fCuT>nlIgMW0<iyj|HA(pcU?{U`bfQE4c=b{!o~qslm6jJ
z1&WX8jri-v_%MaXMl{KjJ?M~en|g-B<329!)09svX-GC}3OuDZ$Z(zQiIeC@TD3n;
zEm8KgX3PPw4EB}@daS{xH4+C(OZuKCiov&>6U8clxKOK^W0-$R05po#r<D`%zU<^O
ziLw0za;%kOjKLW(Mdr{W*KsoQx~t1?osB*4tZ%_En4uf}{(HkJM*n8cqUT-+QoZ%N
zL&83%b+$v*KEHLl=iLH2hWyJpsBe^|8lT}<YR3Anv;BSVu%r35{w@Q*kx=So(WmUU
z5CT(;@Ck7!zCKPQ*|c5u@&~c)=eyd%fvTveqraXjwnob#tt1|YjqfN>?rfFIV`WYH
zZVEy=D~q8INs)u*9<Gf!y$h~AR!i5j5I6bHdi;KEM!xX+$=M}qc!|y*{|AT`$1mw<
zNN;3AD%?RY;Ap?9!K0mub(tuu?2bViX3wWl|DX~jn7MG^F<QgAtX=?0%;z?~R?dbo
zP69Klaunfzw|%z%NwC_dz5D;7CxO30|3SHb1^-v!ze@ip{8#D!Dg3`HlwDo8-D>pp
z8a(MiUpsJQw8LsV6_O!p5F&@sBXfh+{3i`E75cVZ;`hW3Oq0NldrzTsPI%)#p_x=#
zgW)y1cNT*09(<HJIvD3og_XC=AiWFajwkirc0jh}jDmors;)J=yPV7p1F%hk<q2PN
zpn`7ek~f}V>QH1uvH6O{08GF2K-;0WPG7qXE~483(Ly(vJ&ZV61`;{wEI<>+$@NSk
z8+uuChw#nrCq=DQMiEy}Lrl3tmZCEs0U-Z=SpZ1?>7DH_C4T;&r1pgDf0Fc5NdHOy
zjqqQk{}leK_}>WsFS9=>j2bI9Btj45HJ;?Zdo17mV1n1PC?!KaBW}MhB}08%CK5j<
zGxVJ(>h&3Y{)_LRBBHzEJb-N(J9K9%70-@mP9O@+z=myE`vLmZ@r2J)t=03^=7(}m
z@51&o<H_EI-546f*6;oVNy`NQA_tcS)@Hd42fac)c!>w13Y311?qO56Z^DvZYMb1<
zV6fBXENH8j-mtS-qEEChpH#j--pqLR2)%>Jdz#v|{&h`_C^O;mJc{)2pVPgT<i43t
z6;sx!qWdrS{|`a`7fSyr{1@8)g7kku>A#2cZ+W@fr~XIr_c9zT^6CAnvq+WsrX`1}
zf2Bo~S&QWv3!;mCS_-TOi<GSF%Ol{wS-Hn$+XJ%WkUW1na)&h6k~33Z&L~9k%7@7Z
zAQ4ESu<~*|A+(yDp8nNPYfc|_L?KPDTDT3q!9Z->ez7O^C7z?x3(jvzthqQOA?1-W
z7LIp~Pmc>6^j=-<FP1JW&P*G`bUycs={w#Lyt+MXn!v^Q-CpKrE9rXzp%x25bw~j|
zq%VYXECb)l#v<7m7YzHK5Fx+7d&`t6@x;kveUg!b;k}hrFEA4Lc6vN$Bl;117?m^t
ztzp3p+*y_}OVeM-Fpa?keyw4SrCAA49;T3bArhZ6T@E&p7oXDTr6jE$fAP84?l#;C
zAf>iX-zC1ATsl|E21K!GHIL2%^`~mSkhybhjjwLj7#qxqcgciu-$G36(DM=2BEuyT
z?~M>9Op}q6#;`KyEr~xeTly((54f@22C2?h2DRYRB@>pqH7(ldVuOGxy$z=>*OI^X
z!iarhfqXY<cyG6pf7Rb*E$XHGSt6I;$@qOPYZt>|_t)Ublfel25kJ^r;!b{nzg2!#
zmk`Hx@2e!R90ggO3-t0fFuyFoIXnsK{h=T4b$k3x#WO*>o4s^wB{22~`h*IXC6Rt8
zHl+!3j-97`g(Mn-vT?Io(#<<8rpb1P@nTVCiir4vfP&LR>TcH^3wuAa;5~FQ87sNi
zStI^0DsD4oIr#3YPmXZ4XpFrm@lpX8VT8J~zv_A4HpiseV#`AxdAMOn!wTBn8Xoc!
z+B%YNExZV<Q$6!W9a}jiB1@RIKb(4XxK=)4wrqZxB5;fyl&do}o~Tx&jGw@+pCV=j
zQ%@y)eWI&1N@#2>I^6`>39)TSkg7Z#+-#b%%++=kFd-JZ3fBP|d%P4-8}w314sgl3
zj%a5qCcg4|?=w%xnD%$cw?_Sd($wOIUeRTZ&IC<FYJ8w*{j5`mr7KUas|<EzzZ4%1
zydcevE*zR~g>dcU<l{ns_Xy!04{ucTN%wwFulp>Q2yW{y*A=GR%>Qimvj8dupHozB
z7)--wl!=L2?^8oNkOiQ@RmNzA@7O|(3(9$jTHhF~r`k2k@TKG}>E&2R3OOC8x~rly
znN5is{Q%abC8_;2MU7d3QqvTEVOAcz;u#GhR6pWRaXJS+1@>g5M(}xh(Z=0M(t!dh
z-yDO~W&4X+lXP!zXxkj758^}i04=~w-%_ykacjj#X|Al1u#TnSSz+MgMAI`~+Cx7C
z0{y@+vbHDjg_{sa`w#z%<dK+oq5{R+XAkjW3duVa8d4qg-52XjXAQRRa*gkv-`&uw
zh_I(uMP|V#<ogbk#XLG${<sU7`G$$~z7Mq67xb$5s=I#@ILCtH185YuiuB#tzxt}q
zY?gE#S}Dp_Z1z=mqzRKjT1cmCBFul!QBxC6kpaxcR_`0>nRDG?{LBjal$Hsa^osHT
zVl00R^Y32?*&{B95~U1;dr2z#he{T%yW_f+#aBO7)Kh5(z%bHf)5v3=ZVt4YUKwwb
z$SqmFTTpYSTTH|i;7DDJ*9dd}4R_uG*3w*$Q6W!4b5T9wwm=PCHkUmCOG0^yMYL|u
zV{iLcX>fOF@{`$*=VaP*=Z@Bz)Cy6`OukgE2xma;wZf3o-8a22!=1AyU$kHdfOUY{
zPxXYSeH!3?Sk(KO6|=Ld^BMdu2+5}==@qskWAKc)BZJN>g1HbqciL?d5`!GZ9B+Jr
zUBFoQG&ZUj>MPmG8}ye^^-0fy2IgoV$U2VZXe3H8_`aQrbjH3<h%HTW*^J%jFB(d{
zFS|;o5KbFzUp)|^E!wXy?f2haXhK+q1|8`DI`OrUiZHW)NKa?A03SgA+E2&7Q~}+S
zH`~s>b_!I_XJpQy!;kl=<~{)ysk<F)SG*rU7^7BN?NAy1^Ym_8nHBruBU#YI3TYs$
zXg{PjAAIk0v%K;;56ZH2A)(n(S^oDZS-RkfP8PV1ifYuH6n<&{(W5S5bVeIc6a}ka
zKTFC3di5x;HLpYOZ+-grJThfLse4}OFEyn>hvN@$8v`n1wN`fU(iDgH^z6H&FR-FA
zx7|PB@*xNB{hgANpQ4tJMwXx~AhQc%q}Sj9bUi<4S6tRs^SD6Df&x}cerUXw?<mb*
ziHU=quNUYe3K|YRfQlXj>9YQwb^8}MSnFjce+nE)JL<o|!(zDm$7xnvit{gKBvrj7
z#du?DnF<3r5E7VoLddbg;ufs+bkwbn@B&R;um48`eG;i4MQ{icy4^x$@*i~|<O`1j
zz~vW^;@iggP8JuRm$k?5ypI}Yq4qC2znugzQCUmXR3y_=$T5;9-CJ-uRg-%R|ExQZ
zeIzFWP4IYS^#~mCHrF>8C<Za|P8yFv#vz8aPdOuwaZ^71eXTmL4TSA8vWA+(ZHn=~
zP8*9R-RXzv3pjz)Q$IR~K6JMmJX1}6DJb=R%-NSU8N9(h*S>+jP_RUzk~X$V3+|Oh
z#8oQF81K!yJXJuFj6cG&N;A1MNsp2!De|j|CZA$_-q9wwx1hnf3qw_Jqp-wN4Pm;P
zgorBz{9$9uXV*%?Wh>#UEVBqzW@V1-%TKF%*-4q}_v5@TzwB=FF=^OCl!tt$-0m&S
zG_!NZEiMu0$Upm){DOi2KJktl%_M#jvgT%vvdNG}#1Kj9z^ayx!oB-|si^t3h!s%*
z`PP?}@gr+{m3wa5^*_oTBX)2kW`iGoLMc=gq{pQ_w50agZK2O;J14vS-5S+v4HQwX
zz(~xIM5R5TY9^UO1!PUMzXuvoiuQ*EbtsUImdO|1xlSWHHahJmHH2vmTOcj(ZT{Y4
z#ne%1s#wgz`u8EjaMcvK3Z)+v35gS<=iSzd!8kXeFV>fZIMmOGxbuTaF)Sn`3?%AX
r^j4vQ<5e_@q8%d6do&30tM{Xh>M9%?3QzxMfg~rTELkD`Dd>LyaA|;v

diff --git a/modules/web-console/frontend/public/images/query-metadata.png b/modules/web-console/frontend/public/images/query-metadata.png
deleted file mode 100644
index 1b6c73c10f4862a0982018ddca7345b6dd7b8dc4..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 39361
zcmXuKbwFHA?>~wb_aa-gKyeBzE`{Rm?z%X|t+-2(0t<90?i63#q4?rf+}+*n^1Q$A
z{Uc{GIWw6_^2yB6C>14XEc6fPaBy%~vN95CaBzqqI5-3vRKzz)?TUE*TSZevK|}KL
z@{*I2b9i|8_V)JV<m8Bq_WAkw`uaLBFmQT$`uzO->FL?n*jQ6jQ%6UKpP&EW;9zNK
zX>f2bBRwN0Cueqcwxpy)LPCO<o4XvFu%MtYDk^$>d|X6W<m&2betsSTfiyHUyu7|5
zARw%*ty!2`R8&;R%F1qTZua%{IXOA``T35GjVUWD*VNSL8|Zg;cZY?Ab#`{f#l=}$
zTYGwX-rU?ICMNFf?M+Thy1Ke1Cnq;GH4hC9d3$?XSy?$aIM~_QdANJry}q8Eowc^M
zy}sVHwY3F@gq*y-Zf|eDzHE+;j%sjm+1uN{-cMf-xWB%hlB3|)*VnJTyb`ihzg~Bz
zrlxXoNj~n?KX2sE+`a|{1wGG&%tU=*)2ZMRa@fcgb>s)4AOpF$b@Es4&-Yh|jR(!!
zP8u5<MNyDBrTxpqkWz3$c~z44uW!~`J^QX-3p9Zz1qL7hyi)uaSHSe#$z|BoW%uC)
zJhkUB1>Xe+om}nN1uVy7_$pkMb9DW*HOVZkW#vwjZF#I7ksHDf*=%WPiD5xJSJcoc
z*f?x3;{En_S|{N8V7NOk@uo@f9k$vLt5xm!>xor$<JQegbLQ3b^Y6@V8mf=Gf2zeT
zijU=EZ?D&?Yy}?tl`Q0V4(DM%mtWLFs(Vcdx6V$=^hy#1N>y#cQuHPcCwBT>ctXr%
ztk{cv>Ngxy+PnKVE^=q<X4mWJf_ceH(mgaQ2WC7ghEBYuFY3>lX&C?H><+&t%PaSP
ze0<v2*f=^mdVPJpxVYHa*?D+)SYBS1L!Z)xgG;5Cl@QhNGBV?<eO(JOuZ7Puc^?yu
zh;mD5#`k@TJusuj&tVZ2vGKEwhdJmn8{^BspI9|LYq5TwaUOwp?14mdJ3s~(YyA0v
zzFu4aN<Rm)Qtq0vY*oJwRBUIs4+)dQ+0N<G#`T~)U^Y`wR}C!SIFw$oeJ@&>>cj0M
zLb0E~y1<5uwEuM7P&=zU_{JT{&RUCz;^woqWj`Z{{v>`a0HFqFgN(7`vC`AmZyDeC
zBCx&v>BglI=9T?2sbGO6YH$jcF`TE8Eue~5K^gh21=UmuOADXWBU*4u4rpA-7z4Lf
z;d8+i3n4=e>cq}o0>dB>GCJOkN&p14i>7?E)*WUdD6hAetWjAg1Ld#?^OS&Bg+`&y
zGmVHeI5b)s@ii>B_Y1H8fJm%E4HZ9`>k3K3#4z%2_N^frir?uh!eC*j@hX#B94WHQ
z77*;oZZvW$2tM6;k`B=y-jmE45UQRQlmwo{?$>fSxLzFeVKFo$z*5N?kg*<@-V6K=
z96FIkm0kT}GEc>J@;pyJ^fq53)1Gkm0y(Ire|GZU;P!eknFTPNH(FmU$wX?8;^dzK
zrB8sT4rIR#uc2{trm#A1^rQh_+&)WFhAMPs>lW5NTit$hv9(hWQ+aBXmLU=`8g2xC
zl$q{Wv*1Kx!G)c+scyAGjNw$<6z8{@I)0nB(Z~t*W5O5)%?tOrBJW?IoPMGh{Tp46
z(_6|cBR~JwiLs8G{x=VVTpo;%XxR%o(+_WO{CL!?CIN>bm)}SNHVkqGew~k~4Vy*L
zmFLTFiwD69>sydthSNAO(##I=bSl*}t`C(Nygn4iP7i8?qcahq(Vj?%T0eXIPz1>x
zvZ}S!p}IzT)R;C4cC{`P5OG3HAeOCP{s>Si!iw&q=g~AN?;Eixeh{{gj;Gtk)N1&Z
zH-K^mV*j|0L$kue!!uU`xVWws8=k8$MTfwIg}kgPEE(}teB1e$k1=whDTSgZsfOX+
zRiyC>5T`8yT_$Ga6IDi#L`L$4n?YOpfJ{<r)GkRIv((QSQYOz}`|hpqZUNlgoebta
zwP?GoV~t2!phjj8&ijoW2I^sSfh43ph9=M5VgtSLnj1gJTriIOZ7s|Dl4RK#+q+7V
zCKK@|N{L#3op2KtjLWZ`G+H*l=d7kiI#K4!_I2N(G|qq1UXu{ym7S#ZN+xiJ)6*Uy
z+{AIH0Lxim>lKdh7f=m+$z^8oD_>R;Gt}7`KKEc^abhz!hTNohpf%btdhif%<^P2`
zyFcW42;y`<rFbXeIjjtEd{3KCyzYLBR8n^+DNm<j4QrCR)W4I9eyCU|Y7F;nm2#l=
zw@J|*7YF6t88-%?-3_#?J1^xcev0T_wXC|X4&4v_i}s8}rioHT`p1txDeldEJBJD}
zwdN3^u1FK6&4%G5g)QNyls0Z*q$ig;<#`+=bKOMe5r)*5UgKuAOWi`_094-{B%+UM
zoZHKYVX;;TZj9U+m_7R+tL;~ahyLb2{H)oF2R(Y^R+|}qnh`hGRhseaAeCvVwhvZ(
z?JN>L?+12RR8psW{Pis3sB3gXGuVrOjxmmFQFB6Q-eBw@9?jML0e7@G7T<z*eQ%6}
z)qih0kA<LG;mCqb<`Y|ryS!)-R3>?jPC?LGw_g{pFqvmrhvx*F&sw+m3C;DB;Oyg@
zfAXl40zk%euRbrWF<FLCmQnY@->ta!?>-D22_9|Qc48?nF%(+m#6}tQLwnY%6)Z%G
zrwd<n=mZ6W_SUK*OZ<nPn0GWPt`&G6Ir@vz*mQ}O*jcrq+KJ{8XC++gcnVaaUZt@@
zxU0`_v)M)+3i$HWNv~bS0(M%_Ea#XH5^UBJl0h9`qbGwKJ3`7@tA7WXkJ-jITrmFA
z0ZTA?6xR;CE2A3wO84qv7%z9PsOGovhJlS_c~;G_zGx<Uw1OJv_E8grfcNZrI7qdI
z1U)m-xo;+wP(*gv1AhOJAR8RL^#%l|Fv_*}Qa<yPY&=qIV&zJgc)c%oC@WpV(Lj_a
zU5HPMwdTbgO>1X!YV=ePbxMuMH-OLz$9(mxqmmOG7LARlR)Q+b`3Tbek{xjCgU|>H
zP8g6g)p1uB5Y@@=3TQJpl+ae!V$EY010Cj-B1mEtaE5#nj)XcMgR((^!-kjKUxx%`
zH5b?YjEbn6EjYw-)7eSfk)pN#_E2QVHN<~+?_C_U35jmlegT2IL~rwCeeVvh9ZZn|
z?x}Yv%O>*BM86>h7~fQijsrnIDWWu(Qs%LbDXjs)9UGXEwc$~Yw0ktDdhZ7AJjmfh
z&lH`B-w0JN-0_R#IFuI^NpuJoiB<f6REznh1L=SCCcKyKCyARNiV0R_M<<DcpDPL>
zg%c$S%thm%z3vIa`W1x6*Thwr<I@z9%Jl3QQDh89*3A40f*XQMsm5e62XsOW13~q)
z{69gWDM27DeSUJV7Hc1{%D(Q~RM7Qf^jo8(gc(T+d~<+Kib96Y_l6_S=T2ye(hkK?
zColz`#b-{iJBufdB`}LWR6?Dx>4Q>n>YslCe<~6yJ)0d9P8RE#kMCJ(Qo2o9FipsQ
zI%lhNmV8>QW?h7eMT@@AN+M7V<l=&d&t>wBq;{`3UUfem@v{79#P&5!Fry*^XXX0)
z&dB2JPQ&GjKCRY;<7NjPsqZ7ZmHhm8`xpnPK*8r~Btvj4F7l?A+@4K#2Rms!>~Ne1
zbv`u9!x(z%#4{u<?B24R$H6DtSe-mb;wxpf;{?515Qr>Z3zPIS=~C2Z+Q^)i4WJ^V
zz+Stp<KN?ze!`IF!`EL>ZYlihPx9e;m#msc*ezyAOcHj6NGYe4x0vr>{#~NM6pq5&
zKxEu=>6oIm5}muaXQuCqg){BI72T4Jb7lF6F+`&a|5L_3Z%O!u(=WcC=#_tW*GlAM
z#7#|;*?-@p*y6!JQ91vJa>!8!@0KRWGXNjIuP!9(OY%iv4x@DTbor+vhB+BKJ=Y@n
zOMPvQ^`B%~Qh1T$!pM^s$U<VZ%NY&~fahu?_=Z;~NjA0q3W7cOVM}L))OVF(feghv
ze70-U-l@zOU-?IYkJ|%zm>CwIcKD6%;GGt~d_Ug<5&Pc1dw3Ej1Q#gT-sXCCk(Us}
z_Y=^WPl$O68%<6W1Tk{?m9*TEr0k^ZX2|W6D}NhqtH=H<0V6V%P$G|RncACIjHjpM
zLk28pIp<6#AO^&`X%MSv4RZ-=3<RI!$cu{Ijn^=c?w@%zySucQ`yc1)?>;1q|Gkps
z=*V;7PGm?`<#`{e{70q-UlerTwxDjgRMt!Bl}P*E_<&U_m*X#?QEcX4ivfCd9UV#J
z7H^?hSFNxn7XE30%A$HW&WH`p(=9|0?w849Z?ch0QOKl|l3BS^+k^X@&s0c~m&+2d
zSV!n%%1I>>-SJhD*1F||cA%^jKuIhrvtES=AoQf@9MEUhU46-T=scmN^8AHViZCuU
zXj`sNbJjQ&=!-s3U%YSxiUzW;ghSapP0b{cVVN}vE^Iy}8w?kyY;Nv3Q`PagGO)iU
zv-$!Rtd?{JDR_76_(@T+gbaBY?}a;y8r+W9Gbu;{g26dVd!eNTRM#x_^16TMt~BOz
z!;BYMSl(Pk-94c_;lOdF0CS=qr>sk-QR~nQ6b&NXw6#{A-B6E#R%_4_hARc6W5+A@
zPdynOGltj{FXVWAC{RQTGsS-|B;EKIy#~GJx1zYNi}fR9qSL$<r3VhP3Hk-J#IGU3
zA9aO1i$lTCK6i$MT05bv7xsJ57+|0;4ZEZ5o0>LY28p*bvfrHTE!nB;loPEy%<9(#
z+b&@eAjVmAHaVN>ffWp2ZG0e`>1~)_ZKNULn4&dm^=RN~%Aegk!im5uKMc@>``Oww
z(=TG}>T_KxMI=zk(}0W3n8Li0kexh)7`OfeWM2%Hq%P?A3}%x7qy~f0o=v_{nA>2%
z)Z!iYWgxuOTvtw{QlH~i%hQcgfTx?*!wLZ3K(CUsCa?{IJa%0{%madoevxq>g4Ho-
znE%dgVw;xkzuf2<n7UXA31tQSmmRo48Za=BKWae?LX^V}$l6<J9pi9@(|VlVCHXQw
zIA^`W+lVN4zizBWanSSVkY(Qd^?}}u4tC~y7+F62`Wci_*lPbT1vu%!b{XJob*me9
z)@q;kiER<<13s&Fw4zsiLaD{R?H~&{xDf-hw<fmrNP}Bd>*@=mw3m_AWW>gPQIaAU
zj{qcvU{S~O7TXK0l<{h_J#TM-;hYR)9LL&8y5n<kVOyDoZ(}UD%fX8oL;{J6QS#b&
zTQam@dpMRb_T6aNFh-{C&RUIM>U6GO9@{V@7SzI1a(PD+r!D~A08i|D7F_Z#m@wH!
zwnl$FGZ*i_p2Q3_{|hWyYkfL;w~lbsoaQG`-zLiU?j(XR%FB?&{MUp1$NR1pi{a|2
zjMJfD-okjh_D4O7t9(9@+=vyd1{JA5*4cLjEE1d24yFB?%84Jfl_z8cNhFI(GegAV
zOLs~5R?PYp+V<QsO{_It*C&ek4Le2*xy;DE2+1DwG(NAF(*HOpxB<FI`oLeur`po6
zJ-XW&6{d|(9x-$qZ11q>v)Yj~FHN?skq`I?&D<ek3h{#mCAf(2yaC@mypHFZQ)OXC
z7eGe4J=2sc>Dl{n*GnkbLa-X$H4XC&eGuNV#Js&38s&%5amUK{q1RkwzsMqsdfW6E
zncckvm@Hu_0vD+7v~{wn33tGzE0+x(>xmvVFm|>f=^}eZg3+M)(#P{@fh>u;;6oHJ
z7E6BO_>7W>Dd9J|0OW85c#v-*57>J*RimCFcD3|wt7z{2w!x-nucM(Dy8~tYU}fNF
zMdl}F)G`XAIv=R#)B<e7Jc^YZn%+NadrgJbbinKOK4X7yJxpcH*=pjR|K-IHPs+l8
zesy!AcqU2z3kR$E*m4EYG=^%S4t1aW5vVxn!kTxEN+!Vv!fnPi8cjlotS#@Tb_A#z
zqGB4FZb^r-|6(<ZBbi$1iF0R5!pl=MqO&+hC$H$lkct7aB|YFFD?#aC!OwR0b?!7>
ze-f)QTW7HHV^$BtsHM0PxXr-mSy`!VN<A`|xOU<YH|*!&_f@j_Z;<&4(PJjDl8E{H
zmNC|$ZTKWX%9C_v62%tf6YIYasw|AcF+U4djUaKhPjqADa_xdXTa$)<y%v{>@e0-s
zEM&jN3Y&NpRq??f!W$3m@DX`Pw+NJAGvS8Y>guJm?L31`^fJ&;@>2KRvYWLk7||uw
zWIGszLw<wUYAXFr4&y1VZu~#Y6_(C9R`{#zrZMJ?b=}R+#4pa4c|*X-3NdcCMwJQ&
z4h&NjBlS57M<6kgsLK3d<EG|%ifFc*8zN#@Ri=}#x{;-CWtKGHL?q=PLYs+&F$rD&
z7RWyhWbP>4Ifw|d&1PW+P4Z)jrSgIPzU>4^ANnL9y~O{|=$r_QPZ`Q)fpV;1yp&XL
zk%`Ajiao^5V4$Q5kF)P)Uo35(`=j3v+e;&@y`v+I-;ClnB)&NmJf!wc!@iO%a(%cg
z38{cHP~=VJA@i5hOxnTUF(k3KKiYd&ESN#@ndG7acj?kGl!zc{uiWptJ;j%D?Myeo
z-*W!=+8F&fhAMrU!_jeCF!tz!zE(`iibSCLr&oc}t!^82=UAL=Xf##uXN@-z_@*-q
zaZZ<UGubz1^^Ec$QxEwX494@-3>+dQK90_gS!Gt({G4StsJ!Wr8f;){6c^#MV=)++
z&<8d!U6gAF{d?ILyUp~8=F+~n4Epqg#gy<QJ?OkUZ^M_1nPIaX_dyo+%YTVj3)TqX
zVJ2y9map-}#}$~W=#xR6CZSOo#<|z&iEP0svcGF<sWmT$8_Z*nOuGbPfCh|;m>4d3
z88Afo_K)$vc%l$al;;3s0Y|?r0o7HLSm$3JdiDkOhk$9m%(07uephbSh<c|@R=2Wq
ztp}MC5RYTQJ!kwew)It|)Aj_pY_*D%c#QkOL}V1SuwjzZRe>xfvF0MjfJmr!UW#^A
zu0<E8RbW02c&?Rp$Xi8e(jeAM3;O6)8C{CZ$VUV|_vuZOd^l>Dyt{Ov8X}acqf^p^
z!U$8m0nF)0tmv+~z6>@Q+-_{0Mx}NT<a@0T%6cH@QZkI)ElM$bzwBtNq(`wmdIo|A
z4r3ig0!uNPD-=0q-$bgxm<DfSj|ZA|Hhl`A@@3a0?MIZH2M)ZDjZLyGkbJYJ|AQBE
zK=$znnd8V9lHv?b0V~KWn70fS#$(fSSEQQ{Ta6@%#YqZ=;^wI=!ajO;@#>muEWCy#
zThcU_-aAQV@}eC>jktcCm}NvF#Ac?|WzVu`>$_7(oet4SSBYv3s@>_hh8I>TT?ps<
z-+A}uhV2zQYw9Cd+Htzu?R}cYv&9HYCaNAm*FWtSC?%;>alZJS!BQaEQlV8xSdrDc
zPGlUJ>7vJ|cw<|jkQeaC5rO;|HRY4%sxz4NZh$OqV#No$AJK1nvAw-AN2_{uH!!3r
zv18ElQwOIMW=dR!FUoYf+qY-kB5#00ZJFvU=ji5bX~i$OQkpNOSW#lI7~3zN<&8O_
zct|bfTN;~|_`g~tdhT9tJYyPd+KjrNo=5hwYncV|j~_d{!?w#r6jLs;uEN@!WFZ!+
zTB~;vxjZ6VXD)}U{_i!k20h8O*MC&ZP<7-cKhP*Nq-Rb2<xkmrMUa@u4m2)D;Sed^
z%|2W`G+t+Lui5!K`3ghvg1d8@JIC$7m?a6o4q$zff84GN1BBv#liapQ6Wz{0nn+Vr
zaQE*i{$(4pg%5uI-B#mcvlWfN*W#mlHi9merB$B*@*~K?EOVSEo)sfJf!3So6+uGF
z%o~2tNsV(w%SF+Btc}j6B3&B%wLlvJz^4MQBtCdvr#C_3E!CFUcqcLtIeA@A`ERMu
zHhqUPn42gz9>-HC#uOja*|JJolKfl0+#B;TJZ1Y_IIKGnGaQ=k3xg__8c&Cs<pInp
zAHb`Q=+$6fZS}iQah-i?P(>qH(aOm0<!DP91!QAXrG4q8ts$mU!NiF5S4}PCXN(4I
zVPt_yTpunSI3q5-jS+ODlIJ#GVz08JGxI(Wjke{Ky*KB#JVai1we6d#$El+7Y!K82
zv}WRePD&4wtejn>-!16wiuwbl;@=<tov4vr#_&Uv6f~S$R`|zu!;j&T(Av;XBg#k&
zQuk*WL>?a-TXI1!^``IAw1MGw>r+18BSv9mCUQEM8&%FI*u&7a3QSE<t#_o%IMb7&
zcDywdEI_VsZr2rDPaGDPC^f_byNH&$z3GA@H;VgP87%c|m3{U#uNX4x_KRdNs30H-
zw?g$@(l&kpJQuUG)$SGOW7{&}H#B<F9EBZdr6tlh7o$0;X}n6p{9g}n=?_v|g^l3D
zONueGBR#bpwVN@MOC?^CmMPyMb!E?v*MeA}rAo>>SPvMCo51{4L5WZutMNr#S7^6^
zfgpLUm<TYVr32qNI1wldDIQ;tK&u6!7T_o31h`hg;Q;g~zGs&^IpFeZYnz%-D)K~7
zd<4}Bg%#>AKn3*V(n>%4Wq}>Ag*^ltXI>r+vRNk5f$TkZyf$P|gWd^cR`ku;)C<>r
z>_M}?Eg@%d7@Ca-rUHBHp@Q^Y%I(W&D~)~yu0~qm*jAQUaFDpqFD42G%^4-YIbK#g
ziMMV#qaf~TP^swAxFZC-U56|Bb4{9z?W?7vr(8A;Dx!x>I->ziTwtIYlA?JTZV<#j
zEb9KxvG;s^>+j{)Z`?c$<QYTnA{)PgLV$RFIuE<q=|<7GttOlNS7QB3wwU{vz6vmd
z``Q=M-er2}j`P}KNb0i^72Bm{{=1@>J#ay(7rleo7DzQ;L02616V*+Z73K>^B8??I
zp0!;pC7S5nmLvKD>WkJQMxvF*E#(WB6>j!-Z(t}=769!w>W)<}M3iDTkPr!fi8+0D
zb=OaPrKQuv1T%1E^l=HTiy!p{bhly}>S);C0|9rilH4R9vl$noc)i3ApqWZ|$0(RQ
zGmWEf+)i|e-yS)qi*t{gk%kB)*c|q8wDrgZ23L7uXRk-}TqSvfUP}3vC?73!`cHqQ
znlg+2ZqOF_S$U4Z!087e<VdWK#50)Uc=tnyg4O^{QGW)f2U}X)pj!%39cdWe2K*(!
zNT7#Dv-ZA0ItKTXFb-l_%{pq9$>)}UFmTg`8js(VgM*XPKtgKN&4)bHNJ#P%b=*rp
z&#x*~snkdGR(2WQF;Lq>&?ij8h8cV8I6_2P@uWX`EauXNeg;tp9zg2|uj@_c`FzY$
zN~;y|M^f(k><+gbyv;~fwo8_OJXZ((M~SO~3X!VMR7k>j-D$%mDiDD?8i{}1V|IuE
znG!w`z*OjZ!sD&<OY>Qk6u8K1GR~a$^ysO)E;ILNzqK9WwJl~W<&tRIn%v%*s;W>!
zy~$&&mD`#qqC39#!IZjki(q4ML#Y()*^RA>`Q8V9gWz8OZ9O&?ou$uxS2P>}UPn#2
zS!6VCefWrEKNFuN^p~T#>4n&I%yzhVD`ariRD_4xt#jABH4*1en;1lXet8jbcVx#3
zQQebGpbXDhuPno=9kUM7y&lmf4spdO*d9VXwE8ZN8Dw=Blz{zLrK~)!m12)Gypb@g
zhh_fp%(f&j(NxZ#y}2o6W-YQ{=e=b`q_?j80zuQYQ{=6FNTWfe%8r_cPvAa~9VAkA
zetOgWp-35Cu#tZ8xaF&E!wts`4JbqUL5|(6z)W?AZ2tSa^B$erO(izAQ8p3+6+QtO
z=VTzlt+^Z(+}i|Yyt{Git{hDapgb+yjED^1i`~=Imi5r0lEdWepBHJVwUpG+<@KN`
z9D;<!)g~V!3u&@FKMO`Or;?}Reh`Tb`v+(B`4BCw)_<<DN`P+<o)k4uM!}?;78O~6
z$Np^yL?jt!S67sLE({(?7RyUVnfYoCEe{S-5-;MBb$f@s>#NVO#ia$@Q$PxMd}-k)
z@n0gUmM#~6kB_9!a4a?Ch>=Dud`&o5R3_?wYr+D&5UVKX=jIwwL~z&o-t|Zz=LD+9
zkRrAI_@2=u8t9N2&FQJ5_D&~Mkm^%1DdtVlwyP_>rF1_`%gOb27=8!(jVgn^*T=t=
z-V^MI%xehGSBZvw+BTU72kvdefeo=^(X1gBIY55dJ2-Svqng<L$oBQkHeip4dig5#
zLdE-=Dlnr(Uycl|n@`D$0XJMqI1ibo`SKkt6qh|3ldsgAqX4AqI!@sQ<Yo7S&W`gU
za`hZ8V2_#<OsADUGtgYU!cBVfpRZ}tIatgj+gP0TrEp*TJ3&cZA0XBJ=jD0*Cc^^U
zo-ysWHm2*8Rw&S1KHR#*Xv_1a+zt|I6<#Lfa7AwtPW761RMrAx?T@bgZ-qi@WoQ6?
zho49jj19TIPiSftCdYo!$OG06$Z7Fh{~OXPFy=@qEmK%cZX>1Pz^H_V&s_F7eyQYz
zlr;hUw6(~?v~^wHr#g1rF*L2fu8_?giQXC&(`$cZYyJnT1=6Zb;WCpBX<E0A!fK(t
z$mUwikH%APtc(Kl<Le}h3yK1ImIf8$PrUy_a*JcM7mQ?7!N3sJq5Yeo>Eca6TQ7-5
zxNs4*1}K-`pNAHy1>eh=FOSnFykma7GTEU@Qu<<En1d%kANIgltoD4ZT9|xoDGFLM
zqx_SyrsHv|F+bEz<|mqc(4kIL>3ewlcuWi)+#*%?aM)GvWN;jt)rM%%z2k!4aT|vJ
z73?9r!Hh@vbCrIyB4~@pArtM%041W670-djxywc*0)IlqVy%H*1`~dY%6SP9yFr$$
z$EkQ{8ivqDx2r_Shi7rLo>Eo--1nG~A50!=c<cjn+xIGbtf~BOn7&g?Vv-M%*mGO{
zX4LquHp1ORL!L2QRSxNC25Y+ld`Q&1)1$dVLmA|G5%17A(PpwVqO=Cy87@zI!@FjV
z{%*|$9~-2#qG@<NkIQT(qEEn^-r9c9*zxR(ULL;x)}qyteDMowW@8fvlyNyMsg@Qq
zBOT+HLOHDG@aw0`q}#g*Tt<DMK9#;PNDtz~yx6I98;4kiyp$CC5uRZr$Pq!c8HuOI
zg`DJJ`w8caIa^{CFAqpmcLPsCzD3iih4on(Ub;f<hQg^H?h`h;M^ND}3ejtgc`ql+
z-nc3H1o3}A&nuD?jtdAdKurqEOne;s6!P9_kio@;k&`v0jib>&Ru3Q5!|n9lW4YgO
zb{;KRK(=jwje5aJNwxh;5B(PxLPd{DY|HkUs&2?Pcqd{GV71IskYuW0`hcjz?s+>V
z$~zXl=~L2;Lq=SUJ|INFkerN|kUAw9=*I-=O~xNBsI*@{>$XytnyD&{dp7pSs1+)Y
zEf8{yP}F5{T)dwUV4i|W5bKjJv$B8-R!gBcFdkml_6q7xzMvs@6A}%&Os3zLd5ra$
z>O8)ca#YTbkcPL%rlhe_G}q#g5h2EoG#uELvv<x>_EEra@~Q2yQ6Y^qOP*O&@at&w
zNA(?=OGtS&usp_7^}Up7PT6($UkdmyObELOsqc}~%$l$<DJw}@IzK<(6qn5)I8i){
z@pn3{T>zZSJ|_o>Fw`msVoPQ<nfYqz*5Inp{ZL#CQoFGdCuGBXYEW$i-w*7}3fI(W
z7AAx4DWe!CY8yoTK-mCpT^x6yJno}$8Z8v9+FNhuX=hh}J@g>_;A`sdAItG*gjIWj
zO%w19E^fZCVnWEJUi(*vCn@i%(HKBQ@h{Kc3>P>3@FQ6#{t6>jVDJ5#O~Sl@i%i1|
z8OEu?s}}d~>x<?~ngfAN4!i#}AzE6Wa$WhbKg+0!tNz@h`?-*d=u5nnu0re`QvX}x
z&55X=l8R#v0;gU+%SmjRO{k1pq7*wBITweBhe+Vj;Tc*+BK;vwj}a)VrOIoY8^n2O
zE8CQDtRhZLW%<;Ww}GNN=vJ0AtdXK4E^DVRNl#ZGWTJYl=PB^dQ=QJXBH68IT`hcx
z?2kA0VW9%E&-V8=W8ug30nWZ9efMcklEtLtir>P}ZTddorvUnZpI|1=FItu7F08KS
zvam<X!g}o)Wu5(6Xuu7bd2jOAm4osNuMJ3`e7NP{yRwp!GNd5+S^DifMqiq;Qp%J=
zN%C7o<v9SNRX7Dp)NDf6Ov=%GV?SUup&KfTl`0e^OUk(pG*Ik(eEFxIp%p%0f!yBZ
zgWtu>JY@rFKKt*e2T}T8F>j1dqQZ`j&GGgI$(#r&hXJ!&+~oZMKG%D&s3e0#11>cp
z=ZE~9*CWwaZG%=xTH1A$-p6md`lDA*9YC-jj~D=k3JL{+@!kps8nB<_+W`-zBK20}
zunf8D52ZW!Ef?;*iT<HKsd%=g%e&UQvL*f}s6U1H8KxZ!gXOT9+|>w=BzWBoGU#V=
zgIbTadcPgHXZ_Q>^P1_nJ68#ZzVsZKYcefH4yRJr%j}RMgzc@+v&(6GV-5FLEiHRM
z0BPSGn((SJpd(f>CNW2oFoq&4O_NY1+qE)`Q;$-m(WO}eNr}HbAP5(1Qtr9N@`lBZ
zxN(%p-4R?k%8VWi@oQH-Zlo%b%h`}gP$v7$fp}qBX~ey9A`2Kgi@7wyt_YAczDuW0
z1u9U8yvR9saO>5o72GbR;!`xwZ*ts|atnA~o~KcGl0I1|GjA>o=2k867AE+3ZHByP
z7Y)KNtmG_SnAG4VUnCNK)7dV5D;~Yc_|Zex6x^bC8U9CMLdhRrz}14a%?>JlPn`;>
zid9GG$97>EA_Y<!x)7b5Y1jhe*mly7>Y+T7d<5kWYa?qdowVyn>1a*~C_~&`Hz(A$
zf#f3?6Y$<2%mW!$w|K@=mau*fu5ajhP0Gg>L!s!aKdyTRf;*wX<*iuhi-wkP%;abG
zcj_j#&IlPq^p|2!r#YV>(9_352DZo#7i89Co)LvCgr`7RC7k6$;}ktxiR9#r9$2AP
zy|Vg@jQUMty|8LMoIwOZq(QrN8oL%J7FH8{Hp849e4rcVX*mTdsqhdn71<W-#FYm0
zW(7b=bKbHJ8YbNt0(qz?!`V9}HfMX_4f2!PVWQ9e0jSlcs;hVGEQL2?i~g=q?Ct*x
zWj8~-k4Hb)miG8p6V%cQef=(c{h~wq33M9gf!~k79!KzzymRVZ{&&%&<c}$HW_tEN
zPzAygGfe0RzP0`$H8xr8X)wDF^{!y;okZwfA>kFge(Xn8@%er|6r_fu6rK!X$Q;bN
zcEg+bY%=qMG1Ls>BgV&r{aeruQBX>yNL!gQTpEo7cJ>!I_SxW0)_z_F-f+3>9p_72
zw^+*Vq5BY2Ixo#|*8qV(@+!-`WWg{XX{Gtq!`w(+)ocqTwGRxtzql-a*Qx4OMB*ly
zgUJW|VeXV1JAoe*10#8>=K-bOxZ)U9Kpv3ET%+$l?*GKv>PPvE(bN>5;itFp;pDNi
z#(j`cB@?<IlUjXk%1^(m7{W;{*7uLWNwlD4J4if+x@$eoS$<Il>2olZBJ$QHghDEA
z;B<rog9|^}Tf?Cy!{6B=M1pny4zek`5#n||mjWpddk8NIJUnGVfOyyq(MqvF{IC?o
zw%wScB7A?k$5>F;ZtUC3uulLS{sbS*cjjI)^OW^LRf)bBrjdU$fhPOQ>biY-(6%t%
z1*Vi<0oydNqq6fSB0j+AxG^ov<pQCXU`e+UwxfLV5hNWQYQ2bLrs{f|4F5+8L`lsg
zPqe}?gq+RUIRHne{?v%LmONQWa#`nm`I~;d2u&v%x4wJ_htX9?;+{3KcjS!0+nCM_
zR8E}?5@K3>w%W$fMW}F#6g~r?D|%(p3y^|xn3l`G)`j89F{Y)q&lB_IANdHS^6Mx@
z8~F!(F1;>33Xue{xxhR1{rb@P{Jw5sWj?;Dpb`0OPXCAMB1*C!lsq&Ud@l#!s%tDQ
zV-e7}>B<Gx9u73<H%{e~21Kj31?%(1Pdey9rX=T<MkURJ_eVd1ybQ9;g~apAsDm9q
zHJ9I1bD{A>$(meK8mYfT_RX2$=ynE<?6b^~7XBTM=ndDr^nV1g`!1dc=rD(TaujB>
z`x@F2@__Fc4I`<1HqrPG3?#|_f#Ls$#Tyv@2aEqhOFvQRd)|1@$TyVq8h`Ne_tW+@
zwtNGSLNoIy&#IB-V-g*e#i!wzOx0j5f`NI-(5SQOFMB36qY-ptcsI0^MIL_;`BH`!
z9^${hEuiL=K>JGrcRZ^tr>ba17N*DLJg7(ALPjf-e(ut_ZD8Zy9YDz+!N8d*VjV|5
zVKx7671;k;i2`9N!b%>Ky^{HuUn}2HX{5xo!Onuf&<O;Tq%})PT^{q9gL|8j*XONO
zu%An1<w#8|<C^2Pi~RhDP;WhLyi3jwI@kvqxbl`-Xy?W67Fc360cbQt50hdY7KF#f
zy|G7Aw`WojE2Q~1L!cNW4G^|XAIcRjN;Z9J%q#mRZW9<Stk3)#00ZS7mFgl$J=a2W
zxY}1WQ(`h04jGDynTE{0-_!uL|1@fo-ElLXzJ^Juh!XvmetaP+TDlXQBY$3yPdKvy
zfmLBDPt5sX-7o2JLl@4_DO^p}IjuK6T096T7Y@ZNMz%cwxJ0KN7vx{H6MG+%-ogMG
zD``?d<?xc?@A-5@os0E_*an2`e7RZO%AE?&?WwN|^vGp-vAsQ`H?rNTMm1S87+B9+
zhr{Cjuk&>>Y?N85^8o(mm1M2yx7#V$U&?2Y_Wa11wY)CyhAp{QK9<isyF`$mKcdS1
zE%YAddscIr|Hve1xTdeeQ~2eX1p&=-n&AY>vdr>*E8297BAW$L?To~Y4dg;IAM4Ut
z-Z~XB+{}qSc|(8!n*WGv_N1~T6H`Q)%a3$K^cdwmK1h+)??5e%{6KqX%x?x>ykqSM
zX(B@U27y5dlb?$K!q~L%B!1HP$<O~(As=}FuLK}(=di?Q`s6l_KD&!N=CZRPC55r4
zzszMM$<+-^j}dJ($2Oa=4KI{_r6}tKjZpl#vN=2rxq(7c@$M5TG>0}9yYVtBjjii$
z0TK&pT3V!Pgt8;#ltxDd8Fc`!%$msuy8>HM#!qU|qKw-zJFdt4vz#&X{+;YL=#yU@
zcXpP1UR;frvQ8k~*CWR$`n%YU`Z(W6-Qp7V)yJ-<|B?N(^>H~1kKNICp_WbtD$O64
zr)qKDxE$73xTY&)eLt}u<K`<fJmh~Y<jA{E$sE8dh*7`kWFcdtv=-<ex+=N73Pj^1
z{q_>p{2Hm&$G8Yht7FyN{RI)eUa&LitC;!wn=8$+DT+)L#s}Ihvpi1!<2tO6)g1N7
zhVJFMqjIOAAcnLkM&0sd1?OA(+xQ8Mw;}%%wBCl46`TQa0_9J+ieIJYLC<fWnvQ;+
z+drr{w6E@rx(%AYE(|f4?_!jn3xe1uLI?2KFQtq{@OAfMk9q0pu=b%DvUtcawVlRK
z>=;;G%sa`U0aNPaJPX=0e@@%PZ8H_RtZ&s#yU--V9VyN!!=aVC|B%LRZ;!tBe>{1H
z+XPWu#Xl(NbAVFe0><rwGHwo?3Vt-%)7!|Gy*RyJ9l~E+_Y!Ka6#?mb)(U|_Avo^P
zNA(mnGn8HyT?JhWMiV#Mqy1=v<Qr(v+GS?uK29_P;%&APgwJnGmk^X)cJsPB7XOYu
zT|hA>i!KL@1ui)9cWNaA)!WFPJ*nb{Cbv_=u-#50&Ve{{B{fTBjIz4Rr&Z1e5)Ouj
zf^PnEe(<-@!)xtDk&-ZV<J#q??xM%H0OZXyQSc}OSRc1)uSwL13JH&2?5n9k7Mf}|
z;vG|PKSRhW<AGUQ^IzLrYkWv8M%Mj%ZCZyBKOC$pk9=aZ+izd8?e<V7jyoPu?VFqQ
zM*rL(E?;w8KK5cn2ucU02z=)_cI_z}B=a)gyUW`96;~*s{eC%UfDSIdJi*;1rKn6N
zLhKAOC#6iZG{FKl(2RZUh{(gi+7pPdI*aOWi)LW+F6!UEEhU6(g5Ba!h_WtG_0*ps
z>t?RLIwb^q;Q|JNC?FJGlmuHw?DZG!L(%lFb50nznp^IbOawL0?>20F%<k{)=8TF|
zE-Zbz5vasyv5DE&aVF%Py)c0LO`7(g-_T9z!h(dkLigjPThFQg`Dbhs-J_}xI4;)3
zYkn}>P4=%Qz58j`3gl_fDGMufw#zE_o39lPMejlN%p(r&v&!tS&;Hmk&;U4t1Ekg@
z^!Bnipb~(a#ni5zcwOc1xuPRi5Woa|aIiTyCL48e=%3;KbfQ5f)GTA5kePhdO<G+B
za2%14kN<;cPeO-)U?iVGT7A*<!qi@%{Un;Mewz<o3|!)%h;1x=dC`|7qx~<aQmtc%
z3!q4hcvdo2v}7c;LxPIcWXXY(R4}}CwPq-&7M}lQ(T{c{L~Ir0vI)y47Q87M78G3X
zQ8-`GuE9`pdz!6SvK@Zf1CKdQSH#8mpm4UBPri4_D9Nx(4-&G>iKRS7q#5hzhKD_`
zbsXunHVhh{30H;^z(We>`Am6RQukB6m-rNWU&7VE)jxMSb{Ja{ipnU~*p^kaG0t9I
zz|OF$@}9X#qxvG}Z5T_bd?FjO9hgImByo#T7NB1#DZoOzy_^^08zz8;@IwHtV1Q0M
z+5J;r+2Iu|$@E}yA&u;}nwsgyxqqBeic(Vr^!#=A;H4p?+81G5A8l#ln3xa}zE>oP
z;hjieKbw-u3Co=BHO(5p?0!`koUl?%X%KSBa&3;sVx-$kJ4UaFoDWc;@RtYBOiE6x
z&4m8)7DB^C#ZsL!YUMD;<eK!#9cIp{kcC|h2?+=n#r>DX^q*DCZ!^g9$_jr%M@fry
z$w93TU5&aZdgVk^KT22tjPg<T{NTc2ZM6tEYEG5H!=G92iZq3M?`2uP7{!&Y{pFo5
zsMF+kmmA&{zcVTrZYYvhcCo!=Y)k#(=#Y`0;`#HFsRUpkM=}?ay@>!R7CMf*5&TuF
z(EbYZTY%@WaMa9aVs{ujEA8)n!(RN4e{ew(sY<%1WbaDAHs`%y<PXSTE?LMqF#cBf
z;R~FohQj=)bSmpvLq&W}&ZZ`;xgy%p&wv#qWS+l-<y*_X&G&ZS4CQi986)}OE~j32
z_+Gt_OYLxj>r*JLsG;7Xj4s>dCzfOebq(0`y~e|5bMQ#Ec!7N+!=)TVczd*q!CWQw
z&TrD!&hC(HR18Msp|9_m5PMotflT~^p<A^pr>#xl2MW-<PsWnTTu9NQdJPy~h)rc@
zYYDD?`H&3co0`y?{a1%*wb1M!1zWLuy*g$SDTxLu=|<aP%DD)(O3NpfE5ZI@0?Zp-
zVma`5_N*VGwi)tF3=0%ybNs{8f}rn{q9&YT$!rf@zV{6nu@b8n?gK@-xm23^<`&uG
zF#JHD8wYNHJCZO|o-0Z)ko)Pg>wC+XdmT@8%0`skAB=gtL1k)E=gojWlooV<C}&7j
zwFd3sw7+Rd0>((r#lz*^)!HZN$aMw_X^JGKuvZwB2}BCBATZgKeYq;!j5L~$!Bh2>
zq&N0EGZooyvZlo7cPIZxX*GirwvIRAn62^rwPfGRmVdRN{&tIJ`J7>KEoF+U?v5^0
zx0o*PV1oo*Wpcq+?ih4*3Nn^ehn252pnmvETP_qp%gTp_lm%`&L$KS?c|Hp{@)kqO
z=>UQ-3m$VM{}>$!`&vpLJp}fKJbnJBmiFj7`A$nD+_{@nT8D0$dNY`yH$>!%f%tce
zvA%pF$1?x9;>1%ftqaAaN7Fbc&g$rPdvRqj(Ep~}GUTfq5@qtCd4dHtMMSw?@>1H8
z$H4D1EHjjK->5H!J4X>%?n{yadK1ERNGmM3l)1{Yrq5q-5gDA{i>}PIl>9b(&!0vp
z8dc(8IGUSplip1{q<z8qwPpr%(VUN50a8`nRbM}futF#8HbP^*3A*I14n66wXRkCV
zf1dwig)aZ0i$l~T7B67ChLRx&_v&^+TI(!R3|pi(9#Qi`d@OxVG+uvCP>`8j&)`V*
zDCQ|<RllO>Z|E0FM_#Q9c#DV9D-_LV&4tzpw9*U9riJBs_ls=$vO9h$|2WnBQ)nVN
z4_sB$OosV;$zvP0LnS);g*xYjz(j%YHf=%EZK3uK-*xc$)L*F8fpr!U-PO+tb9unq
z)dby?F4hb4sPP2f+nxnXQ1{4>QSS>d?ql5SbDUjoIQ+%??WvH@Uw!F!$J^CL$2EBM
z`}e2Q=QUohOVZZYu@&+w+KMn!GJ5Opj1(WTI}wJr<%D{)Ch-}*!ow>l#5(u?9eMtG
zSsqB=xtkU|EzUe*A!BQK^k`i2Wzjf}<L<q&0jpcr<M47kC=rqH7Q<C51#UOt(iuNI
z*J5dxi0KIbt1j|Sc6GQOg~(fMPVyFw3i%ZtRH>o{(KC_C9;tcH7y-{)^U5;J=@d>x
z+4it$wTF$iz?%A^;l1;^0J>1_vjjb&MXFNSIi3M5#>d}zql?eDFt|FDO^R=97&%9l
zlY9(QKemG%>1d0G7vD8iy>B~ReV3piY|S*!$CS5sd@QExAP89RwWlY53kA}Pw9GLl
zp$uGv{X)CXSFjJ6OIqWRnUa&~9%lLmV^FI<wXIYzRjA&9dkBlWm=jXF36(RgEPF;Q
zw&u%3GlhNL%>KVl11iYhHY;LsTZSPHRh5gt%YGR2=$~c^55U|~ZaCDpd8mgc@(bFx
z5DmG`4rgtt1gKsSrok%w=_)#iz3&sD6i#5ztl8VF|DQhFt|~Z|H+|F~hBUkP?;3vE
zr(7{6Q54xaq#>>4;&FZy1(}gbSZc$Al&31Lh<rfr8O0v~+MhBY8AQIYDou^wtZSB+
zZ&tw%F|=r%!g_dQw5=pNpd(I`6EeE#)F8b4lz=)cD9m1F-k8Gu<wW`l7VR6RhLV3o
zDh<n3-fj*(lS@f%{xL;g)^4ToS(_DhV<-F_^nMxSmg?6n1CV_4j?{mY0Q`@l!K$jK
z{wnz~(O$P9)SziV3neMsp~8oZiAPja`Rz3a@1p>-xQq`=FaW^hRGn(nlx<6-v4Hcz
zfJ{hZnPSBECotMWL>TN4_ygZKko)YMZ@E?j4#Q~AbXG!8=uFOvXCr%dk)ZY4+sG}H
z!l2J-b;^J$jT5+I{x19+SWnk%m;Fxmw+n<noBPurLB_b(lfZ)_d8u)6;Yu4ttE-{0
ztq8_LqB6;^0E*qOOz}B*g$iU#Z=d(eeYjPY`TgO+UKU0juQyv4F*Sonk!bY=fx`dx
zb?EN1GQgZ=%BXws;_OT)^~_9m?m!v_Hsn`^@ck-=xBOOA#*H}6oB$oj(_66HTzHvw
z-@G1ENoi|sEsJik=OBmgYEgI%3LYd;->Pa+AWL73@0W8gvaird>D5#t^44fG9ip$H
z(e$`+jKN_c*RW)s$2vs(bt?vr3h~Yivp7lKM!A}euZ;RL<WimXs8Pr83W1#oT`?cp
zwuWk<d}HQx)`Rfv?WvW2TaW?VMQ}Tv*M9<aD&LI0);Z9iuPwoKlh|X>{FXt3PY7;7
z05OvzzFV|p*Bl<<K~=7n-9#-QLCUn><Uu|t#(o*ZM#IHNu<VOsL^4aqce+pqL>zY5
zo$gyOR_8X3DA;sX%8sd>thCsz(x1wQ(hx;5Yg$Q(_1Ya9e8^$+J?qQdczJPMx1M$U
zPOHo#ZeJUU^`{(Ky0!lWQtbu7_nQG2Mxn*|1)$Md_r3m*N@5`Qui{YUnLQmF>yDuD
zd*bANj#49fa#`?Z!#V6X33wWq^RxF|b87nhxKc2jc{jDWO1b=aIJ?G%Lm?J_KDKI!
zKmhw%QK+C$+-owP?bUBgjTN+4^1~B-f4P#`l4rYtIuiwEyHD0)gl}u7*No8qR~u=c
z8TS}o_LMJ=S-vJCOio+yYGO!Hy8XJp>MOWq1XE<ri{bal7)l66H~!IBn+QmLmE0k;
zPiejpiLphA4$Pt2`dv<=9LV9Ho?JOauzG@?ny$){@49yo@tOpsHS9RJAuga~?~_#A
zX7^Y46Ecy3D>GX0@o7kSkpiTrPz+1+t8#v`8Ry2oqPPbyr*Zs{by(RMX6Amc2djsT
zG=IeH%D92ljHUu<1Yd5SNkl+N&6telW!Xh?4x)Srvj+AA&#YcXwag6kYIm|cz0QqC
zG1wr}JeyCh1$3%<B?!7Hd2}Vzpi}np;cVfsA`j<~M6RtuwDG0H;~pHMxfDAIA9>@x
zGE5*Yx2ZwCP3yBgd-&%*U?^?jF-4LYgKRQ))q*XK5?;LmJ>xH{K_0puS*G+_=e(z{
zCj87bU0qZiniRaUZDxt~*vg@TFD9_Uosc#PrwzHAWE>EOsV+rsnFN|{y_Tb#Klqcv
zFB)%#N1b409gfPF?|*I8Qk`Mr*G9XCQ-rSS(OcIwUkHUNbFxk*n3#{{jbL6ZKX^J#
z&1-hEc(0an7Y<0>Sp6|il2lk>V&{4fk_?GsnOf7i+ib9V4gQQDTcS0#_iBw2vjvR~
z0KPw21w>XXf-(SJJ(}Mk=LjI&AwE0R8j`<w+UXsDJ*YzW_do-O5kL*!L5u!9XkJ()
zcJHfk1SLyrQR0=K99UZME5B@=po3Z)JP4x=8F9cxxDD*w-U>!e1kZudU}=;fGx(hW
z3Yf4vWEzwX#b*uAd>NUG&J>I29k=W)F+UV6+AD~$L&V*{>ZFmKP1zI3H*PV_XR*~<
zj+w|caYRfx8*;Q55>s4UKl^L92<&F>195?}L)qf9{VhHx1`xTp=dh+^1DN-yAJMKq
zt-@!%W2C`ZurQ&?B>vZ}99Z=U)tMubZo*?%3Vi)dMz>Y;9T;ye90&BStsJa)07RHf
z{8&SPI-$<ZYIScDo6WbK#+#BcDt?PNB@Vup5$9j{35fvOTaX=J7NDv7`1YMr6Jvww
zK}4Hd)i_b0Zt#3_V}fK_HBYEE8h5t=MzJEoYy~!h`afNG#=QCIT)Emjb&nby9%^%m
zcFa4HLi+>(a2J&~Bc~o?Xqgdz!Z(^XsFuyWYX4pF;fZk5J<<KTpp--WOjdOF55xiD
zw*6*y`r$p9kbR+NEBNgn*%mC}82m~%8$*i1w|_2GZv5Tn4?e*fKl7&CtB1q?DF3Rs
zTM#4Jkkd$N#X_n$pBR?vIwcN3sK?5>3vd!-{5=iI9jGZiBltq7jt#N4ievDvj>&T&
zX;}1)0+dp8uOJk=F?Mq$cG05!qrELhw`DMQWAS0UJmqkan`lIyO;m)G{^$qNn-9Ze
zbm3SUcLzmThS=FagDnxLAzUxb!&1ZYx`FD2ZkW;C3+$kO4#7W{jbL={fH=!%BY**$
z?ZpYo4Ffiee1!#78x~yu3Ns`$l07{W_B|?1dRZaommPb2H`2N7UBLVvl&$dBqyMsa
zKCQGE4lc2dL-U`Os%x012npCV4y4uH9OEg1U0>qj^5MhtCD4S)uYQ(D;A7;U#8M2!
zmSr5fGGJ{d4$>N4x$Gp{U6&k_4LO(04YGkl57M(KY3>ph1L=gh1k9IHikQ>EDiX+>
zxGeWJQ(du7_2K}<9HTz50@<sIOf!AMAhP3&KF^+3on&+6$%`-<@gdVyeZd53P&DF?
zRCt^uw+kI3iN&pX!EW1g{ra!w-$QaNR|bjC^}Q@jnxrCKB4w!#b|rJ3ha>iGk5E>k
zJ!HR!+ZNH?bY`e$@6&!d`!l_j;mfy+&SQV}mFgADNH!*{gV=sXq>-so_3)k4lcgq}
zWT@(K-DIa^2os}<<b~!tV@vGmBZqe!o+(nTg+Cr{Wk%GQ0L{xbD6IqYEVaA#hf&4~
za{!*IL+47^o*}<j3kfQ_Hp0f5jHw-oxgk#)FeM+`a^XP5@TM02=Xskq<70fApq?Tr
zW;fqSq{Yl9(dkT*>>KeAMGJyU&8e_73wj6B&{zbJId9b>$&r(NLlrZlchzm(J2K;N
zz-Mvm(B1ez`b<<28R3;y{+>2Ujc}ZhllcA+KQc80<=<*I*Si5D{;22Qx{nq|%nq+M
zpB2OA<1gAM)A)lNbW<=9x4M#teDlve4;DQ_<~sfJpz#Oa|39|AGAfRy33qY#;1(dr
zqQTwW-8GQK-Q7Y65S+!`-4+WHLI}DL+}+(>!`=70=l;7t_Vjk2nXanto~mcMs%xTk
zbz5__^t!CHQDVsHr2^ls9tNv9<w|RIe3TZORs+Pr5V$kO!bZ6~VOdE-qRo0VP9aoK
z;MG^Cw>kz!<9cRb%e^!ZVdGgA^lANmD~*ATf9S;E&a1h8uzDIXqx0|)EnGi7Y1WK1
zRnw3%FHjT89#;e@blfie_8ryW*3;>hRNC03OOcaz?m+HdbsDdt<Uf}|Zbk@GCqfyQ
zep)8w22|P|d8O<J2HcR|G^U2?e-0(#@90pzbK`MwI&CG=pOKUB9Y9BECA-nAy`rN3
z98Y^#BdNTj$X7e`u31I%(q_{$_QkQ#g_l!XtpJeaPYN4)sL=pMAen^Ep21U*Ti%xM
zisXiamzEsJ@zf;NzHSs!<Cak<v)IcU&n>L@!Y5?r?n#cI?|d?LkpOdVxwW}+mez<+
zX4DqI$ZhAnm<EV5Gke5SQPn-`=tV&*YV7TMt9bq<>|6;l)}X_@8>cM)e-8-c_56Q+
z5_Q-+4>UM>lVtw_vAAbS>zks&bQD_8T02{L?dMOvY(<s`lW>WE)F@;T5S#+h7Re`9
zOz%ZdI?FD(`2zz-!}{kns3!S~$@D5$xqxtnqxFFH*)Iw^Mf-owEzjR0?9y8IU!w@C
zpm;RuAj6+6O#UEUteocz$yK)$oVm;sPv<mJiH@~nYCamKFK-c3Zw~sKPZ{k|kfwp@
z<EvoTVfhkVgUp@jo`4PWFQoA`8C)G)BGG{p<-_}!?}r7WP#F91VU`F+#Grf>mde0A
zU0W`8Et#uZH2)Iwy_HpokLc7-J}n!c2+!Qgq$6}FA}Abk>T$JOJgyWb|7oZ)tNkxM
z=(LA<6m64KqRPfl3g}Rcl&rm<ltsr_RiL7A$E+`Ds7HD=bdU&PMo~IXpa+$>it7ru
zXFw8Np=A-Hsh@mg6tsw7OqlI5Gd(x_NQ--HIq{ebGSC=!XDHbR<hFz&M+E9v`vCoI
zKQIx1o!kh)l%LVi(EG@n;zEX;)JPu-D8#hyCjL=;c}Rww3DUfI{>LR-T{xGEh|5Sw
zqS6chfQzEqVi&`oEhBF#rO&~{CJmVHv%#ws-*!IW{wMzrbf|S@k0|ZsK1ka^%@-e6
zP?rV+%6`TO2{eTX#s{RU{a4|=^dFo5j9`6lfnSvrpIn}9d0{PiiBO#w0B<bem4sDi
zkCZVyfV%Q%m-Ob+@auehBxCIEM}11b50vVV&>%p?dqR$8P%K+4nAYRkz`)&*JKotz
zb+!~Vy%d%4jTFPCE3LN9ZIiBbW8Zni6yWk{5aC$_MF+_mOE++i*42BnFg~hQw*LGZ
z1YdlYsU|y^H)bm8hr?RJD)9L|b&{v+B^hjvlH&I^<_2)xspe1+h^481@R0#*qS67b
z`n^rpLTZIBI28UGL9nxVAs-uZp#V+f(9CFoF?a>OEbh<~5?)dAss1$1){k1~2sb!a
zTvg4ax1=S!o-0EBBV|EkbVBbBqdMNrUt!EUqWq;&5bayTazGQm7t8Y7;KlsxcrEdJ
zipI6u-|;YhqNG1|pYK%0urVTcrD*<M+Hi8fjBZ+fy>|P(eJm*`#5gc=mIHf#wflKj
zm%9cL^ii7peQhS8Ko3v?;g5i=WiI|FEA)zYFX-IlINIzv8*DS+m1=o}dzAjdx~RB^
zWI@DGmgYtx>izGvR;=uFEN<6A6$k<g;Z$jGADlYumoV|v0TU)OkkwS;JsN)M?Y4a~
zqlLZS#owp^h-Ie;?@||Wd6}4ezNajQajj)mo2_GjHmu-SPn19EG^W-Tq`h}h3+K9+
zh(a+a%lwY;QM2lk0W&BM1-T*vupymkFAt56uXG5N4NWR@g=*Pu*!n=p*5{d6ru#`y
z5&-^Av(2PgS;=rb#4RT(i&c@?d#u>*CmS?E>{t3)&^dKyeW>2gZgD8QBZ5e4!<!zE
z0K96Ikif=HwbO`Ar-Io)iWB?2&M+o8o9k?;(S-&gFIIfg_*vOYcqx<1z^Y)w3Lc%0
znTXe<eDv%S-h{~<gbsv79QOmcPGI6AJjN~U+OL9G2Op?!{docxK@%WB-Jl4Z>5{}H
znHdifkSHUacpq#|v{*VwmiE|F03o<A26Iw=oClYK-um=%HRt;9e0?DVAeDh+t}<Fc
zU@{(WPzof|G~qu=vxM_ZR##l(yi5*YtWV*G(wKZqRzeS^P*2wLI0&oDq*Vk%Ddsvm
zXv;{C8WVZZx5O^y);Ih%{+k#zi46kM>U!~LNgdjrkic&uq}o0vmg1Gnf;J4$%7MLN
zrjYA|%LP6qRxK7(Vvsp&#Hh6cdyc6Z$3)d(fEUyMcpgGNed6gjTB|*vex_<Va?%#e
zMsPlokrwvNWbm>l@rdWi*eAB`bAGx>11U(1E0yaQteDWb%GoPOW+8wtfsv~UE$WEd
zt4gC>Pd7aH=R24oO>RE~I@yeO3ppP_Pbc)wBSdXb26p}#CJ!=aU}CW~fUSWK#xRwj
zw1-;|v+ZlqLz?%(2U_Fx5tXgE45(Ls8K_S9%L~OfSV@8~r`aW1#G&&6tz<H)jl4@(
z6MlPOsXi)GluRHH*DrAH%R{@4O@YR@PwQo)y)F|IqZniY_@5DKIH7AgYdDgIp9>D#
z`64gCy&e0xFz2|$n>v8RO7w49aFK4W`xjcSr~(O`waK5sr&H(eUjDUC_PmOYr?nr$
z*>nHtdtO1!CH-73P#desy(MJ>TLR>EFTP=<yPzB@X}npA)gWQWWVq}!WE77|mxp}B
z2z!3{c}ka=mdNJBOAzOkrJ`U}Wr)N#?~5r>PL+Xw$J7O4LEIi(^`9APeMZ<6@!t6@
zLgwH2K!jK>K_>tcm4|&$@j+S41M+onR6BKdOV;zGRTu(*?3duxAr&>=a$@~90S7&T
z=Y;JhQ`|p?{otV$UPw8<7UlabPH{b%D%rxVIYTm?k#=gD6-0wC!h@}|WD+aQpx-P!
z*CR&k`lqX6_kZ9nc*g%2m{KhiQ{3R029s0vair)m-UoPy2RcxDh%Z4*&t-Xr?vBbr
zsK;M@L2C)pAT>)vrL5YzG0J@&Nu#PNrm7`Ikf9}tS4Zf3*$SvqpWZq+K}Ju9+2@hu
zJ5$ck>k$g=F53Kp|G1)5pNJf;pEw^dz%@284s$>HL+4RqWIDUeQB7Mx4WSTsYq@#x
z$Vr5zck`n{nf5C}bQh=Hw+P_l;J5pn4>B6awC95ipuY}@VaYp<aoOJpK`1H3Hqe!D
zP=n|!&~$D<yinzz%$Hbj8Ytn$A!ToY)~a}4b^1f4aB*u^kU7kz^yR#$Lbd$3oHe_{
z&EOCAVj$HoizsS_{xT5k{`$bfGPF#pw+P5liC0ERBjFT$u3#Zp_n9=(xf5r&5&x$Z
zv~P$Iyn2(e6a9&o2l>b^AP%Ce#U}vz88TVRusn^fxGD07NjuX>8Jc_wp-Z4w%M6ee
z0NqDW+zw}uQFJ@}?|#xS))D)x!Te6i;9N*epaO8D0beraG-+r2-X}AC9<tCV5(9kh
zWil2)?pCRO9t<c`i8Ya2G@+W>HB}Jf6)>6;0+o0f%_y>z7%SSy-A_>OUowIfRf$A^
z9%_b`xe!j%ppbEJbP@~GG4=u%dJhRDY`k(73RwNbX5R#N<8{BB4y*g`FoI<zj}h8Y
zQA!pblOQ99`#Z0~rT>x2R08?3b?>%_5t#~-cn%|jb`)n5c2QdWPDCa|sdHsiGK;um
z$H7S^gGsp`X$5VPS4tek@-VZ`<6%0m!10%RLMuI5H25)~nDy1{L*64KU@hTQThEnN
zi=t~@y|mM)Z(h_U;cY(+defGRM6!j};{~{ga0)H5bTEiO#Q{`#S#-#$PB(OEx>mFB
z9{>S~OqEvNuC5p+el-Bk3RP#FsZk-=_O$H=edzv6ssr4$CtktfCyz;-cCE+nARozj
z9_si@k_{HB^2?~%M70S9;)2hBP*q~tQ=7VpijZZWU+$(`m|!X_IrankjdF!qOG>+{
zkA33Y#FZ9vLySWq`u|x_os)a*xO<^LRVRRSva=Mf+<LyvLrYcoQxqteXZAiW+}Knf
zEZOQ^cL-Z|Y54^m2%u)2;<&>f)IS766zOu=dwlc?qT@*UG7W``*wT9H;X@anSfC#U
zlkQgQ6VxZwnT{VhLFRT%%;aFSpQKQR`9QD2mOE9$rdG7^1r*ziY%u9vVr_6bL@!+^
z)lBQob`Ag))<nil<8l+fNO#LO@W$(bz#Zv^^67<+h?ZhM9M&@_KVk`}owS*AQ`nPR
z%#;s3(j2rRx&|QY47&aLDG38H)rHvu_0Pqu%(u*`O8Z3EjmV_xsTHsJ8r;T=kV@$<
zXAy2MKcR-huS-;v&P0wzC;R3t)F$6b6MsdLe-;Wx+(3F%56Xm_bq355QVsI`YZh!(
z{xix?ACBI-u_cn2)|xFs@hRP?bxO9j<K!D({50<=mF4Zxski9ol9sFApZ&^BRa8Ec
zkr>^9a7gw*rS4~^?dC42*30E(ely};&C(Npr@P`g2ZDZOa!@zKk1CWy@8#}KH5iyu
z@<J3%9Z#HiOfrVNvgKq(WNY^AG*Z-@ei_fB^YB5V$Uf44ZUSdH5n%>-uU_X+o8UE$
z!aKXhycp-dCI{&ibOxS=uJh6GWnwi4{cX~qexDYP*XSF0z~RVm5=Qh}tc0-E^Vb{>
zuWa=<Vq6xb#k&^76Np_x+Un<-mSFaOM9Q~FIAUCW&!_qiVoa%BG!A(#%uWDPY^Tn9
zRF$}-+T2n@c`|m(L-e<7Fk1Alo$tSPFkr~`%@heF70xBO`0qLx6%958Z-2U||JPu>
zmcVRG_|749VH;G1^Ut>__%@u%c+>=Z$VJj(h>g(82R|S4`!CZO<!W0Zn9;#s+)e@!
z#G;>c|Hp05(@VTjFIv>B>}ez6sOwO5zr+!bU5xV0N-C%bd>Myh=khTDn4Q84UD~YQ
z<D=?A_)q!xi8JJ~(|8Cr|KB?mRy~*?pTBT1-@8ka@*d{bXnT84z)n2o!I!uT<!V*r
zuQWk88E^)g&C?Xxw8cV*|I;enk6d<TsCMaI4w?V)oq%$i(YI7`Qdr5S2gWPzQd?|(
zN%Ca1{Q7}_XISH(+6s{%_&nO{Df2*d3Oj!2x!^w)L;9R`b4$Ex#Bk@Ed9<x5_@U%j
zKrI`r%K(j}7=rlw%fNt1=^W5wIf8Z|xWmTN$6%B>VdgY%#T$*Z3g=YJkMV!{jjn@G
zkoy1ek#M9hwU-R;Sm{dpDhg4{q;l3s^DR4BKUrPX+FBq(-Y~QHNv42``yTi)ZZ#D*
zl(M%Tl@@-N9cT|Fz!A7aep1`3vrUUQUV3blOkUJWNUh9locFQBn|tch$ukOz(0|G}
zy7liUK|AwrE9yEE*_p0=<S4)lk#!64(VydOZ{5y6=UdL#Z_U<q$8M2skQX>AUcE);
zfQRzUU8F(~*PpCFi%I<3N<T`PJBx~I?i*&`K;TBYbYStd@n21E)H_u9Alo$0@am#e
zaXmSf+glvavHKh5Cy+DZQ028}V2iZN$xn`I5kUm^Utlip9WGEp&=Hd`Vn9T(m~BgS
zx&Qpg>n$X$el$-v5Hwb1I9c`yWQM5VclD7i-2L`Y!bnq?KbMc|R|HpCEbx1`8Te<?
zYa=u$V<GUr`ulr2d(bKE>Omt;Ni5I``Y-M3yrC$>U{=mW4|U~J|Nl6(f)#Ztq}l@=
zTp*Q9woH3<gei|%k_9|UtX8Qm`wg7q<dtIO*n98XPApmxcb`;x*G0YAew)51eHK=m
zBF%*aR+6n7D5Vg=E;?=&@tHjirdKdh?25NK9&<IW)YZkYRMHz3G=odAdwd*`ykl(k
zGq9?7Q;SNcdiLL&ru9D0Mz62UeE*7rt4YdJr}=SCqDR5!8HhDGXa6IXAt&-{m;<xn
zgDyd~$vM%&5Z#$IYQ)~xPO}lV_<Xn*!|K^6Ez<79^9RlTkgr^2qu@|3V7VgZ!Br9^
zmOHk@CR<DL<yhUl?1=+q7?*84k#Qw}a@X%#ICK%<cSV^IyD?t@ccMVbB8c7Ado5*(
zgm#dqO`^KNgI5Y6YJX0LjoN^+1f9th7g|k(ZEAb2q9*RGy);jggosT{@4~8Yh=$%=
zzs|nx#<=HC`7bveMMNCKN2e|dcv7u^frwaDuYZ_3$5l(>Zt|%*?Hb&(O%KLUo?bZR
zH=P=*97|UCM8pUB9RQQn0hTXe9`O-bTNtaNv?D#7Ox?-%N(25nh{G+Y{5>Zg7Vz*?
zL3aFxaGN^y`M21s0VyBA1dUjeMLO$dDyND_eJL`|<1$!ZhsV(j`0OZ>N-rx+_28}V
zi<L~zHMErM_D6K|LL`5c9<D%c?-0@jZ+-nTAtVi{eaW+B2~5?-Z~-bO#xJ)b0|Mcg
zhU)dv+fjS$EN<i@&;Fv1V+07~-ndG6PFBd&Iacgk(O;s=x(}EpvaXO_uKZl`^w}XS
zuS%YKW84k?2+9#aP9%&bOi;_DjoHGwi1rNaMgtFHmkR@+#3tQZdWlXwqm)>Ioemhm
zU%~(dg$R#Zo3WQUwmpmQCe8rEh#f=?bgC9JA<CqlV!5^fMgo^97_2olsD@`?@+0I;
zzs<T<1syC>2sCZTZw9)@{oc;3xMC@E4J8zF7~y+IC*Azr%8%ulV4?dzKam=*Zg=@M
z!ekCRwA3###)8IH)866NPl=J4154w`M5`axz5FOCraa{?Pw7{2?z66KNRRaXR;3@l
zn#?nD?*A@NU%pAXses&q@8D9MM)?5WB=ihq%4<58Pu0x9Eq)ft&Y?<5go@V+hS>tH
z3j7~0YU}btn3Ul*J|A0tn}=3OM5Cky8R}q~FGe<pmM`<C+KFi9DPZ>8vSB-rGR73O
znvbmHyWN*F@gZFIQhBeKG;I3;q=fB3y}Y6SC}c_h<)g-~QdSytLt4GsNdGyWeY4R1
zd^5%ZD)kq8Y`0w1P~GP>U))rCSrn)^!KIkW(eM|t)$n=f7K|hI#PR&=$5(dFBvi>F
zwAzs!N<hZ?4}rPq)|fn)H`S}zCRQE_sYsG1x0Eon-`9Ea-B>m?XZf?wLBL6@hitko
zC8SmMs}gg;(YXsWV+8kn4@w{V-gJLHqJ<CGBpVx$u{B%xb0xbT_$~>u!*|B4Ou^Dg
z6bSO$2k9a}*T!B2BnsNkBp3adPvZg$mTCka$u#t~)#0IcziH~Yh_xh0RZ*9zp;=f*
zGw!Q|*1Y`pe+);a^lUgOH|XE(STP%CFfw^$X(BP(b7_GZ_c-$~2n00aQwe8n0KWAQ
z7vA@b2yvm1k}FitH!zn4eLb8p1;P*k+YPA|RUFW@N)H$IJlO{!bQ9EB*T7Z7cx<`C
zs;KCB*_<V{3PJ~fLVh8EUvJ)0(~wa>?MPk@ERHS4ql0iD<4SUkRvhe0aL|zw94Nhg
zEtt*NXmx?{oYnJqH&_6w7J=S!yQ$1H^i4-k$7uFw2k1O=ZP*kR1@8aJVq7XzH$*pQ
zqenqH^}G10_797oqXNN?k8^;;#Qn#Umm^<`n(%dW+aNU0ptSm*f@WX56r&I@f-~|j
z*(<WK05b}t$6u!hY6jVLwx>ICW447!|Kwd<1}$5G{In4GwoSE1f6pUI<RjIk|Mh%d
ztnkP!7&g&$L6woQ7`Pf+KI?2qYohbqlJ|5vAc7FNvpbDb$~l_k;_ATA7r7eXBVV^2
zr{kaUN&xM)z9Bb)c$Tw9n>|lDw<b<H+eM1IrBY6bwWhZ<hC+f|6}bDadu<i(|L}(S
zw0gGPeB<&@Jucs4GdE0lg7SgY&R#VV6Dn2SJB!j>7eG(<qt0I|&QWQ;;?WEw;=(fK
z=rmbJk<pYa(97H~9>jG05T^RU<@RZ-lK4Og^$<REzCbbx-4zESGz(D0J%3fd8V*_E
z8_KdQ-gvbkfkkXrs-STqIK9@nk|m*>Xcf^ss+rb>1&GH)3K+v5pwi}}NWmVMGa`u5
zd}tOu`Z{hZ<VGfqcpN~bN}q}p^P;4SqWGibuzHK9X+4SU$^#mRO2;oFFXUFmGPyE6
zV&n}A3)A9nNSk!z=+<u~qX?INX>knt!;vrtx%E`Jhpu#-7#ONAzQ`KpObvwmCu(`@
zD)~@%|Dm+&vFpH4Y3tfh2S${Tvc3c<+wPdX=SHpXYp<qj)PBXfh?bnoL!a5A-|U-i
z9^{_`D3}RVdqCuc3~Fl!rB~0MYcG$r4yX);@Q7DW#?C_Wzd!BaQf#q23pGa$LJzk5
z>sk4ym?8r)D@`RbsnL7EoH*2mh8O;Z!s<bwX3rmHb64QWGFUE&g@&7{d;Q-u|8BV)
zW(uD-gRkwG-=qy88vC`~+?GB338{~T?^Q^zd)&p>OuHc#U!^58xc0(!gJr5&DgM{*
ziE++3#~XIF1d_C*7?e`)KO_(Xv@*?vAqqbWr~xNPW$|r6xVW@0l@0UlZ^=S$1fn7g
zZ=r}F!Qtb>_u&=M8W<R9Yp;!d<?rqZPVr+YmCZjcA;1ki&8w_D@z|TWF*loF`J2#E
zB`$2Y1j9$r9}mMyFQ4hz)fV4=v6RPt*B$^Wcb}mU!$PR;#A}~__XD@Vwa38|Thh|a
zCU;dSEI%pV+7{Cm&SVnI%!AR5={U#3(?X!^p0!t0wLt;pCGvi>Q|oH~#c_9>$+sSt
z0;``T?PPDK9iXjC)Y?-lK^hB<q5I(BBN<|`&nGGA=bCcIzwlr|T55~Is|otis&bF&
z;IV$3?U^h{ZmXB-w^e2~>X(2N2(Iv3&RzgP^<5aY)L=NFR+-5YKOSiI^0WJOfF>-i
z3w(M$9D+IU8W^Zp`l_TdYxdm8*245k))gb&p-QUDGNX4mBKa7{RjJB7B~rq=>@;2Y
zaFeY)AkSVN`)^as^L2_&^*R4g*{VC-rg)W5)^0T1uJd@x6gv9FQLSJGV~r_0a<bD~
z!SLp&xetgS=76<<@yTD+nrBqcrh9{fmvOKnac{iCX!L!1l5nR_tQ=nG61dz;f~f+)
zF7t=8ai-xMULb8}g`btTgC~m@@U44=^}N0Ani(@oPD4Erf={!9+BoLP1_zZ2qA+Cs
znYe%+Vuv*D{p^W5*l{Qd1yB34$9j{wfaCrgJcyK>RPQ_oOK*bV+DhO8r|awMZzEv&
z!pLDjZi5nPt%N7w@UNuWNfvoosQvkH{nkua?1~aDXi>$K4-hvbG2%;-sz{y`@41EG
zWJ;;Kvx~yP(`j|Y@cyYX`Ruf#?%Kb~v|L795^9mmtfz%vP?J|!xWak5+K`rS*4wn2
z=OfQLhj+*hGMo&s_Aq>l`D@htD`%w4k(~vR2fzH}J3C*f7J&py_KfYsJAggbdG*|j
z;=Gj?<v4>xrZ$4yXG90g&qJt8!=Wq~5^Y!3=LoGhpLq$BRHOl{7?fH_bv35*US_%6
z8#y}!B_~;|_@y=P%ch#WO7}Y7Vh$A9fXjB-bKB;ccLt(@3fi42;tLFHI0u7N0qnv>
zvWlp{=P$KVYrou~dHqmHgdgWyq!LA7I9aKgoE3p)&>%uY7M(CV230+fJlg#%uzdU8
zG~rXqNQ}ul|Ex({y0)%mFElqwIuiKY=uajsu2Sp)k$T;3n8v<?v!b8lE2H=hoK+Sb
zu?yTf6*GO{3MyIG{i%vip@6g8|G3`g?x9mx)V3Dt*x`gCN9swJ^hdAMh&IUxb!4<p
zWs572-VRv$XaxiBi}c#1Wc8Y@x||1<qqRNqAr0%zFZlR|jjXp{_XfA~943#A`?K!S
z9b$b_+_SXx?DrOJ4+P^yY8h2Y^M*j8Ad;i~FJ+<OfjOEtv_|CM-R!%HGU7I&;Q3Ti
zk^7+@f-~Op-g{2Wmws-*jp7B*lp37)!MhLS<84p#{bB5S+H#LBa%XlPk+xY}GZ|cE
z;U`X713O8#0TbwKx>hc7!;X8Yz>q14UNGT}Q90v$A+E5O+>-^Qqtfb6EWY0-k`~L0
zgt=D>@N2~b<+hYCGC7vy*^2{r`~J5FPa+G2v9M}}kF=yai?XYIfnOjH20%Py_VV2H
z7Y!eY0N@09LUZ0H7D)^71gK%s=b<7C&sBnG@cWsLpT0A+QFXCW&3~cLadXA8_B|sj
zklWjI-mhPUL5|Z73Qz6ptyW0FEzuPtl!)#6@ps}&W-iiS*<*&V4(efz0pPJ=l-wT&
zN>@b|PAGGx3d66GlU9r_d>p1-%uH}AQ)k>*V5OtiOPzy?Vf<yGq7A8|mL8J7>BdJ3
zo9;ev%MkDOrKo;$7EaHZ$s9Bz5=}NOmmiwPs6|zc`;P9*{XRsXUh}!vJt6`)f(1a-
zdl@D~1DK>Kn8pBw57>>qzI$ct_cnOVc)+gisvCi*?No5m$`7w`o@vZ6mI3^}K};BC
zru7ETJL-VaYhB5yLD2Ux^%buqAgN*+fblq}@nj^iMddOM66THCS7@tA<hu_$byEMW
zBv-{O-+ccW{Ar^P9E3(-s2a*uR{|WdmO`?WyU^WGpHc!tTHCS4GZC6^oS-^ZsAc&j
zqo)YGPWcd<RYYcSxy7*QUZxfkE7%q&cPC(=@i2x<o4p|Tr+<B+<=&of$&a>yAF2Ao
zNL3$FRm_?uso@aFePZEV`H@03bK1<oT-}Vn#FDBic%DyJDDhvrYka;5aVE9U!;xMw
zs;r6<dm+_+jN)b^7yo5o-c``O+GuJmzCKDhU+7KtlaXuaE=SP92SdJlDD=t*;3_TG
z7{_gwPrCH+IWfCf$z-&yqqolKr0QsX_9!*3+eFDUhV}hE=M)O5K0D1Et1aq}#Pe$1
zS;>;|u@WP;#CacBa<To}zDuE$zB7_@cc<~jFnG=QqFJ0E6OAYghIKn)=0VmZ!%sN@
z_O~JQu){}UG}u=-S$NHfCu<3vm^82rohepaDV0x-0<UEzA0jFUr9OWwl`$1IaP$%J
zTIjOj_clSJAp}HW4n&YhK<8ua@R)?I+`5w0mqwBE=$1i6Ik-h}y%u&#QB79loxL<F
z-p%)a#1XSTRY!U#dV_Vvb8&yZN@S#g+doV>G~Y|Fj`RrCV;FOv;TymD3Qh6^MA_1d
z@59CT+LL^-`Pc$4oo8)8tNR`IG}0CNAU0sWkgES-V+-u2uLg@~?!4-tHp$`om(Q`)
z_AP{noBhAWI~QvH-b>^F#7WZ6Dr?&JA7jV*;5#dLYHo%FO)IgU>bAB&{D4J3L!8?w
zR!+3;>}s@*;`K%A#k7P-G#E-%4v|4ANJ09C2eu#n>#yd(&!&_TYUcRWS>ui*qj`GB
zS!?c>E>4;G;d_23@hXB}v=hx!sIa67%om&4=`b&LIc5&m?S~l)AN6kE$<Fd@@Z`L6
z(9xtSRHRY?r<B<H#^3KHni=8fQ6YbKuNs~@WesB&c95h8n9U~j_ns+Rjm1tWeH=uK
zk=jYvp}H6`mA;aw={;ZW$t0QwJo$X|Sk?2*OIi>H4??x|Ix%LKi$`B@0({z1e#{NP
zt;mm>hXpffEQRW>7QVAUzbXx_kjRl>AW#>%=p(;??Kfqx+tnX~G&9&@Kc4^SOc5P^
z%D1q%tYCS8WVMR0OsUq^fKIf;jI`%09pz$KQfjQ5fRxrYvTC1w7B`AL>A=ysseNNg
z+4B(flZm5Rrt6&*InXNNc6;9M7xvB(Wa?7WLKP+gC|^A&pJci!_f4MkUH)#qXlu4*
z{&Tu-@~Z3?oI=(p4F<5RWz)&8^4Oph^zX+Y8)I-IOXXJS)9V%9b=|p<?4l{p11>az
ztMH^aNc;x`3(l(Uq>&o$e=Sby3AkXpw2>(`1~3{(YH@F-Q*z}LT7$pPsmPy=g5>rs
z71sZ2N)#%ns0#Kkx=JcjqYQ8S-8bp!?!1T_yNk4d?u}KGDO4<oxRKWoM#RJL?X`L6
zNS0l^dV}({2=5np#6{p8p!wISWzuw*UHM1#>4lw7>MroUl;h~%Dp=KJ$ij0-pYjV-
zm^weQv$Mwnm6Y~n|3=)}{g|mo|0?D1g^ME<VeAA)j}qq#JWD%U(1QylJ|%)-XNIf&
z0<ZNin!+<a@}|K}z_l$hRw$1qzU0|$<DIPtyhH*Yb2XkmU*h*k|0t-d+IpkPUII*p
zT_?a<ANN?8J9o4Cmpu4HHA|1=JCy2pu7@N=RA^$cm9HFC1$q4$KlU~{8uf#HYX~?$
zpeY4ai<3^G4Z~u+d$~k~yM`h0J=+f507l2#AK1t#qPYQ{dEo8!en3MhH0AVar3}6B
zB|hv{xF?7UJ{d1omNMVAg#D@$lFH1&;w2tF+FR9UlHoy`pgR)WyC9rUDqCFIE{2rl
zu?y`<e$Iwa$cAx*MRokd$+IUT2S47@pHv@hBZJ&3H67sR`Tk*n_Tx2wZbIw+_Lsc(
zu3j2`2&Bb2t^h5(Wp-0b8P2)Q1d|`Etx_KzHl%JQW_{C~9zB_L#2Gvq8YdC_;R4QG
z7%x|(YvZ~|*YykYO{>E<o9d1h!@LlpSQ&&XQ26sl`xT7@vqkL|aG7<<rpH|(HKXF9
zO;jwzVT=d>lK6cJ$Od1YT!sts`gcnj_KR&op-O)>nLn`{lK|POTayaE9FWrYOs5vn
zHT5z=D0%;WuDi`Ye5L_o?LOHA0DfRkvh`{Y_}H|=c~@XJQ*3|(Po4p>9wyzIX>+1I
zinW+4X1DN?VJxH)i}1~(>+-RKU<B+Zi8Cq%A&Fx22OYSMS|_N5;<v^n6GNR<IQ2ua
zWiS>Yj<2o77f6N}ygnZfmpmzRWR0T$U~7z*s|>|}4In^^MUws?qt;azbWRjfe?I6D
zd3h7ySnl5w0<~VC<!rwhVt7oZjVq3w9f|#RuGX5hR%Xf=0kI=LND^*D&X@D%5&>bY
z-`^8M?eRaBlE1IdVAj<G&)On?BV$2m(-$>l)_%b7za1Sk1|hv?fipJ>98Z_XR3`4Z
zUeyB$M#tJLA-Pgwu4uXdKI_aHYPR~q1E*2s?#38Ei|p_qQ@|I!A?S_+pq7S9C_Du3
z;4+%0Dhy(DyLS3www_SbKPyw6UgHi@od^5Hf|QM3KA8VBWd1{C92d~$ghKE3^90;X
z+ul=*hjt(3CTj>U>wi;F2h@Hs#MVb~u7A=aRU31eoe_qzI>hY6a5+?S^?11qp=bob
ztFK-FP9B4uWl2M51$Z$e(D}UmUfPeM@w#~a^1;p2xbE?a#kqnSbwTs)1uEA;zZzs0
z@4Bf8Sbmdq3R%^ZfV9Mc!g|>I=fN33<uzXUKsIB(XWN9@V9_ZZrPKRVG|(4J4aPP#
z&JFU@Z&i+MkeK~ZxVu<g5a~6c2(`|g7NVArhM6`IKULU(3lJgi25qBK^x^}^QOxq@
zFJ{+T@bCLq@aAA}ozSI?onm+T{C!xbyTG*&r&(wh8YcA`1!vk5z?E!tnGdx59DG6P
zd|xg0-FvZ+*HAx=j61JVq?SdNNOvU@0%}Tdf2x<_onQQsh6chCzEouYJ^Rf_it7Cb
zP|{;+ts|O~f8S04*>AkrOWUp)dOD<Z=AKMZxKNU}{y%p|J*y(!Wo~ZWdbhgfCj9$I
z42FX6v8C~RiJ_4}+wc-eYMi9wP=V4_?&-D688B+MHqKh@JHs?n?J`X~*MMNz-Kd}`
zcmvayDE_YNP~~T#c)z<Gt-^^yVCX#e6vO!<(o*S>kIcy@9%*ywtoh`t+W|p<P_w}E
zC$KQkA8uS??UY9@kd`qC;(ovoeBC+a)DPlli{<V98$mdqYHH3&r7o7$_TbAj24>3&
zIvH+U%y_VdY4l*#d1Lm8cJ5mUIV^sXK0j*8l(thE_I|zS8Pl3YSETYq0c*{I^ywkk
zYa`{){LlS8BVAg1Ku?!M<`7~LPRHUnoS-2klFij20mqk!;<V}O+)p>!w9xzLoEC_r
zE;#xCuDfVyxTg|>isd^f@XOG`-{~v$0gTJhRQkv5XDG4^_QrbUm7tWCw{P|Fwmu*s
zzVJh6?9Jya{ji`mP}OTOHG!YxuSZimy^*HPsQ!n{e@DMNAT!sNY!34__=`?1WsZ|4
zbi_F*c+kF}*mK#NR|CZ@8b(=zuN+1VWe(6HsOP^k{Oo!Qgzhw$<SQfWR;L92Gn~Qg
zX`6LvpF?i<U4)fiJ>wgx<IeS~25aglxK$Ukfv{!DRQ@s5T@%O-)8NN=LE>}f$-kQ7
zc{atpYZUeLtfN`MNb@81WLEiu!&3!guw>SM#a0@5NE%0yOa3#K->PD|sly<*t{`z)
zn4O-ZeF}ad0LknAXG&>w34_GLg=^;ASxXoE*VaDK)52P5r7|m+VP)?Z4Tk^fZ&v5d
zJ_NH%9|7wl$!A+~jp0%yGPxBM@4m-u>mTbc&7AQ3TNHmPXZ&b0)nsofBwClQi(EUV
za=%7tJ=N88HrlWO6tP3o*RbCtX4>9V*V|IpdwRpv9}sl~iI|Y;x0*EFU>~ZZ;ph*f
zQZC$Lk2qcuzmjflLVrL!z9QRUA$UbI2PI7-y>4E(f?nFEB(GHKVp*O+rtJ{BM!}|Z
zp0yh}LUY6F?t`7f*Nxx#&o2dPBGLuvfBTz(6Cngtu!~2$4VE~-s<GmB-13Cm7d>VZ
z+IFhb+bByleItWB{7(w0e1E-wwao{={>MfCJnINC^oSi4%fPEtPkiHjkS>~pzHOa0
z^;Nw(J&;Mw66Gdt6Uy#F*^~oUV06{|ZgSF<RO}jQeKh&qC*0#&)8iV+2>H4UQRFoF
zGQ0BDVpp3iRXE8Fh8r0yiSf51O#@bj-9Ax+e^bx^(>7h*l%FaQP)I}~0V3&<(T94H
z<3<~hR{qp?B;F6>*B_|!l4G@*Qx~<4{=80RHkP=c>w8xds*A`=DA)B3>{i4_md=gs
zUw=7g>eS)tneeyL@1){cP_((+JNJ@MAhR`Mn-^S)cN%drT%Gt1ODi|ZyCGP+z=&9R
zXuqDw4fx3T4CR|G%8OIY@%4mi(D+4Y((1NWX!>^-#NgLxkP&x)v8OiEY}*dAds79G
z3e=w}5&%gz#X@8}NTEhO_u36>3oR9vDZYr%L>fC(e<FeTfKAkCY7d6+iaB&Q<*&-l
z4bi7?ZmLMG#mfJYx<aiu(#K=&q&=qAoU^A!=Eee<FHQ)&2pCPM2+9VWe)$&m?59<n
z+BwcsHnq$`eL$Y-28y$KFKwwpUcgVQJ0L{BOOu}9wCDy`xG|*mh48yEC`eH)=e+h|
z-1K|H@IoTDls5MW?*~*GbpTle4~54Bf*NC)V0Dr7fU5S0sc;!vR~f4`d#IH=S$sk;
z7Oxf(#-27{TX2b})r|vSfZmWCTO0(+W;TY4Cu@Pk6f>HZ)i~LIE$BWQ()-l!T~e&P
zbrLWh=y#A-y1&rd4cRsIo6FGhH_e|h&EjSv9O<?3;%ZE=o4qT$rV8?Sm`$kjLQ_Su
zsKR6ULr0o!SyuVa>oc0dh^*!$cSW~G8ssEca{Q#`A;Hw}i*8D@>873shX~Hkur@O|
ziSL+H&ql--$mNG6akD0G0YN3oj~ST(s*71UO|7-ApUZt#8qh`xur0WHm;`;2Q`4uS
zP(Rns%tL$ar!1gLcr41W?fWv5aH*4p#O59?@$ZCxq1W+UAOA{O;})MYni#4$1FV~&
z;pI&LQFrW7)&g~z5*dI$5^VjuX8{DDV=?tUKjkM1P5})gJ(V3n{TU3TJ$dk%yspOp
zIec72q6!={?MTSYk8ZkJixqvn5Iwy?34PhGCUm1qnFtQGQ9+<(d;RZdhWIUzFqE1c
z_IKC#iPqyRiP{^s;Z%?Beo+`4AKeOT4W#PU3ctVd$fSv}WY&m$B)yO^d7u3el)S3R
z$0$Z&c~bY_Kd)g3FQKB0E!(A&hLUB|b19MH#iF4{uRpq74dt?tX;WmQ;D@Rhjz#W6
zEN+l1Mj%)gyozO^)|BSJWj$?5siVD*mY`^F*GOF+L~+^)3t=sFWN7@@-0aP>=0Gai
zE(GwBa}NrD8kNvWtZAp=zKWc&1pGYKBh^aHtO=CP{2*F*K@EnRty1oDUFqT=kahd%
zXFf-PkfH30-;W>K>Wx{_EwZ|MC_A5Pf226!F?ljSJAFkCT~aGJk$czw(y0T}aDuuw
zR_1k_c`{WsdW-$so}pFU)hfIJj=t3pISeLSa~9)6!k!0pXlNY~&Y+9A)o2#5Ktw7r
zCP(g&nf-}Wp{QZntNcjL>y)pKKeQ&AxSj;bf)gn;xM6**n5#}@w*KldGcGMBF0Rxz
z#C*=AK!!PLSrLx*F(PJ7Al(y5_gB;>SB3hPr#NAz(>jBZ#*q8epLL?O<R2}twp4A<
z7Hkd))GZ5Em;oj!!4m_SGm&L;-`DBWhEkB(ZEU0#yww+hc5t(i8L1e{jZW=#v$Ba*
zb`9(4k`2h5sWahQUjwtptFZxKhT)g))rPFo&M}^<*A9gllR@Mu89~49r^19Obi2@g
z4|}`fA@9#7B=l6(wLZnMEFP%eu`1F|DCJ)k6TTj(yL(C1G>dEF$(2n*c4Ecr$s&d<
zq7^&5o7YSmk1Y;**w4M_BS-)jD;6?QXni7obF4#S>4?9uJeCYB^>}?2mIq_g7_D<y
zmq!RZJIOXYRO8}UO3gWk{T&iR;97q?`Gz6EI-(YJ6W{O=bg}SbnYX46T4q;YtaZ0A
z$<v8au+`v#8Z*QD_YR$PHVmjP+e$YpM6<;EhzAmR@rdnW`9%k(KhWv^KMO|m>LlU&
zgRAaDe;qmE8_PDdwG9sMZD9qu1grpgSCM1@+3oDhpSi%8>Qef{+G#b+Lea38)E7Ar
z_17hR5!q&XWpiKgkNJ~D^{*zb;2n3+B5ko}*Q0cX*26pcKh+P@Ga0}U%_q%F3TGxD
zil-fSYH6k0FU?j{Yw|$f9KH(Qv@05QoF5REuhi`(hM%_7S^jW952%+rFBBO6ePDk`
zk=ku;(j{U?A=fB9esj^1#6LW*js5T1(rLkJpV)e^>1+^XN^k{cG%prg9C@iUpT~RC
z;0&-7D0_Fq<c>$I{HA!n72}3qe;~o64)eGr*NlT7n#?-b4UtQ?w!Y+mVuKgIxn@hx
zGbu}lO+D9f)Zj8Nvy&?-D*1;rP5sThPh#1YXMbQBcx&`=>9lh&CJCJf+<0(hxI&QJ
z6qtpPE1W`c;b~A<(<C%!vrTRoI1>2{U1Vt%##=H4)ZFkv>Ch=7-TiuKQ(B}v^L&0_
zsLHQ^{s$iwfj94QFei@@2l0YhZM+`KGCp7Q6ZGYlZDg^};d6Ef!^eNt^#80&3n1;!
zd62)<r^Djuryu4ll=-(Qa{Rt_bA_ySTAl<Lo4!p#VKX?IkNn{uEWQ1Q0@jde<DbQj
zcA1m2o)hM#M2q&F=PcLrKe3eNvS2g(1G2&MzVjZWTE<!{8wd)c@Z^ysXWdR4nJY@O
zcjmY6stF*!*$YYIf}xM?O{lN_e<jrq$l1_H^6@6L_Y|6X=?dy%fnlD@Iv5Dr!}RR$
z226~_$Z?ev|Gr#*4EOO+<GxXNo2}z>z}-qZ&Rc`qr<Bd4ekE}rkmcxAHzao92sMyO
z$5;1N6JxQVgFPHJ^_^)^ptvWQneU&Nh3Sj%yC_Dc&I@8$I>AedpWsb~(_0p04dnl{
z^Hj_qzuu?*23GYn{`+pa-#Y(3@pzBp&Dg0(x}b{xN^T#NqjWv>GZ2RZYf>d|6+p15
zER+97S%nnruY?W|x{)(9c5Yj~^8aKv9I*RA1|(2;exd!y26h8w!cUY6ILj59Eh@+Y
zc6(92BE1rKjxqZV(;|X24sqiH(LHw=LwHh|K3?4qS0Me+=`7wH3p1UZHaH*z&>IF6
zpXi`8n3yD?-$Z;&ZLxdG25v)(uoKHy+8rb7F*Tx@Nj0n6Waw~YzDH(|O4Db(YUoY(
zIr3{faGToLP8zD~6zeLsKZdj7)xyO_4@6pu!<Cj5-B{IBRVnO77JewA25JM*Ry2pH
zSI^j2Rq#SZ;&XEl%F(+#o#9l3B#aKc)|gmO6}TN~0orJ$i_tX3`YMa1S6$kP9YnWM
z{+)E7(*lm;w^Bdxhv|feTT&e|)Q(&XV^Vj|G-YNb`ngF;iWu%T+G>L{CgCY+dn7}w
zcmLMwY9u?9zI16%)F+!W`a&3|c>p{^W3weLJ$>w0wTM%ckNG<CL`RCHFG-MhUWUt&
z;_wnDAc9Kx4L{Z6Ki{1YNESmYsekI~@nU+j@E5;pI*+JvWFl79y@V%J`j7y`NlQx1
z0WvHn5i809D_-Bmx~NSu>5_|z<a#eArh2s*z8Co-y~vstxBz(F@qFNh7zNff_0lMM
zn2^!M&z2(REfV1-rb4xL2krI~b>6#SRA?7i07NldhY{so6Tp%L?v>cQFK#1D$B5uY
zBxuo-5-dJ@vzAME`qeo9eVON$7mX+A4pD(Ty@DzJ{9mY2PuH;bPH*GwA>^{@I=JD4
z_x#($Q?p?!S$4m<o@LVt8&NA(R<-V-BtaH~{$9W85~<B|h^K<HE!4m6cpyL33(-PV
zzsF7wZy1y5cmf&w7XB->kLXn@2<`*@@j`zcL`Z`$w6psIw4>Zn(P?9IPa0GuafKjj
zCRb4LLqhAFtk=HDoopcD`{<&_q8g!IVdxxjPez*UamfuD4a<lL{K4{Ckl{FdDCcq#
zWEXgBFui+W6LS3(RPQGqF}$E}jos|kVDNB4SX#rUhhmh@5->(7TlNkv4UBUi!&$(Z
zfV?V<YYrUg$leKuaki=(S)Ag~_z=&fYFvz~Q5IBzX~14T^vc<~%NOb~??w66jrk*m
z9j?b~6&zyjee>d-ert@w%gpI=X(jprbm#>JDkG(uT8n__=e>?;fX}{b36Zz~dULG0
zF~2R%GqdBkeD_UA0mpa`YKjN48!_GAs#q<NFDb|+RIr(=nbaL0bU_JN1x#?Y`p~H>
z(TQbc(hF6iuGl)QFz)zsF)LOY=eVh%b)-B__7&Bq@MFYIHD-zQG>gfTL@?Wq{Q2eg
zSFin;M!(~vv8GQUpj|O#mju{IcopsgmE-jUo$#3MQrYC@S99+5>rR%2(zE2=l|y}P
zRdJIc2X`s8_5{S939`L0TcI64(y2ScP5P2mHVn)G5=szdzh_zl62bm-t|i9$4l%`a
zDug);&A&2X?H^fH5s{DV(EwOgpE%o!K4v9Z$-X&E4$EEU4!G8RtysTHh2Zms?k@25
zZ;g;}XWc#}&WVe0WZNn%KS};{&iJ<v`l+R!3Ev|-{S<F8pO^QRV+TJh=kz`V+u;{q
zlX+b%;>VKxa%y)6gA)01F9xoYzYFmuGRBK5TyH5*Er+zAy;t11eg33c$eK9$r);-!
zgCR&pweCt_!x$8!4ztIT9jHnuA*p`?8d`s~?U2!-!aJ0v?eH;*nVgVKjbaw1om`ek
zDShNIc=+4X81}76-!!@c1+a|FMvJ5noC-E3)ZpG44M<kcHzeP|YSdlpCY7tgn>=Jo
z+HAw}l>Hvn-TODjAMRpM-tscFu;_b=@Qy9H$9^ZZUxP|itUlEirQ#7sSR=bl2dc8+
znNjkdI^Znsbi59A{T(O~c?41))rP_$3{ibH;e<IkzcrZcju(-GfPCZMQc>SEd1U6_
zG#KtcAAAtt#LiJh#I9kXaMkvllLnbz{*pDfn0HmHsMft?lC{GC$CaPB!Cl6^RW&Q4
z3R%8$oA}m`1LAY$5e&SVBR-hDBtHuqf8Jy<&deOpFld-9Y+V2nbs>-FJ{vVSvy4$&
zM8gc?zn>Q`2|+?10D#a>B*@Mi|7O(&dbHp`q#AOI`qSZD(7G(9VimAgJvf?6AIjuW
zs`Bal#G7oeP{4U`7OVqqZoDp(iemZ7Q}`jGSHgJdQ9?ErH;bHo&)Phcf3voA)!MgC
z)Vo8~ICL?$w(lc!GS*5h{;e_LB_)o0X8kQYHO57cnTdtzxl{)Oiq68s@?QZ(r-J=p
z6(-?biy5C`x@p0bdA!}9smAS{V=ULc(4jZJiDw@-@I0!b$`x`=L5;zPx1oWu*At=w
zG&bw1IJ>>G>|~7c4-(WL{K7)Mk}R9z-_E(a7-=P$SuTY^m1sT>BLKasteJ*3fs=Hh
zd2TEY1(+w|8pwAUkm+!UxpqrIO)r)^8JE8>4FXj%**=^G0W2=mwF=YLN})tG3%sgY
zV!TUnphG@$GTbt{D?&trwr@gTDhw3@(eAc8Yf<;O20^8ypp*k!oSZby$qvdt@%Q<~
zZaa(cTyQNzJLF4B#dwb+-bcZ{)e_(w!+2F~I4|245GN_Q$Wdve!Jzz=H0(TO8d)O|
z9r~bK;K5N`bBMyO3;MD^9q`X-k1v!ffu*t+LDDept2<DVzgdf+ZSNtBUPh?GusTej
z@J}<gkkECw0K;_W4@e1?KW(dV3{7kA4Up7hw?mpyE5fu?+K7&ZTZEcP_%x%0!WHLr
z+DmJz%s4<8?8Q%EgHVWOved%+H!>-%v?V`2bAw@Npf+E3l^EV3r@hKBKS8{2YAhmT
zJWrlMx06f?r(1R`FHB=i%^+%AU-!Dptc|UF)szmzFyL#WC*D^3rbf!j;K+1mPQ@wx
z1KNTab%}ys&i;#`1i=;rz}M|)nvN-ReYPraB;J5!p*Izal!z($a*DeJM`}m<@ib9c
zV#{dQJGR}sOumA7>2HTOu{2T-%JL^r1#`q<CXwQ3O*zaR<-X1KYUYja1s0BoH_H96
zWH9fTVK7*Ce3xJ7D1((jQ=6~jbgmo*3V6FMe_k3ifC<sOi55*1E<WG^t;^lcbz+oV
zuh|<@)RFQNZqU5t1o@e%WB+NGHQP7Zv+^-3pBt|M(f@m2hk^nAJs!>B#wAb+&R%ra
zCQ8rYic06(0js-1AwBHsXbY7En3qDAfpU-ZXc_fB(Iu?&W+kJ$oe+o2r;Yefug?}-
z_>`0!T72hE>^|$bAhd40^kgSmo`%ljEG(k0;g`hle#eZu*{t{aa}uOn4Pw+Gt}Fn-
z$f7yuJl9-Sk<$0VD7u+HqTsd^{$gF|Ro9#>rpKJ(BE8xh4v5fMMAguCjH&a?yR#)0
zGHt_T1$QhWooO-xT_KsoroE)cwZXJ)P1{`(FBv@vPi0tn7G)B4{%bv|p%VSYON#--
zS#2M1+e$<aTVu^kL}S)!hA!IG*n03GHA^y1FK))%O7dw<#6`>l2C<9-0`WP)#D!nf
z<2~>S?jN?3ta$5a%T4{mAf=b<E%vmW-~Q!5IPwc09+3LI@M>8#`Mi-6m`1M29ta{v
zZ?*U2@APPyguPkw9;fe(zN^d9(gkS$Y%m_r<Age;XZ+?YNLGS|CNCRofx|<rD;G9I
zG4;J;716=J6P(oyI~uG+Kq%p{Jp|Rceu}Xuw7~9P^zC=dplaa(a-meCn~lqOP4Cw@
z&AU&>%FVmQ?{IL@U2Xn?3pxKC0OSAE@f*>XIH;yBA)^y9RktjOVt4oVP**C29pZ_p
zDXrIh_H_^Mcunz)=@I%Ner9NcntOwDQ_GhX+~_im6)>%vGk^hRi+j_MA<IhEo?HL6
zrENYpZd6PUwLo}|f{4^l-LcQ9W;$Pgq5CWSt}XStX5RKq!%nDMWoN?e$7IOSHRSz{
zL~QTM`@1j!y?aY%+^_2igg^48D5Wf?F_In8pZ_g5L}V@l>_vhT8ESOY7NR^2eB{0x
z)42sdNgDhie`YAa^+#A@HGh<3&P(O#(yGfnR6WF+7*@V0@eLH(Q_A{WPEF|TbGoWJ
zMCc)!GC$(_)TqoXqOxzw;dYCqIoFn{bV<0hOi><Elf?CS%8KgMfASMh;BO$y>o$Ei
z`8f;{iKHRx3A_N!27u8h<TvcjX*_$bD2wB<T!WDjwIaGD`|gOE2N+a(|0#pnVnZst
z3Pl#>1VPWqT$=A?zfaPFI<6a(7PZWNlVvqJL!TH?Rg1Z0C?$fmrD|fn`fd|^xT@ni
zYMa{lKLKeMmgwr~cRvhK4H=CmgiHstQRjTB_&Q@D8BxxMq%@#3PGytHEoEU@x>abL
zl3s)Q0qH=UfKvidj%kve<nLyWVI80PsX&=2_dEz<cvg*&nn~RSZK3DEfBgoE8gUt$
zES}^CJ_uIG=$1Bm5O~iyBP68$KV7dutD84OnV35#ePkSByN$!OQMRqBYHdc@mqUae
zFfeCU9SxgHlzqVf4&Gh~d|7oySrt5rwRNtdMao=+jzFf~w=}zzQ5N7!!Q+dR<>NmV
zfz4Joqs-AXb73tDT7L-YG_9Srt86j!n6f;tP=vtcx+$w3XOEy3l0yWGMI!C3jIxcx
zG|*q^Jw~=e3R|J<=%bkGn@|Kng4Olz7*{CkV84X470RkA*mx=MhpIVcsODd2^a;u~
zHy;rH50vHO-#UZhF#94GA3^8R`R`?@w6|A+>Vv9!S-UGSx3L|qs^)pETgPZ!luaKd
zPf&IoWt81OU)@)yY((|b6<0x->C_O~`ylIQR%NY}h5j-;=Mh4Iy-XQ%y5jP)l*JuT
zvK~`5wS6G$-&2;4e<YV47iKKA)pYrc5p6^P>MfprbXBm{k+Gth=e4%AKSiDTdh%Ma
zW1)-|c8mc1*Ut#7m*wrXDXVbK!xdjaS*g`m7!FYotER@+`+3>XSL9(IRoYu6q~bf|
z+wox2rYHa>I?s?!t^J#%?=1(04e{^ZUs3H^XR?hHs2oKTVCPWZIoT9IB>@7v!o5r;
z`V^i!R=4s;b%I<RqtV4-DVJTM>;QLAK7JM7WbW=czNuu@z}c3cmofX&buZc|ZzpA1
z?O-_rd0waemg>4Ke@%J)S`Cv3p08uNBTql<4?4y1>;t+`-)@RTR!SeL<DdhS{TfzK
zhSWVR^5Ig|tZ!(|?|yt$p=j)5+|r@qmp(XAr==NCXWJNSvVpQ`TN~+e;8`Trz4qRU
zD%{!@6)ErmV+UHFSk7kyT#?qyPV#XmWKa@m?FOdrF}8Z(G(Te;t;(vhgb;j^kH3Rw
z$Z+sKEWG=XAQUaW(UP?T(`K$*HP5SurLBvy@q5dXZ<6?t>=4~NbDZO;ychHx?XxNp
z$shyP6)q?%-eY{js>NS%7FJMJP*#$Zeg95g(F*(iefV2bR#4Ur9(HQ)$SYc5?<V?h
zOIbnLG^0#a|N4`>niclvU&ilCS)rOy=D9oWU+-RN(^%N|cYm{E_b3p;Kp2K$wwkR>
z;Om4mgJdCCYq1vyUfRE$OFJv=L>T8ez}LMn%;fl|xiYJ4?6Nr2ef60hVb+mZ6jx@I
zefP>1Dd0qZdWK1msQY5H%Bm$w0M`Tg3yr@+^X^g4J0KLvI)99|dOR7kl)&7lfzGvb
zhxXm0!~lXT5<AjcnN>D#?A@Ub!ypjGaZ!?HQy}J=VoqF>sfy*ARFX8Q?F&P=?YVYj
znmG*cdw|dVgFu{9ff&Kqhjg}0S=Us;)$*vY)<|Q3te^E>R_d$iw$d1XrNKv-jNO#5
zw>@e=csxm~t?J5pW$$5@%rr5g_Z~71DB);(6cF)b(m~yC6<5|*(_{9Uqf4C;an{Nu
ztZk1Hk=D|fi*wbL^~zFa!<c@~RQO{FB`j@^1|`1i+o1}?Fbu}=V}T1a>IP>sn4T`w
z_q`)V-fIojovUf`dw{<QpG-|RBm7rhSyXLi3K{uxr<Cf*UT4^SN2`4zF=hZgR$f_D
z`^*#|XGG+Yjc3?=M>&I06<K^`Q6&=52HspS!{s{)YD9>>ztvY3RkERpz9~9G@f}q%
z?EvTLU}aIIPKJuiP<%(F?W=oOHKU!>oi&Q@56n_1FiL@G${xW4D8RoUpeleOjVBOe
zyy3zdU%)rnZ^fUZ?Q<R4QgIBpIi%~tF=S@tW6k%Q7RrB}6=fQ^D&wjx#Pl?zZm@k<
zYG9+TXo4;o`?^}LL3o{OtERXp0mvEb{$@QM92xxR9V?|%S}FoLCd`s7(F<0{u6Um=
zXCgwLUc#fSLPS~kDr$T;*d|0JWrJewADf8sN(ho%6)#t3`H#L#=OY|buLZPZML7^u
z39VAasZ`y+W5$=Jg|RSHMY$kP!OS8>6y?|w#=!oo>k=<tYs!YzYROcQH)?6K9aiUF
z@#|`}dbx!%J5vIR#-6TI%QKL5SKqr<l~1zcq>UPD#x6$q9LI^Lh%tq)196@0intt8
zjdF!He+X8{JdBgISO*bpS#g?0!lMjl^e+PG9J}e3O50rz__d7<GBN-{$~DBsD&@2x
zz$&4gvlh{BV8Q#b+a4bHG&|E~xacl-&Kkp4l~Dw!r+RNp$KT}gB~vzh0Mx|X)+#_p
zdk?D{cm_aD#dfL2o~{!epPGk2nW+K=Dxlv0ob~ufy)Rv?8w%!zF2~fq08Z&VSRuKl
z{R__G4}kGCLZs}R9{zvIepBh^T(?pdR<2<-R-+s$Whl9nz3?aiG({BX_mOshCvgK`
z{SifAYiB^?Vh_py9E&VU6ayY98zN<X`J^XVx!Pc68i;00mK?e-PS%ta$}H89Chp=o
zL4z*3tQ5_2F?vFZh&})eiH4@I-hpggh|2?aR$lO2%@jw`Ev~~ycyB-Hnrp*gJUF`o
zDgwUR_W;IiIVRf%?DY!CbzmS+$2bg~vNKa_-MW+wM1V;Ajv1;V2ic;dIx}m#Q5NhE
z{7@f=$i${l4eCBK^PddoIsojJYnY9dF#t9gvBU{5zvis|@uf)Rh4V&SDm+JJz@W^J
zl$9HB<#g7Eb}I*8H4&o-4rv0k%llE*y#(Ua0TxsXnKB&H#c@ms{h)2w_{}Dcu9dL(
zLfNVDT2Qr;H1Th)o`I65>$I&Jx3T9j-mUS2;>AJMae==8V=LQh9YJYpf8uQ|QH_b*
z51spB*zj6L<UU7N<xBXoEC7A$drYu%{2^E&xi(U5Gu4EvL=It7*3&ZCJiCETN3kNM
z;i{jAO+-@;(tZ}{#Z<(1qb%4TKUC$xD86G;sCLRI+_XgFph+hHcFQ%)#>yuEW^|rX
zWgsK|^vW$|U8K&>e<xq%!e7fkl>bbbaoIP@t6ONl-1DFnqo#gC5uiVMIN`ayD6>Tk
zKsj+7Z-+>k#u&Q=eo+UwG=Ty+ICKDLXSn)8S&rL)nUN;`?0OF0;5xawcgX6WhmYS)
zNB2PK0Z>h#+X5VF0{zSF-4J6#thzxDMz1(lk}3Nh(@3NBO3BsI^b?(Gp82oPDYM3a
zVh*4SJ2(nXC<{#MQjq?;J{6VSC=2!{2-V1J3ANP)a`YI$vs13&QKsCwZY&2hnF6i?
zRZ6dm<T@}DQsMpu$oQEu&O3#MSO9>P^6;t;;7e`IJ<1j)(H-`ptj3}MN(s_lDH0lG
zUB___SCq9Azz5v|m6TxAlqhQxV~o1e#GhSH1IkhWV!Ty!@RhQvf7Cr*a;6zRWeYk7
zUPs$pu`t`fhHKdqdjr53YX+2k#?)u*2*6{u$VwZpWmX@+_!7WigwQEV4gg$Q&^i3K
z2n^jIwU!j5PZ?U*yHOVGPY|k+*%+!%*^sUPvj||fT*EuQHLYv=pUXOQ>fcy!{j<np
z(iqbyE)~8LWjYY`>dXj_jbCili60^cm$G?z{LjQt_oA#jT4l1hB`4fFMv1aCYBaOK
zzEPHPmO7Ru{?&C&8BWjcSY`ud<?;tqiL&xGSGzp2f$yKofJZ(6fMt2{;B(n$OdpPO
z15c*9@pD=J0;|n!y@W=Y@)30@i_4S^l0IeGWu;X)NT0GqemBa3{Ru)fGMhq`D0>00
zTdv_9-z6IeDEqa6vWm0RTq-=D_&a4O74U*R*UsUpquKCu?Oh49c8wOmOpESA*+m;G
zz3_Rcltr5Io)=BvK$9uc#XA7(OW9iu$Z((`QwBww0o~mwo0<nG#8%&hWk1KmR$oE-
zQTmQij;WpZ_AXc<xmNfgFlqoc8-_;NTzdm50<fOH+%eYg0ifXL_MI{dxDuINC>y>9
z`xAs}WVVDVQ?^^KAs^oXWx*ZenkesVlnaMy0(1T|WsQ5_P*mUu9ro}LP?PX=yrecl
zvzrdZfj@M(4`qJuEQU^#We^f&QSZrCzK$E3VSdz#;)FVwX>Xl`mcfFWK+*7eHO{SU
z+PaY^Mk;v9;rvW?#XBE(YdnK_J{bk3iC%V`aQFUp;UC8Z%%@8mvmPl5<JY{MisD}y
z!oEvH16$cu*UqceyI1eI&bW+VR4n79S~Z)&^RjNXD<0&SDr|N#U{0_?PQb8YIz5Hb
zU~AWU>gm=z0(!Li_Css*GW523Ko{wCCvuSJvst#mZV(}7o$AKjDv%TGPY`PB{%O7`
z)M%<bM}-zRG`foHx;@vBDLZ7kOZqV~;y<xbOIa7GinDC)1ybP=%Jq&|{%UgvAWJpi
zL{qazJbqx-tJNNT0C#-=ZfXGR8dVM#Zm%ZC*n={gh5+ECn4u!%-+t5`BFgoNj2;oq
z?1nIF1}hPf-X&uGg0jY^5Awtp6GaoR7vEfos0#p|L4!yDv1cysOF${fCU9dX`p30F
z+GLF!W$uW0fru*iL}p+sXPhKguYhi_@gsuqm^?MV{PrXLJ*JGwTD4$>><VYZl4ssd
zFT$hj_>S}GJAj$#m7^zKgt%7VYH=yp{fQhT`p=dyhUgG!Pr=*Ywp-3>e?CK<aDKBX
zRD;OUiKxVgrjqOSTtjLsPFfKF4v0vt5YdT!&Ds6qmm-%hXsr&U!l81WYf)!&2Q-N2
z8R#TgB8xZj^Z}If05CB9t0GYoy^-LJ#sAo~LnHtoD2k#ZX?CH=2=>1)Q5KBwj|JQj
z-f90vS%f=A)ir+nHBgi?)!f^p>}8ZP)sj-Bj51YK%B(?^GRjmbqfAwmGHXz!j51YK
z%FeR}RmwK|bVvXI0Kz~JOaD&f!ma>{TYy))R5o?7R#p8SEy{ALlu@RtN|`sPQbw67
zWt6F^Qsxb+lu@Qi8D*-flx=Hgx0Af`h4E*R9$3;cNL!`>Ctw1UQU?lP_<a2eMzdo)
z@xT*rz&r7>BoMFbzuVHl{5QJYq$_XIlks_<K5PAkX(iF6sG2^-HO|KOGEt=f2Wv&=
zsFHWAY9|uNFk`)Z3an0q2#QE^XeL)ArM(H!*8qH&Oy;mH+hey4MCH?5QQhSVZOFxC
zJHvS>6VVxxn!6mos&^vi`&@~rpXuyoJz44qxU!6mpmJtTm#<PZ9iTB2-cSQXWVi(@
zx=2Lz#}KjwJW*6%M^M(S)g1ZiyR}*^WPV1zSA6)l-s~I7mX-s|y(3kFPAs6-<@3>A
zXGA5S?+rqEf;5L_HS#|~QbR8scw-=R$J6r$k{Ym9?z$P@kEu|$pDQeJ?#oVwl%Yj6
zq-AmZuEHNELt6|r94OB^(rea}rH+7OyIp`~N7*YsKO0F6hVkctzoJO?{e7W@YVZEu
zl1V^s(+c_caPRN$-6JULZh%)K-+low8fN7i+yY?N@qN|3BUR6Bq=kG2l)<4AG`9>a
z+)0{4r~|%8ff8k)B7o)%cGbB8beaONJ*iT*%N526P-e>vDZ8i5f2XWO&d<4KN?G=k
zrCPmeW=ff?15=td$Oo!D)Tjn*R=5g)WjnqVrUAVw${v7r{;21&hDc5z?_M!m$^eJR
zHd*sDU@`=>x@58ohv<Aq+0<a>LRt75)pG=NU1g5e=c4Ju+<uUEiMmqECcwpw5O-}{
z(sf_(%7YYv_7i0e{Q*wtVn2sMit{Ix0H4|`H|=cN8@OHIQ!Et9lw5oL$ki%q%39Jb
zgb`&!!Rv@?bIH8=S`D&C!84N&zoRVe50X$fL?m%2RFA47CVh)@`>@ySCrh<zc>o(X
zSx*P@L(ZPR_DrtFoCh1lc!dnWBkQ~U;1qBuw_dG*v4l^nHI%JkxdPC9#A^@hW8Z#M
zXUVl!LDrPvldc;%A?&|SJPjpwu3N5y_8w(LFPPR{M_Kr9s<FVjPs31q<>8;v2$R`p
z6W{-UW}j0&21c`sQX#<l(S6yRxJE4}DnC)Sp+zJ+*vsJ~H4^P}z!Oa!8XDg06G_D*
zCTt++K%q>@)#CbJ#ta!#R&whXeb#_QL2jP7MyHoM7cDi&<w%6>6S4UnWodtsP}dti
zpWslan#re)p15q6Nk6EX{bZ?D`#pep@4ndtEb&rm_SQR-HvcG@foeQ30F7Heq&oJN
z0ML+gytsyImH@N2eH!0}6QVCAyKNjrnI@7h{b7>~DRUWPAuyo@aOVM2(oxg_xGk*q
zD64P{ke0IWXVniR0z(NPXaX1Z?!*{RQ*aA78{ky})dHYX8R*P@q6|&rbMr=ny&Rs0
zfJ)hvj8ds!AStn>tR%iv%9LE!9U{5OnzGplSj#aiu>_2}cmZ7iCpT)4xS0V(QT`oe
zX@AmCJ?ooL3ss;(r+}y&RLy#_RIBSSik?+~GFyY!ff|K7liUVcLN)$q0PudGEXTS4
zLPot6Y*Bu=<`eKvXtVkjLYMl$%v`*9qoXJruvL5=HjyD^RbAJMTgo1Lz!!x;*%XY*
z3T2PP7^AAX@MqPOG7~_K7v?(lDXS%0-RX=o*GnjC(|yF_kCZt*as%>rIZRjxz+y}y
zr7Rxg66ra{n3H~%<qI}c%9LD(a^W{e>x?NghX7_BSRBbM0^KM`x1a=>P=?;{ca)|5
zNkjFlgP|srb!nyEx2md6R;Jsx>sGboxoknj<ov<y)FeKWzxlFJjQ<;DZSO7FW(MX7
zHV*JfUFZ|w(cCW3TcU-p2GFJUF_cxe85G9I{KTmCd_XM??Efe;^}p*=UHG%=K4mDj
z?wECeGQ;>n{Xb9!wA&%+yBrp-7N@H_NT16#<VRxRO4oHQeJ*>5w+UCOs%fr8w;SJ5
zHo309RtG2>%7n5fUDs=eC^P!Mqb%)D8mea<3RR(Od{8y($x=63M*~vICI=|1L=%-)
zjQ0%y-=7$@W>Z5_WjDV&sQ|V>TEX@9U~_g1W!I1KWy!i(Q|7nSp3f!Fc2&x3F#&*M
zDeDe^I{znSYqted{}W}vnCEvnL}sDBV}w@+LaOb*V?3KgJ@xvIs+#6Hq4)Uz&nDM=
z#B5n$@5HD9&$n^+liUtawi4EFDC<tr{-mLL)}c^;L0R^brCQyi47?7U(qKoFh&(CA
zlV;|>r|d+wG0i#NX_W4IJ_9!Nvf??QY63v32jJ=%BO7=>>a#5Mt<w{LOn+b2e@25^
z$Oi=tI=!Rib{B?gp-nAd&hr4U;hm>&)=(GvmFV9`OI5{$kK><-1@e8l0)k%YY-B{e
z596DEplKo_Z~VF7ll+^xLXEmu-u1LpR#n9X4S;r6FSGhy4tF<53qvmqy>UcZx752v
zdLAn0u8l;<3wF`#O0G>ae7fPAjB<T8{ZIx#iLA5u?T33+vM3qik@QQ|b2UiZu-4fX
zYrrRcm+HZ8O-WDtlZINj_49{9U72rrl&OuPS6zRjs!mq+logpBk={UEy!>ev);p80
zqIG7Ep&0k6QVLjqb$9}7_r6xU3pDTMDN1+Ymd|Fqh+gyQMOe`fOx>a0Wo1;ofp0$J
z2+Fi=lLj%P*?#xqStp`OhsY}tQA@iLa^NwCi0mp6LsU7#<tH0;;aFlW5j3Hy5>XWZ
zVu33n0pvzp+&X|G**T)viTp?gp=3dYtV<$3C8F{zkxXUPj4z331l?r(bw3BAkF?@1
zhCFHj>Rim)d*A&Sogykm6zsezxi&7jXy&pSvd>G-g9+j{`DO@M##o5q8d0@pg5BzB
zkhmGmgxAG8k?SkBf1q3Xwm)g8IOn`^C{#`~vx&%IL??$-vz#o|DkpcI?6XAka%6fr
z<m~oz&*XB*-NHK*<Ct@97ie*K0whHhKpzSIIEfl7j3>akjlT*ql)$3Eqy-e;lK%gm
avi|{S`j5n<bh}gl0000<MNUMnLSTYNi9Edk

diff --git a/modules/web-console/frontend/public/images/query-table.png b/modules/web-console/frontend/public/images/query-table.png
index 4d63a682f57c6331d3e493db79c234cd6855a26a..be4da8f6b15486d7b0e4080e25735cac20058d1a 100644
GIT binary patch
literal 19943
zcma&MbzB@l6CetKK(G)X$l?TFd?A9nySp>E1b0s$K|+AVg1Zw~B)GGIT_kvLcMI<D
z`0noA``vqY@9vMOsj2R&uIjF;>6+=UYAUjL*yPwKC@6UHa#9*7C};o*$}@%+Xit>V
zId{q@0*acFmh|J}<N5jd>FMe1?d|pT_2J>+{QUeCH{ba9I1-86+1=|RrxOwqdiU<#
z#>NIJQ1keBxcm6Hvb^;8c(wBIc>Tmb9C7maSY;#r__*dJCL$&*`gqgc_xN}KbDm8x
z5SR3nFfULP6YdXFNJ~qr-g^}HZq?DzG09z4mT(ppS5%1_etbmUG<(MUxf7AleSDmL
zTq<}xY!GvT{k^#B^bx;1A5NZqEU&02T74RGQ9EJ!Znd@f^iEaGaw=Zynp?1=v(sE$
zr*w3Gf1u1nUSjM?rJV{vgLDsN(eK(qkG-K$YU253qVjq!edqV$ik8-;8&lJh_SQB|
z5=ygkGs+GXhCXGhh2{y)S~8yX7gIldx*zmi5@KUx8pDhQG`}@0oa;)6Tyty4hxXo0
z#%qgddKoF)%{<uG>?a$EFYlg8%7)!&>v^=C{@L2=-g;8<=wYk>2Ss2heCFZbUorJq
zxp!~x_wUXx*0PeT-`P}(YK9Iht$7wAZ%#W_=ituJ&hju|X{7c_z`@km<dY)*XkFHQ
zq|VmGg`ia6(ZNa%?RVa{$_n!GzuY83q(wvf?~J2=t5}A72DBaqQ|F6Z$y3(;z{2@N
zJdQR;%s<!9HGPBU2h^TD{u*1#%P*|@k+s#6vWxuEmFR3KEMl)M3#)<sIlS+gI4JGd
zu*@7i>kC_0*-HEQXZG<ivisnsaA6+)m(UBIn%gRA_TAs#KLR$`QJ5Yc9#-a|nb<LN
zt@Pzq^{ttaKO#o!em3Jq<n2x4;dD=B^6(CxLFSEx-R;QmU4Fixkaa~J;<mNbAv|TX
z+5gT((6e^wbNzg_ZGThKuZIGw>%^hVFTN$f``gW3S2s5t?AO;W_1Dr8*P*{}!#`8n
zcJ)upcjvVX^71c$xl^VtW}E_Utrhuj7`a2bu1)hda?;^y!3)1S^VG345jS_^yEh?K
zXNP+?@SEz4g^={3^32d{|Bm|{-@p&p8Ap%lBIPk*rFHKDI^6PW+fzy;<K3h^Ite_7
zQBV}}<fX*5d}jCO6&;Aw$>7KQmJaf%iP8(`pW!VUqd9Pi*wweTQkIoC!~_X?v7~CR
z8PTIh!F^H~T<SIUYV6v3u@y0bTrF}^q*Gk9Vg3u)?KVFKJ|sqI#|#yW&=~kZ?7T!b
z*E7uvFlJ_i2<Wsje`LCaGqhNx(S0gQRO)76j-k^UvMQb!d?hhiq5q@*>E`$bN@rEo
zp2ROm6|7dANM{vkFP}C(`j<8qDoLIjA%mGpXQ-u!5TIbRn?RJnRU*FQl`#U}u!4&m
z!@}vT!tE8(&X^G_ERqe|T+n)_7cSNYY7u`#)x7eVI}N=nL9wvj(_&T@$!%}MuxXsK
z3*H9)?jDdhxtf`q^Mm3uY`yQiJ-BrLaEcO|>(^i5I3rUvNoC$ML~Giu?ROfi+rW%y
z<Bx{P&jzf&fYKQjvhv>j(^$b&f?oaom7zW@aS%gw^c9?Y2`d`Z<GI#RvRfcFpW~nG
zi)f+H6gW|oTfp%|<<R}I^mPxlaLpr1gzrxHZ)WUBKH45ecZXWTMQpmeON;!>*oe)g
z{8+cHDU%oorp^SQOS^E6`{g>)$r=1!uy?gq3M^*j)u~{l{=xrRSym;GMsA75%0M@t
z%CzJ8>qN**53je^KTiH*^<xnvGc!&($Gj*E_R!{j;#>}MS5~&U<OrCYI73BzkG;{?
zUk!1)R{DN==A@bt^5vWJa#}HF+V+o=FEv(2e%Wf)6$xz%1sksOkK^BXT*ZgJo<ikk
zklU-TsaT~5Y5LTCvD#!@jLbBwY5hsgH~rZKWl$Y2*~Ibwt86#76{0zilI&k3>MW~$
z<_N}zkqSEg!*24F^0bEU5@|JWOlJd*Q*&cPzC64MX9B1<GLai6ERl6iwpdJUqZNZ<
zaNi|2%NuiUqW*tvxHGJ^5Rtdt&bSGDDzDB|0&&XX5KA-I2Ty|R)h`T-X+C+pD_46L
zj$h2?u!nd1iH|-Kl=fbie;`}6U#A^@hBgBp|K>_WI*1NY5ktgROMD8-sv<1>1hIT;
z4=7F~mP<t{BDVh&gLat{A#9Kgq?)i*X3#5Mi6T`|l0Ht}c-b~7|6YAK!;)T-SIjqY
zNl+lYU_6*L|B?NO44-Fw*-iD1sV!ea|8bXWI)23;R2B|qzFxHTO;F`Jlm3MBKGVH{
zEzSx)OfvyZ(zTLr^;VtuSvsZ~_4#7#EMMBkNswFU{@-XWGXVU?uY>QHeTKvZm$t;<
z^2%5}vg;Q5bLbY{<}a>hy6`L}SvaNlI?;x5>a+0b@C_V22)t&D<r8EVu`+O3;~T&q
zYe*DP=Xm+N_>1o<4x=Ou@x=~edxDi)2fX(1&?W#e;*z`=@r%@+3Wy{YoF!VeXUVph
z^|U?YcnQg^XoXZM@HV_FF|vmG9Au3hYg5o3#;`7?>mQVPTu|%s;_K_rAVcj+36elI
zkonutUE16VfgQnq)5{O-&%JT-UH_1Q6&r7@#X;nlv1yR&8)6aF2ngRxa$bOsZDzBF
z2<wkeNphB4Nbz3IO+_+vMc%bUq?{4(*<~#ou~z^|c!Hi10T$vF%vkLrJh1t<IoT&v
z79}ggH@LX-EV}*o_?iVOv6F?b31p8FNrTo8P)W5gK1l+d&y!GB)Ii*ZKXE&?+F<DO
zp@qeHkohO-%OrJBST3Y97Xp^#PL8k}y2Wk+C(%8lh}(DU(_uN#0!yN-B~3PzBAzwZ
zD*!LdT49cF?4A=KwIWd9Y;)RU<axI)No_P0f-ljSZghbJop{!yB#~*HN0a%{>xNmb
zyT%JsY$C6c{l4}Yc*E<@u6J*A_u+~y4b5rOmA!_ow`Sy!18y$fRnOSJ3~#ico?p}v
z5<Lg6q)JjsQ}h&Ne=w2XE=&HxPiEAD;opaS(z3p#Wz?79$Z|hAY;%fZa{w`39v$+J
zGzo9XHJr|!W^64UL$UgojnqVcV|e#{#_!e7|JcZI-*e-_lK9*}HDh2P&1imACt97U
zrTt!PMlu8DuU^|EXv=VJUBF3nPbUYlw{%7~kNf4Fejtv229lODUlaX+E+e)=B^dI6
z(jfTk7v=JE;cBmxc#nYkma9^#Ts7N%ZK45eW~|7}wugtG);kI(-K^uzh|E@B{CHaZ
zDQI00yPyO2&vONJM1X3L*bMTgv5AZSYnEEsR$7Ie!3Xji>UDJUPj)5Ka}jW#Mq^bP
z7oU&)qh<8IZQ=E1DC{feG6&8H4}UhC9?1*Nyih9eKq^$a*w|U0ioUacP1qN;;z3vw
zJ5}ptQ<@|_0}Sx;-NIE%t0&77Xr&F`ZjrK5@Jr-4?z0@nxDMj8+**4j%87yYdez_&
z<d|j>Zk9$hw?-Xrer*o&&`)C`!e0MESBlPgJ1W50CQQz-ByGv+07>wWP5e4R`dLDe
zgS++g4?=BQSY8yb2k$Byk6AJo=EnCT<8~0Xr({>-#E>4-$7-mDLGiHNL{+M$1MK2!
z9QaqtKnL!cMa4nXNQI374GgO<noI$-YBKQw9lRck{jQ{{patu1w4JHfe*WSq^@5!m
zGJP7>AFAtU|6ozc+LTOKIWf6Jj^JYl@Q6Pu6js&_NO^zC8h3X<sQZxzC6V09z}jlx
zfGjVDufMRSe=GS56}$(pn?yu6G0INBD)4^0N7Ho4rr@Ycu(tH@Y3f#NvP4^M#A!0!
z{&nXR99?Ca98lN!H{%#56N4f-!o$8|L}ETl$U>Bpx0UtK>Vd5N8xt=$Tf>1<Hc2@;
zXBzwk<NUZLrX77^-8hsW-(eF~TX{K@##svQPeVAv`v?j<GOSElVQ47Jlv;wado(UJ
z5+$8VxDlxQrA4azsoKkPxxRY=R*~@AAuGc~I)>A(`j9;-&g75T0-p+NZPFUF24vZ;
znSacR6=}LwPJfbFukv0}vR-o>w`cic$Us|_)Y@7Mht8H}-m5f6Jew(rr^?ChB13~!
zM8Kp$g*=G?c<$kU#`dZ!3yG2Nw+S~tCeAwTwU#KQ%*D5loRT?yb1mj2Vrk_=`%}NQ
zMa2LTThQBQK*%e(IS|;+m9BFUQZh=_Bj;x5^Xns%V)W8j!hI8Qkj4ifpz4a?UEm`6
z`L=PnnaiRL1fCXMG^F<~o;F#vnWxB+M?JKWwwFbd>Z}Fx74fhEcv*8(FfdsYUCtE9
zbvi!@uwm3bnD6nd|6v_BSxOJPAJRG>5}tYRL+r92^~4bTT<-ya3uEH>CVmG<u8jar
z##yN`qZ>Y!6TfvuAxcV0mbz$&#oCunbFZsL@4rB>>VgMOM#Q8oL5SDx7$Z7dl+0Tr
zfZ~)&dY=|3Ybae?>u%t@XE%~U9S+1q8J^p}u-f=oLqG_`-<k}opJ~!%@7b5{rZS8#
zr=!i3QN-$ehzU&RS%os{-qW&fX47Jq2mNuK#3vOs{E$MoF$rZ>lX+9(^#YVH25KAb
zQQpF1yx!S$uj+5whdJwv`ZC7rMrmFM@$!!6GhJX9{K`^|0c=$-qH}S-kJ27P55liL
z7gP(ZG`wu?Zo&<jsLcSM{n(s#TfMZ2Icsk1si*d|VC=ZXD{F4CV7o3kKeLKKBVf`l
zNf>brJAN)Kl$>j3O$fez2FN`FB0lN^@U0=HEZ7`JefUXtY{0o>4z8o7BX?r^*iCet
zy-%#`j${%1LqM01gjWM}i0o(EGD~`mqlo0d+Yf+nDxV3v<!crPTJc>eL>+kwcrH$J
zQ<4H-0~tQ~P}aPZ%`;5(tQi81hfO(OU_lJ;tI<F)MEj*d9-yY;F{})=`Og9D$u+16
z{$MIq5V0gkbKqx_{W@z)J@JjKTy6rFnis_++3|FZ+<>ZNHk$I8@61CpqjwxYIg(jv
zT}I>J?G7cNP$Y*q$xc|pb*kpve25M2UlrpTSR;P}G1nl4HdU4NTTi2u{J{_d57}>o
zzbAO%?LmgAtEJs0BKwp3(6>wz{akN4@nlZMhvH@^-y8)w{*`3mBvuhU@yH;|9Emb5
z@kLlo<G4prXw7(IyK`=<F`8yT4BOPTS1K5&p1Var+HY18Dawq}f?~hzxj{E>_hLQ#
zsmfwM<d8kXUwjcM%~(sGI7oK;4T8M~4@)8Y`|&s*U4c8GPujcHqUiBHyv?eQV}w#?
zK-D-8N?^c@4v8d+I<e1!=!dW_9O2nI8{9cpA=Tl~f9a|O#DBC9O`mC33-2N(e`?B@
ztBP^weMs4Jcj_kpkatyK<)fNDA$P))J^gsD(jGHH#x^O%#gh2Ys;R*RH3{y!o@R|s
zl1eK$OO#4ukWXbmFd*7%y{$WVq+KaH8otJFS|2JEkMnrgvzHp!n?@<{6ej0QO)WxY
z<T=E{@+v>k*2t1Wd)p&075|BK<Iq_Vnh0NEG6=~t1Z|z&?Rq)e>?2oh<@vmaZ&nli
zkD9iExc<asf3F&!z0$06#z%%=+%*D4*T2U+m57p2@vcCn#nx<Vs(%pd&N%;ROu<^L
z#%0*xs@vy!o*dfoTn{4zO%DSZV%-Trp}gZnwjSyYI*}`1eapsCE>|whcO@q9{m!(+
zV>$MwPCv!#AH@pR^tMg*;4t-U9|d^UUL86YAX&}l^1((#z;6^aenA)BKCsd^6|8RQ
zvpe!R%6VM;>i8c#-W>ENN-DW&fpgrWX#wMueXHTRI&U6Q#a<qGU;WX`z7_DbRIY8W
zKsf6+*-?wO%scA`Y|>xa3vmmg3wQ=BI*;^pn*@#A?|LrYY`gW47JbaV*#<fYRCdg}
zAUQM#gz{73V+Cn8rcI`Dqiv>uuUM<I4zWwx)yAH{ADI4>Vlw)Evg@X00N2}8h}&UA
zn96PWb_hm(>8JIH_ZgYvq{*oFwlfgjcDpQ7pRHEA&6dBgP?CF3{WF2$gkih1iTox2
z17uZO!FtNmp3(zb6ohTlUY3c{Yi3+Ud{13Tk8UpLhU8Fj7?QZR43E$bx6Jcsdo(+C
z2`FZM|8k4QQ4DC!Gaeef`}wcwJ}TZFM%ng5|DD_Fiw+m(i<;;MnYC>f(J|in!lIA-
z>}a#2FrX*O*!vTJW8Bx%=x8F(QvB|16R?&yo?aRBsupvGV{K4YmaFN80IZvPX^io8
zBN@AfGmjldg&?_bxH1bH8R8re3n)1U)=enbru|{K%>jN@Ozc&%veqYLA{%uL7U`pM
z4aJkyDHo2nUr2sDQ_|O|b=Tkhjv%PU-G)z_@c6YR8xc|tUjo9bM@TuM#90=$n+mN<
zpdfkZsYd*LWioVxWU1xUwPucZ|C(<h1q;cNcPPw`_Tx$SKAsfCil9cL8jG9Khm>f_
zquhJ2|20t+5B81KSZJJ(-N1hp@oh2=?rsf74D%(CB?Vz4!*=|i93>{1!y!HUD53Xc
zvfcZ5Vt4l<0V|w5J4M?=UOTQ)r}q|ObuyH=7y;Xg2r$vngf)g#J(HH~cooS@LzR4d
z3bE6d@f7F2|0BR9`~l}nn$6cOU<|S*4*HL7oZ_xQgT_Yt-!|S)`hU}va?7ep_7{~H
z9|kzp=ia_E^fE<v*h5G;((B7i){RJ3+FE@h{Ey#c0QS{?0$@r4?<e}5)Y6rGj%4$J
ztg32UTbUNAjRVV`h`w3oG80kYnD#+KDW4kter1N)|J-fLjE4Sd%Pf|%&PlfJEn&RB
zI@Q&M&)4$gW2e@0>>tlDdm#Vf!i29;K7;^I!Z6s;I#Jof0Wn@P5Xwul|EP?LMEQt<
zGz0y?{U2hr6QYN~hn9@GjFS8m0HA7I6wGSeWYnQAzbYmr#DD37cZ(Dv>Hv_b)kpk}
z<L3~o#v~$@$xvXKhH(+1iy6MX{NRc#p<qYLVMH1rHAvafmUZ<vlalK%;QE{VL`kMp
zhK7W!esqDW3G>qN83u}aMPTjN7gcX6RkT07j3vI8h}_*SuFU_^?MLu^Nt(@1N^_F>
zC6$3idZ=N^%O_J5TLJ5Qz&Zl}`Ci&9u!#AeWN@4#7uD!ee3}=9n%cM(_25bF#g*kv
zf-AE(2!cu$CeOVJ=}FD-ok?aI5(Q+VpO|@Nd62#-z0-`&QCil5pl5~2ufP*A<hyYD
zF5V}r*<K<I>TUIXTtjOvz9!YUfeUY=UO97uM2onnyleU)+B5!cBp^43*Pe-{OuQH#
zLfk=vsNXI&?NwVsMqO4tmdxx4D)tmL2N=xfaLb;Lpd&-7dJU(%l2cfmDQsp!y7RnA
zSvy3e)l@zkL0B0SNJzf}Wp15H5CzYtK$FfblNVO~7YNez-&Wv(CP**2t=vMcv2e0-
zOF#dhxCF#Z`(eEvvm?nM2kLfqLEBi9R_Q_(w}FD+aZ+Ei*)@}fIM|ZRV-q2=eTH7Z
z%&Az%R~4iWhI5zunh{1FV<~XPProAl8@zO^h!N;q=r6<c+rwS{u@=)OR$c3aapk)*
zDfv&b7CAxcbvzqnqd@BwxqimRo`F56D}vDxG$K|)B1FWF)|@a1A*7EK5m)!E?R;TV
zMO02E={2~hahedr)7V@4^JaZCVy_!g1w~T4#|iss{quw?cFJVo!Cb8aF(yvW{In83
zB7*b?2NKF@@O*n_Apd=?7#d<+V%;{4xMP;nSStIGF7o|L*@{!sNdTAL8}>B?6JJrP
zHecn6b=CaO5Al(b(p>44>zJg{QHRPPI-QeRJIaQ^r?t$r?hXX~A=$S?|3C2k?|>-{
z!?GaZf-tzM`7t#YOTcf;AGGw09qo~2$!g}Rwq6%a1=*X0r-#w0TH{#y_Y4P#a`iRY
z^?OJZ);5uX{LV`8T83q5L;z6XU>CxUQ(REM{wor%;wO=^(2o1=8|mrNMiY|a^E)5T
z+vU-OemeU+nRv)ll(mtS3*7f9$ZVB0kRRm4`p~bc*4wbq$8B74Up#98oFEX{YbVy{
zT^<UoCEFYF*<LMIhliyy@H+puNrgHFR>OM{a-#<aM2Z~LpO^oew54zyNNC5Wf+zJ*
zexvuZ!GDp=OwxBI9~0+u6y|iuu{J_%QW`A^EU%fE-m31+WD;T^?%()?$gbchl7Eig
zANp{Fl8l-uZx6!6`lu9A=eW2c#{%T^K>QWNnY390;UoKkIhNSRK8=55TgjD&WRF=I
z{<|UDO8+q=#PV4`1hQiWp4q4#Ox9Sg-f#NCXj_CIXW;;szOc^lMnQ%QGt|d~qkmu$
z(@U0l6uMPFTmAb!l_T9=mG!%B%&GIf#?4u}9O9K%pwH`c;Ax(UPF{XA#i(N{1t=Lh
zc~GsAeXKXDqH%E@=YlI5a~*T)*g3ZPI1Lq0D}x+oM4AHf$08vw4t>3yo&Ga!ZOoHl
zT8pUcXa<QBke7tZsVFd|{m1?pzibIP)l8&`+O!WRvMt>hP-P|4kli~ft%zH(`w|g#
zcztm5kL4Y*v42r%gQ^pf^Y(E=L<S`R?Z2f@ks3oxR@`ne6lt~>^{<68Qbq(;#hVoj
z*JOIdFct;HAd*)sm|#tihXkOe4*HZ2u`SWU;M*|7z<eO1BTEvO23~zb<m5qK$o{wX
zk`mnR8QtWDyLRQpv85XI`v$VD9nUVlxHvC_)zA?Ksb~FTsjq-foiV`rbA+OjKQJz~
z+>wX#^k1V{YDz>}(!h@;L(W~hNB2{Yf3`Fi!@@<Sf>CSU=7w92@(!P?Lhxx>w4lZ8
zY>po0pN`ecTu=BueBb(1DHRzllw`)JU!;i}!=I=O-B*Tm!)dV%Q&l3Zxs}CgnX!Tg
zl-@n+!5o*CjxLs+koEoc6<Ev>Q%h}^w}Ku1Ba!}o&OJ4```)DE|MrgZ_QP>!_Q68J
zpP$d%#d*|-z>P)X{VPr1>@Uu~gWl(Tthu*H3anN(>a)b>xFTJ%>~7=p@fz7G{5tP&
ze|r3zgYyR-ph(QwP^dG5IoGSrPC#OHDmrm~qG%)>pI&KNwl}pJ4du{ikehh}TrQEY
z8WuRlL;U{>yd+G?tzdRhpsUG)w72<$Liv+CYn1n7GY`gEf4})P;K6&Y0_`hL_56Fe
zwbTfYUaWF7rt|L7lho;nM-PpuSC1WIufT`pw>Gu$byhd2TGoc5^YHGS?+*H;2r(0-
zrEu_@dr}97XFX!w#vn~jvGqi0YS8pFQXbt%Ls5pncXbW!xoody6YN^b2!CsTZh$VS
zB9w%~1U%KEB<&d;$Mn~~&1i07>yLjtek|TMC_%E!nQZOsT*Fd?0M{jsl){S3B-uqy
zF?U+nImon)ef~KR+PhA9g2xKpn5}<XHmkdp=NMURV>x`9jjiHzIkMmuDr*3qB)=fn
zL;82Z3#^1^O<?9((9g$IwMmz$kn_+KM_tcmL%pS%?jaT;^tH;|xepb^zi^6`OfI_e
z_I<JO`4MI?XwBafK`qAbxsm0|Zx9w>Xy&q|cj)f5?W`H|7iSYr`Mf$82;xoPiLWBh
z6iE7>sd5PfI86K2aE>}0@c?1@O^iUJ8Hpez1~s*0K$R}vr4Jd8#gC@4$}A0BTUi2l
z5ZZpPWj4jHI72^gnDm<epi{UPjJ<LKU%~3y@So7i`|yQG+y4Gn*~_wFkTF8<O(H|<
z1$Yt88=ye^$1Cowq{1=qp_qPxYW1H#VGYZf+gFLS8W%>T;QfSe#+Dr%fwD=mCO^_)
ze<T|lkCI?;(a?&*4)6Z#uh&zkE18<@=^X?m#T`D6bEJd?&cA)0oru&q6guB;M>8jl
z=v0nl1wB@&VU|I*ig0;#yW|VJ1;lRyE%a=Hx%r<-?&Y9J+VCOC=v$!GdOuzmCvD|b
zG_JFvX5K9JxnK)$471LiYz%uWQ=iZeOTGQU;K?}+pDxP(srTv<#>yO$sk}BIk+ry)
zMBb3u%av@8YkKhpe}D!Pv3g0(#W%6XVi|KO3VP+W(`2<&VVleYGwLu>qfxXPo3;EL
z5iN2k21>|gWqnEJLe|}pm+neG-sq}a;u3q-i)-M@y<+kX-C$Ox3M9M1nwX*;bJP$6
z^_?34GewT5C;-O|Zg59+CEGa}Yaw`V?nJ!9rg~r9^;G}Io>wgNm8g!_NJpO7NEL8_
zGd`LCnceqIlp+wRhMj;f;?+duPPxJOq^<y=374(14OIH3wGYCqlBRl4b<g?&Ul#E(
zO)~5i;=JN(rthS~j!6a4qxhcKSn>_Ptc$5$8x$0iyTiO7J7N@6V`kIL|K~ABY~d#R
zRt)@eY2vu5lch%(qjpeUe=NBten5GCV70r5R0RV)ksg;S3k42A)W!MxQRCdkGDY30
z@R9=CdIP~95wKsu0-}hUv6q(0O&^GEm>Z^!V|Y0}5OU2UbuyHFYRfL9ELEUIxr}?o
z7ga3EN(MT{{sS)Sba(N&gCSm_5qP!GHuXn|4)fD0@8&QNK+Pr?)~c7?36Ell_3n)1
zdR;Of4iQg(|G}_``z81Xk~5X##ishc+my#R1?s~A%StCN{5XPcUxZ0Dt(9Fg=#BN5
z3JEAbcE54y75vR>qLpy$M%qPOj;Opn%_4J`*7tE?kn*Wl-+JmSg)A9<MUH5|<;ZJ)
ztno;)TNdoBm(y&7)mRwn^2eEc@+8hzy8oi`LKV3VHUHEmpd)G)F@nwk5vBA>1!7)I
z<jkOwG{J3J(%3h=%l}5Qp3<A416U2|PQQW@@Xu#jj0%*3pA(Hpr%|I0W-1N5es=%X
z2d~JtFV5LSViAi=4c?Od+fSa!4Frfpu6b{Fvz3XZ{lkQ!O>ciXlKgfdlb9kzaKDz*
zd8oIi8*O9#<j)LgHS2U=`86<@LrB%(4-}!%1q)<_XaKuq?{n<U%H{W~j-RRBND+Ed
z(f~F|a>bh~(k1AGBc3D{@IsphE%_<Kc!juN|Ex=V4ru7V#DeqFEi1f!XWA$t^;Qvw
zrPdzL!sf<E#(C|f0yG}O?uXx-t1ZY6XBC9~-k(Pb^=0F5oW8YZqX<_2EvQBzHVG^8
z_lmpPl;N+6nvQe3dnU%gUkU}dm+#2AFKmqF3tJW{vSCW}28!UGcOMZYh8^@ku1MKS
zegw3779CaP<^~*oK7SOotS@VJL`r?#I%i-x(Xm6r!5dG6P$`$X)rf%It$r_#_AaW`
z-k`$!r!BLC@=8Yr%JcKnZs6y3#1wJBj-Fd<!pzGPwJgXwsF6$xbn9ZJE5dMLUjYp@
zC*BX}F7iBU8id<<_mk-t$SliVy=KEB8Isq}C~#~%bGFL0H2?i>L>@FCpaFMUpPx)s
zXRBwotg}yJfXdG&=i~m&fc!Xos$%2p|EaXfE<X#0#o?xStk|7E33wXMciwNnOzwZo
zK}SQp9<z)6TBul}O*2Cbqvg_+pL_t*u%9e#z$&H*DH4?CYy6B^Q%tu)4m5-u^r2<m
zPmr5Yc*VdQUpoR6vzh=b<GNazGb_YL9YT=n$_)v#s+ZC41DKlDbb9kB&dT>qNREE4
zKy#~*lyvx1ENJX8RoCjt10;6>Q>sw;46o>2%&=3+Zg&IWs7s4x9S`TaYxDj0Dj>_j
z9HrRNQg&?GZ?HE2gF^}rA%Iq5D1n==Ww~y^oy9Y(M+U5WjSgw~Bdp*0zu^b?7BX3)
z?RJEq^870N`M6lvRgLb5?<KbS-S8Ne{LofaK6U_BjN3#QZ66Pj{}mLui>VYSQ+3ld
znVJ|GR!`yX3zs|XJ~Pc>3Yp|VBqY#@6Y*E_h*%n03JtAkCqw9N<#sSop)2pcthsqE
z+bucVkx1JlE@i?Db!jzU^TH>^HSi0YGk$u`r-Qekg@)dn4;pY5*PC{olbx>QQf9Xo
zsQri+^lK%esp?NDfh1`oc5wf=$HYmq^&7XF`I95~>TAZ0YVf-wRY+vD=lb&|)cMh#
z6+&z1YO2(I+hWya3@UEA6dC=HubKjfY+1=k9a-pyP*4A^f06|&Ge5TOyT)BjojF2T
zt1=$TYC7n@s5d|=j`k=wgjf{WNTZ0-MHU~u&)5OJW#h456>aPCf(SjpVmee~UAx>s
z)waY?w2bB=>CRSX;Aed0dQW2_YFA_N<moOZe|Onln`bO>6S*3EaW#!rnL+bj%`lpP
zht<$2J0;Wl7_6$VL7JeZmOL?8XJsk|<%IC_GI2~r`=^E0HVLo@0ecB+Un|+(B&y%D
zNS)^c4#sP7s_~H*4_1+r!L1z@{>{9I)*eop-r+#xOTL|{$Utim9Zl!8v%p~R&@Wjf
z`d0+sOO$<5PmA;h7SPaHFw{h8E+q#Kzb?Q1=$?)Xs9327PU{%%1L6vUv;zu=i?JBj
zn=-S&I@<mAzNhqHj>O|=@qV|YX4vm!vd&eT8^fne<X`98Q(A|8z5Dw*xEVoUB4`qv
zcUgj6=q;1hp2L)nf89=mcs1<cQ4x|gTF|7N<|!6E;#HzAXcaRl8yI(2y;1#2_XmO0
z!p#BrTEYm&A6#jrTDZYFPKxgMV6Vx}Wos%0#8HOk8V%de=dgd(J-M{+{kjgmW;vTC
z>pco$*?L@v3%DX=<3`l|ChXZAxuU_Hr{=IO`3vLtosv1!pczqd@!k9Lr3$DacZ#Jq
zcQboZnR@m%eB=ih?VVu$;NLj>oZYprV6TX{{Df?cLzSiKH|~A!E2xF&5mwYREhCp?
zkCsOUL6%Xg|A@cp@$o>v=~_pX_-V1yX|YkC7vZfbf2!-HC)U7OB_`~KyRWD-XoO))
zykZA;g-QL@Gt|36tN46$w|VL5T-?T+Noai)n7;Y+?=<5m!@oU^x7fSg#r_Vfygk#&
zdn~VV=A6<}HXqb|J1u%cnJ>(d5e-HbEc`i+$f8(O8E{J)3YtEZn4Ahew=>tA>8OIC
zDr@Vd12eqW2QsJkL*j2Sdx~I>MnLc*DftDGR9vT{v2;+&8jR!T>OL)x<vz{ho$2T~
z%~SlrJ&L!u@!e0Ss$y2Ns^U$*AC*sV+3Drt=5nUg>=+Afgi%Q8pqX6Q6X@qpMc_s-
z{0C%&|Idl+&#eOw3VryHNNFosOjx+IjFJ)%{4&n3e$`>`Xl<pN?puZj{0N--Ghtw_
zwRcAh%YgZ#Cll4*@w%^RysZJaF$RKP0~+2b<9=ush<Pq~isFMp^zkk0Wmx=#4WSAW
zB21)Lb%$u<O}fW8JL$@wUYTPJRssgI<aA~Zy}z!`ctdtNLmqB_a=3u+$5@j_mx}N2
zJQp=&bxCzRT*+|2HvAOT)s!XPS6*CX@`2VFiksk5-fIpp8*9PE{l~r{n3DF?{CpX#
z|KYSR*cdQA>NK_?CKk}1J4(??UV6qQpqIJ^Kr9Kke~I_sD$^o8p6t^{TO|4!3JxF8
zM{hXSN;Bl|O3A}bBydj}|7G^un=DiLtZph=rBOq>?2!06VW%5h78Cie&MWxsi4>6u
z-x2yX+uM;<b#ELTwu}{!3PGB!|6sKIWGKn;hs;3LjNOl%+C`BC#a-OafbRNdPB}#u
ziuxdm#;Dkz0u9|C7u_#972H1V&n4IPaEqalkQSPCHC6Whc0wZAGAmy5triW`4~UUp
zfa1COzr^QW#Ii;kujLPD{Di{Jl=uSSM8*H2F-d6eO4%E<4=*<+C2#D{ViPQ~U61lU
zEM&sLcmwekn)=M6eyWRVfV*X)F%!D>CQOdJT!faTt2~^4D^+^+{Sm~7ygW*{(leI>
z`Ll%0w#(Y8Ifk-ITMLAbpV>a2UsIM9&z$ft&zISxZnJN6vQR5cDn&>(0<Fs0{|712
zJZd{K`u3uWS~*Em3lL%u#)}v*rSM1DGOa@|A8kY{ZOKA)Wc)mt;i};Fuk$zF+_5hf
z_*|U*4y?KAe@p7QCyPz!T3c4M^6%95^6ht-TS|e?FMe>;o4>k<V~s9!g-x&(vI`Rx
zjqFP&tk&c$j+-Zde(`$E2x5{25|swHlKnU>t?)0X?#6pjex}6wF3Hmb81i1dEFhJ<
zKNOT#2gsC<k787|+ji&U%GWls<Rfr4-BasdIE}jOv%twecE;CKXe1r~;n(pWwU){1
zk5WHXVL(6HP%2Y;iL}_8pLA3`wb1ZDu{lovzE<N?_q6^6zs<-h$d2DrClIo>M(veC
z3cEnX41K=-V>K3ZRGnchG~xEbgulsYyI4dFY-{)BKVyv9S|)_Ld#`^~*SAp8XOCq=
z9{LqR3(X|jcmpvoRvL-9-pApYGuW2(ZKt!a#~+`654`wERv%?E8^yz$^wy`5xqT?y
zO`k;_kW>8r>nw~)@HSY_hz2y4@@p9DQ@#3Y`YTlTy#txX8-Z#CZMftmsoDD15~WiW
zZr~2g7QOO{Yxn<Nt}=8gRQx8UDhTzVwjVj}npQ+p)MkxaU%FTVLV*yhnP5@6-^EEr
zAlJp7!zthNPmU6-$nBQD;fKlnJt83@jFPdKlS&ictcI_O-S@xPki-*+L9>f*<$=*7
zkqx#yQ=YkFTykEkMoY3I9^p#@ieT^Y4zvCh#gN+Ujnxc*DFSYG;#Njy%{NT2lMXB<
z<P4H{PNcMoe3^k753r?)(nrjzw*8y&wo)y)8CltWIF3A4IED7qgMV5=jOuPP0%S%y
z80Icmi6E0p3`Ia0|4TfiiiB_YI{iI&q~7l-g-M+aRq1BG%RKm3t5oh&RLb6S1h_0-
zy$jM6erOH29&_6GN^Du5S{tdtVL002Jdn)m#Dwp_Pj6M&fU%dyMxkE_Udv^#BNv67
zNEPJ!GAF36n(~nRsv?`o+UzUXMY$wv!K@%l%EIW`j;hJn;cFL$-~J)LAlv)r_azRC
zO+Ia+M6aPkAVfL`At!!D&jy#jb>{hAv=BA=6GGAgAAq%(umU@2+<{CJZ+dHF<~3dy
zA;<y~mwaSrM@7mJW%t^CMQl&beBLo8H#K%W)qK0+#nzC=@{>$+%}P3bpvPeN#bJ<9
zX+}3#8-x#B((8=$6Q%s?ZcV&PzNff!l$*Wpgr1Ztomzwm<YXsIbH3FMI^ON1uet4#
z9JA!cg#`R+Nxq>JCCj345OKYaIC=#7tx?UT2AzMr(n$~~&G*f}Ewt!*Ou5#U!G;)Q
z?FR{E^@x!#9b$?&CTPJe>@wKT6<8IKR)XU;|3HU6;zs$qMApA!+P{*#YosDR;MFQ(
zg@owrq-i*U_h!-f?^wD71LiUVCd}NBEj6Zq0TR@e86e~zZ_gY&@SB@nm0{(xFSBMG
zZsqm6+5?pbnUP}1W*@pYida;19ZhRu+^!V2lE=1fL4N1nt+eQXDgUbfg{9OE8kb2U
zQw4*khUFz~ANR>jocP`ynEv^DgL#k*ZR|zb%7HxCvW8D3*^d!1#w*^aEEHID#4VCY
z$giKNmS(;GWrrU__Ld!-eq*#=qAjEur3~<YBOGk(h*MKk7E6`x{p`N!mMz)y?|iT;
z%ks`NKG+WJBDNMKH<C)F&sgr|;DN}IYg**f8=HKMWTN!~M&_BWqw+-kNj^#WE(wIo
z|6DVz>pQw;+6r@v=SN9`*BIt!8wXN()P@|k8muH`IfS>_2aH5pXa9hR58!YK;MLQo
z%M#qx65J?~Bvj=YNaH^;x)?-`8mLzt^co!gKlIt}iLe75d5-*e>>zdYUmC6{+qXJW
zvb22B^Wg!zE!F8_d)E0E6uYbsRyo$Z%QB^2KD)cZUgqpHK>#gNBNV5i`y@w&=gGJW
zc4eGHG!urE#L^#YGhD4EKDM?TSqawGDrydu-P<&u4i63wCLTVV2p!cX-ra{ujC{j(
z{YlpwE+Is1Kiu^al@7IF64*OqE%`hl-@K<;>?aBY5kTy<zPPB#$cYKqc?P0e1;;|n
zE(TQ7Oh4st;(C=&6;}wqY-k99n-<^--b1gSBf`SR$9WkQRPmiX5nr+)9kWFd=*5`L
zn%2$PkSBTduYeFG4xJJMsw+&WCugds%I%S6vPJzI`G2_Ug@D~Zf81&%{v4kE18HDd
zzjS>CEj)Q4HBicXtnEM?HP5~B{qpqVpI<C%sQ$`1I(ZdGUMhd!oG&dwxTSruHg@gS
z5e%=Gj+h~eW-ehHLby>3VUORpXFnE3#;cIUlLmZjl{ST7R!N@r2j|nJ2?45|Uvse7
z0G7R0uW$wQ-mY=%K8!3#-?L0gxvz7?Q9eoQW{WZ0*|~%828vzN>FEyV9KkKpF=L!Z
z2!PCha)dYpyv!g4`tE{POqsuA{pB%3CLEfdK-KeQxpr+z4HXz5X_l8<t3S|*wOd%v
zR`S>{f8~qlGSZZ_N&TekEu`Y^j5!XDPlO$^rQhXRdE~|%GZs41(z<Ij_*XkN?>6hy
zh)aO}BC${T5#-_fPi10avSilyTuVELCQmm*tbUYVg4=IL-a;jhH?897svvj$q_QIK
zLa)wVS@|kxsLh{WdRdU8(Ku0I?+iEm*6v5r0d<lqK`84bO9oWj)eWk>!Bzm>aRv8P
zTy1@z$s1fWbLd<xsGu?a^#%wu|7wc+voF6oJ2=5!$d4`J2Sd02tLo|iTG_Od>K<)^
zs@snS1r77uu->n&cotln`7dhdPOU!qIP~^eu^SO*?|rXo`1XN*B25EPV#ixgLnGSU
z0lBkPx#X7p6VH8Z%KMU-{*KL#JsvcTPo_p5mB%HbU);w*DtowaB|LeB`z<;N{JOYN
z>aAU3LMd5cQw4T(Dn#=W4sa~M3TXe+eP*R^7!-wX%|>gMY_ll*mzkxuqhYGnXbK9O
zqcTBvGMsh^GVdW(KDOBpZmMEAQhC)O{13ZIeH3yE`>^}3S(PTMzI2`v&7}->ra8(Z
z=TbN!>-}2@AIiAY?cKq47nibL2Dj;a+i0jtis17FrFj0JrcnK$yz%$dLw9Iv2e;Pi
zdh4<&D^8qs<GDNwq=fSam-qXdZzKy<q(COj`%fnujh1@DIQ1cQRH(p9u1$kfydzWN
zb_SCTrkkO;*l_wwH@k7Xn&kJ_HMfM!<TP6!Q@!h?efxn$RztaCLa!*+{17U*Y3lm&
z-*N0RzV#-u?3{|OcFH*&2ejlOXra+e01fi4dv#=vI`y#Fh_$pj2o!ffc)JBn38XNz
zHNnt3>--v^O;>+5c06@jy;V$tD0=Zqa-C(3_OHJt*Q0#kw^Pouf&4NMP+`$~r>uCQ
zVvA)ixw*)t6@x&bs(Qw3p7QlC1DZ{f5(5;#fsS_S|EaS;2PO`dpgg<~i|2R}<ZUOM
zU!vycoEjg9!M`1yS<3o9B~XxR7jW#p-vn@Z_z^KF+3S||Egh*4F$`1)YnWWHQqBY^
z3tM;ehZF*qYn49@5{Bx}Kfm-n*lF<O+9sbn1j`8K!tP!B<W6VFW~X>R9`E(6ZS!H`
zq4igHqXA9!p4aQ_NvvVv)QpTvFd>sldJDuqyZsIY9LT$ga3cF14z&n_@y@;v;uhrA
zK@;PePsjY&Z9^e`?v84TM?uak<CIZVl}tGm(bj7FB4bI><B{%P3>9Y!TF=QqzzH#Z
z&wjrv0g$J}BnwhGdya67^%`i6@e<|sQzQ}4q<Ga*>8mj9%xalB^WgI|#hmPgPHhNo
ziDF?D1WNo(Sd<v`Ch#`U@5h_0<D&@z<?Bn{`bk2B+{rh}HQtIDH3@y?E?z1~9u6R#
z3(f;srS3SJkZ`Ioq2J#^C)d3d-zc+4IxsLVsm;9OaR}nHT)>ae=Sq6PfUSqebYjA?
zW_|Mvn7V<#`E?4}#sqEu?Twa2kXbia4<l(`2V8Y*?qX~Ru*qnB`%a2DjFrpNCWm^*
zbMe^79BxF{8$SbXD9XQBucHT$cdHrk$PBf8;%a9DW{fg)L>U`8(y8;^MxLQavl4}o
zut+Y~SwQpmP0DIS?G$FJ(>=F^tB$d1J&L9>QU)|5RN6ykzAzBT?e*xBoXUaf#P-mj
zlSGDXx776C?e&<oVu($^GX2gDo2|Ir?<?7@96Va9igKV{q9jGh#w?F%%T2dk0xx%|
z3Y&+YPfteTV6gQ#EQ#`IB2bJsGy;7NnA-Wn-q+GK)7X26q9dBge~J!%ErPCieH%%6
zFiEDpZ(|R&@-{TivMhjNiSL;Ll@r(tac|eifV^!YX18RA_b$oJ8rBfQ_7y@WUlW~n
z<GcOHo6D^G4?eS4QDvD0x4EMaydyzAcWvw&Q9BRFUcplwqRJ`Euik&|tH%y83->GA
zSA&qACp$5WlX`xnOQ({hHE~1VRN4!bHVPA6AgXIETM=~vm79M-oCi(GxRUtyE%_E&
zGSa{8qTN6J{*WO#HDCkX*^(-r%#A?@_Pql9?_KM+XYjv?+{Op6&7N?CYcQcJmuWa~
zVP)R=eo~*}^Aj65j`A*y5wV8kD|??5-t-Mq<?|8Bpi-uc3|U+IvYVSn_j%D)W9n%*
zM^$>U_2}T#evT#aqV5te|9Sk3#G(@oy*0$|6W@i74t3{yUV&{hM1Sl1N16mo3>>Ur
zlT@X4>l9(G%uNmHy_e;Y?$<$X$8SnJ&X+2KU~EZki@<q%`|Zbyz5UDK!;ah7Vzz7g
zwggjMFCS3>lexec(T9Df&QRi{9YMIrLSk_3jFwI_dAoSI-$=`6Sy{}1;Cmsmmld<X
zeVOBj`Hr@xMaM&uLor}!I4H?wYZ<krVxr)pmN%)3zzejr+S@$zORyIC;Sx<nd3<;u
zGF%;|nZ331TjdG0{A9YoZ>UG_?`_I&NcJxr5kdk+8_b2|d`6(Sm*Cny9o44<qZyRo
zaz`RUJjW4)D0>A-&r{>k_2NHaB&A6JUTSIDTGp6J`qKZuXyg$h8G_gJrT247@Q^gn
z#D30`s6@j|h$#@|SMa7u#8^JuGq9vW?=|FsFymM~tzaaHL%QgxaorQ>lW@Y^yHT65
zdy(P>%5c?~*)N5u8mga^!R%u^wYV~IgiH5-_AbZ7A@Y>=5Ss~ouG%w7vTu07wRxrN
zjsvt;tNAc?5>Eg}{=B$?eLS}#55iVbF?UwapX94QLks5~ElnyEh_cYoF7o&x3+V8d
z?#H+B!ku2f=-EE3ru`!G+RiS%G0d>aY<=oiZYFQzDIU8ko-U;gX3VHxt{l<EW5T70
zg@ldW*%>C;*E+jlp7N<Bqs`k=SjCZQdwYeqXq8G9jRt5MU`m_XT$ZeG{Ks3YH&b&F
zE_y<fk2`N!H&pk|Zp{+t`dvEtdwg+A_J~3jfCz#2A{H^TQb1P2+th4q0i>#HTUz>e
zl_`j3y6#iRT=iwoyEya7>ziXU0X%JfZJpbIQpY0C-GC!RJMU61Ec;nfsRSUPmIl@;
zLi^vLl7~7zTR5FIJ-0{{k#s%%1cnZ^ZU&ngkATt`&v71`noj|x`^XNwAg!meSHFlg
ziFJT|9EyNiCEF<g3J@e;A%rmv_@R;ZbUsHp1wpde;TvW>nW?`CYcOl8NEhwiy^~5&
zqI`z5o)}_RlsR%?Pw`CApTYyyIkME!%zxFg%R6gXFWMzbH@ic+*lQ0injNd21|$vI
zenQx(kyX1G;M(0`VHxYRkjEid4eP8JpaUe^-DCihQu;UV<T<shwXM^NM)coV#fyGc
zZQ|r*uT>gP*@G3Fca#oRKdEK<6A1~<gY-OUdw=As|GvHL{eR`*@)x~XZ!7mZ3`+HM
zp`oW(-ESF$1)I_2>(3-|f&EXCtW?{Jw2ByNl{EhktS?m>AYLS7dG3PsIt(F|3!y7f
zdJQJ}-=Y@9uuH=iTN9ynQwZ!}Z?kkKz)qCJse~%kx^jQt<)ve7WyQZ^pTqw=SAe?H
z(Z(IwkX6U|&XlQ}`ugfgbX9u0KaD3dr|9)89uH+_GpL^EGu-g(Qdkkk4_DzZ#tXJ`
z$(>F<MZm7HxdB`Z+MH*skge`@1}@7y&C&~DP1PU2crIkN=<E@&*XYjh7*Uehl<sdb
z%Yb=@Q~-eYL-lYzjL+yqNK}h4N#EK~fA;NfcBBuxQ@SiLy*s2_NmViknM1gV_9Ue_
zQ76T&1&|4VYjJPMq|IsaUuiL6Ct-;Ld1`6@4?1IdF--PbofbRqU0<H{<eR^Bng#98
zSar$PCw|c#@P2niQ#9%!_&o!vy%-J=8k4$9D9@6_<Omej&UZ#vw*uK&f&Q293}DoF
zx|q(A#1yPR#Jdm`D-aPZhDaa&$txzKAHp;N{J&v?F@_isUtCY1&X1J924kHe)|ccN
zuJdp7w?xOxZ}IW|yh`Up-d<NK=SNtj2OCc*Px=R;h4Aqri;!wpqMdvWG7YXsuNpzi
zBsA*Lhb8QE&K`o!rg2-xT926?SXKL{Z8#JoCu<KXGhMv%0&4taHnlSW_%I(-ViCTe
z%hYbZ02tdF$JA2XVR@1*a^|wl{h?LdvN&_TJa7Ln$0<p1SK$=ldT2}{qOsuU5P({M
z;fapDgpz`D0>-q+x10UGubBc!gB*tX-d%F0M_sPJM;c_K*2pZNbp*8R-~J554*DC|
z#5G)mbOt4`*ADGA?WX~SH156P<8^d1`ADO|5IT#?!ml(!DZ;gmUpje8krlRHM=*9}
z<^!^!S^+M9>zTJkbNnU97`cnb!{2r)H6r$$4cBg$`N8)3B87SvTMH?-tByxJyB3sa
ziYb+G`S^SJcE%ocA&UVkQr;(}f6}@BGwD_!W>^f?0U3Whpj3kV@c)4<q2nMjd{cJ!
zsJuI%<C{J?RyPW6@KBe&(>EhMjlVaTLLdH?SJiY_e?E9NE!X*~i%DlnM+c3wpn-@o
zIIKG7bKwHaEdGrnO~oBtJuKhLndhHKUqt30l<V|h+Fvsb<;qVBBOnS9vaJSsl>?y_
z(8543!Ts_w2^AHLmL~~_h5S#NjK>iu9#Jj0K9OGuzTo~Zx(ZY|`qj$qKZuq2F`@{R
zR4o0CJcfHI0%E#Jc&9qqiY-sVWs{V>WBpYyv;%b+tYz2RmG(_${rSeS(<Yv?>?%FB
zLNq4b<7HBg;ubQ}h##?e$EliyiBZIXMaVTT`L;_^>gc5t{rH&DJo-#3O^>nQ^q3UT
zR7G9MHg23Bd&7r3IQP|lOqLWzU0y@nq<mgfD1GD{M>^OU%U<NcPVLnj+F-<mE`Ifu
zZ2rC@CsW3em5rJxVTh^rg^9|aN?g_OyMS+CHWiBT)I<*Bpy5MhKza2FAr&_`D25Vn
zXZ%k7ys0kT9GIQ3G@R+U*_HGwb=+X9%|G_JL+W~|nQr06blN0QH9F3BUK-5o^_Dmo
zo|<1TeOc291cIts>)F)XVJ1!0>CcwGw)x}4vwlPS@bE~(+hYCW(SpSh<`#$9$26@G
z6N9usR`mn$sa}$qvBr%oTxAQuOI}JU`8A_716LisiI2LF&%fm)n+JF64C|*4o(pc&
z!!t))=HAvem&bE=!$)_;2W_zU$n16ps&>Zr7pA~%I?f#as29U~`?D2-n|Lp?thW2=
zU$ive*5SKL3POIZYPvdqbGW&m-DlpXimdJo@(g?GyC%ADy~Q@fPhX6`Ys{+c*vcY)
ze;qwd%I0v3=zMV(=^k^@`?n<<HzuS!DfVG1g9yc_qT;~8xe%q3*)*hG9kk4^h0$24
zAcgiU#Q&MPI?-#Cd7=t}F5B<kyqz4V#;Mg|Z%4r>hhL`9Ip{dpUUYIKGfIm+`=}{~
zj?x%~Kp{#(&5{H{LJ=q<Pn$6@AY@AoWR?TzeA=CHtNj<@WF@%&o;GQ?S{Oc09*hA-
z6N*Hjp#5*z8kNy2xF7YgJ}UA#KL4a`sZSUp$ZnkaVw`OG4H7%L1bw-vq$B}lUP~Gf
z<5$%tamK}yu+)3jzHR}msBzQ!5fQe%K$o<_2zIN1!E?CDl0adxg#YMC*>LbEjDQ}A
zC;UAd5iyuo9dx%Ns^ev@8G#7t()BvfQW~0VFwz+#jxHH4)WXoJ*q4yCKl#4`unSN0
ztcK^UvS>+dw)@tnc=m~`VC&TF@qQ~?w4=G6$}l9ASGHuWvK>vxf+==i^H-l=FDp6p
zxxdOrUR0{7-l%N9&_*AmGWc6*Wx6I+wDW#hIaeDN=7-0m+Hfo*rPXmyPmwyr<Fzff
zQ;`iHt%{W`TF{YHs%30hWowrG*7=>=^_I%uYh{&fR*sp{_ByiBauUpTPfe+Zmb{v7
zpL!~NbiJ-@&W!F)vFwvdl`R@Kqp2flR5rJ|dfD%h=c=pE9HcV%S!rd5_Czk9jw^Nk
zW?gz!z07M{t=b>A77~nXETnYkRI`4uF0xlwc0M8Hopek8mFdXZuc)Su<}qaxV?MBD
zU-GH(UIt$)sZ2-n{=3{7*X6vq`>&V_H)2Y)FSjF_rk}4kd|%oos@?hzj%1{qTRSh)
zXJt-gN-Z{ajCN-E$ha<pzm->}CMD#x2fDP<t^A2!@$5r6|Lo5ClI^}$tjzw}cV$jw
zO=HZ;#QayfvX|@_;d5mZ<2sekEFYQD6JzOC<bA@aOh3z>3=OZVT$vde?`0U8%<YHH
zY-`-ve#}i>mU(sAOT~z;g<8APvz>{ta%JN$HpHCBxGuv`)N_+RC+TeR^$$7|`GCe{
zzZbV37dxkQr5Ed0)(x<-34fM$B}*bRc8vSkF~awZyV7TkyV7@(bGmQ+N~xBNy~)3I
zC34xyp9gsno1r!itg_n9edC<SxL<}L$UI?nqx^)GY}1>LY{T=6EZmj0AKgyryVG~l
zvj<pN$+*bb4-7?o@&)LD$^eg{GN1<!RrZ&iJI1jPgu*Zi0Vnth3YX^o7c<=y#70LA
z@XqN?{1d+Rx<~-7>%gILDsw3gTs?(j162m73=XIahNCiQs0@aqGH9p_hNCiQJW(0&
ztnd?nC#eh?Dudyu3>qqf;iwE6Dudyu3>qqf;iwE6omG}=S-Vqeo@L63O_vt;@EhOY
zUzO#3`z+o0Y<r$$%85-E7S|X~iglOS+m+QV=A3$y<5nk`(rmg_trx%I-O9?6Qs_=f
zOPOR!v*}hPh6`c|yj$6tyL_1`ud_^PHr;8x`WY`**6GVk>oimTB$fTPcWyCFLva{C
z(MWWe&2F{Md4=k{4zCz+Xms<gC{9JxMQ3!xU=#?NxCnw|F4M&8G$>94QRfBZ0Ru`h
zOz=gb(U_R{rU{Qe`fTF=oVB&<hB#$9mh;7UO1pOZbNb_a{dy?FSCq2UE`?H-4x*H$
zb}5vybdZ8Fdx>dl?P8?R#0h(c)u}mhjQq2!v6ju<REQ=YnUofkjmw0J3f^;Qa4ng^
z4uZ?c9V^Rl+k9Vr)g`u+vLsS=*Ik{_Te4f52xa9#hdJC0J)Rfzdf(V!&GjM!lBC&Z
zHk-X~LTGN_GJI)#QHZXQHPE%!!ubfK-**KAUThB6N;_URgPb<90J^#b?-Y5ToHOt~
zuHUEZ9Ru=fd@e#+)*|hIrC52QGW08z_wsEwo669h4g*TQ;#0QV#xV5<j-tV3e>4a#
z`0sunhIYZd>WrRqTf~-9mNd$Yn-6jb9a^570A>0XXCvctv?Jp_XJbv!+aE*O6_u*y
zOp!rGS=a_^oQtC<yXL5FtqU}0MarDVRJxY%bh4mQm9p2uu2JW2c`YdKMNyS57|#7|
z%B0UeIAxw=BS0(?lmjruXpiTizkHjYP}bbWq2w!;vRTs+(nBRf;uz;%!j!F8XFxe;
z4~s3OENPUj-Ht)XYbqno5gdvB)_qgKD>zeY114-E^7dtB*J{y{-JEo4&JDC90@f2i
z0yAe&QuY`h*|AhkSw2}KqYTzxfn&aeV_*rS&Mz*M4hF9!$l}+Kqb5k1;26iN)vt3=
zMR8`qdsZ{Pp}4@kR8E;>mItQH9vQ>_J7s9si8Y#f=pP#*^?VOc68i^hPatz^HHRJJ
z3N`4)ZJ}uRKE$I;e{lvvn(=eQmQt29%3iJ#Mu>&$YcuNI7BtRXt!tSzed2^um35mI
zX*F5T8tc~2BsN5%3`hVQ)jG1LfkT9oSjq^C3d$BnQ?@W*!GM1W$ti;ka>~dGc7dZG
zzC2(0XOxv^P4lCiO+{7p?tabqt({{YgkmYPx79V>dBkD^+0E+fn)p5gY+KnxE?Z!w
zt%5j4zmMlVW$1Vpf9D~q@m2cT6oX50DeIZ0eX{)hj(tbAXfB>=>|K9ulGs0>`5Sg;
z)UR&Gl$~UlnzJ*sV$nWVEM*w1gXBN4rIaO)GSMq)Msd(jh5996N{nYL6DJ%cc5Ce_
zKw-)>ZIolY?yWt>3bv2H6OS^0D4MeJmT1Sgz#+V~j52uf<0#7Xc83KHSkPITX!^14
zDNrUlMrgMn<M?IbkL*~=_)1kBU#8D#a<SkLtIl_|aaO*s6&J3zclg-|Wd?n7o1Zmy
z^{MLalxcjP&x((gT*~aL3sLzfFa@iYf(KFd5AHHe@54y?p<gjIUo0B7QOXif62~$P
zI7O3srg8JKpX0&2sgpLE)S&%qqE1bHa{#9j*vv(6ibGjmi*t=U{Q!Xxjizk6k}~kC
z;iW?Q(GELdJ?0{Di&K_<tTac&50rlBbh=VjPT2}I*1=fHc&u*L-l0XtE*o)-&=UNt
zp3qOn2$YGA5!%H_<9h`EP!biJG0wiCh41TXlRDe75mPtsh`fI^Q!Y6B77H2UA$L0x
zi%yKP2V;qzQ_6<=x(v>(C+>IL{0*^`y?#Z`SY`ZuGw6Lc@K|OB0Tof{2cx^?=>O$d
zMo8>tA^oU=h>(ly1mOzVq2^4uUx^t%xMP$k+h2w2h%=TQgZb}%aEw1Go8QEPJRtxr
zK4o?u>)%qg{FRQ;;>cpEja|D6D7%d*i|(JBT_sy{F=b$oA=DC!MxglDWh}lP6I&$5
z>^U9Fl94e^+5Z$xyfjmv1);t%=n^SgaPM+VFjwBnkw|cja9swpC^JUL(UBOF7>nQi
zyu^yVE)(Mh0vA4A6zK<8$djd!qq5PM@#yy{n>>{aoC7&V1!cBTAa-ENI!ZB`$)`+t
zT_%h<huoHzZVPsd^8TstgrGa;V9GQ-r*JS&icUOZd;pM9$2gRfp(pO@R(!u~a&~8L
zapp{tegv22ij?82?3=-MS^6Qyf^VH`A{pcL_!%P^#qh#uU@abHur)Sg1S!&wo-1gw
z^C?;2kQ_4}{U&8o&NXUt&hD(L`Iw6-J03s%kUPdlgy#V%+Y{OFKPb~@ovQSeWLu1z
zpW*aF>K~F{HDvUzFT|9AF<}@LClnno{gCdLP0g2DNe`aW>$0Jx3^n7q9|pMhm08dY
zb`g^N*;j3nV?@W#9GLBN$&-9?n~doVJm!S^Wfmmd;^GvPp|{@7R^c|8O~u^uwvz>w
z%1_MZAxCFxT`{nuqzn!QVnxc>(}9L6T|Mk=?bBBDX|0u9A{isHmTaAlti_qxMIlVt
zf>ldn@0W>`<uz4;ee^sWRnr)EEQ94^TwYgO>G`jr|3Mk5fEz3yz~);6rh<>>#QqVI
zd=rHFn6fN2XtqTtI&P9L&i$~7EuH%jQOe9?%nL&L(W#Bh{rKdq=7hjvrh+mOlX$;{
z5R#^GzyheqFU|d^HlP^F%vj-kOc`;esG`MT{?yHq@zXg8UN~g)7_B^!OQcM1=O2+I
zzvE$i%8qw=LL|xW`<IR}zToy5oyVA>8tU1?&E9q~u=J9tQTZqlNqf=R!im&m?fcxj
z{vbywOHmCqW$30jw3IVgrHOUDv=^l;k(&0Dn6eud(>eL6D*Br*sa*>F&6ji#r7X2e
zp_HYA{_g2IBT!0e#OSmsMbZfRbe-X=zoLv`QgQ?s$uQ|simWF3q`2WLpzJRnoSAaM
zkLFC1Qe-vJr=AXD(I<GPaw+r)-svIw*C?o{6rz+-Q4Kp$%BZLmqLfil4LecFsHha8
flu=O)J7xR;OZj$~8Z?he00000NkvXXu0mjfpIHM6

literal 28065
zcmYhi1yEc~7cGjr6J&6A9o#jzyZhi0oZv1A&Y;1a-~&N}2OHd--~@sMces52dv)tp
zon5E8Pj%Jq)vH(U-KRTFLroqNjRXw_1_o16K~@U}1|9?h14oMj|6bF)DO37h2-Hx~
z`FM48#lyoB8X7u3KYv0-cXM;|^71l1K3<DOaR2b|_I58QC@3c<Cm|u>;^LB>lLHF_
zclloX_GV&Y>f`OLrl$7xy2sDUx4g8py0%(dTf6w)=<R%JYHE071Y`qxyPG^YIeMD@
ztgEXV9Ua};+RDqX{dV1#1_t-_^%vyjtMLj{R8#~62D}`#zU`H6ZEd*<@l@*bloS?r
zc62_!y-D%##K*_;@W}iw)co`I_V@HOc;*Hu7k*044}m~<_$@D3>3aU$KCQ%Sm+n1Y
zzwrotx|U%v`|;P9mj`Ip@^-fh)T`Wjemmju)J&M{wPAnGGr2n4;#Euj)A#9=UiaP%
zxcpW8Bad?7&g;cc_-Uh#VB})L+Ee4v>qX(_eZM8K#b}@;P{yYG(O-erW$5AW=!Lpo
zIyd~m_1)m@+jN)=D?B<DJQEyq6d7gR1q5`vUhmg`GB-PYHT9f<9jWE`<$AQ(t8Nil
zs`gYqbAP65*qyiY@{RAKr(1Ua+uM?CV9(voOa1zlJqhgf^}22Um8Bft)X^OxvgG9L
z+uG%YW!QHG6vXtNjoX_O$?P3pW7%j0o})scQ?sVi&B+#J<ff^e{M^P)7Ou(0ba^AM
z)~~k1%ZK}IwW~iO>%2bZ)-M%LoTcjUEew|Z%*_Q?_FlH0Pgb}g5L=(QPY$H$e|6E3
z`*C@8b9+9$|Ih#XdDQZGQEiQ3)g%F#BX?zym%+^b!G)Rs{HK6fh0rGGOuE7J7BSWQ
zc>Cn=#y=VPhVsZ}ny@s<dQpn+n|oLJncl@wD~$xC-^K=8YxgplilE1RySuw*XJ=1O
zPw(V7I5@btxY*d(xShjohk<e2Qk0d_@iVh(`}DfLGS&=RV(}q9yy<Y#+(Izq0xvZ0
z>d9d(7QPMJeFt{&djNM1bU^ReCt-x7g+xAiLfJt{qg01|PH%OV*_6icCs;JnUPsmt
z%wSa9Q58vp`yJAYLy|&yG*GG*M66X-X1Ww&|68PC0|BmZ@uloserK=g{A?^jV*Mdt
zXr37d;o<y&$8uV86bFHi^n;o<qZCdl$In8waH^57OAdf=D*KhIv;4%DxfkY-IL#-a
zsW%wnPri)8KB6Mao{|$d2<(4@(auhmfEEqkC{Pfj#w5$(4R32=gi*sYia?+8yD6#c
z4O=9(D!#Re$pVTyuc+ju5m(VTsY_OoZ2;jJAR-Uhs&#I&3CN%l1@yN*@As+ebI=Nr
zRMlPUA}r@*=Pkt9pXzGTD7p~D!$*{Y2Wnzzqb;3X%cONk=mv66mBE!g?ong_^hdZh
z5c|m2$AFxGl_b!a(eVgr3x?F#HD3x$an|g3S$^uCCoaHe5z<B8_}MfgFBS(+eAZ?~
zNRLeH5DY>wYGFxL?PZ_L`Y9AQc!u~afQm}`NXNSUt3!pw_g+TJzmHFt10N`ivUGpS
z-fOr|a>L^8kC>Y<un<a_1E)zWrC2v+)$iu7qTpdlRxT`>z&t3L@DYUzE-3Iy=#Q;j
z_z$P#TGMP$(F2)gTJ%jRqZl)@%%g`QgO(EfH%pi!V3{_(rizV`;enZ<br=bo$m877
zl2p&t@0XqUckhtysz>!&b~1#{{RY>Fma@z}(P_Yq^z@A>@Lvxo#)?;c89P}Xp<Z<b
zrB@%ajCHh(2py5Ul!zc#`S1QA!54s;{Z<W+*bN^NvJop=b*28Sr15d<OlJ})UVIb1
zrk*VV+kOlWP9NYiyomsPnO{5rhVX6A;Kg$#OZ?SB#>!MM+NBNDEu538YTHJ$@o5=9
zJwmJJGrqMB2za`|KU`Z2d0oaG!)bO_!2qxPzWWz1|5Fw4^8NSE59PbS`$6So4#guK
zuF^jC^DAws*pIy>C#g9_7?vtzVPmG<KgF3%`90$gzTnzK@n{Ww5&q6S5cIjN{MW>d
zS%#hBi2}DhjIKAF^yFjFH!?mZc>jx=lWT7Kf+$5_Cj=Bt&_ut8(ZMa92i?zPM-y&f
z-)ax&yAV$%wS9J`Dgu|a!y~&#CNN_h<GC$xp@AJf4|?-RX2M6AdacN<HkC6fF-_yR
z4(10ttpuBQuK*ijGBZE}KQRjGmh`u?M67NbX$Q}$bSvM=T%vM!=q-{E|GXR{P=-}M
zG?<x3+a_eJq~d}`Cb?I0V3(!@t=?IUYSxeO7iMm&@H0oJvdfqiMZYHdSZ_n6EhD*P
zT?`S27}};c@|+y{*bZifl(Smf3?!2#0a>xkfWL`SBRYreTixjNEW|l?A$GapW^slJ
zuqWOEx}0i(56N|BdkWk7lHwqjZL@QFa0u~>0EMOR_8uPm>M!#f`MKQBDs~as^iqWE
zk$W}=*>N9wVHi76;1x0V>~mg&LL4$XX6G8OQ$f_?Rx8W4+3beZ?c<7LIOD;Q$V(?D
z!&zDU<7BkPd`7HtmHaMdp9j<WD&>S~7K$jNdw=8zH*${M+l)Fik0+M;zD=k}x7v=K
z;C|DOU!BYjY(30!R$8<-JV_AEq|O?BLka#>`oo;C^>%Elh25E+v)j(wztO_YtZzZh
z-U;t(vY`YU?O?L4G2z9&u@TY=wtex^H?}64us@m!hr+%;oP{gZmRFLf7j@TcSL_6u
z`II58sn)We^mDwTCyYH=q*>ryPByU5jI0XhKL+Zy?Wh0B463U}vbhWjxg2^#JV-~K
zm;7=f_8N$6_}PITyR|FJfLYWCPy5c>OlxSIrd^XiOQMNUK*>=q27eH;GMxAeORcuk
z_T^tRL1g4C28K0(WevCCX|*?0X*P6VWs_xUui(4)P^&RRjIC&f&BPPPZgV{t5bkEM
za?b6Ovcj@&JOfSGr-~yvNmdw~sI{$ftmj~_OUCug*#3;7mp6&~TY^F`amdfX`BVu%
zYxlmc*+7xRf`Pgyd}a8{uQXC6ngJP>g^5|C!+wLu@8~V{Zc^s=Wm5b~c1kpw+EL_e
zPPwC-ubq@A-D%iJZOf*t({hoSQz&L!js;mSSo52L8Ug6rL4L->Qe&Z2{1O}FzI(Iv
z_iv!du&uXD5M|~s++#BPcXqamDJczb;t8UJXTZV*n<7i%JZr6MKWmYNXKd@M<2Ikb
z!%0m8-?bd)f5t9@(~KeYpZ!<)x&(?IYexKU&4P;_gSkD*inSz0g4x<qR-twx&XS1H
zYr##IA=)nv!ICI|>AtGH%nSorWP=QB0S32(287i&2reH}Kn8?MZNA|FCYsiOj9Pcu
zFS|{SYy*&%sqCHg+94p@U+y&7H#ft^;P+-3hDaGN%Oddk{E2KZW8Y=HjPp)Rk1&Ml
zS@H5D5m9sYZa<I}7q=x))ghv#Fbx(a<Bt@z==N($!crlta{tQJMlfKQ&La6BYw}&k
zaUKVk9+!x3biX`P!s~p(-`2Ta;Fkl9uC6>@OV`!r&(Ch%c>-?iGeK`GuTBsNjs(?g
zExn+oSuM9x6{FA7zu#VfDQDN#0V+t3-sBYb6sxFych(_qX~Xw`H5sBbXUK|kN%cS5
zzG!x(gDtm)c4XwQ&ha=_cLPM*a_yp4B`(%;i_B>eVadA|p}}WWSkBkUvx<~TR?6vW
zJIW*6SjBxkcR1G8#hxleDbX(CZN0LiD;6T7Yfy(Ii%!>1ZRsoWBfZq$PX8dI36!AP
z${h4Rk~wPPd94yTGm9^+^PipT^x{)zep_9t(%CK@<3Nd6`*&3+t*~lwmhKR=Og&f#
zKH4cBT9g*Sxm1i`_Zuw}q0X&fy+XsrI(lV*;kebh(m?%+*o0k9g7{Ln@k@N0;Go<+
zq=9fR+9VKXHef_D8|@9Q@&xLY2&@|PSif*8$9UDnb&}`I($JfK%6v+fgTj25H>UqB
zh6JS$|Gm2U$R7deG@cfxyA@GCbJ_6ML7=1%U0;2S8Dd^rPAn*K#dHzLo4jgZvwLzy
zm>zcN>jineOdE?VSvpLL_&yi;=8k!`B;bbPGdm%~fy#s}lr3$<)DCSwVR$KAiKG<;
zjp5W61v#bRFv(aKn-Gl{OPfh&RWgI7FXqK7uJTpX&PB*xkMhS)XmwL|*F=^$n8)J*
z^VQl1SO~%ddvnO>!x>hdd=)CE%19l#+J$rasiH)#`$(;b$d+7t+5%YtAG6Wl5X5{E
zka$d`=LH$<JRsU(GdaIkFl61oxZRk>6z;Xq91!5<CdisUUpWL6@BoAm2bv7Ho<bqi
zMC+%*s;8_VGfUUvt`s0Yiiudh@=AykbjbBQaL!rsyRAC3nUB>4b<opBI>8v<h`K?6
zb<umNS}WQYcJm7<!CZXfvjzn;IFJdy*g<l|G*AY}6i_<D>Dos`l9%XzSSQUZWjGSX
zJ2GFp{0HJ3CUGo%2q(C4HjGFTCV`TRLC%j=1|_Yd0cBRndT*U36VZNk-kP0ms3_(%
zLeb2R8kBA4@Zj>3SpEX(`1AD^NckIXtDF5q1f9#m+Ezhzdju3D>{&FC3{RJtC~_c2
zTXk*1=LxkScJ$Gu`<U<!Q&@@?e*74Qtyy2#<uRQK5z)jLKm&=z$lXi6D$!&<kkh)a
z=$7C@N|r4vsIU&QV2X^UdV>Fs2gw~Jrf=cbWvXL1;yf8czOI!{`2e(v_~j+rZ6SAH
zQDqQT@lY7#><d0;(in`!?ZGY36IyDl&ZOf2Rp0>m!3xpk0-eKw4t^z!nGmelt$qGa
zoOM<|hB9cvArncHyx4YHKJ;3wOalx|CDOKFRMS%W>p>fxSi<acP%aRadG|JK5*zFc
zm0pUm$-?qF@<3j-6yu}Fxk++|j*Mv@IH0xKYL-_MA)!Mr1yH?FHMOA@TMqMcQWEof
zN6g&y4O_h%gY-zFvH#7M)eq!n;h(ct^r1id*Llw*_LGXg-cwm%mBe@E_uITxj|JEE
zg@10i_KqlX)nMRbd^7#rJcH;T!o)%9iHrS=KU${i(NMz|0nzmn=&7t9mu8*|#s$6Q
z5FJ9UqBP}1eIE-%SLsho_)|PN0(L$&k24R9luxnz@cMzFA`Nlp7u6!KLj2t%16Oh4
zu%9_Y=)*ypS`B}rnpY04LN8=(<<_hlJG)aH@H##3$~-D|=G?tq`R3eL7MZGOI6M1g
z?tQF~UZ}Di0kr`niWlu9%q2uX<^i6i?_9(heb~(X7=&O=&4LHBAz4*s8TA?0$8Fxn
za8Cd_Ruw9O{P0gJ971&4g68SZ<#k1b%sh=r0a5L!#fyX<Fd%1~?rJpXd^v{@`Dl_|
z!EYuf#)6+>ZEr)vr=>$4`?aKlM6oabE(YsKkIORBz7k;<c<2t234w7f#)PGq{|-KD
zrX3_1ELAAf^R;MmOCcLp^>WVV(Jn4S<VwH(>!?hIZF4mROR29L!PYy@Iwq)gIS&a6
zw{)QcD}2w)*?Fz!OKXjNA1R%VJXd1c4}mUj&{;cz%x)uot|j7B_kz%md-3ueK<00T
zD6K6Go+~b~&c_yhos!VX%ISTqd1(!iUuG@C?HCpMvds9mmM<u3)Bx}Vn-?|6Plf{P
zv!z%&eBOS_hXN<pfrwJi0*N%>@9qf;F#&_Z|3I7ZRPXjuWXT2kb97pCWL|7ZM(zs0
zlhQilP@vmVYUqgra>D9tZ_|PGTmlJCQl%73))B<a@X^!T%k!p)OQmRZDYM^Bd9AMl
z_qEq0bFYu8++rj=QZ~K4lEZ*nMn&2TeP&22Zi72v_rljL46dDAKj9nq;YYJtSz4Y?
zXq<hz2)+*g^X|>Lf1vqSHMHb5MVXz9WYwxZjDJZwMcn8B^;Q4*j!|qz9=9W+ZG00w
zPv8`akUiI0|0w{hddIMrF&@tix-<7rkJtQD>^#@pyycnQkOb&uzRKTN#z?9>a-`F+
zHUG=R$z}bkX}+1;dJpM5k2_U=lsFE}Y2AhszfaLFNs}KX>3y0oY40>0qXXs62o4y$
z`v2jy#bSm7N{-CRH}CQEoVyOInItk{BgFT4&|7TlJ1wuF9%t?2XtfbImBF#`n$lWI
zNx4+I+%+cUmM6i#Szpv6?(nZWUS9D1I0l?Z2#ge;VSCQa^fo^YF=5%6&3OX!B;vu@
z`V#h%cFIbw2x~_y<?ck_zI|t?MYjB<Hj;kTn>r<r7K7J>WKlyY_5$f0@Tm7e^bpvc
ziTc#guGv`Vy#D6nSIRPLpp_agJI09n#ThzZurJ0WlgBJ`Gr;jaz^D-k0ogbU@_eCx
zXn#KTF?oXi?wH~B;ZhVKeKq*>L^#b200XQv|C5DCwMr}%toCIpmgxc+!1f`8M;;ku
zkR}I(68;W2?=l_#>k4tquM8#&xK)`fQnACnWy&yt43OA-xfcr`oB}xp5?-n}|Mj>_
zS-$1?oChfBkXV^EfP7P4sXae`vD@!670Tm&#b*lmiKwb*7w^%L`ewkR)zX+ZUb&~<
zxIkm62q%DLGT2OJRT^)rd1n*LwrPS00;E5Scco&xHGOeX0Uz~S!#YxoR_<ZXQnY=7
z5!p2xVtH5>zbjbG8}$^-Da3F++QwxQ-zh#l05`Vx&$fOvHyK676|wOcx6n(hw?#U%
zi64suI2(-$#QXa&&-iZmWZURp)*}WYnXa?43GT-XwfJHoI|+f<2-n4>vRw<P37<Cy
z)~^x0^e$jwNF&>5C(8MT2$>G_`7DM|zMi}yRE(Qe{dVgyL%dW|I~*m(1k;Ch*`Za`
z@TwZR4Mtj-?}BmF44u&)L(uun1Ali^b`8bPn8q!YwDS!;oEHtb--%WSi_RiFI(U{A
zNY|&*yi_$WkE!G<T?OQKi&UcXRt(ECbhAO}%wAA@Eo>B(kw7czWpl?=HpSfU>W@Sl
zPnzP#;xZSo)oKyy&bg~zmK}5-x~SKx1YxTF5-D%tJI%cHj)m0(&CO25`<@IrPnMXG
zt2NT0j)b(CZNr21(Yj`J9g=gO-2BC8sUJ&%n}@feL%s9WmdO(-`^!Jfw88Kd{i%0t
znK2-~10Iy4{=qn-d<@6ji8_3F$j=lTJ_Hq{V=~kmiEB&-inhHOnIbzA&i%TYpa)b_
z(l2o+3`}Ue*xw#h2NUzi<$d?k$M(7U89-6lpMTxa!86EQJ<AL}s0jU;b@u}lf6&vA
z3gNKdyJ;yVGF!MO^>i5TFq6}n!w}<2g6dT`9Nan>N%8bOvjqsU7sNIsy>U}wWAfjn
zyX$DX9CYxewV#XGS+jld2vEq0%iV6Jl&@F8ox0iSxbb8RXh`j8W(jg^oEH<bYxEl3
zkoZ6pg0*l~J(y7zH?AbUQ}XXPhD4nSW@d-G(Dq{axs#*NS&1?Pag5||-R7757pFyC
z%<q&M814!R;12$3i<{lvNmK_=5K-p@=a}Pf3@&}ip?l*e@?A1xeMAA#o*wR|e{!!K
zq$JNBavx25@(o0JI(><=*8jFrHszR&wsH&=#d6M3di_cF2P7FYdL^e3EJhNU6(;u(
zUhUl6F#mB(n5da4NDO7udF*_{;N-z`cF+HTy1s`tLLBm~hw!!Q<<-gZ%7jXy=Ho}9
z_8g&OBg-B_j!?*3lW%vus;MQJIWzIK?gxY2`5p3VS`_Oqf~_S~hM^q7G;zr;wbt3q
zBlZE~<VSc$81y;xo&42l#48Gm(I8^B27t5!DT-6w2cy6P)z>1i!#)3VdWRa8#_UZ_
zRf<W^$<d&j?FaXBv8co`aMjX5cA$-(HZl_r2wIA<fC)zZsPuwhCihF*qaXd4re#fS
zU~&|!2-vhg7vXy_MF&;M4Tz^%{a(O?+3q8`J&kL?`_=azU0Clj4?^bqETt!0Mpv4&
z7}nF7y_MO66CelGB$m<yWYgd4&8(p+Q2*bn%+O8$gE{X>cDa8PkUL3y(2see_Hdf2
zt&Rx@-nb-x+v|%5R{}VFG=UEu6fS2dm>`m277o1hOo?V;T%WydRIx7`cWGVIA0?H`
zR?NP%uVB*7KQC?jD&Os#0QwubwgpFJN!qWZD{TijwY7Ehbg4Et$b{*VUZ8*a>LrW!
zr4FyGI1HkbK8uU&N<7pDPlBkfSzbrRF){{@{uFkC&<!*$4h*jsEcnHKhKG4L4~Vv8
z1x2i5B@~bq<f_JQu|CJ%S<>eLw_mgPaH-)oKY#pffFU}L3kP$n)GPr1CZ7#{anklV
zN3iZpKNP*WK3=Iew$4_|w)@PpYiUAl1y%a|Ip}i`qP%}DZWBj3&;tB$dKqilP4@w}
zQ8d93jJoh?RjXh3GZM%<t5Cv$b%#ow$oYJFE?1P{(w8X#U(@7cS_?k4%@+=x<wfR|
zyvC;9a<iK_M`2LWZk$AYWZaanOfaRVBRHk#0x<UMhfxRFzLqt~Qj`66;fD*?w7DQr
z7entC1-vQG+X}kR%<eK?0)b{L$5?u+@G}G`Yr#$5oZdHq>T*{1EbyX^$qsyp9R+>z
zXLIFNz-(|f4p>@{5%E7GjrV-9f5WVYW36c-Lp!7qK46E86v{9qZjGA<D|R`0ni6J2
zqs^=##<)pe(sSvVN%`nNMcJYg%<j@={tkpvMmk}^P&U$v>rG&8U+ObyyDf*52nvKe
z^zPRFg$P@i>Wv}`#DmKgzu&7CDmt3pCTsJ@hQzZ^CMp@SE0<vU5VJmfB5U0CD7SHa
z5e8M^dHy0c@HdSH6vsR<`+EcaLU=I<$XXe9#GhGzNwsm|>NC@pjNt$C!rfgSyim^K
zHOlVqCwCRm|CtSB4hYzM!q2(&21NWVqPC`6`7*7PZIp>?uMD698A*=m97=*>(}rs^
zxc!x0G5q<Uk`)-OfL#M8({k+tE-fU1e2r+oIHLaXUocVKi40%MRiIuYt4km*(4XK>
z-K$#BK&@zFLBhcA<)HXK-edMz@E>)2vIeRwV<NJ;rOV>qY}1arY?PsRLUib`Rywda
z?AEg9fDASy+nBJpd*K|m93ZoZV2ICx?b_R@c0fD^_sQO!6{C4sP0&ylw}#wCHqFEZ
zae9sN&nNfFUtE363v@2G9v?1w!Byao2PnGyG<X@7zqHG^H(!?B%%`mY=fOH>9dX+o
zzId6<@r~xQ%uW9K=f;2^uz~H4sD(2x7Hq-JAQRNNUEHzyuMs#F$o(%ofS9@z#g&*B
zc=#9pq*p&|?ankn&(Q$GgAWNP=92D?@3Vy9BG2+?=%&OFC4>0OyaCdRg;6K#Mm`YA
z>l_bqWGrA36sYvMEQu~G`v6k1%OJ@M)W>o>Hg(vnPa0{IW0YV!28WT`nJ6xYLx#pI
zqvJ0!R!3t3tQX_%)6~RaP{*TZ{;7#~qasIoU*ZeU|G~=!pF#|dzi6FHG%JhLZ#}0L
zH;eI{ZE&`fV7On_EuJ6ACmq9h<w8yi8fKNLr2oOI6KP7Ee7h68Mh#Nq?7e24Z{st=
zbo{=6ssA@Q)WSl|`@$?}j&S02dV~X2lRQ#VV6=4HirVBQh8JqGjP!PWaLB2&(&umv
zd&@IfSA4QR$n3;3XWU;ksnq*v(bo3ZuMc*EH4V~Xs);1-?<T2v;(fL}z$`=l+G2O9
zH}zw$tE7S^-~w@q=PP4U<gEwvc1yW%$>XHb+-{3!oE>{pV$~O5j1^`qW;YnbqN1%y
zc6ZY#k0wuaF(Zwv<%1u>^_X@3#{Nj6nNd;4<YhE>%0vobko|-8E;+-KD_wMWQSQX(
z2&aw+K&N%9B2;EJ&|<CwQB;B@`(PQ|7$+U~v@4CJfC9Nq5urZa&lQlOPajOt^4i&L
zu-o3?L(Hw%Fk4egR~(DzRKjF-QfJInzVGDo7Qbxu0YF|fM;8I^cOyo|UuZ>bNBjLg
z@@PWZdO#0}Oy@l0U$UbR0e?dah9p7+2AvzICuipbOU3hp6`gHjzepM%Q2gYVMZ|Hc
z=)ML^Pqm-ApV7o?pte8G*cq;vD6oFZu)lkU?1w%4?#NGmAX=p5Ymi7@^w5c<>~gQ6
z3$7$fd=Wo07P|~Kh-58x4N(!i8wo$f2#(ib+Wt2)b|ctqs5oc9_I<-hvpj7*Q9xZR
z|E4S0VvK?ksmZyVC-Y(I@L-lGgZ}4dUdH8?abhFPLeO>A6#Hl;C4)sDn`#2yju1Ju
zQd#lxlu4;!oRw+q7@l-s`*&Kh>lhG*?TU@tAj**1;Id10mGz22D`?0iWZ;|p*zExh
zN2}35!S#Lz2J7Gy)rPI1<^cNsYeRe=8|eqW2_A{98D95>&f&Hj)w3TpvTCCn1hhX*
zB~;w)r*LuT=tF9U1@}c@c5gop-fO0dWL&68@YP7LS#ynH@N1UD^2^*kLHOt+w6Tyy
zq{)^*npkvAjt*^`mUpy^N5L={OB0=;45$RBosJNZ)Wauu_XZL1z#m&rkhqDxuh#n1
z5uQe;HauS@hCez2B2B;hXkq=YHvnFjVsD#YEUP(7cfm}&S(^Pfi<HQ(+=`n#nIcS#
z<8AffK9{VakzkBM=ixa&XPUe~-*&GbceH`_YWc^6ij3w=$S2}?9K?OdLnANYtDFS1
z5r*Y%AVxb|h9-8gbwtz5jZj}g)i-<Zzi+>N&Pd0I*^*|ajSTDoTdw!XsWE1HBnIM}
z=;J^nWwEmD%?Hv5*L3R)(|!xyf_)zZTR?;%lKLj3=VY-F6^081QS&~JwRDj~q%Gu7
zAzOO-lUqzglNWnTb51<8?p9{eMyDLR*9pktc>|ZOX;?`lD92L$i9N<TiCm--7A91R
z>fkpY4GQI~f19%uC?inA5K~7FCcO&d{n0_Hd^eVFaAvN6dtxb2C{*%)RtS^?6#_L3
zmszdBDB)CJAYPneF7{NU-YwDJ0AIr@!${*}vjBd0ZZrB^`8MJ0;a7j<q^VJ$RO79w
z<z^-?A@ubfluoVr!nY<^*gcPI)V1^@7>T#;V~@fr?qYKq0SqxQ=0Tf^Lq?DjU`;kX
zPB1>X^KW04l#Ni(E{;%i%INmCJ`rpLO8lrl(!aIymF9@^Ih`FnK5N4~nqi%6GK%1v
z<h}+>F1cTuZn(D5A=I-Fkg3g4)_)%m5)rKXeYTNN%E<!@$}d}?6s_az%C{2=>o{*z
zMTqOb_oC<6`-?5}r3}K^E;N={p<$*lM)A9ltZt|0DzWL?J@T;APYz@g``&O6qdMvS
zGeAOc-Nm)&&+<T*v8}BkGo?9rcGLWJwC>@x*}c@=WUWM@)pvHZn^kRpVf1h=TglZW
z-fHX?w$X{p+c~WbDW<KD3Z`b&4I>7I9$e6&F@wSv%cLuf<v$&+P>N($s*vJ8`c>|W
zmkZ+#WyfLmN{s1qQ{`t*Ek-Td?3D-cCXuV`&j*IS-gNn50sNZZAn?i!t7eP*xNvz9
zt6)=oExSQyQ*6}iCPGVFvN{$G7tSlh46`n!y9|vX5>pb8^RcqbVAcF|e6GLSptlR9
zB5egJ&Rp#Q&(y+90%%mE-Em$&d7_|Xk+|J(KyHMVFy~Kn3`NNw29pG5!1EZCM@dIZ
zO}G21mCPENZ<_NQd3j0R_U|suORAc*;ijrOFL(Iv79BrfVMN_dV~*`fR?X=Q!1VV>
zfl4~gTrHcR;%fel7DO54$%Xp{u2L{OMKo?Fd%9gnbm}X_l-b(90~tTCB-S^z*8dFi
zlD+H(WtJ8R>V8<MNjWC%_V0O=nKyPZe|s=H*A<co@E!J^FgRlvB*I}y<h<sIS7M7e
z9sCXjjjr_@_?}-U92n$YLdgB@78Lz69UH{FKE_7`?(=Et)$=A1S69wNfa48bmNSD{
zJO*+C_@N`-Ud^gpLb`Wt5kguT?-^r-aDC@tgod-r8VDm0(h}UNLY1zeXI4@18}qX|
zGWdi?`4go+lPdCGk!2mm`3V{LA2p{95ed>giG+q;Su)Z~bgd)AA8rE&NYK11%gH)Y
zfU4167y0%QX#ZYyN-@&ZQ^iqM$_!d<0tQ@3fLAlR{B8r$+nIKyHi425G4l)teW<3-
z3=CY;s+9o#RnwAeHqTC8A`gEHJFKRT6hC^{_}_oT=*!gt34x}}k)9y8;yaxHk5m@V
zRb%3DM|g44{e5N;aYYgk))Hl!trVcEbtyR+cpyRx;?r_@?=IUsIdAFlV!Eu)67Gp1
z*%$@8(cjy{h}^;!c=Yl4*FRhz^vY{=DqNO;xf?$a>c^P7DO+k;s|wXBFgY1>`#QXR
zJFXPGkXEtmo(>~S$&XH8VZz_@D7qycT(rd-6+uW+SX~f6CC(Etn#{3WfUM0#^QfHw
zTp=1USmYQoTy6Td0{%-<;F(a@zB{2HqJ61YL;UE4!J@>JW;s+cjiyZjSIGKjGJ7a&
zoUujxv!`K4&o2XiLr7T2A$m_k0H-?w$bH}sev;SQZ)eHWQrr@IZ1H7X7kREex+bg#
zcBjp5Jg9WneE!E)l@?zMLgV}_XHWZ3XW`RX)q~xbDe-c^v_vH50pqmPr6V5{ZA;Jy
zscdv@_T)H!L&R>2erqwzPPf!332eByD#QfVFRoF1p3~IJ+UR?=2Q!oU%<VmSP_!sm
zNI4L4)U@39rA73h-;82ncU!0IO~ii4;;;N63#j5=YzRBYhQ!)@<NtzOBt_cEz7g_^
z8GgaA>APn0fv^QNQjsHg+pp|?Ufhc@nec3()_SE|4uA)z9n^-U4AqJbp;PkOT1Rf*
zjRdmI{r%@(BMV;jk?9wS`kwFA(1QyZM5x!QSH#+z!E=L?m!7bu1I!MRp;t+OnA51X
z*_@V@Tz?$gkMEP>-|6IF1e&xlx<)<7cj_FO@$<J?z-Z~a)g%yuvzP7U=-oPQC_`4o
z?~-a431XbK{qPy`)H`mziOmVRF}VKwu_=ZuFhU(LrWi0xWDDGi{k4+|Ty%l!{mS&W
z1ymWd+48v2Qd?itesg9Uscs87iiZJ>x}De1Px!;H%19b_g2(?c4!?cZjAk(O68Xv7
z;bL^}jPEv-I@71@2Gm+O1zEW@w5>ANOB$vHbC#$c7EIj#lFbf8eC*ESm2a&F)aGaP
z%+M!(Yrfx3D?w*B=bi{!KLs$xJo(PKo!{JQvTMpYdWbjp+p*?0-PTc@uRQ*WsEi=o
za1HY33<qyhTP&)O9Oo$8reXV-6zG@LBu|L%u3z)8R$ys=F%bT^;w`W^;hRx5o;_i!
z?2SXEQo%$^&N7UtyS4q{Y1`_Wt2%rPso_hVVxq5uIR;&Cg{B;^^s5k|RtQv!ux)Mr
z7q(`il;4<D&URdWlItTsS7im8c{K0=n3`jP+hk9bn`X!H2x#U;C(cbFw1{Ok2i>!f
znC`a4k}k(skfyuS7TpTN>tS!kgN10Q{Bs~PKeEhS+`__!>a5TAMSIemt*))F>t>Ke
zFab>BQWN4E?t{;K%q0f$@M>>KlGrPKsSpb5AOfi4IcRc3u$w&rXdl0|`KzMSqelS$
zm|zpg9$d6Z=v8z?9-`LJ-`OB~Tw+ulKTq;GIOOBQW^6`7&!ZG=Qc8Visk`IE4-7LP
zWMUzQqU7I(fr3U%n@*uK@o~QHEwQpUi-w$V85lL8B>3(__*XoGk4qKI--dxOl_VfE
z&7mjl1?yiwev6pNX467&^mD2;<x)^SvDE|ee9p2!u@J%dmdGR9fehcvu{L6!+!CWo
zmJZm2_diUI=Caufw5DuDD#<UXKZXGbrktMwn8rsKw4$3@@{Ta=wZgAm*K`vg=y>XG
zIwj6jFhffm1e$sQoV2Gb0f3d=xf~`LJ`Ov~kTuaRpDS8Eu=Idd=xeXBGwwMtzvx%=
z#MMaB1w2q`v=9Tx4IoU<R3bIN?+;bjop(+S%6~EsF?urbiR4Dw=Qb&MigL)CQp<Y2
z%JWa+))X6BPG_3MowQuCY#77*+@l0I0{h3Sh0$dyZlB5~wlN4@5^L`m=luW!X9qpP
z94h_e8@H>;RPu^qwlO}(Z<8f*<nxkz*OEH~S|G)^cA%z8sju{NgC)O_a%R}^l!3#B
zAzqM7%>bYz1ua9~u!4<1(0FB+T=uDJT8BWIs+wUlK%@@96pZ3rEs|+Uh4Y2EnzMY|
zK-RF<6X2k~G4@9rQ5K-g6cH}*^X4_Zh`q8}Df`({du~`F6XJT_9CiyNX_lta{{|eA
zEBM`#u=H%!;xvdZ?y^J;{^y5QOgVY7n+P|)Dc$n>G?J<ixE@y&d9o|Je-3)eut>p*
zX5~VozJ^)h@ZHHShnLm3mQkdp$Sr$el9Leb6fTFzza8%#a^4uF!><gn*$YEf26Q&U
zQUzB*sGDFLg;p0h5;m5B`GPOM$6RF}!7*Vf&4B)W>Z$37i4l~LL$#;c^ZZ^pxcs*4
zGD|!bo==+ZNL-g_-c0d9!M0Lmh{YRQxnQkmj9u~mL*$l<(xl)|QKz$ikm($;clxbA
zG|9Q)GY>2$2JUK4z$l_(a>0^367M1+6h%Truni2Mjzh%8Oq|t0uc9uX!>NMiJaOMM
zlkNqR6Yj)_J#*_|r*q&E4b+Q*R&!{*-<r1}`cRobbjba6lkB{v@a`-Y1!_NC)6xy4
zpw0eRK!#j$;3iRH>U!|&Q_y!znoUiwK3Oo?D|oK61_oA4Z)^KvW}BtwrGbp#$l{Ob
zQ0(K)M}D8)+)x<!1BIBF?G<9lc<wCJ*InG^(;>(BP1Yl-`tV!7+EH(&fn|BXs&qD7
zV+dDy+7WJtpZfK5twspT+wH(wDl!o)jY^|m<(*jkK;Mt~I?-#t!<L`yJgFZx2Rb>d
zQNDwLWa(j&NHRFXcL(t}FR?sy_)`a-!y>IaTn<!t?)a0_tT-Y#+@F8KHegD0wPFk$
z#hUp4OItM~dii8c_uHH~bxLiOl_xCi&u}|tck_4tY<vdwk>H~k5hqFw<gX=^+Z6&^
zvB>SWT_^pdB)rWeFI+rUFeI=Xf0V-pyeC2f`)ertH9lzLBFj`po!g8G?t^}q=Y{C2
z9-7bW2z1i;$f#wXIGGmtA<iqd6~XR_&g}T&y^Ks9SfEEGTK1oNxw#<#b-p(LUDy+y
z+C^Nuo^+|&7@?FU=N+@aE2iA0FHr(G<QjQyoBmATJZ0;#e!lUIFB^XIPie1yxrMLv
zz!-YQB=w2?#Hx|ZOPkG#5z*fwRMU&0f%WXGu_J+zGKh&4ODNH5&dY?Ys@P=jR;nQ0
zn%^>@fApIuC|$3mFqE@xo`UxNTxv<aAWJ1p*`Q?gSCc`fdjB=-9L66?n#W((@6qnU
z{V8%eF3PCC`3`<)@oAYd`;qm=t>i)Tcetq-*WP^7eo29SP{{E8uKE1+uG^?s#;PWq
z^Ohs>IeTtIR;LEa1!`Ts7P1;)&y2hyojKZ-uk*VjZ?n{n?lm0wS`q6a*#p?rVfDxZ
zLa$WEg_d+7w^1KPs=!qwmq&a=XE&R`H+8yNLW&7cxh5_@n0=9_OA^gupp4QxYT-m2
zJo$k%I+*_tlL&6_hYH4Jl*joGt;j1(25CuaIvhVAQRQJt!%(aScH#Kfo94=y9G4Z0
z(<E}n@Q49}F40>#fo^#l$_x(%31@*tsL54kC)OVq@;FOG%vEA6IJ6DmU2hT&fH^Q?
zQyn8SFQAgAF*#rNS>guFg*k6I^(DXrs2Qn*aWEnOgMfd>WN+Avwz|B!7`Yg;>ovmp
z4E|5pll4YS7%B|-J{H6$&Wx00n0DBrKdv$QTJ6|1`FhBX2ggU~2LH^C!tc~m{%=YO
z|DKk@r^CKm7_<%!VvO`(diwvRrT<G!0q^N4;9J+#Uk6%|0YCL!`k8C!W;*h)%&}4W
zj1L{Dv6bV`NOD3r?kyb0!mZG_Pw<Xyu(7>pTah)6G=xQnn)K*ur_T1_{3g7WG%3a9
zD_^NxHPntd=kId+F!BLwBY%7K@?r`{$8#l2#*^-ZlH_x-3qn4d8QZDrWxQELdA*!=
zc6pY~Z2Hbs$Y*1%WD-p}{tZkCnR#?hdFigyuqu~nxtpSCHPA4BT-v;L&@BIZg-$>~
zfR9h`g|p4C9}H`|Ph47!uqK;3x9+>$%rxiSZM73$&Y+eHxg%IP?6}1!D|J4fXPRRc
z@ho3Yt>q0ggihshfsVg*Or|FT+p;1cBy`odb~;CLRPyt&S6?6g_F(6XJQXxGe00Og
z7PI;KyWcAHX9<y$y(8c=5xgOM_(*rpgRB5#s%%dbZ&;Nw6R0d~TBfzC^%|m@tp>1I
z?C$AdHW_-bQEOxh_-eW=_9ozU^+>iOD!KBb{F`9C#cjL1(N^2@Nrh;R2pqrfk5iED
ztVAX{&zc2o*i|F$p$7c<%a5(vmr8JYMfL5SgpJeNK=7(ZAoN5mO}YF{7wQGQ6w6yz
zdByZ<|H01x4G0r^X8eZF?0NBHl@ImDH<J71eod%RJVN^FPw+k=PXx&YQ-|eq7T9pM
zclnx1_`{l7VYP7KIK_o!6xg^$UKlZw2KGU(E;!2V^=GNJoInJnvF(atVSjtPRVO!V
z%V~qfbZ-Q#DJWD13rfso_1D~*EJr@^<6pk{gQnU-Iq8)r+{kGVFkc^%vCfe6t1=}%
za0}^_EgWlhJT$az%)CYWUHYE5d>)gep&HQf{B9((TtpkAc&z}>e(dO_y=&Ua79JEb
z+RUxE%&tT-iOG^tL}#QeMRhj`pNl@V$Z+Et>!Ws<t}S_74K_zM07w^x;1f;tDF_RC
zBK=JrR(1sYv`7H`qbQL=dg!OG>fWA?GyZJvFd=cJw?D{|)0V{pP=j~lM03pAEqMbt
zy3M{N8E)TG9ra*PjK@eBfq#^T_Tq%t(RjP}9px)CJ{TE&uv9P_Ys)FGSFu(9HxvD2
zJc{v=J+Azl%^s#eP;wS?@cIb(c)W<_o_Fs97Y{bIp5a&)pinut%uXvHU`oq>ZYC)P
zX-%SUU4g6jS<|H0jGs>SEUEoTWqtK-y2pD^imB6Hk;=ydP;|pCh2hC#@8O%@Yr+Bz
zt(x3Yeh?`xj=wDXm_4RmSsL&E<e5?JG%fcL&_L8k5d+MFbcA)f{q8npS6FeW>zoIx
zKJ@UtY)>_pYO3quH_Zz3r9S*HEHB0gBkP*oJ-b{=bzT2GFdnM{@Q(j9>)Jm`zl!{?
z;e}&*8lETq%eKt09~V!HNWVeG*s0(#RyGEx`|l20nX2Pd7f;{u888oAPRZ{}3lPB4
zxU)LwMH>Sfj_#Ts`9=z1lL|<L9kzSUK6)0$cYcv%*P=h+anRNy+~H&5slX}D!hBgY
z3I%R(cQ-7MV(1TjDXH-rj+M07$~A`LHq~He0+q!6;eu#uGKg3DH{oG4*HIO>3GL?v
zlJz=5vq&soj)GKpn|JQ_;ql9P+F*WQfV^z!LBD)+8uZw;HClwQuPBS+u~6;sK(smF
zm37G#-4Aw-IGwM&p~?!`{oF`=%zq<8l%Ty=wvO55%)g30?vQEhO_1+TkVG=sTD!Wv
zalfkq{K5wO(mmX6qTTXCP0rs?*0;9Em=4U=>g-eHTgt-W8!avV%(}}IZ{>gDb=2Su
zFq5ZfD9$7KVjbuM)g@VLo#Y)0u=n~bP{ni%#YBQbWQZHLsc}FaP6=9?O#qhiP~VZg
z>2RI|s~T1aIM#R(6|0zKf;6MEkH{$y)!Q;0v~4T=$jx-L<Y3o6gwiieU{YufZY}$X
zzS9@OwAJLMwOXWC8c6Of!L<>sTW;;Q<;r7m5V&B{3iV#fT;>jEd2#n+*wznHmnc(7
zIoMZ{5KOu@S*AehZR1OAp!)`FNI~{SNfP5v;ZPc8n#ffJ2*6FfF57h-$Ghk9NASb}
zg^T9ued2e4pcTNbCbaGFYqwtS`5qH(?0(6(2nYMPV>%etMypSY*00%*1xZ)A0waQC
zOG2MRb)#v(y%0jd@~!KP(B!0J$+AuBg^_*u9!=YYNF5+1$3Yjlth#3L?<U7sNdk($
ztM%BY!iaEjr0>C5E|;60zn}bxv8xe<fDY|Qo-8JqAgF0z5>XI4j)!c<SL1|@)U{)>
zjf2Aw&&ohef{8DNN5->INl8N3u=#Zb_=Q!R)0nm(Q)*hPkr{!g^tL4N&k!(6`6_LM
zmbR2qPw3^((eS#;G}=)NZ~C2fpYd9br0Q?R&u$;!?nac8eBCpN2HqD2^Eqz9atyMP
z_-tZ54LKIWRL11}3Ka~ZLY0b>vw#1%Ow()i;LY(s(ROup%D?zFA@HCLBk+Ruc*|G!
z`0EYqSVqaB&-|BU_oNYYA?T^kMdTw(JsGKiS83y^Cc$^`XmLXHlc{u+cp;wQjL73s
z(!q9!2>~Ok{kI4=T`^B)%yty3cpA=eWY}qeutk2h2bwvxQmN|06<?gXpHi-yzB14I
zCbYPzHtx*zF<5-C&z7om=*z{yQN1jpku3gnLCRE}?JH4Sj+6x%gwNErdFH-PSwxMj
zFdr*yfo3r;fe?VKrr89A7=uY4v(Jc`K`jRuAb3RPk_G3^5$|jx5Z&@yT-^KprRL?t
zC|%X={mDVB&yQf;&gYn{n`;)y$FK5rRu9abmMkjSYKBWbJc2`-Tyc0H3nEyqV`5Kj
zy*D%}R$S0z4AJ7V#-<m6;z~2=sPOywZ{D(soU5SXcQ#r(Y`#o0mlR}n?R0dy^39u5
z-#EeeV)TUmSZ2vol=#z8)3I9%e1Q>FmNi_-y3Th^nPt{?dP@evf~qDe{0GoJFJ-Er
z=`}wQbXR4l8E(Lo??j((cTOZPC6OQ*QIArJrS4f{WG#pd)IEv<c0*R$Wk`{Vm@rD5
znu`@pweJ`TrMxksEnpe?ATWeU)L{t*p$M>?cX6IR_a3-OZ_*$k@dlliGP`!S%PwnY
z@tYeWz3p{KIIFhd!p}DV>j1m9AzzJ?KC(`<@xc^PdON7Glad8}?PcppN%_^7u*VWK
z;E1u-VD<Fl=sqLroO^a^&Luq|EFhk(%o$xm*4Fr*Y<U#<{|Uoj-b1la(@X#d&m3NS
zxF#1G%*h-#E({C`JonkZ`nqW1UN~cxtqO+pM}z9`UpASv%i<w<-&sJLg-QU>cd%yp
zccpBBN-eo;77)1VyB7brQD5S52i{3IBKq2Be+x`$TohhuC>|<mDVxDR&8s|FrTZer
zHR_i(e|Ve8l(Q?EIbcyor^(0ExN{h)%P;m3g3t-Uj|4i(vV<_sGbsaHdIgI?@??`G
z6tPSe_s>X)v5PP`YW{|T@cZfV)j>@7=0p0$k>btP4goOe$AQaHY;KynqvkL;{`&sv
zIucvbg(`*4h&Hm!(HAJ|;Pf<@I+crpqcCco4zS*GGiAnn#%V<*UnnrTOZND!rHWuS
zFh!CKf`nHLCyDEDkD5+RL?I8G;VuxmK8qze+5VX9)EEDTAdEJLd<N@TpFZ8m=xioZ
z_lQ}&4jjw>{MkzsIsj|04Re@e;v*!PC0kVbgfW|iP=@_HAIB8OB@*{bfo>1%p3Lwz
z0*ZHw_i%!@r#VK@aD{&>rd`dziTUJLg@gpvNAhW%o|cpG;U)SM_fah4Ru?W&6w!gS
z>8IodAr9sb%xSOg*kzFnKoo|YHDx&2r9z*1f&>GSa@rQ}JntxoG|Ea0@~Us5$IRQh
z*~frqYcji|J>L~={EG|T%PWr8m07Q!1l^@{yg4)EU<;x!;54ryAC?c>Zn^#=Oy3Ce
z^S%iZ^bjgH6aq&cC8RET43*UL0#C<D-uzysoz-68G~CZPq7FC!o%n%=4XqYKl*bb(
z+vA0qirWq991ep*WB?Huslr8ESB(4j#cJDv%75Y`>mFY4`MAXQFDecRL$F%HLZVck
zMaj&*hlP1jjIn;A@@UiAU6}t8sr(m5p+B|hrwl!tHF2Nvcr=FoiOlqr#?y(kVKK{j
z3r;iCSH=_HgmBbm@$Jjlh}A~|6Xi9sPFS)_nfYZ$GRzN&vTJ2+pGmsClrXhe$OuAj
zwf7|lNax{3;_Q*^hi<N=H|(*8JrDoE>up2h-(aBq7*dhk-No&A@<RtdcEA5_gtE_3
z1sp_)*nZBnu*dvdgvwDuQV~0CcSoEo8St^nhB9g!8@-qcBvIae^7ot1bY*P(NF(NR
zfkk=eiH~(=lnm+|J~_XM5i<2<e#0IczUdI4qvmVhMu=aeGER4Ul%Jz(@mSVP&--O0
z+%}rHEp}On4Uk@Gcex(hp{)4?cNTTRA(Kg$S$6ai$DDyCCm-*wzin6O-6v7IpDgye
zoLLu?l7?9)5VhFKa&Y74)3)Q*8jrCLpsAmqhS`$7ei(*hR`oFa!ax&0!A3P|VVV5V
z@;ZQ;v&Y*|mR(1h2oyf%Uu9IaI?6$N3scKR*sL{D-N1DMTqvYr6B^^q+`!At+M&j2
z(}<FZ0-Fx;sd@pLUhhm88ZiBnDFSaydsTWeCQ@p)!W{>lam$xM36R<)*13PPAbKot
zQKap<WW8D4?Ypnshd{G%2-gb8(CY#zx)Bgp9X!R-Dms#H*=C&~|3$j$On~F6<Vxe>
zwpADM8-aQcrjYQOHS?kIs<O$**|*JzGX^8tR#4p|o3dgC6s#kAy$|E==VMs0hp0Sx
ziA8V%yw|*wiOw~RW)~Vynmz%#p8fV+(a=Qa)+sxfGtDTbVHGas*9|ojYd_K0*}H-A
z{`6WIa%4%C;&C<T#ISuQ0h)m09PbMS@f$8UN~i{J>ko!}#5VD_l(u8vcSNyQcsSR&
z$E>S+E7MFE61dk;=r6xZ3=r};<H%Wg(Dis`a}BYHeizy``dWL<;Z%qo^C*+aJ^k4$
zn;+|zaFPk(J7eYyEOSQgKKXac%!jBCG+zN2p6s8FBAL*lp0?ioeLFK^s9Vw*afOWW
zs#xwuocNSjwyJZiDXkRJm2a1geMb_gKAesxUE+{J8}=mhT!pJd8VW%GSz>8<Pb~q5
zZ0YJA$*b2dWp2mOea9~g@pj~{s}F$^Lxpn0zcq<#f87;qa5K~bS*Z8-E#RAvS3t5Z
z5~qnN7;0V~>@x&sE1+5(1CNZjDG&i(c~-#~Kg#a)0Qqjul}U?meNchZ?6SNE(x-^O
zbeT5;A}~}%wbikYqEwKv%5dpX-k&}z%raqfa5=;RU;%R$!Tk4wI%lZgr_PA2d#6Ck
zGf`g5)>o_U3vTanVR#GwAC$DV1qYDyUKSb?uN(Vv{qC?2pbtp&f^Vy!CPT^EFxC5+
zmvDxptj00>FvsFCEnXdq?XR;_99O^Ii7A`OUKoRsZT8C|S{N_6FM(tCfT1Ja@Xb0?
zH)}>UHu#>hWXQV`x-B{iiYvt@Db|ZqCd!%@l<2JaA^AUN!{|QU=>T0zTan17Of8dw
z>#T+(yZ~=W?q$F7{{LB&$eskTGkd4!FkR*k##sMAbG;>`iPhe-etONxml=*ayom(I
zEcDu)n6iyrmqy6h$BlUNB_-r}a#>$Ig>$635|?lrQ|a6t#Zhby>%s(VX~b-~OI3o)
z8&n!@Gy*ZDHGkP-Pc}E}4BDlq9*s4$O5wMz0L*}LKAo0N)83~I*kRE>3!)eUlJ+_F
z1h3cEOYcB<gF!YgNv17?3T$Y_7eR(KO1rxvL(AX6zuWa+SBqDa_T&a!hco5!|8cIu
z($7sBs>YcT+&-(%`}xBLu~7-clV<+m*z&6r&`_jlAO+a1+fB!Cb?ID*k)tnr2AY?D
zD!z$TrNz@fg;nlcS-wQp<4k%VF6{qMh^g;0V?2E^xZK1pya>}TL0M8j=Ox_U&;KQs
zTQ-2%M_`NfbhL?0u3?<s4ATJrk6gSO#zOF@Y;5R-dUrD^RX73jG*`=C=7Tb+x~O$_
zg5jje`BN8VVkO~?#9eUBsO5d*{Dl*{z+H70<ycSCoppV>vk}cIzt-}#X1dBjK`~Y>
z^n*sq&{$)G^y1z2fbBYRktPt6{KJ0M|0(M$pyGzQbq6W#Rw!1q1Et7dEmGXw87R&`
zad#<NTndA`4G!(#?lw>yibE-`1&Z6Feg9kczI(HhoSl=blf84~%gL6URk9R(p5msu
z0I>bi{<4A=yUlKA9qKIj;y(gDe!J`e=2bg^Ou314l-{*lmayJ=uAY2MgFLSN9oa(;
zZbS|i=#T&a9OSNtbJ>trfIYIc=@pZKMwvRZ=sM1%8*we^qOuWwRV^5(459fy=n7L9
z6Zv1Sz7H0SIOEUs{=Pd&BLx_onP_Q6<a;iD3?0b<P?x=A(i}{Esfy7kN?SGtR`<`l
z)SM6$8hbZUhyB|EC2Tyc&;IayKNYMj@0$<Po+*3ikTPMpZL26N8d0~<s(eadU1fR*
zHi-6&x&=#3+23T!SnCZW8B}h~QUe8RJtg{TTKI}VzOC(1Of#FtUoD`&6z>eQI*7=}
zey!(9dLrk(OT_PNv!1TYeYe2Mv5aXpw*0f(3$N6a=G(`IKq1@kkD>KFn^DX#$>M51
zru?(PdEP```>)9pXeweEC&Yjes|md3V|`*m7UPt$)gm^CRXTjYTFzNA%+mK$%kmuu
z@HL$H>zYHEfd3ogS+(}%{lM+l6ipa&5LSr*VhJt&S+y-<yIbOW8d(QHozeI`g99DQ
z6RI2#qwBumzK&9+2Nd^m&otNb!oAu8-j6#gS3_lNrfR_q0MhH#1|Qw7_*kX0e9Z4@
zff|qm7(H{Oih}!#-n3NV-6_p{C@42YMNA{<PQ7>_1X|(R>IQ4iseLB4o5<R^6xzZ1
z$W>UT6&SRSubQkrW`9u@p)cTfyk7#1$ziLRF?&k{u|Nh?PUlm*u3k<cMG68Q_#o)j
zv3pn0x?M)<Z@;&ubyVzOx=<bTvuwEAVE(l3Rirgu*hX5`QficARCqWIz&ZZ|<|AfB
zA;AN+OoC+>e;`rpZK(A_2}yA{9zb`+J6cq+Be!DJMa81iA|9@b9Dh;_R3JIq3fS_Q
z7^N9w&<N}1nPiXwk+w>L`DuAz?;~VQ1~ru}ho<>{eZar8X14J~fS!Hz8*C*4UYv*e
zubO{bTk?7lV}pexHII+ksa!bmi6zm8=lngyvM6=QFA{Vhdju2T@fk6&1sm>dVs>~V
zIDd}bbSn3E=V;ml(!>1LXs&!&nX=n`$#^L@j%R(5WIwr`Q<0p2KJ|~ZD1|;SH(?v*
z#zv1<4%ioZ1Vbr&%CPR~v{tJheZ%4Vr(YxI@zOJQgOD{heWpHqP-MVn2{MZHY=2@u
zr+<2V=(!3`+Mcv;+gb04f8Kd+Sz5&jOO$ZHi7REODD$b!4yj*av%t^P7(u$V(}OV7
z&J!+C@CK%n`(rprMBasIpxAbZJOhPT9?1yFhL!6z(pS`)Y|suCYZ3)^hmO(&#sLG*
zMWyQxzeHvk<l<M3;Q=-1P*)1YyIP+vXHjw}b*pJA$P6FwUHMZE$Ot)fK_V)@kBX@h
zAoD_+k6N*Jix;tQ?Fu_Rz}9Hv{+cIa_v5Q|&J&BTx5QBtQJ!m0Pjsk_^AGXHW7M3y
zt!Q|N52*CSQR}}Kt?8jvyO=|Iw=7d^Ym~fMQN(r;fc`yDiILm@2VrU&Ju$C))wZ|c
zfq*UPL$9*n`VzA}+qR|YKRo0qX~MLebhwNkIDEoctP!YY&x7H6q7CSCaMVvZ@Q<q{
zHnCf8ki8%qlibm^g1;}S)n-SFxsxn|VSLa{9+YWQRku6yG}>CrhHJy*$qi*O;g+f)
zgH4oz;t5ZM`AbZoPz)WSthaCC7Ft&ao3X4S$3-BcLK}^lAx@eP`hsRZ<I02~sf;qo
zxT<8I@u>_8MsOvbb9SU0+FXI7;cPi*keNVxd>UW;ySigYUk`O|1s8p>7C4!|XD}Jg
z`DF9%{ywWzh!S+pgcvc8xjpf74m<`fYDTac=8Y(d>W%qQP(bU7`e@ad-IzW-VnEKK
z-kpUaA1Ne)mX{9wQ{apx&<6$ZH^qQ68|b^e*z?-9fktbgl_bc!Vsn1azPaNxFaN-`
zlZGQMGB&HD`ae3+>@OkIJN*qv2n%|jQ25>azvm$VYZsrt8UitlYs?yOxl2X}18PnL
zyq=A<t8^CTYh<Y~LsCyz9Ah8C;oSK3Ur@0`TnD&&DnHFfGPy%@9fh7V70K!*cn{-r
z>`X2Tc{+b0y~20-=^K2AW@TEoK6H7Wt(P0l@Z7O=U*3D>VA}(L@o{+&_|6F9eW}he
zP{!%clJe|EU{S;3#I8R_umv~35oKXyd^$4dL9(Ox_sosEMoPzpPg->P>UI^ha?}>R
zO~{a2e!=Bz{~do!$XLS67h%p~xQm;oo(=2bQzP_t*8kgAoC2}$yA`vd=w_S}dGExA
zdE%$=(7&iiRPSU6g_{>W=X{YfwYB%v^TH1q09|VO5o~|Uj>pvMm=smfZD-UfH_x`!
z-BDmJIea}g=0BJaUaL}AB?)bm$wTf_p6TKywBZYh7F2QhN#8Brh1uaLok#YT2-BFg
zDgM<O$u}xdWywW(v_R8X*nw>C4@uWAl4Xg!La`;R#Xc9o#1lxJ$>Dnbv674}#P`|M
zGrW=%K`KJG-vsMDN{RdO0fXJ41cwL5?Z(b_FEEX6#5ZLlk{2Hi@m0!gpN!_xT^`6m
zNyOLb;mXm26zbI6V2kq1WL{AZdOh1R*rMRIU*ZfBN9e>`vP_$}Nm?QRCw(!6>r{gR
zrm}iTTNG~H7vACb10S{gR^Ja?si`TXX=Ta2tAYBdPe0+KaQ?VDpYC5Md_qJ%-2Y%?
zGU(~5y0T<Y2HJlxQX#Eqq)bmsVP51QYd^9;o+R{#wrDOpvfc)=S`gL?5@bTu7r*mP
zGqJP{cQOkny)(_i6wNNH7jVAEf75pOv5+%OE_RzNT5i2c6rKb8TRIesuz>u-XXW63
zNB<46|5uJ}wXSbx8$Cj2Pv2ojjg)LZ7{t@tQO<gd{M!2<nSZT^u=^c-zRV_&D0%v-
z0^9Nw=MV046LgI>kFG|0wR4%RJjQ$!#mx+UrIGM@qg#2&T|`@z2zUy^Hi0NO*SSbp
zsr6*mobR*n=SDOL;WheD9KaR+0&+$0-M#_4BUkZO|K;|orduIlsur+g*+!NGiG>4e
z^X0;UdrOd%WipNPg_Tp+g&`G6`yf4&i9A8izq<k@6a;-|cMXW6o<sA9mEuCz9~yiN
zKEYvxbB4OuU(bQVy1PS?86*8)2MBa>*>pOO^4xwtSzjrNrs%L--({4|KQr4U1@<J<
z|EWN==5wV8XcVjBA>iR;dfi#{+7um__PI2{<U>R5jRtgv=S~<xXlHcW%nI38nQfh6
zHB5pxA`58$EnX>nUv-vIROZ=ugtq}gF~wC*x3xT#HDE<YvF;MTCIW8Lnc(fdCRj$n
z8cA7ys6n@mfl0UwsFkcF`U#*aBrpk!U)KzWYv=D7sns*#H>OqU_Q#blIzF1nUjuZb
zUOv;?dxhwC>reuh>#8rY*o1h(IYSh$z3t6Klg!Kfqu^Y$<QnMo>bjsL4c4?s<;<Mq
zzt3T-@`lQ@^{nriYfoCMl9KqgxMokqNdj^}Vh5Ww&&<$Mg|sn4Xohc~pM<2X{Sf17
zz84{lIC#f|-$sP(>UWqCjvF&GmW%U&4kZ@jEWA?3XWfZhpE+r2=P5#$leR^vXz~k#
z-1+9$z8Y^woRUINX2#M0K}51B0A+y2x~YF=qaZ3B^Iy&t!L=v&EP}rzlTt)JwcxZW
z9&AfnvqX*(_UJ=nuE{mdJI|05U>BGj*(ikWKF-gzeq>j8l`~WK*8sU%JUQ_GOmE0d
zUou<9=pc=sRRa}X0OagJls$%tpv>?PbjkI4X2{kdTjv++mztePAE*pmlCDyL5-i@L
zSr+K++Lb1#FV2?)sg!@kPAF<H1s(#fus$Mp774%X(Sg$8HFxE$b5YCW=K9f|h_Fm0
z04bOeoK?3FL^t_3>ww(HPRHX(B*%>rS(jNEwRV|#?wkqDqfaK*;{c{*fLgn17PE~3
z!j+EQVxZCTW?~~YvQ*9=`NDLzvB}V14$R)#UbuzMQZBVybXO5ftX)*$WN>+vJCgY;
z1}j$)wZK_AO(L30e*1~f`$9IPwUd_~Cq@YOEp-p<cZ-e^V41p~C(j^@0emw(Q(I2C
zW3tb<g!<>)dEV=|kqo>D`ZlS`S9NQENLb1|i|EOuA76=CV&rDPg<zy#0)4i)(-HSl
z8n}=*e!je1M>mPQ9pkXJyb4&syv*<6IdjXEYtYCG6(e<boQT4~Px0_hq}<b+>-MN5
zkLa@z?1f9?PpZj{kJ^`bki<PnidefrWo+oPF5CwrRWw%vG4_B?MahmSr*QlqZntBV
zH%?RO4<CL|<3N5j8|5jk-aJD%Lx6&fRW163Y<^D@*Y^dbOGcpcH;CDsie2%R!$2Kq
z(q+WtCou}g*7J(HuE&&TECGe@Rs=P6-L(+;_d;hMkd`f!=cRA}zYP!z$iMp@S*9|C
z;~z*NqNBXI;9IM<#*rf~8Z<=oT%#G)k9#Z#tYI?q$Yo#-MIE_w=Bi$FKJ0Cq1Xxh$
zj6!MiA))kGwf@2UyNq!@PWJ)%tNf5%D`{{Q;7vOW2e>29GT^D31piQLZ(sZ2$n~|_
ztVsL3Jkw}xSXBZIJ7lh=9*^&pYb~ve3&E*s$o6%hn%AR7gNqgp{ig~qj2)g`q*uT3
zoB%v!J0tFUSci<l&}iqbxVVnvl2E*0e`uDEZ!YqVp2wJPG}@2sSJ#}6?Kn>AR4^5e
zgSV9^WST!<IG_kDEhIwWV}IMS9m%Z_Q&KM_K7OKvLwr&}26_)yw|AAVo5p;lHBcya
znKW*DCxE3rU--S{+^pcUD~#W;$0xm7Z@Kt(qb?kWxNL;-L1nr(Cw-^~)u9Q@H7o~y
zcJFnA;qSNExH^&uwf5iN(izaRG7(;ee_YiOC(X2r*qJ2Y;*LQg9_19_)EOi#yNa?T
zp+P470uW*eSwhzD1GyY0=eOWs^_3;u#fKZ^ELcPD&u+~U9EmNE*-B6Xq`%HR;}oWH
zL>bK^_=c{L8GdI)O{S?+T!9_xhz7Xb&boW~JbxnRvA<A|pGf_4P{Uw2--PSUu+=}I
zxamd}V9xo!#U=HnH`vOb@B^kLma+7~GUD`frPdxbv7uCM=X1@=o(iXp4<C@rQeKG;
zJdJ4FHCzXsUauu@sS31`^88{?WTf6ew9m<h@fGG(BJ~gz*5V|(|K_Jp)6@eVU9JC%
zwf{Tk34Jx)Up&F&0j>L?Zk^proHftS97r|?@p8ex`t*~e%Eid@=w8N)y;v8ch&7Mt
zB2Z1veZDikL4gCdwLDee@r|2qDPK-tl~7RK{r!mh{%uabh{yYVCCCB_;PDd#kc5;A
z3P1o@QAGbIt%>$8zobG3@GxVD=4gugzb|o5i#V7K{30|%1JLA4_-<>$T<L#U^i*I_
zS%lU=BE%sHNpou*>sUh{OqZxs;cYO_<u_QUSe&@|cAl20K+$;$hZwxKq!cP$C>JdM
zg1hNC3wqL^ydXe1`pZUL_2A%EX?#+)#q|xlS1dzA$B?!Rq<l6s$9`QlxlAe>O$y8b
zA9!0~UK3jkpUm@<6$G$ONio6YW^%|m;FZdD2HyJh_+Y>EG-md673-J-S7U!IK1J)9
zau9Cx?E{@p>~bVt03ws4K`y%);mdXg(n`@YrYShB%G9zCI~8jyWg{xK^@G@N&25yr
zBWb$-DBIIa&O*3CpMSTyNMM^u^I$Ev)NZY2OSrWyHvlAs&dl<|Z@}{(KbcNDJsDo`
z*RyY1Pu?K<;ipM4iF#Rn5ED1E=dI{#vEO{t-mrR{o^=do|LQpm`<%8oUKlY$!TkLO
zWOeYPrFgivl4HMk(T?gyL0-7Y8N204vaf@Z?slpX&>s<lW_Q1sjqYP1zshFL_^-8y
zTb%fxXJil#=^VYwnYV2D&4JW9;OTtzCzx8|?AD)jC%fbKBQT&L^nUlZLEi1Yq6kJV
z{I{dPK_sKYxeC(%tLX#Z+<PR>w;}>QGY<eyVL^e13xRQez!m$bizS$1lpFZs0}S`y
z=Nob$DAUqdaBOHup7QAnP8V=(LkORMvL_VVwie}I&?dmGt0L-CG)&RKhu!xHbVB?)
zqDnadi=a->R~$AJHk9E2E4Tj<mk{vPMMT(+%7kj*vQ%h2zLu~JCrq{UkxEaqJOj9B
zK%rA5K@}ZOMOap1x1g!YR_TpNMWvaq$4*~wT^{>|fC!D3i3H4h$|bzf%fv5E=TI`n
zY20a;7htU^*l}#qnr=WVl73!M*FSp^`%xPsFYE9P!<?o$Mj&|nN|T2>WE^&){x=!^
zAxR;wTE9HAe6r`(_L?9FPE5a#XMg`oD{R~L>ut!$Q%==$+X4MF%Pw}Oe1afQzbidN
ze>p#m4izHAVaO7ai-RD5eoy93xVwe`Z>z;s`Px<#9yY{r=gnBpeOp)dj|DX#f#;f*
zKCBX|+9k5<g^EqyroHmz1=W=gHJa{LP~Zx<_@Ma(yP<;<1OtIcaJ_(LUwuYHea;D<
zKTzkGAEKz&WtV4F(h8M;vReX2!hDuF5=745QZ)@ImW9&J043K;rHWReT3Bx^@}tNm
zjN`{90vh0*dQw?J`3LLxIjazV{`R|~q9i&Xc51B=5B=NET*-osUW$gg4f)*B<<n<E
zJp|kQY%R4Hmi<S>&6rcl?=9WG{ER&1?O^m$6jDpPC`>ir8G6*Vu3?kFHhlmU)np@(
zx2`>l4CgQ3=NumuQNCYQacF>pP$3MUW>vcUt6COXtWlsQFw^M0>)FoSYNSsQMsOr<
zvrq^;(o~PbqSaFg^GhnWjAyLh?!*_3y;C}!5yO@9RB}j9jY>Cnw*t6R(E`{G74RM?
z3Ak=oo!0YyBcZybx&YB}YWN*MgNbWdC%d82#P(5VuowNyy_*BDxODf6*}IXjH`HqV
znPhzY+t(yHKgCR-iR*gDx%+vF3=A#R_Hk<5X68EjV2U764;@p5gj_-6oP*~#S+l0@
z@4r37b*|cg5hZG1(?8l(V&Lw=<!)XGo9&X@M^R7U)F5zW{%hi=?*sRJgL?WD-P8r9
z%p{;|RqVwLF{MAW>e|V9BF#ExG>8#84#-3uepB31I_Tjp1AiYn!*FIeydX-^eIz5K
z`)=;uvD+s~#am{T3$G&)BE`Mp5gpN_S`Rx=K8*dgCHMLl)jTn!2W8T;k;^wR6^7va
zEdLa`gN+V3NoRSLw-*~n=)#jNw9o46hheJ8F$gUDE9bUbt}X>?qe}M|4jLWnX@aW#
zmSZPLF^CmPm)E@JhphwQzctfG`*<mxL@Zu$d;q894X!A^kJ+44-i^t(lhWB5*4{}g
zA7~rC_o_>*u70TS%ocxLF{*5MU3jUS<Z*g$1MVWVpBy9PfXt59f+e&cTMw)S-*ZS@
zg(b<dLBj8@E5=od?pf*>h3m{u=c)Rr*IiyW%v{ZMY3#P@4Nn(*^s(iY{-xzMShL3t
zzhLy1gq3F__O#*kH^=BfpyRUjkNrQ*3j9~rSngXC$p?JZ$YYnoK4CKe1?}|aPo7CT
zJiS{L<BHAymL3`9pdEHB0HUKpvz}Vq%@3W9x3B=K;t#D~0AgSl?$L)s0gRtYP8<)G
z4v`WV4#H9x2)&n2J(Uh9fg$$P-#-&T>Hc$gUp|bJVuk%Qs*yj#NkMv2?c?>lB#@jy
z{zA&%85*_J*>-HyFE||GjI4l1AGF@#jV-;ip984Sqj&<y5}bc{s`_8FP(mehX!%s#
zVLhzUs`2#qnL5dL35a2whyM55k`;=18l8+m*ZrjowAZ}w>_N3CSRVW>y!zEAafA9)
zf#TGoEUjRvJ7)phq@s&SZ^JjGQAwpPORi%cxXJKrgxr*BT9fw1tf1vj%M7#{q=sDh
ziOZ8Gqcr3{>jmUE+GuL9{_W`Zpg4X2h#^`(bGeyfobNO`onOrFi#(B}8Po7n<ht6^
zk0s-1omZV$Fig5uSmSzCV193EiCmECJT0#3Akj-~>_VTZI|-ko8K;?@=9ZbxAoM}%
z?d*`ImD{H6%wy{Clm{JspD26>v9ZPu7D&&p@C6>4*=Nx-8xNGK%v81v2J|addE9XX
zsRE#g7D@-N5jQ%Sx32@DxI$egyccB<XwVZfou4-!eQM72$=Et%(k@=9MKkJCKiPMd
zER5H#yon(EUWpdUB5}31cR0$!XW^^sfQCr^z#*Vvp{Z?$?ZL&&2OCi|Jtl9wxcEkq
zVgKvTfur)2Q{G~xzRov^Ff<Qdla4v2_evDw(bQ*Q!%Zj>y7srS^8xVqQGAgUi#ZVq
z@YhUPqocC!%D1J;%4SIrqw*1&W;+B~c6@Oud;Jmg=i1=xK-{B9@<g7;f)wukpwCp+
zHn~s4zIFePmH`^x#vzyLj-AHwT?}cY&quJG^k*O&(fR2hT5LzU2EPJZQ3?ky=_@Q)
zlQDz}66I(aS0A)#BBeweF+<|~`JLm&24tvJe>OfxZOL3Za`_`;tbT>-E%b{t7M($W
zYJbJckR@Ggl!dQ09;)=4<3j0EJgR(h;-V<JH}y~o?ir`wZLnxB$zYO4hZa@POH1gX
zXxi+-)};_n2a1f1=|BvHEN{)seV2lz>Me=zR;>l<MReu)o^Z8;I+=5q<UG}jrb9{y
z3c`401KRuLLFhP~`}OF?V(WgW@r2zS$1#p^!MhW%+@<8)n_2!(ev$U2-{e~L2&GDx
z$@&hOSZxb9PmJJqUh*T%?{tw^2491zA~VZ-ag(P5^05{c^Kfk2<tfN)tK{ZhQSG9{
zQTpJ!y?S>9=~_{wNd8?0a^ZuqU9E|ivdNA-VE7teB)yVhU>t1KRL3Sjtg*{+Wqf$J
zF~c_MF_f|Mr1Q4K53p8An>h3Kr1Qd^3f!LI6dsIujyo=}PNJnFV3GFC2OkqMF`U55
zcIh4W%0azea#t^I*lra_sS4xos50<#5!;%p=2wvrLi9f)+K|R2{&RbI<hz<`8nbe4
zW-b2S>BZ|YcmQ~vcD1VWSzT$%&@f68Ek{{C^W^8=m@&LwOpL@YON0aqkIguAsFbK`
zB6}1JkT`JKx3grWSk6#=APP4t_E_+MR79X4)fp)9cB`=yS>l*uaFC=tjsFoRRt>@V
z$Nx2N%ia0F*8c8w!Wrpo?A3xs|F%JO2zD%v+|^B9+rnrkj~DZxc|}*{3~_P^ZFidV
zqif=QrOuvY+H3ZP`(j&y;gZ>KSARgcVak`bM6g6JjSDVRsT3KMKE@@<N=x%u(AU3y
zOHw<CkCbf14j$is)g^&)$it7-%e7(QaL_J$Xh@03XHJZ;$zux%|9)+bTlEEmp~#5D
zBQ9!sfz7rVE5bh68b2?RXA<Wd;H1e?ApC~CEgaHCy{wvUqmx3%X-SA(WY_5E<6%so
z$6c%G%cx((vk6w&Fpp8w4W_ToyNv6FqlYi3b^s}-AsrQO8(t>Y<=L{MD_nRj>V6K6
z=OTLM^!65u`&_r5YAJNl#DPh#dKefS1&xFV9x+*53Rpx}LU$)Q<FS`w;g$Hb@8vh>
zA1EPu#`tSf0P#8}3VooB^K~J?hw9<4j+$I-4N;KMT1-9;A%_?^C*zwTb|56<2PdoG
z`A?7GJ#dvkd*8Fvf&P^vH!yBZ-pT+xuzv?fHReQ6e5SQ3o$JMM9|Z9-#dC%p!BXpN
z(65sV#GV26JlTJ$C7E~0z!V7+ScYL79M$ZO)rw9zqO)lrDVKiTWf614f<^RqXpsoO
zmr88Mf=vSeQDSeKAlJdB^18dAU?7*!q%0Uv@{l2a5w7X>884FOtl$@Sx+VB9ZdL3!
zXOf(jVDLtCey!D3*lJ;Ug=F@_u}6cOO85<-Afp$l?{R@>+C;1G3rZ}WGTL0Oo09$#
zB_-p2W)1Bh<aXnB6_DsVC&2Ji&1bKzEZqB0j}Duk_o%Q0N9{aQ6M4Ui9eg9!eTjP>
z>~#xE1IeOW@lL%<=56vl^_u5>$8lTD?PWPeu_H?%xv`^$rR1V88^cj@!`1HO2mP()
zx+Qv_diLc6(@W>3Y-4=dTYm;$Z^p}J+uT{<uuGM84~_&$Ek5=qU{k$V;_L}ZaENJL
z!zc8X)eN!9-_R#Mk!GUZACAbx935`m?0vp%iYnP}8<3qs%0A>p(F?DElW*qoCc%+l
zs5C1Gw&87triSgl_Kw>;>F{KFx{wMHpF#54FjX+xYil=ig<Gin5&?1g*l>H$i(w4M
zFE6ll<sW5`Ci`{1)lTcCw9&QKoq-@n=&-KML>hhiV}D)Z-`m5#J_>ue<Si07t6j8u
z)tzmyeC#~#uY5dJWis-2+lI);z-LaLl_nhjl@96)s-BJ56QpHJUes3?K|AJ>|Nbf?
zL9;`Xr{k7zso>bZvo|s#VJG{!R1z(x;L|O7BaCzIo2Vi*m_9{)i!i;YWs8jO>8xgA
zzLfwsFf1G$`;$Zjh%}XJX3s_NIXsV*hNfjA^W5SX^cCJJl=QB|1y=VSvnHw+DB#oT
z-(-cd4=roXJtk!d@`;gS^l(xUD!T@~uC0Z;KO1l)K_|p^BH-uDrUcKwa3;8N5mL!I
z@@x@VeGH>gQ;sxBv-$bc@)nX|;r=F5on;_Wg}I$<<-_IJ?e2k|C&>PEdBedWtZ!3|
zUi&lR?m8coRH_J>b!#S4Yp^W_?F%HY8!~0znA9wxSB$7<&P)Z=)a<L=?^!+Kse-4<
zi&<;dvOLODoujZ<SflX&6Exw!JXDa33@jQI_5V!?jT>={8}RI&!Y%-@7Dfl$qr1td
zOW{dLAxmCFBMJ}M05)mWf4(b!{Z~qG{%)FNN&5=gLa%U_`SryhVOiW3b{{a2f4{EB
zU^V(FFXi}4xoVo7&19bb<y?SfqZ*deqQFanMxq;(YNy1lI%M8g5hTs@@j09#yWj0=
z$2KyD`#uy1ZlNep+E{3ZitylwdBmxV9cT!N2(M(RCr5OAXA+BttY*&@d);hlkm0Uu
zCA=Jj4`9<MjK#f&Osv_lV+OIcm_B#h)JXnLdgo7IP63www~L3EQ-~#><D%lvOC_JE
zb(g6h{E#&T&Z)>!Ha<EbVfHx~Kb@xzZsWR^H!uYoKIhP&h%lNtICdHSyx*YZ<??6}
zxT!B44{2@dGCNgq4}@41RAy7c4_pY$=e5_Ih}f>N#EO11(jj!w-8k}yj`iIg;y!<q
zKq+rCPE^=@V|(^?Y0TUBCMBABJ=eij44qv^uxqk9l}}-P*2OcB34qmq_V%m(C8%V{
zuo%={6@xiIj;)BhM5a(5of~^XAQS4iCzM(Wt`P#kmNDF;J**RMmSdd)5S)x(1g8el
z`2|xDXQF5C2e*_pGg`Gv)C(t_1x0)=VE8q)(VUfz$=7brt+_)eXsbxguI)_rsn@e8
zOfAzmT*?-IHidRC_k2~yu8)2>Y?x#6Q7hct)#f6fT(AmyYb@wxpZ1B_s@8;oQ<|H>
zks9+hZBedYL9wq+Bf_oDQTi8+ujasbl6%S)FUJpZ|CFX!t}oRr@2dO;c80H-FAmyA
z`0d9a`qJZ!B8QvWBlKT+QUH(o_4gBus~aaP50h;r0EagUOxfdE{K-Xjy7L{Ka8l2q
zYJ<1yY=XT3(^yhMH6uUz?3PkmQHSrH-!Ke<-m?hN1lf`WHvFIv^tS~I%#Y`3*Fdal
zwAdb-AcH)7lruAEYDbf;52JA84xgW!GRbld^RxZzO=gUPaP?nY5!gStXzDU-U1{}`
z??Uq`%9HlV2as#H{@lho`K@p<`z+#Fw<PfoRQg?&WI~22sFFpVSWwD1U*a<D?zW6Z
z{Z8&smxi!O;?tiR7UtmB;OH5-zncJ^97S`n8s2Q7xy@y~2daaWyjlsDtW9{OI9HWe
zT4^Hb!bHt{KVMtC;Px1*Zh6;=jor7{%-QBX6(`jGt@NqVj|lckpS>Xhr|qsj6B)*U
zM;Dp>V!mQa8`#Gc`g78Zf>A^Cl4v{m8&8To{a9?Z#-9S5GAdVW)mKEu_ZA8Gx#$5&
zr(O$oq(UXVhUq`pg%F&DA#G8&>aEJB3SaGZOGiFDH(Pgv2C(xH9QA_O7%*-`q=Q}#
z4}yZ&7=|%|*lrW)20`%Xmpg|7sA3M01^j)4e`Ynj_gTcoDO(@Sj$`Ek08w(R1(D@W
zaTo!~k$}+Lsx$ct@UL<8-|hd4zcSCBD-h6!!Rmwi*9`l=nPLZ+9Q%G)c-!-KlpDmt
z4`hs9lDD@Z>&Fx%Q|ja)OscWAp4K#|IQQtQK#+(UOsMkCVnyhFVb-<f9j$HF)(fm7
z(E;x3<*o-zdR*ZjdL5ajr}m0xkj393_xdDWdJ%bw2(8{SuN`|_C~wGlFvC@0a*1g3
zD;mq_yOLXzt3JdOP^-y8%*Sns72uR`TcTJRvr_tnt5>07L&`r|c_Rr<{0`{SC8+gE
z2iq=y2*umIwo(6aN27`zIg(az*m>|{*f83KzteugT=sngXE;(GCdmwpOgdt!NG7Fi
z=L~+p`qlpIZja53fv{+#IGH&H)Ln7%J>!kc8$tR0xA>c|Y0?y-+xy<nW4+F@Ul|}-
zM`G^KlG_oz*~BB{?On&p@eQHtbGfH~reN{&-f6WSr499kHO0MOx-#$^-KHZ)@=!6M
zt?*uh1#YUO^B`gclDf?$SOmqp5qvq)X;`XS`j;~5+Kfq?xnpuFj{4Y!+DyEj8Z6~g
zH9@RP9wd-=TB;-$KF<V0KL3Qb%XZ_RdlkwRL^mOn7zndzC0Hv6F^<rbMp}Cc3dD~g
z>P~}V6ktJ+F|`T7bH%K2RqI%BUbPGN@K%}`m$F&)xdc(Xol-dQt{B*}6E|YC=jE|{
z9a*249+ezC+WmuEaWq*=8KYNZMu<RL^BDPA06C|waDavf(Kr_tSA>uqM|gENCtsWo
zo??aZ_VDN2^t6KY4jRW}`O#fB{J39go{WLv4u^B{1qbTunpwA*v1&!pl&8yIZ+|f7
zr<_asT%RR5j)%kz?`P}%G343Af~#rM${-6C?yjzHo@p<RJTz-cOMH=Mtx<Gr8ha7#
zzPbQAgmr7`uW615%O>&LahvZpI$3FQFbdR={T8{@bI`~!fxDdoCF<Vp6v;VZL6#UN
znN$c}BK(r;`|jU-%Q(N}^&bkM%SmYga<D#m<o?W|{N_dxlH?OT{V0KYSCptayjQT^
z_6EJe{_N%B#!rdq3q{o8oo`9f;JzXo4x{!D-W(8}&{<y%zN03qt7{!dA&cP%DG!T_
zbxAD2u)r{YslI%mbx=ckF+{|>F^>d+A=Ubk=j*u?1mD!;VM2fAGxg`q>_Rg|R|_)2
zADXx*74#OuJS~$dZH}T~D~@D_g=xQT8<d+7J?knMt%W}C3_jh`VYGstsUC`@XijcL
z+c>id#e^8>xocYyl8$S07PXO8?8&z#^<qJh(+J>j7FzB&6S#rey#Av&0i;J%J-@l4
zM$wSnpYYskzaU^;qQBAQpehBT-rq!3ez`qOM?~78%g0$HL6`?qhq)^kRD>L*hcJsO
zSMXg5Jqd)TR!@o+K!PaYF(8EO-O87V?6D~n;RuH6zJldb6IHN}d^2N@0crNRxyHm%
z*!Xu=;LC?te7LUpeBQ8EmF%cu&4%wJ<M5s(W1gd*IDFJTwkaGgmIQKYYHYmXmDp_N
zC^*xn?K|w#G&bd@8p(vX!i=PP)uf5|JQzDJIU&>6zu9cW5FeY#Q$t^sfsThQaCV_}
zYrd?tZ8@NyBI%jqsf`@_PhH@ApqnQzu8jxb`n-xn%kYhrH1-g$j?jjlo5F{)&LW#_
z0`C197g7ar8;CzOL+Lnica5$+JXeDeF{#lS^kFy5SSPOfy8bz?K5r<K74%DtVtkkM
z(Tk76CIcnz%0<z$!NEb))@ea{TWsYC(z~0Q;37pU+jDb){s{@_x#<$UqWHO3f2uTi
zcuq*>irUNyIw3{+;M8t$*Bp8p-z}eMn;iMN(etNw&5`dA$2`?9ZrIL{jH-0p!?9`X
zT5i+WrEeb$M!VW8J@Y3xBmCXQttD7JQz2V@%rVT&`;!aOD;dC49Krq->u=MroUX5#
z>o{D$UxY1}DrJbvSGRhyGG7)J(n>18xoz@T@?o@tf#gX1xUa=ysBhYBL)$reIyV>J
zggF>4^RuDxF{s*c+F$+nlj@(kQbcRFtUG3@J{2dPK~%xm&S67RPc_;pNjtI>DREta
z-9H+VRd7JAP;ju&*%{!hQ?D81TMFH!Au%YS`{CIa1g*1BP3}Dy3VSOp;2Lm(TE;MP
zOH22Ofe^#qEMc`H`1+<IeS?x85L!}rcY%T)&c&{xmJk=w!NvR&1@aTmNdT+BJ)l|r
z42>D<s?{ZM(Bys*4}*Qu3^e2c2*DG;M|s<lmcCJQLvHb@29=byd$^+GzVnYC9iBlJ
zqW$JA$Lg^QLwUjyH|AP4zbPiu5V|FpSo9i~cZgQ*_7L6g;q<Nc(k4>#h#=8D-qxf0
zQGA`rOPtQHm;--Y$=zHs1oYq1catxc>>Q+t+DZD0&O4-%6W0YcCr74=LpIUxB@RjP
z7MKYpqCVcdKYDq=M4K`!_sGT7j6f@`9;w%{V5J57Kn|gcbC2=mK)-fPXE_CoW~ZVT
zw5sYFKEL0q&U>cOF!#60-u3D-vUBot@o8>p4FfeH9G_Gk9gN~8Wpd%aF5#>XG|wZh
zU5`0l7YvD8PB<KRh=6p`XzbWubE}mgmKZm;nrKigMb?Mn6sMYGY34802d;;d+hyH#
zlTNM04>x|WG%>7E75MQPWSgOu*nUK}3VTuq^07(5ww^Hj>g-u(Pt58ql4BTYz`Q$k
z1)TJf3wi0y(z^hk(<g8Df6B)HXr}%<U4rwZv(!X0Zb1L_O^Q!ru|5v)w8%**OIC>+
G2mK%JT5Wp(

diff --git a/modules/web-console/frontend/public/images/summary.png b/modules/web-console/frontend/public/images/summary.png
deleted file mode 100644
index fda0abf6cd3c950f78e685d047a37ac62eba6be0..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 33650
zcmZs?byOQ&)IN%|xD_jbB8B1wN^o~~FIEaPc%gW4TC@<Llt6H|;>C*uD;C_{Deevd
za?|&__gi<}-~A(N&73)tGqcZg_SyT{&wSI?R3gNq#zR9xBUDkA*F{6a1fii}FyUaL
zT56Z&@=*<F+UoiWNF?&|@^WcuDL6P7i9El%ySqUm&rq#5yEHUKXXht<*T_92lAn)%
z@&<X**oa&nJ}j?)rM|dvgOuWgH5(TX4h>|iJs=SaLBiz*bvHsnJ*+Ow3lB)-(R!a_
z8FGEL{R-JSd3!lFc>brO{tWqR>mhUr!Das!dDY`BREpf2lTJK++%GKADq%NXf4m(7
zaaA&iz&DV{g@j*|2gvmFjIP}~A-}zc%_Js;Nox99n=S+rxf48jU$let9=u21oJfQn
zvArk?gk4Cv470HR%wWv4^NPG(3dm5vo1C2B(wf`9MoM{cC6b5Vue8cA)(Y^IiqaNk
zDVE(IHL|&j9F=Nmm7Kp(>ECMTR$*<J4z5ZR{>g7TWyDrvqgcLliewgP(c>?==~QNU
z-F`jPoo8r&F>Y1cb<Q1A-4g%$i$?itZ!;P0;x}J>2QmebC;P>m-{v(#jlG3bbIvyM
zYA4rjc5W^W^$o{^W&Q-$G#CI`IO>bk)wr$KT;G%^yKDb>_d<uEEM0}Vxau@Xxi}<{
z_YHHgVah(AfL1yvXR}jFUsyEVtEr7YT($l0#jlk0)EcOUC{$^*$iDV0E6LecP@-fa
zUzgi?;pVu*{(5PyT9@LJPwk*jxr~o(r&UQq%@$8ecl-)+F0W0;yIat_#$8e{CyMJ}
zY%O)-IuFw6GI*D0UAs3{WW-l@(^<O_NI(CPst`KE5uUx2VLse<va!E1Un#wnz+3It
zlAE;Oys)7gv@Rs;mTYQZTd{dK81i>xuefxv&`G(~S->D>Sl{^bX!WX~T9p6rP8RLg
zkA`Zu+gqD1(Pven%H@k+O!X8sTg^+mj7E+_8b<zBdpZ=K{v257Ofa`ko6Pn}>j>7%
z^xe?*tj_bWc4<2@_>j2XK9?xiR9T>3^Sf0{rh2r&b!+$f5_#O8*ko^S-v>(kts0$S
z5jozG;g;H#Vln3%BPO4coYrypJ9)k{GddtkAJn^)>oQSZTyAADo)_t3p6|;_@(evS
zi37DHa6c&<yQ87ull}Xlqot-(p&GG0R5TQ@R|&{*2%dmO6&2CY7|>MYW%NJKA7tQ~
z=^G$=iM@XFb;UM<aU+8n7)7H(i|ry+zrW`G{sZwNn=y(lwfTqXQHw)<W5JfDO~v2v
z-w-wX!k>xtZ;f3*m6xz-MCEnRw3z$xYl1E?dgits(Wm2dzoV4%))AjfMA|mS>VaDc
zrI7RD3+p1Ybo5V>f7k!S&<H7|46xg-cD~%`8Olli9c<!H-stLHoUsmZ-Pn3x+ReVv
z6QkD?LsJB%5pqrkIxh-?78_mRjjkI#s7vK{x?9Ms=%XY28QQ;?-7#scY@Dq%vjJ;-
zr23Dxi*ZtylNsIkBj@q!Jwx1(--{lsUXG-<UDVeLJ4L|j0ss6)*BX2f_Tp09N_&3B
ztDw)Z_upZJu$z@o=Od`f$@A;56*J?TNh|N`3T~dx%MZl~0}s(h_%6?@G!{=?H*8{o
zIJg()vpKYJJwfd5K@w57zeg5FBs|9+4qidWCytJ`r)gVvAMbRE>WY>sm))C)W)i&@
zE$G#S6wWRG_6!k#9KSyoQ+tE>n$xXI53)Ng=9Hvce?m)TA|3wGZ>3Acl)y)Q<tmW>
zZr`W=8_EJUY+|dLya<9vHHp~(h;`_`eBVgFUXepUJxNaJOy~)yNN;z&B7q^p)-gl$
zlJ*JMdA%nl?Ghq`-xl<=g2udX{7>I=QdLm&h$yYJ6kT@Oup7#jHE!KEDYaHD9(ZE4
zX-BN)h1~alX+At9Rc*?>CZ=ek4`XYN5qkPxPe#PAL}|qudh9;O!SxE^gWOW(@bNEU
zT3nLRyW*vb|7J7+B4VzTaz8=<yv4@h$7WCj9aU<{hRG6WN4;(NY#E@=`c>z@mnmfD
zYvMF-ntezq%53^-mWBjeyK6oCJ`5m+A{34qT$jg!N)Y=YxM96nzI*GZBo8dki;XFT
zgUXhFH!ruX@~!3f6+MjR9~KE)D*gl#a#4I6asM6o%(+Rh|2-KBXoK1);0S2~{s%b!
zgBld9{sHHI0QG-f`d_U5yYwH{3~1qK|3GX1e}M0Q`~EL3CAJ_2oth%2&o+C8@Zjkm
zg`g%c_)zz%r6dfnONK5s8q1|5lp{sNPK#q{gnI9QwzmJqnbuL1NT?;LT~7H2?3!h$
zghF{{IZWN5{c3qO2iJMA1RaDd@4Yt`Ng=+^!i7JYR*pj+pw>O*;kt_N%t&lBd-D5@
z5;Y6`0Y0wP8PCM{(T8?3`H=5_8+13S=N_=%$=xm!95NP_sX@8Niod{h4ZpEu)3%C!
z3#wWy$>%8rUC0e~_ixLl6RBtVBepjC8{T-FKx<Br?qgtCCU}!PCV;WiHi0#0*Ycx_
z1A9<pm`lV}G%?y9vi;Fa?@JYD!ia$4LO%V6$fZVCLOm7{{zjTw+d#6!=1xM3<UNJm
z#D4S63J-E3^~dqJ5fbOC`~wj=rXba3H^DiP2l$+d<_v3)2-TyEpVe9UW+NsQP>-{R
z+nmyCm$8TdN2u+E(?^#N0q-HplAgsEaASi5+ihoYKjb<W*!X$vPlE9ZJ86$5FAC~c
zgSYd9;VDP(a=p4o>#uA}6nc>E%*V6hxVtnvh8L?4*BU8M5s#$zX>BHOdw<)jXGvb&
zJWJe%^~^beCsRJiQ$Wr5xiy@~A7K0po-g?ZUSERt*1f`Y?l+10v?iLdQc%dd@jeww
zvk=IG+vPkVT1AEel5rwa821|H9GVY+pB&Ai9Mr)JXb!F<;F0eZHO{4N!?D)U*H4JE
zVPjU?B20_SQN67X9NeaMslcVI=d4m>I1FVL`4;K^e&Yd+QHbs#a?qV5l`iG^h~^E2
zXlhOF)gOz<#tY6M4tCB)R|*-g%e}g<c{}uSPUDpx=on@ml9-kgODW<$=KQ(pe!gd$
zL(&M=gIK(A#6SiQaYsc)skf_}k}SY?K$KWgr*W<yx?~n<<bdy@3K1zXPjB%gEj)OH
z(hWC}&1gb41_rggRtNIxoC9~i#IuXdy!HM&{}b`N?Tt)|3_Srmr<gb7Gu2xAq4ME3
z$D%c!nYMdPmAR>SrjcCasuWWtWh8L>36g>+y;)~}7%FL}LFHjpil>W(B*zG8<B7{X
zveU|HHe?9TYW(|ToQD*ijo)&5@U4bJQm=8&zT0kdAbmr!i&H^)uEuCs44UCLtq8wA
zmI->Y?LH@Bv{qBYu?E3gcQ>q^hVE)EO3E90R|(J`e&eZFi3L9#Y*Ld!RJ`PVJqbnb
zRkYJHNbx4@%YB-835O3KhDwCA;bYgAe9y~ojxc`-S!;k~Yz%|by4Hc)y0tA`=&Acd
z<=jFe;r(pxz+9&_zCw1n+m8P4)m5D4C-+<mK2iw~Kre_*?xvu0dK8+Wj<m(OKW_R_
zoi`e<GhQJA&`Sz*Wep4~n`i`k<PvQ2wt>MC0ctA7e-vu_g%kT@(qddwfW12hL3due
z-8?uT(Dt7B94^lxMStL*-;v3{7H&D1dZSN0L2i8DU&CsEg$-lx+x<z2dbfPnIhD1V
zgc+A+fX<fxhGGZdpV=uIO}GaZcmT}a>`A8Z`ICh29=Uj=3VgUrqv$3oKF!=e<@5pg
zM=z3WctNhJmNoNcTu#e)REKY#ID}q${Wl8^Sv7f=N@udW_o}+4e8;H0H5_~l<!wTH
zGsv46+Gf6s0cOJ-e){z3OvSrq*AZd+&kts`PTg;t@XM*Did+!w6dRf?dARTsQ<Tdh
z18MEXuuF|sEV{W7N&WKtL6)97jLov*)*1Ahi02`~*;zv{k5Z7tIn><P2)}?LbOhGo
zid_G2pfcL_?0L}7>+Q{QMY>LTGEz|Z^?>uw&WdzL0*QPbH@fUh&d1OrDDB<IW&-0-
z;7!hf(r%)u)m>tO6}8i|hj;r@ZRm&gkgd^-;IaaXgRd=B*uDcfh#7elphSX!B@dB`
z4e;+*8)hDVr1py}Ud|JLnAQ$o=Xo`Qw#&*A<b!0~d<x<j9V(4Z2)YIPl0-vtstyEI
zZfWeKSXREOX-%Q;W3XTzD!m@sj95zXKZ0(0P`NKP#md&4NUaf#o#4FzFw?DBXbXFh
z)*XLYb~{4tHw|#ztv_wus43qFNJ)q%n*D}~VA3*-z&5aTtJ4LL=D)3^tYEOeis*|#
zB5orODv;@PO-aCQgnQfkyT(y(#cwl!CXA~9NcgXB-XO_~glHw};m>A0F{V#Z|2oz-
z>{h5=M)mvO&0jec0lMM)^6t09bWl`S!UgJymUDE`6BB8kMD72kB0X6as3+J60MI}A
z|3py#=O*@FaCLR9+oQADQy!&#)e$!7R7y<C8*q{*+*S6hAI;7d`)nlDlp^=_dM*M%
z_g9By@qPtKNp0YZ`Ae<5NL_5qzk4|_Bc!<*-4aeUlyE22Qx0}Pe(Mu7j0EOPijO^F
z=J_3C?WNG7*-c)AeI~5l+Dq9fz+7Op`))%mUOVdN-8RfPn)$ebT;5h_{cZea8!37E
zCn#e`FX?J%=!`x3;ll3ox1Mq!#2BbL=a-fW(5}z*^!OB~oyJ!40SK43-4?71NMEbe
z5*1VXt2ad`S_<@=w1pJ#uq!W2Lq=XXZxPC3@z&H@nOUa**Cc%pNYIq{<JWxV+BtdN
zaPjc0^-g-ri-44pA{m?<7?h<F^lNcNzY1lECh&JE3IBk2Fyj3=Qsa<)WVZ6s9g}H1
z9N`PgDj|n-$O(+<v0P-Lg;HiDLgJ?juo#D5Q9PQG6TPAcOeF~2{t+b^0+BF!i(>?K
zqCbpQVI6+`OhXS#I&HqnRXOElq`N4FXK4`twBz<;Qo!r(kWDhJF{PfhYEC`nXbVZ1
z+bbR-fJ9&fw>})lAkEa(*xugVE>|vXLYQMuzGVJU67z#{*f{W~Zz?=xFiRAcP*qm5
zaRn^0ReTBZIp5xV6EbVBv|8Y#BLmo6jaZp(ATNmdu%u#QOBmW=QD^b3OOt%(s+GBi
z9DDCd#TBry-SsSlN%aDPo5m7A@i7I6(_m^N*Qj{U;fsBgf3n_5qV0Pc)ax^4>3mhB
zZhGH#?XsFeaVm9@j<pujQkfuQg4{gi@nL>7z#gJ!DqO9m9?FmF0>ekHge_C)nQ2Yh
zN?8>ubv#su39XQZ1*FvmNrsdq1p3nGPVYtk&M2BReBxf~Iggf0-AYL7FgxN*4=cmy
zB4F!6;2^Xul5mh6y?eQ~Z#4k~114%+RrQI5t!r|GS*DQ)DxykZ^BeHGjV;_`f$~vy
zT7}->WFvQiyX6T9h+Z1*PwqMwZ!X&Seo&Qzo0~h7?8T38oW^%QE`Ac{y>=D#pMXIb
z(4=RtNk4TI0yO~ZXtzM0Sp}4x2oQi)z?Uaq(n6=pjKd*00kw$gQ-^64uh&fQyTxX2
zVzkP4scRMOL0ha=qvy1jOQ}l~-g3c5U@5Zjv<D+_fx?Fj{X6{B?7adWR{0dvVDbPD
zu5YugXwvK<l!6FzYm0F7&z)a+64rsgd6>%i;N&o;IJRq$9sLCCH4*Jk9n%TUp67)E
zJrU3YQQx!~+9t0pl=py$RQgE4b+4H23lYwWbJ$4j4&*EN6RLawHz0b=NhzKTp05WV
zcl_2Aa7^6|Dhx<b3t`pC<ynm(deuu6>IMepmDgYqY*d>weh~J``T?Yb_6MX?x%RB8
zQE`z$!bZNpoHVf|P9m#kW$6-KYZ9yfbRQiZnHy}e=MBij(4+a~FzN9GDCH8ge1|%;
z&pm_8%Ne(Kis^u>lS$Jb)n2Qpa=hk1<E=@E5>!F!(<sId$-Bz2TMFM2Z?<3&8gH6J
zgYzDdTE~sPus+@8WBSo$^PGi((!~mq{bHk-86JMG<jCL4h*~Sa2#0x|H*for2v+@F
zOBbWYW*t&H@j##(_0=t5PGq1lVUc>X96?&f5G#fdb}HF;ps+*_By1==TJ=%f%Q<ea
zktOSYk^+G^;97I{5q{;je?InwqVYkHbNZdLCELGTGopg3PC@Hm%Hz%k+Ku-9@8}ii
zmu-E?zYJ95J8-RLCIIbU*Ipf#7jH=M-T%eztRRwVyMS)|e_=gpO*9YD&SU&LCISzr
zAN{|x5n<53K}%>s&uHut9S>WK$^-T899H6;Pg-kI-y&k8YMrIT?y1T2_qW0fR|AX@
zOyvu_HgXD%p}o%6L%bpm&ye?N@R*2)8eAoU<l#*oFN>RK)87#0tz^UwX}~Y81U9lG
zp^El{?rO3_A<4g5C2^dKO+!h^`#QXEM%`7CWCjjeqZ)#N<NQSia1&U|Aq4=3p9Q!N
zL%q%Ud_I!)<R(pd2k-?xmzi@|0hfMos#z5&kUAOqNlaW%Ey!K1148`SL~IH6C@HdX
z`a{14732j8bzfVn*H>2ZlOHe`pTq&GjMq*$TTVjkEhzO@b(=nWGowO^;Rq#7w`oK*
z9#zUS!*@+mfgT(BmhfPALrv=%y+LwNqe8}JG6jUWFdYfI2-kpwl`CSKLTECC9r@Ur
zy-N6l0NK*LvD6~=Iu)hP%bX?E#*J#<Xw2p$ElY!DoFksMj=R+7Re=63e)2JFy28C~
zGmc?L%${s<NZ@SF7{6j&g9G>)tkf5-YM;VG$0Kqm*+bmFm)+47ZnjH2FSyD5nHK_W
zwRp%k_c*D>4vLXB(!nu03W3B+GT{3LgX5h&4lye;0mSYSql5USt$=rjK>@32`_!js
zurZ3dy)^WUFxVxyl;h@BQoSFfr-8i^$_i+~W0GPxM*m1_KD>4by+kwrWx3B_Gra@>
zgD`w-XB|G5fsMXUJ|33PG+AwnzsWwRB%hU@-P7KryC;X{B;SMEY&@;XyTac?t(9`N
zxTE`w8MI_=ca#rXW*7o%v3YCg?zRpl{Y2L{;_+{%FbO|OQqKe3tUizcKnLU>jBo}g
zuLd?%!x3~fIk?Ir#9;xxi7;il#Dj#yeLSsP+<=hf0aZ><bwDeJ{AOE;l=5_-)D>>B
z=}_--EA{1Dy{q(Q1lRe>`eXn!%YCSq1Fp**6hy_9s=Qfi*OqlsmKHdF;7zf2Yrkf|
z&sl_<9!7U#Z3811fhHU0%pZnBI@ke6Nr6np4G$DLXP`z;%Y#7T3mIF`WE|~flm|Gj
z|MDV`|1D-wbPIN^=jY#7-});@e7`5^Sx8V0?<7NJ7DN3o0$5Xiexm<8XMt5Y8^d%)
z$hL3{6~-4liR#-b&IC3ht|RzFRkoEjfa?Q{XF=ZYm_+*jERNqlj3OuK{^Ymhn!Nfr
zImE=8mPqzhQ<Cwo_h-TQrm1b2&-~dTeQef6*}X<#6#{KR4GH%K(LOQ{Pruc~N_u)A
zs%H-?)j7Ig2|jYQBe@R+Z4lWJdA5o2@aGAol8O+}ht2TpG4<QGK_rb<gTMD!OwnkJ
zcjZ<r;-cbsK&}z6w8ee`(0nt%!-<rO3yY|R04p;CQf<~HqX5;q``H;6tv&|AgHv`G
zRuix%*a)Ty1L32u|IAoyE?qRYVo0b{rhfSJBb38e+Ego4>1#xx1c_qD?}%v;-WpLA
zI$nrwxQH?TnO`62B{l&Xf7&Yd?(HQ7BqxaRJ5Sy=YaW4{tVvOS?s4G5_QRqI3Mf?;
zmbW?MCC3Ksqrn-4a;2s*MG~~~q7LKieX>hc_vZ^1BH(aYhJ(U9GWEfqP^8#_pFGT^
z$EVt<+haX|D9#L1C_)$BdFGxsVZ@`S%`z#LT5sM&Srw8`FX15tO8Mf*pHcsHW<MkD
zG)hO<-H=}+#rfTET?fDONxR<n(BGd3YX>8NpFPzV8FUn(re`D=L$bmO&&(7dYs$=H
z-+}&-RT1m}m@53_b*SDee7eu^9<*AE7D;YC2f&=yQe8q!`pcNpzDX{Ms!_+FdCbor
zHC0r;WH4Gj^JC3<EkXbBg$OR~>>F-BiV>$LwU$kodf`9<JBq&H5tPHO;&I~MFu=`^
zXj=&oKJ%N$oJ4>48hr;8cU()Z&f)6r#}|$C&@Rx^zNt2P#&uF8KqH$UhhuJ)z3^2M
zrw5c{$MaQS8Qc5`<0}hFEL^fTh2ao@*_$tGGNTpFcosb@{fwzRU*eT|d^A(~DVScQ
zTIgh|3o|Ee4{SnybvN5v*n9=1@C?{OaIG!<t(T$wY-(Aa*Yg(V?_!b-vWDnG=-yZ;
zR@~KMLp~(=e_vJ!FlI~P8c@Fl-W2H!Kz4Kf5Kcl<vcc*>4`oY|oEM+qoGTNGn=o(>
z=m(Fr9R8%WQWJYXidA9~b2lh?%=q&185v>UEA{8`15f6Z0MG!Zq-vvrP;v>JGz`&m
zZ(|Q{mCe)7x`hwNI?(;-=C^%;(ptn)_ZP@J18{Y%9j|Pv06#a1IR)xL#`<sryo>>)
zpcB2xj}cjd?*RQ9)l1TvDPDfVV}~Ld+UZ$OC4RJEzvfQ!77(f6PrYQoR|KpH=|g~5
zn&R81GNHA3Y%u(^BJhAV^(RACP^^y7u{SBSpPk8F+-llK$9zPKIzFAn7^X|S=?YLD
z6h4M{$wdwHjh5`+wnxXRMh)2QC~3(wWe@#u1Lu%rYuiGQOt^Td)}fY}RG7v&+M@y4
z5qd64^qbe#7VclO?49+KJEC-gjh;7-#P~%V$x`qBjea4|Cb1_=n(~Y$HF=0cN>C?7
z1%C9}%i?qC{jsWP`~!oYOaz$+eTV?NFuvV)>kENCUN~E{%xzct-D~{h<g9b(=D@0$
zo1nP-%;X<`3kVuRdCiMFFi`mE64CK>bZZ*#K2IdH1sh95amDN#CL~)<>=}GkY>gQ9
zsyEWN6qfd@gRM8xHJZQ4qQ?!K@0m^&)=N%Vkp05iUUImmn)%?dFA`#8O#CVK7wjw$
zFwEPeDi_^Empj6n%Lk!;(<oN0-jv7;R%igeO!jyweFW_Y`^96TW?hLrD`^)zMp&(I
zbHpm{V-a$IZSr_)<D_X3gB1sonw23cy7^IXn>+K_(^#93+)!7yaj(IdJ10qU-^Eb+
z51Y$X4n<YzCF}c|T{@5TNCQuJsBM6il8Cy*`|D{kjWKd^=GSgJVV%@8wwpTY%d67?
z8<yKIGymY0o<X`}JFl}!mG(Q?TAcK$&6u{%@I{Nt94e`PyKelWT8Aj%o%O9p|0vg?
zFA!WF&8>gDzBb$n*3$p4Egr?s?cBWh7e*=RdIgmJ$MY-cQhF7Tq1d{OEoHbBW!`*e
zuf%^rms2-R3<4E4{W~%6hN~a^|3p;eAbVUvO2ucf<dj%R4GFd%8<bqn)<)9qNNbP`
z8s4Tv$8*Cynx!=2=8sh$orYN*33JsykKK5s(6VDni!ES)9z?Ob*oWE3OvJxgEbt3F
zByO#SH)gH5Og8*vbxl*xkZ&ujpDL;P0nE8hpeP{7%rb!Bwt8Rh58_W45yR_6MWVAs
zwOMMDr3D*mi}oEq<jqw5sD4h}WC#I%4ot63RWvsGl}=%8v(#Q2O0P#o)H!gV_48+%
z{Avm`^)3nT^)b@=9g>Z~|I^F5AoK;X00)N1WNK8p<YuM6*34M&+U9O(o$*HQ&q02t
zdG3#DnBTf-L@>p<K&mq)V@S^*$0kC1#>s$(n9N@7#Nta#<(Gt&qMy5qoUdxDTrmod
zywo-$4E@#Ky%L-u8n2aNQ6>D`U^?e+qLic1aHTQ&Vnx*M5#!lS-1NoAt5-ORz3FRM
zEtx9<EP*nMnYXF1m+kI-$*{Zy?<cM<BQr1D&uz+Wi4<toPF)D2vevNJ&t2-JMJaB3
zvU5_)N0{O`C&L<X^nC5ZJ-*Ty4bKpq=m`mn>0_=an-~e*)IT)6<6GRx>A&ruu%5|D
zt%&!aCwas6ZbAU>$;DeC50mF!D1+_QS&3h#`M`VX`uCB?o3;4Ik)?NtKim&Yw_z>P
zB>vhL@Byp=J^awqtY~}Or|*@jFJc9lhz|YKz$t-tD-~r~D1Vl(vDyVS=!BW)B8eUb
zXP<p$r;%NNshJ%Q<5{pwemtZWx>mWze8+|tGrun6dv%FPX4_6b#aEfboUeWT+Gn>*
z^QW_^!J>?gLwoZB>w04cKewKcT?t;vg#4^<7i-N-!VjMFy{Typwn#rt)#P#R@tUO^
z5!(4$ikXIJI-w>ll8+MaHJ>+*5$lc)Q>2rwY35yi?-wdb)A*D*PNze8;?L^gv-8BO
zy>zSxy`s}kat{mh{_EP)klG68(Hx6?xpL7lxruW!K_cQ)m+6QjOpJG`bV3c~RA^%2
zzx-ZFNTI=^mY?-)d(`y>wtT9c9tul4jYG<_<h{P?olM+0S67R~So0YzAjImt@jlTH
z=NUleV3a;SWnXv&C#CC_Vhu@tmax1?&c|qc1`3f`kNjY{Qgl6vaq-PIyZk(X(jxx7
zf|`yBAhbk7Ezz=(@u{dx0Bc=xRlTN1spdzMCVpY9ccQ8g!bl7(6rQdoY^7Gqu=Who
zrpmLsc3h;5i4DnmOJhrI)u1+M5~d}#Ucr&|9y#TCHk~i|_iP%WnQ~ZlQ{I{|-eOH_
z63HUT=(`qQ4;jBZVghDP#c|Ae-;kjjdS5%=z)r<Q@8{LTk_7&^!K@^cqluaH4SR{!
zV6Jz;78wj5G=cC*wH&5A*A!p-tK~_2AD_|k){}{_NqyBhDS1n)cMiuLcUz=6E1i&5
za^6m*+(B&JEzEHXb&mO0c-(#I@*yua<gMYW7uSM)lDg}kvNw=DV=|b)W-W(Ph3L3g
z*qW+Ee(hT3?P!1sVU;2qNUKYPZjTti5RnJT!6gc9==Ip2UY>FfR|~oGGrb$xovyLf
z+GXH;&*y-HXEK(f>G;m2ocQ-9jZUhf1c~m}3SJJ66CfY3iuc_A<!kOP35If^;$x8w
z#8}S7=*9gfvn@h%j^*B=AgvMS(>eoml6T4g2`S06QUFMAH%~Mj=R>&IQ%e_?TOfVs
zvRLlsz~dK5XJo-KRleAfdK^#*kS^_5h{%^<x&{@phd|E*eJvDOg>10jodk*0+p4&a
zR_-HPAJn{s?yRWPh+JL^teTsG<Wu5#x*$Fcu{|)bHkLP22a~<|afK);w)oF5NeT89
zswbvhFek?S$;RJ9moV^E@S`GIUWCoIQUKskCJ!L_J~tES7U(Z%L=U%WWB+vP{IZng
z_yZ#XovB8e6w2Jo9iYbi<hdOb3!@UB6tsq|aCzf>f#0KLp!CjA1%mrV8I`+!J?>~6
z>P_b<Rb%9FA<x3`RI$U@rWy{+{uK696s;`l%A@U;R{E3w`6};6g*yyy2b`s_lXv9l
zVHLWVg=`_N#veZMFu@&~arItVuvA2F*3ix!Fx-ts>uyoLIl-TnxK$*FKA>$>3pWbO
z&ty9#`W`(YhbpW!5?rHwpJs;l<?P;~7_epR_@Ycs9mSEjk*z(sl&jii#HnNMUv-#W
zvh{&ETFd#F$^9#xtwfaPj(|KUhsLT4A<Vx4N%Vfys(paaqDT)>I|z9Z<O|9j9+A93
zj-{??gf_hzLKRRCU}UYWe?=BA!|Y!&v=}KNMQCU7RXrw)*ju@!9fGwf`D@b(7i#aE
zV!Nyf><aUp7Be&H%M<)lsICQ_BkM9n^!c6?Jp+K+X?)33)8k$8MHGEQc;0_koS=B*
zcLs?M^ZuawJ`Di<gVBYE;$lc$2<*Ww<KN6Imdr%`N>_hevHq}|-(nw;9sU1k0gg9U
zI3|0r)R$cLR@)yD!~~W?Hymf&`y+8QMT5(b0EHvLCoOr^#2_e!%;@>c*uDE$wyFY`
z4~wQ0-0~J*&QZ118X?e?Gg`%Np@{QTUBUC#?!g#6!^D3jS3RuEI(W|8yng6wtGoB_
zIRte12$*@wsD3@t-iVjGLU3oEH@6+{30v6caFs23=fdBJd;$4VkRMDy-9EypG!Re!
zo=5AZbtnr`eS4k~kmri!(Wm)}1?-|hJXQI#M8gr%eLSI6LlZjkv)n<>nXcVjfIdwB
zd-Zj03^LNi6>8nFL`=9TG*-d}a*SR1<TKr?11qzFu(NztV9UD|FP-oe!!v`W>cM8D
ze+~_<;K8Ux(Qii5S8LtERg1;CiiSQJk(?Y6Lx8VNw1svs=RM*MX3EQVmJ)POyFKr$
z0&$EdB!yqo__FY|*PaTZ2Q+Pe=2^2sdP?YYA@x@8hdzYR>YAB?P_VE}_G`dRu5<X0
zwLQZ;P*VV$Wg(K~aNe-FAjJ5`B$Fe*96`9<(<%FPDQsZa%5SLdR?HK+4#GItFTe%O
zNzfSQ3@?0=Zx?+!Hau33;B%l#<&}9=gd_4B)u{E(ch6J&<mF6&xYnP*yo!(>g2;S_
zKueI2MuT&PLdk-y1!w#9P7|8t09F3VI>f*d#gJycJqTRLgbi|N1itMjy{YtCY2IP+
zdaELcBLyOdKI``hdxF?eD312JaJaEZan8cKno~4-;|l9hRgv&rSj2sA)ohYAlUg%Z
z+B+r{dv%;w1jZ2$ZCd)CU230~fV>V_hF%USdhbsW#{Q03c!hOglAWq6rklJl&AJma
zggz2+;Y>3*6SsFr;cA>WT@k5MvQ-2F?_KTA;C;t2Zh;VfEjlDG<TpS^Iu&U+`$rk6
z{EI5-GwKiXX*YnJ{I@^4$)DRYZNgNhZKaZ|<2Se{a}1QL;w%jL!yJW2lH8Ug$qpuB
zI9B~#)M)JiHafa-%TqI$WyKjH_dDsaG83`Gx4GW#5+HukbR0QgW6-COEwX|}r^N;9
z?>ie$86)s^zA#_*?c>p~KH>LRnq@^((@Z{C{too0+v%FT$Kq716s^#o)KbWlHQzb;
z<`HH(bqNXxK3U)A5bFW`bv(P&bD%dQfo7<P>a?>(cQN*U=z5Vi^)=xBO!j)v<$0M)
z#pq#;bhOhGr$F7S*D8B!;{^Rotg?$;@jM~lscwMOxZw>ZGc&}xg>>dCrnv<31vVK9
zT4z%<9N+O`kEd(4@3+tiC8#9;?|KvW4!;=j5QfL=PQO4{pbQDoS&~e!)45)v*}eMx
zxzV8))Xn-af6%`v&rAar=;>o6c)R6s5Dzu5@i%_@1hLtTjA5R6bh&U2bIv+a5h8`>
zhoyp40^WG}+N=XfN$_ZF8wU?D*~xx6>uLcovTlL0#w4d=U|>mu(OvbHQ)2YwJp_ot
z8HCqD@XN?PN0zntNuYkR24|1$5Es?J%x2yJb}pxt1H(PB9iKsGJadVge+tFdfHTHH
zTGAye6Ef7exL5r>jSUzXKtE5kY303;%;TMd*BHnY{PevaFE;Q(V`^CM0s(|L7QyKH
zpE6i{;Xl?)LmR3SW(4;M7R0tFu{VGmR~gOi8aEuNI*ON9h3t6_fx&HO{V-rVo3$%I
zyj(4<@lC0!Ug5*wQvE^1@5@|+RR8pc(W8)JLK@z;-Yo39n&`^^!&+<C8liVcP8<rA
z9EKlR^m`)otF(lA$`1-suD;>DW%}uo{{>6r`w9>K-c#hf?kn`1CK>kw8fN&YX?U|H
zAaBg$GQkaepq88`kP6G70_nT(i2QY?)@Wb_fYgVG>o4}Ejq{rV6ajk|{6DPI>x1<S
z3)uNuO+o9`O79DwB6==!orD~jFF)Fx+zt)7LcWy90VocFF4Hnn8{X;iIIZQrcaDqT
zscCRkkq|l>Kq;eOgYCQeW(irSdPcQtnI7)5_hISzDhl^Zl?5((-PUF^uUZWJ#VXK#
zIzV>obGB9K$!NC*%5?C1j|Nx}q#BQJ0HFNGDtUkbe%pIRKZ|~oI~I!5*op#|+=nwZ
zK5;1)f}9cb*=S-oKCuzI1&gp)#K--e#>O&6h)(Gf&{TDjWJ0q3YGn9!5y^o&ykslN
zbqGNxsBP23k=@@b9MCB5U-9N=)rnqWe!q|k;90Gn2CNC-pHffrnHVrx!=t*gAvWAD
z%k=;g9-x2qOT?MDgq2x9dT&!tp{3j8u9MjMJ87oM-Z#eO=sK7e2{#Y36RH&QNMaOS
zcVWLD9Cg<}#1otvAttaVsvN?9+vqt_cd;uWz%pAjg1!nd(1BZxnAO{w8KP@Q7O0<r
z^b>fBz?0)eU%kS3&Yjzg-^L}GZpPMxncP87q2?_G&8EwT_n{%qx|sQzk}(6Xr^lZ!
z<3CB6NPVLsq4TGCbh_<$#W>NJ^@;9uAZz2M&CeuQ&7^2jGJ|I?v#{=Rmt1?S-KgDX
z<<F%{`Yy*-)k(1YPAeE#GAl;ACwj*S#l};5V_cnsD8@ETbJca!vhd~;Asr&NpLyo}
zxgKJ<GDF1#e;o1oH4u>#Qor{+V+Z}695)&{=zDdk|4hr2IJh~x0f)wwhihtlrXf~N
z<#O=p+rM*Z7bNy)rWZNB2)l&3i-8=l=~V7P`+mIF5U)vPe}jBV?qijHtBI!Z$Hwn`
zeUc`{1uKOg(_BVI5;3wH8zORTDSu(j8Y!`XC>?gY3d|dvQ)AT`6evz>J@cQ0dsNxl
zo0Jsr)M?7~S7LP&&-LdgI0-15#0eU!W$77auuV{AJh{36QH*YyUVzR_?rhjXj@F(A
z@H(p1U4$TxZnq9-!W%HBiTq6*+Q~tMj=R_wvnu`o3Iyqjoaqb#D4T*ZzwSOiL|!1?
zu~`KbhANr}nXA$B?}5Jw7QX(Wb-|%zynfmvsAGez5jY*$F+cYmSRE%1AbpW`30nR@
z>NADg;c5(RY_g+EIEW}Pxm$B@qNk2pra4i>et}Z1V0)n$6@`>tle@liP>&i!M?w6f
zxn1yjwWcq8aAw@LyvD`H<?LCZw3c{$59p6p{It3H^f)$WQ}ffo-L5AHQjm;iqf@~i
z=xoHpU~U3xAe}&5%R{*W2n$qx&F*i^Y=34Q_lkl&Ym`|hT;R59fpeo7<vlD(ru!!_
zZj&5#@d;D$xkf{;I#)mBQX6voM3*Ho_+wgDz@xr<J2w4=6r}%b#Kb}@@&g|8zXvKR
zpU~Eo5T)BDHja%Ch$gE$LIFwA>AmvI*v^8~+bXRr%wW-C`w@{+-xUA%S{7HzL(TPT
z6uIkl7ZilaCxb-g1#u>ojm23r<3T80he2$0)XaE;dWyB<Zo>3OlFM_GyG7_j3}mhu
zy?1)t+i8FCpDH0n&qm8a@BoxZm?r(#wCE}l?{I=3tMhAwgVVV5eA$*#N}AiX{d9cZ
zi5)w>*cuE~3BU|6zB9_zh<x#@Pis1I^US27d=ysh5Lgmv<dqZmF~o)$^t#jp9lmPs
z2B#AMb*9Wnj5h?FUjG4kWe={C*XE4E0;cY#KPSPpZOl;7DO2vmCPcykZWYagX}JeN
z+yYDbu`PL6{?*evXT5<l>Cz4D$t-PUPnLve1lPOspM{*du;PUZX+3{52BNC<(nmja
z;H#CzwApfpKX-cGcsP~|YdZvJ;BtH|NDDY8xT6mKT8&3BZzSY5Z<P~pQT1E{VVye}
z?A@FwQRJB)^*JmxNJ4ZAqn<c%(vJu*;q8HlWjH##O|^DjF7y2T-lAel>F=2iJLlaH
zx(^^oSB)O#{@4G0a<4{C$KGvRRPTI0`ZRAsUuZ$J@PXBd-tebXc5hP-8?D@MT#eNX
z#m1Kz>>~5p?dyR^7JC01_iB>-aj`Puu^5}e{y6sKSA{b0C?bb7*fZv|Pyr^E)?35r
z3pqq}*x#Z*0u$JL@bC<_(T;#nAt?<?@q{_We(2gXQ&deDlT@ik4K(r_R>9Rrw%WK&
zCQAInJl{_xjUlLa6H2e#z?x`O|G8C^8FnDYe3^MjG_*TfSl$RTzi1o__-eQB)&XNP
zI`<<w&R49v=x@Lv2i@Cq1z2vN8LdgsdxoF~c6@&|-jrH$TVQp6!#~C6h<JKAtA35{
zVU2ch^5-c2;$E23t3ciK;3&`#)Y}NQJL`yGa`>h=AY0fkRsF<;=8A8cHxTC<CyhFi
z{~kp>goY^W5a5D3(~502Mx6T;s+CC}*zySK$aG#5q!G3*OzyzR=X<}zcFQ7Um(X?E
zv(a62Gt+cfjL!MsCVV1=MzlsFN!&w%e)WHv*p8F9YyZtn&ejS}To^n!!R09%{AR-B
z?h#Y2KPvtrxZXK?tM?sQR$-GdcCA}JqIjn#c>A=0Zs*puW<Tn5&@ii(QEZyBFx5X>
zLR+^{t$+ZAnw~t&WRuxjWlL9ASIgBj9!q$r0CSj17%2#`te|}RN4+;oVu7uQz!Hx)
zA7W5ADoNz=nsOfd(Qiv`=9NG@bq=I|>K+Vi`G=(8PxjhDE^0^-Wfz`0?&k*sQ};J-
zeKl%w!BXV~^rK9n-JnM7e9w|0tL~eA`&?@rPqB%B#T~_;Y@Cu(!%-6Ka#J`jbXe()
z)m^h)qc9!MQh1nB?@{CsMgc+VKpBrU4d?m*hqhf-@e&v1yi$@#%dbha=E{%C|8jeT
zwu(f7x`wCj|7|3wKcG<ExF>6J9CdVv{(#z<qD~VgKK#q=ogt9i8`NZ8<*S?4PH~m>
zCGCkO3EVOqLa2DBf~zgr5FPxtueq5Z3hGO*?))@!Avt{xgjXQVVsVMs0k8N*Cl}xX
zPO%G9d2dzYU@lKh?x>_rvnh8i@_)1Sgvon1cw~O2?*E;r|F^7iwys~$$*u!S?2{5t
zi|Q^c%_ze*fVmzlC{4s$-axlhU9<4(XFH09Jz44Ww%rx3#=8pM*9$sV(F4US7V;xA
zq0T$MJM;5pH@%gwLhR4%W8(wc2p$b@*ioFA^69QrJdfH}uTIvhNXyvxk55b*F9;we
z5!f_e+`!2|knZB0EJFRz`RAyl=B_8YE@mhL%7c~Mf*?975_zwWJ}#0tnsd)WOK-<`
zb!z3?vY2=_MjUqIOaXE3;C$4?DFV29w6yfHtp7-4(*Qu5_l<Ljz=>vwR0Jp}D}}q(
z0=>@f_f!mR=~hEMILN1ArqRF^E=`zQw(bx(01+`9atQzgh0PtxV4ys4O$29&8D{L%
z!a4A95hd{IVfT9lE5rM^nd+Yxz)@+<{~(SoR_j~=^b;t0eHex4Ot3=D0LA1c_QDPC
zjI;*Kb^lI-C5yG4olnmAV6t2i6fGo$1sfFAfDxuNtNNmx?bcfj*hh91%@p=SrQ!gA
zTZ0j(fpP+|#X)v|k#}uL1ItR4XC>V}OhIcRNkR+F;G90B<hBw_U#&?^KTvK<y%iQp
zIizqeB@xEeI$v4Z0r9NMy2{(Tt&Xxg8&mr$aQnjnX5x58XnMUjh5ezk0-ni=MqTG+
zFd~0>mpfBGu4nVp^GX<vau7D=s&K8m%OK%ZvRz-1!`Y((if)b)n1ve1C&XEBl=nmR
z&!<chTxOe$;#l75GWunrRcys^Fvw-i^tD5c?J^{Zq^k!rJw;WpWuojyrUe8LuKw?0
zQ05rh+<>Y&o-t4Pdrof-m?$YJI;$Cm7?*6VM1V_BDbqpz-|7Ahv<sO<^~smM-vd9L
zYRkEUXR|JUu8Al*3fx}EJWHI~R4cjNE9-~W#rji;epP|1BY^JZdA;b$GjW#zD>t&D
zeR#s@9Rd^3u;-2b-YvMF2~f#Rs{fg;y@sUTkkPw@O1c=j%WH4Dv%lqjmwRQYv-4B0
zKqqHw)Xt3Dd~>1!FVF%^WIlDVba=|<hfLyHTDocEl{x)FoMx=@o`4MP&gKYnQEy>P
zSv8*Of2n@pgT&H+wfvi#6IHD%arB?4??qnZeVd-!K<k^pyEm0-Xn+;ah=p-k!&;Y`
z3M;5I4h%#5Q+5{=Sp;q=8fEk=NXCP*nF_**W;QuZY$XTFE*%Hj-W}q=OX^alH;trg
zoKt|gsowhA^kFO(V@(S1^=gC1A<<7wjXb5r|KZYL2uYag9}T$`n(z?o$@<<AfKt3H
zLPb6?&1^y)JJ=4X#T6$~dJV4%5AGDaJ@`x>05h8jKd)x(&IqetZf>jbti81V!(Jl>
zDp*OEfzW=IN_9NzurBBq$^}yw4We@@t|=%3N)qVBJY_VlKxPS*lbbbu+n?A94^b+@
z(?doPP;9<>!{L2Upx>`jLIbVeEN$}+XS*4W{m`fG@E2;vv-Rf+wudIBjqqDwMZb~5
znFuIu5qhX7HTl7!N8nafL$c@~z8A#m=xg@hX<*1wA^QdgJ3?>k!pf+q_f>D7z-@S{
ze~yCtDu6y}y7FZ>MB{+2|Jg4-(8zDO{qxU@JR7P0avG9Os|)2W((us4(<QHvZ|55V
zOLw`HijMQQuF41d1FsMN*<J<J)p_b`wZliVgpn9TwbH==N6gPohjf%O=^9nqa<#Z>
zaNbP4cPqz8^`9uRpFhd?5_m4q%*BmoB5s_(o|k$b8GUr@_dgF`sF`3Rr|x+6lcZ;F
zLZ6Cet+Lvco(7_2+&0{?wB+y#mEdGu4QVW}p+?2_Ny5|KV$cXoOG->VJ9`$`!p<~L
zZ0G2JiuO|P@f^;6$$nU*^FR>(mx)z!iq)?3KPWlg2`03#bLlB4QMJ5LlzTFZ623Kk
z>L8@4vAQoqS5G<FXiE?QO-(^vH3FXs+`fIezBH^Qyo1#YGk&|G&rH!LB(7n(L69~0
zP8EIuD~sm^b>0CYY$VI+<!7bEo!<YGB!2#Q$Lsl5CSK2wD*%NKBlQHWuM)@8Uv`A@
zmE`ty4H;HW_@~Mlcpf<A0oo%El$lht@{7`#A){`KQ7w<ysNwm!|Gk3UnM&33>nrVx
z_pc^Bc_N89tw5hsA4xpLOO~k#U%W46!z5d9Ny5@`kOuAyzeDj~6E9FGHLdKQS#hxt
zJF!0qzYOM-AH%b_^0$^P{tBGya>ZPJJF^r=m`nYRg@C$olBF_817}Fx?(EBX>UAC+
z{yzX2+Ml^rMaVL{9=6B6;oc`Tdfp9U`HzLJlH?8<#od4T{X-~WF*ElM@Vn@(mAqbz
zahb(g<BK2EBt0{c#e?}QQm(xIyCo&JFtFoU`NZZmlO(@8xbOjk+>2=ZBE}}dFrDQE
z9R2PE(<~5~=h+91f3_Uyuq(v85&NktZw8Bw;r)9hL-0z_MFV|Z$(VNm*U^8gyqVuD
z!KDJ!^rN81tZvSAXGqOk?8kq=a~Z~1)-l$YE089+5TD_4x3DSwx0nT#5?tb|8M>5y
zxo^P0G!4DqQ3@D(qpcyBhz5tZzvTfZC8-IB9X7Bd*=e}~HoBc$vmpb?C-C?;pgy~*
zQ=NVg79TI0<l`-J4PzI(H@U@KiZrg*?J?mYWJi)pUyTi;>cTZKz})`0hmjb)6)u7R
z0`{5@q5vb59!{{5D&-2(E;_R|5twsfS!@7F{EOHMmmW!3PjjaWzQBh4F1>_wqhx23
z*xzVUvyo(a{sDZezS@2$LvT2Gj|JYy+5IVtxE0CK#dL#P*+{6*Y14O{8o`d08zQId
z=q>e~1L0H3_emD!W13YNpfb;!C_}K5HQ-|sH_MtNU|?1f3>;UBuHTullk67;v9POI
zv`h%XJy!m1o1Om(ua3Mw@0@)I?0m4RVk!Fe#vxzJFiPH7{_%L&(M=-s7t@4M8@wgv
zjo^%d5HXfgG%)b_R0xx9U_^_;I2Hub5o7$D?t8VF(b{(}lydF02gxj&N3yZV<&?FB
zUPs0|cd{c?uKNe%k>`BrLqhj)bHMHZRa$AS6=tyX)^i(aKK1Eg=gWP4HE5W~1+hQ2
zx%VY+sItp1c>$+9ybgK)emB<H(~c@sMNBJ~U_9gzZ`%vb$)O0C#};Z~n~A%tkO%gT
zRj67_KMDQo5>Du^?j2T+pU1!$p%fVR&PuScGeZ<mL8W37;yhUn+q1UnX3GO#R=?N|
zM54EFoTT)yC+M;GUo{a|g#njZ(j7x^Sw64#<lit7c$5b4?a}C9wbE`%vki=#4B=>G
z8LTjHlMx{b^YgRCmvg~Gq}aQ+fRjolrc=|GqBPW9fAg_FN~-`Ci$nDHw!+5cdYmm-
zP(s95#Dmu)(LLF`ii+{;g|~Oo_0;?_DKKc1LT)9G!}m;aG<$EocWV$=_$CAw8~mr!
zu#!rR9%Tv4fGO8ERl`sP>6Xs#Joryd0<J=ScShbvARaDYuO~lo7JvoccR~xSEhrs2
zp<EMrb_WAVzXT_(36ok`0_OV8=(Q_gZK4|v<XN`q!2T*0dRfZD11BcXCyYQSF{{_E
zz!(N-c$hJIz)FxG-58LA6HFD-==7g@g~tm4I=mWSG*0UPz*gjx#Y(clg3`|+cZ~-W
zQ;aX5VODH*W=UkE_0&C}-!B>N!&P$$8o0YbAM%<qiNSaVHf&uj)$eknzsx~4))r|v
zBnN~+sv0ma8sDFrd={@kQvNwIui?7dvAkE#J!NeSP)lN<;O4$NxZoBBXuEqB6oKQr
zs^$h3K4#4W3!t^9lCKyMx`97;D6wJQy~sOOrcYNKY%D244GM+_FZpWZ(|8j5>s+IR
zNdcd{p1CRdeoKB#MhW8$?|^E}20Yei{pTciQw;yUoI}Z1H32ArMni7~N_ycS3K~FK
zYP{pJ64kga(h^A6tAATeOQ8%r(c@~Wvu#T&?fdw_l~bUYBdldUnq7@Y#dj|Lr|g78
z_EKrp(~CDR048JYsx@`sAdu25R*kUO=7=ng3&n^kpQ%OIO3-&K3eJ)~ceJt`u)`TK
zN}O{OowmU~p{WnhD=c1I>@$In+9emM-&lfeB7d!a52UyFJfiJr*W`6kb;eHSdwgWO
zF){pCLfhk5N7tqa+%^T}+E+|=%N|=*r8i6|!yXr-F;)Yh`EhC6*m#Y)u}+I)XSd+K
z&(#dn{-U<Kz7$)ee$4(t(>QHSECYteew}x-z*gg2D2@SFe?HZa0mLs_ottNJRz<MX
zYxgYMv%&`lV-+bp_oj6Ujqr;;`pw(Gf%{n8$4ET~m&O+a!)k{?Bsd?r^I4oQRzoDD
z%x|C`N52q1qDG^BmF1s|STTdjD8&Fu{^c?=-1<{L@k#2z9SRaz7UQJ>&&>+xTwX!C
z%TYmg#ZTqyobJUbzW%=01+uu3H$L@r5```}I8;&ahUQ$~TRRD+(oyb9rE4qQwIUA1
zxzr!|x#k=UFH}@TpD6?m84^|JhCU>{JDl02KYd?4Exs@FKcyjg&-_vLs-rQ{+Yba7
zz67+@M6i(btmjwcGK3O4R5&!vv6)hTD0ErM(x0*G)O)HUwBBa;^M_+u2L}qUy_6of
zsbRd!AC=^E*HVg`ypf9IxDA!ESQ({nzbd1){6;|{$qI=<7kPSBg?_uO<QMGN<XeAb
zef+8>MA9kvGB}eZfD$8{m?j%CMwgsP>p`+xL;?=C3a1GmLMtW3>hCI8=yxYVX*>or
zcXhBnVC2`msc^Q9s@`5)a+$l~8g^|||9duh(0Ga@Z9NnGHh^HoH^uw&7dNB>+k({*
zYQwo+UG>8|j7h;4t{mWD7@eVe^^T!0efo((AteteWYu0Qj9tS3y_K@Ka59$s&f5@_
z%ked|`dhjAOuYlgcz_-6buirr!Ls9{U-bsqt8*}zc0<kM*N8l<pMS)5(P`rUhjjsf
z2C^uaR54`MFW6pk0j?xyLbJxRMvcbAFcO&$T75mhX$vN~gX-lzqz(KIhb5W6X{KZq
zirSl$_BO>5cyT^U=6f$>!vZ(&%}jF6&oWOXXfZ9V5gDsLOr_&?5;5~tCHGS59=`fI
z6?&d^7|oT`VHWU8d$#gC<p=<9<!a#{@@dH`#ZBKRF;fz%^7;`Iql@L1(^yJ?Oy?s;
zd9&%6{APpi&#lAk$C&U11)<9->`cTo-+iSu6D0A!P}IGG{NlK0vBgePzR>&b5BMph
zk|oce6W+_<Kg?=nBWHBQh6zNU{CGzC`0?p~npXA6J*2#i1nAer8u~z+7p5PXzhOsM
zo-g|&LEXvUkP!vTdo?^bX%Qkga=jS!?b(<5VXahfK4o*DMt0_0yy0WF0AK4~JEs?!
zKAmW`602?urCvDP`>Ucx#;#;ptT_8_Y}D_r7{<~Bo;1v;>c(*^LG52w<X5fuR#E#o
z&S8oEA$!Z*O6EaZl%_gxwb?w;#Y`+nN`-(HyWB|YPDtqmKQRA5<r_2>ozPt93b%Me
zb)>R8p(;_4Y!(G-ULtgf>AG{ix4d4`T3WfDXZ<*u??>>oN6c2OeY<h~ug2awEUKvc
z`xd1IkxprWp&43Aq`Rc08M;%tL_h`*7(hU}qy_016r^Toq@}yN>p6Je&-?!Vc;DxG
zxVUDnfivf<z0W@T?6ubSvyOK`XVfN@b)KC4%{5*bSItJ=`7$Bvt9F*AH{!`q6tVw|
z=58T8r;WvVyB{a2w>!bj)zm|y`C_JMX;_dN>^4(3ukVtKS)rleGseeJU{Q2JKqMOa
zZdO^`IW(}wu9M}faCDGM=x1SlwNzhFaXoaYy5_?#n*6m@BY9lZIiKQUpc}n|c_8`>
z2^#O18fe|k#XpRh7cGAgt}zr;k<&@76J)&*Z{hKCz@y^0aV12kpUf3*o_o+3q$L`c
z57BnSsl!S#DF#rMv3Gj_$c9{##J%u4_ofl;J1&JQqJe>GUIk|C*Hn8VrM%@VX7kP;
zZ7$TgdHwm@c>lK+pmq%exnRgIGJQ>+c>&B06ux}E;Fef6u{Ht68GpY(CpiK1jW=<s
z^%I||MQ~Z9)ZT6?Z?81(lJT{r7dF1VJS~+xl=5N)<;k4b<^buf7h*cgZBEB70V+ZU
zV-55SJ5#S-r4l@r$9H>=ePZ?`aV3i;569T*x!6vIM}ZWzz|Sq#(T6U@qof6<2TWjm
zQc9yIK>yRD{>Ha_-!q)le#G^d%IMM|XsuL<4R{jd7UkhMf(gkk&uxzT)RTomFRHBp
zh3vRFvnkq7VT%nUll&B3BbWUf!HNjDL#Ri5N7g`MHCuVHuxSj}Xn&xtG#ux$YU<+U
zSr3xc9gag|u!s!aG&lw$p#J)Dv@s}o&`cATd9nGg0eMw(DfkFqKHEGA#$!mC>fPg;
zzc{uyrc}0{Spb9Suh#s@LByng&oOSb&|vVsPxw}O?S)NqDZd8w?=BUl;!Yih(QQQ!
zcYyiNR4n_Yfh``6zdqyiWIWaHKRciOQ$dKyL`(`#rW6k_I^pJ{9s=bz+?GEHAbN?F
z|B>wKOYgiL-ob$OJuYZqH>65Q#c8F`a~enqSLsKmAtAp68$@Ri*~#hN_Fcqkz8mL8
zeK(z-3&X-4a-$F<tRyy2_VYfcoWSXVSkjz<hS@b4DEPmmJ^4hi|Iu}7&a$RD=1m4J
zD*ns;&=Z5)XYGakZ?WW(zpcQPHQMJ&y9XLbY;;gthEcwh^E=Cve-)@$EQi5{c}!u$
z$4|3HZ-<7w;vH}p5ft71Nr($2SMuUSiDO%`GI;<d^Zx*{srbByk3cn$&LWw%hU`&R
zH9*zZ=IwBNRtR5Vev!|m-<Bnlh<dF`{_kq*T<^x-#put{nGXhwifnYC>)`<}HhB3B
z?3m$uW!P=KQ@_xM86i6gsQ)rF@hnf@=S2(ciAe}U{0O00S!Qn%gRHBr+YMP~s%a(x
zk(b35!V+NA=GbEV_01JDHO(|o<i8sD7wP%vrj401hfjxt#3ZJ%*O^ozb8v=h(4b%#
z1-*VXM=Re!No)1GT$p?;{$58{{2V@cz1-;C7eyY7KmF>LXUs`2=ZJ;7CK(iDG)z?v
zH${e`I$=L0n4Zk0=I4u`$zHzm!SYMCU>wxr*0;|*4Gx6!bpyOn?8Csy8p>`QdVD0^
z{aTGY|Jv3d(in&yCT;L``(F>+M1fB?-Y10R#Rb#Cp(Mja;b*p~egeKwa1Opg$8y+)
zJ5&eC?L!dq73_<>WA^0JVBFawFT7f2jgQtC0h7Vv-H~dEr@`V5k7Bb5129Y5oqfHA
zBw<>zSx<htLGSV+JYJLsEy+Lkr4KQ=6jUlHY%ggs*ojRr6)ODg=VH?KDbu8_PWygl
z?AGerj(Gd(+xG%J^`gF|zulh+LHek)mW+fpI;a;ccP~33r8nu?x6H3D26s&F@i`-^
zvtI@2;@%;|)`PW}0z#jG4mq=BQtZPA2HR2#XR285(d?a7bOo>DzRMs!GzkUW*Xylj
zO8?})1SqR%>M356JCYwPGvg)<Y~RR8A5^{5wHV+Z+HgJLteGqc&z)g)S-#65Ko4RJ
zQi0!!Umw<qH;Ob|=Y+_+LL}8$XX*KiBdu}2@a;N0h3Rv>*P&<(aC{08*!C%>dQ#c)
zc;Hh2OJ15kZHv$p6C$&`5T@_2z>`z<l}GOs&o~Nc%q9yk#Fs~fVgCAmf0KxpUfshY
zoC*tG*7Pa`V7%v0QRzW9z4_bHBJuXMo-4|{t>N8i94WzLT_brFmD>nA=TN*yG&j-x
zd1ed)Fen*L4cAh$cE_iLy*^HC={Ax*&D0#|=2_IYEX4s`+OgAM`9iW3WbZ1B?!}jb
zHA5f@0*X%E3#(<@v`+XW5T%HS%r-i#eO%Jpt){Vu+TvSw?bV}Nr!mWJJnA^@q@b$0
zf8GwSmte{jkrg^)E}|t3=N%JOQtlzXUs)OJcM6)(;_5HLW$6lb%AXyLLrKU<8%cBj
zky<@T_#(?alFRUVIeczEs#Nm{+6U$7aEZ6=t&;^=(V18prZ*Gf0J9j<iiL65wOXA1
z+qL&kN^uI1t4Y<+c}u(Fr}W3Ek(ugJpMPilBw+{7`xgyHME@M)3#7!1Z6akJw70zY
z{RhQ?)6$nzi7YECWhf5vC-LI^X|_kJFk9<=%}(Vpu7=2OCImG|xXx?-OJmPhIe)+F
zR*KX9$ibND@h6W@{9)ZO;aK`fgazAu<_fj7Dm5eY4H4%##H~|8%;&U^yd*f}G}S6Y
zAe-t*b-b6F1iCypqIRwKE18G}AT6J9e|7*_lz7<^fX5*o*@J%t&cAql-}R=YD~Pil
z62<GQ5JoxNU1GxXX7B7QMckATbmM}RsqGSrXPBE#A?Bb!mUct4H+&Z*6P~fM&CjH<
zT|_!Z=h@`&k-WDTvRwi&tP!`$jCk_hEDsFn*D_~#>4iZE<6~Grd9d9P9UJS-$Oo)_
zpz;pjhrBUjWDFr}!$S2A^}u%1XaLBNrL0U*g3_lPSMnqNBf;i48?w`4y+aru*f+ZW
zNL(O>Ibq>Oo2XQa$d)7DwxV368$M<7s{ynn+$z4i7d5Nb1@c{S9&wD%vSD-q_v^W#
z;N6gf_I$!G)uCoeBTCrO(N9fu%v5rs6;)Fi>zV2!idWw8bimx)QX#jZSe#_3OeU)O
zI+{G9Cx+OdLPndEe^!Tcgbf+-<sV4SLvkr`m%w4v7yGGrv#Kus^_gPpg4RXZ&%@bR
z5<z#Hy@a-b?uGi+^SFsIC^PcS%u{oBE89HF(<=e}t*J&V(fZI8Xt&r7bqZFhL;>8$
zbOGs41mAhGft;heMu4J?t||h5^{`N)Tm1OF39aU)&zum?Ie|FF1du0vO+N%eF;CMC
z@G7ugs)*+~a6>2k)onfwtdR4<`wr~Uf`KFhG-HNJgt8&42DlMm4ll-pealumf`I?i
zw<c@IFMKAIvM>|de2mHLy-j!Z{m8fWfu9MrY!QH~Kjr=m!Gm6=cAzhVdJO}EIF(p1
zBffL&n>^xdW!?fYa5`vjO*knc#DUa4RlI-dalBe$c#Xm9P2_in^xkp>^$y1MV1=x2
z^u~0|y6og?9cT=rD^G|>muysofFxFq$M^OWg;1}l3G8o7^Ad&ynIb6Ysu|jZhRxM6
zE|t%v5H4p@@42uw*I4_E7MT2r%j1xixh{qJx^c3xS``;X%v>pJ+aRE4XMvtAe{(P$
z$Q6g!c|$(DGNYR6S@#M13=1OE-GCK*$hBTDLS@n3_7d^Agywd|Q?kv}5*D5T5)aTi
zg3v1pXL7^;m`~N^&oJi(G6~BiWbMT(^HjhR*^;3Ok!(P6T|TKzi71mzG$3(>?CCxk
zakIsL?Us0C+bE_Uiysi7c{LF79Somw3c!;;v%ntzl1G(S=*(QK9`TNd({|iifH>sM
zp_-cUtb;w*6I(drQNlmNyOp!%R;@5htZW6FXSsL-w!=Ay`z=+E9@{T1{Mh%LoKh}f
zWYbAL<RQD+QY3dFgOtL(mSML>FtnTWbz+()CWS_6>b#YOV_{&Yv7vu?upcxKzP4XL
zH6P`x)bZY+2aW?57I457|4?xu0f;h^8vor|Vb>W?$@xw6K$+wbGmHny%aA+fv;W{!
z)PDYAEUp*}BR-Ue-!l<Y^jK4CUNdC%HmGl#=H*S#zN?$xSCG56#CduQEdJG{v@G5T
zXA48c);?kLN!$u)w*6D_NLIek)nYXH5q#;=0QcoHGEH=2A%pF;_Wn9`61Mmsq{WKy
zviKEL8e>`^mRrM56Fke_*oC^X9bLH_QPp=Rf<Gu7J~m{r0E9hnSvbQ~9i6wrTSPa;
zjW0Ie*s&JiOq_k(+j)71<>Kc6O81?rBW5tdpg)HIR}W_Beq;jIb;SFq3XVoszZRB`
zw3|t1K*DNBW@mr)WUQC8r~a4xyJV{76ON?k?}FjGJDwpLXSeSGfZLA>l;?S**4#|f
z#x$)+&KIvE3hv`M<9syd<HHdMe#hemwB%nb=(o;o^tMvt0|wnrV%!o(DTwia;QT+;
zlysyX_7uJyJv;Bc`he^_LfB;7@OL{mh;Da0AU^nPvSqiC8NLY*P!O<}CPy`PueSb&
z98qi@yTi=&u<eZMKfF4dYx_yid!vV%U;SpT`iBY+<aBH<^iQt@pdchO0gPIE4(#23
z*LqL%`d3UkkV`25Dh~apq7!h&0ea})YhNf<Ub6O$3jU9@N1rqIn<WT>CwGAW_DMPr
zx)bpEQdJp=Kv#BFRz7^$X1m^MceD^;GP=*suGouNd;VEo?ECD>dJ#7VRaMu?%?7O-
zTNW^oCLE=TyDZELSVTle`q`hs;lk@Oqd#`>0SO^`k>-yc)OZyTFDzu_IU(+Mt<R2@
zeOTv|DN6djfw$ORV(e{Nmloi09faQY$xWa7*KISGK>e@a?T3}euPI>%L%9WRgz+Vu
z9paso4K<KqVIw6gU2j-Sr)PDmk&{ndHA@WW`U<w!g=&g*@$b*v^pOJbaT{S+)Z@Qp
zt?5hYl%dJ|aB`Poo#g#=SM5qckW$DqyzI9201p=xf;wMb|LsaVe|ICvt0xVWnC1ZH
zy|ta*-+%sM^T~&OP5&TIv>MOZmbRk@-21oP%`@|8+1zDz-(9~tSw4lSvIw}a2#Wal
zey%xwa?&;T%X%zB%WAD`N`?gyQ@lwk6WG!<tIy%-?H5nAfC<6%5NYyvzQ54lOkH-i
z!|%7~p#2B8Y!bS;c$peq+JcJ*ck)8Mi3_2$49nGg=o_K|a!oP)0iPdtzaD;a`>5bu
zICGQDy-|(mOR1-7yb!5?O+H&Zn{5Yt6~Qf2!JyMbPMk{@<_oi-<Ge6=U)$7$h=~MY
z$k2nRkB4*gAE8_%idf+D919=>Xt^w$#=;b8ZJ8A%6zIRGI<7?Tu?vla7{o<3ym@oY
zNM(F3xcPpsg=s>{PJo%w;S~IjJU-(mb~F;)=hyBlV><Z4>d7(eQV<qEvx;DZfGWR{
ztoqv2x2+LceE-ToD9Dx<xbsc-#5u<)%0am;zHH`Mkg5B-Pe=kmKC@Xv*o+YUd{4^o
zF%p3lf-4jkfOhx&)?)LgOV1idm^7!<uDWwa*K0wp32mVjQGp2thtpD7F;iIMmb!a1
zEzj034;u?0gFy4qBdrD_fvaP^TYz(VpILO<Y6ZD*xhQGbTJxS9eown>$5(_S&Aw+z
z_`n;tQ4^ouwTnW8V|uIm#!Oc&zi*6|!#_QKe9_pB*OZtEZm~D#yrVy-zo^>MiJ%bh
zd%tqNfhHJj$h=?11*IT|xbPUwt{eb_xjGW|%cT8Cif}P{(Hy3UKK~u_w37(3bCLur
z=(yw>+KXwz5lqj<xy5S38@Q>W>2?Udm(K`H;-ieuI;yHEw6MrdKj#q>-!3!5!@-b8
zBU!!Lj7a{vX62RU@nIEK6%*|i_lZdd`Vpqk%5E*`OUi*p?TvWWLTyV3Y4Q=CKu^RY
z4>I6!nOY0^a@hZ^@&sbnQ1?@qbqh&tX?d|_?Y{Y>hQNT2z(_}M8)PUeuEwYGp#u&A
zfygTr`;BF_tVhmpE+)Qq)p(u_oJ4?xN1A|W{r-&LAF1PM0*Fhq(`l(b{JYpZQW0U=
zM$!t@`_P}1dT(QasOf3`ov<gvkmWXs3PDb$GSdH~r?o@o-(dQ#_sBmN!M=9x3*ke5
z<<TE3*8-hUKl^DKtlrH-<zYiy)JvA6(IvpD8^*+v=v(}S)bermZ&D|dfO|>RA|Q_L
zIILjEds%?T8`z36qmGBZ`WvFpy(0hjC(CI6wGoV&`vK-9zpD0+0<R%0Q<QsC;stEB
zq2Pu9m1j>~KXkO+`JOKZ3YPj$q3aT2p(#Be#g>*kNYmG=%#HRt0$pi&fU@-?ES#q@
zqjnL+D)j>V3LOYy_+}qK7}>|Jt35W58@6freWo}#B&Sw@MB$uaB%8VU@O@Gze@-t(
zR5#^}B01H`{?Vd*iZLOwm#00tn46^Ezh+BvLhAxw5mpR<xsAEGzod4&$ebbau*HV5
zOfsJBmTwj7)?_KaB59v~JOKdU8c)SkbuR1yRoe<n2s={IN2YC|BjlOU9?LTLeObfE
zh8v5aH1|uGWLC~htkqHOt@xGxmxdg|oGcILHj_zFUKk2ciG(F=d4VqfMfmo>kO$w-
zSAX>d(A^L)F>cIv5>}RquBkhG<d9ddh!{EZC`WTUBX88*VGq}Zw8TYfzlpdL0Q#!H
zT?$JPUWl?*lfs!y0MzAcY^T0hqFOF%J2CsQ7s7=I&Uwq=tI#)iskD!DdA3`Esdx-j
zzc}TMw*V3+xq#?sF9BBIA@h;<)S@QywEs1E(_}MaO5MjoWIh#x0{n8#E8$rJ%KTB9
zJSdUkXVFgq*|LGDgXjG#-+kcMw+rUIf(*+RW@TLd|7J-+i;gFNkyLr5v_&}9D((Z_
zPTjE;Jn*;xeTpZSpETy4bUxy_m;0#Wn&$99VSDFhql+I@&BvVs&7`4}=6GRi_O=2;
zZ6W;(lI*pt)7pQ6+xZ21{@cGhA?Sg;1@dG6C|NwjP}n16edH$JY?e4b;`f?>3LAa<
zC{VqK!XXaju*~t5b4L?H&3l6P6Lp;Uw{N8I0F=Q$!4(1BUWv$XF?m1Ue+aW>E`fr!
zCxjcC(?hGXZDT@){6Y^p)cb4Hq&otX-`Jat{duU_l&cLx!vmO7tM}fXasmMLY8q1*
z5HJcWHnimn@TTy<6etwDAExM+qZ`Mq+T>Z#IR!DK`7icuQ*X||T|^-X`btwXQ|Evp
z{=$X@c~ibnnnS2us{!Y7H=gNWhrHmt<t|UOMSLLPMQ?5)?FW@EvdJ369fF@18rP*(
z_ceteukT^glmnLwEr0GXLWb7f*tIh~5}!OC4G&V_Wk}{cL?~k4v+HP*)Pn<iUdih-
zofsRDL2TULg)t}bD!;E_w3Q~xV_#ARawxs8AL=|kzH?Pk(A20)0Glatsp#T8`VR_=
zf=N*_b=8vp=y<1@8P4y@5@RJVjXoGBHzegeT2G!|#_u04^z9$-dhC44hZ%9d%|WG6
zSD<_C`MG+1q7yG<P6<e16*@;YjXBZwIX-)g-f7<?8}eU@ceswuDA<?jQvCU@@#~B(
z+HW~*>VnPuEP)>>Z6Ez^6*F<~i-`?+P+eRSRV2ego>bzgHrp(pBmaxYt`<+ZF5kcO
zU3DZ#Iaed8BZ@>4qKDzZRmVB|b%e71cS^(y)fgx|@q4_&-jJVoj8`OBDFm1?@Ngj4
z+>~CquO~(Ha?3Im>-RZ$Tzz4mlW`0O(HDzOB7o{oa<r(n@0m~k*3lpQV_ww-(-%Yw
zS~IK9jFx=8yYhIvL+LpQw+0S+p%JiO0XhTfSd<tI*!!2k3m=C$EIA1@*2XM~e4Xs^
zDvkTK4Go$f?=RAs%JJpm@RgKfzX$nHZJ``O+WkUk-3<tHWab@Ead$H_tGy}~VDc1?
zwr6DGC}$&LB!}XP+3-I3ljuqzYJh4)*1ZWESv_S%?8$)}|1&FLOB?%Z0f~_n-F}fm
z1beA^j)LE*VXG$k#4&@$P=r4s5nrS3k-~@2CH|oy7^#kccEy<ad28uAp1=k<D5xTW
z$waSL&<4mi#7MYDw|X*sozE=4rNy+_B|)`zzn~I>?9%`Gl)TZ;cIC^$q9qmglJ7OV
z53hy$?^qD+FeTQu-&xXT)N=^i0%DX<bDRP7qJB%Bi{MLjpry*ztS02U`P;!A%&t~3
zbYY%pF)_#&>FGx;dkfx*DYe8tE{#9Sr_0F(9ZnVUe9f3KQxi{N#VKdqN@PSS1ut>#
zX!*JJ-`LR}3@E9DbJ4Usfr5|}@z5pkFSJy`>u1XQ;+6(M{Fof|p@q9G-X-^_HOX5}
z`LUz@0|NQ%x1H_n`2ZAsnvkzXmjkg*P$dc$`uoc89hN6u$8&<$=7kF%sb7nIhnu|<
zz}HBg;+9UeIEmm+Xz0VGpBBo-m}U5;OOu^nM)S^Tq(OPMOzK_OGXls^%H{s%6bHOc
zT7PZGX_oK(sY|iGwe>^G)RqgtPCtWGfzW{;g{<XJM(gh)*V;TBZHCW4A`HK$%$O(P
zC3FSJ&2Ni^5h5^%1BqgY<nohULvFpvA79>y;4Y}6G~~gSIN?`~Mh2uFtQD~yEF!-h
zoBcTaILW|O+II<7)_;C8F_fZ#2;J^ca(!-s9#JT93c-D8LL%sG|AIUoWk{hS_!%?V
zWwk1{$;YK?SziC2(oxmH)FM?}9TB$0=J1`%t_|AH)v@UCvLxk`KrF7IOZEyKn3)Q^
z&Zg8*55Kfx`Tr1h2l<m9>;Z+B^tCi5R~VFm9m2|O$N(LI$4|VRABxR#29F;@dp&-b
z=oCy4cs`eKEL0S#mVS8-;Xc5kb0sgt>4ZpRHz<oypy|S@v)4sliRA6Q*G&1iTpg1k
zQ)$e{Y5!@dnuRFkbekIMJ`eZzE&j$IbP>e!oC<Nybou4#XPgqs+aFqreK~W>ICs$h
z*7Ec~_gPw!GFs~Nt;fY0M2W}otYO3GKd(?UMpV#*zKXJzdcibQ0{^3)P3l>RMi=h-
zlhoUe;nS*1aasL4AwR!8_wi?;+6&dHql%jE57BSacLJ}dSI1Z&Tj(sJ_D(PZiO(P0
zp{6RDW&)8Qkhs_uL|Yk4m49o1@#QRt+qWBLOd+HXf@sESi2W$etd`x-V=>!TA!Klr
zk3F1;GuU*8F6(sJqhGYD7ZF-oh00SrFI@OQ_sFoz-><n=BR2jom!ZhUOx$?R_+nMF
z9JZEkV2!7#+=;4+u0*k);aX6Q>Jj8Mg6E9B-@UW~-Z$dJKYfFd?8n<+qP52lxT+4d
zFEmV*=8q6xv0BrQd)oo9q116RJlUK?Wmb7KWgEZoq3WR@6^maTIyAy1|2y~4r`NG7
zrr}A)y^d~Slum19xw&@>JQT2FA^|UvF0^R=NB~4oYhHpM<m<ikPh~$gp?kU)Xk%*O
zZ^70g7f@+T@%4+n6mz{yak-aQcK8Mqldc|T%s-kS07dNEk*->5@Nb=89Y#-N2Pbg2
z-$4wxwyeh^d+K^5Z!l!fnO=2I7TgGU@D~EQd5a>aQ;=axqv8eFPh*^caWCR5V@FnB
zy(6z%HK&!avy}%TS&yyS?`XTkLR<)~DKBzx4x$lvE<OAdZzu7Ft)REw(v{HgqHOD4
zi@GT_IeCbQ>-5D!EgwIM@oM*AKk`}%?E^9CK}OxG$^CmeI+wBVKY#uNeLG|NX?yys
z?Tg<FfhFbM$X~Co>EWaTWh|QmJGrk<>=!wcgb`_v?uDCGv_A3~x2HRsU5Y<&%e=D~
z+nNZ)&(cbgsouL%mYwor3{w1tet$5Q&$64Xq2QXSAGP@(Y8batzS%wd&=m0Kz^T4^
zy(ZQdVbIG$*ww~hx9!#`r;=S(imRr>I+#*^d-!Jm{I_;-$m2ImyX`cri^e-6gEE63
zSh(-(Y53n^?Wzbx3UaE=GR7(u!j)V3D3LMb5E(T2rJPweHKC&)K<U8Ems2!*p%TxY
zrMD{32th;~-FUN=C&8l9xynpic<B_FBSo@dtAn5LUxr{K=oJO9>}5Bs`S4FMPA|gM
z1@P<e{_sY9n{Up@avAFXTz!gOZLE|V_!6=!f!~cgt~LG$87`=Dg&lO>_cS(n;ZkdB
z=KNx(91i<MyqsU%X~cGZ112Q7(|VjqR{AgU%sBb_KKHf7%{o1hWw1U;Ko_KcnIG7R
z**G&}lD5@htP_HlvT!H)>?6mpN(^-!BM!>ZBK2;TUUd(g+4gw~?tF0^a)qtoU&t9M
zh$&A9%EM8hr~8Oa#~AE7_4zq$Ue3g;GZ(lIwisgMI><cDr}zbHJ@5~ZIqQIaGf~jD
zm*^m7=z%&rK3~u%q8{3Rv;WKgPDdA^eztredVLM9Y?<57v2DVBdKPeS|LFLm9>x8$
z>7!ds#$~xCxg$q=Me>pJAWl}6QEC>D@~aRK8(AG=YyS6!h64z(4tvu#BG{=-XxjiJ
z0sQ&&n6beWmB$+UX(AC!<<(<D4o;l6m~=U%H*HCn;R3t#r@40kab_1QX)=B`*UAxD
zUF+<ju}0P(bwvyt9$|oibM8^Y@EAnnggohvi#|LS4=Uw=u*C4!#@#N1g}5~5Ik^uY
z;(%fk<pi?-Ibapm$r+&g5cR|jtI}WckR>-b&-v3y2sZf$J4X?FmD9lfO^%rt7Oql`
zK?-e}+W!PL5g?Ow5dUKh$SDpa>PYGomcS>cM66p6f>7H|vbin7M^HevtI-DA4aPJ_
z^n_5<7PzLGvDy(+r`E;{Ev2i?Z^;80P8#EdquE2IXTASFq2yni3>vcUCQJTQ&H`e<
z&aW=gN00$IaTSsgJCsJCe#meq0K%OnGP&H<5zqWlWqRY+nQ(YA_}&PUUUBYabs$wa
zmK-qkdM0k{r<NT+L;{;V9F(La6jK3;g&V&MBJ#Mi_K64>ag)v=U#Ig2@A2I0d1}Mq
zSq{@!6bNCn9O5{IPc~dPRFpbNTNm+-sqLSx{3o5TtTGPq4k5xN)|K$lvdMr_fpFJ{
z?h!)Z838nl56=h)Qi9QGOI-DCx32>wXTI}x7C9@zm8C)5gtGG#b6{CsOXk<GU#au?
zF+||r6d;gX%HLVEzfhUq31i$P2w%rmn`ac3W!y_K9Lj$GrwnG{zdRFX`Yu^mb~G^L
zlg3(9?AY8Gg@%Mf5viifMI7~rXAKHu`9U@fVwt(--^1~?7@J7ZpYG>`E4Qs^P#ChN
z^?qGJ(<wL0n~pmWPA?CP?$5o;esvWdZtC;3Vay*9+`(da>PbvUj>ULLd&3Ppr$FY=
zf%x<(Wf8{^&70Qnd1e5UrGwd~Frn%ko2}&rs!lPSK4`BIRJ+KM2ng{KOx_3CDA9Af
z%wZ8(l%TsJV9UjqUs5WdBI8FooF$Yc(pRFIdiG3#vqYklC+$XYCybG9jh9>+lblbU
z7qvjrrPBfp!ZzKA0#tU0$-lfyfBP&C=bM6<aI0!!A!DO9Iig)$%y;9*a>c^W>tTMG
z#C<))jbX-p-Xkn}g<k1ISZdOm$Oo`55Ch0eC`(}LitNl#&$yW^eoD23q^@Pk(tc(X
zow-1-D6>|)VI-@6_2+RgtMJGhAbr8qul7*0R^hGsnl^J<8JPl)Oi8`m%Woe&%wakG
z=p#1=IfomhS5%LDLfGo?N--6v34+rcfeO8dRl%)SHhsbJIyP;?^@oNLbxwU}bR+mr
z5bE)8ZD)5n9pXHPeplVuy5sxHU75}&e@ly*vNVATIzU?P;(j;dsWQ>TT{oV`mstq8
z^spj7_G1}*6K(u*8EI5j#YahR57FQTgcgqrmulf72DyBOVQ&eZ*zsvA1Y|qrsTe8|
z(<Eu`Iuf={N6h(A+WR>h&ipk_JPS&^7q%Wqoz>SdbtP4Y&r9n}PWkL!_U(HQ_^*4v
z&Fg95Zrw)qa32f^C)^AK>f6jK>;Fc|u@jDq6j=<b+>ZBe-gkqZTKrIxZwt`dC(`-e
z9S%<9&pJo`7Jalvf8_*=Qg{rjK3wGKo`<a-G04qKl@aF-1+H97;({15<B`Cchh*uf
z55+x*X~&VZ*-}zz{}rrr;x6Ovy5=aijAN%mZ=5{SKMbrhKcsJIoHnPDTlxa7!G&{q
z$s>p;bY#yB60)vXQ#0hnjn|5*9u$~+g9BJH_ttCA%Vi%Uug{_18rJLcs88l3U89Q`
z3v>IHakf1=_nr8$fSVl8q8x!Rb6H!)-E-tnfl2B{!gfT`0e2U)^FG`#zf_$>!)N%7
z>=-qq&Md3Ddf;FM33XklD(CYjms#`kO>^cA3ZOUq!BXQ~xD(E9)gLrb{uoSYA=+-m
zU^syNUa76sE&Fqf_YRrSl@x6>#A)1LHT4U|&v(-EVitdxu_9ISL|Q$+>|A5b!&z!*
zrnpLiY*wfmad6Yq-^D4<r9r2+x75R4B~?crV{FzZv}{9zt{@o;03owhjGc@N{?zBk
zj+x!!p3Vf7%?VH-uV4F7K|V8O-CqUOJfmj#aE{EM&n#(h_!7_4WOytLF{ggyjvt_l
z=!)fI!g|iYnEU&CwldK)DXbS_*`^$rb(3FE@ry9aD6)FxSa72H)kz?g#g2^{{2PDe
zJH>q1R1E!bCen$0R-RB+-zICr-2l^Gd^-i?DdSY-uuy7$|7E~Fi1%^+eBc6hbnT9e
zLaJH+gyVUx3DX)1H|cOn(v`&(o+1Y&h#5;AQ5Vi~uME~=db;}*u>y8F-#)2nwiED$
z00&=urZS4axV{e5@Af_NDfbcIkvkU9x18gJTsD*I0*qTaFVi+@bj{kjNzb$!vNcVj
zU+Off;?zO)VmA-l(SZak`JW?)f_d<QugUU^i%q5)7m;ENtBffAb(@VKr>g%pyAPH#
zuC4pRqUGLPJPJ~gf%bzMtOk$gIoi<xrH`(~vF}U#47%oZ`Jse|S6e}?fQX!D*>=nH
zFGkl$wNOYeS2?!6;0iMQ!9R<(sMj|fr8n+{phi)HLj$fID4*pGhk)C<!I+OBcd-z>
zkgmELJ$Fj`O{8i%Y3<1^WU?q-w)M6`GWBEy?_Sv+XP2U?ugz(Z1Z3}u#-V-bCAj0K
zr8IByF#v8GASdl($yCHow8f3cXO=iWi#gK}cmWdz@#0+X78mT3B2IQiK-(ox>f-{t
zg#P}j@E*JS1bMOBi-(*!zFq3L2_$M(E7Ywft+?dg!;0FZG0^Ine|uI?`L}73bLYqk
zVNg*pUv1X5TYEBNI;nd_^-BDhp(Bwi;QA&tjCRlRt?Y?KcZ#Ymk-&aWNsXi1Xzm<5
zvART%9ijHEejW>^Jux$*ray7K649R~g?$ylnv$#V>m#$wOTWdm2}+|u6$#ES=Aexi
za-j70U*0{#_wxB+*4AloRZe!+T3j$+VrH_*JDq7TuI77G;A|4KX4=o>72$uJE|y%N
zrw>~FuD>h=(`>ZWa5jNmktV&c2t-WzVEb5fYj%lV0u9Yr?6nRj3-^on!97(xYv)(v
zx<X#hFql}*60b?TJ#$?S|Bh^iQ~(&i^YHh5BaT}g+-n$f5_Ig>j_?S*|MgO8*q26o
zxfZ?A8F(CLM+s}hVeFT`HGZuDhh(0Iu?c7mRpaq6S2cK>xxV%LqsGmEV~sRDh~*?x
zl;YV4zl7J#<h}$)`tP#3K1k^fWfx(IaAp^oApriTna!sI&7?R>4T%@iq!qnvujk4V
z77A=~G5ZC5Uf1W=<Rzw?l<rkp?<^{-(n0=J?}dEoUhlbKTN$C~<=+#yjmz6PvqMqd
zQp>^01xSbd&R>gqE^3+;C{X0sEX8}~901<y%_q+blLJdX22dl{;%YF}#D@c}cQQgD
zVX&#TK#!eMv)_>r-#B%r>37aqVzMedt5q+b4)Jfx17j%*Ra=nh${JeLvAr^wW}Y+q
zl{<}(IA4VzX{W$md@+f9pTox&dW5>2Qh<rbH=4V}9tO%s=-^yC0_TmSL)=7m^9Q@r
zg8IL~a+Df|o4D>A@bN^Cr05(F+PjUeU3+4yJ%f#HBzbZ}MiM1rS_`jl^lgh{PgWvU
z4_#la7NnX`=fB^4IGnMMIazvcDDIHsf(7(n^X9i7n&&X-ti*9!M|O4zR?IqiS$D!S
z0zfg>z|p9YJ4nHew(8il22xr9$3kUMSzOO9vgKRSV+D4?Cq2}6TQq9$!DwlpB)V;e
z%eg-z&ST#P0iXb8-x{EKN=g07PYA>im{-=Cd81mO42(wLoHd!X-NI70ciAwWBc3Cb
z?b?{z+(E}_VdJ>hcm&97Cn6**ChRg6%mYV}%<hx#^H77kb>}3R1S~Ys=mu&@lA|_i
zFo|CFo>vI|1<~9D!>TkS)zKxX!LqZ7YV@qvwdvKN;tt>5oNL^|5`8`ilexQ{mFCMS
zs17ufXjUy0C7?yQgh|aNgMrPDUPR2u=~ubzxO8|sjYimwg&wTz23c|y<#3$AD0Her
z->$&GN*I3^)CCKa-ybD%QPtE$R9l8oIbB#$%cKc|yT2Y3R+Cz7e>2C6{)PL@Qx5Tg
z+2nFfUC0&>GQQjHTO!qY1>yUa(PPUB$}w}i98?h&4y_g>i+(TxsN&tLt17_zv|iYP
z$S2$BLcm<VA$B361Rpu4zK9;t!~kheRHIW`8oxP<2^`AQuW`B{xvkz=+1)pQO=_Oi
zxa%e6@5nX$%Gr}oJy?<ZGeWv6x#q7CzBuCvb@c*m>T80RtS6xOp-syha!;du7|7ss
zJ%g}c2e^`faggVzArF)1^Aj6QCb3KA7Y3J=blEte{%WaZi8NXi@79H|5wtxL<aq}x
z>*K2EkmoBjhz90+VU=q|G~fd%!b2@^%g_5R;W4lo4XhVX88o_RJp9WK0ey7|?CM79
z!K@pfR~Km(tYBHpBQao%1|f5jf*>fsW^sY~O3{+fQ4zoimwe9h1Tf`5UFT3XJu7Ko
zKI1fPo~Eo$oM&ozo^VZUbFs3@f!=1{!*pA^F1+W^k+WtfHs*cRSx9&tIo-L4c$_AQ
z=o0{J1^Se^@39+8ei+MW(YQnTFuP(}z#lwV!QS~>^P8j6GJZCOt6HDiC!Opm_wR<c
za{cR6EEuRgL;Hs|Ls$ZK-O!poc^6KRd=xPBfQwtCtuOPAzI>+D2wPdy<X&!-3HuqT
z21?BI&n>%FI?pcS=&$7Xv^Bd{XY=v*kI=PqgwqT*Oq@3%l#AcQVud@4iizb|t*b}u
zlJL%T8HI&ygV2bsFgBK>SKs9mi$(gypJCnD+rKC2?yY&_5XZ}DdQj<HhPKb|H?Z>`
z)#9o!E+-f<jgrOao5W1D{jZH?1j5|*OVrkhY<lhM(7nb6m$i2yQLM^83Py?byq#(}
zs7wQvmI!79lJWz;kDX~m{s@66Tas({5zPGZ2~e=~T`UQ=i1c))P+IeHsGS0DRPuz^
zb#jjNZR(Jytr7|rMI?M{5mORSepw75ufu7QK<XZ1=I<yQwpd3ZdZIU-na6WXhzlLD
zFDF>fB)5n7nl>~V@#~rzT~Fv;zPoG-q%82)k&3*$I_woZtwj>EVCYPdOe^SRQcK%o
zU$%@aJvX0%J7c>_z|Dvd!bdw%gp7EJ@M@^W*G1|)-3oIp{cV_bKromSUI6tuN|eYv
zdFKZj-CogMDU&z4O$RPXlXAV!<y0)%Y~bjgr2T4!I!fWBqqB)A#W)8A{Z0?hi#;2#
zqeX5)I)@1qa$b4MtYULEfLZSIN#J89;e8<qY51xNuu6Kd)mOyMADx-A-&(x$!aT~C
zzq!GorDVs@Mwc_(FiTH}w(dO9a(`5zlWo1>6j>%^g5zsT?--u9JKirB!<iTuul9b`
z*-kRZ;?3aqmg22d;{5JvA~qwxKV^ZI{ZO6(y)HjhF*)Wj(N)E%QLmBb9-uTLhxa4o
zpDtu`jZ;fnPWW0eajL#B1+2lMmfhjS=g1S{-%dhhEquR6v?Kx{zlfA1_l4H!BK54q
z#TCdU{FYFKd1dFg2(25sf5R2CKqp&u^rk_W3X?N$y|83@{pHBdS4>36=t0+lS~H+T
z&m%VYQ#xjF$5ac~_^yAofzHthqQ0bhnq?}ZO?;Ze;_aNU9Mg{PSN@l`GL7f#qi-4Z
z^Kj|hA{A$N(wF_^Dxdofd-f;8#JNV;PRXX93zLY+8K84guuIqmY^iLoSznm+!#dTe
z2#hz86g<x1Oo;KJd&3A-@Ri6_#QE^rl*LIf%atp7LKIjWsM^H_0l)m=l(`~HF7o+<
z56JxdFAHJ!y8mc67!iypJT~eWM8z!u5hVr=NVydB35hFA*$7LCiV!m)R>=1xy`8${
ztzERS0|#!nSZ%H%!F#=N_w?cNh{r6JD2_$S<Gqy&qpZY6y&#Y+B8o{&$<1v8nS>Yh
z6hbJ#)TAk}*q+RxzaHu4=siBMc=VF?i-M+Ez^Ai{t`V*_Ypy!)W5MEz3|KvOz18H$
z!}E4uJc{=YR<Z_MN^gz7?xg5vQHUNSK~s6H2QYcGNR%ebj8+_g2CvO*FoT)WBQAyH
zkttIz<*g7_qn=RD0*C5MJ1fh{zaR@*eNtSK=5^UrA@TL}3Ru6DT(!gJNC;2su1C<i
z5Z<pO@cHuSV5X2H;K=d*4dblt3I`L^)^8fBuTcWi>m90<^`9*%G%FNi2-G*jpCS3s
znMv@ND2m0ksE-OM+v=|hse*QnrmP|ZEXeokzF4H5Mme(D-|i@?phW&9QXM~W>jKYd
zs&GuL8aN9YZTD9><b$-`q9N3v#43k5TI#o_@?=Tg4F!5@QDBB-pIoS{qgdFk{U=B!
z3vaA5HnYZ8G1t&NLZmXRpyE<^J&Y#!lQRoprazN)8~e|hVLTn?`Hhrv_2{S7Q-6qv
z`_0MsQ!=vSohhE$9l{zHn+M}>UVHu#>53Lkj+dX}THa-3-*?%<l-2*^I{sQft#Fn~
zaq4$*all7C8?E}})9Rr>efs^BO%BifcIhLMZE{&oD$}j{8+&ti#MO7%7oPql2L86e
zlaW76NV=~98}l|M32oY-U1*SfumU-5Wj?JV!L{1gh``iGy-jYP?4UfKmfee#O(Y%e
zh1`7FWXOMYlO-|hwXt{dLXIxH`Y@w>Wi$&T2^pVhogsgd3ux)^Q?`QVAJ!LEh-u!0
z$|fk!+D@NN1*~swianX!0@}N8t<99MWwh=*Oe<g}cxzGT?}~Bp#c@fFL?M-@i%cNe
zDE6sN!2PNs2@>1UJyBj|@!VuDC;ISOL6%qb)|7-MTI-7Yl|}ORs$Xy?3m5yk!YT0D
zQN1JY1z8?PK=5BoN}|8naXgx){V>oq*r5(eS3+jNh*1~V*c}N$DDk$5HrNo*g!Bp7
zIefs?@4{z>q;IZ6uOV*G_%_f4r_RqVGC_wQJ=YNaaEJQ8Ebb!k5H`}HN<Q;cQOPSa
zgNW+d^F-?g{VvQ)59r}tw5LMnz>9@5QpxNTiUyW}qHF%3maU>?`z-$ee|7I1Wnp5@
z(F}~dopNb6(>=1RUEM8g<j<VJi|vbPIvN_9KjkbIy$KnjLJ$nAD#H%((VsbW&^Y$H
zS8%VM?rsdv`iI2_C1kYuZjl0hRwXr^6%E>Vo+6kX&m3c&A;Mt@uD0?#nh<tQ@$_PM
zf020N_V?$H;W3wfEri!(Y_(teT>0y-B`Jp44tFp;uXV%eRR%E)D`L(-HCAx}dvlXf
z>c~cw9J%0!mDKnP98Aq-S$4n`|0o1WXTQtq`;PQe(cfyE@v%ab%>BeIl>=E&?a6D5
z7nI&@r4&?<;~Up5tP^<gatOLRC=5g}I{2wF;c}NE6&3HkKZnQ>wvFl>&Z?=T4r>GJ
zHWg<*Rm+crZmWWgx=LAX->)y}392~Rwsk=I2C1Igs|cI3?PLZLZ~5P1{AN(V7kqmg
zJ09EFtwjcDG7@KX?p+eIo!O9Un3+|cdOB!LxVr0o%8c<=Vd_K7qUH4QBIErVoz=jx
z7_e>11Sqlo_ouk3Yj4B=j-hOqT#3g`^epfAXVXGx>T5G-&<B=Mt19JD@yl77s<O4B
z!l{%$C0}jA=@QeABV;b=e6H*5;`*ekubN%9&vbjE`m}EPKYhKmbFGU6mZfa(l@ejv
z$*lV=rLt5Pl0NsO$Ptqx@5+lC%v)!T8{~D2<EEF}=FCLQVF_?`lRH{hT)t>Ers!fB
z-96GcUb2ddOBH$DFx!c3`Nf1r<!FMrml>lECez~o8*J8kOxIJN%T;Nbyrzx^JN5T{
zBuq?ik?L_n*=(KLHyQS6pPjo);tk;L-5rgF2qNZBeAtyPgcF+=OKj53hCzafQ{hvC
zmY*iJoVC-)(5h12T`{@8dBrP>n2kdFNc@4EMj|MqUSYZ`1ukwEFfuZ+tiOEkZ~gne
z;Z#HCGR0V~gKRM4BZEiLVyrRG?L47ucYk8;)y#Q+)RrTulXrdJ>8|%!LEC}sz-%(p
zzX;dUugCzXqfcB*OpC0+d_OEERwsiz&*MfU;JN~)P&F#`frlFRY_X{CTVhs=efpq<
zzFyQ`MCl0bH5F64j|F@A-RIWhWiP8`Yq(r1Gd+vNU0@Nj|E4`FAjnwR<ItxRh7>57
z3=U3L+fO7<u>ZXzV^2wf2be(xo7cQgu3UeUE2cW7#BI?~OmXGZ8#T<%dfkPajE{WK
z5EnKDB41@j9;5rL{zw$J(M${dEexd#_xx9okwfR5OgV%q`_MKvw}mLhj+9#J57oe<
ztF|k+i7Z!_aE<pO6td`4wSyf4q1i&Fft2O~2U58e5y>O)DZa4k_p?hhhm!&3Q;ejD
zFF65Rd3_^5VX%d7J`tySg<4F1_nbbN+za(%#V}(qLS$;XNd}s%cH}epJHgg%4x&Xn
z-)d$j*wY+-gBLHneL~BHA&4zJZ71m92cA3p><4GbIR$+lM6;8^)Ag-2by2&?%tYPl
z1PN9mWN{#Vs@S`#RFVi5`=HGqK)R$lwO*+YJI6(Y&fUK<?8RkV(<2d3JKs%bVGy&%
zVWCEVC@)}+-@hZ35Fch{Z?O;0eA2iP%ov#4q(^gUN8<Y_#fnDphU|o1KA$uzc|KSd
zi;bX<il-ZND0ke1Xm%X#tLm#5n^-e*K&{7t;Ya`At*rBx-<8~3VU-bl49EbBcQs^v
z9;-}RiR6(z)2k>OBs5IuxFcP=dY+Tbe%D@cEQo`6E3k5L2s)1e?gK64logk$CM0f%
z9^)BPrxXb%by>hPF-V{JyC}dgKbzp()4$2bf+sv(9Q6dV5_J11#FaH-V!^-JFYgvy
z(=sZS#uO%FA<7yB%k#>qr=axyy1otaR~WiEa+VJ>^Ce?4zFS+m1X#Oeut!0`qd9I!
z7k;KX5=Qmv1XEM?MwWaVS*rv;s`bvyy^B`Hhk3qckDkn&8rW_e^xu7cqTc!7-930=
z0Jf1>`Oe)Y=FAF)c6&*N>M3zczM_P%%(&UQ+Ws}#k*<N>lkM~=wUW1MF&4IRYAtF7
z6`D~OOsWKxR@BFp1oY+1;$Zyq`Mxkq!0UKG^*V9Z6`n`V%MMaKWHeua^%#w|NCeCF
zv%_?Q!dI6+aki2_qg}uNA3F<(t4gL$cplMV+C_o&neQzv%>B!$P!5|^#Ze`9o7r#y
z*Ds6~@ZbSc|HoT~jQ;b1$EcSYk|FtYWJM^UH{TY}35K3SZO|nHqb+~oqU-=-9?3gR
zsYkdLl9DLTQ7;uFC3}bg2bY{d6$KUWEB|kQ@ygYWF+y+j-bIW>KDEP?MI*vIN)M$B
zmFK1nqir)MQGV=7Lfcbm_Xy|#YqZdRPV<A)3i!_d^<Dqq!!rKVNZCJmeDq?68C4RV
z$G#Z!OyzoIYdZLY-`C2U!Dn-E)scR>D72`xiu$u05juA}w#(;#4R#Y<m&He&NNTA&
zm+UE}T;I@rgCIhkZJOW&qA%Yj-)Gzg82L~E9b}>|du@C=A#7`XBJtLqR9Js30X!`7
z6n>qm6ZnH~p{$U<`hHIzDN$y-*+WhTJb3KLo_}}EfO=>v@x4Q{dY~Sj8?L=4!fwwM
zQDJ+!>|`79iF3YuY%8esHvCNc_3J({x!Wu@7ukoemgw^}eY5By=nU3Kdgb^k?3o3W
zr>%}BsxW@K$OEg(Oni_yp688vDeUlToY@)fZErKAHYN&Kjzw9TN_D*0tqhgAWd>~C
z2Q{0pi-6D-IDt;HGwHgB=_9jkE34c{Y1HsK7SQ5**hWs`+1BSMvT+CKa1U9h+-q`F
z9~u3h`~nI0F-ZBqH;~IBl)`I>`AL{Y=i{%GkSL*?=YPPE{yjzB{q7%q*RJ1K7Gv`c
zucp)dU{cB8@zJ*N?w5bdIob?L?r|a~3&i~LF^h;o4RXsGI}n=|mJHOZ5vN%wzb3?&
zgT#=_bNcwVTXwZwj9)b|?SSV5h62Y2!Xp2DdpN`Y36lJe^ZoyxcHocSw$){6F^`J=
z^R@ra?Sa?^D&i})-IqI|m5f3`)STilFc)n)_2hFLpQ4XHH-F;5YK?G1P$WOsW82ug
zHpGB9j-QzN+wQ6#czk`LK5iuz`=`h#>TFpEBO3=60#-*y07pXr*9zjbeoD4-<8`<c
z+k9fA)`$YEQ=~@9LSDL{@CEjv6USxk^jD0@C{o07OCI~_UW03WwEV14MMI?DP5Av~
zQ#hYrI1Yk-V4F80<v!MZM4<}We&l^6vvWiO#<tlF0BLbjxjAnTQiZ9st>2OFy?}Tw
ztG0RXe=XSSnO<0EVvxaKuE0Q_i)QeYQ?%c^tq|}FXM`ZoRGL$NTRr01k4_vBhQ*YO
zQAjBqU3t`6fQ|($YUc8Em;Sm&r6_IkFAmxeNDQdX#M`x!s-TDx?A?8Gp2~KhP|<x;
zCh^%nV(qqo=m;NjUp!l=mF(Y<bK9X>c2aNyRXl9IL__-=9zSTKL!-I{AF2sr)Myc3
zt30f|zkE|Fp`6;ZHneqm!?l!L7;q$_^ctz9yxl0pOY&mn$4Ocf^NDbb6?E`WbWH`k
z)uMA}A6czBKOJCBehn-S+vdfr=`NSiUH)_rzd+EWYydXtYZjK}tzS$`9|<OU+ZtPL
zes%`Ucvasi$fmkg-`E*BHbD$7$9_`Q?8?1OHW4d0%0IGNFzT*bITb9L&@MEj_u|j2
zC>13lFfk0eJw$ZztduQJY_3!8-cXjeiQkI0x)}@AB!3wv^cX+16~Jd}Uj8CoLJo;3
zvS6i@N}LePgl`K;Z4HFih+f4b4sx4stO=gPvMOjMulzGExh2yCSz%zqKFYLM=0O6%
zPa+M<2l`!1O5U?|b~<}RXe^F#pFn@5*H=UXE-b}4D!OB7v7w)Qtv!#kNN(YIn{Am)
z{mff=gtwE&@<RLRAX)!*Joz$ew`GFzwU)yP!+DKw!WnU45?-Ny-RJ%dn2dVaduWM8
z*|3wr!Yo5hc9|nA!%P^JWm_82Gj-Gf19D7(!_iuMLKhSCpuC=qM=K-Hc&{ZEy#b)~
zwE}*vKk7Dl=CfB@=hD%EsXk%}KtgPFF!O1O1+?P@)ZxyDg2=ECirVI#;@LnnF2ztR
wF%&>Uii?;|;2FWW=fSvlbJckTt-i<LqyCuM_(;{{5%8lRqbgnT#x(SQ0cR|!zW@LL

diff --git a/modules/web-console/frontend/public/stylesheets/_bootstrap-variables.scss b/modules/web-console/frontend/public/stylesheets/_bootstrap-variables.scss
index d1cbc0ec8fd1a..ded958e4e6cc0 100644
--- a/modules/web-console/frontend/public/stylesheets/_bootstrap-variables.scss
+++ b/modules/web-console/frontend/public/stylesheets/_bootstrap-variables.scss
@@ -49,7 +49,7 @@ $body-bg:               #f9f9f9 !default;
 $text-color:            #393939 !default;
 
 //** Global textual link color.
-$link-color:            $brand-primary !default;
+$link-color:            $brand-info !default;
 //** Link hover color set via `darken()` function.
 $link-hover-color:      darken($link-color, 15%) !default;
 //** Link hover decoration.
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index 0c978e9ca43aa..be288dc924d5b 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -33,6 +33,14 @@ body {
     flex: 1 0 auto;
 }
 
+.container--responsive {
+    padding: 0 30px;
+
+    .gridster-wrapper {
+        margin: -20px !important;
+    }
+}
+
 hr {
     margin: 20px 0;
 }
@@ -62,7 +70,7 @@ hr {
 }
 
 .table.table-vertical-middle tbody > tr > td {
-  vertical-align: middle;
+    vertical-align: middle;
 }
 
 .table .ui-grid-settings {
@@ -95,7 +103,7 @@ hr {
 }
 
 .tooltip {
-  word-wrap: break-word;
+    word-wrap: break-word;
 }
 
 .theme-line ul.dropdown-menu {
@@ -241,29 +249,6 @@ h1, h2, h3, h4, h5, h6 {
     font-size: 2em;
 }
 
-.login-footer {
-    padding: 15px 0;
-    text-align: right;
-    border-top: 1px solid #e5e5e5;
-
-    label {
-        float: left;
-        margin: 0;
-    }
-
-    .btn:last-child {
-        margin-right: 0;
-    }
-
-    .checkbox {
-        margin: 0;
-    }
-
-    .btn {
-        margin-right: 0;
-    }
-}
-
 body {
     overflow-x: hidden;
     display: flex;
@@ -311,10 +296,6 @@ body > .wrapper > ui-view {
 
 .use-cache {
     display: flex;
-
-    input[type="checkbox"] {
-        width: 20px;
-    }
 }
 
 .group-section {
@@ -1086,7 +1067,28 @@ button.form-control {
     position: absolute;
     z-index: 1030;
     min-width: 305px;
-    max-width: 450px;
+    max-width: 335px;
+
+    treecontrol li {
+        line-height: 16px;
+    }
+
+    treeitem ul {
+        margin-top: -2px;
+        margin-bottom: 2px;
+    }
+
+    .node-display {
+        position: relative;
+        top: 0;
+
+        max-width: 280px;
+
+        text-overflow: ellipsis;
+        line-height: 16px;
+
+        overflow: hidden;
+    }
 
     .popover-title {
         color: black;
@@ -1229,12 +1231,12 @@ label {
 }
 
 .fa-chevron-circle-down {
-    color: $brand-primary;
+    color: $ignite-brand-success;
     margin-right: 5px;
 }
 
 .fa-chevron-circle-right {
-    color: $brand-primary;
+    color: $ignite-brand-success;
     margin-right: 5px;
 }
 
@@ -1410,8 +1412,8 @@ th[st-sort] {
 // element a background, we're telling Safari that it *really* needs to
 // paint the whole area. See https://github.com/ajaxorg/ace/issues/2872
 .ace_scrollbar-inner {
-	background-color: #FFF;
-	opacity: 0.01;
+    background-color: #FFF;
+    opacity: 0.01;
 
     .ace_dark & {
         background-color: #000;
@@ -1664,7 +1666,7 @@ button.select-toggle::after {
     border-right: 0.3em solid transparent;
     border-left: 0.3em solid transparent;
     position: absolute;
-    right: 5px;
+    right: 10px;
     top: 50%;
     vertical-align: middle;
 }
@@ -1902,15 +1904,11 @@ treecontrol.tree-classic {
 .ribbon {
     position: absolute;
     top: 42px;
-    width: 200px;
+    width: 186px;
     padding: 1px 0;
     color: $btn-primary-color;
     background: $btn-primary-border;
 
-    -moz-box-shadow: 0 0 10px rgba(0,0,0,0.5);
-    -webkit-box-shadow: 0 0 10px rgba(0,0,0,0.5);
-    box-shadow: 0 0 10px rgba(0,0,0,0.5);
-
     right: -42px;
     -moz-transform: rotate(45deg);
     -webkit-transform: rotate(45deg);
@@ -1919,16 +1917,15 @@ treecontrol.tree-classic {
     transform: rotate(45deg);
 
     > label {
-        display: block;
         padding: 1px 0;
-        height: 24px;
-        line-height: 18px;
-
+        height: 17px;
+        line-height: 12px;
         text-align: center;
         text-decoration: none;
-        font-family: $font-family-sans-serif;
-        font-size: 20px;
-        font-weight: 500;
+        font-family: Roboto, sans-serif;
+        font-size: 16px;
+        display: block;
+        font-weight: 400;
 
         border: 1px solid rgba(255,255,255,0.3);
 
@@ -1978,10 +1975,10 @@ html, body {
 }
 
 .splash:before {
-  content: '';
-  display: inline-block;
-  height: 100%;
-  vertical-align: middle;
+    content: '';
+    display: inline-block;
+    height: 100%;
+    vertical-align: middle;
 }
 
 .spinner {
@@ -2117,15 +2114,31 @@ header.header-with-selector {
 
     div {
         display: flex;
-    }
 
-    .btn-ignite + .btn-ignite-group {
-        position: relative;
-        margin-left: 20px;
+        &:nth-child(2) {
+            .btn-ignite {
+                margin-left: 20px;
+            }
+
+            .btn-ignite-group {
+                .btn-ignite {
+                    margin-left: 0;
+                }
+            }
 
-        ul.dropdown-menu {
-            min-width: 100%;
-            font-family: Roboto;
+            .btn-ignite + .btn-ignite-group {
+                position: relative;
+                margin-left: 20px;
+
+                button {
+                    margin-left: 0;
+                }
+
+                ul.dropdown-menu {
+                    min-width: 100%;
+                    font-family: Roboto;
+                }
+            }
         }
     }
 }
diff --git a/modules/web-console/frontend/test/check-doc-links/Dockerfile b/modules/web-console/frontend/test/check-doc-links/Dockerfile
index fdf00200ff908..a9cd8217598f4 100644
--- a/modules/web-console/frontend/test/check-doc-links/Dockerfile
+++ b/modules/web-console/frontend/test/check-doc-links/Dockerfile
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-FROM node:8.10-alpine
+FROM node:8-alpine
 
 # Install frontend & backend apps.
 RUN mkdir -p /opt/web-console
diff --git a/modules/web-console/frontend/test/ci/Dockerfile b/modules/web-console/frontend/test/ci/Dockerfile
index 96ee599ba099e..2573534dfb1d3 100644
--- a/modules/web-console/frontend/test/ci/Dockerfile
+++ b/modules/web-console/frontend/test/ci/Dockerfile
@@ -17,23 +17,16 @@
 
 FROM alpine:edge
 
-RUN apk update && apk upgrade && \
-    apk add --no-cache bash git openssh
-
 RUN apk --no-cache --repository http://dl-3.alpinelinux.org/alpine/edge/testing/ add \
  nodejs nodejs-npm chromium xwininfo xvfb dbus eudev ttf-freefont fluxbox
 
 ENV CHROME_BIN /usr/bin/chromium-browser
 
-# Install frontend & backend apps.
-RUN mkdir -p /opt/web-console
+WORKDIR /opt/web-console/frontend
 
-# Copy source.
-WORKDIR /opt/web-console
-COPY . ./frontend
+COPY ./package*.json ./
+RUN npm install --no-optional
 
-# Install node modules.
-WORKDIR /opt/web-console/frontend
-RUN npm install
+COPY . .
 
 ENTRYPOINT ["npm", "test"]
diff --git a/modules/web-console/frontend/test/karma.conf.babel.js b/modules/web-console/frontend/test/karma.conf.babel.js
index b19259421e2bc..dcf6cb0140ff2 100644
--- a/modules/web-console/frontend/test/karma.conf.babel.js
+++ b/modules/web-console/frontend/test/karma.conf.babel.js
@@ -19,7 +19,7 @@ import path from 'path';
 
 import testCfg from '../webpack/webpack.test';
 
-export default (config) => {
+export default (/** @type {import('karma').Config} */ config) => {
     config.set({
         // Base path that will be used to resolve all patterns (eg. files, exclude).
         basePath: path.resolve('./'),
@@ -30,8 +30,10 @@ export default (config) => {
         // List of files / patterns to load in the browser.
         files: [
             'node_modules/babel-polyfill/dist/polyfill.js',
-            'test/**/*.test.js',
-            'app/**/*.spec.js'
+            'node_modules/angular/angular.js',
+            'node_modules/angular-mocks/angular-mocks.js',
+            'app/**/*.spec.js',
+            'test/**/*.test.js'
         ],
 
         plugins: [
@@ -45,7 +47,7 @@ export default (config) => {
         // Preprocess matching files before serving them to the browser
         // available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor.
         preprocessors: {
-            '{app,test}/**/*.js': ['webpack']
+            '+(app|test)/**/*.js': ['webpack']
         },
 
         webpack: testCfg,
diff --git a/modules/web-console/frontend/tsconfig.json b/modules/web-console/frontend/tsconfig.json
index df181203cad9d..382c9a5fe1321 100644
--- a/modules/web-console/frontend/tsconfig.json
+++ b/modules/web-console/frontend/tsconfig.json
@@ -6,5 +6,8 @@
         "allowJs": true,
         "checkJs": true,
         "baseUrl": "."
-    }
+    },
+    "exclude": [
+        "build"
+    ]
 }
\ No newline at end of file
diff --git a/modules/web-console/frontend/views/403.tpl.pug b/modules/web-console/frontend/views/403.tpl.pug
index 7d65442b96b20..491ff6b622651 100644
--- a/modules/web-console/frontend/views/403.tpl.pug
+++ b/modules/web-console/frontend/views/403.tpl.pug
@@ -16,7 +16,7 @@
 
 web-console-header
 
-.container.body-container
+.container--responsive.body-container
     .error-page
         h1.error-page__title 403
         h2.error-page__description You are not authorized
diff --git a/modules/web-console/frontend/views/404.tpl.pug b/modules/web-console/frontend/views/404.tpl.pug
index c4cef12bc3917..d9a0795dc73ad 100644
--- a/modules/web-console/frontend/views/404.tpl.pug
+++ b/modules/web-console/frontend/views/404.tpl.pug
@@ -16,7 +16,7 @@
 
 web-console-header
 
-.container.body-container
+.container--responsive.body-container
     .error-page
         h1.error-page__title 404
         h2.error-page__description Page not found
diff --git a/modules/web-console/frontend/views/base.pug b/modules/web-console/frontend/views/base.pug
index 525e762fd5fd3..967d469b569e0 100644
--- a/modules/web-console/frontend/views/base.pug
+++ b/modules/web-console/frontend/views/base.pug
@@ -20,7 +20,7 @@ web-console-header
     web-console-header-right
         include ./includes/header-right
 
-.container.body-container
-    .main-content(ui-view='')
+.container--responsive.body-container.flex-full-height
+    div(ui-view='').flex-full-height
 
 web-console-footer
diff --git a/modules/web-console/frontend/views/includes/header-left.pug b/modules/web-console/frontend/views/includes/header-left.pug
index 2052bbcf7cf22..bf1ee9b71bd0f 100644
--- a/modules/web-console/frontend/views/includes/header-left.pug
+++ b/modules/web-console/frontend/views/includes/header-left.pug
@@ -24,7 +24,7 @@ a.wch-nav-item(
     a(ui-sref='base.sql.tabs' ui-sref-active='active')
         | Queries
 
-.wch-content(ignite-navbar)
+.wch-additional-nav-items(ignite-navbar)
     .wch-nav-item(ng-repeat='item in navbar.items')
         div(ng-if='$root.user.becomeUsed ? item.canBecomed : item.children' ng-click='$event.stopPropagation()'
                 ng-class='{active: $state.includes(item.sref)}'
diff --git a/modules/web-console/frontend/views/includes/header-right.pug b/modules/web-console/frontend/views/includes/header-right.pug
index b223f2e68422a..d2d89aa616d57 100644
--- a/modules/web-console/frontend/views/includes/header-right.pug
+++ b/modules/web-console/frontend/views/includes/header-right.pug
@@ -23,7 +23,7 @@
 web-console-header-extension
 
 .wch-nav-item(ignite-userbar)
-    div(
+    .user-name-wrapper(
         ng-class='{active: $state.includes("base.settings")}'
         ng-click='$event.stopPropagation()'
         bs-dropdown='userbar.items'
@@ -32,4 +32,4 @@ web-console-header-extension
         data-container='self'
     )
         span {{$root.user.firstName}} {{$root.user.lastName}}
-            span.caret
+        span.caret
diff --git a/modules/web-console/frontend/views/index.pug b/modules/web-console/frontend/views/index.pug
index 6384592156c77..b9ba00bea21a8 100644
--- a/modules/web-console/frontend/views/index.pug
+++ b/modules/web-console/frontend/views/index.pug
@@ -15,7 +15,7 @@
     limitations under the License.
 
 doctype html
-html(ng-app='ignite-console' id='app' ng-strict-di)
+html
     head
         base(href='/')
 
diff --git a/modules/web-console/frontend/views/sql/cache-metadata.tpl.pug b/modules/web-console/frontend/views/sql/cache-metadata.tpl.pug
index 385960a056421..ccffeaee2f319 100644
--- a/modules/web-console/frontend/views/sql/cache-metadata.tpl.pug
+++ b/modules/web-console/frontend/views/sql/cache-metadata.tpl.pug
@@ -24,7 +24,8 @@
             span(ng-switch='node.type')
                 span(ng-switch-when='type' ng-dblclick='dblclickMetadata(paragraph, node)')
                     i.fa.fa-table
-                    label.clickable(ng-bind='node.displayName')
+                    label.clickable
+                        div.node-display(ng-bind='node.displayName' ng-attr-title='{{node.displayName}}')
                 span(ng-switch-when='plain')
                     label {{node.name}}
                 span(ng-switch-when='field' ng-dblclick='dblclickMetadata(paragraph, node)')
diff --git a/modules/web-console/frontend/views/sql/chart-settings.tpl.pug b/modules/web-console/frontend/views/sql/chart-settings.tpl.pug
index 5a1ceed507b0a..054067f9188bd 100644
--- a/modules/web-console/frontend/views/sql/chart-settings.tpl.pug
+++ b/modules/web-console/frontend/views/sql/chart-settings.tpl.pug
@@ -36,5 +36,5 @@
                 ul.chart-settings-columns-list(dnd-list='paragraph.chartValCols' dnd-drop='chartAcceptValColumn(paragraph, item)')
                     li(ng-repeat='col in paragraph.chartValCols track by $index')
                         .btn.btn-default.btn-chart-column(ng-style='chartColor($index)') {{col.label}}
-                            button.btn-chart-column-agg-fx.select-toggle(ng-change='applyChartSettings(paragraph)' ng-show='paragraphTimeSpanVisible(paragraph)' ng-style='chartColor($index)' ng-model='col.aggFx' placeholder='...' bs-select bs-options='item for item in aggregateFxs' data-container='false' tabindex='-1')
+                            button.btn-chart-column-agg-fx.select-toggle(ng-change='applyChartSettings(paragraph)' ng-show='paragraphTimeSpanVisible(paragraph)' ng-style='chartColor($index)' ng-model='col.aggFx' placeholder='...' bs-select bs-options='item for item in aggregateFxs' tabindex='-1')
                             i.fa.fa-close(ng-click='chartRemoveValColumn(paragraph, $index)')
diff --git a/modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug b/modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug
index 03c6497ac0eed..2933a3e4f2257 100644
--- a/modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug
+++ b/modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug
@@ -24,7 +24,7 @@
                 .col-sm-4
                     input.form-control(id='paragraph-rate' ng-init='value = paragraph.rate.value' ng-model='value' type='number' min='1' required ignite-auto-focus)
                 .col-sm-8(style='padding-left: 5px')
-                    button.form-control.select-toggle(id='paragraph-unit' ng-init='unit = paragraph.rate.unit' ng-model='unit' required placeholder='Time unit' bs-select bs-options='item.value as item.label for item in timeUnit' data-container='false' tabindex='0')
+                    button.form-control.select-toggle(id='paragraph-unit' ng-init='unit = paragraph.rate.unit' ng-model='unit' required placeholder='Time unit' bs-select bs-options='item.value as item.label for item in timeUnit' tabindex='0')
             .form-actions(style='margin-top: 30px; padding: 5px')
                 button.btn.btn-primary(id='paragraph-rate-start' ng-disabled='popoverForm.$invalid' ng-click='startRefresh(paragraph, value, unit); $hide()') Start
                 button.btn.btn-primary.btn-default(id='paragraph-rate-stop' ng-click='stopRefresh(paragraph); $hide()' ng-disabled='!paragraph.rate.installed') Stop
diff --git a/modules/web-console/frontend/views/templates/agent-download.tpl.pug b/modules/web-console/frontend/views/templates/agent-download.tpl.pug
index fdb3a059d0a56..4da9fadc9b2f6 100644
--- a/modules/web-console/frontend/views/templates/agent-download.tpl.pug
+++ b/modules/web-console/frontend/views/templates/agent-download.tpl.pug
@@ -14,6 +14,8 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
+include /app/helpers/jade/mixins
+
 .modal.modal--ignite.theme--ignite.center(tabindex='-1' role='dialog')
     .modal-dialog(ng-switch='ctrl.status')
         .modal-content(ng-switch-when='agentMissing')
@@ -30,10 +32,17 @@
                 .modal-advanced-options
                     i.fa(ng-class='showToken ? "fa-chevron-circle-down" : "fa-chevron-circle-right"' ng-click='showToken = !showToken')
                     a(ng-click='showToken = !showToken') {{showToken ? 'Hide security token...' : 'Show security token...'}}
-                p(ng-show='showToken')
-                    label.labelField Security token: {{user.becameToken || user.token}}
-                    i.tipLabel.fa.fa-clipboard(ignite-copy-to-clipboard='{{user.becameToken || user.token}}' bs-tooltip='' data-title='Copy security token to clipboard')
-                    i.tipLabel.icon-help(ng-if=lines bs-tooltip='' data-title='The security token is used for authorization of web agent')
+                div(ng-show='showToken')
+                    +form-field__text({
+                        label: 'Security Token:',
+                        model: 'ctrl.securityToken',
+                        tip: 'The security token is used for authentication of web agent',
+                        name: '"securityToken"'
+                    })(
+                        autocomplete='security-token'
+                        ng-disabled='::true'
+                        copy-input-value-button='Copy security token to clipboard'
+                    )
 
             .modal-footer
                 button.btn-ignite.btn-ignite--link-success(ng-click='ctrl.back()') {{::ctrl.backText}}
diff --git a/modules/web-console/frontend/views/templates/confirm.tpl.pug b/modules/web-console/frontend/views/templates/confirm.tpl.pug
index 1b1560cc3be96..9f5e2bbb2d3cd 100644
--- a/modules/web-console/frontend/views/templates/confirm.tpl.pug
+++ b/modules/web-console/frontend/views/templates/confirm.tpl.pug
@@ -29,6 +29,6 @@
                 button#confirm-btn-cancel.btn-ignite.btn-ignite--link-success(ng-click='confirmCancel()') Cancel
 
                 button#confirm-btn-no.btn-ignite.btn-ignite--link-success(ng-if='yesNo' ng-click='confirmNo()') No
-                button#confirm-btn-yes.btn-ignite.btn-ignite--success(ng-if='yesNo' ng-click='confirmYes()') Yes
+                button#confirm-btn-yes.btn-ignite.btn-ignite--success(ignite-auto-focus ng-if='yesNo' ng-click='confirmYes()') Yes
 
-                button#confirm-btn-ok.btn-ignite.btn-ignite--success(ng-if='!yesNo' ng-click='confirmYes()') Confirm
+                button#confirm-btn-ok.btn-ignite.btn-ignite--success(ignite-auto-focus ng-if='!yesNo' ng-click='confirmYes()') Confirm
diff --git a/modules/web-console/frontend/views/templates/dropdown.tpl.pug b/modules/web-console/frontend/views/templates/dropdown.tpl.pug
index 2ee8616a61ee8..1ddd775c2a68a 100644
--- a/modules/web-console/frontend/views/templates/dropdown.tpl.pug
+++ b/modules/web-console/frontend/views/templates/dropdown.tpl.pug
@@ -18,7 +18,7 @@ ul.dropdown-menu(tabindex='-1' role='menu' ng-show='content && content.length')
     li(role='presentation' ui-sref-active='active' ng-class='{divider: item.divider, active: item.active}' ng-repeat='item in content')
         a(role='menuitem' tabindex='-1' ui-sref='{{item.sref}}' ng-if='!item.action && !item.divider && item.sref' ng-bind='item.text')
         a(role='menuitem' tabindex='-1' ng-href='{{item.href}}' ng-if='!item.action && !item.divider && item.href' target='{{item.target || ""}}' ng-bind='item.text')
-        a(role='menuitem' tabindex='-1' href='javascript:void(0)' ng-if='!item.action && !item.divider && item.click' ng-click='$eval(item.click);$hide()' ng-bind='item.text')
+        a(role='menuitem' tabindex='-1' ng-if='!item.action && !item.divider && item.click' ng-click='$eval(item.click);$hide()' ng-bind='item.text')
         div(role='menuitem' ng-if='item.action')
             i.fa.pull-right(class='{{ item.action.icon }}' ng-click='item.action.click(item.data)' bs-tooltip data-title='{{ item.action.tooltip }}' data-trigger='hover' data-placement='bottom')
             div: a(ui-sref='{{ item.sref }}' ng-bind='item.text')
diff --git a/modules/web-console/frontend/webpack/webpack.common.js b/modules/web-console/frontend/webpack/webpack.common.js
index 3b9aef06c47a5..f2901e1bd870d 100644
--- a/modules/web-console/frontend/webpack/webpack.common.js
+++ b/modules/web-console/frontend/webpack/webpack.common.js
@@ -23,26 +23,24 @@ import presetEs2015 from 'babel-preset-es2015';
 import presetStage1 from 'babel-preset-stage-1';
 
 import CopyWebpackPlugin from 'copy-webpack-plugin';
-import ExtractTextPlugin from 'extract-text-webpack-plugin';
 import HtmlWebpackPlugin from 'html-webpack-plugin';
 import ProgressBarPlugin from 'progress-bar-webpack-plugin';
 
 import eslintFormatter from 'eslint-friendly-formatter';
 
-const basedir = path.resolve('./');
-const contentBase = path.resolve('public');
-const node_modules = path.resolve('node_modules');
+const basedir = path.join(__dirname, '../');
+const contentBase = path.join(basedir, 'public');
+const node_modules = path.join(basedir, 'node_modules');
+const app = path.join(basedir, 'app');
 
-const app = path.resolve('app');
-const IgniteModules = process.env.IGNITE_MODULES ? path.join(process.env.IGNITE_MODULES, 'frontend') : path.resolve('ignite_modules');
-
-export default {
+/** @type {webpack.Configuration} */
+const config = {
     node: {
         fs: 'empty'
     },
     // Entry points.
     entry: {
-        app: path.join(app, 'app.js'),
+        app: path.join(basedir, 'index.js'),
         browserUpdate: path.join(app, 'browserUpdate', 'index.js')
     },
 
@@ -60,9 +58,7 @@ export default {
         alias: {
             app,
             images: path.join(basedir, 'public/images'),
-            views: path.join(basedir, 'views'),
-            Controllers: path.join(basedir, 'controllers'),
-            IgniteModules
+            views: path.join(basedir, 'views')
         }
     },
 
@@ -79,7 +75,12 @@ export default {
             // Exclude tpl.pug files to import in bundle.
             {
                 test: /^(?:(?!tpl\.pug$).)*\.pug$/, // TODO: check this regexp for correct.
-                loader: `pug-html?basedir=${basedir}`
+                use: {
+                    loader: 'pug-html',
+                    options: {
+                        basedir
+                    }
+                }
             },
 
             // Render .tpl.pug files to assets folder.
@@ -94,7 +95,7 @@ export default {
             {
                 test: /\.js$/,
                 enforce: 'pre',
-                exclude: [node_modules],
+                exclude: [/node_modules/],
                 use: [{
                     loader: 'eslint',
                     options: {
@@ -106,7 +107,7 @@ export default {
                 }]
             },
             {
-                test: /\.(js)$/,
+                test: /\.js$/,
                 exclude: [node_modules],
                 use: [{
                     loader: 'babel-loader',
@@ -124,17 +125,27 @@ export default {
             },
             {
                 test: /\.(ttf|eot|svg|woff(2)?)(\?v=[\d.]+)?(\?[a-z0-9#-]+)?$/,
-                exclude: [contentBase, IgniteModules],
-                loader: 'file?name=assets/fonts/[name].[ext]'
+                exclude: [contentBase, /\.icon\.svg$/],
+                use: 'file?name=assets/fonts/[name].[ext]'
             },
             {
-                test: /.*\.svg$/,
-                include: [contentBase, IgniteModules],
-                use: ['svg-sprite-loader']
+                test: /\.icon\.svg$/,
+                use: {
+                    loader: 'svg-sprite-loader',
+                    options: {
+                        symbolRegExp: /\w+(?=\.icon\.\w+$)/,
+                        symbolId: '[0]'
+                    }
+                }
+            },
+            {
+                test: /.*\.url\.svg$/,
+                include: [contentBase],
+                use: 'file?name=assets/fonts/[name].[ext]'
             },
             {
                 test: /\.(jpe?g|png|gif)$/i,
-                loader: 'file?name=assets/images/[name].[hash].[ext]'
+                use: 'file?name=assets/images/[name].[hash].[ext]'
             },
             {
                 test: require.resolve('jquery'),
@@ -145,7 +156,7 @@ export default {
             },
             {
                 test: require.resolve('nvd3'),
-                use: ['expose-loader?nv']
+                use: 'expose-loader?nv'
             }
         ]
     },
@@ -178,18 +189,13 @@ export default {
         }),
         new webpack.optimize.AggressiveMergingPlugin({moveToParents: true}),
         new HtmlWebpackPlugin({
-            template: './views/index.pug'
+            template: path.join(basedir, './views/index.pug')
         }),
-        new ExtractTextPlugin({filename: 'assets/css/[name].[hash].css', allChunks: true}),
         new CopyWebpackPlugin([
-            { context: 'public', from: '**/*.png' },
-            { context: 'public', from: '**/*.svg' },
-            { context: 'public', from: '**/*.ico' },
-            // Ignite modules.
-            { context: IgniteModules, from: '**/*.png', force: true },
-            { context: IgniteModules, from: '**/*.svg', force: true },
-            { context: IgniteModules, from: '**/*.ico', force: true }
+            { context: 'public', from: '**/*.{png,svg,ico}' }
         ]),
         new ProgressBarPlugin()
     ]
 };
+
+export default config;
diff --git a/modules/web-console/frontend/webpack/webpack.dev.babel.js b/modules/web-console/frontend/webpack/webpack.dev.babel.js
index d17fbc723956c..c5950ee6e23e9 100644
--- a/modules/web-console/frontend/webpack/webpack.dev.babel.js
+++ b/modules/web-console/frontend/webpack/webpack.dev.babel.js
@@ -15,14 +15,13 @@
  * limitations under the License.
  */
 
-import webpack from 'webpack';
 import merge from 'webpack-merge';
 
 import path from 'path';
 
 import commonCfg from './webpack.common';
 
-import ExtractTextPlugin from 'extract-text-webpack-plugin';
+const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 
 const backendPort = process.env.BACKEND_PORT || 3000;
 const devServerPort = process.env.PORT || 9000;
@@ -41,33 +40,33 @@ export default merge(commonCfg, {
             },
             {
                 test: /\.scss$/,
-                // Version without extract plugin fails on some machines. https://github.com/sass/node-sass/issues/1895
-                use: ExtractTextPlugin.extract({
-                    fallback: 'style-loader',
-                    use: [
-                        {
-                            loader: 'css',
-                            options: {
-                                sourceMap: true
-                            }
-                        },
-                        {
-                            loader: 'sass',
-                            options: {
-                                sourceMap: true
-                            }
+                use: [
+                    MiniCssExtractPlugin.loader, // style-loader does not work with styles in IgniteModules
+                    {
+                        loader: 'css-loader',
+                        options: {
+                            sourceMap: true
                         }
-                    ]
-                })
+                    },
+                    {
+                        loader: 'sass-loader',
+                        options: {
+                            sourceMap: true,
+                            includePaths: [ path.join(__dirname, '../') ]
+                        }
+                    }
+                ]
             }
         ]
     },
+    plugins: [
+        new MiniCssExtractPlugin({filename: 'assets/css/[name].css'})
+    ],
     devServer: {
         compress: true,
         historyApiFallback: true,
         disableHostCheck: true,
         contentBase: path.resolve('build'),
-        // hot: true,
         inline: true,
         proxy: {
             '/socket.io': {
@@ -78,7 +77,7 @@ export default merge(commonCfg, {
                 target: `http://localhost:${backendPort}`,
                 ws: true
             },
-            '/api/v1/*': {
+            '/api/*': {
                 target: `http://localhost:${backendPort}`
             }
         },
diff --git a/modules/web-console/frontend/webpack/webpack.prod.babel.js b/modules/web-console/frontend/webpack/webpack.prod.babel.js
index cced53d301cd7..a5aa1c61f4735 100644
--- a/modules/web-console/frontend/webpack/webpack.prod.babel.js
+++ b/modules/web-console/frontend/webpack/webpack.prod.babel.js
@@ -15,13 +15,16 @@
  * limitations under the License.
  */
 
+import path from 'path';
 import merge from 'webpack-merge';
 
-import ExtractTextPlugin from 'extract-text-webpack-plugin';
+const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 import UglifyJSPlugin from 'uglifyjs-webpack-plugin';
 
 import commonCfg from './webpack.common';
 
+const basedir = path.join(__dirname, '../');
+
 export default merge(commonCfg, {
     bail: true, // Cancel build on error.
     mode: 'production',
@@ -29,20 +32,22 @@ export default merge(commonCfg, {
         rules: [
             {
                 test: /\.css$/,
-                use: ExtractTextPlugin.extract({
-                    fallback: 'style',
-                    use: ['css']
-                })
+                use: [MiniCssExtractPlugin.loader, 'css-loader']
             },
             {
                 test: /\.scss$/,
-                use: ExtractTextPlugin.extract({
-                    fallback: 'style-loader',
-                    use: ['css', 'sass']
-                })
+                use: [MiniCssExtractPlugin.loader, 'css-loader', {
+                    loader: 'sass',
+                    options: {
+                        includePaths: [basedir]
+                    }
+                }]
             }
         ]
     },
+    plugins: [
+        new MiniCssExtractPlugin({filename: 'assets/css/[name].[hash].css'})
+    ],
     optimization: {
         minimizer: [
             new UglifyJSPlugin({
diff --git a/modules/web-console/pom.xml b/modules/web-console/pom.xml
index 66a8f507c0217..bfd9ffeb0c3cf 100644
--- a/modules/web-console/pom.xml
+++ b/modules/web-console/pom.xml
@@ -35,7 +35,12 @@
     <url>http://ignite.apache.org</url>
 
     <properties>
-        <node.version>v8.9.0</node.version>
+        <node.version>v8.11.2</node.version>
+        <docker.registry.host>docker.io</docker.registry.host>
+        <docker.repository>apacheignite</docker.repository>
+        <docker.backend.image>web-console-backend</docker.backend.image>
+        <docker.frontend.image>web-console-frontend</docker.frontend.image>
+        <docker.standalone.image>web-console-standalone</docker.standalone.image>
     </properties>
 
     <dependencies>
@@ -47,210 +52,370 @@
         </dependency>
     </dependencies>
 
-    <build>
-        <pluginManagement>
-            <plugins>
-                 <plugin>
-                     <groupId>com.github.eirslett</groupId>
-                     <artifactId>frontend-maven-plugin</artifactId>
-                     <version>1.6</version>
-                     <configuration>
-                         <nodeVersion>${node.version}</nodeVersion>
-                         <installDirectory>target</installDirectory>
-                     </configuration>
-                 </plugin>
-            </plugins>
-        </pluginManagement>
+    <profiles>
+        <profile>
+            <id>direct-install</id>
 
-        <plugins>
-            <plugin>
-                <groupId>com.github.eirslett</groupId>
-                <artifactId>frontend-maven-plugin</artifactId>
-
-                <executions>
-                    <execution>
-                        <id>install node and npm for frontend</id>
-                        <goals>
-                            <goal>install-node-and-npm</goal>
-                        </goals>
-                    </execution>
-
-                    <execution>
-                        <id>download dependencies for frontend</id>
-                        <goals>
-                            <goal>npm</goal>
-                        </goals>
-
-                        <configuration>
-                            <workingDirectory>frontend</workingDirectory>
-                            <arguments>install --no-optional --prod</arguments>
-                        </configuration>
-                    </execution>
-
-                    <execution>
-                        <id>download dependencies for backend</id>
-                        <goals>
-                            <goal>npm</goal>
-                        </goals>
-
-                        <configuration>
-                            <workingDirectory>backend</workingDirectory>
-                            <arguments>install --no-optional</arguments>
-                        </configuration>
-                    </execution>
-
-                    <execution>
-                        <id>build frontend</id>
-                        <goals>
-                            <goal>npm</goal>
-                        </goals>
-
-                        <phase>compile</phase>
-
-                        <configuration>
-                            <workingDirectory>frontend</workingDirectory>
-                            <arguments>run build</arguments>
-                            <environmentVariables>
-                                <NODE_ENV>production</NODE_ENV>
-                            </environmentVariables>
-                        </configuration>
-                    </execution>
-
-                    <execution>
-                        <id>build backend</id>
-                        <goals>
-                            <goal>npm</goal>
-                        </goals>
-
-                        <phase>compile</phase>
-
-                        <configuration>
-                            <workingDirectory>backend</workingDirectory>
-                            <arguments>run build</arguments>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
+            <build>
+                <pluginManagement>
+                    <plugins>
+                        <plugin>
+                            <groupId>com.github.eirslett</groupId>
+                            <artifactId>frontend-maven-plugin</artifactId>
+                            <version>1.6</version>
+                            <configuration>
+                                <nodeVersion>${node.version}</nodeVersion>
+                                <installDirectory>target</installDirectory>
+                            </configuration>
+                        </plugin>
+                    </plugins>
+                </pluginManagement>
 
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-clean-plugin</artifactId>
-                <version>2.5</version>
-                <executions>
-                    <execution>
-                        <id>clean-frontend-build</id>
-                        <goals>
-                            <goal>clean</goal>
-                        </goals>
-                        <phase>process-resources</phase>
-                        <configuration>
-                            <excludeDefaultDirectories>true</excludeDefaultDirectories>
-                            <filesets>
-                                <fileset>
-                                    <directory>${project.basedir}/frontend/build</directory>
-                                </fileset>
-                            </filesets>
-                        </configuration>
-                    </execution>
-
-                    <execution>
-                        <id>clean-backend-build</id>
-                        <goals>
-                            <goal>clean</goal>
-                        </goals>
-                        <phase>process-resources</phase>
-                        <configuration>
-                            <excludeDefaultDirectories>true</excludeDefaultDirectories>
-                            <filesets>
-                                <fileset>
-                                    <directory>${project.basedir}/backend/build</directory>
-                                </fileset>
-                            </filesets>
-                        </configuration>
-                    </execution>
-
-                    <execution>
-                        <id>remove-old-getos</id>
-                        <goals>
-                            <goal>clean</goal>
-                        </goals>
-                        <phase>process-resources</phase>
-                        <configuration>
-                            <excludeDefaultDirectories>true</excludeDefaultDirectories>
-                            <filesets>
-                                <fileset>
-                                    <directory>${project.basedir}/backend/node_modules/mongodb-download/node_modules</directory>
-                                </fileset>
-                            </filesets>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
+                <plugins>
+                    <plugin>
+                        <groupId>com.github.eirslett</groupId>
+                        <artifactId>frontend-maven-plugin</artifactId>
 
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-antrun-plugin</artifactId>
-                <version>1.7</version>
-                <executions>
-                    <execution>
-                        <id>fixed-rhel-detection</id>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                        <phase>process-resources</phase>
-                        <configuration>
-                            <target>
-                                <replace dir="${basedir}/backend/node_modules/mongodb-download/built"
-                                         token="/rhel/" value="/Red Hat Linux/i.test(os.dist) || /rhel/">
-                                    <include name="mongodb-download.js"/>
-                                </replace>
-                            </target>
-                        </configuration>
-                    </execution>
-
-                    <execution>
-                        <id>fixed-download-url</id>
-                        <goals>
-                            <goal>run</goal>
-                        </goals>
-                        <phase>process-resources</phase>
-                        <configuration>
-                            <target>
-                                <replace dir="${basedir}/backend/node_modules/mongodb-download/built"
-                                         token="http://downloads.mongodb.org" value="https://fastdl.mongodb.org">
-                                    <include name="mongodb-download.js"/>
-                                </replace>
-                            </target>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
+                        <executions>
+                            <execution>
+                                <id>install node and npm for frontend</id>
+                                <goals>
+                                    <goal>install-node-and-npm</goal>
+                                </goals>
+                            </execution>
 
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.4</version>
-                <inherited>false</inherited>
-
-                <executions>
-                    <execution>
-                        <id>release-direct-install</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                        <configuration>
-                            <descriptors>
-                                <descriptor>assembly/direct-install.xml</descriptor>
-                            </descriptors>
-                            <finalName>ignite-web-console-direct-install-${project.version}</finalName>
-                            <outputDirectory>target</outputDirectory>
-                            <appendAssemblyId>false</appendAssemblyId>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
+                            <execution>
+                                <id>download dependencies for frontend</id>
+                                <goals>
+                                    <goal>npm</goal>
+                                </goals>
+
+                                <configuration>
+                                    <workingDirectory>frontend</workingDirectory>
+                                    <arguments>install --no-optional</arguments>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>download dependencies for backend</id>
+                                <goals>
+                                    <goal>npm</goal>
+                                </goals>
+
+                                <configuration>
+                                    <workingDirectory>backend</workingDirectory>
+                                    <arguments>install --no-optional --production</arguments>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>build frontend</id>
+                                <goals>
+                                    <goal>npm</goal>
+                                </goals>
+
+                                <phase>compile</phase>
+
+                                <configuration>
+                                    <workingDirectory>frontend</workingDirectory>
+                                    <arguments>run build</arguments>
+                                    <environmentVariables>
+                                        <NODE_ENV>production</NODE_ENV>
+                                    </environmentVariables>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>build backend</id>
+                                <goals>
+                                    <goal>npm</goal>
+                                </goals>
 
+                                <phase>compile</phase>
+
+                                <configuration>
+                                    <workingDirectory>backend</workingDirectory>
+                                    <arguments>run build</arguments>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-clean-plugin</artifactId>
+                        <version>2.5</version>
+                        <executions>
+                            <execution>
+                                <id>clean-frontend-build</id>
+                                <goals>
+                                    <goal>clean</goal>
+                                </goals>
+                                <phase>process-resources</phase>
+                                <configuration>
+                                    <excludeDefaultDirectories>true</excludeDefaultDirectories>
+                                    <filesets>
+                                        <fileset>
+                                            <directory>${project.basedir}/frontend/build</directory>
+                                        </fileset>
+                                    </filesets>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>clean-backend-build</id>
+                                <goals>
+                                    <goal>clean</goal>
+                                </goals>
+                                <phase>process-resources</phase>
+                                <configuration>
+                                    <excludeDefaultDirectories>true</excludeDefaultDirectories>
+                                    <filesets>
+                                        <fileset>
+                                            <directory>${project.basedir}/backend/build</directory>
+                                        </fileset>
+                                    </filesets>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-antrun-plugin</artifactId>
+                        <version>1.7</version>
+
+                        <dependencies>
+                            <dependency>
+                                <groupId>ant-contrib</groupId>
+                                <artifactId>ant-contrib</artifactId>
+                                <version>1.0b3</version>
+                                <exclusions>
+                                    <exclusion>
+                                        <groupId>ant</groupId>
+                                        <artifactId>ant</artifactId>
+                                    </exclusion>
+                                </exclusions>
+                            </dependency>
+                        </dependencies>
+                        <executions>
+                            <execution>
+                                <id>fixed-getos-logic-path</id>
+                                <goals>
+                                    <goal>run</goal>
+                                </goals>
+                                <phase>process-resources</phase>
+                                <configuration>
+
+                                    <tasks>
+                                        <taskdef resource="net/sf/antcontrib/antlib.xml"/>
+                                        <if>
+                                            <available
+                                                file="${project.basedir}/backend/node_modules/mongodb-download/node_modules/getos/index.js"/>
+                                            <then>
+                                                <replace dir="${project.basedir}/backend/node_modules/mongodb-download/node_modules/getos"
+                                                         token='"./logic/"' value='__dirname+"/logic/"'>
+                                                    <include name="index.js"/>
+                                                </replace>
+                                            </then>
+                                        </if>
+                                    </tasks>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>fixed-rhel-detection</id>
+                                <goals>
+                                    <goal>run</goal>
+                                </goals>
+                                <phase>process-resources</phase>
+                                <configuration>
+                                    <target>
+                                        <replace dir="${project.basedir}/backend/node_modules/mongodb-download/built"
+                                                 token="/rhel/" value="/Red Hat Linux/i.test(os.dist) || /rhel/">
+                                            <include name="mongodb-download.js"/>
+                                        </replace>
+                                    </target>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>fixed-download-url</id>
+                                <goals>
+                                    <goal>run</goal>
+                                </goals>
+                                <phase>process-resources</phase>
+                                <configuration>
+                                    <target>
+                                        <replace dir="${project.basedir}/backend/node_modules/mongodb-download/built"
+                                                 token="https://downloads.mongodb.org" value="https://fastdl.mongodb.org">
+                                            <include name="mongodb-download.js"/>
+                                        </replace>
+                                    </target>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+
+                    <plugin>
+                        <groupId>org.apache.maven.plugins</groupId>
+                        <artifactId>maven-assembly-plugin</artifactId>
+                        <version>2.4</version>
+                        <inherited>false</inherited>
+
+                        <executions>
+                            <execution>
+                                <id>release-direct-install</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>single</goal>
+                                </goals>
+                                <configuration>
+                                    <descriptors>
+                                        <descriptor>assembly/direct-install.xml</descriptor>
+                                    </descriptors>
+                                    <finalName>ignite-web-console-direct-install-${project.version}</finalName>
+                                    <outputDirectory>target</outputDirectory>
+                                    <appendAssemblyId>false</appendAssemblyId>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+
+        <profile>
+            <id>docker-image</id>
+
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.codehaus.mojo</groupId>
+                        <artifactId>exec-maven-plugin</artifactId>
+                        <executions>
+                            <execution>
+                                <id>docker-build-backend</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>docker</executable>
+                                    <arguments>
+                                        <argument>build</argument>
+                                        <argument>-f</argument>
+                                        <argument>docker/compose/backend/Dockerfile</argument>
+                                        <argument>-t</argument>
+                                        <argument>
+                                            ${docker.registry.host}/${docker.repository}/${docker.backend.image}:${project.version}
+                                        </argument>
+                                        <argument>${project.basedir}</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>docker-build-frontend</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>docker</executable>
+                                    <arguments>
+                                        <argument>build</argument>
+                                        <argument>-f</argument>
+                                        <argument>docker/compose/frontend/Dockerfile</argument>
+                                        <argument>-t</argument>
+                                        <argument>
+                                            ${docker.registry.host}/${docker.repository}/${docker.frontend.image}:${project.version}
+                                        </argument>
+                                        <argument>${project.basedir}</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>docker-build-standalone</id>
+                                <phase>package</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>docker</executable>
+                                    <arguments>
+                                        <argument>build</argument>
+                                        <argument>-f</argument>
+                                        <argument>docker/standalone/Dockerfile</argument>
+                                        <argument>-t</argument>
+                                        <argument>
+                                            ${docker.registry.host}/${docker.repository}/${docker.standalone.image}:${project.version}
+                                        </argument>
+                                        <argument>${project.basedir}</argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>docker-push-backend</id>
+                                <phase>deploy</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>docker</executable>
+                                    <arguments>
+                                        <argument>push</argument>
+                                        <argument>
+                                            ${docker.registry.host}/${docker.repository}/${docker.backend.image}:${project.version}
+                                        </argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>docker-push-frontend</id>
+                                <phase>deploy</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>docker</executable>
+                                    <arguments>
+                                        <argument>push</argument>
+                                        <argument>
+                                            ${docker.registry.host}/${docker.repository}/${docker.frontend.image}:${project.version}
+                                        </argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+
+                            <execution>
+                                <id>docker-push-standalone</id>
+                                <phase>deploy</phase>
+                                <goals>
+                                    <goal>exec</goal>
+                                </goals>
+                                <configuration>
+                                    <executable>docker</executable>
+                                    <arguments>
+                                        <argument>push</argument>
+                                        <argument>
+                                            ${docker.registry.host}/${docker.repository}/${docker.standalone.image}:${project.version}
+                                        </argument>
+                                    </arguments>
+                                </configuration>
+                            </execution>
+                        </executions>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+
+    <build>
+        <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 9340417248da1..579f236536e95 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -114,7 +114,7 @@ public class AgentLauncher {
             Exception ignore = X.cause(e, SSLHandshakeException.class);
 
             if (ignore != null) {
-                log.error("Failed to establish SSL connection to server, due to errors with SSL handshake.");
+                log.error("Failed to establish SSL connection to server, due to errors with SSL handshake:", e);
                 log.error("Add to environment variable JVM_OPTS parameter \"-Dtrust.all=true\" to skip certificate validation in case of using self-signed certificate.");
 
                 System.exit(1);
@@ -123,7 +123,7 @@ public class AgentLauncher {
             ignore = X.cause(e, UnknownHostException.class);
 
             if (ignore != null) {
-                log.error("Failed to establish connection to server, due to errors with DNS or missing proxy settings.");
+                log.error("Failed to establish connection to server, due to errors with DNS or missing proxy settings.", e);
                 log.error("Documentation for proxy configuration can be found here: http://apacheignite.readme.io/docs/web-agent#section-proxy-configuration");
 
                 System.exit(1);
@@ -323,6 +323,8 @@ public static void main(String[] args) throws Exception {
 
         // Workaround for use self-signed certificate
         if (Boolean.getBoolean("trust.all")) {
+            log.info("Trust to all certificates mode is enabled.");
+
             SSLContext ctx = SSLContext.getInstance("TLS");
 
             // Create an SSLContext that uses our TrustManager
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
index 0c7560c207da9..1c93798b28819 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
@@ -65,6 +65,9 @@ public class ClusterListener implements AutoCloseable {
     /** */
     private static final IgniteLogger log = new Slf4jLogger(LoggerFactory.getLogger(ClusterListener.class));
 
+    /** */
+    private static final IgniteProductVersion IGNITE_2_0 = IgniteProductVersion.fromString("2.0.0");
+
     /** */
     private static final IgniteProductVersion IGNITE_2_1 = IgniteProductVersion.fromString("2.1.0");
 
@@ -418,6 +421,10 @@ private RestResult topology(boolean full) throws IOException {
          * @throws IOException If failed to collect cluster active state.
          */
         public boolean active(IgniteProductVersion ver, UUID nid) throws IOException {
+            // 1.x clusters are always active.
+            if (ver.compareTo(IGNITE_2_0) < 0)
+                return true;
+
             Map<String, Object> params = U.newHashMap(10);
 
             boolean v23 = ver.compareTo(IGNITE_2_3) >= 0;
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
index 2555ee15f2dab..6c54786597a46 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentClusterDemo.java
@@ -115,6 +115,8 @@ private static IgniteConfiguration igniteConfiguration(int basePort, int gridIdx
         throws IgniteCheckedException {
         IgniteConfiguration cfg = new IgniteConfiguration();
 
+        cfg.setGridLogger(new Slf4jLogger());
+
         cfg.setIgniteInstanceName((client ? CLN_NODE_NAME : SRV_NODE_NAME) + gridIdx);
         cfg.setLocalHost("127.0.0.1");
         cfg.setEventStorageSpi(new MemoryEventStorageSpi());
@@ -180,8 +182,7 @@ private static IgniteConfiguration igniteConfiguration(int basePort, int gridIdx
 
         cfg.setDataStorageConfiguration(dataStorageCfg);
 
-        if (client)
-            cfg.setClientMode(true);
+        cfg.setClientMode(client);
 
         return cfg;
     }
diff --git a/parent/pom.xml b/parent/pom.xml
index 3decc16612374..5c66c3a0e1538 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -922,19 +922,19 @@
                                         <exclude>**/*.cmd</exclude>
                                         <exclude>**/*.ps1</exclude>
                                         <exclude>**/*.json</exclude>
-                                        <!--web console-->
-                                        <exclude>**/.babelrc</exclude>
-                                        <exclude>**/.eslintrc</exclude>
                                         <exclude>**/.dockerignore</exclude>
-                                        <exclude>**/backend/config/settings.json.sample</exclude>
-                                        <exclude>**/backend/node_modules/**</exclude>
-                                        <exclude>**/e2e/testcafe/node_modules/**</exclude>
-                                        <exclude>**/frontend/build/**</exclude>
-                                        <exclude>**/frontend/public/images/**/*.png</exclude>
-                                        <exclude>**/frontend/public/images/**/*.svg</exclude>
-                                        <exclude>**/frontend/ignite_modules/**</exclude>
-                                        <exclude>**/frontend/ignite_modules_temp/**</exclude>
-                                        <exclude>**/frontend/node_modules/**</exclude>
+                                        <!--web console-->
+                                        <exclude>**/web-console/**/.eslintrc</exclude>
+                                        <exclude>**/web-console/**/.babelrc</exclude>
+                                        <exclude>**/web-console/**/*.json</exclude>
+                                        <exclude>**/web-console/**/*.json.sample</exclude>
+                                        <exclude>**/web-console/backend/build/**</exclude>
+                                        <exclude>**/web-console/backend/node_modules/**</exclude>
+                                        <exclude>**/web-console/e2e/testcafe/node_modules/**</exclude>
+                                        <exclude>**/web-console/frontend/build/**</exclude>
+                                        <exclude>**/web-console/frontend/node_modules/**</exclude>
+                                        <exclude>**/web-console/frontend/**/*.png</exclude>
+                                        <exclude>**/web-console/frontend/**/*.svg</exclude>
                                         <!--RPM packaging, IGNITE-7107-->
                                         <exclude>**/packaging/rpm/SOURCES/name.service</exclude>
                                         <exclude>**/packaging/rpm/SOURCES/service.sh</exclude>

From 8799faa32eb0fc5bb67e5bfe1e7e72b7baab7d97 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 22 Aug 2018 11:51:58 +0700
Subject: [PATCH 316/543] GG-13195 Updated classnames.properties.

---
 .../resources/META-INF/classnames.properties  | 88 ++++++++++++++-----
 1 file changed, 68 insertions(+), 20 deletions(-)

diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index a94048f90fb13..cc571b482723d 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -258,8 +258,12 @@ org.apache.ignite.internal.IgniteMessagingImpl
 org.apache.ignite.internal.IgniteNeedReconnectException
 org.apache.ignite.internal.IgniteSchedulerImpl
 org.apache.ignite.internal.IgniteServicesImpl
+org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$2
+org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$3
 org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance$4
 org.apache.ignite.internal.NodeStoppingException
+org.apache.ignite.internal.SecurityCredentialsAttrFilterPredicate
+org.apache.ignite.internal.TransactionMetricsMxBeanImpl
 org.apache.ignite.internal.binary.BinaryEnumObjectImpl
 org.apache.ignite.internal.binary.BinaryFieldMetadata
 org.apache.ignite.internal.binary.BinaryMetadata
@@ -473,6 +477,7 @@ org.apache.ignite.internal.processors.cache.ClientCacheChangeDiscoveryMessage
 org.apache.ignite.internal.processors.cache.ClientCacheChangeDummyDiscoveryMessage
 org.apache.ignite.internal.processors.cache.ClusterCachesInfo$1$1
 org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch
+org.apache.ignite.internal.processors.cache.DynamicCacheChangeFailureMessage
 org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest
 org.apache.ignite.internal.processors.cache.EntryProcessorResourceInjectorProxy
 org.apache.ignite.internal.processors.cache.ExchangeActions$1
@@ -502,7 +507,6 @@ org.apache.ignite.internal.processors.cache.GridCacheAdapter$50
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$51
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$53
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$54
-org.apache.ignite.internal.processors.cache.GridCacheAdapter$54$1
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$55
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$56
 org.apache.ignite.internal.processors.cache.GridCacheAdapter$6
@@ -644,6 +648,7 @@ org.apache.ignite.internal.processors.cache.KeyCacheObject
 org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl
 org.apache.ignite.internal.processors.cache.QueryCursorImpl$State
 org.apache.ignite.internal.processors.cache.StoredCacheData
+org.apache.ignite.internal.processors.cache.TxTimeoutOnPartitionMapExchangeChangeMessage
 org.apache.ignite.internal.processors.cache.WalStateAbstractMessage
 org.apache.ignite.internal.processors.cache.WalStateAckMessage
 org.apache.ignite.internal.processors.cache.WalStateFinishMessage
@@ -710,6 +715,7 @@ org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartit
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLockFuture$1
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLockFuture$2
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLockFuture$3
+org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLockFuture$4
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLockRequest
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLockResponse
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState
@@ -735,6 +741,7 @@ org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRespo
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal$1
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal$2
+org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal$3
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter$1
 org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter$2
@@ -854,7 +861,8 @@ org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPar
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$6
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$7
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8
-org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$8$1$1
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$9
+org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$9$1$1
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$ExchangeLocalState
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture$ExchangeType
 org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage
@@ -890,7 +898,6 @@ org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$2
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$3
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$4
-org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$5
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$LockTimeoutObject$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockFuture$MiniFuture$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest
@@ -908,6 +915,7 @@ org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticT
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$5
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFuture$MiniFuture$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFutureAdapter$1
+org.apache.ignite.internal.processors.cache.distributed.near.GridNearOptimisticTxPrepareFutureAdapter$2
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearPessimisticTxPrepareFuture$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearPessimisticTxPrepareFuture$2
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearSingleGetRequest
@@ -917,13 +925,13 @@ org.apache.ignite.internal.processors.cache.distributed.near.GridNearTransaction
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTransactionalCache$2
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$2
+org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$3
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishFuture$FinishMiniFuture$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishResponse
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$10
-org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$11
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$12
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$13
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$14
@@ -935,15 +943,19 @@ org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$19
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$2
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$20
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$21
+org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$21$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$22
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$23
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$24
+org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$25
+org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$26
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$3
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$4
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$5
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$6
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$7
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$8
+org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$9
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal$FinishClosure
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareFutureAdapter$1
 org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest
@@ -961,12 +973,12 @@ org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$5
 org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$8
 org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache$9
 org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter$RowData
-org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$12
-org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$6
-org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$9
-org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$CheckpointEntryType
+org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$2
+org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$4
+org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager$7
 org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager$WALHistoricalIterator
 org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager$1
+org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntryType
 org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory
 org.apache.ignite.internal.processors.cache.persistence.file.FileDownloader$1
 org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory
@@ -985,10 +997,12 @@ org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO$Entry
 org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator
 org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator$StartSeekingFilter
 org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer
+org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$7
 org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$FileArchiver$1
 org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$FileArchiver$2
 org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$FileCompressor$1
 org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager$RecordsIterator
+org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager$7
 org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager$FileArchiver$1
 org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager$FileArchiver$2
 org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager$FileCompressor$1
@@ -1024,6 +1038,7 @@ org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$9
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$CacheSqlIndexMetadata
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$CacheSqlMetadata
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$CachedResult$QueueIterator
+org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$InternalScanFilter
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$MetadataJob
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$MetadataJob$1
 org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager$MetadataJob$2
@@ -1081,6 +1096,8 @@ org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore$Stat
 org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore$StoreOperation
 org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore$ValueStatus
 org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx$FinalizationStatus
+org.apache.ignite.internal.processors.cache.transactions.IgniteTransactionsImpl$1
+org.apache.ignite.internal.processors.cache.transactions.IgniteTransactionsImpl$2
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxHandler$1
@@ -1123,8 +1140,12 @@ org.apache.ignite.internal.processors.cache.transactions.IgniteTxMap
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxMap$1
 org.apache.ignite.internal.processors.cache.transactions.IgniteTxMap$1$1
 org.apache.ignite.internal.processors.cache.transactions.TransactionMetricsAdapter
+org.apache.ignite.internal.processors.cache.transactions.TransactionMetricsAdapter$1
+org.apache.ignite.internal.processors.cache.transactions.TransactionMetricsAdapter$2
+org.apache.ignite.internal.processors.cache.transactions.TransactionMetricsAdapter$3
 org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl
 org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl$1
+org.apache.ignite.internal.processors.cache.transactions.TransactionProxyRollbackOnlyImpl
 org.apache.ignite.internal.processors.cache.transactions.TxDeadlockDetection$UniqueDeque
 org.apache.ignite.internal.processors.cache.transactions.TxEntryValueHolder
 org.apache.ignite.internal.processors.cache.transactions.TxLock
@@ -1132,23 +1153,25 @@ org.apache.ignite.internal.processors.cache.transactions.TxLockList
 org.apache.ignite.internal.processors.cache.transactions.TxLocksRequest
 org.apache.ignite.internal.processors.cache.transactions.TxLocksResponse
 org.apache.ignite.internal.processors.cache.verify.CacheInfo
-org.apache.ignite.internal.processors.cache.verify.CacheInfo$1
-org.apache.ignite.internal.processors.cache.verify.CacheInfo$2
 org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysTask
 org.apache.ignite.internal.processors.cache.verify.CollectConflictPartitionKeysTask$CollectPartitionEntryHashesJob
 org.apache.ignite.internal.processors.cache.verify.ContentionClosure
 org.apache.ignite.internal.processors.cache.verify.ContentionInfo
-org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2
 org.apache.ignite.internal.processors.cache.verify.IdleVerifyDumpResult
+org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2
 org.apache.ignite.internal.processors.cache.verify.PartitionEntryHashRecord
 org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord
+org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2
 org.apache.ignite.internal.processors.cache.verify.PartitionKey
+org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2
 org.apache.ignite.internal.processors.cache.verify.RetrieveConflictPartitionValuesTask
 org.apache.ignite.internal.processors.cache.verify.RetrieveConflictPartitionValuesTask$RetrieveConflictValuesJob
+org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsDumpTask
 org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask
 org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask$VerifyBackupPartitionsJob
+org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2
+org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2$VerifyBackupPartitionsJobV2
 org.apache.ignite.internal.processors.cache.verify.ViewCacheClosure
-org.apache.ignite.internal.processors.cache.verify.ViewCacheClosure$1
 org.apache.ignite.internal.processors.cache.version.GridCacheRawVersionedEntry
 org.apache.ignite.internal.processors.cache.version.GridCacheVersion
 org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext$State
@@ -1888,6 +1911,8 @@ org.apache.ignite.internal.visor.baseline.VisorBaselineTask
 org.apache.ignite.internal.visor.baseline.VisorBaselineTask$VisorBaselineJob
 org.apache.ignite.internal.visor.baseline.VisorBaselineTaskArg
 org.apache.ignite.internal.visor.baseline.VisorBaselineTaskResult
+org.apache.ignite.internal.visor.baseline.VisorBaselineViewTask
+org.apache.ignite.internal.visor.baseline.VisorBaselineViewTask$VisorBaselineViewJob
 org.apache.ignite.internal.visor.binary.VisorBinaryMetadata
 org.apache.ignite.internal.visor.binary.VisorBinaryMetadataCollectorTask
 org.apache.ignite.internal.visor.binary.VisorBinaryMetadataCollectorTask$VisorBinaryCollectMetadataJob
@@ -1930,6 +1955,9 @@ org.apache.ignite.internal.visor.cache.VisorCacheModifyTask
 org.apache.ignite.internal.visor.cache.VisorCacheModifyTask$VisorCacheModifyJob
 org.apache.ignite.internal.visor.cache.VisorCacheModifyTaskArg
 org.apache.ignite.internal.visor.cache.VisorCacheModifyTaskResult
+org.apache.ignite.internal.visor.cache.VisorCacheNamesCollectorTask
+org.apache.ignite.internal.visor.cache.VisorCacheNamesCollectorTask$VisorCacheNamesCollectorJob
+org.apache.ignite.internal.visor.cache.VisorCacheNamesCollectorTaskResult
 org.apache.ignite.internal.visor.cache.VisorCacheNearConfiguration
 org.apache.ignite.internal.visor.cache.VisorCacheNodesTask
 org.apache.ignite.internal.visor.cache.VisorCacheNodesTask$VisorCacheNodesJob
@@ -2024,13 +2052,13 @@ org.apache.ignite.internal.visor.misc.VisorAckTaskArg
 org.apache.ignite.internal.visor.misc.VisorChangeGridActiveStateTask
 org.apache.ignite.internal.visor.misc.VisorChangeGridActiveStateTask$VisorChangeGridActiveStateJob
 org.apache.ignite.internal.visor.misc.VisorChangeGridActiveStateTaskArg
+org.apache.ignite.internal.visor.misc.VisorClusterNode
 org.apache.ignite.internal.visor.misc.VisorLatestVersionTask
 org.apache.ignite.internal.visor.misc.VisorLatestVersionTask$VisorLatestVersionJob
 org.apache.ignite.internal.visor.misc.VisorNopTask
 org.apache.ignite.internal.visor.misc.VisorNopTask$VisorNopJob
 org.apache.ignite.internal.visor.misc.VisorResolveHostNameTask
 org.apache.ignite.internal.visor.misc.VisorResolveHostNameTask$VisorResolveHostNameJob
-org.apache.ignite.internal.visor.misc.VisorClusterNode
 org.apache.ignite.internal.visor.misc.VisorWalTask
 org.apache.ignite.internal.visor.misc.VisorWalTask$VisorWalJob
 org.apache.ignite.internal.visor.misc.VisorWalTaskArg
@@ -2133,18 +2161,30 @@ org.apache.ignite.internal.visor.service.VisorCancelServiceTaskArg
 org.apache.ignite.internal.visor.service.VisorServiceDescriptor
 org.apache.ignite.internal.visor.service.VisorServiceTask
 org.apache.ignite.internal.visor.service.VisorServiceTask$VisorServiceJob
-org.apache.ignite.internal.visor.util.VisorClusterGroupEmptyException
-org.apache.ignite.internal.visor.util.VisorEventMapper
-org.apache.ignite.internal.visor.util.VisorExceptionWrapper
-org.apache.ignite.internal.visor.util.VisorTaskUtils$4
 org.apache.ignite.internal.visor.tx.VisorTxInfo
+org.apache.ignite.internal.visor.tx.VisorTxInfo$1
+org.apache.ignite.internal.visor.tx.VisorTxInfo$2
 org.apache.ignite.internal.visor.tx.VisorTxOperation
 org.apache.ignite.internal.visor.tx.VisorTxProjection
 org.apache.ignite.internal.visor.tx.VisorTxSortOrder
 org.apache.ignite.internal.visor.tx.VisorTxTask
+org.apache.ignite.internal.visor.tx.VisorTxTask$1
+org.apache.ignite.internal.visor.tx.VisorTxTask$2
+org.apache.ignite.internal.visor.tx.VisorTxTask$3
+org.apache.ignite.internal.visor.tx.VisorTxTask$4
+org.apache.ignite.internal.visor.tx.VisorTxTask$5
+org.apache.ignite.internal.visor.tx.VisorTxTask$LocalKillClosure
+org.apache.ignite.internal.visor.tx.VisorTxTask$NearKillClosure
+org.apache.ignite.internal.visor.tx.VisorTxTask$RemoteKillClosure
+org.apache.ignite.internal.visor.tx.VisorTxTask$TxKillClosure
+org.apache.ignite.internal.visor.tx.VisorTxTask$VisorTxJob
 org.apache.ignite.internal.visor.tx.VisorTxTaskArg
 org.apache.ignite.internal.visor.tx.VisorTxTaskResult
-org.apache.ignite.internal.visor.verify.VisorViewCacheCmd
+org.apache.ignite.internal.visor.util.VisorClusterGroupEmptyException
+org.apache.ignite.internal.visor.util.VisorEventMapper
+org.apache.ignite.internal.visor.util.VisorExceptionWrapper
+org.apache.ignite.internal.visor.util.VisorTaskUtils$4
+org.apache.ignite.internal.visor.verify.IndexValidationIssue
 org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult
 org.apache.ignite.internal.visor.verify.VisorContentionJobResult
 org.apache.ignite.internal.visor.verify.VisorContentionTask
@@ -2157,15 +2197,22 @@ org.apache.ignite.internal.visor.verify.VisorIdleAnalyzeTask$VisorIdleVerifyJob$
 org.apache.ignite.internal.visor.verify.VisorIdleAnalyzeTask$VisorIdleVerifyJob$2
 org.apache.ignite.internal.visor.verify.VisorIdleAnalyzeTaskArg
 org.apache.ignite.internal.visor.verify.VisorIdleAnalyzeTaskResult
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTask
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyJob
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyJob$1
 org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask
 org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask$VisorIdleVerifyJob
 org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask$VisorIdleVerifyJob$1
 org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg
-org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg
 org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskV2
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskV2$VisorIdleVerifyJobV2
+org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskV2$VisorIdleVerifyJobV2$1
 org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult
 org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskArg
 org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskResult
+org.apache.ignite.internal.visor.verify.VisorViewCacheCmd
 org.apache.ignite.internal.visor.verify.VisorViewCacheTask
 org.apache.ignite.internal.visor.verify.VisorViewCacheTask$VisorViewCacheJob
 org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg
@@ -2190,6 +2237,7 @@ org.apache.ignite.lang.IgniteUuid
 org.apache.ignite.lifecycle.LifecycleEventType
 org.apache.ignite.marshaller.jdk.JdkMarshallerDummySerializable
 org.apache.ignite.messaging.MessagingListenActor
+org.apache.ignite.mxbean.TransactionMetricsMxBean
 org.apache.ignite.platform.dotnet.PlatformDotNetAffinityFunction
 org.apache.ignite.platform.dotnet.PlatformDotNetCacheStoreFactory
 org.apache.ignite.platform.dotnet.PlatformDotNetCacheStoreFactoryNative
@@ -2224,8 +2272,8 @@ org.apache.ignite.spi.checkpoint.sharedfs.SharedFsCheckpointData
 org.apache.ignite.spi.collision.jobstealing.JobStealingRequest
 org.apache.ignite.spi.collision.priorityqueue.PriorityQueueCollisionSpi$PriorityGridCollisionJobContextComparator
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$1
-org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$10
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$11
+org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$12
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$1
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$2
 org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi$2$ConnectClosure

From 60fdc4fbd30eeb5cbc8ce4ffb0e4c8700712395a Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 22 Aug 2018 11:52:48 +0700
Subject: [PATCH 317/543] GG-14081 Fake commit to fix renaming: part 1.

---
 .../e2e/testcafe/page-models/{PageSignIn.js => pageSignIn1.js}  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename modules/web-console/e2e/testcafe/page-models/{PageSignIn.js => pageSignIn1.js} (97%)

diff --git a/modules/web-console/e2e/testcafe/page-models/PageSignIn.js b/modules/web-console/e2e/testcafe/page-models/pageSignIn1.js
similarity index 97%
rename from modules/web-console/e2e/testcafe/page-models/PageSignIn.js
rename to modules/web-console/e2e/testcafe/page-models/pageSignIn1.js
index 57031b6f3a0d0..039e93dc1487d 100644
--- a/modules/web-console/e2e/testcafe/page-models/PageSignIn.js
+++ b/modules/web-console/e2e/testcafe/page-models/pageSignIn1.js
@@ -18,7 +18,7 @@
 import {Selector, t} from 'testcafe';
 import {CustomFormField} from '../components/FormField';
 
-export const pageSignin = {
+export const pageSignIn1 = {
     email: new CustomFormField({model: '$ctrl.data.email'}),
     password: new CustomFormField({model: '$ctrl.data.password'}),
     signinButton: Selector('button').withText('Sign In'),

From 38fa93c8f085392e75f22a68648d368be1039c31 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 22 Aug 2018 13:21:23 +0700
Subject: [PATCH 318/543] GG-14081 Fake commit to fix renaming: part 2.

---
 .../e2e/testcafe/page-models/{pageSignIn1.js => pageSignin.js}  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename modules/web-console/e2e/testcafe/page-models/{pageSignIn1.js => pageSignin.js} (97%)

diff --git a/modules/web-console/e2e/testcafe/page-models/pageSignIn1.js b/modules/web-console/e2e/testcafe/page-models/pageSignin.js
similarity index 97%
rename from modules/web-console/e2e/testcafe/page-models/pageSignIn1.js
rename to modules/web-console/e2e/testcafe/page-models/pageSignin.js
index 039e93dc1487d..57031b6f3a0d0 100644
--- a/modules/web-console/e2e/testcafe/page-models/pageSignIn1.js
+++ b/modules/web-console/e2e/testcafe/page-models/pageSignin.js
@@ -18,7 +18,7 @@
 import {Selector, t} from 'testcafe';
 import {CustomFormField} from '../components/FormField';
 
-export const pageSignIn1 = {
+export const pageSignin = {
     email: new CustomFormField({model: '$ctrl.data.email'}),
     password: new CustomFormField({model: '$ctrl.data.password'}),
     signinButton: Selector('button').withText('Sign In'),

From 373e481c6d98477eb4da664fdb920567c4195e41 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Thu, 14 Jun 2018 16:20:19 +0700
Subject: [PATCH 319/543] GG-14122 Backport IGNITE-8722 REST: Fixed
 BinaryObject serialization to JSON.

(cherry picked from commit e539a0cbde417e718cd4f6840dca8429907e09c5)
---
 .../rest/protocols/http/jetty/GridJettyObjectMapper.java      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyObjectMapper.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyObjectMapper.java
index d8b79cfd4cfde..92ee5c0df02b7 100644
--- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyObjectMapper.java
+++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyObjectMapper.java
@@ -246,8 +246,8 @@ private void writeException(Throwable e, JsonGenerator gen) throws IOException {
                             if (ref.hasCircularReferences())
                                 throw ser.mappingException("Failed convert to JSON object for circular references");
                         }
-                        else
-                            gen.writeObjectField(name, val);
+
+                        gen.writeObjectField(name, val);
                     }
 
                     gen.writeEndObject();

From 53b6a3965767feee882b7a81ab7fddb6403550b7 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Thu, 23 Aug 2018 10:48:41 +0700
Subject: [PATCH 320/543] IGNITE-9350 Web Console: Added checks for invalid web
 socket messages. Fixes #4597.

(cherry picked from commit b6f67f5f33febb0821b9cb5bde963de212c0e504)
---
 .../web-console/backend/app/agentsHandler.js  | 19 +++++++------------
 .../backend/app/browsersHandler.js            | 10 ++++++++--
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/modules/web-console/backend/app/agentsHandler.js b/modules/web-console/backend/app/agentsHandler.js
index 4af81b6699fca..6d8c621474fb4 100644
--- a/modules/web-console/backend/app/agentsHandler.js
+++ b/modules/web-console/backend/app/agentsHandler.js
@@ -244,6 +244,12 @@ module.exports.factory = function(settings, mongo, AgentSocket) {
             });
 
             sock.on('cluster:topology', (top) => {
+                if (_.isNil(top)) {
+                    console.log('Invalid format of message: "cluster:topology"');
+
+                    return;
+                }
+
                 const cluster = this.getOrCreateCluster(top);
 
                 _.forEach(this.topLsnrs, (lsnr) => lsnr(agentSocket, cluster, top));
@@ -283,19 +289,8 @@ module.exports.factory = function(settings, mongo, AgentSocket) {
             _.forEach(tokens, (token) => {
                 this._agentSockets.add(token, agentSocket);
 
-                // TODO start demo if needed.
-                // const browserSockets = _.filter(this._browserSockets[token], 'request._query.IgniteDemoMode');
-                //
-                // // First agent join after user start demo.
-                // if (_.size(browserSockets))
-                //     agentSocket.runDemoCluster(token, browserSockets);
-
                 this._browsersHnd.agentStats(token);
             });
-
-            // ioSocket.on('cluster:topology', (top) => {
-            //
-            // });
         }
 
         /**
@@ -315,7 +310,7 @@ module.exports.factory = function(settings, mongo, AgentSocket) {
                     this.io = socketio(srv, {path: '/agents'});
 
                     this.io.on('connection', (sock) => {
-                        sock.on('agent:auth', ({ver, bt, tokens, disableDemo}, cb) => {
+                        sock.on('agent:auth', ({ver, bt, tokens, disableDemo} = {}, cb) => {
                             if (_.isEmpty(tokens))
                                 return cb('Tokens not set. Please reload agent archive or check settings');
 
diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js
index c3c2ea42dfccc..5aade60fac3e2 100644
--- a/modules/web-console/backend/app/browsersHandler.js
+++ b/modules/web-console/backend/app/browsersHandler.js
@@ -204,7 +204,10 @@ module.exports = {
 
             nodeListeners(sock) {
                 // Return command result from grid to browser.
-                sock.on('node:rest', ({clusterId, params, credentials}, cb) => {
+                sock.on('node:rest', ({clusterId, params, credentials} = {}, cb) => {
+                    if (_.isNil(clusterId) || _.isNil(params))
+                        return cb('Invalid format of message: "node:rest"');
+
                     const demo = sock.request._query.IgniteDemoMode === 'true';
                     const token = sock.request.user.token;
 
@@ -233,7 +236,10 @@ module.exports = {
                 this.registerVisorTask('toggleClusterState', internalVisor('misc.VisorChangeGridActiveStateTask'), internalVisor('misc.VisorChangeGridActiveStateTaskArg'));
 
                 // Return command result from grid to browser.
-                sock.on('node:visor', ({clusterId, params = {}, credentials} = {}, cb) => {
+                sock.on('node:visor', ({clusterId, params, credentials} = {}, cb) => {
+                    if (_.isNil(clusterId) || _.isNil(params))
+                        return cb('Invalid format of message: "node:visor"');
+
                     const demo = sock.request._query.IgniteDemoMode === 'true';
                     const token = sock.request.user.token;
 

From 69b6663277fdde33452c1f2428ab5a045347b016 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Thu, 23 Aug 2018 12:37:28 +0700
Subject: [PATCH 321/543] GG-13195 IGNITE-9337 Refactored from single group to
 collection of groups.

(cherry picked from commit 68374ad52eede916129b82ef769e680ad2bd1a70)
---
 .../visor/node/VisorNodeDataCollectorJob.java |  9 ++++---
 .../node/VisorNodeDataCollectorTaskArg.java   | 26 ++++++++++---------
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
index 578300e813936..5fab8d16d5e94 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
@@ -19,6 +19,7 @@
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 
 import org.apache.ignite.DataRegionMetrics;
@@ -183,7 +184,9 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto
 
             GridCacheProcessor cacheProc = ignite.context().cache();
 
-            String cacheGrpToCollect = arg.getCacheGroup();
+            Set<String> cacheGrps = arg.getCacheGroups();
+
+            boolean all = F.isEmpty(cacheGrps);
 
             int partitions = 0;
             double total = 0;
@@ -219,9 +222,7 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto
                         total += partTotal;
                         ready += partReady;
 
-                        String cacheGrp = ca.configuration().getGroupName();
-
-                        if (F.eq(cacheGrpToCollect, cacheGrp))
+                        if (all || cacheGrps.contains(ca.configuration().getGroupName()))
                             resCaches.add(new VisorCache(ignite, ca, arg.isCollectCacheMetrics()));
                     }
                     catch(IllegalStateException | IllegalArgumentException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskArg.java
index e6f80273fcb74..b707991c5cc8d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskArg.java
@@ -20,6 +20,8 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.util.Set;
+
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
@@ -46,8 +48,8 @@ public class VisorNodeDataCollectorTaskArg extends VisorDataTransferObject {
     /** If {@code false} then cache metrics will not be collected. */
     private boolean collectCacheMetrics;
 
-    /** Optional cache group, if provided, then caches only from that group will be collected. */
-    private String cacheGrp;
+    /** Optional Set of cache groups, if provided, then caches only from that groups will be collected. */
+    private Set<String> cacheGrps;
 
     /**
      * Default constructor.
@@ -64,7 +66,7 @@ public VisorNodeDataCollectorTaskArg() {
      * @param evtThrottleCntrKey Event throttle counter key, unique for Visor instance.
      * @param sysCaches If {@code true} then collect information about system caches.
      * @param collectCacheMetrics If {@code false} then cache metrics will not be collected.
-     * @param cacheGrp Optional cache group, if provided, then caches only from that group will be collected.
+     * @param cacheGrps Optional Set of cache groups, if provided, then caches only from that groups will be collected.
      */
     public VisorNodeDataCollectorTaskArg(
         boolean taskMonitoringEnabled,
@@ -72,14 +74,14 @@ public VisorNodeDataCollectorTaskArg(
         String evtThrottleCntrKey,
         boolean sysCaches,
         boolean collectCacheMetrics,
-        String cacheGrp
+        Set<String> cacheGrps
     ) {
         this.taskMonitoringEnabled = taskMonitoringEnabled;
         this.evtOrderKey = evtOrderKey;
         this.evtThrottleCntrKey = evtThrottleCntrKey;
         this.sysCaches = sysCaches;
         this.collectCacheMetrics = collectCacheMetrics;
-        this.cacheGrp = cacheGrp;
+        this.cacheGrps = cacheGrps;
     }
     /**
      * Create task arguments with given parameters.
@@ -190,15 +192,15 @@ public void setCollectCacheMetrics(boolean collectCacheMetrics) {
     /**
      * @return Optional cache group, if provided, then caches only from that group will be collected.
      */
-    public String getCacheGroup() {
-        return cacheGrp;
+    public Set<String> getCacheGroups() {
+        return cacheGrps;
     }
 
     /**
-     * @param cacheGrp Optional cache group, if provided, then caches only from that group will be collected.
+     * @param cacheGrps Optional Set of cache groups, if provided, then caches only from that groups will be collected.
      */
-    public void setCollectCacheMetrics(String cacheGrp) {
-        this.cacheGrp = cacheGrp;
+    public void setCacheGroups(Set<String> cacheGrps) {
+        this.cacheGrps = cacheGrps;
     }
 
     /** {@inheritDoc} */
@@ -213,7 +215,7 @@ public void setCollectCacheMetrics(String cacheGrp) {
         U.writeString(out, evtThrottleCntrKey);
         out.writeBoolean(sysCaches);
         out.writeBoolean(collectCacheMetrics);
-        U.writeString(out, cacheGrp);
+        U.writeCollection(out, cacheGrps);
     }
 
     /** {@inheritDoc} */
@@ -225,7 +227,7 @@ public void setCollectCacheMetrics(String cacheGrp) {
 
         collectCacheMetrics = protoVer < V2 || in.readBoolean();
 
-        cacheGrp = protoVer < V3 ? null : U.readString(in);
+        cacheGrps = protoVer < V3 ? null : U.readSet(in);
     }
 
     /** {@inheritDoc} */

From 011d8aa0e436ad964f92b8667ed734f81f7210de Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Thu, 23 Aug 2018 18:05:44 +0300
Subject: [PATCH 322/543] Revert "IGNITE-7165 Re-balancing is cancelled if
 client node joins"

This reverts commit ef331f3ba58da49746f34546f1972cf65c083b3d.
---
 .../GridCachePartitionExchangeManager.java    |  68 +++-----
 .../processors/cache/GridCachePreloader.java  |  21 +--
 .../cache/GridCachePreloaderAdapter.java      |   6 -
 .../preloader/GridDhtPartitionDemander.java   |  55 +++----
 .../preloader/GridDhtPartitionSupplier.java   |  26 +--
 .../dht/preloader/GridDhtPreloader.java       |  60 +------
 .../GridDhtPreloaderAssignments.java          |   6 +-
 .../ClusterBaselineNodesMetricsSelfTest.java  |   1 +
 .../cache/CacheValidatorMetricsTest.java      |   4 +-
 .../dht/GridCacheDhtPreloadSelfTest.java      |  68 +++++++-
 .../atomic/IgniteCacheAtomicProtocolTest.java |   1 -
 .../GridCacheRebalancingAsyncSelfTest.java    |   7 +-
 .../GridCacheRebalancingCancelTest.java       | 106 -------------
 ...CacheRebalancingPartitionCountersTest.java |   3 +-
 .../GridCacheRebalancingSyncSelfTest.java     | 149 +++++++++++-------
 ...entAffinityAssignmentWithBaselineTest.java |   4 +-
 ...owHistoricalRebalanceSmallHistoryTest.java |   5 +-
 ...lushMultiNodeFailoverAbstractSelfTest.java |   2 +-
 .../GridMarshallerMappingConsistencyTest.java |   3 +-
 .../junits/common/GridCommonAbstractTest.java | 108 +++----------
 .../testsuites/IgniteCacheTestSuite3.java     |   9 +-
 21 files changed, 273 insertions(+), 439 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index e63c484826594..f5ccedfc961d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
@@ -87,7 +88,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionsToReloadMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.RebalanceReassignExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
@@ -180,14 +180,6 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     private final ConcurrentMap<AffinityTopologyVersion, AffinityReadyFuture> readyFuts =
         new ConcurrentSkipListMap<>();
 
-    /**
-     * Latest started rebalance topology version but possibly not finished yet. Value {@code NONE}
-     * means that previous rebalance is undefined and the new one should be initiated.
-     *
-     * Should not be used to determine latest rebalanced topology.
-     */
-    private volatile AffinityTopologyVersion rebTopVer = AffinityTopologyVersion.NONE;
-
     /** */
     private GridFutureAdapter<?> reconnectExchangeFut;
 
@@ -851,13 +843,6 @@ public AffinityTopologyVersion readyAffinityVersion() {
         return exchFuts.readyTopVer();
     }
 
-    /**
-     * @return Latest rebalance topology version or {@code NONE} if there is no info.
-     */
-    public AffinityTopologyVersion rebalanceTopologyVersion() {
-        return rebTopVer;
-    }
-
     /**
      * @return Last initialized topology future.
      */
@@ -2601,9 +2586,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (grp.isLocal())
                                     continue;
 
-                                if (grp.preloader().rebalanceRequired(rebTopVer, exchFut))
-                                    rebTopVer = AffinityTopologyVersion.NONE;
-
                                 changed |= grp.topology().afterExchange(exchFut);
                             }
 
@@ -2611,11 +2593,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 refreshPartitions();
                         }
 
-                        // Schedule rebalance if force rebalance or force reassign occurs.
-                        if (exchFut == null)
-                            rebTopVer = AffinityTopologyVersion.NONE;
-
-                        if (!cctx.kernalContext().clientNode() && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
+                        if (!cctx.kernalContext().clientNode()) {
                             assignsMap = new HashMap<>();
 
                             IgniteCacheSnapshotManager snp = cctx.snapshot();
@@ -2632,9 +2610,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                     assigns = grp.preloader().generateAssignments(exchId, exchFut);
 
                                 assignsMap.put(grp.groupId(), assigns);
-
-                                if (resVer == null)
-                                    resVer = grp.topology().readyTopologyVersion();
                             }
                         }
                     }
@@ -2643,7 +2618,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         busy = false;
                     }
 
-                    if (assignsMap != null && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
+                    if (assignsMap != null) {
                         int size = assignsMap.size();
 
                         NavigableMap<Integer, List<Integer>> orderMap = new TreeMap<>();
@@ -2681,6 +2656,11 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (assigns != null)
                                     assignsCancelled |= assigns.cancelled();
 
+                                // Cancels previous rebalance future (in case it's not done yet).
+                                // Sends previous rebalance stopped event (if necessary).
+                                // Creates new rebalance future.
+                                // Sends current rebalance started event (if necessary).
+                                // Finishes cache sync future (on empty assignments).
                                 Runnable cur = grp.preloader().addAssignments(assigns,
                                     forcePreload,
                                     cnt,
@@ -2698,7 +2678,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         if (forcedRebFut != null)
                             forcedRebFut.markInitialized();
 
-                        if (assignsCancelled || hasPendingExchange()) {
+                        if (assignsCancelled) { // Pending exchange.
                             U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
                                 "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
@@ -2706,31 +2686,25 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         else if (r != null) {
                             Collections.reverse(rebList);
 
-                            U.log(log, "Rebalancing scheduled [order=" + rebList +
-                                ", top=" + resVer + ", force=" + (exchFut == null) +
-                                ", evt=" + exchId.discoveryEventName() +
-                                ", node=" + exchId.nodeId() + ']');
+                            U.log(log, "Rebalancing scheduled [order=" + rebList + "]");
 
-                            rebTopVer = resVer;
+                            if (!hasPendingExchange()) {
+                                U.log(log, "Rebalancing started " +
+                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
+                                    ", node=" + exchId.nodeId() + ']');
 
-                            // Start rebalancing cache groups chain. Each group will be rebalanced
-                            // sequentially one by one e.g.:
-                            // ignite-sys-cache -> cacheGroupR1 -> cacheGroupP2 -> cacheGroupR3
-                            r.run();
+                                r.run(); // Starts rebalancing routine.
+                            }
+                            else
+                                U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
+                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
+                                    ", node=" + exchId.nodeId() + ']');
                         }
                         else
                             U.log(log, "Skipping rebalancing (nothing scheduled) " +
-                                "[top=" + resVer + ", force=" + (exchFut == null) +
-                                ", evt=" + exchId.discoveryEventName() +
+                                "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
                     }
-                    else
-                        U.log(log, "Skipping rebalancing (no affinity changes) " +
-                            "[top=" + resVer +
-                            ", rebTopVer=" + rebTopVer +
-                            ", evt=" + exchId.discoveryEventName() +
-                            ", evtNode=" + exchId.nodeId() +
-                            ", client=" + cctx.kernalContext().clientNode() + ']');
                 }
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index d629e94db7e84..5fa7a821c69ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -24,7 +24,6 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.ForceRebalanceExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
@@ -63,17 +62,10 @@ public interface GridCachePreloader {
      */
     public void onInitialExchangeComplete(@Nullable Throwable err);
 
-    /**
-     * @param rebTopVer Previous rebalance topology version or {@code NONE} if there is no info.
-     * @param exchFut Completed exchange future.
-     * @return {@code True} if rebalance should be started (previous will be interrupted).
-     */
-    public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer, GridDhtPartitionsExchangeFuture exchFut);
-
     /**
      * @param exchId Exchange ID.
-     * @param exchFut Completed exchange future. Can be {@code null} if forced or reassigned generation occurs.
-     * @return Partition assignments which will be requested from supplier nodes.
+     * @param exchFut Exchange future.
+     * @return Assignments or {@code null} if detected that there are pending exchanges.
      */
     @Nullable public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      @Nullable GridDhtPartitionsExchangeFuture exchFut);
@@ -82,10 +74,10 @@ public interface GridCachePreloader {
      * Adds assignments to preloader.
      *
      * @param assignments Assignments to add.
-     * @param forcePreload {@code True} if preload requested by {@link ForceRebalanceExchangeTask}.
-     * @param rebalanceId Rebalance id created by exchange thread.
-     * @param next Runnable responsible for cache rebalancing chain.
-     * @param forcedRebFut External future for forced rebalance.
+     * @param forcePreload Force preload flag.
+     * @param rebalanceId Rebalance id.
+     * @param next Runnable responsible for cache rebalancing start.
+     * @param forcedRebFut Rebalance future.
      * @return Rebalancing runnable.
      */
     public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
@@ -122,6 +114,7 @@ public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
      * Future result is {@code false} in case rebalancing cancelled or finished with missed partitions and will be
      * restarted at current or pending topology.
      *
+     * Note that topology change creates new futures and finishes previous.
      */
     public IgniteInternalFuture<Boolean> rebalanceFuture();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index c5e4a817d0418..af916791daa3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -151,12 +151,6 @@ public GridCachePreloaderAdapter(CacheGroupContext grp) {
         // No-op.
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
-        GridDhtPartitionsExchangeFuture exchFut) {
-        return true;
-    }
-
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      GridDhtPartitionsExchangeFuture exchFut) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 54d3c93ed2f61..1eeebaef3a79e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -235,12 +235,12 @@ else if (log.isDebugEnabled())
 
     /**
      * @param fut Future.
-     * @return {@code True} if rebalance topology version changed by exchange thread or force
-     * reassing exchange occurs, see {@link RebalanceReassignExchangeTask} for details.
+     * @return {@code True} if topology changed.
      */
     private boolean topologyChanged(RebalanceFuture fut) {
-        return !ctx.exchange().rebalanceTopologyVersion().equals(fut.topVer) ||
-            fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
+        return
+            !grp.affinity().lastVersion().equals(fut.topologyVersion()) || // Topology already changed.
+                fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
     }
 
     /**
@@ -253,21 +253,14 @@ void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
     }
 
     /**
-     * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
-     */
-    Collection<UUID> remainingNodes() {
-        return rebalanceFut.remainingNodes();
-    }
-
-    /**
-     * This method initiates new rebalance process from given {@code assignments} by creating new rebalance
-     * future based on them. Cancels previous rebalance future and sends rebalance started event.
-     * In case of delayed rebalance method schedules the new one with configured delay based on {@code lastExchangeFut}.
+     * Initiates new rebalance process from given {@code assignments}.
+     * If previous rebalance is not finished method cancels it.
+     * In case of delayed rebalance method schedules new with configured delay.
      *
-     * @param assignments Assignments to process.
-     * @param force {@code True} if preload request by {@link ForceRebalanceExchangeTask}.
-     * @param rebalanceId Rebalance id generated from exchange thread.
-     * @param next Runnable responsible for cache rebalancing chain.
+     * @param assignments Assignments.
+     * @param force {@code True} if dummy reassign.
+     * @param rebalanceId Rebalance id.
+     * @param next Runnable responsible for cache rebalancing start.
      * @param forcedRebFut External future for forced rebalance.
      * @return Rebalancing runnable.
      */
@@ -447,7 +440,17 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
             if (fut.isDone())
                 return;
 
-            fut.remaining.forEach((key, value) -> value.set1(U.currentTimeMillis()));
+            // Must add all remaining node before send first request, for avoid race between add remaining node and
+            // processing response, see checkIsDone(boolean).
+            for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assignments.entrySet()) {
+                UUID nodeId = e.getKey().id();
+
+                IgniteDhtDemandedPartitionsMap parts = e.getValue().partitions();
+
+                assert parts != null : "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + nodeId + "]";
+
+                fut.remaining.put(nodeId, new T2<>(U.currentTimeMillis(), parts));
+            }
         }
 
         final CacheConfiguration cfg = grp.config();
@@ -976,13 +979,6 @@ public static class RebalanceFuture extends GridFutureAdapter<Boolean> {
             exchId = assignments.exchangeId();
             topVer = assignments.topologyVersion();
 
-            assignments.forEach((k, v) -> {
-                assert v.partitions() != null :
-                    "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + k.id() + "]";
-
-                remaining.put(k.id(), new T2<>(U.currentTimeMillis(), v.partitions()));
-            });
-
             this.grp = grp;
             this.log = log;
             this.rebalanceId = rebalanceId;
@@ -1221,13 +1217,6 @@ private void checkIsDone(boolean cancelled) {
             }
         }
 
-        /**
-         * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
-         */
-        private synchronized Collection<UUID> remainingNodes() {
-            return remaining.keySet();
-        }
-
         /**
          *
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index ea7f4c9b4aacd..4946d7e3f0eed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -17,14 +17,12 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
@@ -120,20 +118,19 @@ private static void clearContext(
     }
 
     /**
-     * Handle topology change and clear supply context map of outdated contexts.
+     * Handles new topology version and clears supply context map of outdated contexts.
+     *
+     * @param topVer Topology version.
      */
-    void onTopologyChanged() {
+    @SuppressWarnings("ConstantConditions")
+    void onTopologyChanged(AffinityTopologyVersion topVer) {
         synchronized (scMap) {
             Iterator<T3<UUID, Integer, AffinityTopologyVersion>> it = scMap.keySet().iterator();
 
-            Collection<UUID> aliveNodes = grp.shared().discovery().aliveServerNodes().stream()
-                .map(ClusterNode::id)
-                .collect(Collectors.toList());
-
             while (it.hasNext()) {
                 T3<UUID, Integer, AffinityTopologyVersion> t = it.next();
 
-                if (!aliveNodes.contains(t.get1())) { // Clear all obsolete contexts.
+                if (topVer.compareTo(t.get3()) > 0) { // Clear all obsolete contexts.
                     clearContext(scMap.get(t), log);
 
                     it.remove();
@@ -174,6 +171,17 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
         AffinityTopologyVersion curTop = grp.affinity().lastVersion();
         AffinityTopologyVersion demTop = d.topologyVersion();
 
+        if (curTop.compareTo(demTop) > 0) {
+            if (log.isDebugEnabled())
+                log.debug("Demand request outdated [grp=" + grp.cacheOrGroupName()
+                        + ", currentTopVer=" + curTop
+                        + ", demandTopVer=" + demTop
+                        + ", from=" + nodeId
+                        + ", topicId=" + topicId + "]");
+
+            return;
+        }
+
         T3<UUID, Integer, AffinityTopologyVersion> contextId = new T3<>(nodeId, topicId, demTop);
 
         if (d.rebalanceId() < 0) { // Demand node requested context cleanup.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 7cf55a35536fc..77f48661d588b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -24,7 +24,6 @@
 import java.util.UUID;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -40,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
@@ -160,67 +160,11 @@ private IgniteCheckedException stopError() {
 
     /** {@inheritDoc} */
     @Override public void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
-        supplier.onTopologyChanged();
+        supplier.onTopologyChanged(lastFut.initialVersion());
 
         demander.onTopologyChanged(lastFut);
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
-        GridDhtPartitionsExchangeFuture exchFut) {
-        if (ctx.kernalContext().clientNode() || rebTopVer.equals(AffinityTopologyVersion.NONE))
-            return false; // No-op.
-
-        if (exchFut.localJoinExchange())
-            return true; // Required, can have outdated updSeq partition counter if node reconnects.
-
-        if (!grp.affinity().cachedVersions().contains(rebTopVer)) {
-            assert rebTopVer.compareTo(grp.localStartVersion()) <= 0 :
-                "Empty hisroty allowed only for newly started cache group [rebTopVer=" + rebTopVer +
-                    ", localStartTopVer=" + grp.localStartVersion() + ']';
-
-            return true; // Required, since no history info available.
-        }
-
-        final IgniteInternalFuture<Boolean> rebFut = rebalanceFuture();
-
-        if (rebFut.isDone() && !rebFut.result())
-            return true; // Required, previous rebalance cancelled.
-
-        final AffinityTopologyVersion exchTopVer = exchFut.context().events().topologyVersion();
-
-        Collection<UUID> aliveNodes = ctx.discovery().aliveServerNodes().stream()
-            .map(ClusterNode::id)
-            .collect(Collectors.toList());
-
-        return assignmentsChanged(rebTopVer, exchTopVer) ||
-            !aliveNodes.containsAll(demander.remainingNodes()); // Some of nodes left before rabalance compelete.
-    }
-
-    /**
-     * @param oldTopVer Previous topology version.
-     * @param newTopVer New topology version to check result.
-     * @return {@code True} if affinity assignments changed between two versions for local node.
-     */
-    private boolean assignmentsChanged(AffinityTopologyVersion oldTopVer, AffinityTopologyVersion newTopVer) {
-        final AffinityAssignment aff = grp.affinity().readyAffinity(newTopVer);
-
-        // We should get affinity assignments based on previous rebalance to calculate difference.
-        // Whole history size described by IGNITE_AFFINITY_HISTORY_SIZE constant.
-        final AffinityAssignment prevAff = grp.affinity().cachedVersions().contains(oldTopVer) ?
-            grp.affinity().cachedAffinity(oldTopVer) : null;
-
-        if (prevAff == null)
-            return false;
-
-        boolean assignsChanged = false;
-
-        for (int p = 0; !assignsChanged && p < grp.affinity().partitions(); p++)
-            assignsChanged |= aff.get(p).contains(ctx.localNode()) != prevAff.get(p).contains(ctx.localNode());
-
-        return assignsChanged;
-    }
-
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId, GridDhtPartitionsExchangeFuture exchFut) {
         assert exchFut == null || exchFut.isDone();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
index 6e847bb0e1852..41dd076c3f745 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
@@ -20,7 +20,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
@@ -73,9 +73,9 @@ GridDhtPartitionExchangeId exchangeId() {
     }
 
     /**
-     * @return Topology version based on last {@link GridDhtPartitionTopologyImpl#readyTopVer}.
+     * @return Topology version.
      */
-    public AffinityTopologyVersion topologyVersion() {
+    AffinityTopologyVersion topologyVersion() {
         return topVer;
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
index 46b09ac3b1819..565317720d2a9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
@@ -149,6 +149,7 @@ public void testBaselineNodes() throws Exception {
     private void resetBlt() throws Exception {
         resetBaselineTopology();
 
+        waitForRebalancing();
         awaitPartitionMapExchange();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
index 0b441dd94c0e8..5c601a1bd3bbf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
@@ -103,14 +103,14 @@ public void testCacheValidatorMetrics() throws Exception {
 
         startGrid(2);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         assertCacheStatus(CACHE_NAME_1, true, true);
         assertCacheStatus(CACHE_NAME_2, true, true);
 
         stopGrid(1);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         // Invalid for writing due to invalid topology.
         assertCacheStatus(CACHE_NAME_1, true, false);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
index 23ba4b31b4e0a..83eff893b8894 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
@@ -22,23 +22,29 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.CacheRebalancingEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.P1;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -52,6 +58,9 @@
 import static org.apache.ignite.configuration.CacheConfiguration.DFLT_REBALANCE_BATCH_SIZE;
 import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS;
 import static org.apache.ignite.events.EventType.EVTS_CACHE_REBALANCE;
+import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
@@ -132,6 +141,15 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) {
         return cacheCfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+//        resetLog4j(Level.DEBUG, true,
+//            // Categories.
+//            GridDhtPreloader.class.getPackage().getName(),
+//            GridDhtPartitionTopologyImpl.class.getName(),
+//            GridDhtLocalPartition.class.getName());
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         backups = DFLT_BACKUPS;
@@ -209,6 +227,11 @@ public void testActivePartitionTransferAsyncRandomCoordinator() throws Exception
      */
     private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
+//        resetLog4j(Level.DEBUG, true,
+//            // Categories.
+//            GridDhtPreloader.class.getPackage().getName(),
+//            GridDhtPartitionTopologyImpl.class.getName(),
+//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -247,6 +270,8 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
             info(">>> Finished checking nodes [keyCnt=" + keyCnt + ", nodeCnt=" + nodeCnt + ", grids=" +
                 U.grids2names(ignites) + ']');
 
+            Collection<IgniteFuture<?>> futs = new LinkedList<>();
+
             Ignite last = F.last(ignites);
 
             for (Iterator<Ignite> it = ignites.iterator(); it.hasNext(); ) {
@@ -260,8 +285,21 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
 
                 checkActiveState(ignites);
 
+                final UUID nodeId = g.cluster().localNode().id();
+
                 it.remove();
 
+                futs.add(waitForLocalEvent(last.events(), new P1<Event>() {
+                    @Override public boolean apply(Event e) {
+                        CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
+
+                        ClusterNode node = evt.discoveryNode();
+
+                        return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
+                            (evt.discoveryEventType() == EVT_NODE_LEFT || evt.discoveryEventType() == EVT_NODE_FAILED);
+                    }
+                }, EVT_CACHE_REBALANCE_STOPPED));
+
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
@@ -274,6 +312,14 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
                 awaitPartitionMapExchange(); // Need wait, otherwise test logic is broken if EVT_NODE_FAILED exchanges are merged.
             }
 
+            info("Waiting for preload futures: " + F.view(futs, new IgnitePredicate<IgniteFuture<?>>() {
+                @Override public boolean apply(IgniteFuture<?> fut) {
+                    return !fut.isDone();
+                }
+            }));
+
+            X.waitAll(futs);
+
             info("Finished waiting for preload futures.");
 
             assert last != null;
@@ -453,6 +499,11 @@ private void stopGrids(Iterable<Ignite> grids) {
      */
     private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
+//        resetLog4j(Level.DEBUG, true,
+//            // Categories.
+//            GridDhtPreloader.class.getPackage().getName(),
+//            GridDhtPartitionTopologyImpl.class.getName(),
+//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -504,13 +555,28 @@ private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuf
 
                 it.remove();
 
+                Collection<IgniteFuture<?>> futs = new LinkedList<>();
+
+                for (Ignite gg : ignites)
+                    futs.add(waitForLocalEvent(gg.events(), new P1<Event>() {
+                        @Override public boolean apply(Event e) {
+                            CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
+
+                            ClusterNode node = evt.discoveryNode();
+
+                            return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
+                                (evt.discoveryEventType() == EVT_NODE_LEFT ||
+                                    evt.discoveryEventType() == EVT_NODE_FAILED);
+                        }
+                    }, EVT_CACHE_REBALANCE_STOPPED));
+
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
 
                 info(">>> Waiting for preload futures [leftNode=" + g.name() + ", remaining=" + U.grids2names(ignites) + ']');
 
-                awaitPartitionMapExchange();
+                X.waitAll(futs);
 
                 info("After grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java
index 14c85717ae857..0bfb4fbd06471 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/IgniteCacheAtomicProtocolTest.java
@@ -806,7 +806,6 @@ private void readerUpdateDhtFails(boolean updateNearEnabled,
 
         startServers(2);
 
-        // Waiting for minor topology changing because of late affinity assignment.
         awaitPartitionMapExchange();
 
         Ignite srv0 = ignite(0);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
index 0a8698a597cd8..4ebcd5df35271 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
@@ -17,11 +17,10 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
 
-import java.util.Collections;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
@@ -44,7 +43,7 @@ public class GridCacheRebalancingAsyncSelfTest extends GridCacheRebalancingSyncS
      * @throws Exception Exception.
      */
     public void testNodeFailedAtRebalancing() throws Exception {
-        IgniteEx ignite = startGrid(0);
+        Ignite ignite = startGrid(0);
 
         generateData(ignite, 0, 0);
 
@@ -61,7 +60,7 @@ public void testNodeFailedAtRebalancing() throws Exception {
 
         ((TestTcpDiscoverySpi)grid(1).configuration().getDiscoverySpi()).simulateNodeFailure();
 
-        awaitPartitionMapExchange(false, false, Collections.singletonList(ignite.localNode()));
+        waitForRebalancing(0, 3);
 
         checkSupplyContextMapIsEmpty();
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
deleted file mode 100644
index 3965290480a3d..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
-
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.TestRecordingCommunicationSpi;
-import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
-import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Test cases for checking cancellation rebalancing process if some events occurs.
- */
-public class GridCacheRebalancingCancelTest extends GridCommonAbstractTest {
-    /** */
-    private static final String DHT_PARTITIONED_CACHE = "cacheP";
-
-    /** */
-    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
-        IgniteConfiguration dfltCfg = super.getConfiguration(igniteInstanceName);
-
-        ((TcpDiscoverySpi)dfltCfg.getDiscoverySpi()).setIpFinder(ipFinder);
-
-        dfltCfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
-
-        return dfltCfg;
-    }
-
-    /**
-     * Test rebalance not cancelled when client node join to cluster.
-     *
-     * @throws Exception Exception.
-     */
-    public void testClientNodeJoinAtRebalancing() throws Exception {
-        final IgniteEx ignite0 = startGrid(0);
-
-        IgniteCache<Integer, Integer> cache = ignite0.createCache(
-            new CacheConfiguration<Integer, Integer>(DHT_PARTITIONED_CACHE)
-                .setCacheMode(CacheMode.PARTITIONED)
-                .setRebalanceMode(CacheRebalanceMode.ASYNC)
-                .setBackups(1)
-                .setRebalanceOrder(2)
-                .setAffinity(new RendezvousAffinityFunction(false)));
-
-        for (int i = 0; i < 2048; i++)
-            cache.put(i, i);
-
-        TestRecordingCommunicationSpi.spi(ignite0)
-            .blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
-                @Override public boolean apply(ClusterNode node, Message msg) {
-                    return (msg instanceof GridDhtPartitionSupplyMessage)
-                        && ((GridCacheGroupIdMessage)msg).groupId() == groupIdForCache(ignite0, DHT_PARTITIONED_CACHE);
-                }
-            });
-
-        final IgniteEx ignite1 = startGrid(1);
-
-        TestRecordingCommunicationSpi.spi(ignite0).waitForBlocked();
-
-        GridDhtPartitionDemander.RebalanceFuture fut = (GridDhtPartitionDemander.RebalanceFuture)ignite1.context().
-            cache().internalCache(DHT_PARTITIONED_CACHE).preloader().rebalanceFuture();
-
-        String igniteClntName = getTestIgniteInstanceName(2);
-
-        startGrid(igniteClntName, optimize(getConfiguration(igniteClntName).setClientMode(true)));
-
-        // Resend delayed rebalance messages.
-        TestRecordingCommunicationSpi.spi(ignite0).stopBlock(true);
-
-        awaitPartitionMapExchange();
-
-        // Previous rebalance future should not be cancelled.
-        assertTrue(fut.result());
-    }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
index cb414eda13bf5..1280e87f9f091 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
@@ -141,8 +141,7 @@ else if ("index.bin".equals(f.getName())) {
         assertTrue(primaryRemoved);
 
         ignite.cluster().active(true);
-
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         List<String> issues = new ArrayList<>();
         HashMap<Integer, Long> partMap = new HashMap<>();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
index a027a41cfb463..ed51cf392a992 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
@@ -21,9 +21,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cluster.ClusterNode;
@@ -41,13 +42,11 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
-import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -69,9 +68,6 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private static final int TEST_SIZE = 100_000;
 
-    /** */
-    private static final long TOPOLOGY_STILLNESS_TIME = 30_000L;
-
     /** partitioned cache name. */
     protected static final String CACHE_NAME_DHT_PARTITIONED = "cacheP";
 
@@ -93,11 +89,11 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private volatile boolean concurrentStartFinished3;
 
-    /**
-     * Time in milliseconds of last received {@link GridDhtPartitionsSingleMessage}
-     * or {@link GridDhtPartitionsFullMessage} using {@link CollectingCommunicationSpi}.
-     */
-    private static volatile long lastPartMsgTime;
+    /** */
+    private volatile boolean record;
+
+    /** */
+    private final ConcurrentHashMap<Class, AtomicInteger> map = new ConcurrentHashMap<>();
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -106,7 +102,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setIpFinder(ipFinder);
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setForceServerMode(true);
 
-        TcpCommunicationSpi commSpi = new CollectingCommunicationSpi();
+        TcpCommunicationSpi commSpi = new CountingCommunicationSpi();
 
         commSpi.setLocalPort(GridTestUtils.getNextCommPort(getClass()));
         commSpi.setTcpNoDelay(true);
@@ -236,35 +232,47 @@ public void testSimpleRebalancing() throws Exception {
 
         startGrid(1);
 
+        int waitMinorVer = ignite.configuration().isLateAffinityAssignment() ? 1 : 0;
+
+        waitForRebalancing(0, 2, waitMinorVer);
+        waitForRebalancing(1, 2, waitMinorVer);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         stopGrid(0);
 
+        waitForRebalancing(1, 3);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         startGrid(2);
 
+        waitForRebalancing(1, 4, waitMinorVer);
+        waitForRebalancing(2, 4, waitMinorVer);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         stopGrid(2);
 
+        waitForRebalancing(1, 5);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         long spend = (System.currentTimeMillis() - start) / 1000;
 
@@ -323,10 +331,13 @@ public void testLoadRebalancing() throws Exception {
 
         startGrid(4);
 
-        awaitPartitionMapExchange(true, true, null);
+        waitForRebalancing(3, 6);
+        waitForRebalancing(4, 6);
 
         concurrentStartFinished = true;
 
+        awaitPartitionMapExchange(true, true, null);
+
         checkSupplyContextMapIsEmpty();
 
         t1.join();
@@ -431,29 +442,27 @@ protected void checkPartitionMapExchangeFinished() {
     }
 
     /**
-     * Method checks for {@link GridDhtPartitionsSingleMessage} or {@link GridDhtPartitionsFullMessage}
-     * not received within {@code TOPOLOGY_STILLNESS_TIME} bound.
-     *
      * @throws Exception If failed.
      */
-    protected void awaitPartitionMessagesAbsent() throws Exception {
-        log.info("Checking GridDhtPartitions*Message absent (it will take up to " +
-            TOPOLOGY_STILLNESS_TIME + " ms) ... ");
-
-        // Start waiting new messages from current point of time.
-        lastPartMsgTime = U.currentTimeMillis();
-
-        assertTrue("Should not have partition Single or Full messages within bound " +
-                TOPOLOGY_STILLNESS_TIME + " ms.",
-            GridTestUtils.waitForCondition(
-                new GridAbsPredicateX() {
-                    @Override public boolean applyx() {
-                        return lastPartMsgTime + TOPOLOGY_STILLNESS_TIME < U.currentTimeMillis();
-                    }
-                },
-                2 * TOPOLOGY_STILLNESS_TIME // 30 sec to gain stable topology and 30 sec of silence.
-            )
-        );
+    protected void checkPartitionMapMessagesAbsent() throws Exception {
+        map.clear();
+
+        record = true;
+
+        log.info("Checking GridDhtPartitions*Message absent (it will take 30 SECONDS) ... ");
+
+        U.sleep(30_000);
+
+        record = false;
+
+        AtomicInteger iF = map.get(GridDhtPartitionsFullMessage.class);
+        AtomicInteger iS = map.get(GridDhtPartitionsSingleMessage.class);
+
+        Integer fullMap = iF != null ? iF.get() : null;
+        Integer singleMap = iS != null ? iS.get() : null;
+
+        assertTrue("Unexpected full map messages: " + fullMap, fullMap == null || fullMap.equals(1)); // 1 message can be sent right after all checks passed.
+        assertNull("Unexpected single map messages", singleMap);
     }
 
     /** {@inheritDoc} */
@@ -486,7 +495,11 @@ public void testComplexRebalancing() throws Exception {
                     while (!concurrentStartFinished2)
                         U.sleep(10);
 
-                    awaitPartitionMapExchange();
+                    waitForRebalancing(0, 5, 0);
+                    waitForRebalancing(1, 5, 0);
+                    waitForRebalancing(2, 5, 0);
+                    waitForRebalancing(3, 5, 0);
+                    waitForRebalancing(4, 5, 0);
 
                     //New cache should start rebalancing.
                     CacheConfiguration<Integer, Integer> cacheRCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
@@ -539,6 +552,12 @@ public void testComplexRebalancing() throws Exception {
         t2.join();
         t3.join();
 
+        waitForRebalancing(0, 5, 1);
+        waitForRebalancing(1, 5, 1);
+        waitForRebalancing(2, 5, 1);
+        waitForRebalancing(3, 5, 1);
+        waitForRebalancing(4, 5, 1);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
@@ -558,23 +577,35 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(1);
 
+        waitForRebalancing(0, 6);
+        waitForRebalancing(2, 6);
+        waitForRebalancing(3, 6);
+        waitForRebalancing(4, 6);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(0);
 
+        waitForRebalancing(2, 7);
+        waitForRebalancing(3, 7);
+        waitForRebalancing(4, 7);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(2);
 
+        waitForRebalancing(3, 8);
+        waitForRebalancing(4, 8);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         checkSupplyContextMapIsEmpty();
 
@@ -582,7 +613,7 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(3);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing(4, 9);
 
         checkSupplyContextMapIsEmpty();
 
@@ -603,26 +634,36 @@ public void testComplexRebalancing() throws Exception {
     /**
      *
      */
-    private static class CollectingCommunicationSpi extends TcpCommunicationSpi {
-        /** */
-        @LoggerResource
-        private IgniteLogger log;
-
+    private class CountingCommunicationSpi extends TcpCommunicationSpi {
         /** {@inheritDoc} */
         @Override public void sendMessage(final ClusterNode node, final Message msg,
             final IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
             final Object msg0 = ((GridIoMessage)msg).message();
 
-            if (msg0 instanceof GridDhtPartitionsSingleMessage ||
-                msg0 instanceof GridDhtPartitionsFullMessage) {
-                lastPartMsgTime = U.currentTimeMillis();
-
-                log.info("Last seen time of GridDhtPartitionsSingleMessage or GridDhtPartitionsFullMessage updated " +
-                    "[lastPartMsgTime=" + lastPartMsgTime +
-                    ", node=" + node.id() + ']');
-            }
+            recordMessage(msg0);
 
             super.sendMessage(node, msg, ackC);
         }
+
+        /**
+         * @param msg Message.
+         */
+        private void recordMessage(Object msg) {
+            if (record) {
+                Class id = msg.getClass();
+
+                AtomicInteger ai = map.get(id);
+
+                if (ai == null) {
+                    ai = new AtomicInteger();
+
+                    AtomicInteger oldAi = map.putIfAbsent(id, ai);
+
+                    (oldAi != null ? oldAi : ai).incrementAndGet();
+                }
+                else
+                    ai.incrementAndGet();
+            }
+        }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
index ce84156e56dca..15ec41557dfba 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
@@ -383,7 +383,7 @@ public void testRejoinWithCleanLfs() throws Exception {
         startGrid("flaky");
 
         System.out.println("### Starting rebalancing after flaky node join");
-        awaitPartitionMapExchange();
+        waitForRebalancing();
         System.out.println("### Rebalancing is finished after flaky node join");
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
@@ -689,7 +689,7 @@ private void tryChangeBaselineDown(
         ig0.cluster().setBaselineTopology(fullBlt.subList(0, newBaselineSize));
 
         System.out.println("### Starting rebalancing after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
-        awaitPartitionMapExchange();
+        waitForRebalancing();
         System.out.println("### Rebalancing is finished after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
index 3500c8de7f7ed..8f2e7389f8bb3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
@@ -157,8 +157,7 @@ public void testReservation() throws Exception {
 
         SUPPLY_MESSAGE_LATCH.get().countDown();
 
-        // Partition is OWNING on grid(0) and grid(1)
-        awaitPartitionMapExchange();
+        waitForRebalancing(); // Partition is OWNING on grid(0) and grid(1)
 
         for (int i = 0; i < 2; i++) {
             for (int j = 0; i < 500; i++)
@@ -179,7 +178,7 @@ public void testReservation() throws Exception {
 
         startGrid(0);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         assertEquals(2, grid(1).context().discovery().aliveServerNodes().size());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index 0ed39ce003f4e..d585a82e66a2f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -192,7 +192,7 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
 
                     grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
 
-                    awaitPartitionMapExchange();
+                    waitForRebalancing();
                 }
                 catch (Throwable expected) {
                     // There can be any exception. Do nothing.
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index 9de2702d3790e..78f3c03eda818 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -120,8 +120,7 @@ public void testMappingsPersistedOnJoin() throws Exception {
         c1.put(k, new DummyObject(k));
 
         startGrid(2);
-
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         stopAllGrids();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 0414a4173d48a..0d73e1de43c34 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -43,7 +43,6 @@
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteEvents;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.CachePeekMode;
@@ -62,7 +61,6 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityFunctionContextImpl;
@@ -104,9 +102,6 @@
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.testframework.GridTestNode;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.GridAbstractTest;
@@ -686,34 +681,32 @@ protected void awaitPartitionMapExchange(
 
                                 if (affNodesCnt != ownerNodesCnt || !affNodes.containsAll(owners) ||
                                     (waitEvicts && loc != null && loc.state() != GridDhtPartitionState.OWNING)) {
-                                    if (i % 50 == 0)
-                                        LT.warn(log(), "Waiting for topology map update [" +
-                                            "igniteInstanceName=" + g.name() +
-                                            ", cache=" + cfg.getName() +
-                                            ", cacheId=" + dht.context().cacheId() +
-                                            ", topVer=" + top.readyTopologyVersion() +
-                                            ", p=" + p +
-                                            ", affNodesCnt=" + affNodesCnt +
-                                            ", ownersCnt=" + ownerNodesCnt +
-                                            ", affNodes=" + F.nodeIds(affNodes) +
-                                            ", owners=" + F.nodeIds(owners) +
-                                            ", topFut=" + topFut +
-                                            ", locNode=" + g.cluster().localNode() + ']');
-                                }
-                                else
-                                    match = true;
-                            }
-                            else {
-                                if (i % 50 == 0)
                                     LT.warn(log(), "Waiting for topology map update [" +
                                         "igniteInstanceName=" + g.name() +
                                         ", cache=" + cfg.getName() +
                                         ", cacheId=" + dht.context().cacheId() +
                                         ", topVer=" + top.readyTopologyVersion() +
-                                        ", started=" + dht.context().started() +
                                         ", p=" + p +
-                                        ", readVer=" + readyVer +
+                                        ", affNodesCnt=" + affNodesCnt +
+                                        ", ownersCnt=" + ownerNodesCnt +
+                                        ", affNodes=" + F.nodeIds(affNodes) +
+                                        ", owners=" + F.nodeIds(owners) +
+                                        ", topFut=" + topFut +
                                         ", locNode=" + g.cluster().localNode() + ']');
+                                }
+                                else
+                                    match = true;
+                            }
+                            else {
+                                LT.warn(log(), "Waiting for topology map update [" +
+                                    "igniteInstanceName=" + g.name() +
+                                    ", cache=" + cfg.getName() +
+                                    ", cacheId=" + dht.context().cacheId() +
+                                    ", topVer=" + top.readyTopologyVersion() +
+                                    ", started=" + dht.context().started() +
+                                    ", p=" + p +
+                                    ", readVer=" + readyVer +
+                                    ", locNode=" + g.cluster().localNode() + ']');
                             }
 
                             if (!match) {
@@ -875,7 +868,7 @@ protected void printPartitionState(String cacheName, int firstParts) {
                     .append(" res=").append(f.isDone() ? f.get() : "N/A")
                     .append(" topVer=")
                     .append((U.hasField(f, "topVer") ?
-                        String.valueOf(U.<Object>field(f, "topVer")) : "[unknown] may be it is finished future"))
+                        String.valueOf(U.field(f, "topVer")) : "[unknown] may be it is finished future"))
                     .append("\n");
 
                 Map<UUID, T2<Long, Collection<Integer>>> remaining = U.field(f, "remaining");
@@ -959,61 +952,6 @@ protected void printPartitionState(String cacheName, int firstParts) {
         log.info("dump partitions state for <" + cacheName + ">:\n" + sb.toString());
     }
 
-    /**
-     * Use method for manual rebalaincing cache on all nodes. Note that using
-     * <pre name="code" class="java">
-     *   for (int i = 0; i < G.allGrids(); i++)
-     *     grid(i).cache(CACHE_NAME).rebalance().get();
-     * </pre>
-     * for rebalancing cache will lead to flaky test cases.
-     *
-     * @param ignite Ignite server instance for getting {@code compute} facade over all cluster nodes.
-     * @param cacheName Cache name for manual rebalancing on cluster. Usually used when used when
-     * {@link CacheConfiguration#getRebalanceDelay()} configuration parameter set to {@code -1} value.
-     * @throws IgniteCheckedException If fails.
-     */
-    protected void manualCacheRebalancing(Ignite ignite,
-        final String cacheName) throws IgniteCheckedException {
-        if (ignite.configuration().isClientMode())
-            return;
-
-        IgniteFuture<Void> fut =
-            ignite.compute().withTimeout(5_000).broadcastAsync(new IgniteRunnable() {
-                /** */
-                @LoggerResource
-                IgniteLogger log;
-
-                /** */
-                @IgniteInstanceResource
-                private Ignite ignite;
-
-                /** {@inheritDoc} */
-                @Override public void run() {
-                    IgniteCache<?, ?> cache = ignite.cache(cacheName);
-
-                    assertNotNull(cache);
-
-                    while (!(Boolean)cache.rebalance().get()) {
-                        try {
-                            U.sleep(100);
-                        }
-                        catch (IgniteInterruptedCheckedException e) {
-                            throw new IgniteException(e);
-                        }
-                    }
-
-                    if (log.isInfoEnabled())
-                        log.info("Manual rebalance finished [node=" + ignite.name() + ", cache=" + cacheName + "]");
-                }
-            });
-
-        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override public boolean apply() {
-                return fut.isDone();
-            }
-        }, 5_000));
-    }
-
     /**
      * @param id Node id.
      * @param major Major ver.
@@ -1062,13 +1000,13 @@ protected void waitForRebalancing(IgniteEx ignite, AffinityTopologyVersion top)
 
             for (GridCacheAdapter c : ignite.context().cache().internalCaches()) {
                 GridDhtPartitionDemander.RebalanceFuture fut =
-                        (GridDhtPartitionDemander.RebalanceFuture)c.preloader().rebalanceFuture();
+                    (GridDhtPartitionDemander.RebalanceFuture)c.preloader().rebalanceFuture();
 
                 if (fut.topologyVersion() == null || fut.topologyVersion().compareTo(top) < 0) {
                     finished = false;
 
                     log.info("Unexpected future version, will retry [futVer=" + fut.topologyVersion() +
-                            ", expVer=" + top + ']');
+                        ", expVer=" + top + ']');
 
                     U.sleep(100);
 
@@ -1729,8 +1667,6 @@ protected static <T> T doInTransaction(Ignite ignite,
      *
      */
     protected void cleanPersistenceDir() throws Exception {
-        assertTrue("Grids are not stopped", F.isEmpty(G.allGrids()));
-
         U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "cp", false));
         U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
         U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), "marshaller", false));
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index e386beb227acf..b66bf5b5026fa 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -53,10 +53,11 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingAsyncSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingCancelTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheDaemonNodeReplicatedSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicGetAndTransformStoreSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedBasicApiTest;
@@ -83,7 +84,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.replicated.preloader.GridCacheReplicatedPreloadStartStopEventsSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheDaemonNodeLocalSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheLocalByteArrayValuesSelfTest;
-import org.apache.ignite.testframework.junits.GridAbstractTest;
 
 /**
  * Test suite.
@@ -94,8 +94,6 @@ public class IgniteCacheTestSuite3 extends TestSuite {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite() throws Exception {
-//        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
-
         TestSuite suite = new TestSuite("IgniteCache Test Suite part 3");
 
         suite.addTestSuite(IgniteCacheGroupsTest.class);
@@ -153,7 +151,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
         suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
         suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
-        suite.addTestSuite(GridCacheRebalancingCancelTest.class);
+        suite.addTestSuite(GridCacheRebalancingPartitionCountersTest.class);
+        suite.addTestSuite(GridCacheRebalancingWithAsyncClearingTest.class);
 
         // Test for byte array value special case.
         suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);

From 2348d9258e6f636f2a3dc77823fe06fcb8bf702a Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Wed, 22 Aug 2018 21:06:14 +0300
Subject: [PATCH 323/543] IGNITE-9296 Stop walSyncer thread before walWriter
 thread in order to avoid hang up

Signed-off-by: Andrey Gura <agura@apache.org>
(cherry picked from commit d91c6aa58a471451fa2de2557b377ebf674cefab)
---
 .../cache/persistence/wal/FileWriteAheadLogManager.java     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 019497aa3030a..31f7383c1191d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -562,6 +562,9 @@ private void checkWalConfiguration() throws IgniteCheckedException {
             if (currHnd != null)
                 currHnd.close(false);
 
+            if (walSegmentSyncWorker != null)
+                walSegmentSyncWorker.shutdown();
+
             if (walWriter != null)
                 walWriter.shutdown();
 
@@ -573,9 +576,6 @@ private void checkWalConfiguration() throws IgniteCheckedException {
 
             if (decompressor != null)
                 decompressor.shutdown();
-
-            if (walSegmentSyncWorker != null)
-                walSegmentSyncWorker.shutdown();
         }
         catch (Exception e) {
             U.error(log, "Failed to gracefully close WAL segment: " + this.currHnd.fileIO, e);

From 406a8263165bceba3153c98ba990b4f93a7163c0 Mon Sep 17 00:00:00 2001
From: AMedvedev <andrew.y.medvedev>
Date: Thu, 23 Aug 2018 11:07:43 +0300
Subject: [PATCH 324/543] IGNITE-9235: Fixed GridMergeIndexSorted compare
 routine. This closes #4498.

(cherry picked from commit 394d5ea7d2713827a89da53b8a4acd03efae6344)
---
 .../processors/query/h2/twostep/GridMergeIndexSorted.java     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
index 0dc8354a4413b..9f5547ace39c2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMergeIndexSorted.java
@@ -61,7 +61,9 @@ public final class GridMergeIndexSorted extends GridMergeIndex {
     /** */
     private final Comparator<RowStream> streamCmp = new Comparator<RowStream>() {
         @Override public int compare(RowStream o1, RowStream o2) {
-            // Nulls at the beginning.
+            if (o1 == o2) // both nulls
+                return 0;
+
             if (o1 == null)
                 return -1;
 

From 9f18538fe5bb2d339bd0adeb3c806ac8772c2fd2 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Wed, 22 Aug 2018 17:33:39 +0300
Subject: [PATCH 325/543] IGNITE-9188 Fixed unvalid partition eviction if
 rebalance was cancelled by a topology event - Fixes #4578.

---
 .../cache/CacheAffinitySharedManager.java     |  17 +-
 .../GridCachePartitionExchangeManager.java    |   2 +-
 .../GridDhtPartitionsExchangeFuture.java      |   5 +-
 .../CacheDataLossOnPartitionMoveTest.java     | 296 ++++++++++++++++++
 .../junits/common/GridCommonAbstractTest.java |  20 ++
 .../testsuites/IgniteCacheTestSuite7.java     |  52 +++
 6 files changed, 383 insertions(+), 9 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 08a07a457f247..6c03cc79fc0b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -2250,10 +2250,19 @@ private <T> Map<Integer, Map<Integer, List<T>>> initAffinityBasedOnPartitionsAva
                     if (!owners.isEmpty() && !owners.contains(curPrimary))
                         curPrimary = owners.get(0);
 
-                    if (curPrimary != null && newPrimary != null && !curPrimary.equals(newPrimary)) {
-                        if (aliveNodes.contains(curPrimary)) {
-                            GridDhtPartitionState state = top.partitionState(newPrimary.id(), p);
+                    // If new assignment is empty preserve current ownership for alive nodes.
+                    if (curPrimary != null && newPrimary == null) {
+                        newNodes0 = new ArrayList<>(curNodes.size());
+
+                        for (ClusterNode node : curNodes) {
+                            if (aliveNodes.contains(node))
+                                newNodes0.add(node);
+                        }
+                    }
+                    else if (curPrimary != null && !curPrimary.equals(newPrimary)) {
+                        GridDhtPartitionState state = top.partitionState(newPrimary.id(), p);
 
+                        if (aliveNodes.contains(curPrimary)) {
                             if (state != GridDhtPartitionState.OWNING) {
                                 newNodes0 = latePrimaryAssignment(grpHolder.affinity(),
                                     p,
@@ -2263,8 +2272,6 @@ private <T> Map<Integer, Map<Integer, List<T>>> initAffinityBasedOnPartitionsAva
                             }
                         }
                         else {
-                            GridDhtPartitionState state = top.partitionState(newPrimary.id(), p);
-
                             if (state != GridDhtPartitionState.OWNING) {
                                 for (int i = 1; i < curNodes.size(); i++) {
                                     ClusterNode curNode = curNodes.get(i);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index f5ccedfc961d1..aaec9d8790a34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -302,7 +302,7 @@ private void notifyNodeFail(DiscoveryEvent evt) {
      * @param cache Discovery data cache.
      */
     private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) {
-        // Clean local join caches context.
+        // Clear local join caches context.
         cctx.cache().localJoinCachesContext();
 
         if (log.isDebugEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 7e9dd1472d1ae..90829d6f5c55d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -496,7 +496,7 @@ public void onEvent(GridDhtPartitionExchangeId exchId, DiscoveryEvent discoEvt,
         assert exchId.equals(this.exchId);
 
         this.exchId.discoveryEvent(discoEvt);
-        this.firstDiscoEvt= discoEvt;
+        this.firstDiscoEvt = discoEvt;
         this.firstEvtDiscoCache = discoCache;
 
         evtLatch.countDown();
@@ -3807,9 +3807,8 @@ public void onNodeLeft(final ClusterNode node) {
 
                             crd0 = crd;
 
-                            if (crd0 == null) {
+                            if (crd0 == null)
                                 finishState = new FinishState(null, initialVersion(), null);
-                            }
                         }
 
                         if (crd0 == null) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java
new file mode 100644
index 0000000000000..2a992714745f3
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java
@@ -0,0 +1,296 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.AffinityFunctionContext;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.GridCacheUtils;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.testframework.GridTestNode;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+
+/**
+ *
+ */
+public class CacheDataLossOnPartitionMoveTest extends GridCommonAbstractTest {
+    /** */
+    public static final long MB = 1024 * 1024L;
+
+    /** */
+    public static final String GRP_ATTR = "grp";
+
+    /** */
+    public static final int GRIDS_CNT = 2;
+
+    /** */
+    public static final String EVEN_GRP = "event";
+
+    /** */
+    public static final String ODD_GRP = "odd";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        cfg.setPeerClassLoadingEnabled(true);
+
+        Map<String, Object> attrs = new HashMap<>();
+
+        attrs.put(GRP_ATTR, grp(getTestIgniteInstanceIndex(igniteInstanceName)));
+
+        cfg.setUserAttributes(attrs);
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration().setPersistenceEnabled(true).setInitialSize(50 * MB).setMaxSize(50 * MB))
+            .setWalMode(WALMode.LOG_ONLY);
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        cfg.setCacheConfiguration(configuration(DEFAULT_CACHE_NAME));
+
+        return cfg;
+    }
+
+    /**
+     * @param name Name.
+     */
+    private CacheConfiguration configuration(String name) {
+        return new CacheConfiguration(name).
+            setCacheMode(CacheMode.PARTITIONED).
+            setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL).
+            setBackups(1).
+            setRebalanceBatchSize(1).
+            setAffinity(new TestAffinityFunction().setPartitions(32));
+    }
+
+    /**
+     * @param idx Index.
+     */
+    private String grp(int idx) {
+        return idx < GRIDS_CNT / 2 ? EVEN_GRP : ODD_GRP;
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testDataLossOnPartitionMove() throws Exception {
+        try {
+            Ignite ignite = startGridsMultiThreaded(GRIDS_CNT / 2, false);
+
+            ignite.cluster().active(true);
+
+            List<Integer> toCp = movingKeysAfterJoin(ignite, DEFAULT_CACHE_NAME, 1,
+                node -> ((GridTestNode)node).setAttribute(GRP_ATTR, ODD_GRP));
+
+            int blockPartId = ignite.affinity(DEFAULT_CACHE_NAME).partition(toCp.get(0));
+
+            awaitPartitionMapExchange();
+
+            int c = 0;
+
+            for (int i = 0; i < 1000; i++) {
+                if (ignite.affinity(DEFAULT_CACHE_NAME).partition(i) == blockPartId) {
+                    ignite.cache(DEFAULT_CACHE_NAME).put(i, i);
+
+                    c++;
+                }
+            }
+
+            assertEquals(c, ignite.cache(DEFAULT_CACHE_NAME).size());
+
+            startGridsMultiThreaded(GRIDS_CNT / 2, GRIDS_CNT / 2);
+
+            // Prevent rebalancing to new nodes.
+            for (Ignite ig0 : G.allGrids()) {
+                TestRecordingCommunicationSpi.spi(ig0).blockMessages((node, message) -> {
+                    if (message instanceof GridDhtPartitionDemandMessage) {
+                        assertTrue(node.order() <= GRIDS_CNT / 2);
+
+                        GridDhtPartitionDemandMessage msg = (GridDhtPartitionDemandMessage)message;
+
+                        return msg.groupId() == CU.cacheId(DEFAULT_CACHE_NAME);
+                    }
+
+                    return false;
+                });
+            }
+
+            ignite.cluster().setBaselineTopology(GRIDS_CNT);
+
+            for (Ignite ig0 : G.allGrids()) {
+                if (ig0.cluster().localNode().order() <= GRIDS_CNT / 2)
+                    continue;
+
+                TestRecordingCommunicationSpi.spi(ig0).waitForBlocked();
+            }
+
+            assertEquals(c, ignite.cache(DEFAULT_CACHE_NAME).size());
+
+            int i = 0;
+
+            while(i < GRIDS_CNT / 2) {
+                stopGrid(GRIDS_CNT / 2 + i);
+
+                i++;
+            }
+
+            awaitPartitionMapExchange();
+
+            for (Ignite ig : G.allGrids()) {
+                GridDhtLocalPartition locPart = dht(ig.cache(DEFAULT_CACHE_NAME)).topology().localPartition(blockPartId);
+
+                assertNotNull(locPart);
+
+                assertEquals("Unexpected state", OWNING, locPart.state());
+            }
+
+            startGridsMultiThreaded(GRIDS_CNT / 2, GRIDS_CNT / 2);
+
+            awaitPartitionMapExchange(true, true, null);
+
+            for (Ignite ig : G.allGrids()) {
+                GridDhtLocalPartition locPart = dht(ig.cache(DEFAULT_CACHE_NAME)).topology().localPartition(blockPartId);
+
+                assertNotNull(locPart);
+
+                switch ((String)ig.cluster().localNode().attribute(GRP_ATTR)) {
+                    case EVEN_GRP:
+                        assertEquals("Unexpected state", EVICTED, locPart.state());
+
+                        break;
+
+                    case ODD_GRP:
+                        assertEquals("Unexpected state", OWNING, locPart.state());
+
+                        break;
+
+                    default:
+                        fail();
+                }
+            }
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /** */
+    public static class TestAffinityFunction extends RendezvousAffinityFunction {
+        /** */
+        public TestAffinityFunction() {
+        }
+
+        /** */
+        public TestAffinityFunction(boolean exclNeighbors) {
+            super(exclNeighbors);
+        }
+
+        /** */
+        public TestAffinityFunction(boolean exclNeighbors, int parts) {
+            super(exclNeighbors, parts);
+        }
+
+        /** */
+        public TestAffinityFunction(int parts,
+            @Nullable IgniteBiPredicate<ClusterNode, ClusterNode> backupFilter) {
+            super(parts, backupFilter);
+        }
+
+        /** {@inheritDoc} */
+        @Override public List<List<ClusterNode>> assignPartitions(AffinityFunctionContext affCtx) {
+            int parts = partitions();
+
+            List<List<ClusterNode>> assignments = new ArrayList<>(parts);
+
+            Map<UUID, Collection<ClusterNode>> neighborhoodCache = isExcludeNeighbors() ?
+                GridCacheUtils.neighbors(affCtx.currentTopologySnapshot()) : null;
+
+            List<ClusterNode> nodes = affCtx.currentTopologySnapshot();
+
+            Map<Object, List<ClusterNode>> nodesByGrp = U.newHashMap(2);
+
+            for (ClusterNode node : nodes) {
+                Object grp = node.attribute(GRP_ATTR);
+
+                List<ClusterNode> grpNodes = nodesByGrp.get(grp);
+
+                if (grpNodes == null)
+                    nodesByGrp.put(grp, (grpNodes = new ArrayList<>()));
+
+                grpNodes.add(node);
+            }
+
+            boolean split = nodesByGrp.size() == 2;
+
+            for (int i = 0; i < parts; i++) {
+                List<ClusterNode> partAssignment = assignPartition(i, split ?
+                        nodesByGrp.get(i % 2 == 0 ? EVEN_GRP : ODD_GRP) : nodes,
+                    affCtx.backups(), neighborhoodCache);
+
+                assignments.add(partAssignment);
+            }
+
+            return assignments;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return Integer.MAX_VALUE;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 0d73e1de43c34..e6984c4b8919e 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -89,6 +89,7 @@
 import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask;
+import org.apache.ignite.internal.util.lang.GridAbsClosure;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
@@ -101,6 +102,7 @@
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.testframework.GridTestNode;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -1201,6 +1203,21 @@ protected List<Integer> nearKeys(IgniteCache<?, ?> cache, int cnt, int startFrom
      * @return List of keys.
      */
     protected final List<Integer> movingKeysAfterJoin(Ignite ign, String cacheName, int size) {
+        return movingKeysAfterJoin(ign, cacheName, size, null);
+    }
+
+    /**
+     * Return list of keys that are primary for given node on current topology,
+     * but primary node will change after new node will be added.
+     *
+     * @param ign Ignite.
+     * @param cacheName Cache name.
+     * @param size Number of keys.
+     * @param nodeInitializer Node initializer closure.
+     * @return List of keys.
+     */
+    protected final List<Integer> movingKeysAfterJoin(Ignite ign, String cacheName, int size,
+        @Nullable IgniteInClosure<ClusterNode> nodeInitializer) {
         assertEquals("Expected consistentId is set to node name", ign.name(), ign.cluster().localNode().consistentId());
 
         GridCacheContext<Object, Object> cctx = ((IgniteKernal)ign).context().cache().internalCache(cacheName).context();
@@ -1220,6 +1237,9 @@ protected final List<Integer> movingKeysAfterJoin(Ignite ign, String cacheName,
 
         GridTestNode fakeNode = new GridTestNode(UUID.randomUUID(), null);
 
+        if (nodeInitializer != null)
+            nodeInitializer.apply(fakeNode);
+
         fakeNode.consistentId(getTestIgniteInstanceName(nodes.size()));
 
         nodes.add(fakeNode);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index dbd455acf39a5..e16200e0c6e09 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -19,6 +19,25 @@
 
 import java.util.Set;
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.authentication.Authentication1kUsersNodeRestartTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationConfigurationClusterTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationOnNotActiveClusterTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNPEOnStartTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNodeRestartTest;
+import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorSelfTest;
+import org.apache.ignite.internal.processors.cache.CacheDataRegionConfigurationTest;
+import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMBeanTest;
+import org.apache.ignite.internal.processors.cache.CacheMetricsManageTest;
+import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailWithPersistenceTest;
+import org.apache.ignite.internal.processors.cache.WalModeChangeAdvancedSelfTest;
+import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
+import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
+import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.CheckpointBufferDeadlockTest;
 
 /**
@@ -43,6 +62,39 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
 
         suite.addTestSuite(CheckpointBufferDeadlockTest.class);
 
+        suite.addTestSuite(AuthenticationConfigurationClusterTest.class);
+        suite.addTestSuite(AuthenticationProcessorSelfTest.class);
+        suite.addTestSuite(AuthenticationOnNotActiveClusterTest.class);
+        suite.addTestSuite(AuthenticationProcessorNodeRestartTest.class);
+        suite.addTestSuite(AuthenticationProcessorNPEOnStartTest.class);
+        suite.addTestSuite(Authentication1kUsersNodeRestartTest.class);
+
+        suite.addTestSuite(CacheDataRegionConfigurationTest.class);
+
+        suite.addTestSuite(WalModeChangeAdvancedSelfTest.class);
+        suite.addTestSuite(WalModeChangeSelfTest.class);
+        suite.addTestSuite(WalModeChangeCoordinatorNotAffinityNodeSelfTest.class);
+
+        suite.addTestSuite(Cache64kPartitionsTest.class);
+        suite.addTestSuite(GridCacheRebalancingPartitionCountersTest.class);
+        suite.addTestSuite(GridCacheRebalancingWithAsyncClearingTest.class);
+
+        suite.addTestSuite(IgnitePdsCacheAssignmentNodeRestartsTest.class);
+        suite.addTestSuite(TxRollbackAsyncWithPersistenceTest.class);
+
+        suite.addTestSuite(CacheGroupMetricsMBeanTest.class);
+        suite.addTestSuite(CacheMetricsManageTest.class);
+        suite.addTestSuite(PageEvictionMultinodeMixedRegionsTest.class);
+
+        suite.addTestSuite(IgniteDynamicCacheStartFailWithPersistenceTest.class);
+
+        suite.addTestSuite(TxWithSmallTimeoutAndContentionOneKeyTest.class);
+
+        suite.addTestSuite(CacheRentingStateRepairTest.class);
+
+        suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class);
+        suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class);
+
         return suite;
     }
 }

From 0555a9bd53f20941a04f0f7d32c746af09cdb4cc Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Thu, 23 Aug 2018 19:51:04 +0300
Subject: [PATCH 326/543] IGNITE-9188 Fixed unvalid partition eviction if
 rebalance was cancelled by a topology event - Fixes #4578.

---
 .../ignite/testsuites/IgniteCacheTestSuite7.java      | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index e16200e0c6e09..f3b6a9d84f6ac 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -27,18 +27,19 @@
 import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheDataRegionConfigurationTest;
 import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMBeanTest;
-import org.apache.ignite.internal.processors.cache.CacheMetricsManageTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailWithPersistenceTest;
 import org.apache.ignite.internal.processors.cache.WalModeChangeAdvancedSelfTest;
 import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
 import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
-import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheDataLossOnPartitionMoveTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
 import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.CheckpointBufferDeadlockTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncWithPersistenceTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxWithSmallTimeoutAndContentionOneKeyTest;
 
 /**
  * Test suite.
@@ -58,7 +59,7 @@ public static TestSuite suite() throws Exception {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
-        TestSuite suite = new TestSuite("IgniteCache Test Suite part 7");
+        TestSuite suite = new TestSuite("IgniteCache With Persistence Test Suite");
 
         suite.addTestSuite(CheckpointBufferDeadlockTest.class);
 
@@ -83,16 +84,12 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(TxRollbackAsyncWithPersistenceTest.class);
 
         suite.addTestSuite(CacheGroupMetricsMBeanTest.class);
-        suite.addTestSuite(CacheMetricsManageTest.class);
         suite.addTestSuite(PageEvictionMultinodeMixedRegionsTest.class);
 
         suite.addTestSuite(IgniteDynamicCacheStartFailWithPersistenceTest.class);
 
         suite.addTestSuite(TxWithSmallTimeoutAndContentionOneKeyTest.class);
 
-        suite.addTestSuite(CacheRentingStateRepairTest.class);
-
-        suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class);
         suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class);
 
         return suite;

From 288a5078989619579d9abbe669c3ee32e563d45c Mon Sep 17 00:00:00 2001
From: "Andrey V. Mashenkov" <andrey.mashenkov@gmail.com>
Date: Thu, 23 Aug 2018 20:44:39 +0300
Subject: [PATCH 327/543] IGNITE-9363 Jetty tests forget to stop nodes on
 finished. - Fixes #4609.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>

(cherry picked from commit e0c034e)
Signed-off-by: Andrey V. Mashenkov <andrey.mashenkov@gmail.com>

(cherry picked from commit 56bf7fc)
Signed-off-by: Andrey V. Mashenkov <andrey.mashenkov@gmail.com>
---
 .../rest/AbstractRestProcessorSelfTest.java        | 14 +++++++++-----
 .../rest/JettyRestProcessorCommonSelfTest.java     |  2 ++
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/AbstractRestProcessorSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/AbstractRestProcessorSelfTest.java
index 3738915ad8638..e5c658ccfae82 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/AbstractRestProcessorSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/AbstractRestProcessorSelfTest.java
@@ -43,12 +43,9 @@ public abstract class AbstractRestProcessorSelfTest extends GridCommonAbstractTe
 
     /** {@inheritDoc} */
     @Override protected void beforeTestsStarted() throws Exception {
-        startGrids(gridCount());
-    }
+        cleanPersistenceDir();
 
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        stopAllGrids();
+        startGrids(gridCount());
     }
 
     /** {@inheritDoc} */
@@ -63,6 +60,13 @@ public abstract class AbstractRestProcessorSelfTest extends GridCommonAbstractTe
         assertEquals(0, jcache().localSize());
     }
 
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java
index 2076d490ad72f..1b9328443d68d 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorCommonSelfTest.java
@@ -56,6 +56,8 @@ public abstract class JettyRestProcessorCommonSelfTest extends AbstractRestProce
 
     /** {@inheritDoc} */
     @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
         System.clearProperty(IGNITE_JETTY_PORT);
     }
 

From 78bada74fe1b7332b3e8a32539d7aa1dbda24059 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 24 Aug 2018 18:10:23 +0700
Subject: [PATCH 328/543] IGNITE-9337 Added support to passing "null" in
 collections.

(cherry picked from commit 9b000978c84357d0198d5fd09e310a9cb4ea2aa3)
---
 .../apache/ignite/internal/visor/compute/VisorGatewayTask.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
index d14463ff90f6c..c41864fbc6504 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/compute/VisorGatewayTask.java
@@ -246,7 +246,7 @@ private static class VisorGatewayJob extends ComputeJobAdapter {
          * @return Object constructed from string.
          */
         @Nullable private Object toObject(Class cls, String val) {
-            if (val == null  || "null".equals(val))
+            if (val == null  || "null".equals(val) || "nil".equals(val))
                 return null;
 
             if (String.class == cls)

From 60d862550f0dc28d35b8e0db89d878422d05365a Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Fri, 24 Aug 2018 17:02:46 +0300
Subject: [PATCH 329/543] IGNITE-8920 - Fixed transaction hanging on Runtime
 exceptions during commit. - Unified StorageException and
 PersistentStorageIOException. - BPlus tree behavior is overridable in tests.
 - Refactored AccountTransferAmountTest to make it possible to develop any
 transaction failover based on it. - Fixes #4432.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
(cherry picked from commit dde936ace5eb0618edc360ca05006d76e810add8)

(cherry picked from commit 309633e)
---
 .../internal/NodeStoppingException.java       |   2 +-
 .../wal/IgniteWriteAheadLogManager.java       |   1 +
 .../processors/cache/GridCacheMapEntry.java   |   2 +-
 .../cache/IgniteCacheOffheapManagerImpl.java  |   4 +-
 .../GridDistributedTxRemoteAdapter.java       |  43 +-
 .../dht/GridDhtTxFinishFuture.java            |   5 +-
 .../dht/atomic/GridDhtAtomicCache.java        |   2 +-
 .../GridCacheDatabaseSharedManager.java       |  18 +-
 .../persistence/GridCacheOffheapManager.java  |   1 -
 .../cache/persistence}/StorageException.java  |   2 +-
 .../cache/persistence/file/FilePageStore.java |  48 +-
 .../file/FilePageStoreManager.java            |  23 +-
 .../persistence/metastorage/MetaStorage.java  |   2 +-
 .../persistence/pagemem/PageMemoryEx.java     |   2 +-
 .../persistence/pagemem/PageMemoryImpl.java   |   2 +-
 .../cache/persistence/tree/BPlusTree.java     | 111 ++--
 .../CorruptedTreeException.java}              |  27 +-
 .../tree/util/PageHandlerWrapper.java         |  36 ++
 .../wal/FileWriteAheadLogManager.java         |   2 +-
 .../FsyncModeFileWriteAheadLogManager.java    |   2 +-
 .../cache/transactions/IgniteTxAdapter.java   |  37 +-
 .../transactions/IgniteTxLocalAdapter.java    |  59 +-
 .../resources/META-INF/classnames.properties  |   2 +-
 .../AccountTransferTransactionTest.java       | 331 ----------
 .../IgnitePdsCorruptedStoreTest.java          |   2 -
 .../wal/IgniteWalFormatFileFailoverTest.java  |   2 +-
 .../persistence/pagemem/NoOpWALManager.java   |   2 +-
 .../AbstractTransactionIntergrityTest.java    | 595 ++++++++++++++++++
 ...tegrityWithPrimaryIndexCorruptionTest.java | 238 +++++++
 ...ionIntegrityWithSystemWorkerDeathTest.java | 106 ++++
 .../testsuites/IgniteBasicTestSuite.java      |   4 +-
 .../IgniteBasicWithPersistenceTestSuite.java  |   0
 .../testsuites/IgniteCacheTestSuite6.java     |   3 +
 33 files changed, 1180 insertions(+), 536 deletions(-)
 rename modules/core/src/main/java/org/apache/ignite/internal/{pagemem/wal => processors/cache/persistence}/StorageException.java (96%)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/{file/PersistentStorageIOException.java => tree/CorruptedTreeException.java} (65%)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandlerWrapper.java
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/failure/AccountTransferTransactionTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/NodeStoppingException.java b/modules/core/src/main/java/org/apache/ignite/internal/NodeStoppingException.java
index cc39b14cd92d6..75447a14c53fa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/NodeStoppingException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/NodeStoppingException.java
@@ -22,7 +22,7 @@
 /**
  *
  */
-public class NodeStoppingException extends IgniteCheckedException implements InvalidEnvironmentException {
+public class NodeStoppingException extends IgniteCheckedException {
     /** */
     private static final long serialVersionUID = 0L;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 2b6358b6edaa7..7de005cf10c1a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport;
 
 /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index ed3fa34ad3b49..1eac83b3319bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -34,7 +34,7 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.eviction.EvictableEntry;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 07f99312a56d4..6ce8536688efd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -1527,7 +1527,7 @@ private void finishRemove(GridCacheContext cctx, KeyCacheObject key, @Nullable C
                         rowStore.removeRow(row.link());
                     }
                     catch (IgniteCheckedException e) {
-                        U.error(log, "Fail remove row [link=" + row.link() + "]");
+                        U.error(log, "Failed to remove row [link=" + row.link() + "]");
 
                         IgniteCheckedException ex = exception.get();
 
@@ -1540,7 +1540,7 @@ private void finishRemove(GridCacheContext cctx, KeyCacheObject key, @Nullable C
             });
 
             if (exception.get() != null)
-                throw new IgniteCheckedException("Fail destroy store", exception.get());
+                throw new IgniteCheckedException("Failed to destroy store", exception.get());
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index 1b9b3a8ad28fd..c1293fc0772bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -29,9 +29,11 @@
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.InvalidEnvironmentException;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.InvalidEnvironmentException;
+import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
@@ -49,6 +51,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
@@ -744,26 +747,46 @@ else if (op == READ) {
                                     }
                                 }
                                 catch (Throwable ex) {
-                                    boolean hasIOIssue = X.hasCause(ex, InvalidEnvironmentException.class);
+                                    boolean isNodeStopping = X.hasCause(ex, NodeStoppingException.class);
+                                    boolean hasInvalidEnvironmentIssue = X.hasCause(ex, InvalidEnvironmentException.class);
 
                                     // In case of error, we still make the best effort to commit,
                                     // as there is no way to rollback at this point.
                                     err = new IgniteTxHeuristicCheckedException("Commit produced a runtime exception " +
                                         "(all transaction entries will be invalidated): " + CU.txString(this), ex);
 
-                                    if (hasIOIssue) {
+                                    if (isNodeStopping) {
                                         U.warn(log, "Failed to commit transaction, node is stopping [tx=" + this +
                                             ", err=" + ex + ']');
                                     }
+                                    else if (hasInvalidEnvironmentIssue) {
+                                        U.warn(log, "Failed to commit transaction, node is in invalid state and will be stopped [tx=" + this +
+                                            ", err=" + ex + ']');
+                                    }
                                     else
                                         U.error(log, "Commit failed.", err);
 
-                                    uncommit(hasIOIssue);
-
                                     state(UNKNOWN);
 
+                                    if (hasInvalidEnvironmentIssue)
+                                        cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
+                                    else if (!isNodeStopping) { // Skip fair uncommit in case of node stopping or invalidation.
+                                        try {
+                                            // Courtesy to minimize damage.
+                                            uncommit();
+                                        }
+                                        catch (Throwable ex1) {
+                                            U.error(log, "Failed to uncommit transaction: " + this, ex1);
+
+                                            if (ex1 instanceof Error)
+                                                throw ex1;
+                                        }
+                                    }
+
                                     if (ex instanceof Error)
-                                        throw (Error)ex;
+                                        throw (Error) ex;
+
+                                    throw err;
                                 }
                             }
 
@@ -792,12 +815,6 @@ else if (op == READ) {
                     }
                 }
 
-                if (err != null) {
-                    state(UNKNOWN);
-
-                    throw err;
-                }
-
                 cctx.tm().commitTx(this);
 
                 state(COMMITTED);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index 0ed8419b32297..2f36053b9d838 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.internal.IgniteDiagnosticAware;
 import org.apache.ignite.internal.IgniteDiagnosticPrepareContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheFuture;
@@ -168,7 +169,7 @@ public void rollbackOnError(Throwable e) {
         if (ERR_UPD.compareAndSet(this, null, e)) {
             tx.setRollbackOnly();
 
-            if (X.hasCause(e, InvalidEnvironmentException.class))
+            if (X.hasCause(e, InvalidEnvironmentException.class, NodeStoppingException.class))
                 onComplete();
             else
                 finish(false);
@@ -225,7 +226,7 @@ public void onResult(UUID nodeId, GridDhtTxFinishResponse res) {
 
             if (this.tx.onePhaseCommit() && (this.tx.state() == COMMITTING)) {
                 try {
-                    boolean hasInvalidEnvironmentIssue = X.hasCause(err, InvalidEnvironmentException.class);
+                    boolean hasInvalidEnvironmentIssue = X.hasCause(err, InvalidEnvironmentException.class, NodeStoppingException.class);
 
                     this.tx.tmFinish(err == null, hasInvalidEnvironmentIssue, false);
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 44f2b153db3ad..315cec796ca6c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -39,7 +39,7 @@
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 9a60d14722d68..ec1014d07f9e8 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -88,7 +88,6 @@
 import org.apache.ignite.internal.pagemem.PageUtils;
 import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.pagemem.store.PageStore;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.CacheState;
@@ -119,7 +118,6 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
-import org.apache.ignite.internal.processors.cache.persistence.file.PersistentStorageIOException;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker;
@@ -597,7 +595,7 @@ private void removeCheckpointFiles(CheckpointEntry cpEntry) throws IgniteChecked
                 Files.delete(endFile);
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Failed to delete stale checkpoint files: " + cpEntry, e);
+            throw new StorageException("Failed to delete stale checkpoint files: " + cpEntry, e);
         }
     }
 
@@ -854,7 +852,7 @@ private void unRegistrateMetricsMBean() {
             notifyMetastorageReadyForReadWrite();
         }
         catch (IgniteCheckedException e) {
-            if (X.hasCause(e, StorageException.class, PersistentStorageIOException.class, IOException.class))
+            if (X.hasCause(e, StorageException.class, IOException.class))
                 cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
@@ -900,7 +898,7 @@ private void nodeStart(WALPointer ptr) throws IgniteCheckedException {
             Files.move(Paths.get(cpDir.getAbsolutePath(), tmpFileName), Paths.get(cpDir.getAbsolutePath(), fileName));
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Failed to write node start marker: " + ptr, e);
+            throw new StorageException("Failed to write node start marker: " + ptr, e);
         }
     }
 
@@ -943,12 +941,12 @@ public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedExce
                     buf.clear();
                 }
                 catch (IOException e) {
-                    throw new PersistentStorageIOException("Failed to read node started marker file: " + f.getAbsolutePath(), e);
+                    throw new StorageException("Failed to read node started marker file: " + f.getAbsolutePath(), e);
                 }
             }
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Failed to retreive node started files.", e);
+            throw new StorageException("Failed to retreive node started files.", e);
         }
 
         // Sort start markers by file timestamp.
@@ -2632,9 +2630,9 @@ private CheckpointEntry prepareCheckpointEntry(
      * @param entryBuf Checkpoint entry buffer to write.
      * @param cp Checkpoint entry.
      * @param type Checkpoint entry type.
-     * @throws PersistentStorageIOException If failed to write checkpoint entry.
+     * @throws StorageException If failed to write checkpoint entry.
      */
-    public void writeCheckpointEntry(ByteBuffer entryBuf, CheckpointEntry cp, CheckpointEntryType type) throws PersistentStorageIOException {
+    public void writeCheckpointEntry(ByteBuffer entryBuf, CheckpointEntry cp, CheckpointEntryType type) throws StorageException {
         String fileName = checkpointFileName(cp, type);
         String tmpFileName = fileName + FILE_TMP_SUFFIX;
 
@@ -2654,7 +2652,7 @@ public void writeCheckpointEntry(ByteBuffer entryBuf, CheckpointEntry cp, Checkp
                 Files.move(Paths.get(cpDir.getAbsolutePath(), tmpFileName), Paths.get(cpDir.getAbsolutePath(), fileName));
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Failed to write checkpoint entry [ptr=" + cp.checkpointMark()
+            throw new StorageException("Failed to write checkpoint entry [ptr=" + cp.checkpointMark()
                 + ", cpTs=" + cp.timestamp()
                 + ", cpId=" + cp.checkpointId()
                 + ", type=" + type + "]", e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 0a02ca7f0c4ac..ebc1eaf810627 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -35,7 +35,6 @@
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.PageSupport;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/StorageException.java
similarity index 96%
rename from modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/StorageException.java
index debc391271a97..509dee6264d64 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/StorageException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/StorageException.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.pagemem.wal;
+package org.apache.ignite.internal.processors.cache.persistence;
 
 import java.io.IOException;
 import org.apache.ignite.IgniteCheckedException;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 9a1eb30fa58e9..54d3de182fd33 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.store.PageStore;
 import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
@@ -182,7 +183,7 @@ private long initFile(FileIO fileIO) throws IOException {
      * Checks that file store has correct header and size.
      *
      * @return Next available position in the file to store a data.
-     * @throws IOException If check is failed.
+     * @throws IOException If check has failed.
      */
     private long checkFile(FileIO fileIO) throws IOException {
         ByteBuffer hdr = ByteBuffer.allocate(headerSize()).order(ByteOrder.LITTLE_ENDIAN);
@@ -233,10 +234,10 @@ private long checkFile(FileIO fileIO) throws IOException {
     }
 
     /**
-     * @param cleanFile {@code True} to delete file.
-     * @throws PersistentStorageIOException If failed.
+     * @param delete {@code True} to delete file.
+     * @throws StorageException If failed in case of underlying I/O exception.
      */
-    public void stop(boolean cleanFile) throws PersistentStorageIOException {
+    public void stop(boolean delete) throws StorageException {
         lock.writeLock().lock();
 
         try {
@@ -249,11 +250,12 @@ public void stop(boolean cleanFile) throws PersistentStorageIOException {
 
             fileIO = null;
 
-            if (cleanFile)
+            if (delete)
                 Files.delete(cfgFile.toPath());
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException(e);
+            throw new StorageException("Failed to stop serving partition file [file=" + cfgFile.getPath()
+                + ", delete=" + delete + "]", e);
         }
         finally {
             lock.writeLock().unlock();
@@ -264,9 +266,9 @@ public void stop(boolean cleanFile) throws PersistentStorageIOException {
      * Truncates and deletes partition file.
      *
      * @param tag New partition tag.
-     * @throws PersistentStorageIOException If failed
+     * @throws StorageException If failed in case of underlying I/O exception.
      */
-    public void truncate(int tag) throws PersistentStorageIOException {
+    public void truncate(int tag) throws StorageException {
         init();
 
         lock.writeLock().lock();
@@ -283,7 +285,7 @@ public void truncate(int tag) throws PersistentStorageIOException {
             Files.delete(cfgFile.toPath());
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Failed to delete partition file: " + cfgFile.getPath(), e);
+            throw new StorageException("Failed to truncate partition file [file=" + cfgFile.getPath() + "]", e);
         }
         finally {
             allocatedTracker.updateTotalAllocatedPages(-1L * allocated.get() / pageSize);
@@ -311,9 +313,9 @@ public void beginRecover() {
     }
 
     /**
-     *
+     * @throws StorageException If failed in case of underlying I/O exception.
      */
-    public void finishRecover() throws PersistentStorageIOException {
+    public void finishRecover() throws StorageException {
         lock.writeLock().lock();
 
         try {
@@ -332,7 +334,7 @@ public void finishRecover() throws PersistentStorageIOException {
             recover = false;
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Failed to finish recover", e);
+            throw new StorageException("Failed to finish recover partition file [file=" + cfgFile.getAbsolutePath() + "]", e);
         }
         finally {
             lock.writeLock().unlock();
@@ -394,7 +396,7 @@ public void finishRecover() throws PersistentStorageIOException {
                 PageIO.setCrc(pageBuf, savedCrc32);
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Read error", e);
+            throw new StorageException("Failed to read page [file=" + cfgFile.getAbsolutePath() + ", pageId=" + pageId + "]", e);
         }
     }
 
@@ -423,14 +425,14 @@ public void finishRecover() throws PersistentStorageIOException {
             while (len > 0);
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Read error", e);
+            throw new StorageException("Failed to read header [file=" + cfgFile.getAbsolutePath() + "]", e);
         }
     }
 
     /**
-     * @throws PersistentStorageIOException If failed to initialize store file.
+     * @throws StorageException If failed to initialize store file.
      */
-    private void init() throws PersistentStorageIOException {
+    private void init() throws StorageException {
         if (!inited) {
             lock.writeLock().lock();
 
@@ -438,7 +440,7 @@ private void init() throws PersistentStorageIOException {
                 if (!inited) {
                     FileIO fileIO = null;
 
-                    PersistentStorageIOException err = null;
+                    StorageException err = null;
 
                     long newSize;
 
@@ -472,8 +474,8 @@ private void init() throws PersistentStorageIOException {
                         inited = true;
                     }
                     catch (IOException e) {
-                        err = new PersistentStorageIOException(
-                            "Failed to initialize partition file: " + cfgFile.getName(), e);
+                        err = new StorageException(
+                            "Failed to initialize partition file: " + cfgFile.getAbsolutePath(), e);
 
                         throw err;
                     }
@@ -631,8 +633,8 @@ private void reinit(FileIO fileIO) throws IOException {
                     }
                 }
 
-                throw new PersistentStorageIOException("Failed to write the page to the file store [pageId=" + pageId
-                    + ", file=" + cfgFile.getAbsolutePath() + ']', e);
+                throw new StorageException("Failed to write page [file=" + cfgFile.getAbsolutePath()
+                    + ", pageId=" + pageId + ", tag=" + tag + "]", e);
             }
         }
     }
@@ -658,7 +660,7 @@ private static int calcCrc32(ByteBuffer pageBuf, int pageSize) {
     }
 
     /** {@inheritDoc} */
-    @Override public void sync() throws IgniteCheckedException {
+    @Override public void sync() throws StorageException {
         lock.writeLock().lock();
 
         try {
@@ -670,7 +672,7 @@ private static int calcCrc32(ByteBuffer pageBuf, int pageSize) {
                 fileIO.force();
         }
         catch (IOException e) {
-            throw new PersistentStorageIOException("Sync error", e);
+            throw new StorageException("Failed to fsync partition file [file=" + cfgFile.getAbsolutePath() + "]", e);
         }
         finally {
             lock.writeLock().unlock();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index e611303d3c618..cbec04a5ff3e4 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
 import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
@@ -259,7 +260,7 @@ public FilePageStoreManager(GridKernalContext ctx) {
                     partStore.finishRecover();
             }
         }
-        catch (PersistentStorageIOException e) {
+        catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
@@ -267,8 +268,7 @@ public FilePageStoreManager(GridKernalContext ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cacheData)
-        throws IgniteCheckedException {
+    @Override public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cacheData) throws IgniteCheckedException {
         int grpId = grpDesc.groupId();
 
         if (!idxCacheStores.containsKey(grpId)) {
@@ -281,8 +281,7 @@ public FilePageStoreManager(GridKernalContext ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public void initializeForMetastorage()
-        throws IgniteCheckedException {
+    @Override public void initializeForMetastorage() throws IgniteCheckedException {
         int grpId = MetaStorage.METASTORAGE_CACHE_ID;
 
         if (!idxCacheStores.containsKey(grpId)) {
@@ -388,7 +387,7 @@ public void read(int cacheId, long pageId, ByteBuffer pageBuf, boolean keepCrc)
         try {
             store.read(pageId, pageBuf, keepCrc);
         }
-        catch (PersistentStorageIOException e) {
+        catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
@@ -409,7 +408,7 @@ public void read(int cacheId, long pageId, ByteBuffer pageBuf, boolean keepCrc)
         try {
             store.readHeader(buf);
         }
-        catch (PersistentStorageIOException e) {
+        catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
@@ -445,7 +444,7 @@ public PageStore writeInternal(int cacheId, long pageId, ByteBuffer pageBuf, int
         try {
             store.write(pageId, pageBuf, tag, calculateCrc);
         }
-        catch (PersistentStorageIOException e) {
+        catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
@@ -531,7 +530,7 @@ private CacheStoreHolder initDir(File cacheWorkDir,
 
             return new CacheStoreHolder(idxStore, partStores);
         }
-        catch (PersistentStorageIOException e) {
+        catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
@@ -623,7 +622,7 @@ else if (lockF.exists()) {
         try {
             getStore(grpId, partId).sync();
         }
-        catch (PersistentStorageIOException e) {
+        catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
@@ -635,7 +634,7 @@ else if (lockF.exists()) {
         try {
             getStore(grpId, partId).ensure();
         }
-        catch (PersistentStorageIOException e) {
+        catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
@@ -653,7 +652,7 @@ else if (lockF.exists()) {
 
             return PageIdUtils.pageId(partId, flags, (int)pageIdx);
         }
-        catch (PersistentStorageIOException e) {
+        catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index 4a2549b51f8e8..c0fba7308467b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -29,7 +29,7 @@
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
index ed0d30422849b..af204dd70f4dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
@@ -23,7 +23,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
 import org.jetbrains.annotations.Nullable;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 109c26fca1861..0f3a31cb1e78b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -52,7 +52,7 @@
 import org.apache.ignite.internal.pagemem.PageUtils;
 import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
index 38e91bcf949fd..cae19edab17ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandlerWrapper;
 import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.apache.ignite.internal.util.GridArrays;
 import org.apache.ignite.internal.util.GridLongList;
@@ -88,6 +89,9 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
     /** */
     private static final Object[] EMPTY = {};
 
+    /** Wrapper for tree pages operations. Noop by default. Override for test purposes. */
+    public static volatile PageHandlerWrapper<Result> pageHndWrapper = (tree, hnd) -> hnd;
+
     /** */
     private static volatile boolean interrupted;
 
@@ -226,7 +230,7 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements
     };
 
     /** */
-    private final GetPageHandler<Get> askNeighbor = new AskNeighbor();
+    private final PageHandler<Get, Result> askNeighbor;
 
     /**
      *
@@ -257,12 +261,12 @@ private class AskNeighbor extends GetPageHandler<Get> {
     }
 
     /** */
-    private final GetPageHandler<Get> search = new Search();
+    private final PageHandler<Get, Result> search;
 
     /**
      *
      */
-    private class Search extends GetPageHandler<Get> {
+    public class Search extends GetPageHandler<Get> {
         /** {@inheritDoc} */
         @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Get g, int lvl)
             throws IgniteCheckedException {
@@ -346,12 +350,12 @@ else if (needBackIfRouting) {
     }
 
     /** */
-    private final GetPageHandler<Put> replace = new Replace();
+    private final PageHandler<Put, Result> replace;
 
     /**
      *
      */
-    private class Replace extends GetPageHandler<Put> {
+    public class Replace extends GetPageHandler<Put> {
         /** {@inheritDoc} */
         @SuppressWarnings("unchecked")
         @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Put p, int lvl)
@@ -408,12 +412,12 @@ private class Replace extends GetPageHandler<Put> {
     }
 
     /** */
-    private final GetPageHandler<Put> insert = new Insert();
+    private final PageHandler<Put, Result> insert;
 
     /**
      *
      */
-    private class Insert extends GetPageHandler<Put> {
+    public class Insert extends GetPageHandler<Put> {
         /** {@inheritDoc} */
         @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Put p, int lvl)
             throws IgniteCheckedException {
@@ -457,7 +461,7 @@ private class Insert extends GetPageHandler<Put> {
     }
 
     /** */
-    private final GetPageHandler<Remove> rmvFromLeaf = new RemoveFromLeaf();
+    private final PageHandler<Remove, Result> rmvFromLeaf;
 
     /**
      *
@@ -532,7 +536,7 @@ private class RemoveFromLeaf extends GetPageHandler<Remove> {
     }
 
     /** */
-    private final GetPageHandler<Remove> lockBackAndRmvFromLeaf = new LockBackAndRmvFromLeaf();
+    private final PageHandler<Remove, Result> lockBackAndRmvFromLeaf;
 
     /**
      *
@@ -557,7 +561,7 @@ private class LockBackAndRmvFromLeaf extends GetPageHandler<Remove> {
     }
 
     /** */
-    private final GetPageHandler<Remove> lockBackAndTail = new LockBackAndTail();
+    private final PageHandler<Remove, Result> lockBackAndTail;
 
     /**
      *
@@ -581,7 +585,7 @@ private class LockBackAndTail extends GetPageHandler<Remove> {
     }
 
     /** */
-    private final GetPageHandler<Remove> lockTailForward = new LockTailForward();
+    private final PageHandler<Remove, Result> lockTailForward;
 
     /**
      *
@@ -597,7 +601,7 @@ private class LockTailForward extends GetPageHandler<Remove> {
     }
 
     /** */
-    private final GetPageHandler<Remove> lockTail = new LockTail();
+    private final PageHandler<Remove, Result> lockTail;
 
     /**
      *
@@ -782,6 +786,17 @@ protected BPlusTree(
         this.reuseList = reuseList;
         this.globalRmvId = globalRmvId;
         this.failureProcessor = failureProcessor;
+
+        // Initialize page handlers.
+        askNeighbor = (PageHandler<Get, Result>) pageHndWrapper.wrap(this, new AskNeighbor());
+        search = (PageHandler<Get, Result>) pageHndWrapper.wrap(this, new Search());
+        lockTail = (PageHandler<Remove, Result>) pageHndWrapper.wrap(this, new LockTail());
+        lockTailForward = (PageHandler<Remove, Result>) pageHndWrapper.wrap(this, new LockTailForward());
+        lockBackAndTail = (PageHandler<Remove, Result>) pageHndWrapper.wrap(this, new LockBackAndTail());
+        lockBackAndRmvFromLeaf = (PageHandler<Remove, Result>) pageHndWrapper.wrap(this, new LockBackAndRmvFromLeaf());
+        rmvFromLeaf = (PageHandler<Remove, Result>) pageHndWrapper.wrap(this, new RemoveFromLeaf());
+        insert = (PageHandler<Put, Result>) pageHndWrapper.wrap(this, new Insert());
+        replace = (PageHandler<Put, Result>) pageHndWrapper.wrap(this, new Replace());
     }
 
     /**
@@ -979,11 +994,8 @@ public final GridCursor<T> find(L lower, L upper, Object x) throws IgniteChecked
         catch (IgniteCheckedException e) {
             throw new IgniteCheckedException("Runtime failure on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
         }
-        catch (RuntimeException e) {
-            throw new IgniteException("Runtime failure on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
-        }
-        catch (AssertionError e) {
-            throw new AssertionError("Assertion error on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
+        catch (RuntimeException | AssertionError e) {
+            throw new CorruptedTreeException("Runtime failure on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
         }
         finally {
             checkDestroyed();
@@ -1031,11 +1043,8 @@ public final GridCursor<T> find(L lower, L upper, Object x) throws IgniteChecked
         catch (IgniteCheckedException e) {
             throw new IgniteCheckedException("Runtime failure on first row lookup", e);
         }
-        catch (RuntimeException e) {
-            throw new IgniteException("Runtime failure on first row lookup", e);
-        }
-        catch (AssertionError e) {
-            throw new AssertionError("Assertion error on first row lookup", e);
+        catch (RuntimeException | AssertionError e) {
+            throw new CorruptedTreeException("Runtime failure on first row lookup", e);
         }
         finally {
             checkDestroyed();
@@ -1056,12 +1065,9 @@ public final GridCursor<T> find(L lower, L upper, Object x) throws IgniteChecked
         catch (IgniteCheckedException e) {
             throw new IgniteCheckedException("Runtime failure on last row lookup", e);
         }
-        catch (RuntimeException e) {
+        catch (RuntimeException | AssertionError e) {
             throw new IgniteException("Runtime failure on last row lookup", e);
         }
-        catch (AssertionError e) {
-            throw new AssertionError("Assertion error on last row lookup", e);
-        }
         finally {
             checkDestroyed();
         }
@@ -1087,11 +1093,8 @@ public final <R> R findOne(L row, Object x) throws IgniteCheckedException {
         catch (IgniteCheckedException e) {
             throw new IgniteCheckedException("Runtime failure on lookup row: " + row, e);
         }
-        catch (RuntimeException e) {
-            throw new IgniteException("Runtime failure on lookup row: " + row, e);
-        }
-        catch (AssertionError e) {
-            throw new AssertionError("Assertion error on lookup row: " + row, e);
+        catch (RuntimeException | AssertionError e) {
+            throw new CorruptedTreeException("Runtime failure on lookup row: " + row, e);
         }
         finally {
             checkDestroyed();
@@ -1648,11 +1651,8 @@ public final boolean removex(L row) throws IgniteCheckedException {
         catch (IgniteCheckedException e) {
             throw new IgniteCheckedException("Runtime failure on search row: " + row, e);
         }
-        catch (RuntimeException e) {
-            throw new IgniteException("Runtime failure on search row: " + row, e);
-        }
-        catch (AssertionError e) {
-            throw new AssertionError("Assertion error on search row: " + row, e);
+        catch (RuntimeException | AssertionError e) {
+            throw new CorruptedTreeException("Runtime failure on search row: " + row, e);
         }
         finally {
             x.releaseAll();
@@ -1808,11 +1808,8 @@ private T doRemove(L row, boolean needOld) throws IgniteCheckedException {
         catch (IgniteCheckedException e) {
             throw new IgniteCheckedException("Runtime failure on search row: " + row, e);
         }
-        catch (RuntimeException e) {
-            throw new IgniteException("Runtime failure on search row: " + row, e);
-        }
-        catch (AssertionError e) {
-            throw new AssertionError("Assertion error on search row: " + row, e);
+        catch (RuntimeException | AssertionError e) {
+            throw new CorruptedTreeException("Runtime failure on search row: " + row, e);
         }
         finally {
             r.releaseAll();
@@ -2127,11 +2124,8 @@ private T doPut(T row, boolean needOld) throws IgniteCheckedException {
         catch (IgniteCheckedException e) {
             throw new IgniteCheckedException("Runtime failure on row: " + row, e);
         }
-        catch (RuntimeException e) {
-            throw new IgniteException("Runtime failure on row: " + row, e);
-        }
-        catch (AssertionError e) {
-            throw new AssertionError("Assertion error on row: " + row, e);
+        catch (RuntimeException | AssertionError e) {
+            throw new CorruptedTreeException("Runtime failure on row: " + row, e);
         }
         finally {
             checkDestroyed();
@@ -2422,7 +2416,7 @@ private long doAskNeighbor(BPlusIO<L> io, long pageAddr, boolean back) {
     /**
      * Get operation.
      */
-    private abstract class Get {
+    public abstract class Get {
         /** */
         long rmvId;
 
@@ -2537,7 +2531,7 @@ boolean notFound(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCh
          * @param lvl Level.
          * @return {@code true} If we can release the given page.
          */
-        boolean canRelease(long pageId, int lvl) {
+        public boolean canRelease(long pageId, int lvl) {
             return pageId != 0L;
         }
 
@@ -2587,6 +2581,13 @@ void checkLockRetry() throws IgniteCheckedException {
 
             lockRetriesCnt--;
         }
+
+        /**
+         * @return Operation row.
+         */
+        public L row() {
+            return row;
+        }
     }
 
     /**
@@ -2660,11 +2661,13 @@ private final class GetCursor extends Get {
     /**
      * Put operation.
      */
-    private final class Put extends Get {
+    public final class Put extends Get {
         /** Mark of NULL value of page id. It means valid value can't be equal this value. */
         private static final long NULL_PAGE_ID = 0L;
+
         /** Mark of NULL value of page. */
         private static final long NULL_PAGE = 0L;
+
         /** Mark of NULL value of page address. */
         private static final long NULL_PAGE_ADDRESS = 0L;
 
@@ -2749,7 +2752,7 @@ private void tail(long tailId, long tailPage, long tailPageAddr) {
         }
 
         /** {@inheritDoc} */
-        @Override boolean canRelease(long pageId, int lvl) {
+        @Override public boolean canRelease(long pageId, int lvl) {
             return pageId != NULL_PAGE_ID && tailId != pageId;
         }
 
@@ -3008,7 +3011,7 @@ public Result tryReplace(long pageId, long page, long fwdId, int lvl) throws Ign
     /**
      * Invoke operation.
      */
-    private final class Invoke extends Get {
+    public final class Invoke extends Get {
         /** */
         Object x;
 
@@ -3146,7 +3149,7 @@ private void invokeClosure() throws IgniteCheckedException {
         }
 
         /** {@inheritDoc} */
-        @Override boolean canRelease(long pageId, int lvl) {
+        @Override public boolean canRelease(long pageId, int lvl) {
             if (pageId == 0L)
                 return false;
 
@@ -4123,7 +4126,7 @@ private void doReleaseTail(Tail<L> t) {
         }
 
         /** {@inheritDoc} */
-        @Override boolean canRelease(long pageId, int lvl) {
+        @Override public boolean canRelease(long pageId, int lvl) {
             return pageId != 0L && !isTail(pageId, lvl);
         }
 
@@ -4922,7 +4925,7 @@ private static class TreeMetaData {
     /**
      * Operation result.
      */
-    enum Result {
+    public enum Result {
         /** */
         GO_DOWN,
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/PersistentStorageIOException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/CorruptedTreeException.java
similarity index 65%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/PersistentStorageIOException.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/CorruptedTreeException.java
index 7b3c30371b9dd..a6bfb1ffd5bc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/PersistentStorageIOException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/CorruptedTreeException.java
@@ -14,34 +14,25 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.ignite.internal.processors.cache.persistence.file;
 
-import java.io.IOException;
+package org.apache.ignite.internal.processors.cache.persistence.tree;
+
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.InvalidEnvironmentException;
+import org.jetbrains.annotations.Nullable;
 
 /**
- * Exception is needed to distinguish persistent storage I/O errors.
+ * Exception to distinguish {@link BPlusTree} tree broken invariants.
  */
-public class PersistentStorageIOException extends IgniteCheckedException {
+public class CorruptedTreeException extends IgniteCheckedException implements InvalidEnvironmentException {
     /** */
     private static final long serialVersionUID = 0L;
 
     /**
-     * Create an instance of exception.
-     *
-     * @param cause Error cause.
-     */
-    public PersistentStorageIOException(IOException cause) {
-        super(cause);
-    }
-
-    /**
-     * Create an instance of exception.
-     *
-     * @param msg Error message.
-     * @param cause Error cause.
+     * @param msg Message.
+     * @param cause Cause.
      */
-    public PersistentStorageIOException(String msg, IOException cause) {
+    public CorruptedTreeException(String msg, @Nullable Throwable cause) {
         super(msg, cause);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandlerWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandlerWrapper.java
new file mode 100644
index 0000000000000..495eba05dd047
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandlerWrapper.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.util;
+
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+
+/**
+ * Wrapper factory for {@link PageHandler} instances.
+ *
+ * @param <R> Result type of actual {@link PageHandler} class.
+ */
+public interface PageHandlerWrapper<R> {
+    /**
+     * Wraps given {@code hnd}.
+     *
+     * @param tree Instance of {@link BPlusTree} where given {@code} is used.
+     * @param hnd Page handler to wrap.
+     * @return Wrapped version of given {@code hnd}.
+     */
+    public PageHandler<?, R> wrap(BPlusTree<?, ?> tree, PageHandler<?, R> hnd);
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 31f7383c1191d..27ef23248582d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -77,7 +77,7 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 7521f73190c4e..df2878f7645fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -72,7 +72,7 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 3cf1146e80657..d80604540253a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -437,36 +437,27 @@ public void storeEnabled(boolean storeEnabled) {
 
     /**
      * Uncommits transaction by invalidating all of its entries. Courtesy to minimize inconsistency.
-     *
-     * @param nodeStopping {@code True} if tx was cancelled during node stop.
      */
     @SuppressWarnings({"CatchGenericClass"})
-    protected void uncommit(boolean nodeStopping) {
-        try {
-            if (!nodeStopping) {
-                for (IgniteTxEntry e : writeMap().values()) {
-                    try {
-                        GridCacheEntryEx entry = e.cached();
-
-                        if (e.op() != NOOP)
-                            entry.invalidate(xidVer);
-                    }
-                    catch (Throwable t) {
-                        U.error(log, "Failed to invalidate transaction entries while reverting a commit.", t);
+    protected void uncommit() {
+        for (IgniteTxEntry e : writeMap().values()) {
+            try {
+                GridCacheEntryEx entry = e.cached();
 
-                        if (t instanceof Error)
-                            throw (Error)t;
+                if (e.op() != NOOP)
+                    entry.invalidate(xidVer);
+            }
+            catch (Throwable t) {
+                U.error(log, "Failed to invalidate transaction entries while reverting a commit.", t);
 
-                        break;
-                    }
-                }
+                if (t instanceof Error)
+                    throw (Error)t;
 
-                cctx.tm().uncommitTx(this);
+                break;
             }
         }
-        catch (Exception ex) {
-            U.error(log, "Failed to do uncommit.", ex);
-        }
+
+        cctx.tm().uncommitTx(this);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 6f11a57500ec2..b2408286f7695 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -30,9 +30,12 @@
 import javax.cache.processor.EntryProcessor;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.InvalidEnvironmentException;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.NodeStoppingException;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
@@ -46,7 +49,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
 import org.apache.ignite.internal.processors.cache.GridCacheFilterFailedException;
-import org.apache.ignite.internal.processors.cache.GridCacheIndexUpdateException;
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.GridCacheReturn;
@@ -833,39 +835,34 @@ assert ownsLock(txEntry.cached()):
                         // Need to remove version from committed list.
                         cctx.tm().removeCommittedTx(this);
 
-                        if (X.hasCause(ex, GridCacheIndexUpdateException.class) && cacheCtx.cache().isMongoDataCache()) {
-                            if (log.isDebugEnabled())
-                                log.debug("Failed to update mongo document index (transaction entry will " +
-                                    "be ignored): " + txEntry);
+                        boolean isNodeStopping = X.hasCause(ex, NodeStoppingException.class);
+                        boolean hasInvalidEnvironmentIssue = X.hasCause(ex, InvalidEnvironmentException.class);
 
-                            // Set operation to NOOP.
-                            txEntry.op(NOOP);
+                        IgniteCheckedException err = new IgniteTxHeuristicCheckedException("Failed to locally write to cache " +
+                            "(all transaction entries will be invalidated, however there was a window when " +
+                            "entries for this transaction were visible to others): " + this, ex);
 
-                            errorWhenCommitting();
-
-                            throw ex;
+                        if (isNodeStopping) {
+                            U.warn(log, "Failed to commit transaction, node is stopping [tx=" + this +
+                                ", err=" + ex + ']');
                         }
-                        else {
-                            boolean hasInvalidEnvironmentIssue = X.hasCause(ex, InvalidEnvironmentException.class);
-
-                            IgniteCheckedException err = new IgniteTxHeuristicCheckedException("Failed to locally write to cache " +
-                                "(all transaction entries will be invalidated, however there was a window when " +
-                                "entries for this transaction were visible to others): " + this, ex);
-
-                            if (hasInvalidEnvironmentIssue) {
-                                U.warn(log, "Failed to commit transaction, node is stopping " +
-                                    "[tx=" + this + ", err=" + ex + ']');
-                            }
-                            else
-                                U.error(log, "Heuristic transaction failure.", err);
+                        else if (hasInvalidEnvironmentIssue) {
+                            U.warn(log, "Failed to commit transaction, node is in invalid state and will be stopped [tx=" + this +
+                                ", err=" + ex + ']');
+                        }
+                        else
+                            U.error(log, "Commit failed.", err);
 
-                            COMMIT_ERR_UPD.compareAndSet(this, null, err);
+                        COMMIT_ERR_UPD.compareAndSet(this, null, err);
 
-                            state(UNKNOWN);
+                        state(UNKNOWN);
 
+                        if (hasInvalidEnvironmentIssue)
+                            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
+                        else if (!isNodeStopping) { // Skip fair uncommit in case of node stopping or invalidation.
                             try {
                                 // Courtesy to minimize damage.
-                                uncommit(hasInvalidEnvironmentIssue);
+                                uncommit();
                             }
                             catch (Throwable ex1) {
                                 U.error(log, "Failed to uncommit transaction: " + this, ex1);
@@ -873,12 +870,12 @@ assert ownsLock(txEntry.cached()):
                                 if (ex1 instanceof Error)
                                     throw ex1;
                             }
+                        }
 
-                            if (ex instanceof Error)
-                                throw ex;
+                        if (ex instanceof Error)
+                            throw ex;
 
-                            throw err;
-                        }
+                        throw err;
                     }
                 }
 
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index cc571b482723d..8979679ebf7d5 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -384,7 +384,7 @@ org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager$1
 org.apache.ignite.internal.marshaller.optimized.OptimizedFieldType
 org.apache.ignite.internal.mem.IgniteOutOfMemoryException
 org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl$Segment
-org.apache.ignite.internal.pagemem.wal.StorageException
+org.apache.ignite.internal.processors.cache.persistence.StorageException
 org.apache.ignite.internal.pagemem.wal.WALIterator
 org.apache.ignite.internal.pagemem.wal.WALPointer
 org.apache.ignite.internal.pagemem.wal.record.ExchangeRecord$Type
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/AccountTransferTransactionTest.java b/modules/core/src/test/java/org/apache/ignite/failure/AccountTransferTransactionTest.java
deleted file mode 100644
index 8d7cf15608ed2..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/failure/AccountTransferTransactionTest.java
+++ /dev/null
@@ -1,331 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.failure;
-
-import javax.management.MBeanServer;
-import javax.management.MBeanServerInvocationHandler;
-import javax.management.ObjectName;
-import java.lang.management.ManagementFactory;
-import java.util.ArrayList;
-import java.util.Random;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.DataRegionConfiguration;
-import org.apache.ignite.configuration.DataStorageConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.TestRecordingCommunicationSpi;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.internal.worker.WorkersControlMXBeanImpl;
-import org.apache.ignite.mxbean.WorkersControlMXBean;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.transactions.Transaction;
-import org.jetbrains.annotations.NotNull;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
-import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
-import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
-import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
-
-/**
- * Test transfer amount between accounts with enabled {@link StopNodeFailureHandler}.
- */
-public class AccountTransferTransactionTest extends GridCommonAbstractTest {
-    /** */
-    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-    /** Count of accounts in one thread. */
-    private static final int ACCOUNTS_CNT = 20;
-    /** Count of threads and caches. */
-    private static final int THREADS_CNT = 20;
-    /** Count of nodes to start. */
-    private static final int NODES_CNT = 3;
-    /** Count of transaction on cache. */
-    private static final int TRANSACTION_CNT = 10;
-
-    /** {@inheritDoc} */
-    @Override protected FailureHandler getFailureHandler(String igniteInstanceName) {
-        return new StopNodeFailureHandler();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
-        final IgniteConfiguration cfg = super.getConfiguration(name);
-
-        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
-        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
-        cfg.setLocalHost("127.0.0.1");
-        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
-            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
-                .setMaxSize(50 * 1024 * 1024)
-                .setPersistenceEnabled(true))
-        );
-
-        CacheConfiguration[] cacheConfigurations = new CacheConfiguration[THREADS_CNT];
-        for (int i = 0; i < THREADS_CNT; i++) {
-            cacheConfigurations[i] = new CacheConfiguration()
-                .setName(cacheName(i))
-                .setAffinity(new RendezvousAffinityFunction(false, 32))
-                .setBackups(1)
-                .setAtomicityMode(TRANSACTIONAL)
-                .setCacheMode(CacheMode.PARTITIONED)
-                .setWriteSynchronizationMode(FULL_SYNC)
-                .setEvictionPolicy(new FifoEvictionPolicy(1000))
-                .setOnheapCacheEnabled(true);
-        }
-
-        cfg.setCacheConfiguration(cacheConfigurations);
-
-        return cfg;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTest() throws Exception {
-        super.beforeTest();
-
-        stopAllGrids();
-
-        cleanPersistenceDir();
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        super.afterTest();
-
-        stopAllGrids();
-
-        cleanPersistenceDir();
-    }
-
-    /**
-     * Test transfer amount.
-     */
-    public void testTransferAmount() throws Exception {
-        //given: started some nodes with client.
-        startGrids(NODES_CNT);
-
-        IgniteEx igniteClient = startGrid(getClientConfiguration(NODES_CNT));
-
-        igniteClient.cluster().active(true);
-
-        Random random = new Random();
-
-        long[] initAmount = new long[THREADS_CNT];
-
-        //and: fill all accounts on all caches and calculate total amount for every cache.
-        for (int cachePrefixIdx = 0; cachePrefixIdx < THREADS_CNT; cachePrefixIdx++) {
-            IgniteCache<Object, Object> cache = igniteClient.getOrCreateCache(cacheName(cachePrefixIdx));
-
-            try (Transaction tx = igniteClient.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
-                for (int accountId = 0; accountId < ACCOUNTS_CNT; accountId++) {
-                    Long amount = (long)random.nextInt(1000);
-
-                    cache.put(accountId, amount);
-
-                    initAmount[cachePrefixIdx] += amount;
-                }
-
-                tx.commit();
-            }
-        }
-
-        //when: start transfer amount from account to account in different threads.
-        CountDownLatch firstTransactionDone = new CountDownLatch(THREADS_CNT);
-
-        ArrayList<Thread> transferThreads = new ArrayList<>();
-
-        for (int i = 0; i < THREADS_CNT; i++) {
-            transferThreads.add(new TransferAmountTxThread(firstTransactionDone, igniteClient, cacheName(i)));
-
-            transferThreads.get(i).start();
-        }
-
-        firstTransactionDone.await(10, TimeUnit.SECONDS);
-
-        //and: terminate disco-event-worker thread on one node.
-        WorkersControlMXBean bean = workersMXBean(1);
-
-        bean.terminateWorker(
-            bean.getWorkerNames().stream()
-                .filter(name -> name.startsWith("disco-event-worker"))
-                .findFirst()
-                .orElse(null)
-        );
-
-        for (Thread thread : transferThreads) {
-            thread.join();
-        }
-
-        long[] resultAmount = new long[THREADS_CNT];
-
-        //then: calculate total amount for every thread.
-        for (int j = 0; j < THREADS_CNT; j++) {
-            String cacheName = cacheName(j);
-
-            IgniteCache<Object, Object> cache = igniteClient.getOrCreateCache(cacheName);
-
-            try (Transaction tx = igniteClient.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
-
-                for (int i = 0; i < ACCOUNTS_CNT; i++)
-                    resultAmount[j] += getNotNullValue(cache, i);
-                tx.commit();
-            }
-
-            long diffAmount = initAmount[j] - resultAmount[j];
-
-            //and: check that result amount equal to init amount.
-            assertTrue(
-                String.format("Total amount before and after transfer is not same: diff=%s, cache=%s",
-                    diffAmount, cacheName),
-                diffAmount == 0
-            );
-        }
-    }
-
-    /**
-     * Make test cache name by prefix.
-     */
-    @NotNull private String cacheName(int cachePrefixIdx) {
-        return "cache" + cachePrefixIdx;
-    }
-
-    /**
-     * Ignite configuration for client.
-     */
-    @NotNull private IgniteConfiguration getClientConfiguration(int nodesPrefix) throws Exception {
-        IgniteConfiguration clientConf = getConfiguration(getTestIgniteInstanceName(nodesPrefix));
-
-        clientConf.setClientMode(true);
-
-        return clientConf;
-    }
-
-    /**
-     * Extract not null value from cache.
-     */
-    private long getNotNullValue(IgniteCache<Object, Object> cache, int i) {
-        Object value = cache.get(i);
-
-        return value == null ? 0 : ((Long)value);
-    }
-
-    /**
-     * Configure workers mx bean.
-     */
-    private WorkersControlMXBean workersMXBean(int igniteInt) throws Exception {
-        ObjectName mbeanName = U.makeMBeanName(
-            getTestIgniteInstanceName(igniteInt),
-            "Kernal",
-            WorkersControlMXBeanImpl.class.getSimpleName()
-        );
-
-        MBeanServer mbeanSrv = ManagementFactory.getPlatformMBeanServer();
-
-        if (!mbeanSrv.isRegistered(mbeanName))
-            fail("MBean is not registered: " + mbeanName.getCanonicalName());
-
-        return MBeanServerInvocationHandler.newProxyInstance(mbeanSrv, mbeanName, WorkersControlMXBean.class, true);
-    }
-
-    /**
-     *
-     */
-    private static class TransferAmountTxThread extends Thread {
-        /** */
-        private CountDownLatch firstTransactionLatch;
-        /** */
-        private Ignite ignite;
-        /** */
-        private String cacheName;
-        /** */
-        private Random random = new Random();
-
-        /**
-         * @param ignite Ignite.
-         */
-        private TransferAmountTxThread(CountDownLatch firstTransactionLatch, final Ignite ignite, String cacheName) {
-            this.firstTransactionLatch = firstTransactionLatch;
-            this.ignite = ignite;
-            this.cacheName = cacheName;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void run() {
-            for (int i = 0; i < TRANSACTION_CNT; i++) {
-                try {
-                    updateInTransaction(ignite.cache(cacheName));
-                }
-                finally {
-                    if (i == 0)
-                        firstTransactionLatch.countDown();
-                }
-            }
-        }
-
-        /**
-         * @throws IgniteException if fails
-         */
-        @SuppressWarnings("unchecked")
-        private void updateInTransaction(IgniteCache cache) throws IgniteException {
-            int accIdFrom = random.nextInt(ACCOUNTS_CNT);
-            int accIdTo = random.nextInt(ACCOUNTS_CNT);
-
-            if (accIdFrom == accIdTo)
-                accIdTo = (int)getNextAccountId(accIdFrom);
-
-            Long acctFrom;
-            Long acctTo;
-
-            try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
-                acctFrom = (Long)cache.get(accIdFrom);
-                acctTo = (Long)cache.get(accIdTo);
-
-                long transactionAmount = (long)(random.nextDouble() * acctFrom);
-
-                cache.put(accIdFrom, acctFrom - transactionAmount);
-                cache.put(accIdTo, acctTo + transactionAmount);
-
-                tx.commit();
-            }
-        }
-
-        /**
-         * @param curr current
-         * @return random value
-         */
-        private long getNextAccountId(long curr) {
-            long randomVal;
-
-            do {
-                randomVal = random.nextInt(ACCOUNTS_CNT);
-            }
-            while (curr == randomVal);
-
-            return randomVal;
-        }
-    }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
index ff95f97a1fc78..fda25e65d5da3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
@@ -37,10 +37,8 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
-import org.apache.ignite.internal.processors.cache.persistence.file.PersistentStorageIOException;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
index d30289682cbba..2ea269d3f053e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
@@ -32,7 +32,7 @@
 import org.apache.ignite.failure.FailureHandler;
 import org.apache.ignite.failure.TestFailureHandler;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index 0a240ea6ae36f..cdad57555a382 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -20,7 +20,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java
new file mode 100644
index 0000000000000..fe27e6e119d24
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java
@@ -0,0 +1,595 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import com.google.common.collect.Sets;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.jetbrains.annotations.NotNull;
+import org.jsr166.ConcurrentLinkedHashMap;
+import org.junit.Assert;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Test transfer amount between accounts with enabled {@link StopNodeFailureHandler}.
+ *
+ * This test can be extended to emulate failover scenarios during transactional operations on the grid.
+ */
+public class AbstractTransactionIntergrityTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** Count of accounts in one thread. */
+    private static final int DFLT_ACCOUNTS_CNT = 32;
+
+    /** Count of threads and caches. */
+    private static final int DFLT_TX_THREADS_CNT = 20;
+
+    /** Count of nodes to start. */
+    private static final int DFLT_NODES_CNT = 3;
+
+    /** Count of transaction on cache. */
+    private static final int DFLT_TRANSACTIONS_CNT = 10;
+
+    /** Completed transactions map. */
+    private ConcurrentLinkedHashMap[] completedTxs;
+
+    /**
+     *
+     */
+    protected int nodesCount() {
+        return DFLT_NODES_CNT;
+    }
+
+    /**
+     *
+     */
+    protected int accountsCount() {
+        return DFLT_ACCOUNTS_CNT;
+    }
+
+    /**
+     *
+     */
+    protected int transactionsCount() {
+        return DFLT_TRANSACTIONS_CNT;
+    }
+
+    /**
+     *
+     */
+    protected int txThreadsCount() {
+        return DFLT_TX_THREADS_CNT;
+    }
+
+    /**
+     * @return Flag enables secondary index on account caches.
+     */
+    protected boolean indexed() {
+        return false;
+    }
+
+    /**
+     * @return Flag enables persistence on account caches.
+     */
+    protected boolean persistent() {
+        return true;
+    }
+
+    /**
+     * @return Flag enables cross-node transactions,
+     *         when primary partitions participating in transaction spreaded across several cluster nodes.
+     */
+    protected boolean crossNodeTransactions() {
+        // Commit error during cross node transactions breaks transaction integrity
+        // TODO: https://issues.apache.org/jira/browse/IGNITE-9086
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected FailureHandler getFailureHandler(String igniteInstanceName) {
+        return new StopNodeFailureHandler();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        final IgniteConfiguration cfg = super.getConfiguration(name);
+
+        cfg.setConsistentId(name);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+        cfg.setLocalHost("127.0.0.1");
+
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(256 * 1024 * 1024)
+                .setPersistenceEnabled(persistent()))
+        );
+
+        CacheConfiguration[] cacheConfigurations = new CacheConfiguration[txThreadsCount()];
+
+        for (int i = 0; i < txThreadsCount(); i++) {
+            CacheConfiguration ccfg = new CacheConfiguration()
+                .setName(cacheName(i))
+                .setAffinity(new RendezvousAffinityFunction(false, accountsCount()))
+                .setBackups(1)
+                .setAtomicityMode(TRANSACTIONAL)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setWriteSynchronizationMode(FULL_SYNC)
+                .setReadFromBackup(true)
+                .setOnheapCacheEnabled(true);
+
+            if (indexed())
+                ccfg.setIndexedTypes(IgniteUuid.class, AccountState.class);
+
+            cacheConfigurations[i] = ccfg;
+        }
+
+        cfg.setCacheConfiguration(cacheConfigurations);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Make test cache name by prefix.
+     */
+    @NotNull private String cacheName(int cachePrefixIdx) {
+        return "cache" + cachePrefixIdx;
+    }
+
+    /**
+     * Ignite configuration for client.
+     */
+    @NotNull private IgniteConfiguration getClientConfiguration(int nodesPrefix) throws Exception {
+        IgniteConfiguration clientConf = getConfiguration(getTestIgniteInstanceName(nodesPrefix));
+
+        clientConf.setClientMode(true);
+
+        return clientConf;
+    }
+
+    /**
+     * Test transfer amount.
+     */
+    public void doTestTransferAmount(FailoverScenario failoverScenario) throws Exception {
+        failoverScenario.beforeNodesStarted();
+
+        //given: started some nodes with client.
+        startGrids(nodesCount());
+
+        IgniteEx igniteClient = startGrid(getClientConfiguration(nodesCount()));
+
+        igniteClient.cluster().active(true);
+
+        int[] initAmount = new int[txThreadsCount()];
+        completedTxs = new ConcurrentLinkedHashMap[txThreadsCount()];
+
+        //and: fill all accounts on all caches and calculate total amount for every cache.
+        for (int cachePrefixIdx = 0; cachePrefixIdx < txThreadsCount(); cachePrefixIdx++) {
+            IgniteCache<Integer, AccountState> cache = igniteClient.getOrCreateCache(cacheName(cachePrefixIdx));
+
+            AtomicInteger coinsCounter = new AtomicInteger();
+
+            try (Transaction tx = igniteClient.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                for (int accountId = 0; accountId < accountsCount(); accountId++) {
+                    Set<Integer> initialAmount = generateCoins(coinsCounter, 5);
+
+                    cache.put(accountId, new AccountState(accountId, tx.xid(), initialAmount));
+                }
+
+                tx.commit();
+            }
+
+            initAmount[cachePrefixIdx] = coinsCounter.get();
+            completedTxs[cachePrefixIdx] = new ConcurrentLinkedHashMap();
+        }
+
+        //when: start transfer amount from account to account in different threads.
+        CountDownLatch firstTransactionDone = new CountDownLatch(txThreadsCount());
+
+        ArrayList<Thread> transferThreads = new ArrayList<>();
+
+        for (int i = 0; i < txThreadsCount(); i++) {
+            transferThreads.add(new TransferAmountTxThread(firstTransactionDone, igniteClient, cacheName(i), i));
+
+            transferThreads.get(i).start();
+        }
+
+        firstTransactionDone.await(10, TimeUnit.SECONDS);
+
+        failoverScenario.afterFirstTransaction();
+
+        for (Thread thread : transferThreads) {
+            thread.join();
+        }
+
+        failoverScenario.afterTransactionsFinished();
+
+        consistencyCheck(initAmount);
+    }
+
+    /**
+     * Calculates total amount of coins for every thread for every node and checks that coins difference is zero (transaction integrity is saved).
+     */
+    private void consistencyCheck(int[] initAmount) {
+        for (Ignite node : G.allGrids()) {
+            for (int j = 0; j < txThreadsCount(); j++) {
+                List<Integer> totalCoins = new ArrayList<>();
+
+                String cacheName = cacheName(j);
+
+                IgniteCache<Integer, AccountState> cache = node.getOrCreateCache(cacheName);
+
+                AccountState[] accStates = new AccountState[accountsCount()];
+
+                try (Transaction tx = node.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                    for (int i = 0; i < accountsCount(); i++) {
+                        AccountState state = cache.get(i);
+
+                        Assert.assertNotNull("Account state has lost [node=" + node.name() + ", cache=" + cacheName + ", accNo=" + i + "]", state);
+
+                        totalCoins.addAll(state.coins);
+
+                        accStates[i] = state;
+                    }
+
+                    tx.commit();
+                }
+
+                Collections.sort(totalCoins);
+
+                if (initAmount[j] != totalCoins.size()) {
+                    Set<Integer> lostCoins = new HashSet<>();
+                    Set<Integer> duplicateCoins = new HashSet<>();
+
+                    for (int coin = 1; coin <= initAmount[j]; coin++)
+                        if (!totalCoins.contains(coin))
+                            lostCoins.add(coin);
+
+                    for (int coinIdx = 1; coinIdx < totalCoins.size(); coinIdx++)
+                        if (totalCoins.get(coinIdx).equals(totalCoins.get(coinIdx - 1)))
+                            duplicateCoins.add(totalCoins.get(coinIdx));
+
+                    log.error("Transaction integrity failed for [node=" + node.name() + ", cache=" + cacheName + "]");
+
+                    log.error(String.format("Total amount of coins before and after transfers are not same. Lost coins: %s. Duplicate coins: %s.",
+                        Objects.toString(lostCoins),
+                        Objects.toString(duplicateCoins)));
+
+                    ConcurrentLinkedHashMap<IgniteUuid, TxState> txs = completedTxs[j];
+
+                    for (TxState tx : txs.values())
+                        log.error("Tx: " + tx);
+
+                    for (int i = 0; i < accountsCount(); i++)
+                        log.error("Account state " + i + " = " + accStates[i]);
+
+                    assertFalse("Test failed. See messages above", true);
+                }
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    public static class AccountState {
+        /** Account id. */
+        private final int accId;
+
+        /** Last performed transaction id on account state. */
+        @QuerySqlField(index = true)
+        private final IgniteUuid txId;
+
+        /** Set of coins holds in account. */
+        private final Set<Integer> coins;
+
+        /**
+         * @param accId Acc id.
+         * @param txId Tx id.
+         * @param coins Coins.
+         */
+        public AccountState(int accId, IgniteUuid txId, Set<Integer> coins) {
+            this.txId = txId;
+            this.coins = Collections.unmodifiableSet(coins);
+            this.accId = accId;
+        }
+
+        /**
+         * @param random Randomizer.
+         * @return Set of coins need to transfer from.
+         */
+        public Set<Integer> coinsToTransfer(Random random) {
+            int coinsNum = random.nextInt(coins.size());
+
+            return coins.stream().limit(coinsNum).collect(Collectors.toSet());
+        }
+
+        /**
+         * @param txId Transaction id.
+         * @param coinsToAdd Coins to add to current account.
+         * @return Account state with added coins.
+         */
+        public AccountState addCoins(IgniteUuid txId, Set<Integer> coinsToAdd) {
+            return new AccountState(accId, txId, Sets.union(coins, coinsToAdd).immutableCopy());
+        }
+
+        /**
+         * @param txId Transaction id.
+         * @param coinsToRemove Coins to remove from current account.
+         * @return Account state with removed coins.
+         */
+        public AccountState removeCoins(IgniteUuid txId, Set<Integer> coinsToRemove) {
+            return new AccountState(accId, txId, Sets.difference(coins, coinsToRemove).immutableCopy());
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            AccountState that = (AccountState) o;
+            return Objects.equals(txId, that.txId) &&
+                Objects.equals(coins, that.coins);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return Objects.hash(txId, coins);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "AccountState{" +
+                "accId=" + Objects.toString(accId) +
+                ", coins=" + Objects.toString(coins) +
+                '}';
+        }
+    }
+
+    /**
+     * @param coinsNum Coins number.
+     */
+    private Set<Integer> generateCoins(AtomicInteger coinsCounter, int coinsNum) {
+        Set<Integer> res = new HashSet<>();
+
+        for (int i = 0; i < coinsNum; i++)
+            res.add(coinsCounter.incrementAndGet());
+
+        return res;
+    }
+
+    /**
+     * State representing transaction between two accounts.
+     */
+    static class TxState {
+        /**
+         * Account states before transaction.
+         */
+        AccountState before1, before2;
+
+        /**
+         * Account states after transaction.
+         */
+        AccountState after1, after2;
+
+        /**
+         * Transferred coins between accounts during this transaction.
+         */
+        Set<Integer> transferredCoins;
+
+        /**
+         * @param before1 Before 1.
+         * @param before2 Before 2.
+         * @param after1 After 1.
+         * @param after2 After 2.
+         * @param transferredCoins Transferred coins.
+         */
+        public TxState(AccountState before1, AccountState before2, AccountState after1, AccountState after2, Set<Integer> transferredCoins) {
+            this.before1 = before1;
+            this.before2 = before2;
+            this.after1 = after1;
+            this.after2 = after2;
+            this.transferredCoins = transferredCoins;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "TxState{" +
+                "before1=" + before1 +
+                ", before2=" + before2 +
+                ", transferredCoins=" + transferredCoins +
+                ", after1=" + after1 +
+                ", after2=" + after2 +
+                '}';
+        }
+    }
+
+    /**
+     *
+     */
+    private class TransferAmountTxThread extends Thread {
+        /** */
+        private CountDownLatch firstTransactionLatch;
+        /** */
+        private IgniteEx ignite;
+        /** */
+        private String cacheName;
+        /** */
+        private int txIndex;
+        /** */
+        private Random random = new Random();
+
+        /**
+         * @param ignite Ignite.
+         */
+        private TransferAmountTxThread(CountDownLatch firstTransactionLatch, final IgniteEx ignite, String cacheName, int txIndex) {
+            this.firstTransactionLatch = firstTransactionLatch;
+            this.ignite = ignite;
+            this.cacheName = cacheName;
+            this.txIndex = txIndex;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            for (int i = 0; i < transactionsCount(); i++) {
+                try {
+                    updateInTransaction(ignite.cache(cacheName));
+                }
+                finally {
+                    if (i == 0)
+                        firstTransactionLatch.countDown();
+                }
+            }
+        }
+
+        /**
+         * @throws IgniteException if fails
+         */
+        @SuppressWarnings("unchecked")
+        private void updateInTransaction(IgniteCache<Integer, AccountState> cache) throws IgniteException {
+            int accIdFrom;
+            int accIdTo;
+
+            for (;;) {
+                accIdFrom = random.nextInt(accountsCount());
+                accIdTo = random.nextInt(accountsCount());
+
+                if (accIdFrom == accIdTo)
+                    continue;
+
+                ClusterNode primaryForAccFrom = ignite.cachex(cacheName).affinity().mapKeyToNode(accIdFrom);
+                ClusterNode primaryForAccTo = ignite.cachex(cacheName).affinity().mapKeyToNode(accIdTo);
+
+                // Allows only transaction between accounts that primary on the same node if corresponding flag is enabled.
+                if (!crossNodeTransactions() && !primaryForAccFrom.id().equals(primaryForAccTo.id()))
+                    continue;
+
+                break;
+            }
+
+            AccountState acctFrom;
+            AccountState acctTo;
+
+            try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                acctFrom = cache.get(accIdFrom);
+                acctTo = cache.get(accIdTo);
+
+                Set<Integer> coinsToTransfer = acctFrom.coinsToTransfer(random);
+
+                AccountState nextFrom = acctFrom.removeCoins(tx.xid(), coinsToTransfer);
+                AccountState nextTo = acctTo.addCoins(tx.xid(), coinsToTransfer);
+
+                cache.put(accIdFrom, nextFrom);
+                cache.put(accIdTo, nextTo);
+
+                tx.commit();
+
+                completedTxs[txIndex].put(tx.xid(), new TxState(acctFrom, acctTo, nextFrom, nextTo, coinsToTransfer));
+            }
+        }
+
+        /**
+         * @param curr current
+         * @return random value
+         */
+        private long getNextAccountId(long curr) {
+            long randomVal;
+
+            do {
+                randomVal = random.nextInt(accountsCount());
+            }
+            while (curr == randomVal);
+
+            return randomVal;
+        }
+    }
+
+    /**
+     * Interface to implement custom failover scenario during transactional amount transfer.
+     */
+    public interface FailoverScenario {
+        /**
+         * Callback before nodes have started.
+         */
+        public default void beforeNodesStarted() throws Exception { }
+
+        /**
+         * Callback when first transaction has finished.
+         */
+        public default void afterFirstTransaction() throws Exception { }
+
+        /**
+         * Callback when all transactions have finished.
+         */
+        public default void afterTransactionsFinished() throws Exception { }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java
new file mode 100644
index 0000000000000..3260607023a92
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.function.BiFunction;
+import java.util.function.Supplier;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteIllegalStateException;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.processors.cache.tree.SearchRow;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ * Test cases that check transaction data integrity after transaction commit failed.
+ */
+public class TransactionIntegrityWithPrimaryIndexCorruptionTest extends AbstractTransactionIntergrityTest {
+    /** Corruption enabled flag. */
+    private static volatile boolean corruptionEnabled;
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        corruptionEnabled = false;
+
+        super.afterTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 60 * 1000L;
+    }
+
+    /**
+     * Throws a test {@link AssertionError} during tx commit from {@link BPlusTree} and checks after that data is consistent.
+     */
+    public void testPrimaryIndexCorruptionDuringCommitOnPrimaryNode1() throws Exception {
+        doTestTransferAmount(new IndexCorruptionFailoverScenario(
+            true,
+            (hnd, tree) -> hnd instanceof BPlusTree.Search,
+            failoverPredicate(true, () -> new AssertionError("Test")))
+        );
+    }
+
+    /**
+     * Throws a test {@link RuntimeException} during tx commit from {@link BPlusTree} and checks after that data is consistent.
+     */
+    public void testPrimaryIndexCorruptionDuringCommitOnPrimaryNode2() throws Exception {
+        doTestTransferAmount(new IndexCorruptionFailoverScenario(
+            true,
+            (hnd, tree) -> hnd instanceof BPlusTree.Search,
+            failoverPredicate(true, () -> new RuntimeException("Test")))
+        );
+    }
+
+    /**
+     * Throws a test {@link AssertionError} during tx commit from {@link BPlusTree} and checks after that data is consistent.
+     */
+    public void testPrimaryIndexCorruptionDuringCommitOnBackupNode() throws Exception {
+        doTestTransferAmount(new IndexCorruptionFailoverScenario(
+            true,
+            (hnd, tree) -> hnd instanceof BPlusTree.Search,
+            failoverPredicate(false, () -> new AssertionError("Test")))
+        );
+    }
+
+    /**
+     * Throws a test {@link IgniteCheckedException} during tx commit from {@link BPlusTree} and checks after that data is consistent.
+     */
+    public void testPrimaryIndexCorruptionDuringCommitOnPrimaryNode3() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-9082");
+
+        doTestTransferAmount(new IndexCorruptionFailoverScenario(
+            false,
+            (hnd, tree) -> hnd instanceof BPlusTree.Search,
+            failoverPredicate(true, () -> new IgniteCheckedException("Test")))
+        );
+    }
+
+    /**
+     * Creates failover predicate which generates error during transaction commmit.
+     *
+     * @param failOnPrimary If {@code true} index should be failed on transaction primary node.
+     * @param errorSupplier Supplier to create various errors.
+     */
+    private BiFunction<IgniteEx, SearchRow, Throwable> failoverPredicate(
+        boolean failOnPrimary,
+        Supplier<Throwable> errorSupplier
+    ) {
+        return (ignite, row) -> {
+            int cacheId = row.cacheId();
+            int partId = row.key().partition();
+
+            final ClusterNode locNode = ignite.localNode();
+            final AffinityTopologyVersion curTopVer = ignite.context().discovery().topologyVersionEx();
+
+            // Throw exception if current node is primary for given row.
+            return ignite.cachesx(c -> c.context().cacheId() == cacheId)
+                .stream()
+                .filter(c -> c.context().affinity().primaryByPartition(locNode, partId, curTopVer) == failOnPrimary)
+                .map(c -> errorSupplier.get())
+                .findFirst()
+                .orElse(null);
+        };
+    }
+
+    /**
+     * Index corruption failover scenario.
+     */
+    class IndexCorruptionFailoverScenario implements FailoverScenario {
+        /** Failed node index. */
+        static final int failedNodeIdx = 1;
+
+        /** Is node stopping expected after failover. */
+        private final boolean nodeStoppingExpected;
+
+        /** Predicate that will choose an instance of {@link BPlusTree} and page operation
+         * to make further failover in this tree using {@link #failoverPredicate}. */
+        private final BiFunction<PageHandler, BPlusTree, Boolean> treeCorruptionPredicate;
+
+        /** Function that may return error during row insertion into {@link BPlusTree}. */
+        private final BiFunction<IgniteEx, SearchRow, Throwable> failoverPredicate;
+
+        /**
+         * @param nodeStoppingExpected Node stopping expected.
+         * @param treeCorruptionPredicate Tree corruption predicate.
+         * @param failoverPredicate Failover predicate.
+         */
+        IndexCorruptionFailoverScenario(
+            boolean nodeStoppingExpected,
+            BiFunction<PageHandler, BPlusTree, Boolean> treeCorruptionPredicate,
+            BiFunction<IgniteEx, SearchRow, Throwable> failoverPredicate
+        ) {
+            this.nodeStoppingExpected = nodeStoppingExpected;
+            this.treeCorruptionPredicate = treeCorruptionPredicate;
+            this.failoverPredicate = failoverPredicate;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void beforeNodesStarted() {
+            BPlusTree.pageHndWrapper = (tree, hnd) -> {
+                final IgniteEx locIgnite = (IgniteEx) Ignition.localIgnite();
+
+                if (!locIgnite.name().endsWith(String.valueOf(failedNodeIdx)))
+                    return hnd;
+
+                if (treeCorruptionPredicate.apply(hnd, tree)) {
+                    log.info("Created corrupted tree handler for -> " + hnd + " " + tree);
+
+                    PageHandler<Object, BPlusTree.Result> delegate = (PageHandler<Object, BPlusTree.Result>) hnd;
+
+                    return new PageHandler<BPlusTree.Get, BPlusTree.Result>() {
+                        @Override public BPlusTree.Result run(int cacheId, long pageId, long page, long pageAddr, PageIO io, Boolean walPlc, BPlusTree.Get arg, int lvl) throws IgniteCheckedException {
+                            log.info("Invoked " + " " + cacheId + " " + arg.toString() + " for BTree (" + corruptionEnabled + ") -> " + arg.row() + " / " + arg.row().getClass());
+
+                            if (corruptionEnabled && (arg.row() instanceof SearchRow)) {
+                                SearchRow row = (SearchRow) arg.row();
+
+                                // Store cacheId to search row explicitly, as it can be zero if there is one cache in a group.
+                                Throwable res = failoverPredicate.apply(locIgnite, new SearchRow(cacheId, row.key()));
+
+                                if (res != null) {
+                                    if (res instanceof Error)
+                                        throw (Error) res;
+                                    else if (res instanceof RuntimeException)
+                                        throw (RuntimeException) res;
+                                    else if (res instanceof IgniteCheckedException)
+                                        throw (IgniteCheckedException) res;
+                                }
+                            }
+
+                            return delegate.run(cacheId, pageId, page, pageAddr, io, walPlc, arg, lvl);
+                        }
+
+                        @Override public boolean releaseAfterWrite(int cacheId, long pageId, long page, long pageAddr, BPlusTree.Get g, int lvl) {
+                            return g.canRelease(pageId, lvl);
+                        }
+                    };
+                }
+
+                return hnd;
+            };
+        }
+
+        /** {@inheritDoc} */
+        @Override public void afterFirstTransaction() {
+            // Enable BPlus tree corruption after first transactions have finished.
+            corruptionEnabled = true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void afterTransactionsFinished() throws Exception {
+            // Disable index corruption.
+            BPlusTree.pageHndWrapper = (tree, hnd) -> hnd;
+
+            if (nodeStoppingExpected) {
+                // Wait until node with corrupted index will left cluster.
+                GridTestUtils.waitForCondition(() -> {
+                    try {
+                        grid(failedNodeIdx);
+                    }
+                    catch (IgniteIllegalStateException e) {
+                        return true;
+                    }
+
+                    return false;
+                }, getTestTimeout());
+
+                // Failed node should be stopped.
+                GridTestUtils.assertThrows(log, () -> grid(failedNodeIdx), IgniteIllegalStateException.class, "");
+
+                // Re-start failed node.
+                startGrid(failedNodeIdx);
+
+                awaitPartitionMapExchange();
+            }
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java
new file mode 100644
index 0000000000000..25aae4b2a1b47
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.lang.management.ManagementFactory;
+import javax.management.MBeanServer;
+import javax.management.MBeanServerInvocationHandler;
+import javax.management.ObjectName;
+import org.apache.ignite.IgniteIllegalStateException;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.worker.WorkersControlMXBeanImpl;
+import org.apache.ignite.mxbean.WorkersControlMXBean;
+import org.apache.ignite.testframework.GridTestUtils;
+
+/**
+ *
+ */
+public class TransactionIntegrityWithSystemWorkerDeathTest extends AbstractTransactionIntergrityTest {
+    /** {@inheritDoc}. */
+    @Override protected long getTestTimeout() {
+        return 60 * 1000L;
+    }
+
+    /** {@inheritDoc}. */
+    @Override protected boolean persistent() {
+        return false;
+    }
+
+    /**
+     *
+     */
+    public void testFailoverWithDiscoWorkerTermination() throws Exception {
+        doTestTransferAmount(new FailoverScenario() {
+            static final int failedNodeIdx = 1;
+
+            /** {@inheritDoc}. */
+            @Override public void afterFirstTransaction() throws Exception {
+                // Terminate disco-event-worker thread on one node.
+                WorkersControlMXBean bean = workersMXBean(failedNodeIdx);
+
+                bean.terminateWorker(
+                    bean.getWorkerNames().stream()
+                        .filter(name -> name.startsWith("disco-event-worker"))
+                        .findFirst()
+                        .orElse(null)
+                );
+            }
+
+            /** {@inheritDoc}. */
+            @Override public void afterTransactionsFinished() throws Exception {
+                // Wait until node with death worker will left cluster.
+                GridTestUtils.waitForCondition(() -> {
+                    try {
+                        grid(failedNodeIdx);
+                    }
+                    catch (IgniteIllegalStateException e) {
+                        return true;
+                    }
+
+                    return false;
+                }, getTestTimeout());
+
+                // Failed node should be stopped.
+                GridTestUtils.assertThrows(log, () -> grid(failedNodeIdx), IgniteIllegalStateException.class, "");
+
+                // Re-start failed node.
+                startGrid(failedNodeIdx);
+
+                awaitPartitionMapExchange();
+            }
+        });
+    }
+
+    /**
+     * Configure workers mx bean.
+     */
+    private WorkersControlMXBean workersMXBean(int igniteInt) throws Exception {
+        ObjectName mbeanName = U.makeMBeanName(
+            getTestIgniteInstanceName(igniteInt),
+            "Kernal",
+            WorkersControlMXBeanImpl.class.getSimpleName()
+        );
+
+        MBeanServer mbeanSrv = ManagementFactory.getPlatformMBeanServer();
+
+        if (!mbeanSrv.isRegistered(mbeanName))
+            fail("MBean is not registered: " + mbeanName.getCanonicalName());
+
+        return MBeanServerInvocationHandler.newProxyInstance(mbeanSrv, mbeanName, WorkersControlMXBean.class, true);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index 576bd5a64a242..4486fe8c7f69b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -20,7 +20,6 @@
 import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.GridSuppressedExceptionSelfTest;
-import org.apache.ignite.failure.AccountTransferTransactionTest;
 import org.apache.ignite.failure.FailureHandlerTriggeredTest;
 import org.apache.ignite.failure.IoomFailureHandlerTest;
 import org.apache.ignite.failure.OomFailureHandlerTest;
@@ -57,6 +56,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteMarshallerCacheFSRestoreTest;
 import org.apache.ignite.internal.processors.cache.SetTxTimeoutOnPartitionMapExchangeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteRejectConnectOnNodeStopTest;
+import org.apache.ignite.internal.processors.cache.transactions.TransactionIntegrityWithSystemWorkerDeathTest;
 import org.apache.ignite.internal.processors.closure.GridClosureProcessorRemoteTest;
 import org.apache.ignite.internal.processors.closure.GridClosureProcessorSelfTest;
 import org.apache.ignite.internal.processors.closure.GridClosureSerializationTest;
@@ -208,8 +208,8 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
         suite.addTestSuite(IoomFailureHandlerTest.class);
         suite.addTestSuite(OomFailureHandlerTest.class);
-        suite.addTestSuite(AccountTransferTransactionTest.class);
         suite.addTestSuite(SystemWorkersTerminationTest.class);
+        suite.addTestSuite(TransactionIntegrityWithSystemWorkerDeathTest.class);
 
         suite.addTestSuite(CacheRebalanceConfigValidationTest.class);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 890954440f909..1ca35b22e3cba 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgnitePessimisticTxSuspendResumeTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
+import org.apache.ignite.internal.processors.cache.transactions.TransactionIntegrityWithPrimaryIndexCorruptionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxLabelTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxMultiCacheAsyncOpsTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxOnCachesStartTest;
@@ -126,6 +127,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(PartitionsExchangeCoordinatorFailoverTest.class);
 
+        suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class);
+
         //suite.addTestSuite(CacheClientsConcurrentStartTest.class);
         //suite.addTestSuite(GridCacheRebalancingOrderingTest.class);
         //suite.addTestSuite(IgniteCacheClientMultiNodeUpdateTopologyLockTest.class);

From e449961bf221876f3788dc61619bffdeb6b17908 Mon Sep 17 00:00:00 2001
From: AMedvedev <andrew.y.medvedev>
Date: Fri, 24 Aug 2018 18:15:05 +0300
Subject: [PATCH 330/543] IGNITE-8971 Make GridRestProcessor propagate error
 message - Fixes #4604.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit a4e144cc3616f2430843934cf844b4ac29841ba0)
---
 .../JettyRestProcessorAbstractSelfTest.java   |  2 +-
 .../processors/rest/GridRestProcessor.java    | 23 ++++++++++++++++++-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
index 5bd8ea20fbc5d..76190b0efc629 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
@@ -688,7 +688,7 @@ public void testIncorrectPut() throws Exception {
         String ret = content(DEFAULT_CACHE_NAME, GridRestCommand.CACHE_PUT, "key", "key0");
 
         assertResponseContainsError(ret,
-            "Failed to handle request: [req=CACHE_PUT, err=Failed to find mandatory parameter in request: val]");
+            "Failed to find mandatory parameter in request: val");
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
index d7a30f95787e6..ccb92df24dc89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.rest;
 
+import java.io.PrintWriter;
+import java.io.StringWriter;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
@@ -349,7 +351,11 @@ private IgniteInternalFuture<GridRestResponse> handleRequest(final GridRestReque
                             .a(", params=").a(tskReq.params());
                     }
 
-                    sb.a(", err=").a(e.getMessage() != null ? e.getMessage() : e.getClass().getName()).a(']');
+                    sb.a(", err=")
+                        .a(e.getMessage() != null ? e.getMessage() : e.getClass().getName())
+                        .a(", trace=")
+                        .a(getErrorMessage(e))
+                        .a(']');
 
                     res = new GridRestResponse(STATUS_FAILED, sb.toString());
                 }
@@ -366,6 +372,21 @@ private IgniteInternalFuture<GridRestResponse> handleRequest(final GridRestReque
         });
     }
 
+    /**
+     * @param th Th.
+     * @return Stack trace
+     */
+    private String getErrorMessage(Throwable th) {
+        if (th == null)
+            return "";
+
+        StringWriter writer = new StringWriter();
+
+        th.printStackTrace(new PrintWriter(writer));
+
+        return writer.toString();
+    }
+
     /**
      * @param req Request.
      * @return Not null session.

From 25a29f4ac7b0d53fcd7800fe1bb5894c4a0f4bb2 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Mon, 27 Aug 2018 18:54:56 +0300
Subject: [PATCH 331/543] IGNITE-9183: SQL: fixed UUID handling from DDL
 statements. This closes #4483.

(cherry picked from commit a98dcb93ba45f9d2b69585a84653f2e0174ba2a4)
---
 .../apache/ignite/IgniteSystemProperties.java |   3 +
 .../query/h2/ddl/DdlStatementsProcessor.java  |  34 ++++-
 ...H2DynamicColumnsAbstractBasicSelfTest.java | 117 ++++++++++++++++++
 .../cache/index/H2DynamicTableSelfTest.java   |  91 +++++++++++---
 4 files changed, 222 insertions(+), 23 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index aaf8305ec6f3f..dd54ced164f0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -474,6 +474,9 @@ public final class IgniteSystemProperties {
     /** SQL retry timeout. */
     public static final String IGNITE_SQL_RETRY_TIMEOUT = "IGNITE_SQL_RETRY_TIMEOUT";
 
+    /** Enable backward compatible handling of UUID through DDL. */
+    public static final String IGNITE_SQL_UUID_DDL_BYTE_FORMAT = "IGNITE_SQL_UUID_DDL_BYTE_FORMAT";
+
     /** Maximum size for affinity assignment history. */
     public static final String IGNITE_AFFINITY_HISTORY_SIZE = "IGNITE_AFFINITY_HISTORY_SIZE";
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
index bc5c1e0208848..b6db5c550211b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
@@ -25,8 +25,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteCluster;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.cache.QueryIndex;
 import org.apache.ignite.cache.QueryIndexType;
@@ -80,6 +82,7 @@
 import org.h2.command.ddl.DropTable;
 import org.h2.table.Column;
 import org.h2.value.DataType;
+import org.h2.value.Value;
 
 import static org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.UPDATE_RESULT_META;
 import static org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser.PARAM_WRAP_VALUE;
@@ -95,6 +98,10 @@ public class DdlStatementsProcessor {
     /** Indexing. */
     IgniteH2Indexing idx;
 
+    /** Is backward compatible handling of UUID through DDL enabled. */
+    private static final boolean handleUuidAsByte =
+            IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_SQL_UUID_DDL_BYTE_FORMAT, false);
+
     /**
      * Initialize message handlers and this' fields needed for further operation.
      *
@@ -419,7 +426,7 @@ else if (stmt0 instanceof GridSqlAlterTableAddColumn) {
                         }
 
                         QueryField field = new QueryField(col.columnName(),
-                            DataType.getTypeClassName(col.column().getType()),
+                            getTypeClassName(col),
                             col.column().isNullable(), col.defaultValue());
 
                         cols.add(field);
@@ -600,7 +607,7 @@ private static QueryEntity toQueryEntity(GridSqlCreateTable createTbl) {
 
             Column col = gridCol.column();
 
-            res.addQueryField(e.getKey(), DataType.getTypeClassName(col.getType()), null);
+            res.addQueryField(e.getKey(), getTypeClassName(gridCol), null);
 
             if (!col.isNullable()) {
                 if (notNullFields == null)
@@ -633,7 +640,7 @@ private static QueryEntity toQueryEntity(GridSqlCreateTable createTbl) {
         if (!createTbl.wrapKey()) {
             GridSqlColumn pkCol = createTbl.columns().get(createTbl.primaryKeyColumns().iterator().next());
 
-            keyTypeName = DataType.getTypeClassName(pkCol.column().getType());
+            keyTypeName = getTypeClassName(pkCol);
 
             res.setKeyFieldName(pkCol.columnName());
         }
@@ -653,7 +660,7 @@ private static QueryEntity toQueryEntity(GridSqlCreateTable createTbl) {
 
             assert valCol != null;
 
-            valTypeName = DataType.getTypeClassName(valCol.column().getType());
+            valTypeName = getTypeClassName(valCol);
 
             res.setValueFieldName(valCol.columnName());
         }
@@ -680,4 +687,23 @@ public static boolean isDdlStatement(Prepared cmd) {
         return cmd instanceof CreateIndex || cmd instanceof DropIndex || cmd instanceof CreateTable ||
             cmd instanceof DropTable || cmd instanceof AlterTableAlterColumn;
     }
+
+    /**
+     * Helper function for obtaining type class name for H2.
+     *
+     * @param col Column.
+     * @return Type class name.
+     */
+    private static String getTypeClassName(GridSqlColumn col) {
+        int type = col.column().getType();
+
+        switch (type) {
+            case Value.UUID :
+                if (!handleUuidAsByte)
+                    return UUID.class.getName();
+
+            default:
+                return DataType.getTypeClassName(type);
+        }
+    }
 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicColumnsAbstractBasicSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicColumnsAbstractBasicSelfTest.java
index 70f2d85a82818..50f0349e771f2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicColumnsAbstractBasicSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicColumnsAbstractBasicSelfTest.java
@@ -21,6 +21,8 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Random;
+import java.util.UUID;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.binary.BinaryObject;
@@ -288,6 +290,76 @@ public void testAddColumnToNonDynamicCacheWithRealValueType() throws SQLExceptio
         cache.destroy();
     }
 
+    /**
+     * Tests that we can add dynamically UUID column to tables.
+     *
+     * @throws SQLException If failed.
+     */
+    @SuppressWarnings("unchecked")
+    public void testAddColumnUUID() throws SQLException {
+        CacheConfiguration<Integer, Object> ccfg = defaultCacheConfiguration().setName("GuidTest")
+                .setIndexedTypes(Integer.class, GuidTest.class);
+
+        Random rnd = new Random();
+
+        IgniteCache<Integer, Object> cache = ignite(nodeIndex()).getOrCreateCache(ccfg);
+
+        run(cache, "ALTER TABLE \"GuidTest\".GuidTest ADD COLUMN GUID UUID");
+        run(cache, "ALTER TABLE \"GuidTest\".GuidTest ADD COLUMN DATA BINARY(128)");
+
+        doSleep(500);
+
+        QueryField c1 = c("GUID", Object.class.getName());
+        QueryField c2 = c("DATA", byte[].class.getName());
+
+        checkTableState("GuidTest", "GUIDTEST", c1, c2);
+
+        UUID guid1 = UUID.randomUUID();
+        UUID guid2 = UUID.randomUUID();
+
+        // Populate random data for BINARY field.
+        byte[] data1 = new byte[128];
+        rnd.nextBytes(data1);
+        byte[] data2 = new byte[128];
+        rnd.nextBytes(data2);
+
+        run(cache, "INSERT INTO \"GuidTest\".GuidTest (_key, id, guid, data) values " +
+                "(1, 1, ?, ?)", guid1.toString(), data1);
+
+        cache.put(2, new GuidTest(2, guid2, data2));
+
+        List<List<?>> res = run(cache, "select _key, id, guid from \"GuidTest\".GuidTest order by id");
+
+        assertEquals(Arrays.asList(Arrays.asList(1, 1, guid1), Arrays.asList(2, 2, guid2)), res);
+
+        // Additional check for BINARY field content.
+        res = run(cache, "select data from \"GuidTest\".GuidTest order by id");
+
+        assertTrue(Arrays.equals(data1, (byte[])res.get(0).get(0)));
+        assertTrue(Arrays.equals(data2, (byte[])res.get(1).get(0)));
+
+        if (!Boolean.valueOf(GridTestProperties.getProperty(BINARY_MARSHALLER_USE_SIMPLE_NAME_MAPPER))) {
+            GuidTest val1 = (GuidTest)cache.get(1);
+            GuidTest val2 = (GuidTest)cache.get(2);
+
+            assertEquals(guid1, val1.guid());
+            assertEquals(guid2, val2.guid());
+            assertTrue(Arrays.equals(data1, val1.data()));
+            assertTrue(Arrays.equals(data2, val2.data()));
+        }
+        else {
+            BinaryObject val1 = (BinaryObject)cache.withKeepBinary().get(1);
+            BinaryObject val2 = (BinaryObject)cache.withKeepBinary().get(2);
+
+            assertEquals(guid1, val1.field("guid"));
+            assertEquals(guid2, val2.field("guid"));
+            assertTrue(Arrays.equals(data1, val1.field("data")));
+            assertTrue(Arrays.equals(data2, val2.field("data")));
+        }
+
+        cache.destroy();
+    }
+
     /**
      * Test addition of column with not null constraint.
      */
@@ -777,4 +849,49 @@ public void state(String state) {
             this.state = state;
         }
     }
+
+    /**  */
+    private final static class GuidTest {
+        /** */
+        @QuerySqlField
+        private int id;
+
+        /** */
+        private UUID guid;
+
+        /** */
+        private byte[] data;
+
+        /**
+         * @param id   Id.
+         * @param guid Guid.
+         * @param data Data.
+         */
+        public GuidTest(int id, UUID guid, byte[] data) {
+            this.id = id;
+            this.guid = guid;
+            this.data = data;
+        }
+
+        /**
+         * @return Id.
+         */
+        public int id() {
+            return id;
+        }
+
+        /**
+         * @return Guid.
+         */
+        public UUID guid() {
+            return guid;
+        }
+
+        /**
+         * @return Data.
+         */
+        public byte[] data() {
+            return data;
+        }
+    }
 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
index 82247114fc062..424205fed5935 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
@@ -31,6 +31,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import javax.cache.CacheException;
 import org.apache.ignite.Ignite;
@@ -43,6 +44,7 @@
 import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.cache.QueryIndex;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -1248,7 +1250,7 @@ public void testWrappingAlwaysOnWithComplexObjects() {
      * @throws SQLException if failed.
      */
     public void testNoWrap() throws SQLException {
-        doTestKeyValueWrap(false, false);
+        doTestKeyValueWrap(false, false, false);
     }
 
     /**
@@ -1256,7 +1258,7 @@ public void testNoWrap() throws SQLException {
      * @throws SQLException if failed.
      */
     public void testKeyWrap() throws SQLException {
-        doTestKeyValueWrap(true, false);
+        doTestKeyValueWrap(true, false, false);
     }
 
     /**
@@ -1264,7 +1266,7 @@ public void testKeyWrap() throws SQLException {
      * @throws SQLException if failed.
      */
     public void testValueWrap() throws SQLException {
-        doTestKeyValueWrap(false, true);
+        doTestKeyValueWrap(false, true, false);
     }
 
     /**
@@ -1272,29 +1274,73 @@ public void testValueWrap() throws SQLException {
      * @throws SQLException if failed.
      */
     public void testKeyAndValueWrap() throws SQLException {
-        doTestKeyValueWrap(true, true);
+        doTestKeyValueWrap(true, true, false);
+    }
+
+    /**
+     * Test behavior when neither key nor value should be wrapped.
+     * Key and value are UUID.
+     * @throws SQLException if failed.
+     */
+    public void testUuidNoWrap() throws SQLException {
+        doTestKeyValueWrap(false, false, true);
+    }
+
+    /**
+     * Test behavior when only key is wrapped.
+     * Key and value are UUID.
+     * @throws SQLException if failed.
+     */
+    public void testUuidKeyWrap() throws SQLException {
+        doTestKeyValueWrap(true, false, true);
+    }
+
+    /**
+     * Test behavior when only value is wrapped.
+     * Key and value are UUID.
+     * @throws SQLException if failed.
+     */
+    public void testUuidValueWrap() throws SQLException {
+        doTestKeyValueWrap(false, true, true);
+    }
+
+    /**
+     * Test behavior when both key and value is wrapped.
+     * Key and value are UUID.
+     * @throws SQLException if failed.
+     */
+    public void testUuidKeyAndValueWrap() throws SQLException {
+        doTestKeyValueWrap(true, true, true);
     }
 
     /**
      * Test behavior for given combination of wrap flags.
      * @param wrapKey Whether key wrap should be enforced.
      * @param wrapVal Whether value wrap should be enforced.
+     * @param testUuid Whether should test with UUID as key and value.
      * @throws SQLException if failed.
      */
-    private void doTestKeyValueWrap(boolean wrapKey, boolean wrapVal) throws SQLException {
+    private void doTestKeyValueWrap(boolean wrapKey, boolean wrapVal, boolean testUuid) throws SQLException {
         try {
-            String sql = String.format("CREATE TABLE T (\"id\" int primary key, \"x\" varchar) WITH " +
-                "\"wrap_key=%b,wrap_value=%b\"", wrapKey, wrapVal);
+            String sql = testUuid ? String.format("CREATE TABLE T (\"id\" UUID primary key, \"x\" UUID) WITH " +
+                            "\"wrap_key=%b,wrap_value=%b\"", wrapKey, wrapVal) :
+                    String.format("CREATE TABLE T (\"id\" int primary key, \"x\" varchar) WITH " +
+                            "\"wrap_key=%b,wrap_value=%b\"", wrapKey, wrapVal);
+
+            UUID guid = UUID.randomUUID();
 
             if (wrapKey)
-                sql += ",\"key_type=tkey\"";
+                sql += ",\"key_type=" + (testUuid ? "tkey_guid" : "tkey") + "\"";
 
             if (wrapVal)
-                sql += ",\"value_type=tval\"";
+                sql += ",\"value_type=" + (testUuid ? "tval_guid" : "tval") + "\"";
 
             execute(sql);
 
-            execute("INSERT INTO T(\"id\", \"x\") values(1, 'a')");
+            if(testUuid)
+                execute("INSERT INTO T(\"id\", \"x\") values('" + guid.toString() + "', '" + guid.toString() + "')");
+            else
+                execute("INSERT INTO T(\"id\", \"x\") values(1, 'a')");
 
             LinkedHashMap<String, String> resCols = new LinkedHashMap<>();
 
@@ -1320,20 +1366,27 @@ private void doTestKeyValueWrap(boolean wrapKey, boolean wrapVal) throws SQLExce
 
             LinkedHashMap<String, String> expCols = new LinkedHashMap<>();
 
-            expCols.put("id", Integer.class.getName());
-            expCols.put("x", String.class.getName());
+            if (testUuid) {
+                expCols.put("id", Object.class.getName());
+                expCols.put("x", Object.class.getName());
+            }
+            else {
+                expCols.put("id", Integer.class.getName());
+                expCols.put("x", String.class.getName());
+            }
 
             assertEquals(expCols, resCols);
 
-            assertEqualsCollections(Arrays.asList(1, "a"), resData);
+            assertEqualsCollections(testUuid ? Arrays.asList(guid, guid) : Arrays.asList(1, "a")
+                    , resData);
 
-            Object key = createKeyForWrapTest(1, wrapKey);
+            Object key = createKeyForWrapTest(testUuid ? guid : 1, wrapKey);
 
             Object val = client().cache(cacheName("T")).withKeepBinary().get(key);
 
             assertNotNull(val);
 
-            assertEquals(createValueForWrapTest("a", wrapVal), val);
+            assertEquals(createValueForWrapTest(testUuid ? guid : "a", wrapVal), val);
         }
         finally {
             execute("DROP TABLE IF EXISTS T");
@@ -1345,11 +1398,11 @@ private void doTestKeyValueWrap(boolean wrapKey, boolean wrapVal) throws SQLExce
      * @param wrap Whether key should be wrapped.
      * @return (optionally wrapped) key.
      */
-    private Object createKeyForWrapTest(int key, boolean wrap) {
+    private Object createKeyForWrapTest(Object key, boolean wrap) {
         if (!wrap)
             return key;
 
-        return client().binary().builder("tkey").setField("id", key).build();
+        return client().binary().builder(key instanceof UUID ? "tkey_guid" : "tkey").setField("id", key).build();
     }
 
     /**
@@ -1357,11 +1410,11 @@ private Object createKeyForWrapTest(int key, boolean wrap) {
      * @param wrap Whether value should be wrapped.
      * @return (optionally wrapped) value.
      */
-    private Object createValueForWrapTest(String val, boolean wrap) {
+    private Object createValueForWrapTest(Object val, boolean wrap) {
         if (!wrap)
             return val;
 
-        return client().binary().builder("tval").setField("x", val).build();
+        return client().binary().builder(val instanceof UUID ? "tval_guid" : "tval").setField("x", val).build();
     }
 
     /**

From 4f1b5677c62af07a269775f14c38176b374191c8 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Mon, 27 Aug 2018 20:18:04 +0300
Subject: [PATCH 332/543] GG-14131 Backport additional fixes from IGNITE-9147
 and IGNITE-9246 after merge to master.

---
 .../dht/GridDhtTransactionalCacheAdapter.java |   2 +
 .../colocated/GridDhtColocatedLockFuture.java | 553 +++++++++---------
 ...OptimisticSerializableTxPrepareFuture.java |   9 +-
 .../GridNearOptimisticTxPrepareFuture.java    |  10 +-
 ...dNearOptimisticTxPrepareFutureAdapter.java | 122 +---
 .../near/GridNearTxFinishFuture.java          |   4 +-
 .../cache/transactions/IgniteTxHandler.java   |  80 ++-
 .../timeout/GridTimeoutProcessor.java         |  94 +++
 .../ignite/internal/visor/tx/VisorTxInfo.java |  25 +-
 .../ignite/internal/visor/tx/VisorTxTask.java |   2 +-
 .../transactions/TxRollbackAsyncTest.java     | 129 +++-
 .../transactions/TxRollbackOnTimeoutTest.java | 284 ++++-----
 12 files changed, 726 insertions(+), 588 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index e3e29b01e03cd..be6b6fb11b60a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -696,6 +696,8 @@ private boolean waitForExchangeFuture(final ClusterNode node, final GridNearLock
                 final IgniteThread thread = (IgniteThread)curThread;
 
                 if (thread.cachePoolThread()) {
+                    // Near transaction's finish on timeout will unlock topFut if it was held for too long,
+                    // so need to listen with timeout. This is not true for optimistic transactions.
                     topFut.listen(new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
                         @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> fut) {
                             ctx.kernalContext().closure().runLocalWithThreadPolicy(thread, new Runnable() {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index 55a82cbad4bab..7c81f57d37c64 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -23,12 +23,10 @@
 import java.util.Deque;
 import java.util.HashSet;
 import java.util.Iterator;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
@@ -66,7 +64,6 @@
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.lang.GridAbsClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.C1;
@@ -136,7 +133,7 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
 
     /** Timeout object. */
     @GridToStringExclude
-    private LockTimeoutObject timeoutObj;
+    private volatile LockTimeoutObject timeoutObj;
 
     /** Lock timeout. */
     private final long timeout;
@@ -182,7 +179,7 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
     /** */
     private int miniId;
 
-    /** Used for synchronization between lock cancellation and mapping phase. */
+    /** {@code True} when mappings are ready for processing. */
     private boolean mappingsReady;
 
     /**
@@ -449,32 +446,59 @@ public void complete(boolean success) {
      * @param nodeId Sender.
      * @param res Result.
      */
+    @SuppressWarnings("SynchronizeOnNonFinalField")
     void onResult(UUID nodeId, GridNearLockResponse res) {
-        if (!isDone()) {
-            MiniFuture mini = miniFuture(res.miniId());
-
-            if (mini != null) {
-                assert mini.node().id().equals(nodeId);
+        boolean done = isDone();
 
-                mini.onResult(res);
+        if (!done) {
+            // onResult is always called after map() and timeoutObj is never reset to null, so this is
+            // a race-free null check.
+            if (timeoutObj == null) {
+                onResult0(nodeId, res);
 
                 return;
             }
 
-            //  This warning can be triggered by deadlock detection code which clears pending futures.
-            U.warn(msgLog, "Collocated lock fut, failed to find mini future [txId=" + lockVer +
-                ", tx=" + (inTx() ? CU.txString(tx) : "N/A") +
-                ", node=" + nodeId +
-                ", res=" + res +
-                ", fut=" + this + ']');
-        }
-        else {
-            if (msgLog.isDebugEnabled()) {
-                msgLog.debug("Collocated lock fut, response for finished future [txId=" + lockVer +
-                    ", inTx=" + inTx() +
-                    ", node=" + nodeId + ']');
+            synchronized (timeoutObj) {
+                if (!isDone()) {
+                    if (onResult0(nodeId, res))
+                        return;
+                }
+                else
+                    done = true;
             }
         }
+
+        if (done && msgLog.isDebugEnabled()) {
+            msgLog.debug("Collocated lock fut, response for finished future [txId=" + lockVer +
+                ", inTx=" + inTx() +
+                ", node=" + nodeId + ']');
+        }
+    }
+
+    /**
+     * @param nodeId Sender.
+     * @param res Result.
+     */
+    private boolean onResult0(UUID nodeId, GridNearLockResponse res) {
+        MiniFuture mini = miniFuture(res.miniId());
+
+        if (mini != null) {
+            assert mini.node().id().equals(nodeId);
+
+            mini.onResult(res);
+
+            return true;
+        }
+
+        //  This warning can be triggered by deadlock detection code which clears pending futures.
+        U.warn(msgLog, "Collocated lock fut, failed to find mini future [txId=" + lockVer +
+            ", tx=" + (inTx() ? CU.txString(tx) : "N/A") +
+            ", node=" + nodeId +
+            ", res=" + res +
+            ", fut=" + this + ']');
+
+        return false;
     }
 
     /**
@@ -555,6 +579,9 @@ private synchronized void onError(Throwable t) {
         if (inTx()) {
             onError(tx.rollbackException());
 
+            /** Should wait until {@link mappings} are ready before continuing with async rollback
+             * or some primary nodes might not receive tx finish messages because of race.
+             */
             synchronized (this) {
                 while (!mappingsReady)
                     try {
@@ -626,11 +653,14 @@ private boolean onComplete(boolean success, boolean distribute) {
             if (timeoutObj != null)
                 cctx.time().removeTimeoutObject(timeoutObj);
 
-            synchronized (this) {
-                if (!mappingsReady) {
-                    mappingsReady = true;
+            /** Ensures what waiters for ready {@link mappings} will be unblocked if error has occurred while mapping. */
+            if (tx != null) {
+                synchronized (this) {
+                    if (!mappingsReady) {
+                        mappingsReady = true;
 
-                    notifyAll();
+                        notifyAll();
+                    }
                 }
             }
 
@@ -833,14 +863,15 @@ private void mapOnTopology(final boolean remap, @Nullable final Runnable c) {
                 markInitialized();
             }
             else {
-                applyWhenReady(fut, new GridAbsClosureX() {
-                    @Override public void applyx() throws IgniteCheckedException {
-                        try {
-                            mapOnTopology(remap, c);
-                        }
-                        finally {
-                            cctx.shared().txContextReset();
-                        }
+                cctx.time().waitAsync(fut, tx == null ? 0 : tx.remainingTime(), (e, timedOut) -> {
+                    if (errorOrTimeoutOnTopologyVersion(e, timedOut))
+                        return;
+
+                    try {
+                        mapOnTopology(remap, c);
+                    }
+                    finally {
+                        cctx.shared().txContextReset();
                     }
                 });
             }
@@ -869,13 +900,6 @@ private void map(Collection<KeyCacheObject> keys, boolean remap, boolean topLock
         catch (IgniteCheckedException ex) {
             onDone(false, ex);
         }
-        finally {
-            synchronized (this) {
-                mappingsReady = true;
-
-                notifyAll();
-            }
-        }
     }
 
     /**
@@ -889,224 +913,234 @@ private synchronized void map0(
         boolean remap,
         boolean topLocked
     ) throws IgniteCheckedException {
-        AffinityTopologyVersion topVer = this.topVer;
+        try {
+            AffinityTopologyVersion topVer = this.topVer;
 
-        assert topVer != null;
+            assert topVer != null;
 
-        assert topVer.topologyVersion() > 0;
+            assert topVer.topologyVersion() > 0;
 
-        if (CU.affinityNodes(cctx, topVer).isEmpty()) {
-            onDone(new ClusterTopologyServerNotFoundException("Failed to map keys for cache " +
-                "(all partition nodes left the grid): " + cctx.name()));
+            if (CU.affinityNodes(cctx, topVer).isEmpty()) {
+                onDone(new ClusterTopologyServerNotFoundException("Failed to map keys for cache " +
+                    "(all partition nodes left the grid): " + cctx.name()));
 
-            return;
-        }
+                return;
+            }
 
-        boolean clientNode = cctx.kernalContext().clientNode();
+            boolean clientNode = cctx.kernalContext().clientNode();
 
-        assert !remap || (clientNode && (tx == null || !tx.hasRemoteLocks()));
+            assert !remap || (clientNode && (tx == null || !tx.hasRemoteLocks()));
 
-        // First assume this node is primary for all keys passed in.
-        if (!clientNode && mapAsPrimary(keys, topVer))
-            return;
+            // First assume this node is primary for all keys passed in.
+            if (!clientNode && mapAsPrimary(keys, topVer))
+                return;
 
-        mappings = new ArrayDeque<>();
+            mappings = new ArrayDeque<>();
 
-        // Assign keys to primary nodes.
-        GridNearLockMapping map = null;
+            // Assign keys to primary nodes.
+            GridNearLockMapping map = null;
 
-        for (KeyCacheObject key : keys) {
-            GridNearLockMapping updated = map(key, map, topVer);
+            for (KeyCacheObject key : keys) {
+                GridNearLockMapping updated = map(key, map, topVer);
 
-            // If new mapping was created, add to collection.
-            if (updated != map) {
-                mappings.add(updated);
+                // If new mapping was created, add to collection.
+                if (updated != map) {
+                    mappings.add(updated);
 
-                if (tx != null && updated.node().isLocal())
-                    tx.colocatedLocallyMapped(true);
-            }
+                    if (tx != null && updated.node().isLocal())
+                        tx.colocatedLocallyMapped(true);
+                }
 
-            map = updated;
-        }
+                map = updated;
+            }
 
-        if (isDone()) {
-            if (log.isDebugEnabled())
-                log.debug("Abandoning (re)map because future is done: " + this);
+            if (isDone()) {
+                if (log.isDebugEnabled())
+                    log.debug("Abandoning (re)map because future is done: " + this);
 
-            return;
-        }
+                return;
+            }
 
-        if (log.isDebugEnabled())
-            log.debug("Starting (re)map for mappings [mappings=" + mappings + ", fut=" + this + ']');
+            if (log.isDebugEnabled())
+                log.debug("Starting (re)map for mappings [mappings=" + mappings + ", fut=" + this + ']');
 
-        boolean hasRmtNodes = false;
+            boolean hasRmtNodes = false;
 
-        boolean first = true;
+            boolean first = true;
 
-        // Create mini futures.
-        for (Iterator<GridNearLockMapping> iter = mappings.iterator(); iter.hasNext(); ) {
-            GridNearLockMapping mapping = iter.next();
+            // Create mini futures.
+            for (Iterator<GridNearLockMapping> iter = mappings.iterator(); iter.hasNext(); ) {
+                GridNearLockMapping mapping = iter.next();
 
-            ClusterNode node = mapping.node();
-            Collection<KeyCacheObject> mappedKeys = mapping.mappedKeys();
+                ClusterNode node = mapping.node();
+                Collection<KeyCacheObject> mappedKeys = mapping.mappedKeys();
 
-            boolean loc = node.equals(cctx.localNode());
+                boolean loc = node.equals(cctx.localNode());
 
-            assert !mappedKeys.isEmpty();
+                assert !mappedKeys.isEmpty();
 
-            GridNearLockRequest req = null;
+                GridNearLockRequest req = null;
 
-            Collection<KeyCacheObject> distributedKeys = new ArrayList<>(mappedKeys.size());
+                Collection<KeyCacheObject> distributedKeys = new ArrayList<>(mappedKeys.size());
 
-            for (KeyCacheObject key : mappedKeys) {
-                IgniteTxKey txKey = cctx.txKey(key);
+                for (KeyCacheObject key : mappedKeys) {
+                    IgniteTxKey txKey = cctx.txKey(key);
 
-                GridDistributedCacheEntry entry = null;
+                    GridDistributedCacheEntry entry = null;
 
-                if (tx != null) {
-                    IgniteTxEntry txEntry = tx.entry(txKey);
+                    if (tx != null) {
+                        IgniteTxEntry txEntry = tx.entry(txKey);
 
-                    if (txEntry != null) {
-                        entry = (GridDistributedCacheEntry)txEntry.cached();
+                        if (txEntry != null) {
+                            entry = (GridDistributedCacheEntry)txEntry.cached();
 
-                        if (entry != null && loc == entry.detached()) {
-                            entry = cctx.colocated().entryExx(key, topVer, true);
+                            if (entry != null && loc == entry.detached()) {
+                                entry = cctx.colocated().entryExx(key, topVer, true);
 
-                            txEntry.cached(entry);
+                                txEntry.cached(entry);
+                            }
                         }
                     }
-                }
 
-                boolean explicit;
+                    boolean explicit;
 
-                while (true) {
-                    try {
-                        if (entry == null)
-                            entry = cctx.colocated().entryExx(key, topVer, true);
+                    while (true) {
+                        try {
+                            if (entry == null)
+                                entry = cctx.colocated().entryExx(key, topVer, true);
 
-                        if (!cctx.isAll(entry, filter)) {
-                            if (log.isDebugEnabled())
-                                log.debug("Entry being locked did not pass filter (will not lock): " + entry);
+                            if (!cctx.isAll(entry, filter)) {
+                                if (log.isDebugEnabled())
+                                    log.debug("Entry being locked did not pass filter (will not lock): " + entry);
 
-                            onComplete(false, false);
+                                onComplete(false, false);
 
-                            return;
-                        }
+                                return;
+                            }
 
-                        assert loc ^ entry.detached() : "Invalid entry [loc=" + loc + ", entry=" + entry + ']';
+                            assert loc ^ entry.detached() : "Invalid entry [loc=" + loc + ", entry=" + entry + ']';
 
-                        GridCacheMvccCandidate cand = addEntry(entry);
+                            GridCacheMvccCandidate cand = addEntry(entry);
 
-                        // Will either return value from dht cache or null if this is a miss.
-                        IgniteBiTuple<GridCacheVersion, CacheObject> val = entry.detached() ? null :
-                            ((GridDhtCacheEntry)entry).versionedValue(topVer);
+                            // Will either return value from dht cache or null if this is a miss.
+                            IgniteBiTuple<GridCacheVersion, CacheObject> val = entry.detached() ? null :
+                                ((GridDhtCacheEntry)entry).versionedValue(topVer);
 
-                        GridCacheVersion dhtVer = null;
+                            GridCacheVersion dhtVer = null;
 
-                        if (val != null) {
-                            dhtVer = val.get1();
+                            if (val != null) {
+                                dhtVer = val.get1();
 
-                            valMap.put(key, val);
-                        }
+                                valMap.put(key, val);
+                            }
 
-                        if (cand != null && !cand.reentry()) {
-                            if (req == null) {
-                                boolean clientFirst = false;
+                            if (cand != null && !cand.reentry()) {
+                                if (req == null) {
+                                    boolean clientFirst = false;
+
+                                    if (first) {
+                                        clientFirst = clientNode &&
+                                            !topLocked &&
+                                            (tx == null || !tx.hasRemoteLocks());
+
+                                        first = false;
+                                    }
+
+                                    assert !implicitTx() && !implicitSingleTx() : tx;
+
+                                    req = new GridNearLockRequest(
+                                        cctx.cacheId(),
+                                        topVer,
+                                        cctx.nodeId(),
+                                        threadId,
+                                        futId,
+                                        lockVer,
+                                        inTx(),
+                                        read,
+                                        retval,
+                                        isolation(),
+                                        isInvalidate(),
+                                        timeout,
+                                        mappedKeys.size(),
+                                        inTx() ? tx.size() : mappedKeys.size(),
+                                        inTx() && tx.syncMode() == FULL_SYNC,
+                                        inTx() ? tx.subjectId() : null,
+                                        inTx() ? tx.taskNameHash() : 0,
+                                        read ? createTtl : -1L,
+                                        read ? accessTtl : -1L,
+                                        skipStore,
+                                        keepBinary,
+                                        clientFirst,
+                                        false,
+                                        cctx.deploymentEnabled());
+
+                                    mapping.request(req);
+                                }
 
-                                if (first) {
-                                    clientFirst = clientNode &&
-                                        !topLocked &&
-                                        (tx == null || !tx.hasRemoteLocks());
+                                distributedKeys.add(key);
 
-                                    first = false;
-                                }
+                                if (tx != null)
+                                    tx.addKeyMapping(txKey, mapping.node());
 
-                                assert !implicitTx() && !implicitSingleTx() : tx;
-
-                                req = new GridNearLockRequest(
-                                    cctx.cacheId(),
-                                    topVer,
-                                    cctx.nodeId(),
-                                    threadId,
-                                    futId,
-                                    lockVer,
-                                    inTx(),
-                                    read,
+                                req.addKeyBytes(
+                                    key,
                                     retval,
-                                    isolation(),
-                                    isInvalidate(),
-                                    timeout,
-                                    mappedKeys.size(),
-                                    inTx() ? tx.size() : mappedKeys.size(),
-                                    inTx() && tx.syncMode() == FULL_SYNC,
-                                    inTx() ? tx.subjectId() : null,
-                                    inTx() ? tx.taskNameHash() : 0,
-                                    read ? createTtl : -1L,
-                                    read ? accessTtl : -1L,
-                                    skipStore,
-                                    keepBinary,
-                                    clientFirst,
-                                    false,
-                                    cctx.deploymentEnabled());
-
-                                mapping.request(req);
+                                    dhtVer, // Include DHT version to match remote DHT entry.
+                                    cctx);
                             }
 
-                            distributedKeys.add(key);
+                            explicit = inTx() && cand == null;
 
-                            if (tx != null)
+                            if (explicit)
                                 tx.addKeyMapping(txKey, mapping.node());
 
-                            req.addKeyBytes(
-                                key,
-                                retval,
-                                dhtVer, // Include DHT version to match remote DHT entry.
-                                cctx);
+                            break;
                         }
+                        catch (GridCacheEntryRemovedException ignored) {
+                            if (log.isDebugEnabled())
+                                log.debug("Got removed entry in lockAsync(..) method (will retry): " + entry);
 
-                        explicit = inTx() && cand == null;
-
-                        if (explicit)
-                            tx.addKeyMapping(txKey, mapping.node());
-
-                        break;
+                            entry = null;
+                        }
                     }
-                    catch (GridCacheEntryRemovedException ignored) {
-                        if (log.isDebugEnabled())
-                            log.debug("Got removed entry in lockAsync(..) method (will retry): " + entry);
 
-                        entry = null;
+                    // Mark mapping explicit lock flag.
+                    if (explicit) {
+                        boolean marked = tx != null && tx.markExplicit(node.id());
+
+                        assert tx == null || marked;
                     }
                 }
 
-                // Mark mapping explicit lock flag.
-                if (explicit) {
-                    boolean marked = tx != null && tx.markExplicit(node.id());
+                if (!distributedKeys.isEmpty()) {
+                    mapping.distributedKeys(distributedKeys);
 
-                    assert tx == null || marked;
+                    hasRmtNodes |= !mapping.node().isLocal();
                 }
-            }
-
-            if (!distributedKeys.isEmpty()) {
-                mapping.distributedKeys(distributedKeys);
+                else {
+                    assert mapping.request() == null;
 
-                hasRmtNodes |= !mapping.node().isLocal();
+                    iter.remove();
+                }
             }
-            else {
-                assert mapping.request() == null;
 
-                iter.remove();
+            if (hasRmtNodes) {
+                trackable = true;
+
+                if (!remap && !cctx.mvcc().addFuture(this))
+                    throw new IllegalStateException("Duplicate future ID: " + this);
             }
+            else
+                trackable = false;
         }
+        finally {
+            /** Notify ready {@link mappings} waiters. See {@link #cancel()} */
+            if (tx != null) {
+                mappingsReady = true;
 
-        if (hasRmtNodes) {
-            trackable = true;
-
-            if (!remap && !cctx.mvcc().addFuture(this))
-                throw new IllegalStateException("Duplicate future ID: " + this);
+                notifyAll();
+            }
         }
-        else
-            trackable = false;
 
         proceedMapping();
     }
@@ -1136,13 +1170,12 @@ private void proceedMapping0()
         throws IgniteCheckedException {
         GridNearLockMapping map;
 
-        synchronized (this) {
-            map = mappings.poll();
-        }
+        // Fail fast if future is completed (in case of async rollback)
+        if (isDone()) {
+            clear();
 
-        // If there are no more mappings to process or prepare has timed out, complete the future.
-        if (map == null)
             return;
+        }
 
         // Fail fast if the transaction is timed out.
         if (tx != null && tx.remainingTime() == -1) {
@@ -1153,6 +1186,14 @@ private void proceedMapping0()
             return;
         }
 
+        synchronized (this) {
+            map = mappings.poll();
+        }
+
+        // If there are no more mappings to process or prepare has timed out, complete the future.
+        if (map == null)
+            return;
+
         final GridNearLockRequest req = map.request();
         final Collection<KeyCacheObject> mappedKeys = map.distributedKeys();
         final ClusterNode node = map.node();
@@ -1436,59 +1477,20 @@ private ClusterTopologyCheckedException newTopologyException(@Nullable Throwable
     }
 
     /**
-     * @param fut Future.
-     * @param clo Closure.
+     * @param e Exception.
+     * @param timedOut {@code True} if timed out.
      */
-    protected void applyWhenReady(final IgniteInternalFuture<?> fut, GridAbsClosureX clo) {
-        long remaining = tx.remainingTime();
+    private boolean errorOrTimeoutOnTopologyVersion(IgniteCheckedException e, boolean timedOut) {
+        if (e != null || timedOut) {
+            // Can timeout only if tx is not null.
+            assert e != null || tx != null : "Timeout is possible only in transaction";
 
-        if (remaining == -1) {
-            onDone(tx.timeoutException());
+            onDone(e == null ? tx.timeoutException() : e);
 
-            return;
-        }
-
-        if (fut == null || fut.isDone()) {
-            try {
-                clo.applyx();
-            }
-            catch (IgniteCheckedException e) {
-                onDone(e);
-            }
+            return true;
         }
-        else {
-            RemapTimeoutObject timeoutObj = null;
-
-            AtomicBoolean state = new AtomicBoolean();
-
-            if (remaining > 0) {
-                timeoutObj = new RemapTimeoutObject(remaining, fut, state);
-
-                cctx.time().addTimeoutObject(timeoutObj);
-            }
-
-            final RemapTimeoutObject finalTimeoutObj = timeoutObj;
 
-            fut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
-                @Override public void apply(IgniteInternalFuture<?> fut) {
-                    if (!state.compareAndSet(false, true))
-                        return;
-
-                    try {
-                        fut.get();
-
-                        clo.applyx();
-                    }
-                    catch (IgniteCheckedException e) {
-                        onDone(e);
-                    }
-                    finally {
-                        if (finalTimeoutObj != null)
-                            cctx.time().removeTimeoutObject(finalTimeoutObj);
-                    }
-                }
-            });
-        }
+        return false;
     }
 
     /**
@@ -1543,15 +1545,20 @@ private class LockTimeoutObject extends GridTimeoutObjectAdapter {
                                 U.warn(log, "Failed to detect deadlock.", e);
                             }
 
-                            onComplete(false, true);
+                            synchronized (LockTimeoutObject.this) {
+                                onComplete(false, true);
+                            }
                         }
                     });
                 }
                 else
                     err = tx.timeoutException();
             }
-            else
-                onComplete(false, true);
+            else {
+                synchronized (this) {
+                    onComplete(false, true);
+                }
+            }
         }
 
         /** {@inheritDoc} */
@@ -1684,14 +1691,15 @@ void onResult(GridNearLockResponse res) {
                 IgniteInternalFuture<?> affFut =
                     cctx.shared().exchange().affinityReadyFuture(res.clientRemapVersion());
 
-                applyWhenReady(affFut, new GridAbsClosureX() {
-                    @Override public void applyx() throws IgniteCheckedException {
-                        try {
-                            remap();
-                        }
-                        finally {
-                            cctx.shared().txContextReset();
-                        }
+                cctx.time().waitAsync(affFut, tx == null ? 0 : tx.remainingTime(), (e, timedOut) -> {
+                    if (errorOrTimeoutOnTopologyVersion(e, timedOut))
+                        return;
+
+                    try {
+                        remap();
+                    }
+                    finally {
+                        cctx.shared().txContextReset();
                     }
                 });
             }
@@ -1791,39 +1799,4 @@ private void remap() {
             return S.toString(MiniFuture.class, this, "node", node.id(), "super", super.toString());
         }
     }
-
-    /**
-     *
-     */
-    private class RemapTimeoutObject extends GridTimeoutObjectAdapter {
-        /** */
-        private final IgniteInternalFuture<?> fut;
-
-        /** */
-        private final AtomicBoolean state;
-
-        /**
-         * @param timeout Timeout.
-         * @param fut Future.
-         * @param state State.
-         */
-        RemapTimeoutObject(long timeout, IgniteInternalFuture<?> fut, AtomicBoolean state) {
-            super(timeout);
-
-            this.fut = fut;
-
-            this.state = state;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void onTimeout() {
-            if (!fut.isDone() && state.compareAndSet(false, true))
-                onDone(tx.timeoutException());
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return S.toString(RemapTimeoutObject.class, this);
-        }
-    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
index 38055e71eb625..7e85e0592eaf8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticSerializableTxPrepareFuture.java
@@ -45,7 +45,6 @@
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.lang.GridAbsClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.C1;
@@ -55,6 +54,7 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteReducer;
 import org.jetbrains.annotations.Nullable;
 
@@ -913,8 +913,11 @@ void onResult(final GridNearTxPrepareResponse res, boolean updateMapping) {
                                                 parent.remapFut = null;
                                             }
 
-                                            parent.applyWhenReady(affFut, new GridAbsClosureX() {
-                                                @Override public void applyx() throws IgniteCheckedException {
+                                            parent.cctx.time().waitAsync(affFut, parent.tx.remainingTime(), new IgniteBiInClosure<IgniteCheckedException, Boolean>() {
+                                                @Override public void apply(IgniteCheckedException e, Boolean timedOut) {
+                                                    if (parent.errorOrTimeoutOnTopologyVersion(e, timedOut))
+                                                        return;
+
                                                     remap(res);
                                                 }
                                             });
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 820037507eb5a..8e10ad3ac5cd1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -52,7 +52,6 @@
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.lang.GridAbsClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.C1;
@@ -990,10 +989,11 @@ void onResult(final GridNearTxPrepareResponse res) {
                         IgniteInternalFuture<?> affFut =
                             parent.cctx.exchange().affinityReadyFuture(res.clientRemapVersion());
 
-                        parent.applyWhenReady(affFut, new GridAbsClosureX() {
-                            @Override public void applyx() throws IgniteCheckedException {
-                                remap();
-                            }
+                        parent.cctx.time().waitAsync(affFut, parent.tx.remainingTime(), (e, timedOut) -> {
+                            if (parent.errorOrTimeoutOnTopologyVersion(e, timedOut))
+                                return;
+
+                            remap();
                         });
                     }
                     else {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
index ecc02c23e3a7b..28f0184e4cc0b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFutureAdapter.java
@@ -18,17 +18,13 @@
 package org.apache.ignite.internal.processors.cache.distributed.near;
 
 import java.util.Collection;
-import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
-import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.lang.GridAbsClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteInClosure;
@@ -142,14 +138,15 @@ protected final void prepareOnTopology(final boolean remap, @Nullable final Runn
                 c.run();
         }
         else {
-            applyWhenReady(topFut, new GridAbsClosureX() {
-                @Override public void applyx() throws IgniteCheckedException {
-                    try {
-                        prepareOnTopology(remap, c);
-                    }
-                    finally {
-                        cctx.txContextReset();
-                    }
+            cctx.time().waitAsync(topFut, tx.remainingTime(), (e, timedOut) -> {
+                if (errorOrTimeoutOnTopologyVersion(e, timedOut))
+                    return;
+
+                try {
+                    prepareOnTopology(remap, c);
+                }
+                finally {
+                    cctx.txContextReset();
                 }
             });
         }
@@ -162,67 +159,22 @@ protected final void prepareOnTopology(final boolean remap, @Nullable final Runn
     protected abstract void prepare0(boolean remap, boolean topLocked);
 
     /**
-     * @param fut Future.
-     * @param clo Closure.
+     * @param e Exception.
+     * @param timedOut {@code True} if timed out.
      */
-    protected void applyWhenReady(final IgniteInternalFuture<?> fut, GridAbsClosureX clo) {
-        long remaining = tx.remainingTime();
-
-        if (remaining == -1) {
-            IgniteCheckedException e = tx.timeoutException();
+    protected boolean errorOrTimeoutOnTopologyVersion(IgniteCheckedException e, boolean timedOut) {
+        if (e != null || timedOut) {
+            if (timedOut)
+                e = tx.timeoutException();
 
             ERR_UPD.compareAndSet(this, null, e);
 
             onDone(e);
 
-            return;
-        }
-
-        if (fut == null || fut.isDone()) {
-            try {
-                clo.applyx();
-            }
-            catch (IgniteCheckedException e) {
-                ERR_UPD.compareAndSet(this, null, e);
-
-                onDone(e);
-            }
+            return true;
         }
-        else {
-            RemapTimeoutObject timeoutObj = null;
-
-            AtomicBoolean state = new AtomicBoolean();
-
-            if (remaining > 0) {
-                timeoutObj = new RemapTimeoutObject(remaining, fut, state);
 
-                cctx.time().addTimeoutObject(timeoutObj);
-            }
-
-            final RemapTimeoutObject finalTimeoutObj = timeoutObj;
-
-            fut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
-                @Override public void apply(IgniteInternalFuture<?> fut) {
-                    if (!state.compareAndSet(false, true))
-                        return;
-
-                    try {
-                        fut.get();
-
-                        clo.applyx();
-                    }
-                    catch (IgniteCheckedException e) {
-                        ERR_UPD.compareAndSet(GridNearOptimisticTxPrepareFutureAdapter.this, null, e);
-
-                        onDone(e);
-                    }
-                    finally {
-                        if (finalTimeoutObj != null)
-                            cctx.time().removeTimeoutObject(finalTimeoutObj);
-                    }
-                }
-            });
-        }
+        return false;
     }
 
     /**
@@ -288,44 +240,4 @@ private boolean checkLocks() {
             return S.toString(KeyLockFuture.class, this, super.toString());
         }
     }
-
-    /**
-     *
-     */
-    private class RemapTimeoutObject extends GridTimeoutObjectAdapter {
-        /** */
-        private final IgniteInternalFuture<?> fut;
-
-        /** */
-        private final AtomicBoolean state;
-
-        /**
-         * @param timeout Timeout.
-         * @param fut Future.
-         * @param state State.
-         */
-        RemapTimeoutObject(long timeout, IgniteInternalFuture<?> fut, AtomicBoolean state) {
-            super(timeout);
-
-            this.fut = fut;
-
-            this.state = state;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void onTimeout() {
-            if (!fut.isDone() && state.compareAndSet(false, true)) {
-                IgniteCheckedException e = tx.timeoutException();
-
-                ERR_UPD.compareAndSet(GridNearOptimisticTxPrepareFutureAdapter.this, null, e);
-
-                onDone(e);
-            }
-        }
-
-        /** {@inheritDoc} */
-        @Override public String toString() {
-            return S.toString(RemapTimeoutObject.class, this);
-        }
-    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 936b51597511b..297456350dff9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -412,13 +412,13 @@ public void finish(final boolean commit, final boolean clearThreadMap, final boo
         }
 
         if (!commit && !clearThreadMap)
-            rollbackAsyncSafe(onTimeout); // Asynchronous rollback.
+            rollbackAsyncSafe(onTimeout);
         else
             doFinish(commit, clearThreadMap);
     }
 
     /**
-     * Roll back tx when it's safe.
+     * Rollback tx when it's safe.
      * If current future is not lock future (enlist future) wait until completion and tries again.
      * Else cancel lock future (onTimeout=false) or wait for completion due to deadlock detection (onTimeout=true).
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 5b82333ac8a80..f25dc386704a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -67,6 +68,7 @@
 import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
+import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.typedef.C1;
@@ -548,7 +550,7 @@ public IgniteInternalFuture<GridNearTxPrepareResponse> prepareNearTxLocal(final
     /**
      * @param node Sender node.
      * @param req Request.
-     * @return {@code True} if update will be retried from future listener.
+     * @return {@code True} if update will be retried from future listener or topology version future is timed out.
      */
     private boolean waitForExchangeFuture(final ClusterNode node, final GridNearTxPrepareRequest req) {
         assert req.firstClientRequest() : req;
@@ -562,27 +564,37 @@ private boolean waitForExchangeFuture(final ClusterNode node, final GridNearTxPr
                 final IgniteThread thread = (IgniteThread)curThread;
 
                 if (thread.cachePoolThread()) {
-                    topFut.listen(new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
-                        @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> fut) {
-                            ctx.kernalContext().closure().runLocalWithThreadPolicy(thread, new Runnable() {
-                                @Override public void run() {
-                                    try {
-                                        processNearTxPrepareRequest0(node, req);
-                                    }
-                                    finally {
-                                        ctx.io().onMessageProcessed(req);
-                                    }
+                    ctx.time().waitAsync(topFut, req.timeout(), (e, timedOut) -> {
+                            if (e != null || timedOut) {
+                                sendResponseOnTimeoutOrError(e, topFut, node, req);
+
+                                return;
+                            }
+                            ctx.kernalContext().closure().runLocalWithThreadPolicy(thread, () -> {
+                                try {
+                                    processNearTxPrepareRequest0(node, req);
+                                }
+                                finally {
+                                    ctx.io().onMessageProcessed(req);
                                 }
                             });
                         }
-                    });
+                    );
 
                     return true;
                 }
             }
 
             try {
-                topFut.get();
+                if (req.timeout() > 0)
+                    topFut.get(req.timeout());
+                else
+                    topFut.get();
+            }
+            catch (IgniteFutureTimeoutCheckedException e) {
+                sendResponseOnTimeoutOrError(null, topFut, node, req);
+
+                return true;
             }
             catch (IgniteCheckedException e) {
                 U.error(log, "Topology future failed: " + e, e);
@@ -592,6 +604,48 @@ private boolean waitForExchangeFuture(final ClusterNode node, final GridNearTxPr
         return false;
     }
 
+    /**
+     * @param e Exception or null if timed out.
+     * @param topFut Topology future.
+     * @param node Node.
+     * @param req Prepare request.
+     */
+    private void sendResponseOnTimeoutOrError(@Nullable IgniteCheckedException e,
+        GridDhtTopologyFuture topFut,
+        ClusterNode node,
+        GridNearTxPrepareRequest req) {
+        if (e == null)
+            e = new IgniteTxTimeoutCheckedException("Failed to wait topology version for near prepare " +
+                "[txId=" + req.version() +
+                ", topVer=" + topFut.initialVersion() +
+                ", node=" + node.id() +
+                ", req=" + req + ']');
+
+        GridNearTxPrepareResponse res = new GridNearTxPrepareResponse(
+            req.partition(),
+            req.version(),
+            req.futureId(),
+            req.miniId(),
+            req.version(),
+            req.version(),
+            null,
+            e,
+            null,
+            req.onePhaseCommit(),
+            req.deployInfo() != null);
+
+        try {
+            ctx.io().send(node.id(), res, req.policy());
+        }
+        catch (IgniteCheckedException e0) {
+            U.error(txPrepareMsgLog, "Failed to send wait topology version response for near prepare " +
+                "[txId=" + req.version() +
+                ", topVer=" + topFut.initialVersion() +
+                ", node=" + node.id() +
+                ", req=" + req + ']', e0);
+        }
+    }
+
     /**
      * @param expVer Expected topology version.
      * @param curVer Current topology version.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index 25151cf7221bf..de9e8eb8c0a68 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -20,9 +20,11 @@
 import java.io.Closeable;
 import java.util.Comparator;
 import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
 import org.apache.ignite.internal.util.GridConcurrentSkipListSet;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -30,6 +32,8 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.lang.IgniteBiInClosure;
+import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.thread.IgniteThread;
 
@@ -137,6 +141,57 @@ public boolean removeTimeoutObject(GridTimeoutObject timeoutObj) {
         return timeoutObjs.remove(timeoutObj);
     }
 
+    /**
+     * Wait for a future (listen with timeout).
+     * @param fut Future.
+     * @param timeout Timeout millis. -1 means expired timeout, 0 - no timeout.
+     * @param clo Finish closure. First argument contains error on future or null if no errors,
+     * second is {@code true} if wait timed out.
+     */
+    public void waitAsync(final IgniteInternalFuture<?> fut,
+        long timeout,
+        IgniteBiInClosure<IgniteCheckedException, Boolean> clo) {
+        if (timeout == -1) {
+            clo.apply(null, false);
+
+            return;
+        }
+
+        if (fut == null || fut.isDone())
+            clo.apply(null, false);
+        else {
+            WaitFutureTimeoutObject timeoutObj = null;
+
+            if (timeout > 0) {
+                timeoutObj = new WaitFutureTimeoutObject(fut, timeout, clo);
+
+                addTimeoutObject(timeoutObj);
+            }
+
+            final WaitFutureTimeoutObject finalTimeoutObj = timeoutObj;
+
+            fut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
+                @Override public void apply(IgniteInternalFuture<?> fut) {
+                    if (finalTimeoutObj != null && !finalTimeoutObj.finishGuard.compareAndSet(false, true))
+                        return;
+
+                    try {
+                        fut.get();
+
+                        clo.apply(null, false);
+                    }
+                    catch (IgniteCheckedException e) {
+                        clo.apply(e, false);
+                    }
+                    finally {
+                        if (finalTimeoutObj != null)
+                            removeTimeoutObject(finalTimeoutObj);
+                    }
+                }
+            });
+        }
+    }
+
     /**
      * Handles job timeouts.
      */
@@ -309,4 +364,43 @@ public class CancelableTask implements GridTimeoutObject, Closeable {
             return S.toString(CancelableTask.class, this);
         }
     }
+
+    /**
+     *
+     */
+    private static class WaitFutureTimeoutObject extends GridTimeoutObjectAdapter {
+        /** */
+        private final IgniteInternalFuture<?> fut;
+
+        /** */
+        private final AtomicBoolean finishGuard = new AtomicBoolean();
+
+        /** */
+        private final IgniteBiInClosure<IgniteCheckedException, Boolean> clo;
+
+        /**
+         * @param fut Future.
+         * @param timeout Timeout.
+         * @param clo Closure to call on timeout.
+         */
+        WaitFutureTimeoutObject(IgniteInternalFuture<?> fut, long timeout,
+            IgniteBiInClosure<IgniteCheckedException, Boolean> clo) {
+            super(timeout);
+
+            this.fut = fut;
+
+            this.clo = clo;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onTimeout() {
+            if (!fut.isDone() && finishGuard.compareAndSet(false, true))
+                clo.apply(null, true);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(WaitFutureTimeoutObject.class, this);
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
index 6971d7c14060e..be60a5e3033e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxInfo.java
@@ -26,6 +26,7 @@
 import java.util.Collection;
 import java.util.TimeZone;
 import java.util.UUID;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -84,6 +85,9 @@ public class VisorTxInfo extends VisorDataTransferObject {
     /** */
     private Collection<UUID> masterNodeIds;
 
+    /** */
+    private AffinityTopologyVersion topVer;
+
     /**
      * Default constructor.
      */
@@ -105,7 +109,7 @@ public VisorTxInfo() {
      */
     public VisorTxInfo(IgniteUuid xid, long startTime, long duration, TransactionIsolation isolation,
         TransactionConcurrency concurrency, long timeout, String lb, Collection<UUID> primaryNodes,
-        TransactionState state, int size, IgniteUuid nearXid, Collection<UUID> masterNodeIds) {
+        TransactionState state, int size, IgniteUuid nearXid, Collection<UUID> masterNodeIds, AffinityTopologyVersion topVer) {
         this.xid = xid;
         this.startTime = startTime;
         this.duration = duration;
@@ -118,11 +122,12 @@ public VisorTxInfo(IgniteUuid xid, long startTime, long duration, TransactionIso
         this.size = size;
         this.nearXid = nearXid;
         this.masterNodeIds = masterNodeIds;
+        this.topVer = topVer;
     }
 
     /** {@inheritDoc} */
     @Override public byte getProtocolVersion() {
-        return V2;
+        return V3;
     }
 
     /** */
@@ -155,6 +160,11 @@ public TransactionConcurrency getConcurrency() {
         return concurrency;
     }
 
+    /** */
+    public AffinityTopologyVersion getTopologyVersion() {
+        return topVer;
+    }
+
     /** */
     public long getTimeout() {
         return timeout;
@@ -204,6 +214,8 @@ public int getSize() {
         U.writeGridUuid(out, nearXid);
         U.writeCollection(out, masterNodeIds);
         out.writeLong(startTime);
+        out.writeLong(topVer == null ? -1 : topVer.topologyVersion());
+        out.writeInt(topVer == null ? -1 : topVer.minorTopologyVersion());
     }
 
     /** {@inheritDoc} */
@@ -219,12 +231,16 @@ public int getSize() {
         size = in.readInt();
         if (protoVer >= V2) {
             nearXid = U.readGridUuid(in);
-
             masterNodeIds = U.readCollection(in);
-
             startTime = in.readLong();
         }
+        if (protoVer >= V3) {
+            long topVer = in.readLong();
+            int minorTopVer = in.readInt();
 
+            if (topVer != -1)
+                this.topVer = new AffinityTopologyVersion(topVer, minorTopVer);
+        }
     }
 
     /**
@@ -240,6 +256,7 @@ public String toUserString() {
             ", duration=" + getDuration() / 1000 +
             ", isolation=" + getIsolation() +
             ", concurrency=" + getConcurrency() +
+            ", topVer=" + (getTopologyVersion() == null ? "N/A" : getTopologyVersion()) +
             ", timeout=" + getTimeout() +
             ", size=" + getSize() +
             ", dhtNodes=" + (getPrimaryNodes() == null ? "N/A" :
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
index 3f35a26711994..23d1663ef79d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/tx/VisorTxTask.java
@@ -312,7 +312,7 @@ else if (locTx instanceof GridDhtTxRemote) {
 
                 infos.add(new VisorTxInfo(locTx.xid(), locTx.startTime(), duration, locTx.isolation(), locTx.concurrency(),
                     locTx.timeout(), lb, mappings, locTx.state(),
-                    size, locTx.nearXidVersion().asGridUuid(), locTx.masterNodeIds()));
+                    size, locTx.nearXidVersion().asGridUuid(), locTx.masterNodeIds(), locTx.topologyVersionSnapshot()));
 
                 if (arg.getOperation() == VisorTxOperation.KILL)
                     killClo.apply(locTx, tm);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
index 4eda30c4ae885..f2e46d9ea6ccd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
@@ -42,24 +42,34 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.GridCacheFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.tx.VisorTxInfo;
+import org.apache.ignite.internal.visor.tx.VisorTxOperation;
+import org.apache.ignite.internal.visor.tx.VisorTxTask;
+import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
+import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -75,6 +85,7 @@
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
 import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
 import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
 import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
 import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
@@ -861,6 +872,122 @@ public void testRollbackProxy() throws Exception {
         }
     }
 
+    /**
+     *
+     */
+    public void testRollbackOnTopologyLockPessimistic() throws Exception {
+        final Ignite client = startClient();
+
+        Ignite crd = grid(0);
+
+        List<Integer> keys = primaryKeys(grid(1).cache(CACHE_NAME), 1);
+
+        assertTrue(crd.cluster().localNode().order() == 1);
+
+        CountDownLatch txLatch = new CountDownLatch(1);
+        CountDownLatch tx2Latch = new CountDownLatch(1);
+        CountDownLatch commitLatch = new CountDownLatch(1);
+
+        // Start tx holding topology.
+        IgniteInternalFuture txFut = runAsync(new Runnable() {
+            @Override public void run() {
+                List<Integer> keys = primaryKeys(grid(0).cache(CACHE_NAME), 1);
+
+                try (Transaction tx = client.transactions().txStart()) {
+                    client.cache(CACHE_NAME).put(keys.get(0), 0);
+
+                    txLatch.countDown();
+
+                    U.awaitQuiet(commitLatch);
+
+                    tx.commit();
+
+                    fail();
+                }
+                catch (Exception e) {
+                    // Expected.
+                }
+            }
+        });
+
+        U.awaitQuiet(txLatch);
+
+        crd.events().localListen(new IgnitePredicate<Event>() {
+            @Override public boolean apply(Event evt) {
+                runAsync(new Runnable() {
+                    @Override public void run() {
+                        try(Transaction tx = crd.transactions().withLabel("testLbl").txStart()) {
+                            // Wait for node start.
+                            waitForCondition(new GridAbsPredicate() {
+                                @Override public boolean apply() {
+                                    return crd.cluster().topologyVersion() != GRID_CNT +
+                                        /** client node */ 1  + /** stop server node */ 1 + /** start server node */ 1;
+                                }
+                            }, 10_000);
+
+                            tx2Latch.countDown();
+
+                            crd.cache(CACHE_NAME).put(keys.get(0), 0);
+
+                            tx.commit();
+
+                            fail();
+                        }
+                        catch (Exception e) {
+                            // Expected.
+                        }
+                    }
+                });
+
+                return false;
+            }
+        }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
+
+        IgniteInternalFuture restartFut = runAsync(new Runnable() {
+            @Override public void run() {
+                stopGrid(2);
+
+                try {
+                    startGrid(2);
+                }
+                catch (Exception e) {
+                    fail();
+                }
+            }
+        });
+
+        U.awaitQuiet(tx2Latch);
+
+        // Rollback tx using kill task.
+        VisorTxTaskArg arg =
+            new VisorTxTaskArg(VisorTxOperation.KILL, null, null, null, null, null, null, null, null, null);
+
+        Map<ClusterNode, VisorTxTaskResult> res = client.compute(client.cluster().forPredicate(F.alwaysTrue())).
+            execute(new VisorTxTask(), new VisorTaskArgument<>(client.cluster().localNode().id(), arg, false));
+
+        int expCnt = 0;
+
+        for (Map.Entry<ClusterNode, VisorTxTaskResult> entry : res.entrySet()) {
+            if (entry.getValue().getInfos().isEmpty())
+                continue;
+
+            for (VisorTxInfo info : entry.getValue().getInfos()) {
+                log.info(info.toUserString());
+
+                expCnt++;
+            }
+        }
+
+        assertEquals("Expecting 2 transactions", 2, expCnt);
+
+        commitLatch.countDown();
+
+        txFut.get();
+        restartFut.get();
+
+        checkFutures();
+    }
+
     /**
      * Locks entry in tx and delays commit until signalled.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
index 47c00b5f0b929..de01844155c3b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
@@ -22,7 +22,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
-import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -37,14 +36,12 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
-import org.apache.ignite.events.Event;
-import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.GridCacheFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareResponse;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest;
@@ -54,20 +51,10 @@
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.internal.visor.VisorTaskArgument;
-import org.apache.ignite.internal.visor.tx.VisorTxInfo;
-import org.apache.ignite.internal.visor.tx.VisorTxOperation;
-import org.apache.ignite.internal.visor.tx.VisorTxTask;
-import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
-import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
-import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteInClosure;
-import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
@@ -79,6 +66,7 @@
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
 import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.testframework.GridTestUtils.runMultiThreaded;
 import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
 import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
@@ -207,7 +195,7 @@ private void lock(final Ignite node, final boolean retry) throws Exception {
 
         final int KEYS_PER_THREAD = 10_000;
 
-        GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+        runMultiThreaded(new IgniteInClosure<Integer>() {
             @Override public void apply(Integer idx) {
                 int start = idx * KEYS_PER_THREAD;
                 int end = start + KEYS_PER_THREAD;
@@ -320,7 +308,7 @@ private void deadlockUnblockedOnTimeout(final Ignite node1, final Ignite node2)
 
         final CountDownLatch l = new CountDownLatch(2);
 
-        IgniteInternalFuture<?> fut1 = GridTestUtils.runAsync(new Runnable() {
+        IgniteInternalFuture<?> fut1 = runAsync(new Runnable() {
             @Override public void run() {
                 try {
                     try (Transaction tx = node1.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 5000, 2)) {
@@ -344,7 +332,7 @@ private void deadlockUnblockedOnTimeout(final Ignite node1, final Ignite node2)
             }
         }, "First");
 
-        IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Runnable() {
+        IgniteInternalFuture<?> fut2 = runAsync(new Runnable() {
             @Override public void run() {
                 try (Transaction tx = node2.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, 2)) {
                     node2.cache(CACHE_NAME).put(2, 2);
@@ -492,7 +480,19 @@ public void testRandomMixedTxConfigurations() throws Exception {
 
         stop.set(true);
 
-        fut.get(10_000);
+        try {
+            fut.get(30_000);
+        }
+        catch (IgniteFutureTimeoutCheckedException e) {
+            error("Transactions hang", e);
+
+            for (Ignite node : G.allGrids())
+                ((IgniteKernal)node).dumpDebugInfo();
+
+            fut.cancel(); // Try to interrupt hanging threads.
+
+            throw  e;
+        }
 
         log.info("Tx test stats: started=" + cntr0.sum() +
             ", completed=" + cntr1.sum() +
@@ -538,7 +538,7 @@ public void testLockRelease() throws Exception {
                 final int idx0 = idx.getAndIncrement();
 
                 if (idx0 == 0) {
-                    try(final Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, 1)) {
+                    try (final Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, 1)) {
                         client.cache(CACHE_NAME).put(0, 0); // Lock is owned.
 
                         readStartLatch.countDown();
@@ -594,159 +594,100 @@ public void testEnlistManyWrite() throws Exception {
         testEnlistMany(true);
     }
 
-    public void testRollbackOnTopologyLockPessimistic() throws Exception {
-        final Ignite client = startClient();
-
-        Ignite crd = grid(0);
-
-        List<Integer> keys = primaryKeys(grid(1).cache(CACHE_NAME), 1);
-
-        assertTrue(crd.cluster().localNode().order() == 1);
-
-        CountDownLatch txLatch = new CountDownLatch(1);
-        CountDownLatch stopLatch = new CountDownLatch(1);
-        CountDownLatch commitLatch = new CountDownLatch(1);
-
-        // Start tx holding topology.
-        IgniteInternalFuture txFut = runAsync(new Runnable() {
-            @Override public void run() {
-                List<Integer> keys = primaryKeys(grid(0).cache(CACHE_NAME), 1);
-
-                try (Transaction tx = client.transactions().txStart()) {
-                    client.cache(CACHE_NAME).put(keys.get(0), 0);
-
-                    txLatch.countDown();
-
-                    U.awaitQuiet(commitLatch);
-
-                    tx.commit();
-
-                    fail();
-                }
-                catch (Exception e) {
-                    // Expected.
-                }
-            }
-        });
-
-        U.awaitQuiet(txLatch);
-
-        crd.events().localListen(new IgnitePredicate<Event>() {
-            @Override public boolean apply(Event evt) {
-                runAsync(new Runnable() {
-                    @Override public void run() {
-                        try(Transaction tx = crd.transactions().withLabel("testLbl").txStart()) {
-                            stopLatch.countDown();
-
-                            crd.cache(CACHE_NAME).put(keys.get(0), 0);
-
-                            tx.commit();
-
-                            fail();
-                        }
-                        catch (Exception e) {
-                            // Expected.
-                        }
-                    }
-                });
-
-                return false;
-            }
-        }, EventType.EVT_NODE_FAILED, EventType.EVT_NODE_LEFT);
-
-        IgniteInternalFuture restartFut = runAsync(new Runnable() {
-            @Override public void run() {
-                stopGrid(2);
-
-                try {
-                    startGrid(2);
-                }
-                catch (Exception e) {
-                    fail();
-                }
-            }
-        });
-
-        U.awaitQuiet(stopLatch);
-
-        doSleep(500);
-
-        VisorTxTaskArg arg =
-            new VisorTxTaskArg(VisorTxOperation.KILL, null, null, null, null, null, null, null, null, null);
-
-        Map<ClusterNode, VisorTxTaskResult> res = client.compute(client.cluster().forPredicate(F.alwaysTrue())).
-            execute(new VisorTxTask(), new VisorTaskArgument<>(client.cluster().localNode().id(), arg, false));
-
-        int expCnt = 0;
-
-        for (Map.Entry<ClusterNode, VisorTxTaskResult> entry : res.entrySet()) {
-            if (entry.getValue().getInfos().isEmpty())
-                continue;
-
-            for (VisorTxInfo info : entry.getValue().getInfos()) {
-                log.info(info.toUserString());
-
-                expCnt++;
-            }
-        }
-
-        assertEquals("Expecting 2 transactions", 2, expCnt);
-
-        commitLatch.countDown();
-
-        txFut.get();
-        restartFut.get();
-
-        checkFutures();
-    }
-
     /**
      *
      */
     public void testRollbackOnTimeoutTxRemapOptimisticReadCommitted() throws Exception {
-        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, READ_COMMITTED);
+        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, READ_COMMITTED, true);
     }
 
     /**
      *
      */
     public void testRollbackOnTimeoutTxRemapOptimisticRepeatableRead() throws Exception {
-        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, REPEATABLE_READ);
+        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, REPEATABLE_READ, true);
     }
 
     /**
      *
      */
     public void testRollbackOnTimeoutTxRemapOptimisticSerializable() throws Exception {
-        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, SERIALIZABLE);
+        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, SERIALIZABLE, true);
     }
 
     /**
      *
      */
     public void testRollbackOnTimeoutTxRemapPessimisticReadCommitted() throws Exception {
-        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, READ_COMMITTED);
+        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, READ_COMMITTED, true);
     }
 
     /**
      *
      */
     public void testRollbackOnTimeoutTxRemapPessimisticRepeatableRead() throws Exception {
-        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, REPEATABLE_READ);
+        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, REPEATABLE_READ, true);
     }
 
     /**
      *
      */
     public void testRollbackOnTimeoutTxRemapPessimisticSerializable() throws Exception {
-        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, SERIALIZABLE);
+        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, SERIALIZABLE, true);
     }
 
     /**
      *
      */
-    private void doTestRollbackOnTimeoutTxRemap(TransactionConcurrency concurrency, TransactionIsolation isolation) throws Exception {
-        Ignite client = startClient();
+    public void testRollbackOnTimeoutTxServerRemapOptimisticReadCommitted() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, READ_COMMITTED, false);
+    }
+
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxServerRemapOptimisticRepeatableRead() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, REPEATABLE_READ, false);
+    }
+
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxServerRemapOptimisticSerializable() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(OPTIMISTIC, SERIALIZABLE, false);
+    }
+
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxServerRemapPessimisticReadCommitted() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, READ_COMMITTED, false);
+    }
+
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxServerRemapPessimisticRepeatableRead() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, REPEATABLE_READ, false);
+    }
+
+    /**
+     *
+     */
+    public void testRollbackOnTimeoutTxServerRemapPessimisticSerializable() throws Exception {
+        doTestRollbackOnTimeoutTxRemap(PESSIMISTIC, SERIALIZABLE, false);
+    }
+
+
+    /**
+     * @param concurrency Concurrency.
+     * @param isolation Isolation.
+     * @param clientWait {@code True} to wait remap on client, otherwise wait remap on server.
+     */
+    private void doTestRollbackOnTimeoutTxRemap(TransactionConcurrency concurrency,
+        TransactionIsolation isolation,
+        boolean clientWait) throws Exception {
+        IgniteEx client = (IgniteEx)startClient();
 
         Ignite crd = grid(0);
 
@@ -754,21 +695,37 @@ private void doTestRollbackOnTimeoutTxRemap(TransactionConcurrency concurrency,
 
         List<Integer> keys = movingKeysAfterJoin(grid(1), CACHE_NAME, 1);
 
-        // Delay exchange finish on client node.
-        TestRecordingCommunicationSpi.spi(crd).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
-            @Override public boolean apply(ClusterNode node, Message msg) {
-                return node.equals(client.cluster().localNode()) && msg instanceof GridDhtPartitionsFullMessage;
-            }
-        });
+        // Delay exchange finish on server nodes if clientWait=true, or on all nodes otherwise (excluding joining node).
+        TestRecordingCommunicationSpi.spi(crd).blockMessages((node,
+            msg) -> node.order() < 5 && msg instanceof GridDhtPartitionsFullMessage &&
+            (!clientWait || node.order() != grid(1).cluster().localNode().order()));
 
         // Delay prepare until exchange is finished.
-        TestRecordingCommunicationSpi.spi(client).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
-            @Override public boolean apply(ClusterNode node, Message msg) {
-                return concurrency == PESSIMISTIC ?
-                    msg instanceof GridNearLockRequest : msg instanceof GridNearTxPrepareRequest;
+        TestRecordingCommunicationSpi.spi(client).blockMessages((node, msg) -> {
+            boolean block = false;
+
+            if (concurrency == PESSIMISTIC) {
+                if (msg instanceof GridNearLockRequest) {
+                    block = true;
+
+                    assertEquals(GRID_CNT + 1, ((GridNearLockRequest)msg).topologyVersion().topologyVersion());
+                }
             }
+            else {
+                if (msg instanceof GridNearTxPrepareRequest) {
+                    block = true;
+
+                    assertEquals(GRID_CNT + 1, ((GridNearTxPrepareRequest)msg).topologyVersion().topologyVersion());
+                }
+            }
+
+            return block;
         });
 
+        // Start tx and map on topver=GRID_CNT + 1
+        // Delay map until exchange.
+        // Start new node.
+
         IgniteInternalFuture fut0 = runAsync(new Runnable() {
             @Override public void run() {
                 try (Transaction tx = client.transactions().txStart(concurrency, isolation, 5000, 1)) {
@@ -784,14 +741,24 @@ private void doTestRollbackOnTimeoutTxRemap(TransactionConcurrency concurrency,
             }
         });
 
-        IgniteInternalFuture fut = runAsync(new Runnable() {
+        IgniteInternalFuture fut1 = runAsync(new Runnable() {
             @Override public void run() {
                 try {
-                    startGrid(GRID_CNT);
+                    TestRecordingCommunicationSpi.spi(client).waitForBlocked(); // TX is trying to prepare on prev top ver.
 
-                    TestRecordingCommunicationSpi.spi(crd).waitForBlocked();
+                    startGrid(GRID_CNT);
+                }
+                catch (Exception e) {
+                    fail(e.getMessage());
+                }
+            }
+        });
 
-                    TestRecordingCommunicationSpi.spi(client).waitForBlocked();
+        IgniteInternalFuture fut2 = runAsync(new Runnable() {
+            @Override public void run() {
+                try {
+                    // Wait for all full messages to be ready.
+                    TestRecordingCommunicationSpi.spi(crd).waitForBlocked(GRID_CNT + (clientWait ? 0 : 1));
 
                     // Trigger remap.
                     TestRecordingCommunicationSpi.spi(client).stopBlock();
@@ -802,24 +769,13 @@ private void doTestRollbackOnTimeoutTxRemap(TransactionConcurrency concurrency,
             }
         });
 
-        // Allow timeout on future wait.
-        try {
-            fut.get(10_000);
-        }
-        catch (IgniteFutureTimeoutCheckedException e) {
-            // Ignored.
-        }
-
-        try {
-            fut0.get(10_000);
-        }
-        catch (IgniteFutureTimeoutCheckedException e) {
-            // Ignored.
-        }
+        fut0.get(30_000);
+        fut1.get(30_000);
+        fut2.get(30_000);
 
         TestRecordingCommunicationSpi.spi(crd).stopBlock();
 
-        // If using awaitPartitionMapExchange for waiting it some times fail while waiting for owners.
+        // FIXME: If using awaitPartitionMapExchange for waiting it some times fail while waiting for owners.
         IgniteInternalFuture<?> topFut = ((IgniteEx)client).context().cache().context().exchange().
             affinityReadyFuture(new AffinityTopologyVersion(GRID_CNT + 2, 1));
 
@@ -1058,7 +1014,7 @@ private void waitingTxUnblockedOnTimeout(final Ignite near, final Ignite other,
 
         final int recordsCnt = 5;
 
-        IgniteInternalFuture<?> fut1 = GridTestUtils.runAsync(new Runnable() {
+        IgniteInternalFuture<?> fut1 = runAsync(new Runnable() {
             @Override public void run() {
                 try (Transaction tx = near.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, timeout, 0)) {
                     try {
@@ -1106,7 +1062,7 @@ private void waitingTxUnblockedOnTimeout(final Ignite near, final Ignite other,
             }
         }, "First");
 
-        IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Runnable() {
+        IgniteInternalFuture<?> fut2 = runAsync(new Runnable() {
             @Override public void run() {
                 U.awaitQuiet(blocked);
 

From 87d095960f3840947f55c61fc0ba29cb99871a52 Mon Sep 17 00:00:00 2001
From: SGrimstad <sgrimstad@gridgain.com>
Date: Tue, 28 Aug 2018 10:52:33 +0300
Subject: [PATCH 333/543] IGNITE-9256: SQL: Remove reference to H2 result set
 when iteration is finished to avoid potential OOME. This closes #4550.

---
 .../processors/query/h2/H2FieldsIterator.java |   2 +-
 .../query/h2/H2KeyValueIterator.java          |   2 +-
 .../query/h2/H2ResultSetIterator.java         |  33 +-
 ...ResultSetIteratorNullifyOnEndSelfTest.java | 420 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite.java        |   2 +
 5 files changed, 436 insertions(+), 23 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIteratorNullifyOnEndSelfTest.java

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2FieldsIterator.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2FieldsIterator.java
index f300c3f7f4dc3..26cbaee63aa73 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2FieldsIterator.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2FieldsIterator.java
@@ -36,7 +36,7 @@ public class H2FieldsIterator extends H2ResultSetIterator<List<?>> {
      * @throws IgniteCheckedException If failed.
      */
     public H2FieldsIterator(ResultSet data) throws IgniteCheckedException {
-        super(data, false, true);
+        super(data);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2KeyValueIterator.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2KeyValueIterator.java
index 2088e4439ccc8..31add346ee01f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2KeyValueIterator.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2KeyValueIterator.java
@@ -34,7 +34,7 @@ public class H2KeyValueIterator<K, V> extends H2ResultSetIterator<IgniteBiTuple<
      * @throws IgniteCheckedException If failed.
      */
     protected H2KeyValueIterator(ResultSet data) throws IgniteCheckedException {
-        super(data, false, true);
+        super(data);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIterator.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIterator.java
index 1b9aea3f2ab26..8cc396a792451 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIterator.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIterator.java
@@ -56,32 +56,26 @@ public abstract class H2ResultSetIterator<T> extends GridCloseableIteratorAdapte
     private static final long serialVersionUID = 0L;
 
     /** */
-    private final ResultInterface res;
+    private ResultInterface res;
 
     /** */
-    private final ResultSet data;
+    private ResultSet data;
 
     /** */
     protected final Object[] row;
 
-    /** */
-    private final boolean closeStmt;
-
     /** */
     private boolean hasRow;
 
     /**
      * @param data Data array.
-     * @param closeStmt If {@code true} closes result set statement when iterator is closed.
-     * @param needCpy {@code True} if need copy cache object's value.
      * @throws IgniteCheckedException If failed.
      */
-    protected H2ResultSetIterator(ResultSet data, boolean closeStmt, boolean needCpy) throws IgniteCheckedException {
+    protected H2ResultSetIterator(ResultSet data) throws IgniteCheckedException {
         this.data = data;
-        this.closeStmt = closeStmt;
 
         try {
-            res = needCpy ? (ResultInterface)RESULT_FIELD.get(data) : null;
+            res = (ResultInterface)RESULT_FIELD.get(data);
         }
         catch (IllegalAccessException e) {
             throw new IllegalStateException(e); // Must not happen.
@@ -107,8 +101,11 @@ private boolean fetchNext() {
             return false;
 
         try {
-            if (!data.next())
+            if (!data.next()){
+                onClose();
+
                 return false;
+            }
 
             if (res != null) {
                 Value[] values = res.currentRow();
@@ -164,21 +161,15 @@ private boolean fetchNext() {
     }
 
     /** {@inheritDoc} */
-    @Override public void onClose() throws IgniteCheckedException {
+    @Override public void onClose(){
         if (data == null)
             // Nothing to close.
             return;
 
-        if (closeStmt) {
-            try {
-                U.closeQuiet(data.getStatement());
-            }
-            catch (SQLException e) {
-                throw new IgniteCheckedException(e);
-            }
-        }
-
         U.closeQuiet(data);
+
+        res = null;
+        data = null;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIteratorNullifyOnEndSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIteratorNullifyOnEndSelfTest.java
new file mode 100644
index 0000000000000..31b0b97c5b0ed
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/H2ResultSetIteratorNullifyOnEndSelfTest.java
@@ -0,0 +1,420 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
+import javax.cache.Cache;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
+import org.apache.ignite.internal.processors.query.GridQueryCacheObjectsIterator;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.util.lang.GridCloseableIterator;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for iterator data link erasure after closing or completing
+ */
+public class H2ResultSetIteratorNullifyOnEndSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final int NODES_COUNT = 2;
+
+    /** */
+    private static final int PERSON_COUNT = 20;
+
+    /** */
+    private static final String SELECT_ALL_SQL = "SELECT p.* FROM Person p ORDER BY p.salary";
+
+    /** */
+    private static final String SELECT_MAX_SAL_SQLF = "select max(salary) from Person";
+
+    /**
+     * Non local SQL check nullification after close
+     */
+    public void testSqlQueryClose() {
+        SqlQuery<String, Person> qry = new SqlQuery<>(Person.class, SELECT_ALL_SQL);
+
+        QueryCursor<Cache.Entry<String, Person>> qryCurs = cache().query(qry);
+
+        qryCurs.iterator();
+
+        qryCurs.close();
+
+        H2ResultSetIterator h2It = extractIteratorInnerGridIteratorInnerH2ResultSetIterator(qryCurs);
+
+        checkIterator(h2It);
+    }
+
+    /**
+     * Non local SQL check nullification after complete
+     */
+    public void testSqlQueryComplete() {
+        SqlQuery<String, Person> qry = new SqlQuery<>(Person.class, SELECT_ALL_SQL);
+
+        QueryCursor<Cache.Entry<String, Person>> qryCurs = cache().query(qry);
+
+        qryCurs.getAll();
+
+        H2ResultSetIterator h2It = extractIteratorInnerGridIteratorInnerH2ResultSetIterator(qryCurs);
+
+        checkIterator(h2It);
+    }
+
+    /**
+     * Local SQL check nullification after close
+     */
+    public void testSqlQueryLocalClose() {
+        SqlQuery<String, Person> qry = new SqlQuery<>(Person.class, SELECT_ALL_SQL);
+
+        qry.setLocal(true);
+
+        QueryCursor<Cache.Entry<String, Person>> qryCurs = cache().query(qry);
+
+        qryCurs.iterator();
+
+        qryCurs.close();
+
+        H2ResultSetIterator h2It = extractIterableInnerH2ResultSetIterator(qryCurs);
+
+        checkIterator(h2It);
+    }
+
+    /**
+     * Local SQL check nullification after complete
+     */
+    public void testSqlQueryLocalComplete() {
+        SqlQuery<String, Person> qry = new SqlQuery<>(Person.class, SELECT_ALL_SQL);
+
+        qry.setLocal(true);
+
+        QueryCursor<Cache.Entry<String, Person>> qryCurs = cache().query(qry);
+
+        qryCurs.getAll();
+
+        H2ResultSetIterator h2It = extractIterableInnerH2ResultSetIterator(qryCurs);
+
+        checkIterator(h2It);
+    }
+
+    /**
+     * Non local SQL Fields check nullification after close
+     */
+    public void testSqlFieldsQueryClose() {
+        SqlFieldsQuery qry = new SqlFieldsQuery(SELECT_MAX_SAL_SQLF);
+
+        QueryCursor<List<?>> qryCurs = cache().query(qry);
+
+        qryCurs.iterator();
+
+        qryCurs.close();
+
+        H2ResultSetIterator h2It = extractGridIteratorInnerH2ResultSetIterator(qryCurs);
+
+        checkIterator(h2It);
+    }
+
+    /**
+     * Non local SQL Fields check nullification after complete
+     */
+    public void testSqlFieldsQueryComplete() {
+        SqlFieldsQuery qry = new SqlFieldsQuery(SELECT_MAX_SAL_SQLF);
+
+        QueryCursor<List<?>> qryCurs = cache().query(qry);
+
+        qryCurs.getAll();
+
+        H2ResultSetIterator h2It = extractGridIteratorInnerH2ResultSetIterator(qryCurs);
+
+        checkIterator(h2It);
+    }
+
+    /**
+     * Local SQL Fields check nullification after close
+     */
+    public void testSqlFieldsQueryLocalClose() {
+        SqlFieldsQuery qry = new SqlFieldsQuery(SELECT_MAX_SAL_SQLF);
+
+        qry.setLocal(true);
+
+        QueryCursor<List<?>> qryCurs = cache().query(qry);
+
+        qryCurs.iterator();
+
+        qryCurs.close();
+
+        H2ResultSetIterator h2It = extractGridIteratorInnerH2ResultSetIterator(qryCurs);
+
+        checkIterator(h2It);
+    }
+
+    /**
+     * Local SQL Fields check nullification after complete
+     */
+    public void testSqlFieldsQueryLocalComplete() {
+        SqlFieldsQuery qry = new SqlFieldsQuery(SELECT_MAX_SAL_SQLF);
+
+        qry.setLocal(true);
+
+        QueryCursor<List<?>> qryCurs = cache().query(qry);
+
+        qryCurs.getAll();
+
+        H2ResultSetIterator h2It = extractGridIteratorInnerH2ResultSetIterator(qryCurs);
+
+        checkIterator(h2It);
+    }
+
+    /**
+     * Common Assertion
+     * @param h2it target iterator
+     */
+    private void checkIterator(H2ResultSetIterator h2it){
+        if (Objects.nonNull(h2it))
+            assertNull(GridTestUtils.getFieldValue(h2it, H2ResultSetIterator.class, "data"));
+        else
+            fail();
+    }
+
+    /**
+     * Extract H2ResultSetIterator by reflection for non local SQL cases
+     * @param qryCurs source cursor
+     * @return target iterator or null of not extracted
+     */
+    private H2ResultSetIterator extractIteratorInnerGridIteratorInnerH2ResultSetIterator(
+        QueryCursor<Cache.Entry<String, Person>> qryCurs) {
+        if (QueryCursorImpl.class.isAssignableFrom(qryCurs.getClass())) {
+            Iterator inner = GridTestUtils.getFieldValue(qryCurs, QueryCursorImpl.class, "iter");
+
+            GridQueryCacheObjectsIterator it = GridTestUtils.getFieldValue(inner, inner.getClass(), "val$iter0");
+
+            Iterator<List<?>> h2RsIt = GridTestUtils.getFieldValue(it, GridQueryCacheObjectsIterator.class, "iter");
+
+            if (H2ResultSetIterator.class.isAssignableFrom(h2RsIt.getClass()))
+                return (H2ResultSetIterator)h2RsIt;
+        }
+        return null;
+    }
+
+    /**
+     * Extract H2ResultSetIterator by reflection for local SQL cases.
+     *
+     * @param qryCurs source cursor
+     * @return target iterator or null of not extracted
+     */
+    private H2ResultSetIterator extractIterableInnerH2ResultSetIterator(
+        QueryCursor<Cache.Entry<String, Person>> qryCurs) {
+        if (QueryCursorImpl.class.isAssignableFrom(qryCurs.getClass())) {
+            Iterable iterable = GridTestUtils.getFieldValue(qryCurs, QueryCursorImpl.class, "iterExec");
+
+            Iterator h2RsIt = GridTestUtils.getFieldValue(iterable, iterable.getClass(), "val$i");
+
+            if (H2ResultSetIterator.class.isAssignableFrom(h2RsIt.getClass()))
+                return (H2ResultSetIterator)h2RsIt;
+        }
+        return null;
+    }
+
+    /**
+     * Extract H2ResultSetIterator by reflection for SQL Fields cases.
+     *
+     * @param qryCurs source cursor
+     * @return target iterator or null of not extracted
+     */
+    private H2ResultSetIterator extractGridIteratorInnerH2ResultSetIterator(QueryCursor<List<?>> qryCurs) {
+        if (QueryCursorImpl.class.isAssignableFrom(qryCurs.getClass())) {
+            GridQueryCacheObjectsIterator it = GridTestUtils.getFieldValue(qryCurs, QueryCursorImpl.class, "iter");
+
+            Iterator<List<?>> h2RsIt = GridTestUtils.getFieldValue(it, GridQueryCacheObjectsIterator.class, "iter");
+
+            if (H2ResultSetIterator.class.isAssignableFrom(h2RsIt.getClass()))
+                return (H2ResultSetIterator)h2RsIt;
+        }
+        return null;
+    }
+
+    /**
+     * "onClose" should remove links to data.
+     */
+    public void testOnClose() {
+        try {
+            GridCloseableIterator it = indexing().queryLocalSql(
+                indexing().schema(cache().getName()),
+                cache().getName(),
+                SELECT_ALL_SQL,
+                null,
+                Collections.emptySet(),
+                "Person",
+                null,
+                null);
+
+            if (H2ResultSetIterator.class.isAssignableFrom(it.getClass())) {
+                H2ResultSetIterator h2it = (H2ResultSetIterator)it;
+
+                h2it.onClose();
+
+                assertNull(GridTestUtils.getFieldValue(h2it, H2ResultSetIterator.class, "data"));
+            }
+            else
+                fail();
+        }
+        catch (IgniteCheckedException e) {
+            fail(e.getMessage());
+        }
+    }
+
+    /**
+     * Complete iterate should remove links to data.
+     */
+    public void testOnComplete() {
+        try {
+            GridCloseableIterator it = indexing().queryLocalSql(
+                indexing().schema(cache().getName()),
+                cache().getName(),
+                SELECT_ALL_SQL,
+                null,
+                Collections.emptySet(),
+                "Person",
+                null,
+                null);
+
+            if (H2ResultSetIterator.class.isAssignableFrom(it.getClass())) {
+                H2ResultSetIterator h2it = (H2ResultSetIterator)it;
+
+                while (h2it.onHasNext())
+                    h2it.onNext();
+
+                assertNull(GridTestUtils.getFieldValue(h2it, H2ResultSetIterator.class, "data"));
+            }
+            else
+                fail();
+        }
+        catch (IgniteCheckedException e) {
+            fail(e.getMessage());
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        startGrids(NODES_COUNT);
+
+        ignite(0).createCache(
+            new CacheConfiguration<String, Person>("pers").setIndexedTypes(String.class, Person.class)
+        );
+
+        awaitPartitionMapExchange();
+
+        populateDataIntoPerson();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @return H2 indexing instance.
+     */
+    private IgniteH2Indexing indexing() {
+        GridQueryProcessor qryProcessor = grid(0).context().query();
+
+        return GridTestUtils.getFieldValue(qryProcessor, GridQueryProcessor.class, "idx");
+    }
+
+    /**
+     * @return Cache.
+     */
+    private IgniteCache<String, Person> cache() {
+        return grid(0).cache("pers");
+    }
+
+    /**
+     * Populate person cache with test data
+     */
+    private void populateDataIntoPerson() {
+        IgniteCache<String, Person> cache = cache();
+
+        int personId = 0;
+
+        for (int j = 0; j < PERSON_COUNT; j++) {
+            Person prsn = new Person();
+
+            prsn.setId("pers" + personId);
+            prsn.setName("Person name #" + personId);
+
+            cache.put(prsn.getId(), prsn);
+
+            personId++;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class Person {
+        /** */
+        @QuerySqlField(index = true)
+        private String id;
+
+        /** */
+        @QuerySqlField(index = true)
+        private String name;
+
+        /** */
+        @QuerySqlField(index = true)
+        private int salary;
+
+        /** */
+        public String getId() {
+            return id;
+        }
+
+        /** */
+        public void setId(String id) {
+            this.id = id;
+        }
+
+        /** */
+        public String getName() {
+            return name;
+        }
+
+        /** */
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        /** */
+        public int getSalary() {
+            return salary;
+        }
+
+        /** */
+        public void setSalary(int salary) {
+            this.salary = salary;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 1ebe654e2ee6b..cc076681ce057 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -164,6 +164,7 @@
 import org.apache.ignite.internal.processors.query.SqlSchemaSelfTest;
 import org.apache.ignite.internal.processors.query.h2.GridH2IndexingInMemSelfTest;
 import org.apache.ignite.internal.processors.query.h2.GridH2IndexingOffheapSelfTest;
+import org.apache.ignite.internal.processors.query.h2.H2ResultSetIteratorNullifyOnEndSelfTest;
 import org.apache.ignite.internal.processors.query.h2.IgniteSqlBigIntegerKeyTest;
 import org.apache.ignite.internal.processors.query.h2.IgniteSqlQueryMinMaxTest;
 import org.apache.ignite.internal.processors.query.h2.sql.BaseH2CompareQueryTest;
@@ -259,6 +260,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheCollocatedQuerySelfTest.class);
         suite.addTestSuite(IgniteCacheLargeResultSelfTest.class);
         suite.addTestSuite(GridCacheQueryInternalKeysSelfTest.class);
+        suite.addTestSuite(H2ResultSetIteratorNullifyOnEndSelfTest.class);
         suite.addTestSuite(IgniteSqlBigIntegerKeyTest.class);
         suite.addTestSuite(IgniteCacheOffheapEvictQueryTest.class);
         suite.addTestSuite(IgniteCacheOffheapIndexScanTest.class);

From 96f00952b37137b9e39300232281f6afb2dd971a Mon Sep 17 00:00:00 2001
From: kcmvp <kcheng.mvp@gmail.com>
Date: Tue, 10 Jul 2018 00:04:23 +0300
Subject: [PATCH 334/543] IGNITE-8956 Fix of javadoc build failure. - Fixes
 #4328.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
(cherry picked from commit cb38b5d196c7adf27f5e727de039bfc8604abf82)
---
 .../ignite/internal/processors/cache/WalStateManager.java     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
index 4d12eb33bf06f..16b92ce68789b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -1283,7 +1283,9 @@ protected void disableWAL(boolean disable) throws IgniteCheckedException {
             metaStorage = ms;
         }
 
-        /** {@inheritDoc} */
+        /**
+         * @return {@code true} If WAL is disabled.
+         */
         public boolean check() {
             return disableWal;
         }

From 6cca59184f4ace19fead2e98701bd2c65eb09583 Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Fri, 31 Aug 2018 18:41:58 +0300
Subject: [PATCH 335/543] IGNITE-9448 Update ZooKeeper version to 3.4.13 -
 Fixes #4661.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 4d736fc20a6e667e841201ad2205f1440546191b)
---
 parent/pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parent/pom.xml b/parent/pom.xml
index 5c66c3a0e1538..4b76da89a73c2 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -122,7 +122,7 @@
         <yammer.metrics.core.version>2.2.0</yammer.metrics.core.version>
         <yardstick.version>0.8.3</yardstick.version>
         <zkclient.version>0.5</zkclient.version>
-        <zookeeper.version>3.4.6</zookeeper.version>
+        <zookeeper.version>3.4.13</zookeeper.version>
 
         <!-- OSGI Manifest generation default property values -->
         <osgi.import.package>*</osgi.import.package>

From dc4baa598f8c5e72473070ef2d1e9ae5c55e9dbc Mon Sep 17 00:00:00 2001
From: SGrimstad <sgrimstad@gridgain.com>
Date: Tue, 4 Sep 2018 11:57:07 +0300
Subject: [PATCH 336/543] IGNITE-9141: SQL: pass error message from mapper to
 reducer in case of mapping failure. This closes #4536.

---
 .../messages/GridQueryNextPageResponse.java   |  33 +-
 .../h2/twostep/GridMapQueryExecutor.java      | 131 +++---
 .../h2/twostep/GridReduceQueryExecutor.java   |  78 ++--
 .../query/h2/twostep/ReduceQueryRun.java      | 117 ++++-
 ...ppearedCacheCauseRetryMessageSelfTest.java | 134 ++++++
 ...pearedCacheWasNotFoundMessageSelfTest.java | 123 ++++++
 .../query/h2/twostep/JoinSqlTestHelper.java   | 163 +++++++
 .../NonCollocatedRetryMessageSelfTest.java    | 146 ++++++
 .../h2/twostep/RetryCauseMessageSelfTest.java | 416 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite2.java       |   9 +
 10 files changed, 1236 insertions(+), 114 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheCauseRetryMessageSelfTest.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheWasNotFoundMessageSelfTest.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinSqlTestHelper.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/NonCollocatedRetryMessageSelfTest.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
index 4d918a04eacb8..ea83169d37212 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
@@ -67,6 +67,9 @@ public class GridQueryNextPageResponse implements Message {
     /** */
     private AffinityTopologyVersion retry;
 
+    /** Retry cause description*/
+    private String retryCause;
+
     /** Last page flag. */
     private boolean last;
 
@@ -230,6 +233,12 @@ public Collection<?> plainRows() {
                     return false;
 
                 writer.incrementState();
+
+            case 9:
+                if (!writer.writeString("retryCause", retryCause))
+                    return false;
+
+                writer.incrementState();
         }
 
         return true;
@@ -310,6 +319,14 @@ public Collection<?> plainRows() {
             case 8:
                 last = reader.readBoolean("last");
 
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 9:
+                retryCause = reader.readString("retryCause");
+
                 if (!reader.isLastRead())
                     return false;
 
@@ -326,7 +343,7 @@ public Collection<?> plainRows() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 9;
+        return 10;
     }
 
     /**
@@ -343,6 +360,20 @@ public void retry(AffinityTopologyVersion retry) {
         this.retry = retry;
     }
 
+    /**
+     * @return Retry Ccause message.
+     */
+    public String retryCause() {
+        return retryCause;
+    }
+
+    /**
+     * @param retryCause Retry Ccause message.
+     */
+    public void retryCause(String retryCause){
+        this.retryCause = retryCause;
+    }
+
     /**
      * @return Last page flag.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 216a259da73e4..4f09b23676697 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -296,10 +296,10 @@ private GridDhtLocalPartition partition(GridCacheContext<?, ?> cctx, int p) {
      * @param reserved Reserved list.
      * @param nodeId Node ID.
      * @param reqId Request ID.
-     * @return {@code true} If all the needed partitions successfully reserved.
+     * @return String which is null in case of success or with causeMessage if failed
      * @throws IgniteCheckedException If failed.
      */
-    private boolean reservePartitions(
+    private String reservePartitions(
         @Nullable List<Integer> cacheIds,
         AffinityTopologyVersion topVer,
         final int[] explicitParts,
@@ -310,7 +310,7 @@ private boolean reservePartitions(
         assert topVer != null;
 
         if (F.isEmpty(cacheIds))
-            return true;
+            return null;
 
         Collection<Integer> partIds = wrap(explicitParts);
 
@@ -319,11 +319,11 @@ private boolean reservePartitions(
 
             // Cache was not found, probably was not deployed yet.
             if (cctx == null) {
-                logRetry("Failed to reserve partitions for query (cache is not found on local node) [" +
-                    "rmtNodeId=" + nodeId + ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" +
-                    cacheIds.get(i) + "]");
+                final String res = String.format("Failed to reserve partitions for query (cache is not found on " +
+                    "local node) [localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s]",
+                    ctx.localNodeId(), nodeId, reqId, topVer, cacheIds.get(i));
 
-                return false;
+                return res;
             }
 
             if (cctx.isLocal() || !cctx.rebalanceEnabled())
@@ -336,13 +336,10 @@ private boolean reservePartitions(
 
             if (explicitParts == null && r != null) { // Try to reserve group partition if any and no explicits.
                 if (r != MapReplicatedReservation.INSTANCE) {
-                    if (!r.reserve()) {
-                        logRetry("Failed to reserve partitions for query (group reservation failed) [" +
-                            "rmtNodeId=" + nodeId + ", reqId=" + reqId + ", affTopVer=" + topVer +
-                            ", cacheId=" + cacheIds.get(i) + ", cacheName=" + cctx.name() + "]");
-
-                        return false; // We need explicit partitions here -> retry.
-                    }
+                    if (!r.reserve())
+                        return String.format("Failed to reserve partitions for query (group " +
+                            "reservation failed) [localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, " +
+                            "cacheName=%s]",ctx.localNodeId(), nodeId, reqId, topVer, cacheIds.get(i), cctx.name());
 
                     reserved.add(r);
                 }
@@ -358,15 +355,21 @@ private boolean reservePartitions(
                             // We don't need to reserve partitions because they will not be evicted in replicated caches.
                             GridDhtPartitionState partState = part != null ? part.state() : null;
 
-                            if (partState != OWNING) {
-                                logRetry("Failed to reserve partitions for query (partition of " +
-                                    "REPLICATED cache is not in OWNING state) [rmtNodeId=" + nodeId +
-                                    ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
-                                    ", cacheName=" + cctx.name() + ", part=" + p + ", partFound=" + (part != null) +
-                                    ", partState=" + partState + "]");
-
-                                return false;
-                            }
+                            if (partState != OWNING)
+                                return String.format("Failed to reserve partitions for query " +
+                                    "(partition of REPLICATED cache is not in OWNING state) [" +
+                                    "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, cacheName=%s, " +
+                                    "part=%s, partFound=%s, partState=%s]",
+                                    ctx.localNodeId(),
+                                    nodeId,
+                                    reqId,
+                                    topVer,
+                                    cacheIds.get(i),
+                                    cctx.name(),
+                                    p,
+                                    (part != null),
+                                    partState
+                                );
                         }
 
                         // Mark that we checked this replicated cache.
@@ -382,29 +385,41 @@ private boolean reservePartitions(
 
                         GridDhtPartitionState partState = part != null ? part.state() : null;
 
-                        if (partState != OWNING || !part.reserve()) {
-                            logRetry("Failed to reserve partitions for query (partition of " +
-                                "PARTITIONED cache cannot be reserved) [rmtNodeId=" + nodeId + ", reqId=" + reqId +
-                                ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
-                                ", cacheName=" + cctx.name() + ", part=" + partId + ", partFound=" + (part != null) +
-                                ", partState=" + partState + "]");
-
-                            return false;
-                        }
+                        if (partState != OWNING || !part.reserve())
+                            return String.format("Failed to reserve partitions for query " +
+                                "(partition of PARTITIONED cache cannot be reserved) [" +
+                                "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, cacheName=%s, " +
+                                "part=%s, partFound=%s, partState=%s]",
+                                ctx.localNodeId(),
+                                nodeId,
+                                reqId,
+                                topVer,
+                                cacheIds.get(i),
+                                cctx.name(),
+                                partId,
+                                (part != null),
+                                partState
+                            );
 
                         reserved.add(part);
 
                         // Double check that we are still in owning state and partition contents are not cleared.
                         partState = part.state();
 
-                        if (part.state() != OWNING) {
-                            logRetry("Failed to reserve partitions for query (partition of " +
-                                "PARTITIONED cache is not in OWNING state after reservation) [rmtNodeId=" + nodeId +
-                                ", reqId=" + reqId + ", affTopVer=" + topVer + ", cacheId=" + cacheIds.get(i) +
-                                ", cacheName=" + cctx.name() + ", part=" + partId + ", partState=" + partState + "]");
-
-                            return false;
-                        }
+                        if (part.state() != OWNING)
+                            return String.format("Failed to reserve partitions for query " +
+                                "(partition of PARTITIONED cache is not in OWNING state after reservation) [" +
+                                "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, cacheName=%s, " +
+                                "part=%s, partState=%s]",
+                                ctx.localNodeId(),
+                                nodeId,
+                                reqId,
+                                topVer,
+                                cacheIds.get(i),
+                                cctx.name(),
+                                partId,
+                                partState
+                            );
                     }
 
                     if (explicitParts == null) {
@@ -426,16 +441,7 @@ private boolean reservePartitions(
             }
         }
 
-        return true;
-    }
-
-    /**
-     * Load failed partition reservation.
-     *
-     * @param msg Message.
-     */
-    private void logRetry(String msg) {
-        log.info(msg);
+        return null;
     }
 
     /**
@@ -673,12 +679,14 @@ private void onQueryRequest0(
         try {
             if (topVer != null) {
                 // Reserve primary for topology version or explicit partitions.
-                if (!reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId)) {
+                String err = reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId);
+
+                if (!F.isEmpty(err)) {
                     // Unregister lazy worker because re-try may never reach this node again.
                     if (lazy)
                         stopAndUnregisterCurrentLazyWorker();
 
-                    sendRetry(node, reqId, segmentId);
+                    sendRetry(node, reqId, segmentId, err);
 
                     return;
                 }
@@ -793,10 +801,12 @@ private void onQueryRequest0(
             GridH2RetryException retryErr = X.cause(e, GridH2RetryException.class);
 
             if (retryErr != null) {
-                logRetry("Failed to execute non-collocated query (will retry) [nodeId=" + node.id() +
-                    ", reqId=" + reqId + ", errMsg=" + retryErr.getMessage() + ']');
+                final String retryCause = String.format(
+                    "Failed to execute non-collocated query (will retry) [localNodeId=%s, rmtNodeId=%s, reqId=%s, " +
+                    "errMsg=%s]", ctx.localNodeId(), node.id(), reqId, retryErr.getMessage()
+                );
 
-                sendRetry(node, reqId, segmentId);
+                sendRetry(node, reqId, segmentId, retryCause);
             }
             else {
                 U.error(log, "Failed to execute local query.", e);
@@ -845,13 +855,15 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th
 
         List<GridReservable> reserved = new ArrayList<>();
 
-        if (!reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId)) {
+        String err = reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId);
+
+        if (!F.isEmpty(err)) {
             U.error(log, "Failed to reserve partitions for DML request. [localNodeId=" + ctx.localNodeId() +
                 ", nodeId=" + node.id() + ", reqId=" + req.requestId() + ", cacheIds=" + cacheIds +
                 ", topVer=" + topVer + ", parts=" + Arrays.toString(parts) + ']');
 
-            sendUpdateResponse(node, reqId, null, "Failed to reserve partitions for DML request. " +
-                "Explanation (Retry your request when re-balancing is over).");
+            sendUpdateResponse(node, reqId, null,
+                "Failed to reserve partitions for DML request. " + err);
 
             return;
         }
@@ -1081,7 +1093,7 @@ private void sendNextPage(MapNodeResults nodeRess, ClusterNode node, MapQueryRes
      * @param reqId Request ID.
      * @param segmentId Index segment ID.
      */
-    private void sendRetry(ClusterNode node, long reqId, int segmentId) {
+    private void sendRetry(ClusterNode node, long reqId, int segmentId, String retryCause) {
         try {
             boolean loc = node.isLocal();
 
@@ -1092,6 +1104,7 @@ private void sendRetry(ClusterNode node, long reqId, int segmentId) {
                 false);
 
             msg.retry(h2.readyTopologyVersion());
+            msg.retryCause(retryCause);
 
             if (loc)
                 h2.reduceQueryExecutor().onMessage(ctx.localNodeId(), msg);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 39b2bbcfc09ad..51d05845a104c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -223,8 +223,7 @@ public void start(final GridKernalContext ctx, final IgniteH2Indexing h2) throws
      * @param nodeId Left node ID.
      */
     private void handleNodeLeft(ReduceQueryRun r, UUID nodeId) {
-        // Will attempt to retry. If reduce query was started it will fail on next page fetching.
-        retry(r, h2.readyTopologyVersion(), nodeId);
+        r.setStateOnNodeLeave(nodeId, h2.readyTopologyVersion());
     }
 
     /**
@@ -276,12 +275,13 @@ private void onFail(ClusterNode node, GridQueryFailResponse msg) {
      */
     private void fail(ReduceQueryRun r, UUID nodeId, String msg, byte failCode) {
         if (r != null) {
-            CacheException e = new CacheException("Failed to execute map query on the node: " + nodeId + ", " + msg);
+            CacheException e = new CacheException("Failed to execute map query on remote node [nodeId=" + nodeId +
+                ", errMsg=" + msg + ']');
 
             if (failCode == GridQueryFailResponse.CANCELLED_BY_ORIGINATOR)
                 e.addSuppressed(new QueryCancelledException());
 
-            r.state(e, nodeId);
+            r.setStateOnException(nodeId, e);
         }
     }
 
@@ -289,7 +289,7 @@ private void fail(ReduceQueryRun r, UUID nodeId, String msg, byte failCode) {
      * @param node Node.
      * @param msg Message.
      */
-    private void onNextPage(final ClusterNode node, GridQueryNextPageResponse msg) {
+    private void onNextPage(final ClusterNode node, final GridQueryNextPageResponse msg) {
         final long qryReqId = msg.queryRequestId();
         final int qry = msg.query();
         final int seg = msg.segmentId();
@@ -308,20 +308,13 @@ private void onNextPage(final ClusterNode node, GridQueryNextPageResponse msg) {
         try {
             page = new GridResultPage(ctx, node.id(), msg) {
                 @Override public void fetchNextPage() {
-                    Object errState = r.state();
+                    if (r.hasErrorOrRetry()) {
+                        if (r.exception() != null)
+                            throw r.exception();
 
-                    if (errState != null) {
-                        CacheException err0 = errState instanceof CacheException ? (CacheException)errState : null;
+                        assert r.retryCause() != null;
 
-                        if (err0 != null && err0.getCause() instanceof IgniteClientDisconnectedException)
-                            throw err0;
-
-                        CacheException e = new CacheException("Failed to fetch data from node: " + node.id());
-
-                        if (err0 != null)
-                            e.addSuppressed(err0);
-
-                        throw e;
+                        throw new CacheException(r.retryCause());
                     }
 
                     try {
@@ -349,26 +342,21 @@ private void onNextPage(final ClusterNode node, GridQueryNextPageResponse msg) {
         idx.addPage(page);
 
         if (msg.retry() != null)
-            retry(r, msg.retry(), node.id());
-        else if (msg.page() == 0) // Do count down on each first page received.
+            r.setStateOnRetry(node.id(), msg.retry(), msg.retryCause());
+        else if (msg.page() == 0)
+            // Do count down on each first page received.
             r.latch().countDown();
     }
 
-    /**
-     * @param r Query run.
-     * @param retryVer Retry version.
-     * @param nodeId Node ID.
-     */
-    private void retry(ReduceQueryRun r, AffinityTopologyVersion retryVer, UUID nodeId) {
-        r.state(retryVer, nodeId);
-    }
-
     /**
      * @param cacheIds Cache IDs.
      * @return {@code true} If preloading is active.
      */
     private boolean isPreloadingActive(List<Integer> cacheIds) {
         for (Integer cacheId : cacheIds) {
+            if (null == cacheContext(cacheId))
+                throw new CacheException(String.format("Cache not found on local node [cacheId=%d]", cacheId));
+
             if (hasMovingPartitions(cacheContext(cacheId)))
                 return true;
         }
@@ -381,6 +369,8 @@ private boolean isPreloadingActive(List<Integer> cacheIds) {
      * @return {@code True} If cache has partitions in {@link GridDhtPartitionState#MOVING} state.
      */
     private boolean hasMovingPartitions(GridCacheContext<?, ?> cctx) {
+        assert cctx != null;
+
         return !cctx.isLocal() && cctx.topology().hasMovingPartitions();
     }
 
@@ -572,9 +562,18 @@ public Iterator<List<?>> query(
 
         final long startTime = U.currentTimeMillis();
 
+        ReduceQueryRun lastRun = null;
+
         for (int attempt = 0;; attempt++) {
-            if (attempt > 0 && retryTimeout > 0 && (U.currentTimeMillis() - startTime > retryTimeout))
-                throw new CacheException("Failed to map SQL query to topology.");
+            if (attempt > 0 && retryTimeout > 0 && (U.currentTimeMillis() - startTime > retryTimeout)) {
+                UUID retryNodeId = lastRun.retryNodeId();
+                String retryCause = lastRun.retryCause();
+
+                assert !F.isEmpty(retryCause);
+
+                throw new CacheException("Failed to map SQL query to topology on data node [dataNodeId=" + retryNodeId +
+                    ", msg=" + retryCause + ']');
+            }
 
             if (attempt != 0) {
                 try {
@@ -777,26 +776,23 @@ public Iterator<List<?>> query(
                 if (send(nodes, req, parts == null ? null : new ExplicitPartitionsSpecializer(qryMap), false)) {
                     awaitAllReplies(r, nodes, cancel);
 
-                    Object state = r.state();
-
-                    if (state != null) {
-                        if (state instanceof CacheException) {
-                            CacheException err = (CacheException)state;
+                    if (r.hasErrorOrRetry()) {
+                        CacheException err = r.exception();
 
+                        if (err != null) {
                             if (err.getCause() instanceof IgniteClientDisconnectedException)
                                 throw err;
 
                             if (wasCancelled(err))
                                 throw new QueryCancelledException(); // Throw correct exception.
 
-                            throw new CacheException("Failed to run map query remotely." + err.getMessage(), err);
+                            throw new CacheException("Failed to run map query remotely: " + err.getMessage(), err);
                         }
-
-                        if (state instanceof AffinityTopologyVersion) {
+                        else {
                             retry = true;
 
                             // If remote node asks us to retry then we have outdated full partition map.
-                            h2.awaitForReadyTopologyVersion((AffinityTopologyVersion)state);
+                            h2.awaitForReadyTopologyVersion(r.retryTopologyVersion());
                         }
                     }
                 }
@@ -841,8 +837,10 @@ public Iterator<List<?>> query(
                         }
                     }
                 }
+                else {
+                    assert r != null;
+                    lastRun=r;
 
-                if (retry) {
                     if (Thread.currentThread().isInterrupted())
                         throw new IgniteInterruptedCheckedException("Query was interrupted.");
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReduceQueryRun.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReduceQueryRun.java
index 73bb002ee1635..e27fe84403ac8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReduceQueryRun.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/ReduceQueryRun.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.internal.processors.cache.query.GridCacheTwoStepQuery;
 import org.apache.ignite.internal.processors.query.GridQueryCancel;
 import org.apache.ignite.internal.processors.query.GridRunningQueryInfo;
+import org.apache.ignite.internal.util.typedef.F;
 import org.h2.jdbc.JdbcConnection;
 import org.jetbrains.annotations.Nullable;
 
@@ -53,8 +54,8 @@ class ReduceQueryRun {
     /** */
     private final int pageSize;
 
-    /** Can be either CacheException in case of error or AffinityTopologyVersion to retry if needed. */
-    private final AtomicReference<Object> state = new AtomicReference<>();
+    /** */
+    private final AtomicReference<State> state = new AtomicReference<>();
 
     /**
      * Constructor.
@@ -80,30 +81,59 @@ class ReduceQueryRun {
     }
 
     /**
-     * @param o Fail state object.
+     * Set state on exception.
+     *
+     * @param err error.
+     * @param nodeId Node ID.
+     */
+    void setStateOnException(@Nullable UUID nodeId, CacheException err) {
+        setState0(new State(nodeId, err, null, null));
+    }
+
+    /**
+     * Set state on map node leave.
+     *
+     * @param nodeId Node ID.
+     * @param topVer Topology version.
+     */
+    void setStateOnNodeLeave(UUID nodeId, AffinityTopologyVersion topVer) {
+        setState0(new State(nodeId, null, topVer, "Data node has left the grid during query execution [nodeId=" +
+            nodeId + ']'));
+    }
+
+    /**
+     * Set state on retry due to mapping failure.
+     *
      * @param nodeId Node ID.
+     * @param topVer Topology version.
+     * @param retryCause Retry cause.
      */
-    void state(Object o, @Nullable UUID nodeId) {
-        assert o != null;
-        assert o instanceof CacheException || o instanceof AffinityTopologyVersion : o.getClass();
+    void setStateOnRetry(UUID nodeId, AffinityTopologyVersion topVer, String retryCause) {
+        assert !F.isEmpty(retryCause);
 
-        if (!state.compareAndSet(null, o))
+        setState0(new State(nodeId, null, topVer, retryCause));
+    }
+
+    /**
+     *
+     * @param state state
+     */
+    private void setState0(State state){
+        if (!this.state.compareAndSet(null, state))
             return;
 
         while (latch.getCount() != 0) // We don't need to wait for all nodes to reply.
             latch.countDown();
 
-        CacheException e = o instanceof CacheException ? (CacheException) o : null;
-
         for (GridMergeIndex idx : idxs) // Fail all merge indexes.
-            idx.fail(nodeId, e);
+            idx.fail(state.nodeId, state.ex);
     }
 
     /**
      * @param e Error.
      */
     void disconnected(CacheException e) {
-        state(e, null);
+        setStateOnException(null, e);
     }
 
     /**
@@ -127,11 +157,45 @@ JdbcConnection connection() {
         return conn;
     }
 
+    /** */
+    boolean hasErrorOrRetry(){
+        return state.get() != null;
+    }
+
     /**
-     * @return State.
+     * @return Exception.
      */
-    Object state() {
-        return state.get();
+    CacheException exception() {
+        State st = state.get();
+
+        return st != null ? st.ex : null;
+    }
+
+    /**
+     * @return Retry topology version.
+     */
+    AffinityTopologyVersion retryTopologyVersion(){
+        State st = state.get();
+
+        return st != null ? st.retryTopVer : null;
+    }
+
+    /**
+     * @return Retry bode ID.
+     */
+    UUID retryNodeId() {
+        State st = state.get();
+
+        return st != null ? st.nodeId : null;
+    }
+
+    /**
+     * @return Retry cause.
+     */
+    String retryCause(){
+        State st = state.get();
+
+        return st != null ? st.retryCause : null;
     }
 
     /**
@@ -154,4 +218,29 @@ CountDownLatch latch() {
     void latch(CountDownLatch latch) {
         this.latch = latch;
     }
+
+    /**
+     * Error state.
+     */
+    private static class State {
+        /** Affected node (may be null in case of local node failure). */
+        private final UUID nodeId;
+
+        /** Error. */
+        private final CacheException ex;
+
+        /** Retry topology version. */
+        private final AffinityTopologyVersion retryTopVer;
+
+        /** Retry cause. */
+        private final String retryCause;
+
+        /** */
+        private State(UUID nodeId, CacheException ex, AffinityTopologyVersion retryTopVer, String retryCause){
+            this.nodeId = nodeId;
+            this.ex = ex;
+            this.retryTopVer = retryTopVer;
+            this.retryCause = retryCause;
+        }
+    }
 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheCauseRetryMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheCauseRetryMessageSelfTest.java
new file mode 100644
index 0000000000000..8c4358a7fd722
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheCauseRetryMessageSelfTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import javax.cache.CacheException;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.query.h2.twostep.messages.GridQueryCancelRequest;
+import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Organization;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Person;
+
+/**
+ * Failed to reserve partitions for query (cache is not found on local node) Root cause test
+ */
+public class DisappearedCacheCauseRetryMessageSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final int NODES_COUNT = 2;
+    /** */
+    private static final String ORG = "org";
+    /** */
+    private IgniteCache<String, JoinSqlTestHelper.Person> personCache;
+    /** */
+    private IgniteCache<String, JoinSqlTestHelper.Organization> orgCache;
+
+    /** */
+    public void testDisappearedCacheCauseRetryMessage() {
+
+        SqlQuery<String, JoinSqlTestHelper.Person> qry = new SqlQuery<String, JoinSqlTestHelper.Person>(JoinSqlTestHelper.Person.class, JoinSqlTestHelper.JOIN_SQL).setArgs("Organization #0");
+
+        qry.setDistributedJoins(true);
+
+        try {
+            personCache.query(qry).getAll();
+
+            fail("No CacheException emitted.");
+        }
+        catch (CacheException e) {
+            assertTrue(e.getMessage(), e.getMessage().contains("Failed to reserve partitions for query (cache is not found on local node) ["));
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCommunicationSpi(new TcpCommunicationSpi(){
+
+            volatile long reqId = -1;
+            /** {@inheritDoc} */
+            @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+                assert msg != null;
+
+                if ( GridIoMessage.class.isAssignableFrom(msg.getClass())){
+                    GridIoMessage gridMsg = (GridIoMessage)msg;
+
+                    if ( GridH2QueryRequest.class.isAssignableFrom( gridMsg.message().getClass() ) ){
+                        GridH2QueryRequest req = (GridH2QueryRequest) (gridMsg.message());
+                        reqId = req.requestId();
+                        orgCache.destroy();
+                    }
+                    else if ( GridQueryCancelRequest.class.isAssignableFrom( gridMsg.message().getClass() ) ){
+                        GridQueryCancelRequest req = (GridQueryCancelRequest) (gridMsg.message());
+
+                        if (reqId == req.queryRequestId())
+                            orgCache = DisappearedCacheCauseRetryMessageSelfTest.this.ignite(0).getOrCreateCache(new CacheConfiguration<String, Organization>(ORG)
+                                .setCacheMode(CacheMode.REPLICATED)
+                                .setIndexedTypes(String.class, JoinSqlTestHelper.Organization.class)
+                            );
+
+                    }
+                }
+
+                super.sendMessage(node, msg, ackC);
+            }
+        });
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        System.setProperty(IGNITE_SQL_RETRY_TIMEOUT, "5000");
+
+        startGridsMultiThreaded(NODES_COUNT, false);
+
+        personCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Person>("pers")
+            .setIndexedTypes(String.class, JoinSqlTestHelper.Person.class)
+        );
+
+        orgCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Organization>(ORG)
+            .setCacheMode(CacheMode.REPLICATED)
+            .setIndexedTypes(String.class, JoinSqlTestHelper.Organization.class)
+        );
+
+        awaitPartitionMapExchange();
+
+        JoinSqlTestHelper.populateDataIntoOrg(orgCache);
+
+        JoinSqlTestHelper.populateDataIntoPerson(personCache);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheWasNotFoundMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheWasNotFoundMessageSelfTest.java
new file mode 100644
index 0000000000000..9928ed6ff2745
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/DisappearedCacheWasNotFoundMessageSelfTest.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Organization;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Person;
+
+/**
+ * Grid cache context is not registered for cache id root cause message test
+ */
+public class DisappearedCacheWasNotFoundMessageSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final int NODES_COUNT = 2;
+    /** */
+    private static final String ORG = "org";
+    /** */
+    private IgniteCache<String, JoinSqlTestHelper.Person> personCache;
+    /** */
+    private IgniteCache<String, JoinSqlTestHelper.Organization> orgCache;
+
+    /** */
+    public void testDisappearedCacheWasNotFoundMessage() {
+        SqlQuery<String, Person> qry = new SqlQuery<String, Person>(Person.class, JoinSqlTestHelper.JOIN_SQL).setArgs("Organization #0");
+
+        qry.setDistributedJoins(true);
+
+        try {
+            personCache.query(qry).getAll();
+
+            fail("No CacheException emitted.");
+        }
+        catch (CacheException e) {
+            assertTrue(e.getMessage(), e.getMessage().contains("Cache not found on local node"));
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCommunicationSpi(new TcpCommunicationSpi(){
+            /** {@inheritDoc} */
+            @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+                assert msg != null;
+
+                if ( GridIoMessage.class.isAssignableFrom(msg.getClass())){
+                    GridIoMessage gridMsg = (GridIoMessage)msg;
+
+                    if ( GridH2QueryRequest.class.isAssignableFrom( gridMsg.message().getClass() ) ){
+                        GridH2QueryRequest req = (GridH2QueryRequest) (gridMsg.message());
+
+                        req.requestId();
+
+                        orgCache.destroy();
+                    }
+                }
+
+                super.sendMessage(node, msg, ackC);
+            }
+        });
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        System.setProperty(IGNITE_SQL_RETRY_TIMEOUT, "5000");
+
+        startGridsMultiThreaded(NODES_COUNT, false);
+
+        personCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Person>("pers")
+            .setIndexedTypes(String.class, JoinSqlTestHelper.Person.class)
+        );
+
+        orgCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Organization>(ORG)
+                .setCacheMode(CacheMode.REPLICATED)
+                .setIndexedTypes(String.class, Organization.class)
+        );
+
+        awaitPartitionMapExchange();
+
+        JoinSqlTestHelper.populateDataIntoOrg(orgCache);
+
+        JoinSqlTestHelper.populateDataIntoPerson(personCache);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinSqlTestHelper.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinSqlTestHelper.java
new file mode 100644
index 0000000000000..fe7821ae29709
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/JoinSqlTestHelper.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+
+/**
+ * Join sql test helper
+ */
+public class JoinSqlTestHelper {
+    /** */
+    private static final int ORG_COUNT = 100;
+
+    /** */
+    private static final int PERSON_PER_ORG_COUNT = 10;
+
+    /** */
+    static final String JOIN_SQL = "select * from Person, \"org\".Organization as org " +
+        "where Person.orgId = org.id " +
+        "and lower(org.name) = lower(?)";
+
+    /**
+     * Populate organization cache with test data
+     * @param cache @{IgniteCache}
+     */
+    static void populateDataIntoOrg(IgniteCache<String, Organization> cache) {
+        for (int i = 0; i < ORG_COUNT; i++) {
+            Organization org = new Organization();
+
+            org.setId("org" + i);
+
+            org.setName("Organization #" + i);
+
+            cache.put(org.getId(), org);
+        }
+    }
+
+    /**
+     * Populate person cache with test data
+     * @param cache @{IgniteCache}
+     */
+    static void populateDataIntoPerson(IgniteCache<String, Person> cache) {
+        int personId = 0;
+
+        for (int i = 0; i < ORG_COUNT; i++) {
+            Organization org = new Organization();
+
+            org.setId("org" + i);
+
+            org.setName("Organization #" + i);
+
+            for (int j = 0; j < PERSON_PER_ORG_COUNT; j++) {
+                Person prsn = new Person();
+
+                prsn.setId("pers" + personId);
+
+                prsn.setOrgId(org.getId());
+
+                prsn.setName("Person name #" + personId);
+
+                cache.put(prsn.getId(), prsn);
+
+                personId++;
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    public static class Person {
+        /** */
+        @QuerySqlField(index = true)
+        private String id;
+
+        /** */
+        @QuerySqlField(index = true)
+        private String orgId;
+
+        /** */
+        @QuerySqlField(index = true)
+        private String name;
+
+        /** */
+        public String getId() {
+            return id;
+        }
+
+        /** */
+        public void setId(String id) {
+            this.id = id;
+        }
+
+        /** */
+        public String getOrgId() {
+            return orgId;
+        }
+
+        /** */
+        public void setOrgId(String orgId) {
+            this.orgId = orgId;
+        }
+
+        /** */
+        public String getName() {
+            return name;
+        }
+
+        /** */
+        public void setName(String name) {
+            this.name = name;
+        }
+    }
+
+    /**
+     *
+     */
+    public static class Organization {
+        /** */
+        @QuerySqlField(index = true)
+        private String id;
+
+        /** */
+        @QuerySqlField(index = true)
+        private String name;
+
+        /** */
+        public void setId(String id) {
+            this.id = id;
+        }
+
+        /** */
+        public String getId() {
+            return id;
+        }
+
+        /** */
+        public String getName() {
+            return name;
+        }
+
+        /** */
+        public void setName(String name) {
+            this.name = name;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/NonCollocatedRetryMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/NonCollocatedRetryMessageSelfTest.java
new file mode 100644
index 0000000000000..c602225e8e30e
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/NonCollocatedRetryMessageSelfTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import java.util.List;
+import javax.cache.Cache;
+import javax.cache.CacheException;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT;
+
+/**
+ * Failed to execute non-collocated query root cause message test
+ */
+public class NonCollocatedRetryMessageSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final int NODES_COUNT = 3;
+
+    /** */
+    private static final String ORG = "org";
+
+    /** */
+    private IgniteCache<String, JoinSqlTestHelper.Person> personCache;
+
+    /** */
+    public void testNonCollocatedRetryMessage() {
+        SqlQuery<String, JoinSqlTestHelper.Person> qry = new SqlQuery<String, JoinSqlTestHelper.Person>(JoinSqlTestHelper.Person.class, JoinSqlTestHelper.JOIN_SQL).setArgs("Organization #0");
+
+        qry.setDistributedJoins(true);
+
+        try {
+            List<Cache.Entry<String,JoinSqlTestHelper.Person>> prsns = personCache.query(qry).getAll();
+            fail("No CacheException emitted. Collection size="+prsns.size());
+        }
+        catch (CacheException e) {
+            assertTrue(e.getMessage(), e.getMessage().contains("Failed to execute non-collocated query"));
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCommunicationSpi(new TcpCommunicationSpi(){
+            volatile long reqId = -1;
+            /** {@inheritDoc} */
+            @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+                assert msg != null;
+
+                if ( GridIoMessage.class.isAssignableFrom(msg.getClass())){
+                    GridIoMessage gridMsg = (GridIoMessage)msg;
+
+                    if ( GridH2QueryRequest.class.isAssignableFrom( gridMsg.message().getClass() ) ){
+                        GridH2QueryRequest req = (GridH2QueryRequest) (gridMsg.message());
+
+                        if (reqId < 0) {
+                            reqId = req.requestId();
+
+                            String shutName = getTestIgniteInstanceName(1);
+
+                            stopGrid(shutName, true, false);
+                        }
+                        else if( reqId != req.requestId() ){
+                            try {
+                                U.sleep(IgniteSystemProperties.getLong(IGNITE_SQL_RETRY_TIMEOUT, GridReduceQueryExecutor.DFLT_RETRY_TIMEOUT));
+                            }
+                            catch (IgniteInterruptedCheckedException e) {
+                                // no-op
+                            }
+                        }
+                    }
+                }
+                super.sendMessage(node, msg, ackC);
+            }
+        });
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi(){
+            public long getNodesJoined() {
+                return stats.joinedNodesCount();
+            }
+        });
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        System.setProperty(IGNITE_SQL_RETRY_TIMEOUT, "5000");
+
+        startGridsMultiThreaded(NODES_COUNT, false);
+
+        personCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, JoinSqlTestHelper.Person>("pers")
+            .setBackups(1)
+            .setIndexedTypes(String.class, JoinSqlTestHelper.Person.class)
+        );
+
+        final IgniteCache<String, JoinSqlTestHelper.Organization> orgCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, JoinSqlTestHelper.Organization>(ORG)
+            .setBackups(1)
+            .setIndexedTypes(String.class, JoinSqlTestHelper.Organization.class)
+        );
+
+        awaitPartitionMapExchange();
+
+        JoinSqlTestHelper.populateDataIntoOrg(orgCache);
+
+        JoinSqlTestHelper.populateDataIntoPerson(personCache);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+
+}
+
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
new file mode 100644
index 0000000000000..326988739ec57
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import java.util.UUID;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
+import org.apache.ignite.internal.processors.query.GridQueryProcessor;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2RetryException;
+import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2QueryRequest;
+import org.apache.ignite.internal.util.GridSpinBusyLock;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_RETRY_TIMEOUT;
+import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.JOIN_SQL;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Organization;
+import static org.apache.ignite.internal.processors.query.h2.twostep.JoinSqlTestHelper.Person;
+
+/**
+ * Test for 6 retry cases
+ */
+public class RetryCauseMessageSelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final int NODES_COUNT = 2;
+
+    /** */
+    private static final String ORG_SQL = "select * from Organization";
+
+    /** */
+    private static final String ORG = "org";
+
+    /** */
+    private IgniteCache<String, Person> personCache;
+
+    /** */
+    private IgniteCache<String, Organization> orgCache;
+
+    /** */
+    private IgniteH2Indexing h2Idx;
+
+    /** */
+    @Override protected long getTestTimeout() {
+        return 600 * 1000;
+    }
+
+    /**
+     * Failed to reserve partitions for query (cache is not found on local node)
+     */
+    public void testSynthCacheWasNotFoundMessage() {
+        GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec");
+
+        GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec",
+            new MockGridMapQueryExecutor(null) {
+                @Override public void onMessage(UUID nodeId, Object msg) {
+                    if (GridH2QueryRequest.class.isAssignableFrom(msg.getClass())) {
+                        GridH2QueryRequest qryReq = (GridH2QueryRequest)msg;
+
+                        qryReq.caches().add(Integer.MAX_VALUE);
+
+                        startedExecutor.onMessage(nodeId, msg);
+
+                        qryReq.caches().remove(qryReq.caches().size() - 1);
+                    }
+                    else
+                        startedExecutor.onMessage(nodeId, msg);
+                }
+            }.insertRealExecutor(mapQryExec));
+
+        SqlQuery<String, Person> qry = new SqlQuery<String, Person>(Person.class, JOIN_SQL).setArgs("Organization #0");
+
+        qry.setDistributedJoins(true);
+
+        try {
+            personCache.query(qry).getAll();
+        }
+        catch (CacheException e) {
+            assertTrue(e.getMessage(), e.getMessage().contains("Failed to reserve partitions for query (cache is not found on local node) ["));
+
+            return;
+        }
+        finally {
+            GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec", mapQryExec);
+        }
+        fail();
+    }
+
+    /**
+     * Failed to reserve partitions for query (group reservation failed)
+     */
+    public void testGrpReservationFailureMessage() {
+        final GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec");
+
+        final ConcurrentMap<MapReservationKey, GridReservable> reservations = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "reservations");
+
+        GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec",
+            new MockGridMapQueryExecutor(null) {
+                @Override public void onMessage(UUID nodeId, Object msg) {
+                    if (GridH2QueryRequest.class.isAssignableFrom(msg.getClass())) {
+                        final MapReservationKey grpKey = new MapReservationKey(ORG, null);
+
+                        reservations.put(grpKey, new GridReservable() {
+
+                            @Override public boolean reserve() {
+                                return false;
+                            }
+
+                            @Override public void release() {}
+                        });
+                    }
+                    startedExecutor.onMessage(nodeId, msg);
+                }
+            }.insertRealExecutor(mapQryExec));
+
+        SqlQuery<String, Person> qry = new SqlQuery<String, Person>(Person.class, JOIN_SQL).setArgs("Organization #0");
+
+        qry.setDistributedJoins(true);
+
+        try {
+            personCache.query(qry).getAll();
+        }
+        catch (CacheException e) {
+            assertTrue(e.getMessage().contains("Failed to reserve partitions for query (group reservation failed) ["));
+
+            return;
+        }
+        finally {
+            GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec", mapQryExec);
+        }
+        fail();
+    }
+
+    /**
+     * Failed to reserve partitions for query (partition of REPLICATED cache is not in OWNING state)
+     */
+    public void testReplicatedCacheReserveFailureMessage() {
+        GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec");
+
+        final GridKernalContext ctx = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "ctx");
+
+        GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec",
+            new MockGridMapQueryExecutor(null) {
+                @Override public void onMessage(UUID nodeId, Object msg) {
+                    if (GridH2QueryRequest.class.isAssignableFrom(msg.getClass())) {
+                        GridH2QueryRequest qryReq = (GridH2QueryRequest)msg;
+
+                        GridCacheContext<?, ?> cctx = ctx.cache().context().cacheContext(qryReq.caches().get(0));
+
+                        GridDhtLocalPartition part = cctx.topology().localPartition(0, NONE, false);
+
+                        AtomicLong aState = GridTestUtils.getFieldValue(part, GridDhtLocalPartition.class, "state");
+
+                        long stateVal = aState.getAndSet(2);
+
+                        startedExecutor.onMessage(nodeId, msg);
+
+                        aState.getAndSet(stateVal);
+                    }
+                    else 
+                        startedExecutor.onMessage(nodeId, msg);
+                }
+            }.insertRealExecutor(mapQryExec));
+
+        SqlQuery<String, Organization> qry = new SqlQuery<>(Organization.class, ORG_SQL);
+
+        qry.setDistributedJoins(true);
+        try {
+            orgCache.query(qry).getAll();
+        }
+        catch (CacheException e) {
+            assertTrue(e.getMessage().contains("Failed to reserve partitions for query (partition of REPLICATED cache is not in OWNING state) ["));
+
+            return;
+        }
+        finally {
+            GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec", mapQryExec);
+        }
+        fail();
+    }
+
+    /**
+     * Failed to reserve partitions for query (partition of PARTITIONED cache cannot be reserved)
+     */
+    public void testPartitionedCacheReserveFailureMessage() {
+        GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec");
+
+        final GridKernalContext ctx = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "ctx");
+
+        GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec",
+            new MockGridMapQueryExecutor(null) {
+                @Override public void onMessage(UUID nodeId, Object msg) {
+                    if (GridH2QueryRequest.class.isAssignableFrom(msg.getClass())) {
+                        GridH2QueryRequest qryReq = (GridH2QueryRequest)msg;
+
+                        GridCacheContext<?, ?> cctx = ctx.cache().context().cacheContext(qryReq.caches().get(0));
+
+                        GridDhtLocalPartition part = cctx.topology().localPartition(0, NONE, false);
+
+                        AtomicLong aState = GridTestUtils.getFieldValue(part, GridDhtLocalPartition.class, "state");
+
+                        long stateVal = aState.getAndSet(2);
+
+                        startedExecutor.onMessage(nodeId, msg);
+
+                        aState.getAndSet(stateVal);
+                    }
+                    else
+                        startedExecutor.onMessage(nodeId, msg);
+
+                }
+            }.insertRealExecutor(mapQryExec));
+
+        SqlQuery<String, Person> qry = new SqlQuery<String, Person>(Person.class, JOIN_SQL).setArgs("Organization #0");
+
+        qry.setDistributedJoins(true);
+        try {
+            personCache.query(qry).getAll();
+        }
+        catch (CacheException e) {
+            assertTrue(e.getMessage().contains("Failed to reserve partitions for query (partition of PARTITIONED cache cannot be reserved) ["));
+
+            return;
+        }
+        finally {
+            GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec", mapQryExec);
+        }
+        fail();
+    }
+
+    /**
+     * Failed to execute non-collocated query (will retry)
+     */
+    public void testNonCollocatedFailureMessage() {
+        final GridMapQueryExecutor mapQryExec = GridTestUtils.getFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec");
+
+        final ConcurrentMap<MapReservationKey, GridReservable> reservations = GridTestUtils.getFieldValue(mapQryExec, GridMapQueryExecutor.class, "reservations");
+
+        GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec",
+            new MockGridMapQueryExecutor(null) {
+                @Override public void onMessage(UUID nodeId, Object msg) {
+                    if (GridH2QueryRequest.class.isAssignableFrom(msg.getClass())) {
+                        final MapReservationKey grpKey = new MapReservationKey(ORG, null);
+
+                        reservations.put(grpKey, new GridReservable() {
+
+                            @Override public boolean reserve() {
+                                throw new GridH2RetryException("test retry exception");
+                            }
+
+                            @Override public void release() {
+                            }
+                        });
+                    }
+                    startedExecutor.onMessage(nodeId, msg);
+
+                }
+            }.insertRealExecutor(mapQryExec));
+
+        SqlQuery<String, Person> qry = new SqlQuery<String, Person>(Person.class, JOIN_SQL).setArgs("Organization #0");
+
+        qry.setDistributedJoins(true);
+        try {
+            personCache.query(qry).getAll();
+        }
+        catch (CacheException e) {
+            assertTrue(e.getMessage().contains("Failed to execute non-collocated query (will retry) ["));
+
+            return;
+        }
+        finally {
+            GridTestUtils.setFieldValue(h2Idx, IgniteH2Indexing.class, "mapQryExec", mapQryExec);
+        }
+        fail();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setCommunicationSpi(new TcpCommunicationSpi(){
+            /** {@inheritDoc} */
+            @Override public void sendMessage(ClusterNode node, Message msg, IgniteInClosure<IgniteException> ackC) {
+                assert msg != null;
+
+                super.sendMessage(node, msg, ackC);
+            }
+        });
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        System.setProperty(IGNITE_SQL_RETRY_TIMEOUT, "5000");
+
+        Ignite ignite = startGridsMultiThreaded(NODES_COUNT, false);
+
+        GridQueryProcessor qryProc = grid(ignite.name()).context().query();
+
+        h2Idx = GridTestUtils.getFieldValue(qryProc, GridQueryProcessor.class, "idx");
+
+        personCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Person>("pers")
+            .setIndexedTypes(String.class, Person.class)
+        );
+
+        orgCache = ignite(0).getOrCreateCache(new CacheConfiguration<String, Organization>(ORG)
+            .setCacheMode(CacheMode.REPLICATED)
+            .setIndexedTypes(String.class, Organization.class)
+        );
+
+        awaitPartitionMapExchange();
+
+        JoinSqlTestHelper.populateDataIntoOrg(orgCache);
+
+        JoinSqlTestHelper.populateDataIntoPerson(personCache);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+
+    /**
+     * Wrapper around @{GridMapQueryExecutor}
+     */
+    private abstract static class MockGridMapQueryExecutor extends GridMapQueryExecutor {
+
+        /**
+         * Wrapped executor
+         */
+        GridMapQueryExecutor startedExecutor;
+
+        /** */
+        MockGridMapQueryExecutor insertRealExecutor(GridMapQueryExecutor realExecutor) {
+            this.startedExecutor = realExecutor;
+            return this;
+        }
+
+        /**
+         * @param busyLock Busy lock.
+         */
+        MockGridMapQueryExecutor(GridSpinBusyLock busyLock) {
+            super(busyLock);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onMessage(UUID nodeId, Object msg) {
+            startedExecutor.onMessage(nodeId, msg);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void cancelLazyWorkers() {
+            startedExecutor.cancelLazyWorkers();
+        }
+
+        /** {@inheritDoc} */
+        @Override GridSpinBusyLock busyLock() {
+            return startedExecutor.busyLock();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void onCacheStop(String cacheName) {
+            startedExecutor.onCacheStop(cacheName);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void stopAndUnregisterCurrentLazyWorker() {
+            startedExecutor.stopAndUnregisterCurrentLazyWorker();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void unregisterLazyWorker(MapQueryLazyWorker worker) {
+            startedExecutor.unregisterLazyWorker(worker);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int registeredLazyWorkers() {
+            return startedExecutor.registeredLazyWorkers();
+        }
+    }
+
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index 093423dd50f0d..536834cda9346 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -51,6 +51,10 @@
 import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlSegmentedIndexMultiNodeSelfTest;
 import org.apache.ignite.internal.processors.query.IgniteCacheGroupsSqlSegmentedIndexSelfTest;
 import org.apache.ignite.internal.processors.query.h2.twostep.CacheQueryMemoryLeakTest;
+import org.apache.ignite.internal.processors.query.h2.twostep.DisappearedCacheCauseRetryMessageSelfTest;
+import org.apache.ignite.internal.processors.query.h2.twostep.DisappearedCacheWasNotFoundMessageSelfTest;
+import org.apache.ignite.internal.processors.query.h2.twostep.NonCollocatedRetryMessageSelfTest;
+import org.apache.ignite.internal.processors.query.h2.twostep.RetryCauseMessageSelfTest;
 import org.apache.ignite.testframework.IgniteTestSuite;
 
 /**
@@ -110,6 +114,11 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(CacheQueryMemoryLeakTest.class);
 
+        suite.addTestSuite(NonCollocatedRetryMessageSelfTest.class);
+        suite.addTestSuite(RetryCauseMessageSelfTest.class);
+        suite.addTestSuite(DisappearedCacheCauseRetryMessageSelfTest.class);
+        suite.addTestSuite(DisappearedCacheWasNotFoundMessageSelfTest.class);
+
         return suite;
     }
 }

From c981338598510e30ff834cf17abf9cd5ba427072 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 6 Sep 2018 13:21:46 +0300
Subject: [PATCH 337/543] IGNITE-9422 Fixed NPE on clients when new binary meta
 from joined node arrived.

---
 .../CacheObjectBinaryProcessorImpl.java       |   9 +-
 ...eObjectBinaryProcessorOnDiscoveryTest.java | 125 ++++++++++++++++++
 .../ignite/testsuites/IgnitePdsTestSuite.java |   5 +
 3 files changed, 136 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 8e5ec5c924728..2d515764b9d94 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -476,7 +476,8 @@ public GridBinaryMarshaller marshaller() {
         try {
             BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0);
 
-            metadataFileStore.mergeAndWriteMetadata(mergedMeta);
+            if (!ctx.clientNode())
+                metadataFileStore.mergeAndWriteMetadata(mergedMeta);
 
             metadataLocCache.put(typeId, new BinaryMetadataHolder(mergedMeta, 0, 0));
         }
@@ -998,7 +999,8 @@ private IgniteNodeValidationResult validateBinaryMetadata(UUID rmtNodeId, Map<In
                             localMetaHolder.pendingVersion(),
                             localMetaHolder.acceptedVersion()));
 
-                    metadataFileStore.writeMetadata(mergedMeta);
+                    if (!ctx.clientNode())
+                        metadataFileStore.writeMetadata(mergedMeta);
                 }
             }
             else {
@@ -1014,7 +1016,8 @@ private IgniteNodeValidationResult validateBinaryMetadata(UUID rmtNodeId, Map<In
 
                 metadataLocCache.put(metaEntry.getKey(), newMetaHolder);
 
-                metadataFileStore.writeMetadata(newMeta);
+                if (!ctx.clientNode())
+                    metadataFileStore.writeMetadata(newMeta);
             }
         }
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.java
new file mode 100644
index 0000000000000..279d3c8c4e620
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.Arrays;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        if ("client".equals(igniteInstanceName))
+            cfg.setClientMode(true).setFailureHandler(new StopNodeFailureHandler());
+
+        return cfg.setDiscoverySpi(new TcpDiscoverySpi()
+                .setIpFinder(IP_FINDER))
+                .setDataStorageConfiguration(new DataStorageConfiguration()
+                        .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                                .setPersistenceEnabled(true)));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Tests that joining node metadata correctly handled on client.
+     * @throws Exception If fails.
+     */
+    public void testJoiningNodeBinaryMetaOnClient() throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(2);
+
+        ig0.cluster().active(true);
+
+        addBinaryType(ig0, "test_1", new IgniteBiTuple<>("name", String.class));
+
+        stopGrid(0);
+
+        Ignite ig1 = grid(1);
+
+        // Modify existing type.
+        addBinaryType(ig1, "test_1", new IgniteBiTuple<>("id", Integer.class));
+
+        // Add new type.
+        addBinaryType(ig1, "test_2", new IgniteBiTuple<>("name", String.class));
+
+        stopGrid(1);
+
+        startGrid(0);
+
+        IgniteEx client = startGrid(getConfiguration("client"));
+
+        startGrid(1);
+
+        awaitPartitionMapExchange();
+
+        // Check that new metadata from grid_1 was handled without NPE on client.
+        assertNull(client.context().failure().failureContext());
+
+        // Check that metadata from grid_1 correctly loaded on client.
+        assertTrue(client.binary().type("test_1").fieldNames().containsAll(Arrays.asList("id", "name")));
+        assertTrue(client.binary().type("test_2").fieldNames().contains("name"));
+    }
+
+    /**
+     * @param ig Ig.
+     * @param typeName Type name.
+     * @param fields Fields.
+     */
+    @SafeVarargs
+    private final BinaryObject addBinaryType(Ignite ig, String typeName, IgniteBiTuple<String, Class<?>>... fields) {
+        BinaryObjectBuilder builder = ig.binary().builder(typeName);
+
+        if (fields != null) {
+            for (IgniteBiTuple<String,Class<?>> field: fields)
+                builder.setField(field.get1(), field.get2());
+        }
+
+        return builder.build();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index 17a3e41f35987..fb262ce055cb5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheWithoutCheckpointsTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheConfigurationFileConsistencyCheckTest;
@@ -75,6 +76,10 @@ public static TestSuite suite() throws Exception {
         // Metrics
         suite.addTestSuite(FillFactorMetricTest.class);
 
+        // Binary meta tests.
+        suite.addTestSuite(IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.class);
+
+
         return suite;
     }
 

From f85211fcd857d1654ec5768a8a2f6a79328a4089 Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Tue, 4 Sep 2018 15:55:41 +0300
Subject: [PATCH 338/543] IGNITE-9448 Updated ZooKeeper version to 3.4.13 -
 Fixes #4671.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit dfaffe2af8f5e3b51eff3cfe426733311c2c100f)
---
 .../ipfinder/zk/ZookeeperIpFinderTest.java    |   3 +-
 .../curator/FixedTestingQuorumPeerMain.java   |  87 +++++++
 .../ipfinder/zk/curator/TestingCluster.java   | 237 ++++++++++++++++++
 .../zk/curator/TestingZooKeeperServer.java    | 169 +++++++++++++
 ...ookeeperDiscoverySpiAbstractTestSuite.java |   2 +-
 .../zk/ZookeeperDiscoverySpiTestSuite2.java   |   2 +-
 .../zk/internal/ZookeeperClientTest.java      |   2 +-
 .../internal/ZookeeperDiscoverySpiTest.java   |   4 +-
 8 files changed, 500 insertions(+), 6 deletions(-)
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/FixedTestingQuorumPeerMain.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingCluster.java
 create mode 100644 modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingZooKeeperServer.java

diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/ZookeeperIpFinderTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/ZookeeperIpFinderTest.java
index 20947c4fd27c1..d29ab1b49f67d 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/ZookeeperIpFinderTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/ZookeeperIpFinderTest.java
@@ -26,7 +26,6 @@
 import org.apache.curator.retry.ExponentialBackoffRetry;
 import org.apache.curator.retry.RetryNTimes;
 import org.apache.curator.test.InstanceSpec;
-import org.apache.curator.test.TestingCluster;
 import org.apache.curator.utils.CloseableUtils;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -36,6 +35,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -81,6 +81,7 @@ public ZookeeperIpFinderTest() {
 
         // start the ZK cluster
         zkCluster = new TestingCluster(ZK_CLUSTER_SIZE);
+
         zkCluster.start();
 
         // start the Curator client so we can perform assertions on the ZK state later
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/FixedTestingQuorumPeerMain.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/FixedTestingQuorumPeerMain.java
new file mode 100644
index 0000000000000..e0a416e8e9b96
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/FixedTestingQuorumPeerMain.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator;
+
+import org.apache.curator.test.ZooKeeperMainFace;
+import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.apache.zookeeper.server.quorum.QuorumPeer;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.channels.ServerSocketChannel;
+
+/**
+ */
+public class FixedTestingQuorumPeerMain extends QuorumPeerMain implements ZooKeeperMainFace {
+    @Override public void runFromConfig(QuorumPeerConfig config) throws IOException {
+        quorumPeer = QuorumPeer.testingQuorumPeer();
+        super.runFromConfig(config);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void kill() {
+        try {
+            if (quorumPeer != null) {
+                Field cnxnFactoryField = QuorumPeer.class.getDeclaredField("cnxnFactory");
+
+                cnxnFactoryField.setAccessible(true);
+
+                ServerCnxnFactory cnxnFactory = (ServerCnxnFactory)cnxnFactoryField.get(quorumPeer);
+
+                cnxnFactory.closeAll();
+
+                Field ssField = cnxnFactory.getClass().getDeclaredField("ss");
+
+                ssField.setAccessible(true);
+
+                ServerSocketChannel ss = (ServerSocketChannel)ssField.get(cnxnFactory);
+
+                ss.close();
+            }
+            close();
+        }
+        catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public QuorumPeer getQuorumPeer() {
+        return quorumPeer;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close() throws IOException {
+        if (quorumPeer != null)
+            quorumPeer.shutdown();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void blockUntilStarted() throws Exception {
+        while (quorumPeer == null) {
+            try {
+                Thread.sleep(100);
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
+            }
+        }
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingCluster.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingCluster.java
new file mode 100644
index 0000000000000..c6e8d7268cbaf
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingCluster.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.curator.test.ByteCodeRewrite;
+import org.apache.curator.test.InstanceSpec;
+import org.apache.curator.test.QuorumConfigBuilder;
+import org.apache.zookeeper.ZooKeeper;
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Manages an internally running ensemble of ZooKeeper servers. FOR TESTING PURPOSES ONLY.
+ * This class is a copy of {{org.apache.curator.test.TestingCluster}},
+ * but have very small change, that allow to run testing cluster with ZooKeeper 2.4.13 ver.
+ */
+public class TestingCluster implements Closeable {
+    static {
+        ByteCodeRewrite.apply();
+    }
+
+    /** Servers. */
+    private final List<TestingZooKeeperServer> servers;
+
+    /**
+     * Creates an ensemble comprised of <code>n</code> servers. Each server will use
+     * a temp directory and random ports
+     *
+     * @param instanceQty number of servers to create in the ensemble
+     */
+    public TestingCluster(int instanceQty) {
+        this(makeSpecs(instanceQty));
+    }
+
+    /**
+     * Creates an ensemble using the given server specs
+     *
+     * @param specs the server specs
+     */
+    public TestingCluster(InstanceSpec... specs) {
+        this(listToMap(ImmutableList.copyOf(specs)));
+    }
+
+    /**
+     * Creates an ensemble using the given server specs
+     *
+     * @param specs the server specs
+     */
+    public TestingCluster(Collection<InstanceSpec> specs) {
+        this(listToMap(specs));
+    }
+
+    /**
+     * Creates an ensemble using the given server specs
+     *
+     * @param specs map of an instance spec to its set of quorum instances. Allows simulation of an ensemble with
+     * instances having different config peers
+     */
+    public TestingCluster(Map<InstanceSpec, Collection<InstanceSpec>> specs) {
+        ImmutableList.Builder<TestingZooKeeperServer> serverBuilder = ImmutableList.builder();
+        for (Map.Entry<InstanceSpec, Collection<InstanceSpec>> entry : specs.entrySet()) {
+            List<InstanceSpec> instanceSpecs = Lists.newArrayList(entry.getValue());
+            int index = instanceSpecs.indexOf(entry.getKey());
+            Preconditions.checkState(index >= 0, entry.getKey() + " not found in specs");
+            QuorumConfigBuilder builder = new QuorumConfigBuilder(instanceSpecs);
+            serverBuilder.add(new TestingZooKeeperServer(builder, index));
+        }
+        servers = serverBuilder.build();
+    }
+
+    /**
+     * Returns the set of servers in the ensemble
+     *
+     * @return set of servers
+     */
+    public Collection<InstanceSpec> getInstances() {
+        Iterable<InstanceSpec> transformed = Iterables.transform
+            (
+                servers,
+                new Function<TestingZooKeeperServer, InstanceSpec>() {
+                    @Override
+                    public InstanceSpec apply(TestingZooKeeperServer server) {
+                        return server.getInstanceSpec();
+                    }
+                }
+            );
+        return Lists.newArrayList(transformed);
+    }
+
+    public List<TestingZooKeeperServer> getServers() {
+        return Lists.newArrayList(servers);
+    }
+
+    /**
+     * Returns the connection string to pass to the ZooKeeper constructor
+     *
+     * @return connection string
+     */
+    public String getConnectString() {
+        StringBuilder str = new StringBuilder();
+        for (InstanceSpec spec : getInstances()) {
+            if (str.length() > 0)
+                str.append(",");
+
+            str.append(spec.getConnectString());
+        }
+        return str.toString();
+    }
+
+    /**
+     * Start the ensemble. The cluster must be started before use.
+     *
+     * @throws Exception errors
+     */
+    public void start() throws Exception {
+        for (TestingZooKeeperServer server : servers)
+            server.start();
+
+    }
+
+    /**
+     * Shutdown the ensemble WITHOUT freeing resources, etc.
+     */
+    public void stop() throws IOException {
+        for (TestingZooKeeperServer server : servers)
+            server.stop();
+
+    }
+
+    /**
+     * Shutdown the ensemble, free resources, etc. If temp directories were used, they
+     * are deleted. You should call this in a <code>finally</code> block.
+     *
+     * @throws IOException errors
+     */
+    @Override public void close() throws IOException {
+        for (TestingZooKeeperServer server : servers)
+            server.close();
+    }
+
+    /**
+     * Kills the given server. This simulates the server unexpectedly crashing
+     *
+     * @param instance server to kill
+     * @return true if the instance was found
+     * @throws Exception errors
+     */
+    public boolean killServer(InstanceSpec instance) throws Exception {
+        for (TestingZooKeeperServer server : servers) {
+            if (server.getInstanceSpec().equals(instance)) {
+                server.kill();
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Restart the given server of the cluster
+     *
+     * @param instance server instance
+     * @return true of the server was found
+     * @throws Exception errors
+     */
+    public boolean restartServer(InstanceSpec instance) throws Exception {
+        for (TestingZooKeeperServer server : servers) {
+            if (server.getInstanceSpec().equals(instance)) {
+                server.restart();
+                return true;
+            }
+        }
+        return false;
+    }
+
+    /**
+     * Given a ZooKeeper instance, returns which server it is connected to
+     *
+     * @param client ZK instance
+     * @return the server
+     * @throws Exception errors
+     */
+    public InstanceSpec findConnectionInstance(ZooKeeper client) throws Exception {
+        Method m = client.getClass().getDeclaredMethod("testableRemoteSocketAddress");
+        m.setAccessible(true);
+        InetSocketAddress address = (InetSocketAddress)m.invoke(client);
+        if (address != null) {
+            for (TestingZooKeeperServer server : servers) {
+                if (server.getInstanceSpec().getPort() == address.getPort())
+                    return server.getInstanceSpec();
+            }
+        }
+
+        return null;
+    }
+
+    private static Map<InstanceSpec, Collection<InstanceSpec>> makeSpecs(int instanceQty) {
+        ImmutableList.Builder<InstanceSpec> builder = ImmutableList.builder();
+        for (int i = 0; i < instanceQty; ++i)
+            builder.add(InstanceSpec.newInstanceSpec());
+
+        return listToMap(builder.build());
+    }
+
+    private static Map<InstanceSpec, Collection<InstanceSpec>> listToMap(Collection<InstanceSpec> list) {
+        ImmutableMap.Builder<InstanceSpec, Collection<InstanceSpec>> mapBuilder = ImmutableMap.builder();
+        for (InstanceSpec spec : list)
+            mapBuilder.put(spec, list);
+
+        return mapBuilder.build();
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingZooKeeperServer.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingZooKeeperServer.java
new file mode 100644
index 0000000000000..808e617e17edb
--- /dev/null
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/zk/curator/TestingZooKeeperServer.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator;
+
+import org.apache.curator.test.DirectoryUtils;
+import org.apache.curator.test.InstanceSpec;
+import org.apache.curator.test.QuorumConfigBuilder;
+import org.apache.curator.test.TestingZooKeeperMain;
+import org.apache.curator.test.ZooKeeperMainFace;
+import org.apache.zookeeper.server.quorum.QuorumPeer;
+import org.apache.zookeeper.server.quorum.QuorumPeerConfig;
+import org.apache.zookeeper.server.quorum.QuorumPeerMain;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ */
+public class TestingZooKeeperServer extends QuorumPeerMain implements Closeable {
+    /** Logger. */
+    private static final Logger logger = LoggerFactory.getLogger(org.apache.curator.test.TestingZooKeeperServer.class);
+
+    /** Config builder. */
+    private final QuorumConfigBuilder configBuilder;
+    /** This instance index. */
+    private final int thisInstanceIndex;
+    /** Main. */
+    private volatile ZooKeeperMainFace main;
+    /** State. */
+    private final AtomicReference<State> state = new AtomicReference<>(State.LATENT);
+
+    /**
+     * Server state.
+     */
+    private enum State {
+        /** Latent. */
+        LATENT,
+        /** Started. */
+        STARTED,
+        /** Stopped. */
+        STOPPED,
+        /** Closed. */
+        CLOSED
+    }
+
+    /**
+     * @param configBuilder Config builder.
+     */
+    public TestingZooKeeperServer(QuorumConfigBuilder configBuilder) {
+        this(configBuilder, 0);
+    }
+
+    /**
+     * @param configBuilder Config builder.
+     * @param thisInstanceIndex This instance index.
+     */
+    public TestingZooKeeperServer(QuorumConfigBuilder configBuilder, int thisInstanceIndex) {
+        this.configBuilder = configBuilder;
+        this.thisInstanceIndex = thisInstanceIndex;
+        main = (configBuilder.size() > 1) ? new FixedTestingQuorumPeerMain() : new TestingZooKeeperMain();
+    }
+
+    /** {@inheritDoc} */
+    public QuorumPeer getQuorumPeer() {
+        return main.getQuorumPeer();
+    }
+
+    /**
+     *
+     */
+    public Collection<InstanceSpec> getInstanceSpecs() {
+        return configBuilder.getInstanceSpecs();
+    }
+
+    public void kill() {
+        main.kill();
+        state.set(State.STOPPED);
+    }
+
+    /**
+     * Restart the server. If the server is running it will be stopped and then
+     * started again. If it is not running (in a LATENT or STOPPED state) then
+     * it will be restarted. If it is in a CLOSED state then an exception will
+     * be thrown.
+     *
+     * @throws Exception
+     */
+    public void restart() throws Exception {
+        // Can't restart from a closed state as all the temporary data is gone
+        if (state.get() == State.CLOSED)
+            throw new IllegalStateException("Cannot restart a closed instance");
+
+        // If the server's currently running then stop it.
+        if (state.get() == State.STARTED)
+            stop();
+
+        // Set to a LATENT state so we can restart
+        state.set(State.LATENT);
+
+        main = (configBuilder.size() > 1) ? new FixedTestingQuorumPeerMain() : new TestingZooKeeperMain();
+        start();
+    }
+
+    /**
+     *
+     */
+    public void stop() throws IOException {
+        if (state.compareAndSet(State.STARTED, State.STOPPED))
+            main.close();
+    }
+
+    /**
+     *
+     */
+    public InstanceSpec getInstanceSpec() {
+        return configBuilder.getInstanceSpec(thisInstanceIndex);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close() throws IOException {
+        stop();
+
+        if (state.compareAndSet(State.STOPPED, State.CLOSED)) {
+            InstanceSpec spec = getInstanceSpec();
+            if (spec.deleteDataDirectoryOnClose())
+                DirectoryUtils.deleteRecursively(spec.getDataDirectory());
+        }
+    }
+
+    /**
+     *
+     */
+    public void start() throws Exception {
+        if (!state.compareAndSet(State.LATENT, State.STARTED))
+            return;
+
+        new Thread(new Runnable() {
+            public void run() {
+                try {
+                    QuorumPeerConfig config = configBuilder.buildConfig(thisInstanceIndex);
+                    main.runFromConfig(config);
+                }
+                catch (Exception e) {
+                    logger.error(String.format("From testing server (random state: %s) for instance: %s", String.valueOf(configBuilder.isFromRandom()), getInstanceSpec()), e);
+                }
+            }
+        }).start();
+
+        main.blockUntilStarted();
+    }
+}
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
index c5d34884a8c08..9f0cc6d0f7439 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiAbstractTestSuite.java
@@ -22,7 +22,7 @@
 import java.util.List;
 import junit.framework.TestSuite;
 import org.apache.curator.test.InstanceSpec;
-import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.spi.discovery.DiscoverySpi;
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
index 012366f70abe6..3b8ddeefefe1a 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiTestSuite2.java
@@ -18,7 +18,7 @@
 package org.apache.ignite.spi.discovery.zk;
 
 import junit.framework.TestSuite;
-import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster;
 import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest;
 import org.apache.ignite.internal.IgniteClientReconnectCacheTest;
 import org.apache.ignite.internal.processors.cache.datastructures.IgniteClientDataStructuresTest;
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
index d228e03070ca7..0af7e709e2171 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
@@ -26,7 +26,7 @@
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.curator.test.TestingCluster;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.internal.U;
diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
index 035ca3b3d2d85..5e1a10ae52b4c 100644
--- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
+++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoverySpiTest.java
@@ -53,8 +53,8 @@
 import org.apache.curator.framework.CuratorFramework;
 import org.apache.curator.framework.CuratorFrameworkFactory;
 import org.apache.curator.retry.RetryNTimes;
-import org.apache.curator.test.TestingCluster;
-import org.apache.curator.test.TestingZooKeeperServer;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingCluster;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.zk.curator.TestingZooKeeperServer;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;

From 3965d380db205a8439117a7f3f63590a62dd8952 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Thu, 6 Sep 2018 16:30:00 +0300
Subject: [PATCH 339/543] IGNITE-9479 Fixed spontaneous rebalance during cache
 start and improved logging level - Fixes #4691.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../dht/preloader/GridDhtForceKeysFuture.java |  36 ++--
 .../preloader/GridDhtPartitionDemander.java   |  22 +--
 .../preloader/GridDhtPartitionSupplier.java   |   4 +-
 .../GridDhtPartitionsExchangeFuture.java      |  10 +-
 .../dht/preloader/GridDhtPreloader.java       |  43 ++---
 .../dht/IgniteCacheStartWithLoadTest.java     | 165 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite7.java     |   2 +
 7 files changed, 213 insertions(+), 69 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheStartWithLoadTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java
index fe216a00379ac..e1591bb0df6a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java
@@ -271,8 +271,8 @@ private boolean map(Iterable<KeyCacheObject> keys, Collection<ClusterNode> exc)
 
                         assert !n.id().equals(loc.id());
 
-                        if (log.isDebugEnabled())
-                            log.debug("Sending force key request [cacheName=" + cctx.name() + "node=" + n.id() +
+                        if (log.isTraceEnabled())
+                            log.trace("Sending force key request [cacheName=" + cctx.name() + "node=" + n.id() +
                                 ", req=" + req + ']');
 
                         cctx.io().send(n, req, cctx.ioPolicy());
@@ -307,10 +307,10 @@ private Map<ClusterNode, Set<KeyCacheObject>> map(KeyCacheObject key,
 
         try {
             if (e != null && !e.isNewLocked()) {
-                if (log.isDebugEnabled()) {
+                if (log.isTraceEnabled()) {
                     int part = cctx.affinity().partition(key);
 
-                    log.debug("Will not rebalance key (entry is not new) [cacheName=" + cctx.name() +
+                    log.trace("Will not rebalance key (entry is not new) [cacheName=" + cctx.name() +
                         ", key=" + key + ", part=" + part + ", locId=" + cctx.nodeId() + ']');
                 }
 
@@ -319,8 +319,8 @@ private Map<ClusterNode, Set<KeyCacheObject>> map(KeyCacheObject key,
             }
         }
         catch (GridCacheEntryRemovedException ignore) {
-            if (log.isDebugEnabled())
-                log.debug("Received removed DHT entry for force keys request [entry=" + e +
+            if (log.isTraceEnabled())
+                log.trace("Received removed DHT entry for force keys request [entry=" + e +
                     ", locId=" + cctx.nodeId() + ']');
         }
 
@@ -330,8 +330,8 @@ private Map<ClusterNode, Set<KeyCacheObject>> map(KeyCacheObject key,
             new ArrayList<>(F.view(top.owners(part, topVer), F.notIn(exc)));
 
         if (owners.isEmpty() || (owners.contains(loc) && cctx.rebalanceEnabled())) {
-            if (log.isDebugEnabled())
-                log.debug("Will not rebalance key (local node is owner) [key=" + key + ", part=" + part +
+            if (log.isTraceEnabled())
+                log.trace("Will not rebalance key (local node is owner) [key=" + key + ", part=" + part +
                     "topVer=" + topVer + ", locId=" + cctx.nodeId() + ']');
 
             // Key is already rebalanced.
@@ -341,8 +341,8 @@ private Map<ClusterNode, Set<KeyCacheObject>> map(KeyCacheObject key,
         // Create partition.
         GridDhtLocalPartition locPart = top.localPartition(part, topVer, false);
 
-        if (log.isDebugEnabled())
-            log.debug("Mapping local partition [loc=" + cctx.localNodeId() + ", topVer" + topVer +
+        if (log.isTraceEnabled())
+            log.trace("Mapping local partition [loc=" + cctx.localNodeId() + ", topVer" + topVer +
                 ", part=" + locPart + ", owners=" + owners + ", allOwners=" + U.toShortString(top.owners(part)) + ']');
 
         if (locPart == null)
@@ -359,8 +359,8 @@ else if (!cctx.rebalanceEnabled() || locPart.state() == MOVING) {
                 pick = F.first(F.view(owners, F.remoteNodes(loc.id())));
 
             if (pick == null) {
-                if (log.isDebugEnabled())
-                    log.debug("Will not rebalance key (no nodes to request from with rebalancing disabled) [key=" +
+                if (log.isTraceEnabled())
+                    log.trace("Will not rebalance key (no nodes to request from with rebalancing disabled) [key=" +
                         key + ", part=" + part + ", locId=" + cctx.nodeId() + ']');
 
                 return mappings;
@@ -375,15 +375,15 @@ else if (!cctx.rebalanceEnabled() || locPart.state() == MOVING) {
 
             mappedKeys.add(key);
 
-            if (log.isDebugEnabled())
-                log.debug("Will rebalance key from node [cacheName=" + cctx.name() + ", key=" + key + ", part=" +
+            if (log.isTraceEnabled())
+                log.trace("Will rebalance key from node [cacheName=" + cctx.name() + ", key=" + key + ", part=" +
                     part + ", node=" + pick.id() + ", locId=" + cctx.nodeId() + ']');
         }
         else if (locPart.state() != OWNING)
             invalidParts.add(part);
         else {
-            if (log.isDebugEnabled())
-                log.debug("Will not rebalance key (local partition is not MOVING) [cacheName=" + cctx.name() +
+            if (log.isTraceEnabled())
+                log.trace("Will not rebalance key (local partition is not MOVING) [cacheName=" + cctx.name() +
                     ", key=" + key + ", part=" + locPart + ", locId=" + cctx.nodeId() + ']');
         }
 
@@ -556,8 +556,8 @@ void onResult(GridDhtForceKeysResponse res) {
                         return;
                     }
                     catch (GridCacheEntryRemovedException ignore) {
-                        if (log.isDebugEnabled())
-                            log.debug("Trying to rebalance removed entry (will ignore) [cacheName=" +
+                        if (log.isTraceEnabled())
+                            log.trace("Trying to rebalance removed entry (will ignore) [cacheName=" +
                                 cctx.name() + ", entry=" + entry + ']');
                     }
                     finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 1eeebaef3a79e..0375de4271d1a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -761,9 +761,9 @@ public void handleSupplyMessage(
                                         GridCacheEntryInfo entry = infos.next();
 
                                         if (!preloadEntry(node, p, entry, topVer)) {
-                                            if (log.isDebugEnabled())
-                                                log.debug("Got entries for invalid partition during " +
-                                                        "preloading (will skip) [p=" + p + ", entry=" + entry + ']');
+                                            if (log.isTraceEnabled())
+                                                log.trace("Got entries for invalid partition during " +
+                                                    "preloading (will skip) [p=" + p + ", entry=" + entry + ']');
 
                                             break;
                                         }
@@ -874,8 +874,8 @@ private boolean preloadEntry(
 
                 cached = cctx.dhtCache().entryEx(entry.key());
 
-                if (log.isDebugEnabled())
-                    log.debug("Rebalancing key [key=" + entry.key() + ", part=" + p + ", node=" + from.id() + ']');
+                if (log.isTraceEnabled())
+                    log.trace("Rebalancing key [key=" + entry.key() + ", part=" + p + ", node=" + from.id() + ']');
 
                 if (preloadPred == null || preloadPred.apply(entry)) {
                     if (cached.initialValue(
@@ -898,17 +898,17 @@ private boolean preloadEntry(
                     else {
                         cctx.evicts().touch(cached, topVer); // Start tracking.
 
-                        if (log.isDebugEnabled())
-                            log.debug("Rebalancing entry is already in cache (will ignore) [key=" + cached.key() +
+                        if (log.isTraceEnabled())
+                            log.trace("Rebalancing entry is already in cache (will ignore) [key=" + cached.key() +
                                 ", part=" + p + ']');
                     }
                 }
-                else if (log.isDebugEnabled())
-                    log.debug("Rebalance predicate evaluated to false for entry (will ignore): " + entry);
+                else if (log.isTraceEnabled())
+                    log.trace("Rebalance predicate evaluated to false for entry (will ignore): " + entry);
             }
             catch (GridCacheEntryRemovedException ignored) {
-                if (log.isDebugEnabled())
-                    log.debug("Entry has been concurrently removed while rebalancing (will ignore) [key=" +
+                if (log.isTraceEnabled())
+                    log.trace("Entry has been concurrently removed while rebalancing (will ignore) [key=" +
                         cached.key() + ", part=" + p + ']');
             }
             catch (GridDhtInvalidPartitionException ignored) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 4946d7e3f0eed..0033b92242519 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -380,8 +380,8 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                 if (preloadPred == null || preloadPred.apply(info))
                     s.addEntry0(part, iter.historical(part), info, grp.shared(), grp.cacheObjectContext());
                 else {
-                    if (log.isDebugEnabled())
-                        log.debug("Rebalance predicate evaluated to false (will not send " +
+                    if (log.isTraceEnabled())
+                        log.trace("Rebalance predicate evaluated to false (will not send " +
                             "cache entry): " + info);
                 }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 90829d6f5c55d..d3789011e4802 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1173,8 +1173,8 @@ private void distributedExchange() throws IgniteCheckedException {
 
         boolean distributed = true;
 
-        // Do not perform distributed partition release in case of cluster activation or caches start.
-        if (activateCluster() || hasCachesToStart())
+        // Do not perform distributed partition release in case of cluster activation.
+        if (activateCluster())
             distributed = false;
 
         // On first phase we wait for finishing all local tx updates, atomic updates and lock releases on all nodes.
@@ -2854,11 +2854,7 @@ private void finishExchangeOnCoordinator(@Nullable Collection<ClusterNode> sndRe
                 }
             }
 
-            // Don't validate partitions state in case of caches start.
-            boolean skipValidation = hasCachesToStart();
-
-            if (!skipValidation)
-                validatePartitionsState();
+            validatePartitionsState();
 
             if (firstDiscoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT) {
                 assert firstDiscoEvt instanceof DiscoveryCustomEvent;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 77f48661d588b..5a60eb66eda94 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -257,7 +257,7 @@ private IgniteCheckedException stopError() {
                     msg.partitions().addHistorical(p, part.initialUpdateCounter(), countersMap.updateCounter(p), partCnt);
                 }
                 else {
-                    Collection<ClusterNode> picked = pickOwners(p, topVer);
+                    List<ClusterNode> picked = remoteOwners(p, topVer);
 
                     if (picked.isEmpty()) {
                         top.own(part);
@@ -274,7 +274,7 @@ private IgniteCheckedException stopError() {
                             log.debug("Owning partition as there are no other owners: " + part);
                     }
                     else {
-                        ClusterNode n = F.rand(picked);
+                        ClusterNode n = picked.get(0);
 
                         GridDhtPartitionDemandMessage msg = assignments.get(n);
 
@@ -303,42 +303,23 @@ private IgniteCheckedException stopError() {
     }
 
     /**
-     * Picks owners for specified partition {@code p} from affinity.
+     * Returns remote owners (excluding local node) for specified partition {@code p}.
      *
      * @param p Partition.
      * @param topVer Topology version.
-     * @return Picked owners.
+     * @return Nodes owning this partition.
      */
-    private Collection<ClusterNode> pickOwners(int p, AffinityTopologyVersion topVer) {
-        Collection<ClusterNode> affNodes = grp.affinity().cachedAffinity(topVer).get(p);
-
-        int affCnt = affNodes.size();
-
-        Collection<ClusterNode> rmts = remoteOwners(p, topVer);
+    private List<ClusterNode> remoteOwners(int p, AffinityTopologyVersion topVer) {
+        List<ClusterNode> owners = grp.topology().owners(p, topVer);
 
-        int rmtCnt = rmts.size();
+        List<ClusterNode> res = new ArrayList<>(owners.size());
 
-        if (rmtCnt <= affCnt)
-            return rmts;
-
-        List<ClusterNode> sorted = new ArrayList<>(rmts);
-
-        // Sort in descending order, so nodes with higher order will be first.
-        Collections.sort(sorted, CU.nodeComparator(false));
-
-        // Pick newest nodes.
-        return sorted.subList(0, affCnt);
-    }
+        for (ClusterNode owner : owners) {
+            if (!owner.id().equals(ctx.localNodeId()))
+                res.add(owner);
+        }
 
-    /**
-     * Returns remote owners (excluding local node) for specified partition {@code p}.
-     *
-     * @param p Partition.
-     * @param topVer Topology version.
-     * @return Nodes owning this partition.
-     */
-    private Collection<ClusterNode> remoteOwners(int p, AffinityTopologyVersion topVer) {
-        return F.view(grp.topology().owners(p, topVer), F.remoteNodes(ctx.localNodeId()));
+        return res;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheStartWithLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheStartWithLoadTest.java
new file mode 100644
index 0000000000000..acccc5be07fed
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheStartWithLoadTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class IgniteCacheStartWithLoadTest extends GridCommonAbstractTest {
+    /** */
+    static final String CACHE_NAME = "tx_repl";
+
+    @Override
+    protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        CacheConfiguration ccfg = new CacheConfiguration().setName(CACHE_NAME)
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+            .setCacheMode(CacheMode.REPLICATED)
+            .setDataRegionName("ds")
+            .setAffinity(new RendezvousAffinityFunction(false, 32));
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+            .setDataRegionConfigurations(
+                new DataRegionConfiguration()
+                    .setName("ds")
+                    .setPersistenceEnabled(true)
+                    .setMaxSize(1024 * 1024 * 1024)
+            );
+
+        cfg.setDataStorageConfiguration(dsCfg);
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testNoRebalanceDuringCacheStart() throws Exception {
+        IgniteEx crd = (IgniteEx)startGrids(4);
+
+        crd.cluster().active(true);
+
+        AtomicBoolean txLoadStop = new AtomicBoolean();
+
+        AtomicInteger txLoaderNo = new AtomicInteger(0);
+
+        IgniteInternalFuture txLoadFuture = GridTestUtils.runMultiThreadedAsync(() -> {
+            Ignite node = grid(txLoaderNo.getAndIncrement());
+            IgniteCache<Object, Object> cache = node.cache(CACHE_NAME);
+            ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+            final int keys = 5;
+            final int keysSpace = 10_000;
+
+            while (!txLoadStop.get()) {
+                try (Transaction tx = node.transactions().txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+                    for (int it = 0; it < keys; it++) {
+                        int key = rnd.nextInt(keysSpace);
+                        byte[] value = new byte[2048];
+
+                        cache.put(key, value);
+                    }
+                    tx.commit();
+
+                    U.sleep(10);
+                }
+                catch (Throwable t) {
+                    log.warning("Unexpected exception during tx load.", t);
+                }
+            }
+        }, 4, "tx-loader");
+
+        AtomicBoolean hasRebalance = new AtomicBoolean();
+
+        AtomicBoolean cacheRestartStop = new AtomicBoolean();
+
+        IgniteInternalFuture cacheRestartFuture = GridTestUtils.runAsync(() -> {
+            Ignite node = grid(0);
+
+            final String tmpCacheName = "tmp";
+
+            while (!cacheRestartStop.get()) {
+                try {
+                    node.getOrCreateCache(tmpCacheName);
+
+                    boolean hasMoving = false;
+
+                    for (int i = 0; i < 4; i++) {
+                        hasMoving |= grid(i).cachex(CACHE_NAME).context().topology().hasMovingPartitions();
+                    }
+
+                    if (hasMoving) {
+                        log.error("Cache restarter has been stopped because rebalance is triggered for stable caches.");
+
+                        hasRebalance.set(true);
+
+                        return;
+                    }
+
+                    node.destroyCache(tmpCacheName);
+
+                    U.sleep(10_000);
+                }
+                catch (Throwable t) {
+                    log.warning("Unexpected exception during caches restart.", t);
+                }
+            }
+        });
+
+        U.sleep(60_000);
+
+        cacheRestartStop.set(true);
+        txLoadStop.set(true);
+
+        cacheRestartFuture.get();
+        txLoadFuture.get();
+
+        Assert.assertFalse(hasRebalance.get());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 5 * 60 * 1000;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index f3b6a9d84f6ac..fa64c7653ab78 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
 import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.CheckpointBufferDeadlockTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncWithPersistenceTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxWithSmallTimeoutAndContentionOneKeyTest;
@@ -62,6 +63,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         TestSuite suite = new TestSuite("IgniteCache With Persistence Test Suite");
 
         suite.addTestSuite(CheckpointBufferDeadlockTest.class);
+        suite.addTestSuite(IgniteCacheStartWithLoadTest.class);
 
         suite.addTestSuite(AuthenticationConfigurationClusterTest.class);
         suite.addTestSuite(AuthenticationProcessorSelfTest.class);

From 3921967d071bfb552ffcea662d8258c584052192 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 6 Sep 2018 18:03:50 +0300
Subject: [PATCH 340/543] IGNITE-9084 Fixed error handling for historical
 rebalance - Fixes #4437.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../communication/GridIoMessageFactory.java   |   6 +
 .../processors/cache/GridCacheIoManager.java  |   3 +-
 .../preloader/GridDhtPartitionDemander.java   |  14 +-
 .../preloader/GridDhtPartitionSupplier.java   |  69 +++++--
 .../GridDhtPartitionSupplyMessage.java        |   6 +-
 .../GridDhtPartitionSupplyMessageV2.java      | 153 ++++++++++++++
 .../persistence/GridCacheOffheapManager.java  |  10 +
 .../db/wal/IgniteWalRebalanceTest.java        | 186 +++++++++++++++++-
 8 files changed, 426 insertions(+), 21 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 581c32e4b1fb0..a5f0066092f20 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -54,6 +54,7 @@
 import org.apache.ignite.internal.processors.cache.binary.MetadataRequestMessage;
 import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.LatchAckMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTtlUpdateRequest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryRequest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryResponse;
@@ -867,6 +868,11 @@ public GridIoMessageFactory(MessageFactory[] ext) {
 
                 break;
 
+            case 120:
+                msg = new GridDhtPartitionSupplyMessageV2();
+
+                break;
+
             case 124:
                 msg = new GridMessageCollection<>();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 01344211f36e9..1e25c935f8104 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -914,7 +914,8 @@ private void processFailedMessage(UUID nodeId,
 
             break;
 
-            case 114: {
+            case 114:
+            case 120: {
                 processMessage(nodeId, msg, c);// Will be handled by Rebalance Demander.
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 0375de4271d1a..ca828953c4c45 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -688,10 +688,20 @@ public void handleSupplyMessage(
         if (log.isDebugEnabled())
             log.debug("Received supply message [grp=" + grp.cacheOrGroupName() + ", msg=" + supply + ']');
 
-        // Check whether there were class loading errors on unmarshal
+        // Check whether there were error during supply message unmarshalling process.
         if (supply.classError() != null) {
             U.warn(log, "Rebalancing from node cancelled [grp=" + grp.cacheOrGroupName() + ", node=" + nodeId +
-                "]. Class got undeployed during preloading: " + supply.classError());
+                "]. Supply message couldn't be unmarshalled: " + supply.classError());
+
+            fut.cancel(nodeId);
+
+            return;
+        }
+
+        // Check whether there were error during supplying process.
+        if (supply.error() != null) {
+            U.warn(log, "Rebalancing from node cancelled [grp=" + grp.cacheOrGroupName() + ", node=" + nodeId +
+                "]. Supplier has failed with error: " + supply.error());
 
             fut.cancel(nodeId);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 0033b92242519..131d16fc8aa72 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -221,9 +221,11 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
         if (node == null)
             return;
 
-        try {
-            SupplyContext sctx;
+        IgniteRebalanceIterator iter = null;
+
+        SupplyContext sctx = null;
 
+        try {
             synchronized (scMap) {
                 sctx = scMap.remove(contextId);
 
@@ -232,7 +234,7 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                     scMap.put(contextId, sctx);
 
                     if (log.isDebugEnabled())
-                        log.debug("Stale demand message [grp=" + grp.cacheOrGroupName()
+                        log.debug("Stale demand message [cache=" + grp.cacheOrGroupName()
                             + ", actualContext=" + sctx
                             + ", from=" + nodeId
                             + ", demandMsg=" + d + "]");
@@ -244,7 +246,7 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
             // Demand request should not contain empty partitions if no supply context is associated with it.
             if (sctx == null && (d.partitions() == null || d.partitions().isEmpty())) {
                 if (log.isDebugEnabled())
-                    log.debug("Empty demand message [grp=" + grp.cacheOrGroupName()
+                    log.debug("Empty demand message [cache=" + grp.cacheOrGroupName()
                         + ", from=" + nodeId
                         + ", topicId=" + topicId
                         + ", demandMsg=" + d + "]");
@@ -275,8 +277,6 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                     d.topologyVersion(),
                     grp.deploymentEnabled());
 
-            IgniteRebalanceIterator iter;
-
             Set<Integer> remainingParts;
 
             if (sctx == null || sctx.iterator == null) {
@@ -430,13 +430,56 @@ else if (iter.isPartitionMissing(p)) {
                     ", topology=" + demTop + ", rebalanceId=" + d.rebalanceId() +
                     ", topicId=" + topicId + "]");
         }
-        catch (IgniteCheckedException e) {
-            U.error(log, "Failed to send partition supply message to node: " + nodeId, e);
-        }
-        catch (IgniteSpiException e) {
-            if (log.isDebugEnabled())
-                log.debug("Failed to send message to node (current node is stopping?) [node=" + node.id() +
-                    ", msg=" + e.getMessage() + ']');
+        catch (Throwable t) {
+            if (grp.shared().kernalContext().isStopping())
+                return;
+
+            // Sending supply messages with error requires new protocol.
+            boolean sendErrMsg = node.version().compareTo(GridDhtPartitionSupplyMessageV2.AVAILABLE_SINCE) >= 0;
+
+            if (t instanceof IgniteSpiException) {
+                if (log.isDebugEnabled())
+                    log.debug("Failed to send message to node (current node is stopping?) [node=" + node.id() +
+                        ", msg=" + t.getMessage() + ']');
+
+                sendErrMsg = false;
+            }
+            else
+                U.error(log, "Failed to continue supplying process for " +
+                    "[cache=" + grp.cacheOrGroupName() + ", node=" + nodeId
+                    + ", topicId=" + contextId.get2() + ", topVer=" + contextId.get3() + "]", t);
+
+            try {
+                if (sctx != null)
+                    clearContext(sctx, log);
+                else if (iter != null)
+                    iter.close();
+            }
+            catch (Throwable t1) {
+                U.error(log, "Failed to cleanup supplying context " +
+                        "[cache=" + grp.cacheOrGroupName() + ", node=" + nodeId
+                        + ", topicId=" + contextId.get2() + ", topVer=" + contextId.get3() + "]", t1);
+            }
+
+            if (!sendErrMsg)
+                return;
+
+            try {
+                GridDhtPartitionSupplyMessageV2 errMsg = new GridDhtPartitionSupplyMessageV2(
+                    d.rebalanceId(),
+                    grp.groupId(),
+                    d.topologyVersion(),
+                    grp.deploymentEnabled(),
+                    t
+                );
+
+                reply(node, d, errMsg, contextId);
+            }
+            catch (Throwable t1) {
+                U.error(log, "Failed to send supply error message for " +
+                    "[cache=" + grp.cacheOrGroupName() + ", node=" + nodeId
+                        + ", topicId=" + contextId.get2() + ", topVer=" + contextId.get3() + "]", t1);
+            }
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
index 4ecffc492c7b4..284700ad3a44f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
@@ -89,10 +89,12 @@ public class GridDhtPartitionSupplyMessage extends GridCacheGroupIdMessage imple
      * @param topVer Topology version.
      * @param addDepInfo Deployment info flag.
      */
-    GridDhtPartitionSupplyMessage(long rebalanceId,
+    GridDhtPartitionSupplyMessage(
+        long rebalanceId,
         int grpId,
         AffinityTopologyVersion topVer,
-        boolean addDepInfo) {
+        boolean addDepInfo
+    ) {
         this.grpId = grpId;
         this.rebalanceId = rebalanceId;
         this.topVer = topVer;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
new file mode 100644
index 0000000000000..a77576683815f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Supply message with supplier error transfer support.
+ */
+public class GridDhtPartitionSupplyMessageV2 extends GridDhtPartitionSupplyMessage {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Available since. */
+    public static final IgniteProductVersion AVAILABLE_SINCE = IgniteProductVersion.fromString("2.7.0");
+
+    /** Supplying process error. */
+    @GridDirectTransient
+    private Throwable err;
+
+    /** Supplying process error bytes. */
+    private byte[] errBytes;
+
+    /**
+     * Default constructor.
+     */
+    public GridDhtPartitionSupplyMessageV2() {
+    }
+
+    /**
+     * @param rebalanceId Rebalance id.
+     * @param grpId Group id.
+     * @param topVer Topology version.
+     * @param addDepInfo Add dep info.
+     * @param err Supply process error.
+     */
+    public GridDhtPartitionSupplyMessageV2(
+        long rebalanceId,
+        int grpId,
+        AffinityTopologyVersion topVer,
+        boolean addDepInfo,
+        Throwable err
+    ) {
+        super(rebalanceId, grpId, topVer, addDepInfo);
+
+        this.err = err;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
+        super.prepareMarshal(ctx);
+
+        if (err != null && errBytes == null)
+            errBytes = U.marshal(ctx, err);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr) throws IgniteCheckedException {
+        super.finishUnmarshal(ctx, ldr);
+
+        if (errBytes != null && err == null)
+            err = U.unmarshal(ctx, errBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+        writer.setBuffer(buf);
+
+        if (!super.writeTo(buf, writer))
+            return false;
+
+        if (!writer.isHeaderWritten()) {
+            if (!writer.writeHeader(directType(), fieldsCount()))
+                return false;
+
+            writer.onHeaderWritten();
+        }
+
+        switch (writer.state()) {
+            case 12:
+                if (!writer.writeByteArray("errBytes", errBytes))
+                    return false;
+
+                writer.incrementState();
+
+        }
+
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+        reader.setBuffer(buf);
+
+        if (!reader.beforeMessageRead())
+            return false;
+
+        if (!super.readFrom(buf, reader))
+            return false;
+
+        switch (reader.state()) {
+            case 12:
+                errBytes = reader.readByteArray("errBytes");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+        }
+
+        return reader.afterMessageRead(GridDhtPartitionSupplyMessageV2.class);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public Throwable error() {
+        return err;
+    }
+
+    /** {@inheritDoc} */
+    @Override public short directType() {
+        return 120;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte fieldsCount() {
+        return 13;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index ebc1eaf810627..b075c01ec0660 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -862,6 +862,10 @@ private static class WALHistoricalIterator implements IgniteHistoricalIterator {
         /** Flag indicates that partition belongs to current {@link #next} is finished and no longer needs to rebalance. */
         private boolean reachedPartitionEnd;
 
+        /** Flag indicates that update counters for requested partitions have been reached and done.
+         *  It means that no further iteration is needed. */
+        private boolean doneAllPartitions;
+
         /**
          * @param grp Cache context.
          * @param walIt WAL iterator.
@@ -935,6 +939,9 @@ private WALHistoricalIterator(CacheGroupContext grp, CachePartitionPartialCounte
                 doneParts.add(next.partitionId());
 
                 reachedPartitionEnd = false;
+
+                if (doneParts.size() == partMap.size())
+                    doneAllPartitions = true;
             }
 
             advance();
@@ -993,6 +1000,9 @@ private void releasePartitions() {
         private void advance() {
             next = null;
 
+            if (doneAllPartitions)
+                return;
+
             while (true) {
                 if (entryIt != null) {
                     while (entryIt.hasNext()) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
index 28801cd6c20bf..034ea74a5b19f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
@@ -17,6 +17,10 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.db.wal;
 
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.OpenOption;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -38,19 +42,31 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCachePreloader;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
-import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.IgniteSpiException;
-import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.junit.Assert;
 
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
 
 /**
@@ -63,6 +79,9 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
     /** Partitions count. */
     private static final int PARTS_CNT = 32;
 
+    /** Block message predicate to set to Communication SPI in node configuration. */
+    private IgniteBiPredicate<ClusterNode, Message> blockMessagePredicate;
+
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0"); //to make all rebalance wal-based
@@ -89,6 +108,12 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
 
         cfg.setCommunicationSpi(new WalRebalanceCheckingCommunicationSpi());
 
+        if (blockMessagePredicate != null) {
+            TestRecordingCommunicationSpi spi = (TestRecordingCommunicationSpi) cfg.getCommunicationSpi();
+
+            spi.blockMessages(blockMessagePredicate);
+        }
+
         return cfg;
     }
 
@@ -224,6 +249,8 @@ public void testWithLocalWalChange() throws Exception {
                 cache.put(k, new IndexedObject(k - 1));
         }
 
+        forceCheckpoint();
+
         stopAllGrids();
 
         IgniteEx ig0 = (IgniteEx) startGrids(2);
@@ -237,6 +264,8 @@ public void testWithLocalWalChange() throws Exception {
         for (int k = 0; k < entryCnt; k++)
             cache.put(k, new IndexedObject(k));
 
+        forceCheckpoint();
+
         // This node should rebalance data from other nodes and shouldn't have WAL history.
         Ignite ignite = startGrid(2);
 
@@ -255,6 +284,8 @@ else if (k % 3 == 1) // Spread removes across all partitions.
                 cache.remove(k);
         }
 
+        forceCheckpoint();
+
         // Stop grids which have actual WAL history.
         stopGrid(0);
 
@@ -306,6 +337,8 @@ public void testWithGlobalWalChange() throws Exception {
                 cache.put(k, new IndexedObject(k - 1));
         }
 
+        forceCheckpoint();
+
         stopAllGrids();
 
         // Rewrite data with globally disabled WAL.
@@ -322,6 +355,8 @@ public void testWithGlobalWalChange() throws Exception {
         for (int k = 0; k < entryCnt; k++)
             cache.put(k, new IndexedObject(k));
 
+        forceCheckpoint();
+
         crd.cluster().enableWal(CACHE_NAME);
 
         // This node shouldn't rebalance data using WAL, because it was disabled on other nodes.
@@ -361,6 +396,100 @@ public void testWithGlobalWalChange() throws Exception {
         }
     }
 
+    /**
+     * Tests that cache rebalance is cancelled if supplyer node got exception during iteration over WAL.
+     *
+     * @throws Exception If failed.
+     */
+    public void testRebalanceCancelOnSupplyError() throws Exception {
+        // Prepare some data.
+        IgniteEx crd = (IgniteEx) startGrids(3);
+
+        crd.cluster().active(true);
+
+        final int entryCnt = PARTS_CNT * 10;
+
+        {
+            IgniteCache<Object, Object> cache = crd.cache(CACHE_NAME);
+
+            for (int k = 0; k < entryCnt; k++)
+                cache.put(k, new IndexedObject(k - 1));
+        }
+
+        forceCheckpoint();
+
+        stopAllGrids();
+
+        // Rewrite data to trigger further rebalance.
+        IgniteEx supplierNode = (IgniteEx) startGrid(0);
+
+        supplierNode.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = supplierNode.cache(CACHE_NAME);
+
+        for (int k = 0; k < entryCnt; k++)
+            cache.put(k, new IndexedObject(k));
+
+        forceCheckpoint();
+
+        final int groupId = supplierNode.cachex(CACHE_NAME).context().groupId();
+
+        // Delay rebalance process for specified group.
+        blockMessagePredicate = (node, msg) -> {
+            if (msg instanceof GridDhtPartitionDemandMessage)
+                return ((GridDhtPartitionDemandMessage) msg).groupId() == groupId;
+
+            return false;
+        };
+
+        IgniteEx demanderNode = startGrid(2);
+
+        AffinityTopologyVersion curTopVer = demanderNode.context().discovery().topologyVersionEx();
+
+        // Wait for rebalance process start on demander node.
+        final GridCachePreloader preloader = demanderNode.cachex(CACHE_NAME).context().group().preloader();
+
+        GridTestUtils.waitForCondition(() ->
+            ((GridDhtPartitionDemander.RebalanceFuture) preloader.rebalanceFuture()).topologyVersion().equals(curTopVer),
+            getTestTimeout()
+        );
+
+        // Inject I/O factory which can throw exception during WAL read on supplier node.
+        FailingIOFactory ioFactory = new FailingIOFactory(new RandomAccessFileIOFactory());
+
+        ((FileWriteAheadLogManager) supplierNode.cachex(CACHE_NAME).context().shared().wal()).setFileIOFactory(ioFactory);
+
+        ioFactory.throwExceptionOnWalRead();
+
+        // Resume rebalance process.
+        TestRecordingCommunicationSpi spi = (TestRecordingCommunicationSpi) demanderNode.configuration().getCommunicationSpi();
+
+        spi.stopBlock();
+
+        // Wait till rebalance will be failed and cancelled.
+        Boolean result = preloader.rebalanceFuture().get();
+
+        Assert.assertEquals("Rebalance should be cancelled on demander node: " + preloader.rebalanceFuture(), false, result);
+
+        // Stop blocking messages and fail WAL during read.
+        blockMessagePredicate = null;
+
+        ioFactory.reset();
+
+        // Start last grid and wait for rebalance.
+        startGrid(1);
+
+        awaitPartitionMapExchange();
+
+        // Check data consistency.
+        for (Ignite ig : G.allGrids()) {
+            IgniteCache<Object, Object> cache1 = ig.cache(CACHE_NAME);
+
+            for (int k = 0; k < entryCnt; k++)
+                assertEquals(new IndexedObject(k), cache1.get(k));
+        }
+    }
+
     /**
      *
      */
@@ -406,7 +535,7 @@ private IndexedObject(int iVal) {
     /**
      * Wrapper of communication spi to detect on what topology versions WAL rebalance has happened.
      */
-    public static class WalRebalanceCheckingCommunicationSpi extends TcpCommunicationSpi {
+    public static class WalRebalanceCheckingCommunicationSpi extends TestRecordingCommunicationSpi {
         /** (Group ID, Set of topology versions). */
         private static final Map<Integer, Set<Long>> topVers = new HashMap<>();
 
@@ -461,4 +590,55 @@ public static void cleanup() {
             super.sendMessage(node, msg, ackC);
         }
     }
+
+    /**
+     *
+     */
+    static class FailingIOFactory implements FileIOFactory {
+        /** Fail read operations. */
+        private volatile boolean failRead;
+
+        /** Delegate. */
+        private final FileIOFactory delegate;
+
+        /**
+         * @param delegate Delegate.
+         */
+        FailingIOFactory(FileIOFactory delegate) {
+            this.delegate = delegate;
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            return create(file, CREATE, WRITE, READ);
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+            FileIO delegateIO = delegate.create(file, modes);
+
+            if (file.getName().endsWith(".wal") && failRead)
+                return new FileIODecorator(delegateIO) {
+                    @Override public int read(ByteBuffer destBuf) throws IOException {
+                        throw new IgniteException("Test exception.");
+                    }
+                };
+
+            return delegateIO;
+        }
+
+        /**
+         *
+         */
+        public void throwExceptionOnWalRead() {
+            failRead = true;
+        }
+
+        /**
+         *
+         */
+        public void reset() {
+            failRead = false;
+        }
+    }
 }
\ No newline at end of file

From 5c24e4fccb3ddf1d72169449ef860fbb9b929739 Mon Sep 17 00:00:00 2001
From: devozerov <vozerov@gridgain.com>
Date: Fri, 7 Sep 2018 12:19:55 +0300
Subject: [PATCH 341/543] IGNITE-8927: SQL: correct handling of LOST partitions
 on mapper side. Note that for local queries this doesn't work still because
 partition state is not checked (see IGNITE-7039). This closes #4679.

---
 .../messages/GridQueryFailResponse.java       |   2 +-
 ...gniteCachePartitionLossPolicySelfTest.java |  29 +++-
 .../h2/twostep/GridMapQueryExecutor.java      | 123 +++++++++++-----
 .../h2/twostep/GridReduceQueryExecutor.java   |   2 +-
 ...exingCachePartitionLossPolicySelfTest.java | 139 ++++++++++++++++++
 ...teCacheDistributedQueryCancelSelfTest.java |   2 +-
 .../h2/twostep/RetryCauseMessageSelfTest.java |   3 +-
 .../IgniteCacheQuerySelfTestSuite.java        |   4 +
 8 files changed, 261 insertions(+), 43 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryFailResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryFailResponse.java
index 1b759bbfa2e24..ef26d2a8e2988 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryFailResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryFailResponse.java
@@ -59,7 +59,7 @@ public GridQueryFailResponse() {
      */
     public GridQueryFailResponse(long qryReqId, Throwable err) {
         this.qryReqId = qryReqId;
-        this.errMsg = err.getClass() + ":" + err.getMessage();
+        this.errMsg = err.getMessage();
         this.failCode = err instanceof QueryCancelledException ? CANCELLED_BY_ORIGINATOR : GENERAL_ERROR;
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
index 7f35ddba8947d..e803e2c40efe2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCachePartitionLossPolicySelfTest.java
@@ -62,7 +62,7 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe
     private PartitionLossPolicy partLossPlc;
 
     /** */
-    private static final String CACHE_NAME = "partitioned";
+    protected static final String CACHE_NAME = "partitioned";
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
@@ -72,6 +72,15 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe
 
         cfg.setClientMode(client);
 
+        cfg.setCacheConfiguration(cacheConfiguration());
+
+        return cfg;
+    }
+
+    /**
+     * @return Cache configuration.
+     */
+    protected CacheConfiguration<Integer, Integer> cacheConfiguration() {
         CacheConfiguration<Integer, Integer> cacheCfg = new CacheConfiguration<>(CACHE_NAME);
 
         cacheCfg.setCacheMode(PARTITIONED);
@@ -80,9 +89,7 @@ public class IgniteCachePartitionLossPolicySelfTest extends GridCommonAbstractTe
         cacheCfg.setPartitionLossPolicy(partLossPlc);
         cacheCfg.setAffinity(new RendezvousAffinityFunction(false, 32));
 
-        cfg.setCacheConfiguration(cacheCfg);
-
-        return cfg;
+        return cacheCfg;
     }
 
     /** {@inheritDoc} */
@@ -194,6 +201,9 @@ private void checkLostPartition(boolean canWrite, boolean safe) throws Exception
 
             // Check that writing in recover mode does not clear partition state.
             verifyCacheOps(canWrite, safe, part, ig);
+
+            // Validate queries.
+            validateQuery(safe, part, ig);
         }
 
         // Check that partition state does not change after we start a new node.
@@ -353,4 +363,15 @@ private int prepareTopology() throws Exception {
 
         return part;
     }
+
+    /**
+     * Validate query execution on a node.
+     *
+     * @param safe Safe flag.
+     * @param part Partition.
+     * @param node Node.
+     */
+    protected void validateQuery(boolean safe, int part, Ignite node) {
+        // No-op.
+    }
 }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index 4f09b23676697..c9da3dcceb24d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.PartitionLossPolicy;
 import org.apache.ignite.cache.query.QueryCancelledException;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cluster.ClusterNode;
@@ -50,6 +51,7 @@
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
@@ -86,9 +88,12 @@
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SQL_FORCE_LAZY_RESULT_SET;
+import static org.apache.ignite.cache.PartitionLossPolicy.READ_ONLY_SAFE;
+import static org.apache.ignite.cache.PartitionLossPolicy.READ_WRITE_SAFE;
 import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.QUERY_POOL;
 import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.LOST;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
 import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.distributedJoinMode;
@@ -319,11 +324,9 @@ private String reservePartitions(
 
             // Cache was not found, probably was not deployed yet.
             if (cctx == null) {
-                final String res = String.format("Failed to reserve partitions for query (cache is not found on " +
+                return String.format("Failed to reserve partitions for query (cache is not found on " +
                     "local node) [localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s]",
                     ctx.localNodeId(), nodeId, reqId, topVer, cacheIds.get(i));
-
-                return res;
             }
 
             if (cctx.isLocal() || !cctx.rebalanceEnabled())
@@ -380,16 +383,39 @@ private String reservePartitions(
                     if (explicitParts == null)
                         partIds = cctx.affinity().primaryPartitions(ctx.localNodeId(), topVer);
 
+                    int reservedCnt = 0;
+
                     for (int partId : partIds) {
                         GridDhtLocalPartition part = partition(cctx, partId);
 
                         GridDhtPartitionState partState = part != null ? part.state() : null;
 
-                        if (partState != OWNING || !part.reserve())
+                        if (partState != OWNING) {
+                            if (partState == LOST)
+                                ignoreLostPartitionIfPossible(cctx, part);
+                            else {
+                                return String.format("Failed to reserve partitions for query " +
+                                        "(partition of PARTITIONED cache is not found or not in OWNING state) [" +
+                                        "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, " +
+                                        "cacheName=%s, part=%s, partFound=%s, partState=%s]",
+                                    ctx.localNodeId(),
+                                    nodeId,
+                                    reqId,
+                                    topVer,
+                                    cacheIds.get(i),
+                                    cctx.name(),
+                                    partId,
+                                    (part != null),
+                                    partState
+                                );
+                            }
+                        }
+
+                        if (!part.reserve()) {
                             return String.format("Failed to reserve partitions for query " +
-                                "(partition of PARTITIONED cache cannot be reserved) [" +
-                                "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, cacheName=%s, " +
-                                "part=%s, partFound=%s, partState=%s]",
+                                    "(partition of PARTITIONED cache cannot be reserved) [" +
+                                    "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, " +
+                                    "cacheName=%s, part=%s, partFound=%s, partState=%s]",
                                 ctx.localNodeId(),
                                 nodeId,
                                 reqId,
@@ -397,36 +423,44 @@ private String reservePartitions(
                                 cacheIds.get(i),
                                 cctx.name(),
                                 partId,
-                                (part != null),
+                                true,
                                 partState
                             );
+                        }
 
                         reserved.add(part);
 
+                        reservedCnt++;
+
                         // Double check that we are still in owning state and partition contents are not cleared.
                         partState = part.state();
 
-                        if (part.state() != OWNING)
-                            return String.format("Failed to reserve partitions for query " +
-                                "(partition of PARTITIONED cache is not in OWNING state after reservation) [" +
-                                "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, cacheName=%s, " +
-                                "part=%s, partState=%s]",
-                                ctx.localNodeId(),
-                                nodeId,
-                                reqId,
-                                topVer,
-                                cacheIds.get(i),
-                                cctx.name(),
-                                partId,
-                                partState
-                            );
+                        if (partState != OWNING) {
+                            if (partState == LOST)
+                                ignoreLostPartitionIfPossible(cctx, part);
+                            else {
+                                return String.format("Failed to reserve partitions for query " +
+                                    "(partition of PARTITIONED cache is not in OWNING state after reservation) [" +
+                                    "localNodeId=%s, rmtNodeId=%s, reqId=%s, affTopVer=%s, cacheId=%s, " +
+                                    "cacheName=%s, part=%s, partState=%s]",
+                                    ctx.localNodeId(),
+                                    nodeId,
+                                    reqId,
+                                    topVer,
+                                    cacheIds.get(i),
+                                    cctx.name(),
+                                    partId,
+                                    partState
+                                );
+                            }
+                        }
                     }
 
-                    if (explicitParts == null) {
+                    if (explicitParts == null && reservedCnt > 0) {
                         // We reserved all the primary partitions for cache, attempt to add group reservation.
                         GridDhtPartitionsReservation grp = new GridDhtPartitionsReservation(topVer, cctx, "SQL");
 
-                        if (grp.register(reserved.subList(reserved.size() - partIds.size(), reserved.size()))) {
+                        if (grp.register(reserved.subList(reserved.size() - reservedCnt, reserved.size()))) {
                             if (reservations.putIfAbsent(grpKey, grp) != null)
                                 throw new IllegalStateException("Reservation already exists.");
 
@@ -444,6 +478,25 @@ private String reservePartitions(
         return null;
     }
 
+    /**
+     * Decide whether to ignore or proceed with lost partition.
+     *
+     * @param cctx Cache context.
+     * @param part Partition.
+     * @throws IgniteCheckedException If failed.
+     */
+    private static void ignoreLostPartitionIfPossible(GridCacheContext cctx, GridDhtLocalPartition part)
+        throws IgniteCheckedException {
+        PartitionLossPolicy plc = cctx.config().getPartitionLossPolicy();
+
+        if (plc != null) {
+            if (plc == READ_ONLY_SAFE || plc == READ_WRITE_SAFE) {
+                throw new CacheInvalidStateException("Failed to execute query because cache partition has been " +
+                    "lost [cacheName=" + cctx.name() + ", part=" + part + ']');
+            }
+        }
+    }
+
     /**
      * @param ints Integers.
      * @return Collection wrapper.
@@ -855,22 +908,22 @@ private void onDmlRequest(final ClusterNode node, final GridH2DmlRequest req) th
 
         List<GridReservable> reserved = new ArrayList<>();
 
-        String err = reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId);
+        MapNodeResults nodeResults = resultsForNode(node.id());
 
-        if (!F.isEmpty(err)) {
-            U.error(log, "Failed to reserve partitions for DML request. [localNodeId=" + ctx.localNodeId() +
-                ", nodeId=" + node.id() + ", reqId=" + req.requestId() + ", cacheIds=" + cacheIds +
-                ", topVer=" + topVer + ", parts=" + Arrays.toString(parts) + ']');
+        try {
+            String err = reservePartitions(cacheIds, topVer, parts, reserved, node.id(), reqId);
 
-            sendUpdateResponse(node, reqId, null,
-                "Failed to reserve partitions for DML request. " + err);
+            if (!F.isEmpty(err)) {
+                U.error(log, "Failed to reserve partitions for DML request. [localNodeId=" + ctx.localNodeId() +
+                    ", nodeId=" + node.id() + ", reqId=" + req.requestId() + ", cacheIds=" + cacheIds +
+                    ", topVer=" + topVer + ", parts=" + Arrays.toString(parts) + ']');
 
-            return;
-        }
+                sendUpdateResponse(node, reqId, null,
+                    "Failed to reserve partitions for DML request. " + err);
 
-        MapNodeResults nodeResults = resultsForNode(node.id());
+                return;
+            }
 
-        try {
             IndexingQueryFilter filter = h2.backupFilter(topVer, parts);
 
             GridQueryCancel cancel = nodeResults.putUpdate(reqId);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index 51d05845a104c..fbd0729672a66 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -786,7 +786,7 @@ public Iterator<List<?>> query(
                             if (wasCancelled(err))
                                 throw new QueryCancelledException(); // Throw correct exception.
 
-                            throw new CacheException("Failed to run map query remotely: " + err.getMessage(), err);
+                            throw err;
                         }
                         else {
                             retry = true;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java
new file mode 100644
index 0000000000000..f2085994b0cd0
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IndexingCachePartitionLossPolicySelfTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePartitionLossPolicySelfTest;
+
+import java.util.Collection;
+
+/**
+ * Partition loss policy test with enabled indexing.
+ */
+public class IndexingCachePartitionLossPolicySelfTest extends IgniteCachePartitionLossPolicySelfTest {
+    /** {@inheritDoc} */
+    @Override protected CacheConfiguration<Integer, Integer> cacheConfiguration() {
+        CacheConfiguration<Integer, Integer> ccfg = super.cacheConfiguration();
+
+        ccfg.setIndexedTypes(Integer.class, Integer.class);
+
+        return ccfg;
+    }
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override protected void validateQuery(boolean safe, int part, Ignite node) {
+        // Get node lost and remaining partitions.
+        IgniteCache cache = node.cache(CACHE_NAME);
+
+        Collection<Integer> lostParts = cache.lostPartitions();
+
+        Integer remainingPart = null;
+
+        for (int i = 0; i < node.affinity(CACHE_NAME).partitions(); i++) {
+            if (lostParts.contains(i))
+                continue;
+
+            remainingPart = i;
+
+            break;
+        }
+
+        // Determine whether local query should be executed on that node.
+        boolean execLocQry = false;
+
+        for (int nodePrimaryPart : node.affinity(CACHE_NAME).primaryPartitions(node.cluster().localNode())) {
+            if (part == nodePrimaryPart) {
+                execLocQry = true;
+
+                break;
+            }
+        }
+
+        // 1. Check query against all partitions.
+        validateQuery0(safe, node, false);
+
+        // TODO: https://issues.apache.org/jira/browse/IGNITE-7039
+//        if (execLocQry)
+//            validateQuery0(safe, node, true);
+
+        // 2. Check query against LOST partition.
+        validateQuery0(safe, node, false, part);
+
+        // TODO: https://issues.apache.org/jira/browse/IGNITE-7039
+//        if (execLocQry)
+//            validateQuery0(safe, node, true, part);
+
+        // 3. Check query on remaining partition.
+        if (remainingPart != null) {
+            executeQuery(node, false, remainingPart);
+
+            // 4. Check query over two partitions - normal and LOST.
+            validateQuery0(safe, node, false, part, remainingPart);
+        }
+    }
+
+    /**
+     * Query validation routine.
+     *
+     * @param safe Safe flag.
+     * @param node Node.
+     * @param loc Local flag.
+     * @param parts Partitions.
+     */
+    private void validateQuery0(boolean safe, Ignite node, boolean loc, int... parts) {
+        if (safe) {
+            try {
+                executeQuery(node, loc, parts);
+
+                fail("Exception is not thrown.");
+            }
+            catch (Exception e) {
+                assertTrue(e.getMessage(), e.getMessage() != null &&
+                    e.getMessage().contains("Failed to execute query because cache partition has been lost"));
+            }
+        }
+        else {
+            executeQuery(node, loc, parts);
+        }
+    }
+
+    /**
+     * Execute SQL query on a given node.
+     *
+     * @param parts Partitions.
+     * @param node Node.
+     * @param loc Local flag.
+     */
+    private static void executeQuery(Ignite node, boolean loc, int... parts) {
+        IgniteCache cache = node.cache(CACHE_NAME);
+
+        SqlFieldsQuery qry = new SqlFieldsQuery("SELECT * FROM Integer");
+
+        if (parts != null && parts.length != 0)
+            qry.setPartitions(parts);
+
+        if (loc)
+            qry.setLocal(true);
+
+        cache.query(qry).getAll();
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
index e26b2111f90c5..d5ee0e978f874 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/IgniteCacheDistributedQueryCancelSelfTest.java
@@ -158,7 +158,7 @@ public void testQueryResponseFailCode() throws Exception {
                 fail();
             }
             catch (Exception e) {
-                assertTrue(e.getCause() instanceof CacheException);
+                assertTrue(e instanceof CacheException);
             }
         }
     }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
index 326988739ec57..ce385114485a7 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
@@ -250,7 +250,8 @@ public void testPartitionedCacheReserveFailureMessage() {
             personCache.query(qry).getAll();
         }
         catch (CacheException e) {
-            assertTrue(e.getMessage().contains("Failed to reserve partitions for query (partition of PARTITIONED cache cannot be reserved) ["));
+            assertTrue(e.getMessage().contains("Failed to reserve partitions for query (partition of PARTITIONED " +
+                "cache is not found or not in OWNING state) "));
 
             return;
         }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index cc076681ce057..4e865260d3a7b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -76,6 +76,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCrossCachesJoinsQueryTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicSqlRestoreTest;
 import org.apache.ignite.internal.processors.cache.IncorrectQueryEntityTest;
+import org.apache.ignite.internal.processors.cache.IndexingCachePartitionLossPolicySelfTest;
 import org.apache.ignite.internal.processors.cache.QueryEntityCaseMismatchTest;
 import org.apache.ignite.internal.processors.cache.SqlFieldsQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.authentication.SqlUserCommandSelfTest;
@@ -412,6 +413,9 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(SqlParserUserSelfTest.class);
         suite.addTestSuite(SqlUserCommandSelfTest.class);
 
+        // Partition loss.
+        suite.addTestSuite(IndexingCachePartitionLossPolicySelfTest.class);
+
         return suite;
     }
 }

From b7f7f15e1ceb6fa54e01461f3adac4d6b802b88f Mon Sep 17 00:00:00 2001
From: ibessonov <bessonov.ip@gmail.com>
Date: Fri, 7 Sep 2018 17:48:09 +0300
Subject: [PATCH 342/543] IGNITE-9472 Added permissions check on cluster
 activation - Fixes #4686.

Signed-off-by: Dmitriy Govorukhin <dgovorukhin@Dmitriys-MacBook-Pro.local>

(cherry picked from commit 02ba707bb0ea9f230d4b4739af8d842737ed0a8e)
---
 .../internal/processors/rest/GridRestProcessor.java       | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
index ccb92df24dc89..b6c1310a19631 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
@@ -888,6 +888,12 @@ private void authorize(GridRestRequest req, SecurityContext sCtx) throws Securit
 
                 break;
 
+            case CLUSTER_ACTIVE:
+            case CLUSTER_INACTIVE:
+                perm = SecurityPermission.ADMIN_OPS;
+
+                break;
+
             case CACHE_METRICS:
             case CACHE_SIZE:
             case CACHE_METADATA:
@@ -901,8 +907,6 @@ private void authorize(GridRestRequest req, SecurityContext sCtx) throws Securit
             case NAME:
             case LOG:
             case CLUSTER_CURRENT_STATE:
-            case CLUSTER_ACTIVE:
-            case CLUSTER_INACTIVE:
             case AUTHENTICATE:
             case ADD_USER:
             case REMOVE_USER:

From da837f6dafa4bf708f8cd0d901d1904d17f8cbe4 Mon Sep 17 00:00:00 2001
From: ibessonov <bessonov.ip@gmail.com>
Date: Fri, 7 Sep 2018 18:02:12 +0300
Subject: [PATCH 343/543] IGNITE-9475 Fix closures that has been created on
 client does not provide real class name to TASK_* permissions - Fixes #4688.

Signed-off-by: Dmitriy Govorukhin <dgovorukhin@Dmitriys-MacBook-Pro.local>

(cherry picked from commit 4fd6c2d8d085d35ba39f2b6cb042a1bc3155518b)
---
 .../internal/processors/task/GridTaskProcessor.java       | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 2f0aa7b858940..9007472b034f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -537,8 +537,12 @@ private <T, R> ComputeTaskInternalFuture<R> startTask(
 
         String taskClsName;
 
-        if (task != null)
-            taskClsName = task.getClass().getName();
+        if (task != null) {
+            if (task instanceof GridPeerDeployAware)
+                taskClsName = ((GridPeerDeployAware)task).deployClass().getName();
+            else
+                taskClsName = task.getClass().getName();
+        }
         else
             taskClsName = taskCls != null ? taskCls.getName() : taskName;
 

From f8863bf3a840e1a3217741efd81bbda3b96ad5e3 Mon Sep 17 00:00:00 2001
From: ibessonov <bessonov.ip@gmail.com>
Date: Tue, 11 Sep 2018 12:53:56 +0300
Subject: [PATCH 344/543] IGNITE-9518 Fix getPagesFillFactor returns NaN for
 empty region - Fixes #4717.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit 16e969e2f11c3c5c949e23ed7fec82b5d44c3c0f)
---
 .../persistence/DataRegionMetricsImpl.java    |  4 +++-
 .../pagemem/FillFactorMetricTest.java         | 19 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
index a82f73bd7966f..57631722972f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
@@ -170,7 +170,9 @@ public DataRegionMetricsImpl(DataRegionConfiguration memPlcCfg, @Nullable Ignite
 
         long totalAllocated = getPageSize() * totalAllocatedPages.longValue();
 
-        return (float) (totalAllocated - freeSpace) / totalAllocated;
+        return totalAllocated != 0 ?
+            (float) (totalAllocated - freeSpace) / totalAllocated
+            : 0f;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FillFactorMetricTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FillFactorMetricTest.java
index cdea7bcc6fad9..d48f158624934 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FillFactorMetricTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FillFactorMetricTest.java
@@ -90,6 +90,25 @@ protected CacheConfiguration<Object, Object> cacheCfg() {
      */
     private final float[] curFillFactor = new float[NODES];
 
+    /**
+     * Tests that {@link DataRegionMetrics#getPagesFillFactor()} doesn't return NaN for empty cache.
+     *
+     * @throws Exception if failed.
+     */
+    public void testEmptyCachePagesFillFactor() throws Exception {
+        startGrids(1);
+
+        // Cache is created in default region so MY_DATA_REGION will have "empty" metrics.
+        CacheConfiguration<Object, Object> cacheCfg = new CacheConfiguration<>().setName(MY_CACHE);
+        grid(0).getOrCreateCache(cacheCfg);
+
+        DataRegionMetrics m = grid(0).dataRegionMetrics(MY_DATA_REGION);
+
+        assertEquals(0, m.getTotalAllocatedPages());
+
+        assertEquals(0, m.getPagesFillFactor(), Float.MIN_VALUE);
+    }
+
     /**
      * throws if failed.
      */

From c870b280b9c35b2c460229f277e745f15d9cbf05 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Fri, 27 Jul 2018 12:23:08 +0300
Subject: [PATCH 345/543] IGNITE-7700: SQL system view for list of nodes. This
 closes #4185. This closes #4424.

(cherry picked from commit 6263dbe6c45a0656ba55a0eb04d07941fac27c7c)
---
 .../apache/ignite/IgniteSystemProperties.java |   3 +
 .../internal/processors/query/QueryUtils.java |   3 +
 .../processors/query/h2/IgniteH2Indexing.java |  85 +++++-
 .../query/h2/ddl/DdlStatementsProcessor.java  |  28 +-
 .../query/h2/dml/UpdatePlanBuilder.java       |  15 +-
 .../query/h2/sql/GridSqlQuerySplitter.java    |  12 +-
 .../query/h2/sys/SqlSystemIndex.java          | 143 ++++++++++
 .../query/h2/sys/SqlSystemTable.java          | 208 +++++++++++++++
 .../query/h2/sys/SqlSystemTableEngine.java    |  60 +++++
 .../sys/view/SqlAbstractLocalSystemView.java  | 104 ++++++++
 .../h2/sys/view/SqlAbstractSystemView.java    | 134 ++++++++++
 .../query/h2/sys/view/SqlSystemView.java      |  79 ++++++
 .../view/SqlSystemViewColumnCondition.java    | 102 ++++++++
 .../query/h2/sys/view/SqlSystemViewNodes.java | 116 ++++++++
 .../query/SqlSystemViewsSelfTest.java         | 247 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite.java        |   2 +
 16 files changed, 1325 insertions(+), 16 deletions(-)
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemIndex.java
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTable.java
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTableEngine.java
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractSystemView.java
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemView.java
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewColumnCondition.java
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index dd54ced164f0e..c42cae6b7f98d 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -471,6 +471,9 @@ public final class IgniteSystemProperties {
     /** Force all SQL queries to be processed lazily regardless of what clients request. */
     public static final String IGNITE_SQL_FORCE_LAZY_RESULT_SET = "IGNITE_SQL_FORCE_LAZY_RESULT_SET";
 
+    /** Disable SQL system views. */
+    public static final String IGNITE_SQL_DISABLE_SYSTEM_VIEWS = "IGNITE_SQL_DISABLE_SYSTEM_VIEWS";
+
     /** SQL retry timeout. */
     public static final String IGNITE_SQL_RETRY_TIMEOUT = "IGNITE_SQL_RETRY_TIMEOUT";
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
index 2ee37a15ccff1..d71cb7566f5d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/QueryUtils.java
@@ -74,6 +74,9 @@ public class QueryUtils {
     /** Default schema. */
     public static final String DFLT_SCHEMA = "PUBLIC";
 
+    /** Schema for system view. */
+    public static final String SCHEMA_SYS = "IGNITE";
+
     /** Field name for key. */
     public static final String KEY_FIELD_NAME = "_KEY";
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 920a99e59f72b..ecd9c59e7a837 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -113,9 +113,12 @@
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement;
+import org.apache.ignite.internal.processors.query.h2.sys.SqlSystemTableEngine;
+import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodes;
 import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor;
 import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor;
 import org.apache.ignite.internal.processors.query.h2.twostep.MapQueryLazyWorker;
+import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemView;
 import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor;
 import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitorClosure;
 import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitorImpl;
@@ -194,7 +197,6 @@
  * {@link GridQueryTypeDescriptor#fields()}.
  * For each table it will create indexes declared in {@link GridQueryTypeDescriptor#indexes()}.
  */
-@SuppressWarnings({"UnnecessaryFullyQualifiedName", "NonFinalStaticVariableUsedInClassInitialization"})
 public class IgniteH2Indexing implements GridQueryIndexing {
     /** A pattern for commands having internal implementation in Ignite. */
     public static final Pattern INTERNAL_CMD_RE = Pattern.compile(
@@ -239,6 +241,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
     private final Long CLEANUP_STMT_CACHE_PERIOD = Long.getLong(IGNITE_H2_INDEXING_CACHE_CLEANUP_PERIOD, 10_000);
 
     /** The period of clean up the {@link #conns}. */
+    @SuppressWarnings("FieldCanBeLocal")
     private final Long CLEANUP_CONNECTIONS_PERIOD = 2000L;
 
     /** The timeout to remove entry from the {@link #stmtCache} if the thread doesn't perform any queries. */
@@ -296,7 +299,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
 
     /** */
     private final ThreadLocal<H2ConnectionWrapper> connCache = new ThreadLocal<H2ConnectionWrapper>() {
-        @Nullable @Override public H2ConnectionWrapper get() {
+        @Override public H2ConnectionWrapper get() {
             H2ConnectionWrapper c = super.get();
 
             boolean reconnect = true;
@@ -320,7 +323,7 @@ public class IgniteH2Indexing implements GridQueryIndexing {
             return c;
         }
 
-        @Nullable @Override protected H2ConnectionWrapper initialValue() {
+        @Override protected H2ConnectionWrapper initialValue() {
             Connection c;
 
             try {
@@ -1066,7 +1069,7 @@ else if (DdlStatementsProcessor.isDdlStatement(p))
     private static List<Long> zeroBatchedStreamedUpdateResult(int size) {
         Long[] res = new Long[size];
 
-        Arrays.fill(res, 0);
+        Arrays.fill(res, 0L);
 
         return Arrays.asList(res);
     }
@@ -1260,6 +1263,7 @@ public void bindParameters(PreparedStatement stmt,
             qry.isEnforceJoinOrder(), qry.getTimeout(), cancel);
 
         QueryCursorImpl<List<?>> cursor = new QueryCursorImpl<>(new Iterable<List<?>>() {
+            @SuppressWarnings("NullableProblems")
             @Override public Iterator<List<?>> iterator() {
                 try {
                     return new GridQueryCacheObjectsIterator(res.iterator(), objectContext(), keepBinary);
@@ -1290,6 +1294,7 @@ public void bindParameters(PreparedStatement stmt,
             F.asList(params), type, filter, cancel);
 
         return new QueryCursorImpl<>(new Iterable<Cache.Entry<K, V>>() {
+            @SuppressWarnings("NullableProblems")
             @Override public Iterator<Cache.Entry<K, V>> iterator() {
                 return new ClIter<Cache.Entry<K, V>>() {
                     @Override public void close() throws Exception {
@@ -1391,6 +1396,7 @@ private Iterable<List<?>> runQueryTwoStep(
         final boolean lazy
     ) {
         return new Iterable<List<?>>() {
+            @SuppressWarnings("NullableProblems")
             @Override public Iterator<List<?>> iterator() {
                 return rdcQryExec.query(schemaName, qry, keepCacheObj, enforceJoinOrder, timeoutMillis, cancel, params,
                     parts, lazy);
@@ -1683,6 +1689,7 @@ private void checkQueryType(SqlFieldsQuery qry, boolean isQry) {
      * @param keepBinary Whether binary objects must not be deserialized automatically.
      * @param cancel Query cancel state holder.    @return Query result.
      */
+    @SuppressWarnings("unchecked")
     private List<? extends FieldsQueryCursor<List<?>>> doRunPrepared(String schemaName, Prepared prepared,
         SqlFieldsQuery qry, GridCacheTwoStepQuery twoStepQry,
         List<GridQueryFieldMetadata> meta, boolean keepBinary, GridQueryCancel cancel) {
@@ -1704,6 +1711,7 @@ private List<? extends FieldsQueryCursor<List<?>>> doRunPrepared(String schemaNa
                             dmlProc.updateSqlFieldsLocal(schemaName, conn, prepared, qry, filter, cancel);
 
                         return Collections.singletonList(new QueryCursorImpl<>(new Iterable<List<?>>() {
+                            @SuppressWarnings("NullableProblems")
                             @Override public Iterator<List<?>> iterator() {
                                 try {
                                     return new GridQueryCacheObjectsIterator(updRes.iterator(), objectContext(),
@@ -1744,7 +1752,8 @@ private List<? extends FieldsQueryCursor<List<?>>> doRunPrepared(String schemaNa
                 return Collections.singletonList(resCur);
             }
 
-            throw new IgniteSQLException("Unsupported DDL/DML operation: " + prepared.getClass().getName());
+            throw new IgniteSQLException("Unsupported DDL/DML operation: " + prepared.getClass().getName(),
+                IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
         }
 
         if (twoStepQry != null) {
@@ -1935,6 +1944,11 @@ private GridCacheTwoStepQuery split(Prepared prepared, SqlFieldsQuery qry) throw
 
         List<Integer> cacheIds = collectCacheIds(null, res);
 
+        if (!F.isEmpty(cacheIds) && hasSystemViews(res)) {
+            throw new IgniteSQLException("Normal tables and system views cannot be used in the same query.",
+                IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+        }
+
         if (F.isEmpty(cacheIds))
             res.local(true);
         else {
@@ -2529,6 +2543,33 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
 
             dmlProc.start(ctx, this);
             ddlProc.start(ctx, this);
+
+            boolean sysViewsEnabled =
+                !IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_SQL_DISABLE_SYSTEM_VIEWS);
+
+            if (sysViewsEnabled) {
+                try {
+                    synchronized (schemaMux) {
+                        createSchema(QueryUtils.SCHEMA_SYS);
+                    }
+
+                    Connection c = connectionForSchema(QueryUtils.SCHEMA_SYS);
+
+                    for (SqlSystemView view : systemViews(ctx))
+                        SqlSystemTableEngine.registerView(c, view);
+                }
+                catch (SQLException e) {
+                    throw new IgniteCheckedException("Failed to register system view.", e);
+                }
+
+                // Caching this connection in ThreadLocal may lead to memory leaks.
+                connCache.set(null);
+            }
+            else {
+                if (log.isDebugEnabled())
+                    log.debug("SQL system views will not be created because they are disabled (see " +
+                        IgniteSystemProperties.IGNITE_SQL_DISABLE_SYSTEM_VIEWS + " system property)");
+            }
         }
 
         if (JdbcUtils.serializer != null)
@@ -2536,6 +2577,8 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
 
         JdbcUtils.serializer = h2Serializer();
 
+        assert ctx != null;
+
         connCleanupTask = ctx.timeout().schedule(new Runnable() {
             @Override public void run() {
                 cleanupConnections();
@@ -2543,6 +2586,18 @@ public GridReduceQueryExecutor reduceQueryExecutor() {
         }, CLEANUP_CONNECTIONS_PERIOD, CLEANUP_CONNECTIONS_PERIOD);
     }
 
+    /**
+     * @param ctx Context.
+     * @return Predefined system views.
+     */
+    public Collection<SqlSystemView> systemViews(GridKernalContext ctx) {
+        Collection<SqlSystemView> views = new ArrayList<>();
+
+        views.add(new SqlSystemViewNodes(ctx));
+
+        return views;
+    }
+
     /**
      * @return Value object context.
      */
@@ -3007,9 +3062,11 @@ public Map<Thread, Connection> perThreadConnections() {
             for (QueryTable tblKey : twoStepQry.tables()) {
                 GridH2Table tbl = dataTable(tblKey);
 
-                int cacheId = tbl.cacheId();
+                if (tbl != null) {
+                    int cacheId = tbl.cacheId();
 
-                caches0.add(cacheId);
+                    caches0.add(cacheId);
+                }
             }
         }
 
@@ -3025,6 +3082,20 @@ public Map<Thread, Connection> perThreadConnections() {
         }
     }
 
+    /**
+     * @return {@code True} is system views exist.
+     */
+    private boolean hasSystemViews(GridCacheTwoStepQuery twoStepQry) {
+        if (twoStepQry.tablesCount() > 0) {
+            for (QueryTable tbl : twoStepQry.tables()) {
+                if (QueryUtils.SCHEMA_SYS.equals(tbl.schema()))
+                    return true;
+            }
+        }
+
+        return false;
+    }
+
     /**
      * Closeable iterator.
      */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
index b6db5c550211b..6154bbe9bb6e9 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/ddl/DdlStatementsProcessor.java
@@ -36,8 +36,6 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.processors.authentication.AuthorizationContext;
-import org.apache.ignite.internal.processors.authentication.UserManagementOperation;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
@@ -60,7 +58,6 @@
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQueryParser;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement;
 import org.apache.ignite.internal.processors.query.schema.SchemaOperationException;
-import org.apache.ignite.internal.processors.security.SecurityContext;
 import org.apache.ignite.internal.processors.security.SecurityContextHolder;
 import org.apache.ignite.internal.sql.command.SqlAlterTableCommand;
 import org.apache.ignite.internal.sql.command.SqlAlterUserCommand;
@@ -126,6 +123,8 @@ public FieldsQueryCursor<List<?>> runDdlStatement(String sql, SqlCommand cmd) th
         IgniteInternalFuture fut = null;
 
         try {
+            isDdlOnSchemaSupported(cmd.schemaName());
+
             if (cmd instanceof SqlCreateIndexCommand) {
                 SqlCreateIndexCommand cmd0 = (SqlCreateIndexCommand)cmd;
 
@@ -274,6 +273,8 @@ public FieldsQueryCursor<List<?>> runDdlStatement(String sql, Prepared prepared)
             if (stmt0 instanceof GridSqlCreateIndex) {
                 GridSqlCreateIndex cmd = (GridSqlCreateIndex)stmt0;
 
+                isDdlOnSchemaSupported(cmd.schemaName());
+
                 GridH2Table tbl = idx.dataTable(cmd.schemaName(), cmd.tableName());
 
                 if (tbl == null)
@@ -311,6 +312,8 @@ public FieldsQueryCursor<List<?>> runDdlStatement(String sql, Prepared prepared)
             else if (stmt0 instanceof GridSqlDropIndex) {
                 GridSqlDropIndex cmd = (GridSqlDropIndex) stmt0;
 
+                isDdlOnSchemaSupported(cmd.schemaName());
+
                 GridH2Table tbl = idx.dataTableForIndex(cmd.schemaName(), cmd.indexName());
 
                 if (tbl != null) {
@@ -332,6 +335,8 @@ else if (stmt0 instanceof GridSqlCreateTable) {
 
                 GridSqlCreateTable cmd = (GridSqlCreateTable)stmt0;
 
+                isDdlOnSchemaSupported(cmd.schemaName());
+
                 if (!F.eq(QueryUtils.DFLT_SCHEMA, cmd.schemaName()))
                     throw new SchemaOperationException("CREATE TABLE can only be executed on " +
                         QueryUtils.DFLT_SCHEMA + " schema.");
@@ -367,6 +372,8 @@ else if (stmt0 instanceof GridSqlDropTable) {
 
                 GridSqlDropTable cmd = (GridSqlDropTable)stmt0;
 
+                isDdlOnSchemaSupported(cmd.schemaName());
+
                 if (!F.eq(QueryUtils.DFLT_SCHEMA, cmd.schemaName()))
                     throw new SchemaOperationException("DROP TABLE can only be executed on " +
                         QueryUtils.DFLT_SCHEMA + " schema.");
@@ -390,6 +397,8 @@ else if (stmt0 instanceof GridSqlDropTable) {
             else if (stmt0 instanceof GridSqlAlterTableAddColumn) {
                 GridSqlAlterTableAddColumn cmd = (GridSqlAlterTableAddColumn)stmt0;
 
+                isDdlOnSchemaSupported(cmd.schemaName());
+
                 GridH2Table tbl = idx.dataTable(cmd.schemaName(), cmd.tableName());
 
                 if (tbl == null && cmd.ifTableExists()) {
@@ -448,6 +457,8 @@ else if (stmt0 instanceof GridSqlAlterTableAddColumn) {
             else if (stmt0 instanceof GridSqlAlterTableDropColumn) {
                 GridSqlAlterTableDropColumn cmd = (GridSqlAlterTableDropColumn)stmt0;
 
+                isDdlOnSchemaSupported(cmd.schemaName());
+
                 GridH2Table tbl = idx.dataTable(cmd.schemaName(), cmd.tableName());
 
                 if (tbl == null && cmd.ifTableExists()) {
@@ -525,6 +536,17 @@ else if (stmt0 instanceof GridSqlAlterTableDropColumn) {
         }
     }
 
+    /**
+     * Check if schema supports DDL statement.
+     *
+     * @param schemaName Schema name.
+     */
+    private static void isDdlOnSchemaSupported(String schemaName) {
+        if (F.eq(QueryUtils.SCHEMA_SYS, schemaName))
+            throw new IgniteSQLException("DDL statements are not supported on " + schemaName + " schema",
+                IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+    }
+
     /**
      * Check if table supports DDL statement.
      *
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
index d897ac7e9b84c..1079005d42b18 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/dml/UpdatePlanBuilder.java
@@ -137,7 +137,14 @@ private static UpdatePlan planForInsert(GridSqlStatement stmt, boolean loc, Igni
             target = ins.into();
 
             tbl = DmlAstUtils.gridTableForElement(target);
-            desc = tbl.dataTable().rowDescriptor();
+
+            GridH2Table h2Tbl = tbl.dataTable();
+
+            if (h2Tbl == null)
+                throw new IgniteSQLException("Operation not supported for table '" + tbl.tableName() + "'",
+                    IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
+            desc = h2Tbl.rowDescriptor();
 
             cols = ins.columns();
             sel = DmlAstUtils.selectForInsertOrMerge(cols, ins.rows(), ins.query());
@@ -312,6 +319,10 @@ else if (stmt instanceof GridSqlDelete) {
 
         GridH2Table h2Tbl = tbl.dataTable();
 
+        if (h2Tbl == null)
+            throw new IgniteSQLException("Operation not supported for table '" + tbl.tableName() + "'",
+                IgniteQueryErrorCode.UNSUPPORTED_OPERATION);
+
         GridH2RowDescriptor desc = h2Tbl.rowDescriptor();
 
         if (desc == null)
@@ -659,7 +670,7 @@ private static void verifyUpdateColumns(GridSqlStatement statement) {
 
         GridH2Table gridTbl = tbl.dataTable();
 
-        if (updateAffectsKeyColumns(gridTbl, update.set().keySet()))
+        if (gridTbl != null && updateAffectsKeyColumns(gridTbl, update.set().keySet()))
             throw new IgniteSQLException("SQL UPDATE can't modify key or its fields directly",
                 IgniteQueryErrorCode.KEY_UPDATE);
     }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index d2fb149fe509f..5c320055bcdf8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -1546,8 +1546,12 @@ private void splitSelect(
      * @return {@code true} If the given AST has partitioned tables.
      */
     private static boolean hasPartitionedTables(GridSqlAst ast) {
-        if (ast instanceof GridSqlTable)
-            return ((GridSqlTable)ast).dataTable().isPartitioned();
+        if (ast instanceof GridSqlTable) {
+            if (((GridSqlTable)ast).dataTable() != null)
+                return ((GridSqlTable)ast).dataTable().isPartitioned();
+            else
+                return false;
+        }
 
         for (int i = 0; i < ast.size(); i++) {
             if (hasPartitionedTables(ast.child(i)))
@@ -1739,8 +1743,8 @@ private void normalizeFrom(GridSqlAst prnt, int childIdx, boolean prntAlias) {
         if (from instanceof GridSqlTable) {
             GridSqlTable tbl = (GridSqlTable)from;
 
-            String schemaName = tbl.dataTable().identifier().schema();
-            String tblName = tbl.dataTable().identifier().table();
+            String schemaName = tbl.dataTable() != null ? tbl.dataTable().identifier().schema() : tbl.schema();
+            String tblName = tbl.dataTable() != null ? tbl.dataTable().identifier().table() : tbl.tableName();
 
             tbls.add(new QueryTable(schemaName, tblName));
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemIndex.java
new file mode 100644
index 0000000000000..55f5f29e23b25
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemIndex.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys;
+
+import org.apache.ignite.internal.processors.query.h2.opt.GridH2Cursor;
+import org.h2.engine.Session;
+import org.h2.index.BaseIndex;
+import org.h2.index.Cursor;
+import org.h2.index.IndexType;
+import org.h2.message.DbException;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.result.SortOrder;
+import org.h2.table.Column;
+import org.h2.table.IndexColumn;
+import org.h2.table.TableFilter;
+
+import java.util.HashSet;
+
+/**
+ * Meta view H2 index.
+ */
+public class SqlSystemIndex extends BaseIndex {
+    /** Distributed view cost multiplier. */
+    private static final int DISTRIBUTED_MUL = 100;
+
+    /**
+     * @param tbl Table.
+     * @param col Column.
+     */
+    SqlSystemIndex(SqlSystemTable tbl, Column... col) {
+        IndexColumn[] idxCols;
+
+        if (col != null && col.length > 0)
+            idxCols = IndexColumn.wrap(col);
+        else
+            idxCols = new IndexColumn[0];
+
+        initBaseIndex(tbl, 0, null, idxCols, IndexType.createNonUnique(false));
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close(Session ses) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void add(Session ses, Row row) {
+        throw DbException.getUnsupportedException("system view is read-only");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void remove(Session ses, Row row) {
+        throw DbException.getUnsupportedException("system view is read-only");
+    }
+
+    /** {@inheritDoc} */
+    @Override public Cursor find(Session ses, SearchRow first, SearchRow last) {
+        assert table instanceof SqlSystemTable;
+
+        Iterable<Row> rows = ((SqlSystemTable)table).getRows(ses, first, last);
+
+        return new GridH2Cursor(rows.iterator());
+    }
+
+    /** {@inheritDoc} */
+    @Override public double getCost(Session ses, int[] masks, TableFilter[] filters, int filter, SortOrder sortOrder,
+        HashSet<Column> allColsSet) {
+        long rowCnt = getRowCountApproximation();
+
+        double baseCost = getCostRangeIndex(masks, rowCnt, filters, filter, sortOrder, false, allColsSet);
+
+        if (((SqlSystemTable)table).view.isDistributed())
+            baseCost = baseCost * DISTRIBUTED_MUL;
+
+        return baseCost;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void truncate(Session ses) {
+        throw DbException.getUnsupportedException("system view cannot be truncated");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void remove(Session ses) {
+        throw DbException.getUnsupportedException("system view cannot be removed");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void checkRename() {
+        throw DbException.getUnsupportedException("system view cannot be renamed");
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean needRebuild() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String getCreateSQL() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canGetFirstOrLast() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Cursor findFirstOrLast(Session ses, boolean first) {
+        throw DbException.getUnsupportedException("system views cannot be used to get first or last value");
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCount(Session ses) {
+        return table.getRowCount(ses);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCountApproximation() {
+        return table.getRowCountApproximation();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getDiskSpaceUsed() {
+        return 0;
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTable.java
new file mode 100644
index 0000000000000..23106ee5f952a
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTable.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys;
+
+import java.util.ArrayList;
+
+import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemView;
+import org.h2.command.ddl.CreateTableData;
+import org.h2.engine.Session;
+import org.h2.index.Index;
+import org.h2.index.IndexType;
+import org.h2.message.DbException;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.table.Column;
+import org.h2.table.IndexColumn;
+import org.h2.table.TableBase;
+import org.h2.table.TableType;
+
+/**
+ * System H2 table over a view.
+ */
+public class SqlSystemTable extends TableBase {
+    /** Scan index. */
+    protected final SqlSystemIndex scanIdx;
+
+    /** Meta view. */
+    protected final SqlSystemView view;
+
+    /**
+     * Indexes.
+     *
+     * Note: We need ArrayList here by H2 {@link #getIndexes()} method contract.
+     */
+    protected final ArrayList<Index> indexes;
+
+    /**
+     * @param data Data.
+     * @param view Meta view.
+     */
+    public SqlSystemTable(CreateTableData data, SqlSystemView view) {
+        super(data);
+
+        assert view != null;
+
+        this.view = view;
+
+        this.setColumns(view.getColumns());
+
+        scanIdx = new SqlSystemIndex(this);
+
+        indexes = new ArrayList<>();
+        indexes.add(scanIdx);
+
+        for (String index : view.getIndexes()) {
+            String[] indexedCols = index.split(",");
+
+            Column[] cols = new Column[indexedCols.length];
+
+            for (int i = 0; i < indexedCols.length; i++)
+                cols[i] = getColumn(indexedCols[i]);
+
+            SqlSystemIndex idx = new SqlSystemIndex(this, cols);
+
+            indexes.add(idx);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public Index addIndex(Session ses, String idxName, int idxId, IndexColumn[] cols,
+        IndexType idxType, boolean create, String idxComment) {
+        throw DbException.getUnsupportedException("META");
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean lock(Session ses, boolean exclusive, boolean forceLockEvenInMvcc) {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void unlock(Session ses) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isLockedExclusively() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeRow(Session ses, Row row) {
+        throw DbException.getUnsupportedException("META");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void addRow(Session ses, Row row) {
+        throw DbException.getUnsupportedException("META");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void removeChildrenAndResources(Session ses) {
+        throw DbException.getUnsupportedException("META");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close(Session ses) {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public void checkRename() {
+        throw DbException.getUnsupportedException("META");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void checkSupportAlter() {
+        throw DbException.getUnsupportedException("META");
+    }
+
+    /** {@inheritDoc} */
+    @Override public void truncate(Session ses) {
+        throw DbException.getUnsupportedException("META");
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCount(Session ses) {
+        return view.getRowCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canGetRowCount() {
+        return view.canGetRowCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCountApproximation() {
+        return view.getRowCount();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canDrop() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public TableType getTableType() {
+        return TableType.SYSTEM_TABLE;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Index getScanIndex(Session ses) {
+        return scanIdx;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Index getUniqueIndex() {
+        return null;
+    }
+
+    /** {@inheritDoc} */
+    @Override public ArrayList<Index> getIndexes() {
+        return indexes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getMaxDataModificationId() {
+        return Long.MAX_VALUE;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getDiskSpaceUsed() {
+        return 0;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isDeterministic() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canReference() {
+        return false;
+    }
+
+    /**
+     * @param ses Session.
+     * @param first First.
+     * @param last Last.
+     */
+    public Iterable<Row> getRows(Session ses, SearchRow first, SearchRow last) {
+        return view.getRows(ses, first, last);
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTableEngine.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTableEngine.java
new file mode 100644
index 0000000000000..23d77390bc5b5
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTableEngine.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemView;
+import org.h2.api.TableEngine;
+import org.h2.command.ddl.CreateTableData;
+import org.h2.table.Table;
+
+/**
+ * H2 table engine for system views.
+ */
+public class SqlSystemTableEngine implements TableEngine {
+    /** View being created. */
+    private static volatile SqlSystemView curView;
+
+    /**
+     * @param conn Connection.
+     * @param view View.
+     */
+    public static synchronized void registerView(Connection conn, SqlSystemView view)
+        throws SQLException {
+        curView = view;
+
+        String sql = view.getCreateSQL() + " ENGINE \"" + SqlSystemTableEngine.class.getName() + "\"";
+
+        try {
+            try (Statement s = conn.createStatement()) {
+                s.execute(sql);
+            }
+        }
+        finally {
+            curView = null;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public Table createTable(CreateTableData data) {
+        return new SqlSystemTable(data, curView);
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
new file mode 100644
index 0000000000000..c601708ad9c43
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys.view;
+
+import org.apache.ignite.internal.GridKernalContext;
+import org.h2.engine.Session;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.table.Column;
+import org.h2.value.Value;
+import org.h2.value.ValueNull;
+import org.h2.value.ValueString;
+
+/**
+ * Local meta view base class (which uses only local node data).
+ */
+public abstract class SqlAbstractLocalSystemView extends SqlAbstractSystemView {
+    /**
+     * @param tblName Table name.
+     * @param desc Description.
+     * @param ctx Context.
+     * @param indexes Indexed columns.
+     * @param cols Columns.
+     */
+    public SqlAbstractLocalSystemView(String tblName, String desc, GridKernalContext ctx, String[] indexes,
+        Column... cols) {
+        super(tblName, desc, ctx, cols, indexes);
+
+        assert tblName != null;
+        assert ctx != null;
+        assert cols != null;
+        assert indexes != null;
+    }
+
+    /**
+     * @param ses Session.
+     * @param key Key.
+     * @param data Data for each column.
+     */
+    protected Row createRow(Session ses, long key, Object... data) {
+        Value[] values = new Value[data.length];
+
+        for (int i = 0; i < data.length; i++) {
+            Object o = data[i];
+
+            Value v = (o == null) ? ValueNull.INSTANCE :
+                (o instanceof Value) ? (Value)o : ValueString.get(o.toString());
+
+            values[i] = cols[i].convert(v);
+        }
+
+        Row row = ses.getDatabase().createRow(values, 1);
+
+        row.setKey(key);
+
+        return row;
+    }
+
+    /**
+     * Gets column index by name.
+     *
+     * @param colName Column name.
+     */
+    protected int getColumnIndex(String colName) {
+        assert colName != null;
+
+        for (int i = 0; i < cols.length; i++)
+            if (colName.equalsIgnoreCase(cols[i].getName()))
+                return i;
+
+        return -1;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isDistributed() {
+        return false;
+    }
+
+    /**
+     * Parse condition for column.
+     *
+     * @param colName Column name.
+     * @param first First.
+     * @param last Last.
+     */
+    protected SqlSystemViewColumnCondition conditionForColumn(String colName, SearchRow first, SearchRow last) {
+        return SqlSystemViewColumnCondition.forColumn(getColumnIndex(colName), first, last);
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractSystemView.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractSystemView.java
new file mode 100644
index 0000000000000..79d62259e4a0f
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractSystemView.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys.view;
+
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.GridKernalContext;
+import org.h2.table.Column;
+import org.h2.value.Value;
+
+/**
+ * Meta view base class.
+ */
+public abstract class SqlAbstractSystemView implements SqlSystemView {
+    /** Default row count approximation. */
+    protected static final long DEFAULT_ROW_COUNT_APPROXIMATION = 100L;
+
+    /** Table name. */
+    protected final String tblName;
+
+    /** Description. */
+    protected final String desc;
+
+    /** Grid context. */
+    protected final GridKernalContext ctx;
+
+    /** Logger. */
+    protected final IgniteLogger log;
+
+    /** Columns. */
+    protected final Column[] cols;
+
+    /** Indexed column names. */
+    protected final String[] indexes;
+
+    /**
+     * @param tblName Table name.
+     * @param desc Descriptor.
+     * @param ctx Context.
+     * @param cols Columns.
+     * @param indexes Indexes.
+     */
+    public SqlAbstractSystemView(String tblName, String desc, GridKernalContext ctx, Column[] cols,
+        String[] indexes) {
+        this.tblName = tblName;
+        this.desc = desc;
+        this.ctx = ctx;
+        this.cols = cols;
+        this.indexes = indexes;
+        this.log = ctx.log(this.getClass());
+    }
+
+    /**
+     * @param name Name.
+     */
+    protected static Column newColumn(String name) {
+        return newColumn(name, Value.STRING);
+    }
+
+    /**
+     * @param name Name.
+     * @param type Type.
+     */
+    protected static Column newColumn(String name, int type) {
+        return new Column(name, type);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String getTableName() {
+        return tblName;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String getDescription() {
+        return desc;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Column[] getColumns() {
+        return cols;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String[] getIndexes() {
+        return indexes;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCount() {
+        return DEFAULT_ROW_COUNT_APPROXIMATION;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canGetRowCount() {
+        return false;
+    }
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("StringConcatenationInsideStringBufferAppend")
+    @Override public String getCreateSQL() {
+        StringBuilder sql = new StringBuilder();
+
+        sql.append("CREATE TABLE " + getTableName() + '(');
+
+        boolean isFirst = true;
+
+        for (Column col : getColumns()) {
+            if (isFirst)
+                isFirst = false;
+            else
+                sql.append(", ");
+
+            sql.append(col.getCreateSQL());
+        }
+
+        sql.append(')');
+
+        return sql.toString();
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemView.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemView.java
new file mode 100644
index 0000000000000..7eab521d006ad
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemView.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys.view;
+
+import org.h2.engine.Session;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.table.Column;
+
+/**
+ * SQL system view.
+ */
+public interface SqlSystemView {
+    /**
+     * Gets table name.
+     */
+    public String getTableName();
+
+    /**
+     * Gets description.
+     */
+    public String getDescription();
+
+    /**
+     * Gets columns.
+     */
+    public Column[] getColumns();
+
+    /**
+     * Gets indexed column names.
+     */
+    public String[] getIndexes();
+
+    /**
+     * Gets view content.
+     *
+     * @param ses Session.
+     * @param first First.
+     * @param last Last.
+     */
+    public Iterable<Row> getRows(Session ses, SearchRow first, SearchRow last);
+
+    /**
+     * Gets row count for this view (or approximated row count, if real value can't be calculated quickly).
+     */
+    public long getRowCount();
+
+    /**
+     * Check if the row count can be retrieved quickly.
+     *
+     * @return true if it can
+     */
+    public boolean canGetRowCount();
+
+    /**
+     * Gets SQL script for creating table.
+     */
+    public String getCreateSQL();
+
+    /**
+     * @return {@code True} if view is distributed.
+     */
+    public boolean isDistributed();
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewColumnCondition.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewColumnCondition.java
new file mode 100644
index 0000000000000..77cf82b8cf90c
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewColumnCondition.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys.view;
+
+import org.h2.result.SearchRow;
+import org.h2.value.Value;
+
+/**
+ * Column condition.
+ */
+public class SqlSystemViewColumnCondition {
+    /** Is equality. */
+    private final boolean isEquality;
+
+    /** Is range. */
+    private final boolean isRange;
+
+    /** Value 1. */
+    private final Value val;
+
+    /**
+     * @param isEquality Is equality.
+     * @param isRange Is range.
+     * @param val Value for equality.
+     */
+    private SqlSystemViewColumnCondition(boolean isEquality, boolean isRange, Value val) {
+        this.isEquality = isEquality;
+        this.isRange = isRange;
+        this.val = val;
+    }
+
+    /**
+     * Parse condition for column.
+     *
+     * @param colIdx Column index.
+     * @param start Start row values.
+     * @param end End row values.
+     */
+    public static SqlSystemViewColumnCondition forColumn(int colIdx, SearchRow start, SearchRow end) {
+        boolean isEquality = false;
+        boolean isRange = false;
+
+        Value val = null;
+        Value val2 = null;
+
+        if (start != null && colIdx >= 0 && colIdx < start.getColumnCount())
+            val = start.getValue(colIdx);
+
+        if (end != null && colIdx >= 0 && colIdx < end.getColumnCount())
+            val2 = end.getValue(colIdx);
+
+        if (val != null && val2 != null) {
+            if (val.equals(val2))
+                isEquality = true;
+            else
+                isRange = true;
+        }
+        else if (val != null || val2 != null)
+            isRange = true;
+
+        return new SqlSystemViewColumnCondition(isEquality, isRange, val);
+    }
+
+    /**
+     * Checks whether the condition is equality.
+     */
+    public boolean isEquality() {
+        return isEquality;
+    }
+
+    /**
+     * Checks whether the condition is range.
+     */
+    public boolean isRange() {
+        return isRange;
+    }
+
+    /**
+     * Gets value, if condition is equality.
+     */
+    public Value valueForEquality() {
+        if (isEquality)
+            return val;
+
+        return null;
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
new file mode 100644
index 0000000000000..e944b4fdcb402
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys.view;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.F;
+import org.h2.engine.Session;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.value.Value;
+
+/**
+ * Meta view: nodes.
+ */
+public class SqlSystemViewNodes extends SqlAbstractLocalSystemView {
+    /**
+     * @param ctx Grid context.
+     */
+    public SqlSystemViewNodes(GridKernalContext ctx) {
+        super("NODES", "Topology nodes", ctx, new String[] {"ID", "IS_LOCAL"},
+            newColumn("ID", Value.UUID),
+            newColumn("CONSISTENT_ID"),
+            newColumn("VERSION"),
+            newColumn("IS_LOCAL", Value.BOOLEAN),
+            newColumn("IS_CLIENT", Value.BOOLEAN),
+            newColumn("IS_DAEMON", Value.BOOLEAN),
+            newColumn("NODE_ORDER", Value.INT),
+            newColumn("ADDRESSES"),
+            newColumn("HOSTNAMES")
+        );
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterable<Row> getRows(Session ses, SearchRow first, SearchRow last) {
+        List<Row> rows = new ArrayList<>();
+
+        Collection<ClusterNode> nodes;
+
+        SqlSystemViewColumnCondition locCond = conditionForColumn("IS_LOCAL", first, last);
+        SqlSystemViewColumnCondition idCond = conditionForColumn("ID", first, last);
+
+        if (locCond.isEquality() && locCond.valueForEquality().getBoolean())
+            nodes = Collections.singleton(ctx.discovery().localNode());
+        else if (idCond.isEquality()) {
+            UUID nodeId = uuidFromString(idCond.valueForEquality().getString());
+
+            nodes = nodeId == null ? Collections.emptySet() : Collections.singleton(ctx.discovery().node(nodeId));
+        }
+        else
+            nodes = F.concat(false, ctx.discovery().allNodes(), ctx.discovery().daemonNodes());
+
+        for (ClusterNode node : nodes) {
+            if (node != null)
+                rows.add(
+                    createRow(ses, rows.size(),
+                        node.id(),
+                        node.consistentId(),
+                        node.version(),
+                        node.isLocal(),
+                        node.isClient(),
+                        node.isDaemon(),
+                        node.order(),
+                        node.addresses(),
+                        node.hostNames()
+                    )
+                );
+        }
+
+        return rows;
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canGetRowCount() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCount() {
+        return ctx.discovery().allNodes().size() + ctx.discovery().daemonNodes().size();
+    }
+
+    /**
+     * Converts string to UUID safe (suppressing exceptions).
+     *
+     * @param val UUID in string format.
+     */
+    private static UUID uuidFromString(String val) {
+        try {
+            return UUID.fromString(val);
+        }
+        catch (RuntimeException e) {
+            return null;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
new file mode 100644
index 0000000000000..9c7ce38ce662a
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests for ignite SQL meta views.
+ */
+public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @param sql Sql.
+     * @param args Args.
+     */
+    @SuppressWarnings("unchecked")
+    private List<List<?>> execSql(Ignite ignite, String sql, Object ... args) {
+        IgniteCache cache = ignite.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        SqlFieldsQuery qry = new SqlFieldsQuery(sql);
+
+        if (args != null && args.length > 0)
+            qry.setArgs(args);
+
+        return cache.query(qry).getAll();
+    }
+
+    /**
+     * @param sql Sql.
+     * @param args Args.
+     */
+    private List<List<?>> execSql(String sql, Object ... args) {
+        return execSql(grid(), sql, args);
+    }
+
+    /**
+     * @param sql Sql.
+     */
+    private void assertSqlError(final String sql) {
+        Throwable t = GridTestUtils.assertThrowsWithCause(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                execSql(sql);
+
+                return null;
+            }
+        }, IgniteSQLException.class);
+
+        IgniteSQLException sqlE = X.cause(t, IgniteSQLException.class);
+
+        assert sqlE != null;
+
+        assertEquals(IgniteQueryErrorCode.UNSUPPORTED_OPERATION, sqlE.statusCode());
+    }
+    
+    /**
+     * Test meta views modifications.
+     */
+    public void testModifications() throws Exception {
+        startGrid();
+
+        assertSqlError("DROP TABLE IGNITE.NODES");
+
+        assertSqlError("TRUNCATE TABLE IGNITE.NODES");
+
+        assertSqlError("ALTER TABLE IGNITE.NODES RENAME TO IGNITE.N");
+
+        assertSqlError("ALTER TABLE IGNITE.NODES ADD COLUMN C VARCHAR");
+
+        assertSqlError("ALTER TABLE IGNITE.NODES DROP COLUMN ID");
+
+        assertSqlError("ALTER TABLE IGNITE.NODES RENAME COLUMN ID TO C");
+
+        assertSqlError("CREATE INDEX IDX ON IGNITE.NODES(ID)");
+
+        assertSqlError("INSERT INTO IGNITE.NODES (ID) VALUES ('-')");
+
+        assertSqlError("UPDATE IGNITE.NODES SET ID = '-'");
+
+        assertSqlError("DELETE IGNITE.NODES");
+    }
+
+    /**
+     * Test different query modes.
+     */
+    public void testQueryModes() throws Exception {
+        Ignite ignite = startGrid(0);
+        startGrid(1);
+
+        UUID nodeId = ignite.cluster().localNode().id();
+
+        IgniteCache cache = ignite.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        String sql = "SELECT ID FROM IGNITE.NODES WHERE IS_LOCAL = true";
+
+        SqlFieldsQuery qry;
+
+        qry = new SqlFieldsQuery(sql).setDistributedJoins(true);
+
+        assertEquals(nodeId, ((List<?>)cache.query(qry).getAll().get(0)).get(0));
+
+        qry = new SqlFieldsQuery(sql).setReplicatedOnly(true);
+
+        assertEquals(nodeId, ((List<?>)cache.query(qry).getAll().get(0)).get(0));
+
+        qry = new SqlFieldsQuery(sql).setLocal(true);
+
+        assertEquals(nodeId, ((List<?>)cache.query(qry).getAll().get(0)).get(0));
+    }
+
+    /**
+     * Test that we can't use cache tables and meta views in the same query.
+     */
+    public void testCacheToViewJoin() throws Exception {
+        Ignite ignite = startGrid();
+
+        ignite.createCache(new CacheConfiguration<>().setName(DEFAULT_CACHE_NAME).setQueryEntities(
+            Collections.singleton(new QueryEntity(Integer.class.getName(), String.class.getName()))));
+
+        assertSqlError("SELECT * FROM \"" + DEFAULT_CACHE_NAME + "\".String JOIN IGNITE.NODES ON 1=1");
+    }
+
+    /**
+     * @param rowData Row data.
+     * @param colTypes Column types.
+     */
+    private void assertColumnTypes(List<?> rowData, Class<?> ... colTypes) {
+        for (int i = 0; i < colTypes.length; i++) {
+            if (rowData.get(i) != null)
+                assertEquals("Column " + i + " type", rowData.get(i).getClass(), colTypes[i]);
+        }
+    }
+
+    /**
+     * Test nodes meta view.
+     *
+     * @throws Exception If failed.
+     */
+    public void testNodesViews() throws Exception {
+        Ignite ignite1 = startGrid(getTestIgniteInstanceName(), getConfiguration());
+        Ignite ignite2 = startGrid(getTestIgniteInstanceName(1), getConfiguration().setClientMode(true));
+        Ignite ignite3 = startGrid(getTestIgniteInstanceName(2), getConfiguration().setDaemon(true));
+
+        awaitPartitionMapExchange();
+
+        List<List<?>> resAll = execSql("SELECT ID, CONSISTENT_ID, VERSION, IS_LOCAL, IS_CLIENT, IS_DAEMON, " +
+                "NODE_ORDER, ADDRESSES, HOSTNAMES FROM IGNITE.NODES");
+
+        assertColumnTypes(resAll.get(0), UUID.class, String.class, String.class, Boolean.class, Boolean.class,
+            Boolean.class, Integer.class, String.class, String.class);
+
+        assertEquals(3, resAll.size());
+
+        List<List<?>> resSrv = execSql(
+            "SELECT ID, IS_LOCAL, NODE_ORDER FROM IGNITE.NODES WHERE IS_CLIENT = FALSE AND IS_DAEMON = FALSE"
+        );
+
+        assertEquals(1, resSrv.size());
+
+        assertEquals(ignite1.cluster().localNode().id(), resSrv.get(0).get(0));
+
+        assertEquals(true, resSrv.get(0).get(1));
+
+        assertEquals(1, resSrv.get(0).get(2));
+
+        List<List<?>> resCli = execSql(
+            "SELECT ID, IS_LOCAL, NODE_ORDER FROM IGNITE.NODES WHERE IS_CLIENT = TRUE");
+
+        assertEquals(1, resCli.size());
+
+        assertEquals(ignite2.cluster().localNode().id(), resCli.get(0).get(0));
+
+        assertEquals(false, resCli.get(0).get(1));
+
+        assertEquals(2, resCli.get(0).get(2));
+
+        List<List<?>> resDaemon = execSql(
+            "SELECT ID, IS_LOCAL, NODE_ORDER FROM IGNITE.NODES WHERE IS_DAEMON = TRUE");
+
+        assertEquals(1, resDaemon.size());
+
+        assertEquals(ignite3.cluster().localNode().id(), resDaemon.get(0).get(0));
+
+        assertEquals(false, resDaemon.get(0).get(1));
+
+        assertEquals(3, resDaemon.get(0).get(2));
+
+        // Check index on ID column.
+        assertEquals(0, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = '-'").size());
+
+        assertEquals(1, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = ?",
+            ignite1.cluster().localNode().id()).size());
+
+        assertEquals(1, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = ?",
+            ignite3.cluster().localNode().id()).size());
+
+        // Check index on IS_LOCAL column.
+        assertEquals(1, execSql("SELECT ID FROM IGNITE.NODES WHERE IS_LOCAL = true").size());
+
+        // Check index on IS_LOCAL column with disjunction.
+        assertEquals(3, execSql("SELECT ID FROM IGNITE.NODES WHERE IS_LOCAL = true OR node_order=1 OR node_order=2 OR node_order=3").size());
+
+        // Check quick-count.
+        assertEquals(3L, execSql("SELECT COUNT(*) FROM IGNITE.NODES").get(0).get(0));
+
+        // Check joins
+        assertEquals(ignite1.cluster().localNode().id(), execSql("SELECT N1.ID FROM IGNITE.NODES N1 JOIN " +
+            "IGNITE.NODES N2 ON N1.IS_LOCAL = N2.IS_LOCAL JOIN IGNITE.NODES N3 ON N2.ID = N3.ID WHERE N3.IS_LOCAL = true")
+            .get(0).get(0));
+
+        // Check sub-query
+        assertEquals(ignite1.cluster().localNode().id(), execSql("SELECT N1.ID FROM IGNITE.NODES N1 " +
+            "WHERE NOT EXISTS (SELECT 1 FROM IGNITE.NODES N2 WHERE N2.ID = N1.ID AND N2.IS_LOCAL = false)")
+            .get(0).get(0));
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
index 4e865260d3a7b..df7992c57e21c 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite.java
@@ -161,6 +161,7 @@
 import org.apache.ignite.internal.processors.query.IgniteSqlSplitterSelfTest;
 import org.apache.ignite.internal.processors.query.LazyQuerySelfTest;
 import org.apache.ignite.internal.processors.query.MultipleStatementsSqlQuerySelfTest;
+import org.apache.ignite.internal.processors.query.SqlSystemViewsSelfTest;
 import org.apache.ignite.internal.processors.query.SqlPushDownFunctionTest;
 import org.apache.ignite.internal.processors.query.SqlSchemaSelfTest;
 import org.apache.ignite.internal.processors.query.h2.GridH2IndexingInMemSelfTest;
@@ -402,6 +403,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(H2ConnectionLeaksSelfTest.class);
         suite.addTestSuite(IgniteCheckClusterStateBeforeExecuteQueryTest.class);
         suite.addTestSuite(OptimizedMarshallerIndexNameTest.class);
+        suite.addTestSuite(SqlSystemViewsSelfTest.class);
 
         suite.addTestSuite(IgniteSqlDefaultValueTest.class);
 

From cec7f5b2eeef19cc9fa9800e82094e5adcdfa703 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Fri, 17 Aug 2018 17:10:04 +0300
Subject: [PATCH 346/543] IGNITE-7701: SQL: implemented NODE_ATTRIBUTES system
 view. This closes #4445.

(cherry picked from commit 5d63d7f44c343c72deb4ff1b3f5a41362646f1a4)
---
 .../processors/query/h2/IgniteH2Indexing.java |   2 +
 .../query/h2/sys/SqlSystemIndex.java          |   5 +-
 .../query/h2/sys/SqlSystemTable.java          |   4 +-
 .../sys/view/SqlAbstractLocalSystemView.java  |  18 ++-
 .../query/h2/sys/view/SqlSystemView.java      |   3 +-
 .../sys/view/SqlSystemViewNodeAttributes.java | 108 ++++++++++++++++++
 .../query/h2/sys/view/SqlSystemViewNodes.java |  23 +---
 .../query/SqlSystemViewsSelfTest.java         |  38 +++++-
 8 files changed, 171 insertions(+), 30 deletions(-)
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeAttributes.java

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index ecd9c59e7a837..d7e7bdcc2c69f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -114,6 +114,7 @@
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement;
 import org.apache.ignite.internal.processors.query.h2.sys.SqlSystemTableEngine;
+import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodeAttributes;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodes;
 import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor;
 import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor;
@@ -2594,6 +2595,7 @@ public Collection<SqlSystemView> systemViews(GridKernalContext ctx) {
         Collection<SqlSystemView> views = new ArrayList<>();
 
         views.add(new SqlSystemViewNodes(ctx));
+        views.add(new SqlSystemViewNodeAttributes(ctx));
 
         return views;
     }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemIndex.java
index 55f5f29e23b25..2b4896e2b92af 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemIndex.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.query.h2.sys;
 
+import java.util.Iterator;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2Cursor;
 import org.h2.engine.Session;
 import org.h2.index.BaseIndex;
@@ -73,9 +74,9 @@ public class SqlSystemIndex extends BaseIndex {
     @Override public Cursor find(Session ses, SearchRow first, SearchRow last) {
         assert table instanceof SqlSystemTable;
 
-        Iterable<Row> rows = ((SqlSystemTable)table).getRows(ses, first, last);
+        Iterator<Row> rows = ((SqlSystemTable)table).getRows(ses, first, last);
 
-        return new GridH2Cursor(rows.iterator());
+        return new GridH2Cursor(rows);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTable.java
index 23106ee5f952a..664da0f0b56d3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTable.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/SqlSystemTable.java
@@ -18,7 +18,7 @@
 package org.apache.ignite.internal.processors.query.h2.sys;
 
 import java.util.ArrayList;
-
+import java.util.Iterator;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemView;
 import org.h2.command.ddl.CreateTableData;
 import org.h2.engine.Session;
@@ -202,7 +202,7 @@ public SqlSystemTable(CreateTableData data, SqlSystemView view) {
      * @param first First.
      * @param last Last.
      */
-    public Iterable<Row> getRows(Session ses, SearchRow first, SearchRow last) {
+    public Iterator<Row> getRows(Session ses, SearchRow first, SearchRow last) {
         return view.getRows(ses, first, last);
     }
 }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
index c601708ad9c43..e8d52d265d803 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.query.h2.sys.view;
 
+import java.util.UUID;
 import org.apache.ignite.internal.GridKernalContext;
 import org.h2.engine.Session;
 import org.h2.result.Row;
@@ -27,7 +28,7 @@
 import org.h2.value.ValueString;
 
 /**
- * Local meta view base class (which uses only local node data).
+ * Local system view base class (which uses only local node data).
  */
 public abstract class SqlAbstractLocalSystemView extends SqlAbstractSystemView {
     /**
@@ -42,7 +43,6 @@ public SqlAbstractLocalSystemView(String tblName, String desc, GridKernalContext
         super(tblName, desc, ctx, cols, indexes);
 
         assert tblName != null;
-        assert ctx != null;
         assert cols != null;
         assert indexes != null;
     }
@@ -101,4 +101,18 @@ protected int getColumnIndex(String colName) {
     protected SqlSystemViewColumnCondition conditionForColumn(String colName, SearchRow first, SearchRow last) {
         return SqlSystemViewColumnCondition.forColumn(getColumnIndex(colName), first, last);
     }
+
+    /**
+     * Converts value to UUID safe (suppressing exceptions).
+     *
+     * @param val UUID.
+     */
+    protected static UUID uuidFromValue(Value val) {
+        try {
+            return UUID.fromString(val.getString());
+        }
+        catch (RuntimeException e) {
+            return null;
+        }
+    }
 }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemView.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemView.java
index 7eab521d006ad..93fdfa0e563e4 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemView.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemView.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.query.h2.sys.view;
 
+import java.util.Iterator;
 import org.h2.engine.Session;
 import org.h2.result.Row;
 import org.h2.result.SearchRow;
@@ -53,7 +54,7 @@ public interface SqlSystemView {
      * @param first First.
      * @param last Last.
      */
-    public Iterable<Row> getRows(Session ses, SearchRow first, SearchRow last);
+    public Iterator<Row> getRows(Session ses, SearchRow first, SearchRow last);
 
     /**
      * Gets row count for this view (or approximated row count, if real value can't be calculated quickly).
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeAttributes.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeAttributes.java
new file mode 100644
index 0000000000000..1ba0c7f17650c
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeAttributes.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys.view;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.F;
+import org.h2.engine.Session;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.value.Value;
+
+/**
+ * System view: node attributes.
+ */
+public class SqlSystemViewNodeAttributes extends SqlAbstractLocalSystemView {
+    /**
+     * @param ctx Grid context.
+     */
+    public SqlSystemViewNodeAttributes(GridKernalContext ctx) {
+        super("NODE_ATTRIBUTES", "Node attributes", ctx, new String[] {"NODE_ID,NAME", "NAME"},
+            newColumn("NODE_ID", Value.UUID),
+            newColumn("NAME"),
+            newColumn("VALUE")
+        );
+    }
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override public Iterator<Row> getRows(Session ses, SearchRow first, SearchRow last) {
+        Collection<ClusterNode> nodes;
+
+        SqlSystemViewColumnCondition idCond = conditionForColumn("NODE_ID", first, last);
+        SqlSystemViewColumnCondition nameCond = conditionForColumn("NAME", first, last);
+
+        if (idCond.isEquality()) {
+            try {
+                UUID nodeId = uuidFromValue(idCond.valueForEquality());
+
+                ClusterNode node = nodeId == null ? null : ctx.discovery().node(nodeId);
+
+                if (node != null)
+                    nodes = Collections.singleton(node);
+                else
+                    nodes = Collections.emptySet();
+            }
+            catch (Exception e) {
+                nodes = Collections.emptySet();
+            }
+        }
+        else
+            nodes = F.concat(false, ctx.discovery().allNodes(), ctx.discovery().daemonNodes());
+
+        if (nameCond.isEquality()) {
+            String attrName = nameCond.valueForEquality().getString();
+
+            List<Row> rows = new ArrayList<>();
+
+            for (ClusterNode node : nodes) {
+                if (node.attributes().containsKey(attrName))
+                    rows.add(
+                        createRow(ses, rows.size(),
+                            node.id(),
+                            attrName,
+                            node.attribute(attrName)
+                        )
+                    );
+            }
+
+            return rows.iterator();
+        }
+        else {
+            AtomicLong rowKey = new AtomicLong();
+
+            return F.concat(F.iterator(nodes,
+                node -> F.iterator(node.attributes().entrySet(),
+                    attr -> createRow(ses,
+                        rowKey.incrementAndGet(),
+                        node.id(),
+                        attr.getKey(),
+                        attr.getValue()),
+                    true).iterator(),
+                true));
+        }
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
index e944b4fdcb402..3e97faa4cd679 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
@@ -20,6 +20,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Iterator;
 import java.util.List;
 import java.util.UUID;
 import org.apache.ignite.cluster.ClusterNode;
@@ -31,7 +32,7 @@
 import org.h2.value.Value;
 
 /**
- * Meta view: nodes.
+ * System view: nodes.
  */
 public class SqlSystemViewNodes extends SqlAbstractLocalSystemView {
     /**
@@ -52,7 +53,7 @@ public SqlSystemViewNodes(GridKernalContext ctx) {
     }
 
     /** {@inheritDoc} */
-    @Override public Iterable<Row> getRows(Session ses, SearchRow first, SearchRow last) {
+    @Override public Iterator<Row> getRows(Session ses, SearchRow first, SearchRow last) {
         List<Row> rows = new ArrayList<>();
 
         Collection<ClusterNode> nodes;
@@ -63,7 +64,7 @@ public SqlSystemViewNodes(GridKernalContext ctx) {
         if (locCond.isEquality() && locCond.valueForEquality().getBoolean())
             nodes = Collections.singleton(ctx.discovery().localNode());
         else if (idCond.isEquality()) {
-            UUID nodeId = uuidFromString(idCond.valueForEquality().getString());
+            UUID nodeId = uuidFromValue(idCond.valueForEquality());
 
             nodes = nodeId == null ? Collections.emptySet() : Collections.singleton(ctx.discovery().node(nodeId));
         }
@@ -87,7 +88,7 @@ else if (idCond.isEquality()) {
                 );
         }
 
-        return rows;
+        return rows.iterator();
     }
 
     /** {@inheritDoc} */
@@ -99,18 +100,4 @@ else if (idCond.isEquality()) {
     @Override public long getRowCount() {
         return ctx.discovery().allNodes().size() + ctx.discovery().daemonNodes().size();
     }
-
-    /**
-     * Converts string to UUID safe (suppressing exceptions).
-     *
-     * @param val UUID in string format.
-     */
-    private static UUID uuidFromString(String val) {
-        try {
-            return UUID.fromString(val);
-        }
-        catch (RuntimeException e) {
-            return null;
-        }
-    }
 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
index 9c7ce38ce662a..acd2a1c3da4da 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
@@ -26,13 +26,14 @@
 import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
- * Tests for ignite SQL meta views.
+ * Tests for ignite SQL system views.
  */
 public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
     /** {@inheritDoc} */
@@ -70,7 +71,7 @@ private List<List<?>> execSql(String sql, Object ... args) {
      */
     private void assertSqlError(final String sql) {
         Throwable t = GridTestUtils.assertThrowsWithCause(new Callable<Void>() {
-            @Override public Void call() throws Exception {
+            @Override public Void call() {
                 execSql(sql);
 
                 return null;
@@ -85,7 +86,7 @@ private void assertSqlError(final String sql) {
     }
     
     /**
-     * Test meta views modifications.
+     * Test system views modifications.
      */
     public void testModifications() throws Exception {
         startGrid();
@@ -140,7 +141,7 @@ public void testQueryModes() throws Exception {
     }
 
     /**
-     * Test that we can't use cache tables and meta views in the same query.
+     * Test that we can't use cache tables and system views in the same query.
      */
     public void testCacheToViewJoin() throws Exception {
         Ignite ignite = startGrid();
@@ -163,7 +164,7 @@ private void assertColumnTypes(List<?> rowData, Class<?> ... colTypes) {
     }
 
     /**
-     * Test nodes meta view.
+     * Test nodes system view.
      *
      * @throws Exception If failed.
      */
@@ -243,5 +244,32 @@ public void testNodesViews() throws Exception {
         assertEquals(ignite1.cluster().localNode().id(), execSql("SELECT N1.ID FROM IGNITE.NODES N1 " +
             "WHERE NOT EXISTS (SELECT 1 FROM IGNITE.NODES N2 WHERE N2.ID = N1.ID AND N2.IS_LOCAL = false)")
             .get(0).get(0));
+
+        // Check node attributes view
+        UUID cliNodeId = ignite2.cluster().localNode().id();
+
+        String cliAttrName = IgniteNodeAttributes.ATTR_CLIENT_MODE;
+
+        assertColumnTypes(execSql("SELECT NODE_ID, NAME, VALUE FROM IGNITE.NODE_ATTRIBUTES").get(0),
+            UUID.class, String.class, String.class);
+
+        assertEquals(1,
+            execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NAME = ? AND VALUE = 'true'",
+                cliAttrName).size());
+
+        assertEquals(3,
+            execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NAME = ?", cliAttrName).size());
+
+        assertEquals(1,
+            execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NODE_ID = ? AND NAME = ? AND VALUE = 'true'",
+                cliNodeId, cliAttrName).size());
+
+        assertEquals(0,
+            execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NODE_ID = '-' AND NAME = ?",
+                cliAttrName).size());
+
+        assertEquals(0,
+            execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NODE_ID = ? AND NAME = '-'",
+                cliNodeId).size());
     }
 }

From 125909ec130e71ae3433589f3ad3f7a91a237026 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Thu, 23 Aug 2018 11:18:05 +0300
Subject: [PATCH 347/543] IGNITE-9318: SQL: added "BASELINE_NODES" system view.
 This closes #4575.

(cherry picked from commit e2fc48e200189e5f36dcf72de8791f23e9d51c2f)
---
 .../processors/query/h2/IgniteH2Indexing.java |  2 +
 .../sys/view/SqlAbstractLocalSystemView.java  | 10 +++
 .../sys/view/SqlSystemViewBaselineNodes.java  | 87 +++++++++++++++++++
 .../query/SqlSystemViewsSelfTest.java         | 71 +++++++++++++++
 4 files changed, 170 insertions(+)
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewBaselineNodes.java

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index d7e7bdcc2c69f..b6adb7417206d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -114,6 +114,7 @@
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlQuerySplitter;
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement;
 import org.apache.ignite.internal.processors.query.h2.sys.SqlSystemTableEngine;
+import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewBaselineNodes;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodeAttributes;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodes;
 import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor;
@@ -2596,6 +2597,7 @@ public Collection<SqlSystemView> systemViews(GridKernalContext ctx) {
 
         views.add(new SqlSystemViewNodes(ctx));
         views.add(new SqlSystemViewNodeAttributes(ctx));
+        views.add(new SqlSystemViewBaselineNodes(ctx));
 
         return views;
     }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
index e8d52d265d803..ac90b631b8297 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
@@ -47,6 +47,16 @@ public SqlAbstractLocalSystemView(String tblName, String desc, GridKernalContext
         assert indexes != null;
     }
 
+    /**
+     * @param tblName Table name.
+     * @param desc Description.
+     * @param ctx Context.
+     * @param cols Columns.
+     */
+    public SqlAbstractLocalSystemView(String tblName, String desc, GridKernalContext ctx, Column ... cols) {
+        this(tblName, desc, ctx, new String[] {}, cols);
+    }
+
     /**
      * @param ses Session.
      * @param key Key.
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewBaselineNodes.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewBaselineNodes.java
new file mode 100644
index 0000000000000..81a9a77ed2ab4
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewBaselineNodes.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys.view;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.cluster.BaselineTopology;
+import org.apache.ignite.internal.util.typedef.F;
+import org.h2.engine.Session;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.value.Value;
+
+/**
+ * System view: baseline nodes.
+ */
+public class SqlSystemViewBaselineNodes extends SqlAbstractLocalSystemView {
+    /**
+     * @param ctx Grid context.
+     */
+    public SqlSystemViewBaselineNodes(GridKernalContext ctx) {
+        super("BASELINE_NODES", "Baseline topology nodes", ctx,
+            newColumn("CONSISTENT_ID"),
+            newColumn("ONLINE", Value.BOOLEAN)
+        );
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterator<Row> getRows(Session ses, SearchRow first, SearchRow last) {
+        List<Row> rows = new ArrayList<>();
+
+        BaselineTopology blt = ctx.state().clusterState().baselineTopology();
+
+        if (blt == null)
+            return rows.iterator();
+
+        Set<Object> consistentIds = blt.consistentIds();
+
+        Collection<ClusterNode> srvNodes = ctx.discovery().aliveServerNodes();
+
+        Set<Object> aliveNodeIds = new HashSet<>(F.nodeConsistentIds(srvNodes));
+
+        for (Object consistentId : consistentIds) {
+            rows.add(
+                createRow(ses, rows.size(),
+                    consistentId,
+                    aliveNodeIds.contains(consistentId)
+                )
+            );
+        }
+
+        return rows.iterator();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canGetRowCount() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCount() {
+        BaselineTopology blt = ctx.state().clusterState().baselineTopology();
+
+        return blt == null ? 0 : blt.consistentIds().size();
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
index acd2a1c3da4da..bc5afa1df76aa 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
@@ -26,6 +26,9 @@
 import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
 import org.apache.ignite.internal.util.typedef.X;
@@ -36,9 +39,18 @@
  * Tests for ignite SQL system views.
  */
 public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+    }
+
     /** {@inheritDoc} */
     @Override protected void afterTest() throws Exception {
         stopAllGrids();
+
+        cleanPersistenceDir();
     }
 
     /**
@@ -272,4 +284,63 @@ public void testNodesViews() throws Exception {
             execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NODE_ID = ? AND NAME = '-'",
                 cliNodeId).size());
     }
+
+    /**
+     * Test baseline topology system view.
+     */
+    public void testBaselineViews() throws Exception {
+        cleanPersistenceDir();
+
+        Ignite ignite = startGrid(getTestIgniteInstanceName(), getPdsConfiguration("node0"));
+        startGrid(getTestIgniteInstanceName(1), getPdsConfiguration("node1"));
+
+        ignite.cluster().active(true);
+
+        List<List<?>> res = execSql("SELECT CONSISTENT_ID, ONLINE FROM IGNITE.BASELINE_NODES ORDER BY CONSISTENT_ID");
+
+        assertColumnTypes(res.get(0), String.class, Boolean.class);
+
+        assertEquals(2, res.size());
+
+        assertEquals("node0", res.get(0).get(0));
+        assertEquals("node1", res.get(1).get(0));
+
+        assertEquals(true, res.get(0).get(1));
+        assertEquals(true, res.get(1).get(1));
+
+        stopGrid(getTestIgniteInstanceName(1));
+
+        res = execSql("SELECT CONSISTENT_ID FROM IGNITE.BASELINE_NODES WHERE ONLINE = false");
+
+        assertEquals(1, res.size());
+
+        assertEquals("node1", res.get(0).get(0));
+
+        Ignite ignite2 = startGrid(getTestIgniteInstanceName(2), getPdsConfiguration("node2"));
+
+        assertEquals(2, execSql(ignite2, "SELECT CONSISTENT_ID FROM IGNITE.BASELINE_NODES").size());
+
+        res = execSql("SELECT CONSISTENT_ID FROM IGNITE.NODES N WHERE NOT EXISTS (SELECT 1 FROM " +
+            "IGNITE.BASELINE_NODES B WHERE B.CONSISTENT_ID = N.CONSISTENT_ID)");
+
+        assertEquals(1, res.size());
+
+        assertEquals("node2", res.get(0).get(0));
+    }
+
+    /**
+     * Gets ignite configuration with persistance enabled.
+     */
+    private IgniteConfiguration getPdsConfiguration(String consistentId) throws Exception {
+        IgniteConfiguration cfg = getConfiguration();
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration().setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setMaxSize(100L * 1024L * 1024L).setPersistenceEnabled(true))
+        );
+
+        cfg.setConsistentId(consistentId);
+
+        return cfg;
+    }
 }

From dc08ff544290d9e66031adbb1ca8c4517efa1239 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Tue, 28 Aug 2018 11:03:57 +0300
Subject: [PATCH 348/543] IGNITE-9362: SQL: removed NODES.IS_LOCAL attribute.
 This closes #4610.

(cherry picked from commit e2ff347eaaf6a17dc8754fde1a5251967dcff124)
---
 .../query/h2/sys/view/SqlSystemViewNodes.java |  9 +----
 .../query/SqlSystemViewsSelfTest.java         | 37 ++++++++-----------
 2 files changed, 17 insertions(+), 29 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
index 3e97faa4cd679..514f92e9708ee 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
@@ -39,11 +39,10 @@ public class SqlSystemViewNodes extends SqlAbstractLocalSystemView {
      * @param ctx Grid context.
      */
     public SqlSystemViewNodes(GridKernalContext ctx) {
-        super("NODES", "Topology nodes", ctx, new String[] {"ID", "IS_LOCAL"},
+        super("NODES", "Topology nodes", ctx, new String[] {"ID"},
             newColumn("ID", Value.UUID),
             newColumn("CONSISTENT_ID"),
             newColumn("VERSION"),
-            newColumn("IS_LOCAL", Value.BOOLEAN),
             newColumn("IS_CLIENT", Value.BOOLEAN),
             newColumn("IS_DAEMON", Value.BOOLEAN),
             newColumn("NODE_ORDER", Value.INT),
@@ -58,12 +57,9 @@ public SqlSystemViewNodes(GridKernalContext ctx) {
 
         Collection<ClusterNode> nodes;
 
-        SqlSystemViewColumnCondition locCond = conditionForColumn("IS_LOCAL", first, last);
         SqlSystemViewColumnCondition idCond = conditionForColumn("ID", first, last);
 
-        if (locCond.isEquality() && locCond.valueForEquality().getBoolean())
-            nodes = Collections.singleton(ctx.discovery().localNode());
-        else if (idCond.isEquality()) {
+        if (idCond.isEquality()) {
             UUID nodeId = uuidFromValue(idCond.valueForEquality());
 
             nodes = nodeId == null ? Collections.emptySet() : Collections.singleton(ctx.discovery().node(nodeId));
@@ -78,7 +74,6 @@ else if (idCond.isEquality()) {
                         node.id(),
                         node.consistentId(),
                         node.version(),
-                        node.isLocal(),
                         node.isClient(),
                         node.isDaemon(),
                         node.order(),
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
index bc5afa1df76aa..c8374a78ffc2f 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
@@ -135,7 +135,7 @@ public void testQueryModes() throws Exception {
 
         IgniteCache cache = ignite.getOrCreateCache(DEFAULT_CACHE_NAME);
 
-        String sql = "SELECT ID FROM IGNITE.NODES WHERE IS_LOCAL = true";
+        String sql = "SELECT ID FROM IGNITE.NODES WHERE NODE_ORDER = 1";
 
         SqlFieldsQuery qry;
 
@@ -187,47 +187,41 @@ public void testNodesViews() throws Exception {
 
         awaitPartitionMapExchange();
 
-        List<List<?>> resAll = execSql("SELECT ID, CONSISTENT_ID, VERSION, IS_LOCAL, IS_CLIENT, IS_DAEMON, " +
+        List<List<?>> resAll = execSql("SELECT ID, CONSISTENT_ID, VERSION, IS_CLIENT, IS_DAEMON, " +
                 "NODE_ORDER, ADDRESSES, HOSTNAMES FROM IGNITE.NODES");
 
         assertColumnTypes(resAll.get(0), UUID.class, String.class, String.class, Boolean.class, Boolean.class,
-            Boolean.class, Integer.class, String.class, String.class);
+            Integer.class, String.class, String.class);
 
         assertEquals(3, resAll.size());
 
         List<List<?>> resSrv = execSql(
-            "SELECT ID, IS_LOCAL, NODE_ORDER FROM IGNITE.NODES WHERE IS_CLIENT = FALSE AND IS_DAEMON = FALSE"
+            "SELECT ID, NODE_ORDER FROM IGNITE.NODES WHERE IS_CLIENT = FALSE AND IS_DAEMON = FALSE"
         );
 
         assertEquals(1, resSrv.size());
 
         assertEquals(ignite1.cluster().localNode().id(), resSrv.get(0).get(0));
 
-        assertEquals(true, resSrv.get(0).get(1));
-
-        assertEquals(1, resSrv.get(0).get(2));
+        assertEquals(1, resSrv.get(0).get(1));
 
         List<List<?>> resCli = execSql(
-            "SELECT ID, IS_LOCAL, NODE_ORDER FROM IGNITE.NODES WHERE IS_CLIENT = TRUE");
+            "SELECT ID, NODE_ORDER FROM IGNITE.NODES WHERE IS_CLIENT = TRUE");
 
         assertEquals(1, resCli.size());
 
         assertEquals(ignite2.cluster().localNode().id(), resCli.get(0).get(0));
 
-        assertEquals(false, resCli.get(0).get(1));
-
-        assertEquals(2, resCli.get(0).get(2));
+        assertEquals(2, resCli.get(0).get(1));
 
         List<List<?>> resDaemon = execSql(
-            "SELECT ID, IS_LOCAL, NODE_ORDER FROM IGNITE.NODES WHERE IS_DAEMON = TRUE");
+            "SELECT ID, NODE_ORDER FROM IGNITE.NODES WHERE IS_DAEMON = TRUE");
 
         assertEquals(1, resDaemon.size());
 
         assertEquals(ignite3.cluster().localNode().id(), resDaemon.get(0).get(0));
 
-        assertEquals(false, resDaemon.get(0).get(1));
-
-        assertEquals(3, resDaemon.get(0).get(2));
+        assertEquals(3, resDaemon.get(0).get(1));
 
         // Check index on ID column.
         assertEquals(0, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = '-'").size());
@@ -238,23 +232,22 @@ public void testNodesViews() throws Exception {
         assertEquals(1, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = ?",
             ignite3.cluster().localNode().id()).size());
 
-        // Check index on IS_LOCAL column.
-        assertEquals(1, execSql("SELECT ID FROM IGNITE.NODES WHERE IS_LOCAL = true").size());
-
-        // Check index on IS_LOCAL column with disjunction.
-        assertEquals(3, execSql("SELECT ID FROM IGNITE.NODES WHERE IS_LOCAL = true OR node_order=1 OR node_order=2 OR node_order=3").size());
+        // Check index on ID column with disjunction.
+        assertEquals(3, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = ? " +
+            "OR node_order=1 OR node_order=2 OR node_order=3", ignite1.cluster().localNode().id()).size());
 
         // Check quick-count.
         assertEquals(3L, execSql("SELECT COUNT(*) FROM IGNITE.NODES").get(0).get(0));
 
         // Check joins
         assertEquals(ignite1.cluster().localNode().id(), execSql("SELECT N1.ID FROM IGNITE.NODES N1 JOIN " +
-            "IGNITE.NODES N2 ON N1.IS_LOCAL = N2.IS_LOCAL JOIN IGNITE.NODES N3 ON N2.ID = N3.ID WHERE N3.IS_LOCAL = true")
+            "IGNITE.NODES N2 ON N1.NODE_ORDER = N2.NODE_ORDER JOIN IGNITE.NODES N3 ON N2.ID = N3.ID " +
+            "WHERE N3.NODE_ORDER = 1")
             .get(0).get(0));
 
         // Check sub-query
         assertEquals(ignite1.cluster().localNode().id(), execSql("SELECT N1.ID FROM IGNITE.NODES N1 " +
-            "WHERE NOT EXISTS (SELECT 1 FROM IGNITE.NODES N2 WHERE N2.ID = N1.ID AND N2.IS_LOCAL = false)")
+            "WHERE NOT EXISTS (SELECT 1 FROM IGNITE.NODES N2 WHERE N2.ID = N1.ID AND N2.NODE_ORDER <> 1)")
             .get(0).get(0));
 
         // Check node attributes view

From 54acedd5962856705d2157dd347e38a985292086 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Fri, 7 Sep 2018 14:22:48 +0300
Subject: [PATCH 349/543] IGNITE-9366: SQL: Added IGNITE.NODE_METRICS system
 view. This closes #4615.

(cherry picked from commit fd3c50e0df916cd636b98849d02d3a730fcac73d)
---
 .../processors/query/h2/IgniteH2Indexing.java |   2 +
 .../sys/view/SqlAbstractLocalSystemView.java  |  27 +++
 .../h2/sys/view/SqlSystemViewNodeMetrics.java | 210 ++++++++++++++++
 .../query/SqlSystemViewsSelfTest.java         | 226 ++++++++++++++++--
 4 files changed, 448 insertions(+), 17 deletions(-)
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index b6adb7417206d..090975c3b8c2a 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -116,6 +116,7 @@
 import org.apache.ignite.internal.processors.query.h2.sys.SqlSystemTableEngine;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewBaselineNodes;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodeAttributes;
+import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodeMetrics;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodes;
 import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor;
 import org.apache.ignite.internal.processors.query.h2.twostep.GridReduceQueryExecutor;
@@ -2598,6 +2599,7 @@ public Collection<SqlSystemView> systemViews(GridKernalContext ctx) {
         views.add(new SqlSystemViewNodes(ctx));
         views.add(new SqlSystemViewNodeAttributes(ctx));
         views.add(new SqlSystemViewBaselineNodes(ctx));
+        views.add(new SqlSystemViewNodeMetrics(ctx));
 
         return views;
     }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
index ac90b631b8297..d692dbac3dafa 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
@@ -26,6 +26,8 @@
 import org.h2.value.Value;
 import org.h2.value.ValueNull;
 import org.h2.value.ValueString;
+import org.h2.value.ValueTime;
+import org.h2.value.ValueTimestamp;
 
 /**
  * Local system view base class (which uses only local node data).
@@ -125,4 +127,29 @@ protected static UUID uuidFromValue(Value val) {
             return null;
         }
     }
+
+    /**
+     * Converts millis to ValueTime
+     *
+     * @param millis Millis.
+     */
+    protected static Value valueTimeFromMillis(long millis) {
+        if (millis == -1L || millis == Long.MAX_VALUE)
+            return ValueNull.INSTANCE;
+        else
+            // Note: ValueTime.fromMillis(long) method trying to convert time using timezone and return wrong result.
+            return ValueTime.fromNanos(millis * 1_000_000L);
+    }
+
+    /**
+     * Converts millis to ValueTimestamp
+     *
+     * @param millis Millis.
+     */
+    protected static Value valueTimestampFromMillis(long millis) {
+        if (millis <= 0L || millis == Long.MAX_VALUE)
+            return ValueNull.INSTANCE;
+        else
+            return ValueTimestamp.fromMillis(millis);
+    }
 }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java
new file mode 100644
index 0000000000000..01b4e976f0cae
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys.view;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.util.typedef.F;
+import org.h2.engine.Session;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.value.Value;
+
+/**
+ * System view: node metrics.
+ */
+public class SqlSystemViewNodeMetrics extends SqlAbstractLocalSystemView {
+    /**
+     * @param ctx Grid context.
+     */
+    public SqlSystemViewNodeMetrics(GridKernalContext ctx) {
+        super("NODE_METRICS", "Node metrics", ctx, new String[] {"NODE_ID"},
+            newColumn("NODE_ID", Value.UUID),
+            newColumn("LAST_UPDATE_TIME", Value.TIMESTAMP),
+            newColumn("MAX_ACTIVE_JOBS", Value.INT),
+            newColumn("CUR_ACTIVE_JOBS", Value.INT),
+            newColumn("AVG_ACTIVE_JOBS", Value.FLOAT),
+            newColumn("MAX_WAITING_JOBS", Value.INT),
+            newColumn("CUR_WAITING_JOBS", Value.INT),
+            newColumn("AVG_WAITING_JOBS", Value.FLOAT),
+            newColumn("MAX_REJECTED_JOBS", Value.INT),
+            newColumn("CUR_REJECTED_JOBS", Value.INT),
+            newColumn("AVG_REJECTED_JOBS", Value.FLOAT),
+            newColumn("TOTAL_REJECTED_JOBS", Value.INT),
+            newColumn("MAX_CANCELED_JOBS", Value.INT),
+            newColumn("CUR_CANCELED_JOBS", Value.INT),
+            newColumn("AVG_CANCELED_JOBS", Value.FLOAT),
+            newColumn("TOTAL_CANCELED_JOBS", Value.INT),
+            newColumn("MAX_JOBS_WAIT_TIME", Value.TIME),
+            newColumn("CUR_JOBS_WAIT_TIME", Value.TIME),
+            newColumn("AVG_JOBS_WAIT_TIME", Value.TIME),
+            newColumn("MAX_JOBS_EXECUTE_TIME", Value.TIME),
+            newColumn("CUR_JOBS_EXECUTE_TIME", Value.TIME),
+            newColumn("AVG_JOBS_EXECUTE_TIME", Value.TIME),
+            newColumn("TOTAL_JOBS_EXECUTE_TIME", Value.TIME),
+            newColumn("TOTAL_EXECUTED_JOBS", Value.INT),
+            newColumn("TOTAL_EXECUTED_TASKS", Value.INT),
+            newColumn("TOTAL_BUSY_TIME", Value.TIME),
+            newColumn("TOTAL_IDLE_TIME", Value.TIME),
+            newColumn("CUR_IDLE_TIME", Value.TIME),
+            newColumn("BUSY_TIME_PERCENTAGE", Value.FLOAT),
+            newColumn("IDLE_TIME_PERCENTAGE", Value.FLOAT),
+            newColumn("TOTAL_CPU", Value.INT),
+            newColumn("CUR_CPU_LOAD", Value.DOUBLE),
+            newColumn("AVG_CPU_LOAD", Value.DOUBLE),
+            newColumn("CUR_GC_CPU_LOAD", Value.DOUBLE),
+            newColumn("HEAP_MEMORY_INIT", Value.LONG),
+            newColumn("HEAP_MEMORY_USED", Value.LONG),
+            newColumn("HEAP_MEMORY_COMMITED", Value.LONG),
+            newColumn("HEAP_MEMORY_MAX", Value.LONG),
+            newColumn("HEAP_MEMORY_TOTAL", Value.LONG),
+            newColumn("NONHEAP_MEMORY_INIT", Value.LONG),
+            newColumn("NONHEAP_MEMORY_USED", Value.LONG),
+            newColumn("NONHEAP_MEMORY_COMMITED", Value.LONG),
+            newColumn("NONHEAP_MEMORY_MAX", Value.LONG),
+            newColumn("NONHEAP_MEMORY_TOTAL", Value.LONG),
+            newColumn("UPTIME", Value.TIME),
+            newColumn("JVM_START_TIME", Value.TIMESTAMP),
+            newColumn("NODE_START_TIME", Value.TIMESTAMP),
+            newColumn("LAST_DATA_VERSION", Value.LONG),
+            newColumn("CUR_THREAD_COUNT", Value.INT),
+            newColumn("MAX_THREAD_COUNT", Value.INT),
+            newColumn("TOTAL_THREAD_COUNT", Value.LONG),
+            newColumn("CUR_DAEMON_THREAD_COUNT", Value.INT),
+            newColumn("SENT_MESSAGES_COUNT", Value.INT),
+            newColumn("SENT_BYTES_COUNT", Value.LONG),
+            newColumn("RECEIVED_MESSAGES_COUNT", Value.INT),
+            newColumn("RECEIVED_BYTES_COUNT", Value.LONG),
+            newColumn("OUTBOUND_MESSAGES_QUEUE", Value.INT)
+        );
+    }
+
+    /** {@inheritDoc} */
+    @Override public Iterator<Row> getRows(Session ses, SearchRow first, SearchRow last) {
+        List<Row> rows = new ArrayList<>();
+
+        Collection<ClusterNode> nodes;
+
+        SqlSystemViewColumnCondition idCond = conditionForColumn("NODE_ID", first, last);
+
+        if (idCond.isEquality()) {
+            try {
+                UUID nodeId = uuidFromValue(idCond.valueForEquality());
+
+                ClusterNode node = nodeId == null ? null : ctx.discovery().node(nodeId);
+
+                if (node != null)
+                    nodes = Collections.singleton(node);
+                else
+                    nodes = Collections.emptySet();
+            }
+            catch (Exception e) {
+                nodes = Collections.emptySet();
+            }
+        }
+        else
+            nodes = F.concat(false, ctx.discovery().allNodes(), ctx.discovery().daemonNodes());
+
+        for (ClusterNode node : nodes) {
+            if (node != null) {
+                ClusterMetrics metrics = node.metrics();
+
+                rows.add(
+                    createRow(ses, rows.size(),
+                        node.id(),
+                        valueTimestampFromMillis(metrics.getLastUpdateTime()),
+                        metrics.getMaximumActiveJobs(),
+                        metrics.getCurrentActiveJobs(),
+                        metrics.getAverageActiveJobs(),
+                        metrics.getMaximumWaitingJobs(),
+                        metrics.getCurrentWaitingJobs(),
+                        metrics.getAverageWaitingJobs(),
+                        metrics.getMaximumRejectedJobs(),
+                        metrics.getCurrentRejectedJobs(),
+                        metrics.getAverageRejectedJobs(),
+                        metrics.getTotalRejectedJobs(),
+                        metrics.getMaximumCancelledJobs(),
+                        metrics.getCurrentCancelledJobs(),
+                        metrics.getAverageCancelledJobs(),
+                        metrics.getTotalCancelledJobs(),
+                        valueTimeFromMillis(metrics.getMaximumJobWaitTime()),
+                        valueTimeFromMillis(metrics.getCurrentJobWaitTime()),
+                        valueTimeFromMillis((long)metrics.getAverageJobWaitTime()),
+                        valueTimeFromMillis(metrics.getMaximumJobExecuteTime()),
+                        valueTimeFromMillis(metrics.getCurrentJobExecuteTime()),
+                        valueTimeFromMillis((long)metrics.getAverageJobExecuteTime()),
+                        valueTimeFromMillis(metrics.getTotalJobsExecutionTime()),
+                        metrics.getTotalExecutedJobs(),
+                        metrics.getTotalExecutedTasks(),
+                        valueTimeFromMillis(metrics.getTotalBusyTime()),
+                        valueTimeFromMillis(metrics.getTotalIdleTime()),
+                        valueTimeFromMillis(metrics.getCurrentIdleTime()),
+                        metrics.getBusyTimePercentage(),
+                        metrics.getIdleTimePercentage(),
+                        metrics.getTotalCpus(),
+                        metrics.getCurrentCpuLoad(),
+                        metrics.getAverageCpuLoad(),
+                        metrics.getCurrentGcCpuLoad(),
+                        metrics.getHeapMemoryInitialized(),
+                        metrics.getHeapMemoryUsed(),
+                        metrics.getHeapMemoryCommitted(),
+                        metrics.getHeapMemoryMaximum(),
+                        metrics.getHeapMemoryTotal(),
+                        metrics.getNonHeapMemoryInitialized(),
+                        metrics.getNonHeapMemoryUsed(),
+                        metrics.getNonHeapMemoryCommitted(),
+                        metrics.getNonHeapMemoryMaximum(),
+                        metrics.getNonHeapMemoryTotal(),
+                        valueTimeFromMillis(metrics.getUpTime()),
+                        valueTimestampFromMillis(metrics.getStartTime()),
+                        valueTimestampFromMillis(metrics.getNodeStartTime()),
+                        metrics.getLastDataVersion(),
+                        metrics.getCurrentThreadCount(),
+                        metrics.getMaximumThreadCount(),
+                        metrics.getTotalStartedThreadCount(),
+                        metrics.getCurrentDaemonThreadCount(),
+                        metrics.getSentMessagesCount(),
+                        metrics.getSentBytesCount(),
+                        metrics.getReceivedMessagesCount(),
+                        metrics.getReceivedBytesCount(),
+                        metrics.getOutboundMessagesQueueSize()
+                    )
+                );
+            }
+        }
+
+        return rows.iterator();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canGetRowCount() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCount() {
+        return F.concat(false, ctx.discovery().allNodes(), ctx.discovery().daemonNodes()).size();
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
index c8374a78ffc2f..1a4dae794fb10 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
@@ -17,21 +17,31 @@
 
 package org.apache.ignite.internal.processors.query;
 
+import java.sql.Time;
+import java.sql.Timestamp;
 import java.util.Collections;
 import java.util.List;
+import java.util.Random;
+import java.util.TimeZone;
 import java.util.UUID;
 import java.util.concurrent.Callable;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.QueryEntity;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.ClusterMetricsSnapshot;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
@@ -39,6 +49,9 @@
  * Tests for ignite SQL system views.
  */
 public class SqlSystemViewsSelfTest extends GridCommonAbstractTest {
+    /** Metrics check attempts. */
+    private static final int METRICS_CHECK_ATTEMPTS = 10;
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         super.beforeTest();
@@ -171,7 +184,7 @@ public void testCacheToViewJoin() throws Exception {
     private void assertColumnTypes(List<?> rowData, Class<?> ... colTypes) {
         for (int i = 0; i < colTypes.length; i++) {
             if (rowData.get(i) != null)
-                assertEquals("Column " + i + " type", rowData.get(i).getClass(), colTypes[i]);
+                assertEquals("Column " + i + " type", colTypes[i], rowData.get(i).getClass());
         }
     }
 
@@ -181,9 +194,14 @@ private void assertColumnTypes(List<?> rowData, Class<?> ... colTypes) {
      * @throws Exception If failed.
      */
     public void testNodesViews() throws Exception {
-        Ignite ignite1 = startGrid(getTestIgniteInstanceName(), getConfiguration());
-        Ignite ignite2 = startGrid(getTestIgniteInstanceName(1), getConfiguration().setClientMode(true));
-        Ignite ignite3 = startGrid(getTestIgniteInstanceName(2), getConfiguration().setDaemon(true));
+        Ignite igniteSrv = startGrid(getTestIgniteInstanceName(), getConfiguration().setMetricsUpdateFrequency(500L));
+
+        Ignite igniteCli = startGrid(getTestIgniteInstanceName(1), getConfiguration().setMetricsUpdateFrequency(500L)
+            .setClientMode(true));
+
+        startGrid(getTestIgniteInstanceName(2), getConfiguration().setMetricsUpdateFrequency(500L).setDaemon(true));
+
+        UUID nodeId0 = igniteSrv.cluster().localNode().id();
 
         awaitPartitionMapExchange();
 
@@ -201,7 +219,7 @@ public void testNodesViews() throws Exception {
 
         assertEquals(1, resSrv.size());
 
-        assertEquals(ignite1.cluster().localNode().id(), resSrv.get(0).get(0));
+        assertEquals(nodeId0, resSrv.get(0).get(0));
 
         assertEquals(1, resSrv.get(0).get(1));
 
@@ -210,7 +228,7 @@ public void testNodesViews() throws Exception {
 
         assertEquals(1, resCli.size());
 
-        assertEquals(ignite2.cluster().localNode().id(), resCli.get(0).get(0));
+        assertEquals(nodeId(1), resCli.get(0).get(0));
 
         assertEquals(2, resCli.get(0).get(1));
 
@@ -219,7 +237,7 @@ public void testNodesViews() throws Exception {
 
         assertEquals(1, resDaemon.size());
 
-        assertEquals(ignite3.cluster().localNode().id(), resDaemon.get(0).get(0));
+        assertEquals(nodeId(2), resDaemon.get(0).get(0));
 
         assertEquals(3, resDaemon.get(0).get(1));
 
@@ -227,32 +245,30 @@ public void testNodesViews() throws Exception {
         assertEquals(0, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = '-'").size());
 
         assertEquals(1, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = ?",
-            ignite1.cluster().localNode().id()).size());
+            nodeId0).size());
 
         assertEquals(1, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = ?",
-            ignite3.cluster().localNode().id()).size());
+            nodeId(2)).size());
 
         // Check index on ID column with disjunction.
         assertEquals(3, execSql("SELECT ID FROM IGNITE.NODES WHERE ID = ? " +
-            "OR node_order=1 OR node_order=2 OR node_order=3", ignite1.cluster().localNode().id()).size());
+            "OR node_order=1 OR node_order=2 OR node_order=3", nodeId0).size());
 
         // Check quick-count.
         assertEquals(3L, execSql("SELECT COUNT(*) FROM IGNITE.NODES").get(0).get(0));
 
         // Check joins
-        assertEquals(ignite1.cluster().localNode().id(), execSql("SELECT N1.ID FROM IGNITE.NODES N1 JOIN " +
+        assertEquals(nodeId0, execSql("SELECT N1.ID FROM IGNITE.NODES N1 JOIN " +
             "IGNITE.NODES N2 ON N1.NODE_ORDER = N2.NODE_ORDER JOIN IGNITE.NODES N3 ON N2.ID = N3.ID " +
             "WHERE N3.NODE_ORDER = 1")
             .get(0).get(0));
 
         // Check sub-query
-        assertEquals(ignite1.cluster().localNode().id(), execSql("SELECT N1.ID FROM IGNITE.NODES N1 " +
+        assertEquals(nodeId0, execSql("SELECT N1.ID FROM IGNITE.NODES N1 " +
             "WHERE NOT EXISTS (SELECT 1 FROM IGNITE.NODES N2 WHERE N2.ID = N1.ID AND N2.NODE_ORDER <> 1)")
             .get(0).get(0));
 
         // Check node attributes view
-        UUID cliNodeId = ignite2.cluster().localNode().id();
-
         String cliAttrName = IgniteNodeAttributes.ATTR_CLIENT_MODE;
 
         assertColumnTypes(execSql("SELECT NODE_ID, NAME, VALUE FROM IGNITE.NODE_ATTRIBUTES").get(0),
@@ -267,7 +283,7 @@ public void testNodesViews() throws Exception {
 
         assertEquals(1,
             execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NODE_ID = ? AND NAME = ? AND VALUE = 'true'",
-                cliNodeId, cliAttrName).size());
+                nodeId(1), cliAttrName).size());
 
         assertEquals(0,
             execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NODE_ID = '-' AND NAME = ?",
@@ -275,7 +291,169 @@ public void testNodesViews() throws Exception {
 
         assertEquals(0,
             execSql("SELECT NODE_ID FROM IGNITE.NODE_ATTRIBUTES WHERE NODE_ID = ? AND NAME = '-'",
-                cliNodeId).size());
+                nodeId(1)).size());
+
+        // Check node metrics view.
+        String sqlAllMetrics = "SELECT NODE_ID, LAST_UPDATE_TIME, " +
+            "MAX_ACTIVE_JOBS, CUR_ACTIVE_JOBS, AVG_ACTIVE_JOBS, " +
+            "MAX_WAITING_JOBS, CUR_WAITING_JOBS, AVG_WAITING_JOBS, " +
+            "MAX_REJECTED_JOBS, CUR_REJECTED_JOBS, AVG_REJECTED_JOBS, TOTAL_REJECTED_JOBS, " +
+            "MAX_CANCELED_JOBS, CUR_CANCELED_JOBS, AVG_CANCELED_JOBS, TOTAL_CANCELED_JOBS, " +
+            "MAX_JOBS_WAIT_TIME, CUR_JOBS_WAIT_TIME, AVG_JOBS_WAIT_TIME, " +
+            "MAX_JOBS_EXECUTE_TIME, CUR_JOBS_EXECUTE_TIME, AVG_JOBS_EXECUTE_TIME, TOTAL_JOBS_EXECUTE_TIME, " +
+            "TOTAL_EXECUTED_JOBS, TOTAL_EXECUTED_TASKS, " +
+            "TOTAL_BUSY_TIME, TOTAL_IDLE_TIME, CUR_IDLE_TIME, BUSY_TIME_PERCENTAGE, IDLE_TIME_PERCENTAGE, " +
+            "TOTAL_CPU, CUR_CPU_LOAD, AVG_CPU_LOAD, CUR_GC_CPU_LOAD, " +
+            "HEAP_MEMORY_INIT, HEAP_MEMORY_USED, HEAP_MEMORY_COMMITED, HEAP_MEMORY_MAX, HEAP_MEMORY_TOTAL, " +
+            "NONHEAP_MEMORY_INIT, NONHEAP_MEMORY_USED, NONHEAP_MEMORY_COMMITED, NONHEAP_MEMORY_MAX, NONHEAP_MEMORY_TOTAL, " +
+            "UPTIME, JVM_START_TIME, NODE_START_TIME, LAST_DATA_VERSION, " +
+            "CUR_THREAD_COUNT, MAX_THREAD_COUNT, TOTAL_THREAD_COUNT, CUR_DAEMON_THREAD_COUNT, " +
+            "SENT_MESSAGES_COUNT, SENT_BYTES_COUNT, RECEIVED_MESSAGES_COUNT, RECEIVED_BYTES_COUNT, " +
+            "OUTBOUND_MESSAGES_QUEUE FROM IGNITE.NODE_METRICS";
+
+        List<List<?>> resMetrics = execSql(sqlAllMetrics);
+
+        assertColumnTypes(resMetrics.get(0), UUID.class, Timestamp.class,
+            Integer.class, Integer.class, Float.class, // Active jobs.
+            Integer.class, Integer.class, Float.class, // Waiting jobs.
+            Integer.class, Integer.class, Float.class, Integer.class, // Rejected jobs.
+            Integer.class, Integer.class, Float.class, Integer.class, // Canceled jobs.
+            Time.class, Time.class, Time.class, // Jobs wait time.
+            Time.class, Time.class, Time.class, Time.class, // Jobs execute time.
+            Integer.class, Integer.class, // Executed jobs/task.
+            Time.class, Time.class, Time.class, Float.class, Float.class, // Busy/idle time.
+            Integer.class, Double.class, Double.class, Double.class, // CPU.
+            Long.class, Long.class, Long.class, Long.class, Long.class, // Heap memory.
+            Long.class, Long.class, Long.class, Long.class, Long.class, // Nonheap memory.
+            Time.class, Timestamp.class, Timestamp.class, Long.class, // Uptime.
+            Integer.class, Integer.class, Long.class, Integer.class, // Threads.
+            Integer.class, Long.class, Integer.class, Long.class, // Sent/received messages.
+            Integer.class); // Outbound message queue.
+
+        assertEquals(3, resAll.size());
+
+        // Check join with nodes.
+        assertEquals(3, execSql("SELECT NM.LAST_UPDATE_TIME FROM IGNITE.NODES N " +
+            "JOIN IGNITE.NODE_METRICS NM ON N.ID = NM.NODE_ID").size());
+
+        // Check index on NODE_ID column.
+        assertEquals(1, execSql("SELECT LAST_UPDATE_TIME FROM IGNITE.NODE_METRICS WHERE NODE_ID = ?",
+            nodeId(1)).size());
+
+        // Check malformed value for indexed column.
+        assertEquals(0, execSql("SELECT LAST_UPDATE_TIME FROM IGNITE.NODE_METRICS WHERE NODE_ID = ?",
+            "-").size());
+
+        // Check quick-count.
+        assertEquals(3L, execSql("SELECT COUNT(*) FROM IGNITE.NODE_METRICS").get(0).get(0));
+
+        // Check metric values.
+
+        // Broadcast jobs to server and client nodes to get non zero metric values.
+        for (int i = 0; i < 100; i++) {
+            IgniteFuture<Void > fut = igniteSrv.compute(igniteSrv.cluster().forNodeId(nodeId0, nodeId(1)))
+                .broadcastAsync(
+                    new IgniteRunnable() {
+                        @Override public void run() {
+                            Random rnd = new Random();
+
+                            try {
+                                doSleep(rnd.nextInt(100));
+                            }
+                            catch (Throwable ignore) {
+                                // No-op.
+                            }
+                        }
+                    });
+
+            if (i % 10 == 0)
+                fut.cancel();
+        }
+
+        doSleep(igniteSrv.configuration().getMetricsUpdateFrequency() * 3L);
+
+        for (Ignite grid : G.allGrids()) {
+            UUID nodeId = grid.cluster().localNode().id();
+
+            // Metrics for node must be collected from another node to avoid race and get consistent metrics snapshot.
+            Ignite ignite = F.eq(nodeId, nodeId0) ? igniteCli : igniteSrv;
+
+            for (int i = 0; i < METRICS_CHECK_ATTEMPTS; i++) {
+                ClusterMetrics metrics = ignite.cluster().node(nodeId).metrics();
+
+                assertTrue(metrics instanceof ClusterMetricsSnapshot);
+
+                resMetrics = execSql(ignite, sqlAllMetrics + " WHERE NODE_ID = ?", nodeId);
+
+                log.info("Check metrics for node " + grid.name() + ", attempt " + (i + 1));
+
+                if (metrics.getLastUpdateTime() == ((Timestamp)resMetrics.get(0).get(1)).getTime()) {
+                    assertEquals(metrics.getMaximumActiveJobs(), resMetrics.get(0).get(2));
+                    assertEquals(metrics.getCurrentActiveJobs(), resMetrics.get(0).get(3));
+                    assertEquals(metrics.getAverageActiveJobs(), resMetrics.get(0).get(4));
+                    assertEquals(metrics.getMaximumWaitingJobs(), resMetrics.get(0).get(5));
+                    assertEquals(metrics.getCurrentWaitingJobs(), resMetrics.get(0).get(6));
+                    assertEquals(metrics.getAverageWaitingJobs(), resMetrics.get(0).get(7));
+                    assertEquals(metrics.getMaximumRejectedJobs(), resMetrics.get(0).get(8));
+                    assertEquals(metrics.getCurrentRejectedJobs(), resMetrics.get(0).get(9));
+                    assertEquals(metrics.getAverageRejectedJobs(), resMetrics.get(0).get(10));
+                    assertEquals(metrics.getTotalRejectedJobs(), resMetrics.get(0).get(11));
+                    assertEquals(metrics.getMaximumCancelledJobs(), resMetrics.get(0).get(12));
+                    assertEquals(metrics.getCurrentCancelledJobs(), resMetrics.get(0).get(13));
+                    assertEquals(metrics.getAverageCancelledJobs(), resMetrics.get(0).get(14));
+                    assertEquals(metrics.getTotalCancelledJobs(), resMetrics.get(0).get(15));
+                    assertEquals(metrics.getMaximumJobWaitTime(), convertToMilliseconds(resMetrics.get(0).get(16)));
+                    assertEquals(metrics.getCurrentJobWaitTime(), convertToMilliseconds(resMetrics.get(0).get(17)));
+                    assertEquals((long)metrics.getAverageJobWaitTime(), convertToMilliseconds(resMetrics.get(0).get(18)));
+                    assertEquals(metrics.getMaximumJobExecuteTime(), convertToMilliseconds(resMetrics.get(0).get(19)));
+                    assertEquals(metrics.getCurrentJobExecuteTime(), convertToMilliseconds(resMetrics.get(0).get(20)));
+                    assertEquals((long)metrics.getAverageJobExecuteTime(), convertToMilliseconds(resMetrics.get(0).get(21)));
+                    assertEquals(metrics.getTotalJobsExecutionTime(), convertToMilliseconds(resMetrics.get(0).get(22)));
+                    assertEquals(metrics.getTotalExecutedJobs(), resMetrics.get(0).get(23));
+                    assertEquals(metrics.getTotalExecutedTasks(), resMetrics.get(0).get(24));
+                    assertEquals(metrics.getTotalBusyTime(), convertToMilliseconds(resMetrics.get(0).get(25)));
+                    assertEquals(metrics.getTotalIdleTime(), convertToMilliseconds(resMetrics.get(0).get(26)));
+                    assertEquals(metrics.getCurrentIdleTime(), convertToMilliseconds(resMetrics.get(0).get(27)));
+                    assertEquals(metrics.getBusyTimePercentage(), resMetrics.get(0).get(28));
+                    assertEquals(metrics.getIdleTimePercentage(), resMetrics.get(0).get(29));
+                    assertEquals(metrics.getTotalCpus(), resMetrics.get(0).get(30));
+                    assertEquals(metrics.getCurrentCpuLoad(), resMetrics.get(0).get(31));
+                    assertEquals(metrics.getAverageCpuLoad(), resMetrics.get(0).get(32));
+                    assertEquals(metrics.getCurrentGcCpuLoad(), resMetrics.get(0).get(33));
+                    assertEquals(metrics.getHeapMemoryInitialized(), resMetrics.get(0).get(34));
+                    assertEquals(metrics.getHeapMemoryUsed(), resMetrics.get(0).get(35));
+                    assertEquals(metrics.getHeapMemoryCommitted(), resMetrics.get(0).get(36));
+                    assertEquals(metrics.getHeapMemoryMaximum(), resMetrics.get(0).get(37));
+                    assertEquals(metrics.getHeapMemoryTotal(), resMetrics.get(0).get(38));
+                    assertEquals(metrics.getNonHeapMemoryInitialized(), resMetrics.get(0).get(39));
+                    assertEquals(metrics.getNonHeapMemoryUsed(), resMetrics.get(0).get(40));
+                    assertEquals(metrics.getNonHeapMemoryCommitted(), resMetrics.get(0).get(41));
+                    assertEquals(metrics.getNonHeapMemoryMaximum(), resMetrics.get(0).get(42));
+                    assertEquals(metrics.getNonHeapMemoryTotal(), resMetrics.get(0).get(43));
+                    assertEquals(metrics.getUpTime(), convertToMilliseconds(resMetrics.get(0).get(44)));
+                    assertEquals(metrics.getStartTime(), ((Timestamp)resMetrics.get(0).get(45)).getTime());
+                    assertEquals(metrics.getNodeStartTime(), ((Timestamp)resMetrics.get(0).get(46)).getTime());
+                    assertEquals(metrics.getLastDataVersion(), resMetrics.get(0).get(47));
+                    assertEquals(metrics.getCurrentThreadCount(), resMetrics.get(0).get(48));
+                    assertEquals(metrics.getMaximumThreadCount(), resMetrics.get(0).get(49));
+                    assertEquals(metrics.getTotalStartedThreadCount(), resMetrics.get(0).get(50));
+                    assertEquals(metrics.getCurrentDaemonThreadCount(), resMetrics.get(0).get(51));
+                    assertEquals(metrics.getSentMessagesCount(), resMetrics.get(0).get(52));
+                    assertEquals(metrics.getSentBytesCount(), resMetrics.get(0).get(53));
+                    assertEquals(metrics.getReceivedMessagesCount(), resMetrics.get(0).get(54));
+                    assertEquals(metrics.getReceivedBytesCount(), resMetrics.get(0).get(55));
+                    assertEquals(metrics.getOutboundMessagesQueueSize(), resMetrics.get(0).get(56));
+
+                    break;
+                }
+                else {
+                    log.info("Metrics was updated in background, will retry check");
+
+                    if (i == METRICS_CHECK_ATTEMPTS - 1)
+                        fail("Failed to check metrics, attempts limit reached (" + METRICS_CHECK_ATTEMPTS + ')');
+                }
+            }
+        }
     }
 
     /**
@@ -322,7 +500,7 @@ public void testBaselineViews() throws Exception {
     }
 
     /**
-     * Gets ignite configuration with persistance enabled.
+     * Gets ignite configuration with persistence enabled.
      */
     private IgniteConfiguration getPdsConfiguration(String consistentId) throws Exception {
         IgniteConfiguration cfg = getConfiguration();
@@ -336,4 +514,18 @@ private IgniteConfiguration getPdsConfiguration(String consistentId) throws Exce
 
         return cfg;
     }
+
+    /**
+     * Convert Time to milliseconds.
+     *
+     * Note: Returned Time values from SQL it's milliseconds since January 1, 1970, 00:00:00 GMT. To get right interval
+     * in milliseconds this value must be adjusted to current time zone.
+     *
+     * @param sqlTime Time value returned from SQL.
+     */
+    private long convertToMilliseconds(Object sqlTime) {
+        Time time0 = (Time)sqlTime;
+
+        return time0.getTime() + TimeZone.getDefault().getOffset(time0.getTime());
+    }
 }

From 01906da6ac6b8d43ee1ef63473fa102e1c7c8133 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 11 Sep 2018 16:47:56 +0300
Subject: [PATCH 350/543] IGNITE-9084 Fixed conflicting messages order

---
 .../managers/communication/GridIoMessageFactory.java  | 11 ++++++-----
 .../preloader/GridDhtPartitionSupplyMessageV2.java    |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index a5f0066092f20..eda62607a1652 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -868,11 +868,7 @@ public GridIoMessageFactory(MessageFactory[] ext) {
 
                 break;
 
-            case 120:
-                msg = new GridDhtPartitionSupplyMessageV2();
-
-                break;
-
+            // [120..123] - DR
             case 124:
                 msg = new GridMessageCollection<>();
 
@@ -933,6 +929,11 @@ public GridIoMessageFactory(MessageFactory[] ext) {
 
                 break;
 
+            case 158:
+                msg = new GridDhtPartitionSupplyMessageV2();
+
+                break;
+
             // [-3..119] [124..129] [-23..-27] [-36..-55]- this
             // [120..123] - DR
             // [-4..-22, -30..-35] - SQL
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
index a77576683815f..b6bff0e823506 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
@@ -143,7 +143,7 @@ public GridDhtPartitionSupplyMessageV2(
 
     /** {@inheritDoc} */
     @Override public short directType() {
-        return 120;
+        return 158;
     }
 
     /** {@inheritDoc} */

From 693c52924dee7dc34862c720c0c5b81541b0dd5e Mon Sep 17 00:00:00 2001
From: Eduard Shangareev <eshangareev@gridgain.com>
Date: Tue, 31 Jul 2018 16:25:50 +0300
Subject: [PATCH 351/543] IGNITE-9100 Split Basic and Cache TC configurations
 on pure in-memory and with disk usage one

Signed-off-by: Andrey Gura <agura@apache.org>
(cherry picked from commit c5e723c533a336802ba0fa6b9e9f361182db16ff)
---
 .../junits/GridAbstractTest.java              | 96 ++++++++++++++-----
 .../testsuites/IgniteBasicTestSuite.java      |  8 +-
 .../IgniteBasicWithPersistenceTestSuite.java  | 63 ++++++++++++
 ...niteBinaryObjectsComputeGridTestSuite.java |  3 +
 .../IgniteCacheEvictionSelfTestSuite.java     |  3 -
 .../IgniteCacheFullApiSelfTestSuite.java      |  3 +
 .../IgniteCacheMetricsSelfTestSuite.java      |  2 -
 .../testsuites/IgniteCacheTestSuite.java      |  8 +-
 .../testsuites/IgniteCacheTestSuite2.java     |  5 +-
 .../testsuites/IgniteCacheTestSuite3.java     |  7 +-
 .../testsuites/IgniteCacheTestSuite4.java     |  5 +-
 .../testsuites/IgniteCacheTestSuite5.java     | 12 +--
 .../testsuites/IgniteCacheTestSuite6.java     | 28 +-----
 .../testsuites/IgniteCacheTestSuite7.java     | 11 ++-
 .../IgniteComputeGridTestSuite.java           |  2 -
 .../testsuites/IgniteKernalSelfTestSuite.java |  3 +-
 .../IgniteMarshallerSelfTestSuite.java        |  3 +-
 .../testsuites/IgniteUtilSelfTestSuite.java   |  7 +-
 ...GridInternalTaskUnusedWalSegmentsTest.java | 27 +++---
 ...heWithIndexingAndPersistenceTestSuite.java | 38 ++++++++
 .../IgniteCacheWithIndexingTestSuite.java     |  3 +
 21 files changed, 234 insertions(+), 103 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 00929262e69c0..d106b49203445 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -17,31 +17,6 @@
 
 package org.apache.ignite.testframework.junits;
 
-import java.io.ObjectStreamException;
-import java.io.Serializable;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.lang.reflect.Proxy;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import javax.cache.configuration.Factory;
-import javax.cache.configuration.FactoryBuilder;
 import junit.framework.TestCase;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
@@ -55,6 +30,8 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.BinaryConfiguration;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.EventType;
@@ -121,6 +98,32 @@
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.FileSystemXmlApplicationContext;
 
+import javax.cache.configuration.Factory;
+import javax.cache.configuration.FactoryBuilder;
+import java.io.ObjectStreamException;
+import java.io.Serializable;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.Proxy;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CLIENT_CACHE_CHANGE_MESSAGE_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
@@ -138,6 +141,9 @@
     "JUnitTestCaseWithNonTrivialConstructors"
 })
 public abstract class GridAbstractTest extends TestCase {
+    /** Persistence in tests allowed property. */
+    public static final String PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY = "PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY";
+
     /**************************************************************
      * DO NOT REMOVE TRANSIENT - THIS OBJECT MIGHT BE TRANSFERRED *
      *                  TO ANOTHER NODE.                          *
@@ -195,6 +201,12 @@ public abstract class GridAbstractTest extends TestCase {
     /** Number of tests. */
     private int testCnt;
 
+    /**
+     *
+     */
+    private static final boolean PERSISTENCE_ALLOWED =
+            IgniteSystemProperties.getBoolean(PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, true);
+
     /**
      *
      */
@@ -855,6 +867,9 @@ protected Ignite startGrid(String igniteInstanceName, GridSpringResourceContext
      */
     protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, GridSpringResourceContext ctx)
         throws Exception {
+
+        checkConfiguration(cfg);
+
         if (!isRemoteJvm(igniteInstanceName)) {
             startingIgniteInstanceName.set(igniteInstanceName);
 
@@ -899,6 +914,30 @@ protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, G
             return startRemoteGrid(igniteInstanceName, null, ctx);
     }
 
+    /**
+     * @param cfg Config.
+     */
+    protected void checkConfiguration(IgniteConfiguration cfg) {
+        if (cfg == null)
+            return;
+
+        if (!PERSISTENCE_ALLOWED) {
+            String errorMsg = "PERSISTENCE IS NOT ALLOWED IN THIS SUITE, PUT YOUR TEST TO ANOTHER ONE!";
+
+            DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration();
+
+            if (dsCfg != null) {
+                assertFalse(errorMsg, dsCfg.getDefaultDataRegionConfiguration().isPersistenceEnabled());
+
+                DataRegionConfiguration[] dataRegionConfigurations = dsCfg.getDataRegionConfigurations();
+
+                if (dataRegionConfigurations != null)
+                    for (DataRegionConfiguration dataRegionConfiguration : dataRegionConfigurations)
+                        assertFalse(errorMsg, dataRegionConfiguration.isPersistenceEnabled());
+            }
+        }
+    }
+
     /**
      * Starts new grid at another JVM with given name.
      *
@@ -995,6 +1034,8 @@ protected Ignite startRemoteGrid(String igniteInstanceName, IgniteConfiguration
         if (cfg == null)
             cfg = optimize(getConfiguration(igniteInstanceName));
 
+        checkConfiguration(cfg);
+
         if (locNode != null) {
             DiscoverySpi discoverySpi = locNode.configuration().getDiscoverySpi();
 
@@ -1033,7 +1074,8 @@ protected List<String> additionalRemoteJvmArgs() {
      * @throws IgniteCheckedException On error.
      */
     protected IgniteConfiguration optimize(IgniteConfiguration cfg) throws IgniteCheckedException {
-        // TODO: IGNITE-605: propose another way to avoid network overhead in tests.
+        checkConfiguration(cfg);
+
         if (cfg.getLocalHost() == null) {
             if (cfg.getDiscoverySpi() instanceof TcpDiscoverySpi) {
                 cfg.setLocalHost("127.0.0.1");
@@ -1635,6 +1677,8 @@ protected IgniteConfiguration getConfiguration(String igniteInstanceName, Ignite
 
         cfg.setFailureHandler(getFailureHandler(igniteInstanceName));
 
+        checkConfiguration(cfg);
+
         return cfg;
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index 4486fe8c7f69b..f8839bf77f43d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -17,11 +17,9 @@
 
 package org.apache.ignite.testsuites;
 
-import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.GridSuppressedExceptionSelfTest;
 import org.apache.ignite.failure.FailureHandlerTriggeredTest;
-import org.apache.ignite.failure.IoomFailureHandlerTest;
 import org.apache.ignite.failure.OomFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest;
@@ -85,12 +83,15 @@
 import org.apache.ignite.spi.GridSpiLocalHostInjectionTest;
 import org.apache.ignite.startup.properties.NotStringSystemPropertyTest;
 import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
 import org.apache.ignite.testframework.test.ConfigVariationsTestSuiteBuilderTest;
 import org.apache.ignite.testframework.test.ParametersTest;
 import org.apache.ignite.testframework.test.VariationsIteratorTest;
 import org.apache.ignite.util.AttributeNodeFilterSelfTest;
 import org.jetbrains.annotations.Nullable;
 
+import java.util.Set;
+
 /**
  * Basic test suite.
  */
@@ -109,6 +110,8 @@ public static TestSuite suite() throws Exception {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = new TestSuite("Ignite Basic Test Suite");
 
         suite.addTest(IgniteMarshallerSelfTestSuite.suite(ignoredTests));
@@ -206,7 +209,6 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(FailureHandlerTriggeredTest.class);
         suite.addTestSuite(StopNodeFailureHandlerTest.class);
         suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
-        suite.addTestSuite(IoomFailureHandlerTest.class);
         suite.addTestSuite(OomFailureHandlerTest.class);
         suite.addTestSuite(SystemWorkersTerminationTest.class);
         suite.addTestSuite(TransactionIntegrityWithSystemWorkerDeathTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
index e69de29bb2d1d..7b4b4c098b85e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.failure.IoomFailureHandlerTest;
+import org.apache.ignite.failure.SystemWorkersTerminationTest;
+import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest;
+import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest;
+import org.apache.ignite.marshaller.GridMarshallerMappingConsistencyTest;
+import org.apache.ignite.util.GridCommandHandlerTest;
+import org.apache.ignite.util.GridInternalTaskUnusedWalSegmentsTest;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Set;
+
+/**
+ * Basic test suite.
+ */
+public class IgniteBasicWithPersistenceTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        return suite(null);
+    }
+
+    /**
+     * @param ignoredTests Tests don't include in the execution. Providing null means nothing to exclude.
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Exception {
+        TestSuite suite = new TestSuite("Ignite Basic With Persistence Test Suite");
+
+        suite.addTestSuite(IoomFailureHandlerTest.class);
+        suite.addTestSuite(ClusterBaselineNodesMetricsSelfTest.class);
+        suite.addTestSuite(ServiceDeploymentOutsideBaselineTest.class);
+        suite.addTestSuite(GridMarshallerMappingConsistencyTest.class);
+        suite.addTestSuite(SystemWorkersTerminationTest.class);
+
+        suite.addTestSuite(GridCommandHandlerTest.class);
+        suite.addTestSuite(GridInternalTaskUnusedWalSegmentsTest.class);
+
+        return suite;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java
index 8798db9b1d667..4820d455683f5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsComputeGridTestSuite.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.GridComputationBinarylizableClosuresSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
 
 /**
  *
@@ -29,6 +30,8 @@ public class IgniteBinaryObjectsComputeGridTestSuite {
      * @throws Exception If failed.
      */
     public static TestSuite suite() throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = IgniteComputeGridTestSuite.suite();
 
         suite.addTestSuite(GridComputationBinarylizableClosuresSelfTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java
index 1d4fdf7c301c2..b127cfe6a432e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java
@@ -37,7 +37,6 @@
 import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearOnlyNearEvictionPolicySelfTest;
 import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionDataStreamerTest;
 import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMetricTest;
-import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest;
 import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionReadThroughTest;
 import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionTouchOrderTest;
 import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruNearEnabledPageEvictionMultinodeTest;
@@ -90,8 +89,6 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(PageEvictionReadThroughTest.class));
         suite.addTest(new TestSuite(PageEvictionDataStreamerTest.class));
 
-        suite.addTest(new TestSuite(PageEvictionMultinodeMixedRegionsTest.class));
-
         suite.addTest(new TestSuite(PageEvictionMetricTest.class));
 
         return suite;
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
index b380ebc7a89c8..bc46055a2a6b8 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
@@ -78,6 +78,7 @@
 import org.apache.ignite.internal.processors.cache.local.GridCacheLocalFullApiMultithreadedSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheLocalFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheLocalWithGroupFullApiSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
 
 /**
  * Test suite for cache API.
@@ -88,6 +89,8 @@ public class IgniteCacheFullApiSelfTestSuite extends TestSuite {
      * @throws Exception If failed.
      */
     public static TestSuite suite() throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = new TestSuite("Cache Full API Test Suite");
 
         // One node.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
index 237f7e1189282..82be3fa19db75 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
@@ -19,7 +19,6 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.TransactionMetricsMxBeanImplTest;
-import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMBeanTest;
 import org.apache.ignite.internal.processors.cache.CacheGroupsMetricsRebalanceTest;
 import org.apache.ignite.internal.processors.cache.CacheMetricsEnableRuntimeTest;
 import org.apache.ignite.internal.processors.cache.CacheMetricsEntitiesCountTest;
@@ -66,7 +65,6 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheAtomicLocalTckMetricsSelfTestImpl.class);
 
         suite.addTestSuite(CacheGroupsMetricsRebalanceTest.class);
-        suite.addTestSuite(CacheGroupMetricsMBeanTest.class);
         suite.addTestSuite(CacheValidatorMetricsTest.class);
         suite.addTestSuite(CacheMetricsEnableRuntimeTest.class);
         suite.addTestSuite(CacheMetricsEntitiesCountTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 1bf65e0bc7ad7..9c64a9f1ce344 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.testsuites;
 
-import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.cache.IgniteCacheEntryProcessorSequentialCallTest;
 import org.apache.ignite.cache.IgniteWarmupClosureSelfTest;
@@ -52,6 +51,7 @@
 import org.apache.ignite.internal.processors.cache.CacheNamesWithSpecialCharactersTest;
 import org.apache.ignite.internal.processors.cache.CachePutEventListenerErrorSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheTxFastFinishTest;
+import org.apache.ignite.internal.processors.cache.DataStorageConfigurationValidationTest;
 import org.apache.ignite.internal.processors.cache.GridCacheAffinityApiSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheAffinityMapperSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheAffinityRoutingSelfTest;
@@ -83,7 +83,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheTtlManagerSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheTxPartitionedLocalStoreSelfTest;
 import org.apache.ignite.internal.processors.cache.GridDataStorageConfigurationConsistencySelfTest;
-import org.apache.ignite.internal.processors.cache.DataStorageConfigurationValidationTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicInvokeTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalInvokeTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAtomicLocalWithStoreInvokeTest;
@@ -149,6 +148,9 @@
 import org.apache.ignite.internal.processors.datastreamer.DataStreamerTimeoutTest;
 import org.apache.ignite.internal.processors.datastreamer.DataStreamerUpdateAfterLoadTest;
 import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
+
+import java.util.Set;
 
 /**
  * Test suite.
@@ -168,6 +170,8 @@ public static TestSuite suite() throws Exception {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = new TestSuite("IgniteCache Test Suite");
 
         suite.addTestSuite(IgniteCacheEntryListenerAtomicTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
index e9c5e76ee2e96..6eca728ebbc9a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
@@ -31,7 +31,6 @@
 import org.apache.ignite.internal.processors.cache.CacheEnumOperationsTest;
 import org.apache.ignite.internal.processors.cache.CacheExchangeMessageDuplicatedStateTest;
 import org.apache.ignite.internal.processors.cache.CacheGroupLocalConfigurationSelfTest;
-import org.apache.ignite.internal.processors.cache.CacheDataRegionConfigurationTest;
 import org.apache.ignite.internal.processors.cache.CacheOptimisticTransactionsWithFilterSingleServerTest;
 import org.apache.ignite.internal.processors.cache.CacheOptimisticTransactionsWithFilterTest;
 import org.apache.ignite.internal.processors.cache.CrossCacheTxNearEnabledRandomOperationsTest;
@@ -144,6 +143,7 @@
 import org.apache.ignite.internal.processors.cache.local.GridCacheLocalTxTimeoutSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicyInitializationTest;
 import org.apache.ignite.internal.processors.continuous.IgniteNoCustomEventsOnNodeStart;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
 
 /**
  * Test suite.
@@ -154,6 +154,8 @@ public class IgniteCacheTestSuite2 extends TestSuite {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite() throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = new TestSuite("IgniteCache Test Suite part 2");
 
         // Local cache.
@@ -266,7 +268,6 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(CacheConfigurationLeakTest.class));
         suite.addTest(new TestSuite(MemoryPolicyConfigValidationTest.class));
         suite.addTest(new TestSuite(MemoryPolicyInitializationTest.class));
-        suite.addTest(new TestSuite(CacheDataRegionConfigurationTest.class));
         suite.addTest(new TestSuite(CacheGroupLocalConfigurationSelfTest.class));
         suite.addTest(new TestSuite(CacheEnumOperationsSingleNodeTest.class));
         suite.addTest(new TestSuite(CacheEnumOperationsTest.class));
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index b66bf5b5026fa..55436d6da943b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -53,11 +53,9 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingAsyncSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheDaemonNodeReplicatedSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicGetAndTransformStoreSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedBasicApiTest;
@@ -84,6 +82,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.replicated.preloader.GridCacheReplicatedPreloadStartStopEventsSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheDaemonNodeLocalSelfTest;
 import org.apache.ignite.internal.processors.cache.local.GridCacheLocalByteArrayValuesSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
 
 /**
  * Test suite.
@@ -94,6 +93,8 @@ public class IgniteCacheTestSuite3 extends TestSuite {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite() throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = new TestSuite("IgniteCache Test Suite part 3");
 
         suite.addTestSuite(IgniteCacheGroupsTest.class);
@@ -151,8 +152,6 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
         suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
         suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
-        suite.addTestSuite(GridCacheRebalancingPartitionCountersTest.class);
-        suite.addTestSuite(GridCacheRebalancingWithAsyncClearingTest.class);
 
         // Test for byte array value special case.
         suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index b973c9125bb21..321046be20271 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -80,7 +80,6 @@
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheMultinodeTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartCoordinatorFailoverTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailTest;
-import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailWithPersistenceTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartNoExchangeTimeoutTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartStopConcurrentTest;
@@ -151,6 +150,7 @@
 import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryPartitionedTransactionalSelfTest;
 import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryReplicatedAtomicSelfTest;
 import org.apache.ignite.internal.processors.cache.version.CacheVersionedEntryReplicatedTransactionalSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
 
 /**
  * Test suite.
@@ -161,6 +161,8 @@ public class IgniteCacheTestSuite4 extends TestSuite {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite() throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = new TestSuite("IgniteCache Test Suite part 4");
 
         // Multi node update.
@@ -229,7 +231,6 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteDynamicCacheStartSelfTest.class);
         suite.addTestSuite(IgniteDynamicCacheMultinodeTest.class);
         suite.addTestSuite(IgniteDynamicCacheStartFailTest.class);
-        suite.addTestSuite(IgniteDynamicCacheStartFailWithPersistenceTest.class);
         suite.addTestSuite(IgniteDynamicCacheStartCoordinatorFailoverTest.class);
         suite.addTestSuite(IgniteDynamicCacheWithConfigStartSelfTest.class);
         suite.addTestSuite(IgniteCacheDynamicStopSelfTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 7c41e49355513..41fca2a044383 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -35,14 +35,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteCachePutStackOverflowSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheReadThroughEvictionsVariationsSuite;
 import org.apache.ignite.internal.processors.cache.IgniteCacheStoreCollectionTest;
-import org.apache.ignite.internal.processors.cache.PartitionedAtomicCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.PartitionedTransactionalOptimisticCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.PartitionedTransactionalPessimisticCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.PartitionsExchangeOnDiscoveryHistoryOverflowTest;
-import org.apache.ignite.internal.processors.cache.ReplicatedAtomicCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalOptimisticCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalPessimisticCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentNodeJoinValidationTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheLateAffinityAssignmentTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheGroupsPartitionLossPolicySelfTest;
@@ -53,6 +46,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.CacheManualRebalancingTest;
 import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheSyncRebalanceModeSelfTest;
 import org.apache.ignite.internal.processors.cache.store.IgniteCacheWriteBehindNoUpdateSelfTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
 
 /**
  * Test suite.
@@ -63,6 +57,8 @@ public class IgniteCacheTestSuite5 extends TestSuite {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite() throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = new TestSuite("IgniteCache Test Suite part 5");
 
         suite.addTestSuite(CacheSerializableTransactionsTest.class);
@@ -105,8 +101,6 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(ConcurrentCacheStartTest.class);
 
-        suite.addTestSuite(Cache64kPartitionsTest.class);
-
         return suite;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 1ca35b22e3cba..63377ca6fbee8 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -18,11 +18,6 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
-import org.apache.ignite.internal.processors.authentication.AuthenticationConfigurationClusterTest;
-import org.apache.ignite.internal.processors.authentication.AuthenticationOnNotActiveClusterTest;
-import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNPEOnStartTest;
-import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorNodeRestartTest;
-import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorSelfTest;
 import org.apache.ignite.internal.processors.cache.PartitionedAtomicCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.PartitionedTransactionalOptimisticCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.PartitionedTransactionalPessimisticCacheGetsDistributionTest;
@@ -30,9 +25,6 @@
 import org.apache.ignite.internal.processors.cache.ReplicatedAtomicCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalOptimisticCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalPessimisticCacheGetsDistributionTest;
-import org.apache.ignite.internal.processors.cache.WalModeChangeAdvancedSelfTest;
-import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
-import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
 import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchangeLatchManagerCoordinatorFailTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest;
 import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest;
@@ -42,7 +34,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeMultiServerTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgnitePessimisticTxSuspendResumeTest;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
 import org.apache.ignite.internal.processors.cache.transactions.TransactionIntegrityWithPrimaryIndexCorruptionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxLabelTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxMultiCacheAsyncOpsTest;
@@ -51,12 +42,11 @@
 import org.apache.ignite.internal.processors.cache.transactions.TxOptimisticPrepareOnUnstableTopologyTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncNearCacheTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncTest;
-import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncWithPersistenceTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNearCacheTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTopologyChangeTest;
-import org.apache.ignite.internal.processors.cache.transactions.TxWithSmallTimeoutAndContentionOneKeyTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
 
 /**
  * Test suite.
@@ -67,6 +57,8 @@ public class IgniteCacheTestSuite6 extends TestSuite {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite() throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = new TestSuite("IgniteCache Test Suite part 6");
 
         suite.addTestSuite(CachePartitionStateTest.class);
@@ -81,11 +73,9 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TxRollbackOnTimeoutTest.class);
         suite.addTestSuite(TxRollbackOnTimeoutNoDeadlockDetectionTest.class);
         suite.addTestSuite(TxRollbackOnTimeoutNearCacheTest.class);
-        suite.addTestSuite(TxWithSmallTimeoutAndContentionOneKeyTest.class);
         suite.addTestSuite(IgniteCacheThreadLocalTxTest.class);
         suite.addTestSuite(TxRollbackAsyncTest.class);
         suite.addTestSuite(TxRollbackAsyncNearCacheTest.class);
-        suite.addTestSuite(TxRollbackAsyncWithPersistenceTest.class);
         suite.addTestSuite(TxRollbackOnTopologyChangeTest.class);
 
         suite.addTestSuite(TxOptimisticPrepareOnUnstableTopologyTest.class);
@@ -96,23 +86,11 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(TxOnCachesStartTest.class);
 
-        suite.addTestSuite(IgnitePdsCacheAssignmentNodeRestartsTest.class);
-
-        suite.addTestSuite(WalModeChangeSelfTest.class);
-        suite.addTestSuite(WalModeChangeCoordinatorNotAffinityNodeSelfTest.class);
-        suite.addTestSuite(WalModeChangeAdvancedSelfTest.class);
-
         suite.addTestSuite(IgniteCache150ClientsTest.class);
 
 //        TODO enable this test after IGNITE-6753, now it takes too long
 //        suite.addTestSuite(IgniteOutOfMemoryPropagationTest.class);
 
-        suite.addTestSuite(AuthenticationConfigurationClusterTest.class);
-        suite.addTestSuite(AuthenticationProcessorSelfTest.class);
-        suite.addTestSuite(AuthenticationOnNotActiveClusterTest.class);
-        suite.addTestSuite(AuthenticationProcessorNodeRestartTest.class);
-        suite.addTestSuite(AuthenticationProcessorNPEOnStartTest.class);
-
         suite.addTestSuite(ReplicatedAtomicCacheGetsDistributionTest.class);
         suite.addTestSuite(ReplicatedTransactionalOptimisticCacheGetsDistributionTest.class);
         suite.addTestSuite(ReplicatedTransactionalPessimisticCacheGetsDistributionTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index fa64c7653ab78..dddf8f1768b02 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.testsuites;
 
-import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.authentication.Authentication1kUsersNodeRestartTest;
 import org.apache.ignite.internal.processors.authentication.AuthenticationConfigurationClusterTest;
@@ -27,21 +26,24 @@
 import org.apache.ignite.internal.processors.authentication.AuthenticationProcessorSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheDataRegionConfigurationTest;
 import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMBeanTest;
+import org.apache.ignite.internal.processors.cache.CacheMetricsEnableRuntimeTest;
 import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheStartFailWithPersistenceTest;
 import org.apache.ignite.internal.processors.cache.WalModeChangeAdvancedSelfTest;
 import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
 import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheDataLossOnPartitionMoveTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingWithAsyncClearingTest;
 import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.CheckpointBufferDeadlockTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncWithPersistenceTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxWithSmallTimeoutAndContentionOneKeyTest;
 
+import java.util.Set;
+
 /**
  * Test suite.
  */
@@ -65,6 +67,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(CheckpointBufferDeadlockTest.class);
         suite.addTestSuite(IgniteCacheStartWithLoadTest.class);
 
+        suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class);
+
         suite.addTestSuite(AuthenticationConfigurationClusterTest.class);
         suite.addTestSuite(AuthenticationProcessorSelfTest.class);
         suite.addTestSuite(AuthenticationOnNotActiveClusterTest.class);
@@ -86,14 +90,13 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(TxRollbackAsyncWithPersistenceTest.class);
 
         suite.addTestSuite(CacheGroupMetricsMBeanTest.class);
+        suite.addTestSuite(CacheMetricsEnableRuntimeTest.class);
         suite.addTestSuite(PageEvictionMultinodeMixedRegionsTest.class);
 
         suite.addTestSuite(IgniteDynamicCacheStartFailWithPersistenceTest.class);
 
         suite.addTestSuite(TxWithSmallTimeoutAndContentionOneKeyTest.class);
 
-        suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class);
-
         return suite;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
index bab7099e3ebe5..14eb296424a47 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteComputeGridTestSuite.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
-import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest;
 import org.apache.ignite.internal.ClusterNodeMetricsSelfTest;
 import org.apache.ignite.internal.ClusterNodeMetricsUpdateTest;
 import org.apache.ignite.internal.GridAffinityNoCacheSelfTest;
@@ -125,7 +124,6 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridTaskInstanceExecutionSelfTest.class);
         suite.addTestSuite(ClusterNodeMetricsSelfTest.class);
         suite.addTestSuite(ClusterNodeMetricsUpdateTest.class);
-        suite.addTestSuite(ClusterBaselineNodesMetricsSelfTest.class);
         suite.addTestSuite(GridNonHistoryMetricsSelfTest.class);
         suite.addTestSuite(GridCancelledJobsMetricsSelfTest.class);
         suite.addTestSuite(GridCollisionJobsContextSelfTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index 48f10a9b2c22a..ca258dff43658 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.testsuites;
 
-import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.ComputeJobCancelWithServiceSelfTest;
 import org.apache.ignite.internal.GridCommunicationSelfTest;
@@ -76,6 +75,8 @@
 import org.apache.ignite.spi.communication.GridCacheMessageSelfTest;
 import org.apache.ignite.testframework.GridTestUtils;
 
+import java.util.Set;
+
 /**
  * Kernal self test suite.
  */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
index 2d7f64b50b74b..8262ae3cad750 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.testsuites;
 
-import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2ByteOrderSelfTest;
 import org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerEnumSelfTest;
@@ -34,6 +33,8 @@
 import org.apache.ignite.marshaller.jdk.GridJdkMarshallerSelfTest;
 import org.apache.ignite.testframework.GridTestUtils;
 
+import java.util.Set;
+
 /**
  * Test suite for all marshallers.
  */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index 791621ff598b0..309af8cd1193f 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.testsuites;
 
-import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.commandline.CommandHandlerParsingTest;
 import org.apache.ignite.internal.pagemem.impl.PageIdUtilsSelfTest;
@@ -41,9 +40,7 @@
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.thread.GridThreadPoolExecutorServiceSelfTest;
 import org.apache.ignite.thread.IgniteThreadPoolSizeTest;
-import org.apache.ignite.util.GridCommandHandlerTest;
 import org.apache.ignite.util.GridIntListSelfTest;
-import org.apache.ignite.util.GridInternalTaskUnusedWalSegmentsTest;
 import org.apache.ignite.util.GridLongListSelfTest;
 import org.apache.ignite.util.GridMessageCollectionTest;
 import org.apache.ignite.util.GridPartitionMapSelfTest;
@@ -55,6 +52,8 @@
 import org.apache.ignite.util.mbeans.GridMBeanSelfTest;
 import org.apache.ignite.util.mbeans.WorkersControlMXBeanTest;
 
+import java.util.Set;
+
 /**
  * Test suite for Ignite utility classes.
  */
@@ -118,8 +117,6 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
 
         // control.sh
         suite.addTestSuite(CommandHandlerParsingTest.class);
-        suite.addTestSuite(GridCommandHandlerTest.class);
-        suite.addTestSuite(GridInternalTaskUnusedWalSegmentsTest.class);
 
         return suite;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
index 148ecef37d3f9..a39b0c26fa723 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridInternalTaskUnusedWalSegmentsTest.java
@@ -17,12 +17,7 @@
 
 package org.apache.ignite.util;
 
-import java.io.File;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
@@ -40,6 +35,12 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_MAX_CHECKPOINT_MEMORY_HISTORY_SIZE;
 
 /**
@@ -101,15 +102,17 @@ public void testCorrectnessOfDeletionTaskSegments() throws Exception {
 
             ig0.cluster().active(true);
 
-            IgniteCache<Object, Object> cache = ig0.cache(DEFAULT_CACHE_NAME);
-
-            for (int k = 0; k < 10_000; k++)
-                cache.put(k, new byte[1024]);
+            try (IgniteDataStreamer streamer = ig0.dataStreamer(DEFAULT_CACHE_NAME)) {
+                for (int k = 0; k < 10_000; k++)
+                    streamer.addData(k, new byte[1024]);
+            }
 
             forceCheckpoint();
 
-            for (int k = 0; k < 1_000; k++)
-                cache.put(k, new byte[1024]);
+            try (IgniteDataStreamer streamer = ig0.dataStreamer(DEFAULT_CACHE_NAME)) {
+                for (int k = 0; k < 1_000; k++)
+                    streamer.addData(k, new byte[1024]);
+            }
 
             forceCheckpoint();
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
new file mode 100644
index 0000000000000..1bc60f65658c8
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.util.GridCommandHandlerIndexingTest;
+
+/**
+ * Cache tests using indexing.
+ */
+public class IgniteCacheWithIndexingAndPersistenceTestSuite extends TestSuite {
+    /**
+     * @return Test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        TestSuite suite = new TestSuite("Ignite Cache With Indexing And Persistence Test Suite");
+
+        suite.addTestSuite(GridCommandHandlerIndexingTest.class);
+
+        return suite;
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index c8cf92d758fc2..6a376a95217bf 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.internal.processors.cache.ttl.CacheTtlTransactionalPartitionedSelfTest;
 import org.apache.ignite.internal.processors.client.IgniteDataStreamerTest;
 import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelperTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
 import org.apache.ignite.util.GridCommandHandlerIndexingTest;
 
 /**
@@ -51,6 +52,8 @@ public class IgniteCacheWithIndexingTestSuite extends TestSuite {
      * @throws Exception Thrown in case of the failure.
      */
     public static TestSuite suite() throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
         TestSuite suite = new TestSuite("Ignite Cache With Indexing Test Suite");
 
         suite.addTestSuite(InlineIndexHelperTest.class);

From 8ff42a2f4b876d2a5b44153f0abb4757e305fe3e Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Wed, 12 Sep 2018 20:36:34 +0300
Subject: [PATCH 352/543] IGNITE-9100 fix incorrect conflict resolving -
 duplicate test invocations

---
 .../apache/ignite/testsuites/IgniteBasicTestSuite.java   | 5 +----
 .../apache/ignite/testsuites/IgniteCacheTestSuite6.java  | 3 ---
 .../apache/ignite/testsuites/IgniteCacheTestSuite7.java  | 9 +++++----
 .../ignite/testsuites/IgniteKernalSelfTestSuite.java     | 5 +----
 .../ignite/testsuites/IgniteMarshallerSelfTestSuite.java | 5 +----
 5 files changed, 8 insertions(+), 19 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
index f8839bf77f43d..cfdff85a89a44 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicTestSuite.java
@@ -17,13 +17,13 @@
 
 package org.apache.ignite.testsuites;
 
+import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.GridSuppressedExceptionSelfTest;
 import org.apache.ignite.failure.FailureHandlerTriggeredTest;
 import org.apache.ignite.failure.OomFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeFailureHandlerTest;
 import org.apache.ignite.failure.StopNodeOrHaltFailureHandlerTest;
-import org.apache.ignite.failure.SystemWorkersTerminationTest;
 import org.apache.ignite.internal.ClassSetTest;
 import org.apache.ignite.internal.ClusterGroupHostsSelfTest;
 import org.apache.ignite.internal.ClusterGroupSelfTest;
@@ -90,8 +90,6 @@
 import org.apache.ignite.util.AttributeNodeFilterSelfTest;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.Set;
-
 /**
  * Basic test suite.
  */
@@ -210,7 +208,6 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(StopNodeFailureHandlerTest.class);
         suite.addTestSuite(StopNodeOrHaltFailureHandlerTest.class);
         suite.addTestSuite(OomFailureHandlerTest.class);
-        suite.addTestSuite(SystemWorkersTerminationTest.class);
         suite.addTestSuite(TransactionIntegrityWithSystemWorkerDeathTest.class);
 
         suite.addTestSuite(CacheRebalanceConfigValidationTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 63377ca6fbee8..61f516773ab28 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -34,7 +34,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeMultiServerTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgnitePessimisticTxSuspendResumeTest;
-import org.apache.ignite.internal.processors.cache.transactions.TransactionIntegrityWithPrimaryIndexCorruptionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxLabelTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxMultiCacheAsyncOpsTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxOnCachesStartTest;
@@ -105,8 +104,6 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(PartitionsExchangeCoordinatorFailoverTest.class);
 
-        suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class);
-
         //suite.addTestSuite(CacheClientsConcurrentStartTest.class);
         //suite.addTestSuite(GridCacheRebalancingOrderingTest.class);
         //suite.addTestSuite(IgniteCacheClientMultiNodeUpdateTopologyLockTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index dddf8f1768b02..859aecd81c5ae 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.testsuites;
 
+import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.authentication.Authentication1kUsersNodeRestartTest;
 import org.apache.ignite.internal.processors.authentication.AuthenticationConfigurationClusterTest;
@@ -39,11 +40,10 @@
 import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionMultinodeMixedRegionsTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheAssignmentNodeRestartsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.CheckpointBufferDeadlockTest;
+import org.apache.ignite.internal.processors.cache.transactions.TransactionIntegrityWithPrimaryIndexCorruptionTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncWithPersistenceTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxWithSmallTimeoutAndContentionOneKeyTest;
 
-import java.util.Set;
-
 /**
  * Test suite.
  */
@@ -67,8 +67,6 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(CheckpointBufferDeadlockTest.class);
         suite.addTestSuite(IgniteCacheStartWithLoadTest.class);
 
-        suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class);
-
         suite.addTestSuite(AuthenticationConfigurationClusterTest.class);
         suite.addTestSuite(AuthenticationProcessorSelfTest.class);
         suite.addTestSuite(AuthenticationOnNotActiveClusterTest.class);
@@ -97,6 +95,9 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
 
         suite.addTestSuite(TxWithSmallTimeoutAndContentionOneKeyTest.class);
 
+        suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class);
+        suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class);
+
         return suite;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
index ca258dff43658..5306aebd7611a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteKernalSelfTestSuite.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.testsuites;
 
+import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.ComputeJobCancelWithServiceSelfTest;
 import org.apache.ignite.internal.GridCommunicationSelfTest;
@@ -68,15 +69,12 @@
 import org.apache.ignite.internal.processors.service.IgniteServiceDynamicCachesSelfTest;
 import org.apache.ignite.internal.processors.service.IgniteServiceProxyTimeoutInitializedTest;
 import org.apache.ignite.internal.processors.service.IgniteServiceReassignmentTest;
-import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest;
 import org.apache.ignite.internal.processors.service.ServicePredicateAccessCacheTest;
 import org.apache.ignite.internal.util.GridStartupWithUndefinedIgniteHomeSelfTest;
 import org.apache.ignite.services.ServiceThreadPoolSelfTest;
 import org.apache.ignite.spi.communication.GridCacheMessageSelfTest;
 import org.apache.ignite.testframework.GridTestUtils;
 
-import java.util.Set;
-
 /**
  * Kernal self test suite.
  */
@@ -150,7 +148,6 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(ServiceThreadPoolSelfTest.class);
         suite.addTestSuite(GridServiceProcessorBatchDeploySelfTest.class);
         suite.addTestSuite(GridServiceDeploymentCompoundFutureSelfTest.class);
-        suite.addTestSuite(ServiceDeploymentOutsideBaselineTest.class);
 
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingDefaultMarshallerTest.class);
         suite.addTestSuite(IgniteServiceDeploymentClassLoadingJdkMarshallerTest.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
index 8262ae3cad750..98a6bde7e717b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteMarshallerSelfTestSuite.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.testsuites;
 
+import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2ByteOrderSelfTest;
 import org.apache.ignite.internal.marshaller.optimized.OptimizedMarshallerEnumSelfTest;
@@ -29,12 +30,9 @@
 import org.apache.ignite.internal.util.GridHandleTableSelfTest;
 import org.apache.ignite.internal.util.io.GridUnsafeDataInputOutputByteOrderSelfTest;
 import org.apache.ignite.internal.util.io.GridUnsafeDataOutputArraySizingSelfTest;
-import org.apache.ignite.marshaller.GridMarshallerMappingConsistencyTest;
 import org.apache.ignite.marshaller.jdk.GridJdkMarshallerSelfTest;
 import org.apache.ignite.testframework.GridTestUtils;
 
-import java.util.Set;
-
 /**
  * Test suite for all marshallers.
  */
@@ -67,7 +65,6 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         GridTestUtils.addTestIfNeeded(suite, DirectByteBufferStreamImplV2ByteOrderSelfTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, GridHandleTableSelfTest.class, ignoredTests);
         GridTestUtils.addTestIfNeeded(suite, OptimizedMarshallerPooledSelfTest.class, ignoredTests);
-        GridTestUtils.addTestIfNeeded(suite, GridMarshallerMappingConsistencyTest.class, ignoredTests);
 
         return suite;
     }

From c3fc9964a28bd20770997ffe01ceb381f76f4169 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Tue, 21 Aug 2018 12:01:19 +0300
Subject: [PATCH 353/543] IGNITE-7339 Fixed partition eviction after restore
 from storage - Fixes #4138.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 570ee960a337695272cf6a18453f1b68a8d9182e)
---
 .../dht/GridDhtLocalPartition.java            |  35 +++-
 .../dht/GridDhtPartitionTopologyImpl.java     |  32 ++-
 .../CacheRentingStateRepairTest.java          | 190 ++++++++++++++++++
 .../junits/common/GridCommonAbstractTest.java | 113 +++++++++--
 .../testsuites/IgniteCacheTestSuite7.java     |   3 +
 5 files changed, 338 insertions(+), 35 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRentingStateRepairTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index a8adf52d04aeb..25be88fd35413 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -521,7 +521,30 @@ private void release0(int sizeChange) {
      * @param stateToRestore State to restore.
      */
     public void restoreState(GridDhtPartitionState stateToRestore) {
-        state.set(setPartState(state.get(),stateToRestore));
+        state.set(setPartState(state.get(), stateToRestore));
+    }
+
+    /**
+     * For testing purposes only.
+     * @param toState State to set.
+     */
+    public void setState(GridDhtPartitionState toState) {
+        if (grp.persistenceEnabled() && grp.walEnabled()) {
+            synchronized (this) {
+                long state0 = state.get();
+
+                this.state.compareAndSet(state0, setPartState(state0, toState));
+
+                try {
+                    ctx.wal().log(new PartitionMetaStateRecord(grp.groupId(), id, toState, updateCounter()));
+                }
+                catch (IgniteCheckedException e) {
+                    U.error(log, "Error while writing to log", e);
+                }
+            }
+        }
+        else
+            restoreState(toState);
     }
 
     /**
@@ -692,11 +715,13 @@ && getReservations(state) == 0 && !groupReserved()) {
     }
 
     /**
-     * Initiates single clear process if partition is in MOVING state.
+     * Initiates single clear process if partition is in MOVING state or continues cleaning for RENTING state.
      * Method does nothing if clear process is already running.
      */
     public void clearAsync() {
-        if (state() != MOVING)
+        GridDhtPartitionState state0 = state();
+
+        if (state0 != MOVING && state0 != RENTING)
             return;
 
         clear = true;
@@ -734,10 +759,8 @@ private boolean addEvicting() {
             if (cnt != 0)
                 return false;
 
-            if (evictGuard.compareAndSet(cnt, cnt + 1)) {
-
+            if (evictGuard.compareAndSet(cnt, cnt + 1))
                 return true;
-            }
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 78012dfe314c2..7e410a45430b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -654,8 +654,15 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
 
                 if (locPart == null)
                     updateLocal(p, EVICTED, updateSeq, topVer);
-                else
-                    updateLocal(p, locPart.state(), updateSeq, topVer);
+                else {
+                    GridDhtPartitionState state = locPart.state();
+
+                    updateLocal(p, state, updateSeq, topVer);
+
+                    // Restart cleaning.
+                    if (state == RENTING)
+                        locPart.clearAsync();
+                }
             }
         }
         finally {
@@ -1737,7 +1744,8 @@ else if (isStaleUpdate(cur, parts)) {
                     assert cur != null;
 
                     String msg = "Stale update for single partition map update (will ignore) [" +
-                        "grp=" + grp.cacheOrGroupName() +
+                        "nodeId=" + parts.nodeId() +
+                        ", grp=" + grp.cacheOrGroupName() +
                         ", exchId=" + exchId +
                         ", curMap=" + cur +
                         ", newMap=" + parts + ']';
@@ -2244,12 +2252,15 @@ private boolean checkEvictions(long updateSeq, AffinityAssignment aff) {
 
                     // If all affinity nodes are owners, then evict partition from local node.
                     if (nodeIds.containsAll(F.nodeIds(affNodes))) {
-                        IgniteInternalFuture<?> rentFuture = part.rent(false);
-                        rentingFutures.add(rentFuture);
+                        GridDhtPartitionState state0 = part.state();
+
+                        IgniteInternalFuture<?> rentFut = part.rent(false);
+
+                        rentingFutures.add(rentFut);
 
                         updateSeq = updateLocal(part.id(), part.state(), updateSeq, aff.topologyVersion());
 
-                        changed = true;
+                        changed = state0 != part.state();
 
                         if (log.isDebugEnabled()) {
                             log.debug("Evicted local partition (all affinity nodes are owners) [grp=" + grp.cacheOrGroupName() +
@@ -2270,12 +2281,15 @@ private boolean checkEvictions(long updateSeq, AffinityAssignment aff) {
                                 ClusterNode n = nodes.get(i);
 
                                 if (locId.equals(n.id())) {
-                                    IgniteInternalFuture<?> rentFuture = part.rent(false);
-                                    rentingFutures.add(rentFuture);
+                                    GridDhtPartitionState state0 = part.state();
+
+                                    IgniteInternalFuture<?> rentFut = part.rent(false);
+
+                                    rentingFutures.add(rentFut);
 
                                     updateSeq = updateLocal(part.id(), part.state(), updateSeq, aff.topologyVersion());
 
-                                    changed = true;
+                                    changed = state0 != part.state();
 
                                     if (log.isDebugEnabled()) {
                                         log.debug("Evicted local partition (this node is oldest non-affinity node) [" +
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRentingStateRepairTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRentingStateRepairTest.java
new file mode 100644
index 0000000000000..cba3c3cdac094
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRentingStateRepairTest.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
+/**
+ *
+ */
+public class CacheRentingStateRepairTest extends GridCommonAbstractTest {
+    /** */
+    public static final int PARTS = 1024;
+
+    /** */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, PARTS).setPartitions(64));
+
+        ccfg.setOnheapCacheEnabled(false);
+
+        ccfg.setBackups(1);
+
+        ccfg.setRebalanceBatchSize(100);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        cfg.setCacheConfiguration(ccfg);
+
+        cfg.setActiveOnStart(false);
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+        discoSpi.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(discoSpi);
+
+        long sz = 100 * 1024 * 1024;
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration().setPageSize(1024)
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration().setPersistenceEnabled(true).setInitialSize(sz).setMaxSize(sz))
+            .setWalMode(WALMode.LOG_ONLY).setCheckpointFrequency(24L * 60 * 60 * 1000);
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String getTestIgniteInstanceName(int idx) {
+        return "node" + idx;
+    }
+
+    /**
+     *
+     */
+    public void testRentingStateRepairAfterRestart() throws Exception {
+        try {
+            IgniteEx g0 = startGrid(0);
+
+            startGrid(1);
+
+            g0.cluster().active(true);
+
+            awaitPartitionMapExchange();
+
+            List<Integer> parts = evictingPartitionsAfterJoin(g0, g0.cache(DEFAULT_CACHE_NAME), 20);
+
+            int toEvictPart = parts.get(0);
+
+            int k = 0;
+
+            while (g0.affinity(DEFAULT_CACHE_NAME).partition(k) != toEvictPart)
+                k++;
+
+            g0.cache(DEFAULT_CACHE_NAME).put(k, k);
+
+            GridDhtPartitionTopology top = dht(g0.cache(DEFAULT_CACHE_NAME)).topology();
+
+            GridDhtLocalPartition part = top.localPartition(toEvictPart);
+
+            assertNotNull(part);
+
+            // Prevent eviction.
+            part.reserve();
+
+            startGrid(2);
+
+            g0.cluster().setBaselineTopology(3);
+
+            // Wait until all is evicted except first partition.
+            assertTrue("Failed to wait for partition eviction", waitForCondition(() -> {
+                for (int i = 1; i < parts.size(); i++) { // Skip reserved partition.
+                    Integer p = parts.get(i);
+
+                    if (top.localPartition(p).state() != GridDhtPartitionState.EVICTED)
+                        return false;
+                }
+
+                return true;
+            }, 5000));
+
+            /**
+             * Force renting state before node stop.
+             * This also could be achieved by stopping node just after RENTING state is set.
+             */
+            part.setState(GridDhtPartitionState.RENTING);
+
+            assertEquals(GridDhtPartitionState.RENTING, part.state());
+
+            stopGrid(0);
+
+            g0 = startGrid(0);
+
+            awaitPartitionMapExchange();
+
+            part = dht(g0.cache(DEFAULT_CACHE_NAME)).topology().localPartition(toEvictPart);
+
+            assertNotNull(part);
+
+            final GridDhtLocalPartition finalPart = part;
+
+            CountDownLatch clearLatch = new CountDownLatch(1);
+
+            part.onClearFinished(fut -> {
+                assertEquals(GridDhtPartitionState.EVICTED, finalPart.state());
+
+                clearLatch.countDown();
+            });
+
+            assertTrue("Failed to wait for partition eviction after restart",
+                clearLatch.await(5_000, TimeUnit.MILLISECONDS));
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        cleanPersistenceDir();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index e6984c4b8919e..428cf24c4a961 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -19,6 +19,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.BitSet;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
@@ -73,6 +74,7 @@
 import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
+import org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
@@ -97,6 +99,8 @@
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiInClosure;
+import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
@@ -117,6 +121,7 @@
 import static org.apache.ignite.cache.CacheMode.LOCAL;
 import static org.apache.ignite.cache.CacheRebalanceMode.NONE;
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled;
+import static org.apache.ignite.internal.processors.cache.GridCacheUtils.retryTopologySafe;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
 import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
@@ -1220,20 +1225,9 @@ protected final List<Integer> movingKeysAfterJoin(Ignite ign, String cacheName,
         @Nullable IgniteInClosure<ClusterNode> nodeInitializer) {
         assertEquals("Expected consistentId is set to node name", ign.name(), ign.cluster().localNode().consistentId());
 
-        GridCacheContext<Object, Object> cctx = ((IgniteKernal)ign).context().cache().internalCache(cacheName).context();
-
         ArrayList<ClusterNode> nodes = new ArrayList<>(ign.cluster().nodes());
 
-        AffinityFunction func = cctx.config().getAffinity();
-
-        AffinityFunctionContext ctx = new GridAffinityFunctionContextImpl(
-            nodes,
-            null,
-            null,
-            AffinityTopologyVersion.NONE,
-            cctx.config().getBackups());
-
-        List<List<ClusterNode>> calcAff = func.assignPartitions(ctx);
+        List<List<ClusterNode>> calcAff = calcAffinity(ign.cache(cacheName), nodes);
 
         GridTestNode fakeNode = new GridTestNode(UUID.randomUUID(), null);
 
@@ -1244,14 +1238,7 @@ protected final List<Integer> movingKeysAfterJoin(Ignite ign, String cacheName,
 
         nodes.add(fakeNode);
 
-        ctx = new GridAffinityFunctionContextImpl(
-            nodes,
-            null,
-            null,
-            AffinityTopologyVersion.NONE,
-            cctx.config().getBackups());
-
-        List<List<ClusterNode>> calcAff2 = func.assignPartitions(ctx);
+        List<List<ClusterNode>> calcAff2 = calcAffinity(ign.cache(cacheName), nodes);
 
         Set<Integer> movedParts = new HashSet<>();
 
@@ -1282,6 +1269,92 @@ protected final List<Integer> movingKeysAfterJoin(Ignite ign, String cacheName,
         return keys;
     }
 
+    /**
+     * Returns list of partitions what will be evicted after new node's join.
+     *
+     * @param ign Node to find evicting partitions.
+     * @param cache Cach.
+     * @param size Size.
+     * @return List of moving partition
+     */
+    protected List<Integer> evictingPartitionsAfterJoin(Ignite ign, IgniteCache<?, ?> cache, int size) {
+        ArrayList<ClusterNode> nodes = new ArrayList<>(ign.cluster().nodes());
+
+        List<List<ClusterNode>> ideal1 = calcAffinity(cache, nodes);
+
+        GridTestNode fakeNode = new GridTestNode(UUID.randomUUID(), null);
+
+        fakeNode.consistentId(getTestIgniteInstanceName(nodes.size()));
+
+        nodes.add(fakeNode);
+
+        List<List<ClusterNode>> ideal2 = calcAffinity(cache, nodes);
+
+        Map<ClusterNode, BitSet> m1 = U.newHashMap(nodes.size());
+        Map<ClusterNode, BitSet> m2 = U.newHashMap(nodes.size());
+
+        int parts = cache.getConfiguration(CacheConfiguration.class).getAffinity().partitions();
+
+        for (int p = 0; p < parts; p++) {
+            List<ClusterNode> assign1 = new ArrayList<>(ideal1.get(p));
+            List<ClusterNode> assign2 = new ArrayList<>(ideal2.get(p));
+
+            final int finalP = p;
+
+            IgniteBiInClosure<Map<ClusterNode, BitSet>, ClusterNode> updater = (map, node) -> {
+                BitSet set = map.get(node);
+
+                if (set == null)
+                    map.put(node, (set = new BitSet(parts)));
+
+                set.set(finalP);
+            };
+
+            for (ClusterNode node : assign1)
+                updater.apply(m1, node);
+
+            for (ClusterNode node : assign2)
+                updater.apply(m2, node);
+        }
+
+        List<Integer> partsToRet = new ArrayList<>(size);
+
+        BitSet before = m1.get(ign.cluster().localNode());
+        BitSet after = m2.get(ign.cluster().localNode());
+
+        for (int p = before.nextSetBit(0); p >= 0; p = before.nextSetBit(p+1)) {
+            if (!after.get(p)) {
+                partsToRet.add(p);
+
+                if (partsToRet.size() == size)
+                    break;
+            }
+        }
+
+        return partsToRet;
+    }
+
+    /**
+     * @param cache Cache.
+     * @param nodes Nodes.
+     */
+    private List<List<ClusterNode>> calcAffinity(IgniteCache<?, ?> cache, List<ClusterNode> nodes) {
+        IgniteCacheProxyImpl proxy = cache.unwrap(IgniteCacheProxyImpl.class);
+
+        GridCacheContext<?, ?> cctx = proxy.context();
+
+        AffinityFunction func = cctx.config().getAffinity();
+
+        AffinityFunctionContext ctx = new GridAffinityFunctionContextImpl(
+            nodes,
+            null,
+            null,
+            AffinityTopologyVersion.NONE,
+            cctx.config().getBackups());
+
+        return func.assignPartitions(ctx);
+    }
+
     /**
      * @param cache Cache.
      * @return Collection of keys for which given cache is primary.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index 859aecd81c5ae..471f8228034d8 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
 import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheDataLossOnPartitionMoveTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
@@ -95,6 +96,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
 
         suite.addTestSuite(TxWithSmallTimeoutAndContentionOneKeyTest.class);
 
+        suite.addTestSuite(CacheRentingStateRepairTest.class);
+
         suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class);
         suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class);
 

From 31302215fff2d3b2b6a8faf25557d88703cd78d8 Mon Sep 17 00:00:00 2001
From: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Date: Sat, 25 Aug 2018 00:35:55 +0300
Subject: [PATCH 354/543] IGNITE-9374 Replace expired 1-yr certs with new 3-yr
 ones. - Fixes #4617.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
(cherry picked from commit b0175cf644904be66efcc7653b7b193a3fdd3919)
---
 .../clients/src/test/keystore/ca/node01.jks   | Bin 3719 -> 3719 bytes
 .../clients/src/test/keystore/ca/node02.jks   | Bin 4598 -> 4599 bytes
 .../clients/src/test/keystore/ca/node03.jks   | Bin 3754 -> 3753 bytes
 .../clients/src/test/keystore/ca/oneindex.txt |   2 +-
 .../clients/src/test/keystore/ca/twoindex.txt |   4 ++--
 5 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/clients/src/test/keystore/ca/node01.jks b/modules/clients/src/test/keystore/ca/node01.jks
index 23c0643dd24b917efff6b0a8084990b71162f206..7dec684768fe7e20bc945c45a8d7033215b1ee07 100644
GIT binary patch
delta 1887
zcmV-l2cY<e9fuu|7X@W(q7a<196AF9Wo)7}E|XgW9DnV-B%Qh7(ZKIz*ZBb(F^h^#
zGmDs6>nTTv1az>+hKQkMMEc+nrIZi}K@;BqOl7{QwsRqUod#Ury7&8MGtDc|aR%%+
zKwvU^tPH&*MvNX!`en*1q4Ue~mF+Ht!z9bAd>9CHX#cC7r+Y?>(i;Q}_m(6%Uz~iX
zI9A@hW`E5tu<4CZ#X$m*e3!~`#72lt+=%xsDM*uPkNV8c?6YoJC!Eey7uZdFfLx)E
ztVZilPb#t7?2!jya2w2eBc9aydXikeJ&gmyVR|0${JjJHkaj(*I03X?!8QDe$Z&0Q
z3W>D3d3jZ$Z|hhcX!`G!pDRnq0w2x;?N$ckv44G+%@9K{5t0KbVqM=ken!@!>3W&b
zln|#a2)@M=f>xcOUyQG8%h0IKIx_%(mh`Ie=;0{JS{)ly52>w7^|kK4u8Fwc1}Lwr
z3Np^DF#dGtYoX!Uu6BoRb$pyw&XVESv`R*`*NA~j9~|e*X|_?V5iYf%zcAi-aWWfj
zl7GUZgP5#My8NrY9Ztmc-D5Od_ki*WcSFKAYWTEfgBsGeC7V^A$5N-fKRG~exJH5y
zk)=(mpu@POHOXQ9-a=2L0*zi2D(U;9&An?-GP$mw@Fg48we;ec^~z7z-8`90gwuzP
z6@Rui_Pc!}R?(VKM&<sCKSdLJ1w-%RSbyf(;~C!+wanb&4I$!zjs$|r4jd<lS;)od
zq)E=y5O7Rn<ZGo&nf$wm=8d&)v%aw^KpUr3Nb;_Fq6tl+>Zb)O2ZSF|LZpqNoi<x~
zj6+lVJvI<&EcuQruKYFEU&70Y^6;3sRlg398=b=@bT;a6kW2LW8Xu%*3-f_sCVv&u
za=DBJ73+w{BQYH>2}>7BERHekHwJp)5r;hzq8uSnqM&|w2Z^%B0ZWdzUyWgLIbl61
zXbFF01A<$j<o(i>1t*Ct7r=DMaSfu(tL~4c1!vb>13_CQL0Y}+t+7Sv=pA0N?Y`^z
z*%+frnWPMxvc3{%nNJm2wYE1EeSbKxhXv#-hxE$av1Y<eB{l}|R(ab&Ig0<ox)<ID
z7w^YhndAOmP1+??--*0|0fe|XD(h>&9Usr<A+$Xk@p&@#tc;5C>0$N&7T>w=hemoj
zt;|mBFiMBO(F~vCHoit7t~_Iqlv!}J{5~wgHT%@*>k%ReGDKM1FhNu(WPi~NmRCP=
z1(<L*a+NH3!U36uZk4Xx{$oZK7-ao+>VUC&ws5U1KUAsZ1*$X1-`4{iA+YAK#v=fq
z;*s-6L7}fv{e{8DP&62g+bJ^DYeRn(gKBFR2!!eNXUYs91E2#%i6u~)Da>sC7*DF*
z-_Q{Ibi6kMZfIZx(dl)|R(}Iveh!Yr(n@d&q10kXsH*_geDMy@K?G)~A=M*jWeAvI
z)I%F%`G`^%Tz_Hy9n+KNpYwMv=U*as-Jes<Q}S7RNtvRj!lx&~vwc!;P{eZ<1q8e_
zhSV)`%_NqZ>;{DKPok~^%!nlW2I-A9CWQad<q!XBju;S%pG12Tm4Dd70vZ}!GU!%q
z>zF_^V)8MV7Q}HN{mY-_*!xPJMm3n(W9Y>FnLu0s*tkf9`ZzjW)qPOX2Igm5iGXLX
zinq5L76M@J^-NS8gx58Z`)6n&7TPtJ{|cJwc&Nk{FC|>*BAxsmvCf$~1wY6!;7(>M
z0000200mesH842<0F!M9a~U`=I5IRbH8n6aFj^N4GBGeXGBYtXH83=j`3N?Dq-mAy
zUU4|FF_~247vfdC*=*QPF7VF<zKsj&=8j%eqj&yJj@0qFu#z?}f=DAm!Xs#|A#nb*
zaDXff6RdEr%?}VxF`EKcmYiOEQ6V-Z&ldg~jxapwi>A-Fa5!F&tu$Glm!Xbdh(Zk#
za>NA)KO4ola-3mDdT|KtWoHO~v6!4qq&Z4)-IZRm26Dy#(u|DgpEm59zVY4IBt8g?
zo7>yB6I%W}^l--12>6fi`$(IQ>|+pQpQW4C<t-R`98EhJZ`~_v0OiRtH1z4PcNHCz
z9vFP?Gkr1zh7k33gUH&^R#F-MVNQ;r_UB(bwjannY;OA9sAI)c%G{0uoV}AT3K@UT
zXPg4n1a=#Kx4Szj4fbtVk2-g`uYO-wJ8rbksyI|~eT5ZZ``{lvn<%~yNn+cy^C=u3
zOa4uDBu(Bm=MDG0*pMfz#WeP&G0*aPYnDZgF3@ewyIfw9k9CCQWw|c#>XG7}%mfS)
zIF0BvGu$m+%^pyCAd%E`KUN33$vuDkrGi^cGiukiOY=XT3RAz7*he<n-S<CgaWInJ
z7wwuHV2%yK;z9ZQ45~=!Q9Ntj=s*L6d@KZd)--#ffyN3yfP_Hm3Yu{T3kcH!@p5b;
zorO<SH39z?Wg9f8<(1vp_PeBCJHr>{KGZP>)@M7BxLq6`Da<E{#7Nf3pdA8SNwFL&
Z0~8^cnbI6tE^6}CH)kCtS{1Aj0#hZ_ixvO?

delta 1883
zcmV-h2c-Ci9fuu|7X@Au+LvRo96AF9UK84%Ba>SL9DjZQvmsm%A$&Hj;kcw9IageU
zNpYTC-`5$zMboBB3n9>)2M_6P5w_@Ssx`ta#<nh~u(KWIIYwCOGtaBc-)TDLAw{3f
zZL_5FD+NycPJ}r#ZQ4r<F*_F9eQS>zwV>nWcB<wGwbNXg4^r>;1#0_)tfxx|_ulga
zT}^m!R)29_^O<edg+~<$FXV)Lc^Q<bMgPM=qH+=;%p`DUXLrw+MVXJtRy#eF)j-Y?
zFN}5X$s4rR^R)16Ua?)60pIe8vq#70>d!B;C=Mf0z#<?`p}bxkOs1X7^sT75!5Yy*
zl5c<9Pru`Kb5Nw}k<Pd4cDINIK3?jzutz^vA%A^QNx08ebpHZu+HLzaRUif)oz&n*
zS?yV-ktycszI9Y5Ra3&1x7w9cy*|>1S$Jeduv^LpsWL!SW)YoMIFti*;9{1Ro?2fc
zwEu2<AANb$ybgCn?#$HlJ1A1^P#RFsAqR~?5(%lGofo1^A|H+(R1trAifT44N;6Xa
z_kT&w_1nDRhvj#Mx1J1xgNTRUS6+@vvO7fJ`aH}j^I0Mvdl2JCadK0@R6Hrp5G(97
zD-ylVnRr(>MtW(c@?_J?y`?Un+3n309m|5yta2K0fY4NnA3Cd>=S;b*u5ymG^yTs^
z#6XG@Sokdw!q6&Jx^r=xe=smztsj6|(0?tN3KEfK`KUS|pVZG=7D-N`w#aj~?~2&p
z7EiSGS@=z>{eOt4k{%-Vh9RE07K?-7Z%-uHbsxx=EUc>{cd7EDW5CFi8|1^07!9Bb
znZcu^PZXT1<t@dV2(`7w9hSyUlpY5R=!1Ie3$#8b4S)6__<hv~d^#OCgAG5mBY!&a
z*3lc)O^C5P@+tP5O5CciK0E=$7@v0849cC&8iBSCz$Do4Bxz_gEKe|=c5$KK3Kh#j
zZ}89dj7<5_zv4@?<SkghLH+j<T`S_RRBuG{YN&EKte`LH2#K;lJn}Yp9TjKuyWHu}
z8=<qyjRIpGVc4XHVK69xR3NqrZGUC`gYr{z-;VL_70$Fi1Fz$Dq|C@*pnJW`of-`;
zPKWoo`4gFr<@8n#R|O%SDV6xuaw$9f=5G-$fhWj*+C}DB)RX?(RyedN*Gg6Vw{LWP
zs;j9_4Pgq(@MPX`@ZQ=LXqweM@p|@VY<2zv4)^yF!#w#IdC^665h{TX2Y<z8;BwDl
zA%$puo-=g~YWfg#_0Y1u&<2Ml)<0Gc@f}Xx{58=*ZG9<x3e}d)^a;73s7}1+g6lGi
zpK8k#;;X51i&;3K4h^a3>T+S{^8%vc|M24K3?fbYyChoctA9nLt8=niK!B1G2yfS6
z^onGu3|!@+TSS|SL#pzant!W!XXWr~P38F#Y&y;#?iCNco(2SFd+W6me{Nhw#G<Yb
zSwR_lA8hu76NjR=K5_bSO%+rnOJ_Iuj9KVzaUkIqDb)U6q89N!!T6qGX628rVX&tI
zeRU97Ccm(wmb_{S7>-=M8i@2=EL+i~n&)&&6X)8nuh1ZIh$>fILVx=W!0U%1ud?Zn
zD4gzeq^(`e)}AqnZpFfy{NOYqV0mdslosx>MHp$uV^a8M92-O8f#_5-W1ArCpg?e%
z9?KGAa*zCo>4srgEVhoxA>^_<wZ+mCbNnXl3E`Iw;HKT~z@-L`OmaA<F-uc9?T<0Y
z0000200mesH842<0F!M9bCIA25;Qk4Fj^N4F*q<dGBhzTG&eDm`3N?Dzsh1EC5##n
zdL?hGTO{O;W%2MLMZ|q;YLWK*Ov=C);W{mZR<PjL`a5R`;_^24Dh9B}wMpZb`)!)q
zbcZrry5@1({X%j75lAh2a;ijFA!TqczC&%gwE@HmsYfdFpDRf&c=5vTfnz$RP{QD+
zY=1m-Qpb)O%IFHOFRl=OS4n0R?!rC{WW@l6+TO9s)YDZx>4`Y9F#NQP!O6s2dDb2G
zgbs}Ndg);Z%M-^7S(iy`rr?A@-VQt`z`?UlDd?fJCV14nEKV2Aa5Z&iqT!DH70Lj*
za`E9#uM><D?t2_KRwIJIZ!ZX2AW-c0y2A!_4^dEV0n>Es$C-ozUvZN!3K@UQ`^a{#
zq-zWePv4UDj0`WS*u}9T1Dx{f3If$1JDqCR&Z0N?2tqVG@ArvR=CWoGsWdXDpp=(0
z;$aN#PX_RKAhgW0^>O9i$*+OH$P+oad6{Q{n}Fm1^DOy2Hgn{V;%t~X8TXCLW2YAI
zQggyrWd>oQ)%rFJ%|SjIbI5<v0ZRdS181@0o)s9mzg-v5M0za&((=HM%YTi$*P>_J
zXXT--JM_g1<)?!N>iyR(0@k2Zh9;Pe_Tbzd-s~T$VEZn83q>Lj0y%eUHE88hzv+(3
z(mQuy-Iw&&w9fne2&aK=Gj(NltQ8Yb6YdPWd40bDPuvx4WS!?5;;;f&#jzYK0~Dn(
Vr?pHQgXzMM4G3-!X-KF!KrOCNo)rK9

diff --git a/modules/clients/src/test/keystore/ca/node02.jks b/modules/clients/src/test/keystore/ca/node02.jks
index 26da4b5adf49c1dc65e5c1e52b47fd2a2b98d9e4..985abae4b26f7e60092e90a9e42180e9856596d2 100644
GIT binary patch
delta 1927
zcmV;22YC4QBljba7X@W(rd5%%X#+e1lS%^`1!Zifgq*P)E(3o700wSvWMwci0004H
zY^Ijt000F6FoFdAFb)O^D+U1s0V)C!0RaU71cC(V`Kx`IT4$1Bp!<unpd<34=4e00
z$GWJ({L}xA$*c=6n746OF)eT*AhzT^P}H*}y@mWXQJZ8ZBpnblb1WT*7QCn2(ty%1
z=bp2yC%lB`(g%O6#M!uZ5pNu+x{D|L#wg&x{h!c<H%4#adb)&tZ}{|Jo4JEwYK6lJ
zMURx`zLb<-JFCX)PC{gAB->P}7pHvS>NI*^mQsi`t&AAH;`_GYKAH)|^t%?jEFTqa
zb;&?k`Y=n{=U8ng5lwYBkFr#C$`p588UzP|L^Vu7+BAP*Su6^wzkBI6Um1aO+i#1`
zG(rwVK&#a>WjAIJktf>^g?2B|BUEIQ^~*`Lo~U9qBi_a~?|@r)C&P>Gu^t~3r*z{L
z!TNxt08J67p^`&u_#|m}1wcx90S1u<+C66RouHY-vLrX!k?jew)VE7;MU*G@hup@l
z)ZOBcd)0ruAQ*@;q<_=7GH&tI4GliKZu;an0Ulx|i*zK>Fk{WoGdvL#0Cei-4zcWk
zYkS65EyYy=z|pOEoT<?R+s}$RufUm2*#kO@BHp%Sak-ibL|`AZj#n?cy+B}xG6;#?
zj?3lOQ$B_>FB10nu9L`o`j;RRIINE!+z9>3kUoDv;n=ytxE;OQ>F?j&YY%`m3*QL@
zGkWXlrlqflen9S;DU_e9DypeimTH#s=n7p_1i*CKmL~<TZ&MNB37Q!IE=eL4?78v;
z>t)-MOT!-HyR<T2D>vGebf9N-b@HBP_C+L3jqZi7vhmRv)oU7o%U*z%EH4wwL!-61
zt7Ly#tKzMbEN#=CU|DwzEzFt95d1TV6-I*3D43_^X8jWb?)&W}wJQsn-|zoMf*ZFR
z3KWWJIYh3<l~(pu0rJYZ17;n$*L+@=Jv;a4H)npem$yTfuJ7FTPk_ZiCTF66zo6!m
zLRjZNOz}omKL&cklxNDIgQ4uP6*xgjUyFYeqDk>DOV_c5mHt5NaPl^a*8zOd=~L_S
z6g+Yw>&VqG3qO)$PC_352sfK{7|zA|<T|VA)iCzu@Z#A0!nzFM@D-{_{hzJ->>lJz
zK-}BTJe=k27HN+!uje4eS60;D3i1muqZw<k!sm{nx{@LS=)TIY@nb!?)#iS3<wbuZ
z%&m>H>C8f*s8*5J7n$^z$QzA|sndhi{E)uCT4;c0ylnwl>B~?p>=-9uyM_JAeir1Z
zwz0AKJl@rnF=4G3BZAkxXx5#=E!qB-jHS{+GLxO}Iy;}dKqi|D2~rkDLJf^qn6>kx
zU$ctAniLtN(#~L+8SP0RcJQxM<D-AF8fi$}mD{ZJy8>aTlVK6%>tjcslquFnZe{Ue
z;RVlhnC|D&ql(5QO&A6`obqIXN*ijRj~YFVjdt4yJthQ#LeG`acgA3`(E|?4o-;IF
zjibi6Ai9AdFDvg)2WS++U@W3|eB`PWw;KRC+R@9+>)fWN`<ZI3E$Q58_OpL=BIz~g
z#NA6WO4Wt58-IgiX#>Xc#YTiXd-oABZjQc1m1^<)JTtpMvE6L5xq*S4h?u1RvZyMU
zxf`52r!V)Jz|DJSmA7If^qc^*V7ro?YD|CdQ<OzKx+>P?Tux`?hI9B`&Yki0o*K3&
zW=F`{R-dfh_iUoFh8cvlHrya*+;{=AwedC_b+Y;W_WEuODwJ4U^?KK8M6TY?Ub$M6
zx(kOII50ReG%+<bH8e3=7Y#BoFgP+ZF*P+cG%=G~3^jkC;F(eYk`6jO3yWA%tm37U
z_&k2y9A>uu5YpFUEyJysJEMwGhVF$j7g?Hf`{wLLqzveLp5=3Q1Lio?sG1!UMfP!^
zCXZwng6)BK4xSyIu~CGef>~Cx6=Iz@QJG?_C(O{L_cr*299DQdO4}VsZ^{lrwchWG
zon8^M4OD-fU<@3ev}@!@4IjLV$?Gyb*8~_q5SP++XQ?a!bi|!!cF}62niu<tolG8D
zb`=M9Ly|QYdYt&>+(+XiK&!+qtX1HIBH}H^*^PgYxs@5`#|5jPrchYPjA+D<Gnvn?
zeGa5ob%z-wz@A3JMgOA^GRh9&b)9kZnetf`-HZY!wUdwy8GoZj2L%>+(H6bKgl#k;
zJ$NSrdyx^3pHYg%wyQL-^v{v0LnXLBU0Rhvgu2RRUjMW)fWZoMobKP85^FiipPKW>
zZX!0kYj!L?mF}uVlB*8VSpOeHw~cs{6@^)`Ir#FLjf23Q)av})WEKJ_a#wp?G>pZ7
zvY!~N6BgQg$$#Jr+PTGn_;#OoC0rZu=QLT;hkQVRz5k4QG47j0b(Oq2jzW&B<3~=U
zHUWCj1q($qvh@AS!l6OT7?+&YuHn=QmB^<cO@63~mX@Ms*$(`tX$%g6%ejoq2k96*
zkoQi5+Bsw)e>3l`;8B0yBM13x`J)B-`^n8I{O$YbyaHX|vuOi40~AZ&Wk&f00O{lA
N{x()*F5+8|82iB7ymSBn

delta 1947
zcmV;M2W0s7BlaVZ7X@Au+O0IRcLOd19|8arV`Xx5X=Z6-VRU6-b#!QNa%psV0004A
z6WYH_u^c=De*pjnZf|5|FfsrD0bUc@$Hf2u1pqLD1pF`#1_~<%0R#am0uccL1pows
z1n46%u;e$39`zpGW=&8pDz78Oa*SCzERQ_qWAmmQW^U!gRld`$<VP|yS(wQBV5gR{
zEDXvm(dq#`c+~>II_l<C8>HpSYR3T2Fp)$oR;&2~e|h7Ky1X`^1&Bs?eyk61P>9ao
zQ_XQ^dBE&$lU`kFVeLVRII+W%CMO`{cux){;lyKjB<R!GmhO;(F{C9-{ouyzSo2p_
zE)a@103xR1S^?9bJ0N1!HEp48Nr5X-+%WEx76&vhdJ454rUX{wRAf_#X4C>fSM0VP
zTgBjdfBQ05oP?nV_g*8-!5l{?D?3o6`25*qy_zWihNE9_{^2qSmp$4&=m$BAyt^rN
z34C%AeB56T5vsv#&Qq3#muey21UN#ZhmbwZ4^ipR!UYetaqN^3pA0ko=Uab4)qMYD
zQ*+J<#OvX!X=1hcETMH?kO$3dDEo{I-XvG2f6`H$wvTGSFRZFB8Hob+4ycfw=CyB(
zH`uh1MwhLBtpG<I#Ej1Ub=w512i<48)R;s7zj*ipf#?RZ+!9vvgxDJ>;he=M%DR&H
za2`H?kPmWi-Mb>8mhJv2L4Us3h`C-PJ$Z>^fOc*J7T?%?E_(m?rWYViE<e4;gv)TI
ze>Yqn0Kb^XLvT!vpA)?j{zqHGD8sJKF*w(*=`BXLFiyZz@7$B}DRDAT`KR7UUIL*6
zNKYSy!<ujZ#Pq=g9Gk*z3|vD{K@)rd_}@lg{W}WuzN+wAi^PoiDH%Y!T;^4yW8<Xf
zO2-POhXYd))q&NZJ<-{+;t*LTjJVdmf5&BQSm0h{l?)yOvu>(uNjPq%z<XHUh-YSA
zuh~VkfjLsU5i)RWf{W=kjJp-3wg0(#@<B)@@KxBt95H|U(D(~)FP&3}Urm?ZdmoAl
zs)%`5oq6}rpMBd!aD=ZSV$l=CY2*3d8kFiP>n$VqB>@g0eL7l+hYxM;75qs$f28$R
zaosj)YD;&~a7n_p5dH48W{n}bmJ5sZYtOKP-Qc~SHK+zF<M&7BdondTd%2ifGRhH-
z^%$*|SUdCJAeT`MiJh~Ys-*lbyJ$0wr{2b?c@nBa!ht;jiLsZnRN>FPKi%Un8Cpet
zKuUlTLNcQj31+Zyp;4jd<?ppXe~zwPri9?pLz-ru8rW%P>X(X5bD!{q?!kmMuE*P8
zeF$E^&4?r#>Z|BCEST^tp+aol?_AGK8CZ~@eAG$8@jIqu1?|HrXiVQaYA1h4UP8(S
z?TvdN316?gUL)`pA?dRTT?O=t&wzLBKcQbWEAVd~D{+E#(KMC+lB7&)f63ZT%s`n%
z8{76W-5b4V2Ew!^T^VZ1ZAiU%cqutpaIiiW*MYwTN}RoAx7WuLoKyZ~QnZPDJDw&R
zk@agjJ_mn20&DQBfr<ult=?r3bk4|j7ScQ*bs@;<kymnx5rP!u&3wdB)heuhn9OLZ
z6xVY_5>+xf$<f>vWCE)$e+ZncHo>snI>=#B__NQ6zL}*`<ji!cc*!!r+vpm^lJphE
z%qHXs^WDbZdkgflwi8?NUMdHc>K1*26(SWZKw0GWvzSkOaW%W>&$ueVgaS3@8%3%g
zvKpDdHNM&7O9NmW0K@f0(?<0{JJ*Y_XPXZHA5`7r-)XG(Z)i579}9>U8KcxEMPuLx
zDt@2nX@TQ8<*+Xep=Li4B;M&jle-It8aFUFGBhzTG&eCeS{Ds5I50ReG%+wVH!(Jo
zTnsgTi>*MQjR&H2#^TOv|K`!8Vy)r*e8AaGY{xoC_anSSkwc{Tfn`Ew)U;@PC@-@&
z#7);ifc81TB}9oJtxR`3fZ1F}p*8#WxWFJGG0au(4aqY>t)w9eA1qksyXR4uEO4&`
z5{}fVV9`ofsRfN7uH02%zMXzL+O{HXhzQw#5>c--5l-yN+^#zo`w0>~ZN~?KDC;<1
ze<E4Dr3j-2h&Zq#VtmAhD?R1qAA!4l6JKwHi6<(A-@68aCdY+^tf(FpjGQI>UBBt!
z!?O3S3yZ??KFEP6nIMq#c-Se6TPT5O9yM;2!`F1!%01ad^KDoWX^^ka%j-*}6{rgW
z249nr4H<ufRSU4~|2e3`5|$1a-7e@XG})q99TvV&2$R+@4dU*fGjI0Za*Aw!wJx7!
zvqG&mqbj7&3-u}2j0hpre9b{%DmVHCLpgE?6MDAYbuLR3V5+hf7U54KD<jP(m85iU
z_g1OJXKv|zwXbao73F+?JyjPq19?~|v&iN>ljwiO!n39T37(pR1@cI|2hYm0FFlXk
zfo7yE>4qcyX{P~;__DrEfP;?IXi%_Ht%455*NP0rOnKTe2f{SGWV$4!Vk$aOU)JwU
za^XCE%#|HKZ@wp$7mF<skQ-~p+d`ze6vlAFYIS#cedX$l58kVnI61FAYh3eDR@L3M
h2!;Y|m$4i<0~C}MCsn`PO(cS2aYxvH$c%;n0q5Wpo4f!3

diff --git a/modules/clients/src/test/keystore/ca/node03.jks b/modules/clients/src/test/keystore/ca/node03.jks
index 831ca2400b14bee102a504b9d9cfb478c9a278cd..9a6ab40693d937743b451389159ef59037111c48 100644
GIT binary patch
delta 1922
zcmV-|2YvXe9jP6V7X@W(sVF_OT?;t_lT`x`4P|VpYh3^U1pqLD1pJe60~3GfrBR^m
zR_Q#Bk!+GZC=mmONkN=g2nRf{jg`vr)qXUwJm^NN!I*c3hMCg%0{K`Ui+RtTI_DtV
zgnYeNN}4js2-<Q-n|#!k-j7#ZNv4zba*HRBqU5$J4^@$cly&Xzz-l+?ObI_sLW)^G
zFs|t93;Hy#PfD4p%L>4*4Bvm9FOACq90_OErjnw-1|3fv@dLf0`J7HpSc>a>C>fyb
z)VFYW^od}nX=3{5aB(s1OEH~Ok@kzMF@meCe9>y2%Un=v(ytzL+fD4$!5_{qAl5{T
zn%7cRv&lgntAVGfM&Qp4v&Mr=<7Bl<G>L`!Z%;zo4x&p+e+@Y8q?>>K1g#p+!=NcR
z9@!p1<ZRVE&hr_^yfh$nylyiH2)?v`jP03@apd)U(OYeeJ5(t_2r3%Q&0h_Iz6J#_
z#T;r=DbWv4UZnzNVZC1Q!#7_CDnA^BraTj+RMJ#`N5Bz&inKEx|B}1XjX|qNq2#FN
zdCMh7nqM+$);2rB?!SNY5SJ-DLrk|nU*KrqU_ct4p~7{xa0;{O4_tZ`bzTC)m5n~+
zlR8b<AHIjs&RG+dA-Nx5w#e#ZhY#QmYn@W%+T;K0v~PUp{6pp>BEm6~@me<tyw1no
zOw1Bn2l<CCLwTo86Y8%T6D?&Nq>VQVW*<239+Y@~GJAY^rxSl&%i9_?E^lD^!fqQ#
z6}I5kejRQPgH#`m?kC+kAu_^(`&X53ri`q0(_Dng_N>>)PcoIDts14wqK9wLkZGHB
zrp-Y_#j^E<r_zhy&Y|kTCS9&GBnmug*>3}@A~*M^^V8F3K2T?ez_#GpXU+Z9s(12=
z7^G!w0A8Uj66=2!vvGrJnOM9cXeML$t4z6_6P^~bY-gRpa+mM9dYh$d1-)eJ9Ni|?
zn|hrg=#9IsY$@l^()cdH8}Tcfmq(93)T0JY^|`WtwUMaput*733A4R+{$7?NoK%?f
zyIkRoZYCB2qa;GI8qh!^h3&*(<7tKva!x+2kH;=o&HjJi<%<nd{YsrW<n9Ysg}>yc
z?%aYqIZ)vU)Mz917j#tfTM)Y%qm6b_vLg_}M|z-)xFXnO|2rD=!4+YxWq<!3;u;2$
zL<?b7>~n(V1Df<H=*ef84wmm1;q*&F6A;f#>Bn{6d7zS6(#~3-7+wr?_~rXoZ732T
z`QH$BYa4$zge-p-?sf7x!4^Gw%kpNguy&^GF*Y-$&H`{fQ}P{WD_6OQ59r8N&h88)
zAG^1Iytyk86S?ksTbgic`$tlb=?SmO+N&x?YRq<UhcM!PqTPHPtih;eVDgCCW{^i<
zPf!k{GGG8ri~LJ%*vW#$Xt-q`=^-h5z_Mz^*?fQG#t~78YP)$3JfsHIdj<wTn%q^{
zkzfQ)!91ciT~0^lSZ+eCd<_y@0FRIBl|ThOGvRQ{1g}jGky$tCQWC}3R2cvmtd5*6
zs8RSSttH+yVEzkQO_a)EY)yb#Mx3!Ja&|<Kw=Euszz$b2VYZHhii%Ktd)Z{xLxw#U
zIzoTe3Vcf+e4g0vvsA0r_h+kOylE3#=A+<Aibb@q^mk~588U8a;U&NHh-i1U<@vf9
z3?l}?Oi@D7OkFwkFe&*C!iNs~?O?}lDB)RF8L^Ek*P3dmyLlscArvru8z7RCvqNuP
z749u73?Ra%^pmN#5C8xG0ssYAE;TSY005JF2znYgFgP+aF*P+fGc{Tl4Kgt>I5IOa
zH8nUhHIoksHGi>3k9x6M1_00$`=uFBLs}t6xqBq=*>O|oM37{0oAJP-!vXOSc^Z2v
zIGzm|p9qYOv3dpVX>$eybblqI_)RJTJ7x&Omy~tznGwJYo^C}86N}{CH*)DH$#;FY
z%=RU$0}1^PA-|-RjS3!GLLgaqmgP~t#iep0ww#)9aDQL9*e8Zf@P}K^plqA2DRbtj
zhX{k*KT1^m%KWO+23oOzGp|Q@Dv(v)ARd5eCN4d!>;@yZuwz|o0w;RTpm;4u_w*DF
z?wG_caKwfLL6uD}wO@8qH(0js$$0SlpKl8s6AoY$H{YXv)@>z2zlr8vE1|0VZQ9`7
zMZ+zFW`FvB0s{d60Wb{)2`Yw2hW8Bt0R;sB1A+ko03O4BK?Av4B|P}!iYdf(!Vq~w
z#iI(4Sau7Rj%LG*vNw55w{4A6nW*ZONjv#_6~F<wv`gYb7Sp22G#mcbwk5lEYoTB)
zyL5NkICPSx2l)?MJuwPl*!#;?nHY6hKF2+qw0|#WbMeoxykW<>-zs5a>69jB8qoTL
zW!2)`Qy%nQ(IRur&5_8Vd=}^y*3mcLY+V;(%PK_SWgj9a?O`8he^{$&z&Wf-LTYwx
zR&}!G05vY8lLzBB!m{WkutZb_&So5+;p$_2DzU>v?fOmMl+F35jpW_8A;7=cP~0hy
z&K>)0KnK`zAG~KsZz`&V`*&<35U^Wjtzsa-S4jA?T?;t_6iJloQ7oHa2)y7V+~(>|
IiX}Hgq+u_N#sB~S

delta 1916
zcmV-?2ZQ*j9jYCW7X@Au+SUcJ97qEVUK85hnE(I<0Wg9D{gZJ66MyO3O8`Z+ObPdV
z4!T!U<H>5k&&z1}?8!`&uk5ZTSi(Zor_d>g*~UCW{5pKEwh-it2>?M~S~>&Q<l|fx
zVAt;2AxS)6J9Ds+5=nCDdiHgnTWEU#RfO1al23tm^dgMh{<uFsRy{_0U8tEgYLAlu
zlixOE_L!lEMG{*(r+?1qxl_KPU=pI!I#fN4AcRHq*|nvME!eHBot4LpEpMDM_AIqb
z2!3ZYxom;q`jKuNNpI;~!o(`ia=9Iap*ta9f1ZAcQio(NSx_|(sU^MYDW8i<)T(F$
zDRFrxSv9jABoQAS)1RQ0nm_FBEfNCNlMuh(ajcTxyORS}^nbbiLOpw%Tw<#~*s<sW
zkGm3}9=1(RzoJZM!!0*KL+*D_ALI*H6D?OMJJdH~+W0`}l)zxWJX#!m>rAf8>z+nf
ziIiptbB52*luO{?{h=F_vAi`hAk?2~6f_%zvf$R2k0kH~*MK#L*i*602_}O!(@IuG
ze(~FHDXgfAx_<}S*LZ{+ee-KEcuxC}MvSN3(awt?%f1f71spqplU%S|+G7WoG@HpZ
zPGk)u!IHqr3ml*z{SZLi;HN7PaZ_cZ;^<st1d3)y@Cwq8(&N)xX`lONYDzO!|DEjT
z22foW)d~8~1S)WcWf@@Kl1#*9I+`~$nQLgBdL&`QsDG7*i$!m6)Ny?fDxO?j>n7`7
zqg*wrWd(tw*Uf`4)Fs4Htjqta%so7-nwVG*2UJ^yz?FoshNA5ozw5Cg-)!;(OVGWm
zo*4i@W|la-;UrYy`TgBy#1nnp3bL(IFQ0<bY9qvA<@DX{wMk+fKFgwNz!+SZqjst8
z>%iRAjeid~NFib$;OX8+{B~4!w<_?Z_{a~lCO5%2G4m8#A3MH(2p`W32&hOMZVC*>
z3BoluI>OpnNPU_9=pCW)zWG8G^SOTVyF^qb*+X!aex|wYLqRDFwXbd4*_HZ@bSf*z
z03VSWz0(~rbfH3E&EE2UvCJr<XH8T%Wr{vOXn!{_{Uh^^J}0sbgnn}_qf)uij`I!x
z$z06uG=RlA(SnI#qDwo|cK0|X{e=Q8)Qgsc23}ilXd#opi2Xv@FCXS4$Au9j;M<*m
z-ZTu%Q#Tj^u8inOiW7C&;8V!w_Py%e@oK(eTGETatfH(m^%Iu%ZDiop$qqy?@&y-F
z41b<<iv?~mqz|&<bW3tlIu+Jl^wJ!3v?!a|KN_AO)#pbkgB2<*jNI_|^Y}zykJ({p
zXZ7~=*VV*)TA@`?Yw<gH;yoidbjEg+7{3&5Rh_CZ8g?7LbD}mkU>4^Fk4)GG+Q?Ab
zh}0{hNns)sIH7$RMVzaKo3sO}xcMdQ;(uzh@P3Q7vvVWaHcr-m9nXHXgfR6s`M#a5
zKH%0hq)*OHb%PyPTek3hZ_^U{{ECz+3&(472mlJ$xsNTa{nZ*!M!@V<iI|D_is`AP
z_7kF@cF_15UYW`Eb$#1G#sfY?V|1ZDB6T&9#-Nry*1)UDI#eIvelFW*SGrc7<$pU7
zFs78!eZNlve|Loj@smO*TEFsQ%zkKdA`aT|vxCd$k>@Pge>sbLd%n(8BFTkqqYV*D
z{CVI=(Z)oMi%d%#INgFAE%(_mt`l~Z@PvC-;T(UuX9iH~Ev1Nc`WZ?(C4#9RBBzRg
zY*C_!2iOK&6#xJL0ssYAE;TSY0F!wLdm1+|I5IRbFf=zYIa(JDF*q<dGBhzTG&eCh
zlMV?rf4E@(kV+B5sZ!L-?@$=?13`}_Gwu&-`0S_v7s5%VMld2Xnz}61^&pp4%nI*a
z-)M@NF_oHu6M|#!h{8vzlA3bE`P&;y)?UbsEEpg<Lfax-WWHXL>}EG<KAKR>#D@j(
z`kAb`chg=+SQ(ocEKmOQ_Qjg!gt?-jT0DX;f50<MEU`rSiFeFr16#cJO4o1KXz0qc
z1A&)SXqQ;yg@BoyC2UZydT2pj@A@BKpfbyGvAn3%oZ&9Aq}ddk1JHlvG>50F#l!O2
z`>pkU>|aJ%@Cew<8TtIqjba8IPmA&8%v*uEvN^E#s(-T^xw^tJTJ;q}9b-Tuc40+{
ze~}Xc0|5X5FbxI?Duzgg_YDC71qA>Df&l>l2REKd+}j|&Eg|ZaxR|{1kW_9gdcOC#
zks*T*7@C~AwMg<TVG=onxJ0cBbhu<TD4u9hbnrjhw%Q8#bl=JY+c7Lx5Wga|R1xHa
zgXcOXL342^#j(alR(Iz-4aXWRE}1JHe*@|G5>WTmr>wWu6@hQ<pVl}~=jnc+A$rWH
zLsCCSkBQ{KdsW&`beU**(Hu=a>Bi+kaq;Z8XJCJVlu_Mjk8oIhU2lO?ic)9l5r=1}
zwJ8YnH1oKD)Y>rUKmKQP3~42Vg78K4^wZ(jaA2&6bX|XRB5T*h?ykzqWW+2i9sc49
zl94^rB&})yf-|1oz!EAr#BT!HoXB^tvVE}}IRg}rs5%W4XX^yYTy<0w?+Jw-&Y!^<
CO`${p

diff --git a/modules/clients/src/test/keystore/ca/oneindex.txt b/modules/clients/src/test/keystore/ca/oneindex.txt
index 8d347d072ef92..5d0e1c9163a0d 100644
--- a/modules/clients/src/test/keystore/ca/oneindex.txt
+++ b/modules/clients/src/test/keystore/ca/oneindex.txt
@@ -1 +1 @@
-V	180824104710Z		01	unknown	/CN=node01
+V	210823155040Z		01	unknown	/CN=node01
diff --git a/modules/clients/src/test/keystore/ca/twoindex.txt b/modules/clients/src/test/keystore/ca/twoindex.txt
index 00b7307287709..1f9359d2d902e 100644
--- a/modules/clients/src/test/keystore/ca/twoindex.txt
+++ b/modules/clients/src/test/keystore/ca/twoindex.txt
@@ -1,2 +1,2 @@
-V	180824104716Z		01	unknown	/CN=node02
-V	180824104719Z		02	unknown	/CN=node03
+V	210823155541Z		01	unknown	/CN=node02
+V	210823155835Z		02	unknown	/CN=node03

From 0011102c2013ce654c0fb3395f6f83ad65a59cfb Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Fri, 14 Sep 2018 14:27:09 +0300
Subject: [PATCH 355/543] GG-14199 switch on Idle Verify V2 for 2.5.1

---
 .../processors/cache/verify/VerifyBackupPartitionsTaskV2.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
index 826393ac09602..0041bdde786b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
@@ -69,7 +69,7 @@
 @GridInternal
 public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVerifyTaskArg, IdleVerifyResultV2> {
     /** First version of Ignite that is capable of executing Idle Verify V2. */
-    public static final IgniteProductVersion V2_SINCE_VER = IgniteProductVersion.fromString("2.5.3");
+    public static final IgniteProductVersion V2_SINCE_VER = IgniteProductVersion.fromString("2.5.0");
 
     /** */
     private static final long serialVersionUID = 0L;

From afb0a8caefcba706ac595fad8e00f5e71b14aa40 Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Fri, 14 Sep 2018 15:25:30 +0300
Subject: [PATCH 356/543] IGNITE-9100 remove duplicate test

---
 .../ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java      | 2 --
 1 file changed, 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
index 82be3fa19db75..de584e16408db 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMetricsSelfTestSuite.java
@@ -20,7 +20,6 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.TransactionMetricsMxBeanImplTest;
 import org.apache.ignite.internal.processors.cache.CacheGroupsMetricsRebalanceTest;
-import org.apache.ignite.internal.processors.cache.CacheMetricsEnableRuntimeTest;
 import org.apache.ignite.internal.processors.cache.CacheMetricsEntitiesCountTest;
 import org.apache.ignite.internal.processors.cache.CacheMetricsForClusterGroupSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheValidatorMetricsTest;
@@ -66,7 +65,6 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(CacheGroupsMetricsRebalanceTest.class);
         suite.addTestSuite(CacheValidatorMetricsTest.class);
-        suite.addTestSuite(CacheMetricsEnableRuntimeTest.class);
         suite.addTestSuite(CacheMetricsEntitiesCountTest.class);
 
         // Cluster wide metrics.

From cac22a0ceaee40481cbcb7fb192404ff4b86b557 Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Tue, 4 Sep 2018 12:41:14 +0300
Subject: [PATCH 357/543] IGNITE-9433 Refactoring to improve constant usage for
 file suffixes. - Fixes #4652.

---
 .../GridCacheDatabaseSharedManager.java       | 11 +++---
 .../file/FilePageStoreManager.java            | 12 +++++--
 .../wal/FileWriteAheadLogManager.java         | 26 ++++++++------
 .../FsyncModeFileWriteAheadLogManager.java    | 34 +++++++++++--------
 .../IgnitePdsDiskErrorsRecoveringTest.java    |  6 ++--
 ...IgniteNodeStoppedDuringDisableWALTest.java |  4 +--
 .../impl/v2/HadoopV2JobResourceManager.java   | 26 +++++++-------
 7 files changed, 66 insertions(+), 53 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index ec1014d07f9e8..d7b514eb08d62 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -215,9 +215,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Checkpoint file name pattern. */
     public static final Pattern CP_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-(.*)-(START|END)\\.bin");
 
-    /** Checkpoint file temporary suffix. This is needed to safe writing checkpoint markers through temporary file and renaming. */
-    public static final String FILE_TMP_SUFFIX = ".tmp";
-
     /** Node started file suffix. */
     public static final String NODE_STARTED_FILE_NAME_SUFFIX = "-node-started.bin";
 
@@ -487,13 +484,13 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
     }
 
     /**
-     * Cleanup checkpoint directory from all temporary files {@link #FILE_TMP_SUFFIX}.
+     * Cleanup checkpoint directory from all temporary files.
      */
     public void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
         try {
             try (DirectoryStream<Path> files = Files.newDirectoryStream(
                 cpDir.toPath(),
-                path -> path.endsWith(FILE_TMP_SUFFIX))
+                path -> path.endsWith(FilePageStoreManager.TMP_SUFFIX))
             ) {
                 for (Path path : files)
                     Files.delete(path);
@@ -872,7 +869,7 @@ private void nodeStart(WALPointer ptr) throws IgniteCheckedException {
         FileWALPointer p = (FileWALPointer)ptr;
 
         String fileName = U.currentTimeMillis() + NODE_STARTED_FILE_NAME_SUFFIX;
-        String tmpFileName = fileName + FILE_TMP_SUFFIX;
+        String tmpFileName = fileName + FilePageStoreManager.TMP_SUFFIX;
 
         ByteBuffer buf = ByteBuffer.allocate(FileWALPointer.POINTER_SIZE);
         buf.order(ByteOrder.nativeOrder());
@@ -2634,7 +2631,7 @@ private CheckpointEntry prepareCheckpointEntry(
      */
     public void writeCheckpointEntry(ByteBuffer entryBuf, CheckpointEntry cp, CheckpointEntryType type) throws StorageException {
         String fileName = checkpointFileName(cp, type);
-        String tmpFileName = fileName + FILE_TMP_SUFFIX;
+        String tmpFileName = fileName + FilePageStoreManager.TMP_SUFFIX;
 
         try {
             try (FileIO io = ioFactory.create(Paths.get(cpDir.getAbsolutePath(), skipSync ? fileName : tmpFileName).toFile(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index cbec04a5ff3e4..a89ba57bfeba6 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -77,6 +77,12 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
     /** File suffix. */
     public static final String FILE_SUFFIX = ".bin";
 
+    /** Suffix for zip files */
+    public static final String ZIP_SUFFIX = ".zip";
+
+    /** Suffix for tmp files */
+    public static final String TMP_SUFFIX = ".tmp";
+
     /** Partition file prefix. */
     public static final String PART_FILE_PREFIX = "part-";
 
@@ -96,7 +102,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
     public static final String CACHE_DATA_FILENAME = "cache_data.dat";
 
     /** */
-    public static final String CACHE_DATA_TMP_FILENAME = CACHE_DATA_FILENAME + ".tmp";
+    public static final String CACHE_DATA_TMP_FILENAME = CACHE_DATA_FILENAME + TMP_SUFFIX;
 
     /** */
     public static final String DFLT_STORE_DIR = "db";
@@ -313,7 +319,7 @@ public FilePageStoreManager(GridKernalContext ctx) {
 
         if (overwrite || !file.exists() || file.length() == 0) {
             try {
-                File tmp = new File(file.getParent(), file.getName() + ".tmp");
+                File tmp = new File(file.getParent(), file.getName() + TMP_SUFFIX);
 
                 tmp.createNewFile();
 
@@ -568,7 +574,7 @@ private boolean checkAndInitCacheWorkDir(File cacheWorkDir) throws IgniteChecked
 
             Path cacheWorkDirPath = cacheWorkDir.toPath();
 
-            Path tmp = cacheWorkDirPath.getParent().resolve(cacheWorkDir.getName() + ".tmp");
+            Path tmp = cacheWorkDirPath.getParent().resolve(cacheWorkDir.getName() + TMP_SUFFIX);
 
             if (Files.exists(tmp) && Files.isDirectory(tmp) &&
                     Files.exists(tmp.resolve(IgniteCacheSnapshotManager.TEMP_FILES_COMPLETENESS_MARKER))) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 27ef23248582d..d3dc108be6d35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -71,13 +71,11 @@
 import org.apache.ignite.events.WalSegmentArchivedEvent;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.failure.FailureType;
-import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
@@ -89,8 +87,10 @@
 import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
 import org.apache.ignite.internal.processors.cache.persistence.DataStorageMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
 import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator.AbstractFileDescriptor;
@@ -506,7 +506,7 @@ public Collection<File> getAndReserveWalFiles(FileWALPointer low, FileWALPointer
             String segmentName = FileDescriptor.fileName(i);
 
             File file = new File(walArchiveDir, segmentName);
-            File fileZip = new File(walArchiveDir, segmentName + ".zip");
+            File fileZip = new File(walArchiveDir, segmentName + FilePageStoreManager.ZIP_SUFFIX);
 
             if (file.exists())
                 res.add(file);
@@ -881,7 +881,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
     private boolean hasIndex(long absIdx) {
         String segmentName = FileDescriptor.fileName(absIdx);
 
-        String zipSegmentName = FileDescriptor.fileName(absIdx) + ".zip";
+        String zipSegmentName = FileDescriptor.fileName(absIdx) + FilePageStoreManager.ZIP_SUFFIX;
 
         boolean inArchive = new File(walArchiveDir, segmentName).exists() ||
             new File(walArchiveDir, zipSegmentName).exists();
@@ -1452,7 +1452,7 @@ private void createFile(File file) throws StorageException {
         if (log.isDebugEnabled())
             log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
-        File tmp = new File(file.getParent(), file.getName() + ".tmp");
+        File tmp = new File(file.getParent(), file.getName() + FilePageStoreManager.TMP_SUFFIX);
 
         formatFile(tmp);
 
@@ -1834,7 +1834,7 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws StorageException
 
             String name = FileDescriptor.fileName(absIdx);
 
-            File dstTmpFile = new File(walArchiveDir, name + ".tmp");
+            File dstTmpFile = new File(walArchiveDir, name + FilePageStoreManager.TMP_SUFFIX);
 
             File dstFile = new File(walArchiveDir, name);
 
@@ -2027,9 +2027,10 @@ private void deleteObsoleteRawSegments() {
                     if (currReservedSegment == -1)
                         continue;
 
-                    File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + ".zip" + ".tmp");
+                    File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment)
+                        + FilePageStoreManager.ZIP_SUFFIX + FilePageStoreManager.TMP_SUFFIX);
 
-                    File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + ".zip");
+                    File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + FilePageStoreManager.ZIP_SUFFIX);
 
                     File raw = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment));
                     if (!Files.exists(raw.toPath()))
@@ -2180,8 +2181,10 @@ private class FileDecompressor extends GridWorker {
                         if (isCancelled())
                             break;
 
-                        File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".zip");
-                        File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".tmp");
+                        File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress)
+                            + FilePageStoreManager.ZIP_SUFFIX);
+                        File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress)
+                            + FilePageStoreManager.TMP_SUFFIX);
                         File unzip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress));
 
                         try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
@@ -3068,7 +3071,8 @@ private RecordsIterator(
 
             if (!desc.file().exists()) {
                 FileDescriptor zipFile = new FileDescriptor(
-                    new File(walArchiveDir, FileDescriptor.fileName(desc.idx()) + ".zip"));
+                    new File(walArchiveDir, FileDescriptor.fileName(desc.idx())
+                        + FilePageStoreManager.ZIP_SUFFIX));
 
                 if (!zipFile.file.exists()) {
                     throw new FileNotFoundException("Both compressed and raw segment files are missing in archive " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index df2878f7645fc..38681b5f6f6a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -72,7 +72,6 @@
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
@@ -83,8 +82,10 @@
 import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
 import org.apache.ignite.internal.processors.cache.persistence.DataStorageMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
@@ -590,7 +591,7 @@ public Collection<File> getAndReserveWalFiles(FileWALPointer low, FileWALPointer
             String segmentName = FileWriteAheadLogManager.FileDescriptor.fileName(i);
 
             File file = new File(walArchiveDir, segmentName);
-            File fileZip = new File(walArchiveDir, segmentName + ".zip");
+            File fileZip = new File(walArchiveDir, segmentName + FilePageStoreManager.ZIP_SUFFIX);
 
             if (file.exists())
                 res.add(file);
@@ -787,7 +788,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
     private boolean hasIndex(long absIdx) {
         String segmentName = FileDescriptor.fileName(absIdx);
 
-        String zipSegmentName = FileDescriptor.fileName(absIdx) + ".zip";
+        String zipSegmentName = FileDescriptor.fileName(absIdx) + FilePageStoreManager.ZIP_SUFFIX;
 
         boolean inArchive = new File(walArchiveDir, segmentName).exists() ||
             new File(walArchiveDir, zipSegmentName).exists();
@@ -1266,7 +1267,7 @@ private void createFile(File file) throws StorageException {
         if (log.isDebugEnabled())
             log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
 
-        File tmp = new File(file.getParent(), file.getName() + ".tmp");
+        File tmp = new File(file.getParent(), file.getName() + FilePageStoreManager.TMP_SUFFIX);
 
         formatFile(tmp);
 
@@ -1661,7 +1662,7 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedExc
 
             String name = FileDescriptor.fileName(absIdx);
 
-            File dstTmpFile = new File(walArchiveDir, name + ".tmp");
+            File dstTmpFile = new File(walArchiveDir, name + FilePageStoreManager.TMP_SUFFIX);
 
             File dstFile = new File(walArchiveDir, name);
 
@@ -1813,19 +1814,19 @@ private long tryReserveNextSegmentOrWait() throws InterruptedException, IgniteCh
          * Deletes raw WAL segments if they aren't locked and already have compressed copies of themselves.
          */
         private void deleteObsoleteRawSegments() {
-            FsyncModeFileWriteAheadLogManager.FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
+            FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
 
             Set<Long> indices = new HashSet<>();
             Set<Long> duplicateIndices = new HashSet<>();
 
-            for (FsyncModeFileWriteAheadLogManager.FileDescriptor desc : descs) {
+            for (FileDescriptor desc : descs) {
                 if (!indices.add(desc.idx))
                     duplicateIndices.add(desc.idx);
             }
 
             FileArchiver archiver0 = archiver;
 
-            for (FsyncModeFileWriteAheadLogManager.FileDescriptor desc : descs) {
+            for (FileDescriptor desc : descs) {
                 if (desc.isCompressed())
                     continue;
 
@@ -1855,11 +1856,13 @@ private void deleteObsoleteRawSegments() {
                     if (currReservedSegment == -1)
                         continue;
 
-                    File tmpZip = new File(walArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(currReservedSegment) + ".zip" + ".tmp");
+                    File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment)
+                        + FilePageStoreManager.ZIP_SUFFIX + FilePageStoreManager.TMP_SUFFIX);
 
-                    File zip = new File(walArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(currReservedSegment) + ".zip");
+                    File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment)
+                        + FilePageStoreManager.ZIP_SUFFIX);
 
-                    File raw = new File(walArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(currReservedSegment));
+                    File raw = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment));
                     if (!Files.exists(raw.toPath()))
                         throw new IgniteCheckedException("WAL archive segment is missing: " + raw);
 
@@ -1986,8 +1989,10 @@ private class FileDecompressor extends GridWorker {
                         if (isCancelled())
                             break;
 
-                        File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".zip");
-                        File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress) + ".tmp");
+                        File zip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress)
+                            + FilePageStoreManager.ZIP_SUFFIX);
+                        File unzipTmp = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress)
+                            + FilePageStoreManager.TMP_SUFFIX);
                         File unzip = new File(walArchiveDir, FileDescriptor.fileName(segmentToDecompress));
 
                         try (ZipInputStream zis = new ZipInputStream(new BufferedInputStream(new FileInputStream(zip)));
@@ -3179,7 +3184,8 @@ private RecordsIterator(
 
             if (!desc.file().exists()) {
                 FileDescriptor zipFile = new FileDescriptor(
-                        new File(walArchiveDir, FileDescriptor.fileName(desc.idx()) + ".zip"));
+                        new File(walArchiveDir, FileDescriptor.fileName(desc.idx())
+                            + FilePageStoreManager.ZIP_SUFFIX));
 
                 if (!zipFile.file.exists()) {
                     throw new FileNotFoundException("Both compressed and raw segment files are missing in archive " +
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
index 6e9ed1fe9318b..fc0804fad3297 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
@@ -40,10 +40,10 @@
 import org.apache.ignite.internal.GridKernalState;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
@@ -158,7 +158,7 @@ public void testRecoveringOnCacheInitFail() throws Exception {
      */
     public void testRecoveringOnNodeStartMarkerWriteFail() throws Exception {
         // Fail to write node start marker tmp file at the second checkpoint. Pass only initial checkpoint.
-        ioFactory = new FilteringFileIOFactory("started.bin" + GridCacheDatabaseSharedManager.FILE_TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20));
+        ioFactory = new FilteringFileIOFactory("started.bin" + FilePageStoreManager.TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20));
 
         IgniteEx grid = startGrid(0);
         grid.cluster().active(true);
@@ -213,7 +213,7 @@ public void testRecoveringOnNodeStartMarkerWriteFail() throws Exception {
      */
     public void testRecoveringOnCheckpointBeginFail() throws Exception {
         // Fail to write checkpoint start marker tmp file at the second checkpoint. Pass only initial checkpoint.
-        ioFactory = new FilteringFileIOFactory("START.bin" + GridCacheDatabaseSharedManager.FILE_TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20));
+        ioFactory = new FilteringFileIOFactory("START.bin" + FilePageStoreManager.TMP_SUFFIX, new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 20));
 
         final IgniteEx grid = startGrid(0);
         grid.cluster().active(true);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
index 80198e8af14ee..bec277c29bd41 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -47,7 +48,6 @@
 import static java.nio.file.FileVisitResult.CONTINUE;
 import static java.nio.file.Files.walkFileTree;
 import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.CP_FILE_NAME_PATTERN;
-import static org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager.FILE_TMP_SUFFIX;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.META_STORAGE_NAME;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX;
@@ -208,7 +208,7 @@ private void testStopNodeWithDisableWAL(NodeStopPoint nodeStopPoint) throws Exce
 
                     boolean failed = false;
 
-                    if (name.endsWith(FILE_TMP_SUFFIX))
+                    if (name.endsWith(FilePageStoreManager.TMP_SUFFIX))
                         failed = true;
 
                     if (CP_FILE_NAME_PATTERN.matcher(name).matches())
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
index 52e394be96589..6fe1ea0be7e09 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/impl/v2/HadoopV2JobResourceManager.java
@@ -17,6 +17,16 @@
 
 package org.apache.ignite.internal.processors.hadoop.impl.v2;
 
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystemException;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
@@ -26,6 +36,7 @@
 import org.apache.hadoop.util.RunJar;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.hadoop.HadoopCommonUtils;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
 import org.apache.ignite.internal.processors.hadoop.impl.fs.HadoopFileSystemsUtils;
@@ -33,17 +44,6 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.FileSystemException;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-
 /**
  * Provides all resources are needed to the job execution. Downloads the main jar, the configuration and additional
  * files are needed to be placed on local files system.
@@ -234,7 +234,7 @@ private void processFiles(File jobLocDir, @Nullable Object[] files, boolean down
 
                 if (archiveNameLC.endsWith(".jar"))
                     RunJar.unJar(archiveFile, dstPath);
-                else if (archiveNameLC.endsWith(".zip"))
+                else if (archiveNameLC.endsWith(FilePageStoreManager.ZIP_SUFFIX))
                     FileUtil.unZip(archiveFile, dstPath);
                 else if (archiveNameLC.endsWith(".tar.gz") ||
                     archiveNameLC.endsWith(".tgz") ||
@@ -318,4 +318,4 @@ public void cleanupStagingDirectory() {
     @Nullable public URL[] classPath() {
         return clsPath;
     }
-}
\ No newline at end of file
+}

From dde31d917bf5c08c75222df2cf8025b2a4490b96 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Fri, 14 Sep 2018 19:21:45 +0300
Subject: [PATCH 358/543] IGNITE-9398 Reduce time on CustomDiscoveryMessage
 processing by discovery message worker. Fixes #4636.

(cherry picked from commit b13f373c24bf2538603a7f3fee5af70405ad8356)
---
 .../discovery/GridDiscoveryManager.java       | 120 ++++++++++++++++--
 .../cluster/GridClusterStateProcessor.java    |   9 +-
 .../apache/ignite/spi/IgniteSpiAdapter.java   |   3 +-
 .../spi/discovery/DiscoverySpiListener.java   |   5 +-
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  33 +++--
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  64 ++++++----
 ...eMarshallerCacheClassNameConflictTest.java |  10 +-
 .../IgniteMarshallerCacheFSRestoreTest.java   |   8 +-
 ...iteAbstractStandByClientReconnectTest.java |  10 +-
 .../discovery/AbstractDiscoverySelfTest.java  |  21 ++-
 .../ignite/testframework/GridTestUtils.java   |   5 +-
 .../zk/internal/ZookeeperDiscoveryImpl.java   | 113 +++++++++++------
 12 files changed, 300 insertions(+), 101 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 82abdf5213f9e..653d33dffcf44 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -70,6 +70,7 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.cluster.NodeOrderComparator;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
@@ -101,6 +102,7 @@
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -217,6 +219,9 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
     /** Discovery event worker. */
     private final DiscoveryWorker discoWrk = new DiscoveryWorker();
 
+    /** Discovery event notyfier worker. */
+    private final DiscoveryMessageNotifyerWorker discoNotifierWrk = new DiscoveryMessageNotifyerWorker();
+
     /** Network segment check worker. */
     private SegmentCheckWorker segChkWrk;
 
@@ -579,16 +584,23 @@ private void updateClientNodes(UUID leftNodeId) {
                 }
             }
 
-            @Override public void onDiscovery(
+            @Override public IgniteInternalFuture onDiscovery(
                 final int type,
                 final long topVer,
                 final ClusterNode node,
                 final Collection<ClusterNode> topSnapshot,
                 final Map<Long, Collection<ClusterNode>> snapshots,
-                @Nullable DiscoverySpiCustomMessage spiCustomMsg) {
-                synchronized (discoEvtMux) {
-                    onDiscovery0(type, topVer, node, topSnapshot, snapshots, spiCustomMsg);
-                }
+                @Nullable DiscoverySpiCustomMessage spiCustomMsg
+            ) {
+                GridFutureAdapter notificationFut = new GridFutureAdapter();
+
+                discoNotifierWrk.submit(notificationFut, () -> {
+                    synchronized (discoEvtMux) {
+                        onDiscovery0(type, topVer, node, topSnapshot, snapshots, spiCustomMsg);
+                    }
+                });
+
+                return notificationFut;
             }
 
             /**
@@ -913,6 +925,8 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) {
             }
         });
 
+        new IgniteThread(discoNotifierWrk).start();
+
         startSpi();
 
         registeredDiscoSpi = true;
@@ -1679,6 +1693,10 @@ private void topologySnapshotMessage(IgniteClosure<String, Void> clo, long topVe
 
         U.join(discoWrk, log);
 
+        U.cancel(discoNotifierWrk);
+
+        U.join(discoNotifierWrk, log);
+
         // Stop SPI itself.
         stopSpi();
 
@@ -2638,6 +2656,86 @@ public void scheduleSegmentCheck() {
         }
     }
 
+    /**
+     *
+     */
+    private class DiscoveryMessageNotifyerWorker extends GridWorker {
+        /** Queue. */
+        private final BlockingQueue<T2<GridFutureAdapter, Runnable>> queue = new LinkedBlockingQueue<>();
+
+        /**
+         * Default constructor.
+         */
+        protected DiscoveryMessageNotifyerWorker() {
+            super(ctx.igniteInstanceName(), "disco-notyfier-worker", GridDiscoveryManager.this.log, ctx.workersRegistry());
+        }
+
+        /**
+         *
+         */
+        private void body0() throws InterruptedException {
+            T2<GridFutureAdapter, Runnable> notification = queue.take();
+
+            try {
+                notification.get2().run();
+            }
+            finally {
+                notification.get1().onDone();
+            }
+        }
+
+        /**
+         * @param cmd Command.
+         */
+        public synchronized void submit(GridFutureAdapter notificationFut, Runnable cmd) {
+            if (isCancelled()) {
+                notificationFut.onDone();
+
+                return;
+            }
+
+            queue.add(new T2<>(notificationFut, cmd));
+        }
+
+        /**
+         * Cancel thread execution and completes all notification futures.
+         */
+        @Override public synchronized void cancel() {
+            super.cancel();
+
+            while (!queue.isEmpty()) {
+                T2<GridFutureAdapter, Runnable> notification = queue.poll();
+
+                if (notification != null)
+                    notification.get1().onDone();
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+            while (!isCancelled()) {
+                try {
+                    body0();
+                }
+                catch (InterruptedException e) {
+                    if (!isCancelled)
+                        ctx.failure().process(new FailureContext(SYSTEM_WORKER_TERMINATION, e));
+
+                    throw e;
+                }
+                catch (Throwable t) {
+                    U.error(log, "Exception in discovery notyfier worker thread.", t);
+
+                    FailureType type = t instanceof OutOfMemoryError ? CRITICAL_ERROR : SYSTEM_WORKER_TERMINATION;
+
+                    ctx.failure().process(new FailureContext(type, t));
+
+                    throw t;
+                }
+            }
+        }
+    }
+
     /** Worker for discovery events. */
     private class DiscoveryWorker extends GridWorker {
         /** */
@@ -2746,15 +2844,13 @@ void addEvent(
                     throw e;
                 }
                 catch (Throwable t) {
-                    U.error(log, "Exception in discovery worker thread.", t);
+                    U.error(log, "Exception in discovery event worker thread.", t);
 
-                    if (t instanceof Error) {
-                        FailureType type = t instanceof OutOfMemoryError ? CRITICAL_ERROR : SYSTEM_WORKER_TERMINATION;
+                    FailureType type = t instanceof OutOfMemoryError ? CRITICAL_ERROR : SYSTEM_WORKER_TERMINATION;
 
-                        ctx.failure().process(new FailureContext(type, t));
+                    ctx.failure().process(new FailureContext(type, t));
 
-                        throw t;
-                    }
+                    throw t;
                 }
             }
         }
@@ -2769,7 +2865,7 @@ private void body0() throws InterruptedException {
 
             AffinityTopologyVersion topVer = evt.get2();
 
-            if (type == EVT_NODE_METRICS_UPDATED && topVer.compareTo(discoCache.version()) < 0)
+            if (type == EVT_NODE_METRICS_UPDATED && (discoCache == null || topVer.compareTo(discoCache.version()) < 0))
                 return;
 
             ClusterNode node = evt.get3();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 8d2620f531056..72e5d8ab67da4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -510,13 +510,16 @@ protected void afterStateChangeFinished(IgniteUuid msgId, boolean success) {
 
                 transitionFuts.put(msg.requestId(), new GridFutureAdapter<Void>());
 
+                DiscoveryDataClusterState prevState = globalState;
+
                 globalState = DiscoveryDataClusterState.createTransitionState(
-                    globalState,
+                    prevState,
                     msg.activate(),
-                    msg.activate() ? msg.baselineTopology() : globalState.baselineTopology(),
+                    msg.activate() ? msg.baselineTopology() : prevState.baselineTopology(),
                     msg.requestId(),
                     topVer,
-                    nodeIds);
+                    nodeIds
+                );
 
                 if (msg.forceChangeBaselineTopology())
                     globalState.setTransitionResult(msg.requestId(), msg.activate());
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index e8c27d2f90626..97cd95ae9cfcb 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -547,7 +547,8 @@ private void checkConfigurationConsistency(IgniteSpiContext spiCtx, ClusterNode
         if (rmtCls == null) {
             if (!optional && starting)
                 throw new IgniteSpiException("Remote SPI with the same name is not configured" + tipStr +
-                    " [name=" + name + ", loc=" + locCls + ']');
+                    " [name=" + name + ", loc=" + locCls + ", locNode=" + spiCtx.localNode() + ", rmt=" + rmtCls +
+                    ", rmtNode=" + node + ']');
 
             sb.a(format(">>> Remote SPI with the same name is not configured: " + name, locCls));
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiListener.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiListener.java
index 2b2ac94e9414b..519a235ae103b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiListener.java
@@ -21,6 +21,7 @@
 import java.util.Map;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -48,8 +49,10 @@ public interface DiscoverySpiListener {
      *      {@code EVT_NODE_JOINED}, then joined node will be in snapshot).
      * @param topHist Topology snapshots history.
      * @param data Data for custom event.
+     *
+     * @return A future that will be completed when notification process has finished.
      */
-    public void onDiscovery(
+    public IgniteInternalFuture onDiscovery(
         int type,
         long topVer,
         ClusterNode node,
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 934b1da6e3880..ca1c56fb6eda5 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -465,7 +465,12 @@ else if (state == DISCONNECTED) {
 
                 Collection<ClusterNode> top = updateTopologyHistory(topVer + 1, null);
 
-                lsnr.onDiscovery(EVT_NODE_FAILED, topVer, n, top, new TreeMap<>(topHist), null);
+                try {
+                    lsnr.onDiscovery(EVT_NODE_FAILED, topVer, n, top, new TreeMap<>(topHist), null).get();
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IgniteException("Failed to wait for discovery listener notification", e);
+                }
             }
         }
 
@@ -2473,21 +2478,31 @@ private void notifyDiscovery(int type, long topVer, ClusterNode node, Collection
          * @param top Topology snapshot.
          * @param data Optional custom message data.
          */
-        private void notifyDiscovery(int type, long topVer, ClusterNode node, Collection<ClusterNode> top,
-            @Nullable DiscoverySpiCustomMessage data) {
+        private void notifyDiscovery(
+            int type,
+            long topVer,
+            ClusterNode node,
+            Collection<ClusterNode> top,
+            @Nullable DiscoverySpiCustomMessage data
+        ) {
             DiscoverySpiListener lsnr = spi.lsnr;
 
-            DebugLogger log = type == EVT_NODE_METRICS_UPDATED ? traceLog : debugLog;
+            DebugLogger debugLog = type == EVT_NODE_METRICS_UPDATED ? traceLog : ClientImpl.this.debugLog;
 
             if (lsnr != null) {
-                if (log.isDebugEnabled())
-                    log.debug("Discovery notification [node=" + node + ", type=" + U.gridEventName(type) +
+                if (debugLog.isDebugEnabled())
+                    debugLog.debug("Discovery notification [node=" + node + ", type=" + U.gridEventName(type) +
                         ", topVer=" + topVer + ']');
 
-                lsnr.onDiscovery(type, topVer, node, top, new TreeMap<>(topHist), data);
+                try {
+                    lsnr.onDiscovery(type, topVer, node, top, new TreeMap<>(topHist), data).get();
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IgniteException("Failed to wait for discovery listener notification", e);
+                }
             }
-            else if (log.isDebugEnabled())
-                log.debug("Skipped discovery notification [node=" + node + ", type=" + U.gridEventName(type) +
+            else if (debugLog.isDebugEnabled())
+                debugLog.debug("Skipped discovery notification [node=" + node + ", type=" + U.gridEventName(type) +
                     ", topVer=" + topVer + ']');
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 906b97a7e749c..08af4923f879b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -70,6 +70,7 @@
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.IgnitionEx;
@@ -2811,7 +2812,7 @@ else if (msg instanceof TcpDiscoveryDiscardMessage)
                 processDiscardMessage((TcpDiscoveryDiscardMessage)msg);
 
             else if (msg instanceof TcpDiscoveryCustomEventMessage)
-                processCustomMessage((TcpDiscoveryCustomEventMessage)msg);
+                processCustomMessage((TcpDiscoveryCustomEventMessage)msg, false);
 
             else if (msg instanceof TcpDiscoveryClientPingRequest)
                 processClientPingRequest((TcpDiscoveryClientPingRequest)msg);
@@ -5307,8 +5308,9 @@ private void processClientPingRequest(final TcpDiscoveryClientPingRequest msg) {
 
         /**
          * @param msg Message.
+         * @param waitForNotification If {@code true} then thread will wait when discovery event notification has finished.
          */
-        private void processCustomMessage(TcpDiscoveryCustomEventMessage msg) {
+        private void processCustomMessage(TcpDiscoveryCustomEventMessage msg, boolean waitForNotification) {
             if (isLocalNodeCoordinator()) {
                 boolean delayMsg;
 
@@ -5342,12 +5344,12 @@ private void processCustomMessage(TcpDiscoveryCustomEventMessage msg) {
                     msg.topologyVersion(ring.topologyVersion());
 
                     if (pendingMsgs.procCustomMsgs.add(msg.id())) {
-                        notifyDiscoveryListener(msg);
+                        notifyDiscoveryListener(msg, waitForNotification);
 
                         if (sendMessageToRemotes(msg))
                             sendMessageAcrossRing(msg);
                         else
-                            processCustomMessage(msg);
+                            processCustomMessage(msg, waitForNotification);
                     }
 
                     msg.message(null, msg.messageBytes());
@@ -5376,7 +5378,7 @@ private void processCustomMessage(TcpDiscoveryCustomEventMessage msg) {
 
                                 ackMsg.topologyVersion(msg.topologyVersion());
 
-                                processCustomMessage(ackMsg);
+                                processCustomMessage(ackMsg, waitForNotification);
                             }
                             catch (IgniteCheckedException e) {
                                 U.error(log, "Failed to marshal discovery custom message.", e);
@@ -5403,7 +5405,7 @@ private void processCustomMessage(TcpDiscoveryCustomEventMessage msg) {
                     assert msg.topologyVersion() == ring.topologyVersion() :
                         "msg: " + msg + ", topVer=" + ring.topologyVersion();
 
-                    notifyDiscoveryListener(msg);
+                    notifyDiscoveryListener(msg, waitForNotification);
                 }
 
                 if (msg.verified())
@@ -5479,7 +5481,7 @@ private void checkPendingCustomMessages() {
                 TcpDiscoveryCustomEventMessage msg;
 
                 while ((msg = pollPendingCustomeMessage()) != null)
-                    processCustomMessage(msg);
+                    processCustomMessage(msg, true);
             }
         }
 
@@ -5494,8 +5496,9 @@ private void checkPendingCustomMessages() {
 
         /**
          * @param msg Custom message.
+         * @param waitForNotification If {@code true} thread will wait when discovery event notification has finished.
          */
-        private void notifyDiscoveryListener(TcpDiscoveryCustomEventMessage msg) {
+        private void notifyDiscoveryListener(TcpDiscoveryCustomEventMessage msg, boolean waitForNotification) {
             DiscoverySpiListener lsnr = spi.lsnr;
 
             TcpDiscoverySpiState spiState = spiStateCopy();
@@ -5511,23 +5514,40 @@ private void notifyDiscoveryListener(TcpDiscoveryCustomEventMessage msg) {
             if (lsnr != null && (spiState == CONNECTED || spiState == DISCONNECTING)) {
                 TcpDiscoveryNode node = ring.node(msg.creatorNodeId());
 
-                if (node != null) {
-                    try {
-                        DiscoverySpiCustomMessage msgObj = msg.message(spi.marshaller(),
-                            U.resolveClassLoader(spi.ignite().configuration()));
+                if (node == null)
+                    return;
 
-                        lsnr.onDiscovery(DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT,
-                            msg.topologyVersion(),
-                            node,
-                            snapshot,
-                            hist,
-                            msgObj);
+                DiscoverySpiCustomMessage msgObj;
 
-                        if (msgObj.isMutable())
-                            msg.message(msgObj, U.marshal(spi.marshaller(), msgObj));
+                try {
+                    msgObj = msg.message(spi.marshaller(), U.resolveClassLoader(spi.ignite().configuration()));
+                }
+                catch (Throwable t) {
+                    throw new IgniteException("Failed to unmarshal discovery custom message: " + msg, t);
+                }
+
+                IgniteInternalFuture fut = lsnr.onDiscovery(DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT,
+                    msg.topologyVersion(),
+                    node,
+                    snapshot,
+                    hist,
+                    msgObj);
+
+                if (waitForNotification || msgObj.isMutable()) {
+                    try {
+                        fut.get();
                     }
-                    catch (Throwable e) {
-                        U.error(log, "Failed to unmarshal discovery custom message.", e);
+                    catch (IgniteCheckedException e) {
+                        throw new IgniteException("Failed to wait for discovery listener notification", e);
+                    }
+                }
+
+                if (msgObj.isMutable()) {
+                    try {
+                        msg.message(msgObj, U.marshal(spi.marshaller(), msgObj));
+                    }
+                    catch (Throwable t) {
+                        throw new IgniteException("Failed to marshal mutable discovery message: " + msgObj, t);
                     }
                 }
             }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java
index 80d0fd163edcf..64c781741fd51 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java
@@ -31,13 +31,13 @@
 import org.apache.ignite.configuration.BinaryConfiguration;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.Nullable;
 
@@ -193,7 +193,7 @@ private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate) {
             }
 
             /** {@inheritDoc} */
-            @Override public void onDiscovery(
+            @Override public IgniteInternalFuture onDiscovery(
                     int type,
                     long topVer,
                     ClusterNode node,
@@ -219,7 +219,9 @@ else if (conflClsName.contains(BB.class.getSimpleName()))
                 }
 
                 if (delegate != null)
-                    delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg);
+                    return delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg);
+
+                return new GridFinishedFuture();
             }
 
             /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java
index 49f5311bf0e7b..7aa61ebd8f035 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java
@@ -34,8 +34,10 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.PersistentStoreConfiguration;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.processors.marshaller.MappingProposedMessage;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
@@ -243,7 +245,7 @@ private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate) {
             }
 
             /** {@inheritDoc} */
-            @Override public void onDiscovery(
+            @Override public IgniteInternalFuture onDiscovery(
                 int type,
                 long topVer,
                 ClusterNode node,
@@ -267,7 +269,9 @@ private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate) {
                 }
 
                 if (delegate != null)
-                    delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg);
+                    return delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg);
+
+                return new GridFinishedFuture();
             }
 
             /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
index 954e0c63b24ad..40fe0f44e9ac7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -380,7 +381,7 @@ private AwaitDiscoverySpiListener(
         }
 
         /** {@inheritDoc} */
-        @Override public void onDiscovery(
+        @Override public IgniteInternalFuture onDiscovery(
             int type,
             long topVer,
             ClusterNode node,
@@ -388,9 +389,9 @@ private AwaitDiscoverySpiListener(
             @Nullable Map<Long, Collection<ClusterNode>> topHist,
             @Nullable DiscoverySpiCustomMessage data
         ) {
-            delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, data);
+            IgniteInternalFuture fut = delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, data);
 
-            if (type == EVT_CLIENT_NODE_DISCONNECTED)
+            if (type == EVT_CLIENT_NODE_DISCONNECTED) {
                 try {
                     System.out.println("Await cluster change state");
 
@@ -399,6 +400,9 @@ private AwaitDiscoverySpiListener(
                 catch (InterruptedException e) {
                     throw new RuntimeException(e);
                 }
+            }
+
+            return fut;
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/AbstractDiscoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/AbstractDiscoverySelfTest.java
index fa1a2ae212b42..e59d24a2c5afc 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/AbstractDiscoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/AbstractDiscoverySelfTest.java
@@ -34,6 +34,8 @@
 import mx4j.tools.adaptor.http.HttpAdaptor;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.spi.IgniteSpi;
@@ -160,10 +162,17 @@ public boolean isMetricsUpdated() {
         }
 
         /** {@inheritDoc} */
-        @Override public void onDiscovery(int type, long topVer, ClusterNode node, Collection<ClusterNode> topSnapshot,
-            Map<Long, Collection<ClusterNode>> topHist, @Nullable DiscoverySpiCustomMessage data) {
+        @Override public IgniteInternalFuture onDiscovery(
+            int type,
+            long topVer,
+            ClusterNode node,
+            Collection<ClusterNode> topSnapshot,
+            Map<Long, Collection<ClusterNode>> topHist, @Nullable DiscoverySpiCustomMessage data
+        ) {
             if (type == EVT_NODE_METRICS_UPDATED)
                 isMetricsUpdate = true;
+
+            return new GridFinishedFuture();
         }
     }
 
@@ -237,13 +246,15 @@ public void testLocalMetricsUpdate() throws Exception {
                     // No-op.
                 }
 
-                @Override public void onDiscovery(int type, long topVer, ClusterNode node,
+                @Override public IgniteInternalFuture onDiscovery(int type, long topVer, ClusterNode node,
                     Collection<ClusterNode> topSnapshot, Map<Long, Collection<ClusterNode>> topHist,
                     @Nullable DiscoverySpiCustomMessage data) {
                     // If METRICS_UPDATED came from local node
                     if (type == EVT_NODE_METRICS_UPDATED
                         && node.id().equals(spi.getLocalNode().id()))
                         spiCnt.addAndGet(1);
+
+                    return new GridFinishedFuture();
                 }
             };
 
@@ -405,7 +416,7 @@ protected long getMaxMetricsWaitTime() {
                     }
 
                     @SuppressWarnings({"NakedNotify"})
-                    @Override public void onDiscovery(int type, long topVer, ClusterNode node,
+                    @Override public IgniteInternalFuture onDiscovery(int type, long topVer, ClusterNode node,
                         Collection<ClusterNode> topSnapshot, Map<Long, Collection<ClusterNode>> topHist,
                         @Nullable DiscoverySpiCustomMessage data) {
                         info("Discovery event [type=" + type + ", node=" + node + ']');
@@ -413,6 +424,8 @@ protected long getMaxMetricsWaitTime() {
                         synchronized (mux) {
                             mux.notifyAll();
                         }
+
+                        return new GridFinishedFuture();
                     }
                 });
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 9390d6b36fe69..9d99158a57d8b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -156,9 +156,10 @@ private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate, DiscoveryHook
         }
 
         /** {@inheritDoc} */
-        @Override public void onDiscovery(int type, long topVer, ClusterNode node, Collection<ClusterNode> topSnapshot, @Nullable Map<Long, Collection<ClusterNode>> topHist, @Nullable DiscoverySpiCustomMessage spiCustomMsg) {
+        @Override public IgniteInternalFuture onDiscovery(int type, long topVer, ClusterNode node, Collection<ClusterNode> topSnapshot, @Nullable Map<Long, Collection<ClusterNode>> topHist, @Nullable DiscoverySpiCustomMessage spiCustomMsg) {
             hook.handleDiscoveryMessage(spiCustomMsg);
-            delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg);
+
+            return delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index f480933b1a108..3ad8e179c20b8 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -466,12 +466,17 @@ private void doReconnect(UUID newId) {
         if (rtState.joined) {
             assert rtState.evtsData != null;
 
-            lsnr.onDiscovery(EVT_CLIENT_NODE_DISCONNECTED,
-                rtState.evtsData.topVer,
-                locNode,
-                rtState.top.topologySnapshot(),
-                Collections.<Long, Collection<ClusterNode>>emptyMap(),
-                null);
+            try {
+                lsnr.onDiscovery(EVT_CLIENT_NODE_DISCONNECTED,
+                    rtState.evtsData.topVer,
+                    locNode,
+                    rtState.top.topologySnapshot(),
+                    Collections.emptyMap(),
+                    null).get();
+            }
+            catch (IgniteCheckedException e) {
+                throw new IgniteException("Failed to wait for discovery listener notification", e);
+            }
         }
 
         try {
@@ -532,14 +537,19 @@ private void notifySegmented() {
         List<ClusterNode> nodes = rtState.top.topologySnapshot();
 
         if (nodes.isEmpty())
-            nodes = Collections.singletonList((ClusterNode)locNode);
+            nodes = Collections.singletonList(locNode);
 
-        lsnr.onDiscovery(EVT_NODE_SEGMENTED,
-            rtState.evtsData != null ? rtState.evtsData.topVer : 1L,
-            locNode,
-            nodes,
-            Collections.<Long, Collection<ClusterNode>>emptyMap(),
-            null);
+        try {
+            lsnr.onDiscovery(EVT_NODE_SEGMENTED,
+                rtState.evtsData != null ? rtState.evtsData.topVer : 1L,
+                locNode,
+                nodes,
+                Collections.emptyMap(),
+                null).get();
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException("Failed to wait for discovery listener notification", e);
+        }
     }
 
     /**
@@ -2246,12 +2256,19 @@ private void newClusterStarted(@Nullable ZkDiscoveryEventsData prevEvts) throws
 
         final List<ClusterNode> topSnapshot = Collections.singletonList((ClusterNode)locNode);
 
-        lsnr.onDiscovery(EVT_NODE_JOINED,
-            1L,
-            locNode,
-            topSnapshot,
-            Collections.<Long, Collection<ClusterNode>>emptyMap(),
-            null);
+        try {
+            lsnr.onDiscovery(EVT_NODE_JOINED,
+                1L,
+                locNode,
+                topSnapshot,
+                Collections.emptyMap(),
+                null).get();
+        }
+        catch (IgniteCheckedException e) {
+            joinFut.onDone(e);
+
+            throw new IgniteException("Failed to wait for discovery listener notification", e);
+        }
 
         // Reset events (this is also notification for clients left from previous cluster).
         rtState.zkClient.setData(zkPaths.evtsPath, marshalZip(rtState.evtsData), -1);
@@ -2960,16 +2977,16 @@ private void processLocalJoin(ZkDiscoveryEventsData evtsData,
                 joinedEvtData.topVer,
                 locNode,
                 topSnapshot,
-                Collections.<Long, Collection<ClusterNode>>emptyMap(),
-                null);
+                Collections.emptyMap(),
+                null).get();
 
             if (rtState.prevJoined) {
                 lsnr.onDiscovery(EVT_CLIENT_NODE_RECONNECTED,
                     joinedEvtData.topVer,
                     locNode,
                     topSnapshot,
-                    Collections.<Long, Collection<ClusterNode>>emptyMap(),
-                    null);
+                    Collections.emptyMap(),
+                    null).get();
 
                 U.quietAndWarn(log, "Client node was reconnected after it was already considered failed [locId=" + locNode.id() + ']');
             }
@@ -3420,13 +3437,23 @@ private void notifyCustomEvent(final ZkDiscoveryCustomEventData evtData, final D
 
         final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
 
-        lsnr.onDiscovery(
+        IgniteInternalFuture fut = lsnr.onDiscovery(
             DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT,
             evtData.topologyVersion(),
             sndNode,
             topSnapshot,
-            Collections.<Long, Collection<ClusterNode>>emptyMap(),
-            msg);
+            Collections.emptyMap(),
+            msg
+        );
+
+        if (msg != null && msg.isMutable()) {
+            try {
+                fut.get();
+            }
+            catch (IgniteCheckedException e) {
+                throw new IgniteException("Failed to wait for discovery listener notification", e);
+            }
+        }
     }
 
     /**
@@ -3444,12 +3471,17 @@ private void notifyNodeJoin(ZkJoinedNodeEvtData joinedEvtData, ZkJoiningNodeData
 
         final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
 
-        lsnr.onDiscovery(EVT_NODE_JOINED,
-            joinedEvtData.topVer,
-            joinedNode,
-            topSnapshot,
-            Collections.<Long, Collection<ClusterNode>>emptyMap(),
-            null);
+        try {
+            lsnr.onDiscovery(EVT_NODE_JOINED,
+                joinedEvtData.topVer,
+                joinedNode,
+                topSnapshot,
+                Collections.emptyMap(),
+                null).get();
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException("Failed to wait for discovery listener notification", e);
+        }
     }
 
     /**
@@ -3475,12 +3507,17 @@ private void notifyNodeFail(long nodeInternalOrder, long topVer) {
 
         final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
 
-        lsnr.onDiscovery(EVT_NODE_FAILED,
-            topVer,
-            failedNode,
-            topSnapshot,
-            Collections.<Long, Collection<ClusterNode>>emptyMap(),
-            null);
+        try {
+            lsnr.onDiscovery(EVT_NODE_FAILED,
+                topVer,
+                failedNode,
+                topSnapshot,
+                Collections.emptyMap(),
+                null).get();
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException("Failed to wait for discovery listener notification", e);
+        }
 
         stats.onNodeFailed();
     }

From b28f04510738d95090b3705ace0f61aa188e5eb3 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Mon, 20 Aug 2018 12:45:57 +0300
Subject: [PATCH 359/543] IGNITE-9294 StandaloneWalRecordsIterator: support
 iteration from custom pointer - Fixes #4563.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 0a19d010fddae9a48b8a954b46c9864f1d5ceba8)
---
 .../wal/AbstractWalRecordsIterator.java       |   2 +-
 .../wal/reader/IgniteWalIteratorFactory.java  |  70 +++++++
 .../reader/StandaloneWalRecordsIterator.java  |  80 +++++++-
 .../db/wal/reader/IgniteWalReaderTest.java    | 176 +++++++++++++++++-
 4 files changed, 313 insertions(+), 15 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index 27f01022bd110..9fbb53566b904 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -228,7 +228,7 @@ protected abstract AbstractReadFileHandle advanceSegment(
      * @param hnd currently opened read handle.
      * @return next advanced record.
      */
-    private IgniteBiTuple<WALPointer, WALRecord> advanceRecord(
+    protected IgniteBiTuple<WALPointer, WALRecord> advanceRecord(
         @Nullable final AbstractReadFileHandle hnd
     ) throws IgniteCheckedException {
         if (hnd == null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index aae9775207612..cda8d90174281 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -103,6 +103,26 @@ public WALIterator iterator(
         return iterator(new IteratorParametersBuilder().filesOrDirs(filesOrDirs));
     }
 
+    /**
+     * Creates iterator for file by file scan mode.
+     * This method may be used for work folder, file indexes are scanned from the file context.
+     * In this mode only provided WAL segments will be scanned. New WAL files created during iteration will be ignored.
+     *
+     * @param replayFrom File WAL pointer for start replay.
+     * @param filesOrDirs files to scan. A file can be the path to '.wal' file, or directory with '.wal' files.
+     * Order is not important, but it is significant to provide all segments without omissions.
+     * Path should not contain special symbols. Special symbols should be already masked.
+     * @return closable WAL records iterator, should be closed when non needed.
+     * @throws IgniteCheckedException if failed to read files
+     * @throws IllegalArgumentException If parameter validation failed.
+     */
+    public WALIterator iterator(
+        @NotNull FileWALPointer replayFrom,
+        @NotNull File... filesOrDirs
+    ) throws IgniteCheckedException, IllegalArgumentException {
+        return iterator(new IteratorParametersBuilder().from(replayFrom).filesOrDirs(filesOrDirs));
+    }
+
     /**
      * Creates iterator for file by file scan mode.
      * This method may be used for work folder, file indexes are scanned from the file context.
@@ -121,6 +141,26 @@ public WALIterator iterator(
         return iterator(new IteratorParametersBuilder().filesOrDirs(filesOrDirs));
     }
 
+    /**
+     * Creates iterator for file by file scan mode.
+     * This method may be used for work folder, file indexes are scanned from the file context.
+     * In this mode only provided WAL segments will be scanned. New WAL files created during iteration will be ignored.
+     *
+     * @param replayFrom File WAL pointer for start replay.
+     * @param filesOrDirs paths to scan. A path can be direct to '.wal' file, or directory with '.wal' files.
+     * Order is not important, but it is significant to provide all segments without omissions.
+     * Path should not contain special symbols. Special symbols should be already masked.
+     * @return closable WAL records iterator, should be closed when non needed.
+     * @throws IgniteCheckedException If failed to read files.
+     * @throws IllegalArgumentException If parameter validation failed.
+     */
+    public WALIterator iterator(
+        @NotNull FileWALPointer replayFrom,
+        @NotNull String... filesOrDirs
+    ) throws IgniteCheckedException, IllegalArgumentException {
+        return iterator(new IteratorParametersBuilder().from(replayFrom).filesOrDirs(filesOrDirs));
+    }
+
     /**
      * @param iteratorParametersBuilder Iterator parameters builder.
      * @return closable WAL records iterator, should be closed when non needed
@@ -135,6 +175,8 @@ public WALIterator iterator(
             iteratorParametersBuilder.ioFactory,
             resolveWalFiles(iteratorParametersBuilder),
             iteratorParametersBuilder.filter,
+            iteratorParametersBuilder.lowBound,
+            iteratorParametersBuilder.highBound,
             iteratorParametersBuilder.keepBinary,
             iteratorParametersBuilder.bufferSize
         );
@@ -361,6 +403,12 @@ public static class IteratorParametersBuilder {
         /** */
         @Nullable private IgniteBiPredicate<RecordType, WALPointer> filter;
 
+        /** */
+        private FileWALPointer lowBound = new FileWALPointer(Long.MIN_VALUE, 0, 0);
+
+        /** */
+        private FileWALPointer highBound = new FileWALPointer(Long.MAX_VALUE, Integer.MAX_VALUE, 0);
+
         /**
          * @param filesOrDirs Paths to files or directories.
          * @return IteratorParametersBuilder Self reference.
@@ -457,6 +505,26 @@ public IteratorParametersBuilder filter(IgniteBiPredicate<RecordType, WALPointer
             return this;
         }
 
+        /**
+         * @param lowBound WAL pointer to start from.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder from(FileWALPointer lowBound) {
+            this.lowBound = lowBound;
+
+            return this;
+        }
+
+        /**
+         * @param highBound WAL pointer to end of.
+         * @return IteratorParametersBuilder Self reference.
+         */
+        public IteratorParametersBuilder to(FileWALPointer highBound) {
+            this.highBound = highBound;
+
+            return this;
+        }
+
         /**
          * Copy current state of builder to new instance.
          *
@@ -471,6 +539,8 @@ public IteratorParametersBuilder copy() {
                 .ioFactory(ioFactory)
                 .binaryMetadataFileStoreDir(binaryMetadataFileStoreDir)
                 .marshallerMappingFileStoreDir(marshallerMappingFileStoreDir)
+                .from(lowBound)
+                .to(highBound)
                 .filter(filter);
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index d92c0e4029cdb..e934f33bfd2bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.FilteredRecord;
 import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.UnwrapDataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
@@ -48,7 +49,9 @@
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -77,6 +80,12 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
     /** Keep binary. This flag disables converting of non primitive types (BinaryObjects) */
     private boolean keepBinary;
 
+    /** Replay from bound include. */
+    private final FileWALPointer lowBound;
+
+    /** Replay to bound include */
+    private final FileWALPointer highBound;
+
     /**
      * Creates iterator in file-by-file iteration mode. Directory
      * @param log Logger.
@@ -93,6 +102,8 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
         @NotNull FileIOFactory ioFactory,
         @NotNull List<FileDescriptor> walFiles,
         IgniteBiPredicate<RecordType, WALPointer> readTypeFilter,
+        FileWALPointer lowBound,
+        FileWALPointer highBound,
         boolean keepBinary,
         int initialReadBufferSize
     ) throws IgniteCheckedException {
@@ -104,6 +115,9 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
             initialReadBufferSize
         );
 
+        this.lowBound = lowBound;
+        this.highBound = highBound;
+
         this.keepBinary = keepBinary;
 
         walFileDescriptors = walFiles;
@@ -156,14 +170,19 @@ private void init(List<FileDescriptor> walFiles) {
         if (curWalSegment != null)
             curWalSegment.close();
 
-        curWalSegmIdx++;
+        FileDescriptor fd;
 
-        curIdx++;
+        do {
+            curWalSegmIdx++;
 
-        if (curIdx >= walFileDescriptors.size())
-            return null;
+            curIdx++;
 
-        FileDescriptor fd = walFileDescriptors.get(curIdx);
+            if (curIdx >= walFileDescriptors.size())
+                return null;
+
+            fd = walFileDescriptors.get(curIdx);
+        }
+        while (!checkBounds(fd.idx()));
 
         if (log.isDebugEnabled())
             log.debug("Reading next file [absIdx=" + curWalSegmIdx + ", file=" + fd.file().getAbsolutePath() + ']');
@@ -173,7 +192,12 @@ private void init(List<FileDescriptor> walFiles) {
         curRec = null;
 
         try {
-            return initReadHandle(fd, null);
+            FileWALPointer initPtr = null;
+
+            if (lowBound.index() == fd.idx())
+                initPtr = lowBound;
+
+            return initReadHandle(fd, initPtr);
         }
         catch (FileNotFoundException e) {
             if (log.isInfoEnabled())
@@ -183,6 +207,50 @@ private void init(List<FileDescriptor> walFiles) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override protected IgniteBiTuple<WALPointer, WALRecord> advanceRecord(
+        @Nullable AbstractReadFileHandle hnd
+    ) throws IgniteCheckedException {
+        IgniteBiTuple<WALPointer, WALRecord> tup = super.advanceRecord(hnd);
+
+        if (tup == null)
+            return tup;
+
+        if (!checkBounds(tup.get1())) {
+            if (curRec != null) {
+                FileWALPointer prevRecPtr = (FileWALPointer)curRec.get1();
+
+                // Fast stop condition, after high bound reached.
+                if (prevRecPtr != null && prevRecPtr.compareTo(highBound) > 0)
+                    return null;
+            }
+
+            return new T2<>(tup.get1(), FilteredRecord.INSTANCE); // FilteredRecord for mark as filtered.
+        }
+
+        return tup;
+    }
+
+    /**
+     *
+     * @param ptr WAL pointer.
+     * @return {@code True} If pointer between low and high bounds. {@code False} if not.
+     */
+    private boolean checkBounds(WALPointer ptr) {
+        FileWALPointer ptr0 = (FileWALPointer)ptr;
+
+        return ptr0.compareTo(lowBound) >= 0 && ptr0.compareTo(highBound) <= 0;
+    }
+
+    /**
+     *
+     * @param idx WAL segment index.
+     * @return {@code True} If pointer between low and high bounds. {@code False} if not.
+     */
+    private boolean checkBounds(long idx) {
+        return idx >= lowBound.index() && idx <= highBound.index();
+    }
+
     /** {@inheritDoc} */
     @Override protected AbstractReadFileHandle initReadHandle(
         @NotNull AbstractFileDescriptor desc,
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
index a6183a9073b9a..cf427169f30c4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
@@ -23,6 +23,7 @@
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.EnumMap;
 import java.util.HashMap;
@@ -31,6 +32,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Random;
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
@@ -40,6 +42,7 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.IgniteEvents;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryObject;
@@ -64,9 +67,11 @@
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -146,7 +151,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
                 new DataRegionConfiguration()
                     .setMaxSize(1024 * 1024 * 1024)
                     .setPersistenceEnabled(true))
-            .setWalHistorySize(1)
             .setWalSegmentSize(1024 * 1024)
             .setWalSegments(WAL_SEGMENTS)
             .setWalMode(customWalMode != null ? customWalMode : WALMode.BACKGROUND);
@@ -163,7 +167,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
 
             dsCfg.setWalPath(walAbsPath);
             dsCfg.setWalArchivePath(walAbsPath);
-        } else {
+        }
+        else {
             dsCfg.setWalPath(wal.getAbsolutePath());
             dsCfg.setWalArchivePath(new File(wal, "archive").getAbsolutePath());
         }
@@ -309,7 +314,7 @@ public void testArchiveCompletedEventFired() throws Exception {
             return true;
         }, EVT_WAL_SEGMENT_ARCHIVED);
 
-        putDummyRecords(ignite, 500);
+        putDummyRecords(ignite, 5_000);
 
         stopGrid();
 
@@ -472,7 +477,7 @@ public void testTxFillWalAndExtractDataRecords() throws Exception {
 
         String workDir = U.defaultWorkDirectory();
 
-        IteratorParametersBuilder params = createIteratorParametersBuilder(workDir,subfolderName);
+        IteratorParametersBuilder params = createIteratorParametersBuilder(workDir, subfolderName);
 
         params.filesOrDirs(workDir);
 
@@ -738,7 +743,7 @@ else if (val12 instanceof BinaryObject) {
             ctrlMapForBinaryObjects, ctrlMapForBinaryObjects.isEmpty());
 
         assertTrue(" Control Map for strings in entries is not empty after" +
-                " reading records: " + ctrlStringsForBinaryObjSearch, ctrlStringsForBinaryObjSearch.isEmpty());
+            " reading records: " + ctrlStringsForBinaryObjSearch, ctrlStringsForBinaryObjSearch.isEmpty());
     }
 
     /**
@@ -765,7 +770,7 @@ public void testReadEmptyWal() throws Exception {
 
         IteratorParametersBuilder iteratorParametersBuilder =
             createIteratorParametersBuilder(workDir, subfolderName)
-            .filesOrDirs(workDir);
+                .filesOrDirs(workDir);
 
         scanIterateAndCount(
             factory,
@@ -825,12 +830,11 @@ public void testRemoveOperationPresentedForDataEntryForAtomic() throws Exception
         runRemoveOperationTest(CacheAtomicityMode.ATOMIC);
     }
 
-
     /**
      * Test if DELETE operation can be found after mixed cache operations including remove().
      *
-     * @throws Exception if failed.
      * @param mode Cache Atomicity Mode.
+     * @throws Exception if failed.
      */
     private void runRemoveOperationTest(CacheAtomicityMode mode) throws Exception {
         Ignite ignite = startGrid();
@@ -1046,6 +1050,162 @@ public void testTxRecordsReadWoBinaryMeta() throws Exception {
         );
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCheckBoundsIterator() throws Exception {
+        Ignite ignite = startGrid("node0");
+
+        ignite.cluster().active(true);
+
+        try (IgniteDataStreamer<Integer, IndexedObject> st = ignite.dataStreamer(CACHE_NAME)) {
+            st.allowOverwrite(true);
+
+            for (int i = 0; i < 10_000; i++)
+                st.addData(i, new IndexedObject(i));
+        }
+
+        stopAllGrids();
+
+        List<FileWALPointer> wal = new ArrayList<>();
+
+        String workDir = U.defaultWorkDirectory();
+
+        IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory();
+
+        try (WALIterator it = factory.iterator(workDir)) {
+            while (it.hasNext()) {
+                IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+                wal.add((FileWALPointer)tup.get1());
+            }
+        }
+
+        Random rnd = new Random();
+
+        int from0 = rnd.nextInt(wal.size() - 2) + 1;
+        int to0 = wal.size() - 1;
+
+        // +1 for skip first record.
+        FileWALPointer exp0First = wal.get(from0);
+        FileWALPointer exp0Last = wal.get(to0);
+
+        T2<FileWALPointer, WALRecord> actl0First = null;
+        T2<FileWALPointer, WALRecord> actl0Last = null;
+
+        int records0 = 0;
+
+        try (WALIterator it = factory.iterator(exp0First, workDir)) {
+            while (it.hasNext()) {
+                IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+                if (actl0First == null)
+                    actl0First = new T2<>((FileWALPointer)tup.get1(), tup.get2());
+
+                actl0Last = new T2<>((FileWALPointer)tup.get1(), tup.get2());
+
+                records0++;
+            }
+        }
+
+        log.info("Check REPLAY FROM:" + exp0First + "\n" +
+            "expFirst=" + exp0First + " actlFirst=" + actl0First + ", " +
+            "expLast=" + exp0Last + " actlLast=" + actl0Last);
+
+        // +1 because bound include.
+        Assert.assertEquals(to0 - from0 + 1, records0);
+
+        Assert.assertNotNull(actl0First);
+        Assert.assertNotNull(actl0Last);
+
+        Assert.assertEquals(exp0First, actl0First.get1());
+        Assert.assertEquals(exp0Last, actl0Last.get1());
+
+        int from1 = 0;
+        int to1 = rnd.nextInt(wal.size() - 3) + 1;
+
+        // -3 for skip last record.
+        FileWALPointer exp1First = wal.get(from1);
+        FileWALPointer exp1Last = wal.get(to1);
+
+        T2<FileWALPointer, WALRecord> actl1First = null;
+        T2<FileWALPointer, WALRecord> actl1Last = null;
+
+        int records1 = 0;
+
+        try (WALIterator it = factory.iterator(
+            new IteratorParametersBuilder()
+                .filesOrDirs(workDir)
+                .to(exp1Last)
+        )) {
+            while (it.hasNext()) {
+                IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+                if (actl1First == null)
+                    actl1First = new T2<>((FileWALPointer)tup.get1(), tup.get2());
+
+                actl1Last = new T2<>((FileWALPointer)tup.get1(), tup.get2());
+
+                records1++;
+            }
+        }
+
+        log.info("Check REPLAY TO:" + exp1Last + "\n" +
+            "expFirst=" + exp1First + " actlFirst=" + actl1First + ", " +
+            "expLast=" + exp1Last + " actlLast=" + actl1Last);
+
+        // +1 because bound include.
+        Assert.assertEquals(to1 - from1 + 1, records1);
+
+        Assert.assertNotNull(actl1First);
+        Assert.assertNotNull(actl1Last);
+
+        Assert.assertEquals(exp1First, actl1First.get1());
+        Assert.assertEquals(exp1Last, actl1Last.get1());
+
+        int from2 = rnd.nextInt(wal.size() - 2);
+        int to2 = rnd.nextInt((wal.size() - 1) - from2) + from2;
+
+        FileWALPointer exp2First = wal.get(from2);
+        FileWALPointer exp2Last = wal.get(to2);
+
+        T2<FileWALPointer, WALRecord> actl2First = null;
+        T2<FileWALPointer, WALRecord> actl2Last = null;
+
+        int records2 = 0;
+
+        try (WALIterator it = factory.iterator(
+            new IteratorParametersBuilder()
+                .filesOrDirs(workDir)
+                .from(exp2First)
+                .to(exp2Last)
+        )) {
+            while (it.hasNext()) {
+                IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+                if (actl2First == null)
+                    actl2First = new T2<>((FileWALPointer)tup.get1(), tup.get2());
+
+                actl2Last = new T2<>((FileWALPointer)tup.get1(), tup.get2());
+
+                records2++;
+            }
+        }
+
+        log.info("Check REPLAY BETWEEN:" + exp2First + " " + exp2Last+ "\n" +
+            "expFirst=" + exp2First + " actlFirst=" + actl2First + ", " +
+            "expLast=" + exp2Last + " actlLast=" + actl2Last);
+
+        // +1 because bound include.
+        Assert.assertEquals(to2 - from2 + 1, records2);
+
+        Assert.assertNotNull(actl2First);
+        Assert.assertNotNull(actl2Last);
+
+        Assert.assertEquals(exp2First, actl2First.get1());
+        Assert.assertEquals(exp2Last, actl2Last.get1());
+    }
+
     /**
      * @param workDir Work directory.
      * @param subfolderName Subfolder name.

From 179b36c61720253a2d6a9d663f4265894b322f21 Mon Sep 17 00:00:00 2001
From: ibessonov <bessonov.ip@gmail.com>
Date: Tue, 18 Sep 2018 12:44:06 +0300
Subject: [PATCH 360/543] IGNITE-9441 Improved handling of invalid CRC for
 WALIterator, in the end or in middle of WAL - Fixes #4714.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit e153114d4312b9e4def3e8e007720a99cabe6a76)
---
 .../wal/FileWriteAheadLogManager.java         |  46 +++
 .../FsyncModeFileWriteAheadLogManager.java    |  46 +++
 .../wal/reader/IgniteWalIteratorFactory.java  |  10 +-
 .../reader/StandaloneWalRecordsIterator.java  |  20 ++
 ...niteAbstractWalIteratorInvalidCrcTest.java | 279 ++++++++++++++++++
 ...eFsyncReplayWalIteratorInvalidCrcTest.java |  31 ++
 ...IgniteReplayWalIteratorInvalidCrcTest.java |  54 ++++
 ...teStandaloneWalIteratorInvalidCrcTest.java |  50 ++++
 .../testsuites/IgnitePdsTestSuite2.java       |  12 +-
 9 files changed, 543 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteFsyncReplayWalIteratorInvalidCrcTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteReplayWalIteratorInvalidCrcTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteStandaloneWalIteratorInvalidCrcTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index d3dc108be6d35..60ee0a284bcce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -94,6 +94,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
 import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator.AbstractFileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
@@ -110,6 +111,7 @@
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.CO;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
@@ -3209,6 +3211,50 @@ private void init() throws IgniteCheckedException {
             return nextHandle;
         }
 
+        /** {@inheritDoc} */
+        @Override protected IgniteCheckedException handleRecordException(
+            @NotNull Exception e,
+            @Nullable FileWALPointer ptr) {
+
+            if (e instanceof IgniteCheckedException)
+                if (X.hasCause(e, IgniteDataIntegrityViolationException.class))
+                    // This means that there is no explicit last sengment, so we iterate unil the very end.
+                    if (end == null) {
+                        long nextWalSegmentIdx = curWalSegmIdx + 1;
+
+                        // Check that we should not look this segment up in archive directory.
+                        // Basically the same check as in "advanceSegment" method.
+                        if (archiver != null)
+                            if (!canReadArchiveOrReserveWork(nextWalSegmentIdx))
+                                try {
+                                    long workIdx = nextWalSegmentIdx % dsCfg.getWalSegments();
+
+                                    FileDescriptor fd = new FileDescriptor(
+                                        new File(walWorkDir, FileDescriptor.fileName(workIdx)),
+                                        nextWalSegmentIdx
+                                    );
+
+                                    try {
+                                        ReadFileHandle nextHandle = initReadHandle(fd, null);
+
+                                        // "nextHandle == null" is true only if current segment is the last one in the
+                                        // whole history. Only in such case we ignore crc validation error and just stop
+                                        // as if we reached the end of the WAL.
+                                        if (nextHandle == null)
+                                            return null;
+                                    }
+                                    catch (IgniteCheckedException | FileNotFoundException initReadHandleException) {
+                                        e.addSuppressed(initReadHandleException);
+                                    }
+                                }
+                                finally {
+                                    releaseWorkSegment(nextWalSegmentIdx);
+                                }
+                    }
+
+            return super.handleRecordException(e, ptr);
+        }
+
         /**
          * @param absIdx Absolute index to check.
          * @return <ul><li> {@code True} if we can safely read the archive,  </li> <li>{@code false} if the segment has
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 38681b5f6f6a8..45f2eaf5b4d0d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -87,6 +87,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
@@ -102,6 +103,7 @@
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.CO;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -3320,6 +3322,50 @@ private void init() throws IgniteCheckedException {
             return nextHandle;
         }
 
+        /** {@inheritDoc} */
+        @Override protected IgniteCheckedException handleRecordException(
+            @NotNull Exception e,
+            @Nullable FileWALPointer ptr) {
+
+            if (e instanceof IgniteCheckedException)
+                if (X.hasCause(e, IgniteDataIntegrityViolationException.class))
+                    // This means that there is no explicit last sengment, so we iterate unil the very end.
+                    if (end == null) {
+                        long nextWalSegmentIdx = curWalSegmIdx + 1;
+
+                        // Check that we should not look this segment up in archive directory.
+                        // Basically the same check as in "advanceSegment" method.
+                        if (archiver != null)
+                            if (!canReadArchiveOrReserveWork(nextWalSegmentIdx))
+                                try {
+                                    long workIdx = nextWalSegmentIdx % dsCfg.getWalSegments();
+
+                                    FileDescriptor fd = new FileDescriptor(
+                                        new File(walWorkDir, FileDescriptor.fileName(workIdx)),
+                                        nextWalSegmentIdx
+                                    );
+
+                                    try {
+                                        ReadFileHandle nextHandle = initReadHandle(fd, null);
+
+                                        // "nextHandle == null" is true only if current segment is the last one in the
+                                        // whole history. Only in such case we ignore crc validation error and just stop
+                                        // as if we reached the end of the WAL.
+                                        if (nextHandle == null)
+                                            return null;
+                                    }
+                                    catch (IgniteCheckedException | FileNotFoundException initReadHandleException) {
+                                        e.addSuppressed(initReadHandleException);
+                                    }
+                                }
+                                finally {
+                                    releaseWorkSegment(nextWalSegmentIdx);
+                                }
+                    }
+
+            return super.handleRecordException(e, ptr);
+        }
+
         /**
          * @param absIdx Absolute index to check.
          * @return <ul><li> {@code True} if we can safely read the archive,  </li> <li>{@code false} if the segment has
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index cda8d90174281..14bca7fe87fc8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -372,6 +372,12 @@ private FileDescriptor readFileDescriptor(File file, FileIOFactory ioFactory) {
      * Wal iterator parameter builder.
      */
     public static class IteratorParametersBuilder {
+        /** */
+        public static final FileWALPointer DFLT_LOW_BOUND = new FileWALPointer(Long.MIN_VALUE, 0, 0);
+
+        /** */
+        public static final FileWALPointer DFLT_HIGH_BOUND = new FileWALPointer(Long.MAX_VALUE, Integer.MAX_VALUE, 0);
+
         /** */
         private File[] filesOrDirs;
 
@@ -404,10 +410,10 @@ public static class IteratorParametersBuilder {
         @Nullable private IgniteBiPredicate<RecordType, WALPointer> filter;
 
         /** */
-        private FileWALPointer lowBound = new FileWALPointer(Long.MIN_VALUE, 0, 0);
+        private FileWALPointer lowBound = DFLT_LOW_BOUND;
 
         /** */
-        private FileWALPointer highBound = new FileWALPointer(Long.MAX_VALUE, Integer.MAX_VALUE, 0);
+        private FileWALPointer highBound = DFLT_HIGH_BOUND;
 
         /**
          * @param filesOrDirs Paths to files or directories.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index e934f33bfd2bb..04e72df8c6532 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -46,15 +46,18 @@
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.ReadFileHandle;
 import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder.DFLT_HIGH_BOUND;
 import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader;
 
 /**
@@ -299,6 +302,23 @@ private boolean checkBounds(long idx) {
         return super.postProcessRecord(rec);
     }
 
+    /** {@inheritDoc} */
+    @Override protected IgniteCheckedException handleRecordException(
+        @NotNull Exception e,
+        @Nullable FileWALPointer ptr
+    ) {
+        if (e instanceof IgniteCheckedException)
+            if (X.hasCause(e, IgniteDataIntegrityViolationException.class))
+                // "curIdx" is an index in walFileDescriptors list.
+                if (curIdx == walFileDescriptors.size() - 1)
+                    // This means that there is no explicit last sengment, so we stop as if we reached the end
+                    // of the WAL.
+                    if (highBound.equals(DFLT_HIGH_BOUND))
+                        return null;
+
+        return super.handleRecordException(e, ptr);
+    }
+
     /**
      * Performs post processing of lazy data record, converts it to unwrap record.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java
new file mode 100644
index 0000000000000..866b474a1fbaa
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal.crc;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.function.BiFunction;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
+
+import static java.nio.ByteBuffer.allocate;
+import static java.nio.file.StandardOpenOption.WRITE;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.CRC_SIZE;
+
+/**
+ *
+ */
+public abstract class IgniteAbstractWalIteratorInvalidCrcTest extends GridCommonAbstractTest {
+    /** IP finder. */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** Size of inserting dummy value. */
+    private static final int VALUE_SIZE = 4 * 1024;
+
+    /** Size of WAL segment file. */
+    private static final int WAL_SEGMENT_SIZE = 1024 * 1024;
+
+    /** Count of WAL segment files in working directory. */
+    private static final int WAL_SEGMENTS = DataStorageConfiguration.DFLT_WAL_SEGMENTS;
+
+    /** Ignite instance. */
+    protected IgniteEx ignite;
+
+    /** Random instance for utility purposes. */
+    protected Random random = new Random();
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setWalSegmentSize(WAL_SEGMENT_SIZE)
+                .setWalMode(getWalMode())
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                )
+        );
+
+        cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+
+        ignite = (IgniteEx)startGrid();
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Integer, byte[]> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        byte[] val = new byte[VALUE_SIZE];
+
+        // Fill value with random data.
+        random.nextBytes(val);
+
+        // Amount of values that's enough to fill working dir at least twice.
+        int insertingCnt = 2 * WAL_SEGMENT_SIZE * WAL_SEGMENTS / VALUE_SIZE;
+        for (int i = 0; i < insertingCnt; i++)
+            cache.put(i, val);
+
+        ignite.cluster().active(false);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopGrid();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * @return WAL mode that will be used in {@link IgniteConfiguration}.
+     */
+    @NotNull protected abstract WALMode getWalMode();
+
+    /**
+     * Instantiate WAL iterator according to the iterator type of specific implementation.
+     * @param walMgr WAL manager instance.
+     * @param ignoreArchiveDir Do not include archive segments in resulting iterator if this flag is true.
+     * @return WAL iterator instance.
+     * @throws IgniteCheckedException If iterator creation failed for some reason.
+     */
+    @NotNull protected abstract WALIterator getWalIterator(
+        IgniteWriteAheadLogManager walMgr,
+        boolean ignoreArchiveDir
+    ) throws IgniteCheckedException;
+
+    /**
+     * Test that iteration fails if one of archive segments contains record with invalid CRC.
+     * @throws Exception If failed.
+     */
+    public void testArchiveCorruptedPtr() throws Exception {
+        doTest((archiveDescs, descs) -> archiveDescs.get(random.nextInt(archiveDescs.size())), false, true);
+    }
+
+    /**
+     * Test that iteration fails if one of segments in working directory contains record with invalid CRC
+     * and it is not the tail segment.
+     * @throws Exception If failed.
+     */
+    public void testNotTailCorruptedPtr() throws Exception {
+        doTest((archiveDescs, descs) -> descs.get(random.nextInt(descs.size() - 1)), true, true);
+    }
+
+
+    /**
+     * Test that iteration does not fail if tail segment in working directory contains record with invalid CRC.
+     * @throws Exception If failed.
+     */
+    public void testTailCorruptedPtr() throws Exception {
+        doTest((archiveDescs, descs) -> descs.get(descs.size() - 1), false, false);
+    }
+
+    /**
+     * @param descPicker Function that picks WAL segment to corrupt from archive segments list
+     *      and working directory segments list.
+     * @param ignoreArchiveDir Do not iterate over archive segments if this flag is true.
+     * @param shouldFail Whether iteration is axpected to fail or not.
+     * @throws IOException If IO exception.
+     * @throws IgniteCheckedException If iterator failed.
+     */
+    protected void doTest(
+        BiFunction<List<FileDescriptor>, List<FileDescriptor>, FileDescriptor> descPicker,
+        boolean ignoreArchiveDir,
+        boolean shouldFail
+    ) throws IOException, IgniteCheckedException {
+        IgniteWriteAheadLogManager walMgr = ignite.context().cache().context().wal();
+
+        IgniteWalIteratorFactory iterFactory = new IgniteWalIteratorFactory();
+
+        File walArchiveDir = U.field(walMgr, "walArchiveDir");
+        List<FileDescriptor> archiveDescs = iterFactory.resolveWalFiles(
+            new IgniteWalIteratorFactory.IteratorParametersBuilder()
+                .filesOrDirs(walArchiveDir)
+        );
+
+        File walDir = U.field(walMgr, "walWorkDir");
+        List<FileDescriptor> descs = iterFactory.resolveWalFiles(
+            new IgniteWalIteratorFactory.IteratorParametersBuilder()
+                .filesOrDirs(walDir)
+        );
+
+        FileDescriptor corruptedDesc = descPicker.apply(archiveDescs, descs);
+
+        FileWALPointer beforeCorruptedPtr = corruptWalSegmentFile(
+            corruptedDesc,
+            iterFactory
+        );
+
+        if (shouldFail) {
+            FileWALPointer[] lastReadPtrRef = new FileWALPointer[1];
+
+            IgniteException igniteException = (IgniteException) GridTestUtils.assertThrows(log, () -> {
+                try (WALIterator iter = getWalIterator(walMgr, ignoreArchiveDir)) {
+                    for (IgniteBiTuple<WALPointer, WALRecord> tuple : iter) {
+                        FileWALPointer ptr = (FileWALPointer)tuple.get1();
+                        lastReadPtrRef[0] = ptr;
+                    }
+                }
+
+                return null;
+            }, IgniteException.class, "Failed to read WAL record");
+
+            assertTrue(igniteException.hasCause(IgniteDataIntegrityViolationException.class));
+
+            FileWALPointer lastReadPtr = lastReadPtrRef[0];
+            assertNotNull(lastReadPtr);
+
+            // WAL iterator advances to the next record and only then returns current one,
+            // so next record has to be valid as well.
+            assertEquals(lastReadPtr.next(), beforeCorruptedPtr);
+        }
+        else
+            try (WALIterator iter = getWalIterator(walMgr, ignoreArchiveDir)) {
+                while (iter.hasNext())
+                    iter.next();
+            }
+    }
+
+    /**
+     * Put zero CRC in one of records for the specified segment.
+     * @param desc WAL segment descriptor.
+     * @param iterFactory Iterator factory for segment iterating.
+     * @return Descriptor that is located strictly before the corrupted one.
+     * @throws IOException If IO exception.
+     * @throws IgniteCheckedException If iterator failed.
+     */
+    protected FileWALPointer corruptWalSegmentFile(
+        FileDescriptor desc,
+        IgniteWalIteratorFactory iterFactory
+    ) throws IOException, IgniteCheckedException {
+        List<FileWALPointer> pointers = new ArrayList<>();
+
+        try (WALIterator it = iterFactory.iterator(desc.file())) {
+            for (IgniteBiTuple<WALPointer, WALRecord> tuple : it) {
+                pointers.add((FileWALPointer) tuple.get1());
+            }
+        }
+
+        // Should have a previous record to return and another value before that to ensure that "lastReadPtr"
+        // in "doTest" will always exist.
+        int idxCorrupted = 2 + random.nextInt(pointers.size() - 2);
+
+        FileWALPointer pointer = pointers.get(idxCorrupted);
+        int crc32Off = pointer.fileOffset() + pointer.length() - CRC_SIZE;
+
+        ByteBuffer zeroCrc32 = allocate(CRC_SIZE); // Has 0 value by default.
+
+        FileIOFactory ioFactory = new RandomAccessFileIOFactory();
+        try (FileIO io = ioFactory.create(desc.file(), WRITE)) {
+            io.write(zeroCrc32, crc32Off);
+
+            io.force(true);
+        }
+
+        return pointers.get(idxCorrupted - 1);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteFsyncReplayWalIteratorInvalidCrcTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteFsyncReplayWalIteratorInvalidCrcTest.java
new file mode 100644
index 0000000000000..4629acb7f855e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteFsyncReplayWalIteratorInvalidCrcTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal.crc;
+
+import org.apache.ignite.configuration.WALMode;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public class IgniteFsyncReplayWalIteratorInvalidCrcTest extends IgniteReplayWalIteratorInvalidCrcTest {
+    /** {@inheritDoc} */
+    @NotNull @Override protected WALMode getWalMode() {
+        return WALMode.FSYNC;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteReplayWalIteratorInvalidCrcTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteReplayWalIteratorInvalidCrcTest.java
new file mode 100644
index 0000000000000..756ef78798032
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteReplayWalIteratorInvalidCrcTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal.crc;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public class IgniteReplayWalIteratorInvalidCrcTest extends IgniteAbstractWalIteratorInvalidCrcTest {
+    /** {@inheritDoc} */
+    @NotNull @Override protected WALMode getWalMode() {
+        return WALMode.LOG_ONLY;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected WALIterator getWalIterator(
+        IgniteWriteAheadLogManager walMgr,
+        boolean ignoreArchiveDir
+    ) throws IgniteCheckedException {
+        if (ignoreArchiveDir)
+            throw new UnsupportedOperationException(
+                "Cannot invoke \"getWalIterator\" with true \"ignoreArchiveDir\" parameter value."
+            );
+        else
+            return walMgr.replay(null);
+    }
+
+    /**
+     * {@inheritDoc}
+     * Case is not relevant to the replay iterator.
+     */
+    @Override public void testNotTailCorruptedPtr() {
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteStandaloneWalIteratorInvalidCrcTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteStandaloneWalIteratorInvalidCrcTest.java
new file mode 100644
index 0000000000000..8802184336e69
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteStandaloneWalIteratorInvalidCrcTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal.crc;
+
+import java.io.File;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public class IgniteStandaloneWalIteratorInvalidCrcTest extends IgniteAbstractWalIteratorInvalidCrcTest {
+    /** {@inheritDoc} */
+    @NotNull @Override protected WALMode getWalMode() {
+        return WALMode.LOG_ONLY;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected WALIterator getWalIterator(
+        IgniteWriteAheadLogManager walMgr,
+        boolean ignoreArchiveDir
+    ) throws IgniteCheckedException {
+        File walArchiveDir = U.field(walMgr, "walArchiveDir");
+        File walDir = U.field(walMgr, "walWorkDir");
+
+        IgniteWalIteratorFactory iterFactory = new IgniteWalIteratorFactory();
+
+        return ignoreArchiveDir ? iterFactory.iterator(walDir) : iterFactory.iterator(walArchiveDir, walDir);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 08c6d875d099e..0c820e6fbb83d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -32,25 +32,25 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgniteRebalanceScheduleResendPartitionsTest;
 import org.apache.ignite.internal.processors.cache.persistence.LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest;
 import org.apache.ignite.internal.processors.cache.persistence.LocalWalModeChangeDuringRebalancingSelfTest;
-import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAbsentEvictionNodeOutOfBaselineTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.ClientAffinityAssignmentWithBaselineTest;
+import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAbsentEvictionNodeOutOfBaselineTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteAllBaselineNodesOnlineFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOfflineBaselineNodeFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionDuringPartitionClearTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsUnusedWalSegmentsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.SlowHistoricalRebalanceSmallHistoryTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
-import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsUnusedWalSegmentsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.filename.IgniteUidAsConsistentIdMigrationTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteNodeStoppedDuringDisableWALTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWALTailIsReachedDuringIterationOverArchiveTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushBackgroundWithMmapBufferSelfTest;
-import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFailoverTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncWithDedicatedWorkerSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushFsyncWithMmapBufferSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalFlushLogOnlySelfTest;
@@ -62,6 +62,9 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalSerializerVersionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalCompactionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteDataIntegrityTests;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteFsyncReplayWalIteratorInvalidCrcTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteReplayWalIteratorInvalidCrcTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteStandaloneWalIteratorInvalidCrcTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.reader.IgniteWalReaderTest;
 
 /**
@@ -76,6 +79,9 @@ public static TestSuite suite() {
 
         // Integrity test.
         suite.addTestSuite(IgniteDataIntegrityTests.class);
+        suite.addTestSuite(IgniteStandaloneWalIteratorInvalidCrcTest.class);
+        suite.addTestSuite(IgniteReplayWalIteratorInvalidCrcTest.class);
+        suite.addTestSuite(IgniteFsyncReplayWalIteratorInvalidCrcTest.class);
 
         addRealPageStoreTests(suite);
 

From 874c5c3c290e81bd11b0152921d83bd6622a306c Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Tue, 28 Aug 2018 18:18:26 +0300
Subject: [PATCH 361/543] IGNITE-9401 Fixed race in tx rollback test - Fixes
 #4633.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 4924ed430f0adfd9385414d7e341393a00577d1a)
---
 .../processors/cache/transactions/TxRollbackAsyncTest.java   | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
index f2e46d9ea6ccd..85f16192a780c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
@@ -27,6 +27,7 @@
 import java.util.Set;
 import java.util.concurrent.Callable;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicIntegerArray;
 import java.util.concurrent.atomic.AtomicReference;
@@ -898,7 +899,7 @@ public void testRollbackOnTopologyLockPessimistic() throws Exception {
 
                     txLatch.countDown();
 
-                    U.awaitQuiet(commitLatch);
+                    assertTrue(U.await(commitLatch, 10, TimeUnit.SECONDS));
 
                     tx.commit();
 
@@ -929,6 +930,8 @@ public void testRollbackOnTopologyLockPessimistic() throws Exception {
 
                             crd.cache(CACHE_NAME).put(keys.get(0), 0);
 
+                            assertTrue(U.await(commitLatch, 10, TimeUnit.SECONDS));
+
                             tx.commit();
 
                             fail();

From 060961e2aa2969d801064912402eb72f92a8f21e Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Mon, 10 Sep 2018 14:33:40 +0300
Subject: [PATCH 362/543] IGNITE-8509 Fixed and reworkd tx rollback tests -
 Fixes #4150.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit b7a0adc711bb28e7d23f5392bbd588c666cedc22)
---
 .../processors/cache/GridCacheIoManager.java  |   1 -
 .../transactions/TxRollbackAsyncTest.java     | 253 ++++++++++--------
 2 files changed, 139 insertions(+), 115 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 1e25c935f8104..2e66e5bfc3fe4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -611,7 +611,6 @@ private void sendResponseOnFailedMessage(UUID nodeId, GridCacheMessage res, Grid
         }
     }
 
-
     /**
      * @param cacheMsg Cache message.
      * @param nodeId Node ID.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
index 85f16192a780c..4ca8ba37c34ba 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackAsyncTest.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache.transactions;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -25,7 +24,11 @@
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -46,6 +49,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteFutureCancelledCheckedException;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
@@ -55,6 +59,8 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
@@ -81,6 +87,7 @@
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.apache.ignite.transactions.TransactionRollbackException;
 
+import static java.lang.Thread.interrupted;
 import static java.lang.Thread.yield;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
@@ -344,13 +351,12 @@ private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockN
 
         CountDownLatch waitCommit = new CountDownLatch(1);
 
+        // Used for passing tx instance to rollback thread.
         IgniteInternalFuture<?> lockFut = lockInTx(holdLockNode, keyLocked, waitCommit, 0);
 
         U.awaitQuiet(keyLocked);
 
-        final CountDownLatch rollbackLatch = new CountDownLatch(1);
-
-        final int txCnt = 10000;
+        final int txCnt = 1000;
 
         final IgniteKernal k = (IgniteKernal)tryLockNode;
 
@@ -358,7 +364,16 @@ private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockN
 
         final GridCacheContext<Object, Object> cctx = ctx.cacheContext(CU.cacheId(CACHE_NAME));
 
-        final AtomicBoolean stop = new AtomicBoolean();
+        GridFutureAdapter<Transaction> txReadyFut = new GridFutureAdapter<>();
+
+        long seed = System.currentTimeMillis();
+
+        Random r = new Random(seed);
+
+        log.info("Running: node0=" + holdLockNode.cluster().localNode().consistentId() +
+            ", node1=" + tryLockNode.cluster().localNode().consistentId() +
+            ", useTimeout=" + useTimeout +
+            ", seed=" + seed);
 
         IgniteInternalFuture<?> txFut = multithreadedAsync(new Runnable() {
             @Override public void run() {
@@ -369,10 +384,10 @@ private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockN
 
                     assertTrue(tx0 == null || tx0.state() == ROLLED_BACK);
 
-                    rollbackLatch.countDown();
-
                     try (Transaction tx = tryLockNode.transactions().txStart(PESSIMISTIC, REPEATABLE_READ,
-                        useTimeout ? 500 : 0, 1)) {
+                        useTimeout ? 50 : 0, 1)) {
+
+                        txReadyFut.onDone(tx);
 
                         // Will block on lock request until rolled back asynchronously.
                         Object o = tryLockNode.cache(CACHE_NAME).get(0);
@@ -384,29 +399,30 @@ private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockN
                     }
                 }
 
-                stop.set(true);
+                txReadyFut.onDone((Transaction)null);
             }
         }, 1, "tx-get-thread");
 
         IgniteInternalFuture<?> rollbackFut = multithreadedAsync(new Runnable() {
             @Override public void run() {
-                U.awaitQuiet(rollbackLatch);
-
-                doSleep(50);
-
                 Set<IgniteUuid> rolledBackVers = new HashSet<>();
 
                 int proc = 1;
 
-                while(!stop.get()) {
-                    for (Transaction tx : tryLockNode.transactions().localActiveTransactions()) {
+                while(true) {
+                    try {
+                        Transaction tx = txReadyFut.get();
+
+                        txReadyFut.reset();
+
+                        if (tx == null)
+                            break;
+
+                        doSleep(r.nextInt(15)); // Wait a bit to reduce chance of rolling back empty transactions.
+
                         if (rolledBackVers.contains(tx.xid()))
                             fail("Rollback version is expected");
 
-                        // Skip write transaction.
-                        if (LABEL.equals(tx.label()))
-                            continue;
-
                         try {
                             if (proc % 2 == 0)
                                 tx.rollback();
@@ -419,14 +435,15 @@ private void testSynchronousRollback0(Ignite holdLockNode, final Ignite tryLockN
 
                         rolledBackVers.add(tx.xid());
 
-                        if (proc % 1000 == 0)
+                        if (proc % 100 == 0)
                             log.info("Rolled back: " + proc);
 
                         proc++;
                     }
+                    catch (IgniteCheckedException e) {
+                        fail(e.getMessage());
+                    }
                 }
-
-                assertEquals("Unexpected size", txCnt, rolledBackVers.size());
             }
         }, 1, "tx-rollback-thread");
 
@@ -638,134 +655,124 @@ public void testMixedAsyncRollbackTypes() throws Exception {
         final LongAdder failed = new LongAdder();
         final LongAdder rolledBack = new LongAdder();
 
-        IgniteInternalFuture<?> txFut = multithreadedAsync(new Runnable() {
-            @Override public void run() {
-                while (!stop.get()) {
-                    int nodeId = r.nextInt(GRID_CNT + 1);
+        ConcurrentMap<Ignite, BlockingQueue<Transaction>> perNodeTxs = new ConcurrentHashMap<>();
 
-                    // Choose random node to start tx on.
-                    Ignite node = nodeId == GRID_CNT || nearCacheEnabled() ? client : grid(nodeId);
+        for (Ignite ignite : G.allGrids())
+            perNodeTxs.put(ignite, new ArrayBlockingQueue<>(1000));
 
-                    TransactionConcurrency conc = TC_VALS[r.nextInt(TC_VALS.length)];
-                    TransactionIsolation isolation = TI_VALS[r.nextInt(TI_VALS.length)];
+        IgniteInternalFuture<?> txFut = multithreadedAsync(() -> {
+            while (!stop.get()) {
+                int nodeId = r.nextInt(GRID_CNT + 1);
 
-                    long timeout = r.nextInt(50) + 50; // Timeout is necessary to prevent deadlocks.
+                // Choose random node to start tx on.
+                Ignite node = nodeId == GRID_CNT || nearCacheEnabled() ? client : grid(nodeId);
 
-                    try (Transaction tx = node.transactions().txStart(conc, isolation, timeout, txSize)) {
-                        int setSize = r.nextInt(txSize / 2) + 1;
+                TransactionConcurrency conc = TC_VALS[r.nextInt(TC_VALS.length)];
+                TransactionIsolation isolation = TI_VALS[r.nextInt(TI_VALS.length)];
 
-                        for (int i = 0; i < setSize; i++) {
-                            switch (r.nextInt(4)) {
-                                case 0:
-                                    node.cache(CACHE_NAME).remove(r.nextInt(txSize));
+                // Timeout is necessary otherwise deadlock is possible due to randomness of lock acquisition.
+                long timeout = r.nextInt(50) + 50;
 
-                                    break;
+                try (Transaction tx = node.transactions().txStart(conc, isolation, timeout, txSize)) {
+                    BlockingQueue<Transaction> nodeQ = perNodeTxs.get(node);
 
-                                case 1:
-                                    node.cache(CACHE_NAME).get(r.nextInt(txSize));
+                    nodeQ.put(tx);
 
-                                    break;
+                    int setSize = r.nextInt(txSize / 2) + 1;
 
-                                case 2:
-                                    final Integer v = (Integer)node.cache(CACHE_NAME).get(r.nextInt(txSize));
+                    for (int i = 0; i < setSize; i++) {
+                        switch (r.nextInt(4)) {
+                            case 0:
+                                node.cache(CACHE_NAME).remove(r.nextInt(txSize));
 
-                                    node.cache(CACHE_NAME).put(r.nextInt(txSize), (v == null ? 0 : v) + 1);
+                                break;
 
-                                    break;
+                            case 1:
+                                node.cache(CACHE_NAME).get(r.nextInt(txSize));
 
-                                case 3:
-                                    node.cache(CACHE_NAME).put(r.nextInt(txSize), 0);
+                                break;
 
-                                    break;
+                            case 2:
+                                final Integer v = (Integer)node.cache(CACHE_NAME).get(r.nextInt(txSize));
 
-                                default:
-                                    fail("Unexpected opcode");
-                            }
-                        }
+                                node.cache(CACHE_NAME).put(r.nextInt(txSize), (v == null ? 0 : v) + 1);
 
-                        tx.commit();
+                                break;
 
-                        completed.add(1);
-                    }
-                    catch (Throwable e) {
-                        failed.add(1);
+                            case 3:
+                                node.cache(CACHE_NAME).put(r.nextInt(txSize), 0);
+
+                                break;
+
+                            default:
+                                fail("Unexpected opcode");
+                        }
                     }
 
-                    total.add(1);
+                    tx.commit();
+
+                    completed.add(1);
                 }
+                catch (Throwable e) {
+                    failed.add(1);
+                }
+
+                total.add(1);
             }
         }, threadCnt, "tx-thread");
 
         final AtomicIntegerArray idx = new AtomicIntegerArray(GRID_CNT + 1);
 
-        IgniteInternalFuture<?> rollbackFut = multithreadedAsync(new Runnable() {
-            @Override public void run() {
-                int concurrentRollbackCnt = 5;
-
-                List<IgniteFuture<?>> futs = new ArrayList<>(concurrentRollbackCnt);
-
-                while (!stop.get()) {
-                    // Choose node randomly.
-                    final int nodeId = r.nextInt(GRID_CNT + 1);
-
-                    // Reserve node.
-                    if (!idx.compareAndSet(nodeId, 0, 1)) {
-                        yield();
+        CIX1<Transaction> rollbackClo = new CIX1<Transaction>() {
+            @Override public void applyx(Transaction tx) throws IgniteCheckedException {
+                try {
+                    IgniteFuture<Void> rollbackFut = tx.rollbackAsync();
 
-                        continue;
-                    }
+                    rollbackFut.listen(new IgniteInClosure<IgniteFuture<Void>>() {
+                        @Override public void apply(IgniteFuture<Void> fut) {
+                            tx.close();
+                        }
+                    });
+                }
+                catch (Throwable t) {
+                    log.error("Exception on async rollback", t);
 
-                    Ignite node = nodeId == GRID_CNT || nearCacheEnabled() ? client : grid(nodeId);
+                    throw new IgniteCheckedException("Rollback failed", t);
+                }
+            }
+        };
 
-                    Collection<Transaction> transactions = node.transactions().localActiveTransactions();
+        IgniteInternalFuture<?> rollbackFut = multithreadedAsync(() -> {
+            while (!interrupted()) {
+                // Choose node randomly.
+                final int nodeId = r.nextInt(GRID_CNT + 1);
 
-                    for (Transaction tx : transactions) {
-                        rolledBack.add(1);
+                // Reserve node for rollback.
+                if (!idx.compareAndSet(nodeId, 0, 1)) {
+                    yield();
 
-                        if (rolledBack.sum() % 1000 == 0)
-                            info("Processed: " + rolledBack.sum());
+                    continue;
+                }
 
-                        try {
-                            IgniteFuture<Void> rollbackFut = tx.rollbackAsync();
+                Ignite node = nodeId == GRID_CNT || nearCacheEnabled() ? client : grid(nodeId);
 
-                            rollbackFut.listen(new IgniteInClosure<IgniteFuture<Void>>() {
-                                @Override public void apply(IgniteFuture<Void> fut) {
-                                    tx.close();
-                                }
-                            });
+                BlockingQueue<Transaction> nodeQ = perNodeTxs.get(node);
 
-                            futs.add(rollbackFut);
-                        }
-                        catch (Throwable t) {
-                            log.error("Exception on async rollback", t);
+                Transaction tx;
 
-                            fail("Exception is not expected");
-                        }
+                // Rollback all transaction
+                while((tx = nodeQ.poll()) != null) {
+                    rolledBack.add(1);
 
-                        if (futs.size() == concurrentRollbackCnt) {
-                            for (IgniteFuture<?> fut : futs)
-                                try {
-                                    fut.get();
-                                }
-                                catch (IgniteException e) {
-                                    log.warning("Future was rolled back with error", e);
-                                }
+                    doSleep(r.nextInt(50)); // Add random sleep to increase completed txs count.
 
-                            futs.clear();
-                        }
-                    }
+                    if (rolledBack.sum() % 1000 == 0)
+                        info("Rolled back so far: " + rolledBack.sum());
 
-                    idx.set(nodeId, 0);
+                    rollbackClo.apply(tx);
                 }
 
-                for (IgniteFuture<?> fut : futs)
-                    try {
-                        fut.get();
-                    }
-                    catch (Throwable t) {
-                        // No-op.
-                    }
-
+                idx.set(nodeId, 0);
             }
         }, 3, "rollback-thread"); // Rollback by multiple threads.
 
@@ -773,9 +780,27 @@ public void testMixedAsyncRollbackTypes() throws Exception {
 
         stop.set(true);
 
-        txFut.get();
+        txFut.get(); // Stop tx generation.
 
-        rollbackFut.get();
+        rollbackFut.cancel();
+
+        try {
+            rollbackFut.get();
+        }
+        catch (IgniteFutureCancelledCheckedException e) {
+            // Expected.
+        }
+
+        // Rollback remaining transactions.
+        for (BlockingQueue<Transaction> queue : perNodeTxs.values()) {
+            Transaction tx;
+
+            while((tx = queue.poll()) != null) {
+                rolledBack.add(1);
+
+                rollbackClo.apply(tx);
+            }
+        }
 
         log.info("total=" + total.sum() + ", completed=" + completed.sum() + ", failed=" + failed.sum() +
             ", rolledBack=" + rolledBack.sum());

From fab3725639ad79a2066f650e08729adf859e8480 Mon Sep 17 00:00:00 2001
From: Nikolay Izhikov <nizhikov@apache.org>
Date: Fri, 20 Jul 2018 14:37:44 +0300
Subject: [PATCH 363/543] IGNITE-8915: NPE during executing local SqlQuery from
 client node. - Fixes #4378.

Signed-off-by: Nikolay Izhikov <nizhikov@apache.org>
(cherry picked from commit 8633d34ec7542cda739ad38448955ac09238006c)
---
 .../cache/IgniteCacheProxyImpl.java           |  3 +++
 .../IgniteCacheReplicatedQuerySelfTest.java   | 27 +++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index 68e5b850aafed..22a940b9515c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -755,6 +755,9 @@ private void validate(Query qry) {
             (qry instanceof SqlQuery || qry instanceof SqlFieldsQuery || qry instanceof TextQuery))
             throw new CacheException("Failed to execute query. Add module 'ignite-indexing' to the classpath " +
                     "of all Ignite nodes.");
+
+        if (qry.isLocal() && (qry instanceof SqlQuery) && ctx.kernalContext().clientNode())
+            throw new CacheException("Execution of local sql query on client node disallowed.");
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
index 13942c2ede3d4..bd3dffdbded4b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import javax.cache.Cache;
+import javax.cache.CacheException;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
@@ -52,6 +53,7 @@
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
 import static org.apache.ignite.cache.CachePeekMode.ALL;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
 
 /**
  * Tests replicated query.
@@ -160,6 +162,31 @@ public void testClientOnlyNode() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientsLocalQuery() throws Exception {
+        try {
+            Ignite g = startGrid("client");
+
+            IgniteCache<Integer, Integer> c = jcache(g, Integer.class, Integer.class);
+
+            for (int i = 0; i < 10; i++)
+                c.put(i, i);
+
+            assertEquals(0, c.localSize());
+
+            SqlQuery<Integer, Integer> qry = new SqlQuery<>(Integer.class, "_key >= 5 order by _key");
+
+            qry.setLocal(true);
+
+            assertThrowsWithCause(() -> c.query(qry), CacheException.class);
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
+
     /**
      * JUnit.
      *

From 1bb670ece3428a78a06dd99f2f378026f684cd50 Mon Sep 17 00:00:00 2001
From: Nikolay Izhikov <nizhikov@apache.org>
Date: Thu, 6 Sep 2018 13:05:39 +0300
Subject: [PATCH 364/543] IGNITE-8915: Check moved to GridQueryProcessor. Check
 for SqlFieldsQuery added. - Fixes #4414.

Signed-off-by: Nikolay Izhikov <nizhikov@apache.org>
(cherry picked from commit 445d375ac31c64e9f7896f578b96fca43095c6be)
---
 .../cache/IgniteCacheProxyImpl.java           |  3 -
 .../processors/query/GridQueryProcessor.java  | 24 ++++++-
 .../IgniteCacheAbstractQuerySelfTest.java     | 54 +++++++++++++++-
 .../IgniteCacheReplicatedQuerySelfTest.java   | 27 --------
 .../index/H2DynamicIndexAbstractSelfTest.java |  6 ++
 .../local/IgniteCacheLocalQuerySelfTest.java  | 64 ++++++++++++++++++-
 6 files changed, 144 insertions(+), 34 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index 22a940b9515c0..68e5b850aafed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -755,9 +755,6 @@ private void validate(Query qry) {
             (qry instanceof SqlQuery || qry instanceof SqlFieldsQuery || qry instanceof TextQuery))
             throw new CacheException("Failed to execute query. Add module 'ignite-indexing' to the classpath " +
                     "of all Ignite nodes.");
-
-        if (qry.isLocal() && (qry instanceof SqlQuery) && ctx.kernalContext().clientNode())
-            throw new CacheException("Execution of local sql query on client node disallowed.");
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 03e5254183d19..7300393f599ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -2007,7 +2007,7 @@ public List<FieldsQueryCursor<List<?>>> querySqlFields(@Nullable final GridCache
         final boolean failOnMultipleStmts) {
         checkxEnabled();
 
-        validateSqlFieldsQuery(qry);
+        validateSqlFieldsQuery(qry, ctx, cctx);
 
         if (!ctx.state().publicApiActiveState(true)) {
             throw new IgniteException("Can not perform the operation because the cluster is inactive. Note, that " +
@@ -2057,13 +2057,31 @@ public List<FieldsQueryCursor<List<?>>> querySqlFields(@Nullable final GridCache
      * Validate SQL fields query.
      *
      * @param qry Query.
+     * @param ctx Kernal context.
+     * @param cctx Cache context.
      */
-    private static void validateSqlFieldsQuery(SqlFieldsQuery qry) {
+    private static void validateSqlFieldsQuery(SqlFieldsQuery qry, GridKernalContext ctx,
+        @Nullable GridCacheContext<?, ?> cctx) {
         if (qry.isReplicatedOnly() && qry.getPartitions() != null)
             throw new CacheException("Partitions are not supported in replicated only mode.");
 
         if (qry.isDistributedJoins() && qry.getPartitions() != null)
             throw new CacheException("Using both partitions and distributed JOINs is not supported for the same query");
+
+        if (qry.isLocal() && ctx.clientNode() && (cctx == null || cctx.config().getCacheMode() != CacheMode.LOCAL))
+            throw new CacheException("Execution of local SqlFieldsQuery on client node disallowed.");
+    }
+
+    /**
+     * Validate SQL query.
+     *
+     * @param qry Query.
+     * @param ctx Kernal context.
+     * @param cctx Cache context.
+     */
+    private static void validateSqlQuery(SqlQuery qry, GridKernalContext ctx, GridCacheContext<?, ?> cctx) {
+        if (qry.isLocal() && ctx.clientNode() && cctx.config().getCacheMode() != CacheMode.LOCAL)
+            throw new CacheException("Execution of local SqlQuery on client node disallowed.");
     }
 
     /**
@@ -2134,6 +2152,8 @@ public List<Long> streamBatchedUpdateQuery(final String schemaName, final SqlCli
      */
     public <K, V> QueryCursor<Cache.Entry<K,V>> querySql(final GridCacheContext<?,?> cctx, final SqlQuery qry,
         boolean keepBinary) {
+        validateSqlQuery(qry, ctx, cctx);
+
         if (qry.isReplicatedOnly() && qry.getPartitions() != null)
             throw new CacheException("Partitions are not supported in replicated only mode.");
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
index ee1a652348c2d..bae7b820ab4a0 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheAbstractQuerySelfTest.java
@@ -64,6 +64,7 @@
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.cache.store.CacheStoreAdapter;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.CacheQueryExecutedEvent;
@@ -96,6 +97,7 @@
 import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_OBJECT_READ;
 import static org.apache.ignite.internal.processors.cache.query.CacheQueryType.FULL_TEXT;
 import static org.apache.ignite.internal.processors.cache.query.CacheQueryType.SCAN;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
 import static org.junit.Assert.assertArrayEquals;
 
 /**
@@ -142,9 +144,12 @@ protected NearCacheConfiguration nearCacheConfiguration() {
 
         c.setDiscoverySpi(new TcpDiscoverySpi().setForceServerMode(true).setIpFinder(ipFinder));
 
-        if (igniteInstanceName.startsWith("client"))
+        if (igniteInstanceName.startsWith("client")) {
             c.setClientMode(true);
 
+            c.setDataStorageConfiguration(new DataStorageConfiguration());
+        }
+
         return c;
     }
 
@@ -1777,6 +1782,53 @@ public void testFieldsQueryEvents() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLocalSqlQueryFromClient() throws Exception {
+        try {
+            Ignite g = startGrid("client");
+
+            IgniteCache<Integer, Integer> c = jcache(g, Integer.class, Integer.class);
+
+            for (int i = 0; i < 10; i++)
+                c.put(i, i);
+
+            SqlQuery<Integer, Integer> qry = new SqlQuery<>(Integer.class, "_key >= 5 order by _key");
+
+            qry.setLocal(true);
+
+            assertThrowsWithCause(() -> c.query(qry), CacheException.class);
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testLocalSqlFieldsQueryFromClient() throws Exception {
+        try {
+            Ignite g = startGrid("client");
+
+            IgniteCache<UUID, Person> c = jcache(g, UUID.class, Person.class);
+
+            Person p = new Person("Jon", 1500);
+
+            c.put(p.id(), p);
+
+            SqlFieldsQuery qry = new SqlFieldsQuery("select count(*) from Person");
+
+            qry.setLocal(true);
+
+            assertThrowsWithCause(() -> c.query(qry), CacheException.class);
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
+
     /**
      *
      */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
index bd3dffdbded4b..13942c2ede3d4 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/distributed/replicated/IgniteCacheReplicatedQuerySelfTest.java
@@ -28,7 +28,6 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import javax.cache.Cache;
-import javax.cache.CacheException;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
@@ -53,7 +52,6 @@
 import static org.apache.ignite.cache.CacheMode.REPLICATED;
 import static org.apache.ignite.cache.CachePeekMode.ALL;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
-import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
 
 /**
  * Tests replicated query.
@@ -162,31 +160,6 @@ public void testClientOnlyNode() throws Exception {
         }
     }
 
-    /**
-     * @throws Exception If failed.
-     */
-    public void testClientsLocalQuery() throws Exception {
-        try {
-            Ignite g = startGrid("client");
-
-            IgniteCache<Integer, Integer> c = jcache(g, Integer.class, Integer.class);
-
-            for (int i = 0; i < 10; i++)
-                c.put(i, i);
-
-            assertEquals(0, c.localSize());
-
-            SqlQuery<Integer, Integer> qry = new SqlQuery<>(Integer.class, "_key >= 5 order by _key");
-
-            qry.setLocal(true);
-
-            assertThrowsWithCause(() -> c.query(qry), CacheException.class);
-        }
-        finally {
-            stopGrid("client");
-        }
-    }
-
     /**
      * JUnit.
      *
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
index ba848fb5b8b8d..0684f7f6e8e69 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
@@ -95,6 +95,9 @@ public void testCreateIndex() throws Exception {
 
         // Test that local queries on all nodes use new index.
         for (int i = 0 ; i < 4; i++) {
+            if (ignite(i).configuration().isClientMode())
+                continue;
+
             List<List<?>> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " +
                 "\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll();
 
@@ -165,6 +168,9 @@ public void testDropIndex() {
 
         // Test that no local queries on all nodes use new index.
         for (int i = 0 ; i < 4; i++) {
+            if (ignite(i).configuration().isClientMode())
+                continue;
+
             List<List<?>> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " +
                 "\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll();
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java
index 2570bc896c664..2272f27d158dd 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/local/IgniteCacheLocalQuerySelfTest.java
@@ -19,9 +19,12 @@
 
 import java.util.Iterator;
 import java.util.List;
+import java.util.UUID;
 import javax.cache.Cache;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
 import org.apache.ignite.cache.query.QueryCursor;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cache.query.SqlQuery;
@@ -93,4 +96,63 @@ public void testQueryLocal() throws Exception {
             cache.destroy();
         }
     }
-}
\ No newline at end of file
+
+    /** {@inheritDoc} */
+    @Override public void testLocalSqlQueryFromClient() throws Exception {
+        try {
+            Ignite g = startGrid("client");
+
+            IgniteCache<Integer, Integer> c = jcache(g, Integer.class, Integer.class);
+
+            for (int i = 0; i < 10; i++)
+                c.put(i, i);
+
+            SqlQuery<Integer, Integer> qry = new SqlQuery<>(Integer.class, "_key >= 5 order by _key");
+
+            qry.setLocal(true);
+
+            try(QueryCursor<Cache.Entry<Integer, Integer>> qryCursor = c.query(qry)) {
+                assertNotNull(qryCursor);
+
+                List<Cache.Entry<Integer, Integer>> res = qryCursor.getAll();
+
+                assertNotNull(res);
+
+                assertEquals(5, res.size());
+            }
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void testLocalSqlFieldsQueryFromClient() throws Exception {
+        try {
+            Ignite g = startGrid("client");
+
+            IgniteCache<UUID, Person> c = jcache(g, UUID.class, Person.class);
+
+            Person p = new Person("Jon", 1500);
+
+            c.put(p.id(), p);
+
+            SqlFieldsQuery qry = new SqlFieldsQuery("select * from Person");
+
+            qry.setLocal(true);
+
+            try(FieldsQueryCursor<List<?>> qryCursor = c.query(qry)) {
+                assertNotNull(qryCursor);
+
+                List<List<?>> res = qryCursor.getAll();
+
+                assertNotNull(res);
+
+                assertEquals(1, res.size());
+            }
+        }
+        finally {
+            stopGrid("client");
+        }
+    }
+}

From 49c711c85be2528d870fe8714e39309664522dd0 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Wed, 5 Sep 2018 11:33:07 +0300
Subject: [PATCH 365/543] IGNITE-9438 Fix file descriptors leak in
 StandaloneWalRecordsIterator. - Fixes #4658.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
(cherry picked from commit ef4a02dc5d3f66258afa9a3ded8d7671adf6ee73)
---
 .../wal/AbstractWalRecordsIterator.java       | 139 +++++++----
 .../reader/StandaloneWalRecordsIterator.java  |  33 +--
 .../ignite/internal/util/IgniteUtils.java     |  16 ++
 .../StandaloneWalRecordsIteratorTest.java     | 216 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   9 +-
 5 files changed, 351 insertions(+), 62 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index 9fbb53566b904..0b704caecd30c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.SegmentHeader;
 import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
 import org.apache.ignite.internal.util.typedef.P2;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -43,8 +44,8 @@
 import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader;
 
 /**
- * Iterator over WAL segments. This abstract class provides most functionality for reading records in log.
- * Subclasses are to override segment switching functionality
+ * Iterator over WAL segments. This abstract class provides most functionality for reading records in log. Subclasses
+ * are to override segment switching functionality
  */
 public abstract class AbstractWalRecordsIterator
     extends GridCloseableIteratorAdapter<IgniteBiTuple<WALPointer, WALRecord>> implements WALIterator {
@@ -52,14 +53,14 @@ public abstract class AbstractWalRecordsIterator
     private static final long serialVersionUID = 0L;
 
     /**
-     * Current record preloaded, to be returned on next()<br>
-     * Normally this should be not null because advance() method should already prepare some value<br>
+     * Current record preloaded, to be returned on next()<br> Normally this should be not null because advance() method
+     * should already prepare some value<br>
      */
     protected IgniteBiTuple<WALPointer, WALRecord> curRec;
 
     /**
-     * Current WAL segment absolute index. <br>
-     * Determined as lowest number of file at start, is changed during advance segment
+     * Current WAL segment absolute index. <br> Determined as lowest number of file at start, is changed during advance
+     * segment
      */
     protected long curWalSegmIdx = -1;
 
@@ -179,7 +180,6 @@ protected void advance() throws IgniteCheckedException {
     }
 
     /**
-     *
      * @param tailReachedException Tail reached exception.
      * @param currWalSegment Current WAL segment read handler.
      * @return If need to throw exception after validation.
@@ -210,9 +210,8 @@ protected IgniteCheckedException validateTailReachedException(
     }
 
     /**
-     * Switches records iterator to the next WAL segment
-     * as result of this method, new reference to segment should be returned.
-     * Null for current handle means stop of iteration.
+     * Switches records iterator to the next WAL segment as result of this method, new reference to segment should be
+     * returned. Null for current handle means stop of iteration.
      *
      * @param curWalSegment current open WAL segment or null if there is no open segment yet
      * @return new WAL segment to read or null for stop iteration
@@ -263,8 +262,8 @@ protected IgniteBiTuple<WALPointer, WALRecord> advanceRecord(
     }
 
     /**
-     * Performs final conversions with record loaded from WAL.
-     * To be overridden by subclasses if any processing required.
+     * Performs final conversions with record loaded from WAL. To be overridden by subclasses if any processing
+     * required.
      *
      * @param rec record to post process.
      * @return post processed record.
@@ -278,11 +277,11 @@ protected IgniteBiTuple<WALPointer, WALRecord> advanceRecord(
      *
      * @param e problem from records reading
      * @param ptr file pointer was accessed
-     *
-     * @return {@code null} if the error was handled and we can go ahead,
-     *  {@code IgniteCheckedException} if the error was not handled, and we should stop the iteration.
+     * @return {@code null} if the error was handled and we can go ahead, {@code IgniteCheckedException} if the error
+     * was not handled, and we should stop the iteration.
      */
-    protected IgniteCheckedException handleRecordException(@NotNull final Exception e, @Nullable final FileWALPointer ptr) {
+    protected IgniteCheckedException handleRecordException(@NotNull final Exception e,
+        @Nullable final FileWALPointer ptr) {
         if (log.isInfoEnabled())
             log.info("Stopping WAL iteration due to an exception: " + e.getMessage() + ", ptr=" + ptr);
 
@@ -290,45 +289,92 @@ protected IgniteCheckedException handleRecordException(@NotNull final Exception
     }
 
     /**
+     * Assumes fileIO will be closed in this method in case of error occurred.
+     *
      * @param desc File descriptor.
-     * @param start Optional start pointer. Null means read from the beginning
-     * @return Initialized file handle.
-     * @throws FileNotFoundException If segment file is missing.
+     * @param start Optional start pointer. Null means read from the beginning.
+     * @param fileIO fileIO associated with file descriptor
+     * @param segmentHeader read segment header from fileIO
+     * @return Initialized file read header.
      * @throws IgniteCheckedException If initialized failed due to another unexpected error.
      */
     protected AbstractReadFileHandle initReadHandle(
         @NotNull final AbstractFileDescriptor desc,
-        @Nullable final FileWALPointer start
-    ) throws IgniteCheckedException, FileNotFoundException {
+        @Nullable final FileWALPointer start,
+        @NotNull final FileIO fileIO,
+        @NotNull final SegmentHeader segmentHeader
+    ) throws IgniteCheckedException {
         try {
-            FileIO fileIO = desc.isCompressed() ? new UnzipFileIO(desc.file()) : ioFactory.create(desc.file());
+            final boolean isCompacted = segmentHeader.isCompacted();
+
+            if (isCompacted)
+                serializerFactory.skipPositionCheck(true);
+
+            FileInput in = new FileInput(fileIO, buf);
+
+            if (start != null && desc.idx() == start.index()) {
+                if (isCompacted) {
+                    if (start.fileOffset() != 0)
+                        serializerFactory.recordDeserializeFilter(new StartSeekingFilter(start));
+                }
+                else {
+                    // Make sure we skip header with serializer version.
+                    long startOff = Math.max(start.fileOffset(), fileIO.position());
 
+                    in.seek(startOff);
+                }
+            }
+
+            int serVer = segmentHeader.getSerializerVersion();
+
+            return createReadFileHandle(fileIO, desc.idx(), serializerFactory.createSerializer(serVer), in);
+        }
+        catch (SegmentEofException | EOFException ignore) {
             try {
-                SegmentHeader segmentHeader = readSegmentHeader(fileIO, curWalSegmIdx);
+                fileIO.close();
+            }
+            catch (IOException ce) {
+                throw new IgniteCheckedException(ce);
+            }
 
-                boolean isCompacted = segmentHeader.isCompacted();
+            return null;
+        }
+        catch (IgniteCheckedException e) {
+            U.closeWithSuppressingException(fileIO, e);
 
-                if (isCompacted)
-                    serializerFactory.skipPositionCheck(true);
+            throw e;
+        }
+        catch (IOException e) {
+            U.closeWithSuppressingException(fileIO, e);
 
-                FileInput in = new FileInput(fileIO, buf);
+            throw new IgniteCheckedException(
+                "Failed to initialize WAL segment after reading segment header: " + desc.file().getAbsolutePath(), e);
+        }
+    }
 
-                if (start != null && desc.idx() == start.index()) {
-                    if (isCompacted) {
-                        if (start.fileOffset() != 0)
-                            serializerFactory.recordDeserializeFilter(new StartSeekingFilter(start));
-                    }
-                    else {
-                        // Make sure we skip header with serializer version.
-                        long startOff = Math.max(start.fileOffset(), fileIO.position());
+    /**
+     * Assumes file descriptor will be opened in this method. The caller of this method must be responsible for closing
+     * opened file descriptor File descriptor will be closed ONLY in case of error occurred.
+     *
+     * @param desc File descriptor.
+     * @param start Optional start pointer. Null means read from the beginning
+     * @return Initialized file read header.
+     * @throws FileNotFoundException If segment file is missing.
+     * @throws IgniteCheckedException If initialized failed due to another unexpected error.
+     */
+    protected AbstractReadFileHandle initReadHandle(
+        @NotNull final AbstractFileDescriptor desc,
+        @Nullable final FileWALPointer start
+    ) throws IgniteCheckedException, FileNotFoundException {
+        FileIO fileIO = null;
 
-                        in.seek(startOff);
-                    }
-                }
+        try {
+            fileIO = desc.isCompressed() ? new UnzipFileIO(desc.file()) : ioFactory.create(desc.file());
 
-                int serVer = segmentHeader.getSerializerVersion();
+            SegmentHeader segmentHeader;
 
-                return createReadFileHandle(fileIO, desc.idx(), serializerFactory.createSerializer(serVer), in);
+            try {
+                segmentHeader = readSegmentHeader(fileIO, curWalSegmIdx);
             }
             catch (SegmentEofException | EOFException ignore) {
                 try {
@@ -341,20 +387,21 @@ protected AbstractReadFileHandle initReadHandle(
                 return null;
             }
             catch (IOException | IgniteCheckedException e) {
-                try {
-                    fileIO.close();
-                }
-                catch (IOException ce) {
-                    e.addSuppressed(ce);
-                }
+                U.closeWithSuppressingException(fileIO, e);
 
                 throw e;
             }
+
+            return initReadHandle(desc, start, fileIO, segmentHeader);
         }
         catch (FileNotFoundException e) {
+            U.closeQuiet(fileIO);
+
             throw e;
         }
         catch (IOException e) {
+            U.closeQuiet(fileIO);
+
             throw new IgniteCheckedException(
                 "Failed to initialize WAL segment: " + desc.file().getAbsolutePath(), e);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index 04e72df8c6532..eea734fbd2344 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -49,9 +49,11 @@
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
+import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.SegmentHeader;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.NotNull;
@@ -61,8 +63,8 @@
 import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.readSegmentHeader;
 
 /**
- * WAL reader iterator, for creation in standalone WAL reader tool
- * Operates over one directory, does not provide start and end boundaries
+ * WAL reader iterator, for creation in standalone WAL reader tool Operates over one directory, does not provide start
+ * and end boundaries
  */
 class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
     /** Record buffer size */
@@ -91,12 +93,13 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
 
     /**
      * Creates iterator in file-by-file iteration mode. Directory
+     *
      * @param log Logger.
      * @param sharedCtx Shared context. Cache processor is to be configured if Cache Object Key & Data Entry is
      * required.
      * @param ioFactory File I/O factory.
-     * @param keepBinary Keep binary. This flag disables converting of non primitive types
-     * (BinaryObjects will be used instead)
+     * @param keepBinary Keep binary. This flag disables converting of non primitive types (BinaryObjects will be used
+     * instead)
      * @param walFiles Wal files.
      */
     StandaloneWalRecordsIterator(
@@ -131,8 +134,8 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
     }
 
     /**
-     * For directory mode sets oldest file as initial segment,
-     * for file by file mode, converts all files to descriptors and gets oldest as initial.
+     * For directory mode sets oldest file as initial segment, for file by file mode, converts all files to descriptors
+     * and gets oldest as initial.
      *
      * @param walFiles files for file-by-file iteration mode
      */
@@ -235,7 +238,6 @@ private void init(List<FileDescriptor> walFiles) {
     }
 
     /**
-     *
      * @param ptr WAL pointer.
      * @return {@code True} If pointer between low and high bounds. {@code False} if not.
      */
@@ -246,7 +248,6 @@ private boolean checkBounds(WALPointer ptr) {
     }
 
     /**
-     *
      * @param idx WAL segment index.
      * @return {@code True} If pointer between low and high bounds. {@code False} if not.
      */
@@ -261,18 +262,21 @@ private boolean checkBounds(long idx) {
     ) throws IgniteCheckedException, FileNotFoundException {
 
         AbstractFileDescriptor fd = desc;
-
+        FileIO fileIO = null;
+        SegmentHeader segmentHeader;
         while (true) {
             try {
-                FileIO fileIO = fd.isCompressed() ? new UnzipFileIO(fd.file()) : ioFactory.create(fd.file());
+                fileIO = fd.isCompressed() ? new UnzipFileIO(fd.file()) : ioFactory.create(fd.file());
 
-                readSegmentHeader(fileIO, curWalSegmIdx);
+                segmentHeader = readSegmentHeader(fileIO, curWalSegmIdx);
 
                 break;
             }
             catch (IOException | IgniteCheckedException e) {
                 log.error("Failed to init segment curWalSegmIdx=" + curWalSegmIdx + ", curIdx=" + curIdx, e);
 
+                U.closeQuiet(fileIO);
+
                 curIdx++;
 
                 if (curIdx >= walFileDescriptors.size())
@@ -282,13 +286,13 @@ private boolean checkBounds(long idx) {
             }
         }
 
-        return super.initReadHandle(fd, start);
+        return initReadHandle(fd, start, fileIO, segmentHeader);
     }
 
     /** {@inheritDoc} */
     @NotNull @Override protected WALRecord postProcessRecord(@NotNull final WALRecord rec) {
-         GridKernalContext kernalCtx = sharedCtx.kernalContext();
-         IgniteCacheObjectProcessor processor = kernalCtx.cacheObjects();
+        GridKernalContext kernalCtx = sharedCtx.kernalContext();
+        IgniteCacheObjectProcessor processor = kernalCtx.cacheObjects();
 
         if (processor != null && rec.type() == RecordType.DATA_RECORD) {
             try {
@@ -355,6 +359,7 @@ private boolean checkBounds(long idx) {
 
     /**
      * Converts entry or lazy data entry into unwrapped entry
+     *
      * @param processor cache object processor for de-serializing objects.
      * @param fakeCacheObjCtx cache object context for de-serializing binary and unwrapping objects.
      * @param dataEntry entry to process
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 8bfb33fa20c43..508e4ef6216ce 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -3999,6 +3999,22 @@ public static void close(@Nullable AutoCloseable rsrc, @Nullable IgniteLogger lo
             }
     }
 
+    /**
+     * Closes given resource suppressing possible checked exception.
+     *
+     * @param rsrc Resource to close. If it's {@code null} - it's no-op.
+     * @param e Suppressor exception
+     */
+    public static void closeWithSuppressingException(@Nullable AutoCloseable rsrc, @NotNull Exception e) {
+        if (rsrc != null)
+            try {
+                rsrc.close();
+            }
+            catch (Exception suppressed) {
+               e.addSuppressed(suppressed);
+            }
+    }
+
     /**
      * Quietly closes given resource ignoring possible checked exception.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
new file mode 100644
index 0000000000000..b6a04d0a1488e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.reader;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.OpenOption;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.record.SnapshotRecord;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder;
+
+/**
+ * The test check, that StandaloneWalRecordsIterator correctly close file descriptors associated with WAL files.
+ */
+public class StandaloneWalRecordsIteratorTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(name);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration().
+                setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                )
+        ).setDiscoverySpi(
+            new TcpDiscoverySpi()
+                .setIpFinder(IP_FINDER)
+        );
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Check correct closing file descriptors.
+     *
+     * @throws Exception if test failed.
+     */
+    public void testCorrectClosingFileDescriptors() throws Exception {
+        IgniteEx ig = (IgniteEx)startGrid();
+
+        String archiveWalDir = getArchiveWalDirPath(ig);
+
+        ig.cluster().active(true);
+
+        IgniteCacheDatabaseSharedManager sharedMgr = ig.context().cache().context().database();
+
+        IgniteWriteAheadLogManager walMgr = ig.context().cache().context().wal();
+
+        // Generate WAL segments for filling WAL archive folder.
+        for (int i = 0; i < 2 * ig.configuration().getDataStorageConfiguration().getWalSegments(); i++) {
+            sharedMgr.checkpointReadLock();
+
+            try {
+                walMgr.log(new SnapshotRecord(i, false));
+            }
+            finally {
+                sharedMgr.checkpointReadUnlock();
+            }
+        }
+
+        stopGrid();
+
+        // Iterate by all archived WAL segments.
+        createWalIterator(archiveWalDir).forEach(x -> {
+        });
+
+        assertTrue("At least one WAL file must be opened!", CountedFileIO.getCountOpenedWalFiles() > 0);
+
+        assertEquals("All WAL files must be closed!", CountedFileIO.getCountOpenedWalFiles(), CountedFileIO.getCountClosedWalFiles());
+    }
+
+    /**
+     * Creates WALIterator associated with files inside walDir.
+     *
+     * @param walDir - path to WAL directory.
+     * @return WALIterator associated with files inside walDir.
+     * @throws IgniteCheckedException if error occur.
+     */
+    private WALIterator createWalIterator(String walDir) throws IgniteCheckedException {
+        IteratorParametersBuilder params = new IteratorParametersBuilder();
+
+        params.ioFactory(new CountedFileIOFactory());
+
+        return new IgniteWalIteratorFactory(log).iterator(params.filesOrDirs(walDir));
+    }
+
+    /**
+     * Evaluate path to directory with WAL archive.
+     *
+     * @param ignite instance of Ignite.
+     * @return path to directory with WAL archive.
+     * @throws IgniteCheckedException if error occur.
+     */
+    private String getArchiveWalDirPath(Ignite ignite) throws IgniteCheckedException {
+        return U.resolveWorkDirectory(
+            U.defaultWorkDirectory(),
+            ignite.configuration().getDataStorageConfiguration().getWalArchivePath(),
+            false
+        ).getAbsolutePath();
+    }
+
+    /**
+     *
+     */
+    private static class CountedFileIOFactory extends RandomAccessFileIOFactory {
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file) throws IOException {
+            return create(file, StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE);
+        }
+
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+            return new CountedFileIO(file, modes);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class CountedFileIO extends RandomAccessFileIO {
+        /** Wal open counter. */
+        private static final AtomicInteger WAL_OPEN_COUNTER = new AtomicInteger();
+        /** Wal close counter. */
+        private static final AtomicInteger WAL_CLOSE_COUNTER = new AtomicInteger();
+
+        /** File name. */
+        private final String fileName;
+
+        /** */
+        public CountedFileIO(File file, OpenOption... modes) throws IOException {
+            super(file, modes);
+
+            fileName = file.getName();
+
+            if (FileWriteAheadLogManager.WAL_NAME_PATTERN.matcher(fileName).matches())
+                WAL_OPEN_COUNTER.incrementAndGet();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void close() throws IOException {
+            super.close();
+
+            if (FileWriteAheadLogManager.WAL_NAME_PATTERN.matcher(fileName).matches())
+                WAL_CLOSE_COUNTER.incrementAndGet();
+        }
+
+        /**
+         *
+         * @return number of opened files.
+         */
+        public static int getCountOpenedWalFiles() { return WAL_OPEN_COUNTER.get(); }
+
+        /**
+         *
+         * @return number of closed files.
+         */
+        public static int getCountClosedWalFiles() { return WAL_CLOSE_COUNTER.get(); }
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 0c820e6fbb83d..ec3ea8129d475 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteReplayWalIteratorInvalidCrcTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteStandaloneWalIteratorInvalidCrcTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.reader.IgniteWalReaderTest;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneWalRecordsIteratorTest;
 
 /**
  *
@@ -100,7 +101,8 @@ public static TestSuite suite() {
     }
 
     /**
-     * Fills {@code suite} with PDS test subset, which operates with real page store, but requires long time to execute.
+     * Fills {@code suite} with PDS test subset, which operates with real page store, but requires long time to
+     * execute.
      *
      * @param suite suite to add tests into.
      */
@@ -148,7 +150,6 @@ public static void addRealPageStoreTests(TestSuite suite) {
 
         suite.addTestSuite(IgnitePdsWholeClusterRestartTest.class);
 
-
         // Rebalancing test
         suite.addTestSuite(IgniteWalHistoryReservationsTest.class);
 
@@ -195,6 +196,10 @@ public static void addRealPageStoreTests(TestSuite suite) {
 
         suite.addTestSuite(IgniteNodeStoppedDuringDisableWALTest.class);
 
+        suite.addTestSuite(StandaloneWalRecordsIteratorTest.class);
+
+        //suite.addTestSuite(IgniteWalRecoverySeveralRestartsTest.class);
+
         suite.addTestSuite(IgniteRebalanceScheduleResendPartitionsTest.class);
 
         suite.addTestSuite(IgniteWALTailIsReachedDuringIterationOverArchiveTest.class);

From 73b787f8ee84896029acdb41f1281cbfcef598d4 Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Wed, 19 Sep 2018 16:52:43 +0300
Subject: [PATCH 366/543] IGNITE-9100 remove duplicate test

---
 .../ignite/testsuites/IgniteCacheWithIndexingTestSuite.java    | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index 6a376a95217bf..1c7bd795f0f89 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -41,7 +41,6 @@
 import org.apache.ignite.internal.processors.client.IgniteDataStreamerTest;
 import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelperTest;
 import org.apache.ignite.testframework.junits.GridAbstractTest;
-import org.apache.ignite.util.GridCommandHandlerIndexingTest;
 
 /**
  * Cache tests using indexing.
@@ -88,8 +87,6 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteDataStreamerTest.class);
 
-        suite.addTestSuite(GridCommandHandlerIndexingTest.class);
-
         return suite;
     }
 }

From 03f2e84622372abc7b5b8b792d4f281553180ae4 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Sun, 16 Sep 2018 00:06:15 +0700
Subject: [PATCH 367/543] GG-14213: Backport IGNITE-9555 Web Console: Fixed
 handling of unexpected message via web sockets.

(cherry picked from commit 7242e58e38e6bd543db3bc382f0d40973d0f200d)
---
 .../backend/app/browsersHandler.js            | 30 +++++++++++++++----
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js
index 5aade60fac3e2..d0cd1128406d4 100644
--- a/modules/web-console/backend/app/browsersHandler.js
+++ b/modules/web-console/backend/app/browsersHandler.js
@@ -204,11 +204,20 @@ module.exports = {
 
             nodeListeners(sock) {
                 // Return command result from grid to browser.
-                sock.on('node:rest', ({clusterId, params, credentials} = {}, cb) => {
-                    if (_.isNil(clusterId) || _.isNil(params))
+                sock.on('node:rest', (arg, cb) => {
+                    const {clusterId, params, credentials} = arg || {};
+
+                    if (!_.isFunction(cb))
+                        cb = console.log;
+
+                    const demo = _.get(sock, 'request._query.IgniteDemoMode') === 'true';
+
+                    if ((_.isNil(clusterId) && !demo) || _.isNil(params)) {
+                        console.log('Received invalid message: "node:rest" on socket:', JSON.stringify(sock.handshake));
+
                         return cb('Invalid format of message: "node:rest"');
+                    }
 
-                    const demo = sock.request._query.IgniteDemoMode === 'true';
                     const token = sock.request.user.token;
 
                     const agent = this._agentHnd.agent(token, demo, clusterId);
@@ -236,11 +245,20 @@ module.exports = {
                 this.registerVisorTask('toggleClusterState', internalVisor('misc.VisorChangeGridActiveStateTask'), internalVisor('misc.VisorChangeGridActiveStateTaskArg'));
 
                 // Return command result from grid to browser.
-                sock.on('node:visor', ({clusterId, params, credentials} = {}, cb) => {
-                    if (_.isNil(clusterId) || _.isNil(params))
+                sock.on('node:visor', (arg, cb) => {
+                    const {clusterId, params, credentials} = arg || {};
+
+                    if (!_.isFunction(cb))
+                        cb = console.log;
+
+                    const demo = _.get(sock, 'request._query.IgniteDemoMode') === 'true';
+
+                    if ((_.isNil(clusterId) && !demo) || _.isNil(params)) {
+                        console.log('Received invalid message: "node:visor" on socket:', JSON.stringify(sock.handshake));
+
                         return cb('Invalid format of message: "node:visor"');
+                    }
 
-                    const demo = sock.request._query.IgniteDemoMode === 'true';
                     const token = sock.request.user.token;
 
                     const {taskId, nids, args = []} = params;

From f916ffbdd1a29cb42e96fe661a51027629565062 Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Wed, 19 Sep 2018 18:05:55 +0300
Subject: [PATCH 368/543] IGNITE-8869 fix merging

---
 .../cache/PartitionsExchangeOnDiscoveryHistoryOverflowTest.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeOnDiscoveryHistoryOverflowTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeOnDiscoveryHistoryOverflowTest.java
index b9eab780c9822..c0896c87a0c4f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeOnDiscoveryHistoryOverflowTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeOnDiscoveryHistoryOverflowTest.java
@@ -46,7 +46,7 @@
  */
 public class PartitionsExchangeOnDiscoveryHistoryOverflowTest extends IgniteCacheAbstractTest {
     /** */
-    private static final int CACHES_COUNT = 50;
+    private static final int CACHES_COUNT = 30;
 
     /** */
     private static final int DISCOVERY_HISTORY_SIZE = 10;

From 9fd9b2687bc266444ad574b13daca4bef670a1f8 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 19 Sep 2018 22:26:19 +0700
Subject: [PATCH 369/543] GG-14214 Backport WC-756 Web Console: Fixed version
 ranges check for "Collocated" query mode on "Queries" screen.

---
 .../page-queries/components/queries-notebook/controller.js      | 2 +-
 .../frontend/app/modules/agent/AgentManager.service.js          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
index c887f4db27507..87da4fc607304 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
@@ -41,7 +41,7 @@ const ROW_IDX = {value: -2, type: 'java.lang.Integer', label: 'ROW_IDX'};
 
 const NON_COLLOCATED_JOINS_SINCE = '1.7.0';
 
-const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], '2.5.2'];
+const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], ['2.5.1-p13', '2.6.0'], '2.7.0'];
 
 const ENFORCE_JOIN_SINCE = [['1.7.9', '1.8.0'], ['1.8.4', '1.9.0'], '1.9.1'];
 
diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
index a1c0ff966b0b5..06aca74e7b034 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
@@ -39,7 +39,7 @@ const State = {
 
 const IGNITE_2_0 = '2.0.0';
 const LAZY_QUERY_SINCE = [['2.1.4-p1', '2.2.0'], '2.2.1'];
-const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], '2.5.2'];
+const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], ['2.5.1-p13', '2.6.0'], '2.7.0'];
 
 // Error codes from o.a.i.internal.processors.restGridRestResponse.java
 

From 9ae4e087fda24e737fcbf955f73ee1b495d4bc7e Mon Sep 17 00:00:00 2001
From: ibessonov <bessonov.ip@gmail.com>
Date: Wed, 19 Sep 2018 19:05:10 +0300
Subject: [PATCH 370/543] IGNITE-8855 Throttle frequently reconnect client -
 Fixes #4739.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit 58150f8fed16255a48b2d110ce81afd02189d73b)
---
 .../apache/ignite/IgniteSystemProperties.java |  4 ++
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 47 +++++++++++++++++++
 2 files changed, 51 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index c42cae6b7f98d..d664cc084720e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -490,6 +490,10 @@ public final class IgniteSystemProperties {
     public static final String IGNITE_DISCOVERY_CLIENT_RECONNECT_HISTORY_SIZE =
         "IGNITE_DISCOVERY_CLIENT_RECONNECT_HISTORY_SIZE";
 
+    /** Time interval that indicates that client reconnect throttle must be reset to zero. 2 minutes by default. */
+    public static final String CLIENT_THROTTLE_RECONNECT_RESET_TIMEOUT_INTERVAL =
+        "CLIENT_THROTTLE_RECONNECT_RESET_TIMEOUT_INTERVAL";
+
     /** Number of cache operation retries in case of topology exceptions. */
     public static final String IGNITE_CACHE_RETRIES_COUNT = "IGNITE_CACHE_RETRIES_COUNT";
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index ca1c56fb6eda5..1a9921cbc31b0 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -47,6 +47,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.net.ssl.SSLException;
 import org.apache.ignite.Ignite;
@@ -58,6 +59,7 @@
 import org.apache.ignite.cache.CacheMetrics;
 import org.apache.ignite.cluster.ClusterMetrics;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteEx;
@@ -77,6 +79,7 @@
 import org.apache.ignite.internal.worker.WorkersRegistry;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.IgniteSpiAdapter;
 import org.apache.ignite.spi.IgniteSpiContext;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.IgniteSpiOperationTimeoutHelper;
@@ -145,6 +148,12 @@ class ClientImpl extends TcpDiscoveryImpl {
     /** */
     private static final Object SPI_RECONNECT = "SPI_RECONNECT";
 
+    /** */
+    private static final long CLIENT_THROTTLE_RECONNECT_RESET_TIMEOUT = IgniteSystemProperties.getLong(
+        IgniteSystemProperties.CLIENT_THROTTLE_RECONNECT_RESET_TIMEOUT_INTERVAL,
+        2 * 60_000
+    );
+
     /** Remote nodes. */
     private final ConcurrentMap<UUID, TcpDiscoveryNode> rmtNodes = new ConcurrentHashMap<>();
 
@@ -1605,6 +1614,12 @@ protected class MessageWorker extends GridWorker {
         /** */
         private boolean nodeAdded;
 
+        /** */
+        private long lastReconnectTimestamp = -1;
+
+        /** */
+        private long currentReconnectDelay = -1;
+
         /**
          * @param log Logger.
          */
@@ -1687,6 +1702,8 @@ else if (msg == SPI_RECONNECT) {
 
                             locNode.onClientDisconnected(newId);
 
+                            throttleClientReconnect();
+
                             tryJoin();
                         }
                     }
@@ -1875,6 +1892,36 @@ else if (discoMsg instanceof TcpDiscoveryCheckFailedMessage)
             }
         }
 
+        /**
+         * Wait random delay before trying to reconnect. Delay will grow exponentially every time client is forced to
+         * reconnect, but only if all these reconnections happened in small period of time (2 minutes). Maximum delay
+         * could be configured with {@link IgniteSpiAdapter#clientFailureDetectionTimeout()}, default value is
+         * {@link IgniteConfiguration#DFLT_CLIENT_FAILURE_DETECTION_TIMEOUT}.
+         *
+         * @throws InterruptedException If thread is interrupted.
+         */
+        private void throttleClientReconnect() throws InterruptedException {
+            if (U.currentTimeMillis() - lastReconnectTimestamp > CLIENT_THROTTLE_RECONNECT_RESET_TIMEOUT)
+                currentReconnectDelay = 0; // Skip pause on first reconnect.
+            else if (currentReconnectDelay == 0)
+                currentReconnectDelay = 200;
+            else {
+                long maxDelay = spi.failureDetectionTimeoutEnabled()
+                    ? spi.clientFailureDetectionTimeout()
+                    : IgniteConfiguration.DFLT_CLIENT_FAILURE_DETECTION_TIMEOUT;
+
+                currentReconnectDelay = Math.min(maxDelay, (int)(currentReconnectDelay * 1.5));
+            }
+
+            if (currentReconnectDelay != 0) {
+                ThreadLocalRandom random = ThreadLocalRandom.current();
+
+                Thread.sleep(random.nextLong(currentReconnectDelay / 2, currentReconnectDelay));
+            }
+
+            lastReconnectTimestamp = U.currentTimeMillis();
+        }
+
         /**
          *
          */

From 70f6506d18a06dfc35fe7265c9113d55d86312e9 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Mon, 23 Jul 2018 11:25:49 +0300
Subject: [PATCH 371/543] IGNITE-9042 Fixed partial tranasaction state wheh
 transaction is timed out - Fixes #4397.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 33f485aecbca59e7ae776145df830565b0dd1ebd)
---
 ...thSmallTimeoutAndContentionOneKeyTest.java | 20 +++++++-------
 .../junits/common/GridCommonAbstractTest.java | 27 ++++---------------
 2 files changed, 16 insertions(+), 31 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java
index c81714497095f..102f6a1cffb38 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java
@@ -33,9 +33,9 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
-import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
-import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
+import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2;
+import org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -208,7 +208,7 @@ public void test() throws Exception {
 
         f.get();
 
-        Map<PartitionKey, List<PartitionHashRecord>> idleVerifyResult = idleVerify(igClient, DEFAULT_CACHE_NAME);
+        IdleVerifyResultV2 idleVerifyResult = idleVerify(igClient, DEFAULT_CACHE_NAME);
 
         log.info("Current counter value:" + cnt.get());
 
@@ -216,12 +216,13 @@ public void test() throws Exception {
 
         log.info("Last commited value:" + val);
 
-        if (!F.isEmpty(idleVerifyResult)) {
+        if (idleVerifyResult.hasConflicts()){
             SB sb = new SB();
 
             sb.a("\n");
 
-            buildConflicts("Conflicts:\n", sb, idleVerifyResult);
+            buildConflicts("Hash conflicts:\n", sb, idleVerifyResult.hashConflicts());
+            buildConflicts("Counters conflicts:\n", sb, idleVerifyResult.counterConflicts());
 
             System.out.println(sb);
 
@@ -234,14 +235,15 @@ public void test() throws Exception {
      * @param conflicts Conflicts map.
      * @param sb String builder.
      */
-    private void buildConflicts(String msg, SB sb, Map<PartitionKey, List<PartitionHashRecord>> conflicts) {
+    private void buildConflicts(String msg, SB sb, Map<PartitionKeyV2, List<PartitionHashRecordV2>> conflicts) {
         sb.a(msg);
 
-        for (Map.Entry<PartitionKey, List<PartitionHashRecord>> entry : conflicts.entrySet()) {
+        for (Map.Entry<PartitionKeyV2, List<PartitionHashRecordV2>> entry : conflicts.entrySet()) {
             sb.a(entry.getKey()).a("\n");
 
-            for (PartitionHashRecord rec : entry.getValue())
+            for (PartitionHashRecordV2 rec : entry.getValue())
                 sb.a("\t").a(rec).a("\n");
         }
+
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 428cf24c4a961..2aedcf1bca613 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -88,6 +88,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
+import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
 import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTask;
@@ -105,6 +106,7 @@
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTask;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskResult;
+import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskV2;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -797,23 +799,6 @@ protected void awaitPartitionMapExchange(
         log.info("awaitPartitionMapExchange finished");
     }
 
-    /**
-     * Compares checksums between primary and backup partitions of specified caches.
-     * Works properly only on idle cluster - there may be false positive conflict reports if data in cluster is being
-     * concurrently updated.
-     *
-     * @param ig Ignite instance.
-     * @param cacheNames Cache names (if null, all user caches will be verified).
-     * @throws IgniteCheckedException If checksum conflict has been found.
-     */
-    protected void verifyBackupPartitions(Ignite ig, Set<String> cacheNames) throws IgniteCheckedException {
-        Map<PartitionKey, List<PartitionHashRecord>> conflicts = ig.compute().execute(
-            new VerifyBackupPartitionsTask(), cacheNames);
-
-        if (!conflicts.isEmpty())
-            throw new IgniteCheckedException("Conflict partitions: " + conflicts.keySet());
-    }
-
     /**
      * @param top Topology.
      * @param topVer Version to wait for.
@@ -1996,7 +1981,7 @@ protected void forceCheckpoint(Collection<Ignite> nodes) throws IgniteCheckedExc
      * @return Conflicts result.
      * @throws IgniteException If none caches or node found.
      */
-    protected Map<PartitionKey, List<PartitionHashRecord>> idleVerify(Ignite ig, String... caches) {
+    protected IdleVerifyResultV2 idleVerify(Ignite ig, String... caches) {
         IgniteEx ig0 = (IgniteEx)ig;
 
         Set<String> cacheNames = new HashSet<>();
@@ -2016,12 +2001,10 @@ protected Map<PartitionKey, List<PartitionHashRecord>> idleVerify(Ignite ig, Str
 
         VisorIdleVerifyTaskArg taskArg = new VisorIdleVerifyTaskArg(cacheNames);
 
-        VisorIdleVerifyTaskResult res = ig.compute().execute(
-            VisorIdleVerifyTask.class.getName(),
+        return ig.compute().execute(
+            VisorIdleVerifyTaskV2.class.getName(),
             new VisorTaskArgument<>(node.id(), taskArg, false)
         );
-
-        return res.getConflicts();
     }
 
     /**

From ebd289787fbfdcfaa7f4ff8573297255dae74b75 Mon Sep 17 00:00:00 2001
From: Dmitrii Ryabov <SomeFireOne@gmail.com>
Date: Mon, 14 May 2018 17:09:56 +0300
Subject: [PATCH 372/543] IGNITE-8456 Print warning if IGNITE_HOME & WORK and
 persistentStoreDir is not set, but persistence enabled - Fixes #3976.

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit 7637ed99e9321720dd7d0a0a4a833d042e89dee2)
---
 .../file/FilePageStoreManager.java            | 11 +++
 ...ersistenceDirectoryWarningLoggingTest.java | 85 +++++++++++++++++++
 .../IgnitePdsWithIndexingCoreTestSuite.java   |  2 +
 3 files changed, 98 insertions(+)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceDirectoryWarningLoggingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index a89ba57bfeba6..95db1c2efce3f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -170,6 +170,17 @@ public FilePageStoreManager(GridKernalContext ctx) {
         storeWorkDir = new File(folderSettings.persistentStoreRootPath(), folderSettings.folderName());
 
         U.ensureDirectory(storeWorkDir, "page store work directory", log);
+
+        String tmpDir = System.getProperty("java.io.tmpdir");
+
+        if (tmpDir != null && storeWorkDir.getAbsolutePath().contains(tmpDir)) {
+            log.warning("Persistence store directory is in the temp directory and may be cleaned." +
+                "To avoid this set \"IGNITE_HOME\" environment variable properly or " +
+                "change location of persistence directories in data storage configuration " +
+                "(see DataStorageConfiguration#walPath, DataStorageConfiguration#walArchivePath, " +
+                "DataStorageConfiguration#storagePath properties). " +
+                "Current persistence store directory is: [" + tmpDir + "]");
+        }
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceDirectoryWarningLoggingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceDirectoryWarningLoggingTest.java
new file mode 100644
index 0000000000000..7d67402ec0dec
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceDirectoryWarningLoggingTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.GridStringLogger;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests that warning is logged when persistence store directory equals {@code System.getProperty("java.io.tmpdir")}.
+ */
+public class PersistenceDirectoryWarningLoggingTest extends GridCommonAbstractTest {
+    /** Warning message to test. */
+    private static final String WARN_MSG_PREFIX = "Persistence store directory is in the temp " +
+        "directory and may be cleaned.";
+
+    /** String logger to check. */
+    private GridStringLogger log0 = new GridStringLogger();
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setGridLogger(log0);
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+
+        dsCfg.getDefaultDataRegionConfiguration().setPersistenceEnabled(true);
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPdsDirWarningSuppressed() throws Exception {
+        startGrid();
+
+        assertFalse(log0.toString().contains(WARN_MSG_PREFIX));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPdsDirWarningIsLogged() throws Exception {
+        IgniteConfiguration cfg = getConfiguration("0");
+
+        String tempDir = System.getProperty("java.io.tmpdir");
+
+        assertNotNull(tempDir);
+
+        // Emulates that Ignite work directory has not been calculated,
+        // and IgniteUtils#workDirectory resolved directory into "java.io.tmpdir"
+        cfg.setWorkDirectory(tempDir);
+
+        startGrid(cfg);
+
+        assertTrue(log0.toString().contains(WARN_MSG_PREFIX));
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index c47766b02d84a..491bab7dd65bd 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxHistoricalRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreCacheGroupsTest;
+import org.apache.ignite.internal.processors.cache.persistence.PersistenceDirectoryWarningLoggingTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCacheDestroyDuringCheckpointTest;
@@ -55,6 +56,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgnitePdsPageEvictionTest.class);
         suite.addTestSuite(IgnitePdsMultiNodePutGetRestartTest.class);
         suite.addTestSuite(IgnitePersistentStoreCacheGroupsTest.class);
+        suite.addTestSuite(PersistenceDirectoryWarningLoggingTest.class);
         suite.addTestSuite(WalPathsTest.class);
         suite.addTestSuite(WalRecoveryTxLogicalRecordsTest.class);
 

From 53314d5bb8acee226a367ebd1d632f4bb7efcf40 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Wed, 6 Jun 2018 12:22:09 +0300
Subject: [PATCH 373/543] IGNITE-8515 Value of
 CacheGroupMetricsMXBean.getTotalAllocatedPages() is always 0

Signed-off-by: Andrey Gura <agura@apache.org>
(cherry picked from commit d638a095e22e987dd0f5fb55271999494809a956)
---
 .../cache/CacheGroupMetricsMXBeanImpl.java    | 12 ++--
 .../persistence/DataRegionMetricsImpl.java    | 24 ++++++--
 .../cache/CacheGroupMetricsMBeanTest.java     | 55 ++++++++++++++++++-
 3 files changed, 78 insertions(+), 13 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
index e9a2736f34433..639569503f599 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
@@ -47,7 +47,7 @@ public class CacheGroupMetricsMXBeanImpl implements CacheGroupMetricsMXBean {
     private final CacheGroupContext ctx;
 
     /** */
-    private final GroupAllocationTrucker groupPageAllocationTracker;
+    private final GroupAllocationTracker groupPageAllocationTracker;
 
     /** Interface describing a predicate of two integers. */
     private interface IntBiPredicate {
@@ -63,7 +63,7 @@ private interface IntBiPredicate {
     /**
      *
      */
-    public static class GroupAllocationTrucker implements AllocatedPageTracker {
+    public static class GroupAllocationTracker implements AllocatedPageTracker {
         /** */
         private final LongAdder totalAllocatedPages = new LongAdder();
 
@@ -71,9 +71,9 @@ public static class GroupAllocationTrucker implements AllocatedPageTracker {
         private final AllocatedPageTracker delegate;
 
         /**
-         * @param delegate Delegate allocation trucker.
+         * @param delegate Delegate allocation tracker.
          */
-        public GroupAllocationTrucker(AllocatedPageTracker delegate) {
+        public GroupAllocationTracker(AllocatedPageTracker delegate) {
             this.delegate = delegate;
         }
 
@@ -88,7 +88,7 @@ public GroupAllocationTrucker(AllocatedPageTracker delegate) {
     /**
      *
      */
-    private static class NoopAllocationTrucker implements AllocatedPageTracker{
+    private static class NoopAllocationTracker implements AllocatedPageTracker{
         /** {@inheritDoc} */
         @Override public void updateTotalAllocatedPages(long delta) {
             // No-op.
@@ -112,7 +112,7 @@ public CacheGroupMetricsMXBeanImpl(CacheGroupContext ctx) {
             this.groupPageAllocationTracker = dataRegionMetrics.getOrAllocateGroupPageAllocationTracker(ctx.groupId());
         }
         else
-            this.groupPageAllocationTracker = new GroupAllocationTrucker(new NoopAllocationTrucker());
+            this.groupPageAllocationTracker = new GroupAllocationTracker(new NoopAllocationTracker());
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
index 57631722972f0..029cac4ff9e7c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataRegionMetricsImpl.java
@@ -17,12 +17,13 @@
 package org.apache.ignite.internal.processors.cache.persistence;
 
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.LongAdder;
 import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMXBeanImpl.GroupAllocationTrucker;
+import org.apache.ignite.internal.processors.cache.CacheGroupMetricsMXBeanImpl.GroupAllocationTracker;
 import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteOutClosure;
@@ -39,7 +40,7 @@ public class DataRegionMetricsImpl implements DataRegionMetrics, AllocatedPageTr
     private final LongAdder totalAllocatedPages = new LongAdder();
 
     /** */
-    private final ConcurrentHashMap<Integer, GroupAllocationTrucker> groupAllocationTruckers = new ConcurrentHashMap<>();
+    private final ConcurrentMap<Integer, GroupAllocationTracker> grpAllocationTrackers = new ConcurrentHashMap<>();
 
     /**
      * Counter for number of pages occupied by large entries (one entry is larger than one page).
@@ -377,13 +378,24 @@ public void incrementTotalAllocatedPages() {
     }
 
     /**
-     * Get or allocate group allocation trucker.
+     * Get or allocate group allocation tracker.
      *
      * @param grpId Group id.
-     * @return Group allocation trucker.
+     * @return Group allocation tracker.
      */
-    public GroupAllocationTrucker getOrAllocateGroupPageAllocationTracker(int grpId) {
-        return groupAllocationTruckers.getOrDefault(grpId, new GroupAllocationTrucker(this));
+    public GroupAllocationTracker getOrAllocateGroupPageAllocationTracker(int grpId) {
+        GroupAllocationTracker tracker = grpAllocationTrackers.get(grpId);
+
+        if (tracker == null) {
+            tracker = new GroupAllocationTracker(this);
+
+            GroupAllocationTracker old = grpAllocationTrackers.putIfAbsent(grpId, tracker);
+
+            if (old != null)
+                return old;
+        }
+
+        return tracker;
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java
index a50f860a21552..59a73ab58ac9d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java
@@ -30,12 +30,16 @@
 import javax.management.MBeanServerInvocationHandler;
 import javax.management.MalformedObjectNameException;
 import javax.management.ObjectName;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cache.affinity.AffinityFunctionContext;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -46,6 +50,9 @@
  * Cache group JMX metrics test.
  */
 public class CacheGroupMetricsMBeanTest extends GridCommonAbstractTest implements Serializable {
+    /** */
+    private boolean pds = false;
+
     /** */
     private static class RoundRobinVariableSizeAffinityFunction implements AffinityFunction {
         /** {@inheritDoc} */
@@ -137,11 +144,21 @@ private static class RoundRobinVariableSizeAffinityFunction implements AffinityF
 
         cfg.setCacheConfiguration(cCfg1, cCfg2, cCfg3, cCfg4);
 
+        if (pds) {
+            cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                    .setName("default")
+                    .setPersistenceEnabled(true)
+                    .setMetricsEnabled(true)
+                ).setMetricsEnabled(true)
+            );
+        }
+
         return cfg;
     }
 
     /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
+    @Override protected void afterTest() throws Exception {
         stopAllGrids();
     }
 
@@ -212,6 +229,8 @@ private Map<Integer, List<String>> arrayToAssignmentMap(int[][] arr) {
      * @throws Exception If failed.
      */
     public void testCacheGroupMetrics() throws Exception {
+        pds = false;
+
         startGrid(0);
         startGrid(1);
         startGrid(2);
@@ -263,4 +282,38 @@ public void testCacheGroupMetrics() throws Exception {
         assertTrue(mxBean0Grp1.getClusterMovingPartitionsCount() > 0);
 
     }
+
+    /**
+     * Test allocated pages counts for cache groups.
+     */
+    public void testAllocatedPages() throws Exception {
+        pds = true;
+
+        cleanPersistenceDir();
+
+        Ignite ignite = startGrid(0);
+
+        ignite.cluster().active(true);
+
+        CacheGroupMetricsMXBean mxBean0Grp1 = mxBean(0, "group1");
+        CacheGroupMetricsMXBean mxBean0Grp2 = mxBean(0, "group2");
+        CacheGroupMetricsMXBean mxBean0Grp3 = mxBean(0, "cache4");
+
+        long totalPages = ignite.dataRegionMetrics("default").getTotalAllocatedPages();
+
+        assertEquals(totalPages, mxBean0Grp1.getTotalAllocatedPages() + mxBean0Grp2.getTotalAllocatedPages() +
+            mxBean0Grp3.getTotalAllocatedPages());
+
+        for (int cacheIdx = 1; cacheIdx <= 4; cacheIdx++) {
+            IgniteCache cache = ignite.cache("cache" + cacheIdx);
+
+            for (int i = 0; i < 10 * cacheIdx; i++)
+                cache.put(i, new byte[100]);
+        }
+
+        totalPages = ignite.dataRegionMetrics("default").getTotalAllocatedPages();
+
+        assertEquals(totalPages, mxBean0Grp1.getTotalAllocatedPages() + mxBean0Grp2.getTotalAllocatedPages() +
+            mxBean0Grp3.getTotalAllocatedPages());
+    }
 }

From 3ca3ba67897768dad85c6b51d7909ca5728b8536 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Fri, 3 Aug 2018 18:01:50 +0300
Subject: [PATCH 374/543] IGNITE-9157 Optimize memory usage of data regions in
 tests - Fixes #4470.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
(cherry picked from commit c00060ba83c3550c7862c75b691d70112552b5be)
---
 ...dbcThinAuthenticateConnectionSelfTest.java |  5 ++++-
 .../DummyPersistenceCompatibilityTest.java    |  1 +
 ...ngToWalV2SerializerWithCompactionTest.java |  5 ++++-
 .../cache/CacheGroupMetricsMBeanTest.java     |  2 +-
 .../cache/CacheMetricsEnableRuntimeTest.java  |  6 ++++-
 .../WalModeChangeCommonAbstractSelfTest.java  |  4 +++-
 .../distributed/Cache64kPartitionsTest.java   |  6 +++--
 .../IgnitePdsContinuousRestartTest.java       |  2 ++
 ...RebalanceScheduleResendPartitionsTest.java |  1 +
 ...ngeDuringRebalanceOnNonNodeAssignTest.java |  1 +
 ...alModeChangeDuringRebalancingSelfTest.java |  9 ++++----
 ...ersistenceDirectoryWarningLoggingTest.java |  4 +++-
 ...owHistoricalRebalanceSmallHistoryTest.java |  5 +++++
 ...iteCheckpointDirtyPagesForLowLoadTest.java |  6 ++++-
 ...tePdsCacheDestroyDuringCheckpointTest.java | 13 +++++------
 ...pointSimulationWithRealCpDisabledTest.java |  6 ++++-
 .../db/wal/IgniteWalRebalanceTest.java        |  6 ++++-
 ...thSmallTimeoutAndContentionOneKeyTest.java |  1 +
 .../database/IgniteDbAbstractTest.java        |  6 +++--
 .../GridMarshallerMappingConsistencyTest.java |  2 +-
 .../junits/GridAbstractTest.java              | 22 +++++++++++++++++++
 .../SqlUserCommandSelfTest.java               |  8 +++++--
 22 files changed, 92 insertions(+), 29 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAuthenticateConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAuthenticateConnectionSelfTest.java
index 97218fb09162e..cb4d7f3cf70c9 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAuthenticateConnectionSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAuthenticateConnectionSelfTest.java
@@ -60,7 +60,10 @@ public class JdbcThinAuthenticateConnectionSelfTest extends JdbcThinAbstractSelf
 
         cfg.setDataStorageConfiguration(new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
-                .setPersistenceEnabled(true)));
+                .setPersistenceEnabled(true)
+                .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+            )
+        );
 
         return cfg;
     }
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
index b36f563337a63..5a3740cae3fc0 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/DummyPersistenceCompatibilityTest.java
@@ -67,6 +67,7 @@ public class DummyPersistenceCompatibilityTest extends IgnitePersistenceCompatib
                 .setDefaultDataRegionConfiguration(
                     new DataRegionConfiguration()
                         .setPersistenceEnabled(true)
+                        .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
                 ));
 
         cfg.setBinaryConfiguration(
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MigratingToWalV2SerializerWithCompactionTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MigratingToWalV2SerializerWithCompactionTest.java
index d79790ecd7778..c72ce2fe96d9c 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MigratingToWalV2SerializerWithCompactionTest.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MigratingToWalV2SerializerWithCompactionTest.java
@@ -66,7 +66,10 @@ public class MigratingToWalV2SerializerWithCompactionTest extends IgnitePersiste
 
         DataStorageConfiguration memCfg = new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(
-                new DataRegionConfiguration().setPersistenceEnabled(true))
+                new DataRegionConfiguration()
+                    .setPersistenceEnabled(true)
+                    .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+            )
             .setWalSegmentSize(WAL_SEGMENT_SIZE)
             .setWalCompactionEnabled(true)
             .setWalMode(WALMode.LOG_ONLY)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java
index 59a73ab58ac9d..c77df70fa1dec 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMBeanTest.java
@@ -147,8 +147,8 @@ private static class RoundRobinVariableSizeAffinityFunction implements AffinityF
         if (pds) {
             cfg.setDataStorageConfiguration(new DataStorageConfiguration()
                 .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
-                    .setName("default")
                     .setPersistenceEnabled(true)
+                    .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
                     .setMetricsEnabled(true)
                 ).setMetricsEnabled(true)
             );
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsEnableRuntimeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsEnableRuntimeTest.java
index 16b1181249296..a5cd3a67934f1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsEnableRuntimeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsEnableRuntimeTest.java
@@ -356,7 +356,11 @@ private CacheMetricsMXBean mxBean(int nodeIdx, String cacheName, Class<? extends
 
         if (persistence)
             cfg.setDataStorageConfiguration(new DataStorageConfiguration()
-                .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true))
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+                )
                 .setWalMode(WALMode.LOG_ONLY)
             );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java
index 3056a13fb1a13..d4a0078a2cefb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/WalModeChangeCommonAbstractSelfTest.java
@@ -305,7 +305,9 @@ protected IgniteConfiguration config(String name, boolean cli, boolean filter) {
 
         cfg.setDiscoverySpi(new IgniteClientReconnectAbstractTest.TestTcpDiscoverySpi().setIpFinder(IP_FINDER));
 
-        DataRegionConfiguration regionCfg = new DataRegionConfiguration().setPersistenceEnabled(true);
+        DataRegionConfiguration regionCfg = new DataRegionConfiguration()
+            .setPersistenceEnabled(true)
+            .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE);
 
         DataRegionConfiguration volatileRegionCfg = new DataRegionConfiguration().setName(REGION_VOLATILE)
             .setPersistenceEnabled(false);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/Cache64kPartitionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/Cache64kPartitionsTest.java
index 95e90c3dbf684..e54251e2b836c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/Cache64kPartitionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/Cache64kPartitionsTest.java
@@ -23,7 +23,6 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
@@ -48,7 +47,10 @@ public class Cache64kPartitionsTest extends GridCommonAbstractTest {
         if (persistenceEnabled) {
             DataStorageConfiguration memCfg = new DataStorageConfiguration()
                 .setDefaultDataRegionConfiguration(
-                    new DataRegionConfiguration().setPersistenceEnabled(true))
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+                )
                 .setWalMode(WALMode.LOG_ONLY);
 
             cfg.setDataStorageConfiguration(memCfg);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
index 99614ed7ab697..9e450ca34d57c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
@@ -77,6 +77,8 @@ public IgnitePdsContinuousRestartTest(boolean cancel) {
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
+        cfg.setConsistentId(gridName);
+
         DataStorageConfiguration memCfg = new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(
                 new DataRegionConfiguration().setMaxSize(400 * 1024 * 1024).setPersistenceEnabled(true))
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteRebalanceScheduleResendPartitionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteRebalanceScheduleResendPartitionsTest.java
index 274b16d5e2225..bb08a8dc5631f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteRebalanceScheduleResendPartitionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteRebalanceScheduleResendPartitionsTest.java
@@ -73,6 +73,7 @@ public class IgniteRebalanceScheduleResendPartitionsTest extends GridCommonAbstr
                 .setDefaultDataRegionConfiguration(
                     new DataRegionConfiguration()
                         .setPersistenceEnabled(true)
+                        .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
                 )
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest.java
index 675ed310d8063..4f2817f29109c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest.java
@@ -69,6 +69,7 @@ public class LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest extends Grid
                 .setWalArchivePath(walArchivePath(name))
                 .setDefaultDataRegionConfiguration(
                     new DataRegionConfiguration()
+                        .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
                         .setPersistenceEnabled(true)
                 )
         );
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
index 9ec27ce82a886..4dd5f51b28772 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -81,8 +81,8 @@ public class LocalWalModeChangeDuringRebalancingSelfTest extends GridCommonAbstr
                 .setDefaultDataRegionConfiguration(
                     new DataRegionConfiguration()
                         .setPersistenceEnabled(true)
-                        .setMaxSize(200 * 1024 * 1024)
-                        .setInitialSize(200 * 1024 * 1024)
+                        .setInitialSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+                        .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
                 )
                 // Test verifies checkpoint count, so it is essencial that no checkpoint is triggered by timeout
                 .setCheckpointFrequency(999_999_999_999L)
@@ -326,7 +326,7 @@ public void testLocalAndGlobalWalStateInterdependence() throws Exception {
      * @throws Exception If failed.
      */
     public void testWithExchangesMerge() throws Exception {
-        final int nodeCnt = 5;
+        final int nodeCnt = 4;
         final int keyCnt = getKeysCount();
 
         Ignite ignite = startGrids(nodeCnt);
@@ -340,14 +340,13 @@ public void testWithExchangesMerge() throws Exception {
 
         stopGrid(2);
         stopGrid(3);
-        stopGrid(4);
 
         // Rewrite data to trigger further rebalance.
         for (int k = 0; k < keyCnt; k++)
             cache.put(k, k * 2);
 
         // Start several grids in parallel to trigger exchanges merge.
-        startGridsMultiThreaded(2, 3);
+        startGridsMultiThreaded(2, 2);
 
         for (int nodeIdx = 2; nodeIdx < nodeCnt; nodeIdx++) {
             CacheGroupContext grpCtx = grid(nodeIdx).cachex(REPL_CACHE).context().group();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceDirectoryWarningLoggingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceDirectoryWarningLoggingTest.java
index 7d67402ec0dec..bb55c8200ee33 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceDirectoryWarningLoggingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceDirectoryWarningLoggingTest.java
@@ -48,7 +48,9 @@ public class PersistenceDirectoryWarningLoggingTest extends GridCommonAbstractTe
 
         DataStorageConfiguration dsCfg = new DataStorageConfiguration();
 
-        dsCfg.getDefaultDataRegionConfiguration().setPersistenceEnabled(true);
+        dsCfg.getDefaultDataRegionConfiguration()
+            .setPersistenceEnabled(true)
+            .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE);
 
         cfg.setDataStorageConfiguration(dsCfg);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
index 8f2e7389f8bb3..e6ec8a37f6538 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
@@ -67,6 +67,8 @@ public class SlowHistoricalRebalanceSmallHistoryTest extends GridCommonAbstractT
     @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(name);
 
+        cfg.setConsistentId(name);
+
         cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
 
         cfg.setDataStorageConfiguration(
@@ -74,6 +76,7 @@ public class SlowHistoricalRebalanceSmallHistoryTest extends GridCommonAbstractT
                 .setWalHistorySize(WAL_HISTORY_SIZE)
                 .setDefaultDataRegionConfiguration(
                     new DataRegionConfiguration()
+                        .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
                         .setPersistenceEnabled(true)
                 )
                 .setWalSegmentSize(512 * 1024)
@@ -107,6 +110,8 @@ public class SlowHistoricalRebalanceSmallHistoryTest extends GridCommonAbstractT
 
         System.clearProperty(IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
 
+        stopAllGrids();
+
         cleanPersistenceDir();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
index e5f4ac82862d5..8baa1c301512c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/checkpoint/IgniteCheckpointDirtyPagesForLowLoadTest.java
@@ -69,7 +69,11 @@ public class IgniteCheckpointDirtyPagesForLowLoadTest extends GridCommonAbstract
 
         DataStorageConfiguration dsCfg = new DataStorageConfiguration();
         dsCfg.setPageSize(1024); //smaller page to reduce overhead to short values
-        dsCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true));
+        dsCfg.setDefaultDataRegionConfiguration(
+            new DataRegionConfiguration()
+                .setPersistenceEnabled(true)
+                .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+        );
         dsCfg.setCheckpointFrequency(500);
         dsCfg.setWalMode(WALMode.LOG_ONLY);
         dsCfg.setWalHistorySize(1);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java
index fba62dff3b8c3..ad8bfae282682 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheDestroyDuringCheckpointTest.java
@@ -19,7 +19,6 @@
 
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CachePeekMode;
@@ -28,11 +27,8 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
-import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
-
 /**
  * Test for cache creation/deletion with frequent checkpoints.
  */
@@ -68,10 +64,11 @@ private DataStorageConfiguration createDbConfig() {
         DataStorageConfiguration storageCfg = new DataStorageConfiguration();
         storageCfg.setCheckpointFrequency(300);
 
-        DataRegionConfiguration regionConfig = new DataRegionConfiguration();
-        regionConfig.setPersistenceEnabled(true);
-
-        storageCfg.setDefaultDataRegionConfiguration(regionConfig);
+        storageCfg.setDefaultDataRegionConfiguration(
+            new DataRegionConfiguration()
+                .setPersistenceEnabled(true)
+                .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+        );
 
         return storageCfg;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
index 0fbb2556d1caf..5fa618b3b947e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
@@ -113,7 +113,11 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
                 .setCheckpointFrequency(500)
                 .setWalMode(WALMode.LOG_ONLY)
                 .setAlwaysWriteFullPages(true)
-                .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true))
+                .setDefaultDataRegionConfiguration(
+                    new DataRegionConfiguration()
+                        .setPersistenceEnabled(true)
+                        .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+                )
         );
 
         TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
index 034ea74a5b19f..57565bfd58c48 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
@@ -102,7 +102,11 @@ public class IgniteWalRebalanceTest extends GridCommonAbstractTest {
                     .setWalHistorySize(Integer.MAX_VALUE)
                     .setWalMode(WALMode.LOG_ONLY)
                     .setCheckpointFrequency(15 * 60 * 1000)
-                    .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true));
+                    .setDefaultDataRegionConfiguration(
+                        new DataRegionConfiguration()
+                            .setPersistenceEnabled(true)
+                            .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+                    );
 
         cfg.setDataStorageConfiguration(dbCfg);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java
index 102f6a1cffb38..62f46352391bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxWithSmallTimeoutAndContentionOneKeyTest.java
@@ -79,6 +79,7 @@ public class TxWithSmallTimeoutAndContentionOneKeyTest extends GridCommonAbstrac
                 .setDefaultDataRegionConfiguration(
                     new DataRegionConfiguration()
                         .setPersistenceEnabled(true)
+                        .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
                 )
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index 0823ee4fbfef9..9a23502b25725 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -32,7 +32,6 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
@@ -80,7 +79,10 @@ public abstract class IgniteDbAbstractTest extends GridCommonAbstractTest {
         dbCfg.setWalMode(WALMode.LOG_ONLY);
 
         dbCfg.setDefaultDataRegionConfiguration(
-            new DataRegionConfiguration().setPersistenceEnabled(true).setName("default"));
+            new DataRegionConfiguration()
+                .setPersistenceEnabled(true)
+                .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+        );
 
         configure(dbCfg);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index 78f3c03eda818..efb4fd511c4aa 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -55,7 +55,7 @@ public class GridMarshallerMappingConsistencyTest extends GridCommonAbstractTest
         igniteCfg.setConsistentId(igniteInstanceName);
 
         DataRegionConfiguration drCfg = new DataRegionConfiguration();
-        drCfg.setPersistenceEnabled(true);
+        drCfg.setPersistenceEnabled(true).setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE);
 
         DataStorageConfiguration dsCfg = new DataStorageConfiguration();
         dsCfg.setDefaultDataRegionConfiguration(drCfg);
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index d106b49203445..669d6bcf06512 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -857,6 +857,28 @@ protected Ignite startGrid(String igniteInstanceName, GridSpringResourceContext
         return startGrid(igniteInstanceName, optimize(getConfiguration(igniteInstanceName)), ctx);
     }
 
+    /**
+     * @param regionCfg Region config.
+     */
+    private void validateDataRegion(DataRegionConfiguration regionCfg) {
+        if (regionCfg.isPersistenceEnabled() && regionCfg.getMaxSize() == DataStorageConfiguration.DFLT_DATA_REGION_MAX_SIZE)
+            throw new AssertionError("Max size of data region should be set explicitly to avoid memory over usage");
+    }
+
+    /**
+     * @param cfg Config.
+     */
+    private void validateConfiguration(IgniteConfiguration cfg) {
+        if (cfg.getDataStorageConfiguration() != null) {
+            validateDataRegion(cfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration());
+
+            if (cfg.getDataStorageConfiguration().getDataRegionConfigurations() != null) {
+                for (DataRegionConfiguration reg : cfg.getDataStorageConfiguration().getDataRegionConfigurations())
+                    validateDataRegion(reg);
+            }
+        }
+    }
+
     /**
      * Starts new grid with given name.
      *
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/authentication/SqlUserCommandSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/authentication/SqlUserCommandSelfTest.java
index 7d0a5797e12f9..64819f3173449 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/authentication/SqlUserCommandSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/authentication/SqlUserCommandSelfTest.java
@@ -66,8 +66,12 @@ public class SqlUserCommandSelfTest extends GridCommonAbstractTest {
         cfg.setAuthenticationEnabled(true);
 
         cfg.setDataStorageConfiguration(new DataStorageConfiguration()
-            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
-                .setPersistenceEnabled(true)));
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setPersistenceEnabled(true)
+                    .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+            )
+        );
 
         return cfg;
     }

From b66cce09ecfa858a750ae4cd1f19af8d543fcacf Mon Sep 17 00:00:00 2001
From: SGrimstad <sgrimstad@gridgain.com>
Date: Thu, 20 Sep 2018 18:26:37 +0300
Subject: [PATCH 375/543] IGNITE-9162: SQL: fixed class-cast exception in
 partition pruning when SELECT contains subquery. This closes #4779.

---
 .../query/h2/sql/GridSqlQuerySplitter.java    |   3 +-
 .../h2/twostep/TableViewSubquerySelfTest.java | 120 ++++++++++++++++++
 .../IgniteCacheQuerySelfTestSuite2.java       |   3 +
 3 files changed, 125 insertions(+), 1 deletion(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/TableViewSubquerySelfTest.java

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
index 5c320055bcdf8..eddf17c5874ef 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sql/GridSqlQuerySplitter.java
@@ -2335,7 +2335,8 @@ private static CacheQueryPartitionInfo extractPartitionFromEquality(GridSqlOpera
 
         GridSqlColumn column = (GridSqlColumn)left;
 
-        assert column.column().getTable() instanceof GridH2Table;
+        if (!(column.column().getTable() instanceof GridH2Table))
+            return null;
 
         GridH2Table tbl = (GridH2Table) column.column().getTable();
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/TableViewSubquerySelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/TableViewSubquerySelfTest.java
new file mode 100644
index 0000000000000..eaf4243018ff3
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/TableViewSubquerySelfTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.twostep;
+
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class TableViewSubquerySelfTest extends GridCommonAbstractTest {
+    /** */
+    private static final int NODES_COUNT = 1;
+
+    /** */
+    private static Ignite ignite;
+
+    /** */
+    private static IgniteCache<?, ?> initCache;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        ignite = startGridsMultiThreaded(NODES_COUNT, false);
+        initCache = ignite.getOrCreateCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+            .setSqlSchema("PUBLIC")
+        );
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** */
+    public void testSubqueryTableView() {
+        final String cacheName = "a1";
+
+        final String creationQry = "CREATE TABLE t1 ( id INT NOT NULL, int_col1 INT NOT NULL, PRIMARY KEY (id)) " +
+            "WITH \"TEMPLATE=partitioned, cache_name=%s\""; //, WRAP_VALUE=false
+
+        try (FieldsQueryCursor<List<?>> cur = initCache.query(
+            new SqlFieldsQuery(String.format(creationQry,cacheName)))) {
+            assertNotNull(cur);
+
+            List<List<?>> rows = cur.getAll();
+
+            assertEquals(1, rows.size());
+
+            assertEquals(0L, rows.get(0).get(0));
+        }
+
+        final IgniteCache<Integer, Integer> cache = ignite.getOrCreateCache(cacheName);
+
+        try (FieldsQueryCursor<List<?>> cur = cache.query(new SqlFieldsQuery(
+            "INSERT INTO t1 (id,int_col1) VALUES (1,0),(2,2),(3,0),(4,2)"))) {
+            assertNotNull(cur);
+
+            List<List<?>> rows = cur.getAll();
+
+            assertEquals(1, rows.size());
+
+            assertEquals(4L, rows.get(0).get(0));
+        }
+
+        try (FieldsQueryCursor<List<?>> cur = cache.query(new SqlFieldsQuery(
+            "SELECT * FROM ( SELECT * FROM t1 WHERE int_col1 > 0 ORDER BY id ) WHERE int_col1 = 1"))) {
+            assertNotNull(cur);
+
+            List<List<?>> rows = cur.getAll();
+
+            assertEquals(0, rows.size());
+        }
+
+        try (FieldsQueryCursor<List<?>> cur = cache.query(new SqlFieldsQuery(
+            "SELECT * FROM ( SELECT * FROM t1 WHERE int_col1 < 0 ORDER BY id ) WHERE int_col1 = 1"))) {
+            assertNotNull(cur);
+
+            List<List<?>> rows = cur.getAll();
+
+            assertEquals(0, rows.size());
+        }
+
+        try (FieldsQueryCursor<List<?>> cur = cache.query(new SqlFieldsQuery(
+            "SELECT * FROM ( SELECT * FROM t1 WHERE int_col1 > 0 ORDER BY id ) WHERE int_col1 = 2"))) {
+            assertNotNull(cur);
+
+            List<List<?>> rows = cur.getAll();
+
+            assertEquals(2, rows.size());
+
+            assertEquals(2, rows.get(0).get(0));
+
+            assertEquals(2, rows.get(0).get(1));
+
+            assertEquals(4, rows.get(1).get(0));
+
+            assertEquals(2, rows.get(1).get(1));
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
index 536834cda9346..4b91b4307869e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheQuerySelfTestSuite2.java
@@ -55,6 +55,7 @@
 import org.apache.ignite.internal.processors.query.h2.twostep.DisappearedCacheWasNotFoundMessageSelfTest;
 import org.apache.ignite.internal.processors.query.h2.twostep.NonCollocatedRetryMessageSelfTest;
 import org.apache.ignite.internal.processors.query.h2.twostep.RetryCauseMessageSelfTest;
+import org.apache.ignite.internal.processors.query.h2.twostep.TableViewSubquerySelfTest;
 import org.apache.ignite.testframework.IgniteTestSuite;
 
 /**
@@ -119,6 +120,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(DisappearedCacheCauseRetryMessageSelfTest.class);
         suite.addTestSuite(DisappearedCacheWasNotFoundMessageSelfTest.class);
 
+        suite.addTestSuite(TableViewSubquerySelfTest.class);
+
         return suite;
     }
 }

From 24d6be4df8fd9ea2778cebef694e927fe5f068b3 Mon Sep 17 00:00:00 2001
From: EdShangGG <eshangareev@gridgain.com>
Date: Wed, 25 Jul 2018 19:38:16 +0300
Subject: [PATCH 376/543] IGNITE-9080 Activate | Deactivate Cluster is killed
 by Linux OOM killer - Fixes #4430.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>
(cherry picked from commit f13d2ed4c609e9149ae0f5a7a2383612203e8d2c)
---
 .../standbycluster/AbstractNodeJoinTemplate.java          | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java
index 0dee9163dda93..ddb29582e4576 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java
@@ -301,8 +301,12 @@ protected CacheConfiguration[] allCacheConfigurations() {
         return super.getConfiguration(name)
             .setDiscoverySpi(
                 new TcpDiscoverySpi()
-                    .setIpFinder(ipFinder)
-            );
+                    .setIpFinder(ipFinder))
+            .setDataStorageConfiguration(
+                new DataStorageConfiguration()
+                    .setDefaultDataRegionConfiguration(
+                        new DataRegionConfiguration()
+                            .setMaxSize(100 * 1024 * 1024)));
     }
 
     /** {@inheritDoc} */

From f8a8b7c92f74587230269210f637d729b9b870fb Mon Sep 17 00:00:00 2001
From: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Date: Mon, 24 Sep 2018 15:00:42 +0300
Subject: [PATCH 377/543] IGNITE-9544 Fixed excessive memory usage of
 BinaryOutputStream#writeBytes - Fixes #4737.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit b769f25bd43af50393eab765b8bd80f9c0cec011)
---
 .../streams/BinaryAbstractOutputStream.java   |  21 +++-
 .../BinaryAbstractOutputStreamTest.java       | 105 ++++++++++++++++++
 .../IgniteBinaryObjectsTestSuite.java         |   2 +
 3 files changed, 125 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStreamTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStream.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStream.java
index 769031f667b29..023e71c111345 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStream.java
@@ -29,6 +29,15 @@ public abstract class BinaryAbstractOutputStream extends BinaryAbstractStream
     /** Minimal capacity when it is reasonable to start doubling resize. */
     private static final int MIN_CAP = 256;
 
+    /**
+     * The maximum size of array to allocate.
+     * Some VMs reserve some header words in an array.
+     * Attempts to allocate larger arrays may result in
+     * OutOfMemoryError: Requested array size exceeds VM limit
+     * @see java.util.ArrayList#MAX_ARRAY_SIZE
+     */
+    protected static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;
+
     /** {@inheritDoc} */
     @Override public void writeByte(byte val) {
         ensureCapacity(pos + 1);
@@ -285,11 +294,17 @@ protected static int capacity(int curCap, int reqCap) {
 
         if (reqCap < MIN_CAP)
             newCap = MIN_CAP;
+        else if (reqCap > MAX_ARRAY_SIZE)
+            throw new IllegalArgumentException("Required capacity exceeds allowed. Required:" + reqCap);
         else {
-            newCap = curCap << 1;
+            newCap = Math.max(curCap, MIN_CAP);
 
-            if (newCap < reqCap)
-                newCap = reqCap;
+            while (newCap < reqCap) {
+                newCap = newCap << 1;
+
+                if (newCap < 0)
+                    newCap = MAX_ARRAY_SIZE;
+            }
         }
 
         return newCap;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStreamTest.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStreamTest.java
new file mode 100644
index 0000000000000..ed1d3d65ad19d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/streams/BinaryAbstractOutputStreamTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary.streams;
+
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class BinaryAbstractOutputStreamTest extends GridCommonAbstractTest {
+    /**
+     *
+     */
+    public void testCapacity() {
+        assertEquals(256, BinaryAbstractOutputStream.capacity(0, 1));
+
+        assertEquals(256, BinaryAbstractOutputStream.capacity(256, 1));
+
+        assertEquals(256, BinaryAbstractOutputStream.capacity(256, 256));
+
+        assertEquals(512, BinaryAbstractOutputStream.capacity(256, 257));
+
+        assertEquals(512, BinaryAbstractOutputStream.capacity(512, 256));
+
+        assertEquals(1024, BinaryAbstractOutputStream.capacity(512, 513));
+
+        assertEquals(2048, BinaryAbstractOutputStream.capacity(1024, 1025));
+
+        assertEquals(4096, BinaryAbstractOutputStream.capacity(2048, 2049));
+
+        assertEquals(8192, BinaryAbstractOutputStream.capacity(4096, 4097));
+
+        assertEquals(16384, BinaryAbstractOutputStream.capacity(8192, 8193));
+
+        assertEquals(32768, BinaryAbstractOutputStream.capacity(16384, 16385));
+
+        assertEquals(65536, BinaryAbstractOutputStream.capacity(32768, 32769));
+
+        assertEquals(131072, BinaryAbstractOutputStream.capacity(65536, 65537));
+
+        assertEquals(262144, BinaryAbstractOutputStream.capacity(131072, 131073));
+
+        assertEquals(524288, BinaryAbstractOutputStream.capacity(262144, 262145));
+
+        assertEquals(1048576, BinaryAbstractOutputStream.capacity(524288, 524289));
+
+        assertEquals(2097152, BinaryAbstractOutputStream.capacity(1048576, 1048577));
+
+        assertEquals(4194304, BinaryAbstractOutputStream.capacity(2097152, 2097153));
+
+        assertEquals(8388608, BinaryAbstractOutputStream.capacity(4194304, 4194305));
+
+        assertEquals(16777216, BinaryAbstractOutputStream.capacity(8388608, 8388609));
+
+        assertEquals(33554432, BinaryAbstractOutputStream.capacity(16777216, 16777217));
+
+        assertEquals(67108864, BinaryAbstractOutputStream.capacity(33554432, 33554433));
+
+        assertEquals(134217728, BinaryAbstractOutputStream.capacity(67108864, 67108865));
+
+        assertEquals(268435456, BinaryAbstractOutputStream.capacity(134217728, 134217729));
+
+        assertEquals(536870912, BinaryAbstractOutputStream.capacity(268435456, 268435457));
+
+        assertEquals(1073741824, BinaryAbstractOutputStream.capacity(536870912, 536870913));
+
+        final int MAX_SIZE = BinaryAbstractOutputStream.MAX_ARRAY_SIZE;
+
+        assertEquals(MAX_SIZE, BinaryAbstractOutputStream.capacity(1073741824, 1073741825));
+
+        assertEquals(MAX_SIZE, BinaryAbstractOutputStream.capacity(0, 1073741825));
+
+        assertEquals(MAX_SIZE, BinaryAbstractOutputStream.capacity(1073741824, 1500000000));
+
+        assertEquals(MAX_SIZE, BinaryAbstractOutputStream.capacity(1073741824, 2000000000));
+
+        assertEquals(MAX_SIZE, BinaryAbstractOutputStream.capacity(1073741824, Integer.MAX_VALUE - 9));
+
+        assertEquals(MAX_SIZE, BinaryAbstractOutputStream.capacity(1073741824, Integer.MAX_VALUE - 8));
+
+        try {
+            assertEquals(MAX_SIZE, BinaryAbstractOutputStream.capacity(0, Integer.MAX_VALUE - 7));
+
+            fail();
+        }
+        catch (IllegalArgumentException ignored) {
+            // Expected exception.
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
index c0211be4adbd7..240396e4e0518 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
@@ -50,6 +50,7 @@
 import org.apache.ignite.internal.binary.noncompact.BinaryObjectBuilderAdditionalNonCompactSelfTest;
 import org.apache.ignite.internal.binary.noncompact.BinaryObjectBuilderNonCompactDefaultMappersSelfTest;
 import org.apache.ignite.internal.binary.noncompact.BinaryObjectBuilderNonCompactSimpleNameLowerCaseMappersSelfTest;
+import org.apache.ignite.internal.binary.streams.BinaryAbstractOutputStreamTest;
 import org.apache.ignite.internal.binary.streams.BinaryHeapStreamByteOrderSelfTest;
 import org.apache.ignite.internal.binary.streams.BinaryOffheapStreamByteOrderSelfTest;
 import org.apache.ignite.internal.processors.cache.binary.BinaryAtomicCacheLocalEntriesSelfTest;
@@ -144,6 +145,7 @@ public static TestSuite suite() throws Exception {
 
         // Byte order
         suite.addTestSuite(BinaryHeapStreamByteOrderSelfTest.class);
+        suite.addTestSuite(BinaryAbstractOutputStreamTest.class);
         suite.addTestSuite(BinaryOffheapStreamByteOrderSelfTest.class);
 
         suite.addTestSuite(GridCacheBinaryObjectUserClassloaderSelfTest.class);

From 817ce30fbfc0bf0dcb18d3dae1a981830b5f344f Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Mon, 10 Sep 2018 18:43:41 +0300
Subject: [PATCH 378/543] IGNITE-9445 Use valid tag for page write unlock while
 reading cold page from disk - Fixes #4708.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 6a1aa1f061a4399527215ffe71b02086847cd506)
---
 .../persistence/pagemem/PageMemoryImpl.java   |   7 +-
 .../CachePageWriteLockUnlockTest.java         | 196 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite7.java     |   3 +
 3 files changed, 205 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index 0f3a31cb1e78b..ff2d5b23324ba 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -778,9 +778,13 @@ else if (relPtr == OUTDATED_REL_PTR) {
 
                 ByteBuffer buf = wrapPointer(pageAddr, pageSize());
 
+                long actualPageId = 0;
+
                 try {
                     storeMgr.read(grpId, pageId, buf);
 
+                    actualPageId = PageIO.getPageId(buf);
+
                     memMetrics.onPageRead();
                 }
                 catch (IgniteDataIntegrityViolationException ignore) {
@@ -794,7 +798,8 @@ else if (relPtr == OUTDATED_REL_PTR) {
                     memMetrics.onPageRead();
                 }
                 finally {
-                    rwLock.writeUnlock(lockedPageAbsPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
+                    rwLock.writeUnlock(lockedPageAbsPtr + PAGE_LOCK_OFFSET,
+                        actualPageId == 0 ? OffheapReadWriteLock.TAG_LOCK_ALWAYS : PageIdUtils.tag(actualPageId));
                 }
             }
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java
new file mode 100644
index 0000000000000..f6a5ec1d1c20c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.Iterator;
+import javax.cache.Cache;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
+import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+
+/**
+ *
+ */
+public class CachePageWriteLockUnlockTest extends GridCommonAbstractTest {
+    /** */
+    public static final int PARTITION = 0;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME).
+            setAffinity(new RendezvousAffinityFunction(false, 32)));
+
+        cfg.setActiveOnStart(false);
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setPersistenceEnabled(true)
+                    .setMaxSize(DataStorageConfiguration.DFLT_DATA_REGION_INITIAL_SIZE)
+            )
+            .setWalMode(WALMode.LOG_ONLY).setCheckpointFrequency(Integer.MAX_VALUE);
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        return cfg;
+    }
+
+    /**
+     *
+     */
+    public void testPreloadPartition() throws Exception {
+        try {
+            IgniteEx grid0 = startGrid(0);
+
+            grid0.cluster().active(true);
+
+            int total = 512;
+
+            putData(grid0, total, PARTITION);
+
+            grid0.cache(DEFAULT_CACHE_NAME).removeAll();
+
+            forceCheckpoint();
+
+            stopGrid(0);
+
+            grid0 = startGrid(0);
+
+            grid0.cluster().active(true);
+
+            putData(grid0, total, PARTITION); // Will use pages from reuse pool.
+
+            forceCheckpoint();
+
+            stopGrid(0);
+
+            grid0 = startGrid(0);
+
+            preloadPartition(grid0, DEFAULT_CACHE_NAME, PARTITION);
+
+            Iterator<Cache.Entry<Object, Object>> it = grid0.cache(DEFAULT_CACHE_NAME).iterator();
+
+            int c0 = 0;
+
+            while (it.hasNext()) {
+                Cache.Entry<Object, Object> entry = it.next();
+
+                c0++;
+            }
+
+            assertEquals(total, c0);
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
+    /**
+     * @param grid Grid.
+     * @param total Total.
+     * @param part Partition.
+     */
+    private void putData(Ignite grid, int total, int part) {
+        int c = 0, k = 0;
+
+        while(c < total) {
+            if (grid(0).affinity(DEFAULT_CACHE_NAME).partition(k) == part) {
+                grid.cache(DEFAULT_CACHE_NAME).put(k, k);
+
+                c++;
+            }
+
+            k++;
+        }
+    }
+
+    /**
+     * Preload partition fast by iterating on all pages in disk order.
+     *
+     * @param grid Grid.
+     * @param cacheName Cache name.
+     * @param p P.
+     */
+    private void preloadPartition(Ignite grid, String cacheName, int p) throws IgniteCheckedException {
+        GridDhtCacheAdapter<Object, Object> dht = ((IgniteKernal)grid).internalCache(cacheName).context().dht();
+
+        GridDhtLocalPartition part = dht.topology().localPartition(p);
+
+        assertNotNull(part);
+
+        assertTrue(part.state() == OWNING);
+
+        CacheGroupContext grpCtx = dht.context().group();
+
+        if (part.state() != OWNING)
+            return;
+
+        IgnitePageStoreManager pageStoreMgr = grpCtx.shared().pageStore();
+
+        if (pageStoreMgr instanceof FilePageStoreManager) {
+            FilePageStoreManager filePageStoreMgr = (FilePageStoreManager)pageStoreMgr;
+
+            PageStore pageStore = filePageStoreMgr.getStore(grpCtx.groupId(), part.id());
+
+            PageMemoryEx pageMemory = (PageMemoryEx)grpCtx.dataRegion().pageMemory();
+
+            long pageId = pageMemory.partitionMetaPageId(grpCtx.groupId(), part.id());
+
+            for (int pageNo = 0; pageNo < pageStore.pages(); pageId++, pageNo++) {
+                long pagePointer = -1;
+
+                try {
+                    pagePointer = pageMemory.acquirePage(grpCtx.groupId(), pageId);
+                }
+                finally {
+                    if (pagePointer != -1)
+                        pageMemory.releasePage(grpCtx.groupId(), pageId, pagePointer);
+                }
+            }
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        cleanPersistenceDir();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index 471f8228034d8..b2cb6d6d9a714 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.processors.cache.WalModeChangeCoordinatorNotAffinityNodeSelfTest;
 import org.apache.ignite.internal.processors.cache.WalModeChangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
+import org.apache.ignite.internal.processors.cache.distributed.CachePageWriteLockUnlockTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheDataLossOnPartitionMoveTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest;
@@ -101,6 +102,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class);
         suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class);
 
+        suite.addTestSuite(CachePageWriteLockUnlockTest.class);
+
         return suite;
     }
 }

From 2f734c5de3fee0d57acaa7e30fd4ffff7537c1e0 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Tue, 25 Sep 2018 16:02:52 +0300
Subject: [PATCH 379/543] IGNITE-8559 Fix WAL rollOver can be blocked by WAL
 iterator reservation - Fixes #4449.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit 2f72fe7)

(cherry picked from commit 85fedb5)
---
 .../wal/IgniteWriteAheadLogManager.java       |   2 +-
 .../GridCacheDatabaseSharedManager.java       |   4 +-
 .../cache/persistence/file/FileIOFactory.java |   1 -
 .../wal/AbstractWalRecordsIterator.java       |  50 +-
 .../cache/persistence/wal/FileDescriptor.java | 146 +++++
 .../wal/FileWriteAheadLogManager.java         | 606 ++++++------------
 .../FsyncModeFileWriteAheadLogManager.java    | 244 ++-----
 .../wal/SegmentArchivedMonitor.java           |  64 --
 .../cache/persistence/wal/SegmentRouter.java  |  90 +++
 .../SingleSegmentLogicalRecordsIterator.java  |  15 +-
 .../wal/aware/SegmentArchivedStorage.java     | 137 ++++
 .../persistence/wal/aware/SegmentAware.java   | 234 +++++++
 .../wal/aware/SegmentCompressStorage.java     | 116 ++++
 .../wal/aware/SegmentCurrentStateStorage.java | 171 +++++
 .../wal/aware/SegmentLockStorage.java         |  76 +++
 .../wal/aware/SegmentObservable.java          |  46 ++
 .../SegmentReservationStorage.java            |   7 +-
 .../cache/persistence/wal/io/FileInput.java   | 258 ++++++++
 .../wal/io/LockedReadFileInput.java           | 111 ++++
 .../wal/io/LockedSegmentFileInputFactory.java |  68 ++
 .../wal/io/SegmentFileInputFactory.java       |  34 +
 .../cache/persistence/wal/io/SegmentIO.java   |  45 ++
 .../SimpleFileInput.java}                     | 233 +------
 .../wal/io/SimpleSegmentFileInputFactory.java |  33 +
 .../wal/reader/IgniteWalIteratorFactory.java  |  15 +-
 .../reader/StandaloneWalRecordsIterator.java  |  25 +-
 .../wal/serializer/RecordSerializer.java      |   2 +-
 .../wal/serializer/RecordV1Serializer.java    |  16 +-
 .../wal/serializer/RecordV2Serializer.java    |   2 +-
 ...ReachedDuringIterationOverArchiveTest.java |   2 +-
 .../wal/IgniteWalHistoryReservationsTest.java |  87 ++-
 .../IgniteWalIteratorSwitchSegmentTest.java   | 124 +++-
 .../db/wal/IgniteWalRecoveryTest.java         | 139 +++-
 .../persistence/db/wal/WalCompactionTest.java |   7 +-
 ...niteAbstractWalIteratorInvalidCrcTest.java |   2 +-
 .../db/wal/crc/IgniteDataIntegrityTests.java  |   7 +-
 .../persistence/pagemem/NoOpWALManager.java   |   2 +-
 .../wal/aware/SegmentAwareTest.java           | 601 +++++++++++++++++
 .../StandaloneWalRecordsIteratorTest.java     |   2 +-
 .../ignite/testframework/GridTestUtils.java   | 108 ++--
 .../ignite/testsuites/IgnitePdsTestSuite.java |   4 +-
 41 files changed, 2866 insertions(+), 1070 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
 delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentArchivedMonitor.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentRouter.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCurrentStateStorage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/{ => aware}/SegmentReservationStorage.java (93%)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/LockedReadFileInput.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/LockedSegmentFileInputFactory.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SegmentFileInputFactory.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SegmentIO.java
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/{FileInput.java => io/SimpleFileInput.java} (52%)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleSegmentFileInputFactory.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 7de005cf10c1a..26a9bcebf2000 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -115,7 +115,7 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
      *
      * @param ptr Pointer for which it is safe to compact the log.
      */
-    public void allowCompressionUntil(WALPointer ptr);
+    public void notchLastCheckpointPtr(WALPointer ptr);
 
     /**
      * @return Total number of segments in the WAL archive.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index d7b514eb08d62..6d46e5677ac43 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1923,7 +1923,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
             cctx.pageStore().beginRecover();
         }
         else
-            cctx.wal().allowCompressionUntil(status.startPtr);
+            cctx.wal().notchLastCheckpointPtr(status.startPtr);
 
         long start = U.currentTimeMillis();
 
@@ -3679,7 +3679,7 @@ private void markCheckpointEnd(Checkpoint chp) throws IgniteCheckedException {
 
                 writeCheckpointEntry(tmpWriteBuf, cp, CheckpointEntryType.END);
 
-                cctx.wal().allowCompressionUntil(chp.cpEntry.checkpointMark());
+                cctx.wal().notchLastCheckpointPtr(chp.cpEntry.checkpointMark());
             }
 
             List<CheckpointEntry> removedFromHistory = cpHistory.onCheckpointFinished(chp, truncateWalOnCpFinish);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java
index c3a75f5310e4f..27351853269db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java
@@ -44,5 +44,4 @@ public interface FileIOFactory extends Serializable {
      * @throws IOException If I/O interface creation was failed.
      */
     public FileIO create(File file, OpenOption... modes) throws IOException;
-
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index 0b704caecd30c..8a38f28e607ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -30,7 +30,9 @@
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
-import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.SegmentHeader;
@@ -87,24 +89,29 @@ public abstract class AbstractWalRecordsIterator
     /** Utility buffer for reading records */
     private final ByteBufferExpander buf;
 
+    /** Factory to provide I/O interfaces for read primitives with files. */
+    private final SegmentFileInputFactory segmentFileInputFactory;
+
     /**
      * @param log Logger.
      * @param sharedCtx Shared context.
      * @param serializerFactory Serializer of current version to read headers.
      * @param ioFactory ioFactory for file IO access.
      * @param initialReadBufferSize buffer for reading records size.
+     * @param segmentFileInputFactory Factory to provide I/O interfaces for read primitives with files.
      */
     protected AbstractWalRecordsIterator(
         @NotNull final IgniteLogger log,
         @NotNull final GridCacheSharedContext sharedCtx,
         @NotNull final RecordSerializerFactory serializerFactory,
         @NotNull final FileIOFactory ioFactory,
-        final int initialReadBufferSize
-    ) {
+        final int initialReadBufferSize,
+        SegmentFileInputFactory segmentFileInputFactory) {
         this.log = log;
         this.sharedCtx = sharedCtx;
         this.serializerFactory = serializerFactory;
         this.ioFactory = ioFactory;
+        this.segmentFileInputFactory = segmentFileInputFactory;
 
         buf = new ByteBufferExpander(initialReadBufferSize, ByteOrder.nativeOrder());
     }
@@ -134,11 +141,8 @@ protected AbstractWalRecordsIterator(
     }
 
     /**
-     * Switches records iterator to the next record.
-     * <ul>
-     * <li>{@link #curRec} will be updated.</li>
-     * <li> If end of segment reached, switch to new segment is called. {@link #currWalSegment} will be updated.</li>
-     * </ul>
+     * Switches records iterator to the next record. <ul> <li>{@link #curRec} will be updated.</li> <li> If end of
+     * segment reached, switch to new segment is called. {@link #currWalSegment} will be updated.</li> </ul>
      *
      * {@code advance()} runs a step ahead {@link #next()}
      *
@@ -301,16 +305,16 @@ protected IgniteCheckedException handleRecordException(@NotNull final Exception
     protected AbstractReadFileHandle initReadHandle(
         @NotNull final AbstractFileDescriptor desc,
         @Nullable final FileWALPointer start,
-        @NotNull final FileIO fileIO,
+        @NotNull final SegmentIO fileIO,
         @NotNull final SegmentHeader segmentHeader
     ) throws IgniteCheckedException {
         try {
-            final boolean isCompacted = segmentHeader.isCompacted();
+            boolean isCompacted = segmentHeader.isCompacted();
 
             if (isCompacted)
                 serializerFactory.skipPositionCheck(true);
 
-            FileInput in = new FileInput(fileIO, buf);
+            FileInput in = segmentFileInputFactory.createFileInput(fileIO, buf);
 
             if (start != null && desc.idx() == start.index()) {
                 if (isCompacted) {
@@ -327,7 +331,7 @@ protected AbstractReadFileHandle initReadHandle(
 
             int serVer = segmentHeader.getSerializerVersion();
 
-            return createReadFileHandle(fileIO, desc.idx(), serializerFactory.createSerializer(serVer), in);
+            return createReadFileHandle(fileIO, serializerFactory.createSerializer(serVer), in);
         }
         catch (SegmentEofException | EOFException ignore) {
             try {
@@ -366,15 +370,15 @@ protected AbstractReadFileHandle initReadHandle(
         @NotNull final AbstractFileDescriptor desc,
         @Nullable final FileWALPointer start
     ) throws IgniteCheckedException, FileNotFoundException {
-        FileIO fileIO = null;
+        SegmentIO fileIO = null;
 
         try {
-            fileIO = desc.isCompressed() ? new UnzipFileIO(desc.file()) : ioFactory.create(desc.file());
+            fileIO = desc.toIO(ioFactory);
 
             SegmentHeader segmentHeader;
 
             try {
-                segmentHeader = readSegmentHeader(fileIO, curWalSegmIdx);
+                segmentHeader = readSegmentHeader(fileIO, segmentFileInputFactory);
             }
             catch (SegmentEofException | EOFException ignore) {
                 try {
@@ -409,8 +413,7 @@ protected AbstractReadFileHandle initReadHandle(
 
     /** */
     protected abstract AbstractReadFileHandle createReadFileHandle(
-        FileIO fileIO,
-        long idx,
+        SegmentIO fileIO,
         RecordSerializer ser,
         FileInput in
     );
@@ -458,7 +461,9 @@ protected interface AbstractReadFileHandle {
         /** */
         RecordSerializer ser();
 
-        /** */
+        /**
+         *
+         */
         boolean workDir();
     }
 
@@ -472,5 +477,14 @@ protected interface AbstractFileDescriptor {
 
         /** */
         long idx();
+
+        /**
+         * Make fileIo by this description.
+         *
+         * @param fileIOFactory Factory for fileIo creation.
+         * @return One of implementation of {@link FileIO}.
+         * @throws IOException if creation of fileIo was not success.
+         */
+        SegmentIO toIO(FileIOFactory fileIOFactory) throws IOException;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
new file mode 100644
index 0000000000000..f2653765cb1e5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileDescriptor.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal;
+
+import java.io.File;
+import java.io.IOException;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * WAL file descriptor.
+ */
+public class FileDescriptor implements Comparable<FileDescriptor>, AbstractWalRecordsIterator.AbstractFileDescriptor {
+
+    /** file extension of WAL segment. */
+    private static final String WAL_SEGMENT_FILE_EXT = ".wal";
+
+    /** Length of WAL segment file name. */
+    private static final int WAL_SEGMENT_FILE_NAME_LENGTH = 16;
+
+    /** File represented by this class. */
+    protected final File file;
+
+    /** Absolute WAL segment file index. */
+    protected final long idx;
+
+    /**
+     * Creates file descriptor. Index is restored from file name.
+     *
+     * @param file WAL segment file.
+     */
+    public FileDescriptor(@NotNull File file) {
+        this(file, null);
+    }
+
+    /**
+     * @param file WAL segment file.
+     * @param idx Absolute WAL segment file index. For null value index is restored from file name.
+     */
+    public FileDescriptor(@NotNull File file, @Nullable Long idx) {
+        this.file = file;
+
+        String fileName = file.getName();
+
+        assert fileName.contains(WAL_SEGMENT_FILE_EXT);
+
+        this.idx = idx == null ? Long.parseLong(fileName.substring(0, WAL_SEGMENT_FILE_NAME_LENGTH)) : idx;
+    }
+
+    /**
+     * @param segment Segment index.
+     * @return Segment file name.
+     */
+    public static String fileName(long segment) {
+        SB b = new SB();
+
+        String segmentStr = Long.toString(segment);
+
+        for (int i = segmentStr.length(); i < WAL_SEGMENT_FILE_NAME_LENGTH; i++)
+            b.a('0');
+
+        b.a(segmentStr).a(WAL_SEGMENT_FILE_EXT);
+
+        return b.toString();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int compareTo(@NotNull FileDescriptor o) {
+        return Long.compare(idx, o.idx);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+
+        if (!(o instanceof FileDescriptor))
+            return false;
+
+        FileDescriptor that = (FileDescriptor)o;
+
+        return idx == that.idx;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        return (int)(idx ^ (idx >>> 32));
+    }
+
+    /**
+     * @return Absolute WAL segment file index
+     */
+    public long getIdx() {
+        return idx;
+    }
+
+    /**
+     * @return absolute pathname string of this file descriptor pathname.
+     */
+    public String getAbsolutePath() {
+        return file.getAbsolutePath();
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean isCompressed() {
+        return file.getName().endsWith(FilePageStoreManager.ZIP_SUFFIX);
+    }
+
+    /** {@inheritDoc} */
+    @Override public File file() {
+        return file;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long idx() {
+        return idx;
+    }
+
+    /** {@inheritDoc} */
+    @Override public SegmentIO toIO(FileIOFactory fileIOFactory) throws IOException {
+        FileIO fileIO = isCompressed() ? new UnzipFileIO(file()) : fileIOFactory.create(file());
+
+        return new SegmentIO(idx, fileIO);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 60ee0a284bcce..23aa06604bfc8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -96,7 +96,13 @@
 import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator.AbstractFileDescriptor;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.LockedSegmentFileInputFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleSegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
+import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
@@ -238,13 +244,6 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     private static final AtomicLongFieldUpdater<FileWriteHandle> WRITTEN_UPD =
         AtomicLongFieldUpdater.newUpdater(FileWriteHandle.class, "written");
 
-    /** Interrupted flag. */
-    private final ThreadLocal<Boolean> interrupted = new ThreadLocal<Boolean>() {
-        @Override protected Boolean initialValue() {
-            return false;
-        }
-    };
-
     /** */
     private final boolean alwaysWriteFullPages;
 
@@ -288,17 +287,14 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     private final int serializerVer =
         IgniteSystemProperties.getInteger(IGNITE_WAL_SERIALIZER_VERSION, LATEST_SERIALIZER_VERSION);
 
-    /** Latest segment cleared by {@link #truncate(WALPointer, WALPointer)}. */
-    private volatile long lastTruncatedArchiveIdx = -1L;
-
     /** Factory to provide I/O interfaces for read/write operations with files */
     private volatile FileIOFactory ioFactory;
 
-    /** Next WAL segment archived monitor. Manages last archived index, emulates archivation in no-archiver mode. */
-    private final SegmentArchivedMonitor archivedMonitor = new SegmentArchivedMonitor();
+    /** Factory to provide I/O interfaces for read primitives with files */
+    private final SegmentFileInputFactory segmentFileInputFactory;
 
-    /** Segment reservations storage: Protects WAL segments from deletion during WAL log cleanup. */
-    private final SegmentReservationStorage reservationStorage = new SegmentReservationStorage();
+    /** Holder of actual information of latest manipulation on WAL segments. */
+    private final SegmentAware segmentAware;
 
     /** Updater for {@link #currHnd}, used for verify there are no concurrent update for current log segment handle */
     private static final AtomicReferenceFieldUpdater<FileWriteAheadLogManager, FileWriteHandle> CURR_HND_UPD =
@@ -361,6 +357,14 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     /** Wal segment sync worker. */
     private WalSegmentSyncer walSegmentSyncWorker;
 
+    /**
+     * Manage of segment location.
+     */
+    private SegmentRouter segmentRouter;
+
+    /** Segment factory with ability locked segment during reading. */
+    private SegmentFileInputFactory lockedSegmentFileInputFactory;
+
     /**
      * @param ctx Kernal context.
      */
@@ -379,9 +383,11 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) {
         fsyncDelay = dsCfg.getWalFsyncDelayNanos();
         alwaysWriteFullPages = dsCfg.isAlwaysWriteFullPages();
         ioFactory = new RandomAccessFileIOFactory();
+        segmentFileInputFactory = new SimpleSegmentFileInputFactory();
         walAutoArchiveAfterInactivity = dsCfg.getWalAutoArchiveAfterInactivity();
         evt = ctx.event();
         failureProcessor = ctx.failure();
+        segmentAware = new SegmentAware(dsCfg.getWalSegments());
     }
 
     /**
@@ -439,7 +445,7 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
 
             IgniteBiTuple<Long, Long> tup = scanMinMaxArchiveIndices();
 
-            lastTruncatedArchiveIdx = tup == null ? -1 : tup.get1() - 1;
+            segmentAware.lastTruncatedArchiveIdx(tup == null ? -1 : tup.get1() - 1);
 
             long lastAbsArchivedIdx = tup == null ? -1 : tup.get2();
 
@@ -449,7 +455,7 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
                 archiver = null;
 
             if (lastAbsArchivedIdx > 0)
-                archivedMonitor.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx);
+                segmentAware.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx);
 
             if (dsCfg.isWalCompactionEnabled()) {
                 compressor = new FileCompressor();
@@ -461,6 +467,8 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
                 }
             }
 
+            segmentRouter = new SegmentRouter(walWorkDir, walArchiveDir, segmentAware, dsCfg);
+
             walDisableContext = cctx.walState().walDisableContext();
 
             if (mode != WALMode.NONE && mode != WALMode.FSYNC) {
@@ -473,6 +481,12 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
             else
                 U.quietAndWarn(log, "Started write-ahead log manager in NONE mode, persisted data may be lost in " +
                     "a case of unexpected node failure. Make sure to deactivate the cluster before shutdown.");
+
+            lockedSegmentFileInputFactory = new LockedSegmentFileInputFactory(
+                segmentAware,
+                segmentRouter,
+                ioFactory
+            );
         }
     }
 
@@ -497,7 +511,7 @@ private boolean isArchiverEnabled() {
     public Collection<File> getAndReserveWalFiles(FileWALPointer low, FileWALPointer high) throws IgniteCheckedException {
         final long awaitIdx = high.index() - 1;
 
-        archivedMonitor.awaitSegmentArchived(awaitIdx);
+        segmentAware.awaitSegmentArchived(awaitIdx);
 
         if (!reserve(low))
             throw new IgniteCheckedException("WAL archive segment has been deleted [idx=" + low.index() + "]");
@@ -570,6 +584,8 @@ private void checkWalConfiguration() throws IgniteCheckedException {
             if (walWriter != null)
                 walWriter.shutdown();
 
+            segmentAware.interrupt();
+
             if (archiver != null)
                 archiver.shutdown();
 
@@ -764,14 +780,14 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
             if (rec.rollOver()){
                 assert cctx.database().checkpointLockIsHeldByThread();
 
-                long idx = currWrHandle.idx;
+                long idx = currWrHandle.getSegmentId();
 
                 currWrHandle.buf.close();
 
                 currWrHandle = rollOver(currWrHandle);
 
                 if (log != null && log.isInfoEnabled())
-                    log.info("Rollover segment [" + idx + " to " + currWrHandle.idx + "], recordType=" + rec.type());
+                    log.info("Rollover segment [" + idx + " to " + currWrHandle.getSegmentId() + "], recordType=" + rec.type());
             }
 
             WALPointer ptr = currWrHandle.addRecord(rec);
@@ -835,8 +851,8 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
 
         return new RecordsIterator(
             cctx,
-            walWorkDir,
             walArchiveDir,
+            walWorkDir,
             (FileWALPointer)start,
             end,
             dsCfg,
@@ -844,8 +860,10 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
             ioFactory,
             archiver,
             decompressor,
-            log
-        );
+            log,
+            segmentAware,
+            segmentRouter,
+            lockedSegmentFileInputFactory);
     }
 
     /** {@inheritDoc} */
@@ -855,10 +873,10 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
         if (mode == WALMode.NONE)
             return false;
 
-        reservationStorage.reserve(((FileWALPointer)start).index());
+        segmentAware.reserve(((FileWALPointer)start).index());
 
         if (!hasIndex(((FileWALPointer)start).index())) {
-            reservationStorage.release(((FileWALPointer)start).index());
+            segmentAware.release(((FileWALPointer)start).index());
 
             return false;
         }
@@ -873,7 +891,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
         if (mode == WALMode.NONE)
             return;
 
-        reservationStorage.release(((FileWALPointer)start).index());
+        segmentAware.release(((FileWALPointer)start).index());
     }
 
     /**
@@ -896,7 +914,7 @@ private boolean hasIndex(long absIdx) {
 
         FileWriteHandle cur = currHnd;
 
-        return cur != null && cur.idx >= absIdx;
+        return cur != null && cur.getSegmentId() >= absIdx;
     }
 
     /** {@inheritDoc} */
@@ -922,7 +940,7 @@ private boolean hasIndex(long absIdx) {
             if (segmentReservedOrLocked(desc.idx))
                 return deleted;
 
-            long archivedAbsIdx = archivedMonitor.lastArchivedAbsoluteIndex();
+            long archivedAbsIdx = segmentAware.lastArchivedAbsoluteIndex();
 
             long lastArchived = archivedAbsIdx >= 0 ? archivedAbsIdx : lastArchivedIndex();
 
@@ -935,8 +953,8 @@ private boolean hasIndex(long absIdx) {
                     deleted++;
 
                 // Bump up the oldest archive segment index.
-                if (lastTruncatedArchiveIdx < desc.idx)
-                    lastTruncatedArchiveIdx = desc.idx;
+                if (segmentAware.lastTruncatedArchiveIdx() < desc.idx)
+                    segmentAware.lastTruncatedArchiveIdx(desc.idx);
             }
         }
 
@@ -953,22 +971,20 @@ private boolean hasIndex(long absIdx) {
     private boolean segmentReservedOrLocked(long absIdx) {
         FileArchiver archiver0 = archiver;
 
-        return ((archiver0 != null) && archiver0.locked(absIdx))
-            || (reservationStorage.reserved(absIdx));
-
+        return ((archiver0 != null) && segmentAware.locked(absIdx)) || (segmentAware.reserved(absIdx));
     }
 
     /** {@inheritDoc} */
-    @Override public void allowCompressionUntil(WALPointer ptr) {
+    @Override public void notchLastCheckpointPtr(WALPointer ptr) {
         if (compressor != null)
-            compressor.allowCompressionUntil(((FileWALPointer)ptr).index());
+            compressor.keepUncompressedIdxFrom(((FileWALPointer)ptr).index());
     }
 
     /** {@inheritDoc} */
     @Override public int walArchiveSegments() {
-        long lastTruncated = lastTruncatedArchiveIdx;
+        long lastTruncated = segmentAware.lastTruncatedArchiveIdx();
 
-        long lastArchived = archivedMonitor.lastArchivedAbsoluteIndex();
+        long lastArchived = segmentAware.lastArchivedAbsoluteIndex();
 
         if (lastArchived == -1)
             return 0;
@@ -980,7 +996,7 @@ private boolean segmentReservedOrLocked(long absIdx) {
 
     /** {@inheritDoc} */
     @Override public long lastArchivedSegment() {
-        return archivedMonitor.lastArchivedAbsoluteIndex();
+        return segmentAware.lastArchivedAbsoluteIndex();
     }
 
     /** {@inheritDoc} */
@@ -1171,7 +1187,7 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
         int len = lastReadPtr == null ? 0 : lastReadPtr.length();
 
         try {
-            FileIO fileIO = ioFactory.create(curFile);
+            SegmentIO fileIO = new SegmentIO(absIdx, ioFactory.create(curFile));
 
             IgniteInClosure<FileIO> lsnr = createWalFileListener;
 
@@ -1184,7 +1200,7 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
                 // If we have existing segment, try to read version from it.
                 if (lastReadPtr != null) {
                     try {
-                        serVer = readSegmentHeader(fileIO, absIdx).getSerializerVersion();
+                        serVer = readSegmentHeader(fileIO, segmentFileInputFactory).getSerializerVersion();
                     }
                     catch (SegmentEofException | EOFException ignore) {
                         serVer = serializerVer;
@@ -1212,16 +1228,15 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
 
                 FileWriteHandle hnd = new FileWriteHandle(
                     fileIO,
-                    absIdx,
                     off + len,
                     true,
                     ser,
                     rbuf);
 
                 if (archiver0 != null)
-                    archiver0.currentWalIndex(absIdx);
+                    segmentAware.curAbsWalIdx(absIdx);
                 else
-                    archivedMonitor.setLastArchivedAbsoluteIndex(absIdx - 1);
+                    segmentAware.setLastArchivedAbsoluteIndex(absIdx - 1);
 
                 return hnd;
             }
@@ -1256,22 +1271,22 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws IgniteCh
         IgniteCheckedException error = null;
 
         try {
-            File nextFile = pollNextFile(cur.idx);
+            File nextFile = pollNextFile(cur.getSegmentId());
 
             if (log.isDebugEnabled())
                 log.debug("Switching to a new WAL segment: " + nextFile.getAbsolutePath());
 
             SegmentedRingByteBuffer rbuf = null;
 
-            FileIO fileIO = null;
+            SegmentIO fileIO = null;
 
             FileWriteHandle hnd;
 
-            boolean interrupted = this.interrupted.get();
+            boolean interrupted = false;
 
             while (true) {
                 try {
-                    fileIO = ioFactory.create(nextFile);
+                    fileIO = new SegmentIO(cur.getSegmentId() + 1, ioFactory.create(nextFile));
 
                     IgniteInClosure<FileIO> lsnr = createWalFileListener;
                     if (lsnr != null)
@@ -1287,7 +1302,6 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws IgniteCh
 
                     hnd = new FileWriteHandle(
                         fileIO,
-                        cur.idx + 1,
                         0,
                         false,
                         serializer,
@@ -1321,9 +1335,6 @@ private FileWriteHandle initNextWriteHandle(FileWriteHandle cur) throws IgniteCh
                         rbuf = null;
                     }
                 }
-                finally {
-                    this.interrupted.set(false);
-                }
             }
 
             return hnd;
@@ -1477,17 +1488,17 @@ private void createFile(File file) throws StorageException {
      * @return File ready for use as new WAL segment.
      * @throws StorageException If exception occurred in the archiver thread.
      */
-    private File pollNextFile(long curIdx) throws StorageException {
+    private File pollNextFile(long curIdx) throws StorageException, IgniteInterruptedCheckedException {
         FileArchiver archiver0 = archiver;
 
         if (archiver0 == null) {
-            archivedMonitor.setLastArchivedAbsoluteIndex(curIdx);
+            segmentAware.setLastArchivedAbsoluteIndex(curIdx);
 
             return new File(walWorkDir, FileDescriptor.fileName(curIdx + 1));
         }
 
         // Signal to archiver that we are done with the segment and it can be archived.
-        long absNextIdx = archiver0.nextAbsoluteSegmentIndex(curIdx);
+        long absNextIdx = archiver0.nextAbsoluteSegmentIndex();
 
         long segmentIdx = absNextIdx % dsCfg.getWalSegments();
 
@@ -1547,35 +1558,20 @@ public long maxWalSegmentSize() {
      *
      * Monitor of current object is used for notify on: <ul> <li>exception occurred ({@link
      * FileArchiver#cleanErr}!=null)</li> <li>stopping thread ({@link FileArchiver#stopped}==true)</li> <li>current file
-     * index changed ({@link FileArchiver#curAbsWalIdx})</li> <li>last archived file index was changed ({@link
-     * FileArchiver#lastAbsArchivedIdx})</li> <li>some WAL index was removed from {@link FileArchiver#locked} map</li>
+     * index changed </li> <li>last archived file index was changed ({@link
+     * </li> <li>some WAL index was removed from map</li>
      * </ul>
      */
     private class FileArchiver extends GridWorker {
         /** Exception which occurred during initial creation of files or during archiving WAL segment */
         private StorageException cleanErr;
 
-        /**
-         * Absolute current segment index WAL Manager writes to. Guarded by <code>this</code>. Incremented during
-         * rollover. Also may be directly set if WAL is resuming logging after start.
-         */
-        private long curAbsWalIdx = -1;
-
-        /** Last archived file index (absolute, 0-based). Guarded by <code>this</code>. */
-        private volatile long lastAbsArchivedIdx = -1;
-
         /** current thread stopping advice */
         private volatile boolean stopped;
 
         /** Formatted index. */
         private int formatted;
 
-        /**
-         * Maps absolute segment index to locks counter. Lock on segment protects from archiving segment and may come
-         * from {@link RecordsIterator} during WAL replay. Map itself is guarded by <code>this</code>.
-         */
-        private Map<Long, Integer> locked = new HashMap<>();
-
         /**
          *
          */
@@ -1583,7 +1579,7 @@ private FileArchiver(long lastAbsArchivedIdx, IgniteLogger log) {
             super(cctx.igniteInstanceName(), "wal-file-archiver%" + cctx.igniteInstanceName(), log,
                 cctx.kernalContext().workersRegistry());
 
-            this.lastAbsArchivedIdx = lastAbsArchivedIdx;
+            segmentAware.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx);
         }
 
         /**
@@ -1599,27 +1595,6 @@ private void shutdown() throws IgniteInterruptedCheckedException {
             U.join(runner());
         }
 
-        /**
-         * @param curAbsWalIdx Current absolute WAL segment index.
-         */
-        private void currentWalIndex(long curAbsWalIdx) {
-            synchronized (this) {
-                this.curAbsWalIdx = curAbsWalIdx;
-
-                notifyAll();
-            }
-        }
-
-        /**
-         * Check if WAL segment locked (protected from move to archive)
-         *
-         * @param absIdx Index for check reservation.
-         * @return {@code True} if index is locked.
-         */
-        private synchronized boolean locked(long absIdx) {
-            return locked.containsKey(absIdx);
-        }
-
         /** {@inheritDoc} */
         @Override protected void body() {
             try {
@@ -1630,6 +1605,8 @@ private synchronized boolean locked(long absIdx) {
                     // Stop the thread and report to starter.
                     cleanErr = e;
 
+                    segmentAware.forceInterrupt();
+
                     notifyAll();
                 }
 
@@ -1641,57 +1618,36 @@ private synchronized boolean locked(long absIdx) {
             Throwable err = null;
 
             try {
-                synchronized (this) {
-                    while (curAbsWalIdx == -1 && !stopped)
-                        wait();
 
-                    // If the archive directory is empty, we can be sure that there were no WAL segments archived.
-                    // This is ensured by the check in truncate() which will leave at least one file there
-                    // once it was archived.
-                }
+                segmentAware.awaitSegment(0);//wait for init at least one work segments.
 
                 while (!Thread.currentThread().isInterrupted() && !stopped) {
                     long toArchive;
 
-                    synchronized (this) {
-                        assert lastAbsArchivedIdx <= curAbsWalIdx : "lastArchived=" + lastAbsArchivedIdx +
-                            ", current=" + curAbsWalIdx;
-
-                        while (lastAbsArchivedIdx >= curAbsWalIdx - 1 && !stopped)
-                            wait();
-
-                        toArchive = lastAbsArchivedIdx + 1;
-                    }
+                    toArchive = segmentAware.waitNextSegmentForArchivation();
 
                     if (stopped)
                         break;
 
                     final SegmentArchiveResult res = archiveSegment(toArchive);
 
-                    synchronized (this) {
-                        while (locked.containsKey(toArchive) && !stopped)
-                            wait();
-
-                        // Then increase counter to allow rollover on clean working file
-                        changeLastArchivedIndexAndNotifyWaiters(toArchive);
-
-                        notifyAll();
-                    }
+                    segmentAware.markAsMovedToArchive(toArchive);
 
                     if (evt.isRecordable(EVT_WAL_SEGMENT_ARCHIVED)) {
                         evt.record(new WalSegmentArchivedEvent(
-                                cctx.discovery().localNode(),
-                                res.getAbsIdx(),
-                                res.getDstArchiveFile())
+                            cctx.discovery().localNode(),
+                            res.getAbsIdx(),
+                            res.getDstArchiveFile())
                         );
                     }
                 }
             }
-            catch (InterruptedException t) {
+            catch (IgniteInterruptedCheckedException e) {
                 Thread.currentThread().interrupt();
 
-                if (!stopped)
-                    err = t;
+                synchronized (this) {
+                    stopped = true;
+                }
             }
             catch (Throwable t) {
                 err = t;
@@ -1707,63 +1663,39 @@ else if (err != null)
             }
         }
 
-        /**
-         * @param idx Index.
-         */
-        private void changeLastArchivedIndexAndNotifyWaiters(long idx) {
-            lastAbsArchivedIdx = idx;
-
-            if (compressor != null)
-                compressor.onNextSegmentArchived();
-
-            archivedMonitor.setLastArchivedAbsoluteIndex(idx);
-        }
-
         /**
          * Gets the absolute index of the next WAL segment available to write. Blocks till there are available file to
          * write
          *
-         * @param curIdx Current absolute index that we want to increment.
          * @return Next index (curWalSegmIdx+1) when it is ready to be written.
          * @throws StorageException If exception occurred in the archiver thread.
          */
-        private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException {
+        private long nextAbsoluteSegmentIndex() throws StorageException, IgniteInterruptedCheckedException {
             synchronized (this) {
                 if (cleanErr != null)
                     throw cleanErr;
 
-                assert curIdx == curAbsWalIdx;
-
-                curAbsWalIdx++;
-
-                // Notify archiver thread.
-                notifyAll();
+                try {
+                    long nextIdx = segmentAware.nextAbsoluteSegmentIndex();
 
-                while (curAbsWalIdx - lastAbsArchivedIdx > dsCfg.getWalSegments() && cleanErr == null) {
-                    try {
+                    // Wait for formatter so that we do not open an empty file in DEFAULT mode.
+                    while (nextIdx % dsCfg.getWalSegments() > formatted && cleanErr == null)
                         wait();
 
-                        if (cleanErr != null)
-                            throw cleanErr;
-                    }
-                    catch (InterruptedException ignore) {
-                        interrupted.set(true);
-                    }
-                }
-
-                // Wait for formatter so that we do not open an empty file in DEFAULT mode.
-                while (curAbsWalIdx % dsCfg.getWalSegments() > formatted && cleanErr == null)
-                    try {
-                        wait();
+                    if (cleanErr != null)
+                        throw cleanErr;
 
-                        if (cleanErr != null)
-                            throw cleanErr;
-                    }
-                    catch (InterruptedException ignore) {
-                        interrupted.set(true);
-                    }
+                    return nextIdx;
+                }
+                catch (IgniteInterruptedCheckedException e) {
+                    if (cleanErr != null)
+                        throw cleanErr;
 
-                return curAbsWalIdx;
+                    throw e;
+                }
+                catch (InterruptedException e) {
+                    throw new IgniteInterruptedCheckedException(e);
+                }
             }
         }
 
@@ -1773,55 +1705,16 @@ private long nextAbsoluteSegmentIndex(long curIdx) throws StorageException {
          * release segment later, use {@link #releaseWorkSegment} for unlock</li> </ul>
          */
         @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
-        private boolean checkCanReadArchiveOrReserveWorkSegment(long absIdx) {
-            synchronized (this) {
-                if (lastAbsArchivedIdx >= absIdx) {
-                    if (log.isDebugEnabled())
-                        log.debug("Not needed to reserve WAL segment: absIdx=" + absIdx + ";" +
-                            " lastAbsArchivedIdx=" + lastAbsArchivedIdx);
-
-                    return true;
-
-                }
-                Integer cur = locked.get(absIdx);
-
-                cur = cur == null ? 1 : cur + 1;
-
-                locked.put(absIdx, cur);
-
-                if (log.isDebugEnabled())
-                    log.debug("Reserved work segment [absIdx=" + absIdx + ", pins=" + cur + ']');
-
-                return false;
-            }
+        public boolean checkCanReadArchiveOrReserveWorkSegment(long absIdx) {
+            return segmentAware.checkCanReadArchiveOrReserveWorkSegment(absIdx);
         }
 
         /**
          * @param absIdx Segment absolute index.
          */
         @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
-        private void releaseWorkSegment(long absIdx) {
-            synchronized (this) {
-                Integer cur = locked.get(absIdx);
-
-                assert cur != null && cur > 0 : "WAL Segment with Index " + absIdx + " is not locked;" +
-                    " lastAbsArchivedIdx = " + lastAbsArchivedIdx;
-
-                if (cur == 1) {
-                    locked.remove(absIdx);
-
-                    if (log.isDebugEnabled())
-                        log.debug("Fully released work segment (ready to archive) [absIdx=" + absIdx + ']');
-                }
-                else {
-                    locked.put(absIdx, cur - 1);
-
-                    if (log.isDebugEnabled())
-                        log.debug("Partially released work segment [absIdx=" + absIdx + ", pins=" + (cur - 1) + ']');
-                }
-
-                notifyAll();
-            }
+        public void releaseWorkSegment(long absIdx) {
+            segmentAware.releaseWorkSegment(absIdx);
         }
 
         /**
@@ -1911,12 +1804,8 @@ private class FileCompressor extends Thread {
         /** Current thread stopping advice. */
         private volatile boolean stopped;
 
-        /** Last successfully compressed segment. */
-        private volatile long lastCompressedIdx = -1L;
-
         /** All segments prior to this (inclusive) can be compressed. */
-        private volatile long lastAllowedToCompressIdx = -1L;
-
+        private volatile long minUncompressedIdxToKeep = -1L;
         /**
          *
          */
@@ -1940,45 +1829,22 @@ private void init() {
             FileDescriptor[] alreadyCompressed = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_COMPACTED_FILTER));
 
             if (alreadyCompressed.length > 0)
-                lastCompressedIdx = alreadyCompressed[alreadyCompressed.length - 1].idx();
-        }
-
-        /**
-         * @param lastCpStartIdx Segment index to allow compression until (exclusively).
-         */
-        synchronized void allowCompressionUntil(long lastCpStartIdx) {
-            lastAllowedToCompressIdx = lastCpStartIdx - 1;
-
-            notify();
+                segmentAware.lastCompressedIdx(alreadyCompressed[alreadyCompressed.length - 1].idx());
         }
 
         /**
-         * Callback for waking up compressor when new segment is archived.
+         * @param idx Minimum raw segment index that should be preserved from deletion.
          */
-        synchronized void onNextSegmentArchived() {
-            notify();
+        void keepUncompressedIdxFrom(long idx) {
+            minUncompressedIdxToKeep = idx;
         }
 
         /**
          * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation.
          * Waits if there's no segment to archive right now.
          */
-        private long tryReserveNextSegmentOrWait() throws InterruptedException, IgniteCheckedException {
-            long segmentToCompress = lastCompressedIdx + 1;
-
-            synchronized (this) {
-                if (stopped)
-                    return -1;
-
-                while (segmentToCompress > Math.min(lastAllowedToCompressIdx, archivedMonitor.lastArchivedAbsoluteIndex())) {
-                    wait();
-
-                    if (stopped)
-                        return -1;
-                }
-            }
-
-            segmentToCompress = Math.max(segmentToCompress, lastTruncatedArchiveIdx + 1);
+        private long tryReserveNextSegmentOrWait() throws IgniteCheckedException {
+            long segmentToCompress = segmentAware.waitNextSegmentToCompress();
 
             boolean reserved = reserve(new FileWALPointer(segmentToCompress, 0, 0));
 
@@ -2007,7 +1873,7 @@ private void deleteObsoleteRawSegments() {
                 if (segmentReservedOrLocked(desc.idx))
                     return;
 
-                if (desc.idx < lastCompressedIdx && duplicateIndices.contains(desc.idx)) {
+                if (desc.idx < minUncompressedIdxToKeep && duplicateIndices.contains(desc.idx)) {
                     if (!desc.file.delete())
                         U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
                             desc.file.getAbsolutePath() + ", exists: " + desc.file.exists());
@@ -2048,16 +1914,16 @@ private void deleteObsoleteRawSegments() {
                         }
                     }
 
-                    lastCompressedIdx = currReservedSegment;
+                    segmentAware.lastCompressedIdx(currReservedSegment);
+                }
+                catch (IgniteInterruptedCheckedException ignore) {
+                    Thread.currentThread().interrupt();
                 }
                 catch (IgniteCheckedException | IOException e) {
                     U.error(log, "Compression of WAL segment [idx=" + currReservedSegment +
                         "] was skipped due to unexpected error", e);
 
-                    lastCompressedIdx++;
-                }
-                catch (InterruptedException ignore) {
-                    Thread.currentThread().interrupt();
+                    segmentAware.lastCompressedIdx(currReservedSegment);
                 }
                 finally {
                     if (currReservedSegment != -1)
@@ -2076,7 +1942,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
             int segmentSerializerVer;
 
             try (FileIO fileIO = ioFactory.create(raw)) {
-                segmentSerializerVer = readSegmentHeader(fileIO, nextSegment).getSerializerVersion();
+                segmentSerializerVer = readSegmentHeader(new SegmentIO(nextSegment, fileIO), segmentFileInputFactory).getSerializerVersion();
             }
 
             try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(zip)))) {
@@ -2357,115 +2223,25 @@ else if (create)
         return buf;
     }
 
-    /**
-     * WAL file descriptor.
-     */
-    public static class FileDescriptor implements
-        Comparable<FileDescriptor>, AbstractFileDescriptor {
-        /** */
-        protected final File file;
-
-        /** Absolute WAL segment file index */
-        protected final long idx;
-
-        /**
-         * Creates file descriptor. Index is restored from file name
-         *
-         * @param file WAL segment file.
-         */
-        public FileDescriptor(@NotNull File file) {
-            this(file, null);
-        }
-
-        /**
-         * @param file WAL segment file.
-         * @param idx Absolute WAL segment file index. For null value index is restored from file name
-         */
-        public FileDescriptor(@NotNull File file, @Nullable Long idx) {
-            this.file = file;
-
-            String fileName = file.getName();
-
-            assert fileName.contains(WAL_SEGMENT_FILE_EXT);
-
-            this.idx = idx == null ? Long.parseLong(fileName.substring(0, 16)) : idx;
-        }
-
-        /**
-         * @param segment Segment index.
-         * @return Segment file name.
-         */
-        public static String fileName(long segment) {
-            SB b = new SB();
-
-            String segmentStr = Long.toString(segment);
-
-            for (int i = segmentStr.length(); i < 16; i++)
-                b.a('0');
-
-            b.a(segmentStr).a(WAL_SEGMENT_FILE_EXT);
-
-            return b.toString();
-        }
-
-        /** {@inheritDoc} */
-        @Override public int compareTo(@NotNull FileDescriptor o) {
-            return Long.compare(idx, o.idx);
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean equals(Object o) {
-            if (this == o)
-                return true;
-
-            if (!(o instanceof FileDescriptor))
-                return false;
-
-            FileDescriptor that = (FileDescriptor)o;
-
-            return idx == that.idx;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hashCode() {
-            return (int)(idx ^ (idx >>> 32));
-        }
-
-        /**
-         * @return True if segment is ZIP compressed.
-         */
-        @Override public boolean isCompressed() {
-            return file.getName().endsWith(".zip");
-        }
-
-        /** {@inheritDoc} */
-        @Override public File file() {
-            return file;
-        }
-
-        /** {@inheritDoc} */
-        @Override public long idx() {
-            return idx;
-        }
-    }
-
     /**
      *
      */
     private abstract static class FileHandle {
         /** I/O interface for read/write operations with file */
-        FileIO fileIO;
-
-        /** Absolute WAL segment file index (incremental counter) */
-        protected final long idx;
+        SegmentIO fileIO;
 
         /**
-         * @param fileIO I/O interface for read/write operations of FileHandle.
-         * @param idx Absolute WAL segment file index (incremental counter).
+         * @param fileIO I/O interface for read/write operations of FileHandle.         *
          */
-        private FileHandle(FileIO fileIO, long idx) {
+        private FileHandle(SegmentIO fileIO) {
             this.fileIO = fileIO;
-            this.idx = idx;
+        }
+
+        /**
+         * @return Absolute WAL segment file index (incremental counter).
+         */
+        public long getSegmentId(){
+            return fileIO.getSegmentId();
         }
     }
 
@@ -2479,28 +2255,25 @@ public static class ReadFileHandle extends FileHandle implements AbstractWalReco
         /** */
         FileInput in;
 
-        /**
-         * <code>true</code> if this file handle came from work directory. <code>false</code> if this file handle came
-         * from archive directory.
-         */
-        private boolean workDir;
+        /** Holder of actual information of latest manipulation on WAL segments. */
+        private final SegmentAware segmentAware;
 
         /**
          * @param fileIO I/O interface for read/write operations of FileHandle.
-         * @param idx Absolute WAL segment file index (incremental counter).
          * @param ser Entry serializer.
          * @param in File input.
+         * @param aware Segment aware.
          */
         public ReadFileHandle(
-            FileIO fileIO,
-            long idx,
+            SegmentIO fileIO,
             RecordSerializer ser,
-            FileInput in
-        ) {
-            super(fileIO, idx);
+            FileInput in,
+            SegmentAware aware) {
+            super(fileIO);
 
             this.ser = ser;
             this.in = in;
+            segmentAware = aware;
         }
 
         /**
@@ -2509,6 +2282,8 @@ public ReadFileHandle(
         @Override public void close() throws IgniteCheckedException {
             try {
                 fileIO.close();
+
+                in.io().close();
             }
             catch (IOException e) {
                 throw new IgniteCheckedException(e);
@@ -2517,7 +2292,7 @@ public ReadFileHandle(
 
         /** {@inheritDoc} */
         @Override public long idx() {
-            return idx;
+            return getSegmentId();
         }
 
         /** {@inheritDoc} */
@@ -2532,7 +2307,7 @@ public ReadFileHandle(
 
         /** {@inheritDoc} */
         @Override public boolean workDir() {
-            return workDir;
+            return segmentAware != null && segmentAware.lastArchivedAbsoluteIndex() < getSegmentId();
         }
     }
 
@@ -2576,7 +2351,6 @@ private class FileWriteHandle extends FileHandle {
 
         /**
          * @param fileIO I/O file interface to use
-         * @param idx Absolute WAL segment file index for easy access.
          * @param pos Position.
          * @param resume Created on resume logging flag.
          * @param serializer Serializer.
@@ -2584,14 +2358,13 @@ private class FileWriteHandle extends FileHandle {
          * @throws IOException If failed.
          */
         private FileWriteHandle(
-            FileIO fileIO,
-            long idx,
+            SegmentIO fileIO,
             long pos,
             boolean resume,
             RecordSerializer serializer,
             SegmentedRingByteBuffer buf
         ) throws IOException {
-            super(fileIO, idx);
+            super(fileIO);
 
             assert serializer != null;
 
@@ -2615,7 +2388,7 @@ public void writeHeader() {
 
             assert seg != null && seg.position() > 0;
 
-            prepareSerializerVersionBuffer(idx, serializerVersion(), false, seg.buffer());
+            prepareSerializerVersionBuffer(getSegmentId(), serializerVersion(), false, seg.buffer());
 
             seg.release();
         }
@@ -2651,7 +2424,7 @@ public void writeHeader() {
                         if (buf == null)
                             return null; // Can not write to this segment, need to switch to the next one.
 
-                        ptr = new FileWALPointer(idx, pos, rec.size());
+                        ptr = new FileWALPointer(getSegmentId(), pos, rec.size());
 
                         rec.position(ptr);
 
@@ -2694,7 +2467,7 @@ public void writeHeader() {
         private void flushOrWait(FileWALPointer ptr) {
             if (ptr != null) {
                 // If requested obsolete file index, it must be already flushed by close.
-                if (ptr.index() != idx)
+                if (ptr.index() != getSegmentId())
                     return;
             }
 
@@ -2711,7 +2484,7 @@ private void flush(FileWALPointer ptr) {
                 return;
             }
 
-            assert ptr.index() == idx;
+            assert ptr.index() == getSegmentId();
 
             walWriter.flushBuffer(ptr.fileOffset());
         }
@@ -2740,7 +2513,7 @@ private boolean needFsync(FileWALPointer ptr) {
             // If index has changed, it means that the log was rolled over and already sync'ed.
             // If requested position is smaller than last sync'ed, it also means all is good.
             // If position is equal, then our record is the last not synced.
-            return idx == ptr.index() && lastFsyncPos <= ptr.fileOffset();
+            return getSegmentId() == ptr.index() && lastFsyncPos <= ptr.fileOffset();
         }
 
         /**
@@ -2750,7 +2523,7 @@ private FileWALPointer position() {
             lock.lock();
 
             try {
-                return new FileWALPointer(idx, (int)written, 0);
+                return new FileWALPointer(getSegmentId(), (int)written, 0);
             }
             finally {
                 lock.unlock();
@@ -2920,11 +2693,11 @@ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageEx
                         }
                     }
                     catch (IOException e) {
-                        throw new StorageException("Failed to close WAL write handle [idx=" + idx + "]", e);
+                        throw new StorageException("Failed to close WAL write handle [idx=" + getSegmentId() + "]", e);
                     }
 
                     if (log.isDebugEnabled())
-                        log.debug("Closed WAL write handle [idx=" + idx + "]");
+                        log.debug("Closed WAL write handle [idx=" + getSegmentId() + "]");
 
                     return true;
                 }
@@ -2948,7 +2721,7 @@ private void signalNextAvailable() {
             try {
                 assert cctx.kernalContext().invalid() ||
                     written == lastFsyncPos || mode != WALMode.FSYNC :
-                    "fsync [written=" + written + ", lastFsync=" + lastFsyncPos + ", idx=" + idx + ']';
+                    "fsync [written=" + written + ", lastFsync=" + lastFsyncPos + ", idx=" + getSegmentId() + ']';
 
                 fileIO = null;
 
@@ -2998,12 +2771,13 @@ private String safePosition() {
     private static class RecordsIterator extends AbstractWalRecordsIterator {
         /** */
         private static final long serialVersionUID = 0L;
-        /** */
-        private final File walWorkDir;
 
         /** */
         private final File walArchiveDir;
 
+        /** */
+        private final File walWorkDir;
+
         /** See {@link FileWriteAheadLogManager#archiver}. */
         @Nullable private final FileArchiver archiver;
 
@@ -3021,22 +2795,31 @@ private static class RecordsIterator extends AbstractWalRecordsIterator {
         @Nullable
         private FileWALPointer end;
 
+        /** Manager of segment location. */
+        private SegmentRouter segmentRouter;
+
+        /** Holder of actual information of latest manipulation on WAL segments. */
+        private SegmentAware segmentAware;
+
         /**
          * @param cctx Shared context.
-         * @param walWorkDir WAL work dir.
          * @param walArchiveDir WAL archive dir.
+         * @param walWorkDir WAL dir.
          * @param start Optional start pointer.
          * @param end Optional end pointer.
          * @param dsCfg Database configuration.
          * @param serializerFactory Serializer factory.
          * @param archiver File Archiver.
          * @param decompressor Decompressor.
-         *@param log Logger  @throws IgniteCheckedException If failed to initialize WAL segment.
+         * @param log Logger  @throws IgniteCheckedException If failed to initialize WAL segment.
+         * @param segmentAware Segment aware.
+         * @param segmentRouter Segment router.
+         * @param segmentFileInputFactory
          */
         private RecordsIterator(
             GridCacheSharedContext cctx,
-            File walWorkDir,
             File walArchiveDir,
+            File walWorkDir,
             @Nullable FileWALPointer start,
             @Nullable FileWALPointer end,
             DataStorageConfiguration dsCfg,
@@ -3044,20 +2827,28 @@ private RecordsIterator(
             FileIOFactory ioFactory,
             @Nullable FileArchiver archiver,
             FileDecompressor decompressor,
-            IgniteLogger log
+            IgniteLogger log,
+            SegmentAware segmentAware,
+            SegmentRouter segmentRouter,
+            SegmentFileInputFactory segmentFileInputFactory
         ) throws IgniteCheckedException {
             super(log,
                 cctx,
                 serializerFactory,
                 ioFactory,
-                dsCfg.getWalRecordIteratorBufferSize());
-            this.walWorkDir = walWorkDir;
+                dsCfg.getWalRecordIteratorBufferSize(),
+                segmentFileInputFactory
+            );
             this.walArchiveDir = walArchiveDir;
-            this.dsCfg = dsCfg;
+            this.walWorkDir = walWorkDir;
             this.archiver = archiver;
             this.start = start;
             this.end = end;
+            this.dsCfg = dsCfg;
+
             this.decompressor = decompressor;
+            this.segmentRouter = segmentRouter;
+            this.segmentAware = segmentAware;
 
             init();
 
@@ -3096,10 +2887,7 @@ private RecordsIterator(
 
             curRec = null;
 
-            final AbstractReadFileHandle handle = closeCurrentWalSegment();
-
-            if (handle != null && handle.workDir())
-                releaseWorkSegment(curWalSegmIdx);
+            closeCurrentWalSegment();
 
             curWalSegmIdx = Integer.MAX_VALUE;
         }
@@ -3154,42 +2942,24 @@ private void init() throws IgniteCheckedException {
         @Override protected AbstractReadFileHandle advanceSegment(
             @Nullable final AbstractReadFileHandle curWalSegment
         ) throws IgniteCheckedException {
-            if (curWalSegment != null) {
+            if (curWalSegment != null)
                 curWalSegment.close();
 
-                if (curWalSegment.workDir())
-                    releaseWorkSegment(curWalSegment.idx());
-
-            }
-
             // We are past the end marker.
             if (end != null && curWalSegmIdx + 1 > end.index())
                 return null; //stop iteration
 
             curWalSegmIdx++;
 
-            FileDescriptor fd;
-
-            boolean readArchive = canReadArchiveOrReserveWork(curWalSegmIdx);
-
-            if (archiver == null || readArchive) {
-                fd = new FileDescriptor(new File(walArchiveDir,
-                    FileDescriptor.fileName(curWalSegmIdx)));
-            }
-            else {
-                long workIdx = curWalSegmIdx % dsCfg.getWalSegments();
-
-                fd = new FileDescriptor(
-                    new File(walWorkDir, FileDescriptor.fileName(workIdx)),
-                    curWalSegmIdx);
-            }
-
-            if (log.isDebugEnabled())
-                log.debug("Reading next file [absIdx=" + curWalSegmIdx + ", file=" + fd.file.getAbsolutePath() + ']');
+            boolean readArchive = canReadArchiveOrReserveWork(curWalSegmIdx); //lock during creation handle.
 
             ReadFileHandle nextHandle;
-
             try {
+                FileDescriptor fd = segmentRouter.findSegment(curWalSegmIdx);
+
+                if (log.isDebugEnabled())
+                    log.debug("Reading next file [absIdx=" + curWalSegmIdx + ", file=" + fd.file.getAbsolutePath() + ']');
+
                 nextHandle = initReadHandle(fd, start != null && curWalSegmIdx == start.index() ? start : null);
             }
             catch (FileNotFoundException e) {
@@ -3199,12 +2969,8 @@ private void init() throws IgniteCheckedException {
                     nextHandle = null;
             }
 
-            if (nextHandle == null) {
-                if (!readArchive)
-                    releaseWorkSegment(curWalSegmIdx);
-            }
-            else
-                nextHandle.workDir = !readArchive;
+            if (!readArchive)
+                releaseWorkSegment(curWalSegmIdx);
 
             curRec = null;
 
@@ -3274,9 +3040,9 @@ private void releaseWorkSegment(long absIdx) {
         }
 
         /** {@inheritDoc} */
-        @Override protected AbstractReadFileHandle createReadFileHandle(FileIO fileIO, long idx,
+        @Override protected AbstractReadFileHandle createReadFileHandle(SegmentIO fileIO,
             RecordSerializer ser, FileInput in) {
-            return new ReadFileHandle(fileIO, idx, ser, in);
+            return new ReadFileHandle(fileIO, ser, in, segmentAware);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 45f2eaf5b4d0d..9c4a9b30d14e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -89,6 +89,10 @@
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleSegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory;
@@ -237,6 +241,9 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
     /** Factory to provide I/O interfaces for read/write operations with files */
     private volatile FileIOFactory ioFactory;
 
+    /** Factory to provide I/O interfaces for read primitives with files */
+    private final SegmentFileInputFactory segmentFileInputFactory;
+
     /** Updater for {@link #currentHnd}, used for verify there are no concurrent update for current log segment handle */
     private static final AtomicReferenceFieldUpdater<FsyncModeFileWriteAheadLogManager, FileWriteHandle> currentHndUpd =
         AtomicReferenceFieldUpdater.newUpdater(FsyncModeFileWriteAheadLogManager.class, FileWriteHandle.class, "currentHnd");
@@ -319,6 +326,7 @@ public FsyncModeFileWriteAheadLogManager(@NotNull final GridKernalContext ctx) {
         fsyncDelay = dsCfg.getWalFsyncDelayNanos();
         alwaysWriteFullPages = dsCfg.isAlwaysWriteFullPages();
         ioFactory = dsCfg.getFileIOFactory();
+        segmentFileInputFactory = new SimpleSegmentFileInputFactory();
         walAutoArchiveAfterInactivity = dsCfg.getWalAutoArchiveAfterInactivity();
         evt = ctx.event();
 
@@ -590,7 +598,7 @@ public Collection<File> getAndReserveWalFiles(FileWALPointer low, FileWALPointer
         List<File> res = new ArrayList<>();
 
         for (long i = low.index(); i < high.index(); i++) {
-            String segmentName = FileWriteAheadLogManager.FileDescriptor.fileName(i);
+            String segmentName = FileDescriptor.fileName(i);
 
             File file = new File(walArchiveDir, segmentName);
             File fileZip = new File(walArchiveDir, segmentName + FilePageStoreManager.ZIP_SUFFIX);
@@ -741,7 +749,8 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
             ioFactory,
             archiver,
             decompressor,
-            log
+            log,
+            segmentFileInputFactory
         );
     }
 
@@ -803,7 +812,7 @@ private boolean hasIndex(long absIdx) {
 
         FileWriteHandle cur = currentHnd;
 
-        return cur != null && cur.idx >= absIdx;
+        return cur != null && cur.getSegmentId() >= absIdx;
     }
 
     /** {@inheritDoc} */
@@ -851,9 +860,9 @@ private boolean hasIndex(long absIdx) {
     }
 
     /** {@inheritDoc} */
-    @Override public void allowCompressionUntil(WALPointer ptr) {
+    @Override public void notchLastCheckpointPtr(WALPointer ptr) {
         if (compressor != null)
-            compressor.allowCompressionUntil(((FileWALPointer)ptr).index());
+            compressor.keepUncompressedIdxFrom(((FileWALPointer)ptr).index());
     }
 
     /** {@inheritDoc} */
@@ -1052,7 +1061,7 @@ private FileWriteHandle rollOver(FileWriteHandle cur) throws IgniteCheckedExcept
             if (metrics.metricsEnabled())
                 metrics.onWallRollOver();
 
-            FileWriteHandle next = initNextWriteHandle(cur.idx);
+            FileWriteHandle next = initNextWriteHandle(cur.getSegmentId());
 
             boolean swapped = currentHndUpd.compareAndSet(this, hnd, next);
 
@@ -1086,7 +1095,7 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
         int len = lastReadPtr == null ? 0 : lastReadPtr.length();
 
         try {
-            FileIO fileIO = ioFactory.create(curFile);
+            SegmentIO fileIO = new SegmentIO(absIdx, ioFactory.create(curFile));
 
             try {
                 int serVer = serializerVersion;
@@ -1094,7 +1103,7 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
                 // If we have existing segment, try to read version from it.
                 if (lastReadPtr != null) {
                     try {
-                        serVer = readSegmentHeader(fileIO, absIdx).getSerializerVersion();
+                        serVer = readSegmentHeader(fileIO, segmentFileInputFactory).getSerializerVersion();
                     }
                     catch (SegmentEofException | EOFException ignore) {
                         serVer = serializerVersion;
@@ -1109,7 +1118,6 @@ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws St
 
                 FileWriteHandle hnd = new FileWriteHandle(
                     fileIO,
-                    absIdx,
                     offset + len,
                     maxWalSegmentSize,
                     ser);
@@ -1159,11 +1167,10 @@ private FileWriteHandle initNextWriteHandle(long curIdx) throws IgniteCheckedExc
             if (log.isDebugEnabled())
                 log.debug("Switching to a new WAL segment: " + nextFile.getAbsolutePath());
 
-            FileIO fileIO = ioFactory.create(nextFile);
+            SegmentIO fileIO = new SegmentIO(curIdx + 1, ioFactory.create(nextFile));
 
             FileWriteHandle hnd = new FileWriteHandle(
                 fileIO,
-                curIdx + 1,
                 0,
                 maxWalSegmentSize,
                 serializer);
@@ -1742,7 +1749,7 @@ private class FileCompressor extends Thread {
         private volatile long lastCompressedIdx = -1L;
 
         /** All segments prior to this (inclusive) can be compressed. */
-        private volatile long lastAllowedToCompressIdx = -1L;
+        private volatile long minUncompressedIdxToKeep = -1L;
 
         /**
          *
@@ -1767,14 +1774,14 @@ private void init() {
             FileDescriptor[] alreadyCompressed = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_COMPACTED_FILTER));
 
             if (alreadyCompressed.length > 0)
-                lastCompressedIdx = alreadyCompressed[alreadyCompressed.length - 1].getIdx();
+                lastCompressedIdx = alreadyCompressed[alreadyCompressed.length - 1].idx();
         }
 
         /**
-         * @param lastCpStartIdx Segment index to allow compression until (exclusively).
+         * @param idx Minimum raw segment index that should be preserved from deletion.
          */
-        synchronized void allowCompressionUntil(long lastCpStartIdx) {
-            lastAllowedToCompressIdx = lastCpStartIdx - 1;
+        synchronized void keepUncompressedIdxFrom(long idx) {
+            minUncompressedIdxToKeep = idx;
 
             notify();
         }
@@ -1797,7 +1804,7 @@ private long tryReserveNextSegmentOrWait() throws InterruptedException, IgniteCh
                 if (stopped)
                     return -1;
 
-                while (segmentToCompress > Math.min(lastAllowedToCompressIdx, archiver.lastArchivedAbsoluteIndex())) {
+                while (segmentToCompress > archiver.lastArchivedAbsoluteIndex()) {
                     wait();
 
                     if (stopped)
@@ -1836,7 +1843,7 @@ private void deleteObsoleteRawSegments() {
                 if (archiver0 != null && archiver0.reserved(desc.idx))
                     return;
 
-                if (desc.idx < lastCompressedIdx && duplicateIndices.contains(desc.idx)) {
+                if (desc.idx < minUncompressedIdxToKeep && duplicateIndices.contains(desc.idx)) {
                     if (!desc.file.delete())
                         U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
                             desc.file.getAbsolutePath() + ", exists: " + desc.file.exists());
@@ -1912,7 +1919,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
             int segmentSerializerVer;
 
             try (FileIO fileIO = ioFactory.create(raw)) {
-                segmentSerializerVer = readSegmentHeader(fileIO, nextSegment).getSerializerVersion();
+                segmentSerializerVer = readSegmentHeader(new SegmentIO(nextSegment, fileIO), segmentFileInputFactory).getSerializerVersion();
             }
 
             try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(zip)))) {
@@ -2185,143 +2192,25 @@ public static long writeSerializerVersion(FileIO io, long idx, int version, WALM
         return buf;
     }
 
-    /**
-     * WAL file descriptor.
-     */
-    public static class FileDescriptor implements Comparable<FileDescriptor>, AbstractWalRecordsIterator.AbstractFileDescriptor {
-        /** */
-        protected final File file;
-
-        /** Absolute WAL segment file index */
-        protected final long idx;
-
-        /**
-         * Creates file descriptor. Index is restored from file name
-         *
-         * @param file WAL segment file.
-         */
-        public FileDescriptor(@NotNull File file) {
-            this(file, null);
-        }
-
-        /**
-         * @param file WAL segment file.
-         * @param idx Absolute WAL segment file index. For null value index is restored from file name
-         */
-        public FileDescriptor(@NotNull File file, @Nullable Long idx) {
-            this.file = file;
-
-            String fileName = file.getName();
-
-            assert fileName.contains(WAL_SEGMENT_FILE_EXT);
-
-            this.idx = idx == null ? Long.parseLong(fileName.substring(0, 16)) : idx;
-        }
-
-        /**
-         * @param segment Segment index.
-         * @return Segment file name.
-         */
-        public static String fileName(long segment) {
-            SB b = new SB();
-
-            String segmentStr = Long.toString(segment);
-
-            for (int i = segmentStr.length(); i < 16; i++)
-                b.a('0');
-
-            b.a(segmentStr).a(WAL_SEGMENT_FILE_EXT);
-
-            return b.toString();
-        }
-
-        /**
-         * @param segment Segment number as integer.
-         * @return Segment number as aligned string.
-         */
-        private static String segmentNumber(long segment) {
-            SB b = new SB();
-
-            String segmentStr = Long.toString(segment);
-
-            for (int i = segmentStr.length(); i < 16; i++)
-                b.a('0');
-
-            b.a(segmentStr);
-
-            return b.toString();
-        }
-
-        /** {@inheritDoc} */
-        @Override public int compareTo(FileDescriptor o) {
-            return Long.compare(idx, o.idx);
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean equals(Object o) {
-            if (this == o)
-                return true;
-
-            if (!(o instanceof FileDescriptor))
-                return false;
-
-            FileDescriptor that = (FileDescriptor)o;
-
-            return idx == that.idx;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int hashCode() {
-            return (int)(idx ^ (idx >>> 32));
-        }
-
-        /**
-         * @return Absolute WAL segment file index
-         */
-        public long getIdx() {
-            return idx;
-        }
-
-        /**
-         * @return absolute pathname string of this file descriptor pathname.
-         */
-        public String getAbsolutePath() {
-            return file.getAbsolutePath();
-        }
-
-        /** {@inheritDoc} */
-        @Override public boolean isCompressed() {
-            return file.getName().endsWith(".zip");
-        }
-
-        /** {@inheritDoc} */
-        @Override public File file() {
-            return file;
-        }
-
-        /** {@inheritDoc} */
-        @Override public long idx() {
-            return idx;
-        }
-    }
-
     /**
      *
      */
     private abstract static class FileHandle {
         /** I/O interface for read/write operations with file */
-        protected FileIO fileIO;
-
-        /** Absolute WAL segment file index (incremental counter) */
-        protected final long idx;
+        protected SegmentIO fileIO;
 
         /**
          * @param fileIO I/O interface for read/write operations of FileHandle.
-         * @param idx Absolute WAL segment file index (incremental counter).
          */
-        private FileHandle(FileIO fileIO, long idx) {
+        private FileHandle(SegmentIO fileIO) {
             this.fileIO = fileIO;
-            this.idx = idx;
+        }
+
+        /**
+         * @return Current segment id.
+         */
+        public long getSegmentId(){
+            return fileIO.getSegmentId();
         }
     }
 
@@ -2343,17 +2232,15 @@ public static class ReadFileHandle extends FileHandle implements AbstractWalReco
 
         /**
          * @param fileIO I/O interface for read/write operations of FileHandle.
-         * @param idx Absolute WAL segment file index (incremental counter).
          * @param ser Entry serializer.
          * @param in File input.
          */
         ReadFileHandle(
-                FileIO fileIO,
-                long idx,
-                RecordSerializer ser,
-                FileInput in
+            SegmentIO fileIO,
+            RecordSerializer ser,
+            FileInput in
         ) {
-            super(fileIO, idx);
+            super(fileIO);
 
             this.ser = ser;
             this.in = in;
@@ -2373,7 +2260,7 @@ public static class ReadFileHandle extends FileHandle implements AbstractWalReco
 
         /** {@inheritDoc} */
         @Override public long idx() {
-            return idx;
+            return getSegmentId();
         }
 
         /** {@inheritDoc} */
@@ -2440,20 +2327,18 @@ private class FileWriteHandle extends FileHandle {
 
         /**
          * @param fileIO I/O file interface to use
-         * @param idx Absolute WAL segment file index for easy access.
          * @param pos Position.
          * @param maxSegmentSize Max segment size.
          * @param serializer Serializer.
          * @throws IOException If failed.
          */
         private FileWriteHandle(
-            FileIO fileIO,
-            long idx,
+            SegmentIO fileIO,
             long pos,
             long maxSegmentSize,
             RecordSerializer serializer
         ) throws IOException {
-            super(fileIO, idx);
+            super(fileIO);
 
             assert serializer != null;
 
@@ -2462,7 +2347,7 @@ private FileWriteHandle(
             this.maxSegmentSize = maxSegmentSize;
             this.serializer = serializer;
 
-            head.set(new FakeRecord(new FileWALPointer(idx, (int)pos, 0), false));
+            head.set(new FakeRecord(new FileWALPointer(fileIO.getSegmentId(), (int)pos, 0), false));
             written = pos;
             lastFsyncPos = pos;
         }
@@ -2478,15 +2363,15 @@ private void writeSerializerVersion() throws IOException {
                 assert fileIO.position() == 0 : "Serializer version can be written only at the begin of file " +
                     fileIO.position();
 
-                long updatedPosition = FsyncModeFileWriteAheadLogManager.writeSerializerVersion(fileIO, idx,
+                long updatedPosition = FsyncModeFileWriteAheadLogManager.writeSerializerVersion(fileIO, getSegmentId(),
                     serializer.version(), mode);
 
                 written = updatedPosition;
                 lastFsyncPos = updatedPosition;
-                head.set(new FakeRecord(new FileWALPointer(idx, (int)updatedPosition, 0), false));
+                head.set(new FakeRecord(new FileWALPointer(getSegmentId(), (int)updatedPosition, 0), false));
             }
             catch (IOException e) {
-                throw new IOException("Unable to write serializer version for segment " + idx, e);
+                throw new IOException("Unable to write serializer version for segment " + getSegmentId(), e);
             }
         }
 
@@ -2542,7 +2427,7 @@ private boolean stopped(WALRecord record) {
                 rec.previous(h);
 
                 FileWALPointer ptr = new FileWALPointer(
-                    idx,
+                    getSegmentId(),
                     (int)nextPos,
                     rec.size());
 
@@ -2572,7 +2457,7 @@ private void flushOrWait(FileWALPointer ptr, boolean stop) throws StorageExcepti
 
             if (ptr != null) {
                 // If requested obsolete file index, it must be already flushed by close.
-                if (ptr.index() != idx)
+                if (ptr.index() != getSegmentId())
                     return;
 
                 expWritten = ptr.fileOffset();
@@ -2631,7 +2516,7 @@ private boolean flush(FileWALPointer ptr, boolean stop) throws StorageException
                 }
             }
 
-            assert ptr.index() == idx;
+            assert ptr.index() == getSegmentId();
 
             for (; ; ) {
                 WALRecord h = head.get();
@@ -2668,7 +2553,7 @@ private boolean flush(WALRecord expHead, boolean stop) throws StorageException {
             // Fail-fast before CAS.
             checkNode();
 
-            if (!head.compareAndSet(expHead, new FakeRecord(new FileWALPointer(idx, (int)nextPosition(expHead), 0), stop)))
+            if (!head.compareAndSet(expHead, new FakeRecord(new FileWALPointer(getSegmentId(), (int)nextPosition(expHead), 0), stop)))
                 return false;
 
             if (expHead.chainSize() == 0)
@@ -2766,7 +2651,7 @@ private boolean needFsync(FileWALPointer ptr) {
             // If index has changed, it means that the log was rolled over and already sync'ed.
             // If requested position is smaller than last sync'ed, it also means all is good.
             // If position is equal, then our record is the last not synced.
-            return idx == ptr.index() && lastFsyncPos <= ptr.fileOffset();
+            return getSegmentId() == ptr.index() && lastFsyncPos <= ptr.fileOffset();
         }
 
         /**
@@ -2776,7 +2661,7 @@ private FileWALPointer position() {
             lock.lock();
 
             try {
-                return new FileWALPointer(idx, (int)written, 0);
+                return new FileWALPointer(getSegmentId(), (int)written, 0);
             }
             finally {
                 lock.unlock();
@@ -2865,7 +2750,7 @@ private boolean close(boolean rollOver) throws StorageException {
                             if (rollOver && written < (maxSegmentSize - switchSegmentRecSize)) {
                                 final ByteBuffer buf = ByteBuffer.allocate(switchSegmentRecSize);
 
-                                segmentRecord.position(new FileWALPointer(idx, (int)written, switchSegmentRecSize));
+                                segmentRecord.position(new FileWALPointer(getSegmentId(), (int)written, switchSegmentRecSize));
                                 backwardSerializer.writeRecord(segmentRecord, buf);
 
                                 buf.rewind();
@@ -2888,11 +2773,11 @@ private boolean close(boolean rollOver) throws StorageException {
                         }
                     }
                     catch (IOException e) {
-                        throw new StorageException("Failed to close WAL write handle [idx=" + idx + "]", e);
+                        throw new StorageException("Failed to close WAL write handle [idx=" + getSegmentId() + "]", e);
                     }
 
                     if (log.isDebugEnabled())
-                        log.debug("Closed WAL write handle [idx=" + idx + "]");
+                        log.debug("Closed WAL write handle [idx=" + getSegmentId() + "]");
 
                     return true;
                 }
@@ -2927,7 +2812,7 @@ private void signalNextAvailable() {
                         fileIO.close();
                     }
                     catch (IOException e) {
-                        U.error(log, "Failed to close WAL file [idx=" + idx + ", fileIO=" + fileIO + "]", e);
+                        U.error(log, "Failed to close WAL file [idx=" + getSegmentId() + ", fileIO=" + fileIO + "]", e);
                     }
                 }
 
@@ -3144,7 +3029,8 @@ private class RecordsIterator extends AbstractWalRecordsIterator {
          * @param serializerFactory Serializer factory.
          * @param archiver Archiver.
          * @param decompressor Decompressor.
-         *@param log Logger  @throws IgniteCheckedException If failed to initialize WAL segment.
+         * @param log Logger  @throws IgniteCheckedException If failed to initialize WAL segment.
+         * @param segmentFileInputFactory Segment file input factory.
          */
         private RecordsIterator(
             GridCacheSharedContext cctx,
@@ -3157,13 +3043,17 @@ private RecordsIterator(
             FileIOFactory ioFactory,
             FileArchiver archiver,
             FileDecompressor decompressor,
-            IgniteLogger log
+            IgniteLogger log,
+            SegmentFileInputFactory segmentFileInputFactory
         ) throws IgniteCheckedException {
-            super(log,
+            super(
+                log,
                 cctx,
                 serializerFactory,
                 ioFactory,
-                psCfg.getWalRecordIteratorBufferSize());
+                psCfg.getWalRecordIteratorBufferSize(),
+                segmentFileInputFactory
+            );
             this.walWorkDir = walWorkDir;
             this.walArchiveDir = walArchiveDir;
             this.psCfg = psCfg;
@@ -3385,9 +3275,9 @@ private void releaseWorkSegment(long absIdx) {
         }
 
         /** {@inheritDoc} */
-        @Override protected AbstractReadFileHandle createReadFileHandle(FileIO fileIO, long idx,
+        @Override protected AbstractReadFileHandle createReadFileHandle(SegmentIO fileIO,
             RecordSerializer ser, FileInput in) {
-            return new ReadFileHandle(fileIO, idx, ser, in);
+            return new ReadFileHandle(fileIO, ser, in);
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentArchivedMonitor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentArchivedMonitor.java
deleted file mode 100644
index 81ecd410fcfa5..0000000000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentArchivedMonitor.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal.processors.cache.persistence.wal;
-
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-
-/**
- * Next WAL segment archived monitor. Manages last archived index, allows to emulate archivation in no-archiver mode.
- * Monitor which is notified each time WAL segment is archived.
- */
-class SegmentArchivedMonitor {
-    /**
-     * Last archived file absolute index, 0-based. Write is quarded by {@code this}. Negative value indicates there are
-     * no segments archived.
-     */
-    private volatile long lastAbsArchivedIdx = -1;
-
-    /**
-     * @return Last archived segment absolute index.
-     */
-    long lastArchivedAbsoluteIndex() {
-        return lastAbsArchivedIdx;
-    }
-
-    /**
-     * @param lastAbsArchivedIdx new value of last archived segment index
-     */
-    synchronized void setLastArchivedAbsoluteIndex(long lastAbsArchivedIdx) {
-        this.lastAbsArchivedIdx = lastAbsArchivedIdx;
-
-        notifyAll();
-    }
-
-    /**
-     * Method will wait activation of particular WAL segment index.
-     *
-     * @param awaitIdx absolute index  {@link #lastArchivedAbsoluteIndex()} to become true.
-     * @throws IgniteInterruptedCheckedException if interrupted.
-     */
-    synchronized void awaitSegmentArchived(long awaitIdx) throws IgniteInterruptedCheckedException {
-        while (lastArchivedAbsoluteIndex() < awaitIdx) {
-            try {
-                wait(2000);
-            }
-            catch (InterruptedException e) {
-                throw new IgniteInterruptedCheckedException(e);
-            }
-        }
-    }
-}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentRouter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentRouter.java
new file mode 100644
index 0000000000000..2d47e9d491272
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentRouter.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware;
+
+import static org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor.fileName;
+
+/**
+ * Class for manage of segment file location.
+ */
+public class SegmentRouter {
+    /** */
+    public static final String ZIP_SUFFIX = ".zip";
+    /** */
+    private File walWorkDir;
+
+    /** WAL archive directory (including consistent ID as subfolder) */
+    private File walArchiveDir;
+
+    /** Holder of actual information of latest manipulation on WAL segments. */
+    private SegmentAware segmentAware;
+
+    /** */
+    private DataStorageConfiguration dsCfg;
+
+    /**
+     * @param walWorkDir WAL work directory.
+     * @param walArchiveDir WAL archive directory.
+     * @param segmentAware Holder of actual information of latest manipulation on WAL segments.
+     * @param dsCfg Data storage configuration.
+     */
+    public SegmentRouter(
+        File walWorkDir,
+        File walArchiveDir,
+        SegmentAware segmentAware,
+        DataStorageConfiguration dsCfg) {
+        this.walWorkDir = walWorkDir;
+        this.walArchiveDir = walArchiveDir;
+        this.segmentAware = segmentAware;
+        this.dsCfg = dsCfg;
+    }
+
+    /**
+     * Find file which represent given segment.
+     *
+     * @param segmentId Segment for searching.
+     * @return Actual file description.
+     * @throws FileNotFoundException If file does not exist.
+     */
+    public FileDescriptor findSegment(long segmentId) throws FileNotFoundException {
+        FileDescriptor fd;
+
+        if (segmentAware.lastArchivedAbsoluteIndex() >= segmentId)
+            fd = new FileDescriptor(new File(walArchiveDir, fileName(segmentId)));
+        else
+            fd = new FileDescriptor(new File(walWorkDir, fileName(segmentId % dsCfg.getWalSegments())), segmentId);
+
+        if (!fd.file().exists()) {
+            FileDescriptor zipFile = new FileDescriptor(new File(walArchiveDir, fileName(fd.idx()) + ZIP_SUFFIX));
+
+            if (!zipFile.file().exists()) {
+                throw new FileNotFoundException("Both compressed and raw segment files are missing in archive " +
+                    "[segmentIdx=" + fd.idx() + "]");
+            }
+
+            fd = zipFile;
+        }
+
+        return fd;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
index f688bb4b5f783..8d1445c4204c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SingleSegmentLogicalRecordsIterator.java
@@ -25,8 +25,10 @@
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleSegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.RecordTypes;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory;
@@ -71,7 +73,7 @@ public class SingleSegmentLogicalRecordsIterator extends AbstractWalRecordsItera
         File archiveDir,
         CIX1<WALRecord> advanceC
     ) throws IgniteCheckedException {
-        super(log, sharedCtx, initLogicalRecordsSerializerFactory(sharedCtx), ioFactory, bufSize);
+        super(log, sharedCtx, initLogicalRecordsSerializerFactory(sharedCtx), ioFactory, bufSize, new SimpleSegmentFileInputFactory());
 
         curWalSegmIdx = archivedSegIdx;
         this.archiveDir = archiveDir;
@@ -100,8 +102,8 @@ private static RecordSerializerFactory initLogicalRecordsSerializerFactory(GridC
         else {
             segmentInitialized = true;
 
-            FileWriteAheadLogManager.FileDescriptor fd = new FileWriteAheadLogManager.FileDescriptor(
-                new File(archiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(curWalSegmIdx)));
+            FileDescriptor fd = new FileDescriptor(
+                new File(archiveDir, FileDescriptor.fileName(curWalSegmIdx)));
 
             try {
                 return initReadHandle(fd, null);
@@ -121,9 +123,10 @@ private static RecordSerializerFactory initLogicalRecordsSerializerFactory(GridC
     }
 
     /** {@inheritDoc} */
-    @Override protected AbstractReadFileHandle createReadFileHandle(FileIO fileIO, long idx,
+    @Override protected AbstractReadFileHandle createReadFileHandle(
+        SegmentIO fileIO,
         RecordSerializer ser, FileInput in) {
-        return new FileWriteAheadLogManager.ReadFileHandle(fileIO, idx, ser, in);
+        return new FileWriteAheadLogManager.ReadFileHandle(fileIO, ser, in, null);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
new file mode 100644
index 0000000000000..1ed607e9b7dd8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
+
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+
+/**
+ * Manages last archived index, allows to emulate archivation in no-archiver mode. Monitor which is notified each time
+ * WAL segment is archived.
+ *
+ * Class for inner usage.
+ */
+class SegmentArchivedStorage extends SegmentObservable {
+    /** Segment lock storage: Protects WAL work segments from moving. */
+    private final SegmentLockStorage segmentLockStorage;
+    /** Flag of interrupt waiting on this object. */
+    private volatile boolean interrupted;
+    /**
+     * Last archived file absolute index, 0-based. Write is quarded by {@code this}. Negative value indicates there are
+     * no segments archived.
+     */
+    private volatile long lastAbsArchivedIdx = -1;
+
+    /**
+     * @param segmentLockStorage Protects WAL work segments from moving.
+     */
+    private SegmentArchivedStorage(SegmentLockStorage segmentLockStorage) {
+        this.segmentLockStorage = segmentLockStorage;
+    }
+
+    /**
+     * @param segmentLockStorage Protects WAL work segments from moving.
+     */
+    static SegmentArchivedStorage buildArchivedStorage(SegmentLockStorage segmentLockStorage) {
+        SegmentArchivedStorage archivedStorage = new SegmentArchivedStorage(segmentLockStorage);
+
+        segmentLockStorage.addObserver(archivedStorage::onSegmentUnlocked);
+
+        return archivedStorage;
+    }
+
+    /**
+     * @return Last archived segment absolute index.
+     */
+    long lastArchivedAbsoluteIndex() {
+        return lastAbsArchivedIdx;
+    }
+
+    /**
+     * @param lastAbsArchivedIdx New value of last archived segment index.
+     */
+    synchronized void setLastArchivedAbsoluteIndex(long lastAbsArchivedIdx) {
+        this.lastAbsArchivedIdx = lastAbsArchivedIdx;
+
+        notifyAll();
+
+        notifyObservers(lastAbsArchivedIdx);
+    }
+
+    /**
+     * Method will wait activation of particular WAL segment index.
+     *
+     * @param awaitIdx absolute index  {@link #lastArchivedAbsoluteIndex()} to become true.
+     * @throws IgniteInterruptedCheckedException if interrupted.
+     */
+    synchronized void awaitSegmentArchived(long awaitIdx) throws IgniteInterruptedCheckedException {
+        while (lastArchivedAbsoluteIndex() < awaitIdx && !interrupted) {
+            try {
+                wait(2000);
+            }
+            catch (InterruptedException e) {
+                throw new IgniteInterruptedCheckedException(e);
+            }
+        }
+
+        checkInterrupted();
+    }
+
+    /**
+     * Mark segment as moved to archive under lock.
+     *
+     * @param toArchive Segment which was should be moved to archive.
+     * @throws IgniteInterruptedCheckedException if interrupted during waiting.
+     */
+    synchronized void markAsMovedToArchive(long toArchive) throws IgniteInterruptedCheckedException {
+        try {
+            while (segmentLockStorage.locked(toArchive) && !interrupted)
+                wait();
+        }
+        catch (InterruptedException e) {
+            throw new IgniteInterruptedCheckedException(e);
+        }
+
+        //Ignore interrupted flag and force set new value. - legacy logic.
+        //checkInterrupted();
+
+        setLastArchivedAbsoluteIndex(toArchive);
+    }
+
+    /**
+     * Interrupt waiting on this object.
+     */
+    synchronized void interrupt() {
+        interrupted = true;
+
+        notifyAll();
+    }
+
+    /**
+     * Check for interrupt flag was set.
+     */
+    private void checkInterrupted() throws IgniteInterruptedCheckedException {
+        if (interrupted)
+            throw new IgniteInterruptedCheckedException("Interrupt waiting of change archived idx");
+    }
+
+    /**
+     * Callback for waking up waiters of this object when unlocked happened.
+     */
+    private synchronized void onSegmentUnlocked(long segmentId) {
+        notifyAll();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
new file mode 100644
index 0000000000000..3379b74cf5780
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
+
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+
+import static org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentArchivedStorage.buildArchivedStorage;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentCompressStorage.buildCompressStorage;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentCurrentStateStorage.buildCurrentStateStorage;
+
+/**
+ * Holder of actual information of latest manipulation on WAL segments.
+ */
+public class SegmentAware {
+    /** Latest truncated segment. */
+    private volatile long lastTruncatedArchiveIdx = -1L;
+    /** Segment reservations storage: Protects WAL segments from deletion during WAL log cleanup. */
+    private final SegmentReservationStorage reservationStorage = new SegmentReservationStorage();
+    /** Lock on segment protects from archiving segment. */
+    private final SegmentLockStorage segmentLockStorage = new SegmentLockStorage();
+    /** Manages last archived index, emulates archivation in no-archiver mode. */
+    private final SegmentArchivedStorage segmentArchivedStorage = buildArchivedStorage(segmentLockStorage);
+    /** Storage of actual information about current index of compressed segments. */
+    private final SegmentCompressStorage segmentCompressStorage = buildCompressStorage(segmentArchivedStorage);
+    /** Storage of absolute current segment index. */
+    private final SegmentCurrentStateStorage segmentCurrStateStorage;
+
+    /**
+     * @param walSegmentsCnt Total WAL segments count.
+     */
+    public SegmentAware(int walSegmentsCnt) {
+        segmentCurrStateStorage = buildCurrentStateStorage(walSegmentsCnt, segmentArchivedStorage);
+    }
+
+    /**
+     * Waiting until current WAL index will be greater or equal than given one.
+     *
+     * @param absSegIdx Target WAL index.
+     */
+    public void awaitSegment(long absSegIdx) throws IgniteInterruptedCheckedException {
+        segmentCurrStateStorage.awaitSegment(absSegIdx);
+    }
+
+    /**
+     * Calculate next segment index or wait if needed.
+     *
+     * @return Next absolute segment index.
+     */
+    public long nextAbsoluteSegmentIndex() throws IgniteInterruptedCheckedException {
+        return segmentCurrStateStorage.nextAbsoluteSegmentIndex();
+    }
+
+    /**
+     * @return Current WAL index.
+     */
+    public long curAbsWalIdx() {
+        return segmentCurrStateStorage.curAbsWalIdx();
+    }
+
+    /**
+     * Waiting until archivation of next segment will be allowed.
+     */
+    public long waitNextSegmentForArchivation() throws IgniteInterruptedCheckedException {
+        return segmentCurrStateStorage.waitNextSegmentForArchivation();
+    }
+
+    /**
+     * Mark segment as moved to archive under lock.
+     *
+     * @param toArchive Segment which was should be moved to archive.
+     * @throws IgniteInterruptedCheckedException if interrupted during waiting.
+     */
+    public void markAsMovedToArchive(long toArchive) throws IgniteInterruptedCheckedException {
+        segmentArchivedStorage.markAsMovedToArchive(toArchive);
+    }
+
+    /**
+     * Method will wait activation of particular WAL segment index.
+     *
+     * @param awaitIdx absolute index  {@link #lastArchivedAbsoluteIndex()} to become true.
+     * @throws IgniteInterruptedCheckedException if interrupted.
+     */
+    public void awaitSegmentArchived(long awaitIdx) throws IgniteInterruptedCheckedException {
+        segmentArchivedStorage.awaitSegmentArchived(awaitIdx);
+    }
+
+    /**
+     * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation. Waits if
+     * there's no segment to archive right now.
+     */
+    public long waitNextSegmentToCompress() throws IgniteInterruptedCheckedException {
+        return Math.max(segmentCompressStorage.nextSegmentToCompressOrWait(), lastTruncatedArchiveIdx + 1);
+    }
+
+    /**
+     * Force set last compressed segment.
+     *
+     * @param lastCompressedIdx Segment which was last compressed.
+     */
+    public void lastCompressedIdx(long lastCompressedIdx) {
+        segmentCompressStorage.lastCompressedIdx(lastCompressedIdx);
+    }
+
+    /**
+     * @return Last compressed segment.
+     */
+    public long lastCompressedIdx() {
+        return segmentCompressStorage.lastCompressedIdx();
+    }
+
+    /**
+     * Update current WAL index.
+     *
+     * @param curAbsWalIdx New current WAL index.
+     */
+    public void curAbsWalIdx(long curAbsWalIdx) {
+        segmentCurrStateStorage.curAbsWalIdx(curAbsWalIdx);
+    }
+
+    /**
+     * @param lastTruncatedArchiveIdx Last truncated segment;
+     */
+    public void lastTruncatedArchiveIdx(long lastTruncatedArchiveIdx) {
+        this.lastTruncatedArchiveIdx = lastTruncatedArchiveIdx;
+    }
+
+    /**
+     * @return Last truncated segment.
+     */
+    public long lastTruncatedArchiveIdx() {
+        return lastTruncatedArchiveIdx;
+    }
+
+    /**
+     * @param lastAbsArchivedIdx New value of last archived segment index.
+     */
+    public void setLastArchivedAbsoluteIndex(long lastAbsArchivedIdx) {
+        segmentArchivedStorage.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx);
+    }
+
+    /**
+     * @return Last archived segment absolute index.
+     */
+    public long lastArchivedAbsoluteIndex() {
+        return segmentArchivedStorage.lastArchivedAbsoluteIndex();
+    }
+
+    /**
+     * @param absIdx Index for reservation.
+     */
+    public void reserve(long absIdx) {
+        reservationStorage.reserve(absIdx);
+    }
+
+    /**
+     * Checks if segment is currently reserved (protected from deletion during WAL cleanup).
+     *
+     * @param absIdx Index for check reservation.
+     * @return {@code True} if index is reserved.
+     */
+    public boolean reserved(long absIdx) {
+        return reservationStorage.reserved(absIdx);
+    }
+
+    /**
+     * @param absIdx Reserved index.
+     */
+    public void release(long absIdx) {
+        reservationStorage.release(absIdx);
+    }
+
+    /**
+     * Check if WAL segment locked (protected from move to archive)
+     *
+     * @param absIdx Index for check reservation.
+     * @return {@code True} if index is locked.
+     */
+    public boolean locked(long absIdx) {
+        return segmentLockStorage.locked(absIdx);
+    }
+
+    /**
+     * @param absIdx Segment absolute index.
+     * @return <ul><li>{@code True} if can read, no lock is held, </li><li>{@code false} if work segment, need release
+     * segment later, use {@link #releaseWorkSegment} for unlock</li> </ul>
+     */
+    public boolean checkCanReadArchiveOrReserveWorkSegment(long absIdx) {
+        return lastArchivedAbsoluteIndex() >= absIdx || segmentLockStorage.lockWorkSegment(absIdx);
+    }
+
+    /**
+     * @param absIdx Segment absolute index.
+     */
+    public void releaseWorkSegment(long absIdx) {
+        segmentLockStorage.releaseWorkSegment(absIdx);
+    }
+
+    /**
+     * Interrupt waiting on related objects.
+     */
+    public void interrupt() {
+        segmentArchivedStorage.interrupt();
+
+        segmentCompressStorage.interrupt();
+
+        segmentCurrStateStorage.interrupt();
+    }
+
+    /**
+     * Interrupt waiting on related objects.
+     */
+    public void forceInterrupt() {
+        segmentArchivedStorage.interrupt();
+
+        segmentCompressStorage.interrupt();
+
+        segmentCurrStateStorage.forceInterrupt();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
new file mode 100644
index 0000000000000..30c9a2d50d363
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
+
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+
+/**
+ * Storage of actual information about current index of compressed segments.
+ */
+public class SegmentCompressStorage {
+    /** Flag of interrupt waiting on this object. */
+    private volatile boolean interrupted;
+    /** Manages last archived index, emulates archivation in no-archiver mode. */
+    private final SegmentArchivedStorage segmentArchivedStorage;
+    /** Last successfully compressed segment. */
+    private volatile long lastCompressedIdx = -1L;
+
+    /**
+     * @param segmentArchivedStorage Storage of last archived segment.
+     */
+    private SegmentCompressStorage(SegmentArchivedStorage segmentArchivedStorage) {
+        this.segmentArchivedStorage = segmentArchivedStorage;
+
+        this.segmentArchivedStorage.addObserver(this::onSegmentArchived);
+    }
+
+    /**
+     * @param segmentArchivedStorage Storage of last archived segment.
+     */
+    static SegmentCompressStorage buildCompressStorage(SegmentArchivedStorage segmentArchivedStorage) {
+        SegmentCompressStorage storage = new SegmentCompressStorage(segmentArchivedStorage);
+
+        segmentArchivedStorage.addObserver(storage::onSegmentArchived);
+
+        return storage;
+    }
+
+    /**
+     * Force set last compressed segment.
+     *
+     * @param lastCompressedIdx Segment which was last compressed.
+     */
+    void lastCompressedIdx(long lastCompressedIdx) {
+        this.lastCompressedIdx = lastCompressedIdx;
+    }
+
+    /**
+     * @return Last compressed segment.
+     */
+    long lastCompressedIdx() {
+        return lastCompressedIdx;
+    }
+
+    /**
+     * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation. Waits if
+     * there's no segment to archive right now.
+     */
+    synchronized long nextSegmentToCompressOrWait() throws IgniteInterruptedCheckedException {
+        long segmentToCompress = lastCompressedIdx + 1;
+
+        try {
+            while (
+                segmentToCompress > segmentArchivedStorage.lastArchivedAbsoluteIndex()
+                    && !interrupted
+                )
+                wait();
+        }
+        catch (InterruptedException e) {
+            throw new IgniteInterruptedCheckedException(e);
+        }
+
+        checkInterrupted();
+
+        return segmentToCompress;
+    }
+
+    /**
+     * Interrupt waiting on this object.
+     */
+    synchronized void interrupt() {
+        interrupted = true;
+
+        notifyAll();
+    }
+
+    /**
+     * Check for interrupt flag was set.
+     */
+    private void checkInterrupted() throws IgniteInterruptedCheckedException {
+        if (interrupted)
+            throw new IgniteInterruptedCheckedException("Interrupt waiting of change compressed idx");
+    }
+
+    /**
+     * Callback for waking up compressor when new segment is archived.
+     */
+    private synchronized void onSegmentArchived(long lastAbsArchivedIdx) {
+        notifyAll();
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCurrentStateStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCurrentStateStorage.java
new file mode 100644
index 0000000000000..5761ef9fcbb46
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCurrentStateStorage.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
+
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+
+/**
+ * Storage of absolute current segment index.
+ */
+class SegmentCurrentStateStorage {
+    /** Flag of interrupt of waiting on this object. */
+    private volatile boolean interrupted;
+    /** Flag of force interrupt of waiting on this object. Needed for uninterrupted waiters. */
+    private volatile boolean forceInterrupted;
+    /** Total WAL segments count. */
+    private final int walSegmentsCnt;
+    /** Manages last archived index, emulates archivation in no-archiver mode. */
+    private final SegmentArchivedStorage segmentArchivedStorage;
+    /**
+     * Absolute current segment index WAL Manager writes to. Guarded by <code>this</code>. Incremented during rollover.
+     * Also may be directly set if WAL is resuming logging after start.
+     */
+    private volatile long curAbsWalIdx = -1;
+
+    /**
+     * @param walSegmentsCnt Total WAL segments count.
+     * @param segmentArchivedStorage Last archived segment storage.
+     */
+    private SegmentCurrentStateStorage(int walSegmentsCnt, SegmentArchivedStorage segmentArchivedStorage) {
+        this.walSegmentsCnt = walSegmentsCnt;
+        this.segmentArchivedStorage = segmentArchivedStorage;
+    }
+
+    /**
+     * @param walSegmentsCnt Total WAL segments count.
+     * @param segmentArchivedStorage Last archived segment storage.
+     */
+    static SegmentCurrentStateStorage buildCurrentStateStorage(
+        int walSegmentsCnt,
+        SegmentArchivedStorage segmentArchivedStorage
+    ) {
+
+        SegmentCurrentStateStorage currStorage = new SegmentCurrentStateStorage(walSegmentsCnt, segmentArchivedStorage);
+
+        segmentArchivedStorage.addObserver(currStorage::onSegmentArchived);
+
+        return currStorage;
+    }
+
+    /**
+     * Waiting until current WAL index will be greater or equal than given one.
+     *
+     * @param absSegIdx Target WAL index.
+     */
+    synchronized void awaitSegment(long absSegIdx) throws IgniteInterruptedCheckedException {
+        try {
+            while (curAbsWalIdx < absSegIdx && !interrupted)
+                wait();
+        }
+        catch (InterruptedException e) {
+            throw new IgniteInterruptedCheckedException(e);
+        }
+
+        checkInterrupted();
+    }
+
+    /**
+     * Waiting until archivation of next segment will be allowed.
+     */
+    synchronized long waitNextSegmentForArchivation() throws IgniteInterruptedCheckedException {
+        long lastArchivedSegment = segmentArchivedStorage.lastArchivedAbsoluteIndex();
+
+        //We can archive segment if it less than current work segment so for archivate lastArchiveSegment + 1
+        // we should be ensure that currentWorkSegment = lastArchiveSegment + 2
+        awaitSegment(lastArchivedSegment + 2);
+
+        return lastArchivedSegment + 1;
+    }
+
+    /**
+     * Calculate next segment index or wait if needed. Uninterrupted waiting. - for force interrupt used
+     * forceInterrupted flag.
+     *
+     * @return Next absolute segment index.
+     */
+    synchronized long nextAbsoluteSegmentIndex() throws IgniteInterruptedCheckedException {
+        curAbsWalIdx++;
+
+        notifyAll();
+
+        try {
+            while (curAbsWalIdx - segmentArchivedStorage.lastArchivedAbsoluteIndex() > walSegmentsCnt && !forceInterrupted)
+                wait();
+        }
+        catch (InterruptedException e) {
+            throw new IgniteInterruptedCheckedException(e);
+        }
+
+        if (forceInterrupted)
+            throw new IgniteInterruptedCheckedException("Interrupt waiting of change archived idx");
+
+        return curAbsWalIdx;
+    }
+
+    /**
+     * Update current WAL index.
+     *
+     * @param curAbsWalIdx New current WAL index.
+     */
+    synchronized void curAbsWalIdx(long curAbsWalIdx) {
+        this.curAbsWalIdx = curAbsWalIdx;
+
+        notifyAll();
+    }
+
+    /**
+     * @return Current WAL index.
+     */
+    long curAbsWalIdx() {
+        return this.curAbsWalIdx;
+    }
+
+    /**
+     * Interrupt waiting on this object.
+     */
+    synchronized void interrupt() {
+        interrupted = true;
+
+        notifyAll();
+    }
+
+    /**
+     * Interrupt waiting on this object.
+     */
+    synchronized void forceInterrupt() {
+        interrupted = true;
+        forceInterrupted = true;
+
+        notifyAll();
+    }
+
+    /**
+     * Callback for waking up awaiting when new segment is archived.
+     */
+    private synchronized void onSegmentArchived(long lastAbsArchivedIdx) {
+        notifyAll();
+    }
+
+    /**
+     * Check for interrupt flag was set.
+     */
+    private void checkInterrupted() throws IgniteInterruptedCheckedException {
+        if (interrupted)
+            throw new IgniteInterruptedCheckedException("Interrupt waiting of change current idx");
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java
new file mode 100644
index 0000000000000..2e145e7789f43
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
+
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+
+/**
+ * Lock on segment protects from archiving segment.
+ */
+public class SegmentLockStorage extends SegmentObservable {
+    /**
+     * Maps absolute segment index to locks counter. Lock on segment protects from archiving segment and may come from
+     * {@link FileWriteAheadLogManager.RecordsIterator} during WAL replay. Map itself is guarded by <code>this</code>.
+     */
+    private Map<Long, Integer> locked = new HashMap<>();
+
+    /**
+     * Check if WAL segment locked (protected from move to archive)
+     *
+     * @param absIdx Index for check reservation.
+     * @return {@code True} if index is locked.
+     */
+    public synchronized boolean locked(long absIdx) {
+        return locked.containsKey(absIdx);
+    }
+
+    /**
+     * @param absIdx Segment absolute index.
+     * @return <ul><li>{@code True} if can read, no lock is held, </li><li>{@code false} if work segment, need release
+     * segment later, use {@link #releaseWorkSegment} for unlock</li> </ul>
+     */
+    @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
+    synchronized boolean lockWorkSegment(long absIdx) {
+        Integer cur = locked.get(absIdx);
+
+        cur = cur == null ? 1 : cur + 1;
+
+        locked.put(absIdx, cur);
+
+        return false;
+    }
+
+    /**
+     * @param absIdx Segment absolute index.
+     */
+    @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
+    synchronized void releaseWorkSegment(long absIdx) {
+        Integer cur = locked.get(absIdx);
+
+        assert cur != null && cur >= 1 : "cur=" + cur + ", absIdx=" + absIdx;
+
+        if (cur == 1)
+            locked.remove(absIdx);
+        else
+            locked.put(absIdx, cur - 1);
+
+        notifyObservers(absIdx);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java
new file mode 100644
index 0000000000000..ba5ad300cd127
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * Implementation of observer-observable pattern. For handling specific changes of segment.
+ */
+public abstract class SegmentObservable {
+    /** Observers for handle changes of archived index. */
+    private final List<Consumer<Long>> observers = new ArrayList<>();
+
+    /**
+     * @param observer Observer for notification about segment's changes.
+     */
+    synchronized void addObserver(Consumer<Long> observer) {
+        observers.add(observer);
+    }
+
+    /**
+     * Notify observers about changes.
+     *
+     * @param segmentId Segment which was been changed.
+     */
+    synchronized void notifyObservers(long segmentId) {
+        observers.forEach(observer -> observer.accept(segmentId));
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentReservationStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentReservationStorage.java
similarity index 93%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentReservationStorage.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentReservationStorage.java
index 12c4b4f2ffd96..50c2bbf067d84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentReservationStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentReservationStorage.java
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.ignite.internal.processors.cache.persistence.wal;
+package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
 
 import java.util.NavigableMap;
 import java.util.TreeMap;
@@ -24,8 +24,8 @@
  */
 class SegmentReservationStorage {
     /**
-     * Maps absolute segment index to reservation counter. If counter > 0 then we wouldn't delete all segments
-     * which has index >= reserved segment index. Guarded by {@code this}.
+     * Maps absolute segment index to reservation counter. If counter > 0 then we wouldn't delete all segments which has
+     * index >= reserved segment index. Guarded by {@code this}.
      */
     private NavigableMap<Long, Integer> reserved = new TreeMap<>();
 
@@ -38,6 +38,7 @@ synchronized void reserve(long absIdx) {
 
     /**
      * Checks if segment is currently reserved (protected from deletion during WAL cleanup).
+     *
      * @param absIdx Index for check reservation.
      * @return {@code True} if index is reserved.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java
new file mode 100644
index 0000000000000..d19d17b3c7d7f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.io;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * File input, backed by byte buffer file input.
+ * This class allows to read data by chunks from file and then read primitives.
+ */
+public interface FileInput extends ByteBufferBackedDataInput {
+    /**
+     * File I/O.
+     */
+    FileIO io();
+
+    /**
+     * @param pos Position in bytes from file begin.
+     */
+    void seek(long pos) throws IOException;
+
+    /**
+     * @return Position in the stream.
+     */
+    long position();
+
+    /**
+     * @param skipCheck If CRC check should be skipped.
+     * @return autoclosable fileInput, after its closing crc32 will be calculated and compared with saved one
+     */
+    SimpleFileInput.Crc32CheckingFileInput startRead(boolean skipCheck);
+
+    /**
+     * Checking of CRC32.
+     */
+    public class Crc32CheckingFileInput implements ByteBufferBackedDataInput, AutoCloseable {
+        /** */
+        private final PureJavaCrc32 crc32 = new PureJavaCrc32();
+
+        /** Last calc position. */
+        private int lastCalcPosition;
+
+        /** Skip crc check. */
+        private boolean skipCheck;
+
+        /** */
+        private FileInput delegate;
+
+        /**
+         */
+        public Crc32CheckingFileInput(FileInput delegate, boolean skipCheck) {
+            this.delegate = delegate;
+            this.lastCalcPosition = delegate.buffer().position();
+            this.skipCheck = skipCheck;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void ensure(int requested) throws IOException {
+            int available = buffer().remaining();
+
+            if (available >= requested)
+                return;
+
+            updateCrc();
+
+            delegate.ensure(requested);
+
+            lastCalcPosition = 0;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void close() throws Exception {
+            updateCrc();
+
+            int val = crc32.getValue();
+
+            int writtenCrc =  this.readInt();
+
+            if ((val ^ writtenCrc) != 0 && !skipCheck) {
+                // If it last message we will skip it (EOF will be thrown).
+                ensure(5);
+
+                throw new IgniteDataIntegrityViolationException(
+                    "val: " + val + " writtenCrc: " + writtenCrc
+                );
+            }
+        }
+
+        /**
+         *
+         */
+        private void updateCrc() {
+            if (skipCheck)
+                return;
+
+            int oldPos = buffer().position();
+
+            buffer().position(lastCalcPosition);
+
+            crc32.update(delegate.buffer(), oldPos - lastCalcPosition);
+
+            lastCalcPosition = oldPos;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int skipBytes(int n) throws IOException {
+            ensure(n);
+
+            int skipped = Math.min(buffer().remaining(), n);
+
+            buffer().position(buffer().position() + skipped);
+
+            return skipped;
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public void readFully(@NotNull byte[] b) throws IOException {
+            ensure(b.length);
+
+            buffer().get(b);
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
+            ensure(len);
+
+            buffer().get(b, off, len);
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public boolean readBoolean() throws IOException {
+            return readByte() == 1;
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public byte readByte() throws IOException {
+            ensure(1);
+
+            return buffer().get();
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public int readUnsignedByte() throws IOException {
+            return readByte() & 0xFF;
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public short readShort() throws IOException {
+            ensure(2);
+
+            return buffer().getShort();
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public int readUnsignedShort() throws IOException {
+            return readShort() & 0xFFFF;
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public char readChar() throws IOException {
+            ensure(2);
+
+            return buffer().getChar();
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public int readInt() throws IOException {
+            ensure(4);
+
+            return buffer().getInt();
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public long readLong() throws IOException {
+            ensure(8);
+
+            return buffer().getLong();
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public float readFloat() throws IOException {
+            ensure(4);
+
+            return buffer().getFloat();
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public double readDouble() throws IOException {
+            ensure(8);
+
+            return buffer().getDouble();
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public String readLine() throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        /**
+         * {@inheritDoc}
+         */
+        @Override public String readUTF() throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        /** {@inheritDoc} */
+        @Override public ByteBuffer buffer() {
+            return delegate.buffer();
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/LockedReadFileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/LockedReadFileInput.java
new file mode 100644
index 0000000000000..6e5a7a4d2c526
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/LockedReadFileInput.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.io;
+
+import java.io.IOException;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
+import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware;
+
+/**
+ * File input, backed by byte buffer file input. This class allows to read data by chunks from file and then read
+ * primitives.
+ *
+ * This implementation locks segment only for reading to buffer and also can switch reading segment from work directory
+ * to archive directory if needed.
+ */
+final class LockedReadFileInput extends SimpleFileInput {
+    /** Segment for read. */
+    private final long segmentId;
+    /** Holder of actual information of latest manipulation on WAL segments. */
+    private final SegmentAware segmentAware;
+    /** Factory of file I/O for segment. */
+    private final SegmentIoFactory fileIOFactory;
+    /** Last read was from archive or not. */
+    private boolean isLastReadFromArchive;
+
+    /**
+     * @param buf Buffer for reading blocks of data into.
+     * @param initFileIo Initial File I/O for reading.
+     * @param segmentAware Holder of actual information of latest manipulation on WAL segments.
+     * @param segmentIOFactory Factory of file I/O for segment.
+     * @throws IOException if initFileIo would be fail during reading.
+     */
+    LockedReadFileInput(
+        ByteBufferExpander buf,
+        SegmentIO initFileIo,
+        SegmentAware segmentAware,
+        SegmentIoFactory segmentIOFactory
+    ) throws IOException {
+        super(initFileIo, buf);
+        this.segmentAware = segmentAware;
+        this.fileIOFactory = segmentIOFactory;
+        this.segmentId = initFileIo.getSegmentId();
+        isLastReadFromArchive = segmentAware.lastArchivedAbsoluteIndex() >= initFileIo.getSegmentId();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void ensure(int requested) throws IOException {
+        int available = buffer().remaining();
+
+        if (available >= requested)
+            return;
+
+        boolean readArchive = segmentAware.checkCanReadArchiveOrReserveWorkSegment(segmentId);
+        try {
+            if (readArchive && !isLastReadFromArchive) {
+                isLastReadFromArchive = true;
+
+                refreshIO();
+            }
+
+            super.ensure(requested);
+        }
+        finally {
+            if (!readArchive)
+                segmentAware.releaseWorkSegment(segmentId);
+        }
+    }
+
+    /**
+     * Refresh current file io.
+     *
+     * @throws IOException if old fileIO is fail during reading or new file is fail during creation.
+     */
+    private void refreshIO() throws IOException {
+        FileIO io = fileIOFactory.build(segmentId);
+
+        io.position(io().position());
+
+        io().close();
+
+        this.io = io;
+    }
+
+    /**
+     * Resolving fileIo for segment.
+     */
+    interface SegmentIoFactory {
+        /**
+         * @param segmentId Segment for IO action.
+         * @return {@link FileIO}.
+         * @throws IOException if creation would be fail.
+         */
+        FileIO build(long segmentId) throws IOException;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/LockedSegmentFileInputFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/LockedSegmentFileInputFactory.java
new file mode 100644
index 0000000000000..f3cdda737b4c0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/LockedSegmentFileInputFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.io;
+
+import java.io.IOException;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware;
+import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentRouter;
+
+/**
+ * Implementation of factory to provide I/O interfaces for read primitives with files.
+ *
+ * Creating {@link FileInput} with ability locked segment during reading.
+ */
+public class LockedSegmentFileInputFactory implements SegmentFileInputFactory {
+    /** Holder of actual information of latest manipulation on WAL segments. */
+    private final SegmentAware segmentAware;
+    /** Manager of segment location. */
+    private final SegmentRouter segmentRouter;
+    /** {@link FileIO} factory definition.*/
+    private final FileIOFactory fileIOFactory;
+
+    /**
+     * @param segmentAware Holder of actual information of latest manipulation on WAL segments.
+     * @param segmentRouter Manager of segment location.
+     * @param fileIOFactory {@link FileIO} factory definition.
+     */
+    public LockedSegmentFileInputFactory(
+        SegmentAware segmentAware,
+        SegmentRouter segmentRouter,
+        FileIOFactory fileIOFactory) {
+        this.segmentAware = segmentAware;
+        this.segmentRouter = segmentRouter;
+        this.fileIOFactory = fileIOFactory;
+    }
+
+    /** {@inheritDoc} */
+    @Override public FileInput createFileInput(SegmentIO segmentIO, ByteBufferExpander buf) throws IOException {
+        return new LockedReadFileInput(
+            buf,
+            segmentIO,
+            segmentAware,
+            id -> {
+                FileDescriptor segment = segmentRouter.findSegment(id);
+
+                return segment.toIO(fileIOFactory);
+            }
+        );
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SegmentFileInputFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SegmentFileInputFactory.java
new file mode 100644
index 0000000000000..b688f903a661f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SegmentFileInputFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.io;
+
+import java.io.IOException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
+
+/**
+ * Factory to provide I/O interfaces for read primitives with files.
+ */
+public interface SegmentFileInputFactory {
+    /**
+     * @param segmentIO FileIO of segment for reading.
+     * @param buf ByteBuffer wrapper for dynamically expand buffer size.
+     * @return Instance of {@link FileInput}.
+     * @throws IOException If have some trouble with I/O.
+     */
+    FileInput createFileInput(SegmentIO segmentIO, ByteBufferExpander buf) throws IOException;
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SegmentIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SegmentIO.java
new file mode 100644
index 0000000000000..d0a64457e38ba
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SegmentIO.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.io;
+
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+
+/**
+ * Implementation of {@link FileIO} specified for WAL segment file.
+ */
+public class SegmentIO extends FileIODecorator {
+    /** Segment id. */
+    private final long segmentId;
+
+    /**
+     * @param id Segment id.
+     * @param delegate File I/O delegate
+     */
+    public SegmentIO(long id, FileIO delegate) {
+        super(delegate);
+        segmentId = id;
+    }
+
+    /**
+     * @return Segment id.
+     */
+    public long getSegmentId() {
+        return segmentId;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java
similarity index 52%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java
index 303a0231b9d2a..5918b0b34e93e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java
@@ -15,21 +15,20 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.persistence.wal;
+package org.apache.ignite.internal.processors.cache.persistence.wal.io;
 
 import java.io.EOFException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
-import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
 import org.jetbrains.annotations.NotNull;
 
 /**
  * File input, backed by byte buffer file input.
  * This class allows to read data by chunks from file and then read primitives
  */
-public final class FileInput implements ByteBufferBackedDataInput {
+public class SimpleFileInput implements FileInput {
     /**
      * Buffer for reading blocks of data into.
      * <b>Note:</b> biggest block requested from this input can't be longer than buffer capacity
@@ -37,7 +36,7 @@ public final class FileInput implements ByteBufferBackedDataInput {
     private ByteBuffer buf;
 
     /** I/O interface for read/write operations with file */
-    private FileIO io;
+    protected FileIO io;
 
     /** */
     private long pos;
@@ -49,7 +48,7 @@ public final class FileInput implements ByteBufferBackedDataInput {
      * @param io FileIO to read from.
      * @param buf Buffer for reading blocks of data into.
      */
-    public FileInput(FileIO io, ByteBufferExpander buf) throws IOException {
+    public SimpleFileInput(FileIO io, ByteBufferExpander buf) throws IOException {
         assert io != null;
 
         this.io = io;
@@ -62,10 +61,8 @@ public FileInput(FileIO io, ByteBufferExpander buf) throws IOException {
         clearBuffer();
     }
 
-    /**
-     * File I/O.
-     */
-    public FileIO io() {
+    /** {@inheritDoc} */
+    @Override public FileIO io() {
         return io;
     }
 
@@ -79,10 +76,8 @@ private void clearBuffer() {
         assert buf.remaining() == 0; // Buffer is empty.
     }
 
-    /**
-     * @param pos Position in bytes from file begin.
-     */
-    public void seek(long pos) throws IOException {
+    /** {@inheritDoc} */
+    @Override public void seek(long pos) throws IOException {
         if (pos > io.size())
             throw new EOFException();
 
@@ -137,7 +132,7 @@ public void seek(long pos) throws IOException {
     /**
      * @return Position in the stream.
      */
-    public long position() {
+    @Override public long position() {
         return pos - buf.remaining();
     }
 
@@ -262,9 +257,7 @@ public long position() {
         throw new UnsupportedOperationException();
     }
 
-    /**
-     * {@inheritDoc}
-     */
+    /** {@inheritDoc} */
     @Override public String readUTF() throws IOException {
         throw new UnsupportedOperationException();
     }
@@ -274,208 +267,6 @@ public long position() {
      * @return autoclosable fileInput, after its closing crc32 will be calculated and compared with saved one
      */
     public Crc32CheckingFileInput startRead(boolean skipCheck) {
-        return new Crc32CheckingFileInput(buf.position(), skipCheck);
-    }
-
-    /**
-     * Checking of CRC32.
-     */
-    public class Crc32CheckingFileInput implements ByteBufferBackedDataInput, AutoCloseable {
-        /** */
-        private final PureJavaCrc32 crc32 = new PureJavaCrc32();
-
-        /** Last calc position. */
-        private int lastCalcPosition;
-
-        /** Skip crc check. */
-        private boolean skipCheck;
-
-        /**
-         * @param position Position.
-         */
-        public Crc32CheckingFileInput(int position, boolean skipCheck) {
-            this.lastCalcPosition = position;
-            this.skipCheck = skipCheck;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void ensure(int requested) throws IOException {
-            int available = buf.remaining();
-
-            if (available >= requested)
-                return;
-
-            updateCrc();
-
-            FileInput.this.ensure(requested);
-
-            lastCalcPosition = 0;
-        }
-
-        /** {@inheritDoc} */
-        @Override public void close() throws Exception {
-            updateCrc();
-
-            int val = crc32.getValue();
-
-            int writtenCrc =  this.readInt();
-
-            if ((val ^ writtenCrc) != 0 && !skipCheck) {
-                // If it last message we will skip it (EOF will be thrown).
-                ensure(5);
-
-                throw new IgniteDataIntegrityViolationException(
-                    "val: " + val + " writtenCrc: " + writtenCrc
-                );
-            }
-        }
-
-        /**
-         *
-         */
-        private void updateCrc() {
-            if (skipCheck)
-                return;
-
-            int oldPos = buf.position();
-
-            buf.position(lastCalcPosition);
-
-            crc32.update(buf, oldPos - lastCalcPosition);
-
-            lastCalcPosition = oldPos;
-        }
-
-        /** {@inheritDoc} */
-        @Override public int skipBytes(int n) throws IOException {
-            ensure(n);
-
-            int skipped = Math.min(buf.remaining(), n);
-
-            buf.position(buf.position() + skipped);
-
-            return skipped;
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public void readFully(@NotNull byte[] b) throws IOException {
-            ensure(b.length);
-
-            buf.get(b);
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
-            ensure(len);
-
-            buf.get(b, off, len);
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public boolean readBoolean() throws IOException {
-            return readByte() == 1;
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public byte readByte() throws IOException {
-            ensure(1);
-
-            return buf.get();
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public int readUnsignedByte() throws IOException {
-            return readByte() & 0xFF;
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public short readShort() throws IOException {
-            ensure(2);
-
-            return buf.getShort();
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public int readUnsignedShort() throws IOException {
-            return readShort() & 0xFFFF;
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public char readChar() throws IOException {
-            ensure(2);
-
-            return buf.getChar();
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public int readInt() throws IOException {
-            ensure(4);
-
-            return buf.getInt();
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public long readLong() throws IOException {
-            ensure(8);
-
-            return buf.getLong();
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public float readFloat() throws IOException {
-            ensure(4);
-
-            return buf.getFloat();
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public double readDouble() throws IOException {
-            ensure(8);
-
-            return buf.getDouble();
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public String readLine() throws IOException {
-            throw new UnsupportedOperationException();
-        }
-
-        /**
-         * {@inheritDoc}
-         */
-        @Override public String readUTF() throws IOException {
-            throw new UnsupportedOperationException();
-        }
-
-        /** {@inheritDoc} */
-        @Override public ByteBuffer buffer() {
-            return FileInput.this.buffer();
-        }
+        return new Crc32CheckingFileInput(this, skipCheck);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleSegmentFileInputFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleSegmentFileInputFactory.java
new file mode 100644
index 0000000000000..2e5ea7519e6bf
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleSegmentFileInputFactory.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.io;
+
+import java.io.IOException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
+
+/**
+ * Simple implementation of {@link SegmentFileInputFactory}.
+ */
+public class SimpleSegmentFileInputFactory implements SegmentFileInputFactory {
+
+    /** {@inheritDoc} */
+    @Override public FileInput createFileInput(SegmentIO segmentIO,
+        ByteBufferExpander buf) throws IOException {
+        return new SimpleFileInput(segmentIO, buf);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
index 14bca7fe87fc8..f9388eaa15898 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/IgniteWalIteratorFactory.java
@@ -39,13 +39,13 @@
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
-import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleSegmentFileInputFactory;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -67,6 +67,9 @@ public class IgniteWalIteratorFactory {
     /** Logger. */
     private final IgniteLogger log;
 
+    /** */
+    private final SegmentFileInputFactory segmentFileInputFactory = new SimpleSegmentFileInputFactory();
+
     /**
      * Creates WAL files iterator factory.
      * WAL iterator supports automatic converting from CacheObjects and KeyCacheObject into BinaryObjects
@@ -319,10 +322,10 @@ private FileDescriptor readFileDescriptor(File file, FileIOFactory ioFactory) {
         FileDescriptor ds = new FileDescriptor(file);
 
         try (
-            FileIO fileIO = ds.isCompressed() ? new UnzipFileIO(file) : ioFactory.create(file);
+            SegmentIO fileIO = ds.toIO(ioFactory);
             ByteBufferExpander buf = new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())
         ) {
-            final DataInput in = new FileInput(fileIO, buf);
+            final DataInput in = segmentFileInputFactory.createFileInput(fileIO, buf);
 
             // Header record must be agnostic to the serializer version.
             final int type = in.readUnsignedByte();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index eea734fbd2344..b08fde46bf3f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -37,16 +37,17 @@
 import org.apache.ignite.internal.processors.cache.CacheObjectContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
-import org.apache.ignite.internal.processors.cache.persistence.file.UnzipFileIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.ReadFileHandle;
 import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleSegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.SegmentHeader;
@@ -72,6 +73,9 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
 
     /** */
     private static final long serialVersionUID = 0L;
+
+    /** Factory to provide I/O interfaces for read primitives with files. */
+    private static final SegmentFileInputFactory FILE_INPUT_FACTORY = new SimpleSegmentFileInputFactory();
     /**
      * File descriptors remained to scan.
      * <code>null</code> value means directory scan mode
@@ -118,7 +122,8 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
             sharedCtx,
             new RecordSerializerFactoryImpl(sharedCtx, readTypeFilter),
             ioFactory,
-            initialReadBufferSize
+            initialReadBufferSize,
+            FILE_INPUT_FACTORY
         );
 
         this.lowBound = lowBound;
@@ -262,13 +267,13 @@ private boolean checkBounds(long idx) {
     ) throws IgniteCheckedException, FileNotFoundException {
 
         AbstractFileDescriptor fd = desc;
-        FileIO fileIO = null;
+        SegmentIO fileIO = null;
         SegmentHeader segmentHeader;
         while (true) {
             try {
-                fileIO = fd.isCompressed() ? new UnzipFileIO(fd.file()) : ioFactory.create(fd.file());
+                fileIO = fd.toIO(ioFactory);
 
-                segmentHeader = readSegmentHeader(fileIO, curWalSegmIdx);
+                segmentHeader = readSegmentHeader(fileIO, FILE_INPUT_FACTORY);
 
                 break;
             }
@@ -422,8 +427,8 @@ private boolean checkBounds(long idx) {
 
     /** {@inheritDoc} */
     @Override protected AbstractReadFileHandle createReadFileHandle(
-        FileIO fileIO, long idx, RecordSerializer ser, FileInput in
+        SegmentIO fileIO, RecordSerializer ser, FileInput in
     ) {
-        return new ReadFileHandle(fileIO, idx, ser, in);
+        return new ReadFileHandle(fileIO, ser, in, null);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializer.java
index c5760ab3bb30b..a9d793dbc8a0d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordSerializer.java
@@ -22,7 +22,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
 
 /**
  * Record serializer.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
index ca484ce112031..afd770d8efecf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -29,13 +29,15 @@
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType;
-import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleFileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
@@ -246,14 +248,14 @@ public static void putPosition(ByteBuffer buf, FileWALPointer ptr) {
      * NOTE: Method mutates position of {@code io}.
      *
      * @param io I/O interface for file.
-     * @param expectedIdx Expected WAL segment index for readable record.
+     * @param segmentFileInputFactory File input factory.
      * @return Instance of {@link SegmentHeader} extracted from the file.
      * @throws IgniteCheckedException If failed to read serializer version.
      */
-    public static SegmentHeader readSegmentHeader(FileIO io, long expectedIdx)
+    public static SegmentHeader readSegmentHeader(SegmentIO io, SegmentFileInputFactory segmentFileInputFactory)
         throws IgniteCheckedException, IOException {
         try (ByteBufferExpander buf = new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())) {
-            FileInput in = new FileInput(io, buf);
+            ByteBufferBackedDataInput in = segmentFileInputFactory.createFileInput(io, buf);
 
             in.ensure(HEADER_RECORD_SIZE);
 
@@ -270,7 +272,7 @@ public static SegmentHeader readSegmentHeader(FileIO io, long expectedIdx)
             // Read file pointer.
             FileWALPointer ptr = readPosition(in);
 
-            if (expectedIdx != ptr.index())
+            if (io.getSegmentId() != ptr.index())
                 throw new SegmentEofException("Reached logical end of the segment by pointer", null);
 
             assert ptr.fileOffset() == 0 : "Header record should be placed at the beginning of file " + ptr;
@@ -364,7 +366,7 @@ static WALRecord readWithCrc(
     ) throws EOFException, IgniteCheckedException {
         long startPos = -1;
 
-        try (FileInput.Crc32CheckingFileInput in = in0.startRead(skipCrc)) {
+        try (SimpleFileInput.Crc32CheckingFileInput in = in0.startRead(skipCrc)) {
             startPos = in0.position();
 
             WALRecord res = reader.readWithHeaders(in, expPtr);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
index 68e55e0cfeb24..e112522fc31f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV2Serializer.java
@@ -28,7 +28,7 @@
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
index 5fd5a1bee8525..dcc7b70187344 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWALTailIsReachedDuringIterationOverArchiveTest.java
@@ -39,7 +39,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
 import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder;
 import org.apache.ignite.internal.util.typedef.internal.U;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
index e2c8bd1fce568..b69314d83a8dc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
@@ -65,8 +65,8 @@ public class IgniteWalHistoryReservationsTest extends GridCommonAbstractTest {
                 new DataRegionConfiguration()
                     .setMaxSize(200 * 1024 * 1024)
                     .setPersistenceEnabled(true))
-                .setWalMode(WALMode.LOG_ONLY)
-                .setWalSegmentSize(512 * 1024);
+            .setWalMode(WALMode.LOG_ONLY)
+            .setWalSegmentSize(512 * 1024);
 
         cfg.setDataStorageConfiguration(memCfg);
 
@@ -122,8 +122,8 @@ public void testReservedOnExchange() throws Exception {
 
         log.warning("Start loading");
 
-        try (IgniteDataStreamer<Object, Object> st = ig0.dataStreamer("cache1")){
-            for (int k = 0; k < entryCnt; k++){
+        try (IgniteDataStreamer<Object, Object> st = ig0.dataStreamer("cache1")) {
+            for (int k = 0; k < entryCnt; k++) {
                 st.addData(k, k);
 
                 printProgress(k);
@@ -156,10 +156,10 @@ public void testReservedOnExchange() throws Exception {
 
         log.warning("Start loading");
 
-        try (IgniteDataStreamer<Object, Object> st = ig0.dataStreamer("cache1")){
+        try (IgniteDataStreamer<Object, Object> st = ig0.dataStreamer("cache1")) {
             st.allowOverwrite(true);
 
-            for (int k = 0; k < entryCnt; k++){
+            for (int k = 0; k < entryCnt; k++) {
                 st.addData(k, k);
 
                 printProgress(k);
@@ -191,16 +191,8 @@ public void testReservedOnExchange() throws Exception {
                     for (int g = 0; g < initGridCnt; g++) {
                         IgniteEx ig = grid(g);
 
-                        FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
-
-                        Object reservationStorage = GridTestUtils.getFieldValue(wal, "reservationStorage");
-
-                        synchronized (reservationStorage) {
-                            Map reserved = GridTestUtils.getFieldValue(reservationStorage, "reserved");
-
-                            if (reserved.isEmpty())
-                                return false;
-                        }
+                        if (isReserveListEmpty(ig))
+                            return false;
                     }
 
                     return true;
@@ -218,16 +210,8 @@ public void testReservedOnExchange() throws Exception {
                 for (int g = 0; g < initGridCnt; g++) {
                     IgniteEx ig = grid(g);
 
-                    FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
-
-                    Object reservationStorage = GridTestUtils.getFieldValue(wal, "reservationStorage");
-
-                    synchronized (reservationStorage) {
-                        Map reserved = GridTestUtils.getFieldValue(reservationStorage, "reserved");
-
-                        if (!reserved.isEmpty())
-                            return false;
-                    }
+                    if (isReserveListEmpty(ig))
+                        return false;
                 }
 
                 return true;
@@ -237,10 +221,27 @@ public void testReservedOnExchange() throws Exception {
         assert released;
     }
 
+    /**
+     * @return {@code true} if reserve list is empty.
+     */
+    private boolean isReserveListEmpty(IgniteEx ig) {
+        FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
+
+        Object segmentAware = GridTestUtils.getFieldValue(wal, "segmentAware");
+
+        synchronized (segmentAware) {
+            Map reserved = GridTestUtils.getFieldValue(GridTestUtils.getFieldValue(segmentAware, "reservationStorage"), "reserved");
+
+            if (reserved.isEmpty())
+                return true;
+        }
+        return false;
+    }
+
     /**
      *
      */
-    private void printProgress(int k){
+    private void printProgress(int k) {
         if (k % 1000 == 0)
             log.warning("Keys -> " + k);
     }
@@ -251,7 +252,7 @@ private void printProgress(int k){
     public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception {
         int entryCnt = 10_000;
 
-        IgniteEx ig0 = (IgniteEx) startGrids(2);
+        IgniteEx ig0 = (IgniteEx)startGrids(2);
 
         ig0.active(true);
 
@@ -300,7 +301,7 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception {
     public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception {
         int entryCnt = 10_000;
 
-        IgniteEx ig0 = (IgniteEx) startGrids(2);
+        IgniteEx ig0 = (IgniteEx)startGrids(2);
 
         ig0.active(true);
 
@@ -360,7 +361,7 @@ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception {
     public void testWalHistoryPartiallyRemoved() throws Exception {
         int entryCnt = 10_000;
 
-        IgniteEx ig0 = (IgniteEx) startGrids(2);
+        IgniteEx ig0 = (IgniteEx)startGrids(2);
 
         ig0.cluster().active(true);
 
@@ -384,7 +385,7 @@ public void testWalHistoryPartiallyRemoved() throws Exception {
         stopAllGrids();
 
         U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), walArchPath + "/" +
-                nodeId0, false));
+            nodeId0, false));
 
         startGrid(0);
 
@@ -434,16 +435,8 @@ public void testNodeLeftDuringExchange() throws Exception {
                     for (int g = 0; g < initGridCnt; g++) {
                         IgniteEx ig = grid(g);
 
-                        FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
-
-                        Object reservationStorage = GridTestUtils.getFieldValue(wal, "reservationStorage");
-
-                        synchronized (reservationStorage) {
-                            Map reserved = GridTestUtils.getFieldValue(reservationStorage, "reserved");
-
-                            if (reserved.isEmpty())
-                                return false;
-                        }
+                        if (isReserveListEmpty(ig))
+                            return false;
                     }
 
                     return true;
@@ -463,16 +456,8 @@ public void testNodeLeftDuringExchange() throws Exception {
                 for (int g = 0; g < initGridCnt - 1; g++) {
                     IgniteEx ig = grid(g);
 
-                    FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
-
-                    Object reservationStorage = GridTestUtils.getFieldValue(wal, "reservationStorage");
-
-                    synchronized (reservationStorage) {
-                        Map reserved = GridTestUtils.getFieldValue(reservationStorage, "reserved");
-
-                        if (!reserved.isEmpty())
-                            return false;
-                    }
+                    if (isReserveListEmpty(ig))
+                        return false;
                 }
 
                 return true;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
index 9dbef5dcfa181..74db28fed0072 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalIteratorSwitchSegmentTest.java
@@ -18,13 +18,18 @@
 package org.apache.ignite.internal.processors.cache.persistence.db.wal;
 
 import java.io.File;
+import java.nio.channels.Channel;
 import java.nio.file.Paths;
 import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
@@ -40,8 +45,11 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FsyncModeFileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneGridKernalContext;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
@@ -58,6 +66,8 @@
 
 import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.METASTORE_DATA_RECORD;
 import static org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer.HEADER_RECORD_SIZE;
+import static org.apache.ignite.testframework.GridTestUtils.getFieldValueHierarchy;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
 
 /***
  * Test check correct switch segment if in the tail of segment have garbage.
@@ -159,7 +169,7 @@ private void checkInvariantSwitchSegmentSize(int serVer) throws Exception {
      *
      * @throws Exception If some thing failed.
      */
-    public void test() throws Exception {
+    public void testInvariantSwitchSegment() throws Exception {
         for (int serVer : checkSerializerVers) {
             for (Class walMgrClass : checkWalManagers) {
                 try {
@@ -174,6 +184,22 @@ public void test() throws Exception {
         }
     }
 
+    /**
+     * Test for check switch segment from work dir to archive dir during iteration.
+     *
+     * @throws Exception If some thing failed.
+     */
+    public void testSwitchReadingSegmentFromWorkToArchive() throws Exception {
+        for (int serVer : checkSerializerVers) {
+                try {
+                    checkSwitchReadingSegmentDuringIteration(FileWriteAheadLogManager.class, serVer);
+                }
+                finally {
+                    U.delete(Paths.get(U.defaultWorkDirectory()));
+                }
+        }
+    }
+
     /**
      * @param walMgrClass WAL manager class.
      * @param serVer WAL serializer version.
@@ -299,6 +325,101 @@ private void checkInvariantSwitchSegment(Class walMgrClass, int serVer) throws E
         Assert.assertEquals("Not all records read during iteration.", expectedRecords, actualRecords);
     }
 
+    /**
+     * @param walMgrClass WAL manager class.
+     * @param serVer WAL serializer version.
+     * @throws Exception If some thing failed.
+     */
+    private void checkSwitchReadingSegmentDuringIteration(Class walMgrClass, int serVer) throws Exception {
+        String workDir = U.defaultWorkDirectory();
+
+        T2<IgniteWriteAheadLogManager, RecordSerializer> initTup = initiate(walMgrClass, serVer, workDir);
+
+        IgniteWriteAheadLogManager walMgr = initTup.get1();
+
+        RecordSerializer recordSerializer = initTup.get2();
+
+        MetastoreDataRecord rec = new MetastoreDataRecord("0", new byte[100]);
+
+        int recSize = recordSerializer.size(rec);
+
+        // Add more record for rollover to the next segment.
+        int recordsToWrite = SEGMENT_SIZE / recSize + 100;
+
+        SegmentAware segmentAware = GridTestUtils.getFieldValue(walMgr, "segmentAware");
+
+        //guard from archivation before iterator would be created.
+        segmentAware.checkCanReadArchiveOrReserveWorkSegment(0);
+
+        for (int i = 0; i < recordsToWrite; i++)
+            walMgr.log(new MetastoreDataRecord(rec.key(), rec.value()));
+
+        walMgr.flush(null, true);
+
+        int expectedRecords = recordsToWrite;
+        AtomicInteger actualRecords = new AtomicInteger(0);
+
+        AtomicReference<String> startedSegmentPath = new AtomicReference<>();
+        AtomicReference<String> finishedSegmentPath = new AtomicReference<>();
+
+        CountDownLatch startedIteratorLatch = new CountDownLatch(1);
+        CountDownLatch finishedArchivedLatch = new CountDownLatch(1);
+
+        IgniteInternalFuture<Object> future = GridTestUtils.runAsync(
+            () -> {
+                // Check that switch segment works as expected and all record is reachable.
+                try (WALIterator it = walMgr.replay(null)) {
+                    Object handle = getFieldValueHierarchy(it,  "currWalSegment");
+
+                    FileInput in = getFieldValueHierarchy(handle, "in");
+                    Object delegate = getFieldValueHierarchy(in.io(), "delegate");
+                    Channel ch = getFieldValueHierarchy(delegate, "ch");
+                    String path = getFieldValueHierarchy(ch, "path");
+
+                    startedSegmentPath.set(path);
+
+                    startedIteratorLatch.countDown();
+
+                    while (it.hasNext()) {
+                        IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+                        WALRecord rec0 = tup.get2();
+
+                        if (rec0.type() == METASTORE_DATA_RECORD)
+                            actualRecords.incrementAndGet();
+
+                        finishedArchivedLatch.await();
+                    }
+
+                    in = getFieldValueHierarchy(handle, "in");
+                    delegate = getFieldValueHierarchy(in.io(), "delegate");
+                    ch = getFieldValueHierarchy(delegate, "ch");
+                    path = getFieldValueHierarchy(ch, "path");
+
+                    finishedSegmentPath.set(path);
+                }
+
+                return null;
+            }
+        );
+
+        startedIteratorLatch.await();
+
+        segmentAware.releaseWorkSegment(0);
+
+        waitForCondition(() -> segmentAware.lastArchivedAbsoluteIndex() == 0, 5000);
+
+        finishedArchivedLatch.countDown();
+
+        future.get();
+
+        //should started iteration from work directory but finish from archive directory.
+        assertEquals(workDir + WORK_SUB_DIR + "/0000000000000000.wal", startedSegmentPath.get());
+        assertEquals(workDir + ARCHIVE_SUB_DIR + "/0000000000000000.wal", finishedSegmentPath.get());
+
+        Assert.assertEquals("Not all records read during iteration.", expectedRecords, actualRecords.get());
+    }
+
     /***
      * Initiate WAL manager.
      *
@@ -323,6 +444,7 @@ private T2<IgniteWriteAheadLogManager, RecordSerializer> initiate(
                 cfg.setDataStorageConfiguration(
                     new DataStorageConfiguration()
                         .setWalSegmentSize(SEGMENT_SIZE)
+                        .setWalRecordIteratorBufferSize(SEGMENT_SIZE / 2)
                         .setWalMode(WALMode.FSYNC)
                         .setWalPath(workDir + WORK_SUB_DIR)
                         .setWalArchivePath(workDir + ARCHIVE_SUB_DIR)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
index dbb91ee5e5d3a..dde333e00c219 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
@@ -30,6 +30,7 @@
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
 import org.apache.ignite.Ignite;
@@ -74,6 +75,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.CA;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.PAX;
@@ -114,6 +116,9 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
     /** */
     private int walSegmentSize;
 
+    /** */
+    private int walSegments = 10;
+
     /** Log only. */
     private boolean logOnly;
 
@@ -159,6 +164,8 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
         if (walSegmentSize != 0)
             dbCfg.setWalSegmentSize(walSegmentSize);
 
+        dbCfg.setWalSegments(walSegments);
+
         cfg.setDataStorageConfiguration(dbCfg);
 
         cfg.setMarshaller(null);
@@ -448,7 +455,7 @@ public void testWalLargeValue() throws Exception {
 
                 Arrays.fill(data, (byte)i);
 
-                final byte[] loaded = (byte[]) cache.get(i);
+                final byte[] loaded = (byte[])cache.get(i);
 
                 Assert.assertArrayEquals(data, loaded);
 
@@ -906,7 +913,7 @@ public void testMetastorage() throws Exception {
                 sharedCtx0.database().checkpointReadLock();
 
                 try {
-                    storage0.putData(String.valueOf(i), new byte[]{(byte)(i % 256), 2, 3});
+                    storage0.putData(String.valueOf(i), new byte[] {(byte)(i % 256), 2, 3});
                 }
                 finally {
                     sharedCtx0.database().checkpointReadUnlock();
@@ -966,7 +973,7 @@ public void testMetastorageLargeArray() throws Exception {
             for (int i = 0; i < cnt; i++) {
                 byte[] b1 = new byte[arraySize];
                 for (int k = 0; k < arraySize; k++) {
-                    b1[k] = (byte) (k % 100);
+                    b1[k] = (byte)(k % 100);
                 }
 
                 sharedCtx.database().checkpointReadLock();
@@ -984,7 +991,7 @@ public void testMetastorageLargeArray() throws Exception {
                 assertEquals(arraySize, d2.length);
 
                 for (int k = 0; k < arraySize; k++) {
-                    assertEquals((byte) (k % 100), d2[k]);
+                    assertEquals((byte)(k % 100), d2[k]);
                 }
             }
 
@@ -1015,7 +1022,7 @@ public void testMetastorageRemove() throws Exception {
                 sharedCtx0.database().checkpointReadLock();
 
                 try {
-                    storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+                    storage.putData(String.valueOf(i), new byte[] {1, 2, 3});
                 }
                 finally {
                     sharedCtx0.database().checkpointReadUnlock();
@@ -1068,7 +1075,7 @@ public void testMetastorageUpdate() throws Exception {
                 sharedCtx0.database().checkpointReadLock();
 
                 try {
-                    storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+                    storage.putData(String.valueOf(i), new byte[] {1, 2, 3});
                 }
                 finally {
                     sharedCtx0.database().checkpointReadUnlock();
@@ -1079,7 +1086,7 @@ public void testMetastorageUpdate() throws Exception {
                 sharedCtx0.database().checkpointReadLock();
 
                 try {
-                    storage.putData(String.valueOf(i), new byte[]{2, 2, 3, 4});
+                    storage.putData(String.valueOf(i), new byte[] {2, 2, 3, 4});
                 }
                 finally {
                     sharedCtx0.database().checkpointReadUnlock();
@@ -1120,7 +1127,7 @@ public void testMetastorageWalRestore() throws Exception {
                 sharedCtx0.database().checkpointReadLock();
 
                 try {
-                    storage.putData(String.valueOf(i), new byte[]{1, 2, 3});
+                    storage.putData(String.valueOf(i), new byte[] {1, 2, 3});
                 }
                 finally {
                     sharedCtx0.database().checkpointReadUnlock();
@@ -1155,6 +1162,74 @@ public void testMetastorageWalRestore() throws Exception {
         }
     }
 
+    /**
+     * @throws Exception if failed.
+     */
+    public void testAbsentDeadlock_Iterator_RollOver_Archivation() throws Exception {
+        try {
+            walSegments = 2;
+
+            walSegmentSize = 512 * 1024;
+
+            IgniteEx ignite0 = (IgniteEx)startGrid("node0");
+
+            ignite0.active(true);
+
+            IgniteCache<Object, Object> cache0 = ignite0.cache(cacheName);
+
+            for (int i = 0; i < 100; i++)
+                cache0.put(i, new IndexedObject(i));
+
+            GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
+
+            GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
+
+            db.waitForCheckpoint("test");
+            db.enableCheckpoints(false).get();
+
+            // Log something to know where to start.
+            WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+
+            info("Replay marker: " + ptr);
+
+            for (int i = 100; i < 200; i++)
+                cache0.put(i, new IndexedObject(i));
+
+            CountDownLatch insertFinished = new CountDownLatch(1);
+            GridTestUtils.runAsync(
+                () -> {
+                    try (WALIterator it = sharedCtx.wal().replay(ptr)) {
+                        if (it.hasNext()) {
+                            it.next();
+
+                            insertFinished.await();
+                        }
+                    }
+
+                    return null;
+                }
+            );
+
+            IgniteInternalFuture<Object> future = GridTestUtils.runAsync(
+                () -> {
+                    for (int i = 0; i < 10000; i++)
+                        cache0.put(i, new IndexedObject(i));
+
+                    return null;
+                }
+            );
+
+            future.get();
+
+            insertFinished.countDown();
+
+            ignite0.close();
+        }
+        finally {
+            stopAllGrids();
+        }
+    }
+
     /**
      * @throws Exception if failed.
      */
@@ -1303,12 +1378,12 @@ public void testRecoveryOnTransactionalAndPartitionedCache() throws Exception {
             final String cacheName = "transactional";
 
             CacheConfiguration<Object, Object> cacheConfiguration = new CacheConfiguration<>(cacheName)
-                    .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
-                    .setAffinity(new RendezvousAffinityFunction(false, 32))
-                    .setCacheMode(CacheMode.PARTITIONED)
-                    .setRebalanceMode(CacheRebalanceMode.SYNC)
-                    .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
-                    .setBackups(2);
+                .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+                .setAffinity(new RendezvousAffinityFunction(false, 32))
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setRebalanceMode(CacheRebalanceMode.SYNC)
+                .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+                .setBackups(2);
 
             ignite.createCache(cacheConfiguration);
 
@@ -1322,7 +1397,7 @@ public void testRecoveryOnTransactionalAndPartitionedCache() throws Exception {
 
             for (int t = 1; t <= transactions; t++) {
                 Transaction tx = ignite.transactions().txStart(
-                        TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
+                    TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
 
                 Map<Object, Object> changesInTransaction = new HashMap<>();
 
@@ -1385,12 +1460,12 @@ public void testTxRecordsConsistency() throws Exception {
             final String cacheName = "transactional";
 
             CacheConfiguration<Object, Object> cacheConfiguration = new CacheConfiguration<>(cacheName)
-                    .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
-                    .setAffinity(new RendezvousAffinityFunction(false, 32))
-                    .setCacheMode(CacheMode.PARTITIONED)
-                    .setRebalanceMode(CacheRebalanceMode.SYNC)
-                    .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
-                    .setBackups(0);
+                .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+                .setAffinity(new RendezvousAffinityFunction(false, 32))
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setRebalanceMode(CacheRebalanceMode.SYNC)
+                .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+                .setBackups(0);
 
             ignite.createCache(cacheConfiguration);
 
@@ -1413,7 +1488,7 @@ public void testTxRecordsConsistency() throws Exception {
 
             for (int t = 1; t <= transactions; t++) {
                 Transaction tx = ignite.transactions().txStart(
-                        TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
+                    TransactionConcurrency.OPTIMISTIC, TransactionIsolation.READ_COMMITTED);
 
                 for (int op = 0; op < operationsPerTransaction; op++) {
                     int key = random.nextInt(1000) + 1;
@@ -1448,7 +1523,7 @@ public void testTxRecordsConsistency() throws Exception {
                     WALRecord rec = tup.get2();
 
                     if (rec instanceof TxRecord) {
-                        TxRecord txRecord = (TxRecord) rec;
+                        TxRecord txRecord = (TxRecord)rec;
                         GridCacheVersion txId = txRecord.nearXidVersion();
 
                         switch (txRecord.state()) {
@@ -1471,8 +1546,9 @@ public void testTxRecordsConsistency() throws Exception {
                             default:
                                 throw new IllegalStateException("Unknown Tx state of record " + txRecord);
                         }
-                    } else if (rec instanceof DataRecord) {
-                        DataRecord dataRecord = (DataRecord) rec;
+                    }
+                    else if (rec instanceof DataRecord) {
+                        DataRecord dataRecord = (DataRecord)rec;
 
                         for (DataEntry entry : dataRecord.writeEntries()) {
                             GridCacheVersion txId = entry.nearXidVersion();
@@ -1515,16 +1591,18 @@ private static class BigObject {
             // Create pseudo-random array.
             for (int i = 0; i < payload.length; i++)
                 if (i % index == 0)
-                    payload[i] = (byte) index;
+                    payload[i] = (byte)index;
         }
 
         @Override
         public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            BigObject bigObject = (BigObject) o;
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+            BigObject bigObject = (BigObject)o;
             return index == bigObject.index &&
-                    Arrays.equals(payload, bigObject.payload);
+                Arrays.equals(payload, bigObject.payload);
         }
 
         @Override
@@ -1850,7 +1928,6 @@ private static class VerifyLargeCallable implements IgniteCallable<Boolean> {
         }
     }
 
-
     /**
      *
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
index 39513b8c7d6c7..45486a12bdf36 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -163,7 +164,7 @@ private void testApplyingUpdatesFromCompactedWal(boolean switchOffCompressor) th
         File walDir = new File(dbDir, "wal");
         File archiveDir = new File(walDir, "archive");
         File nodeArchiveDir = new File(archiveDir, nodeFolderName);
-        File walSegment = new File(nodeArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(0) + ".zip");
+        File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(0) + ".zip");
 
         assertTrue(walSegment.exists());
         assertTrue(walSegment.length() < WAL_SEGMENT_SIZE / 2); // Should be compressed at least in half.
@@ -267,7 +268,7 @@ private void testCompressorToleratesEmptyWalSegments(WALMode walMode) throws Exc
         File walDir = new File(dbDir, "wal");
         File archiveDir = new File(walDir, "archive");
         File nodeArchiveDir = new File(archiveDir, nodeFolderName);
-        File walSegment = new File(nodeArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(emptyIdx));
+        File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(emptyIdx));
 
         try (RandomAccessFile raf = new RandomAccessFile(walSegment, "rw")) {
             raf.setLength(0); // Clear wal segment, but don't delete.
@@ -357,7 +358,7 @@ public void testSeekingStartInCompactedSegment() throws Exception {
         File walDir = new File(dbDir, "wal");
         File archiveDir = new File(walDir, "archive");
         File nodeArchiveDir = new File(archiveDir, nodeFolderName);
-        File walSegment = new File(nodeArchiveDir, FileWriteAheadLogManager.FileDescriptor.fileName(0) + ".zip");
+        File walSegment = new File(nodeArchiveDir, FileDescriptor.fileName(0) + ".zip");
 
         assertTrue(walSegment.exists());
         assertTrue(walSegment.length() < WAL_SEGMENT_SIZE / 2); // Should be compressed at least in half.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java
index 866b474a1fbaa..98d89308930c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteAbstractWalIteratorInvalidCrcTest.java
@@ -41,7 +41,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.FileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
 import org.apache.ignite.internal.util.typedef.internal.U;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
index c077b27cb8ee3..59dd3b7e7dc9b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
@@ -27,7 +27,8 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleFileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 
@@ -36,7 +37,7 @@
  */
 public class IgniteDataIntegrityTests extends TestCase {
     /** File input. */
-    private FileInput fileInput;
+    private SimpleFileInput fileInput;
 
     /** Buffer expander. */
     private ByteBufferExpander expBuf;
@@ -52,7 +53,7 @@ public class IgniteDataIntegrityTests extends TestCase {
 
         FileIOFactory factory = new RandomAccessFileIOFactory();
 
-        fileInput = new FileInput(
+        fileInput = new SimpleFileInput(
                 factory.create(file),
                 expBuf
         );
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index cdad57555a382..2a9e309d53b27 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -82,7 +82,7 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
     }
 
     /** {@inheritDoc} */
-    @Override public void allowCompressionUntil(WALPointer ptr) {
+    @Override public void notchLastCheckpointPtr(WALPointer ptr) {
         // No-op.
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
new file mode 100644
index 0000000000000..82876845541ed
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
@@ -0,0 +1,601 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
+
+import java.util.concurrent.CountDownLatch;
+import junit.framework.TestCase;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.testframework.GridTestUtils;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+/**
+ * Test for {@link SegmentAware}.
+ */
+public class SegmentAwareTest extends TestCase {
+
+    /**
+     * Waiting finished when work segment is set.
+     */
+    public void testFinishAwaitSegment_WhenExactWaitingSegmentWasSet() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
+
+        //when: set exact awaiting segment.
+        aware.curAbsWalIdx(5);
+
+        //then: waiting should finish immediately
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when work segment greater than expected is set.
+     */
+    public void testFinishAwaitSegment_WhenGreaterThanWaitingSegmentWasSet() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
+
+        //when: set grater than awaiting segment.
+        aware.curAbsWalIdx(10);
+
+        //then: waiting should finish immediately
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when work segment is set.
+     */
+    public void testFinishAwaitSegment_WhenNextSegmentEqualToWaitingOne() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
+
+        //when: set less than awaiting segment.
+        aware.curAbsWalIdx(4);
+
+        //then: thread still waiting the segment
+        assertFutureIsNotFinish(future);
+
+        //when: trigger next segment.
+        aware.nextAbsoluteSegmentIndex();
+
+        //then: waiting should finish immediately
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when interrupt was triggered.
+     */
+    public void testFinishAwaitSegment_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
+
+        //when: interrupt waiting.
+        aware.interrupt();
+
+        //then: IgniteInterruptedCheckedException should be throw.
+        assertTrue(future.get(20) instanceof IgniteInterruptedCheckedException);
+    }
+
+    /**
+     * Waiting finished when next work segment triggered.
+     */
+    public void testFinishWaitSegmentForArchive_WhenWorkSegmentIncremented() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.curAbsWalIdx(5);
+        aware.setLastArchivedAbsoluteIndex(4);
+
+        IgniteInternalFuture future = awaitThread(aware::waitNextSegmentForArchivation);
+
+        //when: next work segment triggered.
+        aware.nextAbsoluteSegmentIndex();
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when work segment is set.
+     */
+    public void testFinishWaitSegmentForArchive_WhenWorkSegmentGreaterValue() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.curAbsWalIdx(5);
+        aware.setLastArchivedAbsoluteIndex(4);
+
+        IgniteInternalFuture future = awaitThread(aware::waitNextSegmentForArchivation);
+
+        //when: set work segment greater than required.
+        aware.curAbsWalIdx(7);
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when interrupt was triggered.
+     */
+    public void testFinishWaitSegmentForArchive_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.curAbsWalIdx(5);
+        aware.setLastArchivedAbsoluteIndex(4);
+
+        IgniteInternalFuture future = awaitThread(aware::waitNextSegmentForArchivation);
+
+        //when: interrupt waiting.
+        aware.interrupt();
+
+        //then: IgniteInterruptedCheckedException should be throw.
+        assertTrue(future.get(20) instanceof IgniteInterruptedCheckedException);
+    }
+
+    /**
+     * Should correct calculate next segment.
+     */
+    public void testCorrectCalculateNextSegmentIndex() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.curAbsWalIdx(5);
+
+        //when: request next work segment.
+        long segmentIndex = aware.nextAbsoluteSegmentIndex();
+
+        //then:
+        assertThat(segmentIndex, is(6L));
+    }
+
+    /**
+     * Waiting finished when segment archived.
+     */
+    public void testFinishWaitNextAbsoluteIndex_WhenMarkAsArchivedFirstSegment() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(2);
+
+        aware.curAbsWalIdx(1);
+        aware.setLastArchivedAbsoluteIndex(-1);
+
+        IgniteInternalFuture future = awaitThread(aware::nextAbsoluteSegmentIndex);
+
+        //when: mark first segment as moved.
+        aware.markAsMovedToArchive(0);
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when segment archived.
+     */
+    public void testFinishWaitNextAbsoluteIndex_WhenSetToArchivedFirst() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(2);
+
+        aware.curAbsWalIdx(1);
+        aware.setLastArchivedAbsoluteIndex(-1);
+
+        IgniteInternalFuture future = awaitThread(aware::nextAbsoluteSegmentIndex);
+
+        //when: mark first segment as moved.
+        aware.setLastArchivedAbsoluteIndex(0);
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when force interrupt was triggered.
+     */
+    public void testFinishWaitNextAbsoluteIndex_WhenOnlyForceInterruptWasCall() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(2);
+
+        aware.curAbsWalIdx(2);
+        aware.setLastArchivedAbsoluteIndex(-1);
+
+        IgniteInternalFuture future = awaitThread(aware::nextAbsoluteSegmentIndex);
+
+        //when: interrupt waiting.
+        aware.interrupt();
+
+        //then: nothing to happen because nextAbsoluteSegmentIndex is not interrupt by "interrupt" call.
+        assertFutureIsNotFinish(future);
+
+        //when: force interrupt waiting.
+        aware.forceInterrupt();
+
+        //then: IgniteInterruptedCheckedException should be throw.
+        assertTrue(future.get(20) instanceof IgniteInterruptedCheckedException);
+    }
+
+    /**
+     * Waiting finished when segment archived.
+     */
+    public void testFinishSegmentArchived_WhenSetExactWaitingSegment() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
+
+        //when: archived exact expected segment.
+        aware.setLastArchivedAbsoluteIndex(5);
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when segment archived.
+     */
+    public void testFinishSegmentArchived_WhenMarkExactWatingSegment() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
+
+        //when: mark exact segment as moved.
+        aware.markAsMovedToArchive(5);
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when segment archived.
+     */
+    public void testFinishSegmentArchived_WhenSetGreaterThanWatingSegment() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
+
+        //when: archived greater than expected segment.
+        aware.setLastArchivedAbsoluteIndex(7);
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when segment archived.
+     */
+    public void testFinishSegmentArchived_WhenMarkGreaterThanWatingSegment() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
+
+        //when: moved greater than expected segment.
+        aware.markAsMovedToArchive(7);
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when interrupt was triggered.
+     */
+    public void testFinishSegmentArchived_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.curAbsWalIdx(5);
+        aware.setLastArchivedAbsoluteIndex(4);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
+
+        //when: interrupt waiting.
+        aware.interrupt();
+
+        //then: IgniteInterruptedCheckedException should be throw.
+        assertTrue(future.get(20) instanceof IgniteInterruptedCheckedException);
+    }
+
+    /**
+     * Waiting finished when release work segment.
+     */
+    public void testMarkAsMovedToArchive_WhenReleaseLockedSegment() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.checkCanReadArchiveOrReserveWorkSegment(5);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.markAsMovedToArchive(5));
+
+        //when: release exact expected work segment.
+        aware.releaseWorkSegment(5);
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished and increment archived segment when interrupt was call.
+     */
+    public void testMarkAsMovedToArchive_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+        aware.checkCanReadArchiveOrReserveWorkSegment(5);
+
+        IgniteInternalFuture future = awaitThread(() -> aware.markAsMovedToArchive(5));
+
+        //when: interrupt waiting.
+        aware.interrupt();
+
+        //then: IgniteInterruptedCheckedException should be throw.
+        assertFalse(future.get(20) instanceof IgniteInterruptedCheckedException);
+
+        //and: last archived segment should be changed.
+        assertEquals(5, aware.lastArchivedAbsoluteIndex());
+    }
+
+    /**
+     * Waiting finished when segment archived.
+     */
+    public void testFinishWaitSegmentToCompress_WhenSetLastArchivedSegment() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.lastCompressedIdx(5);
+
+        IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress);
+
+        //when: archived expected segment.
+        aware.setLastArchivedAbsoluteIndex(6);
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Waiting finished when segment archived.
+     */
+    public void testFinishWaitSegmentToCompress_WhenMarkLastArchivedSegment() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.lastCompressedIdx(5);
+
+        IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress);
+
+        //when: marked expected segment.
+        aware.markAsMovedToArchive(6);
+
+        //then: waiting should finish immediately.
+        future.get(20);
+    }
+
+    /**
+     * Next segment for compress based on truncated archive idx.
+     */
+    public void testCorrectCalculateNextCompressSegment() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.lastCompressedIdx(5);
+        aware.setLastArchivedAbsoluteIndex(6);
+        aware.lastTruncatedArchiveIdx(7);
+
+        //when:
+        long segmentToCompress = aware.waitNextSegmentToCompress();
+
+        //then: segment to compress greater than truncated archive idx
+        assertEquals(8, segmentToCompress);
+    }
+
+    /**
+     * Waiting finished when interrupt was call.
+     */
+    public void testFinishWaitSegmentToCompress_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+        aware.lastCompressedIdx(5);
+
+        IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress);
+
+        //when: interrupt waiting.
+        aware.interrupt();
+
+        //then: IgniteInterruptedCheckedException should be throw.
+        assertTrue(future.get(20) instanceof IgniteInterruptedCheckedException);
+    }
+
+    /**
+     * Segment reserve correctly.
+     */
+    public void testReserveCorrectly() {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        //when: reserve one segment twice and one segment once.
+        aware.reserve(5);
+        aware.reserve(5);
+        aware.reserve(7);
+
+        //then: segments greater than minimum should be reserved.
+        assertTrue(aware.reserved(5));
+        assertTrue(aware.reserved(10));
+        assertFalse(aware.reserved(4));
+
+        //when: release one of twice locked segment.
+        aware.release(5);
+
+        //then: nothing to change.
+        assertTrue(aware.reserved(5));
+        assertTrue(aware.reserved(10));
+        assertFalse(aware.reserved(4));
+
+        //when: again release one of twice locked segment.
+        aware.release(5);
+
+        //then: segments greater than second locked segment should be reserved.
+        assertTrue(aware.reserved(7));
+        assertTrue(aware.reserved(10));
+        assertFalse(aware.reserved(5));
+        assertFalse(aware.reserved(6));
+
+        //when: release last segment.
+        aware.release(7);
+
+        //then: all segments should be released.
+        assertFalse(aware.reserved(7));
+        assertFalse(aware.reserved(10));
+        assertFalse(aware.reserved(6));
+    }
+
+    /**
+     * Should fail when release unreserved segment.
+     */
+    public void testAssertFail_WhenReleaseUnreservedSegment() {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.reserve(5);
+        try {
+
+            aware.release(7);
+        }
+        catch (AssertionError e) {
+            return;
+        }
+
+        fail("Should fail with AssertError because this segment have not reserved");
+    }
+
+    /**
+     * Segment locked correctly.
+     */
+    public void testReserveWorkSegmentCorrectly() {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        //when: lock one segment twice.
+        aware.checkCanReadArchiveOrReserveWorkSegment(5);
+        aware.checkCanReadArchiveOrReserveWorkSegment(5);
+
+        //then: exact one segment should locked.
+        assertTrue(aware.locked(5));
+        assertFalse(aware.locked(6));
+        assertFalse(aware.locked(4));
+
+        //when: release segment once.
+        aware.releaseWorkSegment(5);
+
+        //then: nothing to change, segment still locked.
+        assertTrue(aware.locked(5));
+        assertFalse(aware.locked(6));
+        assertFalse(aware.locked(4));
+
+        //when: release segment.
+        aware.releaseWorkSegment(5);
+
+        //then: all segments should be unlocked.
+        assertFalse(aware.locked(5));
+        assertFalse(aware.locked(6));
+        assertFalse(aware.locked(4));
+    }
+
+    /**
+     * Should fail when release unlocked segment.
+     */
+    public void testAssertFail_WhenReleaseUnreservedWorkSegment() {
+        //given: thread which awaited segment.
+        SegmentAware aware = new SegmentAware(10);
+
+        aware.checkCanReadArchiveOrReserveWorkSegment(5);
+        try {
+
+            aware.releaseWorkSegment(7);
+        }
+        catch (AssertionError e) {
+            return;
+        }
+
+        fail("Should fail with AssertError because this segment have not reserved");
+    }
+
+    /**
+     * Assert that future is still not finished.
+     *
+     * @param future Future to check.
+     */
+    private void assertFutureIsNotFinish(IgniteInternalFuture future) throws IgniteCheckedException {
+        try {
+            future.get(20);
+
+            fail("Timeout should be appear because thread should be still work");
+        }
+        catch (IgniteFutureTimeoutCheckedException ignore) {
+
+        }
+    }
+
+    /**
+     * Create thread for execute waiter.
+     *
+     * @param waiter Waiter for execute in new thread.
+     * @return Future of thread.
+     */
+    private IgniteInternalFuture awaitThread(Waiter waiter) throws IgniteCheckedException, InterruptedException {
+        CountDownLatch latch = new CountDownLatch(1);
+        IgniteInternalFuture<Object> future = GridTestUtils.runAsync(
+            () -> {
+                latch.countDown();
+                try {
+                    waiter.await();
+                }
+                catch (IgniteInterruptedCheckedException e) {
+                    return e;
+                }
+
+                return null;
+            }
+        );
+
+        latch.await();
+
+        assertFutureIsNotFinish(future);
+
+        return future;
+    }
+
+    /**
+     * Represent of command for waiting.
+     */
+    interface Waiter {
+        /**
+         * Some waiting operation.
+         */
+        void await() throws IgniteInterruptedCheckedException;
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
index b6a04d0a1488e..cf660c8b86832 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
@@ -123,7 +123,7 @@ public void testCorrectClosingFileDescriptors() throws Exception {
 
         assertTrue("At least one WAL file must be opened!", CountedFileIO.getCountOpenedWalFiles() > 0);
 
-        assertEquals("All WAL files must be closed!", CountedFileIO.getCountOpenedWalFiles(), CountedFileIO.getCountClosedWalFiles());
+        assertTrue("All WAL files must be closed at least ones!", CountedFileIO.getCountOpenedWalFiles() <= CountedFileIO.getCountClosedWalFiles());
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 9d99158a57d8b..0fb9efef1ac0a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -1274,25 +1274,7 @@ public static <T> T getFieldValue(Object obj, Class cls, String fieldName) throw
         assert fieldName != null;
 
         try {
-            // Resolve inner field.
-            Field field = cls.getDeclaredField(fieldName);
-
-            synchronized (field) {
-                // Backup accessible field state.
-                boolean accessible = field.isAccessible();
-
-                try {
-                    if (!accessible)
-                        field.setAccessible(true);
-
-                    obj = field.get(obj);
-                }
-                finally {
-                    // Recover accessible field state.
-                    if (!accessible)
-                        field.setAccessible(false);
-                }
-            }
+            obj = findField(cls, obj, fieldName);
 
             return (T)obj;
         }
@@ -1322,25 +1304,7 @@ public static <T> T getFieldValue(Object obj, String... fieldNames) throws Ignit
                 Class<?> cls = obj instanceof Class ? (Class)obj : obj.getClass();
 
                 try {
-                    // Resolve inner field.
-                    Field field = cls.getDeclaredField(fieldName);
-
-                    synchronized (field) {
-                        // Backup accessible field state.
-                        boolean accessible = field.isAccessible();
-
-                        try {
-                            if (!accessible)
-                                field.setAccessible(true);
-
-                            obj = field.get(obj);
-                        }
-                        finally {
-                            // Recover accessible field state.
-                            if (!accessible)
-                                field.setAccessible(false);
-                        }
-                    }
+                    obj = findField(cls, obj, fieldName);
                 }
                 catch (NoSuchFieldException e) {
                     // Resolve inner class, if not an inner field.
@@ -1362,6 +1326,74 @@ public static <T> T getFieldValue(Object obj, String... fieldNames) throws Ignit
         }
     }
 
+    /**
+     * Get object field value via reflection(including superclass).
+     *
+     * @param obj Object or class to get field value from.
+     * @param fieldNames Field names to get value for: obj->field1->field2->...->fieldN.
+     * @param <T> Expected field class.
+     * @return Field value.
+     * @throws IgniteException In case of error.
+     */
+    @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+    public static <T> T getFieldValueHierarchy(Object obj, String... fieldNames) throws IgniteException {
+        assert obj != null;
+        assert fieldNames != null;
+        assert fieldNames.length >= 1;
+
+        try {
+            for (String fieldName : fieldNames) {
+                Class<?> cls = obj instanceof Class ? (Class)obj : obj.getClass();
+
+                while (cls != null) {
+                    try {
+                        obj = findField(cls, obj, fieldName);
+
+                        break;
+                    }
+                    catch (NoSuchFieldException e) {
+                        cls = cls.getSuperclass();
+                    }
+                }
+            }
+
+            return (T)obj;
+        }
+        catch (IllegalAccessException e) {
+            throw new IgniteException("Failed to get object field [obj=" + obj +
+                ", fieldNames=" + Arrays.toString(fieldNames) + ']', e);
+        }
+    }
+
+    /**
+     * @param cls Class for searching.
+     * @param obj Target object.
+     * @param fieldName Field name for search.
+     * @return Field from object if it was found.
+     */
+    private static Object findField(Class<?> cls, Object obj,
+        String fieldName) throws NoSuchFieldException, IllegalAccessException {
+        // Resolve inner field.
+        Field field = cls.getDeclaredField(fieldName);
+
+        synchronized (field) {
+            // Backup accessible field state.
+            boolean accessible = field.isAccessible();
+
+            try {
+                if (!accessible)
+                    field.setAccessible(true);
+
+                return field.get(obj);
+            }
+            finally {
+                // Recover accessible field state.
+                if (!accessible)
+                    field.setAccessible(false);
+            }
+        }
+    }
+
     /**
      * Get inner class by its name from the enclosing class.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index fb262ce055cb5..3a421ebdca219 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -19,10 +19,10 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistence;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheConfigurationFileConsistencyCheckTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDestroyCacheWithoutCheckpointsTest;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheConfigurationFileConsistencyCheckTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsCacheRestoreTest;
@@ -39,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteThrottleSmokeTest;
 import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBufferTest;
+import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAwareTest;
 import org.apache.ignite.internal.processors.database.IgniteDbDynamicCacheSelfTest;
 import org.apache.ignite.internal.processors.database.IgniteDbMultiNodePutGetTest;
 import org.apache.ignite.internal.processors.database.IgniteDbPutGetWithCacheStoreTest;
@@ -79,6 +80,7 @@ public static TestSuite suite() throws Exception {
         // Binary meta tests.
         suite.addTestSuite(IgnitePdsCacheObjectBinaryProcessorOnDiscoveryTest.class);
 
+        suite.addTestSuite(SegmentAwareTest.class);
 
         return suite;
     }

From 23b17ab959c21a8a31e7850ddfd7a4e0c7c6543b Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Wed, 26 Sep 2018 16:55:06 +0300
Subject: [PATCH 380/543] IGNITE-9381 Clear class cache on p2p undeploy - Fixes
 #4784.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit d40dcb3a8f25a5bb098662fa7270b69eb3cc3ab5)
---
 .../GridDeploymentPerVersionStore.java        |   6 +
 .../cache/CacheClassLoaderMarker.java         |  24 ++
 .../cache/GridCacheDeploymentManager.java     |   2 +-
 .../ignite/internal/util/IgniteUtils.java     |  22 +-
 .../ignite/p2p/P2PScanQueryUndeployTest.java  | 250 ++++++++++++++++++
 .../testsuites/IgniteP2PSelfTestSuite.java    |   2 +
 .../ignite/tests/p2p/AlwaysTruePredicate.java |  30 +++
 7 files changed, 334 insertions(+), 2 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClassLoaderMarker.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/p2p/P2PScanQueryUndeployTest.java
 create mode 100644 modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/AlwaysTruePredicate.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentPerVersionStore.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentPerVersionStore.java
index 14aa49f60a05b..56a3f3e026fb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentPerVersionStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentPerVersionStore.java
@@ -1290,6 +1290,12 @@ void recordUndeployed(@Nullable UUID leftNodeId) {
 
                 ctx.cache().onUndeployed(ldr);
 
+               // Clear static class cache.
+                U.clearClassFromClassCache(ctx.cache().context().deploy().globalLoader(), sampleClassName());
+
+                for (String alias : deployedClassMap().keySet())
+                    U.clearClassFromClassCache(ctx.cache().context().deploy().globalLoader(), alias);
+
                 // Clear optimized marshaller's cache.
                 if (ctx.config().getMarshaller() instanceof AbstractMarshaller)
                     ((AbstractMarshaller)ctx.config().getMarshaller()).onUndeploy(ldr);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClassLoaderMarker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClassLoaderMarker.java
new file mode 100644
index 0000000000000..b575ec797ed4c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheClassLoaderMarker.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache;
+
+/**
+ * It's a marker interface for detecting GridCacheDeploymentManager$CacheClassLoader.
+ */
+public interface CacheClassLoaderMarker {
+    // Marker interface.
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java
index b70439a7d08ee..b34da627bd7d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java
@@ -760,7 +760,7 @@ public boolean isGlobalLoader() {
     /**
      * Cache class loader.
      */
-    private class CacheClassLoader extends ClassLoader {
+    private class CacheClassLoader extends ClassLoader implements CacheClassLoaderMarker {
         /** */
         private final String[] p2pExclude;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 508e4ef6216ce..7c14bccec9dac 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -193,6 +193,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoManager;
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
 import org.apache.ignite.internal.mxbean.IgniteStandardMXBean;
+import org.apache.ignite.internal.processors.cache.CacheClassLoaderMarker;
 import org.apache.ignite.internal.processors.cache.GridCacheAttributes;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
@@ -8629,7 +8630,13 @@ public static Class<?> forName(String clsName, @Nullable ClassLoader ldr, Ignite
             if (clsFilter != null && !clsFilter.apply(clsName))
                 throw new RuntimeException("Deserialization of class " + clsName + " is disallowed.");
 
-            Class old = ldrMap.putIfAbsent(clsName, cls = Class.forName(clsName, true, ldr));
+            // Avoid class caching inside Class.forName
+            if (ldr instanceof CacheClassLoaderMarker)
+                cls = ldr.loadClass(clsName);
+            else
+                cls = Class.forName(clsName, true, ldr);
+
+            Class old = ldrMap.putIfAbsent(clsName, cls);
 
             if (old != null)
                 cls = old;
@@ -8638,6 +8645,19 @@ public static Class<?> forName(String clsName, @Nullable ClassLoader ldr, Ignite
         return cls;
     }
 
+    /**
+     * Clears class associated with provided class loader from class cache.
+     *
+     * @param ldr Class loader.
+     * @param clsName Class name of clearing class.
+     */
+    public static void clearClassFromClassCache(ClassLoader ldr, String clsName){
+        ConcurrentMap<String, Class> map = classCache.get(ldr);
+
+        if (map!=null)
+            map.remove(clsName);
+    }
+
     /**
      * Clears class cache for provided loader.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/p2p/P2PScanQueryUndeployTest.java b/modules/core/src/test/java/org/apache/ignite/p2p/P2PScanQueryUndeployTest.java
new file mode 100644
index 0000000000000..73ce917d12cd6
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/p2p/P2PScanQueryUndeployTest.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.p2p;
+
+import java.lang.reflect.Field;
+import java.net.URL;
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.query.ScanQuery;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.managers.deployment.GridDeploymentRequest;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestExternalClassLoader;
+import org.apache.ignite.testframework.config.GridTestProperties;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/** */
+public class P2PScanQueryUndeployTest extends GridCommonAbstractTest {
+    /** Predicate classname. */
+    private static final String PREDICATE_CLASSNAME = "org.apache.ignite.tests.p2p.AlwaysTruePredicate";
+
+    /** */
+    private static final String TEST_PREDICATE_RESOURCE_NAME = U.classNameToResourceName(PREDICATE_CLASSNAME);
+
+    /** Cache name. */
+    private static final String CACHE_NAME = "test-cache";
+
+    /** Client instance name. */
+    private static final String CLIENT_INSTANCE_NAME = "client";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setPeerClassLoadingEnabled(true);
+
+        cfg.setCacheConfiguration(
+            new CacheConfiguration()
+                .setName(CACHE_NAME)
+                .setBackups(1)
+        );
+
+        cfg.setDiscoverySpi(
+            new TcpDiscoverySpi()
+                .setIpFinder(
+                    new TcpDiscoveryVmIpFinder(true)
+                        .setAddresses(Collections.singletonList("127.0.0.1:47500..47509"))
+                )
+        );
+
+        cfg.setCommunicationSpi(new MessageCountingCommunicationSpi());
+
+        if (igniteInstanceName.equals(CLIENT_INSTANCE_NAME))
+            cfg.setClientMode(true);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        stopAllGrids();
+
+        MessageCountingCommunicationSpi.resetDeploymentRequestCounter();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /**
+     * Checks, that after client's reconnect to cluster will be redeployed from client node.
+     *
+     * @throws Exception if test failed.
+     */
+    public void testAfterClientDisconnect() throws Exception {
+        ClassLoader extClsLdr = new GridTestExternalClassLoader(new URL[] {new URL(GridTestProperties.getProperty("p2p.uri.cls"))});
+
+        assertFalse(classFound(getClass().getClassLoader(), PREDICATE_CLASSNAME));
+
+        Class predCls = extClsLdr.loadClass(PREDICATE_CLASSNAME);
+
+        startGrid(0);
+
+        Ignite client = startGrid(CLIENT_INSTANCE_NAME);
+
+        client.cluster().active(true);
+
+        awaitPartitionMapExchange();
+
+        IgniteCache<Integer, String> cache = client.getOrCreateCache(CACHE_NAME);
+
+        cache.put(1, "foo");
+
+        invokeScanQueryAndStopClient(client, predCls);
+
+        MessageCountingCommunicationSpi.resetDeploymentRequestCounter();
+
+        client = startGrid(CLIENT_INSTANCE_NAME);
+
+        invokeScanQueryAndStopClient(client, predCls);
+    }
+
+    /**
+     * @param client ignite instance.
+     * @param predCls loaded predicate class.
+     * @throws Exception if failed.
+     */
+    private void invokeScanQueryAndStopClient(Ignite client, Class predCls) throws Exception {
+        IgniteCache<Integer, String> cache = client.getOrCreateCache(CACHE_NAME);
+
+        assertEquals("Invalid number of sent grid deployment requests", 0, MessageCountingCommunicationSpi.deploymentRequestCount());
+
+        assertFalse(PREDICATE_CLASSNAME + " mustn't be cached! ", igniteUtilsCachedClasses().contains(PREDICATE_CLASSNAME));
+
+        cache.query(new ScanQuery<>((IgniteBiPredicate<Integer, String>)predCls.newInstance())).getAll();
+
+        // first request is GridDeployment.java 716 and second is GridDeployment.java 501
+        assertEquals("Invalid number of sent grid deployment requests", 2, MessageCountingCommunicationSpi.deploymentRequestCount());
+
+        assertTrue(PREDICATE_CLASSNAME + " must be cached! ", igniteUtilsCachedClasses().contains(PREDICATE_CLASSNAME));
+
+        client.close();
+
+        assertFalse(PREDICATE_CLASSNAME + " mustn't be cached! ", igniteUtilsCachedClasses().contains(PREDICATE_CLASSNAME));
+    }
+
+    /**
+     * @return All class names cached in IgniteUtils.classCache field
+     * @throws Exception if something wrong.
+     */
+    private Set<String> igniteUtilsCachedClasses() throws Exception {
+        Field f = IgniteUtils.class.getDeclaredField("classCache");
+
+        f.setAccessible(true);
+
+        ConcurrentMap<ClassLoader, ConcurrentMap<String, Class>> map =
+            (ConcurrentMap<ClassLoader, ConcurrentMap<String, Class>>)f.get(null);
+
+        return map.values().stream().flatMap(x -> x.keySet().stream()).collect(Collectors.toSet());
+    }
+
+    /**
+     * @param clsLdr classloader.
+     * @param name classname.
+     * @return true if class loaded by classloader, false if class not found.
+     */
+    private boolean classFound(ClassLoader clsLdr, String name) {
+        try {
+            clsLdr.loadClass(name);
+
+            return true;
+        }
+        catch (ClassNotFoundException e) {
+            return false;
+        }
+    }
+
+    /** */
+    private static class MessageCountingCommunicationSpi extends TcpCommunicationSpi {
+        /** */
+        private static final AtomicInteger reqCnt = new AtomicInteger();
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node,
+            Message msg,
+            IgniteInClosure<IgniteException> ackClosure
+        ) throws IgniteSpiException {
+            if (msg instanceof GridIoMessage && isDeploymentRequestMessage((GridIoMessage)msg))
+                reqCnt.incrementAndGet();
+
+            super.sendMessage(node, msg, ackClosure);
+        }
+
+        /**
+         * @return Number of deployment requests.
+         */
+        public static int deploymentRequestCount() {
+            return reqCnt.get();
+        }
+
+        /**
+         * Reset request counter.
+         */
+        public static void resetDeploymentRequestCounter() {
+            reqCnt.set(0);
+        }
+
+        /**
+         * Checks if it is a p2p deployment request message with test predicate resource.
+         *
+         * @param msg Message to check.
+         * @return {@code True} if this is a p2p message.
+         */
+        private boolean isDeploymentRequestMessage(GridIoMessage msg) {
+            try {
+                if (msg.message() instanceof GridDeploymentRequest) {
+                    // rsrcName getter have only default access level. So, use reflection for access to field value.
+                    GridDeploymentRequest req = (GridDeploymentRequest)msg.message();
+
+                    Field f = GridDeploymentRequest.class.getDeclaredField("rsrcName");
+
+                    f.setAccessible(true);
+
+                    return f.get(req).equals(TEST_PREDICATE_RESOURCE_NAME);
+                }
+            }
+            catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+
+            return false;
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java
index 524283bdbfcc4..1979dbdcd0551 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteP2PSelfTestSuite.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.p2p.GridP2PSameClassLoaderSelfTest;
 import org.apache.ignite.p2p.GridP2PTimeoutSelfTest;
 import org.apache.ignite.p2p.GridP2PUndeploySelfTest;
+import org.apache.ignite.p2p.P2PScanQueryUndeployTest;
 import org.apache.ignite.p2p.P2PStreamingClassLoaderTest;
 import org.apache.ignite.p2p.SharedDeploymentTest;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -76,6 +77,7 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTest(new TestSuite(DeploymentClassLoaderCallableTest.class));
         suite.addTest(new TestSuite(P2PStreamingClassLoaderTest.class));
         suite.addTest(new TestSuite(SharedDeploymentTest.class));
+        suite.addTest(new TestSuite(P2PScanQueryUndeployTest.class));
         GridTestUtils.addTestIfNeeded(suite, GridDeploymentMessageCountSelfTest.class, ignoredTests);
 
         return suite;
diff --git a/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/AlwaysTruePredicate.java b/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/AlwaysTruePredicate.java
new file mode 100644
index 0000000000000..3918f7197767c
--- /dev/null
+++ b/modules/extdata/p2p/src/main/java/org/apache/ignite/tests/p2p/AlwaysTruePredicate.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.tests.p2p;
+
+import org.apache.ignite.lang.IgniteBiPredicate;
+
+/**
+ *
+ */
+public class AlwaysTruePredicate implements IgniteBiPredicate<Object, Object> {
+    /** */
+    @Override public boolean apply(Object k, Object v) {
+        return new CacheDeploymentAlwaysTruePredicate().apply(k,v);
+    }
+}
\ No newline at end of file

From 352c4081471df9ae062e6987e7a26cb80ac8b779 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 26 Sep 2018 18:12:06 +0300
Subject: [PATCH 381/543] IGNITE-9501 Exclude newly joined nodes from exchange
 latch. Fixes #4769.

(cherry picked from commit 77b90dfbf8a6f73ba8c4bfaa2c036eb9f1538a92)
---
 .../GridDhtPartitionsExchangeFuture.java      | 75 ++++++++------
 .../preloader/latch/ExchangeLatchManager.java | 99 +++++++++++++++----
 ...changeLatchManagerCoordinatorFailTest.java | 34 +++++--
 3 files changed, 152 insertions(+), 56 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index d3789011e4802..ad0a9fed88c01 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1171,18 +1171,30 @@ private void distributedExchange() throws IgniteCheckedException {
         // To correctly rebalance when persistence is enabled, it is necessary to reserve history within exchange.
         partHistReserved = cctx.database().reserveHistoryForExchange();
 
-        boolean distributed = true;
+        // Skipping wait on local join is available when all cluster nodes have the same protocol.
+        boolean skipWaitOnLocalJoin = cctx.exchange().latch().canSkipJoiningNodes(initialVersion())
+            && localJoinExchange();
 
-        // Do not perform distributed partition release in case of cluster activation.
-        if (activateCluster())
-            distributed = false;
+        // Skip partition release if node has locally joined (it doesn't have any updates to be finished).
+        if (!skipWaitOnLocalJoin) {
+            boolean distributed = true;
 
-        // On first phase we wait for finishing all local tx updates, atomic updates and lock releases on all nodes.
-        waitPartitionRelease(distributed, true);
+            // Do not perform distributed partition release in case of cluster activation.
+            if (activateCluster())
+                distributed = false;
 
-        // Second phase is needed to wait for finishing all tx updates from primary to backup nodes remaining after first phase.
-        if (distributed)
-            waitPartitionRelease(false, false);
+            // On first phase we wait for finishing all local tx updates, atomic updates and lock releases on all nodes.
+            waitPartitionRelease(distributed, true);
+
+            // Second phase is needed to wait for finishing all tx updates from primary to backup nodes remaining after first phase.
+            if (distributed)
+                waitPartitionRelease(false, false);
+        }
+        else {
+            if (log.isInfoEnabled())
+                log.info("Skipped waiting for partitions release future (local node is joining) " +
+                    "[topVer=" + initialVersion() + "]");
+        }
 
         boolean topChanged = firstDiscoEvt.type() != EVT_DISCOVERY_CUSTOM_EVT || affChangeMsg != null;
 
@@ -1368,9 +1380,11 @@ private void waitPartitionRelease(boolean distributed, boolean doRollback) throw
             String futInfo = RELEASE_FUTURE_DUMP_THRESHOLD > 0 && waitTime > RELEASE_FUTURE_DUMP_THRESHOLD ?
                 partReleaseFut.toString() : "NA";
 
+            String mode = distributed ? "DISTRIBUTED" : "LOCAL";
+
             if (log.isInfoEnabled())
                 log.info("Finished waiting for partition release future [topVer=" + exchangeId().topologyVersion() +
-                    ", waitTime=" + (waitEnd - waitStart) + "ms, futInfo=" + futInfo + "]");
+                    ", waitTime=" + (waitEnd - waitStart) + "ms, futInfo=" + futInfo + ", mode=" + mode + "]");
         }
 
         IgniteInternalFuture<?> locksFut = cctx.mvcc().finishLocks(exchId.topologyVersion());
@@ -1411,33 +1425,38 @@ private void waitPartitionRelease(boolean distributed, boolean doRollback) throw
             }
         }
 
-        if (releaseLatch == null)
+        if (releaseLatch == null) {
+            assert !distributed : "Partitions release latch must be initialized in distributed mode.";
+
             return;
+        }
 
         releaseLatch.countDown();
 
-        if (!localJoinExchange()) {
-            try {
-                while (true) {
-                    try {
-                        releaseLatch.await(waitTimeout, TimeUnit.MILLISECONDS);
+        // For compatibility with old version where joining nodes are not waiting for latch.
+        if (!cctx.exchange().latch().canSkipJoiningNodes(initialVersion()))
+            return;
 
-                        if (log.isInfoEnabled())
-                            log.info("Finished waiting for partitions release latch: " + releaseLatch);
+        try {
+            while (true) {
+                try {
+                    releaseLatch.await(waitTimeout, TimeUnit.MILLISECONDS);
 
-                        break;
-                    }
-                    catch (IgniteFutureTimeoutCheckedException ignored) {
-                        U.warn(log, "Unable to await partitions release latch within timeout: " + releaseLatch);
+                    if (log.isInfoEnabled())
+                        log.info("Finished waiting for partitions release latch: " + releaseLatch);
 
-                        // Try to resend ack.
-                        releaseLatch.countDown();
-                    }
+                    break;
+                }
+                catch (IgniteFutureTimeoutCheckedException ignored) {
+                    U.warn(log, "Unable to await partitions release latch within timeout: " + releaseLatch);
+
+                    // Try to resend ack.
+                    releaseLatch.countDown();
                 }
             }
-            catch (IgniteCheckedException e) {
-                U.warn(log, "Stop waiting for partitions release latch: " + e.getMessage());
-            }
+        }
+        catch (IgniteCheckedException e) {
+            U.warn(log, "Stop waiting for partitions release latch: " + e.getMessage());
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index 57305fbbe4c0c..b9d01271aef2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -19,6 +19,7 @@
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -38,7 +39,6 @@
 import org.apache.ignite.internal.GridTopic;
 import org.apache.ignite.internal.managers.communication.GridIoManager;
 import org.apache.ignite.internal.managers.communication.GridIoPolicy;
-import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.GridConcurrentHashSet;
@@ -46,12 +46,12 @@
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_JOINED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 
 /**
@@ -61,6 +61,12 @@ public class ExchangeLatchManager {
     /** Version since latch management is available. */
     private static final IgniteProductVersion VERSION_SINCE = IgniteProductVersion.fromString("2.5.0");
 
+    /**
+     * Exchange latch V2 protocol introduces following optimization:
+     * Joining nodes are explicitly excluded from possible latch participants.
+     */
+    public static final IgniteProductVersion PROTOCOL_V2_VERSION_SINCE = IgniteProductVersion.fromString("2.5.1-p14");
+
     /** Logger. */
     private final IgniteLogger log;
 
@@ -90,6 +96,10 @@ public class ExchangeLatchManager {
     @GridToStringInclude
     private final ConcurrentMap<CompletableLatchUid, ClientLatch> clientLatches = new ConcurrentHashMap<>();
 
+    /** Map (topology version -> joined node on this version). This map is needed to exclude joined nodes from latch participants. */
+    @GridToStringExclude
+    private final ConcurrentMap<AffinityTopologyVersion, ClusterNode> joinedNodes = new ConcurrentHashMap<>();
+
     /** Lock. */
     private final ReentrantLock lock = new ReentrantLock();
 
@@ -104,7 +114,7 @@ public ExchangeLatchManager(GridKernalContext ctx) {
         this.discovery = ctx.discovery();
         this.io = ctx.io();
 
-        if (!ctx.clientNode()) {
+        if (!ctx.clientNode() && !ctx.isDaemon()) {
             ctx.io().addMessageListener(GridTopic.TOPIC_EXCHANGE, (nodeId, msg, plc) -> {
                 if (msg instanceof LatchAckMessage)
                     processAck(nodeId, (LatchAckMessage) msg);
@@ -121,8 +131,16 @@ public ExchangeLatchManager(GridKernalContext ctx) {
                 assert e.type() == EVT_NODE_LEFT || e.type() == EVT_NODE_FAILED : this;
 
                 // Do not process from discovery thread.
-                ctx.closure().runLocalSafe(() -> processNodeLeft(e.eventNode()));
+                // TODO: Should use queue to guarantee the order of processing left nodes.
+                ctx.closure().runLocalSafe(() -> processNodeLeft(cache.version(), e.eventNode()));
             }, EVT_NODE_LEFT, EVT_NODE_FAILED);
+
+            ctx.event().addDiscoveryEventListener((e, cache) -> {
+                assert e != null;
+                assert e.type() == EVT_NODE_JOINED;
+
+                joinedNodes.put(cache.version(), e.eventNode());
+            }, EVT_NODE_JOINED);
         }
     }
 
@@ -249,10 +267,30 @@ private Collection<ClusterNode> aliveNodesForTopologyVer(AffinityTopologyVersion
     private Collection<ClusterNode> getLatchParticipants(AffinityTopologyVersion topVer) {
         Collection<ClusterNode> aliveNodes = aliveNodesForTopologyVer(topVer);
 
-        return aliveNodes
+        List<ClusterNode> participantNodes = aliveNodes
                 .stream()
                 .filter(node -> node.version().compareTo(VERSION_SINCE) >= 0)
                 .collect(Collectors.toList());
+
+        if (canSkipJoiningNodes(topVer))
+            return excludeJoinedNodes(participantNodes, topVer);
+
+        return participantNodes;
+    }
+
+    /**
+     * Excludes a node that was joined on given {@code topVer} from participant nodes.
+     *
+     * @param participantNodes Participant nodes.
+     * @param topVer Topology version.
+     */
+    private List<ClusterNode> excludeJoinedNodes(List<ClusterNode> participantNodes, AffinityTopologyVersion topVer) {
+        ClusterNode joinedNode = joinedNodes.get(topVer);
+
+        if (joinedNode != null)
+            participantNodes.remove(joinedNode);
+
+        return participantNodes;
     }
 
     /**
@@ -262,12 +300,33 @@ private Collection<ClusterNode> getLatchParticipants(AffinityTopologyVersion top
     @Nullable private ClusterNode getLatchCoordinator(AffinityTopologyVersion topVer) {
         Collection<ClusterNode> aliveNodes = aliveNodesForTopologyVer(topVer);
 
-        return aliveNodes
+        List<ClusterNode> applicableNodes = aliveNodes
             .stream()
             .filter(node -> node.version().compareTo(VERSION_SINCE) >= 0)
             .sorted(Comparator.comparing(ClusterNode::order))
-            .findFirst()
-            .orElse(null);
+            .collect(Collectors.toList());
+
+        if (applicableNodes.isEmpty())
+            return null;
+
+        if (canSkipJoiningNodes(topVer))
+            applicableNodes = excludeJoinedNodes(applicableNodes, topVer);
+
+        return applicableNodes.get(0);
+    }
+
+    /**
+     * Checks that latch manager can use V2 protocol and skip joining nodes from latch participants.
+     *
+     * @param topVer Topology version.
+     */
+    public boolean canSkipJoiningNodes(AffinityTopologyVersion topVer) {
+        Collection<ClusterNode> applicableNodes = topVer.equals(AffinityTopologyVersion.NONE)
+            ? discovery.aliveServerNodes()
+            : discovery.topology(topVer.topologyVersion());
+
+        return applicableNodes.stream()
+            .allMatch(node -> node.version().compareTo(PROTOCOL_V2_VERSION_SINCE) >= 0);
     }
 
     /**
@@ -284,7 +343,7 @@ private void processAck(UUID from, LatchAckMessage message) {
         lock.lock();
 
         try {
-            ClusterNode coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+            ClusterNode coordinator = getLatchCoordinator(message.topVer());
 
             if (coordinator == null)
                 return;
@@ -315,11 +374,8 @@ private void processAck(UUID from, LatchAckMessage message) {
                     if (latch.hasParticipant(from) && !latch.hasAck(from))
                         latch.ack(from);
                 }
-                else {
-                    pendingAcks.computeIfAbsent(latchUid, (id) -> new GridConcurrentHashSet<>());
-
-                    pendingAcks.get(latchUid).add(from);
-                }
+                else
+                    pendingAcks.computeIfAbsent(latchUid, id -> new GridConcurrentHashSet<>()).add(from);
             }
         }
         finally {
@@ -341,7 +397,6 @@ private void becomeNewCoordinator() {
         latchesToRestore.addAll(clientLatches.keySet());
 
         for (CompletableLatchUid latchUid : latchesToRestore) {
-            String id = latchUid.id;
             AffinityTopologyVersion topVer = latchUid.topVer;
             Collection<ClusterNode> participants = getLatchParticipants(topVer);
 
@@ -361,7 +416,7 @@ private void becomeNewCoordinator() {
      *
      * @param left Left node.
      */
-    private void processNodeLeft(ClusterNode left) {
+    private void processNodeLeft(AffinityTopologyVersion topVer, ClusterNode left) {
         assert this.crd != null : "Coordinator is not initialized";
 
         lock.lock();
@@ -370,11 +425,17 @@ private void processNodeLeft(ClusterNode left) {
             if (log.isDebugEnabled())
                 log.debug("Process node left " + left.id());
 
-            ClusterNode coordinator = getLatchCoordinator(AffinityTopologyVersion.NONE);
+            ClusterNode coordinator = getLatchCoordinator(topVer);
 
             if (coordinator == null)
                 return;
 
+            // Removed node from joined nodes map.
+            joinedNodes.entrySet().stream()
+                .filter(e -> e.getValue().equals(left))
+                .map(e -> e.getKey()) // Map to topology version when node has joined.
+                .forEach(joinedNodes::remove);
+
             // Clear pending acks.
             for (Map.Entry<CompletableLatchUid, Set<UUID>> ackEntry : pendingAcks.entrySet())
                 if (ackEntry.getValue().contains(left.id()))
@@ -391,9 +452,9 @@ private void processNodeLeft(ClusterNode left) {
                         /* If new coordinator is not able to take control on the latch,
                            it means that all other latch participants are left from topology
                            and there is no reason to track such latch. */
-                        AffinityTopologyVersion topVer = latchEntry.getKey().topVer;
+                        AffinityTopologyVersion latchTopVer = latchEntry.getKey().topVer;
 
-                        assert getLatchParticipants(topVer).isEmpty();
+                        assert getLatchParticipants(latchTopVer).isEmpty();
 
                         latch.complete(new IgniteCheckedException("All latch participants are left from topology."));
                         clientLatches.remove(latchEntry.getKey());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java
index 52cd0338a87d0..d8f23acdfc1da 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/datastructures/IgniteExchangeLatchManagerCoordinatorFailTest.java
@@ -23,9 +23,10 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiClosure;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -39,7 +40,10 @@ public class IgniteExchangeLatchManagerCoordinatorFailTest extends GridCommonAbs
     private static final String LATCH_NAME = "test";
 
     /** 5 nodes. */
-    private final AffinityTopologyVersion latchTopVer = new AffinityTopologyVersion(5, 0);
+    private final AffinityTopologyVersion latchTopVer = new AffinityTopologyVersion(5, 1);
+
+    /** Latch coordinator index. */
+    private static final int LATCH_CRD_INDEX = 0;
 
     /** Wait before latch creation. */
     private final IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean> beforeCreate = (mgr, syncLatch) -> {
@@ -200,6 +204,8 @@ private void doTestCoordinatorFail(List<IgniteBiClosure<ExchangeLatchManager, Co
         IgniteEx crd = (IgniteEx) startGridsMultiThreaded(5);
         crd.cluster().active(true);
 
+        IgniteEx latchCrd = grid(LATCH_CRD_INDEX);
+
         // Latch to synchronize node states.
         CountDownLatch syncLatch = new CountDownLatch(5);
 
@@ -207,31 +213,41 @@ private void doTestCoordinatorFail(List<IgniteBiClosure<ExchangeLatchManager, Co
 
         AtomicBoolean hasErrors = new AtomicBoolean();
 
-        for (int node = 1; node < 5; node++) {
-            IgniteEx grid = grid(node);
+        int scenarioIdx = 0;
+
+        for (int nodeId = 0; nodeId < 5; nodeId++) {
+            if (nodeId == LATCH_CRD_INDEX)
+                continue;
+
+            IgniteEx grid = grid(nodeId);
+
             ExchangeLatchManager latchMgr = grid.context().cache().context().exchange().latch();
-            final int stateIdx = node - 1;
+
+            IgniteBiClosure<ExchangeLatchManager, CountDownLatch, Boolean> scenario = nodeScenarios.get(scenarioIdx);
 
             IgniteInternalFuture<?> fut = GridTestUtils.runMultiThreadedAsync(() -> {
-                boolean success = nodeScenarios.get(stateIdx).apply(latchMgr, syncLatch);
+                boolean success = scenario.apply(latchMgr, syncLatch);
+
                 if (!success)
                     hasErrors.set(true);
-            }, 1, "latch-runner-" + node);
+            }, 1, "latch-runner-" + nodeId);
 
             finishAllLatches.add(fut);
+
+            scenarioIdx++;
         }
 
         finishAllLatches.markInitialized();
 
         // Wait while all nodes reaches their states.
         while (syncLatch.getCount() != 1) {
-            Thread.sleep(10);
+            U.sleep(10);
 
             if (hasErrors.get())
                 throw new Exception("All nodes should complete latches without errors");
         }
 
-        crd.close();
+        latchCrd.close();
 
         // Resume progress for all nodes.
         syncLatch.countDown();

From 5d5f3429a706df3c76394ca0066ba85b71a8a18a Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 26 Sep 2018 18:20:26 +0300
Subject: [PATCH 382/543] IGNITE-9419 Save cache configuration asynchronously -
 Fixes #4674.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 0b2fc1510b413dea890cd0cc882ef621a98afad2)

(cherry picked from commit 765d407)
---
 .../pagemem/store/IgnitePageStoreManager.java |  10 +
 .../cache/CacheAffinitySharedManager.java     | 203 +++---------
 .../processors/cache/CachesRegistry.java      | 310 ++++++++++++++++++
 .../processors/cache/GridCacheProcessor.java  |  35 +-
 .../cache/GridCacheSharedContext.java         |   2 +-
 .../processors/cache/StoredCacheData.java     |   4 +-
 .../GridDhtPartitionsExchangeFuture.java      |  73 ++++-
 .../file/FilePageStoreManager.java            |  31 +-
 .../cluster/GridClusterStateProcessor.java    |   2 +-
 ...niteAbstractDynamicCacheStartFailTest.java |  13 +-
 .../pagemem/NoOpPageStoreManager.java         |   5 +
 11 files changed, 481 insertions(+), 207 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CachesRegistry.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index 5475bef76ebaa..13bcd7df41d98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -233,4 +233,14 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cac
      * Cleanup persistent space for all caches.
      */
     public void cleanupPersistentSpace() throws IgniteCheckedException;
+
+    /**
+     * Creates and initializes cache work directory retrieved from {@code cacheCfg}.
+     *
+     * @param cacheCfg Cache configuration.
+     * @return {@code True} if work directory already exists.
+     *
+     * @throws IgniteCheckedException If failed.
+     */
+    public boolean checkAndInitCacheWorkDir(CacheConfiguration cacheCfg) throws IgniteCheckedException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 6c03cc79fc0b4..80b4204f0ccaf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -110,7 +110,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
     private AffinityTopologyVersion lastAffVer;
 
     /** Registered caches (updated from exchange thread). */
-    private final CachesInfo caches = new CachesInfo();
+    private CachesRegistry cachesRegistry;
 
     /** */
     private WaitRebalanceInfo waitInfo;
@@ -144,6 +144,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
         super.start0();
 
         cctx.kernalContext().event().addLocalEventListener(discoLsnr, EVT_NODE_LEFT, EVT_NODE_FAILED);
+
+        cachesRegistry = new CachesRegistry(cctx);
     }
 
     /**
@@ -181,14 +183,11 @@ void onDiscoveryEvent(int type,
     /**
      * Must be called from exchange thread.
      */
-    public void initCachesOnLocalJoin(
-        Map<Integer, CacheGroupDescriptor> cacheGroupDescriptors,
-        Map<String, DynamicCacheDescriptor> cacheDescriptors
+    public IgniteInternalFuture<?> initCachesOnLocalJoin(
+        Map<Integer, CacheGroupDescriptor> grpDescs,
+        Map<String, DynamicCacheDescriptor> cacheDescs
     ) {
-        // Clean-up in case of client reconnect.
-        caches.clear();
-
-        caches.init(cacheGroupDescriptors, cacheDescriptors);
+        return cachesRegistry.init(grpDescs, cacheDescs);
     }
 
     /**
@@ -377,7 +376,7 @@ void onCacheGroupCreated(CacheGroupContext grp) {
         List<DynamicCacheDescriptor> startDescs = new ArrayList<>(startReqs.size());
 
         for (DynamicCacheChangeRequest startReq : startReqs.values()) {
-            DynamicCacheDescriptor desc = caches.cache(CU.cacheId(startReq.cacheName()));
+            DynamicCacheDescriptor desc = cachesRegistry.cache(CU.cacheId(startReq.cacheName()));
 
             if (desc == null) {
                 CacheException err = new CacheException("Failed to start client cache " +
@@ -589,7 +588,7 @@ private Set<Integer> processCacheCloseRequests(
 
                     try {
                         grpHolder = CacheGroupHolder2.create(cctx,
-                            caches.group(grpId),
+                            cachesRegistry.group(grpId),
                             topVer,
                             grpHolder.affinity());
 
@@ -642,7 +641,7 @@ void sendClientCacheChangesMessage(ClientCacheUpdateTimeout timeoutObj) {
 
             clientCacheChanges.remove();
 
-            msg.checkCachesExist(caches.registeredCaches.keySet());
+            msg.checkCachesExist(cachesRegistry.allCaches().keySet());
 
             try {
                 if (!msg.empty())
@@ -719,14 +718,14 @@ public void onCustomMessageNoAffinityChange(
      * @param cctx Stopped cache context.
      */
     public void stopCacheOnReconnect(GridCacheContext cctx) {
-        caches.registeredCaches.remove(cctx.cacheId());
+        cachesRegistry.unregisterCache(cctx.cacheId());
     }
 
     /**
      * @param grpCtx Stopped cache group context.
      */
     public void stopCacheGroupOnReconnect(CacheGroupContext grpCtx) {
-        caches.registeredGrps.remove(grpCtx.groupId());
+        cachesRegistry.unregisterGroup(grpCtx.groupId());
     }
 
     /**
@@ -744,7 +743,9 @@ public void forceCloseCaches(
     ) {
         assert exchActions != null && !exchActions.empty() && exchActions.cacheStartRequests().isEmpty(): exchActions;
 
-        caches.updateCachesInfo(exchActions);
+        IgniteInternalFuture<?> res = cachesRegistry.update(exchActions);
+
+        assert res.isDone() : "There should be no caches to start: " + exchActions;
 
         processCacheStopRequests(fut, crd, exchActions, true);
 
@@ -759,14 +760,14 @@ public void forceCloseCaches(
      * @param exchActions Cache change requests.
      * @throws IgniteCheckedException If failed.
      */
-    public void onCacheChangeRequest(
+    public IgniteInternalFuture<?> onCacheChangeRequest(
         GridDhtPartitionsExchangeFuture fut,
         boolean crd,
         final ExchangeActions exchActions
     ) throws IgniteCheckedException {
         assert exchActions != null && !exchActions.empty() : exchActions;
 
-        caches.updateCachesInfo(exchActions);
+        IgniteInternalFuture<?> res = cachesRegistry.update(exchActions);
 
         // Affinity did not change for existing caches.
         onCustomMessageNoAffinityChange(fut, crd, exchActions);
@@ -806,11 +807,13 @@ public void onCacheChangeRequest(
         ClientCacheChangeDiscoveryMessage msg = clientCacheChanges.get();
 
         if (msg != null) {
-            msg.checkCachesExist(caches.registeredCaches.keySet());
+            msg.checkCachesExist(cachesRegistry.allCaches().keySet());
 
             if (msg.empty())
                 clientCacheChanges.remove();
         }
+
+        return res;
     }
 
     /**
@@ -903,7 +906,7 @@ private void processCacheStartRequests(
                     if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.initialVersion())) {
                         assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion();
 
-                        initAffinity(caches.group(grp.groupId()), grp.affinity(), fut);
+                        initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut);
                     }
                 }
             }
@@ -960,10 +963,10 @@ private Set<Integer> processCacheStopRequests(
     /**
      *
      */
-    public void removeAllCacheInfo() {
+    public void clearGroupHoldersAndRegistry() {
         grpHolders.clear();
 
-        caches.clear();
+        cachesRegistry.unregisterAll();
     }
 
     /**
@@ -1050,7 +1053,7 @@ public void onChangeAffinityMessage(final GridDhtPartitionsExchangeFuture exchFu
 
                 assert affTopVer.topologyVersion() > 0 : affTopVer;
 
-                CacheGroupDescriptor desc = caches.group(aff.groupId());
+                CacheGroupDescriptor desc = cachesRegistry.group(aff.groupId());
 
                 assert desc != null : aff.cacheOrGroupName();
 
@@ -1157,7 +1160,7 @@ private void processAffinityAssignmentResponse(UUID nodeId, GridDhtAffinityAssig
      * @throws IgniteCheckedException If failed
      */
     private void forAllRegisteredCacheGroups(IgniteInClosureX<CacheGroupDescriptor> c) throws IgniteCheckedException {
-        for (CacheGroupDescriptor cacheDesc : caches.allGroups()) {
+        for (CacheGroupDescriptor cacheDesc : cachesRegistry.allGroups().values()) {
             if (cacheDesc.config().getCacheMode() == LOCAL)
                 continue;
 
@@ -1230,15 +1233,15 @@ else if (grpHolder.client() && grp != null) {
      * @param descs Cache descriptors.
      * @throws IgniteCheckedException If failed.
      */
-    public void initStartedCaches(
+    public IgniteInternalFuture<?> initStartedCaches(
         boolean crd,
         final GridDhtPartitionsExchangeFuture fut,
         Collection<DynamicCacheDescriptor> descs
     ) throws IgniteCheckedException {
-        caches.initStartedCaches(descs);
+        IgniteInternalFuture<?> res = cachesRegistry.addUnregistered(descs);
 
         if (fut.context().mergeExchanges())
-            return;
+            return res;
 
         if (crd) {
             forAllRegisteredCacheGroups(new IgniteInClosureX<CacheGroupDescriptor>() {
@@ -1254,10 +1257,12 @@ public void initStartedCaches(
             forAllCacheGroups(false, new IgniteInClosureX<GridAffinityAssignmentCache>() {
                 @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException {
                     if (aff.lastVersion().equals(AffinityTopologyVersion.NONE))
-                        initAffinity(caches.group(aff.groupId()), aff, fut);
+                        initAffinity(cachesRegistry.group(aff.groupId()), aff, fut);
                 }
             });
         }
+
+        return res;
     }
 
     /**
@@ -1596,7 +1601,7 @@ private String groupNames(Collection<Integer> grpIds) {
         StringBuilder names = new StringBuilder();
 
         for (Integer grpId : grpIds) {
-            String name = caches.group(grpId).cacheOrGroupName();
+            String name = cachesRegistry.group(grpId).cacheOrGroupName();
 
             if (names.length() != 0)
                 names.append(", ");
@@ -1612,7 +1617,7 @@ private String groupNames(Collection<Integer> grpIds) {
      * @return Group name for debug purpose.
      */
     private String debugGroupName(int grpId) {
-        CacheGroupDescriptor desc = caches.group(grpId);
+        CacheGroupDescriptor desc = cachesRegistry.group(grpId);
 
         if (desc != null)
             return desc.cacheOrGroupName();
@@ -1654,7 +1659,7 @@ private void fetchAffinityOnJoin(GridDhtPartitionsExchangeFuture fut) throws Ign
             }
             else {
                 if (fut.context().fetchAffinityOnJoin()) {
-                    CacheGroupDescriptor grpDesc = caches.group(grp.groupId());
+                    CacheGroupDescriptor grpDesc = cachesRegistry.group(grp.groupId());
 
                     assert grpDesc != null : grp.cacheOrGroupName();
 
@@ -2352,14 +2357,14 @@ else if (curPrimary != null && !curPrimary.equals(newPrimary)) {
      * @return All registered cache groups.
      */
     public Map<Integer, CacheGroupDescriptor> cacheGroups() {
-        return caches.registeredGrps;
+        return cachesRegistry.allGroups();
     }
 
     /**
      * @return All registered cache groups.
      */
     public Map<Integer, DynamicCacheDescriptor> caches() {
-        return caches.registeredCaches;
+        return cachesRegistry.allCaches();
     }
 
     /**
@@ -2644,7 +2649,7 @@ void add(Integer grpId, Integer part, UUID waitNode, List<ClusterNode> assignmen
             if (cacheWaitParts == null) {
                 waitGrps.put(grpId, cacheWaitParts = new HashMap<>());
 
-                deploymentIds.put(grpId, caches.group(grpId).deploymentId());
+                deploymentIds.put(grpId, cachesRegistry.group(grpId).deploymentId());
             }
 
             cacheWaitParts.put(part, waitNode);
@@ -2663,138 +2668,4 @@ void add(Integer grpId, Integer part, UUID waitNode, List<ClusterNode> assignmen
                 ", grps=" + (waitGrps != null ? waitGrps.keySet() : null) + ']';
         }
     }
-
-    /**
-     *
-     */
-    class CachesInfo {
-        /** Registered cache groups (updated from exchange thread). */
-        private final ConcurrentHashMap<Integer, CacheGroupDescriptor> registeredGrps = new ConcurrentHashMap<>();
-
-        /** Registered caches (updated from exchange thread). */
-        private final ConcurrentHashMap<Integer, DynamicCacheDescriptor> registeredCaches = new ConcurrentHashMap<>();
-
-        /**
-         * @param grps Registered groups.
-         * @param caches Registered caches.
-         */
-        void init(Map<Integer, CacheGroupDescriptor> grps, Map<String, DynamicCacheDescriptor> caches) {
-            for (CacheGroupDescriptor grpDesc : grps.values())
-                registerGroup(grpDesc);
-
-            for (DynamicCacheDescriptor cacheDesc : caches.values())
-                registerCache(cacheDesc);
-        }
-
-
-        /**
-         * @param desc Description.
-         */
-        private DynamicCacheDescriptor registerCache(DynamicCacheDescriptor desc) {
-            saveCacheConfiguration(desc.cacheConfiguration(), desc.sql());
-
-            return registeredCaches.put(desc.cacheId(), desc);
-        }
-
-        /**
-         * @param grpDesc Group description.
-         */
-        private CacheGroupDescriptor registerGroup(CacheGroupDescriptor grpDesc) {
-            return registeredGrps.put(grpDesc.groupId(), grpDesc);
-        }
-
-        /**
-         * @return All registered groups.
-         */
-        Collection<CacheGroupDescriptor> allGroups() {
-            return registeredGrps.values();
-        }
-
-        /**
-         * @param grpId Group ID.
-         * @return Group descriptor.
-         */
-        CacheGroupDescriptor group(int grpId) {
-            CacheGroupDescriptor desc = registeredGrps.get(grpId);
-
-            assert desc != null : grpId;
-
-            return desc;
-        }
-
-        /**
-          * @param descs Cache descriptor.
-         */
-        void initStartedCaches(Collection<DynamicCacheDescriptor> descs) {
-            for (DynamicCacheDescriptor desc : descs) {
-                CacheGroupDescriptor grpDesc = desc.groupDescriptor();
-
-                if (!registeredGrps.containsKey(grpDesc.groupId()))
-                    registerGroup(grpDesc);
-
-                if (!registeredCaches.containsKey(desc.cacheId()))
-                    registerCache(desc);
-            }
-        }
-
-        /**
-         * @param exchActions Exchange actions.
-         */
-        void updateCachesInfo(ExchangeActions exchActions) {
-            for (ExchangeActions.CacheGroupActionData stopAction : exchActions.cacheGroupsToStop()) {
-                CacheGroupDescriptor rmvd = registeredGrps.remove(stopAction.descriptor().groupId());
-
-                assert rmvd != null : stopAction.descriptor().cacheOrGroupName();
-            }
-
-            for (ExchangeActions.CacheGroupActionData startAction : exchActions.cacheGroupsToStart()) {
-                CacheGroupDescriptor old = registerGroup(startAction.descriptor());
-
-                assert old == null : old;
-            }
-
-            for (ExchangeActions.CacheActionData req : exchActions.cacheStopRequests())
-                registeredCaches.remove(req.descriptor().cacheId());
-
-            for (ExchangeActions.CacheActionData req : exchActions.cacheStartRequests())
-                registerCache(req.descriptor());
-        }
-
-        /**
-         * @param cacheId Cache ID.
-         * @return Cache descriptor if cache found.
-         */
-        @Nullable DynamicCacheDescriptor cache(Integer cacheId) {
-            return registeredCaches.get(cacheId);
-        }
-
-        /**
-         *
-         */
-        void clear() {
-            registeredGrps.clear();
-
-            registeredCaches.clear();
-        }
-    }
-
-    /**
-     * @param cfg cache configuration
-     * @param sql SQL flag.
-     */
-    private void saveCacheConfiguration(CacheConfiguration<?, ?> cfg, boolean sql) {
-        if (cctx.pageStore() != null && CU.isPersistentCache(cfg, cctx.gridConfig().getDataStorageConfiguration()) &&
-            !cctx.kernalContext().clientNode()) {
-            try {
-                StoredCacheData data = new StoredCacheData(cfg);
-
-                data.sql(sql);
-
-                cctx.pageStore().storeCacheData(data, false);
-            }
-            catch (IgniteCheckedException e) {
-                U.error(log(), "Error while saving cache configuration on disk, cfg = " + cfg, e);
-            }
-        }
-    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CachesRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CachesRegistry.java
new file mode 100644
index 0000000000000..fe55f979a56ec
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CachesRegistry.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Class is responsible to hold and persist cache and cache group descriptors.
+ */
+public class CachesRegistry {
+    /** Logger. */
+    private final IgniteLogger log;
+
+    /** Cache shared context. */
+    private final GridCacheSharedContext cctx;
+
+    /** Registered cache groups (updated from exchange thread). */
+    private final ConcurrentHashMap<Integer, CacheGroupDescriptor> registeredGrps = new ConcurrentHashMap<>();
+
+    /** Registered caches (updated from exchange thread). */
+    private final ConcurrentHashMap<Integer, DynamicCacheDescriptor> registeredCaches = new ConcurrentHashMap<>();
+
+    /** Last registered caches configuration persist future. */
+    private volatile IgniteInternalFuture<?> cachesConfPersistFuture;
+
+    /**
+     * @param cctx Cache shared context.
+     */
+    public CachesRegistry(GridCacheSharedContext cctx) {
+        assert cctx != null;
+
+        this.cctx = cctx;
+        this.log = cctx.logger(getClass());
+    }
+
+    /**
+     * Removes currently registered cache groups and caches.
+     * Adds given cache groups and caches to registry.
+     *
+     * @param groupDescriptors Registered groups.
+     * @param cacheDescriptors Registered caches.
+     * @return Future that will be completed when all caches configurations will be persisted.
+     */
+    public IgniteInternalFuture<?> init(
+        Map<Integer, CacheGroupDescriptor> groupDescriptors,
+        Map<String, DynamicCacheDescriptor> cacheDescriptors
+    ) {
+        unregisterAll();
+
+        return registerAllCachesAndGroups(groupDescriptors.values(), cacheDescriptors.values());
+    }
+
+    /**
+     * Adds cache group to registry.
+     *
+     * @param grpDesc Group description.
+     * @return Previously registered cache group or {@code null} otherwise.
+     */
+    private CacheGroupDescriptor registerGroup(CacheGroupDescriptor grpDesc) {
+        return registeredGrps.put(grpDesc.groupId(), grpDesc);
+    }
+
+    /**
+     * Adds cache to registry.
+     *
+     * @param desc Cache description.
+     * @return Previously registered cache or {@code null} otherwise.
+     */
+    private DynamicCacheDescriptor registerCache(DynamicCacheDescriptor desc) {
+        return registeredCaches.put(desc.cacheId(), desc);
+    }
+
+    /**
+     * Removes cache group from registry.
+     *
+     * @param grpId Group id.
+     * @return Unregistered cache group or {@code null} if group doesn't exist.
+     */
+    public CacheGroupDescriptor unregisterGroup(int grpId) {
+        return registeredGrps.remove(grpId);
+    }
+
+    /**
+     * @return All registered cache groups.
+     */
+    public Map<Integer, CacheGroupDescriptor> allGroups() {
+        return Collections.unmodifiableMap(registeredGrps);
+    }
+
+    /**
+     * @param grpId Group ID.
+     * @return Group descriptor.
+     */
+    public CacheGroupDescriptor group(int grpId) {
+        CacheGroupDescriptor desc = registeredGrps.get(grpId);
+
+        assert desc != null : grpId;
+
+        return desc;
+    }
+
+    /**
+     * @param cacheId Cache ID.
+     * @return Cache descriptor if cache found.
+     */
+    @Nullable public DynamicCacheDescriptor cache(int cacheId) {
+        return registeredCaches.get(cacheId);
+    }
+
+    /**
+     * Removes cache from registry.
+     *
+     * @param cacheId Cache id.
+     * @return Unregistered cache or {@code null} if cache doesn't exist.
+     */
+    @Nullable public DynamicCacheDescriptor unregisterCache(int cacheId) {
+        return registeredCaches.remove(cacheId);
+    }
+
+    /**
+     * @return All registered cache groups.
+     */
+    public Map<Integer, DynamicCacheDescriptor> allCaches() {
+        return Collections.unmodifiableMap(registeredCaches);
+    }
+
+    /**
+     * Adds cache and caches groups that is not registered yet to registry.
+     *
+     * @param descs Cache and cache group descriptors.
+     * @return Future that will be completed when all unregistered cache configurations will be persisted.
+     */
+    public IgniteInternalFuture<?> addUnregistered(Collection<DynamicCacheDescriptor> descs) {
+        Collection<CacheGroupDescriptor> groups = descs.stream()
+            .map(DynamicCacheDescriptor::groupDescriptor)
+            .filter(grpDesc -> !registeredGrps.containsKey(grpDesc.groupId()))
+            .collect(Collectors.toList());
+
+        Collection<DynamicCacheDescriptor> caches = descs.stream()
+            .filter(cacheDesc -> !registeredCaches.containsKey(cacheDesc.cacheId()))
+            .collect(Collectors.toList());
+
+        return registerAllCachesAndGroups(groups, caches);
+    }
+
+    /**
+     * Adds caches and cache groups to start from {@code exchActions}.
+     * Removes caches and caches groups to stop from {@code exchActions}.
+     *
+     * @param exchActions Exchange actions.
+     * @return Future that will be completed when all unregistered cache configurations will be persisted.
+     */
+    public IgniteInternalFuture<?> update(ExchangeActions exchActions) {
+        for (ExchangeActions.CacheGroupActionData stopAction : exchActions.cacheGroupsToStop()) {
+            CacheGroupDescriptor rmvd = unregisterGroup(stopAction.descriptor().groupId());
+
+            assert rmvd != null : stopAction.descriptor().cacheOrGroupName();
+        }
+
+        for (ExchangeActions.CacheActionData req : exchActions.cacheStopRequests())
+            unregisterCache(req.descriptor().cacheId());
+
+        Collection<CacheGroupDescriptor> grpDescs = exchActions.cacheGroupsToStart().stream()
+            .map(ExchangeActions.CacheGroupActionData::descriptor)
+            .collect(Collectors.toList());
+
+        Collection<DynamicCacheDescriptor> cacheDescs = exchActions.cacheStartRequests().stream()
+            .map(ExchangeActions.CacheActionData::descriptor)
+            .collect(Collectors.toList());
+
+        return registerAllCachesAndGroups(grpDescs, cacheDescs);
+    }
+
+    /**
+     *
+     */
+    public void unregisterAll() {
+        registeredGrps.clear();
+
+        registeredCaches.clear();
+    }
+
+    /**
+     * Awaits last registered caches configurations persist future.
+     */
+    private void waitLastRegistration() {
+        IgniteInternalFuture<?> currentFut = cachesConfPersistFuture;
+
+        if (currentFut != null && !currentFut.isDone()) {
+            try {
+                currentFut.get();
+            }
+            catch (IgniteCheckedException e) {
+                throw new IgniteException("Failed to wait for last registered caches registration future", e);
+            }
+
+            if (log.isInfoEnabled())
+                log.info("Successfully awaited for last registered caches registration future");
+        }
+    }
+
+    /**
+     * Registers caches and groups.
+     * Persists caches configurations on disk if needed.
+     *
+     * @param groupDescriptors Cache group descriptors.
+     * @param cacheDescriptors Cache descriptors.
+     * @return Future that will be completed when all unregistered cache configurations will be persisted.
+     */
+    private IgniteInternalFuture<?> registerAllCachesAndGroups(
+        Collection<CacheGroupDescriptor> groupDescriptors,
+        Collection<DynamicCacheDescriptor> cacheDescriptors
+    ) {
+        waitLastRegistration();
+
+        for (CacheGroupDescriptor grpDesc : groupDescriptors)
+            registerGroup(grpDesc);
+
+        for (DynamicCacheDescriptor cacheDesc : cacheDescriptors)
+            registerCache(cacheDesc);
+
+        List<DynamicCacheDescriptor> cachesToPersist = cacheDescriptors.stream()
+            .filter(cacheDesc -> shouldPersist(cacheDesc.cacheConfiguration()))
+            .collect(Collectors.toList());
+
+        if (cachesToPersist.isEmpty())
+            return cachesConfPersistFuture = new GridFinishedFuture<>();
+
+        return cachesConfPersistFuture = persistCacheConfigurations(cachesToPersist);
+    }
+
+    /**
+     * Checks whether given cache configuration should be persisted.
+     *
+     * @param cacheCfg Cache config.
+     * @return {@code True} if cache configuration should be persisted, {@code false} in other case.
+     */
+    private boolean shouldPersist(CacheConfiguration<?, ?> cacheCfg) {
+        return cctx.pageStore() != null &&
+            CU.isPersistentCache(cacheCfg, cctx.gridConfig().getDataStorageConfiguration()) &&
+            !cctx.kernalContext().clientNode();
+    }
+
+    /**
+     * Persists cache configurations from given {@code cacheDescriptors}.
+     *
+     * @param cacheDescriptors Cache descriptors to retrieve cache configurations.
+     * @return Future that will be completed when all cache configurations will be persisted to cache work directory.
+     */
+    private IgniteInternalFuture<?> persistCacheConfigurations(List<DynamicCacheDescriptor> cacheDescriptors) {
+        List<StoredCacheData> cacheConfigsToPersist = cacheDescriptors.stream()
+            .map(cacheDesc -> new StoredCacheData(cacheDesc.cacheConfiguration()).sql(cacheDesc.sql()))
+            .collect(Collectors.toList());
+
+        // Pre-create cache work directories if they don't exist.
+        for (StoredCacheData data : cacheConfigsToPersist) {
+            try {
+                cctx.pageStore().checkAndInitCacheWorkDir(data.config());
+            }
+            catch (IgniteCheckedException e) {
+                if (!cctx.kernalContext().isStopping()) {
+                    cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+                    U.error(log, "Failed to initialize cache work directory for " + data.config(), e);
+                }
+            }
+        }
+
+        return cctx.kernalContext().closure().runLocalSafe(() -> {
+            try {
+                for (StoredCacheData data : cacheConfigsToPersist)
+                    cctx.pageStore().storeCacheData(data, false);
+            }
+            catch (IgniteCheckedException e) {
+                U.error(log, "Error while saving cache configurations on disk", e);
+            }
+        });
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 45d2f4c94df84..57bc716eea551 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1812,29 +1812,32 @@ public CacheMode cacheMode(String cacheName) {
     }
 
     /**
+     * @param exchTopVer Local join exchange future version.
      * @param locJoinCtx Local join cache context.
-     * @param exchTopVer Current exchange version.
      * @throws IgniteCheckedException If failed.
      */
-    public void startCachesOnLocalJoin(
-        LocalJoinCachesContext locJoinCtx,
-        AffinityTopologyVersion exchTopVer
+    public IgniteInternalFuture<?> startCachesOnLocalJoin(
+        AffinityTopologyVersion exchTopVer,
+        LocalJoinCachesContext locJoinCtx
     ) throws IgniteCheckedException {
-        if (locJoinCtx != null) {
-            sharedCtx.affinity().initCachesOnLocalJoin(
-                locJoinCtx.cacheGroupDescriptors(), locJoinCtx.cacheDescriptors());
+        if (locJoinCtx == null)
+            return new GridFinishedFuture<>();
 
-            for (T2<DynamicCacheDescriptor, NearCacheConfiguration> t : locJoinCtx.caches()) {
-                DynamicCacheDescriptor desc = t.get1();
+        IgniteInternalFuture<?> res = sharedCtx.affinity().initCachesOnLocalJoin(
+            locJoinCtx.cacheGroupDescriptors(), locJoinCtx.cacheDescriptors());
 
-                prepareCacheStart(
-                    desc.cacheConfiguration(),
-                    desc,
-                    t.get2(),
-                    exchTopVer,
-                    false);
-            }
+        for (T2<DynamicCacheDescriptor, NearCacheConfiguration> t : locJoinCtx.caches()) {
+            DynamicCacheDescriptor desc = t.get1();
+
+            prepareCacheStart(
+                desc.cacheConfiguration(),
+                desc,
+                t.get2(),
+                exchTopVer,
+                false);
         }
+
+        return res;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index 550c5e746d51b..b0a06b5ce4856 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -416,7 +416,7 @@ void onReconnected(boolean active) throws IgniteCheckedException {
         kernalCtx.query().onCacheReconnect();
 
         if (!active)
-            affinity().removeAllCacheInfo();
+            affinity().clearGroupHoldersAndRegistry();
 
         exchMgr.onKernalStart(active, true);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
index 26b4f9bb92d34..7b8e5344e4680 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StoredCacheData.java
@@ -98,8 +98,10 @@ public boolean sql() {
     /**
      * @param sql SQL flag - {@code true} if cache was created with {@code CREATE TABLE}.
      */
-    public void sql(boolean sql) {
+    public StoredCacheData sql(boolean sql) {
         this.sql = sql;
+
+        return this;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index ad0a9fed88c01..de8e1d917402e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -39,6 +39,7 @@
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
@@ -309,6 +310,9 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     @GridToStringExclude
     private final GridDhtPartitionsStateValidator validator;
 
+    /** Register caches future. Initialized on exchange init. Must be waited on exchange end. */
+    private IgniteInternalFuture<?> registerCachesFuture;
+
     /**
      * @param cctx Cache context.
      * @param busyLock Busy lock.
@@ -699,10 +703,10 @@ else if (msg instanceof WalStateAbstractMessage)
                             firstDiscoEvt.eventNode().id(),
                             topVer);
 
-                        cctx.affinity().initStartedCaches(crdNode, this, receivedCaches);
+                        registerCachesFuture = cctx.affinity().initStartedCaches(crdNode, this, receivedCaches);
                     }
                     else
-                        initCachesOnLocalJoin();
+                        registerCachesFuture = initCachesOnLocalJoin();
                 }
 
                 initCoordinatorCaches(newCrd);
@@ -797,7 +801,7 @@ else if (msg instanceof WalStateAbstractMessage)
     /**
      * @throws IgniteCheckedException If failed.
      */
-    private void initCachesOnLocalJoin() throws IgniteCheckedException {
+    private IgniteInternalFuture<?> initCachesOnLocalJoin() throws IgniteCheckedException {
         if (isLocalNodeNotInBaseline()) {
             cctx.cache().cleanupCachesDirectories();
 
@@ -829,7 +833,9 @@ private void initCachesOnLocalJoin() throws IgniteCheckedException {
             cctx.database().readCheckpointAndRestoreMemory(startDescs);
         }
 
-        cctx.cache().startCachesOnLocalJoin(locJoinCtx, initialVersion());
+        IgniteInternalFuture<?> cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(), locJoinCtx);
+
+        return cachesRegistrationFut;
     }
 
     /**
@@ -944,7 +950,9 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
                         cctx.database().readCheckpointAndRestoreMemory(startDescs);
                     }
 
-                    cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
+                    assert registerCachesFuture == null : "No caches registration should be scheduled before new caches have started.";
+
+                    registerCachesFuture = cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
 
                     if (log.isInfoEnabled()) {
                         log.info("Successfully activated caches [nodeId=" + cctx.localNodeId() +
@@ -978,7 +986,9 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
 
                     cctx.kernalContext().service().onDeActivate(cctx.kernalContext());
 
-                    cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
+                    assert registerCachesFuture == null : "No caches registration should be scheduled before new caches have started.";
+
+                    registerCachesFuture = cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
 
                     if (log.isInfoEnabled()) {
                         log.info("Successfully deactivated data structures, services and caches [" +
@@ -1035,7 +1045,9 @@ private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedExcep
         assert !exchActions.clientOnlyExchange() : exchActions;
 
         try {
-            cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
+            assert registerCachesFuture == null : "No caches registration should be scheduled before new caches have started.";
+
+            registerCachesFuture = cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
         }
         catch (Exception e) {
             if (reconnectOnError(e) || !isRollbackSupported())
@@ -1742,6 +1754,8 @@ public void finishMerged() {
 
         assert res != null || err != null;
 
+        waitUntilNewCachesAreRegistered();
+
         if (err == null &&
             !cctx.kernalContext().clientNode() &&
             (serverNodeDiscoveryEvent() || affChangeMsg != null)) {
@@ -1851,7 +1865,7 @@ public void finishMerged() {
             if (exchCtx != null && exchCtx.events().hasServerLeft()) {
                 ExchangeDiscoveryEvents evts = exchCtx.events();
 
-                for (DiscoveryEvent evt : exchCtx.events().events()) {
+                for (DiscoveryEvent evt : evts.events()) {
                     if (serverLeftEvent(evt)) {
                         for (CacheGroupContext grp : cctx.cache().cacheGroups())
                             grp.affinityFunction().removeNode(evt.eventNode().id());
@@ -1870,7 +1884,7 @@ public void finishMerged() {
                 if (exchCtx != null && (exchCtx.events().hasServerLeft() || exchCtx.events().hasServerJoin())) {
                     ExchangeDiscoveryEvents evts = exchCtx.events();
 
-                    for (DiscoveryEvent evt : exchCtx.events().events()) {
+                    for (DiscoveryEvent evt : evts.events()) {
                         if (serverLeftEvent(evt) || serverJoinEvent(evt))
                             logExchange(evt);
                     }
@@ -1884,6 +1898,40 @@ public void finishMerged() {
         return false;
     }
 
+    /**
+     * Method waits for new caches registration and cache configuration persisting to disk.
+     */
+    private void waitUntilNewCachesAreRegistered() {
+        try {
+            IgniteInternalFuture<?> registerCachesFut = registerCachesFuture;
+
+            if (registerCachesFut != null && !registerCachesFut.isDone()) {
+                final int timeout = Math.max(1000,
+                    (int)(cctx.kernalContext().config().getFailureDetectionTimeout() / 2));
+
+                for (;;) {
+                    try {
+                        registerCachesFut.get(timeout, TimeUnit.SECONDS);
+
+                        break;
+                    }
+                    catch (IgniteFutureTimeoutCheckedException te) {
+                        List<String> cacheNames = exchActions.cacheStartRequests().stream()
+                            .map(req -> req.descriptor().cacheName())
+                            .collect(Collectors.toList());
+
+                        U.warn(log, "Failed to wait for caches configuration registration and saving within timeout. " +
+                            "Probably disk is too busy or slow." +
+                            "[caches=" + cacheNames + "]");
+                    }
+                }
+            }
+        }
+        catch (IgniteCheckedException e) {
+            U.error(log, "Failed to wait for caches registration and saving", e);
+        }
+    }
+
     /**
      * Log exchange event.
      *
@@ -3597,8 +3645,11 @@ public void onDynamicCacheChangeFail(final ClusterNode node, final DynamicCacheC
                             assert msg.error() != null: msg;
 
                             // Try to revert all the changes that were done during initialization phase
-                            cctx.affinity().forceCloseCaches(GridDhtPartitionsExchangeFuture.this,
-                                crd.isLocal(), msg.exchangeActions());
+                            cctx.affinity().forceCloseCaches(
+                                GridDhtPartitionsExchangeFuture.this,
+                                crd.isLocal(),
+                                msg.exchangeActions()
+                            );
 
                             synchronized (mux) {
                                 finishState = new FinishState(crd.id(), initialVersion(), null);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 95db1c2efce3f..268fd106ce22e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -328,8 +328,10 @@ public FilePageStoreManager(GridKernalContext ctx) {
         else
             file = new File(cacheWorkDir, CACHE_DATA_FILENAME);
 
-        if (overwrite || !file.exists() || file.length() == 0) {
-            try {
+        Path filePath = file.toPath();
+
+        try {
+            if (overwrite || !Files.exists(filePath) || Files.size(filePath) == 0) {
                 File tmp = new File(file.getParent(), file.getName() + TMP_SUFFIX);
 
                 tmp.createNewFile();
@@ -344,9 +346,11 @@ public FilePageStoreManager(GridKernalContext ctx) {
 
                 Files.move(tmp.toPath(), file.toPath());
             }
-            catch (IOException ex) {
-                throw new IgniteCheckedException("Failed to persist cache configuration: " + cacheData.config().getName(), ex);
-            }
+        }
+        catch (IOException ex) {
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
+
+            throw new IgniteCheckedException("Failed to persist cache configuration: " + cacheData.config().getName(), ex);
         }
     }
 
@@ -562,19 +566,26 @@ private CacheStoreHolder initDir(File cacheWorkDir,
         return new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId));
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean checkAndInitCacheWorkDir(CacheConfiguration cacheCfg) throws IgniteCheckedException {
+        return checkAndInitCacheWorkDir(cacheWorkDir(cacheCfg));
+    }
+
     /**
      * @param cacheWorkDir Cache work directory.
      */
     private boolean checkAndInitCacheWorkDir(File cacheWorkDir) throws IgniteCheckedException {
         boolean dirExisted = false;
 
-        if (!cacheWorkDir.exists()) {
-            boolean res = cacheWorkDir.mkdirs();
-
-            if (!res)
+        if (!Files.exists(cacheWorkDir.toPath())) {
+            try {
+                Files.createDirectory(cacheWorkDir.toPath());
+            }
+            catch (IOException e) {
                 throw new IgniteCheckedException("Failed to initialize cache working directory " +
                     "(failed to create, make sure the work folder has correct permissions): " +
-                    cacheWorkDir.getAbsolutePath());
+                    cacheWorkDir.getAbsolutePath(), e);
+            }
         }
         else {
             if (cacheWorkDir.isFile())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 72e5d8ab67da4..396cc678599d7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -1104,7 +1104,7 @@ private boolean sendComputeCheckGlobalState() {
 
                     cacheProc.stopCaches(true);
 
-                    sharedCtx.affinity().removeAllCacheInfo();
+                    sharedCtx.affinity().clearGroupHoldersAndRegistry();
 
                     if (!ctx.clientNode())
                         sharedCtx.deactivate();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java
index 9b9e5d70d7079..db13c11d1a947 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteAbstractDynamicCacheStartFailTest.java
@@ -437,11 +437,22 @@ public void testTopologyChangesAfterFailure() throws Exception {
         Ignite serverNode = startGrid(gridCount() + 1);
 
         if (persistenceEnabled()) {
+            // Start a new client node to perform baseline change.
+            // TODO: This change is workaround:
+            // Sometimes problem with caches configuration deserialization from test thread arises.
+            final String clientName1 = "baseline-changer";
+
+            IgniteConfiguration clientCfg = getConfiguration(clientName1);
+
+            clientCfg.setClientMode(true);
+
+            Ignite clientNode = startGrid(clientName1, clientCfg);
+
             List<BaselineNode> baseline = new ArrayList<>(grid(0).cluster().currentBaselineTopology());
 
             baseline.add(serverNode.cluster().localNode());
 
-            grid(0).cluster().setBaselineTopology(baseline);
+            clientNode.cluster().setBaselineTopology(baseline);
         }
 
         awaitPartitionMapExchange();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index c61b3c0a39a2d..222804e150e44 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -226,4 +226,9 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
     @Override public void cleanupPersistentSpace() throws IgniteCheckedException {
         // No-op.
     }
+
+    /** {@inheritDoc} */
+    @Override public boolean checkAndInitCacheWorkDir(CacheConfiguration cacheCfg) throws IgniteCheckedException {
+        return false;
+    }
 }

From 2c2673f90c4a556faba6f3e1578f43aa19e85dc6 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 26 Sep 2018 14:45:15 +0300
Subject: [PATCH 383/543] IGNITE-9492 Do not process background exchange
 messages if active exchange is in progress - Fixes #4773.

(cherry picked from commit ba9e8fd)

(cherry picked from commit 7e6ce8a)
---
 .../GridCachePartitionExchangeManager.java    | 43 ++++++++++++-
 .../preloader/GridDhtPartitionFullMap.java    | 10 ++-
 .../GridDhtPartitionsExchangeFuture.java      | 61 +++++++++++++++++++
 .../GridDhtPartitionsFullMessage.java         | 21 +++++++
 4 files changed, 133 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index aaec9d8790a34..335f953244994 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -338,6 +338,12 @@ private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) {
                             return;
                         }
                     }
+                    else if (exchangeInProgress()) {
+                        if (log.isInfoEnabled())
+                            log.info("Ignore single message without exchange id (there is exchange in progress) [nodeId=" + node.id() + "]");
+
+                        return;
+                    }
 
                     if (!crdInitFut.isDone() && !msg.restoreState()) {
                         GridDhtPartitionExchangeId exchId = msg.exchangeId();
@@ -364,6 +370,17 @@ private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) {
         cctx.io().addCacheHandler(0, GridDhtPartitionsFullMessage.class,
             new MessageHandler<GridDhtPartitionsFullMessage>() {
                 @Override public void onMessage(ClusterNode node, GridDhtPartitionsFullMessage msg) {
+                    if (msg.exchangeId() == null) {
+                        GridDhtPartitionsExchangeFuture currentExchange = lastTopologyFuture();
+
+                        if (currentExchange != null && currentExchange.addOrMergeDelayedFullMessage(node, msg)) {
+                            if (log.isInfoEnabled())
+                                log.info("Delay process full message without exchange id (there is exchange in progress) [nodeId=" + node.id() + "]");
+
+                            return;
+                        }
+                    }
+
                     processFullPartitionUpdate(node, msg);
                 }
             });
@@ -1484,7 +1501,7 @@ private boolean addFuture(GridDhtPartitionsExchangeFuture fut) {
      * @param node Sender cluster node.
      * @param msg Message.
      */
-    private void processFullPartitionUpdate(ClusterNode node, GridDhtPartitionsFullMessage msg) {
+    public void processFullPartitionUpdate(ClusterNode node, GridDhtPartitionsFullMessage msg) {
         if (!enterBusy())
             return;
 
@@ -2162,6 +2179,30 @@ private void waitForTestVersion(AffinityTopologyVersion exchMergeTestWaitVer, Gr
         this.exchMergeTestWaitVer = null;
     }
 
+    /**
+     * @return {@code True} If there is any exchange future in progress.
+     */
+    private boolean exchangeInProgress() {
+        if (exchWorker.hasPendingServerExchange())
+            return true;
+
+        GridDhtPartitionsExchangeFuture current = lastTopologyFuture();
+
+        if (current == null)
+            return false;
+
+        GridDhtTopologyFuture finished = lastFinishedFut.get();
+
+        if (finished == null || finished.result().compareTo(current.initialVersion()) < 0) {
+            ClusterNode triggeredBy = current.firstEvent().eventNode();
+
+            if (current.partitionChangesInProgress() && !triggeredBy.isClient())
+                return true;
+       }
+
+        return false;
+    }
+
     /**
      * Exchange future thread. All exchanges happen only by one thread and next
      * exchange will not start until previous one completes.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java
index 73b7714b8f8e3..4179520431d29 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionFullMap.java
@@ -27,11 +27,12 @@
 import java.util.UUID;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
 
 /**
  * Full partition map.
  */
-public class GridDhtPartitionFullMap extends HashMap<UUID, GridDhtPartitionMap> implements Externalizable {
+public class GridDhtPartitionFullMap extends HashMap<UUID, GridDhtPartitionMap> implements Comparable<GridDhtPartitionFullMap>, Externalizable {
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -251,4 +252,11 @@ public String toFullString() {
     @Override public String toString() {
         return S.toString(GridDhtPartitionFullMap.class, this, "size", size());
     }
+
+    /** {@inheritDoc} */
+    @Override public int compareTo(@NotNull GridDhtPartitionFullMap o) {
+        assert nodeId.equals(o.nodeId);
+
+        return Long.compare(updateSeq, o.updateSeq);
+    }
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index de8e1d917402e..bb8b2d2ee98f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -313,6 +313,15 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     /** Register caches future. Initialized on exchange init. Must be waited on exchange end. */
     private IgniteInternalFuture<?> registerCachesFuture;
 
+    /** Partitions sent flag (for coordinator node). */
+    private volatile boolean partitionsSent;
+
+    /** Partitions received flag (for non-coordinator node). */
+    private volatile boolean partitionsReceived;
+
+    /** Latest (by update sequences) full message with exchangeId == null, need to be processed right after future is done. */
+    private GridDhtPartitionsFullMessage delayedLatestMsg;
+
     /**
      * @param cctx Cache context.
      * @param busyLock Busy lock.
@@ -3035,6 +3044,8 @@ else if (forceAffReassignment)
                 if (!nodes.isEmpty())
                     sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff);
 
+                partitionsSent = true;
+
                 if (!stateChangeExchange())
                     onDone(exchCtx.events().topologyVersion(), null);
 
@@ -3598,6 +3609,8 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa
                 }
             }
         }
+
+        partitionsReceived = true;
     }
 
     /**
@@ -4154,6 +4167,54 @@ public boolean reconnectOnError(Throwable e) {
             && cctx.discovery().reconnectSupported();
     }
 
+    /**
+     * @return {@code True} If partition changes triggered by receiving Single/Full messages are not finished yet.
+     */
+    public boolean partitionChangesInProgress() {
+        boolean isCoordinator = crd.equals(cctx.localNode());
+
+        if (isCoordinator)
+            return !partitionsSent;
+        else
+            return !partitionsReceived;
+    }
+
+    /**
+     * Add or merge updates received from coordinator while exchange in progress.
+     *
+     * @param fullMsg Full message with exchangeId = null.
+     * @return {@code True} if message should be ignored and processed after exchange is done.
+     */
+    public synchronized boolean addOrMergeDelayedFullMessage(ClusterNode node, GridDhtPartitionsFullMessage fullMsg) {
+        assert fullMsg.exchangeId() == null : fullMsg.exchangeId();
+
+        if (isDone())
+            return false;
+
+        GridDhtPartitionsFullMessage prev = delayedLatestMsg;
+
+        if (prev == null) {
+            delayedLatestMsg = fullMsg;
+
+            listen(f -> {
+                GridDhtPartitionsFullMessage msg;
+
+                synchronized (this) {
+                    msg = delayedLatestMsg;
+
+                    delayedLatestMsg = null;
+                }
+
+                if (msg != null)
+                    cctx.exchange().processFullPartitionUpdate(node, msg);
+            });
+        }
+        else
+            delayedLatestMsg.merge(fullMsg);
+
+        return true;
+    }
+
     /** {@inheritDoc} */
     @Override public int compareTo(GridDhtPartitionsExchangeFuture fut) {
         return exchId.compareTo(fut.exchId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index ab45d8be35620..888ab6a642d7b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -797,4 +797,25 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
         return S.toString(GridDhtPartitionsFullMessage.class, this, "partCnt", parts != null ? parts.size() : 0,
             "super", super.toString());
     }
+
+    /**
+     * Merges (replaces with newer) partitions map from given {@code other} full message.
+     *
+     * @param other Other full message.
+     */
+    public void merge(GridDhtPartitionsFullMessage other) {
+        assert other.exchangeId() == null && exchangeId() == null :
+            "Both current and merge full message must have exchangeId == null"
+             + other.exchangeId() + "," + exchangeId();
+
+        for (Map.Entry<Integer, GridDhtPartitionFullMap> groupAndMap : other.partitions().entrySet()) {
+            int grpId = groupAndMap.getKey();
+            GridDhtPartitionFullMap updMap = groupAndMap.getValue();
+
+            GridDhtPartitionFullMap currMap = partitions().get(grpId);
+
+            if (currMap == null || updMap.compareTo(currMap) >= 0)
+                partitions().put(grpId, updMap);
+        }
+    }
 }

From 7a6e6d15d46ef827b547a6545c3f470cfcf6a398 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 26 Sep 2018 18:52:48 +0300
Subject: [PATCH 384/543] IGNITE-9649 Reworked and improved logging for
 critical paths of Ignite internal processes - Fixes #4830.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 9f546f8785c83c20d45f669046d064648a6c1789)

(cherry picked from commit a5ca24b)
---
 .../delta/PartitionMetaStateRecord.java       |   2 +-
 .../cache/CacheAffinitySharedManager.java     |  57 ++++-
 .../processors/cache/CacheGroupContext.java   |   5 +-
 .../cache/CacheGroupMetricsMXBeanImpl.java    |   4 +-
 .../processors/cache/CacheMetricsImpl.java    |   4 +-
 .../processors/cache/GridCacheAdapter.java    |   2 +-
 .../processors/cache/GridCacheContext.java    |   6 +-
 .../processors/cache/GridCacheEntryEx.java    |   2 +-
 .../processors/cache/GridCacheMapEntry.java   |   2 +-
 .../GridCachePartitionExchangeManager.java    |  71 +++---
 .../processors/cache/GridCacheProcessor.java  |  11 +-
 .../cache/GridCacheSharedContext.java         |   9 +-
 .../processors/cache/GridCacheUtils.java      |   2 +-
 .../cache/IgniteCacheOffheapManager.java      |   2 +-
 .../cache/IgniteCacheOffheapManagerImpl.java  |   6 +-
 .../processors/cache/WalStateManager.java     |   6 +-
 .../GridDistributedCacheAdapter.java          |   4 +-
 .../dht/CacheDistributedGetFutureAdapter.java |   4 +-
 .../GridCachePartitionedConcurrentMap.java    |   1 +
 .../distributed/dht/GridDhtCacheAdapter.java  |   3 +
 .../distributed/dht/GridDhtCacheEntry.java    |   1 +
 .../distributed/dht/GridDhtGetFuture.java     |   2 +
 .../dht/GridDhtGetSingleFuture.java           |   2 +
 .../distributed/dht/GridDhtLockFuture.java    |   1 +
 .../dht/GridDhtTransactionalCacheAdapter.java |   3 +
 .../dht/GridDhtTxLocalAdapter.java            |   1 +
 .../distributed/dht/GridDhtTxRemote.java      |   3 +
 .../dht/GridPartitionedGetFuture.java         |   1 +
 .../dht/GridPartitionedSingleGetFuture.java   |   3 +-
 .../dht/atomic/GridDhtAtomicCache.java        |   4 +-
 .../dht/colocated/GridDhtColocatedCache.java  |   2 +-
 .../dht/preloader/GridDhtForceKeysFuture.java |   8 +-
 .../preloader/GridDhtPartitionDemander.java   | 176 ++++++++------
 .../dht/preloader/GridDhtPartitionMap.java    |   4 +-
 .../preloader/GridDhtPartitionSupplier.java   | 229 ++++++++----------
 .../GridDhtPartitionsExchangeFuture.java      | 113 +++++++--
 .../GridDhtPartitionsFullMessage.java         |   2 +-
 .../GridDhtPartitionsSingleMessage.java       |   2 +-
 .../dht/preloader/GridDhtPreloader.java       |  30 +--
 .../GridDhtPreloaderAssignments.java          |   1 +
 .../IgniteDhtDemandedPartitionsMap.java       |  67 +----
 .../dht/{ => topology}/EvictionContext.java   |   2 +-
 .../GridClientPartitionTopology.java          |  10 +-
 .../GridDhtInvalidPartitionException.java     |   2 +-
 .../{ => topology}/GridDhtLocalPartition.java | 110 +++++----
 .../{ => topology}/GridDhtPartitionState.java |   2 +-
 .../GridDhtPartitionTopology.java             |   4 +-
 .../GridDhtPartitionTopologyImpl.java         | 203 +++++++++-------
 .../GridDhtPartitionsReservation.java         |   5 +-
 .../GridDhtPartitionsStateValidator.java      |  20 +-
 .../PartitionsEvictManager.java               |   7 +-
 .../distributed/near/GridNearAtomicCache.java |   2 +-
 .../distributed/near/GridNearGetFuture.java   |   2 +-
 .../GridNearPessimisticTxPrepareFuture.java   |   2 +-
 .../GridCacheDatabaseSharedManager.java       |  47 +++-
 .../persistence/GridCacheOffheapManager.java  |  10 +-
 .../checkpoint/CheckpointEntry.java           |   2 +-
 .../tree/io/PagePartitionMetaIO.java          |   2 +-
 .../serializer/RecordDataV1Serializer.java    |   2 +-
 .../cache/query/GridCacheQueryManager.java    |   4 +-
 .../CacheContinuousQueryHandler.java          |   2 +-
 .../cache/transactions/IgniteTxAdapter.java   |   2 +
 .../cache/transactions/IgniteTxHandler.java   |   6 +-
 .../transactions/IgniteTxLocalAdapter.java    |   3 +
 .../cache/transactions/IgniteTxManager.java   |   2 +-
 .../CollectConflictPartitionKeysTask.java     |   4 +-
 .../RetrieveConflictPartitionValuesTask.java  |   4 +-
 .../verify/VerifyBackupPartitionsTask.java    |   4 +-
 .../verify/VerifyBackupPartitionsTaskV2.java  |   4 +-
 .../datastreamer/DataStreamerImpl.java        |   6 +-
 .../processors/job/GridJobProcessor.java      |   4 +-
 .../schema/SchemaIndexCacheVisitorImpl.java   |  10 +-
 .../internal/util/GridPartitionStateMap.java  |   2 +-
 .../cache/VisorCacheLostPartitionsTask.java   |   2 +-
 .../visor/cache/VisorCachePartitionsTask.java |   6 +-
 .../visor/cache/VisorPartitionMap.java        |   2 +-
 modules/core/src/test/config/log4j-test.xml   |  17 +-
 .../cache/CacheDeferredDeleteQueueTest.java   |   4 +-
 ...eDhtLocalPartitionAfterRemoveSelfTest.java |   2 +-
 .../cache/IgniteCacheGroupsTest.java          |   3 +-
 .../IgniteClientCacheStartFailoverTest.java   |   2 +-
 .../CacheBaselineTopologyTest.java            |   4 +-
 .../CacheDataLossOnPartitionMoveTest.java     |   6 +-
 .../CachePageWriteLockUnlockTest.java         |   4 +-
 .../distributed/CachePartitionStateTest.java  |  10 +-
 .../CacheRentingStateRepairTest.java          |   6 +-
 ...CacheClientNodePartitionsExchangeTest.java |   2 +-
 .../GridCacheDhtPreloadDelayedSelfTest.java   |   2 +
 .../GridCacheDhtPreloadDisabledSelfTest.java  |   1 +
 .../dht/GridCacheDhtPreloadSelfTest.java      |   9 +-
 .../GridCacheDhtPreloadStartStopSelfTest.java |   4 +-
 ...dCachePartitionedUnloadEventsSelfTest.java |   1 +
 ...ridCachePartitionsStateValidationTest.java |   2 +
 ...CachePartitionsStateValidatorSelfTest.java |   4 +
 ...tomicInvalidPartitionHandlingSelfTest.java |   2 +-
 .../near/NoneRebalanceModeSelfTest.java       |   4 +-
 ...CacheRebalancingPartitionCountersTest.java |   4 +-
 .../GridCacheRebalancingSyncSelfTest.java     |   6 +-
 ...CacheRebalancingWithAsyncClearingTest.java |   6 +-
 .../IgniteCacheExpiryPolicyAbstractTest.java  |   2 +-
 ...acheExpiryPolicyWithStoreAbstractTest.java |   3 +-
 ...IgnitePdsCacheRebalancingAbstractTest.java |   2 +-
 .../IgnitePdsCorruptedIndexTest.java          |   3 +-
 .../IgnitePdsPartitionFilesDestroyTest.java   |   6 +-
 ...teAbsentEvictionNodeOutOfBaselineTest.java |   2 +-
 .../wal/IgniteWalHistoryReservationsTest.java |   2 +-
 .../wal/WalRecoveryTxLogicalRecordsTest.java  |   2 +-
 ...ntinuousQueryFailoverAbstractSelfTest.java |   2 +-
 .../hashmap/GridCacheTestContext.java         |   2 +-
 .../ignite/testframework/GridTestUtils.java   |   2 +-
 .../junits/common/GridCommonAbstractTest.java |   6 +-
 .../ignite/util/GridPartitionMapSelfTest.java |   2 +-
 .../h2/twostep/GridMapQueryExecutor.java      |  10 +-
 .../h2/twostep/GridReduceQueryExecutor.java   |   2 +-
 .../visor/verify/ValidateIndexesClosure.java  |   4 +-
 ...cheScanPartitionQueryFallbackSelfTest.java |   4 +-
 ...ockPartitionOnAffinityRunAbstractTest.java |   2 +-
 ...teCacheLockPartitionOnAffinityRunTest.java |   2 +-
 .../h2/twostep/RetryCauseMessageSelfTest.java |   2 +-
 .../util/GridCommandHandlerIndexingTest.java  |   2 +-
 .../impl/cache/CacheBasedDatasetTest.java     |   4 +-
 .../cache/WaitMapExchangeFinishCallable.java  |   4 +-
 .../IgniteFailoverAbstractBenchmark.java      |   3 +-
 ...niteTransactionalWriteInvokeBenchmark.java |   2 +-
 .../ZkCommunicationFailureContext.java        |   3 +-
 125 files changed, 906 insertions(+), 638 deletions(-)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/{ => topology}/EvictionContext.java (98%)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/{ => topology}/GridClientPartitionTopology.java (99%)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/{ => topology}/GridDhtInvalidPartitionException.java (99%)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/{ => topology}/GridDhtLocalPartition.java (94%)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/{ => topology}/GridDhtPartitionState.java (99%)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/{ => topology}/GridDhtPartitionTopology.java (98%)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/{ => topology}/GridDhtPartitionTopologyImpl.java (93%)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/{ => topology}/GridDhtPartitionsReservation.java (98%)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/{ => topology}/GridDhtPartitionsStateValidator.java (96%)
 rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/{ => topology}/PartitionsEvictManager.java (98%)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java
index a89f7be04b406..e1e74c49c51db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PartitionMetaStateRecord.java
@@ -19,7 +19,7 @@
 
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WalRecordCacheGroupAware;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 80b4204f0ccaf..02fc2894efa47 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -48,16 +48,16 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.ClientCacheDhtTopologyFuture;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAssignmentFetchFuture;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CacheGroupAffinityMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridClientPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
@@ -767,11 +767,16 @@ public IgniteInternalFuture<?> onCacheChangeRequest(
     ) throws IgniteCheckedException {
         assert exchActions != null && !exchActions.empty() : exchActions;
 
+        long time = System.currentTimeMillis();
+
         IgniteInternalFuture<?> res = cachesRegistry.update(exchActions);
 
         // Affinity did not change for existing caches.
         onCustomMessageNoAffinityChange(fut, crd, exchActions);
 
+        if (log.isInfoEnabled())
+            log.info("Updating caches registry performed in " + (System.currentTimeMillis() - time) + " ms.");
+
         processCacheStartRequests(fut, crd, exchActions);
 
         Set<Integer> stoppedGrps = processCacheStopRequests(fut, crd, exchActions, false);
@@ -833,6 +838,8 @@ private void processCacheStartRequests(
 
         final ExchangeDiscoveryEvents evts = fut.context().events();
 
+        long time = System.currentTimeMillis();
+
         for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) {
             DynamicCacheDescriptor cacheDesc = action.descriptor();
 
@@ -892,6 +899,11 @@ private void processCacheStartRequests(
             }
         }
 
+        if (log.isInfoEnabled())
+            log.info("Caches starting performed in " + (System.currentTimeMillis() - time) + " ms.");
+
+        time = System.currentTimeMillis();
+
         Set<Integer> gprs = new HashSet<>();
 
         for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) {
@@ -911,6 +923,9 @@ private void processCacheStartRequests(
                 }
             }
         }
+
+        if (log.isInfoEnabled())
+            log.info("Affinity initialization for started caches performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
@@ -1340,6 +1355,8 @@ public void applyAffinityFromFullMessage(final GridDhtPartitionsExchangeFuture f
 
         final Map<Object, List<List<ClusterNode>>> affCache = new HashMap<>();
 
+        long time = System.currentTimeMillis();
+
         forAllCacheGroups(false, new IgniteInClosureX<GridAffinityAssignmentCache>() {
             @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException {
                 ExchangeDiscoveryEvents evts = fut.context().events();
@@ -1373,6 +1390,9 @@ public void applyAffinityFromFullMessage(final GridDhtPartitionsExchangeFuture f
                 aff.initialize(evts.topologyVersion(), cachedAssignment(aff, newAssignment, affCache));
             }
         });
+
+        if (log.isInfoEnabled())
+            log.info("Affinity applying from full message performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
@@ -1395,6 +1415,8 @@ public void onLocalJoin(final GridDhtPartitionsExchangeFuture fut,
                 ", receivedCnt=" + (receivedAff != null ? receivedAff.size() : "none") +
                 ", msg=" + msg + "]");
 
+        long time = System.currentTimeMillis();
+
         forAllCacheGroups(false, new IgniteInClosureX<GridAffinityAssignmentCache>() {
             @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException {
                 ExchangeDiscoveryEvents evts = fut.context().events();
@@ -1434,6 +1456,9 @@ else if (fut.cacheGroupAddedOnExchange(aff.groupId(), grp.receivedFrom()))
                 grp.topology().initPartitionsWhenAffinityReady(resTopVer, fut);
             }
         });
+
+        if (log.isInfoEnabled())
+            log.info("Affinity initialization on local join performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
@@ -1448,6 +1473,8 @@ public void onServerJoinWithExchangeMergeProtocol(GridDhtPartitionsExchangeFutur
         assert fut.context().mergeExchanges();
         assert evts.hasServerJoin() && !evts.hasServerLeft();
 
+        long time = System.currentTimeMillis();
+
         WaitRebalanceInfo waitRebalanceInfo = initAffinityOnNodeJoin(fut, crd);
 
         this.waitInfo = waitRebalanceInfo != null && !waitRebalanceInfo.empty() ? waitRebalanceInfo : null;
@@ -1460,6 +1487,10 @@ public void onServerJoinWithExchangeMergeProtocol(GridDhtPartitionsExchangeFutur
                     ", waitGrps=" + (info != null ? groupNames(info.waitGrps.keySet()) : null) + ']');
             }
         }
+
+        if (log.isInfoEnabled())
+            log.info("Affinity recalculation (on server join) performed in "
+                + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
@@ -1470,12 +1501,20 @@ public void onServerJoinWithExchangeMergeProtocol(GridDhtPartitionsExchangeFutur
     public Map<Integer, CacheGroupAffinityMessage> onServerLeftWithExchangeMergeProtocol(
         final GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException
     {
+        long time = System.currentTimeMillis();
+
         final ExchangeDiscoveryEvents evts = fut.context().events();
 
         assert fut.context().mergeExchanges();
         assert evts.hasServerLeft();
 
-        return onReassignmentEnforced(fut);
+        Map<Integer, CacheGroupAffinityMessage> result = onReassignmentEnforced(fut);
+
+        if (log.isInfoEnabled())
+            log.info("Affinity recalculation (on server left) performed in "
+                + (System.currentTimeMillis() - time) + " ms.");
+
+        return result;
     }
 
     /**
@@ -1490,7 +1529,15 @@ public Map<Integer, CacheGroupAffinityMessage> onCustomEventWithEnforcedAffinity
     {
         assert DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent());
 
-        return onReassignmentEnforced(fut);
+        long time = System.currentTimeMillis();
+
+        Map<Integer, CacheGroupAffinityMessage> result = onReassignmentEnforced(fut);
+
+        if (log.isInfoEnabled())
+            log.info("Affinity recalculation (custom message) performed in "
+                + (System.currentTimeMillis() - time) + " ms.");
+
+        return result;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 434b8fb5dd9db..a7347498e04c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -39,8 +39,9 @@
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.PartitionsEvictManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
index 639569503f599..6a30cdf597bd7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupMetricsMXBeanImpl.java
@@ -30,8 +30,8 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
index 0f6d06f7edd56..43604107a4255 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
@@ -24,8 +24,8 @@
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
 import org.apache.ignite.internal.processors.cache.store.GridCacheWriteBehindStore;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 5121753207c89..fa3ec4543237b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -87,7 +87,7 @@
 import org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityImpl;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteExternalizableExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index c5f4d3c9c993a..fe96a376ad35b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -64,8 +64,8 @@
 import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTransactionalCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
@@ -116,7 +116,7 @@
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STARTED;
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  * Cache context.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index 6cddb2da10a06..e02b7754a65be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -26,7 +26,7 @@
 import org.apache.ignite.cache.eviction.EvictableEntry;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 1eac83b3319bd..26c6fee1ecf34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -41,7 +41,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult.UpdateOutcome;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
 import org.apache.ignite.internal.processors.cache.extras.GridCacheEntryExtras;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 335f953244994..c57553234f49f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -67,8 +67,6 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionFullCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
@@ -88,6 +86,9 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionsToReloadMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.RebalanceReassignExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridClientPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
@@ -900,8 +901,8 @@ public void lastFinishedFuture(GridDhtTopologyFuture fut) {
         GridDhtPartitionsExchangeFuture lastInitializedFut0 = lastInitializedFut;
 
         if (lastInitializedFut0 != null && lastInitializedFut0.initialVersion().compareTo(ver) == 0) {
-            if (log.isDebugEnabled())
-                log.debug("Return lastInitializedFut for topology ready future " +
+            if (log.isTraceEnabled())
+                log.trace("Return lastInitializedFut for topology ready future " +
                     "[ver=" + ver + ", fut=" + lastInitializedFut0 + ']');
 
             return lastInitializedFut0;
@@ -910,8 +911,8 @@ public void lastFinishedFuture(GridDhtTopologyFuture fut) {
         AffinityTopologyVersion topVer = exchFuts.readyTopVer();
 
         if (topVer.compareTo(ver) >= 0) {
-            if (log.isDebugEnabled())
-                log.debug("Return finished future for topology ready future [ver=" + ver + ", topVer=" + topVer + ']');
+            if (log.isTraceEnabled())
+                log.trace("Return finished future for topology ready future [ver=" + ver + ", topVer=" + topVer + ']');
 
             return null;
         }
@@ -925,8 +926,8 @@ public void lastFinishedFuture(GridDhtTopologyFuture fut) {
         topVer = exchFuts.readyTopVer();
 
         if (topVer.compareTo(ver) >= 0) {
-            if (log.isDebugEnabled())
-                log.debug("Completing created topology ready future " +
+            if (log.isTraceEnabled())
+                log.trace("Completing created topology ready future " +
                     "[ver=" + topVer + ", topVer=" + topVer + ", fut=" + fut + ']');
 
             fut.onDone(topVer);
@@ -1084,14 +1085,23 @@ public void refreshPartitions() {
      * @param nodes Nodes.
      * @param msgTopVer Topology version. Will be added to full message.
      */
-    private void sendAllPartitions(Collection<ClusterNode> nodes,
-        AffinityTopologyVersion msgTopVer) {
+    private void sendAllPartitions(
+        Collection<ClusterNode> nodes,
+        AffinityTopologyVersion msgTopVer
+    ) {
+        long time = System.currentTimeMillis();
+
         GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, false, null, null, null, null);
 
         m.topologyVersion(msgTopVer);
 
-        if (log.isDebugEnabled())
-            log.debug("Sending all partitions [nodeIds=" + U.nodeIds(nodes) + ", msg=" + m + ']');
+        if (log.isInfoEnabled())
+            log.info("Full Message creating for " + msgTopVer + " performed in " + (System.currentTimeMillis() - time) + " ms.");
+
+        if (log.isTraceEnabled())
+            log.trace("Sending all partitions [nodeIds=" + U.nodeIds(nodes) + ", msg=" + m + ']');
+
+        time = System.currentTimeMillis();
 
         for (ClusterNode node : nodes) {
             try {
@@ -1108,6 +1118,9 @@ private void sendAllPartitions(Collection<ClusterNode> nodes,
                 U.warn(log, "Failed to send partitions full message [node=" + node + ", err=" + e + ']');
             }
         }
+
+        if (log.isInfoEnabled())
+            log.info("Sending Full Message for " + msgTopVer + " performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
@@ -1255,8 +1268,8 @@ private void sendLocalPartitions(ClusterNode node, @Nullable GridDhtPartitionExc
             false,
             null);
 
-        if (log.isDebugEnabled())
-            log.debug("Sending local partitions [nodeId=" + node.id() + ", msg=" + m + ']');
+        if (log.isTraceEnabled())
+            log.trace("Sending local partitions [nodeId=" + node.id() + ", msg=" + m + ']');
 
         try {
             cctx.io().sendNoRetry(node, m, SYSTEM_POOL);
@@ -1568,8 +1581,8 @@ private void processSinglePartitionUpdate(final ClusterNode node, final GridDhtP
 
         try {
             if (msg.exchangeId() == null) {
-                if (log.isDebugEnabled())
-                    log.debug("Received local partition update [nodeId=" + node.id() + ", parts=" +
+                if (log.isTraceEnabled())
+                    log.trace("Received local partition update [nodeId=" + node.id() + ", parts=" +
                         msg + ']');
 
                 boolean updated = false;
@@ -1597,14 +1610,18 @@ else if (!grp.isLocal())
                     }
                 }
 
-                if (updated)
+                if (updated) {
+                    if (log.isDebugEnabled())
+                        log.debug("Partitions have been scheduled to resend [reason=Single update from " + node.id() + "]");
+
                     scheduleResendPartitions();
+                }
             }
             else {
                 GridDhtPartitionsExchangeFuture exchFut = exchangeFuture(msg.exchangeId());
 
-                if (log.isDebugEnabled())
-                    log.debug("Notifying exchange future about single message: " + exchFut);
+                if (log.isTraceEnabled())
+                    log.trace("Notifying exchange future about single message: " + exchFut);
 
                 if (msg.client()) {
                     AffinityTopologyVersion initVer = exchFut.initialVersion();
@@ -2479,7 +2496,7 @@ private void body0() throws InterruptedException, IgniteCheckedException {
                         timeout = cctx.gridConfig().getNetworkTimeout();
 
                     // After workers line up and before preloading starts we initialize all futures.
-                    if (log.isDebugEnabled()) {
+                    if (log.isTraceEnabled()) {
                         Collection<IgniteInternalFuture> unfinished = new HashSet<>();
 
                         for (GridDhtPartitionsExchangeFuture fut : exchFuts.values()) {
@@ -2487,7 +2504,7 @@ private void body0() throws InterruptedException, IgniteCheckedException {
                                 unfinished.add(fut);
                         }
 
-                        log.debug("Before waiting for exchange futures [futs" + unfinished + ", worker=" + this + ']');
+                        log.trace("Before waiting for exchange futures [futs" + unfinished + ", worker=" + this + ']');
                     }
 
                     // Take next exchange future.
@@ -2610,8 +2627,8 @@ else if (task instanceof ForceRebalanceExchangeTask) {
 
                             removeMergedFutures(resVer, exchFut);
 
-                            if (log.isDebugEnabled())
-                                log.debug("After waiting for exchange future [exchFut=" + exchFut + ", worker=" +
+                            if (log.isTraceEnabled())
+                                log.trace("After waiting for exchange future [exchFut=" + exchFut + ", worker=" +
                                     this + ']');
 
                             if (exchFut.exchangeId().nodeId().equals(cctx.localNodeId()))
@@ -2947,14 +2964,14 @@ private abstract class MessageHandler<M> implements IgniteBiInClosure<UUID, M> {
             ClusterNode node = cctx.node(nodeId);
 
             if (node == null) {
-                if (log.isDebugEnabled())
-                    log.debug("Received message from failed node [node=" + nodeId + ", msg=" + msg + ']');
+                if (log.isTraceEnabled())
+                    log.trace("Received message from failed node [node=" + nodeId + ", msg=" + msg + ']');
 
                 return;
             }
 
-            if (log.isDebugEnabled())
-                log.debug("Received message from node [node=" + nodeId + ", msg=" + msg + ']');
+            if (log.isTraceEnabled())
+                log.trace("Received message from node [node=" + nodeId + ", msg=" + msg + ']');
 
             onMessage(node, msg);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 57bc716eea551..bb86d976f7900 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -79,9 +79,9 @@
 import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionsEvictManager;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.PartitionsEvictManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
@@ -1820,6 +1820,8 @@ public IgniteInternalFuture<?> startCachesOnLocalJoin(
         AffinityTopologyVersion exchTopVer,
         LocalJoinCachesContext locJoinCtx
     ) throws IgniteCheckedException {
+        long time = System.currentTimeMillis();
+
         if (locJoinCtx == null)
             return new GridFinishedFuture<>();
 
@@ -1837,6 +1839,9 @@ public IgniteInternalFuture<?> startCachesOnLocalJoin(
                 false);
         }
 
+        if (log.isInfoEnabled())
+            log.info("Starting caches on local join performed in " + (System.currentTimeMillis() - time) + " ms.");
+
         return res;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index b0a06b5ce4856..c9cea0efa060f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -43,9 +43,9 @@
 import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
-import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionsEvictManager;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.PartitionsEvictManager;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.jta.CacheJtaManagerAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
@@ -267,6 +267,8 @@ public GridCacheSharedContext(
      * @throws IgniteCheckedException If failed.
      */
     public void activate() throws IgniteCheckedException {
+        long time = System.currentTimeMillis();
+
         if (!kernalCtx.clientNode())
             dbMgr.lock();
 
@@ -283,6 +285,9 @@ public void activate() throws IgniteCheckedException {
                 if (!kernalCtx.clientNode())
                     dbMgr.unLock();
             }
+
+            if (msgLog.isInfoEnabled())
+                msgLog.info("Components activation performed in " + (System.currentTimeMillis() - time) + " ms.");
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index 2520f65e54311..de674971d1cbc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -69,7 +69,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
index a12c0334912a5..b2a26b1102008 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
@@ -22,7 +22,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.RootPage;
 import org.apache.ignite.internal.processors.cache.persistence.RowStore;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 6ce8536688efd..5884da2019d4d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -36,8 +36,8 @@
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteHistoricalIterator;
@@ -78,7 +78,7 @@
 
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
index 16b92ce68789b..a01e813cbcc4c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateManager.java
@@ -40,7 +40,7 @@
 import org.apache.ignite.internal.managers.communication.GridMessageListener;
 import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CheckpointFuture;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
@@ -65,8 +65,8 @@
 
 import static org.apache.ignite.internal.GridTopic.TOPIC_WAL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  * Write-ahead log state manager. Manages WAL enable and disable.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheAdapter.java
index dc9e4ecc3a772..028733976b972 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedCacheAdapter.java
@@ -44,7 +44,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -60,7 +60,7 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
 
 /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheDistributedGetFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheDistributedGetFutureAdapter.java
index d9c4b3b8028a0..62b5fe9879c58 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheDistributedGetFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/CacheDistributedGetFutureAdapter.java
@@ -18,11 +18,9 @@
 package org.apache.ignite.internal.processors.cache.distributed.dht;
 
 import java.util.Collection;
-import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
-import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheCompoundIdentityFuture;
@@ -37,7 +35,7 @@
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_NEAR_GET_MAX_REMAPS;
 import static org.apache.ignite.IgniteSystemProperties.getInteger;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedConcurrentMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedConcurrentMap.java
index 3b41ffa067fe0..f34a40f75e623 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedConcurrentMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedConcurrentMap.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index ea99f5d0d8263..4708219992cb9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -62,6 +62,9 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysResponse;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.near.CacheVersionedValue;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
index fe02090a9c2c4..69d0c80453476 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheEntry.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.extras.GridCacheObsoleteEntryExtras;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
index 431937441bd28..e25e2b3985442 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetFuture.java
@@ -37,6 +37,8 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.ReaderArguments;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridCompoundIdentityFuture;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
index 7c6c0208a39db..6a489295a628d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtGetSingleFuture.java
@@ -35,6 +35,8 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.ReaderArguments;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.typedef.F;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
index 529d96575e827..71bcc253361f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockFuture.java
@@ -53,6 +53,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedLockFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index be6b6fb11b60a..3451e1f51371f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -50,6 +50,9 @@
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedUnlockRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysResponse;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
index 604fe0655801e..2665cf3ee5532 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareResponse;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
index 746eb387a9beb..ec023273d9c13 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
@@ -32,6 +32,9 @@
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxRemoteAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxRemoteSingleStateImpl;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index 204a0cecf4729..ad3b2a13b6c7e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -41,6 +41,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMessage;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetResponse;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index e0aea9a7d1b1d..73deb9fc833f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -42,6 +42,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheUtils.BackupPostProcessingClosure;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.near.CacheVersionedValue;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetResponse;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSingleGetRequest;
@@ -59,7 +60,7 @@
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.jetbrains.annotations.Nullable;
 
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 315cec796ca6c..df0ba8f4b0d89 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -66,8 +66,8 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedGetFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedSingleGetFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtForceKeysRequest;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 14d386648ef1c..bb2c7841151af 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -49,7 +49,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtEmbeddedFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFinishedFuture;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLockFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTransactionalCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridPartitionedGetFuture;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java
index e1591bb0df6a8..7fabbe15ea53e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysFuture.java
@@ -40,8 +40,8 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.F0;
 import org.apache.ignite.internal.util.GridLeanSet;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
@@ -58,8 +58,8 @@
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_OBJECT_LOADED;
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.dr.GridDrType.DR_NONE;
 import static org.apache.ignite.internal.processors.dr.GridDrType.DR_PRELOAD;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index ca828953c4c45..e52d8d88a3b84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -49,9 +49,9 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
 import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
@@ -75,7 +75,7 @@
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_LOADED;
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STARTED;
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
 import static org.apache.ignite.internal.processors.dr.GridDrType.DR_NONE;
 import static org.apache.ignite.internal.processors.dr.GridDrType.DR_PRELOAD;
 
@@ -428,8 +428,8 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
         }
 
         if (!ctx.kernalContext().grid().isRebalanceEnabled()) {
-            if (log.isDebugEnabled())
-                log.debug("Cancel partition demand because rebalance disabled on current node.");
+            if (log.isTraceEnabled())
+                log.trace("Cancel partition demand because rebalance disabled on current node.");
 
             fut.cancel();
 
@@ -455,6 +455,8 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
 
         final CacheConfiguration cfg = grp.config();
 
+        int totalStripes = ctx.gridConfig().getRebalanceThreadPoolSize();
+
         for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assignments.entrySet()) {
             final ClusterNode node = e.getKey();
 
@@ -467,13 +469,11 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
 
                 parts = fut.remaining.get(node.id()).get2();
 
-                U.log(log, "Starting rebalancing [grp=" + grp.cacheOrGroupName()
-                        + ", mode=" + cfg.getRebalanceMode() + ", fromNode=" + node.id() + ", partitionsCount=" + parts.size()
-                        + ", topology=" + fut.topologyVersion() + ", rebalanceId=" + fut.rebalanceId + "]");
+                U.log(log, "Prepared rebalancing [grp=" + grp.cacheOrGroupName()
+                        + ", mode=" + cfg.getRebalanceMode() + ", supplier=" + node.id() + ", partitionsCount=" + parts.size()
+                        + ", topVer=" + fut.topologyVersion() + ", parallelism=" + totalStripes + "]");
             }
 
-            int totalStripes = ctx.gridConfig().getRebalanceThreadPoolSize();
-
             int stripes = totalStripes;
 
             final List<IgniteDhtDemandedPartitionsMap> stripePartitions = new ArrayList<>(stripes);
@@ -521,10 +521,11 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
                                     fut.cleanupRemoteContexts(node.id());
                             }
 
-                            if (log.isDebugEnabled())
-                                log.debug("Requested rebalancing [from node=" + node.id() + ", listener index=" +
-                                    topicId + " " + demandMsg.rebalanceId() + ", partitions count=" + stripePartitions.get(topicId).size() +
-                                    " (" + stripePartitions.get(topicId).partitionsList() + ")]");
+                            if (log.isInfoEnabled())
+                                log.info("Started rebalance routine [" + grp.cacheOrGroupName() +
+                                    ", supplier=" + node.id() + ", topic=" + topicId +
+                                    ", fullPartitions=" + S.compact(stripePartitions.get(topicId).fullSet()) +
+                                    ", histPartitions=" + S.compact(stripePartitions.get(topicId).historicalSet()) + "]");
                         }
                         catch (IgniteCheckedException e1) {
                             ClusterTopologyCheckedException cause = e1.getCause(ClusterTopologyCheckedException.class);
@@ -613,16 +614,15 @@ private IgniteInternalFuture<?> clearFullPartitions(RebalanceFuture fut, Set<Int
                             }
 
                             if (log.isDebugEnabled())
-                                log.debug("Remaining clearing partitions [grp=" + grp.cacheOrGroupName()
-                                    + ", remaining=" + remaining + "]");
+                                log.debug("Partition is ready for rebalance [grp=" + grp.cacheOrGroupName()
+                                    + ", p=" + part.id() + ", remaining=" + remaining + "]");
 
                             if (remaining == 0)
                                 clearAllFuture.onDone();
                         }
                     }
-                    else {
+                    else
                         clearAllFuture.onDone();
-                    }
                 });
             }
             else {
@@ -636,10 +636,6 @@ private IgniteInternalFuture<?> clearFullPartitions(RebalanceFuture fut, Set<Int
                     }
                 }
 
-                if (log.isDebugEnabled())
-                    log.debug("Remaining clearing partitions [grp=" + grp.cacheOrGroupName()
-                        + ", remaining=" + remaining + "]");
-
                 if (remaining == 0)
                     clearAllFuture.onDone();
             }
@@ -660,38 +656,41 @@ private IgniteInternalFuture<?> clearFullPartitions(RebalanceFuture fut, Set<Int
      *
      * @param topicId Topic id.
      * @param nodeId Node id.
-     * @param supply Supply message.
+     * @param supplyMsg Supply message.
      */
     public void handleSupplyMessage(
         int topicId,
         final UUID nodeId,
-        final GridDhtPartitionSupplyMessage supply
+        final GridDhtPartitionSupplyMessage supplyMsg
     ) {
-        AffinityTopologyVersion topVer = supply.topologyVersion();
+        AffinityTopologyVersion topVer = supplyMsg.topologyVersion();
 
         final RebalanceFuture fut = rebalanceFut;
 
         ClusterNode node = ctx.node(nodeId);
 
-        if (node == null)
-            return;
+        if (node == null) {
+            if (log.isDebugEnabled())
+                log.debug("Supply message ignored (supplier has left cluster) [" + demandRoutineInfo(topicId, nodeId, supplyMsg) + "]");
 
-        if (topologyChanged(fut)) // Topology already changed (for the future that supply message based on).
             return;
+        }
+
+        // Topology already changed (for the future that supply message based on).
+        if (topologyChanged(fut) || !fut.isActual(supplyMsg.rebalanceId())) {
+            if (log.isDebugEnabled())
+                log.debug("Supply message ignored (topology changed) [" + demandRoutineInfo(topicId, nodeId, supplyMsg) + "]");
 
-        if (!fut.isActual(supply.rebalanceId())) {
-            // Current future have another rebalance id.
-            // Supple message based on another future.
             return;
         }
 
         if (log.isDebugEnabled())
-            log.debug("Received supply message [grp=" + grp.cacheOrGroupName() + ", msg=" + supply + ']');
+            log.debug("Received supply message [" + demandRoutineInfo(topicId, nodeId, supplyMsg) + "]");
 
         // Check whether there were error during supply message unmarshalling process.
-        if (supply.classError() != null) {
-            U.warn(log, "Rebalancing from node cancelled [grp=" + grp.cacheOrGroupName() + ", node=" + nodeId +
-                "]. Supply message couldn't be unmarshalled: " + supply.classError());
+        if (supplyMsg.classError() != null) {
+            U.warn(log, "Rebalancing from node cancelled [" + demandRoutineInfo(topicId, nodeId, supplyMsg) + "]" +
+                ". Supply message couldn't be unmarshalled: " + supplyMsg.classError());
 
             fut.cancel(nodeId);
 
@@ -699,9 +698,9 @@ public void handleSupplyMessage(
         }
 
         // Check whether there were error during supplying process.
-        if (supply.error() != null) {
-            U.warn(log, "Rebalancing from node cancelled [grp=" + grp.cacheOrGroupName() + ", node=" + nodeId +
-                "]. Supplier has failed with error: " + supply.error());
+        if (supplyMsg.error() != null) {
+            U.warn(log, "Rebalancing from node cancelled [" + demandRoutineInfo(topicId, nodeId, supplyMsg) + "]" +
+                "]. Supplier has failed with error: " + supplyMsg.error());
 
             fut.cancel(nodeId);
 
@@ -713,13 +712,13 @@ public void handleSupplyMessage(
         if (grp.sharedGroup()) {
             for (GridCacheContext cctx : grp.caches()) {
                 if (cctx.statisticsEnabled()) {
-                    long keysCnt = supply.keysForCache(cctx.cacheId());
+                    long keysCnt = supplyMsg.keysForCache(cctx.cacheId());
 
                     if (keysCnt != -1)
                         cctx.cache().metrics0().onRebalancingKeysCountEstimateReceived(keysCnt);
 
                     // Can not be calculated per cache.
-                    cctx.cache().metrics0().onRebalanceBatchReceived(supply.messageSize());
+                    cctx.cache().metrics0().onRebalanceBatchReceived(supplyMsg.messageSize());
                 }
             }
         }
@@ -727,10 +726,10 @@ public void handleSupplyMessage(
             GridCacheContext cctx = grp.singleCacheContext();
 
             if (cctx.statisticsEnabled()) {
-                if (supply.estimatedKeysCount() != -1)
-                    cctx.cache().metrics0().onRebalancingKeysCountEstimateReceived(supply.estimatedKeysCount());
+                if (supplyMsg.estimatedKeysCount() != -1)
+                    cctx.cache().metrics0().onRebalancingKeysCountEstimateReceived(supplyMsg.estimatedKeysCount());
 
-                cctx.cache().metrics0().onRebalanceBatchReceived(supply.messageSize());
+                cctx.cache().metrics0().onRebalanceBatchReceived(supplyMsg.messageSize());
             }
         }
 
@@ -738,7 +737,7 @@ public void handleSupplyMessage(
             AffinityAssignment aff = grp.affinity().cachedAffinity(topVer);
 
             // Preload.
-            for (Map.Entry<Integer, CacheEntryInfoCollection> e : supply.infos().entrySet()) {
+            for (Map.Entry<Integer, CacheEntryInfoCollection> e : supplyMsg.infos().entrySet()) {
                 int p = e.getKey();
 
                 if (aff.get(p).contains(ctx.localNode())) {
@@ -746,7 +745,7 @@ public void handleSupplyMessage(
 
                     assert part != null;
 
-                    boolean last = supply.last().containsKey(p);
+                    boolean last = supplyMsg.last().containsKey(p);
 
                     if (part.state() == MOVING) {
                         boolean reserved = part.reserve();
@@ -795,7 +794,8 @@ public void handleSupplyMessage(
                                 fut.partitionDone(nodeId, p, true);
 
                                 if (log.isDebugEnabled())
-                                    log.debug("Finished rebalancing partition: " + part);
+                                    log.debug("Finished rebalancing partition: " +
+                                        "[" + demandRoutineInfo(topicId, nodeId, supplyMsg) + ", p=" + p + "]");
                             }
                         }
                         finally {
@@ -808,29 +808,31 @@ public void handleSupplyMessage(
                             fut.partitionDone(nodeId, p, false);
 
                         if (log.isDebugEnabled())
-                            log.debug("Skipping rebalancing partition (state is not MOVING): " + part);
+                            log.debug("Skipping rebalancing partition (state is not MOVING): " +
+                                "[" + demandRoutineInfo(topicId, nodeId, supplyMsg) + ", p=" + p + "]");
                     }
                 }
                 else {
                     fut.partitionDone(nodeId, p, false);
 
                     if (log.isDebugEnabled())
-                        log.debug("Skipping rebalancing partition (it does not belong on current node): " + p);
+                        log.debug("Skipping rebalancing partition (affinity changed): " +
+                            "[" + demandRoutineInfo(topicId, nodeId, supplyMsg) + ", p=" + p + "]");
                 }
             }
 
             // Only request partitions based on latest topology version.
-            for (Integer miss : supply.missed()) {
+            for (Integer miss : supplyMsg.missed()) {
                 if (aff.get(miss).contains(ctx.localNode()))
                     fut.partitionMissed(nodeId, miss);
             }
 
-            for (Integer miss : supply.missed())
+            for (Integer miss : supplyMsg.missed())
                 fut.partitionDone(nodeId, miss, false);
 
             GridDhtPartitionDemandMessage d = new GridDhtPartitionDemandMessage(
-                supply.rebalanceId(),
-                supply.topologyVersion(),
+                supplyMsg.rebalanceId(),
+                supplyMsg.topologyVersion(),
                 grp.groupId());
 
             d.timeout(grp.config().getRebalanceTimeout());
@@ -842,18 +844,24 @@ public void handleSupplyMessage(
                 try {
                     ctx.io().sendOrderedMessage(node, rebalanceTopics.get(topicId),
                         d.convertIfNeeded(node.version()), grp.ioPolicy(), grp.config().getRebalanceTimeout());
+
+                    if (log.isDebugEnabled())
+                        log.debug("Send next demand message [" + demandRoutineInfo(topicId, nodeId, supplyMsg) + "]");
                 }
                 catch (ClusterTopologyCheckedException e) {
-                    if (log.isDebugEnabled()) {
-                        log.debug("Node left during rebalancing [grp=" + grp.cacheOrGroupName() +
-                            ", node=" + node.id() + ", msg=" + e.getMessage() + ']');
-                    }
+                    if (log.isDebugEnabled())
+                        log.debug("Supplier has left [" + demandRoutineInfo(topicId, nodeId, supplyMsg) +
+                            ", errMsg=" + e.getMessage() + ']');
                 }
             }
+            else {
+                if (log.isDebugEnabled())
+                    log.debug("Will not request next demand message [" + demandRoutineInfo(topicId, nodeId, supplyMsg) +
+                        ", topChanged=" + topologyChanged(fut) + ", rebalanceFuture=" + fut + "]");
+            }
         }
         catch (IgniteSpiException | IgniteCheckedException e) {
-            LT.error(log, e, "Error during rebalancing [grp=" + grp.cacheOrGroupName() +
-                ", srcNode=" + node.id() +
+            LT.error(log, e, "Error during rebalancing [" + demandRoutineInfo(topicId, nodeId, supplyMsg) +
                 ", err=" + e + ']');
         }
     }
@@ -939,6 +947,17 @@ else if (log.isTraceEnabled())
         return true;
     }
 
+    /**
+     * String representation of demand routine.
+     *
+     * @param topicId Topic id.
+     * @param supplier Supplier.
+     * @param supplyMsg Supply message.
+     */
+    private String demandRoutineInfo(int topicId, UUID supplier, GridDhtPartitionSupplyMessage supplyMsg) {
+        return "grp=" + grp.cacheOrGroupName() + ", topVer=" + supplyMsg.topologyVersion() + ", supplier=" + supplier + ", topic=" + topicId;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridDhtPartitionDemander.class, this);
@@ -973,6 +992,9 @@ public static class RebalanceFuture extends GridFutureAdapter<Boolean> {
         /** Unique (per demander) rebalance id. */
         private final long rebalanceId;
 
+        /** The number of rebalance routines. */
+        private final long routines;
+
         /**
          * @param grp Cache group.
          * @param assignments Assignments.
@@ -989,6 +1011,15 @@ public static class RebalanceFuture extends GridFutureAdapter<Boolean> {
             exchId = assignments.exchangeId();
             topVer = assignments.topologyVersion();
 
+            assignments.forEach((k, v) -> {
+                assert v.partitions() != null :
+                    "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + k.id() + "]";
+
+                remaining.put(k.id(), new T2<>(U.currentTimeMillis(), v.partitions()));
+            });
+
+            this.routines = remaining.size();
+
             this.grp = grp;
             this.log = log;
             this.rebalanceId = rebalanceId;
@@ -1006,6 +1037,7 @@ public static class RebalanceFuture extends GridFutureAdapter<Boolean> {
             this.grp = null;
             this.log = null;
             this.rebalanceId = -1;
+            this.routines = 0;
         }
 
         /**
@@ -1040,7 +1072,8 @@ private boolean isInitial() {
                 if (isDone())
                     return true;
 
-                U.log(log, "Cancelled rebalancing from all nodes [topology=" + topologyVersion() + ']');
+                U.log(log, "Cancelled rebalancing from all nodes [grp=" + grp.cacheOrGroupName() +
+                    ", topVer=" + topologyVersion() + "]");
 
                 if (!ctx.kernalContext().isStopping()) {
                     for (UUID nodeId : remaining.keySet())
@@ -1063,8 +1096,8 @@ private void cancel(UUID nodeId) {
                 if (isDone())
                     return;
 
-                U.log(log, ("Cancelled rebalancing [cache=" + grp.cacheOrGroupName() +
-                    ", fromNode=" + nodeId + ", topology=" + topologyVersion() +
+                U.log(log, ("Cancelled rebalancing [grp=" + grp.cacheOrGroupName() +
+                    ", supplier=" + nodeId + ", topVer=" + topologyVersion() +
                     ", time=" + (U.currentTimeMillis() - remaining.get(nodeId).get1()) + " ms]"));
 
                 cleanupRemoteContexts(nodeId);
@@ -1120,7 +1153,7 @@ private void cleanupRemoteContexts(UUID nodeId) {
             }
             catch (IgniteCheckedException ignored) {
                 if (log.isDebugEnabled())
-                    log.debug("Failed to send failover context cleanup request to node");
+                    log.debug("Failed to send failover context cleanup request to node " + nodeId);
             }
         }
 
@@ -1152,11 +1185,14 @@ private void partitionDone(UUID nodeId, int p, boolean updateState) {
                     ", part=" + p + ", left=" + parts + "]";
 
                 if (parts.isEmpty()) {
-                    U.log(log, "Completed " + ((remaining.size() == 1 ? "(final) " : "") +
-                            "rebalancing [fromNode=" + nodeId +
-                            ", cacheOrGroup=" + grp.cacheOrGroupName() +
-                            ", topology=" + topologyVersion() +
-                        ", time=" + (U.currentTimeMillis() - t.get1()) + " ms]"));
+                    int remainingRoutines = remaining.size() - 1;
+
+                    U.log(log, "Completed " + ((remainingRoutines == 0 ? "(final) " : "") +
+                            "rebalancing [grp=" + grp.cacheOrGroupName() +
+                            ", supplier=" + nodeId +
+                            ", topVer=" + topologyVersion() +
+                            ", progress=" + (routines - remainingRoutines) + "/" + routines +
+                            ", time=" + (U.currentTimeMillis() - t.get1()) + " ms]"));
 
                     remaining.remove(nodeId);
                 }
@@ -1201,6 +1237,10 @@ private void checkIsDone(boolean cancelled) {
                 if (log.isInfoEnabled())
                     log.info("Completed rebalance future: " + this);
 
+                if (log.isDebugEnabled())
+                    log.debug("Partitions have been scheduled to resend [reason=" +
+                        "Rebalance is done [grp=" + grp.cacheOrGroupName() + "]");
+
                 ctx.exchange().scheduleResendPartitions();
 
                 Collection<Integer> m = new HashSet<>();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java
index 28c8c8453faad..e84869de7d168 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionMap.java
@@ -27,12 +27,12 @@
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
 
 /**
  * Partition map.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 131d16fc8aa72..5b5f765517fe1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -32,9 +32,9 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
 import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.T3;
@@ -43,7 +43,7 @@
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.IgniteSpiException;
 
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  * Class for supplying partitions to demanding nodes.
@@ -100,9 +100,7 @@ void stop() {
      * @param sc Supply context.
      * @param log Logger.
      */
-    private static void clearContext(
-        final SupplyContext sc,
-        final IgniteLogger log) {
+    private static void clearContext(SupplyContext sc, IgniteLogger log) {
         if (sc != null) {
             final IgniteRebalanceIterator it = sc.iterator;
 
@@ -136,7 +134,7 @@ void onTopologyChanged(AffinityTopologyVersion topVer) {
                     it.remove();
 
                     if (log.isDebugEnabled())
-                        log.debug("Supply context removed [node=" + t.get1() + "]");
+                        log.debug("Supply context removed [grp=" + grp.cacheOrGroupName() + ", demander=" + t.get1() + "]");
                 }
             }
         }
@@ -161,47 +159,29 @@ void preloadPredicate(IgnitePredicate<GridCacheEntryInfo> preloadPred) {
      *
      * @param topicId Id of the topic is used for the supply-demand communication.
      * @param nodeId Id of the node which sent the demand message.
-     * @param d Demand message.
+     * @param demandMsg Demand message.
      */
     @SuppressWarnings("unchecked")
-    public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemandMessage d) {
-        assert d != null;
+    public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemandMessage demandMsg) {
+        assert demandMsg != null;
         assert nodeId != null;
 
-        AffinityTopologyVersion curTop = grp.affinity().lastVersion();
-        AffinityTopologyVersion demTop = d.topologyVersion();
+        T3<UUID, Integer, AffinityTopologyVersion> contextId = new T3<>(nodeId, topicId, demandMsg.topologyVersion());
 
-        if (curTop.compareTo(demTop) > 0) {
-            if (log.isDebugEnabled())
-                log.debug("Demand request outdated [grp=" + grp.cacheOrGroupName()
-                        + ", currentTopVer=" + curTop
-                        + ", demandTopVer=" + demTop
-                        + ", from=" + nodeId
-                        + ", topicId=" + topicId + "]");
-
-            return;
-        }
-
-        T3<UUID, Integer, AffinityTopologyVersion> contextId = new T3<>(nodeId, topicId, demTop);
-
-        if (d.rebalanceId() < 0) { // Demand node requested context cleanup.
+        if (demandMsg.rebalanceId() < 0) { // Demand node requested context cleanup.
             synchronized (scMap) {
                 SupplyContext sctx = scMap.get(contextId);
 
-                if (sctx != null && sctx.rebalanceId == -d.rebalanceId()) {
+                if (sctx != null && sctx.rebalanceId == -demandMsg.rebalanceId()) {
                     clearContext(scMap.remove(contextId), log);
 
                     if (log.isDebugEnabled())
-                        log.debug("Supply context cleaned [grp=" + grp.cacheOrGroupName()
-                            + ", from=" + nodeId
-                            + ", demandMsg=" + d
+                        log.debug("Supply context cleaned [" + supplyRoutineInfo(topicId, nodeId, demandMsg)
                             + ", supplyContext=" + sctx + "]");
                 }
                 else {
                     if (log.isDebugEnabled())
-                        log.debug("Stale supply context cleanup message [grp=" + grp.cacheOrGroupName()
-                            + ", from=" + nodeId
-                            + ", demandMsg=" + d
+                        log.debug("Stale supply context cleanup message [" + supplyRoutineInfo(topicId, nodeId, demandMsg)
                             + ", supplyContext=" + sctx + "]");
                 }
 
@@ -209,17 +189,15 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
             }
         }
 
-        if (log.isDebugEnabled())
-            log.debug("Demand request accepted [grp=" + grp.cacheOrGroupName()
-                + ", from=" + nodeId
-                + ", currentVer=" + curTop
-                + ", demandedVer=" + demTop
-                + ", topicId=" + topicId + "]");
+        ClusterNode demanderNode = grp.shared().discovery().node(nodeId);
 
-        ClusterNode node = grp.shared().discovery().node(nodeId);
+        if (demanderNode == null) {
+            if (log.isDebugEnabled())
+                log.debug("Demand message rejected (demander left cluster) ["
+                    + supplyRoutineInfo(topicId, nodeId, demandMsg) + "]");
 
-        if (node == null)
             return;
+        }
 
         IgniteRebalanceIterator iter = null;
 
@@ -229,62 +207,59 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
             synchronized (scMap) {
                 sctx = scMap.remove(contextId);
 
-                if (sctx != null && d.rebalanceId() < sctx.rebalanceId) {
+                if (sctx != null && demandMsg.rebalanceId() < sctx.rebalanceId) {
                     // Stale message, return context back and return.
                     scMap.put(contextId, sctx);
 
                     if (log.isDebugEnabled())
-                        log.debug("Stale demand message [cache=" + grp.cacheOrGroupName()
-                            + ", actualContext=" + sctx
-                            + ", from=" + nodeId
-                            + ", demandMsg=" + d + "]");
+                        log.debug("Stale demand message [" + supplyRoutineInfo(topicId, nodeId, demandMsg) +
+                            ", actualContext=" + sctx + "]");
 
                     return;
                 }
             }
 
             // Demand request should not contain empty partitions if no supply context is associated with it.
-            if (sctx == null && (d.partitions() == null || d.partitions().isEmpty())) {
+            if (sctx == null && (demandMsg.partitions() == null || demandMsg.partitions().isEmpty())) {
                 if (log.isDebugEnabled())
-                    log.debug("Empty demand message [cache=" + grp.cacheOrGroupName()
-                        + ", from=" + nodeId
-                        + ", topicId=" + topicId
-                        + ", demandMsg=" + d + "]");
+                    log.debug("Empty demand message (no context and partitions) ["
+                        + supplyRoutineInfo(topicId, nodeId, demandMsg) + "]");
 
                 return;
             }
 
-            assert !(sctx != null && !d.partitions().isEmpty());
+            if (log.isDebugEnabled())
+                log.debug("Demand message accepted ["
+                    + supplyRoutineInfo(topicId, nodeId, demandMsg) + "]");
 
-            long batchesCnt = 0;
+            assert !(sctx != null && !demandMsg.partitions().isEmpty());
 
             long maxBatchesCnt = grp.config().getRebalanceBatchesPrefetchCount();
 
-            if (sctx != null) {
-                maxBatchesCnt = 1;
-            }
-            else {
+            if (sctx == null) {
                 if (log.isDebugEnabled())
-                    log.debug("Starting supplying rebalancing [cache=" + grp.cacheOrGroupName() +
-                        ", fromNode=" + node.id() + ", partitionsCount=" + d.partitions().size() +
-                        ", topology=" + demTop + ", rebalanceId=" + d.rebalanceId() +
-                        ", topicId=" + topicId + "]");
+                    log.debug("Starting supplying rebalancing [" + supplyRoutineInfo(topicId, nodeId, demandMsg) +
+                        ", fullPartitions=" + S.compact(demandMsg.partitions().fullSet()) +
+                        ", histPartitions=" + S.compact(demandMsg.partitions().historicalSet()) + "]");
             }
+            else
+                maxBatchesCnt = 1;
 
-            GridDhtPartitionSupplyMessage s = new GridDhtPartitionSupplyMessage(
-                    d.rebalanceId(),
+            GridDhtPartitionSupplyMessage supplyMsg = new GridDhtPartitionSupplyMessage(
+                    demandMsg.rebalanceId(),
                     grp.groupId(),
-                    d.topologyVersion(),
-                    grp.deploymentEnabled());
+                    demandMsg.topologyVersion(),
+                    grp.deploymentEnabled()
+            );
 
             Set<Integer> remainingParts;
 
             if (sctx == null || sctx.iterator == null) {
-                iter = grp.offheap().rebalanceIterator(d.partitions(), d.topologyVersion());
+                iter = grp.offheap().rebalanceIterator(demandMsg.partitions(), demandMsg.topologyVersion());
 
-                remainingParts = new HashSet<>(d.partitions().fullSet());
+                remainingParts = new HashSet<>(demandMsg.partitions().fullSet());
 
-                CachePartitionPartialCountersMap histMap = d.partitions().historicalMap();
+                CachePartitionPartialCountersMap histMap = demandMsg.partitions().historicalMap();
 
                 for (int i = 0; i < histMap.size(); i++) {
                     int p = histMap.partitionAt(i);
@@ -292,16 +267,16 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                     remainingParts.add(p);
                 }
 
-                for (Integer part : d.partitions().fullSet()) {
+                for (Integer part : demandMsg.partitions().fullSet()) {
                     if (iter.isPartitionMissing(part))
                         continue;
 
-                    GridDhtLocalPartition loc = top.localPartition(part, d.topologyVersion(), false);
+                    GridDhtLocalPartition loc = top.localPartition(part, demandMsg.topologyVersion(), false);
 
                     assert loc != null && loc.state() == GridDhtPartitionState.OWNING
                         : "Partition should be in OWNING state: " + loc;
 
-                    s.addEstimatedKeysCount(grp.offheap().totalPartitionEntriesCount(part));
+                    supplyMsg.addEstimatedKeysCount(grp.offheap().totalPartitionEntriesCount(part));
                 }
 
                 for (int i = 0; i < histMap.size(); i++) {
@@ -310,7 +285,7 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                     if (iter.isPartitionMissing(p))
                         continue;
 
-                    s.addEstimatedKeysCount(histMap.updateCounterAt(i) - histMap.initialUpdateCounterAt(i));
+                    supplyMsg.addEstimatedKeysCount(histMap.updateCounterAt(i) - histMap.initialUpdateCounterAt(i));
                 }
             }
             else {
@@ -319,28 +294,30 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                 remainingParts = sctx.remainingParts;
             }
 
-            final int messageMaxSize = grp.config().getRebalanceBatchSize();
+            final int msgMaxSize = grp.config().getRebalanceBatchSize();
+
+            long batchesCnt = 0;
 
             while (iter.hasNext()) {
-                if (s.messageSize() >= messageMaxSize) {
+                if (supplyMsg.messageSize() >= msgMaxSize) {
                     if (++batchesCnt >= maxBatchesCnt) {
                         saveSupplyContext(contextId,
                             iter,
                             remainingParts,
-                            d.rebalanceId()
+                            demandMsg.rebalanceId()
                         );
 
-                        reply(node, d, s, contextId);
+                        reply(topicId, demanderNode, demandMsg, supplyMsg, contextId);
 
                         return;
                     }
                     else {
-                        if (!reply(node, d, s, contextId))
+                        if (!reply(topicId, demanderNode, demandMsg, supplyMsg, contextId))
                             return;
 
-                        s = new GridDhtPartitionSupplyMessage(d.rebalanceId(),
+                        supplyMsg = new GridDhtPartitionSupplyMessage(demandMsg.rebalanceId(),
                             grp.groupId(),
-                            d.topologyVersion(),
+                            demandMsg.topologyVersion(),
                             grp.deploymentEnabled());
                     }
                 }
@@ -349,19 +326,19 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
 
                 int part = row.partition();
 
-                GridDhtLocalPartition loc = top.localPartition(part, d.topologyVersion(), false);
+                GridDhtLocalPartition loc = top.localPartition(part, demandMsg.topologyVersion(), false);
 
                 assert (loc != null && loc.state() == OWNING && loc.reservations() > 0) || iter.isPartitionMissing(part)
                     : "Partition should be in OWNING state and has at least 1 reservation " + loc;
 
                 if (iter.isPartitionMissing(part) && remainingParts.contains(part)) {
-                    s.missed(part);
+                    supplyMsg.missed(part);
 
                     remainingParts.remove(part);
 
                     if (log.isDebugEnabled())
-                        log.debug("Requested partition is marked as missing on local node [part=" + part +
-                            ", demander=" + nodeId + ']');
+                        log.debug("Requested partition is marked as missing ["
+                            + supplyRoutineInfo(topicId, nodeId, demandMsg) + ", p=" + part + "]");
 
                     continue;
                 }
@@ -378,7 +355,7 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                 info.cacheId(row.cacheId());
 
                 if (preloadPred == null || preloadPred.apply(info))
-                    s.addEntry0(part, iter.historical(part), info, grp.shared(), grp.cacheObjectContext());
+                    supplyMsg.addEntry0(part, iter.historical(part), info, grp.shared(), grp.cacheObjectContext());
                 else {
                     if (log.isTraceEnabled())
                         log.trace("Rebalance predicate evaluated to false (will not send " +
@@ -386,7 +363,7 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                 }
 
                 if (iter.isPartitionDone(part)) {
-                    s.last(part, loc.updateCounter());
+                    supplyMsg.last(part, loc.updateCounter());
 
                     remainingParts.remove(part);
                 }
@@ -398,17 +375,17 @@ public void handleDemandMessage(int topicId, UUID nodeId, GridDhtPartitionDemand
                 int p = remainingIter.next();
 
                 if (iter.isPartitionDone(p)) {
-                    GridDhtLocalPartition loc = top.localPartition(p, d.topologyVersion(), false);
+                    GridDhtLocalPartition loc = top.localPartition(p, demandMsg.topologyVersion(), false);
 
                     assert loc != null
                         : "Supply partition is gone: grp=" + grp.cacheOrGroupName() + ", p=" + p;
 
-                    s.last(p, loc.updateCounter());
+                    supplyMsg.last(p, loc.updateCounter());
 
                     remainingIter.remove();
                 }
                 else if (iter.isPartitionMissing(p)) {
-                    s.missed(p);
+                    supplyMsg.missed(p);
 
                     remainingIter.remove();
                 }
@@ -422,32 +399,28 @@ else if (iter.isPartitionMissing(p)) {
             else
                 iter.close();
 
-            reply(node, d, s, contextId);
+            reply(topicId, demanderNode, demandMsg, supplyMsg, contextId);
 
-            if (log.isDebugEnabled())
-                log.debug("Finished supplying rebalancing [cache=" + grp.cacheOrGroupName() +
-                    ", fromNode=" + node.id() +
-                    ", topology=" + demTop + ", rebalanceId=" + d.rebalanceId() +
-                    ", topicId=" + topicId + "]");
+            if (log.isInfoEnabled())
+                log.info("Finished supplying rebalancing [" + supplyRoutineInfo(topicId, nodeId, demandMsg) + "]");
         }
         catch (Throwable t) {
             if (grp.shared().kernalContext().isStopping())
                 return;
 
             // Sending supply messages with error requires new protocol.
-            boolean sendErrMsg = node.version().compareTo(GridDhtPartitionSupplyMessageV2.AVAILABLE_SINCE) >= 0;
+            boolean sendErrMsg = demanderNode.version().compareTo(GridDhtPartitionSupplyMessageV2.AVAILABLE_SINCE) >= 0;
 
             if (t instanceof IgniteSpiException) {
                 if (log.isDebugEnabled())
-                    log.debug("Failed to send message to node (current node is stopping?) [node=" + node.id() +
-                        ", msg=" + t.getMessage() + ']');
+                    log.debug("Failed to send message to node (current node is stopping?) ["
+                        + supplyRoutineInfo(topicId, nodeId, demandMsg) + ", msg=" + t.getMessage() + ']');
 
                 sendErrMsg = false;
             }
             else
-                U.error(log, "Failed to continue supplying process for " +
-                    "[cache=" + grp.cacheOrGroupName() + ", node=" + nodeId
-                    + ", topicId=" + contextId.get2() + ", topVer=" + contextId.get3() + "]", t);
+                U.error(log, "Failed to continue supplying ["
+                    + supplyRoutineInfo(topicId, nodeId, demandMsg) + "]", t);
 
             try {
                 if (sctx != null)
@@ -456,9 +429,8 @@ else if (iter != null)
                     iter.close();
             }
             catch (Throwable t1) {
-                U.error(log, "Failed to cleanup supplying context " +
-                        "[cache=" + grp.cacheOrGroupName() + ", node=" + nodeId
-                        + ", topicId=" + contextId.get2() + ", topVer=" + contextId.get3() + "]", t1);
+                U.error(log, "Failed to cleanup supplying context ["
+                    + supplyRoutineInfo(topicId, nodeId, demandMsg) + "]", t1);
             }
 
             if (!sendErrMsg)
@@ -466,19 +438,18 @@ else if (iter != null)
 
             try {
                 GridDhtPartitionSupplyMessageV2 errMsg = new GridDhtPartitionSupplyMessageV2(
-                    d.rebalanceId(),
+                    demandMsg.rebalanceId(),
                     grp.groupId(),
-                    d.topologyVersion(),
+                    demandMsg.topologyVersion(),
                     grp.deploymentEnabled(),
                     t
                 );
 
-                reply(node, d, errMsg, contextId);
+                reply(topicId, demanderNode, demandMsg, errMsg, contextId);
             }
             catch (Throwable t1) {
-                U.error(log, "Failed to send supply error message for " +
-                    "[cache=" + grp.cacheOrGroupName() + ", node=" + nodeId
-                        + ", topicId=" + contextId.get2() + ", topVer=" + contextId.get3() + "]", t1);
+                U.error(log, "Failed to send supply error message ["
+                    + supplyRoutineInfo(topicId, nodeId, demandMsg) + "]", t1);
             }
         }
     }
@@ -486,23 +457,25 @@ else if (iter != null)
     /**
      * Sends supply message to demand node.
      *
-     * @param node Recipient of supply message.
-     * @param d Demand message.
-     * @param s Supply message.
+     * @param demander Recipient of supply message.
+     * @param demandMsg Demand message.
+     * @param supplyMsg Supply message.
      * @param contextId Supply context id.
      * @return {@code True} if message was sent, {@code false} if recipient left grid.
      * @throws IgniteCheckedException If failed.
      */
-    private boolean reply(ClusterNode node,
-        GridDhtPartitionDemandMessage d,
-        GridDhtPartitionSupplyMessage s,
-        T3<UUID, Integer, AffinityTopologyVersion> contextId)
-        throws IgniteCheckedException {
+    private boolean reply(
+        int topicId,
+        ClusterNode demander,
+        GridDhtPartitionDemandMessage demandMsg,
+        GridDhtPartitionSupplyMessage supplyMsg,
+        T3<UUID, Integer, AffinityTopologyVersion> contextId
+    ) throws IgniteCheckedException {
         try {
             if (log.isDebugEnabled())
-                log.debug("Replying to partition demand [node=" + node.id() + ", demand=" + d + ", supply=" + s + ']');
+                log.debug("Send next supply message [" + supplyRoutineInfo(topicId, demander.id(), demandMsg) + "]");
 
-            grp.shared().io().sendOrderedMessage(node, d.topic(), s, grp.ioPolicy(), d.timeout());
+            grp.shared().io().sendOrderedMessage(demander, demandMsg.topic(), supplyMsg, grp.ioPolicy(), demandMsg.timeout());
 
             // Throttle preloading.
             if (grp.config().getRebalanceThrottle() > 0)
@@ -512,7 +485,7 @@ private boolean reply(ClusterNode node,
         }
         catch (ClusterTopologyCheckedException ignore) {
             if (log.isDebugEnabled())
-                log.debug("Failed to send partition supply message because node left grid: " + node.id());
+                log.debug("Failed to send supply message (demander left): [" + supplyRoutineInfo(topicId, demander.id(), demandMsg) + "]");
 
             synchronized (scMap) {
                 clearContext(scMap.remove(contextId), log);
@@ -522,6 +495,17 @@ private boolean reply(ClusterNode node,
         }
     }
 
+    /**
+     * String representation of supply routine.
+     *
+     * @param topicId Topic id.
+     * @param demander Demander.
+     * @param demandMsg Demand message.
+     */
+    private String supplyRoutineInfo(int topicId, UUID demander, GridDhtPartitionDemandMessage demandMsg) {
+        return "grp=" + grp.cacheOrGroupName() + ", demander=" + demander + ", topVer=" + demandMsg.topologyVersion() + ", topic=" + topicId;
+    }
+
     /**
      * Saves supply context with given parameters to {@code scMap}.
      *
@@ -534,7 +518,8 @@ private void saveSupplyContext(
         T3<UUID, Integer, AffinityTopologyVersion> contextId,
         IgniteRebalanceIterator entryIt,
         Set<Integer> remainingParts,
-        long rebalanceId) {
+        long rebalanceId
+    ) {
         synchronized (scMap) {
             assert scMap.get(contextId) == null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index bb8b2d2ee98f5..373aac4216703 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -82,11 +82,11 @@
 import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
 import org.apache.ignite.internal.processors.cache.StateChangeRequest;
 import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridClientPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsStateValidator;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridClientPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
@@ -1346,8 +1346,8 @@ private void waitPartitionRelease(boolean distributed, boolean doRollback) throw
         if (exchId.isLeft())
             cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion());
 
-        if (log.isDebugEnabled())
-            log.debug("Before waiting for partition release future: " + this);
+        if (log.isTraceEnabled())
+            log.trace("Before waiting for partition release future: " + this);
 
         int dumpCnt = 0;
 
@@ -1551,6 +1551,8 @@ public boolean localJoinExchange() {
     private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException {
         assert node != null;
 
+        long time = System.currentTimeMillis();
+
         GridDhtPartitionsSingleMessage msg;
 
         // Reset lost partitions before sending local partitions to coordinator.
@@ -1585,8 +1587,8 @@ private void sendLocalPartitions(ClusterNode node) throws IgniteCheckedException
         else if (localJoinExchange())
             msg.cacheGroupsAffinityRequest(exchCtx.groupsAffinityRequestOnJoin());
 
-        if (log.isDebugEnabled())
-            log.debug("Sending local partitions [nodeId=" + node.id() + ", exchId=" + exchId + ", msg=" + msg + ']');
+        if (log.isTraceEnabled())
+            log.trace("Sending local partitions [nodeId=" + node.id() + ", exchId=" + exchId + ", msg=" + msg + ']');
 
         try {
             cctx.io().send(node, msg, SYSTEM_POOL);
@@ -1595,6 +1597,9 @@ else if (localJoinExchange())
             if (log.isDebugEnabled())
                 log.debug("Node left during partition exchange [nodeId=" + node.id() + ", exchId=" + exchId + ']');
         }
+
+        if (log.isInfoEnabled())
+            log.info("Sending Single Message performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
@@ -1634,8 +1639,8 @@ private void sendAllPartitions(
     ) {
         assert !nodes.contains(cctx.localNode());
 
-        if (log.isDebugEnabled()) {
-            log.debug("Sending full partition map [nodeIds=" + F.viewReadOnly(nodes, F.node2id()) +
+        if (log.isTraceEnabled()) {
+            log.trace("Sending full partition map [nodeIds=" + F.viewReadOnly(nodes, F.node2id()) +
                 ", exchId=" + exchId + ", msg=" + fullMsg + ']');
         }
 
@@ -1653,6 +1658,8 @@ private void sendAllPartitions(
             .map(singleMessage -> fullMsg.copy().joinedNodeAffinity(affinityForJoinedNodes))
             .orElse(null);
 
+        long time = System.currentTimeMillis();
+
         // Prepare and send full messages for given nodes.
         nodes.stream()
             .map(node -> {
@@ -1705,6 +1712,9 @@ private void sendAllPartitions(
                     U.error(log, "Failed to send partitions [node=" + node + ']', e);
                 }
             });
+
+        if (log.isInfoEnabled())
+            log.info("Sending Full Message performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
@@ -2691,6 +2701,8 @@ else if (cntr == maxCntr.cnt)
     private void detectLostPartitions(AffinityTopologyVersion resTopVer) {
         boolean detected = false;
 
+        long time = System.currentTimeMillis();
+
         synchronized (cctx.exchange().interruptLock()) {
             if (Thread.currentThread().isInterrupted())
                 return;
@@ -2704,8 +2716,16 @@ private void detectLostPartitions(AffinityTopologyVersion resTopVer) {
             }
         }
 
-        if (detected)
+        if (detected) {
+            if (log.isDebugEnabled())
+                log.debug("Partitions have been scheduled to resend [reason=" +
+                    "Lost partitions detect on " + resTopVer + "]");
+
             cctx.exchange().scheduleResendPartitions();
+        }
+
+        if (log.isInfoEnabled())
+            log.info("Detecting lost partitions performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
@@ -2824,6 +2844,8 @@ private void onAllReceived(@Nullable Collection<ClusterNode> sndResNodes) {
                 if (log.isInfoEnabled())
                     log.info("Coordinator received all messages, try merge [ver=" + initialVersion() + ']');
 
+                long time = System.currentTimeMillis();
+
                 boolean finish = cctx.exchange().mergeExchangesOnCoordinator(this);
 
                 // Synchronize in case of changed coordinator (thread switched to sys-*)
@@ -2832,6 +2854,9 @@ private void onAllReceived(@Nullable Collection<ClusterNode> sndResNodes) {
                         updateTopologies(true);
                 }
 
+                if (log.isInfoEnabled())
+                    log.info("Exchanges merging performed in " + (System.currentTimeMillis() - time) + " ms.");
+
                 if (!finish)
                     return;
             }
@@ -2866,6 +2891,8 @@ private void finishExchangeOnCoordinator(@Nullable Collection<ClusterNode> sndRe
 
             Map<Integer, CacheGroupAffinityMessage> idealAffDiff = null;
 
+            long time = System.currentTimeMillis();
+
             if (exchCtx.mergeExchanges()) {
                 synchronized (mux) {
                     if (mergedJoinExchMsgs != null) {
@@ -2899,6 +2926,9 @@ private void finishExchangeOnCoordinator(@Nullable Collection<ClusterNode> sndRe
                 }
             }
 
+            if (log.isInfoEnabled())
+                log.info("Affinity changes (coordinator) applied in " + (System.currentTimeMillis() - time) + " ms.");
+
             Map<Integer, CacheGroupAffinityMessage> joinedNodeAff = null;
 
             for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> e : msgs.entrySet()) {
@@ -2977,6 +3007,8 @@ else if (discoveryCustomMessage instanceof SnapshotDiscoveryMessage
 
             IgniteProductVersion minVer = exchCtx.events().discoveryCache().minimumNodeVersion();
 
+            time = System.currentTimeMillis();
+
             GridDhtPartitionsFullMessage msg = createPartitionsMessage(true,
                 minVer.compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0);
 
@@ -2993,6 +3025,9 @@ else if (forceAffReassignment)
 
             msg.prepareMarshal(cctx);
 
+            if (log.isInfoEnabled())
+                log.info("Preparing Full Message performed in " + (System.currentTimeMillis() - time) + " ms.");
+
             synchronized (mux) {
                 finishState = new FinishState(crd.id(), resTopVer, msg);
 
@@ -3002,6 +3037,8 @@ else if (forceAffReassignment)
             if (centralizedAff) {
                 assert !exchCtx.mergeExchanges();
 
+                time = System.currentTimeMillis();
+
                 IgniteInternalFuture<Map<Integer, Map<Integer, List<UUID>>>> fut = cctx.affinity().initAffinityOnNodeLeft(this);
 
                 if (!fut.isDone()) {
@@ -3013,6 +3050,9 @@ else if (forceAffReassignment)
                 }
                 else
                     onAffinityInitialized(fut);
+
+                if (log.isInfoEnabled())
+                    log.info("Centralized affinity changes are performed in " + (System.currentTimeMillis() - time) + " ms.");
             }
             else {
                 Set<ClusterNode> nodes;
@@ -3041,11 +3081,16 @@ else if (forceAffReassignment)
                         nodes.addAll(sndResNodes);
                 }
 
+                time = System.currentTimeMillis();
+
                 if (!nodes.isEmpty())
                     sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff);
 
                 partitionsSent = true;
 
+                if (log.isInfoEnabled())
+                    log.info("Sending Full Message to all nodes performed in " + (System.currentTimeMillis() - time) + " ms.");
+
                 if (!stateChangeExchange())
                     onDone(exchCtx.events().topologyVersion(), null);
 
@@ -3120,6 +3165,8 @@ else if (forceAffReassignment)
      * Validates that partition update counters and cache sizes for all caches are consistent.
      */
     private void validatePartitionsState() {
+        long time = System.currentTimeMillis();
+
         for (Map.Entry<Integer, CacheGroupDescriptor> e : cctx.affinity().cacheGroups().entrySet()) {
             CacheGroupDescriptor grpDesc = e.getValue();
             if (grpDesc.config().getCacheMode() == CacheMode.LOCAL)
@@ -3150,12 +3197,17 @@ private void validatePartitionsState() {
                 // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833
             }
         }
+
+        if (log.isInfoEnabled())
+            log.info("Partitions validation performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
      *
      */
     private void assignPartitionsStates() {
+        long time = System.currentTimeMillis();
+
         for (Map.Entry<Integer, CacheGroupDescriptor> e : cctx.affinity().cacheGroups().entrySet()) {
             CacheGroupDescriptor grpDesc = e.getValue();
             if (grpDesc.config().getCacheMode() == CacheMode.LOCAL)
@@ -3172,6 +3224,9 @@ private void assignPartitionsStates() {
             else
                 assignPartitionStates(top);
         }
+
+        if (log.isInfoEnabled())
+            log.info("Partitions assignment performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
@@ -3209,17 +3264,22 @@ private void sendAllPartitionsToNode(FinishState finishState, GridDhtPartitionsS
             fullMsg.exchangeId(msg.exchangeId());
         }
 
-            try {
-                cctx.io().send(node, fullMsg, SYSTEM_POOL);
-}
+        try {
+            cctx.io().send(node, fullMsg, SYSTEM_POOL);
 
-            catch (ClusterTopologyCheckedException e) {
-                if (log.isDebugEnabled())
-                    log.debug("Failed to send partitions, node failed: " + node);
+            if (log.isTraceEnabled()) {
+                log.trace("Full message was sent to node: " +
+                    node +
+                    ", fullMsg: " + fullMsg
+                );
             }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to send partitions [node=" + node + ']', e);
-
+        }
+        catch (ClusterTopologyCheckedException e) {
+            if (log.isDebugEnabled())
+                log.debug("Failed to send partitions, node failed: " + node);
+        }
+        catch (IgniteCheckedException e) {
+            U.error(log, "Failed to send partitions [node=" + node + ']', e);
         }
     }
 
@@ -3484,6 +3544,8 @@ private void processFullMessage(boolean checkCrd, ClusterNode node, GridDhtParti
 
             AffinityTopologyVersion resTopVer = initialVersion();
 
+            long time = System.currentTimeMillis();
+
             if (exchCtx.mergeExchanges()) {
                 if (msg.resultTopologyVersion() != null && !initialVersion().equals(msg.resultTopologyVersion())) {
                     if (log.isInfoEnabled()) {
@@ -3530,6 +3592,9 @@ else if (localJoinExchange() && !exchCtx.fetchAffinityOnJoin())
             else if (forceAffReassignment)
                 cctx.affinity().applyAffinityFromFullMessage(this, msg);
 
+            if (log.isInfoEnabled())
+                log.info("Affinity changes applied in " + (System.currentTimeMillis() - time) + " ms.");
+
             if (dynamicCacheStartExchange() && !F.isEmpty(exchangeGlobalExceptions)) {
                 assert cctx.localNode().isClient();
 
@@ -3575,6 +3640,8 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa
 
         partHistSuppliers.putAll(msg.partitionHistorySuppliers());
 
+        long time = System.currentTimeMillis();
+
         for (Map.Entry<Integer, GridDhtPartitionFullMap> entry : msg.partitions().entrySet()) {
             Integer grpId = entry.getKey();
 
@@ -3611,6 +3678,10 @@ private void updatePartitionFullMap(AffinityTopologyVersion resTopVer, GridDhtPa
         }
 
         partitionsReceived = true;
+
+        if (log.isInfoEnabled())
+            log.info("Full map updating for " + msg.partitions().size()
+                + " groups performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index 888ab6a642d7b..b84dc79e821f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -29,7 +29,7 @@
 import org.apache.ignite.internal.GridDirectTransient;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 804cc030489f7..127c471f52ca8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -28,7 +28,7 @@
 import org.apache.ignite.internal.GridDirectMap;
 import org.apache.ignite.internal.GridDirectTransient;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 5a60eb66eda94..6e3d773abbbe3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -19,7 +19,6 @@
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.locks.ReadWriteLock;
@@ -36,27 +35,25 @@
 import org.apache.ignite.internal.processors.cache.GridCachePreloaderAdapter;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridPlainRunnable;
 import org.apache.ignite.internal.util.typedef.CI1;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_UNLOADED;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.LOST;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
 
 /**
  * DHT cache preloader.
@@ -175,7 +172,7 @@ private IgniteCheckedException stopError() {
         if (!grp.rebalanceEnabled())
             return new GridDhtPreloaderAssignments(exchId, top.readyTopologyVersion());
 
-        int partCnt = grp.affinity().partitions();
+        int partitions = grp.affinity().partitions();
 
         AffinityTopologyVersion topVer = top.readyTopologyVersion();
 
@@ -190,7 +187,7 @@ private IgniteCheckedException stopError() {
 
         CachePartitionFullCountersMap countersMap = grp.topology().fullUpdateCounters();
 
-        for (int p = 0; p < partCnt; p++) {
+        for (int p = 0; p < partitions; p++) {
             if (ctx.exchange().hasPendingExchange()) {
                 if (log.isDebugEnabled())
                     log.debug("Skipping assignments creation, exchange worker has pending assignments: " +
@@ -254,7 +251,7 @@ private IgniteCheckedException stopError() {
                         );
                     }
 
-                    msg.partitions().addHistorical(p, part.initialUpdateCounter(), countersMap.updateCounter(p), partCnt);
+                    msg.partitions().addHistorical(p, part.initialUpdateCounter(), countersMap.updateCounter(p), partitions);
                 }
                 else {
                     List<ClusterNode> picked = remoteOwners(p, topVer);
@@ -422,8 +419,13 @@ public void onPartitionEvicted(GridDhtLocalPartition part, boolean updateSeq) {
             if (grp.eventRecordable(EVT_CACHE_REBALANCE_PART_UNLOADED))
                 grp.addUnloadEvent(part.id());
 
-            if (updateSeq)
+            if (updateSeq) {
+                if (log.isDebugEnabled())
+                    log.debug("Partitions have been scheduled to resend [reason=" +
+                        "Eviction [grp" + grp.cacheOrGroupName() + " " + part.id() + "]");
+
                 ctx.exchange().scheduleResendPartitions();
+            }
         }
         finally {
             leaveBusy();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
index 41dd076c3f745..69dc79b5007c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
@@ -21,6 +21,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtDemandedPartitionsMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtDemandedPartitionsMap.java
index d829b53eb28d2..9fe3c647437c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtDemandedPartitionsMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/IgniteDhtDemandedPartitionsMap.java
@@ -19,11 +19,8 @@
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
 import java.io.Serializable;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
 import java.util.Set;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
@@ -153,66 +150,24 @@ public Set<Integer> fullSet() {
         return Collections.unmodifiableSet(full);
     }
 
-    /** {@inheritDoc} */
-    @Override public String toString() {
-        return S.toString(IgniteDhtDemandedPartitionsMap.class, this);
-    }
-
-    /**
-     * @return String representation of partitions list.
-     */
-    String partitionsList() {
-        List<Integer> s = new ArrayList<>(size());
+    /** */
+    public Set<Integer> historicalSet() {
+        if (historical == null)
+            return Collections.emptySet();
 
-        s.addAll(fullSet());
+        Set<Integer> historical = new HashSet<>(historicalMap().size());
 
         for (int i = 0; i < historicalMap().size(); i++) {
             int p = historicalMap().partitionAt(i);
 
-            assert !s.contains(p);
-
-            s.add(p);
+            historical.add(p);
         }
 
-        Collections.sort(s);
-
-        StringBuilder sb = new StringBuilder();
-
-        int start = -1;
-
-        int prev = -1;
-
-        Iterator<Integer> sit = s.iterator();
-
-        while (sit.hasNext()) {
-            int p = sit.next();
-
-            if (start == -1) {
-                start = p;
-                prev = p;
-            }
-
-            if (prev < p - 1) {
-                sb.append(start);
-
-                if (start != prev)
-                    sb.append("-").append(prev);
-
-                sb.append(", ");
-
-                start = p;
-            }
-
-            if (!sit.hasNext()) {
-                sb.append(start);
-
-                if (start != p)
-                    sb.append("-").append(p);
-            }
-
-            prev = p;
-        }
+        return historical;
+    }
 
-        return sb.toString();
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IgniteDhtDemandedPartitionsMap.class, this);
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/EvictionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/EvictionContext.java
similarity index 98%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/EvictionContext.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/EvictionContext.java
index 0964c3c3cc5c1..1498aa6a52c4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/EvictionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/EvictionContext.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.distributed.dht;
+package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
 /**
  * Additional context for partition eviction process.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
similarity index 99%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
index 0d8a9f7cdf467..ebf75efe01156 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.distributed.dht;
+package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -39,6 +39,8 @@
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionFullCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
@@ -55,9 +57,9 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
 
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
 
 /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtInvalidPartitionException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtInvalidPartitionException.java
similarity index 99%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtInvalidPartitionException.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtInvalidPartitionException.java
index 5ef474497cc41..6a2ca6b90ce41 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtInvalidPartitionException.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtInvalidPartitionException.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.distributed.dht;
+package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
 /**
  * Exception thrown whenever entry is created for invalid partition.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java
similarity index 94%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java
index 25be88fd35413..e0bfd960d220d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.distributed.dht;
+package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
 import java.util.Iterator;
 import java.util.List;
@@ -46,6 +46,8 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMapEntryFactory;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
 import org.apache.ignite.internal.processors.cache.extras.GridCacheObsoleteEntryExtras;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
@@ -67,11 +69,11 @@
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_REMOVED_ENTRIES_TTL;
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_OBJECT_UNLOADED;
 import static org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager.CacheDataStore;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.LOST;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
 
 /**
  * Key partition.
@@ -180,7 +182,7 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
      * @param id Partition ID.
      */
     @SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor")
-    GridDhtLocalPartition(GridCacheSharedContext ctx,
+    public GridDhtLocalPartition(GridCacheSharedContext ctx,
         CacheGroupContext grp,
         int id) {
         super(ENTRY_FACTORY);
@@ -236,6 +238,10 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements
             // TODO ignite-db
             throw new IgniteException(e);
         }
+
+        if (log.isDebugEnabled())
+            log.debug("Partition has been created [grp=" + grp.cacheOrGroupName()
+                + ", p=" + id + ", state=" + state() + "]");
     }
 
     /**
@@ -332,7 +338,7 @@ public int id() {
     /**
      * @return Create time.
      */
-    long createTime() {
+    public long createTime() {
         return createTime;
     }
 
@@ -372,7 +378,7 @@ public boolean valid() {
     /**
      * @param entry Entry to remove.
      */
-    void onRemoved(GridDhtCacheEntry entry) {
+    public void onRemoved(GridDhtCacheEntry entry) {
         assert entry.obsolete() : entry;
 
         // Make sure to remove exactly this entry.
@@ -555,27 +561,45 @@ public void setState(GridDhtPartitionState toState) {
     private boolean casState(long state, GridDhtPartitionState toState) {
         if (grp.persistenceEnabled() && grp.walEnabled()) {
             synchronized (this) {
+                GridDhtPartitionState prevState = state();
+
                 boolean update = this.state.compareAndSet(state, setPartState(state, toState));
 
-                if (update)
+                if (update) {
                     try {
                         ctx.wal().log(new PartitionMetaStateRecord(grp.groupId(), id, toState, updateCounter()));
                     }
                     catch (IgniteCheckedException e) {
-                        U.error(log, "Error while writing to log", e);
+                        U.error(log, "Failed to log partition state change to WAL.", e);
                     }
 
+                    if (log.isDebugEnabled())
+                        log.debug("Partition changed state [grp=" + grp.cacheOrGroupName()
+                            + ", p=" + id + ", prev=" + prevState + ", to=" + toState + "]");
+                }
+
                 return update;
             }
         }
-        else
-            return this.state.compareAndSet(state, setPartState(state, toState));
+        else {
+            GridDhtPartitionState prevState = state();
+
+            boolean update = this.state.compareAndSet(state, setPartState(state, toState));
+
+            if (update) {
+                if (log.isDebugEnabled())
+                    log.debug("Partition changed state [grp=" + grp.cacheOrGroupName()
+                        + ", p=" + id + ", prev=" + prevState + ", to=" + toState + "]");
+            }
+
+            return update;
+        }
     }
 
     /**
      * @return {@code True} if transitioned to OWNING state.
      */
-    boolean own() {
+    public boolean own() {
         while (true) {
             long state = this.state.get();
 
@@ -589,12 +613,8 @@ boolean own() {
 
             assert partState == MOVING || partState == LOST;
 
-            if (casState(state, OWNING)) {
-                if (log.isDebugEnabled())
-                    log.debug("Owned partition: " + this);
-
+            if (casState(state, OWNING))
                 return true;
-            }
         }
     }
 
@@ -609,19 +629,15 @@ public void moving() {
 
             assert partState == OWNING || partState == RENTING : "Only partitions in state OWNING or RENTING can be moved to MOVING state";
 
-            if (casState(state, MOVING)) {
-                if (log.isDebugEnabled())
-                    log.debug("Forcibly moved partition to a MOVING state: " + this);
-
+            if (casState(state, MOVING))
                 break;
-            }
         }
     }
 
     /**
      * @return {@code True} if partition state changed.
      */
-    boolean markLost() {
+    public boolean markLost() {
         while (true) {
             long state = this.state.get();
 
@@ -630,12 +646,8 @@ boolean markLost() {
             if (partState == LOST)
                 return false;
 
-            if (casState(state, LOST)) {
-                if (log.isDebugEnabled())
-                    log.debug("Marked partition as LOST: " + this);
-
+            if (casState(state, LOST))
                 return true;
-            }
         }
     }
 
@@ -660,9 +672,6 @@ public IgniteInternalFuture<?> rent(boolean updateSeq) {
         if (getReservations(state0) == 0 && casState(state0, RENTING)) {
             delayedRenting = false;
 
-            if (log.isDebugEnabled())
-                log.debug("Moved partition to RENTING state: " + this);
-
             // Evict asynchronously, as the 'rent' method may be called
             // from within write locks on local partition.
             clearAsync0(updateSeq);
@@ -694,10 +703,8 @@ private void clearAsync0(boolean updateSeq) {
         if (!reinitialized)
             return;
 
-        // Try fast eviction
-        if (isEmpty() && getSize(state) == 0 && !grp.queriesEnabled()
-                && getReservations(state) == 0 && !groupReserved()) {
-
+        // Try fast eviction.
+        if (freeAndEmpty(state) && !grp.queriesEnabled() && !groupReserved()) {
             if (partState == RENTING && casState(state, EVICTED) || clearingRequested) {
                 clearFuture.finish();
 
@@ -707,6 +714,10 @@ && getReservations(state) == 0 && !groupReserved()) {
                     destroy();
                 }
 
+                if (log.isDebugEnabled())
+                    log.debug("Partition has been fast evicted [grp=" + grp.cacheOrGroupName()
+                        + ", p=" + id + ", state=" + state() + "]");
+
                 return;
             }
         }
@@ -733,7 +744,7 @@ public void clearAsync() {
      * Continues delayed clearing of partition if possible.
      * Clearing may be delayed because of existing reservations.
      */
-    void tryContinueClearing() {
+    public void tryContinueClearing() {
         clearAsync0(true);
     }
 
@@ -749,6 +760,14 @@ private boolean groupReserved() {
         return false;
     }
 
+    /**
+     * @param state State.
+     * @return {@code True} if partition has no reservations and empty.
+     */
+    private boolean freeAndEmpty(long state) {
+        return isEmpty() && getSize(state) == 0 && getReservations(state) == 0;
+    }
+
     /**
      * @return {@code true} if evicting thread was added.
      */
@@ -811,12 +830,8 @@ private void finishEviction(boolean updateSeq) {
 
         GridDhtPartitionState state = getPartState(state0);
 
-        if (state == EVICTED || (isEmpty() && getSize(state0) == 0 && getReservations(state0) == 0 && state == RENTING && casState(state0, EVICTED))) {
-            if (log.isDebugEnabled())
-                log.debug("Evicted partition: " + this);
-
+        if (state == EVICTED || (freeAndEmpty(state0) && state == RENTING && casState(state0, EVICTED)))
             updateSeqOnDestroy = updateSeq;
-        }
     }
 
     /**
@@ -900,7 +915,8 @@ public boolean tryClear(EvictionContext evictionCtx) throws NodeStoppingExceptio
                 long clearedEntities = clearAll(evictionCtx);
 
                 if (log.isDebugEnabled())
-                    log.debug("Partition is cleared [clearedEntities=" + clearedEntities + ", part=" + this + "]");
+                    log.debug("Partition has been cleared [grp=" + grp.cacheOrGroupName()
+                        + ", p=" + id + ", state=" + state() + ", clearedCnt=" + clearedEntities + "]");
             }
             catch (NodeStoppingException e) {
                 clearFuture.finish(e);
@@ -934,7 +950,7 @@ private void destroyCacheDataStore() {
      * On partition unlock callback.
      * Tries to continue delayed partition clearing.
      */
-    void onUnlock() {
+    public void onUnlock() {
         tryContinueClearing();
     }
 
@@ -963,7 +979,7 @@ public boolean backup(AffinityTopologyVersion topVer) {
      * @param topVer Topology version for current operation.
      * @return Next update index.
      */
-    long nextUpdateCounter(int cacheId, AffinityTopologyVersion topVer, boolean primary, @Nullable Long primaryCntr) {
+    public long nextUpdateCounter(int cacheId, AffinityTopologyVersion topVer, boolean primary, @Nullable Long primaryCntr) {
         long nextCntr = store.nextUpdateCounter();
 
         if (grp.sharedGroup())
@@ -1266,7 +1282,7 @@ private void clearDeferredDeletes() {
     /**
      * @param cacheId Cache ID.
      */
-    void onCacheStopped(int cacheId) {
+    public void onCacheStopped(int cacheId) {
         assert grp.sharedGroup() : grp.cacheOrGroupName();
 
         for (Iterator<RemovedEntryHolder> it = rmvQueue.iterator(); it.hasNext();) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionState.java
similarity index 99%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionState.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionState.java
index c1a29ed75aa93..4cbce4bab6c28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionState.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.distributed.dht;
+package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
 import org.jetbrains.annotations.Nullable;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java
similarity index 98%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java
index d77d9c35b265b..59ed377d69657 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.distributed.dht;
+package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
 import java.util.Collection;
 import java.util.List;
@@ -29,6 +29,8 @@
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionFullCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
similarity index 93%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
index 7e410a45430b6..3899e126d2928 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.distributed.dht;
+package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -45,6 +45,8 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.ExchangeDiscoveryEvents;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionFullCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
@@ -67,11 +69,11 @@
 
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST;
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.LOST;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
 
 /**
  * Partition topology.
@@ -356,7 +358,7 @@ private boolean initPartitions(AffinityTopologyVersion affVer, List<List<Cluster
 
             GridDhtPartitionExchangeId exchId = exchFut.exchangeId();
 
-            int num = grp.affinity().partitions();
+            int partitions = grp.affinity().partitions();
 
             if (grp.rebalanceEnabled()) {
                 boolean added = exchFut.cacheGroupAddedOnExchange(grp.groupId(), grp.receivedFrom());
@@ -366,7 +368,20 @@ private boolean initPartitions(AffinityTopologyVersion affVer, List<List<Cluster
                 if (first) {
                     assert exchId.isJoined() || added || exchFut.activateCluster();
 
-                    for (int p = 0; p < num; p++) {
+                    if (log.isDebugEnabled()) {
+                        String reason;
+
+                        if (exchId.isJoined())
+                            reason = "First node in cluster";
+                        else if (added)
+                            reason = "Cache group added";
+                        else
+                            reason = "Cluster activate";
+
+                        log.debug("Initialize partitions (" + reason + ")" + " [grp=" + grp.cacheOrGroupName() + "]");
+                    }
+
+                    for (int p = 0; p < partitions; p++) {
                         if (localNode(p, affAssignment)) {
                             // Partition is created first time, so it's safe to own it.
                             boolean shouldOwn = locParts.get(p) == null;
@@ -377,8 +392,8 @@ private boolean initPartitions(AffinityTopologyVersion affVer, List<List<Cluster
                                 locPart.own();
 
                                 if (log.isDebugEnabled())
-                                    log.debug("Owned partition for oldest node [grp=" + grp.cacheOrGroupName() +
-                                        ", part=" + locPart + ']');
+                                    log.debug("Partition has been owned (created first time) " +
+                                        "[grp=" + grp.cacheOrGroupName() + ", p=" + locPart.id() + ']');
                             }
 
                             needRefresh = true;
@@ -393,7 +408,7 @@ private boolean initPartitions(AffinityTopologyVersion affVer, List<List<Cluster
             else {
                 // If preloader is disabled, then we simply clear out
                 // the partitions this node is not responsible for.
-                for (int p = 0; p < num; p++) {
+                for (int p = 0; p < partitions; p++) {
                     GridDhtLocalPartition locPart = localPartition0(p, affVer, false, true);
 
                     boolean belongs = localNode(p, affAssignment);
@@ -443,9 +458,12 @@ private void createPartitions(AffinityTopologyVersion affVer, List<List<ClusterN
         if (!grp.affinityNode())
             return;
 
-        int num = grp.affinity().partitions();
+        int partitions = grp.affinity().partitions();
 
-        for (int p = 0; p < num; p++) {
+        if (log.isDebugEnabled())
+            log.debug("Create non-existing partitions [grp=" + grp.cacheOrGroupName() + "]");
+
+        for (int p = 0; p < partitions; p++) {
             if (node2part != null && node2part.valid()) {
                 if (localNode(p, aff)) {
                     // This will make sure that all non-existing partitions
@@ -463,10 +481,11 @@ else if (localNode(p, aff))
     }
 
     /** {@inheritDoc} */
-    @Override public void beforeExchange(GridDhtPartitionsExchangeFuture exchFut,
+    @Override public void beforeExchange(
+        GridDhtPartitionsExchangeFuture exchFut,
         boolean affReady,
-        boolean updateMoving)
-        throws IgniteCheckedException {
+        boolean updateMoving
+    ) throws IgniteCheckedException {
         ctx.database().checkpointReadLock();
 
         try {
@@ -674,7 +693,7 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
     @Override public boolean afterExchange(GridDhtPartitionsExchangeFuture exchFut) {
         boolean changed = false;
 
-        int num = grp.affinity().partitions();
+        int partitions = grp.affinity().partitions();
 
         AffinityTopologyVersion topVer = exchFut.context().events().topologyVersion();
 
@@ -710,7 +729,7 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
 
                 long updateSeq = this.updateSeq.incrementAndGet();
 
-                for (int p = 0; p < num; p++) {
+                for (int p = 0; p < partitions; p++) {
                     GridDhtLocalPartition locPart = localPartition0(p, topVer, false, true);
 
                     if (partitionLocalNode(p, topVer)) {
@@ -724,7 +743,8 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
                             if (grp.rebalanceEnabled()) {
                                 Collection<ClusterNode> owners = owners(p);
 
-                                // If there are no other owners, then become an owner.
+                                // If there are no other owners, then becomean owner .
+
                                 if (F.isEmpty(owners)) {
                                     boolean owned = locPart.own();
 
@@ -750,9 +770,11 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
                                             ", part=" + locPart + ']');
                                     }
                                 }
-                                else if (log.isDebugEnabled())
-                                    log.debug("Will not own partition (there are owners to rebalance from) [grp=" + grp.cacheOrGroupName() +
-                                        ", locPart=" + locPart + ", owners = " + owners + ']');
+                                elseif (log.isDebugEnabled())
+                                        log.debug("Will not own partition (there are owners to rebalance from) " +
+                                            "[grp=" + grp.cacheOrGroupName() +
+                                            ", p=" + p + ", owners = " + owners + ']');
+
                             }
                             else
                                 updateSeq = updateLocal(p, locPart.state(), updateSeq, topVer);
@@ -771,7 +793,7 @@ else if (log.isDebugEnabled())
 
                                 if (log.isDebugEnabled()) {
                                     log.debug("Evicting " + state + " partition (it does not belong to affinity) [" +
-                                        "grp=" + grp.cacheOrGroupName() + ", part=" + locPart + ']');
+                                        "grp=" + grp.cacheOrGroupName() + ", p=" + locPart.id() + ']');
                                 }
                             }
                         }
@@ -948,9 +970,6 @@ else if (loc != null && state == RENTING && !showRenting) {
                     this.updateSeq.incrementAndGet();
 
                     created = true;
-
-                    if (log.isDebugEnabled())
-                        log.debug("Created local partition [grp=" + grp.cacheOrGroupName() + ", part=" + loc + ']');
                 }
             }
             finally {
@@ -1342,8 +1361,8 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
         @Nullable Map<Integer, Long> partSizes,
         @Nullable AffinityTopologyVersion msgTopVer) {
         if (log.isDebugEnabled()) {
-            log.debug("Updating full partition map [grp=" + grp.cacheOrGroupName() + ", exchVer=" + exchangeVer +
-                ", fullMap=" + fullMapString() + ']');
+            log.debug("Updating full partition map " +
+                "[grp=" + grp.cacheOrGroupName() + ", exchVer=" + exchangeVer + ", fullMap=" + fullMapString() + ']');
         }
 
         assert partMap != null;
@@ -1376,9 +1395,15 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
 
                         if (part.state() == OWNING || part.state() == MOVING) {
                             long updCntr = incomeCntrMap.updateCounter(part.id());
+                            long curCntr = part.updateCounter();
 
-                            if (updCntr != 0 && updCntr > part.updateCounter())
+                            if (updCntr != 0 && updCntr > curCntr) {
                                 part.updateCounter(updCntr);
+
+                                if (log.isDebugEnabled())
+                                    log.debug("Partition update counter has updated [grp=" + grp.cacheOrGroupName() + ", p=" + part.id()
+                                        + ", state=" + part.state() + ", prevCntr=" + curCntr + ", nextCntr=" + updCntr + "]");
+                            }
                         }
                     }
                 }
@@ -1446,8 +1471,8 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
                         UUID nodeId = it.next();
 
                         if (!ctx.discovery().alive(nodeId)) {
-                            if (log.isDebugEnabled())
-                                log.debug("Removing left node from full map update [grp=" + grp.cacheOrGroupName() +
+                            if (log.isTraceEnabled())
+                                log.trace("Removing left node from full map update [grp=" + grp.cacheOrGroupName() +
                                     ", nodeId=" + nodeId + ", partMap=" + partMap + ']');
 
                             it.remove();
@@ -1462,8 +1487,8 @@ private boolean shouldOverridePartitionMap(GridDhtPartitionMap currentMap, GridD
                 }
 
                 if (!fullMapUpdated) {
-                    if (log.isDebugEnabled()) {
-                        log.debug("No updates for full partition map (will ignore) [" +
+                    if (log.isTraceEnabled()) {
+                        log.trace("No updates for full partition map (will ignore) [" +
                             "grp=" + grp.cacheOrGroupName() +
                             ", lastExch=" + lastTopChangeVer +
                             ", exchVer=" + exchangeVer +
@@ -1574,8 +1599,13 @@ else if (state == MOVING) {
                         + ", exchVer=" + exchangeVer + ", states=" + dumpPartitionStates() + ']');
                 }
 
-                if (changed)
+                if (changed) {
+                    if (log.isDebugEnabled())
+                        log.debug("Partitions have been scheduled to resend [reason=" +
+                            "Full map update [grp" + grp.cacheOrGroupName() + "]");
+
                     ctx.exchange().scheduleResendPartitions();
+                }
 
                 return changed;
             } finally {
@@ -1701,8 +1731,8 @@ private boolean isStaleUpdate(GridDhtPartitionMap currentMap, GridDhtPartitionMa
         }
 
         if (!ctx.discovery().alive(parts.nodeId())) {
-            if (log.isDebugEnabled()) {
-                log.debug("Received partition update for non-existing node (will ignore) [grp=" + grp.cacheOrGroupName() +
+            if (log.isTraceEnabled()) {
+                log.trace("Received partition update for non-existing node (will ignore) [grp=" + grp.cacheOrGroupName() +
                     ", exchId=" + exchId + ", parts=" + parts + ']');
             }
 
@@ -1752,8 +1782,8 @@ else if (isStaleUpdate(cur, parts)) {
 
                     // This is usual situation when partition maps are equal, just print debug message.
                     if (cur.compareTo(parts) == 0) {
-                        if (log.isDebugEnabled())
-                            log.debug(msg);
+                        if (log.isTraceEnabled())
+                            log.trace(msg);
                     }
                     else
                         U.warn(log, msg);
@@ -1833,8 +1863,13 @@ else if (isStaleUpdate(cur, parts)) {
                 if (log.isDebugEnabled())
                     log.debug("Partition map after single update [grp=" + grp.cacheOrGroupName() + ", map=" + fullMapString() + ']');
 
-                if (changed && exchId == null)
+                if (changed && exchId == null) {
+                    if (log.isDebugEnabled())
+                        log.debug("Partitions have been scheduled to resend [reason=" +
+                            "Single map update [grp" + grp.cacheOrGroupName() + "]");
+
                     ctx.exchange().scheduleResendPartitions();
+                }
 
                 return changed;
             }
@@ -2229,7 +2264,7 @@ private boolean checkEvictions(long updateSeq, AffinityAssignment aff) {
         if (!ctx.kernalContext().state().evictionsAllowed())
             return false;
 
-        boolean changed = false;
+        boolean hasEvictedPartitions = false;
 
         UUID locId = ctx.localNodeId();
 
@@ -2238,67 +2273,69 @@ private boolean checkEvictions(long updateSeq, AffinityAssignment aff) {
         for (int p = 0; p < locParts.length(); p++) {
             GridDhtLocalPartition part = locParts.get(p);
 
-            if (part == null)
+            if (part == null || !part.state().active())
                 continue;
 
-            GridDhtPartitionState state = part.state();
+            List<ClusterNode> affNodes = aff.get(p);
+
+            // This node is affinity node for partition, no need to run eviction.
+            if (affNodes.contains(ctx.localNode()))
+                continue;
 
-            if (state.active()) {
-                List<ClusterNode> affNodes = aff.get(p);
+            List<ClusterNode> nodes = nodes(p, aff.topologyVersion(), OWNING);
+            Collection<UUID> nodeIds = F.nodeIds(nodes);
 
-                if (!affNodes.contains(ctx.localNode())) {
-                    List<ClusterNode> nodes = nodes(p, aff.topologyVersion(), OWNING, null);
-                    Collection<UUID> nodeIds = F.nodeIds(nodes);
+            // If all affinity nodes are owners, then evict partition from local node.
+            if (nodeIds.containsAll(F.nodeIds(affNodes))) {
+                GridDhtPartitionState state0 = part.state();
 
-                    // If all affinity nodes are owners, then evict partition from local node.
-                    if (nodeIds.containsAll(F.nodeIds(affNodes))) {
-                        GridDhtPartitionState state0 = part.state();
+                IgniteInternalFuture<?> rentFut = part.rent(false);
 
-                        IgniteInternalFuture<?> rentFut = part.rent(false);
+                rentingFutures.add(rentFut);
 
-                        rentingFutures.add(rentFut);
+                updateSeq = updateLocal(part.id(), part.state(), updateSeq, aff.topologyVersion());
 
-                        updateSeq = updateLocal(part.id(), part.state(), updateSeq, aff.topologyVersion());
+                boolean stateChanged = state0 != part.state();
 
-                        changed = state0 != part.state();
+                hasEvictedPartitions |= stateChanged;
 
-                        if (log.isDebugEnabled()) {
-                            log.debug("Evicted local partition (all affinity nodes are owners) [grp=" + grp.cacheOrGroupName() +
-                                ", part=" + part + ']');
-                        }
-                    }
-                    else {
-                        int ownerCnt = nodeIds.size();
-                        int affCnt = affNodes.size();
+                if (stateChanged && log.isDebugEnabled()) {
+                    log.debug("Partition has been scheduled for eviction (all affinity nodes are owners) " +
+                        "[grp=" + grp.cacheOrGroupName() + ", p=" + part.id() + ", prevState=" + state0 + ", state=" + part.state() + "]");
+                }
+            }
+            else {
+                int ownerCnt = nodeIds.size();
+                int affCnt = affNodes.size();
 
-                        if (ownerCnt > affCnt) { //TODO !!! we could loss all owners in such case. Should be fixed by GG-13223
-                            // Sort by node orders in ascending order.
-                            Collections.sort(nodes, CU.nodeComparator(true));
+                if (ownerCnt > affCnt) { //TODO !!! we could loss all owners in such case. Should be fixed by GG-13223
+                    // Sort by node orders in ascending order.
+                    Collections.sort(nodes, CU.nodeComparator(true));
 
-                            int diff = nodes.size() - affCnt;
+                    int diff = nodes.size() - affCnt;
 
-                            for (int i = 0; i < diff; i++) {
-                                ClusterNode n = nodes.get(i);
+                    for (int i = 0; i < diff; i++) {
+                        ClusterNode n = nodes.get(i);
 
-                                if (locId.equals(n.id())) {
-                                    GridDhtPartitionState state0 = part.state();
+                        if (locId.equals(n.id())) {
+                            GridDhtPartitionState state0 = part.state();
 
-                                    IgniteInternalFuture<?> rentFut = part.rent(false);
+                            IgniteInternalFuture<?> rentFut = part.rent(false);
 
-                                    rentingFutures.add(rentFut);
+                            rentingFutures.add(rentFut);
 
-                                    updateSeq = updateLocal(part.id(), part.state(), updateSeq, aff.topologyVersion());
+                            updateSeq = updateLocal(part.id(), part.state(), updateSeq, aff.topologyVersion());
 
-                                    changed = state0 != part.state();
+                            boolean stateChanged = state0 != part.state();
 
-                                    if (log.isDebugEnabled()) {
-                                        log.debug("Evicted local partition (this node is oldest non-affinity node) [" +
-                                            "grp=" + grp.cacheOrGroupName() + ", part=" + part + ']');
-                                    }
+                            hasEvictedPartitions |= stateChanged;
 
-                                    break;
-                                }
+                            if (stateChanged && log.isDebugEnabled()) {
+                                log.debug("Partition has been scheduled for eviction (this node is oldest non-affinity node) " +
+                                    "[grp=" + grp.cacheOrGroupName() + ", p=" + part.id() + ", prevState=" + state0 + ", state=" + part.state() + "]");
                             }
+
+                            break;
                         }
                     }
                 }
@@ -2319,6 +2356,10 @@ private boolean checkEvictions(long updateSeq, AffinityAssignment aff) {
                         try {
                             this.updateSeq.incrementAndGet();
 
+                            if (log.isDebugEnabled())
+                                log.debug("Partitions have been scheduled to resend [reason=" +
+                                    "Evictions are done [grp" + grp.cacheOrGroupName() + "]");
+
                             ctx.exchange().scheduleResendPartitions();
                         }
                         finally {
@@ -2329,7 +2370,7 @@ private boolean checkEvictions(long updateSeq, AffinityAssignment aff) {
             }
         }
 
-        return changed;
+        return hasEvictedPartitions;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsReservation.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java
similarity index 98%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsReservation.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java
index 845d3edd73936..2682a896e7476 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsReservation.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.distributed.dht;
+package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -23,10 +23,11 @@
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
 
 /**
  * Reservation mechanism for multiple partitions allowing to do a reservation in one operation.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
similarity index 96%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
index cc0542c5d0e5b..4ec7e84fc7c63 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionsStateValidator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.distributed.dht;
+package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
 import java.util.HashMap;
 import java.util.HashSet;
@@ -69,9 +69,11 @@ public GridDhtPartitionsStateValidator(GridCacheSharedContext<?, ?> cctx) {
      * @throws IgniteCheckedException If validation failed. Exception message contains
      * full information about all partitions which update counters or cache sizes are not consistent.
      */
-    public void validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture fut,
-                                                  GridDhtPartitionTopology top,
-                                                  Map<UUID, GridDhtPartitionsSingleMessage> messages) throws IgniteCheckedException {
+    public void validatePartitionCountersAndSizes(
+        GridDhtPartitionsExchangeFuture fut,
+        GridDhtPartitionTopology top,
+        Map<UUID, GridDhtPartitionsSingleMessage> messages
+    ) throws IgniteCheckedException {
         final Set<UUID> ignoringNodes = new HashSet<>();
 
         // Ignore just joined nodes.
@@ -148,10 +150,11 @@ public void validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture fu
      * @return Invalid partitions map with following structure: (partId, (nodeId, updateCounter)).
      * If map is empty validation is successful.
      */
-     Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
+     public Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
             GridDhtPartitionTopology top,
             Map<UUID, GridDhtPartitionsSingleMessage> messages,
-            Set<UUID> ignoringNodes) {
+            Set<UUID> ignoringNodes
+     ) {
         Map<Integer, Map<UUID, Long>> invalidPartitions = new HashMap<>();
 
         Map<Integer, T2<UUID, Long>> updateCountersAndNodesByPartitions = new HashMap<>();
@@ -202,10 +205,11 @@ Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
      * @return Invalid partitions map with following structure: (partId, (nodeId, cacheSize)).
      * If map is empty validation is successful.
      */
-     Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
+     public Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
             GridDhtPartitionTopology top,
             Map<UUID, GridDhtPartitionsSingleMessage> messages,
-            Set<UUID> ignoringNodes) {
+            Set<UUID> ignoringNodes
+     ) {
         Map<Integer, Map<UUID, Long>> invalidPartitions = new HashMap<>();
 
         Map<Integer, T2<UUID, Long>> sizesAndNodesByPartitions = new HashMap<>();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java
similarity index 98%
rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java
rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java
index 780ca918593f3..cd010fa89d32d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/PartitionsEvictManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-package org.apache.ignite.internal.processors.cache.distributed.dht;
+package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
 import java.util.Collection;
 import java.util.Comparator;
@@ -45,7 +45,6 @@
  * Multiple partition from group can be evicted at the same time.
  */
 public class PartitionsEvictManager extends GridCacheSharedManagerAdapter {
-
     /** Default eviction progress show frequency. */
     private static final int DEFAULT_SHOW_EVICTION_PROGRESS_FREQ_MS = 2 * 60 * 1000; // 2 Minutes.
 
@@ -123,6 +122,10 @@ public void evictPartitionAsync(CacheGroupContext grp, GridDhtLocalPartition par
         if (evictionTask == null)
             return;
 
+        if (log.isDebugEnabled())
+            log.debug("Partition has been scheduled for eviction [grp=" + grp.cacheOrGroupName()
+                + ", p=" + part.id() + ", state=" + part.state() + "]");
+
         int bucket;
 
         synchronized (mux) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
index 2f832b490ea25..c8e66f295432a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java
@@ -41,7 +41,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicNearResponse;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index b35f524bb735b..c689e2b044178 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -45,7 +45,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.CacheDistributedGetFutureAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.GridLeanMap;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index 7d1ab8598466a..5595c1f785aba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -34,7 +34,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxMapping;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxMapping;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 6d46e5677ac43..6c4965b2613e1 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -108,8 +108,8 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.StoredCacheData;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry;
 import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntryType;
@@ -805,6 +805,8 @@ private void unRegistrateMetricsMBean() {
     ) throws IgniteCheckedException {
         assert !cctx.localNode().isClient();
 
+        long time = System.currentTimeMillis();
+
         checkpointReadLock();
 
         try {
@@ -856,6 +858,9 @@ private void unRegistrateMetricsMBean() {
         }
         finally {
             checkpointReadUnlock();
+
+            if (log.isInfoEnabled())
+                log.info("Binary recovery performed in " + (System.currentTimeMillis() - time) + " ms.");
         }
     }
 
@@ -1303,6 +1308,8 @@ private void shutdownCheckpointer(boolean cancel) {
 
         boolean restored = false;
 
+        long time = System.currentTimeMillis();
+
         // In case of cluster activation or local join restore, restore whole manager state.
         if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode)) {
             restoreState();
@@ -1333,6 +1340,9 @@ else if (acts.localJoinContext() != null && !F.isEmpty(acts.localJoinContext().c
             }
         }
 
+        if (log.isInfoEnabled())
+            log.info("Logical recovery performed in " + (System.currentTimeMillis() - time) + " ms.");
+
         return restored;
     }
 
@@ -1584,12 +1594,17 @@ private void restoreState() throws IgniteCheckedException {
      * @throws IgniteCheckedException If first checkpoint has failed.
      */
     @Override public void onStateRestored() throws IgniteCheckedException {
+        long time = System.currentTimeMillis();
+
         new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
 
         CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started");
 
         if (chp != null)
             chp.cpBeginFut.get();
+
+        if (log.isInfoEnabled())
+            log.info("Checkpointer initilialzation performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /** {@inheritDoc} */
@@ -2348,6 +2363,10 @@ private void restorePartitionStates(
                     if (storeMgr.pages(grpId, i) <= 1)
                         continue;
 
+                    if (log.isDebugEnabled())
+                        log.debug("Creating partition on recovery (exists in page store) " +
+                            "[grp=" + grp.cacheOrGroupName() + ", p=" + i + "]");
+
                     GridDhtLocalPartition part = grp.topology().forceCreatePartition(i);
 
                     assert part != null;
@@ -2383,9 +2402,20 @@ private void restorePartitionStates(
 
                                         changed = true;
                                     }
+
+                                    if (log.isDebugEnabled())
+                                        log.debug("Restored partition state (from WAL) " +
+                                            "[grp=" + grp.cacheOrGroupName() + ", p=" + i + ", state=" + part.state() +
+                                            "updCntr=" + part.initialUpdateCounter() + "]");
+                                }
+                                else {
+                                    updateState(part, (int) io.getPartitionState(pageAddr));
+
+                                    if (log.isDebugEnabled())
+                                        log.debug("Restored partition state (from page memory) " +
+                                            "[grp=" + grp.cacheOrGroupName() + ", p=" + i + ", state=" + part.state() +
+                                            "updCntr=" + part.initialUpdateCounter() + "]");
                                 }
-                                else
-                                    updateState(part, (int)io.getPartitionState(pageAddr));
                             }
                             finally {
                                 pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
@@ -2400,6 +2430,10 @@ private void restorePartitionStates(
                     }
                 }
                 else if (restore != null) {
+                    if (log.isDebugEnabled())
+                        log.debug("Creating partition on recovery (exists in WAL) " +
+                            "[grp=" + grp.cacheOrGroupName() + ", p=" + i + "]");
+
                     GridDhtLocalPartition part = grp.topology().forceCreatePartition(i);
 
                     assert part != null;
@@ -2408,6 +2442,11 @@ else if (restore != null) {
                     grp.offheap().onPartitionInitialCounterUpdated(i, 0);
 
                     updateState(part, restore.get1());
+
+                    if (log.isDebugEnabled())
+                        log.debug("Restored partition state (from WAL) " +
+                            "[grp=" + grp.cacheOrGroupName() + ", p=" + i + ", state=" + part.state() +
+                            "updCntr=" + part.initialUpdateCounter() + "]");
                 }
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index b075c01ec0660..16d218f9df001 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -50,8 +50,8 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteHistoricalIterator;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
@@ -79,9 +79,9 @@
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.Nullable;
 
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
 
 /**
  * Used when persistence enabled.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntry.java
index f6433e156070b..c26e0a3d36cea 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/checkpoint/CheckpointEntry.java
@@ -32,7 +32,7 @@
 import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.Nullable;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
index 3d798841bf473..043c22ddad04d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
@@ -20,7 +20,7 @@
 
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.util.GridStringBuilder;
 
 /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
index a477a131c59c7..e4e9ab77a7920 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordDataV1Serializer.java
@@ -82,7 +82,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 8f0edb73b3231..5d22decad4b29 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -75,7 +75,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtUnreservedPartitionException;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.datastructures.DataStructuresProcessor;
@@ -130,7 +130,7 @@
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.GridClosureCallMode.BROADCAST;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SCAN;
 import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SPI;
 import static org.apache.ignite.internal.processors.cache.query.GridCacheQueryType.SQL;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 9ff4623da1006..74f6e6868b5ea 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -54,7 +54,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheAffinityManager;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheDeploymentManager;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture;
 import org.apache.ignite.internal.processors.cache.query.CacheQueryType;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryManager.JCacheQueryLocalListener;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index d80604540253a..24fd13d8589ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -59,6 +59,8 @@
 import org.apache.ignite.internal.processors.cache.GridCacheReturn;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
 import org.apache.ignite.internal.processors.cache.store.CacheStoreManager;
 import org.apache.ignite.internal.processors.cache.version.GridCacheLazyPlainVersionedEntry;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index f25dc386704a0..ea8e648d7a8c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -42,9 +42,9 @@
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryRequest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryResponse;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedTxRemoteAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxFinishRequest;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index b2408286f7695..c224eaba25750 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -56,6 +56,9 @@
 import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.store.CacheStoreManager;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index d15dbdcdb7238..cc558ef2888b8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -59,7 +59,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxFinishSync;
 import org.apache.ignite.internal.processors.cache.distributed.GridCacheTxRecoveryFuture;
 import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocal;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxOnePhaseCommitAckRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxRemote;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
index 7651f6e49f805..4141d7f7ae0bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CollectConflictPartitionKeysTask.java
@@ -37,8 +37,8 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.lang.GridIterator;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java
index d2ce882bbd0cd..79d5b4b142e9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/RetrieveConflictPartitionValuesTask.java
@@ -35,8 +35,8 @@
 import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.T2;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
index c99681864de0c..ecab8152f052e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTask.java
@@ -45,8 +45,8 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.lang.GridIterator;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
index 0041bdde786b4..d5af22df8f947 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
@@ -44,8 +44,8 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.lang.GridIterator;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index 8cad342a7e696..2b731ee663940 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -85,9 +85,9 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheFutureImpl;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
index ae9cb7e904c77..65224add00ac3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/job/GridJobProcessor.java
@@ -66,7 +66,7 @@
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
 import org.apache.ignite.internal.processors.jobmetrics.GridJobMetricsSnapshot;
 import org.apache.ignite.internal.util.GridAtomicLong;
@@ -101,7 +101,7 @@
 import static org.apache.ignite.internal.GridTopic.TOPIC_TASK;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.MANAGEMENT_POOL;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.jsr166.ConcurrentLinkedHashMap.QueuePolicy.PER_SEGMENT_Q;
 
 /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java
index 1775c79f72a4e..918fac0839e3a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java
@@ -24,8 +24,8 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter;
@@ -39,9 +39,9 @@
 
 import java.util.List;
 
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
 
 /**
  * Traversor operating all primary and backup partitions of given cache.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java
index f5893c02f77b4..43a8bc7c345e3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java
@@ -24,7 +24,7 @@
 import java.util.Iterator;
 import java.util.NoSuchElementException;
 import java.util.Set;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 
 /**
  * Grid partition state map. States are encoded using bits.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java
index 52db55291b262..9785b9cf96921 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheLostPartitionsTask.java
@@ -22,7 +22,7 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.visor.VisorJob;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCachePartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCachePartitionsTask.java
index 76ace1770f2b3..93c9c60c571d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCachePartitionsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCachePartitionsTask.java
@@ -28,9 +28,9 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorPartitionMap.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorPartitionMap.java
index cadad0cd81121..ed0e74aefc02e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorPartitionMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorPartitionMap.java
@@ -22,7 +22,7 @@
 import java.io.ObjectOutput;
 import java.util.HashMap;
 import java.util.Map;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
diff --git a/modules/core/src/test/config/log4j-test.xml b/modules/core/src/test/config/log4j-test.xml
index b0b08e7d1a394..bb5f94ac07e9e 100755
--- a/modules/core/src/test/config/log4j-test.xml
+++ b/modules/core/src/test/config/log4j-test.xml
@@ -84,16 +84,17 @@
     </category>
     -->
 
-    <!-- Enable Ignite debugging.
-     <category name="org.apache.ignite">
-         <level value="DEBUG"/>
-     </category>
+    <!--
+        Uncomment to enable Exchange, Rebalance and Partitions workflow debugging.
     -->
-
     <!--
-    <category name="org.apache.ignite.cache.msg">
-        <level value="DEBUG"/>
-    </category>
+         <category name="org.apache.ignite.internal.processors.cache.distributed.dht.preloader">
+             <level value="DEBUG"/>
+         </category>
+
+        <category name="org.apache.ignite.internal.processors.cache.distributed.dht.topology">
+            <level value="DEBUG"/>
+        </category>
     -->
 
     <!-- Disable all open source debugging. -->
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java
index 18a35c65ed7a6..6aa34c4d52892 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDeferredDeleteQueueTest.java
@@ -23,8 +23,8 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java
index 702b188bd4005..27946c927e482 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheDhtLocalPartitionAfterRemoveSelfTest.java
@@ -20,7 +20,7 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
index 6c570d744ff0e..2104c5d91ecb4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
@@ -78,7 +78,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.platform.cache.expiry.PlatformExpiryPolicyFactory;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
@@ -86,7 +86,6 @@
 import org.apache.ignite.internal.util.lang.GridPlainCallable;
 import org.apache.ignite.internal.util.lang.gridfunc.ContainsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java
index eda0a495c8818..a2d9da719ce98 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheStartFailoverTest.java
@@ -41,7 +41,7 @@
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.internal.U;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index 22cf205d7a1c0..adbb47656d325 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -47,8 +47,8 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java
index 2a992714745f3..9ff072500b6fc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheDataLossOnPartitionMoveTest.java
@@ -36,7 +36,7 @@
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.processors.cache.GridCacheUtils;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -46,8 +46,8 @@
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.Nullable;
 
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java
index f6a5ec1d1c20c..84fd91656f9c2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java
@@ -33,12 +33,12 @@
 import org.apache.ignite.internal.pagemem.store.PageStore;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionStateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionStateTest.java
index 3b05ac35c537f..caede5c9afb68 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionStateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePartitionStateTest.java
@@ -29,8 +29,8 @@
 import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
 import org.apache.ignite.internal.util.typedef.G;
@@ -43,9 +43,9 @@
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.EVICTED;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRentingStateRepairTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRentingStateRepairTest.java
index cba3c3cdac094..68a8a7d865e46 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRentingStateRepairTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRentingStateRepairTest.java
@@ -28,9 +28,9 @@
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodePartitionsExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodePartitionsExchangeTest.java
index 7c13e353f8f2e..510140299931c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodePartitionsExchangeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientNodePartitionsExchangeTest.java
@@ -42,7 +42,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDelayedSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDelayedSelfTest.java
index 0105ece39b505..dd4bbccde0b61 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDelayedSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDelayedSelfTest.java
@@ -38,6 +38,8 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
 import org.apache.ignite.internal.util.typedef.CAX;
 import org.apache.ignite.internal.util.typedef.G;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java
index 7a80e87a61c3e..66efc18fc0a6b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadDisabledSelfTest.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
index 83eff893b8894..7b5cc3e8101dc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
@@ -40,6 +40,9 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.P1;
 import org.apache.ignite.internal.util.typedef.X;
@@ -61,9 +64,9 @@
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
 import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
 import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.MOVING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
 
 /**
  * Test cases for partitioned cache {@link GridDhtPreloader preloader}.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadStartStopSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadStartStopSelfTest.java
index 407f69b76b22e..e77e5c8d1f6f7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadStartStopSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadStartStopSelfTest.java
@@ -32,6 +32,8 @@
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -43,7 +45,7 @@
 import static org.apache.ignite.cache.CacheRebalanceMode.SYNC;
 import static org.apache.ignite.configuration.CacheConfiguration.DFLT_REBALANCE_BATCH_SIZE;
 import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  * Test cases for partitioned cache {@link GridDhtPreloader preloader}.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedUnloadEventsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedUnloadEventsSelfTest.java
index d6cea580b6f1f..d00dd7a71dcc3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedUnloadEventsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionedUnloadEventsSelfTest.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.events.CacheEvent;
 import org.apache.ignite.events.CacheRebalancingEvent;
 import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
index fc617bb8f97e2..fcc1293a959f7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidationTest.java
@@ -43,6 +43,8 @@
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
index 43a23031dd99a..0e0479b654f53 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCachePartitionsStateValidatorSelfTest.java
@@ -25,6 +25,10 @@
 import com.google.common.collect.Sets;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.Nullable;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java
index 3f2fe8a4b7dd9..ba17da04e378d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridCacheAtomicInvalidPartitionHandlingSelfTest.java
@@ -41,7 +41,7 @@
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.internal.CU;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NoneRebalanceModeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NoneRebalanceModeSelfTest.java
index f3fb8140295a6..34e1a82d47f5c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NoneRebalanceModeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/NoneRebalanceModeSelfTest.java
@@ -20,11 +20,11 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 import static org.apache.ignite.cache.CacheRebalanceMode.NONE;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 
 /**
  * Test none rebalance mode.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
index 1280e87f9f091..fa4b655fb41bd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
@@ -32,8 +32,8 @@
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopologyImpl;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
index ed51cf392a992..4d03d7dc7e281 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
@@ -36,9 +36,9 @@
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingTest.java
index c08128b152c0e..1b176ae8a07ba 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingWithAsyncClearingTest.java
@@ -31,12 +31,10 @@
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.junit.Assert;
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
index 2b793674218c5..c04d9d8d9663a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyAbstractTest.java
@@ -49,7 +49,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
 import org.apache.ignite.internal.processors.cache.GridCacheTestStore;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyWithStoreAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyWithStoreAbstractTest.java
index 2b8a9ec83dcca..747cb43536739 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyWithStoreAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheExpiryPolicyWithStoreAbstractTest.java
@@ -31,7 +31,6 @@
 import javax.cache.processor.MutableEntry;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteTransactions;
-import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.store.CacheStore;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -40,7 +39,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
 import org.apache.ignite.internal.processors.cache.IgniteCacheAbstractTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.transactions.Transaction;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
index 368c6094cf7db..389a7feacaaff 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -51,7 +51,7 @@
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
index a1065f6b4b618..14d0fb6d4dfb7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
@@ -39,7 +39,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
@@ -48,7 +48,6 @@
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy;
-import org.junit.Assert;
 
 /**
  * Test to reproduce corrupted indexes problem after partition file eviction and truncation.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
index 5e0ccc9ce1fc8..3605700d739fc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
@@ -34,9 +34,9 @@
 import org.apache.ignite.failure.FailureHandler;
 import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java
index 0c9fb631b39a2..cddd701beea46 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/IgniteAbsentEvictionNodeOutOfBaselineTest.java
@@ -25,7 +25,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
 /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
index b69314d83a8dc..1ab8f3121bae3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
@@ -33,7 +33,7 @@
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.internal.U;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
index 808e737244101..31ee8110b4453 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
@@ -50,8 +50,8 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
 import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
index f91c6896ccc83..a1c3e6c587040 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryFailoverAbstractSelfTest.java
@@ -74,7 +74,7 @@
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
 import org.apache.ignite.internal.processors.continuous.GridContinuousHandler;
 import org.apache.ignite.internal.processors.continuous.GridContinuousMessage;
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
index 64d29e6e0927e..344a1cc0cdbbd 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
@@ -37,7 +37,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheTtlManager;
 import org.apache.ignite.internal.processors.cache.WalStateManager;
 import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
-import org.apache.ignite.internal.processors.cache.distributed.dht.PartitionsEvictManager;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.PartitionsEvictManager;
 import org.apache.ignite.internal.processors.cache.dr.GridOsCacheDrManager;
 import org.apache.ignite.internal.processors.cache.jta.CacheNoopJtaManager;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 0fb9efef1ac0a..f279228f46142 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -83,7 +83,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
 import org.apache.ignite.internal.util.GridBusyLock;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 2aedcf1bca613..7be2545aa9b22 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -76,9 +76,9 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridPartitionMapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridPartitionMapSelfTest.java
index ebb4cd73b31e9..ddd0ac6d1ea1d 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridPartitionMapSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridPartitionMapSelfTest.java
@@ -21,7 +21,7 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
index c9da3dcceb24d..fe32b122d216e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java
@@ -54,9 +54,9 @@
 import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionsReservation;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsReservation;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
 import org.apache.ignite.internal.processors.cache.query.CacheQueryType;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
@@ -93,8 +93,8 @@
 import static org.apache.ignite.events.EventType.EVT_CACHE_QUERY_EXECUTED;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.QUERY_POOL;
 import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.LOST;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.OWNING;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.LOST;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.OFF;
 import static org.apache.ignite.internal.processors.query.h2.opt.DistributedJoinMode.distributedJoinMode;
 import static org.apache.ignite.internal.processors.query.h2.opt.GridH2QueryType.MAP;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
index fbd0729672a66..f3c337c0a5847 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridReduceQueryExecutor.java
@@ -59,7 +59,7 @@
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
 import org.apache.ignite.internal.processors.cache.query.GridCacheSqlQuery;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index e3aebc3121919..503b57c494c70 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -45,8 +45,8 @@
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.processors.query.GridQueryProcessor;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java
index 999b1adcf3358..f13fb44fac42e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheScanPartitionQueryFallbackSelfTest.java
@@ -43,8 +43,8 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryRequest;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
index 725fafde3f852..1a1c287efacec 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunAbstractTest.java
@@ -38,7 +38,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
index 7bddafc88231a..89ef607b86fa6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
@@ -37,7 +37,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.lang.GridCursor;
 import org.apache.ignite.lang.IgniteBiPredicate;
 import org.apache.ignite.lang.IgniteCallable;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
index ce385114485a7..dbb2c59bc2fd7 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/twostep/RetryCauseMessageSelfTest.java
@@ -31,7 +31,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
 import org.apache.ignite.internal.processors.query.GridQueryProcessor;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
index 1a274e5fcc7f0..48e94c1e62cce 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -36,7 +36,7 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.tree.SearchRow;
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java
index 16ba04403310d..529f714f95a78 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/dataset/impl/cache/CacheBasedDatasetTest.java
@@ -33,8 +33,8 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.lang.IgnitePredicate;
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java
index ebd2e4344b622..e6c32ba02667b 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/WaitMapExchangeFinishCallable.java
@@ -23,8 +23,8 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.lang.IgniteCallable;
 import org.apache.ignite.resources.IgniteInstanceResource;
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverAbstractBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverAbstractBenchmark.java
index b90573bbaf3ed..dad0140d56bc5 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverAbstractBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteFailoverAbstractBenchmark.java
@@ -27,7 +27,6 @@
 import javax.cache.Cache;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterGroup;
@@ -36,7 +35,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.util.typedef.internal.U;
diff --git a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteInvokeBenchmark.java b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteInvokeBenchmark.java
index 46ebd8cb421f7..7d7aaf99515bb 100644
--- a/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteInvokeBenchmark.java
+++ b/modules/yardstick/src/main/java/org/apache/ignite/yardstick/cache/failover/IgniteTransactionalWriteInvokeBenchmark.java
@@ -35,7 +35,7 @@
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteKernal;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.typedef.F;
 import org.yardstickframework.BenchmarkConfiguration;
 
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java
index d27b717485e2c..62e83067feaf3 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkCommunicationFailureContext.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.spi.discovery.zk.internal;
 
-import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.Collections;
 import java.util.Comparator;
@@ -34,7 +33,7 @@
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 

From f6532e5f77b803118f1a7f3a258b473126318f4c Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 26 Sep 2018 19:07:16 +0300
Subject: [PATCH 385/543] IGNITE-9649 Compilation fixed.

---
 .../distributed/dht/topology/GridClientPartitionTopology.java   | 2 --
 .../distributed/dht/topology/GridDhtPartitionTopologyImpl.java  | 2 +-
 .../processors/cache/expiry/IgniteCacheTtlCleanupSelfTest.java  | 2 +-
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
index ebf75efe01156..65bb657a34512 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
@@ -52,7 +52,6 @@
 import org.apache.ignite.internal.util.GridPartitionStateMap;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.Nullable;
@@ -60,7 +59,6 @@
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.EVICTED;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
-import static org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState.RENTING;
 
 /**
  * Partition topology for node which does not have any local partitions.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
index 3899e126d2928..56d00d20c21f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
@@ -770,7 +770,7 @@ private boolean partitionLocalNode(int p, AffinityTopologyVersion topVer) {
                                             ", part=" + locPart + ']');
                                     }
                                 }
-                                elseif (log.isDebugEnabled())
+                                else if (log.isDebugEnabled())
                                         log.debug("Will not own partition (there are owners to rebalance from) " +
                                             "[grp=" + grp.cacheOrGroupName() +
                                             ", p=" + p + ", owners = " + owners + ']');
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheTtlCleanupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheTtlCleanupSelfTest.java
index 227fe1ffc0e3d..af765498e9ae9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheTtlCleanupSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/expiry/IgniteCacheTtlCleanupSelfTest.java
@@ -27,7 +27,7 @@
 import org.apache.ignite.internal.processors.cache.CacheObjectContext;
 import org.apache.ignite.internal.processors.cache.GridCacheAbstractSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 
 /**

From cbf31486893e2702713e28303709901b305fc06e Mon Sep 17 00:00:00 2001
From: Mikhail Cherkasov <michael.cherkasov@gmail.com>
Date: Tue, 21 Aug 2018 17:35:33 +0300
Subject: [PATCH 386/543] IGNITE-6167: Add SslContextFactory.protocols and
 SslContextFactory.cipherSuites properties to control allowed encryption
 algorithms. This closes #4440. This closes #4582.

(cherry picked from commit a09b2c0579c96c7ab5b8abfe8fde0149041ff70e)
---
 .../ignite/ssl/DelegatingSSLContextSpi.java   | 103 ++++++
 .../apache/ignite/ssl/SSLContextWrapper.java  |  31 ++
 .../ssl/SSLServerSocketFactoryWrapper.java    |  80 +++++
 .../ignite/ssl/SSLSocketFactoryWrapper.java   | 113 ++++++
 .../apache/ignite/ssl/SslContextFactory.java  |  51 +++
 .../ignite/client/SslParametersTest.java      | 329 ++++++++++++++++++
 .../tcp/TcpDiscoverySslParametersTest.java    | 274 +++++++++++++++
 .../ignite/testframework/GridTestUtils.java   |  25 +-
 .../IgniteSpiDiscoverySelfTestSuite.java      |   2 +
 .../apache/ignite/client/ClientTestSuite.java |   3 +-
 10 files changed, 1001 insertions(+), 10 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/ssl/SSLServerSocketFactoryWrapper.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/ssl/SSLSocketFactoryWrapper.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslParametersTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java b/modules/core/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java
new file mode 100644
index 0000000000000..d8621f2cefd9a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/ssl/DelegatingSSLContextSpi.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ssl;
+
+import java.security.KeyManagementException;
+import java.security.SecureRandom;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLContextSpi;
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLParameters;
+import javax.net.ssl.SSLServerSocketFactory;
+import javax.net.ssl.SSLSessionContext;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManager;
+
+/** */
+class DelegatingSSLContextSpi extends SSLContextSpi {
+
+    /** */
+    private final SSLContext delegate;
+
+    /** */
+    private final SSLParameters parameters;
+
+    /** */
+    DelegatingSSLContextSpi(SSLContext delegate, SSLParameters parameters) {
+        this.delegate = delegate;
+        this.parameters = parameters;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void engineInit(KeyManager[] keyManagers, TrustManager[] trustManagers,
+        SecureRandom secureRandom) throws KeyManagementException {
+        delegate.init(keyManagers, trustManagers, secureRandom);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected SSLSocketFactory engineGetSocketFactory() {
+        return new SSLSocketFactoryWrapper(delegate.getSocketFactory(), parameters);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected SSLServerSocketFactory engineGetServerSocketFactory() {
+        return new SSLServerSocketFactoryWrapper(delegate.getServerSocketFactory(),
+            parameters);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected SSLEngine engineCreateSSLEngine() {
+        final SSLEngine engine = delegate.createSSLEngine();
+
+        if (parameters != null)
+            engine.setSSLParameters(parameters);
+
+        return engine;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected SSLEngine engineCreateSSLEngine(String s, int i) {
+        final SSLEngine engine = delegate.createSSLEngine();
+
+        if (parameters != null)
+            engine.setSSLParameters(parameters);
+
+        return engine;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected SSLSessionContext engineGetServerSessionContext() {
+        return delegate.getServerSessionContext();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected SSLSessionContext engineGetClientSessionContext() {
+        return delegate.getClientSessionContext();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected SSLParameters engineGetDefaultSSLParameters() {
+        return delegate.getDefaultSSLParameters();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected SSLParameters engineGetSupportedSSLParameters() {
+        return delegate.getSupportedSSLParameters();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java b/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java
new file mode 100644
index 0000000000000..901d42b1f4ba9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ssl;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLParameters;
+
+/** */
+class SSLContextWrapper extends SSLContext {
+    /** */
+    SSLContextWrapper(SSLContext delegate, SSLParameters sslParameters) {
+        super(new DelegatingSSLContextSpi(delegate, sslParameters),
+            delegate.getProvider(),
+            delegate.getProtocol());
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/SSLServerSocketFactoryWrapper.java b/modules/core/src/main/java/org/apache/ignite/ssl/SSLServerSocketFactoryWrapper.java
new file mode 100644
index 0000000000000..ad80f3c3dd993
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/ssl/SSLServerSocketFactoryWrapper.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ssl;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.ServerSocket;
+import javax.net.ssl.SSLParameters;
+import javax.net.ssl.SSLServerSocket;
+import javax.net.ssl.SSLServerSocketFactory;
+
+/** */
+class SSLServerSocketFactoryWrapper extends SSLServerSocketFactory {
+
+    /** */
+    private final SSLServerSocketFactory delegate;
+    /** */
+    private final SSLParameters parameters;
+
+    /** */
+    SSLServerSocketFactoryWrapper(SSLServerSocketFactory delegate, SSLParameters parameters) {
+        this.delegate = delegate;
+        this.parameters = parameters;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String[] getDefaultCipherSuites() {
+        return delegate.getDefaultCipherSuites();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String[] getSupportedCipherSuites() {
+        return delegate.getSupportedCipherSuites();
+    }
+
+    /** {@inheritDoc} */
+    @Override public ServerSocket createServerSocket(int port) throws IOException {
+        SSLServerSocket srvSock = (SSLServerSocket)delegate.createServerSocket(port);
+
+        if (parameters != null)
+            srvSock.setSSLParameters(parameters);
+
+        return srvSock;
+    }
+
+    /** {@inheritDoc} */
+    @Override public ServerSocket createServerSocket(int port, int backlog) throws IOException {
+        SSLServerSocket srvSock = (SSLServerSocket)delegate.createServerSocket(port, backlog);
+
+        srvSock.setSSLParameters(parameters);
+
+        return srvSock;
+    }
+
+    /** {@inheritDoc} */
+    @Override public ServerSocket createServerSocket(int port, int backlog, InetAddress locAddr) throws IOException {
+        SSLServerSocket srvSock = (SSLServerSocket)delegate.createServerSocket(port, backlog, locAddr);
+
+        if (parameters != null)
+            srvSock.setSSLParameters(parameters);
+
+        return srvSock;
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/SSLSocketFactoryWrapper.java b/modules/core/src/main/java/org/apache/ignite/ssl/SSLSocketFactoryWrapper.java
new file mode 100644
index 0000000000000..bfe6d0d6f4bca
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/ssl/SSLSocketFactoryWrapper.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.ssl;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.Socket;
+import javax.net.ssl.SSLParameters;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.SSLSocketFactory;
+
+/** */
+class SSLSocketFactoryWrapper extends SSLSocketFactory {
+
+    /** */
+    private final SSLSocketFactory delegate;
+
+    /** */
+    private final SSLParameters parameters;
+
+    /** */
+    SSLSocketFactoryWrapper(SSLSocketFactory delegate, SSLParameters parameters) {
+        this.delegate = delegate;
+        this.parameters = parameters;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String[] getDefaultCipherSuites() {
+        return delegate.getDefaultCipherSuites();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String[] getSupportedCipherSuites() {
+        return delegate.getSupportedCipherSuites();
+    }
+
+    /** {@inheritDoc} */
+    @Override public Socket createSocket() throws IOException {
+        SSLSocket sock = (SSLSocket)delegate.createSocket();
+
+        if (parameters != null)
+            sock.setSSLParameters(parameters);
+
+        return sock;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Socket createSocket(Socket sock, String host, int port, boolean autoClose) throws IOException {
+        SSLSocket sslSock = (SSLSocket)delegate.createSocket(sock, host, port, autoClose);
+
+        if (parameters != null)
+            sslSock.setSSLParameters(parameters);
+
+        return sock;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Socket createSocket(String host, int port) throws IOException {
+        SSLSocket sock = (SSLSocket)delegate.createSocket(host, port);
+
+        if (parameters != null)
+            sock.setSSLParameters(parameters);
+
+        return sock;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Socket createSocket(String host, int port, InetAddress locAddr, int locPort) throws IOException {
+        SSLSocket sock = (SSLSocket)delegate.createSocket(host, port, locAddr, locPort);
+
+        if (parameters != null)
+            sock.setSSLParameters(parameters);
+
+        return sock;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Socket createSocket(InetAddress addr, int port) throws IOException {
+        SSLSocket sock = (SSLSocket)delegate.createSocket(addr, port);
+
+        if (parameters != null)
+            sock.setSSLParameters(parameters);
+
+        return sock;
+    }
+
+    /** {@inheritDoc} */
+    @Override public Socket createSocket(InetAddress addr, int port, InetAddress locAddr,
+        int locPort) throws IOException {
+        SSLSocket sock = (SSLSocket)delegate.createSocket(addr, port, locAddr, locPort);
+
+        if (parameters != null)
+            sock.setSSLParameters(parameters);
+
+        return sock;
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java b/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java
index 06edd7014bb50..c514b0fe2d2c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java
@@ -30,6 +30,7 @@
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLParameters;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.TrustManagerFactory;
 import javax.net.ssl.X509TrustManager;
@@ -89,6 +90,12 @@ public class SslContextFactory implements Factory<SSLContext> {
     /** Trust managers. */
     private TrustManager[] trustMgrs;
 
+    /** Enabled cipher suites. */
+    private String[] cipherSuites;
+
+    /** Enabled cipher suites. */
+    private String[] protocols;
+
     /**
      * Gets key store type used for context creation.
      *
@@ -280,6 +287,38 @@ public static TrustManager getDisabledTrustManager() {
         return new DisabledX509TrustManager();
     }
 
+    /**
+     * Sets enabled cipher suites.
+     * @param cipherSuites enabled cipher suites.
+     */
+    public void setCipherSuites(String... cipherSuites) {
+        this.cipherSuites = cipherSuites;
+    }
+
+    /**
+     * Gets enabled cipher suites
+     * @return enabled cipher suites
+     */
+    public String[] getCipherSuites() {
+        return cipherSuites;
+    }
+
+    /**
+     * Gets enabled cipher suites
+     * @return enabled cipher suites
+     */
+    public String[] getProtocols() {
+        return protocols;
+    }
+
+    /**
+     * Sets enabled protocols.
+     * @param protocols enabled protocols.
+     */
+    public void setProtocols(String... protocols) {
+        this.protocols = protocols;
+    }
+
     /**
      * Creates SSL context based on factory settings.
      *
@@ -310,6 +349,18 @@ private SSLContext createSslContext() throws SSLException {
 
             SSLContext ctx = SSLContext.getInstance(proto);
 
+            if (cipherSuites != null || protocols != null) {
+                SSLParameters sslParameters = new SSLParameters();
+
+                if (cipherSuites != null)
+                    sslParameters.setCipherSuites(cipherSuites);
+
+                if (protocols != null)
+                    sslParameters.setProtocols(protocols);
+
+                ctx = new SSLContextWrapper(ctx, sslParameters);
+            }
+
             ctx.init(keyMgrFactory.getKeyManagers(), mgrs, null);
 
             return ctx;
diff --git a/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java b/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java
new file mode 100644
index 0000000000000..7ac6108197a24
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.client;
+
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ClientConfiguration;
+import org.apache.ignite.configuration.ClientConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.ssl.SslContextFactory;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Tests cases when node connects to cluster with different set of cipher suites.
+ */
+public class SslParametersTest extends GridCommonAbstractTest {
+
+    public static final String TEST_CACHE_NAME = "TEST";
+    /** */
+    private volatile String[] cipherSuites;
+
+    /** */
+    private volatile String[] protocols;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientConnectorConfiguration(new ClientConnectorConfiguration()
+            .setSslEnabled(true)
+            .setUseIgniteSslContextFactory(true));
+
+        cfg.setSslContextFactory(createSslFactory());
+
+        CacheConfiguration ccfg = new CacheConfiguration(TEST_CACHE_NAME);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    protected ClientConfiguration getClientConfiguration() throws Exception {
+        ClientConfiguration cfg = new ClientConfiguration();
+
+        cfg.setAddresses("127.0.0.1:10800");
+
+        cfg.setSslMode(SslMode.REQUIRED);
+
+        cfg.setSslContextFactory(createSslFactory());
+
+        return cfg;
+    }
+
+    @NotNull private SslContextFactory createSslFactory() {
+        SslContextFactory factory = (SslContextFactory)GridTestUtils.sslTrustedFactory(
+            "node01", "trustone");
+
+        factory.setCipherSuites(cipherSuites);
+        factory.setProtocols(protocols);
+
+        return factory;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        protocols = null;
+        cipherSuites = null;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSameCipherSuite() throws Exception {
+        cipherSuites = new String[] {
+            "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+            "TLS_RSA_WITH_AES_128_GCM_SHA256",
+            "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+        };
+
+        startGrid();
+
+        checkSuccessfulClientStart(
+            new String[][] {
+                new String[] {
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneCommonCipherSuite() throws Exception {
+        cipherSuites = new String[] {
+            "TLS_RSA_WITH_AES_128_GCM_SHA256",
+            "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+        };
+
+        startGrid();
+        
+        checkSuccessfulClientStart(
+            new String[][] {
+                new String[] {
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoCommonCipherSuite() throws Exception {
+        cipherSuites = new String[] {
+            "TLS_RSA_WITH_AES_128_GCM_SHA256"
+        };
+
+        startGrid();
+        
+        checkClientStartFailure(
+            new String[][] {
+                new String[] {
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNonExistentCipherSuite() throws Exception {
+        cipherSuites = new String[] {
+            "TLS_RSA_WITH_AES_128_GCM_SHA256"
+        };
+
+        startGrid();
+        
+        checkClientStartFailure(
+            new String[][] {
+                new String[] {
+                    "TLC_FAKE_CIPHER",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null,
+            IllegalArgumentException.class,
+            "Unsupported ciphersuite"
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoCommonProtocols() throws Exception {
+        protocols = new String[] {
+            "TLSv1.1",
+            "SSLv3"
+        };
+
+        startGrid();
+
+        checkClientStartFailure(
+            null,
+            new String[][] {
+                new String[] {
+                    "TLSv1",
+                    "TLSv1.2",
+                }
+            }
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNonExistentProtocol() throws Exception {
+        protocols = new String[] {
+            "SSLv3"
+        };
+
+        startGrid();
+
+        checkClientStartFailure(
+            null,
+            new String[][] {
+                new String[] {
+                    "SSLv3",
+                    "SSLvDoesNotExist"
+                }
+            },
+            IllegalArgumentException.class,
+            "SSLvDoesNotExist"
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSameProtocols() throws Exception {
+        protocols = new String[] {
+            "TLSv1.1",
+            "TLSv1.2",
+        };
+
+        startGrid();
+
+        checkSuccessfulClientStart(null,
+            new String[][] {
+                new String[] {
+                    "TLSv1.1",
+                    "TLSv1.2",
+                }
+            }
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneCommonProtocol() throws Exception {
+        protocols = new String[] {
+            "TLSv1",
+            "TLSv1.1",
+            "TLSv1.2"
+        };
+
+        startGrid();
+
+        checkSuccessfulClientStart(null,
+            new String[][] {
+                new String[] {
+                    "TLSv1.1",
+                    "SSLv3"
+                }
+            }
+        );
+    }
+
+    /**
+     * @param cipherSuites list of cipher suites
+     * @param protocols list of protocols
+     * @throws Exception If failed.
+     */
+    private void checkSuccessfulClientStart(String[][] cipherSuites, String[][] protocols) throws Exception {
+        int n = Math.max(
+            cipherSuites != null ? cipherSuites.length : 0,
+            protocols != null ? protocols.length : 0);
+
+        for (int i = 0; i < n; i++) {
+            this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
+            this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
+
+            IgniteClient client = Ignition.startClient(getClientConfiguration());
+
+            client.getOrCreateCache(TEST_CACHE_NAME);
+
+            client.close();
+        }
+    }
+
+    /**
+     * @param cipherSuites list of cipher suites
+     * @param protocols list of protocols
+     * @throws Exception If failed.
+     */
+    private void checkClientStartFailure(String[][] cipherSuites, String[][] protocols) throws Exception {
+        checkClientStartFailure(cipherSuites, protocols, ClientConnectionException.class, "Ignite cluster is unavailable");
+    }
+
+    /**
+     * @param cipherSuites list of cipher suites
+     * @param protocols list of protocols
+     * @param ex expected exception class
+     * @param msg exception message
+     * @throws Exception If failed.
+     */
+    private void checkClientStartFailure(String[][] cipherSuites, String[][] protocols, Class<? extends Throwable> ex, String msg) throws Exception {
+        int n = Math.max(
+            cipherSuites != null ? cipherSuites.length : 0,
+            protocols != null ? protocols.length : 0);
+
+        for (int i = 0; i < n; i++) {
+            this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
+            this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
+
+            int finalI = i;
+
+            GridTestUtils.assertThrows(null, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    Ignition.startClient(getClientConfiguration());
+
+                    return null;
+                }
+            }, ex, msg);
+        }
+    }
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslParametersTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslParametersTest.java
new file mode 100644
index 0000000000000..f2fc2780de9ef
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySslParametersTest.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.ssl.SslContextFactory;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests cases when node connects to cluster with different set of cipher suites.
+ */
+public class TcpDiscoverySslParametersTest extends GridCommonAbstractTest {
+
+    /** */
+    private volatile String[] cipherSuites;
+
+    /** */
+    private volatile String[] protocols;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        SslContextFactory factory = (SslContextFactory)GridTestUtils.sslTrustedFactory(
+            "node01", "trustone");
+
+        factory.setCipherSuites(cipherSuites);
+
+        factory.setProtocols(protocols);
+
+        cfg.setSslContextFactory(factory);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSameCipherSuite() throws Exception {
+        checkDiscoverySuccess(
+            new String[][] {
+                new String[] {
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                },
+                new String[] {
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneCommonCipherSuite() throws Exception {
+        checkDiscoverySuccess(
+            new String[][] {
+                new String[] {
+                    "TLS_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                },
+                new String[] {
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoCommonCipherSuite() throws Exception {
+        checkDiscoveryFailure(
+            new String[][] {
+                new String[] {
+                    "TLS_RSA_WITH_AES_128_GCM_SHA256",
+                },
+                new String[] {
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNonExistentCipherSuite() throws Exception {
+        checkDiscoveryFailure(
+            new String[][] {
+                new String[] {
+                    "TLS_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                },
+                new String[] {
+                    "TLC_FAKE_CIPHER",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null,
+            IgniteCheckedException.class,
+            "Unsupported ciphersuite"
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoCommonProtocols() throws Exception {
+        checkDiscoveryFailure(
+            null,
+            new String[][] {
+                new String[] {
+                    "TLSv1.1",
+                    "SSLv3"
+                },
+                new String[] {
+                    "TLSv1",
+                    "TLSv1.2",
+                }
+            }
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNonExistentProtocol() throws Exception {
+        checkDiscoveryFailure(
+            null,
+            new String[][] {
+                new String[] {
+                    "SSLv3"
+                },
+                new String[] {
+                    "SSLv3",
+                    "SSLvDoesNotExist"
+                }
+            },
+            IgniteCheckedException.class,
+            "SSLvDoesNotExist"
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSameProtocols() throws Exception {
+        checkDiscoverySuccess(null,
+            new String[][] {
+                new String[] {
+                    "TLSv1.1",
+                    "TLSv1.2",
+                },
+                new String[] {
+                    "TLSv1.1",
+                    "TLSv1.2",
+                }
+            }
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneCommonProtocol() throws Exception {
+        checkDiscoverySuccess(null,
+            new String[][] {
+                new String[] {
+                    "TLSv1",
+                    "TLSv1.1",
+                    "TLSv1.2",
+                },
+                new String[] {
+                    "TLSv1.1",
+                    "SSLv3"
+                }
+            }
+        );
+    }
+
+    /**
+     * @param cipherSuites list of cipher suites
+     * @param protocols list of protocols
+     * @throws Exception If failed.
+     */
+    private void checkDiscoverySuccess(String[][] cipherSuites, String[][] protocols) throws Exception {
+        int n = Math.max(
+            cipherSuites != null ? cipherSuites.length : 0,
+            protocols != null ? protocols.length : 0);
+
+        for (int i = 0; i < n; i++) {
+            this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
+            this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
+
+            startGrid(i);
+        }
+    }
+
+    /**
+     * @param cipherSuites list of cipher suites
+     * @param protocols list of protocols
+     * @throws Exception If failed.
+     */
+    private void checkDiscoveryFailure(String[][] cipherSuites, String[][] protocols) throws Exception {
+        checkDiscoveryFailure(cipherSuites, protocols, IgniteCheckedException.class, "Unable to establish secure connection.");
+    }
+
+    /**
+     * @param cipherSuites list of cipher suites
+     * @param protocols list of protocols
+     * @param ex expected exception class
+     * @param msg exception message
+     * @throws Exception If failed.
+     */
+    private void checkDiscoveryFailure(String[][] cipherSuites, String[][] protocols, Class<? extends Throwable> ex, String msg) throws Exception {
+        this.cipherSuites = cipherSuites != null ? cipherSuites[0] : null;
+        this.protocols = protocols != null ? protocols[0] : null;
+
+        startGrid(0);
+
+        int n = Math.max(
+            cipherSuites != null ? cipherSuites.length : 0,
+            protocols != null ? protocols.length : 0);
+
+        for (int i = 1; i < n; i++) {
+            this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
+            this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
+
+            int finalI = i;
+
+            GridTestUtils.assertThrows(null, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    startGrid(finalI);
+
+                    return null;
+                }
+            }, ex, msg);
+        }
+    }
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index f279228f46142..93a371efb4c2b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -1698,7 +1698,7 @@ public static boolean waitForCondition(GridAbsPredicate cond, long timeout) thro
     public static SSLContext sslContext() throws GeneralSecurityException, IOException {
         SSLContext ctx = SSLContext.getInstance("TLS");
 
-        char[] storePass = GridTestProperties.getProperty("ssl.keystore.password").toCharArray();
+        char[] storePass = keyStorePassword().toCharArray();
 
         KeyManagerFactory keyMgrFactory = KeyManagerFactory.getInstance("SunX509");
 
@@ -1725,7 +1725,7 @@ public static GridSslContextFactory sslContextFactory() {
 
         factory.setKeyStoreFilePath(
             U.resolveIgnitePath(GridTestProperties.getProperty("ssl.keystore.path")).getAbsolutePath());
-        factory.setKeyStorePassword(GridTestProperties.getProperty("ssl.keystore.password").toCharArray());
+        factory.setKeyStorePassword(keyStorePassword().toCharArray());
 
         factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
 
@@ -1743,7 +1743,7 @@ public static Factory<SSLContext> sslFactory() {
 
         factory.setKeyStoreFilePath(
             U.resolveIgnitePath(GridTestProperties.getProperty("ssl.keystore.path")).getAbsolutePath());
-        factory.setKeyStorePassword(GridTestProperties.getProperty("ssl.keystore.password").toCharArray());
+        factory.setKeyStorePassword(keyStorePassword().toCharArray());
 
         factory.setTrustManagers(SslContextFactory.getDisabledTrustManager());
 
@@ -1760,16 +1760,23 @@ public static Factory<SSLContext> sslFactory() {
     public static Factory<SSLContext> sslTrustedFactory(String keyStore, String trustStore) {
         SslContextFactory factory = new SslContextFactory();
 
-        factory.setKeyStoreFilePath(U.resolveIgnitePath(GridTestProperties.getProperty(
-            "ssl.keystore." + keyStore + ".path")).getAbsolutePath());
-        factory.setKeyStorePassword(GridTestProperties.getProperty("ssl.keystore.password").toCharArray());
-        factory.setTrustStoreFilePath(U.resolveIgnitePath(GridTestProperties.getProperty(
-            "ssl.keystore." + trustStore + ".path")).getAbsolutePath());
-        factory.setTrustStorePassword(GridTestProperties.getProperty("ssl.keystore.password").toCharArray());
+        factory.setKeyStoreFilePath(keyStorePath(keyStore));
+        factory.setKeyStorePassword(keyStorePassword().toCharArray());
+        factory.setTrustStoreFilePath(keyStorePath(trustStore));
+        factory.setTrustStorePassword(keyStorePassword().toCharArray());
 
         return factory;
     }
 
+    public static String keyStorePassword() {
+        return GridTestProperties.getProperty("ssl.keystore.password");
+    }
+
+    @NotNull public static String keyStorePath(String keyStore) {
+        return U.resolveIgnitePath(GridTestProperties.getProperty(
+            "ssl.keystore." + keyStore + ".path")).getAbsolutePath();
+    }
+
     /**
      * @param o1 Object 1.
      * @param o2 Object 2.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index c6b461a8c1f12..cd746319812f0 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -45,6 +45,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiReconnectDelayTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpiStartStopSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslParametersTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSecuredUnsecuredTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySslTrustedSelfTest;
@@ -119,6 +120,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(TcpDiscoverySslTrustedSelfTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySslSecuredUnsecuredTest.class));
         suite.addTest(new TestSuite(TcpDiscoverySslTrustedUntrustedTest.class));
+        suite.addTest(new TestSuite(TcpDiscoverySslParametersTest.class));
 
         // Disco cache reuse.
         suite.addTest(new TestSuite(IgniteDiscoveryCacheReuseSelfTest.class));
diff --git a/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java
index 3fb243cfe07c4..623a19ebe66a5 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java
@@ -33,7 +33,8 @@
     ReliabilityTest.class,
     SecurityTest.class,
     FunctionalQueryTest.class,
-    IgniteBinaryQueryTest.class
+    IgniteBinaryQueryTest.class,
+    SslParametersTest.class
 })
 public class ClientTestSuite {
     // No-op.

From 4b1a8f76d0b6eb910073d3d770790f9ca9b396fa Mon Sep 17 00:00:00 2001
From: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Date: Wed, 26 Sep 2018 18:52:05 +0300
Subject: [PATCH 387/543] IGN-11747 [IGNITE-9298] control.sh does not support
 SSL (org.apache.ignite.internal.commandline.CommandHandler) in 5.2.1-p14

(cherry picked from commit 148cc7f9f31e72f561ce181ac1e8e128b31d0111)
---
 .../internal/commandline/Arguments.java       |  86 +++++++++-
 .../internal/commandline/CommandHandler.java  | 149 +++++++++++++++++-
 .../util/GridCommandHandlerSslTest.java       |  90 +++++++++++
 3 files changed, 321 insertions(+), 4 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index 5b8a0dcd6c005..0169ce2061092 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -77,6 +77,33 @@ public class Arguments {
     /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}.*/
     private long pingInterval;
 
+    /** */
+    private boolean sslEnable;
+
+    /** */
+    private String sslProtocol;
+
+    /** */
+    private String sslAlgorithm;
+
+    /** */
+    private String sslKeyStorePath;
+
+    /** */
+    private String sslKeyStoreType;
+
+    /** */
+    private char sslKeyStorePassword[];
+
+    /** */
+    private String sslTrustStorePath;
+
+    /** */
+    private String sslTrustStoreType;
+
+    /** */
+    private char sslTrustStorePassword[];
+
     /**
      * @param cmd Command.
      * @param host Host.
@@ -95,7 +122,10 @@ public class Arguments {
      */
     public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
                      String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
-                     Long pingTimeout, Long pingInterval, boolean autoConfirmation) {
+                     Long pingTimeout, Long pingInterval, boolean autoConfirmation,
+                     boolean sslEnable, String sslProtocol, String sslAlgorithm,
+                     String sslKeyStorePath, String sslKeyStoreType, char sslKeyStorePassword[],
+                     String sslTrustStorePath, String sslTrustStoreType, char sslTrustStorePassword[]) {
         this.cmd = cmd;
         this.host = host;
         this.port = port;
@@ -110,6 +140,15 @@ public Arguments(Command cmd, String host, String port, String user, String pwd,
         this.pingTimeout = pingTimeout;
         this.pingInterval = pingInterval;
         this.autoConfirmation = autoConfirmation;
+        this.sslEnable = sslEnable;
+        this.sslProtocol = sslProtocol;
+        this.sslAlgorithm = sslAlgorithm;
+        this.sslKeyStorePath = sslKeyStorePath;
+        this.sslKeyStoreType = sslKeyStoreType;
+        this.sslKeyStorePassword = sslKeyStorePassword;
+        this.sslTrustStorePath = sslTrustStorePath;
+        this.sslTrustStoreType = sslTrustStoreType;
+        this.sslTrustStorePassword = sslTrustStorePassword;
     }
 
     /**
@@ -213,4 +252,49 @@ public long pingInterval() {
     public boolean autoConfirmation() {
         return autoConfirmation;
     }
+
+    /** */
+    public boolean isSslEnable() {
+        return sslEnable;
+    }
+
+    /** */
+    public String getSslProtocol() {
+        return sslProtocol;
+    }
+
+    /** */
+    public String getSslAlgorithm() {
+        return sslAlgorithm;
+    }
+
+    /** */
+    public String getSslKeyStorePath() {
+        return sslKeyStorePath;
+    }
+
+    /** */
+    public String getSslKeyStoreType() {
+        return sslKeyStoreType;
+    }
+
+    /** */
+    public char[] getSslKeyStorePassword() {
+        return sslKeyStorePassword;
+    }
+
+    /** */
+    public String getSslTrustStorePath() {
+        return sslTrustStorePath;
+    }
+
+    /** */
+    public String getSslTrustStoreType() {
+        return sslTrustStoreType;
+    }
+
+    /** */
+    public char[] getSslTrustStorePassword() {
+        return sslTrustStorePassword;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 092efffe60862..606599502b3f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -49,6 +49,7 @@
 import org.apache.ignite.internal.client.GridClientNode;
 import org.apache.ignite.internal.client.GridServerUnreachableException;
 import org.apache.ignite.internal.client.impl.connection.GridClientConnectionResetException;
+import org.apache.ignite.internal.client.ssl.GridSslBasicContextFactory;
 import org.apache.ignite.internal.commandline.cache.CacheArguments;
 import org.apache.ignite.internal.commandline.cache.CacheCommand;
 import org.apache.ignite.internal.processors.cache.verify.CacheInfo;
@@ -95,10 +96,10 @@
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTask;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskResult;
-import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
+import org.apache.ignite.ssl.SslContextFactory;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
 import static org.apache.ignite.internal.IgniteVersionUtils.ACK_VER_STR;
@@ -161,6 +162,35 @@ public class CommandHandler {
     /** */
     private static final String CMD_SKIP_ZEROS = "--skipZeros";
 
+    // SSL configuration section
+
+    /** */
+    protected static final String CMD_SSL_ENABLED = "--ssl_enabled";
+
+    /** */
+    protected static final String CMD_SSL_PROTOCOL = "--ssl_protocol";
+
+    /** */
+    protected static final String CMD_SSL_ALGORITHM = "--ssl_algorithm";
+
+    /** */
+    protected static final String CMD_SSL_KEY_STORE_PATH = "--ssl_key_store_path";
+
+    /** */
+    protected static final String CMD_SSL_KEY_STORE_TYPE = "--ssl_key_store_type";
+
+    /** */
+    protected static final String CMD_SSL_KEY_STORE_PASSWORD = "--ssl_key_store_password";
+
+    /** */
+    protected static final String CMD_SSL_TRUSTSTORE_PATH = "--ssl_truststore_path";
+
+    /** */
+    protected static final String CMD_SSL_TRUSTSTORE_TYPE = "--ssl_truststore_type";
+
+    /** */
+    protected static final String CMD_SSL_TRUSTSTORE_PASSWORD = "--ssl_truststore_password";
+
     /** List of optional auxiliary commands. */
     private static final Set<String> AUX_COMMANDS = new HashSet<>();
 
@@ -173,6 +203,15 @@ public class CommandHandler {
         AUX_COMMANDS.add(CMD_AUTO_CONFIRMATION);
         AUX_COMMANDS.add(CMD_PING_INTERVAL);
         AUX_COMMANDS.add(CMD_PING_TIMEOUT);
+        AUX_COMMANDS.add(CMD_SSL_ENABLED);
+        AUX_COMMANDS.add(CMD_SSL_PROTOCOL);
+        AUX_COMMANDS.add(CMD_SSL_ALGORITHM);
+        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PATH);
+        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_TYPE);
+        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PASSWORD);
+        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PATH);
+        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_TYPE);
+        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PASSWORD);
     }
 
     /** Broadcast uuid. */
@@ -1240,7 +1279,15 @@ private boolean isConnectionError(Throwable e) {
     private void usage(String desc, Command cmd, String... args) {
         log(desc);
         log("    control.sh [--host HOST_OR_IP] [--port PORT] [--user USER] [--password PASSWORD] " +
-            " [--ping-interval PING_INTERVAL] [--ping-timeout PING_TIMEOUT] " + cmd.text() + String.join("", args));
+            " [--ping-interval PING_INTERVAL] [--ping-timeout PING_TIMEOUT] " +
+            "[" + CMD_SSL_ENABLED + "] " +
+            "[" + CMD_SSL_KEY_STORE_PATH + " PATH] " +
+            "[" + CMD_SSL_KEY_STORE_TYPE + " jks] " +
+            "[" + CMD_SSL_KEY_STORE_PASSWORD + " PASSWORD] " +
+            "[" + CMD_SSL_TRUSTSTORE_PATH + " PATH] " +
+            "[" + CMD_SSL_TRUSTSTORE_TYPE + " jks] " +
+            "[" + CMD_SSL_TRUSTSTORE_PASSWORD + " PASSWORD]" +
+            cmd.text() + String.join("", args));
         nl();
     }
 
@@ -1315,6 +1362,24 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         VisorTxTaskArg txArgs = null;
 
+        boolean sslEnable= false;
+
+        String sslProtocol = SslContextFactory.DFLT_SSL_PROTOCOL;
+
+        String sslAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM;
+
+        String sslKeyStorePath = null;
+
+        String sslKeyStoreType = SslContextFactory.DFLT_STORE_TYPE;
+
+        char sslKeyStorePassword[] = null;
+
+        String sslTrustStorePath = null;
+
+        String sslTrustStoreType = SslContextFactory.DFLT_STORE_TYPE;
+
+        char sslTrustStorePassword[] = null;
+
         while (hasNextArg()) {
             String str = nextArg("").toLowerCase();
 
@@ -1432,6 +1497,51 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
+                    case CMD_SSL_ENABLED:
+                        sslEnable = true;
+
+                        break;
+
+                    case CMD_SSL_PROTOCOL:
+                        sslProtocol = nextArg("Expected ssl protocol");
+
+                        break;
+
+                    case CMD_SSL_ALGORITHM:
+                        sslAlgorithm = nextArg("Expected ssl algorithm");
+
+                        break;
+
+                    case CMD_SSL_KEY_STORE_PATH:
+                        sslKeyStorePath = nextArg("Expected ssl key store path");
+
+                        break;
+
+                    case CMD_SSL_KEY_STORE_TYPE:
+                        sslKeyStoreType = nextArg("Expected ssl key store type");
+
+                        break;
+
+                    case CMD_SSL_KEY_STORE_PASSWORD:
+                        sslKeyStorePassword = nextArg("Expected ssl key store password").toCharArray();
+
+                        break;
+
+                    case CMD_SSL_TRUSTSTORE_PATH:
+                        sslTrustStorePath = nextArg("Expected ssl trust store path");
+
+                        break;
+
+                    case CMD_SSL_TRUSTSTORE_TYPE:
+                        sslTrustStoreType = nextArg("Expected ssl trust store type");
+
+                        break;
+
+                    case CMD_SSL_TRUSTSTORE_PASSWORD:
+                        sslTrustStorePassword = nextArg("Expected ssl trust store password").toCharArray();
+
+                        break;
+
                     default:
                         throw new IllegalArgumentException("Unexpected argument: " + str);
                 }
@@ -1455,7 +1565,10 @@ Arguments parseAndValidate(List<String> rawArgs) {
             throw new IllegalArgumentException("Both user and password should be specified");
 
         return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
-            pingTimeout, pingInterval, autoConfirmation);
+            pingTimeout, pingInterval, autoConfirmation,
+            sslEnable, sslProtocol, sslAlgorithm,
+            sslKeyStorePath, sslKeyStoreType, sslKeyStorePassword,
+            sslTrustStorePath, sslTrustStoreType, sslTrustStorePassword);
     }
 
     /**
@@ -1861,6 +1974,36 @@ public int execute(List<String> rawArgs) {
                     new SecurityCredentialsBasicProvider(new SecurityCredentials(args.user(), args.password())));
             }
 
+            if (args.isSslEnable()){
+                GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
+
+                factory.setProtocol(args.getSslProtocol());
+                factory.setKeyAlgorithm(args.getSslAlgorithm());
+
+                if (args.getSslKeyStorePath()==null)
+                    throw new IllegalArgumentException("SSL key store location is not specified.");
+
+                factory.setKeyStoreFilePath(args.getSslKeyStorePath());
+
+                if (args.getSslKeyStorePassword()!=null)
+                    factory.setKeyStorePassword(args.getSslKeyStorePassword());
+
+                factory.setKeyStoreType(args.getSslKeyStoreType());
+
+                if (args.getSslTrustStorePath()==null)
+                    factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
+                else {
+                    factory.setTrustStoreFilePath(args.getSslTrustStorePath());
+
+                    if (args.getSslTrustStorePassword()!=null)
+                        factory.setTrustStorePassword(args.getSslTrustStorePassword());
+
+                    factory.setTrustStoreType(args.getSslTrustStoreType());
+                }
+
+                clientCfg.setSslContextFactory(factory);
+            }
+
             try (GridClient client = GridClientFactory.start(clientCfg)) {
                 switch (args.command()) {
                     case ACTIVATE:
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
new file mode 100644
index 0000000000000..fb4092af9b968
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.util.Arrays;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.commandline.CommandHandler;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_CONNECTION_FAILED;
+import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+
+/**
+ * Command line handler test.
+ */
+public class GridCommandHandlerSslTest extends GridCommonAbstractTest {
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        cleanPersistenceDir();
+
+        stopAllGrids();
+
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration());
+        cfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration().setMaxSize(100 * 1024 * 1024);
+        cfg.getDataStorageConfiguration().getDefaultDataRegionConfiguration().setPersistenceEnabled(true);
+
+        cfg.setConnectorConfiguration(new ConnectorConfiguration());
+        cfg.getConnectorConfiguration().setSslEnabled(true);
+        cfg.setSslContextFactory(GridTestUtils.sslFactory());
+
+        return cfg;
+    }
+
+    /**
+     * Test activation works via control.sh
+     *
+     * @throws Exception If failed.
+     */
+    public void testActivate() throws Exception {
+        Ignite ignite = startGrids(1);
+
+        assertFalse(ignite.cluster().active());
+
+        final CommandHandler cmd = new CommandHandler();
+        assertEquals(EXIT_CODE_OK, cmd.execute(Arrays.asList(
+            "--activate",
+            "--ssl_enabled",
+            "--ssl_key_store_path", GridTestUtils.keyStorePath("node01"),
+            "--ssl_key_store_password", GridTestUtils.keyStorePassword())));
+
+        assertTrue(ignite.cluster().active());
+
+        assertEquals(EXIT_CODE_CONNECTION_FAILED, cmd.execute(Arrays.asList("--deactivate", "--yes")));
+    }
+
+}

From c13698d3027e1e2dd44dc60be55e9fc69f7ddabb Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <macrergate@gmail.com>
Date: Thu, 27 Sep 2018 16:36:07 +0300
Subject: [PATCH 388/543] =?UTF-8?q?GG-14244=20o.a.i.i.processors.cache.Gri?=
 =?UTF-8?q?dCacheMapEntry#updateTtl(long)=20w=E2=80=A6=20(#103)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../processors/cache/GridCacheMapEntry.java   | 22 +++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 26c6fee1ecf34..3c51a7123bdfc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -34,15 +34,14 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.eviction.EvictableEntry;
-import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult.UpdateOutcome;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
 import org.apache.ignite.internal.processors.cache.extras.GridCacheEntryExtras;
 import org.apache.ignite.internal.processors.cache.extras.GridCacheMvccEntryExtras;
@@ -51,6 +50,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cache.query.continuous.CacheContinuousQueryListener;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
@@ -2463,7 +2463,14 @@ private void updateTtl(long ttl) throws IgniteCheckedException, GridCacheEntryRe
 
         ttlAndExpireTimeExtras(ttl, expireTime);
 
-        storeValue(val, expireTime, ver, null);
+        cctx.shared().database().checkpointReadLock();
+
+        try {
+            storeValue(val, expireTime, ver, null);
+        }
+        finally {
+            cctx.shared().database().checkpointReadUnlock();
+        }
     }
 
     /**
@@ -3407,7 +3414,14 @@ private boolean onExpired(CacheObject expiredVal, GridCacheVersion obsoleteVer)
         if (log.isTraceEnabled())
             log.trace("onExpired clear [key=" + key + ", entry=" + System.identityHashCode(this) + ']');
 
-        removeValue();
+        cctx.shared().database().checkpointReadLock();
+
+        try {
+            removeValue();
+        }
+        finally {
+            cctx.shared().database().checkpointReadUnlock();
+        }
 
         if (cctx.events().isRecordable(EVT_CACHE_OBJECT_EXPIRED)) {
             cctx.events().addEvent(partition(),

From 9134d9eb916f2d668768c23d9745a67183e36d30 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Thu, 27 Sep 2018 14:09:48 +0300
Subject: [PATCH 389/543] IGNITE-9612 Improve checkpoint mark phase speed -
 Fixes #4813.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
(cherry picked from commit 5e220f90d05fcc92eb655329a96f909f444ac75f)
---
 .../ignite/internal/GridKernalContext.java    |   5 +
 .../internal/GridKernalContextImpl.java       |  13 +-
 .../apache/ignite/internal/IgniteKernal.java  |   7 +-
 .../apache/ignite/internal/IgnitionEx.java    |   3 +-
 .../persistence/DbCheckpointListener.java     |   8 +
 .../GridCacheDatabaseSharedManager.java       |  95 +++++++--
 .../persistence/GridCacheOffheapManager.java  |  68 +++++--
 .../persistence/metastorage/MetaStorage.java  |  29 ++-
 .../reader/StandaloneGridKernalContext.java   |   5 +
 .../IgniteTaskTrackingThreadPoolExecutor.java | 180 ++++++++++++++++++
 ...tePersistenceSequentialCheckpointTest.java |   6 +-
 .../junits/GridTestKernalContext.java         |   1 +
 .../testsuites/IgniteUtilSelfTestSuite.java   |   6 +
 ...iteTaskTrackingThreadPoolExecutorTest.java | 140 ++++++++++++++
 14 files changed, 514 insertions(+), 52 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/thread/IgniteTaskTrackingThreadPoolExecutor.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/util/IgniteTaskTrackingThreadPoolExecutorTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index 051978c0fc728..a4560f95389b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -679,4 +679,9 @@ public interface GridKernalContext extends Iterable<GridComponent> {
      * @return subscription processor to manage internal-only (strict node-local) subscriptions between components.
      */
     public GridInternalSubscriptionProcessor internalSubscriptionProcessor();
+
+    /**
+     * @return Default uncaught exception handler used by thread pools.
+     */
+    public Thread.UncaughtExceptionHandler uncaughtExceptionHandler();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index 2be64e5d1330c..dfb51bcf18225 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -366,6 +366,9 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     @GridToStringExclude
     private WorkersRegistry workersRegistry;
 
+    /** */
+    private Thread.UncaughtExceptionHandler hnd;
+
     /** */
     private IgniteEx grid;
 
@@ -433,6 +436,7 @@ public GridKernalContextImpl() {
      * @param customExecSvcs Custom named executors.
      * @param plugins Plugin providers.
      * @param workerRegistry Worker registry.
+     * @param hnd Default uncaught exception handler used by thread pools.
      */
     @SuppressWarnings("TypeMayBeWeakened")
     protected GridKernalContextImpl(
@@ -458,7 +462,8 @@ protected GridKernalContextImpl(
         @Nullable Map<String, ? extends ExecutorService> customExecSvcs,
         List<PluginProvider> plugins,
         IgnitePredicate<String> clsFilter,
-        WorkersRegistry workerRegistry
+        WorkersRegistry workerRegistry,
+        Thread.UncaughtExceptionHandler hnd
     ) {
         assert grid != null;
         assert cfg != null;
@@ -484,6 +489,7 @@ protected GridKernalContextImpl(
         this.schemaExecSvc = schemaExecSvc;
         this.customExecSvcs = customExecSvcs;
         this.workersRegistry = workerRegistry;
+        this.hnd = hnd;
 
         marshCtx = new MarshallerContextImpl(plugins, clsFilter);
 
@@ -1145,6 +1151,11 @@ void disconnected(boolean disconnected) {
         return failureProc;
     }
 
+    /** {@inheritDoc} */
+    public Thread.UncaughtExceptionHandler uncaughtExceptionHandler() {
+        return hnd;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridKernalContextImpl.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 4979e610b9046..306bb66f0cd8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -767,6 +767,7 @@ private void notifyLifecycleBeansEx(LifecycleEventType evt) {
      * @param customExecSvcs Custom named executors.
      * @param errHnd Error handler to use for notification about startup problems.
      * @param workerRegistry Worker registry.
+     * @param hnd Default uncaught exception handler used by thread pools.
      * @throws IgniteCheckedException Thrown in case of any errors.
      */
     @SuppressWarnings({"CatchGenericClass", "unchecked"})
@@ -789,7 +790,8 @@ public void start(
         ExecutorService schemaExecSvc,
         @Nullable final Map<String, ? extends ExecutorService> customExecSvcs,
         GridAbsClosure errHnd,
-        WorkersRegistry workerRegistry
+        WorkersRegistry workerRegistry,
+        Thread.UncaughtExceptionHandler hnd
     )
         throws IgniteCheckedException {
         gw.compareAndSet(null, new GridKernalGatewayImpl(cfg.getIgniteInstanceName()));
@@ -906,7 +908,8 @@ public void start(
                 customExecSvcs,
                 plugins,
                 classNameFilter(),
-                workerRegistry
+                workerRegistry,
+                hnd
             );
 
             cfg.getMarshaller().setContext(ctx.marshallerContext());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 6aa9547e35946..44cb95626e324 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -2040,7 +2040,8 @@ private void start0(GridStartContext startCtx) throws IgniteCheckedException {
                             startLatch.countDown();
                         }
                     },
-                    workerRegistry
+                    workerRegistry,
+                    oomeHnd
                 );
 
                 state = STARTED;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java
index 1c438b86b1daa..36ab1639820bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java
@@ -17,8 +17,11 @@
 
 package org.apache.ignite.internal.processors.cache.persistence;
 
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap;
+import org.jetbrains.annotations.Nullable;
 
 /**
  *
@@ -42,6 +45,11 @@ public interface Context {
          * @param cacheOrGrpName Cache or group name.
          */
         public boolean needToSnapshot(String cacheOrGrpName);
+
+        /**
+         * @return Context executor.
+         */
+        public @Nullable Executor executor();
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 6c4965b2613e1..a6d58a6ea2e06 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -45,6 +45,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.RejectedExecutionException;
@@ -77,6 +78,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.NodeStoppingException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.DirectMemoryRegion;
@@ -155,7 +157,7 @@
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.mxbean.DataStorageMetricsMXBean;
 import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.thread.IgniteThreadPoolExecutor;
+import org.apache.ignite.thread.IgniteTaskTrackingThreadPoolExecutor;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentLinkedHashMap;
@@ -246,6 +248,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Timeout between partition file destroy and checkpoint to handle it. */
     private static final long PARTITION_DESTROY_CHECKPOINT_TIMEOUT = 30 * 1000; // 30 Seconds.
 
+    /** */
+    private static final String CHECKPOINT_RUNNER_THREAD_PREFIX = "checkpoint-runner";
+
     /** Checkpoint thread. Needs to be volatile because it is created in exchange worker. */
     private volatile Checkpointer checkpointer;
 
@@ -283,7 +288,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     private boolean stopping;
 
     /** Checkpoint runner thread pool. If null tasks are to be run in single thread */
-    @Nullable private ExecutorService asyncRunner;
+    @Nullable private IgniteTaskTrackingThreadPoolExecutor asyncRunner;
 
     /** Thread local with buffers for the checkpoint threads. Each buffer represent one page for durable memory. */
     private ThreadLocal<ByteBuffer> threadBuf;
@@ -722,13 +727,15 @@ else if (regCfg.getMaxSize() < 8 * GB)
      */
     private void initDataBase() {
         if (persistenceCfg.getCheckpointThreads() > 1)
-            asyncRunner = new IgniteThreadPoolExecutor(
-                "checkpoint-runner",
+            asyncRunner = new IgniteTaskTrackingThreadPoolExecutor(
+                CHECKPOINT_RUNNER_THREAD_PREFIX,
                 cctx.igniteInstanceName(),
                 persistenceCfg.getCheckpointThreads(),
                 persistenceCfg.getCheckpointThreads(),
-                30_000,
-                new LinkedBlockingQueue<Runnable>()
+                30_000, // A value is ignored if corePoolSize equals to maxPoolSize
+                new LinkedBlockingQueue<Runnable>(),
+                GridIoPolicy.UNDEFINED,
+                cctx.kernalContext().uncaughtExceptionHandler()
             );
     }
 
@@ -1504,7 +1511,8 @@ private void prepareIndexRebuildFuture(int cacheId) {
     @Override public boolean checkpointLockIsHeldByThread() {
         return !ASSERTION_ENABLED ||
             checkpointLock.isWriteLockedByCurrentThread() ||
-            CHECKPOINT_LOCK_HOLD_COUNT.get() > 0;
+            CHECKPOINT_LOCK_HOLD_COUNT.get() > 0 ||
+            Thread.currentThread().getName().startsWith(CHECKPOINT_RUNNER_THREAD_PREFIX);
     }
 
     /**
@@ -3107,7 +3115,7 @@ private void doCheckpoint() {
                     // In case of checkpoint initialization error node should be invalidated and stopped.
                     cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
-                    return;
+                    throw new IgniteException(e); // Re-throw as unchecked exception to force stopping checkpoint thread.
                 }
 
                 currCheckpointPagesCnt = chp.pagesSize;
@@ -3507,6 +3515,9 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
 
                 final PartitionAllocationMap map = new PartitionAllocationMap();
 
+                if (asyncRunner != null)
+                    asyncRunner.reset();
+
                 DbCheckpointListener.Context ctx0 = new DbCheckpointListener.Context() {
                     @Override public boolean nextSnapshot() {
                         return curr.nextSnapshot;
@@ -3517,39 +3528,81 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
                         return map;
                     }
 
+                    /** {@inheritDoc} */
                     @Override public boolean needToSnapshot(String cacheOrGrpName) {
                         return curr.snapshotOperation.cacheGroupIds().contains(CU.cacheId(cacheOrGrpName));
                     }
+
+                    /** {@inheritDoc} */
+                    @Override public Executor executor() {
+                        return asyncRunner == null ? null : cmd -> {
+                            try {
+                                asyncRunner.execute(cmd);
+                            }
+                            catch (RejectedExecutionException e) {
+                                assert false: "A task should never be rejected by async runner";
+                            }
+                        };
+                    }
                 };
 
                 // Listeners must be invoked before we write checkpoint record to WAL.
                 for (DbCheckpointListener lsnr : lsnrs)
                     lsnr.onCheckpointBegin(ctx0);
 
+                if (asyncRunner != null) {
+                    asyncRunner.markInitialized();
+
+                    asyncRunner.awaitDone();
+                }
+
                 if (curr.nextSnapshot)
                     snapFut = snapshotMgr.onMarkCheckPointBegin(curr.snapshotOperation, map);
 
+                if (asyncRunner != null)
+                    asyncRunner.reset();
+
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                     if (grp.isLocal() || !grp.walEnabled())
                         continue;
 
-                    ArrayList<GridDhtLocalPartition> parts = new ArrayList<>();
+                    Runnable r = () -> {
+                        ArrayList<GridDhtLocalPartition> parts = new ArrayList<>(grp.topology().localPartitions().size());
 
-                    for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions())
-                        parts.add(part);
+                        for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions())
+                            parts.add(part);
 
-                    CacheState state = new CacheState(parts.size());
+                        CacheState state = new CacheState(parts.size());
 
-                    for (GridDhtLocalPartition part : parts) {
-                        state.addPartitionState(
-                            part.id(),
-                            part.dataStore().fullSize(),
-                            part.updateCounter(),
-                            (byte)part.state().ordinal()
-                        );
-                    }
+                        for (GridDhtLocalPartition part : parts) {
+                            state.addPartitionState(
+                                part.id(),
+                                part.dataStore().fullSize(),
+                                part.updateCounter(),
+                                (byte)part.state().ordinal()
+                            );
+                        }
+
+                        synchronized (cpRec) {
+                            cpRec.addCacheGroupState(grp.groupId(), state);
+                        }
+                    };
+
+                    if (asyncRunner == null)
+                        r.run();
+                    else
+                        try {
+                            asyncRunner.execute(r);
+                        }
+                        catch (RejectedExecutionException e) {
+                            assert false: "Task should never be rejected by async runner";
+                        }
+                }
+
+                if (asyncRunner != null) {
+                    asyncRunner.markInitialized();
 
-                    cpRec.addCacheGroupState(grp.groupId(), state);
+                    asyncRunner.awaitDone();
                 }
 
                 cpPagesTuple = beginAllCheckpoints();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 16d218f9df001..61cdeb88bf267 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -24,6 +24,7 @@
 import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -161,30 +162,63 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
     @Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
         assert grp.dataRegion().pageMemory() instanceof PageMemoryEx;
 
-        reuseList.saveMetadata();
+        Executor execSvc = ctx.executor();
 
-        boolean metaWasUpdated = false;
+        if (execSvc == null) {
+            reuseList.saveMetadata();
 
-        for (CacheDataStore store : partDataStores.values())
-            metaWasUpdated |= saveStoreMetadata(store, ctx, !metaWasUpdated, false);
+            if (ctx.nextSnapshot())
+                updateSnapshotTag(ctx);
+
+            for (CacheDataStore store : partDataStores.values())
+                saveStoreMetadata(store, ctx, false);
+        }
+        else {
+            execSvc.execute(() -> {
+                try {
+                    reuseList.saveMetadata();
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IgniteException(e);
+                }
+            });
+
+            if (ctx.nextSnapshot()) {
+                execSvc.execute(() -> {
+                    try {
+                        updateSnapshotTag(ctx);
+                    }
+                    catch (IgniteCheckedException e) {
+                        throw new IgniteException(e);
+                    }
+                });
+            }
+
+            for (CacheDataStore store : partDataStores.values())
+                execSvc.execute(() -> {
+                    try {
+                        saveStoreMetadata(store, ctx, false);
+                    }
+                    catch (IgniteCheckedException e) {
+                        throw new IgniteException(e);
+                    }
+                });
+        }
     }
 
     /**
      * @param store Store to save metadata.
      * @throws IgniteCheckedException If failed.
      */
-    private boolean saveStoreMetadata(
+    private void saveStoreMetadata(
         CacheDataStore store,
         Context ctx,
-        boolean saveMeta,
         boolean beforeDestroy
     ) throws IgniteCheckedException {
         RowStore rowStore0 = store.rowStore();
 
         boolean needSnapshot = ctx != null && ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName());
 
-        boolean wasSaveToMeta = false;
-
         if (rowStore0 != null) {
             CacheFreeListImpl freeList = (CacheFreeListImpl)rowStore0.freeList();
 
@@ -215,7 +249,7 @@ private boolean saveStoreMetadata(
 
                     // Do not save meta for evicted partitions on next checkpoints.
                     if (state == null)
-                        return false;
+                        return;
                 }
 
                 int grpId = grp.groupId();
@@ -227,10 +261,10 @@ private boolean saveStoreMetadata(
 
                     if (partMetaPageAddr == 0L) {
                         U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage +
-                            ", saveMeta=" + saveMeta + ", beforeDestroy=" + beforeDestroy + ", size=" + size +
+                            ", beforeDestroy=" + beforeDestroy + ", size=" + size +
                             ", updCntr=" + updCntr + ", state=" + state + ']');
 
-                        return false;
+                        return;
                     }
 
                     boolean changed = false;
@@ -278,12 +312,6 @@ private boolean saveStoreMetadata(
 
                             io.setCandidatePageCount(partMetaPageAddr, size == 0 ? 0: pageCnt);
 
-                            if (saveMeta) {
-                                saveMeta(ctx);
-
-                                wasSaveToMeta = true;
-                            }
-
                             if (state == OWNING) {
                                 assert part != null;
 
@@ -339,8 +367,6 @@ else if (needSnapshot)
         }
         else if (needSnapshot)
             tryAddEmptyPartitionToSnapshot(store, ctx);
-
-        return wasSaveToMeta;
     }
 
     /**
@@ -489,7 +515,7 @@ private static long writeSharedGroupCacheSizes(PageMemory pageMem, int grpId,
     /**
      * @param ctx Context.
      */
-    private void saveMeta(Context ctx) throws IgniteCheckedException {
+    private void updateSnapshotTag(Context ctx) throws IgniteCheckedException {
         int grpId = grp.groupId();
         PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
         IgniteWriteAheadLogManager wal = this.ctx.wal();
@@ -585,7 +611,7 @@ private static boolean addPartition(
         ctx.database().checkpointReadLock();
 
         try {
-            saveStoreMetadata(store, null, false, true);
+            saveStoreMetadata(store, null, true);
         }
         finally {
             ctx.database().checkpointReadUnlock();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index c0fba7308467b..556d9974ecc2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -22,8 +22,10 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -415,9 +417,32 @@ public PageMemory pageMemory() {
 
     /** {@inheritDoc} */
     @Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
-        freeList.saveMetadata();
+        Executor executor = ctx.executor();
 
-        saveStoreMetadata();
+        if (executor == null) {
+            freeList.saveMetadata();
+
+            saveStoreMetadata();
+        }
+        else {
+            executor.execute(() -> {
+                try {
+                    freeList.saveMetadata();
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IgniteException(e);
+                }
+            });
+
+            executor.execute(() -> {
+                try {
+                    saveStoreMetadata();
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IgniteException(e);
+                }
+            });
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index b9ab76a3c7b05..cd4179645a3b3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -635,6 +635,11 @@ protected IgniteConfiguration prepareIgniteConfiguration() {
         return null;
     }
 
+    /** {@inheritDoc} */
+    @Override public Thread.UncaughtExceptionHandler uncaughtExceptionHandler() {
+        return null;
+    }
+
     /** {@inheritDoc} */
     @Override public PdsFoldersResolver pdsFolderResolver() {
         return new PdsFoldersResolver() {
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteTaskTrackingThreadPoolExecutor.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteTaskTrackingThreadPoolExecutor.java
new file mode 100644
index 0000000000000..6cae57eb78e6a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteTaskTrackingThreadPoolExecutor.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.thread;
+
+import java.lang.Thread.UncaughtExceptionHandler;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.LongAdder;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+
+/**
+ * An {@link ExecutorService} that executes submitted tasks using pooled grid threads.
+ *
+ * In addition to what it allows to track all enqueued tasks completion or failure during execution.
+ */
+public class IgniteTaskTrackingThreadPoolExecutor extends IgniteThreadPoolExecutor {
+    /** */
+    private final LongAdder pendingTaskCnt = new LongAdder();
+
+    /** */
+    private final LongAdder completedTaskCnt = new LongAdder();
+
+    /** */
+    private volatile boolean initialized;
+
+    /** */
+    private volatile AtomicReference<Throwable> err = new AtomicReference<>();
+
+    /**
+     * Creates a new service with the given initial parameters.
+     *
+     * @param threadNamePrefix Will be added at the beginning of all created threads.
+     * @param igniteInstanceName Must be the name of the grid.
+     * @param corePoolSize The number of threads to keep in the pool, even if they are idle.
+     * @param maxPoolSize The maximum number of threads to allow in the pool.
+     * @param keepAliveTime When the number of threads is greater than the core, this is the maximum time
+     *      that excess idle threads will wait for new tasks before terminating.
+     * @param workQ The queue to use for holding tasks before they are executed. This queue will hold only
+     *      runnable tasks submitted by the {@link #execute(Runnable)} method.
+     */
+    public IgniteTaskTrackingThreadPoolExecutor(String threadNamePrefix, String igniteInstanceName, int corePoolSize,
+        int maxPoolSize, long keepAliveTime, BlockingQueue<Runnable> workQ) {
+        super(threadNamePrefix, igniteInstanceName, corePoolSize, maxPoolSize, keepAliveTime, workQ);
+    }
+
+    /**
+     * Creates a new service with the given initial parameters.
+     *
+     * @param threadNamePrefix Will be added at the beginning of all created threads.
+     * @param igniteInstanceName Must be the name of the grid.
+     * @param corePoolSize The number of threads to keep in the pool, even if they are idle.
+     * @param maxPoolSize The maximum number of threads to allow in the pool.
+     * @param keepAliveTime When the number of threads is greater than the core, this is the maximum time
+     *      that excess idle threads will wait for new tasks before terminating.
+     * @param workQ The queue to use for holding tasks before they are executed. This queue will hold only
+     *      runnable tasks submitted by the {@link #execute(Runnable)} method.
+     * @param plc {@link GridIoPolicy} for thread pool.
+     * @param eHnd Uncaught exception handler for thread pool.
+     */
+    public IgniteTaskTrackingThreadPoolExecutor(String threadNamePrefix, String igniteInstanceName, int corePoolSize,
+        int maxPoolSize, long keepAliveTime, BlockingQueue<Runnable> workQ, byte plc,
+        UncaughtExceptionHandler eHnd) {
+        super(threadNamePrefix, igniteInstanceName, corePoolSize, maxPoolSize, keepAliveTime, workQ, plc, eHnd);
+    }
+
+    /**
+     * Creates a new service with the given initial parameters.
+     *
+     * @param corePoolSize The number of threads to keep in the pool, even if they are idle.
+     * @param maxPoolSize The maximum number of threads to allow in the pool.
+     * @param keepAliveTime When the number of threads is greater than the core, this is the maximum time
+     *      that excess idle threads will wait for new tasks before terminating.
+     * @param workQ The queue to use for holding tasks before they are executed. This queue will hold only the
+     *      runnable tasks submitted by the {@link #execute(Runnable)} method.
+     * @param threadFactory Thread factory.
+     */
+    public IgniteTaskTrackingThreadPoolExecutor(int corePoolSize, int maxPoolSize, long keepAliveTime,
+        BlockingQueue<Runnable> workQ, ThreadFactory threadFactory) {
+        super(corePoolSize, maxPoolSize, keepAliveTime, workQ, threadFactory);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void execute(Runnable cmd) {
+        pendingTaskCnt.add(1);
+
+        super.execute(cmd);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterExecute(Runnable r, Throwable t) {
+        super.afterExecute(r, t);
+
+        completedTaskCnt.add(1);
+
+        if (t != null && err.compareAndSet(null, t) || isDone()) {
+            synchronized (this) {
+                notifyAll();
+            }
+        }
+    }
+
+    /**
+     * Mark this executor as initialized.
+     * This method should be called when all required tasks are enqueued for execution.
+     */
+    public final void markInitialized() {
+        initialized = true;
+    }
+
+    /**
+     * Check error status.
+     *
+     * @return {@code True} if any task execution resulted in error.
+     */
+    @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+    public final boolean isError() {
+        return err.get() != null;
+    }
+
+    /**
+     * Check done status.
+     *
+     * @return {@code True} when all enqueued task are completed.
+     */
+    public final boolean isDone() {
+        return initialized && completedTaskCnt.sum() == pendingTaskCnt.sum();
+    }
+
+    /**
+     * Wait synchronously until all tasks are completed or error has occurred.
+     *
+     * @throws IgniteCheckedException if task execution resulted in error.
+     */
+    public final synchronized void awaitDone() throws IgniteCheckedException {
+        // There are no guarantee what all enqueued tasks will be finished if an error has occurred.
+        while(!isError() && !isDone()) {
+            try {
+                wait();
+            }
+            catch (InterruptedException e) {
+                err.set(e);
+
+                Thread.currentThread().interrupt();
+            }
+        }
+
+        if (isError())
+            throw new IgniteCheckedException("Task execution resulted in error", err.get());
+    }
+
+    /**
+     * Reset tasks tracking context.
+     * The method should be called before adding new tasks to the executor.
+     */
+    public final void reset() {
+        initialized = false;
+        completedTaskCnt.reset();
+        pendingTaskCnt.reset();
+        err.set(null);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistenceSequentialCheckpointTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistenceSequentialCheckpointTest.java
index 814ee57bfeeac..230b828d5e657 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistenceSequentialCheckpointTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistenceSequentialCheckpointTest.java
@@ -29,10 +29,8 @@ public class IgnitePersistenceSequentialCheckpointTest extends IgnitePersistentS
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
-        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
-            .setWalMode(WALMode.LOG_ONLY)
-            .setCheckpointThreads(4)
-            .setCheckpointWriteOrder(CheckpointWriteOrder.SEQUENTIAL));
+        DataStorageConfiguration dsCfg = cfg.getDataStorageConfiguration();
+        dsCfg.setCheckpointThreads(4).setCheckpointWriteOrder(CheckpointWriteOrder.SEQUENTIAL);
 
         return cfg;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
index d1de34797cc70..1f95b94e5e1d6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridTestKernalContext.java
@@ -80,6 +80,7 @@ public GridTestKernalContext(IgniteLogger log, IgniteConfiguration cfg) {
                 null,
                 U.allPluginProviders(),
                 null,
+                null,
                 null
         );
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index 309af8cd1193f..b345053eb3984 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -47,6 +47,9 @@
 import org.apache.ignite.util.GridQueueSelfTest;
 import org.apache.ignite.util.GridSpinReadWriteLockSelfTest;
 import org.apache.ignite.util.GridStringBuilderFactorySelfTest;
+import org.apache.ignite.util.GridTopologyHeapSizeSelfTest;
+import org.apache.ignite.util.GridTransientTest;
+import org.apache.ignite.util.IgniteTaskTrackingThreadPoolExecutorTest;
 import org.apache.ignite.util.mbeans.GridMBeanDisableSelfTest;
 import org.apache.ignite.util.mbeans.GridMBeanExoticNamesSelfTest;
 import org.apache.ignite.util.mbeans.GridMBeanSelfTest;
@@ -118,6 +121,9 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         // control.sh
         suite.addTestSuite(CommandHandlerParsingTest.class);
 
+        // Thread pool.
+        suite.addTestSuite(IgniteTaskTrackingThreadPoolExecutorTest.class);
+
         return suite;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/IgniteTaskTrackingThreadPoolExecutorTest.java b/modules/core/src/test/java/org/apache/ignite/util/IgniteTaskTrackingThreadPoolExecutorTest.java
new file mode 100644
index 0000000000000..3db02b0409a98
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/util/IgniteTaskTrackingThreadPoolExecutorTest.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.util;
+
+import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.LongAdder;
+import junit.framework.TestCase;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.thread.IgniteTaskTrackingThreadPoolExecutor;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Tests for tracking thread pool executor.
+ */
+public class IgniteTaskTrackingThreadPoolExecutorTest extends TestCase {
+    /** */
+    private IgniteTaskTrackingThreadPoolExecutor executor;
+
+    /** {@inheritDoc} */
+    @Override protected void setUp() throws Exception {
+        int procs = Runtime.getRuntime().availableProcessors();
+
+        executor = new IgniteTaskTrackingThreadPoolExecutor("test", "default",
+            procs * 2, procs * 2, 30_000, new LinkedBlockingQueue<>(), GridIoPolicy.UNDEFINED, (t, e) -> {
+                // No-op.
+            });
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void tearDown() throws Exception {
+        List<Runnable> runnables = executor.shutdownNow();
+
+        assertEquals("Some tasks are not completed", 0, runnables.size());
+    }
+
+    /** */
+    public void testSimple() throws IgniteCheckedException {
+        doTest(null);
+    }
+
+    /** */
+    @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
+    public void testWithException() throws IgniteCheckedException {
+        int fail = 5555;
+
+        try {
+            doTest(fail);
+
+            fail();
+        }
+        catch (Throwable t) {
+            TestException cause = (TestException)X.getCause(t);
+
+            assertEquals(fail, cause.idx);
+        }
+
+        AtomicReference<Throwable> err = U.field(executor, "err");
+        err.set(null);
+
+        executor.awaitDone();
+    }
+
+    /** */
+    public void testReuse() throws IgniteCheckedException {
+        long avg = 0;
+
+        long warmUp = 30;
+
+        int iters = 150;
+
+        for (int i = 0; i < iters; i++) {
+            long t1 = System.nanoTime();
+
+            doTest(null);
+
+            if (i >= warmUp)
+                avg += System.nanoTime() - t1;
+
+            executor.reset();
+        }
+
+        X.print("Average time per iteration: " + (avg / (iters - warmUp)) / 1000 / 1000. + " ms");
+    }
+
+    /** */
+    private void doTest(@Nullable Integer fail) throws IgniteCheckedException {
+        LongAdder cnt = new LongAdder();
+
+        int exp = 100_000;
+
+        for (int i = 0; i < exp; i++) {
+            final int finalI = i;
+            executor.execute(() -> {
+                if (fail != null && fail == finalI)
+                    throw new TestException(finalI);
+                else
+                    cnt.add(1);
+            });
+        }
+
+        executor.markInitialized();
+
+        executor.awaitDone();
+
+        assertEquals("Counter is not as expected", exp, cnt.sum());
+    }
+
+    /** */
+    private static class TestException extends RuntimeException {
+        /** */
+        final int idx;
+
+        /**
+         * @param idx Index.
+         */
+        public TestException(int idx) {
+            this.idx = idx;
+        }
+    }
+}

From a5e5f5fe77b852a1f4b480fea53480be4f6a95e1 Mon Sep 17 00:00:00 2001
From: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Date: Thu, 27 Sep 2018 18:49:07 +0300
Subject: [PATCH 390/543] IGNITE-9599 Added ability to manage compression level
 for compressed WAL archive - Fixes #4763.

---
 .../DataStorageConfiguration.java             | 41 ++++++++++++---
 .../apache/ignite/internal/IgniteKernal.java  |  6 +++
 .../persistence/DataStorageMXBeanImpl.java    | 52 +++++++++++++++++++
 .../wal/FileWriteAheadLogManager.java         | 10 ++--
 .../FsyncModeFileWriteAheadLogManager.java    |  9 ++--
 .../ignite/mxbean/DataStorageMXBean.java      | 39 ++++++++++++++
 .../DataStorageConfigurationParityTest.cs     |  3 +-
 7 files changed, 140 insertions(+), 20 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMXBeanImpl.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMXBean.java

diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
index a433760ce4cc9..6781dd46520c7 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DataStorageConfiguration.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.configuration;
 
 import java.io.Serializable;
+import java.util.zip.Deflater;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
@@ -152,6 +153,9 @@ public class DataStorageConfiguration implements Serializable {
     /** Default wal compaction enabled. */
     public static final boolean DFLT_WAL_COMPACTION_ENABLED = false;
 
+    /** Default wal compaction level. */
+    public static final int DFLT_WAL_COMPACTION_LEVEL = Deflater.BEST_SPEED;
+
     /** Size of a memory chunk reserved for system cache initially. */
     private long sysRegionInitSize = DFLT_SYS_CACHE_INIT_SIZE;
 
@@ -243,7 +247,7 @@ public class DataStorageConfiguration implements Serializable {
     private long metricsRateTimeInterval = DFLT_RATE_TIME_INTERVAL_MILLIS;
 
     /**
-     *  Time interval (in milliseconds) for running auto archiving for incompletely WAL segment
+     * Time interval (in milliseconds) for running auto archiving for incompletely WAL segment
      */
     private long walAutoArchiveAfterInactivity = -1;
 
@@ -258,6 +262,15 @@ public class DataStorageConfiguration implements Serializable {
      */
     private boolean walCompactionEnabled = DFLT_WAL_COMPACTION_ENABLED;
 
+    /**
+     * ZIP level to WAL compaction.
+     *
+     * @see java.util.zip.ZipOutputStream#setLevel(int)
+     * @see java.util.zip.Deflater#BEST_SPEED
+     * @see java.util.zip.Deflater#BEST_COMPRESSION
+     */
+    private int walCompactionLevel = DFLT_WAL_COMPACTION_LEVEL;
+
     /**
      * Initial size of a data region reserved for system cache.
      *
@@ -273,7 +286,6 @@ public long getSystemRegionInitialSize() {
      * Default value is {@link #DFLT_SYS_CACHE_INIT_SIZE}
      *
      * @param sysRegionInitSize Size in bytes.
-     *
      * @return {@code this} for chaining.
      */
     public DataStorageConfiguration setSystemRegionInitialSize(long sysRegionInitSize) {
@@ -388,6 +400,7 @@ public DataRegionConfiguration getDefaultDataRegionConfiguration() {
 
     /**
      * Overrides configuration of default data region which is created automatically.
+     *
      * @param dfltDataRegConf Default data region configuration.
      */
     public DataStorageConfiguration setDefaultDataRegionConfiguration(DataRegionConfiguration dfltDataRegConf) {
@@ -570,7 +583,7 @@ public DataStorageConfiguration setWalPath(String walStorePath) {
     /**
      * Gets a path to the WAL archive directory.
      *
-     *  @return WAL archive directory.
+     * @return WAL archive directory.
      */
     public String getWalArchivePath() {
         return walArchivePath;
@@ -738,8 +751,8 @@ public DataStorageConfiguration setWalBufferSize(int walBuffSize) {
     }
 
     /**
-     *  This property define how often WAL will be fsync-ed in {@code BACKGROUND} mode. Ignored for
-     *  all other WAL modes.
+     * This property define how often WAL will be fsync-ed in {@code BACKGROUND} mode. Ignored for
+     * all other WAL modes.
      *
      * @return WAL flush frequency, in milliseconds.
      */
@@ -748,8 +761,8 @@ public long getWalFlushFrequency() {
     }
 
     /**
-     *  This property define how often WAL will be fsync-ed in {@code BACKGROUND} mode. Ignored for
-     *  all other WAL modes.
+     * This property define how often WAL will be fsync-ed in {@code BACKGROUND} mode. Ignored for
+     * all other WAL modes.
      *
      * @param walFlushFreq WAL flush frequency, in milliseconds.
      */
@@ -909,6 +922,20 @@ public DataStorageConfiguration setWalCompactionEnabled(boolean walCompactionEna
         return this;
     }
 
+    /**
+     * @return ZIP level to WAL compaction.
+     */
+    public int getWalCompactionLevel() {
+        return walCompactionLevel;
+    }
+
+    /**
+     * @param walCompactionLevel New ZIP level to WAL compaction.
+     */
+    public void setWalCompactionLevel(int walCompactionLevel) {
+        this.walCompactionLevel = walCompactionLevel;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(DataStorageConfiguration.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 306bb66f0cd8e..30a491815c9ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -130,6 +130,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DataStorageMXBeanImpl;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.closure.GridClosureProcessor;
@@ -199,6 +200,7 @@
 import org.apache.ignite.marshaller.MarshallerExclusions;
 import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.mxbean.ClusterMetricsMXBean;
+import org.apache.ignite.mxbean.DataStorageMXBean;
 import org.apache.ignite.mxbean.IgniteMXBean;
 import org.apache.ignite.mxbean.StripedExecutorMXBean;
 import org.apache.ignite.mxbean.ThreadPoolMXBean;
@@ -4257,6 +4259,10 @@ private void registerAllMBeans(
             TransactionsMXBean txMXBean = new TransactionsMXBeanImpl(ctx);
             registerMBean("Transactions", txMXBean.getClass().getSimpleName(), txMXBean, TransactionsMXBean.class);
 
+            // Data storage
+            DataStorageMXBean dataStorageMXBean = new DataStorageMXBeanImpl(ctx);
+            registerMBean("DataStorage", dataStorageMXBean.getClass().getSimpleName(), dataStorageMXBean, DataStorageMXBean.class);
+
             // Executors
             registerExecutorMBean("GridUtilityCacheExecutor", utilityCachePool);
             registerExecutorMBean("GridExecutionExecutor", execSvc);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMXBeanImpl.java
new file mode 100644
index 0000000000000..c3eb9cd5490f1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStorageMXBeanImpl.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.internal.GridKernalContextImpl;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.mxbean.DataStorageMXBean;
+
+/**
+ * TransactionsMXBean implementation.
+ */
+public class DataStorageMXBeanImpl implements DataStorageMXBean {
+    /** */
+    private final GridKernalContextImpl ctx;
+
+    /**
+     * @param ctx Context.
+     */
+    public DataStorageMXBeanImpl(GridKernalContextImpl ctx) {
+        this.ctx = ctx;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int getWalCompactionLevel() {
+        return ctx.config().getDataStorageConfiguration().getWalCompactionLevel();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void setWalCompactionLevel(int walCompactionLevel) {
+        ctx.config().getDataStorageConfiguration().setWalCompactionLevel(walCompactionLevel);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DataStorageMXBeanImpl.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 23aa06604bfc8..55fb3d8352056 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1806,16 +1806,13 @@ private class FileCompressor extends Thread {
 
         /** All segments prior to this (inclusive) can be compressed. */
         private volatile long minUncompressedIdxToKeep = -1L;
-        /**
-         *
-         */
+
+        /** */
         FileCompressor() {
             super("wal-file-compressor%" + cctx.igniteInstanceName());
         }
 
-        /**
-         *
-         */
+        /** */
         private void init() {
             File[] toDel = walArchiveDir.listFiles(WAL_SEGMENT_TEMP_FILE_COMPACTED_FILTER);
 
@@ -1946,6 +1943,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
             }
 
             try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(zip)))) {
+                zos.setLevel(dsCfg.getWalCompactionLevel());
                 zos.putNextEntry(new ZipEntry(""));
 
                 ByteBuffer buf = ByteBuffer.allocate(HEADER_RECORD_SIZE);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 9c4a9b30d14e0..4fa96910fa37d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -1751,16 +1751,12 @@ private class FileCompressor extends Thread {
         /** All segments prior to this (inclusive) can be compressed. */
         private volatile long minUncompressedIdxToKeep = -1L;
 
-        /**
-         *
-         */
+        /** */
         FileCompressor() {
             super("wal-file-compressor%" + cctx.igniteInstanceName());
         }
 
-        /**
-         *
-         */
+        /** */
         private void init() {
             File[] toDel = walArchiveDir.listFiles(WAL_SEGMENT_TEMP_FILE_COMPACTED_FILTER);
 
@@ -1923,6 +1919,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
             }
 
             try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(zip)))) {
+                zos.setLevel(dsCfg.getWalCompactionLevel());
                 zos.putNextEntry(new ZipEntry(""));
 
                 zos.write(prepareSerializerVersionBuffer(nextSegment, segmentSerializerVer, true).array());
diff --git a/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMXBean.java b/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMXBean.java
new file mode 100644
index 0000000000000..850b886f933a6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/mxbean/DataStorageMXBean.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.mxbean;
+
+import org.apache.ignite.configuration.DataStorageConfiguration;
+
+/**
+ * An MX bean allowing to monitor and tune persistence.
+ */
+public interface DataStorageMXBean {
+    @MXBeanDescription("ZIP compression level to WAL compaction.")
+    int getWalCompactionLevel();
+
+    /**
+     * Sets ZIP compression level to WAL compaction.
+     * {@link DataStorageConfiguration#setWalCompactionLevel(int)} configuration property.
+     *
+     * @param walCompactionLevel ZIP compression level.
+     */
+    @MXBeanDescription("Sets ZIP compression level to WAL compaction.")
+    @MXBeanParametersNames("walCompactionLevel")
+    @MXBeanParametersDescriptions("ZIP compression level.")
+    void setWalCompactionLevel(int walCompactionLevel);
+}
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageConfigurationParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageConfigurationParityTest.cs
index 51f2865ad5bda..7f1c793af24e5 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageConfigurationParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/DataStorageConfigurationParityTest.cs
@@ -35,7 +35,8 @@ public class DataStorageConfigurationParityTest
         private static readonly string[] MissingProperties =
         {
             // IGNITE-7305
-            "WalBufferSize"
+            "WalBufferSize",
+            "WalCompactionLevel" // IGNITE-9599
         };
 
         /// <summary>

From 070790d3f43ded112a78033d0c13909825149597 Mon Sep 17 00:00:00 2001
From: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Date: Thu, 27 Sep 2018 18:29:52 +0300
Subject: [PATCH 391/543] IGNITE-9549 Added command to collect partition
 distribution information and reset lost partitions - Fixes #4734.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 9e0c1e7eacd4ffdb48f3086a22591e2e4a70757e)
---
 .../internal/commandline/CommandHandler.java  |  82 +++++
 .../commandline/cache/CacheArguments.java     |  19 +-
 .../commandline/cache/CacheCommand.java       |  12 +-
 .../distribution/CacheDistributionGroup.java  | 102 ++++++
 .../distribution/CacheDistributionNode.java   | 127 +++++++
 .../CacheDistributionPartition.java           | 136 +++++++
 .../distribution/CacheDistributionTask.java   | 171 +++++++++
 .../CacheDistributionTaskArg.java             |  94 +++++
 .../CacheDistributionTaskResult.java          | 346 ++++++++++++++++++
 .../CacheResetLostPartitionsTask.java         | 104 ++++++
 .../CacheResetLostPartitionsTaskArg.java      |  72 ++++
 .../CacheResetLostPartitionsTaskResult.java   |  80 ++++
 .../resources/META-INF/classnames.properties  |  14 +
 .../ignite/util/GridCommandHandlerTest.java   |  82 ++++-
 14 files changed, 1437 insertions(+), 4 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionGroup.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionNode.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionPartition.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTaskResult.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 606599502b3f9..d011db9fb48e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -52,6 +52,12 @@
 import org.apache.ignite.internal.client.ssl.GridSslBasicContextFactory;
 import org.apache.ignite.internal.commandline.cache.CacheArguments;
 import org.apache.ignite.internal.commandline.cache.CacheCommand;
+import org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionTask;
+import org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionTaskArg;
+import org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionTaskResult;
+import org.apache.ignite.internal.commandline.cache.reset_lost_partitions.CacheResetLostPartitionsTask;
+import org.apache.ignite.internal.commandline.cache.reset_lost_partitions.CacheResetLostPartitionsTaskArg;
+import org.apache.ignite.internal.commandline.cache.reset_lost_partitions.CacheResetLostPartitionsTaskResult;
 import org.apache.ignite.internal.processors.cache.verify.CacheInfo;
 import org.apache.ignite.internal.processors.cache.verify.ContentionInfo;
 import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
@@ -191,6 +197,10 @@ public class CommandHandler {
     /** */
     protected static final String CMD_SSL_TRUSTSTORE_PASSWORD = "--ssl_truststore_password";
 
+    /** */
+    private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
+
+
     /** List of optional auxiliary commands. */
     private static final Set<String> AUX_COMMANDS = new HashSet<>();
 
@@ -627,6 +637,16 @@ private void cache(GridClient client, CacheArguments cacheArgs) throws Throwable
 
                 break;
 
+            case DISTRIBUTION:
+                cacheDistribution(client, cacheArgs);
+
+                break;
+
+            case RESET_LOST_PARTITIONS:
+                cacheResetLostPartitions(client, cacheArgs);
+
+                break;
+
             default:
                 cacheView(client, cacheArgs);
 
@@ -644,6 +664,8 @@ private void printCacheHelp() {
         usage("  Show hot keys that are point of contention for multiple transactions:", CACHE, " contention minQueueSize [nodeId] [maxPrint]");
         usage("  Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [--dump] [--skipZeros] [cache1,...,cacheN]");
         usage("  Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId] [checkFirst|checkThrough]");
+        usage("  Collect partition distribution information:", CACHE, " distribution nodeId|null [cacheName1,...,cacheNameN] [--user-attributes attributeName1[,...,attributeNameN]]");
+        usage("  Reset lost partitions:", CACHE, " reset_lost_partitions cacheName1[,...,cacheNameN]");
 
         log("  If [nodeId] is not specified, contention and validate_indexes commands will be broadcasted to all server nodes.");
         log("  Another commands where [nodeId] is optional will run on a random server node.");
@@ -821,6 +843,33 @@ private void legacyCacheIdleVerify(GridClient client, CacheArguments cacheArgs)
         }
     }
 
+    /**
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     */
+    private void cacheDistribution(GridClient client, CacheArguments cacheArgs) throws GridClientException {
+        CacheDistributionTaskArg taskArg = new CacheDistributionTaskArg(cacheArgs.caches(), cacheArgs.getUserAttributes());
+
+        UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId();
+
+        CacheDistributionTaskResult res = executeTaskByNameOnNode(client, CacheDistributionTask.class.getName(), taskArg, nodeId);
+
+        res.print(System.out);
+    }
+
+    /**
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     */
+    private void cacheResetLostPartitions(GridClient client, CacheArguments cacheArgs) throws GridClientException {
+
+        CacheResetLostPartitionsTaskArg taskArg = new CacheResetLostPartitionsTaskArg(cacheArgs.caches());
+
+        CacheResetLostPartitionsTaskResult res = executeTaskByNameOnNode(client, CacheResetLostPartitionsTask.class.getName(), taskArg, null);
+
+        res.print(System.out);
+    }
+
     /**
      * @param client Client.
      * @param cacheArgs Cache args.
@@ -1676,6 +1725,39 @@ else if (CMD_SKIP_ZEROS.equals(nextArg))
 
                 break;
 
+            case DISTRIBUTION:
+                String nodeIdStr = nextArg("Node id expected or null");
+                if (!"null".equals(nodeIdStr))
+                    cacheArgs.nodeId(UUID.fromString(nodeIdStr));
+
+                while (hasNextCacheArg()) {
+                    String nextArg = nextArg("");
+
+                    if (CMD_USER_ATTRIBUTES.equals(nextArg)){
+                        nextArg = nextArg("User attributes are expected to be separated by commas");
+
+                        Set<String> userAttributes = new HashSet();
+
+                        for (String userAttribute:nextArg.split(","))
+                            userAttributes.add(userAttribute.trim());
+
+                        cacheArgs.setUserAttributes(userAttributes);
+
+                        nextArg = (hasNextCacheArg()) ? nextArg("") : null;
+
+                    }
+
+                    if (nextArg!=null)
+                        parseCacheNames(nextArg, cacheArgs);
+                }
+
+                break;
+
+            case RESET_LOST_PARTITIONS:
+                parseCacheNames(nextArg("Cache name expected"), cacheArgs);
+
+                break;
+
             default:
                 cacheArgs.regex(nextArg("Regex is expected"));
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
index 353a63ce5269c..97d234aeb879d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
@@ -61,6 +61,9 @@ public class CacheArguments {
     /** Skip zeros partitions. */
     private boolean skipZeros;
 
+    /** Additional user attributes in result. Set of attribute names whose values will be searched in ClusterNode.attributes(). */
+    private Set<String> userAttributes;
+
     /**
      * @return Command.
      */
@@ -174,7 +177,7 @@ public void maxPrint(int maxPrint) {
     }
 
     /**
-     *  @return Max number of entries to be checked.
+     * @return Max number of entries to be checked.
      */
     public int checkFirst() {
         return checkFirst;
@@ -228,4 +231,18 @@ public boolean isSkipZeros() {
     public void skipZeros(boolean skipZeros) {
         this.skipZeros = skipZeros;
     }
+
+    /**
+     * @return Additional user attributes in result. Set of attribute names whose values will be searched in ClusterNode.attributes().
+     */
+    public Set<String> getUserAttributes() {
+        return userAttributes;
+    }
+
+    /**
+     * @param userAttrs New additional user attributes in result.
+     */
+    public void setUserAttributes(Set<String> userAttrs) {
+        userAttributes = userAttrs;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java
index 6aec6d7e9cfa1..af222a8bc0973 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheCommand.java
@@ -46,7 +46,17 @@ public enum CacheCommand {
     /**
      * Prints info about contended keys (the keys concurrently locked from multiple transactions).
      */
-    CONTENTION("contention");
+    CONTENTION("contention"),
+
+    /**
+     * Collect information on the distribution of partitions.
+     */
+    DISTRIBUTION("distribution"),
+
+    /**
+     * Reset lost partitions
+     */
+    RESET_LOST_PARTITIONS("reset_lost_partitions");
 
     /** Enumerated values. */
     private static final CacheCommand[] VALS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionGroup.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionGroup.java
new file mode 100644
index 0000000000000..3e594ff12bf6e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionGroup.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.commandline.cache.distribution;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * DTO for CacheDistributionTask, contains information about group
+ */
+public class CacheDistributionGroup extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Group identifier. */
+    private int grpId;
+
+    /** Group name. */
+    private String grpName;
+
+    /** List of partitions. */
+    private List<CacheDistributionPartition> partitions;
+
+    /** Default constructor. */
+    public CacheDistributionGroup() {
+    }
+
+    /**
+     * @param grpId Group identifier.
+     * @param grpName Group name.
+     * @param partitions List of partitions.
+     */
+    public CacheDistributionGroup(int grpId, String grpName, List<CacheDistributionPartition> partitions) {
+        this.grpId = grpId;
+        this.grpName = grpName;
+        this.partitions = partitions;
+    }
+
+    /** */
+    public int getGroupId() {
+        return grpId;
+    }
+
+    /** */
+    public void setGroupId(int grpId) {
+        this.grpId = grpId;
+    }
+
+    /** */
+    public String getGroupName() {
+        return grpName;
+    }
+
+    /** */
+    public void setGroupName(String grpName) {
+        this.grpName = grpName;
+    }
+
+    /** */
+    public List<CacheDistributionPartition> getPartitions() {
+        return partitions;
+    }
+
+    /** */
+    public void setPartitions(
+        List<CacheDistributionPartition> partitions) {
+        this.partitions = partitions;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        out.writeInt(grpId);
+        U.writeString(out, grpName);
+        U.writeCollection(out, partitions);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer,
+        ObjectInput in) throws IOException, ClassNotFoundException {
+        grpId = in.readInt();
+        grpName = U.readString(in);
+        partitions = U.readList(in);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionNode.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionNode.java
new file mode 100644
index 0000000000000..f53bf34fe877f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionNode.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.commandline.cache.distribution;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * DTO for CacheDistributionTask, contains information about node
+ */
+public class CacheDistributionNode extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Node identifier. */
+    private UUID nodeId;
+
+    /** Network addresses. */
+    private String addrs;
+
+    /** User attribute in result. */
+    private Map<String, String> userAttrs;
+
+    /** Information about groups. */
+    private List<CacheDistributionGroup> groups;
+
+    /** Default constructor. */
+    public CacheDistributionNode() {
+    }
+
+    /**
+     * @param nodeId Node identifier.
+     * @param addrs Network addresses.
+     * @param userAttrs Map node user attribute.
+     * @param groups Information about groups.
+     */
+    public CacheDistributionNode(UUID nodeId, String addrs,
+        Map<String, String> userAttrs,
+        List<CacheDistributionGroup> groups) {
+        this.nodeId = nodeId;
+        this.addrs = addrs;
+        this.userAttrs = userAttrs;
+        this.groups = groups;
+    }
+
+    /** */
+    public UUID getNodeId() {
+        return nodeId;
+    }
+
+    /** */
+    public void setNodeId(UUID nodeId) {
+        this.nodeId = nodeId;
+    }
+
+    /** */
+    public String getAddresses() {
+        return addrs;
+    }
+
+    /** */
+    public void setAddresses(String addrs) {
+        this.addrs = addrs;
+    }
+
+    /**
+     * @return User attribute in result.
+     */
+    public Map<String, String> getUserAttributes() {
+        return userAttrs;
+    }
+
+    /**
+     * @param userAttrs New user attribute in result.
+     */
+    public void setUserAttributes(Map<String, String> userAttrs) {
+        this.userAttrs = userAttrs;
+    }
+
+    /** */
+    public List<CacheDistributionGroup> getGroups() {
+        return groups;
+    }
+
+    /** */
+    public void setGroups(List<CacheDistributionGroup> groups) {
+        this.groups = groups;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeUuid(out, nodeId);
+        U.writeString(out, addrs);
+        U.writeMap(out, userAttrs);
+        U.writeCollection(out, groups);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer,
+        ObjectInput in) throws IOException, ClassNotFoundException {
+        nodeId = U.readUuid(in);
+        addrs = U.readString(in);
+        userAttrs = U.readMap(in);
+        groups = U.readList(in);
+    }
+
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionPartition.java
new file mode 100644
index 0000000000000..e0eea60c54e70
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionPartition.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.commandline.cache.distribution;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * DTO for CacheDistributionTask, contains information about partition
+ */
+public class CacheDistributionPartition extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Partition identifier. */
+    private int partId;
+
+    /** Flag primary or backup partition. */
+    private boolean primary;
+
+    /** Partition status. */
+    private GridDhtPartitionState state;
+
+    /** Partition update counters. */
+    private long updateCntr;
+
+    /** Number of entries in partition. */
+    private long size;
+
+    /** Default constructor. */
+    public CacheDistributionPartition() {
+    }
+
+    /**
+     * @param partId Partition identifier.
+     * @param primary Flag primary or backup partition.
+     * @param state Partition status.
+     * @param updateCntr Partition update counters.
+     * @param size Number of entries in partition.
+     */
+    public CacheDistributionPartition(int partId, boolean primary,
+        GridDhtPartitionState state, long updateCntr, long size) {
+        this.partId = partId;
+        this.primary = primary;
+        this.state = state;
+        this.updateCntr = updateCntr;
+        this.size = size;
+    }
+
+    /** */
+    public int getPartition() {
+        return partId;
+    }
+
+    /** */
+    public void setPartition(int partId) {
+        this.partId = partId;
+    }
+
+    /** */
+    public boolean isPrimary() {
+        return primary;
+    }
+
+    /** */
+    public void setPrimary(boolean primary) {
+        this.primary = primary;
+    }
+
+    /** */
+    public GridDhtPartitionState getState() {
+        return state;
+    }
+
+    /** */
+    public void setState(GridDhtPartitionState state) {
+        this.state = state;
+    }
+
+    /** */
+    public long getUpdateCounter() {
+        return updateCntr;
+    }
+
+    /** */
+    public void setUpdateCounter(long updateCntr) {
+        this.updateCntr = updateCntr;
+    }
+
+    /** */
+    public long getSize() {
+        return size;
+    }
+
+    /** */
+    public void setSize(long size) {
+        this.size = size;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        out.writeInt(partId);
+        out.writeBoolean(primary);
+        U.writeEnum(out, state);
+        out.writeLong(updateCntr);
+        out.writeLong(size);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException {
+        partId = in.readInt();
+        primary = in.readBoolean();
+        state = GridDhtPartitionState.fromOrdinal(in.readByte());
+        updateCntr = in.readLong();
+        size = in.readLong();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTask.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTask.java
new file mode 100644
index 0000000000000..88249dc487aae
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTask.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.commandline.cache.distribution;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Collect information on the distribution of partitions.
+ */
+public class CacheDistributionTask extends VisorMultiNodeTask<CacheDistributionTaskArg,
+    CacheDistributionTaskResult, CacheDistributionNode> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected CacheDistributionTaskResult reduce0(
+        List<ComputeJobResult> list) throws IgniteException {
+        Map<UUID, Exception> exceptions = new HashMap<>();
+        List<CacheDistributionNode> infos = new ArrayList<>();
+
+        for (ComputeJobResult res : list) {
+            if (res.getException() != null)
+                exceptions.put(res.getNode().id(), res.getException());
+            else
+                infos.add(res.getData());
+        }
+
+        return new CacheDistributionTaskResult(infos, exceptions);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<CacheDistributionTaskArg, CacheDistributionNode> job(CacheDistributionTaskArg arg) {
+        return new CacheDistributionJob(arg, debug);
+    }
+
+    /** Job for node. */
+    private static class CacheDistributionJob extends VisorJob<CacheDistributionTaskArg, CacheDistributionNode> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param arg Argument.
+         * @param debug Debug.
+         */
+        public CacheDistributionJob(@Nullable CacheDistributionTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override public CacheDistributionNode run(CacheDistributionTaskArg arg) throws IgniteException {
+            try {
+                final CacheDistributionNode info = new CacheDistributionNode();
+
+                final ClusterNode node = ignite.localNode();
+
+                info.setNodeId(node.id());
+                info.setAddresses(node.addresses().toString());
+
+                if (arg.getUserAttributes() != null) {
+                    info.setUserAttributes(new TreeMap<>());
+
+                    for (String userAttribute : arg.getUserAttributes())
+                        info.getUserAttributes().put(userAttribute, (String)node.attributes().get(userAttribute));
+                }
+
+                info.setGroups(new ArrayList<>());
+
+                Set<Integer> grpIds = new HashSet<>();
+
+                if (arg.getCaches() == null) {
+                    final Collection<CacheGroupContext> ctxs = ignite.context().cache().cacheGroups();
+
+                    for (CacheGroupContext ctx : ctxs)
+                        grpIds.add(ctx.groupId());
+                }
+                else {
+                    for (String cacheName : arg.getCaches())
+                        grpIds.add(CU.cacheId(cacheName));
+                }
+
+                if (grpIds.isEmpty())
+                    return info;
+
+                for (Integer id : grpIds) {
+                    final CacheDistributionGroup grp = new CacheDistributionGroup();
+
+                    info.getGroups().add(grp);
+
+                    grp.setGroupId(id);
+
+                    final DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(id);
+
+                    final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(desc == null ? id : desc.groupId());
+
+                    grp.setGroupName(grpCtx.cacheOrGroupName());
+
+                    grp.setPartitions(new ArrayList<>());
+
+                    GridDhtPartitionTopologyImpl top = (GridDhtPartitionTopologyImpl)grpCtx.topology();
+
+                    final AffinityAssignment assignment = grpCtx.affinity().readyAffinity(top.readyTopologyVersion());
+
+                    List<GridDhtLocalPartition> locParts = top.localPartitions();
+
+                    for (int i = 0; i < locParts.size(); i++) {
+                        GridDhtLocalPartition part = locParts.get(i);
+
+                        if (part == null)
+                            continue;
+
+                        final CacheDistributionPartition partDto = new CacheDistributionPartition();
+
+                        grp.getPartitions().add(partDto);
+
+                        int p = part.id();
+                        partDto.setPartition(p);
+                        partDto.setPrimary(assignment.primaryPartitions(node.id()).contains(p));
+                        partDto.setState(part.state());
+                        partDto.setUpdateCounter(part.updateCounter());
+                        partDto.setSize(desc == null ? part.dataStore().fullSize() : part.dataStore().cacheSize(id));
+                    }
+                }
+                return info;
+            }
+            catch (Exception e) {
+                throw new IgniteException(e);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(CacheDistributionJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskArg.java
new file mode 100644
index 0000000000000..8e9d7235bae0f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskArg.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.commandline.cache.distribution;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Set;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Input params for CacheDistributionTask
+ */
+public class CacheDistributionTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Caches. */
+    private Set<String> caches;
+
+    /** Add user attribute in result. */
+    private Set<String> userAttrs;
+
+    /**
+     * Default constructor.
+     */
+    public CacheDistributionTaskArg() {
+        // No-op.
+    }
+
+    /**
+     * @param caches Caches.
+     * @param userAttrs Add user attribute in result.
+     */
+    public CacheDistributionTaskArg(Set<String> caches, Set<String> userAttrs) {
+        this.caches = caches;
+        this.userAttrs = userAttrs;
+    }
+
+    /**
+     * @return Caches.
+     */
+    public Set<String> getCaches() {
+        return caches;
+    }
+
+    /**
+     * @return Add user attribute in result
+     */
+    public Set<String> getUserAttributes() {
+        return userAttrs;
+    }
+
+    /**
+     * @param userAttrs New add user attribute in result
+     */
+    public void setUserAttributes(Set<String> userAttrs) {
+        this.userAttrs = userAttrs;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeCollection(out, caches);
+        U.writeCollection(out, userAttrs);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer,
+        ObjectInput in) throws IOException, ClassNotFoundException {
+        caches = U.readSet(in);
+        userAttrs = U.readSet(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(CacheDistributionTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java
new file mode 100644
index 0000000000000..71de3bbc0dd05
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.commandline.cache.distribution;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Result of CacheDistributionTask
+ */
+public class CacheDistributionTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Job results. */
+    private List<CacheDistributionNode> nodeResList;
+
+    /** Exceptions. */
+    private Map<UUID, Exception> exceptions;
+
+    /**
+     * @param nodeResList Cluster infos.
+     * @param exceptions Exceptions.
+     */
+    public CacheDistributionTaskResult(List<CacheDistributionNode> nodeResList,
+        Map<UUID, Exception> exceptions) {
+        this.nodeResList = nodeResList;
+        this.exceptions = exceptions;
+    }
+
+    /**
+     * For externalization only.
+     */
+    public CacheDistributionTaskResult() {
+    }
+
+    /**
+     * @return Job results.
+     */
+    public Collection<CacheDistributionNode> jobResults() {
+        return nodeResList;
+    }
+
+    /**
+     * @return Exceptions.
+     */
+    public Map<UUID, Exception> exceptions() {
+        return exceptions;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeCollection(out, nodeResList);
+        U.writeMap(out, exceptions);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in
+    ) throws IOException, ClassNotFoundException {
+        nodeResList = U.readList(in);
+        exceptions = U.readMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(CacheDistributionTaskResult.class, this);
+    }
+
+    /**
+     * Print collect information on the distribution of partitions.
+     *
+     * @param out Print stream.
+     */
+    public void print(PrintStream out) {
+        if (nodeResList.isEmpty())
+            return;
+
+        List<Row> rows = new ArrayList<>();
+
+        for (CacheDistributionNode node : nodeResList) {
+            for (CacheDistributionGroup group : node.getGroups()) {
+                for (CacheDistributionPartition partition : group.getPartitions()) {
+                    final Row row = new Row();
+                    row.setGroupId(group.getGroupId());
+                    row.setGroupName(group.getGroupName());
+                    row.setPartition(partition.getPartition());
+                    row.setNodeId(node.getNodeId());
+                    row.setPrimary(partition.isPrimary());
+                    row.setState(partition.getState());
+                    row.setUpdateCounter(partition.getUpdateCounter());
+                    row.setSize(partition.getSize());
+                    row.setAddresses(node.getAddresses());
+                    row.setUserAttributes(node.getUserAttributes());
+
+                    rows.add(row);
+                }
+            }
+        }
+
+        rows.sort(null);
+
+        StringBuilder userAttrsName = new StringBuilder();
+        if (!rows.isEmpty() && rows.get(0).userAttrs != null) {
+            for (String userAttribute : rows.get(0).userAttrs.keySet()) {
+                userAttrsName.append(',');
+
+                if (userAttribute != null)
+                    userAttrsName.append(userAttribute);
+            }
+        }
+        out.println("[groupId,partition,nodeId,primary,state,updateCounter,partitionSize,nodeAddresses" + userAttrsName + "]");
+
+        int oldGrpId = 0;
+
+        for (Row row : rows) {
+            if (oldGrpId != row.grpId) {
+                out.println("[next group: id=" + row.grpId + ", name=" + row.grpName + ']');
+
+                oldGrpId = row.getGroupId();
+            }
+
+            row.print(out);
+        }
+    }
+
+    /**
+     * Class for
+     */
+    private static class Row implements Comparable {
+        /** */
+        private int grpId;
+
+        /** */
+        private String grpName;
+
+        /** */
+        private int partId;
+
+        /** */
+        private UUID nodeId;
+
+        /** */
+        private boolean primary;
+
+        /** */
+        private GridDhtPartitionState state;
+
+        /** */
+        private long updateCntr;
+
+        /** */
+        private long size;
+
+        /** */
+        private String addrs;
+
+        /** User attribute in result. */
+        private Map<String, String> userAttrs;
+
+        /** */
+        public int getGroupId() {
+            return grpId;
+        }
+
+        /** */
+        public void setGroupId(int grpId) {
+            this.grpId = grpId;
+        }
+
+        /** */
+        public String getGroupName() {
+            return grpName;
+        }
+
+        /** */
+        public void setGroupName(String grpName) {
+            this.grpName = grpName;
+        }
+
+        /** */
+        public int getPartition() {
+            return partId;
+        }
+
+        /** */
+        public void setPartition(int partId) {
+            this.partId = partId;
+        }
+
+        /** */
+        public UUID getNodeId() {
+            return nodeId;
+        }
+
+        /** */
+        public void setNodeId(UUID nodeId) {
+            this.nodeId = nodeId;
+        }
+
+        /** */
+        public boolean isPrimary() {
+            return primary;
+        }
+
+        /** */
+        public void setPrimary(boolean primary) {
+            this.primary = primary;
+        }
+
+        /** */
+        public GridDhtPartitionState getState() {
+            return state;
+        }
+
+        /** */
+        public void setState(GridDhtPartitionState state) {
+            this.state = state;
+        }
+
+        /** */
+        public long getUpdateCounter() {
+            return updateCntr;
+        }
+
+        /** */
+        public void setUpdateCounter(long updateCntr) {
+            this.updateCntr = updateCntr;
+        }
+
+        /** */
+        public long getSize() {
+            return size;
+        }
+
+        /** */
+        public void setSize(long size) {
+            this.size = size;
+        }
+
+        /** */
+        public String getAddresses() {
+            return addrs;
+        }
+
+        /** */
+        public void setAddresses(String addrs) {
+            this.addrs = addrs;
+        }
+
+        /**
+         * @return User attribute in result.
+         */
+        public Map<String, String> getUserAttributes() {
+            return userAttrs;
+        }
+
+        /**
+         * @param userAttrs New user attribute in result.
+         */
+        public void setUserAttributes(Map<String, String> userAttrs) {
+            this.userAttrs = userAttrs;
+        }
+
+        /** {@inheritDoc} */
+        @Override public int compareTo(@NotNull Object o) {
+            assert o instanceof Row;
+
+            Row other = (Row)o;
+
+            int res = grpId - other.grpId;
+
+            if (res == 0) {
+                res = partId - other.partId;
+
+                if (res == 0)
+                    res = nodeId.compareTo(other.nodeId);
+
+            }
+
+            return res;
+        }
+
+        /** */
+        public void print(PrintStream out) {
+            out.print(grpId);
+            out.print(',');
+
+            out.print(partId);
+            out.print(',');
+
+            out.print(U.id8(getNodeId()));
+            out.print(',');
+
+            out.print(primary ? "P" : "B");
+            out.print(',');
+
+            out.print(state);
+            out.print(',');
+
+            out.print(updateCntr);
+            out.print(',');
+
+            out.print(size);
+            out.print(',');
+
+            out.print(addrs);
+
+            if (userAttrs != null) {
+                for (String userAttribute : userAttrs.values()) {
+                    out.print(',');
+                    if (userAttribute != null)
+                        out.print(userAttribute);
+                }
+            }
+
+            out.println();
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTask.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTask.java
new file mode 100644
index 0000000000000..2230a24dc4a0b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTask.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.commandline.cache.reset_lost_partitions;
+
+import java.util.HashMap;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Reset status of lost partitions.
+ */
+public class CacheResetLostPartitionsTask extends VisorOneNodeTask<CacheResetLostPartitionsTaskArg, CacheResetLostPartitionsTaskResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorJob<CacheResetLostPartitionsTaskArg, CacheResetLostPartitionsTaskResult> job(
+        CacheResetLostPartitionsTaskArg arg) {
+        return new CacheResetLostPartitionsJob(arg, debug);
+    }
+
+    /** Job for node. */
+    private static class CacheResetLostPartitionsJob extends VisorJob<CacheResetLostPartitionsTaskArg, CacheResetLostPartitionsTaskResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * @param arg Argument.
+         * @param debug Debug.
+         */
+        public CacheResetLostPartitionsJob(@Nullable CacheResetLostPartitionsTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override public CacheResetLostPartitionsTaskResult run(
+            CacheResetLostPartitionsTaskArg arg) throws IgniteException {
+            try {
+                final CacheResetLostPartitionsTaskResult res = new CacheResetLostPartitionsTaskResult();
+                res.setMessageMap(new HashMap<>());
+
+                if (!F.isEmpty(arg.getCaches())) {
+                    for (String groupName : arg.getCaches()) {
+                        final int grpId = CU.cacheId(groupName);
+
+                        CacheGroupContext grp = ignite.context().cache().cacheGroup(grpId);
+
+                        if (grp != null) {
+                            SortedSet<String> cacheNames = grp.caches().stream()
+                                .map(GridCacheContext::name)
+                                .collect(Collectors.toCollection(TreeSet::new));
+
+                            if (!F.isEmpty(cacheNames)) {
+                                ignite.resetLostPartitions(cacheNames);
+
+                                res.put(groupName, String.format("Reset LOST-partitions performed successfully. " +
+                                        "Cache group (name = '%s', id = %d), caches (%s).",
+                                    groupName, grpId, cacheNames));
+                            }
+                        }
+                        else
+                            res.put(groupName, String.format("Cache group (name = '%s', id = %d) not found.",
+                                groupName, grpId));
+                    }
+                }
+
+                return res;
+            }
+            catch (Exception e) {
+                throw new IgniteException(e);
+            }
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(CacheResetLostPartitionsJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTaskArg.java
new file mode 100644
index 0000000000000..2525ec40e6117
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTaskArg.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.commandline.cache.reset_lost_partitions;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Set;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Input params for CacheResetLostPartitionsTask
+ */
+public class CacheResetLostPartitionsTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Caches. */
+    private Set<String> caches;
+
+    /**
+     * Default constructor.
+     */
+    public CacheResetLostPartitionsTaskArg() {
+        // No-op.
+    }
+
+    /**
+     * @param caches Caches.
+     */
+    public CacheResetLostPartitionsTaskArg(Set<String> caches) {
+        this.caches = caches;
+    }
+
+    /**
+     * @return Caches.
+     */
+    public Set<String> getCaches() {
+        return caches;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeCollection(out, caches);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte ver, ObjectInput in) throws IOException, ClassNotFoundException {
+        caches = U.readSet(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(CacheResetLostPartitionsTaskArg.class, this);
+    }
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTaskResult.java
new file mode 100644
index 0000000000000..7eb1f31d52951
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/reset_lost_partitions/CacheResetLostPartitionsTaskResult.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.commandline.cache.reset_lost_partitions;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.io.PrintStream;
+import java.util.Map;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Result of CacheResetLostPartitionsTask
+ */
+public class CacheResetLostPartitionsTaskResult extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * Map group name to result execute message.
+     */
+    private Map<String, String> msgMap;
+
+    /**
+     * @param groupName - Cache group name.
+     * @param message - Job result message.
+     * @return the previous value associated with <tt>key</tt>, or <tt>null</tt>
+     */
+    public String put(String groupName, String message) {
+        return this.msgMap.put(groupName, message);
+    }
+
+    /**
+     * Print job result.
+     *
+     * @param out Print stream.
+     */
+    public void print(PrintStream out) {
+        if (msgMap == null || msgMap.isEmpty())
+            return;
+
+        for (String message : msgMap.values())
+            out.println(message);
+    }
+
+    /** */
+    public Map<String, String> getMessageMap() {
+        return msgMap;
+    }
+
+    /** */
+    public void setMessageMap(Map<String, String> messageMap) {
+        this.msgMap = messageMap;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, msgMap);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        msgMap = U.readMap(in);
+    }
+}
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 8979679ebf7d5..284b3dbda9521 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -327,6 +327,19 @@ org.apache.ignite.internal.cluster.NodeOrderComparator
 org.apache.ignite.internal.cluster.NodeOrderLegacyComparator
 org.apache.ignite.internal.commandline.Command
 org.apache.ignite.internal.commandline.cache.CacheCommand
+org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionGroup
+org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionNode
+org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionPartition
+org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionTask$CacheDistributionJob
+org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionTask
+org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionTaskArg
+org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionTaskResult$1
+org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionTaskResult$Row
+org.apache.ignite.internal.commandline.cache.distribution.CacheDistributionTaskResult
+org.apache.ignite.internal.commandline.cache.reset_lost_partitions.CacheResetLostPartitionsTaskResult
+org.apache.ignite.internal.commandline.cache.reset_lost_partitions.CacheResetLostPartitionsTask$CacheResetLostPartitionsJob
+org.apache.ignite.internal.commandline.cache.reset_lost_partitions.CacheResetLostPartitionsTask
+org.apache.ignite.internal.commandline.cache.reset_lost_partitions.CacheResetLostPartitionsTaskArg
 org.apache.ignite.internal.compute.ComputeTaskCancelledCheckedException
 org.apache.ignite.internal.compute.ComputeTaskTimeoutCheckedException
 org.apache.ignite.internal.direct.DirectMessageReader$1
@@ -1595,6 +1608,7 @@ org.apache.ignite.internal.processors.rest.handlers.redis.exception.GridRedisGen
 org.apache.ignite.internal.processors.rest.handlers.redis.exception.GridRedisTypeException
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler$2
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler$ExeCallable
+org.apache.ignite.internal.processors.rest.handlers.task.GridTaskCommandHandler$TaskDescriptor
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultRequest
 org.apache.ignite.internal.processors.rest.handlers.task.GridTaskResultResponse
 org.apache.ignite.internal.processors.rest.handlers.top.GridTopologyCommandHandler$1
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 4f922dbab59c9..371afb8e2c5e7 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -64,7 +64,6 @@
 import org.apache.ignite.internal.processors.cache.CacheObjectImpl;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
-import org.apache.ignite.internal.processors.cache.GridCacheFuture;
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCacheOperation;
 import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
@@ -1060,7 +1059,8 @@ public void testCacheIdleVerifyDumpForCorruptedData() throws Exception {
             String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
 
             assertTrue(dumpWithConflicts.contains("found 2 conflict partitions: [counterConflicts=1, hashConflicts=1]"));
-        }else
+        }
+        else
             fail("Should be found dump with conflicts");
     }
 
@@ -1254,6 +1254,84 @@ public void testCacheAffinity() throws Exception {
         assertTrue(testOut.toString().contains("affCls=RendezvousAffinityFunction"));
     }
 
+    /**
+     *
+     */
+    public void testCacheDistribution() throws Exception {
+        Ignite ignite = startGrids(2);
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME));
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, i);
+
+        injectTestSystemOut();
+
+        // Run distribution for all node and all cache
+        assertEquals(EXIT_CODE_OK, execute("--cache", "distribution", "null"));
+
+        String log = testOut.toString();
+
+        // Result include info by cache "default"
+        assertTrue(log.contains("[next group: id=1544803905, name=default]"));
+
+        // Result include info by cache "ignite-sys-cache"
+        assertTrue(log.contains("[next group: id=-2100569601, name=ignite-sys-cache]"));
+
+        // Run distribution for all node and all cache and include additional user attribute
+        assertEquals(EXIT_CODE_OK, execute("--cache", "distribution", "null", "--user-attributes", "ZONE,CELL,DC"));
+
+        log = testOut.toString();
+
+        // Find last row
+        int lastRowIndex = log.lastIndexOf('\n');
+
+        assertTrue(lastRowIndex > 0);
+
+        // Last row is empty, but the previous line contains data
+        lastRowIndex = log.lastIndexOf('\n', lastRowIndex-1);
+
+        assertTrue(lastRowIndex > 0);
+
+        String lastRow = log.substring(lastRowIndex);
+
+        // Since 3 user attributes have been added, the total number of columns in response should be 12 (separators 11)
+        assertEquals(11, lastRow.split(",").length);
+    }
+
+    /**
+     *
+     */
+    public void testCacheResetLostPartitions() throws Exception {
+        Ignite ignite = startGrids(2);
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME));
+
+        for (int i = 0; i < 100; i++)
+            cache.put(i, i);
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "reset_lost_partitions", "ignite-sys-cache,default"));
+
+        final String log = testOut.toString();
+
+        assertTrue(log.contains("Reset LOST-partitions performed successfully. Cache group (name = 'ignite-sys-cache'"));
+
+        assertTrue(log.contains("Reset LOST-partitions performed successfully. Cache group (name = 'default'"));
+
+    }
+
     /**
      * @param h Handler.
      * @param validateClo Validate clo.

From 5bb323b0dc388c163794865ad523e50f84e94e2d Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Thu, 27 Sep 2018 19:40:18 +0300
Subject: [PATCH 392/543] IGNITE-9612 Fix NPE related not thread safe usage
 TreeMap onMarkCheckpointBegin

(cherry picked from commit 4527fe6de8bddea31a8142e78a84eacb534d0384)
---
 .../persistence/partstate/PartitionAllocationMap.java     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
index c2cc171cb103c..cab90c5b3954a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
@@ -17,14 +17,14 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.partstate;
 
-import java.util.HashSet;
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Set;
-import java.util.TreeMap;
+import java.util.concurrent.ConcurrentSkipListMap;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.NotNull;
@@ -37,11 +37,11 @@
 public class PartitionAllocationMap {
     /** Maps following pairs: (groupId, partId) -> (lastAllocatedCount, allocatedCount) */
     @GridToStringInclude
-    private final NavigableMap<GroupPartitionId, PagesAllocationRange> map = new TreeMap<>();
+    private final NavigableMap<GroupPartitionId, PagesAllocationRange> map = new ConcurrentSkipListMap<>();
 
     /** Partitions forced to be skipped. */
     @GridToStringInclude
-    private final Set<GroupPartitionId> skippedParts = new HashSet<>();
+    private final Set<GroupPartitionId> skippedParts = new GridConcurrentHashSet<>();
 
     /**
      * Returns the value to which the specified key is mapped,

From 33ed7edfec38d119d45f8c4ed073596f45868f77 Mon Sep 17 00:00:00 2001
From: AMedvedev <git@andmed.us>
Date: Thu, 27 Sep 2018 22:57:13 +0300
Subject: [PATCH 393/543] IGNITE-9683 Create manual pinger for ZK client -
 Fixes #4839.

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit a8d50e4cd533225060b0805b5c977668ba11ee48)
---
 .../spi/discovery/zk/internal/ZkPinger.java   | 88 +++++++++++++++++++
 .../zk/internal/ZookeeperClient.java          | 30 ++++++-
 .../zk/internal/ZookeeperDiscoveryImpl.java   | 13 +++
 3 files changed, 130 insertions(+), 1 deletion(-)
 create mode 100644 modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkPinger.java

diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkPinger.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkPinger.java
new file mode 100644
index 0000000000000..964b8fc8b4144
--- /dev/null
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkPinger.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.Timer;
+import java.util.TimerTask;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.zookeeper.ZooKeeper;
+
+/**
+ * Periodically pings ZK server with simple request. Prevents connection abort on timeout from ZK side.
+ */
+public class ZkPinger extends TimerTask {
+    /** Ping interval milliseconds. */
+    private static final int PING_INTERVAL_MS = 2000;
+
+    /** Logger. */
+    private final IgniteLogger log;
+
+    /** Zk client. */
+    private final ZooKeeper zkClient;
+
+    /** Paths. */
+    private final ZkIgnitePaths paths;
+
+    /** Scheduler. */
+    private final Timer scheduler = new Timer("ignite-zk-pinger");
+
+    /**
+     * @param log Logger.
+     * @param zkClient Zk client.
+     * @param paths Paths.
+     */
+    public ZkPinger(IgniteLogger log, ZooKeeper zkClient, ZkIgnitePaths paths) {
+        this.log = log;
+        this.zkClient = zkClient;
+        this.paths = paths;
+    }
+
+    /** {@inheritDoc} */
+    @Override public void run() {
+        try {
+            zkClient.exists(paths.clusterDir, false);
+        }
+        catch (Throwable t) {
+            if (zkClient.getState().isAlive())
+                U.warn(log, "Failed to ping Zookeeper.", t);
+            else
+                scheduler.cancel();
+        }
+
+    }
+
+    /**
+     * Starts ping process.
+     */
+    public void start() {
+        scheduler.scheduleAtFixedRate(this, 0, PING_INTERVAL_MS);
+    }
+
+    /**
+     * Stops ping process.
+     */
+    public void stop() {
+        try {
+            scheduler.cancel();
+        }
+        catch (Exception e) {
+            log.warning("Failed to cancel Zookeeper Pinger scheduler.", e);
+        }
+    }
+}
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
index 6cc77a5875eaa..1e57b73753a84 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -51,6 +51,10 @@ public class ZookeeperClient implements Watcher {
     private static final int MAX_RETRY_COUNT =
         IgniteSystemProperties.getInteger("IGNITE_ZOOKEEPER_DISCOVERY_MAX_RETRY_COUNT", 10);
 
+    /** */
+    private static final boolean PINGER_ENABLED =
+        IgniteSystemProperties.getBoolean("IGNITE_ZOOKEEPER_DISCOVERY_PINGER_ENABLED", false);
+
     /** */
     private final AtomicInteger retryCount = new AtomicInteger();
 
@@ -93,6 +97,9 @@ public class ZookeeperClient implements Watcher {
     /** */
     private volatile boolean closing;
 
+    /** */
+    private volatile ZkPinger pinger;
+
     /**
      * @param log Logger.
      * @param connectString ZK connection string.
@@ -162,6 +169,13 @@ boolean connected() {
         }
     }
 
+    /**
+     * @return {@code True} if pinger is enabled
+     */
+    boolean pingerEnabled() {
+        return PINGER_ENABLED;
+    }
+
     /** */
     String state() {
         synchronized (stateMux) {
@@ -557,7 +571,6 @@ void deleteIfExists(String path, int ver)
     }
 
     /**
-     * @param parent Parent path.
      * @param paths Children paths.
      * @param ver Version.
      * @throws KeeperException.NoNodeException If at least one of nodes does not exist.
@@ -764,6 +777,13 @@ void onCloseStart() {
      *
      */
     public void close() {
+        if (PINGER_ENABLED) {
+            ZkPinger pinger0 = pinger;
+
+            if (pinger0 != null)
+                pinger0.stop();
+        }
+
         closeClient();
     }
 
@@ -892,6 +912,14 @@ private void scheduleConnectionCheck() {
         connTimer.schedule(new ConnectionTimeoutTask(connStartTime), connLossTimeout);
     }
 
+    /**
+     * @param pinger Pinger.
+     */
+    void attachPinger(ZkPinger pinger) {
+        if (PINGER_ENABLED)
+            this.pinger = pinger;
+    }
+
     /**
      *
      */
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 3ad8e179c20b8..5d029b8208f3e 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -786,6 +786,19 @@ private void joinTopology(@Nullable ZkRuntimeState prevState) throws Interrupted
             }
 
             startJoin(rtState, prevState, joinDataBytes);
+
+            try {
+                if (rtState.zkClient.pingerEnabled() && !locNode.isClient() && !locNode.isDaemon()) {
+                    ZkPinger pinger = new ZkPinger(log, rtState.zkClient.zk(), zkPaths);
+
+                    rtState.zkClient.attachPinger(pinger);
+
+                    pinger.start();
+                }
+            }
+            catch (Exception e) {
+                log.error("Failed to create and attach Zookeeper pinger", e);
+            }
         }
         finally {
             busyLock.leaveBusy();

From 06cada76030bd9a53515d9de6793fb8379a82277 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Fri, 28 Sep 2018 12:06:52 +0300
Subject: [PATCH 394/543] IGNITE-9501 Backward compatibility fix - Fixes #4860.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit d5f6e50)
---
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java         | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 373aac4216703..b2ccc051cad24 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1455,7 +1455,7 @@ private void waitPartitionRelease(boolean distributed, boolean doRollback) throw
         releaseLatch.countDown();
 
         // For compatibility with old version where joining nodes are not waiting for latch.
-        if (!cctx.exchange().latch().canSkipJoiningNodes(initialVersion()))
+        if (localJoinExchange() && !cctx.exchange().latch().canSkipJoiningNodes(initialVersion()))
             return;
 
         try {
@@ -2462,7 +2462,6 @@ private void processSingleMessage(UUID nodeId, GridDhtPartitionsSingleMessage ms
                 }
             }
         }
-
         if (allReceived) {
             if (!awaitSingleMapUpdates())
                 return;

From f19f715f0250eff2701a13bb8f68f21373f64ba8 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Fri, 28 Sep 2018 18:20:00 +0300
Subject: [PATCH 395/543] IGNITE-9731 Fixed NPE on concurrent WAL flush - Fixes
 #4863.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 69adfd5)
---
 .../wal/FileWriteAheadLogManager.java         |  10 +-
 .../db/wal/WalRolloverRecordLoggingTest.java  | 156 ++++++++++++++++++
 .../IgnitePdsWithIndexingCoreTestSuite.java   |   2 +
 3 files changed, 165 insertions(+), 3 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 55fb3d8352056..86009a8ac412c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -2227,19 +2227,23 @@ else if (create)
     private abstract static class FileHandle {
         /** I/O interface for read/write operations with file */
         SegmentIO fileIO;
+        
+        /** Segment idx corresponded to fileIo*/
+        final long segmentIdx;
 
         /**
-         * @param fileIO I/O interface for read/write operations of FileHandle.         *
+         * @param fileIO I/O interface for read/write operations of FileHandle.
          */
-        private FileHandle(SegmentIO fileIO) {
+        private FileHandle(@NotNull SegmentIO fileIO) {
             this.fileIO = fileIO;
+            segmentIdx = fileIO.getSegmentId();
         }
 
         /**
          * @return Absolute WAL segment file index (incremental counter).
          */
         public long getSegmentId(){
-            return fileIO.getSegmentId();
+            return segmentIdx;
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java
new file mode 100644
index 0000000000000..67caf63506641
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_PATH;
+import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
+/**
+ *
+ */
+public class WalRolloverRecordLoggingTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static class RolloverRecord extends CheckpointRecord {
+        /** */
+        private RolloverRecord() {
+            super(null);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean rollOver() {
+            return true;
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(name);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+                .setPersistenceEnabled(true)
+                .setMaxSize(40 * 1024 * 1024))
+            .setWalMode(LOG_ONLY)
+            .setWalSegmentSize(4 * 1024 * 1024)
+            .setWalArchivePath(DFLT_WAL_PATH));
+
+        cfg.setFailureHandler(new StopNodeOrHaltFailureHandler(false, 0));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** */
+    public void testAvoidInfinityWaitingOnRolloverOfSegment() throws Exception {
+        IgniteEx ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        IgniteCache<Integer, Integer> cache = ig.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+        long startTime = U.currentTimeMillis();
+        long duration = 5_000;
+
+        IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(
+            () -> {
+                ThreadLocalRandom random = ThreadLocalRandom.current();
+
+                while (U.currentTimeMillis() - startTime < duration)
+                    cache.put(random.nextInt(100_000), random.nextInt(100_000));
+            },
+            8, "cache-put-thread");
+
+        Thread t = new Thread(() -> {
+            do {
+                try {
+                    U.sleep(100);
+                }
+                catch (IgniteInterruptedCheckedException e) {
+                    // No-op.
+                }
+
+                ig.context().cache().context().database().wakeupForCheckpoint("test");
+            } while (U.currentTimeMillis() - startTime < duration);
+        });
+
+        t.start();
+
+        IgniteWriteAheadLogManager walMgr = ig.context().cache().context().wal();
+
+        IgniteCacheDatabaseSharedManager dbMgr = ig.context().cache().context().database();
+
+        RolloverRecord rec = new RolloverRecord();
+
+        do {
+            try {
+                dbMgr.checkpointReadLock();
+
+                try {
+                    walMgr.log(rec);
+                }
+                finally {
+                    dbMgr.checkpointReadUnlock();
+                }
+            }
+            catch (IgniteCheckedException e) {
+                log.error(e.getMessage(), e);
+            }
+        } while (U.currentTimeMillis() - startTime < duration);
+
+        fut.get();
+
+        t.join();
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index 491bab7dd65bd..790f14b1202fa 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -40,6 +40,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryWithCompactionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalPathsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRecoveryTxLogicalRecordsTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRolloverRecordLoggingTest;
 
 /**
  * Test suite for tests that cover core PDS features and depend on indexing module.
@@ -59,6 +60,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(PersistenceDirectoryWarningLoggingTest.class);
         suite.addTestSuite(WalPathsTest.class);
         suite.addTestSuite(WalRecoveryTxLogicalRecordsTest.class);
+        suite.addTestSuite(WalRolloverRecordLoggingTest.class);
 
         suite.addTestSuite(IgniteWalRecoveryTest.class);
         suite.addTestSuite(IgniteWalRecoveryWithCompactionTest.class);

From ea12772615ec330ce841db0c8542676eb09ccb01 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Fri, 28 Sep 2018 18:03:45 +0300
Subject: [PATCH 396/543] IGNITE-9693 Scale up wal compression workers to
 increase performance - Fixes #4831.

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit 036bd074d8bfd25a2c4c463a60dde00604d11b9d)
---
 .../apache/ignite/IgniteSystemProperties.java |   6 +
 .../wal/IgniteWriteAheadLogManager.java       |   3 +-
 .../GridCacheDatabaseSharedManager.java       |  11 +-
 .../wal/FileWriteAheadLogManager.java         | 201 +++++++++++-------
 .../FsyncModeFileWriteAheadLogManager.java    |   7 +-
 .../persistence/wal/aware/SegmentAware.java   |  28 ++-
 .../wal/aware/SegmentCompressStorage.java     |  80 +++++--
 ...sReserveWalSegmentsWithCompactionTest.java |  34 +++
 .../persistence/pagemem/NoOpWALManager.java   |   2 +-
 .../wal/aware/SegmentAwareTest.java           |  90 +++++---
 .../testsuites/IgnitePdsTestSuite2.java       |   2 +
 11 files changed, 322 insertions(+), 142 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index d664cc084720e..ba98b785ad5fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -869,6 +869,12 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_LOADED_PAGES_BACKWARD_SHIFT_MAP = "IGNITE_LOADED_PAGES_BACKWARD_SHIFT_MAP";
 
+
+    /**
+     * Count of WAL compressor worker threads. Default value is 4.
+     */
+    public static final String IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT = "IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT";
+
     /**
      * Whenever read load balancing is enabled, that means 'get' requests will be distributed between primary and backup
      * nodes if it is possible and {@link CacheConfiguration#readFromBackup} is {@code true}.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 26a9bcebf2000..70c9ad45129bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -86,9 +86,8 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
      * Invoke this method to reserve WAL history since provided pointer and prevent it's deletion.
      *
      * @param start WAL pointer.
-     * @throws IgniteException If failed to reserve.
      */
-    public boolean reserve(WALPointer start) throws IgniteCheckedException;
+    public boolean reserve(WALPointer start);
 
     /**
      * Invoke this method to release WAL history since provided pointer that was previously reserved.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index a6d58a6ea2e06..6d97e988e2706 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1724,16 +1724,7 @@ private Map<Integer, Set<Integer>> partitionsApplicableForWalRebalance() {
         if (ptr == null)
             return false;
 
-        boolean reserved;
-
-        try {
-            reserved = cctx.wal().reserve(ptr);
-        }
-        catch (IgniteCheckedException e) {
-            U.error(log, "Error while trying to reserve history", e);
-
-            reserved = false;
-        }
+        boolean reserved = cctx.wal().reserve(ptr);
 
         if (reserved)
             reservedForPreloading.put(new T2<>(grpId, partId), new T2<>(cntr, ptr));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 86009a8ac412c..e70785462d831 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -132,6 +132,7 @@
 import static java.nio.file.StandardOpenOption.CREATE;
 import static java.nio.file.StandardOpenOption.READ;
 import static java.nio.file.StandardOpenOption.WRITE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SEGMENT_SYNC_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
@@ -244,6 +245,13 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     private static final AtomicLongFieldUpdater<FileWriteHandle> WRITTEN_UPD =
         AtomicLongFieldUpdater.newUpdater(FileWriteHandle.class, "written");
 
+
+    /**
+     * Number of WAL compressor worker threads.
+     */
+    private final int WAL_COMPRESSOR_WORKER_THREAD_CNT =
+            IgniteSystemProperties.getInteger(IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT, 4);
+
     /** */
     private final boolean alwaysWriteFullPages;
 
@@ -387,7 +395,7 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) {
         walAutoArchiveAfterInactivity = dsCfg.getWalAutoArchiveAfterInactivity();
         evt = ctx.event();
         failureProcessor = ctx.failure();
-        segmentAware = new SegmentAware(dsCfg.getWalSegments());
+        segmentAware = new SegmentAware(dsCfg.getWalSegments(), dsCfg.isWalCompactionEnabled());
     }
 
     /**
@@ -458,7 +466,8 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
                 segmentAware.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx);
 
             if (dsCfg.isWalCompactionEnabled()) {
-                compressor = new FileCompressor();
+                if (compressor == null)
+                    compressor = new FileCompressor(log);
 
                 if (decompressor == null) {  // Preventing of two file-decompressor thread instantiations.
                     decompressor = new FileDecompressor(log);
@@ -867,7 +876,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
+    @Override public boolean reserve(WALPointer start) {
         assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start;
 
         if (mode == WALMode.NONE)
@@ -977,7 +986,7 @@ private boolean segmentReservedOrLocked(long absIdx) {
     /** {@inheritDoc} */
     @Override public void notchLastCheckpointPtr(WALPointer ptr) {
         if (compressor != null)
-            compressor.keepUncompressedIdxFrom(((FileWALPointer)ptr).index());
+            segmentAware.keepUncompressedIdxFrom(((FileWALPointer)ptr).index());
     }
 
     /** {@inheritDoc} */
@@ -1800,16 +1809,13 @@ private void allocateRemainingFiles() throws StorageException {
      * Responsible for compressing WAL archive segments.
      * Also responsible for deleting raw copies of already compressed WAL archive segments if they are not reserved.
      */
-    private class FileCompressor extends Thread {
-        /** Current thread stopping advice. */
-        private volatile boolean stopped;
-
-        /** All segments prior to this (inclusive) can be compressed. */
-        private volatile long minUncompressedIdxToKeep = -1L;
+    private class FileCompressor extends FileCompressorWorker {
+        /** Workers queue. */
+        List<FileCompressorWorker> workers = new ArrayList<>();
 
         /** */
-        FileCompressor() {
-            super("wal-file-compressor%" + cctx.igniteInstanceName());
+        FileCompressor(IgniteLogger log) {
+            super(0, log);
         }
 
         /** */
@@ -1817,7 +1823,7 @@ private void init() {
             File[] toDel = walArchiveDir.listFiles(WAL_SEGMENT_TEMP_FILE_COMPACTED_FILTER);
 
             for (File f : toDel) {
-                if (stopped)
+                if (isCancelled())
                     return;
 
                 f.delete();
@@ -1826,82 +1832,118 @@ private void init() {
             FileDescriptor[] alreadyCompressed = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_COMPACTED_FILTER));
 
             if (alreadyCompressed.length > 0)
-                segmentAware.lastCompressedIdx(alreadyCompressed[alreadyCompressed.length - 1].idx());
+                segmentAware.onSegmentCompressed(alreadyCompressed[alreadyCompressed.length - 1].idx());
+
+            for (int i = 1; i < calculateThreadCount(); i++) {
+                FileCompressorWorker worker = new FileCompressorWorker(i, log);
+
+                worker.start();
+
+                workers.add(worker);
+            }
         }
 
         /**
-         * @param idx Minimum raw segment index that should be preserved from deletion.
+         * Calculate optimal additional compressor worker threads count. If quarter of proc threads greater
+         * than WAL_COMPRESSOR_WORKER_THREAD_CNT, use this value. Otherwise, reduce number of threads.
+         *
+         * @return Optimal number of compressor threads.
          */
-        void keepUncompressedIdxFrom(long idx) {
-            minUncompressedIdxToKeep = idx;
+        private int calculateThreadCount() {
+            int procNum = Runtime.getRuntime().availableProcessors();
+
+            // If quarter of proc threads greater than WAL_COMPRESSOR_WORKER_THREAD_CNT,
+            // use this value. Otherwise, reduce number of threads.
+            if (procNum >> 2 >= WAL_COMPRESSOR_WORKER_THREAD_CNT)
+                return WAL_COMPRESSOR_WORKER_THREAD_CNT;
+            else
+                return procNum >> 2;
         }
 
-        /**
-         * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation.
-         * Waits if there's no segment to archive right now.
-         */
-        private long tryReserveNextSegmentOrWait() throws IgniteCheckedException {
-            long segmentToCompress = segmentAware.waitNextSegmentToCompress();
 
-            boolean reserved = reserve(new FileWALPointer(segmentToCompress, 0, 0));
+        /** {@inheritDoc} */
+        @Override public void body() throws InterruptedException, IgniteInterruptedCheckedException{
+            init();
 
-            return reserved ? segmentToCompress : -1;
+            super.body0();
         }
 
         /**
-         * Deletes raw WAL segments if they aren't locked and already have compressed copies of themselves.
+         * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
          */
-        private void deleteObsoleteRawSegments() {
-            FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
+        private void shutdown() throws IgniteInterruptedCheckedException {
+            synchronized (this) {
+                for (FileCompressorWorker worker: workers)
+                    U.cancel(worker);
 
-            Set<Long> indices = new HashSet<>();
-            Set<Long> duplicateIndices = new HashSet<>();
+                for (FileCompressorWorker worker: workers)
+                    U.join(worker);
 
-            for (FileDescriptor desc : descs) {
-                if (!indices.add(desc.idx))
-                    duplicateIndices.add(desc.idx);
+                U.cancel(this);
             }
 
-            for (FileDescriptor desc : descs) {
-                if (desc.isCompressed())
-                    continue;
+            U.join(this);
+        }
+    }
 
-                // Do not delete reserved or locked segment and any segment after it.
-                if (segmentReservedOrLocked(desc.idx))
-                    return;
+    /** */
+    private class FileCompressorWorker extends GridWorker {
+        /** */
+        private Thread thread;
 
-                if (desc.idx < minUncompressedIdxToKeep && duplicateIndices.contains(desc.idx)) {
-                    if (!desc.file.delete())
-                        U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
-                            desc.file.getAbsolutePath() + ", exists: " + desc.file.exists());
-                }
-            }
+        /** */
+        FileCompressorWorker(int idx,  IgniteLogger log) {
+            super(cctx.igniteInstanceName(), "wal-file-compressor-%" + cctx.igniteInstanceName() + "%-" + idx, log);
+        }
+
+        /** */
+        void start() {
+            thread = new IgniteThread(this);
+
+            thread.start();
+        }
+
+        /**
+         * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation.
+         * Waits if there's no segment to archive right now.
+         */
+        private long tryReserveNextSegmentOrWait() throws IgniteInterruptedCheckedException{
+            long segmentToCompress = segmentAware.waitNextSegmentToCompress();
+
+            boolean reserved = reserve(new FileWALPointer(segmentToCompress, 0, 0));
+
+            return reserved ? segmentToCompress : -1;
         }
 
         /** {@inheritDoc} */
-        @Override public void run() {
-            init();
+        @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+            body0();
+        }
 
-            while (!Thread.currentThread().isInterrupted() && !stopped) {
-                long currReservedSegment = -1;
+        /** */
+        private void body0() {
+            while (!isCancelled()) {
+                long segIdx = -1L;
 
                 try {
-                    deleteObsoleteRawSegments();
+                    segIdx = tryReserveNextSegmentOrWait();
 
-                    currReservedSegment = tryReserveNextSegmentOrWait();
-                    if (currReservedSegment == -1)
+                    if (segIdx <= segmentAware.lastCompressedIdx())
                         continue;
 
-                    File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment)
-                        + FilePageStoreManager.ZIP_SUFFIX + FilePageStoreManager.TMP_SUFFIX);
+                    deleteObsoleteRawSegments();
+
+                    File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(segIdx)
+                            + FilePageStoreManager.ZIP_SUFFIX + FilePageStoreManager.TMP_SUFFIX);
 
-                    File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + FilePageStoreManager.ZIP_SUFFIX);
+                    File zip = new File(walArchiveDir, FileDescriptor.fileName(segIdx) + FilePageStoreManager.ZIP_SUFFIX);
+
+                    File raw = new File(walArchiveDir, FileDescriptor.fileName(segIdx));
 
-                    File raw = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment));
                     if (!Files.exists(raw.toPath()))
                         throw new IgniteCheckedException("WAL archive segment is missing: " + raw);
 
-                    compressSegmentToFile(currReservedSegment, raw, tmpZip);
+                    compressSegmentToFile(segIdx, raw, tmpZip);
 
                     Files.move(tmpZip.toPath(), zip.toPath());
 
@@ -1911,20 +1953,20 @@ private void deleteObsoleteRawSegments() {
                         }
                     }
 
-                    segmentAware.lastCompressedIdx(currReservedSegment);
+                    segmentAware.onSegmentCompressed(segIdx);
                 }
                 catch (IgniteInterruptedCheckedException ignore) {
                     Thread.currentThread().interrupt();
                 }
                 catch (IgniteCheckedException | IOException e) {
-                    U.error(log, "Compression of WAL segment [idx=" + currReservedSegment +
-                        "] was skipped due to unexpected error", e);
+                    U.error(log, "Compression of WAL segment [idx=" + segIdx +
+                            "] was skipped due to unexpected error", e);
 
-                    segmentAware.lastCompressedIdx(currReservedSegment);
+                    segmentAware.onSegmentCompressed(segIdx);
                 }
                 finally {
-                    if (currReservedSegment != -1)
-                        release(new FileWALPointer(currReservedSegment, 0, 0));
+                    if (segIdx != -1L)
+                        release(new FileWALPointer(segIdx, 0, 0));
                 }
             }
         }
@@ -1984,7 +2026,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
          * @param ser Record Serializer.
          */
         @NotNull private ByteBuffer prepareSwitchSegmentRecordBuffer(long nextSegment, RecordSerializer ser)
-            throws IgniteCheckedException {
+                throws IgniteCheckedException {
             SwitchSegmentRecord switchRecord = new SwitchSegmentRecord();
 
             int switchRecordSize = ser.size(switchRecord);
@@ -1999,16 +2041,33 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
         }
 
         /**
-         * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
+         * Deletes raw WAL segments if they aren't locked and already have compressed copies of themselves.
          */
-        private void shutdown() throws IgniteInterruptedCheckedException {
-            synchronized (this) {
-                stopped = true;
+        private void deleteObsoleteRawSegments() {
+            FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
 
-                notifyAll();
+            Set<Long> indices = new HashSet<>();
+            Set<Long> duplicateIndices = new HashSet<>();
+
+            for (FileDescriptor desc : descs) {
+                if (!indices.add(desc.idx))
+                    duplicateIndices.add(desc.idx);
             }
 
-            U.join(this);
+            for (FileDescriptor desc : descs) {
+                if (desc.isCompressed())
+                    continue;
+
+                // Do not delete reserved or locked segment and any segment after it.
+                if (segmentReservedOrLocked(desc.idx))
+                    return;
+
+                if (desc.idx < segmentAware.keepUncompressedIdxFrom() && duplicateIndices.contains(desc.idx)) {
+                    if (desc.file.exists() && !desc.file.delete())
+                        U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
+                                desc.file.getAbsolutePath() + ", exists: " + desc.file.exists());
+                }
+            }
         }
     }
 
@@ -2227,7 +2286,7 @@ else if (create)
     private abstract static class FileHandle {
         /** I/O interface for read/write operations with file */
         SegmentIO fileIO;
-        
+
         /** Segment idx corresponded to fileIo*/
         final long segmentIdx;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 4fa96910fa37d..9ffb16bf45209 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -755,7 +755,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
+    @Override public boolean reserve(WALPointer start) {
         assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start;
 
         if (mode == WALMode.NONE)
@@ -763,8 +763,7 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
 
         FileArchiver archiver0 = archiver;
 
-        if (archiver0 == null)
-            throw new IgniteCheckedException("Could not reserve WAL segment: archiver == null");
+        assert archiver0 != null : "Could not reserve WAL segment: archiver == null";
 
         archiver0.reserve(((FileWALPointer)start).index());
 
@@ -1793,7 +1792,7 @@ synchronized void onNextSegmentArchived() {
          * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation.
          * Waits if there's no segment to archive right now.
          */
-        private long tryReserveNextSegmentOrWait() throws InterruptedException, IgniteCheckedException {
+        private long tryReserveNextSegmentOrWait() throws InterruptedException {
             long segmentToCompress = lastCompressedIdx + 1;
 
             synchronized (this) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
index 3379b74cf5780..6ba03991e58a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
@@ -36,15 +36,17 @@ public class SegmentAware {
     /** Manages last archived index, emulates archivation in no-archiver mode. */
     private final SegmentArchivedStorage segmentArchivedStorage = buildArchivedStorage(segmentLockStorage);
     /** Storage of actual information about current index of compressed segments. */
-    private final SegmentCompressStorage segmentCompressStorage = buildCompressStorage(segmentArchivedStorage);
+    private final SegmentCompressStorage segmentCompressStorage;
     /** Storage of absolute current segment index. */
     private final SegmentCurrentStateStorage segmentCurrStateStorage;
 
     /**
      * @param walSegmentsCnt Total WAL segments count.
+     * @param compactionEnabled Is wal compaction enabled.
      */
-    public SegmentAware(int walSegmentsCnt) {
+    public SegmentAware(int walSegmentsCnt, boolean compactionEnabled) {
         segmentCurrStateStorage = buildCurrentStateStorage(walSegmentsCnt, segmentArchivedStorage);
+        segmentCompressStorage = buildCompressStorage(segmentArchivedStorage, compactionEnabled);
     }
 
     /**
@@ -108,12 +110,12 @@ public long waitNextSegmentToCompress() throws IgniteInterruptedCheckedException
     }
 
     /**
-     * Force set last compressed segment.
+     * Callback after segment compression finish.
      *
-     * @param lastCompressedIdx Segment which was last compressed.
+     * @param compressedIdx Index of compressed segment.
      */
-    public void lastCompressedIdx(long lastCompressedIdx) {
-        segmentCompressStorage.lastCompressedIdx(lastCompressedIdx);
+    public void onSegmentCompressed(long compressedIdx) {
+        segmentCompressStorage.onSegmentCompressed(compressedIdx);
     }
 
     /**
@@ -123,6 +125,20 @@ public long lastCompressedIdx() {
         return segmentCompressStorage.lastCompressedIdx();
     }
 
+    /**
+     * @param idx Minimum raw segment index that should be preserved from deletion.
+     */
+    public void keepUncompressedIdxFrom(long idx) {
+        segmentCompressStorage.keepUncompressedIdxFrom(idx);
+    }
+
+    /**
+     * @return  Minimum raw segment index that should be preserved from deletion.
+     */
+    public long keepUncompressedIdxFrom() {
+        return segmentCompressStorage.keepUncompressedIdxFrom();
+    }
+
     /**
      * Update current WAL index.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
index 30c9a2d50d363..174fb46bb6070 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
@@ -18,6 +18,10 @@
 package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
 
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
 
 /**
  * Storage of actual information about current index of compressed segments.
@@ -25,25 +29,50 @@
 public class SegmentCompressStorage {
     /** Flag of interrupt waiting on this object. */
     private volatile boolean interrupted;
+
     /** Manages last archived index, emulates archivation in no-archiver mode. */
     private final SegmentArchivedStorage segmentArchivedStorage;
+
+    /** If WAL compaction enabled. */
+    private final boolean compactionEnabled;
+
     /** Last successfully compressed segment. */
     private volatile long lastCompressedIdx = -1L;
 
+    /** Last enqueued to compress segment. */
+    private long lastEnqueuedToCompressIdx = -1L;
+
+    /** Segments to compress queue. */
+    private final Queue<Long> segmentsToCompress = new ArrayDeque<>();
+
+    /** List of currently compressing segments. */
+    private final List<Long> compressingSegments = new ArrayList<>();
+
+    /** Compressed segment with maximal index. */
+    private long lastMaxCompressedIdx = -1L;
+
+    /** Min uncompressed index to keep. */
+    private volatile long minUncompressedIdxToKeep = -1L;
+
     /**
      * @param segmentArchivedStorage Storage of last archived segment.
+     * @param compactionEnabled If WAL compaction enabled.
      */
-    private SegmentCompressStorage(SegmentArchivedStorage segmentArchivedStorage) {
+    private SegmentCompressStorage(SegmentArchivedStorage segmentArchivedStorage, boolean compactionEnabled) {
         this.segmentArchivedStorage = segmentArchivedStorage;
 
+        this.compactionEnabled = compactionEnabled;
+
         this.segmentArchivedStorage.addObserver(this::onSegmentArchived);
     }
 
     /**
      * @param segmentArchivedStorage Storage of last archived segment.
+     * @param compactionEnabled If WAL compaction enabled.
      */
-    static SegmentCompressStorage buildCompressStorage(SegmentArchivedStorage segmentArchivedStorage) {
-        SegmentCompressStorage storage = new SegmentCompressStorage(segmentArchivedStorage);
+    static SegmentCompressStorage buildCompressStorage(SegmentArchivedStorage segmentArchivedStorage,
+                                                       boolean compactionEnabled) {
+        SegmentCompressStorage storage = new SegmentCompressStorage(segmentArchivedStorage, compactionEnabled);
 
         segmentArchivedStorage.addObserver(storage::onSegmentArchived);
 
@@ -51,12 +80,20 @@ static SegmentCompressStorage buildCompressStorage(SegmentArchivedStorage segmen
     }
 
     /**
-     * Force set last compressed segment.
+     * Callback after segment compression finish.
      *
-     * @param lastCompressedIdx Segment which was last compressed.
+     * @param compressedIdx Index of compressed segment.
      */
-    void lastCompressedIdx(long lastCompressedIdx) {
-        this.lastCompressedIdx = lastCompressedIdx;
+    synchronized void onSegmentCompressed(long compressedIdx) {
+        if (compressedIdx > lastMaxCompressedIdx)
+            lastMaxCompressedIdx = compressedIdx;
+
+        compressingSegments.remove(compressedIdx);
+
+        if (!compressingSegments.isEmpty())
+            this.lastCompressedIdx = Math.min(lastMaxCompressedIdx, compressingSegments.get(0) - 1);
+        else
+            this.lastCompressedIdx = lastMaxCompressedIdx;
     }
 
     /**
@@ -71,13 +108,8 @@ long lastCompressedIdx() {
      * there's no segment to archive right now.
      */
     synchronized long nextSegmentToCompressOrWait() throws IgniteInterruptedCheckedException {
-        long segmentToCompress = lastCompressedIdx + 1;
-
         try {
-            while (
-                segmentToCompress > segmentArchivedStorage.lastArchivedAbsoluteIndex()
-                    && !interrupted
-                )
+            while (segmentsToCompress.peek() == null && !interrupted)
                 wait();
         }
         catch (InterruptedException e) {
@@ -86,7 +118,11 @@ synchronized long nextSegmentToCompressOrWait() throws IgniteInterruptedCheckedE
 
         checkInterrupted();
 
-        return segmentToCompress;
+        Long idx = segmentsToCompress.poll();
+
+        compressingSegments.add(idx);
+
+        return idx == null ? -1L : idx;
     }
 
     /**
@@ -110,7 +146,23 @@ private void checkInterrupted() throws IgniteInterruptedCheckedException {
      * Callback for waking up compressor when new segment is archived.
      */
     private synchronized void onSegmentArchived(long lastAbsArchivedIdx) {
+        while (lastEnqueuedToCompressIdx < lastAbsArchivedIdx && compactionEnabled)
+            segmentsToCompress.add(++lastEnqueuedToCompressIdx);
+
         notifyAll();
     }
 
+    /**
+     * @param idx Minimum raw segment index that should be preserved from deletion.
+     */
+    void keepUncompressedIdxFrom(long idx) {
+        minUncompressedIdxToKeep = idx;
+    }
+
+    /**
+     * @return  Minimum raw segment index that should be preserved from deletion.
+     */
+    long keepUncompressedIdxFrom() {
+        return minUncompressedIdxToKeep;
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java
new file mode 100644
index 0000000000000..c2c0990cac7ff
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class IgnitePdsReserveWalSegmentsWithCompactionTest extends IgnitePdsUnusedWalSegmentsTest {
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.getDataStorageConfiguration().setWalCompactionEnabled(true);
+
+        return cfg;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index 2a9e309d53b27..196cc7aa3b7a0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -67,7 +67,7 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
+    @Override public boolean reserve(WALPointer start) {
         return false;
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
index 82876845541ed..7840b0979e06c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
@@ -31,13 +31,12 @@
  * Test for {@link SegmentAware}.
  */
 public class SegmentAwareTest extends TestCase {
-
     /**
      * Waiting finished when work segment is set.
      */
     public void testFinishAwaitSegment_WhenExactWaitingSegmentWasSet() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
 
@@ -53,7 +52,7 @@ public void testFinishAwaitSegment_WhenExactWaitingSegmentWasSet() throws Ignite
      */
     public void testFinishAwaitSegment_WhenGreaterThanWaitingSegmentWasSet() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
 
@@ -69,7 +68,7 @@ public void testFinishAwaitSegment_WhenGreaterThanWaitingSegmentWasSet() throws
      */
     public void testFinishAwaitSegment_WhenNextSegmentEqualToWaitingOne() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
 
@@ -91,7 +90,7 @@ public void testFinishAwaitSegment_WhenNextSegmentEqualToWaitingOne() throws Ign
      */
     public void testFinishAwaitSegment_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
 
@@ -107,7 +106,7 @@ public void testFinishAwaitSegment_WhenInterruptWasCall() throws IgniteCheckedEx
      */
     public void testFinishWaitSegmentForArchive_WhenWorkSegmentIncremented() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         aware.curAbsWalIdx(5);
         aware.setLastArchivedAbsoluteIndex(4);
@@ -126,7 +125,7 @@ public void testFinishWaitSegmentForArchive_WhenWorkSegmentIncremented() throws
      */
     public void testFinishWaitSegmentForArchive_WhenWorkSegmentGreaterValue() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         aware.curAbsWalIdx(5);
         aware.setLastArchivedAbsoluteIndex(4);
@@ -145,7 +144,7 @@ public void testFinishWaitSegmentForArchive_WhenWorkSegmentGreaterValue() throws
      */
     public void testFinishWaitSegmentForArchive_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         aware.curAbsWalIdx(5);
         aware.setLastArchivedAbsoluteIndex(4);
@@ -164,7 +163,7 @@ public void testFinishWaitSegmentForArchive_WhenInterruptWasCall() throws Ignite
      */
     public void testCorrectCalculateNextSegmentIndex() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         aware.curAbsWalIdx(5);
 
@@ -180,7 +179,7 @@ public void testCorrectCalculateNextSegmentIndex() throws IgniteCheckedException
      */
     public void testFinishWaitNextAbsoluteIndex_WhenMarkAsArchivedFirstSegment() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(2);
+        SegmentAware aware = new SegmentAware(2, false);
 
         aware.curAbsWalIdx(1);
         aware.setLastArchivedAbsoluteIndex(-1);
@@ -199,7 +198,7 @@ public void testFinishWaitNextAbsoluteIndex_WhenMarkAsArchivedFirstSegment() thr
      */
     public void testFinishWaitNextAbsoluteIndex_WhenSetToArchivedFirst() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(2);
+        SegmentAware aware = new SegmentAware(2, false);
 
         aware.curAbsWalIdx(1);
         aware.setLastArchivedAbsoluteIndex(-1);
@@ -218,7 +217,7 @@ public void testFinishWaitNextAbsoluteIndex_WhenSetToArchivedFirst() throws Igni
      */
     public void testFinishWaitNextAbsoluteIndex_WhenOnlyForceInterruptWasCall() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(2);
+        SegmentAware aware = new SegmentAware(2, false);
 
         aware.curAbsWalIdx(2);
         aware.setLastArchivedAbsoluteIndex(-1);
@@ -243,7 +242,7 @@ public void testFinishWaitNextAbsoluteIndex_WhenOnlyForceInterruptWasCall() thro
      */
     public void testFinishSegmentArchived_WhenSetExactWaitingSegment() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
 
@@ -257,9 +256,9 @@ public void testFinishSegmentArchived_WhenSetExactWaitingSegment() throws Ignite
     /**
      * Waiting finished when segment archived.
      */
-    public void testFinishSegmentArchived_WhenMarkExactWatingSegment() throws IgniteCheckedException, InterruptedException {
+    public void testFinishSegmentArchived_WhenMarkExactWaitingSegment() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
 
@@ -273,9 +272,9 @@ public void testFinishSegmentArchived_WhenMarkExactWatingSegment() throws Ignite
     /**
      * Waiting finished when segment archived.
      */
-    public void testFinishSegmentArchived_WhenSetGreaterThanWatingSegment() throws IgniteCheckedException, InterruptedException {
+    public void testFinishSegmentArchived_WhenSetGreaterThanWaitingSegment() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
 
@@ -289,9 +288,9 @@ public void testFinishSegmentArchived_WhenSetGreaterThanWatingSegment() throws I
     /**
      * Waiting finished when segment archived.
      */
-    public void testFinishSegmentArchived_WhenMarkGreaterThanWatingSegment() throws IgniteCheckedException, InterruptedException {
+    public void testFinishSegmentArchived_WhenMarkGreaterThanWaitingSegment() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
 
@@ -307,7 +306,7 @@ public void testFinishSegmentArchived_WhenMarkGreaterThanWatingSegment() throws
      */
     public void testFinishSegmentArchived_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         aware.curAbsWalIdx(5);
         aware.setLastArchivedAbsoluteIndex(4);
@@ -326,7 +325,7 @@ public void testFinishSegmentArchived_WhenInterruptWasCall() throws IgniteChecke
      */
     public void testMarkAsMovedToArchive_WhenReleaseLockedSegment() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         aware.checkCanReadArchiveOrReserveWorkSegment(5);
 
@@ -344,7 +343,7 @@ public void testMarkAsMovedToArchive_WhenReleaseLockedSegment() throws IgniteChe
      */
     public void testMarkAsMovedToArchive_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
         aware.checkCanReadArchiveOrReserveWorkSegment(5);
 
         IgniteInternalFuture future = awaitThread(() -> aware.markAsMovedToArchive(5));
@@ -364,9 +363,9 @@ public void testMarkAsMovedToArchive_WhenInterruptWasCall() throws IgniteChecked
      */
     public void testFinishWaitSegmentToCompress_WhenSetLastArchivedSegment() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, true);
 
-        aware.lastCompressedIdx(5);
+        aware.onSegmentCompressed(5);
 
         IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress);
 
@@ -382,9 +381,9 @@ public void testFinishWaitSegmentToCompress_WhenSetLastArchivedSegment() throws
      */
     public void testFinishWaitSegmentToCompress_WhenMarkLastArchivedSegment() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, true);
 
-        aware.lastCompressedIdx(5);
+        aware.onSegmentCompressed(5);
 
         IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress);
 
@@ -400,9 +399,9 @@ public void testFinishWaitSegmentToCompress_WhenMarkLastArchivedSegment() throws
      */
     public void testCorrectCalculateNextCompressSegment() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, true);
 
-        aware.lastCompressedIdx(5);
+        aware.onSegmentCompressed(5);
         aware.setLastArchivedAbsoluteIndex(6);
         aware.lastTruncatedArchiveIdx(7);
 
@@ -418,8 +417,8 @@ public void testCorrectCalculateNextCompressSegment() throws IgniteCheckedExcept
      */
     public void testFinishWaitSegmentToCompress_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
-        aware.lastCompressedIdx(5);
+        SegmentAware aware = new SegmentAware(10, true);
+        aware.onSegmentCompressed(5);
 
         IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress);
 
@@ -430,12 +429,35 @@ public void testFinishWaitSegmentToCompress_WhenInterruptWasCall() throws Ignite
         assertTrue(future.get(20) instanceof IgniteInterruptedCheckedException);
     }
 
+    /**
+     * Tests that {@link SegmentAware#onSegmentCompressed} returns segments in proper order.
+     */
+    public void testLastCompressedIdxProperOrdering() throws IgniteInterruptedCheckedException {
+        SegmentAware aware = new SegmentAware(10, true);
+
+        for (int i = 0; i < 5 ; i++) {
+            aware.setLastArchivedAbsoluteIndex(i);
+            aware.waitNextSegmentToCompress();
+        }
+
+        aware.onSegmentCompressed(0);
+
+        aware.onSegmentCompressed(4);
+        assertEquals(0, aware.lastCompressedIdx());
+        aware.onSegmentCompressed(1);
+        assertEquals(1, aware.lastCompressedIdx());
+        aware.onSegmentCompressed(3);
+        assertEquals(1, aware.lastCompressedIdx());
+        aware.onSegmentCompressed(2);
+        assertEquals(4, aware.lastCompressedIdx());
+    }
+
     /**
      * Segment reserve correctly.
      */
     public void testReserveCorrectly() {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         //when: reserve one segment twice and one segment once.
         aware.reserve(5);
@@ -478,7 +500,7 @@ public void testReserveCorrectly() {
      */
     public void testAssertFail_WhenReleaseUnreservedSegment() {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         aware.reserve(5);
         try {
@@ -497,7 +519,7 @@ public void testAssertFail_WhenReleaseUnreservedSegment() {
      */
     public void testReserveWorkSegmentCorrectly() {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         //when: lock one segment twice.
         aware.checkCanReadArchiveOrReserveWorkSegment(5);
@@ -530,7 +552,7 @@ public void testReserveWorkSegmentCorrectly() {
      */
     public void testAssertFail_WhenReleaseUnreservedWorkSegment() {
         //given: thread which awaited segment.
-        SegmentAware aware = new SegmentAware(10);
+        SegmentAware aware = new SegmentAware(10, false);
 
         aware.checkCanReadArchiveOrReserveWorkSegment(5);
         try {
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index ec3ea8129d475..b94589178ea33 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -41,6 +41,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsUnusedWalSegmentsTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsReserveWalSegmentsWithCompactionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.SlowHistoricalRebalanceSmallHistoryTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
@@ -176,6 +177,7 @@ public static void addRealPageStoreTests(TestSuite suite) {
         suite.addTestSuite(IgnitePdsExchangeDuringCheckpointTest.class);
 
         suite.addTestSuite(IgnitePdsUnusedWalSegmentsTest.class);
+        suite.addTestSuite(IgnitePdsReserveWalSegmentsWithCompactionTest.class);
 
         // new style folders with generated consistent ID test
         suite.addTestSuite(IgniteUidAsConsistentIdMigrationTest.class);

From 16602ef85dd8c5d32c8484b9fcc8216a1a16fdab Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Fri, 28 Sep 2018 18:50:05 +0300
Subject: [PATCH 397/543] IGNITE-9612 Fixed checkpoint listener logic - Fixes
 #4864.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 2f3b567f61c2a9a1080ee7fda57f3a18231b1dae)
---
 .../persistence/GridCacheOffheapManager.java  | 29 ++++++++++---------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 61cdeb88bf267..6c3cf00bf6622 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -164,12 +164,24 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
 
         Executor execSvc = ctx.executor();
 
+        if (ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName())) {
+            if (execSvc == null)
+                updateSnapshotTag(ctx);
+            else {
+                execSvc.execute(() -> {
+                    try {
+                        updateSnapshotTag(ctx);
+                    }
+                    catch (IgniteCheckedException e) {
+                        throw new IgniteException(e);
+                    }
+                });
+            }
+        }
+
         if (execSvc == null) {
             reuseList.saveMetadata();
 
-            if (ctx.nextSnapshot())
-                updateSnapshotTag(ctx);
-
             for (CacheDataStore store : partDataStores.values())
                 saveStoreMetadata(store, ctx, false);
         }
@@ -183,17 +195,6 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
                 }
             });
 
-            if (ctx.nextSnapshot()) {
-                execSvc.execute(() -> {
-                    try {
-                        updateSnapshotTag(ctx);
-                    }
-                    catch (IgniteCheckedException e) {
-                        throw new IgniteException(e);
-                    }
-                });
-            }
-
             for (CacheDataStore store : partDataStores.values())
                 execSvc.execute(() -> {
                     try {

From a048bc25bb3760d6fc9d12030ba1eb42c256c0d9 Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Mon, 1 Oct 2018 12:21:56 +0300
Subject: [PATCH 398/543] GG-14253 Revert "IGNITE-5103 Fixed TcpDiscoverySpi
 not to ignore maxMissedHeartbeats property. Fixes #4446."

This reverts commit bc1a1c685ec66be5f6360a36f7f842e79b040412.
---
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  40 +-----
 .../tcp/TcpClientDiscoverySpiSelfTest.java    |  10 +-
 .../TcpDiscoveryClientSuspensionSelfTest.java | 135 ------------------
 .../IgniteSpiDiscoverySelfTestSuite.java      |   2 -
 4 files changed, 3 insertions(+), 184 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 08af4923f879b..a9896df45f1b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -6610,9 +6610,6 @@ private class ClientMessageWorker extends MessageWorker<T2<TcpDiscoveryAbstractM
         /** Current client metrics. */
         private volatile ClusterMetrics metrics;
 
-        /** Last metrics update message receive time. */
-        private volatile long lastMetricsUpdateMsgTime;
-
         /** */
         private final AtomicReference<GridFutureAdapter<Boolean>> pingFut = new AtomicReference<>();
 
@@ -6625,12 +6622,10 @@ private class ClientMessageWorker extends MessageWorker<T2<TcpDiscoveryAbstractM
          * @param log Logger.
          */
         private ClientMessageWorker(Socket sock, UUID clientNodeId, IgniteLogger log) {
-            super("tcp-disco-client-message-worker", log, Math.max(spi.metricsUpdateFreq, 10), null);
+            super("tcp-disco-client-message-worker", log, 2000, null);
 
             this.sock = sock;
             this.clientNodeId = clientNodeId;
-
-            lastMetricsUpdateMsgTime = U.currentTimeMillis();
         }
 
         /**
@@ -6651,8 +6646,6 @@ ClusterMetrics metrics() {
          * @param metrics New current client metrics.
          */
         void metrics(ClusterMetrics metrics) {
-            lastMetricsUpdateMsgTime = U.currentTimeMillis();
-
             this.metrics = metrics;
         }
 
@@ -6833,37 +6826,6 @@ public boolean ping(IgniteSpiOperationTimeoutHelper timeoutHelper) throws Interr
 
             U.closeQuiet(sock);
         }
-
-        /** {@inheritDoc} */
-        @Override protected void noMessageLoop() {
-            if (U.currentTimeMillis() - lastMetricsUpdateMsgTime > spi.clientFailureDetectionTimeout()) {
-                TcpDiscoveryNode clientNode = ring.node(clientNodeId);
-
-                if (clientNode != null) {
-                    boolean failedNode;
-
-                    synchronized (mux) {
-                        failedNode = failedNodes.containsKey(clientNode);
-                    }
-
-                    if (!failedNode) {
-                        String msg = "Client node considered as unreachable " +
-                            "and will be dropped from cluster, " +
-                            "because no metrics update messages received in interval: " +
-                            "TcpDiscoverySpi.clientFailureDetectionTimeout() ms. " +
-                            "It may be caused by network problems or long GC pause on client node, try to increase this " +
-                            "parameter. " +
-                            "[nodeId=" + clientNodeId +
-                            ", clientFailureDetectionTimeout=" + spi.clientFailureDetectionTimeout() +
-                            ']';
-
-                        failNode(clientNodeId, msg);
-
-                        U.warn(log, msg);
-                    }
-                }
-            }
-        }
     }
 
     /** */
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
index c85e94e6c3889..2d130e1db9052 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpClientDiscoverySpiSelfTest.java
@@ -148,7 +148,7 @@ public class TcpClientDiscoverySpiSelfTest extends GridCommonAbstractTest {
     private boolean longSockTimeouts;
 
     /** */
-    protected long clientFailureDetectionTimeout = 5000;
+    protected long clientFailureDetectionTimeout = 1000;
 
     /** */
     private IgniteInClosure2X<TcpDiscoveryAbstractMessage, Socket> afterWrite;
@@ -263,7 +263,7 @@ protected TcpDiscoverySpi getDiscoverySpi() {
         clientIpFinder = null;
         joinTimeout = TcpDiscoverySpi.DFLT_JOIN_TIMEOUT;
         netTimeout = TcpDiscoverySpi.DFLT_NETWORK_TIMEOUT;
-        clientFailureDetectionTimeout = 5000;
+        clientFailureDetectionTimeout = 1000;
         longSockTimeouts = false;
 
         assert G.allGrids().isEmpty();
@@ -538,8 +538,6 @@ public void testPingFailedClientNode() throws Exception {
 
         ((TestTcpDiscoverySpi)client.configuration().getDiscoverySpi()).resumeAll();
 
-        Thread.sleep(2000);
-
         assert ((IgniteEx)srv1).context().discovery().pingNode(client.cluster().localNode().id());
         assert ((IgniteEx)srv0).context().discovery().pingNode(client.cluster().localNode().id());
     }
@@ -585,8 +583,6 @@ public void testClientReconnectOnRouterSuspend() throws Exception {
      * @throws Exception If failed.
      */
     public void testClientReconnectOnRouterSuspendTopologyChange() throws Exception {
-        clientFailureDetectionTimeout = 20_000;
-
         reconnectAfterSuspend(true);
     }
 
@@ -1270,8 +1266,6 @@ public void testDuplicateId() throws Exception {
     public void testTimeoutWaitingNodeAddedMessage() throws Exception {
         longSockTimeouts = true;
 
-        clientFailureDetectionTimeout = 20_000;
-
         startServerNodes(2);
 
         final CountDownLatch cnt = new CountDownLatch(1);
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
deleted file mode 100644
index a519d25d4114f..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryClientSuspensionSelfTest.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.tcp;
-
-import java.util.Timer;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Test for missed client metrics update messages.
- */
-public class TcpDiscoveryClientSuspensionSelfTest extends GridCommonAbstractTest {
-    /** */
-    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(gridName);
-
-        TcpDiscoverySpi disco = new TcpDiscoverySpi();
-
-        disco.setIpFinder(IP_FINDER);
-
-        cfg.setDiscoverySpi(disco);
-
-        cfg.setMetricsUpdateFrequency(100);
-
-        cfg.setClientFailureDetectionTimeout(1000);
-
-        return cfg;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        super.beforeTestsStarted();
-
-        System.setProperty(IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY, "10000");
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        super.afterTestsStopped();
-
-        System.clearProperty(IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        stopAllGrids();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testOneServer() throws Exception {
-        doTestClientSuspension(1);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testTwoServers() throws Exception {
-        doTestClientSuspension(2);
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testThreeServers() throws Exception {
-        doTestClientSuspension(3);
-    }
-
-    /**
-     * @param serverCnt Servers count.
-     * @throws Exception If failed.
-     */
-    private void doTestClientSuspension(int serverCnt) throws Exception {
-        startGrids(serverCnt);
-
-        Ignition.setClientMode(true);
-
-        Ignite client = startGrid("client");
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(1, grid(i).cluster().forClients().nodes().size());
-
-        Thread.sleep(2000);
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(1, grid(i).cluster().forClients().nodes().size());
-
-        suspendClientMetricsUpdate(client);
-
-        Thread.sleep(2000);
-
-        for (int i = 0; i < serverCnt; i++)
-            assertEquals(0, grid(i).cluster().forClients().nodes().size());
-    }
-
-    /**
-     * @param client Client.
-     */
-    private void suspendClientMetricsUpdate(Ignite client) {
-        assert client.cluster().localNode().isClient();
-
-        ClientImpl impl = U.field(client.configuration().getDiscoverySpi(), "impl");
-
-        Timer timer = U.field(impl, "timer");
-
-        timer.cancel();
-
-        System.out.println("Metrics update message suspended");
-    }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index cd746319812f0..a2eb3ac2f9e7a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -29,7 +29,6 @@
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiFailureTimeoutSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiMulticastTest;
 import org.apache.ignite.spi.discovery.tcp.TcpClientDiscoverySpiSelfTest;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryClientSuspensionSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMarshallerCheckSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryMultiThreadedTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeAttributesUpdateOnReconnectTest;
@@ -113,7 +112,6 @@ public static TestSuite suite() throws Exception {
         // Client connect.
         suite.addTest(new TestSuite(IgniteClientConnectTest.class));
         suite.addTest(new TestSuite(IgniteClientReconnectMassiveShutdownTest.class));
-        suite.addTest(new TestSuite(TcpDiscoveryClientSuspensionSelfTest.class));
 
         // SSL.
         suite.addTest(new TestSuite(TcpDiscoverySslSelfTest.class));

From 7b5470146d75bf0c0d3ac29b2f0ae27f90d841b8 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Mon, 1 Oct 2018 11:52:42 +0300
Subject: [PATCH 399/543] IGNITE-9736 Fixed usages of Discovery SPI listener. -
 Fixes #4868.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../discovery/GridDiscoveryManager.java       |  7 +-
 .../ignite/spi/discovery/DiscoverySpi.java    |  6 +-
 .../spi/discovery/DiscoverySpiListener.java   |  4 +-
 .../ignite/spi/discovery/tcp/ClientImpl.java  | 14 +--
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 14 +--
 ...eMarshallerCacheClassNameConflictTest.java |  8 +-
 .../IgniteMarshallerCacheFSRestoreTest.java   |  8 +-
 ...iteAbstractStandByClientReconnectTest.java |  6 +-
 .../discovery/AbstractDiscoverySelfTest.java  | 25 +++---
 .../ignite/testframework/GridTestUtils.java   |  3 +-
 .../zk/internal/ZookeeperDiscoveryImpl.java   | 85 +++++++------------
 11 files changed, 72 insertions(+), 108 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 653d33dffcf44..083990394b3c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -97,6 +97,7 @@
 import org.apache.ignite.internal.util.GridSpinBusyLock;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
 import org.apache.ignite.internal.util.lang.GridTuple6;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -584,7 +585,7 @@ private void updateClientNodes(UUID leftNodeId) {
                 }
             }
 
-            @Override public IgniteInternalFuture onDiscovery(
+            @Override public IgniteFuture<?> onDiscovery(
                 final int type,
                 final long topVer,
                 final ClusterNode node,
@@ -592,7 +593,7 @@ private void updateClientNodes(UUID leftNodeId) {
                 final Map<Long, Collection<ClusterNode>> snapshots,
                 @Nullable DiscoverySpiCustomMessage spiCustomMsg
             ) {
-                GridFutureAdapter notificationFut = new GridFutureAdapter();
+                GridFutureAdapter<?> notificationFut = new GridFutureAdapter<>();
 
                 discoNotifierWrk.submit(notificationFut, () -> {
                     synchronized (discoEvtMux) {
@@ -600,7 +601,7 @@ private void updateClientNodes(UUID leftNodeId) {
                     }
                 });
 
-                return notificationFut;
+                return new IgniteFutureImpl<>(notificationFut);
             }
 
             /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpi.java
index 98222a322f785..545e1a043e733 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpi.java
@@ -99,13 +99,11 @@ public interface DiscoverySpi extends IgniteSpi {
      * {@link org.apache.ignite.events.DiscoveryEvent} for a set of all possible
      * discovery events.
      * <p>
-     * Note that as of Ignite 3.0.2 this method is called <b>before</b>
-     * method {@link #spiStart(String)} is called. This is done to
-     * avoid potential window when SPI is started but the listener is
-     * not registered yet.
+     * TODO: This method should be removed from public API in Apache Ignite 3.0
      *
      * @param lsnr Listener to discovery events or {@code null} to unset the listener.
      */
+    @Deprecated
     public void setListener(@Nullable DiscoverySpiListener lsnr);
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiListener.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiListener.java
index 519a235ae103b..db59de07155a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoverySpiListener.java
@@ -21,7 +21,7 @@
 import java.util.Map;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.DiscoveryEvent;
-import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.lang.IgniteFuture;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -52,7 +52,7 @@ public interface DiscoverySpiListener {
      *
      * @return A future that will be completed when notification process has finished.
      */
-    public IgniteInternalFuture onDiscovery(
+    public IgniteFuture<?> onDiscovery(
         int type,
         long topVer,
         ClusterNode node,
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 1a9921cbc31b0..1a3e33837fa42 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -474,12 +474,7 @@ else if (state == DISCONNECTED) {
 
                 Collection<ClusterNode> top = updateTopologyHistory(topVer + 1, null);
 
-                try {
-                    lsnr.onDiscovery(EVT_NODE_FAILED, topVer, n, top, new TreeMap<>(topHist), null).get();
-                }
-                catch (IgniteCheckedException e) {
-                    throw new IgniteException("Failed to wait for discovery listener notification", e);
-                }
+                lsnr.onDiscovery(EVT_NODE_FAILED, topVer, n, top, new TreeMap<>(topHist), null).get();
             }
         }
 
@@ -2541,12 +2536,7 @@ private void notifyDiscovery(
                     debugLog.debug("Discovery notification [node=" + node + ", type=" + U.gridEventName(type) +
                         ", topVer=" + topVer + ']');
 
-                try {
-                    lsnr.onDiscovery(type, topVer, node, top, new TreeMap<>(topHist), data).get();
-                }
-                catch (IgniteCheckedException e) {
-                    throw new IgniteException("Failed to wait for discovery listener notification", e);
-                }
+                lsnr.onDiscovery(type, topVer, node, top, new TreeMap<>(topHist), data).get();
             }
             else if (debugLog.isDebugEnabled())
                 debugLog.debug("Skipped discovery notification [node=" + node + ", type=" + U.gridEventName(type) +
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index a9896df45f1b9..a01b797f9c575 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -70,7 +70,6 @@
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
-import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.IgnitionEx;
@@ -98,6 +97,7 @@
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.internal.util.worker.GridWorkerListener;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.lang.IgniteUuid;
@@ -5526,21 +5526,15 @@ private void notifyDiscoveryListener(TcpDiscoveryCustomEventMessage msg, boolean
                     throw new IgniteException("Failed to unmarshal discovery custom message: " + msg, t);
                 }
 
-                IgniteInternalFuture fut = lsnr.onDiscovery(DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT,
+                IgniteFuture<?> fut = lsnr.onDiscovery(DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT,
                     msg.topologyVersion(),
                     node,
                     snapshot,
                     hist,
                     msgObj);
 
-                if (waitForNotification || msgObj.isMutable()) {
-                    try {
-                        fut.get();
-                    }
-                    catch (IgniteCheckedException e) {
-                        throw new IgniteException("Failed to wait for discovery listener notification", e);
-                    }
-                }
+                if (waitForNotification || msgObj.isMutable())
+                    fut.get();
 
                 if (msgObj.isMutable()) {
                     try {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java
index 64c781741fd51..b3e0e454eb321 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheClassNameConflictTest.java
@@ -31,10 +31,10 @@
 import org.apache.ignite.configuration.BinaryConfiguration;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
-import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -193,7 +193,7 @@ private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate) {
             }
 
             /** {@inheritDoc} */
-            @Override public IgniteInternalFuture onDiscovery(
+            @Override public IgniteFuture<?> onDiscovery(
                     int type,
                     long topVer,
                     ClusterNode node,
@@ -221,7 +221,7 @@ else if (conflClsName.contains(BB.class.getSimpleName()))
                 if (delegate != null)
                     return delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg);
 
-                return new GridFinishedFuture();
+                return new IgniteFinishedFutureImpl<>();
             }
 
             /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java
index 7aa61ebd8f035..47c01dc47f212 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteMarshallerCacheFSRestoreTest.java
@@ -34,11 +34,11 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.PersistentStoreConfiguration;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.processors.marshaller.MappingProposedMessage;
-import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -245,7 +245,7 @@ private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate) {
             }
 
             /** {@inheritDoc} */
-            @Override public IgniteInternalFuture onDiscovery(
+            @Override public IgniteFuture<?> onDiscovery(
                 int type,
                 long topVer,
                 ClusterNode node,
@@ -271,7 +271,7 @@ private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate) {
                 if (delegate != null)
                     return delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg);
 
-                return new GridFinishedFuture();
+                return new IgniteFinishedFutureImpl<>();
             }
 
             /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
index 40fe0f44e9ac7..ff4e638298ef5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
@@ -31,9 +31,9 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.events.EventType;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
@@ -381,7 +381,7 @@ private AwaitDiscoverySpiListener(
         }
 
         /** {@inheritDoc} */
-        @Override public IgniteInternalFuture onDiscovery(
+        @Override public IgniteFuture<?> onDiscovery(
             int type,
             long topVer,
             ClusterNode node,
@@ -389,7 +389,7 @@ private AwaitDiscoverySpiListener(
             @Nullable Map<Long, Collection<ClusterNode>> topHist,
             @Nullable DiscoverySpiCustomMessage data
         ) {
-            IgniteInternalFuture fut = delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, data);
+            IgniteFuture<?> fut = delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, data);
 
             if (type == EVT_CLIENT_NODE_DISCONNECTED) {
                 try {
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/AbstractDiscoverySelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/AbstractDiscoverySelfTest.java
index e59d24a2c5afc..3e0fb894a3b72 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/AbstractDiscoverySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/AbstractDiscoverySelfTest.java
@@ -34,9 +34,9 @@
 import mx4j.tools.adaptor.http.HttpAdaptor;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.util.future.GridFinishedFuture;
+import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.spi.IgniteSpi;
 import org.apache.ignite.spi.IgniteSpiAdapter;
@@ -162,7 +162,7 @@ public boolean isMetricsUpdated() {
         }
 
         /** {@inheritDoc} */
-        @Override public IgniteInternalFuture onDiscovery(
+        @Override public IgniteFuture<?> onDiscovery(
             int type,
             long topVer,
             ClusterNode node,
@@ -172,7 +172,7 @@ public boolean isMetricsUpdated() {
             if (type == EVT_NODE_METRICS_UPDATED)
                 isMetricsUpdate = true;
 
-            return new GridFinishedFuture();
+            return new IgniteFinishedFutureImpl<>();
         }
     }
 
@@ -246,7 +246,7 @@ public void testLocalMetricsUpdate() throws Exception {
                     // No-op.
                 }
 
-                @Override public IgniteInternalFuture onDiscovery(int type, long topVer, ClusterNode node,
+                @Override public IgniteFuture<?> onDiscovery(int type, long topVer, ClusterNode node,
                     Collection<ClusterNode> topSnapshot, Map<Long, Collection<ClusterNode>> topHist,
                     @Nullable DiscoverySpiCustomMessage data) {
                     // If METRICS_UPDATED came from local node
@@ -254,7 +254,7 @@ public void testLocalMetricsUpdate() throws Exception {
                         && node.id().equals(spi.getLocalNode().id()))
                         spiCnt.addAndGet(1);
 
-                    return new GridFinishedFuture();
+                    return new IgniteFinishedFutureImpl<>();
                 }
             };
 
@@ -416,16 +416,21 @@ protected long getMaxMetricsWaitTime() {
                     }
 
                     @SuppressWarnings({"NakedNotify"})
-                    @Override public IgniteInternalFuture onDiscovery(int type, long topVer, ClusterNode node,
-                        Collection<ClusterNode> topSnapshot, Map<Long, Collection<ClusterNode>> topHist,
-                        @Nullable DiscoverySpiCustomMessage data) {
+                    @Override public IgniteFuture<?> onDiscovery(
+                        int type,
+                        long topVer,
+                        ClusterNode node,
+                        Collection<ClusterNode> topSnapshot,
+                        Map<Long, Collection<ClusterNode>> topHist,
+                        @Nullable DiscoverySpiCustomMessage data
+                    ) {
                         info("Discovery event [type=" + type + ", node=" + node + ']');
 
                         synchronized (mux) {
                             mux.notifyAll();
                         }
 
-                        return new GridFinishedFuture();
+                        return new IgniteFinishedFutureImpl<>();
                     }
                 });
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 93a371efb4c2b..5a754dac4ca06 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -96,6 +96,7 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.plugin.extensions.communication.Message;
@@ -156,7 +157,7 @@ private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate, DiscoveryHook
         }
 
         /** {@inheritDoc} */
-        @Override public IgniteInternalFuture onDiscovery(int type, long topVer, ClusterNode node, Collection<ClusterNode> topSnapshot, @Nullable Map<Long, Collection<ClusterNode>> topHist, @Nullable DiscoverySpiCustomMessage spiCustomMsg) {
+        @Override public IgniteFuture<?> onDiscovery(int type, long topVer, ClusterNode node, Collection<ClusterNode> topSnapshot, @Nullable Map<Long, Collection<ClusterNode>> topHist, @Nullable DiscoverySpiCustomMessage spiCustomMsg) {
             hook.handleDiscoveryMessage(spiCustomMsg);
 
             return delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg);
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 5d029b8208f3e..760d0108052ac 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -69,6 +69,7 @@
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.marshaller.MarshallerUtils;
@@ -466,17 +467,12 @@ private void doReconnect(UUID newId) {
         if (rtState.joined) {
             assert rtState.evtsData != null;
 
-            try {
-                lsnr.onDiscovery(EVT_CLIENT_NODE_DISCONNECTED,
-                    rtState.evtsData.topVer,
-                    locNode,
-                    rtState.top.topologySnapshot(),
-                    Collections.emptyMap(),
-                    null).get();
-            }
-            catch (IgniteCheckedException e) {
-                throw new IgniteException("Failed to wait for discovery listener notification", e);
-            }
+            lsnr.onDiscovery(EVT_CLIENT_NODE_DISCONNECTED,
+                rtState.evtsData.topVer,
+                locNode,
+                rtState.top.topologySnapshot(),
+                Collections.emptyMap(),
+                null).get();
         }
 
         try {
@@ -539,17 +535,12 @@ private void notifySegmented() {
         if (nodes.isEmpty())
             nodes = Collections.singletonList(locNode);
 
-        try {
-            lsnr.onDiscovery(EVT_NODE_SEGMENTED,
-                rtState.evtsData != null ? rtState.evtsData.topVer : 1L,
-                locNode,
-                nodes,
-                Collections.emptyMap(),
-                null).get();
-        }
-        catch (IgniteCheckedException e) {
-            throw new IgniteException("Failed to wait for discovery listener notification", e);
-        }
+        lsnr.onDiscovery(EVT_NODE_SEGMENTED,
+            rtState.evtsData != null ? rtState.evtsData.topVer : 1L,
+            locNode,
+            nodes,
+            Collections.emptyMap(),
+            null).get();
     }
 
     /**
@@ -2277,10 +2268,10 @@ private void newClusterStarted(@Nullable ZkDiscoveryEventsData prevEvts) throws
                 Collections.emptyMap(),
                 null).get();
         }
-        catch (IgniteCheckedException e) {
+        catch (IgniteException e) {
             joinFut.onDone(e);
 
-            throw new IgniteException("Failed to wait for discovery listener notification", e);
+            throw new IgniteException("Failed to wait for discovery listener notification on node join", e);
         }
 
         // Reset events (this is also notification for clients left from previous cluster).
@@ -3450,7 +3441,7 @@ private void notifyCustomEvent(final ZkDiscoveryCustomEventData evtData, final D
 
         final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
 
-        IgniteInternalFuture fut = lsnr.onDiscovery(
+        IgniteFuture<?> fut = lsnr.onDiscovery(
             DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT,
             evtData.topologyVersion(),
             sndNode,
@@ -3459,14 +3450,8 @@ private void notifyCustomEvent(final ZkDiscoveryCustomEventData evtData, final D
             msg
         );
 
-        if (msg != null && msg.isMutable()) {
-            try {
-                fut.get();
-            }
-            catch (IgniteCheckedException e) {
-                throw new IgniteException("Failed to wait for discovery listener notification", e);
-            }
-        }
+        if (msg != null && msg.isMutable())
+            fut.get();
     }
 
     /**
@@ -3484,17 +3469,12 @@ private void notifyNodeJoin(ZkJoinedNodeEvtData joinedEvtData, ZkJoiningNodeData
 
         final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
 
-        try {
-            lsnr.onDiscovery(EVT_NODE_JOINED,
-                joinedEvtData.topVer,
-                joinedNode,
-                topSnapshot,
-                Collections.emptyMap(),
-                null).get();
-        }
-        catch (IgniteCheckedException e) {
-            throw new IgniteException("Failed to wait for discovery listener notification", e);
-        }
+        lsnr.onDiscovery(EVT_NODE_JOINED,
+            joinedEvtData.topVer,
+            joinedNode,
+            topSnapshot,
+            Collections.emptyMap(),
+            null).get();
     }
 
     /**
@@ -3520,17 +3500,12 @@ private void notifyNodeFail(long nodeInternalOrder, long topVer) {
 
         final List<ClusterNode> topSnapshot = rtState.top.topologySnapshot();
 
-        try {
-            lsnr.onDiscovery(EVT_NODE_FAILED,
-                topVer,
-                failedNode,
-                topSnapshot,
-                Collections.emptyMap(),
-                null).get();
-        }
-        catch (IgniteCheckedException e) {
-            throw new IgniteException("Failed to wait for discovery listener notification", e);
-        }
+        lsnr.onDiscovery(EVT_NODE_FAILED,
+            topVer,
+            failedNode,
+            topSnapshot,
+            Collections.emptyMap(),
+            null).get();
 
         stats.onNodeFailed();
     }

From e8dd0dd59e065092b068c4ed73591e77aefc7518 Mon Sep 17 00:00:00 2001
From: ascherbakoff <alexey.scherbakoff@gmail.com>
Date: Mon, 1 Oct 2018 14:54:41 +0300
Subject: [PATCH 400/543] IGNITE-9658 Add an option to reuse memory after
 deactivation - Fixes #4874.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 2f9faf1e0d885185348653d6e3b817d32b992261)
(cherry picked from commit 37b5a5477cf827b75cc30b84ade3d6404ec82fd4)
---
 .../jmh/tree/BPlusTreeBenchmark.java          |   2 +-
 .../apache/ignite/IgniteSystemProperties.java |   5 +
 .../internal/mem/DirectMemoryProvider.java    |   6 +-
 .../mem/file/MappedFileMemoryProvider.java    |   6 +-
 .../mem/unsafe/UnsafeMemoryProvider.java      |  27 +++--
 .../ignite/internal/pagemem/PageMemory.java   |  16 ++-
 .../pagemem/impl/PageMemoryNoStoreImpl.java   |   4 +-
 .../GridCacheDatabaseSharedManager.java       |   6 +-
 .../IgniteCacheDatabaseSharedManager.java     | 102 +++++++++++++-----
 .../persistence/pagemem/PageMemoryImpl.java   |   4 +-
 .../impl/PageMemoryNoLoadSelfTest.java        |  10 +-
 .../IgniteClusterActivateDeactivateTest.java  |   5 +-
 ...vateTestWithPersistenceAndMemoryReuse.java |  40 +++++++
 .../pagemem/FullPageIdTableTest.java          |   4 +-
 ...itePageMemReplaceDelayedWriteUnitTest.java |   4 +-
 .../pagemem/PageIdDistributionTest.java       |   4 +-
 .../pagemem/PageMemoryNoStoreLeakTest.java    |   0
 .../wal/memtracker/PageMemoryTracker.java     |   0
 .../database/BPlusTreeSelfTest.java           |   3 +-
 .../database/CacheFreeListImplSelfTest.java   |   2 +-
 .../database/IndexStorageSelfTest.java        |   2 +-
 .../testsuites/IgnitePdsTestSuite4.java       |  37 +++++++
 .../h2/database/InlineIndexHelperTest.java    |   8 +-
 23 files changed, 233 insertions(+), 64 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java

diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java
index 7ed84cbd8d93e..baca50388310c 100644
--- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java
+++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java
@@ -137,7 +137,7 @@ public void setup() throws Exception {
     public void tearDown() throws Exception {
         tree.destroy();
 
-        pageMem.stop();
+        pageMem.stop(true);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index ba98b785ad5fb..ff8d38a094e69 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -942,6 +942,11 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_EVICTION_PERMITS = "IGNITE_EVICTION_PERMITS";
 
+    /**
+     * Try reuse memory on deactivation. Useful in case of huge page memory region size.
+     */
+    public static final String IGNITE_REUSE_MEMORY_ON_DEACTIVATE = "IGNITE_REUSE_MEMORY_ON_DEACTIVATE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/DirectMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/DirectMemoryProvider.java
index a90c6b80b9e33..03d386bdfc326 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/mem/DirectMemoryProvider.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/DirectMemoryProvider.java
@@ -27,9 +27,11 @@ public interface DirectMemoryProvider {
     public void initialize(long[] chunkSizes);
 
     /**
-     * Shuts down the provider. Will deallocate all previously allocated regions.
+     * Shuts down the provider.
+     *
+     * @param deallocate {@code True} to deallocate memory, {@code false} to allow memory reuse.
      */
-    public void shutdown();
+    public void shutdown(boolean deallocate);
 
     /**
      * Attempts to allocate next memory region. Will return {@code null} if no more regions are available.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java
index 7186b27f08118..ec5bc21886789 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java
@@ -30,7 +30,9 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
- *
+ * Memory provider implementation based on memory mapped file.
+ * <p>
+ * Doesn't support memory reuse semantics.
  */
 public class MappedFileMemoryProvider implements DirectMemoryProvider {
     /** */
@@ -93,7 +95,7 @@ public MappedFileMemoryProvider(IgniteLogger log, File allocationPath) {
     }
 
     /** {@inheritDoc} */
-    @Override public void shutdown() {
+    @Override public void shutdown(boolean deallocate) {
         if (mappedFiles != null) {
             for (MappedFile file : mappedFiles) {
                 try {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
index 276e10e1783c9..b5efbd8cfa1dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
@@ -29,7 +29,9 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 /**
- *
+ * Memory provider implementation based on unsafe memory access.
+ * <p>
+ * Supports memory reuse semantics.
  */
 public class UnsafeMemoryProvider implements DirectMemoryProvider {
     /** */
@@ -41,6 +43,9 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider {
     /** */
     private IgniteLogger log;
 
+    /** */
+    private int used = 0;
+
     /**
      * @param log Ignite logger to use.
      */
@@ -56,24 +61,32 @@ public UnsafeMemoryProvider(IgniteLogger log) {
     }
 
     /** {@inheritDoc} */
-    @Override public void shutdown() {
+    @Override public void shutdown(boolean deallocate) {
         if (regions != null) {
             for (Iterator<DirectMemoryRegion> it = regions.iterator(); it.hasNext(); ) {
                 DirectMemoryRegion chunk = it.next();
 
-                GridUnsafe.freeMemory(chunk.address());
+                if (deallocate) {
+                    GridUnsafe.freeMemory(chunk.address());
 
-                // Safety.
-                it.remove();
+                    // Safety.
+                    it.remove();
+                }
             }
+
+            if (!deallocate)
+                used = 0;
         }
     }
 
     /** {@inheritDoc} */
     @Override public DirectMemoryRegion nextRegion() {
-        if (regions.size() == sizes.length)
+        if (used == sizes.length)
             return null;
 
+        if (used < regions.size())
+            return regions.get(used++);
+
         long chunkSize = sizes[regions.size()];
 
         long ptr;
@@ -103,6 +116,8 @@ public UnsafeMemoryProvider(IgniteLogger log) {
 
         regions.add(region);
 
+        used++;
+
         return region;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java
index 6f2e2c9ec36ed..f7391d2d79a9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageMemory.java
@@ -18,11 +18,25 @@
 package org.apache.ignite.internal.pagemem;
 
 import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.lifecycle.LifecycleAware;
 
 /**
  */
-public interface PageMemory extends LifecycleAware, PageIdAllocator, PageSupport {
+public interface PageMemory extends PageIdAllocator, PageSupport {
+    /**
+     * Start page memory.
+     */
+    public void start() throws IgniteException;
+
+    /**
+     * Stop page memory.
+     *
+     * @param deallocate {@code True} to deallocate memory, {@code false} to allow memory reuse on subsequent {@link #start()}
+     * @throws IgniteException
+     */
+    public void stop(boolean deallocate) throws IgniteException;
+
     /**
      * @return Page size in bytes.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index d4b22a6a2b1cc..ae2134e2611c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -238,11 +238,11 @@ public PageMemoryNoStoreImpl(
 
     /** {@inheritDoc} */
     @SuppressWarnings("OverlyStrongTypeCast")
-    @Override public void stop() throws IgniteException {
+    @Override public void stop(boolean deallocate) throws IgniteException {
         if (log.isDebugEnabled())
             log.debug("Stopping page memory.");
 
-        directMemoryProvider.shutdown();
+        directMemoryProvider.shutdown(deallocate);
 
         if (directMemoryProvider instanceof Closeable) {
             try {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 6d97e988e2706..d17d5d08ed120 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -649,7 +649,7 @@ private void readMetastore() throws IgniteCheckedException {
 
             metaStorage = null;
 
-            storePageMem.stop();
+            storePageMem.stop(true);
         }
         catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
@@ -1130,8 +1130,8 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize, long chpBufSi
                 checkPointBufferIdxCnt.set(chunkSizes.length);
             }
 
-            @Override public void shutdown() {
-                memProvider.shutdown();
+            @Override public void shutdown(boolean deallocate) {
+                memProvider.shutdown(deallocate);
             }
 
             @Override public DirectMemoryRegion nextRegion() {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index f03d37fa83a6e..578f7266d855d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -30,6 +30,7 @@
 import org.apache.ignite.DataStorageMetrics;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.configuration.DataPageEvictionMode;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -67,6 +68,7 @@
 import org.apache.ignite.mxbean.DataRegionMetricsMXBean;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_REUSE_MEMORY_ON_DEACTIVATE;
 import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
 import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE;
 
@@ -84,6 +86,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     /** Maximum initial size on 32-bit JVM */
     private static final long MAX_PAGE_MEMORY_INIT_SIZE_32_BIT = 2L * 1024 * 1024 * 1024;
 
+    /** {@code True} to reuse memory on deactive. */
+    private final boolean reuseMemory = IgniteSystemProperties.getBoolean(IGNITE_REUSE_MEMORY_ON_DEACTIVATE);
+
     /** */
     protected volatile Map<String, DataRegion> dataRegionMap;
 
@@ -105,6 +110,10 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     /** Page size from memory configuration, may be set only for fake(standalone) IgniteCacheDataBaseSharedManager */
     private int pageSize;
 
+
+    /** Stores memory providers eligible for reuse. */
+    private Map<String, DirectMemoryProvider> memProviderMap;
+
     /** {@inheritDoc} */
     @Override protected void start0() throws IgniteCheckedException {
         if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getDataStorageConfiguration() == null)
@@ -226,6 +235,7 @@ protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteChe
 
         dataRegionMap = U.newHashMap(3 + dataRegions);
         memMetricsMap = U.newHashMap(3 + dataRegions);
+        memProviderMap = reuseMemory ? U.newHashMap(3 + dataRegions) : null;
 
         if (dataRegionCfgs != null) {
             for (DataRegionConfiguration dataRegionCfg : dataRegionCfgs)
@@ -248,7 +258,6 @@ protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteChe
             CU.isPersistenceEnabled(memCfg)
         );
 
-
         dataRegionsInitialized = true;
     }
 
@@ -656,21 +665,7 @@ public ReuseList reuseList(String memPlcName) {
 
     /** {@inheritDoc} */
     @Override protected void stop0(boolean cancel) {
-        if (dataRegionMap != null) {
-            for (DataRegion memPlc : dataRegionMap.values()) {
-                memPlc.pageMemory().stop();
-
-                memPlc.evictionTracker().stop();
-
-                unregisterMBean(memPlc.memoryMetrics().getName());
-            }
-
-            dataRegionMap.clear();
-
-            dataRegionMap = null;
-
-            dataRegionsInitialized = false;
-        }
+        onDeActivate(true);
     }
 
     /**
@@ -904,17 +899,49 @@ private DataRegion initMemory(
         DataRegionMetricsImpl memMetrics,
         boolean trackable
     ) throws IgniteCheckedException {
+        PageMemory pageMem = createPageMemory(createOrReuseMemoryProvider(plcCfg), memCfg, plcCfg, memMetrics, trackable);
+
+        return new DataRegion(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem));
+    }
+
+    /**
+     * @param plcCfg Policy config.
+     * @return DirectMemoryProvider provider.
+     */
+    private DirectMemoryProvider createOrReuseMemoryProvider(DataRegionConfiguration plcCfg)
+        throws IgniteCheckedException {
+        if (!supportsMemoryReuse(plcCfg))
+            return createMemoryProvider(plcCfg);
+
+        DirectMemoryProvider memProvider = memProviderMap.get(plcCfg.getName());
+
+        if (memProvider == null)
+            memProviderMap.put(plcCfg.getName(), (memProvider = createMemoryProvider(plcCfg)));
+
+        return memProvider;
+    }
+
+    /**
+     * @param plcCfg Policy config.
+     *
+     * @return {@code True} if policy supports memory reuse.
+     */
+    private boolean supportsMemoryReuse(DataRegionConfiguration plcCfg) {
+        return reuseMemory && plcCfg.getSwapPath() == null;
+    }
+
+    /**
+     * @param plcCfg Policy config.
+     * @return DirectMemoryProvider provider.
+     */
+    private DirectMemoryProvider createMemoryProvider(DataRegionConfiguration plcCfg) throws IgniteCheckedException {
         File allocPath = buildAllocPath(plcCfg);
 
-        DirectMemoryProvider memProvider = allocPath == null ?
+        return allocPath == null ?
             new UnsafeMemoryProvider(log) :
             new MappedFileMemoryProvider(
                 log,
                 allocPath);
-
-        PageMemory pageMem = createPageMemory(memProvider, memCfg, plcCfg, memMetrics, trackable);
-
-        return new DataRegion(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem));
     }
 
     /**
@@ -1013,8 +1040,8 @@ protected DirectMemoryProvider wrapMetricsMemoryProvider(
                 memProvider.initialize(chunkSizes);
             }
 
-            @Override public void shutdown() {
-                memProvider.shutdown();
+            @Override public void shutdown(boolean deallocate) {
+                memProvider.shutdown(deallocate);
             }
 
             @Override public DirectMemoryRegion nextRegion() {
@@ -1066,7 +1093,34 @@ protected File buildPath(String path, String consId) throws IgniteCheckedExcepti
 
     /** {@inheritDoc} */
     @Override public void onDeActivate(GridKernalContext kctx) {
-        stop0(false);
+        onDeActivate(!reuseMemory);
+    }
+
+    /**
+     * @param shutdown Shutdown.
+     */
+    private void onDeActivate(boolean shutdown) {
+        if (dataRegionMap != null) {
+            for (DataRegion memPlc : dataRegionMap.values()) {
+                memPlc.pageMemory().stop(shutdown);
+
+                memPlc.evictionTracker().stop();
+
+                unregisterMBean(memPlc.memoryMetrics().getName());
+            }
+
+            dataRegionMap.clear();
+
+            dataRegionMap = null;
+
+            if (shutdown && memProviderMap != null) {
+                memProviderMap.clear();
+
+                memProviderMap = null;
+            }
+
+            dataRegionsInitialized = false;
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index ff2d5b23324ba..ad32469e2dda9 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -387,7 +387,7 @@ else if (throttlingPlc == ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY)
 
     /** {@inheritDoc} */
     @SuppressWarnings("OverlyStrongTypeCast")
-    @Override public void stop() throws IgniteException {
+    @Override public void stop(boolean deallocate) throws IgniteException {
         if (log.isDebugEnabled())
             log.debug("Stopping page memory.");
 
@@ -398,7 +398,7 @@ else if (throttlingPlc == ThrottlingPolicy.CHECKPOINT_BUFFER_ONLY)
                 seg.close();
         }
 
-        directMemoryProvider.shutdown();
+        directMemoryProvider.shutdown(deallocate);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
index 6ad977f486a36..c5988e3549000 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
@@ -96,7 +96,7 @@ public void testPageTearingInner() throws Exception {
             }
         }
         finally {
-            mem.stop();
+            mem.stop(true);
         }
     }
 
@@ -121,7 +121,7 @@ public void testLoadedPagesCount() throws Exception {
             assertEquals(mem.loadedPages(), expPages);
         }
         finally {
-            mem.stop();
+            mem.stop(true);
         }
     }
 
@@ -173,7 +173,7 @@ public void testPageTearingSequential() throws Exception {
             }
         }
         finally {
-            mem.stop();
+            mem.stop(true);
         }
     }
 
@@ -200,7 +200,7 @@ public void testPageHandleDeallocation() throws Exception {
                 assertFalse(handles.add(allocatePage(mem)));
         }
         finally {
-            mem.stop();
+            mem.stop(true);
         }
     }
 
@@ -304,7 +304,7 @@ public void testPageIdRotation() throws Exception {
             }
         }
         finally {
-            mem.stop();
+            mem.stop(true);
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 501732cfab64f..dd09f1e6fb6c0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -120,11 +120,11 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
         DataStorageConfiguration memCfg = new DataStorageConfiguration();
         memCfg.setPageSize(4 * 1024);
         memCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
-            .setMaxSize(300 * 1024 * 1024)
+            .setMaxSize(150L * 1024 * 1024)
             .setPersistenceEnabled(persistenceEnabled()));
 
         memCfg.setDataRegionConfigurations(new DataRegionConfiguration()
-            .setMaxSize(300 * 1024 * 1024)
+            .setMaxSize(150L * 1024 * 1024)
             .setName(NO_PERSISTENCE_REGION)
             .setPersistenceEnabled(false));
 
@@ -132,6 +132,7 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
             memCfg.setWalMode(WALMode.LOG_ONLY);
 
         cfg.setDataStorageConfiguration(memCfg);
+        cfg.setFailureDetectionTimeout(60_000);
 
         if (testSpi) {
             TestRecordingCommunicationSpi spi = new TestRecordingCommunicationSpi();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.java
new file mode 100644
index 0000000000000..48d17545aa766
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.IgniteSystemProperties;
+
+/**
+ *
+ */
+public class IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse extends
+    IgniteClusterActivateDeactivateTestWithPersistence {
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_REUSE_MEMORY_ON_DEACTIVATE, "true");
+
+        super.beforeTest();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_REUSE_MEMORY_ON_DEACTIVATE);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
index 43b27aa439f2c..daaa92cc4c87a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
@@ -101,7 +101,7 @@ public void testRandomOperations() throws Exception {
             }
         }
         finally {
-            prov.shutdown();
+            prov.shutdown(true);
         }
     }
 
@@ -218,7 +218,7 @@ else if (check.size() >= elementsCnt * 2 / 3) {
         finally {
             long msPassed = U.currentTimeMillis() - seed;
             System.err.println("Seed used [" + seed + "] duration ["+ msPassed+ "] ms");
-            prov.shutdown();
+            prov.shutdown(true);
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
index c6f42e16e87f2..ba7ebee3c4ae6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/IgnitePageMemReplaceDelayedWriteUnitTest.java
@@ -120,7 +120,7 @@ public void testReplacementWithDelayCausesLockForRead() throws IgniteCheckedExce
 
         assert totalEvicted.get() > 0;
 
-        memory.stop();
+        memory.stop(true);
     }
 
     /**
@@ -175,7 +175,7 @@ public void testBackwardCompatibilityMode() throws IgniteCheckedException {
 
         assert totalEvicted.get() > 0;
 
-        memory.stop();
+        memory.stop(true);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java
index 1a8aaa455f44c..181a60b29f296 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java
@@ -29,8 +29,6 @@
 import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.FullPageIdTable;
-import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -228,7 +226,7 @@ public void _testRealHistory() throws Exception {
             }
         }
         finally {
-            prov.shutdown();
+            prov.shutdown(true);
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index 1b18364fd2a96..ca2ded1e4ffa0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -200,7 +200,8 @@ protected ReuseList createReuseList(int cacheId, PageMemory pageMem, long rootId
             assertEquals(0, acquiredPages());
         }
         finally {
-            pageMem.stop();
+            if (pageMem != null)
+                pageMem.stop(true);
 
             MAX_PER_PAGE = 0;
             PUT_INC = 1;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java
index 80daff29633fe..f2a95d0e0355f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/CacheFreeListImplSelfTest.java
@@ -70,7 +70,7 @@ public class CacheFreeListImplSelfTest extends GridCommonAbstractTest {
         super.afterTest();
 
         if (pageMem != null)
-            pageMem.stop();
+            pageMem.stop(true);
 
         pageMem = null;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
index 01222eb2595f3..5cbf41bcf8797 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IndexStorageSelfTest.java
@@ -129,7 +129,7 @@ private void metaAllocation() throws Exception {
             }
         }
         finally {
-            mem.stop();
+            mem.stop(true);
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
new file mode 100644
index 0000000000000..1d9bb2408c58c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse;
+
+/**
+ *
+ */
+public class IgnitePdsTestSuite4 extends TestSuite {
+    /**
+     * @return Suite.
+     */
+    public static TestSuite suite() {
+        TestSuite suite = new TestSuite("Ignite Persistent Store Test Suite 4");
+
+        suite.addTestSuite(IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.class);
+
+        return suite;
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java
index 41dd4f18a8bd4..c7330753794aa 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java
@@ -221,7 +221,7 @@ private int putAndCompare(String v1, String v2, int maxSize) throws Exception {
             if (page != 0L)
                 pageMem.releasePage(CACHE_ID, pageId, page);
 
-            pageMem.stop();
+            pageMem.stop(true);
         }
     }
 
@@ -324,7 +324,7 @@ public void testStringTruncate() throws Exception {
         finally {
             if (page != 0L)
                 pageMem.releasePage(CACHE_ID, pageId, page);
-            pageMem.stop();
+            pageMem.stop(true);
         }
     }
 
@@ -371,7 +371,7 @@ public void testBytes() throws Exception {
         finally {
             if (page != 0L)
                 pageMem.releasePage(CACHE_ID, pageId, page);
-            pageMem.stop();
+            pageMem.stop(true);
         }
     }
 
@@ -490,7 +490,7 @@ private void testPutGet(Value v1, Value v2, Value v3) throws Exception {
             if (page != 0L)
                 pageMem.releasePage(CACHE_ID, pageId, page);
 
-            pageMem.stop();
+            pageMem.stop(true);
         }
     }
 

From 51b952199d27544af1e2a6a3c1d0809bcacaf28e Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Mon, 1 Oct 2018 17:39:24 +0300
Subject: [PATCH 401/543] IGNITE-9741 Fix SegmentArchivedStorage and
 SegmentCompressStorage remain `iterrupted` after de-activation occurs before
 activation - Fixes #4878.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit caa4b0a)
---
 .../wal/FileWriteAheadLogManager.java         |  5 +-
 .../IgniteClusterActivateDeactivateTest.java  | 82 ++++++++++++++++---
 2 files changed, 75 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index e70785462d831..b89de2681d03c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -302,7 +302,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
     private final SegmentFileInputFactory segmentFileInputFactory;
 
     /** Holder of actual information of latest manipulation on WAL segments. */
-    private final SegmentAware segmentAware;
+    private volatile SegmentAware segmentAware;
 
     /** Updater for {@link #currHnd}, used for verify there are no concurrent update for current log segment handle */
     private static final AtomicReferenceFieldUpdater<FileWriteAheadLogManager, FileWriteHandle> CURR_HND_UPD =
@@ -395,7 +395,6 @@ public FileWriteAheadLogManager(@NotNull final GridKernalContext ctx) {
         walAutoArchiveAfterInactivity = dsCfg.getWalAutoArchiveAfterInactivity();
         evt = ctx.event();
         failureProcessor = ctx.failure();
-        segmentAware = new SegmentAware(dsCfg.getWalSegments(), dsCfg.isWalCompactionEnabled());
     }
 
     /**
@@ -453,6 +452,8 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
 
             IgniteBiTuple<Long, Long> tup = scanMinMaxArchiveIndices();
 
+            segmentAware = new SegmentAware(dsCfg.getWalSegments(), dsCfg.isWalCompactionEnabled());
+
             segmentAware.lastTruncatedArchiveIdx(tup == null ? -1 : tup.get1() - 1);
 
             long lastAbsArchivedIdx = tup == null ? -1 : tup.get2();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index dd09f1e6fb6c0..ea4a87db9ed6f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -72,6 +72,8 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
 
     /** Non-persistent data region name. */
     private static final String NO_PERSISTENCE_REGION = "no-persistence-region";
+    /** */
+    private static final int DEFAULT_CACHES_COUNT = 2;
 
     /** */
     boolean client;
@@ -122,6 +124,8 @@ public class IgniteClusterActivateDeactivateTest extends GridCommonAbstractTest
         memCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
             .setMaxSize(150L * 1024 * 1024)
             .setPersistenceEnabled(persistenceEnabled()));
+        memCfg.setWalSegments(2);
+        memCfg.setWalSegmentSize(512 * 1024);
 
         memCfg.setDataRegionConfigurations(new DataRegionConfiguration()
             .setMaxSize(150L * 1024 * 1024)
@@ -227,7 +231,7 @@ private void activateSimple(int srvs, int clients, int activateFrom) throws Exce
             assertTrue(ignite(i).cluster().active());
 
         for (int i = 0; i < srvs + clients; i++) {
-            for (int c = 0; c < 2; c++)
+            for (int c = 0; c < DEFAULT_CACHES_COUNT; c++)
                 checkCache(ignite(i), CACHE_NAME_PREFIX + c, true);
 
             checkCache(ignite(i), CU.UTILITY_CACHE_NAME, true);
@@ -239,7 +243,7 @@ private void activateSimple(int srvs, int clients, int activateFrom) throws Exce
 
         startGrid(srvs + clients);
 
-        for (int c = 0; c < 2; c++)
+        for (int c = 0; c < DEFAULT_CACHES_COUNT; c++)
             checkCache(ignite(srvs + clients), CACHE_NAME_PREFIX + c, true);
 
         checkCaches(srvs + clients + 1, CACHES);
@@ -248,18 +252,76 @@ private void activateSimple(int srvs, int clients, int activateFrom) throws Exce
 
         startGrid(srvs + clients + 1);
 
-        for (int c = 0; c < 2; c++)
+        for (int c = 0; c < DEFAULT_CACHES_COUNT; c++)
             checkCache(ignite(srvs + clients + 1), CACHE_NAME_PREFIX + c, false);
 
         checkCaches(srvs + clients + 2, CACHES);
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReActivateSimple_5_Servers_4_Clients_FromClient() throws Exception {
+        reactivateSimple(5, 4, 6);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReActivateSimple_5_Servers_4_Clients_FromServer() throws Exception {
+        reactivateSimple(5, 4, 0);
+    }
+
+    /**
+     * @param srvs Number of servers.
+     * @param clients Number of clients.
+     * @param activateFrom Index of node stating activation.
+     * @throws Exception If failed.
+     */
+    public void reactivateSimple(int srvs, int clients, int activateFrom) throws Exception {
+        activateSimple(srvs, clients, activateFrom);
+
+        rolloverSegmentAtLeastTwice(activateFrom);
+
+        for (int i = 0; i < srvs + clients; i++) {
+            for (int c = 0; c < DEFAULT_CACHES_COUNT; c++)
+                checkCache(ignite(i), CACHE_NAME_PREFIX + c, true);
+
+            checkCache(ignite(i), CU.UTILITY_CACHE_NAME, true);
+        }
+
+        ignite(activateFrom).cluster().active(false);
+
+        ignite(activateFrom).cluster().active(true);
+
+        rolloverSegmentAtLeastTwice(activateFrom);
+
+        for (int i = 0; i < srvs + clients; i++) {
+            for (int c = 0; c < DEFAULT_CACHES_COUNT; c++)
+                checkCache(ignite(i), CACHE_NAME_PREFIX + c, true);
+
+            checkCache(ignite(i), CU.UTILITY_CACHE_NAME, true);
+        }
+    }
+
+    /**
+     * Work directory have 2 segments by default. This method do full circle.
+     */
+    private void rolloverSegmentAtLeastTwice(int activateFrom) {
+        for (int c = 0; c < DEFAULT_CACHES_COUNT; c++) {
+            IgniteCache<Object, Object> cache = ignite(activateFrom).cache(CACHE_NAME_PREFIX + c);
+            //this should be enough including free-,meta- page and etc.
+            for (int i = 0; i < 1000; i++)
+                cache.put(i, i);
+        }
+    }
+
     /**
      * @param nodes Number of nodes.
      * @param caches Number of caches.
      */
     final void checkCaches(int nodes, int caches) {
-        for (int i  = 0; i < nodes; i++) {
+        for (int i = 0; i < nodes; i++) {
             for (int c = 0; c < caches; c++) {
                 IgniteCache<Integer, Integer> cache = ignite(i).cache(CACHE_NAME_PREFIX + c);
 
@@ -322,7 +384,7 @@ private void joinWhileActivate1(final boolean startClient, final boolean withNew
         activeFut.get();
         startFut.get();
 
-        for (int c = 0; c < 2; c++)
+        for (int c = 0; c < DEFAULT_CACHES_COUNT; c++)
             checkCache(ignite(2), CACHE_NAME_PREFIX + c, true);
 
         if (withNewCache) {
@@ -377,7 +439,7 @@ private IgniteInternalFuture<?> startNodesAndBlockStatusChange(int srvs,
         }
 
         if (blockMsgNodes.length == 0)
-            blockMsgNodes = new int[]{1};
+            blockMsgNodes = new int[] {1};
 
         final AffinityTopologyVersion STATE_CHANGE_TOP_VER = new AffinityTopologyVersion(srvs + clients, minorVer);
 
@@ -472,7 +534,7 @@ private void joinWhileDeactivate1(final boolean startClient, final boolean withN
 
         ignite(2).cluster().active(true);
 
-        for (int c = 0; c < 2; c++)
+        for (int c = 0; c < DEFAULT_CACHES_COUNT; c++)
             checkCache(ignite(2), CACHE_NAME_PREFIX + c, true);
 
         if (withNewCache) {
@@ -644,7 +706,7 @@ private void deactivateSimple(int srvs, int clients, int deactivateFrom) throws
         }
 
         for (int i = 0; i < srvs; i++) {
-            for (int c = 0; c < 2; c++)
+            for (int c = 0; c < DEFAULT_CACHES_COUNT; c++)
                 checkCache(ignite(i), CACHE_NAME_PREFIX + c, true);
         }
 
@@ -937,7 +999,7 @@ private void clientReconnectClusterActivated(final boolean transition) throws Ex
     public void testInactiveTopologyChanges() throws Exception {
         testSpi = true;
 
-        testSpiRecord = new Class[]{GridDhtPartitionsSingleMessage.class, GridDhtPartitionsFullMessage.class};
+        testSpiRecord = new Class[] {GridDhtPartitionsSingleMessage.class, GridDhtPartitionsFullMessage.class};
 
         active = false;
 
@@ -1184,7 +1246,7 @@ public void testClusterStateNotWaitForDeactivation() throws Exception {
 
         final int nodes = 2;
 
-        IgniteEx crd = (IgniteEx) startGrids(nodes);
+        IgniteEx crd = (IgniteEx)startGrids(nodes);
 
         crd.cluster().active(true);
 

From 2d43cff7cefd718c2573c031634f3359605168f8 Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Mon, 1 Oct 2018 21:25:31 +0300
Subject: [PATCH 402/543] IGNITE-9658 fix merge

---
 .../cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java      | 0
 .../cache/persistence/wal/memtracker/PageMemoryTracker.java       | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java
deleted file mode 100644
index e69de29bb2d1d..0000000000000
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java
deleted file mode 100644
index e69de29bb2d1d..0000000000000

From 88c119b3dc489b2776fb3c933c94a786b0c0717a Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Tue, 2 Oct 2018 01:36:19 +0300
Subject: [PATCH 403/543] GG-14259
 IgniteClusterActivateDeactivateTest#testActivateFailover3/testDeactivateFailover3
 hangs in 2.5.1-master

---
 .../processors/cache/IgniteClusterActivateDeactivateTest.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index ea4a87db9ed6f..fbfe4e8c95da8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -1193,6 +1193,8 @@ public void testDeactivateFailover3() throws Exception {
      * @throws Exception If failed.
      */
     private void stateChangeFailover3(boolean activate) throws Exception {
+        fail("https://ggsystems.atlassian.net/browse/GG-14259");
+
         testReconnectSpi = true;
 
         startNodesAndBlockStatusChange(4, 0, 0, !activate);

From d07a152eae871cad7ae4b5557f7f22b56f089c33 Mon Sep 17 00:00:00 2001
From: Dmitriy Pavlov <dpavlov@apache.org>
Date: Tue, 17 Jul 2018 13:04:39 +0300
Subject: [PATCH 404/543] IGNITE-9004 Failing all tests with issue link to
 IGNITE-9004: Failed to move temp file during segment creation

(cherry picked from commit e721619acca9261a76cbc8d8c1fdb9987ef38ceb)
---
 .../standbycluster/IgniteChangeGlobalStateTest.java        | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
index 9152ab90319df..70a9251c7b3d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
@@ -40,6 +40,13 @@
  *
  */
 public class IgniteChangeGlobalStateTest extends IgniteChangeGlobalStateAbstractTest {
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-9004");
+
+        super.beforeTest();
+    }
+
     /**
      * @throws Exception if fail.
      */

From 7fae944669ec15142d7e57521e4f7db92ac1c9aa Mon Sep 17 00:00:00 2001
From: Eduard Shangareev <eshangareev@gridgain.com>
Date: Thu, 9 Aug 2018 15:38:22 +0300
Subject: [PATCH 405/543] IGNITE-9004 Do not allow to start two Ignite nodes on
 the same persistence folder - Fixes #4383.

(cherry picked from commit d3566c1458443555c7813f55f9a82fa64f083226)
---
 .../cache/GridCacheSharedContext.java         |  23 +---
 .../GridCacheDatabaseSharedManager.java       |  77 ++++++------
 .../IgniteCacheDatabaseSharedManager.java     |  14 ---
 .../filename/PdsConsistentIdProcessor.java    |  13 +-
 .../service/GridServiceProcessor.java         |   4 +-
 .../IgniteClusterActivateDeactivateTest.java  |  22 ++--
 .../IgniteChangeGlobalStateAbstractTest.java  |   4 +-
 .../IgniteChangeGlobalStateTest.java          |   7 --
 ...IgniteNoParrallelClusterIsAllowedTest.java | 111 ++++++++++++++++++
 .../testsuites/IgniteStandByClusterSuite.java |  16 +--
 .../development/utils/IgniteWalConverter.java |  10 +-
 11 files changed, 183 insertions(+), 118 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteNoParrallelClusterIsAllowedTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index c9cea0efa060f..9655b14ebe647 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -43,8 +43,8 @@
 import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.PartitionsEvictManager;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.jta.CacheJtaManagerAdapter;
@@ -269,26 +269,11 @@ public GridCacheSharedContext(
     public void activate() throws IgniteCheckedException {
         long time = System.currentTimeMillis();
 
-        if (!kernalCtx.clientNode())
-            dbMgr.lock();
-
-        boolean success = false;
-
-        try {
-            for (IgniteChangeGlobalStateSupport mgr : stateAwareMgrs)
-                mgr.onActivate(kernalCtx);
+        for (IgniteChangeGlobalStateSupport mgr : stateAwareMgrs)
+            mgr.onActivate(kernalCtx);
 
-            success = true;
-        }
-        finally {
-            if (!success) {
-                if (!kernalCtx.clientNode())
-                    dbMgr.unLock();
-            }
-
-            if (msgLog.isInfoEnabled())
+        if (msgLog.isInfoEnabled())
                 msgLog.info("Components activation performed in " + (System.currentTimeMillis() - time) + " ms.");
-        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index d17d5d08ed120..92c7c81b44d66 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -472,14 +472,21 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
             if (!U.mkdirs(cpDir))
                 throw new IgniteCheckedException("Could not create directory for checkpoint metadata: " + cpDir);
 
-            cleanupTempCheckpointDirectory();
-
             final FileLockHolder preLocked = kernalCtx.pdsFolderResolver()
-                .resolveFolders()
-                .getLockedFileLockHolder();
+                    .resolveFolders()
+                    .getLockedFileLockHolder();
+
+            fileLockHolder = preLocked == null ?
+                        new FileLockHolder(storeMgr.workDir().getPath(), kernalCtx, log) : preLocked;
 
-            if (preLocked == null)
-                fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), kernalCtx, log);
+            if (log.isDebugEnabled())
+                log.debug("Try to capture file lock [nodeId=" +
+                        cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
+
+            if (!fileLockHolder.isLocked())
+                fileLockHolder.tryLock(lockWaitTime);
+
+            cleanupTempCheckpointDirectory();
 
             persStoreMetrics.wal(cctx.wal());
 
@@ -710,16 +717,10 @@ else if (regCfg.getMaxSize() < 8 * GB)
 
         onKernalStop0(false);
 
-        stop0(false);
+        super.onDeActivate(kctx);
 
         /* Must be here, because after deactivate we can invoke activate and file lock must be already configured */
         stopping = false;
-
-        if (!cctx.localNode().isClient()) {
-            //we replace lock with new instance (only if we're responsible for locking folders)
-            if (fileLockHolder != null)
-                fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), cctx.kernalContext(), log);
-        }
     }
 
     /**
@@ -964,28 +965,6 @@ public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedExce
         return res;
     }
 
-    /** {@inheritDoc} */
-    @Override public void lock() throws IgniteCheckedException {
-        if (fileLockHolder != null) {
-            if (log.isDebugEnabled())
-                log.debug("Try to capture file lock [nodeId=" +
-                    cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
-
-            fileLockHolder.tryLock(lockWaitTime);
-        }
-    }
-
-    /** {@inheritDoc} */
-    @Override public void unLock() {
-        if (fileLockHolder != null) {
-            if (log.isDebugEnabled())
-                log.debug("Release file lock [nodeId=" +
-                    cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
-
-            fileLockHolder.release();
-        }
-    }
-
     /** {@inheritDoc} */
     @Override protected void onKernalStop0(boolean cancel) {
         checkpointLock.writeLock().lock();
@@ -1003,14 +982,22 @@ public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedExce
 
         super.onKernalStop0(cancel);
 
+        unRegistrateMetricsMBean();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void stop0(boolean cancel) {
+        super.stop0(cancel);
+
         if (!cctx.kernalContext().clientNode()) {
-            unLock();
+            if (fileLockHolder != null) {
+                if (log.isDebugEnabled())
+                    log.debug("Release file lock [nodeId=" +
+                            cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
 
-            if (fileLockHolder != null)
                 fileLockHolder.close();
+            }
         }
-
-        unRegistrateMetricsMBean();
     }
 
     /** */
@@ -4213,7 +4200,7 @@ public static class FileLockHolder implements AutoCloseable {
         private RandomAccessFile lockFile;
 
         /** Lock. */
-        private FileLock lock;
+        private volatile FileLock lock;
 
         /** Kernal context to generate Id of locked node in file. */
         @NotNull private GridKernalContext ctx;
@@ -4286,6 +4273,7 @@ public void tryLock(long lockWaitTimeMillis) throws IgniteCheckedException {
                 for (int i = 0; i < lockWaitTimeMillis; i += 1000) {
                     try {
                         lock = ch.tryLock(0, 1, false);
+
                         if (lock != null && lock.isValid()) {
                             writeContent(sb.toString());
 
@@ -4356,13 +4344,20 @@ private String readContent() throws IOException {
             return content;
         }
 
+        /** Locked or not. */
+        public boolean isLocked() {
+            return lock != null && lock.isValid();
+        }
+
         /** Releases file lock */
         public void release() {
             U.releaseQuiet(lock);
         }
 
         /** Closes file channel */
-        public void close() {
+        @Override public void close() {
+            release();
+
             U.closeQuiet(lockFile);
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 578f7266d855d..67205690f84a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -699,20 +699,6 @@ private void unregisterMBean(String name) {
         return true;
     }
 
-    /**
-     *
-     */
-    public void lock() throws IgniteCheckedException {
-
-    }
-
-    /**
-     *
-     */
-    public void unLock() {
-
-    }
-
     /**
      * No-op for non-persistent storage.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
index ba6d82220e2e6..ffef9af7f350a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsConsistentIdProcessor.java
@@ -211,18 +211,12 @@ private PdsFolderSettings prepareNewSettings() throws IgniteCheckedException {
         }
 
         // was not able to find free slot, allocating new
-        final GridCacheDatabaseSharedManager.FileLockHolder rootDirLock = lockRootDirectory(pstStoreBasePath);
-
-        try {
+        try (final GridCacheDatabaseSharedManager.FileLockHolder rootDirLock = lockRootDirectory(pstStoreBasePath)) {
             final List<FolderCandidate> sortedCandidates = getNodeIndexSortedCandidates(pstStoreBasePath);
             final int nodeIdx = sortedCandidates.isEmpty() ? 0 : (sortedCandidates.get(sortedCandidates.size() - 1).nodeIndex() + 1);
 
             return generateAndLockNewDbStorage(pstStoreBasePath, nodeIdx);
         }
-        finally {
-            rootDirLock.release();
-            rootDirLock.close();
-        }
     }
 
     /**
@@ -505,11 +499,10 @@ private FolderCandidate parseFileName(@NotNull final File subFolderFile) {
         if (settings != null) {
             final GridCacheDatabaseSharedManager.FileLockHolder fileLockHolder = settings.getLockedFileLockHolder();
 
-            if (fileLockHolder != null) {
-                fileLockHolder.release();
+            if (fileLockHolder != null)
                 fileLockHolder.close();
-            }
         }
+
         super.stop(cancel);
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
index 1c83118b6f7a9..eed4fd76bef91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/service/GridServiceProcessor.java
@@ -1755,8 +1755,8 @@ private boolean skipExchange(final AffinityTopologyVersion initTopVer) {
                         affReadyFut.get();
                     }
                     catch (IgniteCheckedException e) {
-                        U.error(log, "Failed to wait for affinity ready future " +
-                            "(the assignment will be recalculated anyway)", e);
+                        U.warn(log, "Failed to wait for affinity ready future " +
+                            "(the assignment will be recalculated anyway):" + e.toString());
                     }
                 }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index fbfe4e8c95da8..8415c692e5b71 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -420,11 +420,13 @@ private void joinWhileActivate1(final boolean startClient, final boolean withNew
      * @return State change future.
      * @throws Exception If failed.
      */
-    private IgniteInternalFuture<?> startNodesAndBlockStatusChange(int srvs,
+    private IgniteInternalFuture<?> startNodesAndBlockStatusChange(
+        int srvs,
         int clients,
         final int stateChangeFrom,
         final boolean initiallyActive,
-        int... blockMsgNodes) throws Exception {
+        int... blockMsgNodes
+    ) throws Exception {
         active = initiallyActive;
         testSpi = true;
 
@@ -1201,20 +1203,16 @@ private void stateChangeFailover3(boolean activate) throws Exception {
 
         client = false;
 
-        IgniteInternalFuture startFut1 = GridTestUtils.runAsync(new Callable() {
-            @Override public Object call() throws Exception {
-                startGrid(4);
+        IgniteInternalFuture startFut1 = GridTestUtils.runAsync((Callable) () -> {
+            startGrid(4);
 
-                return null;
-            }
+            return null;
         }, "start-node1");
 
-        IgniteInternalFuture startFut2 = GridTestUtils.runAsync(new Callable() {
-            @Override public Object call() throws Exception {
-                startGrid(5);
+        IgniteInternalFuture startFut2 = GridTestUtils.runAsync((Callable) () -> {
+            startGrid(5);
 
-                return null;
-            }
+            return null;
         }, "start-node2");
 
         U.sleep(1000);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
index c30c8e90f65fd..c628ad656a25f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
@@ -151,7 +151,7 @@ Ignite backUpClient(int idx) {
      * @param cnt Count.
      * @throws Exception If failed.
      */
-    private void startPrimaryNodes(int cnt) throws Exception {
+    void startPrimaryNodes(int cnt) throws Exception {
         for (int i = 0; i < cnt; i++)
             startPrimary(i);
 
@@ -181,7 +181,7 @@ private void startPrimary(int idx) throws Exception {
      * @param cnt Count.
      * @throws Exception If failed.
      */
-    private void startBackUpNodes(int cnt) throws Exception {
+    void startBackUpNodes(int cnt) throws Exception {
         for (int i = 0; i < cnt; i++)
             startBackUp(i);
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
index 70a9251c7b3d1..9152ab90319df 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
@@ -40,13 +40,6 @@
  *
  */
 public class IgniteChangeGlobalStateTest extends IgniteChangeGlobalStateAbstractTest {
-    /** {@inheritDoc} */
-    @Override protected void beforeTest() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-9004");
-
-        super.beforeTest();
-    }
-
     /**
      * @throws Exception if fail.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteNoParrallelClusterIsAllowedTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteNoParrallelClusterIsAllowedTest.java
new file mode 100644
index 0000000000000..5c986eea23cff
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteNoParrallelClusterIsAllowedTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster;
+
+import junit.framework.AssertionFailedError;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+
+/**
+ *
+ */
+public class IgniteNoParrallelClusterIsAllowedTest extends IgniteChangeGlobalStateAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder vmIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testSimple() throws Exception {
+        startPrimaryNodes(primaryNodes());
+
+        tryToStartBackupClusterWhatShouldFail();
+
+        primary(0).cluster().active(true);
+
+        tryToStartBackupClusterWhatShouldFail();
+
+        primary(0).cluster().active(false);
+
+        tryToStartBackupClusterWhatShouldFail();
+
+        primary(0).cluster().active(true);
+
+        tryToStartBackupClusterWhatShouldFail();
+
+        stopAllPrimary();
+
+        startBackUp(backUpNodes());
+
+        stopAllBackUp();
+
+        startPrimaryNodes(primaryNodes());
+
+        tryToStartBackupClusterWhatShouldFail();
+    }
+
+    /**
+     *
+     */
+    private void tryToStartBackupClusterWhatShouldFail() {
+        try {
+            startBackUpNodes(backUpNodes());
+
+            fail();
+        }
+        catch (AssertionFailedError er) {
+                throw er;
+        }
+        catch (Throwable e) {
+            while (true) {
+                String message = e.getMessage();
+
+                if (message.contains("Failed to acquire file lock during"))
+                    break;
+
+                if (e.getCause() != null)
+                    e = e.getCause();
+                else
+                    fail();
+            }
+        }
+    }
+
+    /**
+     *
+     */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), testName(), true));
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     *
+     */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), testName(), true));
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
index fd124b72b1796..f5244207c89af 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
@@ -21,12 +21,9 @@
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheBaselineTopologyTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteBaselineAffinityTopologyActivationTest;
-import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateCacheTest;
 import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateDataStreamerTest;
-import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateDataStructureTest;
 import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateFailOverTest;
-import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateServiceTest;
-import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateTest;
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteNoParrallelClusterIsAllowedTest;
 import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteStandByClusterTest;
 import org.apache.ignite.internal.processors.cache.persistence.standbycluster.join.JoinActiveNodeToActiveCluster;
 import org.apache.ignite.internal.processors.cache.persistence.standbycluster.join.JoinActiveNodeToInActiveCluster;
@@ -65,12 +62,15 @@ public static TestSuite suite() {
         suite.addTestSuite(JoinInActiveNodeToActiveClusterWithPersistence.class);
         suite.addTestSuite(JoinInActiveNodeToInActiveClusterWithPersistence.class);
 
-        suite.addTestSuite(IgniteChangeGlobalStateTest.class);
-        suite.addTestSuite(IgniteChangeGlobalStateCacheTest.class);
-        suite.addTestSuite(IgniteChangeGlobalStateDataStructureTest.class);
+//TODO https://issues.apache.org/jira/browse/IGNITE-9081 suite.addTestSuite(IgniteChangeGlobalStateTest.class);
+//TODO https://issues.apache.org/jira/browse/IGNITE-9081 suite.addTestSuite(IgniteChangeGlobalStateCacheTest.class);
+//TODO https://issues.apache.org/jira/browse/IGNITE-9081 suite.addTestSuite(IgniteChangeGlobalStateDataStructureTest.class);
+//TODO https://issues.apache.org/jira/browse/IGNITE-9081 suite.addTestSuite(IgniteChangeGlobalStateServiceTest.class);
+
         suite.addTestSuite(IgniteChangeGlobalStateDataStreamerTest.class);
         suite.addTestSuite(IgniteChangeGlobalStateFailOverTest.class);
-        suite.addTestSuite(IgniteChangeGlobalStateServiceTest.class);
+
+        suite.addTestSuite(IgniteNoParrallelClusterIsAllowedTest.class);
 
         suite.addTestSuite(CacheBaselineTopologyTest.class);
         suite.addTestSuite(IgniteBaselineAffinityTopologyActivationTest.class);
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
index eee193ab34965..21a0c618bf279 100644
--- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
@@ -51,8 +51,8 @@ public static void main(String[] args) throws Exception {
         H2ExtrasInnerIO.register();
         H2ExtrasLeafIO.register();
 
-        boolean printRecords = IgniteSystemProperties.getBoolean("PRINT_RECORDS", false);
-        boolean printStat = IgniteSystemProperties.getBoolean("PRINT_STAT", true);
+        boolean printRecords = IgniteSystemProperties.getBoolean("PRINT_RECORDS", false); //TODO read them from argumetns
+        boolean printStat = IgniteSystemProperties.getBoolean("PRINT_STAT", true); //TODO read them from argumetns
 
         final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger());
 
@@ -65,7 +65,11 @@ public static void main(String[] args) throws Exception {
 
         @Nullable final WalStat stat = printStat ? new WalStat() : null;
 
-        try (WALIterator stIt = factory.iterator(workFiles)) {
+        IgniteWalIteratorFactory.IteratorParametersBuilder iteratorParametersBuilder =
+                new IgniteWalIteratorFactory.IteratorParametersBuilder().filesOrDirs(workFiles)
+                    .pageSize(Integer.parseInt(args[0]));
+
+        try (WALIterator stIt = factory.iterator(iteratorParametersBuilder)) {
             while (stIt.hasNextX()) {
                 IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
 

From a7d78964474a303d52d2444658c94e5ef849e292 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Tue, 2 Oct 2018 18:26:20 +0300
Subject: [PATCH 406/543] IGNITE-9761 Fixed deadlock in WAL manager - Fixes
 #4890.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit bd07c83)
---
 .../wal/aware/SegmentArchivedStorage.java     |  8 ++--
 .../persistence/wal/aware/SegmentAware.java   |  9 ++++
 .../wal/aware/SegmentLockStorage.java         | 27 +++++-------
 .../wal/aware/SegmentObservable.java          | 10 ++---
 .../wal/aware/SegmentAwareTest.java           | 42 ++++++++++++++++++-
 5 files changed, 70 insertions(+), 26 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
index 1ed607e9b7dd8..c526ae11e6feb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
@@ -63,10 +63,12 @@ long lastArchivedAbsoluteIndex() {
     /**
      * @param lastAbsArchivedIdx New value of last archived segment index.
      */
-    synchronized void setLastArchivedAbsoluteIndex(long lastAbsArchivedIdx) {
-        this.lastAbsArchivedIdx = lastAbsArchivedIdx;
+    void setLastArchivedAbsoluteIndex(long lastAbsArchivedIdx) {
+        synchronized (this) {
+            this.lastAbsArchivedIdx = lastAbsArchivedIdx;
 
-        notifyAll();
+            notifyAll();
+        }
 
         notifyObservers(lastAbsArchivedIdx);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
index 6ba03991e58a0..e46d93f015947 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
@@ -219,6 +219,15 @@ public boolean checkCanReadArchiveOrReserveWorkSegment(long absIdx) {
         return lastArchivedAbsoluteIndex() >= absIdx || segmentLockStorage.lockWorkSegment(absIdx);
     }
 
+    /**
+     * Visible for test.
+     *
+     * @param absIdx Segment absolute index. segment later, use {@link #releaseWorkSegment} for unlock</li> </ul>
+     */
+    void lockWorkSegment(long absIdx) {
+        segmentLockStorage.lockWorkSegment(absIdx);
+    }
+
     /**
      * @param absIdx Segment absolute index.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java
index 2e145e7789f43..f638d4d57be70 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentLockStorage.java
@@ -17,8 +17,8 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
 
-import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 
 /**
@@ -29,7 +29,7 @@ public class SegmentLockStorage extends SegmentObservable {
      * Maps absolute segment index to locks counter. Lock on segment protects from archiving segment and may come from
      * {@link FileWriteAheadLogManager.RecordsIterator} during WAL replay. Map itself is guarded by <code>this</code>.
      */
-    private Map<Long, Integer> locked = new HashMap<>();
+    private Map<Long, Integer> locked = new ConcurrentHashMap<>();
 
     /**
      * Check if WAL segment locked (protected from move to archive)
@@ -37,7 +37,7 @@ public class SegmentLockStorage extends SegmentObservable {
      * @param absIdx Index for check reservation.
      * @return {@code True} if index is locked.
      */
-    public synchronized boolean locked(long absIdx) {
+    public boolean locked(long absIdx) {
         return locked.containsKey(absIdx);
     }
 
@@ -47,12 +47,8 @@ public synchronized boolean locked(long absIdx) {
      * segment later, use {@link #releaseWorkSegment} for unlock</li> </ul>
      */
     @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
-    synchronized boolean lockWorkSegment(long absIdx) {
-        Integer cur = locked.get(absIdx);
-
-        cur = cur == null ? 1 : cur + 1;
-
-        locked.put(absIdx, cur);
+    boolean lockWorkSegment(long absIdx) {
+        locked.compute(absIdx, (idx, count) -> count == null ? 1 : count + 1);
 
         return false;
     }
@@ -61,15 +57,12 @@ synchronized boolean lockWorkSegment(long absIdx) {
      * @param absIdx Segment absolute index.
      */
     @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
-    synchronized void releaseWorkSegment(long absIdx) {
-        Integer cur = locked.get(absIdx);
-
-        assert cur != null && cur >= 1 : "cur=" + cur + ", absIdx=" + absIdx;
+    void releaseWorkSegment(long absIdx) {
+        locked.compute(absIdx, (idx, count) -> {
+            assert count != null && count >= 1 : "cur=" + count + ", absIdx=" + absIdx;
 
-        if (cur == 1)
-            locked.remove(absIdx);
-        else
-            locked.put(absIdx, cur - 1);
+            return count == 1 ? null : count - 1;
+        });
 
         notifyObservers(absIdx);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java
index ba5ad300cd127..3e915044dd01b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentObservable.java
@@ -17,8 +17,8 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
 
-import java.util.ArrayList;
-import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.function.Consumer;
 
 /**
@@ -26,12 +26,12 @@
  */
 public abstract class SegmentObservable {
     /** Observers for handle changes of archived index. */
-    private final List<Consumer<Long>> observers = new ArrayList<>();
+    private final Queue<Consumer<Long>> observers = new ConcurrentLinkedQueue<>();
 
     /**
      * @param observer Observer for notification about segment's changes.
      */
-    synchronized void addObserver(Consumer<Long> observer) {
+    void addObserver(Consumer<Long> observer) {
         observers.add(observer);
     }
 
@@ -40,7 +40,7 @@ synchronized void addObserver(Consumer<Long> observer) {
      *
      * @param segmentId Segment which was been changed.
      */
-    synchronized void notifyObservers(long segmentId) {
+    void notifyObservers(long segmentId) {
         observers.forEach(observer -> observer.accept(segmentId));
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
index 7840b0979e06c..08693564768c6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
@@ -31,6 +31,46 @@
  * Test for {@link SegmentAware}.
  */
 public class SegmentAwareTest extends TestCase {
+
+    /**
+     * Checking to avoid deadlock SegmentArchivedStorage.markAsMovedToArchive -> SegmentLockStorage.locked <->
+     * SegmentLockStorage.releaseWorkSegment -> SegmentArchivedStorage.onSegmentUnlocked
+     *
+     * @throws IgniteCheckedException if failed.
+     */
+    public void testAvoidDeadlockArchiverAndLockStorage() throws IgniteCheckedException {
+        SegmentAware aware = new SegmentAware(10, false);
+
+        int iterationCnt = 100_000;
+        int segmentToHandle = 1;
+
+        IgniteInternalFuture archiverThread = GridTestUtils.runAsync(() -> {
+            int i = iterationCnt;
+
+            while (i-- > 0) {
+                try {
+                    aware.markAsMovedToArchive(segmentToHandle);
+                }
+                catch (IgniteInterruptedCheckedException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        });
+
+        IgniteInternalFuture lockerThread = GridTestUtils.runAsync(() -> {
+            int i = iterationCnt;
+
+            while (i-- > 0) {
+                aware.lockWorkSegment(segmentToHandle);
+
+                aware.releaseWorkSegment(segmentToHandle);
+            }
+        });
+
+        archiverThread.get();
+        lockerThread.get();
+    }
+
     /**
      * Waiting finished when work segment is set.
      */
@@ -435,7 +475,7 @@ public void testFinishWaitSegmentToCompress_WhenInterruptWasCall() throws Ignite
     public void testLastCompressedIdxProperOrdering() throws IgniteInterruptedCheckedException {
         SegmentAware aware = new SegmentAware(10, true);
 
-        for (int i = 0; i < 5 ; i++) {
+        for (int i = 0; i < 5; i++) {
             aware.setLastArchivedAbsoluteIndex(i);
             aware.waitNextSegmentToCompress();
         }

From 099d57cf2838a477f01264a4375165fe1de4df44 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Tue, 2 Oct 2018 19:07:36 +0300
Subject: [PATCH 407/543] IGNITE-9760 Fixed NPE in WAL manager for FSYNC mode -
 Fixes #4888.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit a03c6e9)
---
 .../FsyncModeFileWriteAheadLogManager.java    |  7 ++--
 .../WalRolloverRecordLoggingFsyncTest.java    | 32 +++++++++++++++++++
 .../WalRolloverRecordLoggingLogOnlyTest.java  | 32 +++++++++++++++++++
 .../db/wal/WalRolloverRecordLoggingTest.java  | 16 ++++++----
 .../IgnitePdsWithIndexingCoreTestSuite.java   |  6 ++--
 5 files changed, 83 insertions(+), 10 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingFsyncTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingLogOnlyTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 9ffb16bf45209..79064ff913a18 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -2194,19 +2194,22 @@ public static long writeSerializerVersion(FileIO io, long idx, int version, WALM
     private abstract static class FileHandle {
         /** I/O interface for read/write operations with file */
         protected SegmentIO fileIO;
+        /** Segment idx corresponded to fileIo*/
+        final long segmentIdx;
 
         /**
          * @param fileIO I/O interface for read/write operations of FileHandle.
          */
-        private FileHandle(SegmentIO fileIO) {
+        private FileHandle(@NotNull SegmentIO fileIO) {
             this.fileIO = fileIO;
+            this.segmentIdx = fileIO.getSegmentId();
         }
 
         /**
          * @return Current segment id.
          */
         public long getSegmentId(){
-            return fileIO.getSegmentId();
+            return segmentIdx;
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingFsyncTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingFsyncTest.java
new file mode 100644
index 0000000000000..7454e5f68dfe2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingFsyncTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import org.apache.ignite.configuration.WALMode;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public class WalRolloverRecordLoggingFsyncTest extends WalRolloverRecordLoggingTest {
+
+    /** {@inheritDoc} */
+    @NotNull @Override public WALMode walMode() {
+        return WALMode.FSYNC;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingLogOnlyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingLogOnlyTest.java
new file mode 100644
index 0000000000000..765fdeb5c4556
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingLogOnlyTest.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import org.apache.ignite.configuration.WALMode;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public class WalRolloverRecordLoggingLogOnlyTest extends WalRolloverRecordLoggingTest {
+
+    /** {@inheritDoc} */
+    @NotNull @Override public WALMode walMode() {
+        return WALMode.LOG_ONLY;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java
index 67caf63506641..395b03ae4367c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -36,13 +37,11 @@
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_PATH;
-import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
+import org.jetbrains.annotations.NotNull;
 /**
  *
  */
-public class WalRolloverRecordLoggingTest extends GridCommonAbstractTest {
+public abstract class WalRolloverRecordLoggingTest extends GridCommonAbstractTest {
     /** */
     private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
 
@@ -69,15 +68,20 @@ private RolloverRecord() {
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
                 .setPersistenceEnabled(true)
                 .setMaxSize(40 * 1024 * 1024))
-            .setWalMode(LOG_ONLY)
+            .setWalMode(walMode())
             .setWalSegmentSize(4 * 1024 * 1024)
-            .setWalArchivePath(DFLT_WAL_PATH));
+        );
 
         cfg.setFailureHandler(new StopNodeOrHaltFailureHandler(false, 0));
 
         return cfg;
     }
 
+    /**
+     * @return Wal mode.
+     */
+    @NotNull public abstract WALMode walMode();
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         stopAllGrids();
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index 790f14b1202fa..ac7dd4033d375 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -40,7 +40,8 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryWithCompactionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalPathsTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRecoveryTxLogicalRecordsTest;
-import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRolloverRecordLoggingTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRolloverRecordLoggingFsyncTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRolloverRecordLoggingLogOnlyTest;
 
 /**
  * Test suite for tests that cover core PDS features and depend on indexing module.
@@ -60,7 +61,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(PersistenceDirectoryWarningLoggingTest.class);
         suite.addTestSuite(WalPathsTest.class);
         suite.addTestSuite(WalRecoveryTxLogicalRecordsTest.class);
-        suite.addTestSuite(WalRolloverRecordLoggingTest.class);
+        suite.addTestSuite(WalRolloverRecordLoggingFsyncTest.class);
+        suite.addTestSuite(WalRolloverRecordLoggingLogOnlyTest.class);
 
         suite.addTestSuite(IgniteWalRecoveryTest.class);
         suite.addTestSuite(IgniteWalRecoveryWithCompactionTest.class);

From 45ddb72eb8f649fa1466d3fb4095df037b81cc3e Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Tue, 2 Oct 2018 18:27:24 +0300
Subject: [PATCH 408/543] IGNITE-9084 fix for 2.5.1

---
 .../dht/preloader/GridDhtPartitionSupplyMessageV2.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
index b6bff0e823506..537dbd0fe819a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
@@ -36,7 +36,7 @@ public class GridDhtPartitionSupplyMessageV2 extends GridDhtPartitionSupplyMessa
     private static final long serialVersionUID = 0L;
 
     /** Available since. */
-    public static final IgniteProductVersion AVAILABLE_SINCE = IgniteProductVersion.fromString("2.7.0");
+    public static final IgniteProductVersion AVAILABLE_SINCE = IgniteProductVersion.fromString("2.5.0");
 
     /** Supplying process error. */
     @GridDirectTransient

From c070ee6a95c0ad1d26b9d8fc7e217e5f83b70b54 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Tue, 7 Aug 2018 18:16:38 +0300
Subject: [PATCH 409/543] IGNITE-9213 Muted hanging tests

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit ca973ad99c6112160a305df05be9458e29f88307)
---
 .../processors/cache/CacheSerializableTransactionsTest.java     | 2 ++
 .../cache/distributed/CacheLockReleaseNodeLeaveTest.java        | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheSerializableTransactionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheSerializableTransactionsTest.java
index 9ca73d920a2d2..97ac10360a9b6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheSerializableTransactionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheSerializableTransactionsTest.java
@@ -2805,6 +2805,8 @@ public void testReadWriteTransactionsNoDeadlockMultinode() throws Exception {
      * @throws Exception If failed.
      */
     private void checkReadWriteTransactionsNoDeadlock(final boolean multiNode) throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-9226");
+
         final Ignite ignite0 = ignite(0);
 
         for (final CacheConfiguration<Integer, Integer> ccfg : cacheConfigurations()) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java
index 019e03028632e..852e855c6db92 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLockReleaseNodeLeaveTest.java
@@ -132,6 +132,8 @@ public void testLockRelease() throws Exception {
      * @throws Exception If failed.
      */
     public void testLockTopologyChange() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-9213");
+
         final int nodeCnt = 5;
         int threadCnt = 8;
         final int keys = 100;

From f23bbf62eaf8045fbdb80b11fcf7ec376ad577b4 Mon Sep 17 00:00:00 2001
From: ibessonov <bessonov.ip@gmail.com>
Date: Thu, 4 Oct 2018 11:29:24 +0300
Subject: [PATCH 410/543] IGNITE-9705 Fix flushes invalid byte buffer into
 snapshot manager - Fixes #4841.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit 76d6ef032f9a634f15398d10e0019419f9798296)
---
 .../cache/persistence/GridCacheDatabaseSharedManager.java       | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 92c7c81b44d66..7b37eab11cde6 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1077,6 +1077,8 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize, long chpBufSi
                 // First of all, write page to disk.
                 storeMgr.write(fullId.groupId(), fullId.pageId(), pageBuf, tag);
 
+                pageBuf.rewind();
+
                 // Only after write we can write page into snapshot.
                 snapshotMgr.flushDirtyPageHandler(fullId, pageBuf, tag);
 

From be73786a3bf0e7ac7419639ac995180a584c3cbf Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 4 Oct 2018 12:46:54 +0300
Subject: [PATCH 411/543] IGNITE-9661 Improved performance of partition state
 validation during PME - Fixes #4850.

---
 .../GridDhtPartitionsStateValidator.java      | 31 +++++++++++--------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
index 4ec7e84fc7c63..18a0674826f56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
@@ -115,7 +115,9 @@ public void validatePartitionCountersAndSizes(
 
         Set<Integer> ignore = null;
 
-        for (int p = 0; p < top.partitions(); p++) {
+        for (int i = 0; i < countersMap.size(); i++) {
+            int p = countersMap.partitionAt(i);
+
             if (top.partitionState(nodeId, p) != GridDhtPartitionState.OWNING) {
                 if (ignore == null)
                     ignore = new HashSet<>();
@@ -125,9 +127,8 @@ public void validatePartitionCountersAndSizes(
                 continue;
             }
 
-            int partIdx = countersMap.partitionIndex(p);
-            long updateCounter = partIdx >= 0 ? countersMap.updateCounterAt(partIdx) : 0;
-            long size = sizesMap.containsKey(p) ? sizesMap.get(p) : 0;
+            long updateCounter = countersMap.updateCounterAt(i);
+            long size = sizesMap.getOrDefault(p, 0L);
 
             // Do not validate partitions with zero update counter and size.
             if (updateCounter == 0 && size == 0) {
@@ -182,14 +183,15 @@ public Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
 
             Set<Integer> ignorePartitions = shouldIgnore(top, nodeId, e.getValue());
 
-            for (int part = 0; part < partitions; part++) {
-                if (ignorePartitions != null && ignorePartitions.contains(part))
+            for (int i = 0; i < countersMap.size(); i++) {
+                int p = countersMap.partitionAt(i);
+
+                if (ignorePartitions != null && ignorePartitions.contains(p))
                     continue;
 
-                int partIdx = countersMap.partitionIndex(part);
-                long currentCounter = partIdx >= 0 ? countersMap.updateCounterAt(partIdx) : 0;
+                long currentCounter = countersMap.updateCounterAt(i);
 
-                process(invalidPartitions, updateCountersAndNodesByPartitions, part, nodeId, currentCounter);
+                process(invalidPartitions, updateCountersAndNodesByPartitions, p, nodeId, currentCounter);
             }
         }
 
@@ -233,17 +235,20 @@ public Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
             if (ignoringNodes.contains(nodeId))
                 continue;
 
+            CachePartitionPartialCountersMap countersMap = e.getValue().partitionUpdateCounters(top.groupId(), partitions);
             Map<Integer, Long> sizesMap = e.getValue().partitionSizes(top.groupId());
 
             Set<Integer> ignorePartitions = shouldIgnore(top, nodeId, e.getValue());
 
-            for (int part = 0; part < partitions; part++) {
-                if (ignorePartitions != null && ignorePartitions.contains(part))
+            for (int i = 0; i < countersMap.size(); i++) {
+                int p = countersMap.partitionAt(i);
+
+                if (ignorePartitions != null && ignorePartitions.contains(p))
                     continue;
 
-                long currentSize = sizesMap.containsKey(part) ? sizesMap.get(part) : 0L;
+                long currentSize = sizesMap.getOrDefault(p, 0L);
 
-                process(invalidPartitions, sizesAndNodesByPartitions, part, nodeId, currentSize);
+                process(invalidPartitions, sizesAndNodesByPartitions, p, nodeId, currentSize);
             }
         }
 

From 3989736b1564c0b987178d45b55fc951b11d78da Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Fri, 5 Oct 2018 18:12:52 +0300
Subject: [PATCH 412/543] fix ignite.version

---
 modules/core/src/main/resources/ignite.properties | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index 785f962378054..e70660b1c6a64 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=2.6.0-SNAPSHOT
+ignite.version=2.5.0-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970

From a4b5664e7682dddd1b8c1a42a433eaa5efc42f42 Mon Sep 17 00:00:00 2001
From: tledkov-gridgain <tledkov@gridgain.com>
Date: Thu, 3 May 2018 16:23:40 +0300
Subject: [PATCH 413/543] IGNITE-8347 Test of Memory leaks on restart Ignite
 node with enabled persistence at ThreadLocal. - Fixes #3889.

Signed-off-by: dpavlov <dpavlov@apache.org>
(cherry picked from commit 83b5c0e0ae493247dc9ba0db0e1014f6c38821aa)
---
 .../ignite/internal/util/GridDebug.java       |  69 +++++++++-
 .../MemoryLeaksOnRestartNodeTest.java         | 118 ++++++++++++++++++
 2 files changed, 186 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java
index a8af0fbfcb435..2fa148e19d589 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.util;
 
+import com.sun.management.HotSpotDiagnosticMXBean;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
@@ -32,6 +33,7 @@
 import java.util.Date;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicReference;
+import javax.management.MBeanServer;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -67,7 +69,13 @@ public class GridDebug {
     /** */
     private static boolean allowLog;
 
-    /** */
+    /** This is the name of the HotSpot Diagnostic MBean */
+    private static final String HOTSPOT_BEAN_NAME = "com.sun.management:type=HotSpotDiagnostic";
+
+    /** field to store the hotspot diagnostic MBean */
+    private static volatile HotSpotDiagnosticMXBean hotspotMBean;
+
+    /* */
     static {
         if (LOGS_PATH != null) {
             File log = new File(new File(LOGS_PATH), new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss-").format(new Date()) +
@@ -302,6 +310,65 @@ private static String formatEntry(long ts, String threadName, long threadId, Obj
             Arrays.deepToString(data);
     }
 
+    /**
+     * Call this method from your application whenever you
+     * want to dump the heap snapshot into a file.
+     *
+     * @param fileName name of the heap dump file
+     * @param live flag that tells whether to dump
+     * only the live objects
+     */
+    public static void dumpHeap(String fileName, boolean live) {
+        // initialize hotspot diagnostic MBean
+        initHotspotMBean();
+
+        File f = new File(fileName);
+
+        if (f.exists())
+            f.delete();
+
+        try {
+            hotspotMBean.dumpHeap(fileName, live);
+        }
+        catch (RuntimeException re) {
+            throw re;
+        }
+        catch (Exception exp) {
+            throw new RuntimeException(exp);
+        }
+    }
+
+    /**
+     * Initialize the hotspot diagnostic MBean field
+     */
+    private static void initHotspotMBean() {
+        if (hotspotMBean == null) {
+            synchronized (GridDebug.class) {
+                if (hotspotMBean == null)
+                    hotspotMBean = getHotspotMBean();
+            }
+        }
+    }
+
+    /**
+     * Gets the hotspot diagnostic MBean from the platform MBean server
+     * @return Diagnostic bean.
+     */
+    private static HotSpotDiagnosticMXBean getHotspotMBean() {
+        try {
+            MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+
+            HotSpotDiagnosticMXBean bean = ManagementFactory.newPlatformMXBeanProxy(server,
+                HOTSPOT_BEAN_NAME, HotSpotDiagnosticMXBean.class);
+
+            return bean;
+        } catch (RuntimeException re) {
+            throw re;
+        } catch (Exception exp) {
+            throw new RuntimeException(exp);
+        }
+    }
+
     /**
      * Debug info queue item.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
new file mode 100644
index 0000000000000..56ab091fe2b7e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import java.io.File;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.GridDebug;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests leaks on node restart with enabled persistence.
+ */
+public class MemoryLeaksOnRestartNodeTest extends GridCommonAbstractTest {
+    /** Heap dump file name. */
+    private static final String HEAP_DUMP_FILE_NAME = "test.hprof";
+
+    /** Restarts count. */
+    private static final int RESTARTS = 10;
+
+    /** Nodes count. */
+    private static final int NODES = 3;
+
+    /** Allow 5Mb leaks on node restart. */
+    private static final int ALLOW_LEAK_ON_RESTART_IN_MB = 1;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration().setName("mem0").setPersistenceEnabled(false))
+            .setDataRegionConfigurations(
+                new DataRegionConfiguration().setName("disk").setPersistenceEnabled(true),
+                new DataRegionConfiguration().setName("mem2").setPersistenceEnabled(false)));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        super.afterTestsStopped();
+    }
+
+    /**
+     * @throws Exception On failed.
+     */
+    public void test() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_DELAYED_REPLACED_PAGE_WRITE, "false");
+
+        // Warmup
+        for (int i = 0; i < RESTARTS / 2; ++i) {
+            startGrids(NODES);
+
+            U.sleep(500);
+
+            stopAllGrids();
+        }
+
+        GridDebug.dumpHeap(HEAP_DUMP_FILE_NAME, true);
+
+        File dumpFile = new File(HEAP_DUMP_FILE_NAME);
+
+        final long size0 = dumpFile.length();
+
+        // Restarts
+        for (int i = 0; i < RESTARTS; ++i) {
+            startGrids(NODES);
+
+            U.sleep(500);
+
+            stopAllGrids();
+
+            GridDebug.dumpHeap(HEAP_DUMP_FILE_NAME, true);
+        }
+
+        GridDebug.dumpHeap(HEAP_DUMP_FILE_NAME, true);
+
+        final float leakSize = (float)(dumpFile.length() - size0) / 1024 / 1024 / NODES / RESTARTS;
+
+        assertTrue("Possible leaks detected. The " + leakSize + "M leaks per node restart after " + RESTARTS
+                + " restarts. See the '" + dumpFile.getAbsolutePath() + "'",
+            leakSize < ALLOW_LEAK_ON_RESTART_IN_MB);
+
+        // Remove dump if successful.
+        dumpFile.delete();
+   }
+}
\ No newline at end of file

From 26162a4ca5f76fde0af13c62ecc9af2e7fc1ecce Mon Sep 17 00:00:00 2001
From: dpavlov <dpavlov@apache.org>
Date: Thu, 3 May 2018 18:26:49 +0300
Subject: [PATCH 414/543] IGNITE-8347 Memory leaks on restart Ignite node with
 enabled persistence at ThreadLocal - Fixes #3891.

Signed-off-by: dpavlov <dpavlov@apache.org>
(cherry picked from commit b1db693436e5a48b870e9ea8727023e7784ca23d)
---
 .../pagemem/DelayedDirtyPageWrite.java           |  6 +++---
 .../pagemem/DelayedPageReplacementTracker.java   | 16 +++++++---------
 .../internal/MemoryLeaksOnRestartNodeTest.java   |  3 ---
 3 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedDirtyPageWrite.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedDirtyPageWrite.java
index 6eec609999d3b..b08ddc2f89146 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedDirtyPageWrite.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedDirtyPageWrite.java
@@ -24,9 +24,9 @@
 import org.jetbrains.annotations.Nullable;
 
 /**
- * Not thread safe and stateful class for replacement of one page with write() delay. This allows to write page content
- * without holding segment lock. Page data is copied into temp buffer during {@link #writePage(FullPageId, ByteBuffer,
- * int)} and then sent to real implementation by {@link #finishReplacement()}.
+ * Not thread safe and stateful class for page replacement of one page with write() delay. This allows to write page
+ * content without holding segment lock. Page data is copied into temp buffer during {@link #writePage(FullPageId,
+ * ByteBuffer, int)} and then sent to real implementation by {@link #finishReplacement()}.
  */
 public class DelayedDirtyPageWrite implements ReplacedPageWriter {
     /** Real flush dirty page implementation. */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedPageReplacementTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedPageReplacementTracker.java
index 9cf5b777240b1..aa1b06161c042 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedPageReplacementTracker.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/DelayedPageReplacementTracker.java
@@ -21,6 +21,8 @@
 import java.nio.ByteOrder;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.pagemem.FullPageId;
@@ -56,15 +58,10 @@ public class DelayedPageReplacementTracker {
 
     /**
      * Dirty page write for replacement operations thread local. Because page write {@link DelayedDirtyPageWrite} is
-     * stateful and not thread safe, this thread local protects from GC pressure on pages replacement.
+     * stateful and not thread safe, this thread local protects from GC pressure on pages replacement. <br> Map is used
+     * instead of build-in thread local to allow GC to remove delayed writers for alive threads after node stop.
      */
-    private final ThreadLocal<DelayedDirtyPageWrite> delayedPageWriteThreadLoc
-        = new ThreadLocal<DelayedDirtyPageWrite>() {
-        @Override protected DelayedDirtyPageWrite initialValue() {
-            return new DelayedDirtyPageWrite(flushDirtyPage, byteBufThreadLoc, pageSize,
-                DelayedPageReplacementTracker.this);
-        }
-    };
+    private final Map<Long, DelayedDirtyPageWrite> delayedPageWriteThreadLocMap = new ConcurrentHashMap<>();
 
     /**
      * @param pageSize Page size.
@@ -87,7 +84,8 @@ public DelayedPageReplacementTracker(int pageSize, ReplacedPageWriter flushDirty
      * @return delayed page write implementation, finish method to be called to actually write page.
      */
     public DelayedDirtyPageWrite delayedPageWrite() {
-        return delayedPageWriteThreadLoc.get();
+        return delayedPageWriteThreadLocMap.computeIfAbsent(Thread.currentThread().getId(),
+            id -> new DelayedDirtyPageWrite(flushDirtyPage, byteBufThreadLoc, pageSize, this));
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
index 56ab091fe2b7e..df055fee53ede 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/MemoryLeaksOnRestartNodeTest.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.internal;
 
 import java.io.File;
-import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -76,8 +75,6 @@ public class MemoryLeaksOnRestartNodeTest extends GridCommonAbstractTest {
      * @throws Exception On failed.
      */
     public void test() throws Exception {
-        System.setProperty(IgniteSystemProperties.IGNITE_DELAYED_REPLACED_PAGE_WRITE, "false");
-
         // Warmup
         for (int i = 0; i < RESTARTS / 2; ++i) {
             startGrids(NODES);

From c7f846a1165c4617c40a1f19430f8a57af64af05 Mon Sep 17 00:00:00 2001
From: Maxim Muzafarov <maxmuzaf@gmail.com>
Date: Mon, 21 May 2018 17:28:22 +0300
Subject: [PATCH 415/543] IGNITE-8469 Fix for non-heap memory leak for calling
 cluster activation multi times. - Fixes #3986.

Signed-off-by: dpavlov <dpavlov@apache.org>
(cherry picked from commit 9a4a145be514e650258715a7e682d427d5812d16)
---
 .../mem/file/MappedFileMemoryProvider.java    |  8 ++
 .../mem/unsafe/UnsafeMemoryProvider.java      |  8 ++
 .../pagemem/impl/PageMemoryNoStoreImpl.java   |  3 +-
 .../ignite/internal/util/GridDebug.java       | 56 +++++++++----
 .../pagemem/PageMemoryNoStoreLeakTest.java    | 82 +++++++++++++++++++
 .../ignite/testsuites/IgnitePdsTestSuite.java |  2 +
 6 files changed, 144 insertions(+), 15 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java
index ec5bc21886789..67e86f5ff779c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/file/MappedFileMemoryProvider.java
@@ -57,6 +57,9 @@ public class MappedFileMemoryProvider implements DirectMemoryProvider {
     /** */
     private List<MappedFile> mappedFiles;
 
+    /** Flag shows if current memory provider have been already initialized. */
+    private boolean isInit;
+
     /**
      * @param allocationPath Allocation path.
      */
@@ -67,6 +70,9 @@ public MappedFileMemoryProvider(IgniteLogger log, File allocationPath) {
 
     /** {@inheritDoc} */
     @Override public void initialize(long[] sizes) {
+        if (isInit)
+            throw new IgniteException("Second initialization does not allowed for current provider");
+
         this.sizes = sizes;
 
         mappedFiles = new ArrayList<>(sizes.length);
@@ -92,6 +98,8 @@ public MappedFileMemoryProvider(IgniteLogger log, File allocationPath) {
                         "opened by another process and current user has enough rights): " + file);
             }
         }
+
+        isInit = true;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
index b5efbd8cfa1dc..f9607f5eaccba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
@@ -43,6 +43,9 @@ public class UnsafeMemoryProvider implements DirectMemoryProvider {
     /** */
     private IgniteLogger log;
 
+    /** Flag shows if current memory provider have been already initialized. */
+    private boolean isInit;
+
     /** */
     private int used = 0;
 
@@ -55,9 +58,14 @@ public UnsafeMemoryProvider(IgniteLogger log) {
 
     /** {@inheritDoc} */
     @Override public void initialize(long[] sizes) {
+        if (isInit)
+            throw new IgniteException("Second initialization does not allowed for current provider");
+
         this.sizes = sizes;
 
         regions = new ArrayList<>();
+
+        isInit = true;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index ae2134e2611c5..87448196a53c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -231,7 +231,8 @@ public PageMemoryNoStoreImpl(
         if (lastIdx != SEG_CNT - 1)
             chunks = Arrays.copyOf(chunks, lastIdx + 1);
 
-        directMemoryProvider.initialize(chunks);
+        if (segments == null)
+            directMemoryProvider.initialize(chunks);
 
         addSegment(null);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java
index 2fa148e19d589..d26e0603c5eca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridDebug.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.util;
 
 import com.sun.management.HotSpotDiagnosticMXBean;
+import com.sun.management.OperatingSystemMXBean;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
@@ -34,6 +35,7 @@
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicReference;
 import javax.management.MBeanServer;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -75,6 +77,12 @@ public class GridDebug {
     /** field to store the hotspot diagnostic MBean */
     private static volatile HotSpotDiagnosticMXBean hotspotMBean;
 
+    /** Platform-specific management interface for the operating system. */
+    private static final String OS_BEAN_NAME = "java.lang:type=OperatingSystem";
+
+    /** Call to {@link #initOSMBean()} before accessing. */
+    private static volatile OperatingSystemMXBean osMBean;
+
     /* */
     static {
         if (LOGS_PATH != null) {
@@ -339,33 +347,53 @@ public static void dumpHeap(String fileName, boolean live) {
     }
 
     /**
-     * Initialize the hotspot diagnostic MBean field
+     * @return Committed VM size in bits.
+     */
+    public static long getCommittedVirtualMemorySize() {
+        initOSMBean();
+
+        return osMBean.getCommittedVirtualMemorySize();
+    }
+
+    /**
+     * Initialize the hotspot diagnostic MBean field.
      */
     private static void initHotspotMBean() {
         if (hotspotMBean == null) {
             synchronized (GridDebug.class) {
                 if (hotspotMBean == null)
-                    hotspotMBean = getHotspotMBean();
+                    hotspotMBean = getMBean(HOTSPOT_BEAN_NAME, HotSpotDiagnosticMXBean.class);
             }
         }
     }
 
     /**
-     * Gets the hotspot diagnostic MBean from the platform MBean server
-     * @return Diagnostic bean.
+     * Initialize field to store OperatingSystem MXBean.
      */
-    private static HotSpotDiagnosticMXBean getHotspotMBean() {
-        try {
-            MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+    private static void initOSMBean() {
+        if (osMBean == null) {
+            synchronized (GridDebug.class) {
+                if (osMBean == null)
+                    osMBean = getMBean(OS_BEAN_NAME, OperatingSystemMXBean.class);
+            }
+        }
+    }
 
-            HotSpotDiagnosticMXBean bean = ManagementFactory.newPlatformMXBeanProxy(server,
-                HOTSPOT_BEAN_NAME, HotSpotDiagnosticMXBean.class);
+    /**
+     * Get MXBean from the platform MBeanServer.
+     *
+     * @param mxbeanName The name for uniquely identifying the MXBean within an MBeanServer.
+     * @param mxbeanItf The MXBean interface.
+     * @return A proxy for a platform MXBean interface.
+     */
+    private static <T> T getMBean(String mxbeanName, Class<T> mxbeanItf) {
+        try {
+            MBeanServer srv = ManagementFactory.getPlatformMBeanServer();
 
-            return bean;
-        } catch (RuntimeException re) {
-            throw re;
-        } catch (Exception exp) {
-            throw new RuntimeException(exp);
+            return ManagementFactory.newPlatformMXBeanProxy(srv, mxbeanName, mxbeanItf);
+        }
+        catch (IOException e) {
+            throw new IgniteException(e);
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java
new file mode 100644
index 0000000000000..65e8c361bc3f8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
+import org.apache.ignite.internal.util.typedef.internal.D;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Base scenario for memory leak:
+ * 1. Start topology with client nodes;
+ * 2. Call active(true) for cluster. This activation should fail by some circumstances (e.g. some locks exists);
+ * 3. IgniteCacheDatabaseSharedManager started and onActive called here. Memory allocated;
+ * 4. Call active(true) again. Activation successfull, non heap memory leak introduced;
+ */
+public class PageMemoryNoStoreLeakTest extends GridCommonAbstractTest {
+    /** */
+    private static final int PAGE_SIZE = 4 * 1024;
+
+    /** */
+    private static final int MAX_MEMORY_SIZE = 10 * 1024 * 1024;
+
+    /** Allow delta between GC executions. */
+    private static final int ALLOWED_DELTA = 10 * 1024 * 1024;
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPageDoubleInitMemoryLeak() throws Exception {
+        long initVMsize = D.getCommittedVirtualMemorySize();
+
+        for (int i = 0; i < 1_000; i++) {
+            final DirectMemoryProvider provider = new UnsafeMemoryProvider(log());
+
+            final DataRegionConfiguration plcCfg = new DataRegionConfiguration()
+                .setMaxSize(MAX_MEMORY_SIZE).setInitialSize(MAX_MEMORY_SIZE);
+
+            PageMemory mem = new PageMemoryNoStoreImpl(
+                log(),
+                provider,
+                null,
+                PAGE_SIZE,
+                plcCfg,
+                new DataRegionMetricsImpl(plcCfg),
+                true);
+
+            try {
+                mem.start();
+
+                //Second initialization, introduces leak
+                mem.start();
+            }
+            finally {
+                mem.stop();
+            }
+
+            long committedVMSize = D.getCommittedVirtualMemorySize();
+
+            assertTrue(committedVMSize - initVMsize <= ALLOWED_DELTA);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index 3a421ebdca219..3d3b1af49cabe 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.IndexStoragePageMemoryImplTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplNoLoadTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryNoStoreLeakTest;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteThrottleSmokeTest;
 import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentedRingByteBufferTest;
 import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAwareTest;
@@ -62,6 +63,7 @@ public static TestSuite suite() throws Exception {
 
         // Basic PageMemory tests.
         suite.addTestSuite(PageMemoryImplNoLoadTest.class);
+        suite.addTestSuite(PageMemoryNoStoreLeakTest.class);
         suite.addTestSuite(IndexStoragePageMemoryImplTest.class);
         suite.addTestSuite(PageMemoryImplTest.class);
 

From 2c4bafce14d4621e11da4d69710d693359f5a489 Mon Sep 17 00:00:00 2001
From: Eduard Shangareev <eshangareev@gridgain.com>
Date: Tue, 25 Sep 2018 16:00:08 +0300
Subject: [PATCH 416/543] GG-12765 Snapshots: add COPY command

(cherry picked from commit c0904e122fd86417414e18dce0d5bad8c14eee04)
---
 .../cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java
index 65e8c361bc3f8..8505a3572c62c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryNoStoreLeakTest.java
@@ -71,7 +71,7 @@ public void testPageDoubleInitMemoryLeak() throws Exception {
                 mem.start();
             }
             finally {
-                mem.stop();
+                mem.stop(true);
             }
 
             long committedVMSize = D.getCommittedVirtualMemorySize();

From 617a2a168b60c7bd74fde92cfc7dd69ef48efce8 Mon Sep 17 00:00:00 2001
From: ibessonov <bessonov.ip@gmail.com>
Date: Sat, 15 Sep 2018 00:25:39 +0300
Subject: [PATCH 417/543] IGNITE-7618 Fix synchronously wait for dchange
 cluster state in validate cache operation - Fixes #4733.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit b263dc150ff49c58b525644361dbd21bdc2c9639)
---
 .../cache/distributed/dht/GridDhtTopologyFutureAdapter.java | 5 ++++-
 .../dht/preloader/GridDhtPartitionsExchangeFuture.java      | 6 ++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
index 3c3150a144751..539fef48bda46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
@@ -44,6 +44,9 @@ public abstract class GridDhtTopologyFutureAdapter extends GridFutureAdapter<Aff
     /** Cache groups validation results. */
     protected volatile Map<Integer, CacheValidation> grpValidRes;
 
+    /** Whether or not cluster is active. */
+    protected volatile boolean clusterIsActive = true;
+
     /**
      * @param grp Cache group.
      * @param topNodes Topology nodes.
@@ -80,7 +83,7 @@ protected final CacheValidation validateCacheGroup(CacheGroupContext grp, Collec
         if (err != null)
             return err;
 
-        if (!cctx.shared().kernalContext().state().publicApiActiveState(true))
+        if (!clusterIsActive)
             return new CacheInvalidStateException(
                 "Failed to perform cache operation (cluster is not activated): " + cctx.name());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index b2ccc051cad24..a067e45c0e960 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -82,13 +82,13 @@
 import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
 import org.apache.ignite.internal.processors.cache.StateChangeRequest;
 import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -347,6 +347,8 @@ public GridDhtPartitionsExchangeFuture(
         this.exchActions = exchActions;
         this.affChangeMsg = affChangeMsg;
         this.validator = new GridDhtPartitionsStateValidator(cctx);
+        if (exchActions != null && exchActions.deactivate())
+            this.clusterIsActive = false;
 
         log = cctx.logger(getClass());
         exchLog = cctx.logger(EXCHANGE_LOG);

From a321e35f95094551e9c8aea609622697b15dd776 Mon Sep 17 00:00:00 2001
From: Eduard Shangareev <eshangareev@gridgain.com>
Date: Thu, 11 Oct 2018 10:58:15 +0300
Subject: [PATCH 418/543] IGNITE-9796 NPE if you call array() method on empty
 GridLongList - Fixes #4917.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 447ce47)
---
 .../processors/cluster/BaselineTopology.java  | 14 ++---
 .../ignite/internal/util/GridLongList.java    |  6 +++
 .../junits/GridAbstractTest.java              | 53 +++++++++----------
 .../ignite/util/GridLongListSelfTest.java     | 23 ++++++++
 4 files changed, 63 insertions(+), 33 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java
index fcb7a397e8936..5a7e66aefc7bb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/BaselineTopology.java
@@ -298,13 +298,15 @@ public ClusterNode baselineNode(Object consId) {
      * @return Sorted list of baseline topology nodes.
      */
     public List<ClusterNode> createBaselineView(
-        List<ClusterNode> aliveNodes,
-        @Nullable IgnitePredicate<ClusterNode> nodeFilter)
-    {
+        Collection<ClusterNode> aliveNodes,
+        @Nullable IgnitePredicate<ClusterNode> nodeFilter
+    ) {
         List<ClusterNode> res = new ArrayList<>(nodeMap.size());
 
+        boolean nullNodeFilter = nodeFilter == null;
+
         for (ClusterNode node : aliveNodes) {
-            if (nodeMap.containsKey(node.consistentId()) && (nodeFilter == null || CU.affinityNode(node, nodeFilter)))
+            if (nodeMap.containsKey(node.consistentId()) && (nullNodeFilter || CU.affinityNode(node, nodeFilter)))
                 res.add(node);
         }
 
@@ -316,7 +318,7 @@ public List<ClusterNode> createBaselineView(
         Map<Object, ClusterNode> consIdMap = new HashMap<>();
 
         for (ClusterNode node : aliveNodes) {
-            if (nodeMap.containsKey(node.consistentId()) && (nodeFilter == null || CU.affinityNode(node, nodeFilter)))
+            if (nodeMap.containsKey(node.consistentId()) && (nullNodeFilter || CU.affinityNode(node, nodeFilter)))
                 consIdMap.put(node.consistentId(), node);
         }
 
@@ -326,7 +328,7 @@ public List<ClusterNode> createBaselineView(
             if (!consIdMap.containsKey(consId)) {
                 DetachedClusterNode node = new DetachedClusterNode(consId, e.getValue());
 
-                if (nodeFilter == null || CU.affinityNode(node, nodeFilter))
+                if (nullNodeFilter || CU.affinityNode(node, nodeFilter))
                     consIdMap.put(consId, node);
             }
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java
index 09fb0983d3462..ded0a5d3f6181 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java
@@ -42,6 +42,9 @@ public class GridLongList implements Message, Externalizable {
     /** */
     private static final long serialVersionUID = 0L;
 
+    /** Empty array. */
+    public static final long[] EMPTY_ARRAY = new long[0];
+
     /** */
     private long[] arr;
 
@@ -390,6 +393,9 @@ public int replaceValue(int startIdx, long oldVal, long newVal) {
      * @return Array copy.
      */
     public long[] array() {
+        if (arr == null)
+            return EMPTY_ARRAY;
+
         long[] res = new long[idx];
 
         System.arraycopy(arr, 0, res, 0, idx);
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 669d6bcf06512..05bf92457a1c4 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -17,6 +17,31 @@
 
 package org.apache.ignite.testframework.junits;
 
+import java.io.ObjectStreamException;
+import java.io.Serializable;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.Proxy;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import javax.cache.configuration.Factory;
+import javax.cache.configuration.FactoryBuilder;
 import junit.framework.TestCase;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
@@ -98,32 +123,6 @@
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.support.FileSystemXmlApplicationContext;
 
-import javax.cache.configuration.Factory;
-import javax.cache.configuration.FactoryBuilder;
-import java.io.ObjectStreamException;
-import java.io.Serializable;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.lang.reflect.Proxy;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CLIENT_CACHE_CHANGE_MESSAGE_TIMEOUT;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_DISCO_FAILED_CLIENT_RECONNECT_DELAY;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
@@ -606,7 +605,7 @@ protected void afterTestsStopped() throws Exception {
             info(">>> Starting test class: " + testClassDescription() + " <<<");
 
             if(isSafeTopology())
-                assert G.allGrids().isEmpty() : "Not all Ignite instances stopped before tests execution";
+                assert G.allGrids().isEmpty() : "Not all Ignite instances stopped before tests execution:" +  G.allGrids();
 
             if (startGrid) {
                 IgniteConfiguration cfg = optimize(getConfiguration());
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridLongListSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridLongListSelfTest.java
index 3b62e32f01f06..8849a3d2727d0 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridLongListSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridLongListSelfTest.java
@@ -150,4 +150,27 @@ public void testSort() {
         assertEquals(asList(1, 3, 4, 5, 0), list);
         assertEquals(asList(0, 1, 3, 4, 5), list.sort());
     }
+
+    /**
+     *
+     */
+    public void testArray() {
+        GridLongList list = new GridLongList();
+
+        long[] array = list.array();
+
+        assertNotNull(array);
+
+        assertEquals(0, array.length);
+
+        list.add(1L);
+
+        array = list.array();
+
+        assertNotNull(array);
+
+        assertEquals(1, array.length);
+
+        assertEquals(1L, array[0]);
+    }
 }
\ No newline at end of file

From b3235b541588b6b846cf2d458213c05bb9847cdb Mon Sep 17 00:00:00 2001
From: EdShangGG <eshangareev@gridgain.com>
Date: Thu, 11 Oct 2018 17:56:46 +0300
Subject: [PATCH 419/543] GG-13842 Move partition without backups during
 snapshot MOVE/COPY

---
 .../main/java/org/apache/ignite/Ignition.java |  8 +--
 .../apache/ignite/cluster/ClusterNode.java    | 12 ++---
 .../DefaultCommunicationFailureResolver.java  | 10 ++--
 .../apache/ignite/internal/IgniteKernal.java  |  2 +-
 .../discovery/GridDiscoveryManager.java       | 13 +++--
 .../managers/discovery/IgniteClusterNode.java |  7 ---
 .../cache/CacheAffinitySharedManager.java     |  2 +-
 .../cache/ExchangeDiscoveryEvents.java        |  9 ++--
 .../GridCachePartitionExchangeManager.java    |  6 +--
 .../processors/cache/GridCacheProcessor.java  |  2 +-
 .../processors/cache/GridCacheUtils.java      | 26 +--------
 .../dht/GridDhtTransactionalCacheAdapter.java |  4 +-
 .../GridDhtPartitionsExchangeFuture.java      | 14 ++---
 .../preloader/latch/ExchangeLatchManager.java |  2 +-
 .../cache/transactions/IgniteTxHandler.java   |  2 +-
 .../apache/ignite/spi/IgniteSpiAdapter.java   |  2 +-
 .../tcp/TcpCommunicationSpi.java              |  2 +-
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 22 ++++----
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |  8 ++-
 .../tcp/internal/TcpDiscoveryNode.java        | 24 ++++-----
 .../tcp/internal/TcpDiscoveryNodesRing.java   |  8 +--
 ...ridDiscoveryManagerAttributesSelfTest.java | 53 ++++++++++++++++---
 ...iteTopologyValidatorAbstractCacheTest.java |  2 +-
 .../CacheLateAffinityAssignmentTest.java      |  2 +-
 .../processors/query/h2/IgniteH2Indexing.java |  2 +-
 .../zk/internal/ZookeeperClusterNode.java     |  5 --
 26 files changed, 123 insertions(+), 126 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/Ignition.java b/modules/core/src/main/java/org/apache/ignite/Ignition.java
index 835896e503c2a..64e855a713863 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignition.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignition.java
@@ -141,30 +141,26 @@ public static boolean isDaemon() {
     }
 
     /**
-     * Sets client mode static flag.
+     * Sets client mode thread-local flag.
      * <p>
      * This flag used when node is started if {@link IgniteConfiguration#isClientMode()}
      * is {@code null}. When {@link IgniteConfiguration#isClientMode()} is set this flag is ignored.
-     * It is recommended to use {@link DiscoverySpi} in client mode too.
      *
      * @param clientMode Client mode flag.
      * @see IgniteConfiguration#isClientMode()
-     * @see TcpDiscoverySpi#setForceServerMode(boolean)
      */
     public static void setClientMode(boolean clientMode) {
         IgnitionEx.setClientMode(clientMode);
     }
 
     /**
-     * Gets client mode static flag.
+     * Gets client mode thread-local flag.
      * <p>
      * This flag used when node is started if {@link IgniteConfiguration#isClientMode()}
      * is {@code null}. When {@link IgniteConfiguration#isClientMode()} is set this flag is ignored.
-     * It is recommended to use {@link DiscoverySpi} in client mode too.
      *
      * @return Client mode flag.
      * @see IgniteConfiguration#isClientMode()
-     * @see TcpDiscoverySpi#setForceServerMode(boolean)
      */
     public static boolean isClientMode() {
         return IgnitionEx.isClientMode();
diff --git a/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java b/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
index 97b31653a9e51..bfc6c1813a246 100644
--- a/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/cluster/ClusterNode.java
@@ -245,17 +245,11 @@ public interface ClusterNode extends BaselineNode {
     public boolean isDaemon();
 
     /**
-     * Tests whether or not this node is connected to cluster as a client.
-     * <p>
-     * Do not confuse client in terms of
-     * discovery {@link DiscoverySpi#isClientMode()} and client in terms of cache
-     * {@link IgniteConfiguration#isClientMode()}. Cache clients cannot carry data,
-     * while topology clients connect to topology in a different way.
+     * Whether this node is cache client (see {@link IgniteConfiguration#isClientMode()}).
+     *
+     * @return {@code True if client}.
      *
-     * @return {@code True} if this node is a client node, {@code false} otherwise.
      * @see IgniteConfiguration#isClientMode()
-     * @see Ignition#isClientMode()
-     * @see DiscoverySpi#isClientMode()
      */
     public boolean isClient();
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
index 9ccadf39b2933..d2a508126c9b6 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/DefaultCommunicationFailureResolver.java
@@ -67,11 +67,11 @@ public class DefaultCommunicationFailureResolver implements CommunicationFailure
     @Nullable private ClusterPart findLargestConnectedCluster(CommunicationFailureContext ctx) {
         List<ClusterNode> srvNodes = ctx.topologySnapshot()
             .stream()
-            .filter(node -> !CU.clientNode(node))
+            .filter(node -> !node.isClient())
             .collect(Collectors.toList());
 
         // Exclude client nodes from analysis.
-        ClusterGraph graph = new ClusterGraph(ctx, CU::clientNode);
+        ClusterGraph graph = new ClusterGraph(ctx, ClusterNode::isClient);
 
         List<BitSet> components = graph.findConnectedComponents();
 
@@ -153,7 +153,7 @@ private void keepCluster(CommunicationFailureContext ctx, ClusterPart clusterPar
             ClusterNode node = allNodes.get(idx);
 
             // Client nodes will be processed separately.
-            if (CU.clientNode(node))
+            if (node.isClient())
                 continue;
 
             if (!clusterPart.srvNodesSet.get(idx))
@@ -164,7 +164,7 @@ private void keepCluster(CommunicationFailureContext ctx, ClusterPart clusterPar
         for (int idx = 0; idx < allNodes.size(); idx++) {
             ClusterNode node = allNodes.get(idx);
 
-            if (CU.clientNode(node) && !clusterPart.connectedClients.contains(node))
+            if (node.isClient() && !clusterPart.connectedClients.contains(node))
                 ctx.killNode(node);
         }
     }
@@ -182,7 +182,7 @@ private Set<ClusterNode> findConnectedClients(CommunicationFailureContext ctx, B
         List<ClusterNode> allNodes = ctx.topologySnapshot();
 
         for (ClusterNode node : allNodes) {
-            if (!CU.clientNode(node))
+            if (!node.isClient())
                 continue;
 
             boolean hasConnections = true;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 30a491815c9ca..7ea3271633dc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1116,7 +1116,7 @@ public void start(
                     catch (IgniteNeedReconnectException e) {
                         ClusterNode locNode = ctx.discovery().localNode();
 
-                        assert CU.clientNode(locNode);
+                        assert locNode.isClient();
 
                         if (!locNode.isClient())
                             throw new IgniteCheckedException("Client node in forceServerMode " +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 083990394b3c6..3ce6dd9b664f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -133,6 +133,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.thread.IgniteThread;
 import org.apache.ignite.thread.OomExceptionHandler;
 import org.jetbrains.annotations.NotNull;
@@ -210,7 +211,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
     /** Predicate filtering client nodes. */
     private static final IgnitePredicate<ClusterNode> FILTER_CLI = new P1<ClusterNode>() {
         @Override public boolean apply(ClusterNode n) {
-            return CU.clientNode(n);
+            return n.isClient();
         }
     };
 
@@ -2332,8 +2333,12 @@ public void failNode(UUID nodeId, @Nullable String warning) {
     public boolean reconnectSupported() {
         DiscoverySpi spi = getSpi();
 
-        return ctx.discovery().localNode().isClient() &&
-            (spi instanceof IgniteDiscoverySpi) &&
+        ClusterNode clusterNode = ctx.discovery().localNode();
+
+        boolean client = (clusterNode instanceof TcpDiscoveryNode) ?
+                (((TcpDiscoveryNode) clusterNode).clientRouterNodeId() != null) : clusterNode.isClient();
+
+        return client && (spi instanceof IgniteDiscoverySpi) &&
             ((IgniteDiscoverySpi)spi).clientReconnectSupported();
     }
 
@@ -2393,7 +2398,7 @@ public void reconnect() {
                 if (!node.isLocal())
                     rmtNodes.add(node);
 
-                if (!CU.clientNode(node)) {
+                if (!node.isClient()) {
                     srvNodes.add(node);
 
                     if (minSrvVer == null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java
index cbc706afbe3b3..a143122c88e9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/IgniteClusterNode.java
@@ -59,11 +59,4 @@ public interface IgniteClusterNode extends ClusterNode {
      * @param cacheMetrics Cache metrics.
      */
     public void setCacheMetrics(Map<Integer, CacheMetrics> cacheMetrics);
-
-    /**
-     * Whether this node is cache client (see {@link IgniteConfiguration#isClientMode()}).
-     *
-     * @return {@code True if client}.
-     */
-    public boolean isCacheClient();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 02fc2894efa47..cd141d3b762c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -169,7 +169,7 @@ void onDiscoveryEvent(int type,
             !DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(customMsg))
             return;
 
-        if ((!CU.clientNode(node) && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) ||
+        if ((!node.isClient() && (type == EVT_NODE_FAILED || type == EVT_NODE_JOINED || type == EVT_NODE_LEFT)) ||
             DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(customMsg)) {
             synchronized (mux) {
                 assert lastAffVer == null || topVer.compareTo(lastAffVer) > 0 :
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
index 0e7e01c131f26..2f7753beabc02 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ExchangeDiscoveryEvents.java
@@ -29,7 +29,6 @@
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
@@ -127,7 +126,7 @@ void addEvent(AffinityTopologyVersion topVer, DiscoveryEvent evt, DiscoCache cac
 
         ClusterNode node = evt.eventNode();
 
-        if (!CU.clientNode(node)) {
+        if (!node.isClient()) {
             lastSrvEvt = evt;
 
             srvEvtTopVer = new AffinityTopologyVersion(evt.topologyVersion(), 0);
@@ -135,7 +134,7 @@ void addEvent(AffinityTopologyVersion topVer, DiscoveryEvent evt, DiscoCache cac
             if (evt.type()== EVT_NODE_JOINED)
                 srvJoin = true;
             else if (evt.type() == EVT_NODE_LEFT || evt.type() == EVT_NODE_FAILED)
-                srvLeft = !CU.clientNode(node);
+                srvLeft = !node.isClient();
         }
     }
 
@@ -151,7 +150,7 @@ public List<DiscoveryEvent> events() {
      * @return {@code True} if given event is {@link EventType#EVT_NODE_FAILED} or {@link EventType#EVT_NODE_LEFT}.
      */
     public static boolean serverLeftEvent(DiscoveryEvent evt) {
-        return  ((evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT) && !CU.clientNode(evt.eventNode()));
+        return  ((evt.type() == EVT_NODE_FAILED || evt.type() == EVT_NODE_LEFT) && !evt.eventNode().isClient());
     }
 
     /**
@@ -159,7 +158,7 @@ public static boolean serverLeftEvent(DiscoveryEvent evt) {
      * @return {@code True} if given event is {@link EventType#EVT_NODE_JOINED}.
      */
     public static boolean serverJoinEvent(DiscoveryEvent evt) {
-        return  (evt.type() == EVT_NODE_JOINED && !CU.clientNode(evt.eventNode()));
+        return  (evt.type() == EVT_NODE_JOINED && !evt.eventNode().isClient());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index c57553234f49f..30a05e721078b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -2016,7 +2016,7 @@ public boolean mergeExchanges(final GridDhtPartitionsExchangeFuture curFut, Grid
                         ", mergedFut=" + fut.initialVersion() +
                         ", evt=" + IgniteUtils.gridEventName(fut.firstEvent().type()) +
                         ", evtNode=" + fut.firstEvent().eventNode().id()+
-                        ", evtNodeClient=" + CU.clientNode(fut.firstEvent().eventNode())+ ']');
+                        ", evtNodeClient=" + fut.firstEvent().eventNode().isClient() + ']');
                 }
 
                 DiscoveryEvent evt = fut.firstEvent();
@@ -2107,7 +2107,7 @@ public boolean mergeExchangesOnCoordinator(GridDhtPartitionsExchangeFuture curFu
                             ", mergedFut=" + fut.initialVersion() +
                             ", evt=" + IgniteUtils.gridEventName(fut.firstEvent().type()) +
                             ", evtNode=" + fut.firstEvent().eventNode().id() +
-                            ", evtNodeClient=" + CU.clientNode(fut.firstEvent().eventNode())+ ']');
+                            ", evtNodeClient=" + fut.firstEvent().eventNode().isClient() + ']');
                     }
 
                     addDiscoEvtForTest(fut.firstEvent());
@@ -2413,7 +2413,7 @@ boolean hasPendingServerExchange() {
                         // because only current exchange future can have multiple discovery events (exchange merge).
                         ClusterNode triggeredBy = ((GridDhtPartitionsExchangeFuture) task).firstEvent().eventNode();
 
-                        if (!CU.clientNode(triggeredBy))
+                        if (!triggeredBy.isClient())
                             return true;
                     }
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index bb86d976f7900..62e73b6b151e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3611,7 +3611,7 @@ private IgniteNodeValidationResult validateRestartingCaches(ClusterNode node) {
      * @return Validation result or {@code null} in case of success.
      */
     @Nullable private IgniteNodeValidationResult validateHashIdResolvers(ClusterNode node) {
-        if (!CU.clientNode(node)) {
+        if (!node.isClient()) {
             for (DynamicCacheDescriptor desc : cacheDescriptors().values()) {
                 CacheConfiguration cfg = desc.cacheConfiguration();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
index de674971d1cbc..67b091f0a14d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUtils.java
@@ -1353,37 +1353,13 @@ public static <T extends CachePluginConfiguration> List<T> cachePluginConfigurat
         return res;
     }
 
-    /**
-     * @param node Node.
-     * @return {@code True} if given node is client node (has flag {@link IgniteConfiguration#isClientMode()} set).
-     */
-    public static boolean clientNode(ClusterNode node) {
-        if (node instanceof IgniteClusterNode)
-            return ((IgniteClusterNode)node).isCacheClient();
-        else
-            return clientNodeDirect(node);
-    }
-
-    /**
-     * @param node Node.
-     * @return {@code True} if given node is client node (has flag {@link IgniteConfiguration#isClientMode()} set).
-     */
-    @SuppressWarnings("ConstantConditions")
-    public static boolean clientNodeDirect(ClusterNode node) {
-        Boolean clientModeAttr = node.attribute(IgniteNodeAttributes.ATTR_CLIENT_MODE);
-
-        assert clientModeAttr != null : node;
-
-        return clientModeAttr != null && clientModeAttr;
-    }
-
     /**
      * @param node Node.
      * @param filter Node filter.
      * @return {@code True} if node is not client node and pass given filter.
      */
     public static boolean affinityNode(ClusterNode node, IgnitePredicate<ClusterNode> filter) {
-        return !node.isDaemon() && !clientNode(node) && filter.apply(node);
+        return !node.isDaemon() && !node.isClient() && filter.apply(node);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index 3451e1f51371f..d5d076f8c6206 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -925,7 +925,7 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                 GridDhtPartitionTopology top = null;
 
                 if (req.firstClientRequest()) {
-                    assert CU.clientNode(nearNode);
+                    assert nearNode.isClient();
 
                     top = topology();
 
@@ -1030,7 +1030,7 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                     GridDhtPartitionTopology top = null;
 
                     if (req.firstClientRequest()) {
-                        assert CU.clientNode(nearNode);
+                        assert nearNode.isClient();
 
                         top = topology();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index a067e45c0e960..b598878509d9a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -736,7 +736,7 @@ else if (msg instanceof WalStateAbstractMessage)
                         }
                     }
                     else {
-                        if (CU.clientNode(firstDiscoEvt.eventNode()))
+                        if (firstDiscoEvt.eventNode().isClient())
                             exchange = onClientNodeEvent(crdNode);
                         else
                             exchange = cctx.kernalContext().clientNode() ? ExchangeType.CLIENT : ExchangeType.ALL;
@@ -746,7 +746,7 @@ else if (msg instanceof WalStateAbstractMessage)
                         onLeft();
                 }
                 else {
-                    exchange = CU.clientNode(firstDiscoEvt.eventNode()) ? onClientNodeEvent(crdNode) :
+                    exchange = firstDiscoEvt.eventNode().isClient() ? onClientNodeEvent(crdNode) :
                         onServerNodeEvent(crdNode);
                 }
             }
@@ -1109,7 +1109,7 @@ private ExchangeType onAffinityChangeRequest(boolean crd) throws IgniteCheckedEx
      * @return Exchange type.
      */
     private ExchangeType onClientNodeEvent(boolean crd) throws IgniteCheckedException {
-        assert CU.clientNode(firstDiscoEvt.eventNode()) : this;
+        assert firstDiscoEvt.eventNode().isClient() : this;
 
         if (firstDiscoEvt.type() == EVT_NODE_LEFT || firstDiscoEvt.type() == EVT_NODE_FAILED) {
             onLeft();
@@ -1130,7 +1130,7 @@ private ExchangeType onClientNodeEvent(boolean crd) throws IgniteCheckedExceptio
      * @return Exchange type.
      */
     private ExchangeType onServerNodeEvent(boolean crd) throws IgniteCheckedException {
-        assert !CU.clientNode(firstDiscoEvt.eventNode()) : this;
+        assert !firstDiscoEvt.eventNode().isClient() : this;
 
         if (firstDiscoEvt.type() == EVT_NODE_LEFT || firstDiscoEvt.type() == EVT_NODE_FAILED) {
             onLeft();
@@ -2039,7 +2039,7 @@ private boolean addMergedJoinExchange(ClusterNode node, @Nullable GridDhtPartiti
 
         boolean wait = false;
 
-        if (CU.clientNode(node)) {
+        if (node.isClient()) {
             if (msg != null)
                 waitAndReplyToNode(nodeId, msg);
         }
@@ -2345,7 +2345,7 @@ public void waitAndReplyToNode(final UUID nodeId, final GridDhtPartitionsSingleM
                 }
 
                 if (finishState0 == null) {
-                    assert firstDiscoEvt.type() == EVT_NODE_JOINED && CU.clientNode(firstDiscoEvt.eventNode()) : this;
+                    assert firstDiscoEvt.type() == EVT_NODE_JOINED && firstDiscoEvt.eventNode().isClient() : this;
 
                     ClusterNode node = cctx.node(nodeId);
 
@@ -3316,7 +3316,7 @@ public void onReceiveFullMessage(final ClusterNode node, final GridDhtPartitions
      */
     public void onReceivePartitionRequest(final ClusterNode node, final GridDhtPartitionsSingleRequest msg) {
         assert !cctx.kernalContext().clientNode() || msg.restoreState();
-        assert !node.isDaemon() && !CU.clientNode(node) : node;
+        assert !node.isDaemon() && !node.isClient() : node;
 
         initFut.listen(new CI1<IgniteInternalFuture<Boolean>>() {
             @Override public void apply(IgniteInternalFuture<Boolean> fut) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index b9d01271aef2e..08adc743d90a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -251,7 +251,7 @@ private Collection<ClusterNode> aliveNodesForTopologyVer(AffinityTopologyVersion
             Collection<ClusterNode> histNodes = discovery.topology(topVer.topologyVersion());
 
             if (histNodes != null)
-                return histNodes.stream().filter(n -> !CU.clientNode(n) && !n.isDaemon() && discovery.alive(n))
+                return histNodes.stream().filter(n -> !n.isClient() && !n.isDaemon() && discovery.alive(n))
                         .collect(Collectors.toList());
             else
                 throw new IgniteException("Topology " + topVer + " not found in discovery history "
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index ea8e648d7a8c6..662eae5cdc52c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -364,7 +364,7 @@ public IgniteInternalFuture<GridNearTxPrepareResponse> prepareNearTxLocal(final
 
             if (req.firstClientRequest()) {
                 assert req.concurrency() == OPTIMISTIC : req;
-                assert CU.clientNode(nearNode) : nearNode;
+                assert nearNode.isClient() : nearNode;
 
                 top = firstEntry.context().topology();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index 97cd95ae9cfcb..a7e6e8c72de42 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -519,7 +519,7 @@ private void checkConfigurationConsistency(IgniteSpiContext spiCtx, ClusterNode
         if (!enabled)
             return;
 
-        if (!checkClient && (CU.clientNode(getLocalNode()) || CU.clientNode(node)))
+        if (!checkClient && (getLocalNode().isClient() || node.isClient()))
             return;
 
         String clsAttr = createSpiAttributeName(IgniteNodeAttributes.ATTR_SPI_CLASS);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 3eab09bb24d5b..7f9d0666212b8 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -3509,7 +3509,7 @@ protected void processClientCreationError(
 
         if (!commErrResolve && enableForcibleNodeKill) {
             if (ctx.node(node.id()) != null
-                && (CU.clientNode(node) ||  !CU.clientNode(getLocalNode())) &&
+                && (node.isClient() || !getLocalNode().isClient()) &&
                 connectionError(errs)) {
                 String msg = "TcpCommunicationSpi failed to establish connection to node, node will be dropped from " +
                     "cluster [" + "rmtNode=" + node + ']';
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index a01b797f9c575..c07d0ce3af30d 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -565,7 +565,7 @@ else if (log.isInfoEnabled()) {
         if (log.isInfoEnabled())
             log.info("Finished node ping [nodeId=" + nodeId + ", res=" + res + ", time=" + (end - start) + "ms]");
 
-        if (!res && !node.isClient() && nodeAlive(nodeId)) {
+        if (!res && node.clientRouterNodeId() == null && nodeAlive(nodeId)) {
             LT.warn(log, "Failed to ping node (status check will be initiated): " + nodeId);
 
             msgWorker.addMessage(new TcpDiscoveryStatusCheckMessage(locNode, node.id()));
@@ -588,7 +588,7 @@ private boolean pingNode(TcpDiscoveryNode node) {
 
         UUID clientNodeId = null;
 
-        if (node.isClient()) {
+        if (node.clientRouterNodeId() != null) {
             clientNodeId = node.id();
 
             node = ring.node(node.clientRouterNodeId());
@@ -1973,7 +1973,7 @@ private void cleanIpFinder() {
                         ring.allNodes(),
                         new C1<TcpDiscoveryNode, Collection<InetSocketAddress>>() {
                             @Override public Collection<InetSocketAddress> apply(TcpDiscoveryNode node) {
-                                return !node.isClient() ? spi.getNodeAddresses(node) :
+                                return node.clientRouterNodeId() == null ? spi.getNodeAddresses(node) :
                                     Collections.<InetSocketAddress>emptyList();
                             }
                         }
@@ -2162,7 +2162,7 @@ void add(TcpDiscoveryAbstractMessage msg) {
 
                 TcpDiscoveryNode node = addedMsg.node();
 
-                if (node.isClient() && !msgs.contains(msg)) {
+                if (node.clientRouterNodeId() != null && !msgs.contains(msg)) {
                     Collection<TcpDiscoveryNode> allNodes = ring.allNodes();
 
                     Collection<TcpDiscoveryNode> top = new ArrayList<>(allNodes.size());
@@ -2251,7 +2251,7 @@ private void clearClientAddFinished(UUID clientId) {
         @Nullable Collection<TcpDiscoveryAbstractMessage> messages(@Nullable IgniteUuid lastMsgId,
             TcpDiscoveryNode node)
         {
-            assert node != null && node.isClient() : node;
+            assert node != null && node.clientRouterNodeId() != null : node;
 
             if (lastMsgId == null) {
                 // Client connection failed before it received TcpDiscoveryNodeAddedMessage.
@@ -3617,7 +3617,7 @@ else if (log.isDebugEnabled())
 
                                 authFailedMsg = "Authentication subject is not serializable";
                             }
-                            else if (!node.isClient() &&
+                            else if (node.clientRouterNodeId() == null &&
                                 !subj.systemOperationAllowed(SecurityPermission.JOIN_AS_SERVER))
                                 authFailedMsg = "Node is not authorised to join as a server node";
 
@@ -4026,13 +4026,13 @@ private void nodeCheckError(TcpDiscoveryNode node, String errMsg, String sndMsg)
          */
         private void trySendMessageDirectly(TcpDiscoveryNode node, TcpDiscoveryAbstractMessage msg)
             throws IgniteSpiException {
-            if (node.isClient()) {
+            if (node.clientRouterNodeId() != null) {
                 TcpDiscoveryNode routerNode = ring.node(node.clientRouterNodeId());
 
                 if (routerNode == null)
                     throw new IgniteSpiException("Router node for client does not exist: " + node);
 
-                if (routerNode.isClient())
+                if (routerNode.clientRouterNodeId() != null)
                     throw new IgniteSpiException("Router node is a client node: " + node);
 
                 if (routerNode.id().equals(getLocalNodeId())) {
@@ -4114,7 +4114,7 @@ private void processNodeAddedMessage(TcpDiscoveryNodeAddedMessage msg) {
                     TcpDiscoveryNodeAddFinishedMessage addFinishMsg = new TcpDiscoveryNodeAddFinishedMessage(locNodeId,
                         node.id());
 
-                    if (node.isClient()) {
+                    if (node.clientRouterNodeId() != null) {
                         addFinishMsg.clientDiscoData(msg.gridDiscoveryData());
 
                         addFinishMsg.clientNodeAttributes(node.attributes());
@@ -4502,7 +4502,7 @@ private void processNodeAddFinishedMessage(TcpDiscoveryNodeAddFinishedMessage ms
                     notifyDiscovery(EVT_NODE_JOINED, topVer, node);
 
                 try {
-                    if (spi.ipFinder.isShared() && locNodeCoord && !node.isClient())
+                    if (spi.ipFinder.isShared() && locNodeCoord && node.clientRouterNodeId() == null)
                         spi.ipFinder.registerAddresses(node.socketAddresses());
                 }
                 catch (IgniteSpiException e) {
@@ -6372,7 +6372,7 @@ private void processClientReconnectMessage(TcpDiscoveryClientReconnectMessage ms
 
             TcpDiscoveryNode node = ring.node(nodeId);
 
-            assert node == null || node.isClient();
+            assert node == null || node.clientRouterNodeId() != null;
 
             if (node != null) {
                 node.clientRouterNodeId(msg.routerNodeId());
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 73f53bf430982..365213b6c6082 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -516,7 +516,9 @@ public void dumpDebugInfo() {
      * of {@link IgniteConfiguration#isClientMode()}
      *
      * @return forceServerMode flag.
+     * @deprecated Will be removed at 3.0.
      */
+    @Deprecated
     public boolean isForceServerMode() {
         return forceSrvMode;
     }
@@ -529,8 +531,10 @@ public boolean isForceServerMode() {
      *
      * @param forceSrvMode forceServerMode flag.
      * @return {@code this} for chaining.
+     * @deprecated Will be removed at 3.0.
      */
     @IgniteSpiConfiguration(optional = true)
+    @Deprecated
     public TcpDiscoverySpi setForceServerMode(boolean forceSrvMode) {
         this.forceSrvMode = forceSrvMode;
 
@@ -1931,9 +1935,9 @@ protected void onExchange(DiscoveryDataPacket dataPacket, ClassLoader clsLdr) {
         DiscoveryDataBag dataBag;
 
         if (dataPacket.joiningNodeId().equals(locNode.id()))
-            dataBag = dataPacket.unmarshalGridData(marshaller(), clsLdr, locNode.isClient(), log);
+            dataBag = dataPacket.unmarshalGridData(marshaller(), clsLdr, locNode.clientRouterNodeId() != null, log);
         else
-            dataBag = dataPacket.unmarshalJoiningNodeData(marshaller(), clsLdr, locNode.isClient(), log);
+            dataBag = dataPacket.unmarshalJoiningNodeData(marshaller(), clsLdr, locNode.clientRouterNodeId() != null, log);
 
 
         exchange.onExchange(dataBag);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
index 55fe4e65dc5cf..6c7d8e144020e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
@@ -40,7 +40,6 @@
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -467,7 +466,15 @@ public void visible(boolean visible) {
 
     /** {@inheritDoc} */
     @Override public boolean isClient() {
-        return clientRouterNodeId != null;
+        if (!cacheCliInit) {
+            Boolean clientModeAttr = ((ClusterNode) this).attribute(IgniteNodeAttributes.ATTR_CLIENT_MODE);
+
+            cacheCli = clientModeAttr != null && clientModeAttr;
+
+            cacheCliInit = true;
+        }
+
+        return cacheCli;
     }
 
     /**
@@ -528,17 +535,6 @@ public TcpDiscoveryNode clientReconnectNode(Map<String, Object> nodeAttrs) {
         return node;
     }
 
-    /** {@inheritDoc} */
-    public boolean isCacheClient() {
-        if (!cacheCliInit) {
-            cacheCli = CU.clientNodeDirect(this);
-
-            cacheCliInit = true;
-        }
-
-        return cacheCli;
-    }
-
     /** {@inheritDoc} */
     @Override public int compareTo(@Nullable TcpDiscoveryNode node) {
         if (node == null)
@@ -627,7 +623,7 @@ public boolean isCacheClient() {
         ver = (IgniteProductVersion)in.readObject();
         clientRouterNodeId = U.readUuid(in);
 
-        if (isClient())
+        if (clientRouterNodeId() != null)
             consistentId = consistentIdAttr != null ? consistentIdAttr : id;
         else
             consistentId = consistentIdAttr != null ? consistentIdAttr : U.consistentId(addrs, discPort);
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNodesRing.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNodesRing.java
index 54ddc9ebff686..06cf89e6b968b 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNodesRing.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNodesRing.java
@@ -61,7 +61,9 @@ public class TcpDiscoveryNodesRing {
     /** Client nodes filter. */
     private static final PN CLIENT_NODES = new PN() {
         @Override public boolean apply(ClusterNode node) {
-            return node.isClient();
+            assert node instanceof TcpDiscoveryNode : node;
+
+            return ((TcpDiscoveryNode) node).clientRouterNodeId() != null;
         }
     };
 
@@ -200,7 +202,7 @@ public boolean hasRemoteServerNodes() {
                 return false;
 
             for (TcpDiscoveryNode node : nodes)
-                if (!node.isClient() && !node.id().equals(locNode.id()))
+                if (node.clientRouterNodeId() == null && !node.id().equals(locNode.id()))
                     return true;
 
             return false;
@@ -607,7 +609,7 @@ private Collection<TcpDiscoveryNode> serverNodes(@Nullable final Collection<TcpD
 
         return F.view(nodes, new P1<TcpDiscoveryNode>() {
             @Override public boolean apply(TcpDiscoveryNode node) {
-                return !node.isClient() && (excludedEmpty || !excluded.contains(node));
+                return node.clientRouterNodeId() == null && (excludedEmpty || !excluded.contains(node));
             }
         });
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
index 6ec8046faee87..69f95e8aab1d9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManagerAttributesSelfTest.java
@@ -123,11 +123,15 @@ public void testPreferIpV4StackDifferentValues() throws Exception {
             Ignite g = startGrid(i);
 
             assert "true".equals(g.cluster().localNode().attribute(PREFER_IPV4));
+
+            checkIsClientFlag((IgniteEx) g);
         }
 
         System.setProperty(PREFER_IPV4, "false");
 
-        startGrid(2);
+        IgniteEx g = startGrid(2);
+
+        checkIsClientFlag(g);
     }
 
     /**
@@ -154,12 +158,18 @@ private void doTestUseDefaultSuid(String first, String second, boolean fail) thr
         try {
             System.setProperty(IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID, first);
 
-            startGrid(0);
+            {
+                IgniteEx g = startGrid(0);
+
+                checkIsClientFlag(g);
+            }
 
             System.setProperty(IGNITE_OPTIMIZED_MARSHALLER_USE_DEFAULT_SUID, second);
 
             try {
-                startGrid(1);
+                IgniteEx g = startGrid(1);
+
+                checkIsClientFlag(g);
 
                 if (fail)
                     fail("Node should not join");
@@ -206,7 +216,11 @@ private void doTestUseStrSerVer2(String first, String second, boolean fail) thro
             else
                 System.clearProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2);
 
-            startGrid(0);
+            {
+                IgniteEx g = startGrid(0);
+
+                checkIsClientFlag(g);
+            }
 
             if (second != null)
                 System.setProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2, second);
@@ -214,7 +228,9 @@ private void doTestUseStrSerVer2(String first, String second, boolean fail) thro
                 System.clearProperty(IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2);
 
             try {
-                startGrid(1);
+                IgniteEx g = startGrid(1);
+
+                checkIsClientFlag(g);
 
                 if (fail)
                     fail("Node should not join");
@@ -317,6 +333,8 @@ private void doTestCompatibilityEnabled(String prop, Object first, Object second
 
             IgniteEx ignite = startGrid(0);
 
+            checkIsClientFlag(ignite);
+
             // Ignore if disabled security plugin used.
             if (IGNITE_SECURITY_COMPATIBILITY_MODE.equals(prop) && !ignite.context().security().enabled())
                 return;
@@ -327,7 +345,9 @@ private void doTestCompatibilityEnabled(String prop, Object first, Object second
                 System.clearProperty(prop);
 
             try {
-                startGrid(1);
+                IgniteEx g = startGrid(1);
+
+                checkIsClientFlag(g);
 
                 if (fail)
                     fail("Node must not join");
@@ -351,7 +371,9 @@ private void doTestCompatibilityEnabled(String prop, Object first, Object second
      * @throws Exception If failed.
      */
     public void testDifferentDeploymentModes() throws Exception {
-        startGrid(0);
+        IgniteEx g = startGrid(0);
+
+        checkIsClientFlag(g);
 
         mode = CONTINUOUS;
 
@@ -370,7 +392,9 @@ public void testDifferentDeploymentModes() throws Exception {
      * @throws Exception If failed.
      */
     public void testDifferentPeerClassLoadingEnabledFlag() throws Exception {
-        startGrid(0);
+        IgniteEx g = startGrid(0);
+
+        checkIsClientFlag(g);
 
         p2pEnabled = true;
 
@@ -398,9 +422,22 @@ private void testPreferIpV4Stack(boolean preferIpV4) throws Exception {
             Ignite g = startGrid(i);
 
             assert val.equals(g.cluster().localNode().attribute(PREFER_IPV4));
+
+            checkIsClientFlag((IgniteEx) g);
         }
     }
 
+    /**
+     *
+     * @param g
+     */
+    protected void checkIsClientFlag(IgniteEx g) {
+        boolean isClientDiscovery = g.context().discovery().localNode().isClient();
+        boolean isClientConfig = g.configuration().isClientMode() == null ? false : g.configuration().isClientMode();
+
+        assertEquals(isClientConfig, isClientDiscovery);
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractCacheTest.java
index 9860199dd416f..e2a4a08e7e8d4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTopologyValidatorAbstractCacheTest.java
@@ -98,7 +98,7 @@ private static int servers(Collection<ClusterNode> nodes) {
         int c = 0;
 
         for (ClusterNode node : nodes) {
-            if (!CU.clientNode(node))
+            if (!node.isClient())
                 c++;
         }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
index 0b6fd83c60c26..33a110891cf2b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheLateAffinityAssignmentTest.java
@@ -2852,7 +2852,7 @@ private boolean calculateAffinity(long topVer,
             IgnitePredicate<ClusterNode> filter = cacheDesc.cacheConfiguration().getNodeFilter();
 
             for (ClusterNode n : allNodes) {
-                if (!CU.clientNode(n) && (filter == null || filter.apply(n)))
+                if (!n.isClient() && (filter == null || filter.apply(n)))
                     affNodes.add(n);
             }
 
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 090975c3b8c2a..fb02bd0abc92e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -2932,7 +2932,7 @@ public boolean serverTopologyChanged(AffinityTopologyVersion readyVer) {
 
         AffinityTopologyVersion initVer = fut.initialVersion();
 
-        return initVer.compareTo(readyVer) > 0 && !CU.clientNode(fut.firstEvent().node());
+        return initVer.compareTo(readyVer) > 0 && !fut.firstEvent().node().isClient();
     }
 
     /**
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
index 1c2a589d934dc..f90bc01dd124d 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
@@ -168,11 +168,6 @@ public void setConsistentId(Serializable consistentId) {
         attrs = Collections.unmodifiableMap(map);
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean isCacheClient() {
-        return isClient();
-    }
-
     /** {@inheritDoc} */
     @Nullable @Override public <T> T attribute(String name) {
         // Even though discovery SPI removes this attribute after authentication, keep this check for safety.

From 3357605ef665ec28f94b11719d55c8389b754be7 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 12 Oct 2018 14:06:06 +0700
Subject: [PATCH 420/543] IGNITE-9863 Refactored VisorDataTransferObject to
 IgniteDataTransferObject.  Deprecated VisorDataTransferObject class.  Minor
 code cleanup.

(cherry picked from commit 789046ba49d61e6b58c89555b8d9b419f00052a1)
---
 .../dto/IgniteDataTransferObject.java         | 130 +++++++++++++++
 .../dto/IgniteDataTransferObjectInput.java    | 156 ++++++++++++++++++
 .../dto/IgniteDataTransferObjectOutput.java   | 141 ++++++++++++++++
 .../visor/VisorDataTransferObject.java        |   5 +-
 .../visor/VisorDataTransferObjectInput.java   |   2 +
 .../visor/VisorDataTransferObjectOutput.java  |   2 +
 .../visor/debug/VisorThreadMonitorInfo.java   |   5 -
 .../visor/event/VisorGridDeploymentEvent.java |   5 -
 .../visor/event/VisorGridDiscoveryEvent.java  |   5 -
 .../visor/event/VisorGridJobEvent.java        |   5 -
 .../visor/event/VisorGridTaskEvent.java       |   5 -
 11 files changed, 435 insertions(+), 26 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObject.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectInput.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectOutput.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObject.java b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObject.java
new file mode 100644
index 0000000000000..3441742dfabd6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObject.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.dto;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Base class for data transfer objects.
+ */
+public abstract class IgniteDataTransferObject implements Externalizable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Magic number to detect correct transfer objects. */
+    private static final int MAGIC = 0x42BEEF00;
+
+    /** Version 1. */
+    protected static final byte V1 = 1;
+
+    /** Version 2. */
+    protected static final byte V2 = 2;
+
+    /** Version 3. */
+    protected static final byte V3 = 3;
+
+    /** Version 4. */
+    protected static final byte V4 = 4;
+
+    /** Version 5. */
+    protected static final byte V5 = 5;
+
+    /**
+     * @param col Source collection.
+     * @param <T> Collection type.
+     * @return List based on passed collection.
+     */
+    @Nullable protected static <T> List<T> toList(Collection<T> col) {
+        if (col != null)
+            return new ArrayList<>(col);
+
+        return null;
+    }
+
+    /**
+     * @param col Source collection.
+     * @param <T> Collection type.
+     * @return List based on passed collection.
+     */
+    @Nullable protected static <T> Set<T> toSet(Collection<T> col) {
+        if (col != null)
+            return new LinkedHashSet<>(col);
+
+        return null;
+    }
+
+    /**
+     * @return Transfer object version.
+     */
+    public byte getProtocolVersion() {
+        return V1;
+    }
+
+    /**
+     * Save object's specific data content.
+     *
+     * @param out Output object to write data content.
+     * @throws IOException If I/O errors occur.
+     */
+    protected abstract void writeExternalData(ObjectOutput out) throws IOException;
+
+    /** {@inheritDoc} */
+    @Override public void writeExternal(ObjectOutput out) throws IOException {
+        int hdr = MAGIC  + getProtocolVersion();
+
+        out.writeInt(hdr);
+
+        try (IgniteDataTransferObjectOutput dtout = new IgniteDataTransferObjectOutput(out)) {
+            writeExternalData(dtout);
+        }
+    }
+
+    /**
+     * Load object's specific data content.
+     *
+     * @param protoVer Input object version.
+     * @param in Input object to load data content.
+     * @throws IOException If I/O errors occur.
+     * @throws ClassNotFoundException If the class for an object being restored cannot be found.
+     */
+    protected abstract void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException;
+
+    /** {@inheritDoc} */
+    @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        int hdr = in.readInt();
+
+        if ((hdr & MAGIC) != MAGIC)
+            throw new IOException("Unexpected IgniteDataTransferObject header " +
+                "[actual=" + Integer.toHexString(hdr) + ", expected=" + Integer.toHexString(MAGIC) + "]");
+
+        byte ver = (byte)(hdr & 0xFF);
+
+        try (IgniteDataTransferObjectInput dtin = new IgniteDataTransferObjectInput(in)) {
+            readExternalData(ver, dtin);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectInput.java b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectInput.java
new file mode 100644
index 0000000000000..c12287520656a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectInput.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.dto;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import org.apache.ignite.internal.util.io.GridByteArrayInputStream;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Wrapper for object input.
+ */
+public class IgniteDataTransferObjectInput implements ObjectInput {
+    /** */
+    private final ObjectInputStream ois;
+
+    /**
+     * @param in Target input.
+     * @throws IOException If an I/O error occurs.
+     */
+    public IgniteDataTransferObjectInput(ObjectInput in) throws IOException {
+        byte[] buf = U.readByteArray(in);
+
+        /* */
+        GridByteArrayInputStream bis = new GridByteArrayInputStream(buf);
+        ois = new ObjectInputStream(bis);
+    }
+
+
+    /** {@inheritDoc} */
+    @Override public Object readObject() throws ClassNotFoundException, IOException {
+        return ois.readObject();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int read() throws IOException {
+        return ois.read();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int read(byte[] b) throws IOException {
+        return ois.read(b);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int read(byte[] b, int off, int len) throws IOException {
+        return ois.read(b, off, len);
+    }
+
+    /** {@inheritDoc} */
+    @Override public long skip(long n) throws IOException {
+        return ois.skip(n);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int available() throws IOException {
+        return ois.available();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close() throws IOException {
+        ois.close();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readFully(@NotNull byte[] b) throws IOException {
+        ois.readFully(b);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
+        ois.readFully(b, off, len);
+    }
+
+    /** {@inheritDoc} */
+    @Override public int skipBytes(int n) throws IOException {
+        return ois.skipBytes(n);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean readBoolean() throws IOException {
+        return ois.readBoolean();
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte readByte() throws IOException {
+        return ois.readByte();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int readUnsignedByte() throws IOException {
+        return ois.readUnsignedByte();
+    }
+
+    /** {@inheritDoc} */
+    @Override public short readShort() throws IOException {
+        return ois.readShort();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int readUnsignedShort() throws IOException {
+        return ois.readUnsignedShort();
+    }
+
+    /** {@inheritDoc} */
+    @Override public char readChar() throws IOException {
+        return ois.readChar();
+    }
+
+    /** {@inheritDoc} */
+    @Override public int readInt() throws IOException {
+        return ois.readInt();
+    }
+
+    /** {@inheritDoc} */
+    @Override public long readLong() throws IOException {
+        return ois.readLong();
+    }
+
+    /** {@inheritDoc} */
+    @Override public float readFloat() throws IOException {
+        return ois.readFloat();
+    }
+
+    /** {@inheritDoc} */
+    @Override public double readDouble() throws IOException {
+        return ois.readDouble();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String readLine() throws IOException {
+        return ois.readLine();
+    }
+
+    /** {@inheritDoc} */
+    @NotNull @Override public String readUTF() throws IOException {
+        return ois.readUTF();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectOutput.java
new file mode 100644
index 0000000000000..db4933cea29c0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/dto/IgniteDataTransferObjectOutput.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.dto;
+
+import java.io.IOException;
+import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
+import org.apache.ignite.internal.util.io.GridByteArrayOutputStream;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Wrapper for object output.
+ */
+public class IgniteDataTransferObjectOutput implements ObjectOutput {
+    /** */
+    private final ObjectOutput out;
+
+    /** */
+    private final GridByteArrayOutputStream bos;
+
+    /** */
+    private final ObjectOutputStream oos;
+
+    /**
+     * Constructor.
+     *
+     * @param out Target stream.
+     * @throws IOException If an I/O error occurs.
+     */
+    public IgniteDataTransferObjectOutput(ObjectOutput out) throws IOException {
+        this.out = out;
+
+        bos = new GridByteArrayOutputStream();
+        oos = new ObjectOutputStream(bos);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeObject(Object obj) throws IOException {
+        oos.writeObject(obj);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void write(int b) throws IOException {
+        oos.write(b);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void write(byte[] b) throws IOException {
+        oos.write(b);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void write(byte[] b, int off, int len) throws IOException {
+        oos.write(b, off, len);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeBoolean(boolean v) throws IOException {
+        oos.writeBoolean(v);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeByte(int v) throws IOException {
+        oos.writeByte(v);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeShort(int v) throws IOException {
+        oos.writeShort(v);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeChar(int v) throws IOException {
+        oos.writeChar(v);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeInt(int v) throws IOException {
+        oos.writeInt(v);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeLong(long v) throws IOException {
+        oos.writeLong(v);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeFloat(float v) throws IOException {
+        oos.writeFloat(v);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeDouble(double v) throws IOException {
+        oos.writeDouble(v);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeBytes(@NotNull String s) throws IOException {
+        oos.writeBytes(s);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeChars(@NotNull String s) throws IOException {
+        oos.writeChars(s);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeUTF(@NotNull String s) throws IOException {
+        oos.writeUTF(s);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void flush() throws IOException {
+        oos.flush();
+    }
+
+    /** {@inheritDoc} */
+    @Override public void close() throws IOException {
+        oos.flush();
+
+        U.writeByteArray(out, bos.internalArray(), bos.size());
+
+        oos.close();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java
index 38d7a0ad0aef5..6ba3aa7d5aed5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObject.java
@@ -26,10 +26,13 @@
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
+import org.apache.ignite.internal.dto.IgniteDataTransferObject;
 import org.jetbrains.annotations.Nullable;
 
 /**
- * Base class for data transfer objects.
+ * Base class for data transfer objects for Visor tasks.
+ *
+ * @deprecated Use {@link IgniteDataTransferObject} instead. This class may be removed in Ignite 3.0.
  */
 public abstract class VisorDataTransferObject implements Externalizable {
     /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectInput.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectInput.java
index 16e933090eb9b..7caf26914ba5e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectInput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectInput.java
@@ -20,12 +20,14 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectInputStream;
+import org.apache.ignite.internal.dto.IgniteDataTransferObjectInput;
 import org.apache.ignite.internal.util.io.GridByteArrayInputStream;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.NotNull;
 
 /**
  * Wrapper for object input.
+ * @deprecated Use {@link IgniteDataTransferObjectInput} instead. This class may be removed in Ignite 3.0.
  */
 public class VisorDataTransferObjectInput implements ObjectInput {
     /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectOutput.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectOutput.java
index 7fa772e75ca13..4dbd25035bac9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectOutput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/VisorDataTransferObjectOutput.java
@@ -20,12 +20,14 @@
 import java.io.IOException;
 import java.io.ObjectOutput;
 import java.io.ObjectOutputStream;
+import org.apache.ignite.internal.dto.IgniteDataTransferObjectOutput;
 import org.apache.ignite.internal.util.io.GridByteArrayOutputStream;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.jetbrains.annotations.NotNull;
 
 /**
  * Wrapper for object output.
+ * @deprecated Use {@link IgniteDataTransferObjectOutput} instead. This class may be removed in Ignite 3.0.
  */
 public class VisorDataTransferObjectOutput implements ObjectOutput {
     /** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadMonitorInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadMonitorInfo.java
index f8ddc68a31e3a..189b8e3cdfba4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadMonitorInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/debug/VisorThreadMonitorInfo.java
@@ -71,11 +71,6 @@ public StackTraceElement getStackFrame() {
         return stackFrame;
     }
 
-    /** {@inheritDoc} */
-    @Override public byte getProtocolVersion() {
-        return 1;
-    }
-
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         try (VisorDataTransferObjectOutput dtout = new VisorDataTransferObjectOutput(out)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDeploymentEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDeploymentEvent.java
index 8b0c2110368b9..e74b54c896c34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDeploymentEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDeploymentEvent.java
@@ -79,11 +79,6 @@ public String getAlias() {
         return alias;
     }
 
-    /** {@inheritDoc} */
-    @Override public byte getProtocolVersion() {
-        return 1;
-    }
-
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         try (VisorDataTransferObjectOutput dtout = new VisorDataTransferObjectOutput(out)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDiscoveryEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDiscoveryEvent.java
index ec2220d1fd7ba..c5f1d30c53d92 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDiscoveryEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridDiscoveryEvent.java
@@ -119,11 +119,6 @@ public long getTopologyVersion() {
         return topVer;
     }
 
-    /** {@inheritDoc} */
-    @Override public byte getProtocolVersion() {
-        return 1;
-    }
-
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         try (VisorDataTransferObjectOutput dtout = new VisorDataTransferObjectOutput(out)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridJobEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridJobEvent.java
index 734d85fa11cb7..2a7ea1862e396 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridJobEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridJobEvent.java
@@ -118,11 +118,6 @@ public IgniteUuid getJobId() {
         return jobId;
     }
 
-    /** {@inheritDoc} */
-    @Override public byte getProtocolVersion() {
-        return 1;
-    }
-
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         try (VisorDataTransferObjectOutput dtout = new VisorDataTransferObjectOutput(out)) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridTaskEvent.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridTaskEvent.java
index 11c9a17d38dd5..a0836c481356f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridTaskEvent.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/event/VisorGridTaskEvent.java
@@ -118,11 +118,6 @@ public boolean isInternal() {
         return internal;
     }
 
-    /** {@inheritDoc} */
-    @Override public byte getProtocolVersion() {
-        return 1;
-    }
-
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         try (VisorDataTransferObjectOutput dtout = new VisorDataTransferObjectOutput(out)) {

From f39915c06b532211841a2654daeb76d067835a30 Mon Sep 17 00:00:00 2001
From: ascherbakoff <alexey.scherbakoff@gmail.com>
Date: Sat, 13 Oct 2018 21:35:36 +0300
Subject: [PATCH 421/543] IGNITE-9612 Improve checkpoint mark phase speed -
 Fixes #4971.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit 6041cda5a0b7331c33ec643d0c8ccbd77a120a86)
Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
---
 .../persistence/GridCacheOffheapManager.java  | 32 ++++++++---
 .../partstate/PartitionAllocationMap.java     | 55 +++++++++++++------
 2 files changed, 62 insertions(+), 25 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 6c3cf00bf6622..d3efab030acad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -164,7 +164,19 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
 
         Executor execSvc = ctx.executor();
 
-        if (ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName())) {
+        boolean needSnapshot = ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName());
+
+        boolean hasNonEmptyGroups = false;
+
+        for (CacheDataStore store : partDataStores.values()) {
+            if (notEmpty(store)) {
+                hasNonEmptyGroups = true;
+
+                break;
+            }
+        }
+
+        if (needSnapshot && hasNonEmptyGroups) {
             if (execSvc == null)
                 updateSnapshotTag(ctx);
             else {
@@ -183,7 +195,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
             reuseList.saveMetadata();
 
             for (CacheDataStore store : partDataStores.values())
-                saveStoreMetadata(store, ctx, false);
+                saveStoreMetadata(store, ctx, false, needSnapshot);
         }
         else {
             execSvc.execute(() -> {
@@ -198,7 +210,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
             for (CacheDataStore store : partDataStores.values())
                 execSvc.execute(() -> {
                     try {
-                        saveStoreMetadata(store, ctx, false);
+                        saveStoreMetadata(store, ctx, false, needSnapshot);
                     }
                     catch (IgniteCheckedException e) {
                         throw new IgniteException(e);
@@ -207,6 +219,13 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
         }
     }
 
+    /**
+     * @return {@code True} is group is not empty.
+     */
+    private boolean notEmpty(CacheDataStore store) {
+        return store.rowStore() != null && (store.fullSize() > 0  || store.updateCounter() > 0);
+    }
+
     /**
      * @param store Store to save metadata.
      * @throws IgniteCheckedException If failed.
@@ -214,12 +233,11 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
     private void saveStoreMetadata(
         CacheDataStore store,
         Context ctx,
-        boolean beforeDestroy
+        boolean beforeDestroy,
+        boolean needSnapshot
     ) throws IgniteCheckedException {
         RowStore rowStore0 = store.rowStore();
 
-        boolean needSnapshot = ctx != null && ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName());
-
         if (rowStore0 != null) {
             CacheFreeListImpl freeList = (CacheFreeListImpl)rowStore0.freeList();
 
@@ -612,7 +630,7 @@ private static boolean addPartition(
         ctx.database().checkpointReadLock();
 
         try {
-            saveStoreMetadata(store, null, true);
+            saveStoreMetadata(store, null, true, false);
         }
         finally {
             ctx.database().checkpointReadUnlock();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
index cab90c5b3954a..808ab41bc0127 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionAllocationMap.java
@@ -20,7 +20,8 @@
 import java.util.Map;
 import java.util.NavigableMap;
 import java.util.Set;
-import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
@@ -37,7 +38,11 @@
 public class PartitionAllocationMap {
     /** Maps following pairs: (groupId, partId) -> (lastAllocatedCount, allocatedCount) */
     @GridToStringInclude
-    private final NavigableMap<GroupPartitionId, PagesAllocationRange> map = new ConcurrentSkipListMap<>();
+    private final Map<GroupPartitionId, PagesAllocationRange> writeMap = new ConcurrentHashMap<>();
+
+    /** Map used by snapshot executor. */
+    @GridToStringInclude
+    private NavigableMap<GroupPartitionId, PagesAllocationRange> readMap;
 
     /** Partitions forced to be skipped. */
     @GridToStringInclude
@@ -51,14 +56,14 @@ public class PartitionAllocationMap {
      * @return value or null
      */
     @Nullable public PagesAllocationRange get(GroupPartitionId key) {
-        return map.get(key);
+        return readMap.get(key);
     }
 
     /**
      * @param fullPageId Full page id.
      */
     @Nullable public PagesAllocationRange get(FullPageId fullPageId) {
-        return map.get(createCachePartId(fullPageId));
+        return readMap.get(createCachePartId(fullPageId));
     }
 
     /**
@@ -73,27 +78,27 @@ public class PartitionAllocationMap {
 
     /** @return <tt>true</tt> if this map contains no key-value mappings */
     public boolean isEmpty() {
-        return map.isEmpty();
+        return readMap.isEmpty();
     }
 
     /** @return the number of key-value mappings in this map. */
     public int size() {
-        return map.size();
+        return readMap.size();
     }
 
     /** @return keys (all caches partitions) */
     public Set<GroupPartitionId> keySet() {
-        return map.keySet();
+        return readMap.keySet();
     }
 
     /** @return values (allocation ranges) */
     public Iterable<PagesAllocationRange> values() {
-        return map.values();
+        return readMap.values();
     }
 
     /** @return Returns the first (lowest) key currently in this map. */
     public GroupPartitionId firstKey() {
-        return map.firstKey();
+        return readMap.firstKey();
     }
 
     /**
@@ -103,11 +108,7 @@ public GroupPartitionId firstKey() {
      * @return {@code true} if skipped partition was added to the ignore list during this call.
      */
     public boolean forceSkipIndexPartition(int grpId) {
-        GroupPartitionId idxId = new GroupPartitionId(grpId, PageIdAllocator.INDEX_PARTITION);
-
-        map.remove(idxId);
-
-        return skippedParts.add(idxId);
+        return skippedParts.add(new GroupPartitionId(grpId, PageIdAllocator.INDEX_PARTITION));
     }
 
     /**
@@ -117,17 +118,17 @@ public boolean forceSkipIndexPartition(int grpId) {
      * @return first found key which is greater than provided
      */
     @Nullable public GroupPartitionId nextKey(@NotNull final GroupPartitionId key) {
-        return map.navigableKeySet().higher(key);
+        return readMap.navigableKeySet().higher(key);
     }
 
     /** @return set view of the mappings contained in this map, sorted in ascending key order */
     public Set<Map.Entry<GroupPartitionId, PagesAllocationRange>> entrySet() {
-        return map.entrySet();
+        return readMap.entrySet();
     }
 
     /** @return <tt>true</tt> if this map contains a mapping for the specified key */
     public boolean containsKey(GroupPartitionId key) {
-        return map.containsKey(key);
+        return readMap.containsKey(key);
     }
 
     /**
@@ -137,7 +138,25 @@ public boolean containsKey(GroupPartitionId key) {
      * <tt>key</tt>.
      */
     public PagesAllocationRange put(GroupPartitionId key, PagesAllocationRange val) {
-        return !skippedParts.contains(key) ? map.put(key, val) : null;
+        return writeMap.put(key, val);
+    }
+
+    /**
+     * Prepare map for snapshot.
+     */
+    public void prepareForSnapshot() {
+        if (readMap != null)
+            return;
+
+        readMap = new TreeMap<>();
+
+        for (Map.Entry<GroupPartitionId, PagesAllocationRange> entry : writeMap.entrySet()) {
+            if (!skippedParts.contains(entry.getKey()))
+                readMap.put(entry.getKey(), entry.getValue());
+        }
+
+        skippedParts.clear();
+        writeMap.clear();
     }
 
     /** {@inheritDoc} */

From 80e01dd9bc992db744cdb171ab47853c9a54e2c9 Mon Sep 17 00:00:00 2001
From: Vitaliy Biryukov <BiryukovVitaliy92@gmail.com>
Date: Fri, 4 May 2018 17:56:00 +0300
Subject: [PATCH 422/543] IGNITE-7986 GridPartitionStateMap.entrySet()
 optimization. - Fixes #3659.

Signed-off-by: dpavlov <dpavlov@apache.org>
(cherry picked from commit bc7814ef31df1ffd71b4aad1df675a62f0d88136)
---
 .../internal/util/GridPartitionStateMap.java  | 46 ++++++---
 .../ignite/util/GridPartitionMapSelfTest.java | 95 ++++++++++++++++++-
 2 files changed, 129 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java
index 43a8bc7c345e3..4f8313d734e96 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridPartitionStateMap.java
@@ -42,7 +42,19 @@ public class GridPartitionStateMap extends AbstractMap<Integer, GridDhtPartition
     private static final int BITS = Integer.SIZE -
         Integer.numberOfLeadingZeros(GridDhtPartitionState.values().length + 1);
 
-    /** */
+    /**
+     * Contains partition map.
+     * For a map containing 3 partitions state with a size of 3 bits storage will be done this way:
+     * <pre>
+     *     +-----------+-----------+-----------+
+     *     |  p0 - 100 |  p1 - 011 |  p2 - 001 |
+     *     +---+---+---+---+---+---+---+---+---+
+     *     | 0 | 0 | 1 | 1 | 1 | 0 | 1 | 0 | 0 |
+     *     +---+---+---+---+---+---+---+---+---+
+     * </pre>
+     * The first element takes the first {@link GridPartitionStateMap#BITS} bits in reverse order,
+     * the second element next {@link GridPartitionStateMap#BITS} bits in reverse order, etc.
+     */
     private final BitSet states;
 
     /** */
@@ -52,25 +64,37 @@ public class GridPartitionStateMap extends AbstractMap<Integer, GridDhtPartition
     @Override public Set<Entry<Integer, GridDhtPartitionState>> entrySet() {
         return new AbstractSet<Entry<Integer, GridDhtPartitionState>>() {
             @Override public Iterator<Entry<Integer, GridDhtPartitionState>> iterator() {
-                final int size = states.isEmpty() ? 0 : (states.length() - 1) / BITS + 1;
-
                 return new Iterator<Entry<Integer, GridDhtPartitionState>>() {
-                    private int next;
+                    /** Current {@link GridPartitionStateMap#states} index. */
+                    private int idx;
+
+                    /** Current key value. */
                     private int cur;
 
                     @Override public boolean hasNext() {
-                        while(state(next) == null && next < size)
-                            next++;
+                        idx = states.nextSetBit(idx);
 
-                        return next < size;
+                        return idx != -1;
                     }
 
                     @Override public Entry<Integer, GridDhtPartitionState> next() {
                         if (!hasNext())
                             throw new NoSuchElementException();
 
-                        cur = next;
-                        next++;
+                        cur = idx / BITS;
+
+                        int bitN = idx % BITS;
+
+                        // Get state value from BitSet like in GridPartitionStateMap#state, but don't process known zero bits.
+                        int st = 1 << bitN;
+
+                        // Accumulating values of remaining bits
+                        for (int i = 1; i < BITS - bitN; i++)
+                            st |= (states.get(idx + i) ? 1 : 0) << i + bitN;
+
+                        final int ordinal = st - 1;
+
+                        idx += (BITS - bitN);
 
                         return new Entry<Integer, GridDhtPartitionState>() {
                             int p = cur;
@@ -80,7 +104,7 @@ public class GridPartitionStateMap extends AbstractMap<Integer, GridDhtPartition
                             }
 
                             @Override public GridDhtPartitionState getValue() {
-                                return state(p);
+                                return GridDhtPartitionState.fromOrdinal(ordinal);
                             }
 
                             @Override public GridDhtPartitionState setValue(GridDhtPartitionState val) {
@@ -219,4 +243,4 @@ private GridDhtPartitionState state(int part) {
     @Override public int hashCode() {
         return 31 * states.hashCode() + size;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridPartitionMapSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridPartitionMapSelfTest.java
index ddd0ac6d1ea1d..9068e829072c3 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridPartitionMapSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridPartitionMapSelfTest.java
@@ -19,8 +19,11 @@
 
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ThreadLocalRandom;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -144,6 +147,96 @@ public void testCopyNoActive() {
         assertEquals(1, cp2.size());
     }
 
+    /**
+     * Tests that entries from {@link Iterator#next()} remain unaltered.
+     */
+    public void testIteratorNext() {
+        GridPartitionStateMap map = new GridPartitionStateMap();
+
+        initMap(map);
+
+        Iterator<Map.Entry<Integer, GridDhtPartitionState>> iter = map.entrySet().iterator();
+
+        for (int i = 0; i < map.size() + 1; i++)
+            assertTrue(iter.hasNext());
+
+        Map.Entry<Integer, GridDhtPartitionState> entry1 = iter.next();
+
+        for (int i = 0; i < map.size() + 1; i++)
+            assertTrue(iter.hasNext());
+
+        Map.Entry<Integer, GridDhtPartitionState> entry2 = iter.next();
+
+        iter.remove();
+
+        assertNotNull(entry1.getValue());
+        assertNotNull(entry2.getValue());
+
+        assertEquals(Integer.valueOf(0), entry1.getKey());
+        assertEquals(Integer.valueOf(1), entry2.getKey());
+
+        assertEquals(GridDhtPartitionState.MOVING, entry1.getValue());
+        assertEquals(GridDhtPartitionState.RENTING, entry2.getValue());
+    }
+
+    /**
+     * Tests {@link GridDhtPartitionState} compatibility with {@link TreeMap} on random operations.
+     */
+    public void testOnRandomOperations() {
+        ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+        Map<Integer, GridDhtPartitionState> treeMap = new TreeMap<>();
+        Map<Integer, GridDhtPartitionState> gridMap = new GridPartitionStateMap();
+
+        int statesNum = GridDhtPartitionState.values().length;
+
+        for (int i = 0; i < 10000; i++) {
+            Integer part = rnd.nextInt(65536);
+
+            GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(rnd.nextInt(statesNum));
+
+            int rndOperation = rnd.nextInt(9);
+
+            if (rndOperation <= 5) {
+                treeMap.put(part, state);
+                gridMap.put(part, state);
+            }
+            else if (rndOperation == 6) {
+                treeMap.remove(part);
+                gridMap.remove(part);
+            }
+            else if (!treeMap.isEmpty()) {
+                int n = rnd.nextInt(0, treeMap.size());
+
+                Iterator<Map.Entry<Integer, GridDhtPartitionState>> iter1 = treeMap.entrySet().iterator();
+                Iterator<Map.Entry<Integer, GridDhtPartitionState>> iter2 = gridMap.entrySet().iterator();
+
+                Map.Entry<Integer, GridDhtPartitionState> entry1 = iter1.next();
+                Map.Entry<Integer, GridDhtPartitionState> entry2 = iter2.next();
+
+                for (int j = 1; j <= n; j++) {
+                    entry1 = iter1.next();
+                    entry2 = iter2.next();
+
+                    assertEquals(entry1.getValue(), entry2.getValue());
+                }
+
+                if (rndOperation == 7) {
+                    entry1.setValue(state);
+                    entry2.setValue(state);
+                }
+                else {
+                    iter1.remove();
+                    iter2.remove();
+                }
+            }
+
+            assertEquals(treeMap.size(), gridMap.size());
+        }
+
+        assertEquals(treeMap, gridMap);
+    }
+
     /** */
     private GridPartitionStateMap initMap(GridPartitionStateMap map) {
         map.put(0, GridDhtPartitionState.MOVING);
@@ -159,4 +252,4 @@ private GridPartitionStateMap initMap(GridPartitionStateMap map) {
 
         return map;
     }
-}
\ No newline at end of file
+}

From 2fe58defd9f25ed539f6fbcda5ec4d55a387e3d9 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 11 Oct 2018 19:25:58 +0300
Subject: [PATCH 423/543] IGNITE-9561 Optimize affinity initialization for
 started cache groups - Fixes #4854.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
---
 .../cache/CacheAffinitySharedManager.java     | 220 ++++++++++--------
 .../cache/CacheGroupDescriptor.java           |  19 ++
 .../preloader/CacheGroupAffinityMessage.java  |  12 +-
 .../GridDhtPartitionsExchangeFuture.java      |  97 +++++---
 .../ignite/internal/util/IgniteUtils.java     | 178 ++++++++++++++
 .../util/InitializationProtector.java         |  79 +++++++
 .../util/ParallelExecutionException.java      |  51 ++++
 .../util/lang/IgniteThrowableConsumer.java    |  37 +++
 .../util/lang/IgniteThrowableRunner.java      |  30 +++
 9 files changed, 583 insertions(+), 140 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/ParallelExecutionException.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableRunner.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index cd141d3b762c1..dd4161c0d7b99 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -29,8 +29,10 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
 import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.affinity.AffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
@@ -904,28 +906,48 @@ private void processCacheStartRequests(
 
         time = System.currentTimeMillis();
 
-        Set<Integer> gprs = new HashSet<>();
+        initAffinityOnCacheGroupsStart(fut, exchActions, crd);
 
-        for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) {
-            int grpId = action.descriptor().groupId();
+        if (log.isInfoEnabled())
+            log.info("Affinity initialization for started caches performed in " + (System.currentTimeMillis() - time) + " ms.");
+    }
 
-            if (gprs.add(grpId)) {
-                if (crd)
-                    initStartedGroupOnCoordinator(fut, action.descriptor().groupDescriptor());
-                else {
-                    CacheGroupContext grp = cctx.cache().cacheGroup(grpId);
+    /**
+     * Initializes affinity for started cache groups received during {@code fut}.
+     *
+     * @param fut Exchange future.
+     * @param exchangeActions Exchange actions.
+     * @param crd {@code True} if local node is coordinator.
+     */
+    private void initAffinityOnCacheGroupsStart(
+        GridDhtPartitionsExchangeFuture fut,
+        ExchangeActions exchangeActions,
+        boolean crd
+    ) throws IgniteCheckedException {
+        List<CacheGroupDescriptor> startedGroups = exchangeActions.cacheStartRequests().stream()
+            .map(action -> action.descriptor().groupDescriptor())
+            .distinct()
+            .collect(Collectors.toList());
+
+        U.doInParallel(
+            cctx.kernalContext().getSystemExecutorService(),
+            startedGroups,
+            new IgniteInClosureX<CacheGroupDescriptor>() {
+                @Override public void applyx(CacheGroupDescriptor grpDesc) throws IgniteCheckedException {
+                    if (crd)
+                        initStartedGroupOnCoordinator(fut, grpDesc);
+                    else {
+                        CacheGroupContext grp = cctx.cache().cacheGroup(grpDesc.groupId());
 
-                    if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.initialVersion())) {
-                        assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion();
+                        if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.initialVersion())) {
+                            assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion();
 
-                        initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut);
+                            initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut);
+                        }
                     }
                 }
-            }
-        }
-
-        if (log.isInfoEnabled())
-            log.info("Affinity initialization for started caches performed in " + (System.currentTimeMillis() - time) + " ms.");
+            },
+            null);
     }
 
     /**
@@ -934,7 +956,7 @@ private void processCacheStartRequests(
      * @param fut Exchange future.
      * @param crd Coordinator flag.
      * @param exchActions Cache change requests.
-     * @param forceClose
+     * @param forceClose Force close flag.
      * @return Set of cache groups to be stopped.
      */
     private Set<Integer> processCacheStopRequests(
@@ -991,9 +1013,11 @@ public void clearGroupHoldersAndRegistry() {
      * @param crd Coordinator flag.
      * @param msg Affinity change message.
      */
-    public void onExchangeChangeAffinityMessage(GridDhtPartitionsExchangeFuture exchFut,
+    public void onExchangeChangeAffinityMessage(
+        GridDhtPartitionsExchangeFuture exchFut,
         boolean crd,
-        CacheAffinityChangeMessage msg) {
+        CacheAffinityChangeMessage msg
+    ) {
         if (log.isDebugEnabled()) {
             log.debug("Process exchange affinity change message [exchVer=" + exchFut.initialVersion() +
                 ", msg=" + msg + ']');
@@ -1007,7 +1031,7 @@ public void onExchangeChangeAffinityMessage(GridDhtPartitionsExchangeFuture exch
 
         assert assignment != null;
 
-        final Map<Object, List<List<ClusterNode>>> affCache = new HashMap<>();
+        final Map<Object, List<List<ClusterNode>>> affCache = new ConcurrentHashMap<>();
 
         forAllCacheGroups(crd, new IgniteInClosureX<GridAffinityAssignmentCache>() {
             @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException {
@@ -1041,10 +1065,11 @@ public void onExchangeChangeAffinityMessage(GridDhtPartitionsExchangeFuture exch
      * @param msg Message.
      * @throws IgniteCheckedException If failed.
      */
-    public void onChangeAffinityMessage(final GridDhtPartitionsExchangeFuture exchFut,
+    public void onChangeAffinityMessage(
+        final GridDhtPartitionsExchangeFuture exchFut,
         boolean crd,
-        final CacheAffinityChangeMessage msg)
-        throws IgniteCheckedException {
+        final CacheAffinityChangeMessage msg
+    ) {
         assert msg.topologyVersion() != null && msg.exchangeId() == null : msg;
 
         final AffinityTopologyVersion topVer = exchFut.initialVersion();
@@ -1060,7 +1085,7 @@ public void onChangeAffinityMessage(final GridDhtPartitionsExchangeFuture exchFu
 
         final Map<Integer, IgniteUuid> deploymentIds = msg.cacheDeploymentIds();
 
-        final Map<Object, List<List<ClusterNode>>> affCache = new HashMap<>();
+        final Map<Object, List<List<ClusterNode>>> affCache = new ConcurrentHashMap<>();
 
         forAllCacheGroups(crd, new IgniteInClosureX<GridAffinityAssignmentCache>() {
             @Override public void applyx(GridAffinityAssignmentCache aff) throws IgniteCheckedException {
@@ -1172,14 +1197,17 @@ private void processAffinityAssignmentResponse(UUID nodeId, GridDhtAffinityAssig
 
     /**
      * @param c Cache closure.
-     * @throws IgniteCheckedException If failed
      */
-    private void forAllRegisteredCacheGroups(IgniteInClosureX<CacheGroupDescriptor> c) throws IgniteCheckedException {
-        for (CacheGroupDescriptor cacheDesc : cachesRegistry.allGroups().values()) {
-            if (cacheDesc.config().getCacheMode() == LOCAL)
-                continue;
+    private void forAllRegisteredCacheGroups(IgniteInClosureX<CacheGroupDescriptor> c) {
+        Collection<CacheGroupDescriptor> affinityCaches = cachesRegistry.allGroups().values().stream()
+            .filter(desc -> desc.config().getCacheMode() != LOCAL)
+            .collect(Collectors.toList());
 
-            c.applyx(cacheDesc);
+        try {
+            U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c, null);
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException("Failed to execute affinity operation on cache groups", e);
         }
     }
 
@@ -1188,17 +1216,25 @@ private void forAllRegisteredCacheGroups(IgniteInClosureX<CacheGroupDescriptor>
      * @param c Closure.
      */
     private void forAllCacheGroups(boolean crd, IgniteInClosureX<GridAffinityAssignmentCache> c) {
+        Collection<GridAffinityAssignmentCache> affinityCaches;
+
         if (crd) {
-            for (CacheGroupHolder grp : grpHolders.values())
-                c.apply(grp.affinity());
+            affinityCaches = grpHolders.values().stream()
+                .map(CacheGroupHolder::affinity)
+                .collect(Collectors.toList());
         }
         else {
-            for (CacheGroupContext grp : cctx.kernalContext().cache().cacheGroups()) {
-                if (grp.isLocal())
-                    continue;
+            affinityCaches = cctx.kernalContext().cache().cacheGroups().stream()
+                .filter(grp -> !grp.isLocal())
+                .map(CacheGroupContext::affinity)
+                .collect(Collectors.toList());
+        }
 
-                c.apply(grp.affinity());
-            }
+        try {
+            U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c, null);
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException("Failed to execute affinity operation on cache groups", e);
         }
     }
 
@@ -1349,11 +1385,14 @@ public GridAffinityAssignmentCache affinity(Integer grpId) {
      * @param fut Current exchange future.
      * @param msg Finish exchange message.
      */
-    public void applyAffinityFromFullMessage(final GridDhtPartitionsExchangeFuture fut,
-        final GridDhtPartitionsFullMessage msg) {
-        final Map<Long, ClusterNode> nodesByOrder = new HashMap<>();
+    public void applyAffinityFromFullMessage(
+        final GridDhtPartitionsExchangeFuture fut,
+        final GridDhtPartitionsFullMessage msg
+    ) {
+        // Please do not use following pattern of code (nodesByOrder, affCache). NEVER.
+        final Map<Long, ClusterNode> nodesByOrder = new ConcurrentHashMap<>();
 
-        final Map<Object, List<List<ClusterNode>>> affCache = new HashMap<>();
+        final Map<Object, List<List<ClusterNode>>> affCache = new ConcurrentHashMap<>();
 
         long time = System.currentTimeMillis();
 
@@ -1400,13 +1439,13 @@ public void applyAffinityFromFullMessage(final GridDhtPartitionsExchangeFuture f
      * @param msg Message finish message.
      * @param resTopVer Result topology version.
      */
-    public void onLocalJoin(final GridDhtPartitionsExchangeFuture fut,
+    public void onLocalJoin(
+        final GridDhtPartitionsExchangeFuture fut,
         GridDhtPartitionsFullMessage msg,
-        final AffinityTopologyVersion resTopVer) {
+        final AffinityTopologyVersion resTopVer
+    ) {
         final Set<Integer> affReq = fut.context().groupsAffinityRequestOnJoin();
 
-        final Map<Long, ClusterNode> nodesByOrder = new HashMap<>();
-
         final Map<Integer, CacheGroupAffinityMessage> receivedAff = msg.joinedNodeAffinity();
 
         assert F.isEmpty(affReq) || (!F.isEmpty(receivedAff) && receivedAff.size() >= affReq.size())
@@ -1415,6 +1454,8 @@ public void onLocalJoin(final GridDhtPartitionsExchangeFuture fut,
                 ", receivedCnt=" + (receivedAff != null ? receivedAff.size() : "none") +
                 ", msg=" + msg + "]");
 
+        final Map<Long, ClusterNode> nodesByOrder = new ConcurrentHashMap<>();
+
         long time = System.currentTimeMillis();
 
         forAllCacheGroups(false, new IgniteInClosureX<GridAffinityAssignmentCache>() {
@@ -1754,8 +1795,8 @@ private GridDhtAffinityAssignmentResponse fetchAffinity(AffinityTopologyVersion
         @Nullable ExchangeDiscoveryEvents events,
         DiscoCache discoCache,
         GridAffinityAssignmentCache affCache,
-        GridDhtAssignmentFetchFuture fetchFut)
-        throws IgniteCheckedException {
+        GridDhtAssignmentFetchFuture fetchFut
+    ) throws IgniteCheckedException {
         assert affCache != null;
 
         GridDhtAffinityAssignmentResponse res = fetchFut.get();
@@ -1833,7 +1874,7 @@ public IgniteInternalFuture<?> initCoordinatorCaches(
         final GridDhtPartitionsExchangeFuture fut,
         final boolean newAff
     ) throws IgniteCheckedException {
-        final List<IgniteInternalFuture<AffinityTopologyVersion>> futs = new ArrayList<>();
+        final List<IgniteInternalFuture<AffinityTopologyVersion>> futs = Collections.synchronizedList(new ArrayList<>());
 
         final AffinityTopologyVersion topVer = fut.initialVersion();
 
@@ -1985,29 +2026,28 @@ private CacheGroupHolder groupHolder(AffinityTopologyVersion topVer, final Cache
     /**
      * @param fut Current exchange future.
      * @param crd Coordinator flag.
-     * @throws IgniteCheckedException If failed.
      * @return Rabalance info.
      */
-    @Nullable private WaitRebalanceInfo initAffinityOnNodeJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd)
-        throws IgniteCheckedException {
+    @Nullable private WaitRebalanceInfo initAffinityOnNodeJoin(final GridDhtPartitionsExchangeFuture fut, boolean crd) {
         final ExchangeDiscoveryEvents evts = fut.context().events();
 
-        final Map<Object, List<List<ClusterNode>>> affCache = new HashMap<>();
+        final Map<Object, List<List<ClusterNode>>> affCache = new ConcurrentHashMap<>();
 
         if (!crd) {
-            for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
-                if (grp.isLocal())
-                    continue;
+            forAllCacheGroups(false, new IgniteInClosureX<GridAffinityAssignmentCache>() {
+                @Override public void applyx(GridAffinityAssignmentCache grpAffCache) throws IgniteCheckedException {
+                    CacheGroupContext grp = cctx.cache().cacheGroup(grpAffCache.groupId());
 
-                boolean latePrimary = grp.rebalanceEnabled();
+                    assert grp != null;
 
-                initAffinityOnNodeJoin(evts,
-                    evts.nodeJoined(grp.receivedFrom()),
-                    grp.affinity(),
-                    null,
-                    latePrimary,
-                    affCache);
-            }
+                    initAffinityOnNodeJoin(evts,
+                        evts.nodeJoined(grp.receivedFrom()),
+                        grp.affinity(),
+                        null,
+                        grp.rebalanceEnabled(),
+                        affCache);
+                }
+            });
 
             return null;
         }
@@ -2049,6 +2089,9 @@ private CacheGroupHolder groupHolder(AffinityTopologyVersion topVer, final Cache
         }
     }
 
+    /**
+     * @param aff Affinity assignment.
+     */
     private Map<UUID, GridDhtPartitionMap> affinityFullMap(AffinityAssignment aff) {
         Map<UUID, GridDhtPartitionMap> map = new HashMap<>();
 
@@ -2214,7 +2257,7 @@ public IgniteInternalFuture<Map<Integer, Map<Integer, List<UUID>>>> initAffinity
                     try {
                         resFut.onDone(initAffinityBasedOnPartitionsAvailability(fut.initialVersion(), fut, NODE_TO_ID, false));
                     }
-                    catch (IgniteCheckedException e) {
+                    catch (Exception e) {
                         resFut.onDone(e);
                     }
                 }
@@ -2234,14 +2277,15 @@ public IgniteInternalFuture<Map<Integer, Map<Integer, List<UUID>>>> initAffinity
      * @param fut Exchange future.
      * @param c Closure converting affinity diff.
      * @param initAff {@code True} if need initialize affinity.
-     * @return Affinity assignment.
-     * @throws IgniteCheckedException If failed.
+     *
+     * @return Affinity assignment for each of registered cache group.
      */
-    private <T> Map<Integer, Map<Integer, List<T>>> initAffinityBasedOnPartitionsAvailability(final AffinityTopologyVersion topVer,
+    private <T> Map<Integer, Map<Integer, List<T>>> initAffinityBasedOnPartitionsAvailability(
+        final AffinityTopologyVersion topVer,
         final GridDhtPartitionsExchangeFuture fut,
         final IgniteClosure<ClusterNode, T> c,
-        final boolean initAff)
-        throws IgniteCheckedException {
+        final boolean initAff
+    ) {
         final boolean enforcedCentralizedAssignment =
             DiscoveryCustomEvent.requiresCentralizedAffinityAssignment(fut.firstEvent());
 
@@ -2251,7 +2295,7 @@ private <T> Map<Integer, Map<Integer, List<T>>> initAffinityBasedOnPartitionsAva
 
         final Collection<ClusterNode> aliveNodes = fut.context().events().discoveryCache().serverNodes();
 
-        final Map<Integer, Map<Integer, List<T>>> assignment = new HashMap<>();
+        final Map<Integer, Map<Integer, List<T>>> assignment = new ConcurrentHashMap<>();
 
         forAllRegisteredCacheGroups(new IgniteInClosureX<CacheGroupDescriptor>() {
             @Override public void applyx(CacheGroupDescriptor desc) throws IgniteCheckedException {
@@ -2647,13 +2691,13 @@ class WaitRebalanceInfo {
         private final AffinityTopologyVersion topVer;
 
         /** */
-        private Map<Integer, Map<Integer, UUID>> waitGrps;
+        private final Map<Integer, Map<Integer, UUID>> waitGrps = new ConcurrentHashMap<>();
 
         /** */
-        private Map<Integer, Map<Integer, List<ClusterNode>>> assignments;
+        private final Map<Integer, Map<Integer, List<ClusterNode>>> assignments = new ConcurrentHashMap<>();
 
         /** */
-        private Map<Integer, IgniteUuid> deploymentIds;
+        private final Map<Integer, IgniteUuid> deploymentIds = new ConcurrentHashMap<>();
 
         /**
          * @param topVer Topology version.
@@ -2666,14 +2710,15 @@ class WaitRebalanceInfo {
          * @return {@code True} if there are partitions waiting for rebalancing.
          */
         boolean empty() {
-            if (waitGrps != null) {
-                assert !waitGrps.isEmpty();
+            boolean isEmpty = waitGrps.isEmpty();
+
+            if (!isEmpty) {
                 assert waitGrps.size() == assignments.size();
 
                 return false;
             }
 
-            return true;
+            return isEmpty;
         }
 
         /**
@@ -2685,28 +2730,11 @@ boolean empty() {
         void add(Integer grpId, Integer part, UUID waitNode, List<ClusterNode> assignment) {
             assert !F.isEmpty(assignment) : assignment;
 
-            if (waitGrps == null) {
-                waitGrps = new HashMap<>();
-                assignments = new HashMap<>();
-                deploymentIds = new HashMap<>();
-            }
-
-            Map<Integer, UUID> cacheWaitParts = waitGrps.get(grpId);
-
-            if (cacheWaitParts == null) {
-                waitGrps.put(grpId, cacheWaitParts = new HashMap<>());
-
-                deploymentIds.put(grpId, cachesRegistry.group(grpId).deploymentId());
-            }
-
-            cacheWaitParts.put(part, waitNode);
-
-            Map<Integer, List<ClusterNode>> cacheAssignment = assignments.get(grpId);
+            deploymentIds.putIfAbsent(grpId, cachesRegistry.group(grpId).deploymentId());
 
-            if (cacheAssignment == null)
-                assignments.put(grpId, cacheAssignment = new HashMap<>());
+            waitGrps.computeIfAbsent(grpId, k -> new HashMap<>()).put(part, waitNode);
 
-            cacheAssignment.put(part, assignment);
+            assignments.computeIfAbsent(grpId, k -> new HashMap<>()).put(part, assignment);
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java
index 70cdcc735af62..e72de28a05674 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupDescriptor.java
@@ -23,6 +23,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.UUID;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -311,4 +312,22 @@ public boolean persistenceEnabled() {
     @Override public String toString() {
         return S.toString(CacheGroupDescriptor.class, this, "cacheName", cacheCfg.getName());
     }
+
+    /** {@inheritDoc} */
+    @Override public boolean equals(Object o) {
+        if (this == o)
+            return true;
+
+        if (o == null || getClass() != o.getClass())
+            return false;
+
+        CacheGroupDescriptor that = (CacheGroupDescriptor) o;
+
+        return grpId == that.grpId;
+    }
+
+    /** {@inheritDoc} */
+    @Override public int hashCode() {
+        return Objects.hash(grpId);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java
index 7da4051e27929..695eadca581a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CacheGroupAffinityMessage.java
@@ -192,16 +192,10 @@ public static List<ClusterNode> toNodes(GridLongList assign, Map<Long, ClusterNo
         for (int n = 0; n < assign.size(); n++) {
             long order = assign.get(n);
 
-            ClusterNode affNode = nodesByOrder.get(order);
+            ClusterNode affNode = nodesByOrder.computeIfAbsent(order, o -> discoCache.serverNodeByOrder(order));
 
-            if (affNode == null) {
-                affNode = discoCache.serverNodeByOrder(order);
-
-                assert affNode != null : "Failed to find node by order [order=" + order +
-                    ", topVer=" + discoCache.version() + ']';
-
-                nodesByOrder.put(order, affNode);
-            }
+            assert affNode != null : "Failed to find node by order [order=" + order +
+                ", topVer=" + discoCache.version() + ']';
 
             assign0.add(affNode);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index b598878509d9a..8702d9e7a9bb1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -42,6 +42,7 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheMode;
@@ -98,6 +99,7 @@
 import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -3162,41 +3164,59 @@ else if (forceAffReassignment)
         }
     }
 
+    /**
+     * Collects non local cache group descriptors.
+     *
+     * @return Collection of non local cache group descriptors.
+     */
+    private List<CacheGroupDescriptor> nonLocalCacheGroupDescriptors() {
+        return cctx.affinity().cacheGroups().values().stream()
+            .filter(grpDesc -> grpDesc.config().getCacheMode() != CacheMode.LOCAL)
+            .collect(Collectors.toList());
+    }
+
     /**
      * Validates that partition update counters and cache sizes for all caches are consistent.
      */
     private void validatePartitionsState() {
         long time = System.currentTimeMillis();
 
-        for (Map.Entry<Integer, CacheGroupDescriptor> e : cctx.affinity().cacheGroups().entrySet()) {
-            CacheGroupDescriptor grpDesc = e.getValue();
-            if (grpDesc.config().getCacheMode() == CacheMode.LOCAL)
-                continue;
-
-            int grpId = e.getKey();
-
-            CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpId);
+        try {
+            U.doInParallel(
+                cctx.kernalContext().getSystemExecutorService(),
+                nonLocalCacheGroupDescriptors(),
+                new IgniteInClosureX<CacheGroupDescriptor>() {
+                    @Override public void applyx(CacheGroupDescriptor grpDesc) {
+                        CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId());
 
-            GridDhtPartitionTopology top = grpCtx != null ?
-                    grpCtx.topology() :
-                    cctx.exchange().clientTopology(grpId, events().discoveryCache());
+                        GridDhtPartitionTopology top = grpCtx != null
+                            ? grpCtx.topology()
+                            : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache());
 
             // Do not validate read or write through caches or caches with disabled rebalance.
+
             if (grpCtx == null
                     || grpCtx.config().isReadThrough()
                     || grpCtx.config().isWriteThrough()
                     || grpCtx.config().getCacheStoreFactory() != null
                     || grpCtx.config().getRebalanceDelay() == -1
-                    || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE)
-                continue;
+                    || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE
+                    )
+                return;
 
-            try {
-                validator.validatePartitionCountersAndSizes(this, top, msgs);
-            }
-            catch (IgniteCheckedException ex) {
-                log.warning("Partition states validation has failed for group: " + grpDesc.cacheOrGroupName() + ". " + ex.getMessage());
-                // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833
-            }
+                        try {
+                            validator.validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture.this, top, msgs);
+                        }
+                        catch (IgniteCheckedException ex) {
+                            log.warning("Partition states validation has failed for group: " + grpCtx.cacheOrGroupName() + ". " + ex.getMessage());
+                            // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833
+                        }
+                    }
+                },
+                null);
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException("Failed to validate partitions state", e);
         }
 
         if (log.isInfoEnabled())
@@ -3209,21 +3229,28 @@ private void validatePartitionsState() {
     private void assignPartitionsStates() {
         long time = System.currentTimeMillis();
 
-        for (Map.Entry<Integer, CacheGroupDescriptor> e : cctx.affinity().cacheGroups().entrySet()) {
-            CacheGroupDescriptor grpDesc = e.getValue();
-            if (grpDesc.config().getCacheMode() == CacheMode.LOCAL)
-                continue;
-
-            CacheGroupContext grpCtx = cctx.cache().cacheGroup(e.getKey());
-
-            GridDhtPartitionTopology top = grpCtx != null ?
-                grpCtx.topology() :
-                cctx.exchange().clientTopology(e.getKey(), events().discoveryCache());
-
-            if (!CU.isPersistentCache(grpDesc.config(), cctx.gridConfig().getDataStorageConfiguration()))
-                assignPartitionSizes(top);
-            else
-                assignPartitionStates(top);
+        try {
+            U.doInParallel(
+                cctx.kernalContext().getSystemExecutorService(),
+                nonLocalCacheGroupDescriptors(),
+                new IgniteInClosureX<CacheGroupDescriptor>() {
+                    @Override public void applyx(CacheGroupDescriptor grpDesc) {
+                        CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId());
+
+                        GridDhtPartitionTopology top = grpCtx != null
+                            ? grpCtx.topology()
+                            : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache());
+
+                        if (!CU.isPersistentCache(grpDesc.config(), cctx.gridConfig().getDataStorageConfiguration()))
+                            assignPartitionSizes(top);
+                        else
+                            assignPartitionStates(top);
+                    }
+                },
+                null);
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException("Failed to assign partition states", e);
         }
 
         if (log.isInfoEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 7c14bccec9dac..26b33b69d9551 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -32,10 +32,13 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.ObjectInput;
+import java.io.ObjectInputStream;
 import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.io.Reader;
+import java.io.Serializable;
 import java.io.StringWriter;
 import java.io.Writer;
 import java.lang.annotation.Annotation;
@@ -137,6 +140,7 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 import java.util.zip.ZipInputStream;
@@ -208,16 +212,20 @@
 import org.apache.ignite.internal.util.lang.GridClosureException;
 import org.apache.ignite.internal.util.lang.GridPeerDeployAware;
 import org.apache.ignite.internal.util.lang.GridTuple;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
+import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer;
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.P1;
+import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteFutureCancelledException;
@@ -10234,6 +10242,43 @@ public static byte[] zip(@Nullable byte[] bytes) throws IgniteCheckedException {
         }
     }
 
+    /**
+     * Serialize object to byte array.
+     *
+     * @param obj Object.
+     * @return Serialized object.
+     */
+    public static byte[] toBytes(Serializable obj) {
+        try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
+             ObjectOutputStream oos = new ObjectOutputStream(bos)) {
+
+            oos.writeObject(obj);
+            oos.flush();
+
+            return bos.toByteArray();
+        }
+        catch (IOException e) {
+            throw new IgniteException(e);
+        }
+    }
+
+    /**
+     * Deserialize object from byte array.
+     *
+     * @param data Serialized object.
+     * @return Object.
+     */
+    public static <T> T fromBytes(byte[] data) {
+        try (ByteArrayInputStream bis = new ByteArrayInputStream(data);
+             ObjectInputStream ois = new ObjectInputStream(bis)) {
+
+            return (T)ois.readObject();
+        }
+        catch (IOException | ClassNotFoundException e) {
+            throw new IgniteException(e);
+        }
+    }
+
     /**
      * Return count of regular file in the directory (including in sub-directories)
      *
@@ -10459,6 +10504,139 @@ public static String toHexString(ByteBuffer buf) {
         return sb.toString();
     }
 
+    /**
+     * @param executorSvc Service for parallel execution.
+     * @param srcDatas List of data for parallelization.
+     * @param consumer Logic for execution of on each item of data.
+     * @param <T> Type of data.
+     * @throws ParallelExecutionException if parallel execution was failed. It contains actual exception in suppressed section.
+     */
+    public static <T> void doInParallel(int parallelismLvl, ExecutorService executorSvc, Collection<T> srcDatas,
+        IgniteThrowableConsumer<T> consumer) throws ParallelExecutionException {
+
+        List<List<T>> batches = new ArrayList<>(parallelismLvl);
+
+        for (int i = 0; i < parallelismLvl; i++)
+            batches.add(new ArrayList<>());
+
+        int i = 0;
+
+        for (T src : srcDatas) {
+            int idx = i % parallelismLvl;
+
+            List<T> batch = batches.get(idx);
+
+            batch.add(src);
+
+            i++;
+        }
+
+        List<T2<List<T>, Future<Object>>> consumerFutures = batches.stream()
+            .filter(batch -> !batch.isEmpty())
+            .map(batch -> new T2<>(
+                batch,
+                executorSvc.submit(() -> {
+                    for (T item : batch)
+                        consumer.accept(item);
+
+                    return null;
+                })))
+            .collect(Collectors.toList());
+
+        ParallelExecutionException executionE = null;
+
+        for (T2<List<T>, Future<Object>> future : consumerFutures) {
+            try {
+                future.get2().get();
+            }
+            catch (Exception e) {
+                if (executionE == null)
+                    executionE = new ParallelExecutionException("Failed during parallel execution.");
+
+                executionE.addSuppressed(e);
+
+                for (T failedData : future.get1())
+                    executionE.addFailedData(failedData);
+            }
+        }
+
+        if (executionE != null)
+            throw executionE;
+    }
+
+    /**
+     * @param executorSvc Service for parallel execution.
+     * @param srcDatas List of data for parallelization.
+     * @param consumer Logic for execution of on each item of data.
+     * @param errHnd Optionan error handler. If not {@code null}, an error of each item execution will be passed to
+     *      this handler. If error handler is not {@code null}, the exception will not be thrown from this method.
+     * @param <T> Type of data.
+     * @return List of (item, execution future) tuples.
+     * @throws IgniteCheckedException If parallel execution failed and {@code errHnd} is {@code null}.
+     */
+    public static <T> List<T2<T, Future<Object>>> doInParallel(
+        ExecutorService executorSvc,
+        Collection<T> srcDatas,
+        IgniteInClosureX<T> consumer,
+        @Nullable IgniteBiInClosure<T, Throwable> errHnd
+    ) throws IgniteCheckedException {
+        List<T2<T, Future<Object>>> consumerFutures = srcDatas.stream()
+            .map(item -> new T2<>(
+                item,
+                executorSvc.submit(() -> {
+                    consumer.apply(item);
+
+                    return null;
+                })))
+            .collect(Collectors.toList());
+
+        IgniteCheckedException composite = null;
+
+        for (T2<T, Future<Object>> tup : consumerFutures) {
+            try {
+                getUninterruptibly(tup.get2());
+            }
+            catch (ExecutionException e) {
+                if (errHnd != null)
+                    errHnd.apply(tup.get1(), e.getCause());
+                else {
+                    if (composite == null)
+                        composite = new IgniteCheckedException("Failed to execute one of the tasks " +
+                            "(see suppressed exception for details)");
+
+                    composite.addSuppressed(e.getCause());
+                }
+            }
+        }
+
+        if (composite != null)
+            throw composite;
+
+        return consumerFutures;
+    }
+
+    /**
+     * @param fut Future to wait for completion.
+     * @throws ExecutionException If the future
+     */
+    private static void getUninterruptibly(Future fut) throws ExecutionException {
+        boolean interrupted = false;
+
+        while (true) {
+            try {
+                fut.get();
+
+                break;
+            }
+            catch (InterruptedException e) {
+                interrupted = true;
+            }
+        }
+
+        if (interrupted)
+            Thread.currentThread().interrupt();
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java b/modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java
new file mode 100644
index 0000000000000..e3fea1ec063e6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import java.util.concurrent.locks.Lock;
+import java.util.function.Supplier;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.util.lang.IgniteThrowableRunner;
+
+/**
+ * Class for avoid multiple initialization of specific value from various threads.
+ */
+public class InitializationProtector {
+    /** Default striped lock concurrency level. */
+    private static final int DEFAULT_CONCURRENCY_LEVEL = 20;
+    /** Striped lock. */
+    GridStripedLock stripedLock = new GridStripedLock(DEFAULT_CONCURRENCY_LEVEL);
+
+    /**
+     * @param protectedKey Unique value by which initialization code should be run only one time.
+     * @param initializedVal Supplier for given already initialized value if it exist or null as sign that
+     * initialization required.
+     * @param initializationCode Code for initialization value corresponding protectedKey.
+     * @param <T> Type of initialization value.
+     * @return Initialized value.
+     * @throws IgniteCheckedException if initialization was failed.
+     */
+    public <T> T protect(Object protectedKey, Supplier<T> initializedVal,
+        IgniteThrowableRunner initializationCode) throws IgniteCheckedException {
+        T value = initializedVal.get();
+
+        if (value != null)
+            return value;
+
+        Lock lock = stripedLock.getLock(protectedKey.hashCode() % stripedLock.concurrencyLevel());
+
+        lock.lock();
+        try {
+            value = initializedVal.get();
+
+            if (value != null)
+                return value;
+
+            initializationCode.run();
+
+            return initializedVal.get();
+        }
+        finally {
+            lock.unlock();
+        }
+    }
+
+    /**
+     * It method allows to avoid simultaneous initialization from various threads.
+     * Garantee protection only for first call.
+     *
+     * @param protectedKey Unique value by which initialization code should be run only from one thread in one time.
+     * @param initializationCode Code for initialization value corresponding protectedKey.
+     * @throws IgniteCheckedException if initialization was failed.
+     */
+    public void protect(Object protectedKey, IgniteThrowableRunner initializationCode) throws IgniteCheckedException {
+        protect(protectedKey, () -> null, initializationCode);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/ParallelExecutionException.java b/modules/core/src/main/java/org/apache/ignite/internal/util/ParallelExecutionException.java
new file mode 100644
index 0000000000000..e7420052d9f95
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/ParallelExecutionException.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * This exception appears during execution of parallel calculation.
+ */
+public class ParallelExecutionException extends IgniteCheckedException {
+    /** */
+    private static final long serialVersionUID = 0L;
+    /** List of failed data. */
+    private final List<Object> failedDatas = new ArrayList<>();
+
+    /** {@inheritDoc} */
+    public ParallelExecutionException(String msg) {
+        super(msg);
+    }
+
+    /**
+     * @param executionData Data for which execution was failed.
+     */
+    public void addFailedData(Object executionData) {
+        failedDatas.add(executionData);
+    }
+
+    /**
+     * @return Datas for which execution was failed.
+     */
+    public List<Object> getFailedDatas() {
+        return failedDatas;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java
new file mode 100644
index 0000000000000..bb746a355304b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.lang;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Represents an operation that accepts a single input argument and returns no result. Unlike most other functional
+ * interfaces, {@code IgniteThrowableConsumer} is expected to operate via side-effects.
+ *
+ * @param <E> Type of closure parameter.
+ */
+public interface IgniteThrowableConsumer<E> extends Serializable {
+    /**
+     * Consumer body.
+     *
+     * @param e Consumer parameter.
+     * @throws IgniteCheckedException if body execution was failed.
+     */
+    public void accept(E e) throws IgniteCheckedException;
+}
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableRunner.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableRunner.java
new file mode 100644
index 0000000000000..a5c95e1a3e575
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableRunner.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.lang;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Represents a throwable runner.
+ */
+public interface IgniteThrowableRunner {
+    /**
+     * Execute a body.
+     */
+    void run() throws IgniteCheckedException;
+}

From f4bd8856a4cd638d07be3428705733f44c296320 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Mon, 8 Oct 2018 14:31:45 +0300
Subject: [PATCH 424/543] GG-14332 : Partial client PME optimizations

---
 .../affinity/GridAffinityAssignmentCache.java |  33 ++--
 .../cache/GridCacheAffinityManager.java       |  14 +-
 .../cache/GridCacheGroupIdMessage.java        |   6 +-
 .../processors/cache/GridCacheIdMessage.java  |   6 +-
 .../processors/cache/GridCacheIoManager.java  |   8 +-
 .../processors/cache/GridCacheMessage.java    |  34 +++-
 .../GridCachePartitionExchangeManager.java    | 149 ++++++++++++------
 .../GridChangeGlobalStateMessageResponse.java |  10 +-
 .../GridCacheTtlUpdateRequest.java            |  26 +--
 .../GridCacheTxRecoveryRequest.java           |  26 +--
 .../GridCacheTxRecoveryResponse.java          |  14 +-
 .../GridDistributedBaseMessage.java           |  16 +-
 .../GridDistributedLockRequest.java           |  54 +++----
 .../GridDistributedLockResponse.java          |  14 +-
 .../GridDistributedTxFinishRequest.java       |  58 +++----
 .../GridDistributedTxFinishResponse.java      |  18 +--
 .../GridDistributedTxPrepareRequest.java      |  54 +++----
 .../GridDistributedTxPrepareResponse.java     |  14 +-
 .../GridDistributedUnlockRequest.java         |   6 +-
 .../dht/ClientCacheDhtTopologyFuture.java     |  10 ++
 .../dht/GridDhtAffinityAssignmentRequest.java |  14 +-
 .../GridDhtAffinityAssignmentResponse.java    |  22 +--
 .../distributed/dht/GridDhtCacheAdapter.java  |  30 ++--
 .../distributed/dht/GridDhtLockRequest.java   |  42 ++---
 .../distributed/dht/GridDhtLockResponse.java  |  18 +--
 .../dht/GridDhtTopologyFuture.java            |  12 ++
 .../dht/GridDhtTransactionalCacheAdapter.java |  18 +--
 .../dht/GridDhtTxFinishRequest.java           |  26 +--
 .../dht/GridDhtTxFinishResponse.java          |  14 +-
 .../GridDhtTxOnePhaseCommitAckRequest.java    |   6 +-
 .../dht/GridDhtTxPrepareRequest.java          |  54 +++----
 .../dht/GridDhtTxPrepareResponse.java         |  22 +--
 .../distributed/dht/GridDhtUnlockRequest.java |   6 +-
 .../GridDhtAtomicAbstractUpdateRequest.java   |  38 ++---
 .../dht/atomic/GridDhtAtomicCache.java        |  21 +--
 .../GridDhtAtomicDeferredUpdateResponse.java  |   6 +-
 .../dht/atomic/GridDhtAtomicNearResponse.java |  22 +--
 .../GridDhtAtomicSingleUpdateRequest.java     |  18 +--
 .../atomic/GridDhtAtomicUpdateRequest.java    |  66 ++++----
 .../atomic/GridDhtAtomicUpdateResponse.java   |  18 +--
 .../GridNearAtomicAbstractUpdateRequest.java  |  30 ++--
 .../GridNearAtomicCheckUpdateRequest.java     |  10 +-
 .../GridNearAtomicFullUpdateRequest.java      |  38 ++---
 ...idNearAtomicSingleUpdateFilterRequest.java |   6 +-
 ...idNearAtomicSingleUpdateInvokeRequest.java |  10 +-
 .../GridNearAtomicSingleUpdateRequest.java    |  10 +-
 .../atomic/GridNearAtomicUpdateResponse.java  |  30 ++--
 .../preloader/GridDhtForceKeysRequest.java    |  18 +--
 .../preloader/GridDhtForceKeysResponse.java   |  22 +--
 .../GridDhtPartitionDemandLegacyMessage.java  |  36 ++---
 .../GridDhtPartitionDemandMessage.java        |  42 ++---
 .../GridDhtPartitionSupplyMessage.java        |  48 +++---
 .../GridDhtPartitionSupplyMessageV2.java      |   6 +-
 .../GridDhtPartitionsAbstractMessage.java     |  14 +-
 .../GridDhtPartitionsExchangeFuture.java      |  31 +++-
 .../GridDhtPartitionsFullMessage.java         |  50 +++---
 .../GridDhtPartitionsSingleMessage.java       |  38 ++---
 .../GridDhtPartitionsSingleRequest.java       |   6 +-
 .../GridDhtPartitionTopologyImpl.java         |  48 ++++--
 .../distributed/near/GridNearGetRequest.java  |  46 +++---
 .../distributed/near/GridNearGetResponse.java |  30 ++--
 .../distributed/near/GridNearLockRequest.java |  38 ++---
 .../near/GridNearLockResponse.java            |  26 +--
 .../near/GridNearSingleGetRequest.java        |  34 ++--
 .../near/GridNearSingleGetResponse.java       |  22 +--
 .../near/GridNearTxFinishRequest.java         |   6 +-
 .../near/GridNearTxFinishResponse.java        |  14 +-
 .../near/GridNearTxPrepareRequest.java        |  26 +--
 .../near/GridNearTxPrepareResponse.java       |  42 ++---
 .../near/GridNearUnlockRequest.java           |   2 +-
 .../cache/query/GridCacheQueryRequest.java    |  82 +++++-----
 .../cache/query/GridCacheQueryResponse.java   |  26 +--
 .../CacheContinuousQueryBatchAck.java         |  16 +-
 .../cache/transactions/IgniteTxHandler.java   |  14 +-
 .../cache/transactions/TxLocksRequest.java    |  10 +-
 .../cache/transactions/TxLocksResponse.java   |  18 +--
 76 files changed, 1082 insertions(+), 906 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index 5b985c262bd6f..e84954b33c257 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -469,7 +469,7 @@ public AffinityTopologyVersion lastVersion() {
      * @return Affinity assignment.
      */
     public List<List<ClusterNode>> assignments(AffinityTopologyVersion topVer) {
-        AffinityAssignment aff = cachedAffinity(topVer);
+        AffinityAssignment aff = cachedAffinity(topVer, AffinityTopologyVersion.NONE);
 
         return aff.assignment();
     }
@@ -536,7 +536,7 @@ public int partitions() {
      */
     public List<ClusterNode> nodes(int part, AffinityTopologyVersion topVer) {
         // Resolve cached affinity nodes.
-        return cachedAffinity(topVer).get(part);
+        return cachedAffinity(topVer, AffinityTopologyVersion.NONE).get(part);
     }
 
     /**
@@ -547,7 +547,7 @@ public List<ClusterNode> nodes(int part, AffinityTopologyVersion topVer) {
      * @return Primary partitions for specified node ID.
      */
     public Set<Integer> primaryPartitions(UUID nodeId, AffinityTopologyVersion topVer) {
-        return cachedAffinity(topVer).primaryPartitions(nodeId);
+        return cachedAffinity(topVer, AffinityTopologyVersion.NONE).primaryPartitions(nodeId);
     }
 
     /**
@@ -558,7 +558,7 @@ public Set<Integer> primaryPartitions(UUID nodeId, AffinityTopologyVersion topVe
      * @return Backup partitions for specified node ID.
      */
     public Set<Integer> backupPartitions(UUID nodeId, AffinityTopologyVersion topVer) {
-        return cachedAffinity(topVer).backupPartitions(nodeId);
+        return cachedAffinity(topVer, AffinityTopologyVersion.NONE).backupPartitions(nodeId);
     }
 
     /**
@@ -618,17 +618,31 @@ public AffinityAssignment readyAffinity(AffinityTopologyVersion topVer) {
      * @return Cached affinity.
      */
     public AffinityAssignment cachedAffinity(AffinityTopologyVersion topVer) {
+        return cachedAffinity(topVer, topVer);
+    }
+
+    /**
+     * Get cached affinity for specified topology version.
+     *
+     * @param topVer Topology version.
+     * @return Cached affinity.
+     */
+    public AffinityAssignment cachedAffinity(AffinityTopologyVersion topVer, AffinityTopologyVersion lastAffChangeTopVer) {
         if (topVer.equals(AffinityTopologyVersion.NONE))
-            topVer = lastVersion();
-        else
-            awaitTopologyVersion(topVer);
+            topVer = lastAffChangeTopVer = lastVersion();
+        else {
+            if (lastAffChangeTopVer.equals(AffinityTopologyVersion.NONE))
+                lastAffChangeTopVer = topVer;
+
+            awaitTopologyVersion(lastAffChangeTopVer);
+        }
 
         assert topVer.topologyVersion() >= 0 : topVer;
 
         AffinityAssignment cache = head.get();
 
         if (!cache.topologyVersion().equals(topVer)) {
-            cache = affCache.get(topVer);
+            cache = affCache.get(lastAffChangeTopVer);
 
             if (cache == null) {
                 throw new IllegalStateException("Getting affinity for topology version earlier than affinity is " +
@@ -641,7 +655,8 @@ public AffinityAssignment cachedAffinity(AffinityTopologyVersion topVer) {
             }
         }
 
-        assert cache.topologyVersion().equals(topVer) : "Invalid cached affinity: " + cache;
+        assert cache.topologyVersion().compareTo(lastAffChangeTopVer) >= 0 &&
+            cache.topologyVersion().compareTo(topVer) <= 0 : "Invalid cached affinity: " + cache;
 
         return cache;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
index c9ee38cf898d6..03f95aef80b50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
@@ -232,15 +232,25 @@ public List<ClusterNode> nodesByPartition(int part, AffinityTopologyVersion topV
      * @return Affinity assignment.
      */
     public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
+        return assignment(topVer, topVer);
+    }
+
+    /**
+     * Get affinity assignment for the given topology version.
+     *
+     * @param topVer Topology version.
+     * @return Affinity assignment.
+     */
+    public AffinityAssignment assignment(AffinityTopologyVersion topVer, AffinityTopologyVersion lastAffChangedTopVer) {
         if (cctx.isLocal())
-            topVer = LOC_CACHE_TOP_VER;
+            topVer = lastAffChangedTopVer = LOC_CACHE_TOP_VER;
 
         GridAffinityAssignmentCache aff0 = aff;
 
         if (aff0 == null)
             throw new IgniteException(FAILED_TO_FIND_CACHE_ERR_MSG + cctx.name());
 
-        return aff0.cachedAffinity(topVer);
+        return aff0.cachedAffinity(topVer, lastAffChangedTopVer);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGroupIdMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGroupIdMessage.java
index 09c143b0c0dd3..bfdce35e86e62 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGroupIdMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheGroupIdMessage.java
@@ -50,7 +50,7 @@ public int groupId() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 3;
+        return 4;
     }
 
     /** {@inheritDoc} */
@@ -68,7 +68,7 @@ public int groupId() {
         }
 
         switch (writer.state()) {
-            case 2:
+            case 3:
                 if (!writer.writeInt("grpId", grpId))
                     return false;
 
@@ -90,7 +90,7 @@ public int groupId() {
             return false;
 
         switch (reader.state()) {
-            case 2:
+            case 3:
                 grpId = reader.readInt("grpId");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIdMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIdMessage.java
index 6c20bdd15bdd6..e0944397ecf3d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIdMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIdMessage.java
@@ -52,7 +52,7 @@ public void cacheId(int cacheId) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 3;
+        return 4;
     }
 
     /** {@inheritDoc} */
@@ -70,7 +70,7 @@ public void cacheId(int cacheId) {
         }
 
         switch (writer.state()) {
-            case 2:
+            case 3:
                 if (!writer.writeInt("cacheId", cacheId))
                     return false;
 
@@ -92,7 +92,7 @@ public void cacheId(int cacheId) {
             return false;
 
         switch (reader.state()) {
-            case 2:
+            case 3:
                 cacheId = reader.readInt("cacheId");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 2e66e5bfc3fe4..9b7d2a01b0501 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -223,8 +223,9 @@ else if (desc.receivedFromStartVersion() != null)
             else {
                 AffinityTopologyVersion locAffVer = cctx.exchange().readyAffinityVersion();
                 AffinityTopologyVersion rmtAffVer = cacheMsg.topologyVersion();
+                AffinityTopologyVersion lastAffChangedVer = cacheMsg.lastAffinityChangedTopologyVersion();
 
-                if (locAffVer.compareTo(rmtAffVer) < 0) {
+                if (locAffVer.compareTo(lastAffChangedVer) < 0) {
                     IgniteLogger log = cacheMsg.messageLogger(cctx);
 
                     if (log.isDebugEnabled()) {
@@ -234,12 +235,13 @@ else if (desc.receivedFromStartVersion() != null)
 
                         msg0.append(", locTopVer=").append(locAffVer).
                             append(", rmtTopVer=").append(rmtAffVer).
+                            append(", lastAffChangedVer=").append(lastAffChangedVer).
                             append(']');
 
                         log.debug(msg0.toString());
                     }
 
-                    fut = cctx.exchange().affinityReadyFuture(rmtAffVer);
+                    fut = cctx.exchange().affinityReadyFuture(lastAffChangedVer);
                 }
             }
 
@@ -1155,6 +1157,8 @@ public boolean checkNodeLeft(UUID nodeId, IgniteCheckedException sndErr, boolean
     public void send(ClusterNode node, GridCacheMessage msg, byte plc) throws IgniteCheckedException {
         assert !node.isLocal() : node;
 
+        msg.lastAffinityChangedTopologyVersion(cctx.exchange().lastAffinityChangedTopologyVersion(msg.topologyVersion()));
+
         if (!onSend(msg, node.id()))
             return;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
index 11916e92270fd..3d13cf3b992ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
@@ -68,6 +68,10 @@ public abstract class GridCacheMessage implements Message {
     @GridToStringInclude
     private GridDeploymentInfoBean depInfo;
 
+    /** */
+    @GridToStringInclude
+    @Nullable private AffinityTopologyVersion lastAffChangedTopVer;
+
     /** */
     @GridDirectTransient
     protected boolean addDepInfo;
@@ -184,6 +188,20 @@ public AffinityTopologyVersion topologyVersion() {
         return AffinityTopologyVersion.NONE;
     }
 
+    /**
+     * @return
+     */
+    public AffinityTopologyVersion lastAffinityChangedTopologyVersion() {
+        if (lastAffChangedTopVer == null)
+            return topologyVersion();
+
+        return lastAffChangedTopVer;
+    }
+
+    public void lastAffinityChangedTopologyVersion(AffinityTopologyVersion topVer) {
+        lastAffChangedTopVer = topVer;
+    }
+
     /**
      *  Deployment enabled flag indicates whether deployment info has to be added to this message.
      *
@@ -637,7 +655,7 @@ public IgniteLogger messageLogger(GridCacheSharedContext ctx) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 2;
+        return 3;
     }
 
     /** {@inheritDoc} */
@@ -659,6 +677,12 @@ public IgniteLogger messageLogger(GridCacheSharedContext ctx) {
                 writer.incrementState();
 
             case 1:
+                if (!writer.writeMessage("lastAffChangedTopVer", lastAffChangedTopVer))
+                    return false;
+
+                writer.incrementState();
+
+            case 2:
                 if (!writer.writeLong("msgId", msgId))
                     return false;
 
@@ -686,6 +710,14 @@ public IgniteLogger messageLogger(GridCacheSharedContext ctx) {
                 reader.incrementState();
 
             case 1:
+                lastAffChangedTopVer = reader.readMessage("lastAffChangedTopVer");
+
+                if (!reader.isLastRead())
+                    return false;
+
+                reader.incrementState();
+
+            case 2:
                 msgId = reader.readLong("msgId");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 30a05e721078b..548d6cc8114e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -98,7 +98,6 @@
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
 import org.apache.ignite.internal.processors.query.schema.SchemaNodeLeaveExchangeWorkerTask;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
-import org.apache.ignite.internal.util.GridListSet;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
@@ -175,7 +174,7 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     @Nullable private volatile GridDhtPartitionsExchangeFuture lastInitializedFut;
 
     /** */
-    private final AtomicReference<GridDhtTopologyFuture> lastFinishedFut = new AtomicReference<>();
+    private final AtomicReference<GridDhtPartitionsExchangeFuture> lastFinishedFut = new AtomicReference<>();
 
     /** */
     private final ConcurrentMap<AffinityTopologyVersion, AffinityReadyFuture> readyFuts =
@@ -878,11 +877,11 @@ public GridDhtPartitionsExchangeFuture lastTopologyFuture() {
     /**
      * @param fut Finished future.
      */
-    public void lastFinishedFuture(GridDhtTopologyFuture fut) {
+    public void lastFinishedFuture(GridDhtPartitionsExchangeFuture fut) {
         assert fut != null && fut.isDone() : fut;
 
         while (true) {
-            GridDhtTopologyFuture cur = lastFinishedFut.get();
+            GridDhtPartitionsExchangeFuture cur = lastFinishedFut.get();
 
             if (cur == null || fut.topologyVersion().compareTo(cur.topologyVersion()) > 0) {
                 if (lastFinishedFut.compareAndSet(cur, fut))
@@ -900,7 +899,8 @@ public void lastFinishedFuture(GridDhtTopologyFuture fut) {
     @Nullable public IgniteInternalFuture<?> affinityReadyFuture(AffinityTopologyVersion ver) {
         GridDhtPartitionsExchangeFuture lastInitializedFut0 = lastInitializedFut;
 
-        if (lastInitializedFut0 != null && lastInitializedFut0.initialVersion().compareTo(ver) == 0) {
+        if (lastInitializedFut0 != null && lastInitializedFut0.initialVersion().compareTo(ver) == 0
+            && lastInitializedFut0.changedAffinity()) {
             if (log.isTraceEnabled())
                 log.trace("Return lastInitializedFut for topology ready future " +
                     "[ver=" + ver + ", fut=" + lastInitializedFut0 + ']');
@@ -972,6 +972,23 @@ public boolean hasPendingExchange() {
         return exchWorker.hasPendingExchange();
     }
 
+    /**
+     *
+     * @param topVer
+     * @return
+     */
+    public AffinityTopologyVersion lastAffinityChangedTopologyVersion(AffinityTopologyVersion topVer) {
+        if (topVer.topologyVersion() <= 0)
+            return topVer;
+
+        GridDhtPartitionsExchangeFuture exchFut = exchFuts.get(topVer);
+
+        if (exchFut == null || exchFut.firstEvent() == null || exchFut.changedAffinity())
+            return topVer;
+
+        return exchFut.lastAffinityChangeTopologyVersion();
+    }
+
     /**
      * @param evt Discovery event.
      * @return Affinity topology version.
@@ -1420,7 +1437,7 @@ private GridDhtPartitionsExchangeFuture exchangeFuture(
     ) {
         GridDhtPartitionsExchangeFuture fut;
 
-        GridDhtPartitionsExchangeFuture old = exchFuts.addx(
+        GridDhtPartitionsExchangeFuture old = exchFuts.putIfAbsent(exchId.topologyVersion(),
             fut = new GridDhtPartitionsExchangeFuture(cctx, busyLock, exchId, exchActions, affChangeMsg));
 
         if (old != null) {
@@ -1454,30 +1471,10 @@ public void onExchangeDone(AffinityTopologyVersion topVer, AffinityTopologyVersi
         if (log.isDebugEnabled())
             log.debug("Exchange done [topVer=" + topVer + ", err=" + err + ']');
 
-        if (err == null) {
+        if (err == null)
             exchFuts.readyTopVer(topVer);
 
-            for (Map.Entry<AffinityTopologyVersion, AffinityReadyFuture> entry : readyFuts.entrySet()) {
-                if (entry.getKey().compareTo(topVer) <= 0) {
-                    if (log.isDebugEnabled())
-                        log.debug("Completing created topology ready future " +
-                            "[ver=" + topVer + ", fut=" + entry.getValue() + ']');
-
-                    entry.getValue().onDone(topVer);
-                }
-            }
-        }
-        else {
-            for (Map.Entry<AffinityTopologyVersion, AffinityReadyFuture> entry : readyFuts.entrySet()) {
-                if (entry.getKey().compareTo(initTopVer) <= 0) {
-                    if (log.isDebugEnabled())
-                        log.debug("Completing created topology ready future with error " +
-                            "[ver=" + entry.getKey() + ", fut=" + entry.getValue() + ']');
-
-                    entry.getValue().onDone(err);
-                }
-            }
-        }
+        completeAffReadyFuts(err == null ? topVer : initTopVer, err);
 
         ExchangeFutureSet exchFuts0 = exchFuts;
 
@@ -1496,6 +1493,28 @@ public void onExchangeDone(AffinityTopologyVersion topVer, AffinityTopologyVersi
         }
     }
 
+    /** */
+    private void completeAffReadyFuts(AffinityTopologyVersion topVer, @Nullable Throwable err) {
+        for (Map.Entry<AffinityTopologyVersion, AffinityReadyFuture> entry : readyFuts.entrySet()) {
+            if (entry.getKey().compareTo(topVer) <= 0) {
+                if (err == null) {
+                    if (log.isDebugEnabled())
+                        log.debug("Completing created topology ready future " +
+                            "[ver=" + topVer + ", fut=" + entry.getValue() + ']');
+
+                    entry.getValue().onDone(topVer);
+                }
+                else {
+                    if (log.isDebugEnabled())
+                        log.debug("Completing created topology ready future with error " +
+                            "[ver=" + entry.getKey() + ", fut=" + entry.getValue() + ']');
+
+                    entry.getValue().onDone(err);
+                }
+            }
+        }
+    }
+
     /**
      * @param fut Future.
      * @return {@code True} if added.
@@ -2087,6 +2106,13 @@ public boolean mergeExchangesOnCoordinator(GridDhtPartitionsExchangeFuture curFu
                         break;
                     }
 
+                    if (!fut.changedAffinity()) {
+                        if (log.isInfoEnabled())
+                            log.info("Stop merge, no-affinity exchange found: " + evt);
+
+                        break;
+                    }
+
                     ClusterNode node = evt.eventNode();
 
                     if (!curFut.context().supportsMergeExchanges(node)) {
@@ -2220,6 +2246,35 @@ private boolean exchangeInProgress() {
         return false;
     }
 
+    /** */
+    public boolean affinityChanged(AffinityTopologyVersion from, AffinityTopologyVersion to) {
+        Collection<GridDhtPartitionsExchangeFuture> history = exchFuts.values();
+
+        boolean fromFound = false;
+
+        for (GridDhtPartitionsExchangeFuture fut : history) {
+            if (!fromFound) {
+                int cmp = fut.initialVersion().compareTo(from);
+
+                if (cmp > 0) // We don't have history, so return true for safety
+                    return true;
+                else if (cmp == 0)
+                    fromFound = true;
+                else if (fut.isDone() && fut.topologyVersion().compareTo(from) >= 0)
+                    return true; // Temporary solution for merge exchange case
+            }
+            else {
+                if (fut.changedAffinity())
+                    return true;
+
+                if (fut.initialVersion().compareTo(to) >= 0)
+                    return false;
+            }
+        }
+
+        return true;
+    }
+
     /**
      * Exchange future thread. All exchanges happen only by one thread and next
      * exchange will not start until previous one completes.
@@ -2565,7 +2620,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 crd = newCrd = !srvNodes.isEmpty() && srvNodes.get(0).isLocal();
                             }
 
-                            exchFut.init(newCrd);
+                            exchFut.init(lastFinishedFut.get(), newCrd);
 
                             int dumpCnt = 0;
 
@@ -2849,7 +2904,7 @@ public boolean started() {
     /**
      *
      */
-    private static class ExchangeFutureSet extends GridListSet<GridDhtPartitionsExchangeFuture> {
+    private static class ExchangeFutureSet extends ConcurrentSkipListMap<AffinityTopologyVersion, GridDhtPartitionsExchangeFuture> {
         /** */
         private static final long serialVersionUID = 0L;
 
@@ -2866,21 +2921,15 @@ private static class ExchangeFutureSet extends GridListSet<GridDhtPartitionsExch
          * @param histSize Max history size.
          */
         private ExchangeFutureSet(int histSize) {
-            super(new Comparator<GridDhtPartitionsExchangeFuture>() {
+            super(new Comparator<AffinityTopologyVersion>() {
                 @Override public int compare(
-                    GridDhtPartitionsExchangeFuture f1,
-                    GridDhtPartitionsExchangeFuture f2
+                    AffinityTopologyVersion t1,
+                    AffinityTopologyVersion t2
                 ) {
-                    AffinityTopologyVersion t1 = f1.exchangeId().topologyVersion();
-                    AffinityTopologyVersion t2 = f2.exchangeId().topologyVersion();
-
-                    assert t1.topologyVersion() > 0;
-                    assert t2.topologyVersion() > 0;
-
                     // Reverse order.
                     return t2.compareTo(t1);
                 }
-            }, /*not strict*/false);
+            });
 
             this.histSize = histSize;
         }
@@ -2889,17 +2938,20 @@ private ExchangeFutureSet(int histSize) {
          * @param fut Future to add.
          * @return {@code True} if added.
          */
-        @Override public synchronized GridDhtPartitionsExchangeFuture addx(
+        @Override public synchronized GridDhtPartitionsExchangeFuture putIfAbsent(
+            AffinityTopologyVersion topVer,
             GridDhtPartitionsExchangeFuture fut) {
-            GridDhtPartitionsExchangeFuture cur = super.addx(fut);
+            assert topVer.topologyVersion() > 0;
+
+            GridDhtPartitionsExchangeFuture cur = super.putIfAbsent(topVer, fut);
 
             while (size() > histSize) {
-                GridDhtPartitionsExchangeFuture last = last();
+                GridDhtPartitionsExchangeFuture last = lastEntry().getValue();
 
                 if (!last.isDone() || Objects.equals(last.initialVersion(), readyTopVer()))
                     break;
 
-                removeLast();
+                remove(last.initialVersion());
             }
 
             // Return the value in the set.
@@ -2930,17 +2982,16 @@ public boolean readyTopVer(AffinityTopologyVersion readyTopVersion) {
         }
 
         /** {@inheritDoc} */
-        @Nullable @Override public synchronized GridDhtPartitionsExchangeFuture removex(
-            GridDhtPartitionsExchangeFuture val) {
-
-            return super.removex(val);
+        @Nullable @Override public synchronized GridDhtPartitionsExchangeFuture remove(
+            Object topVer) {
+            return super.remove(topVer);
         }
 
         /**
          * @return Values.
          */
         @Override public synchronized List<GridDhtPartitionsExchangeFuture> values() {
-            return super.values();
+            return new ArrayList<>(super.values());
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridChangeGlobalStateMessageResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridChangeGlobalStateMessageResponse.java
index e49be4934eeb9..4cf9e2388229c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridChangeGlobalStateMessageResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridChangeGlobalStateMessageResponse.java
@@ -116,13 +116,13 @@ public Throwable getError() {
         }
 
         switch (writer.state()) {
-            case 2:
+            case 3:
                 if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 3:
+            case 4:
                 if (!writer.writeUuid("requestId", requestId))
                     return false;
 
@@ -144,7 +144,7 @@ public Throwable getError() {
             return false;
 
         switch (reader.state()) {
-            case 2:
+            case 3:
                 errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
@@ -152,7 +152,7 @@ public Throwable getError() {
 
                 reader.incrementState();
 
-            case 3:
+            case 4:
                 requestId = reader.readUuid("requestId");
 
                 if (!reader.isLastRead())
@@ -172,7 +172,7 @@ public Throwable getError() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 4;
+        return 5;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java
index c092132192c74..a1c787a932a0e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java
@@ -213,37 +213,37 @@ public List<GridCacheVersion > nearVersions() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeCollection("nearKeys", nearKeys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeCollection("nearVers", nearVers, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeLong("ttl", ttl))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeCollection("vers", vers, MessageCollectionItemType.MSG))
                     return false;
 
@@ -265,7 +265,7 @@ public List<GridCacheVersion > nearVersions() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 keys = reader.readCollection("keys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -273,7 +273,7 @@ public List<GridCacheVersion > nearVersions() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 nearKeys = reader.readCollection("nearKeys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -281,7 +281,7 @@ public List<GridCacheVersion > nearVersions() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 nearVers = reader.readCollection("nearVers", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -289,7 +289,7 @@ public List<GridCacheVersion > nearVersions() {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -297,7 +297,7 @@ public List<GridCacheVersion > nearVersions() {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 ttl = reader.readLong("ttl");
 
                 if (!reader.isLastRead())
@@ -305,7 +305,7 @@ public List<GridCacheVersion > nearVersions() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 vers = reader.readCollection("vers", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -325,7 +325,7 @@ public List<GridCacheVersion > nearVersions() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 9;
+        return 10;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryRequest.java
index 45d1f1a31bf5a..90ce2344d77ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryRequest.java
@@ -148,37 +148,37 @@ public boolean system() {
         }
 
         switch (writer.state()) {
-            case 7:
+            case 8:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeIgniteUuid("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeBoolean("nearTxCheck", nearTxCheck))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeMessage("nearXidVer", nearXidVer))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeBoolean("sys", sys))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeInt("txNum", txNum))
                     return false;
 
@@ -200,7 +200,7 @@ public boolean system() {
             return false;
 
         switch (reader.state()) {
-            case 7:
+            case 8:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -208,7 +208,7 @@ public boolean system() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 miniId = reader.readIgniteUuid("miniId");
 
                 if (!reader.isLastRead())
@@ -216,7 +216,7 @@ public boolean system() {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 nearTxCheck = reader.readBoolean("nearTxCheck");
 
                 if (!reader.isLastRead())
@@ -224,7 +224,7 @@ public boolean system() {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 nearXidVer = reader.readMessage("nearXidVer");
 
                 if (!reader.isLastRead())
@@ -232,7 +232,7 @@ public boolean system() {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 sys = reader.readBoolean("sys");
 
                 if (!reader.isLastRead())
@@ -240,7 +240,7 @@ public boolean system() {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 txNum = reader.readInt("txNum");
 
                 if (!reader.isLastRead())
@@ -260,7 +260,7 @@ public boolean system() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 13;
+        return 14;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryResponse.java
index a9ac26ba4b49c..1ef44a8f21f6b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTxRecoveryResponse.java
@@ -129,19 +129,19 @@ public boolean success() {
         }
 
         switch (writer.state()) {
-            case 7:
+            case 8:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeIgniteUuid("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeBoolean("success", success))
                     return false;
 
@@ -163,7 +163,7 @@ public boolean success() {
             return false;
 
         switch (reader.state()) {
-            case 7:
+            case 8:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -171,7 +171,7 @@ public boolean success() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 miniId = reader.readIgniteUuid("miniId");
 
                 if (!reader.isLastRead())
@@ -179,7 +179,7 @@ public boolean success() {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 success = reader.readBoolean("success");
 
                 if (!reader.isLastRead())
@@ -199,7 +199,7 @@ public boolean success() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 10;
+        return 11;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedBaseMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedBaseMessage.java
index fc209aaa956f0..8536e480489b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedBaseMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedBaseMessage.java
@@ -161,25 +161,25 @@ int keysCount() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeByteArray("candsByIdxBytes", candsByIdxBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeCollection("committedVers", committedVers, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeCollection("rolledbackVers", rolledbackVers, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeMessage("ver", ver))
                     return false;
 
@@ -201,7 +201,7 @@ int keysCount() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 candsByIdxBytes = reader.readByteArray("candsByIdxBytes");
 
                 if (!reader.isLastRead())
@@ -209,7 +209,7 @@ int keysCount() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 committedVers = reader.readCollection("committedVers", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -217,7 +217,7 @@ int keysCount() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 rolledbackVers = reader.readCollection("rolledbackVers", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -225,7 +225,7 @@ int keysCount() {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 ver = reader.readMessage("ver");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
index 25a557c324817..ca78763fc2148 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockRequest.java
@@ -366,79 +366,79 @@ public long timeout() {
         }
 
         switch (writer.state()) {
-            case 7:
+            case 8:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeBoolean("isInTx", isInTx))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeBoolean("isInvalidate", isInvalidate))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeBoolean("isRead", isRead))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeByte("isolation", isolation != null ? (byte)isolation.ordinal() : -1))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeMessage("nearXidVer", nearXidVer))
                     return false;
 
                 writer.incrementState();
 
-            case 15:
+            case 16:
                 if (!writer.writeUuid("nodeId", nodeId))
                     return false;
 
                 writer.incrementState();
 
-            case 16:
+            case 17:
                 if (!writer.writeBooleanArray("retVals", retVals))
                     return false;
 
                 writer.incrementState();
 
-            case 17:
+            case 18:
                 if (!writer.writeLong("threadId", threadId))
                     return false;
 
                 writer.incrementState();
 
-            case 18:
+            case 19:
                 if (!writer.writeLong("timeout", timeout))
                     return false;
 
                 writer.incrementState();
 
-            case 19:
+            case 20:
                 if (!writer.writeInt("txSize", txSize))
                     return false;
 
@@ -460,7 +460,7 @@ public long timeout() {
             return false;
 
         switch (reader.state()) {
-            case 7:
+            case 8:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -468,7 +468,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -476,7 +476,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 isInTx = reader.readBoolean("isInTx");
 
                 if (!reader.isLastRead())
@@ -484,7 +484,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 isInvalidate = reader.readBoolean("isInvalidate");
 
                 if (!reader.isLastRead())
@@ -492,7 +492,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 isRead = reader.readBoolean("isRead");
 
                 if (!reader.isLastRead())
@@ -500,7 +500,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 byte isolationOrd;
 
                 isolationOrd = reader.readByte("isolation");
@@ -512,7 +512,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 keys = reader.readCollection("keys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -520,7 +520,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 nearXidVer = reader.readMessage("nearXidVer");
 
                 if (!reader.isLastRead())
@@ -528,7 +528,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 15:
+            case 16:
                 nodeId = reader.readUuid("nodeId");
 
                 if (!reader.isLastRead())
@@ -536,7 +536,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 16:
+            case 17:
                 retVals = reader.readBooleanArray("retVals");
 
                 if (!reader.isLastRead())
@@ -544,7 +544,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 17:
+            case 18:
                 threadId = reader.readLong("threadId");
 
                 if (!reader.isLastRead())
@@ -552,7 +552,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 18:
+            case 19:
                 timeout = reader.readLong("timeout");
 
                 if (!reader.isLastRead())
@@ -560,7 +560,7 @@ public long timeout() {
 
                 reader.incrementState();
 
-            case 19:
+            case 20:
                 txSize = reader.readInt("txSize");
 
                 if (!reader.isLastRead())
@@ -580,7 +580,7 @@ public long timeout() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 20;
+        return 21;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockResponse.java
index 4b21896b1c051..2d4de9c8156eb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedLockResponse.java
@@ -221,19 +221,19 @@ protected int valuesSize() {
         }
 
         switch (writer.state()) {
-            case 7:
+            case 8:
                 if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeCollection("vals", vals, MessageCollectionItemType.MSG))
                     return false;
 
@@ -255,7 +255,7 @@ protected int valuesSize() {
             return false;
 
         switch (reader.state()) {
-            case 7:
+            case 8:
                 errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
@@ -263,7 +263,7 @@ protected int valuesSize() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -271,7 +271,7 @@ protected int valuesSize() {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 vals = reader.readCollection("vals", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -291,7 +291,7 @@ protected int valuesSize() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 10;
+        return 11;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java
index ea9336b2a55e3..38855a0a9e827 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java
@@ -325,85 +325,85 @@ public boolean replyRequired() {
         }
 
         switch (writer.state()) {
-            case 7:
+            case 8:
                 if (!writer.writeMessage("baseVer", baseVer))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeBoolean("commit", commit))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeMessage("commitVer", commitVer))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeBoolean("invalidate", invalidate))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeByte("plc", plc))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
-            case 15:
+            case 16:
                 if (!writer.writeByte("syncMode", syncMode != null ? (byte)syncMode.ordinal() : -1))
                     return false;
 
                 writer.incrementState();
 
-            case 16:
+            case 17:
                 if (!writer.writeBoolean("sys", sys))
                     return false;
 
                 writer.incrementState();
 
-            case 17:
+            case 18:
                 if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
-            case 18:
+            case 19:
                 if (!writer.writeLong("threadId", threadId))
                     return false;
 
                 writer.incrementState();
 
-            case 19:
+            case 20:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
                 writer.incrementState();
 
-            case 20:
+            case 21:
                 if (!writer.writeInt("txSize", txSize))
                     return false;
 
@@ -425,7 +425,7 @@ public boolean replyRequired() {
             return false;
 
         switch (reader.state()) {
-            case 7:
+            case 8:
                 baseVer = reader.readMessage("baseVer");
 
                 if (!reader.isLastRead())
@@ -433,7 +433,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 commit = reader.readBoolean("commit");
 
                 if (!reader.isLastRead())
@@ -441,7 +441,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 commitVer = reader.readMessage("commitVer");
 
                 if (!reader.isLastRead())
@@ -449,7 +449,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -457,7 +457,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -465,7 +465,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 invalidate = reader.readBoolean("invalidate");
 
                 if (!reader.isLastRead())
@@ -473,7 +473,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 plc = reader.readByte("plc");
 
                 if (!reader.isLastRead())
@@ -481,7 +481,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
@@ -489,7 +489,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 15:
+            case 16:
                 byte syncModeOrd;
 
                 syncModeOrd = reader.readByte("syncMode");
@@ -501,7 +501,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 16:
+            case 17:
                 sys = reader.readBoolean("sys");
 
                 if (!reader.isLastRead())
@@ -509,7 +509,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 17:
+            case 18:
                 taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
@@ -517,7 +517,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 18:
+            case 19:
                 threadId = reader.readLong("threadId");
 
                 if (!reader.isLastRead())
@@ -525,7 +525,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 19:
+            case 20:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -533,7 +533,7 @@ public boolean replyRequired() {
 
                 reader.incrementState();
 
-            case 20:
+            case 21:
                 txSize = reader.readInt("txSize");
 
                 if (!reader.isLastRead())
@@ -553,7 +553,7 @@ public boolean replyRequired() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 21;
+        return 22;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
index c36e6336d4f7c..5fdf970bc0f8d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishResponse.java
@@ -145,25 +145,25 @@ public IgniteUuid futureId() {
         }
 
         switch (writer.state()) {
-            case 2:
+            case 3:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 3:
+            case 4:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeInt("part", part))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeMessage("txId", txId))
                     return false;
 
@@ -185,7 +185,7 @@ public IgniteUuid futureId() {
             return false;
 
         switch (reader.state()) {
-            case 2:
+            case 3:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -193,7 +193,7 @@ public IgniteUuid futureId() {
 
                 reader.incrementState();
 
-            case 3:
+            case 4:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -201,7 +201,7 @@ public IgniteUuid futureId() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 part = reader.readInt("part");
 
                 if (!reader.isLastRead())
@@ -209,7 +209,7 @@ public IgniteUuid futureId() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 txId = reader.readMessage("txId");
 
                 if (!reader.isLastRead())
@@ -229,7 +229,7 @@ public IgniteUuid futureId() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 6;
+        return 7;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java
index 91dcd9e621de9..2d6da9c33d9a7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareRequest.java
@@ -483,79 +483,79 @@ private boolean isFlag(int mask) {
         }
 
         switch (writer.state()) {
-            case 7:
+            case 8:
                 if (!writer.writeByte("concurrency", concurrency != null ? (byte)concurrency.ordinal() : -1))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeCollection("dhtVerKeys", dhtVerKeys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeCollection("dhtVerVals", dhtVerVals, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeByte("isolation", isolation != null ? (byte)isolation.ordinal() : -1))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeByte("plc", plc))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeCollection("reads", reads, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeLong("threadId", threadId))
                     return false;
 
                 writer.incrementState();
 
-            case 15:
+            case 16:
                 if (!writer.writeLong("timeout", timeout))
                     return false;
 
                 writer.incrementState();
 
-            case 16:
+            case 17:
                 if (!writer.writeMap("txNodesMsg", txNodesMsg, MessageCollectionItemType.UUID, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 17:
+            case 18:
                 if (!writer.writeInt("txSize", txSize))
                     return false;
 
                 writer.incrementState();
 
-            case 18:
+            case 19:
                 if (!writer.writeMessage("writeVer", writeVer))
                     return false;
 
                 writer.incrementState();
 
-            case 19:
+            case 20:
                 if (!writer.writeCollection("writes", writes, MessageCollectionItemType.MSG))
                     return false;
 
@@ -577,7 +577,7 @@ private boolean isFlag(int mask) {
             return false;
 
         switch (reader.state()) {
-            case 7:
+            case 8:
                 byte concurrencyOrd;
 
                 concurrencyOrd = reader.readByte("concurrency");
@@ -589,7 +589,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 dhtVerKeys = reader.readCollection("dhtVerKeys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -597,7 +597,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 dhtVerVals = reader.readCollection("dhtVerVals", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -605,7 +605,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -613,7 +613,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 byte isolationOrd;
 
                 isolationOrd = reader.readByte("isolation");
@@ -625,7 +625,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 plc = reader.readByte("plc");
 
                 if (!reader.isLastRead())
@@ -633,7 +633,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 reads = reader.readCollection("reads", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -641,7 +641,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 threadId = reader.readLong("threadId");
 
                 if (!reader.isLastRead())
@@ -649,7 +649,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 15:
+            case 16:
                 timeout = reader.readLong("timeout");
 
                 if (!reader.isLastRead())
@@ -657,7 +657,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 16:
+            case 17:
                 txNodesMsg = reader.readMap("txNodesMsg", MessageCollectionItemType.UUID, MessageCollectionItemType.MSG, false);
 
                 if (!reader.isLastRead())
@@ -665,7 +665,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 17:
+            case 18:
                 txSize = reader.readInt("txSize");
 
                 if (!reader.isLastRead())
@@ -673,7 +673,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 18:
+            case 19:
                 writeVer = reader.readMessage("writeVer");
 
                 if (!reader.isLastRead())
@@ -681,7 +681,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 19:
+            case 20:
                 writes = reader.readCollection("writes", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -701,7 +701,7 @@ private boolean isFlag(int mask) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 20;
+        return 21;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java
index 58e94926ca9e8..c26880e10fed4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxPrepareResponse.java
@@ -178,19 +178,19 @@ public boolean isRollback() {
         }
 
         switch (writer.state()) {
-            case 7:
+            case 8:
                 if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeInt("part", part))
                     return false;
 
@@ -212,7 +212,7 @@ public boolean isRollback() {
             return false;
 
         switch (reader.state()) {
-            case 7:
+            case 8:
                 errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
@@ -220,7 +220,7 @@ public boolean isRollback() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -228,7 +228,7 @@ public boolean isRollback() {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 part = reader.readInt("part");
 
                 if (!reader.isLastRead())
@@ -248,7 +248,7 @@ public boolean isRollback() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 10;
+        return 11;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java
index ca2bdab156bf7..001eb61589569 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedUnlockRequest.java
@@ -122,7 +122,7 @@ public void addKey(KeyCacheObject key, GridCacheContext ctx) throws IgniteChecke
         }
 
         switch (writer.state()) {
-            case 7:
+            case 8:
                 if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG))
                     return false;
 
@@ -144,7 +144,7 @@ public void addKey(KeyCacheObject key, GridCacheContext ctx) throws IgniteChecke
             return false;
 
         switch (reader.state()) {
-            case 7:
+            case 8:
                 keys = reader.readCollection("keys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -164,7 +164,7 @@ public void addKey(KeyCacheObject key, GridCacheContext ctx) throws IgniteChecke
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 8;
+        return 9;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java
index 317037ba69978..56d34acdb6ed6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java
@@ -81,6 +81,16 @@ public void validate(CacheGroupContext grp, Collection<ClusterNode> topNodes) {
         return topVer;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean changedAffinity() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public AffinityTopologyVersion lastAffinityChangeTopologyVersion() {
+        return topVer;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return "ClientCacheDhtTopologyFuture [topVer=" + topVer + ']';
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java
index 44c7b88c029f3..4ae873920d231 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java
@@ -109,7 +109,7 @@ public long futureId() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 6;
+        return 7;
     }
 
     /** {@inheritDoc} */
@@ -127,19 +127,19 @@ public long futureId() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -161,7 +161,7 @@ public long futureId() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -169,7 +169,7 @@ public long futureId() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -177,7 +177,7 @@ public long futureId() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java
index 5b0de08a2ce97..63e8f74b033f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java
@@ -215,7 +215,7 @@ private List<List<UUID>> ids(List<List<ClusterNode>> assignments) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 8;
+        return 9;
     }
 
     /**
@@ -272,31 +272,31 @@ private List<List<UUID>> ids(List<List<ClusterNode>> assignments) {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeByteArray("affAssignmentIdsBytes", affAssignmentIdsBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeByteArray("idealAffAssignmentBytes", idealAffAssignmentBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeByteArray("partBytes", partBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -318,7 +318,7 @@ private List<List<UUID>> ids(List<List<ClusterNode>> assignments) {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 affAssignmentIdsBytes = reader.readByteArray("affAssignmentIdsBytes");
 
                 if (!reader.isLastRead())
@@ -326,7 +326,7 @@ private List<List<UUID>> ids(List<List<ClusterNode>> assignments) {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -334,7 +334,7 @@ private List<List<UUID>> ids(List<List<ClusterNode>> assignments) {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 idealAffAssignmentBytes = reader.readByteArray("idealAffAssignmentBytes");
 
                 if (!reader.isLastRead())
@@ -342,7 +342,7 @@ private List<List<UUID>> ids(List<List<ClusterNode>> assignments) {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 partBytes = reader.readByteArray("partBytes");
 
                 if (!reader.isLastRead())
@@ -350,7 +350,7 @@ private List<List<UUID>> ids(List<List<ClusterNode>> assignments) {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 4708219992cb9..9c00ecb25a6f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -1229,25 +1229,29 @@ private void updateTtl(GridCacheAdapter<K, V> cache,
      * @param curVer Current topology version.
      * @return {@code True} if cache affinity changed and operation should be remapped.
      */
-    protected final boolean needRemap(AffinityTopologyVersion expVer, AffinityTopologyVersion curVer) {
-        if (expVer.equals(curVer))
+    protected final boolean needRemap(AffinityTopologyVersion expVer, AffinityTopologyVersion curVer,
+        AffinityTopologyVersion lastAffChangedTopVer, Collection<KeyCacheObject> keys) {
+        if (curVer.compareTo(lastAffChangedTopVer) >= 0 && curVer.compareTo(expVer) <= 0)
             return false;
 
-        Collection<ClusterNode> cacheNodes0 = ctx.discovery().cacheGroupAffinityNodes(ctx.groupId(), expVer);
-        Collection<ClusterNode> cacheNodes1 = ctx.discovery().cacheGroupAffinityNodes(ctx.groupId(), curVer);
+        // TODO IGNITE-7164 check mvcc crd for mvcc enabled txs.
 
-        if (!cacheNodes0.equals(cacheNodes1) || ctx.affinity().affinityTopologyVersion().compareTo(curVer) < 0)
-            return true;
+        for (KeyCacheObject key : keys) {
+            assert key.partition() != -1;
 
-        try {
-            List<List<ClusterNode>> aff1 = ctx.affinity().assignments(expVer);
-            List<List<ClusterNode>> aff2 = ctx.affinity().assignments(curVer);
+            try {
+                List<ClusterNode> aff1 = ctx.affinity().assignments(expVer).get(key.partition());
+                List<ClusterNode> aff2 = ctx.affinity().assignments(curVer).get(key.partition());
 
-            return !aff1.equals(aff2);
-        }
-        catch (IllegalStateException ignored) {
-            return true;
+                if (!aff1.containsAll(aff2) || aff2.isEmpty() || !aff1.get(0).equals(aff2.get(0)))
+                    return true;
+            }
+            catch (IllegalStateException ignored) {
+                return true;
+            }
         }
+
+        return false;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
index 1ac58182ddc57..4a219f61fd470 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
@@ -363,61 +363,61 @@ public long accessTtl() {
         }
 
         switch (writer.state()) {
-            case 20:
+            case 21:
                 if (!writer.writeLong("accessTtl", accessTtl))
                     return false;
 
                 writer.incrementState();
 
-            case 21:
+            case 22:
                 if (!writer.writeBitSet("invalidateEntries", invalidateEntries))
                     return false;
 
                 writer.incrementState();
 
-            case 22:
+            case 23:
                 if (!writer.writeIgniteUuid("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 23:
+            case 24:
                 if (!writer.writeCollection("nearKeys", nearKeys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 24:
+            case 25:
                 if (!writer.writeObjectArray("ownedKeys", ownedKeys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 25:
+            case 26:
                 if (!writer.writeObjectArray("ownedValues", ownedValues, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 26:
+            case 27:
                 if (!writer.writeBitSet("preloadKeys", preloadKeys))
                     return false;
 
                 writer.incrementState();
 
-            case 27:
+            case 28:
                 if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
-            case 28:
+            case 29:
                 if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
-            case 29:
+            case 30:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -439,7 +439,7 @@ public long accessTtl() {
             return false;
 
         switch (reader.state()) {
-            case 20:
+            case 21:
                 accessTtl = reader.readLong("accessTtl");
 
                 if (!reader.isLastRead())
@@ -447,7 +447,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 21:
+            case 22:
                 invalidateEntries = reader.readBitSet("invalidateEntries");
 
                 if (!reader.isLastRead())
@@ -455,7 +455,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 22:
+            case 23:
                 miniId = reader.readIgniteUuid("miniId");
 
                 if (!reader.isLastRead())
@@ -463,7 +463,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 23:
+            case 24:
                 nearKeys = reader.readCollection("nearKeys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -471,7 +471,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 24:
+            case 25:
                 ownedKeys = reader.readObjectArray("ownedKeys", MessageCollectionItemType.MSG, KeyCacheObject.class);
 
                 if (!reader.isLastRead())
@@ -479,7 +479,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 25:
+            case 26:
                 ownedValues = reader.readObjectArray("ownedValues", MessageCollectionItemType.MSG, GridCacheVersion.class);
 
                 if (!reader.isLastRead())
@@ -487,7 +487,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 26:
+            case 27:
                 preloadKeys = reader.readBitSet("preloadKeys");
 
                 if (!reader.isLastRead())
@@ -495,7 +495,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 27:
+            case 28:
                 subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
@@ -503,7 +503,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 28:
+            case 29:
                 taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
@@ -511,7 +511,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 29:
+            case 30:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -531,7 +531,7 @@ public long accessTtl() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 30;
+        return 31;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java
index 87abd6c2ce994..63c07e82906f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockResponse.java
@@ -207,25 +207,25 @@ public Collection<GridCacheEntryInfo> preloadEntries() {
         }
 
         switch (writer.state()) {
-            case 10:
+            case 11:
                 if (!writer.writeCollection("invalidParts", invalidParts, MessageCollectionItemType.INT))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeIgniteUuid("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeCollection("nearEvicted", nearEvicted, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeCollection("preloadEntries", preloadEntries, MessageCollectionItemType.MSG))
                     return false;
 
@@ -247,7 +247,7 @@ public Collection<GridCacheEntryInfo> preloadEntries() {
             return false;
 
         switch (reader.state()) {
-            case 10:
+            case 11:
                 invalidParts = reader.readCollection("invalidParts", MessageCollectionItemType.INT);
 
                 if (!reader.isLastRead())
@@ -255,7 +255,7 @@ public Collection<GridCacheEntryInfo> preloadEntries() {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 miniId = reader.readIgniteUuid("miniId");
 
                 if (!reader.isLastRead())
@@ -263,7 +263,7 @@ public Collection<GridCacheEntryInfo> preloadEntries() {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 nearEvicted = reader.readCollection("nearEvicted", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -271,7 +271,7 @@ public Collection<GridCacheEntryInfo> preloadEntries() {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 preloadEntries = reader.readCollection("preloadEntries", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -291,7 +291,7 @@ public Collection<GridCacheEntryInfo> preloadEntries() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 14;
+        return 15;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
index 761cb45547365..14dfebbfcd5e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
@@ -86,4 +86,16 @@ public interface GridDhtTopologyFuture extends IgniteInternalFuture<AffinityTopo
         boolean read,
         @Nullable Object key,
         @Nullable Collection<?> keys);
+
+    /**
+     *
+     * @return {@code True} if this exchange changed affinity.
+     */
+    public boolean changedAffinity();
+
+    /**
+     *
+     * @return Topology version of last affinity change.
+     */
+    public AffinityTopologyVersion lastAffinityChangeTopologyVersion();
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index d5d076f8c6206..64d6acb8396dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -938,14 +938,14 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                     }
                 }
 
-                try {
-                    if (top != null && needRemap(req.topologyVersion(), top.readyTopologyVersion())) {
-                        if (log.isDebugEnabled()) {
-                            log.debug("Client topology version mismatch, need remap lock request [" +
-                                "reqTopVer=" + req.topologyVersion() +
-                                ", locTopVer=" + top.readyTopologyVersion() +
-                                ", req=" + req + ']');
-                        }
+            try {
+                if (top != null && needRemap(req.topologyVersion(), top.readyTopologyVersion(), req.lastAffinityChangedTopologyVersion(), req.keys())) {
+                    if (log.isDebugEnabled()) {
+                        log.debug("Client topology version mismatch, need remap lock request [" +
+                            "reqTopVer=" + req.topologyVersion() +
+                            ", locTopVer=" + top.readyTopologyVersion() +
+                            ", req=" + req + ']');
+                    }
 
                         GridNearLockResponse res = sendClientLockRemapResponse(nearNode,
                             req,
@@ -1044,7 +1044,7 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                     }
 
                     try {
-                        if (top != null && needRemap(req.topologyVersion(), top.readyTopologyVersion())) {
+                        if (top != null && needRemap(req.topologyVersion(), top.readyTopologyVersion(), req.lastAffinityChangedTopologyVersion(), req.keys())) {
                             if (log.isDebugEnabled()) {
                                 log.debug("Client topology version mismatch, need remap lock request [" +
                                     "reqTopVer=" + req.topologyVersion() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
index 823b5fee5ed9c..0e4d903519d92 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishRequest.java
@@ -355,37 +355,37 @@ public void needReturnValue(boolean retVal) {
         }
 
         switch (writer.state()) {
-            case 21:
+            case 22:
                 if (!writer.writeByte("isolation", isolation != null ? (byte)isolation.ordinal() : -1))
                     return false;
 
                 writer.incrementState();
 
-            case 22:
+            case 23:
                 if (!writer.writeInt("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 23:
+            case 24:
                 if (!writer.writeUuid("nearNodeId", nearNodeId))
                     return false;
 
                 writer.incrementState();
 
-            case 24:
+            case 25:
                 if (!writer.writeMessage("partUpdateCnt", partUpdateCnt))
                     return false;
 
                 writer.incrementState();
 
-            case 25:
+            case 26:
                 if (!writer.writeCollection("pendingVers", pendingVers, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 26:
+            case 27:
                 if (!writer.writeMessage("writeVer", writeVer))
                     return false;
 
@@ -407,7 +407,7 @@ public void needReturnValue(boolean retVal) {
             return false;
 
         switch (reader.state()) {
-            case 21:
+            case 22:
                 byte isolationOrd;
 
                 isolationOrd = reader.readByte("isolation");
@@ -419,7 +419,7 @@ public void needReturnValue(boolean retVal) {
 
                 reader.incrementState();
 
-            case 22:
+            case 23:
                 miniId = reader.readInt("miniId");
 
                 if (!reader.isLastRead())
@@ -427,7 +427,7 @@ public void needReturnValue(boolean retVal) {
 
                 reader.incrementState();
 
-            case 23:
+            case 24:
                 nearNodeId = reader.readUuid("nearNodeId");
 
                 if (!reader.isLastRead())
@@ -435,7 +435,7 @@ public void needReturnValue(boolean retVal) {
 
                 reader.incrementState();
 
-            case 24:
+            case 25:
                 partUpdateCnt = reader.readMessage("partUpdateCnt");
 
                 if (!reader.isLastRead())
@@ -443,7 +443,7 @@ public void needReturnValue(boolean retVal) {
 
                 reader.incrementState();
 
-            case 25:
+            case 26:
                 pendingVers = reader.readCollection("pendingVers", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -451,7 +451,7 @@ public void needReturnValue(boolean retVal) {
 
                 reader.incrementState();
 
-            case 26:
+            case 27:
                 writeVer = reader.readMessage("writeVer");
 
                 if (!reader.isLastRead())
@@ -471,7 +471,7 @@ public void needReturnValue(boolean retVal) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 27;
+        return 28;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java
index 6d717ebf904e6..d777a2201a149 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishResponse.java
@@ -173,19 +173,19 @@ public GridCacheReturn returnValue() {
         }
 
         switch (writer.state()) {
-            case 6:
+            case 7:
                 if (!writer.writeByteArray("checkCommittedErrBytes", checkCommittedErrBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeInt("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeMessage("retVal", retVal))
                     return false;
 
@@ -207,7 +207,7 @@ public GridCacheReturn returnValue() {
             return false;
 
         switch (reader.state()) {
-            case 6:
+            case 7:
                 checkCommittedErrBytes = reader.readByteArray("checkCommittedErrBytes");
 
                 if (!reader.isLastRead())
@@ -215,7 +215,7 @@ public GridCacheReturn returnValue() {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 miniId = reader.readInt("miniId");
 
                 if (!reader.isLastRead())
@@ -223,7 +223,7 @@ public GridCacheReturn returnValue() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 retVal = reader.readMessage("retVal");
 
                 if (!reader.isLastRead())
@@ -243,7 +243,7 @@ public GridCacheReturn returnValue() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 9;
+        return 10;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxOnePhaseCommitAckRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxOnePhaseCommitAckRequest.java
index 67eacd3f8c5f8..50f2e7947b1c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxOnePhaseCommitAckRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxOnePhaseCommitAckRequest.java
@@ -97,7 +97,7 @@ public Collection<GridCacheVersion> versions() {
         }
 
         switch (writer.state()) {
-            case 2:
+            case 3:
                 if (!writer.writeCollection("vers", vers, MessageCollectionItemType.MSG))
                     return false;
 
@@ -119,7 +119,7 @@ public Collection<GridCacheVersion> versions() {
             return false;
 
         switch (reader.state()) {
-            case 2:
+            case 3:
                 vers = reader.readCollection("vers", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -139,6 +139,6 @@ public Collection<GridCacheVersion> versions() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 3;
+        return 4;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
index 88da7b011ea2c..b48aa4432aa23 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
@@ -395,79 +395,79 @@ public boolean skipCompletedVersion() {
         }
 
         switch (writer.state()) {
-            case 20:
+            case 21:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 21:
+            case 22:
                 if (!writer.writeBitSet("invalidateNearEntries", invalidateNearEntries))
                     return false;
 
                 writer.incrementState();
 
-            case 22:
+            case 23:
                 if (!writer.writeInt("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 23:
+            case 24:
                 if (!writer.writeUuid("nearNodeId", nearNodeId))
                     return false;
 
                 writer.incrementState();
 
-            case 24:
+            case 25:
                 if (!writer.writeCollection("nearWrites", nearWrites, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 25:
+            case 26:
                 if (!writer.writeMessage("nearXidVer", nearXidVer))
                     return false;
 
                 writer.incrementState();
 
-            case 26:
+            case 27:
                 if (!writer.writeCollection("ownedKeys", ownedKeys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 27:
+            case 28:
                 if (!writer.writeCollection("ownedVals", ownedVals, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 28:
+            case 29:
                 if (!writer.writeBitSet("preloadKeys", preloadKeys))
                     return false;
 
                 writer.incrementState();
 
-            case 29:
+            case 30:
                 if (!writer.writeBoolean("skipCompletedVers", skipCompletedVers))
                     return false;
 
                 writer.incrementState();
 
-            case 30:
+            case 31:
                 if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
-            case 31:
+            case 32:
                 if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
-            case 32:
+            case 33:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -489,7 +489,7 @@ public boolean skipCompletedVersion() {
             return false;
 
         switch (reader.state()) {
-            case 20:
+            case 21:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -497,7 +497,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 21:
+            case 22:
                 invalidateNearEntries = reader.readBitSet("invalidateNearEntries");
 
                 if (!reader.isLastRead())
@@ -505,7 +505,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 22:
+            case 23:
                 miniId = reader.readInt("miniId");
 
                 if (!reader.isLastRead())
@@ -513,7 +513,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 23:
+            case 24:
                 nearNodeId = reader.readUuid("nearNodeId");
 
                 if (!reader.isLastRead())
@@ -521,7 +521,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 24:
+            case 25:
                 nearWrites = reader.readCollection("nearWrites", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -529,7 +529,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 25:
+            case 26:
                 nearXidVer = reader.readMessage("nearXidVer");
 
                 if (!reader.isLastRead())
@@ -537,7 +537,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 26:
+            case 27:
                 ownedKeys = reader.readCollection("ownedKeys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -545,7 +545,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 27:
+            case 28:
                 ownedVals = reader.readCollection("ownedVals", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -553,7 +553,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 28:
+            case 29:
                 preloadKeys = reader.readBitSet("preloadKeys");
 
                 if (!reader.isLastRead())
@@ -561,7 +561,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 29:
+            case 30:
                 skipCompletedVers = reader.readBoolean("skipCompletedVers");
 
                 if (!reader.isLastRead())
@@ -569,7 +569,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 30:
+            case 31:
                 subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
@@ -577,7 +577,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 31:
+            case 32:
                 taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
@@ -585,7 +585,7 @@ public boolean skipCompletedVersion() {
 
                 reader.incrementState();
 
-            case 32:
+            case 33:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -605,7 +605,7 @@ public boolean skipCompletedVersion() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 33;
+        return 34;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java
index 0c2bf8198bac6..fcb14a34c58e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareResponse.java
@@ -245,31 +245,31 @@ public void addPreloadEntry(GridCacheEntryInfo info) {
         }
 
         switch (writer.state()) {
-            case 10:
+            case 11:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeMap("invalidParts", invalidParts, MessageCollectionItemType.INT, MessageCollectionItemType.INT_ARR))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeInt("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeCollection("nearEvicted", nearEvicted, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeCollection("preloadEntries", preloadEntries, MessageCollectionItemType.MSG))
                     return false;
 
@@ -291,7 +291,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) {
             return false;
 
         switch (reader.state()) {
-            case 10:
+            case 11:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -299,7 +299,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 invalidParts = reader.readMap("invalidParts", MessageCollectionItemType.INT, MessageCollectionItemType.INT_ARR, false);
 
                 if (!reader.isLastRead())
@@ -307,7 +307,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 miniId = reader.readInt("miniId");
 
                 if (!reader.isLastRead())
@@ -315,7 +315,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 nearEvicted = reader.readCollection("nearEvicted", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -323,7 +323,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) {
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 preloadEntries = reader.readCollection("preloadEntries", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -343,7 +343,7 @@ public void addPreloadEntry(GridCacheEntryInfo info) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 15;
+        return 16;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtUnlockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtUnlockRequest.java
index 5671d7fd0f044..3bc4de01f6b17 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtUnlockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtUnlockRequest.java
@@ -113,7 +113,7 @@ public void addNearKey(KeyCacheObject key)
         }
 
         switch (writer.state()) {
-            case 8:
+            case 9:
                 if (!writer.writeCollection("nearKeys", nearKeys, MessageCollectionItemType.MSG))
                     return false;
 
@@ -135,7 +135,7 @@ public void addNearKey(KeyCacheObject key)
             return false;
 
         switch (reader.state()) {
-            case 8:
+            case 9:
                 nearKeys = reader.readCollection("nearKeys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -155,6 +155,6 @@ public void addNearKey(KeyCacheObject key)
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 9;
+        return 10;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java
index a50b68c5e5495..489fe632b5e62 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java
@@ -472,7 +472,7 @@ final boolean isFlag(int mask) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 12;
+        return 13;
     }
 
     /** {@inheritDoc} */
@@ -490,55 +490,55 @@ final boolean isFlag(int mask) {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeLong("nearFutId", nearFutId))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeUuid("nearNodeId", nearNodeId))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeByte("syncMode", syncMode != null ? (byte)syncMode.ordinal() : -1))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeMessage("writeVer", writeVer))
                     return false;
 
@@ -560,7 +560,7 @@ final boolean isFlag(int mask) {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -568,7 +568,7 @@ final boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -576,7 +576,7 @@ final boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 nearFutId = reader.readLong("nearFutId");
 
                 if (!reader.isLastRead())
@@ -584,7 +584,7 @@ final boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 nearNodeId = reader.readUuid("nearNodeId");
 
                 if (!reader.isLastRead())
@@ -592,7 +592,7 @@ final boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
@@ -600,7 +600,7 @@ final boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 byte syncModeOrd;
 
                 syncModeOrd = reader.readByte("syncMode");
@@ -612,7 +612,7 @@ final boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
@@ -620,7 +620,7 @@ final boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -628,7 +628,7 @@ final boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 writeVer = reader.readMessage("writeVer");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index df0ba8f4b0d89..ad8fa16926829 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1723,14 +1723,15 @@ private void updateAllAsyncInternal0(
 
                     boolean remap = false;
 
-                    // Do not check topology version if topology was locked on near node by
-                    // external transaction or explicit lock.
-                    if (!req.topologyLocked()) {
-                        // Can not wait for topology future since it will break
-                        // GridNearAtomicCheckUpdateRequest processing.
-                        remap = !top.topologyVersionFuture().exchangeDone() ||
-                            needRemap(req.topologyVersion(), top.readyTopologyVersion());
-                    }
+                            // Do not check topology version if topology was locked on near node by
+                            // external transaction or explicit lock.
+                            if (!req.topologyLocked()) {
+                                // Can not wait for topology future since it will break
+                                // GridNearAtomicCheckUpdateRequest processing.
+                                remap = !top.topologyVersionFuture().exchangeDone() ||
+                                    needRemap(req.topologyVersion(), top.readyTopologyVersion(),
+                                        req.lastAffinityChangedTopologyVersion(), req.keys());
+                            }
 
                     if (!remap) {
                         DhtAtomicUpdateResult updRes = update(node, locked, req, res);
@@ -2394,7 +2395,7 @@ private DhtAtomicUpdateResult updateSingle(
 
         boolean intercept = ctx.config().getInterceptor() != null;
 
-        AffinityAssignment affAssignment = ctx.affinity().assignment(topVer);
+        AffinityAssignment affAssignment = ctx.affinity().assignment(topVer, req.lastAffinityChangedTopologyVersion());
 
         // Avoid iterator creation.
         for (int i = 0; i < req.size(); i++) {
@@ -2645,7 +2646,7 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) {
 
             boolean intercept = ctx.config().getInterceptor() != null;
 
-            AffinityAssignment affAssignment = ctx.affinity().assignment(topVer);
+            AffinityAssignment affAssignment = ctx.affinity().assignment(topVer, req.lastAffinityChangedTopologyVersion());
 
             // Avoid iterator creation.
             for (int i = 0; i < entries.size(); i++) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java
index 0c069da80082e..ee5eac15a6c45 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicDeferredUpdateResponse.java
@@ -119,7 +119,7 @@ GridLongList futureIds() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeMessage("futIds", futIds))
                     return false;
 
@@ -141,7 +141,7 @@ GridLongList futureIds() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 futIds = reader.readMessage("futIds");
 
                 if (!reader.isLastRead())
@@ -161,7 +161,7 @@ GridLongList futureIds() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 4;
+        return 5;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicNearResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicNearResponse.java
index 71d23216f0e2b..8f11ead53adff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicNearResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicNearResponse.java
@@ -171,7 +171,7 @@ public long futureId() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 8;
+        return 9;
     }
 
     /** {@inheritDoc} */
@@ -210,31 +210,31 @@ public long futureId() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeMessage("errs", errs))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeInt("partId", partId))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeUuid("primaryId", primaryId))
                     return false;
 
@@ -256,7 +256,7 @@ public long futureId() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 errs = reader.readMessage("errs");
 
                 if (!reader.isLastRead())
@@ -264,7 +264,7 @@ public long futureId() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -272,7 +272,7 @@ public long futureId() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -280,7 +280,7 @@ public long futureId() {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 partId = reader.readInt("partId");
 
                 if (!reader.isLastRead())
@@ -288,7 +288,7 @@ public long futureId() {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 primaryId = reader.readUuid("primaryId");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
index 0ade243ac8da0..392ffc54b9bec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicSingleUpdateRequest.java
@@ -366,25 +366,25 @@ private void near(boolean near) {
         }
 
         switch (writer.state()) {
-            case 12:
+            case 13:
                 if (!writer.writeMessage("key", key))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeMessage("prevVal", prevVal))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeLong("updateCntr", updateCntr))
                     return false;
 
                 writer.incrementState();
 
-            case 15:
+            case 16:
                 if (!writer.writeMessage("val", val))
                     return false;
 
@@ -406,7 +406,7 @@ private void near(boolean near) {
             return false;
 
         switch (reader.state()) {
-            case 12:
+            case 13:
                 key = reader.readMessage("key");
 
                 if (!reader.isLastRead())
@@ -414,7 +414,7 @@ private void near(boolean near) {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 prevVal = reader.readMessage("prevVal");
 
                 if (!reader.isLastRead())
@@ -422,7 +422,7 @@ private void near(boolean near) {
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 updateCntr = reader.readLong("updateCntr");
 
                 if (!reader.isLastRead())
@@ -430,7 +430,7 @@ private void near(boolean near) {
 
                 reader.incrementState();
 
-            case 15:
+            case 16:
                 val = reader.readMessage("val");
 
                 if (!reader.isLastRead())
@@ -480,7 +480,7 @@ private void finishUnmarshalObject(@Nullable CacheObject obj, GridCacheContext c
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 16;
+        return 17;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
index 6f3f53025f7db..56e0058084d28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateRequest.java
@@ -557,97 +557,97 @@ else if (conflictVers != null)
         }
 
         switch (writer.state()) {
-            case 12:
+            case 13:
                 if (!writer.writeMessage("conflictExpireTimes", conflictExpireTimes))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeCollection("conflictVers", conflictVers, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeCollection("entryProcessorsBytes", entryProcessorsBytes, MessageCollectionItemType.BYTE_ARR))
                     return false;
 
                 writer.incrementState();
 
-            case 15:
+            case 16:
                 if (!writer.writeBoolean("forceTransformBackups", forceTransformBackups))
                     return false;
 
                 writer.incrementState();
 
-            case 16:
+            case 17:
                 if (!writer.writeObjectArray("invokeArgsBytes", invokeArgsBytes, MessageCollectionItemType.BYTE_ARR))
                     return false;
 
                 writer.incrementState();
 
-            case 17:
+            case 18:
                 if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 18:
+            case 19:
                 if (!writer.writeCollection("nearEntryProcessorsBytes", nearEntryProcessorsBytes, MessageCollectionItemType.BYTE_ARR))
                     return false;
 
                 writer.incrementState();
 
-            case 19:
+            case 20:
                 if (!writer.writeMessage("nearExpireTimes", nearExpireTimes))
                     return false;
 
                 writer.incrementState();
 
-            case 20:
+            case 21:
                 if (!writer.writeCollection("nearKeys", nearKeys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 21:
+            case 22:
                 if (!writer.writeMessage("nearTtls", nearTtls))
                     return false;
 
                 writer.incrementState();
 
-            case 22:
+            case 23:
                 if (!writer.writeCollection("nearVals", nearVals, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 23:
+            case 24:
                 if (!writer.writeMessage("obsoleteIndexes", obsoleteIndexes))
                     return false;
 
                 writer.incrementState();
 
-            case 24:
+            case 25:
                 if (!writer.writeCollection("prevVals", prevVals, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 25:
+            case 26:
                 if (!writer.writeMessage("ttls", ttls))
                     return false;
 
                 writer.incrementState();
 
-            case 26:
+            case 27:
                 if (!writer.writeMessage("updateCntrs", updateCntrs))
                     return false;
 
                 writer.incrementState();
 
-            case 27:
+            case 28:
                 if (!writer.writeCollection("vals", vals, MessageCollectionItemType.MSG))
                     return false;
 
@@ -669,7 +669,7 @@ else if (conflictVers != null)
             return false;
 
         switch (reader.state()) {
-            case 12:
+            case 13:
                 conflictExpireTimes = reader.readMessage("conflictExpireTimes");
 
                 if (!reader.isLastRead())
@@ -677,7 +677,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 conflictVers = reader.readCollection("conflictVers", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -685,7 +685,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 entryProcessorsBytes = reader.readCollection("entryProcessorsBytes", MessageCollectionItemType.BYTE_ARR);
 
                 if (!reader.isLastRead())
@@ -693,7 +693,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 15:
+            case 16:
                 forceTransformBackups = reader.readBoolean("forceTransformBackups");
 
                 if (!reader.isLastRead())
@@ -701,7 +701,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 16:
+            case 17:
                 invokeArgsBytes = reader.readObjectArray("invokeArgsBytes", MessageCollectionItemType.BYTE_ARR, byte[].class);
 
                 if (!reader.isLastRead())
@@ -709,7 +709,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 17:
+            case 18:
                 keys = reader.readCollection("keys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -717,7 +717,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 18:
+            case 19:
                 nearEntryProcessorsBytes = reader.readCollection("nearEntryProcessorsBytes", MessageCollectionItemType.BYTE_ARR);
 
                 if (!reader.isLastRead())
@@ -725,7 +725,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 19:
+            case 20:
                 nearExpireTimes = reader.readMessage("nearExpireTimes");
 
                 if (!reader.isLastRead())
@@ -733,7 +733,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 20:
+            case 21:
                 nearKeys = reader.readCollection("nearKeys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -741,7 +741,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 21:
+            case 22:
                 nearTtls = reader.readMessage("nearTtls");
 
                 if (!reader.isLastRead())
@@ -749,7 +749,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 22:
+            case 23:
                 nearVals = reader.readCollection("nearVals", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -757,7 +757,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 23:
+            case 24:
                 obsoleteIndexes = reader.readMessage("obsoleteIndexes");
 
                 if (!reader.isLastRead())
@@ -765,7 +765,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 24:
+            case 25:
                 prevVals = reader.readCollection("prevVals", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -773,7 +773,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 25:
+            case 26:
                 ttls = reader.readMessage("ttls");
 
                 if (!reader.isLastRead())
@@ -781,7 +781,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 26:
+            case 27:
                 updateCntrs = reader.readMessage("updateCntrs");
 
                 if (!reader.isLastRead())
@@ -789,7 +789,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 27:
+            case 28:
                 vals = reader.readCollection("vals", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -815,7 +815,7 @@ else if (conflictVers != null)
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 28;
+        return 29;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateResponse.java
index 70bf6f5648b41..21efbb1350b3d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicUpdateResponse.java
@@ -179,25 +179,25 @@ public void nearEvicted(List<KeyCacheObject> nearEvicted) {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeMessage("errs", errs))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeCollection("nearEvicted", nearEvicted, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeInt("partId", partId))
                     return false;
 
@@ -219,7 +219,7 @@ public void nearEvicted(List<KeyCacheObject> nearEvicted) {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 errs = reader.readMessage("errs");
 
                 if (!reader.isLastRead())
@@ -227,7 +227,7 @@ public void nearEvicted(List<KeyCacheObject> nearEvicted) {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -235,7 +235,7 @@ public void nearEvicted(List<KeyCacheObject> nearEvicted) {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 nearEvicted = reader.readCollection("nearEvicted", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -243,7 +243,7 @@ public void nearEvicted(List<KeyCacheObject> nearEvicted) {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 partId = reader.readInt("partId");
 
                 if (!reader.isLastRead())
@@ -263,7 +263,7 @@ public void nearEvicted(List<KeyCacheObject> nearEvicted) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 7;
+        return 8;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java
index 62618f81e6af5..26be658df8f15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java
@@ -528,7 +528,7 @@ abstract void addUpdateEntry(KeyCacheObject key,
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 10;
+        return 11;
     }
 
     /** {@inheritDoc} */
@@ -546,43 +546,43 @@ abstract void addUpdateEntry(KeyCacheObject key,
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeByte("op", op != null ? (byte)op.ordinal() : -1))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeByte("syncMode", syncMode != null ? (byte)syncMode.ordinal() : -1))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -604,7 +604,7 @@ abstract void addUpdateEntry(KeyCacheObject key,
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -612,7 +612,7 @@ abstract void addUpdateEntry(KeyCacheObject key,
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -620,7 +620,7 @@ abstract void addUpdateEntry(KeyCacheObject key,
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 byte opOrd;
 
                 opOrd = reader.readByte("op");
@@ -632,7 +632,7 @@ abstract void addUpdateEntry(KeyCacheObject key,
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
@@ -640,7 +640,7 @@ abstract void addUpdateEntry(KeyCacheObject key,
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 byte syncModeOrd;
 
                 syncModeOrd = reader.readByte("syncMode");
@@ -652,7 +652,7 @@ abstract void addUpdateEntry(KeyCacheObject key,
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
@@ -660,7 +660,7 @@ abstract void addUpdateEntry(KeyCacheObject key,
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicCheckUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicCheckUpdateRequest.java
index 96be0233c308a..a19e28029b89f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicCheckUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicCheckUpdateRequest.java
@@ -101,7 +101,7 @@ GridNearAtomicAbstractUpdateRequest updateRequest() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 5;
+        return 6;
     }
 
     /** {@inheritDoc} */
@@ -119,13 +119,13 @@ GridNearAtomicAbstractUpdateRequest updateRequest() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeInt("partId", partId))
                     return false;
 
@@ -147,7 +147,7 @@ GridNearAtomicAbstractUpdateRequest updateRequest() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -155,7 +155,7 @@ GridNearAtomicAbstractUpdateRequest updateRequest() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 partId = reader.readInt("partId");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
index d6956a64dae69..170586b22d589 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicFullUpdateRequest.java
@@ -435,55 +435,55 @@ else if (conflictVers != null)
         }
 
         switch (writer.state()) {
-            case 10:
+            case 11:
                 if (!writer.writeMessage("conflictExpireTimes", conflictExpireTimes))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeMessage("conflictTtls", conflictTtls))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeCollection("conflictVers", conflictVers, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeCollection("entryProcessorsBytes", entryProcessorsBytes, MessageCollectionItemType.BYTE_ARR))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeByteArray("expiryPlcBytes", expiryPlcBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 15:
+            case 16:
                 if (!writer.writeObjectArray("filter", filter, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 16:
+            case 17:
                 if (!writer.writeObjectArray("invokeArgsBytes", invokeArgsBytes, MessageCollectionItemType.BYTE_ARR))
                     return false;
 
                 writer.incrementState();
 
-            case 17:
+            case 18:
                 if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 18:
+            case 19:
                 if (!writer.writeCollection("vals", vals, MessageCollectionItemType.MSG))
                     return false;
 
@@ -505,7 +505,7 @@ else if (conflictVers != null)
             return false;
 
         switch (reader.state()) {
-            case 10:
+            case 11:
                 conflictExpireTimes = reader.readMessage("conflictExpireTimes");
 
                 if (!reader.isLastRead())
@@ -513,7 +513,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 conflictTtls = reader.readMessage("conflictTtls");
 
                 if (!reader.isLastRead())
@@ -521,7 +521,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 conflictVers = reader.readCollection("conflictVers", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -529,7 +529,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 entryProcessorsBytes = reader.readCollection("entryProcessorsBytes", MessageCollectionItemType.BYTE_ARR);
 
                 if (!reader.isLastRead())
@@ -537,7 +537,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 expiryPlcBytes = reader.readByteArray("expiryPlcBytes");
 
                 if (!reader.isLastRead())
@@ -545,7 +545,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 15:
+            case 16:
                 filter = reader.readObjectArray("filter", MessageCollectionItemType.MSG, CacheEntryPredicate.class);
 
                 if (!reader.isLastRead())
@@ -553,7 +553,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 16:
+            case 17:
                 invokeArgsBytes = reader.readObjectArray("invokeArgsBytes", MessageCollectionItemType.BYTE_ARR, byte[].class);
 
                 if (!reader.isLastRead())
@@ -561,7 +561,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 17:
+            case 18:
                 keys = reader.readCollection("keys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -569,7 +569,7 @@ else if (conflictVers != null)
 
                 reader.incrementState();
 
-            case 18:
+            case 19:
                 vals = reader.readCollection("vals", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -601,7 +601,7 @@ else if (conflictVers != null)
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 19;
+        return 20;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFilterRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFilterRequest.java
index 5c66bc46894b2..c7076988a659f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFilterRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFilterRequest.java
@@ -155,7 +155,7 @@ public GridNearAtomicSingleUpdateFilterRequest() {
         }
 
         switch (writer.state()) {
-            case 12:
+            case 13:
                 if (!writer.writeObjectArray("filter", filter, MessageCollectionItemType.MSG))
                     return false;
 
@@ -177,7 +177,7 @@ public GridNearAtomicSingleUpdateFilterRequest() {
             return false;
 
         switch (reader.state()) {
-            case 12:
+            case 13:
                 filter = reader.readObjectArray("filter", MessageCollectionItemType.MSG, CacheEntryPredicate.class);
 
                 if (!reader.isLastRead())
@@ -197,7 +197,7 @@ public GridNearAtomicSingleUpdateFilterRequest() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 13;
+        return 14;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
index 865d6f8664e9e..ee3d2a4fe036a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateInvokeRequest.java
@@ -225,13 +225,13 @@ public GridNearAtomicSingleUpdateInvokeRequest() {
         }
 
         switch (writer.state()) {
-            case 12:
+            case 13:
                 if (!writer.writeByteArray("entryProcessorBytes", entryProcessorBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeObjectArray("invokeArgsBytes", invokeArgsBytes, MessageCollectionItemType.BYTE_ARR))
                     return false;
 
@@ -253,7 +253,7 @@ public GridNearAtomicSingleUpdateInvokeRequest() {
             return false;
 
         switch (reader.state()) {
-            case 12:
+            case 13:
                 entryProcessorBytes = reader.readByteArray("entryProcessorBytes");
 
                 if (!reader.isLastRead())
@@ -261,7 +261,7 @@ public GridNearAtomicSingleUpdateInvokeRequest() {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 invokeArgsBytes = reader.readObjectArray("invokeArgsBytes", MessageCollectionItemType.BYTE_ARR, byte[].class);
 
                 if (!reader.isLastRead())
@@ -276,7 +276,7 @@ public GridNearAtomicSingleUpdateInvokeRequest() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 14;
+        return 15;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java
index dd3a7bedb7de9..83ec4565f49ce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateRequest.java
@@ -247,13 +247,13 @@ public GridNearAtomicSingleUpdateRequest() {
         }
 
         switch (writer.state()) {
-            case 10:
+            case 11:
                 if (!writer.writeMessage("key", key))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeMessage("val", val))
                     return false;
 
@@ -275,7 +275,7 @@ public GridNearAtomicSingleUpdateRequest() {
             return false;
 
         switch (reader.state()) {
-            case 10:
+            case 11:
                 key = reader.readMessage("key");
 
                 if (!reader.isLastRead())
@@ -283,7 +283,7 @@ public GridNearAtomicSingleUpdateRequest() {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 val = reader.readMessage("val");
 
                 if (!reader.isLastRead())
@@ -311,7 +311,7 @@ public GridNearAtomicSingleUpdateRequest() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 12;
+        return 13;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
index 37fe824a39f99..56eb8d7426500 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
@@ -405,43 +405,43 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeMessage("errs", errs))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeCollection("mapping", mapping, MessageCollectionItemType.UUID))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeMessage("nearUpdates", nearUpdates))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeInt("partId", partId))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeMessage("remapTopVer", remapTopVer))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeMessage("ret", ret))
                     return false;
 
@@ -463,7 +463,7 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 errs = reader.readMessage("errs");
 
                 if (!reader.isLastRead())
@@ -471,7 +471,7 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -479,7 +479,7 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 mapping = reader.readCollection("mapping", MessageCollectionItemType.UUID);
 
                 if (!reader.isLastRead())
@@ -487,7 +487,7 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 nearUpdates = reader.readMessage("nearUpdates");
 
                 if (!reader.isLastRead())
@@ -495,7 +495,7 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 partId = reader.readInt("partId");
 
                 if (!reader.isLastRead())
@@ -503,7 +503,7 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 remapTopVer = reader.readMessage("remapTopVer");
 
                 if (!reader.isLastRead())
@@ -511,7 +511,7 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 ret = reader.readMessage("ret");
 
                 if (!reader.isLastRead())
@@ -531,7 +531,7 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 10;
+        return 11;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java
index 124ae44ca5b30..ccd36a9641eab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java
@@ -167,25 +167,25 @@ private int keyCount() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeIgniteUuid("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -207,7 +207,7 @@ private int keyCount() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -215,7 +215,7 @@ private int keyCount() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 keys = reader.readCollection("keys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -223,7 +223,7 @@ private int keyCount() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 miniId = reader.readIgniteUuid("miniId");
 
                 if (!reader.isLastRead())
@@ -231,7 +231,7 @@ private int keyCount() {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -251,7 +251,7 @@ private int keyCount() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 7;
+        return 8;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java
index 977e9ba41eef2..ab85df3e94622 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysResponse.java
@@ -213,31 +213,31 @@ public void addInfo(GridCacheEntryInfo info) {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeCollection("infos", infos, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeIgniteUuid("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeCollection("missedKeys", missedKeys, MessageCollectionItemType.MSG))
                     return false;
 
@@ -259,7 +259,7 @@ public void addInfo(GridCacheEntryInfo info) {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
@@ -267,7 +267,7 @@ public void addInfo(GridCacheEntryInfo info) {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -275,7 +275,7 @@ public void addInfo(GridCacheEntryInfo info) {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 infos = reader.readCollection("infos", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -283,7 +283,7 @@ public void addInfo(GridCacheEntryInfo info) {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 miniId = reader.readIgniteUuid("miniId");
 
                 if (!reader.isLastRead())
@@ -291,7 +291,7 @@ public void addInfo(GridCacheEntryInfo info) {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 missedKeys = reader.readCollection("missedKeys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -311,7 +311,7 @@ public void addInfo(GridCacheEntryInfo info) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 8;
+        return 9;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java
index 46e9ceb004830..d75bd76739335 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java
@@ -285,49 +285,49 @@ Long partitionCounter(int part) {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeCollection("historicalParts", historicalParts, MessageCollectionItemType.INT))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeCollection("parts", parts, MessageCollectionItemType.INT))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeMap("partsCntrs", partsCntrs, MessageCollectionItemType.INT, MessageCollectionItemType.LONG))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeLong("timeout", timeout))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeByteArray("topicBytes", topicBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeLong("updateSeq", updateSeq))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeInt("workerId", workerId))
                     return false;
 
@@ -349,7 +349,7 @@ Long partitionCounter(int part) {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 historicalParts = reader.readCollection("historicalParts", MessageCollectionItemType.INT);
 
                 if (!reader.isLastRead())
@@ -357,7 +357,7 @@ Long partitionCounter(int part) {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 parts = reader.readCollection("parts", MessageCollectionItemType.INT);
 
                 if (!reader.isLastRead())
@@ -365,7 +365,7 @@ Long partitionCounter(int part) {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 partsCntrs = reader.readMap("partsCntrs", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false);
 
                 if (!reader.isLastRead())
@@ -373,7 +373,7 @@ Long partitionCounter(int part) {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 timeout = reader.readLong("timeout");
 
                 if (!reader.isLastRead())
@@ -381,7 +381,7 @@ Long partitionCounter(int part) {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -389,7 +389,7 @@ Long partitionCounter(int part) {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 topicBytes = reader.readByteArray("topicBytes");
 
                 if (!reader.isLastRead())
@@ -397,7 +397,7 @@ Long partitionCounter(int part) {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 updateSeq = reader.readLong("updateSeq");
 
                 if (!reader.isLastRead())
@@ -405,7 +405,7 @@ Long partitionCounter(int part) {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 workerId = reader.readInt("workerId");
 
                 if (!reader.isLastRead())
@@ -415,7 +415,7 @@ Long partitionCounter(int part) {
 
         }
 
-        return reader.afterMessageRead(GridDhtPartitionDemandMessage.class);
+        return reader.afterMessageRead(GridDhtPartitionDemandLegacyMessage.class);
     }
 
     /** {@inheritDoc} */
@@ -425,7 +425,7 @@ Long partitionCounter(int part) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 11;
+        return 12;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
index dc6162bc321f2..5a8a71117c298 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
@@ -259,37 +259,37 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) {
         }
 
         switch (writer.state()) {
-            case 3:
-                if (!writer.writeByteArray("partsBytes", partsBytes))
-                    return false;
-
-                writer.incrementState();
-
             case 4:
-                if (!writer.writeLong("timeout", timeout))
+                if (!writer.writeByteArray("partsBytes", partsBytes))
                     return false;
 
                 writer.incrementState();
 
             case 5:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeLong("rebalanceId", rebalanceId))
                     return false;
 
                 writer.incrementState();
 
             case 6:
-                if (!writer.writeByteArray("topicBytes", topicBytes))
+                if (!writer.writeLong("timeout", timeout))
                     return false;
 
                 writer.incrementState();
 
             case 7:
-                if (!writer.writeLong("rebalanceId", rebalanceId))
+                if (!writer.writeMessage("topVer", topVer))
                     return false;
 
                 writer.incrementState();
 
             case 8:
+                if (!writer.writeByteArray("topicBytes", topicBytes))
+                    return false;
+
+                writer.incrementState();
+
+            case 9:
                 if (!writer.writeInt("workerId", workerId))
                     return false;
 
@@ -311,7 +311,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 partsBytes = reader.readByteArray("partsBytes");
 
                 if (!reader.isLastRead())
@@ -319,39 +319,39 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) {
 
                 reader.incrementState();
 
-            case 4:
-                timeout = reader.readLong("timeout");
+            case 5:
+                rebalanceId = reader.readLong("rebalanceId");
 
                 if (!reader.isLastRead())
                     return false;
 
                 reader.incrementState();
 
-            case 5:
-                topVer = reader.readMessage("topVer");
+            case 6:
+                timeout = reader.readLong("timeout");
 
                 if (!reader.isLastRead())
                     return false;
 
                 reader.incrementState();
 
-            case 6:
-                topicBytes = reader.readByteArray("topicBytes");
+            case 7:
+                topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
                     return false;
 
                 reader.incrementState();
 
-            case 7:
-                rebalanceId = reader.readLong("rebalanceId");
+            case 8:
+                topicBytes = reader.readByteArray("topicBytes");
 
                 if (!reader.isLastRead())
                     return false;
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 workerId = reader.readInt("workerId");
 
                 if (!reader.isLastRead())
@@ -371,7 +371,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 9;
+        return 10;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
index 284700ad3a44f..ca8a8b5fed9dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
@@ -282,57 +282,56 @@ public int size() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeCollection("clean", clean, MessageCollectionItemType.INT))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeLong("estimatedKeysCnt", estimatedKeysCnt))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeMap("infos", infos, MessageCollectionItemType.INT, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeMap("keysPerCache", keysPerCache, MessageCollectionItemType.INT, MessageCollectionItemType.LONG))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeMap("last", last, MessageCollectionItemType.INT, MessageCollectionItemType.LONG))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeCollection("missed", missed, MessageCollectionItemType.INT))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeInt("msgSize", msgSize))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
-                if (!writer.writeMessage("topVer", topVer))
+            case 11:
+                if (!writer.writeLong("rebalanceId", rebalanceId))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
-                // Keep 'updateSeq' name for compatibility.
-                if (!writer.writeLong("updateSeq", rebalanceId))
+            case 12:
+                if (!writer.writeMessage("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -353,7 +352,7 @@ public int size() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 clean = reader.readCollection("clean", MessageCollectionItemType.INT);
 
                 if (!reader.isLastRead())
@@ -361,7 +360,7 @@ public int size() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 estimatedKeysCnt = reader.readLong("estimatedKeysCnt");
 
                 if (!reader.isLastRead())
@@ -369,7 +368,7 @@ public int size() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 infos = reader.readMap("infos", MessageCollectionItemType.INT, MessageCollectionItemType.MSG, false);
 
                 if (!reader.isLastRead())
@@ -377,7 +376,7 @@ public int size() {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 keysPerCache = reader.readMap("keysPerCache", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false);
 
                 if (!reader.isLastRead())
@@ -385,7 +384,7 @@ public int size() {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 last = reader.readMap("last", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false);
 
                 if (!reader.isLastRead())
@@ -393,7 +392,7 @@ public int size() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 missed = reader.readCollection("missed", MessageCollectionItemType.INT);
 
                 if (!reader.isLastRead())
@@ -401,7 +400,7 @@ public int size() {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 msgSize = reader.readInt("msgSize");
 
                 if (!reader.isLastRead())
@@ -409,17 +408,16 @@ public int size() {
 
                 reader.incrementState();
 
-            case 10:
-                topVer = reader.readMessage("topVer");
+            case 11:
+                rebalanceId = reader.readLong("rebalanceId");
 
                 if (!reader.isLastRead())
                     return false;
 
                 reader.incrementState();
 
-            case 11:
-                // Keep 'updateSeq' name for compatibility.
-                rebalanceId = reader.readLong("updateSeq");
+            case 12:
+                topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
                     return false;
@@ -438,7 +436,7 @@ public int size() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 12;
+        return 13;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
index 537dbd0fe819a..36bae12a96917 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessageV2.java
@@ -101,7 +101,7 @@ public GridDhtPartitionSupplyMessageV2(
         }
 
         switch (writer.state()) {
-            case 12:
+            case 13:
                 if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
@@ -123,7 +123,7 @@ public GridDhtPartitionSupplyMessageV2(
             return false;
 
         switch (reader.state()) {
-            case 12:
+            case 13:
                 errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
@@ -148,6 +148,6 @@ public GridDhtPartitionSupplyMessageV2(
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 13;
+        return 14;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java
index 84cc792fe22ea..e2884e1751508 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsAbstractMessage.java
@@ -145,7 +145,7 @@ public boolean restoreState() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 5;
+        return 6;
     }
 
     /** {@inheritDoc} */
@@ -163,19 +163,19 @@ public boolean restoreState() {
         }
 
         switch (writer.state()) {
-            case 2:
+            case 3:
                 if (!writer.writeMessage("exchId", exchId))
                     return false;
 
                 writer.incrementState();
 
-            case 3:
+            case 4:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeMessage("lastVer", lastVer))
                     return false;
 
@@ -197,7 +197,7 @@ public boolean restoreState() {
             return false;
 
         switch (reader.state()) {
-            case 2:
+            case 3:
                 exchId = reader.readMessage("exchId");
 
                 if (!reader.isLastRead())
@@ -205,7 +205,7 @@ public boolean restoreState() {
 
                 reader.incrementState();
 
-            case 3:
+            case 4:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -213,7 +213,7 @@ public boolean restoreState() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 lastVer = reader.readMessage("lastVer");
 
                 if (!reader.isLastRead())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 8702d9e7a9bb1..d9869759e134c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -85,6 +85,7 @@
 import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.Latch;
+import org.apache.ignite.internal.processors.cache.distributed.dht.ClientCacheDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
@@ -324,6 +325,9 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     /** Latest (by update sequences) full message with exchangeId == null, need to be processed right after future is done. */
     private GridDhtPartitionsFullMessage delayedLatestMsg;
 
+    /** */
+    private volatile AffinityTopologyVersion lastAffChangeTopVer;
+
     /**
      * @param cctx Cache context.
      * @param busyLock Busy lock.
@@ -554,6 +558,26 @@ public boolean changedBaseline() {
         return exchActions != null && exchActions.changedBaseline();
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean changedAffinity() {
+        DiscoveryEvent firstDiscoEvt0 = firstDiscoEvt;
+
+        assert firstDiscoEvt0 != null;
+
+        return firstDiscoEvt0.type() == DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT
+            || !firstDiscoEvt0.eventNode().isClient() || firstDiscoEvt0.eventNode().isLocal();
+    }
+
+    /** {@inheritDoc} */
+    @Override public AffinityTopologyVersion lastAffinityChangeTopologyVersion() {
+        if (changedAffinity())
+            return topologyVersion();
+        else if (!exchangeDone()) // TODO: initialVersion and topologyVersion are always equal now, but it should be fixed later.
+            return initialVersion();
+        else
+            return topologyVersion();
+    }
+
     /**
      * @return {@code True} if there are caches to start.
      */
@@ -562,7 +586,7 @@ public boolean hasCachesToStart() {
     }
 
     /**
-     * @return First event discovery event.
+     * @return First event discovery event.1
      *
      */
     public DiscoveryEvent firstEvent() {
@@ -631,7 +655,7 @@ private void initCoordinatorCaches(boolean newCrd) throws IgniteCheckedException
      * @param newCrd {@code True} if node become coordinator on this exchange.
      * @throws IgniteInterruptedCheckedException If interrupted.
      */
-    public void init(boolean newCrd) throws IgniteInterruptedCheckedException {
+    public void init(@Nullable GridDhtPartitionsExchangeFuture lastFut, boolean newCrd) throws IgniteInterruptedCheckedException {
         if (isDone())
             return;
 
@@ -645,6 +669,9 @@ public void init(boolean newCrd) throws IgniteInterruptedCheckedException {
         assert exchId.nodeId().equals(firstDiscoEvt.eventNode().id()) : this;
 
         try {
+            if (!changedAffinity() && lastFut != null)
+                lastAffChangeTopVer = lastFut.lastAffinityChangeTopologyVersion();
+
             AffinityTopologyVersion topVer = initialVersion();
 
             srvNodes = new ArrayList<>(firstEvtDiscoCache.serverNodes());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index b84dc79e821f3..8d34b340b872b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -593,73 +593,73 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
         }
 
         switch (writer.state()) {
-            case 5:
+            case 6:
                 if (!writer.writeMap("dupPartsData", dupPartsData, MessageCollectionItemType.INT, MessageCollectionItemType.INT))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeByteArray("errsBytes", errsBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeMap("idealAffDiff", idealAffDiff, MessageCollectionItemType.INT, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeMap("joinedNodeAff", joinedNodeAff, MessageCollectionItemType.INT, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeByteArray("partCntrsBytes2", partCntrsBytes2))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeByteArray("partHistSuppliersBytes", partHistSuppliersBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeByteArray("partsBytes", partsBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeByteArray("partsToReloadBytes", partsToReloadBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 15:
+            case 16:
                 if (!writer.writeMessage("resTopVer", resTopVer))
                     return false;
 
                 writer.incrementState();
 
-            case 16:
+            case 17:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -681,7 +681,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
             return false;
 
         switch (reader.state()) {
-            case 5:
+            case 6:
                 dupPartsData = reader.readMap("dupPartsData", MessageCollectionItemType.INT, MessageCollectionItemType.INT, false);
 
                 if (!reader.isLastRead())
@@ -689,7 +689,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 errsBytes = reader.readByteArray("errsBytes");
 
                 if (!reader.isLastRead())
@@ -697,7 +697,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 idealAffDiff = reader.readMap("idealAffDiff", MessageCollectionItemType.INT, MessageCollectionItemType.MSG, false);
 
                 if (!reader.isLastRead())
@@ -705,7 +705,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 joinedNodeAff = reader.readMap("joinedNodeAff", MessageCollectionItemType.INT, MessageCollectionItemType.MSG, false);
 
                 if (!reader.isLastRead())
@@ -713,7 +713,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 partCntrsBytes = reader.readByteArray("partCntrsBytes");
 
                 if (!reader.isLastRead())
@@ -721,7 +721,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 partCntrsBytes2 = reader.readByteArray("partCntrsBytes2");
 
                 if (!reader.isLastRead())
@@ -729,7 +729,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 partHistSuppliersBytes = reader.readByteArray("partHistSuppliersBytes");
 
                 if (!reader.isLastRead())
@@ -737,7 +737,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 partsBytes = reader.readByteArray("partsBytes");
 
                 if (!reader.isLastRead())
@@ -745,7 +745,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 partsSizesBytes = reader.readByteArray("partsSizesBytes");
 
                 if (!reader.isLastRead())
@@ -753,7 +753,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 partsToReloadBytes = reader.readByteArray("partsToReloadBytes");
 
                 if (!reader.isLastRead())
@@ -761,7 +761,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 15:
+            case 16:
                 resTopVer = reader.readMessage("resTopVer");
 
                 if (!reader.isLastRead())
@@ -769,7 +769,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
                 reader.incrementState();
 
-            case 16:
+            case 17:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -789,7 +789,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 17;
+        return 18;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 127c471f52ca8..3ce3b9ffd9c23 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -455,55 +455,55 @@ public void setError(Exception ex) {
         }
 
         switch (writer.state()) {
-            case 5:
+            case 6:
                 if (!writer.writeBoolean("client", client))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeMap("dupPartsData", dupPartsData, MessageCollectionItemType.INT, MessageCollectionItemType.INT))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeMessage("finishMsg", finishMsg))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeCollection("grpsAffRequest", grpsAffRequest, MessageCollectionItemType.INT))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeByteArray("partCntrsBytes", partCntrsBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeByteArray("partHistCntrsBytes", partHistCntrsBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeByteArray("partsBytes", partsBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeByteArray("partsSizesBytes", partsSizesBytes))
                     return false;
 
@@ -524,7 +524,7 @@ public void setError(Exception ex) {
             return false;
 
         switch (reader.state()) {
-            case 5:
+            case 6:
                 client = reader.readBoolean("client");
 
                 if (!reader.isLastRead())
@@ -532,7 +532,7 @@ public void setError(Exception ex) {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 dupPartsData = reader.readMap("dupPartsData", MessageCollectionItemType.INT, MessageCollectionItemType.INT, false);
 
                 if (!reader.isLastRead())
@@ -540,7 +540,7 @@ public void setError(Exception ex) {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
@@ -548,7 +548,7 @@ public void setError(Exception ex) {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 finishMsg = reader.readMessage("finishMsg");
 
                 if (!reader.isLastRead())
@@ -556,7 +556,7 @@ public void setError(Exception ex) {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 grpsAffRequest = reader.readCollection("grpsAffRequest", MessageCollectionItemType.INT);
 
                 if (!reader.isLastRead())
@@ -564,7 +564,7 @@ public void setError(Exception ex) {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 partCntrsBytes = reader.readByteArray("partCntrsBytes");
 
                 if (!reader.isLastRead())
@@ -572,7 +572,7 @@ public void setError(Exception ex) {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 partHistCntrsBytes = reader.readByteArray("partHistCntrsBytes");
 
                 if (!reader.isLastRead())
@@ -580,7 +580,7 @@ public void setError(Exception ex) {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 partsBytes = reader.readByteArray("partsBytes");
 
                 if (!reader.isLastRead())
@@ -588,7 +588,7 @@ public void setError(Exception ex) {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 partsSizesBytes = reader.readByteArray("partsSizesBytes");
 
                 if (!reader.isLastRead())
@@ -607,7 +607,7 @@ public void setError(Exception ex) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 14;
+        return 15;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleRequest.java
index 0be0f37aa22ac..26d3cdeffd684 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleRequest.java
@@ -89,7 +89,7 @@ GridDhtPartitionExchangeId restoreExchangeId() {
         }
 
         switch (writer.state()) {
-            case 5:
+            case 6:
                 if (!writer.writeMessage("restoreExchId", restoreExchId))
                     return false;
 
@@ -111,7 +111,7 @@ GridDhtPartitionExchangeId restoreExchangeId() {
             return false;
 
         switch (reader.state()) {
-            case 5:
+            case 6:
                 restoreExchId = reader.readMessage("restoreExchId");
 
                 if (!reader.isLastRead())
@@ -131,7 +131,7 @@ GridDhtPartitionExchangeId restoreExchangeId() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 6;
+        return 7;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
index 56d00d20c21f9..d84cfd47d5f4c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
@@ -296,9 +296,14 @@ private String mapString(GridDhtPartitionMap map) {
 
     /** {@inheritDoc} */
     @Override public GridDhtTopologyFuture topologyVersionFuture() {
-        assert topReadyFut != null;
+        GridDhtTopologyFuture topReadyFut0 = topReadyFut;
 
-        return topReadyFut;
+        assert topReadyFut0 != null;
+
+        if (topReadyFut0 instanceof GridDhtPartitionsExchangeFuture && !((GridDhtPartitionsExchangeFuture)topReadyFut0).changedAffinity())
+            return ctx.exchange().lastFinishedFuture();
+
+        return topReadyFut0;
     }
 
     /** {@inheritDoc} */
@@ -1138,25 +1143,38 @@ else if (loc != null && state == RENTING && !showRenting) {
 
             List<ClusterNode> nodes = null;
 
-            if (!topVer.equals(diffFromAffinityVer)) {
-                LT.warn(log, "Requested topology version does not match calculated diff, will require full iteration to" +
-                    "calculate mapping [grp=" + grp.cacheOrGroupName() + ", topVer=" + topVer +
-                    ", diffVer=" + diffFromAffinityVer + "]");
+            AffinityTopologyVersion diffVer = diffFromAffinityVer;
 
-                nodes = new ArrayList<>();
+            if (!diffVer.equals(topVer)) {
+                LT.warn(log, "Requested topology version does not match calculated diff, need to check if " +
+                    "affinity has changed [grp=" + grp.cacheOrGroupName() + ", topVer=" + topVer +
+                    ", diffVer=" + diffVer + "]");
 
-                nodes.addAll(affNodes);
+                boolean affChanged;
 
-                for (Map.Entry<UUID, GridDhtPartitionMap> entry : node2part.entrySet()) {
-                    GridDhtPartitionState state = entry.getValue().get(p);
+                if (diffVer.compareTo(topVer) < 0)
+                    affChanged = ctx.exchange().affinityChanged(diffVer, topVer);
+                else
+                    affChanged = ctx.exchange().affinityChanged(topVer, diffVer);
 
-                    ClusterNode n = ctx.discovery().node(entry.getKey());
+                if (affChanged) {
+                    LT.warn(log, "Requested topology version does not match calculated diff, will require full iteration to" +
+                        "calculate mapping [grp=" + grp.cacheOrGroupName() + ", topVer=" + topVer +
+                        ", diffVer=" + diffVer + "]");
 
-                    if (n != null && state != null && (state == MOVING || state == OWNING || state == RENTING)
-                        && !nodes.contains(n) && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion())) {
-                        nodes.add(n);
-                    }
+                    nodes = new ArrayList<>();
 
+                    nodes.addAll(affNodes);
+
+                    for (Map.Entry<UUID, GridDhtPartitionMap> entry : node2part.entrySet()) {
+                        GridDhtPartitionState state = entry.getValue().get(p);
+
+                        ClusterNode n = ctx.discovery().node(entry.getKey());
+
+                        if (n != null && state != null && (state == MOVING || state == OWNING || state == RENTING)
+                            && !nodes.contains(n) && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion()))
+                            nodes.add(n);
+                    }
                 }
 
                 return nodes;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
index dcb167d81a892..8bd5ac16d664b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetRequest.java
@@ -345,67 +345,67 @@ public long accessTtl() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeLong("accessTtl", accessTtl))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeLong("createTtl", createTtl))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeCollection("keys", keys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeIgniteUuid("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeCollection("readersFlags", readersFlags, MessageCollectionItemType.BOOLEAN))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeMessage("ver", ver))
                     return false;
 
@@ -427,7 +427,7 @@ public long accessTtl() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 accessTtl = reader.readLong("accessTtl");
 
                 if (!reader.isLastRead())
@@ -435,7 +435,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 createTtl = reader.readLong("createTtl");
 
                 if (!reader.isLastRead())
@@ -443,7 +443,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -451,7 +451,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -459,7 +459,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 keys = reader.readCollection("keys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -467,7 +467,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 miniId = reader.readIgniteUuid("miniId");
 
                 if (!reader.isLastRead())
@@ -475,7 +475,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 readersFlags = reader.readCollection("readersFlags", MessageCollectionItemType.BOOLEAN);
 
                 if (!reader.isLastRead())
@@ -483,7 +483,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
@@ -491,7 +491,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
@@ -499,7 +499,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -507,7 +507,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 ver = reader.readMessage("ver");
 
                 if (!reader.isLastRead())
@@ -527,7 +527,7 @@ public long accessTtl() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 14;
+        return 15;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java
index b4e4424862c54..8e724b2af2ccc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java
@@ -228,43 +228,43 @@ public void error(IgniteCheckedException err) {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeCollection("entries", entries, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeCollection("invalidParts", invalidParts, MessageCollectionItemType.INT))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeIgniteUuid("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeMessage("ver", ver))
                     return false;
 
@@ -286,7 +286,7 @@ public void error(IgniteCheckedException err) {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 entries = reader.readCollection("entries", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -294,7 +294,7 @@ public void error(IgniteCheckedException err) {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
@@ -302,7 +302,7 @@ public void error(IgniteCheckedException err) {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -310,7 +310,7 @@ public void error(IgniteCheckedException err) {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 invalidParts = reader.readCollection("invalidParts", MessageCollectionItemType.INT);
 
                 if (!reader.isLastRead())
@@ -318,7 +318,7 @@ public void error(IgniteCheckedException err) {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 miniId = reader.readIgniteUuid("miniId");
 
                 if (!reader.isLastRead())
@@ -326,7 +326,7 @@ public void error(IgniteCheckedException err) {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -334,7 +334,7 @@ public void error(IgniteCheckedException err) {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 ver = reader.readMessage("ver");
 
                 if (!reader.isLastRead())
@@ -354,7 +354,7 @@ public void error(IgniteCheckedException err) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 10;
+        return 11;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java
index f736cae61848c..1d18d943aceed 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java
@@ -363,55 +363,55 @@ public long accessTtl() {
         }
 
         switch (writer.state()) {
-            case 20:
+            case 21:
                 if (!writer.writeLong("accessTtl", accessTtl))
                     return false;
 
                 writer.incrementState();
 
-            case 21:
+            case 22:
                 if (!writer.writeLong("createTtl", createTtl))
                     return false;
 
                 writer.incrementState();
 
-            case 22:
+            case 23:
                 if (!writer.writeObjectArray("dhtVers", dhtVers, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 23:
+            case 24:
                 if (!writer.writeObjectArray("filter", filter, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 24:
+            case 25:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 25:
+            case 26:
                 if (!writer.writeInt("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 26:
+            case 27:
                 if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
-            case 27:
+            case 28:
                 if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
-            case 28:
+            case 29:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -433,7 +433,7 @@ public long accessTtl() {
             return false;
 
         switch (reader.state()) {
-            case 20:
+            case 21:
                 accessTtl = reader.readLong("accessTtl");
 
                 if (!reader.isLastRead())
@@ -441,7 +441,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 21:
+            case 22:
                 createTtl = reader.readLong("createTtl");
 
                 if (!reader.isLastRead())
@@ -449,7 +449,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 22:
+            case 23:
                 dhtVers = reader.readObjectArray("dhtVers", MessageCollectionItemType.MSG, GridCacheVersion.class);
 
                 if (!reader.isLastRead())
@@ -457,7 +457,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 23:
+            case 24:
                 filter = reader.readObjectArray("filter", MessageCollectionItemType.MSG, CacheEntryPredicate.class);
 
                 if (!reader.isLastRead())
@@ -465,7 +465,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 24:
+            case 25:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -473,7 +473,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 25:
+            case 26:
                 miniId = reader.readInt("miniId");
 
                 if (!reader.isLastRead())
@@ -481,7 +481,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 26:
+            case 27:
                 subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
@@ -489,7 +489,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 27:
+            case 28:
                 taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
@@ -497,7 +497,7 @@ public long accessTtl() {
 
                 reader.incrementState();
 
-            case 28:
+            case 29:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -517,7 +517,7 @@ public long accessTtl() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 29;
+        return 30;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java
index e88f0a07e0755..34386205fa29e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java
@@ -208,37 +208,37 @@ public void addValueBytes(
         }
 
         switch (writer.state()) {
-            case 10:
+            case 11:
                 if (!writer.writeMessage("clientRemapVer", clientRemapVer))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeObjectArray("dhtVers", dhtVers, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeBooleanArray("filterRes", filterRes))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeObjectArray("mappedVers", mappedVers, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeInt("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 15:
+            case 16:
                 if (!writer.writeCollection("pending", pending, MessageCollectionItemType.MSG))
                     return false;
 
@@ -260,7 +260,7 @@ public void addValueBytes(
             return false;
 
         switch (reader.state()) {
-            case 10:
+            case 11:
                 clientRemapVer = reader.readMessage("clientRemapVer");
 
                 if (!reader.isLastRead())
@@ -268,7 +268,7 @@ public void addValueBytes(
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 dhtVers = reader.readObjectArray("dhtVers", MessageCollectionItemType.MSG, GridCacheVersion.class);
 
                 if (!reader.isLastRead())
@@ -276,7 +276,7 @@ public void addValueBytes(
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 filterRes = reader.readBooleanArray("filterRes");
 
                 if (!reader.isLastRead())
@@ -284,7 +284,7 @@ public void addValueBytes(
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 mappedVers = reader.readObjectArray("mappedVers", MessageCollectionItemType.MSG, GridCacheVersion.class);
 
                 if (!reader.isLastRead())
@@ -292,7 +292,7 @@ public void addValueBytes(
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 miniId = reader.readInt("miniId");
 
                 if (!reader.isLastRead())
@@ -300,7 +300,7 @@ public void addValueBytes(
 
                 reader.incrementState();
 
-            case 15:
+            case 16:
                 pending = reader.readCollection("pending", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -320,7 +320,7 @@ public void addValueBytes(
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 16;
+        return 17;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
index 00ff4bb60bf95..16b514968c8b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
@@ -281,7 +281,7 @@ public boolean recovery() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 accessTtl = reader.readLong("accessTtl");
 
                 if (!reader.isLastRead())
@@ -289,7 +289,7 @@ public boolean recovery() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 createTtl = reader.readLong("createTtl");
 
                 if (!reader.isLastRead())
@@ -297,7 +297,7 @@ public boolean recovery() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -305,7 +305,7 @@ public boolean recovery() {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -313,7 +313,7 @@ public boolean recovery() {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 key = reader.readMessage("key");
 
                 if (!reader.isLastRead())
@@ -321,7 +321,7 @@ public boolean recovery() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
@@ -329,7 +329,7 @@ public boolean recovery() {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
@@ -337,7 +337,7 @@ public boolean recovery() {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -365,49 +365,49 @@ public boolean recovery() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeLong("accessTtl", accessTtl))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeLong("createTtl", createTtl))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeMessage("key", key))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -430,7 +430,7 @@ public boolean recovery() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 11;
+        return 12;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java
index 2cb75c253e290..c71a1d5bdcb4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java
@@ -206,31 +206,31 @@ else if (res instanceof GridCacheEntryInfo)
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeMessage("res", res))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -252,7 +252,7 @@ else if (res instanceof GridCacheEntryInfo)
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
@@ -260,7 +260,7 @@ else if (res instanceof GridCacheEntryInfo)
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -268,7 +268,7 @@ else if (res instanceof GridCacheEntryInfo)
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -276,7 +276,7 @@ else if (res instanceof GridCacheEntryInfo)
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 res = reader.readMessage("res");
 
                 if (!reader.isLastRead())
@@ -284,7 +284,7 @@ else if (res instanceof GridCacheEntryInfo)
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -309,7 +309,7 @@ else if (res instanceof GridCacheEntryInfo)
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 8;
+        return 9;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java
index 00c29e5f7b5c0..30aa572414265 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishRequest.java
@@ -172,7 +172,7 @@ public void miniId(int miniId) {
         }
 
         switch (writer.state()) {
-            case 21:
+            case 22:
                 if (!writer.writeInt("miniId", miniId))
                     return false;
 
@@ -194,7 +194,7 @@ public void miniId(int miniId) {
             return false;
 
         switch (reader.state()) {
-            case 21:
+            case 22:
                 miniId = reader.readInt("miniId");
 
                 if (!reader.isLastRead())
@@ -214,7 +214,7 @@ public void miniId(int miniId) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 22;
+        return 23;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java
index a1a2b5712fcdd..e3dcbf832bd14 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishResponse.java
@@ -133,19 +133,19 @@ public long threadId() {
         }
 
         switch (writer.state()) {
-            case 6:
+            case 7:
                 if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeInt("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeLong("nearThreadId", nearThreadId))
                     return false;
 
@@ -167,7 +167,7 @@ public long threadId() {
             return false;
 
         switch (reader.state()) {
-            case 6:
+            case 7:
                 errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
@@ -175,7 +175,7 @@ public long threadId() {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 miniId = reader.readInt("miniId");
 
                 if (!reader.isLastRead())
@@ -183,7 +183,7 @@ public long threadId() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 nearThreadId = reader.readLong("nearThreadId");
 
                 if (!reader.isLastRead())
@@ -203,7 +203,7 @@ public long threadId() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 9;
+        return 10;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
index a6706093b11cd..40c46ff16f49a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
@@ -301,37 +301,37 @@ private boolean isFlag(int mask) {
         }
 
         switch (writer.state()) {
-            case 20:
+            case 21:
                 if (!writer.writeByte("flags", flags))
                     return false;
 
                 writer.incrementState();
 
-            case 21:
+            case 22:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 22:
+            case 23:
                 if (!writer.writeInt("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 23:
+            case 24:
                 if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
-            case 24:
+            case 25:
                 if (!writer.writeInt("taskNameHash", taskNameHash))
                     return false;
 
                 writer.incrementState();
 
-            case 25:
+            case 26:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
@@ -353,7 +353,7 @@ private boolean isFlag(int mask) {
             return false;
 
         switch (reader.state()) {
-            case 20:
+            case 21:
                 flags = reader.readByte("flags");
 
                 if (!reader.isLastRead())
@@ -361,7 +361,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 21:
+            case 22:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -369,7 +369,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 22:
+            case 23:
                 miniId = reader.readInt("miniId");
 
                 if (!reader.isLastRead())
@@ -377,7 +377,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 23:
+            case 24:
                 subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
@@ -385,7 +385,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 24:
+            case 25:
                 taskNameHash = reader.readInt("taskNameHash");
 
                 if (!reader.isLastRead())
@@ -393,7 +393,7 @@ private boolean isFlag(int mask) {
 
                 reader.incrementState();
 
-            case 25:
+            case 26:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -413,7 +413,7 @@ private boolean isFlag(int mask) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 26;
+        return 27;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
index 8162168136d9d..8b256d19ee9fd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
@@ -358,61 +358,61 @@ public boolean hasOwnedValue(IgniteTxKey key) {
         }
 
         switch (writer.state()) {
-            case 10:
+            case 11:
                 if (!writer.writeMessage("clientRemapVer", clientRemapVer))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeMessage("dhtVer", dhtVer))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeCollection("filterFailedKeys", filterFailedKeys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeIgniteUuid("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeInt("miniId", miniId))
                     return false;
 
                 writer.incrementState();
 
-            case 15:
+            case 16:
                 if (!writer.writeCollection("ownedValKeys", ownedValKeys, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 16:
+            case 17:
                 if (!writer.writeCollection("ownedValVals", ownedValVals, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 17:
+            case 18:
                 if (!writer.writeCollection("pending", pending, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 18:
+            case 19:
                 if (!writer.writeMessage("retVal", retVal))
                     return false;
 
                 writer.incrementState();
 
-            case 19:
+            case 20:
                 if (!writer.writeMessage("writeVer", writeVer))
                     return false;
 
@@ -434,7 +434,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
             return false;
 
         switch (reader.state()) {
-            case 10:
+            case 11:
                 clientRemapVer = reader.readMessage("clientRemapVer");
 
                 if (!reader.isLastRead())
@@ -442,7 +442,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 dhtVer = reader.readMessage("dhtVer");
 
                 if (!reader.isLastRead())
@@ -450,7 +450,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 filterFailedKeys = reader.readCollection("filterFailedKeys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -458,7 +458,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 futId = reader.readIgniteUuid("futId");
 
                 if (!reader.isLastRead())
@@ -466,7 +466,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 miniId = reader.readInt("miniId");
 
                 if (!reader.isLastRead())
@@ -474,7 +474,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 15:
+            case 16:
                 ownedValKeys = reader.readCollection("ownedValKeys", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -482,7 +482,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 16:
+            case 17:
                 ownedValVals = reader.readCollection("ownedValVals", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -490,7 +490,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 17:
+            case 18:
                 pending = reader.readCollection("pending", MessageCollectionItemType.MSG);
 
                 if (!reader.isLastRead())
@@ -498,7 +498,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 18:
+            case 19:
                 retVal = reader.readMessage("retVal");
 
                 if (!reader.isLastRead())
@@ -506,7 +506,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 19:
+            case 20:
                 writeVer = reader.readMessage("writeVer");
 
                 if (!reader.isLastRead())
@@ -526,7 +526,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 20;
+        return 21;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearUnlockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearUnlockRequest.java
index 2b49889cadc27..b2ed241e91fd2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearUnlockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearUnlockRequest.java
@@ -85,7 +85,7 @@ public GridNearUnlockRequest(int cacheId, int keyCnt, boolean addDepInfo) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 8;
+        return 9;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
index 9dc7817889806..fae97668746fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
@@ -500,121 +500,121 @@ public int taskHash() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeBoolean("all", all))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeByteArray("argsBytes", argsBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeString("cacheName", cacheName))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeBoolean("cancel", cancel))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeString("clause", clause))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeString("clsName", clsName))
                     return false;
 
                 writer.incrementState();
 
-            case 9:
+            case 10:
                 if (!writer.writeBoolean("fields", fields))
                     return false;
 
                 writer.incrementState();
 
-            case 10:
+            case 11:
                 if (!writer.writeLong("id", id))
                     return false;
 
                 writer.incrementState();
 
-            case 11:
+            case 12:
                 if (!writer.writeBoolean("incBackups", incBackups))
                     return false;
 
                 writer.incrementState();
 
-            case 12:
+            case 13:
                 if (!writer.writeBoolean("incMeta", incMeta))
                     return false;
 
                 writer.incrementState();
 
-            case 13:
+            case 14:
                 if (!writer.writeBoolean("keepBinary", keepBinary))
                     return false;
 
                 writer.incrementState();
 
-            case 14:
+            case 15:
                 if (!writer.writeByteArray("keyValFilterBytes", keyValFilterBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 15:
+            case 16:
                 if (!writer.writeInt("pageSize", pageSize))
                     return false;
 
                 writer.incrementState();
 
-            case 16:
+            case 17:
                 if (!writer.writeInt("part", part))
                     return false;
 
                 writer.incrementState();
 
-            case 17:
+            case 18:
                 if (!writer.writeByteArray("rdcBytes", rdcBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 18:
+            case 19:
                 if (!writer.writeUuid("subjId", subjId))
                     return false;
 
                 writer.incrementState();
 
-            case 19:
+            case 20:
                 if (!writer.writeInt("taskHash", taskHash))
                     return false;
 
                 writer.incrementState();
 
-            case 20:
+            case 21:
                 if (!writer.writeMessage("topVer", topVer))
                     return false;
 
                 writer.incrementState();
 
-            case 21:
+            case 22:
                 if (!writer.writeByteArray("transBytes", transBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 22:
+            case 23:
                 if (!writer.writeByte("type", type != null ? (byte)type.ordinal() : -1))
                     return false;
 
@@ -636,7 +636,7 @@ public int taskHash() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 all = reader.readBoolean("all");
 
                 if (!reader.isLastRead())
@@ -644,7 +644,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 argsBytes = reader.readByteArray("argsBytes");
 
                 if (!reader.isLastRead())
@@ -652,7 +652,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 cacheName = reader.readString("cacheName");
 
                 if (!reader.isLastRead())
@@ -660,7 +660,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 cancel = reader.readBoolean("cancel");
 
                 if (!reader.isLastRead())
@@ -668,7 +668,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 clause = reader.readString("clause");
 
                 if (!reader.isLastRead())
@@ -676,7 +676,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 clsName = reader.readString("clsName");
 
                 if (!reader.isLastRead())
@@ -684,7 +684,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 9:
+            case 10:
                 fields = reader.readBoolean("fields");
 
                 if (!reader.isLastRead())
@@ -692,7 +692,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 10:
+            case 11:
                 id = reader.readLong("id");
 
                 if (!reader.isLastRead())
@@ -700,7 +700,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 11:
+            case 12:
                 incBackups = reader.readBoolean("incBackups");
 
                 if (!reader.isLastRead())
@@ -708,7 +708,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 12:
+            case 13:
                 incMeta = reader.readBoolean("incMeta");
 
                 if (!reader.isLastRead())
@@ -716,7 +716,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 13:
+            case 14:
                 keepBinary = reader.readBoolean("keepBinary");
 
                 if (!reader.isLastRead())
@@ -724,7 +724,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 14:
+            case 15:
                 keyValFilterBytes = reader.readByteArray("keyValFilterBytes");
 
                 if (!reader.isLastRead())
@@ -732,7 +732,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 15:
+            case 16:
                 pageSize = reader.readInt("pageSize");
 
                 if (!reader.isLastRead())
@@ -740,7 +740,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 16:
+            case 17:
                 part = reader.readInt("part");
 
                 if (!reader.isLastRead())
@@ -748,7 +748,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 17:
+            case 18:
                 rdcBytes = reader.readByteArray("rdcBytes");
 
                 if (!reader.isLastRead())
@@ -756,7 +756,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 18:
+            case 19:
                 subjId = reader.readUuid("subjId");
 
                 if (!reader.isLastRead())
@@ -764,7 +764,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 19:
+            case 20:
                 taskHash = reader.readInt("taskHash");
 
                 if (!reader.isLastRead())
@@ -772,7 +772,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 20:
+            case 21:
                 topVer = reader.readMessage("topVer");
 
                 if (!reader.isLastRead())
@@ -780,7 +780,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 21:
+            case 22:
                 transBytes = reader.readByteArray("transBytes");
 
                 if (!reader.isLastRead())
@@ -788,7 +788,7 @@ public int taskHash() {
 
                 reader.incrementState();
 
-            case 22:
+            case 23:
                 byte typeOrd;
 
                 typeOrd = reader.readByte("type");
@@ -812,7 +812,7 @@ public int taskHash() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 23;
+        return 24;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponse.java
index 13e0915875b92..a1650be162c46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryResponse.java
@@ -287,37 +287,37 @@ public boolean fields() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeCollection("dataBytes", dataBytes, MessageCollectionItemType.BYTE_ARR))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeByteArray("errBytes", errBytes))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeBoolean("fields", fields))
                     return false;
 
                 writer.incrementState();
 
-            case 6:
+            case 7:
                 if (!writer.writeBoolean("finished", finished))
                     return false;
 
                 writer.incrementState();
 
-            case 7:
+            case 8:
                 if (!writer.writeCollection("metaDataBytes", metaDataBytes, MessageCollectionItemType.BYTE_ARR))
                     return false;
 
                 writer.incrementState();
 
-            case 8:
+            case 9:
                 if (!writer.writeLong("reqId", reqId))
                     return false;
 
@@ -339,7 +339,7 @@ public boolean fields() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 dataBytes = reader.readCollection("dataBytes", MessageCollectionItemType.BYTE_ARR);
 
                 if (!reader.isLastRead())
@@ -347,7 +347,7 @@ public boolean fields() {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 errBytes = reader.readByteArray("errBytes");
 
                 if (!reader.isLastRead())
@@ -355,7 +355,7 @@ public boolean fields() {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 fields = reader.readBoolean("fields");
 
                 if (!reader.isLastRead())
@@ -363,7 +363,7 @@ public boolean fields() {
 
                 reader.incrementState();
 
-            case 6:
+            case 7:
                 finished = reader.readBoolean("finished");
 
                 if (!reader.isLastRead())
@@ -371,7 +371,7 @@ public boolean fields() {
 
                 reader.incrementState();
 
-            case 7:
+            case 8:
                 metaDataBytes = reader.readCollection("metaDataBytes", MessageCollectionItemType.BYTE_ARR);
 
                 if (!reader.isLastRead())
@@ -379,7 +379,7 @@ public boolean fields() {
 
                 reader.incrementState();
 
-            case 8:
+            case 9:
                 reqId = reader.readLong("reqId");
 
                 if (!reader.isLastRead())
@@ -399,7 +399,7 @@ public boolean fields() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 9;
+        return 10;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java
index ef0157ee70caa..9ba9ad2d1cd28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryBatchAck.java
@@ -90,15 +90,14 @@ Map<Integer, Long> updateCntrs() {
         }
 
         switch (writer.state()) {
-            case 3:
+            case 4:
                 if (!writer.writeUuid("routineId", routineId))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
-                if (!writer.writeMap("updateCntrs", updateCntrs, MessageCollectionItemType.INT,
-                    MessageCollectionItemType.LONG))
+            case 5:
+                if (!writer.writeMap("updateCntrs", updateCntrs, MessageCollectionItemType.INT, MessageCollectionItemType.LONG))
                     return false;
 
                 writer.incrementState();
@@ -119,7 +118,7 @@ Map<Integer, Long> updateCntrs() {
             return false;
 
         switch (reader.state()) {
-            case 3:
+            case 4:
                 routineId = reader.readUuid("routineId");
 
                 if (!reader.isLastRead())
@@ -127,9 +126,8 @@ Map<Integer, Long> updateCntrs() {
 
                 reader.incrementState();
 
-            case 4:
-                updateCntrs = reader.readMap("updateCntrs", MessageCollectionItemType.INT,
-                    MessageCollectionItemType.LONG, false);
+            case 5:
+                updateCntrs = reader.readMap("updateCntrs", MessageCollectionItemType.INT, MessageCollectionItemType.LONG, false);
 
                 if (!reader.isLastRead())
                     return false;
@@ -153,7 +151,7 @@ Map<Integer, Long> updateCntrs() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 5;
+        return 6;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 662eae5cdc52c..d44728fff96ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -655,23 +655,19 @@ private void sendResponseOnTimeoutOrError(@Nullable IgniteCheckedException e,
     private boolean needRemap(AffinityTopologyVersion expVer,
         AffinityTopologyVersion curVer,
         GridNearTxPrepareRequest req) {
-        if (expVer.equals(curVer))
+        if (curVer.compareTo(expVer) <= 0 && curVer.compareTo(req.lastAffinityChangedTopologyVersion()) >= 0)
             return false;
 
         for (IgniteTxEntry e : F.concat(false, req.reads(), req.writes())) {
             GridCacheContext ctx = e.context();
 
-            Collection<ClusterNode> cacheNodes0 = ctx.discovery().cacheGroupAffinityNodes(ctx.groupId(), expVer);
-            Collection<ClusterNode> cacheNodes1 = ctx.discovery().cacheGroupAffinityNodes(ctx.groupId(), curVer);
-
-            if (!cacheNodes0.equals(cacheNodes1) || ctx.affinity().affinityTopologyVersion().compareTo(curVer) < 0)
-                return true;
+            assert e.key().partition() != -1;
 
             try {
-                List<List<ClusterNode>> aff1 = ctx.affinity().assignments(expVer);
-                List<List<ClusterNode>> aff2 = ctx.affinity().assignments(curVer);
+                List<ClusterNode> aff1 = ctx.affinity().assignments(expVer).get(e.key().partition());
+                List<ClusterNode> aff2 = ctx.affinity().assignments(curVer).get(e.key().partition());
 
-                if (!aff1.equals(aff2))
+                if (!aff1.containsAll(aff2) || aff2.isEmpty() ||!aff1.get(0).equals(aff2.get(0)))
                     return true;
             }
             catch (IllegalStateException ignored) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksRequest.java
index 94fe00527a488..86109c8f323a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksRequest.java
@@ -149,13 +149,13 @@ public Collection<IgniteTxKey> txKeys() {
         }
 
         switch (writer.state()) {
-            case 2:
+            case 3:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 3:
+            case 4:
                 if (!writer.writeObjectArray("txKeysArr", txKeysArr, MessageCollectionItemType.MSG))
                     return false;
 
@@ -177,7 +177,7 @@ public Collection<IgniteTxKey> txKeys() {
             return false;
 
         switch (reader.state()) {
-            case 2:
+            case 3:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -185,7 +185,7 @@ public Collection<IgniteTxKey> txKeys() {
 
                 reader.incrementState();
 
-            case 3:
+            case 4:
                 txKeysArr = reader.readObjectArray("txKeysArr", MessageCollectionItemType.MSG, IgniteTxKey.class);
 
                 if (!reader.isLastRead())
@@ -205,7 +205,7 @@ public Collection<IgniteTxKey> txKeys() {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 4;
+        return 5;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java
index a5c8f0917da8c..df5caa978b609 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/TxLocksResponse.java
@@ -239,25 +239,25 @@ public void addKey(IgniteTxKey key) {
         }
 
         switch (writer.state()) {
-            case 2:
+            case 3:
                 if (!writer.writeLong("futId", futId))
                     return false;
 
                 writer.incrementState();
 
-            case 3:
+            case 4:
                 if (!writer.writeObjectArray("locksArr", locksArr, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 4:
+            case 5:
                 if (!writer.writeObjectArray("nearTxKeysArr", nearTxKeysArr, MessageCollectionItemType.MSG))
                     return false;
 
                 writer.incrementState();
 
-            case 5:
+            case 6:
                 if (!writer.writeObjectArray("txKeysArr", txKeysArr, MessageCollectionItemType.MSG))
                     return false;
 
@@ -279,7 +279,7 @@ public void addKey(IgniteTxKey key) {
             return false;
 
         switch (reader.state()) {
-            case 2:
+            case 3:
                 futId = reader.readLong("futId");
 
                 if (!reader.isLastRead())
@@ -287,7 +287,7 @@ public void addKey(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 3:
+            case 4:
                 locksArr = reader.readObjectArray("locksArr", MessageCollectionItemType.MSG, TxLockList.class);
 
                 if (!reader.isLastRead())
@@ -295,7 +295,7 @@ public void addKey(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 4:
+            case 5:
                 nearTxKeysArr = reader.readObjectArray("nearTxKeysArr", MessageCollectionItemType.MSG, IgniteTxKey.class);
 
                 if (!reader.isLastRead())
@@ -303,7 +303,7 @@ public void addKey(IgniteTxKey key) {
 
                 reader.incrementState();
 
-            case 5:
+            case 6:
                 txKeysArr = reader.readObjectArray("txKeysArr", MessageCollectionItemType.MSG, IgniteTxKey.class);
 
                 if (!reader.isLastRead())
@@ -323,7 +323,7 @@ public void addKey(IgniteTxKey key) {
 
     /** {@inheritDoc} */
     @Override public byte fieldsCount() {
-        return 6;
+        return 7;
     }
 
     /** {@inheritDoc} */

From 2f1d2d63ad93bc2238fbf10c63c653de6594cc64 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Tue, 16 Oct 2018 17:49:40 +0300
Subject: [PATCH 425/543] IGNITE-9895 DiscoveryMessageNotifierWorker must be
 instanceof IgniteDiscoveryThread - Fixes #5000.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../internal/managers/discovery/GridDiscoveryManager.java      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 3ce6dd9b664f8..0d8d9a3a7f8d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -132,6 +132,7 @@
 import org.apache.ignite.spi.discovery.DiscoverySpiMutableCustomMessageSupport;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
+import org.apache.ignite.spi.discovery.IgniteDiscoveryThread;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
 import org.apache.ignite.thread.IgniteThread;
@@ -2665,7 +2666,7 @@ public void scheduleSegmentCheck() {
     /**
      *
      */
-    private class DiscoveryMessageNotifyerWorker extends GridWorker {
+    private class DiscoveryMessageNotifyerWorker extends GridWorker implements IgniteDiscoveryThread {
         /** Queue. */
         private final BlockingQueue<T2<GridFutureAdapter, Runnable>> queue = new LinkedBlockingQueue<>();
 

From 4ceb84525457a9fb960079a71a0ee464b5105a07 Mon Sep 17 00:00:00 2001
From: macrergate <macrergate@gmail.com>
Date: Tue, 16 Oct 2018 18:17:10 +0300
Subject: [PATCH 426/543] IGNITE-9430 Add ability to override all caches's
 "rebalanceThrottle" option via JVM node option - Fixes #4944.

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit 1db895545b303b8945b9b8ade237e17ccd434f5c)
---
 .../org/apache/ignite/IgniteSystemProperties.java   |  6 ++++++
 .../dht/preloader/GridDhtPartitionSupplier.java     | 13 ++++++++++++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index ff8d38a094e69..746c00227b9fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -947,6 +947,12 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_REUSE_MEMORY_ON_DEACTIVATE = "IGNITE_REUSE_MEMORY_ON_DEACTIVATE";
 
+    /**
+     * System property to override {@link CacheConfiguration#rebalanceThrottle} configuration property for all caches.
+     * {@code 0} by default, which means that override is disabled.
+     */
+    public static final String IGNITE_REBALANCE_THROTTLE_OVERRIDE = "IGNITE_REBALANCE_THROTTLE_OVERRIDE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 5b5f765517fe1..3b2a03f91c22b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -25,6 +25,7 @@
 import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
@@ -38,6 +39,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.T3;
+import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -64,6 +66,10 @@ class GridDhtPartitionSupplier {
     /** Supply context map. T3: nodeId, topicId, topVer. */
     private final Map<T3<UUID, Integer, AffinityTopologyVersion>, SupplyContext> scMap = new HashMap<>();
 
+    /** Override for rebalance throttle. */
+    private long rebalanceThrottleOverride =
+        IgniteSystemProperties.getLong(IgniteSystemProperties.IGNITE_REBALANCE_THROTTLE_OVERRIDE, 0);
+
     /**
      * @param grp Cache group.
      */
@@ -75,6 +81,9 @@ class GridDhtPartitionSupplier {
         log = grp.shared().logger(getClass());
 
         top = grp.topology();
+
+        if (rebalanceThrottleOverride > 0)
+            LT.info(log, "Using rebalance throttle override: " + rebalanceThrottleOverride);
     }
 
     /**
@@ -478,7 +487,9 @@ private boolean reply(
             grp.shared().io().sendOrderedMessage(demander, demandMsg.topic(), supplyMsg, grp.ioPolicy(), demandMsg.timeout());
 
             // Throttle preloading.
-            if (grp.config().getRebalanceThrottle() > 0)
+            if (rebalanceThrottleOverride > 0)
+                U.sleep(rebalanceThrottleOverride);
+            else if (grp.config().getRebalanceThrottle() > 0)
                 U.sleep(grp.config().getRebalanceThrottle());
 
             return true;

From fbd83fd251344e3867a5319ae6ae95c15efe6cb0 Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Thu, 11 Oct 2018 12:42:27 +0300
Subject: [PATCH 427/543] IGNITE-9550 fix Get operation returns null for a lost
 partition with READ_SAFE policy - Fixes #4947.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit cbbd9ad4a329d69ca88517ce2c480bffcf413e78)
Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
---
 .../cache/CacheAffinitySharedManager.java     |  22 +-
 .../processors/cache/GridCacheAdapter.java    |   9 +-
 .../GridCachePartitionExchangeManager.java    |  18 +-
 .../processors/cache/GridCacheProcessor.java  | 238 +++++++++++++++---
 .../processors/cache/GridCacheProxyImpl.java  |   7 +-
 .../cache/IgniteCacheProxyImpl.java           |  95 ++++++-
 .../dht/GridPartitionedGetFuture.java         |   4 +-
 .../dht/GridPartitionedSingleGetFuture.java   |   4 +-
 .../GridNearAtomicSingleUpdateFuture.java     |   2 +-
 .../atomic/GridNearAtomicUpdateFuture.java    |   2 +-
 .../GridDhtPartitionsExchangeFuture.java      |  54 +++-
 .../distributed/near/GridNearGetFuture.java   |  27 +-
 .../cache/query/GridCacheQueryAdapter.java    |   2 +-
 .../CacheContinuousQueryHandler.java          |   2 +-
 .../CacheContinuousQueryManager.java          |   2 +-
 .../cluster/DiscoveryDataClusterState.java    |  29 ++-
 .../cluster/GridClusterStateProcessor.java    |   2 +-
 ...cheResultIsNotNullOnPartitionLossTest.java | 213 ++++++++++++++++
 .../testsuites/IgniteCacheTestSuite4.java     |   3 +
 .../kafka/KafkaIgniteStreamerSelfTest.java    |  22 +-
 20 files changed, 639 insertions(+), 118 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index dd4161c0d7b99..9fae87be6f50b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -409,7 +409,8 @@ void onCacheGroupCreated(CacheGroupContext grp) {
         ClientCacheChangeDummyDiscoveryMessage msg,
         boolean crd,
         AffinityTopologyVersion topVer,
-        DiscoCache discoCache) {
+        DiscoCache discoCache
+    ) {
         Map<String, DynamicCacheChangeRequest> startReqs = msg.startRequests();
 
         if (startReqs == null)
@@ -435,11 +436,13 @@ void onCacheGroupCreated(CacheGroupContext grp) {
 
                 DynamicCacheChangeRequest startReq = startReqs.get(desc.cacheName());
 
-                cctx.cache().prepareCacheStart(desc.cacheConfiguration(),
+                cctx.cache().prepareCacheStart(
+                    desc.cacheConfiguration(),
                     desc,
                     startReq.nearCacheConfiguration(),
                     topVer,
-                    startReq.disabledAfterStart());
+                    startReq.disabledAfterStart()
+                );
 
                 startedInfos.put(desc.cacheId(), startReq.nearCacheConfiguration() != null);
 
@@ -556,6 +559,8 @@ else if (!fetchFuts.containsKey(grp.groupId())) {
 
         cctx.cache().initCacheProxies(topVer, null);
 
+        startReqs.keySet().forEach(req -> cctx.cache().completeProxyInitialize(req));
+
         cctx.cache().completeClientCacheChangeFuture(msg.requestId(), null);
 
         return startedInfos;
@@ -570,7 +575,8 @@ else if (!fetchFuts.containsKey(grp.groupId())) {
     private Set<Integer> processCacheCloseRequests(
         ClientCacheChangeDummyDiscoveryMessage msg,
         boolean crd,
-        AffinityTopologyVersion topVer) {
+        AffinityTopologyVersion topVer
+    ) {
         Set<String> cachesToClose = msg.cachesToClose();
 
         if (cachesToClose == null)
@@ -861,7 +867,7 @@ private void processCacheStartRequests(
                 assert cctx.cacheContext(cacheDesc.cacheId()) == null
                     : "Starting cache has not null context: " + cacheDesc.cacheName();
 
-                IgniteCacheProxyImpl cacheProxy = (IgniteCacheProxyImpl) cctx.cache().jcacheProxy(req.cacheName());
+                IgniteCacheProxyImpl cacheProxy = cctx.cache().jcacheProxy(req.cacheName(), false);
 
                 // If it has proxy then try to start it
                 if (cacheProxy != null) {
@@ -879,11 +885,13 @@ private void processCacheStartRequests(
 
             try {
                 if (startCache) {
-                    cctx.cache().prepareCacheStart(req.startCacheConfiguration(),
+                    cctx.cache().prepareCacheStart(
+                        req.startCacheConfiguration(),
                         cacheDesc,
                         nearCfg,
                         evts.topologyVersion(),
-                        req.disabledAfterStart());
+                        req.disabledAfterStart()
+                    );
 
                     if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) {
                         if (fut.events().discoveryCache().cacheGroupAffinityNodes(cacheDesc.groupId()).isEmpty())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index fa3ec4543237b..e43865a56d8d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -4124,7 +4124,10 @@ public void awaitLastFut() {
 
                             assert topVer != null && topVer.topologyVersion() > 0 : tx;
 
-                            ctx.affinity().affinityReadyFuture(topVer.topologyVersion() + 1).get();
+                            AffinityTopologyVersion awaitVer = new AffinityTopologyVersion(
+                                topVer.topologyVersion() + 1, 0);
+
+                            ctx.shared().exchange().affinityReadyFuture(awaitVer).get();
 
                             continue;
                         }
@@ -4873,8 +4876,10 @@ public void execute(boolean retry) {
 
                                 assert topVer != null && topVer.topologyVersion() > 0 : tx;
 
+                                AffinityTopologyVersion awaitVer = new AffinityTopologyVersion(topVer.topologyVersion() + 1, 0);
+
                                 IgniteInternalFuture<?> topFut =
-                                    ctx.affinity().affinityReadyFuture(topVer.topologyVersion() + 1);
+                                    ctx.shared().exchange().affinityReadyFuture(awaitVer);
 
                                 topFut.listen(new IgniteInClosure<IgniteInternalFuture<?>>() {
                                     @Override public void apply(IgniteInternalFuture<?> topFut) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 548d6cc8114e8..afd94a751df64 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -101,6 +101,7 @@
 import org.apache.ignite.internal.util.GridPartitionStateMap;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
+import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -883,7 +884,7 @@ public void lastFinishedFuture(GridDhtPartitionsExchangeFuture fut) {
         while (true) {
             GridDhtPartitionsExchangeFuture cur = lastFinishedFut.get();
 
-            if (cur == null || fut.topologyVersion().compareTo(cur.topologyVersion()) > 0) {
+            if (fut.topologyVersion() != null && (cur == null || fut.topologyVersion().compareTo(cur.topologyVersion()) > 0)) {
                 if (lastFinishedFut.compareAndSet(cur, fut))
                     break;
             }
@@ -896,25 +897,14 @@ public void lastFinishedFuture(GridDhtPartitionsExchangeFuture fut) {
      * @param ver Topology version.
      * @return Future or {@code null} is future is already completed.
      */
-    @Nullable public IgniteInternalFuture<?> affinityReadyFuture(AffinityTopologyVersion ver) {
-        GridDhtPartitionsExchangeFuture lastInitializedFut0 = lastInitializedFut;
-
-        if (lastInitializedFut0 != null && lastInitializedFut0.initialVersion().compareTo(ver) == 0
-            && lastInitializedFut0.changedAffinity()) {
-            if (log.isTraceEnabled())
-                log.trace("Return lastInitializedFut for topology ready future " +
-                    "[ver=" + ver + ", fut=" + lastInitializedFut0 + ']');
-
-            return lastInitializedFut0;
-        }
-
+    @Nullable public IgniteInternalFuture<AffinityTopologyVersion> affinityReadyFuture(AffinityTopologyVersion ver) {
         AffinityTopologyVersion topVer = exchFuts.readyTopVer();
 
         if (topVer.compareTo(ver) >= 0) {
             if (log.isTraceEnabled())
                 log.trace("Return finished future for topology ready future [ver=" + ver + ", topVer=" + topVer + ']');
 
-            return null;
+            return new GridFinishedFuture<>(topVer);
         }
 
         GridFutureAdapter<AffinityTopologyVersion> fut = F.addIfAbsent(readyFuts, ver,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 62e73b6b151e7..b37b000d775dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -34,6 +34,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import javax.management.MBeanServer;
 import org.apache.ignite.IgniteCheckedException;
@@ -1102,6 +1103,9 @@ private void stopCacheOnReconnect(GridCacheContext cctx, List<GridCacheAdapter>
         sharedCtx.removeCacheContext(cctx);
 
         caches.remove(cctx.name());
+
+        completeProxyInitialize(cctx.name());
+
         jCacheProxies.remove(cctx.name());
 
         stoppedCaches.add(cctx.cache());
@@ -1299,9 +1303,10 @@ private void stopCache(GridCacheAdapter<?, ?> cache, boolean cancel, boolean des
             if (destroy && (pageStore = sharedCtx.pageStore()) != null) {
                 try {
                     pageStore.removeCacheData(new StoredCacheData(ctx.config()));
-                } catch (IgniteCheckedException e) {
+                }
+                catch (IgniteCheckedException e) {
                     U.error(log, "Failed to delete cache configuration data while destroying cache" +
-                            "[cache=" + ctx.name() + "]", e);
+                        "[cache=" + ctx.name() + "]", e);
                 }
             }
 
@@ -1694,6 +1699,67 @@ private GridCacheContext createCache(CacheConfiguration<?, ?> cfg,
         return ret;
     }
 
+    /**
+     *
+     * @param reqs Cache requests to start.
+     * @param fut Completable future.
+     */
+    public void registrateProxyRestart(Map<String, DynamicCacheChangeRequest> reqs, GridFutureAdapter<?> fut) {
+        for (IgniteCacheProxyImpl<?, ?> proxy : jCacheProxies.values()) {
+            if (reqs.containsKey(proxy.getName()) &&
+                proxy.isRestarting() &&
+                !reqs.get(proxy.getName()).disabledAfterStart()
+            )
+                proxy.registrateFutureRestart(fut);
+        }
+    }
+
+    /**
+     *
+     * @param reqs Cache requests to start.
+     * @param initVer Init exchange version.
+     * @param doneVer Finish excahnge vertison.
+     */
+    public void completeProxyRestart(
+        Map<String, DynamicCacheChangeRequest> reqs,
+        AffinityTopologyVersion initVer,
+        AffinityTopologyVersion doneVer
+    ) {
+        if (initVer == null || doneVer == null)
+            return;
+
+        for (GridCacheAdapter<?, ?> cache : caches.values()) {
+            GridCacheContext<?, ?> cacheCtx = cache.context();
+
+            if (reqs.containsKey(cache.name()) ||
+                (cacheCtx.startTopologyVersion().compareTo(initVer) <= 0 ||
+                    cacheCtx.startTopologyVersion().compareTo(doneVer) <= 0))
+                completeProxyInitialize(cache.name());
+
+            if (
+                cacheCtx.startTopologyVersion().compareTo(initVer) >= 0 &&
+                    cacheCtx.startTopologyVersion().compareTo(doneVer) <= 0
+            ) {
+                IgniteCacheProxyImpl<?, ?> proxy = jCacheProxies.get(cache.name());
+
+                boolean canRestart = true;
+
+                DynamicCacheChangeRequest req = reqs.get(cache.name());
+
+                if (req != null) {
+                    canRestart = !req.disabledAfterStart();
+                }
+
+                if (proxy != null && proxy.isRestarting() && canRestart) {
+                    proxy.onRestarted(cacheCtx, cache);
+
+                    if (cacheCtx.dataStructuresCache())
+                        ctx.dataStructures().restart(proxy.internalProxy());
+                }
+            }
+        }
+    }
+
     /**
      * Gets a collection of currently started caches.
      *
@@ -1709,8 +1775,8 @@ public Collection<String> cacheNames() {
     }
 
     /**
-     * Gets public cache that can be used for query execution.
-     * If cache isn't created on current node it will be started.
+     * Gets public cache that can be used for query execution. If cache isn't created on current node it will be
+     * started.
      *
      * @param start Start cache.
      * @param inclLoc Include local caches.
@@ -1886,8 +1952,8 @@ public Collection<DynamicCacheDescriptor> startReceivedCaches(UUID nodeId, Affin
      * @param desc Cache descriptor.
      * @param reqNearCfg Near configuration if specified for client cache start request.
      * @param exchTopVer Current exchange version.
-     * @param disabledAfterStart If true, then we will discard restarting state from proxies. If false then we will change
-     *  state of proxies to restarting
+     * @param disabledAfterStart If true, then we will discard restarting state from proxies. If false then we will
+     * change state of proxies to restarting
      * @throws IgniteCheckedException If failed.
      */
     void prepareCacheStart(
@@ -1974,20 +2040,11 @@ else if (CU.affinityNode(ctx.discovery().localNode(), desc.groupDescriptor().con
         grp.onCacheStarted(cacheCtx);
 
         onKernalStart(cache);
-
-        IgniteCacheProxyImpl<?, ?> proxy = jCacheProxies.get(ccfg.getName());
-
-        if (!disabledAfterStart && proxy != null && proxy.isRestarting()) {
-            proxy.onRestarted(cacheCtx, cache);
-
-            if (cacheCtx.dataStructuresCache())
-                ctx.dataStructures().restart(proxy.internalProxy());
-        }
     }
 
     /**
-     * Restarts proxies of caches if they was marked as restarting.
-     * Requires external synchronization - shouldn't be called concurrently with another caches restart.
+     * Restarts proxies of caches if they was marked as restarting. Requires external synchronization - shouldn't be
+     * called concurrently with another caches restart.
      */
     public void restartProxies() {
         for (IgniteCacheProxyImpl<?, ?> proxy : jCacheProxies.values()) {
@@ -2061,7 +2118,7 @@ private CacheGroupContext startCacheGroup(
 
         CacheGroupContext old = cacheGrps.put(desc.groupId(), grp);
 
-        if (!grp.systemCache()  && !U.IGNITE_MBEANS_DISABLED) {
+        if (!grp.systemCache() && !U.IGNITE_MBEANS_DISABLED) {
             try {
                 U.registerMBean(ctx.config().getMBeanServer(), ctx.igniteInstanceName(), CACHE_GRP_METRICS_MBEAN_GRP,
                     grp.cacheOrGroupName(), grp.mxBean(), CacheGroupMetricsMXBean.class);
@@ -2083,7 +2140,7 @@ private CacheGroupContext startCacheGroup(
      */
     void blockGateway(String cacheName, boolean stop, boolean restart) {
         // Break the proxy before exchange future is done.
-        IgniteCacheProxyImpl<?, ?> proxy = jCacheProxies.get(cacheName);
+        IgniteCacheProxyImpl<?, ?> proxy = jcacheProxy(cacheName, false);
 
         if (restart) {
             GridCacheAdapter<?, ?> cache = caches.get(cacheName);
@@ -2127,8 +2184,11 @@ private void stopGateway(DynamicCacheChangeRequest req) {
             if (proxy != null)
                 proxy.restart();
         }
-        else
+        else {
+            completeProxyInitialize(req.cacheName());
+
             proxy = jCacheProxies.remove(req.cacheName());
+        }
 
         if (proxy != null)
             proxy.context().gate().onStopped();
@@ -2169,12 +2229,12 @@ void initCacheProxies(AffinityTopologyVersion startTopVer, @Nullable Throwable e
 
             if (cacheCtx.startTopologyVersion().equals(startTopVer)) {
                 if (!jCacheProxies.containsKey(cacheCtx.name())) {
-                    IgniteCacheProxyImpl newProxy = new IgniteCacheProxyImpl(cache.context(), cache, false);
+                    IgniteCacheProxyImpl<?, ?> newProxy = new IgniteCacheProxyImpl(cache.context(), cache, false);
 
                     if (!cache.active())
                         newProxy.restart();
 
-                    jCacheProxies.putIfAbsent(cacheCtx.name(), newProxy);
+                    addjCacheProxy(cacheCtx.name(), newProxy);
                 }
 
                 if (cacheCtx.preloader() != null)
@@ -2195,6 +2255,8 @@ Set<Integer> closeCaches(Set<String> cachesToClose, boolean retClientCaches) {
 
         try {
             for (String cacheName : cachesToClose) {
+                completeProxyInitialize(cacheName);
+
                 blockGateway(cacheName, false, false);
 
                 GridCacheContext ctx = sharedCtx.cacheContext(CU.cacheId(cacheName));
@@ -2241,12 +2303,16 @@ private void closeCache(GridCacheContext cctx, boolean destroy) {
             assert cache != null : cctx.name();
 
             jCacheProxies.put(cctx.name(), new IgniteCacheProxyImpl(cache.context(), cache, false));
+
+            completeProxyInitialize(cctx.name());
         }
         else {
             jCacheProxies.remove(cctx.name());
 
             cctx.gate().onStopped();
 
+            completeProxyInitialize(cctx.name());
+
             sharedCtx.database().checkpointReadLock();
 
             try {
@@ -2262,8 +2328,8 @@ private void closeCache(GridCacheContext cctx, boolean destroy) {
     }
 
     /**
-     * Called during the rollback of the exchange partitions procedure
-     * in order to stop the given cache even if it's not fully initialized (e.g. failed on cache init stage).
+     * Called during the rollback of the exchange partitions procedure in order to stop the given cache even if it's not
+     * fully initialized (e.g. failed on cache init stage).
      *
      * @param exchActions Stop requests.
      */
@@ -3259,7 +3325,7 @@ public boolean walEnabled(String cacheName) {
     IgniteInternalFuture<?> dynamicCloseCache(String cacheName) {
         assert cacheName != null;
 
-        IgniteCacheProxy<?, ?> proxy = jCacheProxies.get(cacheName);
+        IgniteCacheProxy<?, ?> proxy = jcacheProxy(cacheName, false);
 
         if (proxy == null || proxy.isProxyClosed())
             return new GridFinishedFuture<>(); // No-op.
@@ -3432,6 +3498,7 @@ private Collection<DynamicCacheStartFuture> initiateCacheChanges(
 
     /**
      * Authorize creating cache.
+     *
      * @param cfg Cache configuration.
      * @param secCtx Optional security context.
      */
@@ -3445,6 +3512,7 @@ private void authorizeCacheCreate(CacheConfiguration cfg, SecurityContext secCtx
 
     /**
      * Authorize dynamic cache management for this node.
+     *
      * @param req start/stop cache request.
      */
     private void authorizeCacheChange(DynamicCacheChangeRequest req) {
@@ -3527,7 +3595,7 @@ else if (msg0 instanceof WalStateFinishMessage)
             return cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer);
 
         if (msg instanceof DynamicCacheChangeFailureMessage)
-            cachesInfo.onCacheChangeRequested((DynamicCacheChangeFailureMessage) msg, topVer);
+            cachesInfo.onCacheChangeRequested((DynamicCacheChangeFailureMessage)msg, topVer);
 
         if (msg instanceof ClientCacheChangeDiscoveryMessage)
             cachesInfo.onClientCacheChange((ClientCacheChangeDiscoveryMessage)msg, node);
@@ -3767,11 +3835,60 @@ public <K, V> IgniteInternalCache<K, V> cache(String name) {
         if (log.isDebugEnabled())
             log.debug("Getting cache for name: " + name);
 
-        IgniteCacheProxy<K, V> jcache = (IgniteCacheProxy<K, V>)jCacheProxies.get(name);
+        IgniteCacheProxy<K, V> jcache = (IgniteCacheProxy<K, V>)jcacheProxy(name, true);
 
         return jcache == null ? null : jcache.internalProxy();
     }
 
+    /**
+     * Await proxy initialization.
+     *
+     * @param jcache Cache proxy.
+     */
+    private void awaitInitializeProxy(IgniteCacheProxyImpl<?, ?> jcache) {
+        if (jcache != null) {
+            CountDownLatch initLatch = jcache.getInitLatch();
+
+            try {
+                while (initLatch.getCount() > 0) {
+                    initLatch.await(2000, TimeUnit.MILLISECONDS);
+
+                    if (log.isInfoEnabled())
+                        log.info("Failed to wait proxy initialization, cache=" + jcache.getName() +
+                            ", localNodeId=" + ctx.localNodeId());
+                }
+            }
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                // Ignore intteruption.
+            }
+        }
+    }
+
+    /**
+     * @param name Cache name.
+     */
+    public void completeProxyInitialize(String name) {
+        IgniteCacheProxyImpl<?, ?> jcache = jCacheProxies.get(name);
+
+        if (jcache != null) {
+            CountDownLatch proxyInitLatch = jcache.getInitLatch();
+
+            if (proxyInitLatch.getCount() > 0) {
+                if (log.isInfoEnabled())
+                    log.info("Finish proxy initialization, cacheName=" + name +
+                        ", localNodeId=" + ctx.localNodeId());
+
+                proxyInitLatch.countDown();
+            }
+        }
+        else {
+            if (log.isInfoEnabled())
+                log.info("Can not finish proxy initialization because proxy does not exist, cacheName=" + name +
+                    ", localNodeId=" + ctx.localNodeId());
+        }
+    }
+
     /**
      * @param name Cache name.
      * @return Cache instance for given name.
@@ -3788,18 +3905,21 @@ public <K, V> IgniteInternalCache<K, V> getOrStartCache(String name) throws Igni
      * @throws IgniteCheckedException If failed.
      */
     @SuppressWarnings("unchecked")
-    public <K, V> IgniteInternalCache<K, V> getOrStartCache(String name, CacheConfiguration ccfg) throws IgniteCheckedException {
+    public <K, V> IgniteInternalCache<K, V> getOrStartCache(
+        String name,
+        CacheConfiguration ccfg
+    ) throws IgniteCheckedException {
         assert name != null;
 
         if (log.isDebugEnabled())
             log.debug("Getting cache for name: " + name);
 
-        IgniteCacheProxy<?, ?> cache = jCacheProxies.get(name);
+        IgniteCacheProxy<?, ?> cache = jcacheProxy(name, true);
 
         if (cache == null) {
             dynamicStartCache(ccfg, name, null, false, ccfg == null, true).get();
 
-            cache = jCacheProxies.get(name);
+            cache = jcacheProxy(name, true);
         }
 
         return cache == null ? null : (IgniteInternalCache<K, V>)cache.internalProxy();
@@ -3843,13 +3963,21 @@ public <K, V> IgniteInternalCache<K, V> utilityCache() {
      */
     private <K, V> IgniteInternalCache<K, V> internalCacheEx(String name) {
         if (ctx.discovery().localNode().isClient()) {
-            IgniteCacheProxy<K, V> proxy = (IgniteCacheProxy<K, V>)jCacheProxies.get(name);
+            IgniteCacheProxy<K, V> proxy = (IgniteCacheProxy<K, V>)jcacheProxy(name, true);
 
             if (proxy == null) {
                 GridCacheAdapter<?, ?> cacheAdapter = caches.get(name);
 
-                if (cacheAdapter != null)
+                if (cacheAdapter != null) {
                     proxy = new IgniteCacheProxyImpl(cacheAdapter.context(), cacheAdapter, false);
+
+                    IgniteCacheProxyImpl<?, ?> prev = addjCacheProxy(name, (IgniteCacheProxyImpl<?, ?>)proxy);
+
+                    if (prev != null)
+                        proxy = (IgniteCacheProxy<K, V>)prev;
+
+                    completeProxyInitialize(proxy.getName());
+                }
             }
 
             assert proxy != null : name;
@@ -3881,7 +4009,7 @@ public <K, V> IgniteInternalCache<K, V> publicCache(String name) {
         if (!desc.cacheType().userCache())
             throw new IllegalStateException("Failed to get cache because it is a system cache: " + name);
 
-        IgniteCacheProxy<K, V> jcache = (IgniteCacheProxy<K, V>)jCacheProxies.get(name);
+        IgniteCacheProxy<K, V> jcache = (IgniteCacheProxy<K, V>)jcacheProxy(name, true);
 
         if (jcache == null)
             throw new IllegalArgumentException("Cache is not started: " + name);
@@ -3922,16 +4050,16 @@ public <K, V> IgniteCacheProxy<K, V> publicJCache(String cacheName) throws Ignit
         if (desc != null && !desc.cacheType().userCache())
             throw new IllegalStateException("Failed to get cache because it is a system cache: " + cacheName);
 
-        IgniteCacheProxyImpl<?, ?> cache = jCacheProxies.get(cacheName);
+        IgniteCacheProxyImpl<?, ?> proxy = jcacheProxy(cacheName, true);
 
         // Try to start cache, there is no guarantee that cache will be instantiated.
-        if (cache == null) {
+        if (proxy == null) {
             dynamicStartCache(null, cacheName, null, false, failIfNotStarted, checkThreadTx).get();
 
-            cache = jCacheProxies.get(cacheName);
+            proxy = jcacheProxy(cacheName, true);
         }
 
-        return cache != null ? (IgniteCacheProxy<K, V>)cache.gatewayWrapper() : null;
+        return proxy != null ? (IgniteCacheProxy<K, V>)proxy.gatewayWrapper() : null;
     }
 
     /**
@@ -4048,7 +4176,22 @@ else if (ctx.clientDisconnected()) {
     public <K, V> IgniteCacheProxy<K, V> jcache(String name) {
         assert name != null;
 
-        IgniteCacheProxy<K, V> cache = (IgniteCacheProxy<K, V>) jCacheProxies.get(name);
+        IgniteCacheProxy<K, V> cache = (IgniteCacheProxy<K, V>)jcacheProxy(name, true);
+
+        if (cache == null) {
+            GridCacheAdapter<?, ?> cacheAdapter = caches.get(name);
+
+            if (cacheAdapter != null) {
+                cache = new IgniteCacheProxyImpl(cacheAdapter.context(), cacheAdapter, false);
+
+                IgniteCacheProxyImpl<?, ?> prev = addjCacheProxy(name, (IgniteCacheProxyImpl<?, ?>)cache);
+
+                if (prev != null)
+                    cache = (IgniteCacheProxy<K, V>)prev;
+
+                completeProxyInitialize(cache.getName());
+            }
+        }
 
         if (cache == null)
             throw new IllegalArgumentException("Cache is not configured: " + name);
@@ -4058,10 +4201,25 @@ public <K, V> IgniteCacheProxy<K, V> jcache(String name) {
 
     /**
      * @param name Cache name.
+     * @param awaitInit Await proxy initialization.
      * @return Cache proxy.
      */
-    @Nullable public IgniteCacheProxy jcacheProxy(String name) {
-        return jCacheProxies.get(name);
+    @Nullable public IgniteCacheProxyImpl<?, ?> jcacheProxy(String name, boolean awaitInit) {
+        IgniteCacheProxyImpl<?, ?> cache = jCacheProxies.get(name);
+
+        if (awaitInit)
+            awaitInitializeProxy(cache);
+
+        return cache;
+    }
+
+    /**
+     * @param name Cache name.
+     * @param proxy Cache proxy.
+     * @return Previous cache proxy.
+     */
+    @Nullable public IgniteCacheProxyImpl<?, ?> addjCacheProxy(String name, IgniteCacheProxyImpl<?, ?> proxy) {
+        return jCacheProxies.putIfAbsent(name, proxy);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java
index 30edbeadc9ffd..ed3b47ad68545 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java
@@ -88,8 +88,11 @@ public GridCacheProxyImpl() {
      * @param delegate Delegate object.
      * @param opCtx Optional operation context which will be passed to gateway.
      */
-    public GridCacheProxyImpl(GridCacheContext<K, V> ctx, IgniteInternalCache<K, V> delegate,
-        @Nullable CacheOperationContext opCtx) {
+    public GridCacheProxyImpl(
+        GridCacheContext<K, V> ctx,
+        IgniteInternalCache<K, V> delegate,
+        @Nullable CacheOperationContext opCtx
+    ) {
         assert ctx != null;
         assert delegate != null;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index 68e5b850aafed..284dfcc742266 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -30,6 +30,7 @@
 import java.util.NoSuchElementException;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.locks.Lock;
 import javax.cache.Cache;
@@ -135,16 +136,23 @@ public class IgniteCacheProxyImpl<K, V> extends AsyncSupportAdapter<IgniteCache<
     private CacheManager cacheMgr;
 
     /** Future indicates that cache is under restarting. */
-    private final AtomicReference<GridFutureAdapter<Void>> restartFut;
+    private final AtomicReference<RestartFuture> restartFut;
 
     /** Flag indicates that proxy is closed. */
     private volatile boolean closed;
 
+    /** Proxy initialization latch used for await final completion after proxy created, as an example,
+     * a proxy may be created but the exchange is not completed and if we try to perform some cache
+     * the operation we get last finished exchange future (need for validation)
+     * for the previous version but not for current.
+     */
+    private final CountDownLatch initLatch = new CountDownLatch(1);
+
     /**
      * Empty constructor required for {@link Externalizable}.
      */
     public IgniteCacheProxyImpl() {
-        restartFut = new AtomicReference<GridFutureAdapter<Void>>(null);
+        restartFut = new AtomicReference<>(null);
     }
 
     /**
@@ -157,7 +165,7 @@ public IgniteCacheProxyImpl(
             @NotNull IgniteInternalCache<K, V> delegate,
             boolean async
     ) {
-        this(ctx, delegate, new AtomicReference<GridFutureAdapter<Void>>(null), async);
+        this(ctx, delegate, new AtomicReference<>(null), async);
     }
 
     /**
@@ -168,7 +176,7 @@ public IgniteCacheProxyImpl(
     private IgniteCacheProxyImpl(
         @NotNull GridCacheContext<K, V> ctx,
         @NotNull IgniteInternalCache<K, V> delegate,
-        @NotNull AtomicReference<GridFutureAdapter<Void>> restartFut,
+        @NotNull AtomicReference<RestartFuture> restartFut,
         boolean async
     ) {
         super(async);
@@ -182,6 +190,14 @@ private IgniteCacheProxyImpl(
         this.restartFut = restartFut;
     }
 
+    /**
+     *
+     * @return Init latch.
+     */
+    public CountDownLatch getInitLatch(){
+        return initLatch;
+    }
+
     /**
      * @return Context.
      */
@@ -1758,6 +1774,8 @@ private <R> void setFuture(IgniteInternalFuture<R> fut) {
      * @return Internal proxy.
      */
     @Override public GridCacheProxyImpl<K, V> internalProxy() {
+        checkRestart();
+
         return new GridCacheProxyImpl<>(ctx, delegate, ctx.operationContextPerCall());
     }
 
@@ -1824,11 +1842,10 @@ private <R> void setFuture(IgniteInternalFuture<R> fut) {
      * Throws {@code IgniteCacheRestartingException} if proxy is restarting.
      */
     public void checkRestart() {
-        GridFutureAdapter<Void> currentFut = this.restartFut.get();
+        RestartFuture currentFut = restartFut.get();
 
         if (currentFut != null)
-            throw new IgniteCacheRestartingException(new IgniteFutureImpl<>(currentFut), "Cache is restarting: " +
-                    context().name());
+            currentFut.checkRestartOrAwait();
     }
 
     /**
@@ -1842,9 +1859,9 @@ public boolean isRestarting() {
      * Restarts this cache proxy.
      */
     public boolean restart() {
-        GridFutureAdapter<Void> restartFut = new GridFutureAdapter<>();
+        RestartFuture restartFut = new RestartFuture(ctx.name());
 
-        final GridFutureAdapter<Void> curFut = this.restartFut.get();
+         RestartFuture curFut = this.restartFut.get();
 
         boolean changed = this.restartFut.compareAndSet(curFut, restartFut);
 
@@ -1861,13 +1878,23 @@ public boolean restart() {
         return changed;
     }
 
+    /**
+     * @param fut Finish restart future.
+     */
+    public void registrateFutureRestart(GridFutureAdapter<?> fut){
+        RestartFuture currentFut = restartFut.get();
+
+        if (currentFut != null)
+            currentFut.addRestartFinishedFuture(fut);
+    }
+
     /**
      * If proxy is already being restarted, returns future to wait on, else restarts this cache proxy.
      *
      * @return Future to wait on, or null.
      */
     public GridFutureAdapter<Void> opportunisticRestart() {
-        GridFutureAdapter<Void> restartFut = new GridFutureAdapter<>();
+        RestartFuture restartFut = new RestartFuture(ctx.name());
 
         while (true) {
             if (this.restartFut.compareAndSet(null, restartFut))
@@ -1887,7 +1914,7 @@ public GridFutureAdapter<Void> opportunisticRestart() {
      * @param delegate New delegate.
      */
     public void onRestarted(GridCacheContext ctx, IgniteInternalCache delegate) {
-        GridFutureAdapter<Void> restartFut = this.restartFut.get();
+        RestartFuture restartFut = this.restartFut.get();
 
         assert restartFut != null;
 
@@ -1899,6 +1926,52 @@ public void onRestarted(GridCacheContext ctx, IgniteInternalCache delegate) {
         restartFut.onDone();
     }
 
+    /**
+     *
+     */
+    private class RestartFuture extends GridFutureAdapter<Void> {
+        /** */
+        private final String name;
+
+        /** */
+        private volatile GridFutureAdapter<?> restartFinishFut;
+
+        /** */
+        private RestartFuture(String name) {
+            this.name = name;
+        }
+
+        /**
+         *
+         */
+        void checkRestartOrAwait() {
+            GridFutureAdapter<?> fut = restartFinishFut;
+
+            if (fut != null) {
+                try {
+                    fut.get();
+                }
+                catch (IgniteCheckedException e) {
+                    throw U.convertException(e);
+                }
+
+                return;
+            }
+
+            throw new IgniteCacheRestartingException(
+                new IgniteFutureImpl<>(this),
+                "Cache is restarting: " + name
+            );
+        }
+
+        /**
+         *
+         */
+        void addRestartFinishedFuture(GridFutureAdapter<?> fut) {
+            restartFinishFut = fut;
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(IgniteCacheProxyImpl.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index ad3b2a13b6c7e..e3094ed80a812 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -732,7 +732,7 @@ synchronized void onNodeLeft(ClusterTopologyCheckedException e) {
                 AffinityTopologyVersion updTopVer =
                     new AffinityTopologyVersion(Math.max(topVer.topologyVersion() + 1, cctx.discovery().topologyVersion()));
 
-                cctx.affinity().affinityReadyFuture(updTopVer).listen(
+                cctx.shared().exchange().affinityReadyFuture(updTopVer).listen(
                     new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
                         @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> fut) {
                             try {
@@ -798,7 +798,7 @@ void onResult(final GridNearGetResponse res) {
                 }
 
                 // Need to wait for next topology version to remap.
-                IgniteInternalFuture<AffinityTopologyVersion> topFut = cctx.affinity().affinityReadyFuture(rmtTopVer);
+                IgniteInternalFuture<AffinityTopologyVersion> topFut = cctx.shared().exchange().affinityReadyFuture(rmtTopVer);
 
                 topFut.listen(new CIX1<IgniteInternalFuture<AffinityTopologyVersion>>() {
                     @SuppressWarnings("unchecked")
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index 73deb9fc833f2..487e0a2a322b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -601,7 +601,7 @@ private boolean checkError(@Nullable IgniteCheckedException err,
             }
 
             if (canRemap) {
-                IgniteInternalFuture<AffinityTopologyVersion> topFut = cctx.affinity().affinityReadyFuture(rmtTopVer);
+                IgniteInternalFuture<AffinityTopologyVersion> topFut = cctx.shared().exchange().affinityReadyFuture(rmtTopVer);
 
                 topFut.listen(new CIX1<IgniteInternalFuture<AffinityTopologyVersion>>() {
                     @Override public void applyx(IgniteInternalFuture<AffinityTopologyVersion> fut) {
@@ -726,7 +726,7 @@ private ClusterTopologyServerNotFoundException serverNotFoundError(AffinityTopol
             AffinityTopologyVersion updTopVer = new AffinityTopologyVersion(
                 Math.max(topVer.topologyVersion() + 1, cctx.discovery().topologyVersion()));
 
-            cctx.affinity().affinityReadyFuture(updTopVer).listen(
+            cctx.shared().exchange().affinityReadyFuture(updTopVer).listen(
                 new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
                     @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> fut) {
                         try {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
index b2f9218962eb2..6aac21541302a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicSingleUpdateFuture.java
@@ -357,7 +357,7 @@ private void waitAndRemap(AffinityTopologyVersion remapTopVer) {
             ClusterTopologyCheckedException cause = new ClusterTopologyCheckedException(
                 "Failed to update keys, topology changed while execute atomic update inside transaction.");
 
-            cause.retryReadyFuture(cctx.affinity().affinityReadyFuture(remapTopVer));
+            cause.retryReadyFuture(cctx.shared().exchange().affinityReadyFuture(remapTopVer));
 
             e.add(Collections.singleton(cctx.toCacheKeyObject(key)), cause);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
index e516a99793340..170ff76f55131 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateFuture.java
@@ -485,7 +485,7 @@ private void waitAndRemap(AffinityTopologyVersion remapTopVer) {
             ClusterTopologyCheckedException cause = new ClusterTopologyCheckedException(
                 "Failed to update keys, topology changed while execute atomic update inside transaction.");
 
-            cause.retryReadyFuture(cctx.affinity().affinityReadyFuture(remapTopVer));
+            cause.retryReadyFuture(cctx.shared().exchange().affinityReadyFuture(remapTopVer));
 
             e.add(remapKeys, cause);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index d9869759e134c..7d353546b2ccb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -72,6 +72,7 @@
 import org.apache.ignite.internal.processors.cache.CachePartitionExchangeWorkerTask;
 import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch;
 import org.apache.ignite.internal.processors.cache.DynamicCacheChangeFailureMessage;
+import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.ExchangeActions;
 import org.apache.ignite.internal.processors.cache.ExchangeContext;
@@ -325,6 +326,9 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     /** Latest (by update sequences) full message with exchangeId == null, need to be processed right after future is done. */
     private GridDhtPartitionsFullMessage delayedLatestMsg;
 
+    /** Future for wait all exchange listeners comepleted. */
+    private final GridFutureAdapter<?> afterLsnrCompleteFut = new GridFutureAdapter<>();
+
     /** */
     private volatile AffinityTopologyVersion lastAffChangeTopVer;
 
@@ -780,6 +784,8 @@ else if (msg instanceof WalStateAbstractMessage)
                 }
             }
 
+            cctx.cache().registrateProxyRestart(resolveCacheRequests(exchActions), afterLsnrCompleteFut);
+
             updateTopologies(crdNode);
 
             switch (exchange) {
@@ -1793,6 +1799,8 @@ public void finishMerged() {
 
     /** {@inheritDoc} */
     @Override public boolean onDone(@Nullable AffinityTopologyVersion res, @Nullable Throwable err) {
+        assert res != null || err != null : "TopVer=" + res + ", err=" + err;
+
         if (isDone() || !done.compareAndSet(false, true))
             return false;
 
@@ -1868,16 +1876,8 @@ public void finishMerged() {
 
         cctx.cache().onExchangeDone(initialVersion(), exchActions, err);
 
-        cctx.exchange().onExchangeDone(res, initialVersion(), err);
-
         cctx.kernalContext().authentication().onActivate();
 
-        if (exchActions != null && err == null)
-            exchActions.completeRequestFutures(cctx, null);
-
-        if (stateChangeExchange() && err == null)
-            cctx.kernalContext().state().onStateChangeExchangeDone(exchActions.stateChangeRequest());
-
         Map<T2<Integer, Integer>, Long> localReserved = partHistSuppliers.getReservations(cctx.localNodeId());
 
         if (localReserved != null) {
@@ -1905,7 +1905,29 @@ public void finishMerged() {
             cctx.walState().changeLocalStatesOnExchangeDone(res);
         }
 
+        final Throwable err0 = err;
+
+        // Should execute this listener first, before any external listeners.
+        // Listeners use stack as data structure.
+        listen(f -> {
+            // Update last finished future in the first.
+            cctx.exchange().lastFinishedFuture(this);
+
+            // Complete any affReady futures and update last exchange done version.
+            cctx.exchange().onExchangeDone(res, initialVersion(), err0);
+
+            cctx.cache().completeProxyRestart(resolveCacheRequests(exchActions), initialVersion(), res);
+
+            if (exchActions != null && err0 == null)
+                exchActions.completeRequestFutures(cctx, null);
+
+            if (stateChangeExchange() && err0 == null)
+                cctx.kernalContext().state().onStateChangeExchangeDone(exchActions.stateChangeRequest());
+        });
+
         if (super.onDone(res, err)) {
+            afterLsnrCompleteFut.onDone();
+
             if (log.isDebugEnabled())
                 log.debug("Completed partition exchange [localNode=" + cctx.localNodeId() + ", exchange= " + this +
                     ", durationFromInit=" + (U.currentTimeMillis() - initTs) + ']');
@@ -1929,8 +1951,6 @@ public void finishMerged() {
                 ((DiscoveryCustomEvent)firstDiscoEvt).customMessage(null);
 
             if (err == null) {
-                cctx.exchange().lastFinishedFuture(this);
-
                 if (exchCtx != null && (exchCtx.events().hasServerLeft() || exchCtx.events().hasServerJoin())) {
                     ExchangeDiscoveryEvents evts = exchCtx.events();
 
@@ -1948,6 +1968,20 @@ public void finishMerged() {
         return false;
     }
 
+    /**
+     * @param exchangeActions Exchange actions.
+     * @return Map of cache names and start descriptors.
+     */
+    private Map<String, DynamicCacheChangeRequest> resolveCacheRequests(ExchangeActions exchangeActions) {
+        if (exchangeActions == null)
+            return Collections.emptyMap();
+
+        return exchangeActions.cacheStartRequests()
+            .stream()
+            .map(ExchangeActions.CacheActionData::request)
+            .collect(Collectors.toMap(DynamicCacheChangeRequest::cacheName, r -> r));
+    }
+
     /**
      * Method waits for new caches registration and cache configuration persisting to disk.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
index c689e2b044178..42d3d1b06f98e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetFuture.java
@@ -936,21 +936,18 @@ synchronized void onNodeLeft() {
                 AffinityTopologyVersion updTopVer =
                     new AffinityTopologyVersion(Math.max(topVer.topologyVersion() + 1, cctx.discovery().topologyVersion()));
 
-                cctx.affinity().affinityReadyFuture(updTopVer).listen(
-                    new CI1<IgniteInternalFuture<AffinityTopologyVersion>>() {
-                        @Override public void apply(IgniteInternalFuture<AffinityTopologyVersion> fut) {
-                            try {
-                                // Remap.
-                                map(keys.keySet(), F.t(node, keys), fut.get());
-
-                                onDone(Collections.<K, V>emptyMap());
-                            }
-                            catch (IgniteCheckedException e) {
-                                GridNearGetFuture.this.onDone(e);
-                            }
-                        }
+                cctx.shared().exchange().affinityReadyFuture(updTopVer).listen(f -> {
+                    try {
+                        // Remap.
+                        map(keys.keySet(), F.t(node, keys), f.get());
+
+                        onDone(Collections.<K, V>emptyMap());
+
+                    }
+                    catch (IgniteCheckedException e) {
+                        GridNearGetFuture.this.onDone(e);
                     }
-                );
+                });
             }
         }
 
@@ -1002,7 +999,7 @@ void onResult(final GridNearGetResponse res) {
                 }
 
                 // Need to wait for next topology version to remap.
-                IgniteInternalFuture<AffinityTopologyVersion> topFut = cctx.affinity().affinityReadyFuture(rmtTopVer);
+                IgniteInternalFuture<AffinityTopologyVersion> topFut =  cctx.shared().exchange().affinityReadyFuture(rmtTopVer);
 
                 topFut.listen(new CIX1<IgniteInternalFuture<AffinityTopologyVersion>>() {
                     @Override public void applyx(
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
index 51fdd5866ea82..03529cc5dd6a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryAdapter.java
@@ -803,7 +803,7 @@ private void retryIfPossible(IgniteCheckedException e) {
 
                     assert waitVer != null;
 
-                    retryFut = cctx.affinity().affinityReadyFuture(waitVer);
+                    retryFut = cctx.shared().exchange().affinityReadyFuture(waitVer);
                 }
                 else if (e.hasCause(ClusterTopologyCheckedException.class)) {
                     ClusterTopologyCheckedException topEx = X.cause(e, ClusterTopologyCheckedException.class);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 74f6e6868b5ea..4106dc28eaffe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -641,7 +641,7 @@ void waitTopologyFuture(GridKernalContext ctx) throws IgniteCheckedException {
         if (!cctx.isLocal()) {
             AffinityTopologyVersion topVer = initTopVer;
 
-            cacheContext(ctx).affinity().affinityReadyFuture(topVer).get();
+            cacheContext(ctx).shared().exchange().affinityReadyFuture(topVer).get();
 
             for (int partId = 0; partId < cacheContext(ctx).affinity().partitions(); partId++)
                 getOrCreatePartitionRecovery(ctx, partId, topVer);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index 55c44b489d7a7..95afc56d3696d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -364,7 +364,7 @@ public void onEntryUpdated(
                 topVer,
                 (byte)0);
 
-            IgniteCacheProxy jcache = cctx.kernalContext().cache().jcacheProxy(cctx.name());
+            IgniteCacheProxy jcache = cctx.kernalContext().cache().jcacheProxy(cctx.name(), true);
 
             assert jcache != null : "Failed to get cache proxy [name=" + cctx.name() +
                 ", locStart=" + cctx.startTopologyVersion() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
index b6b301e5fae6a..11ffe4801862d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/DiscoveryDataClusterState.java
@@ -194,17 +194,34 @@ public boolean active() {
     }
 
     /**
-     * @return {@code True} if baseline topology is set in the cluster. {@code False} otherwise.
+     * @return Previous Baseline topology.
      */
-    public boolean hasBaselineTopology() {
-        return baselineTopology != null;
+    @Nullable public BaselineTopology previousBaselineTopology() {
+        return prevState != null ? prevState.baselineTopology() : null;
     }
 
     /**
-     * @return Previous Baseline topology.
+     *
+     * @return {@code True} If baseLine changed, {@code False} if not.
      */
-    @Nullable public BaselineTopology previousBaselineTopology() {
-        return prevState != null ? prevState.baselineTopology() : null;
+    public boolean baselineChanged() {
+        BaselineTopology prevBLT = previousBaselineTopology();
+        BaselineTopology curBLT = baselineTopology();
+
+        if (prevBLT == null && curBLT != null)
+            return true;
+
+        if (prevBLT!= null && curBLT != null)
+            return !prevBLT.equals(curBLT);
+
+        return false;
+    }
+
+    /**
+     * @return {@code True} if baseline topology is set in the cluster. {@code False} otherwise.
+     */
+    public boolean hasBaselineTopology() {
+        return baselineTopology != null;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 396cc678599d7..5a62f69dc6100 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -204,7 +204,7 @@ public boolean compatibilityMode() {
                     return transitionRes;
                 }
                 else
-                    return false;
+                    return globalState.baselineChanged();
             }
         }
         else
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java
new file mode 100644
index 0000000000000..ceafc9e61600d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheResultIsNotNullOnPartitionLossTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.PartitionLossPolicy;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.NodeStoppingException;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.processors.cache.CacheInvalidStateException;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class CacheResultIsNotNullOnPartitionLossTest extends GridCommonAbstractTest {
+    /** IP finder. */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** Number of servers to be started. */
+    private static final int SERVERS = 10;
+
+    /** Index of node that is goning to be the only client node. */
+    private static final int CLIENT_IDX = SERVERS;
+
+    /** Number of cache entries to insert into the test cache. */
+    private static final int CACHE_ENTRIES_CNT = 10_000;
+
+    /** True if {@link #getConfiguration(String)} is expected to configure client node on next invocations. */
+    private boolean isClient;
+
+    /** Client Ignite instance. */
+    private IgniteEx client;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        cfg.setIncludeEventTypes(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST);
+
+        cfg.setCacheConfiguration(
+            new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setBackups(0)
+                .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+                .setPartitionLossPolicy(PartitionLossPolicy.READ_WRITE_SAFE)
+        );
+
+        if (isClient)
+            cfg.setClientMode(true);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        startGrids(SERVERS);
+
+        isClient = true;
+
+        client = startGrid(CLIENT_IDX);
+
+        try (IgniteDataStreamer<Integer, Integer> dataStreamer = client.dataStreamer(DEFAULT_CACHE_NAME)) {
+            dataStreamer.allowOverwrite(true);
+
+            for (int i = 0; i < CACHE_ENTRIES_CNT; i++)
+                dataStreamer.addData(i, i);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCacheResultIsNotNullOnClient() throws Exception {
+        testCacheResultIsNotNull0(client);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCacheResultIsNotNullOnLastServer() throws Exception {
+        testCacheResultIsNotNull0(grid(SERVERS - 1));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCacheResultIsNotNullOnServer() throws Exception {
+        testCacheResultIsNotNull0(grid(SERVERS - 2));
+    }
+    /**
+     * @throws Exception If failed.
+     */
+    private void testCacheResultIsNotNull0(IgniteEx ignite) throws Exception {
+        AtomicBoolean stopReading = new AtomicBoolean();
+
+        AtomicReference<Throwable> unexpectedThrowable = new AtomicReference<>();
+
+        IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
+
+        CountDownLatch readerThreadStarted = new CountDownLatch(1);
+
+        IgniteInternalFuture<Boolean> nullCacheValFoundFut = GridTestUtils.runAsync(() -> {
+            readerThreadStarted.countDown();
+
+            while (!stopReading.get())
+                for (int i = 0; i < CACHE_ENTRIES_CNT && !stopReading.get(); i++) {
+                    try {
+                        if (cache.get(i) == null)
+                            return true;
+                    }
+                    catch (Throwable t) {
+                        if (expectedThrowableClass(t)) {
+                            try {
+                                cache.put(i, i);
+
+                                unexpectedThrowable.set(new RuntimeException("Cache put was successful for entry " + i));
+                            }
+                            catch (Throwable t2) {
+                                if (!expectedThrowableClass(t2))
+                                    unexpectedThrowable.set(t2);
+                            }
+                        }
+                        else
+                            unexpectedThrowable.set(t);
+
+                        break;
+                    }
+                }
+
+            return false;
+        });
+
+        try {
+            readerThreadStarted.await(1, TimeUnit.SECONDS);
+
+            for (int i = 0; i < SERVERS - 1; i++) {
+                Thread.sleep(50L);
+
+                grid(i).close();
+            }
+        }
+        finally {
+            // Ask reader thread to finish its execution.
+            stopReading.set(true);
+        }
+
+        assertFalse("Null value was returned by cache.get instead of exception.", nullCacheValFoundFut.get());
+
+        Throwable throwable = unexpectedThrowable.get();
+        if (throwable != null) {
+            throwable.printStackTrace();
+
+            fail(throwable.getMessage());
+        }
+    }
+
+    /**
+     *
+     */
+    private boolean expectedThrowableClass(Throwable throwable) {
+        return X.hasCause(
+            throwable,
+            CacheInvalidStateException.class,
+            ClusterTopologyCheckedException.class,
+            IllegalStateException.class,
+            NodeStoppingException.class
+        );
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
index 321046be20271..5884c4b800c07 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite4.java
@@ -91,6 +91,7 @@
 import org.apache.ignite.internal.processors.cache.IgniteStartCacheInTransactionSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteSystemCacheOnClientTest;
 import org.apache.ignite.internal.processors.cache.MarshallerCacheJobRunNodeRestartTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheResultIsNotNullOnPartitionLossTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheAffinityEarlyTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheDiscoveryDataConcurrentJoinTest;
@@ -354,6 +355,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteCacheContainsKeyAtomicTest.class);
 
+        suite.addTestSuite(CacheResultIsNotNullOnPartitionLossTest.class);
+
         return suite;
     }
 }
diff --git a/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/KafkaIgniteStreamerSelfTest.java b/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/KafkaIgniteStreamerSelfTest.java
index 00cb4fc77a72d..b39c0cc425ef8 100644
--- a/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/KafkaIgniteStreamerSelfTest.java
+++ b/modules/kafka/src/test/java/org/apache/ignite/stream/kafka/KafkaIgniteStreamerSelfTest.java
@@ -32,9 +32,13 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.events.CacheEvent;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.stream.StreamMultipleTupleExtractor;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.kafka.clients.producer.ProducerRecord;
@@ -203,9 +207,24 @@ private void consumerStream(String topic, Map<String, String> keyValMap)
             final CountDownLatch latch = new CountDownLatch(CNT);
 
             IgniteBiPredicate<UUID, CacheEvent> locLsnr = new IgniteBiPredicate<UUID, CacheEvent>() {
+                @IgniteInstanceResource
+                private Ignite ig;
+
+                @LoggerResource
+                private IgniteLogger log;
+
+                /** {@inheritDoc} */
                 @Override public boolean apply(UUID uuid, CacheEvent evt) {
                     latch.countDown();
 
+                    if (log.isInfoEnabled()) {
+                        IgniteEx igEx = (IgniteEx)ig;
+
+                        UUID nodeId = igEx.localNode().id();
+
+                        log.info("Recive event=" + evt + ", nodeId=" + nodeId);
+                    }
+
                     return true;
                 }
             };
@@ -213,7 +232,8 @@ private void consumerStream(String topic, Map<String, String> keyValMap)
             ignite.events(ignite.cluster().forCacheNodes(DEFAULT_CACHE_NAME)).remoteListen(locLsnr, null, EVT_CACHE_OBJECT_PUT);
 
             // Checks all events successfully processed in 10 seconds.
-            assertTrue(latch.await(10, TimeUnit.SECONDS));
+            assertTrue("Failed to wait latch completion, still wait " + latch.getCount() + " events",
+                latch.await(10, TimeUnit.SECONDS));
 
             for (Map.Entry<String, String> entry : keyValMap.entrySet())
                 assertEquals(entry.getValue(), cache.get(entry.getKey()));

From c51926b27cf0cb0ad740511539a306b7c08ecc4a Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Wed, 17 Oct 2018 15:40:40 +0300
Subject: [PATCH 428/543] IGNITE-9898 fix checkpoint thread hangs on await
 async task completion - Fixes #5002.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit 10c2b10e605d372e832b65da52d67dc9656b53c1)
Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
---
 .../GridCacheDatabaseSharedManager.java       |  47 +++--
 .../ignite/internal/util/IgniteUtils.java     |  22 +++
 .../IgniteTaskTrackingThreadPoolExecutor.java | 180 ------------------
 .../testsuites/IgniteUtilSelfTestSuite.java   |   7 +-
 ...iteTaskTrackingThreadPoolExecutorTest.java | 140 --------------
 5 files changed, 49 insertions(+), 347 deletions(-)
 delete mode 100644 modules/core/src/main/java/org/apache/ignite/thread/IgniteTaskTrackingThreadPoolExecutor.java
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/util/IgniteTaskTrackingThreadPoolExecutorTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 7b37eab11cde6..5277382e9f412 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -138,6 +138,7 @@
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.future.CountDownFuture;
+import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridInClosure3X;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -157,7 +158,7 @@
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.mxbean.DataStorageMetricsMXBean;
 import org.apache.ignite.thread.IgniteThread;
-import org.apache.ignite.thread.IgniteTaskTrackingThreadPoolExecutor;
+import org.apache.ignite.thread.IgniteThreadPoolExecutor;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jsr166.ConcurrentLinkedHashMap;
@@ -288,7 +289,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     private boolean stopping;
 
     /** Checkpoint runner thread pool. If null tasks are to be run in single thread */
-    @Nullable private IgniteTaskTrackingThreadPoolExecutor asyncRunner;
+    @Nullable private IgniteThreadPoolExecutor asyncRunner;
 
     /** Thread local with buffers for the checkpoint threads. Each buffer represent one page for durable memory. */
     private ThreadLocal<ByteBuffer> threadBuf;
@@ -728,15 +729,13 @@ else if (regCfg.getMaxSize() < 8 * GB)
      */
     private void initDataBase() {
         if (persistenceCfg.getCheckpointThreads() > 1)
-            asyncRunner = new IgniteTaskTrackingThreadPoolExecutor(
+            asyncRunner = new IgniteThreadPoolExecutor(
                 CHECKPOINT_RUNNER_THREAD_PREFIX,
                 cctx.igniteInstanceName(),
                 persistenceCfg.getCheckpointThreads(),
                 persistenceCfg.getCheckpointThreads(),
-                30_000, // A value is ignored if corePoolSize equals to maxPoolSize
-                new LinkedBlockingQueue<Runnable>(),
-                GridIoPolicy.UNDEFINED,
-                cctx.kernalContext().uncaughtExceptionHandler()
+                30_000,
+                new LinkedBlockingQueue<Runnable>()
             );
     }
 
@@ -3495,8 +3494,7 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
 
                 final PartitionAllocationMap map = new PartitionAllocationMap();
 
-                if (asyncRunner != null)
-                    asyncRunner.reset();
+                GridCompoundFuture asyncLsnrFut = asyncRunner == null ? null : new GridCompoundFuture();
 
                 DbCheckpointListener.Context ctx0 = new DbCheckpointListener.Context() {
                     @Override public boolean nextSnapshot() {
@@ -3517,10 +3515,14 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
                     @Override public Executor executor() {
                         return asyncRunner == null ? null : cmd -> {
                             try {
-                                asyncRunner.execute(cmd);
+                                GridFutureAdapter<?> res = new GridFutureAdapter<>();
+
+                                asyncRunner.execute(U.wrapIgniteFuture(cmd, res));
+
+                                asyncLsnrFut.add(res);
                             }
                             catch (RejectedExecutionException e) {
-                                assert false: "A task should never be rejected by async runner";
+                                assert false : "A task should never be rejected by async runner";
                             }
                         };
                     }
@@ -3530,17 +3532,16 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
                 for (DbCheckpointListener lsnr : lsnrs)
                     lsnr.onCheckpointBegin(ctx0);
 
-                if (asyncRunner != null) {
-                    asyncRunner.markInitialized();
+                if (asyncLsnrFut != null) {
+                    asyncLsnrFut.markInitialized();
 
-                    asyncRunner.awaitDone();
+                    asyncLsnrFut.get();
                 }
 
                 if (curr.nextSnapshot)
                     snapFut = snapshotMgr.onMarkCheckPointBegin(curr.snapshotOperation, map);
 
-                if (asyncRunner != null)
-                    asyncRunner.reset();
+                GridCompoundFuture grpHandleFut = asyncRunner == null ? null : new GridCompoundFuture();
 
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                     if (grp.isLocal() || !grp.walEnabled())
@@ -3572,17 +3573,21 @@ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws
                         r.run();
                     else
                         try {
-                            asyncRunner.execute(r);
+                            GridFutureAdapter<?> res = new GridFutureAdapter<>();
+
+                            asyncRunner.execute(U.wrapIgniteFuture(r, res));
+
+                            grpHandleFut.add(res);
                         }
                         catch (RejectedExecutionException e) {
-                            assert false: "Task should never be rejected by async runner";
+                            assert false : "Task should never be rejected by async runner";
                         }
                 }
 
-                if (asyncRunner != null) {
-                    asyncRunner.markInitialized();
+                if (grpHandleFut != null) {
+                    grpHandleFut.markInitialized();
 
-                    asyncRunner.awaitDone();
+                    grpHandleFut.get();
                 }
 
                 cpPagesTuple = beginAllCheckpoints();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 26b33b69d9551..2087b10be790c 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -206,6 +206,7 @@
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.future.IgniteFutureImpl;
 import org.apache.ignite.internal.util.io.GridFilenameUtils;
 import org.apache.ignite.internal.util.ipc.shmem.IpcSharedMemoryNativeLoader;
@@ -10637,6 +10638,27 @@ private static void getUninterruptibly(Future fut) throws ExecutionException {
             Thread.currentThread().interrupt();
     }
 
+    /**
+     *
+     * @param r Runnable.
+     * @param fut Grid future apater.
+     * @return Runnable with wrapped future.
+     */
+    public static Runnable wrapIgniteFuture(Runnable r, GridFutureAdapter<?> fut) {
+        return () -> {
+            try {
+                r.run();
+
+                fut.onDone();
+            }
+            catch (Throwable e) {
+                fut.onDone(e);
+
+                throw e;
+            }
+        };
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteTaskTrackingThreadPoolExecutor.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteTaskTrackingThreadPoolExecutor.java
deleted file mode 100644
index 6cae57eb78e6a..0000000000000
--- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteTaskTrackingThreadPoolExecutor.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.thread;
-
-import java.lang.Thread.UncaughtExceptionHandler;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.atomic.LongAdder;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.managers.communication.GridIoPolicy;
-
-/**
- * An {@link ExecutorService} that executes submitted tasks using pooled grid threads.
- *
- * In addition to what it allows to track all enqueued tasks completion or failure during execution.
- */
-public class IgniteTaskTrackingThreadPoolExecutor extends IgniteThreadPoolExecutor {
-    /** */
-    private final LongAdder pendingTaskCnt = new LongAdder();
-
-    /** */
-    private final LongAdder completedTaskCnt = new LongAdder();
-
-    /** */
-    private volatile boolean initialized;
-
-    /** */
-    private volatile AtomicReference<Throwable> err = new AtomicReference<>();
-
-    /**
-     * Creates a new service with the given initial parameters.
-     *
-     * @param threadNamePrefix Will be added at the beginning of all created threads.
-     * @param igniteInstanceName Must be the name of the grid.
-     * @param corePoolSize The number of threads to keep in the pool, even if they are idle.
-     * @param maxPoolSize The maximum number of threads to allow in the pool.
-     * @param keepAliveTime When the number of threads is greater than the core, this is the maximum time
-     *      that excess idle threads will wait for new tasks before terminating.
-     * @param workQ The queue to use for holding tasks before they are executed. This queue will hold only
-     *      runnable tasks submitted by the {@link #execute(Runnable)} method.
-     */
-    public IgniteTaskTrackingThreadPoolExecutor(String threadNamePrefix, String igniteInstanceName, int corePoolSize,
-        int maxPoolSize, long keepAliveTime, BlockingQueue<Runnable> workQ) {
-        super(threadNamePrefix, igniteInstanceName, corePoolSize, maxPoolSize, keepAliveTime, workQ);
-    }
-
-    /**
-     * Creates a new service with the given initial parameters.
-     *
-     * @param threadNamePrefix Will be added at the beginning of all created threads.
-     * @param igniteInstanceName Must be the name of the grid.
-     * @param corePoolSize The number of threads to keep in the pool, even if they are idle.
-     * @param maxPoolSize The maximum number of threads to allow in the pool.
-     * @param keepAliveTime When the number of threads is greater than the core, this is the maximum time
-     *      that excess idle threads will wait for new tasks before terminating.
-     * @param workQ The queue to use for holding tasks before they are executed. This queue will hold only
-     *      runnable tasks submitted by the {@link #execute(Runnable)} method.
-     * @param plc {@link GridIoPolicy} for thread pool.
-     * @param eHnd Uncaught exception handler for thread pool.
-     */
-    public IgniteTaskTrackingThreadPoolExecutor(String threadNamePrefix, String igniteInstanceName, int corePoolSize,
-        int maxPoolSize, long keepAliveTime, BlockingQueue<Runnable> workQ, byte plc,
-        UncaughtExceptionHandler eHnd) {
-        super(threadNamePrefix, igniteInstanceName, corePoolSize, maxPoolSize, keepAliveTime, workQ, plc, eHnd);
-    }
-
-    /**
-     * Creates a new service with the given initial parameters.
-     *
-     * @param corePoolSize The number of threads to keep in the pool, even if they are idle.
-     * @param maxPoolSize The maximum number of threads to allow in the pool.
-     * @param keepAliveTime When the number of threads is greater than the core, this is the maximum time
-     *      that excess idle threads will wait for new tasks before terminating.
-     * @param workQ The queue to use for holding tasks before they are executed. This queue will hold only the
-     *      runnable tasks submitted by the {@link #execute(Runnable)} method.
-     * @param threadFactory Thread factory.
-     */
-    public IgniteTaskTrackingThreadPoolExecutor(int corePoolSize, int maxPoolSize, long keepAliveTime,
-        BlockingQueue<Runnable> workQ, ThreadFactory threadFactory) {
-        super(corePoolSize, maxPoolSize, keepAliveTime, workQ, threadFactory);
-    }
-
-    /** {@inheritDoc} */
-    @Override public void execute(Runnable cmd) {
-        pendingTaskCnt.add(1);
-
-        super.execute(cmd);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterExecute(Runnable r, Throwable t) {
-        super.afterExecute(r, t);
-
-        completedTaskCnt.add(1);
-
-        if (t != null && err.compareAndSet(null, t) || isDone()) {
-            synchronized (this) {
-                notifyAll();
-            }
-        }
-    }
-
-    /**
-     * Mark this executor as initialized.
-     * This method should be called when all required tasks are enqueued for execution.
-     */
-    public final void markInitialized() {
-        initialized = true;
-    }
-
-    /**
-     * Check error status.
-     *
-     * @return {@code True} if any task execution resulted in error.
-     */
-    @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    public final boolean isError() {
-        return err.get() != null;
-    }
-
-    /**
-     * Check done status.
-     *
-     * @return {@code True} when all enqueued task are completed.
-     */
-    public final boolean isDone() {
-        return initialized && completedTaskCnt.sum() == pendingTaskCnt.sum();
-    }
-
-    /**
-     * Wait synchronously until all tasks are completed or error has occurred.
-     *
-     * @throws IgniteCheckedException if task execution resulted in error.
-     */
-    public final synchronized void awaitDone() throws IgniteCheckedException {
-        // There are no guarantee what all enqueued tasks will be finished if an error has occurred.
-        while(!isError() && !isDone()) {
-            try {
-                wait();
-            }
-            catch (InterruptedException e) {
-                err.set(e);
-
-                Thread.currentThread().interrupt();
-            }
-        }
-
-        if (isError())
-            throw new IgniteCheckedException("Task execution resulted in error", err.get());
-    }
-
-    /**
-     * Reset tasks tracking context.
-     * The method should be called before adding new tasks to the executor.
-     */
-    public final void reset() {
-        initialized = false;
-        completedTaskCnt.reset();
-        pendingTaskCnt.reset();
-        err.set(null);
-    }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
index b345053eb3984..5a472a5a2d4f2 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteUtilSelfTestSuite.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.testsuites;
 
+import java.util.Set;
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.commandline.CommandHandlerParsingTest;
 import org.apache.ignite.internal.pagemem.impl.PageIdUtilsSelfTest;
@@ -49,14 +50,11 @@
 import org.apache.ignite.util.GridStringBuilderFactorySelfTest;
 import org.apache.ignite.util.GridTopologyHeapSizeSelfTest;
 import org.apache.ignite.util.GridTransientTest;
-import org.apache.ignite.util.IgniteTaskTrackingThreadPoolExecutorTest;
 import org.apache.ignite.util.mbeans.GridMBeanDisableSelfTest;
 import org.apache.ignite.util.mbeans.GridMBeanExoticNamesSelfTest;
 import org.apache.ignite.util.mbeans.GridMBeanSelfTest;
 import org.apache.ignite.util.mbeans.WorkersControlMXBeanTest;
 
-import java.util.Set;
-
 /**
  * Test suite for Ignite utility classes.
  */
@@ -121,9 +119,6 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         // control.sh
         suite.addTestSuite(CommandHandlerParsingTest.class);
 
-        // Thread pool.
-        suite.addTestSuite(IgniteTaskTrackingThreadPoolExecutorTest.class);
-
         return suite;
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/IgniteTaskTrackingThreadPoolExecutorTest.java b/modules/core/src/test/java/org/apache/ignite/util/IgniteTaskTrackingThreadPoolExecutorTest.java
deleted file mode 100644
index 3db02b0409a98..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/util/IgniteTaskTrackingThreadPoolExecutorTest.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.util;
-
-import java.util.List;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.atomic.LongAdder;
-import junit.framework.TestCase;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.managers.communication.GridIoPolicy;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.thread.IgniteTaskTrackingThreadPoolExecutor;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * Tests for tracking thread pool executor.
- */
-public class IgniteTaskTrackingThreadPoolExecutorTest extends TestCase {
-    /** */
-    private IgniteTaskTrackingThreadPoolExecutor executor;
-
-    /** {@inheritDoc} */
-    @Override protected void setUp() throws Exception {
-        int procs = Runtime.getRuntime().availableProcessors();
-
-        executor = new IgniteTaskTrackingThreadPoolExecutor("test", "default",
-            procs * 2, procs * 2, 30_000, new LinkedBlockingQueue<>(), GridIoPolicy.UNDEFINED, (t, e) -> {
-                // No-op.
-            });
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void tearDown() throws Exception {
-        List<Runnable> runnables = executor.shutdownNow();
-
-        assertEquals("Some tasks are not completed", 0, runnables.size());
-    }
-
-    /** */
-    public void testSimple() throws IgniteCheckedException {
-        doTest(null);
-    }
-
-    /** */
-    @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    public void testWithException() throws IgniteCheckedException {
-        int fail = 5555;
-
-        try {
-            doTest(fail);
-
-            fail();
-        }
-        catch (Throwable t) {
-            TestException cause = (TestException)X.getCause(t);
-
-            assertEquals(fail, cause.idx);
-        }
-
-        AtomicReference<Throwable> err = U.field(executor, "err");
-        err.set(null);
-
-        executor.awaitDone();
-    }
-
-    /** */
-    public void testReuse() throws IgniteCheckedException {
-        long avg = 0;
-
-        long warmUp = 30;
-
-        int iters = 150;
-
-        for (int i = 0; i < iters; i++) {
-            long t1 = System.nanoTime();
-
-            doTest(null);
-
-            if (i >= warmUp)
-                avg += System.nanoTime() - t1;
-
-            executor.reset();
-        }
-
-        X.print("Average time per iteration: " + (avg / (iters - warmUp)) / 1000 / 1000. + " ms");
-    }
-
-    /** */
-    private void doTest(@Nullable Integer fail) throws IgniteCheckedException {
-        LongAdder cnt = new LongAdder();
-
-        int exp = 100_000;
-
-        for (int i = 0; i < exp; i++) {
-            final int finalI = i;
-            executor.execute(() -> {
-                if (fail != null && fail == finalI)
-                    throw new TestException(finalI);
-                else
-                    cnt.add(1);
-            });
-        }
-
-        executor.markInitialized();
-
-        executor.awaitDone();
-
-        assertEquals("Counter is not as expected", exp, cnt.sum());
-    }
-
-    /** */
-    private static class TestException extends RuntimeException {
-        /** */
-        final int idx;
-
-        /**
-         * @param idx Index.
-         */
-        public TestException(int idx) {
-            this.idx = idx;
-        }
-    }
-}

From d30b9c24944f60cd7d6145a56dca53bc63eedc79 Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Wed, 17 Oct 2018 18:20:41 +0300
Subject: [PATCH 429/543] IGNITE-9675 Fixed deadlock on Ignite#active() and
 concurrent node stop - Fixes #4822.  Signed-off-by: Alexey Goncharuk
 <alexey.goncharuk@gmail.com>  (cherry picked from commit
 754c7337de123ac44e2816d2a55ab6f76cd03eac)

---
 .../cluster/GridClusterStateProcessor.java    | 52 ++++++++++++-------
 .../cluster/IGridClusterStateProcessor.java   |  6 +++
 .../processors/task/GridTaskProcessor.java    | 25 ++++++++-
 .../processors/igfs/IgfsIgniteMock.java       | 19 ++++---
 4 files changed, 71 insertions(+), 31 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 5a62f69dc6100..13694384712cb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -56,10 +56,14 @@
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage;
+import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
+import org.apache.ignite.internal.util.future.IgniteFutureImpl;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.F;
@@ -172,6 +176,11 @@ public boolean compatibilityMode() {
 
     /** {@inheritDoc} */
     @Override public boolean publicApiActiveState(boolean waitForTransition) {
+        return publicApiActiveStateAsync(waitForTransition).get();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteFuture<Boolean> publicApiActiveStateAsync(boolean asyncWaitForTransition) {
         if (ctx.isDaemon())
             return sendComputeCheckGlobalState();
 
@@ -183,32 +192,34 @@ public boolean compatibilityMode() {
             Boolean transitionRes = globalState.transitionResult();
 
             if (transitionRes != null)
-                return transitionRes;
+                return new IgniteFinishedFutureImpl<>(transitionRes);
             else {
-                if (waitForTransition) {
-                    GridFutureAdapter<Void> fut = transitionFuts.get(globalState.transitionRequestId());
+                GridFutureAdapter<Void> fut = transitionFuts.get(globalState.transitionRequestId());
+                if (fut != null) {
+                    if (asyncWaitForTransition) {
+                        return new IgniteFutureImpl<>(fut.chain(new C1<IgniteInternalFuture<Void>, Boolean>() {
+                            @Override public Boolean apply(IgniteInternalFuture<Void> fut) {
+                                Boolean res = globalState.transitionResult();
 
-                    if (fut != null) {
-                        try {
-                            fut.get();
-                        }
-                        catch (IgniteCheckedException ex) {
-                            throw new IgniteException(ex);
-                        }
+                                assert res != null;
+
+                                return res;
+                            }
+                        }));
                     }
+                    else
+                        return new IgniteFinishedFutureImpl<>(globalState.baselineChanged());
+                }
 
-                    transitionRes = globalState.transitionResult();
+                transitionRes = globalState.transitionResult();
 
-                    assert transitionRes != null;
+                assert transitionRes != null;
 
-                    return transitionRes;
-                }
-                else
-                    return globalState.baselineChanged();
+                return new IgniteFinishedFutureImpl<>(transitionRes);
             }
         }
         else
-            return globalState.active();
+            return new IgniteFinishedFutureImpl<>(globalState.active());
     }
 
     /** {@inheritDoc} */
@@ -1065,7 +1076,7 @@ private void sendComputeChangeGlobalState(
      *
      *  @return Cluster state, {@code True} if cluster active, {@code False} if inactive.
      */
-    private boolean sendComputeCheckGlobalState() {
+    private IgniteFuture<Boolean> sendComputeCheckGlobalState() {
         AffinityTopologyVersion topVer = ctx.discovery().topologyVersionEx();
 
         if (log.isInfoEnabled()) {
@@ -1078,11 +1089,11 @@ private boolean sendComputeCheckGlobalState() {
         ClusterGroupAdapter clusterGroupAdapter = (ClusterGroupAdapter)ctx.cluster().get().forServers();
 
         if (F.isEmpty(clusterGroupAdapter.nodes()))
-            return false;
+            return new IgniteFinishedFutureImpl<>(false);
 
         IgniteCompute comp = clusterGroupAdapter.compute();
 
-        return comp.call(new IgniteCallable<Boolean>() {
+        return comp.callAsync(new IgniteCallable<Boolean>() {
             @IgniteInstanceResource
             private Ignite ig;
 
@@ -1463,6 +1474,7 @@ private void onAllReceived() {
     /**
      *
      */
+    @GridInternal
     private static class ClientChangeGlobalStateComputeRequest implements IgniteRunnable {
         /** */
         private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java
index bc72a516b860a..d71b4cf88c876 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/IGridClusterStateProcessor.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.processors.GridProcessor;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.StateChangeRequest;
+import org.apache.ignite.lang.IgniteFuture;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -40,6 +41,11 @@ public interface IGridClusterStateProcessor extends GridProcessor {
      */
     boolean publicApiActiveState(boolean waitForTransition);
 
+    /**
+     * @return Cluster state to be used on public API.
+     */
+    IgniteFuture<Boolean> publicApiActiveStateAsync(boolean waitForTransition);
+
     /**
      * @param discoCache Discovery data cache.
      * @return If transition is in progress returns future which is completed when transition finishes.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
index 9007472b034f5..313f6c3cb1519 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/task/GridTaskProcessor.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.LongAdder;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
@@ -54,6 +55,7 @@
 import org.apache.ignite.internal.GridTaskSessionRequest;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteDeploymentCheckedException;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.compute.ComputeTaskCancelledCheckedException;
 import org.apache.ignite.internal.managers.communication.GridIoManager;
@@ -69,6 +71,7 @@
 import org.apache.ignite.internal.util.typedef.C1;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.LT;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
@@ -187,7 +190,24 @@ private IgniteClientDisconnectedCheckedException disconnectedError(@Nullable Ign
     /** {@inheritDoc} */
     @SuppressWarnings("TooBroadScope")
     @Override public void onKernalStop(boolean cancel) {
-        lock.writeLock();
+        boolean interrupted = false;
+
+        while (true) {
+            try {
+                if (lock.tryWriteLock(1, TimeUnit.SECONDS))
+                    break;
+                else {
+                    LT.warn(log, "Still waiting to acquire write lock on stop");
+
+                    U.sleep(50);
+                }
+            }
+            catch (IgniteInterruptedCheckedException | InterruptedException e) {
+                LT.warn(log, "Stopping thread was interrupted while waiting for write lock (will wait anyway)");
+
+                interrupted = true;
+            }
+        }
 
         try {
             stopping = true;
@@ -196,6 +216,9 @@ private IgniteClientDisconnectedCheckedException disconnectedError(@Nullable Ign
         }
         finally {
             lock.writeUnlock();
+
+            if (interrupted)
+                Thread.currentThread().interrupt();
         }
 
         startLatch.countDown();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java
index a0ce28559e847..45bc5cac65a5f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsIgniteMock.java
@@ -17,8 +17,13 @@
 
 package org.apache.ignite.internal.processors.igfs;
 
+import java.util.Collection;
+import java.util.Collections;
+import java.util.concurrent.ExecutorService;
+import javax.cache.CacheException;
 import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.DataRegionMetricsAdapter;
+import org.apache.ignite.DataStorageMetrics;
 import org.apache.ignite.DataStorageMetricsAdapter;
 import org.apache.ignite.IgniteAtomicLong;
 import org.apache.ignite.IgniteAtomicReference;
@@ -41,7 +46,6 @@
 import org.apache.ignite.IgniteServices;
 import org.apache.ignite.IgniteSet;
 import org.apache.ignite.IgniteTransactions;
-import org.apache.ignite.DataStorageMetrics;
 import org.apache.ignite.MemoryMetrics;
 import org.apache.ignite.PersistenceMetrics;
 import org.apache.ignite.cache.affinity.Affinity;
@@ -66,11 +70,6 @@
 import org.apache.ignite.plugin.PluginNotFoundException;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.concurrent.ExecutorService;
-import javax.cache.CacheException;
-
 /**
  * Mocked Ignite implementation for IGFS tests.
  */
@@ -182,13 +181,13 @@ public IgfsIgniteMock(@Nullable String name, IgniteFileSystem igfs) {
         return null;
     }
 
-    @Override
-    public boolean isRebalanceEnabled() {
+    /** {@inheritDoc} */
+    @Override public boolean isRebalanceEnabled() {
         return true;
     }
 
-    @Override
-    public void rebalanceEnabled(boolean rebalanceEnabled) {
+    /** {@inheritDoc} */
+    @Override public void rebalanceEnabled(boolean rebalanceEnabled) {
         throwUnsupported();
     }
 

From bdbf5e7c2df9637f551f071742ce61893e397b07 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Wed, 17 Oct 2018 20:08:34 +0300
Subject: [PATCH 430/543] IGNITE-9868 Improved background full message sending
 - Fixes #4975.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit ce73c9d85ffd5b50e4c3370b13302605392fa572)
---
 .../GridCachePartitionExchangeManager.java    | 181 +++++++++++++-----
 .../GridDhtPartitionsExchangeFuture.java      |  12 +-
 2 files changed, 139 insertions(+), 54 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index afd94a751df64..dd83036182ff6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -1030,11 +1030,13 @@ public void scheduleResendPartitions() {
     }
 
     /**
-     * Partition refresh callback.
+     * Partition refresh callback for selected cache groups.
      * For coordinator causes {@link GridDhtPartitionsFullMessage FullMessages} send,
      * for non coordinator -  {@link GridDhtPartitionsSingleMessage SingleMessages} send
+     *
+     * @param grps Cache groups for partitions refresh.
      */
-    public void refreshPartitions() {
+    public void refreshPartitions(@NotNull Collection<CacheGroupContext> grps) {
         // TODO https://issues.apache.org/jira/browse/IGNITE-6857
         if (cctx.snapshot().snapshotOperationInProgress()) {
             scheduleResendPartitions();
@@ -1042,6 +1044,13 @@ public void refreshPartitions() {
             return;
         }
 
+        if (grps.isEmpty()) {
+            if (log.isDebugEnabled())
+                log.debug("Skip partitions refresh, there are no cache groups for partition refresh.");
+
+            return;
+        }
+
         ClusterNode oldest = cctx.discovery().oldestAliveServerNode(AffinityTopologyVersion.NONE);
 
         if (oldest == null) {
@@ -1051,8 +1060,10 @@ public void refreshPartitions() {
             return;
         }
 
-        if (log.isDebugEnabled())
-            log.debug("Refreshing partitions [oldest=" + oldest.id() + ", loc=" + cctx.localNodeId() + ']');
+        if (log.isDebugEnabled()) {
+            log.debug("Refreshing partitions [oldest=" + oldest.id() + ", loc=" + cctx.localNodeId() +
+                ", cacheGroups= " + grps + ']');
+        }
 
         // If this is the oldest node.
         if (oldest.id().equals(cctx.localNodeId())) {
@@ -1070,46 +1081,66 @@ public void refreshPartitions() {
 
             // No need to send to nodes which did not finish their first exchange.
             AffinityTopologyVersion rmtTopVer =
-                lastFut != null ? (lastFut.isDone() ? lastFut.topologyVersion() : lastFut.initialVersion()) : AffinityTopologyVersion.NONE;
+                lastFut != null ?
+                    (lastFut.isDone() ? lastFut.topologyVersion() : lastFut.initialVersion())
+                    : AffinityTopologyVersion.NONE;
 
             Collection<ClusterNode> rmts = cctx.discovery().remoteAliveNodesWithCaches(rmtTopVer);
 
             if (log.isDebugEnabled())
                 log.debug("Refreshing partitions from oldest node: " + cctx.localNodeId());
 
-            sendAllPartitions(rmts, rmtTopVer);
+            sendAllPartitions(rmts, rmtTopVer, grps);
         }
         else {
             if (log.isDebugEnabled())
                 log.debug("Refreshing local partitions from non-oldest node: " +
                     cctx.localNodeId());
 
-            sendLocalPartitions(oldest, null);
+            sendLocalPartitions(oldest, null, grps);
         }
     }
 
+    /**
+     * Partition refresh callback.
+     * For coordinator causes {@link GridDhtPartitionsFullMessage FullMessages} send,
+     * for non coordinator -  {@link GridDhtPartitionsSingleMessage SingleMessages} send
+     */
+    public void refreshPartitions() { refreshPartitions(cctx.cache().cacheGroups()); }
+
     /**
      * @param nodes Nodes.
      * @param msgTopVer Topology version. Will be added to full message.
+     * @param grps Selected cache groups.
      */
     private void sendAllPartitions(
         Collection<ClusterNode> nodes,
-        AffinityTopologyVersion msgTopVer
+        AffinityTopologyVersion msgTopVer,
+        Collection<CacheGroupContext> grps
     ) {
         long time = System.currentTimeMillis();
 
-        GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, false, null, null, null, null);
+        GridDhtPartitionsFullMessage m = createPartitionsFullMessage(true, false, null, null, null, null, grps);
 
         m.topologyVersion(msgTopVer);
 
-        if (log.isInfoEnabled())
-            log.info("Full Message creating for " + msgTopVer + " performed in " + (System.currentTimeMillis() - time) + " ms.");
+        if (log.isInfoEnabled()) {
+            long latency = System.currentTimeMillis() - time;
+
+            if (latency > 50 || log.isDebugEnabled()) {
+                log.info("Finished full message creation [msgTopVer=" + msgTopVer + ", groups=" + grps +
+                    ", latency=" + latency + "ms]");
+            }
+        }
 
         if (log.isTraceEnabled())
-            log.trace("Sending all partitions [nodeIds=" + U.nodeIds(nodes) + ", msg=" + m + ']');
+            log.trace("Sending all partitions [nodeIds=" + U.nodeIds(nodes) + ", cacheGroups=" + grps +
+                ", msg=" + m + ']');
 
         time = System.currentTimeMillis();
 
+        Collection<ClusterNode> failedNodes = U.newHashSet(nodes.size());
+
         for (ClusterNode node : nodes) {
             try {
                 assert !node.equals(cctx.localNode());
@@ -1117,22 +1148,34 @@ private void sendAllPartitions(
                 cctx.io().sendNoRetry(node, m, SYSTEM_POOL);
             }
             catch (ClusterTopologyCheckedException ignore) {
-                if (log.isDebugEnabled())
-                    log.debug("Failed to send partition update to node because it left grid (will ignore) [node=" +
-                        node.id() + ", msg=" + m + ']');
+                if (log.isDebugEnabled()) {
+                    log.debug("Failed to send partition update to node because it left grid (will ignore) " +
+                        "[node=" + node.id() + ", msg=" + m + ']');
+                }
             }
             catch (IgniteCheckedException e) {
-                U.warn(log, "Failed to send partitions full message [node=" + node + ", err=" + e + ']');
+                failedNodes.add(node);
+
+                U.warn(log, "Failed to send partitions full message [node=" + node + ", err=" + e + ']', e);
             }
         }
 
-        if (log.isInfoEnabled())
-            log.info("Sending Full Message for " + msgTopVer + " performed in " + (System.currentTimeMillis() - time) + " ms.");
+        if (log.isInfoEnabled()) {
+            long latency = System.currentTimeMillis() - time;
+
+            if (latency > 50 || log.isDebugEnabled()) {
+                log.info("Finished sending full message [msgTopVer=" + msgTopVer + ", groups=" + grps +
+                    (failedNodes.isEmpty() ? "" : (", skipped=" + U.nodeIds(failedNodes))) +
+                    ", latency=" + latency + "ms]");
+            }
+        }
     }
 
     /**
+     * Creates partitions full message for all cache groups.
+     *
      * @param compress {@code True} if possible to compress message (properly work only if prepareMarshall/
-     *     finishUnmarshall methods are called).
+     * finishUnmarshall methods are called).
      * @param newCntrMap {@code True} if possible to use {@link CachePartitionFullCountersMap}.
      * @param exchId Non-null exchange ID if message is created for exchange.
      * @param lastVer Last version.
@@ -1148,18 +1191,43 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage(
         @Nullable IgniteDhtPartitionHistorySuppliersMap partHistSuppliers,
         @Nullable IgniteDhtPartitionsToReloadMap partsToReload
     ) {
-        final GridDhtPartitionsFullMessage m = new GridDhtPartitionsFullMessage(exchId,
-            lastVer,
-            exchId != null ? exchId.topologyVersion() : AffinityTopologyVersion.NONE,
-            partHistSuppliers,
-            partsToReload
-            );
+        Collection<CacheGroupContext> grps = cctx.cache().cacheGroups();
+
+        return createPartitionsFullMessage(compress, newCntrMap, exchId, lastVer, partHistSuppliers, partsToReload, grps);
+    }
+
+    /**
+     * Creates partitions full message for selected cache groups.
+     *
+     * @param compress {@code True} if possible to compress message (properly work only if prepareMarshall/
+     *     finishUnmarshall methods are called).
+     * @param newCntrMap {@code True} if possible to use {@link CachePartitionFullCountersMap}.
+     * @param exchId Non-null exchange ID if message is created for exchange.
+     * @param lastVer Last version.
+     * @param partHistSuppliers Partition history suppliers map.
+     * @param partsToReload Partitions to reload map.
+     * @param grps Selected cache groups.
+     * @return Message.
+     */
+    public GridDhtPartitionsFullMessage createPartitionsFullMessage(
+        boolean compress,
+        boolean newCntrMap,
+        @Nullable final GridDhtPartitionExchangeId exchId,
+        @Nullable GridCacheVersion lastVer,
+        @Nullable IgniteDhtPartitionHistorySuppliersMap partHistSuppliers,
+        @Nullable IgniteDhtPartitionsToReloadMap partsToReload,
+        Collection<CacheGroupContext> grps
+    ) {
+        AffinityTopologyVersion ver = exchId != null ? exchId.topologyVersion() : AffinityTopologyVersion.NONE;
+
+        final GridDhtPartitionsFullMessage m =
+            new GridDhtPartitionsFullMessage(exchId, lastVer, ver, partHistSuppliers, partsToReload);
 
         m.compress(compress);
 
         final Map<Object, T2<Integer, GridDhtPartitionFullMap>> dupData = new HashMap<>();
 
-        for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+        for (CacheGroupContext grp : grps) {
             if (!grp.isLocal()) {
                 if (exchId != null) {
                     AffinityTopologyVersion startTopVer = grp.localStartVersion();
@@ -1172,14 +1240,8 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage(
 
                 GridDhtPartitionFullMap locMap = grp.topology().partitionMap(true);
 
-                if (locMap != null) {
-                    addFullPartitionsMap(m,
-                        dupData,
-                        compress,
-                        grp.groupId(),
-                        locMap,
-                        affCache.similarAffinityKey());
-                }
+                if (locMap != null)
+                    addFullPartitionsMap(m, dupData, compress, grp.groupId(), locMap, affCache.similarAffinityKey());
 
                 m.addPartitionSizes(grp.groupId(), grp.topology().globalPartSizes());
 
@@ -1200,14 +1262,8 @@ public GridDhtPartitionsFullMessage createPartitionsFullMessage(
         for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) {
             GridDhtPartitionFullMap map = top.partitionMap(true);
 
-            if (map != null) {
-                addFullPartitionsMap(m,
-                    dupData,
-                    compress,
-                    top.groupId(),
-                    map,
-                    top.similarAffinityKey());
-            }
+            if (map != null)
+                addFullPartitionsMap(m, dupData, compress, top.groupId(), map, top.similarAffinityKey());
 
             if (exchId != null) {
                 CachePartitionFullCountersMap cntrsMap = top.fullUpdateCounters();
@@ -1267,13 +1323,15 @@ private void addFullPartitionsMap(GridDhtPartitionsFullMessage m,
     /**
      * @param node Destination cluster node.
      * @param id Exchange ID.
+     * @param grps Cache groups for send partitions.
      */
-    private void sendLocalPartitions(ClusterNode node, @Nullable GridDhtPartitionExchangeId id) {
-        GridDhtPartitionsSingleMessage m = createPartitionsSingleMessage(id,
-            cctx.kernalContext().clientNode(),
-            false,
-            false,
-            null);
+    private void sendLocalPartitions(
+        ClusterNode node,
+        @Nullable GridDhtPartitionExchangeId id,
+        @NotNull Collection<CacheGroupContext> grps
+    ) {
+        GridDhtPartitionsSingleMessage m =
+            createPartitionsSingleMessage(id, cctx.kernalContext().clientNode(), false, false, null, grps);
 
         if (log.isTraceEnabled())
             log.trace("Sending local partitions [nodeId=" + node.id() + ", msg=" + m + ']');
@@ -1292,6 +1350,8 @@ private void sendLocalPartitions(ClusterNode node, @Nullable GridDhtPartitionExc
     }
 
     /**
+     * Creates partitions single message for all cache groups.
+     *
      * @param exchangeId Exchange ID.
      * @param clientOnlyExchange Client exchange flag.
      * @param sndCounters {@code True} if need send partition update counters.
@@ -1304,6 +1364,29 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(
         boolean sndCounters,
         boolean newCntrMap,
         ExchangeActions exchActions
+    ) {
+        Collection<CacheGroupContext> grps = cctx.cache().cacheGroups();
+
+        return createPartitionsSingleMessage(exchangeId, clientOnlyExchange, sndCounters, newCntrMap, exchActions, grps);
+    }
+
+    /**
+     * Creates partitions single message for selected cache groups.
+     *
+     * @param exchangeId Exchange ID.
+     * @param clientOnlyExchange Client exchange flag.
+     * @param sndCounters {@code True} if need send partition update counters.
+     * @param newCntrMap {@code True} if possible to use {@link CachePartitionPartialCountersMap}.
+     * @param grps Selected cache groups.
+     * @return Message.
+     */
+    public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(
+        @Nullable GridDhtPartitionExchangeId exchangeId,
+        boolean clientOnlyExchange,
+        boolean sndCounters,
+        boolean newCntrMap,
+        ExchangeActions exchActions,
+        Collection<CacheGroupContext> grps
     ) {
         GridDhtPartitionsSingleMessage m = new GridDhtPartitionsSingleMessage(exchangeId,
             clientOnlyExchange,
@@ -1312,7 +1395,7 @@ public GridDhtPartitionsSingleMessage createPartitionsSingleMessage(
 
         Map<Object, T2<Integer, GridPartitionStateMap>> dupData = new HashMap<>();
 
-        for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+        for (CacheGroupContext grp : grps) {
             if (!grp.isLocal() && (exchActions == null || !exchActions.cacheGroupStopping(grp.groupId()))) {
                 GridDhtPartitionMap locMap = grp.topology().localPartitionMap();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 7d353546b2ccb..fee68f7a07cf2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1829,22 +1829,24 @@ public void finishMerged() {
             if (centralizedAff || forceAffReassignment) {
                 assert !exchCtx.mergeExchanges();
 
+                Collection<CacheGroupContext> grpToRefresh = U.newHashSet(cctx.cache().cacheGroups().size());
+
                 for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                     if (grp.isLocal())
                         continue;
 
-                    boolean needRefresh = false;
-
                     try {
-                        needRefresh = grp.topology().initPartitionsWhenAffinityReady(res, this);
+                        if (grp.topology().initPartitionsWhenAffinityReady(res, this))
+                            grpToRefresh.add(grp);
                     }
                     catch (IgniteInterruptedCheckedException e) {
                         U.error(log, "Failed to initialize partitions.", e);
                     }
 
-                    if (needRefresh)
-                        cctx.exchange().refreshPartitions();
                 }
+
+                if (!grpToRefresh.isEmpty())
+                    cctx.exchange().refreshPartitions(grpToRefresh);
             }
 
             for (GridCacheContext cacheCtx : cctx.cacheContexts()) {

From 2713a5492e855b5b8fe9c09bff3886c01c5d5a85 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 18 Oct 2018 14:33:28 +0300
Subject: [PATCH 431/543] IGNITE-9854 Correct remove from dirtyPages and
 segCheckpointPages - Fixes #4988.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit 829dc1f240c07731a1ee98ae18c80ea6074dc6c4)
---
 .../persistence/pagemem/PageMemoryImpl.java      | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index ad32469e2dda9..689530f53ff14 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -841,11 +841,15 @@ private long refreshOutdatedPage(Segment seg, int grpId, long pageId, boolean rm
         if (rmv)
             seg.loadedPages.remove(grpId, PageIdUtils.effectivePageId(pageId));
 
-        if (seg.segCheckpointPages != null)
-            seg.segCheckpointPages.remove(new FullPageId(pageId, grpId));
+        Collection<FullPageId> cpPages = seg.segCheckpointPages;
 
-        if (seg.dirtyPages != null)
-            seg.dirtyPages.remove(new FullPageId(pageId, grpId));
+        if (cpPages != null)
+            cpPages.remove(new FullPageId(pageId, grpId));
+
+        Collection<FullPageId> dirtyPages = seg.dirtyPages;
+
+        if (dirtyPages != null)
+            dirtyPages.remove(new FullPageId(pageId, grpId));
 
         return relPtr;
     }
@@ -1857,7 +1861,7 @@ private class Segment extends ReentrantReadWriteLock {
         private static final int ACQUIRED_PAGES_PADDING = 4;
 
         /** Page ID to relative pointer map. */
-        private LoadedPagesMap loadedPages;
+        private final LoadedPagesMap loadedPages;
 
         /** Pointer to acquired pages integer counter. */
         private long acquiredPagesPtr;
@@ -1869,7 +1873,7 @@ private class Segment extends ReentrantReadWriteLock {
         private long memPerTbl;
 
         /** Pages marked as dirty since the last checkpoint. */
-        private Collection<FullPageId> dirtyPages = new GridConcurrentHashSet<>();
+        private volatile Collection<FullPageId> dirtyPages = new GridConcurrentHashSet<>();
 
         /** */
         private volatile Collection<FullPageId> segCheckpointPages;

From ee1ece0f7bbc87b903f0f5832ba6849cf16fb9b0 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Wed, 17 Oct 2018 17:56:08 +0300
Subject: [PATCH 432/543] IGNITE-9272: j.u.zip.CRC32algo instead of
 PureJavaCrc32 - Fixes #4619.

Signed-off-by: Nikolay Izhikov <nizhikov@apache.org>
(cherry picked from commit 091ace90e56bac1f7bc76886a5dccff796cbe34e)
---
 .../benchmarks/jmh/algo/BenchmarkCRC.java     |  95 ++++++++++++++++
 .../GridCacheDatabaseSharedManager.java       |   4 +-
 .../cache/persistence/file/FilePageStore.java |   7 +-
 .../wal/FileWriteAheadLogManager.java         |   8 +-
 .../FsyncModeFileWriteAheadLogManager.java    |   4 +-
 .../cache/persistence/wal/crc/FastCrc.java    | 101 ++++++++++++++++++
 .../persistence/wal/crc/PureJavaCrc32.java    |   2 +
 .../cache/persistence/wal/io/FileInput.java   |  10 +-
 .../persistence/wal/io/SimpleFileInput.java   |   3 +-
 .../wal/serializer/RecordV1Serializer.java    |   5 +-
 .../db/wal/crc/IgniteDataIntegrityTests.java  |   6 +-
 .../crc/IgnitePureJavaCrcCompatibility.java   |  55 ++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   2 +
 13 files changed, 281 insertions(+), 21 deletions(-)
 create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java

diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java
new file mode 100644
index 0000000000000..5c922fead0c80
--- /dev/null
+++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.benchmarks.jmh.algo;
+
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static org.openjdk.jmh.annotations.Mode.AverageTime;
+import static org.openjdk.jmh.annotations.Scope.Thread;
+
+/**
+ *
+ */
+@State(Thread)
+@OutputTimeUnit(NANOSECONDS)
+@BenchmarkMode(AverageTime)
+@Fork(value = 1, jvmArgsAppend = {"-XX:+UnlockDiagnosticVMOptions"})
+@Warmup(iterations = 5)
+@Measurement(iterations = 5)
+public class BenchmarkCRC {
+    /** */
+    static final int SIZE = 1024;
+
+    /** */
+    static final int BUF_LEN = 4096;
+
+    /** */
+    @State(Thread)
+    public static class Context {
+        /** */
+        final int[] results = new int[SIZE];
+
+        /** */
+        final ByteBuffer bb = ByteBuffer.allocate(BUF_LEN);
+
+        /** */
+        @Setup
+        public void setup() {
+            new Random().ints(BUF_LEN, Byte.MIN_VALUE, Byte.MAX_VALUE).forEach(k -> bb.put((byte) k));
+        }
+    }
+
+    /** */
+    @Benchmark
+    public int[] pureJavaCrc32(Context context) {
+        for (int i = 0; i < SIZE; i++) {
+            context.bb.rewind();
+
+            context.results[i] = PureJavaCrc32.calcCrc32(context.bb, BUF_LEN);
+        }
+
+        return context.results;
+    }
+
+    /** */
+    @Benchmark
+    public int[] crc32(Context context) {
+        for (int i = 0; i < SIZE; i++) {
+            context.bb.rewind();
+
+            context.results[i] = FastCrc.calcCrc(context.bb, BUF_LEN);
+        }
+
+        return context.results;
+    }
+}
+
+
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 5277382e9f412..7332b2a8714aa 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -131,8 +131,8 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.processors.port.GridPortRecord;
 import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
 import org.apache.ignite.internal.util.GridUnsafe;
@@ -3983,7 +3983,7 @@ private List<FullPageId> writePages(Collection<FullPageId> writePageIds) throws
                     }
 
                     if (!skipCrc) {
-                        PageIO.setCrc(writeAddr, PureJavaCrc32.calcCrc32(tmpWriteBuf, pageSize()));
+                        PageIO.setCrc(writeAddr, FastCrc.calcCrc(tmpWriteBuf, pageSize()));
 
                         tmpWriteBuf.rewind();
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 54d3de182fd33..950622f6a191b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -27,6 +27,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -35,8 +36,8 @@
 import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker;
 import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 import static java.nio.file.StandardOpenOption.CREATE;
@@ -379,7 +380,7 @@ public void finishRecover() throws StorageException {
             pageBuf.position(0);
 
             if (!skipCrc) {
-                int curCrc32 = PureJavaCrc32.calcCrc32(pageBuf, pageSize);
+                int curCrc32 = FastCrc.calcCrc(pageBuf, pageSize);
 
                 if ((savedCrc32 ^ curCrc32) != 0)
                     throw new IgniteDataIntegrityViolationException("Failed to read page (CRC validation failed) " +
@@ -647,7 +648,7 @@ private static int calcCrc32(ByteBuffer pageBuf, int pageSize) {
         try {
             pageBuf.position(0);
 
-            return PureJavaCrc32.calcCrc32(pageBuf, pageSize);
+            return FastCrc.calcCrc(pageBuf, pageSize);
         }
         finally {
             pageBuf.position(0);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index b89de2681d03c..2fa2873afd341 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -93,16 +93,15 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
-import org.apache.ignite.internal.processors.cache.persistence.wal.AbstractWalRecordsIterator.AbstractFileDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.LockedSegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleSegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
-import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializer;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordSerializerFactoryImpl;
@@ -118,7 +117,6 @@
 import org.apache.ignite.internal.util.typedef.CO;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -2268,7 +2266,7 @@ else if (create)
             buf.position(0);
 
             // This call will move buffer position to the end of the record again.
-            int crcVal = PureJavaCrc32.calcCrc32(buf, curPos);
+            int crcVal = FastCrc.calcCrc(buf, curPos);
 
             buf.putInt(crcVal);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 79064ff913a18..06ae98708023a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -87,8 +87,8 @@
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
@@ -2175,7 +2175,7 @@ public static long writeSerializerVersion(FileIO io, long idx, int version, WALM
             buf.position(0);
 
             // This call will move buffer position to the end of the record again.
-            int crcVal = PureJavaCrc32.calcCrc32(buf, curPos);
+            int crcVal = FastCrc.calcCrc(buf, curPos);
 
             buf.putInt(crcVal);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java
new file mode 100644
index 0000000000000..0dcbafdb9c978
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.crc;
+
+import java.nio.ByteBuffer;
+import java.util.zip.CRC32;
+
+/**
+ * This CRC calculation implementation workf much faster then {@link PureJavaCrc32}
+ */
+public final class FastCrc {
+    /** CRC algo. */
+    private static final ThreadLocal<CRC32> CRC = ThreadLocal.withInitial(CRC32::new);
+
+    /** */
+    private final CRC32 crc = new CRC32();
+
+    /**
+     * Current value.
+     */
+    private int val;
+
+    /** */
+    public FastCrc() {
+        reset();
+    }
+
+    /**
+     * Preparation for further calculations.
+     */
+    public void reset() {
+        val = 0xffffffff;
+
+        crc.reset();
+    }
+
+    /**
+     * @return crc value.
+     */
+    public int getValue() {
+        return val;
+    }
+
+    /**
+     * @param buf Input buffer.
+     * @param len Data length.
+     */
+    public void update(final ByteBuffer buf, final int len) {
+        val = calcCrc(crc, buf, len);
+    }
+
+    /**
+     * @param buf Input buffer.
+     * @param len Data length.
+     *
+     * @return Crc checksum.
+     */
+    public static int calcCrc(ByteBuffer buf, int len) {
+        CRC32 crcAlgo = CRC.get();
+
+        int res = calcCrc(crcAlgo, buf, len);
+
+        crcAlgo.reset();
+
+        return res;
+    }
+
+    /**
+     * @param crcAlgo CRC algorithm.
+     * @param buf Input buffer.
+     * @param len Buffer length.
+     *
+     * @return Crc checksum.
+     */
+    private static int calcCrc(CRC32 crcAlgo, ByteBuffer buf, int len) {
+        int initLimit = buf.limit();
+
+        buf.limit(buf.position() + len);
+
+        crcAlgo.update(buf);
+
+        buf.limit(initLimit);
+
+        return (int)crcAlgo.getValue() ^ 0xFFFFFFFF;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java
index 6bd4a35b2fdf8..b011f783be005 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java
@@ -29,7 +29,9 @@
  * succession.
  *
  * The current version is ~10x to 1.8x as fast as Sun's native java.util.zip.CRC32 in Java 1.6
+ * @deprecated Use {@link FastCrc} instead.
  */
+@Deprecated
 public class PureJavaCrc32 {
     /**
      * the current CRC value
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java
index d19d17b3c7d7f..c9615f597e0f7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java
@@ -19,10 +19,12 @@
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.zip.CRC32;
+
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.jetbrains.annotations.NotNull;
 
 /**
@@ -56,7 +58,7 @@ public interface FileInput extends ByteBufferBackedDataInput {
      */
     public class Crc32CheckingFileInput implements ByteBufferBackedDataInput, AutoCloseable {
         /** */
-        private final PureJavaCrc32 crc32 = new PureJavaCrc32();
+        private final FastCrc crc = new FastCrc();
 
         /** Last calc position. */
         private int lastCalcPosition;
@@ -93,7 +95,7 @@ public Crc32CheckingFileInput(FileInput delegate, boolean skipCheck) {
         @Override public void close() throws Exception {
             updateCrc();
 
-            int val = crc32.getValue();
+            int val = crc.getValue();
 
             int writtenCrc =  this.readInt();
 
@@ -118,7 +120,7 @@ private void updateCrc() {
 
             buffer().position(lastCalcPosition);
 
-            crc32.update(delegate.buffer(), oldPos - lastCalcPosition);
+            crc.update(delegate.buffer(), oldPos - lastCalcPosition);
 
             lastCalcPosition = oldPos;
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java
index 5918b0b34e93e..1a1562e0909e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java
@@ -20,6 +20,7 @@
 import java.io.EOFException;
 import java.io.IOException;
 import java.nio.ByteBuffer;
+
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
 import org.jetbrains.annotations.NotNull;
@@ -264,7 +265,7 @@ private void clearBuffer() {
 
     /**
      * @param skipCheck If CRC check should be skipped.
-     * @return autoclosable fileInput, after its closing crc32 will be calculated and compared with saved one
+     * @return autoclosable fileInput, after its closing crc will be calculated and compared with saved one
      */
     public Crc32CheckingFileInput startRead(boolean skipCheck) {
         return new Crc32CheckingFileInput(this, skipCheck);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
index afd770d8efecf..e27faa5f02025 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -22,6 +22,7 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
@@ -32,6 +33,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
@@ -39,7 +41,6 @@
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleFileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException;
-import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
 import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io.RecordIO;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -421,7 +422,7 @@ static void writeWithCrc(WALRecord rec, ByteBuffer buf, RecordIO writer) throws
             buf.position(startPos);
 
             // This call will move buffer position to the end of the record again.
-            int crcVal = PureJavaCrc32.calcCrc32(buf, curPos - startPos);
+            int crcVal = FastCrc.calcCrc(buf, curPos - startPos);
 
             buf.putInt(crcVal);
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
index 59dd3b7e7dc9b..3ad4c9007e752 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
@@ -24,13 +24,14 @@
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 import java.util.concurrent.ThreadLocalRandom;
+
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleFileInput;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
 
 /**
  *
@@ -59,6 +60,7 @@ public class IgniteDataIntegrityTests extends TestCase {
         );
 
         ByteBuffer buf = ByteBuffer.allocate(1024);
+
         ThreadLocalRandom curr = ThreadLocalRandom.current();
 
         for (int i = 0; i < 1024; i+=16) {
@@ -66,7 +68,7 @@ public class IgniteDataIntegrityTests extends TestCase {
             buf.putInt(curr.nextInt());
             buf.putInt(curr.nextInt());
             buf.position(i);
-            buf.putInt(PureJavaCrc32.calcCrc32(buf, 12));
+            buf.putInt(FastCrc.calcCrc(buf, 12));
         }
 
         buf.rewind();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java
new file mode 100644
index 0000000000000..faafad214270a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal.crc;
+
+import junit.framework.TestCase;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * PureJavaCrc32 previous crc algo realization vs java.util.zip.crc32 test.
+ */
+public class IgnitePureJavaCrcCompatibility extends TestCase {
+    /**
+     * Test crc algo equality results.
+     * @throws Exception
+     */
+    public void testAlgoEqual() throws Exception {
+        ByteBuffer buf = ByteBuffer.allocate(1024);
+
+        ThreadLocalRandom curr = ThreadLocalRandom.current();
+
+        for (int i = 0; i < 1024; i+=16) {
+            buf.putInt(curr.nextInt());
+            buf.putInt(curr.nextInt());
+            buf.putInt(curr.nextInt());
+            buf.position(i);
+
+            buf.position(i);
+            int crc0 = FastCrc.calcCrc(buf, 12);
+
+            buf.position(i);
+            int crc1 = PureJavaCrc32.calcCrc32(buf, 12);
+
+            assertEquals(crc0, crc1);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index b94589178ea33..c60cd9a3e63b1 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteFsyncReplayWalIteratorInvalidCrcTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteReplayWalIteratorInvalidCrcTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteStandaloneWalIteratorInvalidCrcTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgnitePureJavaCrcCompatibility;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.reader.IgniteWalReaderTest;
 import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneWalRecordsIteratorTest;
 
@@ -84,6 +85,7 @@ public static TestSuite suite() {
         suite.addTestSuite(IgniteStandaloneWalIteratorInvalidCrcTest.class);
         suite.addTestSuite(IgniteReplayWalIteratorInvalidCrcTest.class);
         suite.addTestSuite(IgniteFsyncReplayWalIteratorInvalidCrcTest.class);
+        suite.addTestSuite(IgnitePureJavaCrcCompatibility.class);
 
         addRealPageStoreTests(suite);
 

From ffa3256162e05a53ae68ebc909dae3eec2e53874 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Tue, 9 Oct 2018 14:33:25 +0300
Subject: [PATCH 433/543] IGNITE-9785 Introduce read-only state in local node
 context - Fixes #4907.

Signed-off-by: Ivan Rakov <irakov@apache.org>

(cherry picked from commit 179b09b)
---
 .../cache/GridCacheSharedContext.java         |  17 +++
 .../dht/GridDhtTopologyFutureAdapter.java     |   3 +
 .../datastreamer/DataStreamerImpl.java        |  34 +++--
 .../ClusterReadOnlyModeAbstractTest.java      | 114 +++++++++++++++
 .../cache/ClusterReadOnlyModeTest.java        | 134 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite5.java     |   2 +
 .../cache/ClusterReadOnlyModeSqlTest.java     |  94 ++++++++++++
 .../IgniteCacheWithIndexingTestSuite.java     |   3 +
 8 files changed, 392 insertions(+), 9 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeTest.java
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeSqlTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index 9655b14ebe647..6d30407859f3c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -172,6 +172,9 @@ public class GridCacheSharedContext<K, V> {
     /** */
     private final List<IgniteChangeGlobalStateSupport> stateAwareMgrs;
 
+    /** Cluster is in read-only mode. */
+    private volatile boolean readOnlyMode;
+
     /**
      * @param kernalCtx  Context.
      * @param txMgr Transaction manager.
@@ -1102,4 +1105,18 @@ public void finishDhtAtomicUpdate(GridCacheVersion ver) {
     private int dhtAtomicUpdateIndex(GridCacheVersion ver) {
         return U.safeAbs(ver.hashCode()) % dhtAtomicUpdCnt.length();
     }
+
+    /**
+     * @return {@code true} if cluster is in read-only mode.
+     */
+    public boolean readOnlyMode() {
+        return readOnlyMode;
+    }
+
+    /**
+     * @param readOnlyMode Read-only flag.
+     */
+    public void readOnlyMode(boolean readOnlyMode) {
+        this.readOnlyMode = readOnlyMode;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
index 539fef48bda46..92143080c2409 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFutureAdapter.java
@@ -97,6 +97,9 @@ protected final CacheValidation validateCacheGroup(CacheGroupContext grp, Collec
                     cctx.name());
         }
 
+        if (cctx.shared().readOnlyMode() && !read)
+            return new IgniteCheckedException("Failed to perform cache operation (cluster is in read only mode)" );
+
         if (grp.needsRecovery() || grp.topologyValidator() != null) {
             CacheValidation validation = grpValidRes.get(grp.groupId());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
index 2b731ee663940..64ddeb7695f2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerImpl.java
@@ -85,10 +85,10 @@
 import org.apache.ignite.internal.processors.cache.IgniteCacheFutureImpl;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.dr.GridDrType;
@@ -586,7 +586,7 @@ else if (autoFlushFreq == 0)
                 entries0.add(new DataStreamerEntry(key, val));
             }
 
-            load0(entries0, resFut, keys, 0);
+            load0(entries0, resFut, keys, 0, null, null);
 
             return new IgniteCacheFutureImpl<>(resFut);
         }
@@ -638,7 +638,7 @@ public IgniteFuture<?> addDataInternal(Collection<? extends DataStreamerEntry> e
                     keys.add(new KeyCacheObjectWrapper(entry.getKey()));
             }
 
-            load0(entries, resFut, keys, 0);
+            load0(entries, resFut, keys, 0, null, null);
 
             return new IgniteCacheFutureImpl<>(resFut);
         }
@@ -725,12 +725,17 @@ private void acquireRemapSemaphore() throws IgniteInterruptedCheckedException {
      * @param resFut Result future.
      * @param activeKeys Active keys.
      * @param remaps Remaps count.
+     * @param remapNode Node for remap. In case update with {@code allowOverride() == false} fails on one node,
+     * we don't need to send update request to all affinity nodes again, if topology version does not changed.
+     * @param remapTopVer Topology version.
      */
     private void load0(
         Collection<? extends DataStreamerEntry> entries,
         final GridFutureAdapter<Object> resFut,
         @Nullable final Collection<KeyCacheObjectWrapper> activeKeys,
-        final int remaps
+        final int remaps,
+        ClusterNode remapNode,
+        AffinityTopologyVersion remapTopVer
     ) {
         try {
             assert entries != null;
@@ -803,7 +808,10 @@ else if (rcvr != null)
                         if (key.partition() == -1)
                             key.partition(cctx.affinity().partition(key, false));
 
-                        nodes = nodes(key, topVer, cctx);
+                        if (!allowOverwrite() && remapNode != null && F.eq(topVer, remapTopVer))
+                            nodes = Collections.singletonList(remapNode);
+                        else
+                            nodes = nodes(key, topVer, cctx);
                     }
                     catch (IgniteCheckedException e) {
                         resFut.onDone(e);
@@ -830,6 +838,7 @@ else if (rcvr != null)
                 }
 
                 for (final Map.Entry<ClusterNode, Collection<DataStreamerEntry>> e : mappings.entrySet()) {
+                    final ClusterNode node = e.getKey();
                     final UUID nodeId = e.getKey().id();
 
                     Buffer buf = bufMappings.get(nodeId);
@@ -891,7 +900,7 @@ else if (remaps + 1 > maxRemapCnt) {
                                                     if (cancelled)
                                                         closedException();
 
-                                                    load0(entriesForNode, resFut, activeKeys, remaps + 1);
+                                                    load0(entriesForNode, resFut, activeKeys, remaps + 1, node, topVer);
                                                 }
                                                 catch (Throwable ex) {
                                                     resFut.onDone(
@@ -2057,9 +2066,9 @@ protected static class IsolatedUpdater implements StreamReceiver<KeyCacheObject,
 
             GridCacheContext cctx = internalCache.context();
 
-            AffinityTopologyVersion topVer = cctx.isLocal() ?
-                cctx.affinity().affinityTopologyVersion() :
-                cctx.shared().exchange().readyAffinityVersion();
+            GridDhtTopologyFuture topFut = cctx.shared().exchange().lastFinishedFuture();
+
+            AffinityTopologyVersion topVer = topFut.topologyVersion();
 
             GridCacheVersion ver = cctx.versions().isolatedStreamerVersion();
 
@@ -2120,6 +2129,13 @@ else if (ttl == CU.TTL_NOT_CHANGED)
                             expiryTime = CU.toExpireTime(ttl);
                         }
 
+                        if (topFut != null) {
+                            Throwable err = topFut.validateCache(cctx, false, false, entry.key(), null);
+
+                            if (err != null)
+                                throw new IgniteCheckedException(err);
+                        }
+
                         boolean primary = cctx.affinity().primaryByKey(cctx.localNode(), entry.key(), topVer);
 
                         entry.initialValue(e.getValue(),
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java
new file mode 100644
index 0000000000000..7cb9649a4ffb8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeAbstractTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collection;
+import java.util.Collections;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+
+/**
+ *
+ */
+public class ClusterReadOnlyModeAbstractTest extends GridCommonAbstractTest {
+    /** */
+    private static final int SRVS = 3;
+
+    /** Replicated atomic cache. */
+    private static final String REPL_ATOMIC_CACHE = "repl_atomic_cache";
+
+    /** Replicated transactional cache. */
+    private static final String REPL_TX_CACHE = "repl_tx_cache";
+
+    /** Partitioned atomic cache. */
+    private static final String PART_ATOMIC_CACHE = "part_atomic_cache";
+
+    /** Partitioned transactional cache. */
+    private static final String PART_TX_CACHE = "part_tx_cache";
+
+    /** Cache names. */
+    protected static final Collection<String> CACHE_NAMES = F.asList(REPL_ATOMIC_CACHE, REPL_TX_CACHE,
+        PART_ATOMIC_CACHE, PART_TX_CACHE);
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        startGridsMultiThreaded(SRVS);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        changeClusterReadOnlyMode(false);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setCacheConfiguration(
+            cacheConfiguration(REPL_ATOMIC_CACHE, REPLICATED, ATOMIC, null),
+            cacheConfiguration(REPL_TX_CACHE, REPLICATED, TRANSACTIONAL, null),
+            cacheConfiguration(PART_ATOMIC_CACHE, PARTITIONED, ATOMIC, "part_grp"),
+            cacheConfiguration(PART_TX_CACHE, PARTITIONED, TRANSACTIONAL, "part_grp")
+        );
+
+        return cfg;
+    }
+
+    /**
+     * @param cacheMode Cache mode.
+     * @param atomicityMode Atomicity mode.
+     * @param grpName Cache group name.
+     */
+    private CacheConfiguration<Integer, Integer> cacheConfiguration(String name, CacheMode cacheMode,
+        CacheAtomicityMode atomicityMode, String grpName) {
+        return new CacheConfiguration<Integer, Integer>()
+            .setName(name)
+            .setCacheMode(cacheMode)
+            .setAtomicityMode(atomicityMode)
+            .setGroupName(grpName)
+            .setQueryEntities(Collections.singletonList(new QueryEntity(Integer.class, Integer.class)));
+    }
+
+    /**
+     * Change read only mode on all nodes.
+     *
+     * @param readOnly Read only.
+     */
+    protected void changeClusterReadOnlyMode(boolean readOnly) {
+        for (int idx = 0; idx < SRVS; idx++) {
+            IgniteEx ignite = grid(idx);
+
+            ignite.context().cache().context().readOnlyMode(readOnly);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeTest.java
new file mode 100644
index 0000000000000..ab57614a9e86d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Random;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.processors.datastreamer.DataStreamerImpl;
+import org.apache.ignite.internal.util.typedef.G;
+
+/**
+ * Tests cache get/put/remove and data streaming in read-only cluster mode.
+ */
+public class ClusterReadOnlyModeTest extends ClusterReadOnlyModeAbstractTest {
+    /**
+     * Tests cache get/put/remove.
+     */
+    public void testCacheGetPutRemove() {
+        assertCachesReadOnlyMode(false);
+
+        changeClusterReadOnlyMode(true);
+
+        assertCachesReadOnlyMode(true);
+
+        changeClusterReadOnlyMode(false);
+
+        assertCachesReadOnlyMode(false);
+    }
+
+    /**
+     * Tests data streamer.
+     */
+    public void testDataStreamerReadOnly() {
+        assertDataStreamerReadOnlyMode(false);
+
+        changeClusterReadOnlyMode(true);
+
+        assertDataStreamerReadOnlyMode(true);
+
+        changeClusterReadOnlyMode(false);
+
+        assertDataStreamerReadOnlyMode(false);
+    }
+
+    /**
+     * Asserts that all caches in read-only or in read/write mode on all nodes.
+     *
+     * @param readOnly If {@code true} then cache must be in read only mode, else in read/write mode.
+     */
+    private void assertCachesReadOnlyMode(boolean readOnly) {
+        Random rnd = new Random();
+
+        for (Ignite ignite : G.allGrids()) {
+            for (String cacheName : CACHE_NAMES) {
+                IgniteCache<Integer, Integer> cache = ignite.cache(cacheName);
+
+                for (int i = 0; i < 10; i++) {
+                    cache.get(rnd.nextInt(100)); // All gets must succeed.
+
+                    if (readOnly) {
+                        // All puts must fail.
+                        try {
+                            cache.put(rnd.nextInt(100), rnd.nextInt());
+
+                            fail("Put must fail for cache " + cacheName);
+                        }
+                        catch (Exception e) {
+                            // No-op.
+                        }
+
+                        // All removes must fail.
+                        try {
+                            cache.remove(rnd.nextInt(100));
+
+                            fail("Remove must fail for cache " + cacheName);
+                        }
+                        catch (Exception e) {
+                            // No-op.
+                        }
+                    }
+                    else {
+                        cache.put(rnd.nextInt(100), rnd.nextInt()); // All puts must succeed.
+
+                        cache.remove(rnd.nextInt(100)); // All removes must succeed.
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * @param readOnly If {@code true} then data streamer must fail, else succeed.
+     */
+    private void assertDataStreamerReadOnlyMode(boolean readOnly) {
+        Random rnd = new Random();
+
+        for (Ignite ignite : G.allGrids()) {
+            for (String cacheName : CACHE_NAMES) {
+                boolean failed = false;
+
+                try (IgniteDataStreamer<Integer, Integer> streamer = ignite.dataStreamer(cacheName)) {
+                    for (int i = 0; i < 10; i++) {
+                        ((DataStreamerImpl)streamer).maxRemapCount(5);
+
+                        streamer.addData(rnd.nextInt(1000), rnd.nextInt());
+                    }
+                }
+                catch (CacheException ignored) {
+                    failed = true;
+                }
+
+                if (failed != readOnly)
+                    fail("Streaming to " + cacheName + " must " + (readOnly ? "fail" : "succeed"));
+            }
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
index 41fca2a044383..d2060d4df6645 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite5.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.internal.processors.cache.CacheNearReaderUpdateTest;
 import org.apache.ignite.internal.processors.cache.CacheRebalancingSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheSerializableTransactionsTest;
+import org.apache.ignite.internal.processors.cache.ClusterReadOnlyModeTest;
 import org.apache.ignite.internal.processors.cache.ClusterStatePartitionedSelfTest;
 import org.apache.ignite.internal.processors.cache.ClusterStateReplicatedSelfTest;
 import org.apache.ignite.internal.processors.cache.ConcurrentCacheStartTest;
@@ -78,6 +79,7 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(ClusterStatePartitionedSelfTest.class);
         suite.addTestSuite(ClusterStateReplicatedSelfTest.class);
+        suite.addTestSuite(ClusterReadOnlyModeTest.class);
         suite.addTestSuite(IgniteCachePartitionLossPolicySelfTest.class);
         suite.addTestSuite(IgniteCacheGroupsPartitionLossPolicySelfTest.class);
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeSqlTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeSqlTest.java
new file mode 100644
index 0000000000000..d431f73fb0576
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/ClusterReadOnlyModeSqlTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Random;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.internal.util.typedef.G;
+
+/**
+ * Tests SQL queries in read-only cluster mode.
+ */
+public class ClusterReadOnlyModeSqlTest extends ClusterReadOnlyModeAbstractTest {
+    /**
+     *
+     */
+    public void testSqlReadOnly() {
+        assertSqlReadOnlyMode(false);
+
+        changeClusterReadOnlyMode(true);
+
+        assertSqlReadOnlyMode(true);
+
+        changeClusterReadOnlyMode(false);
+
+        assertSqlReadOnlyMode(false);
+    }
+
+    /**
+     * @param readOnly If {@code true} then data modification SQL queries must fail, else succeed.
+     */
+    private void assertSqlReadOnlyMode(boolean readOnly) {
+        Random rnd = new Random();
+
+        for (Ignite ignite : G.allGrids()) {
+            for (String cacheName : CACHE_NAMES) {
+                IgniteCache<Integer, Integer> cache = ignite.cache(cacheName);
+
+                try (FieldsQueryCursor<?> cur = cache.query(new SqlFieldsQuery("SELECT * FROM Integer"))) {
+                    cur.getAll();
+                }
+
+                boolean failed = false;
+
+                try (FieldsQueryCursor<?> cur = cache.query(new SqlFieldsQuery("DELETE FROM Integer"))) {
+                    cur.getAll();
+                }
+                catch (CacheException ex) {
+                    if (!readOnly)
+                        log.error("Failed to delete data", ex);
+
+                    failed = true;
+                }
+
+                if (failed != readOnly)
+                    fail("SQL delete from " + cacheName + " must " + (readOnly ? "fail" : "succeed"));
+
+                failed = false;
+
+                try (FieldsQueryCursor<?> cur = cache.query(new SqlFieldsQuery(
+                    "INSERT INTO Integer(_KEY, _VAL) VALUES (?, ?)").setArgs(rnd.nextInt(1000), rnd.nextInt()))) {
+                    cur.getAll();
+                }
+                catch (CacheException ex) {
+                    if (!readOnly)
+                        log.error("Failed to insert data", ex);
+
+                    failed = true;
+                }
+
+                if (failed != readOnly)
+                    fail("SQL insert into " + cacheName + " must " + (readOnly ? "fail" : "succeed"));
+            }
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index 1c7bd795f0f89..7c1ee528bcd85 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -26,6 +26,7 @@
 import org.apache.ignite.internal.processors.cache.CacheQueryFilterExpiredTest;
 import org.apache.ignite.internal.processors.cache.CacheRandomOperationsMultithreadedTest;
 import org.apache.ignite.internal.processors.cache.ClientReconnectAfterClusterRestartTest;
+import org.apache.ignite.internal.processors.cache.ClusterReadOnlyModeSqlTest;
 import org.apache.ignite.internal.processors.cache.GridCacheOffHeapSelfTest;
 import org.apache.ignite.internal.processors.cache.GridCacheOffheapIndexEntryEvictTest;
 import org.apache.ignite.internal.processors.cache.GridCacheOffheapIndexGetSelfTest;
@@ -87,6 +88,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgniteDataStreamerTest.class);
 
+        suite.addTestSuite(ClusterReadOnlyModeSqlTest.class);
+
         return suite;
     }
 }

From 29bae6cb016e11a668733074ef360862b4504818 Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Wed, 17 Oct 2018 16:35:32 +0300
Subject: [PATCH 434/543] IGNITE-9875 Optimized GridDhtPartitionsStateValidator
 - Fixes #4983.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 33b9611e72f03b2f6ec8c72600e7d42558a92339)
---
 modules/benchmarks/pom.xml                    |  12 +-
 ...dDhtPartitionsStateValidatorBenchmark.java | 168 ++++++++++++++++++
 .../CachePartitionFullCountersMap.java        |  21 ---
 .../GridDhtPartitionsSingleMessage.java       |  29 +++
 .../GridDhtPartitionsStateValidator.java      |  84 +++++----
 5 files changed, 257 insertions(+), 57 deletions(-)
 create mode 100644 modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java

diff --git a/modules/benchmarks/pom.xml b/modules/benchmarks/pom.xml
index 9e000d46706b7..d6789e639be18 100644
--- a/modules/benchmarks/pom.xml
+++ b/modules/benchmarks/pom.xml
@@ -62,6 +62,16 @@
             <version>${jmh.version}</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-all</artifactId>
+            <version>${mockito.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>${guava.version}</version>
+        </dependency>
     </dependencies>
 
     <build>
@@ -131,4 +141,4 @@
             </plugins>
         </pluginManagement>
     </build>
-</project>
\ No newline at end of file
+</project>
diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java
new file mode 100644
index 0000000000000..151606dab21ba
--- /dev/null
+++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java
@@ -0,0 +1,168 @@
+package org.apache.ignite.internal.benchmarks.jmh.misc;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.ignite.internal.benchmarks.jmh.JmhAbstractBenchmark;
+import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionsStateValidator;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.jetbrains.annotations.Nullable;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.IntStream;
+
+import static org.openjdk.jmh.annotations.Scope.Thread;
+
+/** */
+@State(Scope.Benchmark)
+public class GridDhtPartitionsStateValidatorBenchmark extends JmhAbstractBenchmark {
+    /** */
+    @State(Thread)
+    public static class Context {
+        /** */
+        private final UUID localNodeId = UUID.randomUUID();
+
+        /** */
+        private GridCacheSharedContext cctxMock;
+
+        /** */
+        private GridDhtPartitionTopology topologyMock;
+
+        /** */
+        private GridDhtPartitionsStateValidator validator;
+
+        /** */
+        private Map<UUID, GridDhtPartitionsSingleMessage> messages = new HashMap<>();
+
+        /** */
+        private UUID ignoreNode = UUID.randomUUID();
+
+        /** */
+        private static final int NODES = 3;
+
+        /** */
+        private static final int PARTS = 100;
+
+        /**
+         * @return Partition mock with specified {@code id}, {@code updateCounter} and {@code size}.
+         */
+        private GridDhtLocalPartition partitionMock(int id, long updateCounter, long size) {
+            GridDhtLocalPartition partitionMock = Mockito.mock(GridDhtLocalPartition.class);
+            Mockito.when(partitionMock.id()).thenReturn(id);
+            Mockito.when(partitionMock.updateCounter()).thenReturn(updateCounter);
+            Mockito.when(partitionMock.fullSize()).thenReturn(size);
+            Mockito.when(partitionMock.state()).thenReturn(GridDhtPartitionState.OWNING);
+            return partitionMock;
+        }
+
+        /**
+         * @param countersMap Update counters map.
+         * @param sizesMap Sizes map.
+         * @return Message with specified {@code countersMap} and {@code sizeMap}.
+         */
+        private GridDhtPartitionsSingleMessage from(@Nullable Map<Integer, T2<Long, Long>> countersMap, @Nullable Map<Integer, Long> sizesMap) {
+            GridDhtPartitionsSingleMessage msg = new GridDhtPartitionsSingleMessage();
+            if (countersMap != null)
+                msg.addPartitionUpdateCounters(0, countersMap);
+            if (sizesMap != null)
+                msg.addPartitionSizes(0, sizesMap);
+            return msg;
+        }
+
+        /** */
+        @Setup
+        public void setup() {
+            // Prepare mocks.
+            cctxMock = Mockito.mock(GridCacheSharedContext.class);
+            Mockito.when(cctxMock.localNodeId()).thenReturn(localNodeId);
+
+            topologyMock = Mockito.mock(GridDhtPartitionTopology.class);
+            Mockito.when(topologyMock.partitionState(Matchers.any(), Matchers.anyInt())).thenReturn(GridDhtPartitionState.OWNING);
+            Mockito.when(topologyMock.groupId()).thenReturn(0);
+
+            Mockito.when(topologyMock.partitions()).thenReturn(PARTS);
+
+            List<GridDhtLocalPartition> localPartitions = Lists.newArrayList();
+
+            Map<Integer, T2<Long, Long>> updateCountersMap = new HashMap<>();
+
+            Map<Integer, Long> cacheSizesMap = new HashMap<>();
+
+            IntStream.range(0, PARTS).forEach(k -> { localPartitions.add(partitionMock(k, k + 1, k + 1));
+                long us = k > 20 && k <= 30 ? 0 :k + 2L;
+                updateCountersMap.put(k, new T2<>(k + 2L, us));
+                cacheSizesMap.put(k, us); });
+
+            Mockito.when(topologyMock.localPartitions()).thenReturn(localPartitions);
+            Mockito.when(topologyMock.currentLocalPartitions()).thenReturn(localPartitions);
+
+            // Form single messages map.
+            Map<UUID, GridDhtPartitionsSingleMessage> messages = new HashMap<>();
+
+            for (int n = 0; n < NODES; ++n) {
+                UUID remoteNode = UUID.randomUUID();
+
+                messages.put(remoteNode, from(updateCountersMap, cacheSizesMap));
+            }
+
+            messages.put(ignoreNode, from(updateCountersMap, cacheSizesMap));
+
+            validator = new GridDhtPartitionsStateValidator(cctxMock);
+        }
+    }
+
+    /** */
+    @Benchmark
+    public void testValidatePartitionsUpdateCounters(Context context) {
+        context.validator.validatePartitionsUpdateCounters(context.topologyMock,
+                context.messages, Sets.newHashSet(context.ignoreNode));
+    }
+
+    /** */
+    @Benchmark
+    public void testValidatePartitionsSizes(Context context) {
+        context.validator.validatePartitionsSizes(context.topologyMock, context
+                .messages, Sets.newHashSet(context.ignoreNode));
+    }
+
+    /**
+     * Run benchmarks.
+     *
+     * @param args Arguments.
+     * @throws Exception If failed.
+     */
+    public static void main(String[] args) throws Exception {
+        run(1);
+    }
+
+    /**
+     * Run benchmark.
+     *
+     * @param threads Amount of threads.
+     * @throws Exception If failed.
+     */
+    private static void run(int threads) throws Exception {
+        JmhIdeBenchmarkRunner.create()
+                .forks(1)
+                .threads(threads)
+                .warmupIterations(5)
+                .measurementIterations(10)
+                .benchmarks(GridDhtPartitionsStateValidatorBenchmark.class.getSimpleName())
+                .jvmArguments("-XX:+UseG1GC", "-Xms4g", "-Xmx4g")
+                .run();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java
index 2d5eec3878cf5..008c2766e4a47 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionFullCountersMap.java
@@ -93,27 +93,6 @@ public void updateCounter(int p, long updCntr) {
         updCntrs[p] = updCntr;
     }
 
-    /**
-     * Creates submap for provided partition IDs.
-     *
-     * @param parts Partition IDs.
-     * @return Partial counters map.
-     */
-    public CachePartitionPartialCountersMap subMap(Set<Integer> parts) {
-        CachePartitionPartialCountersMap res = new CachePartitionPartialCountersMap(parts.size());
-
-        for (int p = 0; p < updCntrs.length; p++) {
-            if (!parts.contains(p))
-                continue;
-
-            res.add(p, initialUpdCntrs[p], updCntrs[p]);
-        }
-
-        assert res.size() == parts.size();
-
-        return res;
-    }
-
     /**
      * Clears full counters map.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 3ce3b9ffd9c23..636621da529bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -227,6 +227,35 @@ public CachePartitionPartialCountersMap partitionUpdateCounters(int grpId, int p
         return CachePartitionPartialCountersMap.fromCountersMap(map, partsCnt);
     }
 
+    /**
+     * @param grpId Cache group ID.
+     * @param partsCnt Total cache partitions.
+     * @return Partition update counters.
+     */
+    @SuppressWarnings("unchecked")
+    public CachePartitionPartialCountersMap partitionUpdateCountersUnsorted(int grpId, int partsCnt) {
+        Object res = partCntrs == null ? null : partCntrs.get(grpId);
+
+        if (res == null)
+            return CachePartitionPartialCountersMap.EMPTY;
+
+        if (res instanceof CachePartitionPartialCountersMap)
+            return (CachePartitionPartialCountersMap)res;
+
+        assert res instanceof Map : res;
+
+        Map<Integer, T2<Long, Long>> map = (Map<Integer, T2<Long, Long>>)res;
+
+        CachePartitionPartialCountersMap partCounersMap = new CachePartitionPartialCountersMap(partsCnt);
+
+        for (Map.Entry<Integer, T2<Long, Long>> e : map.entrySet())
+            partCounersMap.add(e.getKey(), e.getValue().get1(), e.getValue().get2());
+
+        partCounersMap.trim();
+
+        return partCounersMap;
+    }
+
     /**
      * Adds partition sizes map for specified {@code grpId} to the current message.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
index 18a0674826f56..5e94096c0f376 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
+import java.util.AbstractMap;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
@@ -32,7 +33,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.jetbrains.annotations.Nullable;
@@ -77,14 +77,16 @@ public void validatePartitionCountersAndSizes(
         final Set<UUID> ignoringNodes = new HashSet<>();
 
         // Ignore just joined nodes.
-        for (DiscoveryEvent evt : fut.events().events())
+        for (DiscoveryEvent evt : fut.events().events()) {
             if (evt.type() == EVT_NODE_JOINED)
                 ignoringNodes.add(evt.eventNode().id());
+        }
 
         AffinityTopologyVersion topVer = fut.context().events().topologyVersion();
 
         // Validate update counters.
         Map<Integer, Map<UUID, Long>> result = validatePartitionsUpdateCounters(top, messages, ignoringNodes);
+
         if (!result.isEmpty())
             throw new IgniteCheckedException("Partitions update counters are inconsistent for " + fold(topVer, result));
 
@@ -106,13 +108,16 @@ public void validatePartitionCountersAndSizes(
      *
      * @param top Topology to validate.
      * @param nodeId Node which sent single message.
-     * @param singleMsg Single message.
+     * @param countersMap Counters map.
+     * @param sizesMap Sizes map.
      * @return Set of partition ids should be excluded from validation.
      */
-    @Nullable private Set<Integer> shouldIgnore(GridDhtPartitionTopology top, UUID nodeId, GridDhtPartitionsSingleMessage singleMsg) {
-        CachePartitionPartialCountersMap countersMap = singleMsg.partitionUpdateCounters(top.groupId(), top.partitions());
-        Map<Integer, Long> sizesMap = singleMsg.partitionSizes(top.groupId());
-
+    @Nullable private Set<Integer> shouldIgnore(
+        GridDhtPartitionTopology top,
+        UUID nodeId,
+        CachePartitionPartialCountersMap countersMap,
+        Map<Integer, Long> sizesMap
+    ) {
         Set<Integer> ignore = null;
 
         for (int i = 0; i < countersMap.size(); i++) {
@@ -151,14 +156,14 @@ public void validatePartitionCountersAndSizes(
      * @return Invalid partitions map with following structure: (partId, (nodeId, updateCounter)).
      * If map is empty validation is successful.
      */
-     public Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
-            GridDhtPartitionTopology top,
-            Map<UUID, GridDhtPartitionsSingleMessage> messages,
-            Set<UUID> ignoringNodes
-     ) {
+    public Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
+        GridDhtPartitionTopology top,
+        Map<UUID, GridDhtPartitionsSingleMessage> messages,
+        Set<UUID> ignoringNodes
+    ) {
         Map<Integer, Map<UUID, Long>> invalidPartitions = new HashMap<>();
 
-        Map<Integer, T2<UUID, Long>> updateCountersAndNodesByPartitions = new HashMap<>();
+        Map<Integer, AbstractMap.Entry<UUID, Long>> updateCountersAndNodesByPartitions = new HashMap<>();
 
         // Populate counters statistics from local node partitions.
         for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
@@ -168,7 +173,7 @@ public Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
             if (part.updateCounter() == 0 && part.fullSize() == 0)
                 continue;
 
-            updateCountersAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.updateCounter()));
+            updateCountersAndNodesByPartitions.put(part.id(), new AbstractMap.SimpleEntry<>(cctx.localNodeId(), part.updateCounter()));
         }
 
         int partitions = top.partitions();
@@ -179,9 +184,13 @@ public Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
             if (ignoringNodes.contains(nodeId))
                 continue;
 
-            CachePartitionPartialCountersMap countersMap = e.getValue().partitionUpdateCounters(top.groupId(), partitions);
+            final GridDhtPartitionsSingleMessage message = e.getValue();
 
-            Set<Integer> ignorePartitions = shouldIgnore(top, nodeId, e.getValue());
+            CachePartitionPartialCountersMap countersMap = message.partitionUpdateCountersUnsorted(top.groupId(), partitions);
+
+            Map<Integer, Long> sizesMap = message.partitionSizes(top.groupId());
+
+            Set<Integer> ignorePartitions = shouldIgnore(top, nodeId, countersMap, sizesMap);
 
             for (int i = 0; i < countersMap.size(); i++) {
                 int p = countersMap.partitionAt(i);
@@ -207,14 +216,14 @@ public Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
      * @return Invalid partitions map with following structure: (partId, (nodeId, cacheSize)).
      * If map is empty validation is successful.
      */
-     public Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
-            GridDhtPartitionTopology top,
-            Map<UUID, GridDhtPartitionsSingleMessage> messages,
-            Set<UUID> ignoringNodes
-     ) {
+    public Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
+        GridDhtPartitionTopology top,
+        Map<UUID, GridDhtPartitionsSingleMessage> messages,
+        Set<UUID> ignoringNodes
+    ) {
         Map<Integer, Map<UUID, Long>> invalidPartitions = new HashMap<>();
 
-        Map<Integer, T2<UUID, Long>> sizesAndNodesByPartitions = new HashMap<>();
+        Map<Integer, AbstractMap.Entry<UUID, Long>> sizesAndNodesByPartitions = new HashMap<>();
 
         // Populate sizes statistics from local node partitions.
         for (GridDhtLocalPartition part : top.currentLocalPartitions()) {
@@ -224,7 +233,7 @@ public Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
             if (part.updateCounter() == 0 && part.fullSize() == 0)
                 continue;
 
-            sizesAndNodesByPartitions.put(part.id(), new T2<>(cctx.localNodeId(), part.fullSize()));
+            sizesAndNodesByPartitions.put(part.id(), new AbstractMap.SimpleEntry<>(cctx.localNodeId(), part.fullSize()));
         }
 
         int partitions = top.partitions();
@@ -235,10 +244,13 @@ public Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
             if (ignoringNodes.contains(nodeId))
                 continue;
 
-            CachePartitionPartialCountersMap countersMap = e.getValue().partitionUpdateCounters(top.groupId(), partitions);
-            Map<Integer, Long> sizesMap = e.getValue().partitionSizes(top.groupId());
+            final GridDhtPartitionsSingleMessage message = e.getValue();
+
+            CachePartitionPartialCountersMap countersMap = message.partitionUpdateCountersUnsorted(top.groupId(), partitions);
+
+            Map<Integer, Long> sizesMap = message.partitionSizes(top.groupId());
 
-            Set<Integer> ignorePartitions = shouldIgnore(top, nodeId, e.getValue());
+            Set<Integer> ignorePartitions = shouldIgnore(top, nodeId, countersMap, sizesMap);
 
             for (int i = 0; i < countersMap.size(); i++) {
                 int p = countersMap.partitionAt(i);
@@ -265,20 +277,22 @@ public Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
      * @param node Node id.
      * @param counter Counter value reported by {@code node}.
      */
-    private void process(Map<Integer, Map<UUID, Long>> invalidPartitions,
-                         Map<Integer, T2<UUID, Long>> countersAndNodes,
-                         int part,
-                         UUID node,
-                         long counter) {
-        T2<UUID, Long> existingData = countersAndNodes.get(part);
+    private void process(
+        Map<Integer, Map<UUID, Long>> invalidPartitions,
+        Map<Integer, AbstractMap.Entry<UUID, Long>> countersAndNodes,
+        int part,
+        UUID node,
+        long counter
+    ) {
+        AbstractMap.Entry<UUID, Long> existingData = countersAndNodes.get(part);
 
         if (existingData == null)
-            countersAndNodes.put(part, new T2<>(node, counter));
+            countersAndNodes.put(part, new AbstractMap.SimpleEntry<>(node, counter));
 
-        if (existingData != null && counter != existingData.get2()) {
+        if (existingData != null && counter != existingData.getValue()) {
             if (!invalidPartitions.containsKey(part)) {
                 Map<UUID, Long> map = new HashMap<>();
-                map.put(existingData.get1(), existingData.get2());
+                map.put(existingData.getKey(), existingData.getValue());
                 invalidPartitions.put(part, map);
             }
 

From 51ca1b645243d8b0b3f684757c0eb7b5484e7ab8 Mon Sep 17 00:00:00 2001
From: ascherbakoff <alexey.scherbakoff@gmail.com>
Date: Thu, 18 Oct 2018 18:18:35 +0300
Subject: [PATCH 435/543] GG-14338 Backport IGNITE-8752 to 2.5.1-X

---
 .../internal/UnregisteredClassException.java  | 74 +++++++++++++++++++
 .../ignite/internal/binary/BinaryContext.java | 20 +++--
 .../internal/binary/BinaryEnumObjectImpl.java |  2 +-
 .../internal/binary/BinaryMarshaller.java     |  2 +-
 .../internal/binary/BinaryReaderExImpl.java   |  2 +-
 .../ignite/internal/binary/BinaryUtils.java   |  4 +-
 .../internal/binary/BinaryWriterExImpl.java   | 28 +++++--
 .../internal/binary/GridBinaryMarshaller.java |  5 +-
 .../binary/builder/BinaryBuilderEnum.java     |  2 +-
 .../builder/BinaryBuilderSerializer.java      |  2 +-
 .../builder/BinaryEnumArrayLazyValue.java     |  2 +-
 .../builder/BinaryObjectArrayLazyValue.java   |  2 +-
 .../builder/BinaryObjectBuilderImpl.java      |  2 +-
 .../client/thin/ClientBinaryMarshaller.java   |  2 +-
 .../CacheDefaultBinaryAffinityKeyMapper.java  |  2 +-
 .../processors/cache/CacheGroupContext.java   | 10 +++
 .../processors/cache/GridCacheContext.java    |  2 +-
 .../processors/cache/GridCacheReturn.java     |  5 ++
 .../binary/CacheObjectBinaryProcessor.java    |  3 +-
 .../CacheObjectBinaryProcessorImpl.java       | 43 ++++++-----
 .../cache/binary/IgniteBinaryImpl.java        |  2 +-
 .../dht/atomic/GridDhtAtomicCache.java        | 56 +++++++++-----
 .../topology/GridClientPartitionTopology.java |  5 ++
 .../topology/GridDhtPartitionTopology.java    |  4 +
 .../GridDhtPartitionTopologyImpl.java         |  5 ++
 .../IgniteCacheObjectProcessor.java           | 11 +++
 .../IgniteCacheObjectProcessorImpl.java       |  6 ++
 .../util/StripedCompositeReadWriteLock.java   | 25 ++++++-
 .../IgniteCacheEntryProcessorCallTest.java    |  5 ++
 29 files changed, 266 insertions(+), 67 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/UnregisteredClassException.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/UnregisteredClassException.java b/modules/core/src/main/java/org/apache/ignite/internal/UnregisteredClassException.java
new file mode 100644
index 0000000000000..6da7daad5941d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/UnregisteredClassException.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.IgniteException;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Exception thrown during serialization if class isn't registered and it's registration isn't allowed.
+ */
+public class UnregisteredClassException extends IgniteException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final Class cls;
+
+    /**
+     * @param cls Class that isn't registered.
+     */
+    public UnregisteredClassException(Class cls) {
+        this.cls = cls;
+    }
+
+    /**
+     * @param msg Error message.
+     * @param cls Class that isn't registered.
+     */
+    public UnregisteredClassException(String msg, Class cls) {
+        super(msg);
+        this.cls = cls;
+    }
+
+    /**
+     * @param cause Exception cause.
+     * @param cls Class that isn't registered.
+     */
+    public UnregisteredClassException(Throwable cause, Class cls) {
+        super(cause);
+        this.cls = cls;
+    }
+
+    /**
+     * @param msg Error message.
+     * @param cause Exception cause.
+     * @param cls Class that isn't registered.
+     */
+    public UnregisteredClassException(String msg, @Nullable Throwable cause, Class cls) {
+        super(msg, cause);
+        this.cls = cls;
+    }
+
+    /**
+     * @return Class that isn't registered.
+     */
+    public Class cls() {
+        return cls;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 233769660c9f9..01215703f4e2c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -48,6 +48,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.UnregisteredClassException;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.binary.BinaryBasicIdMapper;
 import org.apache.ignite.binary.BinaryBasicNameMapper;
@@ -610,17 +611,22 @@ else if (cpElement.isFile()) {
 
     /**
      * @param cls Class.
+     * @param failIfUnregistered Throw exception if class isn't registered.
      * @return Class descriptor.
      * @throws BinaryObjectException In case of error.
      */
-    public BinaryClassDescriptor descriptorForClass(Class<?> cls, boolean deserialize)
+    public BinaryClassDescriptor descriptorForClass(Class<?> cls, boolean deserialize, boolean failIfUnregistered)
         throws BinaryObjectException {
         assert cls != null;
 
         BinaryClassDescriptor desc = descByCls.get(cls);
 
-        if (desc == null)
+        if (desc == null) {
+            if (failIfUnregistered)
+                throw new UnregisteredClassException(cls);
+
             desc = registerClassDescriptor(cls, deserialize);
+        }
         else if (!desc.registered()) {
             if (!desc.userType()) {
                 BinaryClassDescriptor desc0 = new BinaryClassDescriptor(
@@ -652,8 +658,12 @@ else if (!desc.registered()) {
                     return desc0;
                 }
             }
-            else
+            else {
+                if (failIfUnregistered)
+                    throw new UnregisteredClassException(cls);
+
                 desc = registerUserClassDescriptor(desc);
+            }
         }
 
         return desc;
@@ -1176,8 +1186,8 @@ public void registerUserTypesSchema() {
     /**
      * Register "type ID to class name" mapping on all nodes to allow for mapping requests resolution form client.
      * Other {@link BinaryContext}'s "register" methods and method
-     * {@link BinaryContext#descriptorForClass(Class, boolean)} already call this functionality so use this method
-     * only when registering class names whose {@link Class} is unknown.
+     * {@link BinaryContext#descriptorForClass(Class, boolean, boolean)} already call this functionality
+     * so use this method only when registering class names whose {@link Class} is unknown.
      *
      * @param typeId Type ID.
      * @param clsName Class Name.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
index 12a0fc352b99d..275169561fd56 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryEnumObjectImpl.java
@@ -437,6 +437,6 @@ public BinaryEnumObjectImpl(BinaryContext ctx, byte[] arr) {
      * binary enum.
      */
     public boolean isTypeEquals(final Class<?> cls) {
-        return ctx.descriptorForClass(cls, false).typeId() == typeId();
+        return ctx.descriptorForClass(cls, false, false).typeId() == typeId();
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMarshaller.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMarshaller.java
index dfc726e81e8d3..bfb0e1018a91c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMarshaller.java
@@ -79,7 +79,7 @@ private void setBinaryContext(BinaryContext ctx, IgniteConfiguration cfg) {
 
     /** {@inheritDoc} */
     @Override protected byte[] marshal0(@Nullable Object obj) throws IgniteCheckedException {
-        return impl.marshal(obj);
+        return impl.marshal(obj, false);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
index f88e3c3234b41..ab1f874386dd9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
@@ -265,7 +265,7 @@ public BinaryReaderExImpl(BinaryContext ctx,
 
                 if (forUnmarshal) {
                     // Registers class by type ID, at least locally if the cache is not ready yet.
-                    desc = ctx.descriptorForClass(BinaryUtils.doReadClass(in, ctx, ldr, typeId0), false);
+                    desc = ctx.descriptorForClass(BinaryUtils.doReadClass(in, ctx, ldr, typeId0), false, false);
 
                     typeId = desc.typeId();
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 1f167f5f08080..284a4a5448e00 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -1640,7 +1640,7 @@ public static Class doReadClass(BinaryInputStream in, BinaryContext ctx, ClassLo
             }
 
             // forces registering of class by type id, at least locally
-            ctx.descriptorForClass(cls, true);
+            ctx.descriptorForClass(cls, true, false);
         }
 
         return cls;
@@ -1670,7 +1670,7 @@ public static Class resolveClass(BinaryContext ctx, int typeId, @Nullable String
             }
 
             // forces registering of class by type id, at least locally
-            ctx.descriptorForClass(cls, true);
+            ctx.descriptorForClass(cls, true, false);
         }
 
         return cls;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index a7f645c1f4763..3d93e7088af62 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -82,6 +82,9 @@ public class BinaryWriterExImpl implements BinaryWriter, BinaryRawWriterEx, Obje
     /** */
     private BinaryInternalMapper mapper;
 
+    /** */
+    private boolean failIfUnregistered;
+
     /**
      * @param ctx Context.
      */
@@ -112,6 +115,13 @@ public BinaryWriterExImpl(BinaryContext ctx, BinaryOutputStream out, BinaryWrite
         start = out.position();
     }
 
+    /**
+     * @param failIfUnregistered Fail if unregistered.
+     */
+    public void failIfUnregistered(boolean failIfUnregistered) {
+        this.failIfUnregistered = failIfUnregistered;
+    }
+
     /**
      * @param typeId Type ID.
      */
@@ -161,7 +171,7 @@ private void marshal0(Object obj, boolean enableReplace) throws BinaryObjectExce
 
         Class<?> cls = obj.getClass();
 
-        BinaryClassDescriptor desc = ctx.descriptorForClass(cls, false);
+        BinaryClassDescriptor desc = ctx.descriptorForClass(cls, false, failIfUnregistered);
 
         if (desc == null)
             throw new BinaryObjectException("Object is not binary: [class=" + cls + ']');
@@ -724,7 +734,10 @@ void doWriteObjectArray(@Nullable Object[] val) throws BinaryObjectException {
             if (tryWriteAsHandle(val))
                 return;
 
-            BinaryClassDescriptor desc = ctx.descriptorForClass(val.getClass().getComponentType(), false);
+            BinaryClassDescriptor desc = ctx.descriptorForClass(
+                val.getClass().getComponentType(),
+                false,
+                failIfUnregistered);
 
             out.unsafeEnsure(1 + 4);
             out.unsafeWriteByte(GridBinaryMarshaller.OBJ_ARR);
@@ -795,7 +808,7 @@ void doWriteEnum(@Nullable Enum<?> val) {
         if (val == null)
             out.writeByte(GridBinaryMarshaller.NULL);
         else {
-            BinaryClassDescriptor desc = ctx.descriptorForClass(val.getDeclaringClass(), false);
+            BinaryClassDescriptor desc = ctx.descriptorForClass(val.getDeclaringClass(), false, failIfUnregistered);
 
             out.unsafeEnsure(1 + 4);
 
@@ -848,7 +861,10 @@ void doWriteEnumArray(@Nullable Object[] val) {
         if (val == null)
             out.writeByte(GridBinaryMarshaller.NULL);
         else {
-            BinaryClassDescriptor desc = ctx.descriptorForClass(val.getClass().getComponentType(), false);
+            BinaryClassDescriptor desc = ctx.descriptorForClass(
+                val.getClass().getComponentType(),
+                false,
+                failIfUnregistered);
 
             out.unsafeEnsure(1 + 4);
 
@@ -877,7 +893,7 @@ void doWriteClass(@Nullable Class val) {
         if (val == null)
             out.writeByte(GridBinaryMarshaller.NULL);
         else {
-            BinaryClassDescriptor desc = ctx.descriptorForClass(val, false);
+            BinaryClassDescriptor desc = ctx.descriptorForClass(val, false, failIfUnregistered);
 
             out.unsafeEnsure(1 + 4);
 
@@ -906,7 +922,7 @@ public void doWriteProxy(Proxy proxy, Class<?>[] intfs) {
             out.unsafeWriteInt(intfs.length);
 
             for (Class<?> intf : intfs) {
-                BinaryClassDescriptor desc = ctx.descriptorForClass(intf, false);
+                BinaryClassDescriptor desc = ctx.descriptorForClass(intf, false, failIfUnregistered);
 
                 if (desc.registered())
                     out.writeInt(desc.typeId());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java
index d6c8abdcfb35a..743958932f21b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/GridBinaryMarshaller.java
@@ -240,14 +240,17 @@ public GridBinaryMarshaller(BinaryContext ctx) {
 
     /**
      * @param obj Object to marshal.
+     * @param failIfUnregistered Throw exception if class isn't registered.
      * @return Byte array.
      * @throws org.apache.ignite.binary.BinaryObjectException In case of error.
      */
-    public byte[] marshal(@Nullable Object obj) throws BinaryObjectException {
+    public byte[] marshal(@Nullable Object obj, boolean failIfUnregistered) throws BinaryObjectException {
         if (obj == null)
             return new byte[] { NULL };
 
         try (BinaryWriterExImpl writer = new BinaryWriterExImpl(ctx)) {
+            writer.failIfUnregistered(failIfUnregistered);
+
             writer.marshal(obj);
 
             return writer.array();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderEnum.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderEnum.java
index bc5eb9e030b35..3930c463528e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderEnum.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderEnum.java
@@ -63,7 +63,7 @@ public BinaryBuilderEnum(BinaryBuilderReader reader) {
                 throw new BinaryInvalidTypeException("Failed to load the class: " + clsName, e);
             }
 
-            this.typeId = reader.binaryContext().descriptorForClass(cls, false).typeId();
+            this.typeId = reader.binaryContext().descriptorForClass(cls, false, false).typeId();
         }
         else {
             this.typeId = typeId;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
index 018444c65123a..b9b624a32b411 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
@@ -128,7 +128,7 @@ public void writeValue(BinaryWriterExImpl writer, Object val, boolean forceCol,
             writer.context().updateMetadata(typeId, meta);
 
             // Need register class for marshaller to be able to deserialize enum value.
-            writer.context().descriptorForClass(((Enum)val).getDeclaringClass(), false);
+            writer.context().descriptorForClass(((Enum)val).getDeclaringClass(), false, false);
 
             writer.writeByte(GridBinaryMarshaller.ENUM);
             writer.writeInt(typeId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java
index 787ff638b995e..c0e79ec760594 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryEnumArrayLazyValue.java
@@ -56,7 +56,7 @@ protected BinaryEnumArrayLazyValue(BinaryBuilderReader reader) {
                 throw new BinaryInvalidTypeException("Failed to load the class: " + clsName, e);
             }
 
-            compTypeId = reader.binaryContext().descriptorForClass(cls, true).typeId();
+            compTypeId = reader.binaryContext().descriptorForClass(cls, true, false).typeId();
         }
         else {
             compTypeId = typeId;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java
index 8962107c77ac8..d4882dc6fb462 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectArrayLazyValue.java
@@ -55,7 +55,7 @@ protected BinaryObjectArrayLazyValue(BinaryBuilderReader reader) {
                 throw new BinaryInvalidTypeException("Failed to load the class: " + clsName, e);
             }
 
-            compTypeId = reader.binaryContext().descriptorForClass(cls, true).typeId();
+            compTypeId = reader.binaryContext().descriptorForClass(cls, true, false).typeId();
         }
         else {
             compTypeId = typeId;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
index b9eb3e5e0267b..3fc5dc4bee0a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
@@ -157,7 +157,7 @@ public BinaryObjectBuilderImpl(BinaryObjectImpl obj) {
                 throw new BinaryInvalidTypeException("Failed to load the class: " + clsNameToWrite, e);
             }
 
-            this.typeId = ctx.descriptorForClass(cls, false).typeId();
+            this.typeId = ctx.descriptorForClass(cls, false, false).typeId();
 
             registeredType = false;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientBinaryMarshaller.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientBinaryMarshaller.java
index c68b8f909e3fa..aac6873e95fad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientBinaryMarshaller.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientBinaryMarshaller.java
@@ -66,7 +66,7 @@ public <T> T unmarshal(BinaryInputStream in) {
      * Serializes Java object into a byte array.
      */
     public byte[] marshal(Object obj) {
-        return impl.marshal(obj);
+        return impl.marshal(obj, false);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheDefaultBinaryAffinityKeyMapper.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheDefaultBinaryAffinityKeyMapper.java
index 43506873fb427..385ed59cfd36a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheDefaultBinaryAffinityKeyMapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheDefaultBinaryAffinityKeyMapper.java
@@ -63,7 +63,7 @@ public CacheDefaultBinaryAffinityKeyMapper(@Nullable CacheKeyConfiguration[] cac
     /** {@inheritDoc} */
     @Override public Object affinityKey(Object key) {
         try {
-            key = proc.toBinary(key);
+            key = proc.toBinary(key, false);
         }
         catch (IgniteException e) {
             U.error(log, "Failed to marshal key to binary: " + key, e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index a7347498e04c5..39546a715490f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -578,6 +578,16 @@ public GridDhtPartitionTopology topology() {
         return top;
     }
 
+    /**
+     * @return {@code True} if current thread holds lock on topology.
+     */
+    public boolean isTopologyLocked() {
+        if (top == null)
+            return false;
+
+        return top.holdsLock();
+    }
+
     /**
      * @return Offheap manager.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index fe96a376ad35b..751e262f0ca7a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -1799,7 +1799,7 @@ public CacheObjectContext cacheObjectContext() {
     @Nullable public CacheObject toCacheObject(@Nullable Object obj) {
         assert validObjectForCache(obj) : obj;
 
-        return cacheObjects().toCacheObject(cacheObjCtx, obj, true);
+        return cacheObjects().toCacheObject(cacheObjCtx, obj, true, grp.isTopologyLocked());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
index 551d70d83761d..bc859312b371e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.internal.GridDirectCollection;
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.UnregisteredClassException;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -242,6 +243,10 @@ public synchronized void addEntryProcessResult(
                 v = resMap;
             }
 
+            // This exception means that we should register class and call EntryProcessor again.
+            if (err != null && err instanceof UnregisteredClassException)
+                throw (UnregisteredClassException) err;
+
             CacheInvokeResult res0 = err == null ? CacheInvokeResult.fromResult(res) : CacheInvokeResult.fromError(err);
 
             Object resKey = key0 != null ? key0 :
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessor.java
index 14dd5cbc9a35b..c7e2e689f7c06 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessor.java
@@ -140,8 +140,9 @@ public void updateMetadata(int typeId, String typeName, @Nullable String affKeyF
 
     /**
      * @param obj Original object.
+     * @param failIfUnregistered Throw exception if class isn't registered.
      * @return Binary object (in case binary marshaller is used).
      * @throws IgniteException If failed.
      */
-    public Object marshalToBinary(Object obj) throws IgniteException;
+    public Object marshalToBinary(Object obj, boolean failIfUnregistered) throws IgniteException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 2d515764b9d94..f5ffb2a3d15a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -295,7 +295,7 @@ public void addBinaryMetadataUpdateListener(BinaryMetadataUpdatedListener lsnr)
      * @throws BinaryObjectException If failed.
      */
     public byte[] marshal(@Nullable Object obj) throws BinaryObjectException {
-        byte[] arr = binaryMarsh.marshal(obj);
+        byte[] arr = binaryMarsh.marshal(obj, false);
 
         assert arr.length > 0;
 
@@ -330,7 +330,10 @@ public Object unmarshal(long ptr, boolean forceHeap) throws BinaryObjectExceptio
 
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
-    @Override public Object marshalToBinary(@Nullable Object obj) throws BinaryObjectException {
+    @Override public Object marshalToBinary(
+        @Nullable Object obj,
+        boolean failIfUnregistered
+    ) throws BinaryObjectException {
         if (obj == null)
             return null;
 
@@ -343,7 +346,7 @@ public Object unmarshal(long ptr, boolean forceHeap) throws BinaryObjectExceptio
             Object[] pArr = new Object[arr.length];
 
             for (int i = 0; i < arr.length; i++)
-                pArr[i] = marshalToBinary(arr[i]);
+                pArr[i] = marshalToBinary(arr[i], failIfUnregistered);
 
             return pArr;
         }
@@ -352,9 +355,11 @@ public Object unmarshal(long ptr, boolean forceHeap) throws BinaryObjectExceptio
             IgniteBiTuple tup = (IgniteBiTuple)obj;
 
             if (obj instanceof T2)
-                return new T2<>(marshalToBinary(tup.get1()), marshalToBinary(tup.get2()));
+                return new T2<>(marshalToBinary(tup.get1(), failIfUnregistered),
+                    marshalToBinary(tup.get2(), failIfUnregistered));
 
-            return new IgniteBiTuple<>(marshalToBinary(tup.get1()), marshalToBinary(tup.get2()));
+            return new IgniteBiTuple<>(marshalToBinary(tup.get1(), failIfUnregistered),
+                marshalToBinary(tup.get2(), failIfUnregistered));
         }
 
         {
@@ -364,7 +369,7 @@ public Object unmarshal(long ptr, boolean forceHeap) throws BinaryObjectExceptio
                 Collection<?> col = (Collection<?>)obj;
 
                 for (Object item : col)
-                    pCol.add(marshalToBinary(item));
+                    pCol.add(marshalToBinary(item, failIfUnregistered));
 
                 return (pCol instanceof MutableSingletonList) ? U.convertToSingletonList(pCol) : pCol;
             }
@@ -377,7 +382,8 @@ public Object unmarshal(long ptr, boolean forceHeap) throws BinaryObjectExceptio
                 Map<?, ?> map = (Map<?, ?>)obj;
 
                 for (Map.Entry<?, ?> e : map.entrySet())
-                    pMap.put(marshalToBinary(e.getKey()), marshalToBinary(e.getValue()));
+                    pMap.put(marshalToBinary(e.getKey(), failIfUnregistered),
+                        marshalToBinary(e.getValue(), failIfUnregistered));
 
                 return pMap;
             }
@@ -386,13 +392,14 @@ public Object unmarshal(long ptr, boolean forceHeap) throws BinaryObjectExceptio
         if (obj instanceof Map.Entry) {
             Map.Entry<?, ?> e = (Map.Entry<?, ?>)obj;
 
-            return new GridMapEntry<>(marshalToBinary(e.getKey()), marshalToBinary(e.getValue()));
+            return new GridMapEntry<>(marshalToBinary(e.getKey(), failIfUnregistered),
+                marshalToBinary(e.getValue(), failIfUnregistered));
         }
 
         if (binaryMarsh.mustDeserialize(obj))
             return obj; // No need to go through marshal-unmarshal because result will be the same as initial object.
 
-        byte[] arr = binaryMarsh.marshal(obj);
+        byte[] arr = binaryMarsh.marshal(obj, failIfUnregistered);
 
         assert arr.length > 0;
 
@@ -766,7 +773,7 @@ public BinaryContext binaryContext() {
         if (!ctx.binaryEnabled() || binaryMarsh == null)
             return super.marshal(ctx, val);
 
-        byte[] arr = binaryMarsh.marshal(val);
+        byte[] arr = binaryMarsh.marshal(val, false);
 
         assert arr.length > 0;
 
@@ -802,7 +809,7 @@ else if (key.partition() == -1)
             return key;
         }
 
-        obj = toBinary(obj);
+        obj = toBinary(obj, false);
 
         if (obj instanceof BinaryObjectImpl) {
             ((BinaryObjectImpl)obj).partition(partition(ctx, cctx, obj));
@@ -815,14 +822,14 @@ else if (key.partition() == -1)
 
     /** {@inheritDoc} */
     @Nullable @Override public CacheObject toCacheObject(CacheObjectContext ctx, @Nullable Object obj,
-        boolean userObj) {
+        boolean userObj, boolean failIfUnregistered) {
         if (!ctx.binaryEnabled())
-            return super.toCacheObject(ctx, obj, userObj);
+            return super.toCacheObject(ctx, obj, userObj, failIfUnregistered);
 
         if (obj == null || obj instanceof CacheObject)
             return (CacheObject)obj;
 
-        obj = toBinary(obj);
+        obj = toBinary(obj, failIfUnregistered);
 
         if (obj instanceof CacheObject)
             return (CacheObject)obj;
@@ -865,18 +872,20 @@ else if (type == BinaryObjectImpl.TYPE_BINARY_ENUM)
      * @return Binary object.
      * @throws IgniteException In case of error.
      */
-    @Nullable public Object toBinary(@Nullable Object obj) throws IgniteException {
+    @Nullable public Object toBinary(@Nullable Object obj, boolean failIfUnregistered) throws IgniteException {
         if (obj == null)
             return null;
 
         if (isBinaryObject(obj))
             return obj;
 
-        return marshalToBinary(obj);
+        return marshalToBinary(obj, failIfUnregistered);
     }
 
     /** {@inheritDoc} */
-    @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode rmtNode, DiscoveryDataBag.JoiningNodeDiscoveryData discoData) {
+    @Nullable @Override public IgniteNodeValidationResult validateNode(ClusterNode rmtNode,
+        DiscoveryDataBag.JoiningNodeDiscoveryData discoData
+    ) {
         IgniteNodeValidationResult res;
 
         if (getBoolean(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK) || !(marsh instanceof BinaryMarshaller))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/IgniteBinaryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/IgniteBinaryImpl.java
index e88819b0d3189..71475be7b0e23 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/IgniteBinaryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/IgniteBinaryImpl.java
@@ -66,7 +66,7 @@ public IgniteBinaryImpl(GridKernalContext ctx, CacheObjectBinaryProcessor proc)
         guard();
 
         try {
-            return (T)proc.marshalToBinary(obj);
+            return (T)proc.marshalToBinary(obj, false);
         }
         finally {
             unguard();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index ad8fa16926829..81711e88fc5ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.NodeStoppingException;
+import org.apache.ignite.internal.UnregisteredClassException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
 import org.apache.ignite.internal.processors.cache.persistence.StorageException;
@@ -62,6 +63,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
@@ -84,6 +86,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionEx;
+import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.util.GridLongList;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -1708,20 +1711,22 @@ private void updateAllAsyncInternal0(
             Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted = null;
 
             try {
-                GridDhtPartitionTopology top = topology();
+                while (true) {
+                    try {
+                        GridDhtPartitionTopology top = topology();
 
-                top.readLock();
+                        top.readLock();
 
-                try {
-                    if (top.stopping()) {
-                        res.addFailedKeys(req.keys(), new CacheStoppedException(name()));
+                        try {
+                            if (top.stopping()) {
+                                res.addFailedKeys(req.keys(), new CacheStoppedException(name()));
 
-                        completionCb.apply(req, res);
+                                completionCb.apply(req, res);
 
-                        return;
-                    }
+                                return;
+                            }
 
-                    boolean remap = false;
+                            boolean remap = false;
 
                             // Do not check topology version if topology was locked on near node by
                             // external transaction or explicit lock.
@@ -1733,19 +1738,30 @@ private void updateAllAsyncInternal0(
                                         req.lastAffinityChangedTopologyVersion(), req.keys());
                             }
 
-                    if (!remap) {
-                        DhtAtomicUpdateResult updRes = update(node, locked, req, res);
+                            if (!remap) {
+                                DhtAtomicUpdateResult updRes = update(node, locked, req, res);
+
+                                dhtFut = updRes.dhtFuture();
+                                deleted = updRes.deleted();
+                                expiry = updRes.expiryPolicy();
+                            }
+                            else
+                                // Should remap all keys.
+                                res.remapTopologyVersion(top.lastTopologyChangeVersion());
+                        }
+                        finally {
+                            top.readUnlock();
+                        }
+
+                        break;
+                    }
+                    catch (UnregisteredClassException ex) {
+                        IgniteCacheObjectProcessor cacheObjProc = ctx.cacheObjects();
+
+                        assert cacheObjProc instanceof CacheObjectBinaryProcessorImpl;
 
-                        dhtFut = updRes.dhtFuture();
-                        deleted = updRes.deleted();
-                        expiry = updRes.expiryPolicy();
+                        ((CacheObjectBinaryProcessorImpl)cacheObjProc).binaryContext().descriptorForClass(ex.cls(), false, false);
                     }
-                    else
-                        // Should remap all keys.
-                        res.remapTopologyVersion(top.lastTopologyChangeVersion());
-                }
-                finally {
-                    top.readUnlock();
                 }
             }
             catch (GridCacheEntryRemovedException e) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
index 65bb657a34512..66fff02d98e82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridClientPartitionTopology.java
@@ -198,6 +198,11 @@ private String mapString(GridDhtPartitionMap map) {
         lock.readLock().unlock();
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean holdsLock() {
+        return lock.isWriteLockedByCurrentThread() || lock.getReadHoldCount() > 0;
+    }
+
     /** {@inheritDoc} */
     @Override public void updateTopologyVersion(
         GridDhtTopologyFuture exchFut,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java
index 59ed377d69657..9c17287ca4c5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopology.java
@@ -60,6 +60,10 @@ public interface GridDhtPartitionTopology {
      */
     public void readUnlock();
 
+    /**
+     * @return {@code True} if locked by current thread.
+     */
+    public boolean holdsLock();
     /**
      * Updates topology version.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
index d84cfd47d5f4c..652ce58507e78 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
@@ -234,6 +234,11 @@ private String mapString(GridDhtPartitionMap map) {
         lock.readLock().unlock();
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean holdsLock() {
+        return lock.isWriteLockedByCurrentThread() || lock.getReadHoldCount() > 0;
+    }
+
     /** {@inheritDoc} */
     @Override public void updateTopologyVersion(
         GridDhtTopologyFuture exchFut,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
index dad6728573cfc..defb3ccb2ad31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessor.java
@@ -152,6 +152,17 @@ public KeyCacheObject toCacheKeyObject(CacheObjectContext ctx,
      */
     @Nullable public CacheObject toCacheObject(CacheObjectContext ctx, @Nullable Object obj, boolean userObj);
 
+    /**
+     * @param ctx Cache context.
+     * @param obj Object.
+     * @param userObj If {@code true} then given object is object provided by user and should be copied
+     *        before stored in cache.
+     * @param failIfUnregistered Throw exception if class isn't registered.
+     * @return Cache object.
+     */
+    @Nullable public CacheObject toCacheObject(CacheObjectContext ctx, @Nullable Object obj, boolean userObj,
+        boolean failIfUnregistered);
+
     /**
      * @param ctx Cache context.
      * @param type Object type.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java
index 17be90f08cc1a..7f55614e2c593 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cacheobject/IgniteCacheObjectProcessorImpl.java
@@ -225,6 +225,12 @@ protected KeyCacheObject toCacheKeyObject0(CacheObjectContext ctx,
     @Nullable @Override public CacheObject toCacheObject(CacheObjectContext ctx,
         @Nullable Object obj,
         boolean userObj) {
+        return toCacheObject(ctx, obj, userObj, false);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override public CacheObject toCacheObject(CacheObjectContext ctx, @Nullable Object obj, boolean userObj,
+        boolean failIfUnregistered) {
         if (obj == null || obj instanceof CacheObject)
             return (CacheObject)obj;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedCompositeReadWriteLock.java b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedCompositeReadWriteLock.java
index 18ef06cdd5401..42ec39772ac72 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/StripedCompositeReadWriteLock.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/StripedCompositeReadWriteLock.java
@@ -63,8 +63,10 @@ public StripedCompositeReadWriteLock(int concurrencyLvl) {
         writeLock = new WriteLock();
     }
 
-    /** {@inheritDoc} */
-    @NotNull @Override public Lock readLock() {
+    /**
+     * @return Index of current thread stripe.
+     */
+    private int curIdx() {
         int idx;
 
         Thread curThread = Thread.currentThread();
@@ -83,7 +85,12 @@ public StripedCompositeReadWriteLock(int concurrencyLvl) {
         else
             idx = IDX.get();
 
-        return locks[idx % locks.length].readLock();
+        return idx % locks.length;
+    }
+
+    /** {@inheritDoc} */
+    @NotNull @Override public Lock readLock() {
+        return locks[curIdx()].readLock();
     }
 
     /** {@inheritDoc} */
@@ -101,6 +108,18 @@ public boolean isWriteLockedByCurrentThread() {
         return locks[locks.length - 1].isWriteLockedByCurrentThread();
     }
 
+    /**
+     * Queries the number of reentrant read holds on this lock by the
+     * current thread.  A reader thread has a hold on a lock for
+     * each lock action that is not matched by an unlock action.
+     *
+     * @return the number of holds on the read lock by the current thread,
+     *         or zero if the read lock is not held by the current thread
+     */
+    public int getReadHoldCount() {
+        return locks[curIdx()].getReadHoldCount();
+    }
+
     /**
      * Read lock.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorCallTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorCallTest.java
index cd4a78faae481..8a48ac25ffd4e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorCallTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheEntryProcessorCallTest.java
@@ -161,6 +161,11 @@ private void checkEntryProcessorCallCount(CacheConfiguration<Integer, TestValue>
 
         int key = 0;
 
+        // Call EntryProcessor on every node to ensure that binary metadata has been registered everywhere.
+        for (int i = 0; i < 1_000; i++)
+            ignite(i % SRV_CNT).<Integer, TestValue>cache(ccfg.getName())
+                .invoke(key++, new TestEntryProcessor(OP_UPDATE), new TestValue(Integer.MIN_VALUE));
+
         checkEntryProcessCall(key++, clientCache1, null, null, expCallCnt);
 
         if (ccfg.getAtomicityMode() == TRANSACTIONAL) {

From 77ac0c8a8d81464955cc024c7addbbf6098f0dc1 Mon Sep 17 00:00:00 2001
From: ascherbakoff <alexey.scherbakoff@gmail.com>
Date: Thu, 18 Oct 2018 19:37:18 +0300
Subject: [PATCH 436/543] GG-14339 Backport IGNITE-8926 to 2.5.1-X

---
 .../UnregisteredBinaryTypeException.java      |  96 +++++++++++
 .../binary/BinaryCachingMetadataHandler.java  |   2 +-
 .../binary/BinaryClassDescriptor.java         |   9 +-
 .../ignite/internal/binary/BinaryContext.java |  11 +-
 .../internal/binary/BinaryFieldAccessor.java  |   4 +
 .../binary/BinaryMetadataHandler.java         |   3 +-
 .../binary/BinaryNoopMetadataHandler.java     |   2 +-
 .../internal/binary/BinaryWriterExImpl.java   |  13 ++
 .../builder/BinaryBuilderSerializer.java      |   2 +-
 .../builder/BinaryObjectBuilderImpl.java      |   9 +-
 .../internal/client/thin/ClientBinary.java    |   2 +-
 .../internal/client/thin/TcpIgniteClient.java |   6 +-
 .../cache/CacheInvokeDirectResult.java        |  26 +++
 .../processors/cache/CacheInvokeResult.java   |   6 +
 .../processors/cache/GridCacheMapEntry.java   |  24 ++-
 .../processors/cache/GridCacheReturn.java     |  13 +-
 .../binary/CacheObjectBinaryProcessor.java    |   3 +-
 .../CacheObjectBinaryProcessorImpl.java       |  17 +-
 .../dht/GridDhtTxPrepareFuture.java           |  14 +-
 .../dht/atomic/DhtAtomicUpdateResult.java     |  33 +++-
 .../dht/atomic/GridDhtAtomicCache.java        | 143 +++++++++--------
 .../local/atomic/GridLocalAtomicCache.java    |   6 +
 .../cache/persistence/tree/BPlusTree.java     |   5 +
 .../cache/transactions/IgniteTxAdapter.java   |   6 +
 .../cache/transactions/IgniteTxEntry.java     |   6 +
 .../transactions/IgniteTxLocalAdapter.java    |   6 +
 .../platform/PlatformContextImpl.java         |   2 +-
 .../binary/ClientBinaryTypePutRequest.java    |   2 +-
 .../apache/ignite/thread/IgniteThread.java    |  45 +++++-
 .../binary/TestCachingMetadataHandler.java    |   2 +-
 ...aRegistrationInsideEntryProcessorTest.java | 141 ++++++++++++++++
 .../cache/IgniteCacheInvokeAbstractTest.java  | 150 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite.java      |   3 +
 33 files changed, 713 insertions(+), 99 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/UnregisteredBinaryTypeException.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataRegistrationInsideEntryProcessorTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/UnregisteredBinaryTypeException.java b/modules/core/src/main/java/org/apache/ignite/internal/UnregisteredBinaryTypeException.java
new file mode 100644
index 0000000000000..f46de125434fc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/UnregisteredBinaryTypeException.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.binary.BinaryMetadata;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Exception thrown during serialization if binary metadata isn't registered and it's registration isn't allowed.
+ */
+public class UnregisteredBinaryTypeException extends IgniteException {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final int typeId;
+
+    /** */
+    private final BinaryMetadata binaryMetadata;
+
+    /**
+     * @param typeId Type ID.
+     * @param binaryMetadata Binary metadata.
+     */
+    public UnregisteredBinaryTypeException(int typeId, BinaryMetadata binaryMetadata) {
+        this.typeId = typeId;
+        this.binaryMetadata = binaryMetadata;
+    }
+
+    /**
+     * @param msg Error message.
+     * @param typeId Type ID.
+     * @param binaryMetadata Binary metadata.
+     */
+    public UnregisteredBinaryTypeException(String msg, int typeId,
+        BinaryMetadata binaryMetadata) {
+        super(msg);
+        this.typeId = typeId;
+        this.binaryMetadata = binaryMetadata;
+    }
+
+    /**
+     * @param cause Non-null throwable cause.
+     * @param typeId Type ID.
+     * @param binaryMetadata Binary metadata.
+     */
+    public UnregisteredBinaryTypeException(Throwable cause, int typeId,
+        BinaryMetadata binaryMetadata) {
+        super(cause);
+        this.typeId = typeId;
+        this.binaryMetadata = binaryMetadata;
+    }
+
+    /**
+     * @param msg Error message.
+     * @param cause Non-null throwable cause.
+     * @param typeId Type ID.
+     * @param binaryMetadata Binary metadata.
+     */
+    public UnregisteredBinaryTypeException(String msg, @Nullable Throwable cause, int typeId,
+        BinaryMetadata binaryMetadata) {
+        super(msg, cause);
+        this.typeId = typeId;
+        this.binaryMetadata = binaryMetadata;
+    }
+
+    /**
+     * @return Type ID.
+     */
+    public int typeId() {
+        return typeId;
+    }
+
+    /**
+     * @return Binary metadata.
+     */
+    public BinaryMetadata binaryMetadata() {
+        return binaryMetadata;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java
index 535249c80ac18..a0559cbdc8ba1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java
@@ -46,7 +46,7 @@ private BinaryCachingMetadataHandler() {
     }
 
     /** {@inheritDoc} */
-    @Override public synchronized void addMeta(int typeId, BinaryType type) throws BinaryObjectException {
+    @Override public synchronized void addMeta(int typeId, BinaryType type, boolean failIfUnregistered) throws BinaryObjectException {
         synchronized (this) {
             BinaryType oldType = metas.put(typeId, type);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
index 106d238660f2f..cd32120007179 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryClassDescriptor.java
@@ -40,6 +40,8 @@
 import org.apache.ignite.binary.BinaryReflectiveSerializer;
 import org.apache.ignite.binary.BinarySerializer;
 import org.apache.ignite.binary.Binarylizable;
+import org.apache.ignite.internal.UnregisteredClassException;
+import org.apache.ignite.internal.UnregisteredBinaryTypeException;
 import org.apache.ignite.internal.marshaller.optimized.OptimizedMarshaller;
 import org.apache.ignite.internal.processors.cache.CacheObjectImpl;
 import org.apache.ignite.internal.processors.query.QueryUtils;
@@ -773,7 +775,7 @@ void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
                                     BinaryMetadata meta = new BinaryMetadata(typeId, typeName, collector.meta(),
                                         affKeyFieldName, Collections.singleton(newSchema), false, null);
 
-                                    ctx.updateMetadata(typeId, meta);
+                                    ctx.updateMetadata(typeId, meta, writer.failIfUnregistered());
 
                                     schemaReg.addSchema(newSchema.schemaId(), newSchema);
                                 }
@@ -794,7 +796,7 @@ void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
                         BinaryMetadata meta = new BinaryMetadata(typeId, typeName, stableFieldsMeta,
                             affKeyFieldName, Collections.singleton(stableSchema), false, null);
 
-                        ctx.updateMetadata(typeId, meta);
+                        ctx.updateMetadata(typeId, meta, writer.failIfUnregistered());
 
                         schemaReg.addSchema(stableSchema.schemaId(), stableSchema);
 
@@ -823,6 +825,9 @@ void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectException {
             }
         }
         catch (Exception e) {
+            if (e instanceof UnregisteredBinaryTypeException || e instanceof UnregisteredClassException)
+                throw e;
+
             String msg;
 
             if (S.INCLUDE_SENSITIVE && !F.isEmpty(typeName))
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 01215703f4e2c..7885d9575f32b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -653,7 +653,7 @@ else if (!desc.registered()) {
                         schemas, desc0.isEnum(),
                         cls.isEnum() ? enumMap(cls) : null);
 
-                    metaHnd.addMeta(desc0.typeId(), meta.wrap(this));
+                    metaHnd.addMeta(desc0.typeId(), meta.wrap(this), false);
 
                     return desc0;
                 }
@@ -801,7 +801,7 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean
 
         if (!deserialize)
             metaHnd.addMeta(typeId, new BinaryMetadata(typeId, typeName, desc.fieldsMeta(), affFieldName, null,
-                desc.isEnum(), cls.isEnum() ? enumMap(cls) : null).wrap(this));
+                desc.isEnum(), cls.isEnum() ? enumMap(cls) : null).wrap(this), false);
 
         descByCls.put(cls, desc);
 
@@ -1170,7 +1170,7 @@ public void registerUserType(String clsName,
         }
 
         metaHnd.addMeta(id,
-            new BinaryMetadata(id, typeName, fieldsMeta, affKeyFieldName, null, isEnum, enumMap).wrap(this));
+            new BinaryMetadata(id, typeName, fieldsMeta, affKeyFieldName, null, isEnum, enumMap).wrap(this), false);
     }
 
     /**
@@ -1325,10 +1325,11 @@ public BinaryIdentityResolver identity(int typeId) {
     /**
      * @param typeId Type ID.
      * @param meta Meta data.
+     * @param failIfUnregistered Fail if unregistered.
      * @throws BinaryObjectException In case of error.
      */
-    public void updateMetadata(int typeId, BinaryMetadata meta) throws BinaryObjectException {
-        metaHnd.addMeta(typeId, meta.wrap(this));
+    public void updateMetadata(int typeId, BinaryMetadata meta, boolean failIfUnregistered) throws BinaryObjectException {
+        metaHnd.addMeta(typeId, meta.wrap(this), failIfUnregistered);
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
index 32774035671f7..87c4f3e18d8fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryFieldAccessor.java
@@ -26,6 +26,7 @@
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.internal.UnregisteredClassException;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -155,6 +156,9 @@ public void write(Object obj, BinaryWriterExImpl writer) throws BinaryObjectExce
             write0(obj, writer);
         }
         catch (Exception ex) {
+            if (ex instanceof UnregisteredClassException)
+                throw ex;
+
             if (S.INCLUDE_SENSITIVE && !F.isEmpty(name))
                 throw new BinaryObjectException("Failed to write field [name=" + name + ']', ex);
             else
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java
index 5df32e7216ac7..85ab1372f49f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java
@@ -30,9 +30,10 @@ public interface BinaryMetadataHandler {
      *
      * @param typeId Type ID.
      * @param meta Metadata.
+     * @param failIfUnregistered Fail if unregistered.
      * @throws BinaryObjectException In case of error.
      */
-    public void addMeta(int typeId, BinaryType meta) throws BinaryObjectException;
+    public void addMeta(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException;
 
     /**
      * Gets meta data for provided type ID.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java
index bbd931110e30a..4ee24285c7ee5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java
@@ -43,7 +43,7 @@ private BinaryNoopMetadataHandler() {
     }
 
     /** {@inheritDoc} */
-    @Override public void addMeta(int typeId, BinaryType meta) throws BinaryObjectException {
+    @Override public void addMeta(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException {
         // No-op.
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index 3d93e7088af62..e6efb0c509a52 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -115,6 +115,13 @@ public BinaryWriterExImpl(BinaryContext ctx, BinaryOutputStream out, BinaryWrite
         start = out.position();
     }
 
+    /**
+     * @return Fail if unregistered flag value.
+     */
+    public boolean failIfUnregistered() {
+        return failIfUnregistered;
+    }
+
     /**
      * @param failIfUnregistered Fail if unregistered.
      */
@@ -503,6 +510,8 @@ public void doWriteObject(@Nullable Object obj) throws BinaryObjectException {
         else {
             BinaryWriterExImpl writer = new BinaryWriterExImpl(ctx, out, schema, handles());
 
+            writer.failIfUnregistered(failIfUnregistered);
+
             writer.marshal(obj);
         }
     }
@@ -1492,6 +1501,8 @@ void writeBinaryObjectField(@Nullable BinaryObjectImpl po) throws BinaryObjectEx
         else {
             BinaryWriterExImpl writer = new BinaryWriterExImpl(ctx, out, schema, null);
 
+            writer.failIfUnregistered(failIfUnregistered);
+
             writer.marshal(obj);
         }
     }
@@ -1915,6 +1926,8 @@ boolean tryWriteAsHandle(Object obj) {
     public BinaryWriterExImpl newWriter(int typeId) {
         BinaryWriterExImpl res = new BinaryWriterExImpl(ctx, out, schema, handles());
 
+        res.failIfUnregistered(failIfUnregistered);
+
         res.typeId(typeId);
 
         return res;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
index b9b624a32b411..2d769ab9f39c4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryBuilderSerializer.java
@@ -125,7 +125,7 @@ public void writeValue(BinaryWriterExImpl writer, Object val, boolean forceCol,
 
             BinaryMetadata meta = new BinaryMetadata(typeId, typeName, null, null, null, true, enumMap);
 
-            writer.context().updateMetadata(typeId, meta);
+            writer.context().updateMetadata(typeId, meta, writer.failIfUnregistered());
 
             // Need register class for marshaller to be able to deserialize enum value.
             writer.context().descriptorForClass(((Enum)val).getDeclaringClass(), false, false);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
index 3fc5dc4bee0a8..8fffd71c1b230 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.Collection;
@@ -174,6 +175,12 @@ public BinaryObjectBuilderImpl(BinaryObjectImpl obj) {
     /** {@inheritDoc} */
     @Override public BinaryObject build() {
         try (BinaryWriterExImpl writer = new BinaryWriterExImpl(ctx)) {
+            Thread curThread = Thread.currentThread();
+
+            if (curThread instanceof IgniteThread)
+                writer.failIfUnregistered(((IgniteThread)curThread).executingEntryProcessor() &&
+                    ((IgniteThread)curThread).holdsTopLock());
+
             writer.typeId(typeId);
 
             BinaryBuilderSerializer serializationCtx = new BinaryBuilderSerializer();
@@ -360,7 +367,7 @@ else if (readCache == null) {
                 ctx.registerUserClassName(typeId, typeName);
 
                 ctx.updateMetadata(typeId, new BinaryMetadata(typeId, typeName, fieldsMeta, affFieldName0,
-                    Collections.singleton(curSchema), false, null));
+                    Collections.singleton(curSchema), false, null), writer.failIfUnregistered());
 
                 schemaReg.addSchema(curSchema.schemaId(), curSchema);
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientBinary.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientBinary.java
index 8525f5edd37aa..4164532848bc6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientBinary.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/ClientBinary.java
@@ -160,7 +160,7 @@ class ClientBinary implements IgniteBinary {
 
         int typeId = ctx.typeId(typeName);
 
-        ctx.updateMetadata(typeId, new BinaryMetadata(typeId, typeName, null, null, null, true, vals));
+        ctx.updateMetadata(typeId, new BinaryMetadata(typeId, typeName, null, null, null, true, vals), false);
 
         return ctx.metadata(typeId);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java
index 7beeb799cbf79..7a0bc7a1f8160 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java
@@ -233,7 +233,7 @@ private class ClientBinaryMetadataHandler implements BinaryMetadataHandler {
         private final BinaryMetadataHandler cache = BinaryCachingMetadataHandler.create();
 
         /** {@inheritDoc} */
-        @Override public void addMeta(int typeId, BinaryType meta) throws BinaryObjectException {
+        @Override public void addMeta(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException {
             if (cache.metadata(typeId) == null) {
                 try {
                     ch.request(
@@ -246,7 +246,7 @@ private class ClientBinaryMetadataHandler implements BinaryMetadataHandler {
                 }
             }
 
-            cache.addMeta(typeId, meta); // merge
+            cache.addMeta(typeId, meta, failIfUnregistered); // merge
         }
 
         /** {@inheritDoc} */
@@ -259,7 +259,7 @@ private class ClientBinaryMetadataHandler implements BinaryMetadataHandler {
                 if (meta0 != null) {
                     meta = new BinaryTypeImpl(marsh.context(), meta0);
 
-                    cache.addMeta(typeId, meta);
+                    cache.addMeta(typeId, meta, false);
                 }
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
index 17f304eec809d..89a0a0ff7c000 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
@@ -40,6 +40,10 @@ public class CacheInvokeDirectResult implements Message {
     /** */
     private KeyCacheObject key;
 
+    /** */
+    @GridToStringInclude
+    private transient Object unprepareRes;
+
     /** */
     @GridToStringInclude
     private CacheObject res;
@@ -68,6 +72,22 @@ public CacheInvokeDirectResult(KeyCacheObject key, CacheObject res) {
         this.res = res;
     }
 
+    /**
+     * Constructs CacheInvokeDirectResult with unprepared res, to avoid object marshaling while holding topology locks.
+     *
+     * @param key Key.
+     * @param res Result.
+     * @return a new instance of CacheInvokeDirectResult.
+     */
+    static CacheInvokeDirectResult lazyResult(KeyCacheObject key, Object res) {
+        CacheInvokeDirectResult res0 = new CacheInvokeDirectResult();
+
+        res0.key = key;
+        res0.unprepareRes = res;
+
+        return res0;
+    }
+
     /**
      * @param key Key.
      * @param err Exception thrown by {@link EntryProcessor#process(MutableEntry, Object...)}.
@@ -120,6 +140,12 @@ public void prepareMarshal(GridCacheContext ctx) throws IgniteCheckedException {
             }
         }
 
+        if (unprepareRes != null) {
+            res = ctx.toCacheObject(unprepareRes);
+
+            unprepareRes = null;
+        }
+
         if (res != null)
             res.prepareMarshal(ctx.cacheObjectContext());
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeResult.java
index b51c136fa41b7..2e6d64a69cee9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeResult.java
@@ -25,6 +25,9 @@
 import javax.cache.processor.EntryProcessorException;
 import javax.cache.processor.EntryProcessorResult;
 import javax.cache.processor.MutableEntry;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.UnregisteredBinaryTypeException;
+import org.apache.ignite.internal.UnregisteredClassException;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
@@ -96,6 +99,9 @@ public static <T> CacheInvokeResult<T> fromError(Exception err) {
     /** {@inheritDoc} */
     @Override public T get() throws EntryProcessorException {
         if (err != null) {
+            if (err instanceof UnregisteredClassException || err instanceof UnregisteredBinaryTypeException)
+                throw (IgniteException) err;
+
             if (err instanceof EntryProcessorException)
                 throw (EntryProcessorException)err;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 3c51a7123bdfc..4d39e155eb933 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -34,6 +34,8 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.eviction.EvictableEntry;
+import org.apache.ignite.internal.UnregisteredBinaryTypeException;
+import org.apache.ignite.internal.UnregisteredClassException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
@@ -78,6 +80,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.events.EventType.EVT_CACHE_OBJECT_EXPIRED;
@@ -1482,6 +1485,8 @@ else if (ttl == CU.TTL_NOT_CHANGED)
 
                 CacheInvokeEntry<Object, Object> entry = new CacheInvokeEntry<>(key, old, version(), keepBinary, this);
 
+                IgniteThread.onEntryProcessorEntered(false);
+
                 try {
                     Object computed = entryProcessor.process(entry, invokeArgs);
 
@@ -1504,6 +1509,9 @@ else if (ttl == CU.TTL_NOT_CHANGED)
 
                     invokeRes = CacheInvokeResult.fromError(e);
                 }
+                finally {
+                    IgniteThread.onEntryProcessorLeft();
+                }
 
                 if (!entry.modified()) {
                     if (expiryPlc != null && !readFromStore && hasValueUnlocked())
@@ -1796,6 +1804,8 @@ else if (ttl != CU.TTL_ZERO)
                             CacheInvokeEntry<Object, Object> entry =
                                 new CacheInvokeEntry<>(key, prevVal, version(), keepBinary, this);
 
+                            IgniteThread.onEntryProcessorEntered(true);
+
                             try {
                                 entryProcessor.process(entry, invokeArgs);
 
@@ -1805,6 +1815,9 @@ else if (ttl != CU.TTL_ZERO)
                             catch (Exception ignore) {
                                 evtVal = prevVal;
                             }
+                            finally {
+                                IgniteThread.onEntryProcessorLeft();
+                            }
                         }
                         else
                             evtVal = (CacheObject)writeObj;
@@ -3116,7 +3129,8 @@ private GridCacheVersion nextVersion() {
             GridCacheMvcc mvcc = mvccExtras();
 
             return mvcc != null && mvcc.isLocallyOwnedByIdOrThread(lockVer, threadId);
-        } finally {
+        }
+        finally {
             unlockEntry();
         }
     }
@@ -5247,6 +5261,8 @@ private void versionCheck(@Nullable IgniteBiTuple<Object, Exception> invokeRes)
         private IgniteBiTuple<Object, Exception> runEntryProcessor(CacheInvokeEntry<Object, Object> invokeEntry) {
             EntryProcessor<Object, Object, ?> entryProcessor = (EntryProcessor<Object, Object, ?>)writeObj;
 
+            IgniteThread.onEntryProcessorEntered(true);
+
             try {
                 Object computed = entryProcessor.process(invokeEntry, invokeArgs);
 
@@ -5264,10 +5280,16 @@ private IgniteBiTuple<Object, Exception> runEntryProcessor(CacheInvokeEntry<Obje
                 return null;
             }
             catch (Exception e) {
+                if (e instanceof UnregisteredClassException || e instanceof UnregisteredBinaryTypeException)
+                    throw (IgniteException) e;
+
                 writeObj = invokeEntry.valObj;
 
                 return new IgniteBiTuple<>(null, e);
             }
+            finally {
+                IgniteThread.onEntryProcessorLeft();
+            }
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
index bc859312b371e..530f5b6581deb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.internal.GridDirectCollection;
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.UnregisteredBinaryTypeException;
 import org.apache.ignite.internal.UnregisteredClassException;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -243,9 +244,13 @@ public synchronized void addEntryProcessResult(
                 v = resMap;
             }
 
-            // This exception means that we should register class and call EntryProcessor again.
-            if (err != null && err instanceof UnregisteredClassException)
-                throw (UnregisteredClassException) err;
+            // These exceptions mean that we should register class and call EntryProcessor again.
+            if (err != null) {
+                if (err instanceof UnregisteredClassException)
+                    throw (UnregisteredClassException) err;
+                else if (err instanceof UnregisteredBinaryTypeException)
+                    throw (UnregisteredBinaryTypeException) err;
+            }
 
             CacheInvokeResult res0 = err == null ? CacheInvokeResult.fromResult(res) : CacheInvokeResult.fromError(err);
 
@@ -264,7 +269,7 @@ public synchronized void addEntryProcessResult(
                 invokeResCol = new ArrayList<>();
 
             CacheInvokeDirectResult res0 = err == null ?
-                new CacheInvokeDirectResult(key, cctx.toCacheObject(res)) : new CacheInvokeDirectResult(key, err);
+                CacheInvokeDirectResult.lazyResult(key, res) : new CacheInvokeDirectResult(key, err);
 
             invokeResCol.add(res0);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessor.java
index c7e2e689f7c06..ea8f371b950f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessor.java
@@ -49,9 +49,10 @@ public interface CacheObjectBinaryProcessor extends IgniteCacheObjectProcessor {
     /**
      * @param typeId Type ID.
      * @param newMeta New meta data.
+     * @param failIfUnregistered Fail if unregistered.
      * @throws IgniteException In case of error.
      */
-    public void addMeta(int typeId, final BinaryType newMeta) throws IgniteException;
+    public void addMeta(int typeId, final BinaryType newMeta, boolean failIfUnregistered) throws IgniteException;
 
     /**
      * Adds metadata locally without triggering discovery exchange.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index f5ffb2a3d15a9..4c101b290fd54 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -42,6 +42,7 @@
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.UnregisteredBinaryTypeException;
 import org.apache.ignite.internal.binary.BinaryContext;
 import org.apache.ignite.internal.binary.BinaryEnumObjectImpl;
 import org.apache.ignite.internal.binary.BinaryFieldMetadata;
@@ -163,7 +164,7 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
             transport = new BinaryMetadataTransport(metadataLocCache, metadataFileStore, ctx, log);
 
             BinaryMetadataHandler metaHnd = new BinaryMetadataHandler() {
-                @Override public void addMeta(int typeId, BinaryType newMeta) throws BinaryObjectException {
+                @Override public void addMeta(int typeId, BinaryType newMeta, boolean failIfUnregistered) throws BinaryObjectException {
                     assert newMeta != null;
                     assert newMeta instanceof BinaryTypeImpl;
 
@@ -182,7 +183,7 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
 
                     BinaryMetadata newMeta0 = ((BinaryTypeImpl)newMeta).metadata();
 
-                    CacheObjectBinaryProcessorImpl.this.addMeta(typeId, newMeta0.wrap(binaryCtx));
+                    CacheObjectBinaryProcessorImpl.this.addMeta(typeId, newMeta0.wrap(binaryCtx), failIfUnregistered);
                 }
 
                 @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
@@ -436,11 +437,11 @@ public GridBinaryMarshaller marshaller() {
         BinaryMetadata meta = new BinaryMetadata(typeId, typeName, fieldTypeIds, affKeyFieldName, null, isEnum,
             enumMap);
 
-        binaryCtx.updateMetadata(typeId, meta);
+        binaryCtx.updateMetadata(typeId, meta, false);
     }
 
     /** {@inheritDoc} */
-    @Override public void addMeta(final int typeId, final BinaryType newMeta) throws BinaryObjectException {
+    @Override public void addMeta(final int typeId, final BinaryType newMeta, boolean failIfUnregistered) throws BinaryObjectException {
         assert newMeta != null;
         assert newMeta instanceof BinaryTypeImpl;
 
@@ -457,6 +458,14 @@ public GridBinaryMarshaller marshaller() {
             if (mergedMeta == oldMeta)
                 return;
 
+            if (failIfUnregistered)
+                throw new UnregisteredBinaryTypeException(
+                    "Attempted to update binary metadata inside a critical synchronization block (will be " +
+                        "automatically retried). This exception must not be wrapped to any other exception class. " +
+                        "If you encounter this exception outside of EntryProcessor, please report to Apache Ignite " +
+                        "dev-list.",
+                    typeId, mergedMeta);
+
             MetadataUpdateResult res = transport.requestMetadataUpdate(mergedMeta).get();
 
             assert res != null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index d02b8510c7165..c898c870a1864 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -87,6 +87,7 @@
 import org.apache.ignite.lang.IgniteFutureCancelledException;
 import org.apache.ignite.lang.IgniteReducer;
 import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_OBJECT_LOADED;
@@ -412,10 +413,12 @@ private void onEntriesLocked() {
                             txEntry.oldValueOnPrimary(val != null);
 
                             for (T2<EntryProcessor<Object, Object, Object>, Object[]> t : txEntry.entryProcessors()) {
-                                 CacheInvokeEntry<Object, Object> invokeEntry = new CacheInvokeEntry<>(key, val,
-                                     txEntry.cached().version(), keepBinary, txEntry.cached());
+                                CacheInvokeEntry<Object, Object> invokeEntry = new CacheInvokeEntry<>(key, val,
+                                    txEntry.cached().version(), keepBinary, txEntry.cached());
 
-                                 try {
+                                IgniteThread.onEntryProcessorEntered(false);
+
+                                try {
                                     EntryProcessor<Object, Object, Object> processor = t.get1();
 
                                     procRes = processor.process(invokeEntry, t.get2());
@@ -430,8 +433,11 @@ private void onEntriesLocked() {
 
                                     break;
                                 }
+                                finally {
+                                    IgniteThread.onEntryProcessorLeft();
+                                }
 
-                                 modified |= invokeEntry.modified();
+                                modified |= invokeEntry.modified();
                             }
 
                             if (modified)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/DhtAtomicUpdateResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/DhtAtomicUpdateResult.java
index e7d2b1996a105..15db625dc46e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/DhtAtomicUpdateResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/DhtAtomicUpdateResult.java
@@ -44,6 +44,12 @@ class DhtAtomicUpdateResult {
     /** */
     private IgniteCacheExpiryPolicy expiry;
 
+    /**
+     * If batch update was interrupted in the middle, it should be continued from processedEntriesCount to avoid
+     * extra update closure invocation.
+     */
+    private int processedEntriesCount;
+
     /**
      *
      */
@@ -97,10 +103,19 @@ void addDeleted(GridDhtCacheEntry entry,
     /**
      * @return Deleted entries.
      */
-    Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted() {
+    public Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted() {
         return deleted;
     }
 
+    /**
+     * Sets deleted entries.
+     *
+     * @param deleted deleted entries.
+     */
+    void deleted(Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted) {
+        this.deleted = deleted;
+    }
+
     /**
      * @return DHT future.
      */
@@ -128,4 +143,20 @@ GridCacheReturn returnValue() {
     void dhtFuture(@Nullable GridDhtAtomicAbstractUpdateFuture dhtFut) {
         this.dhtFut = dhtFut;
     }
+
+    /**
+     * Sets processed entries count.
+     * @param idx processed entries count.
+     */
+    public void processedEntriesCount(int idx) {
+        processedEntriesCount = idx;
+    }
+
+    /**
+     * Returns processed entries count.
+     * @return processed entries count.
+     */
+    public int processedEntriesCount() {
+        return processedEntriesCount;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 81711e88fc5ef..36b0a6913ac5e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -37,6 +37,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.NodeStoppingException;
+import org.apache.ignite.internal.UnregisteredBinaryTypeException;
 import org.apache.ignite.internal.UnregisteredClassException;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
@@ -110,6 +111,7 @@
 import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.security.SecurityPermission;
+import org.apache.ignite.thread.IgniteThread;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
 
@@ -1710,6 +1712,8 @@ private void updateAllAsyncInternal0(
 
             Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted = null;
 
+            DhtAtomicUpdateResult  updDhtRes = new DhtAtomicUpdateResult();
+
             try {
                 while (true) {
                     try {
@@ -1739,11 +1743,11 @@ private void updateAllAsyncInternal0(
                             }
 
                             if (!remap) {
-                                DhtAtomicUpdateResult updRes = update(node, locked, req, res);
+                                update(node, locked, req, res, updDhtRes);
 
-                                dhtFut = updRes.dhtFuture();
-                                deleted = updRes.deleted();
-                                expiry = updRes.expiryPolicy();
+                                dhtFut = updDhtRes.dhtFuture();
+                                deleted = updDhtRes.deleted();
+                                expiry = updDhtRes.expiryPolicy();
                             }
                             else
                                 // Should remap all keys.
@@ -1760,7 +1764,16 @@ private void updateAllAsyncInternal0(
 
                         assert cacheObjProc instanceof CacheObjectBinaryProcessorImpl;
 
-                        ((CacheObjectBinaryProcessorImpl)cacheObjProc).binaryContext().descriptorForClass(ex.cls(), false, false);
+                        ((CacheObjectBinaryProcessorImpl)cacheObjProc)
+                            .binaryContext().descriptorForClass(ex.cls(), false, false);
+                    }
+                    catch (UnregisteredBinaryTypeException ex) {
+                        IgniteCacheObjectProcessor cacheObjProc = ctx.cacheObjects();
+
+                        assert cacheObjProc instanceof CacheObjectBinaryProcessorImpl;
+
+                        ((CacheObjectBinaryProcessorImpl)cacheObjProc)
+                            .binaryContext().updateMetadata(ex.typeId(), ex.binaryMetadata(), false);
                     }
                 }
             }
@@ -1833,6 +1846,7 @@ private void updateAllAsyncInternal0(
      * @param locked Entries.
      * @param req Request.
      * @param res Response.
+     * @param dhtUpdRes DHT update result
      * @return Operation result.
      * @throws GridCacheEntryRemovedException If got obsolete entry.
      */
@@ -1840,7 +1854,8 @@ private DhtAtomicUpdateResult update(
         ClusterNode node,
         List<GridDhtCacheEntry> locked,
         GridNearAtomicAbstractUpdateRequest req,
-        GridNearAtomicUpdateResponse res)
+        GridNearAtomicUpdateResponse res,
+        DhtAtomicUpdateResult dhtUpdRes)
         throws GridCacheEntryRemovedException
     {
         GridDhtPartitionTopology top = topology();
@@ -1864,14 +1879,13 @@ private DhtAtomicUpdateResult update(
 
         boolean sndPrevVal = !top.rebalanceFinished(req.topologyVersion());
 
-        GridDhtAtomicAbstractUpdateFuture dhtFut = createDhtFuture(ver, req);
+        if (dhtUpdRes.dhtFuture() == null)
+            dhtUpdRes.dhtFuture(createDhtFuture(ver, req));
 
         IgniteCacheExpiryPolicy expiry = expiryPolicy(req.expiry());
 
         GridCacheReturn retVal = null;
 
-        DhtAtomicUpdateResult updRes;
-
         if (req.size() > 1 &&                    // Several keys ...
             writeThrough() && !req.skipStore() && // and store is enabled ...
             !ctx.store().isLocal() &&             // and this is not local store ...
@@ -1879,40 +1893,39 @@ private DhtAtomicUpdateResult update(
             !ctx.dr().receiveEnabled()            // and no DR.
             ) {
             // This method can only be used when there are no replicated entries in the batch.
-            updRes = updateWithBatch(node,
+            updateWithBatch(node,
                 hasNear,
                 req,
                 res,
                 locked,
                 ver,
-                dhtFut,
                 ctx.isDrEnabled(),
                 taskName,
                 expiry,
-                sndPrevVal);
-
-            dhtFut = updRes.dhtFuture();
+                sndPrevVal,
+                dhtUpdRes);
 
             if (req.operation() == TRANSFORM)
-                retVal = updRes.returnValue();
+                retVal = dhtUpdRes.returnValue();
         }
         else {
-            updRes = updateSingle(node,
+            updateSingle(node,
                 hasNear,
                 req,
                 res,
                 locked,
                 ver,
-                dhtFut,
                 ctx.isDrEnabled(),
                 taskName,
                 expiry,
-                sndPrevVal);
+                sndPrevVal,
+                dhtUpdRes);
 
-            retVal = updRes.returnValue();
-            dhtFut = updRes.dhtFuture();
+            retVal = dhtUpdRes.returnValue();
         }
 
+        GridDhtAtomicAbstractUpdateFuture dhtFut = dhtUpdRes.dhtFuture();
+
         if (retVal == null)
             retVal = new GridCacheReturn(ctx, node.isLocal(), true, null, true);
 
@@ -1925,7 +1938,7 @@ private DhtAtomicUpdateResult update(
                 && !dhtFut.isDone()) {
                 final IgniteRunnable tracker = GridNioBackPressureControl.threadTracker();
 
-                if (tracker != null && tracker instanceof GridNioMessageTracker) {
+                if (tracker instanceof GridNioMessageTracker) {
                     ((GridNioMessageTracker)tracker).onMessageReceived();
 
                     dhtFut.listen(new IgniteInClosure<IgniteInternalFuture<Void>>() {
@@ -1939,9 +1952,9 @@ private DhtAtomicUpdateResult update(
             ctx.mvcc().addAtomicFuture(dhtFut.id(), dhtFut);
         }
 
-        updRes.expiryPolicy(expiry);
+        dhtUpdRes.expiryPolicy(expiry);
 
-        return updRes;
+        return dhtUpdRes;
     }
 
     /**
@@ -1953,27 +1966,26 @@ private DhtAtomicUpdateResult update(
      * @param res Update response.
      * @param locked Locked entries.
      * @param ver Assigned version.
-     * @param dhtFut Optional DHT future.
      * @param replicate Whether replication is enabled.
      * @param taskName Task name.
      * @param expiry Expiry policy.
      * @param sndPrevVal If {@code true} sends previous value to backups.
-     * @return Deleted entries.
+     * @param dhtUpdRes DHT update result.
      * @throws GridCacheEntryRemovedException Should not be thrown.
      */
     @SuppressWarnings("unchecked")
-    private DhtAtomicUpdateResult updateWithBatch(
+    private void updateWithBatch(
         final ClusterNode node,
         final boolean hasNear,
         final GridNearAtomicAbstractUpdateRequest req,
         final GridNearAtomicUpdateResponse res,
         final List<GridDhtCacheEntry> locked,
         final GridCacheVersion ver,
-        @Nullable GridDhtAtomicAbstractUpdateFuture dhtFut,
         final boolean replicate,
         final String taskName,
         @Nullable final IgniteCacheExpiryPolicy expiry,
-        final boolean sndPrevVal
+        final boolean sndPrevVal,
+        final DhtAtomicUpdateResult dhtUpdRes
     ) throws GridCacheEntryRemovedException {
         assert !ctx.dr().receiveEnabled(); // Cannot update in batches during DR due to possible conflicts.
         assert !req.returnValue() || req.operation() == TRANSFORM; // Should not request return values for putAll.
@@ -1985,7 +1997,7 @@ private DhtAtomicUpdateResult updateWithBatch(
             catch (IgniteCheckedException e) {
                 res.addFailedKeys(req.keys(), e);
 
-                return new DhtAtomicUpdateResult();
+                return;
             }
         }
 
@@ -1999,8 +2011,6 @@ private DhtAtomicUpdateResult updateWithBatch(
 
         List<CacheObject> writeVals = null;
 
-        DhtAtomicUpdateResult updRes = new DhtAtomicUpdateResult();
-
         List<GridDhtCacheEntry> filtered = new ArrayList<>(size);
 
         GridCacheOperation op = req.operation();
@@ -2011,7 +2021,7 @@ private DhtAtomicUpdateResult updateWithBatch(
 
         boolean intercept = ctx.config().getInterceptor() != null;
 
-        for (int i = 0; i < locked.size(); i++) {
+        for (int i = dhtUpdRes.processedEntriesCount(); i < locked.size(); i++) {
             GridDhtCacheEntry entry = locked.get(i);
 
             try {
@@ -2070,6 +2080,8 @@ private DhtAtomicUpdateResult updateWithBatch(
 
                     boolean validation = false;
 
+                    IgniteThread.onEntryProcessorEntered(true);
+
                     try {
                         Object computed = entryProcessor.process(invokeEntry, req.invokeArguments());
 
@@ -2093,6 +2105,9 @@ private DhtAtomicUpdateResult updateWithBatch(
                         }
                     }
                     catch (Exception e) {
+                        if (e instanceof UnregisteredClassException || e instanceof UnregisteredBinaryTypeException)
+                            throw (IgniteException) e;
+
                         curInvokeRes = CacheInvokeResult.fromError(e);
 
                         updated = old;
@@ -2104,6 +2119,8 @@ private DhtAtomicUpdateResult updateWithBatch(
                         }
                     }
                     finally {
+                        IgniteThread.onEntryProcessorLeft();
+
                         if (curInvokeRes != null) {
                             invokeRes.addEntryProcessResult(ctx, entry.key(), invokeEntry.key(), curInvokeRes.result(),
                                 curInvokeRes.error(), req.keepBinary());
@@ -2122,7 +2139,7 @@ private DhtAtomicUpdateResult updateWithBatch(
 
                         // Update previous batch.
                         if (putMap != null) {
-                            dhtFut = updatePartialBatch(
+                            updatePartialBatch(
                                 hasNear,
                                 firstEntryIdx,
                                 filtered,
@@ -2132,11 +2149,10 @@ private DhtAtomicUpdateResult updateWithBatch(
                                 putMap,
                                 null,
                                 entryProcessorMap,
-                                dhtFut,
                                 req,
                                 res,
                                 replicate,
-                                updRes,
+                                dhtUpdRes,
                                 taskName,
                                 expiry,
                                 sndPrevVal);
@@ -2170,7 +2186,7 @@ private DhtAtomicUpdateResult updateWithBatch(
 
                         // Update previous batch.
                         if (rmvKeys != null) {
-                            dhtFut = updatePartialBatch(
+                            updatePartialBatch(
                                 hasNear,
                                 firstEntryIdx,
                                 filtered,
@@ -2180,11 +2196,10 @@ private DhtAtomicUpdateResult updateWithBatch(
                                 null,
                                 rmvKeys,
                                 entryProcessorMap,
-                                dhtFut,
                                 req,
                                 res,
                                 replicate,
-                                updRes,
+                                dhtUpdRes,
                                 taskName,
                                 expiry,
                                 sndPrevVal);
@@ -2294,7 +2309,7 @@ else if (op == UPDATE) {
 
         // Store final batch.
         if (putMap != null || rmvKeys != null) {
-            dhtFut = updatePartialBatch(
+            updatePartialBatch(
                 hasNear,
                 firstEntryIdx,
                 filtered,
@@ -2304,11 +2319,10 @@ else if (op == UPDATE) {
                 putMap,
                 rmvKeys,
                 entryProcessorMap,
-                dhtFut,
                 req,
                 res,
                 replicate,
-                updRes,
+                dhtUpdRes,
                 taskName,
                 expiry,
                 sndPrevVal);
@@ -2316,11 +2330,7 @@ else if (op == UPDATE) {
         else
             assert filtered.isEmpty();
 
-        updRes.dhtFuture(dhtFut);
-
-        updRes.returnValue(invokeRes);
-
-        return updRes;
+        dhtUpdRes.returnValue(invokeRes);
     }
 
     /**
@@ -2383,29 +2393,30 @@ private void reloadIfNeeded(final List<GridDhtCacheEntry> entries) throws Ignite
      * @param res Update response.
      * @param locked Locked entries.
      * @param ver Assigned update version.
-     * @param dhtFut Optional DHT future.
      * @param replicate Whether DR is enabled for that cache.
      * @param taskName Task name.
      * @param expiry Expiry policy.
      * @param sndPrevVal If {@code true} sends previous value to backups.
-     * @return Return value.
+     * @param dhtUpdRes Dht update result
      * @throws GridCacheEntryRemovedException Should be never thrown.
      */
-    private DhtAtomicUpdateResult updateSingle(
+    private void updateSingle(
         ClusterNode nearNode,
         boolean hasNear,
         GridNearAtomicAbstractUpdateRequest req,
         GridNearAtomicUpdateResponse res,
         List<GridDhtCacheEntry> locked,
         GridCacheVersion ver,
-        @Nullable GridDhtAtomicAbstractUpdateFuture dhtFut,
         boolean replicate,
         String taskName,
         @Nullable IgniteCacheExpiryPolicy expiry,
-        boolean sndPrevVal
+        boolean sndPrevVal,
+        DhtAtomicUpdateResult dhtUpdRes
     ) throws GridCacheEntryRemovedException {
-        GridCacheReturn retVal = null;
-        Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted = null;
+        GridCacheReturn retVal = dhtUpdRes.returnValue();
+        GridDhtAtomicAbstractUpdateFuture dhtFut = dhtUpdRes.dhtFuture();
+        Collection<IgniteBiTuple<GridDhtCacheEntry, GridCacheVersion>> deleted = dhtUpdRes.deleted();
+
 
         AffinityTopologyVersion topVer = req.topologyVersion();
 
@@ -2414,7 +2425,7 @@ private DhtAtomicUpdateResult updateSingle(
         AffinityAssignment affAssignment = ctx.affinity().assignment(topVer, req.lastAffinityChangedTopologyVersion());
 
         // Avoid iterator creation.
-        for (int i = 0; i < req.size(); i++) {
+        for (int i = dhtUpdRes.processedEntriesCount(); i < req.size(); i++) {
             KeyCacheObject k = req.key(i);
 
             GridCacheOperation op = req.operation();
@@ -2576,9 +2587,13 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) {
             catch (IgniteCheckedException e) {
                 res.addFailedKey(k, e);
             }
+
+            dhtUpdRes.processedEntriesCount(i + 1);
         }
 
-        return new DhtAtomicUpdateResult(retVal, deleted, dhtFut);
+        dhtUpdRes.returnValue(retVal);
+        dhtUpdRes.deleted(deleted);
+        dhtUpdRes.dhtFuture(dhtFut);
     }
 
     /**
@@ -2591,18 +2606,16 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) {
      * @param putMap Values to put.
      * @param rmvKeys Keys to remove.
      * @param entryProcessorMap Entry processors.
-     * @param dhtFut DHT update future if has backups.
      * @param req Request.
      * @param res Response.
      * @param replicate Whether replication is enabled.
-     * @param batchRes Batch update result.
+     * @param dhtUpdRes Batch update result.
      * @param taskName Task name.
      * @param expiry Expiry policy.
      * @param sndPrevVal If {@code true} sends previous value to backups.
-     * @return Deleted entries.
      */
     @SuppressWarnings("ForLoopReplaceableByForEach")
-    @Nullable private GridDhtAtomicAbstractUpdateFuture updatePartialBatch(
+    @Nullable private void updatePartialBatch(
         final boolean hasNear,
         final int firstEntryIdx,
         final List<GridDhtCacheEntry> entries,
@@ -2612,11 +2625,10 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) {
         @Nullable final Map<KeyCacheObject, CacheObject> putMap,
         @Nullable final Collection<KeyCacheObject> rmvKeys,
         @Nullable final Map<KeyCacheObject, EntryProcessor<Object, Object, Object>> entryProcessorMap,
-        @Nullable GridDhtAtomicAbstractUpdateFuture dhtFut,
         final GridNearAtomicAbstractUpdateRequest req,
         final GridNearAtomicUpdateResponse res,
         final boolean replicate,
-        final DhtAtomicUpdateResult batchRes,
+        final DhtAtomicUpdateResult dhtUpdRes,
         final String taskName,
         @Nullable final IgniteCacheExpiryPolicy expiry,
         final boolean sndPrevVal
@@ -2664,6 +2676,8 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) {
 
             AffinityAssignment affAssignment = ctx.affinity().assignment(topVer, req.lastAffinityChangedTopologyVersion());
 
+            final GridDhtAtomicAbstractUpdateFuture dhtFut = dhtUpdRes.dhtFuture();
+
             // Avoid iterator creation.
             for (int i = 0; i < entries.size(); i++) {
                 GridDhtCacheEntry entry = entries.get(i);
@@ -2739,7 +2753,7 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) {
                         }
                     }
 
-                    batchRes.addDeleted(entry, updRes, entries);
+                    dhtUpdRes.addDeleted(entry, updRes, entries);
 
                     if (dhtFut != null) {
                         EntryProcessor<Object, Object, Object> entryProcessor =
@@ -2800,7 +2814,10 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) {
 
                     e.printStackTrace();
                 }
+
+                dhtUpdRes.processedEntriesCount(firstEntryIdx + i + 1);
             }
+
         }
         catch (IgniteCheckedException e) {
             res.addFailedKeys(putMap != null ? putMap.keySet() : rmvKeys, e);
@@ -2814,8 +2831,6 @@ else if (GridDhtCacheEntry.ReaderId.contains(readers, nearNode.id())) {
 
             res.addFailedKeys(failed, storeErr.getCause());
         }
-
-        return dhtFut;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index dad105231e36e..cd06a25d1b376 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -72,6 +72,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.plugin.security.SecurityPermission;
+import org.apache.ignite.thread.IgniteThread;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
 
@@ -1064,6 +1065,8 @@ private Map<K, EntryProcessorResult> updateWithBatch(
 
                         boolean validation = false;
 
+                        IgniteThread.onEntryProcessorEntered(false);
+
                         try {
                             Object computed = entryProcessor.process(invokeEntry, invokeArgs);
 
@@ -1091,6 +1094,9 @@ private Map<K, EntryProcessorResult> updateWithBatch(
                                 continue;
                             }
                         }
+                        finally {
+                            IgniteThread.onEntryProcessorLeft();
+                        }
 
                         if (invokeRes != null)
                             invokeResMap.put((K)entry.key().value(ctx.cacheObjectContext(), false), invokeRes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
index cae19edab17ac..c3559848ab656 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -31,6 +31,8 @@
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.UnregisteredBinaryTypeException;
+import org.apache.ignite.internal.UnregisteredClassException;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
@@ -1648,6 +1650,9 @@ public final boolean removex(L row) throws IgniteCheckedException {
                 }
             }
         }
+        catch (UnregisteredClassException | UnregisteredBinaryTypeException e) {
+            throw e;
+        }
         catch (IgniteCheckedException e) {
             throw new IgniteCheckedException("Runtime failure on search row: " + row, e);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 24fd13d8589ca..8a966bcd97eea 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -84,6 +84,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.thread.IgniteThread;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.apache.ignite.transactions.TransactionState;
@@ -1543,6 +1544,8 @@ else if (txEntry.hasOldValue())
                 Object procRes = null;
                 Exception err = null;
 
+                IgniteThread.onEntryProcessorEntered(true);
+
                 try {
                     EntryProcessor<Object, Object, Object> processor = t.get1();
 
@@ -1555,6 +1558,9 @@ else if (txEntry.hasOldValue())
                 catch (Exception e) {
                     err = e;
                 }
+                finally {
+                    IgniteThread.onEntryProcessorLeft();
+                }
 
                 if (ret != null) {
                     if (err != null || procRes != null)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java
index 71c6b65c8ca8e..8e65605f79ec8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java
@@ -52,6 +52,7 @@
 import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
 import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 
 import static org.apache.ignite.internal.processors.cache.GridCacheOperation.READ;
@@ -762,6 +763,8 @@ public CacheObject applyEntryProcessors(CacheObject cacheVal) {
         Object keyVal = null;
 
         for (T2<EntryProcessor<Object, Object, Object>, Object[]> t : entryProcessors()) {
+            IgniteThread.onEntryProcessorEntered(true);
+
             try {
                 CacheInvokeEntry<Object, Object> invokeEntry = new CacheInvokeEntry(key, keyVal, cacheVal, val,
                     ver, keepBinary(), cached());
@@ -777,6 +780,9 @@ public CacheObject applyEntryProcessors(CacheObject cacheVal) {
             catch (Exception ignore) {
                 // No-op.
             }
+            finally {
+                IgniteThread.onEntryProcessorLeft();
+            }
         }
 
         return ctx.toCacheObject(val);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index c224eaba25750..b02f9db13534f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -81,6 +81,7 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiClosure;
 import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.thread.IgniteThread;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionDeadlockException;
 import org.apache.ignite.transactions.TransactionIsolation;
@@ -1241,6 +1242,8 @@ protected final void addInvokeResult(IgniteTxEntry txEntry,
         Object key0 = null;
         Object val0 = null;
 
+        IgniteThread.onEntryProcessorEntered(true);
+
         try {
             Object res = null;
 
@@ -1268,6 +1271,9 @@ protected final void addInvokeResult(IgniteTxEntry txEntry,
         catch (Exception e) {
             ret.addEntryProcessResult(ctx, txEntry.key(), key0, null, e, txEntry.keepBinary());
         }
+        finally {
+            IgniteThread.onEntryProcessorLeft();
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java
index 9e22f38fb53d6..4e22ce9eaff7e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/PlatformContextImpl.java
@@ -333,7 +333,7 @@ public PlatformContextImpl(GridKernalContext ctx, PlatformCallbackGateway gate,
         BinaryContext binCtx = cacheObjProc.binaryContext();
 
         for (BinaryMetadata meta : metas)
-            binCtx.updateMetadata(meta.typeId(), meta);
+            binCtx.updateMetadata(meta.typeId(), meta, false);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/binary/ClientBinaryTypePutRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/binary/ClientBinaryTypePutRequest.java
index 7839d4836bf5b..64c8d80d11d1a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/binary/ClientBinaryTypePutRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/binary/ClientBinaryTypePutRequest.java
@@ -49,7 +49,7 @@ public ClientBinaryTypePutRequest(BinaryRawReaderEx reader) {
     @Override public ClientResponse process(ClientConnectionContext ctx) {
         BinaryContext binCtx = ((CacheObjectBinaryProcessorImpl) ctx.kernalContext().cacheObjects()).binaryContext();
 
-        binCtx.updateMetadata(meta.typeId(), meta);
+        binCtx.updateMetadata(meta.typeId(), meta, false);
 
         return super.process(ctx);
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java
index 70b75e3bc70de..6f65e0e34d39b 100644
--- a/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java
+++ b/modules/core/src/main/java/org/apache/ignite/thread/IgniteThread.java
@@ -56,6 +56,12 @@ public class IgniteThread extends Thread {
     /** */
     private final byte plc;
 
+    /** */
+    private boolean executingEntryProcessor;
+
+    /** */
+    private boolean holdsTopLock;
+
     /**
      * Creates thread with given worker.
      *
@@ -157,10 +163,47 @@ public void compositeRwLockIndex(int compositeRwLockIdx) {
         this.compositeRwLockIdx = compositeRwLockIdx;
     }
 
+    /**
+     * @return {@code True} if thread is currently executing entry processor.
+     */
+    public boolean executingEntryProcessor() {
+        return executingEntryProcessor;
+    }
+
+    /**
+     * @return {@code True} if thread is currently holds topology lock.
+     */
+    public boolean holdsTopLock() {
+        return holdsTopLock;
+    }
+
+    /**
+     * Callback before entry processor execution is started.
+     */
+    public static void onEntryProcessorEntered(boolean holdsTopLock) {
+        Thread curThread = Thread.currentThread();
+
+        if (curThread instanceof IgniteThread) {
+            ((IgniteThread)curThread).executingEntryProcessor = true;
+
+            ((IgniteThread)curThread).holdsTopLock = holdsTopLock;
+        }
+    }
+
+    /**
+     * Callback after entry processor execution is finished.
+     */
+    public static void onEntryProcessorLeft() {
+        Thread curThread = Thread.currentThread();
+
+        if (curThread instanceof IgniteThread)
+            ((IgniteThread)curThread).executingEntryProcessor = false;
+    }
+
     /**
      * @return IgniteThread or {@code null} if current thread is not an instance of IgniteThread.
      */
-    public static IgniteThread current(){
+    public static IgniteThread current() {
         Thread thread = Thread.currentThread();
 
         return thread.getClass() == IgniteThread.class || thread instanceof IgniteThread ?
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java
index 0870153e00f2a..c515f8191766a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java
@@ -30,7 +30,7 @@ public class TestCachingMetadataHandler implements BinaryMetadataHandler {
     private final ConcurrentHashMap<Integer, BinaryType> metas = new ConcurrentHashMap<>();
 
     /** {@inheritDoc} */
-    @Override public void addMeta(int typeId, BinaryType meta) throws BinaryObjectException {
+    @Override public void addMeta(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException {
         BinaryType otherType = metas.put(typeId, meta);
 
         if (otherType != null)
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataRegistrationInsideEntryProcessorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataRegistrationInsideEntryProcessorTest.java
new file mode 100644
index 0000000000000..73dae4bb39a5d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataRegistrationInsideEntryProcessorTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class BinaryMetadataRegistrationInsideEntryProcessorTest extends GridCommonAbstractTest {
+    /** */
+    private static final String CACHE_NAME = "test-cache";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration() {
+        TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder()
+            .setAddresses(Arrays.asList("127.0.0.1:47500..47509"));
+
+        return new IgniteConfiguration()
+            .setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(ipFinder))
+            .setPeerClassLoadingEnabled(true);
+    }
+
+    /**
+     * @throws Exception If failed;
+     */
+    public void test() throws Exception {
+        Ignite ignite = startGrids(2);
+
+        IgniteCache<Integer, Map<Integer, CustomObj>> cache = ignite.createCache(CACHE_NAME);
+
+        try {
+            for (int i = 0; i < 10_000; i++)
+                cache.invoke(i, new CustomProcessor());
+        }
+        catch (Exception e) {
+            Map<Integer, CustomObj> value = cache.get(1);
+
+            if ((value != null) && (value.get(1) != null) && (value.get(1).getObj() == CustomEnum.ONE))
+                System.out.println("Data was saved.");
+            else
+                System.out.println("Data wasn't saved.");
+
+            throw e;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class CustomProcessor implements EntryProcessor<Integer,
+        Map<Integer, CustomObj>, Object> {
+        /** {@inheritDoc} */
+        @Override public Object process(
+            MutableEntry<Integer, Map<Integer, CustomObj>> entry,
+            Object... objects) throws EntryProcessorException {
+            Map<Integer, CustomObj> map = new HashMap<>();
+
+            map.put(1, new CustomObj(CustomEnum.ONE));
+
+            entry.setValue(map);
+
+            return null;
+        }
+    }
+
+    /**
+     *
+     */
+    private static class CustomObj {
+        /** Object. */
+        private final Object obj;
+
+        /**
+         * @param obj Object.
+         */
+        public CustomObj(Object obj) {
+            this.obj = obj;
+        }
+
+        /**
+         * @param val Value.
+         */
+        public static CustomObj valueOf(int val) {
+            return new CustomObj(val);
+        }
+
+        /**
+         *
+         */
+        public Object getObj() {
+            return obj;
+        }
+    }
+
+    /**
+     *
+     */
+    private enum CustomEnum {
+        /** */ONE(1),
+        /** */TWO(2),
+        /** */THREE(3);
+
+        /** Value. */
+        private final Object val;
+
+        /**
+         * @param val Value.
+         */
+        CustomEnum(Object val) {
+            this.val = val;
+        }
+    }
+
+}
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeAbstractTest.java
index b8fc8004bd96f..ef07bf17fb697 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheInvokeAbstractTest.java
@@ -18,12 +18,15 @@
 package org.apache.ignite.internal.processors.cache;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.Callable;
 import javax.cache.processor.EntryProcessor;
@@ -32,10 +35,15 @@
 import javax.cache.processor.MutableEntry;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.binary.BinaryObjectBuilder;
+import org.apache.ignite.cache.CacheEntryProcessor;
 import org.apache.ignite.cache.CachePeekMode;
+import org.apache.ignite.cache.affinity.Affinity;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
@@ -212,6 +220,148 @@ public void testInvokeAll() throws Exception {
         }
     }
 
+    /**
+     *
+     */
+    private static class MyKey {
+        /** */
+        String key;
+
+
+        /** */
+        @AffinityKeyMapped
+        String affkey = "affkey";
+
+        /** */
+        public MyKey(String key) {
+            this.key = key;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+
+            if (!(o instanceof MyKey))
+                return false;
+
+            MyKey key1 = (MyKey)o;
+
+            return Objects.equals(key, key1.key) &&
+                Objects.equals(affkey, key1.affkey);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return Objects.hash(key, affkey);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "MyKey{" +
+                "key='" + key + '\'' +
+                '}';
+        }
+    }
+
+    /** */
+    static class MyClass1{}
+
+    /** */
+    static class MyClass2{}
+
+    /** */
+    static class MyClass3{}
+
+    /** */
+    Object[] results = new Object[] {
+        new MyClass1(),
+        new MyClass2(),
+        new MyClass3()
+    };
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testInvokeAllAppliedOnceOnBinaryTypeRegistration() {
+        IgniteCache<MyKey, Integer> cache = jcache();
+
+        Affinity<Object> affinity = grid(0).affinity(cache.getName());
+
+        for (int i = 0; i < gridCount(); i++) {
+            if(!affinity.isPrimary(grid(i).localNode(), new MyKey(""))) {
+                cache = jcache(i);
+                break;
+            }
+        }
+
+        LinkedHashSet<MyKey> keys = new LinkedHashSet<>(Arrays.asList(
+            new MyKey("remove_0"), new MyKey("1"), new MyKey("2"),
+            new MyKey("remove_3"), new MyKey("remove_4"), new MyKey("register_type_0"),
+            new MyKey("6"), new MyKey("remove_7"), new MyKey("register_type_1"),
+            new MyKey("9"), new MyKey("remove_10"), new MyKey("11"), new MyKey("12"), new MyKey("register_type_2")
+        ));
+
+        for (MyKey key : keys)
+            cache.put(key, 0);
+
+        cache.invokeAll(keys,
+            new CacheEntryProcessor<MyKey, Integer, Object>() {
+
+                @IgniteInstanceResource
+                Ignite ignite;
+
+                @Override public Object process(MutableEntry<MyKey, Integer> entry,
+                    Object... objects) throws EntryProcessorException {
+
+                    String key = entry.getKey().key;
+
+                    if (key.startsWith("register_type")) {
+                        BinaryObjectBuilder bo = ignite.binary().builder(key);
+
+                        bo.build();
+                    }
+
+                    if (key.startsWith("remove")) {
+                        entry.remove();
+                    }
+                    else {
+                        Integer value = entry.getValue() == null ? 0 : entry.getValue();
+
+                        entry.setValue(++value);
+                    }
+
+                    if (key.startsWith("register_type"))
+                        return results[Integer.parseInt(key.substring(key.lastIndexOf("_") + 1))];
+
+                    return null;
+                }
+
+            });
+
+        Map<MyKey, Integer> all = cache.getAll(keys);
+
+        for (Map.Entry<MyKey, Integer> entry : all.entrySet()) {
+            MyKey key = entry.getKey();
+
+            if (key.key.startsWith("remove")) {
+                assertNull(entry.getValue());
+
+                if (cacheStoreFactory() != null)
+                    assertNull(storeMap.get(keys));
+            }
+            else {
+                int value = entry.getValue();
+
+                assertEquals("\"" + key + "' entry has wrong value, exp=1 actl=" + value, 1, value);
+
+                if (cacheStoreFactory() != null)
+                    assertEquals("\"" + key + "' entry has wrong value in cache store, exp=1 actl=" + value,
+                        1, (int)storeMap.get(key));
+            }
+        }
+    }
+
     /**
      * @param cache Cache.
      * @param txMode Not null transaction concurrency mode if explicit transaction should be started.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
index 9c64a9f1ce344..751826cd85a1e 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.internal.managers.communication.IgniteCommunicationSslBalanceTest;
 import org.apache.ignite.internal.managers.communication.IgniteIoTestMessagesTest;
 import org.apache.ignite.internal.managers.communication.IgniteVariousConnectionNumberTest;
+import org.apache.ignite.internal.processors.cache.BinaryMetadataRegistrationInsideEntryProcessorTest;
 import org.apache.ignite.internal.processors.cache.CacheAffinityCallSelfTest;
 import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteQueueTest;
 import org.apache.ignite.internal.processors.cache.CacheDeferredDeleteSanitySelfTest;
@@ -335,6 +336,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
         suite.addTestSuite(CacheStoreReadFromBackupTest.class);
         suite.addTestSuite(CacheTransactionalStoreReadFromBackupTest.class);
 
+        suite.addTestSuite(BinaryMetadataRegistrationInsideEntryProcessorTest.class);
+
         return suite;
     }
 }

From 8e291bd0b85c781aa9c1ade73015848129eb2400 Mon Sep 17 00:00:00 2001
From: ascherbakoff <alexey.scherbakoff@gmail.com>
Date: Thu, 18 Oct 2018 20:07:03 +0300
Subject: [PATCH 437/543] IGNITE-9830 Fixed race in binary metadata
 registration leading to exception on commit - Fixes #4996.

---
 .../apache/ignite/IgniteSystemProperties.java |   5 +
 .../internal/binary/BinaryReaderExImpl.java   |   1 -
 .../internal/binary/BinarySchemaRegistry.java | 114 +++--
 .../ignite/internal/binary/BinaryUtils.java   |  28 +-
 .../cache/binary/BinaryMetadataTransport.java |  93 +++-
 .../CacheObjectBinaryProcessorImpl.java       | 219 ++++++++-
 .../spi/discovery/tcp/TcpDiscoverySpi.java    |   5 +-
 .../CachePageWriteLockUnlockTest.java         |   2 +
 .../transactions/TxRollbackOnTimeoutTest.java |   7 +-
 ...tadataConcurrentUpdateWithIndexesTest.java | 439 ++++++++++++++++++
 .../IgniteBinaryCacheQueryTestSuite.java      |   3 +
 11 files changed, 827 insertions(+), 89 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataConcurrentUpdateWithIndexesTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 746c00227b9fc..05a40ad105ae3 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -953,6 +953,11 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_REBALANCE_THROTTLE_OVERRIDE = "IGNITE_REBALANCE_THROTTLE_OVERRIDE";
 
+    /**
+     * Timeout for waiting schema update if schema was not found for last accepted version.
+     */
+    public static final String IGNITE_WAIT_SCHEMA_UPDATE = "IGNITE_WAIT_SCHEMA_UPDATE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
index ab1f874386dd9..d29bb24b70cf7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryReaderExImpl.java
@@ -2028,7 +2028,6 @@ public BinarySchema getOrCreateSchema() {
                     for (BinarySchema existingSchema : existingSchemas)
                         existingSchemaIds.add(existingSchema.schemaId());
 
-
                     throw new BinaryObjectException("Cannot find schema for object with compact footer" +
                         " [typeName=" + type.typeName() +
                         ", typeId=" + typeId +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchemaRegistry.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchemaRegistry.java
index 91f29b22cfd8a..f22fc4c052121 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchemaRegistry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinarySchemaRegistry.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.binary;
 
+import java.util.ArrayList;
+import java.util.List;
 import org.jetbrains.annotations.Nullable;
 
 import java.util.HashMap;
@@ -98,75 +100,95 @@ else if (schemaId == schemaId4)
      * @param schemaId Schema ID.
      * @param schema Schema.
      */
-    public void addSchema(int schemaId, BinarySchema schema) {
-        synchronized (this) {
-            if (inline) {
-                // Check if this is already known schema.
-                if (schemaId == schemaId1 || schemaId == schemaId2 || schemaId == schemaId3 || schemaId == schemaId4)
-                    return;
+    public synchronized void addSchema(int schemaId, BinarySchema schema) {
+        if (inline) {
+            // Check if this is already known schema.
+            if (schemaId == schemaId1 || schemaId == schemaId2 || schemaId == schemaId3 || schemaId == schemaId4)
+                return;
 
-                // Try positioning new schema in inline mode.
-                if (schemaId1 == EMPTY) {
-                    schemaId1 = schemaId;
+            // Try positioning new schema in inline mode.
+            if (schemaId1 == EMPTY) {
+                schemaId1 = schemaId;
 
-                    schema1 = schema;
+                schema1 = schema;
 
-                    inline = true; // Forcing HB edge just in case.
+                inline = true; // Forcing HB edge just in case.
 
-                    return;
-                }
+                return;
+            }
 
-                if (schemaId2 == EMPTY) {
-                    schemaId2 = schemaId;
+            if (schemaId2 == EMPTY) {
+                schemaId2 = schemaId;
 
-                    schema2 = schema;
+                schema2 = schema;
 
-                    inline = true; // Forcing HB edge just in case.
+                inline = true; // Forcing HB edge just in case.
 
-                    return;
-                }
+                return;
+            }
 
-                if (schemaId3 == EMPTY) {
-                    schemaId3 = schemaId;
+            if (schemaId3 == EMPTY) {
+                schemaId3 = schemaId;
 
-                    schema3 = schema;
+                schema3 = schema;
 
-                    inline = true; // Forcing HB edge just in case.
+                inline = true; // Forcing HB edge just in case.
 
-                    return;
-                }
+                return;
+            }
 
-                if (schemaId4 == EMPTY) {
-                    schemaId4 = schemaId;
+            if (schemaId4 == EMPTY) {
+                schemaId4 = schemaId;
 
-                    schema4 = schema;
+                schema4 = schema;
 
-                    inline = true; // Forcing HB edge just in case.
+                inline = true; // Forcing HB edge just in case.
 
-                    return;
-                }
+                return;
+            }
 
-                // No luck, switching to hash map mode.
-                HashMap<Integer, BinarySchema> newSchemas = new HashMap<>();
+            // No luck, switching to hash map mode.
+            HashMap<Integer, BinarySchema> newSchemas = new HashMap<>();
 
-                newSchemas.put(schemaId1, schema1);
-                newSchemas.put(schemaId2, schema2);
-                newSchemas.put(schemaId3, schema3);
-                newSchemas.put(schemaId4, schema4);
+            newSchemas.put(schemaId1, schema1);
+            newSchemas.put(schemaId2, schema2);
+            newSchemas.put(schemaId3, schema3);
+            newSchemas.put(schemaId4, schema4);
 
-                newSchemas.put(schemaId, schema);
+            newSchemas.put(schemaId, schema);
 
-                schemas = newSchemas;
+            schemas = newSchemas;
 
-                inline = false;
-            }
-            else {
-                HashMap<Integer, BinarySchema> newSchemas = new HashMap<>(schemas);
+            inline = false;
+        }
+        else {
+            HashMap<Integer, BinarySchema> newSchemas = new HashMap<>(schemas);
 
-                newSchemas.put(schemaId, schema);
+            newSchemas.put(schemaId, schema);
 
-                schemas = newSchemas;
-            }
+            schemas = newSchemas;
         }
     }
+
+    /**
+     * @return List of known schemas.
+     */
+    public synchronized List<BinarySchema> schemas() {
+        List<BinarySchema> res = new ArrayList<>();
+
+        if (inline) {
+            if (schemaId1 != EMPTY)
+                res.add(schema1);
+            if (schemaId2 != EMPTY)
+                res.add(schema2);
+            if (schemaId3 != EMPTY)
+                res.add(schema3);
+            if (schemaId4 != EMPTY)
+                res.add(schema4);
+        }
+        else
+            res.addAll(schemas.values());
+
+        return res;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
index 284a4a5448e00..0f7244bce9649 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryUtils.java
@@ -958,10 +958,30 @@ else if (fieldOffsetSize == OFFSET_2)
      * @throws BinaryObjectException If merge failed due to metadata conflict.
      */
     public static BinaryMetadata mergeMetadata(@Nullable BinaryMetadata oldMeta, BinaryMetadata newMeta) {
+        return mergeMetadata(oldMeta, newMeta, null);
+    }
+
+    /**
+     * Merge old and new metas.
+     *
+     * @param oldMeta Old meta.
+     * @param newMeta New meta.
+     * @param changedSchemas Set for holding changed schemas.
+     * @return New meta if old meta was null, old meta if no changes detected, merged meta otherwise.
+     * @throws BinaryObjectException If merge failed due to metadata conflict.
+     */
+    public static BinaryMetadata mergeMetadata(@Nullable BinaryMetadata oldMeta, BinaryMetadata newMeta,
+        @Nullable Set<Integer> changedSchemas) {
         assert newMeta != null;
 
-        if (oldMeta == null)
+        if (oldMeta == null) {
+            if (changedSchemas != null) {
+                for (BinarySchema schema : newMeta.schemas())
+                    changedSchemas.add(schema.schemaId());
+            }
+
             return newMeta;
+        }
         else {
             assert oldMeta.typeId() == newMeta.typeId();
 
@@ -1036,8 +1056,12 @@ public static BinaryMetadata mergeMetadata(@Nullable BinaryMetadata oldMeta, Bin
             Collection<BinarySchema> mergedSchemas = new HashSet<>(oldMeta.schemas());
 
             for (BinarySchema newSchema : newMeta.schemas()) {
-                if (mergedSchemas.add(newSchema))
+                if (mergedSchemas.add(newSchema)) {
                     changed = true;
+
+                    if (changedSchemas != null)
+                        changedSchemas.add(newSchema.schemaId());
+                }
             }
 
             // Return either old meta if no changes detected, or new merged meta.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
index 38450dfec5c78..1c2f6f0e77f84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/BinaryMetadataTransport.java
@@ -16,8 +16,12 @@
  */
 package org.apache.ignite.internal.processors.cache.binary;
 
+import java.util.Iterator;
+import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Queue;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -42,6 +46,7 @@
 import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.jetbrains.annotations.Nullable;
@@ -86,6 +91,9 @@ final class BinaryMetadataTransport {
     /** */
     private final ConcurrentMap<Integer, ClientMetadataRequestFuture> clientReqSyncMap = new ConcurrentHashMap<>();
 
+    /** */
+    private final ConcurrentMap<SyncKey, GridFutureAdapter<?>> schemaWaitFuts = new ConcurrentHashMap<>();
+
     /** */
     private volatile boolean stopping;
 
@@ -206,6 +214,21 @@ GridFutureAdapter<MetadataUpdateResult> awaitMetadataUpdate(int typeId, int ver)
         return resFut;
     }
 
+    /**
+     * Await specific schema update.
+     * @param typeId Type id.
+     * @param schemaId Schema id.
+     * @return Future which will be completed when schema is received.
+     */
+    GridFutureAdapter<?> awaitSchemaUpdate(int typeId, int schemaId) {
+        GridFutureAdapter<Object> fut = new GridFutureAdapter<>();
+
+        // Use version for schemaId.
+        GridFutureAdapter<?> oldFut = schemaWaitFuts.putIfAbsent(new SyncKey(typeId, schemaId), fut);
+
+        return oldFut == null ? fut : oldFut;
+    }
+
     /**
      * Allows client node to request latest version of binary metadata for a given typeId from the cluster
      * in case client is able to detect that it has obsolete metadata in its local cache.
@@ -259,6 +282,13 @@ private final class MetadataUpdateProposedListener implements CustomEventListene
 
         /** {@inheritDoc} */
         @Override public void onCustomEvent(AffinityTopologyVersion topVer, ClusterNode snd, MetadataUpdateProposedMessage msg) {
+            if (log.isDebugEnabled())
+                log.debug("Received MetadataUpdateProposedListener [typeId=" + msg.typeId() +
+                    ", typeName=" + msg.metadata().typeName() +
+                    ", pendingVer=" + msg.pendingVersion() +
+                    ", acceptedVer=" + msg.acceptedVersion() +
+                    ", schemasCnt=" + msg.metadata().schemas().size() + ']');
+
             int typeId = msg.typeId();
 
             BinaryMetadataHolder holder = metaLocCache.get(typeId);
@@ -277,20 +307,23 @@ private final class MetadataUpdateProposedListener implements CustomEventListene
                     acceptedVer = 0;
                 }
 
-                if (log.isDebugEnabled())
-                    log.debug("Versions are stamped on coordinator" +
-                        " [typeId=" + typeId +
-                        ", pendingVer=" + pendingVer +
-                        ", acceptedVer=" + acceptedVer + "]"
-                    );
-
                 msg.pendingVersion(pendingVer);
                 msg.acceptedVersion(acceptedVer);
 
                 BinaryMetadata locMeta = holder != null ? holder.metadata() : null;
 
                 try {
-                    BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(locMeta, msg.metadata());
+                    Set<Integer> changedSchemas = new LinkedHashSet<>();
+
+                    BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(locMeta, msg.metadata(), changedSchemas);
+
+                    if (log.isDebugEnabled())
+                        log.debug("Versions are stamped on coordinator" +
+                                " [typeId=" + typeId +
+                                ", changedSchemas=" + changedSchemas +
+                                ", pendingVer=" + pendingVer +
+                                ", acceptedVer=" + acceptedVer + "]"
+                        );
 
                     msg.metadata(mergedMeta);
                 }
@@ -358,8 +391,10 @@ private final class MetadataUpdateProposedListener implements CustomEventListene
                 if (!msg.rejected()) {
                     BinaryMetadata locMeta = holder != null ? holder.metadata() : null;
 
+                    Set<Integer> changedSchemas = new LinkedHashSet<>();
+
                     try {
-                        BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(locMeta, msg.metadata());
+                        BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(locMeta, msg.metadata(), changedSchemas);
 
                         BinaryMetadataHolder newHolder = new BinaryMetadataHolder(mergedMeta, pendingVer, acceptedVer);
 
@@ -382,7 +417,8 @@ private final class MetadataUpdateProposedListener implements CustomEventListene
                         }
                         else {
                             if (log.isDebugEnabled())
-                                log.debug("Updated metadata on server node: " + newHolder);
+                                log.debug("Updated metadata on server node [holder=" + newHolder +
+                                    ", changedSchemas=" + changedSchemas + ']');
 
                             metaLocCache.put(typeId, newHolder);
                         }
@@ -463,7 +499,7 @@ private final class MetadataUpdateAcceptedListener implements CustomEventListene
                 if (oldAcceptedVer >= newAcceptedVer) {
                     if (log.isDebugEnabled())
                         log.debug("Marking ack as duplicate [holder=" + holder +
-                            ", newAcceptedVer: " + newAcceptedVer + ']');
+                            ", newAcceptedVer=" + newAcceptedVer + ']');
 
                     //this is duplicate ack
                     msg.duplicated(true);
@@ -481,8 +517,26 @@ private final class MetadataUpdateAcceptedListener implements CustomEventListene
 
             GridFutureAdapter<MetadataUpdateResult> fut = syncMap.get(new SyncKey(typeId, newAcceptedVer));
 
+            holder = metaLocCache.get(typeId);
+
             if (log.isDebugEnabled())
-                log.debug("Completing future " + fut + " for " + metaLocCache.get(typeId));
+                log.debug("Completing future " + fut + " for " + holder);
+
+            if (!schemaWaitFuts.isEmpty()) {
+                Iterator<Map.Entry<SyncKey, GridFutureAdapter<?>>> iter = schemaWaitFuts.entrySet().iterator();
+
+                while (iter.hasNext()) {
+                    Map.Entry<SyncKey, GridFutureAdapter<?>> entry = iter.next();
+
+                    SyncKey key = entry.getKey();
+
+                    if (key.typeId() == typeId && holder.metadata().hasSchema(key.version())) {
+                        entry.getValue().onDone();
+
+                        iter.remove();
+                    }
+                }
+            }
 
             if (fut != null)
                 fut.onDone(MetadataUpdateResult.createSuccessfulResult());
@@ -527,6 +581,11 @@ private final class MetadataUpdateResultFuture extends GridFutureAdapter<Metadat
         void key(SyncKey key) {
             this.key = key;
         }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(MetadataUpdateResultFuture.class, this);
+        }
     }
 
     /**
@@ -580,6 +639,11 @@ int version() {
 
             return (typeId == that.typeId) && (ver == that.ver);
         }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(SyncKey.class, this);
+        }
     }
 
     /**
@@ -615,7 +679,7 @@ private final class MetadataRequestListener implements GridMessageListener {
                     binMetaBytes = U.marshal(ctx, metaHolder);
                 }
                 catch (IgniteCheckedException e) {
-                    U.error(log, "Failed to marshal binary metadata for [typeId: " + typeId + "]", e);
+                    U.error(log, "Failed to marshal binary metadata for [typeId=" + typeId + ']', e);
 
                     resp.markErrorOnRequest();
                 }
@@ -670,7 +734,8 @@ private final class MetadataResponseListener implements GridMessageListener {
                     do {
                         oldHolder = metaLocCache.get(typeId);
 
-                        if (oldHolder != null && obsoleteUpdate(
+                        // typeId metadata cannot be removed after initialization.
+                        if (obsoleteUpdate(
                                 oldHolder.pendingVersion(),
                                 oldHolder.acceptedVersion(),
                                 newHolder.pendingVersion(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 4c101b290fd54..137db9f887dc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -19,10 +19,14 @@
 
 import java.io.File;
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -30,6 +34,8 @@
 import org.apache.ignite.IgniteBinary;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.binary.BinaryField;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.binary.BinaryObjectBuilder;
@@ -39,8 +45,10 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.BinaryConfiguration;
 import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.UnregisteredBinaryTypeException;
 import org.apache.ignite.internal.binary.BinaryContext;
@@ -65,6 +73,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheUtils;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessorImpl;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.IgniteUtils;
@@ -76,7 +85,9 @@
 import org.apache.ignite.internal.util.typedef.T1;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiClosure;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteFuture;
@@ -88,7 +99,11 @@
 import org.apache.ignite.thread.IgniteThread;
 import org.jetbrains.annotations.Nullable;
 
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_TEST_FEATURES_ENABLED;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAIT_SCHEMA_UPDATE;
 import static org.apache.ignite.IgniteSystemProperties.getBoolean;
 import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
 import static org.apache.ignite.internal.GridComponent.DiscoveryDataExchangeType.BINARY_PROC;
@@ -120,6 +135,12 @@ public class CacheObjectBinaryProcessorImpl extends IgniteCacheObjectProcessorIm
      */
     @Nullable private File binaryMetadataFileStoreDir;
 
+    /** How long to wait for schema if no updates in progress. */
+    private long waitSchemaTimeout = IgniteSystemProperties.getLong(IGNITE_WAIT_SCHEMA_UPDATE, 30_000);
+
+    /** For tests. */
+    public static boolean useTestBinaryCtx = false;
+
     /** */
     @GridToStringExclude
     private IgniteBinary binaries;
@@ -205,7 +226,9 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
 
             BinaryMarshaller bMarsh0 = (BinaryMarshaller)marsh;
 
-            binaryCtx = new BinaryContext(metaHnd, ctx.config(), ctx.log(BinaryContext.class));
+            binaryCtx = useTestBinaryCtx ?
+                new TestBinaryContext(metaHnd, ctx.config(), ctx.log(BinaryContext.class)) :
+                new BinaryContext(metaHnd, ctx.config(), ctx.log(BinaryContext.class));
 
             IgniteUtils.invoke(BinaryMarshaller.class, bMarsh0, "setBinaryContext", binaryCtx, ctx.config());
 
@@ -452,11 +475,12 @@ public GridBinaryMarshaller marshaller() {
 
             BinaryMetadata oldMeta = metaHolder != null ? metaHolder.metadata() : null;
 
-            BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0);
+            Set<Integer> changedSchemas = new LinkedHashSet<>();
+
+            BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta0, changedSchemas);
 
-            //metadata requested to be added is exactly the same as already presented in the cache
-            if (mergedMeta == oldMeta)
-                return;
+            if (oldMeta != null && mergedMeta == oldMeta && metaHolder.pendingVersion() == metaHolder.acceptedVersion())
+                return; // Safe to use existing schemas.
 
             if (failIfUnregistered)
                 throw new UnregisteredBinaryTypeException(
@@ -466,7 +490,24 @@ public GridBinaryMarshaller marshaller() {
                         "dev-list.",
                     typeId, mergedMeta);
 
-            MetadataUpdateResult res = transport.requestMetadataUpdate(mergedMeta).get();
+            long t0 = System.nanoTime();
+
+            GridFutureAdapter<MetadataUpdateResult> fut = transport.requestMetadataUpdate(mergedMeta);
+
+            MetadataUpdateResult res = fut.get();
+
+            if (log.isDebugEnabled()) {
+                IgniteInternalTx tx = ctx.cache().context().tm().tx();
+
+                log.debug("Completed metadata update [typeId=" + typeId +
+                    ", typeName=" + newMeta.typeName() +
+                    ", changedSchemas=" + changedSchemas +
+                    ", waitTime=" + MILLISECONDS.convert(System.nanoTime() - t0, NANOSECONDS) + "ms" +
+                    ", holder=" + metaHolder +
+                    ", fut=" + fut +
+                    ", tx=" + CU.txString(tx) +
+                    ']');
+            }
 
             assert res != null;
 
@@ -541,9 +582,9 @@ public GridBinaryMarshaller marshaller() {
 
                 if (log.isDebugEnabled() && !fut.isDone())
                     log.debug("Waiting for update for" +
-                            " [typeId=" + typeId +
-                            ", pendingVer=" + holder.pendingVersion() +
-                            ", acceptedVer=" + holder.acceptedVersion() + "]");
+                        " [typeId=" + typeId +
+                        ", pendingVer=" + holder.pendingVersion() +
+                        ", acceptedVer=" + holder.acceptedVersion() + "]");
 
                 try {
                     fut.get();
@@ -565,40 +606,99 @@ public GridBinaryMarshaller marshaller() {
 
         if (ctx.clientNode()) {
             if (holder == null || !holder.metadata().hasSchema(schemaId)) {
+                if (log.isDebugEnabled())
+                    log.debug("Waiting for client metadata update" +
+                        " [typeId=" + typeId
+                        + ", schemaId=" + schemaId
+                        + ", pendingVer=" + (holder == null ? "NA" : holder.pendingVersion())
+                        + ", acceptedVer=" + (holder == null ? "NA" :holder.acceptedVersion()) + ']');
+
                 try {
                     transport.requestUpToDateMetadata(typeId).get();
-
-                    holder = metadataLocCache.get(typeId);
                 }
                 catch (IgniteCheckedException ignored) {
                     // No-op.
                 }
+
+                holder = metadataLocCache.get(typeId);
+
+                if (log.isDebugEnabled())
+                    log.debug("Finished waiting for client metadata update" +
+                        " [typeId=" + typeId
+                        + ", schemaId=" + schemaId
+                        + ", pendingVer=" + (holder == null ? "NA" : holder.pendingVersion())
+                        + ", acceptedVer=" + (holder == null ? "NA" :holder.acceptedVersion()) + ']');
             }
         }
-        else if (holder != null) {
-            if (IgniteThread.current() instanceof IgniteDiscoveryThread)
+        else {
+            if (holder != null && IgniteThread.current() instanceof IgniteDiscoveryThread)
                 return holder.metadata().wrap(binaryCtx);
+            else if (holder != null && (holder.pendingVersion() - holder.acceptedVersion() > 0)) {
+                if (log.isDebugEnabled())
+                    log.debug("Waiting for metadata update" +
+                        " [typeId=" + typeId
+                        + ", schemaId=" + schemaId
+                        + ", pendingVer=" + holder.pendingVersion()
+                        + ", acceptedVer=" + holder.acceptedVersion() + ']');
 
-            if (holder.pendingVersion() - holder.acceptedVersion() > 0) {
-                GridFutureAdapter<MetadataUpdateResult> fut = transport.awaitMetadataUpdate(
-                        typeId,
-                        holder.pendingVersion());
+                long t0 = System.nanoTime();
 
-                if (log.isDebugEnabled() && !fut.isDone())
-                    log.debug("Waiting for update for" +
-                            " [typeId=" + typeId
-                            + ", schemaId=" + schemaId
-                            + ", pendingVer=" + holder.pendingVersion()
-                            + ", acceptedVer=" + holder.acceptedVersion() + "]");
+                GridFutureAdapter<MetadataUpdateResult> fut = transport.awaitMetadataUpdate(
+                    typeId,
+                    holder.pendingVersion());
 
                 try {
                     fut.get();
                 }
+                catch (IgniteCheckedException e) {
+                    log.error("Failed to wait for metadata update [typeId=" + typeId + ", schemaId=" + schemaId + ']', e);
+                }
+
+                if (log.isDebugEnabled())
+                    log.debug("Finished waiting for metadata update" +
+                        " [typeId=" + typeId
+                        + ", waitTime=" + NANOSECONDS.convert(System.nanoTime() - t0, MILLISECONDS) + "ms"
+                        + ", schemaId=" + schemaId
+                        + ", pendingVer=" + holder.pendingVersion()
+                        + ", acceptedVer=" + holder.acceptedVersion() + ']');
+
+                holder = metadataLocCache.get(typeId);
+            }
+            else if (holder == null || !holder.metadata().hasSchema(schemaId)) {
+                // Last resort waiting.
+                U.warn(log,
+                    "Schema is missing while no metadata updates are in progress " +
+                        "(will wait for schema update within timeout defined by IGNITE_BINARY_META_UPDATE_TIMEOUT system property)" +
+                        " [typeId=" + typeId
+                        + ", missingSchemaId=" + schemaId
+                        + ", pendingVer=" + (holder == null ? "NA" : holder.pendingVersion())
+                        + ", acceptedVer=" + (holder == null ? "NA" : holder.acceptedVersion())
+                        + ", binMetaUpdateTimeout=" + waitSchemaTimeout +']');
+
+                long t0 = System.nanoTime();
+
+                GridFutureAdapter<?> fut = transport.awaitSchemaUpdate(typeId, schemaId);
+
+                try {
+                    fut.get(waitSchemaTimeout);
+                }
+                catch (IgniteFutureTimeoutCheckedException e) {
+                    log.error("Timed out while waiting for schema update [typeId=" + typeId + ", schemaId=" +
+                        schemaId + ']');
+                }
                 catch (IgniteCheckedException ignored) {
                     // No-op.
                 }
 
                 holder = metadataLocCache.get(typeId);
+
+                if (log.isDebugEnabled() && holder != null && holder.metadata().hasSchema(schemaId))
+                    log.debug("Found the schema after wait" +
+                        " [typeId=" + typeId
+                        + ", waitTime=" + NANOSECONDS.convert(System.nanoTime() - t0, MILLISECONDS) + "ms"
+                        + ", schemaId=" + schemaId
+                        + ", pendingVer=" + holder.pendingVersion()
+                        + ", acceptedVer=" + holder.acceptedVersion() + ']');
             }
         }
 
@@ -903,7 +1003,7 @@ else if (type == BinaryObjectImpl.TYPE_BINARY_ENUM)
         if ((res = validateBinaryConfiguration(rmtNode)) != null)
             return res;
 
-        return validateBinaryMetadata(rmtNode.id(), (Map<Integer, BinaryMetadataHolder>) discoData.joiningNodeData());
+        return validateBinaryMetadata(rmtNode.id(), (Map<Integer, BinaryMetadataHolder>)discoData.joiningNodeData());
     }
 
     /** */
@@ -1070,4 +1170,75 @@ private IgniteNodeValidationResult validateBinaryMetadata(UUID rmtNodeId, Map<In
     public void setBinaryMetadataFileStoreDir(@Nullable File binaryMetadataFileStoreDir) {
         this.binaryMetadataFileStoreDir = binaryMetadataFileStoreDir;
     }
+
+    /** */
+    public static class TestBinaryContext extends BinaryContext {
+        /** */
+        private List<TestBinaryContextListener> listeners;
+
+        /**
+         * @param metaHnd Meta handler.
+         * @param igniteCfg Ignite config.
+         * @param log Logger.
+         */
+        public TestBinaryContext(BinaryMetadataHandler metaHnd, IgniteConfiguration igniteCfg,
+            IgniteLogger log) {
+            super(metaHnd, igniteCfg, log);
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
+            BinaryType metadata = super.metadata(typeId);
+
+            if (listeners != null) {
+                for (TestBinaryContextListener listener : listeners)
+                    listener.onAfterMetadataRequest(typeId, metadata);
+            }
+
+            return metadata;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void updateMetadata(int typeId, BinaryMetadata meta,
+            boolean failIfUnregistered) throws BinaryObjectException {
+            if (listeners != null) {
+                for (TestBinaryContextListener listener : listeners)
+                    listener.onBeforeMetadataUpdate(typeId, meta);
+            }
+
+            super.updateMetadata(typeId, meta, failIfUnregistered);
+        }
+
+        /** */
+        public interface TestBinaryContextListener {
+            /**
+             * @param typeId Type id.
+             * @param type Type.
+             */
+            void onAfterMetadataRequest(int typeId, BinaryType type);
+
+            /**
+             * @param typeId Type id.
+             * @param metadata Metadata.
+             */
+            void onBeforeMetadataUpdate(int typeId, BinaryMetadata metadata);
+        }
+
+        /**
+         * @param lsnr Listener.
+         */
+        public void addListener(TestBinaryContextListener lsnr) {
+            if (listeners == null)
+                listeners = new ArrayList<>();
+
+            if (!listeners.contains(lsnr))
+                listeners.add(lsnr);
+        }
+
+        /** */
+        public void clearAllListener() {
+            if (listeners != null)
+                listeners.clear();
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
index 365213b6c6082..a4abd4ffec997 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoverySpi.java
@@ -417,6 +417,9 @@ public class TcpDiscoverySpi extends IgniteSpiAdapter implements IgniteDiscovery
     /** */
     private IgniteDiscoverySpiInternalListener internalLsnr;
 
+    /** For test purposes. */
+    private boolean skipAddrsRandomization = false;
+
     /**
      * Gets current SPI state.
      *
@@ -1813,7 +1816,7 @@ protected Collection<InetSocketAddress> resolvedAddresses() throws IgniteSpiExce
             }
         }
 
-        if (!res.isEmpty())
+        if (!res.isEmpty() && !skipAddrsRandomization)
             Collections.shuffle(res);
 
         return res;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java
index 84fd91656f9c2..41c588268a8f2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CachePageWriteLockUnlockTest.java
@@ -100,6 +100,8 @@ public void testPreloadPartition() throws Exception {
 
             grid0 = startGrid(0);
 
+            grid0.cluster().active(true);
+
             preloadPartition(grid0, DEFAULT_CACHE_NAME, PARTITION);
 
             Iterator<Cache.Entry<Object, Object>> it = grid0.cache(DEFAULT_CACHE_NAME).iterator();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
index de01844155c3b..46578606feb1d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
@@ -806,7 +806,12 @@ private void testEnlistMany(boolean write) throws Exception {
             tx.commit();
         }
         catch (Throwable t) {
-            assertTrue(X.hasCause(t, TransactionTimeoutException.class));
+            boolean timedOut = X.hasCause(t, TransactionTimeoutException.class);
+
+            if (!timedOut)
+                log.error("Got unexpected exception", t);
+
+            assertTrue(timedOut);
         }
 
         assertEquals(0, client.cache(CACHE_NAME).size());
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataConcurrentUpdateWithIndexesTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataConcurrentUpdateWithIndexesTest.java
new file mode 100644
index 0000000000000..05c876dd97680
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/BinaryMetadataConcurrentUpdateWithIndexesTest.java
@@ -0,0 +1,439 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.Field;
+import java.net.Socket;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.binary.BinaryObject;
+import org.apache.ignite.binary.BinaryObjectBuilder;
+import org.apache.ignite.binary.BinaryObjectException;
+import org.apache.ignite.binary.BinaryType;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.cache.QueryIndexType;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.binary.BinaryMetadata;
+import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
+import org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiClosure;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryCustomEventMessage;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Tests scenario for too early metadata update completion in case of multiple concurrent updates for the same schema.
+ * <p>
+ * Scenario is the following:
+ *
+ * <ul>
+ *     <li>Start 4 nodes, connect client to node 2 in topology order (starting from 1).</li>
+ *     <li>Start two concurrent transactions from client node producing same schema update.</li>
+ *     <li>Delay second update until first update will return to client with stamped propose message and writes new
+ *     schema to local metadata cache</li>
+ *     <li>Unblock second update. It should correctly wait until the metadata is applied on all
+ *     nodes or tx will fail on commit.</li>
+ * </ul>
+ */
+public class BinaryMetadataConcurrentUpdateWithIndexesTest extends GridCommonAbstractTest {
+    /** */
+    private static final int FIELDS = 2;
+
+    /** */
+    private static final int MB = 1024 * 1024;
+
+    /** */
+    private static final TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+        cfg.setIncludeEventTypes(EventType.EVTS_DISCOVERY);
+
+        BlockTcpDiscoverySpi spi = new BlockTcpDiscoverySpi();
+
+        Field rndAddrsField = U.findField(BlockTcpDiscoverySpi.class, "skipAddrsRandomization");
+
+        assertNotNull(rndAddrsField);
+
+        rndAddrsField.set(spi, true);
+
+        cfg.setDiscoverySpi(spi.setIpFinder(ipFinder));
+
+        cfg.setClientMode(igniteInstanceName.startsWith("client"));
+
+        QueryEntity qryEntity = new QueryEntity("java.lang.Integer", "Value");
+
+        LinkedHashMap<String, String> fields = new LinkedHashMap<>();
+
+        Collection<QueryIndex> indexes = new ArrayList<>(FIELDS);
+
+        for (int i = 0; i < FIELDS; i++) {
+            String name = "s" + i;
+
+            fields.put(name, "java.lang.String");
+
+            indexes.add(new QueryIndex(name, QueryIndexType.SORTED));
+        }
+
+        qryEntity.setFields(fields);
+
+        qryEntity.setIndexes(indexes);
+
+        cfg.setDataStorageConfiguration(new DataStorageConfiguration().
+            setDefaultDataRegionConfiguration(new DataRegionConfiguration().setMaxSize(50 * MB)));
+
+        cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME).
+            setBackups(0).
+            setQueryEntities(Collections.singleton(qryEntity)).
+            setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL).
+            setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC).
+            setCacheMode(CacheMode.PARTITIONED));
+
+        return cfg;
+    }
+
+    /** Flag to start syncing metadata requests. Should skip on exchange. */
+    private volatile boolean syncMeta;
+
+    /** Metadata init latch. Both threads must request initial metadata. */
+    private CountDownLatch initMetaReq = new CountDownLatch(2);
+
+    /** Thread local flag for need of waiting local metadata update. */
+    private ThreadLocal<Boolean> delayMetadataUpdateThreadLoc = new ThreadLocal<>();
+
+    /** Latch for waiting local metadata update. */
+    public static final CountDownLatch localMetaUpdatedLatch = new CountDownLatch(1);
+
+    /** */
+    public void testMissingSchemaUpdate() throws Exception {
+        // Start order is important.
+        Ignite node0 = startGrid("node0");
+
+        Ignite node1 = startGrid("node1");
+
+        IgniteEx client0 = (IgniteEx)startGrid("client0");
+
+        CacheObjectBinaryProcessorImpl.TestBinaryContext clientCtx =
+            (CacheObjectBinaryProcessorImpl.TestBinaryContext)((CacheObjectBinaryProcessorImpl)client0.context().
+                cacheObjects()).binaryContext();
+
+        clientCtx.addListener(new CacheObjectBinaryProcessorImpl.TestBinaryContext.TestBinaryContextListener() {
+            @Override public void onAfterMetadataRequest(int typeId, BinaryType type) {
+                if (syncMeta) {
+                    try {
+                        initMetaReq.countDown();
+
+                        initMetaReq.await();
+                    }
+                    catch (Exception e) {
+                        throw new BinaryObjectException(e);
+                    }
+                }
+            }
+
+            @Override public void onBeforeMetadataUpdate(int typeId, BinaryMetadata metadata) {
+                // Delay one of updates until schema is locally updated on propose message.
+                if (delayMetadataUpdateThreadLoc.get() != null)
+                    await(localMetaUpdatedLatch, 5000);
+            }
+        });
+
+        Ignite node2 = startGrid("node2");
+
+        Ignite node3 = startGrid("node3");
+
+        startGrid("node4");
+
+        node0.cluster().active(true);
+
+        awaitPartitionMapExchange();
+
+        syncMeta = true;
+
+        CountDownLatch clientProposeMsgBlockedLatch = new CountDownLatch(1);
+
+        AtomicBoolean clientWait = new AtomicBoolean();
+        final Object clientMux = new Object();
+
+        AtomicBoolean srvWait = new AtomicBoolean();
+        final Object srvMux = new Object();
+
+        ((BlockTcpDiscoverySpi)node1.configuration().getDiscoverySpi()).setClosure((snd, msg) -> {
+            if (msg instanceof MetadataUpdateProposedMessage) {
+                if (Thread.currentThread().getName().contains("client")) {
+                    log.info("Block custom message to client0: [locNode=" + snd + ", msg=" + msg + ']');
+
+                    clientProposeMsgBlockedLatch.countDown();
+
+                    // Message to client
+                    synchronized (clientMux) {
+                        while (!clientWait.get())
+                            try {
+                                clientMux.wait();
+                            }
+                            catch (InterruptedException e) {
+                                fail();
+                            }
+                    }
+                }
+            }
+
+            return null;
+        });
+
+        ((BlockTcpDiscoverySpi)node2.configuration().getDiscoverySpi()).setClosure((snd, msg) -> {
+            if (msg instanceof MetadataUpdateProposedMessage) {
+                MetadataUpdateProposedMessage msg0 = (MetadataUpdateProposedMessage)msg;
+
+                int pendingVer = U.field(msg0, "pendingVer");
+
+                // Should not block propose messages until they reach coordinator.
+                if (pendingVer == 0)
+                    return null;
+
+                log.info("Block custom message to next server: [locNode=" + snd + ", msg=" + msg + ']');
+
+                // Message to client
+                synchronized (srvMux) {
+                    while (!srvWait.get())
+                        try {
+                            srvMux.wait();
+                        }
+                        catch (InterruptedException e) {
+                            fail();
+                        }
+                }
+            }
+
+            return null;
+        });
+
+        Integer key = primaryKey(node3.cache(DEFAULT_CACHE_NAME));
+
+        IgniteInternalFuture fut0 = runAsync(() -> {
+            try (Transaction tx = client0.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+                client0.cache(DEFAULT_CACHE_NAME).put(key, build(client0, "val", 0));
+
+                tx.commit();
+            }
+            catch (Throwable t) {
+                log.error("err", t);
+            }
+
+        });
+
+        // Implements test logic.
+        IgniteInternalFuture fut1 = runAsync(() -> {
+            // Wait for initial metadata received. It should be initial version: pending=0, accepted=0
+            await(initMetaReq, 5000);
+
+            // Wait for blocking proposal message to client node.
+            await(clientProposeMsgBlockedLatch, 5000);
+
+            // Unblock proposal message to client.
+            clientWait.set(true);
+
+            synchronized (clientMux) {
+                clientMux.notify();
+            }
+
+            // Give some time to apply update.
+            doSleep(3000);
+
+            // Unblock second metadata update.
+            localMetaUpdatedLatch.countDown();
+
+            // Give some time for tx to complete (success or fail). fut2 will throw an error if tx has failed on commit.
+            doSleep(3000);
+
+            // Unblock metadata message and allow for correct version acceptance.
+            srvWait.set(true);
+
+            synchronized (srvMux) {
+                srvMux.notify();
+            }
+        });
+
+        IgniteInternalFuture fut2 = runAsync(() -> {
+            delayMetadataUpdateThreadLoc.set(true);
+
+            try (Transaction tx = client0.transactions().
+                txStart(PESSIMISTIC, REPEATABLE_READ, 0, 1)) {
+                client0.cache(DEFAULT_CACHE_NAME).put(key, build(client0, "val", 0));
+
+                tx.commit();
+            }
+        });
+
+        fut0.get();
+        fut1.get();
+        fut2.get();
+    }
+
+    /**
+     * @param latch Latch.
+     * @param timeout Timeout.
+     */
+    private void await(CountDownLatch latch, long timeout) {
+        try {
+            latch.await(5000, MILLISECONDS);
+        }
+        catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+
+        long cnt = initMetaReq.getCount();
+
+        if (cnt != 0)
+            throw new RuntimeException("Invalid latch count after wait: " + cnt);
+    }
+
+    /**
+     * @param ignite Ignite.
+     * @param prefix Value prefix.
+     * @param fields Fields.
+     */
+    protected BinaryObject build(Ignite ignite, String prefix, int... fields) {
+        BinaryObjectBuilder builder = ignite.binary().builder("Value");
+
+        for (int field : fields) {
+            assertTrue(field < FIELDS);
+
+            builder.setField("i" + field, field);
+            builder.setField("s" + field, prefix + field);
+        }
+
+        return builder.build();
+    }
+
+    /**
+     * Discovery SPI which can simulate network split.
+     */
+    protected class BlockTcpDiscoverySpi extends TcpDiscoverySpi {
+        /** Closure. */
+        private volatile IgniteBiClosure<ClusterNode, DiscoveryCustomMessage, Void> clo;
+
+        /**
+         * @param clo Closure.
+         */
+        public void setClosure(IgniteBiClosure<ClusterNode, DiscoveryCustomMessage, Void> clo) {
+            this.clo = clo;
+        }
+
+        /**
+         * @param addr Address.
+         * @param msg Message.
+         */
+        private synchronized void apply(ClusterNode addr, TcpDiscoveryAbstractMessage msg) {
+            if (!(msg instanceof TcpDiscoveryCustomEventMessage))
+                return;
+
+            TcpDiscoveryCustomEventMessage cm = (TcpDiscoveryCustomEventMessage)msg;
+
+            DiscoveryCustomMessage delegate;
+
+            try {
+                DiscoverySpiCustomMessage custMsg = cm.message(marshaller(), U.resolveClassLoader(ignite().configuration()));
+
+                assertNotNull(custMsg);
+
+                delegate = ((CustomMessageWrapper)custMsg).delegate();
+
+            }
+            catch (Throwable throwable) {
+                throw new RuntimeException(throwable);
+            }
+
+            if (clo != null)
+                clo.apply(addr, delegate);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(
+            Socket sock,
+            TcpDiscoveryAbstractMessage msg,
+            byte[] data,
+            long timeout
+        ) throws IOException {
+            if (spiCtx != null)
+                apply(spiCtx.localNode(), msg);
+
+            super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock,
+            OutputStream out,
+            TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (spiCtx != null)
+                apply(spiCtx.localNode(), msg);
+
+            super.writeToSocket(sock, out, msg, timeout);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        CacheObjectBinaryProcessorImpl.useTestBinaryCtx = true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        CacheObjectBinaryProcessorImpl.useTestBinaryCtx = false;
+
+        stopAllGrids();
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
index b44ff2dbc3517..8a60c7d72ac70 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.BinaryMetadataConcurrentUpdateWithIndexesTest;
 import org.apache.ignite.internal.processors.cache.BinarySerializationQuerySelfTest;
 import org.apache.ignite.internal.processors.cache.BinarySerializationQueryWithReflectiveSerializerSelfTest;
 import org.apache.ignite.internal.processors.cache.IgniteCacheBinaryObjectsScanSelfTest;
@@ -42,6 +43,8 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(IgniteCacheBinaryObjectsScanWithEventsSelfTest.class);
         suite.addTestSuite(BigEntryQueryTest.class);
 
+        suite.addTestSuite(BinaryMetadataConcurrentUpdateWithIndexesTest.class);
+
         //Should be adjusted. Not ready to be used with BinaryMarshaller.
         //suite.addTestSuite(GridCacheBinarySwapScanQuerySelfTest.class);
 

From 43a6c739cec78d8920e199f4b1f44983c00fd008 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Fri, 19 Oct 2018 12:45:06 +0300
Subject: [PATCH 438/543] GG-14341 Fix compilation and license.

---
 ...ridDhtPartitionsStateValidatorBenchmark.java | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java
index 151606dab21ba..f3bbcb96d6cb9 100644
--- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java
+++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/misc/GridDhtPartitionsStateValidatorBenchmark.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.ignite.internal.benchmarks.jmh.misc;
 
 import com.google.common.collect.Lists;

From b8cdc405d335c867ae8d3d5fda6f48f19f46e17a Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Fri, 19 Oct 2018 16:26:33 +0300
Subject: [PATCH 439/543] IGNITE-9082 Throwing checked exception during tx
 commit without node stopping leads to data corruption - Fixes #4809.

Signed-off-by: Ivan Rakov <irakov@apache.org>

# Conflicts:
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
#	modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
#	modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java
---
 .../processors/cache/GridCacheMapEntry.java   |   4 +-
 .../cache/GridCacheSharedContext.java         |   9 +
 .../GridDistributedTxRemoteAdapter.java       | 439 +++++++------
 .../dht/GridDhtTxFinishFuture.java            |  11 +-
 .../cache/distributed/dht/GridDhtTxLocal.java |  10 +-
 .../dht/GridDhtTxPrepareFuture.java           |  59 +-
 .../near/GridNearTxFinishFuture.java          |  42 +-
 .../distributed/near/GridNearTxLocal.java     |   8 +-
 .../cache/transactions/IgniteTxAdapter.java   |  33 +
 .../cache/transactions/IgniteTxHandler.java   | 119 ++--
 .../transactions/IgniteTxLocalAdapter.java    | 575 +++++++++---------
 .../processors/failure/FailureProcessor.java  |   8 +
 .../apache/ignite/spi/IgniteSpiAdapter.java   |   1 -
 .../cache/GridCacheAbstractSelfTest.java      |   9 +-
 .../cache/query/IndexingSpiQuerySelfTest.java |  66 +-
 .../query/IndexingSpiQueryTxSelfTest.java     |  74 ++-
 .../AbstractTransactionIntergrityTest.java    | 111 ++--
 ...tegrityWithPrimaryIndexCorruptionTest.java | 268 ++++----
 ...ionIntegrityWithSystemWorkerDeathTest.java |   6 +-
 .../TxDataConsistencyOnCommitFailureTest.java | 234 +++++++
 .../junits/common/GridCommonAbstractTest.java |  19 +-
 .../testsuites/IgniteCacheTestSuite9.java     |  57 ++
 22 files changed, 1264 insertions(+), 898 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDataConsistencyOnCommitFailureTest.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 4d39e155eb933..0f96b1935a75e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -2382,8 +2382,6 @@ protected final boolean markObsolete0(GridCacheVersion ver, boolean clear, GridC
             ver = newVer;
             flags &= ~IS_EVICT_DISABLED;
 
-            removeValue();
-
             onInvalidate();
 
             return obsoleteVersionExtras() != null;
@@ -3608,7 +3606,7 @@ protected boolean storeValue(@Nullable CacheObject val,
     }
 
     /**
-     * Stores value in offheap.
+     * Stores value in off-heap.
      *
      * @param val Value.
      * @param expireTime Expire time.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index 6d30407859f3c..a97d431c8a58b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -1119,4 +1119,13 @@ public boolean readOnlyMode() {
     public void readOnlyMode(boolean readOnlyMode) {
         this.readOnlyMode = readOnlyMode;
     }
+
+    /**
+     * For test purposes.
+     * @param txMgr Tx manager.
+     */
+    public void setTxManager(IgniteTxManager txMgr) {
+        this.txMgr = txMgr;
+    }
+
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index c1293fc0772bd..4f05ec1195f20 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -32,7 +32,6 @@
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.InvalidEnvironmentException;
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
@@ -51,7 +50,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheEntry;
-import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxAdapter;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
@@ -502,166 +500,127 @@ private void commitIfLocked() throws IgniteCheckedException {
 
                                 boolean replicate = cacheCtx.isDrEnabled();
 
-                                try {
-                                    while (true) {
-                                        try {
-                                            GridCacheEntryEx cached = txEntry.cached();
+                                while (true) {
+                                    try {
+                                        GridCacheEntryEx cached = txEntry.cached();
 
-                                            if (cached == null)
-                                                txEntry.cached(cached = cacheCtx.cache().entryEx(txEntry.key(), topologyVersion()));
+                                        if (cached == null)
+                                            txEntry.cached(cached = cacheCtx.cache().entryEx(txEntry.key(), topologyVersion()));
 
-                                            if (near() && cacheCtx.dr().receiveEnabled()) {
-                                                cached.markObsolete(xidVer);
+                                        if (near() && cacheCtx.dr().receiveEnabled()) {
+                                            cached.markObsolete(xidVer);
 
-                                                break;
-                                            }
-
-                                            GridNearCacheEntry nearCached = null;
+                                            break;
+                                        }
 
-                                            if (updateNearCache(cacheCtx, txEntry.key(), topVer))
-                                                nearCached = cacheCtx.dht().near().peekExx(txEntry.key());
+                                        GridNearCacheEntry nearCached = null;
 
-                                            if (!F.isEmpty(txEntry.entryProcessors()))
-                                                txEntry.cached().unswap(false);
+                                        if (updateNearCache(cacheCtx, txEntry.key(), topVer))
+                                            nearCached = cacheCtx.dht().near().peekExx(txEntry.key());
 
-                                            IgniteBiTuple<GridCacheOperation, CacheObject> res =
-                                                applyTransformClosures(txEntry, false, ret);
+                                        if (!F.isEmpty(txEntry.entryProcessors()))
+                                            txEntry.cached().unswap(false);
 
-                                            GridCacheOperation op = res.get1();
-                                            CacheObject val = res.get2();
+                                        IgniteBiTuple<GridCacheOperation, CacheObject> res =
+                                            applyTransformClosures(txEntry, false, ret);
 
-                                            GridCacheVersion explicitVer = txEntry.conflictVersion();
+                                        GridCacheOperation op = res.get1();
+                                        CacheObject val = res.get2();
 
-                                            if (explicitVer == null)
-                                                explicitVer = writeVersion();
+                                        GridCacheVersion explicitVer = txEntry.conflictVersion();
 
-                                            if (txEntry.ttl() == CU.TTL_ZERO)
-                                                op = DELETE;
+                                        if (explicitVer == null)
+                                            explicitVer = writeVersion();
 
-                                            boolean conflictNeedResolve = cacheCtx.conflictNeedResolve();
+                                        if (txEntry.ttl() == CU.TTL_ZERO)
+                                            op = DELETE;
 
-                                            GridCacheVersionConflictContext conflictCtx = null;
+                                        boolean conflictNeedResolve = cacheCtx.conflictNeedResolve();
 
-                                            if (conflictNeedResolve) {
-                                                IgniteBiTuple<GridCacheOperation, GridCacheVersionConflictContext>
-                                                    drRes = conflictResolve(op, txEntry, val, explicitVer, cached);
+                                        GridCacheVersionConflictContext conflictCtx = null;
 
-                                                assert drRes != null;
+                                        if (conflictNeedResolve) {
+                                            IgniteBiTuple<GridCacheOperation, GridCacheVersionConflictContext>
+                                                drRes = conflictResolve(op, txEntry, val, explicitVer, cached);
 
-                                                conflictCtx = drRes.get2();
+                                            assert drRes != null;
 
-                                                if (conflictCtx.isUseOld())
-                                                    op = NOOP;
-                                                else if (conflictCtx.isUseNew()) {
-                                                    txEntry.ttl(conflictCtx.ttl());
-                                                    txEntry.conflictExpireTime(conflictCtx.expireTime());
-                                                }
-                                                else if (conflictCtx.isMerge()) {
-                                                    op = drRes.get1();
-                                                    val = txEntry.context().toCacheObject(conflictCtx.mergeValue());
-                                                    explicitVer = writeVersion();
+                                            conflictCtx = drRes.get2();
 
-                                                    txEntry.ttl(conflictCtx.ttl());
-                                                    txEntry.conflictExpireTime(conflictCtx.expireTime());
-                                                }
-                                            }
-                                            else
-                                                // Nullify explicit version so that innerSet/innerRemove will work as usual.
-                                                explicitVer = null;
-
-                                            GridCacheVersion dhtVer = cached.isNear() ? writeVersion() : null;
-
-                                            if (!near() && cacheCtx.group().persistenceEnabled() && cacheCtx.group().walEnabled() &&
-                                                op != NOOP && op != RELOAD && (op != READ || cctx.snapshot().needTxReadLogging())) {
-                                                if (dataEntries == null)
-                                                    dataEntries = new ArrayList<>(entries.size());
-
-                                                dataEntries.add(
-                                                        new T2<>(
-                                                                new DataEntry(
-                                                                        cacheCtx.cacheId(),
-                                                                        txEntry.key(),
-                                                                        val,
-                                                                        op,
-                                                                        nearXidVersion(),
-                                                                        writeVersion(),
-                                                                        0,
-                                                                        txEntry.key().partition(),
-                                                                        txEntry.updateCounter()
-                                                                ),
-                                                                txEntry
-                                                        )
-                                                );
+                                            if (conflictCtx.isUseOld())
+                                                op = NOOP;
+                                            else if (conflictCtx.isUseNew()) {
+                                                txEntry.ttl(conflictCtx.ttl());
+                                                txEntry.conflictExpireTime(conflictCtx.expireTime());
                                             }
+                                            else if (conflictCtx.isMerge()) {
+                                                op = drRes.get1();
+                                                val = txEntry.context().toCacheObject(conflictCtx.mergeValue());
+                                                explicitVer = writeVersion();
 
-                                            if (op == CREATE || op == UPDATE) {
-                                                // Invalidate only for near nodes (backups cannot be invalidated).
-                                                if (isSystemInvalidate() || (isInvalidate() && cacheCtx.isNear()))
-                                                    cached.innerRemove(this,
-                                                        eventNodeId(),
-                                                        nodeId,
-                                                        false,
-                                                        true,
-                                                        true,
-                                                        txEntry.keepBinary(),
-                                                        txEntry.hasOldValue(),
-                                                        txEntry.oldValue(),
-                                                        topVer,
-                                                        null,
-                                                        replicate ? DR_BACKUP : DR_NONE,
-                                                        near() ? null : explicitVer,
-                                                        CU.subjectId(this, cctx),
-                                                        resolveTaskName(),
-                                                        dhtVer,
-                                                        txEntry.updateCounter());
-                                                else {
-                                                    assert val != null : txEntry;
-
-                                                    GridCacheUpdateTxResult updRes = cached.innerSet(this,
-                                                        eventNodeId(),
-                                                        nodeId,
-                                                        val,
-                                                        false,
-                                                        false,
-                                                        txEntry.ttl(),
-                                                        true,
-                                                        true,
-                                                        txEntry.keepBinary(),
-                                                        txEntry.hasOldValue(),
-                                                        txEntry.oldValue(),
-                                                        topVer,
-                                                        null,
-                                                        replicate ? DR_BACKUP : DR_NONE,
-                                                        txEntry.conflictExpireTime(),
-                                                        near() ? null : explicitVer,
-                                                        CU.subjectId(this, cctx),
-                                                        resolveTaskName(),
-                                                        dhtVer,
-                                                        txEntry.updateCounter());
-
-                                                    txEntry.updateCounter(updRes.updatePartitionCounter());
-
-                                                    if (updRes.loggedPointer() != null)
-                                                        ptr = updRes.loggedPointer();
-
-                                                    // Keep near entry up to date.
-                                                    if (nearCached != null) {
-                                                        CacheObject val0 = cached.valueBytes();
-
-                                                        nearCached.updateOrEvict(xidVer,
-                                                            val0,
-                                                            cached.expireTime(),
-                                                            cached.ttl(),
-                                                            nodeId,
-                                                            topVer);
-                                                    }
-                                                }
+                                                txEntry.ttl(conflictCtx.ttl());
+                                                txEntry.conflictExpireTime(conflictCtx.expireTime());
                                             }
-                                            else if (op == DELETE) {
-                                                GridCacheUpdateTxResult updRes = cached.innerRemove(this,
+                                        }
+                                        else
+                                            // Nullify explicit version so that innerSet/innerRemove will work as usual.
+                                            explicitVer = null;
+
+                                        GridCacheVersion dhtVer = cached.isNear() ? writeVersion() : null;
+
+                                        if (!near() && cacheCtx.group().persistenceEnabled() && cacheCtx.group().walEnabled() &&
+                                            op != NOOP && op != RELOAD && (op != READ || cctx.snapshot().needTxReadLogging())) {
+                                            if (dataEntries == null)
+                                                dataEntries = new ArrayList<>(entries.size());
+
+                                            dataEntries.add(
+                                                new T2<>(
+                                                    new DataEntry(
+                                                        cacheCtx.cacheId(),
+                                                        txEntry.key(),
+                                                        val,
+                                                        op,
+                                                        nearXidVersion(),
+                                                        writeVersion(),
+                                                        0,
+                                                        txEntry.key().partition(),
+                                                        txEntry.updateCounter()
+                                                    ),
+                                                    txEntry
+                                                )
+                                            );
+                                        }
+
+                                        if (op == CREATE || op == UPDATE) {
+                                            // Invalidate only for near nodes (backups cannot be invalidated).
+                                            if (isSystemInvalidate() || (isInvalidate() && cacheCtx.isNear()))
+                                                cached.innerRemove(this,
+                                                    eventNodeId(),
+                                                    nodeId,
+                                                    false,
+                                                    true,
+                                                    true,
+                                                    txEntry.keepBinary(),
+                                                    txEntry.hasOldValue(),
+                                                    txEntry.oldValue(),
+                                                    topVer,
+                                                    null,
+                                                    replicate ? DR_BACKUP : DR_NONE,
+                                                    near() ? null : explicitVer,
+                                                    CU.subjectId(this, cctx),
+                                                    resolveTaskName(),
+                                                    dhtVer,
+                                                    txEntry.updateCounter());
+                                            else {
+                                                assert val != null : txEntry;
+
+                                                GridCacheUpdateTxResult updRes = cached.innerSet(this,
                                                     eventNodeId(),
                                                     nodeId,
+                                                    val,
                                                     false,
+                                                    false,
+                                                    txEntry.ttl(),
                                                     true,
                                                     true,
                                                     txEntry.keepBinary(),
@@ -670,6 +629,7 @@ else if (op == DELETE) {
                                                     topVer,
                                                     null,
                                                     replicate ? DR_BACKUP : DR_NONE,
+                                                    txEntry.conflictExpireTime(),
                                                     near() ? null : explicitVer,
                                                     CU.subjectId(this, cctx),
                                                     resolveTaskName(),
@@ -682,111 +642,105 @@ else if (op == DELETE) {
                                                     ptr = updRes.loggedPointer();
 
                                                 // Keep near entry up to date.
-                                                if (nearCached != null)
-                                                    nearCached.updateOrEvict(xidVer, null, 0, 0, nodeId, topVer);
-                                            }
-                                            else if (op == RELOAD) {
-                                                CacheObject reloaded = cached.innerReload();
-
                                                 if (nearCached != null) {
-                                                    nearCached.innerReload();
+                                                    CacheObject val0 = cached.valueBytes();
 
-                                                    nearCached.updateOrEvict(cached.version(),
-                                                        reloaded,
+                                                    nearCached.updateOrEvict(xidVer,
+                                                        val0,
                                                         cached.expireTime(),
                                                         cached.ttl(),
                                                         nodeId,
                                                         topVer);
                                                 }
                                             }
-                                            else if (op == READ) {
-                                                assert near();
+                                        }
+                                        else if (op == DELETE) {
+                                            GridCacheUpdateTxResult updRes = cached.innerRemove(this,
+                                                eventNodeId(),
+                                                nodeId,
+                                                false,
+                                                true,
+                                                true,
+                                                txEntry.keepBinary(),
+                                                txEntry.hasOldValue(),
+                                                txEntry.oldValue(),
+                                                topVer,
+                                                null,
+                                                replicate ? DR_BACKUP : DR_NONE,
+                                                near() ? null : explicitVer,
+                                                CU.subjectId(this, cctx),
+                                                resolveTaskName(),
+                                                dhtVer,
+                                                txEntry.updateCounter());
+
+                                            txEntry.updateCounter(updRes.updatePartitionCounter());
+
+                                            if (updRes.loggedPointer() != null)
+                                                ptr = updRes.loggedPointer();
+
+                                            // Keep near entry up to date.
+                                            if (nearCached != null)
+                                                nearCached.updateOrEvict(xidVer, null, 0, 0, nodeId, topVer);
+                                        }
+                                        else if (op == RELOAD) {
+                                            CacheObject reloaded = cached.innerReload();
 
-                                                if (log.isDebugEnabled())
-                                                    log.debug("Ignoring READ entry when committing: " + txEntry);
-                                            }
-                                            // No-op.
-                                            else {
-                                                if (conflictCtx == null || !conflictCtx.isUseOld()) {
-                                                    if (txEntry.ttl() != CU.TTL_NOT_CHANGED)
-                                                        cached.updateTtl(null, txEntry.ttl());
-
-                                                    if (nearCached != null) {
-                                                        CacheObject val0 = cached.valueBytes();
-
-                                                        nearCached.updateOrEvict(xidVer,
-                                                            val0,
-                                                            cached.expireTime(),
-                                                            cached.ttl(),
-                                                            nodeId,
-                                                            topVer);
-                                                    }
-                                                }
-                                            }
+                                            if (nearCached != null) {
+                                                nearCached.innerReload();
 
-                                            // Assert after setting values as we want to make sure
-                                            // that if we replaced removed entries.
-                                            assert
-                                                txEntry.op() == READ || onePhaseCommit() ||
-                                                    // If candidate is not there, then lock was explicit
-                                                    // and we simply allow the commit to proceed.
-                                                    !cached.hasLockCandidateUnsafe(xidVer) || cached.lockedByUnsafe(xidVer) :
-                                                "Transaction does not own lock for commit [entry=" + cached +
-                                                    ", tx=" + this + ']';
-
-                                            // Break out of while loop.
-                                            break;
+                                                nearCached.updateOrEvict(cached.version(),
+                                                    reloaded,
+                                                    cached.expireTime(),
+                                                    cached.ttl(),
+                                                    nodeId,
+                                                    topVer);
+                                            }
                                         }
-                                        catch (GridCacheEntryRemovedException ignored) {
-                                            if (log.isDebugEnabled())
-                                                log.debug("Attempting to commit a removed entry (will retry): " + txEntry);
+                                        else if (op == READ) {
+                                            assert near();
 
-                                            // Renew cached entry.
-                                            txEntry.cached(cacheCtx.cache().entryEx(txEntry.key(), topologyVersion()));
+                                            if (log.isDebugEnabled())
+                                                log.debug("Ignoring READ entry when committing: " + txEntry);
                                         }
-                                    }
-                                }
-                                catch (Throwable ex) {
-                                    boolean isNodeStopping = X.hasCause(ex, NodeStoppingException.class);
-                                    boolean hasInvalidEnvironmentIssue = X.hasCause(ex, InvalidEnvironmentException.class);
-
-                                    // In case of error, we still make the best effort to commit,
-                                    // as there is no way to rollback at this point.
-                                    err = new IgniteTxHeuristicCheckedException("Commit produced a runtime exception " +
-                                        "(all transaction entries will be invalidated): " + CU.txString(this), ex);
-
-                                    if (isNodeStopping) {
-                                        U.warn(log, "Failed to commit transaction, node is stopping [tx=" + this +
-                                            ", err=" + ex + ']');
-                                    }
-                                    else if (hasInvalidEnvironmentIssue) {
-                                        U.warn(log, "Failed to commit transaction, node is in invalid state and will be stopped [tx=" + this +
-                                            ", err=" + ex + ']');
-                                    }
-                                    else
-                                        U.error(log, "Commit failed.", err);
+                                        // No-op.
+                                        else {
+                                            if (conflictCtx == null || !conflictCtx.isUseOld()) {
+                                                if (txEntry.ttl() != CU.TTL_NOT_CHANGED)
+                                                    cached.updateTtl(null, txEntry.ttl());
 
-                                    state(UNKNOWN);
+                                                if (nearCached != null) {
+                                                    CacheObject val0 = cached.valueBytes();
 
-                                    if (hasInvalidEnvironmentIssue)
-                                        cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
-                                    else if (!isNodeStopping) { // Skip fair uncommit in case of node stopping or invalidation.
-                                        try {
-                                            // Courtesy to minimize damage.
-                                            uncommit();
+                                                    nearCached.updateOrEvict(xidVer,
+                                                        val0,
+                                                        cached.expireTime(),
+                                                        cached.ttl(),
+                                                        nodeId,
+                                                        topVer);
+                                                }
+                                            }
                                         }
-                                        catch (Throwable ex1) {
-                                            U.error(log, "Failed to uncommit transaction: " + this, ex1);
 
-                                            if (ex1 instanceof Error)
-                                                throw ex1;
-                                        }
+                                        // Assert after setting values as we want to make sure
+                                        // that if we replaced removed entries.
+                                        assert
+                                            txEntry.op() == READ || onePhaseCommit() ||
+                                                // If candidate is not there, then lock was explicit
+                                                // and we simply allow the commit to proceed.
+                                                !cached.hasLockCandidateUnsafe(xidVer) || cached.lockedByUnsafe(xidVer) :
+                                            "Transaction does not own lock for commit [entry=" + cached +
+                                                ", tx=" + this + ']';
+
+                                        // Break out of while loop.
+                                        break;
                                     }
+                                    catch (GridCacheEntryRemovedException ignored) {
+                                        if (log.isDebugEnabled())
+                                            log.debug("Attempting to commit a removed entry (will retry): " + txEntry);
 
-                                    if (ex instanceof Error)
-                                        throw (Error) ex;
-
-                                    throw err;
+                                        // Renew cached entry.
+                                        txEntry.cached(cacheCtx.cache().entryEx(txEntry.key(), topologyVersion()));
+                                    }
                                 }
                             }
 
@@ -802,9 +756,26 @@ else if (!isNodeStopping) { // Skip fair uncommit in case of node stopping or in
                             if (ptr != null && !cctx.tm().logTxRecords())
                                 cctx.wal().flush(ptr, false);
                         }
-                        catch (StorageException e) {
-                            throw new IgniteCheckedException("Failed to log transaction record " +
-                                "(transaction will be rolled back): " + this, e);
+                        catch (Throwable ex) {
+                            state(UNKNOWN);
+
+                            if (X.hasCause(ex, NodeStoppingException.class)) {
+                                U.warn(log, "Failed to commit transaction, node is stopping [tx=" + CU.txString(this) +
+                                    ", err=" + ex + ']');
+
+                                return;
+                            }
+
+                            err = heuristicException(ex);
+
+                            try {
+                                uncommit();
+                            }
+                            catch (Throwable e) {
+                                err.addSuppressed(e);
+                            }
+
+                            throw err;
                         }
                     }
                     finally {
@@ -843,9 +814,19 @@ else if (!isNodeStopping) { // Skip fair uncommit in case of node stopping or in
                 throw new IgniteCheckedException("Invalid transaction state for commit [state=" + state + ", tx=" + this + ']');
 
             rollbackRemoteTx();
+
+            return;
         }
 
-        commitIfLocked();
+        try {
+            commitIfLocked();
+        }
+        catch (IgniteTxHeuristicCheckedException e) {
+            // Treat heuristic exception as critical.
+            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+
+            throw e;
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index 2f36053b9d838..e98893ca2d335 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -169,10 +169,13 @@ public void rollbackOnError(Throwable e) {
         if (ERR_UPD.compareAndSet(this, null, e)) {
             tx.setRollbackOnly();
 
-            if (X.hasCause(e, InvalidEnvironmentException.class, NodeStoppingException.class))
+            if (X.hasCause(e, NodeStoppingException.class) || cctx.kernalContext().failure().nodeStopping())
                 onComplete();
-            else
+            else {
+                // Rolling back a remote transaction may result in partial commit.
+                // This is only acceptable in tests with no-op failure handler.
                 finish(false);
+            }
         }
     }
 
@@ -226,9 +229,9 @@ public void onResult(UUID nodeId, GridDhtTxFinishResponse res) {
 
             if (this.tx.onePhaseCommit() && (this.tx.state() == COMMITTING)) {
                 try {
-                    boolean hasInvalidEnvironmentIssue = X.hasCause(err, InvalidEnvironmentException.class, NodeStoppingException.class);
+                    boolean nodeStopping = X.hasCause(err, NodeStoppingException.class);
 
-                    this.tx.tmFinish(err == null, hasInvalidEnvironmentIssue, false);
+                    this.tx.tmFinish(err == null, nodeStopping || cctx.kernalContext().failure().nodeStopping(), false);
                 }
                 catch (IgniteCheckedException finishErr) {
                     U.error(log, "Failed to finish tx: " + tx, e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
index 2e19df2911509..bec83cdda1bc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
@@ -24,6 +24,8 @@
 import java.util.UUID;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -39,6 +41,7 @@
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
@@ -46,6 +49,7 @@
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
@@ -468,7 +472,11 @@ private void finishTx(boolean commit, @Nullable IgniteInternalFuture prepFut, Gr
                     ", tx=" + CU.txString(this) + ']');
         }
         catch (IgniteCheckedException e) {
-            U.error(log, "Failed to finish transaction [commit=" + commit + ", tx=" + this + ']', e);
+            logTxFinishErrorSafe(log, commit, e);
+
+            // Treat heuristic exception as critical.
+            if (X.hasCause(e, IgniteTxHeuristicCheckedException.class))
+                cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             err = e;
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index c898c870a1864..1d68c4a3528aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -36,6 +36,8 @@
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.IgniteDiagnosticAware;
 import org.apache.ignite.internal.IgniteDiagnosticPrepareContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -723,8 +725,6 @@ private boolean mapIfLocked() {
 
             if (tx.commitOnPrepare()) {
                 if (tx.markFinalizing(IgniteInternalTx.FinalizationStatus.USER_FINISH)) {
-                    IgniteInternalFuture<IgniteInternalTx> fut = null;
-
                     CIX1<IgniteInternalFuture<IgniteInternalTx>> resClo =
                         new CIX1<IgniteInternalFuture<IgniteInternalTx>>() {
                             @Override public void applyx(IgniteInternalFuture<IgniteInternalTx> fut) {
@@ -736,42 +736,43 @@ private boolean mapIfLocked() {
                             }
                         };
 
-                    if (prepErr == null) {
-                        try {
-                            fut = tx.commitAsync();
-                        }
-                        catch (RuntimeException | Error e) {
-                            Exception hEx = new IgniteTxHeuristicCheckedException("Commit produced a runtime " +
-                                "exception: " + CU.txString(tx), e);
-
-                            res.error(hEx);
+                    try {
+                        if (prepErr == null) {
+                            try {
+                                tx.commitAsync().listen(resClo);
+                            }
+                            catch (Throwable e) {
+                                res.error(e);
 
-                            tx.systemInvalidate(true);
+                                tx.systemInvalidate(true);
 
-                            try {
-                                fut = tx.rollbackAsync();
+                                try {
+                                    tx.rollbackAsync().listen(resClo);
+                                }
+                                catch (Throwable e1) {
+                                    e.addSuppressed(e1);
+                                }
 
-                                fut.listen(resClo);
+                                throw e;
                             }
-                            catch (Throwable e1) {
-                                e.addSuppressed(e1);
+                        }
+                        else if (!cctx.kernalContext().isStopping()) {
+                            try {
+                                tx.rollbackAsync().listen(resClo);
                             }
+                            catch (Throwable e) {
+                                if (err != null)
+                                    err.addSuppressed(e);
 
-                            throw e;
+                                throw err;
+                            }
                         }
-
                     }
-                    else if (!cctx.kernalContext().isStopping())
-                        try {
-                            fut = tx.rollbackAsync();
-                        }
-                        catch (Throwable e) {
-                            err.addSuppressed(e);
-                            fut = null;
-                        }
+                    catch (Throwable e){
+                        tx.logTxFinishErrorSafe(log, true, e);
 
-                    if (fut != null)
-                        fut.listen(resClo);
+                        cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+                    }
                 }
             }
             else {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 297456350dff9..9e73d98d67b25 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -300,7 +300,7 @@ void forceFinish() {
             if (err != null) {
                 tx.setRollbackOnly();
 
-                nodeStop = err instanceof NodeStoppingException;
+                nodeStop = err instanceof NodeStoppingException || cctx.kernalContext().failure().nodeStopping();
             }
 
             if (commit) {
@@ -346,29 +346,6 @@ else if (err != null)
                 }
 
                 if (super.onDone(tx0, err)) {
-                    if (error() instanceof IgniteTxHeuristicCheckedException && !nodeStop) {
-                        AffinityTopologyVersion topVer = tx.topologyVersion();
-
-                        for (IgniteTxEntry e : tx.writeMap().values()) {
-                            GridCacheContext cacheCtx = e.context();
-
-                            try {
-                                if (e.op() != NOOP && !cacheCtx.affinity().keyLocalNode(e.key(), topVer)) {
-                                    GridCacheEntryEx entry = cacheCtx.cache().peekEx(e.key());
-
-                                    if (entry != null)
-                                        entry.invalidate(tx.xidVersion());
-                                }
-                            }
-                            catch (Throwable t) {
-                                U.error(log, "Failed to invalidate entry.", t);
-
-                                if (t instanceof Error)
-                                    throw (Error)t;
-                            }
-                        }
-                    }
-
                     // Don't forget to clean up.
                     cctx.mvcc().removeFuture(futId);
 
@@ -465,18 +442,25 @@ private void rollbackAsyncSafe(boolean onTimeout) {
     private void doFinish(boolean commit, boolean clearThreadMap) {
         try {
             if (tx.localFinish(commit, clearThreadMap) || (!commit && tx.state() == UNKNOWN)) {
+                // Cleanup transaction if heuristic failure.
+                if (tx.state() == UNKNOWN)
+                    cctx.tm().rollbackTx(tx, clearThreadMap, false);
+
                 if ((tx.onePhaseCommit() && needFinishOnePhase(commit)) || (!tx.onePhaseCommit() && mappings != null)) {
                     if (mappings.single()) {
                         GridDistributedTxMapping mapping = mappings.singleMapping();
 
                         if (mapping != null) {
-                            assert !hasFutures() : futures();
+                            assert !hasFutures() || isDone() : futures();
 
                             finish(1, mapping, commit, !clearThreadMap);
                         }
                     }
-                    else
+                    else {
+                        assert !hasFutures() || isDone() : futures();
+                        
                         finish(mappings.mappings(), commit, !clearThreadMap);
+                    }
                 }
 
                 markInitialized();
@@ -729,11 +713,9 @@ private void readyNearMappingFromBackup(GridDistributedTxMapping mapping) {
     /**
      * @param mappings Mappings.
      * @param commit Commit flag.
-     * @param {@code true} If need to add completed version on finish.
+     * @param useCompletedVer {@code True} if need to add completed version on finish.
      */
     private void finish(Iterable<GridDistributedTxMapping> mappings, boolean commit, boolean useCompletedVer) {
-        assert !hasFutures() : futures();
-
         int miniId = 0;
 
         // Create mini futures.
@@ -978,7 +960,7 @@ public GridDistributedTxMapping mapping() {
         }
 
         /** {@inheritDoc} */
-        boolean onNodeLeft(UUID nodeId, boolean discoThread) {
+        @Override boolean onNodeLeft(UUID nodeId, boolean discoThread) {
             if (nodeId.equals(m.primary().id())) {
                 if (msgLog.isDebugEnabled()) {
                     msgLog.debug("Near finish fut, mini future node left [txId=" + tx.nearXidVersion() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 16653e0170f56..e338c61307c06 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -1653,9 +1653,9 @@ private <K, V> IgniteInternalFuture<GridCacheReturn> removeAllAsync0(
                 -1L);
 
             PLC1<GridCacheReturn> plc1 = new PLC1<GridCacheReturn>(ret) {
+                /** {@inheritDoc} */
                 @Override protected GridCacheReturn postLock(GridCacheReturn ret)
-                    throws IgniteCheckedException
-                {
+                    throws IgniteCheckedException {
                     if (log.isDebugEnabled())
                         log.debug("Acquired transaction lock for remove on keys: " + enlisted);
 
@@ -2387,8 +2387,8 @@ private IgniteInternalFuture<Void> loadMissing(
         GridInClosure3<KeyCacheObject, Object, GridCacheVersion> c =
             new GridInClosure3<KeyCacheObject, Object, GridCacheVersion>() {
                 @Override public void apply(KeyCacheObject key,
-                                            @Nullable Object val,
-                                            @Nullable GridCacheVersion loadVer) {
+                    @Nullable Object val,
+                    @Nullable GridCacheVersion loadVer) {
                     if (log.isDebugEnabled())
                         log.debug("Loaded value from remote node [key=" + key + ", val=" + val + ']');
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index 8a966bcd97eea..bdef653d415df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -67,6 +67,7 @@
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersionedEntryEx;
+import org.apache.ignite.internal.transactions.IgniteTxHeuristicCheckedException;
 import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
 import org.apache.ignite.internal.processors.cluster.BaselineTopology;
 import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
@@ -712,6 +713,36 @@ public final IgniteCheckedException rollbackException() {
             "[timeout=" + timeout() + ", tx=" + CU.txString(this) + ']');
     }
 
+    /**
+     * @param ex Root cause.
+     */
+    public final IgniteCheckedException heuristicException(Throwable ex) {
+        return new IgniteTxHeuristicCheckedException("Committing a transaction has produced runtime exception", ex);
+    }
+
+    /**
+     * @param log Log.
+     * @param commit Commit.
+     * @param e Exception.
+     */
+    public void logTxFinishErrorSafe(@Nullable IgniteLogger log, boolean commit, Throwable e) {
+        assert e != null : "Exception is expected";
+
+        final String fmt = "Failed completing the transaction: [commit=%s, tx=%s, plc=%s]";
+
+        try {
+            // First try printing a full transaction. This is error prone.
+            U.error(log, String.format(fmt, commit, this,
+                cctx.gridConfig().getFailureHandler().getClass().getSimpleName()), e);
+        }
+        catch (Throwable e0) {
+            e.addSuppressed(e0);
+
+            U.error(log, String.format(fmt, commit, CU.txString(this),
+                cctx.gridConfig().getFailureHandler().getClass().getSimpleName()), e);
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public GridCacheVersion xidVersion() {
         return xidVer;
@@ -1577,6 +1608,8 @@ else if (txEntry.hasOldValue())
 
             GridCacheOperation op = modified ? (cacheVal == null ? DELETE : UPDATE) : NOOP;
 
+            txEntry.entryProcessorCalculatedValue(new T2<>(op, op == NOOP ? null : cacheVal));
+
             if (op == NOOP) {
                 ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index d44728fff96ab..45308a9fe7d62 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -1007,45 +1007,34 @@ private IgniteInternalFuture<IgniteInternalTx> finishDhtLocal(UUID nodeId,
         }
         catch (Throwable e) {
             try {
-                U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e);
-            }
-            catch (Throwable e0) {
-                ClusterNode node0 = ctx.discovery().node(nodeId);
-
-                U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" +
-                        CU.txString(tx) + ']', e);
-
-                U.error(log, "Failed to log message due to an error: ", e0);
+                if (tx != null) {
+                    tx.commitError(e);
 
-                if (node0 != null && (!node0.isClient() || node0.isLocal())) {
-                    ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+                    tx.systemInvalidate(true);
 
-                    throw e;
-                }
-            }
-
-            if (tx != null) {
-                tx.commitError(e);
-
-                tx.systemInvalidate(true);
+                    try {
+                        IgniteInternalFuture<IgniteInternalTx> res = tx.rollbackDhtLocalAsync();
 
-                try {
-                    IgniteInternalFuture<IgniteInternalTx> res = tx.rollbackDhtLocalAsync();
+                        // Only for error logging.
+                        res.listen(CU.errorLogger(log));
 
-                    // Only for error logging.
-                    res.listen(CU.errorLogger(log));
+                        return res;
+                    }
+                    catch (Throwable e1) {
+                        e.addSuppressed(e1);
+                    }
 
-                    return res;
+                    tx.logTxFinishErrorSafe(log, req.commit(), e);
                 }
-                catch (Throwable e1) {
-                    e.addSuppressed(e1);
-                }
-            }
 
-            if (e instanceof Error)
-                throw (Error)e;
+                if (e instanceof Error)
+                    throw (Error)e;
 
-            return new GridFinishedFuture<>(e);
+                return new GridFinishedFuture<>(e);
+            }
+            finally {
+                ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+            }
         }
     }
 
@@ -1070,20 +1059,26 @@ public IgniteInternalFuture<IgniteInternalTx> finishColocatedLocal(boolean commi
                 return tx.rollbackAsyncLocal();
         }
         catch (Throwable e) {
-            U.error(log, "Failed completing transaction [commit=" + commit + ", tx=" + tx + ']', e);
-
-            if (e instanceof Error)
-                throw e;
+            try {
+                if (tx != null) {
+                    try {
+                        return tx.rollbackNearTxLocalAsync();
+                    }
+                    catch (Throwable e1) {
+                        e.addSuppressed(e1);
+                    }
 
-            if (tx != null)
-                try {
-                    return tx.rollbackNearTxLocalAsync();
-                }
-                catch (Throwable e1) {
-                    e.addSuppressed(e1);
+                    tx.logTxFinishErrorSafe(log, commit, e);
                 }
 
-            return new GridFinishedFuture<>(e);
+                if (e instanceof Error)
+                    throw e;
+
+                return new GridFinishedFuture<>(e);
+            }
+            finally {
+                ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+            }
         }
     }
 
@@ -1167,10 +1162,6 @@ else if (e instanceof IgniteTxOptimisticCheckedException) {
                 if (log.isDebugEnabled())
                     log.debug("Optimistic failure for remote transaction (will rollback): " + req);
             }
-            else if (e instanceof IgniteTxHeuristicCheckedException) {
-                U.warn(log, "Failed to commit transaction (all transaction entries were invalidated): " +
-                    CU.txString(dhtTx));
-            }
             else
                 U.error(log, "Failed to process prepare request: " + req, e);
 
@@ -1389,9 +1380,10 @@ else if (log.isDebugEnabled())
                 tx.rollbackRemoteTx();
             }
         }
+        catch (IgniteTxHeuristicCheckedException e) {
+            // Already uncommitted.
+        }
         catch (Throwable e) {
-            U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e);
-
             // Mark transaction for invalidate.
             tx.invalidate(true);
             tx.systemInvalidate(true);
@@ -1409,6 +1401,8 @@ else if (log.isDebugEnabled())
     }
 
     /**
+     * Finish for one-phase distributed tx.
+     *
      * @param tx Transaction.
      * @param req Request.
      */
@@ -1431,22 +1425,27 @@ protected void finish(
             throw e;
         }
         catch (Throwable e) {
-            U.error(log, "Failed committing transaction [tx=" + tx + ']', e);
+            try {
+                // Mark transaction for invalidate.
+                tx.invalidate(true);
 
-            // Mark transaction for invalidate.
-            tx.invalidate(true);
-            tx.systemInvalidate(true);
+                tx.systemInvalidate(true);
 
-            try {
-                tx.rollbackRemoteTx();
+                try {
+                    tx.rollbackRemoteTx();
+                }
+                catch (Throwable e1) {
+                    e.addSuppressed(e1);
+                }
+
+                tx.logTxFinishErrorSafe(log, true, e);
+
+                if (e instanceof Error)
+                    throw (Error)e;
             }
-            catch (Throwable e1) {
-                e.addSuppressed(e1);
-                U.error(log, "Failed to automatically rollback transaction: " + tx, e1);
+            finally {
+                ctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
             }
-
-            if (e instanceof Error)
-                throw (Error)e;
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index b02f9db13534f..e403b6e4c2ac8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -509,6 +509,8 @@ protected GridCacheEntryEx entryEx(GridCacheContext cacheCtx, IgniteTxKey key, A
 
             WALPointer ptr = null;
 
+            IgniteCheckedException err = null;
+
             cctx.database().checkpointReadLock();
 
             try {
@@ -526,369 +528,344 @@ protected GridCacheEntryEx entryEx(GridCacheContext cacheCtx, IgniteTxKey key, A
 
                     UUID nodeId = txEntry.nodeId() == null ? this.nodeId : txEntry.nodeId();
 
-                    try {
-                        while (true) {
-                            try {
-                                GridCacheEntryEx cached = txEntry.cached();
+                    while (true) {
+                        try {
+                            GridCacheEntryEx cached = txEntry.cached();
 
-                                // Must try to evict near entries before committing from
-                                // transaction manager to make sure locks are held.
-                                if (!evictNearEntry(txEntry, false)) {
-                                    if (cacheCtx.isNear() && cacheCtx.dr().receiveEnabled()) {
-                                        cached.markObsolete(xidVer);
+                            // Must try to evict near entries before committing from
+                            // transaction manager to make sure locks are held.
+                            if (!evictNearEntry(txEntry, false)) {
+                                if (cacheCtx.isNear() && cacheCtx.dr().receiveEnabled()) {
+                                    cached.markObsolete(xidVer);
 
-                                        break;
-                                    }
+                                    break;
+                                }
 
-                                    if (cached.detached())
-                                        break;
+                                if (cached.detached())
+                                    break;
 
-                                    boolean updateNearCache = updateNearCache(cacheCtx, txEntry.key(), topVer);
+                                boolean updateNearCache = updateNearCache(cacheCtx, txEntry.key(), topVer);
 
-                                    boolean metrics = true;
+                                boolean metrics = true;
 
-                                    if (!updateNearCache && cacheCtx.isNear() && txEntry.locallyMapped())
-                                        metrics = false;
+                                if (!updateNearCache && cacheCtx.isNear() && txEntry.locallyMapped())
+                                    metrics = false;
 
-                                    boolean evt = !isNearLocallyMapped(txEntry, false);
+                                boolean evt = !isNearLocallyMapped(txEntry, false);
 
-                                    if (!F.isEmpty(txEntry.entryProcessors()) || !F.isEmpty(txEntry.filters()))
-                                        txEntry.cached().unswap(false);
+                                if (!F.isEmpty(txEntry.entryProcessors()) || !F.isEmpty(txEntry.filters()))
+                                    txEntry.cached().unswap(false);
 
-                                    IgniteBiTuple<GridCacheOperation, CacheObject> res = applyTransformClosures(txEntry,
-                                        true, null);
+                                IgniteBiTuple<GridCacheOperation, CacheObject> res = applyTransformClosures(txEntry,
+                                    true, null);
 
-                                    GridCacheVersion dhtVer = null;
+                                GridCacheVersion dhtVer = null;
 
-                                    // For near local transactions we must record DHT version
-                                    // in order to keep near entries on backup nodes until
-                                    // backup remote transaction completes.
-                                    if (cacheCtx.isNear()) {
-                                        if (txEntry.op() == CREATE || txEntry.op() == UPDATE ||
-                                            txEntry.op() == DELETE || txEntry.op() == TRANSFORM)
-                                            dhtVer = txEntry.dhtVersion();
+                                // For near local transactions we must record DHT version
+                                // in order to keep near entries on backup nodes until
+                                // backup remote transaction completes.
+                                if (cacheCtx.isNear()) {
+                                    if (txEntry.op() == CREATE || txEntry.op() == UPDATE ||
+                                        txEntry.op() == DELETE || txEntry.op() == TRANSFORM)
+                                        dhtVer = txEntry.dhtVersion();
 
-                                        if ((txEntry.op() == CREATE || txEntry.op() == UPDATE) &&
-                                            txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) {
-                                            ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
+                                    if ((txEntry.op() == CREATE || txEntry.op() == UPDATE) &&
+                                        txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) {
+                                        ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
 
-                                            if (expiry != null) {
-                                                txEntry.cached().unswap(false);
+                                        if (expiry != null) {
+                                            txEntry.cached().unswap(false);
 
-                                                Duration duration = cached.hasValue() ?
-                                                    expiry.getExpiryForUpdate() : expiry.getExpiryForCreation();
+                                            Duration duration = cached.hasValue() ?
+                                                expiry.getExpiryForUpdate() : expiry.getExpiryForCreation();
 
-                                                txEntry.ttl(CU.toTtl(duration));
-                                            }
+                                            txEntry.ttl(CU.toTtl(duration));
                                         }
                                     }
+                                }
 
-                                    GridCacheOperation op = res.get1();
-                                    CacheObject val = res.get2();
+                                GridCacheOperation op = res.get1();
+                                CacheObject val = res.get2();
 
-                                    // Deal with conflicts.
-                                    GridCacheVersion explicitVer = txEntry.conflictVersion() != null ?
-                                        txEntry.conflictVersion() : writeVersion();
+                                // Deal with conflicts.
+                                GridCacheVersion explicitVer = txEntry.conflictVersion() != null ?
+                                    txEntry.conflictVersion() : writeVersion();
 
-                                    if ((op == CREATE || op == UPDATE) &&
-                                        txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) {
-                                        ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
+                                if ((op == CREATE || op == UPDATE) &&
+                                    txEntry.conflictExpireTime() == CU.EXPIRE_TIME_CALCULATE) {
+                                    ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
 
-                                        if (expiry != null) {
-                                            Duration duration = cached.hasValue() ?
-                                                expiry.getExpiryForUpdate() : expiry.getExpiryForCreation();
+                                    if (expiry != null) {
+                                        Duration duration = cached.hasValue() ?
+                                            expiry.getExpiryForUpdate() : expiry.getExpiryForCreation();
 
-                                            long ttl = CU.toTtl(duration);
+                                        long ttl = CU.toTtl(duration);
 
-                                            txEntry.ttl(ttl);
+                                        txEntry.ttl(ttl);
 
-                                            if (ttl == CU.TTL_ZERO)
-                                                op = DELETE;
-                                        }
+                                        if (ttl == CU.TTL_ZERO)
+                                            op = DELETE;
                                     }
+                                }
 
-                                    boolean conflictNeedResolve = cacheCtx.conflictNeedResolve();
-
-                                    GridCacheVersionConflictContext<?, ?> conflictCtx = null;
-
-                                    if (conflictNeedResolve) {
-                                        IgniteBiTuple<GridCacheOperation, GridCacheVersionConflictContext> conflictRes =
-                                            conflictResolve(op, txEntry, val, explicitVer, cached);
+                                boolean conflictNeedResolve = cacheCtx.conflictNeedResolve();
 
-                                        assert conflictRes != null;
+                                GridCacheVersionConflictContext<?, ?> conflictCtx = null;
 
-                                        conflictCtx = conflictRes.get2();
+                                if (conflictNeedResolve) {
+                                    IgniteBiTuple<GridCacheOperation, GridCacheVersionConflictContext> conflictRes =
+                                        conflictResolve(op, txEntry, val, explicitVer, cached);
 
-                                        if (conflictCtx.isUseOld())
-                                            op = NOOP;
-                                        else if (conflictCtx.isUseNew()) {
-                                            txEntry.ttl(conflictCtx.ttl());
-                                            txEntry.conflictExpireTime(conflictCtx.expireTime());
-                                        }
-                                        else {
-                                            assert conflictCtx.isMerge();
+                                    assert conflictRes != null;
 
-                                            op = conflictRes.get1();
-                                            val = txEntry.context().toCacheObject(conflictCtx.mergeValue());
-                                            explicitVer = writeVersion();
+                                    conflictCtx = conflictRes.get2();
 
-                                            txEntry.ttl(conflictCtx.ttl());
-                                            txEntry.conflictExpireTime(conflictCtx.expireTime());
-                                        }
+                                    if (conflictCtx.isUseOld())
+                                        op = NOOP;
+                                    else if (conflictCtx.isUseNew()) {
+                                        txEntry.ttl(conflictCtx.ttl());
+                                        txEntry.conflictExpireTime(conflictCtx.expireTime());
                                     }
-                                    else
-                                        // Nullify explicit version so that innerSet/innerRemove will work as usual.
-                                        explicitVer = null;
+                                    else {
+                                        assert conflictCtx.isMerge();
 
-                                    if (sndTransformedVals || conflictNeedResolve) {
-                                        assert sndTransformedVals && cacheCtx.isReplicated() || conflictNeedResolve;
+                                        op = conflictRes.get1();
+                                        val = txEntry.context().toCacheObject(conflictCtx.mergeValue());
+                                        explicitVer = writeVersion();
 
-                                        txEntry.value(val, true, false);
-                                        txEntry.op(op);
-                                        txEntry.entryProcessors(null);
-                                        txEntry.conflictVersion(explicitVer);
+                                        txEntry.ttl(conflictCtx.ttl());
+                                        txEntry.conflictExpireTime(conflictCtx.expireTime());
                                     }
+                                }
+                                else
+                                    // Nullify explicit version so that innerSet/innerRemove will work as usual.
+                                    explicitVer = null;
 
-                                    if (dhtVer == null)
-                                        dhtVer = explicitVer != null ? explicitVer : writeVersion();
+                                if (sndTransformedVals || conflictNeedResolve) {
+                                    assert sndTransformedVals && cacheCtx.isReplicated() || conflictNeedResolve;
 
-                                    if (op == CREATE || op == UPDATE) {
-                                        assert val != null : txEntry;
+                                    txEntry.value(val, true, false);
+                                    txEntry.op(op);
+                                    txEntry.entryProcessors(null);
+                                    txEntry.conflictVersion(explicitVer);
+                                }
 
-                                        GridCacheUpdateTxResult updRes = cached.innerSet(
-                                            this,
-                                            eventNodeId(),
-                                            txEntry.nodeId(),
-                                            val,
-                                            false,
-                                            false,
-                                            txEntry.ttl(),
-                                            evt,
-                                            metrics,
-                                            txEntry.keepBinary(),
-                                            txEntry.hasOldValue(),
-                                            txEntry.oldValue(),
-                                            topVer,
-                                            null,
-                                            cached.detached() ? DR_NONE : drType,
-                                            txEntry.conflictExpireTime(),
-                                            cached.isNear() ? null : explicitVer,
-                                            CU.subjectId(this, cctx),
-                                            resolveTaskName(),
-                                            dhtVer,
-                                            null);
-
-                                        if (updRes.success())
-                                            txEntry.updateCounter(updRes.updatePartitionCounter());
-
-                                        if (updRes.loggedPointer() != null)
-                                            ptr = updRes.loggedPointer();
-
-                                        if (updRes.success() && updateNearCache) {
-                                            final CacheObject val0 = val;
-                                            final boolean metrics0 = metrics;
-                                            final GridCacheVersion dhtVer0 = dhtVer;
-
-                                            updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerSet(
-                                                    null,
-                                                    eventNodeId(),
-                                                    nodeId,
-                                                    val0,
-                                                    false,
-                                                    false,
-                                                    txEntry.ttl(),
-                                                    false,
-                                                    metrics0,
-                                                    txEntry.keepBinary(),
-                                                    txEntry.hasOldValue(),
-                                                    txEntry.oldValue(),
-                                                    topVer,
-                                                    CU.empty0(),
-                                                    DR_NONE,
-                                                    txEntry.conflictExpireTime(),
-                                                    null,
-                                                    CU.subjectId(this, cctx),
-                                                    resolveTaskName(),
-                                                    dhtVer0,
-                                                    null)
-                                            );
-                                        }
+                                if (dhtVer == null)
+                                    dhtVer = explicitVer != null ? explicitVer : writeVersion();
+
+                                if (op == CREATE || op == UPDATE) {
+                                    assert val != null : txEntry;
+
+                                    GridCacheUpdateTxResult updRes = cached.innerSet(
+                                        this,
+                                        eventNodeId(),
+                                        txEntry.nodeId(),
+                                        val,
+                                        false,
+                                        false,
+                                        txEntry.ttl(),
+                                        evt,
+                                        metrics,
+                                        txEntry.keepBinary(),
+                                        txEntry.hasOldValue(),
+                                        txEntry.oldValue(),
+                                        topVer,
+                                        null,
+                                        cached.detached() ? DR_NONE : drType,
+                                        txEntry.conflictExpireTime(),
+                                        cached.isNear() ? null : explicitVer,
+                                        CU.subjectId(this, cctx),
+                                        resolveTaskName(),
+                                        dhtVer,
+                                        null);
+
+                                    if (updRes.success())
+                                        txEntry.updateCounter(updRes.updatePartitionCounter());
+
+                                    if (updRes.loggedPointer() != null)
+                                        ptr = updRes.loggedPointer();
+
+                                    if (updRes.success() && updateNearCache) {
+                                        final CacheObject val0 = val;
+                                        final boolean metrics0 = metrics;
+                                        final GridCacheVersion dhtVer0 = dhtVer;
+
+                                        updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerSet(
+                                                null,
+                                                eventNodeId(),
+                                                nodeId,
+                                                val0,
+                                                false,
+                                                false,
+                                                txEntry.ttl(),
+                                                false,
+                                                metrics0,
+                                                txEntry.keepBinary(),
+                                                txEntry.hasOldValue(),
+                                                txEntry.oldValue(),
+                                                topVer,
+                                                CU.empty0(),
+                                                DR_NONE,
+                                                txEntry.conflictExpireTime(),
+                                                null,
+                                                CU.subjectId(this, cctx),
+                                                resolveTaskName(),
+                                                dhtVer0,
+                                                null)
+                                        );
                                     }
-                                    else if (op == DELETE) {
-                                        GridCacheUpdateTxResult updRes = cached.innerRemove(
-                                            this,
-                                            eventNodeId(),
-                                            txEntry.nodeId(),
-                                            false,
-                                            evt,
-                                            metrics,
-                                            txEntry.keepBinary(),
-                                            txEntry.hasOldValue(),
-                                            txEntry.oldValue(),
-                                            topVer,
-                                            null,
-                                            cached.detached()  ? DR_NONE : drType,
-                                            cached.isNear() ? null : explicitVer,
-                                            CU.subjectId(this, cctx),
-                                            resolveTaskName(),
-                                            dhtVer,
-                                            null);
-
-                                        if (updRes.success())
-                                            txEntry.updateCounter(updRes.updatePartitionCounter());
-
-                                        if (updRes.loggedPointer() != null)
-                                            ptr = updRes.loggedPointer();
-
-                                        if (updRes.success() && updateNearCache) {
-                                            final boolean metrics0 = metrics;
-                                            final GridCacheVersion dhtVer0 = dhtVer;
-
-                                            updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerRemove(
-                                                    null,
-                                                    eventNodeId(),
-                                                    nodeId,
-                                                    false,
-                                                    false,
-                                                    metrics0,
-                                                    txEntry.keepBinary(),
-                                                    txEntry.hasOldValue(),
-                                                    txEntry.oldValue(),
-                                                    topVer,
-                                                    CU.empty0(),
-                                                    DR_NONE,
-                                                    null,
-                                                    CU.subjectId(this, cctx),
-                                                    resolveTaskName(),
-                                                    dhtVer0,
-                                                    null)
-                                            );
-                                        }
+                                }
+                                else if (op == DELETE) {
+                                    GridCacheUpdateTxResult updRes = cached.innerRemove(
+                                        this,
+                                        eventNodeId(),
+                                        txEntry.nodeId(),
+                                        false,
+                                        evt,
+                                        metrics,
+                                        txEntry.keepBinary(),
+                                        txEntry.hasOldValue(),
+                                        txEntry.oldValue(),
+                                        topVer,
+                                        null,
+                                        cached.detached() ? DR_NONE : drType,
+                                        cached.isNear() ? null : explicitVer,
+                                        CU.subjectId(this, cctx),
+                                        resolveTaskName(),
+                                        dhtVer,
+                                        null);
+
+                                    if (updRes.success())
+                                        txEntry.updateCounter(updRes.updatePartitionCounter());
+
+                                    if (updRes.loggedPointer() != null)
+                                        ptr = updRes.loggedPointer();
+
+                                    if (updRes.success() && updateNearCache) {
+                                        final boolean metrics0 = metrics;
+                                        final GridCacheVersion dhtVer0 = dhtVer;
+
+                                        updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerRemove(
+                                                null,
+                                                eventNodeId(),
+                                                nodeId,
+                                                false,
+                                                false,
+                                                metrics0,
+                                                txEntry.keepBinary(),
+                                                txEntry.hasOldValue(),
+                                                txEntry.oldValue(),
+                                                topVer,
+                                                CU.empty0(),
+                                                DR_NONE,
+                                                null,
+                                                CU.subjectId(this, cctx),
+                                                resolveTaskName(),
+                                                dhtVer0,
+                                                null)
+                                        );
                                     }
-                                    else if (op == RELOAD) {
-                                        cached.innerReload();
+                                }
+                                else if (op == RELOAD) {
+                                    cached.innerReload();
 
-                                        if (updateNearCache)
-                                            updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerReload());
+                                    if (updateNearCache)
+                                        updateNearEntrySafely(cacheCtx, txEntry.key(), entry -> entry.innerReload());
+                                }
+                                else if (op == READ) {
+                                    CacheGroupContext grp = cacheCtx.group();
+
+                                    if (grp.persistenceEnabled() && grp.walEnabled() &&
+                                        cctx.snapshot().needTxReadLogging()) {
+                                        ptr = cctx.wal().log(new DataRecord(new DataEntry(
+                                            cacheCtx.cacheId(),
+                                            txEntry.key(),
+                                            val,
+                                            op,
+                                            nearXidVersion(),
+                                            writeVersion(),
+                                            0,
+                                            txEntry.key().partition(),
+                                            txEntry.updateCounter())));
                                     }
-                                    else if (op == READ) {
-                                        CacheGroupContext grp = cacheCtx.group();
-
-                                        if (grp.persistenceEnabled() && grp.walEnabled() &&
-                                            cctx.snapshot().needTxReadLogging()) {
-                                            ptr = cctx.wal().log(new DataRecord(new DataEntry(
-                                                cacheCtx.cacheId(),
-                                                txEntry.key(),
-                                                val,
-                                                op,
-                                                nearXidVersion(),
-                                                writeVersion(),
-                                                0,
-                                                txEntry.key().partition(),
-                                                txEntry.updateCounter())));
-                                        }
 
-                                        ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
+                                    ExpiryPolicy expiry = cacheCtx.expiryForTxEntry(txEntry);
 
-                                        if (expiry != null) {
-                                            Duration duration = expiry.getExpiryForAccess();
+                                    if (expiry != null) {
+                                        Duration duration = expiry.getExpiryForAccess();
 
-                                            if (duration != null)
-                                                cached.updateTtl(null, CU.toTtl(duration));
-                                        }
-
-                                        if (log.isDebugEnabled())
-                                            log.debug("Ignoring READ entry when committing: " + txEntry);
+                                        if (duration != null)
+                                            cached.updateTtl(null, CU.toTtl(duration));
                                     }
-                                    else {
-                                        assert ownsLock(txEntry.cached()):
-                                            "Transaction does not own lock for group lock entry during  commit [tx=" +
-                                                this + ", txEntry=" + txEntry + ']';
-
-                                        if (conflictCtx == null || !conflictCtx.isUseOld()) {
-                                            if (txEntry.ttl() != CU.TTL_NOT_CHANGED)
-                                                cached.updateTtl(null, txEntry.ttl());
-                                        }
 
-                                        if (log.isDebugEnabled())
-                                            log.debug("Ignoring NOOP entry when committing: " + txEntry);
+                                    if (log.isDebugEnabled())
+                                        log.debug("Ignoring READ entry when committing: " + txEntry);
+                                }
+                                else {
+                                    assert ownsLock(txEntry.cached()) :
+                                        "Transaction does not own lock for group lock entry during  commit [tx=" +
+                                            this + ", txEntry=" + txEntry + ']';
+
+                                    if (conflictCtx == null || !conflictCtx.isUseOld()) {
+                                        if (txEntry.ttl() != CU.TTL_NOT_CHANGED)
+                                            cached.updateTtl(null, txEntry.ttl());
                                     }
+
+                                    if (log.isDebugEnabled())
+                                        log.debug("Ignoring NOOP entry when committing: " + txEntry);
                                 }
+                            }
 
-                                // Check commit locks after set, to make sure that
-                                // we are not changing obsolete entries.
-                                // (innerSet and innerRemove will throw an exception
-                                // if an entry is obsolete).
-                                if (txEntry.op() != READ)
-                                    checkCommitLocks(cached);
+                            // Check commit locks after set, to make sure that
+                            // we are not changing obsolete entries.
+                            // (innerSet and innerRemove will throw an exception
+                            // if an entry is obsolete).
+                            if (txEntry.op() != READ)
+                                checkCommitLocks(cached);
 
-                                // Break out of while loop.
-                                break;
-                            }
-                            // If entry cached within transaction got removed.
-                            catch (GridCacheEntryRemovedException ignored) {
-                                if (log.isDebugEnabled())
-                                    log.debug("Got removed entry during transaction commit (will retry): " + txEntry);
+                            // Break out of while loop.
+                            break;
+                        }
+                        // If entry cached within transaction got removed.
+                        catch (GridCacheEntryRemovedException ignored) {
+                            if (log.isDebugEnabled())
+                                log.debug("Got removed entry during transaction commit (will retry): " + txEntry);
 
-                                txEntry.cached(entryEx(cacheCtx, txEntry.txKey(), topologyVersion()));
-                            }
+                            txEntry.cached(entryEx(cacheCtx, txEntry.txKey(), topologyVersion()));
                         }
                     }
-                    catch (Throwable ex) {
-                        // We are about to initiate transaction rollback when tx has started to committing.
-                        // Need to remove version from committed list.
-                        cctx.tm().removeCommittedTx(this);
-
-                        boolean isNodeStopping = X.hasCause(ex, NodeStoppingException.class);
-                        boolean hasInvalidEnvironmentIssue = X.hasCause(ex, InvalidEnvironmentException.class);
 
-                        IgniteCheckedException err = new IgniteTxHeuristicCheckedException("Failed to locally write to cache " +
-                            "(all transaction entries will be invalidated, however there was a window when " +
-                            "entries for this transaction were visible to others): " + this, ex);
+                }
 
-                        if (isNodeStopping) {
-                            U.warn(log, "Failed to commit transaction, node is stopping [tx=" + this +
-                                ", err=" + ex + ']');
-                        }
-                        else if (hasInvalidEnvironmentIssue) {
-                            U.warn(log, "Failed to commit transaction, node is in invalid state and will be stopped [tx=" + this +
-                                ", err=" + ex + ']');
-                        }
-                        else
-                            U.error(log, "Commit failed.", err);
+                if (ptr != null && !cctx.tm().logTxRecords())
+                    cctx.wal().flush(ptr, false);
+            }
+            catch (Throwable ex) {
+                // We are about to initiate transaction rollback when tx has started to committing.
+                // Need to remove version from committed list.
+                cctx.tm().removeCommittedTx(this);
 
-                        COMMIT_ERR_UPD.compareAndSet(this, null, err);
+                if (X.hasCause(ex, NodeStoppingException.class)) {
+                    U.warn(log, "Failed to commit transaction, node is stopping [tx=" + CU.txString(this) +
+                        ", err=" + ex + ']');
 
-                        state(UNKNOWN);
+                    return;
+                }
 
-                        if (hasInvalidEnvironmentIssue)
-                            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, ex));
-                        else if (!isNodeStopping) { // Skip fair uncommit in case of node stopping or invalidation.
-                            try {
-                                // Courtesy to minimize damage.
-                                uncommit();
-                            }
-                            catch (Throwable ex1) {
-                                U.error(log, "Failed to uncommit transaction: " + this, ex1);
+                err = heuristicException(ex);
 
-                                if (ex1 instanceof Error)
-                                    throw ex1;
-                            }
-                        }
+                COMMIT_ERR_UPD.compareAndSet(this, null, err);
 
-                        if (ex instanceof Error)
-                            throw ex;
+                state(UNKNOWN);
 
-                        throw err;
-                    }
+                try {
+                    uncommit();
+                }
+                catch (Throwable e) {
+                    err.addSuppressed(e);
                 }
 
-                if (ptr != null && !cctx.tm().logTxRecords())
-                    cctx.wal().flush(ptr, false);
-            }
-            catch (StorageException e) {
-                throw new IgniteCheckedException("Failed to log transaction record " +
-                    "(transaction will be rolled back): " + this, e);
+                throw err;
             }
             finally {
                 cctx.database().checkpointReadUnlock();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
index 722de1870c6ef..c06af54fc57c3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.failure.NoOpFailureHandler;
 import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
@@ -77,6 +78,13 @@ public FailureProcessor(GridKernalContext ctx) {
         U.quietAndInfo(log, "Configured failure handler: [hnd=" + hnd + ']');
     }
 
+    /**
+     * @return @{code True} if a node will be stopped by current handler in near time.
+     */
+    public boolean nodeStopping() {
+        return failureCtx != null && !(hnd instanceof NoOpFailureHandler);
+    }
+
     /**
      * This method is used to initialize local failure handler if {@link IgniteConfiguration} don't contain configured one.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
index a7e6e8c72de42..f68ecd68e2591 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/IgniteSpiAdapter.java
@@ -41,7 +41,6 @@
 import org.apache.ignite.internal.processors.timeout.GridSpiTimeoutObject;
 import org.apache.ignite.internal.util.IgniteExceptionRegistry;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteBiPredicate;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
index 299dbf4a6eb69..bced8cc969379 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractSelfTest.java
@@ -260,7 +260,7 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) throw
             cfg.setIndexedTypes(idxTypes);
 
         if (cacheMode() == PARTITIONED)
-            cfg.setBackups(1);
+            cfg.setBackups(backups());
 
         return cfg;
     }
@@ -365,6 +365,13 @@ protected IgniteTransactions transactions() {
         return grid(0).transactions();
     }
 
+    /**
+     * @return Backups.
+     */
+    protected int backups() {
+        return 1;
+    }
+
     /**
      * @param idx Index of grid.
      * @return Default cache.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
index b6e32d57cee76..02987d655dde7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQuerySelfTest.java
@@ -26,12 +26,10 @@
 import java.util.TreeMap;
 import java.util.concurrent.Callable;
 import javax.cache.Cache;
-import junit.framework.TestCase;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.IgniteTransactions;
-import org.apache.ignite.Ignition;
 import org.apache.ignite.binary.BinaryObject;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.query.QueryCursor;
@@ -47,6 +45,7 @@
 import org.apache.ignite.spi.indexing.IndexingQueryFilter;
 import org.apache.ignite.spi.indexing.IndexingSpi;
 import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
@@ -57,25 +56,19 @@
 /**
  * Indexing Spi query only test
  */
-public class IndexingSpiQuerySelfTest extends TestCase {
-    public static final String CACHE_NAME = "test-cache";
+public class IndexingSpiQuerySelfTest extends GridCommonAbstractTest {
+    private IndexingSpi indexingSpi;
 
     /** {@inheritDoc} */
-    @Override public void tearDown() throws Exception {
-        Ignition.stopAll(true);
-    }
-
-    /**
-     * @return Configuration.
-     */
-    protected IgniteConfiguration configuration() {
-        IgniteConfiguration cfg = new IgniteConfiguration();
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
         TcpDiscoveryVmIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
         TcpDiscoverySpi disco = new TcpDiscoverySpi();
 
         disco.setIpFinder(ipFinder);
 
+        cfg.setIndexingSpi(indexingSpi);
         cfg.setDiscoverySpi(disco);
 
         cfg.setFailureDetectionTimeout(Integer.MAX_VALUE);
@@ -88,17 +81,22 @@ protected <K,V> CacheConfiguration<K, V> cacheConfiguration(String cacheName) {
         return new CacheConfiguration<>(cacheName);
     }
 
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
     /**
      * @throws Exception If failed.
      */
     public void testSimpleIndexingSpi() throws Exception {
-        IgniteConfiguration cfg = configuration();
-
-        cfg.setIndexingSpi(new MyIndexingSpi());
+        indexingSpi = new MyIndexingSpi();
 
-        Ignite ignite = Ignition.start(cfg);
+        Ignite ignite = startGrid(0);
 
-        CacheConfiguration<Integer, Integer> ccfg = cacheConfiguration(CACHE_NAME);
+        CacheConfiguration<Integer, Integer> ccfg = cacheConfiguration(DEFAULT_CACHE_NAME);
 
         IgniteCache<Integer, Integer> cache = ignite.createCache(ccfg);
 
@@ -115,13 +113,11 @@ public void testSimpleIndexingSpi() throws Exception {
      * @throws Exception If failed.
      */
     public void testIndexingSpiWithDisabledQueryProcessor() throws Exception {
-        IgniteConfiguration cfg = configuration();
-
-        cfg.setIndexingSpi(new MyIndexingSpi());
+        indexingSpi = new MyIndexingSpi();
 
-        Ignite ignite = Ignition.start(cfg);
+        Ignite ignite = startGrid(0);
 
-        CacheConfiguration<Integer, Integer> ccfg = cacheConfiguration(CACHE_NAME);
+        CacheConfiguration<Integer, Integer> ccfg = cacheConfiguration(DEFAULT_CACHE_NAME);
 
         IgniteCache<Integer, Integer> cache = ignite.createCache(ccfg);
 
@@ -138,13 +134,11 @@ public void testIndexingSpiWithDisabledQueryProcessor() throws Exception {
      * @throws Exception If failed.
      */
     public void testBinaryIndexingSpi() throws Exception {
-        IgniteConfiguration cfg = configuration();
+        indexingSpi = new MyBinaryIndexingSpi();
 
-        cfg.setIndexingSpi(new MyBinaryIndexingSpi());
+        Ignite ignite = startGrid(0);
 
-        Ignite ignite = Ignition.start(cfg);
-
-        CacheConfiguration<PersonKey, Person> ccfg = cacheConfiguration(CACHE_NAME);
+        CacheConfiguration<PersonKey, Person> ccfg = cacheConfiguration(DEFAULT_CACHE_NAME);
 
         IgniteCache<PersonKey, Person> cache = ignite.createCache(ccfg);
 
@@ -170,13 +164,11 @@ public void testBinaryIndexingSpi() throws Exception {
     public void testNonBinaryIndexingSpi() throws Exception {
         System.setProperty(IgniteSystemProperties.IGNITE_UNWRAP_BINARY_FOR_INDEXING_SPI, "true");
 
-        IgniteConfiguration cfg = configuration();
-
-        cfg.setIndexingSpi(new MyIndexingSpi());
+        indexingSpi = new MyIndexingSpi();
 
-        Ignite ignite = Ignition.start(cfg);
+        Ignite ignite = startGrid(0);
 
-        CacheConfiguration<PersonKey, Person> ccfg = cacheConfiguration(CACHE_NAME);
+        CacheConfiguration<PersonKey, Person> ccfg = cacheConfiguration(DEFAULT_CACHE_NAME);
 
         IgniteCache<PersonKey, Person> cache = ignite.createCache(ccfg);
 
@@ -200,13 +192,11 @@ public void testNonBinaryIndexingSpi() throws Exception {
      */
     @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
     public void testIndexingSpiFailure() throws Exception {
-        IgniteConfiguration cfg = configuration();
-
-        cfg.setIndexingSpi(new MyBrokenIndexingSpi());
+        indexingSpi = new MyBrokenIndexingSpi();
 
-        Ignite ignite = Ignition.start(cfg);
+        Ignite ignite = startGrid(0);
 
-        CacheConfiguration<Integer, Integer> ccfg = cacheConfiguration(CACHE_NAME);
+        CacheConfiguration<Integer, Integer> ccfg = cacheConfiguration(DEFAULT_CACHE_NAME);
 
         ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryTxSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryTxSelfTest.java
index e59deed2eb767..9bc0540afbd1a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryTxSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/query/IndexingSpiQueryTxSelfTest.java
@@ -17,6 +17,11 @@
 
 package org.apache.ignite.internal.processors.cache.query;
 
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+import javax.cache.Cache;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteTransactions;
 import org.apache.ignite.cache.CacheAtomicityMode;
@@ -37,61 +42,64 @@
 import org.apache.ignite.transactions.TransactionState;
 import org.jetbrains.annotations.Nullable;
 
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicInteger;
-import javax.cache.Cache;
-
 /**
  * Indexing Spi transactional query test
  */
 public class IndexingSpiQueryTxSelfTest extends GridCacheAbstractSelfTest {
-    /** */
-    private static AtomicInteger cnt;
-
     /** {@inheritDoc} */
     @Override protected int gridCount() {
         return 4;
     }
 
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        cnt = new AtomicInteger();
-
-        super.beforeTestsStarted();
-    }
-
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
         ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setForceServerMode(true);
 
-        if (cnt.getAndIncrement() == 0)
-            cfg.setClientMode(true);
-        else {
-            cfg.setIndexingSpi(new MyBrokenIndexingSpi());
+        cfg.setClientMode("client".equals(igniteInstanceName));
+        cfg.setIndexingSpi(new MyBrokenIndexingSpi());
 
-            CacheConfiguration ccfg = cacheConfiguration(igniteInstanceName);
-            ccfg.setName("test-cache");
-            ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+        CacheConfiguration ccfg = cacheConfiguration(igniteInstanceName);
+        ccfg.setName(DEFAULT_CACHE_NAME);
+        ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
 
-            ccfg.setIndexedTypes(Integer.class, Integer.class);
+        ccfg.setIndexedTypes(Integer.class, Integer.class);
+
+        cfg.setCacheConfiguration(ccfg);
 
-            cfg.setCacheConfiguration(ccfg);
-        }
         return cfg;
     }
 
+    /** */
+    public void testIndexingSpiWithTxClient() throws Exception {
+        IgniteEx client = (IgniteEx)startGrid("client");
+
+        assertNotNull(client.cache(DEFAULT_CACHE_NAME));
+
+        doTestIndexingSpiWithTx(client, 0);
+    }
+
+    /** */
+    public void testIndexingSpiWithTxLocal() throws Exception {
+        IgniteEx ignite = (IgniteEx)primaryNode(0, DEFAULT_CACHE_NAME);
+
+        doTestIndexingSpiWithTx(ignite, 0);
+    }
+
+    /** */
+    public void testIndexingSpiWithTxNotLocal() throws Exception {
+        IgniteEx ignite = (IgniteEx)primaryNode(0, DEFAULT_CACHE_NAME);
+
+        doTestIndexingSpiWithTx(ignite, 1);
+    }
+
     /**
      * @throws Exception If failed.
      */
     @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
-    public void testIndexingSpiWithTx() throws Exception {
-        IgniteEx ignite = grid(0);
-
-        final IgniteCache<Integer, Integer> cache = ignite.cache("test-cache");
+    private void doTestIndexingSpiWithTx(IgniteEx ignite, int key) throws Exception {
+        final IgniteCache<Integer, Integer> cache = ignite.cache(DEFAULT_CACHE_NAME);
 
         final IgniteTransactions txs = ignite.transactions();
 
@@ -104,7 +112,7 @@ public void testIndexingSpiWithTx() throws Exception {
                         Transaction tx;
 
                         try (Transaction tx0 = tx = txs.txStart(concurrency, isolation)) {
-                            cache.put(1, 1);
+                            cache.put(key, key);
 
                             tx0.commit();
                         }
@@ -114,6 +122,8 @@ public void testIndexingSpiWithTx() throws Exception {
                         return null;
                     }
                 }, IgniteTxHeuristicCheckedException.class);
+
+                checkFutures();
             }
         }
     }
@@ -135,7 +145,7 @@ private static class MyBrokenIndexingSpi extends IgniteSpiAdapter implements Ind
         /** {@inheritDoc} */
         @Override public Iterator<Cache.Entry<?, ?>> query(@Nullable String cacheName, Collection<Object> params,
             @Nullable IndexingQueryFilter filters) throws IgniteSpiException {
-           return null;
+            return null;
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java
index fe27e6e119d24..01db74726405f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/AbstractTransactionIntergrityTest.java
@@ -33,6 +33,7 @@
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cache.query.annotations.QuerySqlField;
 import org.apache.ignite.cluster.ClusterNode;
@@ -40,10 +41,10 @@
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.failure.FailureHandler;
 import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -73,7 +74,7 @@ public class AbstractTransactionIntergrityTest extends GridCommonAbstractTest {
     private static final int DFLT_ACCOUNTS_CNT = 32;
 
     /** Count of threads and caches. */
-    private static final int DFLT_TX_THREADS_CNT = 20;
+    private static final int DFLT_TX_THREADS_CNT = Runtime.getRuntime().availableProcessors();
 
     /** Count of nodes to start. */
     private static final int DFLT_NODES_CNT = 3;
@@ -126,16 +127,6 @@ protected boolean persistent() {
         return true;
     }
 
-    /**
-     * @return Flag enables cross-node transactions,
-     *         when primary partitions participating in transaction spreaded across several cluster nodes.
-     */
-    protected boolean crossNodeTransactions() {
-        // Commit error during cross node transactions breaks transaction integrity
-        // TODO: https://issues.apache.org/jira/browse/IGNITE-9086
-        return false;
-    }
-
     /** {@inheritDoc} */
     @Override protected FailureHandler getFailureHandler(String igniteInstanceName) {
         return new StopNodeFailureHandler();
@@ -148,14 +139,15 @@ protected boolean crossNodeTransactions() {
         cfg.setConsistentId(name);
 
         ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
-        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
-        cfg.setLocalHost("127.0.0.1");
 
         cfg.setDataStorageConfiguration(new DataStorageConfiguration()
             .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
-                .setMaxSize(256 * 1024 * 1024)
-                .setPersistenceEnabled(persistent()))
-        );
+                    .setPersistenceEnabled(persistent())
+                    .setMaxSize(50 * 1024 * 1024)
+            )
+            .setWalSegmentSize(16 * 1024 * 1024)
+            .setPageSize(1024)
+            .setWalMode(WALMode.LOG_ONLY));
 
         CacheConfiguration[] cacheConfigurations = new CacheConfiguration[txThreadsCount()];
 
@@ -178,6 +170,8 @@ protected boolean crossNodeTransactions() {
 
         cfg.setCacheConfiguration(cacheConfigurations);
 
+        cfg.setFailureDetectionTimeout(30_000);
+
         return cfg;
     }
 
@@ -219,8 +213,11 @@ protected boolean crossNodeTransactions() {
 
     /**
      * Test transfer amount.
+     *
+     * @param failoverScenario Scenario.
+     * @param colocatedAccounts {@code True} to use colocated on same primary node accounts.
      */
-    public void doTestTransferAmount(FailoverScenario failoverScenario) throws Exception {
+    public void doTestTransferAmount(FailoverScenario failoverScenario, boolean colocatedAccounts) throws Exception {
         failoverScenario.beforeNodesStarted();
 
         //given: started some nodes with client.
@@ -230,26 +227,26 @@ public void doTestTransferAmount(FailoverScenario failoverScenario) throws Excep
 
         igniteClient.cluster().active(true);
 
-        int[] initAmount = new int[txThreadsCount()];
+        int[] initAmounts = new int[txThreadsCount()];
         completedTxs = new ConcurrentLinkedHashMap[txThreadsCount()];
 
         //and: fill all accounts on all caches and calculate total amount for every cache.
         for (int cachePrefixIdx = 0; cachePrefixIdx < txThreadsCount(); cachePrefixIdx++) {
             IgniteCache<Integer, AccountState> cache = igniteClient.getOrCreateCache(cacheName(cachePrefixIdx));
 
-            AtomicInteger coinsCounter = new AtomicInteger();
+            AtomicInteger coinsCntr = new AtomicInteger();
 
             try (Transaction tx = igniteClient.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
                 for (int accountId = 0; accountId < accountsCount(); accountId++) {
-                    Set<Integer> initialAmount = generateCoins(coinsCounter, 5);
+                    Set<Integer> initAmount = generateCoins(coinsCntr, 5);
 
-                    cache.put(accountId, new AccountState(accountId, tx.xid(), initialAmount));
+                    cache.put(accountId, new AccountState(accountId, tx.xid(), initAmount));
                 }
 
                 tx.commit();
             }
 
-            initAmount[cachePrefixIdx] = coinsCounter.get();
+            initAmounts[cachePrefixIdx] = coinsCntr.get();
             completedTxs[cachePrefixIdx] = new ConcurrentLinkedHashMap();
         }
 
@@ -259,7 +256,8 @@ public void doTestTransferAmount(FailoverScenario failoverScenario) throws Excep
         ArrayList<Thread> transferThreads = new ArrayList<>();
 
         for (int i = 0; i < txThreadsCount(); i++) {
-            transferThreads.add(new TransferAmountTxThread(firstTransactionDone, igniteClient, cacheName(i), i));
+            transferThreads.add(new TransferAmountTxThread(firstTransactionDone,
+                igniteClient, cacheName(i), i, colocatedAccounts));
 
             transferThreads.get(i).start();
         }
@@ -268,13 +266,12 @@ public void doTestTransferAmount(FailoverScenario failoverScenario) throws Excep
 
         failoverScenario.afterFirstTransaction();
 
-        for (Thread thread : transferThreads) {
+        for (Thread thread : transferThreads)
             thread.join();
-        }
 
         failoverScenario.afterTransactionsFinished();
 
-        consistencyCheck(initAmount);
+        consistencyCheck(initAmounts);
     }
 
     /**
@@ -385,11 +382,11 @@ public AccountState addCoins(IgniteUuid txId, Set<Integer> coinsToAdd) {
 
         /**
          * @param txId Transaction id.
-         * @param coinsToRemove Coins to remove from current account.
+         * @param coinsToRmv Coins to remove from current account.
          * @return Account state with removed coins.
          */
-        public AccountState removeCoins(IgniteUuid txId, Set<Integer> coinsToRemove) {
-            return new AccountState(accId, txId, Sets.difference(coins, coinsToRemove).immutableCopy());
+        public AccountState removeCoins(IgniteUuid txId, Set<Integer> coinsToRmv) {
+            return new AccountState(accId, txId, Sets.difference(coins, coinsToRmv).immutableCopy());
         }
 
         /** {@inheritDoc} */
@@ -418,11 +415,11 @@ public AccountState removeCoins(IgniteUuid txId, Set<Integer> coinsToRemove) {
     /**
      * @param coinsNum Coins number.
      */
-    private Set<Integer> generateCoins(AtomicInteger coinsCounter, int coinsNum) {
+    private Set<Integer> generateCoins(AtomicInteger coinsCntr, int coinsNum) {
         Set<Integer> res = new HashSet<>();
 
         for (int i = 0; i < coinsNum; i++)
-            res.add(coinsCounter.incrementAndGet());
+            res.add(coinsCntr.incrementAndGet());
 
         return res;
     }
@@ -479,23 +476,35 @@ public TxState(AccountState before1, AccountState before2, AccountState after1,
     private class TransferAmountTxThread extends Thread {
         /** */
         private CountDownLatch firstTransactionLatch;
+
         /** */
         private IgniteEx ignite;
+
         /** */
         private String cacheName;
+
         /** */
-        private int txIndex;
+        private int workerIdx;
+
         /** */
         private Random random = new Random();
 
+        /** */
+        private final boolean colocatedAccounts;
+
         /**
          * @param ignite Ignite.
          */
-        private TransferAmountTxThread(CountDownLatch firstTransactionLatch, final IgniteEx ignite, String cacheName, int txIndex) {
+        private TransferAmountTxThread(CountDownLatch firstTransactionLatch,
+            final IgniteEx ignite,
+            String cacheName,
+            int workerIdx,
+            boolean colocatedAccounts) {
             this.firstTransactionLatch = firstTransactionLatch;
             this.ignite = ignite;
             this.cacheName = cacheName;
-            this.txIndex = txIndex;
+            this.workerIdx = workerIdx;
+            this.colocatedAccounts = colocatedAccounts;
         }
 
         /** {@inheritDoc} */
@@ -514,7 +523,6 @@ private TransferAmountTxThread(CountDownLatch firstTransactionLatch, final Ignit
         /**
          * @throws IgniteException if fails
          */
-        @SuppressWarnings("unchecked")
         private void updateInTransaction(IgniteCache<Integer, AccountState> cache) throws IgniteException {
             int accIdFrom;
             int accIdTo;
@@ -526,11 +534,16 @@ private void updateInTransaction(IgniteCache<Integer, AccountState> cache) throw
                 if (accIdFrom == accIdTo)
                     continue;
 
-                ClusterNode primaryForAccFrom = ignite.cachex(cacheName).affinity().mapKeyToNode(accIdFrom);
-                ClusterNode primaryForAccTo = ignite.cachex(cacheName).affinity().mapKeyToNode(accIdTo);
+                Affinity<Object> affinity = ignite.affinity(cacheName);
+
+                ClusterNode primaryForAccFrom = affinity.mapKeyToNode(accIdFrom);
+                assertNotNull(primaryForAccFrom);
+
+                ClusterNode primaryForAccTo = affinity.mapKeyToNode(accIdTo);
+                assertNotNull(primaryForAccTo);
 
                 // Allows only transaction between accounts that primary on the same node if corresponding flag is enabled.
-                if (!crossNodeTransactions() && !primaryForAccFrom.id().equals(primaryForAccTo.id()))
+                if (colocatedAccounts && !primaryForAccFrom.id().equals(primaryForAccTo.id()))
                     continue;
 
                 break;
@@ -541,7 +554,10 @@ private void updateInTransaction(IgniteCache<Integer, AccountState> cache) throw
 
             try (Transaction tx = ignite.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
                 acctFrom = cache.get(accIdFrom);
+                assertNotNull(acctFrom);
+
                 acctTo = cache.get(accIdTo);
+                assertNotNull(acctTo);
 
                 Set<Integer> coinsToTransfer = acctFrom.coinsToTransfer(random);
 
@@ -553,23 +569,8 @@ private void updateInTransaction(IgniteCache<Integer, AccountState> cache) throw
 
                 tx.commit();
 
-                completedTxs[txIndex].put(tx.xid(), new TxState(acctFrom, acctTo, nextFrom, nextTo, coinsToTransfer));
-            }
-        }
-
-        /**
-         * @param curr current
-         * @return random value
-         */
-        private long getNextAccountId(long curr) {
-            long randomVal;
-
-            do {
-                randomVal = random.nextInt(accountsCount());
+                completedTxs[workerIdx].put(tx.xid(), new TxState(acctFrom, acctTo, nextFrom, nextTo, coinsToTransfer));
             }
-            while (curr == randomVal);
-
-            return randomVal;
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java
index 3260607023a92..473eaf5ce97bc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java
@@ -17,20 +17,26 @@
 
 package org.apache.ignite.internal.processors.cache.transactions;
 
+import java.util.Collection;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.function.BiFunction;
+import java.util.function.Consumer;
 import java.util.function.Supplier;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteIllegalStateException;
 import org.apache.ignite.Ignition;
-import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
 import org.apache.ignite.internal.processors.cache.tree.SearchRow;
 import org.apache.ignite.testframework.GridTestUtils;
 
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
+
 /**
  * Test cases that check transaction data integrity after transaction commit failed.
  */
@@ -45,81 +51,96 @@ public class TransactionIntegrityWithPrimaryIndexCorruptionTest extends Abstract
         super.afterTest();
     }
 
-    /** {@inheritDoc} */
-    @Override protected long getTestTimeout() {
-        return 60 * 1000L;
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitPrimaryColocatedThrowsError() throws Exception {
+        doTestTransferAmount0(true, true, () -> new AssertionError("Test"));
     }
 
-    /**
-     * Throws a test {@link AssertionError} during tx commit from {@link BPlusTree} and checks after that data is consistent.
-     */
-    public void testPrimaryIndexCorruptionDuringCommitOnPrimaryNode1() throws Exception {
-        doTestTransferAmount(new IndexCorruptionFailoverScenario(
-            true,
-            (hnd, tree) -> hnd instanceof BPlusTree.Search,
-            failoverPredicate(true, () -> new AssertionError("Test")))
-        );
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitPrimaryColocatedThrowsUnchecked() throws Exception {
+        doTestTransferAmount0(true, true, () -> new RuntimeException("Test"));
     }
 
-    /**
-     * Throws a test {@link RuntimeException} during tx commit from {@link BPlusTree} and checks after that data is consistent.
-     */
-    public void testPrimaryIndexCorruptionDuringCommitOnPrimaryNode2() throws Exception {
-        doTestTransferAmount(new IndexCorruptionFailoverScenario(
-            true,
-            (hnd, tree) -> hnd instanceof BPlusTree.Search,
-            failoverPredicate(true, () -> new RuntimeException("Test")))
-        );
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitPrimaryColocatedThrowsChecked() throws Exception {
+        doTestTransferAmount0(true, true, () -> new IgniteCheckedException("Test"));
     }
 
-    /**
-     * Throws a test {@link AssertionError} during tx commit from {@link BPlusTree} and checks after that data is consistent.
-     */
-    public void testPrimaryIndexCorruptionDuringCommitOnBackupNode() throws Exception {
-        doTestTransferAmount(new IndexCorruptionFailoverScenario(
-            true,
-            (hnd, tree) -> hnd instanceof BPlusTree.Search,
-            failoverPredicate(false, () -> new AssertionError("Test")))
-        );
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitPrimaryNonColocatedThrowsError() throws Exception {
+        doTestTransferAmount0(false, true, () -> new AssertionError("Test"));
     }
 
-    /**
-     * Throws a test {@link IgniteCheckedException} during tx commit from {@link BPlusTree} and checks after that data is consistent.
-     */
-    public void testPrimaryIndexCorruptionDuringCommitOnPrimaryNode3() throws Exception {
-        fail("https://issues.apache.org/jira/browse/IGNITE-9082");
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitPrimaryNonColocatedThrowsUnchecked() throws Exception {
+        doTestTransferAmount0(false, true, () -> new RuntimeException("Test"));
+    }
 
-        doTestTransferAmount(new IndexCorruptionFailoverScenario(
-            false,
-            (hnd, tree) -> hnd instanceof BPlusTree.Search,
-            failoverPredicate(true, () -> new IgniteCheckedException("Test")))
-        );
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitPrimaryNonColocatedThrowsChecked() throws Exception {
+        doTestTransferAmount0(false, true, () -> new IgniteCheckedException("Test"));
+    }
+
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitBackupColocatedThrowsError() throws Exception {
+        doTestTransferAmount0(true, false, () -> new AssertionError("Test"));
+    }
+
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitBackupColocatedThrowsUnchecked() throws Exception {
+        doTestTransferAmount0(true, false, () -> new RuntimeException("Test"));
+    }
+
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitBackupColocatedThrowsChecked() throws Exception {
+        doTestTransferAmount0(true, false, () -> new IgniteCheckedException("Test"));
+    }
+
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitBackupNonColocatedThrowsError() throws Exception {
+        doTestTransferAmount0(false, false, () -> new AssertionError("Test"));
+    }
+
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitBackupNonColocatedThrowsUnchecked() throws Exception {
+        doTestTransferAmount0(false, false, () -> new RuntimeException("Test"));
+    }
+
+    /** */
+    public void testPrimaryIndexCorruptionDuringCommitBackupNonColocatedThrowsChecked() throws Exception {
+        doTestTransferAmount0(false, false, () -> new IgniteCheckedException("Test"));
     }
 
     /**
      * Creates failover predicate which generates error during transaction commmit.
      *
-     * @param failOnPrimary If {@code true} index should be failed on transaction primary node.
+     * @param failOnPrimary If {@code true} index should be failed on transaction primary node, otherwise on backup.
      * @param errorSupplier Supplier to create various errors.
+     * @param errorConsumer Consumer to track unexpected errors while committing.
      */
     private BiFunction<IgniteEx, SearchRow, Throwable> failoverPredicate(
         boolean failOnPrimary,
-        Supplier<Throwable> errorSupplier
+        Supplier<Throwable> errorSupplier,
+        Consumer<Throwable> errorConsumer
     ) {
         return (ignite, row) -> {
-            int cacheId = row.cacheId();
-            int partId = row.key().partition();
-
-            final ClusterNode locNode = ignite.localNode();
-            final AffinityTopologyVersion curTopVer = ignite.context().discovery().topologyVersionEx();
-
-            // Throw exception if current node is primary for given row.
-            return ignite.cachesx(c -> c.context().cacheId() == cacheId)
-                .stream()
-                .filter(c -> c.context().affinity().primaryByPartition(locNode, partId, curTopVer) == failOnPrimary)
-                .map(c -> errorSupplier.get())
-                .findFirst()
-                .orElse(null);
+            try {
+                int cacheId = row.cacheId();
+                int partId = row.key().partition();
+
+                GridDhtPartitionTopology top = ignite.context().cache().cacheGroup(cacheId).topology();
+
+                GridDhtLocalPartition part = top.localPartition(partId);
+
+                assertTrue("Illegal partition state for mapped tx: " + part, part != null && part.state() == OWNING);
+
+                return part.primary(top.readyTopologyVersion()) == failOnPrimary ? errorSupplier.get() : null;
+            }
+            catch (Throwable e) {
+                errorConsumer.accept(e);
+
+                throw e;
+            }
         };
     }
 
@@ -130,68 +151,68 @@ class IndexCorruptionFailoverScenario implements FailoverScenario {
         /** Failed node index. */
         static final int failedNodeIdx = 1;
 
-        /** Is node stopping expected after failover. */
-        private final boolean nodeStoppingExpected;
-
-        /** Predicate that will choose an instance of {@link BPlusTree} and page operation
-         * to make further failover in this tree using {@link #failoverPredicate}. */
-        private final BiFunction<PageHandler, BPlusTree, Boolean> treeCorruptionPredicate;
+        /**
+         * Predicate that will choose an instance of {@link BPlusTree} and page operation to make further failover in
+         * this tree using {@link #failoverPred}.
+         */
+        private final BiFunction<PageHandler, BPlusTree, Boolean> treeCorruptionPred;
 
         /** Function that may return error during row insertion into {@link BPlusTree}. */
-        private final BiFunction<IgniteEx, SearchRow, Throwable> failoverPredicate;
+        private final BiFunction<IgniteEx, SearchRow, Throwable> failoverPred;
 
         /**
-         * @param nodeStoppingExpected Node stopping expected.
-         * @param treeCorruptionPredicate Tree corruption predicate.
-         * @param failoverPredicate Failover predicate.
+         * @param treeCorruptionPred Tree corruption predicate.
+         * @param failoverPred Failover predicate.
          */
         IndexCorruptionFailoverScenario(
-            boolean nodeStoppingExpected,
-            BiFunction<PageHandler, BPlusTree, Boolean> treeCorruptionPredicate,
-            BiFunction<IgniteEx, SearchRow, Throwable> failoverPredicate
+            BiFunction<PageHandler, BPlusTree, Boolean> treeCorruptionPred,
+            BiFunction<IgniteEx, SearchRow, Throwable> failoverPred
         ) {
-            this.nodeStoppingExpected = nodeStoppingExpected;
-            this.treeCorruptionPredicate = treeCorruptionPredicate;
-            this.failoverPredicate = failoverPredicate;
+            this.treeCorruptionPred = treeCorruptionPred;
+            this.failoverPred = failoverPred;
         }
 
         /** {@inheritDoc} */
         @Override public void beforeNodesStarted() {
             BPlusTree.pageHndWrapper = (tree, hnd) -> {
-                final IgniteEx locIgnite = (IgniteEx) Ignition.localIgnite();
+                final IgniteEx locIgnite = (IgniteEx)Ignition.localIgnite();
 
-                if (!locIgnite.name().endsWith(String.valueOf(failedNodeIdx)))
+                if (getTestIgniteInstanceIndex(locIgnite.name()) != failedNodeIdx)
                     return hnd;
 
-                if (treeCorruptionPredicate.apply(hnd, tree)) {
-                    log.info("Created corrupted tree handler for -> " + hnd + " " + tree);
+                if (treeCorruptionPred.apply(hnd, tree)) {
+                    log.info("Created corrupted tree handler [nodeOrder=" + locIgnite.localNode().order() + ", hnd=" + hnd +
+                        ", tree=" + tree + ']');
 
-                    PageHandler<Object, BPlusTree.Result> delegate = (PageHandler<Object, BPlusTree.Result>) hnd;
+                    PageHandler<Object, BPlusTree.Result> delegate = (PageHandler<Object, BPlusTree.Result>)hnd;
 
                     return new PageHandler<BPlusTree.Get, BPlusTree.Result>() {
-                        @Override public BPlusTree.Result run(int cacheId, long pageId, long page, long pageAddr, PageIO io, Boolean walPlc, BPlusTree.Get arg, int lvl) throws IgniteCheckedException {
-                            log.info("Invoked " + " " + cacheId + " " + arg.toString() + " for BTree (" + corruptionEnabled + ") -> " + arg.row() + " / " + arg.row().getClass());
+                        @Override public BPlusTree.Result run(int cacheId, long pageId, long page, long pageAddr, PageIO io,
+                            Boolean walPlc, BPlusTree.Get arg, int lvl) throws IgniteCheckedException {
+                            log.info("Invoked [cachedId=" + cacheId + ", hnd=" + arg.toString() +
+                                ", corruption=" + corruptionEnabled + ", row=" + arg.row() + ", rowCls=" + arg.row().getClass() + ']');
 
                             if (corruptionEnabled && (arg.row() instanceof SearchRow)) {
-                                SearchRow row = (SearchRow) arg.row();
+                                SearchRow row = (SearchRow)arg.row();
 
                                 // Store cacheId to search row explicitly, as it can be zero if there is one cache in a group.
-                                Throwable res = failoverPredicate.apply(locIgnite, new SearchRow(cacheId, row.key()));
+                                Throwable res = failoverPred.apply(locIgnite, new SearchRow(cacheId, row.key()));
 
                                 if (res != null) {
                                     if (res instanceof Error)
-                                        throw (Error) res;
+                                        throw (Error)res;
                                     else if (res instanceof RuntimeException)
-                                        throw (RuntimeException) res;
+                                        throw (RuntimeException)res;
                                     else if (res instanceof IgniteCheckedException)
-                                        throw (IgniteCheckedException) res;
+                                        throw (IgniteCheckedException)res;
                                 }
                             }
 
                             return delegate.run(cacheId, pageId, page, pageAddr, io, walPlc, arg, lvl);
                         }
 
-                        @Override public boolean releaseAfterWrite(int cacheId, long pageId, long page, long pageAddr, BPlusTree.Get g, int lvl) {
+                        @Override public boolean releaseAfterWrite(int cacheId, long pageId, long page, long pageAddr,
+                            BPlusTree.Get g, int lvl) {
                             return g.canRelease(pageId, lvl);
                         }
                     };
@@ -212,27 +233,68 @@ else if (res instanceof IgniteCheckedException)
             // Disable index corruption.
             BPlusTree.pageHndWrapper = (tree, hnd) -> hnd;
 
-            if (nodeStoppingExpected) {
-                // Wait until node with corrupted index will left cluster.
-                GridTestUtils.waitForCondition(() -> {
-                    try {
-                        grid(failedNodeIdx);
-                    }
-                    catch (IgniteIllegalStateException e) {
-                        return true;
-                    }
+            // Wait until node with corrupted index will left cluster.
+            GridTestUtils.waitForCondition(() -> {
+                try {
+                    grid(failedNodeIdx);
+                }
+                catch (IgniteIllegalStateException e) {
+                    return true;
+                }
 
-                    return false;
-                }, getTestTimeout());
+                return false;
+            }, getTestTimeout());
 
-                // Failed node should be stopped.
-                GridTestUtils.assertThrows(log, () -> grid(failedNodeIdx), IgniteIllegalStateException.class, "");
+            // Failed node should be stopped.
+            GridTestUtils.assertThrows(log, () -> grid(failedNodeIdx), IgniteIllegalStateException.class, null);
 
-                // Re-start failed node.
-                startGrid(failedNodeIdx);
+            // Re-start failed node.
+            startGrid(failedNodeIdx);
 
-                awaitPartitionMapExchange();
-            }
+            awaitPartitionMapExchange();
+        }
+    }
+
+    /**
+     * Test transfer amount with extended error recording.
+     *
+     * @param colocatedAccount Colocated account.
+     * @param failOnPrimary {@code True} if fail on primary, else on backup.
+     * @param supplier Fail reason supplier.
+     * @throws Exception If failover predicate execution is failed.
+     */
+    private void doTestTransferAmount0(boolean colocatedAccount, boolean failOnPrimary, Supplier<Throwable> supplier) throws Exception {
+        ErrorTracker errTracker = new ErrorTracker();
+
+        doTestTransferAmount(
+            new IndexCorruptionFailoverScenario(
+                (hnd, tree) -> hnd instanceof BPlusTree.Search,
+                failoverPredicate(failOnPrimary, supplier, errTracker)),
+            colocatedAccount
+        );
+
+        for (Throwable throwable : errTracker.errors())
+            log.error("Recorded error", throwable);
+
+        if (!errTracker.errors().isEmpty())
+            fail("Test run has error");
+    }
+
+    /** */
+    private static class ErrorTracker implements Consumer<Throwable> {
+        /** Queue. */
+        private final Queue<Throwable> q = new ConcurrentLinkedQueue<>();
+
+        /** {@inheritDoc} */
+        @Override public void accept(Throwable throwable) {
+            q.add(throwable);
+        }
+
+        /**
+         * @return Recorded errors.
+         */
+        public Collection<Throwable> errors() {
+            return q;
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java
index 25aae4b2a1b47..551335f2fee6f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithSystemWorkerDeathTest.java
@@ -41,9 +41,7 @@ public class TransactionIntegrityWithSystemWorkerDeathTest extends AbstractTrans
         return false;
     }
 
-    /**
-     *
-     */
+    /** */
     public void testFailoverWithDiscoWorkerTermination() throws Exception {
         doTestTransferAmount(new FailoverScenario() {
             static final int failedNodeIdx = 1;
@@ -83,7 +81,7 @@ public void testFailoverWithDiscoWorkerTermination() throws Exception {
 
                 awaitPartitionMapExchange();
             }
-        });
+        }, true);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDataConsistencyOnCommitFailureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDataConsistencyOnCommitFailureTest.java
new file mode 100644
index 0000000000000..98c77a8a76fe5
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxDataConsistencyOnCommitFailureTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.UUID;
+import java.util.function.Supplier;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteTransactions;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testsuites.IgniteIgnore;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.jetbrains.annotations.Nullable;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+/**
+ * Tests data consistency if transaction is failed due to heuristic exception on originating node.
+ */
+public class TxDataConsistencyOnCommitFailureTest extends GridCommonAbstractTest {
+    /** */
+    public static final int KEY = 0;
+
+    /** */
+    public static final String CLIENT = "client";
+
+    /** */
+    private int nodesCnt;
+
+    /** */
+    private int backups;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setClientMode(igniteInstanceName.startsWith(CLIENT));
+
+        cfg.setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME).
+            setCacheMode(CacheMode.PARTITIONED).
+            setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL).
+            setBackups(backups).
+            setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /** */
+    public void testCommitErrorOnNearNode2PC() throws Exception {
+        nodesCnt = 3;
+
+        backups = 2;
+
+        doTestCommitError(() -> {
+            try {
+                return startGrid("client");
+            }
+            catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    /** */
+    public void testCommitErrorOnNearNode1PC() throws Exception {
+        nodesCnt = 2;
+
+        backups = 1;
+
+        doTestCommitError(() -> {
+            try {
+                return startGrid("client");
+            }
+            catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        });
+    }
+
+    /** */
+    @IgniteIgnore(value = "https://issues.apache.org/jira/browse/IGNITE-9806", forceFailure = false)
+    public void testCommitErrorOnColocatedNode2PC() throws Exception {
+        nodesCnt = 3;
+
+        backups = 2;
+
+        doTestCommitError(() -> primaryNode(KEY, DEFAULT_CACHE_NAME));
+    }
+
+    /**
+     * @param factory Factory.
+     */
+    private void doTestCommitError(Supplier<Ignite> factory) throws Exception {
+        Ignite crd = startGridsMultiThreaded(nodesCnt);
+
+        crd.cache(DEFAULT_CACHE_NAME).put(KEY, KEY);
+
+        Ignite ignite = factory.get();
+
+        if (ignite == null)
+            ignite = startGrid("client");
+
+        assertNotNull(ignite.cache(DEFAULT_CACHE_NAME));
+
+        injectMockedTxManager(ignite);
+
+        checkKey();
+
+        IgniteTransactions transactions = ignite.transactions();
+
+        try(Transaction tx = transactions.txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ, 0, 1)) {
+            assertNotNull(transactions.tx());
+
+            ignite.cache(DEFAULT_CACHE_NAME).put(KEY, KEY + 1);
+
+            tx.commit();
+
+            fail();
+        }
+        catch (Exception t) {
+            // No-op.
+        }
+
+        checkKey();
+
+        checkFutures();
+    }
+
+    /**
+     * @param ignite Ignite.
+     */
+    private void injectMockedTxManager(Ignite ignite) {
+        IgniteEx igniteEx = (IgniteEx)ignite;
+
+        GridCacheSharedContext<Object, Object> ctx = igniteEx.context().cache().context();
+
+        IgniteTxManager tm = ctx.tm();
+
+        IgniteTxManager mockTm = Mockito.spy(tm);
+
+        MockGridNearTxLocal locTx = new MockGridNearTxLocal(ctx, false, false, false, GridIoPolicy.SYSTEM_POOL,
+            TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ, 0, true, 1, null, 0, null);
+
+        Mockito.doAnswer(new Answer<GridNearTxLocal>() {
+            @Override public GridNearTxLocal answer(InvocationOnMock invocation) throws Throwable {
+                mockTm.onCreated(null, locTx);
+
+                return locTx;
+            }
+        }).when(mockTm).
+            newTx(locTx.implicit(), locTx.implicitSingle(), null, locTx.concurrency(),
+                locTx.isolation(), locTx.timeout(), locTx.storeEnabled(), locTx.size(), locTx.label());
+
+        ctx.setTxManager(mockTm);
+    }
+
+    /** */
+    private void checkKey() {
+        for (Ignite ignite : G.allGrids()) {
+            if (!ignite.configuration().isClientMode())
+                assertNotNull(ignite.cache(DEFAULT_CACHE_NAME).localPeek(KEY));
+        }
+    }
+
+    /** */
+    private static class MockGridNearTxLocal extends GridNearTxLocal {
+        /** Empty constructor. */
+        public MockGridNearTxLocal() {
+        }
+
+        /**
+         * @param ctx Context.
+         * @param implicit Implicit.
+         * @param implicitSingle Implicit single.
+         * @param sys System.
+         * @param plc Policy.
+         * @param concurrency Concurrency.
+         * @param isolation Isolation.
+         * @param timeout Timeout.
+         * @param storeEnabled Store enabled.
+         * @param txSize Tx size.
+         * @param subjId Subj id.
+         * @param taskNameHash Task name hash.
+         * @param lb Label.
+         */
+        public MockGridNearTxLocal(GridCacheSharedContext ctx, boolean implicit, boolean implicitSingle, boolean sys,
+            byte plc, TransactionConcurrency concurrency, TransactionIsolation isolation, long timeout,
+            boolean storeEnabled, int txSize, @Nullable UUID subjId, int taskNameHash,
+            @Nullable String lb) {
+            super(ctx, implicit, implicitSingle, sys, plc, concurrency, isolation, timeout, storeEnabled, txSize,
+                subjId, taskNameHash, lb);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void userCommit() throws IgniteCheckedException {
+            throw new IgniteCheckedException("Force failure");
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index 7be2545aa9b22..e6c9076a23496 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -2016,17 +2016,26 @@ protected void checkFutures() {
 
             final Collection<GridCacheFuture<?>> futs = ig.context().cache().context().mvcc().activeFutures();
 
-            for (GridCacheFuture<?> fut : futs)
-                log.info("Waiting for future: " + fut);
+            boolean hasFutures = false;
 
-            assertTrue("Expecting no active futures: node=" + ig.localNode().id(), futs.isEmpty());
+            for (GridCacheFuture<?> fut : futs) {
+                if (!fut.isDone()) {
+                    log.error("Expecting no active future [node=" + ig.localNode().id() + ", fut=" + fut + ']');
+
+                    hasFutures = true;
+                }
+            }
+
+            if (hasFutures)
+                fail("Some mvcc futures are not finished");
 
             Collection<IgniteInternalTx> txs = ig.context().cache().context().tm().activeTransactions();
 
             for (IgniteInternalTx tx : txs)
-                log.info("Waiting for tx: " + tx);
+                log.error("Expecting no active transaction [node=" + ig.localNode().id() + ", tx=" + tx + ']');
 
-            assertTrue("Expecting no active transactions: node=" + ig.localNode().id(), txs.isEmpty());
+            if (!txs.isEmpty())
+                fail("Some transaction are not finished");
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java
new file mode 100644
index 0000000000000..7dba46114ccbc
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite9.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.CachePutIfAbsentTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheGetCustomCollectionsSelfTest;
+import org.apache.ignite.internal.processors.cache.IgniteCacheLoadRebalanceEvictionSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheAtomicPrimarySyncBackPressureTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCachePrimarySyncTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCachePrimarySyncTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteTxCacheWriteSynchronizationModesMultithreadedTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxDataConsistencyOnCommitFailureTest;
+import org.apache.ignite.testframework.junits.GridAbstractTest;
+
+/**
+ * Test suite.
+ */
+public class IgniteCacheTestSuite9 extends TestSuite {
+    /**
+     * @return IgniteCache test suite.
+     * @throws Exception Thrown in case of the failure.
+     */
+    public static TestSuite suite() throws Exception {
+        System.setProperty(GridAbstractTest.PERSISTENCE_IN_TESTS_IS_ALLOWED_PROPERTY, "false");
+
+        TestSuite suite = new TestSuite("IgniteCache Test Suite part 9");
+
+        suite.addTestSuite(IgniteCacheGetCustomCollectionsSelfTest.class);
+        suite.addTestSuite(IgniteCacheLoadRebalanceEvictionSelfTest.class);
+        suite.addTestSuite(IgniteCachePrimarySyncTest.class);
+        suite.addTestSuite(IgniteTxCachePrimarySyncTest.class);
+        suite.addTestSuite(IgniteTxCacheWriteSynchronizationModesMultithreadedTest.class);
+        suite.addTestSuite(CachePutIfAbsentTest.class);
+
+        suite.addTestSuite(CacheAtomicPrimarySyncBackPressureTest.class);
+
+        suite.addTestSuite(TxDataConsistencyOnCommitFailureTest.class);
+
+        return suite;
+    }
+}

From b37894050a2a6420e6f973696b831ad79fdfd75c Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Fri, 19 Oct 2018 18:05:21 +0300
Subject: [PATCH 440/543] IGN-12001 Fixed pom.xml and compatibility tests and
 gridgain.properties with actual incompatible versions.

---
 examples/pom.xml                                              | 4 ++--
 modules/aop/pom.xml                                           | 2 +-
 modules/apache-license-gen/pom.xml                            | 2 +-
 modules/aws/pom.xml                                           | 2 +-
 modules/benchmarks/pom.xml                                    | 2 +-
 modules/camel/pom.xml                                         | 2 +-
 modules/cassandra/pom.xml                                     | 2 +-
 modules/cassandra/serializers/pom.xml                         | 4 ++--
 modules/cassandra/store/pom.xml                               | 4 ++--
 modules/clients/pom.xml                                       | 2 +-
 modules/cloud/pom.xml                                         | 2 +-
 modules/codegen/pom.xml                                       | 2 +-
 modules/compatibility/pom.xml                                 | 2 +-
 modules/core/pom.xml                                          | 2 +-
 .../distributed/dht/preloader/latch/ExchangeLatchManager.java | 3 +--
 modules/core/src/main/resources/ignite.properties             | 2 +-
 modules/dev-utils/pom.xml                                     | 2 +-
 modules/direct-io/pom.xml                                     | 2 +-
 modules/extdata/p2p/pom.xml                                   | 2 +-
 modules/extdata/platform/pom.xml                              | 2 +-
 modules/extdata/uri/modules/uri-dependency/pom.xml            | 4 ++--
 modules/extdata/uri/pom.xml                                   | 2 +-
 modules/flink/pom.xml                                         | 2 +-
 modules/flume/pom.xml                                         | 2 +-
 modules/gce/pom.xml                                           | 2 +-
 modules/geospatial/pom.xml                                    | 2 +-
 modules/hadoop/pom.xml                                        | 2 +-
 modules/hibernate-4.2/pom.xml                                 | 2 +-
 modules/hibernate-5.1/pom.xml                                 | 2 +-
 modules/hibernate-core/pom.xml                                | 2 +-
 modules/ignored-tests/pom.xml                                 | 2 +-
 modules/indexing/pom.xml                                      | 2 +-
 modules/jcl/pom.xml                                           | 2 +-
 modules/jms11/pom.xml                                         | 2 +-
 modules/jta/pom.xml                                           | 2 +-
 modules/kafka/pom.xml                                         | 2 +-
 modules/kubernetes/pom.xml                                    | 2 +-
 modules/log4j/pom.xml                                         | 2 +-
 modules/log4j2/pom.xml                                        | 2 +-
 modules/mesos/pom.xml                                         | 4 ++--
 modules/ml/pom.xml                                            | 2 +-
 modules/mqtt/pom.xml                                          | 2 +-
 modules/osgi-karaf/pom.xml                                    | 2 +-
 modules/osgi-paxlogging/pom.xml                               | 2 +-
 modules/osgi/pom.xml                                          | 2 +-
 modules/rest-http/pom.xml                                     | 2 +-
 modules/rocketmq/pom.xml                                      | 2 +-
 modules/scalar-2.10/pom.xml                                   | 2 +-
 modules/scalar/pom.xml                                        | 2 +-
 modules/schedule/pom.xml                                      | 2 +-
 modules/slf4j/pom.xml                                         | 2 +-
 modules/spark-2.10/pom.xml                                    | 2 +-
 modules/spark/pom.xml                                         | 2 +-
 modules/spring-data/pom.xml                                   | 2 +-
 modules/spring/pom.xml                                        | 2 +-
 modules/sqlline/pom.xml                                       | 2 +-
 modules/ssh/pom.xml                                           | 2 +-
 modules/storm/pom.xml                                         | 2 +-
 modules/tools/pom.xml                                         | 2 +-
 modules/twitter/pom.xml                                       | 2 +-
 modules/urideploy/pom.xml                                     | 2 +-
 modules/visor-console-2.10/pom.xml                            | 2 +-
 modules/visor-console/pom.xml                                 | 2 +-
 modules/visor-plugins/pom.xml                                 | 2 +-
 modules/web-console/pom.xml                                   | 2 +-
 modules/web-console/web-agent/pom.xml                         | 2 +-
 modules/web/ignite-appserver-test/pom.xml                     | 2 +-
 modules/web/ignite-websphere-test/pom.xml                     | 2 +-
 modules/web/pom.xml                                           | 2 +-
 modules/yardstick/pom.xml                                     | 2 +-
 modules/yarn/pom.xml                                          | 2 +-
 modules/zeromq/pom.xml                                        | 2 +-
 modules/zookeeper/pom.xml                                     | 2 +-
 pom.xml                                                       | 2 +-
 74 files changed, 79 insertions(+), 80 deletions(-)

diff --git a/examples/pom.xml b/examples/pom.xml
index cab1a89bb61e2..299363ac31316 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -27,7 +27,7 @@
     </parent>
 
     <artifactId>ignite-examples</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
@@ -298,4 +298,4 @@
             </plugin>
         </plugins>
     </build>
-</project>
\ No newline at end of file
+</project>
diff --git a/modules/aop/pom.xml b/modules/aop/pom.xml
index 3e727d985083a..6f10f6ec31630 100644
--- a/modules/aop/pom.xml
+++ b/modules/aop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aop</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/apache-license-gen/pom.xml b/modules/apache-license-gen/pom.xml
index 31ab20e14eb19..68494fc00b488 100644
--- a/modules/apache-license-gen/pom.xml
+++ b/modules/apache-license-gen/pom.xml
@@ -31,7 +31,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>ignite-apache-license-gen</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <build>
diff --git a/modules/aws/pom.xml b/modules/aws/pom.xml
index 8540b84e87c8b..d2fb36cd76425 100644
--- a/modules/aws/pom.xml
+++ b/modules/aws/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aws</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/benchmarks/pom.xml b/modules/benchmarks/pom.xml
index d6789e639be18..865ad53d5b688 100644
--- a/modules/benchmarks/pom.xml
+++ b/modules/benchmarks/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-benchmarks</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/camel/pom.xml b/modules/camel/pom.xml
index a5017e18e263b..a295a46928ef3 100644
--- a/modules/camel/pom.xml
+++ b/modules/camel/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-camel</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/pom.xml b/modules/cassandra/pom.xml
index e01a95516ce71..4914c77d0c6ed 100644
--- a/modules/cassandra/pom.xml
+++ b/modules/cassandra/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-cassandra</artifactId>
     <packaging>pom</packaging>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencyManagement>
diff --git a/modules/cassandra/serializers/pom.xml b/modules/cassandra/serializers/pom.xml
index 58c08a9369243..8703d77367dd3 100644
--- a/modules/cassandra/serializers/pom.xml
+++ b/modules/cassandra/serializers/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>2.5.0-SNAPSHOT</version>
+        <version>2.5.1-p150-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-serializers</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/store/pom.xml b/modules/cassandra/store/pom.xml
index b62050b1c1218..3a0da5191ed36 100644
--- a/modules/cassandra/store/pom.xml
+++ b/modules/cassandra/store/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>2.5.0-SNAPSHOT</version>
+        <version>2.5.1-p150-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-store</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/clients/pom.xml b/modules/clients/pom.xml
index e75600b925e79..8df6a8c85fc5c 100644
--- a/modules/clients/pom.xml
+++ b/modules/clients/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-clients</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/cloud/pom.xml b/modules/cloud/pom.xml
index 00ec33b78f833..c750f609ecc47 100644
--- a/modules/cloud/pom.xml
+++ b/modules/cloud/pom.xml
@@ -29,7 +29,7 @@
     </parent>
 
     <artifactId>ignite-cloud</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/codegen/pom.xml b/modules/codegen/pom.xml
index 345f7130485dc..2320f56b42b42 100644
--- a/modules/codegen/pom.xml
+++ b/modules/codegen/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-codegen</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/compatibility/pom.xml b/modules/compatibility/pom.xml
index a04d7348f59b5..1794f4375865a 100644
--- a/modules/compatibility/pom.xml
+++ b/modules/compatibility/pom.xml
@@ -33,7 +33,7 @@
     </parent>
 
     <artifactId>ignite-compatibility</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 68c19a7bada45..491c85c0dc3db 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-core</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <repositories>
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
index 08adc743d90a2..67c3629139ae4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/ExchangeLatchManager.java
@@ -45,7 +45,6 @@
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.jetbrains.annotations.Nullable;
@@ -65,7 +64,7 @@ public class ExchangeLatchManager {
      * Exchange latch V2 protocol introduces following optimization:
      * Joining nodes are explicitly excluded from possible latch participants.
      */
-    public static final IgniteProductVersion PROTOCOL_V2_VERSION_SINCE = IgniteProductVersion.fromString("2.5.1-p14");
+    public static final IgniteProductVersion PROTOCOL_V2_VERSION_SINCE = IgniteProductVersion.fromString("2.5.1");
 
     /** Logger. */
     private final IgniteLogger log;
diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index e70660b1c6a64..db63ec1062b28 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=2.5.0-SNAPSHOT
+ignite.version=2.5.1-p150-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970
diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml
index 2a0cf48e37036..afd6d8858f8e6 100644
--- a/modules/dev-utils/pom.xml
+++ b/modules/dev-utils/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-dev-utils</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/direct-io/pom.xml b/modules/direct-io/pom.xml
index d65b6aea858b6..628a23ee95290 100644
--- a/modules/direct-io/pom.xml
+++ b/modules/direct-io/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-direct-io</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/extdata/p2p/pom.xml b/modules/extdata/p2p/pom.xml
index 02bed30fd7d74..46ae0fccde7ec 100644
--- a/modules/extdata/p2p/pom.xml
+++ b/modules/extdata/p2p/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-extdata-p2p</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/extdata/platform/pom.xml b/modules/extdata/platform/pom.xml
index 6fd61899943bf..523f8aa6ccd43 100644
--- a/modules/extdata/platform/pom.xml
+++ b/modules/extdata/platform/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-platform</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/extdata/uri/modules/uri-dependency/pom.xml b/modules/extdata/uri/modules/uri-dependency/pom.xml
index 100b0c2701dcb..06d208ab07aef 100644
--- a/modules/extdata/uri/modules/uri-dependency/pom.xml
+++ b/modules/extdata/uri/modules/uri-dependency/pom.xml
@@ -27,7 +27,7 @@
     <artifactId>ignite-extdata-uri-dep</artifactId>
     <packaging>jar</packaging>
 
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <modelVersion>4.0.0</modelVersion>
 
     <dependencies>
@@ -37,4 +37,4 @@
             <version>${project.version}</version>
         </dependency>
     </dependencies>
-</project>
\ No newline at end of file
+</project>
diff --git a/modules/extdata/uri/pom.xml b/modules/extdata/uri/pom.xml
index 283e07e521e0c..f3a6a3d64fd0a 100644
--- a/modules/extdata/uri/pom.xml
+++ b/modules/extdata/uri/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-uri</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/flink/pom.xml b/modules/flink/pom.xml
index 1013e4c8f4487..deaa81bf757e2 100644
--- a/modules/flink/pom.xml
+++ b/modules/flink/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flink</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/flume/pom.xml b/modules/flume/pom.xml
index dcee0dbb316df..b67d341beddab 100644
--- a/modules/flume/pom.xml
+++ b/modules/flume/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flume</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/gce/pom.xml b/modules/gce/pom.xml
index ba6d484c4957d..b740bb7a6d0ef 100644
--- a/modules/gce/pom.xml
+++ b/modules/gce/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-gce</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/geospatial/pom.xml b/modules/geospatial/pom.xml
index 395c8db9124c4..feaea2061855b 100644
--- a/modules/geospatial/pom.xml
+++ b/modules/geospatial/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-geospatial</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index 5ed32789e0c85..52599df3bb03c 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hadoop</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-4.2/pom.xml b/modules/hibernate-4.2/pom.xml
index 7765690a59c3f..75f99322e9adc 100644
--- a/modules/hibernate-4.2/pom.xml
+++ b/modules/hibernate-4.2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate_4.2</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-5.1/pom.xml b/modules/hibernate-5.1/pom.xml
index 20390dde95181..fa675f339c5fd 100644
--- a/modules/hibernate-5.1/pom.xml
+++ b/modules/hibernate-5.1/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate_5.1</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-core/pom.xml b/modules/hibernate-core/pom.xml
index f939f3f47328c..c31b546306bdd 100644
--- a/modules/hibernate-core/pom.xml
+++ b/modules/hibernate-core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate-core</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ignored-tests/pom.xml b/modules/ignored-tests/pom.xml
index affa6fb8a86c1..bacbecc968338 100644
--- a/modules/ignored-tests/pom.xml
+++ b/modules/ignored-tests/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ignored-tests</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml
index 29d7a3b911ee5..46bfadaa7625f 100644
--- a/modules/indexing/pom.xml
+++ b/modules/indexing/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-indexing</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jcl/pom.xml b/modules/jcl/pom.xml
index d973db4bf5601..2208eda8afe63 100644
--- a/modules/jcl/pom.xml
+++ b/modules/jcl/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jcl</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jms11/pom.xml b/modules/jms11/pom.xml
index 321cde4d9ab0f..29f29d44d6360 100644
--- a/modules/jms11/pom.xml
+++ b/modules/jms11/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jms11</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jta/pom.xml b/modules/jta/pom.xml
index c7cfa25e8c290..c0f8deea21d16 100644
--- a/modules/jta/pom.xml
+++ b/modules/jta/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jta</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kafka/pom.xml b/modules/kafka/pom.xml
index 04cb839195679..ef9dc5ab24538 100644
--- a/modules/kafka/pom.xml
+++ b/modules/kafka/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kafka</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kubernetes/pom.xml b/modules/kubernetes/pom.xml
index eef79a03b714e..39376e0f4edf3 100644
--- a/modules/kubernetes/pom.xml
+++ b/modules/kubernetes/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kubernetes</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j/pom.xml b/modules/log4j/pom.xml
index 88e965c41bad3..a84b8175f5ae4 100644
--- a/modules/log4j/pom.xml
+++ b/modules/log4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j2/pom.xml b/modules/log4j2/pom.xml
index fc8be7fb629a6..c36d48ec6ac4e 100644
--- a/modules/log4j2/pom.xml
+++ b/modules/log4j2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j2</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml
index 620e1f1bf4119..8926a82a0ca10 100644
--- a/modules/mesos/pom.xml
+++ b/modules/mesos/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mesos</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
@@ -128,4 +128,4 @@
         </plugins>
     </build>
 
-</project>
\ No newline at end of file
+</project>
diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml
index f578cdfdf0ac5..bdb87628e4c24 100644
--- a/modules/ml/pom.xml
+++ b/modules/ml/pom.xml
@@ -30,7 +30,7 @@
     </parent>
 
     <artifactId>ignite-ml</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mqtt/pom.xml b/modules/mqtt/pom.xml
index b9ec5d7a9539f..fe7b9f3419425 100644
--- a/modules/mqtt/pom.xml
+++ b/modules/mqtt/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mqtt</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/osgi-karaf/pom.xml b/modules/osgi-karaf/pom.xml
index 42d89be5196b3..5c531760d88f6 100644
--- a/modules/osgi-karaf/pom.xml
+++ b/modules/osgi-karaf/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-karaf</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <build>
diff --git a/modules/osgi-paxlogging/pom.xml b/modules/osgi-paxlogging/pom.xml
index b732e94f651b6..0ae571685816d 100644
--- a/modules/osgi-paxlogging/pom.xml
+++ b/modules/osgi-paxlogging/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-paxlogging</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <dependencies>
diff --git a/modules/osgi/pom.xml b/modules/osgi/pom.xml
index 4d92be2cd6163..b539c94f3a1db 100644
--- a/modules/osgi/pom.xml
+++ b/modules/osgi/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/rest-http/pom.xml b/modules/rest-http/pom.xml
index d653bfd91e930..ffa3385c48716 100644
--- a/modules/rest-http/pom.xml
+++ b/modules/rest-http/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-rest-http</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/rocketmq/pom.xml b/modules/rocketmq/pom.xml
index c28793e30672b..7949aa3b21edb 100644
--- a/modules/rocketmq/pom.xml
+++ b/modules/rocketmq/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-rocketmq</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar-2.10/pom.xml b/modules/scalar-2.10/pom.xml
index b233fdb89a015..54d43ac75a02b 100644
--- a/modules/scalar-2.10/pom.xml
+++ b/modules/scalar-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar_2.10</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml
index a38719b74819f..1f63009e0520f 100644
--- a/modules/scalar/pom.xml
+++ b/modules/scalar/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/schedule/pom.xml b/modules/schedule/pom.xml
index f21d4ff0a0806..82abbf2a8d866 100644
--- a/modules/schedule/pom.xml
+++ b/modules/schedule/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-schedule</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/slf4j/pom.xml b/modules/slf4j/pom.xml
index f5060220f5511..3a6754469f13e 100644
--- a/modules/slf4j/pom.xml
+++ b/modules/slf4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-slf4j</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark-2.10/pom.xml b/modules/spark-2.10/pom.xml
index 1f40f0f3cda20..e74488245dbe7 100644
--- a/modules/spark-2.10/pom.xml
+++ b/modules/spark-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark_2.10</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index e338c8823da6c..99198cd3923b1 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring-data/pom.xml b/modules/spring-data/pom.xml
index cf0c0356d1b98..e849f3a6ffd8a 100644
--- a/modules/spring-data/pom.xml
+++ b/modules/spring-data/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring-data</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring/pom.xml b/modules/spring/pom.xml
index 09375a6b31f6d..bc849af8cce05 100644
--- a/modules/spring/pom.xml
+++ b/modules/spring/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/sqlline/pom.xml b/modules/sqlline/pom.xml
index 9c26a15fe516f..34aaaffc41b3b 100644
--- a/modules/sqlline/pom.xml
+++ b/modules/sqlline/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-sqlline</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ssh/pom.xml b/modules/ssh/pom.xml
index 7b3aee62a1fe6..4504279c876fc 100644
--- a/modules/ssh/pom.xml
+++ b/modules/ssh/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ssh</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/storm/pom.xml b/modules/storm/pom.xml
index 7484c1db36e63..3077b168ac639 100644
--- a/modules/storm/pom.xml
+++ b/modules/storm/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-storm</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/tools/pom.xml b/modules/tools/pom.xml
index d685c206eab22..32656daf86e12 100644
--- a/modules/tools/pom.xml
+++ b/modules/tools/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-tools</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/twitter/pom.xml b/modules/twitter/pom.xml
index 60708dc070405..ef491ab283578 100644
--- a/modules/twitter/pom.xml
+++ b/modules/twitter/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-twitter</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/urideploy/pom.xml b/modules/urideploy/pom.xml
index d24ead3b977ba..8d194e1485f3b 100644
--- a/modules/urideploy/pom.xml
+++ b/modules/urideploy/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-urideploy</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console-2.10/pom.xml b/modules/visor-console-2.10/pom.xml
index 30c480fa7c1b7..f858f588a0b98 100644
--- a/modules/visor-console-2.10/pom.xml
+++ b/modules/visor-console-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console_2.10</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console/pom.xml b/modules/visor-console/pom.xml
index 54c2eb3c9e9c1..39a44113cfe35 100644
--- a/modules/visor-console/pom.xml
+++ b/modules/visor-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-plugins/pom.xml b/modules/visor-plugins/pom.xml
index 21ef092a02c6c..3484fffe83058 100644
--- a/modules/visor-plugins/pom.xml
+++ b/modules/visor-plugins/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-plugins</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web-console/pom.xml b/modules/web-console/pom.xml
index bfd9ffeb0c3cf..57cda7ba9361a 100644
--- a/modules/web-console/pom.xml
+++ b/modules/web-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web-console</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml
index 17deeef51b2e2..b7518c0955742 100644
--- a/modules/web-console/web-agent/pom.xml
+++ b/modules/web-console/web-agent/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-web-agent</artifactId>
     <packaging>jar</packaging>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web/ignite-appserver-test/pom.xml b/modules/web/ignite-appserver-test/pom.xml
index 945a2744205d3..397b4f6135f9e 100644
--- a/modules/web/ignite-appserver-test/pom.xml
+++ b/modules/web/ignite-appserver-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-appserver-test</artifactId>
     <packaging>jar</packaging>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/ignite-websphere-test/pom.xml b/modules/web/ignite-websphere-test/pom.xml
index 77ea6514e4769..8fc17b83f164f 100644
--- a/modules/web/ignite-websphere-test/pom.xml
+++ b/modules/web/ignite-websphere-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-websphere-test</artifactId>
     <packaging>war</packaging>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/pom.xml b/modules/web/pom.xml
index d5e82c6e19378..e2e7e210a5ec7 100644
--- a/modules/web/pom.xml
+++ b/modules/web/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index 9923bb7c7ddf9..f55c5c495258a 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yardstick</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml
index ce7217b0d37e9..a38757919eb78 100644
--- a/modules/yarn/pom.xml
+++ b/modules/yarn/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yarn</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zeromq/pom.xml b/modules/zeromq/pom.xml
index 5ecc6aaf76b5b..bf8caae879433 100644
--- a/modules/zeromq/pom.xml
+++ b/modules/zeromq/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zeromq</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index 0777f1e36edbd..9e36018c4e0eb 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zookeeper</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/pom.xml b/pom.xml
index 086d241f829af..a28464d1e87d4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>apache-ignite</artifactId>
-    <version>2.5.0-SNAPSHOT</version>
+    <version>2.5.1-p150-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <properties>

From ba585c59e0988ec898d518d52bcdf514ae493f27 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Fri, 19 Oct 2018 17:51:41 +0300
Subject: [PATCH 441/543] IGNITE-5795 Register binary metadata during cache
 start - Fixes #4852.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 3bb0344)

(cherry picked from commit 8f85561)
---
 .../apache/ignite/internal/IgniteKernal.java  |   2 +-
 .../binary/BinaryCachingMetadataHandler.java  |  25 +-
 .../ignite/internal/binary/BinaryContext.java |  59 +++-
 .../binary/BinaryMetadataHandler.java         |  10 +
 .../binary/BinaryNoopMetadataHandler.java     |   6 +
 .../builder/BinaryObjectBuilderImpl.java      |   2 +-
 .../internal/client/thin/TcpIgniteClient.java |   6 +
 .../processors/cache/GridCacheProcessor.java  |   9 +-
 .../CacheObjectBinaryProcessorImpl.java       |   5 +
 .../processors/query/GridQueryProcessor.java  |  77 ++++-
 .../binary/TestCachingMetadataHandler.java    |   6 +
 .../CacheRegisterMetadataLocallyTest.java     | 287 ++++++++++++++++++
 .../cache/index/AbstractSchemaSelfTest.java   |   6 +-
 .../index/H2DynamicIndexAbstractSelfTest.java |  48 +--
 .../IgniteCacheWithIndexingTestSuite.java     |   2 +
 15 files changed, 494 insertions(+), 56 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 7ea3271633dc3..f39b14022feee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -998,6 +998,7 @@ public void start(
             // Start processors before discovery manager, so they will
             // be able to start receiving messages once discovery completes.
             try {
+                startProcessor(new GridMarshallerMappingProcessor(ctx));
                 startProcessor(new PdsConsistentIdProcessor(ctx));
                 startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx));
                 startProcessor(new GridAffinityProcessor(ctx));
@@ -1020,7 +1021,6 @@ public void start(
                 startProcessor(createHadoopComponent());
                 startProcessor(new DataStructuresProcessor(ctx));
                 startProcessor(createComponent(PlatformProcessor.class, ctx));
-                startProcessor(new GridMarshallerMappingProcessor(ctx));
 
                 // Start plugins.
                 for (PluginProvider provider : ctx.plugins().allProviders()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java
index a0559cbdc8ba1..b60dc097aaa1e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java
@@ -46,23 +46,28 @@ private BinaryCachingMetadataHandler() {
     }
 
     /** {@inheritDoc} */
-    @Override public synchronized void addMeta(int typeId, BinaryType type, boolean failIfUnregistered) throws BinaryObjectException {
-        synchronized (this) {
-            BinaryType oldType = metas.put(typeId, type);
+    @Override public synchronized void addMeta(int typeId, BinaryType type,
+        boolean failIfUnregistered) throws BinaryObjectException {
+        BinaryType oldType = metas.put(typeId, type);
 
-            if (oldType != null) {
-                BinaryMetadata oldMeta = ((BinaryTypeImpl)oldType).metadata();
-                BinaryMetadata newMeta = ((BinaryTypeImpl)type).metadata();
+        if (oldType != null) {
+            BinaryMetadata oldMeta = ((BinaryTypeImpl)oldType).metadata();
+            BinaryMetadata newMeta = ((BinaryTypeImpl)type).metadata();
 
-                BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta);
+            BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta);
 
-                BinaryType mergedType = mergedMeta.wrap(((BinaryTypeImpl)oldType).context());
+            BinaryType mergedType = mergedMeta.wrap(((BinaryTypeImpl)oldType).context());
 
-                metas.put(typeId, mergedType);
-            }
+            metas.put(typeId, mergedType);
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public synchronized void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered)
+        throws BinaryObjectException {
+        addMeta(typeId, meta, failIfUnregistered);
+    }
+
     /** {@inheritDoc} */
     @Override public synchronized BinaryType metadata(int typeId) throws BinaryObjectException {
         return metas.get(typeId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 7885d9575f32b..7ab74e09b82d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -617,6 +617,18 @@ else if (cpElement.isFile()) {
      */
     public BinaryClassDescriptor descriptorForClass(Class<?> cls, boolean deserialize, boolean failIfUnregistered)
         throws BinaryObjectException {
+        return descriptorForClass(cls, deserialize, failIfUnregistered, false);
+    }
+
+    /**
+     * @param cls Class.
+     * @param failIfUnregistered Throw exception if class isn't registered.
+     * @param onlyLocReg {@code true} if descriptor need to register only locally when registration is required at all.
+     * @return Class descriptor.
+     * @throws BinaryObjectException In case of error.
+     */
+    public BinaryClassDescriptor descriptorForClass(Class<?> cls, boolean deserialize, boolean failIfUnregistered,
+        boolean onlyLocReg) throws BinaryObjectException {
         assert cls != null;
 
         BinaryClassDescriptor desc = descByCls.get(cls);
@@ -625,7 +637,7 @@ public BinaryClassDescriptor descriptorForClass(Class<?> cls, boolean deserializ
             if (failIfUnregistered)
                 throw new UnregisteredClassException(cls);
 
-            desc = registerClassDescriptor(cls, deserialize);
+            desc = registerClassDescriptor(cls, deserialize, onlyLocReg);
         }
         else if (!desc.registered()) {
             if (!desc.userType()) {
@@ -662,7 +674,7 @@ else if (!desc.registered()) {
                 if (failIfUnregistered)
                     throw new UnregisteredClassException(cls);
 
-                desc = registerUserClassDescriptor(desc);
+                desc = registerUserClassDescriptor(desc, onlyLocReg);
             }
         }
 
@@ -715,7 +727,7 @@ public BinaryClassDescriptor descriptorForTypeId(
         }
 
         if (desc == null) {
-            desc = registerClassDescriptor(cls, deserialize);
+            desc = registerClassDescriptor(cls, deserialize, false);
 
             assert desc.typeId() == typeId : "Duplicate typeId [typeId=" + typeId + ", cls=" + cls
                 + ", desc=" + desc + "]";
@@ -728,9 +740,10 @@ public BinaryClassDescriptor descriptorForTypeId(
      * Creates and registers {@link BinaryClassDescriptor} for the given {@code class}.
      *
      * @param cls Class.
+     * @param onlyLocReg {@code true} if descriptor need to register only locally when registration is required at all.
      * @return Class descriptor.
      */
-    private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean deserialize) {
+    private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean deserialize, boolean onlyLocReg) {
         BinaryClassDescriptor desc;
 
         String clsName = cls.getName();
@@ -759,7 +772,7 @@ private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean dese
                 desc = old;
         }
         else
-            desc = registerUserClassDescriptor(cls, deserialize);
+            desc = registerUserClassDescriptor(cls, deserialize, onlyLocReg);
 
         return desc;
     }
@@ -768,9 +781,10 @@ private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean dese
      * Creates and registers {@link BinaryClassDescriptor} for the given user {@code class}.
      *
      * @param cls Class.
+     * @param onlyLocReg {@code true} if descriptor need to register only locally.
      * @return Class descriptor.
      */
-    private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean deserialize) {
+    private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean deserialize, boolean onlyLocReg) {
         boolean registered;
 
         final String clsName = cls.getName();
@@ -781,7 +795,7 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean
 
         final int typeId = mapper.typeId(clsName);
 
-        registered = registerUserClassName(typeId, cls.getName());
+        registered = registerUserClassName(typeId, cls.getName(), onlyLocReg);
 
         BinarySerializer serializer = serializerForClass(cls);
 
@@ -799,9 +813,22 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean
             registered
         );
 
-        if (!deserialize)
-            metaHnd.addMeta(typeId, new BinaryMetadata(typeId, typeName, desc.fieldsMeta(), affFieldName, null,
-                desc.isEnum(), cls.isEnum() ? enumMap(cls) : null).wrap(this), false);
+        if (!deserialize) {
+            BinaryMetadata binaryMetadata = new BinaryMetadata(
+                typeId,
+                typeName,
+                desc.fieldsMeta(),
+                affFieldName,
+                null,
+                desc.isEnum(),
+                cls.isEnum() ? enumMap(cls) : null
+            );
+
+            if (onlyLocReg)
+                metaHnd.addMetaLocally(typeId, binaryMetadata.wrap(this), false);
+            else
+                metaHnd.addMeta(typeId, binaryMetadata.wrap(this), false);
+        }
 
         descByCls.put(cls, desc);
 
@@ -814,12 +841,13 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean
      * Creates and registers {@link BinaryClassDescriptor} for the given user {@code class}.
      *
      * @param desc Old descriptor that should be re-registered.
+     * @param onlyLocReg {@code true} if descriptor need to register only locally.
      * @return Class descriptor.
      */
-    private BinaryClassDescriptor registerUserClassDescriptor(BinaryClassDescriptor desc) {
+    private BinaryClassDescriptor registerUserClassDescriptor(BinaryClassDescriptor desc, boolean onlyLocReg) {
         boolean registered;
 
-        registered = registerUserClassName(desc.typeId(), desc.describedClass().getName());
+        registered = registerUserClassName(desc.typeId(), desc.describedClass().getName(), onlyLocReg);
 
         if (registered) {
             BinarySerializer serializer = desc.initialSerializer();
@@ -1191,15 +1219,18 @@ public void registerUserTypesSchema() {
      *
      * @param typeId Type ID.
      * @param clsName Class Name.
+     * @param onlyLocReg {@code true} if descriptor need to register only locally.
      * @return {@code True} if the mapping was registered successfully.
      */
-    public boolean registerUserClassName(int typeId, String clsName) {
+    public boolean registerUserClassName(int typeId, String clsName, boolean onlyLocReg) {
         IgniteCheckedException e = null;
 
         boolean res = false;
 
         try {
-            res = marshCtx.registerClassName(JAVA_ID, typeId, clsName);
+            res = onlyLocReg
+                ? marshCtx.registerClassNameLocally(JAVA_ID, typeId, clsName)
+                : marshCtx.registerClassName(JAVA_ID, typeId, clsName);
         }
         catch (DuplicateTypeIdException dupEx) {
             // Ignore if trying to register mapped type name of the already registered class name and vise versa
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java
index 85ab1372f49f2..d1336bf6ae4a0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java
@@ -35,6 +35,16 @@ public interface BinaryMetadataHandler {
      */
     public void addMeta(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException;
 
+    /**
+     * Adds meta data locally on current node without sending any messages.
+     *
+     * @param typeId Type ID.
+     * @param meta Metadata.
+     * @param failIfUnregistered Fail if unregistered.
+     * @throws BinaryObjectException In case of error.
+     */
+    public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException;
+
     /**
      * Gets meta data for provided type ID.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java
index 4ee24285c7ee5..a552d611ea990 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java
@@ -47,6 +47,12 @@ private BinaryNoopMetadataHandler() {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered)
+        throws BinaryObjectException {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
         return null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
index 8fffd71c1b230..f860564844056 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
@@ -364,7 +364,7 @@ else if (readCache == null) {
                 if (affFieldName0 == null)
                     affFieldName0 = ctx.affinityKeyFieldName(typeId);
 
-                ctx.registerUserClassName(typeId, typeName);
+                ctx.registerUserClassName(typeId, typeName, false);
 
                 ctx.updateMetadata(typeId, new BinaryMetadata(typeId, typeName, fieldsMeta, affFieldName0,
                     Collections.singleton(curSchema), false, null), writer.failIfUnregistered());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java
index 7a0bc7a1f8160..a108347be4a35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java
@@ -249,6 +249,12 @@ private class ClientBinaryMetadataHandler implements BinaryMetadataHandler {
             cache.addMeta(typeId, meta, failIfUnregistered); // merge
         }
 
+        /** {@inheritDoc} */
+        @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered)
+            throws BinaryObjectException {
+            throw new UnsupportedOperationException("Can't register metadata locally for thin client.");
+        }
+
         /** {@inheritDoc} */
         @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
             BinaryType meta = cache.metadata(typeId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index b37b000d775dc..98cf2115fcf1a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3591,8 +3591,13 @@ else if (msg0 instanceof WalStateFinishMessage)
             return msg0.needExchange();
         }
 
-        if (msg instanceof DynamicCacheChangeBatch)
-            return cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer);
+        if (msg instanceof DynamicCacheChangeBatch) {
+            boolean changeRequested = cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer);
+
+            ctx.query().onCacheChangeRequested((DynamicCacheChangeBatch)msg);
+
+            return changeRequested;
+        }
 
         if (msg instanceof DynamicCacheChangeFailureMessage)
             cachesInfo.onCacheChangeRequested((DynamicCacheChangeFailureMessage)msg, topVer);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index 137db9f887dc5..f3078cb18df80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -207,6 +207,11 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
                     CacheObjectBinaryProcessorImpl.this.addMeta(typeId, newMeta0.wrap(binaryCtx), failIfUnregistered);
                 }
 
+                @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered)
+                    throws BinaryObjectException {
+                    CacheObjectBinaryProcessorImpl.this.addMetaLocally(typeId, meta);
+                }
+
                 @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
                     return CacheObjectBinaryProcessorImpl.this.metadata(typeId);
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index 7300393f599ee..f2b277585fd12 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.query;
 
+import javax.cache.Cache;
+import javax.cache.CacheException;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -34,8 +36,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
-import javax.cache.Cache;
-import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.IgniteException;
@@ -61,15 +61,19 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheObjectContext;
+import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch;
+import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.query.CacheQueryFuture;
 import org.apache.ignite.internal.processors.cache.query.CacheQueryType;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.query.property.QueryBinaryProperty;
 import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheFilter;
 import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor;
@@ -251,6 +255,8 @@ public GridQueryProcessor(GridKernalContext ctx) throws IgniteCheckedException {
                     ctxs.queries().evictDetailMetrics();
             }
         }, QRY_DETAIL_METRICS_EVICTION_FREQ, QRY_DETAIL_METRICS_EVICTION_FREQ);
+
+        registerMetadataForRegisteredCaches();
     }
 
     /** {@inheritDoc} */
@@ -897,6 +903,73 @@ public void skipFieldLookup(boolean skipFieldLookup) {
         this.skipFieldLookup = skipFieldLookup;
     }
 
+    /**
+     * Register metadata locally for already registered caches.
+     */
+    private void registerMetadataForRegisteredCaches() {
+        for (DynamicCacheDescriptor cacheDescriptor : ctx.cache().cacheDescriptors().values()) {
+            registerBinaryMetadata(cacheDescriptor.cacheConfiguration(), cacheDescriptor.schema());
+        }
+    }
+
+    /**
+     * Handle of cache change request.
+     *
+     * @param batch Dynamic cache change batch request.
+     */
+    public void onCacheChangeRequested(DynamicCacheChangeBatch batch) {
+        for (DynamicCacheChangeRequest req : batch.requests()) {
+            if (!req.start())
+                continue;
+
+            registerBinaryMetadata(req.startCacheConfiguration(), req.schema());
+        }
+    }
+
+    /**
+     * Register binary metadata locally.
+     *
+     * @param ccfg Cache configuration.
+     * @param schema Schema for which register metadata is required.
+     */
+    private void registerBinaryMetadata(CacheConfiguration ccfg, QuerySchema schema) {
+        if (schema != null) {
+            Collection<QueryEntity> qryEntities = schema.entities();
+
+            if (!F.isEmpty(qryEntities)) {
+                boolean binaryEnabled = ctx.cacheObjects().isBinaryEnabled(ccfg);
+
+                if (binaryEnabled) {
+                    for (QueryEntity qryEntity : qryEntities) {
+                        Class<?> keyCls = U.box(U.classForName(qryEntity.findKeyType(), null, true));
+                        Class<?> valCls = U.box(U.classForName(qryEntity.findValueType(), null, true));
+
+                        if (keyCls != null)
+                            registerDescriptorLocallyIfNeeded(keyCls);
+
+                        if (valCls != null)
+                            registerDescriptorLocallyIfNeeded(valCls);
+                    }
+                }
+            }
+        }
+    }
+
+    /**
+     * Register class metadata locally if it didn't do it earlier.
+     *
+     * @param cls Class for which the metadata should be registered.
+     */
+    private void registerDescriptorLocallyIfNeeded(Class<?> cls) {
+        IgniteCacheObjectProcessor cacheObjProc = ctx.cacheObjects();
+
+        if (cacheObjProc instanceof CacheObjectBinaryProcessorImpl) {
+            ((CacheObjectBinaryProcessorImpl)cacheObjProc)
+                .binaryContext()
+                .descriptorForClass(cls, false, false, true);
+        }
+    }
+
     /**
      * Handle custom discovery message.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java
index c515f8191766a..47138ddeb0806 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java
@@ -38,6 +38,12 @@ public class TestCachingMetadataHandler implements BinaryMetadataHandler {
                 TestCachingMetadataHandler.class.getSimpleName() + '.');
     }
 
+    /** {@inheritDoc} */
+    @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered)
+        throws BinaryObjectException {
+        addMeta(typeId, meta, failIfUnregistered);
+    }
+
     /** {@inheritDoc} */
     @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
         return metas.get(typeId);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java
new file mode 100644
index 0000000000000..d4066c2f546f6
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collections;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.binary.BinaryType;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.affinity.AffinityKeyMapped;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
+import org.apache.ignite.internal.processors.cache.binary.MetadataRequestMessage;
+import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage;
+import org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Tests, that binary metadata is registered correctly during the start without extra request to grid.
+ */
+public class CacheRegisterMetadataLocallyTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final String STATIC_CACHE_NAME = "staticCache";
+
+    /** */
+    private static final String DYNAMIC_CACHE_NAME = "dynamicCache";
+
+    /** Holder of sent custom messages. */
+    private final ConcurrentLinkedQueue<Object> customMessages = new ConcurrentLinkedQueue<>();
+
+    /** Holder of sent communication messages. */
+    private final ConcurrentLinkedQueue<Object> communicationMessages = new ConcurrentLinkedQueue<>();
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi() {
+            @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException {
+                if (msg instanceof CustomMessageWrapper)
+                    customMessages.add(((CustomMessageWrapper)msg).delegate());
+                else
+                    customMessages.add(msg);
+
+                super.sendCustomEvent(msg);
+            }
+        });
+
+        cfg.setCommunicationSpi(new TcpCommunicationSpi() {
+            @Override public void sendMessage(ClusterNode node, Message msg,
+                IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+                if (msg instanceof GridIoMessage)
+                    communicationMessages.add(((GridIoMessage)msg).message());
+
+                super.sendMessage(node, msg, ackC);
+            }
+
+            @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+                if (msg instanceof GridIoMessage)
+                    communicationMessages.add(((GridIoMessage)msg).message());
+
+                super.sendMessage(node, msg);
+            }
+        });
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        if (igniteInstanceName.equals("client"))
+            cfg.setClientMode(true);
+
+        cfg.setCacheConfiguration(cacheConfiguration(STATIC_CACHE_NAME, StaticKey.class, StaticValue.class));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        customMessages.clear();
+        communicationMessages.clear();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAffinityKeyRegisteredStaticCache() throws Exception {
+        Ignite ignite = startGrid();
+
+        assertEquals("affKey", getAffinityKey(ignite, StaticKey.class));
+        assertEquals("affKey", getAffinityKey(ignite, StaticValue.class));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testAffinityKeyRegisteredDynamicCache() throws Exception {
+        Ignite ignite = startGrid();
+
+        ignite.createCache(cacheConfiguration(DYNAMIC_CACHE_NAME, DynamicKey.class, DynamicValue.class));
+
+        assertEquals("affKey", getAffinityKey(ignite, DynamicKey.class));
+        assertEquals("affKey", getAffinityKey(ignite, DynamicValue.class));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientFindsValueByAffinityKeyStaticCacheWithoutExtraRequest() throws Exception {
+        Ignite srv = startGrid();
+        IgniteCache<StaticKey, StaticValue> cache = srv.cache(STATIC_CACHE_NAME);
+
+        testClientFindsValueByAffinityKey(cache, new StaticKey(1), new StaticValue(2));
+
+        assertCustomMessages(2); //MetadataUpdateProposedMessage for update schema.
+        assertCommunicationMessages();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testClientFindsValueByAffinityKeyDynamicCacheWithoutExtraRequest() throws Exception {
+        Ignite srv = startGrid();
+        IgniteCache<DynamicKey, DynamicValue> cache =
+            srv.createCache(cacheConfiguration(DYNAMIC_CACHE_NAME, DynamicKey.class, DynamicValue.class));
+
+        testClientFindsValueByAffinityKey(cache, new DynamicKey(3), new DynamicValue(4));
+
+        //Expected only DynamicCacheChangeBatch for start cache and MetadataUpdateProposedMessage for update schema.
+        assertCustomMessages(3);
+        assertCommunicationMessages();
+    }
+
+    /**
+     * @param ignite Ignite instance.
+     * @param keyCls Key class.
+     * @return Name of affinity key field of the given class.
+     */
+    private <K> String getAffinityKey(Ignite ignite, Class<K> keyCls) {
+        BinaryType binType = ignite.binary().type(keyCls);
+
+        return binType.affinityKeyFieldName();
+    }
+
+    /**
+     * @param cache Cache instance.
+     * @param key Test key.
+     * @param val Test value.
+     * @throws Exception If failed.
+     */
+    private <K, V> void testClientFindsValueByAffinityKey(IgniteCache<K, V> cache, K key, V val) throws Exception {
+        cache.put(key, val);
+
+        assertTrue(cache.containsKey(key));
+
+        Ignite client = startGrid("client");
+
+        IgniteCache<K, V> clientCache = client.cache(cache.getName());
+
+        assertTrue(clientCache.containsKey(key));
+    }
+
+    /**
+     * @param name Cache name.
+     * @param keyCls Key {@link Class}.
+     * @param valCls Value {@link Class}.
+     * @param <K> Key type.
+     * @param <V> Value type.
+     * @return Cache configuration
+     */
+    private static <K, V> CacheConfiguration<K, V> cacheConfiguration(String name, Class<K> keyCls, Class<V> valCls) {
+        CacheConfiguration<K, V> cfg = new CacheConfiguration<>(name);
+        cfg.setQueryEntities(Collections.singleton(new QueryEntity(keyCls, valCls)));
+        return cfg;
+    }
+
+    /**
+     * Expecting that "proposed binary metadata"( {@link org.apache.ignite.internal.processors.marshaller.MappingProposedMessage},
+     * {@link org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage}) will be skipped because
+     * it should be register locally during the start.
+     *
+     * @param expMsgCnt Count of expected messages.
+     */
+    private void assertCustomMessages(int expMsgCnt) {
+        assertEquals(customMessages.toString(), expMsgCnt, customMessages.size());
+
+        customMessages.forEach(cm -> assertTrue(cm.toString(), cm instanceof DynamicCacheChangeBatch || cm instanceof MetadataUpdateProposedMessage));
+    }
+
+    /**
+     * Expecting that extra request to binary metadata( {@link MetadataRequestMessage}, {@link MetadataResponseMessage})
+     * will be skipped because it should be register locally during the start.
+     */
+    private void assertCommunicationMessages() {
+        communicationMessages.forEach(cm ->
+            assertFalse(cm.toString(), cm instanceof MetadataRequestMessage || cm instanceof MetadataResponseMessage)
+        );
+    }
+
+    /** */
+    private static class StaticKey {
+        /** */
+        @AffinityKeyMapped
+        private int affKey;
+
+        /**
+         * @param affKey Affinity key.
+         */
+        StaticKey(int affKey) {
+            this.affKey = affKey;
+        }
+    }
+
+    /** */
+    private static class StaticValue {
+        /** */
+        @AffinityKeyMapped
+        private int affKey;
+
+        /**
+         * @param affKey Affinity key.
+         */
+        StaticValue(int affKey) {
+        }
+    }
+
+    /** */
+    private static class DynamicKey {
+        /** */
+        @AffinityKeyMapped
+        private int affKey;
+
+        /**
+         * @param affKey Affinity key.
+         */
+        DynamicKey(int affKey) {
+            this.affKey = affKey;
+        }
+    }
+
+    /** */
+    private static class DynamicValue {
+        /** */
+        @AffinityKeyMapped
+        private int affKey;
+
+        /**
+         * @param affKey Affinity key.
+         */
+        DynamicValue(int affKey) {
+            this.affKey = affKey;
+        }
+    }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java
index 0a0efc72d373d..d3a86b9f47d64 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java
@@ -531,21 +531,21 @@ public long id() {
     public static class ValueClass {
         /** Field 1. */
         @QuerySqlField
-        private String field1;
+        private Long field1;
 
         /**
          * Constructor.
          *
          * @param field1 Field 1.
          */
-        public ValueClass(String field1) {
+        public ValueClass(Long field1) {
             this.field1 = field1;
         }
 
         /**
          * @return Field 1
          */
-        public String field1() {
+        public Long field1() {
             return field1;
         }
     }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
index 0684f7f6e8e69..a6982cdee3eb4 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
@@ -70,9 +70,9 @@ public abstract class H2DynamicIndexAbstractSelfTest extends AbstractSchemaSelfT
 
         IgniteCache<KeyClass, ValueClass> cache = client().cache(CACHE_NAME);
 
-        cache.put(new KeyClass(1), new ValueClass("val1"));
-        cache.put(new KeyClass(2), new ValueClass("val2"));
-        cache.put(new KeyClass(3), new ValueClass("val3"));
+        cache.put(new KeyClass(1), new ValueClass(1L));
+        cache.put(new KeyClass(2), new ValueClass(2L));
+        cache.put(new KeyClass(3), new ValueClass(3L));
     }
 
     /** {@inheritDoc} */
@@ -99,14 +99,14 @@ public void testCreateIndex() throws Exception {
                 continue;
 
             List<List<?>> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " +
-                "\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll();
+                "\"cache\".\"ValueClass\" where \"field1\" = 1").setLocal(true)).getAll();
 
             assertEquals(F.asList(
                 Collections.singletonList("SELECT\n" +
                     "    \"id\"\n" +
                     "FROM \"cache\".\"ValueClass\"\n" +
-                    "    /* \"cache\".\"idx_1\": \"field1\" = 'A' */\n" +
-                    "WHERE \"field1\" = 'A'")
+                    "    /* \"cache\".\"idx_1\": \"field1\" = 1 */\n" +
+                    "WHERE \"field1\" = 1")
             ), locRes);
         }
 
@@ -116,7 +116,7 @@ public void testCreateIndex() throws Exception {
 
         assertSize(2);
 
-        cache.put(new KeyClass(4), new ValueClass("someVal"));
+        cache.put(new KeyClass(4), new ValueClass(1L));
 
         assertSize(3);
     }
@@ -172,14 +172,14 @@ public void testDropIndex() {
                 continue;
 
             List<List<?>> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " +
-                "\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll();
+                "\"cache\".\"ValueClass\" where \"field1\" = 1").setLocal(true)).getAll();
 
             assertEquals(F.asList(
                 Collections.singletonList("SELECT\n" +
                     "    \"id\"\n" +
                     "FROM \"cache\".\"ValueClass\"\n" +
                     "    /* \"cache\".\"ValueClass\".__SCAN_ */\n" +
-                    "WHERE \"field1\" = 'A'")
+                    "WHERE \"field1\" = 1")
             ), locRes);
         }
 
@@ -214,38 +214,39 @@ public void testDropMissingIndexIfExists() {
     public void testIndexState() {
         IgniteCache<KeyClass, ValueClass> cache = cache();
 
-        assertColumnValues("val1", "val2", "val3");
+        assertColumnValues(1L, 2L, 3L);
 
         cache.query(new SqlFieldsQuery("CREATE INDEX \"" + IDX_NAME_1_ESCAPED + "\" ON \"" + TBL_NAME_ESCAPED + "\"(\""
             + FIELD_NAME_1_ESCAPED + "\" ASC)"));
 
-        assertColumnValues("val1", "val2", "val3");
+        assertColumnValues(1L, 2L, 3L);
 
         cache.remove(new KeyClass(2));
 
-        assertColumnValues("val1", "val3");
+        assertColumnValues(1L, 3L);
 
-        cache.put(new KeyClass(0), new ValueClass("someVal"));
+        cache.put(new KeyClass(0), new ValueClass(0L));
 
-        assertColumnValues("someVal", "val1", "val3");
+        assertColumnValues(0L, 1L, 3L);
 
         cache.query(new SqlFieldsQuery("DROP INDEX \"" + IDX_NAME_1_ESCAPED + "\""));
 
-        assertColumnValues("someVal", "val1", "val3");
+        assertColumnValues(0L, 1L, 3L);
     }
 
     /**
      * Check that values of {@code field1} match what we expect.
      * @param vals Expected values.
      */
-    private void assertColumnValues(String... vals) {
+    private void assertColumnValues(Long... vals) {
         List<List<?>> expRes = new ArrayList<>(vals.length);
 
-        for (String v : vals)
+        for (Long v : vals)
             expRes.add(Collections.singletonList(v));
 
-        assertEquals(expRes, cache().query(new SqlFieldsQuery("SELECT \"" + FIELD_NAME_1_ESCAPED + "\" FROM \"" +
-            TBL_NAME_ESCAPED + "\" ORDER BY \"id\"")).getAll());
+        List<List<?>> all = cache().query(new SqlFieldsQuery("SELECT \"" + FIELD_NAME_1_ESCAPED + "\" FROM \"" +
+            TBL_NAME_ESCAPED + "\" ORDER BY \"id\"")).getAll();
+        assertEquals(expRes, all);
     }
 
     /**
@@ -255,8 +256,9 @@ private void assertColumnValues(String... vals) {
     private void assertSize(long expSize) {
         assertEquals(expSize, cache().size());
 
-        assertEquals(expSize, cache().query(new SqlFieldsQuery("SELECT COUNT(*) from \"ValueClass\""))
-            .getAll().get(0).get(0));
+        Object actual = cache().query(new SqlFieldsQuery("SELECT COUNT(*) from \"ValueClass\""))
+            .getAll().get(0).get(0);
+        assertEquals(expSize, actual);
     }
 
     /**
@@ -338,8 +340,8 @@ private CacheConfiguration cacheConfiguration() {
         entity.setValueType(ValueClass.class.getName());
 
         entity.addQueryField("id", Long.class.getName(), null);
-        entity.addQueryField(FIELD_NAME_1_ESCAPED, String.class.getName(), null);
-        entity.addQueryField(FIELD_NAME_2_ESCAPED, String.class.getName(), null);
+        entity.addQueryField(FIELD_NAME_1_ESCAPED, Long.class.getName(), null);
+        entity.addQueryField(FIELD_NAME_2_ESCAPED, Long.class.getName(), null);
 
         entity.setKeyFields(Collections.singleton("id"));
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index 7c1ee528bcd85..f088fb777d445 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.CacheRegisterMetadataLocallyTest;
 import org.apache.ignite.internal.processors.cache.CacheBinaryKeyConcurrentQueryTest;
 import org.apache.ignite.internal.processors.cache.CacheConfigurationP2PTest;
 import org.apache.ignite.internal.processors.cache.CacheIndexStreamerTest;
@@ -79,6 +80,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheOperationsWithExpirationTest.class);
         suite.addTestSuite(CacheBinaryKeyConcurrentQueryTest.class);
         suite.addTestSuite(CacheQueryFilterExpiredTest.class);
+        suite.addTestSuite(CacheRegisterMetadataLocallyTest.class);
 
         suite.addTestSuite(ClientReconnectAfterClusterRestartTest.class);
 

From 0193e3f5e09debf4da5f748032ec56dce628fa86 Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Mon, 22 Oct 2018 12:19:36 +0300
Subject: [PATCH 442/543] IGNITE-9822 Corrected WAL archiver thread stop logic
 - Fixes #4930.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit c7d6f110fc39749307e3f8ce203beb5f72f40994)
---
 .../wal/FileWriteAheadLogManager.java         | 27 +++++++++----------
 .../FsyncModeFileWriteAheadLogManager.java    | 24 +++++++----------
 2 files changed, 23 insertions(+), 28 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 2fa2873afd341..82bd9d38b0f88 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -97,8 +97,8 @@
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput;
-import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.LockedSegmentFileInputFactory;
+import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleSegmentFileInputFactory;
 import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
@@ -1564,19 +1564,18 @@ public long maxWalSegmentSize() {
      * the absolute index of last archived segment is denoted by A and the absolute index of next segment we want to
      * write is denoted by W, then we can allow write to S(W) if W - A <= walSegments. <br>
      *
-     * Monitor of current object is used for notify on: <ul> <li>exception occurred ({@link
-     * FileArchiver#cleanErr}!=null)</li> <li>stopping thread ({@link FileArchiver#stopped}==true)</li> <li>current file
-     * index changed </li> <li>last archived file index was changed ({@link
-     * </li> <li>some WAL index was removed from map</li>
+     * Monitor of current object is used for notify on: <ul>
+     *     <li>exception occurred ({@link FileArchiver#cleanErr}!=null)</li>
+     *     <li>stopping thread ({@link FileArchiver#isCancelled==true})</li>
+     *     <li>current file index changed </li>
+     *     <li>last archived file index was changed</li>
+     *     <li>some WAL index was removed from map</li>
      * </ul>
      */
     private class FileArchiver extends GridWorker {
         /** Exception which occurred during initial creation of files or during archiving WAL segment */
         private StorageException cleanErr;
 
-        /** current thread stopping advice */
-        private volatile boolean stopped;
-
         /** Formatted index. */
         private int formatted;
 
@@ -1595,7 +1594,7 @@ private FileArchiver(long lastAbsArchivedIdx, IgniteLogger log) {
          */
         private void shutdown() throws IgniteInterruptedCheckedException {
             synchronized (this) {
-                stopped = true;
+                isCancelled = true;
 
                 notifyAll();
             }
@@ -1629,12 +1628,12 @@ private void shutdown() throws IgniteInterruptedCheckedException {
 
                 segmentAware.awaitSegment(0);//wait for init at least one work segments.
 
-                while (!Thread.currentThread().isInterrupted() && !stopped) {
+                while (!Thread.currentThread().isInterrupted() && !isCancelled()) {
                     long toArchive;
 
                     toArchive = segmentAware.waitNextSegmentForArchivation();
 
-                    if (stopped)
+                    if (isCancelled())
                         break;
 
                     final SegmentArchiveResult res = archiveSegment(toArchive);
@@ -1654,14 +1653,14 @@ private void shutdown() throws IgniteInterruptedCheckedException {
                 Thread.currentThread().interrupt();
 
                 synchronized (this) {
-                    stopped = true;
+                    isCancelled = true;
                 }
             }
             catch (Throwable t) {
                 err = t;
             }
             finally {
-                if (err == null && !stopped)
+                if (err == null && !isCancelled())
                     err = new IllegalStateException("Worker " + name() + " is terminated unexpectedly");
 
                 if (err instanceof OutOfMemoryError)
@@ -1775,7 +1774,7 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws StorageException
          *
          */
         private boolean checkStop() {
-            return stopped;
+            return isCancelled();
         }
 
         /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 06ae98708023a..f65e6b89a2c75 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -109,7 +109,6 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
 import org.apache.ignite.lang.IgniteBiTuple;
@@ -1349,7 +1348,7 @@ private void checkNode() throws StorageException {
      * Monitor of current object is used for notify on:
      * <ul>
      * <li>exception occurred ({@link FileArchiver#cleanException}!=null)</li>
-     * <li>stopping thread ({@link FileArchiver#stopped}==true)</li>
+     * <li>stopping thread ({@link FileArchiver#isCancelled}==true)</li>
      * <li>current file index changed ({@link FileArchiver#curAbsWalIdx})</li>
      * <li>last archived file index was changed ({@link FileArchiver#lastAbsArchivedIdx})</li>
      * <li>some WAL index was removed from {@link FileArchiver#locked} map</li>
@@ -1368,9 +1367,6 @@ private class FileArchiver extends GridWorker {
         /** Last archived file index (absolute, 0-based). Guarded by <code>this</code>. */
         private volatile long lastAbsArchivedIdx = -1;
 
-        /** current thread stopping advice */
-        private volatile boolean stopped;
-
         /** */
         private NavigableMap<Long, Integer> reserved = new TreeMap<>();
 
@@ -1405,7 +1401,7 @@ private long lastArchivedAbsoluteIndex() {
          */
         private void shutdown() throws IgniteInterruptedCheckedException {
             synchronized (this) {
-                stopped = true;
+                isCancelled = true;
 
                 notifyAll();
             }
@@ -1482,7 +1478,7 @@ private synchronized void release(long absIdx) {
 
             try {
                 synchronized (this) {
-                    while (curAbsWalIdx == -1 && !stopped)
+                    while (curAbsWalIdx == -1 && !isCancelled())
                         wait();
 
                     // If the archive directory is empty, we can be sure that there were no WAL segments archived.
@@ -1490,26 +1486,26 @@ private synchronized void release(long absIdx) {
                     // once it was archived.
                 }
 
-                while (!Thread.currentThread().isInterrupted() && !stopped) {
+                while (!Thread.currentThread().isInterrupted() && !isCancelled()) {
                     long toArchive;
 
                     synchronized (this) {
                         assert lastAbsArchivedIdx <= curAbsWalIdx : "lastArchived=" + lastAbsArchivedIdx +
                             ", current=" + curAbsWalIdx;
 
-                        while (lastAbsArchivedIdx >= curAbsWalIdx - 1 && !stopped)
+                        while (lastAbsArchivedIdx >= curAbsWalIdx - 1 && !isCancelled())
                             wait();
 
                         toArchive = lastAbsArchivedIdx + 1;
                     }
 
-                    if (stopped)
+                    if (isCancelled())
                         break;
 
                     final SegmentArchiveResult res = archiveSegment(toArchive);
 
                     synchronized (this) {
-                        while (locked.containsKey(toArchive) && !stopped)
+                        while (locked.containsKey(toArchive) && !isCancelled())
                             wait();
 
                         changeLastArchivedIndexAndWakeupCompressor(toArchive);
@@ -1526,14 +1522,14 @@ private synchronized void release(long absIdx) {
             catch (InterruptedException t) {
                 Thread.currentThread().interrupt();
 
-                if (!stopped)
+                if (!isCancelled())
                     err = t;
             }
             catch (Throwable t) {
                 err = t;
             }
             finally {
-                if (err == null && !stopped)
+                if (err == null && !isCancelled())
                     err = new IllegalStateException("Worker " + name() + " is terminated unexpectedly");
 
                 if (err instanceof OutOfMemoryError)
@@ -1708,7 +1704,7 @@ private SegmentArchiveResult archiveSegment(long absIdx) throws IgniteCheckedExc
          *
          */
         private boolean checkStop() {
-            return stopped;
+            return isCancelled();
         }
 
         /**

From 3e9d14ae6f547483e9d89be00d393cafc21f5b36 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Mon, 22 Oct 2018 18:14:40 +0300
Subject: [PATCH 443/543] IGNITE-8006 Parallelize cache groups start - Fixes
 #4752.

(cherry picked from commit e1f8f46)

(cherry picked from commit b98ba88)
---
 .../apache/ignite/IgniteSystemProperties.java |   7 +
 .../cache/CacheAffinitySharedManager.java     | 115 ++--
 .../processors/cache/GridCacheIoManager.java  |  22 +-
 .../GridCachePartitionExchangeManager.java    |   4 +-
 .../processors/cache/GridCacheProcessor.java  | 512 +++++++++++++-----
 .../processors/cache/StartCacheInfo.java      | 113 ++++
 .../GridDhtPartitionsExchangeFuture.java      |  76 ++-
 .../ignite/internal/util/IgniteUtils.java     | 149 +++--
 .../util/InitializationProtector.java         |   3 +-
 .../distributed/CacheStartInParallelTest.java | 219 ++++++++
 .../IgniteCrossCacheTxStoreSelfTest.java      |  44 +-
 .../internal/util/IgniteUtilsSelfTest.java    |  74 +++
 .../testsuites/IgniteCacheTestSuite7.java     |   4 +
 13 files changed, 1005 insertions(+), 337 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StartCacheInfo.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 05a40ad105ae3..b9408ef54dd8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -958,6 +958,13 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_WAIT_SCHEMA_UPDATE = "IGNITE_WAIT_SCHEMA_UPDATE";
 
+    /**
+     * Enables start caches in parallel.
+     *
+     * Default is {@code true}.
+     */
+    public static final String IGNITE_ALLOW_START_CACHES_IN_PARALLEL = "IGNITE_ALLOW_START_CACHES_IN_PARALLEL";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 9fae87be6f50b..20d852977932e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -17,12 +17,14 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import javax.cache.CacheException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -30,7 +32,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.stream.Collectors;
-import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
@@ -426,25 +427,43 @@ void onCacheGroupCreated(CacheGroupContext grp) {
 
         Map<Integer, GridDhtAssignmentFetchFuture> fetchFuts = U.newHashMap(startDescs.size());
 
-        Set<String> startedCaches = U.newHashSet(startDescs.size());
-
         Map<Integer, Boolean> startedInfos = U.newHashMap(startDescs.size());
 
-        for (DynamicCacheDescriptor desc : startDescs) {
-            try {
-                startedCaches.add(desc.cacheName());
+        List<StartCacheInfo> startCacheInfos = startDescs.stream()
+            .map(desc -> {
+                DynamicCacheChangeRequest changeReq = startReqs.get(desc.cacheName());
 
-                DynamicCacheChangeRequest startReq = startReqs.get(desc.cacheName());
-
-                cctx.cache().prepareCacheStart(
-                    desc.cacheConfiguration(),
+                return new StartCacheInfo(
                     desc,
-                    startReq.nearCacheConfiguration(),
+                    changeReq.nearCacheConfiguration(),
                     topVer,
-                    startReq.disabledAfterStart()
+                    changeReq.disabledAfterStart()
                 );
+            })
+            .collect(Collectors.toList());
+
+        Set<String> startedCaches = startCacheInfos.stream()
+            .map(info -> info.getCacheDescriptor().cacheName())
+            .collect(Collectors.toSet());
+
+        try {
+            cctx.cache().prepareStartCaches(startCacheInfos);
+        }
+        catch (IgniteCheckedException e) {
+            cctx.cache().closeCaches(startedCaches, false);
+
+            cctx.cache().completeClientCacheChangeFuture(msg.requestId(), e);
+
+            return null;
+        }
+
+        for (StartCacheInfo startCacheInfo : startCacheInfos) {
+            try {
+                DynamicCacheDescriptor desc = startCacheInfo.getCacheDescriptor();
+
+                startedCaches.add(desc.cacheName());
 
-                startedInfos.put(desc.cacheId(), startReq.nearCacheConfiguration() != null);
+                startedInfos.put(desc.cacheId(), startCacheInfo.getReqNearCfg() != null);
 
                 CacheGroupContext grp = cctx.cache().cacheGroup(desc.groupId());
 
@@ -848,6 +867,8 @@ private void processCacheStartRequests(
 
         long time = System.currentTimeMillis();
 
+        Map<StartCacheInfo, DynamicCacheChangeRequest> startCacheInfos = new LinkedHashMap<>();
+
         for (ExchangeActions.CacheActionData action : exchActions.cacheStartRequests()) {
             DynamicCacheDescriptor cacheDesc = action.descriptor();
 
@@ -883,29 +904,41 @@ private void processCacheStartRequests(
                 }
             }
 
-            try {
-                if (startCache) {
-                    cctx.cache().prepareCacheStart(
+            if (startCache) {
+                startCacheInfos.put(
+                    new StartCacheInfo(
                         req.startCacheConfiguration(),
                         cacheDesc,
                         nearCfg,
                         evts.topologyVersion(),
                         req.disabledAfterStart()
-                    );
-
-                    if (fut.cacheAddedOnExchange(cacheDesc.cacheId(), cacheDesc.receivedFrom())) {
-                        if (fut.events().discoveryCache().cacheGroupAffinityNodes(cacheDesc.groupId()).isEmpty())
-                            U.quietAndWarn(log, "No server nodes found for cache client: " + req.cacheName());
-                    }
-                }
+                    ),
+                    req
+                );
             }
-            catch (IgniteCheckedException e) {
-                U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
-                    "[cacheName=" + req.cacheName() + ']', e);
+        }
+
+        Map<StartCacheInfo, IgniteCheckedException> failedCaches = cctx.cache().prepareStartCachesIfPossible(startCacheInfos.keySet());
 
-                cctx.cache().closeCaches(Collections.singleton(req.cacheName()), false);
+        failedCaches.forEach((cacheInfo, exception) -> {
+            U.error(log, "Failed to initialize cache. Will try to rollback cache start routine. " +
+                "[cacheName=" + cacheInfo.getStartedConfiguration().getName() + ']', exception);
 
-                cctx.cache().completeCacheStartFuture(req, false, e);
+            cctx.cache().closeCaches(Collections.singleton(cacheInfo.getStartedConfiguration().getName()), false);
+
+            cctx.cache().completeCacheStartFuture(startCacheInfos.get(cacheInfo), false, exception);
+        });
+
+        Set<StartCacheInfo> failedCacheInfos = failedCaches.keySet();
+
+        List<StartCacheInfo> cacheInfos = startCacheInfos.keySet().stream()
+            .filter(failedCacheInfos::contains)
+            .collect(Collectors.toList());
+
+        for (StartCacheInfo info : cacheInfos) {
+            if (fut.cacheAddedOnExchange(info.getCacheDescriptor().cacheId(), info.getCacheDescriptor().receivedFrom())) {
+                if (fut.events().discoveryCache().cacheGroupAffinityNodes(info.getCacheDescriptor().groupId()).isEmpty())
+                    U.quietAndWarn(log, "No server nodes found for cache client: " + info.getCacheDescriptor().cacheName());
             }
         }
 
@@ -940,22 +973,20 @@ private void initAffinityOnCacheGroupsStart(
         U.doInParallel(
             cctx.kernalContext().getSystemExecutorService(),
             startedGroups,
-            new IgniteInClosureX<CacheGroupDescriptor>() {
-                @Override public void applyx(CacheGroupDescriptor grpDesc) throws IgniteCheckedException {
-                    if (crd)
-                        initStartedGroupOnCoordinator(fut, grpDesc);
-                    else {
-                        CacheGroupContext grp = cctx.cache().cacheGroup(grpDesc.groupId());
+            grpDesc -> {
+                if (crd)
+                    initStartedGroupOnCoordinator(fut, grpDesc);
+                else {
+                    CacheGroupContext grp = cctx.cache().cacheGroup(grpDesc.groupId());
 
-                        if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.initialVersion())) {
-                            assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion();
+                    if (grp != null && !grp.isLocal() && grp.localStartVersion().equals(fut.initialVersion())) {
+                        assert grp.affinity().lastVersion().equals(AffinityTopologyVersion.NONE) : grp.affinity().lastVersion();
 
-                            initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut);
-                        }
+                        initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut);
                     }
                 }
-            },
-            null);
+            }
+        );
     }
 
     /**
@@ -1212,7 +1243,7 @@ private void forAllRegisteredCacheGroups(IgniteInClosureX<CacheGroupDescriptor>
             .collect(Collectors.toList());
 
         try {
-            U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c, null);
+            U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c::applyx);
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException("Failed to execute affinity operation on cache groups", e);
@@ -1239,7 +1270,7 @@ private void forAllCacheGroups(boolean crd, IgniteInClosureX<GridAffinityAssignm
         }
 
         try {
-            U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c, null);
+            U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c::applyx);
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException("Failed to execute affinity operation on cache groups", e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 9b7d2a01b0501..5e8b2461576c6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -1344,22 +1344,22 @@ private void addHandler(
         if (msgIdx != -1) {
             Map<Integer, IgniteBiInClosure[]> idxClsHandlers0 = msgHandlers.idxClsHandlers;
 
-            IgniteBiInClosure[] cacheClsHandlers = idxClsHandlers0.get(hndId);
+            IgniteBiInClosure[] cacheClsHandlers = idxClsHandlers0.compute(hndId, (key, clsHandlers) -> {
+                if (clsHandlers == null)
+                    clsHandlers = new IgniteBiInClosure[GridCacheMessage.MAX_CACHE_MSG_LOOKUP_INDEX];
 
-            if (cacheClsHandlers == null) {
-                cacheClsHandlers = new IgniteBiInClosure[GridCacheMessage.MAX_CACHE_MSG_LOOKUP_INDEX];
+                if(clsHandlers[msgIdx] != null)
+                    return null;
 
-                idxClsHandlers0.put(hndId, cacheClsHandlers);
-            }
+                clsHandlers[msgIdx] = c;
+
+                return clsHandlers;
+            });
 
-            if (cacheClsHandlers[msgIdx] != null)
+            if (cacheClsHandlers == null)
                 throw new IgniteException("Duplicate cache message ID found [hndId=" + hndId +
                     ", type=" + type + ']');
 
-            cacheClsHandlers[msgIdx] = c;
-
-            msgHandlers.idxClsHandlers = idxClsHandlers0;
-
             return;
         }
         else {
@@ -1576,7 +1576,7 @@ else if (msg instanceof GridCacheGroupIdMessage)
      */
     static class MessageHandlers {
         /** Indexed class handlers. */
-        volatile Map<Integer, IgniteBiInClosure[]> idxClsHandlers = new HashMap<>();
+        volatile Map<Integer, IgniteBiInClosure[]> idxClsHandlers = new ConcurrentHashMap<>();
 
         /** Handler registry. */
         ConcurrentMap<ListenerKey, IgniteBiInClosure<UUID, GridCacheMessage>>
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index dd83036182ff6..05ebd11251434 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -109,6 +109,7 @@
 import org.apache.ignite.internal.util.typedef.CI2;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -2580,7 +2581,8 @@ void dumpExchangeDebugInfo() {
                     err = e;
             }
             catch (Throwable e) {
-                err = e;
+                if (!(stop && X.hasCause(e, IgniteInterruptedCheckedException.class)))
+                    err = e;
             }
             finally {
                 if (err == null && !stop && !reconnectNeeded)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 98cf2115fcf1a..a536e68621c02 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.processors.cache;
 
+import javax.management.MBeanServer;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -64,6 +65,7 @@
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteComponentType;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.IgniteTransactionsEx;
 import org.apache.ignite.internal.binary.BinaryContext;
@@ -129,14 +131,15 @@
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
 import org.apache.ignite.internal.suggestions.GridPerformanceSuggestions;
 import org.apache.ignite.internal.util.F0;
+import org.apache.ignite.internal.util.InitializationProtector;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.IgniteOutClosureX;
+import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CIX1;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.CU;
@@ -182,6 +185,7 @@
 import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_TX_CONFIG;
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled;
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isPersistentCache;
+import static org.apache.ignite.internal.util.IgniteUtils.doInParallel;
 
 /**
  * Cache processor.
@@ -203,6 +207,10 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     private final boolean walFsyncWithDedicatedWorker =
         IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_WAL_FSYNC_WITH_DEDICATED_WORKER, false);
 
+    /** Enables start caches in parallel. */
+    private final boolean IGNITE_ALLOW_START_CACHES_IN_PARALLEL =
+        IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, true);
+
     /** Shared cache context. */
     private GridCacheSharedContext<?, ?> sharedCtx;
 
@@ -255,6 +263,9 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     /** MBean group for cache group metrics */
     private final String CACHE_GRP_METRICS_MBEAN_GRP = "Cache groups";
 
+    /** Protector of initialization of specific value. */
+    private final InitializationProtector initializationProtector = new InitializationProtector();
+
     /**
      * @param ctx Kernal context.
      */
@@ -1170,69 +1181,6 @@ private void stopCacheOnReconnect(GridCacheContext cctx, List<GridCacheAdapter>
         return null;
     }
 
-    /**
-     * @param cache Cache to start.
-     * @param schema Cache schema.
-     * @throws IgniteCheckedException If failed to start cache.
-     */
-    @SuppressWarnings({"TypeMayBeWeakened", "unchecked"})
-    private void startCache(GridCacheAdapter<?, ?> cache, QuerySchema schema) throws IgniteCheckedException {
-        GridCacheContext<?, ?> cacheCtx = cache.context();
-
-        CacheConfiguration cfg = cacheCtx.config();
-
-        // Intentionally compare Boolean references using '!=' below to check if the flag has been explicitly set.
-        if (cfg.isStoreKeepBinary() && cfg.isStoreKeepBinary() != CacheConfiguration.DFLT_STORE_KEEP_BINARY
-            && !(ctx.config().getMarshaller() instanceof BinaryMarshaller))
-            U.warn(log, "CacheConfiguration.isStoreKeepBinary() configuration property will be ignored because " +
-                "BinaryMarshaller is not used");
-
-        // Start managers.
-        for (GridCacheManager mgr : F.view(cacheCtx.managers(), F.notContains(dhtExcludes(cacheCtx))))
-            mgr.start(cacheCtx);
-
-        cacheCtx.initConflictResolver();
-
-        if (cfg.getCacheMode() != LOCAL && GridCacheUtils.isNearEnabled(cfg)) {
-            GridCacheContext<?, ?> dhtCtx = cacheCtx.near().dht().context();
-
-            // Start DHT managers.
-            for (GridCacheManager mgr : dhtManagers(dhtCtx))
-                mgr.start(dhtCtx);
-
-            dhtCtx.initConflictResolver();
-
-            // Start DHT cache.
-            dhtCtx.cache().start();
-
-            if (log.isDebugEnabled())
-                log.debug("Started DHT cache: " + dhtCtx.cache().name());
-        }
-
-        ctx.continuous().onCacheStart(cacheCtx);
-
-        cacheCtx.cache().start();
-
-        ctx.query().onCacheStart(cacheCtx, schema);
-
-        cacheCtx.onStarted();
-
-        String memPlcName = cfg.getDataRegionName();
-
-        if (memPlcName == null && ctx.config().getDataStorageConfiguration() != null)
-            memPlcName = ctx.config().getDataStorageConfiguration().getDefaultDataRegionConfiguration().getName();
-
-        if (log.isInfoEnabled()) {
-            log.info("Started cache [name=" + cfg.getName() +
-                ", id=" + cacheCtx.cacheId() +
-                (cfg.getGroupName() != null ? ", group=" + cfg.getGroupName() : "") +
-                ", memoryPolicyName=" + memPlcName +
-                ", mode=" + cfg.getCacheMode() +
-                ", atomicity=" + cfg.getAtomicityMode() +
-                ", backups=" + cfg.getBackups() + ']');
-        }
-    }
-
     /**
      * @param cache Cache to stop.
      * @param cancel Cancel flag.
@@ -1481,7 +1429,13 @@ private GridCacheContext createCache(CacheConfiguration<?, ?> cfg,
         GridCacheDrManager drMgr = pluginMgr.createComponent(GridCacheDrManager.class);
         CacheStoreManager storeMgr = pluginMgr.createComponent(CacheStoreManager.class);
 
-        storeMgr.initialize(cfgStore, sesHolders);
+        if (cfgStore == null)
+            storeMgr.initialize(cfgStore, sesHolders);
+        else
+            initializationProtector.protect(
+                cfgStore,
+                () -> storeMgr.initialize(cfgStore, sesHolders)
+            );
 
         GridCacheContext<?, ?> cacheCtx = new GridCacheContext(
             ctx,
@@ -1894,16 +1848,11 @@ public IgniteInternalFuture<?> startCachesOnLocalJoin(
         IgniteInternalFuture<?> res = sharedCtx.affinity().initCachesOnLocalJoin(
             locJoinCtx.cacheGroupDescriptors(), locJoinCtx.cacheDescriptors());
 
-        for (T2<DynamicCacheDescriptor, NearCacheConfiguration> t : locJoinCtx.caches()) {
-            DynamicCacheDescriptor desc = t.get1();
+        List<StartCacheInfo> startCacheInfos = locJoinCtx.caches().stream()
+            .map(cacheInfo -> new StartCacheInfo(cacheInfo.get1(), cacheInfo.get2(), exchTopVer, false))
+            .collect(Collectors.toList());
 
-            prepareCacheStart(
-                desc.cacheConfiguration(),
-                desc,
-                t.get2(),
-                exchTopVer,
-                false);
-        }
+        prepareStartCaches(startCacheInfos);
 
         if (log.isInfoEnabled())
             log.info("Starting caches on local join performed in " + (System.currentTimeMillis() - time) + " ms.");
@@ -1929,22 +1878,156 @@ boolean hasCachesReceivedFromJoin(ClusterNode node) {
      */
     public Collection<DynamicCacheDescriptor> startReceivedCaches(UUID nodeId, AffinityTopologyVersion exchTopVer)
         throws IgniteCheckedException {
-        List<DynamicCacheDescriptor> started = cachesInfo.cachesReceivedFromJoin(nodeId);
+        List<DynamicCacheDescriptor> receivedCaches = cachesInfo.cachesReceivedFromJoin(nodeId);
 
-        for (DynamicCacheDescriptor desc : started) {
-            IgnitePredicate<ClusterNode> filter = desc.groupDescriptor().config().getNodeFilter();
+        List<StartCacheInfo> startCacheInfos = receivedCaches.stream()
+            .filter(desc -> isLocalAffinity(desc.groupDescriptor().config()))
+            .map(desc -> new StartCacheInfo(desc, null, exchTopVer, false))
+            .collect(Collectors.toList());
 
-            if (CU.affinityNode(ctx.discovery().localNode(), filter)) {
-                prepareCacheStart(
-                    desc.cacheConfiguration(),
-                    desc,
-                    null,
-                    exchTopVer,
-                    false);
+        prepareStartCaches(startCacheInfos);
+
+        return receivedCaches;
+    }
+
+    /**
+     * @param cacheConfiguration Checked configuration.
+     * @return {@code true} if local node is affinity node for cache.
+     */
+    private boolean isLocalAffinity(CacheConfiguration cacheConfiguration) {
+        return CU.affinityNode(ctx.discovery().localNode(), cacheConfiguration.getNodeFilter());
+    }
+
+    /**
+     * Start all input caches in parallel.
+     *
+     * @param startCacheInfos All caches information for start.
+     */
+    void prepareStartCaches(Collection<StartCacheInfo> startCacheInfos) throws IgniteCheckedException {
+        prepareStartCaches(startCacheInfos, (data, operation) -> {
+            operation.accept(data);// PROXY
+        });
+    }
+
+    /**
+     * Trying to start all input caches in parallel and skip failed caches.
+     *
+     * @param startCacheInfos Caches info for start.
+     * @return Caches which was failed.
+     * @throws IgniteCheckedException if failed.
+     */
+    Map<StartCacheInfo, IgniteCheckedException> prepareStartCachesIfPossible(Collection<StartCacheInfo> startCacheInfos) throws IgniteCheckedException {
+        HashMap<StartCacheInfo, IgniteCheckedException> failedCaches = new HashMap<>();
+
+        prepareStartCaches(startCacheInfos, (data, operation) -> {
+            try {
+                operation.accept(data);
+            }
+            catch (IgniteInterruptedCheckedException e) {
+                throw e;
+            }
+            catch (IgniteCheckedException e) {
+                log.warning("Cache can not be started : cache=" + data.getStartedConfiguration().getName());
+
+                failedCaches.put(data, e);
+            }
+        });
+
+        return failedCaches;
+    }
+
+    /**
+     * Start all input caches in parallel.
+     *
+     * @param startCacheInfos All caches information for start.
+     * @param cacheStartFailHandler Fail handler for one cache start.
+     */
+    private void prepareStartCaches(
+        Collection<StartCacheInfo> startCacheInfos,
+        StartCacheFailHandler<StartCacheInfo> cacheStartFailHandler
+    ) throws IgniteCheckedException {
+        if (!IGNITE_ALLOW_START_CACHES_IN_PARALLEL || startCacheInfos.size() <= 1) {
+            for (StartCacheInfo startCacheInfo : startCacheInfos) {
+                cacheStartFailHandler.handle(
+                    startCacheInfo,
+                    cacheInfo -> prepareCacheStart(
+                        cacheInfo.getCacheDescriptor().cacheConfiguration(),
+                        cacheInfo.getCacheDescriptor(),
+                        cacheInfo.getReqNearCfg(),
+                        cacheInfo.getExchangeTopVer(),
+                        cacheInfo.isDisabledAfterStart()
+                    )
+                );
             }
         }
+        else {
+            Map<StartCacheInfo, GridCacheContext> cacheContexts = new ConcurrentHashMap<>();
+
+            int parallelismLvl = sharedCtx.kernalContext().config().getSystemThreadPoolSize();
+
+            // Reserve at least 2 threads for system operations.
+            parallelismLvl = Math.max(1, parallelismLvl - 2);
+
+            doInParallel(
+                parallelismLvl,
+                sharedCtx.kernalContext().getSystemExecutorService(),
+                startCacheInfos,
+                startCacheInfo ->
+                    cacheStartFailHandler.handle(
+                        startCacheInfo,
+                        cacheInfo -> {
+                            GridCacheContext cacheCtx = prepareCacheContext(
+                                cacheInfo.getCacheDescriptor().cacheConfiguration(),
+                                cacheInfo.getCacheDescriptor(),
+                                cacheInfo.getReqNearCfg(),
+                                cacheInfo.getExchangeTopVer(),
+                                cacheInfo.isDisabledAfterStart()
+                            );
+                            cacheContexts.put(cacheInfo, cacheCtx);
+                        }
+                    )
+            );
 
-        return started;
+            /*
+             * This hack required because we can't start sql schema in parallel by folowing reasons:
+             * * checking index to duplicate(and other checking) require one order on every nodes.
+             * * onCacheStart and createSchema contains a lot of mutex.
+             *
+             * TODO IGNITE-9729
+             */
+            Set<StartCacheInfo> successfullyPreparedCaches = cacheContexts.keySet();
+
+            List<StartCacheInfo> cacheInfosInOriginalOrder = startCacheInfos.stream()
+                .filter(successfullyPreparedCaches::contains)
+                .collect(Collectors.toList());
+
+            for (StartCacheInfo startCacheInfo : cacheInfosInOriginalOrder) {
+                cacheStartFailHandler.handle(
+                    startCacheInfo,
+                    cacheInfo -> {
+                        ctx.query().onCacheStart(
+                            cacheContexts.get(cacheInfo),
+                            cacheInfo.getCacheDescriptor().schema() != null
+                                ? cacheInfo.getCacheDescriptor().schema()
+                                : new QuerySchema()
+                        );
+                    }
+                );
+            }
+
+            doInParallel(
+                parallelismLvl,
+                sharedCtx.kernalContext().getSystemExecutorService(),
+                cacheContexts.entrySet(),
+                cacheCtxEntry ->
+                    cacheStartFailHandler.handle(
+                        cacheCtxEntry.getKey(),
+                        cacheInfo -> {
+                            onCacheStarted(cacheCtxEntry.getValue());
+                        }
+                    )
+            );
+        }
     }
 
     /**
@@ -1962,6 +2045,32 @@ void prepareCacheStart(
         @Nullable NearCacheConfiguration reqNearCfg,
         AffinityTopologyVersion exchTopVer,
         boolean disabledAfterStart
+    ) throws IgniteCheckedException {
+        GridCacheContext cacheCtx = prepareCacheContext(startCfg, desc, reqNearCfg, exchTopVer, disabledAfterStart);
+
+        ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema());
+
+        onCacheStarted(cacheCtx);
+    }
+
+    /**
+     * Preparing cache context to start.
+     *
+     * @param startCfg Cache configuration to use.
+     * @param desc Cache descriptor.
+     * @param reqNearCfg Near configuration if specified for client cache start request.
+     * @param exchTopVer Current exchange version.
+     * @param disabledAfterStart If true, then we will discard restarting state from proxies. If false then we will change
+     *  state of proxies to restarting
+     * @return Created {@link GridCacheContext}.
+     * @throws IgniteCheckedException if failed.
+     */
+    private GridCacheContext prepareCacheContext(
+        CacheConfiguration startCfg,
+        DynamicCacheDescriptor desc,
+        @Nullable NearCacheConfiguration reqNearCfg,
+        AffinityTopologyVersion exchTopVer,
+        boolean disabledAfterStart
     ) throws IgniteCheckedException {
         assert !caches.containsKey(startCfg.getName()) : startCfg.getName();
 
@@ -1969,65 +2078,124 @@ void prepareCacheStart(
 
         CacheObjectContext cacheObjCtx = ctx.cacheObjects().contextForCache(ccfg);
 
-        boolean affNode;
+        boolean affNode = checkForAffinityNode(desc, reqNearCfg, ccfg);
 
-        if (ccfg.getCacheMode() == LOCAL) {
-            affNode = true;
+        preparePageStore(desc, affNode);
 
+        CacheGroupContext grp = prepareCacheGroup(desc, exchTopVer, cacheObjCtx, affNode, startCfg.getGroupName());
+
+        GridCacheContext cacheCtx = createCache(ccfg,
+            grp,
+            null,
+            desc,
+            exchTopVer,
+            cacheObjCtx,
+            affNode,
+            true,
+            disabledAfterStart
+        );
+
+        initCacheContext(cacheCtx, ccfg, desc.deploymentId());
+
+        return cacheCtx;
+    }
+
+    /**
+     * Check for affinity node and customize near configuration if needed.
+     *
+     * @param desc Cache descriptor.
+     * @param reqNearCfg Near configuration if specified for client cache start request.
+     * @param ccfg Cache configuration to use.
+     * @return {@code true} if it is affinity node for cache.
+     */
+    private boolean checkForAffinityNode(
+        DynamicCacheDescriptor desc,
+        @Nullable NearCacheConfiguration reqNearCfg,
+        CacheConfiguration ccfg
+    ) {
+        if (ccfg.getCacheMode() == LOCAL) {
             ccfg.setNearConfiguration(null);
-        }
-        else if (CU.affinityNode(ctx.discovery().localNode(), desc.groupDescriptor().config().getNodeFilter()))
-            affNode = true;
-        else {
-            affNode = false;
 
-            ccfg.setNearConfiguration(reqNearCfg);
+            return true;
         }
 
-        if (sharedCtx.pageStore() != null && affNode)
-            sharedCtx.pageStore().initializeForCache(desc.groupDescriptor(), desc.toStoredData());
-
-        String grpName = startCfg.getGroupName();
+        if (isLocalAffinity(desc.groupDescriptor().config()))
+            return true;
 
-        CacheGroupContext grp = null;
+        ccfg.setNearConfiguration(reqNearCfg);
 
-        if (grpName != null) {
-            for (CacheGroupContext grp0 : cacheGrps.values()) {
-                if (grp0.sharedGroup() && grpName.equals(grp0.name())) {
-                    grp = grp0;
+        return false;
+    }
 
-                    break;
-                }
-            }
+    /**
+     * Prepare page store for start cache.
+     *
+     * @param desc Cache descriptor.
+     * @param affNode {@code true} if it is affinity node for cache.
+     * @throws IgniteCheckedException if failed.
+     */
+    private void preparePageStore(DynamicCacheDescriptor desc, boolean affNode) throws IgniteCheckedException {
+        if (sharedCtx.pageStore() != null && affNode)
+            initializationProtector.protect(
+                desc.groupDescriptor().groupId(),
+                () -> sharedCtx.pageStore().initializeForCache(desc.groupDescriptor(), desc.toStoredData())
+            );
+    }
 
-            if (grp == null) {
-                grp = startCacheGroup(desc.groupDescriptor(),
+    /**
+     * Prepare cache group to start cache.
+     *
+     * @param desc Cache descriptor.
+     * @param exchTopVer Current exchange version.
+     * @param cacheObjCtx Cache object context.
+     * @param affNode {@code true} if it is affinity node for cache.
+     * @param grpName Group name.
+     * @return Prepared cache group context.
+     * @throws IgniteCheckedException if failed.
+     */
+    private CacheGroupContext prepareCacheGroup(
+        DynamicCacheDescriptor desc,
+        AffinityTopologyVersion exchTopVer,
+        CacheObjectContext cacheObjCtx,
+        boolean affNode,
+        String grpName
+    ) throws IgniteCheckedException {
+        if (grpName != null) {
+            return initializationProtector.protect(
+                desc.groupId(),
+                () -> findCacheGroup(grpName),
+                () -> startCacheGroup(
+                    desc.groupDescriptor(),
                     desc.cacheType(),
                     affNode,
                     cacheObjCtx,
-                    exchTopVer);
-            }
-        }
-        else {
-            grp = startCacheGroup(desc.groupDescriptor(),
-                desc.cacheType(),
-                affNode,
-                cacheObjCtx,
-                exchTopVer);
+                    exchTopVer
+                )
+            );
         }
 
-        GridCacheContext cacheCtx = createCache(ccfg,
-            grp,
-            null,
-            desc,
-            exchTopVer,
-            cacheObjCtx,
+        return startCacheGroup(desc.groupDescriptor(),
+            desc.cacheType(),
             affNode,
-            true,
-            disabledAfterStart
+            cacheObjCtx,
+            exchTopVer
         );
+    }
 
-        cacheCtx.dynamicDeploymentId(desc.deploymentId());
+    /**
+     * Initialize created cache context.
+     *
+     * @param cacheCtx Cache context to initializtion.
+     * @param cfg Cache configuration.
+     * @param deploymentId Dynamic deployment ID.
+     * @throws IgniteCheckedException if failed.
+     */
+    private void initCacheContext(
+        GridCacheContext<?, ?> cacheCtx,
+        CacheConfiguration cfg,
+        IgniteUuid deploymentId
+    ) throws IgniteCheckedException {
+        cacheCtx.dynamicDeploymentId(deploymentId);
 
         GridCacheAdapter cache = cacheCtx.cache();
 
@@ -2035,13 +2203,83 @@ else if (CU.affinityNode(ctx.discovery().localNode(), desc.groupDescriptor().con
 
         caches.put(cacheCtx.name(), cache);
 
-        startCache(cache, desc.schema() != null ? desc.schema() : new QuerySchema());
+        // Intentionally compare Boolean references using '!=' below to check if the flag has been explicitly set.
+        if (cfg.isStoreKeepBinary() && cfg.isStoreKeepBinary() != CacheConfiguration.DFLT_STORE_KEEP_BINARY
+            && !(ctx.config().getMarshaller() instanceof BinaryMarshaller))
+            U.warn(log, "CacheConfiguration.isStoreKeepBinary() configuration property will be ignored because " +
+                "BinaryMarshaller is not used");
+
+        // Start managers.
+        for (GridCacheManager mgr : F.view(cacheCtx.managers(), F.notContains(dhtExcludes(cacheCtx))))
+            mgr.start(cacheCtx);
+
+        cacheCtx.initConflictResolver();
+
+        if (cfg.getCacheMode() != LOCAL && GridCacheUtils.isNearEnabled(cfg)) {
+            GridCacheContext<?, ?> dhtCtx = cacheCtx.near().dht().context();
+
+            // Start DHT managers.
+            for (GridCacheManager mgr : dhtManagers(dhtCtx))
+                mgr.start(dhtCtx);
+
+            dhtCtx.initConflictResolver();
+
+            // Start DHT cache.
+            dhtCtx.cache().start();
+
+            if (log.isDebugEnabled())
+                log.debug("Started DHT cache: " + dhtCtx.cache().name());
+        }
+
+        ctx.continuous().onCacheStart(cacheCtx);
+
+        cacheCtx.cache().start();
+    }
+
+    /**
+     * Handle of cache context which was fully prepared.
+     *
+     * @param cacheCtx Fully prepared context.
+     * @throws IgniteCheckedException if failed.
+     */
+    private void onCacheStarted(GridCacheContext cacheCtx) throws IgniteCheckedException {
+        GridCacheAdapter cache = cacheCtx.cache();
+        CacheConfiguration cfg = cacheCtx.config();
+        CacheGroupContext grp = cacheGrps.get(cacheCtx.groupId());
+
+        cacheCtx.onStarted();
+
+        String dataRegion = cfg.getDataRegionName();
+
+        if (dataRegion == null && ctx.config().getDataStorageConfiguration() != null)
+            dataRegion = ctx.config().getDataStorageConfiguration().getDefaultDataRegionConfiguration().getName();
+
+        if (log.isInfoEnabled()) {
+            log.info("Started cache [name=" + cfg.getName() +
+                ", id=" + cacheCtx.cacheId() +
+                (cfg.getGroupName() != null ? ", group=" + cfg.getGroupName() : "") +
+                ", dataRegionName=" + dataRegion +
+                ", mode=" + cfg.getCacheMode() +
+                ", atomicity=" + cfg.getAtomicityMode() +
+                ", backups=" + cfg.getBackups() + ']');
+        }
 
         grp.onCacheStarted(cacheCtx);
 
         onKernalStart(cache);
     }
 
+    /**
+     * @param grpName Group name.
+     * @return Found group or null.
+     */
+    private CacheGroupContext findCacheGroup(String grpName) {
+        return cacheGrps.values().stream()
+            .filter(grp -> grp.sharedGroup() && grpName.equals(grp.name()))
+            .findAny()
+            .orElse(null);
+    }
+
     /**
      * Restarts proxies of caches if they was marked as restarting. Requires external synchronization - shouldn't be
      * called concurrently with another caches restart.
@@ -4600,7 +4838,7 @@ private DynamicCacheChangeRequest prepareCacheChangeRequest(
 
                     // Check if we were asked to start a near cache.
                     if (nearCfg != null) {
-                        if (CU.affinityNode(ctx.discovery().localNode(), descCfg.getNodeFilter())) {
+                        if (isLocalAffinity(descCfg)) {
                             // If we are on a data node and near cache was enabled, return success, else - fail.
                             if (descCfg.getNearConfiguration() != null)
                                 return null;
@@ -4612,7 +4850,7 @@ private DynamicCacheChangeRequest prepareCacheChangeRequest(
                             // If local node has near cache, return success.
                             req.clientStartOnly(true);
                     }
-                    else if (!CU.affinityNode(ctx.discovery().localNode(), descCfg.getNodeFilter()))
+                    else if (!isLocalAffinity(descCfg))
                         req.clientStartOnly(true);
 
                     req.deploymentId(desc.deploymentId());
@@ -4737,6 +4975,22 @@ public <T> T clone(final T obj) throws IgniteCheckedException {
         });
     }
 
+    /**
+     * Handle of fail during cache start.
+     *
+     * @param <T> Type of started data.
+     */
+    private static interface StartCacheFailHandler<T> {
+        /**
+         * Handle of fail.
+         *
+         * @param data Start data.
+         * @param startCacheOperation Operation for start cache.
+         * @throws IgniteCheckedException if failed.
+         */
+        void handle(T data, IgniteThrowableConsumer<T> startCacheOperation) throws IgniteCheckedException;
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StartCacheInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StartCacheInfo.java
new file mode 100644
index 0000000000000..a5aea26453ff6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/StartCacheInfo.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Specific cache information for start.
+ */
+public class StartCacheInfo {
+    /** Cache configuration for start. */
+    private final CacheConfiguration startedConf;
+
+    /** Cache descriptor for start. */
+    private final DynamicCacheDescriptor desc;
+
+    /** Near cache configuration for start. */
+    private final @Nullable NearCacheConfiguration reqNearCfg;
+
+    /** Exchange topology version in which starting happened. */
+    private final AffinityTopologyVersion exchTopVer;
+
+    /** Disable started cache after start or not. */
+    private final boolean disabledAfterStart;
+
+    /**
+     * @param desc Cache configuration for start.
+     * @param reqNearCfg Near cache configuration for start.
+     * @param exchTopVer Exchange topology version in which starting happened.
+     * @param disabledAfterStart Disable started cache after start or not.
+     */
+    public StartCacheInfo(DynamicCacheDescriptor desc,
+        NearCacheConfiguration reqNearCfg,
+        AffinityTopologyVersion exchTopVer, boolean disabledAfterStart) {
+        this(desc.cacheConfiguration(), desc, reqNearCfg, exchTopVer, disabledAfterStart);
+    }
+
+    /**
+     * @param conf Cache configuration for start.
+     * @param desc Cache descriptor for start.
+     * @param reqNearCfg Near cache configuration for start.
+     * @param exchTopVer Exchange topology version in which starting happened.
+     * @param disabledAfterStart Disable started cache after start or not.
+     */
+    public StartCacheInfo(CacheConfiguration conf, DynamicCacheDescriptor desc,
+        NearCacheConfiguration reqNearCfg,
+        AffinityTopologyVersion exchTopVer, boolean disabledAfterStart) {
+        startedConf = conf;
+        this.desc = desc;
+        this.reqNearCfg = reqNearCfg;
+        this.exchTopVer = exchTopVer;
+        this.disabledAfterStart = disabledAfterStart;
+    }
+
+    /**
+     * @return Cache configuration for start.
+     */
+    public CacheConfiguration getStartedConfiguration() {
+        return startedConf;
+    }
+
+    /**
+     * @return Cache descriptor for start.
+     */
+    public DynamicCacheDescriptor getCacheDescriptor() {
+        return desc;
+    }
+
+    /**
+     * @return Near cache configuration for start.
+     */
+    @Nullable public NearCacheConfiguration getReqNearCfg() {
+        return reqNearCfg;
+    }
+
+    /**
+     * @return Exchange topology version in which starting happened.
+     */
+    public AffinityTopologyVersion getExchangeTopVer() {
+        return exchTopVer;
+    }
+
+    /**
+     * @return Disable started cache after start or not.
+     */
+    public boolean isDisabledAfterStart() {
+        return disabledAfterStart;
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(StartCacheInfo.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index fee68f7a07cf2..938b9fe429f6c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -3248,35 +3248,33 @@ private void validatePartitionsState() {
             U.doInParallel(
                 cctx.kernalContext().getSystemExecutorService(),
                 nonLocalCacheGroupDescriptors(),
-                new IgniteInClosureX<CacheGroupDescriptor>() {
-                    @Override public void applyx(CacheGroupDescriptor grpDesc) {
-                        CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId());
-
-                        GridDhtPartitionTopology top = grpCtx != null
-                            ? grpCtx.topology()
-                            : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache());
-
-            // Do not validate read or write through caches or caches with disabled rebalance.
-
-            if (grpCtx == null
-                    || grpCtx.config().isReadThrough()
-                    || grpCtx.config().isWriteThrough()
-                    || grpCtx.config().getCacheStoreFactory() != null
-                    || grpCtx.config().getRebalanceDelay() == -1
-                    || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE
-                    )
-                return;
+                grpDesc -> {
+                    CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId());
+
+                    GridDhtPartitionTopology top = grpCtx != null
+                        ? grpCtx.topology()
+                        : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache());
+
+                    // Do not validate read or write through caches or caches with disabled rebalance.
+
+                    if (grpCtx == null
+                        || grpCtx.config().isReadThrough()
+                        || grpCtx.config().isWriteThrough()
+                        || grpCtx.config().getCacheStoreFactory() != null
+                        || grpCtx.config().getRebalanceDelay() == -1
+                        || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE
+                        )
+                        return;
 
-                        try {
-                            validator.validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture.this, top, msgs);
-                        }
-                        catch (IgniteCheckedException ex) {
-                            log.warning("Partition states validation has failed for group: " + grpCtx.cacheOrGroupName() + ". " + ex.getMessage());
-                            // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833
-                        }
+                    try {
+                        validator.validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture.this, top, msgs);
                     }
-                },
-                null);
+                    catch (IgniteCheckedException ex) {
+                        log.warning("Partition states validation has failed for group: " + grpCtx.cacheOrGroupName() + ". " + ex.getMessage());
+                        // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833
+                    }
+                }
+            );
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException("Failed to validate partitions state", e);
@@ -3296,21 +3294,19 @@ private void assignPartitionsStates() {
             U.doInParallel(
                 cctx.kernalContext().getSystemExecutorService(),
                 nonLocalCacheGroupDescriptors(),
-                new IgniteInClosureX<CacheGroupDescriptor>() {
-                    @Override public void applyx(CacheGroupDescriptor grpDesc) {
-                        CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId());
+                grpDesc -> {
+                    CacheGroupContext grpCtx = cctx.cache().cacheGroup(grpDesc.groupId());
 
-                        GridDhtPartitionTopology top = grpCtx != null
-                            ? grpCtx.topology()
-                            : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache());
+                    GridDhtPartitionTopology top = grpCtx != null
+                        ? grpCtx.topology()
+                        : cctx.exchange().clientTopology(grpDesc.groupId(), events().discoveryCache());
 
-                        if (!CU.isPersistentCache(grpDesc.config(), cctx.gridConfig().getDataStorageConfiguration()))
-                            assignPartitionSizes(top);
-                        else
-                            assignPartitionStates(top);
-                    }
-                },
-                null);
+                    if (!CU.isPersistentCache(grpDesc.config(), cctx.gridConfig().getDataStorageConfiguration()))
+                        assignPartitionSizes(top);
+                    else
+                        assignPartitionStates(top);
+                }
+            );
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException("Failed to assign partition states", e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 2087b10be790c..69a5a4ec0b284 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -141,6 +141,7 @@
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 import java.util.zip.ZipInputStream;
@@ -220,13 +221,11 @@
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.A;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.util.worker.GridWorker;
-import org.apache.ignite.lang.IgniteBiInClosure;
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteClosure;
 import org.apache.ignite.lang.IgniteFutureCancelledException;
@@ -234,6 +233,7 @@
 import org.apache.ignite.lang.IgniteOutClosure;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.lifecycle.LifecycleAware;
 import org.apache.ignite.marshaller.Marshaller;
@@ -10506,114 +10506,91 @@ public static String toHexString(ByteBuffer buf) {
     }
 
     /**
+     * Execute operation on data in parallel.
+     *
      * @param executorSvc Service for parallel execution.
      * @param srcDatas List of data for parallelization.
-     * @param consumer Logic for execution of on each item of data.
+     * @param operation Logic for execution of on each item of data.
      * @param <T> Type of data.
-     * @throws ParallelExecutionException if parallel execution was failed. It contains actual exception in suppressed section.
+     * @throws IgniteCheckedException if parallel execution was failed.
      */
-    public static <T> void doInParallel(int parallelismLvl, ExecutorService executorSvc, Collection<T> srcDatas,
-        IgniteThrowableConsumer<T> consumer) throws ParallelExecutionException {
-
-        List<List<T>> batches = new ArrayList<>(parallelismLvl);
-
-        for (int i = 0; i < parallelismLvl; i++)
-            batches.add(new ArrayList<>());
-
-        int i = 0;
-
-        for (T src : srcDatas) {
-            int idx = i % parallelismLvl;
-
-            List<T> batch = batches.get(idx);
-
-            batch.add(src);
-
-            i++;
-        }
-
-        List<T2<List<T>, Future<Object>>> consumerFutures = batches.stream()
-            .filter(batch -> !batch.isEmpty())
-            .map(batch -> new T2<>(
-                batch,
-                executorSvc.submit(() -> {
-                    for (T item : batch)
-                        consumer.accept(item);
-
-                    return null;
-                })))
-            .collect(Collectors.toList());
-
-        ParallelExecutionException executionE = null;
-
-        for (T2<List<T>, Future<Object>> future : consumerFutures) {
-            try {
-                future.get2().get();
-            }
-            catch (Exception e) {
-                if (executionE == null)
-                    executionE = new ParallelExecutionException("Failed during parallel execution.");
-
-                executionE.addSuppressed(e);
-
-                for (T failedData : future.get1())
-                    executionE.addFailedData(failedData);
-            }
-        }
-
-        if (executionE != null)
-            throw executionE;
+    public static <T> void doInParallel(ExecutorService executorSvc, Collection<T> srcDatas,
+        IgniteThrowableConsumer<T> operation) throws IgniteCheckedException, IgniteInterruptedCheckedException {
+        doInParallel(srcDatas.size(), executorSvc, srcDatas, operation);
     }
 
     /**
+     * Execute operation on data in parallel.
+     *
+     * @param parallelismLvl Number of threads on which it should be executed.
      * @param executorSvc Service for parallel execution.
      * @param srcDatas List of data for parallelization.
-     * @param consumer Logic for execution of on each item of data.
-     * @param errHnd Optionan error handler. If not {@code null}, an error of each item execution will be passed to
-     *      this handler. If error handler is not {@code null}, the exception will not be thrown from this method.
+     * @param operation Logic for execution of on each item of data.
      * @param <T> Type of data.
-     * @return List of (item, execution future) tuples.
-     * @throws IgniteCheckedException If parallel execution failed and {@code errHnd} is {@code null}.
+     * @throws IgniteCheckedException if parallel execution was failed.
      */
-    public static <T> List<T2<T, Future<Object>>> doInParallel(
+    public static <T> void doInParallel(
+        int parallelismLvl,
         ExecutorService executorSvc,
         Collection<T> srcDatas,
-        IgniteInClosureX<T> consumer,
-        @Nullable IgniteBiInClosure<T, Throwable> errHnd
-    ) throws IgniteCheckedException {
-        List<T2<T, Future<Object>>> consumerFutures = srcDatas.stream()
-            .map(item -> new T2<>(
-                item,
-                executorSvc.submit(() -> {
-                    consumer.apply(item);
+        IgniteThrowableConsumer<T> operation
+    ) throws IgniteCheckedException, IgniteInterruptedCheckedException {
+        List<List<T>> batches = IntStream.range(0, parallelismLvl)
+            .mapToObj(i -> new ArrayList<T>())
+            .collect(Collectors.toList());
 
-                    return null;
-                })))
+        int i = 0;
+
+        for (T src : srcDatas)
+            batches.get(i++ % parallelismLvl).add(src);
+
+        List<Future<Object>> consumerFutures = batches.stream()
+            .filter(batch -> !batch.isEmpty())
+            .map(batch -> executorSvc.submit(() -> {
+                for (T item : batch)
+                    operation.accept(item);
+
+                return null;
+            }))
             .collect(Collectors.toList());
 
-        IgniteCheckedException composite = null;
+        Throwable error =null;
 
-        for (T2<T, Future<Object>> tup : consumerFutures) {
+        for (Future<Object> future : consumerFutures) {
             try {
-                getUninterruptibly(tup.get2());
+                future.get();
             }
-            catch (ExecutionException e) {
-                if (errHnd != null)
-                    errHnd.apply(tup.get1(), e.getCause());
-                else {
-                    if (composite == null)
-                        composite = new IgniteCheckedException("Failed to execute one of the tasks " +
-                            "(see suppressed exception for details)");
+            catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
 
-                    composite.addSuppressed(e.getCause());
-                }
+                throw new IgniteInterruptedCheckedException(e);
+            }
+            catch (ExecutionException e) {
+                if(error == null)
+                    error = e.getCause();
+                else
+                    error.addSuppressed(e.getCause());
+            }
+            catch (CancellationException e) {
+                if(error == null)
+                    error = e;
+                else
+                    error.addSuppressed(e);
             }
         }
 
-        if (composite != null)
-            throw composite;
+        if (error != null) {
+            if (error instanceof IgniteCheckedException)
+                throw (IgniteCheckedException)error;
 
-        return consumerFutures;
+            if (error instanceof RuntimeException)
+                throw (RuntimeException)error;
+
+            if (error instanceof Error)
+                throw (Error)error;
+
+            throw new IgniteCheckedException(error);
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java b/modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java
index e3fea1ec063e6..aad2b187fea6c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/InitializationProtector.java
@@ -67,10 +67,9 @@ public <T> T protect(Object protectedKey, Supplier<T> initializedVal,
 
     /**
      * It method allows to avoid simultaneous initialization from various threads.
-     * Garantee protection only for first call.
      *
      * @param protectedKey Unique value by which initialization code should be run only from one thread in one time.
-     * @param initializationCode Code for initialization value corresponding protectedKey.
+     * @param initializationCode Code for initialization value corresponding protectedKey. Should be idempotent.
      * @throws IgniteCheckedException if initialization was failed.
      */
     public void protect(Object protectedKey, IgniteThrowableRunner initializationCode) throws IgniteCheckedException {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java
new file mode 100644
index 0000000000000..4e30d1cb6bab9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheStartInParallelTest.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.ArrayList;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class CacheStartInParallelTest extends GridCommonAbstractTest {
+    /** */
+    private static final int CACHES_COUNT = 40;
+
+    /** */
+    private static final String STATIC_CACHE_PREFIX = "static-cache-";
+
+    /** */
+    private static final String DYNAMIC_CACHE_PREFIX = "dynamic-cache-";
+
+    /** */
+    private static boolean isStaticCache = true;
+
+    /** */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setSystemThreadPoolSize(10);
+
+        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+        discoSpi.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(discoSpi);
+
+        long sz = 100 * 1024 * 1024;
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration().setPageSize(1024)
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration().setPersistenceEnabled(true).setInitialSize(sz).setMaxSize(sz))
+            .setWalMode(WALMode.LOG_ONLY).setCheckpointFrequency(24L * 60 * 60 * 1000);
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        if (isStaticCache) {
+            ArrayList<Object> staticCaches = new ArrayList<>(CACHES_COUNT);
+
+            for (int i = 0; i < CACHES_COUNT; i++)
+                staticCaches.add(cacheConfiguration(STATIC_CACHE_PREFIX + i));
+
+            cfg.setCacheConfiguration(staticCaches.toArray(new CacheConfiguration[CACHES_COUNT]));
+        }
+
+        return cfg;
+    }
+
+    /**
+     * @param cacheName Cache name.
+     * @return Cache configuration.
+     */
+    private CacheConfiguration cacheConfiguration(String cacheName) {
+        CacheConfiguration cfg = defaultCacheConfiguration();
+
+        cfg.setName(cacheName);
+        cfg.setBackups(1);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanupTestData();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        cleanupTestData();
+    }
+
+    /** */
+    private void cleanupTestData() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL);
+
+        isStaticCache = true;
+    }
+
+    /**
+     * Checking that start static caches in parallel faster than consistenly.
+     *
+     * @throws Exception if fail.
+     */
+    public void testParallelizationAcceleratesStartOfStaticCaches() throws Exception {
+        //start caches consistently.
+        System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "false");
+
+        long startTime = System.currentTimeMillis();
+
+        IgniteEx igniteEx = startGrid(0);
+
+        igniteEx.cluster().active(true);
+
+        long totalStartTimeConsistently = System.currentTimeMillis() - startTime;
+
+        //check cache started.
+        for (int i = 0; i < CACHES_COUNT; i++)
+            igniteEx.cache(STATIC_CACHE_PREFIX + i).put(i, i);
+
+        stopAllGrids();
+
+        //start caches in parallel.
+        System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "true");
+
+        startTime = System.currentTimeMillis();
+
+        igniteEx = startGrid(0);
+
+        igniteEx.cluster().active(true);
+
+        long totalStartTimeInParallel = System.currentTimeMillis() - startTime;
+
+        for (int i = 0; i < CACHES_COUNT; i++)
+            igniteEx.cache(STATIC_CACHE_PREFIX + i).put(i, i);
+
+        stopAllGrids();
+
+        assertTrue("Consistently cache stat time : " + totalStartTimeConsistently +
+                "Parallelization cache stat time : " + totalStartTimeInParallel,
+            totalStartTimeConsistently > totalStartTimeInParallel);
+    }
+
+    /**
+     * Checking that start dynamic caches in parallel faster than consistenly.
+     *
+     * @throws Exception if fail.
+     */
+    public void testParallelizationAcceleratesStartOfCaches2() throws Exception {
+        //prepare dynamic caches.
+        isStaticCache = false;
+
+        IgniteEx igniteEx = startGrid(0);
+
+        igniteEx.cluster().active(true);
+
+        for (int i = 0; i < CACHES_COUNT; i++)
+            igniteEx.getOrCreateCache(DYNAMIC_CACHE_PREFIX + i);
+
+        stopAllGrids();
+
+        //start caches consistently.
+        System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "false");
+
+        igniteEx = startGrid(0);
+        long startTime = System.currentTimeMillis();
+
+        igniteEx.cluster().active(true);
+
+        long totalStartTimeConsistently = System.currentTimeMillis() - startTime;
+
+        for (int i = 0; i < CACHES_COUNT; i++)
+            igniteEx.cache(DYNAMIC_CACHE_PREFIX + i);
+
+        stopAllGrids();
+
+        //start caches in parallel.
+        System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, "true");
+
+        startTime = System.currentTimeMillis();
+
+        igniteEx = startGrid(0);
+
+        igniteEx.cluster().active(true);
+
+        long totalStartTimeInParallel = System.currentTimeMillis() - startTime;
+
+        for (int i = 0; i < CACHES_COUNT; i++)
+            igniteEx.cache(DYNAMIC_CACHE_PREFIX + i).put(i, i);
+
+        stopAllGrids();
+
+        assertTrue("Consistently cache stat time : " + totalStartTimeConsistently +
+                "Parallelization cache stat time : " + totalStartTimeInParallel,
+            totalStartTimeConsistently > totalStartTimeInParallel);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java
index bf5ba61beea64..8085a49166e4f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCrossCacheTxStoreSelfTest.java
@@ -17,15 +17,15 @@
 
 package org.apache.ignite.internal.processors.cache.distributed;
 
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
+import javax.cache.integration.CacheLoaderException;
+import javax.cache.integration.CacheWriterException;
 import java.util.Collection;
 import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedDeque;
-import javax.cache.Cache;
-import javax.cache.configuration.Factory;
-import javax.cache.integration.CacheLoaderException;
-import javax.cache.integration.CacheWriterException;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.store.CacheStore;
@@ -61,7 +61,7 @@ public class IgniteCrossCacheTxStoreSelfTest extends GridCommonAbstractTest {
         CacheConfiguration cfg3 = cacheConfiguration("cacheC", new SecondStoreFactory());
         CacheConfiguration cfg4 = cacheConfiguration("cacheD", null);
 
-        cfg.setCacheConfiguration(cfg1, cfg2, cfg3, cfg4);
+        cfg.setCacheConfiguration(cfg4, cfg2, cfg3, cfg1);
 
         return cfg;
     }
@@ -92,6 +92,8 @@ private CacheConfiguration cacheConfiguration(String cacheName, Factory<CacheSto
     @Override protected void beforeTestsStarted() throws Exception {
         super.beforeTestsStarted();
 
+        cleanPersistenceDir();
+
         startGrids(4);
     }
 
@@ -162,8 +164,8 @@ public void testSameStore() throws Exception {
             "delete cacheA",
             "write cacheB",
             "sessionEnd true"
-        ),
-        firstStoreEvts);
+            ),
+            firstStoreEvts);
 
         assertEquals(0, secondStoreEvts.size());
     }
@@ -211,16 +213,16 @@ public void testDifferentStores() throws Exception {
             "write cacheA",
             "delete cacheA",
             "sessionEnd true"
-        ),
-        firstStoreEvts);
+            ),
+            firstStoreEvts);
 
         assertEqualsCollections(F.asList(
             "writeAll cacheC 2",
             "deleteAll cacheC 2",
             "write cacheC",
             "sessionEnd true"
-        ),
-        secondStoreEvts);
+            ),
+            secondStoreEvts);
     }
 
     /**
@@ -266,8 +268,8 @@ public void testNonPersistentCache() throws Exception {
             "write cacheA",
             "delete cacheA",
             "sessionEnd true"
-        ),
-        firstStoreEvts);
+            ),
+            firstStoreEvts);
 
         assertEquals(0, secondStoreEvts.size());
     }
@@ -363,15 +365,10 @@ private static class FirstStoreFactory implements Factory<CacheStore> {
         private Ignite ignite;
 
         /** {@inheritDoc} */
-        @Override public CacheStore create() {
+        @Override public synchronized CacheStore create() {
             String igniteInstanceName = ignite.name();
 
-            CacheStore store = firstStores.get(igniteInstanceName);
-
-            if (store == null)
-                store = F.addIfAbsent(firstStores, igniteInstanceName, new TestStore());
-
-            return store;
+            return firstStores.computeIfAbsent(igniteInstanceName, (k) -> new TestStore());
         }
     }
 
@@ -386,12 +383,7 @@ private static class SecondStoreFactory implements Factory<CacheStore> {
         @Override public CacheStore create() {
             String igniteInstanceName = ignite.name();
 
-            CacheStore store = secondStores.get(igniteInstanceName);
-
-            if (store == null)
-                store = F.addIfAbsent(secondStores, igniteInstanceName, new TestStore());
-
-            return store;
+            return secondStores.computeIfAbsent(igniteInstanceName, (k) -> new TestStore());
         }
     }
 }
\ No newline at end of file
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
index 963c1d94de230..3d4a425850601 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
@@ -44,6 +44,10 @@
 import java.util.List;
 import java.util.Random;
 import java.util.UUID;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterGroup;
@@ -847,6 +851,76 @@ public void testCeilPow2() throws Exception {
             assertEquals(0, U.ceilPow2(i));
     }
 
+    /**
+     *
+     */
+    public void testDoInParallel() throws Throwable {
+        CyclicBarrier barrier = new CyclicBarrier(3);
+
+        IgniteUtils.doInParallel(3,
+            Executors.newFixedThreadPool(3),
+            Arrays.asList(1, 2, 3),
+            i -> {
+                try {
+                    barrier.await(1, TimeUnit.SECONDS);
+                }
+                catch (Exception e) {
+                    throw new IgniteCheckedException(e);
+                }
+            }
+        );
+    }
+
+    /**
+     *
+     */
+    public void testDoInParallelBatch() {
+        CyclicBarrier barrier = new CyclicBarrier(3);
+
+        try {
+            IgniteUtils.doInParallel(2,
+                Executors.newFixedThreadPool(3),
+                Arrays.asList(1, 2, 3),
+                i -> {
+                    try {
+                        barrier.await(400, TimeUnit.MILLISECONDS);
+                    }
+                    catch (Exception e) {
+                        throw new IgniteCheckedException(e);
+                    }
+                }
+            );
+
+            fail("Should throw timeout exception");
+        }
+        catch (Exception e) {
+            assertTrue(e.getCause() instanceof TimeoutException);
+        }
+    }
+
+    /**
+     *
+     */
+    public void testDoInParallelException() {
+        String expectedException = "ExpectedException";
+
+        try {
+            IgniteUtils.doInParallel(3,
+                Executors.newFixedThreadPool(1),
+                Arrays.asList(1, 2, 3),
+                i -> {
+                    if (i == 1)
+                        throw new IgniteCheckedException(expectedException);
+                }
+            );
+
+            fail("Should throw ParallelExecutionException");
+        }
+        catch (IgniteCheckedException e) {
+            assertEquals(expectedException, e.getMessage());
+        }
+    }
+
     /**
      * Test enum.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
index b2cb6d6d9a714..4ab6b6690f697 100755
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite7.java
@@ -35,6 +35,8 @@
 import org.apache.ignite.internal.processors.cache.distributed.Cache64kPartitionsTest;
 import org.apache.ignite.internal.processors.cache.distributed.CachePageWriteLockUnlockTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheRentingStateRepairTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheStartInParallelTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheDataLossOnPartitionMoveTest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheStartWithLoadTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingPartitionCountersTest;
@@ -99,6 +101,8 @@ public static TestSuite suite(Set<Class> ignoredTests) throws Exception {
 
         suite.addTestSuite(CacheRentingStateRepairTest.class);
 
+        suite.addTestSuite(CacheStartInParallelTest.class);
+
         suite.addTestSuite(TransactionIntegrityWithPrimaryIndexCorruptionTest.class);
         suite.addTestSuite(CacheDataLossOnPartitionMoveTest.class);
 

From 0de7bec33047b26edd5f393b9c1503e745435f7f Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Mon, 22 Oct 2018 18:19:53 +0300
Subject: [PATCH 444/543] IGNITE-9738 Client node can suddenly fail on start -
 Fixes #4968.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
(cherry picked from commit d82b21ec56a956fa7cc5374e3f15e279e7c492ac)
---
 .../ignite/spi/discovery/tcp/ClientImpl.java  |  10 +-
 .../LongClientConnectToClusterTest.java       | 173 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   2 +
 3 files changed, 180 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/LongClientConnectToClusterTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
index 1a3e33837fa42..b2d80918a18b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ClientImpl.java
@@ -303,6 +303,11 @@ class ClientImpl extends TcpDiscoveryImpl {
             }
         }.start();
 
+        timer.schedule(
+            new MetricsSender(),
+            spi.metricsUpdateFreq,
+            spi.metricsUpdateFreq);
+
         try {
             joinLatch.await();
 
@@ -315,11 +320,6 @@ class ClientImpl extends TcpDiscoveryImpl {
             throw new IgniteSpiException("Thread has been interrupted.", e);
         }
 
-        timer.schedule(
-            new MetricsSender(),
-            spi.metricsUpdateFreq,
-            spi.metricsUpdateFreq);
-
         spi.printStartInfo();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/LongClientConnectToClusterTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/LongClientConnectToClusterTest.java
new file mode 100644
index 0000000000000..95b1f502f606b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/LongClientConnectToClusterTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.future.IgniteFinishedFutureImpl;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Test client connects to two nodes cluster during time more than the
+ * {@link org.apache.ignite.configuration.IgniteConfiguration#clientFailureDetectionTimeout}.
+ */
+public class LongClientConnectToClusterTest extends GridCommonAbstractTest {
+    /** Client instance name. */
+    public static final String CLIENT_INSTANCE_NAME = "client";
+    /** Client metrics update count. */
+    private static volatile int clientMetricsUpdateCnt;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        TcpDiscoverySpi discoSpi = getTestIgniteInstanceName(0).equals(igniteInstanceName)
+            ? new DelayedTcpDiscoverySpi()
+            : getTestIgniteInstanceName(1).equals(igniteInstanceName)
+            ? new UpdateMetricsInterceptorTcpDiscoverySpi()
+            : new TcpDiscoverySpi();
+
+        return super.getConfiguration(igniteInstanceName)
+            .setClientMode(igniteInstanceName.startsWith(CLIENT_INSTANCE_NAME))
+            .setClientFailureDetectionTimeout(1_000)
+            .setMetricsUpdateFrequency(500)
+            .setDiscoverySpi(discoSpi
+                .setReconnectCount(1)
+                .setLocalAddress("127.0.0.1")
+                .setIpFinder(new TcpDiscoveryVmIpFinder()
+                    .setAddresses(Collections.singletonList(igniteInstanceName.startsWith(CLIENT_INSTANCE_NAME)
+                        ? "127.0.0.1:47501"
+                        : "127.0.0.1:47500..47502"))));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        startGrids(2);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /**
+     * Test method.
+     *
+     * @throws Exception If failed.
+     */
+    public void testClientConnectToCluster() throws Exception {
+        clientMetricsUpdateCnt = 0;
+
+        IgniteEx client = (IgniteEx)startGrid(CLIENT_INSTANCE_NAME);
+
+        assertTrue(clientMetricsUpdateCnt > 0);
+
+        assertTrue(client.localNode().isClient());
+
+        assertEquals(client.cluster().nodes().size(), 3);
+    }
+
+    /** Discovery SPI which intercept TcpDiscoveryClientMetricsUpdateMessage. */
+    private static class UpdateMetricsInterceptorTcpDiscoverySpi extends TcpDiscoverySpi {
+        /** */
+        private class DiscoverySpiListenerWrapper implements DiscoverySpiListener {
+            /** */
+            private DiscoverySpiListener delegate;
+
+            /**
+             * @param delegate Delegate.
+             */
+            private DiscoverySpiListenerWrapper(DiscoverySpiListener delegate) {
+                this.delegate = delegate;
+            }
+
+            /** {@inheritDoc} */
+            @Override public IgniteFuture<?> onDiscovery(
+                int type,
+                long topVer,
+                ClusterNode node,
+                Collection<ClusterNode> topSnapshot,
+                @Nullable Map<Long, Collection<ClusterNode>> topHist,
+                @Nullable DiscoverySpiCustomMessage spiCustomMsg
+            ) {
+                if (EventType.EVT_NODE_METRICS_UPDATED == type) {
+                    log.info("Metrics update message catched from node " + node);
+
+                    assertFalse(locNode.isClient());
+
+                    if (node.isClient())
+                        clientMetricsUpdateCnt++;
+                }
+
+                if (delegate != null)
+                    return delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, spiCustomMsg);
+
+                return new IgniteFinishedFutureImpl<>();
+            }
+
+            /** {@inheritDoc} */
+            @Override public void onLocalNodeInitialized(ClusterNode locNode) {
+                if (delegate != null)
+                    delegate.onLocalNodeInitialized(locNode);
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public void setListener(@Nullable DiscoverySpiListener lsnr) {
+            super.setListener(new DiscoverySpiListenerWrapper(lsnr));
+        }
+    }
+
+    /** Discovery SPI delayed TcpDiscoveryNodeAddFinishedMessage. */
+    private static class DelayedTcpDiscoverySpi extends TcpDiscoverySpi {
+        /** Delay message period millis. */
+        public static final int DELAY_MSG_PERIOD_MILLIS = 2_000;
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(ClusterNode node, Socket sock, OutputStream out,
+            TcpDiscoveryAbstractMessage msg, long timeout) throws IOException, IgniteCheckedException {
+            if (msg instanceof TcpDiscoveryNodeAddFinishedMessage && msg.topologyVersion() == 3) {
+                log.info("Catched discovery message: " + msg);
+
+                try {
+                    Thread.sleep(DELAY_MSG_PERIOD_MILLIS);
+                }
+                catch (InterruptedException e) {
+                    log.error("Interrupt on DelayedTcpDiscoverySpi.", e);
+
+                    Thread.currentThread().interrupt();
+                }
+            }
+
+            super.writeToSocket(node, sock, out, msg, timeout);
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index a2eb3ac2f9e7a..230fea5794574 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -22,6 +22,7 @@
 import org.apache.ignite.spi.discovery.AuthenticationRestartTest;
 import org.apache.ignite.spi.discovery.FilterDataForClientNodeDiscoveryTest;
 import org.apache.ignite.spi.discovery.IgniteDiscoveryCacheReuseSelfTest;
+import org.apache.ignite.spi.discovery.LongClientConnectToClusterTest;
 import org.apache.ignite.spi.discovery.tcp.DiscoveryUnmarshalVulnerabilityTest;
 import org.apache.ignite.spi.discovery.tcp.IgniteClientConnectTest;
 import org.apache.ignite.spi.discovery.tcp.IgniteClientReconnectMassiveShutdownTest;
@@ -90,6 +91,7 @@ public static TestSuite suite() throws Exception {
         suite.addTest(new TestSuite(GridTcpSpiForwardingSelfTest.class));
 
         suite.addTest(new TestSuite(TcpClientDiscoverySpiSelfTest.class));
+        suite.addTest(new TestSuite(LongClientConnectToClusterTest.class));
         suite.addTest(new TestSuite(TcpClientDiscoveryMarshallerCheckSelfTest.class));
         suite.addTest(new TestSuite(TcpClientDiscoverySpiMulticastTest.class));
         suite.addTest(new TestSuite(TcpClientDiscoverySpiFailureTimeoutSelfTest.class));

From d54d38235610b9a6d6f127f077cf16243e70407f Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Tue, 23 Oct 2018 17:56:56 +0300
Subject: [PATCH 445/543] Revert "IGNITE-5795 Register binary metadata during
 cache start - Fixes #4852."

This reverts commit ba585c5
---
 .../apache/ignite/internal/IgniteKernal.java  |   2 +-
 .../binary/BinaryCachingMetadataHandler.java  |  25 +-
 .../ignite/internal/binary/BinaryContext.java |  59 +---
 .../binary/BinaryMetadataHandler.java         |  10 -
 .../binary/BinaryNoopMetadataHandler.java     |   6 -
 .../builder/BinaryObjectBuilderImpl.java      |   2 +-
 .../internal/client/thin/TcpIgniteClient.java |   6 -
 .../processors/cache/GridCacheProcessor.java  |   9 +-
 .../CacheObjectBinaryProcessorImpl.java       |   5 -
 .../processors/query/GridQueryProcessor.java  |  77 +----
 .../binary/TestCachingMetadataHandler.java    |   6 -
 .../CacheRegisterMetadataLocallyTest.java     | 287 ------------------
 .../cache/index/AbstractSchemaSelfTest.java   |   6 +-
 .../index/H2DynamicIndexAbstractSelfTest.java |  48 ++-
 .../IgniteCacheWithIndexingTestSuite.java     |   2 -
 15 files changed, 56 insertions(+), 494 deletions(-)
 delete mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index f39b14022feee..7ea3271633dc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -998,7 +998,6 @@ public void start(
             // Start processors before discovery manager, so they will
             // be able to start receiving messages once discovery completes.
             try {
-                startProcessor(new GridMarshallerMappingProcessor(ctx));
                 startProcessor(new PdsConsistentIdProcessor(ctx));
                 startProcessor(createComponent(DiscoveryNodeValidationProcessor.class, ctx));
                 startProcessor(new GridAffinityProcessor(ctx));
@@ -1021,6 +1020,7 @@ public void start(
                 startProcessor(createHadoopComponent());
                 startProcessor(new DataStructuresProcessor(ctx));
                 startProcessor(createComponent(PlatformProcessor.class, ctx));
+                startProcessor(new GridMarshallerMappingProcessor(ctx));
 
                 // Start plugins.
                 for (PluginProvider provider : ctx.plugins().allProviders()) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java
index b60dc097aaa1e..a0559cbdc8ba1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryCachingMetadataHandler.java
@@ -46,28 +46,23 @@ private BinaryCachingMetadataHandler() {
     }
 
     /** {@inheritDoc} */
-    @Override public synchronized void addMeta(int typeId, BinaryType type,
-        boolean failIfUnregistered) throws BinaryObjectException {
-        BinaryType oldType = metas.put(typeId, type);
+    @Override public synchronized void addMeta(int typeId, BinaryType type, boolean failIfUnregistered) throws BinaryObjectException {
+        synchronized (this) {
+            BinaryType oldType = metas.put(typeId, type);
 
-        if (oldType != null) {
-            BinaryMetadata oldMeta = ((BinaryTypeImpl)oldType).metadata();
-            BinaryMetadata newMeta = ((BinaryTypeImpl)type).metadata();
+            if (oldType != null) {
+                BinaryMetadata oldMeta = ((BinaryTypeImpl)oldType).metadata();
+                BinaryMetadata newMeta = ((BinaryTypeImpl)type).metadata();
 
-            BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta);
+                BinaryMetadata mergedMeta = BinaryUtils.mergeMetadata(oldMeta, newMeta);
 
-            BinaryType mergedType = mergedMeta.wrap(((BinaryTypeImpl)oldType).context());
+                BinaryType mergedType = mergedMeta.wrap(((BinaryTypeImpl)oldType).context());
 
-            metas.put(typeId, mergedType);
+                metas.put(typeId, mergedType);
+            }
         }
     }
 
-    /** {@inheritDoc} */
-    @Override public synchronized void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered)
-        throws BinaryObjectException {
-        addMeta(typeId, meta, failIfUnregistered);
-    }
-
     /** {@inheritDoc} */
     @Override public synchronized BinaryType metadata(int typeId) throws BinaryObjectException {
         return metas.get(typeId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
index 7ab74e09b82d0..7885d9575f32b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryContext.java
@@ -617,18 +617,6 @@ else if (cpElement.isFile()) {
      */
     public BinaryClassDescriptor descriptorForClass(Class<?> cls, boolean deserialize, boolean failIfUnregistered)
         throws BinaryObjectException {
-        return descriptorForClass(cls, deserialize, failIfUnregistered, false);
-    }
-
-    /**
-     * @param cls Class.
-     * @param failIfUnregistered Throw exception if class isn't registered.
-     * @param onlyLocReg {@code true} if descriptor need to register only locally when registration is required at all.
-     * @return Class descriptor.
-     * @throws BinaryObjectException In case of error.
-     */
-    public BinaryClassDescriptor descriptorForClass(Class<?> cls, boolean deserialize, boolean failIfUnregistered,
-        boolean onlyLocReg) throws BinaryObjectException {
         assert cls != null;
 
         BinaryClassDescriptor desc = descByCls.get(cls);
@@ -637,7 +625,7 @@ public BinaryClassDescriptor descriptorForClass(Class<?> cls, boolean deserializ
             if (failIfUnregistered)
                 throw new UnregisteredClassException(cls);
 
-            desc = registerClassDescriptor(cls, deserialize, onlyLocReg);
+            desc = registerClassDescriptor(cls, deserialize);
         }
         else if (!desc.registered()) {
             if (!desc.userType()) {
@@ -674,7 +662,7 @@ else if (!desc.registered()) {
                 if (failIfUnregistered)
                     throw new UnregisteredClassException(cls);
 
-                desc = registerUserClassDescriptor(desc, onlyLocReg);
+                desc = registerUserClassDescriptor(desc);
             }
         }
 
@@ -727,7 +715,7 @@ public BinaryClassDescriptor descriptorForTypeId(
         }
 
         if (desc == null) {
-            desc = registerClassDescriptor(cls, deserialize, false);
+            desc = registerClassDescriptor(cls, deserialize);
 
             assert desc.typeId() == typeId : "Duplicate typeId [typeId=" + typeId + ", cls=" + cls
                 + ", desc=" + desc + "]";
@@ -740,10 +728,9 @@ public BinaryClassDescriptor descriptorForTypeId(
      * Creates and registers {@link BinaryClassDescriptor} for the given {@code class}.
      *
      * @param cls Class.
-     * @param onlyLocReg {@code true} if descriptor need to register only locally when registration is required at all.
      * @return Class descriptor.
      */
-    private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean deserialize, boolean onlyLocReg) {
+    private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean deserialize) {
         BinaryClassDescriptor desc;
 
         String clsName = cls.getName();
@@ -772,7 +759,7 @@ private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean dese
                 desc = old;
         }
         else
-            desc = registerUserClassDescriptor(cls, deserialize, onlyLocReg);
+            desc = registerUserClassDescriptor(cls, deserialize);
 
         return desc;
     }
@@ -781,10 +768,9 @@ private BinaryClassDescriptor registerClassDescriptor(Class<?> cls, boolean dese
      * Creates and registers {@link BinaryClassDescriptor} for the given user {@code class}.
      *
      * @param cls Class.
-     * @param onlyLocReg {@code true} if descriptor need to register only locally.
      * @return Class descriptor.
      */
-    private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean deserialize, boolean onlyLocReg) {
+    private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean deserialize) {
         boolean registered;
 
         final String clsName = cls.getName();
@@ -795,7 +781,7 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean
 
         final int typeId = mapper.typeId(clsName);
 
-        registered = registerUserClassName(typeId, cls.getName(), onlyLocReg);
+        registered = registerUserClassName(typeId, cls.getName());
 
         BinarySerializer serializer = serializerForClass(cls);
 
@@ -813,22 +799,9 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean
             registered
         );
 
-        if (!deserialize) {
-            BinaryMetadata binaryMetadata = new BinaryMetadata(
-                typeId,
-                typeName,
-                desc.fieldsMeta(),
-                affFieldName,
-                null,
-                desc.isEnum(),
-                cls.isEnum() ? enumMap(cls) : null
-            );
-
-            if (onlyLocReg)
-                metaHnd.addMetaLocally(typeId, binaryMetadata.wrap(this), false);
-            else
-                metaHnd.addMeta(typeId, binaryMetadata.wrap(this), false);
-        }
+        if (!deserialize)
+            metaHnd.addMeta(typeId, new BinaryMetadata(typeId, typeName, desc.fieldsMeta(), affFieldName, null,
+                desc.isEnum(), cls.isEnum() ? enumMap(cls) : null).wrap(this), false);
 
         descByCls.put(cls, desc);
 
@@ -841,13 +814,12 @@ private BinaryClassDescriptor registerUserClassDescriptor(Class<?> cls, boolean
      * Creates and registers {@link BinaryClassDescriptor} for the given user {@code class}.
      *
      * @param desc Old descriptor that should be re-registered.
-     * @param onlyLocReg {@code true} if descriptor need to register only locally.
      * @return Class descriptor.
      */
-    private BinaryClassDescriptor registerUserClassDescriptor(BinaryClassDescriptor desc, boolean onlyLocReg) {
+    private BinaryClassDescriptor registerUserClassDescriptor(BinaryClassDescriptor desc) {
         boolean registered;
 
-        registered = registerUserClassName(desc.typeId(), desc.describedClass().getName(), onlyLocReg);
+        registered = registerUserClassName(desc.typeId(), desc.describedClass().getName());
 
         if (registered) {
             BinarySerializer serializer = desc.initialSerializer();
@@ -1219,18 +1191,15 @@ public void registerUserTypesSchema() {
      *
      * @param typeId Type ID.
      * @param clsName Class Name.
-     * @param onlyLocReg {@code true} if descriptor need to register only locally.
      * @return {@code True} if the mapping was registered successfully.
      */
-    public boolean registerUserClassName(int typeId, String clsName, boolean onlyLocReg) {
+    public boolean registerUserClassName(int typeId, String clsName) {
         IgniteCheckedException e = null;
 
         boolean res = false;
 
         try {
-            res = onlyLocReg
-                ? marshCtx.registerClassNameLocally(JAVA_ID, typeId, clsName)
-                : marshCtx.registerClassName(JAVA_ID, typeId, clsName);
+            res = marshCtx.registerClassName(JAVA_ID, typeId, clsName);
         }
         catch (DuplicateTypeIdException dupEx) {
             // Ignore if trying to register mapped type name of the already registered class name and vise versa
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java
index d1336bf6ae4a0..85ab1372f49f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryMetadataHandler.java
@@ -35,16 +35,6 @@ public interface BinaryMetadataHandler {
      */
     public void addMeta(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException;
 
-    /**
-     * Adds meta data locally on current node without sending any messages.
-     *
-     * @param typeId Type ID.
-     * @param meta Metadata.
-     * @param failIfUnregistered Fail if unregistered.
-     * @throws BinaryObjectException In case of error.
-     */
-    public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered) throws BinaryObjectException;
-
     /**
      * Gets meta data for provided type ID.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java
index a552d611ea990..4ee24285c7ee5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/BinaryNoopMetadataHandler.java
@@ -47,12 +47,6 @@ private BinaryNoopMetadataHandler() {
         // No-op.
     }
 
-    /** {@inheritDoc} */
-    @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered)
-        throws BinaryObjectException {
-        // No-op.
-    }
-
     /** {@inheritDoc} */
     @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
         return null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
index f860564844056..8fffd71c1b230 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/binary/builder/BinaryObjectBuilderImpl.java
@@ -364,7 +364,7 @@ else if (readCache == null) {
                 if (affFieldName0 == null)
                     affFieldName0 = ctx.affinityKeyFieldName(typeId);
 
-                ctx.registerUserClassName(typeId, typeName, false);
+                ctx.registerUserClassName(typeId, typeName);
 
                 ctx.updateMetadata(typeId, new BinaryMetadata(typeId, typeName, fieldsMeta, affFieldName0,
                     Collections.singleton(curSchema), false, null), writer.failIfUnregistered());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java
index a108347be4a35..7a0bc7a1f8160 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/thin/TcpIgniteClient.java
@@ -249,12 +249,6 @@ private class ClientBinaryMetadataHandler implements BinaryMetadataHandler {
             cache.addMeta(typeId, meta, failIfUnregistered); // merge
         }
 
-        /** {@inheritDoc} */
-        @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered)
-            throws BinaryObjectException {
-            throw new UnsupportedOperationException("Can't register metadata locally for thin client.");
-        }
-
         /** {@inheritDoc} */
         @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
             BinaryType meta = cache.metadata(typeId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index a536e68621c02..be2a698fa1909 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -3829,13 +3829,8 @@ else if (msg0 instanceof WalStateFinishMessage)
             return msg0.needExchange();
         }
 
-        if (msg instanceof DynamicCacheChangeBatch) {
-            boolean changeRequested = cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer);
-
-            ctx.query().onCacheChangeRequested((DynamicCacheChangeBatch)msg);
-
-            return changeRequested;
-        }
+        if (msg instanceof DynamicCacheChangeBatch)
+            return cachesInfo.onCacheChangeRequested((DynamicCacheChangeBatch)msg, topVer);
 
         if (msg instanceof DynamicCacheChangeFailureMessage)
             cachesInfo.onCacheChangeRequested((DynamicCacheChangeFailureMessage)msg, topVer);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
index f3078cb18df80..137db9f887dc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/binary/CacheObjectBinaryProcessorImpl.java
@@ -207,11 +207,6 @@ public CacheObjectBinaryProcessorImpl(GridKernalContext ctx) {
                     CacheObjectBinaryProcessorImpl.this.addMeta(typeId, newMeta0.wrap(binaryCtx), failIfUnregistered);
                 }
 
-                @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered)
-                    throws BinaryObjectException {
-                    CacheObjectBinaryProcessorImpl.this.addMetaLocally(typeId, meta);
-                }
-
                 @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
                     return CacheObjectBinaryProcessorImpl.this.metadata(typeId);
                 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index f2b277585fd12..7300393f599ee 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -17,8 +17,6 @@
 
 package org.apache.ignite.internal.processors.query;
 
-import javax.cache.Cache;
-import javax.cache.CacheException;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -36,6 +34,8 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicBoolean;
+import javax.cache.Cache;
+import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.IgniteException;
@@ -61,19 +61,15 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheObjectContext;
-import org.apache.ignite.internal.processors.cache.DynamicCacheChangeBatch;
-import org.apache.ignite.internal.processors.cache.DynamicCacheChangeRequest;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.query.CacheQueryFuture;
 import org.apache.ignite.internal.processors.cache.query.CacheQueryType;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
-import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
 import org.apache.ignite.internal.processors.query.property.QueryBinaryProperty;
 import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheFilter;
 import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor;
@@ -255,8 +251,6 @@ public GridQueryProcessor(GridKernalContext ctx) throws IgniteCheckedException {
                     ctxs.queries().evictDetailMetrics();
             }
         }, QRY_DETAIL_METRICS_EVICTION_FREQ, QRY_DETAIL_METRICS_EVICTION_FREQ);
-
-        registerMetadataForRegisteredCaches();
     }
 
     /** {@inheritDoc} */
@@ -903,73 +897,6 @@ public void skipFieldLookup(boolean skipFieldLookup) {
         this.skipFieldLookup = skipFieldLookup;
     }
 
-    /**
-     * Register metadata locally for already registered caches.
-     */
-    private void registerMetadataForRegisteredCaches() {
-        for (DynamicCacheDescriptor cacheDescriptor : ctx.cache().cacheDescriptors().values()) {
-            registerBinaryMetadata(cacheDescriptor.cacheConfiguration(), cacheDescriptor.schema());
-        }
-    }
-
-    /**
-     * Handle of cache change request.
-     *
-     * @param batch Dynamic cache change batch request.
-     */
-    public void onCacheChangeRequested(DynamicCacheChangeBatch batch) {
-        for (DynamicCacheChangeRequest req : batch.requests()) {
-            if (!req.start())
-                continue;
-
-            registerBinaryMetadata(req.startCacheConfiguration(), req.schema());
-        }
-    }
-
-    /**
-     * Register binary metadata locally.
-     *
-     * @param ccfg Cache configuration.
-     * @param schema Schema for which register metadata is required.
-     */
-    private void registerBinaryMetadata(CacheConfiguration ccfg, QuerySchema schema) {
-        if (schema != null) {
-            Collection<QueryEntity> qryEntities = schema.entities();
-
-            if (!F.isEmpty(qryEntities)) {
-                boolean binaryEnabled = ctx.cacheObjects().isBinaryEnabled(ccfg);
-
-                if (binaryEnabled) {
-                    for (QueryEntity qryEntity : qryEntities) {
-                        Class<?> keyCls = U.box(U.classForName(qryEntity.findKeyType(), null, true));
-                        Class<?> valCls = U.box(U.classForName(qryEntity.findValueType(), null, true));
-
-                        if (keyCls != null)
-                            registerDescriptorLocallyIfNeeded(keyCls);
-
-                        if (valCls != null)
-                            registerDescriptorLocallyIfNeeded(valCls);
-                    }
-                }
-            }
-        }
-    }
-
-    /**
-     * Register class metadata locally if it didn't do it earlier.
-     *
-     * @param cls Class for which the metadata should be registered.
-     */
-    private void registerDescriptorLocallyIfNeeded(Class<?> cls) {
-        IgniteCacheObjectProcessor cacheObjProc = ctx.cacheObjects();
-
-        if (cacheObjProc instanceof CacheObjectBinaryProcessorImpl) {
-            ((CacheObjectBinaryProcessorImpl)cacheObjProc)
-                .binaryContext()
-                .descriptorForClass(cls, false, false, true);
-        }
-    }
-
     /**
      * Handle custom discovery message.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java b/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java
index 47138ddeb0806..c515f8191766a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/binary/TestCachingMetadataHandler.java
@@ -38,12 +38,6 @@ public class TestCachingMetadataHandler implements BinaryMetadataHandler {
                 TestCachingMetadataHandler.class.getSimpleName() + '.');
     }
 
-    /** {@inheritDoc} */
-    @Override public void addMetaLocally(int typeId, BinaryType meta, boolean failIfUnregistered)
-        throws BinaryObjectException {
-        addMeta(typeId, meta, failIfUnregistered);
-    }
-
     /** {@inheritDoc} */
     @Override public BinaryType metadata(int typeId) throws BinaryObjectException {
         return metas.get(typeId);
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java
deleted file mode 100644
index d4066c2f546f6..0000000000000
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/CacheRegisterMetadataLocallyTest.java
+++ /dev/null
@@ -1,287 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache;
-
-import java.util.Collections;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.binary.BinaryType;
-import org.apache.ignite.cache.QueryEntity;
-import org.apache.ignite.cache.affinity.AffinityKeyMapped;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.managers.communication.GridIoMessage;
-import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
-import org.apache.ignite.internal.processors.cache.binary.MetadataRequestMessage;
-import org.apache.ignite.internal.processors.cache.binary.MetadataResponseMessage;
-import org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage;
-import org.apache.ignite.lang.IgniteInClosure;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.spi.IgniteSpiException;
-import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
-import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Tests, that binary metadata is registered correctly during the start without extra request to grid.
- */
-public class CacheRegisterMetadataLocallyTest extends GridCommonAbstractTest {
-    /** */
-    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
-    /** */
-    private static final String STATIC_CACHE_NAME = "staticCache";
-
-    /** */
-    private static final String DYNAMIC_CACHE_NAME = "dynamicCache";
-
-    /** Holder of sent custom messages. */
-    private final ConcurrentLinkedQueue<Object> customMessages = new ConcurrentLinkedQueue<>();
-
-    /** Holder of sent communication messages. */
-    private final ConcurrentLinkedQueue<Object> communicationMessages = new ConcurrentLinkedQueue<>();
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
-        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
-
-        cfg.setDiscoverySpi(new TcpDiscoverySpi() {
-            @Override public void sendCustomEvent(DiscoverySpiCustomMessage msg) throws IgniteException {
-                if (msg instanceof CustomMessageWrapper)
-                    customMessages.add(((CustomMessageWrapper)msg).delegate());
-                else
-                    customMessages.add(msg);
-
-                super.sendCustomEvent(msg);
-            }
-        });
-
-        cfg.setCommunicationSpi(new TcpCommunicationSpi() {
-            @Override public void sendMessage(ClusterNode node, Message msg,
-                IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
-                if (msg instanceof GridIoMessage)
-                    communicationMessages.add(((GridIoMessage)msg).message());
-
-                super.sendMessage(node, msg, ackC);
-            }
-
-            @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
-                if (msg instanceof GridIoMessage)
-                    communicationMessages.add(((GridIoMessage)msg).message());
-
-                super.sendMessage(node, msg);
-            }
-        });
-
-        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
-
-        if (igniteInstanceName.equals("client"))
-            cfg.setClientMode(true);
-
-        cfg.setCacheConfiguration(cacheConfiguration(STATIC_CACHE_NAME, StaticKey.class, StaticValue.class));
-
-        return cfg;
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTest() throws Exception {
-        stopAllGrids();
-
-        cleanPersistenceDir();
-
-        customMessages.clear();
-        communicationMessages.clear();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testAffinityKeyRegisteredStaticCache() throws Exception {
-        Ignite ignite = startGrid();
-
-        assertEquals("affKey", getAffinityKey(ignite, StaticKey.class));
-        assertEquals("affKey", getAffinityKey(ignite, StaticValue.class));
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testAffinityKeyRegisteredDynamicCache() throws Exception {
-        Ignite ignite = startGrid();
-
-        ignite.createCache(cacheConfiguration(DYNAMIC_CACHE_NAME, DynamicKey.class, DynamicValue.class));
-
-        assertEquals("affKey", getAffinityKey(ignite, DynamicKey.class));
-        assertEquals("affKey", getAffinityKey(ignite, DynamicValue.class));
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testClientFindsValueByAffinityKeyStaticCacheWithoutExtraRequest() throws Exception {
-        Ignite srv = startGrid();
-        IgniteCache<StaticKey, StaticValue> cache = srv.cache(STATIC_CACHE_NAME);
-
-        testClientFindsValueByAffinityKey(cache, new StaticKey(1), new StaticValue(2));
-
-        assertCustomMessages(2); //MetadataUpdateProposedMessage for update schema.
-        assertCommunicationMessages();
-    }
-
-    /**
-     * @throws Exception If failed.
-     */
-    public void testClientFindsValueByAffinityKeyDynamicCacheWithoutExtraRequest() throws Exception {
-        Ignite srv = startGrid();
-        IgniteCache<DynamicKey, DynamicValue> cache =
-            srv.createCache(cacheConfiguration(DYNAMIC_CACHE_NAME, DynamicKey.class, DynamicValue.class));
-
-        testClientFindsValueByAffinityKey(cache, new DynamicKey(3), new DynamicValue(4));
-
-        //Expected only DynamicCacheChangeBatch for start cache and MetadataUpdateProposedMessage for update schema.
-        assertCustomMessages(3);
-        assertCommunicationMessages();
-    }
-
-    /**
-     * @param ignite Ignite instance.
-     * @param keyCls Key class.
-     * @return Name of affinity key field of the given class.
-     */
-    private <K> String getAffinityKey(Ignite ignite, Class<K> keyCls) {
-        BinaryType binType = ignite.binary().type(keyCls);
-
-        return binType.affinityKeyFieldName();
-    }
-
-    /**
-     * @param cache Cache instance.
-     * @param key Test key.
-     * @param val Test value.
-     * @throws Exception If failed.
-     */
-    private <K, V> void testClientFindsValueByAffinityKey(IgniteCache<K, V> cache, K key, V val) throws Exception {
-        cache.put(key, val);
-
-        assertTrue(cache.containsKey(key));
-
-        Ignite client = startGrid("client");
-
-        IgniteCache<K, V> clientCache = client.cache(cache.getName());
-
-        assertTrue(clientCache.containsKey(key));
-    }
-
-    /**
-     * @param name Cache name.
-     * @param keyCls Key {@link Class}.
-     * @param valCls Value {@link Class}.
-     * @param <K> Key type.
-     * @param <V> Value type.
-     * @return Cache configuration
-     */
-    private static <K, V> CacheConfiguration<K, V> cacheConfiguration(String name, Class<K> keyCls, Class<V> valCls) {
-        CacheConfiguration<K, V> cfg = new CacheConfiguration<>(name);
-        cfg.setQueryEntities(Collections.singleton(new QueryEntity(keyCls, valCls)));
-        return cfg;
-    }
-
-    /**
-     * Expecting that "proposed binary metadata"( {@link org.apache.ignite.internal.processors.marshaller.MappingProposedMessage},
-     * {@link org.apache.ignite.internal.processors.cache.binary.MetadataUpdateProposedMessage}) will be skipped because
-     * it should be register locally during the start.
-     *
-     * @param expMsgCnt Count of expected messages.
-     */
-    private void assertCustomMessages(int expMsgCnt) {
-        assertEquals(customMessages.toString(), expMsgCnt, customMessages.size());
-
-        customMessages.forEach(cm -> assertTrue(cm.toString(), cm instanceof DynamicCacheChangeBatch || cm instanceof MetadataUpdateProposedMessage));
-    }
-
-    /**
-     * Expecting that extra request to binary metadata( {@link MetadataRequestMessage}, {@link MetadataResponseMessage})
-     * will be skipped because it should be register locally during the start.
-     */
-    private void assertCommunicationMessages() {
-        communicationMessages.forEach(cm ->
-            assertFalse(cm.toString(), cm instanceof MetadataRequestMessage || cm instanceof MetadataResponseMessage)
-        );
-    }
-
-    /** */
-    private static class StaticKey {
-        /** */
-        @AffinityKeyMapped
-        private int affKey;
-
-        /**
-         * @param affKey Affinity key.
-         */
-        StaticKey(int affKey) {
-            this.affKey = affKey;
-        }
-    }
-
-    /** */
-    private static class StaticValue {
-        /** */
-        @AffinityKeyMapped
-        private int affKey;
-
-        /**
-         * @param affKey Affinity key.
-         */
-        StaticValue(int affKey) {
-        }
-    }
-
-    /** */
-    private static class DynamicKey {
-        /** */
-        @AffinityKeyMapped
-        private int affKey;
-
-        /**
-         * @param affKey Affinity key.
-         */
-        DynamicKey(int affKey) {
-            this.affKey = affKey;
-        }
-    }
-
-    /** */
-    private static class DynamicValue {
-        /** */
-        @AffinityKeyMapped
-        private int affKey;
-
-        /**
-         * @param affKey Affinity key.
-         */
-        DynamicValue(int affKey) {
-            this.affKey = affKey;
-        }
-    }
-}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java
index d3a86b9f47d64..0a0efc72d373d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/AbstractSchemaSelfTest.java
@@ -531,21 +531,21 @@ public long id() {
     public static class ValueClass {
         /** Field 1. */
         @QuerySqlField
-        private Long field1;
+        private String field1;
 
         /**
          * Constructor.
          *
          * @param field1 Field 1.
          */
-        public ValueClass(Long field1) {
+        public ValueClass(String field1) {
             this.field1 = field1;
         }
 
         /**
          * @return Field 1
          */
-        public Long field1() {
+        public String field1() {
             return field1;
         }
     }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
index a6982cdee3eb4..0684f7f6e8e69 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicIndexAbstractSelfTest.java
@@ -70,9 +70,9 @@ public abstract class H2DynamicIndexAbstractSelfTest extends AbstractSchemaSelfT
 
         IgniteCache<KeyClass, ValueClass> cache = client().cache(CACHE_NAME);
 
-        cache.put(new KeyClass(1), new ValueClass(1L));
-        cache.put(new KeyClass(2), new ValueClass(2L));
-        cache.put(new KeyClass(3), new ValueClass(3L));
+        cache.put(new KeyClass(1), new ValueClass("val1"));
+        cache.put(new KeyClass(2), new ValueClass("val2"));
+        cache.put(new KeyClass(3), new ValueClass("val3"));
     }
 
     /** {@inheritDoc} */
@@ -99,14 +99,14 @@ public void testCreateIndex() throws Exception {
                 continue;
 
             List<List<?>> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " +
-                "\"cache\".\"ValueClass\" where \"field1\" = 1").setLocal(true)).getAll();
+                "\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll();
 
             assertEquals(F.asList(
                 Collections.singletonList("SELECT\n" +
                     "    \"id\"\n" +
                     "FROM \"cache\".\"ValueClass\"\n" +
-                    "    /* \"cache\".\"idx_1\": \"field1\" = 1 */\n" +
-                    "WHERE \"field1\" = 1")
+                    "    /* \"cache\".\"idx_1\": \"field1\" = 'A' */\n" +
+                    "WHERE \"field1\" = 'A'")
             ), locRes);
         }
 
@@ -116,7 +116,7 @@ public void testCreateIndex() throws Exception {
 
         assertSize(2);
 
-        cache.put(new KeyClass(4), new ValueClass(1L));
+        cache.put(new KeyClass(4), new ValueClass("someVal"));
 
         assertSize(3);
     }
@@ -172,14 +172,14 @@ public void testDropIndex() {
                 continue;
 
             List<List<?>> locRes = ignite(i).cache("cache").query(new SqlFieldsQuery("explain select \"id\" from " +
-                "\"cache\".\"ValueClass\" where \"field1\" = 1").setLocal(true)).getAll();
+                "\"cache\".\"ValueClass\" where \"field1\" = 'A'").setLocal(true)).getAll();
 
             assertEquals(F.asList(
                 Collections.singletonList("SELECT\n" +
                     "    \"id\"\n" +
                     "FROM \"cache\".\"ValueClass\"\n" +
                     "    /* \"cache\".\"ValueClass\".__SCAN_ */\n" +
-                    "WHERE \"field1\" = 1")
+                    "WHERE \"field1\" = 'A'")
             ), locRes);
         }
 
@@ -214,39 +214,38 @@ public void testDropMissingIndexIfExists() {
     public void testIndexState() {
         IgniteCache<KeyClass, ValueClass> cache = cache();
 
-        assertColumnValues(1L, 2L, 3L);
+        assertColumnValues("val1", "val2", "val3");
 
         cache.query(new SqlFieldsQuery("CREATE INDEX \"" + IDX_NAME_1_ESCAPED + "\" ON \"" + TBL_NAME_ESCAPED + "\"(\""
             + FIELD_NAME_1_ESCAPED + "\" ASC)"));
 
-        assertColumnValues(1L, 2L, 3L);
+        assertColumnValues("val1", "val2", "val3");
 
         cache.remove(new KeyClass(2));
 
-        assertColumnValues(1L, 3L);
+        assertColumnValues("val1", "val3");
 
-        cache.put(new KeyClass(0), new ValueClass(0L));
+        cache.put(new KeyClass(0), new ValueClass("someVal"));
 
-        assertColumnValues(0L, 1L, 3L);
+        assertColumnValues("someVal", "val1", "val3");
 
         cache.query(new SqlFieldsQuery("DROP INDEX \"" + IDX_NAME_1_ESCAPED + "\""));
 
-        assertColumnValues(0L, 1L, 3L);
+        assertColumnValues("someVal", "val1", "val3");
     }
 
     /**
      * Check that values of {@code field1} match what we expect.
      * @param vals Expected values.
      */
-    private void assertColumnValues(Long... vals) {
+    private void assertColumnValues(String... vals) {
         List<List<?>> expRes = new ArrayList<>(vals.length);
 
-        for (Long v : vals)
+        for (String v : vals)
             expRes.add(Collections.singletonList(v));
 
-        List<List<?>> all = cache().query(new SqlFieldsQuery("SELECT \"" + FIELD_NAME_1_ESCAPED + "\" FROM \"" +
-            TBL_NAME_ESCAPED + "\" ORDER BY \"id\"")).getAll();
-        assertEquals(expRes, all);
+        assertEquals(expRes, cache().query(new SqlFieldsQuery("SELECT \"" + FIELD_NAME_1_ESCAPED + "\" FROM \"" +
+            TBL_NAME_ESCAPED + "\" ORDER BY \"id\"")).getAll());
     }
 
     /**
@@ -256,9 +255,8 @@ private void assertColumnValues(Long... vals) {
     private void assertSize(long expSize) {
         assertEquals(expSize, cache().size());
 
-        Object actual = cache().query(new SqlFieldsQuery("SELECT COUNT(*) from \"ValueClass\""))
-            .getAll().get(0).get(0);
-        assertEquals(expSize, actual);
+        assertEquals(expSize, cache().query(new SqlFieldsQuery("SELECT COUNT(*) from \"ValueClass\""))
+            .getAll().get(0).get(0));
     }
 
     /**
@@ -340,8 +338,8 @@ private CacheConfiguration cacheConfiguration() {
         entity.setValueType(ValueClass.class.getName());
 
         entity.addQueryField("id", Long.class.getName(), null);
-        entity.addQueryField(FIELD_NAME_1_ESCAPED, Long.class.getName(), null);
-        entity.addQueryField(FIELD_NAME_2_ESCAPED, Long.class.getName(), null);
+        entity.addQueryField(FIELD_NAME_1_ESCAPED, String.class.getName(), null);
+        entity.addQueryField(FIELD_NAME_2_ESCAPED, String.class.getName(), null);
 
         entity.setKeyFields(Collections.singleton("id"));
 
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
index f088fb777d445..7c1ee528bcd85 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingTestSuite.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
-import org.apache.ignite.internal.processors.cache.CacheRegisterMetadataLocallyTest;
 import org.apache.ignite.internal.processors.cache.CacheBinaryKeyConcurrentQueryTest;
 import org.apache.ignite.internal.processors.cache.CacheConfigurationP2PTest;
 import org.apache.ignite.internal.processors.cache.CacheIndexStreamerTest;
@@ -80,7 +79,6 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(CacheOperationsWithExpirationTest.class);
         suite.addTestSuite(CacheBinaryKeyConcurrentQueryTest.class);
         suite.addTestSuite(CacheQueryFilterExpiredTest.class);
-        suite.addTestSuite(CacheRegisterMetadataLocallyTest.class);
 
         suite.addTestSuite(ClientReconnectAfterClusterRestartTest.class);
 

From 733293185c8f1c645e74a02701c32f40777bde53 Mon Sep 17 00:00:00 2001
From: Maxim Muzafarov <maxmuzaf@gmail.com>
Date: Wed, 1 Aug 2018 18:39:54 +0300
Subject: [PATCH 446/543] IGNITE-7165 Re-balancing is cancelled if client node
 joins

Signed-off-by: Anton Vinogradov <av@apache.org>
(cherry picked from commit 137dd06aaee9cc84104e6b4bf48306b050341e3a)
---
 .../GridCachePartitionExchangeManager.java    |  67 +++++---
 .../processors/cache/GridCachePreloader.java  |  21 ++-
 .../cache/GridCachePreloaderAdapter.java      |   6 +
 .../preloader/GridDhtPartitionDemander.java   |  48 +++---
 .../preloader/GridDhtPartitionSupplier.java   |  15 +-
 .../dht/preloader/GridDhtPreloader.java       |  60 ++++++-
 .../GridDhtPreloaderAssignments.java          |   5 +-
 .../ClusterBaselineNodesMetricsSelfTest.java  |   1 -
 .../cache/CacheValidatorMetricsTest.java      |   4 +-
 .../dht/GridCacheDhtPreloadSelfTest.java      |  68 +-------
 .../GridCacheRebalancingAsyncSelfTest.java    |   7 +-
 .../GridCacheRebalancingCancelTest.java       | 106 +++++++++++++
 ...CacheRebalancingPartitionCountersTest.java |   3 +-
 .../GridCacheRebalancingSyncSelfTest.java     | 149 +++++++-----------
 ...entAffinityAssignmentWithBaselineTest.java |   4 +-
 ...owHistoricalRebalanceSmallHistoryTest.java |   5 +-
 ...lushMultiNodeFailoverAbstractSelfTest.java |   2 +-
 .../GridMarshallerMappingConsistencyTest.java |   3 +-
 .../junits/common/GridCommonAbstractTest.java | 145 ++++++++---------
 .../testsuites/IgniteCacheTestSuite3.java     |   2 +
 20 files changed, 406 insertions(+), 315 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 05ebd11251434..73d5b63e5b274 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -66,7 +66,6 @@
 import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionFullCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
@@ -182,6 +181,14 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     private final ConcurrentMap<AffinityTopologyVersion, AffinityReadyFuture> readyFuts =
         new ConcurrentSkipListMap<>();
 
+    /**
+     * Latest started rebalance topology version but possibly not finished yet. Value {@code NONE}
+     * means that previous rebalance is undefined and the new one should be initiated.
+     *
+     * Should not be used to determine latest rebalanced topology.
+     */
+    private volatile AffinityTopologyVersion rebTopVer = AffinityTopologyVersion.NONE;
+
     /** */
     private GridFutureAdapter<?> reconnectExchangeFut;
 
@@ -862,6 +869,13 @@ public AffinityTopologyVersion readyAffinityVersion() {
         return exchFuts.readyTopVer();
     }
 
+    /**
+     * @return Latest rebalance topology version or {@code NONE} if there is no info.
+     */
+    public AffinityTopologyVersion rebalanceTopologyVersion() {
+        return rebTopVer;
+    }
+
     /**
      * @return Last initialized topology future.
      */
@@ -2774,6 +2788,9 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (grp.isLocal())
                                     continue;
 
+                                if (grp.preloader().rebalanceRequired(rebTopVer, exchFut))
+                                    rebTopVer = AffinityTopologyVersion.NONE;
+
                                 changed |= grp.topology().afterExchange(exchFut);
                             }
 
@@ -2781,7 +2798,11 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 refreshPartitions();
                         }
 
-                        if (!cctx.kernalContext().clientNode()) {
+                        // Schedule rebalance if force rebalance or force reassign occurs.
+                        if (exchFut == null)
+                            rebTopVer = AffinityTopologyVersion.NONE;
+
+                        if (!cctx.kernalContext().clientNode() && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
                             assignsMap = new HashMap<>();
 
                             IgniteCacheSnapshotManager snp = cctx.snapshot();
@@ -2798,6 +2819,9 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                     assigns = grp.preloader().generateAssignments(exchId, exchFut);
 
                                 assignsMap.put(grp.groupId(), assigns);
+
+                                if (resVer == null)
+                                    resVer = grp.topology().readyTopologyVersion();
                             }
                         }
                     }
@@ -2806,7 +2830,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         busy = false;
                     }
 
-                    if (assignsMap != null) {
+                    if (assignsMap != null && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
                         int size = assignsMap.size();
 
                         NavigableMap<Integer, List<Integer>> orderMap = new TreeMap<>();
@@ -2844,11 +2868,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (assigns != null)
                                     assignsCancelled |= assigns.cancelled();
 
-                                // Cancels previous rebalance future (in case it's not done yet).
-                                // Sends previous rebalance stopped event (if necessary).
-                                // Creates new rebalance future.
-                                // Sends current rebalance started event (if necessary).
-                                // Finishes cache sync future (on empty assignments).
                                 Runnable cur = grp.preloader().addAssignments(assigns,
                                     forcePreload,
                                     cnt,
@@ -2866,7 +2885,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         if (forcedRebFut != null)
                             forcedRebFut.markInitialized();
 
-                        if (assignsCancelled) { // Pending exchange.
+                        if (assignsCancelled || hasPendingExchange()) {
                             U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
                                 "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
@@ -2874,25 +2893,31 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         else if (r != null) {
                             Collections.reverse(rebList);
 
-                            U.log(log, "Rebalancing scheduled [order=" + rebList + "]");
+                            U.log(log, "Rebalancing scheduled [order=" + rebList +
+                                ", top=" + resVer + ", force=" + (exchFut == null) +
+                                ", evt=" + exchId.discoveryEventName() +
+                                ", node=" + exchId.nodeId() + ']');
 
-                            if (!hasPendingExchange()) {
-                                U.log(log, "Rebalancing started " +
-                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
-                                    ", node=" + exchId.nodeId() + ']');
+                            rebTopVer = resVer;
 
-                                r.run(); // Starts rebalancing routine.
-                            }
-                            else
-                                U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
-                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
-                                    ", node=" + exchId.nodeId() + ']');
+                            // Start rebalancing cache groups chain. Each group will be rebalanced
+                            // sequentially one by one e.g.:
+                            // ignite-sys-cache -> cacheGroupR1 -> cacheGroupP2 -> cacheGroupR3
+                            r.run();
                         }
                         else
                             U.log(log, "Skipping rebalancing (nothing scheduled) " +
-                                "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
+                                "[top=" + resVer + ", force=" + (exchFut == null) +
+                                ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
                     }
+                    else
+                        U.log(log, "Skipping rebalancing (no affinity changes) " +
+                            "[top=" + resVer +
+                            ", rebTopVer=" + rebTopVer +
+                            ", evt=" + exchId.discoveryEventName() +
+                            ", evtNode=" + exchId.nodeId() +
+                            ", client=" + cctx.kernalContext().clientNode() + ']');
                 }
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index 5fa7a821c69ef..d629e94db7e84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -24,6 +24,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.ForceRebalanceExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
@@ -62,10 +63,17 @@ public interface GridCachePreloader {
      */
     public void onInitialExchangeComplete(@Nullable Throwable err);
 
+    /**
+     * @param rebTopVer Previous rebalance topology version or {@code NONE} if there is no info.
+     * @param exchFut Completed exchange future.
+     * @return {@code True} if rebalance should be started (previous will be interrupted).
+     */
+    public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer, GridDhtPartitionsExchangeFuture exchFut);
+
     /**
      * @param exchId Exchange ID.
-     * @param exchFut Exchange future.
-     * @return Assignments or {@code null} if detected that there are pending exchanges.
+     * @param exchFut Completed exchange future. Can be {@code null} if forced or reassigned generation occurs.
+     * @return Partition assignments which will be requested from supplier nodes.
      */
     @Nullable public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      @Nullable GridDhtPartitionsExchangeFuture exchFut);
@@ -74,10 +82,10 @@ public interface GridCachePreloader {
      * Adds assignments to preloader.
      *
      * @param assignments Assignments to add.
-     * @param forcePreload Force preload flag.
-     * @param rebalanceId Rebalance id.
-     * @param next Runnable responsible for cache rebalancing start.
-     * @param forcedRebFut Rebalance future.
+     * @param forcePreload {@code True} if preload requested by {@link ForceRebalanceExchangeTask}.
+     * @param rebalanceId Rebalance id created by exchange thread.
+     * @param next Runnable responsible for cache rebalancing chain.
+     * @param forcedRebFut External future for forced rebalance.
      * @return Rebalancing runnable.
      */
     public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
@@ -114,7 +122,6 @@ public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
      * Future result is {@code false} in case rebalancing cancelled or finished with missed partitions and will be
      * restarted at current or pending topology.
      *
-     * Note that topology change creates new futures and finishes previous.
      */
     public IgniteInternalFuture<Boolean> rebalanceFuture();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index af916791daa3e..c5e4a817d0418 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -151,6 +151,12 @@ public GridCachePreloaderAdapter(CacheGroupContext grp) {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
+        GridDhtPartitionsExchangeFuture exchFut) {
+        return true;
+    }
+
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      GridDhtPartitionsExchangeFuture exchFut) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index e52d8d88a3b84..7f0d46c3d06f6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -235,12 +235,12 @@ else if (log.isDebugEnabled())
 
     /**
      * @param fut Future.
-     * @return {@code True} if topology changed.
+     * @return {@code True} if rebalance topology version changed by exchange thread or force
+     * reassing exchange occurs, see {@link RebalanceReassignExchangeTask} for details.
      */
     private boolean topologyChanged(RebalanceFuture fut) {
-        return
-            !grp.affinity().lastVersion().equals(fut.topologyVersion()) || // Topology already changed.
-                fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
+        return !ctx.exchange().rebalanceTopologyVersion().equals(fut.topVer) ||
+            fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
     }
 
     /**
@@ -253,14 +253,21 @@ void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
     }
 
     /**
-     * Initiates new rebalance process from given {@code assignments}.
-     * If previous rebalance is not finished method cancels it.
-     * In case of delayed rebalance method schedules new with configured delay.
+     * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
+     */
+    Collection<UUID> remainingNodes() {
+        return rebalanceFut.remainingNodes();
+    }
+
+    /**
+     * This method initiates new rebalance process from given {@code assignments} by creating new rebalance
+     * future based on them. Cancels previous rebalance future and sends rebalance started event.
+     * In case of delayed rebalance method schedules the new one with configured delay based on {@code lastExchangeFut}.
      *
-     * @param assignments Assignments.
-     * @param force {@code True} if dummy reassign.
-     * @param rebalanceId Rebalance id.
-     * @param next Runnable responsible for cache rebalancing start.
+     * @param assignments Assignments to process.
+     * @param force {@code True} if preload request by {@link ForceRebalanceExchangeTask}.
+     * @param rebalanceId Rebalance id generated from exchange thread.
+     * @param next Runnable responsible for cache rebalancing chain.
      * @param forcedRebFut External future for forced rebalance.
      * @return Rebalancing runnable.
      */
@@ -440,17 +447,7 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
             if (fut.isDone())
                 return;
 
-            // Must add all remaining node before send first request, for avoid race between add remaining node and
-            // processing response, see checkIsDone(boolean).
-            for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assignments.entrySet()) {
-                UUID nodeId = e.getKey().id();
-
-                IgniteDhtDemandedPartitionsMap parts = e.getValue().partitions();
-
-                assert parts != null : "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + nodeId + "]";
-
-                fut.remaining.put(nodeId, new T2<>(U.currentTimeMillis(), parts));
-            }
+            fut.remaining.forEach((key, value) -> value.set1(U.currentTimeMillis()));
         }
 
         final CacheConfiguration cfg = grp.config();
@@ -1267,6 +1264,13 @@ private void checkIsDone(boolean cancelled) {
             }
         }
 
+        /**
+         * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
+         */
+        private synchronized Collection<UUID> remainingNodes() {
+            return remaining.keySet();
+        }
+
         /**
          *
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 3b2a03f91c22b..5d9cee99630ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -17,12 +17,14 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
@@ -125,19 +127,20 @@ private static void clearContext(SupplyContext sc, IgniteLogger log) {
     }
 
     /**
-     * Handles new topology version and clears supply context map of outdated contexts.
-     *
-     * @param topVer Topology version.
+     * Handle topology change and clear supply context map of outdated contexts.
      */
-    @SuppressWarnings("ConstantConditions")
-    void onTopologyChanged(AffinityTopologyVersion topVer) {
+    void onTopologyChanged() {
         synchronized (scMap) {
             Iterator<T3<UUID, Integer, AffinityTopologyVersion>> it = scMap.keySet().iterator();
 
+            Collection<UUID> aliveNodes = grp.shared().discovery().aliveServerNodes().stream()
+                .map(ClusterNode::id)
+                .collect(Collectors.toList());
+
             while (it.hasNext()) {
                 T3<UUID, Integer, AffinityTopologyVersion> t = it.next();
 
-                if (topVer.compareTo(t.get3()) > 0) { // Clear all obsolete contexts.
+                if (!aliveNodes.contains(t.get1())) { // Clear all obsolete contexts.
                     clearContext(scMap.get(t), log);
 
                     it.remove();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 6e3d773abbbe3..034bf72cecf4e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -23,6 +23,7 @@
 import java.util.UUID;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -37,7 +38,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -157,11 +157,67 @@ private IgniteCheckedException stopError() {
 
     /** {@inheritDoc} */
     @Override public void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
-        supplier.onTopologyChanged(lastFut.initialVersion());
+        supplier.onTopologyChanged();
 
         demander.onTopologyChanged(lastFut);
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
+        GridDhtPartitionsExchangeFuture exchFut) {
+        if (ctx.kernalContext().clientNode() || rebTopVer.equals(AffinityTopologyVersion.NONE))
+            return false; // No-op.
+
+        if (exchFut.localJoinExchange())
+            return true; // Required, can have outdated updSeq partition counter if node reconnects.
+
+        if (!grp.affinity().cachedVersions().contains(rebTopVer)) {
+            assert rebTopVer.compareTo(grp.localStartVersion()) <= 0 :
+                "Empty hisroty allowed only for newly started cache group [rebTopVer=" + rebTopVer +
+                    ", localStartTopVer=" + grp.localStartVersion() + ']';
+
+            return true; // Required, since no history info available.
+        }
+
+        final IgniteInternalFuture<Boolean> rebFut = rebalanceFuture();
+
+        if (rebFut.isDone() && !rebFut.result())
+            return true; // Required, previous rebalance cancelled.
+
+        final AffinityTopologyVersion exchTopVer = exchFut.context().events().topologyVersion();
+
+        Collection<UUID> aliveNodes = ctx.discovery().aliveServerNodes().stream()
+            .map(ClusterNode::id)
+            .collect(Collectors.toList());
+
+        return assignmentsChanged(rebTopVer, exchTopVer) ||
+            !aliveNodes.containsAll(demander.remainingNodes()); // Some of nodes left before rabalance compelete.
+    }
+
+    /**
+     * @param oldTopVer Previous topology version.
+     * @param newTopVer New topology version to check result.
+     * @return {@code True} if affinity assignments changed between two versions for local node.
+     */
+    private boolean assignmentsChanged(AffinityTopologyVersion oldTopVer, AffinityTopologyVersion newTopVer) {
+        final AffinityAssignment aff = grp.affinity().readyAffinity(newTopVer);
+
+        // We should get affinity assignments based on previous rebalance to calculate difference.
+        // Whole history size described by IGNITE_AFFINITY_HISTORY_SIZE constant.
+        final AffinityAssignment prevAff = grp.affinity().cachedVersions().contains(oldTopVer) ?
+            grp.affinity().cachedAffinity(oldTopVer) : null;
+
+        if (prevAff == null)
+            return false;
+
+        boolean assignsChanged = false;
+
+        for (int p = 0; !assignsChanged && p < grp.affinity().partitions(); p++)
+            assignsChanged |= aff.get(p).contains(ctx.localNode()) != prevAff.get(p).contains(ctx.localNode());
+
+        return assignsChanged;
+    }
+
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId, GridDhtPartitionsExchangeFuture exchFut) {
         assert exchFut == null || exchFut.isDone();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
index 69dc79b5007c2..6f988890c6fb3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
@@ -20,7 +20,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
@@ -74,9 +73,9 @@ GridDhtPartitionExchangeId exchangeId() {
     }
 
     /**
-     * @return Topology version.
+     * @return Topology version based on last {@link GridDhtPartitionTopologyImpl#readyTopVer}.
      */
-    AffinityTopologyVersion topologyVersion() {
+    public AffinityTopologyVersion topologyVersion() {
         return topVer;
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
index 565317720d2a9..46b09ac3b1819 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
@@ -149,7 +149,6 @@ public void testBaselineNodes() throws Exception {
     private void resetBlt() throws Exception {
         resetBaselineTopology();
 
-        waitForRebalancing();
         awaitPartitionMapExchange();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
index 5c601a1bd3bbf..0b441dd94c0e8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
@@ -103,14 +103,14 @@ public void testCacheValidatorMetrics() throws Exception {
 
         startGrid(2);
 
-        waitForRebalancing();
+        awaitPartitionMapExchange();
 
         assertCacheStatus(CACHE_NAME_1, true, true);
         assertCacheStatus(CACHE_NAME_2, true, true);
 
         stopGrid(1);
 
-        waitForRebalancing();
+        awaitPartitionMapExchange();
 
         // Invalid for writing due to invalid topology.
         assertCacheStatus(CACHE_NAME_1, true, false);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
index 7b5cc3e8101dc..82ed95e52531f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
@@ -22,20 +22,17 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.events.CacheRebalancingEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
@@ -44,10 +41,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -61,9 +55,6 @@
 import static org.apache.ignite.configuration.CacheConfiguration.DFLT_REBALANCE_BATCH_SIZE;
 import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS;
 import static org.apache.ignite.events.EventType.EVTS_CACHE_REBALANCE;
-import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
-import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
-import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
@@ -144,15 +135,6 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) {
         return cacheCfg;
     }
 
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-//        resetLog4j(Level.DEBUG, true,
-//            // Categories.
-//            GridDhtPreloader.class.getPackage().getName(),
-//            GridDhtPartitionTopologyImpl.class.getName(),
-//            GridDhtLocalPartition.class.getName());
-    }
-
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         backups = DFLT_BACKUPS;
@@ -230,11 +212,6 @@ public void testActivePartitionTransferAsyncRandomCoordinator() throws Exception
      */
     private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
-//        resetLog4j(Level.DEBUG, true,
-//            // Categories.
-//            GridDhtPreloader.class.getPackage().getName(),
-//            GridDhtPartitionTopologyImpl.class.getName(),
-//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -273,8 +250,6 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
             info(">>> Finished checking nodes [keyCnt=" + keyCnt + ", nodeCnt=" + nodeCnt + ", grids=" +
                 U.grids2names(ignites) + ']');
 
-            Collection<IgniteFuture<?>> futs = new LinkedList<>();
-
             Ignite last = F.last(ignites);
 
             for (Iterator<Ignite> it = ignites.iterator(); it.hasNext(); ) {
@@ -288,21 +263,8 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
 
                 checkActiveState(ignites);
 
-                final UUID nodeId = g.cluster().localNode().id();
-
                 it.remove();
 
-                futs.add(waitForLocalEvent(last.events(), new P1<Event>() {
-                    @Override public boolean apply(Event e) {
-                        CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
-
-                        ClusterNode node = evt.discoveryNode();
-
-                        return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
-                            (evt.discoveryEventType() == EVT_NODE_LEFT || evt.discoveryEventType() == EVT_NODE_FAILED);
-                    }
-                }, EVT_CACHE_REBALANCE_STOPPED));
-
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
@@ -315,14 +277,6 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
                 awaitPartitionMapExchange(); // Need wait, otherwise test logic is broken if EVT_NODE_FAILED exchanges are merged.
             }
 
-            info("Waiting for preload futures: " + F.view(futs, new IgnitePredicate<IgniteFuture<?>>() {
-                @Override public boolean apply(IgniteFuture<?> fut) {
-                    return !fut.isDone();
-                }
-            }));
-
-            X.waitAll(futs);
-
             info("Finished waiting for preload futures.");
 
             assert last != null;
@@ -502,11 +456,6 @@ private void stopGrids(Iterable<Ignite> grids) {
      */
     private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
-//        resetLog4j(Level.DEBUG, true,
-//            // Categories.
-//            GridDhtPreloader.class.getPackage().getName(),
-//            GridDhtPartitionTopologyImpl.class.getName(),
-//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -558,28 +507,13 @@ private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuf
 
                 it.remove();
 
-                Collection<IgniteFuture<?>> futs = new LinkedList<>();
-
-                for (Ignite gg : ignites)
-                    futs.add(waitForLocalEvent(gg.events(), new P1<Event>() {
-                        @Override public boolean apply(Event e) {
-                            CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
-
-                            ClusterNode node = evt.discoveryNode();
-
-                            return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
-                                (evt.discoveryEventType() == EVT_NODE_LEFT ||
-                                    evt.discoveryEventType() == EVT_NODE_FAILED);
-                        }
-                    }, EVT_CACHE_REBALANCE_STOPPED));
-
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
 
                 info(">>> Waiting for preload futures [leftNode=" + g.name() + ", remaining=" + U.grids2names(ignites) + ']');
 
-                X.waitAll(futs);
+                awaitPartitionMapExchange();
 
                 info("After grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
index 4ebcd5df35271..0a8698a597cd8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
@@ -17,10 +17,11 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
 
-import org.apache.ignite.Ignite;
+import java.util.Collections;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
@@ -43,7 +44,7 @@ public class GridCacheRebalancingAsyncSelfTest extends GridCacheRebalancingSyncS
      * @throws Exception Exception.
      */
     public void testNodeFailedAtRebalancing() throws Exception {
-        Ignite ignite = startGrid(0);
+        IgniteEx ignite = startGrid(0);
 
         generateData(ignite, 0, 0);
 
@@ -60,7 +61,7 @@ public void testNodeFailedAtRebalancing() throws Exception {
 
         ((TestTcpDiscoverySpi)grid(1).configuration().getDiscoverySpi()).simulateNodeFailure();
 
-        waitForRebalancing(0, 3);
+        awaitPartitionMapExchange(false, false, Collections.singletonList(ignite.localNode()));
 
         checkSupplyContextMapIsEmpty();
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
new file mode 100644
index 0000000000000..3965290480a3d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
@@ -0,0 +1,106 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test cases for checking cancellation rebalancing process if some events occurs.
+ */
+public class GridCacheRebalancingCancelTest extends GridCommonAbstractTest {
+    /** */
+    private static final String DHT_PARTITIONED_CACHE = "cacheP";
+
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration dfltCfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)dfltCfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        dfltCfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        return dfltCfg;
+    }
+
+    /**
+     * Test rebalance not cancelled when client node join to cluster.
+     *
+     * @throws Exception Exception.
+     */
+    public void testClientNodeJoinAtRebalancing() throws Exception {
+        final IgniteEx ignite0 = startGrid(0);
+
+        IgniteCache<Integer, Integer> cache = ignite0.createCache(
+            new CacheConfiguration<Integer, Integer>(DHT_PARTITIONED_CACHE)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setRebalanceMode(CacheRebalanceMode.ASYNC)
+                .setBackups(1)
+                .setRebalanceOrder(2)
+                .setAffinity(new RendezvousAffinityFunction(false)));
+
+        for (int i = 0; i < 2048; i++)
+            cache.put(i, i);
+
+        TestRecordingCommunicationSpi.spi(ignite0)
+            .blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+                @Override public boolean apply(ClusterNode node, Message msg) {
+                    return (msg instanceof GridDhtPartitionSupplyMessage)
+                        && ((GridCacheGroupIdMessage)msg).groupId() == groupIdForCache(ignite0, DHT_PARTITIONED_CACHE);
+                }
+            });
+
+        final IgniteEx ignite1 = startGrid(1);
+
+        TestRecordingCommunicationSpi.spi(ignite0).waitForBlocked();
+
+        GridDhtPartitionDemander.RebalanceFuture fut = (GridDhtPartitionDemander.RebalanceFuture)ignite1.context().
+            cache().internalCache(DHT_PARTITIONED_CACHE).preloader().rebalanceFuture();
+
+        String igniteClntName = getTestIgniteInstanceName(2);
+
+        startGrid(igniteClntName, optimize(getConfiguration(igniteClntName).setClientMode(true)));
+
+        // Resend delayed rebalance messages.
+        TestRecordingCommunicationSpi.spi(ignite0).stopBlock(true);
+
+        awaitPartitionMapExchange();
+
+        // Previous rebalance future should not be cancelled.
+        assertTrue(fut.result());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
index fa4b655fb41bd..258e36e531322 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
@@ -141,7 +141,8 @@ else if ("index.bin".equals(f.getName())) {
         assertTrue(primaryRemoved);
 
         ignite.cluster().active(true);
-        waitForRebalancing();
+
+        awaitPartitionMapExchange();
 
         List<String> issues = new ArrayList<>();
         HashMap<Integer, Long> partMap = new HashMap<>();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
index 4d03d7dc7e281..8b88befee5281 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
@@ -21,10 +21,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cluster.ClusterNode;
@@ -42,11 +41,13 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -68,6 +69,9 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private static final int TEST_SIZE = 100_000;
 
+    /** */
+    private static final long TOPOLOGY_STILLNESS_TIME = 30_000L;
+
     /** partitioned cache name. */
     protected static final String CACHE_NAME_DHT_PARTITIONED = "cacheP";
 
@@ -89,11 +93,11 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private volatile boolean concurrentStartFinished3;
 
-    /** */
-    private volatile boolean record;
-
-    /** */
-    private final ConcurrentHashMap<Class, AtomicInteger> map = new ConcurrentHashMap<>();
+    /**
+     * Time in milliseconds of last received {@link GridDhtPartitionsSingleMessage}
+     * or {@link GridDhtPartitionsFullMessage} using {@link CollectingCommunicationSpi}.
+     */
+    private static volatile long lastPartMsgTime;
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -102,7 +106,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setIpFinder(ipFinder);
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setForceServerMode(true);
 
-        TcpCommunicationSpi commSpi = new CountingCommunicationSpi();
+        TcpCommunicationSpi commSpi = new CollectingCommunicationSpi();
 
         commSpi.setLocalPort(GridTestUtils.getNextCommPort(getClass()));
         commSpi.setTcpNoDelay(true);
@@ -232,47 +236,35 @@ public void testSimpleRebalancing() throws Exception {
 
         startGrid(1);
 
-        int waitMinorVer = ignite.configuration().isLateAffinityAssignment() ? 1 : 0;
-
-        waitForRebalancing(0, 2, waitMinorVer);
-        waitForRebalancing(1, 2, waitMinorVer);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         stopGrid(0);
 
-        waitForRebalancing(1, 3);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         startGrid(2);
 
-        waitForRebalancing(1, 4, waitMinorVer);
-        waitForRebalancing(2, 4, waitMinorVer);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         stopGrid(2);
 
-        waitForRebalancing(1, 5);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         long spend = (System.currentTimeMillis() - start) / 1000;
 
@@ -331,13 +323,10 @@ public void testLoadRebalancing() throws Exception {
 
         startGrid(4);
 
-        waitForRebalancing(3, 6);
-        waitForRebalancing(4, 6);
+        awaitPartitionMapExchange(true, true, null);
 
         concurrentStartFinished = true;
 
-        awaitPartitionMapExchange(true, true, null);
-
         checkSupplyContextMapIsEmpty();
 
         t1.join();
@@ -442,27 +431,29 @@ protected void checkPartitionMapExchangeFinished() {
     }
 
     /**
+     * Method checks for {@link GridDhtPartitionsSingleMessage} or {@link GridDhtPartitionsFullMessage}
+     * not received within {@code TOPOLOGY_STILLNESS_TIME} bound.
+     *
      * @throws Exception If failed.
      */
-    protected void checkPartitionMapMessagesAbsent() throws Exception {
-        map.clear();
-
-        record = true;
-
-        log.info("Checking GridDhtPartitions*Message absent (it will take 30 SECONDS) ... ");
-
-        U.sleep(30_000);
-
-        record = false;
-
-        AtomicInteger iF = map.get(GridDhtPartitionsFullMessage.class);
-        AtomicInteger iS = map.get(GridDhtPartitionsSingleMessage.class);
-
-        Integer fullMap = iF != null ? iF.get() : null;
-        Integer singleMap = iS != null ? iS.get() : null;
-
-        assertTrue("Unexpected full map messages: " + fullMap, fullMap == null || fullMap.equals(1)); // 1 message can be sent right after all checks passed.
-        assertNull("Unexpected single map messages", singleMap);
+    protected void awaitPartitionMessagesAbsent() throws Exception {
+        log.info("Checking GridDhtPartitions*Message absent (it will take up to " +
+            TOPOLOGY_STILLNESS_TIME + " ms) ... ");
+
+        // Start waiting new messages from current point of time.
+        lastPartMsgTime = U.currentTimeMillis();
+
+        assertTrue("Should not have partition Single or Full messages within bound " +
+                TOPOLOGY_STILLNESS_TIME + " ms.",
+            GridTestUtils.waitForCondition(
+                new GridAbsPredicateX() {
+                    @Override public boolean applyx() {
+                        return lastPartMsgTime + TOPOLOGY_STILLNESS_TIME < U.currentTimeMillis();
+                    }
+                },
+                2 * TOPOLOGY_STILLNESS_TIME // 30 sec to gain stable topology and 30 sec of silence.
+            )
+        );
     }
 
     /** {@inheritDoc} */
@@ -495,11 +486,7 @@ public void testComplexRebalancing() throws Exception {
                     while (!concurrentStartFinished2)
                         U.sleep(10);
 
-                    waitForRebalancing(0, 5, 0);
-                    waitForRebalancing(1, 5, 0);
-                    waitForRebalancing(2, 5, 0);
-                    waitForRebalancing(3, 5, 0);
-                    waitForRebalancing(4, 5, 0);
+                    awaitPartitionMapExchange();
 
                     //New cache should start rebalancing.
                     CacheConfiguration<Integer, Integer> cacheRCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
@@ -552,12 +539,6 @@ public void testComplexRebalancing() throws Exception {
         t2.join();
         t3.join();
 
-        waitForRebalancing(0, 5, 1);
-        waitForRebalancing(1, 5, 1);
-        waitForRebalancing(2, 5, 1);
-        waitForRebalancing(3, 5, 1);
-        waitForRebalancing(4, 5, 1);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
@@ -577,35 +558,23 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(1);
 
-        waitForRebalancing(0, 6);
-        waitForRebalancing(2, 6);
-        waitForRebalancing(3, 6);
-        waitForRebalancing(4, 6);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(0);
 
-        waitForRebalancing(2, 7);
-        waitForRebalancing(3, 7);
-        waitForRebalancing(4, 7);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(2);
 
-        waitForRebalancing(3, 8);
-        waitForRebalancing(4, 8);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         checkSupplyContextMapIsEmpty();
 
@@ -613,7 +582,7 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(3);
 
-        waitForRebalancing(4, 9);
+        awaitPartitionMapExchange();
 
         checkSupplyContextMapIsEmpty();
 
@@ -634,36 +603,26 @@ public void testComplexRebalancing() throws Exception {
     /**
      *
      */
-    private class CountingCommunicationSpi extends TcpCommunicationSpi {
+    private static class CollectingCommunicationSpi extends TcpCommunicationSpi {
+        /** */
+        @LoggerResource
+        private IgniteLogger log;
+
         /** {@inheritDoc} */
         @Override public void sendMessage(final ClusterNode node, final Message msg,
             final IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
             final Object msg0 = ((GridIoMessage)msg).message();
 
-            recordMessage(msg0);
+            if (msg0 instanceof GridDhtPartitionsSingleMessage ||
+                msg0 instanceof GridDhtPartitionsFullMessage) {
+                lastPartMsgTime = U.currentTimeMillis();
 
-            super.sendMessage(node, msg, ackC);
-        }
-
-        /**
-         * @param msg Message.
-         */
-        private void recordMessage(Object msg) {
-            if (record) {
-                Class id = msg.getClass();
-
-                AtomicInteger ai = map.get(id);
-
-                if (ai == null) {
-                    ai = new AtomicInteger();
-
-                    AtomicInteger oldAi = map.putIfAbsent(id, ai);
-
-                    (oldAi != null ? oldAi : ai).incrementAndGet();
-                }
-                else
-                    ai.incrementAndGet();
+                log.info("Last seen time of GridDhtPartitionsSingleMessage or GridDhtPartitionsFullMessage updated " +
+                    "[lastPartMsgTime=" + lastPartMsgTime +
+                    ", node=" + node.id() + ']');
             }
+
+            super.sendMessage(node, msg, ackC);
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
index 15ec41557dfba..ce84156e56dca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
@@ -383,7 +383,7 @@ public void testRejoinWithCleanLfs() throws Exception {
         startGrid("flaky");
 
         System.out.println("### Starting rebalancing after flaky node join");
-        waitForRebalancing();
+        awaitPartitionMapExchange();
         System.out.println("### Rebalancing is finished after flaky node join");
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
@@ -689,7 +689,7 @@ private void tryChangeBaselineDown(
         ig0.cluster().setBaselineTopology(fullBlt.subList(0, newBaselineSize));
 
         System.out.println("### Starting rebalancing after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
-        waitForRebalancing();
+        awaitPartitionMapExchange();
         System.out.println("### Rebalancing is finished after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
index e6ec8a37f6538..bd1a1a9cc3575 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
@@ -162,7 +162,8 @@ public void testReservation() throws Exception {
 
         SUPPLY_MESSAGE_LATCH.get().countDown();
 
-        waitForRebalancing(); // Partition is OWNING on grid(0) and grid(1)
+        // Partition is OWNING on grid(0) and grid(1)
+        awaitPartitionMapExchange();
 
         for (int i = 0; i < 2; i++) {
             for (int j = 0; i < 500; i++)
@@ -183,7 +184,7 @@ public void testReservation() throws Exception {
 
         startGrid(0);
 
-        waitForRebalancing();
+        awaitPartitionMapExchange();
 
         assertEquals(2, grid(1).context().discovery().aliveServerNodes().size());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index d585a82e66a2f..0ed39ce003f4e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -192,7 +192,7 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
 
                     grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
 
-                    waitForRebalancing();
+                    awaitPartitionMapExchange();
                 }
                 catch (Throwable expected) {
                     // There can be any exception. Do nothing.
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index efb4fd511c4aa..ba39e366e9f85 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -120,7 +120,8 @@ public void testMappingsPersistedOnJoin() throws Exception {
         c1.put(k, new DummyObject(k));
 
         startGrid(2);
-        waitForRebalancing();
+
+        awaitPartitionMapExchange();
 
         stopAllGrids();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index e6c9076a23496..ed2d6ed782b8d 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -44,6 +44,7 @@
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteEvents;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteMessaging;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.cache.CachePeekMode;
@@ -62,6 +63,7 @@
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityFunctionContextImpl;
@@ -110,6 +112,9 @@
 import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.testframework.GridTestNode;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.GridAbstractTest;
@@ -690,32 +695,34 @@ protected void awaitPartitionMapExchange(
 
                                 if (affNodesCnt != ownerNodesCnt || !affNodes.containsAll(owners) ||
                                     (waitEvicts && loc != null && loc.state() != GridDhtPartitionState.OWNING)) {
+                                    if (i % 50 == 0)
+                                        LT.warn(log(), "Waiting for topology map update [" +
+                                            "igniteInstanceName=" + g.name() +
+                                            ", cache=" + cfg.getName() +
+                                            ", cacheId=" + dht.context().cacheId() +
+                                            ", topVer=" + top.readyTopologyVersion() +
+                                            ", p=" + p +
+                                            ", affNodesCnt=" + affNodesCnt +
+                                            ", ownersCnt=" + ownerNodesCnt +
+                                            ", affNodes=" + F.nodeIds(affNodes) +
+                                            ", owners=" + F.nodeIds(owners) +
+                                            ", topFut=" + topFut +
+                                            ", locNode=" + g.cluster().localNode() + ']');
+                                }
+                                else
+                                    match = true;
+                            }
+                            else {
+                                if (i % 50 == 0)
                                     LT.warn(log(), "Waiting for topology map update [" +
                                         "igniteInstanceName=" + g.name() +
                                         ", cache=" + cfg.getName() +
                                         ", cacheId=" + dht.context().cacheId() +
                                         ", topVer=" + top.readyTopologyVersion() +
+                                        ", started=" + dht.context().started() +
                                         ", p=" + p +
-                                        ", affNodesCnt=" + affNodesCnt +
-                                        ", ownersCnt=" + ownerNodesCnt +
-                                        ", affNodes=" + F.nodeIds(affNodes) +
-                                        ", owners=" + F.nodeIds(owners) +
-                                        ", topFut=" + topFut +
+                                        ", readVer=" + readyVer +
                                         ", locNode=" + g.cluster().localNode() + ']');
-                                }
-                                else
-                                    match = true;
-                            }
-                            else {
-                                LT.warn(log(), "Waiting for topology map update [" +
-                                    "igniteInstanceName=" + g.name() +
-                                    ", cache=" + cfg.getName() +
-                                    ", cacheId=" + dht.context().cacheId() +
-                                    ", topVer=" + top.readyTopologyVersion() +
-                                    ", started=" + dht.context().started() +
-                                    ", p=" + p +
-                                    ", readVer=" + readyVer +
-                                    ", locNode=" + g.cluster().localNode() + ']');
                             }
 
                             if (!match) {
@@ -945,78 +952,58 @@ protected void printPartitionState(String cacheName, int firstParts) {
     }
 
     /**
-     * @param id Node id.
-     * @param major Major ver.
-     * @param minor Minor ver.
-     * @throws IgniteCheckedException If failed.
-     */
-    protected void waitForRebalancing(int id, int major, int minor) throws IgniteCheckedException {
-        waitForRebalancing(grid(id), new AffinityTopologyVersion(major, minor));
-    }
-
-    /**
-     * @param id Node id.
-     * @param major Major ver.
-     * @throws IgniteCheckedException If failed.
-     */
-    protected void waitForRebalancing(int id, int major) throws IgniteCheckedException {
-        waitForRebalancing(grid(id), new AffinityTopologyVersion(major));
-    }
-
-    /**
-     * @throws IgniteCheckedException If failed.
-     */
-    protected void waitForRebalancing() throws IgniteCheckedException {
-        for (Ignite ignite : G.allGrids())
-            waitForRebalancing((IgniteEx)ignite, null);
-    }
-
-    /**
-     * @param ignite Node.
-     * @param top Topology version.
-     * @throws IgniteCheckedException If failed.
+     * Use method for manual rebalaincing cache on all nodes. Note that using
+     * <pre name="code" class="java">
+     *   for (int i = 0; i < G.allGrids(); i++)
+     *     grid(i).cache(CACHE_NAME).rebalance().get();
+     * </pre>
+     * for rebalancing cache will lead to flaky test cases.
+     *
+     * @param ignite Ignite server instance for getting {@code compute} facade over all cluster nodes.
+     * @param cacheName Cache name for manual rebalancing on cluster. Usually used when used when
+     * {@link CacheConfiguration#getRebalanceDelay()} configuration parameter set to {@code -1} value.
+     * @throws IgniteCheckedException If fails.
      */
-    protected void waitForRebalancing(IgniteEx ignite, AffinityTopologyVersion top) throws IgniteCheckedException {
+    protected void manualCacheRebalancing(Ignite ignite,
+        final String cacheName) throws IgniteCheckedException {
         if (ignite.configuration().isClientMode())
             return;
 
-        boolean finished = false;
-
-        long stopTime = System.currentTimeMillis() + 60_000;
-
-        while (!finished && (System.currentTimeMillis() < stopTime)) {
-            finished = true;
-
-            if (top == null)
-                top = ignite.context().discovery().topologyVersionEx();
+        IgniteFuture<Void> fut =
+            ignite.compute().withTimeout(5_000).broadcastAsync(new IgniteRunnable() {
+                /** */
+                @LoggerResource
+                IgniteLogger log;
 
-            for (GridCacheAdapter c : ignite.context().cache().internalCaches()) {
-                GridDhtPartitionDemander.RebalanceFuture fut =
-                    (GridDhtPartitionDemander.RebalanceFuture)c.preloader().rebalanceFuture();
+                /** */
+                @IgniteInstanceResource
+                private Ignite ignite;
 
-                if (fut.topologyVersion() == null || fut.topologyVersion().compareTo(top) < 0) {
-                    finished = false;
+                /** {@inheritDoc} */
+                @Override public void run() {
+                    IgniteCache<?, ?> cache = ignite.cache(cacheName);
 
-                    log.info("Unexpected future version, will retry [futVer=" + fut.topologyVersion() +
-                        ", expVer=" + top + ']');
+                    assertNotNull(cache);
 
-                    U.sleep(100);
+                    while (!(Boolean)cache.rebalance().get()) {
+                        try {
+                            U.sleep(100);
+                        }
+                        catch (IgniteInterruptedCheckedException e) {
+                            throw new IgniteException(e);
+                        }
+                    }
 
-                    break;
+                    if (log.isInfoEnabled())
+                        log.info("Manual rebalance finished [node=" + ignite.name() + ", cache=" + cacheName + "]");
                 }
-                else if (!fut.get()) {
-                    finished = false;
-
-                    log.warning("Rebalancing finished with missed partitions.");
-
-                    U.sleep(100);
+            });
 
-                    break;
-                }
+        assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                return fut.isDone();
             }
-        }
-
-        assertTrue(finished);
+        }, 5_000));
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index 55436d6da943b..5e94052f25013 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -53,6 +53,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingAsyncSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingCancelTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
@@ -152,6 +153,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
         suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
         suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
+        suite.addTestSuite(GridCacheRebalancingCancelTest.class);
 
         // Test for byte array value special case.
         suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);

From eeaf790f31a8dcd0ecb88681645e10eb347f75e5 Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Thu, 25 Oct 2018 05:52:06 +0300
Subject: [PATCH 447/543] IGNITE-7165 Fixed compilation.

---
 .../junits/common/GridCommonAbstractTest.java | 75 +++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index ed2d6ed782b8d..ed9ba931ba6a0 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -951,6 +951,81 @@ protected void printPartitionState(String cacheName, int firstParts) {
         log.info("dump partitions state for <" + cacheName + ">:\n" + sb.toString());
     }
 
+    /**
+     * @param id Node id.
+     * @param major Major ver.
+     * @param minor Minor ver.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void waitForRebalancing(int id, int major, int minor) throws IgniteCheckedException {
+        waitForRebalancing(grid(id), new AffinityTopologyVersion(major, minor));
+    }
+
+    /**
+     * @param id Node id.
+     * @param major Major ver.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void waitForRebalancing(int id, int major) throws IgniteCheckedException {
+        waitForRebalancing(grid(id), new AffinityTopologyVersion(major));
+    }
+
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void waitForRebalancing() throws IgniteCheckedException {
+        for (Ignite ignite : G.allGrids())
+            waitForRebalancing((IgniteEx)ignite, null);
+    }
+
+    /**
+     * @param ignite Node.
+     * @param top Topology version.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected void waitForRebalancing(IgniteEx ignite, AffinityTopologyVersion top) throws IgniteCheckedException {
+        if (ignite.configuration().isClientMode())
+            return;
+
+        boolean finished = false;
+
+        long stopTime = System.currentTimeMillis() + 60_000;
+
+        while (!finished && (System.currentTimeMillis() < stopTime)) {
+            finished = true;
+
+            if (top == null)
+                top = ignite.context().discovery().topologyVersionEx();
+
+            for (GridCacheAdapter c : ignite.context().cache().internalCaches()) {
+                GridDhtPartitionDemander.RebalanceFuture fut =
+                        (GridDhtPartitionDemander.RebalanceFuture)c.preloader().rebalanceFuture();
+
+                if (fut.topologyVersion() == null || fut.topologyVersion().compareTo(top) < 0) {
+                    finished = false;
+
+                    log.info("Unexpected future version, will retry [futVer=" + fut.topologyVersion() +
+                            ", expVer=" + top + ']');
+
+                    U.sleep(100);
+
+                    break;
+                }
+                else if (!fut.get()) {
+                    finished = false;
+
+                    log.warning("Rebalancing finished with missed partitions.");
+
+                    U.sleep(100);
+
+                    break;
+                }
+            }
+        }
+
+        assertTrue(finished);
+    }
+
     /**
      * Use method for manual rebalaincing cache on all nodes. Note that using
      * <pre name="code" class="java">

From b50f776a616cce18598b1d897d20d7ce6bb6cd82 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Thu, 25 Oct 2018 11:23:58 +0700
Subject: [PATCH 448/543] IGNITE-9853 control.sh: Added output of cache
 configurations. Fixes #4956.

(cherry picked from commit a61e4a394af649a146a3f43489181db6fd991967)
---
 .../internal/commandline/CommandHandler.java  | 623 +++++++++++++++---
 .../internal/commandline/OutputFormat.java    |  66 ++
 .../commandline/cache/CacheArguments.java     |  27 +
 .../processors/cache/verify/CacheInfo.java    |  71 +-
 .../visor/cache/VisorCacheConfiguration.java  |   5 +-
 .../VisorCacheConfigurationCollectorJob.java  |  23 +-
 ...sorCacheConfigurationCollectorTaskArg.java |  32 +-
 .../visor/verify/VisorViewCacheTaskArg.java   |  35 +-
 .../ignite/util/GridCommandHandlerTest.java   |  70 +-
 9 files changed, 826 insertions(+), 126 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/commandline/OutputFormat.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index d011db9fb48e2..1ab65ba432c2c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -21,14 +21,15 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
 import java.util.Set;
 import java.util.UUID;
-import java.util.logging.Logger;
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 import java.util.stream.Collectors;
@@ -66,6 +67,7 @@
 import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.apache.ignite.internal.visor.baseline.VisorBaselineNode;
@@ -73,11 +75,20 @@
 import org.apache.ignite.internal.visor.baseline.VisorBaselineTask;
 import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskArg;
 import org.apache.ignite.internal.visor.baseline.VisorBaselineTaskResult;
+import org.apache.ignite.internal.visor.cache.VisorCacheAffinityConfiguration;
+import org.apache.ignite.internal.visor.cache.VisorCacheConfiguration;
+import org.apache.ignite.internal.visor.cache.VisorCacheConfigurationCollectorTask;
+import org.apache.ignite.internal.visor.cache.VisorCacheConfigurationCollectorTaskArg;
+import org.apache.ignite.internal.visor.cache.VisorCacheEvictionConfiguration;
+import org.apache.ignite.internal.visor.cache.VisorCacheNearConfiguration;
+import org.apache.ignite.internal.visor.cache.VisorCacheRebalanceConfiguration;
+import org.apache.ignite.internal.visor.cache.VisorCacheStoreConfiguration;
 import org.apache.ignite.internal.visor.misc.VisorClusterNode;
 import org.apache.ignite.internal.visor.misc.VisorWalTask;
 import org.apache.ignite.internal.visor.misc.VisorWalTaskArg;
 import org.apache.ignite.internal.visor.misc.VisorWalTaskOperation;
 import org.apache.ignite.internal.visor.misc.VisorWalTaskResult;
+import org.apache.ignite.internal.visor.query.VisorQueryConfiguration;
 import org.apache.ignite.internal.visor.tx.VisorTxInfo;
 import org.apache.ignite.internal.visor.tx.VisorTxOperation;
 import org.apache.ignite.internal.visor.tx.VisorTxProjection;
@@ -99,6 +110,7 @@
 import org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult;
 import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskResult;
+import org.apache.ignite.internal.visor.verify.VisorViewCacheCmd;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTask;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskResult;
@@ -117,11 +129,21 @@
 import static org.apache.ignite.internal.commandline.Command.STATE;
 import static org.apache.ignite.internal.commandline.Command.TX;
 import static org.apache.ignite.internal.commandline.Command.WAL;
+import static org.apache.ignite.internal.commandline.OutputFormat.MULTI_LINE;
+import static org.apache.ignite.internal.commandline.OutputFormat.SINGLE_LINE;
+import static org.apache.ignite.internal.commandline.cache.CacheCommand.CONTENTION;
+import static org.apache.ignite.internal.commandline.cache.CacheCommand.DISTRIBUTION;
+import static org.apache.ignite.internal.commandline.cache.CacheCommand.HELP;
+import static org.apache.ignite.internal.commandline.cache.CacheCommand.IDLE_VERIFY;
+import static org.apache.ignite.internal.commandline.cache.CacheCommand.LIST;
+import static org.apache.ignite.internal.commandline.cache.CacheCommand.RESET_LOST_PARTITIONS;
+import static org.apache.ignite.internal.commandline.cache.CacheCommand.VALIDATE_INDEXES;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.ADD;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.COLLECT;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.REMOVE;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.SET;
 import static org.apache.ignite.internal.visor.baseline.VisorBaselineOperation.VERSION;
+import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.CACHES;
 import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.GROUPS;
 import static org.apache.ignite.internal.visor.verify.VisorViewCacheCmd.SEQ;
 
@@ -129,9 +151,6 @@
  * Class that execute several commands passed via command line.
  */
 public class CommandHandler {
-    /** Logger. */
-    private static final Logger log = Logger.getLogger(CommandHandler.class.getName());
-
     /** */
     static final String DFLT_HOST = "127.0.0.1";
 
@@ -320,6 +339,27 @@ public class CommandHandler {
     /** */
     private static final String TX_KILL = "kill";
 
+    /** */
+    private static final String OUTPUT_FORMAT = "--output-format";
+
+    /** */
+    private static final String CONFIG = "--config";
+
+    /** Utility name. */
+    private static final String UTILITY_NAME = "control.sh";
+
+    /** Common options. */
+    private static final String COMMON_OPTIONS = String.join(" ", op(CMD_HOST, "HOST_OR_IP"), op(CMD_PORT, "PORT"), op(CMD_USER, "USER"), op(CMD_PASSWORD, "PASSWORD"), op(CMD_PING_INTERVAL, "PING_INTERVAL"), op(CMD_PING_TIMEOUT, "PING_TIMEOUT"), op(CMD_SSL_ENABLED), op(CMD_SSL_KEY_STORE_PATH,"PATH"),op(CMD_SSL_KEY_STORE_TYPE, "jks"), op(CMD_SSL_KEY_STORE_PASSWORD, "PASSWORD"), op(CMD_SSL_TRUSTSTORE_PATH, "PATH"), op(CMD_SSL_TRUSTSTORE_TYPE, "jks"), op(CMD_SSL_TRUSTSTORE_PASSWORD, "PASSWORD"));
+
+    /** Utility name with common options. */
+    private static final String UTILITY_NAME_WITH_COMMON_OPTIONS = String.join(" ", UTILITY_NAME, COMMON_OPTIONS);
+
+    /** Indent for help output. */
+    private static final String INDENT = "  ";
+
+    /** */
+    private static final String NULL = "null";
+
     /** */
     private Iterator<String> argsIt;
 
@@ -344,6 +384,54 @@ private void log(String s) {
         System.out.println(s);
     }
 
+    /**
+     * Adds indent to begin of input string.
+     *
+     * @param s Input string.
+     * @return Indented string.
+     */
+    private static String i(String s) {
+        return i(s, 1);
+    }
+
+    /**
+     * Adds specified indents to begin of input string.
+     *
+     * @param s Input string.
+     * @param indentCnt Number of indents.
+     * @return Indented string.
+     */
+    private static String i(String s, int indentCnt) {
+        assert indentCnt >= 0;
+
+        switch (indentCnt) {
+            case 0:
+                return s;
+
+            case 1:
+                return INDENT + s;
+
+            default:
+                SB sb = new SB(s.length() + indentCnt * INDENT.length());
+
+                for (int i = 0; i < indentCnt; i++)
+                    sb.a(INDENT);
+
+                return sb.a(s).toString();
+        }
+
+    }
+
+    /**
+     * Format and output specified string to console.
+     *
+     * @param format A format string as described in Format string syntax.
+     * @param args Arguments referenced by the format specifiers in the format string.
+     */
+    private void log(String format, Object... args) {
+        System.out.printf(format, args);
+    }
+
     /**
      * Provides a prompt, then reads a single line of text from the console.
      *
@@ -360,7 +448,7 @@ private String readLine(String prompt) {
      * Output empty line.
      */
     private void nl() {
-        System.out.println("");
+        System.out.println();
     }
 
     /**
@@ -658,19 +746,21 @@ private void cache(GridClient client, CacheArguments cacheArgs) throws Throwable
      *
      */
     private void printCacheHelp() {
-        log("--cache subcommand allows to do the following operations:");
-
-        usage("  Show information about caches, groups or sequences that match a regex:", CACHE, " list regexPattern [groups|seq] [nodeId]");
-        usage("  Show hot keys that are point of contention for multiple transactions:", CACHE, " contention minQueueSize [nodeId] [maxPrint]");
-        usage("  Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [--dump] [--skipZeros] [cache1,...,cacheN]");
-        usage("  Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId] [checkFirst|checkThrough]");
-        usage("  Collect partition distribution information:", CACHE, " distribution nodeId|null [cacheName1,...,cacheNameN] [--user-attributes attributeName1[,...,attributeNameN]]");
-        usage("  Reset lost partitions:", CACHE, " reset_lost_partitions cacheName1[,...,cacheNameN]");
-
-        log("  If [nodeId] is not specified, contention and validate_indexes commands will be broadcasted to all server nodes.");
-        log("  Another commands where [nodeId] is optional will run on a random server node.");
-        log("  checkFirst numeric parameter for validate_indexes specifies number of first K keys to be validated.");
-        log("  checkThrough numeric parameter for validate_indexes allows to check each Kth key.");
+        log(i("The '" + CACHE.text() + " subcommand' is used to get information about and perform actions with caches. The command has the following syntax:"));
+        nl();
+        log(i(UTILITY_NAME_WITH_COMMON_OPTIONS + " " + CACHE.text() + "[subcommand] <subcommand_parameters>"));
+        nl();
+        log(i("The subcommands that take [nodeId] as an argument ('" + LIST.text() + "', '" + CONTENTION.text() + "' and '" + VALIDATE_INDEXES.text() + "') will be executed on the given node or on all server nodes if the option is not specified. Other commands will run on a random server node."));
+        nl();
+        nl();
+        log(i("Subcommands:"));
+
+        usageCache(LIST, "regexPattern", "[groups|seq]", "[nodeId]", op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE.text()));
+        usageCache(CONTENTION, "minQueueSize", "[nodeId]", "[maxPrint]");
+        usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]");
+        usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", "[nodeId]", op(or(VI_CHECK_FIRST + " N", VI_CHECK_THROUGH + " K")));
+        usageCache(DISTRIBUTION, or("nodeId", NULL), "[cacheName1,...,cacheNameN]", op(CMD_USER_ATTRIBUTES, "attName1,...,attrNameN"));
+        usageCache(RESET_LOST_PARTITIONS, "cacheName1,...,cacheNameN");
         nl();
     }
 
@@ -781,8 +871,11 @@ private void cacheView(GridClient client, CacheArguments cacheArgs) throws GridC
         VisorViewCacheTaskResult res = executeTaskByNameOnNode(
             client, VisorViewCacheTask.class.getName(), taskArg, cacheArgs.nodeId());
 
-        for (CacheInfo info : res.cacheInfos())
-            info.print(cacheArgs.cacheCommand());
+        if (cacheArgs.fullConfig() && cacheArgs.cacheCommand() == CACHES)
+            cachesConfig(client, cacheArgs, res);
+        else
+            printCacheInfos(res.cacheInfos(), cacheArgs.cacheCommand());
+
     }
 
     /**
@@ -857,6 +950,101 @@ private void cacheDistribution(GridClient client, CacheArguments cacheArgs) thro
         res.print(System.out);
     }
 
+    /**
+     * @param client Client.
+     * @param cacheArgs Cache args.
+     * @param viewRes Cache view task result.
+     */
+    private void cachesConfig(GridClient client, CacheArguments cacheArgs,
+        VisorViewCacheTaskResult viewRes) throws GridClientException {
+        VisorCacheConfigurationCollectorTaskArg taskArg = new VisorCacheConfigurationCollectorTaskArg(cacheArgs.regex());
+
+        UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId();
+
+        Map<String, VisorCacheConfiguration> res =
+            executeTaskByNameOnNode(client, VisorCacheConfigurationCollectorTask.class.getName(), taskArg, nodeId);
+
+        Map<String, Integer> cacheToMapped =
+            viewRes.cacheInfos().stream().collect(Collectors.toMap(x -> x.getCacheName(), x -> x.getMapped()));
+
+        printCachesConfig(res, cacheArgs.outputFormat(), cacheToMapped);
+
+    }
+
+    /**
+     * Prints caches info.
+     *
+     * @param infos Caches info.
+     * @param cmd Command.
+     */
+    private void printCacheInfos(Collection<CacheInfo> infos, VisorViewCacheCmd cmd) {
+        for (CacheInfo info : infos) {
+            Map<String, Object> map = info.toMap(cmd);
+
+            SB sb = new SB("[");
+
+            for (Map.Entry<String, Object> e : map.entrySet())
+                sb.a(e.getKey()).a("=").a(e.getValue()).a(", ");
+
+            sb.setLength(sb.length() - 2);
+
+            sb.a("]");
+
+            log(sb.toString());
+        }
+    }
+
+    /**
+     * Prints caches config.
+     *
+     * @param caches Caches config.
+     * @param outputFormat Output format.
+     * @param cacheToMapped Map cache name to mapped.
+     */
+    private void printCachesConfig(
+        Map<String, VisorCacheConfiguration> caches,
+        OutputFormat outputFormat,
+        Map<String, Integer> cacheToMapped
+    ) {
+
+        for (Map.Entry<String, VisorCacheConfiguration> entry : caches.entrySet()) {
+            String cacheName = entry.getKey();
+
+            switch (outputFormat) {
+                case MULTI_LINE:
+                    Map<String, Object> params = mapToPairs(entry.getValue());
+
+                    params.put("Mapped", cacheToMapped.get(cacheName));
+
+                    log("[cache = '%s']%n", cacheName);
+
+                    for (Map.Entry<String, Object> innerEntry : params.entrySet())
+                        log("%s: %s%n", innerEntry.getKey(), innerEntry.getValue());
+
+                    nl();
+
+                    break;
+
+                default:
+                    int mapped = cacheToMapped.get(cacheName);
+
+                    log("%s: %s %s=%s%n", entry.getKey(), toString(entry.getValue()), "mapped", mapped);
+
+                    break;
+            }
+        }
+    }
+
+    /**
+     * Invokes toString() method and cuts class name from result string.
+     *
+     * @param cfg Visor cache configuration for invocation.
+     * @return String representation without class name in begin of string.
+     */
+    private String toString(VisorCacheConfiguration cfg) {
+        return cfg.toString().substring(cfg.getClass().getSimpleName().length() + 1);
+    }
+
     /**
      * @param client Client.
      * @param cacheArgs Cache args.
@@ -995,8 +1183,9 @@ private void baselinePrint0(VisorBaselineTaskResult res) {
             log("Baseline nodes:");
 
             for (VisorBaselineNode node : baseline.values()) {
-                log("    ConsistentID=" + node.getConsistentId() + ", STATE=" +
-                    (srvs.containsKey(node.getConsistentId()) ? "ONLINE" : "OFFLINE"));
+                boolean online = srvs.containsKey(node.getConsistentId());
+
+                log(i("ConsistentID=" + node.getConsistentId() + ", STATE=" + (online ? "ONLINE" : "OFFLINE"), 2));
             }
 
             log(DELIM);
@@ -1017,7 +1206,7 @@ private void baselinePrint0(VisorBaselineTaskResult res) {
                 log("Other nodes:");
 
                 for (VisorBaselineNode node : others)
-                    log("    ConsistentID=" + node.getConsistentId());
+                    log(i("ConsistentID=" + node.getConsistentId(), 2));
 
                 log("Number of other nodes: " + others.size());
             }
@@ -1250,10 +1439,10 @@ private void printUnusedWalSegments0(VisorWalTaskResult taskRes) {
             VisorClusterNode node = nodesInfo.get(entry.getKey());
 
             log("Node=" + node.getConsistentId());
-            log("     addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
+            log(i("addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()), 2));
 
             for (String fileName : entry.getValue())
-                log("   " + fileName);
+                log(i(fileName));
 
             nl();
         }
@@ -1262,8 +1451,8 @@ private void printUnusedWalSegments0(VisorWalTaskResult taskRes) {
             VisorClusterNode node = nodesInfo.get(entry.getKey());
 
             log("Node=" + node.getConsistentId());
-            log("     addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
-            log("   failed with error: " + entry.getValue().getMessage());
+            log(i("addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames())), 2);
+            log(i("failed with error: " + entry.getValue().getMessage()));
             nl();
         }
     }
@@ -1285,7 +1474,7 @@ private void printDeleteWalSegments0(VisorWalTaskResult taskRes) {
             VisorClusterNode node = nodesInfo.get(entry.getKey());
 
             log("Node=" + node.getConsistentId());
-            log("     addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
+            log(i("addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames())), 2);
             nl();
         }
 
@@ -1293,8 +1482,8 @@ private void printDeleteWalSegments0(VisorWalTaskResult taskRes) {
             VisorClusterNode node = nodesInfo.get(entry.getKey());
 
             log("Node=" + node.getConsistentId());
-            log("     addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames()));
-            log("   failed with error: " + entry.getValue().getMessage());
+            log(i("addresses " + U.addressesAsString(node.getAddresses(), node.getHostNames())), 2);
+            log(i("failed with error: " + entry.getValue().getMessage()));
             nl();
         }
     }
@@ -1327,19 +1516,170 @@ private boolean isConnectionError(Throwable e) {
      */
     private void usage(String desc, Command cmd, String... args) {
         log(desc);
-        log("    control.sh [--host HOST_OR_IP] [--port PORT] [--user USER] [--password PASSWORD] " +
-            " [--ping-interval PING_INTERVAL] [--ping-timeout PING_TIMEOUT] " +
-            "[" + CMD_SSL_ENABLED + "] " +
-            "[" + CMD_SSL_KEY_STORE_PATH + " PATH] " +
-            "[" + CMD_SSL_KEY_STORE_TYPE + " jks] " +
-            "[" + CMD_SSL_KEY_STORE_PASSWORD + " PASSWORD] " +
-            "[" + CMD_SSL_TRUSTSTORE_PATH + " PATH] " +
-            "[" + CMD_SSL_TRUSTSTORE_TYPE + " jks] " +
-            "[" + CMD_SSL_TRUSTSTORE_PASSWORD + " PASSWORD]" +
-            cmd.text() + String.join("", args));
+        log(i(UTILITY_NAME_WITH_COMMON_OPTIONS + " " + cmd.text() + " " + String.join(" ", args), 2));
         nl();
     }
 
+    /**
+     * Print cache command usage with default indention.
+     *
+     * @param cmd Cache command.
+     * @param args Cache command arguments.
+     */
+    private void usageCache(CacheCommand cmd, String... args) {
+        usageCache(1, cmd, args);
+    }
+
+    /**
+     * Print cache command usage.
+     *
+     * @param indentsNum Number of indents.
+     * @param cmd Cache command.
+     * @param args Cache command arguments.
+     */
+    private void usageCache(int indentsNum, CacheCommand cmd, String... args) {
+        log(i(DELIM, indentsNum));
+        nl();
+        log(i(CACHE.text() + " " + cmd.text() + " " + String.join(" ", args), indentsNum++));
+        nl();
+        log(i(getCacheSubcommandDesc(cmd), indentsNum));
+        nl();
+
+        Map<String, String> paramsDesc = createCacheArgsDesc(cmd);
+
+        if (!paramsDesc.isEmpty()) {
+            log(i("Parameters:", indentsNum));
+
+            usageCacheParams(paramsDesc, indentsNum + 1);
+
+            nl();
+        }
+    }
+
+    /**
+     * Print cache command arguments usage.
+     *
+     * @param paramsDesc Cache command arguments description.
+     * @param indentsNum Number of indents.
+     */
+    private void usageCacheParams(Map<String, String> paramsDesc, int indentsNum) {
+        int maxParamLen = paramsDesc.keySet().stream().max(Comparator.comparingInt(String::length)).get().length();
+
+        for (Map.Entry<String, String> param : paramsDesc.entrySet())
+            log(i(extendToLen(param.getKey(), maxParamLen) + INDENT + "- " + param.getValue(), indentsNum));
+    }
+
+    /**
+     * Appends spaces to end of input string for extending to needed length.
+     *
+     * @param s Input string.
+     * @param targetLen Needed length.
+     * @return String with appended spaces on the end.
+     */
+    private String extendToLen(String s, int targetLen) {
+        assert targetLen >= 0;
+        assert s.length() <= targetLen;
+
+        if (s.length() == targetLen)
+            return s;
+
+        SB sb = new SB(targetLen);
+
+        sb.a(s);
+
+        for (int i = 0; i < targetLen - s.length(); i++)
+            sb.a(" ");
+
+        return sb.toString();
+    }
+
+    /**
+     * Gets cache command description by cache command.
+     *
+     * @param cmd Cache command.
+     * @return Cache command description.
+     */
+    private String getCacheSubcommandDesc(CacheCommand cmd) {
+        switch (cmd) {
+            case LIST:
+                return "Show information about caches, groups or sequences that match a regular expression. When executed without parameters, this subcommand prints the list of caches.";
+
+            case CONTENTION:
+                return "Show the keys that are point of contention for multiple transactions.";
+
+            case IDLE_VERIFY:
+                return "Verify counters and hash sums of primary and backup partitions for the specified caches on an idle cluster and print out the differences, if any.";
+
+            case VALIDATE_INDEXES:
+                return "Validate indexes on an idle cluster and print out the keys that are missing in the indexes.";
+
+            case DISTRIBUTION:
+                return "Prints the information about partition distribution.";
+
+            case RESET_LOST_PARTITIONS:
+                return "Reset the state of lost partitions for the specified caches.";
+
+            default:
+                throw new IllegalArgumentException("Unknown command: " + cmd);
+        }
+    }
+
+    /**
+     * Gets cache command arguments description by cache command.
+     *
+     * @param cmd Cache command.
+     * @return Cache command arguments description.
+     */
+    private Map<String, String> createCacheArgsDesc(CacheCommand cmd) {
+        Map<String, String> map = U.newLinkedHashMap(16);
+        switch (cmd) {
+            case LIST:
+                map.put(CONFIG, "print a all configuration parameters for each cache.");
+                map.put(OUTPUT_FORMAT + " " + MULTI_LINE.text(), "print configuration parameters per line. This option has effect only when used with " + CONFIG + " and without [groups|seq].");
+
+                break;
+            case VALIDATE_INDEXES:
+                map.put(VI_CHECK_FIRST + " N", "validate only the first N keys");
+                map.put(VI_CHECK_THROUGH + " K", "validate every Kth key");
+
+                break;
+        }
+        return map;
+    }
+
+    /**
+     * Join input parameters with space and wrap optional braces {@code []}.
+     *
+     * @param param First input parameter.
+     * @param params Other input parameter.
+     * @return Joined parameters wrapped optional braces.
+     */
+    private static String op(String param, String... params) {
+        if (params == null || params.length == 0)
+            return "[" + param + "]";
+
+        return "[" + param + " " + String.join(" ", params) + "]";
+    }
+
+    /**
+     * Concatenates input parameters to single string with OR delimiter {@code |}.
+     *
+     * @param param1 First parameter.
+     * @param params Remaining parameters.
+     * @return Concatenated string.
+     */
+    private static String or(String param1, String... params) {
+        if (params.length == 0)
+            return param1;
+
+        SB sb = new SB(param1);
+
+        for (String param : params)
+            sb.a("|").a(param);
+
+        return sb.toString();
+    }
+
     /**
      * Extract next argument.
      *
@@ -1727,27 +2067,27 @@ else if (CMD_SKIP_ZEROS.equals(nextArg))
 
             case DISTRIBUTION:
                 String nodeIdStr = nextArg("Node id expected or null");
-                if (!"null".equals(nodeIdStr))
+                if (!NULL.equals(nodeIdStr))
                     cacheArgs.nodeId(UUID.fromString(nodeIdStr));
 
                 while (hasNextCacheArg()) {
                     String nextArg = nextArg("");
 
-                    if (CMD_USER_ATTRIBUTES.equals(nextArg)){
+                    if (CMD_USER_ATTRIBUTES.equals(nextArg)) {
                         nextArg = nextArg("User attributes are expected to be separated by commas");
 
-                        Set<String> userAttributes = new HashSet();
+                        Set<String> userAttrs = new HashSet<>();
 
-                        for (String userAttribute:nextArg.split(","))
-                            userAttributes.add(userAttribute.trim());
+                        for (String userAttribute : nextArg.split(","))
+                            userAttrs.add(userAttribute.trim());
 
-                        cacheArgs.setUserAttributes(userAttributes);
+                        cacheArgs.setUserAttributes(userAttrs);
 
                         nextArg = (hasNextCacheArg()) ? nextArg("") : null;
 
                     }
 
-                    if (nextArg!=null)
+                    if (nextArg != null)
                         parseCacheNames(nextArg, cacheArgs);
                 }
 
@@ -1758,20 +2098,36 @@ else if (CMD_SKIP_ZEROS.equals(nextArg))
 
                 break;
 
-            default:
+            case LIST:
                 cacheArgs.regex(nextArg("Regex is expected"));
 
-                if (hasNextCacheArg()) {
-                    String tmp = nextArg("");
+                VisorViewCacheCmd cacheCmd = CACHES;
+
+                OutputFormat outputFormat = SINGLE_LINE;
+
+                while (hasNextCacheArg()) {
+                    String tmp = nextArg("").toLowerCase();
 
                     switch (tmp) {
                         case "groups":
-                            cacheArgs.cacheCommand(GROUPS);
+                            cacheCmd = GROUPS;
 
                             break;
 
                         case "seq":
-                            cacheArgs.cacheCommand(SEQ);
+                            cacheCmd = SEQ;
+
+                            break;
+
+                        case OUTPUT_FORMAT:
+                            String tmp2 = nextArg("output format must be defined!").toLowerCase();
+
+                            outputFormat = OutputFormat.fromConsoleName(tmp2);
+
+                            break;
+
+                        case CONFIG:
+                            cacheArgs.fullConfig(true);
 
                             break;
 
@@ -1780,7 +2136,13 @@ else if (CMD_SKIP_ZEROS.equals(nextArg))
                     }
                 }
 
+                cacheArgs.cacheCommand(cacheCmd);
+                cacheArgs.outputFormat(outputFormat);
+
                 break;
+
+            default:
+                throw new IllegalArgumentException("Unknown --cache subcommand " + cmd);
         }
 
         if (hasNextCacheArg())
@@ -1975,6 +2337,107 @@ private boolean isCommandOrOption(String raw) {
         return raw != null && raw.contains("--");
     }
 
+    /**
+     * Maps VisorCacheConfiguration to key-value pairs.
+     *
+     * @param cfg Visor cache configuration.
+     * @return map of key-value pairs.
+     */
+    private Map<String, Object> mapToPairs(VisorCacheConfiguration cfg) {
+        Map<String, Object> params = new LinkedHashMap<>();
+
+        VisorCacheAffinityConfiguration affinityCfg = cfg.getAffinityConfiguration();
+        VisorCacheNearConfiguration nearCfg = cfg.getNearConfiguration();
+        VisorCacheRebalanceConfiguration rebalanceCfg = cfg.getRebalanceConfiguration();
+        VisorCacheEvictionConfiguration evictCfg = cfg.getEvictionConfiguration();
+        VisorCacheStoreConfiguration storeCfg = cfg.getStoreConfiguration();
+        VisorQueryConfiguration qryCfg = cfg.getQueryConfiguration();
+
+        params.put("Name", cfg.getName());
+        params.put("Group", cfg.getGroupName());
+        params.put("Dynamic Deployment ID", cfg.getDynamicDeploymentId());
+        params.put("System", cfg.isSystem());
+
+        params.put("Mode", cfg.getMode());
+        params.put("Atomicity Mode", cfg.getAtomicityMode());
+        params.put("Statistic Enabled", cfg.isStatisticsEnabled());
+        params.put("Management Enabled", cfg.isManagementEnabled());
+
+        params.put("On-heap cache enabled", cfg.isOnheapCacheEnabled());
+        params.put("Partition Loss Policy", cfg.getPartitionLossPolicy());
+        params.put("Query Parallelism", cfg.getQueryParallelism());
+        params.put("Copy On Read", cfg.isCopyOnRead());
+        params.put("Listener Configurations", cfg.getListenerConfigurations());
+        params.put("Load Previous Value", cfg.isLoadPreviousValue());
+        params.put("Memory Policy Name", cfg.getMemoryPolicyName());
+        params.put("Node Filter", cfg.getNodeFilter());
+        params.put("Read From Backup", cfg.isReadFromBackup());
+        params.put("Topology Validator", cfg.getTopologyValidator());
+
+        params.put("Time To Live Eager Flag", cfg.isEagerTtl());
+
+        params.put("Write Synchronization Mode", cfg.getWriteSynchronizationMode());
+        params.put("Invalidate", cfg.isInvalidate());
+
+        params.put("Affinity Function", affinityCfg.getFunction());
+        params.put("Affinity Backups", affinityCfg.getPartitionedBackups());
+        params.put("Affinity Partitions", affinityCfg.getPartitions());
+        params.put("Affinity Exclude Neighbors", affinityCfg.isExcludeNeighbors());
+        params.put("Affinity Mapper", affinityCfg.getMapper());
+
+        params.put("Rebalance Mode", rebalanceCfg.getMode());
+        params.put("Rebalance Batch Size", rebalanceCfg.getBatchSize());
+        params.put("Rebalance Timeout", rebalanceCfg.getTimeout());
+        params.put("Rebalance Delay", rebalanceCfg.getPartitionedDelay());
+        params.put("Time Between Rebalance Messages", rebalanceCfg.getThrottle());
+        params.put("Rebalance Batches Count", rebalanceCfg.getBatchesPrefetchCnt());
+        params.put("Rebalance Cache Order", rebalanceCfg.getRebalanceOrder());
+
+        params.put("Eviction Policy Enabled", (evictCfg.getPolicy() != null));
+        params.put("Eviction Policy Factory", evictCfg.getPolicy());
+        params.put("Eviction Policy Max Size", evictCfg.getPolicyMaxSize());
+        params.put("Eviction Filter", evictCfg.getFilter());
+
+        params.put("Near Cache Enabled", nearCfg.isNearEnabled());
+        params.put("Near Start Size", nearCfg.getNearStartSize());
+        params.put("Near Eviction Policy Factory", nearCfg.getNearEvictPolicy());
+        params.put("Near Eviction Policy Max Size", nearCfg.getNearEvictMaxSize());
+
+        params.put("Default Lock Timeout", cfg.getDefaultLockTimeout());
+        params.put("Query Entities", cfg.getQueryEntities());
+        params.put("Cache Interceptor", cfg.getInterceptor());
+
+        params.put("Store Enabled", storeCfg.isEnabled());
+        params.put("Store Class", storeCfg.getStore());
+        params.put("Store Factory Class", storeCfg.getStoreFactory());
+        params.put("Store Keep Binary", storeCfg.isStoreKeepBinary());
+        params.put("Store Read Through", storeCfg.isReadThrough());
+        params.put("Store Write Through", storeCfg.isWriteThrough());
+        params.put("Store Write Coalescing", storeCfg.getWriteBehindCoalescing());
+
+        params.put("Write-Behind Enabled", storeCfg.isWriteBehindEnabled());
+        params.put("Write-Behind Flush Size", storeCfg.getFlushSize());
+        params.put("Write-Behind Frequency", storeCfg.getFlushFrequency());
+        params.put("Write-Behind Flush Threads Count", storeCfg.getFlushThreadCount());
+        params.put("Write-Behind Batch Size", storeCfg.getBatchSize());
+
+        params.put("Concurrent Asynchronous Operations Number", cfg.getMaxConcurrentAsyncOperations());
+
+        params.put("Loader Factory Class Name", cfg.getLoaderFactory());
+        params.put("Writer Factory Class Name", cfg.getWriterFactory());
+        params.put("Expiry Policy Factory Class Name", cfg.getExpiryPolicyFactory());
+
+        params.put("Query Execution Time Threshold", qryCfg.getLongQueryWarningTimeout());
+        params.put("Query Escaped Names", qryCfg.isSqlEscapeAll());
+        params.put("Query SQL Schema", qryCfg.getSqlSchema());
+        params.put("Query SQL functions", qryCfg.getSqlFunctionClasses());
+        params.put("Query Indexed Types", qryCfg.getIndexedTypes());
+        params.put("Maximum payload size for offheap indexes", cfg.getSqlIndexMaxInlineSize());
+        params.put("Query Metrics History Size", cfg.getQueryDetailMetricsSize());
+
+        return params;
+    }
+
     /**
      * Parse and execute command.
      *
@@ -1991,27 +2454,23 @@ public int execute(List<String> rawArgs) {
             if (F.isEmpty(rawArgs) || (rawArgs.size() == 1 && CMD_HELP.equalsIgnoreCase(rawArgs.get(0)))) {
                 log("This utility can do the following commands:");
 
-                usage("  Activate cluster:", ACTIVATE);
-                usage("  Deactivate cluster:", DEACTIVATE, " [" + CMD_AUTO_CONFIRMATION + "]");
-                usage("  Print current cluster state:", STATE);
-                usage("  Print cluster baseline topology:", BASELINE);
-                usage("  Add nodes into baseline topology:", BASELINE, " add consistentId1[,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
-                usage("  Remove nodes from baseline topology:", BASELINE, " remove consistentId1[,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
-                usage("  Set baseline topology:", BASELINE, " set consistentId1[,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
-                usage("  Set baseline topology based on version:", BASELINE, " version topologyVersion [" + CMD_AUTO_CONFIRMATION + "]");
-                usage("  List or kill transactions:", TX, " [xid XID] [minDuration SECONDS] " +
-                    "[minSize SIZE] [label PATTERN_REGEX] [servers|clients] " +
-                    "[nodes consistentId1[,consistentId2,....,consistentIdN] [limit NUMBER] [order DURATION|SIZE|", CMD_TX_ORDER_START_TIME, "] [kill] [" + CMD_AUTO_CONFIRMATION + "]");
+                usage(i("Activate cluster:"), ACTIVATE);
+                usage(i("Deactivate cluster:"), DEACTIVATE, op(CMD_AUTO_CONFIRMATION));
+                usage(i("Print current cluster state:"), STATE);
+                usage(i("Print cluster baseline topology:"), BASELINE);
+                usage(i("Add nodes into baseline topology:"), BASELINE, BASELINE_ADD, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION));
+                usage(i("Remove nodes from baseline topology:"), BASELINE, BASELINE_REMOVE, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION));
+                usage(i("Set baseline topology:"), BASELINE, BASELINE_SET, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION));
+                usage(i("Set baseline topology based on version:"), BASELINE, BASELINE_SET_VERSION + " topologyVersion", op(CMD_AUTO_CONFIRMATION));
+                usage(i("List or kill transactions:"), TX, op(TX_XID, "XID"), op(TX_DURATION, "SECONDS"), op(TX_SIZE, "SIZE"), op(TX_LABEL, "PATTERN_REGEX"), op(or(TX_SERVERS, TX_CLIENTS)), op(TX_NODES, "consistentId1[,consistentId2,....,consistentIdN]"), op(TX_LIMIT, "NUMBER"), op(TX_ORDER, or("DURATION", "SIZE", CMD_TX_ORDER_START_TIME)), op(TX_KILL), op(CMD_AUTO_CONFIRMATION));
 
                 if (enableExperimental) {
-                    usage("  Print absolute paths of unused archived wal segments on each node:", WAL,
-                        " print [consistentId1,consistentId2,....,consistentIdN]");
-                    usage("  Delete unused archived wal segments on each node:", WAL,
-                        " delete [consistentId1,consistentId2,....,consistentIdN] [" + CMD_AUTO_CONFIRMATION + "]");
+                    usage(i("Print absolute paths of unused archived wal segments on each node:"), WAL, WAL_PRINT, "[consistentId1,consistentId2,....,consistentIdN]");
+                    usage(i("Delete unused archived wal segments on each node:"), WAL, WAL_DELETE, "[consistentId1,consistentId2,....,consistentIdN] ", op(CMD_AUTO_CONFIRMATION));
                 }
 
-                log("  View caches information in a cluster. For more details type:");
-                log("    control.sh --cache help");
+                log(i("View caches information in a cluster. For more details type:"));
+                log(i(String.join(" ", UTILITY_NAME, CACHE.text(), HELP.text()), 2));
                 nl();
 
                 log("By default commands affecting the cluster require interactive confirmation.");
@@ -2019,24 +2478,30 @@ public int execute(List<String> rawArgs) {
                 nl();
 
                 log("Default values:");
-                log("    HOST_OR_IP=" + DFLT_HOST);
-                log("    PORT=" + DFLT_PORT);
-                log("    PING_INTERVAL=" + DFLT_PING_INTERVAL);
-                log("    PING_TIMEOUT=" + DFLT_PING_TIMEOUT);
+                log(i("HOST_OR_IP=" + DFLT_HOST, 2));
+                log(i("PORT=" + DFLT_PORT, 2));
+                log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2));
+                log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2));
                 nl();
 
                 log("Exit codes:");
-                log("    " + EXIT_CODE_OK + " - successful execution.");
-                log("    " + EXIT_CODE_INVALID_ARGUMENTS + " - invalid arguments.");
-                log("    " + EXIT_CODE_CONNECTION_FAILED + " - connection failed.");
-                log("    " + ERR_AUTHENTICATION_FAILED + " - authentication failed.");
-                log("    " + EXIT_CODE_UNEXPECTED_ERROR + " - unexpected error.");
+                log(i(EXIT_CODE_OK + " - successful execution.", 2));
+                log(i(EXIT_CODE_INVALID_ARGUMENTS + " - invalid arguments.", 2));
+                log(i(EXIT_CODE_CONNECTION_FAILED + " - connection failed.", 2));
+                log(i(ERR_AUTHENTICATION_FAILED + " - authentication failed.", 2));
+                log(i(EXIT_CODE_UNEXPECTED_ERROR + " - unexpected error.", 2));
 
                 return EXIT_CODE_OK;
             }
 
             Arguments args = parseAndValidate(rawArgs);
 
+            if (args.command() == CACHE && args.cacheArgs().command() == HELP) {
+                printCacheHelp();
+
+                return EXIT_CODE_OK;
+            }
+
             if (!args.autoConfirmation() && !confirm(args)) {
                 log("Operation cancelled.");
 
@@ -2125,7 +2590,7 @@ public int execute(List<String> rawArgs) {
                 }
             }
 
-            return 0;
+            return EXIT_CODE_OK;
         }
         catch (IllegalArgumentException e) {
             return error(EXIT_CODE_INVALID_ARGUMENTS, "Check arguments.", e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/OutputFormat.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/OutputFormat.java
new file mode 100644
index 0000000000000..356cb4b2e88e5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/OutputFormat.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.commandline;
+
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public enum OutputFormat {
+    /** Single line. */
+    SINGLE_LINE("single-line"),
+
+    /** Multi line. */
+    MULTI_LINE("multi-line");
+
+    /** */
+    private final String text;
+
+    /** */
+    OutputFormat(String text) {
+        this.text = text;
+    }
+
+    /**
+     * @return Text.
+     */
+    public String text() {
+        return text;
+    }
+
+    /**
+     * Converts format name in console to enumerated value.
+     *
+     * @param text Format name in console.
+     * @return Enumerated value.
+     * @throws IllegalArgumentException If enumerated value not found.
+     */
+    public static OutputFormat fromConsoleName(@NotNull String text) {
+        for (OutputFormat format : values()) {
+            if (format.text.equals(text))
+                return format;
+        }
+
+        throw new IllegalArgumentException("Unknown output format " + text);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return text;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
index 97d234aeb879d..f7146103a086d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
@@ -18,6 +18,7 @@
 
 import java.util.Set;
 import java.util.UUID;
+import org.apache.ignite.internal.commandline.OutputFormat;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheCmd;
 import org.jetbrains.annotations.Nullable;
 
@@ -64,6 +65,22 @@ public class CacheArguments {
     /** Additional user attributes in result. Set of attribute names whose values will be searched in ClusterNode.attributes(). */
     private Set<String> userAttributes;
 
+    /** Output format. */
+    private OutputFormat outputFormat;
+
+    /** Full config flag. */
+    private boolean fullConfig;
+
+    /**
+     * @return Full config flag.
+     */
+    public boolean fullConfig(){ return fullConfig; }
+
+    /**
+     * @param fullConfig New full config flag.
+     */
+    public void fullConfig(boolean fullConfig) { this.fullConfig = fullConfig; }
+
     /**
      * @return Command.
      */
@@ -245,4 +262,14 @@ public Set<String> getUserAttributes() {
     public void setUserAttributes(Set<String> userAttrs) {
         userAttributes = userAttrs;
     }
+
+    /**
+     * @return Output format.
+     */
+    public OutputFormat outputFormat() { return outputFormat; }
+
+    /**
+     * @param outputFormat New output format.
+     */
+    public void outputFormat(OutputFormat outputFormat) { this.outputFormat = outputFormat; }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java
index 31c0b3fcbe2d0..01231990050bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/CacheInfo.java
@@ -20,7 +20,8 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
-
+import java.util.LinkedHashMap;
+import java.util.Map;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -238,7 +239,7 @@ public CacheAtomicityMode getAtomicityMode() {
     }
 
     /**
-     * @param atomicityMode
+     * @param atomicityMode Atomicity mode.
      */
     public void setAtomicityMode(CacheAtomicityMode atomicityMode) {
         this.atomicityMode = atomicityMode;
@@ -272,32 +273,74 @@ public void setAffinityClsName(String affinityClsName) {
         this.affinityClsName = affinityClsName;
     }
 
+    /**
+     * Gets name of info for multi line output depending on cache command.
+     *
+     * @param cmd Cache command.
+     * @return Header.
+     */
+    public Object name(VisorViewCacheCmd cmd) {
+        switch (cmd) {
+            case CACHES:
+                return getCacheName();
+
+            case GROUPS:
+                return getGrpName();
+
+            case SEQ:
+                return getSeqName();
+
+            default:
+                throw new IllegalArgumentException("Unknown cache subcommand " + cmd);
+        }
+    }
+
     /**
      * @param cmd Command.
      */
-    public void print(VisorViewCacheCmd cmd) {
-        if (cmd == null)
-            cmd = VisorViewCacheCmd.CACHES;
+    public Map<String, Object> toMap(VisorViewCacheCmd cmd) {
+        Map<String, Object> map;
 
         switch (cmd) {
             case SEQ:
-                System.out.println("[seqName=" + getSeqName() + ", curVal=" + seqVal + ']');
+                map = new LinkedHashMap<>(2);
+
+                map.put("seqName", getSeqName());
+                map.put("curVal", seqVal);
 
                 break;
 
             case GROUPS:
-                System.out.println("[grpName=" + getGrpName() + ", grpId=" + getGrpId() + ", cachesCnt=" + getCachesCnt() +
-                    ", prim=" + getPartitions() + ", mapped=" + getMapped() + ", mode=" + getMode() +
-                    ", atomicity=" + getAtomicityMode() + ", backups=" + getBackupsCnt() + ", affCls=" + getAffinityClsName() + ']');
+                map = new LinkedHashMap<>(10);
+
+                map.put("grpName", getGrpName());
+                map.put("grpId", getGrpId());
+                map.put("cachesCnt", getCachesCnt());
+                map.put("prim", getPartitions());
+                map.put("mapped", getMapped());
+                map.put("mode", getMode());
+                map.put("atomicity", getAtomicityMode());
+                map.put("backups", getBackupsCnt());
+                map.put("affCls", getAffinityClsName());
 
                 break;
 
             default:
-                System.out.println("[cacheName=" + getCacheName() + ", cacheId=" + getCacheId() +
-                    ", grpName=" + getGrpName() + ", grpId=" + getGrpId() + ", prim=" + getPartitions() +
-                    ", mapped=" + getMapped() + ", mode=" + getMode() + ", atomicity=" + getAtomicityMode() +
-                    ", backups=" + getBackupsCnt() + ", affCls=" + getAffinityClsName() + ']');
+                map = new LinkedHashMap<>(10);
+
+                map.put("cacheName", getCacheName());
+                map.put("cacheId", getCacheId());
+                map.put("grpName", getGrpName());
+                map.put("grpId", getGrpId());
+                map.put("prim", getPartitions());
+                map.put("mapped", getMapped());
+                map.put("mode", getMode());
+                map.put("atomicity", getAtomicityMode());
+                map.put("backups", getBackupsCnt());
+                map.put("affCls", getAffinityClsName());
         }
+
+        return map;
     }
 
     /** {@inheritDoc} */
@@ -345,4 +388,4 @@ public void print(VisorViewCacheCmd cmd) {
     @Override public String toString() {
         return S.toString(CacheInfo.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java
index b0126788db819..1e9faaabaded2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfiguration.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.lang.IgniteUuid;
 import org.jetbrains.annotations.Nullable;
 
+
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.compactClass;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.compactIterable;
 
@@ -489,8 +490,8 @@ public int getQueryDetailMetricsSize() {
     }
 
     /**
-     * @return {@code true} if data can be read from backup node or {@code false} if data always
-     *      should be read from primary node and never from backup.
+     * @return {@code true} if data can be read from backup node or {@code false} if data always should be read from
+     * primary node and never from backup.
      */
     public boolean isReadFromBackup() {
         return readFromBackup;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorJob.java
index f72cd7de09421..9cf2c89b8855d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorJob.java
@@ -19,6 +19,7 @@
 
 import java.util.Collection;
 import java.util.Map;
+import java.util.regex.Pattern;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
 import org.apache.ignite.internal.util.typedef.F;
@@ -28,7 +29,7 @@
 import org.apache.ignite.lang.IgniteUuid;
 
 /**
- * Job that collect cache metrics from node.
+ * Job that collect cache configuration from node.
  */
 public class VisorCacheConfigurationCollectorJob
     extends VisorJob<VisorCacheConfigurationCollectorTaskArg, Map<String, VisorCacheConfiguration>> {
@@ -38,7 +39,7 @@ public class VisorCacheConfigurationCollectorJob
     /**
      * Create job with given argument.
      *
-     * @param arg Whether to collect metrics for all caches or for specified cache name only.
+     * @param arg Whether to collect metrics for all caches or for specified cache name only or by regex.
      * @param debug Debug flag.
      */
     public VisorCacheConfigurationCollectorJob(VisorCacheConfigurationCollectorTaskArg arg, boolean debug) {
@@ -49,18 +50,24 @@ public VisorCacheConfigurationCollectorJob(VisorCacheConfigurationCollectorTaskA
     @Override protected Map<String, VisorCacheConfiguration> run(VisorCacheConfigurationCollectorTaskArg arg) {
         Collection<IgniteCacheProxy<?, ?>> caches = ignite.context().cache().jcaches();
 
-        Collection<String> cacheNames = arg.getCacheNames();
+        Pattern ptrn = arg.getRegex() != null ? Pattern.compile(arg.getRegex()) : null;
 
-        boolean all = F.isEmpty(cacheNames);
+        boolean all = F.isEmpty(arg.getCacheNames());
 
-        Map<String, VisorCacheConfiguration> res = U.newHashMap(all ? caches.size() : cacheNames.size());
+        boolean hasPtrn = ptrn != null;
+
+        Map<String, VisorCacheConfiguration> res = U.newHashMap(caches.size());
 
         for (IgniteCacheProxy<?, ?> cache : caches) {
             String cacheName = cache.getName();
 
-            if (all || cacheNames.contains(cacheName)) {
-                res.put(cacheName, config(cache.getConfiguration(CacheConfiguration.class),
-                    cache.context().dynamicDeploymentId()));
+            boolean matched = hasPtrn ? ptrn.matcher(cacheName).find() : all || arg.getCacheNames().contains(cacheName);
+
+            if (matched) {
+                VisorCacheConfiguration cfg =
+                    config(cache.getConfiguration(CacheConfiguration.class), cache.context().dynamicDeploymentId());
+
+                res.put(cacheName, cfg);
             }
         }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java
index a0b43dbcf890d..a91f0ca4e2e70 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java
@@ -21,6 +21,7 @@
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import java.util.Collection;
+import java.util.regex.Pattern;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
@@ -32,9 +33,12 @@ public class VisorCacheConfigurationCollectorTaskArg extends VisorDataTransferOb
     /** */
     private static final long serialVersionUID = 0L;
 
-    /** Collection of cache deployment IDs. */
+    /** Collection of cache names. */
     private Collection<String> cacheNames;
 
+    /** Cache name regexp. */
+    private String regex;
+
     /**
      * Default constructor.
      */
@@ -49,6 +53,16 @@ public VisorCacheConfigurationCollectorTaskArg(Collection<String> cacheNames) {
         this.cacheNames = cacheNames;
     }
 
+    /**
+     * @param regex Cache name regexp.
+     */
+    public VisorCacheConfigurationCollectorTaskArg(String regex) {
+        // Checks, that regex is correct.
+        Pattern.compile(regex);
+
+        this.regex = regex;
+    }
+
     /**
      * @return Collection of cache deployment IDs.
      */
@@ -56,14 +70,30 @@ public Collection<String> getCacheNames() {
         return cacheNames;
     }
 
+    /**
+     * @return Cache name regexp.
+     */
+    public String getRegex() {
+        return regex;
+    }
+
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeCollection(out, cacheNames);
+        U.writeString(out, regex);
     }
 
     /** {@inheritDoc} */
     @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
         cacheNames = U.readCollection(in);
+
+        if (getProtocolVersion() > V1)
+            regex = U.readString(in);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java
index 5fcd66d18d082..6bc2369406ee9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorViewCacheTaskArg.java
@@ -1,19 +1,19 @@
 /*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License.  You may obtain a copy of the License at
-*
-*      http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.ignite.internal.visor.verify;
 
@@ -23,7 +23,6 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
-import org.jetbrains.annotations.Nullable;
 
 /**
  *
@@ -36,13 +35,13 @@ public class VisorViewCacheTaskArg extends VisorDataTransferObject {
     private String regex;
 
     /** Type. */
-    private @Nullable VisorViewCacheCmd cmd;
+    private VisorViewCacheCmd cmd;
 
     /**
      * @param regex Regex.
      * @param cmd Command.
      */
-    public VisorViewCacheTaskArg(String regex, @Nullable VisorViewCacheCmd cmd) {
+    public VisorViewCacheTaskArg(String regex, VisorViewCacheCmd cmd) {
         this.regex = regex;
         this.cmd = cmd;
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 371afb8e2c5e7..c8fb1d2b06d2d 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -17,9 +17,6 @@
 
 package org.apache.ignite.util;
 
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.MutableEntry;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.PrintStream;
@@ -42,6 +39,9 @@
 import java.util.concurrent.atomic.LongAdder;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.MutableEntry;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteAtomicSequence;
 import org.apache.ignite.IgniteCache;
@@ -58,6 +58,7 @@
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.commandline.CommandHandler;
+import org.apache.ignite.internal.commandline.OutputFormat;
 import org.apache.ignite.internal.commandline.cache.CacheCommand;
 import org.apache.ignite.internal.managers.communication.GridIoMessage;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
@@ -81,6 +82,7 @@
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.T2;
 import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.tx.VisorTxInfo;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
@@ -102,6 +104,8 @@
 import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_UNEXPECTED_ERROR;
+import static org.apache.ignite.internal.commandline.OutputFormat.MULTI_LINE;
+import static org.apache.ignite.internal.commandline.OutputFormat.SINGLE_LINE;
 import static org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsDumpTask.IDLE_DUMP_FILE_PREMIX;
 import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
@@ -231,6 +235,10 @@ protected int execute(ArrayList<String> args) {
         // Add force to avoid interactive confirmation
         args.add(CMD_AUTO_CONFIRMATION);
 
+        SB sb = new SB();
+
+        args.forEach(arg -> sb.a(arg).a(" "));
+
         return new CommandHandler().execute(args);
     }
 
@@ -1254,6 +1262,60 @@ public void testCacheAffinity() throws Exception {
         assertTrue(testOut.toString().contains("affCls=RendezvousAffinityFunction"));
     }
 
+    /** */
+    public void testCacheConfigNoOutputFormat() throws Exception {
+        testCacheConfig(null);
+    }
+
+    /** */
+    public void testCacheConfigSingleLineOutputFormat() throws Exception {
+        testCacheConfig(SINGLE_LINE);
+    }
+
+    /** */
+    public void testCacheConfigMultiLineOutputFormat() throws Exception {
+        testCacheConfig(MULTI_LINE);
+    }
+
+    /** */
+    private void testCacheConfig(OutputFormat outputFormat) throws Exception {
+        Ignite ignite = startGrid();
+
+        ignite.cluster().active(true);
+
+        IgniteCache<Object, Object> cache1 = ignite.createCache(new CacheConfiguration<>()
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1)
+            .setName(DEFAULT_CACHE_NAME));
+
+        for (int i = 0; i < 100; i++)
+            cache1.put(i, i);
+
+        injectTestSystemOut();
+
+        int exitCode;
+
+        if (outputFormat == null)
+            exitCode = execute("--cache", "list", ".*", "--config");
+        else
+            exitCode = execute("--cache", "list", ".*", "--config", "--output-format", outputFormat.text());
+
+        assertEquals(EXIT_CODE_OK, exitCode);
+
+        String outStr = testOut.toString();
+
+        if (outputFormat == null || outputFormat == SINGLE_LINE) {
+            assertTrue(outStr.contains("name=" + DEFAULT_CACHE_NAME));
+            assertTrue(outStr.contains("partitions=32"));
+            assertTrue(outStr.contains("function=o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction"));
+        }
+        else if (outputFormat == MULTI_LINE) {
+            assertTrue(outStr.contains("[cache = '" + DEFAULT_CACHE_NAME + "']"));
+            assertTrue(outStr.contains("Affinity Partitions: 32"));
+            assertTrue(outStr.contains("Affinity Function: o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction"));
+        }
+    }
+
     /**
      *
      */
@@ -1294,7 +1356,7 @@ public void testCacheDistribution() throws Exception {
         assertTrue(lastRowIndex > 0);
 
         // Last row is empty, but the previous line contains data
-        lastRowIndex = log.lastIndexOf('\n', lastRowIndex-1);
+        lastRowIndex = log.lastIndexOf('\n', lastRowIndex - 1);
 
         assertTrue(lastRowIndex > 0);
 

From 102bbae724241c1223b0513504176daed43b84d4 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Tue, 23 Oct 2018 12:33:24 +0300
Subject: [PATCH 449/543] IGNITE-8879 Fix blinking baseline node sometimes
 unable to connect to cluster - Fixes #4893.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit 3f7109ff52bdfc3157987b2230216cc982392eff)
---
 .../GridDhtPartitionsExchangeFuture.java      | 20 ++++-----
 .../GridCacheDatabaseSharedManager.java       | 22 ++++-----
 .../IgniteCacheDatabaseSharedManager.java     |  6 ++-
 .../IgniteMetaStorageBasicTest.java           | 45 +++++++++++++++++++
 4 files changed, 66 insertions(+), 27 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 938b9fe429f6c..20125195407f8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -848,14 +848,10 @@ else if (msg instanceof WalStateAbstractMessage)
      * @throws IgniteCheckedException If failed.
      */
     private IgniteInternalFuture<?> initCachesOnLocalJoin() throws IgniteCheckedException {
-        if (isLocalNodeNotInBaseline()) {
-            cctx.cache().cleanupCachesDirectories();
+        boolean baselineNode = isLocalNodeInBaseline();
 
-            cctx.database().cleanupCheckpointDirectory();
-
-            if (cctx.wal() != null)
-                cctx.wal().cleanupWalDirectories();
-        }
+        if (!baselineNode)
+                cctx.cache().cleanupCachesDirectories();
 
         cctx.activate();
 
@@ -876,7 +872,7 @@ private IgniteInternalFuture<?> initCachesOnLocalJoin() throws IgniteCheckedExce
                 }
             }
 
-            cctx.database().readCheckpointAndRestoreMemory(startDescs);
+            cctx.database().readCheckpointAndRestoreMemory(startDescs, !baselineNode);
         }
 
         IgniteInternalFuture<?> cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(), locJoinCtx);
@@ -885,12 +881,12 @@ private IgniteInternalFuture<?> initCachesOnLocalJoin() throws IgniteCheckedExce
     }
 
     /**
-     * @return {@code true} if local node is not in baseline and {@code false} otherwise.
+     * @return {@code true} if local node is in baseline and {@code false} otherwise.
      */
-    private boolean isLocalNodeNotInBaseline() {
+    private boolean isLocalNodeInBaseline() {
         BaselineTopology topology = cctx.discovery().discoCache().state().baselineTopology();
 
-        return topology!= null && !topology.consistentIds().contains(cctx.localNode().consistentId());
+        return topology != null && topology.consistentIds().contains(cctx.localNode().consistentId());
     }
 
     /**
@@ -993,7 +989,7 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
                                 startDescs.add(desc);
                         }
 
-                        cctx.database().readCheckpointAndRestoreMemory(startDescs);
+                        cctx.database().readCheckpointAndRestoreMemory(startDescs, false);
                     }
 
                     assert registerCachesFuture == null : "No caches registration should be scheduled before new caches have started.";
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 7332b2a8714aa..2d90dadd56e9e 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -639,7 +639,7 @@ private void readMetastore() throws IgniteCheckedException {
             checkpointReadLock();
 
             try {
-                restoreMemory(status, true, storePageMem);
+                restoreMemory(status, true, storePageMem, false);
 
                 metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true);
 
@@ -808,7 +808,8 @@ private void unRegistrateMetricsMBean() {
 
     /** {@inheritDoc} */
     @Override public void readCheckpointAndRestoreMemory(
-        List<DynamicCacheDescriptor> cachesToStart
+            List<DynamicCacheDescriptor> cachesToStart,
+            boolean restoreMetastorageOnly
     ) throws IgniteCheckedException {
         assert !cctx.localNode().isClient();
 
@@ -834,7 +835,7 @@ private void unRegistrateMetricsMBean() {
                 (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME)
             );
 
-            WALPointer restore = restoreMemory(status);
+            WALPointer restore = restoreMemory(status, restoreMetastorageOnly, (PageMemoryEx) metaStorage.pageMemory(), true);
 
             if (restore == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) {
                 throw new StorageException("Restore wal pointer = " + restore + ", while status.endPtr = " +
@@ -1889,26 +1890,19 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
         }
     }
 
-    /**
-     * @param status Checkpoint status.
-     * @throws IgniteCheckedException If failed.
-     * @throws StorageException In case I/O error occurred during operations with storage.
-     */
-    @Nullable private WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException {
-        return restoreMemory(status, false, (PageMemoryEx)metaStorage.pageMemory());
-    }
-
     /**
      * @param status Checkpoint status.
      * @param metastoreOnly If {@code True} restores Metastorage only.
      * @param storePageMem Metastore page memory.
+     * @param finalizeCp If {@code True}, finalizes checkpoint on recovery.
      * @throws IgniteCheckedException If failed.
      * @throws StorageException In case I/O error occurred during operations with storage.
      */
     @Nullable private WALPointer restoreMemory(
         CheckpointStatus status,
         boolean metastoreOnly,
-        PageMemoryEx storePageMem
+        PageMemoryEx storePageMem,
+        boolean finalizeCp
     ) throws IgniteCheckedException {
         assert !metastoreOnly || storePageMem != null;
 
@@ -2068,7 +2062,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
             }
         }
 
-        if (metastoreOnly)
+        if (!finalizeCp)
             return null;
 
         WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 67205690f84a6..3c2f9b8cde60f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -600,9 +600,13 @@ public DataStorageMetrics persistentStoreMetrics() {
 
     /**
      * @param cachesToStart Started caches.
+     * @param restoreMetastorageOnly Apply updates only for metastorage.
      * @throws IgniteCheckedException If failed.
      */
-    public void readCheckpointAndRestoreMemory(List<DynamicCacheDescriptor> cachesToStart) throws IgniteCheckedException {
+    public void readCheckpointAndRestoreMemory(
+            List<DynamicCacheDescriptor> cachesToStart,
+            boolean restoreMetastorageOnly
+    ) throws IgniteCheckedException {
         // No-op.
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
index 18375150d06d4..e5a53fac1833b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
@@ -17,12 +17,14 @@
 package org.apache.ignite.internal.processors.cache.persistence.metastorage;
 
 import java.io.Serializable;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -106,6 +108,34 @@ public void testMetaStorageMassivePutUpdateRestart() throws Exception {
         verifyKeys(ig, KEYS_CNT, KEY_PREFIX, UPDATED_VAL_PREFIX);
     }
 
+    /**
+     * @throws Exception If fails.
+     */
+    public void testRecoveryOfMetastorageWhenNodeNotInBaseline() throws Exception {
+        IgniteEx ig0 = startGrid(0);
+
+        ig0.cluster().active(true);
+
+        final byte KEYS_CNT = 100;
+        final String KEY_PREFIX = "test.key.";
+        final String NEW_VAL_PREFIX = "new.val.";
+        final String UPDATED_VAL_PREFIX = "updated.val.";
+
+        startGrid(1);
+
+        // Disable checkpoints in order to check whether recovery works.
+        forceCheckpoint(grid(1));
+        disableCheckpoints(grid(1));
+
+        loadKeys(grid(1), KEYS_CNT, KEY_PREFIX, NEW_VAL_PREFIX, UPDATED_VAL_PREFIX);
+
+        stopGrid(1, true);
+
+        startGrid(1);
+
+        verifyKeys(grid(1), KEYS_CNT, KEY_PREFIX, UPDATED_VAL_PREFIX);
+    }
+
     /** */
     private void loadKeys(IgniteEx ig,
         byte keysCnt,
@@ -144,4 +174,19 @@ private void verifyKeys(IgniteEx ig,
             Assert.assertEquals(valPrefix + i, val);
         }
     }
+
+    /**
+     * Disable checkpoints on a specific node.
+     *
+     * @param node Ignite node.h
+     * @throws IgniteCheckedException If failed.
+     */
+    private void disableCheckpoints(Ignite node) throws IgniteCheckedException {
+        assert !node.cluster().localNode().isClient();
+
+        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)node).context()
+                .cache().context().database();
+
+        dbMgr.enableCheckpoints(false).get();
+    }
 }

From 6c0b56df0bb2ccebf1cf4ef96e5fbf8d186d49c2 Mon Sep 17 00:00:00 2001
From: Aleksei Scherbakov <alexey.scherbakoff@gmail.com>
Date: Thu, 25 Oct 2018 14:10:43 +0300
Subject: [PATCH 450/543] GG-14354 Backport IGNITE-9326 to 2.5.1-X

---
 .../cache/CacheInvokeDirectResult.java        | 21 ++++++++++++++-----
 .../processors/cache/GridCacheReturn.java     | 17 ++++++++++++++-
 .../dht/atomic/GridDhtAtomicCache.java        |  4 ++++
 3 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
index 89a0a0ff7c000..3b463afe8a25e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
@@ -140,16 +140,27 @@ public void prepareMarshal(GridCacheContext ctx) throws IgniteCheckedException {
             }
         }
 
-        if (unprepareRes != null) {
-            res = ctx.toCacheObject(unprepareRes);
-
-            unprepareRes = null;
-        }
+        assert unprepareRes == null : "marshalResult() was not called for the result: " + this;
 
         if (res != null)
             res.prepareMarshal(ctx.cacheObjectContext());
     }
 
+    /**
+     * Converts the entry processor unprepared result to a cache object instance.
+     *
+     * @param ctx Cache context.
+     */
+    public void marshalResult(GridCacheContext ctx) {
+        try {
+            if (unprepareRes != null)
+                res = ctx.toCacheObject(unprepareRes);
+        }
+        finally {
+            unprepareRes = null;
+        }
+    }
+
     /**
      * @param ctx Cache context.
      * @param ldr Class loader.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
index 530f5b6581deb..2ae0f7c09eacc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheReturn.java
@@ -269,7 +269,10 @@ else if (err instanceof UnregisteredBinaryTypeException)
                 invokeResCol = new ArrayList<>();
 
             CacheInvokeDirectResult res0 = err == null ?
-                CacheInvokeDirectResult.lazyResult(key, res) : new CacheInvokeDirectResult(key, err);
+                cctx.transactional() ?
+                    new CacheInvokeDirectResult(key, cctx.toCacheObject(res)) :
+                    CacheInvokeDirectResult.lazyResult(key, res) :
+                new CacheInvokeDirectResult(key, err);
 
             invokeResCol.add(res0);
         }
@@ -307,6 +310,18 @@ public synchronized void mergeEntryProcessResults(GridCacheReturn other) {
         resMap.putAll((Map<Object, EntryProcessorResult>)other.v);
     }
 
+    /**
+     * Converts entry processor invokation results to cache object instances.
+     *
+     * @param ctx Cache context.
+     */
+    public void marshalResult(GridCacheContext ctx) {
+        if (invokeRes && invokeResCol != null) {
+            for (CacheInvokeDirectResult directRes : invokeResCol)
+                directRes.marshalResult(ctx);
+        }
+    }
+
     /**
      * @param ctx Cache context.
      * @throws IgniteCheckedException If failed.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 36b0a6913ac5e..4dacde6a066eb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1757,6 +1757,10 @@ private void updateAllAsyncInternal0(
                             top.readUnlock();
                         }
 
+                        // This call will convert entry processor invocation results to cache object instances.
+                        // Must be done outside topology read lock to avoid deadlocks.
+                        res.returnValue().marshalResult(ctx);
+
                         break;
                     }
                     catch (UnregisteredClassException ex) {

From c3b83c4b0ca7b564904a5cfccd3acf237fcf4a2e Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Thu, 25 Oct 2018 18:10:13 +0700
Subject: [PATCH 451/543] IGNITE-9853 control.sh: Added output of cache
 configurations. Fixes #4956.

(cherry picked from commit a18f71ce5c89465e7b32fca71755208ac8976128)
---
 .../visor/cache/VisorCacheConfigurationCollectorTaskArg.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java
index a91f0ca4e2e70..b4b8143be0de0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTaskArg.java
@@ -92,7 +92,7 @@ public String getRegex() {
     @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
         cacheNames = U.readCollection(in);
 
-        if (getProtocolVersion() > V1)
+        if (protoVer > V1)
             regex = U.readString(in);
     }
 

From b5ea4153975351cfbd1fea032c045e1792c7d87d Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Wed, 26 Sep 2018 20:39:46 +0300
Subject: [PATCH 452/543] IGN-12070 Muted test for deprecated functionality.

---
 .../IgniteCacheClientReconnectTest.java       | 46 ++++---------------
 1 file changed, 9 insertions(+), 37 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java
index a0796a3e22ec7..bed58dc0d646f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheClientReconnectTest.java
@@ -17,14 +17,8 @@
 
 package org.apache.ignite.internal.processors.cache.distributed;
 
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
@@ -34,13 +28,18 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
 import static org.apache.ignite.cache.CacheMode.PARTITIONED;
@@ -161,35 +160,8 @@ public void testClientReconnectOnExchangeHistoryExhaustion() throws Exception {
      *
      * @throws Exception If failed
      */
-    public void testClientInForceServerModeStopsOnExchangeHistoryExhaustion() throws Exception {
-        System.setProperty(IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE, "1");
-
-        try {
-            startGrids(SRV_CNT);
-
-            client = true;
-
-            forceServerMode = true;
-
-            int clientNodes = 10;
-
-            try {
-                startGridsMultiThreaded(SRV_CNT, clientNodes);
-            }
-            catch (IgniteCheckedException e) {
-                //Ignored: it is expected to get exception here
-            }
-
-            awaitPartitionMapExchange();
-
-            int topSize = G.allGrids().size();
-
-            assertTrue("Actual size: " + topSize, topSize < SRV_CNT + clientNodes);
-
-        }
-        finally {
-            System.clearProperty(IgniteSystemProperties.IGNITE_EXCHANGE_HISTORY_SIZE);
-        }
+    public void testClientInForceServerModeStopsOnExchangeHistoryExhaustion() {
+        fail("https://ggsystems.atlassian.net/browse/IGN-12070");
     }
 
     /**
@@ -342,4 +314,4 @@ private void putGet(Ignite ignite) {
             assertEquals(key, cache.get(key));
         }
     }
-}
\ No newline at end of file
+}

From 3b4205bf2dc8a5e4fd60f1aa29d7e5dba1266d01 Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Wed, 26 Sep 2018 20:39:46 +0300
Subject: [PATCH 453/543] IGNITE-8873 Added methods to preload partitions to
 page memory - Fixes #5053.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

cherry-picked from 28e3dec5b4c9bffcca5391cac8bee824973fc7a4
---
 .../java/org/apache/ignite/IgniteCache.java   |  90 ++-
 .../cache/GatewayProtectedCacheProxy.java     |  73 ++-
 .../processors/cache/GridCacheAdapter.java    | 198 +++++--
 .../processors/cache/GridCacheProxyImpl.java  |  63 ++-
 .../cache/IgniteCacheOffheapManager.java      |  22 +-
 .../cache/IgniteCacheOffheapManagerImpl.java  |  43 +-
 .../cache/IgniteCacheProxyImpl.java           |  85 ++-
 .../processors/cache/IgniteInternalCache.java |  48 +-
 .../GridDhtPartitionsReservation.java         |  13 +-
 .../cache/local/GridLocalCache.java           |  30 +-
 .../persistence/GridCacheOffheapManager.java  |  74 ++-
 .../db/IgnitePdsPartitionPreloadTest.java     | 512 ++++++++++++++++++
 .../multijvm/IgniteCacheProcessProxy.java     |  46 +-
 .../testsuites/IgnitePdsTestSuite4.java       |   3 +
 .../cache/hibernate/HibernateCacheProxy.java  |  39 +-
 .../ApiParity/CacheParityTest.cs              |   7 +-
 16 files changed, 1153 insertions(+), 193 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPartitionPreloadTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
index cd8264bb05a3a..83a4ba21023d2 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteCache.java
@@ -17,26 +17,6 @@
 
 package org.apache.ignite;
 
-import java.io.Serializable;
-import java.sql.Timestamp;
-import java.util.Collection;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import javax.cache.Cache;
-import javax.cache.CacheException;
-import javax.cache.configuration.Configuration;
-import javax.cache.event.CacheEntryRemovedListener;
-import javax.cache.expiry.ExpiryPolicy;
-import javax.cache.integration.CacheLoader;
-import javax.cache.integration.CacheWriter;
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.EntryProcessorResult;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheEntry;
 import org.apache.ignite.cache.CacheEntryProcessor;
@@ -70,6 +50,27 @@
 import org.apache.ignite.transactions.TransactionTimeoutException;
 import org.jetbrains.annotations.Nullable;
 
+import javax.cache.Cache;
+import javax.cache.CacheException;
+import javax.cache.configuration.Configuration;
+import javax.cache.event.CacheEntryRemovedListener;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.integration.CacheLoader;
+import javax.cache.integration.CacheWriter;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import java.io.Serializable;
+import java.sql.Timestamp;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+
 /**
  * Main entry point for all <b>Data Grid APIs.</b> You can get a named cache by calling {@link Ignite#cache(String)}
  * method.
@@ -1504,7 +1505,7 @@ public <T> IgniteFuture<Map<K, EntryProcessorResult<T>>> invokeAllAsync(Set<? ex
     /**
      * Gets a collection of lost partition IDs.
      *
-     * @return Lost paritions.
+     * @return Lost partitions.
      */
     public Collection<Integer> lostPartitions();
 
@@ -1514,4 +1515,51 @@ public <T> IgniteFuture<Map<K, EntryProcessorResult<T>>> invokeAllAsync(Set<? ex
      * @param enabled Statistics enabled flag.
      */
     public void enableStatistics(boolean enabled);
+
+    /**
+     * Efficiently preloads cache primary partition into page memory.
+     * <p>
+     * This is useful for fast iteration over cache partition data if persistence is enabled and the data is "cold".
+     * <p>
+     * Preload will reduce available amount of page memory for subsequent operations and may lead to earlier page
+     * replacement.
+     * <p>
+     * This method is irrelevant for in-memory caches. Calling this method on an in-memory cache will result in
+     * exception.
+     *
+     * @param partition Partition.
+     */
+    public void preloadPartition(int partition);
+
+    /**
+     * Efficiently preloads cache partition into page memory.
+     * <p>
+     * This is useful for fast iteration over cache partition data if persistence is enabled and the data is "cold".
+     * <p>
+     * Preload will reduce available amount of page memory for subsequent operations and may lead to earlier page
+     * replacement.
+     * <p>
+     * This method is irrelevant for in-memory caches. Calling this method on an in-memory cache will result in
+     * exception.
+     *
+     * @param partition Partition.
+     * @return A future representing pending completion of the partition preloading.
+     */
+    public IgniteFuture<Void> preloadPartitionAsync(int partition);
+
+    /**
+     * Efficiently preloads cache partition into page memory if it exists on the local node.
+     * <p>
+     * This is useful for fast iteration over cache partition data if persistence is enabled and the data is "cold".
+     * <p>
+     * Preload will reduce available amount of page memory for subsequent operations and may lead to earlier page
+     * replacement.
+     * <p>
+     * This method is irrelevant for in-memory caches. Calling this method on an in-memory cache will result in
+     * exception.
+     *
+     * @param partition Partition.
+     * @return {@code True} if partition was preloaded, {@code false} if it doesn't belong to local node.
+     */
+    public boolean localPreloadPartition(int partition);
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java
index 2e8120ba7bde2..7d835c70599ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GatewayProtectedCacheProxy.java
@@ -17,24 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.Lock;
-import javax.cache.CacheException;
-import javax.cache.CacheManager;
-import javax.cache.configuration.CacheEntryListenerConfiguration;
-import javax.cache.configuration.Configuration;
-import javax.cache.expiry.ExpiryPolicy;
-import javax.cache.integration.CompletionListener;
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorResult;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheEntry;
@@ -61,6 +43,25 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import javax.cache.CacheException;
+import javax.cache.CacheManager;
+import javax.cache.configuration.CacheEntryListenerConfiguration;
+import javax.cache.configuration.Configuration;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.integration.CompletionListener;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorResult;
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+
 /**
  * Cache proxy wrapper with gateway lock provided operations and possibility to change cache operation context.
  */
@@ -1450,6 +1451,42 @@ public void setCacheManager(org.apache.ignite.cache.CacheManager cacheMgr) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void preloadPartition(int part) {
+        CacheOperationGate opGate = onEnter();
+
+        try {
+            delegate.preloadPartition(part);
+        }
+        finally {
+            onLeave(opGate);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteFuture<Void> preloadPartitionAsync(int part) {
+        CacheOperationGate opGate = onEnter();
+
+        try {
+            return delegate.preloadPartitionAsync(part);
+        }
+        finally {
+            onLeave(opGate);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean localPreloadPartition(int part) {
+        CacheOperationGate opGate = onEnter();
+
+        try {
+            return delegate.localPreloadPartition(part);
+        }
+        finally {
+            onLeave(opGate);
+        }
+    }
+
     /**
      * Safely get CacheGateway.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index e43865a56d8d9..1b7c9a50730d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -17,36 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.InvalidObjectException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.io.ObjectStreamException;
-import java.util.AbstractSet;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.LongAdder;
-import java.util.concurrent.locks.ReentrantLock;
-import javax.cache.Cache;
-import javax.cache.expiry.ExpiryPolicy;
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.EntryProcessorResult;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
@@ -87,9 +57,11 @@
 import org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityImpl;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteExternalizableExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
@@ -134,19 +106,53 @@
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.lang.IgniteOutClosure;
 import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteProductVersion;
+import org.apache.ignite.lang.IgniteRunnable;
 import org.apache.ignite.mxbean.CacheMetricsMXBean;
 import org.apache.ignite.plugin.security.SecurityPermission;
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.apache.ignite.resources.JobContextResource;
-import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode;
+import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.transactions.Transaction;
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
 
+import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.InvalidObjectException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.io.ObjectStreamException;
+import java.util.AbstractSet;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.LongAdder;
+import java.util.concurrent.locks.ReentrantLock;
+
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_KEY_VALIDATION_DISABLED;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_CACHE_RETRIES_COUNT;
 import static org.apache.ignite.internal.GridClosureCallMode.BROADCAST;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.dr.GridDrType.DR_LOAD;
 import static org.apache.ignite.internal.processors.dr.GridDrType.DR_NONE;
 import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
@@ -176,6 +182,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
     /** Maximum number of retries when topology changes. */
     public static final int MAX_RETRIES = IgniteSystemProperties.getInteger(IGNITE_CACHE_RETRIES_COUNT, 100);
 
+    /** Minimum version supporting partition preloading. */
+    private static final IgniteProductVersion PRELOAD_PARTITION_SINCE = IgniteProductVersion.fromString("2.5.0");
+
     /** Deserialization stash. */
     private static final ThreadLocal<IgniteBiTuple<String, String>> stash = new ThreadLocal<IgniteBiTuple<String,
         String>>() {
@@ -289,6 +298,9 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
     /** Asynchronous operations limit semaphore. */
     private Semaphore asyncOpsSem;
 
+    /** {@code True} if attempted to use partition preloading on outdated node. */
+    private volatile boolean partPreloadBadVerWarned;
+
     /** Active. */
     private volatile boolean active;
 
@@ -742,7 +754,7 @@ else if (modes.heap) {
                 return it;
             }
 
-            public String toString() {
+            @Override public String toString() {
                 return "CacheLocalEntries []";
             }
         };
@@ -1199,6 +1211,31 @@ private IgniteInternalFuture<?> executeClearTask(@Nullable Set<? extends K> keys
             return new GridFinishedFuture<>();
     }
 
+    /**
+     * @param part Partition id.
+     * @return Future.
+     */
+    private IgniteInternalFuture<?> executePreloadTask(int part) throws IgniteCheckedException {
+        ClusterGroup grp = ctx.grid().cluster().forDataNodes(ctx.name());
+
+        @Nullable ClusterNode targetNode = ctx.affinity().primaryByPartition(part, ctx.topology().readyTopologyVersion());
+
+        if (targetNode == null || targetNode.version().compareTo(PRELOAD_PARTITION_SINCE) < 0) {
+            if (!partPreloadBadVerWarned) {
+                U.warn(log(), "Attempting to execute partition preloading task on outdated or not mapped node " +
+                    "[targetNodeVer=" + (targetNode == null ? "NA" : targetNode.version()) +
+                    ", minSupportedNodeVer=" + PRELOAD_PARTITION_SINCE + ']');
+
+                partPreloadBadVerWarned = true;
+            }
+
+            return new GridFinishedFuture<>();
+        }
+
+        return ctx.closures().affinityRun(Collections.singleton(name()), part,
+            new PartitionPreloadJob(ctx.name(), part), grp.nodes(), null);
+    }
+
     /**
      * @param keys Keys.
      * @param readers Readers flag.
@@ -4804,6 +4841,55 @@ private void advance() {
         return new CacheEntryImpl<>((K)key0, (V)val0, entry.version());
     }
 
+    /** {@inheritDoc} */
+    @Override public void preloadPartition(int part) throws IgniteCheckedException {
+        if (isLocal())
+            ctx.offheap().preloadPartition(part);
+        else
+            executePreloadTask(part).get();
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> preloadPartitionAsync(int part) throws IgniteCheckedException {
+        if (isLocal()) {
+            return ctx.kernalContext().closure().runLocalSafe(() -> {
+                try {
+                    ctx.offheap().preloadPartition(part);
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IgniteException(e);
+                }
+            });
+        }
+        else
+            return executePreloadTask(part);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean localPreloadPartition(int part) throws IgniteCheckedException {
+        if (!ctx.affinityNode())
+            return false;
+
+        GridDhtPartitionTopology top = ctx.group().topology();
+
+        @Nullable GridDhtLocalPartition p = top.localPartition(part, top.readyTopologyVersion(), false);
+
+        if (p == null)
+            return false;
+
+        try {
+            if (!p.reserve() || p.state() != OWNING)
+                return false;
+
+            p.dataStore().preload();
+        }
+        finally {
+            p.release();
+        }
+
+        return true;
+    }
+
     /**
      *
      */
@@ -6512,6 +6598,52 @@ public ClearTask(String cacheName, AffinityTopologyVersion topVer, Set<? extends
         }
     }
 
+    /**
+     * Partition preload job.
+     */
+    @GridInternal
+    private static class PartitionPreloadJob implements IgniteRunnable {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /** */
+        @IgniteInstanceResource
+        private IgniteEx ignite;
+
+        /** */
+        @LoggerResource
+        private IgniteLogger log;
+
+        /** */
+        private final String name;
+
+        /** Cache name. */
+        private final int part;
+
+        /**
+         * @param name Name.
+         * @param part Partition.
+         */
+        public PartitionPreloadJob(String name, int part) {
+            this.name = name;
+            this.part = part;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            IgniteInternalCache cache = ignite.context().cache().cache(name);
+
+            try {
+                cache.context().offheap().preloadPartition(part);
+            }
+            catch (IgniteCheckedException e) {
+                log.error("Failed to preload the partition [cache=" + name + ", partition=" + part + ']', e);
+
+                throw new IgniteException(e);
+            }
+        }
+    }
+
     /**
      * Iterator implementation for KeySet.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java
index ed3b47ad68545..728c633445015 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProxyImpl.java
@@ -17,19 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import javax.cache.Cache;
-import javax.cache.expiry.ExpiryPolicy;
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorResult;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheEntry;
 import org.apache.ignite.cache.CacheMetrics;
@@ -52,6 +39,20 @@
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
 
+import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorResult;
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
 /**
  * Cache proxy.
  */
@@ -236,6 +237,42 @@ public IgniteInternalCache<K, V> delegate() {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void preloadPartition(int part) throws IgniteCheckedException {
+        CacheOperationContext prev = gate.enter(opCtx);
+
+        try {
+            delegate.preloadPartition(part);
+        }
+        finally {
+            gate.leave(prev);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> preloadPartitionAsync(int part) throws IgniteCheckedException {
+        CacheOperationContext prev = gate.enter(opCtx);
+
+        try {
+            return delegate.preloadPartitionAsync(part);
+        }
+        finally {
+            gate.leave(prev);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean localPreloadPartition(int part) throws IgniteCheckedException {
+        CacheOperationContext prev = gate.enter(opCtx);
+
+        try {
+            return delegate.localPreloadPartition(part);
+        }
+        finally {
+            gate.leave(prev);
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public GridCacheProxyImpl<K, V> forSubjectId(UUID subjId) {
         return new GridCacheProxyImpl<>(ctx, delegate,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
index b2a26b1102008..99ef2c1844793 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
@@ -17,8 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import java.util.Map;
-import javax.cache.Cache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
@@ -37,6 +35,10 @@
 import org.apache.ignite.internal.util.lang.IgniteInClosure2X;
 import org.jetbrains.annotations.Nullable;
 
+import javax.cache.Cache;
+import java.util.Map;
+
+
 /**
  *
  */
@@ -346,6 +348,14 @@ public long cacheEntriesCount(int cacheId, boolean primary, boolean backup, Affi
      */
     public long totalPartitionEntriesCount(int part);
 
+    /**
+     * Preload a partition. Must be called under partition reservation for DHT caches.
+     *
+     * @param part Partition.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void preloadPartition(int part) throws IgniteCheckedException;
+
     /**
      *
      */
@@ -529,7 +539,7 @@ public GridCursor<? extends CacheDataRow> cursor(int cacheId, KeyCacheObject low
         /**
          * @param cntr Counter.
          */
-        void updateInitialCounter(long cntr);
+        public void updateInitialCounter(long cntr);
 
         /**
          * Inject rows cache cleaner.
@@ -537,5 +547,11 @@ public GridCursor<? extends CacheDataRow> cursor(int cacheId, KeyCacheObject low
          * @param rowCacheCleaner Rows cache cleaner.
          */
         public void setRowCacheCleaner(GridQueryRowCacheCleaner rowCacheCleaner);
+
+        /**
+         * Preload a store into page memory.
+         * @throws IgniteCheckedException If failed.
+         */
+        public void preload() throws IgniteCheckedException;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 5884da2019d4d..8864e7d4e7646 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -17,31 +17,18 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import javax.cache.Cache;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.NodeStoppingException;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtDemandedPartitionsMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteHistoricalIterator;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtInvalidPartitionException;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter;
 import org.apache.ignite.internal.processors.cache.persistence.CacheSearchRow;
@@ -76,6 +63,20 @@
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
+import javax.cache.Cache;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
@@ -262,11 +263,16 @@ public CacheDataStore dataStore(GridDhtLocalPartition part) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void preloadPartition(int p) throws IgniteCheckedException {
+        throw new IgniteCheckedException("Operation only applicable to caches with enabled persistence");
+    }
+
     /**
      * @param p Partition.
      * @return Partition data.
      */
-    @Nullable private CacheDataStore partitionData(int p) {
+    @Nullable protected CacheDataStore partitionData(int p) {
         if (grp.isLocal())
             return locCacheDataStore;
         else {
@@ -1627,6 +1633,11 @@ private void finishRemove(GridCacheContext cctx, KeyCacheObject key, @Nullable C
             }
         }
 
+        /** {@inheritDoc} */
+        @Override public void preload() throws IgniteCheckedException {
+            // No-op.
+        }
+
         /**
          * @param cctx Cache context.
          * @param key Key.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index 284dfcc742266..a399046e87260 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -17,33 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.Lock;
-import javax.cache.Cache;
-import javax.cache.CacheException;
-import javax.cache.configuration.CacheEntryListenerConfiguration;
-import javax.cache.configuration.Configuration;
-import javax.cache.configuration.Factory;
-import javax.cache.event.CacheEntryUpdatedListener;
-import javax.cache.expiry.ExpiryPolicy;
-import javax.cache.integration.CompletionListener;
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorException;
-import javax.cache.processor.EntryProcessorResult;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCacheRestartingException;
 import org.apache.ignite.IgniteCheckedException;
@@ -106,6 +79,34 @@
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import javax.cache.Cache;
+import javax.cache.CacheException;
+import javax.cache.configuration.CacheEntryListenerConfiguration;
+import javax.cache.configuration.Configuration;
+import javax.cache.configuration.Factory;
+import javax.cache.event.CacheEntryUpdatedListener;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.integration.CompletionListener;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorException;
+import javax.cache.processor.EntryProcessorResult;
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+
 /**
  * Cache proxy implementation.
  */
@@ -1808,6 +1809,36 @@ private <R> void setFuture(IgniteInternalFuture<R> fut) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void preloadPartition(int part) {
+        try {
+            delegate.preloadPartition(part);
+        }
+        catch (IgniteCheckedException e) {
+            throw cacheException(e);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteFuture<Void> preloadPartitionAsync(int part) {
+        try {
+            return (IgniteFuture<Void>)createFuture(delegate.preloadPartitionAsync(part));
+        }
+        catch (IgniteCheckedException e) {
+            throw cacheException(e);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean localPreloadPartition(int part) {
+        try {
+            return delegate.localPreloadPartition(part);
+        }
+        catch (IgniteCheckedException e) {
+            throw cacheException(e);
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void writeExternal(ObjectOutput out) throws IOException {
         out.writeObject(ctx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java
index d01d5369b1723..002aaff8de604 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteInternalCache.java
@@ -17,18 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import java.io.Serializable;
-import java.sql.Timestamp;
-import java.util.Collection;
-import java.util.Date;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import javax.cache.Cache;
-import javax.cache.expiry.ExpiryPolicy;
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorResult;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.CacheAtomicityMode;
@@ -55,6 +43,19 @@
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
 
+import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorResult;
+import java.io.Serializable;
+import java.sql.Timestamp;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
 /**
  * This interface provides a rich API for working with distributed caches. It includes the following
  * main functionality:
@@ -1827,4 +1828,27 @@ public void localLoadCache(@Nullable IgniteBiPredicate<K, V> p, @Nullable Object
      * @return A collection of lost partitions if a cache is in recovery state.
      */
     public Collection<Integer> lostPartitions();
+
+    /**
+     * Preload cache partition.
+     * @param part Partition.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void preloadPartition(int part) throws IgniteCheckedException;
+
+    /**
+     * Preload cache partition.
+     * @param part Partition.
+     * @return Future to be completed whenever preloading completes.
+     * @throws IgniteCheckedException If failed.
+     */
+    public IgniteInternalFuture<?> preloadPartitionAsync(int part) throws IgniteCheckedException;
+
+    /**
+     * Preloads cache partition if it exists on local node.
+     * @param part Partition.
+     * @return {@code True} if partition was preloaded, {@code false} if it doesn't belong to local node.
+     * @throws IgniteCheckedException If failed.
+     */
+    public boolean localPreloadPartition(int part) throws IgniteCheckedException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java
index 2682a896e7476..4a6e50ea92ec8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsReservation.java
@@ -17,16 +17,17 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.topology;
 
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridReservable;
 import org.apache.ignite.internal.util.typedef.CI1;
 import org.apache.ignite.internal.util.typedef.F;
 
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
 
 /**
@@ -179,7 +180,7 @@ public void onPublish(CI1<GridDhtPartitionsReservation> unpublish) {
      */
     private static void tryEvict(GridDhtLocalPartition[] parts) {
         if (parts == null)  // Can be not initialized yet.
-            return ;
+            return;
 
         for (GridDhtLocalPartition part : parts)
             tryEvict(part);
@@ -291,4 +292,4 @@ public boolean invalidate() {
 
         return result;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
index 0de53c013c7e5..c6a11cb2764a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/GridLocalCache.java
@@ -17,9 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache.local;
 
-import java.io.Externalizable;
-import java.util.Collection;
-import java.util.concurrent.Callable;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -42,6 +39,10 @@
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
 
+import java.io.Externalizable;
+import java.util.Collection;
+import java.util.concurrent.Callable;
+
 /**
  * Local cache implementation.
  */
@@ -232,4 +233,27 @@ else if (modes.heap)
     @Override public long localSizeLong(int part, CachePeekMode[] peekModes) throws IgniteCheckedException {
         return localSizeLong(peekModes);
     }
+
+    /** {@inheritDoc} */
+    @Override public void preloadPartition(int part) throws IgniteCheckedException {
+        ctx.offheap().preloadPartition(part);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> preloadPartitionAsync(int part) throws IgniteCheckedException {
+        return ctx.closures().callLocalSafe(new Callable<Void>() {
+            @Override public Void call() throws Exception {
+                preloadPartition(part);
+
+                return null;
+            }
+        });
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean localPreloadPartition(int part) throws IgniteCheckedException {
+        ctx.offheap().preloadPartition(part);
+
+        return true;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index d3efab030acad..68a41cc7523e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -17,15 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache.persistence;
 
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Executor;
-import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.failure.FailureContext;
@@ -35,6 +26,7 @@
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.PageSupport;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
@@ -51,10 +43,10 @@
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteHistoricalIterator;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
 import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
@@ -80,6 +72,16 @@
 import org.apache.ignite.lang.IgniteBiTuple;
 import org.jetbrains.annotations.Nullable;
 
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicBoolean;
+
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
@@ -851,6 +853,21 @@ private Metas getOrAllocateCacheMetas() throws IgniteCheckedException {
         return iterator;
     }
 
+    /** {@inheritDoc} */
+    @Override public void preloadPartition(int part) throws IgniteCheckedException {
+        if (grp.isLocal()) {
+            partitionData(part).preload();
+
+            return;
+        }
+
+        GridDhtLocalPartition locPart = grp.topology().localPartition(part, AffinityTopologyVersion.NONE, false, false);
+
+        assert locPart != null && locPart.reservations() > 0;
+
+        locPart.dataStore().preload();
+    }
+
     /**
      * Calculates free space of all partition data stores - number of bytes available for use in allocated pages.
      *
@@ -1282,7 +1299,32 @@ private CacheDataStore init0(boolean checkExists) throws IgniteCheckedException
 
                     PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
 
-                    delegate0 = new CacheDataStoreImpl(partId, name, rowStore, dataTree);
+                    delegate0 = new CacheDataStoreImpl(partId, name, rowStore, dataTree) {
+                        /** {@inheritDoc} */
+                        @Override public void preload() throws IgniteCheckedException {
+                            IgnitePageStoreManager pageStoreMgr = ctx.pageStore();
+
+                            if (pageStoreMgr == null)
+                                return;
+
+                            final int pages = pageStoreMgr.pages(grp.groupId(), partId);
+
+                            long pageId = pageMem.partitionMetaPageId(grp.groupId(), partId);
+
+                            // For each page sequentially pin/unpin.
+                            for (int pageNo = 0; pageNo < pages; pageId++, pageNo++) {
+                                long pagePointer = -1;
+
+                                try {
+                                    pagePointer = pageMem.acquirePage(grp.groupId(), pageId);
+                                }
+                                finally {
+                                    if (pagePointer != -1)
+                                        pageMem.releasePage(grp.groupId(), pageId, pagePointer);
+                                }
+                            }
+                        }
+                    };
 
                     int grpId = grp.groupId();
                     long partMetaId = pageMem.partitionMetaPageId(grpId, partId);
@@ -1660,6 +1702,14 @@ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException {
             if (delegate != null)
                 delegate.clear(cacheId);
         }
+
+        /** {@inheritDoc} */
+        @Override public void preload() throws IgniteCheckedException {
+            CacheDataStore delegate0 = init0(true);
+
+            if (delegate0 != null)
+                delegate0.preload();
+        }
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPartitionPreloadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPartitionPreloadTest.java
new file mode 100644
index 0000000000000..b71f10432572c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPartitionPreloadTest.java
@@ -0,0 +1,512 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import javax.cache.Cache;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.LOCAL;
+
+/**
+ * Test partition preload for varios cache modes.
+ */
+public class IgnitePdsPartitionPreloadTest extends GridCommonAbstractTest {
+    /** IP finder. */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** Test entry count. */
+    public static final int ENTRY_CNT = 500;
+
+    /** Grid count. */
+    private static final int GRIDS_CNT = 3;
+
+    /** */
+    private static final String CLIENT_GRID_NAME = "client";
+
+    /** */
+    public static final String DEFAULT_REGION = "default";
+
+    /** */
+    private Supplier<CacheConfiguration> cfgFactory;
+
+    /** */
+    private static final String TEST_ATTR = "testId";
+
+    /** */
+    private static final String NO_CACHE_NODE = "node0";
+
+    /** */
+    private static final String PRIMARY_NODE = "node1";
+
+    /** */
+    private static final String BACKUP_NODE = "node2";
+
+    /** */
+    public static final String MEM = "mem";
+
+    /** */
+    public static final int MB = 1024 * 1024;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        cfg.setClientMode(CLIENT_GRID_NAME.equals(gridName));
+
+        if (!cfg.isClientMode()) {
+            String val = "node" + getTestIgniteInstanceIndex(gridName);
+            cfg.setUserAttributes(Collections.singletonMap(TEST_ATTR, val));
+            cfg.setConsistentId(val);
+        }
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration()
+            .setDataRegionConfigurations(new DataRegionConfiguration().setName(MEM).setInitialSize(10 * MB))
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration().
+                    setMetricsEnabled(true).
+                    setMaxSize(50L * MB).
+                    setPersistenceEnabled(true).
+                    setName(DEFAULT_REGION))
+            .setWalMode(WALMode.LOG_ONLY)
+            .setWalSegmentSize(16 * MB)
+            .setPageSize(1024)
+            .setMetricsEnabled(true);
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        cfg.setCacheConfiguration(cfgFactory.get());
+
+        cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+        return cfg;
+    }
+
+    /**
+     * @param atomicityMode Atomicity mode.
+     */
+    private CacheConfiguration<Integer, Integer> cacheConfiguration(CacheAtomicityMode atomicityMode) {
+        CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
+
+        ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+        ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+        ccfg.setBackups(1);
+        ccfg.setNodeFilter(new TestIgnitePredicate());
+        ccfg.setAtomicityMode(atomicityMode);
+
+        return ccfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** */
+    public void testLocalPreloadPartitionClient() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL).setDataRegionName(MEM);
+
+        startGridsMultiThreaded(GRIDS_CNT);
+
+        Ignite client = startGrid("client");
+
+        assertNotNull(client.cache(DEFAULT_CACHE_NAME));
+
+        assertFalse(client.cache(DEFAULT_CACHE_NAME).localPreloadPartition(0));
+        assertFalse(grid(0).cache(DEFAULT_CACHE_NAME).localPreloadPartition(0));
+    }
+
+    /** */
+    public void testLocalPreloadPartitionPrimary() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.LOCAL);
+    }
+
+    /** */
+    public void testLocalPreloadPartitionBackup() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(BackupNodePredicate.INSTANCE).findFirst().get(), PreloadMode.LOCAL);
+    }
+
+    /** */
+    public void testPreloadPartitionInMemoryRemote() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL).setDataRegionName(MEM);
+
+        startGridsMultiThreaded(GRIDS_CNT);
+
+        Ignite client = startGrid("client");
+
+        assertNotNull(client.cache(DEFAULT_CACHE_NAME));
+
+        try {
+            client.cache(DEFAULT_CACHE_NAME).preloadPartition(0);
+
+            fail("Exception is expected");
+        }
+        catch (Exception e) {
+            log.error("Expected", e);
+        }
+    }
+
+    /** */
+    public void testPreloadPartitionInMemoryLocal() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL).setDataRegionName(MEM);
+
+        startGridsMultiThreaded(GRIDS_CNT);
+
+        int key = 0;
+
+        Ignite prim = primaryNode(key, DEFAULT_CACHE_NAME);
+
+        int part = prim.affinity(DEFAULT_CACHE_NAME).partition(key);
+
+        try {
+            prim.cache(DEFAULT_CACHE_NAME).preloadPartition(part);
+
+            fail("Exception is expected");
+        }
+        catch (Exception e) {
+            log.error("Expected", e);
+        }
+    }
+
+    /** */
+    public void testPreloadPartitionTransactionalClientSync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL);
+
+        preloadPartition(() -> {
+            try {
+                return startGrid(CLIENT_GRID_NAME);
+            }
+            catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }, PreloadMode.SYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionTransactionalClientAsync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL);
+
+        preloadPartition(() -> {
+            try {
+                return startGrid(CLIENT_GRID_NAME);
+            }
+            catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }, PreloadMode.ASYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionTransactionalNodeFilteredSync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL);
+
+        preloadPartition(() -> grid(0), PreloadMode.SYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionTransactionalNodeFilteredAsync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL);
+
+        preloadPartition(() -> grid(0), PreloadMode.ASYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionTransactionalPrimarySync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.SYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionTransactionalPrimaryAsync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.ASYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionTransactionalBackupSync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(BackupNodePredicate.INSTANCE).findFirst().get(), PreloadMode.SYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionTransactionalBackupAsync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(BackupNodePredicate.INSTANCE).findFirst().get(), PreloadMode.ASYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionAtomicClientSync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(ATOMIC);
+
+        preloadPartition(() -> {
+            try {
+                return startGrid(CLIENT_GRID_NAME);
+            }
+            catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }, PreloadMode.SYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionAtomicClientAsync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(ATOMIC);
+
+        preloadPartition(() -> {
+            try {
+                return startGrid(CLIENT_GRID_NAME);
+            }
+            catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }, PreloadMode.ASYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionAtomicNodeFilteredSync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(ATOMIC);
+
+        preloadPartition(() -> grid(0), PreloadMode.SYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionAtomicNodeFilteredAsync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(ATOMIC);
+
+        preloadPartition(() -> grid(0), PreloadMode.ASYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionAtomicPrimarySync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(ATOMIC);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.SYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionAtomicPrimaryAsync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(ATOMIC);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.ASYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionAtomicBackupSync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(ATOMIC);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(BackupNodePredicate.INSTANCE).findFirst().get(), PreloadMode.SYNC);
+    }
+
+    /** */
+    public void testPreloadPartitionAtomicBackupAsync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(ATOMIC);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(BackupNodePredicate.INSTANCE).findFirst().get(), PreloadMode.ASYNC);
+    }
+
+    /** */
+    public void testPreloadLocalTransactionalSync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL).setCacheMode(LOCAL);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.SYNC);
+    }
+
+    /** */
+    public void testPreloadLocalTransactionalAsync() throws Exception {
+        cfgFactory = () -> cacheConfiguration(TRANSACTIONAL).setCacheMode(LOCAL);
+
+        preloadPartition(
+            () -> G.allGrids().stream().filter(PrimaryNodePredicate.INSTANCE).findFirst().get(), PreloadMode.ASYNC);
+    }
+
+    /**
+     * @param execNodeFactory Test node factory.
+     * @param preloadMode Preload mode.
+     */
+    private void preloadPartition(Supplier<Ignite> execNodeFactory, PreloadMode preloadMode) throws Exception {
+        Ignite crd = startGridsMultiThreaded(GRIDS_CNT);
+
+        Ignite testNode = grid(1);
+
+        Object consistentId = testNode.cluster().localNode().consistentId();
+
+        assertEquals(PRIMARY_NODE, testNode.cluster().localNode().consistentId());
+
+        boolean locCacheMode = testNode.cache(DEFAULT_CACHE_NAME).getConfiguration(CacheConfiguration.class).getCacheMode() == LOCAL;
+
+        Integer key = primaryKey(testNode.cache(DEFAULT_CACHE_NAME));
+
+        int preloadPart = crd.affinity(DEFAULT_CACHE_NAME).partition(key);
+
+        int cnt = 0;
+
+        try (IgniteDataStreamer<Integer, Integer> streamer = testNode.dataStreamer(DEFAULT_CACHE_NAME)) {
+            int k = 0;
+
+            while (cnt < ENTRY_CNT) {
+                if (testNode.affinity(DEFAULT_CACHE_NAME).partition(k) == preloadPart) {
+                    streamer.addData(k, k);
+
+                    cnt++;
+                }
+
+                k++;
+            }
+        }
+
+        forceCheckpoint();
+
+        stopAllGrids();
+
+        startGridsMultiThreaded(GRIDS_CNT);
+
+        testNode = G.allGrids().stream().
+            filter(ignite -> PRIMARY_NODE.equals(ignite.cluster().localNode().consistentId())).findFirst().get();
+
+        if (!locCacheMode)
+            assertEquals(testNode, primaryNode(key, DEFAULT_CACHE_NAME));
+
+        Ignite execNode = execNodeFactory.get();
+
+        switch (preloadMode) {
+            case SYNC:
+                execNode.cache(DEFAULT_CACHE_NAME).preloadPartition(preloadPart);
+
+                if (locCacheMode) {
+                    testNode = G.allGrids().stream().filter(ignite ->
+                        ignite.cluster().localNode().consistentId().equals(consistentId)).findFirst().get();
+                }
+
+                break;
+            case ASYNC:
+                execNode.cache(DEFAULT_CACHE_NAME).preloadPartitionAsync(preloadPart).get();
+
+                if (locCacheMode) {
+                    testNode = G.allGrids().stream().filter(ignite ->
+                        ignite.cluster().localNode().consistentId().equals(consistentId)).findFirst().get();
+                }
+
+                break;
+            case LOCAL:
+                assertTrue(execNode.cache(DEFAULT_CACHE_NAME).localPreloadPartition(preloadPart));
+
+                testNode = execNode; // For local preloading testNode == execNode
+
+                break;
+        }
+
+        long c0 = testNode.dataRegionMetrics(DEFAULT_REGION).getPagesRead();
+
+        // After partition preloading no pages should be read from store.
+        List<Cache.Entry<Object, Object>> list = U.arrayList(testNode.cache(DEFAULT_CACHE_NAME).localEntries(), 1000);
+
+        assertEquals(ENTRY_CNT, list.size());
+
+        assertEquals("Read pages count must be same", c0, testNode.dataRegionMetrics(DEFAULT_REGION).getPagesRead());
+    }
+
+    /** */
+    private static class TestIgnitePredicate implements IgnitePredicate<ClusterNode> {
+        /** {@inheritDoc} */
+        @Override public boolean apply(ClusterNode node) {
+            return !NO_CACHE_NODE.equals(node.attribute(TEST_ATTR));
+        }
+    }
+
+    /** */
+    private static class PrimaryNodePredicate implements Predicate<Ignite> {
+        /** */
+        private static final PrimaryNodePredicate INSTANCE = new PrimaryNodePredicate();
+
+        /** {@inheritDoc} */
+        @Override public boolean test(Ignite ignite) {
+            return PRIMARY_NODE.equals(ignite.cluster().localNode().consistentId());
+        }
+    }
+
+    /** */
+    private static class BackupNodePredicate implements Predicate<Ignite> {
+        /** */
+        private static final BackupNodePredicate INSTANCE = new BackupNodePredicate();
+
+        /** {@inheritDoc} */
+        @Override public boolean test(Ignite ignite) {
+            return BACKUP_NODE.equals(ignite.cluster().localNode().consistentId());
+        }
+    }
+
+    /** */
+    private enum PreloadMode {
+        /** Sync. */ SYNC,
+        /** Async. */ASYNC,
+        /** Local. */LOCAL;
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java
index ff1005cf1337c..bb51eb3aa1a5f 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteCacheProcessProxy.java
@@ -17,21 +17,6 @@
 
 package org.apache.ignite.testframework.junits.multijvm;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.Lock;
-import javax.cache.CacheException;
-import javax.cache.CacheManager;
-import javax.cache.configuration.CacheEntryListenerConfiguration;
-import javax.cache.configuration.Configuration;
-import javax.cache.expiry.ExpiryPolicy;
-import javax.cache.integration.CompletionListener;
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorResult;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCompute;
@@ -56,6 +41,22 @@
 import org.apache.ignite.resources.IgniteInstanceResource;
 import org.jetbrains.annotations.Nullable;
 
+import javax.cache.CacheException;
+import javax.cache.CacheManager;
+import javax.cache.configuration.CacheEntryListenerConfiguration;
+import javax.cache.configuration.Configuration;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.integration.CompletionListener;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorResult;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.Lock;
+
 /**
  * Ignite cache proxy for ignite instance at another JVM.
  */
@@ -682,6 +683,21 @@ private IgniteCacheProcessProxy(String name, boolean async, ExpiryPolicy plc, Ig
         throw new UnsupportedOperationException("Method should be supported.");
     }
 
+    /** {@inheritDoc} */
+    @Override public void preloadPartition(int partId) {
+        throw new UnsupportedOperationException("Method should be supported.");
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteFuture<Void> preloadPartitionAsync(int partId) {
+        throw new UnsupportedOperationException("Method should be supported.");
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean localPreloadPartition(int partition) {
+        throw new UnsupportedOperationException("Method should be supported.");
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
index 1d9bb2408c58c..e078c53195a31 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPartitionPreloadTest;
 
 /**
  *
@@ -32,6 +33,8 @@ public static TestSuite suite() {
 
         suite.addTestSuite(IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse.class);
 
+        suite.addTestSuite(IgnitePdsPartitionPreloadTest.class);
+
         return suite;
     }
 }
diff --git a/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java b/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
index 0fc2c2d3cede2..83b3ff56f2700 100644
--- a/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
+++ b/modules/hibernate-core/src/main/java/org/apache/ignite/cache/hibernate/HibernateCacheProxy.java
@@ -17,17 +17,6 @@
 
 package org.apache.ignite.cache.hibernate;
 
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import javax.cache.Cache;
-import javax.cache.expiry.ExpiryPolicy;
-import javax.cache.processor.EntryProcessor;
-import javax.cache.processor.EntryProcessorResult;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cache.CacheEntry;
 import org.apache.ignite.cache.CacheMetrics;
@@ -37,7 +26,6 @@
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
@@ -49,6 +37,18 @@
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.jetbrains.annotations.Nullable;
 
+import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
+import javax.cache.processor.EntryProcessor;
+import javax.cache.processor.EntryProcessorResult;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
 /**
  * Hibernate cache proxy used to substitute hibernate keys with ignite keys.
  */
@@ -652,6 +652,21 @@ public HibernateKeyTransformer keyTransformer(){
         return delegate.lostPartitions();
     }
 
+    /** {@inheritDoc} */
+    @Override public void preloadPartition(int part) throws IgniteCheckedException {
+        delegate.preloadPartition(part);
+    }
+
+    /** {@inheritDoc} */
+    @Override public IgniteInternalFuture<?> preloadPartitionAsync(int part) throws IgniteCheckedException {
+        return delegate.preloadPartitionAsync(part);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean localPreloadPartition(int part) throws IgniteCheckedException {
+        return delegate.localPreloadPartition(part);
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public EntryProcessorResult invoke(
         @Nullable AffinityTopologyVersion topVer,
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs
index 68b822c38f73a..aea0397519335 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs
@@ -58,7 +58,10 @@ public class CacheParityTest
             "sizeLong",  // 	IGNITE-6563
             "sizeLongAsync",  // IGNITE-6563
             "localSizeLong",  // IGNITE-6563
-            "enableStatistics"  // IGNITE-7276
+            "enableStatistics"  // IGNITE-7276,
+            "preloadPartition",  // IGNITE-9998
+            "preloadPartitionAsync",  // IGNITE-9998
+            "localPreloadPartition",  // IGNITE-9998
         };
 
         /// <summary>
@@ -74,4 +77,4 @@ public void TestCache()
                 MissingMembers);
         }
     }
-}
\ No newline at end of file
+}

From e8c30f06e6382e2182ce6fdac0fdf96702452422 Mon Sep 17 00:00:00 2001
From: luchnikovnsk <luchnikov.bsk@gmail.com>
Date: Tue, 23 Oct 2018 11:16:40 +0300
Subject: [PATCH 454/543] IGNITE-9719 Extra rebalanceThreadPoolSize check on
 client node - Fixes #4911.

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit 962d6a29716195c13519ca48b6d79f3b8541653c)
---
 .../apache/ignite/internal/IgniteKernal.java  | 32 ++++++++------
 .../client/ClientConfigurationTest.java       | 42 +++++++++++++++++++
 2 files changed, 61 insertions(+), 13 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 7ea3271633dc3..5f2b7bc16ba8c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2532,19 +2532,25 @@ private void ackSpis() {
      *
      */
     private void ackRebalanceConfiguration() throws IgniteCheckedException {
-        if (cfg.getSystemThreadPoolSize() <= cfg.getRebalanceThreadPoolSize())
-            throw new IgniteCheckedException("Rebalance thread pool size exceed or equals System thread pool size. " +
-                "Change IgniteConfiguration.rebalanceThreadPoolSize property before next start.");
-
-        if (cfg.getRebalanceThreadPoolSize() < 1)
-            throw new IgniteCheckedException("Rebalance thread pool size minimal allowed value is 1. " +
-                "Change IgniteConfiguration.rebalanceThreadPoolSize property before next start.");
-
-        for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) {
-            if (ccfg.getRebalanceBatchesPrefetchCount() < 1)
-                throw new IgniteCheckedException("Rebalance batches prefetch count minimal allowed value is 1. " +
-                    "Change CacheConfiguration.rebalanceBatchesPrefetchCount property before next start. " +
-                    "[cache=" + ccfg.getName() + "]");
+        if (cfg.isClientMode()) {
+            if (cfg.getRebalanceThreadPoolSize() != IgniteConfiguration.DFLT_REBALANCE_THREAD_POOL_SIZE)
+                U.warn(log, "Setting the rebalance pool size has no effect on the client mode");
+        }
+        else {
+            if (cfg.getSystemThreadPoolSize() <= cfg.getRebalanceThreadPoolSize())
+                throw new IgniteCheckedException("Rebalance thread pool size exceed or equals System thread pool size. " +
+                    "Change IgniteConfiguration.rebalanceThreadPoolSize property before next start.");
+
+            if (cfg.getRebalanceThreadPoolSize() < 1)
+                throw new IgniteCheckedException("Rebalance thread pool size minimal allowed value is 1. " +
+                    "Change IgniteConfiguration.rebalanceThreadPoolSize property before next start.");
+
+            for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) {
+                if (ccfg.getRebalanceBatchesPrefetchCount() < 1)
+                    throw new IgniteCheckedException("Rebalance batches prefetch count minimal allowed value is 1. " +
+                        "Change CacheConfiguration.rebalanceBatchesPrefetchCount property before next start. " +
+                        "[cache=" + ccfg.getName() + "]");
+            }
         }
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/client/ClientConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/client/ClientConfigurationTest.java
index e6ab4d73b1c08..e3190f46bf36a 100644
--- a/modules/core/src/test/java/org/apache/ignite/client/ClientConfigurationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/client/ClientConfigurationTest.java
@@ -25,8 +25,18 @@
 import java.io.ObjectOutput;
 import java.io.ObjectOutputStream;
 import java.util.Collections;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.BinaryConfiguration;
 import org.apache.ignite.configuration.ClientConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.testframework.GridStringLogger;
+import org.junit.Assert;
 import org.junit.Test;
 
 import static org.junit.Assert.assertTrue;
@@ -66,4 +76,36 @@ public void testSerialization() throws IOException, ClassNotFoundException {
 
         assertTrue(Comparers.equal(target, desTarget));
     }
+
+    /**
+     * Test check the case when {@link IgniteConfiguration#getRebalanceThreadPoolSize()} is equal to {@link
+     * IgniteConfiguration#getSystemThreadPoolSize()}
+     */
+    @Test
+    public void testRebalanceThreadPoolSize() {
+        GridStringLogger gridStrLog = new GridStringLogger();
+        gridStrLog.logLength(1024 * 100);
+
+        IgniteConfiguration cci = Config.getServerConfiguration().setClientMode(true);
+        cci.setRebalanceThreadPoolSize(cci.getSystemThreadPoolSize());
+        cci.setGridLogger(gridStrLog);
+
+        try (
+            Ignite si = Ignition.start(Config.getServerConfiguration());
+            Ignite ci = Ignition.start(cci)) {
+            Set<ClusterNode> collect = si.cluster().nodes().stream()
+                .filter(new Predicate<ClusterNode>() {
+                    @Override public boolean test(ClusterNode clusterNode) {
+                        return clusterNode.isClient();
+                    }
+                })
+                .collect(Collectors.toSet());
+
+            String log = gridStrLog.toString();
+            boolean containsMsg = log.contains("Setting the rebalance pool size has no effect on the client mode");
+
+            Assert.assertTrue(containsMsg);
+            Assert.assertEquals(1, collect.size());
+        }
+    }
 }

From 460289423aadffe0d4e7d17050a74ca4a1c75d58 Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Sat, 27 Oct 2018 06:50:34 +0300
Subject: [PATCH 455/543] IGNITE-8873 NET compilation fixes.

---
 .../Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs
index aea0397519335..b789f3e3cc7dd 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/CacheParityTest.cs
@@ -58,10 +58,10 @@ public class CacheParityTest
             "sizeLong",  // 	IGNITE-6563
             "sizeLongAsync",  // IGNITE-6563
             "localSizeLong",  // IGNITE-6563
-            "enableStatistics"  // IGNITE-7276,
+            "enableStatistics",  // IGNITE-7276,
             "preloadPartition",  // IGNITE-9998
             "preloadPartitionAsync",  // IGNITE-9998
-            "localPreloadPartition",  // IGNITE-9998
+            "localPreloadPartition"  // IGNITE-9998
         };
 
         /// <summary>

From 9f96505435858a4589d046992d92d35e76223e75 Mon Sep 17 00:00:00 2001
From: Maxim Muzafarov <maxmuzaf@gmail.com>
Date: Fri, 26 Oct 2018 22:29:24 +0300
Subject: [PATCH 456/543] IGNITE-7196 Restore binary state before node join to
 topology - Fixes #4520.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
---
 .../apache/ignite/internal/IgniteKernal.java  |   2 +
 .../pagemem/store/IgnitePageStoreManager.java |  12 +-
 .../wal/IgniteWriteAheadLogManager.java       |   7 +-
 .../wal/record/MemoryRecoveryRecord.java      |   7 +-
 .../processors/cache/GridCacheProcessor.java  |   2 +-
 .../GridDhtPartitionsExchangeFuture.java      |  61 ++---
 .../GridCacheDatabaseSharedManager.java       | 251 ++++++++++++------
 .../IgniteCacheDatabaseSharedManager.java     |  99 +++++--
 .../file/FilePageStoreManager.java            |  35 ++-
 .../persistence/metastorage/MetaStorage.java  |   5 -
 .../wal/FileWriteAheadLogManager.java         |  27 +-
 .../FsyncModeFileWriteAheadLogManager.java    |  27 +-
 .../IgnitePdsDiskErrorsRecoveringTest.java    |  12 +-
 .../db/wal/IgniteWalRecoveryTest.java         | 126 +++++++++
 .../persistence/db/wal/WalCompactionTest.java |  15 ++
 .../pagemem/NoOpPageStoreManager.java         |   6 +
 .../persistence/pagemem/NoOpWALManager.java   |   5 -
 17 files changed, 486 insertions(+), 213 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 5f2b7bc16ba8c..b16e95fc89f50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1036,6 +1036,8 @@ public void start(
                 ctx.cluster().initDiagnosticListeners();
 
                 fillNodeAttributes(clusterProc.updateNotifierEnabled());
+
+                ctx.cache().context().database().startMemoryRestore(ctx);
             }
             catch (Throwable e) {
                 U.error(
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index 13bcd7df41d98..1cf488eeb18b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -19,6 +19,7 @@
 
 import java.nio.ByteBuffer;
 import java.util.Map;
+import java.util.function.Predicate;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.pagemem.PageMemory;
@@ -230,10 +231,19 @@ public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cac
     public void cleanupPersistentSpace(CacheConfiguration cacheConfiguration) throws IgniteCheckedException;
 
     /**
-     * Cleanup persistent space for all caches.
+     * Cleanup persistent space for all caches except metastore.
      */
     public void cleanupPersistentSpace() throws IgniteCheckedException;
 
+    /**
+     * Cleanup cache store whether it matches the provided predicate and if matched
+     * store was previously initizlized.
+     *
+     * @param cacheGrpPred Predicate to match by id cache group stores to clean.
+     * @param cleanFiles {@code True} to delete all persisted files related to particular store.
+     */
+    public void cleanupPageStoreIfMatch(Predicate<Integer> cacheGrpPred, boolean cleanFiles);
+
     /**
      * Creates and initializes cache work directory retrieved from {@code cacheCfg}.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 70c9ad45129bb..a9b2ddb97ce8c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -46,6 +46,8 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
     /**
      * Resumes logging after start. When WAL manager is started, it will skip logging any updates until this
      * method is called to avoid logging changes induced by the state restore procedure.
+     *
+     * @throws IgniteCheckedException If fails.
      */
     public void resumeLogging(WALPointer lastWrittenPtr) throws IgniteCheckedException;
 
@@ -149,9 +151,4 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
      * @param grpId Group id.
      */
     public boolean disabled(int grpId);
-
-    /**
-     * Cleanup all directories relating to WAL (e.g. work WAL dir, archive WAL dir).
-     */
-    public void cleanupWalDirectories() throws IgniteCheckedException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java
index 8843eeedef90e..92658cc4ce788 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java
@@ -20,8 +20,13 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
- * Marker that we start memory recovering
+ * Marker that we start memory recovering.
+ *
+ * @deprecated Previously, used to track node started\stopped states. But in fact only
+ * mark files created by method GridCacheDatabaseSharedManager#nodeStart(WALPointer)
+ * used. Should be removed in 3.0 release.
  */
+@Deprecated
 public class MemoryRecoveryRecord extends WALRecord {
     /** Create timestamp, millis */
     private long time;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index be2a698fa1909..f20c8e3c8a006 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2134,7 +2134,7 @@ private boolean checkForAffinityNode(
      * @param affNode {@code true} if it is affinity node for cache.
      * @throws IgniteCheckedException if failed.
      */
-    private void preparePageStore(DynamicCacheDescriptor desc, boolean affNode) throws IgniteCheckedException {
+    public void preparePageStore(DynamicCacheDescriptor desc, boolean affNode) throws IgniteCheckedException {
         if (sharedCtx.pageStore() != null && affNode)
             initializationProtector.protect(
                 desc.groupDescriptor().groupId(),
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 20125195407f8..2e938c622a2b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -49,7 +49,6 @@
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.NearCacheConfiguration;
 import org.apache.ignite.events.DiscoveryEvent;
 import org.apache.ignite.internal.IgniteClientDisconnectedCheckedException;
 import org.apache.ignite.internal.IgniteDiagnosticAware;
@@ -81,6 +80,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.GridCacheUtils;
 import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
 import org.apache.ignite.internal.processors.cache.StateChangeRequest;
 import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage;
@@ -848,34 +848,28 @@ else if (msg instanceof WalStateAbstractMessage)
      * @throws IgniteCheckedException If failed.
      */
     private IgniteInternalFuture<?> initCachesOnLocalJoin() throws IgniteCheckedException {
-        boolean baselineNode = isLocalNodeInBaseline();
-
-        if (!baselineNode)
-                cctx.cache().cleanupCachesDirectories();
-
-        cctx.activate();
-
-        LocalJoinCachesContext locJoinCtx = exchActions == null ? null : exchActions.localJoinContext();
-
-        List<T2<DynamicCacheDescriptor, NearCacheConfiguration>> caches = locJoinCtx == null ? null :
-            locJoinCtx.caches();
-
-        if (!cctx.kernalContext().clientNode()) {
-            List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
-
-            if (caches != null) {
-                for (T2<DynamicCacheDescriptor, NearCacheConfiguration> c : caches) {
-                    DynamicCacheDescriptor startDesc = c.get1();
-
-                    if (CU.isPersistentCache(startDesc.cacheConfiguration(), cctx.gridConfig().getDataStorageConfiguration()))
-                        startDescs.add(startDesc);
+        if (!isLocalNodeInBaseline()) {
+            cctx.database().cleanupRestoredCaches();
+
+            for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) {
+                if (CU.isPersistentCache(desc.cacheConfiguration(),
+                    cctx.gridConfig().getDataStorageConfiguration())) {
+                    // Perform cache init from scratch.
+                    cctx.cache().preparePageStore(desc, true);
                 }
             }
 
-            cctx.database().readCheckpointAndRestoreMemory(startDescs, !baselineNode);
+            // Set initial node started marker.
+            cctx.database().nodeStart(null);
         }
 
-        IgniteInternalFuture<?> cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(), locJoinCtx);
+        cctx.activate();
+
+        if (!cctx.kernalContext().clientNode())
+            cctx.database().onDoneRestoreBinaryMemory();
+
+        IgniteInternalFuture<?> cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(),
+            exchActions == null ? null : exchActions.localJoinContext());
 
         return cachesRegistrationFut;
     }
@@ -978,19 +972,8 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
                 try {
                     cctx.activate();
 
-                    if (!cctx.kernalContext().clientNode()) {
-                        List<DynamicCacheDescriptor> startDescs = new ArrayList<>();
-
-                        for (ExchangeActions.CacheActionData startReq : exchActions.cacheStartRequests()) {
-                            DynamicCacheDescriptor desc = startReq.descriptor();
-
-                            if (CU.isPersistentCache(desc.cacheConfiguration(),
-                                cctx.gridConfig().getDataStorageConfiguration()))
-                                startDescs.add(desc);
-                        }
-
-                        cctx.database().readCheckpointAndRestoreMemory(startDescs, false);
-                    }
+                    if (!cctx.kernalContext().clientNode())
+                        cctx.database().onDoneRestoreBinaryMemory();
 
                     assert registerCachesFuture == null : "No caches registration should be scheduled before new caches have started.";
 
@@ -1892,9 +1875,9 @@ public void finishMerged() {
 
         cctx.database().releaseHistoryForExchange();
 
-        cctx.database().rebuildIndexesIfNeeded(this);
-
         if (err == null) {
+            cctx.database().rebuildIndexesIfNeeded(this);
+
             for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
                 if (!grp.isLocal())
                     grp.topology().onExchangeDone(this, grp.affinity().readyAffinity(res), false);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 2d90dadd56e9e..8fcd2b5898b61 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -53,6 +53,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.LongAdder;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -96,7 +97,6 @@
 import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
-import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
 import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
@@ -166,6 +166,7 @@
 import static java.nio.file.StandardOpenOption.READ;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
 import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
 import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD;
@@ -288,6 +289,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** */
     private boolean stopping;
 
+    /**
+     * The position of last seen WAL pointer. Used for resumming logging from this pointer.
+     *
+     * If binary memory recovery pefrormed on node start, the checkpoint END pointer will store
+     * not the last WAL pointer and can't be used for resumming logging.
+     */
+    private volatile WALPointer walTail;
+
     /** Checkpoint runner thread pool. If null tasks are to be run in single thread */
     @Nullable private IgniteThreadPoolExecutor asyncRunner;
 
@@ -336,7 +345,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Number of pages in current checkpoint at the beginning of checkpoint. */
     private volatile int currCheckpointPagesCnt;
 
-    /** */
+    /**
+     * MetaStorage instance. Value {@code null} means storage not initialized yet.
+     * Guarded by {@link GridCacheDatabaseSharedManager#checkpointReadLock()}
+     */
     private MetaStorage metaStorage;
 
     /** */
@@ -412,8 +424,8 @@ public IgniteInternalFuture<Void> enableCheckpoints(boolean enable) {
     }
 
     /** {@inheritDoc} */
-    @Override protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteCheckedException {
-        super.initDataRegions(memCfg);
+    @Override protected void initDataRegions0(DataStorageConfiguration memCfg) throws IgniteCheckedException {
+        super.initDataRegions0(memCfg);
 
         addDataRegion(
             memCfg,
@@ -425,6 +437,8 @@ public IgniteInternalFuture<Void> enableCheckpoints(boolean enable) {
     }
 
     /**
+     * Create metastorage data region configuration with enabled persistence by default.
+     *
      * @param storageCfg Data storage configuration.
      * @return Data region configuration.
      */
@@ -514,10 +528,43 @@ public void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
         }
     }
 
-    /**
-     * Cleanup checkpoint directory.
-     */
-    public void cleanupCheckpointDirectory() throws IgniteCheckedException {
+    /** {@inheritDoc} */
+    @Override public void cleanupRestoredCaches() {
+        if (dataRegionMap == null)
+            return;
+
+        for (CacheGroupDescriptor grpDesc : cctx.cache().cacheGroupDescriptors().values()) {
+            String regionName = grpDesc.config().getDataRegionName();
+
+            DataRegion region = dataRegionMap.get(regionName == null ? DFLT_DATA_REG_DEFAULT_NAME : regionName);
+
+            if (region == null)
+                continue;
+
+            int partitions = grpDesc.config().getAffinity().partitions();
+
+            if (region.pageMemory() instanceof PageMemoryEx) {
+                PageMemoryEx memEx = (PageMemoryEx)region.pageMemory();
+
+                for (int partId = 0; partId < partitions; partId++)
+                    memEx.invalidate(grpDesc.groupId(), partId);
+            }
+        }
+
+        storeMgr.cleanupPageStoreIfMatch(
+            new Predicate<Integer>() {
+                @Override public boolean test(Integer grpId) {
+                    return MetaStorage.METASTORAGE_CACHE_ID != grpId;
+                }
+            },
+            true);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void cleanupCheckpointDirectory() throws IgniteCheckedException {
+        if (cpHistory != null)
+            cpHistory = new CheckpointHistory(cctx.kernalContext());
+
         try {
             try (DirectoryStream<Path> files = Files.newDirectoryStream(cpDir.toPath())) {
                 for (Path path : files)
@@ -639,7 +686,7 @@ private void readMetastore() throws IgniteCheckedException {
             checkpointReadLock();
 
             try {
-                restoreMemory(status, true, storePageMem, false);
+                restoreMemory(status, true, storePageMem, Collections.emptySet());
 
                 metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true);
 
@@ -652,12 +699,18 @@ private void readMetastore() throws IgniteCheckedException {
                 notifyMetastorageReadyForRead();
             }
             finally {
-                checkpointReadUnlock();
-            }
+                metaStorage = null;
 
-            metaStorage = null;
+                storePageMem.stop(true);
 
-            storePageMem.stop(true);
+                cctx.pageStore().cleanupPageStoreIfMatch(new Predicate<Integer>() {
+                    @Override public boolean test(Integer grpId) {
+                        return MetaStorage.METASTORAGE_CACHE_ID == grpId;
+                    }
+                }, false);
+
+                checkpointReadUnlock();
+            }
         }
         catch (StorageException e) {
             cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
@@ -698,7 +751,7 @@ else if (regCfg.getMaxSize() < 8 * GB)
 
         snapshotMgr = cctx.snapshot();
 
-        if (!cctx.localNode().isClient()) {
+        if (!cctx.kernalContext().clientNode()) {
             initDataBase();
 
             registrateMetricsMBean();
@@ -807,56 +860,88 @@ private void unRegistrateMetricsMBean() {
     }
 
     /** {@inheritDoc} */
-    @Override public void readCheckpointAndRestoreMemory(
-            List<DynamicCacheDescriptor> cachesToStart,
-            boolean restoreMetastorageOnly
-    ) throws IgniteCheckedException {
-        assert !cctx.localNode().isClient();
+    @Override public void onDoneRestoreBinaryMemory() throws IgniteCheckedException {
+        assert !cctx.kernalContext().clientNode();
 
         long time = System.currentTimeMillis();
 
         checkpointReadLock();
 
         try {
-            if (!F.isEmpty(cachesToStart)) {
-                for (DynamicCacheDescriptor desc : cachesToStart) {
-                    if (CU.affinityNode(cctx.localNode(), desc.cacheConfiguration().getNodeFilter()))
-                        storeMgr.initializeForCache(desc.groupDescriptor(), new StoredCacheData(desc.cacheConfiguration()));
-                }
-            }
+            cctx.pageStore().initializeForMetastorage();
 
             CheckpointStatus status = readCheckpointStatus();
 
-            cctx.pageStore().initializeForMetastorage();
+            // Binary memory should be recovered at startup.
+            assert !status.needRestoreMemory() : status;
+
+            WALPointer statusEndPtr = CheckpointStatus.NULL_PTR.equals(status.endPtr) ? null : status.endPtr;
+
+            // If binary memory recovery occurs resume from the last walTail in the other case from END checkpoint.
+            WALPointer walPtr = walTail == null ? statusEndPtr : walTail;
+
+            cctx.wal().resumeLogging(walPtr);
+
+            walTail = null;
 
             metaStorage = new MetaStorage(
                 cctx,
                 dataRegionMap.get(METASTORE_DATA_REGION_NAME),
-                (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME)
+                (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME),
+                false
             );
 
-            WALPointer restore = restoreMemory(status, restoreMetastorageOnly, (PageMemoryEx) metaStorage.pageMemory(), true);
+            // Init metastore only after WAL logging resumed. Can't do it earlier because
+            // MetaStorage first initialization also touches WAL, look at #isWalDeltaRecordNeeded.
+            metaStorage.init(this);
 
-            if (restore == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) {
-                throw new StorageException("Restore wal pointer = " + restore + ", while status.endPtr = " +
-                    status.endPtr + ". Can't restore memory - critical part of WAL archive is missing.");
-            }
+            notifyMetastorageReadyForReadWrite();
+        }
+        catch (IgniteCheckedException e) {
+            if (X.hasCause(e, StorageException.class, IOException.class))
+                cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
-            // First, bring memory to the last consistent checkpoint state if needed.
-            // This method should return a pointer to the last valid record in the WAL.
-            cctx.wal().resumeLogging(restore);
+            throw e;
+        }
+        finally {
+            checkpointReadUnlock();
+
+            U.log(log, "Resume logging performed in " + (System.currentTimeMillis() - time) + " ms.");
+        }
+    }
 
-            WALPointer ptr = cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+    /**
+     * @param cacheGrps Cache groups to restore.
+     * @return Last seen WAL pointer during binary memory recovery.
+     * @throws IgniteCheckedException If failed.
+     */
+    protected WALPointer restoreBinaryMemory(Set<Integer> cacheGrps) throws IgniteCheckedException {
+        assert !cctx.kernalContext().clientNode();
 
-            if (ptr != null) {
-                cctx.wal().flush(ptr, true);
+        long time = System.currentTimeMillis();
 
-                nodeStart(ptr);
+        checkpointReadLock();
+
+        try {
+            cctx.pageStore().initializeForMetastorage();
+
+            CheckpointStatus status = readCheckpointStatus();
+
+            // First, bring memory to the last consistent checkpoint state if needed.
+            // This method should return a pointer to the last valid record in the WAL.
+            WALPointer tailWalPtr = restoreMemory(status,
+                false,
+                (PageMemoryEx)dataRegionMap.get(METASTORE_DATA_REGION_NAME).pageMemory(),
+                cacheGrps);
+
+            if (tailWalPtr == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) {
+                throw new StorageException("Restore wal pointer = " + tailWalPtr + ", while status.endPtr = " +
+                    status.endPtr + ". Can't restore memory - critical part of WAL archive is missing.");
             }
 
-            metaStorage.init(this);
+            nodeStart(tailWalPtr);
 
-            notifyMetastorageReadyForReadWrite();
+            return tailWalPtr;
         }
         catch (IgniteCheckedException e) {
             if (X.hasCause(e, StorageException.class, IOException.class))
@@ -872,14 +957,9 @@ private void unRegistrateMetricsMBean() {
         }
     }
 
-    /**
-     * Creates file with current timestamp and specific "node-started.bin" suffix
-     * and writes into memory recovery pointer.
-     *
-     * @param ptr Memory recovery wal pointer.
-     */
-    private void nodeStart(WALPointer ptr) throws IgniteCheckedException {
-        FileWALPointer p = (FileWALPointer)ptr;
+    /** {@inheritDoc} */
+    @Override public void nodeStart(@Nullable WALPointer ptr) throws IgniteCheckedException {
+        FileWALPointer p = (FileWALPointer)(ptr == null ? CheckpointStatus.NULL_PTR : ptr);
 
         String fileName = U.currentTimeMillis() + NODE_STARTED_FILE_NAME_SUFFIX;
         String tmpFileName = fileName + FilePageStoreManager.TMP_SUFFIX;
@@ -1890,11 +1970,33 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void startMemoryRestore(GridKernalContext kctx) throws IgniteCheckedException {
+        if (kctx.clientNode())
+            return;
+
+        // Preform early regions startup before restoring state.
+        initAndStartRegions(kctx.config().getDataStorageConfiguration());
+
+        // Only presistence caches to start.
+        for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) {
+            if (CU.isPersistentCache(desc.cacheConfiguration(), cctx.gridConfig().getDataStorageConfiguration()))
+                storeMgr.initializeForCache(desc.groupDescriptor(), new StoredCacheData(desc.cacheConfiguration()));
+        }
+
+        final WALPointer restoredPtr = restoreBinaryMemory(cctx.cache().cacheGroupDescriptors().keySet());
+
+        walTail = restoredPtr;
+
+        if (restoredPtr != null)
+            U.log(log, "Binary memory state restored at node startup [restoredPtr=" + restoredPtr + ']');
+    }
+
     /**
      * @param status Checkpoint status.
      * @param metastoreOnly If {@code True} restores Metastorage only.
      * @param storePageMem Metastore page memory.
-     * @param finalizeCp If {@code True}, finalizes checkpoint on recovery.
+     * @param cacheGrps Cache groups to restore.
      * @throws IgniteCheckedException If failed.
      * @throws StorageException In case I/O error occurred during operations with storage.
      */
@@ -1902,7 +2004,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
         CheckpointStatus status,
         boolean metastoreOnly,
         PageMemoryEx storePageMem,
-        boolean finalizeCp
+        Set<Integer> cacheGrps
     ) throws IgniteCheckedException {
         assert !metastoreOnly || storePageMem != null;
 
@@ -1927,8 +2029,14 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
 
         RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, lastArchivedSegment, log);
 
-        Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() :
-            F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps);
+        // Always perform recovery at least meta storage cache.
+        Set<Integer> restoreGrps = new HashSet<>(Collections.singletonList(METASTORAGE_CACHE_ID));
+
+        if (!metastoreOnly && !F.isEmpty(cacheGrps)) {
+            restoreGrps.addAll(cacheGrps.stream()
+                .filter(g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g))
+                .collect(Collectors.toSet()));
+        }
 
         int applied = 0;
 
@@ -1948,10 +2056,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
                             // several repetitive restarts and the same pages may have changed several times.
                             int grpId = pageRec.fullPageId().groupId();
 
-                            if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
-                                continue;
-
-                            if (!ignoreGrps.contains(grpId)) {
+                            if (restoreGrps.contains(grpId)) {
                                 long pageId = pageRec.fullPageId().pageId();
 
                                 PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
@@ -1984,10 +2089,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
                         {
                             int grpId = metaStateRecord.groupId();
 
-                            if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
-                                continue;
-
-                            if (ignoreGrps.contains(grpId))
+                            if (!restoreGrps.contains(grpId))
                                 continue;
 
                             int partId = metaStateRecord.partitionId();
@@ -2008,10 +2110,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
                         {
                             int grpId = destroyRecord.groupId();
 
-                            if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
-                                continue;
-
-                            if (ignoreGrps.contains(grpId))
+                            if (!restoreGrps.contains(grpId))
                                 continue;
 
                             PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
@@ -2029,10 +2128,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
 
                             int grpId = r.groupId();
 
-                            if (metastoreOnly && grpId != METASTORAGE_CACHE_ID)
-                                continue;
-
-                            if (!ignoreGrps.contains(grpId)) {
+                            if (restoreGrps.contains(grpId)) {
                                 long pageId = r.pageId();
 
                                 PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
@@ -2062,7 +2158,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
             }
         }
 
-        if (!finalizeCp)
+        if (metastoreOnly)
             return null;
 
         WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer();
@@ -2188,6 +2284,7 @@ else if (log != null)
 
     /**
      * @param status Last registered checkpoint status.
+     * @param metastoreOnly If {@code True} only records related to metastorage will be processed.
      * @throws IgniteCheckedException If failed to apply updates.
      * @throws StorageException If IO exception occurred while reading write-ahead log.
      */
@@ -2228,9 +2325,13 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
                         for (DataEntry dataEntry : dataRec.writeEntries()) {
                             int cacheId = dataEntry.cacheId();
 
-                            int grpId = cctx.cache().cacheDescriptor(cacheId).groupId();
+                            DynamicCacheDescriptor cacheDesc = cctx.cache().cacheDescriptor(cacheId);
 
-                            if (!ignoreGrps.contains(grpId)) {
+                            // Can empty in case recovery node on blt changed.
+                            if (cacheDesc == null)
+                                continue;
+
+                            if (!ignoreGrps.contains(cacheDesc.groupId())) {
                                 GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
                                 applyUpdate(cacheCtx, dataEntry);
@@ -2709,8 +2810,9 @@ private static String checkpointFileName(long cpTs, UUID cpId, CheckpointEntryTy
     /**
      * @param cp Checkpoint entry.
      * @param type Checkpoint type.
+     * @return Checkpoint file name.
      */
-    private static String checkpointFileName(CheckpointEntry cp, CheckpointEntryType type) {
+    public static String checkpointFileName(CheckpointEntry cp, CheckpointEntryType type) {
         return checkpointFileName(cp.timestamp(), cp.checkpointId(), type);
     }
 
@@ -4101,7 +4203,8 @@ private CheckpointStatus(long cpStartTs, UUID cpStartId, WALPointer startPtr, UU
         }
 
         /**
-         * @return {@code True} if need to apply page log to restore tree structure.
+         * @return {@code True} if need perform binary memory recovery. Only records {@link PageDeltaRecord}
+         * and {@link PageSnapshot} needs to be applyed from {@link #cpStartId}.
          */
         public boolean needRestoreMemory() {
             return !F.eq(cpStartId, cpEndId) && !F.eq(NULL_UUID, cpStartId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 3c2f9b8cde60f..096a84e5cbb31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -37,14 +37,15 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.DirectMemoryRegion;
 import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
 import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
@@ -95,6 +96,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     /** */
     private volatile boolean dataRegionsInitialized;
 
+    /** */
+    private volatile boolean dataRegionsStarted;
+
     /** */
     protected Map<String, DataRegionMetrics> memMetricsMap;
 
@@ -110,7 +114,6 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     /** Page size from memory configuration, may be set only for fake(standalone) IgniteCacheDataBaseSharedManager */
     private int pageSize;
 
-
     /** Stores memory providers eligible for reuse. */
     private Map<String, DirectMemoryProvider> memProviderMap;
 
@@ -229,6 +232,18 @@ protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteChe
         if (dataRegionsInitialized)
             return;
 
+        initDataRegions0(memCfg);
+
+        dataRegionsInitialized = true;
+
+        U.log(log, "Configured data regions initialized successfully [total=" + dataRegionMap.size() + ']');
+    }
+
+    /**
+     * @param memCfg Database config.
+     * @throws IgniteCheckedException If failed to initialize swap path.
+     */
+    protected void initDataRegions0(DataStorageConfiguration memCfg) throws IgniteCheckedException {
         DataRegionConfiguration[] dataRegionCfgs = memCfg.getDataRegionConfigurations();
 
         int dataRegions = dataRegionCfgs == null ? 0 : dataRegionCfgs.length;
@@ -257,8 +272,6 @@ protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteChe
             ),
             CU.isPersistenceEnabled(memCfg)
         );
-
-        dataRegionsInitialized = true;
     }
 
     /**
@@ -266,7 +279,7 @@ protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteChe
      * @param dataRegionCfg Data region config.
      * @throws IgniteCheckedException If failed to initialize swap path.
      */
-    protected void addDataRegion(
+    public void addDataRegion(
         DataStorageConfiguration dataStorageCfg,
         DataRegionConfiguration dataRegionCfg,
         boolean trackable
@@ -599,14 +612,19 @@ public DataStorageMetrics persistentStoreMetrics() {
     }
 
     /**
-     * @param cachesToStart Started caches.
-     * @param restoreMetastorageOnly Apply updates only for metastorage.
-     * @throws IgniteCheckedException If failed.
+     * @throws IgniteCheckedException If fails.
      */
-    public void readCheckpointAndRestoreMemory(
-            List<DynamicCacheDescriptor> cachesToStart,
-            boolean restoreMetastorageOnly
-    ) throws IgniteCheckedException {
+    public void onDoneRestoreBinaryMemory() throws IgniteCheckedException {
+        // No-op.
+    }
+
+    /**
+     * Creates file with current timestamp and specific "node-started.bin" suffix
+     * and writes into memory recovery pointer.
+     *
+     * @param ptr Memory recovery wal pointer.
+     */
+    public void nodeStart(@Nullable WALPointer ptr) throws IgniteCheckedException {
         // No-op.
     }
 
@@ -718,7 +736,15 @@ public void checkpointReadUnlock() {
     }
 
     /**
-     * No-op for non-persistent storage.
+     * Method will perform cleanup cache page memory and each cache partition store.
+     */
+    public void cleanupRestoredCaches() {
+        // No-op.
+    }
+
+    /**
+     * Clean checkpoint directory {@link GridCacheDatabaseSharedManager#cpDir}. The operation
+     * is necessary when local node joined to baseline topology with different consistentId.
      */
     public void cleanupCheckpointDirectory() throws IgniteCheckedException {
         // No-op.
@@ -765,6 +791,16 @@ public boolean beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws I
         return false;
     }
 
+    /**
+     * Perform memory restore before {@link GridDiscoveryManager} start.
+     *
+     * @param kctx Current kernal context.
+     * @throws IgniteCheckedException If fails.
+     */
+    public void startMemoryRestore(GridKernalContext kctx) throws IgniteCheckedException {
+        // No-op.
+    }
+
     /**
      * Called when all partitions have been fully restored and pre-created on node start.
      *
@@ -1065,20 +1101,43 @@ protected File buildPath(String path, String consId) throws IgniteCheckedExcepti
 
     /** {@inheritDoc} */
     @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
-        if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getDataStorageConfiguration() == null)
+        if (kctx.clientNode() && kctx.config().getDataStorageConfiguration() == null)
             return;
 
-        DataStorageConfiguration memCfg = cctx.kernalContext().config().getDataStorageConfiguration();
+        initAndStartRegions(kctx.config().getDataStorageConfiguration());
+    }
 
-        assert memCfg != null;
+    /**
+     * @param cfg Current data storage configuration.
+     * @throws IgniteCheckedException If fails.
+     */
+    protected void initAndStartRegions(DataStorageConfiguration cfg) throws IgniteCheckedException {
+        assert cfg != null;
 
-        initDataRegions(memCfg);
+        initDataRegions(cfg);
+
+        startDataRegions(cfg);
+    }
+
+    /**
+     * @param cfg Regions configuration.
+     * @throws IgniteCheckedException If fails.
+     */
+    private void startDataRegions(DataStorageConfiguration cfg) throws IgniteCheckedException {
+        if (dataRegionsStarted)
+            return;
+
+        assert cfg != null;
 
         registerMetricsMBeans();
 
         startMemoryPolicies();
 
-        initPageMemoryDataStructures(memCfg);
+        initPageMemoryDataStructures(cfg);
+
+        dataRegionsStarted = true;
+
+        U.log(log, "Configured data regions started successfully [total=" + dataRegionMap.size() + ']');
     }
 
     /** {@inheritDoc} */
@@ -1087,7 +1146,7 @@ protected File buildPath(String path, String consId) throws IgniteCheckedExcepti
     }
 
     /**
-     * @param shutdown Shutdown.
+     * @param shutdown {@code True} to force memory regions shutdown.
      */
     private void onDeActivate(boolean shutdown) {
         if (dataRegionMap != null) {
@@ -1110,6 +1169,8 @@ private void onDeActivate(boolean shutdown) {
             }
 
             dataRegionsInitialized = false;
+
+            dataRegionsStarted = false;
         }
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 268fd106ce22e..93cd1bc1e4d1b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -32,11 +32,14 @@
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -226,15 +229,29 @@ public FilePageStoreManager(GridKernalContext ctx) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void cleanupPageStoreIfMatch(Predicate<Integer> cacheGrpPred, boolean cleanFiles) {
+        Map<Integer, CacheStoreHolder> filteredStores = idxCacheStores.entrySet().stream()
+            .filter(e -> cacheGrpPred.test(e.getKey()))
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+        IgniteCheckedException ex = shutdown(filteredStores.values(), cleanFiles);
+
+        if (ex != null)
+            U.error(log, "Failed to gracefully stop page store managers", ex);
+
+        idxCacheStores.entrySet().removeIf(e -> cacheGrpPred.test(e.getKey()));
+
+        U.log(log, "Cleanup cache stores [total=" + filteredStores.keySet().size() +
+            ", left=" + idxCacheStores.size() + ", cleanFiles=" + cleanFiles + ']');
+    }
+
     /** {@inheritDoc} */
     @Override public void stop0(boolean cancel) {
         if (log.isDebugEnabled())
             log.debug("Stopping page store manager.");
 
-        IgniteCheckedException ex = shutdown(false);
-
-        if (ex != null)
-            U.error(log, "Failed to gracefully stop page store manager", ex);
+        cleanupPageStoreIfMatch(p -> true, false);
     }
 
     /** {@inheritDoc} */
@@ -253,8 +270,6 @@ public FilePageStoreManager(GridKernalContext ctx) {
                 " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
 
         stop0(true);
-
-        idxCacheStores.clear();
     }
 
     /** {@inheritDoc} */
@@ -286,6 +301,8 @@ public FilePageStoreManager(GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cacheData) throws IgniteCheckedException {
+        assert storeWorkDir != null;
+
         int grpId = grpDesc.groupId();
 
         if (!idxCacheStores.containsKey(grpId)) {
@@ -299,6 +316,8 @@ public FilePageStoreManager(GridKernalContext ctx) {
 
     /** {@inheritDoc} */
     @Override public void initializeForMetastorage() throws IgniteCheckedException {
+        assert storeWorkDir != null;
+
         int grpId = MetaStorage.METASTORAGE_CACHE_ID;
 
         if (!idxCacheStores.containsKey(grpId)) {
@@ -849,10 +868,10 @@ public File cacheWorkDir(boolean isSharedGroup, String cacheOrGroupName) {
     /**
      * @param cleanFiles {@code True} if the stores should delete it's files upon close.
      */
-    private IgniteCheckedException shutdown(boolean cleanFiles) {
+    private IgniteCheckedException shutdown(Collection<CacheStoreHolder> holders, boolean cleanFiles) {
         IgniteCheckedException ex = null;
 
-        for (CacheStoreHolder holder : idxCacheStores.values())
+        for (CacheStoreHolder holder : holders)
             ex = shutdown(holder, cleanFiles, ex);
 
         return ex;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index 556d9974ecc2e..6ce4757c8b911 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -135,11 +135,6 @@ public MetaStorage(
         this.failureProcessor = cctx.kernalContext().failure();
     }
 
-    /** */
-    public MetaStorage(GridCacheSharedContext cctx, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
-        this(cctx, memPlc, memMetrics, false);
-    }
-
     /** */
     public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedException {
         getOrAllocateMetas();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 82bd9d38b0f88..5ebb8cfa668bc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -33,10 +33,8 @@
 import java.nio.ByteOrder;
 import java.nio.MappedByteBuffer;
 import java.nio.channels.ClosedByInterruptException;
-import java.nio.file.DirectoryStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.sql.Time;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -656,6 +654,8 @@ private void checkWalConfiguration() throws IgniteCheckedException {
     @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
         assert currHnd == null;
         assert lastPtr == null || lastPtr instanceof FileWALPointer;
+        assert (isArchiverEnabled() && archiver != null) || (!isArchiverEnabled() && archiver == null) :
+            "Trying to restore FileWriteHandle on deactivated write ahead log manager";
 
         FileWALPointer filePtr = (FileWALPointer)lastPtr;
 
@@ -1397,29 +1397,6 @@ private void checkOrPrepareFiles() throws StorageException {
             checkFiles(0, false, null, null);
     }
 
-    /** {@inheritDoc} */
-    @Override public void cleanupWalDirectories() throws IgniteCheckedException {
-        try {
-            try (DirectoryStream<Path> files = Files.newDirectoryStream(walWorkDir.toPath())) {
-                for (Path path : files)
-                    Files.delete(path);
-            }
-        }
-        catch (IOException e) {
-            throw new IgniteCheckedException("Failed to cleanup wal work directory: " + walWorkDir, e);
-        }
-
-        try {
-            try (DirectoryStream<Path> files = Files.newDirectoryStream(walArchiveDir.toPath())) {
-                for (Path path : files)
-                    Files.delete(path);
-            }
-        }
-        catch (IOException e) {
-            throw new IgniteCheckedException("Failed to cleanup wal archive directory: " + walArchiveDir, e);
-        }
-    }
-
     /**
      * Clears whole the file, fills with zeros for Default mode.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index f65e6b89a2c75..45935c3b50760 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -28,10 +28,8 @@
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
-import java.nio.file.DirectoryStream;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.sql.Time;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -507,6 +505,8 @@ private void checkWalConfiguration() throws IgniteCheckedException {
     @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
         assert currentHnd == null;
         assert lastPtr == null || lastPtr instanceof FileWALPointer;
+        assert (isArchiverEnabled() && archiver != null) || (!isArchiverEnabled() && archiver == null) :
+            "Trying to restore FileWriteHandle on deactivated write ahead log manager";
 
         FileWALPointer filePtr = (FileWALPointer)lastPtr;
 
@@ -926,29 +926,6 @@ private boolean hasIndex(long absIdx) {
         return cctx.walState().isDisabled(grpId);
     }
 
-    /** {@inheritDoc} */
-    @Override public void cleanupWalDirectories() throws IgniteCheckedException {
-        try {
-            try (DirectoryStream<Path> files = Files.newDirectoryStream(walWorkDir.toPath())) {
-                for (Path path : files)
-                    Files.delete(path);
-            }
-        }
-        catch (IOException e) {
-            throw new IgniteCheckedException("Failed to cleanup wal work directory: " + walWorkDir, e);
-        }
-
-        try {
-            try (DirectoryStream<Path> files = Files.newDirectoryStream(walArchiveDir.toPath())) {
-                for (Path path : files)
-                    Files.delete(path);
-            }
-        }
-        catch (IOException e) {
-            throw new IgniteCheckedException("Failed to cleanup wal archive directory: " + walArchiveDir, e);
-        }
-    }
-
     /**
      * Lists files in archive directory and returns the index of last archived file.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
index fc0804fad3297..c64c17b21c24e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
@@ -25,7 +25,6 @@
 import java.util.Arrays;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -176,15 +175,18 @@ public void testRecoveringOnNodeStartMarkerWriteFail() throws Exception {
         boolean activationFailed = false;
         try {
             grid = startGrid(0);
-            grid.cluster().active(true);
         }
-        catch (IgniteException e) {
-            log.warning("Activation test exception", e);
+        catch (IgniteCheckedException e) {
+            boolean interrupted = Thread.interrupted();
+
+            if (interrupted)
+                log.warning("Ignore interrupted excpetion [" +
+                    "thread=" + Thread.currentThread().getName() + ']', e);
 
             activationFailed = true;
         }
 
-        Assert.assertTrue("Activation must be failed", activationFailed);
+        Assert.assertTrue("Ignite instance startup must be failed", activationFailed);
 
         // Grid should be automatically stopped after checkpoint fail.
         awaitStop(grid);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
index dde333e00c219..c43396c625787 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
@@ -18,8 +18,12 @@
 package org.apache.ignite.internal.processors.cache.persistence.db.wal;
 
 import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -38,6 +42,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteCompute;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheMode;
@@ -53,9 +58,11 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.DiscoverySpiTestListener;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.discovery.IgniteDiscoverySpi;
 import org.apache.ignite.internal.pagemem.FullPageId;
 import org.apache.ignite.internal.pagemem.PageUtils;
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
@@ -67,14 +74,21 @@
 import org.apache.ignite.internal.pagemem.wal.record.TxRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
 import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry;
+import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntryType;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
 import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.lang.IgniteInClosureX;
 import org.apache.ignite.internal.util.typedef.CA;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.PA;
@@ -95,6 +109,8 @@
 import org.apache.ignite.transactions.TransactionIsolation;
 import org.junit.Assert;
 
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_IGNITE_INSTANCE_NAME;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DATA_FILENAME;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
 
 /**
@@ -110,6 +126,12 @@ public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
     /** */
     private boolean fork;
 
+    /** */
+    private static final String CACHE_NAME = "partitioned";
+
+    /** */
+    private static final String CACHE_TO_DESTROY_NAME = "destroyCache";
+
     /** */
     private String cacheName;
 
@@ -468,6 +490,110 @@ public void testWalLargeValue() throws Exception {
         }
     }
 
+    /**
+     * Check binary recover completes successfully when node stopped at the middle of checkpoint.
+     * Destroy cache_data.bin file for particular cache to emulate missing {@link DynamicCacheDescriptor}
+     * file (binary recovery should complete successfully in this case).
+     *
+     * @throws Exception if failed.
+     */
+    public void testBinaryRecoverBeforePMEWhenMiddleCheckpoint() throws Exception {
+        startGrids(3);
+
+        IgniteEx ig2 = grid(2);
+
+        ig2.cluster().active(true);
+
+        IgniteCache<Object, Object> cache = ig2.cache(CACHE_NAME);
+
+        for (int i = 1; i <= 4_000; i++)
+            cache.put(i, new BigObject(i));
+
+        BigObject objToCheck;
+
+        ig2.getOrCreateCache(CACHE_TO_DESTROY_NAME).put(1, objToCheck = new BigObject(1));
+
+        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ig2
+            .context().cache().context().database();
+
+        IgniteInternalFuture<?> cpFinishFut = dbMgr.forceCheckpoint("force checkpoint").finishFuture();
+
+        // Delete checkpoint END file to emulate node stopped at the middle of checkpoint.
+        cpFinishFut.listen(new IgniteInClosureX<IgniteInternalFuture>() {
+            @Override public void applyx(IgniteInternalFuture fut0) throws IgniteCheckedException {
+                try {
+                    CheckpointEntry cpEntry = dbMgr.checkpointHistory().lastCheckpoint();
+
+                    String cpEndFileName = GridCacheDatabaseSharedManager.checkpointFileName(cpEntry,
+                        CheckpointEntryType.END);
+
+                    Files.delete(Paths.get(dbMgr.checkpointDirectory().getAbsolutePath(), cpEndFileName));
+
+                    log.info("Checkpoint marker removed [cpEndFileName=" + cpEndFileName + ']');
+                }
+                catch (IOException e) {
+                    throw new IgniteCheckedException(e);
+                }
+            }
+        });
+
+        // Resolve cache directory. Emulating cache destroy in the middle of checkpoint.
+        IgniteInternalCache<Object, Object> destoryCache = ig2.cachex(CACHE_TO_DESTROY_NAME);
+
+        FilePageStoreManager pageStoreMgr = (FilePageStoreManager)destoryCache.context().shared().pageStore();
+
+        File destroyCacheWorkDir = pageStoreMgr.cacheWorkDir(destoryCache.configuration());
+
+        // Stop the whole cluster
+        stopAllGrids();
+
+        // Delete cache_data.bin file for this cache. Binary recovery should complete successfully after it.
+        final File[] files = destroyCacheWorkDir.listFiles(new FilenameFilter() {
+            @Override public boolean accept(final File dir, final String name) {
+                return name.endsWith(CACHE_DATA_FILENAME);
+            }
+        });
+
+        assertTrue(files.length > 0);
+
+        for (final File file : files)
+            assertTrue("Can't remove " + file.getAbsolutePath(), file.delete());
+
+        startGrids(2);
+
+        // Preprare Ignite instance configuration with additional Discovery checks.
+        final String ig2Name = getTestIgniteInstanceName(2);
+
+        final IgniteConfiguration onJoinCfg = optimize(getConfiguration(ig2Name));
+
+        // Check restore beeing called before PME and joining node to cluster.
+        ((IgniteDiscoverySpi)onJoinCfg.getDiscoverySpi())
+            .setInternalListener(new DiscoverySpiTestListener() {
+                @Override public void beforeJoin(ClusterNode locNode, IgniteLogger log) {
+                    String nodeName = locNode.attribute(ATTR_IGNITE_INSTANCE_NAME);
+
+                    GridCacheSharedContext sharedCtx = ((IgniteEx)ignite(getTestIgniteInstanceIndex(nodeName)))
+                        .context()
+                        .cache()
+                        .context();
+
+                    if (nodeName.equals(ig2Name)) {
+                        // Checkpoint history initialized on node start.
+                        assertFalse(((GridCacheDatabaseSharedManager)sharedCtx.database())
+                            .checkpointHistory().checkpoints().isEmpty());
+                    }
+
+                    super.beforeJoin(locNode, log);
+                }
+            });
+
+        Ignite restoredIg2 = startGrid(ig2Name, onJoinCfg);
+
+        awaitPartitionMapExchange();
+
+        assertEquals(restoredIg2.cache(CACHE_TO_DESTROY_NAME).get(1), objToCheck);
+    }
+
     /**
      * @throws Exception if failed.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
index 45486a12bdf36..c51a76739bd41 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalCompactionTest.java
@@ -21,6 +21,7 @@
 import java.io.RandomAccessFile;
 import java.util.Arrays;
 import java.util.Comparator;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
@@ -218,6 +219,20 @@ else if (arr[i] != 1) {
         }
 
         assertFalse(fail);
+
+        // Check compation successfully reset on blt changed.
+        stopAllGrids();
+
+        Ignite ignite = startGrids(2);
+
+        ignite.cluster().active(true);
+
+        resetBaselineTopology();
+
+        // This node will join to different blt.
+        startGrid(2);
+
+        awaitPartitionMapExchange();
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index 222804e150e44..a9fc179c6fb27 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -23,6 +23,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.internal.GridKernalContext;
@@ -227,6 +228,11 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public void cleanupPageStoreIfMatch(Predicate<Integer> cacheGrpPred, boolean cleanFiles) {
+        // No-op.
+    }
+
     /** {@inheritDoc} */
     @Override public boolean checkAndInitCacheWorkDir(CacheConfiguration cacheCfg) throws IgniteCheckedException {
         return false;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index 196cc7aa3b7a0..f1e2895c5e27e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -101,11 +101,6 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
         return false;
     }
 
-    /** {@inheritDoc} */
-    @Override public void cleanupWalDirectories() throws IgniteCheckedException {
-        // No-op.
-    }
-
     /** {@inheritDoc} */
     @Override public void start(GridCacheSharedContext cctx) throws IgniteCheckedException {
         // No-op.

From 64a614b0f16609d3e16e81f7c461cace5eb6481e Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Sat, 27 Oct 2018 17:55:54 +0300
Subject: [PATCH 457/543] GNITE-8006 Fixed flaky test

---
 .../org/apache/ignite/internal/util/IgniteUtilsSelfTest.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
index 3d4a425850601..c06922ea1be91 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
@@ -894,7 +894,7 @@ public void testDoInParallelBatch() {
             fail("Should throw timeout exception");
         }
         catch (Exception e) {
-            assertTrue(e.getCause() instanceof TimeoutException);
+            assertTrue(e.toString(), X.hasCause(e, TimeoutException.class));
         }
     }
 

From 3250d80222dc42d48c79163e54a15b3fd3bfadd8 Mon Sep 17 00:00:00 2001
From: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Date: Sat, 27 Oct 2018 14:23:06 +0300
Subject: [PATCH 458/543] IGNITE-10020 Fixed control.sh error comparison method
 violates its general contract - Fixes #5087.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit e111d895e7f7f32d6f764683ba8412d61d370455)
Signed-off-by: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
---
 .../cache/distribution/CacheDistributionTaskResult.java       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java
index 71de3bbc0dd05..52c6eec308afb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/distribution/CacheDistributionTaskResult.java
@@ -294,10 +294,10 @@ public void setUserAttributes(Map<String, String> userAttrs) {
 
             Row other = (Row)o;
 
-            int res = grpId - other.grpId;
+            int res = Integer.compare(grpId, other.grpId);
 
             if (res == 0) {
-                res = partId - other.partId;
+                res = Integer.compare(partId, other.partId);
 
                 if (res == 0)
                     res = nodeId.compareTo(other.nodeId);

From c4536f8e0f3d6284ec62abc511f9f7b260aac325 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Mon, 29 Oct 2018 11:34:04 +0700
Subject: [PATCH 459/543] GG-14362 Backport latest Web Console 2.5.3 to
 2.5.1-master.

---
 modules/web-console/assembly/README.txt       |  92 ++-
 .../docker/compose/frontend/nginx/nginx.conf  |  18 +-
 .../e2e/testcafe/components/FormField.js      |   6 +-
 .../e2e/testcafe/components/modalInput.js     |   8 +-
 .../clusterFormChangeDetection.js             |  12 +-
 .../PageConfigurationAdvancedCluster.js       |  11 +
 .../page-models/PageConfigurationBasic.js     |   2 +-
 modules/web-console/frontend/.babelrc         |  18 +-
 modules/web-console/frontend/.eslintrc        |  11 +-
 modules/web-console/frontend/app/app.js       | 217 ++++---
 .../frontend/app/browserUpdate/index.js       |   4 +-
 .../activities-user-dialog.controller.js      |   8 +-
 .../activities-user-dialog.tpl.pug            |  30 +-
 .../activities-user-dialog/index.js           |  33 +-
 .../app/components/bs-select-menu/index.js    |   6 +-
 .../components/bs-select-menu/index.spec.js   |   2 +-
 .../components/bs-select-menu/strip.filter.js |  22 +
 .../app/components/bs-select-menu/style.scss  |   4 +-
 .../components/bs-select-menu/template.pug    |   2 +-
 .../transcludeToBody.directive.js             |   2 +-
 .../connected-clusters-badge/controller.js    |   9 +-
 .../components/cell-logout/index.js           |   9 +-
 .../components/list/controller.js             |   1 +
 .../template.tpl.pug                          |   8 +-
 .../directives.js                             |  53 --
 .../components/form-field-size}/controller.js |   7 +-
 .../components/form-field-size/index.js}      |   2 +-
 .../components/form-field-size/style.scss     |  20 +
 .../components/form-field-size/template.pug   |  77 +++
 .../app/components/form-field/index.js        |   3 +
 .../grid-column-selector/component.js         |   2 +-
 .../grid-column-selector/style.scss           |   1 +
 .../app/components/grid-export/component.js   |   9 +-
 .../app/components/grid-export/index.js       |   2 +-
 .../grid-export/style.scss}                   |  35 +-
 .../app/components/grid-export/template.pug   |   6 +-
 .../grid-item-selected/component.js           |   2 +-
 .../components/grid-item-selected/index.js    |   1 +
 .../components/grid-item-selected/style.scss  |  23 +
 .../grid-item-selected/template.pug           |   5 +-
 .../app/components/grid-no-data/component.js  |   2 +-
 .../app/components/grid-no-data/controller.js |   7 +-
 .../components/grid-showing-rows/component.js |  29 +
 .../grid-showing-rows/controller.js}          |  51 +-
 .../index.js                                  |   5 +-
 .../components/grid-showing-rows/style.scss   |  20 +
 .../ignite-chart-series-selector/component.js |  28 +
 .../controller.js                             |  63 ++
 .../index.js                                  |   7 +-
 .../template.pug}                             |  22 +-
 .../app/components/ignite-chart/controller.js | 369 +++++++++++
 .../ignite-chart/index.js}                    |  37 +-
 .../app/components/ignite-chart/style.scss    |  85 +++
 .../ignite-chart/template.pug}                |  30 +-
 .../app/components/ignite-icon/directive.js   |  19 +-
 .../app/components/ignite-icon/service.js     |  17 +
 .../app/components/ignite-status/index.js     |  22 +
 .../types.ts => ignite-status/style.scss}     |  13 +-
 .../input-dialog/input-dialog.controller.js   |  15 +-
 .../input-dialog/input-dialog.service.js      |  59 +-
 .../input-dialog/input-dialog.tpl.pug         |  38 +-
 .../list-editable-cols/cols.style.scss        |   8 +-
 .../list-editable-cols/cols.template.pug      |   2 +-
 .../components/list-editable-cols/index.js    |   4 +-
 .../components/list-editable-one-way/index.js |   2 +-
 .../list-editable-save-on-changes/index.js    |   6 +-
 .../list-editable-transclude/index.js         |   2 +-
 .../app/components/list-editable/style.scss   |   6 +-
 .../list-of-registered-users/controller.js    |   2 +-
 .../list-of-registered-users/style.scss       |  14 +-
 .../list-of-registered-users/template.tpl.pug |  67 +-
 .../cache-edit-form/templates/affinity.pug    | 121 +++-
 .../cache-edit-form/templates/concurrency.pug |  78 ++-
 .../cache-edit-form/templates/general.pug     | 143 +++--
 .../cache-edit-form/templates/memory.pug      |  97 ++-
 .../templates/near-cache-client.pug           |  40 +-
 .../templates/near-cache-server.pug           |  42 +-
 .../cache-edit-form/templates/node-filter.pug |  23 +-
 .../cache-edit-form/templates/query.pug       | 105 ++--
 .../cache-edit-form/templates/rebalance.pug   |  85 ++-
 .../cache-edit-form/templates/statistics.pug  |  17 +-
 .../cache-edit-form/templates/store.pug       | 384 +++++++-----
 .../cluster-edit-form/templates/atomic.pug    | 107 +++-
 .../templates/attributes.pug                  |  20 +-
 .../cluster-edit-form/templates/binary.pug    | 126 ++--
 .../templates/cache-key-cfg.pug               |  68 ++-
 .../templates/checkpoint.pug                  | 102 ++--
 .../templates/checkpoint/fs.pug               |  10 +-
 .../templates/checkpoint/jdbc.pug             | 107 +++-
 .../templates/checkpoint/s3.pug               | 489 +++++++++++----
 .../templates/client-connector.pug            | 163 ++++-
 .../cluster-edit-form/templates/collision.pug |  27 +-
 .../templates/collision/custom.pug            |   9 +-
 .../templates/collision/fifo-queue.pug        |  20 +-
 .../templates/collision/job-stealing.pug      |  76 ++-
 .../templates/collision/priority-queue.pug    |  66 +-
 .../templates/communication.pug               | 200 +++++-
 .../cluster-edit-form/templates/connector.pug | 230 +++++--
 .../templates/data-storage.pug                | 386 ++++++++----
 .../templates/deployment.pug                  | 152 +++--
 .../cluster-edit-form/templates/discovery.pug | 200 +++++-
 .../cluster-edit-form/templates/events.pug    |  92 ++-
 .../cluster-edit-form/templates/failover.pug  | 117 ++--
 .../cluster-edit-form/templates/general.pug   |  51 +-
 .../templates/general/discovery/cloud.pug     |  48 +-
 .../templates/general/discovery/google.pug    |  45 +-
 .../templates/general/discovery/jdbc.pug      |  29 +-
 .../general/discovery/kubernetes.pug          |  43 +-
 .../templates/general/discovery/multicast.pug |  97 ++-
 .../templates/general/discovery/s3.pug        |  31 +-
 .../templates/general/discovery/shared.pug    |   8 +-
 .../templates/general/discovery/vm.pug        |  54 +-
 .../templates/general/discovery/zookeeper.pug | 103 ++--
 .../bounded-exponential-backoff.pug           |  28 +-
 .../zookeeper/retrypolicy/custom.pug          |  11 +-
 .../retrypolicy/exponential-backoff.pug       |  28 +-
 .../zookeeper/retrypolicy/forever.pug         |   9 +-
 .../zookeeper/retrypolicy/n-times.pug         |  18 +-
 .../zookeeper/retrypolicy/one-time.pug        |  11 +-
 .../zookeeper/retrypolicy/until-elapsed.pug   |  18 +-
 .../cluster-edit-form/templates/hadoop.pug    | 119 +++-
 .../templates/load-balancing.pug              | 210 ++++---
 .../cluster-edit-form/templates/logger.pug    |  31 +-
 .../templates/logger/custom.pug               |   9 +-
 .../templates/logger/log4j.pug                |  67 +-
 .../templates/logger/log4j2.pug               |  40 +-
 .../templates/marshaller.pug                  |  99 ++-
 .../cluster-edit-form/templates/memory.pug    | 252 ++++----
 .../cluster-edit-form/templates/metrics.pug   |  52 +-
 .../cluster-edit-form/templates/misc.pug      |  73 ++-
 .../cluster-edit-form/templates/odbc.pug      |  75 ++-
 .../templates/persistence.pug                 | 191 +++++-
 .../cluster-edit-form/templates/service.pug   | 122 ++--
 .../templates/sql-connector.pug               |  88 ++-
 .../cluster-edit-form/templates/ssl.pug       | 110 ++--
 .../cluster-edit-form/templates/swap.pug      |  74 ++-
 .../cluster-edit-form/templates/thread.pug    | 198 +++---
 .../cluster-edit-form/templates/time.pug      |  47 +-
 .../templates/transactions.pug                |  81 ++-
 .../igfs-edit-form/templates/dual.pug         |  26 +-
 .../igfs-edit-form/templates/fragmentizer.pug |  37 +-
 .../igfs-edit-form/templates/general.pug      |  10 +-
 .../igfs-edit-form/templates/ipc.pug          |  77 ++-
 .../igfs-edit-form/templates/misc.pug         | 198 ++++--
 .../igfs-edit-form/templates/secondary.pug    |  35 +-
 .../components/model-edit-form/controller.js  |   3 +-
 .../model-edit-form/templates/general.pug     |  55 +-
 .../model-edit-form/templates/query.pug       | 386 ++++++------
 .../model-edit-form/templates/store.pug       |  86 ++-
 .../controller.js                             |   7 +-
 .../page-configure-advanced-caches/index.js   |   2 +-
 .../controller.js                             |   2 +-
 .../controller.js                             |   3 +-
 .../controller.js                             |   3 +-
 .../page-configure-advanced/style.scss        |   6 -
 .../page-configure-basic/controller.js        |   6 +-
 .../page-configure-basic/style.scss           |  29 +-
 .../page-configure-basic/template.pug         |  78 ++-
 .../page-configure-overview/controller.js     |  10 +-
 .../components/formUICanExitGuard.js          |   6 +-
 .../modal-import-models/component.js          |  20 +-
 .../components/modal-import-models/index.js   |  12 +-
 .../component.js                              |   2 +-
 .../step-indicator/component.js               |   1 -
 .../tables-action-cell/component.js           |   1 -
 .../tables-action-cell/style.scss             |   1 -
 .../tables-action-cell/template.pug           |  34 +-
 .../modal-import-models/template.tpl.pug      | 141 ++++-
 .../components/modal-preview-project/index.js |   2 +-
 .../modal-preview-project/template.pug        |   5 +-
 .../components/pc-form-field-size/style.scss  |  52 --
 .../pc-form-field-size/template.pug           |  61 --
 .../components/pc-items-table/style.scss      |   3 -
 .../components/pc-items-table/template.pug    |  37 +-
 .../app/components/page-configure/index.js    |   8 +-
 .../services/ConfigChangesGuard.js            |   4 +-
 .../services/ConfigurationResource.js         |   4 +-
 .../services/ConfigurationResource.spec.js    |  78 +++
 .../page-configure/services/PageConfigure.js  |   5 +-
 .../app/components/page-configure/states.js   |   3 +-
 .../page-configure/store/effects.js           |  17 +-
 .../app/components/page-configure/style.scss  |  58 +-
 .../components/page-configure/template.pug    |   2 +-
 .../page-password-reset/controller.js         |   6 +-
 .../app/components/page-profile/controller.js |  21 +-
 .../app/components/page-profile/template.pug  |   2 +-
 .../app/components/page-queries/component.js  |  11 +-
 .../components/queries-notebook/controller.js |  42 +-
 .../components/queries-notebook/style.scss    |  52 ++
 .../queries-notebook/template.tpl.pug         | 270 ++++++---
 .../queries-notebooks-list/template.tpl.pug   |  28 +-
 .../components/page-queries/notebook.data.js  |   5 +
 .../page-queries/notebook.service.js          |   8 +-
 .../app/components/page-queries/style.scss    |   4 +
 .../{component.js => component.ts}            |   0
 .../{controller.js => controller.ts}          |  38 +-
 .../page-signin/{index.js => index.ts}        |   0
 .../components/page-signin/{run.js => run.ts} |  11 +-
 .../app/components/page-signup/controller.js  |   3 +
 .../panel-collapsible/index.spec.js           |   2 +-
 .../components/ui-grid-filters/directive.js   |   3 +-
 .../ui-grid/component.js}                     |  49 +-
 .../app/components/ui-grid/controller.js      | 219 +++++++
 .../ui-grid/decorator.js}                     |  35 +-
 .../frontend/app/components/ui-grid/index.js  |  25 +
 .../app/components/ui-grid/style.scss         | 164 +++++
 .../app/components/ui-grid/template.pug       |  60 ++
 .../components/user-notifications/service.js  |  11 +-
 .../components/user-notifications/style.scss  |  39 --
 .../user-notifications/template.tpl.pug       |  28 +-
 .../components/version-picker/component.js    |   5 +
 .../frontend/app/core/admin/Admin.data.js     |  21 +-
 .../frontend/app/data/getting-started.json    |  56 +-
 .../app/directives/auto-focus.directive.js    |  15 +-
 .../directives/bs-affix-update.directive.js   |  10 +-
 .../app/directives/btn-ignite-link.js         |  22 +-
 .../directives/centered/centered.directive.js |   4 +-
 .../directives/copy-to-clipboard.directive.js |  15 +-
 .../hide-on-state-change.directive.js         |  15 +-
 .../information/information.directive.js      |   4 +-
 .../directives/on-click-focus.directive.js    |  22 +-
 .../on-enter-focus-move.directive.js          |  22 +-
 .../app/directives/on-enter.directive.js      |  22 +-
 .../app/directives/on-escape.directive.js     |  22 +-
 .../restore-input-focus.directive.js          |  12 +-
 .../directives/retain-selection.directive.js  |  21 +-
 .../ui-ace-docker/ui-ace-docker.controller.js |   6 +-
 .../ui-ace-docker/ui-ace-docker.directive.js  |   4 +-
 .../ui-ace-java/ui-ace-java.directive.js      |   4 +-
 .../ui-ace-pojos/ui-ace-pojos.controller.js   |   7 +-
 .../ui-ace-pojos/ui-ace-pojos.directive.js    |   4 +-
 .../ui-ace-pom/ui-ace-pom.controller.js       |   6 +-
 .../ui-ace-pom/ui-ace-pom.directive.js        |   4 +-
 .../ui-ace-sharp/ui-ace-sharp.controller.js   |  11 +-
 .../ui-ace-sharp/ui-ace-sharp.directive.js    |  20 +-
 .../ui-ace-spring/ui-ace-spring.directive.js  |   4 +-
 .../app/directives/ui-ace-tabs.directive.js   |   6 +-
 .../frontend/app/filters/byName.filter.js     |   6 +-
 .../bytes.filter.js}                          |  36 +-
 .../bytes.filter.spec.js}                     |  41 +-
 .../app/filters/default-name.filter.js        |  16 +-
 .../app/filters/domainsValidation.filter.js   |  42 +-
 .../frontend/app/filters/duration.filter.js   |  10 +-
 .../frontend/app/filters/hasPojo.filter.js    |   4 +-
 .../app/filters/uiGridSubcategories.filter.js |   6 +-
 .../frontend/app/helpers/jade/form.pug        |  26 -
 .../helpers/jade/form/form-field-checkbox.pug |  44 --
 .../app/helpers/jade/form/form-field-down.pug |  18 -
 .../helpers/jade/form/form-field-dropdown.pug |  60 --
 .../helpers/jade/form/form-field-feedback.pug |  18 -
 .../helpers/jade/form/form-field-number.pug   |  59 --
 .../helpers/jade/form/form-field-password.pug |  47 --
 .../app/helpers/jade/form/form-field-text.pug |  53 --
 .../app/helpers/jade/form/form-field-up.pug   |  18 -
 .../frontend/app/helpers/jade/mixins.pug      | 571 +++++++-----------
 .../frontend/app/modules/ace.module.js        |   2 +-
 .../app/modules/agent/AgentManager.service.js |  31 +-
 .../app/modules/agent/AgentModal.service.js   |  38 +-
 .../app/modules/agent/agent.module.js         |   4 +-
 .../components/cluster-login/component.js     |   7 +-
 .../agent/components/cluster-login/index.js   |   7 +-
 .../agent/components/cluster-login/service.js |  30 +-
 .../components/cluster-login/template.pug     |   7 +-
 .../app/modules/branding/branding.module.js   |  12 +-
 .../app/modules/branding/branding.service.js  |   3 +
 .../modules/branding/features.directive.js    |   9 +-
 .../app/modules/branding/footer.directive.js  |   9 +-
 .../modules/branding/header-logo.directive.js |   9 +-
 .../branding/header-title.directive.js        |   9 +-
 .../branding/powered-by-apache.directive.js   |   9 +-
 .../app/modules/branding/terms.directive.js   |   9 +-
 .../configuration/configuration.module.js     |   6 +-
 .../generator/ConfigurationGenerator.js       |   2 +-
 .../generator/PlatformGenerator.js            |  13 +-
 .../frontend/app/modules/demo/Demo.module.js  | 178 +++---
 .../dialog/dialog-content.directive.js        |   4 +-
 .../modules/dialog/dialog-title.directive.js  |   4 +-
 .../app/modules/dialog/dialog.directive.js    |   4 +-
 .../app/modules/dialog/dialog.factory.js      |   9 +-
 .../app/modules/dialog/dialog.module.js       |   8 +-
 .../field/bs-select-placeholder.directive.js  |  10 +-
 .../form/field/input/autofocus.directive.js   |  16 +-
 .../app/modules/form/field/label.directive.js |  47 --
 .../modules/form/field/tooltip.directive.js   |  49 --
 .../frontend/app/modules/form/form.module.js  |  52 +-
 .../app/modules/form/group/add.directive.js   |  38 --
 .../modules/form/group/tooltip.directive.js   |  38 --
 .../modules/form/panel/chevron.directive.js   |  56 --
 .../modules/form/services/FormGUID.service.js |   4 +-
 .../form/validator/ipaddress.directive.js     |  17 +-
 .../java-built-in-class.directive.js          |  17 +-
 .../validator/java-identifier.directive.js    |  17 +-
 .../form/validator/java-keywords.directive.js |  17 +-
 .../validator/java-package-name.directive.js  |  17 +-
 .../java-package-specified.directive.js       |  17 +-
 .../validator/property-unique.directive.js    |  17 +-
 .../property-value-specified.directive.js     |  12 +-
 .../form/validator/unique.directive.js        |  11 +-
 .../modules/form/validator/uuid.directive.js  |  17 +-
 .../GettingStarted.provider.js                | 161 +++--
 .../app/modules/loading/loading.directive.js  |  10 +-
 .../app/modules/loading/loading.module.js     |   4 +-
 .../app/modules/loading/loading.service.js    |  10 +-
 .../app/modules/navbar/Navbar.provider.js     |  10 +-
 .../app/modules/navbar/Userbar.provider.js    |  10 +-
 .../app/modules/navbar/navbar.directive.js    |   6 +-
 .../app/modules/navbar/navbar.module.js       |   8 +-
 .../app/modules/navbar/userbar.directive.js   |  57 +-
 .../app/modules/nodes/Nodes.service.js        |   4 +-
 .../modules/nodes/nodes-dialog.controller.js  |   6 +-
 .../app/modules/nodes/nodes-dialog.tpl.pug    |  11 +-
 .../frontend/app/modules/socket.module.js     |  16 +-
 .../app/modules/states/admin.state.js         |   2 +-
 .../app/modules/states/errors.state.js        |   2 +-
 .../app/modules/states/logout.state.js        |   4 +-
 .../app/modules/states/settings.state.js      |   2 +-
 .../frontend/app/modules/user/Auth.service.js |   4 +
 .../frontend/app/modules/user/User.service.js |  31 +-
 .../frontend/app/modules/user/user.module.js  |  48 +-
 .../frontend/app/primitives/btn/index.scss    |  14 +-
 .../app/primitives/checkbox/index.scss        |  20 -
 .../app/primitives/datepicker/index.pug       |  30 +-
 .../app/primitives/datepicker/index.scss      |  55 --
 .../app/primitives/form-field/checkbox.pug    |   4 +-
 .../app/primitives/form-field/dropdown.pug    |  18 +-
 .../app/primitives/form-field/email.pug       |   4 +-
 .../app/primitives/form-field/error.pug       |   1 +
 .../app/primitives/form-field/index.pug       |   2 +
 .../app/primitives/form-field/index.scss      | 286 ++++++++-
 .../app/primitives/form-field/input.pug       |   7 +-
 .../app/primitives/form-field/label.pug       |  27 +-
 .../app/primitives/form-field/number.pug      |  21 +-
 .../app/primitives/form-field/password.pug    |   4 +-
 .../app/primitives/form-field/phone.pug       |   2 +-
 .../{file/index.pug => form-field/radio.pug}  |  29 +-
 .../app/primitives/form-field/text.pug        |   9 +-
 .../app/primitives/form-field/tooltip.pug     |  12 +-
 .../form-field/typeahead.pug}                 |  31 +-
 .../frontend/app/primitives/index.js          |   2 +-
 .../frontend/app/primitives/modal/index.scss  |  45 +-
 .../frontend/app/primitives/panel/index.scss  |  28 +-
 .../frontend/app/primitives/radio/index.pug   |  37 --
 .../frontend/app/primitives/radio/index.scss  |  78 ---
 .../app/primitives/spinner-circle/index.scss  |  59 ++
 .../app/primitives/timepicker/index.pug       |  33 +-
 .../app/primitives/timepicker/index.scss      | 130 ++--
 .../app/primitives/ui-grid/index.scss         |   6 +-
 .../services/AngularStrapSelect.decorator.js  |   2 +-
 .../services/AngularStrapTooltip.decorator.js |   2 +-
 .../app/services/ChartColors.service.js       |   4 +-
 .../frontend/app/services/Clusters.js         |   2 +-
 .../frontend/app/services/Clusters.spec.js    |   2 +-
 .../frontend/app/services/Confirm.service.js  |  17 +-
 .../app/services/CopyToClipboard.service.js   |  18 +-
 .../app/services/Countries.service.js         |  16 +-
 .../frontend/app/services/DefaultState.js     |   3 +
 .../app/services/ErrorParser.service.js       |   2 +-
 .../app/services/ErrorPopover.service.js      |  14 +-
 .../frontend/app/services/Focus.service.js    |  14 +-
 .../app/services/FormUtils.service.js         |  22 +-
 .../frontend/app/services/IGFSs.js            |   8 +-
 .../app/services/InetAddress.service.js       |  10 +-
 .../app/services/LegacyTable.service.js       | 338 ++++++-----
 .../app/services/LegacyUtils.service.js       |  10 +-
 .../frontend/app/services/Messages.service.js |  19 +-
 .../app/services/ModelNormalizer.service.js   |   6 +-
 .../frontend/app/services/SqlTypes.service.js |   6 +-
 .../services/UnsavedChangesGuard.service.js   |  38 --
 .../frontend/app/services/Version.service.js  |  10 +-
 .../frontend/app/services/exceptionHandler.js |   3 +
 modules/web-console/frontend/app/style.scss   |  22 +
 modules/web-console/frontend/app/vendor.js    |   1 -
 modules/web-console/frontend/package.json     |  90 +--
 .../frontend/public/images/icons/index.js     |  56 +-
 .../frontend/public/stylesheets/style.scss    |  90 ++-
 .../public/stylesheets/variables.scss         |   6 +-
 .../frontend/test/karma.conf.babel.js         | 113 ----
 .../web-console/frontend/test/karma.conf.js   |  97 ++-
 .../test/unit/JavaTransformer.test.js         |   2 +-
 .../frontend/test/unit/JavaTypes.test.js      |   2 +-
 .../test/unit/SpringTransformer.test.js       |   2 +-
 .../frontend/test/unit/SqlTypes.test.js       |   2 +-
 .../test/unit/defaultName.filter.test.js      |   3 +-
 modules/web-console/frontend/tsconfig.json    |   1 +
 .../frontend/views/sql/paragraph-rate.tpl.pug |  41 +-
 .../views/templates/agent-download.tpl.pug    |   9 +-
 .../views/templates/batch-confirm.tpl.pug     |   2 +-
 .../frontend/views/templates/confirm.tpl.pug  |  12 +-
 .../views/templates/demo-info.tpl.pug         |  38 +-
 .../views/templates/getting-started.tpl.pug   |  31 +-
 .../frontend/views/templates/message.tpl.pug  |  17 +-
 .../frontend/webpack/webpack.common.js        |  78 +--
 .../{webpack.dev.babel.js => webpack.dev.js}  |  10 +-
 ...{webpack.prod.babel.js => webpack.prod.js} |  12 +-
 .../frontend/webpack/webpack.test.js          |   7 +-
 395 files changed, 10417 insertions(+), 5682 deletions(-)
 create mode 100644 modules/web-console/frontend/app/components/bs-select-menu/strip.filter.js
 delete mode 100644 modules/web-console/frontend/app/components/expose-ignite-form-field-control/directives.js
 rename modules/web-console/frontend/app/components/{page-configure/components/pc-form-field-size => form-field/components/form-field-size}/controller.js (95%)
 rename modules/web-console/frontend/app/components/{page-configure/components/pc-form-field-size/component.js => form-field/components/form-field-size/index.js} (100%)
 create mode 100644 modules/web-console/frontend/app/components/form-field/components/form-field-size/style.scss
 create mode 100644 modules/web-console/frontend/app/components/form-field/components/form-field-size/template.pug
 rename modules/web-console/frontend/app/{modules/form/field/input/text.scss => components/grid-export/style.scss} (69%)
 create mode 100644 modules/web-console/frontend/app/components/grid-item-selected/style.scss
 create mode 100644 modules/web-console/frontend/app/components/grid-showing-rows/component.js
 rename modules/web-console/frontend/app/{primitives/file/index.scss => components/grid-showing-rows/controller.js} (55%)
 rename modules/web-console/frontend/app/components/{page-configure/components/pc-form-field-size => grid-showing-rows}/index.js (89%)
 create mode 100644 modules/web-console/frontend/app/components/grid-showing-rows/style.scss
 create mode 100644 modules/web-console/frontend/app/components/ignite-chart-series-selector/component.js
 create mode 100644 modules/web-console/frontend/app/components/ignite-chart-series-selector/controller.js
 rename modules/web-console/frontend/app/components/{expose-ignite-form-field-control => ignite-chart-series-selector}/index.js (81%)
 rename modules/web-console/frontend/app/{helpers/jade/form/form-field-label.pug => components/ignite-chart-series-selector/template.pug} (64%)
 create mode 100644 modules/web-console/frontend/app/components/ignite-chart/controller.js
 rename modules/web-console/frontend/app/{modules/form/field/feedback.scss => components/ignite-chart/index.js} (58%)
 create mode 100644 modules/web-console/frontend/app/components/ignite-chart/style.scss
 rename modules/web-console/frontend/app/{primitives/tooltip/index.pug => components/ignite-chart/template.pug} (54%)
 create mode 100644 modules/web-console/frontend/app/components/ignite-status/index.js
 rename modules/web-console/frontend/app/components/{page-signin/types.ts => ignite-status/style.scss} (79%)
 delete mode 100644 modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/style.scss
 delete mode 100644 modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/template.pug
 create mode 100644 modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.spec.js
 rename modules/web-console/frontend/app/components/page-signin/{component.js => component.ts} (100%)
 rename modules/web-console/frontend/app/components/page-signin/{controller.js => controller.ts} (73%)
 rename modules/web-console/frontend/app/components/page-signin/{index.js => index.ts} (100%)
 rename modules/web-console/frontend/app/components/page-signin/{run.js => run.ts} (90%)
 rename modules/web-console/frontend/app/{modules/form/field/field.scss => components/ui-grid/component.js} (56%)
 create mode 100644 modules/web-console/frontend/app/components/ui-grid/controller.js
 rename modules/web-console/frontend/app/{modules/form/field/up.directive.js => components/ui-grid/decorator.js} (50%)
 create mode 100644 modules/web-console/frontend/app/components/ui-grid/index.js
 create mode 100644 modules/web-console/frontend/app/components/ui-grid/style.scss
 create mode 100644 modules/web-console/frontend/app/components/ui-grid/template.pug
 rename modules/web-console/frontend/app/{modules/form/field/form-control-feedback.directive.js => filters/bytes.filter.js} (57%)
 rename modules/web-console/frontend/app/{modules/form/field/down.directive.js => filters/bytes.filter.spec.js} (52%)
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form.pug
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form/form-field-down.pug
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form/form-field-number.pug
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
 delete mode 100644 modules/web-console/frontend/app/helpers/jade/form/form-field-up.pug
 delete mode 100644 modules/web-console/frontend/app/modules/form/field/label.directive.js
 delete mode 100644 modules/web-console/frontend/app/modules/form/field/tooltip.directive.js
 delete mode 100644 modules/web-console/frontend/app/modules/form/group/add.directive.js
 delete mode 100644 modules/web-console/frontend/app/modules/form/group/tooltip.directive.js
 delete mode 100644 modules/web-console/frontend/app/modules/form/panel/chevron.directive.js
 rename modules/web-console/frontend/app/primitives/{file/index.pug => form-field/radio.pug} (53%)
 rename modules/web-console/frontend/app/{helpers/jade/form/form-field-datalist.pug => primitives/form-field/typeahead.pug} (64%)
 delete mode 100644 modules/web-console/frontend/app/primitives/radio/index.pug
 delete mode 100644 modules/web-console/frontend/app/primitives/radio/index.scss
 create mode 100644 modules/web-console/frontend/app/primitives/spinner-circle/index.scss
 delete mode 100644 modules/web-console/frontend/app/services/UnsavedChangesGuard.service.js
 create mode 100644 modules/web-console/frontend/app/style.scss
 delete mode 100644 modules/web-console/frontend/test/karma.conf.babel.js
 rename modules/web-console/frontend/webpack/{webpack.dev.babel.js => webpack.dev.js} (93%)
 rename modules/web-console/frontend/webpack/{webpack.prod.babel.js => webpack.prod.js} (87%)

diff --git a/modules/web-console/assembly/README.txt b/modules/web-console/assembly/README.txt
index 9699a48496d03..dd323993e5231 100644
--- a/modules/web-console/assembly/README.txt
+++ b/modules/web-console/assembly/README.txt
@@ -1,54 +1,90 @@
 Requirements
 -------------------------------------
-1. JDK 8 bit for your platform, or newer.
+1. JDK 8 suitable for your platform.
 2. Supported browsers: Chrome, Firefox, Safari, Edge.
-3. Ignite cluster should be started with `ignite-rest-http` module in classpath. For this copy `ignite-rest-http` folder from `libs\optional` to `libs` folder.
-
+3. Ignite cluster should be started with `ignite-rest-http` module in classpath.
+ For this copy `ignite-rest-http` folder from `libs\optional` to `libs` folder.
 
 How to run
 -------------------------------------
 1. Unpack ignite-web-console-x.x.x.zip to some folder.
 2. Change work directory to folder where Web Console was unpacked.
 3. Start ignite-web-console-xxx executable for you platform:
-	For Linux: ignite-web-console-linux
-	For MacOS: ignite-web-console-macos
-	For Windows: ignite-web-console-win.exe
+    For Linux: `sudo ./ignite-web-console-linux`
+    For macOS: `sudo ./ignite-web-console-macos`
+    For Windows: `ignite-web-console-win.exe`
 
-Note: on Linux and Mac OS X `root` permission is required to bind to 80 port, but you may always start Web Console on another port if you don't have such permission.
+NOTE: For Linux and macOS `sudo` required because non-privileged users are not allowed to bind to privileged ports (port numbers below 1024).
 
 4. Open URL `localhost` in browser.
 5. Login with user `admin@admin` and password `admin`.
 6. Start web agent from folder `web agent`. For Web Agent settings see `web-agent\README.txt`.
-Cluster URL should be specified in `web-agent\default.properties` in `node-uri` parameter.
+
+NOTE: Cluster URL should be specified in `web-agent\default.properties` in `node-uri` parameter.
 
 Technical details
 -------------------------------------
 1. Package content:
-	`libs` - this folder contains Web Console and MongoDB binaries.
-	`user_data` - this folder contains all Web Console data (registered users, created objects, ...) and should be preserved in case of update to new version.
-2. Package already contains MongoDB for Mac OS X, Windows, RHEL, CentOs and Ubuntu on other platforms MongoDB will be downloaded on first start. MongoDB executables will be downloaded to `libs\mogodb` folder.
+    `libs` - this folder contains Web Console and MongoDB binaries.
+    `user_data` - this folder contains all Web Console data (registered users, created objects, ...) and
+     should be preserved in case of update to new version.
+    `web-agent` - this folder contains Web Agent.
+2. Package already contains MongoDB for macOS, Windows, RHEL, CentOs and Ubuntu on other platforms MongoDB will be downloaded on first start. MongoDB executables will be downloaded to `libs\mogodb` folder.
 3. Web console will start on default HTTP port `80` and bind to all interfaces `0.0.0.0`.
 3. To bind Web Console to specific network interface:
-	On Linux: `./ignite-web-console-linux --server:host 192.168.0.1`
-	On Windows: `ignite-web-console-win.exe --server:host 192.168.0.1`
+    On Linux: `./ignite-web-console-linux --server:host 192.168.0.1`
+    On macOS: `sudo ./ignite-web-console-macos --server:host 192.168.0.1`
+    On Windows: `ignite-web-console-win.exe --server:host 192.168.0.1`
 4. To start Web Console on another port, for example `3000`:
-	On Linux: `sudo ./ignite-web-console-linux --server:port 3000`
-	On Windows: `ignite-web-console-win.exe --server:port 3000`
+    On Linux: `sudo ./ignite-web-console-linux --server:port 3000`
+    On macOS: `./ignite-web-console-macos --server:port 3000`
+    On Windows: `ignite-web-console-win.exe --server:port 3000`
 
 All available parameters with defaults:
-	Web Console host:           --server:host 0.0.0.0
-	Web Console port:           --server:port 80
-	Enable HTTPS:               --server:ssl false
-	HTTPS key:                  --server:key "serve/keys/test.key"
-	HTTPS certificate:          --server:cert "serve/keys/test.crt"
-	HTTPS passphrase:           --server:keyPassphrase "password"
-	MongoDB URL:                --mongodb:url mongodb://localhost/console
-	Mail service:               --mail:service "gmail"
-	Signature text:             --mail:sign "Kind regards, Apache Ignite Team"
-	Greeting text:              --mail:greeting "Apache Ignite Web Console"
-	Mail FROM:                  --mail:from "Apache Ignite Web Console <someusername@somecompany.somedomain>"
-	User to send e-mail:        --mail:auth:user "someusername@somecompany.somedomain"
-	E-mail service password:    --mail:auth:pass ""
+    Web Console host:           --server:host 0.0.0.0
+    Web Console port:           --server:port 80
+    Enable HTTPS:               --server:ssl false
+    HTTPS key:                  --server:key "serve/keys/test.key"
+    HTTPS certificate:          --server:cert "serve/keys/test.crt"
+    HTTPS passphrase:           --server:keyPassphrase "password"
+    MongoDB URL:                --mongodb:url mongodb://localhost/console
+    Mail service:               --mail:service "gmail"
+    Signature text:             --mail:sign "Kind regards, Apache Ignite Team"
+    Greeting text:              --mail:greeting "Apache Ignite Web Console"
+    Mail FROM:                  --mail:from "Apache Ignite Web Console <someusername@somecompany.somedomain>"
+    User to send e-mail:        --mail:auth:user "someusername@somecompany.somedomain"
+    E-mail service password:    --mail:auth:pass ""
+
+Sample usage:
+    `ignite-web-console-win.exe --mail:auth:user "my_user@gmail.com"  --mail:auth:pass "my_password"`
+
+Advanced configuration of SMTP for Web Console.
+-------------------------------------
+1. Create sub-folder "config" in folder with Web Console executable.
+2. Create in config folder file "settings.json".
+3. Specify SMTP settings in settings.json (updating to your specific names and passwords):
+
+Sample "settings.json":
+{
+    "mail": {
+        "service": "gmail",
+        "greeting": "My Company Greeting",
+        "from": "My Company Web Console <some_name@gmail.com>",
+        "sign": "Kind regards,<br>My Company Team",
+        "auth": {
+            "user": "some_name@gmail.com",
+            "pass": "my_password"
+        }
+    }
+}
+
+Web Console sends e-mails with help of NodeMailer: https://nodemailer.com.
+
+Documentation available here:
+   https://nodemailer.com/smtp
+   https://nodemailer.com/smtp/well-known
+
+In case of non GMail SMTP server it may require to change options in "settings.json" according to NodeMailer documentation.
 
 Troubleshooting
 -------------------------------------
diff --git a/modules/web-console/docker/compose/frontend/nginx/nginx.conf b/modules/web-console/docker/compose/frontend/nginx/nginx.conf
index dc208f51ed630..c98875f914d5f 100644
--- a/modules/web-console/docker/compose/frontend/nginx/nginx.conf
+++ b/modules/web-console/docker/compose/frontend/nginx/nginx.conf
@@ -16,23 +16,25 @@
 #
 
 user  nginx;
-worker_processes  1;
+worker_processes auto;
 
 error_log  /var/log/nginx/error.log  warn;
 pid        /var/run/nginx.pid;
 
 events {
-  worker_connections  128;
+  use epoll;
+  worker_connections   512;
+  multi_accept         on;
 }
 
 http {
-  server_tokens off;
-  sendfile            on;
-  aio                 on;
-  tcp_nopush          on;
+  server_tokens        off;
+  sendfile             on;
+  aio                  on;
+  tcp_nopush           on;
 
-  keepalive_timeout   60;
-  tcp_nodelay         on;
+  keepalive_timeout    60;
+  tcp_nodelay          on;
 
   client_max_body_size 100m;
 
diff --git a/modules/web-console/e2e/testcafe/components/FormField.js b/modules/web-console/e2e/testcafe/components/FormField.js
index 71d951c74a7db..4ddfc6199124f 100644
--- a/modules/web-console/e2e/testcafe/components/FormField.js
+++ b/modules/web-console/e2e/testcafe/components/FormField.js
@@ -19,10 +19,10 @@ import {Selector, t} from 'testcafe';
 import {AngularJSSelector} from 'testcafe-angular-selectors';
 
 export class FormField {
-    static ROOT_SELECTOR = '.ignite-form-field';
-    static LABEL_SELECTOR = '.ignite-form-field__label';
+    static ROOT_SELECTOR = '.form-field';
+    static LABEL_SELECTOR = '.form-field__label';
     static CONTROL_SELECTOR = '[ng-model]';
-    static ERRORS_SELECTOR = '.ignite-form-field__errors';
+    static ERRORS_SELECTOR = '.form-field__errors';
 
     /** @type {ReturnType<Selector>} */
     _selector;
diff --git a/modules/web-console/e2e/testcafe/components/modalInput.js b/modules/web-console/e2e/testcafe/components/modalInput.js
index d9ab39930dd13..845a3d73f556d 100644
--- a/modules/web-console/e2e/testcafe/components/modalInput.js
+++ b/modules/web-console/e2e/testcafe/components/modalInput.js
@@ -14,16 +14,16 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
-import {Selector, t} from 'testcafe';
+import {FormField} from './FormField';
+import {t} from 'testcafe';
 
 export class ModalInput {
     constructor() {
-        this.valueInput = Selector('#input-fieldInput');
+        this.valueInput = new FormField({ id: 'inputDialogFieldInput' });
     }
 
     async enterValue(value) {
-        await t.typeText(this.valueInput, value);
+        await t.typeText(this.valueInput.control, value);
     }
 
     async confirm() {
diff --git a/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js b/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js
index a862172a8e282..693add4acb104 100644
--- a/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js
+++ b/modules/web-console/e2e/testcafe/fixtures/configuration/clusterFormChangeDetection.js
@@ -36,21 +36,23 @@ fixture('Cluster configuration form change detection')
     })
     .after(dropTestDB);
 
-test('New cluster change detection', async(t) => {
+test.skip('New cluster change detection', async(t) => {
     const overview = new PageConfigurationOverview();
     const advanced = new PageConfigurationAdvancedCluster();
 
     await t
         .navigateTo(resolveUrl(`/configuration/overview`))
         .click(overview.createClusterConfigButton)
-        .click(advancedNavButton)
-        .click(advanced.sections.connectorConfiguration.panel.heading);
+        .click(advancedNavButton);
 
-    await scrollIntoView.with({dependencies: {el: advanced.sections.connectorConfiguration.panel.heading}})();
+    await t.click(advanced.sections.connectorConfiguration.panel.heading);
+
+    // IODO: Investigate why this code doesn't work in headless mode;
+    await scrollIntoView.with({dependencies: {el: advanced.sections.connectorConfiguration.inputs.enable.control}})();
 
     await t
         .click(advanced.sections.connectorConfiguration.inputs.enable.control)
         .click(advanced.saveButton)
         .click(pageAdvancedConfiguration.cachesNavButton)
-        .expect(confirmation.body.exists).notOk(`Doesn't show changes confiramtion after saving new cluster`);
+        .expect(confirmation.body.exists).notOk(`Doesn't show changes confirmation after saving new cluster`);
 });
diff --git a/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js b/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js
index 1f1f5590f34b4..e326100039a10 100644
--- a/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js
+++ b/modules/web-console/e2e/testcafe/page-models/PageConfigurationAdvancedCluster.js
@@ -16,10 +16,21 @@
  */
 
 import {Selector, t} from 'testcafe';
+import {PanelCollapsible} from '../components/PanelCollapsible';
+import {FormField} from '../components/FormField';
 
 export class PageConfigurationAdvancedCluster {
     constructor() {
         this.saveButton = Selector('.pc-form-actions-panel .btn-ignite').withText('Save');
+
+        this.sections = {
+            connectorConfiguration: {
+                panel: new PanelCollapsible('Connector configuration'),
+                inputs: {
+                    enable: new FormField({id: 'restEnabledInput'})
+                }
+            }
+        };
     }
 
     async save() {
diff --git a/modules/web-console/e2e/testcafe/page-models/PageConfigurationBasic.js b/modules/web-console/e2e/testcafe/page-models/PageConfigurationBasic.js
index bef6606d71512..e979da028a55a 100644
--- a/modules/web-console/e2e/testcafe/page-models/PageConfigurationBasic.js
+++ b/modules/web-console/e2e/testcafe/page-models/PageConfigurationBasic.js
@@ -40,7 +40,7 @@ export class PageConfigurationBasic {
     constructor() {
         this._selector = Selector('page-configure-basic');
         this.versionPicker = new VersionPicker();
-        this.totalOffheapSizeInput = Selector('pc-form-field-size#memory');
+        this.totalOffheapSizeInput = Selector('form-field-size#memory');
         this.mainFormAction = Selector('.pc-form-actions-panel .btn-ignite-group .btn-ignite:nth-of-type(1)');
         this.contextFormActionsButton = Selector('.pc-form-actions-panel .btn-ignite-group .btn-ignite:nth-of-type(2)');
         this.contextSaveButton = Selector('a[role=menuitem]').withText(new RegExp(`^${PageConfigurationBasic.SAVE_CHANGES_LABEL}$`));
diff --git a/modules/web-console/frontend/.babelrc b/modules/web-console/frontend/.babelrc
index 1759c449c5380..b68592cb8ab5e 100644
--- a/modules/web-console/frontend/.babelrc
+++ b/modules/web-console/frontend/.babelrc
@@ -1,4 +1,16 @@
 {
-  "presets": ["es2015", "stage-1"],
-  "plugins": ["add-module-exports", "transform-object-rest-spread"]
-}
+    "presets": [
+        ["@babel/env", {
+            "targets": {
+                "browsers": [">1%", "not ie 11", "not op_mini all"]
+            }
+        }],
+        "@babel/preset-typescript"
+    ],
+    "plugins": [
+        ["@babel/plugin-proposal-class-properties", { "loose" : true }],
+        "@babel/plugin-proposal-object-rest-spread",
+        "@babel/plugin-syntax-dynamic-import",
+        "@babel/plugin-transform-parameters"
+    ]
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/.eslintrc b/modules/web-console/frontend/.eslintrc
index 805b339b23e31..ac1a37f55df81 100644
--- a/modules/web-console/frontend/.eslintrc
+++ b/modules/web-console/frontend/.eslintrc
@@ -1,7 +1,7 @@
-parser: "babel-eslint"
+parser: "typescript-eslint-parser"
 
 plugins:
-    - babel
+    - typescript
 
 env:
     es6: true
@@ -150,17 +150,19 @@ rules:
     no-sparse-arrays: 1
     no-sync: 0
     no-ternary: 0
-    no-trailing-spaces: 2
+    no-trailing-spaces: ["error", {"ignoreComments": true}]
     no-throw-literal: 0
     no-this-before-super: 2
     no-unexpected-multiline: 2
-    no-undef: 2
+    // The rule produces undesired results with TS
+    // no-undef: 2
     no-undef-init: 2
     no-undefined: 2
     no-unneeded-ternary: 2
     no-unreachable: 2
     no-unused-expressions: [2, { allowShortCircuit: true }]
     no-unused-vars: [0, {"vars": "all", "args": "after-used"}]
+    typescript/no-unused-vars: [0]
     no-use-before-define: 2
     no-useless-call: 2
     no-void: 0
@@ -195,4 +197,3 @@ rules:
     wrap-iife: 0
     wrap-regex: 0
     yoda: [2, "never"]
-    babel/semi: 2
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index b9d72ab4878e2..98792c4c50bb8 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -15,6 +15,10 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
+import './style.scss';
+
 import './vendor';
 import '../public/stylesheets/style.scss';
 import '../app/primitives';
@@ -49,19 +53,19 @@ import servicesModule from './services';
 import i18n from './data/i18n';
 
 // Directives.
-import igniteAutoFocus from './directives/auto-focus.directive.js';
+import igniteAutoFocus from './directives/auto-focus.directive';
 import igniteBsAffixUpdate from './directives/bs-affix-update.directive';
-import igniteCentered from './directives/centered/centered.directive.js';
-import igniteCopyToClipboard from './directives/copy-to-clipboard.directive.js';
+import igniteCentered from './directives/centered/centered.directive';
+import igniteCopyToClipboard from './directives/copy-to-clipboard.directive';
 import igniteHideOnStateChange from './directives/hide-on-state-change/hide-on-state-change.directive';
 import igniteInformation from './directives/information/information.directive';
-import igniteMatch from './directives/match.directive.js';
-import igniteOnClickFocus from './directives/on-click-focus.directive.js';
-import igniteOnEnter from './directives/on-enter.directive.js';
-import igniteOnEnterFocusMove from './directives/on-enter-focus-move.directive.js';
-import igniteOnEscape from './directives/on-escape.directive.js';
-import igniteOnFocusOut from './directives/on-focus-out.directive.js';
-import igniteRestoreInputFocus from './directives/restore-input-focus.directive.js';
+import igniteMatch from './directives/match.directive';
+import igniteOnClickFocus from './directives/on-click-focus.directive';
+import igniteOnEnter from './directives/on-enter.directive';
+import igniteOnEnterFocusMove from './directives/on-enter-focus-move.directive';
+import igniteOnEscape from './directives/on-escape.directive';
+import igniteOnFocusOut from './directives/on-focus-out.directive';
+import igniteRestoreInputFocus from './directives/restore-input-focus.directive';
 import igniteUiAceCSharp from './directives/ui-ace-sharp/ui-ace-sharp.directive';
 import igniteUiAcePojos from './directives/ui-ace-pojos/ui-ace-pojos.directive';
 import igniteUiAcePom from './directives/ui-ace-pom/ui-ace-pom.directive';
@@ -69,12 +73,11 @@ import igniteUiAceDocker from './directives/ui-ace-docker/ui-ace-docker.directiv
 import igniteUiAceTabs from './directives/ui-ace-tabs.directive';
 import igniteRetainSelection from './directives/retain-selection.directive';
 import btnIgniteLink from './directives/btn-ignite-link';
-import exposeInput from './components/expose-ignite-form-field-control';
 
 // Services.
 import ChartColors from './services/ChartColors.service';
-import {default as IgniteConfirm, Confirm} from './services/Confirm.service.js';
-import ConfirmBatch from './services/ConfirmBatch.service.js';
+import {default as IgniteConfirm, Confirm} from './services/Confirm.service';
+import ConfirmBatch from './services/ConfirmBatch.service';
 import CopyToClipboard from './services/CopyToClipboard.service';
 import Countries from './services/Countries.service';
 import ErrorPopover from './services/ErrorPopover.service';
@@ -87,11 +90,10 @@ import LegacyTable from './services/LegacyTable.service';
 import LegacyUtils from './services/LegacyUtils.service';
 import Messages from './services/Messages.service';
 import ErrorParser from './services/ErrorParser.service';
-import ModelNormalizer from './services/ModelNormalizer.service.js';
-import UnsavedChangesGuard from './services/UnsavedChangesGuard.service';
+import ModelNormalizer from './services/ModelNormalizer.service';
 import Caches from './services/Caches';
 import {CSV} from './services/CSV';
-import {$exceptionHandler} from './services/exceptionHandler.js';
+import {$exceptionHandler} from './services/exceptionHandler';
 import IGFSs from './services/IGFSs';
 import Models from './services/Models';
 
@@ -100,6 +102,7 @@ import AngularStrapSelect from './services/AngularStrapSelect.decorator';
 
 // Filters.
 import byName from './filters/byName.filter';
+import bytes from './filters/bytes.filter';
 import defaultName from './filters/default-name.filter';
 import domainsValidation from './filters/domainsValidation.filter';
 import duration from './filters/duration.filter';
@@ -126,8 +129,10 @@ import gridColumnSelector from './components/grid-column-selector';
 import gridItemSelected from './components/grid-item-selected';
 import gridNoData from './components/grid-no-data';
 import gridExport from './components/grid-export';
+import gridShowingRows from './components/grid-showing-rows';
 import bsSelectMenu from './components/bs-select-menu';
 import protectFromBsSelectRender from './components/protect-from-bs-select-render';
+import uiGrid from './components/ui-grid';
 import uiGridHovering from './components/ui-grid-hovering';
 import uiGridFilters from './components/ui-grid-filters';
 import uiGridColumnResizer from './components/ui-grid-column-resizer';
@@ -141,6 +146,9 @@ import pageLanding from './components/page-landing';
 import passwordVisibility from './components/password-visibility';
 import progressLine from './components/progress-line';
 import formField from './components/form-field';
+import igniteChart from './components/ignite-chart';
+import igniteChartSelector from './components/ignite-chart-series-selector';
+import igniteStatus from './components/ignite-status';
 
 import pageProfile from './components/page-profile';
 import pagePasswordChanged from './components/page-password-changed';
@@ -221,7 +229,9 @@ export default angular.module('ignite-console', [
     gridItemSelected.name,
     gridNoData.name,
     gridExport.name,
+    gridShowingRows.name,
     bsSelectMenu.name,
+    uiGrid.name,
     uiGridHovering.name,
     uiGridFilters.name,
     uiGridColumnResizer.name,
@@ -236,7 +246,6 @@ export default angular.module('ignite-console', [
     connectedClustersDialog.name,
     igniteListOfRegisteredUsers.name,
     pageProfile.name,
-    exposeInput.name,
     pageLanding.name,
     pagePasswordChanged.name,
     pagePasswordReset.name,
@@ -247,28 +256,31 @@ export default angular.module('ignite-console', [
     uiAceSpring.name,
     breadcrumbs.name,
     passwordVisibility.name,
+    igniteChart.name,
+    igniteChartSelector.name,
+    igniteStatus.name,
     progressLine.name,
     formField.name
 ])
-.service($exceptionHandler.name, $exceptionHandler)
+.service('$exceptionHandler', $exceptionHandler)
 // Directives.
-.directive(...igniteAutoFocus)
-.directive(...igniteBsAffixUpdate)
-.directive(...igniteCentered)
-.directive(...igniteCopyToClipboard)
-.directive(...igniteHideOnStateChange)
-.directive(...igniteInformation)
+.directive('igniteAutoFocus', igniteAutoFocus)
+.directive('igniteBsAffixUpdate', igniteBsAffixUpdate)
+.directive('centered', igniteCentered)
+.directive('igniteCopyToClipboard', igniteCopyToClipboard)
+.directive('hideOnStateChange', igniteHideOnStateChange)
+.directive('igniteInformation', igniteInformation)
 .directive('igniteMatch', igniteMatch)
-.directive(...igniteOnClickFocus)
-.directive(...igniteOnEnter)
-.directive(...igniteOnEnterFocusMove)
-.directive(...igniteOnEscape)
-.directive(...igniteUiAceCSharp)
-.directive(...igniteUiAcePojos)
-.directive(...igniteUiAcePom)
-.directive(...igniteUiAceDocker)
-.directive(...igniteUiAceTabs)
-.directive(...igniteRetainSelection)
+.directive('igniteOnClickFocus', igniteOnClickFocus)
+.directive('igniteOnEnter', igniteOnEnter)
+.directive('igniteOnEnterFocusMove', igniteOnEnterFocusMove)
+.directive('igniteOnEscape', igniteOnEscape)
+.directive('igniteUiAceSharp', igniteUiAceCSharp)
+.directive('igniteUiAcePojos', igniteUiAcePojos)
+.directive('igniteUiAcePom', igniteUiAcePom)
+.directive('igniteUiAceDocker', igniteUiAceDocker)
+.directive('igniteUiAceTabs', igniteUiAceTabs)
+.directive('igniteRetainSelection', igniteRetainSelection)
 .directive('igniteOnFocusOut', igniteOnFocusOut)
 .directive('igniteRestoreInputFocus', igniteRestoreInputFocus)
 .directive('btnIgniteLinkDashedSuccess', btnIgniteLink)
@@ -277,83 +289,113 @@ export default angular.module('ignite-console', [
 .service('IgniteErrorPopover', ErrorPopover)
 .service('JavaTypes', JavaTypes)
 .service('SqlTypes', SqlTypes)
-.service(...ChartColors)
-.service(...IgniteConfirm)
-.service(Confirm.name, Confirm)
+.service('IgniteChartColors', ChartColors)
+.service('IgniteConfirm', IgniteConfirm)
+.service('Confirm', Confirm)
 .service('IgniteConfirmBatch', ConfirmBatch)
-.service(...CopyToClipboard)
-.service(...Countries)
-.service(...Focus)
-.service(...InetAddress)
-.service(...Messages)
+.service('IgniteCopyToClipboard', CopyToClipboard)
+.service('IgniteCountries', Countries)
+.service('IgniteFocus', Focus)
+.service('IgniteInetAddress', InetAddress)
+.service('IgniteMessages', Messages)
 .service('IgniteErrorParser', ErrorParser)
-.service(...ModelNormalizer)
-.service(...LegacyTable)
-.service(...FormUtils)
-.service(...LegacyUtils)
-.service(...UnsavedChangesGuard)
+.service('IgniteModelNormalizer', ModelNormalizer)
+.service('IgniteLegacyTable', LegacyTable)
+.service('IgniteFormUtils', FormUtils)
+.service('IgniteLegacyUtils', LegacyUtils)
 .service('IgniteActivitiesUserDialog', IgniteActivitiesUserDialog)
 .service('Caches', Caches)
-.service(CSV.name, CSV)
+.service('CSV', CSV)
 .service('IGFSs', IGFSs)
 .service('Models', Models)
 // Filters.
 .filter('byName', byName)
+.filter('bytes', bytes)
 .filter('defaultName', defaultName)
 .filter('domainsValidation', domainsValidation)
 .filter('duration', duration)
 .filter('hasPojo', hasPojo)
 .filter('uiGridSubcategories', uiGridSubcategories)
 .filter('id8', id8)
-.config(['$translateProvider', '$stateProvider', '$locationProvider', '$urlRouterProvider', ($translateProvider, $stateProvider, $locationProvider, $urlRouterProvider) => {
-    $translateProvider.translations('en', i18n);
-    $translateProvider.preferredLanguage('en');
+.config(['$translateProvider', '$stateProvider', '$locationProvider', '$urlRouterProvider',
+    /**
+     * @param {angular.translate.ITranslateProvider} $translateProvider
+     * @param {import('@uirouter/angularjs').StateProvider} $stateProvider
+     * @param {ng.ILocationProvider} $locationProvider
+     * @param {import('@uirouter/angularjs').UrlRouterProvider} $urlRouterProvider
+     */
+    ($translateProvider, $stateProvider, $locationProvider, $urlRouterProvider) => {
+        $translateProvider.translations('en', i18n);
+        $translateProvider.preferredLanguage('en');
 
-    // Set up the states.
-    $stateProvider
+        // Set up the states.
+        $stateProvider
         .state('base', {
             url: '',
             abstract: true,
             template: baseTemplate
         });
 
-    $urlRouterProvider.otherwise('/404');
-    $locationProvider.html5Mode(true);
-}])
-.run(['$rootScope', '$state', 'gettingStarted', ($root, $state, gettingStarted) => {
-    $root._ = _;
-    $root.$state = $state;
-    $root.gettingStarted = gettingStarted;
-}])
-.run(['$rootScope', 'AgentManager', ($root, agentMgr) => {
-    let lastUser;
+        $urlRouterProvider.otherwise('/404');
+        $locationProvider.html5Mode(true);
+    }])
+.run(['$rootScope', '$state', 'gettingStarted',
+    /**
+     * @param {ng.IRootScopeService} $root
+     * @param {import('@uirouter/angularjs').StateService} $state
+     * @param {ReturnType<typeof import('./modules/getting-started/GettingStarted.provider').service>} gettingStarted
+     */
+    ($root, $state, gettingStarted) => {
+        $root._ = _;
+        $root.$state = $state;
+        $root.gettingStarted = gettingStarted;
+    }
+])
+.run(['$rootScope', 'AgentManager',
+    /**
+     * @param {ng.IRootScopeService} $root
+     * @param {import('./modules/agent/AgentManager.service').default} agentMgr
+     */
+    ($root, agentMgr) => {
+        let lastUser;
 
-    $root.$on('user', (e, user) => {
-        if (lastUser)
-            return;
+        $root.$on('user', (e, user) => {
+            if (lastUser)
+                return;
 
-        lastUser = user;
+            lastUser = user;
 
-        agentMgr.connect();
-    });
-}])
-.run(['$transitions', ($transitions) => {
-    $transitions.onSuccess({ }, (trans) => {
-        try {
-            const {name, unsaved} = trans.$to();
-            const params = trans.params();
+            agentMgr.connect();
+        });
+    }
+])
+.run(['$transitions',
+    /**
+     * @param {import('@uirouter/angularjs').TransitionService} $transitions
+     */
+    ($transitions) => {
+        $transitions.onSuccess({ }, (trans) => {
+            try {
+                const {name, unsaved} = trans.$to();
+                const params = trans.params();
 
-            if (unsaved)
-                localStorage.removeItem('lastStateChangeSuccess');
-            else
-                localStorage.setItem('lastStateChangeSuccess', JSON.stringify({name, params}));
-        }
-        catch (ignored) {
+                if (unsaved)
+                    localStorage.removeItem('lastStateChangeSuccess');
+                else
+                    localStorage.setItem('lastStateChangeSuccess', JSON.stringify({name, params}));
+            }
+            catch (ignored) {
             // No-op.
-        }
-    });
-}])
+            }
+        });
+    }
+])
 .run(['$rootScope', '$http', '$state', 'IgniteMessages', 'User', 'IgniteNotebookData',
+    /**
+     * @param {ng.IRootScopeService} $root
+     * @param {ng.IHttpService} $http
+     * @param {ReturnType<typeof import('./services/Messages.service').default>} Messages
+     */
     ($root, $http, $state, Messages, User, Notebook) => { // eslint-disable-line no-shadow
         $root.revertIdentity = () => {
             $http.get('/api/v1/admin/revert/identity')
@@ -364,4 +406,9 @@ export default angular.module('ignite-console', [
         };
     }
 ])
-.run(['IgniteIcon', (IgniteIcon) => IgniteIcon.registerIcons(icons)]);
+.run(['IgniteIcon',
+    /**
+     * @param {import('./components/ignite-icon/service').default} IgniteIcon
+     */
+    (IgniteIcon) => IgniteIcon.registerIcons(icons)
+]);
diff --git a/modules/web-console/frontend/app/browserUpdate/index.js b/modules/web-console/frontend/app/browserUpdate/index.js
index 85df64e93ed78..3930ac787ac5f 100644
--- a/modules/web-console/frontend/app/browserUpdate/index.js
+++ b/modules/web-console/frontend/app/browserUpdate/index.js
@@ -20,7 +20,7 @@ import './style.scss';
 
 browserUpdate({
     notify: {
-        i: 10,
+        i: 11,
         f: '-18m',
         s: 9,
         c: '-18m',
@@ -30,5 +30,7 @@ browserUpdate({
     l: 'en',
     mobile: false,
     api: 5,
+    // This should work in older browsers
+    text: '<b>Outdated or unsupported browser detected.</b> Web Console may work incorrectly. Please update to one of modern fully supported browsers! <a {up_but}>Update</a> <a {ignore_but}>Ignore</a>',
     reminder: 0
 });
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
index 078f725fd8e2c..fb9d15a46b666 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.controller.js
@@ -22,6 +22,12 @@ export default class ActivitiesCtrl {
         const $ctrl = this;
 
         $ctrl.user = user;
-        $ctrl.data = _.map(user.activitiesDetail, (amount, action) => ({ action, amount }));
+        $ctrl.data = _.map(user.activitiesDetail, (amount, action) => ({action, amount}));
+
+        $ctrl.columnDefs = [
+            { displayName: 'Description', field: 'action', enableFiltering: false, cellFilter: 'translate'},
+            { displayName: 'Action', field: 'action', enableFiltering: false},
+            { displayName: 'Visited', field: 'amount', enableFiltering: false}
+        ];
     }
 }
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug
index 33d5f62e814d0..7e9d3f5142364 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/activities-user-dialog.tpl.pug
@@ -14,29 +14,21 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-.modal.modal--ignite(tabindex='-1' role='dialog')
+.modal.modal--ignite.theme--ignite(tabindex='-1' role='dialog')
     .modal-dialog
         .modal-content
             .modal-header
-                h4.modal-title
-                    i.fa.fa-info-circle
-                    | Activity details: {{ ctrl.user.userName }}
+                h4.modal-title Activity details: {{ ctrl.user.userName }}
                 button.close(type='button' aria-label='Close' ng-click='$hide()')
                      svg(ignite-icon='cross')
-            .modal-body.modal-body-with-scroll(id='activities-user-dialog')
+            .modal-body.modal-body-with-scroll
                 .panel--ignite
-                    table.table--ignite(scrollable-container='#activities-user-dialog' st-table='displayedRows' st-safe-src='ctrl.data')
-                        thead
-                            th(st-sort='action | translate') Description
-                            th(st-sort='action') Action
-                            th(st-sort='amount') Visited
-                        tbody
-                            tr(ng-repeat='row in displayedRows')
-                                td
-                                    .text-overflow {{ row.action | translate }}
-                                td
-                                    .text-overflow {{ row.action }}
-                                td.text-right
-                                    .text-overflow {{ row.amount }}
+                    ignite-grid-table(
+                        items='ctrl.data'
+                        column-defs='ctrl.columnDefs'
+                        grid-thin='true'
+                    )
+
             .modal-footer
-                button.btn-ignite.btn-ignite--success(id='confirm-btn-confirm' ng-click='$hide()') Close
+                div
+                    button.btn-ignite.btn-ignite--success(ng-click='$hide()') Close
diff --git a/modules/web-console/frontend/app/components/activities-user-dialog/index.js b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
index 13c1d95c74c61..ace3821dd9620 100644
--- a/modules/web-console/frontend/app/components/activities-user-dialog/index.js
+++ b/modules/web-console/frontend/app/components/activities-user-dialog/index.js
@@ -18,17 +18,24 @@
 import controller from './activities-user-dialog.controller';
 import templateUrl from './activities-user-dialog.tpl.pug';
 
-export default ['$modal', ($modal) => ({ show = true, user }) => {
-    const ActivitiesUserDialog = $modal({
-        templateUrl,
-        show,
-        resolve: {
-            user: () => user
-        },
-        controller,
-        controllerAs: 'ctrl'
-    });
+/**
+ * @param {mgcrea.ngStrap.modal.IModalService} $modal
+ */
+export default function service($modal) {
+    return function({ show = true, user }) {
+        const ActivitiesUserDialog = $modal({
+            templateUrl,
+            show,
+            resolve: {
+                user: () => user
+            },
+            controller,
+            controllerAs: 'ctrl'
+        });
+
+        return ActivitiesUserDialog.$promise
+             .then(() => ActivitiesUserDialog);
+    };
+}
 
-    return ActivitiesUserDialog.$promise
-         .then(() => ActivitiesUserDialog);
-}];
+service.$inject = ['$modal'];
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/index.js b/modules/web-console/frontend/app/components/bs-select-menu/index.js
index a9bafb250b465..0f2a258cae882 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/index.js
+++ b/modules/web-console/frontend/app/components/bs-select-menu/index.js
@@ -18,9 +18,11 @@
 import angular from 'angular';
 
 import directive from './directive';
-import {directive as transcludeToBody} from './transcludeToBody.directive';
+import {default as transcludeToBody} from './transcludeToBody.directive';
+import stripFilter from './strip.filter';
 
 export default angular
     .module('ignite-console.bs-select-menu', [])
     .directive('bssmTranscludeToBody', transcludeToBody)
-    .directive('bsSelectMenu', directive);
+    .directive('bsSelectMenu', directive)
+    .filter('bsSelectStrip', stripFilter);
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/index.spec.js b/modules/web-console/frontend/app/components/bs-select-menu/index.spec.js
index 9b9fa3968c9cd..06a292fd1e48c 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/index.spec.js
+++ b/modules/web-console/frontend/app/components/bs-select-menu/index.spec.js
@@ -18,7 +18,7 @@
 import 'mocha';
 import {assert} from 'chai';
 import angular from 'angular';
-import componentModule from './index.js';
+import componentModule from './index';
 
 suite('bs-select-menu', () => {
     /** @type {ng.IScope} */
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/strip.filter.js b/modules/web-console/frontend/app/components/bs-select-menu/strip.filter.js
new file mode 100644
index 0000000000000..f2a18e4d286d2
--- /dev/null
+++ b/modules/web-console/frontend/app/components/bs-select-menu/strip.filter.js
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default function() {
+    return function(val) {
+        return val ? val.replace(/(<\/?\w+>)/igm, '') : '';
+    };
+}
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/style.scss b/modules/web-console/frontend/app/components/bs-select-menu/style.scss
index bfa0063340732..ac3991b4957dc 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/style.scss
+++ b/modules/web-console/frontend/app/components/bs-select-menu/style.scss
@@ -48,9 +48,8 @@
     }
 
     .bssm-item-text {
-        overflow: hidden;
+        overflow: visible;
         white-space: nowrap;
-        text-overflow: ellipsis;
     }
 
     &>li {
@@ -63,6 +62,7 @@
             padding-bottom: 9px;
             background-color: transparent;
             border-radius: 0;
+            padding-right: 30px;
 
             &:hover {
                 background-color: #eeeeee;
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/template.pug b/modules/web-console/frontend/app/components/bs-select-menu/template.pug
index 4ffa144ae5ce2..7cd2195291732 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/template.pug
+++ b/modules/web-console/frontend/app/components/bs-select-menu/template.pug
@@ -37,7 +37,7 @@ ul.bs-select-menu(
             ng-click='$select($index, $event); $event.stopPropagation();'
             ng-class=`{ 'bssm-item-button__active': $isActive($index) }`
             data-placement='right auto'
-            title='{{ ::match.label }}'
+            title='{{ ::match.label | bsSelectStrip }}'
         )
             img.bssm-active-indicator.icon-left(
                 ng-src='{{ $isActive($index) ? "/images/checkbox-active.svg" : "/images/checkbox.svg" }}'
diff --git a/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js b/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js
index b415719e87aab..de4fbfa0ff287 100644
--- a/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js
+++ b/modules/web-console/frontend/app/components/bs-select-menu/transcludeToBody.directive.js
@@ -40,7 +40,7 @@ class Controller {
     }
 }
 
-export function directive() {
+export default function directive() {
     return {
         restrict: 'E',
         transclude: true,
diff --git a/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js b/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js
index 64b23cf01a895..807c3ba8b7d65 100644
--- a/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js
+++ b/modules/web-console/frontend/app/components/connected-clusters-badge/controller.js
@@ -15,17 +15,14 @@
  * limitations under the License.
  */
 
-import AgentManager from 'app/modules/agent/AgentManager.service';
-
 export default class {
-    static $inject = [AgentManager.name, 'ConnectedClustersDialog'];
+    static $inject = ['AgentManager', 'ConnectedClustersDialog'];
 
-    /** @type {Number} */
     connectedClusters = 0;
 
     /**
-     * @param {AgentManager} agentMgr
-     * @param connectedClustersDialog
+     * @param {import('app/modules/agent/AgentManager.service').default} agentMgr
+     * @param {import('../connected-clusters-dialog/service').default} connectedClustersDialog
      */
     constructor(agentMgr, connectedClustersDialog) {
         this.agentMgr = agentMgr;
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/index.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/index.js
index 26f2b12d4f8bb..e5bdd2c6a883b 100644
--- a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/index.js
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/cell-logout/index.js
@@ -17,13 +17,14 @@
 
 import template from './template.pug';
 
-import AgentManager from 'app/modules/agent/AgentManager.service';
-
 class controller {
-    static $inject = [AgentManager.name];
+    /** @type {string} */
+    clusterId;
+
+    static $inject = ['AgentManager'];
 
     /**
-     * @param {AgentManager} agentMgr
+     * @param {import('app/modules/agent/AgentManager.service').default} agentMgr
      */
     constructor(agentMgr) {
         this.agentMgr = agentMgr;
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js
index c4b0f93a5f752..2d0ca266b9a2e 100644
--- a/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/components/list/controller.js
@@ -24,6 +24,7 @@ export default class SnapshotsListCachesCtrl {
 
     /**
      * @param {ng.IScope} $scope
+     * @param {import('app/modules/agent/AgentManager.service').default} agentMgr
      */
     constructor($scope, agentMgr) {
         this.$scope = $scope;
diff --git a/modules/web-console/frontend/app/components/connected-clusters-dialog/template.tpl.pug b/modules/web-console/frontend/app/components/connected-clusters-dialog/template.tpl.pug
index dd7b4fcb87939..5ea83fd3b43ee 100644
--- a/modules/web-console/frontend/app/components/connected-clusters-dialog/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/connected-clusters-dialog/template.tpl.pug
@@ -16,7 +16,7 @@
 
 include /app/helpers/jade/mixins
 
-.modal.modal--ignite.theme--ignite.connected-clusters-dialog(tabindex='-1' role='dialog')
+.modal.modal--ignite.theme-ignite.connected-clusters-dialog(tabindex='-1' role='dialog')
     .modal-dialog.modal-dialog--adjust-height
         form.modal-content(name='$ctrl.form' novalidate)
             .modal-header
@@ -26,9 +26,9 @@ include /app/helpers/jade/mixins
                      svg(ignite-icon="cross")
             .modal-body.modal-body-with-scroll
                 .panel--ignite
-                    ul.tabs.tabs--blue
                     connected-clusters-list(data-options='$ctrl.clusters')
 
             .modal-footer
-                button.btn-ignite.btn-ignite--success(type='button' ng-click='$hide()') Ok
-                
+                div
+                    button.btn-ignite.btn-ignite--success(type='button' ng-click='$hide()') Ok
+
diff --git a/modules/web-console/frontend/app/components/expose-ignite-form-field-control/directives.js b/modules/web-console/frontend/app/components/expose-ignite-form-field-control/directives.js
deleted file mode 100644
index 5184032be70e7..0000000000000
--- a/modules/web-console/frontend/app/components/expose-ignite-form-field-control/directives.js
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// eslint-disable-next-line
-import {IgniteFormField} from 'app/components/page-configure/components/pcValidation'
-
-/**
- * Exposes input to .ignite-form-field scope
- */
-class ExposeIgniteFormFieldControl {
-    /** @type {IgniteFormField} */
-    formField;
-    /** @type {ng.INgModelController} */
-    ngModel;
-    /** 
-     * Name used to access control from $scope.
-     * @type {string}
-     */
-    name;
-
-    $onInit() {
-        if (this.formField && this.ngModel) this.formField.exposeControl(this.ngModel, this.name);
-    }
-}
-
-export function exposeIgniteFormFieldControl() {
-    return {
-        restrict: 'A',
-        controller: ExposeIgniteFormFieldControl,
-        bindToController: {
-            name: '@exposeIgniteFormFieldControl'
-        },
-        require: {
-            formField: '^^?igniteFormField',
-            ngModel: '?ngModel'
-        },
-        scope: false
-    };
-}
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js b/modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js
similarity index 95%
rename from modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js
rename to modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js
index 0d751e849d299..7663e9d0037fc 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/controller.js
+++ b/modules/web-console/frontend/app/components/form-field/components/form-field-size/controller.js
@@ -42,6 +42,11 @@ export default class PCFormFieldSizeController {
             {label: 'ns', value: 1 / 1000},
             {label: 'ms', value: 1},
             {label: 's', value: 1000}
+        ],
+        time: [
+            {label: 'sec', value: 1},
+            {label: 'min', value: 60},
+            {label: 'hour', value: 60 * 60}
         ]
     };
 
@@ -62,7 +67,7 @@ export default class PCFormFieldSizeController {
     $onInit() {
         if (!this.min) this.min = 0;
         if (!this.sizesMenu) this.setDefaultSizeType();
-        this.$element.addClass('ignite-form-field');
+        this.$element.addClass('form-field');
         this.ngModel.$render = () => this.assignValue(this.ngModel.$viewValue);
     }
 
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/component.js b/modules/web-console/frontend/app/components/form-field/components/form-field-size/index.js
similarity index 100%
rename from modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/component.js
rename to modules/web-console/frontend/app/components/form-field/components/form-field-size/index.js
index e90a2cfb4aaf0..5e08df2921194 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/component.js
+++ b/modules/web-console/frontend/app/components/form-field/components/form-field-size/index.js
@@ -15,9 +15,9 @@
  * limitations under the License.
  */
 
+import './style.scss';
 import template from './template.pug';
 import controller from './controller';
-import './style.scss';
 
 export default {
     controller,
diff --git a/modules/web-console/frontend/app/components/form-field/components/form-field-size/style.scss b/modules/web-console/frontend/app/components/form-field/components/form-field-size/style.scss
new file mode 100644
index 0000000000000..c522200d66e9b
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-field/components/form-field-size/style.scss
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+form-field-size {
+	display: block;
+}
diff --git a/modules/web-console/frontend/app/components/form-field/components/form-field-size/template.pug b/modules/web-console/frontend/app/components/form-field/components/form-field-size/template.pug
new file mode 100644
index 0000000000000..b712a6717a266
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-field/components/form-field-size/template.pug
@@ -0,0 +1,77 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins
+
++form-field__label({
+    label: '{{ ::$ctrl.label }}',
+    name: '$ctrl.id',
+    required: '$ctrl.required',
+    disabled: '$ctrl.ngDisabled'
+})
+    +form-field__tooltip({title: '{{$ctrl.tip}}'})(
+        ng-if='$ctrl.tip'
+    )
+
+.form-field__control.form-field__control-group(ng-form='$ctrl.innerForm')
+    input(
+        type='number'
+        id='{{::$ctrl.id}}Input'
+        ng-model='$ctrl.value'
+        ng-model-options='{allowInvalid: true}'
+        ng-change='$ctrl.onValueChange()'
+        name='numberInput'
+        placeholder='{{$ctrl.placeholder}}'
+        min='{{ $ctrl.min ? $ctrl.min / $ctrl.sizeScale.value : "" }}'
+        max='{{ $ctrl.max ? $ctrl.max / $ctrl.sizeScale.value : "" }}'
+        ng-required='$ctrl.required'
+        ng-disabled='$ctrl.ngDisabled'
+    )
+    button.select-toggle(
+        bs-select
+        bs-options='size as size.label for size in $ctrl.sizesMenu'
+        ng-model='$ctrl.sizeScale'
+        protect-from-bs-select-render
+        ng-disabled='$ctrl.ngDisabled'
+        type='button'
+    )
+        | {{ $ctrl.sizeScale.label }}
+
+.form-field__errors(
+    ng-messages='$ctrl.ngModel.$error'
+    ng-show=`($ctrl.ngModel.$dirty || $ctrl.ngModel.$touched || $ctrl.ngModel.$submitted) && $ctrl.ngModel.$invalid`
+)
+    div(ng-transclude)
+    +form-field__error({
+        error: 'required',
+        message: 'This field could not be empty'
+    })
+    +form-field__error({
+        error: 'min',
+        message: 'Value is less than allowable minimum: {{ $ctrl.min/$ctrl.sizeScale.value }} {{$ctrl.sizeScale.label}}'
+    })
+    +form-field__error({
+        error: 'max',
+        message: 'Value is more than allowable maximum: {{ $ctrl.max/$ctrl.sizeScale.value }} {{$ctrl.sizeScale.label}}'
+    })
+    +form-field__error({
+        error: 'number',
+        message: 'Only numbers allowed'
+    })
+    +form-field__error({
+        error: 'step',
+        message: 'Invalid step'
+    })
diff --git a/modules/web-console/frontend/app/components/form-field/index.js b/modules/web-console/frontend/app/components/form-field/index.js
index 077ace0a01a1c..b1ee7535cc61f 100644
--- a/modules/web-console/frontend/app/components/form-field/index.js
+++ b/modules/web-console/frontend/app/components/form-field/index.js
@@ -20,7 +20,10 @@ import './style.scss';
 import {directive as showValidationError} from './showValidationError.directive';
 import {directive as copyInputValue} from './copyInputValueButton.directive';
 
+import { default as formFieldSize } from './components/form-field-size';
+
 export default angular
     .module('ignite-console.form-field', [])
+    .component('formFieldSize', formFieldSize)
     .directive('ngModel', showValidationError)
     .directive('copyInputValueButton', copyInputValue);
diff --git a/modules/web-console/frontend/app/components/grid-column-selector/component.js b/modules/web-console/frontend/app/components/grid-column-selector/component.js
index 957c821f0e7df..0ad37111859dc 100644
--- a/modules/web-console/frontend/app/components/grid-column-selector/component.js
+++ b/modules/web-console/frontend/app/components/grid-column-selector/component.js
@@ -16,7 +16,7 @@
  */
 
 import template from './template.pug';
-import controller from './controller.js';
+import controller from './controller';
 import './style.scss';
 
 export default {
diff --git a/modules/web-console/frontend/app/components/grid-column-selector/style.scss b/modules/web-console/frontend/app/components/grid-column-selector/style.scss
index 5e0d5d4c2b987..872d727f37e9f 100644
--- a/modules/web-console/frontend/app/components/grid-column-selector/style.scss
+++ b/modules/web-console/frontend/app/components/grid-column-selector/style.scss
@@ -17,6 +17,7 @@
 
 grid-column-selector {
     display: inline-block;
+    line-height: initial;
 
     .btn-ignite, .icon {
         margin: 0 !important;
diff --git a/modules/web-console/frontend/app/components/grid-export/component.js b/modules/web-console/frontend/app/components/grid-export/component.js
index d312959b16d9a..d4cfb290ac2a0 100644
--- a/modules/web-console/frontend/app/components/grid-export/component.js
+++ b/modules/web-console/frontend/app/components/grid-export/component.js
@@ -21,7 +21,7 @@ import {CSV} from 'app/services/CSV';
 export default {
     template,
     controller: class {
-        static $inject = ['$scope', 'uiGridGroupingConstants', 'uiGridExporterService', 'uiGridExporterConstants', CSV.name];
+        static $inject = ['$scope', 'uiGridGroupingConstants', 'uiGridExporterService', 'uiGridExporterConstants', 'CSV'];
 
         /**
          * @param {CSV} CSV
@@ -58,10 +58,13 @@ export default {
 
             const csvContent = this.uiGridExporterService.formatAsCsv(exportColumnHeaders, data, this.CSV.getSeparator());
 
-            this.uiGridExporterService.downloadFile(this.gridApi.grid.options.exporterCsvFilename, csvContent, this.gridApi.grid.options.exporterOlderExcelCompatibility);
+            const csvFileName = this.fileName || 'export.csv';
+
+            this.uiGridExporterService.downloadFile(csvFileName, csvContent, this.gridApi.grid.options.exporterOlderExcelCompatibility);
         }
     },
     bindings: {
-        gridApi: '<'
+        gridApi: '<',
+        fileName: '<'
     }
 };
diff --git a/modules/web-console/frontend/app/components/grid-export/index.js b/modules/web-console/frontend/app/components/grid-export/index.js
index 9fa8835b8197f..00e2bced22a7d 100644
--- a/modules/web-console/frontend/app/components/grid-export/index.js
+++ b/modules/web-console/frontend/app/components/grid-export/index.js
@@ -16,7 +16,7 @@
  */
 
 import angular from 'angular';
-
+import './style.scss';
 import component from './component';
 
 export default angular
diff --git a/modules/web-console/frontend/app/modules/form/field/input/text.scss b/modules/web-console/frontend/app/components/grid-export/style.scss
similarity index 69%
rename from modules/web-console/frontend/app/modules/form/field/input/text.scss
rename to modules/web-console/frontend/app/components/grid-export/style.scss
index 658d740b8ebd2..29e35fde5e38d 100644
--- a/modules/web-console/frontend/app/modules/form/field/input/text.scss
+++ b/modules/web-console/frontend/app/components/grid-export/style.scss
@@ -15,27 +15,20 @@
  * limitations under the License.
  */
 
-.checkbox label .input-tip {
-	position: initial;
-    overflow: visible;
-}
-
-.input-tip .fa-floppy-o {
-	position: absolute;
-    top: 0;
-    right: 0;
-    z-index: 2;
-
-    width: 34px;
-    height: 34px;
+grid-export {
+	&:not(.link-success) {
+		button:nth-child(2) {
+			display: none;
+		}
+	}
 
-    text-align: center;
-
-    display: inline-block;
-    line-height: 28px;
-    pointer-events: initial;
-}
+	&.link-success {
+		button:nth-child(1) {
+			display: none;
+		}
+	}
 
-.input-tip .form-control-feedback {
-    height: auto;
+	[ignite-icon='download'] {
+		margin-right: 5px;
+	}
 }
diff --git a/modules/web-console/frontend/app/components/grid-export/template.pug b/modules/web-console/frontend/app/components/grid-export/template.pug
index fac10df42a839..a085db6f0fd66 100644
--- a/modules/web-console/frontend/app/components/grid-export/template.pug
+++ b/modules/web-console/frontend/app/components/grid-export/template.pug
@@ -14,5 +14,9 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-button.btn-ignite.btn-ignite--link-dashed-secondary(ng-click='$ctrl.export()' bs-tooltip='' data-title='Export filtered rows to CSV' data-placement='top')
+button.btn-ignite.btn-ignite--primary-outline(ng-click='$ctrl.export()' bs-tooltip='' data-title='Export filtered rows to CSV' data-placement='top')
+    svg(ignite-icon='csv')
+
+button.btn-ignite.btn-ignite--link-success.link-success(ng-click='$ctrl.export()' bs-tooltip='' data-title='Export filtered rows to CSV' data-placement='top')
     svg(ignite-icon='download')
+    i Export to .csv
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/grid-item-selected/component.js b/modules/web-console/frontend/app/components/grid-item-selected/component.js
index d25ad74803fc3..c5b10d38ffefc 100644
--- a/modules/web-console/frontend/app/components/grid-item-selected/component.js
+++ b/modules/web-console/frontend/app/components/grid-item-selected/component.js
@@ -16,7 +16,7 @@
  */
 
 import template from './template.pug';
-import controller from './controller.js';
+import controller from './controller';
 
 export default {
     template,
diff --git a/modules/web-console/frontend/app/components/grid-item-selected/index.js b/modules/web-console/frontend/app/components/grid-item-selected/index.js
index 583d871d14525..4d76e6e56c68f 100644
--- a/modules/web-console/frontend/app/components/grid-item-selected/index.js
+++ b/modules/web-console/frontend/app/components/grid-item-selected/index.js
@@ -17,6 +17,7 @@
 
 import angular from 'angular';
 
+import './style.scss';
 import component from './component';
 
 export default angular
diff --git a/modules/web-console/frontend/app/components/grid-item-selected/style.scss b/modules/web-console/frontend/app/components/grid-item-selected/style.scss
new file mode 100644
index 0000000000000..b131d82203760
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-item-selected/style.scss
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grid-item-selected {
+    display: inline-flex;
+    align-items: center;
+
+    font-size: 14px;
+}
diff --git a/modules/web-console/frontend/app/components/grid-item-selected/template.pug b/modules/web-console/frontend/app/components/grid-item-selected/template.pug
index eca079e2b9a12..cf0695b038d0e 100644
--- a/modules/web-console/frontend/app/components/grid-item-selected/template.pug
+++ b/modules/web-console/frontend/app/components/grid-item-selected/template.pug
@@ -14,4 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-i {{ $ctrl.selected }} of {{ $ctrl.count }} selected
+span(ng-if='$ctrl.count')
+    i {{ $ctrl.selected }} of {{ $ctrl.count }} selected
+span(ng-if='!$ctrl.count')
+    i Showing: 0 rows
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/grid-no-data/component.js b/modules/web-console/frontend/app/components/grid-no-data/component.js
index aa219dfad35eb..e7e26b8eda60a 100644
--- a/modules/web-console/frontend/app/components/grid-no-data/component.js
+++ b/modules/web-console/frontend/app/components/grid-no-data/component.js
@@ -16,7 +16,7 @@
  */
 
 import './style.scss';
-import controller from './controller.js';
+import controller from './controller';
 
 export default {
     template: `
diff --git a/modules/web-console/frontend/app/components/grid-no-data/controller.js b/modules/web-console/frontend/app/components/grid-no-data/controller.js
index 2451e033962a9..d252c580165ae 100644
--- a/modules/web-console/frontend/app/components/grid-no-data/controller.js
+++ b/modules/web-console/frontend/app/components/grid-no-data/controller.js
@@ -15,9 +15,7 @@
  * limitations under the License.
  */
 
-import filter from 'lodash/fp/filter';
-
-const rowsFiltered = filter(({ visible }) => visible);
+import _ from 'lodash';
 
 export default class {
     static $inject = ['$scope', 'uiGridConstants'];
@@ -47,7 +45,6 @@ export default class {
 
         this.noData = false;
 
-        const filtered = rowsFiltered(this.gridApi.grid.rows);
-        this.noDataFiltered = !filtered.length;
+        this.noDataFiltered = _.sumBy(this.gridApi.grid.rows, 'visible') === 0;
     }
 }
diff --git a/modules/web-console/frontend/app/components/grid-showing-rows/component.js b/modules/web-console/frontend/app/components/grid-showing-rows/component.js
new file mode 100644
index 0000000000000..6cab628cacae2
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-showing-rows/component.js
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './style.scss';
+import controller from './controller';
+
+export default {
+    template: `
+        <i ng-pluralize count="$ctrl.count" when="{'one': 'Showing: 1 row', 'other': 'Showing: {} rows'}"></i>
+    `,
+    controller,
+    bindings: {
+        gridApi: '<'
+    }
+};
diff --git a/modules/web-console/frontend/app/primitives/file/index.scss b/modules/web-console/frontend/app/components/grid-showing-rows/controller.js
similarity index 55%
rename from modules/web-console/frontend/app/primitives/file/index.scss
rename to modules/web-console/frontend/app/components/grid-showing-rows/controller.js
index dbd32a4e480bd..67c407c3bfc3f 100644
--- a/modules/web-console/frontend/app/primitives/file/index.scss
+++ b/modules/web-console/frontend/app/components/grid-showing-rows/controller.js
@@ -15,46 +15,33 @@
  * limitations under the License.
  */
 
-.file--ignite {
-    label {
-        cursor: pointer;
-    }
+import _ from 'lodash';
 
-    &.choosen {
-        .ignite-form-field__control {
-            .btn-ignite {
-                display: none;
-            }
+export default class {
+    static $inject = ['$scope', 'uiGridConstants'];
 
-            .tipField {
-                display: block;
-            }
-        }
-    }
+    constructor($scope, uiGridConstants) {
+        Object.assign(this, {$scope, uiGridConstants});
 
-    &:not(.choosen) {
-        .ignite-form-field__label {
-            font-size: 0;
-            line-height: 0;
-            margin-bottom: 0px;
-        }
+        this.count = 0;
     }
 
-    .ignite-form-field__control {
-        display: flex;
+    $onChanges(changes) {
+        if (changes && 'gridApi' in changes && changes.gridApi.currentValue) {
+            this.applyValues();
 
-        input[type='file'] {
-            position: absolute;
-            opacity: 0;
-            z-index: -1;
+            this.gridApi.core.on.rowsVisibleChanged(this.$scope, () => {
+                this.applyValues();
+            });
         }
+    }
 
-        .tipField {
-            display: none;
+    applyValues() {
+        if (!this.gridApi.grid.rows.length) {
+            this.count = 0;
+            return;
         }
 
-        .input-tip {
-            order: -1;
-        }
+        this.count = _.sumBy(this.gridApi.grid.rows, (row) => Number(row.visible));
     }
-}
\ No newline at end of file
+}
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/index.js b/modules/web-console/frontend/app/components/grid-showing-rows/index.js
similarity index 89%
rename from modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/index.js
rename to modules/web-console/frontend/app/components/grid-showing-rows/index.js
index 1fdc37911398c..9f8c9212dbc04 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/index.js
+++ b/modules/web-console/frontend/app/components/grid-showing-rows/index.js
@@ -16,8 +16,9 @@
  */
 
 import angular from 'angular';
+
 import component from './component';
 
 export default angular
-    .module('ignite-console.page-configure.form-field-size', [])
-    .component('pcFormFieldSize', component);
+    .module('ignite-console.grid-showing-rows', [])
+    .component('gridShowingRows', component);
diff --git a/modules/web-console/frontend/app/components/grid-showing-rows/style.scss b/modules/web-console/frontend/app/components/grid-showing-rows/style.scss
new file mode 100644
index 0000000000000..a37f1b51df35c
--- /dev/null
+++ b/modules/web-console/frontend/app/components/grid-showing-rows/style.scss
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+grid-showing-rows {
+	color: #757575;
+}
diff --git a/modules/web-console/frontend/app/components/ignite-chart-series-selector/component.js b/modules/web-console/frontend/app/components/ignite-chart-series-selector/component.js
new file mode 100644
index 0000000000000..0ca54026b0215
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart-series-selector/component.js
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+import controller from './controller';
+
+export default {
+    template,
+    controller,
+    transclude: true,
+    bindings: {
+        chartApi: '<'
+    }
+};
diff --git a/modules/web-console/frontend/app/components/ignite-chart-series-selector/controller.js b/modules/web-console/frontend/app/components/ignite-chart-series-selector/controller.js
new file mode 100644
index 0000000000000..f46d8da5c9034
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart-series-selector/controller.js
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export default class IgniteChartSeriesSelectorController {
+    constructor() {
+        this.charts = [];
+        this.selectedCharts = [];
+    }
+
+    $onChanges(changes) {
+        if (changes && 'chartApi' in changes && changes.chartApi.currentValue) {
+            this.applyValues();
+            this.setSelectedCharts();
+        }
+    }
+
+    applyValues() {
+        this.charts = this._makeMenu();
+        this.selectedCharts = this.charts.filter((chart) => !chart.hidden).map(({ key }) => key);
+    }
+
+    setSelectedCharts() {
+        const selectedDataset = ({ label }) => this.selectedCharts.includes(label);
+
+        this.chartApi.config.data.datasets
+            .forEach((dataset) => {
+                dataset.hidden = true;
+
+                if (!selectedDataset(dataset))
+                    return;
+
+                dataset.hidden = false;
+            });
+
+        this.chartApi.update();
+    }
+
+    _makeMenu() {
+        const labels = this.chartApi.config.datasetLegendMapping;
+
+        return Object.keys(this.chartApi.config.datasetLegendMapping).map((key) => {
+            return {
+                key,
+                label: labels[key].name || labels[key],
+                hidden: labels[key].hidden
+            };
+        });
+    }
+}
diff --git a/modules/web-console/frontend/app/components/expose-ignite-form-field-control/index.js b/modules/web-console/frontend/app/components/ignite-chart-series-selector/index.js
similarity index 81%
rename from modules/web-console/frontend/app/components/expose-ignite-form-field-control/index.js
rename to modules/web-console/frontend/app/components/ignite-chart-series-selector/index.js
index 9a22478c71963..7ad3da0444139 100644
--- a/modules/web-console/frontend/app/components/expose-ignite-form-field-control/index.js
+++ b/modules/web-console/frontend/app/components/ignite-chart-series-selector/index.js
@@ -16,8 +16,9 @@
  */
 
 import angular from 'angular';
-import {igniteFormField, exposeIgniteFormFieldControl} from './directives';
+
+import component from './component';
 
 export default angular
-.module('expose-ignite-form-field-control', [])
-.directive('exposeIgniteFormFieldControl', exposeIgniteFormFieldControl);
+    .module('ignite-console.ignite-chart-series-selector', [])
+    .component('igniteChartSeriesSelector', component);
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-label.pug b/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug
similarity index 64%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-label.pug
rename to modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug
index 2edd115650014..203f12f64e8a1 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-label.pug
+++ b/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug
@@ -14,12 +14,16 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin ignite-form-field__label(label, name, required, disabled)
-    label.ignite-form-field__label(
-        id=`{{ ${name} }}Label`
-        for=`{{ ${name} }}Input`
-        ng-class=disabled && `{'ignite-form-field__label-disabled': ${disabled}}`
-    )
-        span(class=`{{ ${required} ? 'required' : '' }}`) !{label}
-        if block
-            block
\ No newline at end of file
+button.btn-ignite.btn-ignite--link-dashed-secondary(
+    protect-from-bs-select-render
+    bs-select
+    ng-model='$ctrl.selectedCharts'
+    ng-change='$ctrl.setSelectedCharts()'
+    ng-model-options='{debounce: {default: 5}}',
+    bs-options='chart.key as chart.label for chart in $ctrl.charts'
+    bs-on-before-show='$ctrl.onShow'
+    data-multiple='true'
+    ng-transclude
+    ng-disabled='!($ctrl.charts.length)'
+)
+    svg(ignite-icon='gear').icon
diff --git a/modules/web-console/frontend/app/components/ignite-chart/controller.js b/modules/web-console/frontend/app/components/ignite-chart/controller.js
new file mode 100644
index 0000000000000..79156fc8110c1
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart/controller.js
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import _ from 'lodash';
+
+/**
+ * @typedef {{x: number, y: {[key: string]: number}}} IgniteChartDataPoint
+ */
+
+const RANGE_RATE_PRESET = [{
+    label: '1 min',
+    value: 1
+}, {
+    label: '5 min',
+    value: 5
+}, {
+    label: '10 min',
+    value: 10
+}, {
+    label: '15 min',
+    value: 15
+}, {
+    label: '30 min',
+    value: 30
+}];
+
+export class IgniteChartController {
+    /** @type {import('chart.js').ChartConfiguration} */
+    chartOptions;
+    /** @type {string} */
+    chartTitle;
+    /** @type {IgniteChartDataPoint} */
+    chartDataPoint;
+    /** @type {Array<IgniteChartDataPoint>} */
+    chartHistory;
+    newPoints = [];
+
+    static $inject = ['$element', 'IgniteChartColors', '$filter'];
+
+    /**
+     * @param {JQLite} $element
+     * @param {Array<string>} IgniteChartColors
+     * @param {ng.IFilterService} $filter
+     */
+    constructor($element, IgniteChartColors, $filter) {
+        this.$element = $element;
+        this.IgniteChartColors = IgniteChartColors;
+
+        this.datePipe = $filter('date');
+        this.ranges = RANGE_RATE_PRESET;
+        this.currentRange = this.ranges[0];
+        this.maxRangeInMilliseconds = RANGE_RATE_PRESET[RANGE_RATE_PRESET.length - 1].value * 60 * 1000;
+        this.ctx = this.$element.find('canvas')[0].getContext('2d');
+
+        this.localHistory = [];
+        this.updateIsBusy = false;
+    }
+
+    $onDestroy() {
+        if (this.chart)
+            this.chart.destroy();
+
+        this.$element = this.ctx = this.chart = null;
+    }
+
+    $onInit() {
+        this.chartColors = _.get(this.chartOptions, 'chartColors', this.IgniteChartColors);
+    }
+
+    _refresh() {
+        this.onRefresh();
+        this.rerenderChart();
+    }
+
+    /**
+     * @param {{chartOptions: ng.IChangesObject<import('chart.js').ChartConfiguration>, chartTitle: ng.IChangesObject<string>, chartDataPoint: ng.IChangesObject<IgniteChartDataPoint>, chartHistory: ng.IChangesObject<Array<IgniteChartDataPoint>>}} changes
+     */
+    async $onChanges(changes) {
+        if (this.chart && _.get(changes, 'refreshRate.currentValue'))
+            this.onRefreshRateChanged(_.get(changes, 'refreshRate.currentValue'));
+
+        if ((changes.chartDataPoint && _.isNil(changes.chartDataPoint.currentValue)) ||
+            (changes.chartHistory && _.isEmpty(changes.chartHistory.currentValue))) {
+            this.clearDatasets();
+
+            return;
+        }
+
+        if (changes.chartHistory && changes.chartHistory.currentValue && changes.chartHistory.currentValue.length !== changes.chartHistory.previousValue.length) {
+            if (!this.chart)
+                await this.initChart();
+
+            this.clearDatasets();
+            this.localHistory = [...changes.chartHistory.currentValue];
+
+            this.newPoints.splice(0, this.newPoints.length, ...changes.chartHistory.currentValue);
+
+            this._refresh();
+
+            return;
+        }
+
+        if (this.chartDataPoint && changes.chartDataPoint) {
+            if (!this.chart)
+                this.initChart();
+
+            this.newPoints.push(this.chartDataPoint);
+            this.localHistory.push(this.chartDataPoint);
+
+            this._refresh();
+        }
+    }
+
+    async initChart() {
+        /** @type {import('chart.js').ChartConfiguration} */
+        this.config = {
+            type: 'LineWithVerticalCursor',
+            data: {
+                datasets: []
+            },
+            options: {
+                elements: {
+                    line: {
+                        tension: 0
+                    },
+                    point: {
+                        radius: 2,
+                        pointStyle: 'rectRounded'
+                    }
+                },
+                animation: {
+                    duration: 0 // general animation time
+                },
+                hover: {
+                    animationDuration: 0 // duration of animations when hovering an item
+                },
+                responsiveAnimationDuration: 0, // animation duration after a resize
+                maintainAspectRatio: false,
+                responsive: true,
+                legend: {
+                    display: false
+                },
+                scales: {
+                    xAxes: [{
+                        type: 'realtime',
+                        display: true,
+                        time: {
+                            displayFormats: {
+                                second: 'HH:mm:ss',
+                                minute: 'HH:mm:ss',
+                                hour: 'HH:mm:ss'
+                            }
+                        },
+                        ticks: {
+                            maxRotation: 0,
+                            minRotation: 0
+                        }
+                    }],
+                    yAxes: [{
+                        type: 'linear',
+                        display: true,
+                        ticks: {
+                            min: 0,
+                            beginAtZero: true,
+                            maxTicksLimit: 4,
+                            callback: (value, index, labels) => {
+                                if (value === 0)
+                                    return 0;
+
+                                if (_.max(labels) <= 4000 && value <= 4000)
+                                    return value;
+
+                                if (_.max(labels) <= 1000000 && value <= 1000000)
+                                    return `${value / 1000}K`;
+
+                                if ((_.max(labels) <= 4000000 && value >= 500000) || (_.max(labels) > 4000000))
+                                    return `${value / 1000000}M`;
+
+                                return value;
+                            }
+                        }
+                    }]
+                },
+                tooltips: {
+                    mode: 'index',
+                    position: 'yCenter',
+                    intersect: false,
+                    yAlign: 'center',
+                    xPadding: 20,
+                    yPadding: 20,
+                    bodyFontSize: 13,
+                    callbacks: {
+                        title: (tooltipItem) => {
+                            return tooltipItem[0].xLabel.slice(0, -7);
+                        },
+                        label: (tooltipItem, data) => {
+                            const label = data.datasets[tooltipItem.datasetIndex].label || '';
+
+                            return `${_.startCase(label)}: ${tooltipItem.yLabel} per sec`;
+                        },
+                        labelColor: (tooltipItem) => {
+                            return {
+                                borderColor: 'rgba(255,255,255,0.5)',
+                                borderWidth: 0,
+                                boxShadow: 'none',
+                                backgroundColor: this.chartColors[tooltipItem.datasetIndex]
+                            };
+                        }
+                    }
+                },
+                plugins: {
+                    streaming: {
+                        duration: this.currentRange.value * 1000 * 60,
+                        frameRate: 1000 / this.refreshRate || 1 / 3,
+                        refresh: this.refreshRate || 3000,
+                        ttl: this.maxRangeInMilliseconds,
+                        onRefresh: () => {
+                            this.onRefresh();
+                        }
+                    }
+                }
+            }
+        };
+
+        this.config = _.merge(this.config, this.chartOptions);
+
+        const chartModule = await import('chart.js');
+        const Chart = chartModule.default;
+
+        Chart.Tooltip.positioners.yCenter = (elements) => {
+            const chartHeight = elements[0]._chart.height;
+            const tooltipHeight = 60;
+
+            return {x: elements[0].getCenterPoint().x, y: Math.floor(chartHeight / 2) - Math.floor(tooltipHeight / 2) };
+        };
+
+
+        // Drawing vertical cursor
+        Chart.defaults.LineWithVerticalCursor = Chart.defaults.line;
+        Chart.controllers.LineWithVerticalCursor = Chart.controllers.line.extend({
+            draw(ease) {
+                Chart.controllers.line.prototype.draw.call(this, ease);
+
+                if (this.chart.tooltip._active && this.chart.tooltip._active.length) {
+                    const activePoint = this.chart.tooltip._active[0];
+                    const ctx = this.chart.ctx;
+                    const x = activePoint.tooltipPosition().x;
+                    const topY = this.chart.scales['y-axis-0'].top;
+                    const bottomY = this.chart.scales['y-axis-0'].bottom;
+
+                    // draw line
+                    ctx.save();
+                    ctx.beginPath();
+                    ctx.moveTo(x, topY);
+                    ctx.lineTo(x, bottomY);
+                    ctx.lineWidth = 0.5;
+                    ctx.strokeStyle = '#0080ff';
+                    ctx.stroke();
+                    ctx.restore();
+                }
+            }
+        });
+
+        await import('chartjs-plugin-streaming');
+
+        this.chart = new Chart(this.ctx, this.config);
+        this.changeXRange(this.currentRange);
+    }
+
+    onRefresh() {
+        this.newPoints.forEach((point) => {
+            this.appendChartPoint(point);
+        });
+
+        this.newPoints.splice(0, this.newPoints.length);
+    }
+
+    /**
+     * @param {IgniteChartDataPoint} dataPoint
+     */
+    appendChartPoint(dataPoint) {
+        Object.keys(dataPoint.y).forEach((key) => {
+            if (this.checkDatasetCanBeAdded(key)) {
+                let datasetIndex = this.findDatasetIndex(key);
+
+                if (datasetIndex < 0) {
+                    datasetIndex = this.config.data.datasets.length;
+                    this.addDataset(key);
+                }
+
+                this.config.data.datasets[datasetIndex].data.push({x: dataPoint.x, y: dataPoint.y[key]});
+                this.config.data.datasets[datasetIndex].borderColor = this.chartColors[datasetIndex];
+                this.config.data.datasets[datasetIndex].borderWidth = 2;
+                this.config.data.datasets[datasetIndex].fill = false;
+            }
+        });
+    }
+
+    /**
+     * Checks if a key of dataset can be added to chart or should be ignored.
+     * @param dataPointKey {String}
+     * @return {Boolean}
+     */
+    checkDatasetCanBeAdded(dataPointKey) {
+        // If datasetLegendMapping is empty all keys are allowed.
+        if (!this.config.datasetLegendMapping)
+            return true;
+
+        return Object.keys(this.config.datasetLegendMapping).includes(dataPointKey);
+    }
+
+    clearDatasets() {
+        if (!_.isNil(this.config))
+            this.config.data.datasets.forEach((dataset) => dataset.data = []);
+    }
+
+    addDataset(datasetName) {
+        if (this.findDatasetIndex(datasetName) >= 0)
+            throw new Error(`Dataset with name ${datasetName} is already in chart`);
+        else {
+            const datasetIsHidden = _.isNil(this.config.datasetLegendMapping[datasetName].hidden)
+                ? false
+                : this.config.datasetLegendMapping[datasetName].hidden;
+
+            this.config.data.datasets.push({ label: datasetName, data: [], hidden: datasetIsHidden });
+        }
+    }
+
+    findDatasetIndex(searchedDatasetLabel) {
+        return this.config.data.datasets.findIndex((dataset) => dataset.label === searchedDatasetLabel);
+    }
+
+    changeXRange(range) {
+        if (this.chart) {
+            this.chart.config.options.plugins.streaming.duration = range.value * 60 * 1000;
+
+            this.clearDatasets();
+            this.newPoints.splice(0, this.newPoints.length, ...this.localHistory);
+
+            this.onRefresh();
+            this.rerenderChart();
+        }
+    }
+
+    onRefreshRateChanged(refreshRate) {
+        this.chart.config.options.plugins.streaming.frameRate = 1000 / refreshRate;
+        this.chart.config.options.plugins.streaming.refresh = refreshRate;
+        this.rerenderChart();
+    }
+
+    rerenderChart() {
+        if (this.chart)
+            this.chart.update();
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/form/field/feedback.scss b/modules/web-console/frontend/app/components/ignite-chart/index.js
similarity index 58%
rename from modules/web-console/frontend/app/modules/form/field/feedback.scss
rename to modules/web-console/frontend/app/components/ignite-chart/index.js
index 08d0aefd8b8f2..6900814f4a352 100644
--- a/modules/web-console/frontend/app/modules/form/field/feedback.scss
+++ b/modules/web-console/frontend/app/components/ignite-chart/index.js
@@ -15,23 +15,24 @@
  * limitations under the License.
  */
 
-@import "../../../../public/stylesheets/variables";
+import angular from 'angular';
 
-.form-field-feedback {
-    position: relative;
-    width: 0;
-    height: 28px;
-    float: right;
-    z-index: 2;
+import { IgniteChartController } from './controller';
+import template from './template.pug';
+import './style.scss';
 
-    color: $brand-primary;
-    line-height: $input-height;
-    pointer-events: initial;
-    text-align: center;
-
-    &:before {
-        position: absolute;
-        right: 0;
-        width: 38px;
-    }
-}
+export default angular
+    .module('ignite-console.ignite-chart', [])
+    .component('igniteChart', {
+        controller: IgniteChartController,
+        template,
+        bindings: {
+            chartOptions: '<',
+            chartDataPoint: '<',
+            chartHistory: '<',
+            chartTitle: '<',
+            chartColors: '<',
+            chartHeaderText: '<',
+            refreshRate: '<'
+        }
+    });
diff --git a/modules/web-console/frontend/app/components/ignite-chart/style.scss b/modules/web-console/frontend/app/components/ignite-chart/style.scss
new file mode 100644
index 0000000000000..3a07bd5542986
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-chart/style.scss
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ignite-chart {
+  display: flex;
+  flex-direction: column;
+
+  height: 100%;
+
+  header {
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    box-sizing: content-box;
+
+    height: 36px;
+    min-height: 36px;
+    padding: 7px 21px;
+
+    border-bottom: 1px solid #d4d4d4;
+
+    h5 {
+      margin: 0;
+
+      font-size: 16px;
+      font-weight: normal;
+      line-height: 36px;
+    }
+
+    ignite-chart-series-selector {
+      margin: 0 2px;
+    }
+
+    > div {
+      &:first-child {
+        width: calc(100% - 120px);
+        white-space: nowrap;
+      }
+
+      display: flex;
+      align-items: center;
+      flex-wrap: nowrap;
+      flex-grow: 0;
+
+      .chart-text {
+        overflow: hidden;
+        text-overflow: ellipsis;
+      }
+    }
+  }
+
+  .ignite-chart-placeholder {
+    display: block;
+    height: calc(100% - 71px);
+    margin-top: 20px;
+  }
+
+  .no-data {
+    position: absolute;
+    top: 50%;
+
+    width: 100%;
+    padding: 0 20px;
+
+    border-radius: 0 0 4px 4px;
+
+    font-style: italic;
+    line-height: 16px;
+    text-align: center;
+  }
+}
diff --git a/modules/web-console/frontend/app/primitives/tooltip/index.pug b/modules/web-console/frontend/app/components/ignite-chart/template.pug
similarity index 54%
rename from modules/web-console/frontend/app/primitives/tooltip/index.pug
rename to modules/web-console/frontend/app/components/ignite-chart/template.pug
index ea6a344c58c60..5108853b46d6a 100644
--- a/modules/web-console/frontend/app/primitives/tooltip/index.pug
+++ b/modules/web-console/frontend/app/components/ignite-chart/template.pug
@@ -14,14 +14,24 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin tooltip(title, options, tipClass = 'tipField')
-    if title
-        svg(
-            ignite-icon='info'
-            bs-tooltip=''
+header
+    div
+        h5 {{ $ctrl.chartTitle }}
+        ignite-chart-series-selector(chart-api='$ctrl.chart')
+        .chart-text(ng-if='$ctrl.chartHeaderText') {{$ctrl.chartHeaderText}}
 
-            data-title=title
-            data-container=options && options.container || false
-            data-placement=options && options.placement || false
-            class=`${tipClass}`
-        )
+    div
+        span Range:
+            button.btn-ignite.btn-ignite--link-success.link-primary(
+                ng-model='$ctrl.currentRange'
+                ng-change='$ctrl.changeXRange($ctrl.currentRange)'
+                bs-options='item as item.label for item in $ctrl.ranges'
+                bs-select
+            )
+
+.ignite-chart-placeholder
+    canvas
+
+.no-data(ng-if='!$ctrl.config.data.datasets')
+    | No Data #[br]
+    | Make sure you are connected to the right grid.
diff --git a/modules/web-console/frontend/app/components/ignite-icon/directive.js b/modules/web-console/frontend/app/components/ignite-icon/directive.js
index b72c4f9ebd897..9838fe2a51f73 100644
--- a/modules/web-console/frontend/app/components/ignite-icon/directive.js
+++ b/modules/web-console/frontend/app/components/ignite-icon/directive.js
@@ -21,8 +21,21 @@ export default function() {
         controller: class {
             static $inject = ['$scope', '$attrs', '$sce', '$element', '$window', 'IgniteIcon'];
 
+            /**
+             * @param {ng.IScope} $scope     
+             * @param {ng.IAttributes} $attrs     
+             * @param {ng.ISCEService} $sce       
+             * @param {JQLite} $element   
+             * @param {ng.IWindowService} $window    
+             * @param {import('./service').default} IgniteIcon 
+             */
             constructor($scope, $attrs, $sce, $element, $window, IgniteIcon) {
-                Object.assign(this, {$scope, $attrs, $sce, $element, $window, IgniteIcon});
+                this.$scope = $scope;
+                this.$attrs = $attrs;
+                this.$sce = $sce;
+                this.$element = $element;
+                this.$window = $window;
+                this.IgniteIcon = IgniteIcon;
             }
 
             $onInit() {
@@ -40,6 +53,7 @@ export default function() {
             }
 
             $postLink() {
+                /** @type {string} */
                 this.name = this.$attrs.igniteIcon;
                 this.$element.attr('viewBox', this.IgniteIcon.getIcon(this.name).viewBox);
 
@@ -52,6 +66,9 @@ export default function() {
                 return `${url.split('#')[0]}#${this.name}`;
             }
 
+            /**
+             * @param {string} url 
+             */
             render(url) {
                 // templateNamespace: 'svg' does not work in IE11
                 this.wrapper.innerHTML = `<svg><use xlink:href="${url}" href="${url}" /></svg>`;
diff --git a/modules/web-console/frontend/app/components/ignite-icon/service.js b/modules/web-console/frontend/app/components/ignite-icon/service.js
index e142a2d480639..34ea0744e1a0d 100644
--- a/modules/web-console/frontend/app/components/ignite-icon/service.js
+++ b/modules/web-console/frontend/app/components/ignite-icon/service.js
@@ -15,13 +15,30 @@
  * limitations under the License.
  */
 
+/**
+ * @typedef {{id: string, viewBox: string, content: string}} SpriteSymbol
+ */
+
+/**
+ * @typedef {{[name: string]: SpriteSymbol}} Icons
+ */
+
 export default class IgniteIcon {
+    /**
+     * @type {Icons}
+     */
     _icons = {};
 
+    /**
+     * @param {Icons} icons
+     */
     registerIcons(icons) {
         return Object.assign(this._icons, icons);
     }
 
+    /**
+     * @param {string} name
+     */
     getIcon(name) {
         return this._icons[name];
     }
diff --git a/modules/web-console/frontend/app/components/ignite-status/index.js b/modules/web-console/frontend/app/components/ignite-status/index.js
new file mode 100644
index 0000000000000..a8520bc876952
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ignite-status/index.js
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import './style.scss';
+
+export default angular
+    .module('ignite-console.ignite-status', []);
diff --git a/modules/web-console/frontend/app/components/page-signin/types.ts b/modules/web-console/frontend/app/components/ignite-status/style.scss
similarity index 79%
rename from modules/web-console/frontend/app/components/page-signin/types.ts
rename to modules/web-console/frontend/app/components/ignite-status/style.scss
index 96cca75a5e41d..d2877fc5a71db 100644
--- a/modules/web-console/frontend/app/components/page-signin/types.ts
+++ b/modules/web-console/frontend/app/components/ignite-status/style.scss
@@ -15,12 +15,13 @@
  * limitations under the License.
  */
 
-export interface ISiginData {
-    email: string,
-    password: string
+@import "../../../public/stylesheets/variables";
+
+// Statuses coloring
+.ignite-status__active {
+  color: $ignite-status-active !important;
 }
 
-export interface ISigninFormController extends ng.IFormController {
-    email: ng.INgModelController,
-    password: ng.INgModelController
+.ignite-status__inactive {
+  color: $ignite-status-inactive;
 }
diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js b/modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js
index 3f6e97b96c9a6..1de142c10c091 100644
--- a/modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.controller.js
@@ -15,21 +15,20 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
 export default class InputDialogController {
     static $inject = ['deferred', 'ui'];
 
-    constructor(deferred, {title, label, value, toValidValue}) {
+    constructor(deferred, options) {
         this.deferred = deferred;
-        this.title = title;
-        this.label = label;
-        this.value = value;
-        this.toValidValue = toValidValue;
+        this.options = options;
     }
 
     confirm() {
-        if (_.isFunction(this.toValidValue))
-            return this.deferred.resolve(this.toValidValue(this.value));
+        if (_.isFunction(this.options.toValidValue))
+            return this.deferred.resolve(this.options.toValidValue(this.options.value));
 
-        this.deferred.resolve(this.value);
+        this.deferred.resolve(this.options.value);
     }
 }
diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js
index cfd61714df123..6ddf051290e13 100644
--- a/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.js
@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
 import controller from './input-dialog.controller';
 import templateUrl from './input-dialog.tpl.pug';
 import {CancellationError} from 'app/errors/CancellationError';
@@ -22,33 +23,39 @@ import {CancellationError} from 'app/errors/CancellationError';
 export default class InputDialog {
     static $inject = ['$modal', '$q'];
 
+    /**
+     * @param {mgcrea.ngStrap.modal.IModalService} $modal
+     * @param {ng.IQService} $q
+     */
     constructor($modal, $q) {
         this.$modal = $modal;
         this.$q = $q;
     }
 
     /**
-     * Open input dialog to configure custom value.
+     * Fabric for creating modal instance with different input types.
      *
-     * @param {String} title Dialog title.
-     * @param {String} label Input field label.
-     * @param {String} value Default value.
-     * @param {Function} [toValidValue] Validator function.
-     * @returns {Promise.<String>} User input.
+     * @param {Object} args Options for rendering inputs:
+     * @param {'text'|'number'} args.mode Input type.
+     * @param {String} args.title Dialog title.
+     * @param {String} args.label Input field label.
+     * @param {String} args.tip Message for tooltip in label.
+     * @param {String|Number} args.value Default value.
+     * @param {String} args.placeholder Placeholder for input.
+     * @param {Function} [args.toValidValue] Validator function.
+     * @param {Number} args.min Min value for number input.
+     * @param {Number} args.max Max value for number input.
+     * @param {String} args.postfix Postfix for units in numer input.
+     * @return {Promise.<String>} User input.
      */
-    input(title, label, value, toValidValue) {
+    dialogFabric(args) {
         const deferred = this.$q.defer();
 
         const modal = this.$modal({
             templateUrl,
             resolve: {
                 deferred: () => deferred,
-                ui: () => ({
-                    title,
-                    label,
-                    value,
-                    toValidValue
-                })
+                ui: () => args
             },
             controller,
             controllerAs: 'ctrl'
@@ -62,12 +69,26 @@ export default class InputDialog {
             .finally(modalHide);
     }
 
+    /**
+     * Open input dialog to configure custom value.
+     *
+     * @param {String} title Dialog title.
+     * @param {String} label Input field label.
+     * @param {String} value Default value.
+     * @param {Function} [toValidValue] Validator function.
+     * @param {'text'|'number'} mode Input type.
+     * @returns {ng.IPromise<string>} User input.
+     */
+    input(title, label, value, toValidValue, mode = 'text') {
+        return this.dialogFabric({title, label, value, toValidValue, mode});
+    }
+
     /**
      * Open input dialog to configure cloned object name.
      *
      * @param {String} srcName Name of source object.
      * @param {Array.<String>} names List of already exist names.
-     * @returns {Promise.<String>} New name
+     * @returns {ng.IPromise<string>} New name.
      */
     clone(srcName, names) {
         const uniqueName = (value) => {
@@ -85,4 +106,14 @@ export default class InputDialog {
 
         return this.input('Clone', 'New name', uniqueName(srcName), uniqueName);
     }
+
+    /**
+     * Open input dialog to configure custom number value.
+     *
+     * @param {Object} options Object with settings for rendering number input.
+     * @returns {Promise.<String>} User input.
+     */
+    number(options) {
+        return this.dialogFabric({mode: 'number', ...options});
+    }
 }
diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
index 5c6e1736e4dd9..95cb059ba6ff9 100644
--- a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
@@ -16,24 +16,44 @@
 
 include /app/helpers/jade/mixins
 
-.modal.modal--ignite.theme--ignite(tabindex='-1' role='dialog')
+.modal.modal--ignite.theme--ignite.input-dialog(tabindex='-1' role='dialog')
     .modal-dialog
         form.modal-content(name='ctrl.form' novalidate)
             .modal-header
-                h4.modal-title 
+                h4.modal-title
                     i.fa.fa-clone
-                    span {{ ctrl.title }}
+                    span {{ ctrl.options.title }}
                 button.close(type='button' aria-label='Close' ng-click='$hide()')
                      svg(ignite-icon="cross")
 
-            .modal-body
-                .row
+            .modal-body(ng-switch='ctrl.options.mode')
+                .row(ng-switch-when='text')
                     .col-100
-                        +ignite-form-field-text('{{ ctrl.label }}', 'ctrl.value', '"input-field"', false, true, 'Enter value')(
-                            data-ignite-form-field-input-autofocus='true'
+                        +form-field__text({
+                            label: '{{ ctrl.options.label }}',
+                            model: 'ctrl.options.value',
+                            name: '"inputDialogField"',
+                            required: true,
+                            placeholder: 'Enter value'
+                        })(
+                            ignite-form-field-input-autofocus='true'
                             ignite-on-enter='form.$valid && ctrl.confirm()'
                         )
+                .row(ng-switch-when='number')
+                    .col-100
+                        +form-field__number({
+                            label: '{{ ctrl.options.label }}',
+                            model: 'ctrl.options.value',
+                            name: '"number"',
+                            placeholder: '{{ ctrl.options.placeholder }}',
+                            min: '{{ ctrl.options.min }}',
+                            max: '{{ ctrl.options.max  }}',
+                            tip: '{{ ctrl.options.tip  }}',
+                            postfix: '{{ ctrl.options.postfix }}',
+                            required: true
+                        })
 
             .modal-footer
-                button#copy-btn-cancel.btn-ignite.btn-ignite--link-success(ng-click='$hide()') Cancel
-                button#copy-btn-confirm.btn-ignite.btn-ignite--success(ng-disabled='ctrl.form.$invalid' ng-click='ctrl.confirm()') Confirm
+                div
+                    button#copy-btn-cancel.btn-ignite.btn-ignite--link-success(ng-click='$hide()') Cancel
+                    button#copy-btn-confirm.btn-ignite.btn-ignite--success(ng-disabled='ctrl.form.$invalid' ng-click='ctrl.confirm()') Confirm
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.style.scss b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.style.scss
index 5df29cc540b4e..5735be366717d 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.style.scss
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.style.scss
@@ -19,7 +19,6 @@
     $index-column-width: 46px;
     $remove-column-width: 36px;
 
-    margin-left: 10px;
     margin-right: $remove-column-width;
     transition: 0.2s opacity;
 
@@ -27,7 +26,7 @@
         margin-left: $index-column-width;
     }
 
-    .ignite-form-field__label {
+    .form-field__label {
         padding-left: 0;
         padding-right: 0;
         float: none;
@@ -41,10 +40,15 @@
     }
 
     &+list-editable {
+        .form-field__label,
         .ignite-form-field__label {
             display: none;
         }
 
+        .form-field:not(.form-field__checkbox) {
+            margin-left: -11px;
+        }
+
         .le-row-item-view:nth-last-child(2) {
             display: none;
         }
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug
index d060f45802e40..f1aff2e8e2fcf 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/cols.template.pug
@@ -19,7 +19,7 @@
     ng-disabled='$ctrl.ngDisabled'
 )
     .list-editable-cols__header-cell(ng-repeat='col in ::$ctrl.colDefs' ng-class='::col.cellClass')
-        span.ignite-form-field__label
+        span.form-field__label
             | {{ ::col.name }}
             svg(
                 ng-if='::col.tip'
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/index.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/index.js
index 93df2532191b6..1e3cfbf3ced2d 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/index.js
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-cols/index.js
@@ -17,8 +17,8 @@
 
 import angular from 'angular';
 
-import cols from './cols.directive.js';
-import row from './row.directive.js';
+import cols from './cols.directive';
+import row from './row.directive';
 
 export default angular
     .module('list-editable-cols', [])
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-one-way/index.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-one-way/index.js
index 3c49003bc42ef..652ac0ae04c13 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-one-way/index.js
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-one-way/index.js
@@ -21,4 +21,4 @@ import directive from './directive';
 
 export default angular
     .module('ignite-console.list-editable.one-way', [])
-    .directive(directive.name, directive);
+    .directive('listEditableOneWay', directive);
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/index.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/index.js
index 642e84aa55d0f..18e26db524049 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/index.js
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-save-on-changes/index.js
@@ -19,6 +19,6 @@ import angular from 'angular';
 import {ngModel, listEditableTransclude} from './directives';
 
 export default angular
-.module('list-editable.save-on-changes', [])
-.directive(ngModel.name, ngModel)
-.directive(listEditableTransclude.name, listEditableTransclude);
+    .module('list-editable.save-on-changes', [])
+    .directive('ngModel', ngModel)
+    .directive('listEditableTransclude', listEditableTransclude);
diff --git a/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/index.js b/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/index.js
index 39d5b7337a59b..6d681a4af5487 100644
--- a/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/index.js
+++ b/modules/web-console/frontend/app/components/list-editable/components/list-editable-transclude/index.js
@@ -20,4 +20,4 @@ import {listEditableTransclude} from './directive';
 
 export default angular
     .module('list-editable.transclude', [])
-    .directive(listEditableTransclude.name, listEditableTransclude);
+    .directive('listEditableTransclude', listEditableTransclude);
diff --git a/modules/web-console/frontend/app/components/list-editable/style.scss b/modules/web-console/frontend/app/components/list-editable/style.scss
index 4d6052870aff8..0db6f7a13b142 100644
--- a/modules/web-console/frontend/app/components/list-editable/style.scss
+++ b/modules/web-console/frontend/app/components/list-editable/style.scss
@@ -53,7 +53,7 @@ list-editable {
     }
 
     .le-body {
-        box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.2);
+        box-shadow: 0 1px 0 0 rgba(0, 0, 0, 0.2);
     }
 
     .le-row-sort {
@@ -113,10 +113,6 @@ list-editable {
                 min-height: 36px;
                 align-items: center;
             }
-
-            &-edit {
-                margin-left: -11px;
-            }
         }
 
         &--editable {
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/controller.js
index 53a552188776d..f8b2797359f52 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/controller.js
@@ -210,7 +210,7 @@ export default class IgniteListOfRegisteredUsersCtrl {
             const sdt = $ctrl.params.startDate;
             const edt = $ctrl.params.endDate;
 
-            $ctrl.gridOptions.exporterCsvFilename = `web_console_users_${dtFilter(sdt, 'yyyy_MM')}.csv`;
+            $ctrl.exporterCsvFilename = `web_console_users_${dtFilter(sdt, 'yyyy_MM')}.csv`;
 
             const startDate = Date.UTC(sdt.getFullYear(), sdt.getMonth(), 1);
             const endDate = Date.UTC(edt.getFullYear(), edt.getMonth() + 1, 1);
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/style.scss b/modules/web-console/frontend/app/components/list-of-registered-users/style.scss
index 5a0c71310de34..359a19dceb479 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/style.scss
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/style.scss
@@ -29,17 +29,7 @@ ignite-list-of-registered-users {
         }
     }
 
-    .ui-grid-settings--heading {
-        display: flex;
-    }
-
-    & > a {
-        display: inline-block;
-        margin: 10px;
-        margin-left: 0;
-
-        &.active {
-            font-weight: bold;
-        }
+    .form-field--inline:first-child {
+        margin-right: 20px;
     }
 }
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug b/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug
index 6656c90a01144..d32b64d082f12 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug
@@ -17,15 +17,15 @@
 include /app/helpers/jade/mixins
 
 ul.tabs.tabs--blue
-    li(role='presentation' ng-class='{ active: $ctrl.groupBy === "user" }') 
-        a(ng-click='$ctrl.groupByUser()') 
+    li(role='presentation' ng-class='{ active: $ctrl.groupBy === "user" }')
+        a(ng-click='$ctrl.groupByUser()')
             span Users
             span.badge.badge--blue(ng-hide='$ctrl.groupBy === "user"')
                 | {{ $ctrl.gridOptions.data.length }}
             span.badge.badge--blue(ng-show='$ctrl.groupBy === "user"')
                 | {{ $ctrl.filteredRows.length }}
     li(role='presentation' ng-class='{ active: $ctrl.groupBy === "company" }')
-        a(ng-click='$ctrl.groupByCompany()') 
+        a(ng-click='$ctrl.groupByCompany()')
             span Companies
             span.badge.badge--blue {{ $ctrl.companies.length }}
     li(role='presentation' ng-class='{ active: $ctrl.groupBy === "country" }')
@@ -34,32 +34,53 @@ ul.tabs.tabs--blue
             span.badge.badge--blue {{ $ctrl.countries.length }}
 
 .panel--ignite
-    .panel-heading.ui-grid-settings.ui-grid-ignite__panel
-        .panel-title
+    header.header-with-selector
+        div(ng-if='!$ctrl.selected.length')
+            span(ng-if='$ctrl.groupBy === "user"') List of registered users
+            span(ng-if='$ctrl.groupBy === "company"') List of registered companies
+            span(ng-if='$ctrl.groupBy === "country"') List of registered countries
+            grid-column-selector(grid-api='$ctrl.gridApi')
+
+        div(ng-if='$ctrl.selected.length')
+            grid-item-selected(grid-api='$ctrl.gridApi')
+
+        div
+            .form-field--inline
+                +form-field__text({
+                    label: 'Exclude:',
+                    model: '$ctrl.params.companiesExclude',
+                    name: '"exclude"',
+                    placeholder: 'Exclude by company name...'
+                })
+
+            .form-field--inline
+                +form-field__datepicker({
+                    label: 'Period: from',
+                    model: '$ctrl.params.startDate',
+                    name: '"startDate"',
+                    maxdate: '$ctrl.params.endDate'
+                })
+            .form-field--inline
+                +form-field__datepicker({
+                    label: 'to',
+                    model: '$ctrl.params.endDate',
+                    name: '"endDate"',
+                    mindate: '$ctrl.params.startDate'
+                })
+
+            grid-export(file-name='$ctrl.exporterCsvFilename' grid-api='$ctrl.gridApi')
+
             +ignite-form-field-bsdropdown({
                 label: 'Actions',
                 model: '$ctrl.action',
                 name: 'action',
                 disabled: '!$ctrl.selected.length',
-                required: false,
                 options: '$ctrl.actionOptions'
             })
-            grid-export(grid-api='$ctrl.gridApi')
-            form.ui-grid-settings-dateperiod(name=form novalidate)
-                -var form = 'admin'
-                +ignite-form-field-datepicker('Period: from', '$ctrl.params.startDate', '"startDate"', null, '$ctrl.params.endDate')
-                +ignite-form-field-datepicker('to', '$ctrl.params.endDate', '"endDate"', '$ctrl.params.startDate', null)
-            form.ui-grid-settings-filter
-                -var form = 'admin'
-                +ignite-form-field-text('Exclude:', '$ctrl.params.companiesExclude', '"exclude"', false, false, 'Exclude by company name...')
-
-            .ui-grid-settings--heading(ng-hide='$ctrl.selected.length')
-                span(ng-if='$ctrl.groupBy === "user"') List of registered users
-                span(ng-if='$ctrl.groupBy === "company"') List of registered companies
-                span(ng-if='$ctrl.groupBy === "country"') List of registered countries
-                grid-column-selector(grid-api='$ctrl.gridApi')
-            .panel-selected(ng-show='$ctrl.selected.length')
-                grid-item-selected(grid-api='$ctrl.gridApi')
 
-    .panel-collapse
+    .ignite-grid-table
         .grid.ui-grid--ignite.ui-grid-disabled-group-selection(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-exporter ui-grid-pinning ui-grid-grouping ui-grid-hovering)
+
+    grid-no-data(grid-api='$ctrl.gridApi')
+        grid-no-data-filtered
+           | Nothing to display. Check your filters.
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/affinity.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/affinity.pug
index ce2cad542eb66..68ee4d5e21719 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/affinity.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/affinity.pug
@@ -19,68 +19,123 @@ include /app/helpers/jade/mixins
 -var form = 'affinity'
 -var model = '$ctrl.clonedCache'
 -var affModel = model + '.affinity'
--var affMapModel = model + '.affinityMapper'
 -var rendezvousAff = affModel + '.kind === "Rendezvous"'
 -var fairAff = affModel + '.kind === "Fair"'
 -var customAff = affModel + '.kind === "Custom"'
--var customAffMapper = affMapModel + '.kind === "Custom"'
 -var rendPartitionsRequired = rendezvousAff + ' && ' + affModel + '.Rendezvous.affinityBackupFilter'
 -var fairPartitionsRequired = fairAff + ' && ' + affModel + '.Fair.affinityBackupFilter'
 
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Affinity Collocation
     panel-description
-        | Collocate data with data to improve performance and scalability of your application. 
+        | Collocate data with data to improve performance and scalability of your application.
         a.link-success(href="https://apacheignite.readme.io/docs/affinity-collocation" target="_blank") More info
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60(ng-if='$ctrl.available(["1.0.0", "2.0.0"])')
-                +dropdown('Function:', `${affModel}.kind`, '"AffinityKind"', 'true', 'Default', 'affinityFunction',
-                    'Key topology resolver to provide mapping from keys to nodes<br/>\
-                    <ul>\
-                        <li>Rendezvous - Based on Highest Random Weight algorithm</li>\
-                        <li>Fair - Tries to ensure that all nodes get equal number of partitions with minimum amount of reassignments between existing nodes</li>\
-                        <li>Custom - Custom implementation of key affinity fynction</li>\
-                        <li>Default - By default rendezvous affinity function  with 1024 partitions is used</li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Function:',
+                    model: `${affModel}.kind`,
+                    name: '"AffinityKind"',
+                    placeholder: 'Default',
+                    options: 'affinityFunction',
+                    tip: 'Key topology resolver to provide mapping from keys to nodes<br/>\
+                                        <ul>\
+                                            <li>Rendezvous - Based on Highest Random Weight algorithm</li>\
+                                            <li>Fair - Tries to ensure that all nodes get equal number of partitions with minimum amount of reassignments between existing nodes</li>\
+                                            <li>Custom - Custom implementation of key affinity fynction</li>\
+                                            <li>Default - By default rendezvous affinity function  with 1024 partitions is used</li>\
+                                        </ul>'
+                })
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.0.0")')
-                +dropdown('Function:', `${affModel}.kind`, '"AffinityKind"', 'true', 'Default', 'affinityFunction',
-                    'Key topology resolver to provide mapping from keys to nodes<br/>\
-                    <ul>\
-                        <li>Rendezvous - Based on Highest Random Weight algorithm</li>\
-                        <li>Custom - Custom implementation of key affinity fynction</li>\
-                        <li>Default - By default rendezvous affinity function  with 1024 partitions is used</li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Function:',
+                    model: `${affModel}.kind`,
+                    name: '"AffinityKind"',
+                    placeholder: 'Default',
+                    options: 'affinityFunction',
+                    tip: 'Key topology resolver to provide mapping from keys to nodes<br/>\
+                                       <ul>\
+                                           <li>Rendezvous - Based on Highest Random Weight algorithm</li>\
+                                           <li>Custom - Custom implementation of key affinity fynction</li>\
+                                           <li>Default - By default rendezvous affinity function  with 1024 partitions is used</li>\
+                                       </ul>'
+                })
             .pc-form-group
                 .pc-form-grid-row(ng-if=rendezvousAff)
                     .pc-form-grid-col-60
-                        +number-required('Partitions', `${affModel}.Rendezvous.partitions`, '"RendPartitions"', 'true', rendPartitionsRequired, '1024', '1', 'Number of partitions')
+                        +form-field__number({
+                            label: 'Partitions',
+                            model: `${affModel}.Rendezvous.partitions`,
+                            name: '"RendPartitions"',
+                            required: rendPartitionsRequired,
+                            placeholder: '1024',
+                            min: '1',
+                            tip: 'Number of partitions'
+                        })
                     .pc-form-grid-col-60
-                        +java-class('Backup filter', `${affModel}.Rendezvous.affinityBackupFilter`, '"RendAffinityBackupFilter"', 'true', 'false',
-                            'Backups will be selected from all nodes that pass this filter')
+                        +form-field__java-class({
+                            label: 'Backup filter',
+                            model: `${affModel}.Rendezvous.affinityBackupFilter`,
+                            name: '"RendAffinityBackupFilter"',
+                            tip: 'Backups will be selected from all nodes that pass this filter'
+                        })
                     .pc-form-grid-col-60
-                        +checkbox('Exclude neighbors', `${affModel}.Rendezvous.excludeNeighbors`, '"RendExcludeNeighbors"',
-                            'Exclude same - host - neighbors from being backups of each other and specified number of backups')
+                        +form-field__checkbox({
+                            label: 'Exclude neighbors',
+                            model: `${affModel}.Rendezvous.excludeNeighbors`,
+                            name: '"RendExcludeNeighbors"',
+                            tip: 'Exclude same - host - neighbors from being backups of each other and specified number of backups'
+                        })
                 .pc-form-grid-row(ng-if=fairAff)
                     .pc-form-grid-col-60
-                        +number-required('Partitions', `${affModel}.Fair.partitions`, '"FairPartitions"', 'true', fairPartitionsRequired, '256', '1', 'Number of partitions')
+                        +form-field__number({
+                            label: 'Partitions',
+                            model: `${affModel}.Fair.partitions`,
+                            name: '"FairPartitions"',
+                            required: fairPartitionsRequired,
+                            placeholder: '256',
+                            min: '1',
+                            tip: 'Number of partitions'
+                        })
                     .pc-form-grid-col-60
-                        +java-class('Backup filter', `${affModel}.Fair.affinityBackupFilter`, '"FairAffinityBackupFilter"', 'true', 'false',
-                            'Backups will be selected from all nodes that pass this filter')
+                        +form-field__java-class({
+                            label: 'Backup filter',
+                            model: `${affModel}.Fair.affinityBackupFilter`,
+                            name: '"FairAffinityBackupFilter"',
+                            tip: 'Backups will be selected from all nodes that pass this filter'
+                        })
                     .pc-form-grid-col-60
-                        +checkbox('Exclude neighbors', `${affModel}.Fair.excludeNeighbors`, '"FairExcludeNeighbors"',
-                            'Exclude same - host - neighbors from being backups of each other and specified number of backups')
+                        +form-field__checkbox({
+                            label: 'Exclude neighbors',
+                            model: `${affModel}.Fair.excludeNeighbors`,
+                            name: '"FairExcludeNeighbors"',
+                            tip: 'Exclude same - host - neighbors from being backups of each other and specified number of backups'
+                        })
                 .pc-form-grid-row(ng-if=customAff)
                     .pc-form-grid-col-60
-                        +java-class('Class name:', `${affModel}.Custom.className`, '"AffCustomClassName"', 'true', customAff,
-                            'Custom key affinity function implementation class name')
+                        +form-field__java-class({
+                            label: 'Class name:',
+                            model: `${affModel}.Custom.className`,
+                            name: '"AffCustomClassName"',
+                            required: customAff,
+                            tip: 'Custom key affinity function implementation class name'
+                        })
             .pc-form-grid-col-60
-                +java-class('Mapper:', model + '.affinityMapper', '"AffMapCustomClassName"', 'true', 'false',
-                    'Provide custom affinity key for any given key')
+                +form-field__java-class({
+                    label: 'Mapper:',
+                    model: `${model}.affinityMapper`,
+                    name: '"AffMapCustomClassName"',
+                    tip: 'Provide custom affinity key for any given key'
+                })
 
             //- Since ignite 2.0
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.0.0")')
-                +java-class('Topology validator:', model + '.topologyValidator', '"topologyValidator"', 'true', 'false')
+                +form-field__java-class({
+                    label: 'Topology validator:',
+                    model: `${model}.topologyValidator`,
+                    name: '"topologyValidator"'
+                })
 
         .pca-form-column-6
             +preview-xml-java(model, 'cacheAffinity')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/concurrency.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/concurrency.pug
index d99f894ddf1a0..bb355f01b3eb3 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/concurrency.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/concurrency.pug
@@ -26,39 +26,61 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-30
-                +number('Max async operations:', `${model}.maxConcurrentAsyncOperations`, '"maxConcurrentAsyncOperations"', 'true', '500', '0',
-                    'Maximum number of allowed concurrent asynchronous operations<br/>\
-                    If <b>0</b> then number of concurrent asynchronous operations is unlimited')
+                +form-field__number({
+                    label: 'Max async operations:',
+                    model: `${model}.maxConcurrentAsyncOperations`,
+                    name: '"maxConcurrentAsyncOperations"',
+                    placeholder: '500',
+                    min: '0',
+                    tip: 'Maximum number of allowed concurrent asynchronous operations<br/>\
+                         If <b>0</b> then number of concurrent asynchronous operations is unlimited'
+                })
             .pc-form-grid-col-30
-                +number('Default lock timeout:', `${model}.defaultLockTimeout`, '"defaultLockTimeout"', 'true', '0', '0',
-                    'Default lock acquisition timeout in milliseconds<br/>\
-                    If <b>0</b> then lock acquisition will never timeout')
+                +form-field__number({
+                    label: 'Default lock timeout:',
+                    model: `${model}.defaultLockTimeout`,
+                    name: '"defaultLockTimeout"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Default lock acquisition timeout in milliseconds<br/>\
+                         If <b>0</b> then lock acquisition will never timeout'
+                })
 
             //- Removed in ignite 2.0
             .pc-form-grid-col-60(ng-if='$ctrl.available(["1.0.0", "2.0.0"])' ng-hide=`${model}.atomicityMode === 'TRANSACTIONAL'`)
-                +dropdown('Entry versioning:', `${model}.atomicWriteOrderMode`, '"atomicWriteOrderMode"', 'true', 'Choose versioning',
-                    '[\
-                        {value: "CLOCK", label: "CLOCK"},\
-                        {value: "PRIMARY", label: "PRIMARY"}\
-                    ]',
-                    'Write ordering mode determines which node assigns the write version, sender or the primary node\
-                    <ul>\
-                        <li>CLOCK - in this mode write versions are assigned on a sender node which generally leads to better performance</li>\
-                        <li>PRIMARY - in this mode version is assigned only on primary node. This means that sender will only send write request to primary node, which in turn will assign write version and forward it to backups</li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Entry versioning:',
+                    model: `${model}.atomicWriteOrderMode`,
+                    name: '"atomicWriteOrderMode"',
+                    placeholder: 'Choose versioning',
+                    options: '[\
+                                            {value: "CLOCK", label: "CLOCK"},\
+                                            {value: "PRIMARY", label: "PRIMARY"}\
+                                        ]',
+                    tip: 'Write ordering mode determines which node assigns the write version, sender or the primary node\
+                                        <ul>\
+                                            <li>CLOCK - in this mode write versions are assigned on a sender node which generally leads to better performance</li>\
+                                            <li>PRIMARY - in this mode version is assigned only on primary node. This means that sender will only send write request to primary node, which in turn will assign write version and forward it to backups</li>\
+                                        </ul>'
+                })
 
             .pc-form-grid-col-60
-                +dropdown('Write synchronization mode:', `${model}.writeSynchronizationMode`, '"writeSynchronizationMode"', 'true', 'PRIMARY_SYNC',
-                    '[\
-                        {value: "FULL_SYNC", label: "FULL_SYNC"},\
-                        {value: "FULL_ASYNC", label: "FULL_ASYNC"},\
-                        {value: "PRIMARY_SYNC", label: "PRIMARY_SYNC"}\
-                    ]',
-                    'Write synchronization mode\
-                    <ul>\
-                        <li>FULL_SYNC - Ignite will wait for write or commit replies from all nodes</li>\
-                        <li>FULL_ASYNC - Ignite will not wait for write or commit responses from participating nodes</li>\
-                        <li>PRIMARY_SYNC - Makes sense for PARTITIONED mode. Ignite will wait for write or commit to complete on primary node</li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Write synchronization mode:',
+                    model: `${model}.writeSynchronizationMode`,
+                    name: '"writeSynchronizationMode"',
+                    placeholder: 'PRIMARY_SYNC',
+                    options: '[\
+                                            {value: "FULL_SYNC", label: "FULL_SYNC"},\
+                                            {value: "FULL_ASYNC", label: "FULL_ASYNC"},\
+                                            {value: "PRIMARY_SYNC", label: "PRIMARY_SYNC"}\
+                                        ]',
+                    tip: 'Write synchronization mode\
+                                        <ul>\
+                                            <li>FULL_SYNC - Ignite will wait for write or commit replies from all nodes</li>\
+                                            <li>FULL_ASYNC - Ignite will not wait for write or commit responses from participating nodes</li>\
+                                            <li>PRIMARY_SYNC - Makes sense for PARTITIONED mode. Ignite will wait for write or commit to complete on primary node</li>\
+                                        </ul>'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'cacheConcurrency')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/general.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/general.pug
index 29977bee093b2..955239693354a 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/general.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/general.pug
@@ -22,12 +22,12 @@ include /app/helpers/jade/mixins
 panel-collapsible(opened=`::true` ng-form=form)
     panel-title General
     panel-description
-        | Common cache configuration. 
+        | Common cache configuration.
         a.link-success(href="https://apacheignite.readme.io/docs/data-grid" target="_blank") More info
     panel-content.pca-form-row
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +sane-ignite-form-field-text({
+                +form-field__text({
                     label: 'Name:',
                     model: `${model}.name`,
                     name: '"cacheName"',
@@ -38,9 +38,9 @@ panel-collapsible(opened=`::true` ng-form=form)
                     ignite-unique-property='name'
                     ignite-unique-skip=`["_id", ${model}]`
                 )
-                    +unique-feedback(`${model}.name`, 'Cache name should be unique')
+                    +form-field__error({ error: 'igniteUnique', message: 'Cache name should be unique' })
             .pc-form-grid-col-60
-                +sane-ignite-form-field-dropdown({
+                +form-field__dropdown({
                     label: 'Domain models:',
                     model: `${model}.domains`,
                     name: '"domains"',
@@ -51,63 +51,102 @@ panel-collapsible(opened=`::true` ng-form=form)
                     tip: 'Select domain models to describe types in cache'
                 })
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.1.0")')
-                +text('Group:', `${model}.groupName`, '"groupName"', 'false', 'Input group name',
-                    'Cache group name.<br/>\
-                    Caches with the same group name share single underlying "physical" cache (partition set), but are logically isolated.')
+                +form-field__text({
+                    label: 'Group:',
+                    model: `${model}.groupName`,
+                    name: '"groupName"',
+                    placeholder: 'Input group name',
+                    tip: 'Cache group name.<br/>\
+                          Caches with the same group name share single underlying "physical" cache (partition set), but are logically isolated.'
+                })
             .pc-form-grid-col-30
-                +cacheMode('Mode:', `${model}.cacheMode`, '"cacheMode"', 'PARTITIONED')
+                +form-field__cache-modes({
+                    label: 'Mode:',
+                    model: `${model}.cacheMode`,
+                    name: '"cacheMode"',
+                    placeholder: 'PARTITIONED'
+                })
 
             .pc-form-grid-col-30
-                +dropdown('Atomicity:', `${model}.atomicityMode`, '"atomicityMode"', 'true', 'ATOMIC',
-                    '[\
-                        {value: "ATOMIC", label: "ATOMIC"},\
-                        {value: "TRANSACTIONAL", label: "TRANSACTIONAL"}\
-                    ]',
-                    'Atomicity:\
-                    <ul>\
-                        <li>ATOMIC - in this mode distributed transactions and distributed locking are not supported</li>\
-                        <li>TRANSACTIONAL - in this mode specified fully ACID-compliant transactional cache behavior</li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Atomicity:',
+                    model: `${model}.atomicityMode`,
+                    name: '"atomicityMode"',
+                    placeholder: 'ATOMIC',
+                    options: '[\
+                                            {value: "ATOMIC", label: "ATOMIC"},\
+                                            {value: "TRANSACTIONAL", label: "TRANSACTIONAL"}\
+                                        ]',
+                    tip: 'Atomicity:\
+                                        <ul>\
+                                            <li>ATOMIC - in this mode distributed transactions and distributed locking are not supported</li>\
+                                            <li>TRANSACTIONAL - in this mode specified fully ACID-compliant transactional cache behavior</li>\
+                                        </ul>'
+                })
             .pc-form-grid-col-30(ng-is=`${model}.cacheMode === 'PARTITIONED'`)
-                +number('Backups:', `${model}.backups`, '"backups"', 'true', '0', '0', 'Number of nodes used to back up single partition for partitioned cache')
+                +form-field__number({
+                    label: 'Backups:',
+                    model: `${model}.backups`,
+                    name: '"checkpointS3ClientExecutionTimeout"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Number of nodes used to back up single partition for partitioned cache'
+                })
             //- Since ignite 2.0
             .pc-form-grid-col-30(ng-if='$ctrl.available("2.0.0")')
-                +dropdown('Partition loss policy:', `${model}.partitionLossPolicy`, '"partitionLossPolicy"', 'true', 'IGNORE',
-                '[\
-                    {value: "READ_ONLY_SAFE", label: "READ_ONLY_SAFE"},\
-                    {value: "READ_ONLY_ALL", label: "READ_ONLY_ALL"},\
-                    {value: "READ_WRITE_SAFE", label: "READ_WRITE_SAFE"},\
-                    {value: "READ_WRITE_ALL", label: "READ_WRITE_ALL"},\
-                    {value: "IGNORE", label: "IGNORE"}\
-                ]',
-                'Partition loss policies:\
-                <ul>\
-                    <li>READ_ONLY_SAFE - in this mode all writes to the cache will be failed with an exception,\
-                        reads will only be allowed for keys in  non-lost partitions.\
-                        Reads from lost partitions will be failed with an exception.</li>\
-                    <li>READ_ONLY_ALL - in this mode all writes to the cache will be failed with an exception.\
-                        All reads will proceed as if all partitions were in a consistent state.\
-                        The result of reading from a lost partition is undefined and may be different on different nodes in the cluster.</li>\
-                    <li>READ_WRITE_SAFE - in this mode all reads and writes will be allowed for keys in valid partitions.\
-                        All reads and writes for keys in lost partitions will be failed with an exception.</li>\
-                    <li>READ_WRITE_ALL - in this mode all reads and writes will proceed as if all partitions were in a consistent state.\
-                        The result of reading from a lost partition is undefined and may be different on different nodes in the cluster.</li>\
-                    <li>IGNORE - in this mode if partition is lost, reset it state and do not clear intermediate data.\
-                        The result of reading from a previously lost and not cleared partition is undefined and may be different\
-                        on different nodes in the cluster.</li>\
-                </ul>')
+                +form-field__dropdown({
+                    label:'Partition loss policy:',
+                    model: `${model}.partitionLossPolicy`,
+                    name: '"partitionLossPolicy"',
+                    placeholder: 'IGNORE',
+                    options: '[\
+                                        {value: "READ_ONLY_SAFE", label: "READ_ONLY_SAFE"},\
+                                        {value: "READ_ONLY_ALL", label: "READ_ONLY_ALL"},\
+                                        {value: "READ_WRITE_SAFE", label: "READ_WRITE_SAFE"},\
+                                        {value: "READ_WRITE_ALL", label: "READ_WRITE_ALL"},\
+                                        {value: "IGNORE", label: "IGNORE"}\
+                                    ]',
+                    tip: 'Partition loss policies:\
+                                    <ul>\
+                                        <li>READ_ONLY_SAFE - in this mode all writes to the cache will be failed with an exception,\
+                                            reads will only be allowed for keys in  non-lost partitions.\
+                                            Reads from lost partitions will be failed with an exception.</li>\
+                                        <li>READ_ONLY_ALL - in this mode all writes to the cache will be failed with an exception.\
+                                            All reads will proceed as if all partitions were in a consistent state.\
+                                            The result of reading from a lost partition is undefined and may be different on different nodes in the cluster.</li>\
+                                        <li>READ_WRITE_SAFE - in this mode all reads and writes will be allowed for keys in valid partitions.\
+                                            All reads and writes for keys in lost partitions will be failed with an exception.</li>\
+                                        <li>READ_WRITE_ALL - in this mode all reads and writes will proceed as if all partitions were in a consistent state.\
+                                            The result of reading from a lost partition is undefined and may be different on different nodes in the cluster.</li>\
+                                        <li>IGNORE - in this mode if partition is lost, reset it state and do not clear intermediate data.\
+                                            The result of reading from a previously lost and not cleared partition is undefined and may be different\
+                                            on different nodes in the cluster.</li>\
+                                    </ul>'
+                })
             .pc-form-grid-col-60(ng-show=`${model}.cacheMode === 'PARTITIONED' && ${model}.backups`)
-                +checkbox('Read from backup', `${model}.readFromBackup`, '"readFromBackup"',
-                    'Flag indicating whether data can be read from backup<br/>\
-                    If not set then always get data from primary node (never from backup)')
+                +form-field__checkbox({
+                    label: 'Read from backup',
+                    model: `${model}.readFromBackup`,
+                    name: '"readFromBackup"',
+                    tip: 'Flag indicating whether data can be read from backup<br/>\
+                          If not set then always get data from primary node (never from backup)'
+                })
             .pc-form-grid-col-60
-                +checkbox('Copy on read', `${model}.copyOnRead`, '"copyOnRead"',
-                    'Flag indicating whether copy of the value stored in cache should be created for cache operation implying return value<br/>\
-                    Also if this flag is set copies are created for values passed to CacheInterceptor and to CacheEntryProcessor')
+                +form-field__checkbox({
+                    label: 'Copy on read',
+                    model: `${model}.copyOnRead`,
+                    name: '"copyOnRead"',
+                    tip: 'Flag indicating whether copy of the value stored in cache should be created for cache operation implying return value<br/>\
+                          Also if this flag is set copies are created for values passed to CacheInterceptor and to CacheEntryProcessor'
+                })
             .pc-form-grid-col-60(ng-show=`${model}.cacheMode === 'PARTITIONED' && ${model}.atomicityMode === 'TRANSACTIONAL'`)
-                +checkbox('Invalidate near cache', `${model}.isInvalidate`, '"isInvalidate"',
-                    'Invalidation flag for near cache entries in transaction<br/>\
-                    If set then values will be invalidated (nullified) upon commit in near cache')
+                +form-field__checkbox({
+                    label: 'Invalidate near cache',
+                    model: `${model}.isInvalidate`,
+                    name: '"isInvalidate"',
+                    tip: 'Invalidation flag for near cache entries in transaction<br/>\
+                          If set then values will be invalidated (nullified) upon commit in near cache'
+                })
 
         .pca-form-column-6
             +preview-xml-java(model, 'cacheGeneral')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/memory.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/memory.pug
index 10eb488aa08cd..571e8ec65d1ba 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/memory.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/memory.pug
@@ -22,7 +22,7 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Memory
     panel-description
-        | Cache memory settings. 
+        | Cache memory settings.
         a.link-success(
             href="https://apacheignite.readme.io/v1.9/docs/off-heap-memory"
             target="_blank"
@@ -37,21 +37,35 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             //- Since ignite 2.0
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.0.0")')
-                +checkbox('Onheap cache enabled', model + '.onheapCacheEnabled', '"OnheapCacheEnabled"', 'Checks if the on-heap cache is enabled for the off-heap based page memory')
-
+                +form-field__checkbox({
+                    label: 'Onheap cache enabled',
+                    model: model + '.onheapCacheEnabled',
+                    name: '"OnheapCacheEnabled"',
+                    tip: 'Checks if the on-heap cache is enabled for the off-heap based page memory'
+                })
             //- Since ignite 2.0 deprecated in ignite 2.3
             .pc-form-grid-col-60(ng-if='$ctrl.available(["2.0.0", "2.3.0"])')
-                +text('Memory policy name:', model + '.memoryPolicyName', '"MemoryPolicyName"', 'false', 'default',
-                    'Name of memory policy configuration for this cache')
+                +form-field__text({
+                    label: 'Memory policy name:',
+                    model: `${model}.memoryPolicyName`,
+                    name: '"MemoryPolicyName"',
+                    placeholder: 'default',
+                    tip: 'Name of memory policy configuration for this cache'
+                })
 
             //- Since ignite 2.3
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.3.0")')
-                +text('Data region name:', model + '.dataRegionName', '"DataRegionName"', 'false', 'default',
-                    'Name of data region configuration for this cache')
+                +form-field__text({
+                    label: 'Data region name:',
+                    model: `${model}.dataRegionName`,
+                    name: '"DataRegionName"',
+                    placeholder: 'default',
+                    tip: 'Name of data region configuration for this cache'
+                })
 
             //- Removed in ignite 2.0
             .pc-form-grid-col-60(ng-if-start='$ctrl.available(["1.0.0", "2.0.0"])')
-                +sane-ignite-form-field-dropdown({
+                +form-field__dropdown({
                     label: 'Mode:',
                     model: `${model}.memoryMode`,
                     name: '"memoryMode"',
@@ -88,9 +102,9 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     ui-validate-watch=`"${model}.domains.length"`
                     ng-model-options='{allowInvalid: true}'
                 )
-                    +form-field-feedback(null, 'offheapAndDomains', 'Query indexing could not be enabled while values are stored off-heap')
+                    +form-field__error({ error: 'offheapAndDomains', message: 'Query indexing could not be enabled while values are stored off-heap' })
             .pc-form-grid-col-60(ng-if=`${model}.memoryMode !== 'OFFHEAP_VALUES'`)
-                +sane-ignite-form-field-dropdown({
+                +form-field__dropdown({
                     label: 'Off-heap memory:',
                     model: `${model}.offHeapMode`,
                     name: '"offHeapMode"',
@@ -111,12 +125,12 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     ui-validate-watch=`'${model}.memoryMode'`
                     ng-model-options='{allowInvalid: true}'
                 )
-                    +form-field-feedback(null, 'offheapDisabled', 'Off-heap storage can\'t be disabled when memory mode is OFFHEAP_TIERED')
+                    +form-field__error({ error: 'offheapDisabled', message: 'Off-heap storage can\'t be disabled when memory mode is OFFHEAP_TIERED' })
             .pc-form-grid-col-60(
                 ng-if=`${model}.offHeapMode === 1 && ${model}.memoryMode !== 'OFFHEAP_VALUES'`
                 ng-if-end
             )
-                pc-form-field-size(
+                form-field-size(
                     label='Off-heap memory max size:'
                     ng-model=`${model}.offHeapMaxMemory`
                     name='offHeapMaxMemory'
@@ -127,32 +141,53 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     size-type='bytes'
                     required='true'
                 )
-            +evictionPolicy(`${model}.evictionPolicy`, '"evictionPolicy"', 'true',
-                `$ctrl.Caches.evictionPolicy.required(${model})`,
-                'Optional cache eviction policy<br/>\
-                Must be set for entries to be evicted from on-heap to off-heap or swap\
-                <ul>\
-                    <li>Least Recently Used(LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
-                    <li>First In First Out (FIFO) - Eviction policy based on FIFO algorithm and supports batch eviction</li>\
-                    <li>SORTED - Eviction policy which will select the minimum cache entry for eviction</li>\
-                </ul>')
+
+            +form-field__eviction-policy({
+                model: `${model}.evictionPolicy`,
+                name: '"evictionPolicy"',
+                enabled: 'true',
+                required: `$ctrl.Caches.evictionPolicy.required(${model})`,
+                tip: 'Optional cache eviction policy<br/>\
+                      Must be set for entries to be evicted from on-heap to off-heap or swap\
+                      <ul>\
+                          <li>Least Recently Used(LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
+                          <li>First In First Out (FIFO) - Eviction policy based on FIFO algorithm and supports batch eviction</li>\
+                          <li>SORTED - Eviction policy which will select the minimum cache entry for eviction</li>\
+                      </ul>'
+            })
 
             //- Since ignite 2.0
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.0.0")')
-                +java-class('Eviction filter:', model + '.evictionFilter', '"EvictionFilter"', 'true', 'false', 'Eviction filter to specify which entries should not be evicted')
+                +form-field__java-class({
+                    label: 'Eviction filter:',
+                    model: `${model}.evictionFilter`,
+                    name: '"EvictionFilter"',
+                    tip: 'Eviction filter to specify which entries should not be evicted'
+                })
 
             //- Removed in ignite 2.0
             .pc-form-grid-col-60(ng-if-start='$ctrl.available(["1.0.0", "2.0.0"])')
-                +number('Start size:', `${model}.startSize`, '"startSize"', 'true', '1500000', '0',
-                    'In terms of size and capacity, Ignite internal cache map acts exactly like a normal Java HashMap: it has some initial capacity\
-                    (which is pretty small by default), which doubles as data arrives. The process of internal cache map resizing is CPU-intensive\
-                    and time-consuming, and if you load a huge dataset into cache (which is a normal use case), the map will have to resize a lot of times.\
-                    To avoid that, you can specify the initial cache map capacity, comparable to the expected size of your dataset.\
-                    This will save a lot of CPU resources during the load time, because the map would not have to resize.\
-                    For example, if you expect to load 10 million entries into cache, you can set this property to 10 000 000.\
-                    This will save you from cache internal map resizes.')
+                +form-field__number({
+                    label: 'Start size:',
+                    model: `${model}.startSize`,
+                    name: '"startSize"',
+                    placeholder: '1500000',
+                    min: '0',
+                    tip: 'In terms of size and capacity, Ignite internal cache map acts exactly like a normal Java HashMap: it has some initial capacity\
+                          (which is pretty small by default), which doubles as data arrives. The process of internal cache map resizing is CPU-intensive\
+                          and time-consuming, and if you load a huge dataset into cache (which is a normal use case), the map will have to resize a lot of times.\
+                          To avoid that, you can specify the initial cache map capacity, comparable to the expected size of your dataset.\
+                          This will save a lot of CPU resources during the load time, because the map would not have to resize.\
+                          For example, if you expect to load 10 million entries into cache, you can set this property to 10 000 000.\
+                          This will save you from cache internal map resizes.'
+                })
             .pc-form-grid-col-60(ng-if-end)
-                +checkbox('Swap enabled', `${model}.swapEnabled`, '"swapEnabled"', 'Flag indicating whether swap storage is enabled or not for this cache')
+                +form-field__checkbox({
+                    label: 'Swap enabled',
+                    model: `${model}.swapEnabled`,
+                    name: '"swapEnabled"',
+                    tip: 'Flag indicating whether swap storage is enabled or not for this cache'
+                })
 
         .pca-form-column-6
             +preview-xml-java(model, 'cacheMemory')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-client.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-client.pug
index 2b6705dbc9146..ed0e38a031ee9 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-client.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-client.pug
@@ -26,8 +26,8 @@ panel-collapsible(
 )
     panel-title Near cache on client node
     panel-description
-        | Near cache settings for client nodes. 
-        | Near cache is a small local cache that stores most recently or most frequently accessed data. 
+        | Near cache settings for client nodes.
+        | Near cache is a small local cache that stores most recently or most frequently accessed data.
         | Should be used in case when it is impossible to send computations to remote nodes.
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
@@ -35,16 +35,32 @@ panel-collapsible(
             -var enabled = `${nearCfg}.enabled`
 
             .pc-form-grid-col-60
-                +checkbox('Enabled', enabled, '"clientNearEnabled"', 'Flag indicating whether to configure near cache')
+                +form-field__checkbox({
+                    label: 'Enabled',
+                    model: enabled,
+                    name: '"clientNearEnabled"',
+                    tip: 'Flag indicating whether to configure near cache'
+                })
             .pc-form-grid-col-60
-                +number('Start size:', `${nearCfg}.nearStartSize`, '"clientNearStartSize"', enabled, '375000', '0',
-                    'Initial cache size for near cache which will be used to pre-create internal hash table after start')
-            +evictionPolicy(`${nearCfg}.nearEvictionPolicy`, '"clientNearCacheEvictionPolicy"', enabled, 'false',
-                'Near cache eviction policy\
-                <ul>\
-                    <li>Least Recently Used (LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
-                    <li>First In First Out (FIFO) - Eviction policy based on FIFO algorithm and supports batch eviction</li>\
-                    <li>SORTED - Eviction policy which will select the minimum cache entry for eviction</li>\
-                </ul>')
+                +form-field__number({
+                    label: 'Start size:',
+                    model: `${nearCfg}.nearStartSize`,
+                    name: '"clientNearStartSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '375000',
+                    min: '0',
+                    tip: 'Initial cache size for near cache which will be used to pre-create internal hash table after start'
+                })
+            +form-field__eviction-policy({
+                model: `${nearCfg}.nearEvictionPolicy`,
+                name: '"clientNearCacheEvictionPolicy"',
+                enabled: enabled,
+                tip: 'Near cache eviction policy\
+                     <ul>\
+                         <li>Least Recently Used (LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
+                         <li>First In First Out (FIFO) - Eviction policy based on FIFO algorithm and supports batch eviction</li>\
+                         <li>SORTED - Eviction policy which will select the minimum cache entry for eviction</li>\
+                     </ul>'
+            })
         .pca-form-column-6
             +preview-xml-java(model, 'cacheNearClient')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-server.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-server.pug
index 3a91fd2795c85..3d2043ac96474 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-server.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/near-cache-server.pug
@@ -26,9 +26,9 @@ panel-collapsible(
 )
     panel-title Near cache on server node
     panel-description
-        | Near cache settings. 
-        | Near cache is a small local cache that stores most recently or most frequently accessed data. 
-        | Should be used in case when it is impossible to send computations to remote nodes. 
+        | Near cache settings.
+        | Near cache is a small local cache that stores most recently or most frequently accessed data.
+        | Should be used in case when it is impossible to send computations to remote nodes.
         a.link-success(href="https://apacheignite.readme.io/docs/near-caches" target="_blank") More info
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
@@ -36,16 +36,32 @@ panel-collapsible(
             -var enabled = `${nearCfg}.enabled`
 
             .pc-form-grid-col-60
-                +checkbox('Enabled', enabled, '"nearCacheEnabled"', 'Flag indicating whether to configure near cache')
+                +form-field__checkbox({
+                    label: 'Enabled',
+                    model: enabled,
+                    name: '"nearCacheEnabled"',
+                    tip: 'Flag indicating whether to configure near cache'
+                })
             .pc-form-grid-col-60
-                +number('Start size:', `${nearCfg}.nearStartSize`, '"nearStartSize"', enabled, '375000', '0',
-                    'Initial cache size for near cache which will be used to pre-create internal hash table after start')
-            +evictionPolicy(`${model}.nearConfiguration.nearEvictionPolicy`, '"nearCacheEvictionPolicy"', enabled, 'false',
-                'Near cache eviction policy\
-                <ul>\
-                    <li>Least Recently Used (LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
-                    <li>First In First Out (FIFO) - Eviction policy based on FIFO algorithm and supports batch eviction</li>\
-                    <li>SORTED - Eviction policy which will select the minimum cache entry for eviction</li>\
-                </ul>')
+                +form-field__number({
+                    label: 'Start size:',
+                    model: `${nearCfg}.nearStartSize`,
+                    name: '"nearStartSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '375000',
+                    min: '0',
+                    tip: 'Initial cache size for near cache which will be used to pre-create internal hash table after start'
+                })
+            +form-field__eviction-policy({
+                model: `${model}.nearConfiguration.nearEvictionPolicy`,
+                name: '"nearCacheEvictionPolicy"',
+                enabled: enabled,
+                tip: 'Near cache eviction policy\
+                     <ul>\
+                         <li>Least Recently Used (LRU) - Eviction policy based on LRU algorithm and supports batch eviction</li>\
+                         <li>First In First Out (FIFO) - Eviction policy based on FIFO algorithm and supports batch eviction</li>\
+                         <li>SORTED - Eviction policy which will select the minimum cache entry for eviction</li>\
+                     </ul>'
+            })
         .pca-form-column-6
             +preview-xml-java(model, 'cacheNearServer')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/node-filter.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/node-filter.pug
index 11938f919596e..32afac1499892 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/node-filter.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/node-filter.pug
@@ -29,11 +29,18 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +dropdown('Node filter:', nodeFilterKind, '"nodeFilter"', 'true', 'Not set', '::$ctrl.Caches.nodeFilterKinds', 'Node filter variant')
+                +form-field__dropdown({
+                    label: 'Node filter:',
+                    model: nodeFilterKind,
+                    name: '"nodeFilter"',
+                    placeholder: 'Not set',
+                    options: '::$ctrl.Caches.nodeFilterKinds',
+                    tip: 'Node filter variant'
+                })
             .pc-form-grid-col-60(
                 ng-if=igfsFilter
             )
-                +sane-ignite-form-field-dropdown({
+                +form-field__dropdown({
                     label: 'IGFS:',
                     model: `${nodeFilter}.IGFS.igfs`,
                     name: '"igfsNodeFilter"',
@@ -45,9 +52,15 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                 })(
                     pc-is-in-collection='$ctrl.igfsIDs'
                 )
-                    +form-field-feedback(_, 'isInCollection', `Cluster doesn't have such an IGFS`)
+                    +form-field__error({ error: 'isInCollection',  message: `Cluster doesn't have such an IGFS` })
             .pc-form-grid-col-60(ng-show=customFilter)
-                +java-class('Class name:', `${nodeFilter}.Custom.className`, '"customNodeFilter"',
-                    'true', customFilter, 'Class name of custom node filter implementation', customFilter)
+                +form-field__java-class({
+                    label: 'Class name:',
+                    model: `${nodeFilter}.Custom.className`,
+                    name: '"customNodeFilter"',
+                    required: customFilter,
+                    tip: 'Class name of custom node filter implementation',
+                    validationActive: customFilter
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'cacheNodeFilter', 'igfss')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/query.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/query.pug
index 20869f5296728..8fedfc075d98a 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/query.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/query.pug
@@ -21,45 +21,68 @@ include /app/helpers/jade/mixins
 
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Queries & Indexing
-    panel-description 
-        | Cache queries settings. 
+    panel-description
+        | Cache queries settings.
         a.link-success(href="https://apacheignite-sql.readme.io/docs/select" target="_blank") More info
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +text('SQL schema name:', `${model}.sqlSchema`, '"sqlSchema"', 'false', 'Input schema name',
-                    'Specify any custom name to be used as SQL schema for current cache. This name will correspond to SQL ANSI-99 standard.\
-                    Nonquoted identifiers are not case sensitive. Quoted identifiers are case sensitive.\
-                    When SQL schema is not specified, quoted cache name should used instead.<br/>\
-                    For example:\
-                    <ul>\
-                        <li>\
-                            Query without schema names (quoted cache names will be used):\
-                            SELECT * FROM "PersonsCache".Person p INNER JOIN "OrganizationsCache".Organization o on p.org = o.id\
-                        </li>\
-                        <li>\
-                            The same query using schema names "Persons" and "Organizations":\
-                            SELECT * FROM Persons.Person p INNER JOIN Organizations.Organization o on p.org = o.id\
-                        </li>\
-                    </ul>')
+                +form-field__text({
+                    label: 'SQL schema name:',
+                    model: `${model}.sqlSchema`,
+                    name: '"sqlSchema"',
+                    placeholder: 'Input schema name',
+                    tip: 'Cache group name.<br/>\
+                          Caches with the same group name share single underlying "physical" cache (partition set), but are logically isolated.'
+                })
 
             //- Removed in ignite 2.0
             .pc-form-grid-col-60(ng-if='$ctrl.available(["1.0.0", "2.0.0"])')
-                +number('On-heap cache for off-heap indexes:', `${model}.sqlOnheapRowCacheSize`, '"sqlOnheapRowCacheSize"', 'true', '10240', '1',
-                    'Number of SQL rows which will be cached onheap to avoid deserialization on each SQL index access')
+                +form-field__number({
+                    label: 'On-heap cache for off-heap indexes:',
+                    model: `${model}.sqlOnheapRowCacheSize`,
+                    name: '"sqlOnheapRowCacheSize"',
+                    placeholder: '10240',
+                    min: '1',
+                    tip: 'Specify any custom name to be used as SQL schema for current cache. This name will correspond to SQL ANSI-99 standard.\
+                          Nonquoted identifiers are not case sensitive. Quoted identifiers are case sensitive.\
+                          When SQL schema is not specified, quoted cache name should used instead.<br/>\
+                          For example:\
+                          <ul>\
+                            <li>\
+                            Query without schema names (quoted cache names will be used):\
+                            SELECT * FROM "PersonsCache".Person p INNER JOIN "OrganizationsCache".Organization o on p.org = o.id\
+                            </li>\
+                            <li>\
+                                The same query using schema names "Persons" and "Organizations":\
+                                SELECT * FROM Persons.Person p INNER JOIN Organizations.Organization o on p.org = o.id\
+                            </li>\
+                          </ul>'
+                })
 
             //- Deprecated in ignite 2.1
             .pc-form-grid-col-60(ng-if='$ctrl.available(["1.0.0", "2.1.0"])')
-                +number('Long query timeout:', `${model}.longQueryWarningTimeout`, '"longQueryWarningTimeout"', 'true', '3000', '0',
-                    'Timeout in milliseconds after which long query warning will be printed')
+                +form-field__number({
+                    label: 'Long query timeout:',
+                    model: `${model}.longQueryWarningTimeout`,
+                    name: '"longQueryWarningTimeout"',
+                    placeholder: '3000',
+                    min: '0',
+                    tip: 'Timeout in milliseconds after which long query warning will be printed'
+                })
             .pc-form-grid-col-60
-                +number('History size:', `${model}.queryDetailMetricsSize`, '"queryDetailMetricsSize"', 'true', '0', '0',
-                    'Size of queries detail metrics that will be stored in memory for monitoring purposes')
+                +form-field__number({
+                    label: 'History size:',
+                    model: `${model}.queryDetailMetricsSize`,
+                    name: '"queryDetailMetricsSize"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Size of queries detail metrics that will be stored in memory for monitoring purposes'
+                })
             .pc-form-grid-col-60
                 mixin caches-query-list-sql-functions()
                     .ignite-form-field
                         -let items = `${model}.sqlFunctionClasses`;
-                        -let uniqueTip = 'SQL function with such class name already exists!'
 
                         list-editable(
                             ng-model=items
@@ -72,7 +95,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
 
                             list-editable-item-edit
                                 +list-java-class-field('SQL function', '$item', '"sqlFunction"', items)
-                                    +unique-feedback('"sqlFunction"', uniqueTip)
+                                    +form-field__error({ error: 'igniteUnique', message: 'SQL function with such class name already exists!' })
 
                             list-editable-no-items
                                 list-editable-add-item-button(
@@ -87,21 +110,35 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
 
             //- Removed in ignite 2.0
             .pc-form-grid-col-60(ng-if='$ctrl.available(["1.0.0", "2.0.0"])')
-                +checkbox('Snapshotable index', `${model}.snapshotableIndex`, '"snapshotableIndex"',
-                    'Flag indicating whether SQL indexes should support snapshots')
+                +form-field__checkbox({
+                    label: 'Snapshotable index',
+                    model: `${model}.snapshotableIndex`,
+                    name: '"snapshotableIndex"',
+                    tip: 'Flag indicating whether SQL indexes should support snapshots'
+                })
 
             .pc-form-grid-col-60
-                +checkbox('Escape table and filed names', `${model}.sqlEscapeAll`, '"sqlEscapeAll"',
-                    'If enabled than all schema, table and field names will be escaped with double quotes (for example: "tableName"."fieldName").<br/>\
-                    This enforces case sensitivity for field names and also allows having special characters in table and field names.<br/>\
-                    Escaped names will be used for creation internal structures in Ignite SQL engine.')
+                +form-field__checkbox({
+                    label: 'Escape table and filed names',
+                    model: `${model}.sqlEscapeAll`,
+                    name: '"sqlEscapeAll"',
+                    tip: 'If enabled than all schema, table and field names will be escaped with double quotes (for example: "tableName"."fieldName").<br/>\
+                         This enforces case sensitivity for field names and also allows having special characters in table and field names.<br/>\
+                         Escaped names will be used for creation internal structures in Ignite SQL engine.'
+                })
 
             //- Since ignite 2.0
             .pc-form-grid-col-30(ng-if-start='$ctrl.available("2.0.0")')
-                +number('Query parallelism', model + '.queryParallelism', '"queryParallelism"', 'true', '1', '1',
-                    'A hint to query execution engine on desired degree of parallelism within a single node')
+                +form-field__number({
+                    label: 'Query parallelism',
+                    model: `${model}.queryParallelism`,
+                    name: '"queryParallelism"',
+                    placeholder: '1',
+                    min: '1',
+                    tip: 'A hint to query execution engine on desired degree of parallelism within a single node'
+                })
             .pc-form-grid-col-30(ng-if-end)
-                +sane-ignite-form-field-number({
+                +form-field__number({
                     label: 'SQL index max inline size:',
                     model: `${model}.sqlIndexMaxInlineSize`,
                     name: '"sqlIndexMaxInlineSize"',
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/rebalance.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/rebalance.pug
index 7563435e965d3..feb769901402e 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/rebalance.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/rebalance.pug
@@ -26,41 +26,82 @@ panel-collapsible(
 )
     panel-title Rebalance
     panel-description
-        | Cache rebalance settings. 
+        | Cache rebalance settings.
         a.link-success(href="https://apacheignite.readme.io/docs/rebalancing" target="_blank") More info
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-30
-                +dropdown('Mode:', `${model}.rebalanceMode`, '"rebalanceMode"', 'true', 'ASYNC',
-                    '[\
+                +form-field__dropdown({
+                    label: 'Mode:',
+                    model: `${model}.rebalanceMode`,
+                    name: '"rebalanceMode"',
+                    placeholder: 'ASYNC',
+                    options: '[\
                         {value: "SYNC", label: "SYNC"},\
                         {value: "ASYNC", label: "ASYNC"},\
                         {value: "NONE", label: "NONE"}\
                     ]',
-                    'Rebalance modes\
-                    <ul>\
-                        <li>Synchronous - in this mode distributed caches will not start until all necessary data is loaded from other available grid nodes</li>\
-                        <li>Asynchronous - in this mode distributed caches will start immediately and will load all necessary data from other available grid nodes in the background</li>\
-                        <li>None - in this mode no rebalancing will take place which means that caches will be either loaded on demand from persistent store whenever data is accessed, or will be populated explicitly</li>\
-                    </ul>')
+                    tip: 'Rebalance modes\
+                        <ul>\
+                            <li>Synchronous - in this mode distributed caches will not start until all necessary data is loaded from other available grid nodes</li>\
+                            <li>Asynchronous - in this mode distributed caches will start immediately and will load all necessary data from other available grid nodes in the background</li>\
+                            <li>None - in this mode no rebalancing will take place which means that caches will be either loaded on demand from persistent store whenever data is accessed, or will be populated explicitly</li>\
+                        </ul>'
+                })
             .pc-form-grid-col-30
-                +number('Batch size:', `${model}.rebalanceBatchSize`, '"rebalanceBatchSize"', 'true', '512 * 1024', '1',
-                    'Size (in bytes) to be loaded within a single rebalance message<br/>\
-                    Rebalancing algorithm will split total data set on every node into multiple batches prior to sending data')
+                +form-field__number({
+                    label: 'Batch size:',
+                    model: `${model}.rebalanceBatchSize`,
+                    name: '"rebalanceBatchSize"',
+                    placeholder: '512 * 1024',
+                    min: '1',
+                    tip: 'Size (in bytes) to be loaded within a single rebalance message<br/>\
+                          Rebalancing algorithm will split total data set on every node into multiple batches prior to sending data'
+                })
             .pc-form-grid-col-30
-                +number('Batches prefetch count:', `${model}.rebalanceBatchesPrefetchCount`, '"rebalanceBatchesPrefetchCount"', 'true', '2', '1',
-                    'Number of batches generated by supply node at rebalancing start')
+                +form-field__number({
+                    label: 'Batches prefetch count:',
+                    model: `${model}.rebalanceBatchesPrefetchCount`,
+                    name: '"rebalanceBatchesPrefetchCount"',
+                    placeholder: '2',
+                    min: '1',
+                    tip: 'Number of batches generated by supply node at rebalancing start'
+                })
             .pc-form-grid-col-30
-                +number('Order:', `${model}.rebalanceOrder`, '"rebalanceOrder"', 'true', '0', Number.MIN_SAFE_INTEGER,
-                    'If cache rebalance order is positive, rebalancing for this cache will be started only when rebalancing for all caches with smaller rebalance order (except caches with rebalance order 0) will be completed')
+                +form-field__number({
+                    label: 'Order:',
+                    model: `${model}.rebalanceOrder`,
+                    name: '"rebalanceOrder"',
+                    placeholder: '0',
+                    min: 'Number.MIN_SAFE_INTEGER',
+                    tip: 'If cache rebalance order is positive, rebalancing for this cache will be started only when rebalancing for all caches with smaller rebalance order (except caches with rebalance order 0) will be completed'
+                })
             .pc-form-grid-col-20
-                +number('Delay:', `${model}.rebalanceDelay`, '"rebalanceDelay"', 'true', '0', '0',
-                    'Delay in milliseconds upon a node joining or leaving topology (or crash) after which rebalancing should be started automatically')
+                +form-field__number({
+                    label: 'Delay:',
+                    model: `${model}.rebalanceDelay`,
+                    name: '"rebalanceDelay"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Delay in milliseconds upon a node joining or leaving topology (or crash) after which rebalancing should be started automatically'
+                })
             .pc-form-grid-col-20
-                +number('Timeout:', `${model}.rebalanceTimeout`, '"rebalanceTimeout"', 'true', '10000', '0',
-                    'Rebalance timeout in milliseconds')
+                +form-field__number({
+                    label: 'Timeout:',
+                    model: `${model}.rebalanceTimeout`,
+                    name: '"rebalanceTimeout"',
+                    placeholder: '10000',
+                    min: '0',
+                    tip: 'Rebalance timeout in milliseconds'
+                })
             .pc-form-grid-col-20
-                +number('Throttle:', `${model}.rebalanceThrottle`, '"rebalanceThrottle"', 'true', '0', '0',
-                    'Time in milliseconds to wait between rebalance messages to avoid overloading of CPU or network')
+                +form-field__number({
+                    label: 'Throttle:',
+                    model: `${model}.rebalanceThrottle`,
+                    name: '"rebalanceThrottle"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Time in milliseconds to wait between rebalance messages to avoid overloading of CPU or network'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'cacheRebalance')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/statistics.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/statistics.pug
index bf58354f1b032..a6c55ab3cb4d0 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/statistics.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/statistics.pug
@@ -25,10 +25,19 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +checkbox('Statistics enabled', `${model}.statisticsEnabled`, '"statisticsEnabled"', 'Flag indicating whether statistics gathering is enabled on this cache')
+                +form-field__checkbox({
+                    label: 'Statistics enabled',
+                    model: `${model}.statisticsEnabled`,
+                    name: '"statisticsEnabled"',
+                    tip: 'Flag indicating whether statistics gathering is enabled on this cache'
+                })
             .pc-form-grid-col-60
-                +checkbox('Management enabled', `${model}.managementEnabled`, '"managementEnabled"',
-                'Flag indicating whether management is enabled on this cache<br/>\
-                If enabled the CacheMXBean for each cache is registered in the platform MBean server')
+                +form-field__checkbox({
+                    label: 'Management enabled',
+                    model: `${model}.managementEnabled`,
+                    name: '"managementEnabled"',
+                    tip: 'Flag indicating whether management is enabled on this cache<br/>\
+                         If enabled the CacheMXBean for each cache is registered in the platform MBean server'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'cacheStatistics')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/store.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/store.pug
index 32d966aff4001..903ea0888e861 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/store.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cache-edit-form/templates/store.pug
@@ -19,40 +19,17 @@ include /app/helpers/jade/mixins
 -var form = 'store'
 -var model = '$ctrl.clonedCache'
 
-mixin hibernateField(name, model, items, valid, save, newItem)
-    -var resetOnEnter = newItem ? '(stopblur = true) && (group.add = [{}])' : '(field.edit = false)'
-    -var onEnter = `${valid} && (${save}); ${valid} && ${resetOnEnter};`
-
-    -var onEscape = newItem ? 'group.add = []' : 'field.edit = false'
-
-    -var resetOnBlur = newItem ? '!stopblur && (group.add = [])' : 'field.edit = false'
-    -var onBlur = `${valid} && (${save}); ${resetOnBlur};`
-
-    div(ignite-on-focus-out=onBlur)
-        if block
-            block
-
-        .input-tip
-            +ignite-form-field-input(name, model, false, 'true', 'key=value')(
-                data-ignite-property-unique=items
-                data-ignite-property-value-specified
-                data-ignite-form-field-input-autofocus='true'
-
-                ignite-on-enter=onEnter
-                ignite-on-escape=onEscape
-            )
-
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Store
-    panel-description 
-        | Cache store settings. 
+    panel-description
+        | Cache store settings.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/3rd-party-store" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             -var storeFactory = `${model}.cacheStoreFactory`;
             -var storeFactoryKind = `${storeFactory}.kind`;
             .pc-form-grid-col-60
-                +sane-ignite-form-field-dropdown({
+                +form-field__dropdown({
                     label: 'Store factory:',
                     model: storeFactoryKind,
                     name: '"cacheStoreFactory"',
@@ -65,24 +42,24 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                         <li>Hibernate BLOB store factory - Objects are stored in underlying database in BLOB format backed by Hibernate</li>
                     </ul>`
                 })(
-                    ui-validate=`{
+                ui-validate=`{
                         writeThroughOn: '$ctrl.Caches.cacheStoreFactory.storeDisabledValueOff(${model}, ${model}.writeThrough)',
                         readThroughOn: '$ctrl.Caches.cacheStoreFactory.storeDisabledValueOff(${model}, ${model}.readThrough)',
                         writeBehindOn: '$ctrl.Caches.cacheStoreFactory.storeDisabledValueOff(${model}, ${model}.writeBehindEnabled)'
                     }`
-                    ui-validate-watch-collection=`"[${model}.readThrough, ${model}.writeThrough, ${model}.writeBehindEnabled]"`
-                    ng-model-options='{allowInvalid: true}'
+                ui-validate-watch-collection=`"[${model}.readThrough, ${model}.writeThrough, ${model}.writeBehindEnabled]"`
+                ng-model-options='{allowInvalid: true}'
                 )
-                    +form-field-feedback(null, 'writeThroughOn', 'Write through is enabled but store is not set')
-                    +form-field-feedback(null, 'readThroughOn', 'Read through is enabled but store is not set')
-                    +form-field-feedback(null, 'writeBehindOn', 'Write-behind is enabled but store is not set')
+                    +form-field__error({ error: 'writeThroughOn', message: 'Write through is enabled but store is not set' })
+                    +form-field__error({ error: 'readThroughOn', message: 'Read through is enabled but store is not set' })
+                    +form-field__error({ error: 'writeBehindOn', message: 'Write-behind is enabled but store is not set' })
             .pc-form-group(ng-if=storeFactoryKind)
                 .pc-form-grid-row(ng-if=`${storeFactoryKind} === 'CacheJdbcPojoStoreFactory'`)
                     -var pojoStoreFactory = `${storeFactory}.CacheJdbcPojoStoreFactory`
                     -var required = `${storeFactoryKind} === 'CacheJdbcPojoStoreFactory'`
 
                     .pc-form-grid-col-60
-                        +sane-ignite-form-field-text({
+                        +form-field__text({
                             label: 'Data source bean name:',
                             model: `${pojoStoreFactory}.dataSourceBean`,
                             name: '"pojoDataSourceBean"',
@@ -93,66 +70,130 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                             is-valid-java-identifier
                             not-java-reserved-word
                         )
-                            +form-field-feedback(null, 'required', 'Data source bean name is required')
-                            +form-field-feedback(null, 'isValidJavaIdentifier', 'Data source bean name is not a valid Java identifier')
-                            +form-field-feedback(null, 'notJavaReservedWord', 'Data source bean name should not be a Java reserved word')
+                            +form-field__error({ error: 'required', message: 'Data source bean name is required' })
+                            +form-field__error({ error: 'isValidJavaIdentifier', message: 'Data source bean name is not a valid Java identifier' })
+                            +form-field__error({ error: 'notJavaReservedWord', message: 'Data source bean name should not be a Java reserved word' })
                     .pc-form-grid-col-60
-                        +dialect('Dialect:', `${pojoStoreFactory}.dialect`, '"pojoDialect"', required,
-                            'Dialect of SQL implemented by a particular RDBMS:', 'Generic JDBC dialect',
-                            'Choose JDBC dialect')
+                        +form-field__dialect({
+                            label: 'Dialect:',
+                            model: `${pojoStoreFactory}.dialect`,
+                            name: '"pojoDialect"',
+                            required,
+                            tip: 'Dialect of SQL implemented by a particular RDBMS:',
+                            genericDialectName: 'Generic JDBC dialect',
+                            placeholder: 'Choose JDBC dialect'
+                        })
                     .pc-form-grid-col-30
-                        +number('Batch size:', `${pojoStoreFactory}.batchSize`, '"pojoBatchSize"', true, '512', '1',
-                            'Maximum batch size for writeAll and deleteAll operations')
+                        +form-field__number({
+                            label:'Batch size:',
+                            model: `${pojoStoreFactory}.batchSize`,
+                            name: '"pojoBatchSize"',
+                            placeholder: '512',
+                            min: '1',
+                            tip: 'Maximum batch size for writeAll and deleteAll operations'
+                        })
                     .pc-form-grid-col-30
-                        +number('Thread count:', `${pojoStoreFactory}.maximumPoolSize`, '"pojoMaximumPoolSize"', true, 'availableProcessors', '1',
-                            'Maximum workers thread count.<br/>\
-                            These threads are responsible for load cache.')
+                        +form-field__number({
+                            label: 'Thread count:',
+                            model: `${pojoStoreFactory}.maximumPoolSize`,
+                            name: '"pojoMaximumPoolSize"',
+                            placeholder: 'availableProcessors',
+                            min: '1',
+                            tip: 'Maximum workers thread count.<br/>\
+                                 These threads are responsible for load cache.'
+                        })
                     .pc-form-grid-col-30
-                        +number('Maximum write attempts:', `${pojoStoreFactory}.maximumWriteAttempts`, '"pojoMaximumWriteAttempts"', true, '2', '0',
-                            'Maximum write attempts in case of database error')
+                        +form-field__number({
+                            label: 'Maximum write attempts:',
+                            model: `${pojoStoreFactory}.maximumWriteAttempts`,
+                            name: '"pojoMaximumWriteAttempts"',
+                            placeholder: '2',
+                            min: '0',
+                            tip: 'Maximum write attempts in case of database error'
+                        })
                     .pc-form-grid-col-30
-                        +number('Parallel load threshold:', `${pojoStoreFactory}.parallelLoadCacheMinimumThreshold`, '"pojoParallelLoadCacheMinimumThreshold"', true, '512', '0',
-                            'Parallel load cache minimum threshold.<br/>\
-                            If <b>0</b> then load sequentially.')
+                        +form-field__number({
+                            label: 'Parallel load threshold:',
+                            model: `${pojoStoreFactory}.parallelLoadCacheMinimumThreshold`,
+                            name: '"pojoParallelLoadCacheMinimumThreshold"',
+                            placeholder: '512',
+                            min: '0',
+                            tip: 'Parallel load cache minimum threshold.<br/>\
+                                 If <b>0</b> then load sequentially.'
+                        })
                     .pc-form-grid-col-60
-                        +java-class('Hasher', `${pojoStoreFactory}.hasher`, '"pojoHasher"', 'true', 'false', 'Hash calculator', required)
+                        +form-field__java-class({
+                            label: 'Hasher:',
+                            model: `${pojoStoreFactory}.hasher`,
+                            name: '"pojoHasher"',
+                            tip: 'Hash calculator',
+                            validationActive: required
+                        })
                     .pc-form-grid-col-60
-                        +java-class('Transformer', `${pojoStoreFactory}.transformer`, '"pojoTransformer"', 'true', 'false', 'Types transformer', required)
+                        +form-field__java-class({
+                            label: 'Transformer:',
+                            model: `${pojoStoreFactory}.transformer`,
+                            name: '"pojoTransformer"',
+                            tip: 'Types transformer',
+                            validationActive: required
+                        })
                     .pc-form-grid-col-60
-                        +checkbox('Escape table and filed names', `${pojoStoreFactory}.sqlEscapeAll`, '"sqlEscapeAll"',
-                            'If enabled than all schema, table and field names will be escaped with double quotes (for example: "tableName"."fieldName").<br/>\
-                            This enforces case sensitivity for field names and also allows having special characters in table and field names.<br/>\
-                            Escaped names will be used for CacheJdbcPojoStore internal SQL queries.')
+                        +form-field__checkbox({
+                            label: 'Escape table and filed names',
+                            model:`${pojoStoreFactory}.sqlEscapeAll`,
+                            name: '"sqlEscapeAll"',
+                            tip: 'If enabled than all schema, table and field names will be escaped with double quotes (for example: "tableName"."fieldName").<br/>\
+                                  This enforces case sensitivity for field names and also allows having special characters in table and field names.<br/>\
+                                  Escaped names will be used for CacheJdbcPojoStore internal SQL queries.'
+                        })
                 .pc-form-grid-row(ng-if=`${storeFactoryKind} === 'CacheJdbcBlobStoreFactory'`)
                     -var blobStoreFactory = `${storeFactory}.CacheJdbcBlobStoreFactory`
                     -var blobStoreFactoryVia = `${blobStoreFactory}.connectVia`
 
                     .pc-form-grid-col-60
-                        +dropdown('Connect via:', blobStoreFactoryVia, '"connectVia"', 'true', 'Choose connection method',
-                            '[\
-                                {value: "URL", label: "URL"},\
-                                {value: "DataSource", label: "Data source"}\
-                            ]',
-                            'You can connect to database via:\
-                            <ul>\
-                                <li>JDBC URL, for example: jdbc:h2:mem:myDatabase</li>\
-                                <li>Configured data source</li>\
-                            </ul>')
+                        +form-field__dropdown({
+                            label: 'Connect via:',
+                            model: blobStoreFactoryVia,
+                            name: '"connectVia"',
+                            placeholder: 'Choose connection method',
+                            options: '[\
+                                                        {value: "URL", label: "URL"},\
+                                                        {value: "DataSource", label: "Data source"}\
+                                                    ]',
+                            tip: 'You can connect to database via:\
+                                                    <ul>\
+                                                        <li>JDBC URL, for example: jdbc:h2:mem:myDatabase</li>\
+                                                        <li>Configured data source</li>\
+                                                    </ul>'
+                        })
 
                     -var required = `${storeFactoryKind} === 'CacheJdbcBlobStoreFactory' && ${blobStoreFactoryVia} === 'URL'`
 
                     .pc-form-grid-col-60(ng-if-start=`${blobStoreFactoryVia} === 'URL'`)
-                        +text('Connection URL:', `${blobStoreFactory}.connectionUrl`, '"connectionUrl"', required, 'Input URL',
-                            'URL for database access, for example: jdbc:h2:mem:myDatabase')
+                        +form-field__text({
+                            label: 'Connection URL:',
+                            model: `${blobStoreFactory}.connectionUrl`,
+                            name: '"connectionUrl"',
+                            required: required,
+                            placeholder: 'Input URL',
+                            tip: 'URL for database access, for example: jdbc:h2:mem:myDatabase'
+                        })
                     .pc-form-grid-col-30
-                        +text('User:', `${blobStoreFactory}.user`, '"user"', required, 'Input user name', 'User name for database access')
+                        +form-field__text({
+                            label: 'User:',
+                            model: `${blobStoreFactory}.user`,
+                            name: '"user"',
+                            required: required,
+                            placeholder: 'Input user name',
+                            tip: 'User name for database access'
+                        })
                     .pc-form-grid-col-30(ng-if-end)
                         .pc-form-grid__text-only-item Password will be generated as stub.
 
                     -var required = `${storeFactoryKind} === 'CacheJdbcBlobStoreFactory' && ${blobStoreFactoryVia} !== 'URL'`
 
                     .pc-form-grid-col-60(ng-if-start=`${blobStoreFactoryVia} !== 'URL'`)
-                        +sane-ignite-form-field-text({
+                        +form-field__text({
                             label: 'Data source bean name:',
                             model: `${blobStoreFactory}.dataSourceBean`,
                             name: '"blobDataSourceBean"',
@@ -160,105 +201,150 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                             placeholder: 'Input bean name',
                             tip: 'Name of the data source bean in Spring context'
                         })(
-                            is-valid-java-identifier
-                            not-java-reserved-word
+                        is-valid-java-identifier
+                        not-java-reserved-word
                         )
-                            +form-field-feedback(null, 'required', 'Data source bean name is required')
-                            +form-field-feedback(null, 'isValidJavaIdentifier', 'Data source bean name is not a valid Java identifier')
-                            +form-field-feedback(null, 'notJavaReservedWord', 'Data source bean name should not be a Java reserved word')
+                            +form-field__error({ error: 'required', message: 'Data source bean name is required' })
+                            +form-field__error({ error: 'isValidJavaIdentifier', message: 'Data source bean name is not a valid Java identifier' })
+                            +form-field__error({ error: 'notJavaReservedWord', message: 'Data source bean name should not be a Java reserved word' })
                     .pc-form-grid-col-60(ng-if-end)
-                        +dialect('Database:', `${blobStoreFactory}.dialect`, '"blobDialect"', required, 'Supported databases:', 'Generic database', 'Choose database')
+                        +form-field__dialect({
+                            label: 'Database:',
+                            model: `${blobStoreFactory}.dialect`,
+                            name: '"blobDialect"',
+                            required,
+                            tip: 'Supported databases:',
+                            genericDialectName: 'Generic database',
+                            placeholder: 'Choose database'
+                        })
 
                     .pc-form-grid-col-60
-                        +checkbox('Init schema', `${blobStoreFactory}.initSchema`, '"initSchema"',
-                            'Flag indicating whether DB schema should be initialized by Ignite (default behaviour) or was explicitly created by user')
+                        +form-field__checkbox({
+                            label: 'Init schema',
+                            model: `${blobStoreFactory}.initSchema`,
+                            name: '"initSchema"',
+                            tip: 'Flag indicating whether DB schema should be initialized by Ignite (default behaviour) or was explicitly created by user'
+                        })
                     .pc-form-grid-col-60
-                        +text('Create query:', `${blobStoreFactory}.createTableQuery`, '"createTableQuery"', 'false', 'SQL for table creation',
-                            'Query for table creation in underlying database<br/>\
-                            Default value: create table if not exists ENTRIES (key binary primary key, val binary)')
+                        +form-field__text({
+                            label: 'Create query:',
+                            model: `${blobStoreFactory}.createTableQuery`,
+                            name: '"createTableQuery"',
+                            placeholder: 'SQL for table creation',
+                            tip: 'Query for table creation in underlying database<br/>\
+                                 Default value: create table if not exists ENTRIES (key binary primary key, val binary)'
+                        })
                     .pc-form-grid-col-60
-                        +text('Load query:', `${blobStoreFactory}.loadQuery`, '"loadQuery"', 'false', 'SQL for load entry',
-                            'Query for entry load from underlying database<br/>\
-                            Default value: select * from ENTRIES where key=?')
+                        +form-field__text({
+                            label: 'Load query:',
+                            model: `${blobStoreFactory}.loadQuery`,
+                            name: '"loadQuery"',
+                            placeholder: 'SQL for load entry',
+                            tip: 'Query for entry load from underlying database<br/>\
+                                 Default value: select * from ENTRIES where key=?'
+                        })
                     .pc-form-grid-col-60
-                        +text('Insert query:', `${blobStoreFactory}.insertQuery`, '"insertQuery"', 'false', 'SQL for insert entry',
-                            'Query for insert entry into underlying database<br/>\
-                            Default value: insert into ENTRIES (key, val) values (?, ?)')
+                        +form-field__text({
+                            label: 'Insert query:',
+                            model: `${blobStoreFactory}.insertQuery`,
+                            name: '"insertQuery"',
+                            placeholder: 'SQL for insert entry',
+                            tip: 'Query for insert entry into underlying database<br/>\
+                                 Default value: insert into ENTRIES (key, val) values (?, ?)'
+                        })
                     .pc-form-grid-col-60
-                        +text('Update query:', `${blobStoreFactory}.updateQuery`, '"updateQuery"', 'false', 'SQL for update entry',
-                            'Query for update entry in underlying database<br/>\
-                            Default value: update ENTRIES set val=? where key=?')
+                        +form-field__text({
+                            label: 'Update query:',
+                            model: `${blobStoreFactory}.updateQuery`,
+                            name: '"updateQuery"',
+                            placeholder: 'SQL for update entry',
+                            tip: 'Query for update entry in underlying database<br/>\
+                                 Default value: update ENTRIES set val=? where key=?'
+                        })
                     .pc-form-grid-col-60
-                        +text('Delete query:', `${blobStoreFactory}.deleteQuery`, '"deleteQuery"', 'false', 'SQL for delete entry',
-                            'Query for delete entry from underlying database<br/>\
-                            Default value: delete from ENTRIES where key=?')
+                        +form-field__text({
+                            label: 'Delete query:',
+                            model: `${blobStoreFactory}.deleteQuery`,
+                            name: '"deleteQuery"',
+                            placeholder: 'SQL for delete entry',
+                            tip: 'Query for delete entry from underlying database<br/>\
+                                 Default value: delete from ENTRIES where key=?'
+                        })
 
                 .pc-form-grid-row(ng-if=`${storeFactoryKind} === 'CacheHibernateBlobStoreFactory'`)
                     -var hibernateStoreFactory = `${storeFactory}.CacheHibernateBlobStoreFactory`
 
                     .pc-form-grid-col-60
                         .ignite-form-field
-                            +ignite-form-field__label('Hibernate properties:', '"hibernateProperties"')
-                                +tooltip(`List of Hibernate properties<bt />
-                                    For example: connection.url=jdbc:h2:mem:exampleDb`)
-                            .ignite-form-field__control
-                                +list-pair-edit({
-                                    items: `${hibernateStoreFactory}.hibernateProperties`,
-                                    keyLbl: 'Property name', 
-                                    valLbl: 'Property value',
-                                    itemName: 'property',
-                                    itemsName: 'properties'
-                                })
+                            +form-field__label({ label: 'Hibernate properties:', name: '"hibernateProperties"' })
+                                +form-field__tooltip({ title: `List of Hibernate properties<bt />
+                                    For example: connection.url=jdbc:h2:mem:exampleDb` })
+
+                            +list-pair-edit({
+                                items: `${hibernateStoreFactory}.hibernateProperties`,
+                                keyLbl: 'Property name',
+                                valLbl: 'Property value',
+                                itemName: 'property',
+                                itemsName: 'properties'
+                            })
 
             - form = 'store'
             .pc-form-grid-col-60
-                +checkbox('Keep binary in store', `${model}.storeKeepBinary`, '"storeKeepBinary"',
-                    'Flag indicating that CacheStore implementation is working with binary objects instead of Java objects')
+                +form-field__checkbox({
+                    label: 'Keep binary in store',
+                    model: `${model}.storeKeepBinary`,
+                    name: '"storeKeepBinary"',
+                    tip: 'Flag indicating that CacheStore implementation is working with binary objects instead of Java objects'
+                })
             .pc-form-grid-col-60
-                +checkbox('Load previous value', `${model}.loadPreviousValue`, '"loadPreviousValue"',
-                    'Flag indicating whether value should be loaded from store if it is not in the cache for following cache operations: \
-                    <ul> \
-                        <li>IgniteCache.putIfAbsent()</li> \
-                        <li>IgniteCache.replace()</li> \
-                        <li>IgniteCache.remove()</li> \
-                        <li>IgniteCache.getAndPut()</li> \
-                        <li>IgniteCache.getAndRemove()</li> \
-                        <li>IgniteCache.getAndReplace()</li> \
-                        <li> IgniteCache.getAndPutIfAbsent()</li>\
-                    </ul>')
+                +form-field__checkbox({
+                    label: 'Load previous value',
+                    model: `${model}.loadPreviousValue`,
+                    name: '"loadPreviousValue"',
+                    tip: 'Flag indicating whether value should be loaded from store if it is not in the cache for following cache operations: \
+                        <ul> \
+                            <li>IgniteCache.putIfAbsent()</li> \
+                            <li>IgniteCache.replace()</li> \
+                            <li>IgniteCache.remove()</li> \
+                            <li>IgniteCache.getAndPut()</li> \
+                            <li>IgniteCache.getAndRemove()</li> \
+                            <li>IgniteCache.getAndReplace()</li> \
+                            <li> IgniteCache.getAndPutIfAbsent()</li>\
+                        </ul>'
+                })
             .pc-form-grid-col-60
-                +sane-form-field-checkbox({
+                +form-field__checkbox({
                     label: 'Read-through',
                     model: `${model}.readThrough`,
                     name: '"readThrough"',
                     tip: 'Flag indicating whether read-through caching should be used'
                 })(
-                    ng-model-options='{allowInvalid: true}'
-                    ui-validate=`{
+                ng-model-options='{allowInvalid: true}'
+                ui-validate=`{
                         storeEnabledReadOrWriteOn: '$ctrl.Caches.cacheStoreFactory.storeEnabledReadOrWriteOn(${model})'
                     }`
-                    ui-validate-watch-collection=`"[${storeFactoryKind}, ${model}.writeThrough, ${model}.readThrough]"`
+                ui-validate-watch-collection=`"[${storeFactoryKind}, ${model}.writeThrough, ${model}.readThrough]"`
                 )
-                    +form-field-feedback(0, 'storeEnabledReadOrWriteOn', 'Read or write through should be turned on when store kind is set')
+                    +form-field__error({ error: 'storeEnabledReadOrWriteOn', message: 'Read or write through should be turned on when store kind is set' })
             .pc-form-grid-col-60
-                +sane-form-field-checkbox({
+                +form-field__checkbox({
                     label: 'Write-through',
                     model: `${model}.writeThrough`,
                     name: '"writeThrough"',
                     tip: 'Flag indicating whether write-through caching should be used'
                 })(
-                    ng-model-options='{allowInvalid: true}'
-                    ui-validate=`{
+                ng-model-options='{allowInvalid: true}'
+                ui-validate=`{
                         storeEnabledReadOrWriteOn: '$ctrl.Caches.cacheStoreFactory.storeEnabledReadOrWriteOn(${model})'
                     }`
-                    ui-validate-watch-collection=`"[${storeFactoryKind}, ${model}.writeThrough, ${model}.readThrough]"`
+                ui-validate-watch-collection=`"[${storeFactoryKind}, ${model}.writeThrough, ${model}.readThrough]"`
                 )
-                    +form-field-feedback(0, 'storeEnabledReadOrWriteOn', 'Read or write through should be turned on when store kind is set')
+                    +form-field__error({ error: 'storeEnabledReadOrWriteOn', message: 'Read or write through should be turned on when store kind is set' })
 
             -var enabled = `${model}.writeBehindEnabled`
 
             .pc-form-grid-col-60.pc-form-group__text-title
-                +sane-form-field-checkbox({
+                +form-field__checkbox({
                     label: 'Write-behind',
                     model: enabled,
                     name: '"writeBehindEnabled"',
@@ -267,16 +353,23 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                         Write-behind is a special mode when updates to cache accumulated and then asynchronously flushed to persistent store as a bulk operation.
                     `
                 })(
-                    ng-model-options='{allowInvalid: true}'
+                ng-model-options='{allowInvalid: true}'
                 )
-                    +form-field-feedback(0, 'storeDisabledValueOff', 'Write-behind is enabled but store kind is not set')
+                    +form-field__error({ error: 'storeDisabledValueOff', message: 'Write-behind is enabled but store kind is not set' })
             .pc-form-group.pc-form-grid-row(ng-if=enabled)
                 .pc-form-grid-col-30
-                    +number('Batch size:', `${model}.writeBehindBatchSize`, '"writeBehindBatchSize"', enabled, '512', '1',
-                        'Maximum batch size for write-behind cache store operations<br/>\
-                         Store operations(get or remove) are combined in a batch of this size to be passed to cache store')
+                    +form-field__number({
+                        label: 'Batch size:',
+                        model: `${model}.writeBehindBatchSize`,
+                        name: '"writeBehindBatchSize"',
+                        disabled: `!(${enabled})`,
+                        placeholder: '512',
+                        min: '1',
+                        tip: 'Maximum batch size for write-behind cache store operations<br/>\
+                              Store operations(get or remove) are combined in a batch of this size to be passed to cache store'
+                    })
                 .pc-form-grid-col-30
-                    +sane-ignite-form-field-number({
+                    +form-field__number({
                         label: 'Flush size:',
                         model: `${model}.writeBehindFlushSize`,
                         name: '"writeBehindFlushSize"',
@@ -285,10 +378,10 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                         tip: `Maximum size of the write-behind cache<br/>
                          If cache size exceeds this value, all cached items are flushed to the cache store and write cache is cleared`
                     })(
-                        ng-model-options='{allowInvalid: true}'
+                    ng-model-options='{allowInvalid: true}'
                     )
                 .pc-form-grid-col-30
-                    +sane-ignite-form-field-number({
+                    +form-field__number({
                         label: 'Flush frequency:',
                         model: `${model}.writeBehindFlushFrequency`,
                         name: '"writeBehindFlushFrequency"',
@@ -296,15 +389,28 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                         min: `{{ $ctrl.Caches.writeBehindFlush.min(${model}) }}`,
                         tip: `Frequency with which write-behind cache is flushed to the cache store in milliseconds`
                     })(
-                        ng-model-options='{allowInvalid: true}'
+                    ng-model-options='{allowInvalid: true}'
                     )
                 .pc-form-grid-col-30
-                    +number('Flush threads count:', `${model}.writeBehindFlushThreadCount`, '"writeBehindFlushThreadCount"', enabled, '1', '1',
-                        'Number of threads that will perform cache flushing')
+                    +form-field__number({
+                        label: 'Flush threads count:',
+                        model: `${model}.writeBehindFlushThreadCount`,
+                        name: '"writeBehindFlushThreadCount"',
+                        disabled: `!(${enabled})`,
+                        placeholder: '1',
+                        min: '1',
+                        tip: 'Number of threads that will perform cache flushing'
+                    })
 
                 //- Since ignite 2.0
                 .pc-form-grid-col-60(ng-if='$ctrl.available("2.0.0")')
-                    +checkbox-enabled('Write coalescing', model + '.writeBehindCoalescing', '"WriteBehindCoalescing"', enabled, 'Write coalescing flag for write-behind cache store')
+                    +form-field__checkbox({
+                        label: 'Write coalescing',
+                        model: model + '.writeBehindCoalescing',
+                        name: '"WriteBehindCoalescing"',
+                        disabled: `!${enabled}`,
+                        tip: 'Write coalescing flag for write-behind cache store'
+                    })
 
         .pca-form-column-6
             +preview-xml-java(model, 'cacheStore', 'domains')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/atomic.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/atomic.pug
index 13424f84c261b..9f5f1385b2ca1 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/atomic.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/atomic.pug
@@ -26,50 +26,95 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Atomic configuration
     panel-description
         | Configuration for atomic data structures.
-        | Atomics are distributed across the cluster, essentially enabling performing atomic operations (such as increment-and-get or compare-and-set) with the same globally-visible value. 
+        | Atomics are distributed across the cluster, essentially enabling performing atomic operations (such as increment-and-get or compare-and-set) with the same globally-visible value.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/atomic-types" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-30
-                +dropdown('Cache mode:', `${model}.cacheMode`, '"cacheMode"', 'true', 'PARTITIONED',
-                    '[\
-                        {value: "LOCAL", label: "LOCAL"},\
-                        {value: "REPLICATED", label: "REPLICATED"},\
-                        {value: "PARTITIONED", label: "PARTITIONED"}\
-                    ]',
-                    'Cache modes:\
-                    <ul>\
-                        <li>Partitioned - in this mode the overall key set will be divided into partitions and all partitions will be split equally between participating nodes</li>\
-                        <li>Replicated - in this mode all the keys are distributed to all participating nodes</li>\
-                        <li>Local - in this mode caches residing on different grid nodes will not know about each other</li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Cache mode:',
+                    model: `${model}.cacheMode`,
+                    name: '"cacheMode"',
+                    placeholder: 'PARTITIONED',
+                    options: '[\
+                                            {value: "LOCAL", label: "LOCAL"},\
+                                            {value: "REPLICATED", label: "REPLICATED"},\
+                                            {value: "PARTITIONED", label: "PARTITIONED"}\
+                                        ]',
+                    tip: 'Cache modes:\
+                                        <ul>\
+                                            <li>Partitioned - in this mode the overall key set will be divided into partitions and all partitions will be split equally between participating nodes</li>\
+                                            <li>Replicated - in this mode all the keys are distributed to all participating nodes</li>\
+                                            <li>Local - in this mode caches residing on different grid nodes will not know about each other</li>\
+                                        </ul>'
+                })
             .pc-form-grid-col-30
-                +number('Sequence reserve:', `${model}.atomicSequenceReserveSize`, '"atomicSequenceReserveSize"', 'true', '1000', '0',
-                    'Default number of sequence values reserved for IgniteAtomicSequence instances<br/>\
-                    After a certain number has been reserved, consequent increments of sequence will happen locally, without communication with other nodes, until the next reservation has to be made')
+                +form-field__number({
+                    label: 'Sequence reserve:',
+                    model: `${model}.atomicSequenceReserveSize`,
+                    name: '"atomicSequenceReserveSize"',
+                    placeholder: '1000',
+                    min: '0',
+                    tip: 'Default number of sequence values reserved for IgniteAtomicSequence instances<br/>\
+                          After a certain number has been reserved, consequent increments of sequence will happen locally, without communication with other nodes, until the next reservation has to be made'
+                })
             .pc-form-grid-col-60(ng-show=`!(${model}.cacheMode && ${model}.cacheMode != "PARTITIONED")`)
-                +number('Backups:', model + '.backups', '"backups"', 'true', '0', '0', 'Number of backup nodes')
+                +form-field__number({
+                    label: 'Backups:',
+                    model: model + '.backups',
+                    name: '"backups"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Number of backup nodes'
+                })
 
             .pc-form-grid-col-60(ng-if-start='$ctrl.available("2.1.0")')
-                +dropdown('Function:', `${affModel}.kind`, '"AffinityKind"', 'true', 'Default', '$ctrl.Clusters.affinityFunctions',
-                    'Key topology resolver to provide mapping from keys to nodes\
-                    <ul>\
-                        <li>Rendezvous - Based on Highest Random Weight algorithm<br/></li>\
-                        <li>Custom - Custom implementation of key affinity function<br/></li>\
-                        <li>Default - By default rendezvous affinity function  with 1024 partitions is used<br/></li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Function:',
+                    model: `${affModel}.kind`,
+                    name: '"AffinityKind"',
+                    placeholder: 'Default',
+                    options: '$ctrl.Clusters.affinityFunctions',
+                    tip: 'Key topology resolver to provide mapping from keys to nodes\
+                                        <ul>\
+                                            <li>Rendezvous - Based on Highest Random Weight algorithm<br/></li>\
+                                            <li>Custom - Custom implementation of key affinity function<br/></li>\
+                                            <li>Default - By default rendezvous affinity function  with 1024 partitions is used<br/></li>\
+                                        </ul>'
+                })
             .pc-form-group(ng-if-end ng-if=rendezvousAff + ' || ' + customAff)
                 .pc-form-grid-row
                     .pc-form-grid-col-30(ng-if-start=rendezvousAff)
-                        +number-required('Partitions', `${affModel}.Rendezvous.partitions`, '"RendPartitions"', 'true', rendPartitionsRequired, '1024', '1', 'Number of partitions')
+                        +form-field__number({
+                            label: 'Partitions',
+                            model: `${affModel}.Rendezvous.partitions`,
+                            name: '"RendPartitions"',
+                            required: rendPartitionsRequired,
+                            placeholder: '1024',
+                            min: '1',
+                            tip: 'Number of partitions'
+                        })
                     .pc-form-grid-col-30
-                        +java-class('Backup filter', `${affModel}.Rendezvous.affinityBackupFilter`, '"RendAffinityBackupFilter"', 'true', 'false',
-                            'Backups will be selected from all nodes that pass this filter')
+                        +form-field__java-class({
+                            label: 'Backup filter',
+                            model: `${affModel}.Rendezvous.affinityBackupFilter`,
+                            name: '"RendAffinityBackupFilter"',
+                            tip: 'Backups will be selected from all nodes that pass this filter'
+                        })
                     .pc-form-grid-col-60(ng-if-end)
-                        +checkbox('Exclude neighbors', `${affModel}.Rendezvous.excludeNeighbors`, '"RendExcludeNeighbors"',
-                            'Exclude same - host - neighbors from being backups of each other and specified number of backups')
+                        +form-field__checkbox({
+                            label: 'Exclude neighbors',
+                            model: `${affModel}.Rendezvous.excludeNeighbors`,
+                            name: '"RendExcludeNeighbors"',
+                            tip: 'Exclude same - host - neighbors from being backups of each other and specified number of backups'
+                        })
                     .pc-form-grid-col-60(ng-if=customAff)
-                        +java-class('Class name:', `${affModel}.Custom.className`, '"AffCustomClassName"', 'true', customAff,
-                            'Custom key affinity function implementation class name')
+                        +form-field__java-class({
+                            label: 'Class name:',
+                            model: `${affModel}.Custom.className`,
+                            name: '"AffCustomClassName"',
+                            required: customAff,
+                            tip: 'Custom key affinity function implementation class name'
+                        })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterAtomics')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/attributes.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/attributes.pug
index b57f1dad86286..f704bc987a93f 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/attributes.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/attributes.pug
@@ -25,16 +25,16 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6
             .ignite-form-field
-                +ignite-form-field__label('User attributes:', '"userAttributes"')
-                    +tooltip(`User-defined attributes to add to node`)
-                .ignite-form-field__control
-                    +list-pair-edit({
-                        items: `${model}.attributes`,
-                        keyLbl: 'Attribute name', 
-                        valLbl: 'Attribute value',
-                        itemName: 'attribute',
-                        itemsName: 'attributes'
-                    })
+                +form-field__label({ label: 'User attributes:', name: '"userAttributes"'})
+                    +form-field__tooltip({ title: `User-defined attributes to add to node` })
+
+                +list-pair-edit({
+                    items: `${model}.attributes`,
+                    keyLbl: 'Attribute name', 
+                    valLbl: 'Attribute value',
+                    itemName: 'attribute',
+                    itemsName: 'attributes'
+                })
 
         .pca-form-column-6
             +preview-xml-java(model, 'clusterUserAttributes')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/binary.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/binary.pug
index 17fe4cd66f03c..a20a3fd93a9e5 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/binary.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/binary.pug
@@ -22,59 +22,103 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Binary configuration
     panel-description
-        | Configuration of specific binary types. 
+        | Configuration of specific binary types.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/binary-marshaller" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +java-class('ID mapper:', model + '.idMapper', '"idMapper"', 'true', 'false',
-                    'Maps given from BinaryNameMapper type and filed name to ID that will be used by Ignite in internals<br/>\
-                    Ignite never writes full strings for field or type names. Instead, for performance reasons, Ignite writes integer hash codes for type/class and field names. It has been tested that hash code conflicts for the type/class names or the field names within the same type are virtually non - existent and, to gain performance, it is safe to work with hash codes. For the cases when hash codes for different types or fields actually do collide <b>BinaryIdMapper</b> allows to override the automatically generated hash code IDs for the type and field names')
+                +form-field__java-class({
+                    label: 'ID mapper:',
+                    model: model + '.idMapper',
+                    name: '"idMapper"',
+                    tip: 'Maps given from BinaryNameMapper type and filed name to ID that will be used by Ignite in internals<br/>\
+                          Ignite never writes full strings for field or type names. Instead, for performance reasons, Ignite writes integer hash codes for type/class and field names. It has been tested that hash code conflicts for the type/class names or the field names within the same type are virtually non - existent and, to gain performance, it is safe to work with hash codes. For the cases when hash codes for different types or fields actually do collide <b>BinaryIdMapper</b> allows to override the automatically generated hash code IDs for the type and field names'
+                })
             .pc-form-grid-col-60
-                +java-class('Name mapper:', model + '.nameMapper', '"nameMapper"', 'true', 'false', 'Maps type/class and field names to different names')
+                +form-field__java-class({
+                    label: 'Name mapper:',
+                    model: model + '.nameMapper',
+                    name: '"nameMapper"',
+                    tip: 'Maps type/class and field names to different names'
+                })
             .pc-form-grid-col-60
-                +java-class('Serializer:', model + '.serializer', '"serializer"', 'true', 'false', 'Class with custom serialization logic for binary objects')
+                +form-field__java-class({
+                    label: 'Serializer:',
+                    model: model + '.serializer',
+                    name: '"serializer"',
+                    tip: 'Class with custom serialization logic for binary objects'
+                })
             .pc-form-grid-col-60
                 .ignite-form-field
-                    +ignite-form-field__label('Type configurations:', '"typeConfigurations"')
-                        +tooltip(`Configuration properties for binary types`)
-                    .ignite-form-field__control
-                        -var items = model + '.typeConfigurations'
-                        list-editable.pc-list-editable-with-form-grid(ng-model=items name='typeConfigurations')
-                            list-editable-item-edit.pc-form-grid-row
-                                - form = '$parent.form'
-                                .pc-form-grid-col-60
-                                    +java-class-autofocus('Type name:', '$item.typeName', '"typeName"', 'true', 'true', 'true', 'Type name')(
-                                        ignite-unique=items
-                                        ignite-unique-property='typeName'
-                                    )
-                                        +unique-feedback(`$item.typeName`, 'Type name should be unique.')
-                                .pc-form-grid-col-60
-                                    +java-class('ID mapper:', '$item.idMapper', '"idMapper"', 'true', 'false',
-                                        'Maps given from BinaryNameMapper type and filed name to ID that will be used by Ignite in internals<br/>\
-                                        Ignite never writes full strings for field or type/class names.\
-                                        Instead, for performance reasons, Ignite writes integer hash codes for type/class and field names.\
-                                        It has been tested that hash code conflicts for the type/class names or the field names within the same type are virtually non - existent and,\
-                                        to gain performance, it is safe to work with hash codes.\
-                                        For the cases when hash codes for different types or fields actually do collide <b>BinaryIdMapper</b> allows to override the automatically generated hash code IDs for the type and field names')
-                                .pc-form-grid-col-60
-                                    +java-class('Name mapper:', '$item.nameMapper', '"nameMapper"', 'true', 'false',
-                                        'Maps type/class and field names to different names')
-                                .pc-form-grid-col-60
-                                    +java-class('Serializer:', '$item.serializer', '"serializer"', 'true', 'false',
-                                        'Class with custom serialization logic for binary object')
-                                .pc-form-grid-col-60
-                                    +checkbox('Enum', '$item.enum', 'enum', 'Flag indicating that this type is the enum')
+                    +form-field__label({ label: 'Type configurations:', name: '"typeConfigurations"' })
+                        +form-field__tooltip({ title: `Configuration properties for binary types`})
 
-                            list-editable-no-items
-                                list-editable-add-item-button(
-                                    add-item=`$ctrl.Clusters.addBinaryTypeConfiguration($ctrl.clonedCluster)`
-                                    label-single='configuration'
-                                    label-multiple='configurations'
+                    -var items = model + '.typeConfigurations'
+                    list-editable.pc-list-editable-with-form-grid(ng-model=items name='typeConfigurations')
+                        list-editable-item-edit.pc-form-grid-row
+                            - form = '$parent.form'
+                            .pc-form-grid-col-60
+                                +form-field__java-class({
+                                    label: 'Type name:',
+                                    model: '$item.typeName',
+                                    name: '"typeName"',
+                                    required: 'true',
+                                    tip: 'Type name'
+                                })(
+                                    ignite-form-field-input-autofocus='true'
+                                    ignite-unique=items
+                                    ignite-unique-property='typeName'
                                 )
+                                    +form-field__error({ error: 'igniteUnique', message: 'Type name should be unique.' })
+                            .pc-form-grid-col-60
+                                +form-field__java-class({
+                                    label: 'ID mapper:',
+                                    model: '$item.idMapper',
+                                    name: '"idMapper"',
+                                    tip: 'Maps given from BinaryNameMapper type and filed name to ID that will be used by Ignite in internals<br/>\
+                                          Ignite never writes full strings for field or type/class names.\
+                                          Instead, for performance reasons, Ignite writes integer hash codes for type/class and field names.\
+                                          It has been tested that hash code conflicts for the type/class names or the field names within the same type are virtually non - existent and,\
+                                          to gain performance, it is safe to work with hash codes.\
+                                          For the cases when hash codes for different types or fields actually do collide <b>BinaryIdMapper</b> allows to override the automatically generated hash code IDs for the type and field names'
+                                })
+                            .pc-form-grid-col-60
+                                +form-field__java-class({
+                                    label: 'Name mapper:',
+                                    model: '$item.nameMapper',
+                                    name: '"nameMapper"',
+                                    tip: 'Maps type/class and field names to different names'
+                                })
+
+                            .pc-form-grid-col-60
+                                +form-field__java-class({
+                                    label: 'Serializer:',
+                                    model: '$item.serializer',
+                                    name: '"serializer"',
+                                    tip: 'Class with custom serialization logic for binary object'
+                                })
+                            .pc-form-grid-col-60
+                                +form-field__checkbox({
+                                    label: 'Enum',
+                                    model: '$item.enum',
+                                    name: 'enum',
+                                    tip: 'Flag indicating that this type is the enum'
+                                })
+
+                        list-editable-no-items
+                            list-editable-add-item-button(
+                                add-item=`$ctrl.Clusters.addBinaryTypeConfiguration($ctrl.clonedCluster)`
+                                label-single='configuration'
+                                label-multiple='configurations'
+                            )
 
             - form = 'binary'
             .pc-form-grid-col-60
-                +checkbox('Compact footer', model + '.compactFooter', '"compactFooter"', 'When enabled, Ignite will not write fields metadata when serializing objects (this will increase serialization performance), because internally <b>BinaryMarshaller</b> already distribute metadata inside cluster')
+                +form-field__checkbox({
+                    label: 'Compact footer',
+                    model: model + '.compactFooter',
+                    name: '"compactFooter"',
+                    tip: 'When enabled, Ignite will not write fields metadata when serializing objects (this will increase serialization performance), because internally <b>BinaryMarshaller</b> already distribute metadata inside cluster'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterBinary')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/cache-key-cfg.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/cache-key-cfg.pug
index 0b34ce44f5252..a17e52aaf5833 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/cache-key-cfg.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/cache-key-cfg.pug
@@ -25,39 +25,41 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
         | Cache key configuration allows to collocate objects in a partitioned cache based on field in cache key without explicit usage of annotations on user classes.
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6
-            mixin clusters-cache-key-cfg
-                .ignite-form-field
-                    +ignite-form-field__label('Cache key configuration:', '"cacheKeyConfiguration"')
-                    .ignite-form-field__control
-                        -let items = model
-                        list-editable.pc-list-editable-with-form-grid(ng-model=items name='cacheKeyConfiguration')
-                            list-editable-item-edit.pc-form-grid-row
-                                - form = '$parent.form'
-                                .pc-form-grid-col-60
-                                    +java-class-autofocus('Type name:', '$item.typeName', '"cacheKeyTypeName"', 'true', 'true', 'true', 'Type name')(
-                                        ignite-unique=items
-                                        ignite-unique-property='typeName'
-                                    )
-                                        +unique-feedback(`cacheKeyTypeName`, 'Type name should be unique.')
-                                .pc-form-grid-col-60
-                                    +sane-ignite-form-field-text({
-                                        label: 'Affinity key field name:',
-                                        model: '$item.affinityKeyFieldName',
-                                        name: '"affinityKeyFieldName"',
-                                        disabled: 'false',
-                                        placeholder: 'Enter field name',
-                                        tip: 'Affinity key field name',
-                                        required: true
-                                    })
-
-                            list-editable-no-items
-                                list-editable-add-item-button(
-                                    add-item=`(${items} = ${items} || []).push({})`
-                                    label-single='configuration'
-                                    label-multiple='configurations'
-                                )
-
-            +clusters-cache-key-cfg
+            .ignite-form-field
+                +form-field__label({ label: 'Cache key configuration:', name: '"cacheKeyConfiguration"' })
+
+                list-editable.pc-list-editable-with-form-grid(ng-model=model name='cacheKeyConfiguration')
+                    list-editable-item-edit.pc-form-grid-row
+                        - form = '$parent.form'
+                        .pc-form-grid-col-60
+                            +form-field__java-class({
+                                label: 'Type name:',
+                                model: '$item.typeName',
+                                name: '"cacheKeyTypeName"',
+                                required: 'true',
+                                tip: 'Type name'
+                            })(
+                                ignite-form-field-input-autofocus='true'
+                                ignite-unique=model
+                                ignite-unique-property='typeName'
+                            )
+                                +form-field__error({ error: 'igniteUnique', message: 'Type name should be unique.' })
+                        .pc-form-grid-col-60
+                            +form-field__text({
+                                label: 'Affinity key field name:',
+                                model: '$item.affinityKeyFieldName',
+                                name: '"affinityKeyFieldName"',
+                                placeholder: 'Enter field name',
+                                tip: 'Affinity key field name',
+                                required: true
+                            })
+
+                    list-editable-no-items
+                        list-editable-add-item-button(
+                            add-item=`(${model} = ${model} || []).push({})`
+                            label-single='configuration'
+                            label-multiple='configurations'
+                        )
 
         .pca-form-column-6
             +preview-xml-java(model, 'clusterCacheKeyConfiguration')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint.pug
index 7d56f14e7b209..760f99661aac6 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint.pug
@@ -24,59 +24,85 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form)
     panel-title Checkpointing
     panel-description
-        | Checkpointing provides an ability to save an intermediate job state. 
+        | Checkpointing provides an ability to save an intermediate job state.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/checkpointing" target="_blank") More info]
     panel-content.pca-form-row
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
                 .ignite-form-field
-                    +ignite-form-field__label('Checkpoint SPI configurations:', '"checkpointSPIConfigurations"')
-                    .ignite-form-field__control
-                        list-editable.pc-list-editable-with-form-grid(ng-model=model name='checkpointSPIConfigurations')
-                            list-editable-item-edit(item-name='$checkpointSPI').pc-form-grid-row
-                                .pc-form-grid-col-60
-                                    +dropdown-required('Checkpoint SPI:', '$checkpointSPI.kind', '"checkpointKind"', 'true', 'true', 'Choose checkpoint configuration variant', '[\
-                                            {value: "FS", label: "File System"},\
-                                            {value: "Cache", label: "Cache"},\
-                                            {value: "S3", label: "Amazon S3"},\
-                                            {value: "JDBC", label: "Database"},\
-                                            {value: "Custom", label: "Custom"}\
-                                        ]',
-                                        'Provides an ability to save an intermediate job state\
-                                        <ul>\
+                    +form-field__label({ label: 'Checkpoint SPI configurations:', name: '"checkpointSPIConfigurations"' })
+
+                    list-editable.pc-list-editable-with-form-grid(ng-model=model name='checkpointSPIConfigurations')
+                        list-editable-item-edit(item-name='$checkpointSPI').pc-form-grid-row
+                            .pc-form-grid-col-60
+                                +form-field__dropdown({
+                                    label: 'Checkpoint SPI:',
+                                    model: '$checkpointSPI.kind',
+                                    name: '"checkpointKind"',
+                                    required: 'true',
+                                    placeholder: 'Choose checkpoint configuration variant',
+                                    options: '[\
+                                                {value: "FS", label: "File System"},\
+                                                {value: "Cache", label: "Cache"},\
+                                                {value: "S3", label: "Amazon S3"},\
+                                                {value: "JDBC", label: "Database"},\
+                                                {value: "Custom", label: "Custom"}\
+                                               ]',
+                                    tip: 'Provides an ability to save an intermediate job state\
+                                          <ul>\
                                             <li>File System - Uses a shared file system to store checkpoints</li>\
                                             <li>Cache - Uses a cache to store checkpoints</li>\
                                             <li>Amazon S3 - Uses Amazon S3 to store checkpoints</li>\
                                             <li>Database - Uses a database to store checkpoints</li>\
                                             <li>Custom - Custom checkpoint SPI implementation</li>\
-                                        </ul>')
+                                          </ul>'
+                                })
+
+                            include ./checkpoint/fs
 
-                                include ./checkpoint/fs
+                            .pc-form-grid-col-60(ng-if-start=CacheCheckpoint)
+                                +form-field__dropdown({
+                                    label: 'Cache:',
+                                    model: '$checkpointSPI.Cache.cache',
+                                    name: '"checkpointCacheCache"',
+                                    required: CacheCheckpoint,
+                                    placeholder: 'Choose cache',
+                                    placeholderEmpty: 'No caches configured for current cluster',
+                                    options: '$ctrl.cachesMenu',
+                                    tip: 'Cache to use for storing checkpoints'
+                                })(
+                                    pc-is-in-collection='$ctrl.clonedCluster.caches'
+                                )
+                                    +form-field__error({ error: 'isInCollection', message: `Cluster doesn't have such a cache` })
+                            .pc-form-grid-col-60(ng-if-end)
+                                +form-field__java-class({
+                                    label: 'Listener:',
+                                    model: '$checkpointSPI.Cache.checkpointListener',
+                                    name: '"checkpointCacheListener"',
+                                    tip: 'Checkpoint listener implementation class name',
+                                    validationActive: CacheCheckpoint
+                                })
 
-                                .pc-form-grid-col-60(ng-if-start=CacheCheckpoint)
-                                    +dropdown-required-empty('Cache:', '$checkpointSPI.Cache.cache', '"checkpointCacheCache"', 'true', CacheCheckpoint,
-                                        'Choose cache', 'No caches configured for current cluster', '$ctrl.cachesMenu', 'Cache to use for storing checkpoints')(
-                                        pc-is-in-collection='$ctrl.clonedCluster.caches'
-                                    )
-                                        +form-field-feedback(form, 'isInCollection', `Cluster doesn't have such a cache`)
-                                .pc-form-grid-col-60(ng-if-end)
-                                    +java-class('Listener:', '$checkpointSPI.Cache.checkpointListener', '"checkpointCacheListener"', 'true', 'false',
-                                        'Checkpoint listener implementation class name', CacheCheckpoint)
+                            include ./checkpoint/s3
 
-                                include ./checkpoint/s3
+                            include ./checkpoint/jdbc
 
-                                include ./checkpoint/jdbc
+                            .pc-form-grid-col-60(ng-if=CustomCheckpoint)
+                                +form-field__java-class({
+                                    label: 'Class name:',
+                                    model: '$checkpointSPI.Custom.className',
+                                    name: '"checkpointCustomClassName"',
+                                    required: CustomCheckpoint,
+                                    tip: 'Custom CheckpointSpi implementation class',
+                                    validationActive: CustomCheckpoint
+                                })
 
-                                .pc-form-grid-col-60(ng-if=CustomCheckpoint)
-                                    +java-class('Class name:', '$checkpointSPI.Custom.className', '"checkpointCustomClassName"', 'true', CustomCheckpoint,
-                                    'Custom CheckpointSpi implementation class', CustomCheckpoint)
+                        list-editable-no-items
+                            list-editable-add-item-button(
+                                add-item=`$edit($ctrl.Clusters.addCheckpointSPI($ctrl.clonedCluster))`
+                                label-single='checkpoint SPI configuration'
+                                label-multiple='checkpoint SPI configurations'
+                            )
 
-                            list-editable-no-items
-                                list-editable-add-item-button(
-                                    add-item=`$edit($ctrl.Clusters.addCheckpointSPI($ctrl.clonedCluster))`
-                                    label-single='checkpoint SPI configuration'
-                                    label-multiple='checkpoint SPI configurations'
-                                )
-        
         .pca-form-column-6
             +preview-xml-java('$ctrl.clonedCluster', 'clusterCheckpoint', '$ctrl.caches')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/fs.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/fs.pug
index 0359cf309959f..0cda6fa7914b2 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/fs.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/fs.pug
@@ -30,7 +30,13 @@ include /app/helpers/jade/mixins
                 tip: 'Paths to a shared directory where checkpoints will be stored'
             }]`
         )
-            +unique-feedback(_, 'Such path already exists!')
+            +form-field__error({ error: 'igniteUnique', message: 'Such path already exists!' })
 
 .pc-form-grid-col-60(ng-if-end)
-    +java-class('Listener:', '$checkpointSPI.FS.checkpointListener', '"checkpointFsListener"', 'true', 'false', 'Checkpoint listener implementation class name', '$checkpointSPI.kind === "FS"')
+    +form-field__java-class({
+        label: 'Listener:',
+        model: '$checkpointSPI.FS.checkpointListener',
+        name: '"checkpointFsListener"',
+        tip: 'Checkpoint listener implementation class name',
+        validationActive: '$checkpointSPI.kind === "FS"'
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/jdbc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/jdbc.pug
index 00a868165a434..be4afc4ace0f2 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/jdbc.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/jdbc.pug
@@ -19,29 +19,106 @@ include /app/helpers/jade/mixins
 -var jdbcCheckpoint = '$checkpointSPI.kind === "JDBC"'
 
 .pc-form-grid-col-30(ng-if-start='$checkpointSPI.kind === "JDBC"')
-    +text('Data source bean name:', '$checkpointSPI.JDBC.dataSourceBean', '"checkpointJdbcDataSourceBean"', jdbcCheckpoint, 'Input bean name',
-    'Name of the data source bean in Spring context')
+    +form-field__text({
+        label: 'Data source bean name:',
+        model: '$checkpointSPI.JDBC.dataSourceBean',
+        name: '"checkpointJdbcDataSourceBean"',
+        required: jdbcCheckpoint,
+        placeholder: 'Input bean name',
+        tip: 'Name of the data source bean in Spring context'
+    })
 .pc-form-grid-col-30
-    +dialect('Dialect:', '$checkpointSPI.JDBC.dialect', '"checkpointJdbcDialect"', jdbcCheckpoint,
-    'Dialect of SQL implemented by a particular RDBMS:', 'Generic JDBC dialect', 'Choose JDBC dialect')
+    +form-field__dialect({
+        label: 'Dialect:',
+        model: '$checkpointSPI.JDBC.dialect',
+        name: '"checkpointJdbcDialect"',
+        required: jdbcCheckpoint,
+        tip: 'Dialect of SQL implemented by a particular RDBMS:',
+        genericDialectName: 'Generic JDBC dialect',
+        placeholder: 'Choose JDBC dialect'
+    })
+
 .pc-form-grid-col-60
-    +java-class('Listener:', '$checkpointSPI.JDBC.checkpointListener', '"checkpointJdbcListener"', 'true', 'false',
-        'Checkpoint listener implementation class name', jdbcCheckpoint)
+    +form-field__java-class({
+        label: 'Listener:',
+        model: '$checkpointSPI.JDBC.checkpointListener',
+        name: '"checkpointJdbcListener"',
+        tip: 'Checkpoint listener implementation class name',
+        validationActive: jdbcCheckpoint
+    })
 .pc-form-grid-col-60
-    +text('User:', '$checkpointSPI.JDBC.user', '"checkpointJdbcUser"', 'false', 'Input user name', 'Checkpoint jdbc user name')
+    +form-field__text({
+        label: 'User:',
+        model: '$checkpointSPI.JDBC.user',
+        name: '"checkpointJdbcUser"',
+        placeholder: 'Input user name',
+        tip: 'Checkpoint jdbc user name'
+    })
 .pc-form-grid-col-30
-    +text('Table name:', '$checkpointSPI.JDBC.checkpointTableName', '"checkpointJdbcCheckpointTableName"', 'false', 'CHECKPOINTS', 'Checkpoint table name')
+    +form-field__text({
+        label: 'Table name:',
+        model: '$checkpointSPI.JDBC.checkpointTableName',
+        name: '"checkpointJdbcCheckpointTableName"',
+        placeholder: 'CHECKPOINTS',
+        tip: 'Checkpoint table name'
+    })
 .pc-form-grid-col-30
-    +number('Number of retries:', '$checkpointSPI.JDBC.numberOfRetries', '"checkpointJdbcNumberOfRetries"', 'true', '2', '0', 'Number of retries in case of DB failure')
+    +form-field__number({
+        label: 'Number of retries:',
+        model: '$checkpointSPI.JDBC.numberOfRetries',
+        name: '"checkpointJdbcNumberOfRetries"',
+        placeholder: '2',
+        min: '0',
+        tip: 'Number of retries in case of DB failure'
+    })
 .pc-form-grid-col-30
-    +text('Key field name:', '$checkpointSPI.JDBC.keyFieldName', '"checkpointJdbcKeyFieldName"', 'false', 'NAME', 'Checkpoint key field name')
+    +form-field__text({
+        label: 'Key field name:',
+        model: '$checkpointSPI.JDBC.keyFieldName',
+        name: '"checkpointJdbcKeyFieldName"',
+        placeholder: 'NAME',
+        tip: 'Checkpoint key field name'
+    })
 .pc-form-grid-col-30
-    +dropdown('Key field type:', '$checkpointSPI.JDBC.keyFieldType', '"checkpointJdbcKeyFieldType"', 'true', 'VARCHAR', '::$ctrl.supportedJdbcTypes', 'Checkpoint key field type')
+    +form-field__dropdown({
+        label: 'Key field type:',
+        model: '$checkpointSPI.JDBC.keyFieldType',
+        name: '"checkpointJdbcKeyFieldType"',
+        placeholder: 'VARCHAR',
+        options: '::$ctrl.supportedJdbcTypes',
+        tip: 'Checkpoint key field type'
+    })
 .pc-form-grid-col-30
-    +text('Value field name:', '$checkpointSPI.JDBC.valueFieldName', '"checkpointJdbcValueFieldName"', 'false', 'VALUE', 'Checkpoint value field name')
+    +form-field__text({
+        label: 'Value field name:',
+        model: '$checkpointSPI.JDBC.valueFieldName',
+        name: '"checkpointJdbcValueFieldName"',
+        placeholder: 'VALUE',
+        tip: 'Checkpoint value field name'
+    })
 .pc-form-grid-col-30
-    +dropdown('Value field type:', '$checkpointSPI.JDBC.valueFieldType', '"checkpointJdbcValueFieldType"', 'true', 'BLOB', '::$ctrl.supportedJdbcTypes', 'Checkpoint value field type')
+    +form-field__dropdown({
+        label: 'Value field type:',
+        model: '$checkpointSPI.JDBC.valueFieldType',
+        name: '"checkpointJdbcValueFieldType"',
+        placeholder: 'BLOB',
+        options: '::$ctrl.supportedJdbcTypes',
+        tip: 'Checkpoint value field type'
+    })
 .pc-form-grid-col-30
-    +text('Expire date field name:', '$checkpointSPI.JDBC.expireDateFieldName', '"checkpointJdbcExpireDateFieldName"', 'false', 'EXPIRE_DATE', 'Checkpoint expire date field name')
+    +form-field__text({
+        label:'Expire date field name:',
+        model: '$checkpointSPI.JDBC.expireDateFieldName',
+        name: '"checkpointJdbcExpireDateFieldName"',
+        placeholder: 'EXPIRE_DATE',
+        tip: 'Checkpoint expire date field name'
+    })
 .pc-form-grid-col-30(ng-if-end)
-    +dropdown('Expire date field type:', '$checkpointSPI.JDBC.expireDateFieldType', '"checkpointJdbcExpireDateFieldType"', 'true', 'DATETIME', '::$ctrl.supportedJdbcTypes', 'Checkpoint expire date field type')
+    +form-field__dropdown({
+        label: 'Expire date field type:',
+        model: '$checkpointSPI.JDBC.expireDateFieldType',
+        name: '"checkpointJdbcExpireDateFieldType"',
+        placeholder: 'DATETIME',
+        options: '::$ctrl.supportedJdbcTypes',
+        tip: 'Checkpoint expire date field type'
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/s3.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/s3.pug
index 8e284fcb4f9b7..1f6eef2dd5160 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/s3.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/checkpoint/s3.pug
@@ -28,177 +28,416 @@ include /app/helpers/jade/mixins
 -var checkpointS3CustomRetry = checkpointS3 + ' && ' + clientRetryModel + '.kind === "Custom"'
 
 .pc-form-grid-col-60(ng-if-start='$checkpointSPI.kind === "S3"')
-    +dropdown-required('AWS credentials:', '$checkpointSPI.S3.awsCredentials.kind', '"checkpointS3AwsCredentials"', 'true', checkpointS3, 'Custom', '[\
-        {value: "Basic", label: "Basic"},\
-        {value: "Properties", label: "Properties"},\
-        {value: "Anonymous", label: "Anonymous"},\
-        {value: "BasicSession", label: "Basic with session"},\
-        {value: "Custom", label: "Custom"}\
-    ]',
-    'AWS credentials\
-    <ul>\
-        <li>Basic - Allows callers to pass in the AWS access key and secret access in the constructor</li>\
-        <li>Properties - Reads in AWS access keys from a properties file</li>\
-        <li>Anonymous - Allows use of "anonymous" credentials</li>\
-        <li>Database - Session credentials with keys and session token</li>\
-        <li>Custom - Custom AWS credentials provider</li>\
-    </ul>')
+    +form-field__dropdown({
+        label: 'AWS credentials:',
+        model: '$checkpointSPI.S3.awsCredentials.kind',
+        name: '"checkpointS3AwsCredentials"',
+        required: checkpointS3,
+        placeholder: 'Custom',
+        options: '[\
+                {value: "Basic", label: "Basic"},\
+                {value: "Properties", label: "Properties"},\
+                {value: "Anonymous", label: "Anonymous"},\
+                {value: "BasicSession", label: "Basic with session"},\
+                {value: "Custom", label: "Custom"}\
+            ]',
+        tip: 'AWS credentials\
+            <ul>\
+                <li>Basic - Allows callers to pass in the AWS access key and secret access in the constructor</li>\
+                <li>Properties - Reads in AWS access keys from a properties file</li>\
+                <li>Anonymous - Allows use of "anonymous" credentials</li>\
+                <li>Database - Session credentials with keys and session token</li>\
+                <li>Custom - Custom AWS credentials provider</li>\
+            </ul>'
+    })
+
 .pc-form-group.pc-form-grid-row(ng-if=checkpointS3Path)
     .pc-form-grid-col-60
-        +text('Path:', credentialsModel + '.Properties.path', '"checkpointS3PropertiesPath"', checkpointS3Path, 'Input properties file path',
-        'The file from which to read the AWS credentials properties')
+        +form-field__text({
+            label: 'Path:',
+            model: `${credentialsModel}.Properties.path`,
+            name: '"checkpointS3PropertiesPath"',
+            required: checkpointS3Path,
+            placeholder: 'Input properties file path',
+            tip: 'The file from which to read the AWS credentials properties'
+        })
 .pc-form-group.pc-form-grid-row(ng-if=checkpointS3Custom)
     .pc-form-grid-col-60
-        +java-class('Class name:', credentialsModel + '.Custom.className', '"checkpointS3CustomClassName"', 'true', checkpointS3Custom,
-        'Custom AWS credentials provider implementation class', checkpointS3Custom)
+        +form-field__java-class({
+            label: 'Class name:',
+            model: credentialsModel + '.Custom.className',
+            name: '"checkpointS3CustomClassName"',
+            required: checkpointS3Custom,
+            tip: 'Custom AWS credentials provider implementation class',
+            validationActive:checkpointS3Custom
+        })
 .pc-form-grid-col-60
     label Note, AWS credentials will be generated as stub
 .pc-form-grid-col-60
-    +text('Bucket name suffix:', '$checkpointSPI.S3.bucketNameSuffix', '"checkpointS3BucketNameSuffix"', 'false', 'default-bucket')
+    +form-field__text({
+        label: 'Bucket name suffix:',
+        model: '$checkpointSPI.S3.bucketNameSuffix',
+        name: '"checkpointS3BucketNameSuffix"',
+        placeholder: 'default-bucket'
+    })
 .pc-form-grid-col-60(ng-if-start=`$ctrl.available("2.4.0")`)
-    +text('Bucket endpoint:', `$checkpointSPI.S3.bucketEndpoint`, '"checkpointS3BucketEndpoint"', false, 'Input bucket endpoint',
-    'Bucket endpoint for IP finder<br/> \
-    For information about possible endpoint names visit <a href="http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region">docs.aws.amazon.com</a>')
+    +form-field__text({
+        label: 'Bucket endpoint:',
+        model: `$checkpointSPI.S3.bucketEndpoint`,
+        name: '"checkpointS3BucketEndpoint"',
+        placeholder: 'Input bucket endpoint',
+        tip: 'Bucket endpoint for IP finder<br/> \
+            For information about possible endpoint names visit <a href="http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region">docs.aws.amazon.com</a>'
+    })
 .pc-form-grid-col-60(ng-if-end)
-    +text('SSE algorithm:', `$checkpointSPI.S3.SSEAlgorithm`, '"checkpointS3SseAlgorithm"', false, 'Input SSE algorithm',
-    'Server-side encryption algorithm for Amazon S3-managed encryption keys<br/> \
-    For information about possible S3-managed encryption keys visit <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html">docs.aws.amazon.com</a>')
+    +form-field__text({
+        label: 'SSE algorithm:',
+        model: `$checkpointSPI.S3.SSEAlgorithm`,
+        name: '"checkpointS3SseAlgorithm"',
+        placeholder: 'Input SSE algorithm',
+        tip: 'Server-side encryption algorithm for Amazon S3-managed encryption keys<br/> \
+              For information about possible S3-managed encryption keys visit <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html">docs.aws.amazon.com</a>'
+    })
 .pc-form-grid-col-60
-    +java-class('Listener:', '$checkpointSPI.S3.checkpointListener', '"checkpointS3Listener"', 'true', 'false',
-        'Checkpoint listener implementation class name', checkpointS3)
+    +form-field__java-class({
+        label: 'Listener:',
+        model: '$checkpointSPI.S3.checkpointListener',
+        name: '"checkpointS3Listener"',
+        tip: 'Checkpoint listener implementation class name',
+        validationActive: checkpointS3
+    })
 .pc-form-grid-col-60.pc-form-group__text-title
     span Client configuration
 .pc-form-group.pc-form-grid-row(ng-if-end)
     .pc-form-grid-col-30
-        +dropdown('Protocol:', clientCfgModel + '.protocol', '"checkpointS3Protocol"', 'true', 'HTTPS', '[\
-                {value: "HTTP", label: "HTTP"},\
-                {value: "HTTPS", label: "HTTPS"}\
-            ]',
-        'Provides an ability to save an intermediate job state\
-        <ul>\
-            <li>HTTP - Using the HTTP protocol is less secure than HTTPS, but can slightly reduce\
-                the system resources used when communicating with AWS</li>\
-            <li>HTTPS - Using the HTTPS protocol is more secure than using the HTTP protocol, but\
-                may use slightly more system resources. AWS recommends using HTTPS for maximize security</li>\
-        </ul>')
+        +form-field__dropdown({
+            label: 'Protocol:',
+            model: clientCfgModel + '.protocol',
+            name: '"checkpointS3Protocol"',
+            placeholder: 'HTTPS',
+            options: '[\
+                            {value: "HTTP", label: "HTTP"},\
+                            {value: "HTTPS", label: "HTTPS"}\
+                        ]',
+            tip: 'Provides an ability to save an intermediate job state\
+                    <ul>\
+                        <li>HTTP - Using the HTTP protocol is less secure than HTTPS, but can slightly reduce\
+                            the system resources used when communicating with AWS</li>\
+                        <li>HTTPS - Using the HTTPS protocol is more secure than using the HTTP protocol, but\
+                            may use slightly more system resources. AWS recommends using HTTPS for maximize security</li>\
+                    </ul>'
+        })
     .pc-form-grid-col-30
-        +number('Maximum connections:', clientCfgModel + '.maxConnections', '"checkpointS3MaxConnections"',
-        'true', '50', '1', 'Maximum number of allowed open HTTP connections')
+        +form-field__number({
+            label:'Maximum connections:',
+            model:clientCfgModel + '.maxConnections',
+            name: '"checkpointS3MaxConnections"',
+            placeholder: '50',
+            min: '1',
+            tip: 'Maximum number of allowed open HTTP connections'
+        })
     .pc-form-grid-col-60
-        +text('User agent prefix:', clientCfgModel + '.userAgentPrefix', '"checkpointS3UserAgentPrefix"', 'false', 'System specific header',
-        'HTTP user agent prefix to send with all requests')
+        +form-field__text({
+            label: 'User agent prefix:',
+            model: `${clientCfgModel}.userAgentPrefix`,
+            name: '"checkpointS3UserAgentPrefix"',
+            placeholder: 'System specific header',
+            tip: 'HTTP user agent prefix to send with all requests'
+        })
     .pc-form-grid-col-60
-        +text('User agent suffix:', clientCfgModel + '.userAgentSuffix', '"checkpointS3UserAgentSuffix"', 'false', 'System specific header',
-        'HTTP user agent suffix to send with all requests')
+        +form-field__text({
+            label: 'User agent suffix:',
+            model: `${clientCfgModel}.userAgentSuffix`,
+            name: '"checkpointS3UserAgentSuffix"',
+            placeholder: 'System specific header',
+            tip: 'HTTP user agent suffix to send with all requests'
+        })
     .pc-form-grid-col-60
-        +text-ip-address('Local address:', clientCfgModel + '.localAddress', '"checkpointS3LocalAddress"', 'true', 'Not specified',
-        'Optionally specifies the local address to bind to')
+        +form-field__ip-address({
+            label: 'Local address:',
+            model: clientCfgModel + '.localAddress',
+            name: '"checkpointS3LocalAddress"',
+            enabled: 'true',
+            placeholder: 'Not specified',
+            tip: 'Optionally specifies the local address to bind to'
+        })
     .pc-form-grid-col-40
-        +text('Proxy host:', clientCfgModel + '.proxyHost', '"checkpointS3ProxyHost"', 'false', 'Not specified',
-        'Optional proxy host the client will connect through')
+        +form-field__text({
+            label: 'Proxy host:',
+            model: `${clientCfgModel}.proxyHost`,
+            name: '"checkpointS3ProxyHost"',
+            placeholder: 'Not specified',
+            tip: 'Optional proxy host the client will connect through'
+        })
     .pc-form-grid-col-20
-        +number('Proxy port:', clientCfgModel + '.proxyPort', '"checkpointS3ProxyPort"', 'true', 'Not specified', '0',
-        'Optional proxy port the client will connect through')
+        +form-field__number({
+            label: 'Proxy port:',
+            model: clientCfgModel + '.proxyPort',
+            name: '"checkpointS3ProxyPort"',
+            placeholder: 'Not specified',
+            min: '0',
+            tip: 'Optional proxy port the client will connect through'
+        })
     .pc-form-grid-col-30
-        +text('Proxy user:', clientCfgModel + '.proxyUsername', '"checkpointS3ProxyUsername"', 'false', 'Not specified',
-        'Optional proxy user name to use if connecting through a proxy')
+        +form-field__text({
+            label: 'Proxy user:',
+            model: clientCfgModel + '.proxyUsername',
+            name: '"checkpointS3ProxyUsername"',
+            placeholder: 'Not specified',
+            tip: 'Optional proxy user name to use if connecting through a proxy'
+        })
     .pc-form-grid-col-30
-        +text('Proxy domain:', clientCfgModel + '.proxyDomain', '"checkpointS3ProxyDomain"', 'false', 'Not specified',
-        'Optional Windows domain name for configuring an NTLM proxy')
-    .pc-form-grid-col-60
-        +text('Proxy workstation:', clientCfgModel + '.proxyWorkstation', '"checkpointS3ProxyWorkstation"', 'false', 'Not specified',
-        'Optional Windows workstation name for configuring NTLM proxy support')
-    .pc-form-grid-col-60
-        +text('Non proxy hosts:', clientCfgModel + '.nonProxyHosts', '"checkpointS3NonProxyHosts"', 'false', 'Not specified',
-        'Optional hosts the client will access without going through the proxy')
-    .pc-form-grid-col-60
-        +dropdown('Retry policy:', clientRetryModel + '.kind', '"checkpointS3RetryPolicy"', 'true', 'Default', '[\
-                                            {value: "Default", label: "Default SDK retry policy"},\
-                                            {value: "DefaultMaxRetries", label: "Default with the specified max retry count"},\
-                                            {value: "DynamoDB", label: "Default for DynamoDB client"},\
-                                            {value: "DynamoDBMaxRetries", label: "DynamoDB with the specified max retry count"},\
-                                            {value: "Custom", label: "Custom configured"}\
-                                        ]',
-        'Provides an ability to save an intermediate job state\
-        <ul>\
-            <li>SDK default retry policy - This policy will honor the maxErrorRetry set in ClientConfiguration</li>\
-            <li>Default with the specified max retry count - Default SDK retry policy with the specified max retry count</li>\
-            <li>Default for DynamoDB client - This policy will honor the maxErrorRetry set in ClientConfiguration</li>\
-            <li>DynamoDB with the specified max retry count - This policy will honor the maxErrorRetry set in ClientConfiguration with the specified max retry count</li>\
-            <li>Custom configured - Custom configured SDK retry policy</li>\
-        </ul>')
+        +form-field__text({
+            label: 'Proxy domain:',
+            model: `${clientCfgModel}.proxyDomain`,
+            name: '"checkpointS3ProxyDomain"',
+            placeholder: 'Not specified',
+            tip: 'Optional Windows domain name for configuring an NTLM proxy'
+        })
+    .pc-form-grid-col-60
+        +form-field__text({
+            label: 'Proxy workstation:',
+            model: `${clientCfgModel}.proxyWorkstation`,
+            name: '"checkpointS3ProxyWorkstation"',
+            placeholder: 'Not specified',
+            tip: 'Optional Windows workstation name for configuring NTLM proxy support'
+        })
+    .pc-form-grid-col-60
+        +form-field__text({
+            label: 'Non proxy hosts:',
+            model: `${clientCfgModel}.nonProxyHosts`,
+            name: '"checkpointS3NonProxyHosts"',
+            placeholder: 'Not specified',
+            tip: 'Optional hosts the client will access without going through the proxy'
+        })
+    .pc-form-grid-col-60
+        +form-field__dropdown({
+            label: 'Retry policy:',
+            model: `${clientRetryModel}.kind`,
+            name: '"checkpointS3RetryPolicy"',
+            placeholder: 'Default',
+            options: '[\
+                                                        {value: "Default", label: "Default SDK retry policy"},\
+                                                        {value: "DefaultMaxRetries", label: "Default with the specified max retry count"},\
+                                                        {value: "DynamoDB", label: "Default for DynamoDB client"},\
+                                                        {value: "DynamoDBMaxRetries", label: "DynamoDB with the specified max retry count"},\
+                                                        {value: "Custom", label: "Custom configured"}\
+                                                    ]',
+            tip: 'Provides an ability to save an intermediate job state\
+                    <ul>\
+                        <li>SDK default retry policy - This policy will honor the maxErrorRetry set in ClientConfiguration</li>\
+                        <li>Default with the specified max retry count - Default SDK retry policy with the specified max retry count</li>\
+                        <li>Default for DynamoDB client - This policy will honor the maxErrorRetry set in ClientConfiguration</li>\
+                        <li>DynamoDB with the specified max retry count - This policy will honor the maxErrorRetry set in ClientConfiguration with the specified max retry count</li>\
+                        <li>Custom configured - Custom configured SDK retry policy</li>\
+                    </ul>'
+        })
     .pc-form-group.pc-form-grid-row(ng-if=checkpointS3DefaultMaxRetry)
         .pc-form-grid-col-60
-            +number-required('Maximum retry attempts:', clientRetryModel + '.DefaultMaxRetries.maxErrorRetry', '"checkpointS3DefaultMaxErrorRetry"', 'true', checkpointS3DefaultMaxRetry, '-1', '1',
-            'Maximum number of retry attempts for failed requests')
+            +form-field__number({
+                label: 'Maximum retry attempts:',
+                model: clientRetryModel + '.DefaultMaxRetries.maxErrorRetry',
+                name: '"checkpointS3DefaultMaxErrorRetry"',
+                required: checkpointS3DefaultMaxRetry,
+                placeholder: '-1',
+                min: '1',
+                tip: 'Maximum number of retry attempts for failed requests'
+            })
     .pc-form-group.pc-form-grid-row(ng-if=checkpointS3DynamoDbMaxRetry)
         .pc-form-grid-col-60
-            +number-required('Maximum retry attempts:', clientRetryModel + '.DynamoDBMaxRetries.maxErrorRetry', '"checkpointS3DynamoDBMaxErrorRetry"', 'true', checkpointS3DynamoDbMaxRetry, '-1', '1',
-            'Maximum number of retry attempts for failed requests')
+            +form-field__number({
+                label: 'Maximum retry attempts:',
+                model: clientRetryModel + '.DynamoDBMaxRetries.maxErrorRetry',
+                name: '"checkpointS3DynamoDBMaxErrorRetry"',
+                required: checkpointS3DynamoDbMaxRetry,
+                placeholder: '-1',
+                min: '1',
+                tip: 'Maximum number of retry attempts for failed requests'
+            })
     .pc-form-group.pc-form-grid-row(ng-if=checkpointS3CustomRetry)
         .pc-form-grid-col-60
-            +java-class('Retry condition:', clientRetryModel + '.Custom.retryCondition', '"checkpointS3CustomRetryPolicy"', 'true', checkpointS3CustomRetry,
-            'Retry condition on whether a specific request and exception should be retried', checkpointS3CustomRetry)
+            +form-field__java-class({
+                label: 'Retry condition:',
+                model: clientRetryModel + '.Custom.retryCondition',
+                name: '"checkpointS3CustomRetryPolicy"',
+                required: checkpointS3CustomRetry,
+                tip: 'Retry condition on whether a specific request and exception should be retried',
+                validationActive: checkpointS3CustomRetry
+            })
         .pc-form-grid-col-60
-            +java-class('Backoff strategy:', clientRetryModel + '.Custom.backoffStrategy', '"checkpointS3CustomBackoffStrategy"', 'true', checkpointS3CustomRetry,
-            'Back-off strategy for controlling how long the next retry should wait', checkpointS3CustomRetry)
+            +form-field__java-class({
+                label: 'Backoff strategy:',
+                model: clientRetryModel + '.Custom.backoffStrategy',
+                name: '"checkpointS3CustomBackoffStrategy"',
+                required: checkpointS3CustomRetry,
+                tip: 'Back-off strategy for controlling how long the next retry should wait',
+                validationActive: checkpointS3CustomRetry
+            })
         .pc-form-grid-col-60
-            +number-required('Maximum retry attempts:', clientRetryModel + '.Custom.maxErrorRetry', '"checkpointS3CustomMaxErrorRetry"', 'true', checkpointS3CustomRetry, '-1', '1',
-            'Maximum number of retry attempts for failed requests')
+            +form-field__number({
+                label: 'Maximum retry attempts:',
+                model: clientRetryModel + '.Custom.maxErrorRetry',
+                name: '"checkpointS3CustomMaxErrorRetry"',
+                required: checkpointS3CustomRetry,
+                placeholder: '-1',
+                min: '1',
+                tip: 'Maximum number of retry attempts for failed requests'
+            })
         .pc-form-grid-col-60
-            +checkbox('Honor the max error retry set', clientRetryModel + '.Custom.honorMaxErrorRetryInClientConfig', '"checkpointS3CustomHonorMaxErrorRetryInClientConfig"',
-            'Whether this retry policy should honor the max error retry set by ClientConfiguration#setMaxErrorRetry(int)')
+            +form-field__checkbox({
+                label: 'Honor the max error retry set',
+                model: clientRetryModel + '.Custom.honorMaxErrorRetryInClientConfig',
+                name: '"checkpointS3CustomHonorMaxErrorRetryInClientConfig"',
+                tip: 'Whether this retry policy should honor the max error retry set by ClientConfiguration#setMaxErrorRetry(int)'
+            })
     .pc-form-grid-col-60
-        +number('Maximum retry attempts:', clientCfgModel + '.maxErrorRetry', '"checkpointS3MaxErrorRetry"', 'true', '-1', '0',
-        'Maximum number of retry attempts for failed retryable requests<br/>\
-        If -1 the configured RetryPolicy will be used to control the retry count')
+        +form-field__number({
+            label: 'Maximum retry attempts:',
+            model: `${clientCfgModel}.maxErrorRetry`,
+            name: '"checkpointS3MaxErrorRetry"',
+            placeholder: '-1',
+            min: '0',
+            tip: 'Maximum number of retry attempts for failed retryable requests<br/>\
+                  If -1 the configured RetryPolicy will be used to control the retry count'
+        })
     .pc-form-grid-col-30
-        +number('Socket timeout:', clientCfgModel + '.socketTimeout', '"checkpointS3SocketTimeout"', 'true', '50000', '0',
-        'Amount of time in milliseconds to wait for data to be transfered over an established, open connection before the connection times out and is closed<br/>\
-        A value of <b>0</b> means infinity')
+        +form-field__number({
+            label: 'Socket timeout:',
+            model: `${clientCfgModel}.socketTimeout`,
+            name: '"checkpointS3SocketTimeout"',
+            placeholder: '50000',
+            min: '0',
+            tip: 'Amount of time in milliseconds to wait for data to be transfered over an established, open connection before the connection times out and is closed<br/>\
+                  A value of <b>0</b> means infinity'
+        })
     .pc-form-grid-col-30
-        +number('Connection timeout:', clientCfgModel + '.connectionTimeout', '"checkpointS3ConnectionTimeout"', 'true', '50000', '0',
-        'Amount of time in milliseconds to wait when initially establishing a connection before giving up and timing out<br/>\
-        A value of <b>0</b> means infinity')
+        +form-field__number({
+            label: 'Connection timeout:',
+            model: `${clientCfgModel}.connectionTimeout`,
+            name: '"checkpointS3ConnectionTimeout"',
+            placeholder: '50000',
+            min: '0',
+            tip: 'Amount of time in milliseconds to wait when initially establishing a connection before giving up and timing out<br/>\
+                  A value of <b>0</b> means infinity'
+        })
     .pc-form-grid-col-30
-        +number('Request timeout:', clientCfgModel + '.requestTimeout', '"checkpointS3RequestTimeout"', 'true', '0', '-1',
-        'Amount of time in milliseconds to wait for the request to complete before giving up and timing out<br/>\
-        A non - positive value means infinity')
+        +form-field__number({
+            label: 'Request timeout:',
+            model: `${clientCfgModel}.requestTimeout`,
+            name: '"checkpointS3RequestTimeout"',
+            placeholder: '0',
+            min: '-1',
+            tip: 'Amount of time in milliseconds to wait for the request to complete before giving up and timing out<br/>\
+                  A non - positive value means infinity'
+        })
     .pc-form-grid-col-30
-        +number('Idle timeout:', clientCfgModel + '.connectionMaxIdleMillis', '"checkpointS3ConnectionMaxIdleMillis"', 'true', '60000', '0',
-        'Maximum amount of time that an idle connection may sit in the connection pool and still be eligible for reuse')
+        +form-field__number({
+            label: 'Idle timeout:',
+            model: `${clientCfgModel}.connectionMaxIdleMillis`,
+            name: '"checkpointS3ConnectionMaxIdleMillis"',
+            placeholder: '60000',
+            min: '0',
+            tip: 'Maximum amount of time that an idle connection may sit in the connection pool and still be eligible for reuse'
+        })
     .pc-form-grid-col-30
-        +text('Signature algorithm:', clientCfgModel + '.signerOverride', '"checkpointS3SignerOverride"', 'false', 'Not specified',
-        'Name of the signature algorithm to use for signing requests made by this client')
+        +form-field__text({
+            label: 'Signature algorithm:',
+            model: `${clientCfgModel}.signerOverride`,
+            name: '"checkpointS3SignerOverride"',
+            placeholder: 'Not specified',
+            tip: 'Name of the signature algorithm to use for signing requests made by this client'
+        })
     .pc-form-grid-col-30
-        +number('Connection TTL:', clientCfgModel + '.connectionTTL', '"checkpointS3ConnectionTTL"', 'true', '-1', '-1',
-        'Expiration time in milliseconds for a connection in the connection pool<br/>\
-        By default, it is set to <b>-1</b>, i.e. connections do not expire')
+        +form-field__number({
+            label: 'Connection TTL:',
+            model: `${clientCfgModel}.connectionTTL`,
+            name: '"checkpointS3ConnectionTTL"',
+            placeholder: '-1',
+            min: '-1',
+            tip: 'Expiration time in milliseconds for a connection in the connection pool<br/>\
+                  By default, it is set to <b>-1</b>, i.e. connections do not expire'
+        })
     .pc-form-grid-col-60
-        +java-class('DNS resolver:', clientCfgModel + '.dnsResolver', '"checkpointS3DnsResolver"', 'true', 'false',
-        'DNS Resolver that should be used to for resolving AWS IP addresses', checkpointS3)
+        +form-field__java-class({
+            label: 'DNS resolver:',
+            model: clientCfgModel + '.dnsResolver',
+            name: '"checkpointS3DnsResolver"',
+            tip: 'DNS Resolver that should be used to for resolving AWS IP addresses',
+            validationActive: checkpointS3
+        })
     .pc-form-grid-col-60
-        +number('Response metadata cache size:', clientCfgModel + '.responseMetadataCacheSize', '"checkpointS3ResponseMetadataCacheSize"', 'true', '50', '0',
-        'Response metadata cache size')
+        +form-field__number({
+            label: 'Response metadata cache size:',
+            model: `${clientCfgModel}.responseMetadataCacheSize`,
+            name: '"checkpointS3ResponseMetadataCacheSize"',
+            placeholder: '50',
+            min: '0',
+            tip: 'Response metadata cache size'
+        })
     .pc-form-grid-col-60
-        +java-class('SecureRandom class name:', clientCfgModel + '.secureRandom', '"checkpointS3SecureRandom"', 'true', 'false',
-        'SecureRandom to be used by the SDK class name', checkpointS3)
+        +form-field__java-class({
+            label: 'SecureRandom class name:',
+            model: clientCfgModel + '.secureRandom',
+            name: '"checkpointS3SecureRandom"',
+            tip: 'SecureRandom to be used by the SDK class name',
+            validationActive: checkpointS3
+        })
     .pc-form-grid-col-60
-        +number('Client execution timeout:', clientCfgModel + '.clientExecutionTimeout', '"checkpointS3ClientExecutionTimeout"', 'true', '0', '0',
-        'Amount of time in milliseconds to allow the client to complete the execution of an API call<br/>\
-        <b>0</b> value disables that feature')
+        +form-field__number({
+            label: 'Client execution timeout:',
+            model: `${clientCfgModel}.clientExecutionTimeout`,
+            name: '"checkpointS3ClientExecutionTimeout"',
+            placeholder: '0',
+            min: '0',
+            tip: 'Amount of time in milliseconds to allow the client to complete the execution of an API call<br/>\
+                  <b>0</b> value disables that feature'
+        })
     .pc-form-grid-col-60
-        +checkbox('Cache response metadata', clientCfgModel + '.cacheResponseMetadata', '"checkpointS3CacheResponseMetadata"', 'Cache response metadata')
+        +form-field__checkbox({
+            label: 'Cache response metadata',
+            model: clientCfgModel + '.cacheResponseMetadata',
+            name: '"checkpointS3CacheResponseMetadata"',
+            tip: 'Cache response metadata'
+        })
     .pc-form-grid-col-60
-        +checkbox('Use expect continue', clientCfgModel + '.useExpectContinue', '"checkpointS3UseExpectContinue"', 'Optional override to enable/disable support for HTTP/1.1 handshake utilizing EXPECT: 100-Continue')
+        +form-field__checkbox({
+            label: 'Use expect continue',
+            model: clientCfgModel + '.useExpectContinue',
+            name: '"checkpointS3UseExpectContinue"',
+            tip: 'Optional override to enable/disable support for HTTP/1.1 handshake utilizing EXPECT: 100-Continue'
+        })
     .pc-form-grid-col-60
-        +checkbox('Use throttle retries', clientCfgModel + '.useThrottleRetries', '"checkpointS3UseThrottleRetries"', 'Retry throttling will be used')
+        +form-field__checkbox({
+            label: 'Use throttle retries',
+            model: clientCfgModel + '.useThrottleRetries',
+            name: '"checkpointS3UseThrottleRetries"',
+            tip: 'Retry throttling will be used'
+        })
     .pc-form-grid-col-60
-        +checkbox('Use reaper', clientCfgModel + '.useReaper', '"checkpointS3UseReaper"', 'Checks if the IdleConnectionReaper is to be started')
+        +form-field__checkbox({
+            label: 'Use reaper',
+            model: clientCfgModel + '.useReaper',
+            name: '"checkpointS3UseReaper"',
+            tip: 'Checks if the IdleConnectionReaper is to be started'
+        })
     .pc-form-grid-col-60
-        +checkbox('Use GZIP', clientCfgModel + '.useGzip', '"checkpointS3UseGzip"', 'Checks if gzip compression is used')
+        +form-field__checkbox({
+            label: 'Use GZIP',
+            model: clientCfgModel + '.useGzip',
+            name: '"checkpointS3UseGzip"',
+            tip: 'Checks if gzip compression is used'
+        })
     .pc-form-grid-col-60
-        +checkbox('Preemptively basic authentication', clientCfgModel + '.preemptiveBasicProxyAuth', '"checkpointS3PreemptiveBasicProxyAuth"',
-        'Attempt to authenticate preemptively against proxy servers using basic authentication')
+        +form-field__checkbox({
+            label: 'Preemptively basic authentication',
+            model: clientCfgModel + '.preemptiveBasicProxyAuth',
+            name: '"checkpointS3PreemptiveBasicProxyAuth"',
+            tip: 'Attempt to authenticate preemptively against proxy servers using basic authentication'
+        })
     .pc-form-grid-col-60
-        +checkbox('TCP KeepAlive', clientCfgModel + '.useTcpKeepAlive', '"checkpointS3UseTcpKeepAlive"', 'TCP KeepAlive support is enabled')
+        +form-field__checkbox({
+            label: 'TCP KeepAlive',
+            model: clientCfgModel + '.useTcpKeepAlive',
+            name: '"checkpointS3UseTcpKeepAlive"',
+            tip: 'TCP KeepAlive support is enabled'
+        })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/client-connector.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/client-connector.pug
index 620137b9d923c..00c35634f6a3b 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/client-connector.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/client-connector.pug
@@ -28,49 +28,156 @@ panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.lo
     panel-content.pca-form-row(ng-if=`$ctrl.available("2.3.0") && ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +checkbox('Enabled', connectionEnabled, '"ClientConnectorEnabled"', 'Flag indicating whether to configure client connector configuration')
+                +form-field__checkbox({
+                    label: 'Enabled',
+                    model: connectionEnabled,
+                    name: '"ClientConnectorEnabled"',
+                    tip: 'Flag indicating whether to configure client connector configuration'
+                })
             .pc-form-grid-col-60
-                +text-enabled('Host:', `${connectionModel}.host`, '"ClientConnectorHost"', connectionEnabled, 'false', 'localhost')
+                +form-field__text({
+                    label: 'Host:',
+                    model: `${connectionModel}.host`,
+                    name: '"ClientConnectorHost"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: 'localhost'
+                })
             .pc-form-grid-col-30
-                +number('Port:', `${connectionModel}.port`, '"ClientConnectorPort"', connectionEnabled, '10800', '1025')
+                +form-field__number({
+                    label: 'Port:',
+                    model: `${connectionModel}.port`,
+                    name: '"ClientConnectorPort"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '10800',
+                    min: '1025'
+                })
             .pc-form-grid-col-30
-                +number('Port range:', `${connectionModel}.portRange`, '"ClientConnectorPortRange"', connectionEnabled, '100', '0')
+                +form-field__number({
+                    label: 'Port range:',
+                    model: `${connectionModel}.portRange`,
+                    name: '"ClientConnectorPortRange"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '100',
+                    min: '0'
+                })
             .pc-form-grid-col-30
-                +number('Socket send buffer size:', `${connectionModel}.socketSendBufferSize`, '"ClientConnectorSocketSendBufferSize"', connectionEnabled, '0', '0',
-                    'Socket send buffer size<br/>\
-                    When set to <b>0</b>, operation system default will be used')
+                +form-field__number({
+                    label: 'Socket send buffer size:',
+                    model: `${connectionModel}.socketSendBufferSize`,
+                    name: '"ClientConnectorSocketSendBufferSize"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Socket send buffer size<br/>\
+                          When set to <b>0</b>, operation system default will be used'
+                })
             .pc-form-grid-col-30
-                +number('Socket receive buffer size:', `${connectionModel}.socketReceiveBufferSize`, '"ClientConnectorSocketReceiveBufferSize"', connectionEnabled, '0', '0',
-                    'Socket receive buffer size<br/>\
-                    When set to <b>0</b>, operation system default will be used')
+                +form-field__number({
+                    label: 'Socket receive buffer size:',
+                    model: `${connectionModel}.socketReceiveBufferSize`,
+                    name: '"ClientConnectorSocketReceiveBufferSize"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Socket receive buffer size<br/>\
+                          When set to <b>0</b>, operation system default will be used'
+                })
             .pc-form-grid-col-30
-                +number('Max connection cursors:', `${connectionModel}.maxOpenCursorsPerConnection`, '"ClientConnectorMaxOpenCursorsPerConnection"', connectionEnabled, '128', '0',
-                    'Max number of opened cursors per connection')
+                +form-field__number({
+                    label: 'Max connection cursors:',
+                    model: `${connectionModel}.maxOpenCursorsPerConnection`,
+                    name: '"ClientConnectorMaxOpenCursorsPerConnection"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '128',
+                    min: '0',
+                    tip: 'Max number of opened cursors per connection'
+                })
             .pc-form-grid-col-30
-                +number('Pool size:', `${connectionModel}.threadPoolSize`, '"ClientConnectorThreadPoolSize"', connectionEnabled, 'max(8, availableProcessors)', '1',
-                    'Size of thread pool that is in charge of processing SQL requests')
+                +form-field__number({
+                    label: 'Pool size:',
+                    model: `${connectionModel}.threadPoolSize`,
+                    name: '"ClientConnectorThreadPoolSize"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: 'max(8, availableProcessors)',
+                    min: '1',
+                    tip: 'Size of thread pool that is in charge of processing SQL requests'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('TCP_NODELAY option', `${connectionModel}.tcpNoDelay`, '"ClientConnectorTcpNoDelay"', connectionEnabled)
+                +form-field__checkbox({
+                    label: 'TCP_NODELAY option',
+                    model: `${connectionModel}.tcpNoDelay`,
+                    name: '"ClientConnectorTcpNoDelay"',
+                    disabled: `!${connectionEnabled}`
+                })
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.4.0")')
-                +number('Idle timeout:', `${connectionModel}.idleTimeout`, '"ClientConnectorIdleTimeout"', connectionEnabled, '0', '-1',
-                    'Idle timeout for client connections<br/>\
-                    Zero or negative means no timeout')
+                +form-field__number({
+                    label: 'Idle timeout:',
+                    model: `${connectionModel}.idleTimeout`,
+                    name: '"ClientConnectorIdleTimeout"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '0',
+                    min: '-1',
+                    tip: 'Idle timeout for client connections<br/>\
+                         Zero or negative means no timeout'
+                })
             .pc-form-grid-col-60(ng-if-start='$ctrl.available("2.5.0")')
-                +checkbox-enabled('Enable SSL', `${connectionModel}.sslEnabled`, '"ClientConnectorSslEnabled"', connectionEnabled, 'Enable secure socket layer on client connector')
+                +form-field__checkbox({
+                    label: 'Enable SSL',
+                    model: `${connectionModel}.sslEnabled`,
+                    name: '"ClientConnectorSslEnabled"',
+                    disabled: `!${connectionEnabled}`,
+                    tip: 'Enable secure socket layer on client connector'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('Enable SSL client auth', `${connectionModel}.sslClientAuth`, '"ClientConnectorSslClientAuth"', sslEnabled, 'Flag indicating whether or not SSL client authentication is required')
+                +form-field__checkbox({
+                    label: 'Enable SSL client auth',
+                    model: `${connectionModel}.sslClientAuth`,
+                    name: '"ClientConnectorSslClientAuth"',
+                    disabled: `!(${sslEnabled})`,
+                    tip: 'Flag indicating whether or not SSL client authentication is required'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('Use Ignite SSL', `${connectionModel}.useIgniteSslContextFactory`, '"ClientConnectorUseIgniteSslContextFactory"', sslEnabled, 'Use SSL factory Ignite configuration')
+                +form-field__checkbox({
+                    label: 'Use Ignite SSL',
+                    model: `${connectionModel}.useIgniteSslContextFactory`,
+                    name: '"ClientConnectorUseIgniteSslContextFactory"',
+                    disabled: `!(${sslEnabled})`,
+                    tip: 'Use SSL factory Ignite configuration'
+                })
             .pc-form-grid-col-60(ng-if-end)
-                +java-class('SSL factory:', `${connectionModel}.sslContextFactory`, '"ClientConnectorSslContextFactory"', sslFactoryEnabled, sslFactoryEnabled,
-                'If SSL factory specified then replication will be performed through secure SSL channel created with this factory<br/>\
-                If not present <b>isUseIgniteSslContextFactory()</b> flag will be evaluated<br/>\
-                If set to <b>true</b> and <b>IgniteConfiguration#getSslContextFactory()</b> exists, then Ignite SSL context factory will be used to establish secure connection')
+                +form-field__java-class({
+                    label:'SSL factory:',
+                    model: `${connectionModel}.sslContextFactory`,
+                    name: '"ClientConnectorSslContextFactory"',
+                    disabled: `!(${sslFactoryEnabled})`,
+                    required: sslFactoryEnabled,
+                    tip: 'If SSL factory specified then replication will be performed through secure SSL channel created with this factory<br/>\
+                          If not present <b>isUseIgniteSslContextFactory()</b> flag will be evaluated<br/>\
+                          If set to <b>true</b> and <b>IgniteConfiguration#getSslContextFactory()</b> exists, then Ignite SSL context factory will be used to establish secure connection'
+                })
             .pc-form-grid-col-60(ng-if-start='$ctrl.available("2.4.0")')
-                +checkbox-enabled('JDBC Enabled', `${connectionModel}.jdbcEnabled`, '"ClientConnectorJdbcEnabled"', connectionEnabled, 'Access through JDBC is enabled')
+                +form-field__checkbox({
+                    label: 'JDBC Enabled',
+                    model: `${connectionModel}.jdbcEnabled`,
+                    name: '"ClientConnectorJdbcEnabled"',
+                    disabled: `!${connectionEnabled}`,
+                    tip: 'Access through JDBC is enabled'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('ODBC Enabled', `${connectionModel}.odbcEnabled`, '"ClientConnectorOdbcEnabled"', connectionEnabled, 'Access through ODBC is enabled')
+                +form-field__checkbox({
+                    label: 'ODBC Enabled',
+                    model: `${connectionModel}.odbcEnabled`,
+                    name: '"ClientConnectorOdbcEnabled"',
+                    disabled: `!${connectionEnabled}`,
+                    tip: 'Access through ODBC is enabled'
+                })
             .pc-form-grid-col-60(ng-if-end)
-                +checkbox-enabled('Thin client enabled', `${connectionModel}.thinClientEnabled`, '"ClientConnectorThinCliEnabled"', connectionEnabled, 'Access through thin client is enabled')
+                +form-field__checkbox({
+                    label: 'Thin client enabled',
+                    model: `${connectionModel}.thinClientEnabled`,
+                    name: '"ClientConnectorThinCliEnabled"',
+                    disabled: `!${connectionEnabled}`,
+                    tip: 'Access through thin client is enabled'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterClientConnector')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision.pug
index c315af18f3acf..e3cacd3cb2b04 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision.pug
@@ -23,27 +23,32 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Collision configuration
     panel-description
-        | Configuration Collision SPI allows to regulate how grid jobs get executed when they arrive on a destination node for execution. 
+        | Configuration Collision SPI allows to regulate how grid jobs get executed when they arrive on a destination node for execution.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/job-scheduling" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +dropdown('CollisionSpi:', modelCollisionKind, '"collisionKind"', 'true', '',
-                    '[\
+                +form-field__dropdown({
+                    label:'CollisionSpi:',
+                    model: modelCollisionKind,
+                    name: '"collisionKind"',
+                    placeholder: 'Choose discovery',
+                    options: '[\
                         {value: "JobStealing", label: "Job stealing"},\
                         {value: "FifoQueue", label: "FIFO queue"},\
                         {value: "PriorityQueue", label: "Priority queue"},\
                         {value: "Custom", label: "Custom"},\
                         {value: "Noop", label: "Default"}\
                     ]',
-                    'Regulate how grid jobs get executed when they arrive on a destination node for execution\
-                    <ul>\
-                        <li>Job stealing - supports job stealing from over-utilized nodes to under-utilized nodes</li>\
-                        <li>FIFO queue - jobs are ordered as they arrived</li>\
-                        <li>Priority queue - jobs are first ordered by their priority</li>\
-                        <li>Custom - custom CollisionSpi implementation</li>\
-                        <li>Default - jobs are activated immediately on arrival to mapped node</li>\
-                    </ul>')
+                    tip: 'Regulate how grid jobs get executed when they arrive on a destination node for execution\
+                       <ul>\
+                           <li>Job stealing - supports job stealing from over-utilized nodes to under-utilized nodes</li>\
+                           <li>FIFO queue - jobs are ordered as they arrived</li>\
+                           <li>Priority queue - jobs are first ordered by their priority</li>\
+                           <li>Custom - custom CollisionSpi implementation</li>\
+                           <li>Default - jobs are activated immediately on arrival to mapped node</li>\
+                       </ul>'
+                })
             .pc-form-group(ng-show=`${modelCollisionKind} !== 'Noop'`)
                 .pc-form-grid-row(ng-show=`${modelCollisionKind} === 'JobStealing'`)
                     include ./collision/job-stealing
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/custom.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/custom.pug
index c1d11d5cde726..64bd5e4089537 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/custom.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/custom.pug
@@ -20,4 +20,11 @@ include /app/helpers/jade/mixins
 -var required = '$ctrl.clonedCluster.collision.kind === "Custom"'
 
 .pc-form-grid-col-60
-    +java-class('Class:', `${model}.class`, '"collisionCustom"', 'true', required, 'CollisionSpi implementation class', required)
+    +form-field__java-class({
+        label: 'Class:',
+        model: `${model}.class`,
+        name: '"collisionCustom"',
+        required: required,
+        tip: 'CollisionSpi implementation class',
+        validationActive: required
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/fifo-queue.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/fifo-queue.pug
index c00938628bfd3..de795b74d3489 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/fifo-queue.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/fifo-queue.pug
@@ -19,8 +19,20 @@ include /app/helpers/jade/mixins
 -var model = '$ctrl.clonedCluster.collision.FifoQueue'
 
 .pc-form-grid-col-30
-    +number('Parallel jobs number:', `${model}.parallelJobsNumber`, '"fifoParallelJobsNumber"', 'true', 'availableProcessors * 2', '1',
-        'Number of jobs that can be executed in parallel')
+    +form-field__number({
+        label: 'Parallel jobs number:',
+        model: `${model}.parallelJobsNumber`,
+        name: '"fifoParallelJobsNumber"',
+        placeholder: 'availableProcessors * 2',
+        min: '1',
+        tip: 'Number of jobs that can be executed in parallel'
+    })
 .pc-form-grid-col-30
-    +number('Wait jobs number:', `${model}.waitingJobsNumber`, '"fifoWaitingJobsNumber"', 'true', 'Integer.MAX_VALUE', '0',
-        'Maximum number of jobs that are allowed to wait in waiting queue')
+    +form-field__number({
+        label: 'Wait jobs number:',
+        model: `${model}.waitingJobsNumber`,
+        name: '"fifoWaitingJobsNumber"',
+        placeholder: 'Integer.MAX_VALUE',
+        min: '0',
+        tip: 'Maximum number of jobs that are allowed to wait in waiting queue'
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/job-stealing.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/job-stealing.pug
index f1b0a5619c22f..872254411575c 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/job-stealing.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/job-stealing.pug
@@ -20,32 +20,64 @@ include /app/helpers/jade/mixins
 -var stealingAttributes = `${model}.stealingAttributes`
 
 .pc-form-grid-col-30
-    +number('Active jobs threshold:', `${model}.activeJobsThreshold`, '"jsActiveJobsThreshold"', 'true', '95', '0',
-        'Number of jobs that can be executed in parallel')
+    +form-field__number({
+        label: 'Active jobs threshold:',
+        model: `${model}.activeJobsThreshold`,
+        name: '"jsActiveJobsThreshold"',
+        placeholder: '95',
+        min: '0',
+        tip: 'Number of jobs that can be executed in parallel'
+    })
 .pc-form-grid-col-30
-    +number('Wait jobs threshold:', `${model}.waitJobsThreshold`, '"jsWaitJobsThreshold"', 'true', '0', '0',
-        'Job count threshold at which this node will start stealing jobs from other nodes')
+    +form-field__number({
+        label: 'Wait jobs threshold:',
+        model: `${model}.waitJobsThreshold`,
+        name: '"jsWaitJobsThreshold"',
+        placeholder: '0',
+        min: '0',
+        tip: 'Job count threshold at which this node will start stealing jobs from other nodes'
+    })
 .pc-form-grid-col-30
-    +number('Message expire time:', `${model}.messageExpireTime`, '"jsMessageExpireTime"', 'true', '1000', '1',
-        'Message expire time in ms')
+    +form-field__number({
+        label: 'Message expire time:',
+        model: `${model}.messageExpireTime`,
+        name: '"jsMessageExpireTime"',
+        placeholder: '1000',
+        min: '1',
+        tip: 'Message expire time in ms'
+    })
 .pc-form-grid-col-30
-    +number('Maximum stealing attempts:', `${model}.maximumStealingAttempts`, '"jsMaximumStealingAttempts"', 'true', '5', '1',
-        'Maximum number of attempts to steal job by another node')
+    +form-field__number({
+        label: 'Maximum stealing attempts:',
+        model: `${model}.maximumStealingAttempts`,
+        name: '"jsMaximumStealingAttempts"',
+        placeholder: '5',
+        min: '1',
+        tip: 'Maximum number of attempts to steal job by another node'
+    })
 .pc-form-grid-col-60
-    +checkbox('Stealing enabled', `${model}.stealingEnabled`, '"jsStealingEnabled"',
-        'Node should attempt to steal jobs from other nodes')
+    +form-field__checkbox({
+        label: 'Stealing enabled',
+        model: `${model}.stealingEnabled`,
+        name: '"jsStealingEnabled"',
+        tip: 'Node should attempt to steal jobs from other nodes'
+    })
 .pc-form-grid-col-60
-    +java-class('External listener:', `${model}.externalCollisionListener`, '"jsExternalCollisionListener"', 'true', 'false',
-        'Listener to be set for notification of external collision events', '$ctrl.clonedCluster.collision.kind === "JobStealing"')
+    +form-field__java-class({
+        label: 'External listener:',
+        model: `${model}.externalCollisionListener`,
+        name: '"jsExternalCollisionListener"',
+        tip: 'Listener to be set for notification of external collision events',
+        validationActive: '$ctrl.clonedCluster.collision.kind === "JobStealing"'
+    })
 .pc-form-grid-col-60
     .ignite-form-field
-        +ignite-form-field__label('Stealing attributes:', '"stealingAttributes"')
-            +tooltip(`Configuration parameter to enable stealing to/from only nodes that have these attributes set`)
-        .ignite-form-field__control
-            +list-pair-edit({
-                items: stealingAttributes,
-                keyLbl: 'Attribute name', 
-                valLbl: 'Attribute value',
-                itemName: 'stealing attribute',
-                itemsName: 'stealing attributes'
-            })
+        +form-field__label({ label: 'Stealing attributes:', name: '"stealingAttributes"' })
+            +form-field__tooltip(`Configuration parameter to enable stealing to/from only nodes that have these attributes set`)
+        +list-pair-edit({
+            items: stealingAttributes,
+            keyLbl: 'Attribute name',
+            valLbl: 'Attribute value',
+            itemName: 'stealing attribute',
+            itemsName: 'stealing attributes'
+        })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/priority-queue.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/priority-queue.pug
index fd198cef859e6..c8ae73323da8e 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/priority-queue.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/collision/priority-queue.pug
@@ -19,23 +19,61 @@ include /app/helpers/jade/mixins
 -var model = '$ctrl.clonedCluster.collision.PriorityQueue'
 
 .pc-form-grid-col-30
-    +number('Parallel jobs number:', `${model}.parallelJobsNumber`, '"priorityParallelJobsNumber"', 'true', 'availableProcessors * 2', '1',
-        'Number of jobs that can be executed in parallel')
+    +form-field__number({
+        label: 'Parallel jobs number:',
+        model: `${model}.parallelJobsNumber`,
+        name: '"priorityParallelJobsNumber"',
+        placeholder: 'availableProcessors * 2',
+        min: '1',
+        tip: 'Number of jobs that can be executed in parallel'
+    })
 .pc-form-grid-col-30
-    +number('Waiting jobs number:', `${model}.waitingJobsNumber`, '"priorityWaitingJobsNumber"', 'true', 'Integer.MAX_VALUE', '0',
-        'Maximum number of jobs that are allowed to wait in waiting queue')
+    +form-field__number({
+        label: 'Waiting jobs number:',
+        model: `${model}.waitingJobsNumber`,
+        name: '"priorityWaitingJobsNumber"',
+        placeholder: 'Integer.MAX_VALUE',
+        min: '0',
+        tip: 'Maximum number of jobs that are allowed to wait in waiting queue'
+    })
 .pc-form-grid-col-30
-    +text('Priority attribute key:', `${model}.priorityAttributeKey`, '"priorityPriorityAttributeKey"', 'false', 'grid.task.priority',
-        'Task priority attribute key')
+    +form-field__text({
+        label: 'Priority attribute key:',
+        model: `${model}.priorityAttributeKey`,
+        name: '"priorityPriorityAttributeKey"',
+        placeholder: 'grid.task.priority',
+        tip: 'Task priority attribute key'
+    })
 .pc-form-grid-col-30
-    +text('Job priority attribute key:', `${model}.jobPriorityAttributeKey`, '"priorityJobPriorityAttributeKey"', 'false', 'grid.job.priority',
-        'Job priority attribute key')
+    +form-field__text({
+        label: 'Job priority attribute key:',
+        model: `${model}.jobPriorityAttributeKey`,
+        name: '"priorityJobPriorityAttributeKey"',
+        placeholder: 'grid.job.priority',
+        tip: 'Job priority attribute key'
+    })
 .pc-form-grid-col-30
-    +number('Default priority:', `${model}.defaultPriority`, '"priorityDefaultPriority"', 'true', '0', '0',
-        'Default priority to use if a job does not have priority attribute set')
+    +form-field__number({
+        label: 'Default priority:',
+        model: `${model}.defaultPriority`,
+        name: '"priorityDefaultPriority"',
+        placeholder: '0',
+        min: '0',
+        tip: 'Default priority to use if a job does not have priority attribute set'
+    })
 .pc-form-grid-col-30
-    +number('Starvation increment:', `${model}.starvationIncrement`, '"priorityStarvationIncrement"', 'true', '1', '0',
-        'Value to increment job priority by every time a lower priority job gets behind a higher priority job')
+    +form-field__number({
+        label: 'Starvation increment:',
+        model: `${model}.starvationIncrement`,
+        name: '"priorityStarvationIncrement"',
+        placeholder: '1',
+        min: '0',
+        tip: 'Value to increment job priority by every time a lower priority job gets behind a higher priority job'
+    })
 .pc-form-grid-col-60
-    +checkbox('Starvation prevention enabled', `${model}.starvationPreventionEnabled`, '"priorityStarvationPreventionEnabled"',
-        'Job starvation prevention is enabled')
+    +form-field__checkbox({
+        label: 'Starvation prevention enabled',
+        model: `${model}.starvationPreventionEnabled`,
+        name: '"priorityStarvationPreventionEnabled"',
+        tip: 'Job starvation prevention is enabled'
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/communication.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/communication.pug
index 8b43521ff5a0a..cdf473a60a8ac 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/communication.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/communication.pug
@@ -24,30 +24,84 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Communication
     panel-description
         | Configuration of communication with other nodes by TCP/IP.
-        | Provide basic plumbing to send and receive grid messages and is utilized for all distributed grid operations. 
+        | Provide basic plumbing to send and receive grid messages and is utilized for all distributed grid operations.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/network-config" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-30
-                +number('Timeout:', `${model}.networkTimeout`, '"commNetworkTimeout"', 'true', '5000', '1', 'Maximum timeout in milliseconds for network requests')
+                +form-field__number({
+                    label: 'Timeout:',
+                    model: `${model}.networkTimeout`,
+                    name: '"commNetworkTimeout"',
+                    placeholder: '5000',
+                    min: '1',
+                    tip: 'Maximum timeout in milliseconds for network requests'
+                })
             .pc-form-grid-col-30
-                +number('Send retry delay:', `${model}.networkSendRetryDelay`, '"networkSendRetryDelay"', 'true', '1000', '1', 'Interval in milliseconds between message send retries')
+                +form-field__number({
+                    label: 'Send retry delay:',
+                    model: `${model}.networkSendRetryDelay`,
+                    name: '"networkSendRetryDelay"',
+                    placeholder: '1000',
+                    min: '1',
+                    tip: 'Interval in milliseconds between message send retries'
+                })
             .pc-form-grid-col-30
-                +number('Send retry count:', `${model}.networkSendRetryCount`, '"networkSendRetryCount"', 'true', '3', '1', 'Message send retries count')
+                +form-field__number({
+                    label: 'Send retry count:',
+                    model: `${model}.networkSendRetryCount`,
+                    name: '"networkSendRetryCount"',
+                    placeholder: '3',
+                    min: '1',
+                    tip: 'Message send retries count'
+                })
             .pc-form-grid-col-30(ng-if='$ctrl.available(["1.0.0", "2.3.0"])')
-                +number('Discovery startup delay:', `${model}.discoveryStartupDelay`, '"discoveryStartupDelay"', 'true', '60000', '1', 'This value is used to expire messages from waiting list whenever node discovery discrepancies happen')
+                +form-field__number({
+                    label: 'Discovery startup delay:',
+                    model: `${model}.discoveryStartupDelay`,
+                    name: '"discoveryStartupDelay"',
+                    placeholder: '60000',
+                    min: '1',
+                    tip: 'This value is used to expire messages from waiting list whenever node discovery discrepancies happen'
+                })
             .pc-form-grid-col-60
-                +java-class('Communication listener:', `${communication}.listener`, '"comListener"', 'true', 'false', 'Listener of communication events')
+                +form-field__java-class({
+                    label: 'Communication listener:',
+                    model: `${communication}.listener`,
+                    name: '"comListener"',
+                    tip: 'Listener of communication events'
+                })
             .pc-form-grid-col-30
-                +text-ip-address('Local IP address:', `${communication}.localAddress`, '"comLocalAddress"', 'true', '0.0.0.0',
-                    'Local host address for socket binding<br/>\
-                    If not specified use all available addres on local host')
+                +form-field__ip-address({
+                    label: 'Local IP address:',
+                    model: `${communication}.localAddress`,
+                    name: '"comLocalAddress"',
+                    enabled: 'true',
+                    placeholder: '0.0.0.0',
+                    tip: 'Local host address for socket binding<br/>\
+                         If not specified use all available addres on local host'
+                })
             .pc-form-grid-col-30
-                +number-min-max('Local port:', `${communication}.localPort`, '"comLocalPort"', 'true', '47100', '1024', '65535', 'Local port for socket binding')
+                +form-field__number({
+                    label: 'Local port:',
+                    model: `${communication}.localPort`,
+                    name: '"comLocalPort"',
+                    placeholder: '47100',
+                    min: '1024',
+                    max: '65535',
+                    tip: 'Local port for socket binding'
+                })
             .pc-form-grid-col-30
-                +number('Local port range:', `${communication}.localPortRange`, '"comLocalPortRange"', 'true', '100', '1', 'Local port range for local host ports')
+                +form-field__number({
+                    label: 'Local port range:',
+                    model: `${communication}.localPortRange`,
+                    name: '"comLocalPortRange"',
+                    placeholder: '100',
+                    min: '1',
+                    tip: 'Local port range for local host ports'
+                })
             .pc-form-grid-col-30
-                +sane-ignite-form-field-number({
+                +form-field__number({
                     label: 'Shared memory port:',
                     model: `${communication}.sharedMemoryPort`,
                     name: '"sharedMemoryPort"',
@@ -58,25 +112,72 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                 })(
                     pc-not-in-collection='::$ctrl.Clusters.sharedMemoryPort.invalidValues'
                 )
-                    +form-field-feedback('"sharedMemoryPort"', 'notInCollection', 'Shared memory port should be more than "{{ ::$ctrl.Clusters.sharedMemoryPort.invalidValues[0] }}" or equal to "{{ ::$ctrl.Clusters.sharedMemoryPort.min }}"')
+                    +form-field__error({ error: 'notInCollection', message: 'Shared memory port should be more than "{{ ::$ctrl.Clusters.sharedMemoryPort.invalidValues[0] }}" or equal to "{{ ::$ctrl.Clusters.sharedMemoryPort.min }}"' })
             .pc-form-grid-col-30
-                +number('Idle connection timeout:', `${communication}.idleConnectionTimeout`, '"idleConnectionTimeout"', 'true', '30000', '1',
-                    'Maximum idle connection timeout upon which a connection to client will be closed')
+                +form-field__number({
+                    label: 'Idle connection timeout:',
+                    model: `${communication}.idleConnectionTimeout`,
+                    name: '"idleConnectionTimeout"',
+                    placeholder: '30000',
+                    min: '1',
+                    tip: 'Maximum idle connection timeout upon which a connection to client will be closed'
+                })
             .pc-form-grid-col-30
-                +number('Connect timeout:', `${communication}.connectTimeout`, '"connectTimeout"', 'true', '5000', '0', 'Connect timeout used when establishing connection with remote nodes')
+                +form-field__number({
+                    label: 'Connect timeout:',
+                    model: `${communication}.connectTimeout`,
+                    name: '"connectTimeout"',
+                    placeholder: '5000',
+                    min: '0',
+                    tip: 'Connect timeout used when establishing connection with remote nodes'
+                })
             .pc-form-grid-col-30
-                +number('Max. connect timeout:', `${communication}.maxConnectTimeout`, '"maxConnectTimeout"', 'true', '600000', '0', 'Maximum connect timeout')
+                +form-field__number({
+                    label: 'Max. connect timeout:',
+                    model: `${communication}.maxConnectTimeout`,
+                    name: '"maxConnectTimeout"',
+                    placeholder: '600000',
+                    min: '0',
+                    tip: 'Maximum connect timeout'
+                })
             .pc-form-grid-col-30
-                +number('Reconnect count:', `${communication}.reconnectCount`, '"comReconnectCount"', 'true', '10', '1',
-                    'Maximum number of reconnect attempts used when establishing connection with remote nodes')
+                +form-field__number({
+                    label: 'Reconnect count:',
+                    model: `${communication}.reconnectCount`,
+                    name: '"comReconnectCount"',
+                    placeholder: '10',
+                    min: '1',
+                    tip: 'Maximum number of reconnect attempts used when establishing connection with remote nodes'
+                })
             .pc-form-grid-col-30
-                +number('Socket send buffer:', `${communication}.socketSendBuffer`, '"socketSendBuffer"', 'true', '32768', '0', 'Send buffer size for sockets created or accepted by this SPI')
+                +form-field__number({
+                    label: 'Socket send buffer:',
+                    model: `${communication}.socketSendBuffer`,
+                    name: '"socketSendBuffer"',
+                    placeholder: '32768',
+                    min: '0',
+                    tip: 'Send buffer size for sockets created or accepted by this SPI'
+                })
             .pc-form-grid-col-30
-                +number('Socket receive buffer:', `${communication}.socketReceiveBuffer`, '"socketReceiveBuffer"', 'true', '32768', '0', 'Receive buffer size for sockets created or accepted by this SPI')
+                +form-field__number({
+                    label: 'Socket receive buffer:',
+                    model: `${communication}.socketReceiveBuffer`,
+                    name: '"socketReceiveBuffer"',
+                    placeholder: '32768',
+                    min: '0',
+                    tip: 'Receive buffer size for sockets created or accepted by this SPI'
+                })
             .pc-form-grid-col-30
-                +number('Slow client queue limit:', `${communication}.slowClientQueueLimit`, '"slowClientQueueLimit"', 'true', '0', '0', 'Slow client queue limit')
+                +form-field__number({
+                    label: 'Slow client queue limit:',
+                    model: `${communication}.slowClientQueueLimit`,
+                    name: '"slowClientQueueLimit"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Slow client queue limit'
+                })
             .pc-form-grid-col-30
-                +sane-ignite-form-field-number({
+                +form-field__number({
                     label: 'Ack send threshold:',
                     model: `${communication}.ackSendThreshold`,
                     name: '"ackSendThreshold"',
@@ -85,7 +186,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     tip: 'Number of received messages per connection to node after which acknowledgment message is sent'
                 })
             .pc-form-grid-col-30
-                +sane-ignite-form-field-number({
+                +form-field__number({
                     label: 'Message queue limit:',
                     model: `${communication}.messageQueueLimit`,
                     name: '"messageQueueLimit"',
@@ -95,7 +196,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                 })
             .pc-form-grid-col-30
                 //- allowInvalid: true prevents from infinite digest loop when old value was 0 and becomes less than allowed minimum
-                +sane-ignite-form-field-number({
+                +form-field__number({
                     label: 'Unacknowledged messages:',
                     model: `${communication}.unacknowledgedMessagesBufferSize`,
                     name: '"unacknowledgedMessagesBufferSize"',
@@ -117,18 +218,51 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     }`
                 )
             .pc-form-grid-col-30
-                +number('Socket write timeout:', `${communication}.socketWriteTimeout`, '"socketWriteTimeout"', 'true', '2000', '0', 'Socket write timeout')
+                +form-field__number({
+                    label: 'Socket write timeout:',
+                    model: `${communication}.socketWriteTimeout`,
+                    name: '"socketWriteTimeout"',
+                    placeholder: '2000',
+                    min: '0',
+                    tip: 'Socket write timeout'
+                })
             .pc-form-grid-col-30
-                +number('Selectors count:', `${communication}.selectorsCount`, '"selectorsCount"', 'true', 'min(4, availableProcessors)', '1', 'Count of selectors te be used in TCP server')
+                +form-field__number({
+                    label: 'Selectors count:',
+                    model: `${communication}.selectorsCount`,
+                    name: '"selectorsCount"',
+                    placeholder: 'min(4, availableProcessors)',
+                    min: '1',
+                    tip: 'Count of selectors te be used in TCP server'
+                })
             .pc-form-grid-col-60
-                +java-class('Address resolver:', `${communication}.addressResolver`, '"comAddressResolver"', 'true', 'false', 'Provides resolution between external and internal addresses')
+                +form-field__java-class({
+                    label: 'Address resolver:',
+                    model: `${communication}.addressResolver`,
+                    name: '"comAddressResolver"',
+                    tip: 'Provides resolution between external and internal addresses'
+                })
             .pc-form-grid-col-60
-                +checkbox('Direct buffer', `${communication}.directBuffer`, '"directBuffer"',
-                'If value is true, then SPI will use ByteBuffer.allocateDirect(int) call<br/>\
-                Otherwise, SPI will use ByteBuffer.allocate(int) call')
+                +form-field__checkbox({
+                    label: 'Direct buffer',
+                    model: `${communication}.directBuffer`,
+                    name: '"directBuffer"',
+                    tip: 'If value is true, then SPI will use ByteBuffer.allocateDirect(int) call<br/>\
+                          Otherwise, SPI will use ByteBuffer.allocate(int) call'
+                })
             .pc-form-grid-col-60
-                +checkbox('Direct send buffer', `${communication}.directSendBuffer`, '"directSendBuffer"', 'Flag defining whether direct send buffer should be used')
+                +form-field__checkbox({
+                    label: 'Direct send buffer',
+                    model: `${communication}.directSendBuffer`,
+                    name: '"directSendBuffer"',
+                    tip: 'Flag defining whether direct send buffer should be used'
+                })
             .pc-form-grid-col-60
-                +checkbox('TCP_NODELAY option', `${communication}.tcpNoDelay`, '"tcpNoDelay"', 'Value for TCP_NODELAY socket option')
+                +form-field__checkbox({
+                    label: 'TCP_NODELAY option',
+                    model: `${communication}.tcpNoDelay`,
+                    name: '"tcpNoDelay"',
+                    tip: 'Value for TCP_NODELAY socket option'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterCommunication')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/connector.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/connector.pug
index 76c501689355e..fd52e5cfcd678 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/connector.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/connector.pug
@@ -24,77 +24,209 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Connector configuration
     panel-description
-        | Configure HTTP REST configuration to enable HTTP server features. 
+        | Configure HTTP REST configuration to enable HTTP server features.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/rest-api#general-configuration" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +checkbox('Enabled', enabled, '"restEnabled"', 'Flag indicating whether to configure connector configuration')
+                +form-field__checkbox({
+                    label: 'Enabled',
+                    model: enabled,
+                    name: '"restEnabled"',
+                    tip: 'Flag indicating whether to configure connector configuration'
+                })
             .pc-form-grid-col-60
-                +text-enabled('Jetty configuration path:', `${model}.jettyPath`, '"connectorJettyPath"', enabled, 'false', 'Input path to Jetty configuration',
-                    'Path, either absolute or relative to IGNITE_HOME, to Jetty XML configuration file<br/>\
-                    Jetty is used to support REST over HTTP protocol for accessing Ignite APIs remotely<br/>\
-                    If not provided, Jetty instance with default configuration will be started picking IgniteSystemProperties.IGNITE_JETTY_HOST and IgniteSystemProperties.IGNITE_JETTY_PORT as host and port respectively')
+                +form-field__text({
+                    label: 'Jetty configuration path:',
+                    model: `${model}.jettyPath`,
+                    name: '"connectorJettyPath"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'Input path to Jetty configuration',
+                    tip: 'Path, either absolute or relative to IGNITE_HOME, to Jetty XML configuration file<br/>\
+                          Jetty is used to support REST over HTTP protocol for accessing Ignite APIs remotely<br/>\
+                          If not provided, Jetty instance with default configuration will be started picking IgniteSystemProperties.IGNITE_JETTY_HOST and IgniteSystemProperties.IGNITE_JETTY_PORT as host and port respectively'
+                })
             .pc-form-grid-col-20
-                +text-ip-address('TCP host:', `${model}.host`, '"connectorHost"', enabled, 'IgniteConfiguration#getLocalHost()',
-                    'Host for TCP binary protocol server<br/>\
-                    This can be either an IP address or a domain name<br/>\
-                    If not defined, system - wide local address will be used IgniteConfiguration#getLocalHost()<br/>\
-                    You can also use "0.0.0.0" value to bind to all locally - available IP addresses')
+                +form-field__ip-address({
+                    label:'TCP host:',
+                    model: `${model}.host`,
+                    name: '"connectorHost"',
+                    enabled: enabled,
+                    placeholder: 'IgniteConfiguration#getLocalHost()',
+                    tip: 'Host for TCP binary protocol server<br/>\
+                         This can be either an IP address or a domain name<br/>\
+                         If not defined, system - wide local address will be used IgniteConfiguration#getLocalHost()<br/>\
+                         You can also use "0.0.0.0" value to bind to all locally - available IP addresses'
+                })
             .pc-form-grid-col-20
-                +number-min-max('TCP port:', `${model}.port`, '"connectorPort"', enabled, '11211', '1024', '65535', 'Port for TCP binary protocol server')
+                +form-field__number({
+                    label: 'TCP port:',
+                    model: `${model}.port`,
+                    name: '"connectorPort"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '11211',
+                    min: '1024',
+                    max: '65535',
+                    tip: 'Port for TCP binary protocol server'
+                })
             .pc-form-grid-col-20
-                +number('TCP port range:', `${model}.portRange`, '"connectorPortRange"', enabled, '100', '1', 'Number of ports for TCP binary protocol server to try if configured port is already in use')
+                +form-field__number({
+                    label: 'TCP port range:',
+                    model: `${model}.portRange`,
+                    name: '"connectorPortRange"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '100',
+                    min: '1',
+                    tip: 'Number of ports for TCP binary protocol server to try if configured port is already in use'
+                })
             .pc-form-grid-col-60
-                +number('Idle query cursor timeout:', `${model}.idleQueryCursorTimeout`, '"connectorIdleQueryCursorTimeout"', enabled, '600000', '0',
-                    'Reject open query cursors that is not used timeout<br/>\
-                    If no fetch query request come within idle timeout, it will be removed on next check for old query cursors')
+                +form-field__number({
+                    label: 'Idle query cursor timeout:',
+                    model: `${model}.idleQueryCursorTimeout`,
+                    name: '"connectorIdleQueryCursorTimeout"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '600000',
+                    min: '0',
+                    tip: 'Reject open query cursors that is not used timeout<br/>\
+                          If no fetch query request come within idle timeout, it will be removed on next check for old query cursors'
+                })
             .pc-form-grid-col-60
-                +number('Idle query cursor check frequency:', `${model}.idleQueryCursorCheckFrequency`, '"connectorIdleQueryCursorCheckFrequency"', enabled, '60000', '0',
-                    'Idle query cursors check frequency<br/>\
-                    This setting is used to reject open query cursors that is not used')
+                +form-field__number({
+                    label: 'Idle query cursor check frequency:',
+                    model: `${model}.idleQueryCursorCheckFrequency`,
+                    name: '"connectorIdleQueryCursorCheckFrequency"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '60000',
+                    min: '0',
+                    tip: 'Idle query cursors check frequency<br/>\
+                          This setting is used to reject open query cursors that is not used'
+                })
             .pc-form-grid-col-30
-                +number('Idle timeout:', `${model}.idleTimeout`, '"connectorIdleTimeout"', enabled, '7000', '0',
-                    'Idle timeout for REST server<br/>\
-                    This setting is used to reject half - opened sockets<br/>\
-                    If no packets come within idle timeout, the connection is closed')
+                +form-field__number({
+                    label: 'Idle timeout:',
+                    model: `${model}.idleTimeout`,
+                    name: '"connectorIdleTimeout"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '7000',
+                    min: '0',
+                    tip: 'Idle timeout for REST server<br/>\
+                          This setting is used to reject half - opened sockets<br/>\
+                          If no packets come within idle timeout, the connection is closed'
+                })
             .pc-form-grid-col-30
-                +number('Receive buffer size:', `${model}.receiveBufferSize`, '"connectorReceiveBufferSize"', enabled, '32768', '0', 'REST TCP server receive buffer size')
+                +form-field__number({
+                    label: 'Receive buffer size:',
+                    model: `${model}.receiveBufferSize`,
+                    name: '"connectorReceiveBufferSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '32768',
+                    min: '0',
+                    tip: 'REST TCP server receive buffer size'
+                })
             .pc-form-grid-col-30
-                +number('Send buffer size:', `${model}.sendBufferSize`, '"connectorSendBufferSize"', enabled, '32768', '0', 'REST TCP server send buffer size')
+                +form-field__number({
+                    label: 'Send buffer size:',
+                    model: `${model}.sendBufferSize`,
+                    name: '"connectorSendBufferSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '32768',
+                    min: '0',
+                    tip: 'REST TCP server send buffer size'
+                })
             .pc-form-grid-col-30
-                +number('Send queue limit:', `${model}.sendQueueLimit`, '"connectorSendQueueLimit"', enabled, 'unlimited', '0',
-                    'REST TCP server send queue limit<br/>\
-                    If the limit exceeds, all successive writes will block until the queue has enough capacity')
+                +form-field__number({
+                    label: 'Send queue limit:',
+                    model: `${model}.sendQueueLimit`,
+                    name: '"connectorSendQueueLimit"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'unlimited',
+                    min: '0',
+                    tip: 'REST TCP server send queue limit<br/>\
+                         If the limit exceeds, all successive writes will block until the queue has enough capacity'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('Direct buffer', `${model}.directBuffer`, '"connectorDirectBuffer"', enabled,
-                    'Flag indicating whether REST TCP server should use direct buffers<br/>\
-                    A direct buffer is a buffer that is allocated and accessed using native system calls, without using JVM heap<br/>\
-                    Enabling direct buffer may improve performance and avoid memory issues(long GC pauses due to huge buffer size)')
+                +form-field__checkbox({
+                    label: 'Direct buffer',
+                    model: `${model}.directBuffer`,
+                    name: '"connectorDirectBuffer"',
+                    disabled: `!${enabled}`,
+                    tip: 'Flag indicating whether REST TCP server should use direct buffers<br/>\
+                          A direct buffer is a buffer that is allocated and accessed using native system calls, without using JVM heap<br/>\
+                          Enabling direct buffer may improve performance and avoid memory issues(long GC pauses due to huge buffer size)'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('TCP_NODELAY option', `${model}.noDelay`, '"connectorNoDelay"', enabled,
-                    'Flag indicating whether TCP_NODELAY option should be set for accepted client connections<br/>\
-                    Setting this option reduces network latency and should be enabled in majority of cases<br/>\
-                    For more information, see Socket#setTcpNoDelay(boolean)')
+                +form-field__checkbox({
+                    label: 'TCP_NODELAY option',
+                    model: `${model}.noDelay`,
+                    name: '"connectorNoDelay"',
+                    disabled: `!${enabled}`,
+                    tip: 'Flag indicating whether TCP_NODELAY option should be set for accepted client connections<br/>\
+                          Setting this option reduces network latency and should be enabled in majority of cases<br/>\
+                          For more information, see Socket#setTcpNoDelay(boolean)'
+                })
             .pc-form-grid-col-30
-                +number('Selector count:', `${model}.selectorCount`, '"connectorSelectorCount"', enabled, 'min(4, availableProcessors)', '1',
-                    'Number of selector threads in REST TCP server<br/>\
-                    Higher value for this parameter may increase throughput, but also increases context switching')
+                +form-field__number({
+                    label: 'Selector count:',
+                    model: `${model}.selectorCount`,
+                    name: '"connectorSelectorCount"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'min(4, availableProcessors)',
+                    min: '1',
+                    tip: 'Number of selector threads in REST TCP server<br/>\
+                          Higher value for this parameter may increase throughput, but also increases context switching'
+                })
             .pc-form-grid-col-30
-                +number('Thread pool size:', `${model}.threadPoolSize`, '"connectorThreadPoolSize"', enabled, 'max(8, availableProcessors) * 2', '1',
-                    'Thread pool size to use for processing of client messages (REST requests)')
+                +form-field__number({
+                    label: 'Thread pool size:',
+                    model: `${model}.threadPoolSize`,
+                    name: '"connectorThreadPoolSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'max(8, availableProcessors) * 2',
+                    min: '1',
+                    tip: 'Thread pool size to use for processing of client messages (REST requests)'
+                })
             .pc-form-grid-col-60
-                +java-class('Message interceptor:', `${model}.messageInterceptor`, '"connectorMessageInterceptor"', enabled, 'false',
-                    'Interceptor allows to transform all objects exchanged via REST protocol<br/>\
-                    For example if you use custom serialisation on client you can write interceptor to transform binary representations received from client to Java objects and later access them from java code directly')
+                +form-field__java-class({
+                    label: 'Message interceptor:',
+                    model: `${model}.messageInterceptor`,
+                    name: '"connectorMessageInterceptor"',
+                    disabled: `!(${enabled})`,
+                    tip: 'Interceptor allows to transform all objects exchanged via REST protocol<br/>\
+                         For example if you use custom serialisation on client you can write interceptor to transform binary representations received from client to Java objects and later access them from java code directly'
+                })
             .pc-form-grid-col-60
-                +text-enabled('Secret key:', `${model}.secretKey`, '"connectorSecretKey"', enabled, 'false', 'Specify to enable authentication', 'Secret key to authenticate REST requests')
+                +form-field__text({
+                    label: 'Secret key:',
+                    model: `${model}.secretKey`,
+                    name: '"connectorSecretKey"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'Specify to enable authentication',
+                    tip: 'Secret key to authenticate REST requests'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('Enable SSL', `${model}.sslEnabled`, '"connectorSslEnabled"', enabled, 'Enables/disables SSL for REST TCP binary protocol')
+                +form-field__checkbox({
+                    label: 'Enable SSL',
+                    model: `${model}.sslEnabled`,
+                    name: '"connectorSslEnabled"',
+                    disabled: `!${enabled}`,
+                    tip: 'Enables/disables SSL for REST TCP binary protocol'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('Enable SSL client auth', `${model}.sslClientAuth`, '"connectorSslClientAuth"', sslEnabled, 'Flag indicating whether or not SSL client authentication is required')
+                +form-field__checkbox({
+                    label: 'Enable SSL client auth',
+                    model: `${model}.sslClientAuth`,
+                    name: '"connectorSslClientAuth"',
+                    disabled: `!(${sslEnabled})`,
+                    tip: 'Flag indicating whether or not SSL client authentication is required'
+                })
             .pc-form-grid-col-60
-                +java-class('SSL factory:', `${model}.sslFactory`, '"connectorSslFactory"', sslEnabled, sslEnabled,
-                    'Instance of Factory that will be used to create an instance of SSLContext for Secure Socket Layer on TCP binary protocol')
+                +form-field__java-class({
+                    label: 'SSL factory:',
+                    model: `${model}.sslFactory`,
+                    name: '"connectorSslFactory"',
+                    disabled: `!(${sslEnabled})`,
+                    required: sslEnabled,
+                    tip: 'Instance of Factory that will be used to create an instance of SSLContext for Secure Socket Layer on TCP binary protocol'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterConnector')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug
index ea27c3c632bdb..e18b0cbabaf78 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/data-storage.pug
@@ -23,7 +23,7 @@ include /app/helpers/jade/mixins
 
 mixin data-region-form({modelAt, namePlaceholder, dataRegionsAt})
     .pc-form-grid-col-60
-        +sane-ignite-form-field-text({
+        +form-field__text({
             label: 'Name:',
             model: `${modelAt}.name`,
             name: '"name"',
@@ -36,11 +36,11 @@ mixin data-region-form({modelAt, namePlaceholder, dataRegionsAt})
             ignite-unique-property='name'
             ignite-unique-skip=`["_id", ${modelAt}]`
         )
-            +form-field-feedback(_, 'notInCollection', '{{::$ctrl.Clusters.dataRegion.name.invalidValues[0]}} is reserved for internal use')
-            +form-field-feedback(_, 'igniteUnique', 'Name should be unique')
+            +form-field__error({ error: 'notInCollection', message: '{{::$ctrl.Clusters.dataRegion.name.invalidValues[0]}} is reserved for internal use' })
+            +form-field__error({ error: 'igniteUnique', message: 'Name should be unique' })
 
     .pc-form-grid-col-30
-        pc-form-field-size(
+        form-field-size(
             label='Initial size:'
             ng-model=`${modelAt}.initialSize`
             name='initialSize'
@@ -50,7 +50,7 @@ mixin data-region-form({modelAt, namePlaceholder, dataRegionsAt})
         )
 
     .pc-form-grid-col-30
-        pc-form-field-size(
+        form-field-size(
             ng-model=`${modelAt}.maxSize`
             ng-model-options='{allowInvalid: true}'
             name='maxSize'
@@ -60,27 +60,45 @@ mixin data-region-form({modelAt, namePlaceholder, dataRegionsAt})
         )
 
     .pc-form-grid-col-60(ng-if=`!${modelAt}.persistenceEnabled || ${modelAt}.swapPath`)
-        +text('Swap file path:', `${modelAt}.swapPath`, '"swapPath"', 'false', 'Input swap file path', 'An optional path to a memory mapped file for this data region')
-        
+        +form-field__text({
+            label: 'Swap file path:',
+            model: `${modelAt}.swapPath`,
+            name: '"swapPath"',
+            placeholder: 'Input swap file path',
+            tip: 'An optional path to a memory mapped file for this data region'
+        })
+
     .pc-form-grid-col-60
-        +number('Checkpoint page buffer:', `${modelAt}.checkpointPageBufferSize`, '"checkpointPageBufferSize"', 'true', '0', '0', 'Amount of memory allocated for a checkpoint temporary buffer in bytes')
+        +form-field__number({
+            label: 'Checkpoint page buffer:',
+            model: `${modelAt}.checkpointPageBufferSize`,
+            name: '"checkpointPageBufferSize"',
+            placeholder: '0',
+            min: '0',
+            tip: 'Amount of memory allocated for a checkpoint temporary buffer in bytes'
+        })
 
     .pc-form-grid-col-60
-        +dropdown('Eviction mode:', `${modelAt}.pageEvictionMode`, '"pageEvictionMode"', 'true', 'DISABLED',
-            '[\
+        +form-field__dropdown({
+            label: 'Entry versioning:',
+            model: `${modelAt}.pageEvictionMode`,
+            name: '"pageEvictionMode"',
+            placeholder: 'DISABLED',
+            options: '[\
                 {value: "DISABLED", label: "DISABLED"},\
                 {value: "RANDOM_LRU", label: "RANDOM_LRU"},\
                 {value: "RANDOM_2_LRU", label: "RANDOM_2_LRU"}\
             ]',
-            `An algorithm for memory pages eviction
-            <ul>
-                <li>DISABLED - Eviction is disabled</li>
-                <li>RANDOM_LRU - Once a memory region defined by a data region is configured, an off-heap array is allocated to track last usage timestamp for every individual data page</li>
-                <li>RANDOM_2_LRU - Differs from Random - LRU only in a way that two latest access timestamps are stored for every data page</li>
-            </ul>`)
+            tip: `An algorithm for memory pages eviction
+                <ul>
+                    <li>DISABLED - Eviction is disabled</li>
+                    <li>RANDOM_LRU - Once a memory region defined by a data region is configured, an off-heap array is allocated to track last usage timestamp for every individual data page</li>
+                    <li>RANDOM_2_LRU - Differs from Random - LRU only in a way that two latest access timestamps are stored for every data page</li>
+                </ul>`
+        })
 
     .pc-form-grid-col-30
-        +sane-ignite-form-field-number({
+        +form-field__number({
             label: 'Eviction threshold:',
             model: `${modelAt}.evictionThreshold`,
             name: '"evictionThreshold"',
@@ -92,7 +110,7 @@ mixin data-region-form({modelAt, namePlaceholder, dataRegionsAt})
         })
 
     .pc-form-grid-col-30
-        +sane-ignite-form-field-number({
+        +form-field__number({
             label: 'Empty pages pool size:',
             model: `${modelAt}.emptyPagesPoolSize`,
             name: '"emptyPagesPoolSize"',
@@ -103,7 +121,7 @@ mixin data-region-form({modelAt, namePlaceholder, dataRegionsAt})
         })
 
     .pc-form-grid-col-30
-        +sane-ignite-form-field-number({
+        +form-field__number({
             label: 'Metrics sub interval count:',
             model: `${modelAt}.metricsSubIntervalCount`,
             name: '"metricsSubIntervalCount"',
@@ -114,7 +132,7 @@ mixin data-region-form({modelAt, namePlaceholder, dataRegionsAt})
         })
 
     .pc-form-grid-col-30
-        pc-form-field-size(
+        form-field-size(
             ng-model=`${modelAt}.metricsRateTimeInterval`
             ng-model-options='{allowInvalid: true}'
             name='metricsRateTimeInterval'
@@ -126,24 +144,32 @@ mixin data-region-form({modelAt, namePlaceholder, dataRegionsAt})
             on-scale-change='_metricsRateTimeIntervalScale = $event'
             size-scale-label='s'
         )
-            
+
     .pc-form-grid-col-60
-        +checkbox('Metrics enabled', `${modelAt}.metricsEnabled`, '"MemoryPolicyMetricsEnabled"',
-            'Whether memory metrics are enabled by default on node startup')
+        +form-field__checkbox({
+            label: 'Metrics enabled',
+            model: `${modelAt}.metricsEnabled`,
+            name: '"MemoryPolicyMetricsEnabled"',
+            tip: 'Whether memory metrics are enabled by default on node startup'
+        })
 
     .pc-form-grid-col-60(ng-if=`!${modelAt}.swapPath`)
-        +checkbox('Persistence enabled', `${modelAt}.persistenceEnabled`, '"RegionPersistenceEnabled" + $index',
-            'Enable Ignite Native Persistence')
+        +form-field__checkbox({
+            label: 'Persistence enabled',
+            model: `${modelAt}.persistenceEnabled`,
+            name: '"RegionPersistenceEnabled" + $index',
+            tip: 'Enable Ignite Native Persistence'
+        })
 
 panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Data storage configuration
     panel-description
-        | Page memory is a manageable off-heap based memory architecture that is split into pages of fixed size. 
+        | Page memory is a manageable off-heap based memory architecture that is split into pages of fixed size.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/distributed-persistent-store" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`$ctrl.available("2.3.0") && ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-30
-                +sane-ignite-form-field-dropdown({
+                +form-field__dropdown({
                     label: 'Page size:',
                     model: `${model}.pageSize`,
                     name: '"DataStorageConfigurationPageSize"',
@@ -151,13 +177,19 @@ panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.lo
                     tip: 'Every memory region is split on pages of fixed size'
                 })
             .pc-form-grid-col-30
-                +number('Concurrency level:', model + '.concurrencyLevel', '"DataStorageConfigurationConcurrencyLevel"',
-                    'true', 'availableProcessors', '2', 'The number of concurrent segments in Ignite internal page mapping tables')
+                +form-field__number({
+                    label: 'Concurrency level:',
+                    model: model + '.concurrencyLevel',
+                    name: '"DataStorageConfigurationConcurrencyLevel"',
+                    placeholder: 'availableProcessors',
+                    min: '2',
+                    tip: 'The number of concurrent segments in Ignite internal page mapping tables'
+                })
             .pc-form-grid-col-60.pc-form-group__text-title
                 span System region
             .pc-form-group.pc-form-grid-row
                 .pc-form-grid-col-30
-                    pc-form-field-size(
+                    form-field-size(
                         label='Initial size:'
                         ng-model=`${model}.systemRegionInitialSize`
                         name='DataStorageSystemRegionInitialSize'
@@ -167,7 +199,7 @@ panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.lo
                         on-scale-change='systemRegionInitialSizeScale = $event'
                     )
                 .pc-form-grid-col-30
-                    pc-form-field-size(
+                    form-field-size(
                         label='Max size:'
                         ng-model=`${model}.systemRegionMaxSize`
                         name='DataStorageSystemRegionMaxSize'
@@ -186,119 +218,259 @@ panel-collapsible(ng-show='$ctrl.available("2.3.0")' ng-form=form on-open=`ui.lo
                 })
             .pc-form-grid-col-60
                 .ignite-form-field
-                    .ignite-form-field__label Data region configurations
-                    .ignite-form-field__control
-                        list-editable.pc-list-editable-with-form-grid(
-                            name='dataRegionConfigurations'
-                            ng-model=dataRegionConfigurations
-                        )
-                            list-editable-item-edit.pc-form-grid-row
-                                - form = '$parent.form'
-                                +data-region-form({
-                                    modelAt: '$item',
-                                    namePlaceholder: 'Data region name',
-                                    dataRegionsAt: dataRegionConfigurations
-                                })
-                                - form = 'dataStorageConfiguration'
-                            list-editable-no-items
-                                list-editable-add-item-button(
-                                    add-item=`$ctrl.Clusters.addDataRegionConfiguration($ctrl.clonedCluster)`
-                                    label-single='data region configuration'
-                                    label-multiple='data region configurations'
-                                )
+                    +form-field__label({ label: 'Data region configurations' })
+
+                    list-editable.pc-list-editable-with-form-grid(
+                        name='dataRegionConfigurations'
+                        ng-model=dataRegionConfigurations
+                    )
+                        list-editable-item-edit.pc-form-grid-row
+                            - form = '$parent.form'
+                            +data-region-form({
+                                modelAt: '$item',
+                                namePlaceholder: 'Data region name',
+                                dataRegionsAt: dataRegionConfigurations
+                            })
+                            - form = 'dataStorageConfiguration'
+                        list-editable-no-items
+                            list-editable-add-item-button(
+                                add-item=`$ctrl.Clusters.addDataRegionConfiguration($ctrl.clonedCluster)`
+                                label-single='data region configuration'
+                                label-multiple='data region configurations'
+                            )
 
             .pc-form-grid-col-60
-                +text-enabled('Storage path:', `${model}.storagePath`, '"DataStoragePath"', 'true', 'false', 'db',
-                    'Directory where index and partition files are stored')
+                +form-field__text({
+                    label: 'Storage path:',
+                    model: `${model}.storagePath`,
+                    name: '"DataStoragePath"',
+                    placeholder: 'db',
+                    tip: 'Directory where index and partition files are stored'
+                })
             .pc-form-grid-col-60
-                +number('Checkpoint frequency:', `${model}.checkpointFrequency`, '"DataStorageCheckpointFrequency"', 'true', '180000', '1',
-                    'Frequency which is a minimal interval when the dirty pages will be written to the Persistent Store')
+                +form-field__number({
+                    label: 'Checkpoint frequency:',
+                    model: `${model}.checkpointFrequency`,
+                    name: '"DataStorageCheckpointFrequency"',
+                    placeholder: '180000',
+                    min: '1',
+                    tip: 'Frequency which is a minimal interval when the dirty pages will be written to the Persistent Store'
+                })
             .pc-form-grid-col-20
-                +number('Checkpoint threads:', `${model}.checkpointThreads`, '"DataStorageCheckpointThreads"', 'true', '4', '1', 'A number of threads to use for the checkpoint purposes')
+                +form-field__number({
+                    label: 'Checkpoint threads:',
+                    model: `${model}.checkpointThreads`,
+                    name: '"DataStorageCheckpointThreads"',
+                    placeholder: '4',
+                    min: '1',
+                    tip: 'A number of threads to use for the checkpoint purposes'
+                })
             .pc-form-grid-col-20
-                +dropdown('Checkpoint write order:', `${model}.checkpointWriteOrder`, '"DataStorageCheckpointWriteOrder"', 'true', 'SEQUENTIAL',
-                    '[\
+                +form-field__dropdown({
+                    label: 'Checkpoint write order:',
+                    model: `${model}.checkpointWriteOrder`,
+                    name: '"DataStorageCheckpointWriteOrder"',
+                    placeholder: 'SEQUENTIAL',
+                    options: '[\
                         {value: "RANDOM", label: "RANDOM"},\
                         {value: "SEQUENTIAL", label: "SEQUENTIAL"}\
                     ]',
-                    'Order of writing pages to disk storage during checkpoint.\
-                    <ul>\
-                        <li>RANDOM - Pages are written in order provided by checkpoint pages collection iterator</li>\
-                        <li>SEQUENTIAL - All checkpoint pages are collected into single list and sorted by page index</li>\
-                    </ul>')
+                    tip: 'Order of writing pages to disk storage during checkpoint.\
+                        <ul>\
+                            <li>RANDOM - Pages are written in order provided by checkpoint pages collection iterator</li>\
+                            <li>SEQUENTIAL - All checkpoint pages are collected into single list and sorted by page index</li>\
+                        </ul>'
+                })
             .pc-form-grid-col-20
-                +dropdown('WAL mode:', `${model}.walMode`, '"DataStorageWalMode"', 'true', 'DEFAULT',
-                    '[\
+                +form-field__dropdown({
+                    label: 'WAL mode:',
+                    model: `${model}.walMode`,
+                    name: '"DataStorageWalMode"',
+                    placeholder: 'DEFAULT',
+                    options: '[\
                         {value: "DEFAULT", label: "DEFAULT"},\
                         {value: "LOG_ONLY", label: "LOG_ONLY"},\
                         {value: "BACKGROUND", label: "BACKGROUND"},\
                         {value: "NONE", label: "NONE"}\
                     ]',
-                    'Type define behavior wal fsync.\
-                    <ul>\
-                        <li>DEFAULT - full-sync disk writes</li>\
-                        <li>LOG_ONLY - flushes application buffers</li>\
-                        <li>BACKGROUND - does not force application&#39;s buffer flush</li>\
-                        <li>NONE - WAL is disabled</li>\
-                    </ul>')
+                    tip: 'Type define behavior wal fsync.\
+                        <ul>\
+                            <li>DEFAULT - full-sync disk writes</li>\
+                            <li>LOG_ONLY - flushes application buffers</li>\
+                            <li>BACKGROUND - does not force application&#39;s buffer flush</li>\
+                            <li>NONE - WAL is disabled</li>\
+                        </ul>'
+                })
             .pc-form-grid-col-60
-                +text-enabled('WAL path:', `${model}.walPath`, '"DataStorageWalPath"', 'true', 'false', 'db/wal', 'A path to the directory where WAL is stored')
+                +form-field__text({
+                    label: 'WAL path:',
+                    model: `${model}.walPath`,
+                    name: '"DataStorageWalPath"',
+                    placeholder: 'db/wal',
+                    tip: 'A path to the directory where WAL is stored'
+                })
             .pc-form-grid-col-60
-                +text-enabled('WAL archive path:', `${model}.walArchivePath`, '"DataStorageWalArchivePath"', 'true', 'false', 'db/wal/archive', 'A path to the WAL archive directory')
+                +form-field__text({
+                    label: 'WAL archive path:',
+                    model: `${model}.walArchivePath`,
+                    name: '"DataStorageWalArchivePath"',
+                    placeholder: 'db/wal/archive',
+                    tip: 'A path to the WAL archive directory'
+                })
             .pc-form-grid-col-20
-                +number('WAL segments:', `${model}.walSegments`, '"DataStorageWalSegments"', 'true', '10', '1', 'A number of WAL segments to work with')
+                +form-field__number({
+                    label: 'WAL segments:',
+                    model: `${model}.walSegments`,
+                    name: '"DataStorageWalSegments"',
+                    placeholder: '10',
+                    min: '1',
+                    tip: 'A number of WAL segments to work with'
+                })
             .pc-form-grid-col-20
-                +number('WAL segment size:', `${model}.walSegmentSize`, '"DataStorageWalSegmentSize"', 'true', '67108864', '0', 'Size of a WAL segment')
+                +form-field__number({
+                    label: 'WAL segment size:',
+                    model: `${model}.walSegmentSize`,
+                    name: '"DataStorageWalSegmentSize"',
+                    placeholder: '67108864',
+                    min: '0',
+                    tip: 'Size of a WAL segment'
+                })
             .pc-form-grid-col-20
-                +number('WAL history size:', `${model}.walHistorySize`, '"DataStorageWalHistorySize"', 'true', '20', '1', 'A total number of checkpoints to keep in the WAL history')
+                +form-field__number({
+                    label: 'WAL history size:',
+                    model: `${model}.walHistorySize`,
+                    name: '"DataStorageWalHistorySize"',
+                    placeholder: '20',
+                    min: '1',
+                    tip: 'A total number of checkpoints to keep in the WAL history'
+                })
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.4.0")')
-                +number('WAL buffer size:', `${model}.walBufferSize`, '"DataStorageWalBufferSize"', 'true', 'WAL segment size / 4', '1',
-                    'Size of WAL buffer')
+                +form-field__number({
+                    label: 'WAL buffer size:',
+                    model: `${model}.walBufferSize`,
+                    name: '"DataStorageWalBufferSize"',
+                    placeholder: 'WAL segment size / 4',
+                    min: '1',
+                    tip: 'Size of WAL buffer'
+                })
             .pc-form-grid-col-30
-                +number('WAL flush frequency:', `${model}.walFlushFrequency`, '"DataStorageWalFlushFrequency"', 'true', '2000', '1',
-                    'How often will be fsync, in milliseconds. In background mode, exist thread which do fsync by timeout')
+                +form-field__number({
+                    label: 'WAL flush frequency:',
+                    model: `${model}.walFlushFrequency`,
+                    name: '"DataStorageWalFlushFrequency"',
+                    placeholder: '2000',
+                    min: '1',
+                    tip: 'How often will be fsync, in milliseconds. In background mode, exist thread which do fsync by timeout'
+                })
             .pc-form-grid-col-30
-                +number('WAL fsync delay:', `${model}.walFsyncDelayNanos`, '"DataStorageWalFsyncDelay"', 'true', '1000', '1', 'WAL fsync delay, in nanoseconds')
+                +form-field__number({
+                    label: 'WAL fsync delay:',
+                    model: `${model}.walFsyncDelayNanos`,
+                    name: '"DataStorageWalFsyncDelay"',
+                    placeholder: '1000',
+                    min: '1',
+                    tip: 'WAL fsync delay, in nanoseconds'
+                })
             .pc-form-grid-col-60
-                +number('WAL record iterator buffer size:', `${model}.walRecordIteratorBufferSize`, '"DataStorageWalRecordIteratorBufferSize"', 'true', '67108864', '1',
-                    'How many bytes iterator read from disk(for one reading), during go ahead WAL')
+                +form-field__number({
+                    label: 'WAL record iterator buffer size:',
+                    model: `${model}.walRecordIteratorBufferSize`,
+                    name: '"DataStorageWalRecordIteratorBufferSize"',
+                    placeholder: '67108864',
+                    min: '1',
+                    tip: 'How many bytes iterator read from disk(for one reading), during go ahead WAL'
+                })
             .pc-form-grid-col-30
-                +number('Lock wait time:', `${model}.lockWaitTime`, '"DataStorageLockWaitTime"', 'true', '10000', '1',
-                    'Time out in milliseconds, while wait and try get file lock for start persist manager')
+                +form-field__number({
+                    label: 'Lock wait time:',
+                    model: `${model}.lockWaitTime`,
+                    name: '"DataStorageLockWaitTime"',
+                    placeholder: '10000',
+                    min: '1',
+                    tip: 'Time out in milliseconds, while wait and try get file lock for start persist manager'
+                })
             .pc-form-grid-col-30
-                +number('WAL thread local buffer size:', `${model}.walThreadLocalBufferSize`, '"DataStorageWalThreadLocalBufferSize"', 'true', '131072', '1',
-                    'Define size thread local buffer. Each thread which write to WAL have thread local buffer for serialize recode before write in WAL')
+                +form-field__number({
+                    label: 'WAL thread local buffer size:',
+                    model: `${model}.walThreadLocalBufferSize`,
+                    name: '"DataStorageWalThreadLocalBufferSize"',
+                    placeholder: '131072',
+                    min: '1',
+                    tip: 'Define size thread local buffer. Each thread which write to WAL have thread local buffer for serialize recode before write in WAL'
+                })
             .pc-form-grid-col-30
-                +number('Metrics sub interval count:', `${model}.metricsSubIntervalCount`, '"DataStorageMetricsSubIntervalCount"', 'true', '5', '1',
-                    'Number of sub - intervals the whole rate time interval will be split into to calculate rate - based metrics')
+                +form-field__number({
+                    label: 'Metrics sub interval count:',
+                    model: `${model}.metricsSubIntervalCount`,
+                    name: '"DataStorageMetricsSubIntervalCount"',
+                    placeholder: '5',
+                    min: '1',
+                    tip: 'Number of sub - intervals the whole rate time interval will be split into to calculate rate - based metrics'
+                })
             .pc-form-grid-col-30
-                +number('Metrics rate time interval:', `${model}.metricsRateTimeInterval`, '"DataStorageMetricsRateTimeInterval"', 'true', '60000', '1000',
-                    'The length of the time interval for rate - based metrics. This interval defines a window over which hits will be tracked')
+                +form-field__number({
+                    label: 'Metrics rate time interval:',
+                    model: `${model}.metricsRateTimeInterval`,
+                    name: '"DataStorageMetricsRateTimeInterval"',
+                    placeholder: '60000',
+                    min: '1000',
+                    tip: 'The length of the time interval for rate - based metrics. This interval defines a window over which hits will be tracked'
+                })
             .pc-form-grid-col-30
-                +dropdown('File IO factory:', `${model}.fileIOFactory`, '"DataStorageFileIOFactory"', 'true', 'Default',
-                    '[\
+                +form-field__dropdown({
+                    label: 'File IO factory:',
+                    model: `${model}.fileIOFactory`,
+                    name: '"DataStorageFileIOFactory"',
+                    placeholder: 'Default',
+                    options: '[\
                         {value: "RANDOM", label: "RANDOM"},\
                         {value: "ASYNC", label: "ASYNC"},\
                         {value: null, label: "Default"},\
                     ]',
-                    'Order of writing pages to disk storage during checkpoint.\
-                    <ul>\
-                        <li>RANDOM - Pages are written in order provided by checkpoint pages collection iterator</li>\
-                        <li>SEQUENTIAL - All checkpoint pages are collected into single list and sorted by page index</li>\
-                    </ul>')
+                    tip: 'Order of writing pages to disk storage during checkpoint.\
+                        <ul>\
+                            <li>RANDOM - Pages are written in order provided by checkpoint pages collection iterator</li>\
+                            <li>SEQUENTIAL - All checkpoint pages are collected into single list and sorted by page index</li>\
+                        </ul>'
+                })
+
             .pc-form-grid-col-30
-                +number('WAL auto archive after inactivity:', `${model}.walAutoArchiveAfterInactivity`, '"DataStorageWalAutoArchiveAfterInactivity"', 'true', '-1', '-1',
-                    'Time in millis to run auto archiving segment after last record logging')
+                +form-field__number({
+                    label: 'WAL auto archive after inactivity:',
+                    model: `${model}.walAutoArchiveAfterInactivity`,
+                    name: '"DataStorageWalAutoArchiveAfterInactivity"',
+                    placeholder: '-1',
+                    min: '-1',
+                    tip: 'Time in millis to run auto archiving segment after last record logging'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('Metrics enabled', `${model}.metricsEnabled`, '"DataStorageMetricsEnabled"', 'true', 'Flag indicating whether persistence metrics collection is enabled')
+                +form-field__checkbox({
+                    label: 'Metrics enabled',
+                    model: `${model}.metricsEnabled`,
+                    name: '"DataStorageMetricsEnabled"',
+                    tip: 'Flag indicating whether persistence metrics collection is enabled'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('Always write full pages', `${model}.alwaysWriteFullPages`, '"DataStorageAlwaysWriteFullPages"', 'true', 'Flag indicating whether always write full pages')
+                +form-field__checkbox({
+                    label: 'Always write full pages',
+                    model: `${model}.alwaysWriteFullPages`,
+                    name: '"DataStorageAlwaysWriteFullPages"',
+                    tip: 'Flag indicating whether always write full pages'
+                })
             .pc-form-grid-col-60
-                +checkbox('Write throttling enabled', `${model}.writeThrottlingEnabled`, '"DataStorageWriteThrottlingEnabled"',
-                    'Throttle threads that generate dirty pages too fast during ongoing checkpoint')
+                +form-field__checkbox({
+                    label: 'Write throttling enabled',
+                    model: `${model}.writeThrottlingEnabled`,
+                    name: '"DataStorageWriteThrottlingEnabled"',
+                    tip: 'Throttle threads that generate dirty pages too fast during ongoing checkpoint'
+                })
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.4.0")')
-                +checkbox('Enable WAL compaction', `${model}.walCompactionEnabled`, '"DataStorageWalCompactionEnabled"',
-                    'If true, system filters and compresses WAL archive in background')
+                +form-field__checkbox({
+                    label: 'Enable WAL compaction',
+                    model: `${model}.walCompactionEnabled`,
+                    name: '"DataStorageWalCompactionEnabled"',
+                    tip: 'If true, system filters and compresses WAL archive in background'
+                })
 
         .pca-form-column-6
             +preview-xml-java(model, 'clusterDataStorageConfiguration')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/deployment.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/deployment.pug
index 1f0b61555ad19..1d9388682466e 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/deployment.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/deployment.pug
@@ -35,34 +35,58 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +dropdown('Deployment mode:', `${model}.deploymentMode`, '"deploymentMode"', 'true', 'SHARED',
-                    '[\
+                +form-field__dropdown({
+                    label: 'Deployment mode:',
+                    model: `${model}.deploymentMode`,
+                    name: '"deploymentMode"',
+                    placeholder: 'SHARED',
+                    options: '[\
                         {value: "PRIVATE", label: "PRIVATE"},\
                         {value: "ISOLATED", label: "ISOLATED"}, \
                         {value: "SHARED", label: "SHARED"},\
                         {value: "CONTINUOUS", label: "CONTINUOUS"}\
                     ]',
-                    'Task classes and resources sharing mode<br/>\
+                    tip: 'Task classes and resources sharing mode<br/>\
                     The following deployment modes are supported:\
-                    <ul>\
-                        <li>PRIVATE - in this mode deployed classes do not share resources</li>\
-                        <li>ISOLATED - in this mode tasks or classes deployed within the same class loader will share the same instances of resources</li>\
-                        <li>SHARED - same as ISOLATED, but now tasks from different master nodes with the same user version and same class loader will share the same class loader on remote nodes</li>\
-                        <li>CONTINUOUS - same as SHARED deployment mode, but resources will not be undeployed even after all master nodes left grid</li>\
-                    </ul>')
+                        <ul>\
+                            <li>PRIVATE - in this mode deployed classes do not share resources</li>\
+                            <li>ISOLATED - in this mode tasks or classes deployed within the same class loader will share the same instances of resources</li>\
+                            <li>SHARED - same as ISOLATED, but now tasks from different master nodes with the same user version and same class loader will share the same class loader on remote nodes</li>\
+                            <li>CONTINUOUS - same as SHARED deployment mode, but resources will not be undeployed even after all master nodes left grid</li>\
+                        </ul>'
+                })
             .pc-form-grid-col-60
-                +checkbox('Enable peer class loading', `${model}.peerClassLoadingEnabled`, '"peerClassLoadingEnabled"', 'Enables/disables peer class loading')
+                +form-field__checkbox({
+                    label: 'Enable peer class loading',
+                    model: `${model}.peerClassLoadingEnabled`,
+                    name: '"peerClassLoadingEnabled"',
+                    tip: 'Enables/disables peer class loading'
+                })
             .pc-form-grid-col-60
-                +number('Missed resources cache size:', `${model}.peerClassLoadingMissedResourcesCacheSize`, '"peerClassLoadingMissedResourcesCacheSize"', enabled, '100', '0',
-                    'If size greater than 0, missed resources will be cached and next resource request ignored<br/>\
-                    If size is 0, then request for the resource will be sent to the remote node every time this resource is requested')
+                +form-field__number({
+                    label: 'Missed resources cache size:',
+                    model: `${model}.peerClassLoadingMissedResourcesCacheSize`,
+                    name: '"peerClassLoadingMissedResourcesCacheSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '100',
+                    min: '0',
+                    tip: 'If size greater than 0, missed resources will be cached and next resource request ignored<br/>\
+                          If size is 0, then request for the resource will be sent to the remote node every time this resource is requested'
+                })
             .pc-form-grid-col-60
-                +number('Pool size:', `${model}.peerClassLoadingThreadPoolSize`, '"peerClassLoadingThreadPoolSize"', enabled, '2', '1', 'Thread pool size to use for peer class loading')
+                +form-field__number({
+                    label: 'Pool size:',
+                    model: `${model}.peerClassLoadingThreadPoolSize`,
+                    name: '"peerClassLoadingThreadPoolSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '2',
+                    min: '1',
+                    tip: 'Thread pool size to use for peer class loading'
+                })
             .pc-form-grid-col-60
                 mixin clusters-deployment-packages
                     .ignite-form-field
                         -let items = exclude
-                        -var uniqueTip = 'Such package already exists!'
 
                         list-editable(
                             ng-model=items
@@ -77,10 +101,17 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                             list-editable-item-view {{ $item }}
 
                             list-editable-item-edit
-                                +list-java-package-field('Package name', '$item', '"packageName"', items)(
-                                    ignite-auto-focus
+                                +form-field__java-package({
+                                    label: 'Package name',
+                                    model: '$item',
+                                    name: '"packageName"',
+                                    placeholder: 'Enter package name',
+                                    required: enabled
+                                })(
+                                    ignite-unique=items
+                                    ignite-form-field-input-autofocus='true'
                                 )
-                                    +unique-feedback('"packageName"', uniqueTip)
+                                    +form-field__error({ error: 'igniteUnique', message: 'Such package already exists!' })
 
                             list-editable-no-items
                                 list-editable-add-item-button(
@@ -95,30 +126,38 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
 
             //- Since ignite 2.0
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.0.0")')
-                +java-class('Class loader:', model + '.classLoader', '"classLoader"', 'true', 'false',
-                    'Loader which will be used for instantiating execution context')
+                +form-field__java-class({
+                    label: 'Class loader:',
+                    model: `${model}.classLoader`,
+                    name: '"classLoader"',
+                    tip: 'Loader which will be used for instantiating execution context'
+                })
 
             .pc-form-grid-col-60
-                +dropdown('Deployment variant:', modelDeployment + '.kind', '"deploymentKind"', 'true', 'Default',
-                    '[\
+                +form-field__dropdown({
+                    label: 'Deployment variant:',
+                    model: `${modelDeployment}.kind`,
+                    name: '"deploymentKind"',
+                    placeholder: 'Default',
+                    options: '[\
                         {value: "URI", label: "URI"},\
                         {value: "Local", label: "Local"}, \
                         {value: "Custom", label: "Custom"},\
                         {value: null, label: "Default"}\
                     ]',
-                    'Grid deployment SPI is in charge of deploying tasks and classes from different sources:\
-                    <ul>\
-                        <li>URI - Deploy tasks from different sources like file system folders, email and HTTP</li>\
-                        <li>Local - Only within VM deployment on local node</li>\
-                        <li>Custom - Custom implementation of DeploymentSpi</li>\
-                        <li>Default - Default configuration of LocalDeploymentSpi will be used</li>\
-                    </ul>')
+                    tip: 'Grid deployment SPI is in charge of deploying tasks and classes from different sources:\
+                        <ul>\
+                            <li>URI - Deploy tasks from different sources like file system folders, email and HTTP</li>\
+                            <li>Local - Only within VM deployment on local node</li>\
+                            <li>Custom - Custom implementation of DeploymentSpi</li>\
+                            <li>Default - Default configuration of LocalDeploymentSpi will be used</li>\
+                        </ul>'
+                })
             .pc-form-group(ng-show=uriDeployment).pc-form-grid-row
                 .pc-form-grid-col-60
                     mixin clusters-deployment-uri
                         .ignite-form-field
                             -let items = uriListModel
-                            -var uniqueTip = 'Such URI already configured!'
 
                             list-editable(
                                 ng-model=items
@@ -132,7 +171,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
 
                                 list-editable-item-edit
                                     +list-url-field('URL', '$item', '"url"', items)
-                                        +unique-feedback('"url"', uniqueTip)
+                                        +form-field__error({ error: 'igniteUnique', message: 'Such URI already configured!' })
 
                                 list-editable-no-items
                                     list-editable-add-item-button(
@@ -146,13 +185,17 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     - var form = 'deployment'
 
                 .pc-form-grid-col-60
-                    +text('Temporary directory path:', modelDeployment + '.URI.temporaryDirectoryPath', '"DeploymentURITemporaryDirectoryPath"', 'false', 'Temporary directory path',
-                    'Absolute path to temporary directory which will be used by deployment SPI to keep all deployed classes in')
+                    +form-field__text({
+                        label: 'Temporary directory path:',
+                        model: modelDeployment + '.URI.temporaryDirectoryPath',
+                        name: '"DeploymentURITemporaryDirectoryPath"',
+                        placeholder: 'Temporary directory path',
+                        tip: 'Absolute path to temporary directory which will be used by deployment SPI to keep all deployed classes in'
+                    })
                 .pc-form-grid-col-60
                     mixin clusters-deployment-scanner
                         .ignite-form-field
                             -let items = scannerModel
-                            -var uniqueTip = 'Such scanner already configured!'
 
                             list-editable(
                                 ng-model=items
@@ -163,7 +206,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
 
                                 list-editable-item-edit
                                     +list-java-class-field('Scanner', '$item', '"scanner"', items)
-                                        +unique-feedback('"scanner"', uniqueTip)
+                                        +form-field__error({ error: 'igniteUnique', message: 'Such scanner already configured!' })
 
                                 list-editable-no-items
                                     list-editable-add-item-button(
@@ -177,16 +220,45 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     - var form = 'deployment'
 
                 .pc-form-grid-col-60
-                    +java-class('Listener:', `${modelDeployment}.URI.listener`, '"DeploymentURIListener"', 'true', 'false', 'Deployment event listener', uriDeployment)
+                    +form-field__java-class({
+                        label: 'Listener:',
+                        model: `${modelDeployment}.URI.listener`,
+                        name: '"DeploymentURIListener"',
+                        tip: 'Deployment event listener',
+                        validationActive: uriDeployment
+                    })
                 .pc-form-grid-col-60
-                    +checkbox('Check MD5', `${modelDeployment}.URI.checkMd5`, '"DeploymentURICheckMd5"', 'Exclude files with same md5s from deployment')
+                    +form-field__checkbox({
+                        label: 'Check MD5',
+                        model: `${modelDeployment}.URI.checkMd5`,
+                        name: '"DeploymentURICheckMd5"',
+                        tip: 'Exclude files with same md5s from deployment'
+                    })
                 .pc-form-grid-col-60
-                    +checkbox('Encode URI', `${modelDeployment}.URI.encodeUri`, '"DeploymentURIEncodeUri"', 'URI must be encoded before usage')
+                    +form-field__checkbox({
+                        label: 'Encode URI',
+                        model: `${modelDeployment}.URI.encodeUri`,
+                        name: '"DeploymentURIEncodeUri"',
+                        tip: 'URI must be encoded before usage'
+                    })
             .pc-form-group(ng-show=localDeployment).pc-form-grid-row
                 .pc-form-grid-col-60
-                    +java-class('Listener:', `${modelDeployment}.Local.listener`, '"DeploymentLocalListener"', 'true', 'false', 'Deployment event listener', localDeployment)
+                    +form-field__java-class({
+                        label: 'Listener:',
+                        model: `${modelDeployment}.Local.listener`,
+                        name: '"DeploymentLocalListener"',
+                        tip: 'Deployment event listener',
+                        validationActive: localDeployment
+                    })
             .pc-form-group(ng-show=customDeployment).pc-form-grid-row
                 .pc-form-grid-col-60
-                    +java-class('Class:', `${modelDeployment}.Custom.className`, '"DeploymentCustom"', 'true', customDeployment, 'DeploymentSpi implementation class', customDeployment)
+                    +form-field__java-class({
+                        label: 'Class:',
+                        model: `${modelDeployment}.Custom.className`,
+                        name: '"DeploymentCustom"',
+                        required: customDeployment,
+                        tip: 'DeploymentSpi implementation class',
+                        validationActive: customDeployment
+                    })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterDeployment')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/discovery.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/discovery.pug
index d0a9102bbc1c6..0f777f9929aa9 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/discovery.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/discovery.pug
@@ -27,71 +27,205 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-20
-                +text-ip-address('Local address:', `${model}.localAddress`, '"discoLocalAddress"', 'true', '228.1.2.4',
-                    'Local host IP address that discovery SPI uses<br/>\
-                    If not provided a first found non-loopback address will be used')
+                +form-field__ip-address({
+                    label: 'Local address:',
+                    model: `${model}.localAddress`,
+                    name: '"discoLocalAddress"',
+                    enabled: 'true',
+                    placeholder: '228.1.2.4',
+                    tip: 'Local host IP address that discovery SPI uses<br/>\
+                         If not provided a first found non-loopback address will be used'
+                })
             .pc-form-grid-col-20
-                +number-min-max('Local port:', `${model}.localPort`, '"discoLocalPort"', 'true', '47500', '1024', '65535', 'Local port which node uses')
+                +form-field__number({
+                    label: 'Local port:',
+                    model: `${model}.localPort`,
+                    name: '"discoLocalPort"',
+                    placeholder: '47500',
+                    min: '1024',
+                    max: '65535',
+                    tip: 'Local port which node uses'
+                })
             .pc-form-grid-col-20
-                +number('Local port range:', `${model}.localPortRange`, '"discoLocalPortRange"', 'true', '100', '1', 'Local port range')
+                +form-field__number({
+                    label: 'Local port range:',
+                    model: `${model}.localPortRange`,
+                    name: '"discoLocalPortRange"',
+                    placeholder: '100',
+                    min: '1',
+                    tip: 'Local port range'
+                })
             .pc-form-grid-col-60
-                +java-class('Address resolver:', `${model}.addressResolver`, '"discoAddressResolver"', 'true', 'false',
-                    'Provides resolution between external and internal addresses')
+                +form-field__java-class({
+                    label:'Address resolver:',
+                    model: `${model}.addressResolver`,
+                    name: '"discoAddressResolver"',
+                    tip: 'Provides resolution between external and internal addresses'
+                })
             .pc-form-grid-col-30
-                +number('Socket timeout:', `${model}.socketTimeout`, '"socketTimeout"', 'true', '5000', '0', 'Socket operations timeout')
+                +form-field__number({
+                    label: 'Socket timeout:',
+                    model: `${model}.socketTimeout`,
+                    name: '"socketTimeout"',
+                    placeholder: '5000',
+                    min: '0',
+                    tip: 'Socket operations timeout'
+                })
             .pc-form-grid-col-30
-                +sane-ignite-form-field-number({
+                +form-field__number({
                     label: 'Acknowledgement timeout:',
                     model: `${model}.ackTimeout`,
                     name: '"ackTimeout"',
-                    disabled: 'false',
                     placeholder: '5000',
                     min: '0',
                     max: `{{ ${model}.maxAckTimeout || 600000 }}`,
                     tip: 'Message acknowledgement timeout'
                 })
-                    +form-field-feedback('"ackTimeout"', 'max', `Acknowledgement timeout should be less than max acknowledgement timeout ({{ ${model}.maxAckTimeout || 60000 }}).`)
+                    +form-field__error({ error: 'max', message: `Acknowledgement timeout should be less than max acknowledgement timeout ({{ ${model}.maxAckTimeout || 60000 }}).` })
             .pc-form-grid-col-30
-                +number('Max acknowledgement timeout:', `${model}.maxAckTimeout`, '"maxAckTimeout"', 'true', '600000', '0', 'Maximum message acknowledgement timeout')
+                +form-field__number({
+                    label: 'Max acknowledgement timeout:',
+                    model: `${model}.maxAckTimeout`,
+                    name: '"maxAckTimeout"',
+                    placeholder: '600000',
+                    min: '0',
+                    tip: 'Maximum message acknowledgement timeout'
+                })
             .pc-form-grid-col-30
-                +number('Network timeout:', `${model}.networkTimeout`, '"discoNetworkTimeout"', 'true', '5000', '1', 'Timeout to use for network operations')
+                +form-field__number({
+                    label: 'Network timeout:',
+                    model: `${model}.networkTimeout`,
+                    name: '"discoNetworkTimeout"',
+                    placeholder: '5000',
+                    min: '1',
+                    tip: 'Timeout to use for network operations'
+                })
             .pc-form-grid-col-30
-                +number('Join timeout:', `${model}.joinTimeout`, '"joinTimeout"', 'true', '0', '0',
-                    'Join timeout<br/>' +
-                    '0 means wait forever')
+                +form-field__number({
+                    label: 'Join timeout:',
+                    model: `${model}.joinTimeout`,
+                    name: '"joinTimeout"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Join timeout<br/>' +
+                          '0 means wait forever'
+                })
             .pc-form-grid-col-30
-                +number('Thread priority:', `${model}.threadPriority`, '"threadPriority"', 'true', '10', '1', 'Thread priority for all threads started by SPI')
+                +form-field__number({
+                    label: 'Thread priority:',
+                    model: `${model}.threadPriority`,
+                    name: '"threadPriority"',
+                    placeholder: '10',
+                    min: '1',
+                    tip: 'Thread priority for all threads started by SPI'
+                })
 
             //- Removed in ignite 2.0
             .pc-form-grid-col-60(ng-if-start='$ctrl.available(["1.0.0", "2.0.0"])')
-                +number('Heartbeat frequency:', `${model}.heartbeatFrequency`, '"heartbeatFrequency"', 'true', '2000', '1', 'Heartbeat messages issuing frequency')
+                +form-field__number({
+                    label: 'Heartbeat frequency:',
+                    model: `${model}.heartbeatFrequency`,
+                    name: '"heartbeatFrequency"',
+                    placeholder: '2000',
+                    min: '1',
+                    tip: 'Heartbeat messages issuing frequency'
+                })
             .pc-form-grid-col-30
-                +number('Max heartbeats miss w/o init:', `${model}.maxMissedHeartbeats`, '"maxMissedHeartbeats"', 'true', '1', '1',
-                    'Max heartbeats count node can miss without initiating status check')
+                +form-field__number({
+                    label: 'Max heartbeats miss w/o init:',
+                    model: `${model}.maxMissedHeartbeats`,
+                    name: '"maxMissedHeartbeats"',
+                    placeholder: '1',
+                    min: '1',
+                    tip: 'Max heartbeats count node can miss without initiating status check'
+                })
             .pc-form-grid-col-30(ng-if-end)
-                +number('Max missed client heartbeats:', `${model}.maxMissedClientHeartbeats`, '"maxMissedClientHeartbeats"', 'true', '5', '1',
-                    'Max heartbeats count node can miss without failing client node')
+                +form-field__number({
+                    label: 'Max missed client heartbeats:',
+                    model: `${model}.maxMissedClientHeartbeats`,
+                    name: '"maxMissedClientHeartbeats"',
+                    placeholder: '5',
+                    min: '1',
+                    tip: 'Max heartbeats count node can miss without failing client node'
+                })
 
             .pc-form-grid-col-60
-                +number('Topology history:', `${model}.topHistorySize`, '"topHistorySize"', 'true', '1000', '0', 'Size of topology snapshots history')
+                +form-field__number({
+                    label: 'Topology history:',
+                    model: `${model}.topHistorySize`,
+                    name: '"topHistorySize"',
+                    placeholder: '1000',
+                    min: '0',
+                    tip: 'Size of topology snapshots history'
+                })
             .pc-form-grid-col-60
-                +java-class('Discovery listener:', `${model}.listener`, '"discoListener"', 'true', 'false', 'Listener for grid node discovery events')
+                +form-field__java-class({
+                    label: 'Discovery listener:',
+                    model: `${model}.listener`,
+                    name: '"discoListener"',
+                    tip: 'Listener for grid node discovery events'
+                })
             .pc-form-grid-col-60
-                +java-class('Data exchange:', `${model}.dataExchange`, '"dataExchange"', 'true', 'false', 'Class name of handler for initial data exchange between Ignite nodes')
+                +form-field__java-class({
+                    label: 'Data exchange:',
+                    model: `${model}.dataExchange`,
+                    name: '"dataExchange"',
+                    tip: 'Class name of handler for initial data exchange between Ignite nodes'
+                })
             .pc-form-grid-col-60
-                +java-class('Metrics provider:', `${model}.metricsProvider`, '"metricsProvider"', 'true', 'false', 'Class name of metric provider to discovery SPI')
+                +form-field__java-class({
+                    label: 'Metrics provider:',
+                    model: `${model}.metricsProvider`,
+                    name: '"metricsProvider"',
+                    tip: 'Class name of metric provider to discovery SPI'
+                })
             .pc-form-grid-col-30
-                +number('Reconnect count:', `${model}.reconnectCount`, '"discoReconnectCount"', 'true', '10', '1', 'Reconnect attempts count')
+                +form-field__number({
+                    label: 'Reconnect count:',
+                    model: `${model}.reconnectCount`,
+                    name: '"discoReconnectCount"',
+                    placeholder: '10',
+                    min: '1',
+                    tip: 'Reconnect attempts count'
+                })
             .pc-form-grid-col-30
-                +number('Statistics frequency:', `${model}.statisticsPrintFrequency`, '"statisticsPrintFrequency"', 'true', '0', '1', 'Statistics print frequency')
+                +form-field__number({
+                    label: 'Statistics frequency:',
+                    model: `${model}.statisticsPrintFrequency`,
+                    name: '"statisticsPrintFrequency"',
+                    placeholder: '0',
+                    min: '1',
+                    tip: 'Statistics print frequency'
+                })
             .pc-form-grid-col-60
-                +number('IP finder clean frequency:', `${model}.ipFinderCleanFrequency`, '"ipFinderCleanFrequency"', 'true', '60000', '1', 'IP finder clean frequency')
+                +form-field__number({
+                    label: 'IP finder clean frequency:',
+                    model: `${model}.ipFinderCleanFrequency`,
+                    name: '"ipFinderCleanFrequency"',
+                    placeholder: '60000',
+                    min: '1',
+                    tip: 'IP finder clean frequency'
+                })
             .pc-form-grid-col-60
-                +java-class('Node authenticator:', `${model}.authenticator`, '"authenticator"', 'true', 'false', 'Class name of node authenticator implementation')
+                +form-field__java-class({
+                    label: 'Node authenticator:',
+                    model: `${model}.authenticator`,
+                    name: '"authenticator"',
+                    tip: 'Class name of node authenticator implementation'
+                })
             .pc-form-grid-col-60
-                +checkbox('Force server mode', `${model}.forceServerMode`, '"forceServerMode"', 'Force start TCP/IP discovery in server mode')
+                +form-field__checkbox({
+                    label: 'Force server mode',
+                    model: `${model}.forceServerMode`,
+                    name: '"forceServerMode"',
+                    tip: 'Force start TCP/IP discovery in server mode'
+                })
             .pc-form-grid-col-60
-                +checkbox('Client reconnect disabled', `${model}.clientReconnectDisabled`, '"clientReconnectDisabled"',
-                    'Disable try of client to reconnect after server detected client node failure')
+                +form-field__checkbox({
+                    label: 'Client reconnect disabled',
+                    model: `${model}.clientReconnectDisabled`,
+                    name: '"clientReconnectDisabled"',
+                    tip: 'Disable try of client to reconnect after server detected client node failure'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterDiscovery')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/events.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/events.pug
index a41999e09f37b..9967af4f66a69 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/events.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/events.pug
@@ -26,41 +26,87 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Events
     panel-description
-        | Grid events are used for notification about what happens within the grid. 
+        | Grid events are used for notification about what happens within the grid.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/events" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60(ng-if='$ctrl.available(["1.0.0", "2.0.0"])')
-                +dropdown('Event storage:', modelEventStorageKind, '"eventStorageKind"', 'true', 'Disabled', '$ctrl.eventStorage',
-                'Regulate how grid store events locally on node\
-                <ul>\
-                    <li>Memory - All events are kept in the FIFO queue in-memory</li>\
-                    <li>Custom - Custom implementation of event storage SPI</li>\
-                </ul>')
+                +form-field__dropdown({
+                    label: 'Event storage:',
+                    model: modelEventStorageKind,
+                    name: '"eventStorageKind"',
+                    placeholder: 'Disabled',
+                    options: '$ctrl.eventStorage',
+                    tip: 'Regulate how grid store events locally on node\
+                        <ul>\
+                            <li>Memory - All events are kept in the FIFO queue in-memory</li>\
+                            <li>Custom - Custom implementation of event storage SPI</li>\
+                        </ul>'
+                })
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.0.0")')
-                +dropdown('Event storage:', modelEventStorageKind, '"eventStorageKind"', 'true', 'Disabled', '$ctrl.eventStorage',
-                'Regulate how grid store events locally on node\
-                <ul>\
-                    <li>Memory - All events are kept in the FIFO queue in-memory</li>\
-                    <li>Custom - Custom implementation of event storage SPI</li>\
-                    <li>Disabled - Events are not collected</li>\
-                </ul>')
+                +form-field__dropdown({
+                    label: 'Event storage:',
+                    model: modelEventStorageKind,
+                    name: '"eventStorageKind"',
+                    placeholder: 'Disabled',
+                    options: '$ctrl.eventStorage',
+                    tip: 'Regulate how grid store events locally on node\
+                        <ul>\
+                            <li>Memory - All events are kept in the FIFO queue in-memory</li>\
+                            <li>Custom - Custom implementation of event storage SPI</li>\
+                            <li>Disabled - Events are not collected</li>\
+                        </ul>'
+                })
             .pc-form-group.pc-form-grid-row(ng-if=modelEventStorageKind)
                 .pc-form-grid-col-30(ng-if-start=eventStorageMemory)
-                    +number('Events expiration time:', `${modelEventStorage}.Memory.expireAgeMs`, '"EventStorageExpireAgeMs"', 'true', 'Long.MAX_VALUE', '1', 'All events that exceed this value will be removed from the queue when next event comes')
+                    +form-field__number({
+                        label: 'Events expiration time:',
+                        model: `${modelEventStorage}.Memory.expireAgeMs`,
+                        name: '"writeBehindBatchSize"',
+                        placeholder: 'Long.MAX_VALUE',
+                        min: '1',
+                        tip: 'All events that exceed this value will be removed from the queue when next event comes'
+                    })
                 .pc-form-grid-col-30
-                    +number('Events queue size:', `${modelEventStorage}.Memory.expireCount`, '"EventStorageExpireCount"', 'true', '10000', '1', 'Events will be filtered out when new request comes')
+                    +form-field__number({
+                        label: 'Events queue size:',
+                        model: `${modelEventStorage}.Memory.expireCount`,
+                        name: '"EventStorageExpireCount"',
+                        placeholder: '10000',
+                        min: '1',
+                        tip: 'Events will be filtered out when new request comes'
+                    })
                 .pc-form-grid-col-60(ng-if-end)
-                    +java-class('Filter:', `${modelEventStorage}.Memory.filter`, '"EventStorageFilter"', 'true', 'false',
-                    'Filter for events to be recorded<br/>\
-                    Should be implementation of o.a.i.lang.IgnitePredicate&lt;o.a.i.events.Event&gt;', eventStorageMemory)
+                    +form-field__java-class({
+                        label: 'Filter:',
+                        model: `${modelEventStorage}.Memory.filter`,
+                        name: '"EventStorageFilter"',
+                        tip: 'Filter for events to be recorded<br/>\
+                             Should be implementation of o.a.i.lang.IgnitePredicate&lt;o.a.i.events.Event&gt;',
+                        validationActive: eventStorageMemory
+                    })
 
                 .pc-form-grid-col-60(ng-if=eventStorageCustom)
-                    +java-class('Class:', `${modelEventStorage}.Custom.className`, '"EventStorageCustom"', 'true', eventStorageCustom, 'Event storage implementation class name', eventStorageCustom)
+                    +form-field__java-class({
+                        label: 'Class:',
+                        model: `${modelEventStorage}.Custom.className`,
+                        name: '"EventStorageCustom"',
+                        required: eventStorageCustom,
+                        tip: 'Event storage implementation class name',
+                        validationActive: eventStorageCustom
+                    })
 
                 .pc-form-grid-col-60
-                    +dropdown-multiple('Include type:', `${model}.includeEventTypes`, '"includeEventTypes"', true, 'Choose recorded event types', '', '$ctrl.eventGroups',
-                    'Array of event types, which will be recorded by GridEventStorageManager#record(Event)<br/>\
-                    Note, that either the include event types or the exclude event types can be established')
+                    +form-field__dropdown({
+                        label: 'Include type:',
+                        model: `${model}.includeEventTypes`,
+                        name: '"includeEventTypes"',
+                        multiple: true,
+                        placeholder: 'Choose recorded event types',
+                        placeholderEmpty: '',
+                        options: '$ctrl.eventGroups',
+                        tip: 'Array of event types, which will be recorded by GridEventStorageManager#record(Event)<br/>\
+                             Note, that either the include event types or the exclude event types can be established'
+                    })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterEvents')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug
index 85c441e8fd37d..3fafe9f86f845 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/failover.pug
@@ -24,63 +24,92 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Failover configuration
     panel-description
-        | Failover SPI provides ability to supply custom logic for handling failed execution of a grid job. 
+        | Failover SPI provides ability to supply custom logic for handling failed execution of a grid job.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/fault-tolerance" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             //- Since ignite 2.0
             .pc-form-grid-col-60(ng-if-start='$ctrl.available("2.0.0")')
-                +number('Failure detection timeout:', model + '.failureDetectionTimeout', '"failureDetectionTimeout"', 'true',
-                    '10000', '1', 'Failure detection timeout is used to determine how long the communication or discovery SPIs should wait before considering a remote connection failed')
+                +form-field__number({
+                    label: 'Failure detection timeout:',
+                    model: model + '.failureDetectionTimeout',
+                    name: '"failureDetectionTimeout"',
+                    placeholder: '10000',
+                    min: '1',
+                    tip: 'Failure detection timeout is used to determine how long the communication or discovery SPIs should wait before considering a remote connection failed'
+                })
             .pc-form-grid-col-60(ng-if-end)
-                +number('Client failure detection timeout:', model + '.clientFailureDetectionTimeout', '"clientFailureDetectionTimeout"', 'true',
-                    '30000', '1', 'Failure detection timeout is used to determine how long the communication or discovery SPIs should wait before considering a remote connection failed')
+                +form-field__number({
+                    label: 'Client failure detection timeout:',
+                    model: model + '.clientFailureDetectionTimeout',
+                    name: '"clientFailureDetectionTimeout"',
+                    placeholder: '30000',
+                    min: '1',
+                    tip: 'Failure detection timeout is used to determine how long the communication or discovery SPIs should wait before considering a remote connection failed'
+                })
 
             .pc-form-grid-col-60
                 mixin clusters-failover-spi
                     .ignite-form-field
-                        +ignite-form-field__label('Failover SPI configurations:', '"failoverSpi"')
-                            +tooltip(`Failover SPI configurations`)
-                        .ignite-form-field__control
-                            -let items = failoverSpi
+                        +form-field__label({ label: 'Failover SPI configurations:', name: '"failoverSpi"' })
+                            +form-field__tooltip({ title: `Failover SPI configurations` })
+                        -let items = failoverSpi
 
-                            list-editable.pc-list-editable-with-form-grid(ng-model=items name='failoverSpi')
-                                list-editable-item-edit.pc-form-grid-row
-                                    .pc-form-grid-col-60
-                                        +sane-ignite-form-field-dropdown({
-                                            required: true,
-                                            label: 'Failover SPI:',
-                                            model: '$item.kind',
-                                            name: '"failoverKind"',
-                                            placeholder: 'Choose Failover SPI',
-                                            options: '::$ctrl.Clusters.failoverSpis',
-                                            tip: `
-                                            Provides ability to supply custom logic for handling failed execution of a grid job
-                                            <ul>
-                                                <li>Job stealing - Supports job stealing from over-utilized nodes to under-utilized nodes</li>
-                                                <li>Never - Jobs are ordered as they arrived</li>
-                                                <li>Always - Jobs are first ordered by their priority</li>
-                                                <li>Custom - Jobs are activated immediately on arrival to mapped node</li>
-                                                <li>Default - Default FailoverSpi implementation</li>
-                                            </ul>`
-                                        })
+                        list-editable.pc-list-editable-with-form-grid(ng-model=items name='failoverSpi')
+                            list-editable-item-edit.pc-form-grid-row
+                                .pc-form-grid-col-60
+                                    +form-field__dropdown({
+                                        required: true,
+                                        label: 'Failover SPI:',
+                                        model: '$item.kind',
+                                        name: '"failoverKind"',
+                                        placeholder: 'Choose Failover SPI',
+                                        options: '::$ctrl.Clusters.failoverSpis',
+                                        tip: `
+                                        Provides ability to supply custom logic for handling failed execution of a grid job
+                                        <ul>
+                                            <li>Job stealing - Supports job stealing from over-utilized nodes to under-utilized nodes</li>
+                                            <li>Never - Jobs are ordered as they arrived</li>
+                                            <li>Always - Jobs are first ordered by their priority</li>
+                                            <li>Custom - Jobs are activated immediately on arrival to mapped node</li>
+                                            <li>Default - Default FailoverSpi implementation</li>
+                                        </ul>`
+                                    })
 
-                                    .pc-form-grid-col-60(ng-show='$item.kind === "JobStealing"')
-                                        +number('Maximum failover attempts:', '$item.JobStealing.maximumFailoverAttempts', '"jsMaximumFailoverAttempts"', 'true', '5', '0',
-                                            'Maximum number of attempts to execute a failed job on another node')
-                                    .pc-form-grid-col-60(ng-show='$item.kind === "Always"')
-                                        +number('Maximum failover attempts:', '$item.Always.maximumFailoverAttempts', '"alwaysMaximumFailoverAttempts"', 'true', '5', '0',
-                                            'Maximum number of attempts to execute a failed job on another node')
-                                    .pc-form-grid-col-60(ng-show=failoverCustom)
-                                        +java-class('SPI implementation', '$item.Custom.class', '"failoverSpiClass"', 'true', failoverCustom,
-                                            'Custom FailoverSpi implementation class name.', failoverCustom)
+                                .pc-form-grid-col-60(ng-show='$item.kind === "JobStealing"')
+                                    +form-field__number({
+                                        label: 'Maximum failover attempts:',
+                                        model: '$item.JobStealing.maximumFailoverAttempts',
+                                        name: '"jsMaximumFailoverAttempts"',
+                                        placeholder: '5',
+                                        min: '0',
+                                        tip: 'Maximum number of attempts to execute a failed job on another node'
+                                    })
+                                .pc-form-grid-col-60(ng-show='$item.kind === "Always"')
+                                    +form-field__number({
+                                        label: 'Maximum failover attempts:',
+                                        model: '$item.Always.maximumFailoverAttempts',
+                                        name: '"alwaysMaximumFailoverAttempts"',
+                                        placeholder: '5',
+                                        min: '0',
+                                        tip: 'Maximum number of attempts to execute a failed job on another node'
+                                    })
+                                .pc-form-grid-col-60(ng-show=failoverCustom)
+                                    +form-field__java-class({
+                                        label: 'SPI implementation',
+                                        model: '$item.Custom.class',
+                                        name: '"failoverSpiClass"',
+                                        required: failoverCustom,
+                                        tip: 'Custom FailoverSpi implementation class name.',
+                                        validationActive: failoverCustom
+                                    })
 
-                                list-editable-no-items
-                                    list-editable-add-item-button(
-                                        add-item=`(${items} = ${items} || []).push({})`
-                                        label-single='failover SPI'
-                                        label-multiple='failover SPIs'
-                                    )
+                            list-editable-no-items
+                                list-editable-add-item-button(
+                                    add-item=`(${items} = ${items} || []).push({})`
+                                    label-single='failover SPI'
+                                    label-multiple='failover SPIs'
+                                )
 
                 +clusters-failover-spi
 
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general.pug
index 86f6384e473ae..26a949a7ca97e 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general.pug
@@ -38,11 +38,10 @@ panel-collapsible(opened=`::true` ng-form=form)
     panel-content.pca-form-row
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-30
-                +sane-ignite-form-field-text({
+                +form-field__text({
                     label: 'Name:',
                     model: `${model}.name`,
                     name: '"clusterName"',
-                    disabled: 'false',
                     placeholder: 'Input name',
                     required: true,
                     tip: 'Instance name allows to indicate to what grid this particular grid instance belongs to'
@@ -51,28 +50,40 @@ panel-collapsible(opened=`::true` ng-form=form)
                     ignite-unique-property='name'
                     ignite-unique-skip=`["_id", ${model}]`
                 )
-                    +unique-feedback(`${model}.name`, 'Cluster name should be unique.')
+                    +form-field__error({ error: 'igniteUnique', message: 'Cluster name should be unique.' })
 
             .pc-form-grid-col-30
-                +text-ip-address('Local host:', `${model}.localHost`, '"localHost"', 'true', '0.0.0.0',
-                    'System-wide local address or host for all Ignite components to bind to<br/>\
-                    If not defined then Ignite tries to use local wildcard address<br/>\
-                    That means that all services will be available on all network interfaces of the host machine')
+                +form-field__ip-address({
+                    label: 'Local host:',
+                    model: `${model}.localHost`,
+                    name: '"localHost"',
+                    enabled: 'true',
+                    placeholder: '0.0.0.0',
+                    tip: 'System-wide local address or host for all Ignite components to bind to<br/>\
+                          If not defined then Ignite tries to use local wildcard address<br/>\
+                          That means that all services will be available on all network interfaces of the host machine'
+                })
 
             .pc-form-grid-col-60
-                +dropdown('Discovery:', `${model}.discovery.kind`, '"discovery"', 'true', 'Choose discovery', '$ctrl.Clusters.discoveries',
-                'Discovery allows to discover remote nodes in grid\
-                <ul>\
-                    <li>Static IPs - IP Finder which works only with pre configured list of IP addresses specified</li>\
-                    <li>Multicast - Multicast based IP finder</li>\
-                    <li>AWS S3 - AWS S3 based IP finder that automatically discover cluster nodes on Amazon EC2 cloud</li>\
-                    <li>Apache jclouds - Apache jclouds multi cloud toolkit based IP finder for cloud platforms with unstable IP addresses</li>\
-                    <li>Google cloud storage - Google Cloud Storage based IP finder that automatically discover cluster nodes on Google Compute Engine cluster</li>\
-                    <li>JDBC - JDBC based IP finder that use database to store node IP address</li>\
-                    <li>Shared filesystem - Shared filesystem based IP finder that use file to store node IP address</li>\
-                    <li>Apache ZooKeeper - Apache ZooKeeper based IP finder when you use ZooKeeper to coordinate your distributed environment</li>\
-                    <li>Kubernetes - IP finder for automatic lookup of Ignite nodes running in Kubernetes environment</li>\
-                </ul>')
+                +form-field__dropdown({
+                    label: 'Discovery:',
+                    model: `${model}.discovery.kind`,
+                    name: '"discovery"',
+                    placeholder: 'Choose discovery',
+                    options: '$ctrl.Clusters.discoveries',
+                    tip: 'Discovery allows to discover remote nodes in grid\
+                        <ul>\
+                            <li>Static IPs - IP Finder which works only with pre configured list of IP addresses specified</li>\
+                            <li>Multicast - Multicast based IP finder</li>\
+                            <li>AWS S3 - AWS S3 based IP finder that automatically discover cluster nodes on Amazon EC2 cloud</li>\
+                            <li>Apache jclouds - Apache jclouds multi cloud toolkit based IP finder for cloud platforms with unstable IP addresses</li>\
+                            <li>Google cloud storage - Google Cloud Storage based IP finder that automatically discover cluster nodes on Google Compute Engine cluster</li>\
+                            <li>JDBC - JDBC based IP finder that use database to store node IP address</li>\
+                            <li>Shared filesystem - Shared filesystem based IP finder that use file to store node IP address</li>\
+                            <li>Apache ZooKeeper - Apache ZooKeeper based IP finder when you use ZooKeeper to coordinate your distributed environment</li>\
+                            <li>Kubernetes - IP finder for automatic lookup of Ignite nodes running in Kubernetes environment</li>\
+                        </ul>'
+                })
             .pc-form-group
                 +discovery-cloud()(ng-if=`${modelDiscoveryKind} === 'Cloud'`)
                 +discovery-google()(ng-if=`${modelDiscoveryKind} === 'GoogleStorage'`)
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/cloud.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/cloud.pug
index 074756e0bfe8d..800302ad0c65a 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/cloud.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/cloud.pug
@@ -28,19 +28,42 @@ mixin discovery-cloud(modelAt='$ctrl.clonedCluster')
 
     div.pc-form-grid-row&attributes(attributes=attributes)
         .pc-form-grid-col-30
-            +text('Credential:', `${model}.credential`, '"credential"', 'false', 'Input cloud credential',
-                'Credential that is used during authentication on the cloud<br/>\
-                Depending on a cloud platform it can be a password or access key')
+            +form-field__text({
+                label: 'Credential:',
+                model: `${model}.credential`,
+                name: '"credential"',
+                placeholder: 'Input cloud credential',
+                tip: 'Credential that is used during authentication on the cloud<br/>\
+                      Depending on a cloud platform it can be a password or access key'
+            })
         .pc-form-grid-col-30
-            +text('Path to credential:', `${model}.credentialPath`, '"credentialPath"', 'false', 'Input path to credential',
-                'Path to a credential that is used during authentication on the cloud<br/>\
-                Access key or private key should be stored in a plain or PEM file without a passphrase')
+            +form-field__text({
+                label: 'Path to credential:',
+                model: `${model}.credentialPath`,
+                name: '"credentialPath"',
+                placeholder: 'Input path to credential',
+                tip: 'Path to a credential that is used during authentication on the cloud<br/>\
+                     Access key or private key should be stored in a plain or PEM file without a passphrase'
+            })
         .pc-form-grid-col-30
-            +text('Identity:', `${model}.identity`, '"' + discoveryKind + 'Identity"', required, 'Input identity',
-                'Identity that is used as a user name during a connection to the cloud<br/>\
-                Depending on a cloud platform it can be an email address, user name, etc')
+            +form-field__text({
+                label: 'Identity:',
+                model: `${model}.identity`,
+                name: '"' + discoveryKind + 'Identity"',
+                required: required,
+                placeholder: 'Input identity',
+                tip: 'Identity that is used as a user name during a connection to the cloud<br/>\
+                     Depending on a cloud platform it can be an email address, user name, etc'
+            })
         .pc-form-grid-col-30
-            +text('Provider:', `${model}.provider`, '"' + discoveryKind + 'Provider"', required, 'Input provider', 'Cloud provider to use')
+            +form-field__text({
+                label:'Provider:',
+                model: `${model}.provider`,
+                name: '"' + discoveryKind + 'Provider"',
+                required: required,
+                placeholder: 'Input provider',
+                tip: 'Cloud provider to use'
+            })
         .pc-form-grid-col-60
             .ignite-form-field
                 +list-text-field({
@@ -57,8 +80,7 @@ mixin discovery-cloud(modelAt='$ctrl.clonedCluster')
                         Note, that some cloud providers, like Google Compute Engine, doesn't have a notion of a region. For such providers regions are redundant"
                     }]`
                 )
-                    +unique-feedback(_, 'Such region already exists!')
-
+                    +form-field__error({ error: 'igniteUnique', message: 'Such region already exists!' })
         .pc-form-grid-col-60
             .ignite-form-field
                 +list-text-field({
@@ -75,4 +97,4 @@ mixin discovery-cloud(modelAt='$ctrl.clonedCluster')
                         Note, that some cloud providers, like Rackspace, doesn't have a notion of a zone. For such providers zones are redundant"
                     }]`
                 )
-                    +unique-feedback(_, 'Such zone already exists!')
+                    +form-field__error({ error: 'igniteUnique', message: 'Such zone already exists!' })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/google.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/google.pug
index 7de38435311b0..01996ac99830e 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/google.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/google.pug
@@ -23,16 +23,41 @@ mixin discovery-google(modelAt = '$ctrl.clonedCluster')
 
     .pc-form-grid-row&attributes(attributes=attributes)
         .pc-form-grid-col-30
-            +text('Project name:', `${model}.projectName`, `'${discoveryKind}ProjectName'`, required, 'Input project name', '' +
-                'Google Cloud Platforms project name<br/>\
-                Usually this is an auto generated project number(ex. 208709979073) that can be found in "Overview" section of Google Developer Console')
+            +form-field__text({
+                label: 'Project name:',
+                model: `${model}.projectName`,
+                name: `'${discoveryKind}ProjectName'`,
+                required: required,
+                placeholder: 'Input project name',
+                tip: 'Google Cloud Platforms project name<br/>\
+                     Usually this is an auto generated project number(ex. 208709979073) that can be found in "Overview" section of Google Developer Console'
+            })
         .pc-form-grid-col-30
-            +text('Bucket name:', `${model}.bucketName`, `'${discoveryKind}BucketName'`, required, 'Input bucket name',
-                'Google Cloud Storage bucket name<br/>\
-                If the bucket does not exist Ignite will automatically create it<br/>\
-                However the name must be unique across whole Google Cloud Storage and Service Account Id must be authorized to perform this operation')
+            +form-field__text({
+                label: 'Bucket name:',
+                model: `${model}.bucketName`,
+                name: `'${discoveryKind}BucketName'`,
+                required: required,
+                placeholder: 'Input bucket name',
+                tip: 'Google Cloud Storage bucket name<br/>\
+                     If the bucket does not exist Ignite will automatically create it<br/>\
+                     However the name must be unique across whole Google Cloud Storage and Service Account Id must be authorized to perform this operation'
+            })
         .pc-form-grid-col-30
-            +text('Private key path:', `${model}.serviceAccountP12FilePath`, `'${discoveryKind}ServiceAccountP12FilePath'`, required, 'Input private key path',
-                'Full path to the private key in PKCS12 format of the Service Account')
+            +form-field__text({
+                label: 'Private key path:',
+                model: `${model}.serviceAccountP12FilePath`,
+                name: `'${discoveryKind}ServiceAccountP12FilePath'`,
+                required: required,
+                placeholder: 'Input private key path',
+                tip: 'Full path to the private key in PKCS12 format of the Service Account'
+            })
         .pc-form-grid-col-30
-            +text('Account id:', `${model}.serviceAccountId`, `'${discoveryKind}ServiceAccountId'`, required, 'Input account id', 'Service account ID (typically an e-mail address)')
+            +form-field__text({
+                label: 'Account id:',
+                model: `${model}.serviceAccountId`,
+                name: `'${discoveryKind}ServiceAccountId'`,
+                required: required,
+                placeholder: 'Input account id',
+                tip: 'Service account ID (typically an e-mail address)'
+            })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/jdbc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/jdbc.pug
index 7b23a225b9af2..eb9f0aa5147c5 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/jdbc.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/jdbc.pug
@@ -22,14 +22,31 @@ mixin discovery-jdbc(modelAt = '$ctrl.clonedCluster')
 
     .pc-form-grid-row&attributes(attributes=attributes)
         .pc-form-grid-col-30
-            +text('Data source bean name:', `${model}.dataSourceBean`,
-                '"dataSourceBean"', required, 'Input bean name', 'Name of the data source bean in Spring context')
+            +form-field__text({
+                label: 'Data source bean name:',
+                model: `${model}.dataSourceBean`,
+                name: '"dataSourceBean"',
+                required: required,
+                placeholder:'Input bean name',
+                tip: 'Name of the data source bean in Spring context'
+            })
         .pc-form-grid-col-30
-            +dialect('Dialect:', `${model}.dialect`, '"dialect"', required,
-                'Dialect of SQL implemented by a particular RDBMS:', 'Generic JDBC dialect', 'Choose JDBC dialect')
+            +form-field__dialect({
+                label: 'Dialect:',
+                model: `${model}.dialect`,
+                name: '"dialect"',
+                required,
+                tip: 'Dialect of SQL implemented by a particular RDBMS:',
+                genericDialectName: 'Generic JDBC dialect',
+                placeholder: 'Choose JDBC dialect'
+            })
         .pc-form-grid-col-60
-            +checkbox('DB schema should be initialized by Ignite', `${model}.initSchema`, '"initSchema"',
-                'Flag indicating whether DB schema should be initialized by Ignite or was explicitly created by user')
+            +form-field__checkbox({
+                label: 'DB schema should be initialized by Ignite',
+                model: `${model}.initSchema`,
+                name: '"initSchema"',
+                tip: 'Flag indicating whether DB schema should be initialized by Ignite or was explicitly created by user'
+            })
         .pc-form-grid-col-30(ng-if=`$ctrl.Clusters.requiresProprietaryDrivers(${modelAt})`)
             a.link-success(ng-href=`{{ $ctrl.Clusters.JDBCDriverURL(${modelAt}) }}` target='_blank')
                 | Download JDBC drivers?
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/kubernetes.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/kubernetes.pug
index 9232022303e66..32d94fcbc8348 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/kubernetes.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/kubernetes.pug
@@ -22,17 +22,38 @@ mixin discovery-kubernetes(modelAt = '$ctrl.clonedCluster')
 
     .pc-form-grid-row&attributes(attributes=attributes)
         .pc-form-grid-col-30
-            +text('Service name:', `${model}.serviceName`, `'${discoveryKind}ServiceName'`, 'false', 'ignite',
-                "The name of Kubernetes service for Ignite pods' IP addresses lookup.<br/>\
-                The name of the service must be equal to the name set in service's Kubernetes configuration.<br/>\
-                If this parameter is not changed then the name of the service has to be set to 'ignite' in the corresponding Kubernetes configuration.")
+            +form-field__text({
+                label: 'Service name:',
+                model: `${model}.serviceName`,
+                name: `'${discoveryKind}ServiceName'`,
+                placeholder: 'ignite',
+                tip: "The name of Kubernetes service for Ignite pods' IP addresses lookup.<br/>\
+                     The name of the service must be equal to the name set in service's Kubernetes configuration.<br/>\
+                     If this parameter is not changed then the name of the service has to be set to 'ignite' in the corresponding Kubernetes configuration."
+            })
         .pc-form-grid-col-30
-            +text('Namespace:', `${model}.namespace`, `'${discoveryKind}Namespace'`, 'false', 'default',
-                "The namespace the Kubernetes service belongs to.<br/>\
-                By default, it's supposed that the service is running under Kubernetes `default` namespace.")
+            +form-field__text({
+                label: 'Namespace:',
+                model: `${model}.namespace`,
+                name: `'${discoveryKind}Namespace'`,
+                placeholder: 'default',
+                tip: "The namespace the Kubernetes service belongs to.<br/>\
+                      By default, it's supposed that the service is running under Kubernetes `default` namespace."
+            })
         .pc-form-grid-col-60
-            +url('Kubernetes server:', `${model}.masterUrl`, `'${discoveryKind}MasterUrl'`, 'true', 'false', 'https://kubernetes.default.svc.cluster.local:443',
-                'The host name of the Kubernetes API server')
+            +form-field__url({
+                label: 'Kubernetes server:',
+                model: `${model}.masterUrl`,
+                name: `'${discoveryKind}MasterUrl'`,
+                enabled: 'true',
+                placeholder: 'https://kubernetes.default.svc.cluster.local:443',
+                tip: 'The host name of the Kubernetes API server'
+            })
         .pc-form-grid-col-60
-            +text('Service token file:', `${model}.accountToken`, `'${discoveryKind}AccountToken'`, 'false', '/var/run/secrets/kubernetes.io/serviceaccount/token',
-                'The path to the service token file')
+            +form-field__text({
+                label: 'Service token file:',
+                model: `${model}.accountToken`,
+                name: `'${discoveryKind}AccountToken'`,
+                placeholder: '/var/run/secrets/kubernetes.io/serviceaccount/token',
+                tip: 'The path to the service token file'
+            })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast.pug
index 2d7aa4bd6b578..b767e9cfd7943 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/multicast.pug
@@ -22,42 +22,73 @@ mixin discovery-multicast(modelAt = '$ctrl.clonedCluster')
 
     .pc-form-grid-row&attributes(attributes=attributes)
         .pc-form-grid-col-30
-            +text-ip-address('IP address:', `${model}.multicastGroup`, '"multicastGroup"', 'true', '228.1.2.4', 'IP address of multicast group')
+            +form-field__ip-address({
+                label: 'IP address:',
+                model: `${model}.multicastGroup`,
+                name: '"multicastGroup"',
+                enabled: 'true',
+                placeholder: '228.1.2.4',
+                tip: 'IP address of multicast group'
+            })
         .pc-form-grid-col-30
-            +number-min-max('Port number:', `${model}.multicastPort`, '"multicastPort"', 'true', '47400', '0', '65535', 'Port number which multicast messages are sent to')
+            +form-field__number({
+                label: 'Port number:',
+                model: `${model}.multicastPort`,
+                name: '"multicastPort"',
+                placeholder: '47400',
+                min: '0',
+                max: '65535',
+                tip: 'Port number which multicast messages are sent to'
+            })
         .pc-form-grid-col-20
-            +number('Waits for reply:', `${model}.responseWaitTime`, '"responseWaitTime"', 'true', '500', '0',
-                'Time in milliseconds IP finder waits for reply to multicast address request')
+            +form-field__number({
+                label: 'Waits for reply:',
+                model: `${model}.responseWaitTime`,
+                name: '"responseWaitTime"',
+                placeholder: '500',
+                min: '0',
+                tip: 'Time in milliseconds IP finder waits for reply to multicast address request'
+            })
         .pc-form-grid-col-20
-            +number('Attempts count:', `${model}.addressRequestAttempts`, '"addressRequestAttempts"', 'true', '2', '0',
-                'Number of attempts to send multicast address request<br/>\
-                IP finder re - sends request only in case if no reply for previous request is received')
+            +form-field__number({
+                label: 'Attempts count:',
+                model: `${model}.addressRequestAttempts`,
+                name: '"addressRequestAttempts"',
+                placeholder: '2',
+                min: '0',
+                tip: 'Number of attempts to send multicast address request<br/>\
+                     IP finder re - sends request only in case if no reply for previous request is received'
+            })
         .pc-form-grid-col-20.pc-form-grid-col-free
-            +text-ip-address('Local address:', `${model}.localAddress`, '"localAddress"', 'true', '0.0.0.0',
-                'Local host address used by this IP finder<br/>\
-                If provided address is non - loopback then multicast socket is bound to this interface<br/>\
-                If local address is not set or is any local address then IP finder creates multicast sockets for all found non - loopback addresses')
+            +form-field__ip-address({
+                label: 'Local address:',
+                model: `${model}.localAddress`,
+                name: '"localAddress"',
+                enabled: 'true',
+                placeholder: '0.0.0.0',
+                tip: 'Local host address used by this IP finder<br/>\
+                     If provided address is non - loopback then multicast socket is bound to this interface<br/>\
+                     If local address is not set or is any local address then IP finder creates multicast sockets for all found non - loopback addresses'
+            })
         .pc-form-grid-col-60
             .ignite-form-field
-                .ignite-form-field__control
-                    +list-addresses({
-                        items: addresses,
-                        name: 'multicastAddresses',
-                        tip: `Addresses may be represented as follows:
-                        <ul>
-                            <li>IP address (e.g. 127.0.0.1, 9.9.9.9, etc)</li>
-                            <li>IP address and port (e.g. 127.0.0.1:47500, 9.9.9.9:47501, etc)</li>
-                            <li>IP address and port range (e.g. 127.0.0.1:47500..47510, 9.9.9.9:47501..47504, etc)</li>
-                            <li>Hostname (e.g. host1.com, host2, etc)</li>
-                            <li>Hostname and port (e.g. host1.com:47500, host2:47502, etc)</li>
-                            <li>Hostname and port range (e.g. host1.com:47500..47510, host2:47502..47508, etc)</li>
-                        </ul>
-                        If port is 0 or not provided then default port will be used (depends on discovery SPI configuration)<br />
-                        If port range is provided (e.g. host:port1..port2) the following should be considered:
-                        </ul>
-                        <ul>
-                            <li> port1 &lt; port2 should be true</li>
-                            <li> Both port1 and port2 should be greater than 0</li>
-                        </ul>`
-                    })
-                    
+                +list-addresses({
+                    items: addresses,
+                    name: 'multicastAddresses',
+                    tip: `Addresses may be represented as follows:
+                    <ul>
+                        <li>IP address (e.g. 127.0.0.1, 9.9.9.9, etc)</li>
+                        <li>IP address and port (e.g. 127.0.0.1:47500, 9.9.9.9:47501, etc)</li>
+                        <li>IP address and port range (e.g. 127.0.0.1:47500..47510, 9.9.9.9:47501..47504, etc)</li>
+                        <li>Hostname (e.g. host1.com, host2, etc)</li>
+                        <li>Hostname and port (e.g. host1.com:47500, host2:47502, etc)</li>
+                        <li>Hostname and port range (e.g. host1.com:47500..47510, host2:47502..47508, etc)</li>
+                    </ul>
+                    If port is 0 or not provided then default port will be used (depends on discovery SPI configuration)<br />
+                    If port range is provided (e.g. host:port1..port2) the following should be considered:
+                    </ul>
+                    <ul>
+                        <li> port1 &lt; port2 should be true</li>
+                        <li> Both port1 and port2 should be greater than 0</li>
+                    </ul>`
+                })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/s3.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/s3.pug
index 41d45ac1704e2..dc1882468d6f4 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/s3.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/s3.pug
@@ -24,15 +24,32 @@ mixin discovery-s3(modelAt = '$ctrl.clonedCluster')
 
     .pc-form-grid-row&attributes(attributes=attributes)
         .pc-form-grid-col-30
-            +text('Bucket name:', `${model}.bucketName`, `'${discoveryKind}BucketName'`, required, 'Input bucket name', 'Bucket name for IP finder')
+            +form-field__text({
+                label: 'Bucket name:',
+                model: `${model}.bucketName`,
+                name: `'${discoveryKind}BucketName'`,
+                required: required,
+                placeholder: 'Input bucket name',
+                tip: 'Bucket name for IP finder'
+            })
         .pc-form-grid-col-30
             .pc-form-grid__text-only-item(style='font-style: italic;color: #424242;')
                 | AWS credentials will be generated as stub
         .pc-form-grid-col-40(ng-if-start=`$ctrl.available("2.4.0")`)
-            +text('Bucket endpoint:', `${model}.bucketEndpoint`, `'${discoveryKind}BucketEndpoint'`, false, 'Input bucket endpoint',
-            'Bucket endpoint for IP finder<br/> \
-            For information about possible endpoint names visit <a href="http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region">docs.aws.amazon.com</a>')
+            +form-field__text({
+                label: 'Bucket endpoint:',
+                model: `${model}.bucketEndpoint`,
+                name: `'${discoveryKind}BucketEndpoint'`,
+                placeholder: 'Input bucket endpoint',
+                tip: 'Bucket endpoint for IP finder<br/> \
+                      For information about possible endpoint names visit <a href="http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region">docs.aws.amazon.com</a>'
+            })
         .pc-form-grid-col-20(ng-if-end)
-            +text('SSE algorithm:', `${model}.SSEAlgorithm`, `'${discoveryKind}SSEAlgorithm'`, false, 'Input SSE algorithm',
-            'Server-side encryption algorithm for Amazon S3-managed encryption keys<br/> \
-            For information about possible S3-managed encryption keys visit <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html">docs.aws.amazon.com</a>')
\ No newline at end of file
+            +form-field__text({
+                label: 'SSE algorithm:',
+                model: `${model}.SSEAlgorithm`,
+                name: `'${discoveryKind}SSEAlgorithm'`,
+                placeholder: 'Input SSE algorithm',
+                tip: 'Server-side encryption algorithm for Amazon S3-managed encryption keys<br/> \
+                      For information about possible S3-managed encryption keys visit <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html">docs.aws.amazon.com</a>'
+            })
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/shared.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/shared.pug
index 83e8f2a15d541..e5b86c3e1c01c 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/shared.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/shared.pug
@@ -21,4 +21,10 @@ mixin discovery-shared(modelAt = '$ctrl.clonedCluster')
 
     .pc-form-grid-row&attributes(attributes=attributes)
         .pc-form-grid-col-60
-            +text('File path:', `${model}.path`, '"path"', 'false', 'disco/tcp', 'Shared path')
+            +form-field__text({
+                label: 'File path:',
+                model: `${model}.path`,
+                name: '"path"',
+                placeholder: 'disco/tcp',
+                tip: 'Shared path'
+            })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/vm.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/vm.pug
index 1266f8652d172..aee3a1b0b4e5f 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/vm.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/vm.pug
@@ -23,33 +23,33 @@ mixin discovery-vm(modelAt = '$ctrl.clonedCluster')
 
     .pc-form-grid-row&attributes(attributes=attributes)
         .pc-form-grid-col-60
-            .ignite-form-field
-                .ignite-form-field__control
-                    +list-addresses({
-                        items: addresses,
-                        name: 'vmAddresses',
-                        tip: `Addresses may be represented as follows:
-                            <ul>
-                                <li>IP address (e.g. 127.0.0.1, 9.9.9.9, etc)</li>
-                                <li>IP address and port (e.g. 127.0.0.1:47500, 9.9.9.9:47501, etc)</li>
-                                <li>IP address and port range (e.g. 127.0.0.1:47500..47510, 9.9.9.9:47501..47504, etc)</li>
-                                <li>Hostname (e.g. host1.com, host2, etc)</li>
-                                <li>Hostname and port (e.g. host1.com:47500, host2:47502, etc)</li>
-                                <li>Hostname and port range (e.g. host1.com:47500..47510, host2:47502..47508, etc)</li>
-                            </ul>
-                            If port is 0 or not provided then default port will be used (depends on discovery SPI configuration)<br />
-                            If port range is provided (e.g. host:port1..port2) the following should be considered:
-                            </ul>
-                            <ul>
-                                <li> port1 &lt; port2 should be true</li>
-                                <li> Both port1 and port2 should be greater than 0</li>
-                            </ul>`
-                    })(
-                        ng-required='true'
-                        expose-ignite-form-field-control='$vmAddresses'
-                    )
-                .ignite-form-field__errors(
+            .form-field.ignite-form-field
+                +list-addresses({
+                    items: addresses,
+                    name: 'vmAddresses',
+                    tip: `Addresses may be represented as follows:
+                        <ul>
+                            <li>IP address (e.g. 127.0.0.1, 9.9.9.9, etc)</li>
+                            <li>IP address and port (e.g. 127.0.0.1:47500, 9.9.9.9:47501, etc)</li>
+                            <li>IP address and port range (e.g. 127.0.0.1:47500..47510, 9.9.9.9:47501..47504, etc)</li>
+                            <li>Hostname (e.g. host1.com, host2, etc)</li>
+                            <li>Hostname and port (e.g. host1.com:47500, host2:47502, etc)</li>
+                            <li>Hostname and port range (e.g. host1.com:47500..47510, host2:47502..47508, etc)</li>
+                        </ul>
+                        If port is 0 or not provided then default port will be used (depends on discovery SPI configuration)<br />
+                        If port range is provided (e.g. host:port1..port2) the following should be considered:
+                        </ul>
+                        <ul>
+                            <li> port1 &lt; port2 should be true</li>
+                            <li> Both port1 and port2 should be greater than 0</li>
+                        </ul>`
+                })(
+                    ng-required='true'
+                    ng-ref='$vmAddresses'
+                    ng-ref-read='ngModel'
+                )
+                .form-field__errors(
                     ng-messages=`$vmAddresses.$error`
                     ng-show=`$vmAddresses.$invalid`
                 )
-                    +form-field-feedback(_, 'required', 'Addresses should be configured')
+                    +form-field__error({ error: 'required', message: 'Addresses should be configured' })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper.pug
index 826e09ba0d234..53d704f945fd1 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper.pug
@@ -25,36 +25,52 @@ mixin discovery-zookeeper(modelAt = '$ctrl.clonedCluster')
 
     .pc-form-grid-row&attributes(attributes=attributes)
         .pc-form-grid-col-60
-            +java-class('Curator:', `${model}.curator`, '"curator"', 'true', 'false',
-                'The Curator framework in use<br/>\
-                By default generates curator of org.apache.curator. framework.imps.CuratorFrameworkImpl\
-                class with configured connect string, retry policy, and default session and connection timeouts', required)
+            +form-field__java-class({
+                label: 'Curator:',
+                model: `${model}.curator`,
+                name: '"curator"',
+                tip: 'The Curator framework in use<br/>\
+                     By default generates curator of org.apache.curator. framework.imps.CuratorFrameworkImpl\
+                     class with configured connect string, retry policy, and default session and connection timeouts',
+                validationActive: required
+            })
         .pc-form-grid-col-60
-            +text('Connect string:', `${model}.zkConnectionString`, `'${discoveryKind}ConnectionString'`, required, 'host:port[chroot][,host:port[chroot]]',
-                'When <b>IGNITE_ZK_CONNECTION_STRING</b> system property is not configured this property will be used.<br><br>This should be a comma separated host:port pairs, each corresponding to a zk server. e.g. "127.0.0.1:3000,127.0.0.1:3001".<br>If the optional chroot suffix is used the example would look like: "127.0.0.1:3000,127.0.0.1:3002/app/a".<br><br>Where the client would be rooted at "/app/a" and all paths would be relative to this root - ie getting/setting/etc... "/foo/bar" would result in operations being run on "/app/a/foo/bar" (from the server perspective).<br><br><a href="https://zookeeper.apache.org/doc/r3.2.2/api/org/apache/zookeeper/ZooKeeper.html#ZooKeeper(java.lang.String,%20int,%20org.apache.zookeeper.Watcher)">Zookeeper docs</a>')
+            +form-field__text({
+                label: 'Connect string:',
+                model: `${model}.zkConnectionString`,
+                name: `'${discoveryKind}ConnectionString'`,
+                required: required,
+                placeholder: 'host:port[chroot][,host:port[chroot]]',
+                tip: 'When <b>IGNITE_ZK_CONNECTION_STRING</b> system property is not configured this property will be used.<br><br>This should be a comma separated host:port pairs, each corresponding to a zk server. e.g. "127.0.0.1:3000,127.0.0.1:3001".<br>If the optional chroot suffix is used the example would look like: "127.0.0.1:3000,127.0.0.1:3002/app/a".<br><br>Where the client would be rooted at "/app/a" and all paths would be relative to this root - ie getting/setting/etc... "/foo/bar" would result in operations being run on "/app/a/foo/bar" (from the server perspective).<br><br><a href="https://zookeeper.apache.org/doc/r3.2.2/api/org/apache/zookeeper/ZooKeeper.html#ZooKeeper(java.lang.String,%20int,%20org.apache.zookeeper.Watcher)">Zookeeper docs</a>'
+            })
         .pc-form-grid-col-60
-            +dropdown('Retry policy:', `${model}.retryPolicy.kind`, '"retryPolicy"', 'true', 'Default',
-            '[\
-                {value: "ExponentialBackoff", label: "Exponential backoff"},\
-                {value: "BoundedExponentialBackoff", label: "Bounded exponential backoff"},\
-                {value: "UntilElapsed", label: "Until elapsed"},\
-                {value: "NTimes", label: "Max number of times"},\
-                {value: "OneTime", label: "Only once"},\
-                {value: "Forever", label: "Always allow retry"},\
-                {value: "Custom", label: "Custom"},\
-                {value: null, label: "Default"}\
-            ]',
-            'Available retry policies:\
-            <ul>\
-                <li>Exponential backoff - retries a set number of times with increasing sleep time between retries</li>\
-                <li>Bounded exponential backoff - retries a set number of times with an increasing (up to a maximum bound) sleep time between retries</li>\
-                <li>Until elapsed - retries until a given amount of time elapses</li>\
-                <li>Max number of times - retries a max number of times</li>\
-                <li>Only once - retries only once</li>\
-                <li>Always allow retry - retries infinitely</li>\
-                <li>Custom - custom retry policy implementation</li>\
-                <li>Default - exponential backoff retry policy with configured base sleep time equal to 1000ms and max retry count equal to 10</li>\
-            </ul>')
+            +form-field__dropdown({
+                label: 'Retry policy:',
+                model: `${model}.retryPolicy.kind`,
+                name: '"retryPolicy"',
+                placeholder: 'Default',
+                options: '[\
+                                {value: "ExponentialBackoff", label: "Exponential backoff"},\
+                                {value: "BoundedExponentialBackoff", label: "Bounded exponential backoff"},\
+                                {value: "UntilElapsed", label: "Until elapsed"},\
+                                {value: "NTimes", label: "Max number of times"},\
+                                {value: "OneTime", label: "Only once"},\
+                                {value: "Forever", label: "Always allow retry"},\
+                                {value: "Custom", label: "Custom"},\
+                                {value: null, label: "Default"}\
+                            ]',
+                tip: 'Available retry policies:\
+                            <ul>\
+                                <li>Exponential backoff - retries a set number of times with increasing sleep time between retries</li>\
+                                <li>Bounded exponential backoff - retries a set number of times with an increasing (up to a maximum bound) sleep time between retries</li>\
+                                <li>Until elapsed - retries until a given amount of time elapses</li>\
+                                <li>Max number of times - retries a max number of times</li>\
+                                <li>Only once - retries only once</li>\
+                                <li>Always allow retry - retries infinitely</li>\
+                                <li>Custom - custom retry policy implementation</li>\
+                                <li>Default - exponential backoff retry policy with configured base sleep time equal to 1000ms and max retry count equal to 10</li>\
+                            </ul>'
+            })
 
         .pc-form-grid__break
 
@@ -69,16 +85,31 @@ mixin discovery-zookeeper(modelAt = '$ctrl.clonedCluster')
         .pc-form-grid-col-30
             -var model = `${modelAt}.discovery.ZooKeeper`
 
-            +text('Base path:', `${model}.basePath`, '"basePath"', 'false', '/services', 'Base path for service registration')
+            +form-field__text({
+                label: 'Base path:',
+                model: `${model}.basePath`,
+                name: '"basePath"',
+                placeholder: '/services',
+                tip: 'Base path for service registration'
+            })
         .pc-form-grid-col-30
-            +text('Service name:', `${model}.serviceName`, '"serviceName"', 'false', 'ignite',
-                'Service name to use, as defined by Curator&#39;s ServiceDiscovery recipe<br/>\
-                In physical ZooKeeper terms, it represents the node under basePath, under which services will be registered')
+            +form-field__text({
+                label:'Service name:',
+                model: `${model}.serviceName`,
+                name: '"serviceName"',
+                placeholder: 'ignite',
+                tip: 'Service name to use, as defined by Curator&#39;s ServiceDiscovery recipe<br/>\
+                      In physical ZooKeeper terms, it represents the node under basePath, under which services will be registered'
+            })
 
         .pc-form-grid__break
 
         .pc-form-grid-col-60
-            +checkbox('Allow duplicate registrations', `${model}.allowDuplicateRegistrations`, '"allowDuplicateRegistrations"',
-                'Whether to register each node only once, or if duplicate registrations are allowed<br/>\
-                Nodes will attempt to register themselves, plus those they know about<br/>\
-                By default, duplicate registrations are not allowed, but you might want to set this property to <b>true</b> if you have multiple network interfaces or if you are facing troubles')
+            +form-field__checkbox({
+                label: 'Allow duplicate registrations',
+                model: `${model}.allowDuplicateRegistrations`,
+                name: '"allowDuplicateRegistrations"',
+                tip: 'Whether to register each node only once, or if duplicate registrations are allowed<br/>\
+                     Nodes will attempt to register themselves, plus those they know about<br/>\
+                     By default, duplicate registrations are not allowed, but you might want to set this property to <b>true</b> if you have multiple network interfaces or if you are facing troubles'
+            })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
index 0ddc1e9d16bfc..84f7f2d80ddaf 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/bounded-exponential-backoff.pug
@@ -19,8 +19,30 @@ include /app/helpers/jade/mixins
 -var model = `${modelAt}.discovery.ZooKeeper.retryPolicy.BoundedExponentialBackoff`
 
 .pc-form-grid-col-20(ng-if-start=`${modelRetryPolicyKind} === 'BoundedExponentialBackoff'`)
-    +number('Base interval:', `${model}.baseSleepTimeMs`, '"beBaseSleepTimeMs"', 'true', '1000', '0', 'Initial amount of time in ms to wait between retries')
+    +form-field__number({
+        label: 'Base interval:',
+        model: `${model}.baseSleepTimeMs`,
+        name: '"beBaseSleepTimeMs"',
+        placeholder: '1000',
+        min: '0',
+        tip: 'Initial amount of time in ms to wait between retries'
+    })
 .pc-form-grid-col-20
-    +number('Max interval:', `${model}.maxSleepTimeMs`, '"beMaxSleepTimeMs"', 'true', 'Integer.MAX_VALUE', '0', 'Max time in ms to sleep on each retry')
+    +form-field__number({
+        label: 'Max interval:',
+        model: `${model}.maxSleepTimeMs`,
+        name: '"beMaxSleepTimeMs"',
+        placeholder: 'Integer.MAX_VALUE',
+        min: '0',
+        tip: 'Max time in ms to sleep on each retry'
+    })
 .pc-form-grid-col-20(ng-if-end)
-    +number-min-max('Max retries:', `${model}.maxRetries`, '"beMaxRetries"', 'true', '10', '0', '29', 'Max number of times to retry')
+    +form-field__number({
+        label: 'Max retries:',
+        model: `${model}.maxRetries`,
+        name: '"beMaxRetries"',
+        placeholder: '10',
+        min: '0',
+        max: '29',
+        tip: 'Max number of times to retry'
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/custom.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/custom.pug
index 6a1bcfb78c411..1cac6b804a9b4 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/custom.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/custom.pug
@@ -21,5 +21,12 @@ include /app/helpers/jade/mixins
 -var required = `${modelAt}.discovery.kind === "ZooKeeper" && ${modelAt}.discovery.ZooKeeper.retryPolicy.kind === "Custom"`
 
 .pc-form-grid-col-60(ng-if-start=`${modelRetryPolicyKind} === 'Custom'`)
-    +java-class('Class name:', `${retry}.className`, '"customClassName"', 'true', required, 'Custom retry policy implementation class name', required)
-.pc-form-grid__break(ng-if-end)
\ No newline at end of file
+    +form-field__java-class({
+        label: 'Class name:',
+        model: `${retry}.className`,
+        name: '"customClassName"',
+        required: required,
+        tip: 'Custom retry policy implementation class name',
+        validationActive: required
+    })
+.pc-form-grid__break(ng-if-end)
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
index bfc3c025dc3ba..ed0b5ffd7e600 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/exponential-backoff.pug
@@ -19,8 +19,30 @@ include /app/helpers/jade/mixins
 -var model = `${modelAt}.discovery.ZooKeeper.retryPolicy.ExponentialBackoff`
 
 .pc-form-grid-col-20(ng-if-start=`${modelRetryPolicyKind} === 'ExponentialBackoff'`)
-    +number('Base interval:', `${model}.baseSleepTimeMs`, '"expBaseSleepTimeMs"', 'true', '1000', '0', 'Initial amount of time in ms to wait between retries')
+    +form-field__number({
+        label: 'Base interval:',
+        model: `${model}.baseSleepTimeMs`,
+        name: '"expBaseSleepTimeMs"',
+        placeholder: '1000',
+        min: '0',
+        tip: 'Initial amount of time in ms to wait between retries'
+    })
 .pc-form-grid-col-20
-    +number-min-max('Max retries:', `${model}.maxRetries`, '"expMaxRetries"', 'true', '10', '0', '29', 'Max number of times to retry')
+    +form-field__number({
+        label: 'Max retries:',
+        model: `${model}.maxRetries`,
+        name: '"expMaxRetries"',
+        placeholder: '10',
+        min: '0',
+        max: '29',
+        tip: 'Max number of times to retry'
+    })
 .pc-form-grid-col-20(ng-if-end)
-    +number('Max interval:', `${model}.maxSleepMs`, '"expMaxSleepMs"', 'true', 'Integer.MAX_VALUE', '0', 'Max time in ms to sleep on each retry')
+    +form-field__number({
+        label: 'Max interval:',
+        model: `${model}.maxSleepMs`,
+        name: '"expMaxSleepMs"',
+        placeholder: 'Integer.MAX_VALUE',
+        min: '0',
+        tip: 'Max time in ms to sleep on each retry'
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/forever.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/forever.pug
index 575106b0e37dc..e61a3c6f15d29 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/forever.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/forever.pug
@@ -19,5 +19,12 @@ include /app/helpers/jade/mixins
 -var model = `${modelAt}.discovery.ZooKeeper.retryPolicy.Forever`
 
 .pc-form-grid-col-30(ng-if-start=`${modelRetryPolicyKind} === 'Forever'`)
-    +number('Interval:', `${model}.retryIntervalMs`, '"feRetryIntervalMs"', 'true', '1000', '0', 'Time in ms between retry attempts')
+    +form-field__number({
+        label: 'Interval:',
+        model: `${model}.retryIntervalMs`,
+        name: '"feRetryIntervalMs"',
+        placeholder: '1000',
+        min: '0',
+        tip: 'Time in ms between retry attempts'
+    })
 .pc-form-grid__break(ng-if-end)
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/n-times.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/n-times.pug
index dbb54e5ff1aa3..e44d030ffa9f5 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/n-times.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/n-times.pug
@@ -19,6 +19,20 @@ include /app/helpers/jade/mixins
 -var model = `${modelAt}.discovery.ZooKeeper.retryPolicy.NTimes`
 
 .pc-form-grid-col-30(ng-if-start=`${modelRetryPolicyKind} === 'NTimes'`)
-    +number('Retries:', `${model}.n`, '"n"', 'true', '10', '0', 'Number of times to retry')
+    +form-field__number({
+        label: 'Retries:',
+        model: `${model}.n`,
+        name: '"n"',
+        placeholder: '10',
+        min: '0',
+        tip: 'Number of times to retry'
+    })
 .pc-form-grid-col-30(ng-if-end)
-    +number('Interval:', `${model}.sleepMsBetweenRetries`, '"ntSleepMsBetweenRetries"', 'true', '1000', '0', 'Time in ms between retry attempts')
+    +form-field__number({
+        label: 'Interval:',
+        model: `${model}.sleepMsBetweenRetries`,
+        name: '"ntSleepMsBetweenRetries"',
+        placeholder: '1000',
+        min: '0',
+        tip: 'Time in ms between retry attempts'
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/one-time.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/one-time.pug
index 4ff16448cbd4e..4d86f5c3a88f3 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/one-time.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/one-time.pug
@@ -19,5 +19,12 @@ include /app/helpers/jade/mixins
 -var model = `${modelAt}.discovery.ZooKeeper.retryPolicy.OneTime`
 
 .pc-form-grid-col-30(ng-if-start=`${modelRetryPolicyKind} === 'OneTime'`)
-    +number('Interval:', `${model}.sleepMsBetweenRetry`, '"oneSleepMsBetweenRetry"', 'true', '1000', '0', 'Time in ms to retry attempt')
-.pc-form-grid__break(ng-if-end)
\ No newline at end of file
+    +form-field__number({
+        label: 'Interval:',
+        model: `${model}.sleepMsBetweenRetry`,
+        name: '"oneSleepMsBetweenRetry"',
+        placeholder: '1000',
+        min: '0',
+        tip: 'Time in ms to retry attempt'
+    })
+.pc-form-grid__break(ng-if-end)
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/until-elapsed.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
index ebde01c2c5fd2..acb1dff9e085d 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/general/discovery/zookeeper/retrypolicy/until-elapsed.pug
@@ -19,6 +19,20 @@ include /app/helpers/jade/mixins
 -var model = `${modelAt}.discovery.ZooKeeper.retryPolicy.UntilElapsed`
 
 .pc-form-grid-col-30(ng-if-start=`${modelRetryPolicyKind} === 'UntilElapsed'`)
-    +number('Total time:', `${model}.maxElapsedTimeMs`, '"ueMaxElapsedTimeMs"', 'true', '60000', '0', 'Total time in ms for execution of retry attempt')
+    +form-field__number({
+        label: 'Total time:',
+        model: `${model}.maxElapsedTimeMs`,
+        name: '"ueMaxElapsedTimeMs"',
+        placeholder: '60000',
+        min: '0',
+        tip: 'Total time in ms for execution of retry attempt'
+    })
 .pc-form-grid-col-30(ng-if-end)
-    +number('Interval:', `${model}.sleepMsBetweenRetries`, '"ueSleepMsBetweenRetries"', 'true', '1000', '0', 'Time in ms between retry attempts')
+    +form-field__number({
+        label: 'Interval:',
+        model: `${model}.sleepMsBetweenRetries`,
+        name: '"ueSleepMsBetweenRetries"',
+        placeholder: '1000',
+        min: '0',
+        tip: 'Time in ms between retry attempts'
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/hadoop.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/hadoop.pug
index 16a072c80e0f0..082f7bdc3af5f 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/hadoop.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/hadoop.pug
@@ -25,51 +25,110 @@ include /app/helpers/jade/mixins
 -var libs = model + '.nativeLibraryNames'
 
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
-    -var uniqueTip = 'Such native library already exists!'
-
     panel-title Hadoop configuration
     panel-description Hadoop Accelerator configuration.
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +dropdown('Map reduce planner:', plannerModel + '.kind', '"MapReducePlanner"', 'true', 'Default', '[\
-                    {value: "Weighted", label: "Weighted"},\
-                    {value: "Custom", label: "Custom"},\
-                    {value: null, label: "Default"}\
-                ]', 'Implementation of map reduce planner\
-                <ul>\
-                    <li>Weighted - Planner which assigns mappers and reducers based on their "weights"</li>\
-                    <li>Custom - Custom planner implementation</li>\
-                    <li>Default - Default planner implementation</li>\
-                </ul>')
+                +form-field__dropdown({
+                    label: 'Map reduce planner:',
+                    model: `${plannerModel}.kind`,
+                    name: '"MapReducePlanner"',
+                    placeholder: 'Default',
+                    options: '[\
+                        {value: "Weighted", label: "Weighted"},\
+                        {value: "Custom", label: "Custom"},\
+                        {value: null, label: "Default"}\
+                    ]',
+                    tip: 'Implementation of map reduce planner\
+                        <ul>\
+                            <li>Weighted - Planner which assigns mappers and reducers based on their "weights"</li>\
+                            <li>Custom - Custom planner implementation</li>\
+                            <li>Default - Default planner implementation</li>\
+                        </ul>'
+                })
             .pc-form-group.pc-form-grid-row(ng-show=weightedPlanner)
                 .pc-form-grid-col-20
-                    +number('Local mapper weight:', weightedModel + '.localMapperWeight', '"LocalMapperWeight"', 'true', 100, '0',
-                        'This weight is added to a node when a mapper is assigned and it is input split data is located on this node')
+                    +form-field__number({
+                        label: 'Local mapper weight:',
+                        model: `${weightedModel}.localMapperWeight`,
+                        name: '"LocalMapperWeight"',
+                        placeholder: '100',
+                        min: '0',
+                        tip: 'This weight is added to a node when a mapper is assigned and it is input split data is located on this node'
+                    })
                 .pc-form-grid-col-20
-                    +number('Remote mapper weight:', weightedModel + '.remoteMapperWeight', '"remoteMapperWeight"', 'true', 100, '0',
-                        'This weight is added to a node when a mapper is assigned, but it is input split data is not located on this node')
+                    +form-field__number({
+                        label: 'Remote mapper weight:',
+                        model: `${weightedModel}.remoteMapperWeight`,
+                        name: '"remoteMapperWeight"',
+                        placeholder: '100',
+                        min: '0',
+                        tip: 'This weight is added to a node when a mapper is assigned, but it is input split data is not located on this node'
+                    })
                 .pc-form-grid-col-20
-                    +number('Local reducer weight:', weightedModel + '.localReducerWeight', '"localReducerWeight"', 'true', 100, '0',
-                        'This weight is added to a node when a reducer is assigned and the node have at least one assigned mapper')
+                    +form-field__number({
+                        label: 'Local reducer weight:',
+                        model: `${weightedModel}.localReducerWeight`,
+                        name: '"localReducerWeight"',
+                        placeholder: '100',
+                        min: '0',
+                        tip: 'This weight is added to a node when a reducer is assigned and the node have at least one assigned mapper'
+                    })
                 .pc-form-grid-col-30
-                    +number('Remote reducer weight:', weightedModel + '.remoteReducerWeight', '"remoteReducerWeight"', 'true', 100, '0',
-                        'This weight is added to a node when a reducer is assigned, but the node does not have any assigned mappers')
+                    +form-field__number({
+                        label: 'Remote reducer weight:',
+                        model: `${weightedModel}.remoteReducerWeight`,
+                        name: '"remoteReducerWeight"',
+                        placeholder: '100',
+                        min: '0',
+                        tip: 'This weight is added to a node when a reducer is assigned, but the node does not have any assigned mappers'
+                    })
                 .pc-form-grid-col-30
-                    +number('Local mapper weight:', weightedModel + '.preferLocalReducerThresholdWeight', '"preferLocalReducerThresholdWeight"', 'true', 200, '0',
-                        "When threshold is reached, a node with mappers is no longer considered as preferred for further reducer assignments")
+                    +form-field__number({
+                        label: 'Local mapper weight:',
+                        model: `${weightedModel}.preferLocalReducerThresholdWeight`,
+                        name: '"preferLocalReducerThresholdWeight"',
+                        placeholder: '200',
+                        min: '0',
+                        tip: 'When threshold is reached, a node with mappers is no longer considered as preferred for further reducer assignments'
+                    })
             .pc-form-group.pc-form-grid-row(ng-show=customPlanner)
                 .pc-form-grid-col-60
-                    +java-class('Class name:', plannerModel + '.Custom.className', '"MapReducePlannerCustomClass"', 'true', customPlanner,
-                        'Custom planner implementation')
+                    +form-field__java-class({
+                        label: 'Class name:',
+                        model: `${plannerModel}.Custom.className`,
+                        name: '"MapReducePlannerCustomClass"',
+                        required: customPlanner,
+                        tip: 'Custom planner implementation'
+                    })
             .pc-form-grid-col-30
-                +number('Finished job info TTL:', model + '.finishedJobInfoTtl', '"finishedJobInfoTtl"', 'true', '30000', '0',
-                    'Finished job info time-to-live in milliseconds')
+                +form-field__number({
+                    label: 'Finished job info TTL:',
+                    model: `${model}.finishedJobInfoTtl`,
+                    name: '"finishedJobInfoTtl"',
+                    placeholder: '30000',
+                    min: '0',
+                    tip: 'Finished job info time-to-live in milliseconds'
+                })
             .pc-form-grid-col-30
-                +number('Max parallel tasks:', model + '.maxParallelTasks', '"maxParallelTasks"', 'true', 'availableProcessors * 2', '1',
-                    'Max number of local tasks that may be executed in parallel')
+                +form-field__number({
+                    label: 'Max parallel tasks:',
+                    model: `${model}.maxParallelTasks`,
+                    name: '"maxParallelTasks"',
+                    placeholder: 'availableProcessors * 2',
+                    min: '1',
+                    tip: 'Max number of local tasks that may be executed in parallel'
+                })
             .pc-form-grid-col-30
-                +number('Max task queue size:', model + '.maxTaskQueueSize', '"maxTaskQueueSize"', 'true', '8192', '1', 'Max task queue size')
+                +form-field__number({
+                    label: 'Max task queue size:',
+                    model: `${model}.maxTaskQueueSize`,
+                    name: '"maxTaskQueueSize"',
+                    placeholder: '8192',
+                    min: '1',
+                    tip: 'Max task queue size'
+                })
             .pc-form-grid-col-60
                 .ignite-form-field
                     +list-text-field({
@@ -81,7 +140,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     })(
                         list-editable-cols=`::[{name: 'Native libraries:'}]`
                     )
-                        +unique-feedback(_, `${uniqueTip}`)
+                        +form-field__error({ error: 'igniteUnique', message: 'Such native library already exists!' })
 
         .pca-form-column-6
             +preview-xml-java(model, 'clusterHadoop')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/load-balancing.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/load-balancing.pug
index 20ea1f0ed85fe..e96b01697f59f 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/load-balancing.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/load-balancing.pug
@@ -25,92 +25,156 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Load balancing configuration
     panel-description
-        | Load balancing component balances job distribution among cluster nodes. 
+        | Load balancing component balances job distribution among cluster nodes.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/load-balancing" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6
             mixin clusters-load-balancing-spi
                 .ignite-form-field(ng-init='loadBalancingSpiTbl={type: "loadBalancingSpi", model: "loadBalancingSpi", focusId: "kind", ui: "load-balancing-table"}')
-                    +ignite-form-field__label('Load balancing configurations:', '"loadBalancingConfigurations"')
-                        +tooltip(`Load balancing component balances job distribution among cluster nodes`)
-                    .ignite-form-field__control
-                        -let items = loadBalancingSpi
+                    +form-field__label({ label: 'Load balancing configurations:', name: '"loadBalancingConfigurations"' })
+                        +form-field__tooltip(`Load balancing component balances job distribution among cluster nodes`)
 
-                        list-editable.pc-list-editable-with-legacy-settings-rows(
-                            ng-model=items
-                            name='loadBalancingConfigurations'
-                        )
-                            list-editable-item-edit
-                                - form = '$parent.form'
-                                .settings-row
-                                    +sane-ignite-form-field-dropdown({
-                                        label: 'Load balancing:',
-                                        model: '$item.kind',
-                                        name: '"loadBalancingKind"',
-                                        required: true,
-                                        options: '::$ctrl.Clusters.loadBalancingKinds',
-                                        tip: `Provides the next best balanced node for job execution
-                                        <ul>
-                                            <li>Round-robin - Iterates through nodes in round-robin fashion and pick the next sequential node</li>
-                                            <li>Adaptive - Adapts to overall node performance</li>
-                                            <li>Random - Picks a random node for job execution</li>
-                                            <li>Custom - Custom load balancing implementation</li>
-                                        </ul>`
-                                    })(
-                                        ignite-unique=`${loadBalancingSpi}`
-                                        ignite-unique-property='kind'
-                                    )
-                                        +unique-feedback('"loadBalancingKind"', 'Load balancing SPI of that type is already configured')
-                                .settings-row(ng-show='$item.kind === "RoundRobin"')
-                                    +checkbox('Per task', '$item.RoundRobin.perTask', '"loadBalancingRRPerTask"', 'A new round robin order should be created for every task flag')
-                                .settings-row(ng-show='$item.kind === "Adaptive"')
-                                    +dropdown('Load probe:', '$item.Adaptive.loadProbe.kind', '"loadBalancingAdaptiveLoadProbeKind"', 'true', 'Default', '[\
-                                            {value: "Job", label: "Job count"},\
-                                            {value: "CPU", label: "CPU load"},\
-                                            {value: "ProcessingTime", label: "Processing time"},\
-                                            {value: "Custom", label: "Custom"},\
-                                            {value: null, label: "Default"}\
-                                        ]', 'Implementation of node load probing\
+                    -let items = loadBalancingSpi
+                    list-editable.pc-list-editable-with-legacy-settings-rows(
+                        ng-model=items
+                        name='loadBalancingConfigurations'
+                    )
+                        list-editable-item-edit
+                            - form = '$parent.form'
+                            .settings-row
+                                +form-field__dropdown({
+                                    label: 'Load balancing:',
+                                    model: '$item.kind',
+                                    name: '"loadBalancingKind"',
+                                    required: true,
+                                    options: '::$ctrl.Clusters.loadBalancingKinds',
+                                    tip: `Provides the next best balanced node for job execution
+                                    <ul>
+                                        <li>Round-robin - Iterates through nodes in round-robin fashion and pick the next sequential node</li>
+                                        <li>Adaptive - Adapts to overall node performance</li>
+                                        <li>Random - Picks a random node for job execution</li>
+                                        <li>Custom - Custom load balancing implementation</li>
+                                    </ul>`
+                                })(
+                                    ignite-unique=`${loadBalancingSpi}`
+                                    ignite-unique-property='kind'
+                                )
+                                    +form-field__error({ error: 'igniteUnique', message: 'Load balancing SPI of that type is already configured' })
+                            .settings-row(ng-show='$item.kind === "RoundRobin"')
+                                +form-field__checkbox({
+                                    label: 'Per task',
+                                    model: '$item.RoundRobin.perTask',
+                                    name: '"loadBalancingRRPerTask"',
+                                    tip: 'A new round robin order should be created for every task flag'
+                                })
+                            .settings-row(ng-show='$item.kind === "Adaptive"')
+                                +form-field__dropdown({
+                                    label: 'Load probe:',
+                                    model: '$item.Adaptive.loadProbe.kind',
+                                    name: '"loadBalancingAdaptiveLoadProbeKind"',
+                                    placeholder: 'Default',
+                                    options: '[\
+                                        {value: "Job", label: "Job count"},\
+                                        {value: "CPU", label: "CPU load"},\
+                                        {value: "ProcessingTime", label: "Processing time"},\
+                                        {value: "Custom", label: "Custom"},\
+                                        {value: null, label: "Default"}\
+                                    ]',
+                                    tip: 'Implementation of node load probing\
                                         <ul>\
                                             <li>Job count - Based on active and waiting job count</li>\
                                             <li>CPU load - Based on CPU load</li>\
                                             <li>Processing time - Based on total job processing time</li>\
                                             <li>Custom - Custom load probing implementation</li>\
                                             <li>Default - Default load probing implementation</li>\
-                                        </ul>')
-                                .settings-row(ng-show='$item.kind === "Adaptive" && $item.Adaptive.loadProbe.kind')
-                                    .panel-details(ng-show='$item.Adaptive.loadProbe.kind === "Job"')
-                                        .details-row
-                                            +checkbox('Use average', '$item.Adaptive.loadProbe.Job.useAverage', '"loadBalancingAdaptiveJobUseAverage"', 'Use average CPU load vs. current')
-                                    .panel-details(ng-show='$item.Adaptive.loadProbe.kind === "CPU"')
-                                        .details-row
-                                            +checkbox('Use average', '$item.Adaptive.loadProbe.CPU.useAverage', '"loadBalancingAdaptiveCPUUseAverage"', 'Use average CPU load vs. current')
-                                        .details-row
-                                            +checkbox('Use processors', '$item.Adaptive.loadProbe.CPU.useProcessors', '"loadBalancingAdaptiveCPUUseProcessors"', "divide each node's CPU load by the number of processors on that node")
-                                        .details-row
-                                            +number-min-max-step('Processor coefficient:', '$item.Adaptive.loadProbe.CPU.processorCoefficient',
-                                                '"loadBalancingAdaptiveCPUProcessorCoefficient"', 'true', '1', '0.001', '1', '0.001', 'Coefficient of every CPU')
-                                    .panel-details(ng-show='$item.Adaptive.loadProbe.kind === "ProcessingTime"')
-                                        .details-row
-                                            +checkbox('Use average', '$item.Adaptive.loadProbe.ProcessingTime.useAverage', '"loadBalancingAdaptiveJobUseAverage"', 'Use average execution time vs. current')
-                                    .panel-details(ng-show=loadProbeCustom)
-                                        .details-row
-                                            +java-class('Load brobe implementation:', '$item.Adaptive.loadProbe.Custom.className', '"loadBalancingAdaptiveJobUseClass"', 'true', loadProbeCustom,
-                                                'Custom load balancing SPI implementation class name.', loadProbeCustom)
-                                .settings-row(ng-show='$item.kind === "WeightedRandom"')
-                                    +number('Node weight:', '$item.WeightedRandom.nodeWeight', '"loadBalancingWRNodeWeight"', 'true', 10, '1', 'Weight of node')
-                                .settings-row(ng-show='$item.kind === "WeightedRandom"')
-                                    +checkbox('Use weights', '$item.WeightedRandom.useWeights', '"loadBalancingWRUseWeights"', 'Node weights should be checked when doing random load balancing')
-                                .settings-row(ng-show=loadBalancingCustom)
-                                    +java-class('Load balancing SPI implementation:', '$item.Custom.className', '"loadBalancingClass"', 'true', loadBalancingCustom,
-                                        'Custom load balancing SPI implementation class name.', loadBalancingCustom)
+                                        </ul>'
+                                })
+                            .settings-row(ng-show='$item.kind === "Adaptive" && $item.Adaptive.loadProbe.kind')
+                                .panel-details(ng-show='$item.Adaptive.loadProbe.kind === "Job"')
+                                    .details-row
+                                        +form-field__checkbox({
+                                            label: 'Use average',
+                                            model: '$item.Adaptive.loadProbe.Job.useAverage',
+                                            name: '"loadBalancingAdaptiveJobUseAverage"',
+                                            tip: 'Use average CPU load vs. current'
+                                        })
+                                .panel-details(ng-show='$item.Adaptive.loadProbe.kind === "CPU"')
+                                    .details-row
+                                        +form-field__checkbox({
+                                            label: 'Use average',
+                                            model: '$item.Adaptive.loadProbe.CPU.useAverage',
+                                            name: '"loadBalancingAdaptiveCPUUseAverage"',
+                                            tip: 'Use average CPU load vs. current'
+                                        })
+                                    .details-row
+                                        +form-field__checkbox({
+                                            label: 'Use processors',
+                                            model: '$item.Adaptive.loadProbe.CPU.useProcessors',
+                                            name: '"loadBalancingAdaptiveCPUUseProcessors"',
+                                            tip: 'Divide each node\'s CPU load by the number of processors on that node'
+                                        })
+                                    .details-row
+                                        +form-field__number({
+                                            label: 'Processor coefficient:',
+                                            model: '$item.Adaptive.loadProbe.CPU.processorCoefficient',
+                                            name: '"loadBalancingAdaptiveCPUProcessorCoefficient"',
+                                            placeholder: '1',
+                                            min: '0.001',
+                                            max: '1',
+                                            step: '0.05',
+                                            tip: 'Coefficient of every CPU'
+                                        })
+                                .panel-details(ng-show='$item.Adaptive.loadProbe.kind === "ProcessingTime"')
+                                    .details-row
+                                        +form-field__checkbox({
+                                            label: 'Use average',
+                                            model: '$item.Adaptive.loadProbe.ProcessingTime.useAverage',
+                                            name: '"loadBalancingAdaptiveJobUseAverage"',
+                                            tip: 'Use average execution time vs. current'
+                                        })
+                                .panel-details(ng-show=loadProbeCustom)
+                                    .details-row
+                                        +form-field__java-class({
+                                            label: 'Load brobe implementation:',
+                                            model: '$item.Adaptive.loadProbe.Custom.className',
+                                            name: '"loadBalancingAdaptiveJobUseClass"',
+                                            required: loadProbeCustom,
+                                            tip: 'Custom load balancing SPI implementation class name.',
+                                            validationActive: loadProbeCustom
+                                        })
 
-                            list-editable-no-items
-                                list-editable-add-item-button(
-                                    add-item=`$ctrl.Clusters.addLoadBalancingSpi(${model})`
-                                    label-single='load balancing configuration'
-                                    label-multiple='load balancing configurations'
-                                )
+                            .settings-row(ng-show='$item.kind === "WeightedRandom"')
+                                +form-field__number({
+                                    label: 'Node weight:',
+                                    model: '$item.WeightedRandom.nodeWeight',
+                                    name: '"loadBalancingWRNodeWeight"',
+                                    placeholder: '10',
+                                    min: '1',
+                                    tip: 'Weight of node'
+                                })
+                            .settings-row(ng-show='$item.kind === "WeightedRandom"')
+                                +form-field__checkbox({
+                                    label: 'Use weights',
+                                    model: '$item.WeightedRandom.useWeights',
+                                    name: '"loadBalancingWRUseWeights"',
+                                    tip: 'Node weights should be checked when doing random load balancing'
+                                })
+                            .settings-row(ng-show=loadBalancingCustom)
+                                +form-field__java-class({
+                                    label: 'Load balancing SPI implementation:',
+                                    model: '$item.Custom.className',
+                                    name: '"loadBalancingClass"',
+                                    required: loadBalancingCustom,
+                                    tip: 'Custom load balancing SPI implementation class name.',
+                                    validationActive: loadBalancingCustom
+                                })
+
+                        list-editable-no-items
+                            list-editable-add-item-button(
+                                add-item=`$ctrl.Clusters.addLoadBalancingSpi(${model})`
+                                label-single='load balancing configuration'
+                                label-multiple='load balancing configurations'
+                            )
 
             +clusters-load-balancing-spi
 
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger.pug
index 7b4b9aaf7c965..c30448ce6a6be 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger.pug
@@ -26,8 +26,12 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +dropdown('Logger:', kind, '"logger"', 'true', 'Default',
-                    '[\
+                +form-field__dropdown({
+                    label: 'Logger:',
+                    model: kind,
+                    name: '"logger"',
+                    placeholder: 'Default',
+                    options: '[\
                         {value: "Log4j", label: "Apache Log4j"},\
                         {value: "Log4j2", label: "Apache Log4j 2"},\
                         {value: "SLF4J", label: "Simple Logging Facade (SLF4J)"},\
@@ -37,17 +41,18 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                         {value: "Custom", label: "Custom"},\
                         {value: null, label: "Default"}\
                     ]',
-                    'Logger implementations\
-                    <ul>\
-                        <li>Apache Log4j - log4j-based logger</li>\
-                        <li>Apache Log4j 2 - Log4j2-based logger</li>\
-                        <li>Simple Logging Facade (SLF4J) - SLF4j-based logger</li>\
-                        <li>Java logger (JUL) - built in java logger</li>\
-                        <li>Jakarta Commons Logging (JCL) - wraps any JCL (Jakarta Commons Logging) loggers</li>\
-                        <li>Null logger - logger which does not output anything</li>\
-                        <li>Custom - custom logger implementation</li>\
-                        <li>Default - Apache Log4j if awailable on classpath or Java logger otherwise</li>\
-                    </ul>')
+                    tip: 'Logger implementations\
+                       <ul>\
+                           <li>Apache Log4j - log4j-based logger</li>\
+                           <li>Apache Log4j 2 - Log4j2-based logger</li>\
+                           <li>Simple Logging Facade (SLF4J) - SLF4j-based logger</li>\
+                           <li>Java logger (JUL) - built in java logger</li>\
+                           <li>Jakarta Commons Logging (JCL) - wraps any JCL (Jakarta Commons Logging) loggers</li>\
+                           <li>Null logger - logger which does not output anything</li>\
+                           <li>Custom - custom logger implementation</li>\
+                           <li>Default - Apache Log4j if awailable on classpath or Java logger otherwise</li>\
+                       </ul>'
+                })
             .pc-form-group(ng-show=`${kind} && (${kind} === 'Log4j2' || ${kind} === 'Log4j' || ${kind} === 'Custom')`)
                 .pc-form-grid-row(ng-show=`${kind} === 'Log4j2'`)
                     include ./logger/log4j2
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/custom.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/custom.pug
index a7177545fb40c..589a9c038f050 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/custom.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/custom.pug
@@ -21,4 +21,11 @@ include /app/helpers/jade/mixins
 -var required = '$ctrl.clonedCluster.logger.kind === "Custom"'
 
 .pc-form-grid-col-60
-    +java-class('Class:', `${model}.class`, '"customLogger"', 'true', required, 'Logger implementation class name', required)
+    +form-field__java-class({
+        label: 'Class:',
+        model: `${model}.class`,
+        name: '"customLogger"',
+        required: required,
+        tip: 'Logger implementation class name',
+        validationActive: required
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j.pug
index a1cab60eb06f2..1f216b7771f16 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j.pug
@@ -21,29 +21,48 @@ include /app/helpers/jade/mixins
 -var pathRequired = model + '.mode === "Path" && $ctrl.clonedCluster.logger.kind === "Log4j"'
 
 .pc-form-grid-col-30
-    +dropdown('Level:', `${model}.level`, '"log4jLevel"', 'true', 'Default',
-        '[\
-            {value: "OFF", label: "OFF"},\
-            {value: "FATAL", label: "FATAL"},\
-            {value: "ERROR", label: "ERROR"},\
-            {value: "WARN", label: "WARN"},\
-            {value: "INFO", label: "INFO"},\
-            {value: "DEBUG", label: "DEBUG"},\
-            {value: "TRACE", label: "TRACE"},\
-            {value: "ALL", label: "ALL"},\
-            {value: null, label: "Default"}\
-        ]',
-        'Level for internal log4j implementation')
+    +form-field__dropdown({
+        label: 'Level:',
+        model: `${model}.level`,
+        name: '"log4jLevel"',
+        placeholder: 'Default',
+        options: '[\
+                    {value: "OFF", label: "OFF"},\
+                    {value: "FATAL", label: "FATAL"},\
+                    {value: "ERROR", label: "ERROR"},\
+                    {value: "WARN", label: "WARN"},\
+                    {value: "INFO", label: "INFO"},\
+                    {value: "DEBUG", label: "DEBUG"},\
+                    {value: "TRACE", label: "TRACE"},\
+                    {value: "ALL", label: "ALL"},\
+                    {value: null, label: "Default"}\
+                ]',
+        tip: 'Level for internal log4j implementation'
+    })
+
 .pc-form-grid-col-30
-    +dropdown-required('Logger configuration:', `${model}.mode`, '"log4jMode"', 'true', 'true', 'Choose logger mode',
-        '[\
-            {value: "Default", label: "Default"},\
-            {value: "Path", label: "Path"}\
-        ]',
-        'Choose logger configuration\
-        <ul>\
-            <li>Default - default logger</li>\
-            <li>Path - path or URI to XML configuration</li>\
-        </ul>')
+    +form-field__dropdown({
+        label: 'Logger configuration:',
+        model: `${model}.mode`,
+        name: '"log4jMode"',
+        required: 'true',
+        placeholder: 'Choose logger mode',
+        options: '[\
+                    {value: "Default", label: "Default"},\
+                    {value: "Path", label: "Path"}\
+                ]',
+        tip: 'Choose logger configuration\
+                <ul>\
+                    <li>Default - default logger</li>\
+                    <li>Path - path or URI to XML configuration</li>\
+                </ul>'
+    })
 .pc-form-grid-col-60(ng-show=pathRequired)
-    +text('Path:', `${model}.path`, '"log4jPath"', pathRequired, 'Input path', 'Path or URI to XML configuration')
+    +form-field__text({
+        label: 'Path:',
+        model: `${model}.path`,
+        name: '"log4jPath"',
+        required: pathRequired,
+        placeholder: 'Input path',
+        tip: 'Path or URI to XML configuration'
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j2.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j2.pug
index fc94e06d672ed..c5b785b47a3ad 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j2.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/logger/log4j2.pug
@@ -21,18 +21,30 @@ include /app/helpers/jade/mixins
 -var log4j2Required = '$ctrl.clonedCluster.logger.kind === "Log4j2"'
 
 .pc-form-grid-col-60
-    +dropdown('Level:', `${model}.level`, '"log4j2Level"', 'true', 'Default',
-        '[\
-            {value: "OFF", label: "OFF"},\
-            {value: "FATAL", label: "FATAL"},\
-            {value: "ERROR", label: "ERROR"},\
-            {value: "WARN", label: "WARN"},\
-            {value: "INFO", label: "INFO"},\
-            {value: "DEBUG", label: "DEBUG"},\
-            {value: "TRACE", label: "TRACE"},\
-            {value: "ALL", label: "ALL"},\
-            {value: null, label: "Default"}\
-        ]',
-        'Level for internal log4j2 implementation')
+    +form-field__dropdown({
+        label: 'Level:',
+        model: `${model}.level`,
+        name: '"log4j2Level"',
+        placeholder: 'Default',
+        options: '[\
+                    {value: "OFF", label: "OFF"},\
+                    {value: "FATAL", label: "FATAL"},\
+                    {value: "ERROR", label: "ERROR"},\
+                    {value: "WARN", label: "WARN"},\
+                    {value: "INFO", label: "INFO"},\
+                    {value: "DEBUG", label: "DEBUG"},\
+                    {value: "TRACE", label: "TRACE"},\
+                    {value: "ALL", label: "ALL"},\
+                    {value: null, label: "Default"}\
+                ]',
+        tip: 'Level for internal log4j2 implementation'
+    })
 .pc-form-grid-col-60
-    +text('Path:', `${model}.path`, '"log4j2Path"', log4j2Required, 'Input path', 'Path or URI to XML configuration')
+    +form-field__text({
+        label: 'Path:',
+        model: `${model}.path`,
+        name: '"log4j2Path"',
+        required: log4j2Required,
+        placeholder: 'Input path',
+        tip: 'Path or URI to XML configuration'
+    })
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/marshaller.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/marshaller.pug
index baa4956431264..23393b8e0d469 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/marshaller.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/marshaller.pug
@@ -24,52 +24,91 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Marshaller
     panel-description
-        | Marshaller allows to marshal or unmarshal objects in grid. 
-        | It provides serialization/deserialization mechanism for all instances that are sent across networks or are otherwise serialized. 
-        | By default BinaryMarshaller will be used. 
+        | Marshaller allows to marshal or unmarshal objects in grid.
+        | It provides serialization/deserialization mechanism for all instances that are sent across networks or are otherwise serialized.
+        | By default BinaryMarshaller will be used.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/binary-marshaller" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60(ng-if='$ctrl.available(["1.0.0", "2.0.0"])')
-                +dropdown('Marshaller:', marshaller + '.kind', '"kind"', 'true', 'Default', '$ctrl.marshallerVariant',
-                    'Instance of marshaller to use in grid<br/>\
-                    <ul>\
-                        <li>OptimizedMarshaller - Optimized implementation of marshaller</li>\
-                        <li>JdkMarshaller - Marshaller based on JDK serialization mechanism</li>\
-                        <li>Default - BinaryMarshaller serialize and deserialize all objects in the binary format</li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Marshaller:',
+                    model: marshaller + '.kind',
+                    name: '"kind"',
+                    placeholder: 'Default',
+                    options: '$ctrl.marshallerVariant',
+                    tip: 'Instance of marshaller to use in grid<br/>\
+                       <ul>\
+                           <li>OptimizedMarshaller - Optimized implementation of marshaller</li>\
+                           <li>JdkMarshaller - Marshaller based on JDK serialization mechanism</li>\
+                           <li>Default - BinaryMarshaller serialize and deserialize all objects in the binary format</li>\
+                       </ul>'
+                })
             .pc-form-grid-col-60(ng-if='$ctrl.available(["2.0.0", "2.1.0"])')
-                +dropdown('Marshaller:', marshaller + '.kind', '"kind"', 'true', 'Default', '$ctrl.marshallerVariant',
-                    'Instance of marshaller to use in grid<br/>\
-                    <ul>\
-                        <li>JdkMarshaller - Marshaller based on JDK serialization mechanism</li>\
-                        <li>Default - BinaryMarshaller serialize and deserialize all objects in the binary format</li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Marshaller:',
+                    model: marshaller + '.kind',
+                    name: '"kind"',
+                    placeholder: 'Default',
+                    options: '$ctrl.marshallerVariant',
+                    tip: 'Instance of marshaller to use in grid<br/>\
+                        <ul>\
+                            <li>JdkMarshaller - Marshaller based on JDK serialization mechanism</li>\
+                            <li>Default - BinaryMarshaller serialize and deserialize all objects in the binary format</li>\
+                        </ul>'
+                })
             .pc-form-group.pc-form-grid-row(
                 ng-show=`${marshaller}.kind === 'OptimizedMarshaller'`
                 ng-if='$ctrl.available(["1.0.0", "2.1.0"])'
             )
                 .pc-form-grid-col-60
-                    +number('Streams pool size:', `${optMarshaller}.poolSize`, '"poolSize"', 'true', '0', '0',
-                        'Specifies size of cached object streams used by marshaller<br/>\
-                        Object streams are cached for performance reason to avoid costly recreation for every serialization routine<br/>\
-                        If 0 (default), pool is not used and each thread has its own cached object stream which it keeps reusing<br/>\
-                        Since each stream has an internal buffer, creating a stream for each thread can lead to high memory consumption if many large messages are marshalled or unmarshalled concurrently<br/>\
-                        Consider using pool in this case. This will limit number of streams that can be created and, therefore, decrease memory consumption<br/>\
-                        NOTE: Using streams pool can decrease performance since streams will be shared between different threads which will lead to more frequent context switching')
+                    +form-field__number({
+                        label: 'Streams pool size:',
+                        model: `${optMarshaller}.poolSize`,
+                        name: '"poolSize"',
+                        placeholder: '0',
+                        min: '0',
+                        tip: 'Specifies size of cached object streams used by marshaller<br/>\
+                             Object streams are cached for performance reason to avoid costly recreation for every serialization routine<br/>\
+                             If 0 (default), pool is not used and each thread has its own cached object stream which it keeps reusing<br/>\
+                             Since each stream has an internal buffer, creating a stream for each thread can lead to high memory consumption if many large messages are marshalled or unmarshalled concurrently<br/>\
+                             Consider using pool in this case. This will limit number of streams that can be created and, therefore, decrease memory consumption<br/>\
+                             NOTE: Using streams pool can decrease performance since streams will be shared between different threads which will lead to more frequent context switching'
+                    })
                 .pc-form-grid-col-60
-                    +checkbox('Require serializable', `${optMarshaller}.requireSerializable`, '"requireSerializable"',
-                        'Whether marshaller should require Serializable interface or not')
+                    +form-field__checkbox({
+                        label: 'Require serializable',
+                        model: `${optMarshaller}.requireSerializable`,
+                        name: '"requireSerializable"',
+                        tip: 'Whether marshaller should require Serializable interface or not'
+                    })
             .pc-form-grid-col-60
-                +checkbox('Marshal local jobs', `${model}.marshalLocalJobs`, '"marshalLocalJobs"', 'If this flag is enabled, jobs mapped to local node will be marshalled as if it was remote node')
+                +form-field__checkbox({
+                    label: 'Marshal local jobs',
+                    model: `${model}.marshalLocalJobs`,
+                    name: '"marshalLocalJobs"',
+                    tip: 'If this flag is enabled, jobs mapped to local node will be marshalled as if it was remote node'
+                })
 
             //- Removed in ignite 2.0
             .pc-form-grid-col-30(ng-if-start='$ctrl.available(["1.0.0", "2.0.0"])')
-                +number('Keep alive time:', `${model}.marshallerCacheKeepAliveTime`, '"marshallerCacheKeepAliveTime"', 'true', '10000', '0',
-                    'Keep alive time of thread pool that is in charge of processing marshaller messages')
+                +form-field__number({
+                    label: 'Keep alive time:',
+                    model: `${model}.marshallerCacheKeepAliveTime`,
+                    name: '"marshallerCacheKeepAliveTime"',
+                    placeholder: '10000',
+                    min: '0',
+                    tip: 'Keep alive time of thread pool that is in charge of processing marshaller messages'
+                })
             .pc-form-grid-col-30(ng-if-end)
-                +number('Pool size:', `${model}.marshallerCacheThreadPoolSize`, '"marshallerCacheThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
-                    'Default size of thread pool that is in charge of processing marshaller messages')
+                +form-field__number({
+                    label: 'Pool size:',
+                    model: `${model}.marshallerCacheThreadPoolSize`,
+                    name: '"marshallerCacheThreadPoolSize"',
+                    placeholder: 'max(8, availableProcessors) * 2',
+                    min: '1',
+                    tip: 'Default size of thread pool that is in charge of processing marshaller messages'
+                })
 
         .pca-form-column-6
             +preview-xml-java(model, 'clusterMarshaller')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/memory.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/memory.pug
index 181ae4561b30e..7712cb763922d 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/memory.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/memory.pug
@@ -21,18 +21,18 @@ include /app/helpers/jade/mixins
 -var memoryPolicies = model + '.memoryPolicies'
 
 panel-collapsible(
-    ng-form=form
-    on-open=`ui.loadPanel('${form}')`
-    ng-show='$ctrl.available(["2.0.0", "2.3.0"])'
+ng-form=form
+on-open=`ui.loadPanel('${form}')`
+ng-show='$ctrl.available(["2.0.0", "2.3.0"])'
 )
     panel-title Memory configuration
     panel-description
-        | Page memory is a manageable off-heap based memory architecture that is split into pages of fixed size. 
+        | Page memory is a manageable off-heap based memory architecture that is split into pages of fixed size.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/durable-memory" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`$ctrl.available(["2.0.0", "2.3.0"]) && ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +sane-ignite-form-field-dropdown({
+                +form-field__dropdown({
                     label: 'Page size:',
                     model: `${model}.pageSize`,
                     name: '"MemoryConfigurationPageSize"',
@@ -40,13 +40,19 @@ panel-collapsible(
                     tip: 'Every memory region is split on pages of fixed size'
                 })
             .pc-form-grid-col-60
-                +number('Concurrency level:', model + '.concurrencyLevel', '"MemoryConfigurationConcurrencyLevel"',
-                'true', 'availableProcessors', '2', 'The number of concurrent segments in Ignite internal page mapping tables')
+                +form-field__number({
+                    label: 'Concurrency level:',
+                    model: `${model}.concurrencyLevel`,
+                    name: '"MemoryConfigurationConcurrencyLevel"',
+                    placeholder: 'availableProcessors',
+                    min: '2',
+                    tip: 'The number of concurrent segments in Ignite internal page mapping tables'
+                })
             .pc-form-grid-col-60.pc-form-group__text-title
                 span System cache
             .pc-form-group.pc-form-grid-row
                 .pc-form-grid-col-30
-                    pc-form-field-size(
+                    form-field-size(
                         label='Initial size:'
                         ng-model=`${model}.systemCacheInitialSize`
                         name='systemCacheInitialSize'
@@ -56,7 +62,7 @@ panel-collapsible(
                         on-scale-change='systemCacheInitialSizeScale = $event'
                     )
                 .pc-form-grid-col-30
-                    pc-form-field-size(
+                    form-field-size(
                         label='Max size:'
                         ng-model=`${model}.systemCacheMaxSize`
                         name='systemCacheMaxSize'
@@ -69,7 +75,7 @@ panel-collapsible(
                 span Memory policies
             .pc-form-group.pc-form-grid-row
                 .pc-form-grid-col-60
-                    +sane-ignite-form-field-text({
+                    +form-field__text({
                         label: 'Default memory policy name:',
                         model: `${model}.defaultMemoryPolicyName`,
                         name: '"defaultMemoryPolicyName"',
@@ -78,116 +84,154 @@ panel-collapsible(
                     })(
                         pc-not-in-collection='::$ctrl.Clusters.memoryPolicy.name.invalidValues'
                         ui-validate=`{
-                            defaultMemoryPolicyExists: '$ctrl.Clusters.memoryPolicy.customValidators.defaultMemoryPolicyExists($value, ${memoryPolicies})'
-                        }`
+                                defaultMemoryPolicyExists: '$ctrl.Clusters.memoryPolicy.customValidators.defaultMemoryPolicyExists($value, ${memoryPolicies})'
+                            }`
                         ui-validate-watch=`"${memoryPolicies}"`
                         ui-validate-watch-object-equality='true'
                         ng-model-options='{allowInvalid: true}'
                     )
-                        +form-field-feedback('"MemoryPolicyName"', 'notInCollection', '{{::$ctrl.Clusters.memoryPolicy.name.invalidValues[0]}} is reserved for internal use')
-                        +form-field-feedback('"MemoryPolicyName"', 'defaultMemoryPolicyExists', 'Memory policy with that name should be configured')
+                        +form-field__error({ error: 'notInCollection', message: '{{::$ctrl.Clusters.memoryPolicy.name.invalidValues[0]}} is reserved for internal use' })
+                        +form-field__error({ error: 'defaultMemoryPolicyExists', message: 'Memory policy with that name should be configured' })
                 .pc-form-grid-col-60(ng-hide='(' + model + '.defaultMemoryPolicyName || "default") !== "default"')
-                    +number('Default memory policy size:', model + '.defaultMemoryPolicySize', '"defaultMemoryPolicySize"',
-                    'true', '0.8 * totalMemoryAvailable', '10485760',
-                    'Specify desired size of default memory policy without having to use more verbose syntax of MemoryPolicyConfiguration elements')
+                    +form-field__number({
+                        label: 'Default memory policy size:',
+                        model: `${model}.defaultMemoryPolicySize`,
+                        name: '"defaultMemoryPolicySize"',
+                        placeholder: '0.8 * totalMemoryAvailable',
+                        min: '10485760',
+                        tip: 'Specify desired size of default memory policy without having to use more verbose syntax of MemoryPolicyConfiguration elements'
+                    })
                 .pc-form-grid-col-60
                     mixin clusters-memory-policies
                         .ignite-form-field(ng-init='memoryPoliciesTbl={type: "memoryPolicies", model: "memoryPolicies", focusId: "name", ui: "memory-policies-table"}')
-                            +ignite-form-field__label('Configured policies:', '"configuredPolicies"')
-                                +tooltip(`List of configured policies`)
-                            .ignite-form-field__control
-                                -let items = memoryPolicies
+                            +form-field__label({ label: 'Configured policies:', name: '"configuredPolicies"' })
+                                +form-field__tooltip({ title: `List of configured policies` })
 
-                                list-editable.pc-list-editable-with-form-grid(ng-model=items name='memoryPolicies')
-                                    list-editable-item-edit.pc-form-grid-row
-                                        - form = '$parent.form'
-                                        .pc-form-grid-col-60
-                                            +sane-ignite-form-field-text({
-                                                label: 'Name:',
-                                                model: '$item.name',
-                                                name: '"MemoryPolicyName"',
-                                                placeholder: '{{ ::$ctrl.Clusters.memoryPolicy.name.default }}',
-                                                tip: 'Memory policy name'
-                                            })(
-                                                ui-validate=`{
+                            -let items = memoryPolicies
+                            list-editable.pc-list-editable-with-form-grid(ng-model=items name='memoryPolicies')
+                                list-editable-item-edit.pc-form-grid-row
+                                    - form = '$parent.form'
+                                    .pc-form-grid-col-60
+                                        +form-field__text({
+                                            label: 'Name:',
+                                            model: '$item.name',
+                                            name: '"MemoryPolicyName"',
+                                            placeholder: '{{ ::$ctrl.Clusters.memoryPolicy.name.default }}',
+                                            tip: 'Memory policy name'
+                                        })(
+                                            ui-validate=`{
                                                     uniqueMemoryPolicyName: '$ctrl.Clusters.memoryPolicy.customValidators.uniqueMemoryPolicyName($item, ${items})'
                                                 }`
-                                                ui-validate-watch=`"${items}"`
-                                                ui-validate-watch-object-equality='true'
-                                                pc-not-in-collection='::$ctrl.Clusters.memoryPolicy.name.invalidValues'
-                                                ng-model-options='{allowInvalid: true}'
-                                            )
-                                                +form-field-feedback('"MemoryPolicyName', 'uniqueMemoryPolicyName', 'Memory policy with that name is already configured')
-                                                +form-field-feedback('"MemoryPolicyName', 'notInCollection', '{{::$ctrl.Clusters.memoryPolicy.name.invalidValues[0]}} is reserved for internal use')
-                                        .pc-form-grid-col-60
-                                            pc-form-field-size(
-                                                label='Initial size:'
-                                                ng-model='$item.initialSize'
-                                                ng-model-options='{allowInvalid: true}'
-                                                name='MemoryPolicyInitialSize'
-                                                placeholder='{{ $ctrl.Clusters.memoryPolicy.initialSize.default / scale.value }}'
-                                                min='{{ ::$ctrl.Clusters.memoryPolicy.initialSize.min }}'
-                                                tip='Initial memory region size defined by this memory policy'
-                                                on-scale-change='scale = $event'
-                                            )
-                                        .pc-form-grid-col-60
-                                            pc-form-field-size(
-                                                ng-model='$item.maxSize'
-                                                ng-model-options='{allowInvalid: true}'
-                                                name='MemoryPolicyMaxSize'
-                                                label='Maximum size:'
-                                                placeholder='{{ ::$ctrl.Clusters.memoryPolicy.maxSize.default }}'
-                                                min='{{ $ctrl.Clusters.memoryPolicy.maxSize.min($item) }}'
-                                                tip='Maximum memory region size defined by this memory policy'
-                                            )
-                                        .pc-form-grid-col-60
-                                            +text('Swap file path:', '$item.swapFilePath', '"MemoryPolicySwapFilePath"', 'false',
-                                            'Input swap file path', 'An optional path to a memory mapped file for this memory policy')
-                                        .pc-form-grid-col-60
-                                            +dropdown('Eviction mode:', '$item.pageEvictionMode', '"MemoryPolicyPageEvictionMode"', 'true', 'DISABLED',
-                                            '[\
+                                            ui-validate-watch=`"${items}"`
+                                            ui-validate-watch-object-equality='true'
+                                            pc-not-in-collection='::$ctrl.Clusters.memoryPolicy.name.invalidValues'
+                                            ng-model-options='{allowInvalid: true}'
+                                        )
+                                            +form-field__error({ error: 'uniqueMemoryPolicyName', message: 'Memory policy with that name is already configured' })
+                                            +form-field__error({ error: 'notInCollection', message: '{{::$ctrl.Clusters.memoryPolicy.name.invalidValues[0]}} is reserved for internal use' })
+                                    .pc-form-grid-col-60
+                                        form-field-size(
+                                            label='Initial size:'
+                                            ng-model='$item.initialSize'
+                                            ng-model-options='{allowInvalid: true}'
+                                            name='MemoryPolicyInitialSize'
+                                            placeholder='{{ $ctrl.Clusters.memoryPolicy.initialSize.default / scale.value }}'
+                                            min='{{ ::$ctrl.Clusters.memoryPolicy.initialSize.min }}'
+                                            tip='Initial memory region size defined by this memory policy'
+                                            on-scale-change='scale = $event'
+                                        )
+                                    .pc-form-grid-col-60
+                                        form-field-size(
+                                            ng-model='$item.maxSize'
+                                            ng-model-options='{allowInvalid: true}'
+                                            name='MemoryPolicyMaxSize'
+                                            label='Maximum size:'
+                                            placeholder='{{ ::$ctrl.Clusters.memoryPolicy.maxSize.default }}'
+                                            min='{{ $ctrl.Clusters.memoryPolicy.maxSize.min($item) }}'
+                                            tip='Maximum memory region size defined by this memory policy'
+                                        )
+                                    .pc-form-grid-col-60
+                                        +form-field__text({
+                                            label: 'Swap file path:',
+                                            model: '$item.swapFilePath',
+                                            name: '"MemoryPolicySwapFilePath"',
+                                            placeholder: 'Input swap file path',
+                                            tip: 'An optional path to a memory mapped file for this memory policy'
+                                        })
+                                    .pc-form-grid-col-60
+                                        +form-field__dropdown({
+                                            label: 'Eviction mode:',
+                                            model: '$item.pageEvictionMode',
+                                            name: '"MemoryPolicyPageEvictionMode"',
+                                            placeholder: 'DISABLED',
+                                            options: '[\
                                                 {value: "DISABLED", label: "DISABLED"},\
                                                 {value: "RANDOM_LRU", label: "RANDOM_LRU"},\
-                                                {value: "RANDOM_2_LRU", label: "RANDOM_2_LRU"}\
+                                                [value: "RANDOM_2_LRU", label: "RANDOM_2_LRU"}\
                                             ]',
-                                            'An algorithm for memory pages eviction\
-                                            <ul>\
-                                                <li>DISABLED - Eviction is disabled</li>\
-                                                <li>RANDOM_LRU - Once a memory region defined by a memory policy is configured, an off - heap array is allocated to track last usage timestamp for every individual data page</li>\
-                                                <li>RANDOM_2_LRU - Differs from Random - LRU only in a way that two latest access timestamps are stored for every data page</li>\
-                                            </ul>')
-                                        .pc-form-grid-col-30
-                                            +number-min-max-step('Eviction threshold:', '$item.evictionThreshold', '"MemoryPolicyEvictionThreshold"',
-                                            'true', '0.9', '0.5', '0.999', '0.001', 'A threshold for memory pages eviction initiation')
-                                        .pc-form-grid-col-30
-                                            +sane-ignite-form-field-number({
-                                                label: 'Empty pages pool size:',
-                                                model: '$item.emptyPagesPoolSize',
-                                                name: '"MemoryPolicyEmptyPagesPoolSize"',
-                                                placeholder: '{{ ::$ctrl.Clusters.memoryPolicy.emptyPagesPoolSize.default }}',
-                                                min: '{{ ::$ctrl.Clusters.memoryPolicy.emptyPagesPoolSize.min }}',
-                                                max: '{{ $ctrl.Clusters.memoryPolicy.emptyPagesPoolSize.max($ctrl.clonedCluster, $item) }}',
-                                                tip: 'The minimal number of empty pages to be present in reuse lists for this memory policy'
-                                            })
+                                            tip: 'An algorithm for memory pages eviction\
+                                                 <ul>\
+                                                    <li>DISABLED - Eviction is disabled</li>\
+                                                    <li>RANDOM_LRU - Once a memory region defined by a memory policy is configured, an off - heap array is allocated to track last usage timestamp for every individual data page</li>\
+                                                    <li>RANDOM_2_LRU - Differs from Random - LRU only in a way that two latest access timestamps are stored for every data page</li>\
+                                                 </ul>'
+                                        })
+                                    .pc-form-grid-col-30
+                                        +form-field__number({
+                                            label: 'Eviction threshold:',
+                                            model: '$item.evictionThreshold',
+                                            name: '"MemoryPolicyEvictionThreshold"',
+                                            placeholder: '0.9',
+                                            min: '0.5',
+                                            max: '0.999',
+                                            step: '0.05',
+                                            tip: 'A threshold for memory pages eviction initiation'
+                                        })
+                                    .pc-form-grid-col-30
+                                        +form-field__number({
+                                            label: 'Empty pages pool size:',
+                                            model: '$item.emptyPagesPoolSize',
+                                            name: '"MemoryPolicyEmptyPagesPoolSize"',
+                                            placeholder: '{{ ::$ctrl.Clusters.memoryPolicy.emptyPagesPoolSize.default }}',
+                                            min: '{{ ::$ctrl.Clusters.memoryPolicy.emptyPagesPoolSize.min }}',
+                                            max: '{{ $ctrl.Clusters.memoryPolicy.emptyPagesPoolSize.max($ctrl.clonedCluster, $item) }}',
+                                            tip: 'The minimal number of empty pages to be present in reuse lists for this memory policy'
+                                        })
 
-                                        //- Since ignite 2.1
-                                        .pc-form-grid-col-30(ng-if-start='$ctrl.available("2.1.0")')
-                                            +number('Sub intervals:', '$item.subIntervals', '"MemoryPolicySubIntervals"',
-                                                'true', '5', '1', 'A number of sub-intervals the whole rate time interval will be split into to calculate allocation and eviction rates')
-                                        .pc-form-grid-col-30(ng-if-end)
-                                            +number('Rate time interval:', '$item.rateTimeInterval', '"MemoryPolicyRateTimeInterval"',
-                                                'true', '60000', '1000', 'Time interval for allocation rate and eviction rate monitoring purposes')
-                                                
-                                        .pc-form-grid-col-60
-                                            +checkbox('Metrics enabled', '$item.metricsEnabled', '"MemoryPolicyMetricsEnabled"',
-                                            'Whether memory metrics are enabled by default on node startup')
+                                    //- Since ignite 2.1
+                                    .pc-form-grid-col-30(ng-if-start='$ctrl.available("2.1.0")')
+                                        +form-field__number({
+                                            label: 'Sub intervals:',
+                                            model: '$item.subIntervals',
+                                            name: '"MemoryPolicySubIntervals"',
+                                            placeholder: '5',
+                                            min: '1',
+                                            tip: 'A number of sub-intervals the whole rate time interval will be split into to calculate allocation and eviction rates'
+                                        })
+                                    .pc-form-grid-col-30(ng-if-end)
+                                        +form-field__number({
+                                            label: 'Rate time interval:',
+                                            model: '$item.rateTimeInterval',
+                                            name: '"MemoryPolicyRateTimeInterval"',
+                                            placeholder: '60000',
+                                            min: '1000',
+                                            tip: 'Time interval for allocation rate and eviction rate monitoring purposes'
+                                        })
 
-                                    list-editable-no-items
-                                        list-editable-add-item-button(
-                                            add-item=`$ctrl.Clusters.addMemoryPolicy($ctrl.clonedCluster)`
-                                            label-single='memory policy configuration'
-                                            label-multiple='memory policy configurations'
-                                        )
+                                    .pc-form-grid-col-60
+                                        +form-field__checkbox({
+                                            label: 'Metrics enabled',
+                                            model: '$item.metricsEnabled',
+                                            name: '"MemoryPolicyMetricsEnabled"',
+                                            tip: 'Whether memory metrics are enabled by default on node startup'
+                                        })
+
+                                list-editable-no-items
+                                    list-editable-add-item-button(
+                                        add-item=`$ctrl.Clusters.addMemoryPolicy($ctrl.clonedCluster)`
+                                        label-single='memory policy configuration'
+                                        label-multiple='memory policy configurations'
+                                    )
 
                     +clusters-memory-policies
 
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/metrics.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/metrics.pug
index c4c92608a239b..f99efbbd0989e 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/metrics.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/metrics.pug
@@ -25,22 +25,46 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-30
-                +number('Expire time:', `${model}.metricsExpireTime`, '"metricsExpireTime"', 'true', 'Long.MAX_VALUE', '1',
-                    'Time in milliseconds after which a certain metric value is considered expired')
+                +form-field__number({
+                    label: 'Expire time:',
+                    model: `${model}.metricsExpireTime`,
+                    name: '"metricsExpireTime"',
+                    placeholder: 'Long.MAX_VALUE',
+                    min: '0',
+                    tip: 'Time in milliseconds after which a certain metric value is considered expired'
+                })
             .pc-form-grid-col-30
-                +number('History size:', `${model}.metricsHistorySize`, '"metricsHistorySize"', 'true', '10000', '1',
-                    'Number of metrics kept in history to compute totals and averages')
+                +form-field__number({
+                    label: 'History size:',
+                    model: `${model}.metricsHistorySize`,
+                    name: '"metricsHistorySize"',
+                    placeholder: '10000',
+                    min: '1',
+                    tip: 'Number of metrics kept in history to compute totals and averages'
+                })
             .pc-form-grid-col-30
-                +number('Log frequency:', `${model}.metricsLogFrequency`, '"metricsLogFrequency"', 'true', '60000', '0',
-                    'Frequency of metrics log print out<br/>\ ' +
-                    'When <b>0</b> log print of metrics is disabled')
+                +form-field__number({
+                    label: 'Log frequency:',
+                    model: `${model}.metricsLogFrequency`,
+                    name: '"metricsLogFrequency"',
+                    placeholder: '60000',
+                    min: '0',
+                    tip: 'Frequency of metrics log print out<br/>\ ' +
+                    'When <b>0</b> log print of metrics is disabled'
+                })
             .pc-form-grid-col-30
-                +number('Update frequency:', `${model}.metricsUpdateFrequency`, '"metricsUpdateFrequency"', 'true', '2000', '-1',
-                    'Job metrics update frequency in milliseconds\
-                    <ul>\
-                        <li>If set to -1 job metrics are never updated</li>\
-                        <li>If set to 0 job metrics are updated on each job start and finish</li>\
-                        <li>Positive value defines the actual update frequency</li>\
-                    </ul>')
+                +form-field__number({
+                    label: 'Update frequency:',
+                    model: `${model}.metricsUpdateFrequency`,
+                    name: '"metricsUpdateFrequency"',
+                    placeholder: '2000',
+                    min: '-1',
+                    tip: 'Job metrics update frequency in milliseconds\
+                        <ul>\
+                            <li>If set to -1 job metrics are never updated</li>\
+                            <li>If set to 0 job metrics are updated on each job start and finish</li>\
+                            <li>Positive value defines the actual update frequency</li>\
+                        </ul>'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterMetrics')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/misc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/misc.pug
index cdc7258fa07bb..d0e5d9fe7ab24 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/misc.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/misc.pug
@@ -25,34 +25,69 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +text('Work directory:', model + '.workDirectory', '"workDirectory"', 'false', 'Input work directory',
-                    'Ignite work directory.<br/>\
-                    If not provided, the method will use work directory under IGNITE_HOME specified by IgniteConfiguration#setIgniteHome(String)\
-                    or IGNITE_HOME environment variable or system property.')
+                +form-field__text({
+                    label: 'Work directory:',
+                    model: `${model}.workDirectory`,
+                    name: '"workDirectory"',
+                    placeholder: 'Input work directory',
+                    tip: 'Ignite work directory.<br/>\
+                          If not provided, the method will use work directory under IGNITE_HOME specified by IgniteConfiguration#setIgniteHome(String)\
+                          or IGNITE_HOME environment variable or system property.'
+                })
 
             //- Since ignite 2.0
             .pc-form-grid-col-60(ng-if-start='$ctrl.available("2.0.0")')
-                +text('Consistent ID:', model + '.consistentId', '"ConsistentId"', 'false', 'Input consistent ID', 'Consistent globally unique node ID which survives node restarts')
+                +form-field__text({
+                    label: 'Consistent ID:',
+                    model: `${model}.consistentId`,
+                    name: '"ConsistentId"',
+                    placeholder: 'Input consistent ID',
+                    tip: 'Consistent globally unique node ID which survives node restarts'
+                })
             .pc-form-grid-col-60
-                +java-class('Warmup closure:', model + '.warmupClosure', '"warmupClosure"', 'true', 'false', 'This closure will be executed before actual grid instance start')
+                +form-field__java-class({
+                    label: 'Warmup closure:',
+                    model: `${model}.warmupClosure`,
+                    name: '"warmupClosure"',
+                    tip: 'This closure will be executed before actual grid instance start'
+                })
+
             .pc-form-grid-col-60
-                +checkbox('Active on start', model + '.activeOnStart', '"activeOnStart"',
-                    'If cluster is not active on start, there will be no cache partition map exchanges performed until the cluster is activated')
+                +form-field__checkbox({
+                    label: 'Active on start',
+                    model: model + '.activeOnStart',
+                    name: '"activeOnStart"',
+                    tip: 'If cluster is not active on start, there will be no cache partition map exchanges performed until the cluster is activated'
+                })
             .pc-form-grid-col-60(ng-if-end)
-                +checkbox('Cache sanity check enabled', model + '.cacheSanityCheckEnabled', '"cacheSanityCheckEnabled"',
-                    'If enabled, then Ignite will perform the following checks and throw an exception if check fails<br/>\
-                    <ul>\
-                    <li>Cache entry is not externally locked with lock or lockAsync methods when entry is enlisted to transaction</li>\
-                    <li>Each entry in affinity group - lock transaction has the same affinity key as was specified on affinity transaction start</li>\
-                    <li>Each entry in partition group - lock transaction belongs to the same partition as was specified on partition transaction start</li>\
-                    </ul>')
+                +form-field__checkbox({
+                    label: 'Cache sanity check enabled',
+                    model: model + '.cacheSanityCheckEnabled',
+                    name: '"cacheSanityCheckEnabled"',
+                    tip: 'If enabled, then Ignite will perform the following checks and throw an exception if check fails<br/>\
+                          <ul>\
+                              <li>Cache entry is not externally locked with lock or lockAsync methods when entry is enlisted to transaction</li>\
+                              <li>Each entry in affinity group - lock transaction has the same affinity key as was specified on affinity transaction start</li>\
+                              <li>Each entry in partition group - lock transaction belongs to the same partition as was specified on partition transaction start</li>\
+                          </ul>'
+                })
 
             .pc-form-grid-col-60(ng-if='$ctrl.available(["1.0.0", "2.1.0"])')
-                +checkbox('Late affinity assignment', model + '.lateAffinityAssignment', '"lateAffinityAssignment"',
-                    'With late affinity assignment mode if primary node was changed for some partition this nodes becomes primary only when rebalancing for all assigned primary partitions is finished')
+                +form-field__checkbox({
+                    label: 'Late affinity assignment',
+                    model: model + '.lateAffinityAssignment',
+                    name: '"lateAffinityAssignment"',
+                    tip: 'With late affinity assignment mode if primary node was changed for some partition this nodes becomes primary only when rebalancing for all assigned primary partitions is finished'
+                })
 
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.1.0")')
-                +number('Long query timeout:', `${model}.longQueryWarningTimeout`, '"LongQueryWarningTimeout"', 'true', '3000', '0',
-                'Timeout in milliseconds after which long query warning will be printed')
+                +form-field__number({
+                    label: 'Long query timeout:',
+                    model: `${model}.longQueryWarningTimeout`,
+                    name: '"LongQueryWarningTimeout"',
+                    placeholder: '3000',
+                    min: '0',
+                    tip: 'Timeout in milliseconds after which long query warning will be printed'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterMisc', 'caches')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/odbc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/odbc.pug
index 481a9aa984c1d..3f1bca58dfe89 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/odbc.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/odbc.pug
@@ -27,12 +27,12 @@ panel-collapsible(
 )
     panel-title ODBC configuration
     panel-description
-        | ODBC server configuration. 
+        | ODBC server configuration.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/odbc-driver" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`$ctrl.available(["1.0.0", "2.1.0"]) && ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +sane-form-field-checkbox({
+                +form-field__checkbox({
                     label: 'Enabled',
                     model: enabled,
                     name: '"odbcEnabled"',
@@ -43,28 +43,63 @@ panel-collapsible(
                     }`
                     ui-validate-watch='$ctrl.Clusters.odbc.odbcEnabled.correctMarshallerWatch("$ctrl.clonedCluster")'
                 )
-                    +form-field-feedback(null, 'correctMarshaller', 'ODBC can only be used with BinaryMarshaller')
+                    +form-field__error({ error: 'correctMarshaller', message: 'ODBC can only be used with BinaryMarshaller' })
             .pc-form-grid-col-60
-                +text-ip-address-with-port-range('ODBC endpoint address:', `${model}.endpointAddress`, '"endpointAddress"', enabled, '0.0.0.0:10800..10810',
-                    'ODBC endpoint address. <br/>\
-                    The following address formats are permitted:\
-                    <ul>\
-                        <li>hostname - will use provided hostname and default port range</li>\
-                        <li>hostname:port - will use provided hostname and port</li>\
-                        <li>hostname:port_from..port_to - will use provided hostname and port range</li>\
-                    </ul>')
+                +form-field__ip-address-with-port-range({
+                    label: `${model}.endpointAddress`,
+                    model: '$item.localOutboundHost',
+                    name: '"endpointAddress"',
+                    enabled,
+                    placeholder: '0.0.0.0:10800..10810',
+                    tip: 'ODBC endpoint address. <br/>\
+                          The following address formats are permitted:\
+                          <ul>\
+                              <li>hostname - will use provided hostname and default port range</li>\
+                              <li>hostname:port - will use provided hostname and port</li>\
+                              <li>hostname:port_from..port_to - will use provided hostname and port range</li>\
+                          </ul>'
+                })
             .pc-form-grid-col-30
-                +number('Send buffer size:', `${model}.socketSendBufferSize`, '"ODBCSocketSendBufferSize"', enabled, '0', '0',
-                    'Socket send buffer size.<br/>\
-                    When set to <b>0</b>, operation system default will be used')
+                +form-field__number({
+                    label: 'Send buffer size:',
+                    model: `${model}.socketSendBufferSize`,
+                    name: '"ODBCSocketSendBufferSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Socket send buffer size.<br/>\
+                          When set to <b>0</b>, operation system default will be used'
+                })
             .pc-form-grid-col-30
-                +number('Socket receive buffer size:', `${model}.socketReceiveBufferSize`, '"ODBCSocketReceiveBufferSize"', enabled, '0', '0',
-                    'Socket receive buffer size.<br/>\
-                    When set to <b>0</b>, operation system default will be used')
+                +form-field__number({
+                    label:'Socket receive buffer size:',
+                    model: `${model}.socketReceiveBufferSize`,
+                    name: '"ODBCSocketReceiveBufferSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Socket receive buffer size.<br/>\
+                          When set to <b>0</b>, operation system default will be used'
+                })
             .pc-form-grid-col-30
-                +number('Maximum open cursors', `${model}.maxOpenCursors`, '"maxOpenCursors"', enabled, '128', '1', 'Maximum number of opened cursors per connection')
+                +form-field__number({
+                    label: 'Maximum open cursors',
+                    model: `${model}.maxOpenCursors`,
+                    name: '"maxOpenCursors"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '128',
+                    min: '1',
+                    tip: 'Maximum number of opened cursors per connection'
+                })
             .pc-form-grid-col-30
-                +number('Pool size:', `${model}.threadPoolSize`, '"ODBCThreadPoolSize"', enabled, 'max(8, availableProcessors)', '1',
-                    'Size of thread pool that is in charge of processing ODBC tasks')
+                +form-field__number({
+                    label: 'Pool size:',
+                    model: `${model}.threadPoolSize`,
+                    name: '"ODBCThreadPoolSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'max(8, availableProcessors)',
+                    min: '1',
+                    tip: 'Size of thread pool that is in charge of processing ODBC tasks'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterODBC')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/persistence.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/persistence.pug
index 2c8d10ac6287b..a15707a46532f 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/persistence.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/persistence.pug
@@ -27,56 +27,189 @@ panel-collapsible(
 )
     panel-title Persistence store
     panel-description
-        | Configures Apache Ignite Native Persistence. 
+        | Configures Apache Ignite Native Persistence.
         a.link-success(href='https://apacheignite.readme.io/docs/distributed-persistent-store' target='_blank') More info
     panel-content.pca-form-row(ng-if=`$ctrl.available(["2.1.0", "2.3.0"]) && ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +checkbox('Enabled', enabled, '"PersistenceEnabled"', 'Flag indicating whether to configure persistent configuration')
+                +form-field__checkbox({
+                    label: 'Enabled',
+                    model: enabled,
+                    name: '"PersistenceEnabled"',
+                    tip: 'Flag indicating whether to configure persistent configuration'
+                })
             .pc-form-grid-col-60
-                +text-enabled('Store path:', `${model}.persistentStorePath`, '"PersistenceStorePath"', enabled, 'false', 'Input store path',
-                'A path the root directory where the Persistent Store will persist data and indexes')
+                +form-field__text({
+                    label: 'Store path:',
+                    model: `${model}.persistentStorePath`,
+                    name: '"PersistenceStorePath"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'Input store path',
+                    tip: 'A path the root directory where the Persistent Store will persist data and indexes'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('Metrics enabled', `${model}.metricsEnabled`, '"PersistenceMetricsEnabled"', enabled, 'Flag indicating whether persistence metrics collection is enabled')
+                +form-field__checkbox({
+                    label: 'Metrics enabled',
+                    model: `${model}.metricsEnabled`,
+                    name: '"PersistenceMetricsEnabled"',
+                    disabled: `!${enabled}`,
+                    tip: 'Flag indicating whether persistence metrics collection is enabled'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('Always write full pages', `${model}.alwaysWriteFullPages`, '"PersistenceAlwaysWriteFullPages"', enabled, 'Flag indicating whether always write full pages')
+                +form-field__checkbox({
+                    label: 'Always write full pages',
+                    model: `${model}.alwaysWriteFullPages`,
+                    name: '"PersistenceAlwaysWriteFullPages"',
+                    disabled: `!${enabled}`,
+                    tip: 'Flag indicating whether always write full pages'
+                })
             .pc-form-grid-col-60
-                +number('Checkpointing frequency:', `${model}.checkpointingFrequency`, '"PersistenceCheckpointingFrequency"', enabled, '180000', '1',
-                'Frequency which is a minimal interval when the dirty pages will be written to the Persistent Store')
+                +form-field__number({
+                    label: 'Checkpointing frequency:',
+                    model: `${model}.checkpointingFrequency`,
+                    name: '"PersistenceCheckpointingFrequency"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '180000',
+                    min: '1',
+                    tip: 'Frequency which is a minimal interval when the dirty pages will be written to the Persistent Store'
+                })
             .pc-form-grid-col-60
-                +number('Checkpointing page buffer size:', `${model}.checkpointingPageBufferSize`, '"PersistenceCheckpointingPageBufferSize"', enabled, '268435456', '0',
-                'Amount of memory allocated for a checkpointing temporary buffer')
+                +form-field__number({
+                    label: 'Checkpointing page buffer size:',
+                    model: `${model}.checkpointingPageBufferSize`,
+                    name: '"PersistenceCheckpointingPageBufferSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '268435456',
+                    min: '0',
+                    tip: 'Amount of memory allocated for a checkpointing temporary buffer'
+                })
             .pc-form-grid-col-60
-                +number('Checkpointing threads:', `${model}.checkpointingThreads`, '"PersistenceCheckpointingThreads"', enabled, '1', '1', 'A number of threads to use for the checkpointing purposes')
+                +form-field__number({
+                    label: 'Checkpointing threads:',
+                    model: `${model}.checkpointingThreads`,
+                    name: '"PersistenceCheckpointingThreads"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '1',
+                    min: '1',
+                    tip: 'A number of threads to use for the checkpointing purposes'
+                })
             .pc-form-grid-col-60
-                +text-enabled('WAL store path:', `${model}.walStorePath`, '"PersistenceWalStorePath"', enabled, 'false', 'Input store path', 'A path to the directory where WAL is stored')
+                +form-field__text({
+                    label: 'WAL store path:',
+                    model: `${model}.walStorePath`,
+                    name: '"PersistenceWalStorePath"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'Input store path',
+                    tip: 'A path to the directory where WAL is stored'
+                })
             .pc-form-grid-col-60
-                +text-enabled('WAL archive path:', `${model}.walArchivePath`, '"PersistenceWalArchivePath"', enabled, 'false', 'Input archive path', 'A path to the WAL archive directory')
+                +form-field__text({
+                    label: 'WAL archive path:',
+                    model: `${model}.walArchivePath`,
+                    name: '"PersistenceWalArchivePath"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'Input archive path',
+                    tip: 'A path to the WAL archive directory'
+                })
             .pc-form-grid-col-30
-                +number('WAL segments:', `${model}.walSegments`, '"PersistenceWalSegments"', enabled, '10', '1', 'A number of WAL segments to work with')
+                +form-field__number({
+                    label: 'WAL segments:',
+                    model: `${model}.walSegments`,
+                    name: '"PersistenceWalSegments"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '10',
+                    min: '1',
+                    tip: 'A number of WAL segments to work with'
+                })
             .pc-form-grid-col-30
-                +number('WAL segment size:', `${model}.walSegmentSize`, '"PersistenceWalSegmentSize"', enabled, '67108864', '0', 'Size of a WAL segment')
+                +form-field__number({
+                    label: 'WAL segment size:',
+                    model: `${model}.walSegmentSize`,
+                    name: '"PersistenceWalSegmentSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '67108864',
+                    min: '0',
+                    tip: 'Size of a WAL segment'
+                })
             .pc-form-grid-col-30
-                +number('WAL history size:', `${model}.walHistorySize`, '"PersistenceWalHistorySize"', enabled, '20', '1', 'A total number of checkpoints to keep in the WAL history')
+                +form-field__number({
+                    label: 'WAL history size:',
+                    model: `${model}.walHistorySize`,
+                    name: '"PersistenceWalHistorySize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '20',
+                    min: '1',
+                    tip: 'A total number of checkpoints to keep in the WAL history'
+                })
             .pc-form-grid-col-30
-                +number('WAL flush frequency:', `${model}.walFlushFrequency`, '"PersistenceWalFlushFrequency"', enabled, '2000', '1',
-                'How often will be fsync, in milliseconds. In background mode, exist thread which do fsync by timeout')
+                +form-field__number({
+                    label: 'WAL flush frequency:',
+                    model: `${model}.walFlushFrequency`,
+                    name: '"PersistenceWalFlushFrequency"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '2000',
+                    min: '1',
+                    tip:'How often will be fsync, in milliseconds. In background mode, exist thread which do fsync by timeout'
+                })
             .pc-form-grid-col-30
-                +number('WAL fsync delay:', `${model}.walFsyncDelayNanos`, '"PersistenceWalFsyncDelay"', enabled, '1000', '1', 'WAL fsync delay, in nanoseconds')
+                +form-field__number({
+                    label: 'WAL fsync delay:',
+                    model: `${model}.walFsyncDelayNanos`,
+                    name: '"PersistenceWalFsyncDelay"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '1000',
+                    min: '1',
+                    tip: 'WAL fsync delay, in nanoseconds'
+                })
             .pc-form-grid-col-30
-                +number('WAL record iterator buffer size:', `${model}.walRecordIteratorBufferSize`, '"PersistenceWalRecordIteratorBufferSize"', enabled, '67108864', '1',
-                'How many bytes iterator read from disk(for one reading), during go ahead WAL')
+                +form-field__number({
+                    label: 'WAL record iterator buffer size:',
+                    model: `${model}.walRecordIteratorBufferSize`,
+                    name: '"PersistenceWalRecordIteratorBufferSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '67108864',
+                    min: '1',
+                    tip: 'How many bytes iterator read from disk(for one reading), during go ahead WAL'
+                })
             .pc-form-grid-col-30
-                +number('Lock wait time:', `${model}.lockWaitTime`, '"PersistenceLockWaitTime"', enabled, '10000', '1',
-                'Time out in second, while wait and try get file lock for start persist manager')
+                +form-field__number({
+                    label: 'Lock wait time:',
+                    model: `${model}.lockWaitTime`,
+                    name: '"PersistenceLockWaitTime"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '10000',
+                    min: '1',
+                    tip: 'Time out in second, while wait and try get file lock for start persist manager'
+                })
             .pc-form-grid-col-30
-                +number('Rate time interval:', `${model}.rateTimeInterval`, '"PersistenceRateTimeInterval"', enabled, '60000', '1000',
-                'The length of the time interval for rate - based metrics. This interval defines a window over which hits will be tracked.')
+                +form-field__number({
+                    label: 'Rate time interval:' ,
+                    model: `${model}.rateTimeInterval`,
+                    name: '"PersistenceRateTimeInterval"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '60000',
+                    min: '1000',
+                    tip: 'The length of the time interval for rate - based metrics. This interval defines a window over which hits will be tracked.'
+                })
             .pc-form-grid-col-30
-                +number('Thread local buffer size:', `${model}.tlbSize`, '"PersistenceTlbSize"', enabled, '131072', '1',
-                'Define size thread local buffer. Each thread which write to WAL have thread local buffer for serialize recode before write in WAL')
+                +form-field__number({
+                    label: 'Thread local buffer size:',
+                    model: `${model}.tlbSize`,
+                    name: '"PersistenceTlbSize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '131072',
+                    min: '1',
+                    tip: 'Define size thread local buffer. Each thread which write to WAL have thread local buffer for serialize recode before write in WAL'
+                })
             .pc-form-grid-col-30
-                +number('Sub intervals:', `${model}.subIntervals`, '"PersistenceSubIntervals"', enabled, '5', '1',
-                'Number of sub - intervals the whole rate time interval will be split into to calculate rate - based metrics')
+                +form-field__number({
+                    label: 'Sub intervals:',
+                    model: `${model}.subIntervals`,
+                    name: '"PersistenceSubIntervals"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '5',
+                    min: '1',
+                    tip: 'Number of sub - intervals the whole rate time interval will be split into to calculate rate - based metrics'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterPersistence')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug
index b37067bb6b6eb..10b5dd8385324 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/service.pug
@@ -22,60 +22,90 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Service configuration
     panel-description
-        | Service Grid allows for deployments of arbitrary user-defined services on the cluster. 
+        | Service Grid allows for deployments of arbitrary user-defined services on the cluster.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/fault-tolerance" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6
             mixin clusters-service-configurations
                 .ignite-form-field(ng-init='serviceConfigurationsTbl={type: "serviceConfigurations", model: "serviceConfigurations", focusId: "kind", ui: "failover-table"}')
-                    +ignite-form-field__label('Service configurations:', '"serviceConfigurations"')
-                    .ignite-form-field__control
-                        -let items = model
+                    +form-field__label({ label: 'Service configurations:', name: '"serviceConfigurations"' })
 
-                        list-editable.pc-list-editable-with-form-grid(ng-model=items name='serviceConfigurations')
-                            list-editable-item-edit.pc-form-grid-row
-                                .pc-form-grid-col-60
-                                    +sane-ignite-form-field-text({
-                                        label: 'Name:',
-                                        model: '$item.name',
-                                        name: '"serviceName"',
-                                        required: true,
-                                        placeholder: 'Input service name'
-                                    })(
-                                        ui-validate=`{
-                                            uniqueName: '$ctrl.Clusters.serviceConfigurations.serviceConfiguration.name.customValidators.uniqueName($item, ${items})'
-                                        }`
-                                        ui-validate-watch=`"${items}"`
-                                        ui-validate-watch-object-equality='true'
-                                        ng-model-options='{allowInvalid: true}'
-                                    )
-                                        +form-field-feedback('"serviceName', 'uniqueName', 'Service with that name is already configured')
-                                .pc-form-grid-col-60
-                                    +java-class('Service class', '$item.service', '"serviceService"', 'true', 'true', 'Service implementation class name')
-                                .pc-form-grid-col-60
-                                    +number('Max per node count:', '$item.maxPerNodeCount', '"ServiceMaxPerNodeCount"', 'true', 'Unlimited', '0',
-                                        'Maximum number of deployed service instances on each node.<br/>' +
-                                        'Zero for unlimited')
-                                .pc-form-grid-col-60
-                                    +number('Total count:', '$item.totalCount', '"serviceTotalCount"', 'true', 'Unlimited', '0',
-                                        'Total number of deployed service instances in the cluster.<br/>' +
-                                        'Zero for unlimited')
-                                .pc-form-grid-col-60
-                                    +dropdown-required-empty('Cache:', '$item.cache', '"serviceCache"', 'true', 'false',
-                                        'Choose cache', 'No caches configured for current cluster', '$ctrl.cachesMenu', 'Cache name used for key-to-node affinity calculation')(
-                                        pc-is-in-collection='$ctrl.clonedCluster.caches'
-                                    )
-                                        +form-field-feedback(_, 'isInCollection', `Cluster doesn't have such a cache`)
-                                .pc-form-grid-col-60
-                                    +text('Affinity key:', '$item.affinityKey', '"serviceAffinityKey"', 'false', 'Input affinity key',
-                                        'Affinity key used for key-to-node affinity calculation')
+                    -let items = model
 
-                            list-editable-no-items
-                                list-editable-add-item-button(
-                                    add-item=`$ctrl.Clusters.addServiceConfiguration($ctrl.clonedCluster)`
-                                    label-single='service configuration'
-                                    label-multiple='service configurations'
+                    list-editable.pc-list-editable-with-form-grid(ng-model=items name='serviceConfigurations')
+                        list-editable-item-edit.pc-form-grid-row
+                            .pc-form-grid-col-60
+                                +form-field__text({
+                                    label: 'Name:',
+                                    model: '$item.name',
+                                    name: '"serviceName"',
+                                    required: true,
+                                    placeholder: 'Input service name'
+                                })(
+                                    ui-validate=`{
+                                        uniqueName: '$ctrl.Clusters.serviceConfigurations.serviceConfiguration.name.customValidators.uniqueName($item, ${items})'
+                                    }`
+                                    ui-validate-watch=`"${items}"`
+                                    ui-validate-watch-object-equality='true'
+                                    ng-model-options='{allowInvalid: true}'
                                 )
+                                    +form-field__error({ error: 'uniqueName', message: 'Service with that name is already configured' })
+                            .pc-form-grid-col-60
+                                +form-field__java-class({
+                                    label: 'Service class',
+                                    model: '$item.service',
+                                    name: '"serviceService"',
+                                    required: 'true',
+                                    tip: 'Service implementation class name'
+                                })
+                            .pc-form-grid-col-60
+                                +form-field__number({
+                                    label: 'Max per node count:',
+                                    model: '$item.maxPerNodeCount',
+                                    name: '"ServiceMaxPerNodeCount"',
+                                    placeholder: 'Unlimited',
+                                    min: '0',
+                                    tip: 'Maximum number of deployed service instances on each node.<br/>\
+                                          Zero for unlimited'
+                                })
+                            .pc-form-grid-col-60
+                                +form-field__number({
+                                    label: 'Total count:',
+                                    model: '$item.totalCount',
+                                    name: '"serviceTotalCount"',
+                                    placeholder: 'Unlimited',
+                                    min: '0',
+                                    tip: 'Total number of deployed service instances in the cluster.<br/>\
+                                        Zero for unlimited'
+                                })
+                            .pc-form-grid-col-60
+                                +form-field__dropdown({
+                                    label: 'Cache:',
+                                    model: '$item.cache',
+                                    name: '"serviceCache"',
+                                    placeholder: 'Choose cache',
+                                    placeholderEmpty: 'No caches configured for current cluster',
+                                    options: '$ctrl.cachesMenu',
+                                    tip: 'Cache name used for key-to-node affinity calculation'
+                                })(
+                                    pc-is-in-collection='$ctrl.clonedCluster.caches'
+                                )
+                                    +form-field__error({ error: 'isInCollection', message: `Cluster doesn't have such a cache` })
+                            .pc-form-grid-col-60
+                                +form-field__text({
+                                    label: 'Affinity key:',
+                                    model: '$item.affinityKey',
+                                    name: '"serviceAffinityKey"',
+                                    placeholder: 'Input affinity key',
+                                    tip: 'Affinity key used for key-to-node affinity calculation'
+                                })
+
+                        list-editable-no-items
+                            list-editable-add-item-button(
+                                add-item=`$ctrl.Clusters.addServiceConfiguration($ctrl.clonedCluster)`
+                                label-single='service configuration'
+                                label-multiple='service configurations'
+                            )
 
             +clusters-service-configurations
 
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/sql-connector.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/sql-connector.pug
index 708aa0d8a3b17..3b2ca278bb9c6 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/sql-connector.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/sql-connector.pug
@@ -31,28 +31,86 @@ panel-collapsible(
     panel-content.pca-form-row(ng-if=`$ctrl.available(["2.1.0", "2.3.0"]) && ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +checkbox('Enabled', connectionEnabled, '"SqlConnectorEnabled"', 'Flag indicating whether to configure SQL connector configuration')
+                +form-field__checkbox({
+                    label: 'Enabled',
+                    model: connectionEnabled,
+                    name: '"SqlConnectorEnabled"',
+                    tip: 'Flag indicating whether to configure SQL connector configuration'
+                })
             .pc-form-grid-col-60
-                +text-enabled('Host:', `${connectionModel}.host`, '"SqlConnectorHost"', connectionEnabled, 'false', 'localhost')
+                +form-field__text({
+                    label: 'Host:',
+                    model: `${connectionModel}.host`,
+                    name: '"SqlConnectorHost"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: 'localhost'
+                })
             .pc-form-grid-col-30
-                +number('Port:', `${connectionModel}.port`, '"SqlConnectorPort"', connectionEnabled, '10800', '1025')
+                +form-field__number({
+                    label: 'Port:',
+                    model: `${connectionModel}.port`,
+                    name: '"SqlConnectorPort"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '10800',
+                    min: '1025'
+                })
             .pc-form-grid-col-30
-                +number('Port range:', `${connectionModel}.portRange`, '"SqlConnectorPortRange"', connectionEnabled, '100', '0')
+                +form-field__number({
+                    label: 'Port range:',
+                    model: `${connectionModel}.portRange`,
+                    name: '"SqlConnectorPortRange"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '100',
+                    min: '0'
+                })
             .pc-form-grid-col-30
-                +number('Socket send buffer size:', `${connectionModel}.socketSendBufferSize`, '"SqlConnectorSocketSendBufferSize"', connectionEnabled, '0', '0',
-                    'Socket send buffer size.<br/>\
-                    When set to <b>0</b>, operation system default will be used')
+                +form-field__number({
+                    label: 'Socket send buffer size:',
+                    model: `${connectionModel}.socketSendBufferSize`,
+                    name: '"SqlConnectorSocketSendBufferSize"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Socket send buffer size.<br/>\
+                          When set to <b>0</b>, operation system default will be used'
+                })
             .pc-form-grid-col-30
-                +number('Socket receive buffer size:', `${connectionModel}.socketReceiveBufferSize`, '"SqlConnectorSocketReceiveBufferSize"', connectionEnabled, '0', '0',
-                    'Socket receive buffer size.<br/>\
-                    When set to <b>0</b>, operation system default will be used')
+                +form-field__number({
+                    label: 'Socket receive buffer size:',
+                    model: `${connectionModel}.socketReceiveBufferSize`,
+                    name: '"SqlConnectorSocketReceiveBufferSize"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Socket receive buffer size.<br/>\
+                         When set to <b>0</b>, operation system default will be used'
+                })
             .pc-form-grid-col-30
-                +number('Max connection cursors:', `${connectionModel}.maxOpenCursorsPerConnection`, '"SqlConnectorMaxOpenCursorsPerConnection"', connectionEnabled, '128', '0',
-                    'Max number of opened cursors per connection')
+                +form-field__number({
+                    label: 'Max connection cursors:',
+                    model: `${connectionModel}.maxOpenCursorsPerConnection`,
+                    name: '"SqlConnectorMaxOpenCursorsPerConnection"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: '128',
+                    min: '0',
+                    tip: 'Max number of opened cursors per connection'
+                })
             .pc-form-grid-col-30
-                +number('Pool size:', `${connectionModel}.threadPoolSize`, '"SqlConnectorThreadPoolSize"', connectionEnabled, 'max(8, availableProcessors)', '1',
-                    'Size of thread pool that is in charge of processing SQL requests')
+                +form-field__number({
+                    label: 'Pool size:',
+                    model: `${connectionModel}.threadPoolSize`,
+                    name: '"SqlConnectorThreadPoolSize"',
+                    disabled: `!(${connectionEnabled})`,
+                    placeholder: 'max(8, availableProcessors)',
+                    min: '1',
+                    tip: 'Size of thread pool that is in charge of processing SQL requests'
+                })
             .pc-form-grid-col-60
-                +checkbox-enabled('TCP_NODELAY option', `${connectionModel}.tcpNoDelay`, '"SqlConnectorTcpNoDelay"', connectionEnabled)
+                +form-field__checkbox({
+                    label: 'TCP_NODELAY option',
+                    model: `${connectionModel}.tcpNoDelay`,
+                    name: '"SqlConnectorTcpNoDelay"',
+                    disabled: `!${connectionEnabled}`
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterQuery')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/ssl.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/ssl.pug
index 2745f53939f3f..61c722ee2d7a5 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/ssl.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/ssl.pug
@@ -25,55 +25,89 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title SSL configuration
     panel-description
-        | Settings for SSL configuration for creating a secure socket layer. 
+        | Settings for SSL configuration for creating a secure socket layer.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/ssltls" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +checkbox('Enabled', enabled, '"sslEnabled"', 'Flag indicating whether to configure SSL configuration')
+                +form-field__checkbox({
+                    label: 'Enabled',
+                    model: enabled,
+                    name: '"sslEnabled"',
+                    tip: 'Flag indicating whether to configure SSL configuration'
+                })
             .pc-form-grid-col-60
-                +text-options('Algorithm to create a key manager:', `${model}.keyAlgorithm`, '"keyAlgorithm"', '["SumX509", "X509"]', enabled, 'false', 'SumX509',
-                    'Sets key manager algorithm that will be used to create a key manager<br/>\
-                    Notice that in most cased default value suites well, however, on Android platform this value need to be set to X509')
+                +form-field__typeahead({
+                    label: 'Algorithm to create a key manager:',
+                    model: `${model}.keyAlgorithm`,
+                    name: '"keyAlgorithm"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'SumX509',
+                    options: '["SumX509", "X509"]',
+                    tip: 'Sets key manager algorithm that will be used to create a key manager<br/>\
+                         Notice that in most cased default value suites well, however, on Android platform this value need to be set to X509'
+                })
+
             .pc-form-grid-col-60
-                +text-enabled('Key store file:', `${model}.keyStoreFilePath`, '"keyStoreFilePath"', enabled, enabled, 'Path to the key store file',
-                    'Path to the key store file<br/>\
-                    This is a mandatory parameter since ssl context could not be initialized without key manager')
+                +form-field__text({
+                    label: 'Key store file:',
+                    model: `${secondaryFileSystem}.keyStoreFilePath`,
+                    name: '"keyStoreFilePath"',
+                    disabled: `!(${enabled})`,
+                    required: enabled,
+                    placeholder: 'Path to the key store file',
+                    tip: 'Path to the key store file<br/>\
+                          This is a mandatory parameter since ssl context could not be initialized without key manager'
+                })
             .pc-form-grid-col-30
-                +text-options('Key store type:', `${model}.keyStoreType`, '"keyStoreType"', '["JKS", "PCKS11", "PCKS12"]', enabled, 'false', 'JKS',
-                    'Key store type used in context initialization')
+                +form-field__typeahead({
+                    label: 'Key store type:',
+                    model: `${model}.keyStoreType`,
+                    name: '"keyStoreType"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'JKS',
+                    options: '["JKS", "PCKS11", "PCKS12"]',
+                    tip: 'Key store type used in context initialization'
+                })
             .pc-form-grid-col-30
-                +text-options('Protocol:', `${model}.protocol`, '"protocol"', '["TSL", "SSL"]', enabled, 'false', 'TSL', 'Protocol for secure transport')
+                +form-field__typeahead({
+                    label: 'Protocol:',
+                    model: `${model}.protocol`,
+                    name: '"protocol"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'TSL',
+                    options: '["TSL", "SSL"]',
+                    tip: 'Protocol for secure transport'
+                })
             .pc-form-grid-col-60
                 .ignite-form-field
-                    .ignite-form-field__control
-                        list-editable(
-                            ng-model=trust
-                            name='trustManagers'
-                            list-editable-cols=`::[{name: "Pre-configured trust managers:"}]`
-                            ng-disabled=enabledToDisabled(enabled)
-                            ng-required=`${enabled} && !${model}.trustStoreFilePath`
-                        )
-                            list-editable-item-view {{ $item }}
+                    list-editable(
+                        ng-model=trust
+                        name='trustManagers'
+                        list-editable-cols=`::[{name: "Pre-configured trust managers:"}]`
+                        ng-disabled=enabledToDisabled(enabled)
+                        ng-required=`${enabled} && !${model}.trustStoreFilePath`
+                    )
+                        list-editable-item-view {{ $item }}
 
-                            list-editable-item-edit
-                                +list-java-class-field('Trust manager', '$item', '"trustManager"', trust)
-                                    +unique-feedback('"trustManager"', 'Such trust manager already exists!')
+                        list-editable-item-edit
+                            +list-java-class-field('Trust manager', '$item', '"trustManager"', trust)
+                                +form-field__error({ error: 'igniteUnique', message: 'Such trust manager already exists!' })
 
-                            list-editable-no-items
-                                list-editable-add-item-button(
-                                    add-item=`$editLast((${trust} = ${trust} || []).push(''))`
-                                    label-single='trust manager'
-                                    label-multiple='trust managers'
-                                )
-                    .ignite-form-field__errors(
+                        list-editable-no-items
+                            list-editable-add-item-button(
+                                add-item=`$editLast((${trust} = ${trust} || []).push(''))`
+                                label-single='trust manager'
+                                label-multiple='trust managers'
+                            )
+                    .form-field__errors(
                         ng-messages=`sslConfiguration.trustManagers.$error`
                         ng-show=`sslConfiguration.trustManagers.$invalid`
                     )
-                        +form-field-feedback(_, 'required', 'Trust managers or trust store file should be configured')
+                        +form-field__error({ error: 'required', message: 'Trust managers or trust store file should be configured' })
 
             .pc-form-grid-col-30(ng-if-start=`!${trust}.length`)
-                +sane-ignite-form-field-text({
+                +form-field__text({
                     label: 'Trust store file:',
                     model: `${model}.trustStoreFilePath`,
                     name: '"trustStoreFilePath"',
@@ -82,8 +116,16 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     placeholder: 'Path to the trust store file',
                     tip: 'Path to the trust store file'
                 })
-                    +form-field-feedback(_, 'required', 'Trust store file or trust managers should be configured')
+                    +form-field__error({ error: 'required', message: 'Trust store file or trust managers should be configured' })
             .pc-form-grid-col-30(ng-if-end)
-                +text-options('Trust store type:', `${model}.trustStoreType`, '"trustStoreType"', '["JKS", "PCKS11", "PCKS12"]', enabled, 'false', 'JKS', 'Trust store type used in context initialization')
+                +form-field__typeahead({
+                    label: 'Trust store type:',
+                    model: `${model}.trustStoreType`,
+                    name: '"trustStoreType"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'JKS',
+                    options: '["JKS", "PCKS11", "PCKS12"]',
+                    tip: 'Trust store type used in context initialization'
+                })
         .pca-form-column-6
             +preview-xml-java(cluster, 'clusterSsl')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/swap.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/swap.pug
index d314296356f19..d39dae61404f4 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/swap.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/swap.pug
@@ -28,25 +28,35 @@ panel-collapsible(
 )
     panel-title Swap
     panel-description
-        | Settings for overflow data to disk if it cannot fit in memory. 
+        | Settings for overflow data to disk if it cannot fit in memory.
         | #[a.link-success(href="https://apacheignite.readme.io/v1.9/docs/off-heap-memory#swap-space" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`$ctrl.available(["1.0.0", "2.0.0"]) && ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +dropdown('Swap space SPI:', `${swapModel}.kind`, '"swapSpaceSpi"', 'true', 'Choose swap SPI',
-                    '::$ctrl.Clusters.swapSpaceSpis',
-                    'Provides a mechanism in grid for storing data on disk<br/>\
-                    Ignite cache uses swap space to overflow data to disk if it cannot fit in memory\
-                    <ul>\
-                        <li>File-based swap - File-based swap space SPI implementation which holds keys in memory</li>\
-                        <li>Not set - File-based swap space SPI with default configuration when it needed</li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Swap space SPI:',
+                    model: `${swapModel}.kind`,
+                    name: '"swapSpaceSpi"',
+                    placeholder: 'Choose swap SPI',
+                    options: '::$ctrl.Clusters.swapSpaceSpis',
+                    tip: 'Provides a mechanism in grid for storing data on disk<br/>\
+                        Ignite cache uses swap space to overflow data to disk if it cannot fit in memory\
+                        <ul>\
+                            <li>File-based swap - File-based swap space SPI implementation which holds keys in memory</li>\
+                            <li>Not set - File-based swap space SPI with default configuration when it needed</li>\
+                        </ul>'
+                })
             .pc-form-group.pc-form-grid-row(ng-show=`${swapModel}.kind`)
                 .pc-form-grid-col-60
-                    +text('Base directory:', `${fileSwapModel}.baseDirectory`, '"baseDirectory"', 'false', 'swapspace',
-                        'Base directory where to write files')
+                    +form-field__text({
+                        label: 'Base directory:',
+                        model: `${fileSwapModel}.baseDirectory`,
+                        name: '"baseDirectory"',
+                        placeholder: 'swapspace',
+                        tip: 'Base directory where to write files'
+                    })
                 .pc-form-grid-col-30
-                    +sane-ignite-form-field-number({
+                    +form-field__number({
                         label: 'Read stripe size:',
                         model: `${fileSwapModel}.readStripesNumber`,
                         name: '"readStripesNumber"',
@@ -57,18 +67,38 @@ panel-collapsible(
                             powerOfTwo: '$ctrl.Clusters.swapSpaceSpi.readStripesNumber.customValidators.powerOfTwo($value)'
                         }`
                     )
-                        +form-field-feedback('"readStripesNumber"', 'powerOfTwo', 'Read stripe size must be positive and power of two')
+                        +form-field__error({ error: 'powerOfTwo', message: 'Read stripe size must be positive and power of two' })
                 .pc-form-grid-col-30
-                    +number-min-max-step('Maximum sparsity:', `${fileSwapModel}.maximumSparsity`, '"maximumSparsity"', 'true', '0.5', '0', '0.999', '0.001',
-                        'This property defines maximum acceptable wasted file space to whole file size ratio<br/>\
-                        When this ratio becomes higher than specified number compacting thread starts working')
+                    +form-field__number({
+                        label: 'Maximum sparsity:',
+                        model: `${fileSwapModel}.maximumSparsity`,
+                        name: '"maximumSparsity"',
+                        placeholder: '0.5',
+                        min: '0',
+                        max: '0.999',
+                        step: '0.001',
+                        tip: 'This property defines maximum acceptable wasted file space to whole file size ratio<br/>\
+                             When this ratio becomes higher than specified number compacting thread starts working'
+                    })
                 .pc-form-grid-col-30
-                    +number('Max write queue size:', `${fileSwapModel}.maxWriteQueueSize`, '"maxWriteQueueSize"', 'true', '1024 * 1024', '0',
-                        'Max write queue size in bytes<br/>\
-                        If there are more values are waiting for being written to disk then specified size, SPI will block on store operation')
+                    +form-field__number({
+                        label: 'Max write queue size:',
+                        model: `${fileSwapModel}.maxWriteQueueSize`,
+                        name: '"maxWriteQueueSize"',
+                        placeholder: '1024 * 1024',
+                        min: '0',
+                        tip: 'Max write queue size in bytes<br/>\
+                              If there are more values are waiting for being written to disk then specified size, SPI will block on store operation'
+                    })
                 .pc-form-grid-col-30
-                    +number('Write buffer size:', `${fileSwapModel}.writeBufferSize`, '"writeBufferSize"', 'true', '64 * 1024', '0',
-                        'Write buffer size in bytes<br/>\
-                        Write to disk occurs only when this buffer is full')
+                    +form-field__number({
+                        label: 'Write buffer size:',
+                        model: `${fileSwapModel}.writeBufferSize`,
+                        name: '"writeBufferSize"',
+                        placeholder: '64 * 1024',
+                        min: '0',
+                        tip: 'Write buffer size in bytes<br/>\
+                              Write to disk occurs only when this buffer is full'
+                    })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterSwap')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/thread.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/thread.pug
index ebe3bcdbad239..76633f3b26b9c 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/thread.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/thread.pug
@@ -26,10 +26,16 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-30
-                +number('Public:', model + '.publicThreadPoolSize', '"publicThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
-                    'Thread pool that is in charge of processing ComputeJob, GridJobs and user messages sent to node')
+                +form-field__number({
+                    label: 'Public:',
+                    model: model + '.publicThreadPoolSize',
+                    name: '"publicThreadPoolSize"',
+                    placeholder: 'max(8, availableProcessors) * 2',
+                    min: '1',
+                    tip: 'Thread pool that is in charge of processing ComputeJob, GridJobs and user messages sent to node'
+                })
             .pc-form-grid-col-30
-                +sane-ignite-form-field-number({
+                +form-field__number({
                     label: 'System:',
                     model: `${model}.systemThreadPoolSize`,
                     name: '"systemThreadPoolSize"',
@@ -38,16 +44,34 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     tip: 'Thread pool that is in charge of processing internal system messages'
                 })
             .pc-form-grid-col-30
-                +number('Service:', model + '.serviceThreadPoolSize', '"serviceThreadPoolSize"', 'true', 'max(8, availableProcessors) * 2', '1',
-                    'Thread pool that is in charge of processing proxy invocation')
+                +form-field__number({
+                    label: 'Service:',
+                    model: model + '.serviceThreadPoolSize',
+                    name: '"serviceThreadPoolSize"',
+                    placeholder: 'max(8, availableProcessors) * 2',
+                    min: '1',
+                    tip: 'Thread pool that is in charge of processing proxy invocation'
+                })
             .pc-form-grid-col-30
-                +number('Management:', model + '.managementThreadPoolSize', '"managementThreadPoolSize"', 'true', '4', '1',
-                    'Thread pool that is in charge of processing internal and Visor ComputeJob, GridJobs')
+                +form-field__number({
+                    label: 'Management:',
+                    model: model + '.managementThreadPoolSize',
+                    name: '"managementThreadPoolSize"',
+                    placeholder: '4',
+                    min: '1',
+                    tip: 'Thread pool that is in charge of processing internal and Visor ComputeJob, GridJobs'
+                })
             .pc-form-grid-col-30
-                +number('IGFS:', model + '.igfsThreadPoolSize', '"igfsThreadPoolSize"', 'true', 'availableProcessors', '1',
-                    'Thread pool that is in charge of processing outgoing IGFS messages')
+                +form-field__number({
+                    label: 'IGFS:',
+                    model: model + '.igfsThreadPoolSize',
+                    name: '"igfsThreadPoolSize"',
+                    placeholder: 'availableProcessors',
+                    min: '1',
+                    tip: 'Thread pool that is in charge of processing outgoing IGFS messages'
+                })
             .pc-form-grid-col-30
-                +sane-ignite-form-field-number({
+                +form-field__number({
                     label: 'Rebalance:',
                     model: `${model}.rebalanceThreadPoolSize`,
                     name: '"rebalanceThreadPoolSize"',
@@ -56,12 +80,18 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     max: `{{ $ctrl.Clusters.rebalanceThreadPoolSize.max(${model}) }}`,
                     tip: 'Max count of threads can be used at rebalancing'
                 })
-                    +form-field-feedback('max', 'Rebalance thread pool size should not exceed or be equal to System thread pool size')
+                    +form-field__error({ error: 'max', message: 'Rebalance thread pool size should not exceed or be equal to System thread pool size' })
             .pc-form-grid-col-30
-                +number('Utility cache:', model + '.utilityCacheThreadPoolSize', '"utilityCacheThreadPoolSize"', 'true', 'max(8, availableProcessors)', '1',
-                    'Default thread pool size that will be used to process utility cache messages')
+                +form-field__number({
+                    label: 'Utility cache:',
+                    model: model + '.utilityCacheThreadPoolSize',
+                    name: '"utilityCacheThreadPoolSize"',
+                    placeholder: 'max(8, availableProcessors)',
+                    min: '1',
+                    tip: 'Default thread pool size that will be used to process utility cache messages'
+                })
             .pc-form-grid-col-30
-                pc-form-field-size(
+                form-field-size(
                     label='Utility cache keep alive time:'
                     ng-model=`${model}.utilityCacheKeepAliveTime`
                     name='utilityCacheKeepAliveTime'
@@ -73,72 +103,104 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     on-scale-change='_s1 = $event'
                 )
             .pc-form-grid-col-30
-                +number('Async callback:', model + '.asyncCallbackPoolSize', '"asyncCallbackPoolSize"', 'true', 'max(8, availableProcessors)', '1',
-                    'Size of thread pool that is in charge of processing asynchronous callbacks')
+                +form-field__number({
+                    label:'Async callback:',
+                    model: model + '.asyncCallbackPoolSize',
+                    name: '"asyncCallbackPoolSize"',
+                    placeholder: 'max(8, availableProcessors)',
+                    min: '1',
+                    tip: 'Size of thread pool that is in charge of processing asynchronous callbacks'
+                })
             .pc-form-grid-col-30
-                +number('Striped:', model + '.stripedPoolSize', '"stripedPoolSize"', 'true', 'max(8, availableProcessors)', '1',
-                    'Striped pool size that should be used for cache requests processing')
+                +form-field__number({
+                    label: 'Striped:',
+                    model: model + '.stripedPoolSize',
+                    name: '"stripedPoolSize"',
+                    placeholder: 'max(8, availableProcessors)',
+                    min: '1',
+                    tip: 'Striped pool size that should be used for cache requests processing'
+                })
 
             //- Since ignite 2.0
             .pc-form-grid-col-30(ng-if-start='$ctrl.available("2.0.0")')
-                +number('Data streamer:', model + '.dataStreamerThreadPoolSize', '"dataStreamerThreadPoolSize"', 'true', 'max(8, availableProcessors)', '1',
-                    'Size of thread pool that is in charge of processing data stream messages')
+                +form-field__number({
+                    label: 'Data streamer:',
+                    model: model + '.dataStreamerThreadPoolSize',
+                    name: '"dataStreamerThreadPoolSize"',
+                    placeholder: 'max(8, availableProcessors)',
+                    min: '1',
+                    tip: 'Size of thread pool that is in charge of processing data stream messages'
+                })
             .pc-form-grid-col-30
-                +number('Query:', model + '.queryThreadPoolSize', '"queryThreadPoolSize"', 'true', 'max(8, availableProcessors)', '1',
-                    'Size of thread pool that is in charge of processing query messages')
+                +form-field__number({
+                    label: 'Query:',
+                    model: model + '.queryThreadPoolSize',
+                    name: '"queryThreadPoolSize"',
+                    placeholder: 'max(8, availableProcessors)',
+                    min: '1',
+                    tip: 'Size of thread pool that is in charge of processing query messages'
+                })
             .pc-form-grid-col-60(ng-if-end)
                 .ignite-form-field
-                    +ignite-form-field__label('Executor configurations:', '"executorConfigurations"')
-                        +tooltip(`Custom thread pool configurations for compute tasks`)
-                    .ignite-form-field__control
-                        list-editable(
-                            ng-model=executors
-                            ng-model-options='{allowInvalid: true}'
-                            name='executorConfigurations'
-                            ui-validate=`{
-                                allNamesExist: '$ctrl.Clusters.executorConfigurations.allNamesExist($value)',
-                                allNamesUnique: '$ctrl.Clusters.executorConfigurations.allNamesUnique($value)'
-                            }`
-                        )
-                            list-editable-item-view
-                                | {{ $item.name }} / 
-                                | {{ $item.size || 'max(8, availableProcessors)'}}
+                    +form-field__label({ label: 'Executor configurations:', name: '"executorConfigurations"' })
+                        +form-field__tooltip({ title: `Custom thread pool configurations for compute tasks` })
+
+                    list-editable(
+                        ng-model=executors
+                        ng-model-options='{allowInvalid: true}'
+                        name='executorConfigurations'
+                        ui-validate=`{
+                            allNamesExist: '$ctrl.Clusters.executorConfigurations.allNamesExist($value)',
+                            allNamesUnique: '$ctrl.Clusters.executorConfigurations.allNamesUnique($value)'
+                        }`
+                    )
+                        list-editable-item-view
+                            | {{ $item.name }} /
+                            | {{ $item.size || 'max(8, availableProcessors)'}}
+
+                        list-editable-item-edit
+                            .pc-form-grid-row
+                                .pc-form-grid-col-30
+                                    +form-field__text({
+                                        label: 'Name:',
+                                        model: '$item.name',
+                                        name: '"ExecutorName"',
+                                        required: true,
+                                        placeholder: 'Input executor name',
+                                        tip: 'Thread pool name'
+                                    })(
+                                        ui-validate=`{
+                                            uniqueName: '$ctrl.Clusters.executorConfiguration.name.customValidators.uniqueName($item, ${executors})'
+                                        }`
+                                        ui-validate-watch=`"${executors}"`
+                                        ui-validate-watch-object-equality='true'
+                                        ng-model-options='{allowInvalid: true}'
+                                        ignite-form-field-input-autofocus='true'
+                                    )
+                                        +form-field__error({ error: 'uniqueName', message: 'Service with that name is already configured' })
+                                .pc-form-grid-col-30
+                                    +form-field__number({
+                                        label: 'Pool size:',
+                                        model: '$item.size',
+                                        name: '"ExecutorPoolSize"',
+                                        placeholder: 'max(8, availableProcessors)',
+                                        min: '1',
+                                        tip: 'Thread pool size'
+                                    })
 
-                            list-editable-item-edit
-                                .pc-form-grid-row
-                                    .pc-form-grid-col-30
-                                        +sane-ignite-form-field-text({
-                                            label: 'Name:',
-                                            model: '$item.name',
-                                            name: '"ExecutorName"',
-                                            required: true,
-                                            placeholder: 'Input executor name',
-                                            tip: 'Thread pool name'
-                                        })(
-                                            ui-validate=`{
-                                                uniqueName: '$ctrl.Clusters.executorConfiguration.name.customValidators.uniqueName($item, ${executors})'
-                                            }`
-                                            ui-validate-watch=`"${executors}"`
-                                            ui-validate-watch-object-equality='true'
-                                            ng-model-options='{allowInvalid: true}'
-                                            data-ignite-form-field-input-autofocus='true'
-                                        )
-                                            +form-field-feedback(null, 'uniqueName', 'Service with that name is already configured')
-                                    .pc-form-grid-col-30
-                                        +number('Pool size:', '$item.size', '"ExecutorPoolSize"', 'true', 'max(8, availableProcessors)', '1', 'Thread pool size')
+                        list-editable-no-items
+                            list-editable-add-item-button(
+                                add-item=`$edit($ctrl.Clusters.addExecutorConfiguration(${model}))`
+                                label-single='executor configuration'
+                                label-multiple='executor configurations'
+                            )
 
-                            list-editable-no-items
-                                list-editable-add-item-button(
-                                    add-item=`$edit($ctrl.Clusters.addExecutorConfiguration(${model}))`
-                                    label-single='executor configuration'
-                                    label-multiple='executor configurations'
-                                )
-                    .ignite-form-field__errors(
+                    .form-field__errors(
                         ng-messages=`pools.executorConfigurations.$error`
                         ng-show=`pools.executorConfigurations.$invalid`
                     )
-                        +form-field-feedback(_, 'allNamesExist', 'All executor configurations should have a name')
-                        +form-field-feedback(_, 'allNamesUnique', 'All executor configurations should have a unique name')
+                        +form-field__error({ error: 'allNamesExist', message: 'All executor configurations should have a name' })
+                        +form-field__error({ error: 'allNamesUnique', message: 'All executor configurations should have a unique name' })
 
         .pca-form-column-6
             +preview-xml-java(model, 'clusterPools')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/time.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/time.pug
index fa85a5d34bf00..7cfff3cec6793 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/time.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/time.pug
@@ -26,19 +26,46 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             //- Removed in ignite 2.0
             .pc-form-grid-col-30(ng-if-start='$ctrl.available(["1.0.0", "2.0.0"])')
-                +number('Samples size:', `${model}.clockSyncSamples`, '"clockSyncSamples"', 'true', '8', '0',
-                    'Number of samples used to synchronize clocks between different nodes<br/>\
-                    Clock synchronization is used for cache version assignment in CLOCK order mode')
+                +form-field__number({
+                    label: 'Samples size:',
+                    model: `${model}.clockSyncSamples`,
+                    name: '"clockSyncSamples"',
+                    placeholder: '8',
+                    min: '0',
+                    tip: 'Number of samples used to synchronize clocks between different nodes<br/>\
+                          Clock synchronization is used for cache version assignment in CLOCK order mode'
+                })
             .pc-form-grid-col-30(ng-if-end)
-                +number('Frequency:', `${model}.clockSyncFrequency`, '"clockSyncFrequency"', 'true', '120000', '0',
-                    'Frequency at which clock is synchronized between nodes, in milliseconds<br/>\
-                    Clock synchronization is used for cache version assignment in CLOCK order mode')
+                +form-field__number({
+                    label: 'Frequency:',
+                    model: `${model}.clockSyncFrequency`,
+                    name: '"clockSyncFrequency"',
+                    placeholder: '120000',
+                    min: '0',
+                    tip: 'Frequency at which clock is synchronized between nodes, in milliseconds<br/>\
+                          Clock synchronization is used for cache version assignment in CLOCK order mode'
+                })
 
             .pc-form-grid-col-30
-                +number-min-max('Port base:', `${model}.timeServerPortBase`, '"timeServerPortBase"', 'true', '31100', '0', '65535',
-                    'Time server provides clock synchronization between nodes<br/>\
-                    Base UPD port number for grid time server. Time server will be started on one of free ports in range')
+                +form-field__number({
+                    label: 'Port base:',
+                    model: `${model}.timeServerPortBase`,
+                    name: '"timeServerPortBase"',
+                    placeholder: '31100',
+                    min: '0',
+                    max: '65535',
+                    tip: 'Time server provides clock synchronization between nodes<br/>\
+                         Base UPD port number for grid time server. Time server will be started on one of free ports in range'
+                })
+
             .pc-form-grid-col-30
-                +number('Port range:', `${model}.timeServerPortRange`, '"timeServerPortRange"', 'true', '100', '1', 'Time server port range')
+                +form-field__number({
+                    label: 'Port range:',
+                    model: `${model}.timeServerPortRange`,
+                    name: '"timeServerPortRange"',
+                    placeholder: '100',
+                    min: '1',
+                    tip: 'Time server port range'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterTime')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/transactions.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/transactions.pug
index b5f80dffa806f..48c8391d77828 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/transactions.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/cluster-edit-form/templates/transactions.pug
@@ -21,45 +21,78 @@ include /app/helpers/jade/mixins
 
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Transactions
-    panel-description 
-        | Settings for transactions. 
+    panel-description
+        | Settings for transactions.
         | #[a.link-success(href="https://apacheignite.readme.io/docs/transactions" target="_blank") More info]
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-30
-                +dropdown('Concurrency:', `${model}.defaultTxConcurrency`, '"defaultTxConcurrency"', 'true', 'PESSIMISTIC',
-                    '[\
+                +form-field__dropdown({
+                    label: 'Concurrency:',
+                    model: `${model}.defaultTxConcurrency`,
+                    name: '"defaultTxConcurrency"',
+                    placeholder: 'PESSIMISTIC',
+                    options: '[\
                         {value: "OPTIMISTIC", label: "OPTIMISTIC"},\
                         {value: "PESSIMISTIC", label: "PESSIMISTIC"}\
                     ]',
-                    'Cache transaction concurrency to use when one is not explicitly specified\
-                    <ul>\
-                        <li>OPTIMISTIC - All cache operations are not distributed to other nodes until commit is called</li>\
-                        <li>PESSIMISTIC - A lock is acquired on all cache operations with exception of read operations in READ_COMMITTED mode</li>\
-                    </ul>')
+                    tip: 'Cache transaction concurrency to use when one is not explicitly specified\
+                        <ul>\
+                            <li>OPTIMISTIC - All cache operations are not distributed to other nodes until commit is called</li>\
+                            <li>PESSIMISTIC - A lock is acquired on all cache operations with exception of read operations in READ_COMMITTED mode</li>\
+                        </ul>'
+                })
             .pc-form-grid-col-30
-                +dropdown('Isolation:', `${model}.defaultTxIsolation`, '"defaultTxIsolation"', 'true', 'REPEATABLE_READ',
-                    '[\
+                +form-field__dropdown({
+                    label: 'Isolation:',
+                    model: `${model}.defaultTxIsolation`,
+                    name: '"defaultTxIsolation"',
+                    placeholder: 'REPEATABLE_READ',
+                    options: '[\
                         {value: "READ_COMMITTED", label: "READ_COMMITTED"},\
                         {value: "REPEATABLE_READ", label: "REPEATABLE_READ"},\
                         {value: "SERIALIZABLE", label: "SERIALIZABLE"}\
                     ]',
-                    'Default transaction isolation\
-                    <ul>\
-                        <li>READ_COMMITTED - Always a committed value will be provided for read operations</li>\
-                        <li>REPEATABLE_READ - If a value was read once within transaction, then all consecutive reads will provide the same in-transaction value</li>\
-                        <li>SERIALIZABLE - All transactions occur in a completely isolated fashion, as if all transactions in the system had executed serially, one after the other.</li>\
-                    </ul>')
+                    tip: 'Default transaction isolation\
+                        <ul>\
+                            <li>READ_COMMITTED - Always a committed value will be provided for read operations</li>\
+                            <li>REPEATABLE_READ - If a value was read once within transaction, then all consecutive reads will provide the same in-transaction value</li>\
+                            <li>SERIALIZABLE - All transactions occur in a completely isolated fashion, as if all transactions in the system had executed serially, one after the other.</li>\
+                        </ul>'
+                })
             .pc-form-grid-col-60
-                +number('Default timeout:', `${model}.defaultTxTimeout`, '"defaultTxTimeout"', 'true', '0', '0', 'Default transaction timeout')
+                +form-field__number({
+                    label: 'Default timeout:',
+                    model: `${model}.defaultTxTimeout`,
+                    name: '"defaultTxTimeout"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Default transaction timeout'
+                })
             .pc-form-grid-col-30
-                +number('Pessimistic log cleanup delay:', `${model}.pessimisticTxLogLinger`, '"pessimisticTxLogLinger"', 'true', '10000', '0',
-                    'Delay, in milliseconds, after which pessimistic recovery entries will be cleaned up for failed node')
+                +form-field__number({
+                    label: 'Pessimistic log cleanup delay:',
+                    model: `${model}.pessimisticTxLogLinger`,
+                    name: '"pessimisticTxLogLinger"',
+                    placeholder: '10000',
+                    min: '0',
+                    tip: 'Delay, in milliseconds, after which pessimistic recovery entries will be cleaned up for failed node'
+                })
             .pc-form-grid-col-30
-                +number('Pessimistic log size:', `${model}.pessimisticTxLogSize`, '"pessimisticTxLogSize"', 'true', '0', '0',
-                    'Size of pessimistic transactions log stored on node in order to recover transaction commit if originating node has left grid before it has sent all messages to transaction nodes')
+                +form-field__number({
+                    label: 'Pessimistic log size:',
+                    model: `${model}.pessimisticTxLogSize`,
+                    name: '"pessimisticTxLogSize"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Size of pessimistic transactions log stored on node in order to recover transaction commit if originating node has left grid before it has sent all messages to transaction nodes'
+                })
             .pc-form-grid-col-60
-                +java-class('Manager factory:', `${model}.txManagerFactory`, '"txManagerFactory"', 'true', 'false',
-                    'Class name of transaction manager factory for integration with JEE app servers')
+                +form-field__java-class({
+                    label: 'Manager factory:',
+                    model: `${model}.txManagerFactory`,
+                    name: '"txManagerFactory"',
+                    tip: 'Class name of transaction manager factory for integration with JEE app servers'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'clusterTransactions')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/dual.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/dual.pug
index 67a37adacb8f4..67839e812dcae 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/dual.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/dual.pug
@@ -31,12 +31,28 @@ panel-collapsible(
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6
             .settings-row
-                +number('Maximum pending puts size:', `${model}.dualModeMaxPendingPutsSize`, '"dualModeMaxPendingPutsSize"', 'true', '0', 'Number.MIN_SAFE_INTEGER',
-                    'Maximum amount of pending data read from the secondary file system and waiting to be written to data cache<br/>\
-                    Zero or negative value stands for unlimited size')
+                +form-field__number({
+                    label: 'Maximum pending puts size:',
+                    model: `${model}.dualModeMaxPendingPutsSize`,
+                    name: '"dualModeMaxPendingPutsSize"',
+                    placeholder: '0',
+                    min: 'Number.MIN_SAFE_INTEGER',
+                    tip: 'Maximum amount of pending data read from the secondary file system and waiting to be written to data cache<br/>\
+                         Zero or negative value stands for unlimited size'
+                })
             .settings-row
-                +java-class('Put executor service:', `${model}.dualModePutExecutorService`, '"dualModePutExecutorService"', 'true', 'false', 'DUAL mode put operation executor service')
+                +form-field__java-class({
+                    label: 'Put executor service:',
+                    model: `${model}.dualModePutExecutorService`,
+                    name: '"dualModePutExecutorService"',
+                    tip: 'DUAL mode put operation executor service'
+                })
             .settings-row
-                +checkbox('Put executor service shutdown', `${model}.dualModePutExecutorServiceShutdown`, '"dualModePutExecutorServiceShutdown"', 'DUAL mode put operation executor service shutdown flag')
+                +form-field__checkbox({
+                    label: 'Put executor service shutdown',
+                    model: `${model}.dualModePutExecutorServiceShutdown`,
+                    name: '"dualModePutExecutorServiceShutdown"',
+                    tip: 'DUAL mode put operation executor service shutdown flag'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'igfsDualMode')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/fragmentizer.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/fragmentizer.pug
index a8194c28f0f39..d1fa76ab4efe8 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/fragmentizer.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/fragmentizer.pug
@@ -26,12 +26,41 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
             -var enabled = `${model}.fragmentizerEnabled`
 
             .pc-form-grid-col-60
-                +checkbox('Enabled', enabled, '"fragmentizerEnabled"', 'Fragmentizer enabled flag')
+                +form-field__checkbox({
+                    label: 'Enabled',
+                    model: enabled,
+                    name: '"fragmentizerEnabled"',
+                    tip: 'Fragmentizer enabled flag'
+                })
             .pc-form-grid-col-30
-                +number('Concurrent files:', `${model}.fragmentizerConcurrentFiles`, '"fragmentizerConcurrentFiles"', enabled, '0', '0', 'Number of files to process concurrently by fragmentizer')
+                +form-field__number({
+                    label: 'Concurrent files:',
+                    model: `${model}.fragmentizerConcurrentFiles`,
+                    name: '"fragmentizerConcurrentFiles"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Number of files to process concurrently by fragmentizer'
+                })
             .pc-form-grid-col-30
-                +number('Throttling block length:', `${model}.fragmentizerThrottlingBlockLength`, '"fragmentizerThrottlingBlockLength"', enabled, '16777216', '1', 'Length of file chunk to transmit before throttling is delayed')
+                +form-field__number({
+                    label: 'Throttling block length:',
+                    model: `${model}.fragmentizerThrottlingBlockLength`,
+                    name: '"fragmentizerThrottlingBlockLength"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '16777216',
+                    min: '1',
+                    tip: 'Length of file chunk to transmit before throttling is delayed'
+                })
             .pc-form-grid-col-60
-                +number('Throttling delay:', `${model}.fragmentizerThrottlingDelay`, '"fragmentizerThrottlingDelay"', enabled, '200', '0', 'Delay in milliseconds for which fragmentizer is paused')
+                +form-field__number({
+                    label: 'Throttling delay:',
+                    model: `${model}.fragmentizerThrottlingDelay`,
+                    name: '"fragmentizerThrottlingDelay"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '200',
+                    min: '0',
+                    tip: 'Delay in milliseconds for which fragmentizer is paused'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'igfsFragmentizer')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/general.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/general.pug
index b9eb8fc913c73..777c1233fd9b2 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/general.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/general.pug
@@ -22,12 +22,12 @@ include /app/helpers/jade/mixins
 panel-collapsible(opened=`::true` ng-form=form)
     panel-title General
     panel-description
-        | General IGFS configuration. 
+        | General IGFS configuration.
         a.link-success(href="https://apacheignite-fs.readme.io/docs/in-memory-file-system" target="_blank") More info
     panel-content.pca-form-row
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +sane-ignite-form-field-text({
+                +form-field__text({
                     label: 'Name:',
                     model: `${model}.name`,
                     name: '"igfsName"',
@@ -38,9 +38,9 @@ panel-collapsible(opened=`::true` ng-form=form)
                     ignite-unique-property='name'
                     ignite-unique-skip=`["_id", ${model}]`
                 )
-                    +unique-feedback(`${model}.name`, 'IGFS name should be unique.')
+                    +form-field__error({ error: 'igniteUnique', message: 'IGFS name should be unique.' })
             .pc-form-grid-col-30
-                +sane-ignite-form-field-dropdown({
+                +form-field__dropdown({
                     label: 'IGFS mode:',
                     model: `${model}.defaultMode`,
                     name: '"defaultMode"',
@@ -57,7 +57,7 @@ panel-collapsible(opened=`::true` ng-form=form)
                     `
                 })
             .pc-form-grid-col-30
-                +sane-ignite-form-field-number({
+                +form-field__number({
                     label: 'Group size:',
                     model: `${model}.affinnityGroupSize`,
                     name: '"affinnityGroupSize"',
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/ipc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/ipc.pug
index ef024b4814869..efae60cd615e7 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/ipc.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/ipc.pug
@@ -28,28 +28,77 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
             -var enabled = `${model}.ipcEndpointEnabled`
 
             .pc-form-grid-col-60
-                +checkbox('Enabled', enabled, '"ipcEndpointEnabled"', 'IPC endpoint enabled flag')
+                +form-field__checkbox({
+                    label: 'Enabled',
+                    model: enabled,
+                    name: '"ipcEndpointEnabled"',
+                    tip: 'IPC endpoint enabled flag'
+                })
             .pc-form-grid-col-60
-                +dropdown('Type:', `${ipcEndpointConfiguration}.type`, '"ipcEndpointConfigurationType"', enabled, 'TCP',
-                    '[\
+                +form-field__dropdown({
+                    label: 'Type:',
+                    model: `${ipcEndpointConfiguration}.type`,
+                    name: '"ipcEndpointConfigurationType"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'TCP',
+                    options: '[\
                         {value: "SHMEM", label: "SHMEM"},\
                         {value: "TCP", label: "TCP"}\
                     ]',
-                    'IPC endpoint type\
-                    <ul>\
-                        <li>SHMEM - shared memory endpoint</li>\
-                        <li>TCP - TCP endpoint</li>\
-                    </ul>')
+                    tip: 'IPC endpoint type\
+                        <ul>\
+                            <li>SHMEM - shared memory endpoint</li>\
+                            <li>TCP - TCP endpoint</li>\
+                        </ul>'
+                })
             .pc-form-grid-col-30
-                +text-ip-address('Host:', `${ipcEndpointConfiguration}.host`, '"ipcEndpointConfigurationHost"', enabled, '127.0.0.1', 'Host endpoint is bound to')
+                +form-field__ip-address({
+                    label: 'Host:',
+                    model: `${ipcEndpointConfiguration}.host`,
+                    name: '"ipcEndpointConfigurationHost"',
+                    enabled: enabled,
+                    placeholder: '127.0.0.1',
+                    tip: 'Host endpoint is bound to'
+                })
             .pc-form-grid-col-30
-                +number-min-max('Port:', `${ipcEndpointConfiguration}.port`, '"ipcEndpointConfigurationPort"', enabled, '10500', '1', '65535', 'Port endpoint is bound to')
+                +form-field__number({
+                    label: 'Port:',
+                    model: `${ipcEndpointConfiguration}.port`,
+                    name: '"ipcEndpointConfigurationPort"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '10500',
+                    min: '1',
+                    max: '65535',
+                    tip: 'Port endpoint is bound to'
+                })
             .pc-form-grid-col-30
-                +number('Memory size:', `${ipcEndpointConfiguration}.memorySize`, '"ipcEndpointConfigurationMemorySize"', enabled, '262144', '1', 'Shared memory size in bytes allocated for endpoint communication')
+                +form-field__number({
+                    label: 'Memory size:',
+                    model: `${ipcEndpointConfiguration}.memorySize`,
+                    name: '"ipcEndpointConfigurationMemorySize"',
+                    disabled: `!(${enabled})`,
+                    placeholder: '262144',
+                    min: '1',
+                    tip: 'Shared memory size in bytes allocated for endpoint communication'
+                })
             .pc-form-grid-col-30
-                +number('Thread count:', `${ipcEndpointConfiguration}.threadCount`, '"ipcEndpointConfigurationThreadCount"', enabled, 'availableProcessors', '1',
-                    'Number of threads used by this endpoint to process incoming requests')
+                +form-field__number({
+                    label: 'Thread count:',
+                    model: `${ipcEndpointConfiguration}.threadCount`,
+                    name: '"ipcEndpointConfigurationThreadCount"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'availableProcessors',
+                    min: '1',
+                    tip: 'Number of threads used by this endpoint to process incoming requests'
+                })
             .pc-form-grid-col-60
-                +text-enabled('Token directory:', `${ipcEndpointConfiguration}.tokenDirectoryPath`, '"ipcEndpointConfigurationTokenDirectoryPath"', enabled, 'false', 'ipc/shmem', 'Directory where shared memory tokens are stored')
+                +form-field__text({
+                    label: 'Token directory:',
+                    model: `${ipcEndpointConfiguration}.tokenDirectoryPath`,
+                    name: '"ipcEndpointConfigurationTokenDirectoryPath"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'ipc/shmem',
+                    tip: 'Directory where shared memory tokens are stored'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'igfsIPC')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/misc.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/misc.pug
index 9a39b3a39f4d2..cf68e723ed15d 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/misc.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/misc.pug
@@ -26,83 +26,181 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +number('Block size:', `${model}.blockSize`, '"blockSize"', 'true', '65536', '0', 'File data block size in bytes')
+                +form-field__number({
+                    label: 'Block size:',
+                    model: `${model}.blockSize`,
+                    name: '"blockSize"',
+                    placeholder: '65536',
+                    min: '0',
+                    tip: 'File data block size in bytes'
+                })
 
             //- Since ignite 2.0
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.0.0")')
-                +number('Buffer size:', `${model}.streamBufferSize`, '"streamBufferSize"', 'true', '65536', '0', 'Read/write buffer size for IGFS stream operations in bytes')
+                +form-field__number({
+                    label: 'Buffer size:',
+                    model: `${model}.streamBufferSize`,
+                    name: '"streamBufferSize"',
+                    placeholder: '65536',
+                    min: '0',
+                    tip: 'Read/write buffer size for IGFS stream operations in bytes'
+                })
 
             //- Removed in ignite 2.0
             .pc-form-grid-col-60(ng-if-start='$ctrl.available(["1.0.0", "2.0.0"])')
-                +number('Stream buffer size:', `${model}.streamBufferSize`, '"streamBufferSize"', 'true', '65536', '0', 'Read/write buffer size for IGFS stream operations in bytes')
+                +form-field__number({
+                    label: 'Stream buffer size:',
+                    model: `${model}.streamBufferSize`,
+                    name: '"streamBufferSize"',
+                    placeholder: '65536',
+                    min: '0',
+                    tip: 'Read/write buffer size for IGFS stream operations in bytes'
+                })
             .pc-form-grid-col-60(ng-if-end)
-                +number('Maximum space size:', `${model}.maxSpaceSize`, '"maxSpaceSize"', 'true', '0', '0', 'Maximum space available for data cache to store file system entries')
+                +form-field__number({
+                    label: 'Maximum space size:',
+                    model: `${model}.maxSpaceSize`,
+                    name: '"maxSpaceSize"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Maximum space available for data cache to store file system entries'
+                })
 
             .pc-form-grid-col-30
-                +number('Maximum task range length:', `${model}.maximumTaskRangeLength`, '"maximumTaskRangeLength"', 'true', '0', '0', 'Maximum default range size of a file being split during IGFS task execution')
+                +form-field__number({
+                    label: 'Maximum task range length:',
+                    model: `${model}.maximumTaskRangeLength`,
+                    name: '"maximumTaskRangeLength"',
+                    placeholder: '0',
+                    min: '0',
+                    tip: 'Maximum default range size of a file being split during IGFS task execution'
+                })
             .pc-form-grid-col-30
-                +number-min-max('Management port:', `${model}.managementPort`, '"managementPort"', 'true', '11400', '0', '65535', 'Port number for management endpoint')
+                +form-field__number({
+                    label: 'Management port:',
+                    model: `${model}.managementPort`,
+                    name: '"managementPort"',
+                    placeholder: '11400',
+                    min: '0',
+                    max: '65535',
+                    tip: 'Port number for management endpoint'
+                })
             .pc-form-grid-col-30
-                +number('Per node batch size:', `${model}.perNodeBatchSize`, '"perNodeBatchSize"', 'true', '100', '0', 'Number of file blocks collected on local node before sending batch to remote node')
+                +form-field__number({
+                    label: 'Per node batch size:',
+                    model: `${model}.perNodeBatchSize`,
+                    name: '"perNodeBatchSize"',
+                    placeholder: '100',
+                    min: '0',
+                    tip: 'Number of file blocks collected on local node before sending batch to remote node'
+                })
             .pc-form-grid-col-30
-                +number('Per node parallel batch count:', `${model}.perNodeParallelBatchCount`, '"perNodeParallelBatchCount"', 'true', '8', '0', 'Number of file block batches that can be concurrently sent to remote node')
+                +form-field__number({
+                    label: 'Per node parallel batch count:',
+                    model: `${model}.perNodeParallelBatchCount`,
+                    name: '"perNodeParallelBatchCount"',
+                    placeholder: '8',
+                    min: '0',
+                    tip: 'Number of file block batches that can be concurrently sent to remote node'
+                })
             .pc-form-grid-col-60
-                +number('Prefetch blocks:', `${model}.prefetchBlocks`, '"prefetchBlocks"', 'true', '0', '0', 'Number of pre-fetched blocks if specific file chunk is requested')
+                +form-field__number({
+                    label: 'Prefetch blocks:',
+                    model: `${model}.prefetchBlocks`,
+                    name: '"prefetchBlocks"',
+                    placeholder: '8',
+                    min: '0',
+                    tip: 'Number of pre-fetched blocks if specific file chunk is requested'
+                })
             .pc-form-grid-col-60
-                +number('Sequential reads before prefetch:', `${model}.sequentialReadsBeforePrefetch`, '"sequentialReadsBeforePrefetch"', 'true', '0', '0', 'Amount of sequential block reads before prefetch is triggered')
+                +form-field__number({
+                    label: 'Sequential reads before prefetch:',
+                    model: `${model}.sequentialReadsBeforePrefetch`,
+                    name: '"sequentialReadsBeforePrefetch"',
+                    placeholder: '8',
+                    min: '0',
+                    tip: 'Amount of sequential block reads before prefetch is triggered'
+                })
 
             //- Removed in ignite 2.0
             .pc-form-grid-col-60(ng-if='$ctrl.available(["1.0.0", "2.0.0"])')
-                +number('Trash purge timeout:', `${model}.trashPurgeTimeout`, '"trashPurgeTimeout"', 'true', '1000', '0', 'Maximum timeout awaiting for trash purging in case data cache oversize is detected')
+                +form-field__number({
+                    label: 'Trash purge timeout:',
+                    model: `${model}.trashPurgeTimeout`,
+                    name: '"trashPurgeTimeout"',
+                    placeholder: '1000',
+                    min: '0',
+                    tip: 'Maximum timeout awaiting for trash purging in case data cache oversize is detected'
+                })
 
             .pc-form-grid-col-60
-                +checkbox('Colocate metadata', `${model}.colocateMetadata`, '"colocateMetadata"', 'Whether to co-locate metadata on a single node')
+                +form-field__checkbox({
+                    label: 'Colocate metadata',
+                    model: `${model}.colocateMetadata`,
+                    name: '"colocateMetadata"',
+                    tip: 'Whether to co-locate metadata on a single node'
+                })
             .pc-form-grid-col-60
-                +checkbox('Relaxed consistency', `${model}.relaxedConsistency`, '"relaxedConsistency"',
-                    'If value of this flag is <b>true</b>, IGFS will skip expensive consistency checks<br/>\
-                    It is recommended to set this flag to <b>false</b> if your application has conflicting\
-                    operations, or you do not know how exactly users will use your system')
+                +form-field__checkbox({
+                    label: 'Relaxed consistency',
+                    model: `${model}.relaxedConsistency`,
+                    name: '"relaxedConsistency"',
+                    tip: 'If value of this flag is <b>true</b>, IGFS will skip expensive consistency checks<br/>\
+                         It is recommended to set this flag to <b>false</b> if your application has conflicting\
+                         operations, or you do not know how exactly users will use your system'
+                })
 
             //- Since ignite 2.0
             .pc-form-grid-col-60(ng-if='$ctrl.available("2.0.0")')
-                +checkbox('Update file length on flush', model + '.updateFileLengthOnFlush', '"updateFileLengthOnFlush"', 'Update file length on flush flag')
+                +form-field__checkbox({
+                    label: 'Update file length on flush',
+                    model: model + '.updateFileLengthOnFlush',
+                    name: '"updateFileLengthOnFlush"',
+                    tip: 'Update file length on flush flag'
+                })
 
             .pc-form-grid-col-60
                 mixin igfs-misc-path-modes
                     .ignite-form-field
-                        +ignite-form-field__label('Path modes:', '"pathModes"')
-                            +tooltip(`Map of path prefixes to IGFS modes used for them`)
-                        .ignite-form-field__control
-                            -let items = pathModes
-
-                            list-editable(ng-model=items)
-                                list-editable-item-view
-                                    | {{ $item.path + " [" + $item.mode + "]"}}
-
-                                list-editable-item-edit
-                                    - form = '$parent.form'
-
-                                    .pc-form-grid-row
-                                        .pc-form-grid-col-30
-                                            +ignite-form-field-text('Path:', '$item.path', '"path"', false, true, 'Enter path')(ignite-auto-focus)
-                                        .pc-form-grid-col-30
-                                            +sane-ignite-form-field-dropdown({
-                                                label: 'Mode:',
-                                                model: `$item.mode`,
-                                                name: '"mode"',
-                                                required: true,
-                                                placeholder: 'Choose igfs mode',
-                                                options: '{{::$ctrl.IGFSs.defaultMode.values}}'
-                                            })(
-                                                ng-model-options='{allowInvalid: true}'
-                                            )
-
-                                list-editable-no-items
-                                    list-editable-add-item-button(
-                                        add-item=`$editLast((${items} = ${items} || []).push({}))`
-                                        label-single='path mode'
-                                        label-multiple='path modes'
-                                    )
+                        +form-field__label({ label: 'Path modes:', name: '"pathModes"' })
+                            +form-field__tooltip({ title: `Map of path prefixes to IGFS modes used for them` })
+
+                        -let items = pathModes
+
+                        list-editable(ng-model=items)
+                            list-editable-item-view
+                                | {{ $item.path + " [" + $item.mode + "]"}}
+
+                            list-editable-item-edit
+                                - form = '$parent.form'
+
+                                .pc-form-grid-row
+                                    .pc-form-grid-col-30
+                                        +form-field__text({
+                                            label: 'Path:',
+                                            model: '$item.path',
+                                            name: '"path"',
+                                            required: true,
+                                            placeholder: 'Enter path'
+                                        })(ignite-auto-focus)
+                                    .pc-form-grid-col-30
+                                        +form-field__dropdown({
+                                            label: 'Mode:',
+                                            model: `$item.mode`,
+                                            name: '"mode"',
+                                            required: true,
+                                            placeholder: 'Choose igfs mode',
+                                            options: '{{::$ctrl.IGFSs.defaultMode.values}}'
+                                        })(
+                                            ng-model-options='{allowInvalid: true}'
+                                        )
+
+                            list-editable-no-items
+                                list-editable-add-item-button(
+                                    add-item=`$editLast((${items} = ${items} || []).push({}))`
+                                    label-single='path mode'
+                                    label-multiple='path modes'
+                                )
 
                 +igfs-misc-path-modes
 
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/secondary.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/secondary.pug
index 92c82109d727c..4d779f13191bb 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/secondary.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/igfs-edit-form/templates/secondary.pug
@@ -22,7 +22,7 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title Secondary file system
     panel-description
-        | Secondary file system is provided for pass-through, write-through, and read-through purposes. 
+        | Secondary file system is provided for pass-through, write-through, and read-through purposes.
         a.link-success(href="https://apacheignite-fs.readme.io/docs/secondary-file-system" target="_blank") More info
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
@@ -30,7 +30,7 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
             -var secondaryFileSystem = `${model}.secondaryFileSystem`
 
             .pc-form-grid-col-60
-                +sane-form-field-checkbox({
+                +form-field__checkbox({
                     label: 'Enabled',
                     name: '"secondaryFileSystemEnabled"',
                     model: enabled
@@ -43,13 +43,34 @@ panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
                     ui-validate-watch-collection=`"[${model}.defaultMode, ${model}.pathModes]"`
                     ui-validate-watch-object-equality='true'
                 )
-                    +form-field-feedback(null, 'requiredWhenIGFSProxyMode', 'Secondary file system should be configured for "PROXY" IGFS mode')
-                    +form-field-feedback(null, 'requiredWhenPathModeProxyMode', 'Secondary file system should be configured for "PROXY" path mode')
+                    +form-field__error({ error: 'requiredWhenIGFSProxyMode', message: 'Secondary file system should be configured for "PROXY" IGFS mode' })
+                    +form-field__error({ error: 'requiredWhenPathModeProxyMode', message: 'Secondary file system should be configured for "PROXY" path mode' })
             .pc-form-grid-col-60
-                +text-enabled('URI:', `${secondaryFileSystem}.uri`, '"hadoopURI"', enabled, 'false', 'hdfs://[namenodehost]:[port]/[path]', 'URI of file system')
+                +form-field__text({
+                    label: 'URI:',
+                    model: `${secondaryFileSystem}.uri`,
+                    name: '"hadoopURI"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'hdfs://[namenodehost]:[port]/[path]',
+                    tip: 'URI of file system'
+                })
             .pc-form-grid-col-60
-                +text-enabled('Config path:', `${secondaryFileSystem}.cfgPath`, '"cfgPath"', enabled, 'false', 'Path to additional config', 'Additional path to Hadoop configuration')
+                +form-field__text({
+                    label: 'Config path:',
+                    model: `${secondaryFileSystem}.cfgPath`,
+                    name: '"cfgPath"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'Path to additional config',
+                    tip: 'Additional path to Hadoop configuration'
+                })
             .pc-form-grid-col-60
-                +text-enabled('User name:', `${secondaryFileSystem}.userName`, '"userName"', enabled, 'false', 'Input user name', 'User name')
+                +form-field__text({
+                    label: 'User name:',
+                    model: `${secondaryFileSystem}.userName`,
+                    name: '"userName"',
+                    disabled: `!(${enabled})`,
+                    placeholder: 'Input user name',
+                    tip: 'User name'
+                })
         .pca-form-column-6
             +preview-xml-java(model, 'igfsSecondFS')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js
index b7d4ebe3b513d..2b53d97b3c658 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/controller.js
@@ -31,7 +31,8 @@ export default class ModelEditFormController {
     /** @type {ng.ICompiledExpression} */
     onSave;
 
-    static $inject = ['ModalImportModels', 'IgniteErrorPopover', 'IgniteLegacyUtils', Confirm.name, 'ConfigChangesGuard', IgniteVersion.name, '$scope', Models.name, 'IgniteFormUtils'];
+    static $inject = ['ModalImportModels', 'IgniteErrorPopover', 'IgniteLegacyUtils', 'Confirm', 'ConfigChangesGuard', 'IgniteVersion', '$scope', 'Models', 'IgniteFormUtils'];
+
     /**
      * @param {ModalImportModels} ModalImportModels
      * @param {Confirm} Confirm
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/general.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/general.pug
index a6c8194fcc66f..589760eeed40b 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/general.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/general.pug
@@ -23,15 +23,20 @@ include /app/helpers/jade/mixins
 panel-collapsible(opened=`::true` ng-form=form)
     panel-title General
     panel-description
-        | Domain model properties common for Query and Store. 
-        a.link-success(href="https://apacheignite.readme.io/docs/cache-queries" target="_blank") More info about query configuration. 
+        | Domain model properties common for Query and Store.
+        a.link-success(href="https://apacheignite.readme.io/docs/cache-queries" target="_blank") More info about query configuration.
         a.link-success(href="https://apacheignite.readme.io/docs/3rd-party-store" target="_blank") More info about store.
     panel-content.pca-form-row
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-60
-                +checkbox('Generate POJO classes', generatePojo, '"generatePojo"', 'If selected then POJO classes will be generated from database tables')
+                +form-field__checkbox({
+                    label: 'Generate POJO classes',
+                    model: generatePojo,
+                    name: '"generatePojo"',
+                    tip: 'If selected then POJO classes will be generated from database tables'
+                })
             .pc-form-grid-col-30
-                +sane-ignite-form-field-dropdown({
+                +form-field__dropdown({
                     label: 'Caches:',
                     model: `${model}.caches`,
                     name: '"caches"',
@@ -42,16 +47,42 @@ panel-collapsible(opened=`::true` ng-form=form)
                     tip: 'Select caches to describe types in cache'
                 })
             .pc-form-grid-col-30
-                +dropdown-required('Query metadata:', `${model}.queryMetadata`, '"queryMetadata"', 'true', 'true', '', '::$ctrl.Models.queryMetadata.values',
-                    'Query metadata configured with:\
-                    <ul>\
-                        <li>Java annotations like @QuerySqlField</li>\
-                        <li>Configuration via QueryEntity class</li>\
-                    </ul>')
+                +form-field__dropdown({
+                    label: 'Query metadata:',
+                    model: `${model}.queryMetadata`,
+                    name: '"queryMetadata"',
+                    required: 'true',
+                    placeholder: '',
+                    options: '::$ctrl.Models.queryMetadata.values',
+                    tip: 'Query metadata configured with:\
+                          <ul>\
+                            <li>Java annotations like @QuerySqlField</li>\
+                            <li>Configuration via QueryEntity class</li>\
+                          </ul>'
+                })
+
             .pc-form-grid-col-60
-                +java-class-typeahead('Key type:', `${model}.keyType`, '"keyType"', '$ctrl.javaBuiltInClassesBase', 'true', 'true', '{{ ' + generatePojo + ' ? "Full class name for Key" : "Key type name" }}', 'Key class used to store key in cache', generatePojo)
+                +form-field__java-class--typeahead({
+                    label: 'Key type:',
+                    model: `${model}.keyType`,
+                    name: '"keyType"',
+                    options: '$ctrl.javaBuiltInClassesBase',
+                    required: 'true',
+                    placeholder: '{{ ' + generatePojo + ' ? "Full class name for Key" : "Key type name" }}',
+                    tip: 'Key class used to store key in cache',
+                    validationActive: generatePojo
+                })
             .pc-form-grid-col-60
-                +java-class-autofocus-placholder('Value type:', `${model}.valueType`, '"valueType"', 'true', 'true', 'false', '{{ ' + generatePojo +' ? "Enter fully qualified class name" : "Value type name" }}', 'Value class used to store value in cache', generatePojo)
+                +form-field__java-class({
+                    label: 'Value type:',
+                    model: `${model}.valueType`,
+                    name: '"valueType"',
+                    placeholder: '{{ ' + generatePojo +' ? "Enter fully qualified class name" : "Value type name" }}',
+                    tip: 'Value class used to store value in cache',
+                    validationActive: generatePojo
+                })(
+                    ignite-form-field-input-autofocus=autofocus
+                )
 
         .pca-form-column-6
             +preview-xml-java(model, 'domainModelGeneral')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/query.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/query.pug
index ed91ec421121c..ec9f1d4896f6d 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/query.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/query.pug
@@ -26,7 +26,7 @@ include /app/helpers/jade/mixins
 panel-collapsible(ng-form=form opened=`!!${model}.queryMetadata`)
     panel-title#query-title Domain model for SQL query
     panel-description
-        | Domain model properties for fields queries. 
+        | Domain model properties for fields queries.
         a.link-success(href='https://apacheignite.readme.io/docs/cache-queries' target='_blank') More info
     panel-content.pca-form-row
         .pca-form-column-6.pc-form-grid-row
@@ -37,59 +37,87 @@ panel-collapsible(ng-form=form opened=`!!${model}.queryMetadata`)
                 label Not available for annotated types
 
             .pc-form-grid-col-60(ng-if-start=`${model}.queryMetadata === 'Configuration'`)
-                +text('Table name:', `${model}.tableName`, '"tableName"', 'false', 'Enter table name')
+                +form-field__text({
+                    label: 'Table name:',
+                    model: `${model}.tableName`,
+                    name: '"tableName"',
+                    placeholder: 'Enter table name'
+                })
 
             .pc-form-grid-col-30(ng-if-start='$ctrl.available("2.0.0")')
-                +text('Key field name:', `${model}.keyFieldName`, '"keyFieldName"', 'false', 'Enter key field name',
-                    'Key name.<br/>' +
-                    'Can be used in field list to denote the key as a whole')
+                +form-field__text({
+                    label: 'Key field name:',
+                    model: `${model}.keyFieldName`,
+                    name: '"keyFieldName"',
+                    placeholder: 'Enter key field name',
+                    tip: 'Key name.<br/>' +
+                        'Can be used in field list to denote the key as a whole'
+                })
             .pc-form-grid-col-30(ng-if-end)
-                +text('Value field name:', `${model}.valueFieldName`, '"valueFieldName"', 'false', 'Enter value field name',
-                    'Value name.<br/>' +
-                    'Can be used in field list to denote the entire value')
+                +form-field__text({
+                    label: 'Value field name:',
+                    model: `${model}.valueFieldName`,
+                    name: '"valueFieldName"',
+                    placeholder: 'Enter value field name',
+                    tip: 'Value name.<br/>' +
+                        'Can be used in field list to denote the entire value'
+                })
 
             .pc-form-grid-col-60
                 mixin domains-query-fields
                     .ignite-form-field
-                        +ignite-form-field__label('Fields:', '"fields"')
-                            +tooltip(`Collection of name-to-type mappings to be queried, in addition to indexed fields`)
-                        .ignite-form-field__control
-                            -let items = queryFields
-                            list-editable(
-                                ng-model=items
-                                name='queryFields'
-                                ng-change=`$ctrl.onQueryFieldsChange(${model})`
-                            )
-                                list-editable-item-view
-                                    | {{ $item.name}} / {{ $item.className}}
-
-                                list-editable-item-edit
-                                    - form = '$parent.form'
-                                    .pc-form-grid-row
-                                        .pc-form-grid-col-30(divider='/')
-                                            +ignite-form-field-text('Field name:', '$item.name', '"name"', false, true, 'Enter field name')(
-                                                data-ignite-unique=items
-                                                data-ignite-unique-property='name'
-                                                ignite-auto-focus
-                                            )
-                                                +unique-feedback('"name"', 'Property with such name already exists!')
-                                        .pc-form-grid-col-30
-                                            +java-class-typeahead('Field full class name:', `$item.className`, '"className"', '$ctrl.queryFieldTypes', true, true, 'Enter field full class name')(
-                                                ng-model-options='{allowInvalid: true}'
-                                                extra-valid-java-identifiers='$ctrl.queryFieldTypes'
-                                            )
-
-                                list-editable-no-items
-                                    list-editable-add-item-button(
-                                        add-item=`$editLast((${items} = ${items} || []).push({}))`
-                                        label-single='field to query'
-                                        label-multiple='fields'
-                                    )
+                        +form-field__label({ label: 'Fields:', name: '"fields"' })
+                            +form-field__tooltip({ title: `Collection of name-to-type mappings to be queried, in addition to indexed fields` })
+
+                        -let items = queryFields
+                        list-editable(
+                            ng-model=items
+                            name='queryFields'
+                            ng-change=`$ctrl.onQueryFieldsChange(${model})`
+                        )
+                            list-editable-item-view
+                                | {{ $item.name}} / {{ $item.className}}
+
+                            list-editable-item-edit
+                                - form = '$parent.form'
+                                .pc-form-grid-row
+                                    .pc-form-grid-col-30(divider='/')
+                                        +form-field__text({
+                                            label: 'Field name:',
+                                            model: '$item.name',
+                                            name: '"name"',
+                                            required: true,
+                                            placeholder: 'Enter field name'
+                                        })(
+                                            ignite-unique=items
+                                            ignite-unique-property='name'
+                                            ignite-auto-focus
+                                        )
+                                            +form-field__error({ error: 'igniteUnique', message: 'Property with such name already exists!' })
+                                    .pc-form-grid-col-30
+                                        +form-field__java-class--typeahead({
+                                            label: 'Field full class name:',
+                                            model: `$item.className`,
+                                            name: '"className"',
+                                            options: '$ctrl.queryFieldTypes',
+                                            required: 'true',
+                                            placeholder: 'Enter field full class name'
+                                        })(
+                                            ng-model-options='{allowInvalid: true}'
+                                            extra-valid-java-identifiers='$ctrl.queryFieldTypes'
+                                        )
+
+                            list-editable-no-items
+                                list-editable-add-item-button(
+                                    add-item=`$editLast((${items} = ${items} || []).push({}))`
+                                    label-single='field to query'
+                                    label-multiple='fields'
+                                )
 
                 +domains-query-fields
 
             .pc-form-grid-col-60
-                +sane-ignite-form-field-dropdown({
+                +form-field__dropdown({
                     label: 'Key fields:',
                     model: queryKeyFields,
                     name: '"queryKeyFields"',
@@ -103,153 +131,165 @@ panel-collapsible(ng-form=form opened=`!!${model}.queryMetadata`)
             .pc-form-grid-col-60
                 mixin domains-query-aliases
                     .ignite-form-field
-                        +ignite-form-field__label('Aliases:', '"aliases"')
-                            +tooltip(`Mapping from full property name in dot notation to an alias that will be used as SQL column name<br />
-                                For example: "parent.name" as "parentName"`)
-                        .ignite-form-field__control
-                            -let items = queryAliases
-
-                            list-editable(ng-model=items name='queryAliases')
-                                list-editable-item-view
-                                    | {{ $item.field }} &rarr; {{ $item.alias }}
-
-                                list-editable-item-edit
-                                    - form = '$parent.form'
-                                    .pc-form-grid-row
-                                        .pc-form-grid-col-30(divider='/')
-                                            +ignite-form-field-text('Field name', '$item.field', '"field"', false, true, 'Enter field name')(
-                                                data-ignite-unique=items
-                                                data-ignite-unique-property='field'
-                                                ignite-auto-focus
-                                            )
-                                                +unique-feedback('"field"', 'Such field already exists!')
-                                        .pc-form-grid-col-30
-                                            +ignite-form-field-text('Field alias', '$item.alias', '"alias"', false, true, 'Enter field alias')
-
-                                list-editable-no-items
-                                    list-editable-add-item-button(
-                                        add-item=`$editLast((${items} = ${items} || []).push({}))`
-                                        label-single='alias to query'
-                                        label-multiple='aliases'
-                                    )
+                        +form-field__label({ label: 'Aliases:', name: '"aliases"' })
+                            +form-field__tooltip({ title: `Mapping from full property name in dot notation to an alias that will be used as SQL column name<br />
+                                For example: "parent.name" as "parentName"` })
 
-                +domains-query-aliases
+                        -let items = queryAliases
 
-            .pc-form-grid-col-60(ng-if-end)
-                .ignite-form-field
-                    +ignite-form-field__label('Indexes:', '"indexes"')
-                    .ignite-form-field__control
-                        list-editable(
-                            ng-model=queryIndexes
-                            ng-model-options='{allowInvalid: true}'
-                            name='queryIndexes'
-                            ui-validate=`{
-                                complete: '$ctrl.Models.queryIndexes.complete($value)',
-                                fieldsExist: '$ctrl.Models.queryIndexes.fieldsExist($value, ${queryFields})',
-                                indexFieldsHaveUniqueNames: '$ctrl.Models.queryIndexes.indexFieldsHaveUniqueNames($value)'
-                            }`
-                            ui-validate-watch=`"[${queryIndexes}, ${queryFields}]"`
-                            ui-validate-watch-object-equality='true'
-                        )
-                            list-editable-item-view(item-name='queryIndex')
-                                div {{ queryIndex.name }} [{{ queryIndex.indexType }}]
-                                div(ng-repeat='field in queryIndex.fields track by field._id')
-                                    span {{ field.name }}
-                                    span(ng-if='queryIndex.indexType == "SORTED"')
-                                        |  / {{ field.direction ? 'ASC' : 'DESC'}}
-
-                            list-editable-item-edit(item-name='queryIndex')
+                        list-editable(ng-model=items name='queryAliases')
+                            list-editable-item-view
+                                | {{ $item.field }} &rarr; {{ $item.alias }}
+
+                            list-editable-item-edit
+                                - form = '$parent.form'
                                 .pc-form-grid-row
                                     .pc-form-grid-col-30(divider='/')
-                                        +sane-ignite-form-field-text({
-                                            label: 'Index name:',
-                                            model: 'queryIndex.name',
-                                            name: '"name"',
+                                        +form-field__text({
+                                            label: 'Field name',
+                                            model: '$item.field',
+                                            name: '"field"',
                                             required: true,
-                                            placeholder: 'Enter index name'
+                                            placeholder: 'Enter field name'
                                         })(
-                                            ignite-unique=queryIndexes
-                                            ignite-unique-property='name'
-                                            ignite-form-field-input-autofocus='true'
+                                            ignite-unique=items
+                                            ignite-unique-property='field'
+                                            ignite-auto-focus
                                         )
-                                            +unique-feedback(_, 'Such index already exists!')
+                                            +form-field__error({ error: 'igniteUnique', message: 'Such field already exists!' })
                                     .pc-form-grid-col-30
-                                        +sane-ignite-form-field-dropdown({
-                                            label: 'Index type:',
-                                            model: `queryIndex.indexType`,
-                                            name: '"indexType"',
+                                        +form-field__text({
+                                            label: 'Field alias',
+                                            model: '$item.alias',
+                                            name: '"alias"',
                                             required: true,
-                                            placeholder: 'Select index type',
-                                            options: '::$ctrl.Models.indexType.values'
+                                            placeholder: 'Enter field alias'
                                         })
-                                    .pc-form-grid-col-60
-                                        .ignite-form-field
-                                            +ignite-form-field__label('Index fields:', '"indexFields"', true)
-                                            .ignite-form-field__control
-                                                list-editable(
-                                                    ng-model='queryIndex.fields'
-                                                    ng-model-options='{allowInvalid: true}'
-                                                    name='indexFields'
-                                                    ng-required='true'
-                                                )
-                                                    list-editable-item-view(item-name='indexField')
-                                                        | {{ indexField.name }} 
-                                                        span(ng-if='queryIndex.indexType === "SORTED"')
-                                                            |  / {{ indexField.direction ? "ASC" : "DESC" }}
-
-                                                    list-editable-item-edit(item-name='indexField')
-                                                        .pc-form-grid-row
-                                                            .pc-form-grid-col-60
-                                                                +sane-ignite-form-field-dropdown({
-                                                                    label: 'Index field:',
-                                                                    model: 'indexField.name',
-                                                                    name: '"indexName"',
-                                                                    placeholder: `{{ ${queryFields}.length > 0 ? 'Choose index field' : 'No fields configured' }}`,
-                                                                    options: queryFields
-                                                                })(
-                                                                    bs-options=`queryField.name as queryField.name for queryField in ${queryFields}`
-                                                                    ng-disabled=`${queryFields}.length === 0`
-                                                                    ng-model-options='{allowInvalid: true}'
-                                                                    ignite-unique='queryIndex.fields'
-                                                                    ignite-unique-property='name'
-                                                                    ignite-auto-focus
-                                                                )
-                                                                    +unique-feedback(_, 'Such field already exists!')
-                                                            .pc-form-grid-col-60(
-                                                                ng-if='queryIndex.indexType === "SORTED"'
-                                                            )
-                                                                +sane-ignite-form-field-dropdown({
-                                                                    label: 'Sort direction:',
-                                                                    model: 'indexField.direction',
-                                                                    name: '"indexDirection"',
-                                                                    required: true,
-                                                                    options: '::$ctrl.Models.indexSortDirection.values'
-                                                                })
-                                                    list-editable-no-items
-                                                        list-editable-add-item-button(
-                                                            add-item=`$edit($ctrl.Models.addIndexField(queryIndex.fields))`
-                                                            label-single='field to index'
-                                                            label-multiple='fields in index'
-                                                        )
-                                            .ignite-form-field__errors(
-                                                ng-messages=`$form.indexFields.$error`
-                                                ng-show=`$form.indexFields.$invalid`
-                                            )
-                                                +form-field-feedback(_, 'required', 'Index fields should be configured')
 
                             list-editable-no-items
                                 list-editable-add-item-button(
-                                    add-item=`$edit($ctrl.Models.addIndex(${model}))`
-                                    label-single='index'
-                                    label-multiple='fields'
+                                    add-item=`$editLast((${items} = ${items} || []).push({}))`
+                                    label-single='alias to query'
+                                    label-multiple='aliases'
                                 )
-                    .ignite-form-field__errors(
+
+                +domains-query-aliases
+
+            .pc-form-grid-col-60(ng-if-end)
+                .ignite-form-field
+                    +form-field__label({ label: 'Indexes:', name: '"indexes"' })
+
+                    list-editable(
+                        ng-model=queryIndexes
+                        ng-model-options='{allowInvalid: true}'
+                        name='queryIndexes'
+                        ui-validate=`{
+                            complete: '$ctrl.Models.queryIndexes.complete($value)',
+                            fieldsExist: '$ctrl.Models.queryIndexes.fieldsExist($value, ${queryFields})',
+                            indexFieldsHaveUniqueNames: '$ctrl.Models.queryIndexes.indexFieldsHaveUniqueNames($value)'
+                        }`
+                        ui-validate-watch=`"[${queryIndexes}, ${queryFields}]"`
+                        ui-validate-watch-object-equality='true'
+                    )
+                        list-editable-item-view(item-name='queryIndex')
+                            div {{ queryIndex.name }} [{{ queryIndex.indexType }}]
+                            div(ng-repeat='field in queryIndex.fields track by field._id')
+                                span {{ field.name }}
+                                span(ng-if='queryIndex.indexType == "SORTED"')
+                                    |  / {{ field.direction ? 'ASC' : 'DESC'}}
+
+                        list-editable-item-edit(item-name='queryIndex')
+                            .pc-form-grid-row
+                                .pc-form-grid-col-30(divider='/')
+                                    +form-field__text({
+                                        label: 'Index name:',
+                                        model: 'queryIndex.name',
+                                        name: '"name"',
+                                        required: true,
+                                        placeholder: 'Enter index name'
+                                    })(
+                                        ignite-unique=queryIndexes
+                                        ignite-unique-property='name'
+                                        ignite-form-field-input-autofocus='true'
+                                    )
+                                        +form-field__error({ error: 'igniteUnique', message: 'Such index already exists!' })
+                                .pc-form-grid-col-30
+                                    +form-field__dropdown({
+                                        label: 'Index type:',
+                                        model: `queryIndex.indexType`,
+                                        name: '"indexType"',
+                                        required: true,
+                                        placeholder: 'Select index type',
+                                        options: '::$ctrl.Models.indexType.values'
+                                    })
+                                .pc-form-grid-col-60
+                                    .ignite-form-field
+                                        +form-field__label({ label: 'Index fields:', name: '"indexFields"', required: true })
+
+                                        list-editable(
+                                            ng-model='queryIndex.fields'
+                                            ng-model-options='{allowInvalid: true}'
+                                            name='indexFields'
+                                            ng-required='true'
+                                        )
+                                            list-editable-item-view(item-name='indexField')
+                                                | {{ indexField.name }}
+                                                span(ng-if='queryIndex.indexType === "SORTED"')
+                                                    |  / {{ indexField.direction ? "ASC" : "DESC" }}
+
+                                            list-editable-item-edit(item-name='indexField')
+                                                .pc-form-grid-row
+                                                    .pc-form-grid-col-60
+                                                        +form-field__dropdown({
+                                                            label: 'Index field:',
+                                                            model: 'indexField.name',
+                                                            name: '"indexName"',
+                                                            placeholder: `{{ ${queryFields}.length > 0 ? 'Choose index field' : 'No fields configured' }}`,
+                                                            options: queryFields
+                                                        })(
+                                                            bs-options=`queryField.name as queryField.name for queryField in ${queryFields}`
+                                                            ng-disabled=`${queryFields}.length === 0`
+                                                            ng-model-options='{allowInvalid: true}'
+                                                            ignite-unique='queryIndex.fields'
+                                                            ignite-unique-property='name'
+                                                            ignite-auto-focus
+                                                        )
+                                                            +form-field__error({ error: 'igniteUnique', message: 'Such field already exists!' })
+                                                    .pc-form-grid-col-60(
+                                                        ng-if='queryIndex.indexType === "SORTED"'
+                                                    )
+                                                        +form-field__dropdown({
+                                                            label: 'Sort direction:',
+                                                            model: 'indexField.direction',
+                                                            name: '"indexDirection"',
+                                                            required: true,
+                                                            options: '::$ctrl.Models.indexSortDirection.values'
+                                                        })
+                                            list-editable-no-items
+                                                list-editable-add-item-button(
+                                                    add-item=`$edit($ctrl.Models.addIndexField(queryIndex.fields))`
+                                                    label-single='field to index'
+                                                    label-multiple='fields in index'
+                                                )
+                                        .form-field__errors(
+                                            ng-messages=`$form.indexFields.$error`
+                                            ng-show=`$form.indexFields.$invalid`
+                                        )
+                                            +form-field__error({ error: 'required', message: 'Index fields should be configured' })
+
+                        list-editable-no-items
+                            list-editable-add-item-button(
+                                add-item=`$edit($ctrl.Models.addIndex(${model}))`
+                                label-single='index'
+                                label-multiple='fields'
+                            )
+                    .form-field__errors(
                         ng-messages=`query.queryIndexes.$error`
                         ng-show=`query.queryIndexes.$invalid`
                     )
-                        +form-field-feedback(_, 'complete', 'Some indexes are incomplete')
-                        +form-field-feedback(_, 'fieldsExist', 'Some indexes use unknown fields')
-                        +form-field-feedback(_, 'indexFieldsHaveUniqueNames', 'Each query index field name should be unique')
+                        +form-field__error({ error: 'complete', message: 'Some indexes are incomplete' })
+                        +form-field__error({ error: 'fieldsExist', message: 'Some indexes use unknown fields' })
+                        +form-field__error({ error: 'indexFieldsHaveUniqueNames', message: 'Each query index field name should be unique' })
 
         .pca-form-column-6
             +preview-xml-java(model, 'domainModelQuery')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/store.pug b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/store.pug
index 811c0d78d72f2..0e1a44d584173 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/store.pug
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/model-edit-form/templates/store.pug
@@ -37,7 +37,7 @@ mixin list-db-field-edit({ items, itemName, itemsName })
         list-editable-item-edit
             .pc-form-grid-row
                 .pc-form-grid-col-30(divider='/')
-                    +sane-ignite-form-field-text({
+                    +form-field__text({
                         label: 'DB name:',
                         model: '$item.databaseFieldName',
                         name: '"databaseFieldName"',
@@ -49,11 +49,18 @@ mixin list-db-field-edit({ items, itemName, itemsName })
                         ignite-unique=items
                         ignite-unique-property='databaseFieldName'
                     )
-                        +unique-feedback(_, 'DB name should be unique')
+                        +form-field__error({ error: 'igniteUnique', message: 'DB name should be unique' })
                 .pc-form-grid-col-30
-                    +dropdown-required('DB type:', '$item.databaseFieldType', '"databaseFieldType"', true, true, 'Choose DB type', 'supportedJdbcTypes')
+                    +form-field__dropdown({
+                        label: 'DB type:',
+                        model:'$item.databaseFieldType',
+                        name: '"databaseFieldType"',
+                        required: 'true',
+                        placeholder: 'Choose DB type',
+                        options: 'supportedJdbcTypes'
+                    })
                 .pc-form-grid-col-30(divider='/')
-                    +sane-ignite-form-field-text({
+                    +form-field__text({
                         label: 'Java name:',
                         model: '$item.javaFieldName',
                         name: '"javaFieldName"',
@@ -64,9 +71,16 @@ mixin list-db-field-edit({ items, itemName, itemsName })
                         ignite-unique=items
                         ignite-unique-property='javaFieldName'
                     )
-                        +unique-feedback(_, 'Java name should be unique')
+                        +form-field__error({ error: 'igniteUnique', message: 'Java name should be unique' })
                 .pc-form-grid-col-30
-                    +dropdown-required('Java type:', '$item.javaFieldType', '"javaFieldType"', true, true, 'Choose Java type', 'supportedJavaTypes')
+                    +form-field__dropdown({
+                        label: 'Java type:',
+                        model: '$item.javaFieldType',
+                        name: '"javaFieldType"',
+                        required: 'true',
+                        placeholder: 'Choose Java type',
+                        options: 'supportedJavaTypes'
+                    })
 
         list-editable-no-items
             list-editable-add-item-button(
@@ -78,45 +92,59 @@ mixin list-db-field-edit({ items, itemName, itemsName })
 panel-collapsible(ng-form=form on-open=`ui.loadPanel('${form}')`)
     panel-title#store-title Domain model for cache store
     panel-description
-        | Domain model properties for binding database with cache via POJO cache store. 
+        | Domain model properties for binding database with cache via POJO cache store.
         a.link-success(href="https://apacheignite.readme.io/docs/3rd-party-store" target="_blank") More info
     panel-content.pca-form-row(ng-if=`ui.isPanelLoaded('${form}')`)
         .pca-form-column-6.pc-form-grid-row
             .pc-form-grid-col-30
-                +text('Database schema:', model + '.databaseSchema', '"databaseSchema"', 'false', 'Input DB schema name', 'Schema name in database')
+                +form-field__text({
+                    label: 'Database schema:',
+                    model: model + '.databaseSchema',
+                    name: '"databaseSchema"',
+                    placeholder: 'Input DB schema name',
+                    tip: 'Schema name in database'
+                })
             .pc-form-grid-col-30
-                +text('Database table:', model + '.databaseTable', '"databaseTable"', 'false', 'Input DB table name', 'Table name in database')
+                +form-field__text({
+                    label: 'Database table:',
+                    model: model + '.databaseTable',
+                    name: '"databaseTable"',
+                    placeholder: 'Input DB table name',
+                    tip: 'Table name in database'
+                })
             .pc-form-grid-col-60
                 .ignite-form-field
-                    +ignite-form-field__label('Key fields:', '"keyFields"')
-                        +tooltip(`Collection of key fields descriptions for CacheJdbcPojoStore`)
-                    .ignite-form-field__control
-                        +list-db-field-edit({
-                            items: keyFields,
-                            itemName: 'key field',
-                            itemsName: 'key fields'
-                        })(name='keyFields')
-                    .ignite-form-field__errors(
+                    +form-field__label({ label: 'Key fields:', name: '"keyFields"' })
+                        +form-field__tooltip({ title: `Collection of key fields descriptions for CacheJdbcPojoStore` })
+
+                    +list-db-field-edit({
+                        items: keyFields,
+                        itemName: 'key field',
+                        itemsName: 'key fields'
+                    })(name='keyFields')
+
+                    .form-field__errors(
                         ng-messages=`store.keyFields.$error`
                         ng-show=`store.keyFields.$invalid`
                     )
-                        +form-field-feedback(_, 'dbFieldUnique', 'Each key field DB name and Java name should be unique')
+                        +form-field__error({ error: 'dbFieldUnique', message: 'Each key field DB name and Java name should be unique' })
 
             .pc-form-grid-col-60
                 .ignite-form-field
-                    +ignite-form-field__label('Value fields:', '"valueFields"')
-                        +tooltip(`Collection of value fields descriptions for CacheJdbcPojoStore`)
-                    .ignite-form-field__control
-                        +list-db-field-edit({
-                            items: valueFields,
-                            itemName: 'value field',
-                            itemsName: 'value fields'
-                        })(name='valueFields')
-                    .ignite-form-field__errors(
+                    +form-field__label({ label: 'Value fields:', name: '"valueFields"' })
+                        +form-field__tooltip({ title: `Collection of value fields descriptions for CacheJdbcPojoStore` })
+
+                    +list-db-field-edit({
+                        items: valueFields,
+                        itemName: 'value field',
+                        itemsName: 'value fields'
+                    })(name='valueFields')
+
+                    .form-field__errors(
                         ng-messages=`store.valueFields.$error`
                         ng-show=`store.valueFields.$invalid`
                     )
-                        +form-field-feedback(_, 'dbFieldUnique', 'Each value field DB name and Java name should be unique')
+                        +form-field__error({ error: 'dbFieldUnique', message: 'Each value field DB name and Java name should be unique' })
 
         .pca-form-column-6
             +preview-xml-java(model, 'domainStore')
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js
index ecb7a15a3efad..da17c678fe6c2 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/controller.js
@@ -27,16 +27,17 @@ import Caches from 'app/services/Caches';
 // Controller for Caches screen.
 export default class Controller {
     static $inject = [
-        ConfigSelectors.name,
+        'ConfigSelectors',
         'configSelectionManager',
         '$uiRouter',
         '$transitions',
-        ConfigureState.name,
+        'ConfigureState',
         '$state',
         'IgniteFormUtils',
         'IgniteVersion',
-        Caches.name
+        'Caches'
     ];
+
     /**
      * @param {ConfigSelectors} ConfigSelectors
      * @param {object} configSelectionManager
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/index.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/index.js
index 818c2639ca634..c6c751cd20c68 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/index.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-caches/index.js
@@ -20,4 +20,4 @@ import component from './component';
 
 export default angular
     .module('ignite-console.page-configure-advanced.caches', [])
-    .component(component.name, component);
+    .component('pageConfigureAdvancedCaches', component);
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js
index 2b05940eb4ae2..f0348c353629e 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-cluster/controller.js
@@ -22,7 +22,7 @@ import 'rxjs/add/operator/publishReplay';
 
 // Controller for Clusters screen.
 export default class PageConfigureAdvancedCluster {
-    static $inject = ['$uiRouter', ConfigSelectors.name, ConfigureState.name];
+    static $inject = ['$uiRouter', 'ConfigSelectors', 'ConfigureState'];
 
     /**
      * @param {uirouter.UIRouter} $uiRouter
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js
index dd0e5daac5c45..50eee7cad0dce 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-igfs/controller.js
@@ -27,7 +27,8 @@ import ConfigSelectors from 'app/components/page-configure/store/selectors';
 import IGFSs from 'app/services/IGFSs';
 
 export default class PageConfigureAdvancedIGFS {
-    static $inject = [ConfigSelectors.name, ConfigureState.name, '$uiRouter', IGFSs.name, '$state', 'configSelectionManager'];
+    static $inject = ['ConfigSelectors', 'ConfigureState', '$uiRouter', 'IGFSs', '$state', 'configSelectionManager'];
+
     /**
      * @param {ConfigSelectors} ConfigSelectors        
      * @param {ConfigureState} ConfigureState         
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js
index d2a07e879f731..fd1ccaa703a72 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/components/page-configure-advanced-models/controller.js
@@ -32,7 +32,8 @@ import {default as ConfigureState} from 'app/components/page-configure/services/
 import {default as Models} from 'app/services/Models';
 
 export default class PageConfigureAdvancedModels {
-    static $inject = [ConfigSelectors.name, ConfigureState.name, '$uiRouter', Models.name, '$state', 'configSelectionManager'];
+    static $inject = ['ConfigSelectors', 'ConfigureState', '$uiRouter', 'Models', '$state', 'configSelectionManager'];
+
     /**
      * @param {ConfigSelectors} ConfigSelectors
      * @param {ConfigureState} ConfigureState
diff --git a/modules/web-console/frontend/app/components/page-configure-advanced/style.scss b/modules/web-console/frontend/app/components/page-configure-advanced/style.scss
index 9480486c9e790..40cd7130eb4bc 100644
--- a/modules/web-console/frontend/app/components/page-configure-advanced/style.scss
+++ b/modules/web-console/frontend/app/components/page-configure-advanced/style.scss
@@ -114,12 +114,6 @@ page-configure-advanced {
         user-select: none;
         cursor: pointer;
 
-        ignite-form-panel-chevron {
-            margin-right: 10px;
-            position: relative;
-            top: -3px;
-        }
-
         .pca-panel-heading-title {
             font-size: 16px;
             margin-right: 8px;
diff --git a/modules/web-console/frontend/app/components/page-configure-basic/controller.js b/modules/web-console/frontend/app/components/page-configure-basic/controller.js
index 105765a3a423d..88c61636f07ad 100644
--- a/modules/web-console/frontend/app/components/page-configure-basic/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-basic/controller.js
@@ -40,7 +40,7 @@ export default class PageConfigureBasicController {
     form;
 
     static $inject = [
-        Confirm.name, '$uiRouter', ConfigureState.name, ConfigSelectors.name, Clusters.name, Caches.name, IgniteVersion.name, '$element', 'ConfigChangesGuard', 'IgniteFormUtils', '$scope'
+        'Confirm', '$uiRouter', 'ConfigureState', 'ConfigSelectors', 'Clusters', 'Caches', 'IgniteVersion', '$element', 'ConfigChangesGuard', 'IgniteFormUtils', '$scope'
     ];
 
     /**
@@ -81,7 +81,9 @@ export default class PageConfigureBasicController {
     }
 
     _uiCanExit($transition$) {
-        if ($transition$.options().custom.justIDUpdate)
+        const options = $transition$.options();
+
+        if (options.custom.justIDUpdate || options.redirectedFrom)
             return true;
 
         $transition$.onSuccess({}, () => this.reset());
diff --git a/modules/web-console/frontend/app/components/page-configure-basic/style.scss b/modules/web-console/frontend/app/components/page-configure-basic/style.scss
index 7e0d8baac1b01..4814aa4b9c986 100644
--- a/modules/web-console/frontend/app/components/page-configure-basic/style.scss
+++ b/modules/web-console/frontend/app/components/page-configure-basic/style.scss
@@ -59,33 +59,6 @@ page-configure-basic {
         line-height: 19px;
     }
 
-    .pcb-memory-size {
-        .input-tip {
-            display: flex;
-            flex-direction: row;
-
-            .form-control {
-                border-top-right-radius: 0;
-                border-bottom-right-radius: 0;
-            }
-
-            .btn-ignite {
-                border-top-left-radius: 0;
-                border-bottom-left-radius: 0;
-                padding-top: 0;
-                padding-bottom: 0;
-                flex: 0 0 auto;
-                width: 60px !important;
-            }
-        }
-    }
-
-    .pcb-form-field-size {
-        .form-field-feedback {
-            left: -63px;
-        }
-    }
-
     .pcb-form-main-buttons {
         display: flex;
         flex-direction: row;
@@ -101,7 +74,7 @@ page-configure-basic {
         box-shadow: 0px -2px 4px -1px rgba(0, 0, 0, 0.2);
     }
 
-    .form-field-checkbox {
+    .form-field__checkbox {
         margin-top: auto;
         margin-bottom: 8px;
     }
diff --git a/modules/web-console/frontend/app/components/page-configure-basic/template.pug b/modules/web-console/frontend/app/components/page-configure-basic/template.pug
index e85b3d8e56965..996ddcc029fff 100644
--- a/modules/web-console/frontend/app/components/page-configure-basic/template.pug
+++ b/modules/web-console/frontend/app/components/page-configure-basic/template.pug
@@ -28,7 +28,6 @@ include ./../page-configure-advanced/components/cluster-edit-form/templates/gene
 - const model = '$ctrl.clonedCluster'
 - const modelDiscoveryKind = `${model}.discovery.kind`
 - let form = '$ctrl.form'
-- const tipOpts = {placement: 'top'}
 
 form(novalidate name=form)
     h2.pcb-section-header.pcb-inner-padding Step 1. Cluster Configuration
@@ -41,11 +40,10 @@ form(novalidate name=form)
 
     .pc-form-grid-row.pcb-form-grid-row
         .pc-form-grid-col-60
-            +sane-ignite-form-field-text({
+            +form-field__text({
                 label: 'Name:',
                 model: `${model}.name`,
                 name: '"clusterName"',
-                disabled: 'false',
                 placeholder: 'Input name',
                 required: true,
                 tip: 'Instance name allows to indicate to what grid this particular grid instance belongs to'
@@ -54,23 +52,29 @@ form(novalidate name=form)
                 ignite-unique-property='name'
                 ignite-unique-skip=`["_id", ${model}]`
             )
-                +unique-feedback(`${model}.name`, 'Cluster name should be unique.')
+                +form-field__error({ error: 'igniteUnique', message: 'Cluster name should be unique.' })
 
         .pc-form-grid__break
         .pc-form-grid-col-60
-            +dropdown('Discovery:', modelDiscoveryKind, '"discovery"', 'true', 'Choose discovery', '$ctrl.Clusters.discoveries',
-            'Discovery allows to discover remote nodes in grid\
-            <ul>\
-                <li>Static IPs - IP Finder which works only with pre configured list of IP addresses specified</li>\
-                <li>Multicast - Multicast based IP finder</li>\
-                <li>AWS S3 - AWS S3 based IP finder that automatically discover cluster nodes on Amazon EC2 cloud</li>\
-                <li>Apache jclouds - Apache jclouds multi cloud toolkit based IP finder for cloud platforms with unstable IP addresses</li>\
-                <li>Google cloud storage - Google Cloud Storage based IP finder that automatically discover cluster nodes on Google Compute Engine cluster</li>\
-                <li>JDBC - JDBC based IP finder that use database to store node IP address</li>\
-                <li>Shared filesystem - Shared filesystem based IP finder that use file to store node IP address</li>\
-                <li>Apache ZooKeeper - Apache ZooKeeper based IP finder when you use ZooKeeper to coordinate your distributed environment</li>\
-                <li>Kubernetes - IP finder for automatic lookup of Ignite nodes running in Kubernetes environment</li>\
-            </ul>')
+            +form-field__dropdown({
+                label: 'Discovery:',
+                model: modelDiscoveryKind,
+                name: '"discovery"',
+                placeholder: 'Choose discovery',
+                options: '$ctrl.Clusters.discoveries',
+                tip: 'Discovery allows to discover remote nodes in grid\
+                <ul>\
+                    <li>Static IPs - IP Finder which works only with pre configured list of IP addresses specified</li>\
+                    <li>Multicast - Multicast based IP finder</li>\
+                    <li>AWS S3 - AWS S3 based IP finder that automatically discover cluster nodes on Amazon EC2 cloud</li>\
+                    <li>Apache jclouds - Apache jclouds multi cloud toolkit based IP finder for cloud platforms with unstable IP addresses</li>\
+                    <li>Google cloud storage - Google Cloud Storage based IP finder that automatically discover cluster nodes on Google Compute Engine cluster</li>\
+                    <li>JDBC - JDBC based IP finder that use database to store node IP address</li>\
+                    <li>Shared filesystem - Shared filesystem based IP finder that use file to store node IP address</li>\
+                    <li>Apache ZooKeeper - Apache ZooKeeper based IP finder when you use ZooKeeper to coordinate your distributed environment</li>\
+                    <li>Kubernetes - IP finder for automatic lookup of Ignite nodes running in Kubernetes environment</li>\
+                </ul>'
+            })
         .pc-form-grid__break
         .pc-form-group
             +discovery-vm(model)(class='pcb-form-grid-row' ng-if=`${modelDiscoveryKind} === 'Vm'`)
@@ -93,7 +97,7 @@ form(novalidate name=form)
                 $ctrl.memorySizeInputVisible$|async:this
             `
         )
-            pc-form-field-size(
+            form-field-size(
                 ng-model='$ctrl.defaultMemoryPolicy.maxSize'
                 ng-model-options='{allowInvalid: true}'
                 id='memory'
@@ -106,10 +110,10 @@ form(novalidate name=form)
                 tip='“default” cluster memory policy off-heap max memory size. Leave empty to use 80% of physical memory available on current machine. Should be at least 10Mb.'
                 on-scale-change='scale = $event'
             )
-                +form-field-feedback('"memory"', 'min', 'Maximum size should be equal to or more than initial size ({{ $ctrl.Clusters.memoryPolicy.maxSize.min($ctrl.defaultMemoryPolicy) / scale.value}} {{scale.label}}).')
+                +form-field__error({ error: 'min', message: 'Maximum size should be equal to or more than initial size ({{ $ctrl.Clusters.memoryPolicy.maxSize.min($ctrl.defaultMemoryPolicy) / scale.value}} {{scale.label}}).' })
 
         .pc-form-grid-col-60(ng-if=`$ctrl.IgniteVersion.available('2.3.0')`)
-            pc-form-field-size(
+            form-field-size(
                 ng-model=`${model}.dataStorageConfiguration.defaultDataRegionConfiguration.maxSize`
                 ng-model-options='{allowInvalid: true}'
                 id='memory'
@@ -122,11 +126,7 @@ form(novalidate name=form)
                 tip='Default data region off-heap max memory size. Leave empty to use 20% of physical memory available on current machine. Should be at least 10Mb.'
                 on-scale-change='scale = $event'
             )
-                +form-field-feedback(
-                    _,
-                    'min',
-                    `Maximum size should be equal to or more than initial size ({{ $ctrl.Clusters.dataRegion.maxSize.min(${model}.dataStorageConfiguration.defaultDataRegionConfiguration) / scale.value}} {{scale.label}}).`
-                )
+                +form-field__error({ error: 'min', message: `Maximum size should be equal to or more than initial size ({{ $ctrl.Clusters.dataRegion.maxSize.min(${model}.dataStorageConfiguration.defaultDataRegionConfiguration) / scale.value}} {{scale.label}}).` })
         .pc-form-grid-col-120
             .ignite-form-field
                 list-editable.pcb-caches-list(
@@ -144,16 +144,26 @@ form(novalidate name=form)
                         div {{ $ctrl.Caches.getCacheBackupsCount($item) }}
                     list-editable-item-edit
                         div
-                            +ignite-form-field-text('Name', '$item.name', '"name"', false, true)(
+                            +form-field__text({
+                                label: 'Name',
+                                model: '$item.name',
+                                name: '"name"',
+                                required: true
+                            })(
                                 ignite-unique='$ctrl.shortCaches'
                                 ignite-unique-property='name'
                                 ignite-form-field-input-autofocus='true'
                             )
-                                +unique-feedback('"name"', 'Cache name should be unqiue')
+                                +form-field__error({ error: 'igniteUnique', message: 'Cache name should be unqiue' })
                         div
-                            +cacheMode('Mode:', '$item.cacheMode', '"cacheMode"', 'PARTITIONED')
+                            +form-field__cache-modes({
+                                label: 'Mode:',
+                                model: '$item.cacheMode',
+                                name: '"cacheMode"',
+                                placeholder: 'PARTITIONED'
+                            })
                         div
-                            +sane-ignite-form-field-dropdown({
+                            +form-field__dropdown({
                                 label: 'Atomicity:',
                                 model: '$item.atomicityMode',
                                 name: '"atomicityMode"',
@@ -161,14 +171,20 @@ form(novalidate name=form)
                                 options: '::$ctrl.Caches.atomicityModes'
                             })
                         div(ng-show='$ctrl.Caches.shouldShowCacheBackupsCount($item)')
-                            +number('Backups:', '$item.backups', '"backups"', 'true', '0', '0')
+                            +form-field__number({
+                                label: 'Backups:',
+                                model: '$item.backups',
+                                name: '"backups"',
+                                placeholder: '0',
+                                min: 0
+                            })
                     list-editable-no-items
                         list-editable-add-item-button(
                             add-item='$ctrl.addCache()'
                             label-single='cache'
                             label-multiple='caches'
                         )
-        
+
     .pc-form-actions-panel
         button-preview-project(ng-hide='$ctrl.isNew$|async:this' cluster=model)
 
diff --git a/modules/web-console/frontend/app/components/page-configure-overview/controller.js b/modules/web-console/frontend/app/components/page-configure-overview/controller.js
index db21d9afe23a2..e49e5c68a8bb5 100644
--- a/modules/web-console/frontend/app/components/page-configure-overview/controller.js
+++ b/modules/web-console/frontend/app/components/page-configure-overview/controller.js
@@ -39,11 +39,11 @@ import {confirmClustersRemoval} from '../page-configure/store/actionCreators';
 export default class PageConfigureOverviewController {
     static $inject = [
         '$uiRouter',
-        ModalPreviewProject.name,
-        Clusters.name,
-        ConfigureState.name,
-        ConfigSelectors.name,
-        ConfigurationDownload.name
+        'ModalPreviewProject',
+        'Clusters',
+        'ConfigureState',
+        'ConfigSelectors',
+        'ConfigurationDownload'
     ];
 
     /**
diff --git a/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js b/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js
index c96216143d7cc..78755466deaef 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/formUICanExitGuard.js
@@ -18,7 +18,7 @@
 import {default as ConfigChangesGuard} from '../services/ConfigChangesGuard';
 
 class FormUICanExitGuardController {
-    static $inject = ['$element', ConfigChangesGuard.name];
+    static $inject = ['$element', 'ConfigChangesGuard'];
 
     /**
      * @param {JQLite} $element
@@ -43,7 +43,9 @@ class FormUICanExitGuardController {
             return;
 
         controller.uiCanExit = ($transition$) => {
-            if ($transition$.options().custom.justIDUpdate)
+            const options = $transition$.options();
+
+            if (options.custom.justIDUpdate || options.redirectedFrom)
                 return true;
 
             $transition$.onSuccess({}, controller.reset);
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
index 1d4ba7c48dbd3..e4b82530cdaca 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/component.js
@@ -93,7 +93,8 @@ export class ModalImportModels {
     /** @type {ng.ICompiledExpression} */
     onHide;
 
-    static $inject = ['$uiRouter', ConfigSelectors.name, ConfigEffects.name, ConfigureState.name, '$http', 'IgniteConfirm', IgniteConfirmBatch.name, 'IgniteFocus', SqlTypes.name, JavaTypes.name, 'IgniteMessages', '$scope', '$rootScope', 'AgentManager', 'IgniteActivitiesData', 'IgniteLoading', 'IgniteFormUtils', 'IgniteLegacyUtils'];
+    static $inject = ['$uiRouter', 'ConfigSelectors', 'ConfigEffects', 'ConfigureState', '$http', 'IgniteConfirm', 'IgniteConfirmBatch', 'IgniteFocus', 'SqlTypes', 'JavaTypes', 'IgniteMessages', '$scope', '$rootScope', 'AgentManager', 'IgniteActivitiesData', 'IgniteLoading', 'IgniteFormUtils', 'IgniteLegacyUtils'];
+
     /**
      * @param {UIRouter} $uiRouter
      * @param {ConfigSelectors} ConfigSelectors
@@ -154,16 +155,20 @@ export class ModalImportModels {
         })
         .take(1);
     }
+
     saveBatch(batch) {
         if (!batch.length)
             return;
 
+        this.$scope.importDomain.loadingOptions = SAVING_DOMAINS;
         this.Loading.start('importDomainFromDb');
+
         this.ConfigureState.dispatchAction({
             type: 'ADVANCED_SAVE_COMPLETE_CONFIGURATION',
             changedItems: this.batchActionsToRequestBody(batch),
             prevActions: []
         });
+
         this.saveSubscription = Observable.race(
             this.ConfigureState.actions$.filter((a) => a.type === 'ADVANCED_SAVE_COMPLETE_CONFIGURATION_OK')
                 .do(() => this.onHide()),
@@ -175,6 +180,7 @@ export class ModalImportModels {
         })
         .subscribe();
     }
+
     batchActionsToRequestBody(batch) {
         const result = batch.reduce((req, action) => {
             return {
@@ -199,21 +205,25 @@ export class ModalImportModels {
         result.cluster.caches = [...new Set(result.cluster.caches)];
         return result;
     }
+
     onTableSelectionChange(selected) {
         this.$scope.$applyAsync(() => {
             this.$scope.importDomain.tablesToUse = selected;
             this.selectedTablesIDs = selected.map((t) => t.id);
         });
     }
+
     onSchemaSelectionChange(selected) {
         this.$scope.$applyAsync(() => {
             this.$scope.importDomain.schemasToUse = selected;
             this.selectedSchemasIDs = selected.map((i) => i.name);
         });
     }
+
     onVisibleRowsChange(rows) {
         return this.visibleTables = rows.map((r) => r.entity);
     }
+
     onCacheSelect(cacheID) {
         if (cacheID < 0)
             return;
@@ -236,11 +246,13 @@ export class ModalImportModels {
         )
         .subscribe();
     }
+
     $onDestroy() {
         this.subscription.unsubscribe();
         if (this.onCacheSelectSubcription) this.onCacheSelectSubcription.unsubscribe();
         if (this.saveSubscription) this.saveSubscription.unsubscribe();
     }
+
     $onInit() {
         // Restores old behavior
         const {$http, Confirm, ConfirmBatch, Focus, SqlTypes, JavaTypes, Messages, $scope, $root, agentMgr, ActivitiesData, Loading, FormUtils, LegacyUtils} = this;
@@ -1025,13 +1037,9 @@ export class ModalImportModels {
 
                             $scope.ui.selectedJdbcDriverJar = $scope.jdbcDriverJars[0].value;
 
-                            // FormUtils.confirmUnsavedChanges(dirty, () => {
                             $scope.importDomain.action = 'connect';
                             $scope.importDomain.tables = [];
                             this.selectedTables = [];
-
-                            // Focus.move('jdbcUrl');
-                            // });
                         }
                         else {
                             $scope.importDomain.jdbcDriversNotFound = true;
@@ -1140,7 +1148,7 @@ export class ModalImportModels {
                 ></tables-action-cell>
             `,
             visible: true,
-            minWidth: 500
+            minWidth: 450
         }
     ];
 }
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/index.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/index.js
index b75c89ae2f303..3bc71da9a2502 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/index.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/index.js
@@ -23,9 +23,9 @@ import {component as tablesActionCell} from './tables-action-cell/component';
 import {component as amountIndicator} from './selected-items-amount-indicator/component';
 
 export default angular
-.module('configuration.modal-import-models', [])
-.service(service.name, service)
-.component(tablesActionCell.name, tablesActionCell)
-.component(stepIndicator.name, stepIndicator)
-.component(amountIndicator.name, amountIndicator)
-.component(component.name, component);
+    .module('configuration.modal-import-models', [])
+    .service('ModalImportModels', service)
+    .component('tablesActionCell', tablesActionCell)
+    .component('modalImportModelsStepIndicator', stepIndicator)
+    .component('selectedItemsAmountIndicator', amountIndicator)
+    .component('modalImportModels', component);
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/selected-items-amount-indicator/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/selected-items-amount-indicator/component.js
index 26a6ea05159c3..abee6b042943a 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/selected-items-amount-indicator/component.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/selected-items-amount-indicator/component.js
@@ -17,8 +17,8 @@
 
 import template from './template.pug';
 import './style.scss';
+
 export const component = {
-    name: 'selectedItemsAmountIndicator',
     template,
     bindings: {
         selectedAmount: '<',
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/step-indicator/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/step-indicator/component.js
index 4f2b141d81f8d..c37662c499fb4 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/step-indicator/component.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/step-indicator/component.js
@@ -25,7 +25,6 @@ export class ModalImportModelsStepIndicator {
 }
 
 export const component = {
-    name: 'modalImportModelsStepIndicator',
     template,
     controller: ModalImportModelsStepIndicator,
     bindings: {
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js
index 7fde5baad212f..17d4dc1ba5ab3 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/component.js
@@ -59,7 +59,6 @@ export class TablesActionCell {
 }
 
 export const component = {
-    name: 'tablesActionCell',
     controller: TablesActionCell,
     bindings: {
         onEditStart: '&',
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/style.scss b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/style.scss
index 49a7b9194ce5c..e6e63332fa6c1 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/style.scss
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/style.scss
@@ -28,7 +28,6 @@ tables-action-cell {
 
         &:hover {
             background: white;
-            box-shadow: inset 0 1px 3px 0 rgba(0, 0, 0, 0.5);
             border: solid 1px #c5c5c5;
         }
     }
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/template.pug b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/template.pug
index 2f51114cd8f76..c64d97381c68c 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/template.pug
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/tables-action-cell/template.pug
@@ -14,6 +14,8 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
+include /app/helpers/jade/mixins
+
 button.table-action-cell__edit-button.btn-ignite(
     type='button'
     b_s-tooltip=''
@@ -25,21 +27,17 @@ button.table-action-cell__edit-button.btn-ignite(
 )
     | {{ $ctrl.tableActionView($ctrl.table) }}
 .table-action-cell__edit-form(ng-if='$ctrl.table.edit')
-    .ignite-form-field.ignite-form-field-dropdown.table-action-cell__action-select
-        .ignite-form-field__control
-            .input-tip
-                button.select-toggle.form-control(
-                    type='button'
-                    bs-select
-                    ng-model='$ctrl.table.action'
-                    bs-options='item.value as item.shortLabel for item in $ctrl.importActions'
-                )
-    .ignite-form-field.ignite-form-field-dropdown.table-action-cell__cache-select
-        .ignite-form-field__control
-            .input-tip
-                button.select-toggle.form-control(
-                    bs-select
-                    ng-model='$ctrl.table.cacheOrTemplate'
-                    ng-change='$ctrl.onCacheSelect({$event: $ctrl.table.cacheOrTemplate})'
-                    bs-options='item.value as item.label for item in $ctrl.table.cachesOrTemplates'
-                )
\ No newline at end of file
+    .table-action-cell__action-select
+        +form-field__dropdown({
+            model: '$ctrl.table.action',
+            options: '$ctrl.importActions',
+            optionLabel: 'shortLabel'
+        })
+
+    .table-action-cell__cache-select
+        +form-field__dropdown({
+            model: '$ctrl.table.cacheOrTemplate',
+            options: '$ctrl.table.cachesOrTemplates'
+        })(
+            ng-change='$ctrl.onCacheSelect({$event: $ctrl.table.cacheOrTemplate})'
+        )
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/template.tpl.pug b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/template.tpl.pug
index 1762ecc8aa383..8aef347a80250 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-import-models/template.tpl.pug
@@ -16,26 +16,23 @@
 
 include /app/helpers/jade/mixins
 
-mixin chk(mdl, change, tip)
-    input(type='checkbox' ng-model=mdl ng-change=change bs-tooltip='' data-title=tip data-trigger='hover' data-placement='top')
-
 mixin td-ellipses-lbl(w, lbl)
     td.td-ellipsis(width=`${w}` style=`min-width: ${w}; max-width: ${w}`)
         label #{lbl}
 
-.modal--ignite.modal.modal-domain-import.center(role='dialog')
+.modal--ignite.modal--wide.modal.modal-domain-import.center(role='dialog')
     -var tipOpts = {};
     - tipOpts.container = '.modal-content'
-    - tipOpts.placement = 'top'
+    - tipOpts.placement = 'right'
     .modal-dialog
         .modal-content(ignite-loading='importDomainFromDb' ignite-loading-text='{{importDomain.loadingOptions.text}}')
             #errors-container.modal-header.header
                 button.close(type='button' ng-click='$hide()' aria-hidden='true')
                     svg(ignite-icon="cross")
-                h4.modal-title() 
+                h4.modal-title()
                     span(ng-if='!importDomain.demo') Import domain models from database
                     span(ng-if='importDomain.demo') Import domain models from demo database
-            .modal-body.theme--ignite
+            .modal-body
                 modal-import-models-step-indicator(
                     steps='$ctrl.actions'
                     current-step='importDomain.action'
@@ -61,26 +58,61 @@ mixin td-ellipses-lbl(w, lbl)
                                 li Copy h2-x.x.x.jar into agent 'jdbc-drivers' folder and try again
                                 li Refer to agent README.txt for more information
                 .import-domain-model-wizard-page(ng-if='importDomain.action == "connect" && !importDomain.demo')
-                    -var form = 'connectForm'
-
                     form.pc-form-grid-row(name=form novalidate)
                         .pc-form-grid-col-30
-                            +ignite-form-field-dropdown('Driver JAR:', 'ui.selectedJdbcDriverJar', '"jdbcDriverJar"', false, true, false,
-                                'Choose JDBC driver', '', 'jdbcDriverJars',
-                                'Select appropriate JAR with JDBC driver<br> To add another driver you need to place it into "/jdbc-drivers" folder of Ignite Web Agent<br> Refer to Ignite Web Agent README.txt for for more information'
-                            )
+                            +form-field__dropdown({
+                                label: 'Driver JAR:',
+                                model: 'ui.selectedJdbcDriverJar',
+                                name: '"jdbcDriverJar"',
+                                required: true,
+                                placeholder: 'Choose JDBC driver',
+                                options: 'jdbcDriverJars',
+                                tip: 'Select appropriate JAR with JDBC driver<br> To add another driver you need to place it into "/jdbc-drivers" folder of Ignite Web Agent<br> Refer to Ignite Web Agent README.txt for for more information'
+                            })
                         .pc-form-grid-col-30
-                            +java-class('JDBC driver:', 'selectedPreset.jdbcDriverClass', '"jdbcDriverClass"', true, true, 'Fully qualified class name of JDBC driver that will be used to connect to database')
+                            +form-field__java-class({
+                                label: 'JDBC driver:',
+                                model: 'selectedPreset.jdbcDriverClass',
+                                name: '"jdbcDriverClass"',
+                                required: true,
+                                tip: 'Fully qualified class name of JDBC driver that will be used to connect to database'
+                            })
                         .pc-form-grid-col-60
-                            +text-enabled-autofocus('JDBC URL:', 'selectedPreset.jdbcUrl', '"jdbcUrl"', true, true, 'JDBC URL', 'JDBC URL for connecting to database<br>Refer to your database documentation for details')
+                            +form-field__text({
+                                label: 'JDBC URL:',
+                                model: 'selectedPreset.jdbcUrl',
+                                name: '"jdbcUrl"',
+                                required: true,
+                                placeholder: 'JDBC URL',
+                                tip: 'JDBC URL for connecting to database<br>Refer to your database documentation for details'
+                            })(
+                                ignite-form-field-input-autofocus='true'
+                            )
+
                         .pc-form-grid-col-30
-                            +text('User:', 'selectedPreset.user', '"jdbcUser"', false, '', 'User name for connecting to database')
+                            +form-field__text({
+                                label: 'User:',
+                                model: 'selectedPreset.user',
+                                name: '"jdbcUser"',
+                                tip: 'User name for connecting to database'
+                            })
                         .pc-form-grid-col-30
-                            +password('Password:', 'selectedPreset.password', '"jdbcPassword"', false, '', 'Password for connecting to database<br>Note, password would not be saved in preferences for security reasons')(ignite-on-enter='importDomainNext()')
+                            +form-field__password({
+                                label: 'Password:',
+                                model: 'selectedPreset.password',
+                                name: '"jdbcPassword"',
+                                tip: 'Password for connecting to database<br>Note, password would not be saved in preferences for security reasons'
+                            })(
+                                ignite-on-enter='importDomainNext()'
+                            )
                         .pc-form-grid-col-60
-                            - tipOpts.placement = 'auto'
-                            +checkbox('Tables only', 'selectedPreset.tablesOnly', '"tablesOnly"', 'If selected, then only tables metadata will be parsed<br>Otherwise table and view metadata will be parsed')
-                            - tipOpts.placement = 'top'
+                            +form-field__checkbox({
+                                label: 'Tables only',
+                                model: 'selectedPreset.tablesOnly',
+                                name: '"tablesOnly"',
+                                tip: 'If selected, then only tables metadata will be parsed<br>Otherwise table and view metadata will be parsed',
+                                tipOpts
+                            })
 
                 .import-domain-model-wizard-page(ng-if='importDomain.action == "schemas"')
                     pc-items-table(
@@ -95,14 +127,14 @@ mixin td-ellipses-lbl(w, lbl)
                 .import-domain-model-wizard-page(ng-if='importDomain.action == "tables"')
                     form.pc-form-grid-row(novalidate)
                         .pc-form-grid-col-30
-                            +sane-ignite-form-field-dropdown({
+                            +form-field__dropdown({
                                 label: 'Action:',
                                 model: 'importCommon.action'
                             })(
                                 bs-options='item.value as item.label for item in importActions'
                             )
                         .pc-form-grid-col-30
-                            +sane-ignite-form-field-dropdown({
+                            +form-field__dropdown({
                                 label: 'Cache:',
                                 model: 'importCommon.cacheOrTemplate'
                             })(
@@ -110,8 +142,8 @@ mixin td-ellipses-lbl(w, lbl)
                                 ng-change='$ctrl.onCacheSelect(importCommon.cacheOrTemplate)'
                             )
                         .pc-form-grid-col-60.pc-form-grid__text-only-item
-                            | Defaults to be applied for filtered tables
-                            +tooltip('Select and apply options for caches generation')
+                            +form-field__label({ label: 'Defaults to be applied for filtered tables' })
+                                +form-field__tooltip({ title: 'Select and apply options for caches generation' })
                             button.btn-ignite.btn-ignite--success(
                                 type='button'
                                 ng-click='applyDefaults()'
@@ -133,23 +165,66 @@ mixin td-ellipses-lbl(w, lbl)
 
                     form.pc-form-grid-row(name=form novalidate)
                         .pc-form-grid-col-60
-                            +checkbox('Use Java built-in types for keys', 'ui.builtinKeys', '"domainBuiltinKeys"', 'Use Java built-in types like "Integer", "Long", "String" instead of POJO generation in case when table primary key contains only one field')
+                            +form-field__checkbox({
+                                label: 'Use Java built-in types for keys',
+                                model: 'ui.builtinKeys',
+                                name: '"domainBuiltinKeys"',
+                                tip: 'Use Java built-in types like "Integer", "Long", "String" instead of POJO generation in case when table primary key contains only one field',
+                                tipOpts
+                            })
                         .pc-form-grid-col-60
-                            +checkbox('Use primitive types for NOT NULL table columns', 'ui.usePrimitives', '"domainUsePrimitives"', 'Use primitive types like "int", "long", "double" for POJOs fields generation in case of NOT NULL columns')
+                            +form-field__checkbox({
+                                label: 'Use primitive types for NOT NULL table columns',
+                                model: 'ui.usePrimitives',
+                                name: '"domainUsePrimitives"',
+                                tip: 'Use primitive types like "int", "long", "double" for POJOs fields generation in case of NOT NULL columns',
+                                tipOpts
+                            })
                         .pc-form-grid-col-60
-                            +checkbox('Generate query entity key fields', 'ui.generateKeyFields', '"generateKeyFields"',
-                                'Generate key fields for query entity.<br\>\
+                            +form-field__checkbox({
+                                label: 'Generate query entity key fields',
+                                model: 'ui.generateKeyFields',
+                                name: '"generateKeyFields"',
+                                tip: 'Generate key fields for query entity.<br\>\
                                 We need this for the cases when no key-value classes\
                                 are present on cluster nodes, and we need to build/modify keys and values during SQL DML operations.\
-                                Thus, setting this parameter is not mandatory and should be based on particular use case.')
+                                Thus, setting this parameter is not mandatory and should be based on particular use case.',
+                                tipOpts
+                            })
                         .pc-form-grid-col-60
-                            +checkbox('Generate POJO classes', generatePojo, '"domainGeneratePojo"', 'If selected then POJO classes will be generated from database tables')
+                            +form-field__checkbox({
+                                label: 'Generate POJO classes',
+                                model: generatePojo,
+                                name: '"domainGeneratePojo"',
+                                tip: 'If selected then POJO classes will be generated from database tables',
+                                tipOpts
+                            })
                         .pc-form-grid-col-60(ng-if=generatePojo)
-                            +checkbox('Generate aliases for query entity', 'ui.generateTypeAliases', '"domainGenerateTypeAliases"', 'Generate aliases for query entity if table name is invalid Java identifier')
+                            +form-field__checkbox({
+                                label: 'Generate aliases for query entity',
+                                model: 'ui.generateTypeAliases',
+                                name: '"domainGenerateTypeAliases"',
+                                tip: 'Generate aliases for query entity if table name is invalid Java identifier',
+                                tipOpts
+                            })
                         .pc-form-grid-col-60(ng-if=generatePojo)
-                            +checkbox('Generate aliases for query fields', 'ui.generateFieldAliases', '"domainGenerateFieldAliases"', 'Generate aliases for query fields with database field names when database field name differ from Java field name')
+                            +form-field__checkbox({
+                                label: 'Generate aliases for query fields',
+                                model: 'ui.generateFieldAliases',
+                                name: '"domainGenerateFieldAliases"',
+                                tip: 'Generate aliases for query fields with database field names when database field name differ from Java field name',
+                                tipOpts
+                            })
                         .pc-form-grid-col-60(ng-if=generatePojo)
-                            +java-package('Package:', 'ui.packageName', '"domainPackageName"', true, true, 'Package that will be used for POJOs generation')
+                            +form-field__java-package({
+                                label: 'Package:',
+                                model: 'ui.packageName',
+                                name: '"domainPackageName"',
+                                required: true,
+                                tip: 'Package that will be used for POJOs generation',
+                                tipOpts
+                            })
+
             .modal-footer
                 button.btn-ignite.btn-ignite--success.modal-import-models__prev-button(
                     type='button'
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/index.js b/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/index.js
index a0dc92e59e6b3..a8aebb5af02ca 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/index.js
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/index.js
@@ -23,5 +23,5 @@ import service from './service';
 
 export default angular
     .module('ignite-console.page-configure.modal-preview-project', [])
-    .service(service.name, service)
+    .service('ModalPreviewProject', service)
     .component(component.name, component);
diff --git a/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/template.pug b/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/template.pug
index 34992773650e0..536db2839e389 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/template.pug
+++ b/modules/web-console/frontend/app/components/page-configure/components/modal-preview-project/template.pug
@@ -20,7 +20,7 @@ include /app/helpers/jade/mixins
     .modal-dialog
         .modal-content
             .modal-header
-                h4.modal-title 
+                h4.modal-title
                     svg(ignite-icon="structure")
                     span See Project Structure
                 button.close(type='button' aria-label='Close' ng-click='$ctrl.onHide()')
@@ -44,4 +44,5 @@ include /app/helpers/jade/mixins
                 .pane-right
                     div.file-preview(ignite-ace='{mode: $ctrl.fileExt, readonly: true}' ng-model='$ctrl.fileText')
             .modal-footer
-                button.btn-ignite.btn-ignite--success(ng-click='$ctrl.onHide()') Close
\ No newline at end of file
+                div
+                    button.btn-ignite.btn-ignite--success(ng-click='$ctrl.onHide()') Close
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/style.scss b/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/style.scss
deleted file mode 100644
index 737b2a05b55d9..0000000000000
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/style.scss
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-pc-form-field-size {
-    @import "./../../../../../public/stylesheets/variables.scss";
-
-    .input-tip {
-        display: flex;
-        flex-direction: row;
-
-        .form-control {
-            border-top-right-radius: 0;
-            border-bottom-right-radius: 0;
-        }
-
-        input {
-            border-top-right-radius: 0 !important;
-            border-bottom-right-radius: 0 !important;
-            min-width: 0;
-        }
-
-        .btn-ignite {
-            border-top-left-radius: 0 !important;
-            border-bottom-left-radius: 0 !important;
-            flex: 0 0 auto;
-            width: 60px !important;
-            line-height: initial !important;
-        }
-    }
-
-    &.ng-invalid:not(.ng-pristine),
-    &.ng-invalid.ng-touched {
-        input, .btn-ignite {
-            border-color: $ignite-brand-primary !important;
-            box-shadow: inset 0 1px 3px 0 rgba($ignite-brand-primary, .5) !important;
-        }
-    }
-}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/template.pug b/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/template.pug
deleted file mode 100644
index de62d35f3d698..0000000000000
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-form-field-size/template.pug
+++ /dev/null
@@ -1,61 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-include /app/helpers/jade/mixins
-
-+ignite-form-field__label('{{ ::$ctrl.label }}', '$ctrl.id', '$ctrl.required', '$ctrl.ngDisabled')
-    span(ng-if='::$ctrl.tip')
-        +tooltip('{{::$ctrl.tip}}')
-.ignite-form-field__control(ng-form='$ctrl.innerForm')
-    .input-tip
-        input.form-control(
-            type='number'
-            id='{{::$ctrl.id}}Input'
-            ng-model='$ctrl.value'
-            ng-model-options='{allowInvalid: true}'
-            ng-change='$ctrl.onValueChange()'
-            name='numberInput'
-            placeholder='{{$ctrl.placeholder}}'
-            min='{{ $ctrl.min ? $ctrl.min / $ctrl.sizeScale.value : "" }}'
-            max='{{ $ctrl.max ? $ctrl.max / $ctrl.sizeScale.value : "" }}'
-            ng-required='$ctrl.required'
-            ng-disabled='$ctrl.ngDisabled'
-        )
-        button.btn-ignite.btn-ignite--secondary(
-            bs-select
-            bs-options='size as size.label for size in $ctrl.sizesMenu'
-            ng-model='$ctrl.sizeScale'
-            protect-from-bs-select-render
-            ng-disabled='$ctrl.ngDisabled'
-            type='button'
-        )
-            | {{ $ctrl.sizeScale.label }}
-            span.fa.fa-caret-down.icon-right
-.ignite-form-field__errors(
-    ng-messages='$ctrl.ngModel.$error'
-    ng-show=`($ctrl.ngModel.$dirty || $ctrl.ngModel.$touched || $ctrl.ngModel.$submitted) && $ctrl.ngModel.$invalid`
-)
-    div(ng-transclude)
-    div(ng-message='required')
-        | This field could not be empty
-    div(ng-message='min')
-        | Value is less than allowable minimum: {{ $ctrl.min/$ctrl.sizeScale.value }} {{$ctrl.sizeScale.label}}
-    div(ng-message='max')
-        | Value is more than allowable maximum: {{ $ctrl.max/$ctrl.sizeScale.value }} {{$ctrl.sizeScale.label}}
-    div(ng-message='number')
-        | Only numbers allowed
-    div(ng-message='step')
-        | Invalid step
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/style.scss b/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/style.scss
index 227f23cd2d5a2..2147714389524 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/style.scss
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/style.scss
@@ -24,9 +24,6 @@ pc-items-table {
         display: flex;
         flex-direction: row;
     }
-    .ui-grid-settings--heading {
-        flex: 1;
-    }
 
     // Removes unwanted box-shadow and border-right from checkboxes column
     .ui-grid.ui-grid--ignite .ui-grid-pinned-container.ui-grid-pinned-container-left .ui-grid-render-container-left:before {
diff --git a/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/template.pug b/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/template.pug
index 0dbb760d2d408..75c683f48a58c 100644
--- a/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/template.pug
+++ b/modules/web-console/frontend/app/components/page-configure/components/pc-items-table/template.pug
@@ -17,25 +17,22 @@
 include /app/helpers/jade/mixins
 
 .panel--ignite
-    .panel-heading.ui-grid-settings.ui-grid-ignite__panel(ng-if='!$ctrl.hideHeader')
-        .panel-title
-            .pc-items-table__table-name.ng-animate-disabled(
-                ng-hide='$ctrl.gridAPI.selection.getSelectedCount()'
-            )
-                | {{ $ctrl.tableTitle }}
-                grid-column-selector(grid-api='$ctrl.gridAPI')
-            .pc-items-table__selection-count.ng-animate-disabled(
-                ng-show='$ctrl.gridAPI.selection.getSelectedCount()'
-            )
-                i {{ $ctrl.gridAPI.selection.getSelectedCount() }} of {{ $ctrl.items.length }} selected
-            .pco-clusters-table__actions-button
-                +ignite-form-field-bsdropdown({
-                    label: 'Actions',
-                    name: 'action',
-                    disabled: '!$ctrl.gridAPI.selection.getSelectedCount()',
-                    required: false,
-                    options: '$ctrl.actionsMenu'
-                })
+    header.header-with-selector(ng-if='!$ctrl.hideHeader')
+        div(ng-hide='$ctrl.gridAPI.selection.getSelectedCount()')
+            span {{ $ctrl.tableTitle }}
+            grid-column-selector(grid-api='$ctrl.gridAPI')
+
+        div(ng-show='$ctrl.gridAPI.selection.getSelectedCount()')
+            grid-item-selected(grid-api='$ctrl.gridAPI')
+
+        div
+            +ignite-form-field-bsdropdown({
+                label: 'Actions',
+                name: 'action',
+                disabled: '!$ctrl.gridAPI.selection.getSelectedCount()',
+                options: '$ctrl.actionsMenu'
+            })
+
     .grid.ui-grid--ignite(
         ui-grid='$ctrl.grid'
         ui-grid-selection
@@ -46,4 +43,4 @@ include /app/helpers/jade/mixins
     )
 
     div(ng-transclude='footerSlot' ng-hide='$ctrl.showFilterNotification')
-    footer-slot(ng-if='$ctrl.showFilterNotification' style='font-style:italic') Nothing to display. Check your filters.
\ No newline at end of file
+    footer-slot(ng-if='$ctrl.showFilterNotification' style='font-style:italic') Nothing to display. Check your filters.
diff --git a/modules/web-console/frontend/app/components/page-configure/index.js b/modules/web-console/frontend/app/components/page-configure/index.js
index 9beea7a57ad0d..c1d51ee094be7 100644
--- a/modules/web-console/frontend/app/components/page-configure/index.js
+++ b/modules/web-console/frontend/app/components/page-configure/index.js
@@ -36,7 +36,6 @@ import effects from './store/effects';
 import projectStructurePreview from './components/modal-preview-project';
 import itemsTable from './components/pc-items-table';
 import pcUiGridFilters from './components/pc-ui-grid-filters';
-import pcFormFieldSize from './components/pc-form-field-size';
 import isInCollection from './components/pcIsInCollection';
 import pcValidation from './components/pcValidation';
 import fakeUiCanExit from './components/fakeUICanExit';
@@ -99,7 +98,6 @@ export default angular
         'ui.router',
         'asyncFilter',
         uiValidate,
-        pcFormFieldSize.name,
         pcUiGridFilters.name,
         projectStructurePreview.name,
         itemsTable.name,
@@ -179,9 +177,9 @@ export default angular
         ConfigEffects.connect();
     }])
     .component('pageConfigure', component)
-    .directive(isInCollection.name, isInCollection)
-    .directive(fakeUiCanExit.name, fakeUiCanExit)
-    .directive(formUICanExitGuard.name, formUICanExitGuard)
+    .directive('pcIsInCollection', isInCollection)
+    .directive('fakeUiCanExit', fakeUiCanExit)
+    .directive('formUiCanExitGuard', formUICanExitGuard)
     .factory('configSelectionManager', ConfigSelectionManager)
     .service('IgniteSummaryZipper', SummaryZipper)
     .service('IgniteConfigurationResource', ConfigurationResource)
diff --git a/modules/web-console/frontend/app/components/page-configure/services/ConfigChangesGuard.js b/modules/web-console/frontend/app/components/page-configure/services/ConfigChangesGuard.js
index cca0e9efe2ed7..7e2df806f9d05 100644
--- a/modules/web-console/frontend/app/components/page-configure/services/ConfigChangesGuard.js
+++ b/modules/web-console/frontend/app/components/page-configure/services/ConfigChangesGuard.js
@@ -52,7 +52,7 @@ export class IgniteObjectDiffer {
 }
 
 export default class ConfigChangesGuard {
-    static $inject = [Confirm.name, '$sce'];
+    static $inject = ['Confirm', '$sce'];
 
     /**
      * @param {Confirm} Confirm.
@@ -74,7 +74,7 @@ export default class ConfigChangesGuard {
             You have unsaved changes.
             Are you sure you want to discard them?
             </p>
-            <details>
+            <details class='config-changes-guard__details'>
                 <summary>Click here to see changes</summary>
                 <div style='max-height: 400px; overflow: auto;'>${html.format(changes)}</div>                
             </details>
diff --git a/modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.js b/modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.js
index 2dab8a3d2334e..269eb938a819a 100644
--- a/modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.js
+++ b/modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.js
@@ -27,7 +27,9 @@ export default function ConfigurationResourceService($http) {
                 .then(({data}) => data)
                 .catch(({data}) => Promise.reject(data));
         },
-        populate({spaces, clusters, caches, igfss, domains}) {
+        populate(data) {
+            const {spaces, clusters, caches, igfss, domains} = _.cloneDeep(data);
+
             _.forEach(clusters, (cluster) => {
                 cluster.caches = _.filter(caches, ({_id}) => _.includes(cluster.caches, _id));
 
diff --git a/modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.spec.js b/modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.spec.js
new file mode 100644
index 0000000000000..d52d94a2991ac
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-configure/services/ConfigurationResource.spec.js
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import configurationResource from './ConfigurationResource';
+
+import { suite, test } from 'mocha';
+import { assert } from 'chai';
+
+const CHECKED_CONFIGURATION = {
+    spaces: [{
+        _id: '1space',
+        name: 'Test space'
+    }],
+    clusters: [{
+        _id: '1cluster',
+        space: '1space',
+        name: 'Test cluster',
+        caches: ['1cache'],
+        models: ['1model'],
+        igfss: ['1igfs']
+    }],
+    caches: [{
+        _id: '1cache',
+        space: '1space',
+        name: 'Test cache',
+        clusters: ['1cluster'],
+        models: ['1model']
+    }],
+    domains: [{
+        _id: '1model',
+        space: '1space',
+        name: 'Test model',
+        clusters: ['1cluster'],
+        caches: ['1cache']
+    }],
+    igfss: [{
+        _id: '1igfs',
+        space: '1space',
+        name: 'Test IGFS',
+        clusters: ['1cluster']
+    }]
+};
+
+suite('ConfigurationResourceTestsSuite', () => {
+    test('ConfigurationResourceService correctly populate data', async() => {
+        const service = configurationResource(null);
+        const converted = _.cloneDeep(CHECKED_CONFIGURATION);
+        const res = await service.populate(converted);
+
+        assert.notEqual(res.clusters[0], converted.clusters[0]);
+
+        assert.deepEqual(converted.clusters[0].caches, CHECKED_CONFIGURATION.clusters[0].caches);
+        assert.deepEqual(converted.clusters[0].models, CHECKED_CONFIGURATION.clusters[0].models);
+        assert.deepEqual(converted.clusters[0].igfss, CHECKED_CONFIGURATION.clusters[0].igfss);
+
+        assert.deepEqual(converted.caches[0].clusters, CHECKED_CONFIGURATION.caches[0].clusters);
+        assert.deepEqual(converted.caches[0].models, CHECKED_CONFIGURATION.caches[0].models);
+
+        assert.deepEqual(converted.domains[0].clusters, CHECKED_CONFIGURATION.domains[0].clusters);
+        assert.deepEqual(converted.domains[0].caches, CHECKED_CONFIGURATION.domains[0].caches);
+
+        assert.deepEqual(converted.igfss[0].clusters, CHECKED_CONFIGURATION.igfss[0].clusters);
+    });
+});
diff --git a/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js b/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js
index 10200bee1cdeb..d81921ea013be 100644
--- a/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js
+++ b/modules/web-console/frontend/app/components/page-configure/services/PageConfigure.js
@@ -38,7 +38,8 @@ import {default as ConfigureState} from 'app/components/page-configure/services/
 import {default as ConfigSelectors} from 'app/components/page-configure/store/selectors';
 
 export default class PageConfigure {
-    static $inject = [ConfigureState.name, ConfigSelectors.name];
+    static $inject = ['ConfigureState', 'ConfigSelectors'];
+
     /**
      * @param {ConfigureState} ConfigureState
      * @param {ConfigSelectors} ConfigSelectors
@@ -55,7 +56,7 @@ export default class PageConfigure {
                 .take(1)
                 .do(() => this.ConfigureState.dispatchAction({type: 'LOAD_COMPLETE_CONFIGURATION', clusterID, isDemo}))
                 .ignoreElements(),
-            this.ConfigureState.actions$.let(ofType('LOAD_COMPLETE_CONFIGURATION_ERR')).take(1).map((e) => {throw e;}),
+            this.ConfigureState.actions$.let(ofType('LOAD_COMPLETE_CONFIGURATION_ERR')).take(1).pluck('error').map((e) => Promise.reject(e)),
             this.ConfigureState.state$
                 .let(this.ConfigSelectors.selectCompleteClusterConfiguration({clusterID, isDemo}))
                 .filter((c) => c.__isComplete)
diff --git a/modules/web-console/frontend/app/components/page-configure/states.js b/modules/web-console/frontend/app/components/page-configure/states.js
index ed43f48c47bbc..3ba5bb70f18bc 100644
--- a/modules/web-console/frontend/app/components/page-configure/states.js
+++ b/modules/web-console/frontend/app/components/page-configure/states.js
@@ -24,10 +24,9 @@ import {Observable} from 'rxjs/Observable';
 
 const idRegex = `new|[a-z0-9]+`;
 
-const shortCachesResolve = ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', (ConfigSelectors, ConfigureState, {etp}, $transition$) => {
+const shortCachesResolve = ['ConfigSelectors', 'ConfigureState', 'ConfigEffects', '$transition$', function(ConfigSelectors, ConfigureState, {etp}, $transition$) {
     if ($transition$.params().clusterID === 'new')
         return Promise.resolve();
-
     return Observable.fromPromise($transition$.injector().getAsync('_cluster'))
         .switchMap(() => ConfigureState.state$.let(ConfigSelectors.selectCluster($transition$.params().clusterID)).take(1))
         .switchMap((cluster) => {
diff --git a/modules/web-console/frontend/app/components/page-configure/store/effects.js b/modules/web-console/frontend/app/components/page-configure/store/effects.js
index 9647461776522..014076b2b5a76 100644
--- a/modules/web-console/frontend/app/components/page-configure/store/effects.js
+++ b/modules/web-console/frontend/app/components/page-configure/store/effects.js
@@ -85,18 +85,19 @@ export const ofType = (type) => (s) => s.filter((a) => a.type === type);
 
 export default class ConfigEffects {
     static $inject = [
-        ConfigureState.name,
-        Caches.name,
-        IGFSs.name,
-        Models.name,
-        ConfigSelectors.name,
-        Clusters.name,
+        'ConfigureState',
+        'Caches',
+        'IGFSs',
+        'Models',
+        'ConfigSelectors',
+        'Clusters',
         '$state',
         'IgniteMessages',
         'IgniteConfirm',
-        Confirm.name,
-        ConfigurationDownload.name
+        'Confirm',
+        'ConfigurationDownload'
     ];
+
     /**
      * @param {ConfigureState} ConfigureState
      * @param {Caches} Caches
diff --git a/modules/web-console/frontend/app/components/page-configure/style.scss b/modules/web-console/frontend/app/components/page-configure/style.scss
index 36ae7524da51e..365e058cbbe31 100644
--- a/modules/web-console/frontend/app/components/page-configure/style.scss
+++ b/modules/web-console/frontend/app/components/page-configure/style.scss
@@ -33,10 +33,10 @@ page-configure {
         display: flex;
         flex-direction: row;
         padding: 10px 20px 10px 30px;
-        box-shadow: 0 0px 4px 0 rgba(0, 0, 0, 0.2), 0px 3px 4px -1px rgba(0, 0, 0, 0.2);
+        box-shadow: 0 0 4px 0 rgba(0, 0, 0, 0.2), 0 3px 4px -1px rgba(0, 0, 0, 0.2);
         position: -webkit-sticky;
         position: sticky;
-        bottom: 0px;
+        bottom: 0;
         // margin: 20px -30px -30px;
         background: white;
         border-radius: 0 0 4px 4px;
@@ -103,6 +103,11 @@ page-configure {
             margin-left: 10px !important;
         }
     }
+
+    .form-field__label.required:after {
+        content: '*';
+        margin-left: 0.25em;
+    }
 }
 
 .pc-form-group {
@@ -130,30 +135,15 @@ page-configure {
         background: var(--pc-form-group-title-bg-color);
     }
 
-    &>.form-field-checkbox .ignite-form-field__control {
-        & > span {
-            position: relative;
-
-            &:after {
-                content: '';
-                display: block;
-                position: absolute;
-                background-color: var(--pc-form-group-title-bg-color);
-                z-index: -1;
-                top: 0;
-                bottom: 0;
-                left: -26px;
-                right: -5px;
-            }
-        }
-        [ignite-icon] {
-            background-color: var(--pc-form-group-title-bg-color);            
-        }
-    }
-
     &+.pc-form-group {
         padding-top: 10px;
     }
+
+    .form-field__checkbox {
+        background-color: white;
+        padding: 0 5px;
+        margin: 0 -5px;
+    }
 }
 
 .pc-form-grid-row > .pc-form-group__text-title[class*='pc-form-grid-col-'] {
@@ -322,4 +312,24 @@ list-editable .pc-form-group__text-title {
     align-items: center;
     background: white;
     z-index: 2;
-}
\ No newline at end of file
+}
+
+.config-changes-guard__details {
+    cursor: pointer;
+
+    summary {
+        list-style: none;
+    }
+
+    summary::-webkit-details-marker {
+        display: none;
+    }
+
+    summary:before {
+        content: '▶ ';
+    }
+
+    &[open] summary:before {
+        content: '▼ ';
+    }
+}
diff --git a/modules/web-console/frontend/app/components/page-configure/template.pug b/modules/web-console/frontend/app/components/page-configure/template.pug
index bd299bec740cf..86180fa0d4c28 100644
--- a/modules/web-console/frontend/app/components/page-configure/template.pug
+++ b/modules/web-console/frontend/app/components/page-configure/template.pug
@@ -39,7 +39,7 @@ div.pc-content-container
         input(type='checkbox' ng-model='$ctrl.tooltipsVisible')
         div
 
-    ui-view.theme--ignite(
+    ui-view.theme--ignite.theme--ignite-errors-horizontal(
         ignite-loading='configuration'
         ignite-loading-text='{{ $ctrl.loadingText }}'
         ignite-loading-position='top'
diff --git a/modules/web-console/frontend/app/components/page-password-reset/controller.js b/modules/web-console/frontend/app/components/page-password-reset/controller.js
index 6b402282e31b5..53d6c69393a95 100644
--- a/modules/web-console/frontend/app/components/page-password-reset/controller.js
+++ b/modules/web-console/frontend/app/components/page-password-reset/controller.js
@@ -20,9 +20,9 @@ export default class {
 
     /**
      * @param {mgcrea.ngStrap.modal.IModalService} $modal
-     * @param $http
-     * @param {StateProvider} $state
-     * @param Messages
+     * @param {ng.IHttpService} $http
+     * @param {import('@uirouter/angularjs').StateService} $state
+     * @param {ReturnType<typeof import('app/services/Messages.service').default>} Messages
      */
     constructor($modal, $http, $state, Messages) {
         this.$http = $http;
diff --git a/modules/web-console/frontend/app/components/page-profile/controller.js b/modules/web-console/frontend/app/components/page-profile/controller.js
index 2c786f029b1ee..8586656743991 100644
--- a/modules/web-console/frontend/app/components/page-profile/controller.js
+++ b/modules/web-console/frontend/app/components/page-profile/controller.js
@@ -22,8 +22,27 @@ export default class PageProfileController {
         '$rootScope', '$scope', '$http', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteFocus', 'IgniteConfirm', 'IgniteCountries', 'User'
     ];
 
+    /**
+     * @param {ng.IRootScopeService} $root       
+     * @param {ng.IScope} $scope      
+     * @param {ng.IHttpService} $http       
+     * @param {ReturnType<typeof import('app/services/LegacyUtils.service').default>} LegacyUtils
+     * @param {ReturnType<typeof import('app/services/Messages.service').default>} Messages
+     * @param {ReturnType<typeof import('app/services/Focus.service').default>} Focus
+     * @param {import('app/services/Confirm.service').Confirm} Confirm
+     * @param {ReturnType<typeof import('app/services/Countries.service').default>} Countries
+     * @param {ReturnType<typeof import('app/modules/user/User.service').default>} User
+     */
     constructor($root, $scope, $http, LegacyUtils, Messages, Focus, Confirm, Countries, User) {
-        Object.assign(this, {$root, $scope, $http, LegacyUtils, Messages, Focus, Confirm, Countries, User});
+        this.$root = $root;
+        this.$scope = $scope;
+        this.$http = $http;
+        this.LegacyUtils = LegacyUtils;
+        this.Messages = Messages;
+        this.Focus = Focus;
+        this.Confirm = Confirm;
+        this.Countries = Countries;
+        this.User = User;
     }
 
     $onInit() {
diff --git a/modules/web-console/frontend/app/components/page-profile/template.pug b/modules/web-console/frontend/app/components/page-profile/template.pug
index 2ec6c90c78386..3c5fb52c980ba 100644
--- a/modules/web-console/frontend/app/components/page-profile/template.pug
+++ b/modules/web-console/frontend/app/components/page-profile/template.pug
@@ -118,7 +118,7 @@ div
             .col-100
                 panel-collapsible(
                     opened='$ctrl.ui.expandedPassword'
-                    on-open='$ctrl.ui.expandedToken = false'
+                    on-open='$ctrl.ui.expandedPassword = true'
                     on-close='$ctrl.onPasswordPanelClose()'
                 )
                     panel-title
diff --git a/modules/web-console/frontend/app/components/page-queries/component.js b/modules/web-console/frontend/app/components/page-queries/component.js
index 056b12014b59d..e0c1083b8e6da 100644
--- a/modules/web-console/frontend/app/components/page-queries/component.js
+++ b/modules/web-console/frontend/app/components/page-queries/component.js
@@ -27,8 +27,17 @@ export default {
     controller: class Ctrl {
         static $inject = ['$element', '$rootScope', '$state', 'IgniteNotebook'];
 
+        /**
+         * @param {JQLite} $element       
+         * @param {ng.IRootScopeService} $rootScope     
+         * @param {import('@uirouter/angularjs').StateService} $state         
+         * @param {import('./notebook.service').default} IgniteNotebook
+         */
         constructor($element, $rootScope, $state, IgniteNotebook) {
-            Object.assign(this, {$element, $rootScope, $state, IgniteNotebook});
+            this.$element = $element;
+            this.$rootScope = $rootScope;
+            this.$state = $state;
+            this.IgniteNotebook = IgniteNotebook;
         }
 
         $onInit() {
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
index 87da4fc607304..3ee08b31a9e56 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
@@ -251,16 +251,16 @@ class Paragraph {
 
 // Controller for SQL notebook screen.
 export class NotebookCtrl {
-    static $inject = ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'AgentManager', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData', 'JavaTypes', 'IgniteCopyToClipboard', CSV.name, 'IgniteErrorParser'];
+    static $inject = ['$rootScope', '$scope', '$http', '$q', '$timeout', '$interval', '$animate', '$location', '$anchorScroll', '$state', '$filter', '$modal', '$popover', 'IgniteLoading', 'IgniteLegacyUtils', 'IgniteMessages', 'IgniteConfirm', 'AgentManager', 'IgniteChartColors', 'IgniteNotebook', 'IgniteNodes', 'uiGridExporterConstants', 'IgniteVersion', 'IgniteActivitiesData', 'JavaTypes', 'IgniteCopyToClipboard', 'CSV', 'IgniteErrorParser', 'DemoInfo'];
 
     /**
      * @param {CSV} CSV
      */
-    constructor($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMgr, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes, IgniteCopyToClipboard, CSV, errorParser) {
+    constructor($root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMgr, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes, IgniteCopyToClipboard, CSV, errorParser, DemoInfo) {
         const $ctrl = this;
 
         this.CSV = CSV;
-        Object.assign(this, { $root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMgr, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes, errorParser });
+        Object.assign(this, { $root, $scope, $http, $q, $timeout, $interval, $animate, $location, $anchorScroll, $state, $filter, $modal, $popover, Loading, LegacyUtils, Messages, Confirm, agentMgr, IgniteChartColors, Notebook, Nodes, uiGridExporterConstants, Version, ActivitiesData, JavaTypes, errorParser, DemoInfo });
 
         // Define template urls.
         $ctrl.paragraphRateTemplateUrl = paragraphRateTemplateUrl;
@@ -283,7 +283,15 @@ export class NotebookCtrl {
 
         $scope.caches = [];
 
-        $scope.pageSizes = [50, 100, 200, 400, 800, 1000];
+        $scope.pageSizesOptions = [
+            {value: 50, label: '50'},
+            {value: 100, label: '100'},
+            {value: 200, label: '200'},
+            {value: 400, label: '400'},
+            {value: 800, label: '800'},
+            {value: 1000, label: '1000'}
+        ];
+
         $scope.maxPages = [
             {label: 'Unlimited', value: 0},
             {label: '1', value: 1},
@@ -980,7 +988,14 @@ export class NotebookCtrl {
                 else
                     $scope.rebuildScrollParagraphs();
             })
-            .then(() => _startWatch())
+            .then(() => {
+                if ($root.IgniteDemoMode && sessionStorage.showDemoInfo !== 'true') {
+                    sessionStorage.showDemoInfo = 'true';
+
+                    this.DemoInfo.show().then(_startWatch);
+                } else
+                    _startWatch();
+            })
             .catch(() => {
                 $scope.notebookLoadFailed = true;
 
@@ -1048,7 +1063,7 @@ export class NotebookCtrl {
             const paragraph = _newParagraph({
                 name: 'Query' + (sz === 0 ? '' : sz),
                 query: '',
-                pageSize: $scope.pageSizes[1],
+                pageSize: $scope.pageSizesOptions[1].value,
                 timeLineSpan: $scope.timeLineSpans[0],
                 result: 'none',
                 rate: {
@@ -1077,7 +1092,7 @@ export class NotebookCtrl {
             const paragraph = _newParagraph({
                 name: 'Scan' + (sz === 0 ? '' : sz),
                 query: '',
-                pageSize: $scope.pageSizes[1],
+                pageSize: $scope.pageSizesOptions[1].value,
                 timeLineSpan: $scope.timeLineSpans[0],
                 result: 'none',
                 rate: {
@@ -1556,8 +1571,8 @@ export class NotebookCtrl {
             if (!$scope.queryAvailable(paragraph))
                 return;
 
-            Notebook.save($scope.notebook)
-                .catch(Messages.showError);
+            if (!paragraph.partialQuery)
+                Notebook.save($scope.notebook).catch(Messages.showError);
 
             _cancelRefresh(paragraph);
 
@@ -1569,7 +1584,7 @@ export class NotebookCtrl {
                     const args = paragraph.queryArgs = {
                         type: 'EXPLAIN',
                         cacheName: $scope.cacheNameForSql(paragraph),
-                        query: 'EXPLAIN ' + paragraph.query,
+                        query: 'EXPLAIN ' + (paragraph.partialQuery || paragraph.query),
                         pageSize: paragraph.pageSize
                     };
 
@@ -1956,11 +1971,8 @@ export class NotebookCtrl {
         };
     }
 
-    $onInit() {
-
-    }
-
     $onDestroy() {
-        this.refresh$.unsubscribe();
+        if (this.refresh$)
+            this.refresh$.unsubscribe();
     }
 }
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss
index a5fd50ab46124..fe0ede1b962e5 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/style.scss
@@ -121,4 +121,56 @@ queries-notebook {
     .btn.btn-default.select-toggle.tipLabel {
         padding-right: 25px;
     }
+
+    .form-field__sensitive {
+        input[type='checkbox'] {
+            height: 0;
+        }
+
+        input:checked + span {
+            color: #0067b9;
+        }
+    }
+
+    .queries-notebook-displayed-caches {
+        max-height: 210px;
+        padding: 0 5px;
+        margin-top: 10px;
+        margin-left: -5px;
+        margin-right: -5px;
+
+        overflow-y: auto;
+    }
+}
+
+.popover.settings.refresh-rate {
+    width: 244px;
+
+    [ignite-icon] {
+        height: 12px;
+    }
+
+    .popover-title {
+        padding: 10px;
+        font-size: 14px;
+    }
+
+    .actions {
+        width: 100%;
+        text-align: right;
+
+        button {
+            margin-top: 20px;
+            margin-right: 0;
+        }
+    }
+
+    .ignite-form-field {
+        display: flex;
+        padding: 5px;
+
+        input {
+            margin-right: 10px;
+        }
+    }
 }
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
index 713f83e540f35..781ce5190268f 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
@@ -16,6 +16,24 @@
 
 include /app/helpers/jade/mixins
 
+mixin form-field__sensitive({ label, modelFilter, modelSensitive, name, placeholder })
+    .form-field.form-field__sensitive.ignite-form-field
+        +form-field__label({ label, name })
+            +form-field__tooltip({ title: 'You can set case sensitive search' })
+        .form-field__control.form-field__control-group
+            +form-field__input({ name, model: modelFilter, placeholder })(
+                type='text'
+            )
+            label.btn-ignite.btn-ignite--secondary
+                +form-field__input({ name: `${ name } + "Sensitive"`, model: modelSensitive, placeholder })(
+                    type='checkbox'
+                )
+                span Cs
+        .form-field__errors(
+            ng-messages=`$input.$error`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
+        )
+
 mixin btn-toolbar(btn, click, tip, focusId)
     i.btn.btn-default.fa(class=btn ng-click=click bs-tooltip='' data-title=tip ignite-on-click-focus=focusId data-trigger='hover' data-placement='bottom')
 
@@ -72,59 +90,120 @@ mixin paragraph-rename
             input.form-control(id='paragraph-name-{{paragraph.id}}' ng-model='paragraph.editName' required ng-click='$event.stopPropagation();' ignite-on-enter='renameParagraph(paragraph, paragraph.editName)' ignite-on-escape='paragraph.edit = false')
 
 mixin query-settings
-    .panel-top-align
-        label.tipLabel(bs-tooltip data-placement='bottom' data-title='Configure periodical execution of last successfully executed query') Refresh rate:
-            button.btn.btn-default.fa.fa-clock-o.tipLabel(ng-class='{"btn-info": paragraph.rate && paragraph.rate.installed}' bs-popover data-template-url='{{ $ctrl.paragraphRateTemplateUrl }}' data-placement='left' data-auto-close='1' data-trigger='click') {{rateAsString(paragraph)}}
-
-        label.tipLabel(bs-tooltip data-placement='bottom' data-title='Max number of rows to show in query result as one page') Page size:
-            button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-select bs-options='item for item in pageSizes')
-
-        label.tipLabel(bs-tooltip data-placement='bottom' data-title='Limit query max results to specified number of pages') Max pages:
-            button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.maxPages' bs-select bs-options='item.value as item.label for item in maxPages')
-
-        .panel-tip-container
-            .row(ng-if='nonCollocatedJoinsAvailable(paragraph)')
-                label.tipLabel(bs-tooltip data-placement='bottom' data-title='Non-collocated joins is a special mode that allow to join data across cluster without collocation.<br/>\
-                    Nested joins are not supported for now.<br/>\
-                    <b>NOTE</b>: In some cases it may consume more heap memory or may take a long time than collocated joins.' data-trigger='hover')
-                    input(type='checkbox' ng-model='paragraph.nonCollocatedJoins')
-                    span Allow non-collocated joins
-            .row(ng-if='collocatedJoinsAvailable(paragraph)')
-                label.tipLabel(bs-tooltip data-placement='bottom' data-title='Used For Optimization Purposes Of Queries With GROUP BY Statements.<br/>\
-                    <b>NOTE:</b> Whenever Ignite executes a distributed query, it sends sub-queries to individual cluster members.<br/>\
-                    If you know in advance that the elements of your query selection are collocated together on the same node\
-                    and you group by collocated key (primary or affinity key), then Ignite can make significant performance and\
-                    network optimizations by grouping data on remote nodes.' data-trigger='hover')
-                    input(type='checkbox' ng-model='paragraph.collocated')
-                    span Collocated Query
-            .row(ng-if='enforceJoinOrderAvailable(paragraph)')
-                label.tipLabel(bs-tooltip data-placement='bottom' data-title='Enforce join order of tables in the query.<br/>\
-                    If <b>set</b>, then query optimizer will not reorder tables within join.<br/>\
-                    <b>NOTE:</b> It is not recommended to enable this property unless you have verified that\
-                    indexes are not selected in optimal order.' data-trigger='hover')
-                    input(type='checkbox' ng-model='paragraph.enforceJoinOrder')
-                    span Enforce join order
-            .row(ng-if='lazyQueryAvailable(paragraph)')
-                label.tipLabel(bs-tooltip data-placement='bottom' data-title='By default Ignite attempts to fetch the whole query result set to memory and send it to the client.<br/>\
-                    For small and medium result sets this provides optimal performance and minimize duration of internal database locks, thus increasing concurrency.<br/>\
-                    If result set is too big to fit in available memory this could lead to excessive GC pauses and even OutOfMemoryError.<br/>\
-                    Use this flag as a hint for Ignite to fetch result set lazily, thus minimizing memory consumption at the cost of moderate performance hit.' data-trigger='hover')
-                    input(type='checkbox' ng-model='paragraph.lazy')
-                    span Lazy result set
+    div
+        .form-field--inline(
+            bs-tooltip
+            data-placement='top'
+            data-title='Max number of rows to show in query result as one page'
+        )
+            +form-field__dropdown({
+                label: 'Rows per page:',
+                model: 'paragraph.pageSize',
+                name: '"pageSize" + paragraph.id',
+                options: 'pageSizesOptions'
+            })
+
+        .form-field--inline(
+            bs-tooltip
+            data-placement='top'
+            data-title='Limit query max results to specified number of pages'
+        )
+            +form-field__dropdown({
+                label: 'Max pages:',
+                model: 'paragraph.maxPages',
+                name: '"maxPages" + paragraph.id',
+                options: 'maxPages'
+            })
+
+        .form-field--inline(
+            bs-tooltip
+            data-placement='bottom'
+            data-title='Configure periodical execution of last successfully executed query'
+        )
+            button.btn-ignite-group(
+                bs-popover
+                data-template-url='{{ $ctrl.paragraphRateTemplateUrl }}'
+                data-placement='bottom-right'
+                data-auto-close='1'
+                data-trigger='click'
+            )
+                .btn-ignite(
+                    ng-class='{\
+                        "btn-ignite--primary": paragraph.rate && paragraph.rate.installed,\
+                        "btn-ignite--secondary": !(paragraph.rate && paragraph.rate.installed),\
+                    }'
+                )
+                    svg(ignite-icon='clock')
+                    | &nbsp; {{ rateAsString(paragraph) }}
+                .btn-ignite(
+                    ng-class='{\
+                        "btn-ignite--primary": paragraph.rate && paragraph.rate.installed,\
+                        "btn-ignite--secondary": !(paragraph.rate && paragraph.rate.installed),\
+                    }'
+                )
+                    span.icon.fa.fa-caret-down
+    div
+        .row(ng-if='nonCollocatedJoinsAvailable(paragraph)')
+            +form-field__checkbox({
+                label: 'Allow non-collocated joins',
+                model: 'paragraph.nonCollocatedJoins',
+                name: '"nonCollocatedJoins" + paragraph.id',
+                tip: 'Non-collocated joins is a special mode that allow to join data across cluster without collocation.<br/>\
+                Nested joins are not supported for now.<br/>\
+                <b>NOTE</b>: In some cases it may consume more heap memory or may take a long time than collocated joins.',
+                tipOpts: { placement: 'top' }
+            })
+
+        .row(ng-if='collocatedJoinsAvailable(paragraph)')
+            +form-field__checkbox({
+                label: 'Collocated Query',
+                model: 'paragraph.collocated',
+                name: '"collocated" + paragraph.id',
+                tip: 'Used For Optimization Purposes Of Queries With GROUP BY Statements.<br/>\
+                <b>NOTE:</b> Whenever Ignite executes a distributed query, it sends sub-queries to individual cluster members.<br/>\
+                If you know in advance that the elements of your query selection are collocated together on the same node\
+                and you group by collocated key (primary or affinity key), then Ignite can make significant performance and\
+                network optimizations by grouping data on remote nodes.',
+                tipOpts: { placement: 'top' }
+            })
+
+        .row(ng-if='enforceJoinOrderAvailable(paragraph)')
+            +form-field__checkbox({
+                label: 'Enforce join order',
+                model: 'paragraph.enforceJoinOrder',
+                name: '"enforceJoinOrder" + paragraph.id',
+                tip: 'Enforce join order of tables in the query.<br/>\
+                If <b>set</b>, then query optimizer will not reorder tables within join.<br/>\
+                <b>NOTE:</b> It is not recommended to enable this property unless you have verified that\
+                indexes are not selected in optimal order.',
+                tipOpts: { placement: 'top' }
+            })
+
+        .row(ng-if='lazyQueryAvailable(paragraph)')
+            +form-field__checkbox({
+                label: 'Lazy result set',
+                model: 'paragraph.lazy',
+                name: '"lazy" + paragraph.id',
+                tip: 'By default Ignite attempts to fetch the whole query result set to memory and send it to the client.<br/>\
+                For small and medium result sets this provides optimal performance and minimize duration of internal database locks, thus increasing concurrency.<br/>\
+                If result set is too big to fit in available memory this could lead to excessive GC pauses and even OutOfMemoryError.<br/>\
+                Use this flag as a hint for Ignite to fetch result set lazily, thus minimizing memory consumption at the cost of moderate performance hit.',
+                tipOpts: { placement: 'top' }
+            })
 
 mixin query-actions
-    button.btn.btn-primary(ng-disabled='!queryAvailable(paragraph)' ng-click='execute(paragraph)')
-        div
-            i.fa.fa-fw.fa-play(ng-hide='paragraph.executionInProgress(false)')
-            i.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.executionInProgress(false)')
-            span.tipLabelExecute Execute
-    button.btn.btn-primary(ng-disabled='!queryAvailable(paragraph)' ng-click='execute(paragraph, true)')
-        div
-            i.fa.fa-fw.fa-play(ng-hide='paragraph.executionInProgress(true)')
-            i.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.executionInProgress(true)')
-            span.tipLabelExecute Execute on selected node
-
-    a.btn.btn-default(ng-disabled='!queryAvailable(paragraph)' ng-click='explain(paragraph)' data-placement='bottom' bs-tooltip='' data-title='{{queryTooltip(paragraph, "explain query")}}') Explain
+    button.btn-ignite.btn-ignite--primary(ng-disabled='!queryAvailable(paragraph)' ng-click='execute(paragraph)')
+        span.icon-left.fa.fa-fw.fa-play(ng-hide='paragraph.executionInProgress(false)')
+        span.icon-left.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.executionInProgress(false)')
+        | Execute
+
+    button.btn-ignite.btn-ignite--primary(ng-disabled='!queryAvailable(paragraph)' ng-click='execute(paragraph, true)')
+        span.icon-left.fa.fa-fw.fa-play(ng-hide='paragraph.executionInProgress(true)')
+        span.icon-left.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.executionInProgress(true)')
+        | Execute on selected node
+
+    button.btn-ignite.btn-ignite--secondary(ng-disabled='!queryAvailable(paragraph)' ng-click='explain(paragraph)' data-placement='bottom' bs-tooltip='' data-title='{{queryTooltip(paragraph, "explain query")}}')
+        | Explain
 
 mixin table-result-heading-query
     .total.row
@@ -241,26 +320,44 @@ mixin paragraph-scan
     .panel-collapse(role='tabpanel' bs-collapse-target)
         .col-sm-12.sql-controls
             .col-sm-3
-                +dropdown-required('Cache:', 'paragraph.cacheName', '"cache"', 'true', 'false', 'Choose cache', 'caches')
+                +form-field__dropdown({
+                    label: 'Cache:',
+                    model: 'paragraph.cacheName',
+                    name: '"cache"',
+                    placeholder: 'Choose cache',
+                    options: 'caches'
+                })
             .col-sm-3
-                +text-enabled('Filter:', 'paragraph.filter', '"filter"', true, false, 'Enter filter')
-                    label.btn.btn-default.ignite-form-field__btn(ng-click='paragraph.caseSensitive = !paragraph.caseSensitive')
-                        input(type='checkbox' ng-model='paragraph.caseSensitive')
-                        span(bs-tooltip data-title='Select this checkbox for case sensitive search') Cs
-            label.tipLabel(bs-tooltip data-placement='bottom' data-title='Max number of rows to show in query result as one page') Page size:
-                button.btn.btn-default.select-toggle.tipLabel(ng-model='paragraph.pageSize' bs-select bs-options='item for item in pageSizes')
+                +form-field__sensitive({
+                    label: 'Filter:',
+                    modelFilter: 'paragraph.filter',
+                    modelSensitive: 'paragraph.caseSensitive',
+                    name: '"filter"',
+                    placeholder: 'Enter filter'
+                })
 
-        .col-sm-12.sql-controls
-            button.btn.btn-primary(ng-disabled='!scanAvailable(paragraph)' ng-click='scan(paragraph)')
-                div
-                    i.fa.fa-fw.fa-play(ng-hide='paragraph.checkScanInProgress(false)')
-                    i.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.checkScanInProgress(false)')
-                    span.tipLabelExecute Scan
+            .col-sm-3
+                +form-field__dropdown({
+                    label: 'Rows per page:',
+                    model: 'paragraph.pageSize',
+                    name: '"pageSize" + paragraph.id',
+                    options: 'pageSizesOptions',
+                    tip: 'Max number of rows to show in query result as one page',
+                    tipOpts: { placement: 'top' }
+                })
 
-            button.btn.btn-primary(ng-disabled='!scanAvailable(paragraph)' ng-click='scan(paragraph, true)')
-                i.fa.fa-fw.fa-play(ng-hide='paragraph.checkScanInProgress(true)')
-                i.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.checkScanInProgress(true)')
-                span.tipLabelExecute Scan on selected node
+        .col-sm-12.sql-controls
+            div
+                button.btn-ignite.btn-ignite--primary(ng-disabled='!scanAvailable(paragraph)' ng-click='scan(paragraph)')
+                    span.icon-left.fa.fa-fw.fa-play(ng-hide='paragraph.checkScanInProgress(false)')
+                    span.icon-left.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.checkScanInProgress(false)')
+                    | Scan
+
+                button.btn-ignite.btn-ignite--primary(ng-disabled='!scanAvailable(paragraph)' ng-click='scan(paragraph, true)')
+                    span.icon-left.fa.fa-fw.fa-play(ng-hide='paragraph.checkScanInProgress(true)')
+                    span.icon-left.fa.fa-fw.fa-refresh.fa-spin(ng-show='paragraph.checkScanInProgress(true)')
+                    | Scan on selected node
+            div
 
         .col-sm-12.sql-result(ng-if='paragraph.queryExecuted() && !paragraph.scanningInProgress' ng-switch='paragraph.resultType()')
             .error(ng-switch-when='error') Error: {{paragraph.error.message}}
@@ -295,31 +392,36 @@ mixin paragraph-query
                     i.fa.fa-database.tipField(title='Click to show cache types metadata dialog' bs-popover data-template-url='{{ $ctrl.cacheMetadataTemplateUrl }}' data-placement='bottom-right' data-trigger='click' data-container='#{{ paragraph.id }}')
                     .input-tip
                         input.form-control(type='text' st-search='label' placeholder='Filter caches...')
-                    table.links
-                        tbody.scrollable-y(style='max-height: 15em; display: block;')
-                            tr(ng-repeat='cache in displayedCaches track by cache.name')
-                                td(style='width: 100%')
-                                    input.labelField(id='cache_{{ [paragraph.id, $index].join("_") }}' type='radio' value='{{cache.name}}' ng-model='paragraph.cacheName')
-                                    label(for='cache_{{ [paragraph.id, $index].join("_") }} ' ng-bind-html='cache.label')
+
+                    .queries-notebook-displayed-caches
+                        div(ng-repeat='cache in displayedCaches track by cache.name')
+                            +form-field__radio({
+                                label: '{{ cache.label }}',
+                                model: 'paragraph.cacheName',
+                                name: '"cache_" + [paragraph.id, $index].join("_")',
+                                value: 'cache.name'
+                            })
+
                     .settings-row
                         .row(ng-if='ddlAvailable(paragraph)')
-                            label.tipLabel.use-cache(bs-tooltip data-placement='bottom'
-                                data-title=
-                                    'Use selected cache as default schema name.<br/>\
+                            +form-field__checkbox({
+                                label: 'Use selected cache as default schema name',
+                                model: 'paragraph.useAsDefaultSchema',
+                                name: '"useAsDefaultSchema" + paragraph.id',
+                                tip: 'Use selected cache as default schema name.<br/>\
                                     This will allow to execute query on specified cache without specify schema name.<br/>\
-                                    <b>NOTE:</b> In future version of Ignite this feature will be removed.'
-                                data-trigger='hover')
-                                input(type='checkbox' ng-model='paragraph.useAsDefaultSchema')
-                                span Use selected cache as default schema name
+                                    <b>NOTE:</b> In future version of Ignite this feature will be removed.',
+                                tipOpts: { placement: 'top' }
+                            })
                 .empty-caches(ng-show='displayedCaches.length == 0 && caches.length != 0')
                     label Wrong caches filter
                 .empty-caches(ng-show='caches.length == 0')
                     label No caches
         .col-sm-12.sql-controls
-            +query-actions
+            div
+                +query-actions
 
-            .pull-right
-                +query-settings
+            +query-settings
         .col-sm-12.sql-result(ng-if='paragraph.queryExecuted()' ng-switch='paragraph.resultType()')
             .error(ng-switch-when='error')
                 label Error: {{paragraph.error.message}}
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/template.tpl.pug
index 614a6a6d067bd..bbf6df756d36b 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebooks-list/template.tpl.pug
@@ -27,22 +27,18 @@ page-queries-slot(slot-name="'queriesButtons'" ng-if="!$root.IgniteDemoMode")
 
 .queries-notebooks-list
     .panel--ignite
-        .panel-heading.ui-grid-settings.ui-grid-ignite__panel
-            .panel-title
-                div(ng-if="!$root.IgniteDemoMode")
-                    +ignite-form-field-bsdropdown({
-                        label: 'Actions',
-                        model: '$ctrl.action',
-                        name: 'action',
-                        disabled: '$ctrl.gridApi.selection.legacyGetSelectedRows().length === 0',
-                        required: false,
-                        options: '$ctrl.actionOptions'
-                    })
-
-                .ui-grid-settings--heading
-                    span Notebooks
-
-
+        header.header-with-selector
+            div
+                span Notebooks
+
+            div(ng-if="!$root.IgniteDemoMode")
+                +ignite-form-field-bsdropdown({
+                    label: 'Actions',
+                    model: '$ctrl.action',
+                    name: 'action',
+                    disabled: '$ctrl.gridApi.selection.legacyGetSelectedRows().length === 0',
+                    options: '$ctrl.actionOptions'
+                })
 
         .panel-collapse(ignite-loading='notebooksLoading' ignite-loading-text='Loading notebooks...')
             .grid.ui-grid--ignite#queriesNotebooksList(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-hovering)
diff --git a/modules/web-console/frontend/app/components/page-queries/notebook.data.js b/modules/web-console/frontend/app/components/page-queries/notebook.data.js
index d83101d5ac322..6c784bfc3ea11 100644
--- a/modules/web-console/frontend/app/components/page-queries/notebook.data.js
+++ b/modules/web-console/frontend/app/components/page-queries/notebook.data.js
@@ -80,6 +80,11 @@ const DEMO_NOTEBOOK = {
 export default class NotebookData {
     static $inject = ['$rootScope', '$http', '$q'];
 
+    /**
+     * @param {ng.IRootScopeService} $root 
+     * @param {ng.IHttpService} $http 
+     * @param {ng.IQService} $q    
+     */
     constructor($root, $http, $q) {
         this.demo = $root.IgniteDemoMode;
 
diff --git a/modules/web-console/frontend/app/components/page-queries/notebook.service.js b/modules/web-console/frontend/app/components/page-queries/notebook.service.js
index 300444734e4f6..dbfd6a6febf77 100644
--- a/modules/web-console/frontend/app/components/page-queries/notebook.service.js
+++ b/modules/web-console/frontend/app/components/page-queries/notebook.service.js
@@ -19,10 +19,10 @@ export default class Notebook {
     static $inject = ['$state', 'IgniteConfirm', 'IgniteMessages', 'IgniteNotebookData'];
 
     /**
-     * @param $state
-     * @param confirmModal
-     * @param Messages
-     * @param {NotebookData} NotebookData
+     * @param {import('@uirouter/angularjs').StateService} $state
+     * @param {ReturnType<typeof import('app/services/Confirm.service').default>} confirmModal
+     * @param {ReturnType<typeof import('app/services/Messages.service').default>} Messages
+     * @param {import('./notebook.data').default} NotebookData
      */
     constructor($state, confirmModal, Messages, NotebookData) {
         this.$state = $state;
diff --git a/modules/web-console/frontend/app/components/page-queries/style.scss b/modules/web-console/frontend/app/components/page-queries/style.scss
index 8f13c2e563ae8..818fb1cbed58d 100644
--- a/modules/web-console/frontend/app/components/page-queries/style.scss
+++ b/modules/web-console/frontend/app/components/page-queries/style.scss
@@ -18,3 +18,7 @@
 button.select-toggle.btn-chart-column-agg-fx::after {
 	right: 0;
 }
+
+.sql-controls {
+	flex-wrap: wrap;
+}
diff --git a/modules/web-console/frontend/app/components/page-signin/component.js b/modules/web-console/frontend/app/components/page-signin/component.ts
similarity index 100%
rename from modules/web-console/frontend/app/components/page-signin/component.js
rename to modules/web-console/frontend/app/components/page-signin/component.ts
diff --git a/modules/web-console/frontend/app/components/page-signin/controller.js b/modules/web-console/frontend/app/components/page-signin/controller.ts
similarity index 73%
rename from modules/web-console/frontend/app/components/page-signin/controller.js
rename to modules/web-console/frontend/app/components/page-signin/controller.ts
index f2d28e886c07e..724da41dadeb9 100644
--- a/modules/web-console/frontend/app/components/page-signin/controller.js
+++ b/modules/web-console/frontend/app/components/page-signin/controller.ts
@@ -15,30 +15,33 @@
  * limitations under the License.
  */
 
+import AuthService from 'app/modules/user/Auth.service';
+
+interface ISiginData {
+    email: string,
+    password: string
+}
+
+interface ISigninFormController extends ng.IFormController {
+    email: ng.INgModelController,
+    password: ng.INgModelController
+}
+
 export default class {
-    /** @type {import('./types').ISiginData} */
-    data = {
+    data: ISiginData = {
         email: null,
         password: null
     };
-    /** @type {import('./types').ISigninFormController} */
-    form;
-    /** @type {string} */
-    serverError = null;
+
+    form: ISigninFormController;
+
+    serverError: string = null;
 
     static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils'];
 
-    /**
-     * @param {import('app/modules/user/Auth.service').default} Auth
-     */
-    constructor(Auth, IgniteMessages, IgniteFormUtils) {
-        this.Auth = Auth;
-        this.IgniteMessages = IgniteMessages;
-        this.IgniteFormUtils = IgniteFormUtils;
-    }
+    constructor(private Auth: AuthService, private IgniteMessages, private IgniteFormUtils) {}
 
-    /** @param {import('./types').ISigninFormController} form */
-    canSubmitForm(form) {
+    canSubmitForm(form: ISigninFormController) {
         return form.$error.server ? true : !form.$invalid;
     }
 
@@ -47,8 +50,7 @@ export default class {
         this.form.password.$validators.server = () => !this.serverError;
     }
 
-    /** @param {string} error */
-    setServerError(error) {
+    setServerError(error: string) {
         this.serverError = error;
         this.form.email.$validate();
         this.form.password.$validate();
diff --git a/modules/web-console/frontend/app/components/page-signin/index.js b/modules/web-console/frontend/app/components/page-signin/index.ts
similarity index 100%
rename from modules/web-console/frontend/app/components/page-signin/index.js
rename to modules/web-console/frontend/app/components/page-signin/index.ts
diff --git a/modules/web-console/frontend/app/components/page-signin/run.js b/modules/web-console/frontend/app/components/page-signin/run.ts
similarity index 90%
rename from modules/web-console/frontend/app/components/page-signin/run.js
rename to modules/web-console/frontend/app/components/page-signin/run.ts
index 457e488912234..aa5647c44a363 100644
--- a/modules/web-console/frontend/app/components/page-signin/run.js
+++ b/modules/web-console/frontend/app/components/page-signin/run.ts
@@ -15,12 +15,11 @@
  * limitations under the License.
  */
 
-/**
- * @param {import("@uirouter/angularjs").UIRouter} $uiRouter
- */
-export function registerState($uiRouter) {
-    /** @type {import("app/types").IIgniteNg1StateDeclaration} */
-    const state = {
+import {UIRouter} from '@uirouter/angularjs';
+import {IIgniteNg1StateDeclaration} from 'app/types';
+
+export function registerState($uiRouter: UIRouter) {
+    const state: IIgniteNg1StateDeclaration = {
         url: '/signin',
         name: 'signin',
         component: 'pageSignin',
diff --git a/modules/web-console/frontend/app/components/page-signup/controller.js b/modules/web-console/frontend/app/components/page-signup/controller.js
index 57f404a558c74..10397760c3d71 100644
--- a/modules/web-console/frontend/app/components/page-signup/controller.js
+++ b/modules/web-console/frontend/app/components/page-signup/controller.js
@@ -33,7 +33,10 @@ export default class PageSignup {
     static $inject = ['IgniteCountries', 'Auth', 'IgniteMessages', 'IgniteFormUtils'];
 
     /**
+     * @param {ReturnType<typeof import('app/services/Countries.service').default>} Countries
      * @param {import('app/modules/user/Auth.service').default} Auth
+     * @param {ReturnType<typeof import('app/services/Messages.service').default>} IgniteMessages
+     * @param {ReturnType<typeof import('app/services/FormUtils.service').default>} IgniteFormUtils
      */
     constructor(Countries, Auth, IgniteMessages, IgniteFormUtils) {
         this.Auth = Auth;
diff --git a/modules/web-console/frontend/app/components/panel-collapsible/index.spec.js b/modules/web-console/frontend/app/components/panel-collapsible/index.spec.js
index b5c00cd5a3933..94e9def14f069 100644
--- a/modules/web-console/frontend/app/components/panel-collapsible/index.spec.js
+++ b/modules/web-console/frontend/app/components/panel-collapsible/index.spec.js
@@ -19,7 +19,7 @@ import 'mocha';
 import {assert} from 'chai';
 import angular from 'angular';
 import {spy} from 'sinon';
-import componentModule from './index.js';
+import componentModule from './index';
 
 suite('panel-collapsible', () => {
     const ICON_COLLAPSE = 'collapse';
diff --git a/modules/web-console/frontend/app/components/ui-grid-filters/directive.js b/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
index c9b84af4219dc..e22530ba2dfdc 100644
--- a/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
+++ b/modules/web-console/frontend/app/components/ui-grid-filters/directive.js
@@ -27,7 +27,8 @@ export default function uiGridFilters(uiGridConstants) {
                     return;
 
                 const applyMultiselectFilter = (cd) => {
-                    cd.headerCellTemplate = template;
+                    if (!cd.headerCellTemplate)
+                        cd.headerCellTemplate = template;
 
                     cd.filter = {
                         type: uiGridConstants.filter.SELECT,
diff --git a/modules/web-console/frontend/app/modules/form/field/field.scss b/modules/web-console/frontend/app/components/ui-grid/component.js
similarity index 56%
rename from modules/web-console/frontend/app/modules/form/field/field.scss
rename to modules/web-console/frontend/app/components/ui-grid/component.js
index 57177664546e3..f0c3b06309055 100644
--- a/modules/web-console/frontend/app/modules/form/field/field.scss
+++ b/modules/web-console/frontend/app/components/ui-grid/component.js
@@ -15,29 +15,32 @@
  * limitations under the License.
  */
 
-@import "../../../../public/stylesheets/variables";
+import './style.scss';
+import template from './template.pug';
+import controller from './controller';
 
-.indexField {
-    float: left;
-    line-height: 28px;
-    margin-right: 5px;
-    color: $brand-primary;
-}
+export default {
+    template,
+    controller,
+    bindings: {
+        gridApi: '=?',
+        gridTreeView: '<?',
+        gridGrouping: '<?',
+        gridThin: '<?',
+        gridHeight: '<?',
+        tabName: '<?',
+        tableTitle: '<?',
 
-.form-field-save {
-    position: relative;
-    width: 0;
-    height: 28px;
-    float: right;
-    z-index: 2;
+        // Input Events.
+        items: '<',
+        columnDefs: '<',
+        categories: '<?',
+        oneWaySelection: '<?',
+        rowIdentityKey: '@?',
+        selectedRows: '<?',
+        selectedRowsId: '<?',
 
-    line-height: $input-height;
-    pointer-events: initial;
-    text-align: center;
-
-    &:before {
-        position: absolute;
-        right: 0;
-        width: 38px;
-    }    
-}
+        // Output events.
+        onSelectionChange: '&?'
+    }
+};
diff --git a/modules/web-console/frontend/app/components/ui-grid/controller.js b/modules/web-console/frontend/app/components/ui-grid/controller.js
new file mode 100644
index 0000000000000..a6406044f10df
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid/controller.js
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import debounce from 'lodash/debounce';
+import headerTemplate from 'app/primitives/ui-grid-header/index.tpl.pug';
+
+export default class IgniteUiGrid {
+    /** @type */
+    gridApi;
+
+    /** @type */
+    gridThin;
+
+    /** @type */
+    gridHeight;
+
+    /** @type */
+    items;
+
+    /** @type */
+    columnDefs;
+
+    /** @type */
+    categories;
+
+    /** @type */
+    onSelectionChange;
+
+    /** @type */
+    selectedRows;
+
+    /** @type */
+    selectedRowsId;
+
+    /** @type */
+    _selected;
+
+    static $inject = ['$scope', '$element', '$timeout', 'gridUtil'];
+
+    /**
+     * @param {ng.IScope} $scope
+     */
+    constructor($scope, $element, $timeout, gridUtil) {
+        this.$scope = $scope;
+        this.$element = $element;
+        this.$timeout = $timeout;
+        this.gridUtil = gridUtil;
+
+        this.rowIdentityKey = '_id';
+
+        this.rowHeight = 48;
+        this.headerRowHeight = 70;
+    }
+
+    $onInit() {
+        this.SCROLLBAR_WIDTH = this.gridUtil.getScrollbarWidth();
+
+        if (this.gridThin) {
+            this.rowHeight = 36;
+            this.headerRowHeight = 48;
+        }
+
+        this.grid = {
+            data: this.items,
+            columnDefs: this.columnDefs,
+            categories: this.categories,
+            rowHeight: this.rowHeight,
+            headerRowHeight: this.headerRowHeight,
+            columnVirtualizationThreshold: 30,
+            enableColumnMenus: false,
+            enableFullRowSelection: true,
+            enableFiltering: true,
+            enableRowHashing: false,
+            fastWatch: true,
+            showTreeExpandNoChildren: false,
+            modifierKeysToMultiSelect: true,
+            selectionRowHeaderWidth: 52,
+            exporterCsvColumnSeparator: ';',
+            onRegisterApi: (api) => {
+                this.gridApi = api;
+
+                api.core.on.rowsVisibleChanged(this.$scope, () => {
+                    this.adjustHeight();
+
+                    // Without property existence check non-set selectedRows or selectedRowsId
+                    // binding might cause unwanted behavior,
+                    // like unchecking rows during any items change,
+                    // even if nothing really changed.
+                    if (this._selected && this._selected.length && this.onSelectionChange) {
+                        this.applyIncomingSelectionRows(this._selected);
+
+                        // Change selected rows if filter was changed.
+                        this.onRowsSelectionChange([]);
+                    }
+                });
+
+                if (this.onSelectionChange) {
+                    api.selection.on.rowSelectionChanged(this.$scope, (row, e) => {
+                        this.onRowsSelectionChange([row], e);
+                    });
+
+                    api.selection.on.rowSelectionChangedBatch(this.$scope, (rows, e) => {
+                        this.onRowsSelectionChange(rows, e);
+                    });
+                }
+
+                api.core.on.filterChanged(this.$scope, (column) => {
+                    this.onFilterChange(column);
+                });
+
+                this.$timeout(() => {
+                    if (this.selectedRowsId) this.applyIncomingSelectionRowsId(this.selectedRowsId);
+                });
+            }
+        };
+
+        if (this.grid.categories)
+            this.grid.headerTemplate = headerTemplate;
+    }
+
+    $onChanges(changes) {
+        const hasChanged = (binding) =>
+            binding in changes && changes[binding].currentValue !== changes[binding].previousValue;
+
+        if (hasChanged('items') && this.grid)
+            this.grid.data = changes.items.currentValue;
+
+        if (hasChanged('selectedRows') && this.grid && this.grid.data && this.onSelectionChange)
+            this.applyIncomingSelectionRows(changes.selectedRows.currentValue);
+
+        if (hasChanged('selectedRowsId') && this.grid && this.grid.data)
+            this.applyIncomingSelectionRowsId(changes.selectedRowsId.currentValue);
+
+        if (hasChanged('gridHeight') && this.grid)
+            this.adjustHeight();
+    }
+
+    applyIncomingSelectionRows = (selected = []) => {
+        this.gridApi.selection.clearSelectedRows({ ignore: true });
+
+        const visibleRows = this.gridApi.core.getVisibleRows(this.gridApi.grid)
+            .map(({ entity }) => entity);
+
+        const rows = visibleRows.filter((r) =>
+            selected.map((row) => row[this.rowIdentityKey]).includes(r[this.rowIdentityKey]));
+
+        rows.forEach((r) => {
+            this.gridApi.selection.selectRow(r, { ignore: true });
+        });
+    };
+
+    applyIncomingSelectionRowsId = (selected = []) => {
+        if (this.onSelectionChange) {
+            this.gridApi.selection.clearSelectedRows({ ignore: true });
+
+            const visibleRows = this.gridApi.core.getVisibleRows(this.gridApi.grid)
+                .map(({ entity }) => entity);
+
+            const rows = visibleRows.filter((r) =>
+                selected.includes(r[this.rowIdentityKey]));
+
+            rows.forEach((r) => {
+                this.gridApi.selection.selectRow(r, { ignore: true });
+            });
+        }
+    };
+
+    onRowsSelectionChange = debounce((rows, e = {}) => {
+        if (e.ignore)
+            return;
+
+        this._selected = this.gridApi.selection.legacyGetSelectedRows();
+
+        if (this.onSelectionChange)
+            this.onSelectionChange({ $event: this._selected });
+    });
+
+    onFilterChange = debounce((column) => {
+        if (!this.gridApi.selection)
+            return;
+
+        if (this.selectedRows && this.onSelectionChange)
+            this.applyIncomingSelectionRows(this.selectedRows);
+
+        if (this.selectedRowsId)
+            this.applyIncomingSelectionRowsId(this.selectedRowsId);
+    });
+
+    adjustHeight() {
+        let height = this.gridHeight;
+
+        if (!height) {
+            const maxRowsToShow = this.maxRowsToShow || 5;
+            const headerBorder = 1;
+            const visibleRows = this.gridApi.core.getVisibleRows().length;
+            const header = this.grid.headerRowHeight + headerBorder;
+            const optionalScroll = (visibleRows ? this.gridUtil.getScrollbarWidth() : 0);
+
+            height = Math.min(visibleRows, maxRowsToShow) * this.grid.rowHeight + header + optionalScroll;
+        }
+
+        this.gridApi.grid.element.css('height', height + 'px');
+        this.gridApi.core.handleWindowResize();
+    }
+}
diff --git a/modules/web-console/frontend/app/modules/form/field/up.directive.js b/modules/web-console/frontend/app/components/ui-grid/decorator.js
similarity index 50%
rename from modules/web-console/frontend/app/modules/form/field/up.directive.js
rename to modules/web-console/frontend/app/components/ui-grid/decorator.js
index 6f87180b67929..a82f702abfddf 100644
--- a/modules/web-console/frontend/app/modules/form/field/up.directive.js
+++ b/modules/web-console/frontend/app/components/ui-grid/decorator.js
@@ -15,29 +15,24 @@
  * limitations under the License.
  */
 
-export default ['igniteFormFieldUp', ['$tooltip', ($tooltip) => {
-    const controller = ['$element', function($element) {
-        const ctrl = this;
+export default ['$delegate', 'uiGridSelectionService', ($delegate, uiGridSelectionService) => {
+    $delegate[0].require = ['^uiGrid', '?^igniteGridTable'];
+    $delegate[0].compile = () => ($scope, $el, $attr, [uiGridCtrl, igniteGridTable]) => {
+        const self = uiGridCtrl.grid;
 
-        this.$onInit = () => {
-            $tooltip($element, { title: 'Move item up' });
+        $delegate[0].link($scope, $el, $attr, uiGridCtrl);
 
-            this.up = () => {
-                const idx = ctrl.models.indexOf(ctrl.model);
+        const mySelectButtonClick = (row, evt) => {
+            evt.stopPropagation();
 
-                ctrl.models.splice(idx, 1);
-                ctrl.models.splice(idx - 1, 0, ctrl.model);
-            };
+            if (evt.shiftKey)
+                uiGridSelectionService.shiftSelect(self, row, evt, self.options.multiSelect);
+            else
+                uiGridSelectionService.toggleRowSelection(self, row, evt, self.options.multiSelect, self.options.noUnselect);
         };
-    }];
 
-    return {
-        restrict: 'A',
-        bindToController: {
-            model: '=ngModel',
-            models: '=models'
-        },
-        controller,
-        controllerAs: 'vm'
+        if (igniteGridTable)
+            $scope.selectButtonClick = mySelectButtonClick;
     };
-}]];
+    return $delegate;
+}];
diff --git a/modules/web-console/frontend/app/components/ui-grid/index.js b/modules/web-console/frontend/app/components/ui-grid/index.js
new file mode 100644
index 0000000000000..fce526846527b
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid/index.js
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import angular from 'angular';
+import component from './component';
+import decorator from './decorator';
+
+export default angular
+    .module('ignite-console.ui-grid', [])
+    .component('igniteGridTable', component)
+    .decorator('uiGridSelectionRowHeaderButtonsDirective', decorator);
diff --git a/modules/web-console/frontend/app/components/ui-grid/style.scss b/modules/web-console/frontend/app/components/ui-grid/style.scss
new file mode 100644
index 0000000000000..dba5d359ea290
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid/style.scss
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.ignite-grid-table,
+ignite-grid-table {
+	@import 'public/stylesheets/variables';
+
+    .ui-grid.ui-grid--ignite.ui-grid--thin {
+        // Start section row height.
+        .ui-grid-row {
+            height: 36px;
+
+            .ui-grid-cell {
+                height: 100%;
+            }
+        }
+
+        .ui-grid-cell .ui-grid-cell-contents {
+            padding: 8px 20px;
+            min-height: 35px;
+            max-height: 35px;
+        }
+
+        // Set force header height.
+        // Fix hide border bottom of pinned column without data.
+        .ui-grid-header-canvas {
+            height: 48px;
+        }
+    }
+
+    .ui-grid.ui-grid--ignite.ui-grid--thin-rows {
+        .ui-grid-row {
+            height: 36px !important;
+
+            .ui-grid-cell {
+                height: 100% !important;
+            }
+        }
+
+        .ui-grid-cell .ui-grid-cell-contents {
+            padding: 8px 20px !important;
+            min-height: 35px !important;
+            max-height: 35px !important;
+        }
+    }
+
+    .ui-grid.ui-grid--ignite:not(.ui-grid--thin) {
+        // Start section row height.
+        .ui-grid-row {
+            height: 48px;
+
+            .ui-grid-cell {
+                height: 100%;
+            }
+        }
+
+        .ui-grid-cell .ui-grid-cell-contents {
+            padding: 14px 20px;
+            min-height: 47px;
+            max-height: 47px;
+        }
+
+        // Set force header height.
+        // Fix hide border bottom of pinned column without data.
+        .ui-grid-header-canvas {
+            height: 70px;
+        }
+
+        [role="columnheader"] {
+            margin: 11px 0;
+        }
+
+        // Fix checkbox position.
+        .ui-grid-header-cell  .ui-grid-selection-row-header-buttons {
+            margin-top: 12px;
+        }
+    }
+
+    .ui-grid.ui-grid--ignite {
+        .ui-grid-header .ui-grid-tree-base-row-header-buttons.ui-grid-icon-plus-squared,
+        .ui-grid-header .ui-grid-tree-base-row-header-buttons.ui-grid-icon-minus-squared {
+            top: 14px;
+        }
+
+        [role="columnheader"] {
+            display: flex;
+            align-items: center;
+        }
+
+        .ui-grid-header--subcategories [role="columnheader"] {
+            margin: 0;
+            background-color: white;
+        }
+
+        // Removes unwanted box-shadow and border-right from checkboxes column
+        .ui-grid-pinned-container.ui-grid-pinned-container-left .ui-grid-render-container-left:before {
+            box-shadow: none;
+        }
+        .ui-grid-pinned-container.ui-grid-pinned-container-left .ui-grid-cell:last-child {
+            border-right: none;
+        }
+
+        .ui-grid-pinned-container-left .ui-grid-header--subcategories .ui-grid-header-span.ui-grid-header-cell {
+            box-shadow: none;
+        }
+
+        .ui-grid-header:not(.ui-grid-header--subcategories) .ui-grid-filters[role="columnheader"] {
+            padding-top: 6px;
+        }
+
+        // End section row height.
+        .ui-grid-header-cell:last-child .ui-grid-column-resizer.right {
+            border-right: none;
+        }
+
+        input[type="text"].ui-grid-filter-input {
+            &::placeholder {
+                color: rgba(66, 66, 66, 0.5);
+                font-weight: normal;
+                text-align: left;
+            }
+
+            &:focus {
+                border-color: $ignite-brand-success;
+                box-shadow: none;
+            }
+
+            font-family: Roboto;
+            outline: none;
+            overflow: visible;
+
+            box-sizing: border-box;
+            width: 100%;
+            max-width: initial;
+            height: 29px;
+            padding: 0 10px;
+            margin-right: 0;
+
+            border: solid 1px #c5c5c5;
+            border-radius: 4px;
+            background-color: #ffffff;
+            box-shadow: none;
+
+            color: $text-color;
+            text-align: left;
+            font-weight: normal;
+            line-height: 16px;
+        }
+    }
+}
diff --git a/modules/web-console/frontend/app/components/ui-grid/template.pug b/modules/web-console/frontend/app/components/ui-grid/template.pug
new file mode 100644
index 0000000000000..4a0f8b353472a
--- /dev/null
+++ b/modules/web-console/frontend/app/components/ui-grid/template.pug
@@ -0,0 +1,60 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+div(ng-if='::$ctrl.gridTreeView')
+    .grid.ui-grid--ignite(
+        ui-grid='$ctrl.grid'
+        ui-grid-resize-columns
+        ui-grid-filters
+        ui-grid-selection
+        ui-grid-exporter
+        ui-grid-pinning
+        ui-grid-tree-view
+        ng-class='{ "ui-grid--thin": $ctrl.gridThin }'
+    )
+
+div(ng-if='::$ctrl.gridGrouping')
+    .grid.ui-grid--ignite(
+        ui-grid='$ctrl.grid'
+        ui-grid-resize-columns
+        ui-grid-filters
+        ui-grid-selection
+        ui-grid-exporter
+        ui-grid-pinning
+        ui-grid-grouping
+        ng-class='{ "ui-grid--thin": $ctrl.gridThin }'
+    )
+
+div(ng-if='::(!$ctrl.gridGrouping && !$ctrl.gridTreeView && $ctrl.onSelectionChange)')
+    .grid.ui-grid--ignite(
+        ui-grid='$ctrl.grid'
+        ui-grid-resize-columns
+        ui-grid-filters
+        ui-grid-selection
+        ui-grid-exporter
+        ui-grid-pinning
+        ng-class='{ "ui-grid--thin": $ctrl.gridThin }'
+    )
+
+div(ng-if='::(!$ctrl.gridGrouping && !$ctrl.gridTreeView && !$ctrl.onSelectionChange)')
+    .grid.ui-grid--ignite(
+        ui-grid='$ctrl.grid'
+        ui-grid-resize-columns
+        ui-grid-filters
+        ui-grid-exporter
+        ui-grid-pinning
+        ng-class='{ "ui-grid--thin": $ctrl.gridThin }'
+    )
diff --git a/modules/web-console/frontend/app/components/user-notifications/service.js b/modules/web-console/frontend/app/components/user-notifications/service.js
index d008482d66d08..5ee3257392408 100644
--- a/modules/web-console/frontend/app/components/user-notifications/service.js
+++ b/modules/web-console/frontend/app/components/user-notifications/service.js
@@ -27,8 +27,17 @@ export default class UserNotificationsService {
     /** @type {ng.IQService} */
     $q;
 
+    /**
+     * @param {ng.IHttpService} $http    
+     * @param {mgcrea.ngStrap.modal.IModalService} $modal   
+     * @param {ng.IQService} $q       
+     * @param {ReturnType<typeof import('app/services/Messages.service').default>} Messages
+     */
     constructor($http, $modal, $q, Messages) {
-        Object.assign(this, {$http, $modal, $q, Messages});
+        this.$http = $http;
+        this.$modal = $modal;
+        this.$q = $q;
+        this.Messages = Messages;
 
         this.message = null;
         this.isShown = false;
diff --git a/modules/web-console/frontend/app/components/user-notifications/style.scss b/modules/web-console/frontend/app/components/user-notifications/style.scss
index e4fa39eda5e51..a1dd94f95a51e 100644
--- a/modules/web-console/frontend/app/components/user-notifications/style.scss
+++ b/modules/web-console/frontend/app/components/user-notifications/style.scss
@@ -21,43 +21,4 @@ $disabled-color: #c5c5c5;
 
 #user-notifications-dialog {
     min-height: 160px;
-
-    > .ignite-form-field {
-        display: flex;
-        flex-direction: column;
-
-        > .ignite-form-field__label {
-            color: $gray-light;
-            font-size: 12px;
-
-            margin-left: 10px;
-            margin-bottom: 5px;
-        }
-
-        > .ignite-form-field__control {
-            width: 100%;
-
-            .ace_editor {
-                border-radius: 4px;
-                box-shadow: inset 0 1px 3px 0 rgba(0, 0, 0, 0.5);
-                border: solid 1px $disabled-color;
-
-                margin: 0;
-
-                .ace_content {
-                    padding-left: 2px;
-                }
-            }
-        }
-    }
-}
-
-.modal-footer {
-    .show-message {
-        display: flex;
-
-        span {
-            margin-left: 5px;
-        }
-    }
 }
diff --git a/modules/web-console/frontend/app/components/user-notifications/template.tpl.pug b/modules/web-console/frontend/app/components/user-notifications/template.tpl.pug
index a8c0394c46b75..a680dea6f229d 100644
--- a/modules/web-console/frontend/app/components/user-notifications/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/user-notifications/template.tpl.pug
@@ -16,7 +16,7 @@
 
 include /app/helpers/jade/mixins
 
-.modal.modal--ignite(tabindex='-1' role='dialog')
+.modal.modal--ignite.theme--ignite(tabindex='-1' role='dialog')
     .modal-dialog
         .modal-content
             .modal-header
@@ -30,18 +30,18 @@ include /app/helpers/jade/mixins
                     | Enter the text, which will show for all users of the Web Console about an important event or
                     | warning about ongoing technical works. It will appear #[b on the yellow bar] in the header.
 
-                .ignite-form-field
-                    +ignite-form-field__label('Your notification:', 'notification', true)
-
-                    .ignite-form-field__control
-                        .input-tip
-                            div(ignite-ace='{onLoad: $ctrl.onLoad, mode: "xml"}' ng-trim='true' ng-model='$ctrl.message')
+                .form-field__ace.ignite-form-field
+                    +form-field__label({ label: 'Your notification:', name: 'notification', required: true})
+                    .form-field__control
+                        div(ignite-ace='{onLoad: $ctrl.onLoad, mode: "xml"}' ng-trim='true' ng-model='$ctrl.message')
 
             .modal-footer
-                .pull-left
-                    label.show-message
-                        input(type='checkbox' ng-model='$ctrl.isShown')
-                        span Show message
-
-                button.btn-ignite.btn-ignite--link-success(id='btn-cancel' ng-click='$hide()') Cancel
-                button.btn-ignite.btn-ignite--success(id='btn-submit' ng-click='$ctrl.submit()') Submit
+                +form-field__checkbox({
+                    label: 'Show message',
+                    name: 'showMessages',
+                    model: '$ctrl.isShown'
+                })
+
+                div
+                    button.btn-ignite.btn-ignite--link-success(id='btn-cancel' ng-click='$hide()') Cancel
+                    button.btn-ignite.btn-ignite--success(id='btn-submit' ng-click='$ctrl.submit()') Submit
diff --git a/modules/web-console/frontend/app/components/version-picker/component.js b/modules/web-console/frontend/app/components/version-picker/component.js
index 68382ee07c7cc..16fcb7ebe7485 100644
--- a/modules/web-console/frontend/app/components/version-picker/component.js
+++ b/modules/web-console/frontend/app/components/version-picker/component.js
@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
 import template from './template.pug';
 import './style.scss';
 
@@ -23,6 +24,10 @@ export default {
     controller: class {
         static $inject = ['IgniteVersion', '$scope'];
 
+        /**
+         * @param {import('app/services/Version.service').default} Version
+         * @param {ng.IRootScopeService} $scope
+         */
         constructor(Version, $scope) {
             this.currentSbj = Version.currentSbj;
             this.supportedVersions = Version.supportedVersions;
diff --git a/modules/web-console/frontend/app/core/admin/Admin.data.js b/modules/web-console/frontend/app/core/admin/Admin.data.js
index 5c4fe101119eb..9ebe59937d0b3 100644
--- a/modules/web-console/frontend/app/core/admin/Admin.data.js
+++ b/modules/web-console/frontend/app/core/admin/Admin.data.js
@@ -15,15 +15,25 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
 export default class IgniteAdminData {
     static $inject = ['$http', 'IgniteMessages', 'IgniteCountries'];
 
+    /**
+     * @param {ng.IHttpService} $http     
+     * @param {ReturnType<typeof import('app/services/Messages.service').default>} Messages
+     * @param {ReturnType<typeof import('app/services/Countries.service').default>} Countries
+     */
     constructor($http, Messages, Countries) {
         this.$http = $http;
         this.Messages = Messages;
         this.Countries = Countries;
     }
 
+    /**
+     * @param {string} viewedUserId
+     */
     becomeUser(viewedUserId) {
         return this.$http.get('/api/v1/admin/become', {
             params: {viewedUserId}
@@ -31,6 +41,9 @@ export default class IgniteAdminData {
         .catch(this.Messages.showError);
     }
 
+    /**
+     * @param {import('app/modules/user/User.service').User} user
+     */
     removeUser(user) {
         return this.$http.post('/api/v1/admin/remove', {
             userId: user._id
@@ -46,6 +59,9 @@ export default class IgniteAdminData {
         });
     }
 
+    /**
+     * @param {import('app/modules/user/User.service').User} user
+     */
     toggleAdmin(user) {
         const adminFlag = !user.admin;
 
@@ -59,10 +75,13 @@ export default class IgniteAdminData {
             this.Messages.showInfo(`Admin rights was successfully ${adminFlag ? 'granted' : 'revoked'} for user: "${user.userName}"`);
         })
         .catch((res) => {
-            this.Messages.showError(`Failed to ${adminFlag ? 'grant' : 'revok'} admin rights for user: "${user.userName}"`, res);
+            this.Messages.showError(`Failed to ${adminFlag ? 'grant' : 'revoke'} admin rights for user: "${user.userName}"`, res);
         });
     }
 
+    /**
+     * @param {import('app/modules/user/User.service').User} user
+     */
     prepareUsers(user) {
         const { Countries } = this;
 
diff --git a/modules/web-console/frontend/app/data/getting-started.json b/modules/web-console/frontend/app/data/getting-started.json
index 1802b554aaa82..58ca36795ff73 100644
--- a/modules/web-console/frontend/app/data/getting-started.json
+++ b/modules/web-console/frontend/app/data/getting-started.json
@@ -2,11 +2,11 @@
     {
         "title": "With Apache Ignite Web Console You Can",
         "message": [
-            "<div class='col-xs-4'>",
-            " <img src='/images/ignite-puzzle.png' width='80%' class='getting-started-puzzle' />",
+            "<div class='col-40 align-center'>",
+            " <img src='/images/ignite-puzzle.png' width='170px' class='getting-started-puzzle' />",
             "</div>",
-            "<div class='col-xs-8'>",
-            " <ul>",
+            "<div class='col-60'>",
+            " <ul class='align-top'>",
             "  <li>Generate cluster configuration</li>",
             "  <li>Import domain model from database</li>",
             "  <li>Configure all needed caches</li>",
@@ -20,11 +20,11 @@
     {
         "title": "Quick cluster configuration",
         "message": [
-            "<div class='col-xs-7'>",
+            "<div class='col-60'>",
             " <img src='/images/cluster-quick.png' width='100%' />",
             "</div>",
-            "<div class='col-xs-5'>",
-            " <ul>",
+            "<div class='col-40'>",
+            " <ul class='align-top'>",
             "  <li>Quick configuration of cluster and it's caches</li>",
             " </ul>",
             "</div>"
@@ -33,11 +33,11 @@
     {
         "title": "Clusters",
         "message": [
-            "<div class='col-xs-7'>",
+            "<div class='col-60'>",
             " <img src='/images/cluster.png' width='100%' />",
             "</div>",
-            "<div class='col-xs-5'>",
-            " <ul>",
+            "<div class='col-40'>",
+            " <ul class='align-top'>",
             "  <li>Configure cluster properties</li>",
             " </ul>",
             "</div>"
@@ -46,11 +46,11 @@
     {
         "title": "Domain Model",
         "message": [
-            "<div class='col-xs-7'>",
+            "<div class='col-60'>",
             " <img src='/images/domains.png' width='100%' />",
             "</div>",
-            "<div class='col-xs-5'>",
-            " <ul>",
+            "<div class='col-40'>",
+            " <ul class='align-top'>",
             "  <li>Import database schemas</li>",
             "  <li>Try in <span class='getting-started-demo'>Demo</span> mode</li>",
             " </ul>",
@@ -60,11 +60,11 @@
     {
         "title": "Caches",
         "message": [
-            "<div class='col-xs-7'>",
+            "<div class='col-60'>",
             " <img src='/images/cache.png' width='100%' />",
             "</div>",
-            "<div class='col-xs-5'>",
-            " <ul>",
+            "<div class='col-40'>",
+            " <ul class='align-top'>",
             "  <li>Configure memory settings</li>",
             " </ul>",
             "</div>"
@@ -73,11 +73,11 @@
     {
         "title": "In-memory File System",
         "message": [
-            "<div class='col-xs-7'>",
+            "<div class='col-60'>",
             " <img src='/images/igfs.png' width='100%' />",
             "</div>",
-            "<div class='col-xs-5'>",
-            " <ul>",
+            "<div class='col-40'>",
+            " <ul class='align-top'>",
             "  <li>Configure IGFS properties</li>",
             " </ul>",
             "</div>"
@@ -86,11 +86,11 @@
     {
         "title": "Preview configuration result",
         "message": [
-            "<div class='col-xs-7'>",
+            "<div class='col-60'>",
             " <img src='/images/preview.png' width='100%' />",
             "</div>",
-            "<div class='col-xs-5'>",
-            " <ul>",
+            "<div class='col-40'>",
+            " <ul class='align-top'>",
             "  <li>Preview configured project files</li>",
             "  <li>Download configured project files</li>",
             " </ul>",
@@ -100,11 +100,11 @@
     {
         "title": "SQL Queries",
         "message": [
-            "<div class='col-xs-7'>",
+            "<div class='col-60'>",
             " <img src='/images/query-table.png' width='100%' />",
             "</div>",
-            "<div class='col-xs-5'>",
-            " <ul>",
+            "<div class='col-40'>",
+            " <ul class='align-top'>",
             "  <li>Execute SQL Queries</li>",
             "  <li>View Execution Paln</li>",
             "  <li>View In-Memory Schema</li>",
@@ -116,11 +116,11 @@
     {
         "title": "Multicluster support",
         "message": [
-            "<div class='col-xs-7'>",
+            "<div class='col-60'>",
             " <img src='/images/multicluster.png' width='100%' />",
             "</div>",
-            "<div class='col-xs-5'>",
-            " <ul>",
+            "<div class='col-40'>",
+            " <ul class='align-top'>",
             "  <li>Execute queries on different clusters</li>",
             " </ul>",
             "</div>"
diff --git a/modules/web-console/frontend/app/directives/auto-focus.directive.js b/modules/web-console/frontend/app/directives/auto-focus.directive.js
index 326fe1f35456a..8d269ce6d2cca 100644
--- a/modules/web-console/frontend/app/directives/auto-focus.directive.js
+++ b/modules/web-console/frontend/app/directives/auto-focus.directive.js
@@ -15,12 +15,21 @@
  * limitations under the License.
  */
 
-// Directive to auto-focus specified element.
-export default ['igniteAutoFocus', ['$timeout', ($timeout) => {
+/**
+ * Directive to auto-focus specified element.
+ * @param {ng.ITimeoutService} $timeout
+ */
+export default function directive($timeout) {
     return {
         restrict: 'AC',
+        /**
+         * @param {ng.IScope} scope
+         * @param {JQLite} element
+         */
         link(scope, element) {
             $timeout(() => element[0].focus());
         }
     };
-}]];
+}
+
+directive.$inject = ['$timeout'];
diff --git a/modules/web-console/frontend/app/directives/bs-affix-update.directive.js b/modules/web-console/frontend/app/directives/bs-affix-update.directive.js
index 925722c827d30..a462dcb3e7919 100644
--- a/modules/web-console/frontend/app/directives/bs-affix-update.directive.js
+++ b/modules/web-console/frontend/app/directives/bs-affix-update.directive.js
@@ -17,7 +17,11 @@
 
 import angular from 'angular';
 
-export default ['igniteBsAffixUpdate', ['$window', '$timeout', ($window, $timeout) => {
+/**
+ * @param {ng.IWindowService} $window
+ * @param {ng.ITimeoutService} $timeout
+ */
+export default function directive($window, $timeout) {
     let update = null;
 
     const link = ({$last}) => {
@@ -31,4 +35,6 @@ export default ['igniteBsAffixUpdate', ['$window', '$timeout', ($window, $timeou
         restrict: 'A',
         link
     };
-}]];
+}
+
+directive.$inject = ['$window', '$timeout'];
diff --git a/modules/web-console/frontend/app/directives/btn-ignite-link.js b/modules/web-console/frontend/app/directives/btn-ignite-link.js
index 5180c62b1bc9b..02aaf8ffb6f2e 100644
--- a/modules/web-console/frontend/app/directives/btn-ignite-link.js
+++ b/modules/web-console/frontend/app/directives/btn-ignite-link.js
@@ -15,13 +15,15 @@
  * limitations under the License.
  */
 
-export default () => ({
-    restrict: 'C',
-    link: (scope, $element) => {
-        $element.contents()
-            .filter(function() {
-                return this.nodeType === 3;
-            })
-            .wrap('<span></span>');
-    }
-});
+export default function() {
+    return {
+        restrict: 'C',
+        link: (scope, $element) => {
+            $element.contents()
+                .filter(function() {
+                    return this.nodeType === 3;
+                })
+                .wrap('<span></span>');
+        }
+    };
+}
diff --git a/modules/web-console/frontend/app/directives/centered/centered.directive.js b/modules/web-console/frontend/app/directives/centered/centered.directive.js
index 77bbb940fe937..967e0c2f0171e 100644
--- a/modules/web-console/frontend/app/directives/centered/centered.directive.js
+++ b/modules/web-console/frontend/app/directives/centered/centered.directive.js
@@ -17,10 +17,10 @@
 
 import './centered.scss';
 
-export default ['centered', [() => {
+export default function() {
     return {
         restrict: 'E',
         transclude: true,
         template: '<div class="center-container"><div class="centered"><div ng-transclude></div></div></div>'
     };
-}]];
+}
diff --git a/modules/web-console/frontend/app/directives/copy-to-clipboard.directive.js b/modules/web-console/frontend/app/directives/copy-to-clipboard.directive.js
index ee2110ee12d21..399e1f3e15539 100644
--- a/modules/web-console/frontend/app/directives/copy-to-clipboard.directive.js
+++ b/modules/web-console/frontend/app/directives/copy-to-clipboard.directive.js
@@ -15,10 +15,17 @@
  * limitations under the License.
  */
 
-// Directive for copy to clipboard.
-export default ['igniteCopyToClipboard', ['IgniteCopyToClipboard', (CopyToClipboard) => {
+/**
+ * @param {ReturnType<typeof import('../services/CopyToClipboard.service').default>} CopyToClipboard
+ */
+export default function directive(CopyToClipboard) {
     return {
         restrict: 'A',
+        /**
+         * @param {ng.IScope} scope
+         * @param {JQLite} element
+         * @param {ng.IAttributes} attrs
+         */
         link(scope, element, attrs) {
             element.bind('click', () => CopyToClipboard.copy(attrs.igniteCopyToClipboard));
 
@@ -26,4 +33,6 @@ export default ['igniteCopyToClipboard', ['IgniteCopyToClipboard', (CopyToClipbo
                 element.hide();
         }
     };
-}]];
+}
+
+directive.$inject = ['IgniteCopyToClipboard'];
diff --git a/modules/web-console/frontend/app/directives/hide-on-state-change/hide-on-state-change.directive.js b/modules/web-console/frontend/app/directives/hide-on-state-change/hide-on-state-change.directive.js
index 152e9420a36a2..cd1f45a5b6858 100644
--- a/modules/web-console/frontend/app/directives/hide-on-state-change/hide-on-state-change.directive.js
+++ b/modules/web-console/frontend/app/directives/hide-on-state-change/hide-on-state-change.directive.js
@@ -15,13 +15,22 @@
  * limitations under the License.
  */
 
-export default ['hideOnStateChange', ['$transitions', ($transitions) => {
+/**
+ * @param {import('@uirouter/angularjs').TransitionService} $transitions
+ */
+export default function directive($transitions) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} element
+     */
     const link = (scope, element) => {
-        $transitions.onSuccess({}, () => element.fadeOut('slow'));
+        $transitions.onSuccess({}, () => {element.fadeOut('slow');});
     };
 
     return {
         restrict: 'AE',
         link
     };
-}]];
+}
+
+directive.$inject = ['$transitions'];
diff --git a/modules/web-console/frontend/app/directives/information/information.directive.js b/modules/web-console/frontend/app/directives/information/information.directive.js
index 6f304ef8a9b0b..8e95791135f5a 100644
--- a/modules/web-console/frontend/app/directives/information/information.directive.js
+++ b/modules/web-console/frontend/app/directives/information/information.directive.js
@@ -17,7 +17,7 @@
 
 import template from './information.pug';
 
-export default ['igniteInformation', [() => {
+export default function() {
     return {
         scope: {
             title: '@'
@@ -27,4 +27,4 @@ export default ['igniteInformation', [() => {
         replace: true,
         transclude: true
     };
-}]];
+}
diff --git a/modules/web-console/frontend/app/directives/on-click-focus.directive.js b/modules/web-console/frontend/app/directives/on-click-focus.directive.js
index 5c9ee883a1ce9..bbc46d186e8ac 100644
--- a/modules/web-console/frontend/app/directives/on-click-focus.directive.js
+++ b/modules/web-console/frontend/app/directives/on-click-focus.directive.js
@@ -15,12 +15,24 @@
  * limitations under the License.
  */
 
-// Directive to describe element that should be focused on click.
-export default ['igniteOnClickFocus', ['IgniteFocus', (Focus) => {
-    return function(scope, elem, attrs) {
+/**
+ * Directive to describe element that should be focused on click.
+ * @param {ReturnType<typeof import('../services/Focus.service').default>} Focus
+ */
+export default function directive(Focus) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} elem
+     * @param {ng.IAttributes} attrs
+     */
+    function directive(scope, elem, attrs) {
         elem.on('click', () => Focus.move(attrs.igniteOnClickFocus));
 
         // Removes bound events in the element itself when the scope is destroyed
         scope.$on('$destroy', () => elem.off('click'));
-    };
-}]];
+    }
+
+    return directive;
+}
+
+directive.$inject = ['IgniteFocus'];
diff --git a/modules/web-console/frontend/app/directives/on-enter-focus-move.directive.js b/modules/web-console/frontend/app/directives/on-enter-focus-move.directive.js
index 2dd28842264e4..9cb9913a823a2 100644
--- a/modules/web-console/frontend/app/directives/on-enter-focus-move.directive.js
+++ b/modules/web-console/frontend/app/directives/on-enter-focus-move.directive.js
@@ -15,9 +15,17 @@
  * limitations under the License.
  */
 
-// Directive to move focus to specified element on ENTER key.
-export default ['igniteOnEnterFocusMove', ['IgniteFocus', (Focus) => {
-    return function(scope, elem, attrs) {
+/**
+ * Directive to move focus to specified element on ENTER key.
+ * @param {ReturnType<typeof import('../services/Focus.service').default>} Focus
+ */
+export default function directive(Focus) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} elem
+     * @param {ng.IAttributes} attrs
+     */
+    function directive(scope, elem, attrs) {
         elem.on('keydown keypress', (event) => {
             if (event.which === 13) {
                 event.preventDefault();
@@ -25,5 +33,9 @@ export default ['igniteOnEnterFocusMove', ['IgniteFocus', (Focus) => {
                 Focus.move(attrs.igniteOnEnterFocusMove);
             }
         });
-    };
-}]];
+    }
+
+    return directive;
+}
+
+directive.$inject = ['IgniteFocus'];
diff --git a/modules/web-console/frontend/app/directives/on-enter.directive.js b/modules/web-console/frontend/app/directives/on-enter.directive.js
index 459220e382603..1bc3e48da317d 100644
--- a/modules/web-console/frontend/app/directives/on-enter.directive.js
+++ b/modules/web-console/frontend/app/directives/on-enter.directive.js
@@ -15,9 +15,17 @@
  * limitations under the License.
  */
 
-// Directive to bind ENTER key press with some user action.
-export default ['igniteOnEnter', ['$timeout', ($timeout) => {
-    return function(scope, elem, attrs) {
+/**
+ * Directive to bind ENTER key press with some user action.
+ * @param {ng.ITimeoutService} $timeout
+ */
+export default function directive($timeout) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} elem
+     * @param {ng.IAttributes} attrs
+     */
+    function directive(scope, elem, attrs) {
         elem.on('keydown keypress', (event) => {
             if (event.which === 13) {
                 scope.$apply(() => $timeout(() => scope.$eval(attrs.igniteOnEnter)));
@@ -28,5 +36,9 @@ export default ['igniteOnEnter', ['$timeout', ($timeout) => {
 
         // Removes bound events in the element itself when the scope is destroyed.
         scope.$on('$destroy', () => elem.off('keydown keypress'));
-    };
-}]];
+    }
+
+    return directive;
+}
+
+directive.$inject = ['$timeout'];
diff --git a/modules/web-console/frontend/app/directives/on-escape.directive.js b/modules/web-console/frontend/app/directives/on-escape.directive.js
index aa1accded5d2a..b1f16489bcdfe 100644
--- a/modules/web-console/frontend/app/directives/on-escape.directive.js
+++ b/modules/web-console/frontend/app/directives/on-escape.directive.js
@@ -15,9 +15,17 @@
  * limitations under the License.
  */
 
-// Directive to bind ESC key press with some user action.
-export default ['igniteOnEscape', ['$timeout', ($timeout) => {
-    return function(scope, elem, attrs) {
+/**
+ * Directive to bind ESC key press with some user action.
+ * @param {ng.ITimeoutService} $timeout
+ */
+export default function directive($timeout) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} elem
+     * @param {ng.IAttributes} attrs
+     */
+    function directive(scope, elem, attrs) {
         elem.on('keydown keypress', (event) => {
             if (event.which === 27) {
                 scope.$apply(() => $timeout(() => scope.$eval(attrs.igniteOnEscape)));
@@ -28,5 +36,9 @@ export default ['igniteOnEscape', ['$timeout', ($timeout) => {
 
         // Removes bound events in the element itself when the scope is destroyed.
         scope.$on('$destroy', () => elem.off('keydown keypress'));
-    };
-}]];
+    }
+
+    return directive;
+}
+
+directive.$inject = ['$timeout'];
diff --git a/modules/web-console/frontend/app/directives/restore-input-focus.directive.js b/modules/web-console/frontend/app/directives/restore-input-focus.directive.js
index 32e66224482ad..829b8884c3d46 100644
--- a/modules/web-console/frontend/app/directives/restore-input-focus.directive.js
+++ b/modules/web-console/frontend/app/directives/restore-input-focus.directive.js
@@ -15,10 +15,16 @@
  * limitations under the License.
  */
 
-export default [() => {
-    return ($scope, $element) => {
+export default function() {
+    /**
+     * @param {ng.IScope} $scope
+     * @param {JQuery} $element
+     */
+    const directive = ($scope, $element) => {
         $element.on('click', () => {
             $element.siblings('.input-tip').find('input').focus();
         });
     };
-}];
+
+    return directive;
+}
diff --git a/modules/web-console/frontend/app/directives/retain-selection.directive.js b/modules/web-console/frontend/app/directives/retain-selection.directive.js
index 74d6872ef1ba7..fb7321a55178d 100644
--- a/modules/web-console/frontend/app/directives/retain-selection.directive.js
+++ b/modules/web-console/frontend/app/directives/retain-selection.directive.js
@@ -15,11 +15,18 @@
  * limitations under the License.
  */
 
-// Directive to workaround known issue with type ahead edit lost cursor position.
-export default ['igniteRetainSelection', ['$timeout', ($timeout) => {
+/**
+ * Directive to workaround known issue with type ahead edit lost cursor position.
+ * @param {ng.ITimeoutService} $timeout
+ */
+export default function directive($timeout) {
     let promise;
 
-    return function(scope, elem) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} elem  [description]
+     */
+    function directive(scope, elem) {
         elem.on('keydown', function(evt) {
             const key = evt.which;
             const ctrlDown = evt.ctrlKey || evt.metaKey;
@@ -63,5 +70,9 @@ export default ['igniteRetainSelection', ['$timeout', ($timeout) => {
         scope.$on('$destroy', function() {
             elem.off('keydown');
         });
-    };
-}]];
+    }
+
+    return directive;
+}
+
+directive.$inject = ['$timeout'];
diff --git a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.controller.js b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.controller.js
index 4f443aead3c1a..c3928c722e448 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.controller.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.controller.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-export default ['$scope', 'IgniteVersion', 'IgniteDockerGenerator', function($scope, Version, docker) {
+export default function controller($scope, Version, docker) {
     const ctrl = this;
 
     this.$onInit = () => {
@@ -36,4 +36,6 @@ export default ['$scope', 'IgniteVersion', 'IgniteDockerGenerator', function($sc
 
         $scope.$watch('cluster', clusterWatcher);
     };
-}];
+}
+
+controller.$inject = ['$scope', 'IgniteVersion', 'IgniteDockerGenerator'];
diff --git a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js
index 9042acb767f53..188f11f0d4ff6 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-docker/ui-ace-docker.directive.js
@@ -18,7 +18,7 @@
 import template from './ui-ace-docker.pug';
 import controller from './ui-ace-docker.controller';
 
-export default ['igniteUiAceDocker', [() => {
+export default function() {
     const link = ($scope, $el, $attrs, [igniteUiAceTabs]) => {
         if (igniteUiAceTabs.onLoad)
             $scope.onLoad = igniteUiAceTabs.onLoad;
@@ -43,4 +43,4 @@ export default ['igniteUiAceDocker', [() => {
         controllerAs: 'ctrl',
         require: ['?^igniteUiAceTabs']
     };
-}]];
+}
diff --git a/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js b/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js
index 62eb3766c2993..b4f945d08f0d4 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-java/ui-ace-java.directive.js
@@ -18,7 +18,7 @@
 import template from './ui-ace-java.pug';
 import IgniteUiAceJava from './ui-ace-java.controller';
 
-export default () => {
+export default function() {
     return {
         priority: 1,
         restrict: 'E',
@@ -41,4 +41,4 @@ export default () => {
             ngModelCtrl: '?ngModel'
         }
     };
-};
+}
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.controller.js b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.controller.js
index 3f9580f1ca727..bb4d1c4b9ebd4 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.controller.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.controller.js
@@ -15,9 +15,10 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
 import {nonNil} from 'app/utils/lodashMixins';
 
-export default ['$scope', 'JavaTypes', 'JavaTransformer', function($scope, JavaTypes, generator) {
+export default function controller($scope, JavaTypes, generator) {
     const ctrl = this;
 
     this.$onInit = () => {
@@ -96,4 +97,6 @@ export default ['$scope', 'JavaTypes', 'JavaTransformer', function($scope, JavaT
         $scope.$watch('ctrl.pojos', updatePojosData);
         $scope.$watch('ctrl.class', updatePojosData);
     };
-}];
+}
+
+controller.$inject = ['$scope', 'JavaTypes', 'JavaTransformer'];
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js
index 8a8d047de561c..c9f97cc85c884 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pojos/ui-ace-pojos.directive.js
@@ -18,7 +18,7 @@
 import template from './ui-ace-pojos.pug';
 import controller from './ui-ace-pojos.controller';
 
-export default ['igniteUiAcePojos', [() => {
+export default function() {
     const link = ($scope, $el, $attrs, [igniteUiAceTabs]) => {
         if (igniteUiAceTabs.onLoad)
             $scope.onLoad = igniteUiAceTabs.onLoad;
@@ -43,4 +43,4 @@ export default ['igniteUiAcePojos', [() => {
         controllerAs: 'ctrl',
         require: ['?^igniteUiAceTabs']
     };
-}]];
+}
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
index 2e421b2d6b82d..ae64022a99a12 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.controller.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-export default ['$scope', 'IgniteVersion', 'IgniteMavenGenerator', function($scope, Version, maven) {
+export default function controller($scope, Version, maven) {
     const ctrl = this;
 
     this.$onInit = () => {
@@ -36,4 +36,6 @@ export default ['$scope', 'IgniteVersion', 'IgniteMavenGenerator', function($sco
 
         $scope.$watch('cluster', clusterWatcher);
     };
-}];
+}
+
+controller.$inject = ['$scope', 'IgniteVersion', 'IgniteMavenGenerator'];
diff --git a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js
index 664d3a000fd4a..f2ba0ae212328 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-pom/ui-ace-pom.directive.js
@@ -18,7 +18,7 @@
 import template from './ui-ace-pom.pug';
 import controller from './ui-ace-pom.controller';
 
-export default ['igniteUiAcePom', [() => {
+export default function() {
     const link = ($scope, $el, $attrs, [igniteUiAceTabs]) => {
         if (igniteUiAceTabs.onLoad)
             $scope.onLoad = igniteUiAceTabs.onLoad;
@@ -38,4 +38,4 @@ export default ['igniteUiAcePom', [() => {
         controllerAs: 'ctrl',
         require: ['?^igniteUiAceTabs']
     };
-}]];
+}
diff --git a/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.controller.js b/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.controller.js
index e87cacad214eb..09f7a017914a4 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.controller.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.controller.js
@@ -18,7 +18,12 @@
 const SERVER_CFG = 'ServerConfigurationFactory';
 const CLIENT_CFG = 'ClientConfigurationFactory';
 
-export default ['$scope', 'IgniteSharpTransformer', function($scope, generator) {
+/**
+ * @param {ng.IScope} $scope
+ * @param {import('app/modules/configuration/generator/SharpTransformer.service').default} generator
+ */
+export default function controller($scope, generator) {
+    /** @type {ThisType} */
     const ctrl = this;
 
     this.$onInit = () => {
@@ -31,4 +36,6 @@ export default ['$scope', 'IgniteSharpTransformer', function($scope, generator)
             return generator.cluster(cluster, 'config', type, $scope.cfg);
         };
     };
-}];
+}
+
+controller.$inject = ['$scope', 'IgniteSharpTransformer'];
diff --git a/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js b/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js
index 5a37b80742ea8..d6e366bb9978b 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-sharp/ui-ace-sharp.directive.js
@@ -15,11 +15,23 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
 import template from './ui-ace-sharp.pug';
 import controller from './ui-ace-sharp.controller';
 
-export default ['igniteUiAceSharp', ['IgniteSharpTransformer', (generator) => {
-    const link = (scope, $el, attrs, [ctrl, igniteUiAceTabs, formCtrl, ngModelCtrl]) => {
+/**
+ * @param {import('app/modules/configuration/generator/SharpTransformer.service').default} generator
+ */
+export default function directive(generator) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} $el
+     * @param {ng.IAttributes} attrs
+     * @param {[typeof controller, any?, ng.IFormController?, ng.INgModelController?]} controllers
+     */
+    const link = (scope, $el, attrs, controllers) => {
+        const [ctrl, igniteUiAceTabs, formCtrl, ngModelCtrl] = controllers;
         if (formCtrl && ngModelCtrl)
             formCtrl.$removeControl(ngModelCtrl);
 
@@ -130,4 +142,6 @@ export default ['igniteUiAceSharp', ['IgniteSharpTransformer', (generator) => {
         controllerAs: 'ctrl',
         require: ['igniteUiAceSharp', '?^igniteUiAceTabs', '?^form', '?ngModel']
     };
-}]];
+}
+
+directive.$inject = ['IgniteSharpTransformer'];
diff --git a/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js b/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js
index 8655fd105d934..532ba42929141 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-spring/ui-ace-spring.directive.js
@@ -18,7 +18,7 @@
 import template from './ui-ace-spring.pug';
 import IgniteUiAceSpring from './ui-ace-spring.controller';
 
-export default () => {
+export default function() {
     return {
         priority: 1,
         restrict: 'E',
@@ -41,4 +41,4 @@ export default () => {
             ngModelCtrl: '?ngModel'
         }
     };
-};
+}
diff --git a/modules/web-console/frontend/app/directives/ui-ace-tabs.directive.js b/modules/web-console/frontend/app/directives/ui-ace-tabs.directive.js
index 2b90a72ebddc8..a174b98ae9875 100644
--- a/modules/web-console/frontend/app/directives/ui-ace-tabs.directive.js
+++ b/modules/web-console/frontend/app/directives/ui-ace-tabs.directive.js
@@ -15,10 +15,12 @@
  * limitations under the License.
  */
 
-export default ['igniteUiAceTabs', [() => {
+import _ from 'lodash';
+
+export default function() {
     return {
         scope: true,
         restrict: 'AE',
         controller: _.noop
     };
-}]];
+}
diff --git a/modules/web-console/frontend/app/filters/byName.filter.js b/modules/web-console/frontend/app/filters/byName.filter.js
index 27691894feacd..6ec56f4a01dfb 100644
--- a/modules/web-console/frontend/app/filters/byName.filter.js
+++ b/modules/web-console/frontend/app/filters/byName.filter.js
@@ -15,9 +15,11 @@
  * limitations under the License.
  */
 
-export default [() => (arr, search) => {
+import _ from 'lodash';
+
+export default () => (arr, search) => {
     if (!(arr && arr.length) || !search)
         return arr;
 
     return _.filter(arr, ({ name }) => name.indexOf(search) >= 0);
-}];
+};
diff --git a/modules/web-console/frontend/app/modules/form/field/form-control-feedback.directive.js b/modules/web-console/frontend/app/filters/bytes.filter.js
similarity index 57%
rename from modules/web-console/frontend/app/modules/form/field/form-control-feedback.directive.js
rename to modules/web-console/frontend/app/filters/bytes.filter.js
index 797ba6988fb4f..71e77d4fb4c2e 100644
--- a/modules/web-console/frontend/app/modules/form/field/form-control-feedback.directive.js
+++ b/modules/web-console/frontend/app/filters/bytes.filter.js
@@ -15,26 +15,26 @@
  * limitations under the License.
  */
 
-export default ['formFieldFeedback', [() => {
-    const link = ($scope, $element, $attrs, [form]) => {
-        let name = $scope.name;
+export default () => {
+    /**
+     * @param {number} bytes
+     * @param {number} [precision]
+     */
+    const filter = (bytes, precision) => {
+        if (bytes === 0)
+            return '0 bytes';
 
-        if (_.isNil(name))
-            name = $attrs.name;
+        if (isNaN(parseFloat(bytes)) || !isFinite(bytes))
+            return '-';
 
-        const err = $attrs.igniteError;
-        const msg = $attrs.igniteErrorMessage;
+        if (typeof precision === 'undefined')
+            precision = 1;
 
-        if (name && err && msg) {
-            form.$errorMessages = form.$errorMessages || {};
-            form.$errorMessages[name] = form.$errorMessages[name] || {};
-            form.$errorMessages[name][err] = msg;
-        }
-    };
+        const units = ['bytes', 'kB', 'MB', 'GB', 'TB'];
+        const number = Math.floor(Math.log(bytes) / Math.log(1024));
 
-    return {
-        restrict: 'C',
-        link,
-        require: ['^form']
+        return (bytes / Math.pow(1024, Math.floor(number))).toFixed(precision) + ' ' + units[number];
     };
-}]];
+
+    return filter;
+};
diff --git a/modules/web-console/frontend/app/modules/form/field/down.directive.js b/modules/web-console/frontend/app/filters/bytes.filter.spec.js
similarity index 52%
rename from modules/web-console/frontend/app/modules/form/field/down.directive.js
rename to modules/web-console/frontend/app/filters/bytes.filter.spec.js
index c957e9788d94b..884e43efd9ea6 100644
--- a/modules/web-console/frontend/app/modules/form/field/down.directive.js
+++ b/modules/web-console/frontend/app/filters/bytes.filter.spec.js
@@ -15,31 +15,22 @@
  * limitations under the License.
  */
 
-export default ['igniteFormFieldDown', ['$tooltip', ($tooltip) => {
-    const controller = ['$element', function($element) {
-        const ctrl = this;
+import bytesFilter from './bytes.filter';
 
-        this.$onInit = () => {
-            $tooltip($element, { title: 'Move item down' });
+import { suite, test } from 'mocha';
+import { assert } from 'chai';
 
-            ctrl.down = () => {
-                const i = ctrl.models.indexOf(ctrl.model);
+const bytesFilterInstance = bytesFilter();
 
-                ctrl.models.splice(i, 1);
-                ctrl.models.splice(i + 1, 0, ctrl.model);
-            };
-        };
-
-
-    }];
-
-    return {
-        restrict: 'A',
-        bindToController: {
-            model: '=ngModel',
-            models: '=models'
-        },
-        controller,
-        controllerAs: 'vm'
-    };
-}]];
+suite('bytes filter', () => {
+    test('bytes filter', () => {
+        assert.equal(bytesFilterInstance(0), '0 bytes');
+        assert.equal(bytesFilterInstance(1000), '1000.0 bytes');
+        assert.equal(bytesFilterInstance(1024), '1.0 kB');
+        assert.equal(bytesFilterInstance(5000), '4.9 kB');
+        assert.equal(bytesFilterInstance(1048576), '1.0 MB');
+        assert.equal(bytesFilterInstance(104857600), '100.0 MB');
+        assert.equal(bytesFilterInstance(1073741824), '1.0 GB');
+        assert.equal(bytesFilterInstance(1099511627776), '1.0 TB');
+    });
+});
diff --git a/modules/web-console/frontend/app/filters/default-name.filter.js b/modules/web-console/frontend/app/filters/default-name.filter.js
index 687ff84c36ad0..a8b182e2b0664 100644
--- a/modules/web-console/frontend/app/filters/default-name.filter.js
+++ b/modules/web-console/frontend/app/filters/default-name.filter.js
@@ -15,7 +15,15 @@
  * limitations under the License.
  */
 
-// Filter that will check name and return `<default>` if needed.
-export default [() => {
-    return (name, html) => _.isEmpty(name) ? (html ? '&lt;default&gt;' : '<default>') : name;
-}];
+import _ from 'lodash';
+
+export default () => {
+    /**
+     * Filter that will check name and return `<default>` if needed.
+     * @param {string} name
+     * @param {string} html
+     */
+    const filter = (name, html) => _.isEmpty(name) ? (html ? '&lt;default&gt;' : '<default>') : name;
+
+    return filter;
+};
diff --git a/modules/web-console/frontend/app/filters/domainsValidation.filter.js b/modules/web-console/frontend/app/filters/domainsValidation.filter.js
index eec0ff1bfb5f4..d551ca5a2e777 100644
--- a/modules/web-console/frontend/app/filters/domainsValidation.filter.js
+++ b/modules/web-console/frontend/app/filters/domainsValidation.filter.js
@@ -15,19 +15,37 @@
  * limitations under the License.
  */
 
-// Filter domain models with key fields configuration.
-export default ['IgniteLegacyUtils', (LegacyUtils) => (domains, valid, invalid) => {
-    if (valid && invalid)
-        return domains;
+import _ from 'lodash';
 
-    const out = [];
+/**
+ * @param {ReturnType<typeof import('../services/LegacyUtils.service').default>} LegacyUtils [description]
+ */
+export default function factory(LegacyUtils) {
+    /**
+     * Filter domain models with key fields configuration.
+     * @template T
+     * @param {Array<T>} domains
+     * @param {boolean} valid
+     * @param {boolean} invalid
+     */
+    const filter = (domains, valid, invalid) => {
+        if (valid && invalid)
+            return domains;
+
+        /** @type {Array<T>} */
+        const out = [];
+
+        _.forEach(domains, function(domain) {
+            const _valid = !LegacyUtils.domainForStoreConfigured(domain) || LegacyUtils.isJavaBuiltInClass(domain.keyType) || !_.isEmpty(domain.keyFields);
+
+            if (valid && _valid || invalid && !_valid)
+                out.push(domain);
+        });
 
-    _.forEach(domains, function(domain) {
-        const _valid = !LegacyUtils.domainForStoreConfigured(domain) || LegacyUtils.isJavaBuiltInClass(domain.keyType) || !_.isEmpty(domain.keyFields);
+        return out;
+    };
 
-        if (valid && _valid || invalid && !_valid)
-            out.push(domain);
-    });
+    return filter;
+}
 
-    return out;
-}];
+factory.$inject = ['IgniteLegacyUtils'];
diff --git a/modules/web-console/frontend/app/filters/duration.filter.js b/modules/web-console/frontend/app/filters/duration.filter.js
index 55ec7e0b7db3a..810cc4d44678f 100644
--- a/modules/web-console/frontend/app/filters/duration.filter.js
+++ b/modules/web-console/frontend/app/filters/duration.filter.js
@@ -15,11 +15,11 @@
  * limitations under the License.
  */
 
-export default [() => {
+export default () => {
     /**
-     * @param {Number} t Time in ms.
+     * @param {number} t Time in ms.
      */
-    return (t) => {
+    const filter = (t) => {
         if (t === 9223372036854775807)
             return 'Infinite';
 
@@ -38,4 +38,6 @@ export default [() => {
 
         return a(d, 'd') + a(h, 'h') + a(m, 'm') + a(s, 's') + (t < 1000 || (t < cm && ms !== 0) ? ms + 'ms' : '');
     };
-}];
+
+    return filter;
+};
diff --git a/modules/web-console/frontend/app/filters/hasPojo.filter.js b/modules/web-console/frontend/app/filters/hasPojo.filter.js
index 57530db045eaa..4186928acdbc5 100644
--- a/modules/web-console/frontend/app/filters/hasPojo.filter.js
+++ b/modules/web-console/frontend/app/filters/hasPojo.filter.js
@@ -16,6 +16,6 @@
  */
 
 // Filter that return 'true' if caches has at least one domain with 'generatePojo' flag.
-export default [() => ({caches} = []) =>
+export default () => ({caches} = []) =>
     _.find(caches, (cache) => cache.domains && cache.domains.length &&
-        cache.domains.find((domain) => domain.generatePojo))];
+        cache.domains.find((domain) => domain.generatePojo));
diff --git a/modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js b/modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js
index f36ae6e057023..859192dcc5795 100644
--- a/modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js
+++ b/modules/web-console/frontend/app/filters/uiGridSubcategories.filter.js
@@ -15,10 +15,12 @@
  * limitations under the License.
  */
 
-export default [() => {
+import _ from 'lodash';
+
+export default () => {
     return (arr, category) => {
         return _.filter(arr, (item) => {
             return item.colDef.categoryDisplayName === category;
         });
     };
-}];
+};
diff --git a/modules/web-console/frontend/app/helpers/jade/form.pug b/modules/web-console/frontend/app/helpers/jade/form.pug
deleted file mode 100644
index 44eaed94d93ae..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form.pug
+++ /dev/null
@@ -1,26 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-include ./form/form-field-feedback
-include ./form/form-field-label
-include ./form/form-field-text
-include ./form/form-field-password
-include ./form/form-field-dropdown
-include ./form/form-field-datalist
-include ./form/form-field-checkbox
-include ./form/form-field-number
-include ./form/form-field-up
-include ./form/form-field-down
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug
deleted file mode 100644
index a8236a9d455dc..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-checkbox.pug
+++ /dev/null
@@ -1,44 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-mixin form-field-checkbox(label, model, name, disabled, required, tip)
-        label.form-field-checkbox.ignite-form-field
-            .ignite-form-field__control
-                input(
-                    id=`{{ ${name} }}Input`
-                    name=`{{ ${name} }}`
-                    type='checkbox'
-
-                    ng-model=model
-                    ng-required=required && `${required}`
-                    ng-disabled=disabled && `${disabled}`
-                    expose-ignite-form-field-control='$input'
-                )&attributes(attributes ? attributes.attributes ? attributes.attributes : attributes : {})
-                span #{label}
-                +tooltip(tip, tipOpts, 'tipLabel')
-            .ignite-form-field__errors(
-                ng-messages=`$input.$error`
-                ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
-            )
-                if block
-                    block
-                if required
-                    +form-field-feedback(name, 'required', `${errLbl} could not be empty!`)
-
-mixin sane-form-field-checkbox({label, model, name, disabled, required, tip})
-    +form-field-checkbox(label, model, name, disabled = false, required = false, tip)&attributes(attributes)
-        if block
-            block
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-down.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-down.pug
deleted file mode 100644
index 1ced54c7524a4..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-down.pug
+++ /dev/null
@@ -1,18 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-mixin ignite-form-field-down()
-    i.tipField.fa.fa-arrow-down(ignite-form-field-down ng-click='vm.down()')&attributes(attributes)
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug
deleted file mode 100644
index c6579e38bed41..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-dropdown.pug
+++ /dev/null
@@ -1,60 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-mixin ignite-form-field-dropdown(label, model, name, disabled, required, multiple, placeholder, placeholderEmpty, options, tip)
-    mixin form-field-input()
-        -var errLbl = label.substring(0, label.length - 1)
-
-        button.select-toggle.form-control(
-            type='button'
-            id=`{{ ${name} }}Input`
-            name=`{{ ${name} }}`
-
-            data-placeholder=placeholderEmpty ? `{{ ${options}.length > 0 ? '${placeholder}' : '${placeholderEmpty}' }}` : placeholder
-            
-            ng-model=model
-            ng-disabled=disabled && `${disabled}`
-            ng-required=required && `${required}`
-
-            bs-select
-            bs-options=`item.value as item.label for item in ${options}`
-            expose-ignite-form-field-control='$input'
-
-            data-multiple=multiple ? '1' : false
-
-            tabindex='0'
-        )&attributes(attributes.attributes)
-
-    .ignite-form-field.ignite-form-field-dropdown
-        +ignite-form-field__label(label, name, required, disabled)
-            +tooltip(tip, tipOpts)
-        .ignite-form-field__control
-            .input-tip
-                +form-field-input(attributes=attributes)
-        .ignite-form-field__errors(
-            ng-messages=`$input.$error`
-            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
-        )
-            if block
-                block
-
-            if required
-                +form-field-feedback(name, 'required', multiple ? 'At least one option should be selected' : 'An option should be selected')
-
-mixin sane-ignite-form-field-dropdown({label, model, name, disabled = false, required = false, multiple = false, placeholder, placeholderEmpty, options, tip})
-    +ignite-form-field-dropdown(label, model, name, disabled, required, multiple, placeholder, placeholderEmpty, options, tip)&attributes(attributes)
-        if block
-            block
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
deleted file mode 100644
index dcdcf0e179f28..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-feedback.pug
+++ /dev/null
@@ -1,18 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-mixin form-field-feedback(name, error, message)
-    div(ng-message=error) #{message}
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-number.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-number.pug
deleted file mode 100644
index 75f2a20398088..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-number.pug
+++ /dev/null
@@ -1,59 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-mixin ignite-form-field-number(label, model, name, disabled, required, placeholder, min, max, step, tip)
-    -var errLbl = label.substring(0, label.length - 1)
-
-    mixin form-field-input()
-        input.form-control(
-            id=`{{ ${name} }}Input`
-            name=`{{ ${name} }}`
-            placeholder=placeholder
-            type='number'
-
-            min=min ? min : '0'
-            max=max ? max : '{{ Number.MAX_VALUE }}'
-            step=step ? step : '1'
-
-            ng-model=model
-
-            ng-required=required && `${required}`
-            ng-disabled=disabled && `${disabled}`
-            expose-ignite-form-field-control='$input'
-        )&attributes(attributes.attributes)
-
-    .ignite-form-field
-        +ignite-form-field__label(label, name, required, disabled)
-            +tooltip(tip, tipOpts)
-        .ignite-form-field__control
-            .input-tip
-                +form-field-input(attributes=attributes)
-        .ignite-form-field__errors(
-            ng-messages=`$input.$error`
-            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
-        )
-            if block
-                block
-            +form-field-feedback(name, 'required', `${errLbl} could not be empty`)
-            +form-field-feedback(name, 'min', `${errLbl} is less than allowable minimum: ${min || 0}`)
-            +form-field-feedback(name, 'max', `${errLbl} is more than allowable maximum: ${max}`)
-            +form-field-feedback(name, 'number', `Only numbers are allowed`)
-            +form-field-feedback(name, 'step', `${errLbl} step should be ${step || 1}`)
-
-mixin sane-ignite-form-field-number({label, model, name, disabled = 'false', required = false, placeholder, min = '0', max, step = '1', tip})
-    +ignite-form-field-number(label, model, name, disabled, required, placeholder, min, max, step, tip)&attributes(attributes)
-        if block
-            block
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
deleted file mode 100644
index 3e3597476eff8..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-password.pug
+++ /dev/null
@@ -1,47 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-mixin ignite-form-field-password-input(name, model, disabled, required, placeholder)
-    input.form-control(
-        id=`{{ ${name} }}Input`
-        name=`{{ ${name} }}`
-        placeholder=placeholder
-        type='password'
-
-        ng-model=model
-
-        ng-required=required && `${required}`
-        ng-disabled=disabled && `${disabled}`
-        expose-ignite-form-field-control='$input'
-    )&attributes(attributes ? attributes.attributes ? attributes.attributes : attributes : {})
-
-mixin ignite-form-field-password(label, model, name, disabled, required, placeholder, tip)
-    -var errLbl = label.substring(0, label.length - 1)
-
-    .ignite-form-field
-        +ignite-form-field__label(label, name, required, disabled)
-            +tooltip(tip, tipOpts)
-        .ignite-form-field__control
-            .input-tip
-                +ignite-form-field-password-input(name, model, disabled, required, placeholder)(attributes=attributes)
-        .ignite-form-field__errors(
-            ng-messages=`$input.$error`
-            ng-if=`!$input.$pristine && $input.$invalid`
-        )
-            if block
-                block
-
-            +form-field-feedback(name, 'required', `${errLbl} could not be empty!`)
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
deleted file mode 100644
index 3d28e178e7464..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-text.pug
+++ /dev/null
@@ -1,53 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-mixin ignite-form-field-input(name, model, disabled, required, placeholder)
-    input.form-control(
-        id=`{{ ${name} }}Input`
-        name=`{{ ${name} }}`
-        placeholder=placeholder
-
-        ng-model=model
-
-        ng-required=required && `${required}`
-        ng-disabled=disabled && `${disabled}`
-        expose-ignite-form-field-control='$input'
-
-    )&attributes(attributes ? attributes.attributes ? attributes.attributes : attributes : {})
-
-mixin ignite-form-field-text(lbl, model, name, disabled, required, placeholder, tip)
-    -let errLbl = lbl[lbl.length - 1] === ':' ? lbl.substring(0, lbl.length - 1) : lbl
-
-    .ignite-form-field
-        +ignite-form-field__label(lbl, name, required, disabled)
-            +tooltip(tip, tipOpts)
-        .ignite-form-field__control
-            .input-tip
-                +ignite-form-field-input(name, model, disabled, required, placeholder)(attributes=attributes)
-        .ignite-form-field__errors(
-            ng-messages=`$input.$error`
-            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
-        )
-            if block
-                block
-
-            if required
-                +form-field-feedback(name, 'required', `${errLbl} could not be empty!`)
-
-mixin sane-ignite-form-field-text({label, model, name, disabled, required, placeholder, tip})
-    +ignite-form-field-text(label, model, name, disabled, required, placeholder, tip)&attributes(attributes)
-        if block
-            block
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-up.pug b/modules/web-console/frontend/app/helpers/jade/form/form-field-up.pug
deleted file mode 100644
index 3522fb5f5349f..0000000000000
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-up.pug
+++ /dev/null
@@ -1,18 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-mixin ignite-form-field-up()
-    i.tipField.fa.fa-arrow-up.ng-scope(ignite-form-field-up ng-click='vm.up()')&attributes(attributes)
diff --git a/modules/web-console/frontend/app/helpers/jade/mixins.pug b/modules/web-console/frontend/app/helpers/jade/mixins.pug
index 62290c48f13b3..b2aa8e5441b93 100644
--- a/modules/web-console/frontend/app/helpers/jade/mixins.pug
+++ b/modules/web-console/frontend/app/helpers/jade/mixins.pug
@@ -14,296 +14,138 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-include ./form
 include ../../primitives/btn-group/index
 include ../../primitives/datepicker/index
 include ../../primitives/timepicker/index
 include ../../primitives/dropdown/index
-include ../../primitives/tooltip/index
-include ../../primitives/radio/index
 include ../../primitives/switcher/index
 include ../../primitives/form-field/index
 
-//- Mixin for advanced options toggle.
-mixin advanced-options-toggle(click, cond, showMessage, hideMessage)
-    .advanced-options
-        i.fa(
-            ng-click=`${click}`
-            ng-class=`${cond} ? 'fa-chevron-circle-down' : 'fa-chevron-circle-right'`
-        )
-        a(ng-click=click) {{ #{cond} ? '#{hideMessage}' : '#{showMessage}' }}
-
-//- Mixin for advanced options toggle with default settings.
-mixin advanced-options-toggle-default
-    +advanced-options-toggle('toggleExpanded()', 'ui.expanded', 'Show advanced settings...', 'Hide advanced settings...')
-
-//- Mixin for main table on screen with list of items.
-mixin main-table(title, rows, focusId, click, rowTemplate, searchField)
-    .padding-bottom-dflt(ng-show=`${rows} && ${rows}.length > 0`)
-        table.links(st-table='displayedRows' st-safe-src=`${rows}`)
-            thead
-                tr
-                    th
-                        label.labelHeader.labelFormField #{title}:
-                        .col-sm-3.pull-right(style='padding: 0')
-                            input.form-control(type='text' st-search=`${searchField}` placeholder=`Filter ${title}...`)
-            tbody
-                tr
-                    td
-                        .scrollable-y(ng-show='displayedRows.length > 0' style='max-height: 200px')
-                            table
-                                tbody
-                                    tr(ng-repeat='row in displayedRows track by row._id' ignite-bs-affix-update)
-                                        td
-                                            a(ng-class='{active: row._id == selectedItem._id}' ignite-on-click-focus=focusId ng-click=click) #{rowTemplate}
-                        label.placeholder(ng-show='displayedRows.length == 0') No #{title} found
-
-//- Mixin with save, remove, clone and undo buttons.
-mixin save-remove-clone-undo-buttons(objectName)
-    -var removeTip = 'Remove current ' + objectName
-    -var cloneTip = '"Clone current ' + objectName + '"'
-    -var undoTip = '"Undo all changes for current ' + objectName + '"'
-
-    button.btn-ignite.btn-ignite--success(
-        ng-disabled='!ui.inputForm.$dirty'
-        ng-click='ui.inputForm.$dirty && saveItem()'
-    ) Save
-    button.btn-ignite.btn-ignite--success(
-        ng-show='backupItem._id && contentVisible()'
-        type='button'
-        id='clone-item'
-        ng-click='cloneItem()'
-    ) Clone
-
-    .btn-ignite-group(ng-show='backupItem._id && contentVisible()')
-        button.btn-ignite.btn-ignite--success(
-            ng-click='removeItem()'
-            type='button'
-        )
-            | Remove
-        button.btn-ignite.btn-ignite--success(
-            bs-dropdown='$ctrl.extraFormActions'
-            data-placement='top-right'
-            type='button'
-        )
-            span.icon.fa.fa-caret-up
-
-    button.btn-ignite.btn-ignite--success(
-        ng-show='contentVisible()'
-        id='undo-item'
-        ng-disabled='!ui.inputForm.$dirty'
-        ng-click='ui.inputForm.$dirty && resetAll()'
-        bs-tooltip=undoTip
-        data-placement='top'
-        data-trigger='hover'
-    )
-        i.icon.fa.fa-undo()
-
-//- Mixin for feedback on specified error.
-mixin error-feedback(visible, error, errorMessage, name)
-    i.fa.fa-exclamation-triangle.form-control-feedback(
-        ng-if=visible
-        bs-tooltip=`'${errorMessage}'`
-        ignite-error=error
-        ignite-error-message=errorMessage
-        name=name
-    )
-
-//- Mixin for feedback on unique violation.
-mixin unique-feedback(name, errorMessage)
-    +form-field-feedback(name, 'igniteUnique', errorMessage)
-
-//- Mixin for feedback on IP address violation.
-mixin ipaddress-feedback(name)
-    +form-field-feedback(name, 'ipaddress', 'Invalid address!')
-
-//- Mixin for feedback on port of IP address violation.
-mixin ipaddress-port-feedback(name)
-    +form-field-feedback(name, 'ipaddressPort', 'Invalid port!')
-
-//- Mixin for feedback on port range violation.
-mixin ipaddress-port-range-feedback(name)
-    +form-field-feedback(name, 'ipaddressPortRange', 'Invalid port range!')
-
-//- Mixin for feedback on UUID violation.
-mixin uuid-feedback(name)
-    +form-field-feedback(name, 'uuid', 'Invalid node ID!')
-
 //- Function that convert enabled state to corresponding disabled state.
 -var enabledToDisabled = function (enabled) {
 -    return (enabled === false || enabled === true) ? !enabled : '!(' + enabled + ')';
 -}
 
-//- Mixin for checkbox.
-mixin checkbox(lbl, model, name, tip)
-    +form-field-checkbox(lbl, model, name, false, false, tip)
-
-//- Mixin for checkbox with enabled condition.
-mixin checkbox-enabled(lbl, model, name, enabled, tip)
-    +form-field-checkbox(lbl, model, name, enabledToDisabled(enabled), false, tip)
-
-//- Mixin for Java class name field with auto focus condition.
-mixin java-class-autofocus-placholder(lbl, model, name, enabled, required, autofocus, placeholder, tip, validationActive)
-    -var errLbl = lbl.substring(0, lbl.length - 1)
-
-    +ignite-form-field-text(lbl, model, name, enabledToDisabled(enabled), required, placeholder, tip)(
+mixin form-field__java-class({ label, model, name, disabled, required, tip, placeholder, validationActive })
+    -var errLbl = label.substring(0, label.length - 1)
+
+    +form-field__text({
+        label,
+        model,
+        name,
+        disabled,
+        required,
+        placeholder: placeholder || 'Enter fully qualified class name',
+        tip
+    })(
         data-java-identifier='true'
         data-java-package-specified='true'
         data-java-keywords='true'
         data-java-built-in-class='true'
-        data-ignite-form-field-input-autofocus=autofocus
         data-validation-active=validationActive ? `{{ ${validationActive} }}` : `'always'`
     )&attributes(attributes)
         if  block
             block
 
-        +form-field-feedback(name, 'javaBuiltInClass', errLbl + ' should not be the Java built-in class!')
-        +form-field-feedback(name, 'javaKeywords', errLbl + ' could not contains reserved Java keyword!')
-        +form-field-feedback(name, 'javaPackageSpecified', errLbl + ' does not have package specified!')
-        +form-field-feedback(name, 'javaIdentifier', errLbl + ' is invalid Java identifier!')
-
-//- Mixin for Java class name field with auto focus condition.
-mixin java-class-autofocus(lbl, model, name, enabled, required, autofocus, tip, validationActive)
-    +java-class-autofocus-placholder(lbl, model, name, enabled, required, autofocus, 'Enter fully qualified class name', tip, validationActive)&attributes(attributes)
-        if  block
-            block
-
-//- Mixin for Java class name field.
-mixin java-class(lbl, model, name, enabled, required, tip, validationActive)
-    +java-class-autofocus(lbl, model, name, enabled, required, 'false', tip, validationActive)
-        if  block
-            block
+        +form-field__error({ error: 'javaBuiltInClass', message: `${ errLbl } should not be the Java built-in class!` })
+        +form-field__error({ error: 'javaKeywords', message: `${ errLbl } could not contains reserved Java keyword!` })
+        +form-field__error({ error: 'javaPackageSpecified', message: `${ errLbl } does not have package specified!` })
+        +form-field__error({ error: 'javaIdentifier', message: `${ errLbl } is invalid Java identifier!` })
 
 //- Mixin for text field with enabled condition with options.
-mixin java-class-typeahead(lbl, model, name, options, enabled, required, placeholder, tip, validationActive)
-    -var errLbl = lbl.substring(0, lbl.length - 1)
-
-    +form-field-datalist(lbl, model, name, enabledToDisabled(enabled), required, placeholder, options, tip)&attributes(attributes)(
+mixin form-field__java-class--typeahead({ label, model, name, options, disabled, required, placeholder, tip, validationActive })
+    -var errLbl = label.substring(0, label.length - 1)
+
+    +form-field__typeahead({
+        label,
+        model,
+        name,
+        disabled,
+        required,
+        placeholder,
+        options,
+        tip
+    })&attributes(attributes)(
         data-java-identifier='true'
         data-java-package-specified='allow-built-in'
         data-java-keywords='true'
         data-validation-active=validationActive ? `{{ ${validationActive} }}` : `'always'`
     )
-        +form-field-feedback(name, 'javaKeywords', errLbl + ' could not contains reserved Java keyword!')
-        +form-field-feedback(name, 'javaPackageSpecified', errLbl + ' does not have package specified!')
-        +form-field-feedback(name, 'javaIdentifier', errLbl + ' is invalid Java identifier!')
-
-//- Mixin for java package field with enabled condition.
-mixin java-package(lbl, model, name, enabled, required, tip)
-    +ignite-form-field-text(lbl, model, name, enabledToDisabled(enabled), required, 'Enter package name', tip)(
+        +form-field__error({ error: 'javaKeywords', message: `${ errLbl } could not contains reserved Java keyword!` })
+        +form-field__error({ error: 'javaPackageSpecified', message: `${ errLbl } does not have package specified!` })
+        +form-field__error({ error: 'javaIdentifier', message: `${ errLbl } is invalid Java identifier!` })
+
+
+mixin form-field__java-package({ label, model, name, disabled, required, tip, tipOpts, placeholder })
+    +form-field__text({
+        label,
+        model,
+        name,
+        disabled,
+        required,
+        tip,
+        tipOpts,
+        placeholder
+    })(
         data-java-keywords='true'
         data-java-package-name='package-only'
-    )
-        +form-field-feedback(name, 'javaPackageName', 'Package name is invalid')
-        +form-field-feedback(name, 'javaKeywords', 'Package name could not contains reserved java keyword')
-
-//- Mixin for text field with IP address check.
-mixin text-ip-address(lbl, model, name, enabled, placeholder, tip)
-    +ignite-form-field-text(lbl, model, name, enabledToDisabled(enabled), false, placeholder, tip)(data-ipaddress='true')
-        +ipaddress-feedback(name)
-
-//- Mixin for text field with IP address and port range check.
-mixin text-ip-address-with-port-range(lbl, model, name, enabled, placeholder, tip)
-    +ignite-form-field-text(lbl, model, name, enabledToDisabled(enabled), false, placeholder, tip)(data-ipaddress='true' data-ipaddress-with-port='true' data-ipaddress-with-port-range='true')
-        +ipaddress-feedback(name)
-        +ipaddress-port-feedback(name)
-        +ipaddress-port-range-feedback(name)
-
-//- Mixin for text field.
-mixin text-enabled(lbl, model, name, enabled, required, placeholder, tip)
-    +ignite-form-field-text(lbl, model, name, enabledToDisabled(enabled), required, placeholder, tip)
-        if  block
-            block
-
-//- Mixin for text field with autofocus.
-mixin text-enabled-autofocus(lbl, model, name, enabled, required, placeholder, tip)
-    +ignite-form-field-text(lbl, model, name, enabledToDisabled(enabled), required, placeholder, tip)(
-        data-ignite-form-field-input-autofocus='true'
     )&attributes(attributes)
         if  block
             block
 
-//- Mixin for text field.
-mixin text(lbl, model, name, required, placeholder, tip)
-    +ignite-form-field-text(lbl, model, name, false, required, placeholder, tip)
-        if  block
-            block
-
-//- Mixin for url field.
-mixin url(lbl, model, name, enabled, required, placeholder, tip)
-    -var errLbl = lbl.substring(0, lbl.length - 1)
-
-    +ignite-form-field-text(lbl, model, name, enabledToDisabled(enabled), required, placeholder, tip)(type='url')
-        if  block
-            block
-
-        +form-field-feedback(name, 'url', errLbl + ' should be a valid URL!')
-
-//- Mixin for password field.
-mixin password(lbl, model, name, required, placeholder, tip)
-    +ignite-form-field-password(lbl, model, name, false, required, placeholder, tip)
-        if  block
-            block
-
-//- Mixin for text field with enabled condition with options.
-mixin text-options(lbl, model, name, options, enabled, required, placeholder, tip)
-    +form-field-datalist(lbl, model, name, enabledToDisabled(enabled), required, placeholder, options, tip)
-
-//- Mixin for required numeric field.
-mixin number-required(lbl, model, name, enabled, required, placeholder, min, tip)
-    +ignite-form-field-number(lbl, model, name, enabledToDisabled(enabled), required, placeholder, min, false, false, tip)
-
-//- Mixin for required numeric field with maximum and minimum limit.
-mixin number-min-max(lbl, model, name, enabled, placeholder, min, max, tip)
-    +ignite-form-field-number(lbl, model, name, enabledToDisabled(enabled), false, placeholder, min, max, '1', tip)
+        +form-field__error({ error: 'javaPackageName', message: 'Package name is invalid!' })
+        +form-field__error({ error: 'javaKeywords', message: 'Package name could not contains reserved java keyword!' })
 
-//- Mixin for required numeric field with maximum and minimum limit.
-mixin number-min-max-step(lbl, model, name, enabled, placeholder, min, max, step, tip)
-    +ignite-form-field-number(lbl, model, name, enabledToDisabled(enabled), false, placeholder, min, max, step, tip)
-
-//- Mixin for numeric field.
-mixin number(lbl, model, name, enabled, placeholder, min, tip)
-    +ignite-form-field-number(lbl, model, name, enabledToDisabled(enabled), false, placeholder, min, false, false, tip)
-
-//- Mixin for required dropdown field.
-mixin dropdown-required-empty(lbl, model, name, enabled, required, placeholder, placeholderEmpty, options, tip)
-    +ignite-form-field-dropdown(lbl, model, name, enabledToDisabled(enabled), required, false, placeholder, placeholderEmpty, options, tip)&attributes(attributes)
-        if  block
-            block
+//- Mixin for text field with IP address check.
+mixin form-field__ip-address({ label, model, name, enabled, placeholder, tip })
+    +form-field__text({
+        label,
+        model,
+        name,
+        disabled: enabledToDisabled(enabled),
+        placeholder,
+        tip
+    })(data-ipaddress='true')
+        +form-field__error({ error: 'ipaddress', message: 'Invalid address!' })
 
-//- Mixin for required dropdown field with autofocus.
-mixin dropdown-required-empty-autofocus(lbl, model, name, enabled, required, placeholder, placeholderEmpty, options, tip)
-    +ignite-form-field-dropdown(lbl, model, name, enabledToDisabled(enabled), required, false, placeholder, placeholderEmpty, options, tip)(
-        data-ignite-form-field-input-autofocus='true'
+//- Mixin for text field with IP address and port range check.
+mixin form-field__ip-address-with-port-range({ label, model, name, enabled, placeholder, tip })
+    +form-field__text({
+        label,
+        model,
+        name,
+        disabled: enabledToDisabled(enabled),
+        placeholder,
+        tip
+    })(
+        data-ipaddress='true'
+        data-ipaddress-with-port='true'
+        data-ipaddress-with-port-range='true'
     )
-        if  block
-            block
-
-//- Mixin for required dropdown field.
-mixin dropdown-required(lbl, model, name, enabled, required, placeholder, options, tip)
-    +ignite-form-field-dropdown(lbl, model, name, enabledToDisabled(enabled), required, false, placeholder, '', options, tip)&attributes(attributes)
-        if  block
-            block
+        +form-field__error({ error: 'ipaddress', message: 'Invalid address!' })
+        +form-field__error({ error: 'ipaddressPort', message: 'Invalid port!' })
+        +form-field__error({ error: 'ipaddressPortRange', message: 'Invalid port range!' })
 
-//- Mixin for required dropdown field with autofocus.
-mixin dropdown-required-autofocus(lbl, model, name, enabled, required, placeholder, options, tip)
-    +ignite-form-field-dropdown(lbl, model, name, enabledToDisabled(enabled), required, false, placeholder, '', options, tip)(
-        data-ignite-form-field-input-autofocus='true'
+//- Mixin for url field.
+mixin form-field__url({ label, model, name, enabled, required, placeholder, tip })
+    -var errLbl = label.substring(0, label.length - 1)
+
+    +form-field__text({
+        label,
+        model,
+        name,
+        disabled: enabledToDisabled(enabled),
+        required,
+        placeholder,
+        tip
+    })(
+        type='url'
     )
         if  block
             block
 
-//- Mixin for dropdown field.
-mixin dropdown(lbl, model, name, enabled, placeholder, options, tip)
-    +ignite-form-field-dropdown(lbl, model, name, enabledToDisabled(enabled), false, false, placeholder, '', options, tip)
-        if  block
-            block
+        +form-field__error({ error: 'url', message: `${ errLbl } should be a valid URL!` })
 
-//- Mixin for dropdown-multiple field.
-mixin dropdown-multiple(lbl, model, name, enabled, placeholder, placeholderEmpty, options, tip)
-    +ignite-form-field-dropdown(lbl, model, name, enabledToDisabled(enabled), false, true, placeholder, placeholderEmpty, options, tip)
-        if  block
-            block
 
 mixin list-text-field({ items, lbl, name, itemName, itemsName })
     list-editable(ng-model=items)&attributes(attributes)
@@ -311,9 +153,15 @@ mixin list-text-field({ items, lbl, name, itemName, itemsName })
             | {{ $item }}
 
         list-editable-item-edit
-            +ignite-form-field-text(lbl, '$item', `"${name}"`, false, true, `Enter ${lbl.toLowerCase()}`)(
-                data-ignite-unique=items
-                data-ignite-form-field-input-autofocus='true'
+            +form-field__text({
+                label: lbl,
+                model: '$item',
+                name: `"${name}"`,
+                required: true,
+                placeholder: `Enter ${lbl.toLowerCase()}`
+            })(
+                ignite-unique=items
+                ignite-form-field-input-autofocus='true'
             )
                 if  block
                     block
@@ -325,46 +173,44 @@ mixin list-text-field({ items, lbl, name, itemName, itemsName })
                 label-multiple=itemsName
             )
 
-mixin list-java-class-field(lbl, model, name, items)
-    +ignite-form-field-text(lbl, model, name, false, true, 'Enter fully qualified class name')(
-        data-java-identifier='true'
-        data-java-package-specified='true'
-        data-java-keywords='true'
-        data-java-built-in-class='true'
-
-        data-ignite-unique=items
-        data-ignite-form-field-input-autofocus='true'
+mixin list-java-class-field(label, model, name, items)
+    +form-field__text({
+        label,
+        model,
+        name,
+        required: true,
+        placeholder: 'Enter fully qualified class name'
+    })(
+        java-identifier='true'
+        java-package-specified='true'
+        java-keywords='true'
+        java-built-in-class='true'
+
+        ignite-unique=items
+        ignite-form-field-input-autofocus='true'
     )
-        +form-field-feedback(name, 'javaBuiltInClass', lbl + ' should not be the Java built-in class!')
-        +form-field-feedback(name, 'javaKeywords', lbl + ' could not contains reserved Java keyword!')
-        +form-field-feedback(name, 'javaPackageSpecified', lbl + ' does not have package specified!')
-        +form-field-feedback(name, 'javaIdentifier', lbl + ' is invalid Java identifier!')
-
-        if block
-            block
-
-mixin list-java-package-field(lbl, model, name, items)
-    +ignite-form-field-text(lbl, model, name, false, true, 'Enter package name')(
-        data-java-keywords='true'
-        data-java-package-name='package-only'
-
-        data-ignite-unique=items
-        data-ignite-form-field-input-autofocus='true'
-    )&attributes(attributes)
-        +form-field-feedback(name, 'javaKeywords', 'Package name could not contains reserved Java keyword!')
-        +form-field-feedback(name, 'javaPackageName', 'Package name is invalid!')
+        +form-field__error({ error: 'javaBuiltInClass', message: `${ label } should not be the Java built-in class!` })
+        +form-field__error({ error: 'javaKeywords', message: `${ label } could not contains reserved Java keyword!` })
+        +form-field__error({ error: 'javaPackageSpecified', message: `${ label } does not have package specified!` })
+        +form-field__error({ error: 'javaIdentifier', message: `${ label } is invalid Java identifier!` })
 
         if block
             block
 
-mixin list-url-field(lbl, model, name, items)
-    +ignite-form-field-text(lbl, model, name, false, true, 'Enter URL')(
+mixin list-url-field(label, model, name, items)
+    +form-field__text({
+        label,
+        model,
+        name,
+        required: true,
+        placeholder: 'Enter URL'
+    })(
         type='url'
 
-        data-ignite-unique=items
-        data-ignite-form-field-input-autofocus='true'
+        ignite-unique=items
+        ignite-form-field-input-autofocus='true'
     )
-        +form-field-feedback(name, 'url', 'URL should be valid!')
+        +form-field__error({ error: 'url', message: 'URL should be valid!' })
 
         if block
             block
@@ -377,18 +223,24 @@ mixin list-addresses({ items, name, tip, withPortRange = true })
     )&attributes(attributes)
         list-editable-item-view {{ $item }}
         list-editable-item-edit(item-name='address')
-            +ignite-form-field-text('Address', 'address', '"address"', false, true, 'IP address:port')(
-                data-ipaddress='true'
-                data-ipaddress-with-port='true'
-                data-ipaddress-with-port-range=withPortRange
-                data-ignite-unique=items
-                data-ignite-form-field-input-autofocus='true'
+            +form-field__text({
+                label: 'Address',
+                model: 'address',
+                name: '"address"',
+                required: true,
+                placeholder: 'IP address:port'
+            })(
+                ipaddress='true'
+                ipaddress-with-port='true'
+                ipaddress-with-port-range=withPortRange
+                ignite-unique=items
+                ignite-form-field-input-autofocus='true'
             )
-                +unique-feedback('"address"', 'Such IP address already exists!')
-                +ipaddress-feedback('"address"')
-                +ipaddress-port-feedback('"address"')
-                +ipaddress-port-range-feedback('"address"')
-                +form-field-feedback('"address"', 'required', 'IP address:port could not be empty!')
+                +form-field__error({ error: 'igniteUnique', message: 'Such IP address already exists!' })
+                +form-field__error({ error: 'ipaddress', message: 'Invalid address!' })
+                +form-field__error({ error: 'ipaddressPort', message: 'Invalid port!' })
+                +form-field__error({ error: 'ipaddressPortRange', message: 'Invalid port range!' })
+                +form-field__error({ error: 'required', message: 'IP address:port could not be empty!' })
 
         list-editable-no-items
             list-editable-add-item-button(
@@ -397,29 +249,32 @@ mixin list-addresses({ items, name, tip, withPortRange = true })
                 label-single='address'
             )
 
-//- Mixin for cache mode.
-mixin cacheMode(lbl, model, name, placeholder)
-    +dropdown(lbl, model, name, 'true', placeholder,
-        '[\
+
+mixin form-field__cache-modes({ label, model, name, placeholder })
+    +form-field__dropdown({
+        label, model, name, placeholder,
+        options: '[\
             {value: "LOCAL", label: "LOCAL"},\
             {value: "REPLICATED", label: "REPLICATED"},\
             {value: "PARTITIONED", label: "PARTITIONED"}\
         ]',
-        'Cache modes:\
+        tip: 'Cache modes:\
         <ul>\
             <li>PARTITIONED - in this mode the overall key set will be divided into partitions and all partitions will be split equally between participating nodes</li>\
             <li>REPLICATED - in this mode all the keys are distributed to all participating nodes</li>\
             <li>LOCAL - in this mode caches residing on different grid nodes will not know about each other</li>\
         </ul>'
-    )
+    })&attributes(attributes)
+        if  block
+            block
 
 //- Mixin for eviction policy.
-mixin evictionPolicy(model, name, enabled, required, tip)
+mixin form-field__eviction-policy({ model, name, enabled, required, tip })
     -var kind = model + '.kind'
     -var policy = model + '[' + kind + ']'
 
     .pc-form-grid-col-60
-        +sane-ignite-form-field-dropdown({
+        +form-field__dropdown({
             label: 'Eviction policy:',
             model: kind,
             name: `${name}+"Kind"`,
@@ -431,10 +286,17 @@ mixin evictionPolicy(model, name, enabled, required, tip)
         })
     .pc-form-group.pc-form-grid-row(ng-if=kind)
         .pc-form-grid-col-30
-            +number('Batch size', policy + '.batchSize', name + '+ "batchSize"', enabled, '1', '1',
-                'Number of entries to remove on shrink')
+            +form-field__number({
+                label: 'Batch size',
+                model: policy + '.batchSize',
+                name: name + '+ "batchSize"',
+                disabled: enabledToDisabled(enabled),
+                placeholder: '1',
+                min: '1',
+                tip: 'Number of entries to remove on shrink'
+            })
         .pc-form-grid-col-30
-            pc-form-field-size(
+            form-field-size(
                 label='Max memory size:'
                 ng-model=`${policy}.maxMemorySize`
                 ng-model-options='{allowInvalid: true}'
@@ -446,9 +308,9 @@ mixin evictionPolicy(model, name, enabled, required, tip)
                 size-scale-label='mb'
                 size-type='bytes'
             )
-                +form-field-feedback(null, 'min', 'Either maximum memory size or maximum size should be greater than 0')
+                +form-field__error({ error: 'min', message: 'Either maximum memory size or maximum size should be greater than 0' })
         .pc-form-grid-col-60
-            +sane-ignite-form-field-number({
+            +form-field__number({
                 label: 'Max size:',
                 model: policy + '.maxSize',
                 name: name + '+ "maxSize"',
@@ -459,33 +321,7 @@ mixin evictionPolicy(model, name, enabled, required, tip)
             })(
                 ng-model-options='{allowInvalid: true}'
             )
-                +form-field-feedback(null, 'min', 'Either maximum memory size or maximum size should be greater than 0')
-
-//- Mixin for clusters dropdown.
-mixin clusters(model, tip)
-    +dropdown-multiple('Clusters:',
-        model + '.clusters', '"clusters"', true, 'Choose clusters', 'No clusters configured', 'clusters', tip)
-
-//- Mixin for caches dropdown.
-mixin caches(model, tip)
-    +dropdown-multiple('Caches:',
-        model + '.caches', '"caches"', true, 'Choose caches', 'No caches configured', 'caches', tip)
-
-//- Mixin for XML, Java, .Net preview.
-mixin preview(master, generator, detail)
-    ignite-ui-ace-tabs
-        .preview-panel(ng-init='mode = "spring"')
-            .preview-legend
-                a(ng-class='{active: mode === "spring"}' ng-click='mode = "spring"') Spring
-                a(ng-class='{active: mode === "java"}' ng-click='mode = "java"') Java
-                a(ng-class='{active: mode === "csharp"}' ng-click='mode = "csharp"') C#
-                //a(ng-class='{active: mode === "app.config"}' ng-click='mode = "app.config"') app.config
-            .preview-content(ng-switch='mode')
-                ignite-ui-ace-spring(ng-switch-when='spring' data-master=master data-generator=generator ng-model='$parent.data' data-detail=detail)
-                ignite-ui-ace-java(ng-switch-when='java' data-master=master data-generator=generator ng-model='$parent.data' data-detail=detail)
-                ignite-ui-ace-sharp(ng-switch-when='csharp' data-master=master data-generator=generator ng-model='$parent.data' data-detail=detail)
-            .preview-content-empty(ng-if='!data')
-                label All Defaults
+                +form-field__error({ error: 'min', message: 'Either maximum memory size or maximum size should be greater than 0' })
 
 //- Mixin for XML and Java preview.
 mixin preview-xml-java(master, generator, detail)
@@ -503,21 +339,6 @@ mixin preview-xml-java(master, generator, detail)
             .preview-content-empty(ng-if='!data')
                 label All Defaults
 
-//- LEGACY mixin for LEGACY tables.
-mixin btn-save(show, click)
-    i.tipField.fa.fa-floppy-o(ng-show=show ng-click=click bs-tooltip='' data-title='Click icon or press [Enter] to save item' data-trigger='hover')
-
-//- LEGACY mixin for LEGACY tables.
-mixin btn-add(click, tip)
-    i.tipField.fa.fa-plus(ng-click=click bs-tooltip=tip data-trigger = 'hover')
-
-//- LEGACY mixin for LEGACY tables.
-mixin btn-remove(click, tip)
-    i.tipField.fa.fa-remove(ng-click=click bs-tooltip=tip data-trigger='hover')
-
-//- LEGACY mixin for LEGACY tables.
-mixin btn-remove-cond(cond, click, tip)
-    i.tipField.fa.fa-remove(ng-show=cond ng-click=click bs-tooltip=tip data-trigger='hover')
 
 mixin list-pair-edit({ items, keyLbl, valLbl, itemName, itemsName })
     list-editable(ng-model=items)
@@ -528,14 +349,26 @@ mixin list-pair-edit({ items, keyLbl, valLbl, itemName, itemsName })
             - form = '$parent.form'
             .pc-form-grid-row
                 .pc-form-grid-col-30(divider='=')
-                    +ignite-form-field-text(keyLbl, '$item.name', '"name"', false, true, keyLbl)(
-                        data-ignite-unique=items
-                        data-ignite-unique-property='name'
+                    +form-field__text({
+                        label: keyLbl,
+                        model: '$item.name',
+                        name: '"name"',
+                        required: true,
+                        placeholder: keyLbl
+                    })(
+                        ignite-unique=items
+                        ignite-unique-property='name'
                         ignite-auto-focus
                     )
-                        +unique-feedback('"name"', 'Property with such name already exists!')
+                        +form-field__error({ error: 'igniteUnique', message: 'Property with such name already exists!' })
                 .pc-form-grid-col-30
-                    +ignite-form-field-text(valLbl, '$item.value', '"value"', false, true, valLbl)
+                    +form-field__text({
+                        label: valLbl,
+                        model: '$item.value',
+                        name: '"value"',
+                        required: true,
+                        placeholder: valLbl
+                    })
 
         list-editable-no-items
             list-editable-add-item-button(
@@ -544,9 +377,14 @@ mixin list-pair-edit({ items, keyLbl, valLbl, itemName, itemsName })
                 label-multiple=itemsName
             )
 
-//- Mixin for DB dialect.
-mixin dialect(lbl, model, name, required, tipTitle, genericDialectName, placeholder)
-    +dropdown-required(lbl, model, name, 'true', required, placeholder, '[\
+mixin form-field__dialect({ label, model, name, required, tip, genericDialectName, placeholder })
+    +form-field__dropdown({
+        label,
+        model,
+        name,
+        required,
+        placeholder,
+        options: '[\
                 {value: "Generic", label: "' + genericDialectName + '"},\
                 {value: "Oracle", label: "Oracle"},\
                 {value: "DB2", label: "IBM DB2"},\
@@ -555,13 +393,14 @@ mixin dialect(lbl, model, name, required, tipTitle, genericDialectName, placehol
                 {value: "PostgreSQL", label: "PostgreSQL"},\
                 {value: "H2", label: "H2 database"}\
         ]',
-        tipTitle +
-        '<ul>\
-            <li>' + genericDialectName + '</li>\
-            <li>Oracle database</li>\
-            <li>IBM DB2</li>\
-            <li>Microsoft SQL Server</li>\
-            <li>MySQL</li>\
-            <li>PostgreSQL</li>\
-            <li>H2 database</li>\
-        </ul>')
+        tip: `${ tip }
+            <ul>
+                <li>${ genericDialectName }</li>
+                <li>Oracle database</li>
+                <li>IBM DB2</li>
+                <li>Microsoft SQL Server</li>
+                <li>MySQL</li>
+                <li>PostgreSQL</li>
+                <li>H2 database</li>
+            </ul>`
+    })
diff --git a/modules/web-console/frontend/app/modules/ace.module.js b/modules/web-console/frontend/app/modules/ace.module.js
index 44e51caf0a015..4decbe1e6bd2b 100644
--- a/modules/web-console/frontend/app/modules/ace.module.js
+++ b/modules/web-console/frontend/app/modules/ace.module.js
@@ -21,7 +21,7 @@ import _ from 'lodash';
 angular
     .module('ignite-console.ace', [])
     .constant('igniteAceConfig', {})
-    .directive('igniteAce', ['igniteAceConfig', (aceConfig) => {
+    .directive('igniteAce', ['igniteAceConfig', function(aceConfig) {
         if (_.isUndefined(window.ace))
             throw new Error('ignite-ace need ace to work... (o rly?)');
 
diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
index 06aca74e7b034..f667374bbec2f 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
@@ -20,6 +20,9 @@ import {nonEmpty, nonNil} from 'app/utils/lodashMixins';
 
 import { BehaviorSubject } from 'rxjs/BehaviorSubject';
 import 'rxjs/add/operator/first';
+import 'rxjs/add/operator/partition';
+import 'rxjs/add/operator/takeUntil';
+import 'rxjs/add/operator/pluck';
 
 import AgentModal from './AgentModal.service';
 // @ts-ignore
@@ -115,7 +118,7 @@ class ConnectionState {
 }
 
 export default class AgentManager {
-    static $inject = ['$rootScope', '$q', '$transitions', 'igniteSocketFactory', AgentModal.name, 'UserNotifications', 'IgniteVersion', ClusterLoginService.name];
+    static $inject = ['$rootScope', '$q', '$transitions', 'igniteSocketFactory', 'AgentModal', 'UserNotifications', 'IgniteVersion', 'ClusterLoginService'];
 
     /** @type {ng.IScope} */
     $root;
@@ -139,7 +142,7 @@ export default class AgentManager {
 
     pool = new SimpleWorkerPool('decompressor', Worker, 4);
 
-    /** @type {Set<ng.IDifferend>} */
+    /** @type {Set<ng.IPromise<unknown>>} */
     promises = new Set();
 
     socket = null;
@@ -156,8 +159,25 @@ export default class AgentManager {
         }
     }
 
+    /**
+     * @param {ng.IRootScopeService} $root
+     * @param {ng.IQService} $q
+     * @param {import('@uirouter/angularjs').TransitionService} $transitions
+     * @param {unknown} socketFactory
+     * @param {import('./AgentModal.service').default} agentModal
+     * @param {import('app/components/user-notifications/service').default} UserNotifications
+     * @param {import('app/services/Version.service').default} Version
+     * @param {import('./components/cluster-login/service').default} ClusterLoginSrv
+     */
     constructor($root, $q, $transitions, socketFactory, agentModal, UserNotifications, Version, ClusterLoginSrv) {
-        Object.assign(this, {$root, $q, $transitions, socketFactory, agentModal, UserNotifications, Version, ClusterLoginSrv});
+        this.$root = $root;
+        this.$q = $q;
+        this.$transitions = $transitions;
+        this.socketFactory = socketFactory;
+        this.agentModal = agentModal;
+        this.UserNotifications = UserNotifications;
+        this.Version = Version;
+        this.ClusterLoginSrv = ClusterLoginSrv;
 
         let prevCluster;
 
@@ -165,6 +185,11 @@ export default class AgentManager {
             .distinctUntilChanged(({ cluster }) => prevCluster === cluster)
             .do(({ cluster }) => prevCluster = cluster);
 
+        this.clusterIsActive$ = this.connectionSbj
+            .map(({ cluster }) => cluster)
+            .filter((cluster) => Boolean(cluster))
+            .pluck('active');
+
         if (!this.isDemoMode()) {
             this.connectionSbj.subscribe({
                 next: ({cluster}) => {
diff --git a/modules/web-console/frontend/app/modules/agent/AgentModal.service.js b/modules/web-console/frontend/app/modules/agent/AgentModal.service.js
index 15a08a2287784..e4b41ac26c3a8 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentModal.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentModal.service.js
@@ -15,29 +15,37 @@
  * limitations under the License.
  */
 
+import angular from 'angular';
+import _ from 'lodash';
 import templateUrl from 'views/templates/agent-download.tpl.pug';
 
 export default class AgentModal {
     static $inject = ['$rootScope', '$state', '$modal', 'IgniteMessages'];
 
+    /**
+     * @param {ng.IRootScopeService} $root
+     * @param {import('@uirouter/angularjs').StateService} $state
+     * @param {mgcrea.ngStrap.modal.IModalService} $modal
+     * @param {ReturnType<typeof import('app/services/Messages.service').default>} Messages
+     */
     constructor($root, $state, $modal, Messages) {
         const self = this;
 
         this.$root = $root;
-        self.$state = $state;
-        self.Messages = Messages;
+        this.$state = $state;
+        this.Messages = Messages;
 
         // Pre-fetch modal dialogs.
-        self.modal = $modal({
+        this.modal = $modal({
             templateUrl,
             show: false,
             backdrop: 'static',
             keyboard: false,
-            controller: () => self,
+            controller() { return self;},
             controllerAs: 'ctrl'
         });
 
-        $root.$on('user', (event, user) => self.user = user);
+        $root.$on('user', (event, user) => this.user = user);
     }
 
     hide() {
@@ -63,14 +71,12 @@ export default class AgentModal {
      * @param {String} [backText]
      */
     agentDisconnected(backText, backState) {
-        const self = this;
+        this.backText = backText;
+        this.backState = backState;
 
-        self.backText = backText;
-        self.backState = backState;
+        this.status = 'agentMissing';
 
-        self.status = 'agentMissing';
-
-        self.modal.$promise.then(self.modal.show);
+        this.modal.$promise.then(() => this.modal.show());
     }
 
     /**
@@ -78,14 +84,12 @@ export default class AgentModal {
      * @param {String} [backText]
      */
     clusterDisconnected(backText, backState) {
-        const self = this;
-
-        self.backText = backText;
-        self.backState = backState;
+        this.backText = backText;
+        this.backState = backState;
 
-        self.status = 'nodeMissing';
+        this.status = 'nodeMissing';
 
-        self.modal.$promise.then(self.modal.show);
+        this.modal.$promise.then(() => this.modal.show());
     }
 
     get securityToken() {
diff --git a/modules/web-console/frontend/app/modules/agent/agent.module.js b/modules/web-console/frontend/app/modules/agent/agent.module.js
index 1812af088e619..9189d92c33312 100644
--- a/modules/web-console/frontend/app/modules/agent/agent.module.js
+++ b/modules/web-console/frontend/app/modules/agent/agent.module.js
@@ -26,5 +26,5 @@ angular
     .module('ignite-console.agent', [
         clusterLogin.name
     ])
-    .service(AgentModal.name, AgentModal)
-    .service(AgentManager.name, AgentManager);
+    .service('AgentModal', AgentModal)
+    .service('AgentManager', AgentManager);
diff --git a/modules/web-console/frontend/app/modules/agent/components/cluster-login/component.js b/modules/web-console/frontend/app/modules/agent/components/cluster-login/component.js
index a8311ed76ee55..e932bad494c07 100644
--- a/modules/web-console/frontend/app/modules/agent/components/cluster-login/component.js
+++ b/modules/web-console/frontend/app/modules/agent/components/cluster-login/component.js
@@ -19,7 +19,6 @@ import template from './template.pug';
 import {ClusterSecrets} from '../../types/ClusterSecrets';
 
 export const component = {
-    name: 'clusterLogin',
     bindings: {
         secrets: '=',
         onLogin: '&',
@@ -28,6 +27,12 @@ export const component = {
     controller: class {
         /** @type {ClusterSecrets} */
         secrets;
+        /** @type {ng.ICompiledExpression} */
+        onLogin;
+        /** @type {ng.ICompiledExpression} */
+        onHide;
+        /** @type {ng.IFormController} */
+        form;
 
         login() {
             if (this.form.$invalid)
diff --git a/modules/web-console/frontend/app/modules/agent/components/cluster-login/index.js b/modules/web-console/frontend/app/modules/agent/components/cluster-login/index.js
index 24cce4ff123f8..177ccdaae0c6a 100644
--- a/modules/web-console/frontend/app/modules/agent/components/cluster-login/index.js
+++ b/modules/web-console/frontend/app/modules/agent/components/cluster-login/index.js
@@ -20,7 +20,6 @@ import {component} from './component';
 import service from './service';
 
 export default angular
-    .module('ignite-console.agent.cluster-login', [
-    ])
-    .service(service.name, service)
-    .component(component.name, component);
+    .module('ignite-console.agent.cluster-login', [])
+    .service('ClusterLoginService', service)
+    .component('clusterLogin', component);
diff --git a/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js b/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js
index a066bcf60ded6..955d0a3307e6c 100644
--- a/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js
+++ b/modules/web-console/frontend/app/modules/agent/components/cluster-login/service.js
@@ -17,23 +17,33 @@
 
 import _ from 'lodash';
 
-import {ClusterSecrets} from '../../types/ClusterSecrets';
 import {CancellationError} from 'app/errors/CancellationError';
 
 export default class ClusterLoginService {
     static $inject = ['$modal', '$q'];
 
+    deferred;
+
+    /**
+     * @param {mgcrea.ngStrap.modal.IModalService} $modal
+     * @param {ng.IQService} $q
+     */
     constructor($modal, $q) {
         this.$modal = $modal;
         this.$q = $q;
     }
 
     /**
-     * @param {ClusterSecrets} baseSecrets
-     * @return {ng.IDifferend<ClusterSecrets>}
+     * @param {import('../../types/ClusterSecrets').ClusterSecrets} baseSecrets
+     * @returns {ng.IPromise<import('../../types/ClusterSecrets').ClusterSecrets>}
      */
     askCredentials(baseSecrets) {
-        const deferred = this.$q.defer();
+        if (this.deferred)
+            return this.deferred.promise;
+
+        this.deferred = this.$q.defer();
+
+        const self = this;
 
         const modal = this.$modal({
             template: `
@@ -47,11 +57,11 @@ export default class ClusterLoginService {
                 this.secrets = _.clone(baseSecrets);
 
                 this.onLogin = () => {
-                    deferred.resolve(this.secrets);
+                    self.deferred.resolve(this.secrets);
                 };
 
                 this.onHide = () => {
-                    deferred.reject(new CancellationError());
+                    self.deferred.reject(new CancellationError());
                 };
             }],
             controllerAs: '$ctrl',
@@ -60,7 +70,11 @@ export default class ClusterLoginService {
         });
 
         return modal.$promise
-            .then(() => deferred.promise)
-            .finally(modal.hide);
+            .then(() => this.deferred.promise)
+            .finally(() => {
+                this.deferred = null;
+
+                modal.hide();
+            });
     }
 }
diff --git a/modules/web-console/frontend/app/modules/agent/components/cluster-login/template.pug b/modules/web-console/frontend/app/modules/agent/components/cluster-login/template.pug
index c6bc474947393..19ab6f43372b8 100644
--- a/modules/web-console/frontend/app/modules/agent/components/cluster-login/template.pug
+++ b/modules/web-console/frontend/app/modules/agent/components/cluster-login/template.pug
@@ -23,7 +23,6 @@ include /app/helpers/jade/mixins
        form.modal-content(name=form novalidate ng-submit='$ctrl.login()')
             .modal-header
                 h4.modal-title
-                    i.icon-confirm
                     span Cluster Authentication
                 button.close(type='button' aria-label='Close' ng-click='$ctrl.onHide()')
                     svg(ignite-icon="cross")
@@ -53,5 +52,7 @@ include /app/helpers/jade/mixins
                             autocomplete='node-password'
                         )
             .modal-footer
-                button#btn-cancel.btn-ignite.btn-ignite--link-success(type='button' ng-click='$ctrl.onHide()') Cancel
-                button#btn-login.btn-ignite.btn-ignite--success Login
+                div
+                    button#btn-cancel.btn-ignite.btn-ignite--link-success(type='button' ng-click='$ctrl.onHide()') Cancel
+                    button#btn-login.btn-ignite.btn-ignite--success Login
+
diff --git a/modules/web-console/frontend/app/modules/branding/branding.module.js b/modules/web-console/frontend/app/modules/branding/branding.module.js
index 2313728c837c4..20cd93e015f2b 100644
--- a/modules/web-console/frontend/app/modules/branding/branding.module.js
+++ b/modules/web-console/frontend/app/modules/branding/branding.module.js
@@ -41,9 +41,9 @@ angular
 
     tfMetaTagsProvider.setTitleSuffix(' – Apache Ignite Web Console');
 }])
-.directive(...ignitePoweredByApache)
-.directive(...igniteHeaderLogo)
-.directive(...igniteHeaderTitle)
-.directive(...igniteTerms)
-.directive(...igniteFeatures)
-.directive(...igniteFooter);
+.directive('ignitePoweredByApache', ignitePoweredByApache)
+.directive('igniteHeaderLogo', igniteHeaderLogo)
+.directive('igniteHeaderTitle', igniteHeaderTitle)
+.directive('igniteTerms', igniteTerms)
+.directive('igniteFeatures', igniteFeatures)
+.directive('igniteFooter', igniteFooter);
diff --git a/modules/web-console/frontend/app/modules/branding/branding.service.js b/modules/web-console/frontend/app/modules/branding/branding.service.js
index ebef881182e5f..667dbfd0f6182 100644
--- a/modules/web-console/frontend/app/modules/branding/branding.service.js
+++ b/modules/web-console/frontend/app/modules/branding/branding.service.js
@@ -18,6 +18,9 @@
 export default class {
     static $inject = ['IgniteVersion'];
 
+    /**
+     * @param {import('app/services/Version.service').default} Version
+     */
     constructor(Version) {
         this.titleSuffix = ' - Apache Ignite Web Console';
 
diff --git a/modules/web-console/frontend/app/modules/branding/features.directive.js b/modules/web-console/frontend/app/modules/branding/features.directive.js
index 9226a3f6c98c6..39c639c9f8f18 100644
--- a/modules/web-console/frontend/app/modules/branding/features.directive.js
+++ b/modules/web-console/frontend/app/modules/branding/features.directive.js
@@ -17,7 +17,10 @@
 
 const template = '<div class="features" ng-bind-html="features.html"></div>';
 
-export default ['igniteFeatures', ['IgniteBranding', (branding) => {
+/**
+ * @param {import('./branding.service').default} branding
+ */
+export default function factory(branding) {
     function controller() {
         const ctrl = this;
 
@@ -31,5 +34,7 @@ export default ['igniteFeatures', ['IgniteBranding', (branding) => {
         controllerAs: 'features',
         replace: true
     };
-}]];
+}
+
+factory.$inject = ['IgniteBranding'];
 
diff --git a/modules/web-console/frontend/app/modules/branding/footer.directive.js b/modules/web-console/frontend/app/modules/branding/footer.directive.js
index f0b199458c10f..3c0bb83e740f5 100644
--- a/modules/web-console/frontend/app/modules/branding/footer.directive.js
+++ b/modules/web-console/frontend/app/modules/branding/footer.directive.js
@@ -17,7 +17,10 @@
 
 const template = '<div class="footer" ng-bind-html="footer.html"></div>';
 
-export default ['igniteFooter', ['IgniteBranding', (branding) => {
+/**
+ * @param {import('./branding.service').default} branding
+ */
+export default function factory(branding) {
     function controller() {
         const ctrl = this;
 
@@ -31,4 +34,6 @@ export default ['igniteFooter', ['IgniteBranding', (branding) => {
         controllerAs: 'footer',
         replace: true
     };
-}]];
+}
+
+factory.$inject = ['IgniteBranding'];
diff --git a/modules/web-console/frontend/app/modules/branding/header-logo.directive.js b/modules/web-console/frontend/app/modules/branding/header-logo.directive.js
index 231411b818601..60e5d5d68aec0 100644
--- a/modules/web-console/frontend/app/modules/branding/header-logo.directive.js
+++ b/modules/web-console/frontend/app/modules/branding/header-logo.directive.js
@@ -17,7 +17,10 @@
 
 import template from './header-logo.pug';
 
-export default ['igniteHeaderLogo', ['IgniteBranding', (branding) => {
+/**
+ * @param {import('./branding.service').default} branding
+ */
+export default function factory(branding) {
     function controller() {
         const ctrl = this;
 
@@ -31,4 +34,6 @@ export default ['igniteHeaderLogo', ['IgniteBranding', (branding) => {
         controllerAs: 'logo',
         replace: true
     };
-}]];
+}
+
+factory.$inject = ['IgniteBranding'];
diff --git a/modules/web-console/frontend/app/modules/branding/header-title.directive.js b/modules/web-console/frontend/app/modules/branding/header-title.directive.js
index aedd4b9350a02..f67439c0a64da 100644
--- a/modules/web-console/frontend/app/modules/branding/header-title.directive.js
+++ b/modules/web-console/frontend/app/modules/branding/header-title.directive.js
@@ -21,7 +21,10 @@ const template = `
     >{{::title.text}}</h1>
 `;
 
-export default ['igniteHeaderTitle', ['IgniteBranding', (branding) => {
+/**
+ * @param {import('./branding.service').default} branding
+ */
+export default function factory(branding) {
     function controller() {
         const ctrl = this;
 
@@ -35,5 +38,7 @@ export default ['igniteHeaderTitle', ['IgniteBranding', (branding) => {
         controllerAs: 'title',
         replace: true
     };
-}]];
+}
+
+factory.$inject = ['IgniteBranding'];
 
diff --git a/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js b/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js
index e88f71f74bc46..3f8b3deb5bfc9 100644
--- a/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js
+++ b/modules/web-console/frontend/app/modules/branding/powered-by-apache.directive.js
@@ -17,7 +17,10 @@
 
 import template from './powered-by-apache.pug';
 
-export default ['ignitePoweredByApache', ['IgniteBranding', (branding) => {
+/**
+ * @param {import('./branding.service').default} branding
+ */
+export default function factory(branding) {
     function controller() {
         const ctrl = this;
 
@@ -30,5 +33,7 @@ export default ['ignitePoweredByApache', ['IgniteBranding', (branding) => {
         controller,
         controllerAs: 'poweredBy'
     };
-}]];
+}
+
+factory.$inject = ['IgniteBranding'];
 
diff --git a/modules/web-console/frontend/app/modules/branding/terms.directive.js b/modules/web-console/frontend/app/modules/branding/terms.directive.js
index 020774516faaf..8126a39074a9c 100644
--- a/modules/web-console/frontend/app/modules/branding/terms.directive.js
+++ b/modules/web-console/frontend/app/modules/branding/terms.directive.js
@@ -15,7 +15,10 @@
  * limitations under the License.
  */
 
-export default ['igniteTerms', ['IgniteBranding', (branding) => {
+/**
+ * @param {import('./branding.service').default} branding
+ */
+export default function factory(branding) {
     function controller() {
         const ctrl = this;
 
@@ -27,4 +30,6 @@ export default ['igniteTerms', ['IgniteBranding', (branding) => {
         controller,
         controllerAs: 'terms'
     };
-}]];
+}
+
+factory.$inject = ['IgniteBranding'];
diff --git a/modules/web-console/frontend/app/modules/configuration/configuration.module.js b/modules/web-console/frontend/app/modules/configuration/configuration.module.js
index a35087124b222..844c87f9da3a7 100644
--- a/modules/web-console/frontend/app/modules/configuration/configuration.module.js
+++ b/modules/web-console/frontend/app/modules/configuration/configuration.module.js
@@ -42,10 +42,10 @@ angular
 .module('ignite-console.configuration', [
 
 ])
-.service('IgniteConfigurationGenerator', () => IgniteConfigurationGenerator)
+.service('IgniteConfigurationGenerator', function() { return IgniteConfigurationGenerator;})
 .service('IgnitePlatformGenerator', IgnitePlatformGenerator)
-.service('SpringTransformer', () => IgniteSpringTransformer)
-.service('JavaTransformer', () => IgniteJavaTransformer)
+.service('SpringTransformer', function() { return IgniteSpringTransformer;})
+.service('JavaTransformer', function() { return IgniteJavaTransformer;})
 .service('IgniteSharpTransformer', SharpTransformer)
 .service('IgniteEventGroups', IgniteEventGroups)
 .service('IgniteClusterDefaults', IgniteClusterDefaults)
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
index c3efea5db8438..2a1a506d8b939 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/ConfigurationGenerator.js
@@ -1698,7 +1698,7 @@ export default class IgniteConfigurationGenerator {
                 .emptyBeanProperty('service')
                 .intProperty('maxPerNodeCount')
                 .intProperty('totalCount')
-                .stringProperty('cache', 'cacheName', (_id) => _id ? _.find(caches, {_id}).name : null)
+                .stringProperty('cache', 'cacheName', (_id) => _id ? _.get(_.find(caches, {_id}), 'name', null) : null)
                 .stringProperty('affinityKey');
 
             srvBeans.push(bean);
diff --git a/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js b/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js
index 99b93ccb01c0d..5f7eed2857e99 100644
--- a/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js
+++ b/modules/web-console/frontend/app/modules/configuration/generator/PlatformGenerator.js
@@ -15,10 +15,17 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
 import {nonEmpty} from 'app/utils/lodashMixins';
 import { EmptyBean, Bean } from './Beans';
 
-export default ['JavaTypes', 'igniteClusterPlatformDefaults', 'igniteCachePlatformDefaults', (JavaTypes, clusterDflts, cacheDflts) => {
+/**
+ * @param {import('app/services/JavaTypes.service').default} JavaTypes
+ * @param {import('./defaults/Cluster.service').default} clusterDflts
+ * @param {import('./defaults/Cache.service').default} cacheDflts
+ */
+export default function service(JavaTypes, clusterDflts, cacheDflts) {
     class PlatformGenerator {
         static igniteConfigurationBean(cluster) {
             return new Bean('Apache.Ignite.Core.IgniteConfiguration', 'cfg', cluster, clusterDflts);
@@ -519,4 +526,6 @@ export default ['JavaTypes', 'igniteClusterPlatformDefaults', 'igniteCachePlatfo
     }
 
     return PlatformGenerator;
-}];
+}
+
+service.$inject = ['JavaTypes', 'igniteClusterPlatformDefaults', 'igniteCachePlatformDefaults'];
diff --git a/modules/web-console/frontend/app/modules/demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js
index 2e1a6275322f0..6c14e4a4de518 100644
--- a/modules/web-console/frontend/app/modules/demo/Demo.module.js
+++ b/modules/web-console/frontend/app/modules/demo/Demo.module.js
@@ -20,47 +20,14 @@ import angular from 'angular';
 import DEMO_INFO from 'app/data/demo-info.json';
 import templateUrl from 'views/templates/demo-info.tpl.pug';
 
-angular
-.module('ignite-console.demo', [
-    'ignite-console.socket'
-])
-.config(['$stateProvider', ($stateProvider) => {
-    $stateProvider
-        .state('demo', {
-            abstract: true,
-            url: '/demo',
-            template: '<ui-view></ui-view>'
-        })
-        .state('demo.resume', {
-            url: '/resume',
-            permission: 'demo',
-            redirectTo: 'default-state',
-            unsaved: true,
-            tfMetaTags: {
-                title: 'Demo resume'
-            }
-        })
-        .state('demo.reset', {
-            url: '/reset',
-            permission: 'demo',
-            redirectTo: (trans) => {
-                const $http = trans.injector().get('$http');
+const DEMO_QUERY_STATE = {state: 'base.sql.notebook', params: {noteId: 'demo'}};
 
-                return $http.post('/api/v1/demo/reset')
-                    .then(() => 'default-state')
-                    .catch((err) => {
-                        trans.injector().get('IgniteMessages').showError(err);
-
-                        return 'default-state';
-                    });
-            },
-            unsaved: true,
-            tfMetaTags: {
-                title: 'Demo reset'
-            }
-        });
-}])
-.provider('Demo', ['$stateProvider', '$httpProvider', 'igniteSocketFactoryProvider', function($state, $http, socketFactory) {
+/**
+ * @param {import('@uirouter/angularjs').StateProvider} $state
+ * @param {ng.IHttpProvider} $http
+ * @param {unknown} socketFactory
+ */
+export function DemoProvider($state, $http, socketFactory) {
     if (/(\/demo.*)/ig.test(location.pathname))
         sessionStorage.setItem('IgniteDemoMode', 'true');
 
@@ -72,13 +39,24 @@ angular
         $http.interceptors.push('demoInterceptor');
     }
 
-    this.$get = ['$rootScope', ($root) => {
+    function service($root) {
         $root.IgniteDemoMode = enabled;
 
         return {enabled};
-    }];
-}])
-.factory('demoInterceptor', ['Demo', (Demo) => {
+    }
+    service.$inject = ['$rootScope'];
+
+    this.$get = service;
+    return this;
+}
+
+DemoProvider.$inject = ['$stateProvider', '$httpProvider', 'igniteSocketFactoryProvider'];
+
+/**
+ * @param {{enabled: boolean}} Demo
+ * @returns {ng.IHttpInterceptor}
+ */
+function demoInterceptor(Demo) {
     const isApiRequest = (url) => /\/api\/v1/ig.test(url);
 
     return {
@@ -89,8 +67,17 @@ angular
             return cfg;
         }
     };
-}])
-.controller('demoController', ['$scope', '$state', '$window', 'IgniteConfirm', ($scope, $state, $window, Confirm) => {
+}
+
+demoInterceptor.$inject = ['Demo'];
+
+/**
+ * @param {ng.IScope} $scope
+ * @param {import('@uirouter/angularjs').StateService} $state
+ * @param {ng.IWindowService} $window
+ * @param {ReturnType<typeof import('app/services/Confirm.service').default>} Confirm
+ */
+function demoController($scope, $state, $window, Confirm) {
     const _openTab = (stateName) => $window.open($state.href(stateName), '_blank');
 
     $scope.startDemo = () => {
@@ -109,17 +96,30 @@ angular
     $scope.closeDemo = () => {
         $window.close();
     };
-}])
-.provider('igniteDemoInfo', [function() {
+}
+
+demoController.$inject = ['$scope', '$state', '$window', 'IgniteConfirm'];
+
+function igniteDemoInfoProvider() {
     const items = DEMO_INFO;
 
     this.update = (data) => items[0] = data;
 
-    this.$get = [() => {
+    this.$get = () => {
         return items;
-    }];
-}])
-.service('DemoInfo', ['$rootScope', '$modal', '$state', '$q', 'igniteDemoInfo', 'AgentManager', ($rootScope, $modal, $state, $q, igniteDemoInfo, agentMgr) => {
+    };
+    return this;
+}
+
+/**
+ * @param {ng.IRootScopeService} $rootScope
+ * @param {mgcrea.ngStrap.modal.IModalScope} $modal
+ * @param {import('@uirouter/angularjs').StateService} $state
+ * @param {ng.IQService} $q
+ * @param {Array<{title: string, message: Array<string>}>} igniteDemoInfo
+ * @param {import('app/modules/agent/AgentManager.service').default} agentMgr
+ */
+function DemoInfo($rootScope, $modal, $state, $q, igniteDemoInfo, agentMgr) {
     const scope = $rootScope.$new();
 
     let closePromise = null;
@@ -138,27 +138,14 @@ angular
         backdrop: 'static'
     });
 
+    scope.downloadAgentHref = '/api/v1/downloads/agent';
+
     scope.close = () => {
         dialog.hide();
 
         closePromise && closePromise.resolve();
     };
 
-    scope.downloadAgent = () => {
-        const lnk = document.createElement('a');
-
-        lnk.setAttribute('href', '/api/v1/agent/downloads/agent');
-        lnk.setAttribute('target', '_self');
-        lnk.setAttribute('download', null);
-        lnk.style.display = 'none';
-
-        document.body.appendChild(lnk);
-
-        lnk.click();
-
-        document.body.removeChild(lnk);
-    };
-
     return {
         show: () => {
             closePromise = $q.defer();
@@ -171,4 +158,59 @@ angular
                 .then(() => scope.hasAgents = true);
         }
     };
-}]);
+}
+
+DemoInfo.$inject = ['$rootScope', '$modal', '$state', '$q', 'igniteDemoInfo', 'AgentManager'];
+
+/**
+ * @param {import('@uirouter/angularjs').StateProvider} $stateProvider
+ */
+function config($stateProvider) {
+    $stateProvider
+        .state('demo', {
+            abstract: true,
+            url: '/demo',
+            template: '<ui-view></ui-view>'
+        })
+        .state('demo.resume', {
+            url: '/resume',
+            permission: 'demo',
+            redirectTo: DEMO_QUERY_STATE,
+            unsaved: true,
+            tfMetaTags: {
+                title: 'Demo resume'
+            }
+        })
+        .state('demo.reset', {
+            url: '/reset',
+            permission: 'demo',
+            redirectTo: (trans) => {
+                const $http = trans.injector().get('$http');
+
+                return $http.post('/api/v1/demo/reset')
+                    .then(() => DEMO_QUERY_STATE)
+                    .catch((err) => {
+                        trans.injector().get('IgniteMessages').showError(err);
+
+                        return DEMO_QUERY_STATE;
+                    });
+            },
+            unsaved: true,
+            tfMetaTags: {
+                title: 'Demo reset'
+            }
+        });
+}
+
+config.$inject = ['$stateProvider'];
+
+angular
+.module('ignite-console.demo', [
+    'ignite-console.socket'
+])
+.config(config)
+.provider('Demo', DemoProvider)
+.factory('demoInterceptor', demoInterceptor)
+.controller('demoController', demoController)
+.provider('igniteDemoInfo', igniteDemoInfoProvider)
+.service('DemoInfo', DemoInfo);
diff --git a/modules/web-console/frontend/app/modules/dialog/dialog-content.directive.js b/modules/web-console/frontend/app/modules/dialog/dialog-content.directive.js
index 98e9903e18ace..fb5d9724dd171 100644
--- a/modules/web-console/frontend/app/modules/dialog/dialog-content.directive.js
+++ b/modules/web-console/frontend/app/modules/dialog/dialog-content.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-export default ['igniteDialogContent', [() => {
+export default () => {
     const link = ($scope, $element, $attrs, igniteDialog) => {
         igniteDialog.content = $element.html();
 
@@ -28,4 +28,4 @@ export default ['igniteDialogContent', [() => {
         link,
         require: '^igniteDialog'
     };
-}]];
+};
diff --git a/modules/web-console/frontend/app/modules/dialog/dialog-title.directive.js b/modules/web-console/frontend/app/modules/dialog/dialog-title.directive.js
index ed4adb83ad016..ebd2d9402782d 100644
--- a/modules/web-console/frontend/app/modules/dialog/dialog-title.directive.js
+++ b/modules/web-console/frontend/app/modules/dialog/dialog-title.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-export default ['igniteDialogTitle', [() => {
+export default () => {
     const link = ($scope, $element, $attrs, igniteDialog) => {
         igniteDialog.title = $element.text();
 
@@ -28,4 +28,4 @@ export default ['igniteDialogTitle', [() => {
         link,
         require: '^igniteDialog'
     };
-}]];
+};
diff --git a/modules/web-console/frontend/app/modules/dialog/dialog.directive.js b/modules/web-console/frontend/app/modules/dialog/dialog.directive.js
index 7aab10f1770ab..96a96e2c16ab0 100644
--- a/modules/web-console/frontend/app/modules/dialog/dialog.directive.js
+++ b/modules/web-console/frontend/app/modules/dialog/dialog.directive.js
@@ -19,7 +19,7 @@ import controller from './dialog.controller';
 
 const template = '<a ng-click="ctrl.show()"><span ng-transclude=""></span></a>';
 
-export default ['igniteDialog', [() => {
+export default () => {
     return {
         restrict: 'E',
         template,
@@ -29,4 +29,4 @@ export default ['igniteDialog', [() => {
         transclude: true,
         require: '^igniteDialog'
     };
-}]];
+};
diff --git a/modules/web-console/frontend/app/modules/dialog/dialog.factory.js b/modules/web-console/frontend/app/modules/dialog/dialog.factory.js
index 599433af817d5..14afcdf6404af 100644
--- a/modules/web-console/frontend/app/modules/dialog/dialog.factory.js
+++ b/modules/web-console/frontend/app/modules/dialog/dialog.factory.js
@@ -17,7 +17,10 @@
 
 import templateUrl from './dialog.tpl.pug';
 
-export default ['IgniteDialog', ['$modal', ($modal) => {
+/**
+ * @param {mgcrea.ngStrap.modal.IModalService} $modal
+ */
+export default function factory($modal) {
     const defaults = {
         templateUrl,
         show: false
@@ -28,4 +31,6 @@ export default ['IgniteDialog', ['$modal', ($modal) => {
 
         return $modal(options);
     };
-}]];
+}
+
+factory.$inject = ['$modal'];
diff --git a/modules/web-console/frontend/app/modules/dialog/dialog.module.js b/modules/web-console/frontend/app/modules/dialog/dialog.module.js
index c9ba9f9db736a..6d9324f392dc3 100644
--- a/modules/web-console/frontend/app/modules/dialog/dialog.module.js
+++ b/modules/web-console/frontend/app/modules/dialog/dialog.module.js
@@ -26,7 +26,7 @@ angular
 .module('ignite-console.dialog', [
 
 ])
-.factory(...IgniteDialog)
-.directive(...igniteDialog)
-.directive(...igniteDialogTitle)
-.directive(...igniteDialogContent);
+.factory('IgniteDialog', IgniteDialog)
+.directive('igniteDialog', igniteDialog)
+.directive('igniteDialogTitle', igniteDialogTitle)
+.directive('igniteDialogContent', igniteDialogContent);
diff --git a/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js b/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js
index cecb49c54a770..9d7f59f748e73 100644
--- a/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js
+++ b/modules/web-console/frontend/app/modules/form/field/bs-select-placeholder.directive.js
@@ -16,7 +16,13 @@
  */
 
 // Override AngularStrap "bsSelect" in order to dynamically change placeholder and class.
-export default ['bsSelect', [() => {
+export default () => {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} $element
+     * @param {ng.IAttributes} attrs
+     * @param {[ng.INgModelController]} [ngModel]
+     */
     const link = (scope, $element, attrs, [ngModel]) => {
         if (!ngModel)
             return;
@@ -48,4 +54,4 @@ export default ['bsSelect', [() => {
         link,
         require: ['?ngModel']
     };
-}]];
+};
diff --git a/modules/web-console/frontend/app/modules/form/field/input/autofocus.directive.js b/modules/web-console/frontend/app/modules/form/field/input/autofocus.directive.js
index 8ffc9a052c307..f54eaee03181a 100644
--- a/modules/web-console/frontend/app/modules/form/field/input/autofocus.directive.js
+++ b/modules/web-console/frontend/app/modules/form/field/input/autofocus.directive.js
@@ -15,7 +15,17 @@
  * limitations under the License.
  */
 
-export default ['igniteFormFieldInputAutofocus', ['$timeout', ($timeout) => {
+import _ from 'lodash';
+
+/**
+ * @param {ng.ITimeoutService} $timeout
+ */
+export default function factory($timeout) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} el
+     * @param {ng.IAttributes} attrs
+     */
     const link = (scope, el, attrs) => {
         if (_.isUndefined(attrs.igniteFormFieldInputAutofocus) || attrs.igniteFormFieldInputAutofocus !== 'true')
             return;
@@ -27,4 +37,6 @@ export default ['igniteFormFieldInputAutofocus', ['$timeout', ($timeout) => {
         restrict: 'A',
         link
     };
-}]];
+}
+
+factory.$inject = ['$timeout'];
diff --git a/modules/web-console/frontend/app/modules/form/field/label.directive.js b/modules/web-console/frontend/app/modules/form/field/label.directive.js
deleted file mode 100644
index 94f7889107270..0000000000000
--- a/modules/web-console/frontend/app/modules/form/field/label.directive.js
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-export default ['igniteFormFieldLabel', [() => {
-    return {
-        restrict: 'E',
-        compile() {
-            return {
-                post($scope, $element, $attrs, [field], $transclude) {
-                    $transclude($scope, function(clone) {
-                        const text = clone.text();
-
-                        if (/(.*):$/.test(text))
-                            field.name = /(.*):$/.exec(text)[1];
-
-                        const $label = $element.parent().parent().find('.group-legend > label, .ignite-field > label');
-
-                        if ($label[0] && $element[0].id) {
-                            const id = $element[0].id;
-
-                            $label[0].id = id.indexOf('+') >= 0 ? $scope.$eval(id) : id;
-                        }
-
-                        $label.append(clone);
-                    });
-                }
-            };
-        },
-        replace: true,
-        transclude: true,
-        require: ['?^igniteFormField']
-    };
-}]];
diff --git a/modules/web-console/frontend/app/modules/form/field/tooltip.directive.js b/modules/web-console/frontend/app/modules/form/field/tooltip.directive.js
deleted file mode 100644
index 9e764bcfd0901..0000000000000
--- a/modules/web-console/frontend/app/modules/form/field/tooltip.directive.js
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const template = '<i class="tipField icon-help"></i>';
-
-export default ['igniteFormFieldTooltip', ['$tooltip', ($tooltip) => {
-    const link = ($scope, $element, $attrs, [field], $transclude) => {
-        const content = Array.prototype.slice
-            .apply($transclude($scope))
-            .reduce((html, el) => html += el.outerHTML || el.textContent || el, '');
-
-        $tooltip($element, { title: content });
-
-        if (field)
-            $element.attr('id', field.for + 'Tooltip');
-
-        // TODO cleanup css styles.
-        if ($element.hasClass('tipLabel'))
-            $element.removeClass('tipField');
-
-        if ($element.parent('label').length)
-            $element.addClass('tipLabel').removeClass('tipField');
-    };
-
-    return {
-        priority: 1,
-        restrict: 'E',
-        scope: {},
-        template,
-        link,
-        replace: true,
-        transclude: true,
-        require: ['?^igniteFormField']
-    };
-}]];
diff --git a/modules/web-console/frontend/app/modules/form/form.module.js b/modules/web-console/frontend/app/modules/form/form.module.js
index 59fedff244d7b..29afa595bc544 100644
--- a/modules/web-console/frontend/app/modules/form/form.module.js
+++ b/modules/web-console/frontend/app/modules/form/form.module.js
@@ -17,23 +17,9 @@
 
 import angular from 'angular';
 
-// Fields styles.
-import './field/field.scss';
-import './field/feedback.scss';
-import './field/input/text.scss';
-
-// Panel.
-import igniteFormPanelChevron from './panel/chevron.directive';
-
 // Field.
-import igniteFormFieldLabel from './field/label.directive';
-import igniteFormFieldTooltip from './field/tooltip.directive';
 import placeholder from './field/bs-select-placeholder.directive';
 
-// Group.
-import igniteFormGroupAdd from './group/add.directive';
-import igniteFormGroupTooltip from './group/tooltip.directive';
-
 // Validators.
 import ipaddress from './validator/ipaddress.directive';
 import javaKeywords from './validator/java-keywords.directive';
@@ -48,41 +34,27 @@ import uuid from './validator/uuid.directive';
 
 // Helpers.
 import igniteFormFieldInputAutofocus from './field/input/autofocus.directive';
-import igniteFormControlFeedback from './field/form-control-feedback.directive';
-import igniteFormFieldUp from './field/up.directive';
-import igniteFormFieldDown from './field/down.directive';
-
 import IgniteFormGUID from './services/FormGUID.service.js';
 
 angular
 .module('ignite-console.Form', [
 
 ])
-// Panel.
-.directive(...igniteFormPanelChevron)
 // Field.
-.directive(...igniteFormFieldLabel)
-.directive(...igniteFormFieldTooltip)
-.directive(...placeholder)
-// Group.
-.directive(...igniteFormGroupAdd)
-.directive(...igniteFormGroupTooltip)
+.directive('bsSelect', placeholder)
 // Validators.
-.directive(...ipaddress)
-.directive(...javaKeywords)
-.directive(...javaPackageSpecified)
-.directive(...javaBuiltInClass)
-.directive(...javaIdentifier)
-.directive(...javaPackageName)
-.directive(...propertyValueSpecified)
-.directive(...propertyUnique)
-.directive(...unique)
-.directive(...uuid)
+.directive('ipaddress', ipaddress)
+.directive('javaKeywords', javaKeywords)
+.directive('javaPackageSpecified', javaPackageSpecified)
+.directive('javaBuiltInClass', javaBuiltInClass)
+.directive('javaIdentifier', javaIdentifier)
+.directive('javaPackageName', javaPackageName)
+.directive('ignitePropertyValueSpecified', propertyValueSpecified)
+.directive('ignitePropertyUnique', propertyUnique)
+.directive('igniteUnique', unique)
+.directive('uuid', uuid)
 // Helpers.
-.directive(...igniteFormFieldInputAutofocus)
-.directive(...igniteFormControlFeedback)
-.directive(...igniteFormFieldUp)
-.directive(...igniteFormFieldDown)
+.directive('igniteFormFieldInputAutofocus', igniteFormFieldInputAutofocus)
 
 // Generator of globally unique identifier.
 .service('IgniteFormGUID', IgniteFormGUID);
diff --git a/modules/web-console/frontend/app/modules/form/group/add.directive.js b/modules/web-console/frontend/app/modules/form/group/add.directive.js
deleted file mode 100644
index 71070cc55644d..0000000000000
--- a/modules/web-console/frontend/app/modules/form/group/add.directive.js
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const template = '<i class="group-legend-btn fa fa-plus"></i>';
-
-export default ['igniteFormGroupAdd', ['$tooltip', ($tooltip) => {
-    return {
-        restrict: 'E',
-        scope: {},
-        template,
-        link($scope, $el, $attr, $ctrl, $transclude) {
-            $transclude((clone) => {
-                const title = Array.from(clone)
-                    .reduce((html, el) => html += el.outerHTML || el.textContent || el, '');
-                const legend = $el.closest('.group').find('.group-legend');
-
-                $tooltip($el, {title});
-                legend.append($el);
-            });
-        },
-        replace: true,
-        transclude: true
-    };
-}]];
diff --git a/modules/web-console/frontend/app/modules/form/group/tooltip.directive.js b/modules/web-console/frontend/app/modules/form/group/tooltip.directive.js
deleted file mode 100644
index 4190deeac9b3e..0000000000000
--- a/modules/web-console/frontend/app/modules/form/group/tooltip.directive.js
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const template = '<i class="group-legend-btn icon-help"></i>';
-
-export default ['igniteFormGroupTooltip', ['$tooltip', ($tooltip) => {
-    return {
-        restrict: 'E',
-        scope: {},
-        template,
-        link($scope, $el, $attr, $ctrl, $transclude) {
-            $transclude((clone) => {
-                const title = Array.from(clone)
-                    .reduce((html, el) => html += el.outerHTML || el.textContent || el, '');
-                const legend = $el.closest('.group').find('.group-legend');
-
-                $tooltip($el, {title});
-                legend.append($el);
-            });
-        },
-        replace: true,
-        transclude: true
-    };
-}]];
diff --git a/modules/web-console/frontend/app/modules/form/panel/chevron.directive.js b/modules/web-console/frontend/app/modules/form/panel/chevron.directive.js
deleted file mode 100644
index f5ad95705c505..0000000000000
--- a/modules/web-console/frontend/app/modules/form/panel/chevron.directive.js
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const template = `<img ng-src="{{ isOpen ? '/images/collapse.svg' : '/images/expand.svg' }}" style='width:13px;height:13px;' />`;
-
-export default ['igniteFormPanelChevron', ['$timeout', ($timeout) => {
-    const controller = [() => {}];
-
-    const link = ($scope, $element, $attrs, [bsCollapseCtrl]) => {
-        const $target = $element.parent().parent().find('[bs-collapse-target]');
-
-        const listener = function() {
-            const index = bsCollapseCtrl.$targets.reduce((acc, el, i) => {
-                if (el[0] === $target[0])
-                    acc.push(i);
-
-                return acc;
-            }, [])[0];
-
-            $scope.isOpen = false;
-
-            const active = bsCollapseCtrl.$activeIndexes();
-
-            if ((active instanceof Array) && active.indexOf(index) !== -1 || active === index)
-                $scope.isOpen = true;
-        };
-
-        bsCollapseCtrl.$viewChangeListeners.push(listener);
-        $timeout(listener);
-    };
-
-    return {
-        restrict: 'E',
-        scope: {},
-        link,
-        template,
-        controller,
-        // replace: true,
-        // transclude: true,
-        require: ['^bsCollapse']
-    };
-}]];
diff --git a/modules/web-console/frontend/app/modules/form/services/FormGUID.service.js b/modules/web-console/frontend/app/modules/form/services/FormGUID.service.js
index b886851766ffb..6b8df3c8d29ef 100644
--- a/modules/web-console/frontend/app/modules/form/services/FormGUID.service.js
+++ b/modules/web-console/frontend/app/modules/form/services/FormGUID.service.js
@@ -15,8 +15,8 @@
  * limitations under the License.
  */
 
-export default [() => {
+export default function() {
     let guid = 0;
 
     return () => `form-field-${guid++}`;
-}];
+}
diff --git a/modules/web-console/frontend/app/modules/form/validator/ipaddress.directive.js b/modules/web-console/frontend/app/modules/form/validator/ipaddress.directive.js
index 77e63f6034f6f..531e68271444b 100644
--- a/modules/web-console/frontend/app/modules/form/validator/ipaddress.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/ipaddress.directive.js
@@ -15,7 +15,12 @@
  * limitations under the License.
  */
 
-export default ['ipaddress', ['IgniteInetAddress', (InetAddress) => {
+import _ from 'lodash';
+
+/**
+ * @param {ReturnType<typeof import('app/services/InetAddress.service').default>} InetAddress
+ */
+export default function factory(InetAddress) {
     const onlyDigits = (str) => (/^\d+$/.test(str));
 
     const strictParseInt = (str) => onlyDigits(str) ? parseInt(str, 10) : Number.NaN;
@@ -27,6 +32,12 @@ export default ['ipaddress', ['IgniteInetAddress', (InetAddress) => {
         return {ipOrHost, ports};
     };
 
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} el
+     * @param {ng.IAttributes} attrs
+     * @param {[ng.INgModelController]} [ngModel]
+     */
     const link = (scope, el, attrs, [ngModel]) => {
         const isEmpty = (modelValue) => {
             return ngModel.$isEmpty(modelValue) || _.isUndefined(attrs.ipaddress) || attrs.ipaddress !== 'true';
@@ -83,4 +94,6 @@ export default ['ipaddress', ['IgniteInetAddress', (InetAddress) => {
         link,
         require: ['ngModel']
     };
-}]];
+}
+
+factory.$inject = ['IgniteInetAddress'];
diff --git a/modules/web-console/frontend/app/modules/form/validator/java-built-in-class.directive.js b/modules/web-console/frontend/app/modules/form/validator/java-built-in-class.directive.js
index f9aadc4aba0e1..736179c1055ed 100644
--- a/modules/web-console/frontend/app/modules/form/validator/java-built-in-class.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/java-built-in-class.directive.js
@@ -15,7 +15,18 @@
  * limitations under the License.
  */
 
-export default ['javaBuiltInClass', ['JavaTypes', (JavaTypes) => {
+import _ from 'lodash';
+
+/**
+ * @param {import('app/services/JavaTypes.service').default} JavaTypes
+ */
+export default function factory(JavaTypes) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} el
+     * @param {ng.IAttributes} attrs
+     * @param {[ng.INgModelController]} [ngModel]
+     */
     const link = (scope, el, attrs, [ngModel]) => {
         if (_.isUndefined(attrs.javaBuiltInClass) || !attrs.javaBuiltInClass)
             return;
@@ -32,4 +43,6 @@ export default ['javaBuiltInClass', ['JavaTypes', (JavaTypes) => {
         link,
         require: ['ngModel']
     };
-}]];
+}
+
+factory.$inject = ['JavaTypes'];
diff --git a/modules/web-console/frontend/app/modules/form/validator/java-identifier.directive.js b/modules/web-console/frontend/app/modules/form/validator/java-identifier.directive.js
index a61d309239dce..1b5a700776b74 100644
--- a/modules/web-console/frontend/app/modules/form/validator/java-identifier.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/java-identifier.directive.js
@@ -15,7 +15,18 @@
  * limitations under the License.
  */
 
-export default ['javaIdentifier', ['JavaTypes', (JavaTypes) => {
+import _ from 'lodash';
+
+/**
+ * @param {import('app/services/JavaTypes.service').default} JavaTypes
+ */
+export default function factory(JavaTypes) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} el
+     * @param {ng.IAttributes} attrs
+     * @param {[ng.INgModelController]} [ngModel]
+     */
     const link = (scope, el, attrs, [ngModel]) => {
         if (_.isNil(attrs.javaIdentifier) || attrs.javaIdentifier !== 'true')
             return;
@@ -35,4 +46,6 @@ export default ['javaIdentifier', ['JavaTypes', (JavaTypes) => {
         link,
         require: ['ngModel']
     };
-}]];
+}
+
+factory.$inject = ['JavaTypes'];
diff --git a/modules/web-console/frontend/app/modules/form/validator/java-keywords.directive.js b/modules/web-console/frontend/app/modules/form/validator/java-keywords.directive.js
index 9d5c2aa65f78d..9581ee9122afe 100644
--- a/modules/web-console/frontend/app/modules/form/validator/java-keywords.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/java-keywords.directive.js
@@ -15,7 +15,18 @@
  * limitations under the License.
  */
 
-export default ['javaKeywords', ['JavaTypes', (JavaTypes) => {
+import _ from 'lodash';
+
+/**
+ * @param {import('app/services/JavaTypes.service').default} JavaTypes
+ */
+export default function factory(JavaTypes) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} el
+     * @param {ng.IAttributes} attrs
+     * @param {[ng.INgModelController]} [ngModel]
+     */
     const link = (scope, el, attrs, [ngModel]) => {
         if (_.isNil(attrs.javaKeywords) || attrs.javaKeywords === 'false')
             return;
@@ -36,4 +47,6 @@ export default ['javaKeywords', ['JavaTypes', (JavaTypes) => {
         link,
         require: ['ngModel']
     };
-}]];
+}
+
+factory.$inject = ['JavaTypes'];
diff --git a/modules/web-console/frontend/app/modules/form/validator/java-package-name.directive.js b/modules/web-console/frontend/app/modules/form/validator/java-package-name.directive.js
index 49d4b551cdd14..905911208976b 100644
--- a/modules/web-console/frontend/app/modules/form/validator/java-package-name.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/java-package-name.directive.js
@@ -15,7 +15,18 @@
  * limitations under the License.
  */
 
-export default ['javaPackageName', ['JavaTypes', (JavaTypes) => {
+import _ from 'lodash';
+
+/**
+ * @param {import('app/services/JavaTypes.service').default} JavaTypes
+ */
+export default function factory(JavaTypes) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} el
+     * @param {ng.IAttributes} attrs
+     * @param {[ng.INgModelController]} [ngModel]
+     */
     const link = (scope, el, attrs, [ngModel]) => {
         if (_.isNil(attrs.javaPackageName) || attrs.javaPackageName === 'false')
             return;
@@ -28,4 +39,6 @@ export default ['javaPackageName', ['JavaTypes', (JavaTypes) => {
         link,
         require: ['ngModel']
     };
-}]];
+}
+
+factory.$inject = ['JavaTypes'];
diff --git a/modules/web-console/frontend/app/modules/form/validator/java-package-specified.directive.js b/modules/web-console/frontend/app/modules/form/validator/java-package-specified.directive.js
index d3274ae3bbf59..5ff25d5e147f0 100644
--- a/modules/web-console/frontend/app/modules/form/validator/java-package-specified.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/java-package-specified.directive.js
@@ -15,7 +15,18 @@
  * limitations under the License.
  */
 
-export default ['javaPackageSpecified', ['JavaTypes', (JavaTypes) => {
+import _ from 'lodash';
+
+/**
+ * @param {import('app/services/JavaTypes.service').default} JavaTypes
+ */
+export default function factory(JavaTypes) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} el
+     * @param {ng.IAttributes} attrs
+     * @param {[ng.INgModelController]} [ngModel]
+     */
     const link = (scope, el, attrs, [ngModel]) => {
         if (_.isNil(attrs.javaPackageSpecified) || attrs.javaPackageSpecified === 'false')
             return;
@@ -36,4 +47,6 @@ export default ['javaPackageSpecified', ['JavaTypes', (JavaTypes) => {
         link,
         require: ['ngModel']
     };
-}]];
+}
+
+factory.$inject = ['JavaTypes'];
diff --git a/modules/web-console/frontend/app/modules/form/validator/property-unique.directive.js b/modules/web-console/frontend/app/modules/form/validator/property-unique.directive.js
index 8cfae89fb1773..fbcf734ae347c 100644
--- a/modules/web-console/frontend/app/modules/form/validator/property-unique.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/property-unique.directive.js
@@ -15,7 +15,18 @@
  * limitations under the License.
  */
 
-export default ['ignitePropertyUnique', ['$parse', ($parse) => {
+import _ from 'lodash';
+
+/**
+ * @param {ng.IParseService} $parse
+ */
+export default function factory($parse) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} el
+     * @param {ng.IAttributes} attrs
+     * @param {[ng.INgModelController]} [ngModel]
+     */
     const link = (scope, el, attrs, [ngModel]) => {
         if (_.isUndefined(attrs.ignitePropertyUnique) || !attrs.ignitePropertyUnique)
             return;
@@ -44,4 +55,6 @@ export default ['ignitePropertyUnique', ['$parse', ($parse) => {
         link,
         require: ['ngModel']
     };
-}]];
+}
+
+factory.$inject = ['$parse'];
diff --git a/modules/web-console/frontend/app/modules/form/validator/property-value-specified.directive.js b/modules/web-console/frontend/app/modules/form/validator/property-value-specified.directive.js
index d113a4f331d89..5136d1a4e60d5 100644
--- a/modules/web-console/frontend/app/modules/form/validator/property-value-specified.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/property-value-specified.directive.js
@@ -15,7 +15,15 @@
  * limitations under the License.
  */
 
-export default ['ignitePropertyValueSpecified', [() => {
+import _ from 'lodash';
+
+export default () => {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} el
+     * @param {ng.IAttributes} attrs
+     * @param {[ng.INgModelController]} [ngModel]
+     */
     const link = (scope, el, attrs, [ngModel]) => {
         if (_.isUndefined(attrs.ignitePropertyValueSpecified) || !attrs.ignitePropertyValueSpecified)
             return;
@@ -28,4 +36,4 @@ export default ['ignitePropertyValueSpecified', [() => {
         link,
         require: ['ngModel']
     };
-}]];
+};
diff --git a/modules/web-console/frontend/app/modules/form/validator/unique.directive.js b/modules/web-console/frontend/app/modules/form/validator/unique.directive.js
index 318b8b6c828a0..ac6787f5c8ba2 100644
--- a/modules/web-console/frontend/app/modules/form/validator/unique.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/unique.directive.js
@@ -26,9 +26,16 @@ class Controller {
     listEditableTransclude;
     /** @type {Array} */
     items;
+    /** @type {string?} */
+    key;
+    /** @type {Array<string>} */
+    skip;
 
     static $inject = ['$scope'];
 
+    /**
+     * @param {ng.IScope} $scope
+     */
     constructor($scope) {
         this.$scope = $scope;
     }
@@ -69,7 +76,7 @@ class Controller {
     }
 }
 
-export default ['igniteUnique', () => {
+export default () => {
     return {
         controller: Controller,
         require: {
@@ -82,4 +89,4 @@ export default ['igniteUnique', () => {
             skip: '<?igniteUniqueSkip'
         }
     };
-}];
+};
diff --git a/modules/web-console/frontend/app/modules/form/validator/uuid.directive.js b/modules/web-console/frontend/app/modules/form/validator/uuid.directive.js
index 94c6e3f7aa38d..cb2098168c063 100644
--- a/modules/web-console/frontend/app/modules/form/validator/uuid.directive.js
+++ b/modules/web-console/frontend/app/modules/form/validator/uuid.directive.js
@@ -15,7 +15,18 @@
  * limitations under the License.
  */
 
-export default ['uuid', ['JavaTypes', (JavaTypes) => {
+import _ from 'lodash';
+
+/**
+ * @param {import('app/services/JavaTypes.service').default} JavaTypes
+ */
+export default function factory(JavaTypes) {
+    /**
+     * @param {ng.IScope} scope
+     * @param {JQLite} el
+     * @param {ng.IAttributes} attrs
+     * @param {[ng.INgModelController]} [ngModel]
+     */
     const link = (scope, el, attrs, [ngModel]) => {
         if (_.isNil(attrs.uuid) || attrs.uuid !== 'true')
             return;
@@ -28,4 +39,6 @@ export default ['uuid', ['JavaTypes', (JavaTypes) => {
         link,
         require: ['ngModel']
     };
-}]];
+}
+
+factory.$inject = ['JavaTypes'];
diff --git a/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js b/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js
index 2608bd28c1da9..14dd26a6dc535 100644
--- a/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js
+++ b/modules/web-console/frontend/app/modules/getting-started/GettingStarted.provider.js
@@ -17,97 +17,130 @@
 
 import angular from 'angular';
 
-// Getting started pages.
+/**
+ * @typedef GettingStartedItem
+ * @prop {string} title
+ * @prop {Array<string>} message
+ */
+
+/**
+ * @typedef {Array<GettingStartedItem>} GettingStartedItems
+ */
+
 import PAGES from 'app/data/getting-started.json';
 import templateUrl from 'views/templates/getting-started.tpl.pug';
 
-angular
-    .module('ignite-console.getting-started', [])
-    .provider('igniteGettingStarted', function() {
-        const items = PAGES;
+export function provider() {
+    /**
+     * Getting started pages.
+     * @type {GettingStartedItems}
+     */
+    const items = PAGES;
 
-        this.push = (before, data) => {
-            const idx = _.findIndex(items, {title: before});
+    this.push = (before, data) => {
+        const idx = _.findIndex(items, {title: before});
 
-            if (idx < 0)
-                items.push(data);
-            else
-                items.splice(idx, 0, data);
-        };
+        if (idx < 0)
+            items.push(data);
+        else
+            items.splice(idx, 0, data);
+    };
 
-        this.update = (before, data) => {
-            const idx = _.findIndex(items, {title: before});
+    this.update = (before, data) => {
+        const idx = _.findIndex(items, {title: before});
 
-            if (idx >= 0)
-                items[idx] = data;
-        };
+        if (idx >= 0)
+            items[idx] = data;
+    };
 
-        this.$get = [function() {
-            return items;
-        }];
-    })
-    .service('gettingStarted', ['$rootScope', '$modal', 'igniteGettingStarted', function($root, $modal, igniteGettingStarted) {
-        const _model = igniteGettingStarted;
+    this.$get = function() {
+        return items;
+    };
 
-        let _page = 0;
+    return this;
+}
 
-        const scope = $root.$new();
+/**
+ * @param {ng.IRootScopeService} $root
+ * @param {mgcrea.ngStrap.modal.IModalService} $modal
+ * @param {GettingStartedItems} igniteGettingStarted
+ */
+export function service($root, $modal, igniteGettingStarted) {
+    const _model = igniteGettingStarted;
 
-        scope.ui = {
-            showGettingStarted: false
-        };
+    let _page = 0;
 
-        function _fillPage() {
-            scope.title = _model[_page].title;
-            scope.message = _model[_page].message.join(' ');
-        }
+    const scope = $root.$new();
+
+    scope.ui = {
+        dontShowGettingStarted: false
+    };
+
+    function _fillPage() {
+        if (_page === 0)
+            scope.title = `${_model[_page].title}`;
+        else
+            scope.title = `${_page}. ${_model[_page].title}`;
 
-        scope.isFirst = () => _page === 0;
+        scope.message = _model[_page].message.join(' ');
+    }
 
-        scope.isLast = () => _page === _model.length - 1;
+    scope.isFirst = () => _page === 0;
 
-        scope.next = () => {
-            _page += 1;
+    scope.isLast = () => _page === _model.length - 1;
 
-            _fillPage();
-        };
+    scope.next = () => {
+        _page += 1;
 
-        scope.prev = () => {
-            _page -= 1;
+        _fillPage();
+    };
 
-            _fillPage();
-        };
+    scope.prev = () => {
+        _page -= 1;
 
-        const dialog = $modal({ templateUrl, scope, show: false, backdrop: 'static'});
+        _fillPage();
+    };
 
-        scope.close = () => {
+    const dialog = $modal({ templateUrl, scope, show: false, backdrop: 'static'});
+
+    scope.close = () => {
+        try {
+            localStorage.showGettingStarted = !scope.ui.dontShowGettingStarted;
+        }
+        catch (ignore) {
+            // No-op.
+        }
+
+        dialog.hide();
+    };
+
+    return {
+        /**
+         * @param {boolean} force
+         */
+        tryShow: (force) => {
             try {
-                localStorage.showGettingStarted = scope.ui.showGettingStarted;
+                scope.ui.dontShowGettingStarted = !(_.isNil(localStorage.showGettingStarted)
+                        || localStorage.showGettingStarted === 'true');
             }
             catch (ignore) {
                 // No-op.
             }
 
-            dialog.hide();
-        };
+            if (force || !scope.ui.dontShowGettingStarted) {
+                _page = 0;
 
-        return {
-            tryShow: (force) => {
-                try {
-                    scope.ui.showGettingStarted = _.isNil(localStorage.showGettingStarted)
-                        || localStorage.showGettingStarted === 'true';
-                }
-                catch (ignore) {
-                    // No-op.
-                }
+                _fillPage();
 
-                if (force || scope.ui.showGettingStarted) {
-                    _page = 0;
+                dialog.$promise.then(dialog.show);
+            }
+        }
+    };
+}
 
-                    _fillPage();
+service.$inject = ['$rootScope', '$modal', 'igniteGettingStarted'];
 
-                    dialog.$promise.then(dialog.show);
-                }
-            }
-        };
-    }]);
+export default angular
+    .module('ignite-console.getting-started', [])
+    .provider('igniteGettingStarted', provider)
+    .service('gettingStarted', service);
diff --git a/modules/web-console/frontend/app/modules/loading/loading.directive.js b/modules/web-console/frontend/app/modules/loading/loading.directive.js
index 8a4ca9f5fab8c..983e59784fb8a 100644
--- a/modules/web-console/frontend/app/modules/loading/loading.directive.js
+++ b/modules/web-console/frontend/app/modules/loading/loading.directive.js
@@ -18,7 +18,11 @@
 import template from './loading.pug';
 import './loading.scss';
 
-export default ['igniteLoading', ['IgniteLoading', '$compile', (Loading, $compile) => {
+/**
+ * @param {ReturnType<typeof import('./loading.service').default>} Loading
+ * @param {ng.ICompileService} $compile
+ */
+export default function factory(Loading, $compile) {
     const link = (scope, element) => {
         const compiledTemplate = $compile(template);
 
@@ -48,4 +52,6 @@ export default ['igniteLoading', ['IgniteLoading', '$compile', (Loading, $compil
         restrict: 'A',
         link
     };
-}]];
+}
+
+factory.$inject = ['IgniteLoading', '$compile'];
diff --git a/modules/web-console/frontend/app/modules/loading/loading.module.js b/modules/web-console/frontend/app/modules/loading/loading.module.js
index 889cd6bcfa126..ac9e127644be9 100644
--- a/modules/web-console/frontend/app/modules/loading/loading.module.js
+++ b/modules/web-console/frontend/app/modules/loading/loading.module.js
@@ -22,5 +22,5 @@ import IgniteLoadingService from './loading.service';
 
 angular
     .module('ignite-console.loading', [])
-    .directive(...IgniteLoadingDirective)
-    .service(...IgniteLoadingService);
+    .directive('igniteLoading', IgniteLoadingDirective)
+    .service('IgniteLoading', IgniteLoadingService);
diff --git a/modules/web-console/frontend/app/modules/loading/loading.service.js b/modules/web-console/frontend/app/modules/loading/loading.service.js
index bdc80b8f5360a..0aa02a389ceba 100644
--- a/modules/web-console/frontend/app/modules/loading/loading.service.js
+++ b/modules/web-console/frontend/app/modules/loading/loading.service.js
@@ -15,9 +15,12 @@
  * limitations under the License.
  */
 
-export default ['IgniteLoading', [() => {
+export default function() {
     const _overlays = {};
 
+    /**
+     * @param {string} key
+     */
     const start = (key) => {
         setTimeout(() => {
             const loadingOverlay = _overlays[key];
@@ -26,6 +29,9 @@ export default ['IgniteLoading', [() => {
         });
     };
 
+    /**
+     * @param {string} key
+     */
     const finish = (key) => {
         setTimeout(() => {
             const loadingOverlay = _overlays[key];
@@ -45,4 +51,4 @@ export default ['IgniteLoading', [() => {
         start,
         finish
     };
-}]];
+}
diff --git a/modules/web-console/frontend/app/modules/navbar/Navbar.provider.js b/modules/web-console/frontend/app/modules/navbar/Navbar.provider.js
index f1ae1b2299cbc..9b905d7285aa8 100644
--- a/modules/web-console/frontend/app/modules/navbar/Navbar.provider.js
+++ b/modules/web-console/frontend/app/modules/navbar/Navbar.provider.js
@@ -15,14 +15,16 @@
  * limitations under the License.
  */
 
-export default ['IgniteNavbar', [function() {
+export default function() {
     const items = [];
 
     this.push = function(data) {
         items.push(data);
     };
 
-    this.$get = [function() {
+    this.$get = function() {
         return items;
-    }];
-}]];
+    };
+
+    return this;
+}
diff --git a/modules/web-console/frontend/app/modules/navbar/Userbar.provider.js b/modules/web-console/frontend/app/modules/navbar/Userbar.provider.js
index 9513641da841c..9b905d7285aa8 100644
--- a/modules/web-console/frontend/app/modules/navbar/Userbar.provider.js
+++ b/modules/web-console/frontend/app/modules/navbar/Userbar.provider.js
@@ -15,14 +15,16 @@
  * limitations under the License.
  */
 
-export default ['IgniteUserbar', [function() {
+export default function() {
     const items = [];
 
     this.push = function(data) {
         items.push(data);
     };
 
-    this.$get = [function() {
+    this.$get = function() {
         return items;
-    }];
-}]];
+    };
+
+    return this;
+}
diff --git a/modules/web-console/frontend/app/modules/navbar/navbar.directive.js b/modules/web-console/frontend/app/modules/navbar/navbar.directive.js
index 020cfe430c47d..e400cf7d2f54a 100644
--- a/modules/web-console/frontend/app/modules/navbar/navbar.directive.js
+++ b/modules/web-console/frontend/app/modules/navbar/navbar.directive.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-export default ['igniteNavbar', ['IgniteNavbar', (IgniteNavbar) => {
+export default function factory(IgniteNavbar) {
     function controller() {
         const ctrl = this;
 
@@ -27,4 +27,6 @@ export default ['igniteNavbar', ['IgniteNavbar', (IgniteNavbar) => {
         controller,
         controllerAs: 'navbar'
     };
-}]];
+}
+
+factory.$inject = ['IgniteNavbar'];
diff --git a/modules/web-console/frontend/app/modules/navbar/navbar.module.js b/modules/web-console/frontend/app/modules/navbar/navbar.module.js
index b845cbcf446c0..63588c7ac4509 100644
--- a/modules/web-console/frontend/app/modules/navbar/navbar.module.js
+++ b/modules/web-console/frontend/app/modules/navbar/navbar.module.js
@@ -27,7 +27,7 @@ angular
 .module('ignite-console.navbar', [
 
 ])
-.provider(...IgniteNavbar)
-.provider(...IgniteUserbar)
-.directive(...igniteNavbar)
-.directive(...igniteUserbar);
+.provider('IgniteNavbar', IgniteNavbar)
+.provider('IgniteUserbar', IgniteUserbar)
+.directive('igniteNavbar', igniteNavbar)
+.directive('igniteUserbar', igniteUserbar);
diff --git a/modules/web-console/frontend/app/modules/navbar/userbar.directive.js b/modules/web-console/frontend/app/modules/navbar/userbar.directive.js
index 51f46d4c5187b..b636a34d82ff5 100644
--- a/modules/web-console/frontend/app/modules/navbar/userbar.directive.js
+++ b/modules/web-console/frontend/app/modules/navbar/userbar.directive.js
@@ -15,36 +15,45 @@
  * limitations under the License.
  */
 
-export default ['igniteUserbar', [function() {
-    return {
-        restrict: 'A',
-        controller: ['$rootScope', 'IgniteUserbar', 'AclService', function($root, IgniteUserbar, AclService) {
-            const ctrl = this;
+/**
+ * @param {ng.IRootScopeService} $root
+ * @param {unknown} IgniteUserbar
+ * @param {unknown} AclService
+ */
+function controller($root, IgniteUserbar, AclService) {
+    const ctrl = this;
+
+    this.$onInit = () => {
+        ctrl.items = [
+            {text: 'Profile', sref: 'base.settings.profile'},
+            {text: 'Getting started', click: 'gettingStarted.tryShow(true)'}
+        ];
 
-            this.$onInit = () => {
-                ctrl.items = [
-                    {text: 'Profile', sref: 'base.settings.profile'},
-                    {text: 'Getting started', click: 'gettingStarted.tryShow(true)'}
-                ];
+        const _rebuildSettings = () => {
+            ctrl.items.splice(2);
 
-                const _rebuildSettings = () => {
-                    ctrl.items.splice(2);
+            if (AclService.can('admin_page'))
+                ctrl.items.push({text: 'Admin panel', sref: 'base.settings.admin'});
 
-                    if (AclService.can('admin_page'))
-                        ctrl.items.push({text: 'Admin panel', sref: 'base.settings.admin'});
+            ctrl.items.push(...IgniteUserbar);
 
-                    ctrl.items.push(...IgniteUserbar);
+            if (AclService.can('logout'))
+                ctrl.items.push({text: 'Log out', sref: 'logout'});
+        };
 
-                    if (AclService.can('logout'))
-                        ctrl.items.push({text: 'Log out', sref: 'logout'});
-                };
+        if ($root.user)
+            _rebuildSettings(null, $root.user);
 
-                if ($root.user)
-                    _rebuildSettings(null, $root.user);
+        $root.$on('user', _rebuildSettings);
+    };
+}
+
+controller.$inject = ['$rootScope', 'IgniteUserbar', 'AclService'];
 
-                $root.$on('user', _rebuildSettings);
-            };
-        }],
+export default function() {
+    return {
+        restrict: 'A',
+        controller,
         controllerAs: 'userbar'
     };
-}]];
+}
diff --git a/modules/web-console/frontend/app/modules/nodes/Nodes.service.js b/modules/web-console/frontend/app/modules/nodes/Nodes.service.js
index e2317536d9fe5..9130fc6472c54 100644
--- a/modules/web-console/frontend/app/modules/nodes/Nodes.service.js
+++ b/modules/web-console/frontend/app/modules/nodes/Nodes.service.js
@@ -27,8 +27,8 @@ class Nodes {
     static $inject = ['$q', '$modal'];
 
     /**
-     * @param $q
-     * @param $modal
+     * @param {ng.IQService} $q
+     * @param {mgcrea.ngStrap.modal.IModalService} $modal
      */
     constructor($q, $modal) {
         this.$q = $q;
diff --git a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js
index 2296aedccd374..a0cf80abad5cd 100644
--- a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js
+++ b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.controller.js
@@ -25,7 +25,7 @@ const COLUMNS_DEFS = [
     {displayName: 'OS information', field: 'os', headerTooltip: 'OS information for node\'s host', minWidth: 125}
 ];
 
-export default ['$scope', '$animate', 'uiGridConstants', 'nodes', 'options', function($scope, $animate, uiGridConstants, nodes, options) {
+export default function controller($scope, $animate, uiGridConstants, nodes, options) {
     const $ctrl = this;
 
     const updateSelected = () => {
@@ -65,4 +65,6 @@ export default ['$scope', '$animate', 'uiGridConstants', 'nodes', 'options', fun
         },
         ...options.grid
     };
-}];
+}
+
+controller.$inject = ['$scope', '$animate', 'uiGridConstants', 'nodes', 'options'];
diff --git a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
index 22f3b1d057c9e..7bd8c7a669307 100644
--- a/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
+++ b/modules/web-console/frontend/app/modules/nodes/nodes-dialog.tpl.pug
@@ -14,7 +14,7 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-.modal.modal--ignite.theme--ignite.ignite-nodes-dialog(tabindex='-1' role='dialog')
+.modal.modal--ignite.ignite-nodes-dialog(tabindex='-1' role='dialog')
     .modal-dialog.modal-dialog--adjust-height
         form.modal-content
             .modal-header
@@ -30,12 +30,13 @@
                             span Cache Nodes
                             span.badge.badge--blue {{ $ctrl.data.length }}
 
-                .panel--ignite
+                .panel--ignite.panel--ignite__without-border
                     .grid.ui-grid--ignite(ui-grid='$ctrl.gridOptions' ui-grid-resize-columns ui-grid-selection ui-grid-pinning ui-grid-hovering)
 
             .modal-footer
-                button.pull-left.btn-ignite(disabled)
+                div
                     grid-item-selected(class='pull-left' grid-api='$ctrl.gridApi')
 
-                button.btn-ignite.btn-ignite--link-success(id='confirm-btn-close' ng-click='$cancel()') Cancel
-                button.btn-ignite.btn-ignite--success(id='confirm-btn-confirm' ng-click='$ok($ctrl.selected)' ng-disabled='$ctrl.selected.length === 0') Select node
+                div
+                    button.btn-ignite.btn-ignite--link-success(id='confirm-btn-close' ng-click='$cancel()') Cancel
+                    button.btn-ignite.btn-ignite--success(id='confirm-btn-confirm' ng-click='$ok($ctrl.selected)' ng-disabled='$ctrl.selected.length === 0') Select node
diff --git a/modules/web-console/frontend/app/modules/socket.module.js b/modules/web-console/frontend/app/modules/socket.module.js
index 17856c48e07c5..5a38bdb765134 100644
--- a/modules/web-console/frontend/app/modules/socket.module.js
+++ b/modules/web-console/frontend/app/modules/socket.module.js
@@ -21,7 +21,7 @@ import io from 'socket.io-client'; // eslint-disable-line no-unused-vars
 angular
 .module('ignite-console.socket', [
 ])
-.provider('igniteSocketFactory', [function() {
+.provider('igniteSocketFactory', function() {
     let _options = {};
 
     /**
@@ -31,11 +31,17 @@ angular
         _options = options;
     };
 
-    this.$get = ['socketFactory', function(socketFactory) {
-        return () => {
+    function factory(socketFactory) {
+        return function() {
             const ioSocket = io.connect(_options);
 
             return socketFactory({ioSocket});
         };
-    }];
-}]);
+    }
+
+    factory.$inject = ['socketFactory'];
+
+    this.$get = factory;
+
+    return this;
+});
diff --git a/modules/web-console/frontend/app/modules/states/admin.state.js b/modules/web-console/frontend/app/modules/states/admin.state.js
index f45421aedcfbf..68e65cf9a5250 100644
--- a/modules/web-console/frontend/app/modules/states/admin.state.js
+++ b/modules/web-console/frontend/app/modules/states/admin.state.js
@@ -21,7 +21,7 @@ angular
 .module('ignite-console.states.admin', [
     'ui.router'
 ])
-.config(['$stateProvider', function($stateProvider) {
+.config(['$stateProvider', /** @param {import('@uirouter/angularjs').StateProvider} $stateProvider */ function($stateProvider) {
     // set up the states
     $stateProvider
     .state('base.settings.admin', {
diff --git a/modules/web-console/frontend/app/modules/states/errors.state.js b/modules/web-console/frontend/app/modules/states/errors.state.js
index fcc6a5bbc51ac..19a1947b60334 100644
--- a/modules/web-console/frontend/app/modules/states/errors.state.js
+++ b/modules/web-console/frontend/app/modules/states/errors.state.js
@@ -23,7 +23,7 @@ angular
     .module('ignite-console.states.errors', [
         'ui.router'
     ])
-    .config(['$stateProvider', function($stateProvider) {
+    .config(['$stateProvider', /** @param {import('@uirouter/angularjs').StateProvider} $stateProvider */ function($stateProvider) {
         // set up the states
         $stateProvider
             .state('404', {
diff --git a/modules/web-console/frontend/app/modules/states/logout.state.js b/modules/web-console/frontend/app/modules/states/logout.state.js
index 9f9c7c6774842..9adc81b940340 100644
--- a/modules/web-console/frontend/app/modules/states/logout.state.js
+++ b/modules/web-console/frontend/app/modules/states/logout.state.js
@@ -20,12 +20,12 @@ import angular from 'angular';
 angular.module('ignite-console.states.logout', [
     'ui.router'
 ])
-.config(['$stateProvider', function($stateProvider) {
+.config(['$stateProvider', /** @param {import('@uirouter/angularjs').StateProvider} $stateProvider */ function($stateProvider) {
     // set up the states
     $stateProvider.state('logout', {
         url: '/logout',
         permission: 'logout',
-        controller: ['Auth', (Auth) => Auth.logout()],
+        controller: ['Auth', function(Auth) {Auth.logout();}],
         tfMetaTags: {
             title: 'Logout'
         }
diff --git a/modules/web-console/frontend/app/modules/states/settings.state.js b/modules/web-console/frontend/app/modules/states/settings.state.js
index 1651376cdd015..e967badf523e3 100644
--- a/modules/web-console/frontend/app/modules/states/settings.state.js
+++ b/modules/web-console/frontend/app/modules/states/settings.state.js
@@ -22,7 +22,7 @@ angular
     .module('ignite-console.states.settings', [
         'ui.router'
     ])
-    .config(['$stateProvider', function($stateProvider) {
+    .config(['$stateProvider', /** @param {import('@uirouter/angularjs').StateProvider} $stateProvider */ function($stateProvider) {
         // Set up the states.
         $stateProvider
             .state('base.settings', {
diff --git a/modules/web-console/frontend/app/modules/user/Auth.service.js b/modules/web-console/frontend/app/modules/user/Auth.service.js
index 744943ad05c98..8c3cc4ea4e2db 100644
--- a/modules/web-console/frontend/app/modules/user/Auth.service.js
+++ b/modules/web-console/frontend/app/modules/user/Auth.service.js
@@ -30,7 +30,11 @@ export default class AuthService {
     /**
      * @param {ng.IHttpService} $http
      * @param {ng.IRootScopeService} $root
+     * @param {import('@uirouter/angularjs').StateService} $state
      * @param {ng.IWindowService} $window
+     * @param {ReturnType<typeof import('app/services/Messages.service').default>} Messages
+     * @param {ReturnType<typeof import('app/modules/getting-started/GettingStarted.provider').service>} gettingStarted
+     * @param {ReturnType<typeof import('./User.service').default>} User
      */
     constructor($http, $root, $state, $window, Messages, gettingStarted, User) {
         this.$http = $http;
diff --git a/modules/web-console/frontend/app/modules/user/User.service.js b/modules/web-console/frontend/app/modules/user/User.service.js
index 8b9a1e71245c1..3fdb9b9919fed 100644
--- a/modules/web-console/frontend/app/modules/user/User.service.js
+++ b/modules/web-console/frontend/app/modules/user/User.service.js
@@ -15,10 +15,35 @@
  * limitations under the License.
  */
 
-export default ['User', ['$q', '$injector', '$rootScope', '$state', '$http', function($q, $injector, $root, $state, $http) {
+/**
+ * @typedef User
+ * @prop {string} _id
+ * @prop {boolean} admin
+ * @prop {string} country
+ * @prop {string} email
+ * @prop {string} firstName
+ * @prop {string} lastName
+ * @prop {string} lastActivity
+ * @prop {string} lastLogin
+ * @prop {string} registered
+ * @prop {string} token
+ */
+
+/**
+ * @param {ng.IQService} $q
+ * @param {ng.auto.IInjectorService} $injector
+ * @param {ng.IRootScopeService} $root
+ * @param {import('@uirouter/angularjs').StateService} $state
+ * @param {ng.IHttpService} $http
+ */
+export default function User($q, $injector, $root, $state, $http) {
+    /** @type {ng.IPromise<User>} */
     let user;
 
     return {
+        /**
+         * @returns {ng.IPromise<User>}
+         */
         load() {
             return user = $http.post('/api/v1/user')
                 .then(({data}) => {
@@ -48,4 +73,6 @@ export default ['User', ['$q', '$injector', '$rootScope', '$state', '$http', fun
             sessionStorage.removeItem('IgniteDemoMode');
         }
     };
-}]];
+}
+
+User.$inject = ['$q', '$injector', '$rootScope', '$state', '$http'];
diff --git a/modules/web-console/frontend/app/modules/user/user.module.js b/modules/web-console/frontend/app/modules/user/user.module.js
index 889ee5edb2476..9591a98cc3f17 100644
--- a/modules/web-console/frontend/app/modules/user/user.module.js
+++ b/modules/web-console/frontend/app/modules/user/user.module.js
@@ -21,12 +21,12 @@ import aclData from './permissions';
 import Auth from './Auth.service';
 import User from './User.service';
 
-angular.module('ignite-console.user', [
-    'mm.acl',
-    'ignite-console.config',
-    'ignite-console.core'
-])
-.factory('sessionRecoverer', ['$injector', '$q', ($injector, $q) => {
+/**
+ * @param {ng.auto.IInjectorService} $injector
+ * @param {ng.IQService} $q
+ */
+function sessionRecoverer($injector, $q) {
+    /** @type {ng.IHttpInterceptor} */
     return {
         responseError: (response) => {
             // Session has expired
@@ -42,13 +42,18 @@ angular.module('ignite-console.user', [
             return $q.reject(response);
         }
     };
-}])
-.config(['$httpProvider', ($httpProvider) => {
-    $httpProvider.interceptors.push('sessionRecoverer');
-}])
-.service('Auth', Auth)
-.service(...User)
-.run(['$rootScope', '$transitions', 'AclService', 'User', 'IgniteActivitiesData', ($root, $transitions, AclService, User, Activities) => {
+}
+
+sessionRecoverer.$inject = ['$injector', '$q'];
+
+/**
+ * @param {ng.IRootScopeService} $root
+ * @param {import('@uirouter/angularjs').TransitionService} $transitions
+ * @param {unknown} AclService
+ * @param {ReturnType<typeof import('./User.service').default>} User
+ * @param {ReturnType<typeof import('app/components/activities-user-dialog/index').default>} Activities
+ */
+function run($root, $transitions, AclService, User, Activities) {
     AclService.setAbilities(aclData);
     AclService.attachRole('guest');
 
@@ -90,4 +95,19 @@ angular.module('ignite-console.user', [
                 return $state.target(trans.to().failState || '403');
             });
     });
-}]);
+}
+
+run.$inject = ['$rootScope', '$transitions', 'AclService', 'User', 'IgniteActivitiesData'];
+
+angular.module('ignite-console.user', [
+    'mm.acl',
+    'ignite-console.config',
+    'ignite-console.core'
+])
+.factory('sessionRecoverer', sessionRecoverer)
+.config(['$httpProvider', ($httpProvider) => {
+    $httpProvider.interceptors.push('sessionRecoverer');
+}])
+.service('Auth', Auth)
+.service('User', User)
+.run(run);
diff --git a/modules/web-console/frontend/app/primitives/btn/index.scss b/modules/web-console/frontend/app/primitives/btn/index.scss
index 2d9e9c43428f8..4401a81bf972f 100644
--- a/modules/web-console/frontend/app/primitives/btn/index.scss
+++ b/modules/web-console/frontend/app/primitives/btn/index.scss
@@ -271,7 +271,7 @@ $btn-content-padding-with-border: 9px 11px;
 .btn-ignite--secondary {
     background-color: white;
     color: #424242;
-    border: 1px solid #dedede;
+    border: 1px solid #c5c5c5;
     padding: $btn-content-padding-with-border;
 
     &:hover, &.hover,
@@ -319,10 +319,22 @@ $btn-content-padding-with-border: 9px 11px;
         $line-color: $ignite-brand-success;
         border-right-color: change-color($line-color, $saturation: 63%, $lightness: 33%);
     }
+    .btn-ignite.btn-ignite--secondary + .btn-ignite.btn-ignite--secondary {
+        border-left: 0;
+    }
+
+    &[disabled] .btn-ignite.btn-ignite--primary {
+        border-right-color: change-color($ignite-brand-primary, $lightness: 83%);
+    }
+
+    &[disabled] .btn-ignite.btn-ignite--success {
+        border-right-color: change-color($ignite-brand-success, $lightness: 83%);
+    }
 }
 
 @mixin ignite-link($color, $color-hover) {
     color: $color;
+    text-decoration: none;
 
     &:hover, &.hover,
     &:focus, &.focus {
diff --git a/modules/web-console/frontend/app/primitives/checkbox/index.scss b/modules/web-console/frontend/app/primitives/checkbox/index.scss
index d1e1e837387c2..847e33c4ff7ab 100644
--- a/modules/web-console/frontend/app/primitives/checkbox/index.scss
+++ b/modules/web-console/frontend/app/primitives/checkbox/index.scss
@@ -15,26 +15,6 @@
  * limitations under the License.
  */
 
-input[type='checkbox'] {
-    background-image: url(/images/checkbox.svg);
-    width: 12px !important;
-    height: 12px !important;
-    -webkit-appearance: none;
-    -moz-appearance: none;
-    appearance: none;
-    background-repeat: no-repeat;
-    background-size: 100%;
-    padding: 0;
-    border: none;
-
-    &:checked {
-        background-image: url(/images/checkbox-active.svg);
-    }
-    &:disabled {
-        opacity: 0.5;
-    }
-}
-
 .theme--ignite {
     .form-field-checkbox {
         z-index: 2;
diff --git a/modules/web-console/frontend/app/primitives/datepicker/index.pug b/modules/web-console/frontend/app/primitives/datepicker/index.pug
index 7120111aef19e..28cd1a06c24c2 100644
--- a/modules/web-console/frontend/app/primitives/datepicker/index.pug
+++ b/modules/web-console/frontend/app/primitives/datepicker/index.pug
@@ -14,9 +14,9 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin ignite-form-field-datepicker(label, model, name, mindate, maxdate, minview = 1, format = 'MMM yyyy', disabled, required, placeholder, tip)
-    mixin form-field-input()
-        input.form-control(
+mixin form-field__datepicker({ label, model, name, mindate, maxdate, minview = 1, format = 'MMM yyyy', disabled, required, placeholder, tip })
+    mixin __form-field__datepicker()
+        input(
             id=`{{ ${name} }}Input`
             name=`{{ ${name} }}`
 
@@ -41,19 +41,25 @@ mixin ignite-form-field-datepicker(label, model, name, mindate, maxdate, minview
 
             tabindex='0'
 
-            onkeydown='return false'
+            onkeydown='return false',
+            ng-ref='$input'
+            ng-ref-read='ngModel'
         )&attributes(attributes.attributes)
 
-    .datepicker--ignite.ignite-form-field
-        if name
-            +ignite-form-field__label(label, name, required)
+    .form-field.form-field__datepicker.ignite-form-field(id=`{{ ${name} }}Field`)
+        +form-field__label({ label, name, required, disabled })
+            +form-field__tooltip({ title: tip, options: tipOpts })
 
-        .ignite-form-field__control
-            if tip
-                i.tipField.icon-help(bs-tooltip='' data-title=tip)
+        .form-field__control
+            - attributes.type='button'
+            +__form-field__datepicker(attributes=attributes)
 
+        .form-field__errors(
+            ng-messages=`$input.$error`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
+        )
             if block
                 block
 
-            .input-tip
-                +form-field-input(attributes=attributes)
+            if required
+                +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` })
diff --git a/modules/web-console/frontend/app/primitives/datepicker/index.scss b/modules/web-console/frontend/app/primitives/datepicker/index.scss
index b0c13e6ea6b70..c8edab58f3558 100644
--- a/modules/web-console/frontend/app/primitives/datepicker/index.scss
+++ b/modules/web-console/frontend/app/primitives/datepicker/index.scss
@@ -39,58 +39,3 @@
         }
     }
 }
-
-.datepicker--ignite {
-    $height: 36px;
-
-    display: inline-block;
-    width: auto;
-
-    font-size: 14px;
-
-    label.ignite-form-field__label {
-        width: auto;
-        max-width: initial;
-
-        font-size: inherit;
-        line-height: $height;
-    }
-
-    .ignite-form-field__control {
-        @import "./../../../public/stylesheets/variables.scss";
-
-        width: auto;
-
-        input {
-            width: auto;
-            height: $height;
-            min-width: 70px;
-            max-width: 70px;
-            padding: 0;
-            padding-left: 5px;
-
-            cursor: pointer;
-            color: transparent;
-            font-size: inherit;
-            line-height: $height;
-            text-align: left;
-            text-shadow: 0 0 0 $ignite-brand-success;
-
-            border: none;
-            box-shadow: none;
-
-            &:hover, &:focus {
-                text-shadow: 0 0 0 change-color($ignite-brand-success, $lightness: 26%);
-            }
-        }
-    }
-}
-
-.theme--ignite {
-    .datepicker--ignite {
-        display: block;
-        width: 100%;
-
-        font-size: 14px;
-    }
-}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/primitives/form-field/checkbox.pug b/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
index b498cbd51de6a..88b8f5a2f9618 100644
--- a/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
@@ -14,9 +14,9 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin form-field__checkbox({ label, model, name, disabled, required, tip })
+mixin form-field__checkbox({ label, model, name, disabled, required, tip, tipOpts })
     .form-field.form-field__checkbox(id=`{{ ${name} }}Field`)
-        +form-field__label({ label, name, required })
+        +form-field__label({ label, name, required, disabled })
             +form-field__tooltip({ title: tip, options: tipOpts })
 
         .form-field__control
diff --git a/modules/web-console/frontend/app/primitives/form-field/dropdown.pug b/modules/web-console/frontend/app/primitives/form-field/dropdown.pug
index de83bf9229ce3..96d8482cfc8af 100644
--- a/modules/web-console/frontend/app/primitives/form-field/dropdown.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/dropdown.pug
@@ -14,8 +14,9 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin form-field__dropdown({ label, model, name, disabled, required, multiple, placeholder, placeholderEmpty, options, tip })
-    -var errLbl = label.substring(0, label.length - 1)
+mixin form-field__dropdown({ label, model, name, disabled, required, multiple, placeholder, placeholderEmpty, options, optionLabel = 'label', tip, tipOpts })
+    -var errLbl = label ? label.substring(0, label.length - 1) : 'Field';
+
     mixin __form-field__input()
         button.select-toggle(
             type='button'
@@ -27,27 +28,30 @@ mixin form-field__dropdown({ label, model, name, disabled, required, multiple, p
             ng-model=model
             ng-disabled=disabled && `${disabled}`
             ng-required=required && `${required}`
+            ng-ref='$input'
+            ng-ref-read='ngModel'
 
             bs-select
-            bs-options=`item.value as item.label for item in ${options}`
+            bs-options=`item.value as item.${optionLabel} for item in ${options}`
 
             data-multiple=multiple ? '1' : false
 
             tabindex='0'
         )&attributes(attributes.attributes)
 
-    .form-field(id=`{{ ${name} }}Field`)
-        +form-field__label({ label, name, required })
+    .form-field.form-field__dropdown.ignite-form-field(id=`{{ ${name} }}Field`)
+        +form-field__label({ label, name, required, disabled })
             +form-field__tooltip({ title: tip, options: tipOpts })
 
         .form-field__control
             +__form-field__input(attributes=attributes)
 
         .form-field__errors(
-            ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}`
+            ng-messages=`$input.$error`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
         )
             if block
                 block
 
             if required
-                +form-field__error({ name, error: 'required', message: `${errLbl} could not be empty!` })
+                +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` })
diff --git a/modules/web-console/frontend/app/primitives/form-field/email.pug b/modules/web-console/frontend/app/primitives/form-field/email.pug
index 5fccdb29e363a..b68a520811741 100644
--- a/modules/web-console/frontend/app/primitives/form-field/email.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/email.pug
@@ -32,6 +32,6 @@ mixin form-field__email({ label, model, name, disabled, required, placeholder, t
                 block
 
             if required
-                +form-field__error({ name, error: 'required', message: `${errLbl} could not be empty!` })
+                +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` })
 
-            +form-field__error({ name, error: 'email', message: `${errLbl} has invalid format!` })
+            +form-field__error({ error: 'email', message: `${errLbl} has invalid format!` })
diff --git a/modules/web-console/frontend/app/primitives/form-field/error.pug b/modules/web-console/frontend/app/primitives/form-field/error.pug
index 34b03c2e203c4..9b44c3c70770d 100644
--- a/modules/web-console/frontend/app/primitives/form-field/error.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/error.pug
@@ -16,6 +16,7 @@
 
 mixin form-field__error({ error, message })
     .form-field__error(ng-message=error)
+        div #{ message }
         div(
             bs-tooltip=''
             data-title=message
diff --git a/modules/web-console/frontend/app/primitives/form-field/index.pug b/modules/web-console/frontend/app/primitives/form-field/index.pug
index 9b67d6c8e791e..5a54f21aff9a6 100644
--- a/modules/web-console/frontend/app/primitives/form-field/index.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/index.pug
@@ -25,3 +25,5 @@ include ./password
 include ./phone
 include ./dropdown
 include ./checkbox
+include ./typeahead
+include ./radio
diff --git a/modules/web-console/frontend/app/primitives/form-field/index.scss b/modules/web-console/frontend/app/primitives/form-field/index.scss
index 070dfe722649b..0330e5886a42a 100644
--- a/modules/web-console/frontend/app/primitives/form-field/index.scss
+++ b/modules/web-console/frontend/app/primitives/form-field/index.scss
@@ -194,11 +194,24 @@
             font-style: normal;
             color: $gray-light;
         }
+
+        svg {
+            flex: 0 0 auto;
+            margin-left: 4px;
+        }
+
+        [ignite-icon='info'] {
+            position: relative;
+            top: 1px;
+
+            color: $ignite-brand-success;
+        }
     }
 
     &__control {
         overflow: visible;
         display: flex;
+        flex-direction: row;
         width: 100%;
 
         & > input::placeholder,
@@ -226,6 +239,7 @@
             box-shadow: none;
 
             color: $text-color;
+            font-size: 14px;
             text-align: left;
             line-height: 16px;
 
@@ -264,6 +278,27 @@
         }
     }
 
+    &__control-group {
+        input {
+            min-width: 0;
+            margin-right: -1px;
+
+            border-top-right-radius: 0 !important;
+            border-bottom-right-radius: 0 !important;
+
+            &:focus {
+                z-index: 1;
+            }
+        }
+
+        input + * {
+            border-top-left-radius: 0 !important;
+            border-bottom-left-radius: 0 !important;
+            flex: 0 0 auto;
+            width: 60px !important;
+        }
+    }
+
     &__errors {
         position: absolute;
         right: 0;
@@ -302,7 +337,11 @@
             width: 38px;
         }
 
-        div {
+        div:first-child {
+            display: none;
+        }
+
+        [bs-tooltip] {
             z-index: 1;
             position: absolute;
             top: 0;
@@ -311,7 +350,7 @@
             height: 36px;
         }
 
-        svg {
+        [ignite-icon] {
             position: absolute;
             top: 10px;
             right: 0;
@@ -324,14 +363,93 @@
     }
 }
 
+.theme--ignite-errors-horizontal {
+    .form-field__control {
+        // Reset offset to appearance of input for invalid password
+        & > input[type='email'].ng-invalid.ng-touched,
+        & > input[type='text'].ng-invalid.ng-touched,
+        & > input[type='password'].ng-invalid.ng-touched {
+            padding-right: 0;
+        }
+        // Reset offset to appearance of dropdown for invalid data
+        & > button.select-toggle.ng-invalid.ng-touched {
+            &:after {
+                right: 10px;
+            }
+        }
+    }
+
+    .form-field__errors {
+        position: relative;
+        
+        padding: 5px 10px 0px;
+
+        color: $ignite-brand-primary;
+        font-size: 12px;
+        line-height: 14px;
+
+        &:empty {
+            display: none;
+        }
+
+        [ng-message] + [ng-message] {
+            margin-top: 10px;
+        }
+    }
+
+    .form-field__error {
+        float: none;
+        width: auto;
+        height: auto;
+
+        text-align: left;
+        line-height: 14px;
+
+        div:first-child {
+            display: block;
+        }
+
+        [bs-tooltip],
+        [ignite-icon] {
+            display: none;
+        }
+    }
+
+    .form-field__error + .form-field__error {
+        margin-top: 10px;
+    }
+
+    .form-field__checkbox {
+        flex-wrap: wrap;
+
+        .form-field__errors {
+            margin-left: -10px;
+            flex-basis: 100%;
+
+            .form-field__error {
+                width: auto;
+
+                div {
+                    width: auto;
+                }
+            }
+        }
+    }
+}
+
+.form-field__radio,
 .form-field__checkbox {
     $errorSize: 16px;
-    display: flex;
+    display: inline-flex;
+    width: auto;
+
+    сursor: pointer;
 
     .form-field {
         &__label {
             order: 1;
             margin: 0;
+            cursor: pointer;
         }
 
         &__control {
@@ -376,6 +494,132 @@
     }
 }
 
+.form-field__radio {
+    .form-field__control {
+        padding: 2px 0;
+    }
+
+    .form-field__control > input[type='radio'] {
+        -webkit-appearance: none;
+
+        width: 13px;
+        height: 13px;
+        padding: 0;
+
+        background: white;
+        border: none;
+        border-radius: 50%;
+        box-shadow: inset 0 0 0 1px rgb(197, 197, 197);
+
+        &:focus {
+            outline: none;
+            border: none;
+            box-shadow: 0 0 0 2px rgba(0, 103, 185, .3),
+                  inset 0 0 0 1px rgb(197, 197, 197);
+        }
+
+        &:checked {
+            border: none;
+            box-shadow: inset 0 0 0 5px rgba(0, 103, 185, 1); 
+
+            &:focus {
+                box-shadow: 0 0 0 2px rgba(0, 103, 185, .3),
+                      inset 0 0 0 5px rgba(0, 103, 185, 1); 
+            }
+        }
+    }
+}
+
+.form-field__checkbox {
+    .form-field__control > input[type='checkbox'] {
+        border-radius: 2px;
+
+        background-image: url(/images/checkbox.svg);
+        width: 12px !important;
+        height: 12px !important;
+        -webkit-appearance: none;
+        -moz-appearance: none;
+        appearance: none;
+        background-repeat: no-repeat;
+        background-size: 100%;
+        padding: 0;
+        border: none;
+
+        &:checked {
+            background-image: url(/images/checkbox-active.svg);
+        }
+
+        &:disabled {
+            opacity: 0.5;
+        }
+
+        &:focus {
+            outline: none;
+            box-shadow: 0 0 0 2px rgba(0, 103, 185, .3);
+        }
+    }
+}
+
+.form-field--inline {
+    display: inline-block;
+    width: auto;
+
+    .form-field {
+        display: flex;
+        align-items: baseline;
+    }
+
+    .form-field__label {
+        white-space: nowrap;
+    }
+
+    form-field-size,
+    .form-field__text {
+        .form-field__control {
+            margin-left: 10px;
+        }
+    }
+
+    .form-field__dropdown,
+    .form-field__datepicker,
+    .form-field__timepicker {
+        .form-field__control {
+            width: auto;
+
+            input,
+            button {
+                color: transparent;
+
+                text-shadow: 0 0 0 $ignite-brand-success;
+
+                border: none;
+                box-shadow: none;
+
+                background: linear-gradient(to right, rgb(0, 103, 185), transparent) 0px 25px / 0px, 
+                            linear-gradient(to right, rgb(0, 103, 185) 70%, transparent 0%) 0% 0% / 8px 1px repeat-x,
+                            0% 0% / 0px, 0% 0% / 4px;
+                background-size: 0, 8px 1px, 0, 0;
+                background-position: 1px 25px;
+
+                padding-left: 0px;
+                padding-right: 0px;
+                margin-left: 10px;
+                margin-right: 10px;
+
+                &:hover, &:focus {
+                    text-shadow: 0 0 0 change-color($ignite-brand-success, $lightness: 26%);
+                }
+            }
+        }
+    }
+
+    .form-field__dropdown {
+        button::after {
+            display: none;
+        }
+    }
+}
+
 .form-field__password {
     // Validation error notification will overlap with visibility button if it's not moved more to the left
     input[type='password'].ng-invalid.ng-touched,
@@ -400,10 +644,44 @@
     }
 }
 
+.form-field__dropdown {
+    .form-field__control {
+        > button:not(.btn-ignite) {
+            padding-top: 10px;
+        }
+    }
+}
+
+.form-field__ace {
+    .ace_editor {
+        width: 100%;
+        min-height: 70px;
+        margin: 0;
+
+        border: solid 1px #c5c5c5;
+        border-radius: 4px;
+        background-color: #ffffff;
+        box-shadow: none;
+
+        .ace_content {
+            padding-left: 2px;
+        }
+
+        &.ace_focus {
+            border-color: $ignite-brand-success;
+            box-shadow: none;
+        }
+    }
+}
+
+.form-field.ignite-form-field label.required {
+    margin-left: 0 !important;
+}
+
 .form-fieldset {
     padding: 10px;
 
-    border: 1px solid hsla(0,0%,77%,.5);
+    border: 1px solid hsla(0, 0%, 77%, .5);
     border-radius: 4px;
 
     legend {
diff --git a/modules/web-console/frontend/app/primitives/form-field/input.pug b/modules/web-console/frontend/app/primitives/form-field/input.pug
index 0fee77b33ac97..0551101f73a57 100644
--- a/modules/web-console/frontend/app/primitives/form-field/input.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/input.pug
@@ -14,9 +14,9 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin form-field__input({ name, model, disabled, required, placeholder })
+mixin form-field__input({ name, model, disabled, required, placeholder, namePostfix = '' })
     input(
-        id=`{{ ${name} }}Input`
+        id=`{{ ${name} }}${ namePostfix }Input`
         name=`{{ ${name} }}`
         placeholder=placeholder
 
@@ -24,5 +24,6 @@ mixin form-field__input({ name, model, disabled, required, placeholder })
 
         ng-required=required && `${required}`
         ng-disabled=disabled && `${disabled}`
-        ng-focus='tableReset()'
+        ng-ref='$input'
+        ng-ref-read='ngModel'
     )&attributes(attributes ? attributes.attributes ? attributes.attributes : attributes : {})
diff --git a/modules/web-console/frontend/app/primitives/form-field/label.pug b/modules/web-console/frontend/app/primitives/form-field/label.pug
index d725f9d65afea..74ddecb764d2d 100644
--- a/modules/web-console/frontend/app/primitives/form-field/label.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/label.pug
@@ -14,17 +14,18 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin form-field__label({ label, name, required, optional, disabled })
-    -var colon = label[label.length-1] === ':' ? ':' : '';
-    - label = label[label.length-1] === ':' ? label.substring(0, label.length - 1) : label
-    - optional = optional ? ' <i>(optional)</i>' : '';
+mixin form-field__label({ label, name, required, optional, disabled, namePostfix = '' })
+    if label
+        -var colon = label[label.length-1] === ':' ? ':' : '';
+        - label = label[label.length-1] === ':' ? label.substring(0, label.length - 1) : label
+        - optional = optional ? ' <i>(optional)</i>' : '';
 
-    label.form-field__label(
-        id=name && `{{ ${name} }}Label`
-        for=name && `{{ ${name} }}Input`
-        class=`{{ ${required} ? 'required' : '' }}`
-        ng-disabled=disabled && `${disabled}`
-    )
-        span !{label}!{optional}!{colon}
-        if block
-            block
+        label.form-field__label(
+            id=name && `{{ ${name} }}Label`
+            for=name && `{{ ${name} }}${ namePostfix }Input`
+            class=`{{ ${required} ? 'required' : '' }}`
+            ng-disabled=disabled && `${disabled}`
+        )
+            span !{label}!{optional}!{colon}
+            if block
+                block
diff --git a/modules/web-console/frontend/app/primitives/form-field/number.pug b/modules/web-console/frontend/app/primitives/form-field/number.pug
index 11f8e22bc2ee8..ea907090e69d2 100644
--- a/modules/web-console/frontend/app/primitives/form-field/number.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/number.pug
@@ -14,23 +14,24 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin form-field__number({ label, model, name, disabled, required, placeholder, tip, min, max, step, postfix })
-    -var errLbl = label.substring(0, label.length - 1)
+mixin form-field__number({ label, model, name, disabled, required, placeholder, tip, min, max, step = '1', postfix })
+    -var errLbl = label[label.length - 1] === ':' ? label.substring(0, label.length - 1) : label
 
-    .form-field
-        +form-field__label({ label, name, required })
+    .form-field.ignite-form-field
+        +form-field__label({ label, name, required, disabled })
             +form-field__tooltip({ title: tip, options: tipOpts })
 
         .form-field__control(class=postfix && 'form-field__control--postfix' data-postfix=postfix)
             - attributes.type = 'number'
             - attributes.min = min ? min : '0'
             - attributes.max = max ? max : '{{ Number.MAX_VALUE }}'
-            - attributes.step = step ? step : '1'
+            - attributes.step = step
             +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes)
 
         .form-field__errors(
             data-postfix=postfix
-            ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}`
+            ng-messages=`$input.$error`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
         )
             if block
                 block
@@ -38,10 +39,10 @@ mixin form-field__number({ label, model, name, disabled, required, placeholder,
             if required
                 +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` })
 
-            if min
-                +form-field__error({ error: 'min', message: `${errLbl} is less than allowable minimum: ${ min || 0 }`})
+            +form-field__error({ error: 'min', message: `${errLbl} is less than allowable minimum: ${ min || 0 }`})
 
-            if max
-                +form-field__error({ error: 'max', message: `${errLbl} is more than allowable maximum: ${ max }`})
+            +form-field__error({ error: 'max', message: `${errLbl} is more than allowable maximum: ${ max }`})
+
+            +form-field__error({ error: 'step', message: `${errLbl} step should be ${step || 1}` })
 
             +form-field__error({ error: 'number', message: 'Only numbers allowed' })
diff --git a/modules/web-console/frontend/app/primitives/form-field/password.pug b/modules/web-console/frontend/app/primitives/form-field/password.pug
index 40e1aa90b87d7..6b9818bc91f43 100644
--- a/modules/web-console/frontend/app/primitives/form-field/password.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/password.pug
@@ -42,6 +42,6 @@ mixin form-field__password({ label, model, name, disabled, required, placeholder
                 block
 
             if required
-                +form-field__error({ name, error: 'required', message: `${errLbl} could not be empty!` })
+                +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` })
                 
-            +form-field__error({ name, error: 'mismatch', message: `Password does not match the confirm password!` })
+            +form-field__error({ error: 'mismatch', message: `Password does not match the confirm password!` })
diff --git a/modules/web-console/frontend/app/primitives/form-field/phone.pug b/modules/web-console/frontend/app/primitives/form-field/phone.pug
index b65c5d22305c7..8b503018d3bda 100644
--- a/modules/web-console/frontend/app/primitives/form-field/phone.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/phone.pug
@@ -32,4 +32,4 @@ mixin form-field__phone({ label, model, name, disabled, required, optional, plac
                 block
 
             if required
-                +form-field__error({ name, error: 'required', message: `${errLbl} could not be empty!` })
+                +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` })
diff --git a/modules/web-console/frontend/app/primitives/file/index.pug b/modules/web-console/frontend/app/primitives/form-field/radio.pug
similarity index 53%
rename from modules/web-console/frontend/app/primitives/file/index.pug
rename to modules/web-console/frontend/app/primitives/form-field/radio.pug
index 7bdd3cc411c9b..57ae097e5c1f3 100644
--- a/modules/web-console/frontend/app/primitives/file/index.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/radio.pug
@@ -14,24 +14,19 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin ignite-form-field-file(label, model, name, disabled, required, options, tip)
-    .file--ignite.ignite-form-field(ng-class='{ "choosen": `${model}` }')
-        label.ignite-form-field__label(for=`{{ ${name} }}Input`)
-            span Folder name:
-        .ignite-form-field__control
-            label.btn-ignite.btn-ignite--primary(for=`{{ ${name} }}Input`)
-                | Choose Destination Folder
-            label.tipField.link-primary(for=`{{ ${name} }}Input`) Change folder
+mixin form-field__radio({ label, model, name, value, disabled, required, tip })
+    .form-field.form-field__radio
+        +form-field__label({ label, name, required, disabled, namePostfix: value })
+            +form-field__tooltip({ title: tip, options: tipOpts })
 
-            +tooltip(tip, tipOpts)
+        .form-field__control
+            - attributes.type='radio'
+            - attributes['ng-value'] = value
+            +form-field__input({ name, model, disabled, required, placeholder, namePostfix: value })(attributes=attributes)
 
+        .form-field__errors(
+            ng-messages=`$input.$error`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
+        )
             if block
                 block
-
-            .input-tip
-                input(
-                    id=`{{ ${name} }}Input`
-                    type='file'
-                    ng-model=model
-                )&attributes(attributes)
-                | {{ `${model}` }}
diff --git a/modules/web-console/frontend/app/primitives/form-field/text.pug b/modules/web-console/frontend/app/primitives/form-field/text.pug
index 8f63f19f2aeaa..ab8d14d2cbd26 100644
--- a/modules/web-console/frontend/app/primitives/form-field/text.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/text.pug
@@ -15,9 +15,9 @@
     limitations under the License.
 
 mixin form-field__text({ label, model, name, disabled, required, placeholder, tip })
-    -var errLbl = label.substring(0, label.length - 1)
+    -let errLbl = label[label.length - 1] === ':' ? label.substring(0, label.length - 1) : label
 
-    .form-field(id=`{{ ${name} }}Field`)
+    .form-field.form-field__text.ignite-form-field(id=`{{ ${name} }}Field`)
         +form-field__label({ label, name, required, disabled })
             +form-field__tooltip({ title: tip, options: tipOpts })
 
@@ -26,10 +26,11 @@ mixin form-field__text({ label, model, name, disabled, required, placeholder, ti
             +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes)
 
         .form-field__errors(
-            ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}`
+            ng-messages=`$input.$error`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
         )
             if block
                 block
 
             if required
-                +form-field__error({ name, error: 'required', message: `${errLbl} could not be empty!` })
+                +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` })
diff --git a/modules/web-console/frontend/app/primitives/form-field/tooltip.pug b/modules/web-console/frontend/app/primitives/form-field/tooltip.pug
index 08ffd83c39f33..34376fdfaa517 100644
--- a/modules/web-console/frontend/app/primitives/form-field/tooltip.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/tooltip.pug
@@ -14,5 +14,13 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin form-field__tooltip({ title, options })
-    +tooltip(title, options)
+mixin form-field__tooltip({ title, options = { placement: 'auto' }})
+    if title
+        svg(
+            ignite-icon='info'
+            bs-tooltip=''
+
+            data-title=title
+            data-container=options && options.container || false
+            data-placement=options && options.placement || false
+        )&attributes(attributes)
diff --git a/modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.pug b/modules/web-console/frontend/app/primitives/form-field/typeahead.pug
similarity index 64%
rename from modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.pug
rename to modules/web-console/frontend/app/primitives/form-field/typeahead.pug
index 888634b72ca30..b2c62ae3d464e 100644
--- a/modules/web-console/frontend/app/helpers/jade/form/form-field-datalist.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/typeahead.pug
@@ -14,11 +14,10 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin form-field-datalist(label, model, name, disabled, required, placeholder, options, tip)
+mixin form-field__typeahead({ label, model, name, disabled, required, placeholder, options, tip })
     -var errLbl = label.substring(0, label.length - 1)
-
-    mixin form-field-input()
-        input.form-control(
+    mixin __form-field__typeahead()
+        input(
             id=`{{ ${name} }}Input`
             name=`{{ ${name} }}`
             placeholder=placeholder
@@ -33,20 +32,24 @@ mixin form-field-datalist(label, model, name, disabled, required, placeholder, o
             container='body'
             data-min-length='1'
             ignite-retain-selection
-            expose-ignite-form-field-control='$input'
+            ng-ref='$input'
+            ng-ref-read='ngModel'
         )&attributes(attributes.attributes)
 
-    .ignite-form-field
-        +ignite-form-field__label(label, name, required, disabled)
-            +tooltip(tip, tipOpts)
-        .ignite-form-field__control
-            .input-tip
-                +form-field-input(attributes=attributes)
-        .ignite-form-field__errors(
+    .form-field.form-field__typeahead.ignite-form-field(id=`{{ ${name} }}Field`)
+        +form-field__label({ label, name, required, disabled })
+            +form-field__tooltip({ title: tip, options: tipOpts })
+
+        .form-field__control
+            - attributes.type='text'
+            +__form-field__typeahead(attributes=attributes)
+
+        .form-field__errors(
             ng-messages=`$input.$error`
-            ng-if=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
         )
             if block
                 block
 
-            +form-field-feedback(name, 'required', `${errLbl} could not be empty!`)
+            if required
+                +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` })
diff --git a/modules/web-console/frontend/app/primitives/index.js b/modules/web-console/frontend/app/primitives/index.js
index f9d85917d08b1..a9ff053e9c68c 100644
--- a/modules/web-console/frontend/app/primitives/index.js
+++ b/modules/web-console/frontend/app/primitives/index.js
@@ -28,7 +28,7 @@ import './ui-grid/index.scss';
 import './ui-grid-header/index.scss';
 import './ui-grid-settings/index.scss';
 import './page/index.scss';
-import './radio/index.scss';
+import './spinner-circle/index.scss';
 import './switcher/index.scss';
 import './form-field/index.scss';
 import './typography/index.scss';
diff --git a/modules/web-console/frontend/app/primitives/modal/index.scss b/modules/web-console/frontend/app/primitives/modal/index.scss
index 802a24195ca5e..2be2ecdc2c276 100644
--- a/modules/web-console/frontend/app/primitives/modal/index.scss
+++ b/modules/web-console/frontend/app/primitives/modal/index.scss
@@ -67,14 +67,6 @@
     }
 }
 
-.modal .modal-content {
-    background-color: $gray-lighter;
-
-    .input-tip {
-        padding-top: 1px;
-    }
-}
-
 .modal .modal-content .modal-header {
     background-color: $ignite-background-color;
     text-align: center;
@@ -130,7 +122,7 @@
         opacity: 1;
         background: none;
         color: $gray-light;
-        
+
         [ignite-icon] {
             height: 12px;
         }
@@ -182,12 +174,22 @@
         }
 
         .modal-body {
-            margin: 0;
             padding: 5px 20px;
+            margin: 0;
 
             .input-tip {
                 padding-top: 0;
             }
+
+            .modal-body--inner-content {
+                max-height: 300px;
+                overflow-x: auto;
+
+                p {
+                    text-align: left;
+                    white-space: nowrap;
+                }
+            }
         }
 
         .modal-footer {
@@ -223,3 +225,26 @@
         }
     }
 }
+
+.modal-footer {
+    display: flex;
+    align-items: center;
+
+    > div {
+        display: flex;
+        flex: 1;
+
+        &:last-child {
+            flex: 1;
+            justify-content: flex-end;
+
+            &.modal-footer--no-grow {
+                flex-grow: 0;
+            }
+        }
+    }
+}
+
+.modal--wide .modal-dialog {
+    width: 900px;
+}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/primitives/panel/index.scss b/modules/web-console/frontend/app/primitives/panel/index.scss
index 5dab39f31fb01..6afcd5c86778b 100644
--- a/modules/web-console/frontend/app/primitives/panel/index.scss
+++ b/modules/web-console/frontend/app/primitives/panel/index.scss
@@ -33,11 +33,17 @@
 
             background-color: initial;
             font-size: 16px;
-            line-height: 36px;
 
             &:hover {
                 text-decoration: none;
             }
+
+            h5 {
+                margin: 0;
+                font-size: 16px;
+                font-weight: normal;
+                line-height: 36px;
+            }
         }
 
         & > hr {
@@ -72,6 +78,18 @@
                 }
             }
         }
+
+        & > header.header-with-selector {
+            margin: 0;
+            padding: 14px 20px;
+            min-height: 65px;
+
+            border-bottom: 1px solid #ddd;
+
+            sub {
+                bottom: 0;
+            }
+        }
     }
 
     &--collapse {
@@ -106,3 +124,11 @@
         }
     }
 }
+
+// Adding top border for panels in modals
+.modal-body {
+    .panel--ignite:not(.panel--ignite__without-border) {
+        border-top: 1px solid #d4d4d4;
+        border-radius: 4px 4px 0 0 ;
+    }
+}
diff --git a/modules/web-console/frontend/app/primitives/radio/index.pug b/modules/web-console/frontend/app/primitives/radio/index.pug
deleted file mode 100644
index f47fd176f10f9..0000000000000
--- a/modules/web-console/frontend/app/primitives/radio/index.pug
+++ /dev/null
@@ -1,37 +0,0 @@
-//-
-    Licensed to the Apache Software Foundation (ASF) under one or more
-    contributor license agreements.  See the NOTICE file distributed with
-    this work for additional information regarding copyright ownership.
-    The ASF licenses this file to You under the Apache License, Version 2.0
-    (the "License"); you may not use this file except in compliance with
-    the License.  You may obtain a copy of the License at
-
-         http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-
-mixin form-field-radio(label, model, name, value, disabled, required, tip)
-    .radio--ignite.ignite-form-field
-        label(id=`{{ ${name} }}Label`)
-            .input-tip
-                if block
-                    block
-                else
-                    input(
-                        id=`{{ ${name} }}Input`
-                        name=`{{ ${name} }}`
-                        type='radio'
-
-                        ng-model=model
-                        ng-value=value
-                        ng-required=required && `${required}`
-                        ng-disabled=disabled && `${disabled}`
-                    )
-                    div
-            span #{label}
-
-            +tooltip(tip, tipOpts)
diff --git a/modules/web-console/frontend/app/primitives/radio/index.scss b/modules/web-console/frontend/app/primitives/radio/index.scss
deleted file mode 100644
index ff9b5b3a43a1a..0000000000000
--- a/modules/web-console/frontend/app/primitives/radio/index.scss
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-@import '../../../public/stylesheets/variables';
-
-.radio--ignite {
-    label {
-        padding-left: 20px;
-
-        line-height: 20px;
-        vertical-align: middle;
-
-        cursor: pointer;
-
-        .input-tip {
-            float: left;
-            width: 14px;
-            margin-left: -20px;
-
-            input[type="radio"] {
-                position: relative;
-                left: -20px;
-
-                & + div {
-                    position: absolute;
-                    top: 50%;
-
-                    width: 14px;
-                    height: 14px;
-                    margin-top: -8px;
-
-                    border-radius: 50%;
-                    box-shadow: inset 0 1px 3px 0 rgba(0, 0, 0, 0.35);
-                }
-
-                &:checked + div {
-                    background-color: #0098ff;
-                    box-shadow: none;
-
-                    &:before {
-                        content: '';
-
-                        position: absolute;
-                        top: 50%;
-                        left: 50%;
-
-                        width: 4px;
-                        height: 4px;
-                        margin-top: -2px;
-                        margin-left: -2px;
-
-                        border-radius: 50%;
-                        background-color: #ffffff;
-                        box-shadow: 0 1px 1px 0 rgba(12, 50, 76, 0.3);
-                    }
-                }
-            }
-        }
-    }
-
-    & + .radio--ignite {
-        margin-left: 45px;
-    }
-}
\ No newline at end of file
diff --git a/modules/web-console/frontend/app/primitives/spinner-circle/index.scss b/modules/web-console/frontend/app/primitives/spinner-circle/index.scss
new file mode 100644
index 0000000000000..88152fab6063c
--- /dev/null
+++ b/modules/web-console/frontend/app/primitives/spinner-circle/index.scss
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+$color-inactive-primary: #C5C5C5;
+$color-inactive-secondary: #FFFFFF;
+$color-active-primary: #EE2B27;
+$color-active-secondary: #FF8485;
+
+.spinner-circle {
+  display: inline-block;
+
+  &:before {
+    content: '';
+
+    display: block;
+
+    width: 20px;
+    height: 20px;
+
+    border-width: 1px;
+    border-style: solid;
+    border-radius: 50%;
+    border-color: $color-inactive-primary;
+    border-left-color: $color-active-primary;
+  }
+}
+
+.spinner-circle:before {
+  border-left-width: 2px;
+  border-left-color: $color-active-primary;
+
+  animation-name: spinner-circle--animation;
+  animation-duration: 1s;
+  animation-iteration-count: infinite;
+  animation-timing-function: linear;
+}
+
+@keyframes spinner-circle--animation {
+  0% {
+    transform: rotate(0deg);
+  }
+  100% {
+    transform: rotate(360deg);
+  }
+}
diff --git a/modules/web-console/frontend/app/primitives/timepicker/index.pug b/modules/web-console/frontend/app/primitives/timepicker/index.pug
index 54ce8c1dfce0f..5e3936c737c25 100644
--- a/modules/web-console/frontend/app/primitives/timepicker/index.pug
+++ b/modules/web-console/frontend/app/primitives/timepicker/index.pug
@@ -14,10 +14,9 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-mixin ignite-form-field-timepicker(label, model, name, mindate, maxdate, disabled, required, placeholder, tip)
-
-    mixin form-field-input()
-        input.form-control(
+mixin form-field__timepicker({ label, model, name, mindate, maxdate, disabled, required, placeholder, tip })
+    mixin __form-field__timepicker()
+        input(
             id=`{{ ${name} }}Input`
             name=`{{ ${name} }}`
 
@@ -35,23 +34,29 @@ mixin ignite-form-field-timepicker(label, model, name, mindate, maxdate, disable
             data-arrow-behavior='picker'
 
             data-placement='bottom'
-            data-container=`.timepicker--ignite`
+            data-container='body'
 
             tabindex='0'
 
-            onkeydown="return false"
+            onkeydown='return false'
+            ng-ref='$input'
+            ng-ref-read='ngModel'
         )&attributes(attributes.attributes)
 
-    .timepicker--ignite.ignite-form-field
-        if name
-            +ignite-form-field__label(label, name, required)
+    .form-field.form-field__timepicker.ignite-form-field(id=`{{ ${name} }}Field`)
+        +form-field__label({ label, name, required, disabled })
+            +form-field__tooltip({ title: tip, options: tipOpts })
 
-        .ignite-form-field__control
-            if tip
-                i.tipField.icon-help(bs-tooltip='' data-title=tip)
+        .form-field__control
+            - attributes.type='button'
+            +__form-field__timepicker(attributes=attributes)
 
+        .form-field__errors(
+            ng-messages=`$input.$error`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
+        )
             if block
                 block
 
-            .input-tip
-                +form-field-input(attributes=attributes)
+            if required
+                +form-field__error({ error: 'required', message: `${errLbl} could not be empty!` })
diff --git a/modules/web-console/frontend/app/primitives/timepicker/index.scss b/modules/web-console/frontend/app/primitives/timepicker/index.scss
index 5be534e10f2dd..643c741b5f8f3 100644
--- a/modules/web-console/frontend/app/primitives/timepicker/index.scss
+++ b/modules/web-console/frontend/app/primitives/timepicker/index.scss
@@ -17,6 +17,7 @@
 
 .timepicker.dropdown-menu {
     padding: 0 4px;
+    line-height: 30px;
 
     button {
         outline: none;
@@ -27,113 +28,62 @@
         height: 100%;
         padding: 6px;
     }
-}
-
-.timepicker--ignite {
-    $height: 36px;
-
-    display: inline-block;
-    width: auto;
-
-    font-size: 14px;
-
-    label.ignite-form-field__label {
-        width: auto;
-        max-width: initial;
-
-        font-size: inherit;
-        line-height: $height;
-    }
-
-    .ignite-form-field__control {
-        @import "./../../../public/stylesheets/variables.scss";
-
-        width: auto;
-
-        input {
-            width: auto;
-            height: $height;
-            min-width: 40px;
-            max-width: 70px;
-            padding: 0;
-            padding-left: 5px;
-
-            cursor: pointer;
-            color: transparent;
-            font-size: inherit;
-            line-height: $height;
-            text-align: left;
-            text-shadow: 0 0 0 $ignite-brand-success;
-
-            border: none;
-            box-shadow: none;
-            background: none;
 
-            &:hover, &:focus {
-                text-shadow: 0 0 0 change-color($ignite-brand-success, $lightness: 26%);
-            }
+    thead, tfoot {
+        th {
+            text-align: center;
+            line-height: 0;
         }
-    }
 
-    .dropdown-menu {
-        line-height: 30px;
+        .btn.btn-default {
+            float: none !important;
+            display: inline-block;
+            margin-right: 0;
 
-        thead, tfoot {
-            th {
-                text-align: center;
-                line-height: 0;
+            &:active {
+                box-shadow: none;
+                background: none;
             }
 
-            .btn.btn-default {
-                float: none !important;
-                display: inline-block;
-                margin-right: 0;
-
-                &:active {
-                    box-shadow: none;
-                    background: none;
-                }
+            &:before {
+                content: '';
 
-                &:before {
-                    content: '';
+                display: block;
+                width: 10px;
+                height: 10px;
 
-                    display: block;
-                    width: 10px;
-                    height: 10px;
-
-                    border: 2px solid #757575;
-                    transform: rotate(45deg);
-                }
+                border: 2px solid #757575;
+                transform: rotate(45deg);
+            }
 
-                &:hover {
-                    background: none;
-                }
+            &:hover {
+                background: none;
             }
         }
+    }
 
-        thead {
-            th {
-                padding-top: 10px;
-            }
+    thead {
+        th {
+            padding-top: 10px;
+        }
 
-            .btn.btn-default {
-                &:before {
-                    border-width: 2px 0 0 2px;
-                }
+        .btn.btn-default {
+            &:before {
+                border-width: 2px 0 0 2px;
             }
         }
+    }
 
-        tfoot {
-            th {
-                padding-top: 2px;
-                padding-bottom: 10px;
-            }
+    tfoot {
+        th {
+            padding-top: 2px;
+            padding-bottom: 10px;
+        }
 
-            .btn.btn-default {
-                &:before {
-                    border-width: 0 2px 2px 0;
-                }
+        .btn.btn-default {
+            &:before {
+                border-width: 0 2px 2px 0;
             }
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/web-console/frontend/app/primitives/ui-grid/index.scss b/modules/web-console/frontend/app/primitives/ui-grid/index.scss
index c450dadaad201..274e21da8ffc5 100644
--- a/modules/web-console/frontend/app/primitives/ui-grid/index.scss
+++ b/modules/web-console/frontend/app/primitives/ui-grid/index.scss
@@ -208,10 +208,6 @@
                 }
 
                 .ui-grid-header--subcategories {
-                    .ui-grid-tree-base-row-header-buttons {
-                        margin-top: 10px;
-                    }
-
                     .ui-grid-selection-row-header-buttons {
                         margin-top: 12px;
 
@@ -449,7 +445,7 @@
                     z-index: 1;
 
                     width: 4px;
-                    height: 46px;
+                    height: 47px;
 
                     background: #0067b9;
                     box-shadow: 0 -1px 0 0 rgba(0, 0, 0, .3), 0 -1px 0 0 rgba(0, 103, 185, 1);
diff --git a/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js b/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js
index 32fa167f2d1af..44ed8eda2dd2a 100644
--- a/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js
+++ b/modules/web-console/frontend/app/services/AngularStrapSelect.decorator.js
@@ -23,7 +23,7 @@ import _ from 'lodash';
  * If this problem will be fixed in AngularStrap we can remove this delegate.
  */
 export default angular.module('mgcrea.ngStrap.select')
-    .decorator('$select', ['$delegate', ($delegate) => {
+    .decorator('$select', ['$delegate', function($delegate) {
         function SelectFactoryDecorated(element, controller, config) {
             const delegate = $delegate(element, controller, config);
 
diff --git a/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js b/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js
index fa59f32765111..f1f8673c6460b 100644
--- a/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js
+++ b/modules/web-console/frontend/app/services/AngularStrapTooltip.decorator.js
@@ -26,7 +26,7 @@ export default angular
     /**
      * Don't hide tooltip when mouse move from element to tooltip.
      */
-    .decorator('$tooltip', ['$delegate', ($delegate) => {
+    .decorator('$tooltip', ['$delegate', function($delegate) {
         function TooltipFactoryDecorated(element, config) {
             let tipElementEntered = false;
 
diff --git a/modules/web-console/frontend/app/services/ChartColors.service.js b/modules/web-console/frontend/app/services/ChartColors.service.js
index 843aa5cb3b038..061ff3f258e56 100644
--- a/modules/web-console/frontend/app/services/ChartColors.service.js
+++ b/modules/web-console/frontend/app/services/ChartColors.service.js
@@ -17,6 +17,6 @@
 
 import COLORS from 'app/data/colors.json';
 
-export default ['IgniteChartColors', function() {
+export default function() {
     return COLORS;
-}];
+}
diff --git a/modules/web-console/frontend/app/services/Clusters.js b/modules/web-console/frontend/app/services/Clusters.js
index 0228a77b4d2cd..e0a2ec7ca40b4 100644
--- a/modules/web-console/frontend/app/services/Clusters.js
+++ b/modules/web-console/frontend/app/services/Clusters.js
@@ -266,7 +266,7 @@ export default class Clusters {
             }
         },
         evictionThreshold: {
-            step: 0.05,
+            step: 0.001,
             max: 0.999,
             min: 0.5,
             default: 0.9
diff --git a/modules/web-console/frontend/app/services/Clusters.spec.js b/modules/web-console/frontend/app/services/Clusters.spec.js
index 92bfa2ee73e26..ffa0be1462c37 100644
--- a/modules/web-console/frontend/app/services/Clusters.spec.js
+++ b/modules/web-console/frontend/app/services/Clusters.spec.js
@@ -19,7 +19,7 @@ import {suite, test} from 'mocha';
 import {assert} from 'chai';
 import {spy} from 'sinon';
 
-import Provider from './Clusters.js';
+import Provider from './Clusters';
 
 const mocks = () => new Map([
     ['$http', {
diff --git a/modules/web-console/frontend/app/services/Confirm.service.js b/modules/web-console/frontend/app/services/Confirm.service.js
index c2eaf35e683b5..953bdcd2d5f02 100644
--- a/modules/web-console/frontend/app/services/Confirm.service.js
+++ b/modules/web-console/frontend/app/services/Confirm.service.js
@@ -39,7 +39,7 @@ export class Confirm {
                 templateUrl,
                 backdrop: true,
                 onBeforeHide: () => reject(new CancellationError()),
-                controller: ['$scope', ($scope) => {
+                controller: ['$scope', function($scope) {
                     $scope.yesNo = yesNo;
                     $scope.content = content;
                     $scope.confirmCancel = $scope.confirmNo = () => {
@@ -56,8 +56,15 @@ export class Confirm {
     }
 }
 
-// Confirm popup service.
-export default ['IgniteConfirm', ['$rootScope', '$q', '$modal', '$animate', ($root, $q, $modal, $animate) => {
+/**
+ * Confirm popup service.
+ * @deprecated Use Confirm instead
+ * @param {ng.IRootScopeService} $root
+ * @param {ng.IQService} $q
+ * @param {mgcrea.ngStrap.modal.IModalService} $modal
+ * @param {ng.animate.IAnimateService} $animate
+ */
+export default function IgniteConfirm($root, $q, $modal, $animate) {
     const scope = $root.$new();
 
     const modal = $modal({templateUrl, scope, show: false, backdrop: true});
@@ -106,4 +113,6 @@ export default ['IgniteConfirm', ['$rootScope', '$q', '$modal', '$animate', ($ro
     };
 
     return modal;
-}]];
+}
+
+IgniteConfirm.$inject = ['$rootScope', '$q', '$modal', '$animate'];
diff --git a/modules/web-console/frontend/app/services/CopyToClipboard.service.js b/modules/web-console/frontend/app/services/CopyToClipboard.service.js
index df0bb8a579ab6..f3c63090a7832 100644
--- a/modules/web-console/frontend/app/services/CopyToClipboard.service.js
+++ b/modules/web-console/frontend/app/services/CopyToClipboard.service.js
@@ -15,10 +15,17 @@
  * limitations under the License.
  */
 
-// Service to copy some value to OS clipboard.
-export default ['IgniteCopyToClipboard', ['$window', 'IgniteMessages', ($window, Messages) => {
+import angular from 'angular';
+
+/**
+ * Service to copy some value to OS clipboard.
+ * @param {ng.IWindowService} $window
+ * @param {ReturnType<typeof import('./Messages.service').default>} Messages
+ */
+export default function factory($window, Messages) {
     const body = angular.element($window.document.body);
 
+    /** @type {JQuery<HTMLTextAreaElement>} */
     const textArea = angular.element('<textarea/>');
 
     textArea.css({
@@ -27,6 +34,9 @@ export default ['IgniteCopyToClipboard', ['$window', 'IgniteMessages', ($window,
     });
 
     return {
+        /**
+         * @param {string} toCopy
+         */
         copy(toCopy) {
             textArea.val(toCopy);
 
@@ -47,4 +57,6 @@ export default ['IgniteCopyToClipboard', ['$window', 'IgniteMessages', ($window,
             textArea.remove();
         }
     };
-}]];
+}
+
+factory.$inject = ['$window', 'IgniteMessages'];
diff --git a/modules/web-console/frontend/app/services/Countries.service.js b/modules/web-console/frontend/app/services/Countries.service.js
index 575951ec136a8..2de671635148e 100644
--- a/modules/web-console/frontend/app/services/Countries.service.js
+++ b/modules/web-console/frontend/app/services/Countries.service.js
@@ -15,17 +15,29 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
 import COUNTRIES from 'app/data/countries.json';
 
-export default ['IgniteCountries', function() {
+/**
+ * @typedef {{label:string,value:string,code:string}} Country
+ */
+
+export default function() {
     const indexByName = _.keyBy(COUNTRIES, 'label');
     const UNDEFINED_COUNTRY = {label: '', value: '', code: ''};
 
+    /**
+     * @param {string} name
+     * @return {Country}
+     */
     const getByName = (name) => (indexByName[name] || UNDEFINED_COUNTRY);
+    /**
+     * @returns {Array<Country>}
+     */
     const getAll = () => (COUNTRIES);
 
     return {
         getByName,
         getAll
     };
-}];
+}
diff --git a/modules/web-console/frontend/app/services/DefaultState.js b/modules/web-console/frontend/app/services/DefaultState.js
index db7b3e3439219..b46bfec8febac 100644
--- a/modules/web-console/frontend/app/services/DefaultState.js
+++ b/modules/web-console/frontend/app/services/DefaultState.js
@@ -15,6 +15,9 @@
  * limitations under the License.
  */
 
+/**
+ * @param {import('@uirouter/angularjs').StateProvider} $stateProvider
+ */
 function DefaultState($stateProvider) {
     const stateName = 'default-state';
 
diff --git a/modules/web-console/frontend/app/services/ErrorParser.service.js b/modules/web-console/frontend/app/services/ErrorParser.service.js
index 9cda8ed1d60b2..88642b9ce096f 100644
--- a/modules/web-console/frontend/app/services/ErrorParser.service.js
+++ b/modules/web-console/frontend/app/services/ErrorParser.service.js
@@ -22,7 +22,7 @@ export default class {
     static $inject = ['JavaTypes'];
 
     /**
-     * @param JavaTypes service.
+     * @param {import('./JavaTypes.service').default} JavaTypes
      */
     constructor(JavaTypes) {
         this.JavaTypes = JavaTypes;
diff --git a/modules/web-console/frontend/app/services/ErrorPopover.service.js b/modules/web-console/frontend/app/services/ErrorPopover.service.js
index bddf43622b81e..3b02a7e7ce4cc 100644
--- a/modules/web-console/frontend/app/services/ErrorPopover.service.js
+++ b/modules/web-console/frontend/app/services/ErrorPopover.service.js
@@ -22,10 +22,10 @@ export default class ErrorPopover {
     static $inject = ['$popover', '$anchorScroll', '$timeout', 'IgniteFormUtils'];
 
     /**
-     * @param $popover
-     * @param $anchorScroll
-     * @param $timeout
-     * @param FormUtils
+     * @param {mgcrea.ngStrap.popover.IPopoverService} $popover
+     * @param {ng.IAnchorScrollService} $anchorScroll
+     * @param {ng.ITimeoutService} $timeout
+     * @param {ReturnType<typeof import('app/services/FormUtils.service').default>} FormUtils
      */
     constructor($popover, $anchorScroll, $timeout, FormUtils) {
         this.$popover = $popover;
@@ -41,7 +41,7 @@ export default class ErrorPopover {
     /**
      * Check that element is document area.
      *
-     * @param el Element to check.
+     * @param {HTMLElement} el Element to check.
      * @returns {boolean} True when element in document area.
      */
     static _isElementInViewport(el) {
@@ -58,8 +58,8 @@ export default class ErrorPopover {
     /**
      * Internal show popover message with detected properties.
      *
-     * @param id Id element to show popover message.
-     * @param message Message to show.
+     * @param {string }id Id element to show popover message.
+     * @param {string} message Message to show.
      * @param showTime Time before popover will be hidden.
      */
     _show(id, message, showTime = 5000) {
diff --git a/modules/web-console/frontend/app/services/Focus.service.js b/modules/web-console/frontend/app/services/Focus.service.js
index a07e18191b714..a152ff112b0b8 100644
--- a/modules/web-console/frontend/app/services/Focus.service.js
+++ b/modules/web-console/frontend/app/services/Focus.service.js
@@ -15,9 +15,15 @@
  * limitations under the License.
  */
 
-// Service to transfer focus for specified element.
-export default ['IgniteFocus', ['$timeout', ($timeout) => {
+/**
+ * Service to transfer focus for specified element.
+ * @param {ng.ITimeoutService} $timeout
+ */
+export default function factory($timeout) {
     return {
+        /**
+         * @param {string} id Element id
+         */
         move(id) {
             // Timeout makes sure that is invoked after any other event has been triggered.
             // E.g. click events that need to run before the focus or inputs elements that are
@@ -30,4 +36,6 @@ export default ['IgniteFocus', ['$timeout', ($timeout) => {
             }, 100);
         }
     };
-}]];
+}
+
+factory.$inject = ['$timeout'];
diff --git a/modules/web-console/frontend/app/services/FormUtils.service.js b/modules/web-console/frontend/app/services/FormUtils.service.js
index 2c81c5740da92..c8ce4b04f448d 100644
--- a/modules/web-console/frontend/app/services/FormUtils.service.js
+++ b/modules/web-console/frontend/app/services/FormUtils.service.js
@@ -16,7 +16,12 @@
  */
 import _ from 'lodash';
 
-export default ['IgniteFormUtils', ['$window', 'IgniteFocus', '$rootScope', ($window, Focus, $rootScope) => {
+/**
+ * @param {ng.IWindowService} $window
+ * @param {ReturnType<typeof import('./Focus.service').default>} Focus
+ * @param {ng.IRootScopeService} $rootScope
+ */
+export default function service($window, Focus, $rootScope) {
     function ensureActivePanel(ui, pnl, focusId) {
         if (ui && ui.loadPanel) {
             const collapses = $('[bs-collapse-target]');
@@ -55,12 +60,13 @@ export default ['IgniteFormUtils', ['$window', 'IgniteFocus', '$rootScope', ($wi
         }
     }
 
+    /** @type {CanvasRenderingContext2D} */
     let context = null;
 
     /**
      * Calculate width of specified text in body's font.
      *
-     * @param text Text to calculate width.
+     * @param {string} text Text to calculate width.
      * @returns {Number} Width of text in pixels.
      */
     function measureText(text) {
@@ -427,14 +433,6 @@ export default ['IgniteFormUtils', ['$window', 'IgniteFocus', '$rootScope', ($wi
         ensureActivePanel(panels, id, focusId) {
             ensureActivePanel(panels, id, focusId);
         },
-        confirmUnsavedChanges(dirty, selectFunc) {
-            if (dirty) {
-                if ($window.confirm('You have unsaved changes.\n\nAre you sure you want to discard them?'))
-                    selectFunc();
-            }
-            else
-                selectFunc();
-        },
         saveBtnTipText(dirty, objectName) {
             if (dirty)
                 return 'Save ' + objectName;
@@ -461,4 +459,6 @@ export default ['IgniteFormUtils', ['$window', 'IgniteFocus', '$rootScope', ($wi
         },
         triggerValidation
     };
-}]];
+}
+
+service.$inject = ['$window', 'IgniteFocus', '$rootScope'];
diff --git a/modules/web-console/frontend/app/services/IGFSs.js b/modules/web-console/frontend/app/services/IGFSs.js
index de9e43d253b36..c9bde77aaf840 100644
--- a/modules/web-console/frontend/app/services/IGFSs.js
+++ b/modules/web-console/frontend/app/services/IGFSs.js
@@ -29,10 +29,16 @@ export default class IGFSs {
         {value: 'DUAL_ASYNC', label: 'DUAL_ASYNC'}
     ];
 
+    /**
+     * @param {ng.IHttpService} $http
+     */
     constructor($http) {
-        Object.assign(this, {$http});
+        this.$http = $http;
     }
 
+    /**
+     * @param {string} igfsID
+     */
     getIGFS(igfsID) {
         return this.$http.get(`/api/v1/configuration/igfs/${igfsID}`);
     }
diff --git a/modules/web-console/frontend/app/services/InetAddress.service.js b/modules/web-console/frontend/app/services/InetAddress.service.js
index abdd8a363d02a..1f50bdfae0e68 100644
--- a/modules/web-console/frontend/app/services/InetAddress.service.js
+++ b/modules/web-console/frontend/app/services/InetAddress.service.js
@@ -15,7 +15,9 @@
  * limitations under the License.
  */
 
-export default ['IgniteInetAddress', function() {
+import _ from 'lodash';
+
+export default function() {
     return {
         /**
          * @param {String} ip IP address to check.
@@ -36,18 +38,18 @@ export default ['IgniteInetAddress', function() {
             return regexp.test(hostNameOrIp) || this.validIp(hostNameOrIp);
         },
         /**
-         * @param {int} port Port value to check.
+         * @param {number} port Port value to check.
          * @returns boolean 'true' if given port is valid tcp/udp port range.
          */
         validPort(port) {
             return _.isInteger(port) && port > 0 && port <= 65535;
         },
         /**
-         * @param {int} port Port value to check.
+         * @param {number} port Port value to check.
          * @returns {boolean} 'true' if given port in non system port range(user+dynamic).
          */
         validNonSystemPort(port) {
             return _.isInteger(port) && port >= 1024 && port <= 65535;
         }
     };
-}];
+}
diff --git a/modules/web-console/frontend/app/services/LegacyTable.service.js b/modules/web-console/frontend/app/services/LegacyTable.service.js
index 38b041a0ada38..3086f4fb50654 100644
--- a/modules/web-console/frontend/app/services/LegacyTable.service.js
+++ b/modules/web-console/frontend/app/services/LegacyTable.service.js
@@ -16,214 +16,220 @@
  */
 
 // TODO: Refactor this service for legacy tables with more than one input field.
-export default ['IgniteLegacyTable',
-    ['IgniteLegacyUtils', 'IgniteFocus', 'IgniteErrorPopover', (LegacyUtils, Focus, ErrorPopover) => {
-        function _model(item, field) {
-            let path = field.path;
-
-            if (_.isNil(path) || _.isNil(item))
-                return item;
+/**
+ * @param {ReturnType<typeof import('./LegacyUtils.service').default>} LegacyUtils
+ * @param {ReturnType<typeof import('./Focus.service').default>} Focus
+ * @param {import('./ErrorPopover.service').default} ErrorPopover
+ */
+export default function service(LegacyUtils, Focus, ErrorPopover) {
+    function _model(item, field) {
+        let path = field.path;
 
-            path = path.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
-            path = path.replace(/^\./, ''); // strip a leading dot
+        if (_.isNil(path) || _.isNil(item))
+            return item;
 
-            const segs = path.split('.');
-            let root = item;
+        path = path.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties
+        path = path.replace(/^\./, ''); // strip a leading dot
 
-            while (segs.length > 0) {
-                const pathStep = segs.shift();
+        const segs = path.split('.');
+        let root = item;
 
-                if (typeof root[pathStep] === 'undefined')
-                    root[pathStep] = {};
+        while (segs.length > 0) {
+            const pathStep = segs.shift();
 
-                root = root[pathStep];
-            }
+            if (typeof root[pathStep] === 'undefined')
+                root[pathStep] = {};
 
-            return root;
+            root = root[pathStep];
         }
 
-        const table = {name: 'none', editIndex: -1};
+        return root;
+    }
 
-        function _tableReset() {
-            delete table.field;
-            table.name = 'none';
-            table.editIndex = -1;
+    const table = {name: 'none', editIndex: -1};
 
-            ErrorPopover.hide();
-        }
+    function _tableReset() {
+        delete table.field;
+        table.name = 'none';
+        table.editIndex = -1;
 
-        function _tableSaveAndReset() {
-            const field = table.field;
+        ErrorPopover.hide();
+    }
 
-            const save = LegacyUtils.isDefined(field) && LegacyUtils.isDefined(field.save);
+    function _tableSaveAndReset() {
+        const field = table.field;
 
-            if (!save || !LegacyUtils.isDefined(field) || field.save(field, table.editIndex, true)) {
-                _tableReset();
+        const save = LegacyUtils.isDefined(field) && LegacyUtils.isDefined(field.save);
 
-                return true;
-            }
+        if (!save || !LegacyUtils.isDefined(field) || field.save(field, table.editIndex, true)) {
+            _tableReset();
 
-            return false;
+            return true;
         }
 
-        function _tableState(field, editIndex, specName) {
-            table.field = field;
-            table.name = specName || field.model;
-            table.editIndex = editIndex;
-        }
+        return false;
+    }
 
-        function _tableUI(tbl) {
-            const ui = tbl.ui;
+    function _tableState(field, editIndex, specName) {
+        table.field = field;
+        table.name = specName || field.model;
+        table.editIndex = editIndex;
+    }
 
-            return ui ? ui : tbl.type;
-        }
+    function _tableUI(tbl) {
+        const ui = tbl.ui;
 
-        function _tableFocus(focusId, index) {
-            Focus.move((index < 0 ? 'new' : 'cur') + focusId + (index >= 0 ? index : ''));
-        }
+        return ui ? ui : tbl.type;
+    }
 
-        function _tablePairValue(filed, index) {
-            return index < 0 ? {key: filed.newKey, value: filed.newValue} : {
-                key: filed.curKey,
-                value: filed.curValue
-            };
-        }
+    function _tableFocus(focusId, index) {
+        Focus.move((index < 0 ? 'new' : 'cur') + focusId + (index >= 0 ? index : ''));
+    }
 
-        function _tableStartEdit(item, tbl, index, save) {
-            _tableState(tbl, index);
+    function _tablePairValue(filed, index) {
+        return index < 0 ? {key: filed.newKey, value: filed.newValue} : {
+            key: filed.curKey,
+            value: filed.curValue
+        };
+    }
 
-            const val = _.get(_model(item, tbl), tbl.model)[index];
+    function _tableStartEdit(item, tbl, index, save) {
+        _tableState(tbl, index);
 
-            const ui = _tableUI(tbl);
+        const val = _.get(_model(item, tbl), tbl.model)[index];
 
-            tbl.save = save;
+        const ui = _tableUI(tbl);
 
-            if (ui === 'table-pair') {
-                tbl.curKey = val[tbl.keyName];
-                tbl.curValue = val[tbl.valueName];
+        tbl.save = save;
 
-                _tableFocus('Key' + tbl.focusId, index);
-            }
-            else if (ui === 'table-db-fields') {
-                tbl.curDatabaseFieldName = val.databaseFieldName;
-                tbl.curDatabaseFieldType = val.databaseFieldType;
-                tbl.curJavaFieldName = val.javaFieldName;
-                tbl.curJavaFieldType = val.javaFieldType;
+        if (ui === 'table-pair') {
+            tbl.curKey = val[tbl.keyName];
+            tbl.curValue = val[tbl.valueName];
 
-                _tableFocus('DatabaseFieldName' + tbl.focusId, index);
-            }
-            else if (ui === 'table-indexes') {
-                tbl.curIndexName = val.name;
-                tbl.curIndexType = val.indexType;
-                tbl.curIndexFields = val.fields;
+            _tableFocus('Key' + tbl.focusId, index);
+        }
+        else if (ui === 'table-db-fields') {
+            tbl.curDatabaseFieldName = val.databaseFieldName;
+            tbl.curDatabaseFieldType = val.databaseFieldType;
+            tbl.curJavaFieldName = val.javaFieldName;
+            tbl.curJavaFieldType = val.javaFieldType;
 
-                _tableFocus(tbl.focusId, index);
-            }
+            _tableFocus('DatabaseFieldName' + tbl.focusId, index);
         }
+        else if (ui === 'table-indexes') {
+            tbl.curIndexName = val.name;
+            tbl.curIndexType = val.indexType;
+            tbl.curIndexFields = val.fields;
 
-        function _tableNewItem(tbl) {
-            _tableState(tbl, -1);
+            _tableFocus(tbl.focusId, index);
+        }
+    }
 
-            const ui = _tableUI(tbl);
+    function _tableNewItem(tbl) {
+        _tableState(tbl, -1);
 
-            if (ui === 'table-pair') {
-                tbl.newKey = null;
-                tbl.newValue = null;
+        const ui = _tableUI(tbl);
 
-                _tableFocus('Key' + tbl.focusId, -1);
-            }
-            else if (ui === 'table-db-fields') {
-                tbl.newDatabaseFieldName = null;
-                tbl.newDatabaseFieldType = null;
-                tbl.newJavaFieldName = null;
-                tbl.newJavaFieldType = null;
+        if (ui === 'table-pair') {
+            tbl.newKey = null;
+            tbl.newValue = null;
 
-                _tableFocus('DatabaseFieldName' + tbl.focusId, -1);
-            }
-            else if (ui === 'table-indexes') {
-                tbl.newIndexName = null;
-                tbl.newIndexType = 'SORTED';
-                tbl.newIndexFields = null;
+            _tableFocus('Key' + tbl.focusId, -1);
+        }
+        else if (ui === 'table-db-fields') {
+            tbl.newDatabaseFieldName = null;
+            tbl.newDatabaseFieldType = null;
+            tbl.newJavaFieldName = null;
+            tbl.newJavaFieldType = null;
 
-                _tableFocus(tbl.focusId, -1);
-            }
+            _tableFocus('DatabaseFieldName' + tbl.focusId, -1);
         }
+        else if (ui === 'table-indexes') {
+            tbl.newIndexName = null;
+            tbl.newIndexType = 'SORTED';
+            tbl.newIndexFields = null;
 
-        return {
-            tableState: _tableState,
-            tableReset: _tableReset,
-            tableSaveAndReset: _tableSaveAndReset,
-            tableNewItem: _tableNewItem,
-            tableNewItemActive(tbl) {
-                return table.name === tbl.model && table.editIndex < 0;
-            },
-            tableEditing(tbl, index) {
-                return table.name === tbl.model && table.editIndex === index;
-            },
-            tableEditedRowIndex() {
-                return table.editIndex;
-            },
-            tableField() {
-                return table.field;
-            },
-            tableStartEdit: _tableStartEdit,
-            tableRemove(item, field, index) {
-                _tableReset();
-
-                _.get(_model(item, field), field.model).splice(index, 1);
-            },
-            tablePairValue: _tablePairValue,
-            tablePairSave(pairValid, item, field, index, stopEdit) {
-                const valid = pairValid(item, field, index, stopEdit);
-
-                if (valid) {
-                    const pairValue = _tablePairValue(field, index);
-
-                    let pairModel = {};
-
-                    const container = _.get(item, field.model);
-
-                    if (index < 0) {
-                        pairModel[field.keyName] = pairValue.key;
-                        pairModel[field.valueName] = pairValue.value;
-
-                        if (container)
-                            container.push(pairModel);
-                        else
-                            _.set(item, field.model, [pairModel]);
+            _tableFocus(tbl.focusId, -1);
+        }
+    }
+
+    return {
+        tableState: _tableState,
+        tableReset: _tableReset,
+        tableSaveAndReset: _tableSaveAndReset,
+        tableNewItem: _tableNewItem,
+        tableNewItemActive(tbl) {
+            return table.name === tbl.model && table.editIndex < 0;
+        },
+        tableEditing(tbl, index) {
+            return table.name === tbl.model && table.editIndex === index;
+        },
+        tableEditedRowIndex() {
+            return table.editIndex;
+        },
+        tableField() {
+            return table.field;
+        },
+        tableStartEdit: _tableStartEdit,
+        tableRemove(item, field, index) {
+            _tableReset();
+
+            _.get(_model(item, field), field.model).splice(index, 1);
+        },
+        tablePairValue: _tablePairValue,
+        tablePairSave(pairValid, item, field, index, stopEdit) {
+            const valid = pairValid(item, field, index, stopEdit);
+
+            if (valid) {
+                const pairValue = _tablePairValue(field, index);
+
+                let pairModel = {};
+
+                const container = _.get(item, field.model);
+
+                if (index < 0) {
+                    pairModel[field.keyName] = pairValue.key;
+                    pairModel[field.valueName] = pairValue.value;
+
+                    if (container)
+                        container.push(pairModel);
+                    else
+                        _.set(item, field.model, [pairModel]);
+
+                    if (!stopEdit)
+                        _tableNewItem(field);
+                }
+                else {
+                    pairModel = container[index];
+
+                    pairModel[field.keyName] = pairValue.key;
+                    pairModel[field.valueName] = pairValue.value;
 
-                        if (!stopEdit)
+                    if (!stopEdit) {
+                        if (index < container.length - 1)
+                            _tableStartEdit(item, field, index + 1);
+                        else
                             _tableNewItem(field);
                     }
-                    else {
-                        pairModel = container[index];
-
-                        pairModel[field.keyName] = pairValue.key;
-                        pairModel[field.valueName] = pairValue.value;
-
-                        if (!stopEdit) {
-                            if (index < container.length - 1)
-                                _tableStartEdit(item, field, index + 1);
-                            else
-                                _tableNewItem(field);
-                        }
-                    }
                 }
+            }
 
-                return valid || stopEdit;
-            },
-            tablePairSaveVisible(field, index) {
-                const pairValue = _tablePairValue(field, index);
+            return valid || stopEdit;
+        },
+        tablePairSaveVisible(field, index) {
+            const pairValue = _tablePairValue(field, index);
 
-                return !LegacyUtils.isEmptyString(pairValue.key) && !LegacyUtils.isEmptyString(pairValue.value);
-            },
-            tableFocusInvalidField(index, id) {
-                _tableFocus(id, index);
+            return !LegacyUtils.isEmptyString(pairValue.key) && !LegacyUtils.isEmptyString(pairValue.value);
+        },
+        tableFocusInvalidField(index, id) {
+            _tableFocus(id, index);
 
-                return false;
-            },
-            tableFieldId(index, id) {
-                return (index < 0 ? 'new' : 'cur') + id + (index >= 0 ? index : '');
-            }
-        };
-    }]];
+            return false;
+        },
+        tableFieldId(index, id) {
+            return (index < 0 ? 'new' : 'cur') + id + (index >= 0 ? index : '');
+        }
+    };
+}
+
+service.$inject = ['IgniteLegacyUtils', 'IgniteFocus', 'IgniteErrorPopover'];
diff --git a/modules/web-console/frontend/app/services/LegacyUtils.service.js b/modules/web-console/frontend/app/services/LegacyUtils.service.js
index a16934354f431..8362669af088b 100644
--- a/modules/web-console/frontend/app/services/LegacyUtils.service.js
+++ b/modules/web-console/frontend/app/services/LegacyUtils.service.js
@@ -16,9 +16,13 @@
  */
 
 import saver from 'file-saver';
+import _ from 'lodash';
 
 // TODO: Refactor this service for legacy tables with more than one input field.
-export default ['IgniteLegacyUtils', ['IgniteErrorPopover', (ErrorPopover) => {
+/**
+ * @param {import('./ErrorPopover.service').default} ErrorPopover
+ */
+export default function service(ErrorPopover) {
     function isDefined(v) {
         return !_.isNil(v);
     }
@@ -547,4 +551,6 @@ export default ['IgniteLegacyUtils', ['IgniteErrorPopover', (ErrorPopover) => {
             return true;
         }
     };
-}]];
+}
+
+service.$inject = ['IgniteErrorPopover'];
diff --git a/modules/web-console/frontend/app/services/Messages.service.js b/modules/web-console/frontend/app/services/Messages.service.js
index 1337e24c1e0f6..05c058b9cb45c 100644
--- a/modules/web-console/frontend/app/services/Messages.service.js
+++ b/modules/web-console/frontend/app/services/Messages.service.js
@@ -17,8 +17,12 @@
 
 import {CancellationError} from 'app/errors/CancellationError';
 
-// Service to show various information and error messages.
-export default ['IgniteMessages', ['$alert', 'IgniteErrorParser', ($alert, errorParser) => {
+/**
+ * Service to show various information and error messages.
+ * @param {mgcrea.ngStrap.alert.IAlertService} $alert
+ * @param {import('./ErrorParser.service').default} errorParser
+ */
+export default function factory($alert, errorParser) {
     // Common instance of alert modal.
     let msgModal;
 
@@ -47,6 +51,10 @@ export default ['IgniteMessages', ['$alert', 'IgniteErrorParser', ($alert, error
     return {
         errorMessage,
         hideAlert,
+        /**
+         * @param {string|CancellationError} message
+         * @param [err]
+         */
         showError(message, err, duration = 10) {
             if (message instanceof CancellationError)
                 return false;
@@ -55,8 +63,13 @@ export default ['IgniteMessages', ['$alert', 'IgniteErrorParser', ($alert, error
 
             return false;
         },
+        /**
+         * @param {string} message
+         */
         showInfo(message, duration = 5) {
             _showMessage(message, null, 'success', duration);
         }
     };
-}]];
+}
+
+factory.$inject = ['$alert', 'IgniteErrorParser'];
diff --git a/modules/web-console/frontend/app/services/ModelNormalizer.service.js b/modules/web-console/frontend/app/services/ModelNormalizer.service.js
index 4c7052b75774b..fbbff9b6e2b21 100644
--- a/modules/web-console/frontend/app/services/ModelNormalizer.service.js
+++ b/modules/web-console/frontend/app/services/ModelNormalizer.service.js
@@ -15,8 +15,10 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
 // Service to normalize objects for dirty checks.
-export default ['IgniteModelNormalizer', () => {
+export default function() {
     /**
      * Normalize object for dirty checks.
      *
@@ -56,4 +58,4 @@ export default ['IgniteModelNormalizer', () => {
             return _.isEqual(prev, normalize(cur));
         }
     };
-}];
+}
diff --git a/modules/web-console/frontend/app/services/SqlTypes.service.js b/modules/web-console/frontend/app/services/SqlTypes.service.js
index e42d903fd3a2c..389d8e7c0d9e8 100644
--- a/modules/web-console/frontend/app/services/SqlTypes.service.js
+++ b/modules/web-console/frontend/app/services/SqlTypes.service.js
@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import _ from 'lodash';
+
 // List of H2 reserved SQL keywords.
 import H2_SQL_KEYWORDS from 'app/data/sql-keywords.json';
 
@@ -36,7 +38,7 @@ const UNKNOWN_JDBC_TYPE = {
  */
 export default class SqlTypes {
     /**
-     * @param value {String} Value to check.
+     * @param {String} value Value to check.
      * @returns {boolean} 'true' if given text is valid Java class name.
      */
     validIdentifier(value) {
@@ -54,7 +56,7 @@ export default class SqlTypes {
     /**
      * Find JDBC type descriptor for specified JDBC type and options.
      *
-     * @param dbType {Number} Column db type.
+     * @param {Number} dbType  Column db type.
      * @return {String} Java type.
      */
     findJdbcType(dbType) {
diff --git a/modules/web-console/frontend/app/services/UnsavedChangesGuard.service.js b/modules/web-console/frontend/app/services/UnsavedChangesGuard.service.js
deleted file mode 100644
index 91244b0249876..0000000000000
--- a/modules/web-console/frontend/app/services/UnsavedChangesGuard.service.js
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const MSG = 'You have unsaved changes.\n\nAre you sure you want to discard them?';
-
-// Service that show confirmation about unsaved changes on user change location.
-export default ['IgniteUnsavedChangesGuard', ['$rootScope', ($root) => {
-    return {
-        install(scope, customDirtyCheck = () => scope.ui.inputForm.$dirty) {
-            scope.$on('$destroy', () => window.onbeforeunload = null);
-
-            const unbind = $root.$on('$stateChangeStart', (event) => {
-                if (_.get(scope, 'ui.inputForm', false) && customDirtyCheck()) {
-                    if (!confirm(MSG)) // eslint-disable-line no-alert
-                        event.preventDefault();
-                    else
-                        unbind();
-                }
-            });
-
-            window.onbeforeunload = () => _.get(scope, 'ui.inputForm.$dirty', false) ? MSG : null;
-        }
-    };
-}]];
diff --git a/modules/web-console/frontend/app/services/Version.service.js b/modules/web-console/frontend/app/services/Version.service.js
index 33de64de4e243..c40073aced1c9 100644
--- a/modules/web-console/frontend/app/services/Version.service.js
+++ b/modules/web-console/frontend/app/services/Version.service.js
@@ -49,8 +49,8 @@ const numberComparator = (a, b) => a > b ? 1 : a < b ? -1 : 0;
 
 /**
  * Compare to version.
- * @param a {Object} first compared version.
- * @param b {Object} second compared version.
+ * @param {Object} a first compared version.
+ * @param {Object} b second compared version.
  * @returns {Number} 1 if a > b, 0 if versions equals, -1 if a < b
  */
 const compare = (a, b) => {
@@ -74,9 +74,13 @@ const compare = (a, b) => {
 
 export default class IgniteVersion {
     constructor() {
-        this.webConsole = '2.5.0';
+        this.webConsole = '2.6.0';
 
         this.supportedVersions = [
+            {
+                label: 'Ignite 2.6',
+                ignite: '2.6.0'
+            },
             {
                 label: 'Ignite 2.5',
                 ignite: '2.5.0'
diff --git a/modules/web-console/frontend/app/services/exceptionHandler.js b/modules/web-console/frontend/app/services/exceptionHandler.js
index 82a88e15be481..c6a09c3324495 100644
--- a/modules/web-console/frontend/app/services/exceptionHandler.js
+++ b/modules/web-console/frontend/app/services/exceptionHandler.js
@@ -17,6 +17,9 @@
 
 import {CancellationError} from 'app/errors/CancellationError';
 
+/**
+ * @param {ng.ILogService} $log
+ */
 export function $exceptionHandler($log) {
     return function(exception, cause) {
         if (exception instanceof CancellationError)
diff --git a/modules/web-console/frontend/app/style.scss b/modules/web-console/frontend/app/style.scss
new file mode 100644
index 0000000000000..c7eb726c1ac44
--- /dev/null
+++ b/modules/web-console/frontend/app/style.scss
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.flex-full-height > [ui-view] {
+	display: flex;
+    flex-direction: column;
+    flex: 1 0 auto;
+}
diff --git a/modules/web-console/frontend/app/vendor.js b/modules/web-console/frontend/app/vendor.js
index 84dea92ee812c..6961e962e30b2 100644
--- a/modules/web-console/frontend/app/vendor.js
+++ b/modules/web-console/frontend/app/vendor.js
@@ -15,7 +15,6 @@
  * limitations under the License.
  */
 
-import 'babel-polyfill';
 import 'jquery';
 import 'angular';
 import 'angular-acl';
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 44bcd6f1bd951..0aef839a2dfb8 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -5,9 +5,9 @@
   "private": true,
   "main": "index.js",
   "scripts": {
-    "start": "webpack-dev-server --config ./webpack/webpack.dev.babel.js",
+    "start": "webpack-dev-server --config ./webpack/webpack.dev.js",
     "dev": "npm start",
-    "build": "webpack --config ./webpack/webpack.prod.babel.js",
+    "build": "webpack --config ./webpack/webpack.prod.js",
     "test": "karma start ./test/karma.conf.js",
     "test-watch": "npm test -- --no-single-run",
     "eslint": "eslint --format node_modules/eslint-friendly-formatter app/ -- --eff-by-issue"
@@ -27,86 +27,89 @@
     "win32"
   ],
   "dependencies": {
-    "@uirouter/angularjs": "1.0.18",
+    "@babel/plugin-transform-parameters": "7.0.0",
+    "@types/angular-translate": "2.16.0",
+    "@uirouter/angularjs": "1.0.20",
     "@uirouter/core": "5.0.19",
     "@uirouter/rx": "0.4.1",
     "@uirouter/visualizer": "4.0.2",
-    "angular": "1.7.2",
+    "angular": "1.7.4",
     "angular-acl": "0.1.10",
-    "angular-animate": "1.7.2",
-    "angular-aria": "1.6.6",
-    "angular-cookies": "1.7.2",
+    "angular-animate": "1.7.4",
+    "angular-aria": "1.7.4",
+    "angular-cookies": "1.7.4",
     "angular-drag-and-drop-lists": "1.4.0",
     "angular-gridster": "0.13.14",
-    "angular-messages": "1.6.9",
+    "angular-messages": "1.7.4",
     "angular-motion": "0.4.4",
     "angular-nvd3": "1.0.9",
-    "angular-sanitize": "1.7.2",
-    "angular-smart-table": "2.1.8",
+    "angular-sanitize": "1.7.4",
+    "angular-smart-table": "2.1.11",
     "angular-socket-io": "0.7.0",
     "angular-strap": "2.3.12",
-    "angular-translate": "2.16.0",
+    "angular-translate": "2.18.1",
     "angular-tree-control": "0.2.28",
     "angular-ui-carousel": "0.1.10",
     "angular-ui-grid": "4.6.1",
     "angular-ui-validate": "1.2.3",
     "angular1-async-filter": "1.1.0",
     "brace": "0.10.0",
-    "browser-update": "2.1.9",
+    "browser-update": "3.1.13",
     "bson-objectid": "1.1.5",
+    "chart.js": "2.7.2",
+    "chartjs-plugin-streaming": "1.6.1",
     "file-saver": "1.3.3",
     "font-awesome": "4.7.0",
     "jquery": "3.2.1",
     "json-bigint": "0.2.3",
     "jsondiffpatch": "0.2.5",
     "jszip": "3.1.5",
-    "lodash": "4.17.10",
+    "lodash": "4.17.11",
     "natural-compare-lite": "1.4.0",
     "nvd3": "1.8.6",
     "outdent": "0.5.0",
     "pako": "1.0.6",
     "roboto-font": "0.1.0",
-    "rxjs": "5.4.2",
+    "rxjs": "5.5.11",
     "socket.io-client": "1.7.3",
     "tf-metatags": "2.0.0"
   },
   "devDependencies": {
-    "@types/angular": "1.6.48",
+    "@babel/core": "7.0.1",
+    "@babel/plugin-proposal-class-properties": "7.0.0",
+    "@babel/plugin-proposal-object-rest-spread": "7.0.0",
+    "@babel/plugin-syntax-dynamic-import": "7.0.0",
+    "@babel/preset-env": "7.0.0",
+    "@babel/preset-typescript": "7.1.0",
+    "@types/angular": "1.6.51",
     "@types/angular-animate": "1.5.10",
     "@types/angular-mocks": "1.5.12",
     "@types/angular-strap": "2.3.1",
     "@types/chai": "4.1.4",
-    "@types/copy-webpack-plugin": "4.4.1",
+    "@types/copy-webpack-plugin": "4.4.2",
     "@types/karma": "1.7.4",
     "@types/lodash": "4.14.110",
     "@types/mini-css-extract-plugin": "0.2.0",
     "@types/mocha": "2.2.48",
+    "@types/node": "10.5.1",
     "@types/sinon": "4.0.0",
     "@types/socket.io-client": "1.4.32",
     "@types/ui-grid": "0.0.38",
-    "@types/webpack": "4.4.5",
+    "@types/webpack": "4.4.11",
     "@types/webpack-merge": "4.1.3",
-    "angular-mocks": "1.6.9",
+    "angular-mocks": "1.7.4",
     "app-root-path": "2.0.1",
-    "babel-core": "6.26.0",
-    "babel-eslint": "7.2.3",
-    "babel-loader": "7.1.4",
-    "babel-plugin-add-module-exports": "0.2.1",
-    "babel-plugin-transform-object-rest-spread": "6.26.0",
-    "babel-plugin-transform-runtime": "6.23.0",
-    "babel-polyfill": "6.26.0",
-    "babel-preset-es2015": "6.24.1",
-    "babel-preset-stage-1": "6.24.1",
-    "babel-runtime": "6.26.0",
+    "babel-loader": "8.0.2",
     "bootstrap-sass": "3.3.7",
     "chai": "4.1.0",
     "chalk": "2.1.0",
     "copy-webpack-plugin": "4.5.2",
     "css-loader": "0.28.7",
-    "eslint": "4.3.0",
-    "eslint-friendly-formatter": "3.0.0",
-    "eslint-loader": "1.9.0",
-    "eslint-plugin-babel": "4.1.1",
+    "eslint": "4.19.1",
+    "eslint-formatter-friendly": "6.0.0",
+    "eslint-loader": "2.1.0",
+    "eslint-plugin-babel": "5.2.0",
+    "eslint-plugin-typescript": "0.12.0",
     "expose-loader": "0.7.5",
     "file-loader": "1.1.11",
     "glob": "7.1.2",
@@ -117,17 +120,17 @@
     "jasmine-core": "2.6.4",
     "json-loader": "0.5.7",
     "karma": "2.0.0",
-    "karma-babel-preprocessor": "6.0.1",
+    "karma-babel-preprocessor": "8.0.0-beta.0",
     "karma-chrome-launcher": "2.2.0",
     "karma-mocha": "1.3.0",
     "karma-mocha-reporter": "2.2.3",
     "karma-teamcity-reporter": "1.0.0",
-    "karma-webpack": "4.0.0-beta.0",
-    "mini-css-extract-plugin": "0.4.1",
+    "karma-webpack": "4.0.0-rc.2",
+    "mini-css-extract-plugin": "0.4.2",
     "mocha": "3.4.2",
     "mocha-teamcity-reporter": "1.1.1",
     "node-fetch": "1.7.3",
-    "node-sass": "4.9.0",
+    "node-sass": "4.9.3",
     "progress": "2.0.0",
     "progress-bar-webpack-plugin": "1.11.0",
     "pug-html-loader": "1.1.0",
@@ -137,14 +140,15 @@
     "sinon": "2.3.8",
     "slash": "1.0.0",
     "style-loader": "0.19.0",
-    "svg-sprite-loader": "3.9.0",
+    "svg-sprite-loader": "3.9.2",
     "teamcity-service-messages": "0.1.9",
-    "type-detect": "4.0.3",
-    "uglifyjs-webpack-plugin": "1.2.4",
-    "webpack": "4.12.0",
-    "webpack-cli": "2.0.14",
-    "webpack-dev-server": "3.1.4",
-    "webpack-merge": "4.1.3",
+    "typescript": "3.0.3",
+    "typescript-eslint-parser": "19.0.1",
+    "uglifyjs-webpack-plugin": "1.3.0",
+    "webpack": "4.19.1",
+    "webpack-cli": "3.1.0",
+    "webpack-dev-server": "3.1.8",
+    "webpack-merge": "4.1.4",
     "worker-loader": "2.0.0",
     "yargs": "9.0.1"
   }
diff --git a/modules/web-console/frontend/public/images/icons/index.js b/modules/web-console/frontend/public/images/icons/index.js
index 24ffac14e8ab6..0951d32550bc8 100644
--- a/modules/web-console/frontend/public/images/icons/index.js
+++ b/modules/web-console/frontend/public/images/icons/index.js
@@ -15,31 +15,31 @@
  * limitations under the License.
  */
 
-export alert from './alert.icon.svg';
-export attention from './attention.icon.svg';
-export check from './check.icon.svg';
-export checkmark from './checkmark.icon.svg';
-export clock from './clock.icon.svg';
-export collapse from './collapse.icon.svg';
-export connectedClusters from './connectedClusters.icon.svg';
-export copy from './copy.icon.svg';
-export cross from './cross.icon.svg';
-export csv from './csv.icon.svg';
-export download from './download.icon.svg';
-export exclamation from './exclamation.icon.svg';
-export exit from './exit.icon.svg';
-export expand from './expand.icon.svg';
-export eyeClosed from './eyeClosed.icon.svg';
-export eyeOpened from './eyeOpened.icon.svg';
-export filter from './filter.icon.svg';
-export gear from './gear.icon.svg';
-export home from './home.icon.svg';
-export info from './info.icon.svg';
-export lockClosed from './lockClosed.icon.svg';
-export lockOpened from './lockOpened.icon.svg';
-export manual from './manual.icon.svg';
-export plus from './plus.icon.svg';
-export refresh from './refresh.icon.svg';
-export search from './search.icon.svg';
-export sort from './sort.icon.svg';
-export structure from './structure.icon.svg';
+export {default as alert} from './alert.icon.svg';
+export {default as attention} from './attention.icon.svg';
+export {default as check} from './check.icon.svg';
+export {default as checkmark} from './checkmark.icon.svg';
+export {default as clock} from './clock.icon.svg';
+export {default as collapse} from './collapse.icon.svg';
+export {default as connectedClusters} from './connectedClusters.icon.svg';
+export {default as copy} from './copy.icon.svg';
+export {default as cross} from './cross.icon.svg';
+export {default as csv} from './csv.icon.svg';
+export {default as download} from './download.icon.svg';
+export {default as exclamation} from './exclamation.icon.svg';
+export {default as exit} from './exit.icon.svg';
+export {default as expand} from './expand.icon.svg';
+export {default as eyeClosed} from './eyeClosed.icon.svg';
+export {default as eyeOpened} from './eyeOpened.icon.svg';
+export {default as filter} from './filter.icon.svg';
+export {default as gear} from './gear.icon.svg';
+export {default as home} from './home.icon.svg';
+export {default as info} from './info.icon.svg';
+export {default as lockClosed} from './lockClosed.icon.svg';
+export {default as lockOpened} from './lockOpened.icon.svg';
+export {default as manual} from './manual.icon.svg';
+export {default as plus} from './plus.icon.svg';
+export {default as refresh} from './refresh.icon.svg';
+export {default as search} from './search.icon.svg';
+export {default as sort} from './sort.icon.svg';
+export {default as structure} from './structure.icon.svg';
diff --git a/modules/web-console/frontend/public/stylesheets/style.scss b/modules/web-console/frontend/public/stylesheets/style.scss
index be288dc924d5b..a00a5857ba1bc 100644
--- a/modules/web-console/frontend/public/stylesheets/style.scss
+++ b/modules/web-console/frontend/public/stylesheets/style.scss
@@ -496,62 +496,40 @@ button.form-control {
         border-top: 1px solid $ignite-border-color;
     }
 
+    .btn-ignite-group {
+        padding: 0;
+        border: none;
+        margin-right: 0;
+        background: transparent;
+    }
+
     .sql-controls {
-        position: relative;
-        top: -1px;
+        display: flex;
+        justify-content: space-between;
         border-top: 1px solid #ddd;
 
-        padding: 10px 10px;
-
-        input[type="checkbox"] {
-            line-height: 20px;
-            margin-right: 5px;
-        }
+        & > div {
+            display: flex;
+            padding: 10px;
+            align-items: flex-start;
 
-        label {
-            line-height: 28px;
-            vertical-align: middle;
-        }
-
-        .btn {
-            line-height: 20px;
-        }
-
-        .ignite-form-field {
-            margin-right: 10px;
-
-            .ignite-form-field__label {
-                float: left;
-                width: auto;
-                margin-right: 5px;
-                line-height: 28px;
+            &:nth-child(2) {
+                flex: 1;
+                justify-content: flex-end;
             }
 
-            .ignite-form-field__label + div {
-                display: block;
-                float: none;
-                width: auto;
+            &:last-child {
+                flex-direction: column;
+                flex-basis: 25%;
             }
         }
 
-        .tipLabel .btn {
-            float: right;
+        button + button {
+            margin-left: 20px;
         }
 
-        .pull-right {
-            margin-left: 10px;
-
-            .ignite-form-field {
-                margin-right: -24px;
-
-                label {
-                    margin-left: 5px;
-                }
-            }
-        }
-
-        .col-sm-3 + .tipLabel {
-            margin-left: 0;
+        .form-field--inline + .form-field--inline {
+            margin-left: 20px;
         }
     }
 
@@ -621,14 +599,6 @@ button.form-control {
     font-size: $font-size-large;
     line-height: 24px;
 
-    label {
-        overflow: hidden;
-        text-overflow: ellipsis;
-        white-space: nowrap;
-        max-width: calc(100% - 85px);
-        cursor: pointer;
-    }
-
     .btn-group {
         vertical-align: top;
         margin-left: 10px;
@@ -1782,7 +1752,20 @@ treecontrol.tree-classic {
 }
 
 .getting-started {
+    min-height: 240px;
     margin: 15px 15px 300px;
+
+    ul {
+        line-height: 20px;
+    }
+
+    [class*="col-"] {
+        align-self: flex-start !important;
+    }
+
+    .align-center {
+        justify-content: center !important;
+    }
 }
 
 .getting-started-demo {
@@ -2114,6 +2097,7 @@ header.header-with-selector {
 
     div {
         display: flex;
+        align-items: center;
 
         &:nth-child(2) {
             .btn-ignite {
diff --git a/modules/web-console/frontend/public/stylesheets/variables.scss b/modules/web-console/frontend/public/stylesheets/variables.scss
index 22e4b9b86bf0d..bc0c2d54e4807 100644
--- a/modules/web-console/frontend/public/stylesheets/variables.scss
+++ b/modules/web-console/frontend/public/stylesheets/variables.scss
@@ -32,4 +32,8 @@ $ignite-button-border-radius: 4px;
 
 // Color palette
 $ignite-brand-primary: #ee2b27; // Red
-$ignite-brand-success: #0067b9; // Blue
\ No newline at end of file
+$ignite-brand-success: #0067b9; // Blue
+
+// Color for statuses
+$ignite-status-active: #417505;
+$ignite-status-inactive: #ee2b27;
\ No newline at end of file
diff --git a/modules/web-console/frontend/test/karma.conf.babel.js b/modules/web-console/frontend/test/karma.conf.babel.js
deleted file mode 100644
index dcf6cb0140ff2..0000000000000
--- a/modules/web-console/frontend/test/karma.conf.babel.js
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import path from 'path';
-
-import testCfg from '../webpack/webpack.test';
-
-export default (/** @type {import('karma').Config} */ config) => {
-    config.set({
-        // Base path that will be used to resolve all patterns (eg. files, exclude).
-        basePath: path.resolve('./'),
-
-        // Frameworks to use available frameworks: https://npmjs.org/browse/keyword/karma-adapter
-        frameworks: ['mocha'],
-
-        // List of files / patterns to load in the browser.
-        files: [
-            'node_modules/babel-polyfill/dist/polyfill.js',
-            'node_modules/angular/angular.js',
-            'node_modules/angular-mocks/angular-mocks.js',
-            'app/**/*.spec.js',
-            'test/**/*.test.js'
-        ],
-
-        plugins: [
-            require('karma-chrome-launcher'),
-            require('karma-teamcity-reporter'),
-            require('karma-mocha-reporter'),
-            require('karma-webpack'),
-            require('karma-mocha')
-        ],
-
-        // Preprocess matching files before serving them to the browser
-        // available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor.
-        preprocessors: {
-            '+(app|test)/**/*.js': ['webpack']
-        },
-
-        webpack: testCfg,
-
-        webpackMiddleware: {
-            noInfo: true
-        },
-
-        // Test results reporter to use
-        // possible values: 'dots', 'progress'
-        // available reporters: https://npmjs.org/browse/keyword/karma-reporter.
-        reporters: [process.env.TEST_REPORTER || 'mocha'],
-
-        mochaReporter: {
-            showDiff: true
-        },
-
-        // web server port
-        port: 9876,
-
-        // enable / disable colors in the output (reporters and logs)
-        colors: true,
-
-        // level of logging
-        // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG
-        logLevel: config.LOG_INFO,
-
-        // enable / disable watching file and executing tests whenever any file changes
-        autoWatch: true,
-
-        // start these browsers
-        // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
-        browsers: ['ChromeHeadlessNoSandbox'],
-        customLaunchers: {
-            ChromeHeadlessNoSandbox: {
-                base: 'ChromeHeadless',
-                flags: ['--no-sandbox']
-            },
-            ChromeDebug: {
-                base: 'Chrome',
-                flags: [
-                    '--start-maximized',
-                    '--auto-open-devtools-for-tabs'
-                ],
-                debug: true
-            }
-        },
-
-        // Continuous Integration mode
-        // if true, Karma captures browsers, runs the tests and exits
-        singleRun: true,
-
-        // Concurrency level
-        // how many browser should be started simultaneous
-        concurrency: Infinity,
-
-        client: {
-            mocha: {
-                ui: 'tdd'
-            }
-        }
-    });
-};
diff --git a/modules/web-console/frontend/test/karma.conf.js b/modules/web-console/frontend/test/karma.conf.js
index e3544823c4393..05b7e45e84cfc 100644
--- a/modules/web-console/frontend/test/karma.conf.js
+++ b/modules/web-console/frontend/test/karma.conf.js
@@ -15,5 +15,98 @@
  * limitations under the License.
  */
 
-require('babel-core/register');
-module.exports = require('./karma.conf.babel.js');
+const path = require('path');
+
+const testCfg = require('../webpack/webpack.test');
+
+module.exports = (/** @type {import('karma').Config} */ config) => {
+    config.set({
+        // Base path that will be used to resolve all patterns (eg. files, exclude).
+        basePath: path.resolve('./'),
+
+        // Frameworks to use available frameworks: https://npmjs.org/browse/keyword/karma-adapter
+        frameworks: ['mocha'],
+
+        // List of files / patterns to load in the browser.
+        files: [
+            'node_modules/angular/angular.js',
+            'node_modules/angular-mocks/angular-mocks.js',
+            'app/**/*.spec.+(js|ts)',
+            'test/**/*.test.js'
+        ],
+
+        plugins: [
+            require('karma-chrome-launcher'),
+            require('karma-teamcity-reporter'),
+            require('karma-mocha-reporter'),
+            require('karma-webpack'),
+            require('karma-mocha')
+        ],
+
+        // Preprocess matching files before serving them to the browser
+        // available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor.
+        preprocessors: {
+            '+(app|test)/**/*.+(js|ts)': ['webpack']
+        },
+
+        webpack: testCfg,
+
+        webpackMiddleware: {
+            noInfo: true
+        },
+
+        // Test results reporter to use
+        // possible values: 'dots', 'progress'
+        // available reporters: https://npmjs.org/browse/keyword/karma-reporter.
+        reporters: [process.env.TEST_REPORTER || 'mocha'],
+
+        mochaReporter: {
+            showDiff: true
+        },
+
+        // web server port
+        port: 9876,
+
+        // enable / disable colors in the output (reporters and logs)
+        colors: true,
+
+        // level of logging
+        // possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG
+        logLevel: config.LOG_INFO,
+
+        // enable / disable watching file and executing tests whenever any file changes
+        autoWatch: true,
+
+        // start these browsers
+        // available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
+        browsers: ['ChromeHeadlessNoSandbox'],
+        customLaunchers: {
+            ChromeHeadlessNoSandbox: {
+                base: 'ChromeHeadless',
+                flags: ['--no-sandbox']
+            },
+            ChromeDebug: {
+                base: 'Chrome',
+                flags: [
+                    '--start-maximized',
+                    '--auto-open-devtools-for-tabs'
+                ],
+                debug: true
+            }
+        },
+
+        // Continuous Integration mode
+        // if true, Karma captures browsers, runs the tests and exits
+        singleRun: true,
+
+        // Concurrency level
+        // how many browser should be started simultaneous
+        concurrency: Infinity,
+
+        client: {
+            mocha: {
+                ui: 'tdd'
+            }
+        }
+    });
+};
diff --git a/modules/web-console/frontend/test/unit/JavaTransformer.test.js b/modules/web-console/frontend/test/unit/JavaTransformer.test.js
index 7dacc1681e093..214377f04f0c1 100644
--- a/modules/web-console/frontend/test/unit/JavaTransformer.test.js
+++ b/modules/web-console/frontend/test/unit/JavaTransformer.test.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import JavaTypes from '../../app/services/JavaTypes.service.js';
+import JavaTypes from '../../app/services/JavaTypes.service';
 
 import generator from '../../app/modules/configuration/generator/ConfigurationGenerator';
 import transformer from '../../app/modules/configuration/generator/JavaTransformer.service';
diff --git a/modules/web-console/frontend/test/unit/JavaTypes.test.js b/modules/web-console/frontend/test/unit/JavaTypes.test.js
index e42d687e15eaf..2a465e2a0baaf 100644
--- a/modules/web-console/frontend/test/unit/JavaTypes.test.js
+++ b/modules/web-console/frontend/test/unit/JavaTypes.test.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import JavaTypes from '../../app/services/JavaTypes.service.js';
+import JavaTypes from '../../app/services/JavaTypes.service';
 
 import ClusterDflts from '../../app/modules/configuration/generator/defaults/Cluster.service';
 import CacheDflts from '../../app/modules/configuration/generator/defaults/Cache.service';
diff --git a/modules/web-console/frontend/test/unit/SpringTransformer.test.js b/modules/web-console/frontend/test/unit/SpringTransformer.test.js
index e96cab3c2d2f4..95671158918f6 100644
--- a/modules/web-console/frontend/test/unit/SpringTransformer.test.js
+++ b/modules/web-console/frontend/test/unit/SpringTransformer.test.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import JavaTypes from '../../app/services/JavaTypes.service.js';
+import JavaTypes from '../../app/services/JavaTypes.service';
 
 import generator from '../../app/modules/configuration/generator/ConfigurationGenerator';
 import transformer from '../../app/modules/configuration/generator/SpringTransformer.service';
diff --git a/modules/web-console/frontend/test/unit/SqlTypes.test.js b/modules/web-console/frontend/test/unit/SqlTypes.test.js
index 17f899cf90900..18b7bb64dd5d9 100644
--- a/modules/web-console/frontend/test/unit/SqlTypes.test.js
+++ b/modules/web-console/frontend/test/unit/SqlTypes.test.js
@@ -15,7 +15,7 @@
  * limitations under the License.
  */
 
-import SqlTypes from '../../app/services/SqlTypes.service.js';
+import SqlTypes from '../../app/services/SqlTypes.service';
 
 const INSTANCE = new SqlTypes();
 
diff --git a/modules/web-console/frontend/test/unit/defaultName.filter.test.js b/modules/web-console/frontend/test/unit/defaultName.filter.test.js
index 2eaf2238e17ee..7ecbd6fb51492 100644
--- a/modules/web-console/frontend/test/unit/defaultName.filter.test.js
+++ b/modules/web-console/frontend/test/unit/defaultName.filter.test.js
@@ -17,10 +17,9 @@
 
 import defaultName from '../../app/filters/default-name.filter';
 
-import { suite, test } from 'mocha';
 import { assert } from 'chai';
 
-const instance = defaultName[0]();
+const instance = defaultName();
 
 suite('defaultName', () => {
     test('defaultName filter', () => {
diff --git a/modules/web-console/frontend/tsconfig.json b/modules/web-console/frontend/tsconfig.json
index 382c9a5fe1321..7419d8f46176f 100644
--- a/modules/web-console/frontend/tsconfig.json
+++ b/modules/web-console/frontend/tsconfig.json
@@ -3,6 +3,7 @@
         "allowSyntheticDefaultImports": true,
         "target": "ES2017",
         "moduleResolution": "Node",
+        "noEmit": true,
         "allowJs": true,
         "checkJs": true,
         "baseUrl": "."
diff --git a/modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug b/modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug
index 2933a3e4f2257..bd652088a8cb2 100644
--- a/modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug
+++ b/modules/web-console/frontend/views/sql/paragraph-rate.tpl.pug
@@ -14,18 +14,33 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-.popover.settings(tabindex='-1' style='width: 200px')
-    .arrow
-    h3.popover-title(style='color: black') Refresh rate
-    button.close(id='paragraph-rate-close' ng-click='$hide()') &times;
+.popover.settings.refresh-rate(tabindex='-1')
+    h3.popover-title Refresh rate
+    button.close(ng-click='$hide()')
+        svg(ignite-icon='cross')
     .popover-content
-        form(name='popoverForm' novalidate)
-            .form-group(style='padding: 5px')
-                .col-sm-4
-                    input.form-control(id='paragraph-rate' ng-init='value = paragraph.rate.value' ng-model='value' type='number' min='1' required ignite-auto-focus)
-                .col-sm-8(style='padding-left: 5px')
-                    button.form-control.select-toggle(id='paragraph-unit' ng-init='unit = paragraph.rate.unit' ng-model='unit' required placeholder='Time unit' bs-select bs-options='item.value as item.label for item in timeUnit' tabindex='0')
-            .form-actions(style='margin-top: 30px; padding: 5px')
-                button.btn.btn-primary(id='paragraph-rate-start' ng-disabled='popoverForm.$invalid' ng-click='startRefresh(paragraph, value, unit); $hide()') Start
-                button.btn.btn-primary.btn-default(id='paragraph-rate-stop' ng-click='stopRefresh(paragraph); $hide()' ng-disabled='!paragraph.rate.installed') Stop
+        form.theme--ignite(name='popoverForm' novalidate ng-init='refreshRate = {}')
+            .form-field.form-field__text.ignite-form-field
+                .form-field__control
+                    input(ng-init='refreshRate.value = paragraph.rate.value' ng-model='refreshRate.value' type='number' min='1' required ignite-auto-focus)
+                .form-field__control
+                    button.select-toggle(ng-init='refreshRate.unit = paragraph.rate.unit'  ng-model='refreshRate.unit' required placeholder='Time unit' bs-select bs-options='item.value as item.label for item in timeUnit' tabindex='0')
+            .form-field.form-field__text.ignite-form-field
+                .actions
+                    button.btn-ignite.btn-ignite--primary(
+                        ng-disabled='popoverForm.$invalid'
+                        ng-click='startRefresh(paragraph, refreshRate.value, refreshRate.unit); $hide()'
+                        ng-hide='paragraph.rate.installed'
+                    ) Start
+
+                    button.btn-ignite.btn-ignite--primary(
+                        ng-disabled='popoverForm.$invalid || (refreshRate.unit === paragraph.rate.unit && refreshRate.value === paragraph.rate.value)'
+                        ng-click='startRefresh(paragraph, refreshRate.value, refreshRate.unit); $hide()'
+                        ng-hide='!paragraph.rate.installed'
+                    ) Start new
+
+                    button.btn-ignite.btn-ignite--primary(
+                        ng-click='stopRefresh(paragraph); $hide()'
+                        ng-hide='!paragraph.rate.installed'
+                    ) Stop
 
diff --git a/modules/web-console/frontend/views/templates/agent-download.tpl.pug b/modules/web-console/frontend/views/templates/agent-download.tpl.pug
index 4da9fadc9b2f6..46acfed248ea0 100644
--- a/modules/web-console/frontend/views/templates/agent-download.tpl.pug
+++ b/modules/web-console/frontend/views/templates/agent-download.tpl.pug
@@ -21,7 +21,6 @@ include /app/helpers/jade/mixins
         .modal-content(ng-switch-when='agentMissing')
             .modal-header.header
                 h4.modal-title
-                    i.fa.fa-download
                     span Connection to Ignite Web Agent is not established
             .modal-body.agent-download
                 p Please download and run #[a(href='/api/v1/downloads/agent' target='_self') ignite-web-agent] to use this functionality:
@@ -45,8 +44,9 @@ include /app/helpers/jade/mixins
                     )
 
             .modal-footer
-                button.btn-ignite.btn-ignite--link-success(ng-click='ctrl.back()') {{::ctrl.backText}}
-                a.btn-ignite.btn-ignite--success(href='/api/v1/downloads/agent' target='_self') Download agent
+                div
+                    button.btn-ignite.btn-ignite--link-success(ng-click='ctrl.back()') {{::ctrl.backText}}
+                    a.btn-ignite.btn-ignite--success(href='/api/v1/downloads/agent' target='_self') Download agent
 
         .modal-content(ng-switch-when='nodeMissing')
             .modal-header.header
@@ -65,4 +65,5 @@ include /app/helpers/jade/mixins
                         li Refer to #[b README.txt] in the ignite-web-agent folder for more information.
 
             .modal-footer
-                button.btn-ignite.btn-ignite--link-success(ng-click='ctrl.back()') {{::ctrl.backText}}
+                div
+                    button.btn-ignite.btn-ignite--link-success(ng-click='ctrl.back()') {{::ctrl.backText}}
diff --git a/modules/web-console/frontend/views/templates/batch-confirm.tpl.pug b/modules/web-console/frontend/views/templates/batch-confirm.tpl.pug
index ad8741bb3f360..874979bc4513c 100644
--- a/modules/web-console/frontend/views/templates/batch-confirm.tpl.pug
+++ b/modules/web-console/frontend/views/templates/batch-confirm.tpl.pug
@@ -18,7 +18,7 @@
     .modal-dialog
         .modal-content
             .modal-header
-                h4.modal-title 
+                h4.modal-title
                     svg(ignite-icon='attention')
                     | Confirmation
                 button.close(type='button' aria-label='Close' ng-click='cancel()')
diff --git a/modules/web-console/frontend/views/templates/confirm.tpl.pug b/modules/web-console/frontend/views/templates/confirm.tpl.pug
index 9f5e2bbb2d3cd..581aa37f01472 100644
--- a/modules/web-console/frontend/views/templates/confirm.tpl.pug
+++ b/modules/web-console/frontend/views/templates/confirm.tpl.pug
@@ -18,17 +18,17 @@
     .modal-dialog.modal-dialog--adjust-height
         .modal-content
             .modal-header
-                h4.modal-title 
-                    i.icon-confirm
+                h4.modal-title
                     span Confirmation
                 button.close(type='button' aria-label='Close' ng-click='confirmCancel()')
                      svg(ignite-icon="cross")
             .modal-body(ng-show='content')
                 p(ng-bind-html='content')
             .modal-footer
-                button#confirm-btn-cancel.btn-ignite.btn-ignite--link-success(ng-click='confirmCancel()') Cancel
+                div
+                    button#confirm-btn-cancel.btn-ignite.btn-ignite--link-success(ng-click='confirmCancel()') Cancel
 
-                button#confirm-btn-no.btn-ignite.btn-ignite--link-success(ng-if='yesNo' ng-click='confirmNo()') No
-                button#confirm-btn-yes.btn-ignite.btn-ignite--success(ignite-auto-focus ng-if='yesNo' ng-click='confirmYes()') Yes
+                    button#confirm-btn-no.btn-ignite.btn-ignite--link-success(ng-if='yesNo' ng-click='confirmNo()') No
+                    button#confirm-btn-yes.btn-ignite.btn-ignite--success(ignite-auto-focus ng-if='yesNo' ng-click='confirmYes()') Yes
 
-                button#confirm-btn-ok.btn-ignite.btn-ignite--success(ignite-auto-focus ng-if='!yesNo' ng-click='confirmYes()') Confirm
+                    button#confirm-btn-ok.btn-ignite.btn-ignite--success(ignite-auto-focus ng-if='!yesNo' ng-click='confirmYes()') Confirm
diff --git a/modules/web-console/frontend/views/templates/demo-info.tpl.pug b/modules/web-console/frontend/views/templates/demo-info.tpl.pug
index 62642b39bdb74..437c8a0b03645 100644
--- a/modules/web-console/frontend/views/templates/demo-info.tpl.pug
+++ b/modules/web-console/frontend/views/templates/demo-info.tpl.pug
@@ -14,34 +14,40 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-.modal.center(role='dialog')
+.modal.modal--ignite.theme--ignite.center(role='dialog')
     .modal-dialog
         .modal-content
             #errors-container.modal-header.header
-                button.close(ng-click='close()' aria-hidden='true') &times;
+                button.close(ng-click='close()' aria-hidden='true')
+                    svg(ignite-icon="cross")
                 h4.modal-title
-                    i.fa.fa-info-circle
+                    svg(ignite-icon="attention")
                     | {{title}}
             .modal-body
                 div(ng-bind-html='message')
+
                 div(ng-hide='hasAgents')
-                    p &nbsp;
-                    div
-                        h4
-                            i.fa.fa-download.fa-cursor-default
-                            | &nbsp;How To Start Demo
-                        ul
-                            li
-                                a(ng-click='downloadAgent()') #[b Download]
-                                | &nbsp; and unzip ignite-web-agent archive
-                            li #[b Run] shell file ignite-web-agent.{sh|bat}
-                div(ng-show='hasAgents')
+                    h4
+                        i.fa.fa-download.fa-cursor-default
+                        | &nbsp;How To Start Demo
+                    ul
+                        li
+                            a(ng-href='{{downloadAgentHref}}' target='_self') #[b Download]
+                            | &nbsp; and unzip ignite-web-agent archive
+                        li #[b Run] shell file ignite-web-agent.{sh|bat}
+
+                div(ng-hide='!hasAgents')
                     h4
                         i.fa.fa-star-o.fa-cursor-default
                         | &nbsp;Start Demo
                     ul
                         li Web Agent is already started
                         li Close dialog and try Web Console
+
             .modal-footer
-                button.btn.btn-default(ng-class='hasAgents ? "btn-primary" : "btn-default"' ng-click='close()') Close
-                a.btn.btn-primary(ng-hide='hasAgents' href='/api/v1/downloads/agent' target='_self') Download agent
+                .ng-animate-disabled(ng-if='!hasAgents')
+                    button.btn-ignite.btn-ignite--link-success(ng-click='close()') Close
+                    a.btn-ignite.btn-ignite--success(ng-href='{{downloadAgentHref}}' target='_self') Download agent
+
+                .ng-animate-disabled(ng-if='hasAgents')
+                    button.btn-ignite.btn-ignite--success(ng-click='close()') Close
diff --git a/modules/web-console/frontend/views/templates/getting-started.tpl.pug b/modules/web-console/frontend/views/templates/getting-started.tpl.pug
index 3a89035d27ab0..8b0a03c5b892b 100644
--- a/modules/web-console/frontend/views/templates/getting-started.tpl.pug
+++ b/modules/web-console/frontend/views/templates/getting-started.tpl.pug
@@ -14,21 +14,26 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-.modal.center(role='dialog')
+include /app/helpers/jade/mixins
+
+.modal.modal--ignite.theme--ignite.center(role='dialog')
     .modal-dialog
         .modal-content
             #errors-container.modal-header.header
-                button.close(ng-click='close()' aria-hidden='true') &times;
-                h4.modal-title 
-                    i.fa.fa-book
+                button.close(ng-click='close()' aria-hidden='true')
+                    svg(ignite-icon="cross")
+                h4.modal-title
                     | {{title}}
-            .getting-started
-                .col-xs-12(ng-bind-html='message')
+            .modal-body
+                .getting-started.row(ng-bind-html='message')
             .modal-footer
-                .checkbox
-                    label
-                        input(type='checkbox' ng-model='ui.showGettingStarted')
-                        | Show getting started on next login
-                a.btn.btn-primary(ng-disabled='isFirst()' ng-click='!isFirst() && prev()') Prev
-                a.btn.btn-primary(ng-disabled='isLast()' ng-click='!isLast() && next()') Next
-                a.btn.btn-primary(ng-click='close()') Close
+                div
+                    +form-field__checkbox({
+                        label: 'Do not show this window again',
+                        model: 'ui.dontShowGettingStarted',
+                        name: '"dontShowGettingStarted"'
+                    })
+                div
+                    a.btn-ignite.btn-ignite--link-success(ng-disabled='isFirst()' ng-click='!isFirst() && prev()') Prev
+                    a.btn-ignite.btn-ignite--link-success(ng-disabled='isLast()' ng-click='!isLast() && next()') Next
+                    a.btn-ignite.btn-ignite--success(ng-click='close()') Close
diff --git a/modules/web-console/frontend/views/templates/message.tpl.pug b/modules/web-console/frontend/views/templates/message.tpl.pug
index 3cdb3c8619840..bfe4c184a13eb 100644
--- a/modules/web-console/frontend/views/templates/message.tpl.pug
+++ b/modules/web-console/frontend/views/templates/message.tpl.pug
@@ -14,16 +14,19 @@
     See the License for the specific language governing permissions and
     limitations under the License.
 
-.modal(tabindex='-1' role='dialog')
+.modal.modal--ignite.theme--ignite.center(tabindex='-1' role='dialog')
     .modal-dialog
         .modal-content
             .modal-header
-                button.close(ng-click='$hide()' aria-hidden='true') &times;
+                button.close(ng-click='$hide()' aria-hidden='true')
+                    svg(ignite-icon="cross")
                 h4.modal-title
-                    i.fa.fa-info-circle
+                    svg(ignite-icon="attention")
                     | {{title}}
-            .modal-body(ng-show='content' style='overflow: auto; max-height: 300px;')
-                p(ng-bind-html='content.join("<br/>")' style='text-align: left; white-space: nowrap;')
+            .modal-body(ng-show='content')
+                .modal-body--inner-content
+                    p(ng-bind-html='content.join("<br/>")')
             .modal-footer
-                .pull-left(ng-show='meta') {{meta}}
-                button.btn.btn-primary(id='confirm-btn-confirm' ng-click='$hide()') Ok
+                div(ng-show='meta') {{meta}}
+                div
+                    button.btn-ignite.btn-ignite--success(ng-click='$hide()') Ok
diff --git a/modules/web-console/frontend/webpack/webpack.common.js b/modules/web-console/frontend/webpack/webpack.common.js
index f2901e1bd870d..327fe7a2c1893 100644
--- a/modules/web-console/frontend/webpack/webpack.common.js
+++ b/modules/web-console/frontend/webpack/webpack.common.js
@@ -15,22 +15,17 @@
  * limitations under the License.
  */
 
-import path from 'path';
-import webpack from 'webpack';
+const path = require('path');
+const webpack = require('webpack');
 
-import transformRuntime from 'babel-plugin-transform-runtime';
-import presetEs2015 from 'babel-preset-es2015';
-import presetStage1 from 'babel-preset-stage-1';
+const CopyWebpackPlugin = require('copy-webpack-plugin');
+const HtmlWebpackPlugin = require('html-webpack-plugin');
+const ProgressBarPlugin = require('progress-bar-webpack-plugin');
 
-import CopyWebpackPlugin from 'copy-webpack-plugin';
-import HtmlWebpackPlugin from 'html-webpack-plugin';
-import ProgressBarPlugin from 'progress-bar-webpack-plugin';
-
-import eslintFormatter from 'eslint-friendly-formatter';
+const eslintFormatter = require('eslint-formatter-friendly');
 
 const basedir = path.join(__dirname, '../');
 const contentBase = path.join(basedir, 'public');
-const node_modules = path.join(basedir, 'node_modules');
 const app = path.join(basedir, 'app');
 
 /** @type {webpack.Configuration} */
@@ -53,21 +48,13 @@ const config = {
 
     // Resolves modules.
     resolve: {
-        modules: [node_modules],
         // A list of module source folders.
         alias: {
             app,
             images: path.join(basedir, 'public/images'),
             views: path.join(basedir, 'views')
-        }
-    },
-
-    // Resolve loader use postfix.
-    resolveLoader: {
-        modules: [
-            node_modules
-        ],
-        moduleExtensions: ['-loader']
+        },
+        extensions: ['.wasm', '.mjs', '.js', '.ts', '.json']
     },
 
     module: {
@@ -76,7 +63,7 @@ const config = {
             {
                 test: /^(?:(?!tpl\.pug$).)*\.pug$/, // TODO: check this regexp for correct.
                 use: {
-                    loader: 'pug-html',
+                    loader: 'pug-html-loader',
                     options: {
                         basedir
                     }
@@ -87,46 +74,32 @@ const config = {
             {
                 test: /\.tpl\.pug$/,
                 use: [
-                    'file?exports=false&name=assets/templates/[name].[hash].html',
-                    `pug-html?exports=false&basedir=${basedir}`
+                    'file-loader?exports=false&name=assets/templates/[name].[hash].html',
+                    `pug-html-loader?exports=false&basedir=${basedir}`
                 ]
             },
             { test: /\.worker\.js$/, use: { loader: 'worker-loader' } },
             {
-                test: /\.js$/,
+                test: /\.(js|ts)$/,
                 enforce: 'pre',
                 exclude: [/node_modules/],
                 use: [{
-                    loader: 'eslint',
+                    loader: 'eslint-loader',
                     options: {
-                        failOnWarning: false,
-                        failOnError: false,
                         formatter: eslintFormatter,
                         context: process.cwd()
                     }
                 }]
             },
             {
-                test: /\.js$/,
-                exclude: [node_modules],
-                use: [{
-                    loader: 'babel-loader',
-                    options: {
-                        cacheDirectory: true,
-                        plugins: [
-                            transformRuntime
-                        ],
-                        presets: [
-                            presetEs2015,
-                            presetStage1
-                        ]
-                    }
-                }]
+                test: /\.(js|ts)$/,
+                exclude: /node_modules/,
+                use: 'babel-loader'
             },
             {
                 test: /\.(ttf|eot|svg|woff(2)?)(\?v=[\d.]+)?(\?[a-z0-9#-]+)?$/,
                 exclude: [contentBase, /\.icon\.svg$/],
-                use: 'file?name=assets/fonts/[name].[ext]'
+                use: 'file-loader?name=assets/fonts/[name].[ext]'
             },
             {
                 test: /\.icon\.svg$/,
@@ -141,11 +114,11 @@ const config = {
             {
                 test: /.*\.url\.svg$/,
                 include: [contentBase],
-                use: 'file?name=assets/fonts/[name].[ext]'
+                use: 'file-loader?name=assets/fonts/[name].[ext]'
             },
             {
                 test: /\.(jpe?g|png|gif)$/i,
-                use: 'file?name=assets/images/[name].[hash].[ext]'
+                use: 'file-loader?name=assets/images/[name].[hash].[ext]'
             },
             {
                 test: require.resolve('jquery'),
@@ -169,17 +142,6 @@ const config = {
 
     // Load plugins.
     plugins: [
-        new webpack.LoaderOptionsPlugin({
-            options: {
-                pug: {
-                    basedir
-                },
-                eslint: {
-                    configFile: path.join(basedir, '.eslintrc')
-                },
-                target: 'web'
-            }
-        }),
         new webpack.ProvidePlugin({
             $: 'jquery',
             'window.jQuery': 'jquery',
@@ -198,4 +160,4 @@ const config = {
     ]
 };
 
-export default config;
+module.exports = config;
diff --git a/modules/web-console/frontend/webpack/webpack.dev.babel.js b/modules/web-console/frontend/webpack/webpack.dev.js
similarity index 93%
rename from modules/web-console/frontend/webpack/webpack.dev.babel.js
rename to modules/web-console/frontend/webpack/webpack.dev.js
index c5950ee6e23e9..3af63776b4189 100644
--- a/modules/web-console/frontend/webpack/webpack.dev.babel.js
+++ b/modules/web-console/frontend/webpack/webpack.dev.js
@@ -15,11 +15,11 @@
  * limitations under the License.
  */
 
-import merge from 'webpack-merge';
+const merge = require('webpack-merge');
 
-import path from 'path';
+const path = require('path');
 
-import commonCfg from './webpack.common';
+const commonCfg = require('./webpack.common');
 
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 
@@ -27,7 +27,7 @@ const backendPort = process.env.BACKEND_PORT || 3000;
 const devServerPort = process.env.PORT || 9000;
 const devServerHost = process.env.HOST || '0.0.0.0';
 
-export default merge(commonCfg, {
+module.exports = merge(commonCfg, {
     mode: 'development',
     devtool: 'source-map',
     watch: true,
@@ -36,7 +36,7 @@ export default merge(commonCfg, {
         rules: [
             {
                 test: /\.css$/,
-                use: ['style', 'css']
+                use: ['style-loader', 'css-loader']
             },
             {
                 test: /\.scss$/,
diff --git a/modules/web-console/frontend/webpack/webpack.prod.babel.js b/modules/web-console/frontend/webpack/webpack.prod.js
similarity index 87%
rename from modules/web-console/frontend/webpack/webpack.prod.babel.js
rename to modules/web-console/frontend/webpack/webpack.prod.js
index a5aa1c61f4735..fa6374ea09411 100644
--- a/modules/web-console/frontend/webpack/webpack.prod.babel.js
+++ b/modules/web-console/frontend/webpack/webpack.prod.js
@@ -15,17 +15,17 @@
  * limitations under the License.
  */
 
-import path from 'path';
-import merge from 'webpack-merge';
+const path = require('path');
+const merge = require('webpack-merge');
 
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
-import UglifyJSPlugin from 'uglifyjs-webpack-plugin';
+const UglifyJSPlugin = require('uglifyjs-webpack-plugin');
 
-import commonCfg from './webpack.common';
+const commonCfg = require('./webpack.common');
 
 const basedir = path.join(__dirname, '../');
 
-export default merge(commonCfg, {
+module.exports = merge(commonCfg, {
     bail: true, // Cancel build on error.
     mode: 'production',
     module: {
@@ -37,7 +37,7 @@ export default merge(commonCfg, {
             {
                 test: /\.scss$/,
                 use: [MiniCssExtractPlugin.loader, 'css-loader', {
-                    loader: 'sass',
+                    loader: 'sass-loader',
                     options: {
                         includePaths: [basedir]
                     }
diff --git a/modules/web-console/frontend/webpack/webpack.test.js b/modules/web-console/frontend/webpack/webpack.test.js
index 2ade625654655..c6d90df5351c2 100644
--- a/modules/web-console/frontend/webpack/webpack.test.js
+++ b/modules/web-console/frontend/webpack/webpack.test.js
@@ -15,11 +15,10 @@
  * limitations under the License.
  */
 
-import merge from 'webpack-merge';
+const merge = require('webpack-merge');
+const commonCfg = require('./webpack.common');
 
-import commonCfg from './webpack.common';
-
-export default merge(commonCfg, {
+module.exports = merge(commonCfg, {
     mode: 'development',
     cache: true,
     node: {

From 8deb207e6366284f66bb4762c6407520c32de427 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Mon, 29 Oct 2018 16:01:12 +0700
Subject: [PATCH 460/543] IGNITE-10031 REST: Added "caches" parameter to "top"
 command to include/exclude information about caches.

(cherry picked from commit 594aac83c39eec1d12f2e9cef0ed9af35e9a9dba)
---
 .../JettyRestProcessorAbstractSelfTest.java   | 41 +++++++++++++++++++
 .../top/GridTopologyCommandHandler.java       | 28 +++++++------
 .../rest/request/GridRestTopologyRequest.java | 19 ++++++++-
 .../http/jetty/GridJettyRestHandler.java      |  3 ++
 .../app/modules/agent/AgentManager.service.js |  2 +-
 .../agent/handlers/ClusterListener.java       |  1 +
 6 files changed, 80 insertions(+), 14 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
index 76190b0efc629..75c9c48b5e96b 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
@@ -1445,6 +1445,25 @@ public void testTopology() throws Exception {
                 assertEquals(publicCache.getConfiguration(CacheConfiguration.class).getCacheMode(), cacheMode);
             }
         }
+
+        // Test that caches not included.
+        ret = content(null, GridRestCommand.TOPOLOGY,
+            "attr", "false",
+            "mtr", "false",
+            "caches", "false"
+        );
+
+        info("Topology command result: " + ret);
+
+        res = jsonResponse(ret);
+
+        assertEquals(gridCount(), res.size());
+
+        for (JsonNode node : res) {
+            assertTrue(node.get("attributes").isNull());
+            assertTrue(node.get("metrics").isNull());
+            assertTrue(node.get("caches").isNull());
+        }
     }
 
     /**
@@ -1464,6 +1483,12 @@ public void testNode() throws Exception {
         assertTrue(res.get("attributes").isObject());
         assertTrue(res.get("metrics").isObject());
 
+        JsonNode caches = res.get("caches");
+
+        assertTrue(caches.isArray());
+        assertFalse(caches.isNull());
+        assertEquals(grid(0).context().cache().publicCaches().size(), caches.size());
+
         ret = content(null, GridRestCommand.NODE,
             "attr", "false",
             "mtr", "false",
@@ -1489,6 +1514,22 @@ public void testNode() throws Exception {
         res = jsonResponse(ret);
 
         assertTrue(res.isNull());
+
+        // Check that caches not included.
+        ret = content(null, GridRestCommand.NODE,
+            "id", grid(0).localNode().id().toString(),
+            "attr", "false",
+            "mtr", "false",
+            "caches", "false"
+        );
+
+        info("Topology command result: " + ret);
+
+        res = jsonResponse(ret);
+
+        assertTrue(res.get("attributes").isNull());
+        assertTrue(res.get("metrics").isNull());
+        assertTrue(res.get("caches").isNull());
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
index 0390936e77395..edcb3741bc331 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/top/GridTopologyCommandHandler.java
@@ -97,6 +97,7 @@ public GridTopologyCommandHandler(GridKernalContext ctx) {
 
         boolean mtr = req0.includeMetrics();
         boolean attr = req0.includeAttributes();
+        boolean caches = req0.includeCaches();
 
         switch (req.command()) {
             case TOPOLOGY: {
@@ -107,7 +108,7 @@ public GridTopologyCommandHandler(GridKernalContext ctx) {
                     new ArrayList<>(allNodes.size());
 
                 for (ClusterNode node : allNodes)
-                    top.add(createNodeBean(node, mtr, attr));
+                    top.add(createNodeBean(node, mtr, attr, caches));
 
                 res.setResponse(top);
 
@@ -143,7 +144,7 @@ public boolean apply(ClusterNode n) {
                     });
 
                 if (node != null)
-                    res.setResponse(createNodeBean(node, mtr, attr));
+                    res.setResponse(createNodeBean(node, mtr, attr, caches));
                 else
                     res.setResponse(null);
 
@@ -196,14 +197,15 @@ public GridClientCacheBean createCacheBean(CacheConfiguration ccfg) {
     }
 
     /**
-     * Creates node bean out of grid node. Notice that cache attribute is handled separately.
+     * Creates node bean out of cluster node. Notice that cache attribute is handled separately.
      *
-     * @param node Grid node.
-     * @param mtr {@code true} to add metrics.
-     * @param attr {@code true} to add attributes.
+     * @param node Cluster node.
+     * @param mtr Whether to include node metrics.
+     * @param attr Whether to include node attributes.
+     * @param caches Whether to include node caches.
      * @return Grid Node bean.
      */
-    private GridClientNodeBean createNodeBean(ClusterNode node, boolean mtr, boolean attr) {
+    private GridClientNodeBean createNodeBean(ClusterNode node, boolean mtr, boolean attr, boolean caches) {
         assert node != null;
 
         GridClientNodeBean nodeBean = new GridClientNodeBean();
@@ -216,14 +218,16 @@ private GridClientNodeBean createNodeBean(ClusterNode node, boolean mtr, boolean
         nodeBean.setTcpAddresses(nonEmptyList(node.<Collection<String>>attribute(ATTR_REST_TCP_ADDRS)));
         nodeBean.setTcpHostNames(nonEmptyList(node.<Collection<String>>attribute(ATTR_REST_TCP_HOST_NAMES)));
 
-        Map<String, CacheConfiguration> nodeCaches = ctx.discovery().nodePublicCaches(node);
+        if (caches) {
+            Map<String, CacheConfiguration> nodeCaches = ctx.discovery().nodePublicCaches(node);
 
-        Collection<GridClientCacheBean> caches = new ArrayList<>(nodeCaches.size());
+            Collection<GridClientCacheBean> cacheBeans = new ArrayList<>(nodeCaches.size());
 
-        for (CacheConfiguration ccfg : nodeCaches.values())
-            caches.add(createCacheBean(ccfg));
+            for (CacheConfiguration ccfg : nodeCaches.values())
+                cacheBeans.add(createCacheBean(ccfg));
 
-        nodeBean.setCaches(caches);
+            nodeBean.setCaches(cacheBeans);
+        }
 
         if (mtr) {
             ClusterMetrics metrics = node.metrics();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestTopologyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestTopologyRequest.java
index b02836743f67b..fadd178945e2b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestTopologyRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/request/GridRestTopologyRequest.java
@@ -36,6 +36,9 @@ public class GridRestTopologyRequest extends GridRestRequest {
     /** Include node attributes flag. */
     private boolean includeAttrs;
 
+    /** Include caches flag. With default value for compatibility. */
+    private boolean includeCaches = true;
+
     /**
      * @return Include metrics flag.
      */
@@ -64,6 +67,20 @@ public void includeAttributes(boolean includeAttrs) {
         this.includeAttrs = includeAttrs;
     }
 
+    /**
+     * @return Include caches flag.
+     */
+    public boolean includeCaches() {
+        return includeCaches;
+    }
+
+    /**
+     * @param includeCaches Include caches flag.
+     */
+    public void includeCaches(boolean includeCaches) {
+        this.includeCaches = includeCaches;
+    }
+
     /**
      * @return Node identifier, if specified, {@code null} otherwise.
      */
@@ -96,4 +113,4 @@ public void nodeIp(String nodeIp) {
     @Override public String toString() {
         return S.toString(GridRestTopologyRequest.class, this, super.toString());
     }
-}
\ No newline at end of file
+}
diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
index 99a884411cfd9..4ef9012dbbabe 100644
--- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
+++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
@@ -674,6 +674,9 @@ private Object convert(String type, Object obj) throws IgniteCheckedException {
                 restReq0.includeMetrics(Boolean.parseBoolean((String)params.get("mtr")));
                 restReq0.includeAttributes(Boolean.parseBoolean((String)params.get("attr")));
 
+                String caches = (String)params.get("caches");
+                restReq0.includeCaches(caches == null || Boolean.parseBoolean(caches));
+
                 restReq0.nodeIp((String)params.get("ip"));
 
                 restReq0.nodeId(uuidValue("id", params));
diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
index f667374bbec2f..8d2ced2ad8a0e 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
@@ -545,7 +545,7 @@ export default class AgentManager {
      * @returns {Promise}
      */
     topology(attr = false, mtr = false) {
-        return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr});
+        return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr, caches: false});
     }
 
     /**
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
index 1c93798b28819..779b458b61cae 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
@@ -410,6 +410,7 @@ private RestResult topology(boolean full) throws IOException {
             params.put("cmd", "top");
             params.put("attr", true);
             params.put("mtr", full);
+            params.put("caches", false);
 
             return restCommand(params);
         }

From 717890007d62dc81c7d2e1ea6071c75240f9feb9 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Mon, 29 Oct 2018 12:14:54 +0300
Subject: [PATCH 461/543] IGNITE-9753 Several optimization of validate_indexes
 - Fixes #5063.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit 1fe02df8daa94ca0761049cd41659f4ae18f580c)
---
 .../internal/commandline/CommandHandler.java  |  34 ++-
 .../verify/IndexIntegrityCheckIssue.java      |  74 +++++++
 .../verify/VisorValidateIndexesJobResult.java |  30 ++-
 .../resources/META-INF/classnames.properties  |   1 +
 .../visor/verify/ValidateIndexesClosure.java  | 205 +++++++++++++++---
 .../util/GridCommandHandlerIndexingTest.java  | 152 ++++++++-----
 6 files changed, 398 insertions(+), 98 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexIntegrityCheckIssue.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 1ab65ba432c2c..a66c0eb295878 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -96,6 +96,7 @@
 import org.apache.ignite.internal.visor.tx.VisorTxTask;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
 import org.apache.ignite.internal.visor.tx.VisorTxTaskResult;
+import org.apache.ignite.internal.visor.verify.IndexIntegrityCheckIssue;
 import org.apache.ignite.internal.visor.verify.IndexValidationIssue;
 import org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult;
 import org.apache.ignite.internal.visor.verify.VisorContentionTask;
@@ -821,9 +822,20 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
             }
         }
 
-        boolean errors = false;
-
         for (Map.Entry<UUID, VisorValidateIndexesJobResult> nodeEntry : taskRes.results().entrySet()) {
+            boolean errors = false;
+
+            log("validate_indexes result on node " + nodeEntry.getKey() + ":");
+
+            Collection<IndexIntegrityCheckIssue> integrityCheckFailures = nodeEntry.getValue().integrityCheckFailures();
+
+            if (!integrityCheckFailures.isEmpty()) {
+                errors = true;
+
+                for (IndexIntegrityCheckIssue is : integrityCheckFailures)
+                    log("\t" + is.toString());
+            }
+
             Map<PartitionKey, ValidateIndexesPartitionResult> partRes = nodeEntry.getValue().partitionResult();
 
             for (Map.Entry<PartitionKey, ValidateIndexesPartitionResult> e : partRes.entrySet()) {
@@ -832,10 +844,10 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
                 if (!res.issues().isEmpty()) {
                     errors = true;
 
-                    log(e.getKey().toString() + " " + e.getValue().toString());
+                    log("\t" + e.getKey().toString() + " " + e.getValue().toString());
 
                     for (IndexValidationIssue is : res.issues())
-                        log(is.toString());
+                        log("\t\t" + is.toString());
                 }
             }
 
@@ -847,18 +859,18 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
                 if (!res.issues().isEmpty()) {
                     errors = true;
 
-                    log("SQL Index " + e.getKey() + " " + e.getValue().toString());
+                    log("\tSQL Index " + e.getKey() + " " + e.getValue().toString());
 
                     for (IndexValidationIssue is : res.issues())
-                        log(is.toString());
+                        log("\t\t" + is.toString());
                 }
             }
-        }
 
-        if (!errors)
-            log("validate_indexes has finished, no issues found.");
-        else
-            log("validate_indexes has finished with errors (listed above).");
+            if (!errors)
+                log("no issues found.\n");
+            else
+                log("issues found (listed above).\n");
+        }
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexIntegrityCheckIssue.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexIntegrityCheckIssue.java
new file mode 100644
index 0000000000000..ec6e5b2440765
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/IndexIntegrityCheckIssue.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.verify;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ *
+ */
+public class IndexIntegrityCheckIssue extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Cache group name. */
+    private String grpName;
+
+    /** T. */
+    @GridToStringExclude
+    private Throwable t;
+
+    /**
+     *
+     */
+    public IndexIntegrityCheckIssue() {
+        // Default constructor required for Externalizable.
+    }
+
+    /**
+     * @param grpName Group name.
+     * @param t Data integrity check error.
+     */
+    public IndexIntegrityCheckIssue(String grpName, Throwable t) {
+        this.grpName = grpName;
+        this.t = t;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeString(out, this.grpName);
+        out.writeObject(t);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        this.grpName = U.readString(in);
+        this.t = (Throwable)in.readObject();
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(IndexIntegrityCheckIssue.class, this) + ", " + t.getClass() + ": " + t.getMessage();
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
index aa74323898bd8..f84fc1a259027 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
@@ -20,11 +20,14 @@
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Map;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
+import org.jetbrains.annotations.NotNull;
 
 /**
  *
@@ -39,14 +42,22 @@ public class VisorValidateIndexesJobResult extends VisorDataTransferObject {
     /** Results of reverse indexes validation from node. */
     private Map<String, ValidateIndexesPartitionResult> idxRes;
 
+    /** Integrity check issues. */
+    private Collection<IndexIntegrityCheckIssue> integrityCheckFailures;
+
     /**
      * @param partRes Results of indexes validation from node.
      * @param idxRes Results of reverse indexes validation from node.
+     * @param  integrityCheckFailures Collection of indexes integrity check failures.
      */
-    public VisorValidateIndexesJobResult(Map<PartitionKey, ValidateIndexesPartitionResult> partRes,
-        Map<String, ValidateIndexesPartitionResult> idxRes) {
+    public VisorValidateIndexesJobResult(
+            @NotNull Map<PartitionKey, ValidateIndexesPartitionResult> partRes,
+            @NotNull Map<String, ValidateIndexesPartitionResult> idxRes,
+            @NotNull Collection<IndexIntegrityCheckIssue> integrityCheckFailures
+    ) {
         this.partRes = partRes;
         this.idxRes = idxRes;
+        this.integrityCheckFailures = integrityCheckFailures;
     }
 
     /**
@@ -57,7 +68,7 @@ public VisorValidateIndexesJobResult() {
 
     /** {@inheritDoc} */
     @Override public byte getProtocolVersion() {
-        return V2;
+        return V3;
     }
 
     /**
@@ -71,13 +82,21 @@ public Map<PartitionKey, ValidateIndexesPartitionResult> partitionResult() {
      * @return Results of reverse indexes validation from node.
      */
     public Map<String, ValidateIndexesPartitionResult> indexResult() {
-        return idxRes;
+        return idxRes == null ? Collections.emptyMap() : idxRes;
+    }
+
+    /**
+     * @return Collection of failed integrity checks.
+     */
+    public Collection<IndexIntegrityCheckIssue> integrityCheckFailures() {
+        return integrityCheckFailures == null ? Collections.emptyList() : integrityCheckFailures;
     }
 
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeMap(out, partRes);
         U.writeMap(out, idxRes);
+        U.writeCollection(out, integrityCheckFailures);
     }
 
     /** {@inheritDoc} */
@@ -86,6 +105,9 @@ public Map<String, ValidateIndexesPartitionResult> indexResult() {
 
         if (protoVer >= V2)
             idxRes = U.readMap(in);
+
+        if (protoVer >= V3)
+            integrityCheckFailures = U.readCollection(in);
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 284b3dbda9521..3c866f49c2d5b 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -2199,6 +2199,7 @@ org.apache.ignite.internal.visor.util.VisorEventMapper
 org.apache.ignite.internal.visor.util.VisorExceptionWrapper
 org.apache.ignite.internal.visor.util.VisorTaskUtils$4
 org.apache.ignite.internal.visor.verify.IndexValidationIssue
+org.apache.ignite.internal.visor.verify.IndexIntegrityCheckIssue
 org.apache.ignite.internal.visor.verify.ValidateIndexesPartitionResult
 org.apache.ignite.internal.visor.verify.VisorContentionJobResult
 org.apache.ignite.internal.visor.verify.VisorContentionTask
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index 503b57c494c70..ec02c25c3ca36 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -18,6 +18,8 @@
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -37,22 +39,29 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.store.PageStore;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheObjectContext;
 import org.apache.ignite.internal.processors.cache.CacheObjectUtils;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.KeyCacheObject;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.processors.query.GridQueryProcessor;
 import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
 import org.apache.ignite.internal.processors.query.QueryTypeDescriptorImpl;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.processors.query.h2.database.H2TreeIndex;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
 import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
@@ -104,9 +113,15 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
     /** Counter of processed indexes. */
     private final AtomicInteger processedIndexes = new AtomicInteger(0);
 
+    /** Counter of integrity checked indexes. */
+    private final AtomicInteger integrityCheckedIndexes = new AtomicInteger(0);
+
     /** Total partitions. */
     private volatile int totalIndexes;
 
+    /** Total cache groups. */
+    private volatile  int totalCacheGrps;
+
     /** Last progress print timestamp. */
     private final AtomicLong lastProgressPrintTs = new AtomicLong(0);
 
@@ -182,10 +197,14 @@ private VisorValidateIndexesJobResult call0() throws Exception {
         List<T2<CacheGroupContext, GridDhtLocalPartition>> partArgs = new ArrayList<>();
         List<T2<GridCacheContext, Index>> idxArgs = new ArrayList<>();
 
+        totalCacheGrps = grpIds.size();
+
+        Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = integrityCheckIndexesPartitions(grpIds);
+
         for (Integer grpId : grpIds) {
             CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
 
-            if (grpCtx == null)
+            if (grpCtx == null || integrityCheckResults.containsKey(grpId))
                 continue;
 
             List<GridDhtLocalPartition> parts = grpCtx.topology().localPartitions();
@@ -210,7 +229,8 @@ private VisorValidateIndexesJobResult call0() throws Exception {
                         ArrayList<Index> indexes = gridH2Tbl.getIndexes();
 
                         for (Index idx : indexes)
-                            idxArgs.add(new T2<>(ctx, idx));
+                            if (idx instanceof H2TreeIndex)
+                                idxArgs.add(new T2<>(ctx, idx));
                     }
                 }
             }
@@ -220,15 +240,15 @@ private VisorValidateIndexesJobResult call0() throws Exception {
         Collections.shuffle(partArgs);
         Collections.shuffle(idxArgs);
 
+        totalPartitions = partArgs.size();
+        totalIndexes = idxArgs.size();
+
         for (T2<CacheGroupContext, GridDhtLocalPartition> t2 : partArgs)
             procPartFutures.add(processPartitionAsync(t2.get1(), t2.get2()));
 
         for (T2<GridCacheContext, Index> t2 : idxArgs)
             procIdxFutures.add(processIndexAsync(t2.get1(), t2.get2()));
 
-        totalPartitions = procPartFutures.size();
-        totalIndexes = procIdxFutures.size();
-
         Map<PartitionKey, ValidateIndexesPartitionResult> partResults = new HashMap<>();
         Map<String, ValidateIndexesPartitionResult> idxResults = new HashMap<>();
 
@@ -250,6 +270,9 @@ private VisorValidateIndexesJobResult call0() throws Exception {
 
                 idxResults.putAll(idxRes);
             }
+
+            log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and "
+                    + totalIndexes + " indexes.");
         }
         catch (InterruptedException | ExecutionException e) {
             for (int j = curPart; j < procPartFutures.size(); j++)
@@ -258,15 +281,102 @@ private VisorValidateIndexesJobResult call0() throws Exception {
             for (int j = curIdx; j < procIdxFutures.size(); j++)
                 procIdxFutures.get(j).cancel(false);
 
-            if (e instanceof InterruptedException)
-                throw new IgniteInterruptedException((InterruptedException)e);
-            else if (e.getCause() instanceof IgniteException)
-                throw (IgniteException)e.getCause();
-            else
-                throw new IgniteException(e.getCause());
+            throw unwrapFutureException(e);
+        }
+
+        return new VisorValidateIndexesJobResult(partResults, idxResults, integrityCheckResults.values());
+    }
+
+    /**
+     * @param grpIds Group ids.
+     */
+    private Map<Integer, IndexIntegrityCheckIssue> integrityCheckIndexesPartitions(Set<Integer> grpIds) {
+        List<Future<T2<Integer, IndexIntegrityCheckIssue>>> integrityCheckFutures = new ArrayList<>(grpIds.size());
+
+        for (Integer grpId: grpIds) {
+            final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
+
+            if (grpCtx == null) {
+                integrityCheckedIndexes.incrementAndGet();
+
+                continue;
+            }
+
+            Future<T2<Integer, IndexIntegrityCheckIssue>> checkFut =
+                    calcExecutor.submit(new Callable<T2<Integer, IndexIntegrityCheckIssue>>() {
+                        @Override public T2<Integer, IndexIntegrityCheckIssue> call() throws Exception {
+                            IndexIntegrityCheckIssue issue = integrityCheckIndexPartition(grpCtx);
+
+                            return new T2<>(grpCtx.groupId(), issue);
+                        }
+                    });
+
+            integrityCheckFutures.add(checkFut);
+        }
+
+        Map<Integer, IndexIntegrityCheckIssue> integrityCheckResults = new HashMap<>();
+
+        int curFut = 0;
+        try {
+            for (Future<T2<Integer, IndexIntegrityCheckIssue>> fut : integrityCheckFutures) {
+                T2<Integer, IndexIntegrityCheckIssue> res = fut.get();
+
+                if (res.getValue() != null)
+                    integrityCheckResults.put(res.getKey(), res.getValue());
+            }
+        }
+        catch (InterruptedException | ExecutionException e) {
+            for (int j = curFut; j < integrityCheckFutures.size(); j++)
+                integrityCheckFutures.get(j).cancel(false);
+
+            throw unwrapFutureException(e);
+        }
+
+        return integrityCheckResults;
+    }
+
+    /**
+     * @param gctx Cache group context.
+     */
+    private IndexIntegrityCheckIssue integrityCheckIndexPartition(CacheGroupContext gctx) {
+        GridKernalContext ctx = ignite.context();
+        GridCacheSharedContext cctx = ctx.cache().context();
+
+        try {
+            FilePageStoreManager pageStoreMgr = (FilePageStoreManager)cctx.pageStore();
+
+            if (pageStoreMgr == null)
+                return null;
+
+            int pageSz = gctx.dataRegion().pageMemory().pageSize();
+
+            PageStore pageStore = pageStoreMgr.getStore(gctx.groupId(), PageIdAllocator.INDEX_PARTITION);
+
+            long pageId = PageIdUtils.pageId(PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX, 0);
+
+            ByteBuffer buf = ByteBuffer.allocateDirect(pageSz);
+
+            buf.order(ByteOrder.nativeOrder());
+
+            for (int pageNo = 0; pageNo < pageStore.pages(); pageId++, pageNo++) {
+                buf.clear();
+
+                pageStore.read(pageId, buf, true);
+            }
+
+            return null;
         }
+        catch (Throwable t) {
+            log.error("Integrity check of index partition of cache group " + gctx.cacheOrGroupName() + " failed", t);
+
+            return new IndexIntegrityCheckIssue(gctx.cacheOrGroupName(), t);
+        }
+        finally {
+            integrityCheckedIndexes.incrementAndGet();
 
-        return new VisorValidateIndexesJobResult(partResults, idxResults);
+            printProgressIfNeeded("Current progress of ValidateIndexesClosure: checked integrity of "
+                    + integrityCheckedIndexes.get() + " index partitions of " + totalCacheGrps + " cache groups");
+        }
     }
 
     /**
@@ -371,6 +481,19 @@ else if (current++ % checkThrough > 0)
                 if (cacheCtx == null)
                     throw new IgniteException("Unknown cacheId of CacheDataRow: " + cacheId);
 
+                if (row.link() == 0L) {
+                    String errMsg = "Invalid partition row, possibly deleted";
+
+                    log.error(errMsg);
+
+                    IndexValidationIssue is = new IndexValidationIssue(null, cacheCtx.name(), null,
+                            new IgniteCheckedException(errMsg));
+
+                    enoughIssues |= partRes.reportIssue(is);
+
+                    continue;
+                }
+
                 try {
                     QueryTypeDescriptorImpl res = (QueryTypeDescriptorImpl)m.invoke(
                         qryProcessor, cacheCtx.name(), cacheCtx.cacheObjectContext(), row.key(), row.value(), true);
@@ -392,6 +515,9 @@ else if (current++ % checkThrough > 0)
                     ArrayList<Index> indexes = gridH2Tbl.getIndexes();
 
                     for (Index idx : indexes) {
+                        if (!(idx instanceof H2TreeIndex))
+                            continue;
+
                         try {
                             Cursor cursor = idx.find((Session) null, h2Row, h2Row);
 
@@ -434,7 +560,7 @@ else if (current++ % checkThrough > 0)
         finally {
             part.release();
 
-            printProgressIfNeeded();
+            printProgressOfIndexValidationIfNeeded();
         }
 
         PartitionKey partKey = new PartitionKey(grpCtx.groupId(), part.id(), grpCtx.cacheOrGroupName());
@@ -447,16 +573,21 @@ else if (current++ % checkThrough > 0)
     /**
      *
      */
-    private void printProgressIfNeeded() {
-        long curTs = U.currentTimeMillis();
+    private void printProgressOfIndexValidationIfNeeded() {
+        printProgressIfNeeded("Current progress of ValidateIndexesClosure: processed " +
+                processedPartitions.get() + " of " + totalPartitions + " partitions, " +
+                processedIndexes.get() + " of " + totalIndexes + " SQL indexes");
+    }
 
+    /**
+     *
+     */
+    private void printProgressIfNeeded(String msg) {
+        long curTs = U.currentTimeMillis();
         long lastTs = lastProgressPrintTs.get();
 
-        if (curTs - lastTs >= 60_000 && lastProgressPrintTs.compareAndSet(lastTs, curTs)) {
-            log.warning("Current progress of ValidateIndexesClosure: processed " +
-                processedPartitions.get() + " of " + totalPartitions + " partitions, " +
-                processedIndexes.get() + " of " + totalIndexes + " SQL indexes");
-        }
+        if (curTs - lastTs >= 60_000 && lastProgressPrintTs.compareAndSet(lastTs, curTs))
+            log.warning(msg);
     }
 
     /**
@@ -546,12 +677,14 @@ else if (current++ % checkThrough > 0)
 
                 h2key = h2Row.key();
 
-                CacheDataRow cacheDataStoreRow = ctx.group().offheap().read(ctx, h2key);
+                if (h2Row.link() != 0L) {
+                    CacheDataRow cacheDataStoreRow = ctx.group().offheap().read(ctx, h2key);
 
-                if (cacheDataStoreRow == null)
-                    throw new IgniteCheckedException("Key is present in SQL index, but can't be found in CacheDataTree.");
-
-                previousKey = h2key;
+                    if (cacheDataStoreRow == null)
+                        throw new IgniteCheckedException("Key is present in SQL index, but can't be found in CacheDataTree.");
+                }
+                else
+                    throw new IgniteCheckedException("Invalid index row, possibly deleted " + h2Row);
             }
             catch (Throwable t) {
                 Object o = CacheObjectUtils.unwrapBinaryIfNeeded(
@@ -564,14 +697,34 @@ else if (current++ % checkThrough > 0)
 
                 enoughIssues |= idxValidationRes.reportIssue(is);
             }
+            finally {
+                previousKey = h2key;
+            }
         }
 
         String uniqueIdxName = "[cache=" + ctx.name() + ", idx=" + idx.getName() + "]";
 
         processedIndexes.incrementAndGet();
 
-        printProgressIfNeeded();
+        printProgressOfIndexValidationIfNeeded();
 
         return Collections.singletonMap(uniqueIdxName, idxValidationRes);
     }
+
+    /**
+     * @param e Future result exception.
+     * @return Unwrapped exception.
+     */
+    private IgniteException unwrapFutureException(Exception e) {
+        assert e instanceof InterruptedException || e instanceof ExecutionException : "Expecting either InterruptedException " +
+                "or ExecutionException";
+
+        if (e instanceof InterruptedException)
+            return new IgniteInterruptedException((InterruptedException)e);
+        else if (e.getCause() instanceof IgniteException)
+            return  (IgniteException)e.getCause();
+        else
+            return new IgniteException(e.getCause());
+    }
+
 }
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
index 48e94c1e62cce..c7693d214b0ac 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -17,6 +17,9 @@
 
 package org.apache.ignite.util;
 
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -24,8 +27,8 @@
 import java.util.concurrent.ThreadLocalRandom;
 import javax.cache.Cache;
 import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.CacheAtomicityMode;
 import org.apache.ignite.cache.CacheWriteSynchronizationMode;
 import org.apache.ignite.cache.QueryEntity;
@@ -39,6 +42,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.tree.SearchRow;
 import org.apache.ignite.internal.processors.query.GridQueryProcessor;
 import org.apache.ignite.internal.util.lang.GridIterator;
@@ -47,57 +51,35 @@
 import org.apache.ignite.internal.util.typedef.internal.U;
 
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
 
 /**
  *
  */
 public class GridCommandHandlerIndexingTest extends GridCommandHandlerTest {
+    /** Test cache name. */
+    private static final String CACHE_NAME = "persons-cache-vi";
+
     /**
      * Tests that validation doesn't fail if nothing is broken.
      */
     public void testValidateIndexesNoErrors() throws Exception {
-        Ignite ignite = startGrids(2);
-
-        ignite.cluster().active(true);
-
-        Ignite client = startGrid("client");
-
-        String cacheName = "persons-cache-vi";
-
-        IgniteCache<Integer, Person> personCache = createPersonCache(client, cacheName);
-
-        ThreadLocalRandom rand = ThreadLocalRandom.current();
-
-        for (int i = 0; i < 10_000; i++)
-            personCache.put(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong())));
+        prepareGridForTest();
 
         injectTestSystemOut();
 
-        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", cacheName));
+        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", CACHE_NAME));
 
-        assertTrue(testOut.toString().contains("validate_indexes has finished, no issues found"));
+        assertTrue(testOut.toString().contains("no issues found"));
     }
 
     /**
      * Tests that missing rows in CacheDataTree are detected.
      */
     public void testBrokenCacheDataTreeShouldFailValidation() throws Exception {
-        Ignite ignite = startGrids(2);
+        Ignite ignite = prepareGridForTest();
 
-        ignite.cluster().active(true);
-
-        Ignite client = startGrid("client");
-
-        String cacheName = "persons-cache-vi";
-
-        IgniteCache<Integer, Person> personCache = createPersonCache(client, cacheName);
-
-        ThreadLocalRandom rand = ThreadLocalRandom.current();
-
-        for (int i = 0; i < 10_000; i++)
-            personCache.put(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong())));
-
-        breakCacheDataTree(ignite, cacheName, 1);
+        breakCacheDataTree(ignite, CACHE_NAME, 1);
 
         injectTestSystemOut();
 
@@ -105,11 +87,11 @@ public void testBrokenCacheDataTreeShouldFailValidation() throws Exception {
             execute(
                 "--cache",
                 "validate_indexes",
-                cacheName,
+                CACHE_NAME,
                 "checkFirst", "10000",
                 "checkThrough", "10"));
 
-        assertTrue(testOut.toString().contains("validate_indexes has finished with errors"));
+        assertTrue(testOut.toString().contains("issues found (listed above)"));
 
         assertTrue(testOut.toString().contains(
             "Key is present in SQL index, but is missing in corresponding data page."));
@@ -119,6 +101,46 @@ public void testBrokenCacheDataTreeShouldFailValidation() throws Exception {
      * Tests that missing rows in H2 indexes are detected.
      */
     public void testBrokenSqlIndexShouldFailValidation() throws Exception {
+        Ignite ignite = prepareGridForTest();
+
+        breakSqlIndex(ignite, CACHE_NAME);
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", CACHE_NAME));
+
+        assertTrue(testOut.toString().contains("issues found (listed above)"));
+    }
+
+    /**
+     * Tests that missing rows in H2 indexes are detected.
+     */
+    public void testCorruptedIndexPartitionShouldFailValidation() throws Exception {
+        Ignite ignite = prepareGridForTest();
+
+        forceCheckpoint();
+
+        File idxPath = indexPartition(ignite, CACHE_NAME);
+
+        stopAllGrids();
+
+        corruptIndexPartition(idxPath);
+
+        startGrids(2);
+
+        awaitPartitionMapExchange();
+
+        injectTestSystemOut();
+
+        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", CACHE_NAME));
+
+        assertTrue(testOut.toString().contains("issues found (listed above)"));
+    }
+
+    /**
+     *
+     */
+    private Ignite prepareGridForTest() throws Exception{
         Ignite ignite = startGrids(2);
 
         ignite.cluster().active(true);
@@ -127,20 +149,52 @@ public void testBrokenSqlIndexShouldFailValidation() throws Exception {
 
         String cacheName = "persons-cache-vi";
 
-        IgniteCache<Integer, Person> personCache = createPersonCache(client, cacheName);
+        client.getOrCreateCache(new CacheConfiguration<Integer, Person>()
+                .setName(cacheName)
+                .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+                .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+                .setBackups(1)
+                .setQueryEntities(F.asList(personEntity(true, true)))
+                .setAffinity(new RendezvousAffinityFunction(false, 32)));
 
         ThreadLocalRandom rand = ThreadLocalRandom.current();
 
-        for (int i = 0; i < 10_000; i++)
-            personCache.put(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong())));
+        try (IgniteDataStreamer<Integer, Person> streamer = client.dataStreamer(CACHE_NAME);) {
+            for (int i = 0; i < 10_000; i++)
+                streamer.addData(i, new Person(rand.nextInt(), String.valueOf(rand.nextLong())));
+        }
 
-        breakSqlIndex(ignite, cacheName);
+        return ignite;
+    }
 
-        injectTestSystemOut();
+    /**
+     * Get index partition file for specific node and cache.
+     */
+    private File indexPartition(Ignite ig, String cacheName) {
+        IgniteEx ig0 = (IgniteEx)ig;
 
-        assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", cacheName));
+        FilePageStoreManager pageStoreManager = ((FilePageStoreManager) ig0.context().cache().context().pageStore());
 
-        assertTrue(testOut.toString().contains("validate_indexes has finished with errors"));
+        return new File(pageStoreManager.cacheWorkDir(false, cacheName), INDEX_FILE_NAME);
+    }
+
+    /**
+     * Write some random trash in index partition.
+     */
+    private void corruptIndexPartition(File path) throws IOException {
+        assertTrue(path.exists());
+
+        ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+        try (RandomAccessFile idx = new RandomAccessFile(path, "rw")) {
+            byte[] trash = new byte[1024];
+
+            rand.nextBytes(trash);
+
+            idx.seek(4096);
+
+            idx.write(trash);
+        }
     }
 
     /**
@@ -241,22 +295,6 @@ private void breakSqlIndex(Ignite ig, String cacheName) throws Exception {
         }
     }
 
-    /**
-     * Dynamically creates cache with SQL indexes.
-     *
-     * @param ig Client.
-     * @param cacheName Cache name.
-     */
-    private IgniteCache<Integer, Person> createPersonCache(Ignite ig, String cacheName) {
-        return ig.getOrCreateCache(new CacheConfiguration<Integer, Person>()
-            .setName(cacheName)
-            .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
-            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
-            .setBackups(1)
-            .setQueryEntities(F.asList(personEntity(true, true)))
-            .setAffinity(new RendezvousAffinityFunction(false, 32)));
-    }
-
     /**
      * @param idxName Index name.
      * @param idxOrgId Index org id.

From a190e9ee1dc324ab94f57834b00d4962983bcd3c Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Mon, 29 Oct 2018 13:40:24 +0300
Subject: [PATCH 462/543] IGNITE-9790 Fixed delayed full message handling in
 case of changed coordinator. - Fixes #5190.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 2906a16522279802a6002f9fba009978225afad3)
---
 .../GridDhtPartitionsExchangeFuture.java      | 14 ++--
 .../GridDhtPartitionsFullMessage.java         | 16 ++++-
 ...itionsExchangeCoordinatorFailoverTest.java | 64 +++++++++++++++++++
 3 files changed, 85 insertions(+), 9 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 2e938c622a2b4..3239782a17c49 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -174,7 +174,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     private List<ClusterNode> srvNodes;
 
     /** */
-    private ClusterNode crd;
+    private volatile ClusterNode crd;
 
     /** ExchangeFuture id. */
     private final GridDhtPartitionExchangeId exchId;
@@ -4308,12 +4308,12 @@ public boolean reconnectOnError(Throwable e) {
      * @return {@code True} If partition changes triggered by receiving Single/Full messages are not finished yet.
      */
     public boolean partitionChangesInProgress() {
-        boolean isCoordinator = crd.equals(cctx.localNode());
+        ClusterNode crd0 = crd;
 
-        if (isCoordinator)
-            return !partitionsSent;
-        else
-            return !partitionsReceived;
+        if (crd0 == null)
+            return false;
+
+        return crd0.equals(cctx.localNode()) ? !partitionsSent : !partitionsReceived;
     }
 
     /**
@@ -4347,7 +4347,7 @@ public synchronized boolean addOrMergeDelayedFullMessage(ClusterNode node, GridD
             });
         }
         else
-            delayedLatestMsg.merge(fullMsg);
+            delayedLatestMsg.merge(fullMsg, cctx.discovery());
 
         return true;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index 8d34b340b872b..d915426619449 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -25,8 +25,10 @@
 import java.util.Set;
 import java.util.UUID;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridDirectMap;
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
@@ -803,7 +805,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
      *
      * @param other Other full message.
      */
-    public void merge(GridDhtPartitionsFullMessage other) {
+    public void merge(GridDhtPartitionsFullMessage other, GridDiscoveryManager discovery) {
         assert other.exchangeId() == null && exchangeId() == null :
             "Both current and merge full message must have exchangeId == null"
              + other.exchangeId() + "," + exchangeId();
@@ -814,8 +816,18 @@ public void merge(GridDhtPartitionsFullMessage other) {
 
             GridDhtPartitionFullMap currMap = partitions().get(grpId);
 
-            if (currMap == null || updMap.compareTo(currMap) >= 0)
+            if (currMap == null)
                 partitions().put(grpId, updMap);
+            else {
+                ClusterNode currentMapSentBy = discovery.node(currMap.nodeId());
+                ClusterNode newMapSentBy = discovery.node(updMap.nodeId());
+
+                if (newMapSentBy == null)
+                    return;
+
+                if (currentMapSentBy == null || newMapSentBy.order() > currentMapSentBy.order() || updMap.compareTo(currMap) >= 0)
+                    partitions().put(grpId, updMap);
+            }
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java
index a2adcf77a7bac..1c847f9fea843 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java
@@ -152,4 +152,68 @@ public void testNewCoordinatorCompletedExchange() throws Exception {
             cache.put(0, 0);
         }
     }
+
+    /**
+     * Test checks that delayed full messages are processed correctly in case of changed coordinator.
+     *
+     * @throws Exception If failed.
+     */
+    public void testDelayedFullMessageReplacedIfCoordinatorChanged() throws Exception {
+        IgniteEx crd = startGrid("crd");
+
+        IgniteEx newCrd = startGrid(1);
+
+        IgniteEx problemNode = startGrid(2);
+
+        crd.cluster().active(true);
+
+        awaitPartitionMapExchange();
+
+        blockSendingFullMessage(crd, problemNode);
+
+        IgniteInternalFuture joinNextNodeFut = GridTestUtils.runAsync(() -> startGrid(3));
+
+        joinNextNodeFut.get();
+
+        U.sleep(5000);
+
+        blockSendingFullMessage(newCrd, problemNode);
+
+        IgniteInternalFuture stopCoordinatorFut = GridTestUtils.runAsync(() -> stopGrid("crd"));
+
+        stopCoordinatorFut.get();
+
+        U.sleep(5000);
+
+        TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(newCrd);
+
+        spi.stopBlock(true);
+
+        awaitPartitionMapExchange();
+    }
+
+    /**
+     * Blocks sending full message from coordinator to non-coordinator node.
+     * @param from Coordinator node.
+     * @param to Non-coordinator node.
+     */
+    private void blockSendingFullMessage(IgniteEx from, IgniteEx to) {
+        // Block FullMessage for newly joined nodes.
+        TestRecordingCommunicationSpi spi = TestRecordingCommunicationSpi.spi(from);
+
+        // Delay sending full messages (without exchange id).
+        spi.blockMessages((node, msg) -> {
+            if (msg instanceof GridDhtPartitionsFullMessage) {
+                GridDhtPartitionsFullMessage fullMsg = (GridDhtPartitionsFullMessage) msg;
+
+                if (fullMsg.exchangeId() != null && node.order() == to.localNode().order()) {
+                    log.warning("Blocked sending " + msg + " to " + to.localNode());
+
+                    return true;
+                }
+            }
+
+            return false;
+        });
+    }
 }

From 8a1a9aad9bf27a7c4f6f947730db00a7e3a82d0f Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Mon, 29 Oct 2018 14:57:10 +0300
Subject: [PATCH 463/543] IGNITE-9790 Fixed compilation.

---
 .../cache/PartitionsExchangeCoordinatorFailoverTest.java        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java
index 1c847f9fea843..c63450db4664a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/PartitionsExchangeCoordinatorFailoverTest.java
@@ -159,7 +159,7 @@ public void testNewCoordinatorCompletedExchange() throws Exception {
      * @throws Exception If failed.
      */
     public void testDelayedFullMessageReplacedIfCoordinatorChanged() throws Exception {
-        IgniteEx crd = startGrid("crd");
+        IgniteEx crd = (IgniteEx) startGrid("crd");
 
         IgniteEx newCrd = startGrid(1);
 

From 561bc56a497a84846f362d360dc28f113136d511 Mon Sep 17 00:00:00 2001
From: Ilya Borisov <iborisov@gridgain.com>
Date: Mon, 29 Oct 2018 20:13:57 +0700
Subject: [PATCH 464/543] IGNITE-9941 Web Console: Added configuration
 parameter to disable signup and possibility to create user accounts by
 administrator.

---
 modules/web-console/DEVNOTES.txt              |  17 ++
 modules/web-console/assembly/README.txt       |   1 +
 modules/web-console/backend/app/settings.js   |   5 +-
 .../backend/config/settings.json.sample       |   5 +-
 modules/web-console/backend/routes/public.js  |  22 +-
 modules/web-console/backend/services/mails.js |  24 +-
 modules/web-console/backend/services/users.js |  12 +-
 modules/web-console/frontend/.eslintrc        |   1 -
 modules/web-console/frontend/app/app.js       |   6 +-
 .../dialog-admin-create-user/component.ts     |  27 +++
 .../dialog-admin-create-user/controller.ts    |  78 +++++++
 .../dialog-admin-create-user/index.ts         |  23 ++
 .../dialog-admin-create-user/state.ts         |  29 +++
 .../dialog-admin-create-user/template.pug     |  37 +++
 .../app/components/form-signup/component.ts   |  32 +++
 .../app/components/form-signup/controller.ts  |  46 ++++
 .../types.ts => form-signup/index.ts}         |   5 +
 .../app/components/form-signup/style.scss     |  31 +++
 .../app/components/form-signup/template.pug   | 105 +++++++++
 .../input-dialog/input-dialog.service.ts      | 172 ++++++++++++++
 .../input-dialog/input-dialog.tpl.pug         |  14 ++
 .../list-of-registered-users/column-defs.js   |  23 --
 .../list-of-registered-users/controller.js    | 214 ++++++++++--------
 .../list-of-registered-users/template.tpl.pug |   1 -
 .../app/components/page-admin/controller.ts   |  28 +++
 .../app/components/page-admin/index.js        |  17 +-
 .../app/components/page-signup/component.js   |   2 -
 .../{controller.js => controller.ts}          |  48 ++--
 .../app/components/page-signup/index.js       |   3 +-
 .../app/components/page-signup/style.scss     |  10 -
 .../app/components/page-signup/template.pug   |  94 +-------
 .../user/{Auth.service.js => Auth.service.ts} |  90 ++++----
 .../app/primitives/dropdown/index.pug         |   3 +-
 .../app/primitives/form-field/checkbox.pug    |   5 +-
 .../app/primitives/form-field/email.pug       |   7 +-
 .../app/primitives/form-field/password.pug    |  12 +-
 .../app/primitives/form-field/phone.pug       |   5 +-
 .../frontend/app/utils/dialogState.ts         |  56 +++++
 38 files changed, 963 insertions(+), 347 deletions(-)
 create mode 100644 modules/web-console/frontend/app/components/dialog-admin-create-user/component.ts
 create mode 100644 modules/web-console/frontend/app/components/dialog-admin-create-user/controller.ts
 create mode 100644 modules/web-console/frontend/app/components/dialog-admin-create-user/index.ts
 create mode 100644 modules/web-console/frontend/app/components/dialog-admin-create-user/state.ts
 create mode 100644 modules/web-console/frontend/app/components/dialog-admin-create-user/template.pug
 create mode 100644 modules/web-console/frontend/app/components/form-signup/component.ts
 create mode 100644 modules/web-console/frontend/app/components/form-signup/controller.ts
 rename modules/web-console/frontend/app/components/{page-signup/types.ts => form-signup/index.ts} (89%)
 create mode 100644 modules/web-console/frontend/app/components/form-signup/style.scss
 create mode 100644 modules/web-console/frontend/app/components/form-signup/template.pug
 create mode 100644 modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts
 create mode 100644 modules/web-console/frontend/app/components/page-admin/controller.ts
 rename modules/web-console/frontend/app/components/page-signup/{controller.js => controller.ts} (53%)
 rename modules/web-console/frontend/app/modules/user/{Auth.service.js => Auth.service.ts} (51%)
 create mode 100644 modules/web-console/frontend/app/utils/dialogState.ts

diff --git a/modules/web-console/DEVNOTES.txt b/modules/web-console/DEVNOTES.txt
index ba61150b41bd9..a06608d14ad8a 100644
--- a/modules/web-console/DEVNOTES.txt
+++ b/modules/web-console/DEVNOTES.txt
@@ -139,3 +139,20 @@ Unit tests are performed with Mocha framework - https://mochajs.org
 To launch tests on your local machine you will need:
 1. In new terminal change directory to 'modules/web-console/backend' folder and execute: "npm install".
 2. To start test environment and tests execute: "npm run test".
+
+
+Web Console settings
+====================
+Web Console backend could be configured with custom parameters.
+
+See "backend/config/settings.json.sample" for list of parameters with example value.
+
+If you need custom parameters, you will need create "backend/config/settings.json" and adjust values.
+
+Web Console settings for Docker
+===============================
+Web Console backend could be configured with custom parameters.
+
+You may pass custom parameters with help of "-e" option.
+
+For example: docker run -e "server_disable_signup=true" -p 9090:80 $IMAGE_ID
diff --git a/modules/web-console/assembly/README.txt b/modules/web-console/assembly/README.txt
index dd323993e5231..f1e114c523b0a 100644
--- a/modules/web-console/assembly/README.txt
+++ b/modules/web-console/assembly/README.txt
@@ -47,6 +47,7 @@ All available parameters with defaults:
     HTTPS key:                  --server:key "serve/keys/test.key"
     HTTPS certificate:          --server:cert "serve/keys/test.crt"
     HTTPS passphrase:           --server:keyPassphrase "password"
+    Disable self registration:  --server:disable:signup false
     MongoDB URL:                --mongodb:url mongodb://localhost/console
     Mail service:               --mail:service "gmail"
     Signature text:             --mail:sign "Kind regards, Apache Ignite Team"
diff --git a/modules/web-console/backend/app/settings.js b/modules/web-console/backend/app/settings.js
index d206107843908..067864475af7c 100644
--- a/modules/web-console/backend/app/settings.js
+++ b/modules/web-console/backend/app/settings.js
@@ -53,6 +53,8 @@ module.exports = {
         const dfltHost = packaged ? '0.0.0.0' : '127.0.0.1';
         const dfltPort = packaged ? 80 : 3000;
 
+        const _disableSignup = nconf.get('server:disable:signup');
+
         return {
             agent: {
                 dists: nconf.get('agent:dists') || dfltAgentDists
@@ -67,7 +69,8 @@ module.exports = {
                     key: fs.readFileSync(nconf.get('server:key')),
                     cert: fs.readFileSync(nconf.get('server:cert')),
                     passphrase: nconf.get('server:keyPassphrase')
-                }
+                },
+                disableSignup: _disableSignup === 'true' || _disableSignup === true
             },
             mail,
             mongoUrl: nconf.get('mongodb:url') || 'mongodb://127.0.0.1/console',
diff --git a/modules/web-console/backend/config/settings.json.sample b/modules/web-console/backend/config/settings.json.sample
index aa93e07c57eba..aa20ab46f5c92 100644
--- a/modules/web-console/backend/config/settings.json.sample
+++ b/modules/web-console/backend/config/settings.json.sample
@@ -5,7 +5,10 @@
         "ssl": false,
         "key": "serve/keys/test.key",
         "cert": "serve/keys/test.crt",
-        "keyPassphrase": "password"
+        "keyPassphrase": "password",
+        "disable": {
+            "signup": false
+        }
     },
     "mongodb": {
         "url": "mongodb://localhost/console"
diff --git a/modules/web-console/backend/routes/public.js b/modules/web-console/backend/routes/public.js
index 155b2e9f268e4..8aa3b4338db4b 100644
--- a/modules/web-console/backend/routes/public.js
+++ b/modules/web-console/backend/routes/public.js
@@ -17,6 +17,7 @@
 
 'use strict';
 
+const _ = require('lodash');
 const express = require('express');
 const passport = require('passport');
 
@@ -49,15 +50,22 @@ module.exports.factory = function(mongo, mailsService, usersService, authService
          * Register new account.
          */
         router.post('/signup', (req, res) => {
-            usersService.create(req.origin(), req.body)
-                .then((user) => new Promise((resolve, reject) => {
-                    req.logIn(user, {}, (err) => {
-                        if (err)
-                            reject(err);
+            const createdByAdmin = _.get(req, 'user.admin', false);
 
-                        resolve(user);
+            usersService.create(req.origin(), req.body, createdByAdmin)
+                .then((user) => {
+                    if (createdByAdmin)
+                        return user;
+
+                    return new Promise((resolve, reject) => {
+                        req.logIn(user, {}, (err) => {
+                            if (err)
+                                reject(err);
+
+                            resolve(user);
+                        });
                     });
-                }))
+                })
                 .then(res.api.ok)
                 .catch(res.api.error);
         });
diff --git a/modules/web-console/backend/services/mails.js b/modules/web-console/backend/services/mails.js
index e0737cd2a7500..75da128ad9dca 100644
--- a/modules/web-console/backend/services/mails.js
+++ b/modules/web-console/backend/services/mails.js
@@ -81,17 +81,27 @@ module.exports.factory = (settings) => {
         }
 
         /**
-         * Send email to user for password reset.
-         * @param host
-         * @param user
+         * Send email when user signed up.
+         *
+         * @param host Web Console host.
+         * @param user User that signed up.
+         * @param createdByAdmin Whether user was created by admin.
          */
-        emailUserSignUp(host, user) {
+        emailUserSignUp(host, user, createdByAdmin) {
             const resetLink = `${host}/password/reset?token=${user.resetPasswordToken}`;
 
-            return this.send(user, `Thanks for signing up for ${settings.mail.greeting}.`,
+            const sbj = createdByAdmin
+                ? 'Account was created for'
+                : 'Thanks for signing up for';
+
+            const reason = createdByAdmin
+                ? 'administrator created account for you'
+                : 'you have signed up';
+
+            return this.send(user, `${sbj} ${settings.mail.greeting}.`,
                 `Hello ${user.firstName} ${user.lastName}!<br><br>` +
-                `You are receiving this email because you have signed up to use <a href="${host}">${settings.mail.greeting}</a>.<br><br>` +
-                'If you have not done the sign up and do not know what this email is about, please ignore it.<br>' +
+                `You are receiving this email because ${reason} to use <a href="${host}">${settings.mail.greeting}</a>.<br><br>` +
+                'If you do not know what this email is about, please ignore it.<br>' +
                 'You may reset the password by clicking on the following link, or paste this into your browser:<br><br>' +
                 `<a href="${resetLink}">${resetLink}</a>`);
         }
diff --git a/modules/web-console/backend/services/users.js b/modules/web-console/backend/services/users.js
index 43bceda3cd6d5..79578c5b45995 100644
--- a/modules/web-console/backend/services/users.js
+++ b/modules/web-console/backend/services/users.js
@@ -42,17 +42,21 @@ module.exports.factory = (errors, settings, mongo, spacesService, mailsService,
         /**
          * Save profile information.
          *
-         * @param {String} host - The host
-         * @param {Object} user - The user
+         * @param {String} host - The host.
+         * @param {Object} user - The user.
+         * @param {Object} createdByAdmin - Whether user created by admin.
          * @returns {Promise.<mongo.ObjectId>} that resolves account id of merge operation.
          */
-        static create(host, user) {
+        static create(host, user, createdByAdmin) {
             return mongo.Account.count().exec()
                 .then((cnt) => {
                     user.admin = cnt === 0;
                     user.registered = new Date();
                     user.token = utilsService.randomString(settings.tokenLength);
 
+                    if (settings.server.disableSignup && !user.admin && !createdByAdmin)
+                        throw new errors.ServerErrorException('Sign-up is not allowed. Ask your Web Console administrator to create account for you.');
+
                     return new mongo.Account(user);
                 })
                 .then((created) => {
@@ -74,7 +78,7 @@ module.exports.factory = (errors, settings, mongo, spacesService, mailsService,
                     return registered.save()
                         .then(() => mongo.Space.create({name: 'Personal space', owner: registered._id}))
                         .then(() => {
-                            mailsService.emailUserSignUp(host, registered);
+                            mailsService.emailUserSignUp(host, registered, createdByAdmin);
 
                             return registered;
                         });
diff --git a/modules/web-console/frontend/.eslintrc b/modules/web-console/frontend/.eslintrc
index ac1a37f55df81..8f6bf20970b31 100644
--- a/modules/web-console/frontend/.eslintrc
+++ b/modules/web-console/frontend/.eslintrc
@@ -163,7 +163,6 @@ rules:
     no-unused-expressions: [2, { allowShortCircuit: true }]
     no-unused-vars: [0, {"vars": "all", "args": "after-used"}]
     typescript/no-unused-vars: [0]
-    no-use-before-define: 2
     no-useless-call: 2
     no-void: 0
     no-var: 2
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 98792c4c50bb8..939fa71667749 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -112,6 +112,7 @@ import id8 from './filters/id8.filter';
 
 // Components
 import igniteListOfRegisteredUsers from './components/list-of-registered-users';
+import dialogAdminCreateUser from './components/dialog-admin-create-user';
 import IgniteActivitiesUserDialog from './components/activities-user-dialog';
 import './components/input-dialog';
 import webConsoleHeader from './components/web-console-header';
@@ -156,6 +157,7 @@ import pagePasswordReset from './components/page-password-reset';
 import pageSignup from './components/page-signup';
 import pageSignin from './components/page-signin';
 import pageForgotPassword from './components/page-forgot-password';
+import formSignup from './components/form-signup';
 
 import igniteServices from './services';
 
@@ -245,6 +247,7 @@ export default angular.module('ignite-console', [
     connectedClusters.name,
     connectedClustersDialog.name,
     igniteListOfRegisteredUsers.name,
+    dialogAdminCreateUser.name,
     pageProfile.name,
     pageLanding.name,
     pagePasswordChanged.name,
@@ -260,7 +263,8 @@ export default angular.module('ignite-console', [
     igniteChartSelector.name,
     igniteStatus.name,
     progressLine.name,
-    formField.name
+    formField.name,
+    formSignup.name
 ])
 .service('$exceptionHandler', $exceptionHandler)
 // Directives.
diff --git a/modules/web-console/frontend/app/components/dialog-admin-create-user/component.ts b/modules/web-console/frontend/app/components/dialog-admin-create-user/component.ts
new file mode 100644
index 0000000000000..3a60d93077030
--- /dev/null
+++ b/modules/web-console/frontend/app/components/dialog-admin-create-user/component.ts
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+import {DialogAdminCreateUser} from './controller';
+
+export default {
+    template,
+    controller: DialogAdminCreateUser,
+    bindings: {
+        close: '&onHide'
+    }
+};
diff --git a/modules/web-console/frontend/app/components/dialog-admin-create-user/controller.ts b/modules/web-console/frontend/app/components/dialog-admin-create-user/controller.ts
new file mode 100644
index 0000000000000..e8a29e14212ad
--- /dev/null
+++ b/modules/web-console/frontend/app/components/dialog-admin-create-user/controller.ts
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Auth from '../../modules/user/Auth.service';
+import MessagesFactory from '../../services/Messages.service';
+import FormUtilsFactoryFactory from '../../services/FormUtils.service';
+import LoadingServiceFactory from '../../modules/loading/loading.service';
+import {ISignupData} from '../form-signup';
+
+export class DialogAdminCreateUser {
+    close: ng.ICompiledExpression;
+
+    form: ng.IFormController;
+
+    data: ISignupData = {
+        email: null,
+        password: null,
+        firstName: null,
+        lastName: null,
+        company: null,
+        country: null
+    };
+
+    serverError: string | null = null;
+
+    static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils', 'IgniteLoading'];
+
+    constructor(
+        private Auth: Auth,
+        private IgniteMessages: ReturnType<typeof MessagesFactory>,
+        private IgniteFormUtils: ReturnType<typeof FormUtilsFactoryFactory>,
+        private loading: ReturnType<typeof LoadingServiceFactory>
+    ) {}
+
+    canSubmitForm(form: DialogAdminCreateUser['form']) {
+        return form.$error.server ? true : !form.$invalid;
+    }
+
+    setServerError(error: DialogAdminCreateUser['serverError']) {
+        this.serverError = error;
+    }
+
+    createUser() {
+        this.IgniteFormUtils.triggerValidation(this.form);
+
+        this.setServerError(null);
+
+        if (!this.canSubmitForm(this.form))
+            return;
+
+        this.loading.start('createUser');
+
+        this.Auth.signup(this.data, false)
+            .then(() => {
+                this.IgniteMessages.showInfo(`User ${this.data.email} created`);
+                this.close({});
+            })
+            .catch((res) => {
+                this.loading.finish('createUser');
+                this.IgniteMessages.showError(null, res.data);
+                this.setServerError(res.data);
+            });
+    }
+}
diff --git a/modules/web-console/frontend/app/components/dialog-admin-create-user/index.ts b/modules/web-console/frontend/app/components/dialog-admin-create-user/index.ts
new file mode 100644
index 0000000000000..5a23368229030
--- /dev/null
+++ b/modules/web-console/frontend/app/components/dialog-admin-create-user/index.ts
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import component from './component';
+import {registerState} from './state';
+
+export default angular.module('ignite-console.dialog-admin-create-user', [])
+    .run(registerState)
+    .component('dialogAdminCreateUser', component);
diff --git a/modules/web-console/frontend/app/components/dialog-admin-create-user/state.ts b/modules/web-console/frontend/app/components/dialog-admin-create-user/state.ts
new file mode 100644
index 0000000000000..c64238e6c882d
--- /dev/null
+++ b/modules/web-console/frontend/app/components/dialog-admin-create-user/state.ts
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {UIRouter} from '@uirouter/angularjs';
+import {dialogState} from '../../utils/dialogState';
+
+registerState.$inject = ['$uiRouter'];
+
+export function registerState(router: UIRouter) {
+    router.stateRegistry.register({
+        ...dialogState('dialog-admin-create-user'),
+        name: 'base.settings.admin.createUser',
+        url: '/create-user'
+    });
+}
diff --git a/modules/web-console/frontend/app/components/dialog-admin-create-user/template.pug b/modules/web-console/frontend/app/components/dialog-admin-create-user/template.pug
new file mode 100644
index 0000000000000..0a9f2b4c4b67c
--- /dev/null
+++ b/modules/web-console/frontend/app/components/dialog-admin-create-user/template.pug
@@ -0,0 +1,37 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+.modal-dialog(ng-click='$event.stopPropagation()' )
+    form.modal-content(
+        name='$ctrl.form' novalidate
+        ignite-loading='createUser'
+        ignite-loading-text='Creating user…'
+    )
+        .modal-header
+            h4.modal-title Create User
+
+            button.close(type='button' aria-label='Close' ng-click='$ctrl.close()')
+                svg(ignite-icon="cross")
+        .modal-body
+            form-signup(
+                outer-form='$ctrl.form'
+                ng-model='$ctrl.data'
+                server-error='$ctrl.serverError'
+            )
+        .modal-footer
+            div
+                button.btn-ignite.btn-ignite--link-success(ng-click='$ctrl.close()') Cancel
+                button.btn-ignite.btn-ignite--success(ng-click='$ctrl.createUser()') Create user
diff --git a/modules/web-console/frontend/app/components/form-signup/component.ts b/modules/web-console/frontend/app/components/form-signup/component.ts
new file mode 100644
index 0000000000000..79863db09c84b
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-signup/component.ts
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import template from './template.pug';
+import './style.scss';
+import {FormSignup} from './controller';
+
+export const component: ng.IComponentOptions = {
+    template,
+    controller: FormSignup,
+    bindings: {
+        outerForm: '<',
+        serverError: '<'
+    },
+    require: {
+        ngModel: 'ngModel'
+    }
+};
diff --git a/modules/web-console/frontend/app/components/form-signup/controller.ts b/modules/web-console/frontend/app/components/form-signup/controller.ts
new file mode 100644
index 0000000000000..eedf6d0ca0b07
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-signup/controller.ts
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CountriesService from '../../services/Countries.service';
+import {ISignupFormController} from '.';
+
+export class FormSignup implements ng.IPostLink, ng.IOnDestroy, ng.IOnChanges {
+    static $inject = ['IgniteCountries'];
+
+    constructor(private Countries: ReturnType<typeof CountriesService>) {}
+
+    countries = this.Countries.getAll();
+
+    innerForm: ISignupFormController;
+    outerForm: ng.IFormController;
+    ngModel: ng.INgModelController;
+    serverError: string | null = null;
+
+    $postLink() {
+        this.outerForm.$addControl(this.innerForm);
+        this.innerForm.email.$validators.server = () => !this.serverError;
+    }
+
+    $onDestroy() {
+        this.outerForm.$removeControl(this.innerForm);
+    }
+
+    $onChanges(changes: {serverError: ng.IChangesObject<FormSignup['serverError']>}) {
+        if (changes.serverError && this.innerForm)
+            this.innerForm.email.$validate();
+    }
+}
diff --git a/modules/web-console/frontend/app/components/page-signup/types.ts b/modules/web-console/frontend/app/components/form-signup/index.ts
similarity index 89%
rename from modules/web-console/frontend/app/components/page-signup/types.ts
rename to modules/web-console/frontend/app/components/form-signup/index.ts
index 49e8d5d15072b..1798aa0ae0ea6 100644
--- a/modules/web-console/frontend/app/components/page-signup/types.ts
+++ b/modules/web-console/frontend/app/components/form-signup/index.ts
@@ -15,6 +15,11 @@
  * limitations under the License.
  */
 
+import {component} from './component';
+
+export default angular.module('ignite-console.form-signup', [])
+    .component('formSignup', component);
+
 export interface ISignupData {
     email: string,
     password: string,
diff --git a/modules/web-console/frontend/app/components/form-signup/style.scss b/modules/web-console/frontend/app/components/form-signup/style.scss
new file mode 100644
index 0000000000000..e80b5108131ef
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-signup/style.scss
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+form-signup {
+    .form-signup__grid {
+        display: grid;
+        grid-gap: 10px;
+        grid-template-columns: 1fr 1fr;
+
+        .span-1 {
+            grid-column: span 1;
+        }
+        .span-2 {
+            grid-column: span 2;
+        }
+    }
+}
diff --git a/modules/web-console/frontend/app/components/form-signup/template.pug b/modules/web-console/frontend/app/components/form-signup/template.pug
new file mode 100644
index 0000000000000..375ea958e3d18
--- /dev/null
+++ b/modules/web-console/frontend/app/components/form-signup/template.pug
@@ -0,0 +1,105 @@
+//-
+    Licensed to the Apache Software Foundation (ASF) under one or more
+    contributor license agreements.  See the NOTICE file distributed with
+    this work for additional information regarding copyright ownership.
+    The ASF licenses this file to You under the Apache License, Version 2.0
+    (the "License"); you may not use this file except in compliance with
+    the License.  You may obtain a copy of the License at
+
+         http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+include /app/helpers/jade/mixins
+
+.form-signup__grid(ng-form='signup' ng-ref='$ctrl.innerForm' ng-ref-read='form')
+    .span-2
+        +form-field__email({
+            label: 'Email:',
+            model: '$ctrl.ngModel.$viewValue.email',
+            name: '"email"',
+            placeholder: 'Input email',
+            required: true
+        })(
+            ng-model-options='{allowInvalid: true}'
+            autocomplete='email'
+            ignite-auto-focus
+        )
+            +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`})
+    .span-1
+        +form-field__password({
+            label: 'Password:',
+            model: '$ctrl.ngModel.$viewValue.password',
+            name: '"password"',
+            placeholder: 'Input password',
+            required: true
+        })(
+            autocomplete='new-password'
+        )
+    .span-1
+        +form-field__password({
+            label: 'Confirm:',
+            model: 'confirm',
+            name: '"confirm"',
+            placeholder: 'Confirm password',
+            required: true
+        })(
+            ignite-match='$ctrl.ngModel.$viewValue.password'
+            autocomplete='off'
+        )
+    .span-1
+        +form-field__text({
+            label: 'First name:',
+            model: '$ctrl.ngModel.$viewValue.firstName',
+            name: '"firstName"',
+            placeholder: 'Input first name',
+            required: true
+        })(
+            autocomplete='given-name'
+        )
+    .span-1
+        +form-field__text({
+            label: 'Last name:',
+            model: '$ctrl.ngModel.$viewValue.lastName',
+            name: '"lastName"',
+            placeholder: 'Input last name',
+            required: true
+        })(
+            autocomplete='family-name'
+        )
+    .span-1
+        +form-field__phone({
+            label: 'Phone:',
+            model: '$ctrl.ngModel.$viewValue.phone',
+            name: '"phone"',
+            placeholder: 'Input phone (ex.: +15417543010)',
+            optional: true
+        })(
+            autocomplete='tel'
+        )
+    .span-1
+        +form-field__dropdown({
+            label: 'Country:',
+            model: '$ctrl.ngModel.$viewValue.country',
+            name: '"country"',
+            required: true,
+            placeholder: 'Choose your country',
+            options: '$ctrl.countries'
+        })(
+            autocomplete='country'
+        )
+    .span-2
+        +form-field__text({
+            label: 'Company:',
+            model: '$ctrl.ngModel.$viewValue.company',
+            name: '"company"',
+            placeholder: 'Input company name',
+            required: true
+        })(
+            ignite-on-enter-focus-move='countryInput'
+            autocomplete='organization'
+        )
diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts
new file mode 100644
index 0000000000000..b2a241fb9206a
--- /dev/null
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.service.ts
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import _ from 'lodash';
+import controller from './input-dialog.controller';
+import templateUrl from './input-dialog.tpl.pug';
+import {CancellationError} from 'app/errors/CancellationError';
+
+type InputModes = 'text' | 'number' | 'email' | 'date' | 'time' | 'date-and-time';
+
+interface ValidationFunction<T> {
+    (value: T): boolean
+}
+
+/**
+ * Options for rendering inputs.
+ */
+interface InputOptions<T> {
+    /** Input type. */
+    mode?: InputModes,
+    /** Dialog title. */
+    title?: string,
+    /** Input field label. */
+    label?: string,
+    /** Message for tooltip in label. */
+    tip?: string,
+    /** Default value. */
+    value: T,
+    /** Placeholder for input. */
+    placeholder?: string,
+    /** Validator function. */
+    toValidValue?: ValidationFunction<T>,
+    /** Min value for number input. */
+    min?: number,
+    /** Max value for number input. */
+    max?: number,
+    /** Postfix for units in number input. */
+    postfix?: string
+}
+
+export default class InputDialog {
+    static $inject = ['$modal', '$q'];
+
+    constructor(private $modal: mgcrea.ngStrap.modal.IModalService, private $q: ng.IQService) {}
+
+    /**
+     * Fabric for creating modal instance with different input types.
+     *
+     * @returns User input.
+     */
+    private dialogFabric<T>(args: InputOptions<T>) {
+        const deferred = this.$q.defer<T>();
+
+        const modal = this.$modal({
+            templateUrl,
+            resolve: {
+                deferred: () => deferred,
+                ui: () => args
+            },
+            controller,
+            controllerAs: 'ctrl'
+        });
+
+        const modalHide = modal.hide;
+
+        modal.hide = () => deferred.reject(new CancellationError());
+
+        return deferred.promise
+            .finally(modalHide);
+    }
+
+    /**
+     * Open input dialog to configure custom value.
+     *
+     * @param title Dialog title.
+     * @param label Input field label.
+     * @param value Default value.
+     * @param toValidValue Validator function.
+     * @param mode Input type.
+     */
+    input<T>(title: string, label: string, value: T, toValidValue?: ValidationFunction<T>, mode: InputModes = 'text') {
+        return this.dialogFabric<T>({title, label, value, toValidValue, mode});
+    }
+
+    /**
+     * Open input dialog to configure cloned object name.
+     *
+     * @param srcName Name of source object.
+     * @param names List of already exist names.
+     * @returns New name.
+     */
+    clone(srcName: string, names: Array<string>) {
+        const uniqueName = (value) => {
+            let num = 1;
+            let tmpName = value;
+
+            while (_.includes(names, tmpName)) {
+                tmpName = `${value}_${num}`;
+
+                num++;
+            }
+
+            return tmpName;
+        };
+
+        return this.input<string>('Clone', 'New name', uniqueName(srcName), uniqueName);
+    }
+
+    /**
+     * Open input dialog to configure custom number value.
+     *
+     * @param options Object with settings for rendering number input.
+     * @returns User input.
+     */
+    number(options: InputOptions<number>) {
+        return this.dialogFabric({mode: 'number', ...options});
+    }
+
+    /**
+     * Open input dialog to configure custom e-mail.
+     *
+     * @param options Object with settings for rendering e-mail input.
+     * @return User input.
+     */
+    email(options: InputOptions<string>) {
+        return this.dialogFabric({mode: 'email', ...options});
+    }
+
+    /**
+     * Open input dialog to configure custom date value.
+     *
+     * @param options Settings for rendering date input.
+     * @returns User input.
+     */
+    date(options: InputOptions<Date>) {
+        return this.dialogFabric({mode: 'date', ...options});
+    }
+
+    /**
+     * Open input dialog to configure custom time value.
+     *
+     * @param options Settings for rendering time input.
+     * @returns User input.
+     */
+    time(options: InputOptions<Date>) {
+        return this.dialogFabric({mode: 'time', ...options});
+    }
+
+    /**
+     * Open input dialog to configure custom date and time value.
+     *
+     * @param options Settings for rendering date and time inputs.
+     * @returns User input.
+     */
+    dateTime(options: InputOptions<Date>) {
+        return this.dialogFabric({mode: 'date-and-time', ...options});
+    }
+}
diff --git a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
index 95cb059ba6ff9..bc6aa0b4f9d20 100644
--- a/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
+++ b/modules/web-console/frontend/app/components/input-dialog/input-dialog.tpl.pug
@@ -39,6 +39,7 @@ include /app/helpers/jade/mixins
                             ignite-form-field-input-autofocus='true'
                             ignite-on-enter='form.$valid && ctrl.confirm()'
                         )
+
                 .row(ng-switch-when='number')
                     .col-100
                         +form-field__number({
@@ -53,6 +54,19 @@ include /app/helpers/jade/mixins
                             required: true
                         })
 
+                .row(ng-switch-when='email')
+                    .col-100
+                        +form-field__email({
+                            label: '{{ ctrl.options.label }}',
+                            model: 'ctrl.options.value',
+                            name: '"email"',
+                            required: true,
+                            placeholder: 'Input email'
+                        })(
+                            ignite-form-field-input-autofocus='true'
+                            ignite-on-enter='form.$valid && ctrl.confirm()'
+                        )
+
             .modal-footer
                 div
                     button#copy-btn-cancel.btn-ignite.btn-ignite--link-success(ng-click='$hide()') Cancel
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js b/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js
index 419a0636f23aa..2094f0c7909c7 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/column-defs.js
@@ -25,34 +25,11 @@ const MODEL_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip dat
 const CACHE_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-database'></i>${ICON_SORT}</div>`;
 const IGFS_HEADER_TEMPLATE = `<div class='ui-grid-cell-contents' bs-tooltip data-title='{{ col.headerTooltip(col) }}' data-placement='top'><i class='fa fa-folder-o'></i>${ICON_SORT}</div>`;
 
-const ACTIONS_TEMPLATE = `
-<div class='text-center ui-grid-cell-actions'>
-    <a class='btn btn-default dropdown-toggle' bs-dropdown='' data-placement='bottom-right' data-container='.panel'>
-        <i class='fa fa-gear'></i>&nbsp;
-        <span class='caret'></span>
-    </a>
-    <ul class='dropdown-menu' role='menu'>
-        <li ng-show='row.entity._id != $root.user._id'>
-            <a ng-click='grid.api.becomeUser(row.entity)'>Become this user</a>
-        </li>
-        <li ng-show='row.entity._id != $root.user._id'>
-            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='row.entity.admin && row.entity._id !== $root.user._id'>Revoke admin</a>
-            <a ng-click='grid.api.toggleAdmin(row.entity)' ng-if='!row.entity.admin && row.entity._id !== $root.user._id'>Grant admin</a>
-        </li>
-        <li ng-show='row.entity._id != $root.user._id'>
-            <a ng-click='grid.api.removeUser(row.entity)'>Remove user</a>
-        </li>
-        <li>
-            <a ng-click='grid.api.showActivities(row.entity)'>Activity detail</a>
-        </li>
-</div>`;
-
 const EMAIL_TEMPLATE = '<div class="ui-grid-cell-contents"><a bs-tooltip data-title="{{ COL_FIELD }}" ng-href="mailto:{{ COL_FIELD }}">{{ COL_FIELD }}</a></div>';
 const DATE_WITH_TITLE = '<div class="ui-grid-cell-contents"><label bs-tooltip data-title="{{ COL_FIELD | date:\'M/d/yy HH:mm\' }}">{{ COL_FIELD | date:"M/d/yy HH:mm" }}</label></div>';
 const VALUE_WITH_TITLE = '<div class="ui-grid-cell-contents"><label bs-tooltip data-title="{{ COL_FIELD }}">{{ COL_FIELD }}</label></div>';
 
 export default [
-    {name: 'actions', enableHiding: false, displayName: 'Actions', categoryDisplayName: 'Actions', cellTemplate: ACTIONS_TEMPLATE, field: 'actions', minWidth: 70, width: 70, enableFiltering: false, enableSorting: false, visible: false},
     {name: 'user', enableHiding: false, displayName: 'User', categoryDisplayName: 'User', field: 'userName', cellTemplate: USER_TEMPLATE, minWidth: 160, enableFiltering: true, pinnedLeft: true, filter: { placeholder: 'Filter by name...' }},
     {name: 'email', displayName: 'Email', categoryDisplayName: 'Email', field: 'email', cellTemplate: EMAIL_TEMPLATE, minWidth: 160, width: 220, enableFiltering: true, filter: { placeholder: 'Filter by email...' }},
     {name: 'company', displayName: 'Company', categoryDisplayName: 'Company', field: 'company', cellTemplate: VALUE_WITH_TITLE, minWidth: 180, enableFiltering: true, filter: { placeholder: 'Filter by company...' }},
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/controller.js b/modules/web-console/frontend/app/components/list-of-registered-users/controller.js
index f8b2797359f52..6a7e098df0f55 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/controller.js
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/controller.js
@@ -15,11 +15,13 @@
  * limitations under the License.
  */
 
-import headerTemplate from 'app/primitives/ui-grid-header/index.tpl.pug';
+import _ from 'lodash';
 
 import columnDefs from './column-defs';
 import categories from './categories';
 
+import headerTemplate from 'app/primitives/ui-grid-header/index.tpl.pug';
+
 const rowTemplate = `<div
   ng-repeat="(colRenderIndex, col) in colContainer.renderedColumns track by col.uid"
   ui-grid-one-bind-id-grid="rowRenderIndex + '-' + col.uid + '-cell'"
@@ -36,120 +38,80 @@ export default class IgniteListOfRegisteredUsersCtrl {
     static $inject = ['$scope', '$state', '$filter', 'User', 'uiGridGroupingConstants', 'uiGridPinningConstants', 'IgniteAdminData', 'IgniteNotebookData', 'IgniteConfirm', 'IgniteActivitiesUserDialog'];
 
     constructor($scope, $state, $filter, User, uiGridGroupingConstants, uiGridPinningConstants, AdminData, NotebookData, Confirm, ActivitiesUserDialog) {
-        const $ctrl = this;
+        this.$state = $state;
+        this.AdminData = AdminData;
+        this.ActivitiesDialogFactory = ActivitiesUserDialog;
+        this.Confirm = Confirm;
+        this.User = User;
+        this.NotebookData = NotebookData;
 
         const dtFilter = $filter('date');
 
-        $ctrl.groupBy = 'user';
+        this.groupBy = 'user';
 
-        $ctrl.selected = [];
+        this.selected = [];
 
-        $ctrl.params = {
+        this.params = {
             startDate: new Date(),
             endDate: new Date()
         };
 
-        $ctrl.uiGridPinningConstants = uiGridPinningConstants;
-        $ctrl.uiGridGroupingConstants = uiGridGroupingConstants;
-
-        User.read().then((user) => $ctrl.user = user);
-
-        const becomeUser = () => {
-            const user = this.gridApi.selection.legacyGetSelectedRows()[0];
-
-            AdminData.becomeUser(user._id)
-                .then(() => User.load())
-                .then(() => $state.go('default-state'))
-                .then(() => NotebookData.load());
-        };
-
-        const removeUser = () => {
-            const user = this.gridApi.selection.legacyGetSelectedRows()[0];
-
-            Confirm.confirm(`Are you sure you want to remove user: "${user.userName}"?`)
-                .then(() => AdminData.removeUser(user))
-                .then(() => {
-                    const i = _.findIndex($ctrl.gridOptions.data, (u) => u._id === user._id);
+        this.uiGridPinningConstants = uiGridPinningConstants;
+        this.uiGridGroupingConstants = uiGridGroupingConstants;
 
-                    if (i >= 0) {
-                        $ctrl.gridOptions.data.splice(i, 1);
-                        $ctrl.gridApi.selection.clearSelectedRows();
-                    }
-
-                    $ctrl.adjustHeight($ctrl.gridOptions.data.length);
-
-                    return $ctrl._refreshRows();
-                });
-        };
-
-        const toggleAdmin = () => {
-            const user = this.gridApi.selection.legacyGetSelectedRows()[0];
-
-            if (user.adminChanging)
-                return;
-
-            user.adminChanging = true;
-
-            AdminData.toggleAdmin(user)
-                .finally(() => {
-                    $ctrl._updateSelected();
-
-                    user.adminChanging = false;
-                });
-        };
-
-        const showActivities = () => {
-            const user = this.gridApi.selection.legacyGetSelectedRows()[0];
-
-            return new ActivitiesUserDialog({ user });
-        };
+        User.read().then((user) => this.user = user);
 
         const companiesExcludeFilter = (renderableRows) => {
-            if (_.isNil($ctrl.params.companiesExclude))
+            if (_.isNil(this.params.companiesExclude))
                 return renderableRows;
 
             _.forEach(renderableRows, (row) => {
-                row.visible = _.isEmpty($ctrl.params.companiesExclude) ||
-                    row.entity.company.toLowerCase().indexOf($ctrl.params.companiesExclude.toLowerCase()) === -1;
+                row.visible = _.isEmpty(this.params.companiesExclude) ||
+                    row.entity.company.toLowerCase().indexOf(this.params.companiesExclude.toLowerCase()) === -1;
             });
 
             return renderableRows;
         };
 
-        $ctrl.actionOptions = [
+        this.actionOptions = [
             {
                 action: 'Become this user',
-                click: becomeUser.bind(this),
+                click: () => this.becomeUser(),
                 available: true
             },
             {
                 action: 'Revoke admin',
-                click: toggleAdmin.bind(this),
+                click: () => this.toggleAdmin(),
                 available: true
             },
             {
                 action: 'Grant admin',
-                click: toggleAdmin.bind(this),
+                click: () => this.toggleAdmin(),
                 available: false
             },
+            {
+                action: 'Add user',
+                sref: '.createUser',
+                available: true
+            },
             {
                 action: 'Remove user',
-                click: removeUser.bind(this),
+                click: () => this.removeUser(),
                 available: true
             },
             {
                 action: 'Activity detail',
-                click: showActivities.bind(this),
+                click: () => this.showActivities(),
                 available: true
             }
         ];
 
-        $ctrl._userGridOptions = {
+        this._userGridOptions = {
             columnDefs,
             categories
         };
 
-        $ctrl.gridOptions = {
+        this.gridOptions = {
             data: [],
 
             columnDefs,
@@ -176,18 +138,18 @@ export default class IgniteListOfRegisteredUsersCtrl {
             rowIdentity: (row) => row._id,
             getRowIdentity: (row) => row._id,
             onRegisterApi: (api) => {
-                $ctrl.gridApi = api;
+                this.gridApi = api;
 
-                api.selection.on.rowSelectionChanged($scope, $ctrl._updateSelected.bind($ctrl));
-                api.selection.on.rowSelectionChangedBatch($scope, $ctrl._updateSelected.bind($ctrl));
+                api.selection.on.rowSelectionChanged($scope, this._updateSelected.bind(this));
+                api.selection.on.rowSelectionChangedBatch($scope, this._updateSelected.bind(this));
 
-                api.core.on.filterChanged($scope, $ctrl._filteredRows.bind($ctrl));
-                api.core.on.rowsVisibleChanged($scope, $ctrl._filteredRows.bind($ctrl));
+                api.core.on.filterChanged($scope, this._filteredRows.bind(this));
+                api.core.on.rowsVisibleChanged($scope, this._filteredRows.bind(this));
 
                 api.grid.registerRowsProcessor(companiesExcludeFilter, 50);
 
-                $scope.$watch(() => $ctrl.gridApi.grid.getVisibleRows().length, (rows) => $ctrl.adjustHeight(rows));
-                $scope.$watch(() => $ctrl.params.companiesExclude, () => $ctrl.gridApi.grid.refreshRows());
+                $scope.$watch(() => this.gridApi.grid.getVisibleRows().length, (rows) => this.adjustHeight(rows));
+                $scope.$watch(() => this.params.companiesExclude, () => this.gridApi.grid.refreshRows());
             }
         };
 
@@ -197,20 +159,20 @@ export default class IgniteListOfRegisteredUsersCtrl {
         const reloadUsers = (params) => {
             AdminData.loadUsers(params)
                 .then((data) => {
-                    $ctrl.gridOptions.data = data;
+                    this.gridOptions.data = data;
 
-                    $ctrl.companies = _.values(_.groupBy(data, 'company'));
-                    $ctrl.countries = _.values(_.groupBy(data, 'countryCode'));
+                    this.companies = _.values(_.groupBy(data, 'company'));
+                    this.countries = _.values(_.groupBy(data, 'countryCode'));
 
-                    $ctrl._refreshRows();
+                    this._refreshRows();
                 });
         };
 
         const filterDates = _.debounce(() => {
-            const sdt = $ctrl.params.startDate;
-            const edt = $ctrl.params.endDate;
+            const sdt = this.params.startDate;
+            const edt = this.params.endDate;
 
-            $ctrl.exporterCsvFilename = `web_console_users_${dtFilter(sdt, 'yyyy_MM')}.csv`;
+            this.exporterCsvFilename = `web_console_users_${dtFilter(sdt, 'yyyy_MM')}.csv`;
 
             const startDate = Date.UTC(sdt.getFullYear(), sdt.getMonth(), 1);
             const endDate = Date.UTC(edt.getFullYear(), edt.getMonth() + 1, 1);
@@ -218,8 +180,9 @@ export default class IgniteListOfRegisteredUsersCtrl {
             reloadUsers({ startDate, endDate });
         }, 250);
 
-        $scope.$watch(() => $ctrl.params.startDate, filterDates);
-        $scope.$watch(() => $ctrl.params.endDate, filterDates);
+        $scope.$on('userCreated', filterDates);
+        $scope.$watch(() => this.params.startDate, filterDates);
+        $scope.$watch(() => this.params.endDate, filterDates);
     }
 
     adjustHeight(rows) {
@@ -233,36 +196,95 @@ export default class IgniteListOfRegisteredUsersCtrl {
 
     _filteredRows() {
         const filtered = _.filter(this.gridApi.grid.rows, ({ visible}) => visible);
-        const entities = _.map(filtered, 'entity');
 
-        this.filteredRows = entities;
+        this.filteredRows = _.map(filtered, 'entity');
     }
 
     _updateSelected() {
         const ids = this.gridApi.selection.legacyGetSelectedRows().map(({ _id }) => _id).sort();
 
+        if (!_.isEqual(ids, this.selected))
+            this.selected = ids;
+
         if (ids.length) {
             const user = this.gridApi.selection.legacyGetSelectedRows()[0];
             const other = this.user._id !== user._id;
 
-            this.actionOptions[1].available = other && user.admin;
-            this.actionOptions[2].available = other && !user.admin;
-
-            this.actionOptions[0].available = other;
-            this.actionOptions[3].available = other;
+            this.actionOptions[0].available = other; // Become this user.
+            this.actionOptions[1].available = other && user.admin; // Revoke admin.
+            this.actionOptions[2].available = other && !user.admin; // Grant admin.
+            this.actionOptions[4].available = other; // Remove user.
+            this.actionOptions[5].available = true; // Activity detail.
+        }
+        else {
+            this.actionOptions[0].available = false; // Become this user.
+            this.actionOptions[1].available = false; // Revoke admin.
+            this.actionOptions[2].available = false; // Grant admin.
+            this.actionOptions[4].available = false; // Remove user.
+            this.actionOptions[5].available = false; // Activity detail.
         }
-
-        if (!_.isEqual(ids, this.selected))
-            this.selected = ids;
     }
 
     _refreshRows() {
         if (this.gridApi) {
             this.gridApi.grid.refreshRows()
-                .then(() => this.selected.length && this._updateSelected());
+                .then(() => this._updateSelected());
         }
     }
 
+    becomeUser() {
+        const user = this.gridApi.selection.legacyGetSelectedRows()[0];
+
+        this.AdminData.becomeUser(user._id)
+            .then(() => this.User.load())
+            .then(() => this.$state.go('default-state'))
+            .then(() => this.NotebookData.load());
+    }
+
+    toggleAdmin() {
+        if (!this.gridApi)
+            return;
+
+        const user = this.gridApi.selection.legacyGetSelectedRows()[0];
+
+        if (user.adminChanging)
+            return;
+
+        user.adminChanging = true;
+
+        this.AdminData.toggleAdmin(user)
+            .finally(() => {
+                this._updateSelected();
+
+                user.adminChanging = false;
+            });
+    }
+
+    removeUser() {
+        const user = this.gridApi.selection.legacyGetSelectedRows()[0];
+
+        this.Confirm.confirm(`Are you sure you want to remove user: "${user.userName}"?`)
+            .then(() => this.AdminData.removeUser(user))
+            .then(() => {
+                const i = _.findIndex(this.gridOptions.data, (u) => u._id === user._id);
+
+                if (i >= 0) {
+                    this.gridOptions.data.splice(i, 1);
+                    this.gridApi.selection.clearSelectedRows();
+                }
+
+                this.adjustHeight(this.gridOptions.data.length);
+
+                return this._refreshRows();
+            });
+    }
+
+    showActivities() {
+        const user = this.gridApi.selection.legacyGetSelectedRows()[0];
+
+        return new this.ActivitiesDialogFactory({ user });
+    }
+
     groupByUser() {
         this.groupBy = 'user';
 
diff --git a/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug b/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug
index d32b64d082f12..ff1c851817f65 100644
--- a/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/list-of-registered-users/template.tpl.pug
@@ -74,7 +74,6 @@ ul.tabs.tabs--blue
                 label: 'Actions',
                 model: '$ctrl.action',
                 name: 'action',
-                disabled: '!$ctrl.selected.length',
                 options: '$ctrl.actionOptions'
             })
 
diff --git a/modules/web-console/frontend/app/components/page-admin/controller.ts b/modules/web-console/frontend/app/components/page-admin/controller.ts
new file mode 100644
index 0000000000000..45a5c0a5547e8
--- /dev/null
+++ b/modules/web-console/frontend/app/components/page-admin/controller.ts
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import UserNotificationsService from '../user-notifications/service';
+
+export default class PageAdminCtrl {
+    static $inject = ['UserNotifications'];
+
+    constructor(private UserNotifications: UserNotificationsService) {}
+
+    changeUserNotifications() {
+        this.UserNotifications.editor();
+    }
+}
diff --git a/modules/web-console/frontend/app/components/page-admin/index.js b/modules/web-console/frontend/app/components/page-admin/index.js
index 232d889eb46e1..8b26f2c410290 100644
--- a/modules/web-console/frontend/app/components/page-admin/index.js
+++ b/modules/web-console/frontend/app/components/page-admin/index.js
@@ -17,23 +17,12 @@
 
 import './style.scss';
 
+import controller from './controller';
 import templateUrl from './template.tpl.pug';
 
-class PageAdminCtrl {
-    static $inject = ['UserNotifications'];
-
-    constructor(UserNotifications) {
-        this.UserNotifications = UserNotifications;
-    }
-
-    changeUserNotifications() {
-        this.UserNotifications.editor();
-    }
-}
-
 export default angular
     .module('ignite-console.page-admin', [])
     .component('pageAdmin', {
-        templateUrl,
-        controller: PageAdminCtrl
+        controller,
+        templateUrl
     });
diff --git a/modules/web-console/frontend/app/components/page-signup/component.js b/modules/web-console/frontend/app/components/page-signup/component.js
index 789a68187e828..968ff396e298d 100644
--- a/modules/web-console/frontend/app/components/page-signup/component.js
+++ b/modules/web-console/frontend/app/components/page-signup/component.js
@@ -15,8 +15,6 @@
  * limitations under the License.
  */
 
-import angular from 'angular';
-
 import template from './template.pug';
 import controller from './controller';
 import './style.scss';
diff --git a/modules/web-console/frontend/app/components/page-signup/controller.js b/modules/web-console/frontend/app/components/page-signup/controller.ts
similarity index 53%
rename from modules/web-console/frontend/app/components/page-signup/controller.js
rename to modules/web-console/frontend/app/components/page-signup/controller.ts
index 10397760c3d71..ddf77be099ca9 100644
--- a/modules/web-console/frontend/app/components/page-signup/controller.js
+++ b/modules/web-console/frontend/app/components/page-signup/controller.ts
@@ -15,11 +15,15 @@
  * limitations under the License.
  */
 
+import Auth from '../../modules/user/Auth.service';
+import MessagesFactory from '../../services/Messages.service';
+import FormUtilsFactoryFactory from '../../services/FormUtils.service';
+import {ISignupData} from '../form-signup';
+
 export default class PageSignup {
-    /** @type {import('./types').ISignupFormController} */
-    form;
-    /** @type {import('./types').ISignupData} */
-    data = {
+    form: ng.IFormController;
+
+    data: ISignupData = {
         email: null,
         password: null,
         firstName: null,
@@ -27,37 +31,23 @@ export default class PageSignup {
         company: null,
         country: null
     };
-    /** @type {string} */
-    serverError = null;
 
-    static $inject = ['IgniteCountries', 'Auth', 'IgniteMessages', 'IgniteFormUtils'];
+    serverError: string | null = null;
 
-    /**
-     * @param {ReturnType<typeof import('app/services/Countries.service').default>} Countries
-     * @param {import('app/modules/user/Auth.service').default} Auth
-     * @param {ReturnType<typeof import('app/services/Messages.service').default>} IgniteMessages
-     * @param {ReturnType<typeof import('app/services/FormUtils.service').default>} IgniteFormUtils
-     */
-    constructor(Countries, Auth, IgniteMessages, IgniteFormUtils) {
-        this.Auth = Auth;
-        this.IgniteMessages = IgniteMessages;
-        this.countries = Countries.getAll();
-        this.IgniteFormUtils = IgniteFormUtils;
-    }
+    static $inject = ['Auth', 'IgniteMessages', 'IgniteFormUtils'];
 
-    /** @param {import('./types').ISignupFormController} form */
-    canSubmitForm(form) {
-        return form.$error.server ? true : !form.$invalid;
-    }
+    constructor(
+        private Auth: Auth,
+        private IgniteMessages: ReturnType<typeof MessagesFactory>,
+        private IgniteFormUtils: ReturnType<typeof FormUtilsFactoryFactory>
+    ) {}
 
-    $postLink() {
-        this.form.email.$validators.server = () => !this.serverError;
+    canSubmitForm(form: PageSignup['form']) {
+        return form.$error.server ? true : !form.$invalid;
     }
 
-    /** @param {string} error */
-    setServerError(error) {
+    setServerError(error: PageSignup['serverError']) {
         this.serverError = error;
-        this.form.email.$validate();
     }
 
     signup() {
@@ -68,7 +58,7 @@ export default class PageSignup {
         if (!this.canSubmitForm(this.form))
             return;
 
-        return this.Auth.signnup(this.data).catch((res) => {
+        return this.Auth.signup(this.data).catch((res) => {
             this.IgniteMessages.showError(null, res.data);
             this.setServerError(res.data);
         });
diff --git a/modules/web-console/frontend/app/components/page-signup/index.js b/modules/web-console/frontend/app/components/page-signup/index.js
index 4efadb5f47af5..001d26987d29a 100644
--- a/modules/web-console/frontend/app/components/page-signup/index.js
+++ b/modules/web-console/frontend/app/components/page-signup/index.js
@@ -22,7 +22,8 @@ import {registerState} from './run';
 export default angular
     .module('ignite-console.page-signup', [
         'ui.router',
-        'ignite-console.user'
+        'ignite-console.user',
+        'ignite-console.form-signup'
     ])
     .component('pageSignup', component)
     .run(registerState);
diff --git a/modules/web-console/frontend/app/components/page-signup/style.scss b/modules/web-console/frontend/app/components/page-signup/style.scss
index 93167ce6856da..98c4729bf5ad5 100644
--- a/modules/web-console/frontend/app/components/page-signup/style.scss
+++ b/modules/web-console/frontend/app/components/page-signup/style.scss
@@ -41,16 +41,6 @@ page-signup {
                 margin-left: auto;
             }
         }
-
-        form {
-            display: grid;
-            grid-gap: 10px;
-            grid-template-columns: 1fr 1fr;
-
-            .full-width {
-                grid-column: 1 / 3;
-            }
-        }
     }
 
     .page-signup__has-account-message {
diff --git a/modules/web-console/frontend/app/components/page-signup/template.pug b/modules/web-console/frontend/app/components/page-signup/template.pug
index 11bb50a15c733..5584dfcb67129 100644
--- a/modules/web-console/frontend/app/components/page-signup/template.pug
+++ b/modules/web-console/frontend/app/components/page-signup/template.pug
@@ -22,95 +22,13 @@ web-console-header
 
 .container--responsive.body-container
     section
-        -var form = '$ctrl.form'
         h3 Don't Have An Account?
-        form(name=form novalidate ng-submit='$ctrl.signup()')
-            .full-width
-                +form-field__email({
-                    label: 'Email:',
-                    model: '$ctrl.data.email',
-                    name: '"email"',
-                    placeholder: 'Input email',
-                    required: true
-                })(
-                    ng-model-options='{allowInvalid: true}'
-                    autocomplete='email'
-                    ignite-auto-focus
-                )
-                    +form-field__error({error: 'server', message: `{{$ctrl.serverError}}`})
-            div
-                +form-field__password({
-                    label: 'Password:',
-                    model: '$ctrl.data.password',
-                    name: '"password"',
-                    placeholder: 'Input password',
-                    required: true
-                })(
-                    autocomplete='new-password'
-                )
-            div
-                +form-field__password({
-                    label: 'Confirm:',
-                    model: 'confirm',
-                    name: '"confirm"',
-                    placeholder: 'Confirm password',
-                    required: true
-                })(
-                    ignite-match='$ctrl.data.password'
-                    autocomplete='off'
-                )
-            div
-                +form-field__text({
-                    label: 'First name:',
-                    model: '$ctrl.data.firstName',
-                    name: '"firstName"',
-                    placeholder: 'Input first name',
-                    required: true
-                })(
-                    autocomplete='given-name'
-                )
-            div
-                +form-field__text({
-                    label: 'Last name:',
-                    model: '$ctrl.data.lastName',
-                    name: '"lastName"',
-                    placeholder: 'Input last name',
-                    required: true
-                })(
-                    autocomplete='family-name'
-                )
-            div
-                +form-field__phone({
-                    label: 'Phone:',
-                    model: '$ctrl.data.phone',
-                    name: '"phone"',
-                    placeholder: 'Input phone (ex.: +15417543010)',
-                    optional: true
-                })(
-                    autocomplete='tel'
-                )
-            div
-                +form-field__dropdown({
-                    label: 'Country:',
-                    model: '$ctrl.data.country',
-                    name: '"country"',
-                    required: true,
-                    placeholder: 'Choose your country',
-                    options: '$ctrl.countries'
-                })(
-                    autocomplete='country'
-                )
-            .full-width
-                +form-field__text({
-                    label: 'Company:',
-                    model: '$ctrl.data.company',
-                    name: '"company"',
-                    placeholder: 'Input company name',
-                    required: true
-                })(
-                    ignite-on-enter-focus-move='countryInput'
-                    autocomplete='organization'
-                )
+        form(name='$ctrl.form' novalidate ng-submit='$ctrl.signup()')
+            form-signup(
+                outer-form='$ctrl.form'
+                ng-model='$ctrl.data'
+                server-error='$ctrl.serverError'
+            )
             footer.full-width.form-footer
                 button.btn-ignite.btn-ignite--primary(
                     type='submit'
diff --git a/modules/web-console/frontend/app/modules/user/Auth.service.js b/modules/web-console/frontend/app/modules/user/Auth.service.ts
similarity index 51%
rename from modules/web-console/frontend/app/modules/user/Auth.service.js
rename to modules/web-console/frontend/app/modules/user/Auth.service.ts
index 8c3cc4ea4e2db..55956ad990b47 100644
--- a/modules/web-console/frontend/app/modules/user/Auth.service.js
+++ b/modules/web-console/frontend/app/modules/user/Auth.service.ts
@@ -15,64 +15,53 @@
  * limitations under the License.
  */
 
-/**
- * @typedef {object} SignupUserInfo
- * @prop {string} email
- * @prop {string} password
- * @prop {string} firstName
- * @prop {string} lastName
- * @prop {string} company
- * @prop {string} country
- */
+import {StateService} from '@uirouter/angularjs';
+import MessagesFactory from '../../services/Messages.service';
+import {service as GettingsStartedFactory} from '../../modules/getting-started/GettingStarted.provider';
+import UserServiceFactory from './User.service';
+
+type SignupUserInfo = {
+    email: string,
+    password: string,
+    firstName: string,
+    lastName: string,
+    company: string,
+    country: string,
+};
+
+type AuthActions = 'signin' | 'signup' | 'password/forgot';
+type AuthOptions = {email:string, password:string}|SignupUserInfo|{email:string};
 
 export default class AuthService {
     static $inject = ['$http', '$rootScope', '$state', '$window', 'IgniteMessages', 'gettingStarted', 'User'];
-    /**
-     * @param {ng.IHttpService} $http
-     * @param {ng.IRootScopeService} $root
-     * @param {import('@uirouter/angularjs').StateService} $state
-     * @param {ng.IWindowService} $window
-     * @param {ReturnType<typeof import('app/services/Messages.service').default>} Messages
-     * @param {ReturnType<typeof import('app/modules/getting-started/GettingStarted.provider').service>} gettingStarted
-     * @param {ReturnType<typeof import('./User.service').default>} User
-     */
-    constructor($http, $root, $state, $window, Messages, gettingStarted, User) {
-        this.$http = $http;
-        this.$root = $root;
-        this.$state = $state;
-        this.$window = $window;
-        this.Messages = Messages;
-        this.gettingStarted = gettingStarted;
-        this.User = User;
-    }
-    /**
-     * @param {SignupUserInfo} userInfo
-     */
-    signnup(userInfo) {
-        return this._auth('signup', userInfo);
+
+    constructor(
+        private $http: ng.IHttpService,
+        private $root: ng.IRootScopeService,
+        private $state: StateService,
+        private $window: ng.IWindowService,
+        private Messages: ReturnType<typeof MessagesFactory>,
+        private gettingStarted: ReturnType<typeof GettingsStartedFactory>,
+        private User: ReturnType<typeof UserServiceFactory>
+    ) {}
+
+    signup(userInfo: SignupUserInfo, loginAfterSignup: boolean = true) {
+        return this._auth('signup', userInfo, loginAfterSignup);
     }
-    /**
-     * @param {string} email
-     * @param {string} password
-     */
-    signin(email, password) {
+
+    signin(email: string, password: string) {
         return this._auth('signin', {email, password});
     }
-    /**
-     * @param {string} email
-     */
-    remindPassword(email) {
+
+    remindPassword(email: string) {
         return this._auth('password/forgot', {email}).then(() => this.$state.go('password.send'));
     }
 
     // TODO IGNITE-7994: Remove _auth and move API calls to corresponding methods
     /**
      * Performs the REST API call.
-     * @private
-     * @param {('signin'|'signup'|'password/forgot')} action
-     * @param {{email:string,password:string}|SignupUserInfo|{email:string}} userInfo
      */
-    _auth(action, userInfo) {
+    private _auth(action: AuthActions, userInfo: AuthOptions, loginAfterwards: boolean = true) {
         return this.$http.post('/api/v1/' + action, userInfo)
             .then(() => {
                 if (action === 'password/forgot')
@@ -80,11 +69,12 @@ export default class AuthService {
 
                 this.User.read()
                     .then((user) => {
-                        this.$root.$broadcast('user', user);
-
-                        this.$state.go('default-state');
-
-                        this.$root.gettingStarted.tryShow();
+                        if (loginAfterwards) {
+                            this.$root.$broadcast('user', user);
+                            this.$state.go('default-state');
+                            this.$root.gettingStarted.tryShow();
+                        } else
+                            this.$root.$broadcast('userCreated');
                     });
             });
     }
diff --git a/modules/web-console/frontend/app/primitives/dropdown/index.pug b/modules/web-console/frontend/app/primitives/dropdown/index.pug
index 0099457be59ab..aad6efd003605 100644
--- a/modules/web-console/frontend/app/primitives/dropdown/index.pug
+++ b/modules/web-console/frontend/app/primitives/dropdown/index.pug
@@ -37,5 +37,6 @@ mixin ignite-form-field-bsdropdown({label, model, name, disabled, required, opti
 
         ul.dropdown-menu(role='menu')
             li(ng-repeat=`item in ${options}` ng-if='item.available')
-                a(ng-click='item.click()') {{ item.action }}
+                a(ng-if='item.click' ng-click='item.click()') {{ item.action }}
+                a(ng-if='item.sref' ui-sref='{{:: item.sref}}') {{ item.action }}
 
diff --git a/modules/web-console/frontend/app/primitives/form-field/checkbox.pug b/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
index 88b8f5a2f9618..fe0f808574b3b 100644
--- a/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/checkbox.pug
@@ -15,7 +15,7 @@
     limitations under the License.
 
 mixin form-field__checkbox({ label, model, name, disabled, required, tip, tipOpts })
-    .form-field.form-field__checkbox(id=`{{ ${name} }}Field`)
+    .form-field.ignite-form-field.form-field__checkbox(id=`{{ ${name} }}Field`)
         +form-field__label({ label, name, required, disabled })
             +form-field__tooltip({ title: tip, options: tipOpts })
 
@@ -24,7 +24,8 @@ mixin form-field__checkbox({ label, model, name, disabled, required, tip, tipOpt
             +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes)
 
         .form-field__errors(
-            ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}`
+            ng-messages=`$input.$error`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
         )
             if block
                 block
diff --git a/modules/web-console/frontend/app/primitives/form-field/email.pug b/modules/web-console/frontend/app/primitives/form-field/email.pug
index b68a520811741..5fb0c3019fc67 100644
--- a/modules/web-console/frontend/app/primitives/form-field/email.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/email.pug
@@ -15,9 +15,9 @@
     limitations under the License.
 
 mixin form-field__email({ label, model, name, disabled, required, placeholder, tip })
-    -var errLbl = label.substring(0, label.length - 1)
+    -let errLbl = label[label.length - 1] === ':' ? label.substring(0, label.length - 1) : label
 
-    .form-field
+    .form-field.ignite-form-field
         +form-field__label({ label, name, required, disabled })
             +form-field__tooltip({ title: tip, options: tipOpts })
 
@@ -26,7 +26,8 @@ mixin form-field__email({ label, model, name, disabled, required, placeholder, t
             +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes)
 
         .form-field__errors(
-            ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}`
+            ng-messages=`$input.$error`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
         )
             if block
                 block
diff --git a/modules/web-console/frontend/app/primitives/form-field/password.pug b/modules/web-console/frontend/app/primitives/form-field/password.pug
index 6b9818bc91f43..374d768213639 100644
--- a/modules/web-console/frontend/app/primitives/form-field/password.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/password.pug
@@ -17,9 +17,9 @@
 mixin form-field__password({ label, model, name, disabled, required, placeholder, tip })
     -var errLbl = label.substring(0, label.length - 1)
 
-    .form-field.form-field__password(
+    .form-field.form-field__password.ignite-form-field(
         password-visibility-root
-        on-password-visibility-toggle=`${form}[${name}].$setTouched()`
+        on-password-visibility-toggle=`$input1.$setTouched(); $input2.$setTouched()`
     )
         +form-field__label({ label, name, required })
             +form-field__tooltip({ title: tip, options: tipOpts })
@@ -27,16 +27,22 @@ mixin form-field__password({ label, model, name, disabled, required, placeholder
         .form-field__control
             - attributes.type='password'
             - attributes.class = 'password-visibility__password-hidden'
+            - attributes['ng-ref'] = '$input1'
             +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes)
             - attributes.class = 'password-visibility__password-visible'
             - attributes.type='text'
             - attributes.autocomplete = 'off'
+            - attributes['ng-ref'] = '$input2'
             +form-field__input({ name: name + `+"Text"`, model, disabled, required, placeholder })(attributes=attributes)
 
             password-visibility-toggle-button
 
         .form-field__errors(
-            ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}`
+            ng-messages=`$input1.$error || $input2.$error`
+            ng-show=`
+                ($input1.$dirty || $input1.$touched || $input1.$submitted) && $input1.$invalid ||
+                ($input2.$dirty || $input2.$touched || $input2.$submitted) && $input2.$invalid
+            `
         )
             if block
                 block
diff --git a/modules/web-console/frontend/app/primitives/form-field/phone.pug b/modules/web-console/frontend/app/primitives/form-field/phone.pug
index 8b503018d3bda..9b60a40457b0b 100644
--- a/modules/web-console/frontend/app/primitives/form-field/phone.pug
+++ b/modules/web-console/frontend/app/primitives/form-field/phone.pug
@@ -17,7 +17,7 @@
 mixin form-field__phone({ label, model, name, disabled, required, optional, placeholder, tip })
     -var errLbl = label.substring(0, label.length - 1)
 
-    .form-field
+    .form-field.ignite-form-field
         +form-field__label({ label, name, required, optional })
             +form-field__tooltip({ title: tip, options: tipOpts })
 
@@ -26,7 +26,8 @@ mixin form-field__phone({ label, model, name, disabled, required, optional, plac
             +form-field__input({ name, model, disabled, required, placeholder })(attributes=attributes)
 
         .form-field__errors(
-            ng-messages=`(${form}[${name}].$dirty || ${form}[${name}].$touched || ${form}[${name}].$submitted) && ${form}[${name}].$invalid ? ${form}[${name}].$error : {}`
+            ng-messages=`$input.$error`
+            ng-show=`($input.$dirty || $input.$touched || $input.$submitted) && $input.$invalid`
         )
             if block
                 block
diff --git a/modules/web-console/frontend/app/utils/dialogState.ts b/modules/web-console/frontend/app/utils/dialogState.ts
new file mode 100644
index 0000000000000..d83c911035eb9
--- /dev/null
+++ b/modules/web-console/frontend/app/utils/dialogState.ts
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {UIRouter, StateDeclaration, Transition} from '@uirouter/angularjs';
+
+export function dialogState(component: string): Partial<StateDeclaration> {
+    let dialog: mgcrea.ngStrap.modal.IModal | undefined;
+    let hide: (() => void) | undefined;
+
+    onEnter.$inject = ['$transition$'];
+
+    function onEnter(transition: Transition) {
+        const modal = transition.injector().get<mgcrea.ngStrap.modal.IModalService>('$modal');
+        const router = transition.injector().get<UIRouter>('$uiRouter');
+
+        dialog = modal({
+            template: `
+                <${component}
+                    class='modal center modal--ignite theme--ignite'
+                    tabindex='-1'
+                    role='dialog'
+                    on-hide=$hide()
+                ></${component}>
+            `,
+            onHide(modal) {
+                modal.destroy();
+            }
+        });
+
+        hide = dialog.hide;
+
+        dialog.hide = () => router.stateService.go('.^');
+    }
+
+    return {
+        onEnter,
+        onExit() {
+            if (hide) hide();
+            dialog = hide = void 0;
+        }
+    };
+}

From 70abdadd7f27611da750ecaeff383b40f412c00d Mon Sep 17 00:00:00 2001
From: pereslegin-pa <xxtern@gmail.com>
Date: Tue, 14 Aug 2018 21:20:50 +0300
Subject: [PATCH 465/543] IGNITE-9195 Split PDS 2 TC configuration into 3
 suites. - Fixes #4496.

Signed-off-by: Dmitriy Pavlov <dpavlov@apache.org>

(cherry picked from commit a14ee5cd8a36a764fed8baa46de9c8abe3fcf14a)
---
 .../testsuites/IgnitePdsTestSuite2.java       | 16 ++----
 .../testsuites/IgnitePdsTestSuite3.java       | 49 +++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite4.java       |  3 ++
 3 files changed, 55 insertions(+), 13 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java

diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index c60cd9a3e63b1..1f2effbd141c5 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -19,15 +19,12 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteDataStorageMetricsSelfTest;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest2;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCorruptedStoreTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchangeDuringCheckpointTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPartitionFilesDestroyTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
-import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteRebalanceScheduleResendPartitionsTest;
 import org.apache.ignite.internal.processors.cache.persistence.LocalWacModeNoChangeDuringRebalanceOnNonNodeAssignTest;
@@ -39,9 +36,9 @@
 import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionDuringPartitionClearTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsReserveWalSegmentsWithCompactionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsUnusedWalSegmentsTest;
-import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsReserveWalSegmentsWithCompactionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.SlowHistoricalRebalanceSmallHistoryTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
@@ -64,9 +61,9 @@
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalCompactionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteDataIntegrityTests;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteFsyncReplayWalIteratorInvalidCrcTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgnitePureJavaCrcCompatibility;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteReplayWalIteratorInvalidCrcTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteStandaloneWalIteratorInvalidCrcTest;
-import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgnitePureJavaCrcCompatibility;
 import org.apache.ignite.internal.processors.cache.persistence.db.wal.reader.IgniteWalReaderTest;
 import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneWalRecordsIteratorTest;
 
@@ -98,14 +95,11 @@ public static TestSuite suite() {
         suite.addTestSuite(ClientAffinityAssignmentWithBaselineTest.class);
         suite.addTestSuite(IgniteAbsentEvictionNodeOutOfBaselineTest.class);
 
-        suite.addTestSuite(IgnitePdsTaskCancelingTest.class);
-
         return suite;
     }
 
     /**
-     * Fills {@code suite} with PDS test subset, which operates with real page store, but requires long time to
-     * execute.
+     * Fills {@code suite} with PDS test subset, which operates with real page store, but requires long time to execute.
      *
      * @param suite suite to add tests into.
      */
@@ -114,10 +108,6 @@ private static void addRealPageStoreTestsLongRunning(TestSuite suite) {
 
         suite.addTestSuite(IgnitePdsPageEvictionDuringPartitionClearTest.class);
 
-        // Rebalancing test
-        suite.addTestSuite(IgnitePdsContinuousRestartTest.class);
-        suite.addTestSuite(IgnitePdsContinuousRestartTest2.class);
-
         suite.addTestSuite(IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes.class);
 
         // Integrity test.
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java
new file mode 100644
index 0000000000000..ce012627cf551
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite3.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest2;
+
+/**
+ *
+ */
+public class IgnitePdsTestSuite3 extends TestSuite {
+    /**
+     * @return Suite.
+     */
+    public static TestSuite suite() {
+        TestSuite suite = new TestSuite("Ignite Persistent Store Test Suite 3");
+
+        addRealPageStoreTestsNotForDirectIo(suite);
+
+        return suite;
+    }
+
+    /**
+     * Fills {@code suite} with PDS test subset, which operates with real page store, but requires long time to execute.
+     *
+     * @param suite suite to add tests into.
+     */
+    public static void addRealPageStoreTestsNotForDirectIo(TestSuite suite) {
+        // Rebalancing test
+        suite.addTestSuite(IgnitePdsContinuousRestartTest.class);
+        suite.addTestSuite(IgnitePdsContinuousRestartTest2.class);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
index e078c53195a31..d5cc0ef233e46 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPartitionPreloadTest;
 
 /**
@@ -35,6 +36,8 @@ public static TestSuite suite() {
 
         suite.addTestSuite(IgnitePdsPartitionPreloadTest.class);
 
+        suite.addTestSuite(IgnitePdsTaskCancelingTest.class);
+
         return suite;
     }
 }

From 248c623b24a721acc3fa9e181f3675b86f93c7fb Mon Sep 17 00:00:00 2001
From: ascherbakoff <alexey.scherbakoff@gmail.com>
Date: Mon, 29 Oct 2018 16:39:44 +0300
Subject: [PATCH 466/543] IGNITE-8902 Fixed rollback of GridDhtTxRemote for
 one-phase commit - Fixes #5089.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 996a1d5ed8af0034dd1541b206bda0ac9467f981)
(cherry picked from commit 2b8cf2dfabb2c65cc5662ccec52ddef6aa01e382)
---
 .../cache/GridCacheSharedContext.java         |   1 -
 .../dht/GridDhtTxPrepareFuture.java           | 104 +++++----
 .../cache/transactions/IgniteTxHandler.java   |   1 +
 .../cache/transactions/IgniteTxManager.java   |   3 +-
 .../TestRecordingCommunicationSpi.java        |  20 ++
 ...TxRollbackOnTimeoutOnePhaseCommitTest.java | 215 ++++++++++++++++++
 .../testsuites/IgniteCacheTestSuite6.java     |   2 +
 7 files changed, 299 insertions(+), 47 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutOnePhaseCommitTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index a97d431c8a58b..69ec807ae03e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -1127,5 +1127,4 @@ public void readOnlyMode(boolean readOnlyMode) {
     public void setTxManager(IgniteTxManager txMgr) {
         this.txMgr = txMgr;
     }
-
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
index 1d68c4a3528aa..95058898de2c1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareFuture.java
@@ -26,6 +26,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
@@ -212,6 +213,9 @@ public final class GridDhtTxPrepareFuture extends GridCacheCompoundFuture<Ignite
     /** Timeout object. */
     private final PrepareTimeoutObject timeoutObj;
 
+    /** */
+    private CountDownLatch timeoutAddedLatch;
+
     /**
      * @param cctx Context.
      * @param tx Transaction.
@@ -255,6 +259,9 @@ public GridDhtTxPrepareFuture(
         assert nearMap != null;
 
         timeoutObj = timeout > 0 ? new PrepareTimeoutObject(timeout) : null;
+
+        if (tx.onePhaseCommit())
+            timeoutAddedLatch = new CountDownLatch(1);
     }
 
     /** {@inheritDoc} */
@@ -675,6 +682,14 @@ private boolean mapIfLocked() {
             if (!MAPPED_UPD.compareAndSet(this, 0, 1))
                 return false;
 
+            if (timeoutObj != null && tx.onePhaseCommit()) {
+                U.awaitQuiet(timeoutAddedLatch);
+
+                // Disable timeouts after all locks are acquired for one-phase commit or partition desync will occur.
+                if (!cctx.time().removeTimeoutObject(timeoutObj))
+                    return true; // Should not proceed with prepare if tx is already timed out.
+            }
+
             if (forceKeysFut == null || (forceKeysFut.isDone() && forceKeysFut.error() == null))
                 prepare0();
             else {
@@ -713,7 +728,7 @@ private boolean mapIfLocked() {
             tx.clearPrepareFuture(this);
 
         // Do not commit one-phase commit transaction if originating node has near cache enabled.
-        if (tx.onePhaseCommit() && tx.commitOnPrepare()) {
+        if (tx.commitOnPrepare()) {
             assert last;
 
             Throwable prepErr = this.err;
@@ -723,61 +738,55 @@ private boolean mapIfLocked() {
 
             onComplete(res);
 
-            if (tx.commitOnPrepare()) {
-                if (tx.markFinalizing(IgniteInternalTx.FinalizationStatus.USER_FINISH)) {
-                    CIX1<IgniteInternalFuture<IgniteInternalTx>> resClo =
-                        new CIX1<IgniteInternalFuture<IgniteInternalTx>>() {
-                            @Override public void applyx(IgniteInternalFuture<IgniteInternalTx> fut) {
-                                if(res.error() == null && fut.error() != null)
-                                    res.error(fut.error());
+            if (tx.markFinalizing(IgniteInternalTx.FinalizationStatus.USER_FINISH)) {
+                CIX1<IgniteInternalFuture<IgniteInternalTx>> resClo =
+                    new CIX1<IgniteInternalFuture<IgniteInternalTx>>() {
+                        @Override public void applyx(IgniteInternalFuture<IgniteInternalTx> fut) {
+                            if (res.error() == null && fut.error() != null)
+                                res.error(fut.error());
 
-                                if (REPLIED_UPD.compareAndSet(GridDhtTxPrepareFuture.this, 0, 1))
-                                    sendPrepareResponse(res);
-                            }
-                        };
+                            if (REPLIED_UPD.compareAndSet(GridDhtTxPrepareFuture.this, 0, 1))
+                                sendPrepareResponse(res);
+                        }
+                    };
 
-                    try {
-                        if (prepErr == null) {
-                            try {
-                                tx.commitAsync().listen(resClo);
-                            }
-                            catch (Throwable e) {
-                                res.error(e);
+                try {
+                    if (prepErr == null) {
+                        try {
+                            tx.commitAsync().listen(resClo);
+                        }
+                        catch (Throwable e) {
+                            res.error(e);
 
-                                tx.systemInvalidate(true);
+                            tx.systemInvalidate(true);
 
-                                try {
-                                    tx.rollbackAsync().listen(resClo);
-                                }
-                                catch (Throwable e1) {
-                                    e.addSuppressed(e1);
-                                }
-
-                                throw e;
-                            }
-                        }
-                        else if (!cctx.kernalContext().isStopping()) {
                             try {
                                 tx.rollbackAsync().listen(resClo);
                             }
-                            catch (Throwable e) {
-                                if (err != null)
-                                    err.addSuppressed(e);
-
-                                throw err;
+                            catch (Throwable e1) {
+                                e.addSuppressed(e1);
                             }
+
+                            throw e;
                         }
                     }
-                    catch (Throwable e){
-                        tx.logTxFinishErrorSafe(log, true, e);
+                    else if (!cctx.kernalContext().isStopping()) {
+                        try {
+                            tx.rollbackAsync().listen(resClo);
+                        }
+                        catch (Throwable e) {
+                            if (err != null)
+                                err.addSuppressed(e);
 
-                        cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+                            throw err;
+                        }
                     }
                 }
-            }
-            else {
-                if (REPLIED_UPD.compareAndSet(this, 0, 1))
-                    sendPrepareResponse(res);
+                catch (Throwable e) {
+                    tx.logTxFinishErrorSafe(log, true, e);
+
+                    cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+                }
             }
 
             return true;
@@ -1047,9 +1056,14 @@ public void prepare(GridNearTxPrepareRequest req) {
 
         readyLocks();
 
-        if (timeoutObj != null && !isDone()) {
-            // Start timeout tracking after 'readyLocks' to avoid race with timeout processing.
+        // Start timeout tracking after 'readyLocks' to avoid race with timeout processing.
+        if (timeoutObj != null) {
             cctx.time().addTimeoutObject(timeoutObj);
+
+            // Fix race with add/remove timeout object if locks are mapped from another
+            // thread before timeout object is enqueued.
+            if (tx.onePhaseCommit())
+                timeoutAddedLatch.countDown();
         }
 
         mapIfLocked();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 45308a9fe7d62..9075ccfce417f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -1631,6 +1631,7 @@ private void sendReply(UUID nodeId, GridDhtTxFinishRequest req, boolean committe
                     single,
                     req.storeWriteThrough());
 
+                tx.onePhaseCommit(req.onePhaseCommit());
                 tx.writeVersion(req.writeVersion());
 
                 tx = ctx.tm().onCreated(null, tx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index cc558ef2888b8..52f95b871edf5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -862,7 +862,8 @@ public void prepareTx(IgniteInternalTx tx, @Nullable Collection<IgniteTxEntry> e
             throw new IgniteCheckedException("Transaction is marked for rollback: " + tx);
         }
 
-        if (tx.remainingTime() == -1) {
+        // One-phase commit tx cannot timeout on prepare because it is expected to be committed.
+        if (tx.remainingTime() == -1 && !tx.onePhaseCommit()) {
             tx.setRollbackOnly();
 
             throw new IgniteTxTimeoutCheckedException("Transaction timed out: " + this);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
index b36bf16dc9fdb..f7256cd12fea2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/TestRecordingCommunicationSpi.java
@@ -214,6 +214,26 @@ public void waitForBlocked(int cnt) throws InterruptedException {
         }
     }
 
+    /**
+     * @param size Size
+     * @param timeout Timeout.
+     * @throws InterruptedException
+     */
+    public boolean waitForBlocked(int size, long timeout) throws InterruptedException {
+        long t0 = U.currentTimeMillis() + timeout;
+
+        synchronized (this) {
+            while (blockedMsgs.size() < size) {
+                wait(1000);
+
+                if (U.currentTimeMillis() >= t0)
+                    return false;
+            }
+        }
+
+        return true;
+    }
+
     /**
      * @throws InterruptedException If interrupted.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutOnePhaseCommitTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutOnePhaseCommitTest.java
new file mode 100644
index 0000000000000..2375d9ed03231
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutOnePhaseCommitTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrepareResponse;
+import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionTimeoutException;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Tests rollback on timeout scenarios for one-phase commit protocol.
+ */
+public class TxRollbackOnTimeoutOnePhaseCommitTest extends GridCommonAbstractTest {
+    /** IP finder. */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private static final int GRID_CNT = 2;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        boolean client = igniteInstanceName.startsWith("client");
+
+        cfg.setClientMode(client);
+
+        if (!client) {
+            CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+            ccfg.setAtomicityMode(TRANSACTIONAL);
+            ccfg.setBackups(1);
+            ccfg.setWriteSynchronizationMode(FULL_SYNC);
+            ccfg.setOnheapCacheEnabled(false);
+
+            cfg.setCacheConfiguration(ccfg);
+        }
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        startGridsMultiThreaded(GRID_CNT);
+
+        startGrid("client");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+    }
+
+    /** */
+    public void testRollbackOnTimeoutPartitionDesyncPessimistic() throws Exception {
+        doTestRollbackOnTimeoutPartitionDesync(PESSIMISTIC);
+    }
+
+    /** */
+    public void testRollbackOnTimeoutPartitionDesyncOptimistic() throws Exception {
+        doTestRollbackOnTimeoutPartitionDesync(OPTIMISTIC);
+    }
+
+    /** */
+    public void testUnlockOptimistic() throws IgniteCheckedException {
+        IgniteEx client = grid("client");
+
+        assertNotNull(client.cache(DEFAULT_CACHE_NAME));
+
+        int key = 0;
+
+        CountDownLatch lock = new CountDownLatch(1);
+        CountDownLatch finish = new CountDownLatch(1);
+
+        IgniteInternalFuture fut = runAsync(() -> {
+            try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, 1)) {
+                client.cache(DEFAULT_CACHE_NAME).put(key, key + 1);
+
+                lock.countDown();
+
+                try {
+                    assertTrue(U.await(finish, 30, TimeUnit.SECONDS));
+                }
+                catch (IgniteInterruptedCheckedException e) {
+                    fail();
+                }
+
+                tx.commit();
+            }
+        });
+
+        try (Transaction tx = client.transactions().txStart(OPTIMISTIC, REPEATABLE_READ, 200, 1)) {
+            try {
+                assertTrue(U.await(lock, 30, TimeUnit.SECONDS));
+            }
+            catch (IgniteInterruptedCheckedException e) {
+                fail();
+            }
+
+            client.cache(DEFAULT_CACHE_NAME).put(key, key);
+
+            tx.commit();
+
+            // fail(); // TODO IGNITE-10027 throw timeout exception for optimistic timeout.
+        }
+        catch (Exception e) {
+            assertTrue(e.getClass().getName(), X.hasCause(e, TransactionTimeoutException.class));
+        }
+
+        assertNull(client.cache(DEFAULT_CACHE_NAME).get(key));
+
+        finish.countDown();
+
+        fut.get();
+
+        assertEquals(1, client.cache(DEFAULT_CACHE_NAME).get(key));
+    }
+
+    /** */
+    private void doTestRollbackOnTimeoutPartitionDesync(TransactionConcurrency concurrency) throws Exception {
+        IgniteEx client = grid("client");
+
+        assertNotNull(client.cache(DEFAULT_CACHE_NAME));
+
+        int key = 0;
+
+        Ignite primary = primaryNode(key, DEFAULT_CACHE_NAME);
+        Ignite backup = backupNode(key, DEFAULT_CACHE_NAME);
+
+        TestRecordingCommunicationSpi backupSpi = TestRecordingCommunicationSpi.spi(backup);
+        backupSpi.blockMessages(GridDhtTxPrepareResponse.class, primary.name());
+
+        IgniteInternalFuture fut = runAsync(() -> {
+            try {
+                backupSpi.waitForBlocked(1, 5000);
+            }
+            catch (InterruptedException e) {
+                fail();
+            }
+
+            doSleep(500);
+
+            backupSpi.stopBlock();
+        });
+
+        try (Transaction tx = client.transactions().txStart(concurrency, REPEATABLE_READ, 500, 1)) {
+            client.cache(DEFAULT_CACHE_NAME).put(key, key);
+
+            tx.commit();
+        }
+        catch (Exception e) {
+            assertTrue(e.getClass().getName(), X.hasCause(e, TransactionTimeoutException.class));
+        }
+
+        fut.get();
+
+        IdleVerifyResultV2 res = idleVerify(client, DEFAULT_CACHE_NAME);
+
+        if (res.hasConflicts()) {
+            StringBuilder b = new StringBuilder();
+
+            res.print(b::append);
+
+            fail(b.toString());
+        }
+
+        checkFutures();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 61f516773ab28..058f6243f58ba 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -43,6 +43,7 @@
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackAsyncTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNearCacheTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutOnePhaseCommitTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest;
 import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTopologyChangeTest;
 import org.apache.ignite.testframework.junits.GridAbstractTest;
@@ -76,6 +77,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(TxRollbackAsyncTest.class);
         suite.addTestSuite(TxRollbackAsyncNearCacheTest.class);
         suite.addTestSuite(TxRollbackOnTopologyChangeTest.class);
+        suite.addTestSuite(TxRollbackOnTimeoutOnePhaseCommitTest.class);
 
         suite.addTestSuite(TxOptimisticPrepareOnUnstableTopologyTest.class);
 

From c8dba6d7db166001e45dc31f198b5dec1b8d306d Mon Sep 17 00:00:00 2001
From: Ilya Borisov <iborisov@gridgain.com>
Date: Tue, 30 Oct 2018 22:33:07 +0700
Subject: [PATCH 467/543] IGNITE-9941 Fixed E2E tests.

---
 .../e2e/testcafe/page-models/pageSignup.js         | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/modules/web-console/e2e/testcafe/page-models/pageSignup.js b/modules/web-console/e2e/testcafe/page-models/pageSignup.js
index c380abed41907..83f2932b2c362 100644
--- a/modules/web-console/e2e/testcafe/page-models/pageSignup.js
+++ b/modules/web-console/e2e/testcafe/page-models/pageSignup.js
@@ -19,13 +19,13 @@ import {Selector, t} from 'testcafe';
 import {CustomFormField} from '../components/FormField';
 
 export const pageSignup = {
-    email: new CustomFormField({model: '$ctrl.data.email'}),
-    password: new CustomFormField({model: '$ctrl.data.password'}),
-    passwordConfirm: new CustomFormField({model: 'confirm'}),
-    firstName: new CustomFormField({model: '$ctrl.data.firstName'}),
-    lastName: new CustomFormField({model: '$ctrl.data.lastName'}),
-    company: new CustomFormField({model: '$ctrl.data.company'}),
-    country: new CustomFormField({model: '$ctrl.data.country'}),
+    email: new CustomFormField({id: 'emailInput'}),
+    password: new CustomFormField({id: 'passwordInput'}),
+    passwordConfirm: new CustomFormField({id: 'confirmInput'}),
+    firstName: new CustomFormField({id: 'firstNameInput'}),
+    lastName: new CustomFormField({id: 'lastNameInput'}),
+    company: new CustomFormField({id: 'companyInput'}),
+    country: new CustomFormField({id: 'countryInput'}),
     signupButton: Selector('button').withText('Sign Up'),
     async fillSignupForm({
         email,

From 51f2a60f991a9135b1aa5d3c16f6c42f34178018 Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Wed, 31 Oct 2018 11:54:36 +0300
Subject: [PATCH 468/543] IGNITE-10028 Fixed incorrect handling of page during
 replacement - Fixes #5093.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit e4881e343afaf4df1b42c2b7b4a15885489e4a72)
---
 .../GridCacheDatabaseSharedManager.java       | 26 ++++---------------
 1 file changed, 5 insertions(+), 21 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 8fcd2b5898b61..9058bdca0c2b0 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1154,13 +1154,11 @@ private long[] calculateFragmentSizes(int concLvl, long cacheSize, long chpBufSi
             (fullId, pageBuf, tag) -> {
                 memMetrics.onPageWritten();
 
-                // First of all, write page to disk.
-                storeMgr.write(fullId.groupId(), fullId.pageId(), pageBuf, tag);
-
-                pageBuf.rewind();
+                // We can write only page from disk into snapshot.
+                snapshotMgr.beforePageWrite(fullId);
 
-                // Only after write we can write page into snapshot.
-                snapshotMgr.flushDirtyPageHandler(fullId, pageBuf, tag);
+                // Write page to disk.
+                storeMgr.write(fullId.groupId(), fullId.pageId(), pageBuf, tag);
 
                 AtomicInteger cntr = evictedPagesCntr;
 
@@ -4025,8 +4023,6 @@ private WriteCheckpointPages(
         private List<FullPageId> writePages(Collection<FullPageId> writePageIds) throws IgniteCheckedException {
             ByteBuffer tmpWriteBuf = threadBuf.get();
 
-            long writeAddr = GridUnsafe.bufferAddress(tmpWriteBuf);
-
             List<FullPageId> pagesToRetry = new ArrayList<>();
 
             for (FullPageId fullId : writePageIds) {
@@ -4078,19 +4074,7 @@ private List<FullPageId> writePages(Collection<FullPageId> writePageIds) throws
                             tracker.onDataPageWritten();
                     }
 
-                    if (!skipCrc) {
-                        PageIO.setCrc(writeAddr, FastCrc.calcCrc(tmpWriteBuf, pageSize()));
-
-                        tmpWriteBuf.rewind();
-                    }
-
-                    int curWrittenPages = writtenPagesCntr.incrementAndGet();
-
-                    snapshotMgr.onPageWrite(fullId, tmpWriteBuf, curWrittenPages, totalPagesToWrite);
-
-                    tmpWriteBuf.rewind();
-
-                    PageStore store = storeMgr.writeInternal(grpId, fullId.pageId(), tmpWriteBuf, tag, false);
+                    PageStore store = storeMgr.writeInternal(grpId, fullId.pageId(), tmpWriteBuf, tag, true);
 
                     updStores.computeIfAbsent(store, k -> new LongAdder()).increment();
                 }

From a186c95aad41c69c8e5224e96b627df2706ef6a8 Mon Sep 17 00:00:00 2001
From: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Date: Tue, 14 Aug 2018 21:20:50 +0300
Subject: [PATCH 469/543] IGNITE-9990 Control.sh utility should request a
 password if necessary - Fixes #5079.

(cherry picked from commit a8713f6)
---
 .../internal/commandline/Arguments.java       | 22 +++++-
 .../internal/commandline/CommandHandler.java  | 79 +++++++++++++++----
 .../CommandHandlerParsingTest.java            | 22 +-----
 3 files changed, 83 insertions(+), 40 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index 0169ce2061092..32e1b662b3e71 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -71,10 +71,10 @@ public class Arguments {
      */
     private String walArgs;
 
-    /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}.*/
+    /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}. */
     private long pingTimeout;
 
-    /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}.*/
+    /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}. */
     private long pingInterval;
 
     /** */
@@ -175,17 +175,31 @@ public String port() {
     /**
      * @return user name
      */
-    public String user() {
+    public String getUserName() {
         return user;
     }
 
+    /**
+     * @param user New user name.
+     */
+    public void setUserName(String user) {
+        this.user = user;
+    }
+
     /**
      * @return password
      */
-    public String password() {
+    public String getPassword() {
         return pwd;
     }
 
+    /**
+     * @param pwd New password.
+     */
+    public void setPassword(String pwd) {
+        this.pwd = pwd;
+    }
+
     /**
      * @return baseline action
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index a66c0eb295878..a5afcc69f331d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.commandline;
 
+import java.io.Console;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -118,6 +119,7 @@
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
+import org.apache.ignite.plugin.security.SecurityCredentialsProvider;
 import org.apache.ignite.ssl.SslContextFactory;
 
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
@@ -1959,12 +1961,6 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         Command cmd = commands.get(0);
 
-        boolean hasUsr = F.isEmpty(user);
-        boolean hasPwd = F.isEmpty(pwd);
-
-        if (hasUsr != hasPwd)
-            throw new IllegalArgumentException("Both user and password should be specified");
-
         return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
             pingTimeout, pingInterval, autoConfirmation,
             sslEnable, sslProtocol, sslAlgorithm,
@@ -2528,11 +2524,6 @@ public int execute(List<String> rawArgs) {
 
             clientCfg.setServers(Collections.singletonList(args.host() + ":" + args.port()));
 
-            if (!F.isEmpty(args.user())) {
-                clientCfg.setSecurityCredentialsProvider(
-                    new SecurityCredentialsBasicProvider(new SecurityCredentials(args.user(), args.password())));
-            }
-
             if (args.isSslEnable()){
                 GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
 
@@ -2563,10 +2554,32 @@ public int execute(List<String> rawArgs) {
                 clientCfg.setSslContextFactory(factory);
             }
 
-            try (GridClient client = GridClientFactory.start(clientCfg)) {
-                switch (args.command()) {
-                    case ACTIVATE:
-                        activate(client);
+            boolean tryConnectAgain = true;
+
+            int tryConnectMaxCount=3;
+
+            while (tryConnectAgain) {
+                tryConnectAgain = false;
+
+                if (!F.isEmpty(args.getUserName())) {
+                    SecurityCredentialsProvider securityCredential = clientCfg.getSecurityCredentialsProvider();
+
+                    if (securityCredential == null) {
+                        securityCredential = new SecurityCredentialsBasicProvider(
+                            new SecurityCredentials(args.getUserName(), args.getPassword())
+                        );
+
+                        clientCfg.setSecurityCredentialsProvider(securityCredential);
+                    }
+                    final SecurityCredentials credential = securityCredential.credentials();
+                    credential.setLogin(args.getUserName());
+                    credential.setPassword(args.getPassword());
+                }
+
+                try (GridClient client = GridClientFactory.start(clientCfg)) {
+                    switch (args.command()) {
+                        case ACTIVATE:
+                            activate(client);
 
                         break;
 
@@ -2598,7 +2611,41 @@ public int execute(List<String> rawArgs) {
                     case WAL:
                         wal(client, args.walAction(), args.walArguments());
 
-                        break;
+                            break;
+                    }
+                }
+                catch (Throwable e) {
+                    if (tryConnectMaxCount > 0 && isAuthError(e)) {
+                        System.out.println("Authentication error, try connection again.");
+
+                        final Console console = System.console();
+
+                        if (console != null) {
+                            if (F.isEmpty(args.getUserName()))
+                                args.setUserName(console.readLine("user: "));
+
+                            args.setPassword(new String(console.readPassword("password: ")));
+                        }
+                        else {
+                            Scanner scanner = new Scanner(System.in);
+
+                            if (F.isEmpty(args.getUserName())){
+                                System.out.println("user: ");
+
+                                args.setUserName(scanner.next());
+                            }
+
+                            System.out.println("password: ");
+
+                            args.setPassword(scanner.next());
+                        }
+
+                        tryConnectAgain = true;
+
+                        tryConnectMaxCount--;
+                    }
+                    else
+                        throw e;
                 }
             }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 0ac5d1a4483b0..988b1d7e94a40 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -194,28 +194,10 @@ public void testParseAndValidateUserAndPassword() {
                 e.printStackTrace();
             }
 
-            try {
-                hnd.parseAndValidate(asList("--user", "testUser", cmd.text()));
-
-                fail("expected exception: Both user and password should be specified");
-            }
-            catch (IllegalArgumentException e) {
-                e.printStackTrace();
-            }
-
-            try {
-                hnd.parseAndValidate(asList("--password", "testPass", cmd.text()));
-
-                fail("expected exception: Both user and password should be specified");
-            }
-            catch (IllegalArgumentException e) {
-                e.printStackTrace();
-            }
-
             Arguments args = hnd.parseAndValidate(asList("--user", "testUser", "--password", "testPass", cmd.text()));
 
-            assertEquals("testUser", args.user());
-            assertEquals("testPass", args.password());
+            assertEquals("testUser", args.getUserName());
+            assertEquals("testPass", args.getPassword());
             assertEquals(cmd, args.command());
         }
     }

From ad0ca4c92e226264b127d3587b3b8cb187f439ac Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 31 Oct 2018 22:04:02 +0700
Subject: [PATCH 470/543] GG-14352 Web Console: fixes and optimizations to be
 able to work on large clusters with many caches.

---
 modules/web-console/backend/package.json      |  2 +-
 modules/web-console/frontend/app/app.js       |  3 -
 .../app/modules/agent/AgentManager.service.js | 21 ++--
 .../app/modules/agent/decompress.worker.js    |  8 +-
 .../frontend/app/modules/demo/Demo.module.js  | 26 ++---
 .../frontend/app/modules/socket.module.js     | 47 ---------
 .../frontend/app/utils/SimpleWorkerPool.js    |  2 +-
 modules/web-console/frontend/app/vendor.js    |  1 -
 modules/web-console/frontend/package.json     |  5 +-
 modules/web-console/web-agent/pom.xml         |  2 +-
 .../ignite/console/agent/AgentLauncher.java   |  2 +-
 .../db/dialect/OracleMetadataDialect.java     |  1 -
 .../agent/handlers/AbstractListener.java      |  9 +-
 .../agent/handlers/ClusterListener.java       | 97 +++++++++++++------
 .../console/agent/handlers/RestListener.java  |  2 +-
 .../console/agent/rest/RestExecutor.java      |  5 +-
 .../demo/service/DemoComputeLoadService.java  |  1 -
 .../console/demo/task/DemoComputeTask.java    |  1 -
 18 files changed, 113 insertions(+), 122 deletions(-)
 delete mode 100644 modules/web-console/frontend/app/modules/socket.module.js

diff --git a/modules/web-console/backend/package.json b/modules/web-console/backend/package.json
index 896b5fa8cd935..e2fc2ddf23f27 100644
--- a/modules/web-console/backend/package.json
+++ b/modules/web-console/backend/package.json
@@ -68,7 +68,7 @@
     "passport-local-mongoose": "4.0.0",
     "passport.socketio": "3.7.0",
     "pkg": "4.3.1",
-    "socket.io": "1.7.3",
+    "socket.io": "2.1.1",
     "uuid": "3.1.0"
   },
   "devDependencies": {
diff --git a/modules/web-console/frontend/app/app.js b/modules/web-console/frontend/app/app.js
index 939fa71667749..17e00b6aef13a 100644
--- a/modules/web-console/frontend/app/app.js
+++ b/modules/web-console/frontend/app/app.js
@@ -44,7 +44,6 @@ import './modules/configuration/configuration.module';
 import './modules/getting-started/GettingStarted.provider';
 import './modules/dialog/dialog.module';
 import './modules/ace.module';
-import './modules/socket.module';
 import './modules/loading/loading.module';
 import servicesModule from './services';
 // endignite
@@ -173,7 +172,6 @@ export default angular.module('ignite-console', [
     'ngSanitize',
     'ngMessages',
     // Third party libs.
-    'btford.socket-io',
     'dndLists',
     'gridster',
     'mgcrea.ngStrap',
@@ -197,7 +195,6 @@ export default angular.module('ignite-console', [
     'ignite-console.input-dialog',
     'ignite-console.user',
     'ignite-console.branding',
-    'ignite-console.socket',
     'ignite-console.agent',
     'ignite-console.nodes',
     'ignite-console.demo',
diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
index 8d2ced2ad8a0e..3a4817e6ba072 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
@@ -24,6 +24,8 @@ import 'rxjs/add/operator/partition';
 import 'rxjs/add/operator/takeUntil';
 import 'rxjs/add/operator/pluck';
 
+import io from 'socket.io-client';
+
 import AgentModal from './AgentModal.service';
 // @ts-ignore
 import Worker from './decompress.worker';
@@ -118,7 +120,7 @@ class ConnectionState {
 }
 
 export default class AgentManager {
-    static $inject = ['$rootScope', '$q', '$transitions', 'igniteSocketFactory', 'AgentModal', 'UserNotifications', 'IgniteVersion', 'ClusterLoginService'];
+    static $inject = ['$rootScope', '$q', '$transitions', 'AgentModal', 'UserNotifications', 'IgniteVersion', 'ClusterLoginService'];
 
     /** @type {ng.IScope} */
     $root;
@@ -163,17 +165,15 @@ export default class AgentManager {
      * @param {ng.IRootScopeService} $root
      * @param {ng.IQService} $q
      * @param {import('@uirouter/angularjs').TransitionService} $transitions
-     * @param {unknown} socketFactory
      * @param {import('./AgentModal.service').default} agentModal
      * @param {import('app/components/user-notifications/service').default} UserNotifications
      * @param {import('app/services/Version.service').default} Version
      * @param {import('./components/cluster-login/service').default} ClusterLoginSrv
      */
-    constructor($root, $q, $transitions, socketFactory, agentModal, UserNotifications, Version, ClusterLoginSrv) {
+    constructor($root, $q, $transitions, agentModal, UserNotifications, Version, ClusterLoginSrv) {
         this.$root = $root;
         this.$q = $q;
         this.$transitions = $transitions;
-        this.socketFactory = socketFactory;
         this.agentModal = agentModal;
         this.UserNotifications = UserNotifications;
         this.Version = Version;
@@ -216,7 +216,9 @@ export default class AgentManager {
         if (nonNil(this.socket))
             return;
 
-        this.socket = this.socketFactory();
+        const options = this.isDemoMode() ? {query: 'IgniteDemoMode=true'} : {};
+
+        this.socket = io.connect(options);
 
         const onDisconnect = () => {
             const conn = this.connectionSbj.getValue();
@@ -475,8 +477,13 @@ export default class AgentManager {
                         if (cluster.secured)
                             this.clustersSecrets.get(cluster.id).sessionToken = res.sessionToken;
 
-                        if (res.zipped)
-                            return this.pool.postMessage(res.data);
+                        if (res.zipped) {
+                            const taskId = _.get(params, 'taskId', '');
+
+                            const useBigIntJson = taskId.startsWith('querySql') || taskId.startsWith('queryFetch');
+
+                            return this.pool.postMessage({payload: res.data, useBigIntJson});
+                        }
 
                         return res;
 
diff --git a/modules/web-console/frontend/app/modules/agent/decompress.worker.js b/modules/web-console/frontend/app/modules/agent/decompress.worker.js
index 2fd294da5f1e1..e3de4a1c6e570 100644
--- a/modules/web-console/frontend/app/modules/agent/decompress.worker.js
+++ b/modules/web-console/frontend/app/modules/agent/decompress.worker.js
@@ -23,12 +23,16 @@ import bigIntJSON from 'json-bigint';
 // eslint-disable-next-line no-undef
 onmessage = function(e) {
     const data = e.data;
+    const payload = data.payload;
+    const useBigIntJson = data.useBigIntJson;
 
-    const binaryString = atob(data); // Decode from BASE64
+    const binaryString = atob(payload); // Decode from BASE64
 
     const unzipped = pako.inflate(binaryString, {to: 'string'});
 
-    const res = bigIntJSON({storeAsString: true}).parse(unzipped);
+    const res = useBigIntJson
+        ? bigIntJSON({storeAsString: true}).parse(unzipped)
+        : JSON.parse(unzipped);
 
     postMessage(_.get(res, 'result', res));
 };
diff --git a/modules/web-console/frontend/app/modules/demo/Demo.module.js b/modules/web-console/frontend/app/modules/demo/Demo.module.js
index 6c14e4a4de518..eee977e28a004 100644
--- a/modules/web-console/frontend/app/modules/demo/Demo.module.js
+++ b/modules/web-console/frontend/app/modules/demo/Demo.module.js
@@ -25,19 +25,15 @@ const DEMO_QUERY_STATE = {state: 'base.sql.notebook', params: {noteId: 'demo'}};
 /**
  * @param {import('@uirouter/angularjs').StateProvider} $state
  * @param {ng.IHttpProvider} $http
- * @param {unknown} socketFactory
  */
-export function DemoProvider($state, $http, socketFactory) {
+export function DemoProvider($state, $http) {
     if (/(\/demo.*)/ig.test(location.pathname))
         sessionStorage.setItem('IgniteDemoMode', 'true');
 
     const enabled = sessionStorage.getItem('IgniteDemoMode') === 'true';
 
-    if (enabled) {
-        socketFactory.set({query: 'IgniteDemoMode=true'});
-
+    if (enabled)
         $http.interceptors.push('demoInterceptor');
-    }
 
     function service($root) {
         $root.IgniteDemoMode = enabled;
@@ -50,7 +46,7 @@ export function DemoProvider($state, $http, socketFactory) {
     return this;
 }
 
-DemoProvider.$inject = ['$stateProvider', '$httpProvider', 'igniteSocketFactoryProvider'];
+DemoProvider.$inject = ['$stateProvider', '$httpProvider'];
 
 /**
  * @param {{enabled: boolean}} Demo
@@ -205,12 +201,10 @@ function config($stateProvider) {
 config.$inject = ['$stateProvider'];
 
 angular
-.module('ignite-console.demo', [
-    'ignite-console.socket'
-])
-.config(config)
-.provider('Demo', DemoProvider)
-.factory('demoInterceptor', demoInterceptor)
-.controller('demoController', demoController)
-.provider('igniteDemoInfo', igniteDemoInfoProvider)
-.service('DemoInfo', DemoInfo);
+    .module('ignite-console.demo', [])
+    .config(config)
+    .provider('Demo', DemoProvider)
+    .factory('demoInterceptor', demoInterceptor)
+    .controller('demoController', demoController)
+    .provider('igniteDemoInfo', igniteDemoInfoProvider)
+    .service('DemoInfo', DemoInfo);
diff --git a/modules/web-console/frontend/app/modules/socket.module.js b/modules/web-console/frontend/app/modules/socket.module.js
deleted file mode 100644
index 5a38bdb765134..0000000000000
--- a/modules/web-console/frontend/app/modules/socket.module.js
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import angular from 'angular';
-import io from 'socket.io-client'; // eslint-disable-line no-unused-vars
-
-angular
-.module('ignite-console.socket', [
-])
-.provider('igniteSocketFactory', function() {
-    let _options = {};
-
-    /**
-     * @param {Object} options Socket io options.
-     */
-    this.set = (options) => {
-        _options = options;
-    };
-
-    function factory(socketFactory) {
-        return function() {
-            const ioSocket = io.connect(_options);
-
-            return socketFactory({ioSocket});
-        };
-    }
-
-    factory.$inject = ['socketFactory'];
-
-    this.$get = factory;
-
-    return this;
-});
diff --git a/modules/web-console/frontend/app/utils/SimpleWorkerPool.js b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js
index 8e7aab74e849c..1beb418469ba8 100644
--- a/modules/web-console/frontend/app/utils/SimpleWorkerPool.js
+++ b/modules/web-console/frontend/app/utils/SimpleWorkerPool.js
@@ -90,7 +90,7 @@ export default class SimpleWorkerPool {
         worker.postMessage(task.data);
 
         if (this.__dbg)
-            console.timeEnd('Post message');
+            console.timeEnd(`Post message[pool=${this._name}]`);
     }
 
     terminate() {
diff --git a/modules/web-console/frontend/app/vendor.js b/modules/web-console/frontend/app/vendor.js
index 6961e962e30b2..3fe8e3c42948f 100644
--- a/modules/web-console/frontend/app/vendor.js
+++ b/modules/web-console/frontend/app/vendor.js
@@ -22,7 +22,6 @@ import 'angular-animate';
 import 'angular-sanitize';
 import 'angular-strap';
 import 'angular-strap/dist/angular-strap.tpl';
-import 'angular-socket-io';
 
 import 'angular-messages';
 import '@uirouter/angularjs';
diff --git a/modules/web-console/frontend/package.json b/modules/web-console/frontend/package.json
index 0aef839a2dfb8..cea2692f5b4ea 100644
--- a/modules/web-console/frontend/package.json
+++ b/modules/web-console/frontend/package.json
@@ -45,7 +45,6 @@
     "angular-nvd3": "1.0.9",
     "angular-sanitize": "1.7.4",
     "angular-smart-table": "2.1.11",
-    "angular-socket-io": "0.7.0",
     "angular-strap": "2.3.12",
     "angular-translate": "2.18.1",
     "angular-tree-control": "0.2.28",
@@ -61,7 +60,7 @@
     "file-saver": "1.3.3",
     "font-awesome": "4.7.0",
     "jquery": "3.2.1",
-    "json-bigint": "0.2.3",
+    "json-bigint": "0.3.0",
     "jsondiffpatch": "0.2.5",
     "jszip": "3.1.5",
     "lodash": "4.17.11",
@@ -71,7 +70,7 @@
     "pako": "1.0.6",
     "roboto-font": "0.1.0",
     "rxjs": "5.5.11",
-    "socket.io-client": "1.7.3",
+    "socket.io-client": "2.1.1",
     "tf-metatags": "2.0.0"
   },
   "devDependencies": {
diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml
index b7518c0955742..80284f1eb6c81 100644
--- a/modules/web-console/web-agent/pom.xml
+++ b/modules/web-console/web-agent/pom.xml
@@ -43,7 +43,7 @@
         <dependency>
             <groupId>io.socket</groupId>
             <artifactId>socket.io-client</artifactId>
-            <version>0.8.3</version>
+            <version>1.0.0</version>
         </dependency>
 
         <dependency>
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
index 579f236536e95..371ac7dc245ce 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/AgentLauncher.java
@@ -330,7 +330,7 @@ public static void main(String[] args) throws Exception {
             // Create an SSLContext that uses our TrustManager
             ctx.init(null, new TrustManager[] {trustManager()}, null);
 
-            opts.sslContext = ctx;
+            // TODO IGNITE-5617 Check SSL for socket-io 1.0.0 opts.sslContext = ctx;
         }
 
         final Socket client = IO.socket(uri, opts);
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/dialect/OracleMetadataDialect.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/dialect/OracleMetadataDialect.java
index b8b72d5422b94..f7c183f2461d7 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/dialect/OracleMetadataDialect.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/db/dialect/OracleMetadataDialect.java
@@ -39,7 +39,6 @@
 
 import static java.sql.Types.BIGINT;
 import static java.sql.Types.BLOB;
-import static java.sql.Types.BOOLEAN;
 import static java.sql.Types.CHAR;
 import static java.sql.Types.CLOB;
 import static java.sql.Types.DATE;
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
index 33e4c2ba3becb..e0cd5953af485 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/AbstractListener.java
@@ -95,12 +95,17 @@ else if (args.length == 1)
                     }
 
                     cb.call(null, toJSON(res));
-                } catch (Exception e) {
+                }
+                catch (Throwable e) {
+                    log.error("Failed to process event in pool", e);
+
                     cb.call(e, null);
                 }
             });
         }
-        catch (Exception e) {
+        catch (Throwable e) {
+            log.error("Failed to process event", e);
+
             cb.call(e, null);
         }
     }
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
index 779b458b61cae..5b9cda328f025 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/ClusterListener.java
@@ -20,16 +20,18 @@
 import io.socket.client.Socket;
 import java.io.IOException;
 import java.net.ConnectException;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledFuture;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteLogger;
 
 import com.fasterxml.jackson.core.type.TypeReference;
@@ -192,7 +194,7 @@ private static class TopologySnapshot {
         private String clusterName;
 
         /** */
-        private Collection<UUID> nids;
+        private Set<UUID> nids;
 
         /** */
         private Map<UUID, String> addrs;
@@ -229,7 +231,7 @@ private static <T> T attribute(Map<String, Object> attrs, String name) {
         TopologySnapshot(Collection<GridClientNodeBean> nodes) {
             int sz = nodes.size();
 
-            nids = new ArrayList<>(sz);
+            nids = new HashSet<>(sz);
             addrs = U.newHashMap(sz);
             clients = U.newHashMap(sz);
             active = false;
@@ -313,7 +315,7 @@ public void setSecured(boolean secured) {
         /**
          * @return Cluster nodes IDs.
          */
-        public Collection<UUID> getNids() {
+        public Set<UUID> getNids() {
             return nids;
         }
 
@@ -384,11 +386,11 @@ else if (!F.isEmpty(cfg.nodeLogin()) && !F.isEmpty(cfg.nodePassword())) {
                     sesTok = res.getSessionToken();
 
                     return res;
-                    
+
                 case STATUS_FAILED:
                     if (res.getError().startsWith(EXPIRED_SES_ERROR_MSG)) {
                         sesTok = null;
-                        
+
                         params.remove("sessionToken");
 
                         return restCommand(params);
@@ -402,14 +404,15 @@ else if (!F.isEmpty(cfg.nodeLogin()) && !F.isEmpty(cfg.nodePassword())) {
         /**
          * Collect topology.
          *
-         * @param full Full.
+         * @param attrs Whether to collect attributes.
+         * @return REST result.
          */
-        private RestResult topology(boolean full) throws IOException {
+        private RestResult topology(boolean attrs) throws IOException {
             Map<String, Object> params = U.newHashMap(3);
 
             params.put("cmd", "top");
-            params.put("attr", true);
-            params.put("mtr", full);
+            params.put("attr", attrs);
+            params.put("mtr", false);
             params.put("caches", false);
 
             return restCommand(params);
@@ -464,43 +467,75 @@ public boolean active(IgniteProductVersion ver, UUID nid) throws IOException {
             }
         }
 
+        /**
+         * @param res Response from topology command.
+         * @return List of cluster nodes.
+         * @throws Exception if failed to extract nodes list from response.
+         */
+        private List<GridClientNodeBean> nodes(RestResult res) throws Exception {
+            switch (res.getStatus()) {
+                case STATUS_SUCCESS:
+                    return MAPPER.readValue(res.getData(), new TypeReference<List<GridClientNodeBean>>() {});
 
-        /** {@inheritDoc} */
-        @Override public void run() {
-            try {
-                RestResult res = topology(false);
+                default:
+                    LT.warn(log, res.getError());
 
-                switch (res.getStatus()) {
-                    case STATUS_SUCCESS:
-                        List<GridClientNodeBean> nodes = MAPPER.readValue(res.getData(),
-                            new TypeReference<List<GridClientNodeBean>>() {});
+                    throw new ConnectException();
+            }
+        }
 
-                        TopologySnapshot newTop = new TopologySnapshot(nodes);
+        /**
+         * Collect light weight topology without attributes.
+         *
+         * @return Set of node IDs.
+         * @throws Exception If failed to collect topology.
+         */
+        private Set<UUID> newTopologyNids() throws Exception {
+            if (top == null)
+                return Collections.emptySet();
+
+            RestResult res = topology(false);
+
+            List<GridClientNodeBean> nodes = nodes(res);
+
+            return nodes.stream().map(GridClientNodeBean::getNodeId).collect(Collectors.toSet());
+        }
 
-                        if (newTop.differentCluster(top))
-                            log.info("Connection successfully established to cluster with nodes: " + newTop.nid8());
+        /** {@inheritDoc} */
+        @Override public void run() {
+            try {
+                // If topology changed, collect with attributes.
+                if (top == null || !top.getNids().equals(newTopologyNids())) {
+                    RestResult res = topology(true);
 
-                        boolean active = active(newTop.clusterVersion(), F.first(newTop.getNids()));
+                    List<GridClientNodeBean> nodes = nodes(res);
 
-                        newTop.setActive(active);
-                        newTop.setSecured(!F.isEmpty(res.getSessionToken()));
+                    TopologySnapshot newTop = new TopologySnapshot(nodes);
 
-                        top = newTop;
+                    if (newTop.differentCluster(top))
+                        log.info("Connection successfully established to cluster with nodes: " + newTop.nid8());
 
-                        client.emit(EVENT_CLUSTER_TOPOLOGY, toJSON(top));
+                    boolean active = active(newTop.clusterVersion(), F.first(newTop.getNids()));
 
-                        break;
+                    newTop.setActive(active);
+                    newTop.setSecured(!F.isEmpty(res.getSessionToken()));
 
-                    default:
-                        LT.warn(log, res.getError());
+                    top = newTop;
+                }
+                else {
+                    // If topology stable, just poll active state only.
+                    boolean active = active(top.clusterVersion(), F.first(top.getNids()));
 
-                        clusterDisconnect();
+                    if (active != top.isActive())
+                        top.setActive(active);
                 }
+
+                client.emit(EVENT_CLUSTER_TOPOLOGY, toJSON(top));
             }
             catch (ConnectException ignored) {
                 clusterDisconnect();
             }
-            catch (Exception e) {
+            catch (Throwable e) {
                 log.error("WatchTask failed", e);
 
                 clusterDisconnect();
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
index 24c2097531c6a..6b43b6a964bea 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/handlers/RestListener.java
@@ -84,7 +84,7 @@ public RestListener(AgentConfiguration cfg, RestExecutor restExecutor) {
 
             return restExecutor.sendRequest(this.cfg.nodeURIs(), params, headers);
         }
-        catch (Exception e) {
+        catch (Throwable e) {
             U.error(log, "Failed to execute REST command with parameters: " + params, e);
 
             return RestResult.fail(404, e.getMessage());
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java
index bb06c32579b86..4c48c916b5ed1 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/agent/rest/RestExecutor.java
@@ -94,7 +94,8 @@ public RestExecutor() {
                 builder.sslSocketFactory(ctx.getSocketFactory(), trustManager());
 
                 builder.hostnameVerifier((hostname, session) -> true);
-            } catch (Exception ignored) {
+            }
+            catch (Throwable ignored) {
                 LT.warn(log, "Failed to initialize the Trust Manager for \"-Dtrust.all\" option to skip certificate validation.");
             }
         }
@@ -136,7 +137,7 @@ private RestResult parseResponse(Response res) throws IOException {
         if (res.code() == 404)
             return RestResult.fail(STATUS_FAILED, "Failed connect to cluster.");
 
-        return RestResult.fail(STATUS_FAILED, "Failed to execute REST command: " + res.message());
+        return RestResult.fail(STATUS_FAILED, "Failed to execute REST command: " + res);
     }
 
     /** */
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoComputeLoadService.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoComputeLoadService.java
index e775a9a6dd337..3c767c85dac91 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoComputeLoadService.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/service/DemoComputeLoadService.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.console.demo.service;
 
-import java.util.Random;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import org.apache.ignite.Ignite;
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/task/DemoComputeTask.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/task/DemoComputeTask.java
index 14f2508d82e9b..7dba1b73465c8 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/task/DemoComputeTask.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/task/DemoComputeTask.java
@@ -22,7 +22,6 @@
 import java.util.Map;
 import java.util.Random;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobAdapter;

From 548a81a972459c55f229333b4e83e9684026bf8f Mon Sep 17 00:00:00 2001
From: Evgeny Stanilovskiy <estanilovskiy@gridgain.com>
Date: Thu, 1 Nov 2018 10:11:44 +0300
Subject: [PATCH 471/543] IGNITE-9418 Avoid initialize file page store manager
 for caches during PME synchronously - Fixes #4827.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
(cherry picked from commit 31bf38d37ff0b73b24a8fa0034630d833eee1f75)
---
 .../processors/cache/GridCacheProcessor.java  |  2 -
 .../GridDhtPartitionsExchangeFuture.java      |  8 ---
 .../file/FilePageStoreManager.java            | 57 +++++++++++++++----
 3 files changed, 46 insertions(+), 21 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index f20c8e3c8a006..ee814ed358926 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2080,8 +2080,6 @@ private GridCacheContext prepareCacheContext(
 
         boolean affNode = checkForAffinityNode(desc, reqNearCfg, ccfg);
 
-        preparePageStore(desc, affNode);
-
         CacheGroupContext grp = prepareCacheGroup(desc, exchTopVer, cacheObjCtx, affNode, startCfg.getGroupName());
 
         GridCacheContext cacheCtx = createCache(ccfg,
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 3239782a17c49..6d63b5b460b24 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -851,14 +851,6 @@ private IgniteInternalFuture<?> initCachesOnLocalJoin() throws IgniteCheckedExce
         if (!isLocalNodeInBaseline()) {
             cctx.database().cleanupRestoredCaches();
 
-            for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) {
-                if (CU.isPersistentCache(desc.cacheConfiguration(),
-                    cctx.gridConfig().getDataStorageConfiguration())) {
-                    // Perform cache init from scratch.
-                    cctx.cache().preparePageStore(desc, true);
-                }
-            }
-
             // Set initial node started marker.
             cctx.database().nodeStart(null);
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 93cd1bc1e4d1b..e0516482ea304 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -41,6 +41,7 @@
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
@@ -63,6 +64,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.marshaller.Marshaller;
@@ -379,16 +381,15 @@ public FilePageStoreManager(GridKernalContext ctx) {
 
         CacheStoreHolder old = idxCacheStores.remove(grp.groupId());
 
-        assert old != null : "Missing cache store holder [cache=" + grp.cacheOrGroupName() +
-            ", locNodeId=" + cctx.localNodeId() + ", gridName=" + cctx.igniteInstanceName() + ']';
+        if (old != null) {
+            IgniteCheckedException ex = shutdown(old, /*clean files if destroy*/destroy, null);
 
-        IgniteCheckedException ex = shutdown(old, /*clean files if destroy*/destroy, null);
+            if (destroy)
+                removeCacheGroupConfigurationData(grp);
 
-        if (destroy)
-            removeCacheGroupConfigurationData(grp);
-
-        if (ex != null)
-            throw ex;
+            if (ex != null)
+                throw ex;
+        }
     }
 
     /** {@inheritDoc} */
@@ -570,8 +571,9 @@ private CacheStoreHolder initDir(File cacheWorkDir,
 
             return new CacheStoreHolder(idxStore, partStores);
         }
-        catch (StorageException e) {
-            cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
+        catch (IgniteCheckedException e) {
+            if (X.hasCause(e, StorageException.class, IOException.class))
+                cctx.kernalContext().failure().process(new FailureContext(FailureType.CRITICAL_ERROR, e));
 
             throw e;
         }
@@ -959,6 +961,39 @@ private IgniteCheckedException shutdown(FilePageStore store, boolean cleanFile,
         return aggr;
     }
 
+    /**
+     * Return cache store holedr.
+     *
+     * @param grpId Cache group ID.
+     * @return Cache store holder.
+     */
+    private CacheStoreHolder getHolder(int grpId) throws IgniteCheckedException {
+        try {
+            return idxCacheStores.computeIfAbsent(grpId, (key) -> {
+                CacheGroupDescriptor gDesc = cctx.cache().cacheGroupDescriptors().get(grpId);
+
+                CacheStoreHolder holder0 = null;
+
+                if (gDesc != null) {
+                    if (CU.isPersistentCache(gDesc.config(), cctx.gridConfig().getDataStorageConfiguration())) {
+                        try {
+                            holder0 = initForCache(gDesc, gDesc.config());
+                        } catch (IgniteCheckedException e) {
+                            throw new IgniteException(e);
+                        }
+                    }
+                }
+
+                return holder0;
+            });
+        } catch (IgniteException ex) {
+            if (X.hasCause(ex, IgniteCheckedException.class))
+                throw ex.getCause(IgniteCheckedException.class);
+            else
+                throw ex;
+        }
+    }
+
     /**
      * @param grpId Cache group ID.
      * @param partId Partition ID.
@@ -968,7 +1003,7 @@ private IgniteCheckedException shutdown(FilePageStore store, boolean cleanFile,
      * Note: visible for testing.
      */
     public PageStore getStore(int grpId, int partId) throws IgniteCheckedException {
-        CacheStoreHolder holder = idxCacheStores.get(grpId);
+        CacheStoreHolder holder = getHolder(grpId);
 
         if (holder == null)
             throw new IgniteCheckedException("Failed to get page store for the given cache ID " +

From b46a69df05396df0935d30c8c20ee15f1b934cc6 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Thu, 1 Nov 2018 21:26:10 +0300
Subject: [PATCH 472/543] IGNITE-9420 Move logical recovery outside of PME -
 Fixes #5067.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit c076aee4eed3c121ec92314dd95ac9d5fcaaccb7)
(cherry picked from commit ed1493281d52c9003670d2503d9112b65ad1a3c1)
---
 .../ignite/internal/GridKernalContext.java    |   5 +
 .../internal/GridKernalContextImpl.java       |  15 +
 .../apache/ignite/internal/IgniteKernal.java  |   2 +
 .../eventstorage/GridEventStorageManager.java |   3 +
 .../mem/unsafe/UnsafeMemoryProvider.java      |   4 +-
 .../pagemem/wal/record/DataRecord.java        |   7 +
 .../wal/record/MemoryRecoveryRecord.java      |   9 +-
 .../wal/record/MetastoreDataRecord.java       |   8 +-
 .../MetaPageUpdatePartitionDataRecord.java    |   3 +-
 .../cache/CacheAffinitySharedManager.java     |   9 +-
 .../processors/cache/CacheGroupContext.java   | 287 +++++-
 .../processors/cache/GridCacheContext.java    |  44 +-
 .../cache/GridCacheEventManager.java          |   4 +
 .../processors/cache/GridCacheProcessor.java  | 345 +++++--
 .../GridCacheSharedTtlCleanupManager.java     |  14 +-
 .../GridDhtPartitionsExchangeFuture.java      |  28 +-
 .../GridDhtPartitionTopologyImpl.java         |   2 -
 .../DatabaseLifecycleListener.java            |  66 ++
 .../GridCacheDatabaseSharedManager.java       | 939 +++++++++---------
 .../IgniteCacheDatabaseSharedManager.java     | 216 ++--
 .../persistence/file/AsyncFileIOFactory.java  |   9 -
 .../cache/persistence/file/FileIOFactory.java |  10 +-
 .../cache/persistence/file/FilePageStore.java |   7 +-
 .../file/RandomAccessFileIOFactory.java       |   9 -
 .../MetastorageLifecycleListener.java         |   4 +-
 .../partstate/PartitionRecoverState.java      |  52 +
 .../snapshot/IgniteCacheSnapshotManager.java  |   4 -
 .../wal/FileWriteAheadLogManager.java         |  56 +-
 .../FsyncModeFileWriteAheadLogManager.java    |  48 +-
 .../reader/StandaloneGridKernalContext.java   |   5 +
 .../GridInternalSubscriptionProcessor.java    |  18 +
 .../IgniteClusterActivateDeactivateTest.java  |   2 +-
 ...PdsBinaryMetadataOnClusterRestartTest.java |   6 +-
 .../IgnitePdsCorruptedIndexTest.java          |  10 -
 .../IgnitePdsCorruptedStoreTest.java          |   3 +
 .../IgnitePdsPartitionFilesDestroyTest.java   |  10 -
 .../IgnitePdsTaskCancelingTest.java           |   5 -
 ...alModeChangeDuringRebalancingSelfTest.java |   5 -
 .../db/CheckpointBufferDeadlockTest.java      |   5 -
 .../db/IgniteLogicalRecoveryTest.java         | 560 +++++++++++
 .../db/IgnitePdsDataRegionMetricsTest.java    |  23 +-
 .../IgnitePdsDiskErrorsRecoveringTest.java    |  71 +-
 ...IgniteNodeStoppedDuringDisableWALTest.java |  10 +-
 .../db/wal/IgniteWalFlushFailoverTest.java    |   5 -
 ...lushMultiNodeFailoverAbstractSelfTest.java |   5 -
 .../wal/IgniteWalFormatFileFailoverTest.java  |   9 +-
 .../wal/IgniteWalHistoryReservationsTest.java |  16 +-
 .../db/wal/IgniteWalRebalanceTest.java        |   5 -
 .../wal/WalRecoveryTxLogicalRecordsTest.java  |   4 +-
 .../pagemem/PagesWriteThrottleSmokeTest.java  |   5 -
 .../AbstractNodeJoinTemplate.java             |   4 +-
 .../StandaloneWalRecordsIteratorTest.java     |   5 -
 ...tegrityWithPrimaryIndexCorruptionTest.java |   3 -
 .../hashmap/GridCacheTestContext.java         |   1 +
 .../junits/GridAbstractTest.java              |   7 +-
 .../AlignedBuffersDirectFileIOFactory.java    |   5 -
 .../IgnitePdsWithIndexingCoreTestSuite.java   |   3 +
 57 files changed, 2124 insertions(+), 895 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionRecoverState.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
index a4560f95389b5..e30a49c9eb4c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java
@@ -684,4 +684,9 @@ public interface GridKernalContext extends Iterable<GridComponent> {
      * @return Default uncaught exception handler used by thread pools.
      */
     public Thread.UncaughtExceptionHandler uncaughtExceptionHandler();
+
+    /**
+     * @return {@code True} if node is in recovery mode (before join to topology).
+     */
+    public boolean recoveryMode();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
index dfb51bcf18225..444f40828f495 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java
@@ -405,6 +405,9 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable
     /** Failure processor. */
     private FailureProcessor failureProc;
 
+    /** Recovery mode flag. Flag is set to {@code false} when discovery manager started. */
+    private boolean recoveryMode = true;
+
     /**
      * No-arg constructor is required by externalization.
      */
@@ -1156,6 +1159,18 @@ public Thread.UncaughtExceptionHandler uncaughtExceptionHandler() {
         return hnd;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean recoveryMode() {
+        return recoveryMode;
+    }
+
+    /**
+     * @param recoveryMode Recovery mode.
+     */
+    public void recoveryMode(boolean recoveryMode) {
+        this.recoveryMode = recoveryMode;
+    }
+
     /** {@inheritDoc} */
     @Override public String toString() {
         return S.toString(GridKernalContextImpl.class, this);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index b16e95fc89f50..d594ce8fc57ad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1038,6 +1038,8 @@ public void start(
                 fillNodeAttributes(clusterProc.updateNotifierEnabled());
 
                 ctx.cache().context().database().startMemoryRestore(ctx);
+
+                ctx.recoveryMode(false);
             }
             catch (Throwable e) {
                 U.error(
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
index 68084500757ca..e06d1100facd3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageManager.java
@@ -316,6 +316,9 @@ public void record(DiscoveryEvent evt, DiscoCache discoCache) {
     private void record0(Event evt, Object... params) {
         assert evt != null;
 
+        if (ctx.recoveryMode())
+            return;
+
         if (!enterBusy())
             return;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
index f9607f5eaccba..8cb8119001a23 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/mem/unsafe/UnsafeMemoryProvider.java
@@ -59,7 +59,7 @@ public UnsafeMemoryProvider(IgniteLogger log) {
     /** {@inheritDoc} */
     @Override public void initialize(long[] sizes) {
         if (isInit)
-            throw new IgniteException("Second initialization does not allowed for current provider");
+            return;
 
         this.sizes = sizes;
 
@@ -106,7 +106,7 @@ public UnsafeMemoryProvider(IgniteLogger log) {
             String msg = "Failed to allocate next memory chunk: " + U.readableSize(chunkSize, true) +
                 ". Check if chunkSize is too large and 32-bit JVM is used.";
 
-            if (regions.size() == 0)
+            if (regions.isEmpty())
                 throw new IgniteException(msg, e);
 
             U.error(log, msg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
index 7a4d6b8793a21..d5ab53a7226c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/DataRecord.java
@@ -76,6 +76,13 @@ public DataRecord(List<DataEntry> writeEntries, long timestamp) {
         this.writeEntries = writeEntries;
     }
 
+    /**
+     * @param writeEntries Write entries.
+     */
+    public void setWriteEntries(List<DataEntry> writeEntries) {
+        this.writeEntries = writeEntries;
+    }
+
     /**
      * @return Collection of write entries.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java
index 92658cc4ce788..5a48b340d0ca4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MemoryRecoveryRecord.java
@@ -20,16 +20,11 @@
 import org.apache.ignite.internal.util.typedef.internal.S;
 
 /**
- * Marker that we start memory recovering.
- *
- * @deprecated Previously, used to track node started\stopped states. But in fact only
- * mark files created by method GridCacheDatabaseSharedManager#nodeStart(WALPointer)
- * used. Should be removed in 3.0 release.
+ * Marker indicates that binary memory recovery has finished.
  */
-@Deprecated
 public class MemoryRecoveryRecord extends WALRecord {
     /** Create timestamp, millis */
-    private long time;
+    private final long time;
 
     /**
      * Default constructor.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
index e269de2adc012..9e734244c7b84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
@@ -18,13 +18,14 @@
 
 package org.apache.ignite.internal.pagemem.wal.record;
 
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
 
 /**
  *
  */
-public class MetastoreDataRecord extends WALRecord {
+public class MetastoreDataRecord extends WALRecord implements WalRecordCacheGroupAware {
     /** */
     private final String key;
 
@@ -59,4 +60,9 @@ public String key() {
     @Override public String toString() {
         return S.toString(MetastoreDataRecord.class, this, "super", super.toString());
     }
+
+    /** {@inheritDoc} */
+    @Override public int groupId() {
+        return MetaStorage.METASTORAGE_CACHE_ID;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
index e5bd343bcb5ee..28294a9f2cd5b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
@@ -55,7 +55,8 @@ public MetaPageUpdatePartitionDataRecord(
         long updateCntr,
         long globalRmvId,
         int partSize,
-        long cntrsPageId, byte state,
+        long cntrsPageId,
+        byte state,
         int allocatedIdxCandidate
     ) {
         super(grpId, pageId);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 20d852977932e..76be2b6eda23f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import javax.cache.CacheException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -32,6 +31,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.stream.Collectors;
+import javax.cache.CacheException;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteSystemProperties;
@@ -361,11 +361,7 @@ public Set<Integer> waitGroups() {
     void onCacheGroupCreated(CacheGroupContext grp) {
         if (!grpHolders.containsKey(grp.groupId())) {
             cctx.io().addCacheGroupHandler(grp.groupId(), GridDhtAffinityAssignmentResponse.class,
-                new IgniteBiInClosure<UUID, GridDhtAffinityAssignmentResponse>() {
-                    @Override public void apply(UUID nodeId, GridDhtAffinityAssignmentResponse res) {
-                        processAffinityAssignmentResponse(nodeId, res);
-                    }
-                });
+                (IgniteBiInClosure<UUID, GridDhtAffinityAssignmentResponse>) this::processAffinityAssignmentResponse);
         }
     }
 
@@ -1265,6 +1261,7 @@ private void forAllCacheGroups(boolean crd, IgniteInClosureX<GridAffinityAssignm
         else {
             affinityCaches = cctx.kernalContext().cache().cacheGroups().stream()
                 .filter(grp -> !grp.isLocal())
+                .filter(grp -> !grp.isRecoveryMode())
                 .map(CacheGroupContext::affinity)
                 .collect(Collectors.toList());
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index 39546a715490f..e5ec83da90444 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -22,8 +22,10 @@
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.affinity.AffinityFunction;
@@ -39,6 +41,9 @@
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentRequest;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.PartitionsEvictManager;
@@ -46,6 +51,10 @@
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionRecoverState;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
 import org.apache.ignite.internal.processors.cache.query.continuous.CounterSkipContext;
 import org.apache.ignite.internal.processors.query.QueryUtils;
@@ -77,13 +86,13 @@ public class CacheGroupContext {
     private final int grpId;
 
     /** Node ID cache group was received from. */
-    private final UUID rcvdFrom;
+    private volatile UUID rcvdFrom;
 
     /** Flag indicating that this cache group is in a recovery mode due to partitions loss. */
     private boolean needsRecovery;
 
     /** */
-    private final AffinityTopologyVersion locStartVer;
+    private volatile AffinityTopologyVersion locStartVer;
 
     /** */
     private final CacheConfiguration<?, ?> ccfg;
@@ -92,7 +101,7 @@ public class CacheGroupContext {
     private final GridCacheSharedContext ctx;
 
     /** */
-    private final boolean affNode;
+    private volatile boolean affNode;
 
     /** */
     private final CacheType cacheType;
@@ -116,17 +125,16 @@ public class CacheGroupContext {
     private final IgniteLogger log;
 
     /** */
-    private GridAffinityAssignmentCache aff;
+    private volatile GridAffinityAssignmentCache aff;
 
     /** */
-    private GridDhtPartitionTopologyImpl top;
+    private volatile GridDhtPartitionTopologyImpl top;
 
     /** */
-    private IgniteCacheOffheapManager offheapMgr;
+    private volatile IgniteCacheOffheapManager offheapMgr;
 
     /** */
-    private GridCachePreloader preldr;
-
+    private volatile GridCachePreloader preldr;
     /** */
     private final DataRegion dataRegion;
 
@@ -157,6 +165,12 @@ public class CacheGroupContext {
     /** */
     private volatile boolean globalWalEnabled;
 
+    /** Flag indicates that cache group is under recovering and not attached to topology. */
+    private final AtomicBoolean recoveryMode;
+
+    /** Flag indicates that all group partitions have restored their state from page memory / disk. */
+    private volatile boolean partitionStatesRestored;
+
     /**
      * @param ctx Context.
      * @param grpId Group ID.
@@ -185,7 +199,8 @@ public class CacheGroupContext {
         ReuseList reuseList,
         AffinityTopologyVersion locStartVer,
         boolean persistenceEnabled,
-        boolean walEnabled
+        boolean walEnabled,
+        boolean recoveryMode
     ) {
         assert ccfg != null;
         assert dataRegion != null || !affNode;
@@ -205,8 +220,7 @@ public class CacheGroupContext {
         this.globalWalEnabled = walEnabled;
         this.persistenceEnabled = persistenceEnabled;
         this.localWalEnabled = true;
-
-        persistGlobalWalState(walEnabled);
+        this.recoveryMode = new AtomicBoolean(recoveryMode);
 
         ioPlc = cacheType.ioPolicy();
 
@@ -704,9 +718,11 @@ public boolean sharedGroup() {
      *
      */
     public void onKernalStop() {
-        aff.cancelFutures(new IgniteCheckedException("Failed to wait for topology update, node is stopping."));
+        if (!isRecoveryMode()) {
+            aff.cancelFutures(new IgniteCheckedException("Failed to wait for topology update, node is stopping."));
 
-        preldr.onKernalStop();
+            preldr.onKernalStop();
+        }
 
         offheapMgr.onKernalStop();
     }
@@ -728,6 +744,11 @@ void stopCache(GridCacheContext cctx, boolean destroy) {
      *
      */
     void stopGroup() {
+        offheapMgr.stop();
+
+        if (isRecoveryMode())
+            return;
+
         IgniteCheckedException err =
             new IgniteCheckedException("Failed to wait for topology update, cache (or node) is stopping.");
 
@@ -737,11 +758,204 @@ void stopGroup() {
 
         preldr.onKernalStop();
 
-        offheapMgr.stop();
-
         ctx.io().removeCacheGroupHandlers(grpId);
     }
 
+    /**
+     * Finishes recovery for current cache group.
+     * Attaches topology version and initializes I/O.
+     *
+     * @param startVer Cache group start version.
+     * @param originalReceivedFrom UUID of node that was first who initiated cache group creating.
+     *                             This is needed to decide should node calculate affinity locally or fetch from other nodes.
+     * @param affinityNode Flag indicates, is local node affinity node or not. This may be calculated only after node joined to topology.
+     * @throws IgniteCheckedException If failed.
+     */
+    public void finishRecovery(
+        AffinityTopologyVersion startVer,
+        UUID originalReceivedFrom,
+        boolean affinityNode
+    ) throws IgniteCheckedException {
+        if (recoveryMode.compareAndSet(true, false)) {
+            affNode = affinityNode;
+
+            rcvdFrom = originalReceivedFrom;
+
+            locStartVer = startVer;
+
+            persistGlobalWalState(globalWalEnabled);
+
+            initializeIO();
+
+            ctx.affinity().onCacheGroupCreated(this);
+        }
+    }
+
+    /**
+     * Pre-create partitions that resides in page memory or WAL and restores their state.
+     */
+    public long restorePartitionStates(Map<GroupPartitionId, PartitionRecoverState> partitionRecoveryStates) throws IgniteCheckedException {
+        if (isLocal() || !affinityNode() || !dataRegion().config().isPersistenceEnabled())
+            return 0;
+
+        if (partitionStatesRestored)
+            return 0;
+
+        long processed = 0;
+
+        PageMemoryEx pageMem = (PageMemoryEx)dataRegion().pageMemory();
+
+        for (int p = 0; p < affinity().partitions(); p++) {
+            PartitionRecoverState recoverState = partitionRecoveryStates.get(new GroupPartitionId(grpId, p));
+
+            if (ctx.pageStore().exists(grpId, p)) {
+                ctx.pageStore().ensure(grpId, p);
+
+                if (ctx.pageStore().pages(grpId, p) <= 1) {
+                    if (log.isDebugEnabled())
+                        log.debug("Skipping partition on recovery (pages less than 1) " +
+                            "[grp=" + cacheOrGroupName() + ", p=" + p + "]");
+
+                    continue;
+                }
+
+                if (log.isDebugEnabled())
+                    log.debug("Creating partition on recovery (exists in page store) " +
+                        "[grp=" + cacheOrGroupName() + ", p=" + p + "]");
+
+                processed++;
+
+                GridDhtLocalPartition part = topology().forceCreatePartition(p);
+
+                offheap().onPartitionInitialCounterUpdated(p, 0);
+
+                ctx.database().checkpointReadLock();
+
+                try {
+                    long partMetaId = pageMem.partitionMetaPageId(grpId, p);
+                    long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
+
+                    try {
+                        long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
+
+                        boolean changed = false;
+
+                        try {
+                            PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
+
+                            if (recoverState != null) {
+                                io.setPartitionState(pageAddr, (byte) recoverState.stateId());
+
+                                changed = updateState(part, recoverState.stateId());
+
+                                if (recoverState.stateId() == GridDhtPartitionState.OWNING.ordinal()
+                                    || (recoverState.stateId() == GridDhtPartitionState.MOVING.ordinal()
+                                    && part.initialUpdateCounter() < recoverState.updateCounter())) {
+                                    part.initialUpdateCounter(recoverState.updateCounter());
+
+                                    changed = true;
+                                }
+
+                                if (log.isInfoEnabled())
+                                    log.warning("Restored partition state (from WAL) " +
+                                        "[grp=" + cacheOrGroupName() + ", p=" + p + ", state=" + part.state() +
+                                        ", updCntr=" + part.initialUpdateCounter() + "]");
+                            }
+                            else {
+                                int stateId = (int) io.getPartitionState(pageAddr);
+
+                                changed = updateState(part, stateId);
+
+                                if (log.isDebugEnabled())
+                                    log.debug("Restored partition state (from page memory) " +
+                                        "[grp=" + cacheOrGroupName() + ", p=" + p + ", state=" + part.state() +
+                                        ", updCntr=" + part.initialUpdateCounter() + ", stateId=" + stateId + "]");
+                            }
+                        }
+                        finally {
+                            pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
+                        }
+                    }
+                    finally {
+                        pageMem.releasePage(grpId, partMetaId, partMetaPage);
+                    }
+                }
+                finally {
+                    ctx.database().checkpointReadUnlock();
+                }
+            }
+            else if (recoverState != null) {
+                GridDhtLocalPartition part = topology().forceCreatePartition(p);
+
+                offheap().onPartitionInitialCounterUpdated(p, recoverState.updateCounter());
+
+                updateState(part, recoverState.stateId());
+
+                processed++;
+
+                if (log.isDebugEnabled())
+                    log.debug("Restored partition state (from WAL) " +
+                        "[grp=" + cacheOrGroupName() + ", p=" + p + ", state=" + part.state() +
+                        ", updCntr=" + part.initialUpdateCounter() + "]");
+            }
+            else {
+                if (log.isDebugEnabled())
+                    log.debug("Skipping partition on recovery (no page store OR wal state) " +
+                        "[grp=" + cacheOrGroupName() + ", p=" + p + "]");
+            }
+        }
+
+        partitionStatesRestored = true;
+
+        return processed;
+    }
+
+    /**
+     * @param part Partition to restore state for.
+     * @param stateId State enum ordinal.
+     * @return Updated flag.
+     */
+    private boolean updateState(GridDhtLocalPartition part, int stateId) {
+        if (stateId != -1) {
+            GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(stateId);
+
+            assert state != null;
+
+            part.restoreState(state == GridDhtPartitionState.EVICTED ? GridDhtPartitionState.RENTING : state);
+
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * @return {@code True} if current cache group is in recovery mode.
+     */
+    public boolean isRecoveryMode() {
+        return recoveryMode.get();
+    }
+
+    /**
+     * Initializes affinity and rebalance I/O handlers.
+     */
+    private void initializeIO() throws IgniteCheckedException {
+        assert !recoveryMode.get() : "Couldn't initialize I/O handlers, recovery mode is on for group " + this;
+
+        if (ccfg.getCacheMode() != LOCAL) {
+            if (!ctx.kernalContext().clientNode()) {
+                ctx.io().addCacheGroupHandler(groupId(), GridDhtAffinityAssignmentRequest.class,
+                    (IgniteBiInClosure<UUID, GridDhtAffinityAssignmentRequest>) this::processAffinityAssignmentRequest);
+            }
+
+            preldr = new GridDhtPreloader(this);
+
+            preldr.start();
+        }
+        else
+            preldr = new GridCachePreloaderAdapter(this);
+    }
+
     /**
      * @return IDs of caches in this group.
      */
@@ -766,7 +980,7 @@ public List<GridCacheContext> caches() {
     /**
      * @return {@code True} if group contains caches.
      */
-    boolean hasCaches() {
+    public boolean hasCaches() {
         List<GridCacheContext> caches = this.caches;
 
         return !caches.isEmpty();
@@ -888,39 +1102,25 @@ public void start() throws IgniteCheckedException {
             ccfg.getCacheMode() == LOCAL,
             persistenceEnabled());
 
-        if (ccfg.getCacheMode() != LOCAL) {
+        if (ccfg.getCacheMode() != LOCAL)
             top = new GridDhtPartitionTopologyImpl(ctx, this);
 
-            if (!ctx.kernalContext().clientNode()) {
-                ctx.io().addCacheGroupHandler(groupId(), GridDhtAffinityAssignmentRequest.class,
-                    new IgniteBiInClosure<UUID, GridDhtAffinityAssignmentRequest>() {
-                        @Override public void apply(UUID nodeId, GridDhtAffinityAssignmentRequest msg) {
-                            processAffinityAssignmentRequest(nodeId, msg);
-                        }
-                    });
-            }
-
-            preldr = new GridDhtPreloader(this);
-
-            preldr.start();
+        try {
+            offheapMgr = persistenceEnabled
+                ? new GridCacheOffheapManager()
+                : new IgniteCacheOffheapManagerImpl();
         }
-        else
-            preldr = new GridCachePreloaderAdapter(this);
-
-        if (persistenceEnabled()) {
-            try {
-                offheapMgr = new GridCacheOffheapManager();
-            }
-            catch (Exception e) {
-                throw new IgniteCheckedException("Failed to initialize offheap manager", e);
-            }
+        catch (Exception e) {
+            throw new IgniteCheckedException("Failed to initialize offheap manager", e);
         }
-        else
-            offheapMgr = new IgniteCacheOffheapManagerImpl();
 
         offheapMgr.start(ctx, this);
 
-        ctx.affinity().onCacheGroupCreated(this);
+        if (!isRecoveryMode()) {
+            initializeIO();
+
+            ctx.affinity().onCacheGroupCreated(this);
+        }
     }
 
     /**
@@ -934,8 +1134,7 @@ public boolean persistenceEnabled() {
      * @param nodeId Node ID.
      * @param req Request.
      */
-    private void processAffinityAssignmentRequest(final UUID nodeId,
-        final GridDhtAffinityAssignmentRequest req) {
+    private void processAffinityAssignmentRequest(UUID nodeId, GridDhtAffinityAssignmentRequest req) {
         if (log.isDebugEnabled())
             log.debug("Processing affinity assignment request [node=" + nodeId + ", req=" + req + ']');
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 751e262f0ca7a..0161e4f1acfe3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -224,7 +224,7 @@ public class GridCacheContext<K, V> implements Externalizable {
     private CacheWeakQueryIteratorsHolder<Map.Entry<K, V>> itHolder;
 
     /** Affinity node. */
-    private boolean affNode;
+    private volatile boolean affNode;
 
     /** Conflict resolver. */
     private CacheVersionConflictResolver conflictRslvr;
@@ -236,7 +236,7 @@ public class GridCacheContext<K, V> implements Externalizable {
     private CountDownLatch startLatch = new CountDownLatch(1);
 
     /** Topology version when cache was started on local node. */
-    private AffinityTopologyVersion locStartTopVer;
+    private volatile AffinityTopologyVersion locStartTopVer;
 
     /** Dynamic cache deployment ID. */
     private IgniteUuid dynamicDeploymentId;
@@ -269,7 +269,10 @@ public class GridCacheContext<K, V> implements Externalizable {
     private boolean readFromBackup = CacheConfiguration.DFLT_READ_FROM_BACKUP;
 
     /** Local node's MAC address. */
-    private String locMacs;
+    private volatile String locMacs;
+
+    /** Recovery mode flag. */
+    private volatile boolean recoveryMode;
 
     /**
      * Empty constructor required for {@link Externalizable}.
@@ -309,6 +312,7 @@ public GridCacheContext(
         AffinityTopologyVersion locStartTopVer,
         boolean affNode,
         boolean updatesAllowed,
+        boolean recoveryMode,
 
         /*
          * Managers in starting order!
@@ -393,11 +397,45 @@ public GridCacheContext(
 
         readFromBackup = cacheCfg.isReadFromBackup();
 
+        this.recoveryMode = recoveryMode;
+
+        assert kernalContext().recoveryMode() == recoveryMode;
+
+        if (!recoveryMode) {
+            locMacs = localNode().attribute(ATTR_MACS);
+
+            assert locMacs != null;
+        }
+    }
+
+    /**
+     * Called when cache was restored during recovery and node has joined to topology.
+     *
+     * @param topVer Cache topology join version.
+     * @param statisticsEnabled Flag indicates is statistics enabled or not for that cache.
+     *                          Value may be changed after node joined to topology.
+     */
+    public void finishRecovery(AffinityTopologyVersion topVer, boolean statisticsEnabled) {
+        assert recoveryMode : this;
+
+        recoveryMode = false;
+
+        locStartTopVer = topVer;
+
         locMacs = localNode().attribute(ATTR_MACS);
 
+        this.statisticsEnabled = statisticsEnabled;
+
         assert locMacs != null;
     }
 
+    /**
+     * @return {@code True} if cache is in recovery mode.
+     */
+    public boolean isRecoveryMode() {
+        return recoveryMode;
+    }
+
     /**
      * @return Cache group ID.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java
index 3c5cf1e944ad1..ef350160eb11a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEventManager.java
@@ -372,6 +372,10 @@ public void addEvent(
     public boolean isRecordable(int type) {
         GridCacheContext cctx0 = cctx;
 
+        // Event recording is impossible in recovery mode.
+        if (cctx0 != null && cctx0.kernalContext().recoveryMode())
+            return false;
+
         return cctx0 != null && cctx0.userCache() && cctx0.gridEvents().isRecordable(type)
             && !cctx0.config().isEventsDisabled();
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index ee814ed358926..4755f33cd2ce6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -17,7 +17,6 @@
 
 package org.apache.ignite.internal.processors.cache;
 
-import javax.management.MBeanServer;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -82,12 +81,12 @@
 import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
-import org.apache.ignite.internal.processors.cache.distributed.dht.topology.PartitionsEvictManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
+import org.apache.ignite.internal.processors.cache.distributed.dht.topology.PartitionsEvictManager;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearAtomicCache;
 import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTransactionalCache;
 import org.apache.ignite.internal.processors.cache.dr.GridCacheDrManager;
@@ -95,12 +94,15 @@
 import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
 import org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
+import org.apache.ignite.internal.processors.cache.persistence.DatabaseLifecycleListener;
 import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotDiscoveryMessage;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
@@ -266,6 +268,9 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     /** Protector of initialization of specific value. */
     private final InitializationProtector initializationProtector = new InitializationProtector();
 
+    /** Cache recovery lifecycle state and actions. */
+    private final CacheRecoveryLifecycle recovery = new CacheRecoveryLifecycle();
+
     /**
      * @param ctx Kernal context.
      */
@@ -683,9 +688,35 @@ private void cleanup(CacheConfiguration cfg, @Nullable Object rsrc, boolean near
         }
     }
 
+    /**
+     * @throws IgniteCheckedException If failed.
+     */
+    private void restoreCacheConfigurations() throws IgniteCheckedException {
+        if (ctx.isDaemon())
+            return;
+
+        Map<String, CacheInfo> caches = new HashMap<>();
+
+        Map<String, CacheInfo> templates = new HashMap<>();
+
+        addCacheOnJoinFromConfig(caches, templates);
+
+        CacheJoinNodeDiscoveryData discoData = new CacheJoinNodeDiscoveryData(
+                IgniteUuid.randomUuid(),
+                caches,
+                templates,
+                startAllCachesOnClientStart()
+        );
+
+        cachesInfo.onStart(discoData);
+    }
+
     /** {@inheritDoc} */
     @SuppressWarnings({"unchecked"})
     @Override public void start() throws IgniteCheckedException {
+        ctx.internalSubscriptionProcessor().registerMetastorageListener(recovery);
+        ctx.internalSubscriptionProcessor().registerDatabaseListener(recovery);
+
         cachesInfo = new ClusterCachesInfo(ctx);
 
         DeploymentMode depMode = ctx.config().getDeploymentMode();
@@ -709,25 +740,11 @@ private void cleanup(CacheConfiguration cfg, @Nullable Object rsrc, boolean near
         for (GridCacheSharedManager mgr : sharedCtx.managers())
             mgr.start(sharedCtx);
 
-        if (!ctx.isDaemon()) {
-            Map<String, CacheInfo> caches = new HashMap<>();
-
-            Map<String, CacheInfo> templates = new HashMap<>();
-
-            addCacheOnJoinFromConfig(caches, templates);
-
-            CacheJoinNodeDiscoveryData discoData = new CacheJoinNodeDiscoveryData(
-                IgniteUuid.randomUuid(),
-                caches,
-                templates,
-                startAllCachesOnClientStart()
-            );
-
-            cachesInfo.onStart(discoData);
+        if (!ctx.isDaemon() && (!CU.isPersistenceEnabled(ctx.config())) || ctx.config().isClientMode())
+            restoreCacheConfigurations();
 
-            if (log.isDebugEnabled())
-                log.debug("Started cache processor.");
-        }
+        if (log.isDebugEnabled())
+            log.debug("Started cache processor.");
 
         ctx.state().cacheProcessorStarted();
         ctx.authentication().cacheProcessorStarted();
@@ -1348,7 +1365,7 @@ private void onKernalStop(GridCacheAdapter<?, ?> cache, boolean cancel) {
 
         cache.onKernalStop();
 
-        if (ctx.events().isRecordable(EventType.EVT_CACHE_STOPPED))
+        if (!ctx.isRecoveryMode() && ctx.events().isRecordable(EventType.EVT_CACHE_STOPPED))
             ctx.events().addEvent(EventType.EVT_CACHE_STOPPED);
     }
 
@@ -1366,7 +1383,8 @@ private void onKernalStop(GridCacheAdapter<?, ?> cache, boolean cancel) {
      * @return Cache context.
      * @throws IgniteCheckedException If failed to create cache.
      */
-    private GridCacheContext createCache(CacheConfiguration<?, ?> cfg,
+    private GridCacheContext<?, ?> createCacheContext(
+        CacheConfiguration<?, ?> cfg,
         CacheGroupContext grp,
         @Nullable CachePluginManager pluginMgr,
         DynamicCacheDescriptor desc,
@@ -1374,8 +1392,9 @@ private GridCacheContext createCache(CacheConfiguration<?, ?> cfg,
         CacheObjectContext cacheObjCtx,
         boolean affNode,
         boolean updatesAllowed,
-        boolean disabledAfterStart)
-        throws IgniteCheckedException {
+        boolean disabledAfterStart,
+        boolean recoveryMode
+    ) throws IgniteCheckedException {
         assert cfg != null;
 
         if (cfg.getCacheStoreFactory() instanceof GridCacheLoaderWriterStoreFactory) {
@@ -1446,6 +1465,7 @@ private GridCacheContext createCache(CacheConfiguration<?, ?> cfg,
             locStartTopVer,
             affNode,
             updatesAllowed,
+            recoveryMode,
             /*
              * Managers in starting order!
              * ===========================
@@ -1578,6 +1598,7 @@ private GridCacheContext createCache(CacheConfiguration<?, ?> cfg,
                 locStartTopVer,
                 affNode,
                 true,
+                recoveryMode,
                 /*
                  * Managers in starting order!
                  * ===========================
@@ -2005,12 +2026,16 @@ private void prepareStartCaches(
                 cacheStartFailHandler.handle(
                     startCacheInfo,
                     cacheInfo -> {
-                        ctx.query().onCacheStart(
-                            cacheContexts.get(cacheInfo),
-                            cacheInfo.getCacheDescriptor().schema() != null
-                                ? cacheInfo.getCacheDescriptor().schema()
-                                : new QuerySchema()
-                        );
+                        GridCacheContext<?, ?> cctx = cacheContexts.get(cacheInfo);
+
+                        if (!cctx.isRecoveryMode()) {
+                            ctx.query().onCacheStart(
+                                cctx,
+                                cacheInfo.getCacheDescriptor().schema() != null
+                                    ? cacheInfo.getCacheDescriptor().schema()
+                                    : new QuerySchema()
+                            );
+                        }
                     }
                 );
             }
@@ -2023,7 +2048,12 @@ private void prepareStartCaches(
                     cacheStartFailHandler.handle(
                         cacheCtxEntry.getKey(),
                         cacheInfo -> {
-                            onCacheStarted(cacheCtxEntry.getValue());
+                            GridCacheContext<?, ?> cacheContext = cacheCtxEntry.getValue();
+
+                            if (cacheContext.isRecoveryMode())
+                                finishRecovery(cacheInfo.getExchangeTopVer(), cacheContext);
+                            else
+                                onCacheStarted(cacheCtxEntry.getValue());
                         }
                     )
             );
@@ -2039,7 +2069,7 @@ private void prepareStartCaches(
      * change state of proxies to restarting
      * @throws IgniteCheckedException If failed.
      */
-    void prepareCacheStart(
+    public void prepareCacheStart(
         CacheConfiguration startCfg,
         DynamicCacheDescriptor desc,
         @Nullable NearCacheConfiguration reqNearCfg,
@@ -2050,7 +2080,10 @@ void prepareCacheStart(
 
         ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema());
 
-        onCacheStarted(cacheCtx);
+        if (cacheCtx.isRecoveryMode())
+            finishRecovery(exchTopVer, cacheCtx);
+        else
+            onCacheStarted(cacheCtx);
     }
 
     /**
@@ -2072,6 +2105,24 @@ private GridCacheContext prepareCacheContext(
         AffinityTopologyVersion exchTopVer,
         boolean disabledAfterStart
     ) throws IgniteCheckedException {
+        if (caches.containsKey(startCfg.getName())) {
+            GridCacheAdapter<?, ?> existingCache = caches.get(startCfg.getName());
+
+            GridCacheContext<?, ?> cctx = existingCache.context();
+
+            assert cctx.isRecoveryMode();
+
+            QuerySchema localSchema = recovery.querySchemas.get(desc.cacheId());
+
+            QuerySchemaPatch localSchemaPatch = localSchema.makePatch(desc.schema().entities());
+
+            // Cache schema is changed after restart, workaround is stop existing cache and start new.
+            if (!localSchemaPatch.isEmpty() || localSchemaPatch.hasConflicts())
+                stopCacheSafely(cctx);
+            else
+                return existingCache.context();
+        }
+
         assert !caches.containsKey(startCfg.getName()) : startCfg.getName();
 
         CacheConfiguration ccfg = new CacheConfiguration(startCfg);
@@ -2080,9 +2131,9 @@ private GridCacheContext prepareCacheContext(
 
         boolean affNode = checkForAffinityNode(desc, reqNearCfg, ccfg);
 
-        CacheGroupContext grp = prepareCacheGroup(desc, exchTopVer, cacheObjCtx, affNode, startCfg.getGroupName());
+        CacheGroupContext grp = getOrCreateCacheGroupContext(desc, exchTopVer, cacheObjCtx, affNode, startCfg.getGroupName(), false);
 
-        GridCacheContext cacheCtx = createCache(ccfg,
+        GridCacheContext cacheCtx = createCacheContext(ccfg,
             grp,
             null,
             desc,
@@ -2090,7 +2141,8 @@ private GridCacheContext prepareCacheContext(
             cacheObjCtx,
             affNode,
             true,
-            disabledAfterStart
+            disabledAfterStart,
+            false
         );
 
         initCacheContext(cacheCtx, ccfg, desc.deploymentId());
@@ -2098,6 +2150,74 @@ private GridCacheContext prepareCacheContext(
         return cacheCtx;
     }
 
+    /**
+     * Stops cache under checkpoint lock.
+     * @param cctx Cache context.
+     */
+    private void stopCacheSafely(GridCacheContext<?, ?> cctx) {
+        sharedCtx.database().checkpointReadLock();
+
+        try {
+            prepareCacheStop(cctx.name(), false);
+
+            if (!cctx.group().hasCaches())
+                stopCacheGroup(cctx.group().groupId());
+        }
+        finally {
+            sharedCtx.database().checkpointReadUnlock();
+        }
+
+    }
+
+    /**
+     * Finishes recovery for given cache context.
+     *
+     * @param cacheStartVer Cache join to topology version.
+     * @param cacheContext Cache context.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void finishRecovery(AffinityTopologyVersion cacheStartVer, GridCacheContext<?, ?> cacheContext) throws IgniteCheckedException {
+        CacheGroupContext groupContext = cacheContext.group();
+
+        // Take cluster-wide cache descriptor and try to update local cache and cache group parameters.
+        DynamicCacheDescriptor updatedDescriptor = cacheDescriptor(cacheContext.cacheId());
+
+        groupContext.finishRecovery(
+            cacheStartVer,
+            updatedDescriptor.receivedFrom(),
+            isLocalAffinity(updatedDescriptor.cacheConfiguration())
+        );
+
+        cacheContext.finishRecovery(cacheStartVer, updatedDescriptor.cacheConfiguration().isStatisticsEnabled());
+
+        onKernalStart(cacheContext.cache());
+
+        if (log.isInfoEnabled())
+            log.info("Finished recovery for cache [cache=" + cacheContext.name()
+                + ", grp=" + groupContext.cacheOrGroupName() + ", startVer=" + cacheStartVer + "]");
+    }
+
+    /**
+     * Stops all caches and groups, that was recovered, but not activated on node join.
+     * Such caches can remain only if it was filtered by node filter on current node.
+     * It's impossible to check whether current node is affinity node for given cache before join to topology.
+     */
+    public void shutdownNotFinishedRecoveryCaches() {
+        for (GridCacheAdapter cacheAdapter : caches.values()) {
+            GridCacheContext cacheContext = cacheAdapter.context();
+
+            if (cacheContext.isLocal())
+                continue;
+
+            if (cacheContext.isRecoveryMode()) {
+                assert !isLocalAffinity(cacheContext.config())
+                    : "Cache " + cacheAdapter.context() + " is still in recovery mode after start, but not activated.";
+
+                stopCacheSafely(cacheContext);
+            }
+        }
+    }
+
     /**
      * Check for affinity node and customize near configuration if needed.
      *
@@ -2151,12 +2271,13 @@ public void preparePageStore(DynamicCacheDescriptor desc, boolean affNode) throw
      * @return Prepared cache group context.
      * @throws IgniteCheckedException if failed.
      */
-    private CacheGroupContext prepareCacheGroup(
+    private CacheGroupContext getOrCreateCacheGroupContext(
         DynamicCacheDescriptor desc,
         AffinityTopologyVersion exchTopVer,
         CacheObjectContext cacheObjCtx,
         boolean affNode,
-        String grpName
+        String grpName,
+        boolean recoveryMode
     ) throws IgniteCheckedException {
         if (grpName != null) {
             return initializationProtector.protect(
@@ -2167,7 +2288,8 @@ private CacheGroupContext prepareCacheGroup(
                     desc.cacheType(),
                     affNode,
                     cacheObjCtx,
-                    exchTopVer
+                    exchTopVer,
+                    recoveryMode
                 )
             );
         }
@@ -2176,7 +2298,8 @@ private CacheGroupContext prepareCacheGroup(
             desc.cacheType(),
             affNode,
             cacheObjCtx,
-            exchTopVer
+            exchTopVer,
+            recoveryMode
         );
     }
 
@@ -2267,6 +2390,66 @@ private void onCacheStarted(GridCacheContext cacheCtx) throws IgniteCheckedExcep
         onKernalStart(cache);
     }
 
+    /**
+     * @param desc Cache descriptor.
+     * @throws IgniteCheckedException If failed.
+     */
+    private GridCacheContext<?, ?> startCacheInRecoveryMode(
+        DynamicCacheDescriptor desc
+    ) throws IgniteCheckedException {
+        CacheConfiguration cfg = desc.cacheConfiguration();
+
+        CacheObjectContext cacheObjCtx = ctx.cacheObjects().contextForCache(cfg);
+
+        preparePageStore(desc, true);
+
+        CacheGroupContext grp = getOrCreateCacheGroupContext(
+            desc,
+            AffinityTopologyVersion.NONE,
+            cacheObjCtx,
+            true,
+            cfg.getGroupName(),
+            true
+        );
+
+        GridCacheContext cacheCtx = createCacheContext(cfg,
+            grp,
+            null,
+            desc,
+            AffinityTopologyVersion.NONE,
+            cacheObjCtx,
+            true,
+            true,
+            false,
+            true
+        );
+
+        initCacheContext(cacheCtx, cfg, desc.deploymentId());
+
+        cacheCtx.onStarted();
+
+        String dataRegion = cfg.getDataRegionName();
+
+        if (dataRegion == null && ctx.config().getDataStorageConfiguration() != null)
+            dataRegion = ctx.config().getDataStorageConfiguration().getDefaultDataRegionConfiguration().getName();
+
+        grp.onCacheStarted(cacheCtx);
+
+        ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema());
+
+        if (log.isInfoEnabled()) {
+            log.info("Started cache in recovery mode [name=" + cfg.getName() +
+                    ", id=" + cacheCtx.cacheId() +
+                    (cfg.getGroupName() != null ? ", group=" + cfg.getGroupName() : "") +
+                    ", dataRegionName=" + dataRegion +
+                    ", mode=" + cfg.getCacheMode() +
+                    ", atomicity=" + cfg.getAtomicityMode() +
+                    ", backups=" + cfg.getBackups() + ']');
+        }
+
+        return cacheCtx;
+    }
+
     /**
      * @param grpName Group name.
      * @return Found group or null.
@@ -2317,8 +2500,9 @@ private CacheGroupContext startCacheGroup(
         CacheType cacheType,
         boolean affNode,
         CacheObjectContext cacheObjCtx,
-        AffinityTopologyVersion exchTopVer)
-        throws IgniteCheckedException {
+        AffinityTopologyVersion exchTopVer,
+        boolean recoveryMode
+    ) throws IgniteCheckedException {
         CacheConfiguration cfg = new CacheConfiguration(desc.config());
 
         String memPlcName = cfg.getDataRegionName();
@@ -2327,7 +2511,7 @@ private CacheGroupContext startCacheGroup(
         FreeList freeList = sharedCtx.database().freeList(memPlcName);
         ReuseList reuseList = sharedCtx.database().reuseList(memPlcName);
 
-        boolean persistenceEnabled = sharedCtx.localNode().isClient() ? desc.persistenceEnabled() :
+        boolean persistenceEnabled = recoveryMode || sharedCtx.localNode().isClient() ? desc.persistenceEnabled() :
             dataRegion != null && dataRegion.config().isPersistenceEnabled();
 
         CacheGroupContext grp = new CacheGroupContext(sharedCtx,
@@ -2342,7 +2526,8 @@ private CacheGroupContext startCacheGroup(
             reuseList,
             exchTopVer,
             persistenceEnabled,
-            desc.walEnabled()
+            desc.walEnabled(),
+            recoveryMode
         );
 
         for (Object obj : grp.configuredUserObjects())
@@ -2435,7 +2620,7 @@ private void stopGateway(DynamicCacheChangeRequest req) {
      * @param destroy Cache data destroy flag. Setting to <code>true</code> will remove all cache data.
      * @return Stopped cache context.
      */
-    private GridCacheContext<?, ?> prepareCacheStop(String cacheName, boolean destroy) {
+    public GridCacheContext<?, ?> prepareCacheStop(String cacheName, boolean destroy) {
         assert sharedCtx.database().checkpointLockIsHeldByThread();
 
         GridCacheAdapter<?, ?> cache = caches.remove(cacheName);
@@ -2543,23 +2728,21 @@ private void closeCache(GridCacheContext cctx, boolean destroy) {
             completeProxyInitialize(cctx.name());
         }
         else {
-            jCacheProxies.remove(cctx.name());
-
             cctx.gate().onStopped();
 
-            completeProxyInitialize(cctx.name());
-
-            sharedCtx.database().checkpointReadLock();
+            // Do not close client cache while requests processing is in progress.
+            sharedCtx.io().writeLock();
 
             try {
-                prepareCacheStop(cctx.name(), destroy);
+                completeProxyInitialize(cctx.name());
+
+                jCacheProxies.remove(cctx.name());
+
+                stopCacheSafely(cctx);
             }
             finally {
-                sharedCtx.database().checkpointReadUnlock();
+                sharedCtx.io().writeUnlock();
             }
-
-            if (!cctx.group().hasCaches())
-                stopCacheGroup(cctx.group().groupId());
         }
     }
 
@@ -4332,6 +4515,26 @@ public Map<String, DynamicCacheDescriptor> cacheDescriptors() {
         return cachesInfo.registeredCaches();
     }
 
+    /**
+     * @return Collection of persistent cache descriptors.
+     */
+    public Collection<DynamicCacheDescriptor> persistentCaches() {
+        return cachesInfo.registeredCaches().values()
+                .stream()
+                .filter(desc -> isPersistentCache(desc.cacheConfiguration(), ctx.config().getDataStorageConfiguration()))
+                .collect(Collectors.toList());
+    }
+
+    /**
+     * @return Collection of persistent cache group descriptors.
+     */
+    public Collection<CacheGroupDescriptor> persistentGroups() {
+        return cachesInfo.registeredCacheGroups().values()
+                .stream()
+                .filter(CacheGroupDescriptor::persistenceEnabled)
+                .collect(Collectors.toList());
+    }
+
     /**
      * @return Cache group descriptors.
      */
@@ -4968,6 +5171,34 @@ public <T> T clone(final T obj) throws IgniteCheckedException {
         });
     }
 
+    /**
+     * Recovery lifecycle for caches.
+     */
+    private class CacheRecoveryLifecycle implements MetastorageLifecycleListener, DatabaseLifecycleListener {
+        /** Set of QuerySchema's saved on recovery. It's needed if cache query schema has changed after node joined to topology.*/
+        private final Map<Integer, QuerySchema> querySchemas = new ConcurrentHashMap<>();
+
+        /** {@inheritDoc} */
+        @Override public void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException {
+            restoreCacheConfigurations();
+        }
+
+        /** {@inheritDoc} */
+        @Override public void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {
+            for (DynamicCacheDescriptor cacheDescriptor : persistentCaches())
+                preparePageStore(cacheDescriptor, true);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void afterBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {
+            for (DynamicCacheDescriptor cacheDescriptor : persistentCaches()) {
+                startCacheInRecoveryMode(cacheDescriptor);
+
+                querySchemas.put(cacheDescriptor.cacheId(), cacheDescriptor.schema().copy());
+            }
+        }
+    }
+
     /**
      * Handle of fail during cache start.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
index 7adabc3694e9f..a85215b4f24a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedTtlCleanupManager.java
@@ -133,12 +133,16 @@ private class CleanupWorker extends GridWorker {
                 while (!isCancelled()) {
                     boolean expiredRemains = false;
 
-                    for (GridCacheTtlManager mgr : mgrs) {
-                        if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT))
-                            expiredRemains = true;
+                    // TTL cleanup is allowed only when node joined to topology.
+                    if (!cctx.kernalContext().recoveryMode()) {
+                        for (GridCacheTtlManager mgr : mgrs) {
 
-                        if (isCancelled())
-                            return;
+                            if (mgr.expire(CLEANUP_WORKER_ENTRIES_PROCESS_LIMIT))
+                                expiredRemains = true;
+
+                            if (isCancelled())
+                                return;
+                        }
                     }
 
                     if (!expiredRemains)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 6d63b5b460b24..9cce8ee5b01d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -80,8 +80,6 @@
 import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
 import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.GridCacheUtils;
-import org.apache.ignite.internal.processors.cache.LocalJoinCachesContext;
 import org.apache.ignite.internal.processors.cache.StateChangeRequest;
 import org.apache.ignite.internal.processors.cache.WalStateAbstractMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFutureAdapter;
@@ -101,7 +99,6 @@
 import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.lang.IgniteInClosureX;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.CI1;
@@ -848,7 +845,12 @@ else if (msg instanceof WalStateAbstractMessage)
      * @throws IgniteCheckedException If failed.
      */
     private IgniteInternalFuture<?> initCachesOnLocalJoin() throws IgniteCheckedException {
-        if (!isLocalNodeInBaseline()) {
+        if (!cctx.kernalContext().clientNode() && !isLocalNodeInBaseline()) {
+            // Stop all recovered caches and groups.
+            cctx.cache().onKernalStopCaches(true);
+
+            cctx.cache().stopCaches(true);
+
             cctx.database().cleanupRestoredCaches();
 
             // Set initial node started marker.
@@ -857,12 +859,12 @@ private IgniteInternalFuture<?> initCachesOnLocalJoin() throws IgniteCheckedExce
 
         cctx.activate();
 
-        if (!cctx.kernalContext().clientNode())
-            cctx.database().onDoneRestoreBinaryMemory();
-
         IgniteInternalFuture<?> cachesRegistrationFut = cctx.cache().startCachesOnLocalJoin(initialVersion(),
             exchActions == null ? null : exchActions.localJoinContext());
 
+        if (!cctx.kernalContext().clientNode())
+            cctx.cache().shutdownNotFinishedRecoveryCaches();
+
         return cachesRegistrationFut;
     }
 
@@ -964,13 +966,13 @@ private ExchangeType onClusterStateChangeRequest(boolean crd) {
                 try {
                     cctx.activate();
 
-                    if (!cctx.kernalContext().clientNode())
-                        cctx.database().onDoneRestoreBinaryMemory();
-
                     assert registerCachesFuture == null : "No caches registration should be scheduled before new caches have started.";
 
                     registerCachesFuture = cctx.affinity().onCacheChangeRequest(this, crd, exchActions);
 
+                    if (!cctx.kernalContext().clientNode())
+                        cctx.cache().shutdownNotFinishedRecoveryCaches();
+
                     if (log.isInfoEnabled()) {
                         log.info("Successfully activated caches [nodeId=" + cctx.localNodeId() +
                             ", client=" + cctx.kernalContext().clientNode() +
@@ -1241,7 +1243,7 @@ private void distributedExchange() throws IgniteCheckedException {
            In case of persistent store is enabled we first restore partitions presented on disk.
            We need to guarantee that there are no partition state changes logged to WAL before this callback
            to make sure that we correctly restored last actual states. */
-        boolean restored = cctx.database().beforeExchange(this);
+        cctx.database().beforeExchange(this);
 
         // Pre-create missing partitions using current affinity.
         if (!exchCtx.mergeExchanges()) {
@@ -1256,8 +1258,8 @@ private void distributedExchange() throws IgniteCheckedException {
         }
 
         // After all partitions have been restored and pre-created it's safe to make first checkpoint.
-        if (restored)
-            cctx.database().onStateRestored();
+        if (localJoinExchange() || activateCluster())
+            cctx.database().onStateRestored(initialVersion());
 
         changeWalModeIfNeeded();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
index 652ce58507e78..ba5059ed4098a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
@@ -902,8 +902,6 @@ private GridDhtLocalPartition getOrCreatePartition(int p) {
 
             locParts.set(p, part);
 
-            ctx.pageStore().onPartitionCreated(grp.groupId(), p);
-
             return part;
         }
         finally {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java
new file mode 100644
index 0000000000000..1f7ba84a05d02
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DatabaseLifecycleListener.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ *
+ */
+public interface DatabaseLifecycleListener {
+    /**
+     * Callback executed when data regions become to start-up.
+     *
+     * @param mgr Database shared manager.
+     * @throws IgniteCheckedException If failed.
+     */
+    default void onInitDataRegions(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {};
+
+    /**
+     * Callback executed right before node become perform binary recovery.
+     *
+     * @param mgr Database shared manager.
+     * @throws IgniteCheckedException If failed.
+     */
+    default void beforeBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {};
+
+    /**
+     * Callback executed when binary memory has fully restored and WAL logging is resumed.
+     *
+     * @param mgr Database shared manager.
+     * @throws IgniteCheckedException If failed.
+     */
+    default void afterBinaryMemoryRestore(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {};
+
+    /**
+     *
+     * @param mgr
+     * @throws IgniteCheckedException
+     */
+    default void beforeResumeWalLogging(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {};
+
+    /**
+     * @param mgr Database shared manager.
+     */
+    default void afterInitialise(IgniteCacheDatabaseSharedManager mgr) throws IgniteCheckedException {};
+
+    /**
+     * @param mgr Database shared manager.
+     */
+    default void beforeStop(IgniteCacheDatabaseSharedManager mgr) {};
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 9058bdca0c2b0..d0166e7bb7f4d 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -71,21 +71,17 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.NearCacheConfiguration;
-import org.apache.ignite.events.DiscoveryEvent;
-import org.apache.ignite.events.EventType;
 import org.apache.ignite.failure.FailureContext;
 import org.apache.ignite.failure.FailureType;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.NodeStoppingException;
-import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.mem.DirectMemoryProvider;
 import org.apache.ignite.internal.mem.DirectMemoryRegion;
-import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
-import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
 import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.PageUtils;
@@ -97,22 +93,24 @@
 import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
 import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
 import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WalRecordCacheGroupAware;
 import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
 import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord;
 import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
 import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
 import org.apache.ignite.internal.processors.cache.ExchangeActions;
 import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.StoredCacheData;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
 import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntry;
 import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointEntryType;
 import org.apache.ignite.internal.processors.cache.persistence.checkpoint.CheckpointHistory;
@@ -125,17 +123,16 @@
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId;
 import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionAllocationMap;
+import org.apache.ignite.internal.processors.cache.persistence.partstate.PartitionRecoverState;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotOperation;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc;
 import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
 import org.apache.ignite.internal.processors.port.GridPortRecord;
 import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
-import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.future.CountDownFuture;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
@@ -166,11 +163,9 @@
 import static java.nio.file.StandardOpenOption.READ;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
-import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_DATA_REG_DEFAULT_NAME;
 import static org.apache.ignite.failure.FailureType.CRITICAL_ERROR;
 import static org.apache.ignite.failure.FailureType.SYSTEM_WORKER_TERMINATION;
 import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.CHECKPOINT_RECORD;
-import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_ID;
 
 /**
  *
@@ -180,6 +175,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** */
     public static final String IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC = "IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC";
 
+    /** */
+    public static final String IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP = "IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP";
+
     /** MemoryPolicyConfiguration name reserved for meta store. */
     private static final String METASTORE_DATA_REGION_NAME = "metastoreMemPlc";
 
@@ -206,6 +204,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     private final String throttlingPolicyOverride = IgniteSystemProperties.getString(
         IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED);
 
+    /** */
+    private final boolean skipCheckpointOnNodeStop = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP, false);
+
     /** Checkpoint lock hold count. */
     private static final ThreadLocal<Integer> CHECKPOINT_LOCK_HOLD_COUNT = new ThreadLocal<Integer>() {
         @Override protected Integer initialValue() {
@@ -429,7 +430,7 @@ public IgniteInternalFuture<Void> enableCheckpoints(boolean enable) {
 
         addDataRegion(
             memCfg,
-            createDataRegionConfiguration(memCfg),
+            createMetastoreDataRegionConfig(memCfg),
             false
         );
 
@@ -442,13 +443,14 @@ public IgniteInternalFuture<Void> enableCheckpoints(boolean enable) {
      * @param storageCfg Data storage configuration.
      * @return Data region configuration.
      */
-    private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfiguration storageCfg) {
+    private DataRegionConfiguration createMetastoreDataRegionConfig(DataStorageConfiguration storageCfg) {
         DataRegionConfiguration cfg = new DataRegionConfiguration();
 
         cfg.setName(METASTORE_DATA_REGION_NAME);
         cfg.setInitialSize(storageCfg.getSystemRegionInitialSize());
         cfg.setMaxSize(storageCfg.getSystemRegionMaxSize());
         cfg.setPersistenceEnabled(true);
+
         return cfg;
     }
 
@@ -491,15 +493,7 @@ private DataRegionConfiguration createDataRegionConfiguration(DataStorageConfigu
                     .resolveFolders()
                     .getLockedFileLockHolder();
 
-            fileLockHolder = preLocked == null ?
-                        new FileLockHolder(storeMgr.workDir().getPath(), kernalCtx, log) : preLocked;
-
-            if (log.isDebugEnabled())
-                log.debug("Try to capture file lock [nodeId=" +
-                        cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
-
-            if (!fileLockHolder.isLocked())
-                fileLockHolder.tryLock(lockWaitTime);
+            acquireFileLock(preLocked);
 
             cleanupTempCheckpointDirectory();
 
@@ -530,17 +524,20 @@ public void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
 
     /** {@inheritDoc} */
     @Override public void cleanupRestoredCaches() {
-        if (dataRegionMap == null)
+        if (dataRegionMap.isEmpty())
             return;
 
         for (CacheGroupDescriptor grpDesc : cctx.cache().cacheGroupDescriptors().values()) {
             String regionName = grpDesc.config().getDataRegionName();
 
-            DataRegion region = dataRegionMap.get(regionName == null ? DFLT_DATA_REG_DEFAULT_NAME : regionName);
+            DataRegion region = regionName != null ? dataRegionMap.get(regionName) : dfltDataRegion;
 
             if (region == null)
                 continue;
 
+            if (log.isInfoEnabled())
+                log.info("Page memory " + region + " for " + grpDesc + " has invalidated.");
+
             int partitions = grpDesc.config().getAffinity().partitions();
 
             if (region.pageMemory() instanceof PageMemoryEx) {
@@ -548,6 +545,8 @@ public void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
 
                 for (int partId = 0; partId < partitions; partId++)
                     memEx.invalidate(grpDesc.groupId(), partId);
+
+                memEx.invalidate(grpDesc.groupId(), PageIdAllocator.INDEX_PARTITION);
             }
         }
 
@@ -576,6 +575,39 @@ public void cleanupTempCheckpointDirectory() throws IgniteCheckedException {
         }
     }
 
+    /**
+     * @param preLocked Pre-locked file lock holder.
+     */
+    private void acquireFileLock(FileLockHolder preLocked) throws IgniteCheckedException {
+        if (cctx.kernalContext().clientNode())
+            return;
+
+        fileLockHolder = preLocked == null ?
+            new FileLockHolder(storeMgr.workDir().getPath(), cctx.kernalContext(), log) : preLocked;
+
+        if (!fileLockHolder.isLocked()) {
+            if (log.isDebugEnabled())
+                log.debug("Try to capture file lock [nodeId=" +
+                    cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
+
+            fileLockHolder.tryLock(lockWaitTime);
+        }
+    }
+
+    /**
+     *
+     */
+    private void releaseFileLock() {
+        if (cctx.kernalContext().clientNode() || fileLockHolder == null)
+            return;
+
+        if (log.isDebugEnabled())
+            log.debug("Release file lock [nodeId=" +
+                cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
+
+        fileLockHolder.close();
+    }
+
     /**
      * Retreives checkpoint history form specified {@code dir}.
      *
@@ -659,40 +691,18 @@ private void removeCheckpointFiles(CheckpointEntry cpEntry) throws IgniteChecked
     /** */
     private void readMetastore() throws IgniteCheckedException {
         try {
-            DataStorageConfiguration memCfg = cctx.kernalContext().config().getDataStorageConfiguration();
-
-            DataRegionConfiguration plcCfg = createDataRegionConfiguration(memCfg);
-
-            File allocPath = buildAllocPath(plcCfg);
-
-            DirectMemoryProvider memProvider = allocPath == null ?
-                new UnsafeMemoryProvider(log) :
-                new MappedFileMemoryProvider(
-                    log,
-                    allocPath);
-
-            DataRegionMetricsImpl memMetrics = new DataRegionMetricsImpl(plcCfg);
-
-            PageMemoryEx storePageMem = (PageMemoryEx)createPageMemory(memProvider, memCfg, plcCfg, memMetrics, false);
-
-            DataRegion regCfg = new DataRegion(storePageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, storePageMem));
-
             CheckpointStatus status = readCheckpointStatus();
 
-            cctx.pageStore().initializeForMetastorage();
-
-            storePageMem.start();
-
             checkpointReadLock();
 
             try {
-                restoreMemory(status, true, storePageMem, Collections.emptySet());
+                dataRegion(METASTORE_DATA_REGION_NAME).pageMemory().start();
 
-                metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true);
+                performBinaryMemoryRestore(status, g -> MetaStorage.METASTORAGE_CACHE_ID == g, false);
 
-                metaStorage.init(this);
+                metaStorage = createMetastorage(true);
 
-                applyLastUpdates(status, true);
+                applyLogicalUpdates(status, g -> MetaStorage.METASTORAGE_CACHE_ID == g, false);
 
                 fillWalDisabledGroups();
 
@@ -701,7 +711,7 @@ private void readMetastore() throws IgniteCheckedException {
             finally {
                 metaStorage = null;
 
-                storePageMem.stop(true);
+                dataRegion(METASTORE_DATA_REGION_NAME).pageMemory().stop(false);
 
                 cctx.pageStore().cleanupPageStoreIfMatch(new Predicate<Integer>() {
                     @Override public boolean test(Integer grpId) {
@@ -751,16 +761,16 @@ else if (regCfg.getMaxSize() < 8 * GB)
 
         snapshotMgr = cctx.snapshot();
 
-        if (!cctx.kernalContext().clientNode()) {
-            initDataBase();
-
-            registrateMetricsMBean();
-        }
-
-        if (checkpointer == null)
+        if (!cctx.kernalContext().clientNode() && checkpointer == null)
             checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log);
 
         super.onActivate(ctx);
+
+        if (!cctx.kernalContext().clientNode()) {
+            initializeCheckpointPool();
+
+            finishRecovery();
+        }
     }
 
     /** {@inheritDoc} */
@@ -780,7 +790,7 @@ else if (regCfg.getMaxSize() < 8 * GB)
     /**
      *
      */
-    private void initDataBase() {
+    private void initializeCheckpointPool() {
         if (persistenceCfg.getCheckpointThreads() > 1)
             asyncRunner = new IgniteThreadPoolExecutor(
                 CHECKPOINT_RUNNER_THREAD_PREFIX,
@@ -792,27 +802,17 @@ private void initDataBase() {
             );
     }
 
-    /**
-     * Try to register Metrics MBean.
-     *
-     * @throws IgniteCheckedException If failed.
-     */
-    private void registrateMetricsMBean() throws IgniteCheckedException {
-        if (U.IGNITE_MBEANS_DISABLED)
-            return;
-
-        try {
-            persistenceMetricsMbeanName = U.registerMBean(
-                cctx.kernalContext().config().getMBeanServer(),
-                cctx.kernalContext().igniteInstanceName(),
-                MBEAN_GROUP,
-                MBEAN_NAME,
-                persStoreMetrics,
-                DataStorageMetricsMXBean.class);
-        }
-        catch (Throwable e) {
-            throw new IgniteCheckedException("Failed to register " + MBEAN_NAME + " MBean.", e);
-        }
+    /** {@inheritDoc} */
+    @Override protected void registerMetricsMBeans(IgniteConfiguration cfg) {
+        super.registerMetricsMBeans(cfg);
+
+        registerMetricsMBean(
+            cctx.kernalContext().config(),
+            MBEAN_GROUP,
+            MBEAN_NAME,
+            persStoreMetrics,
+            DataStorageMetricsMXBean.class
+        );
     }
 
     /**
@@ -859,8 +859,13 @@ private void unRegistrateMetricsMBean() {
         };
     }
 
-    /** {@inheritDoc} */
-    @Override public void onDoneRestoreBinaryMemory() throws IgniteCheckedException {
+    /**
+     * Restores last valid WAL pointer and resumes logging from that pointer.
+     * Re-creates metastorage if needed.
+     *
+     * @throws IgniteCheckedException If failed.
+     */
+    private void finishRecovery() throws IgniteCheckedException {
         assert !cctx.kernalContext().clientNode();
 
         long time = System.currentTimeMillis();
@@ -868,34 +873,27 @@ private void unRegistrateMetricsMBean() {
         checkpointReadLock();
 
         try {
-            cctx.pageStore().initializeForMetastorage();
-
-            CheckpointStatus status = readCheckpointStatus();
-
-            // Binary memory should be recovered at startup.
-            assert !status.needRestoreMemory() : status;
+            for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext()))
+                lsnr.beforeResumeWalLogging(this);
 
-            WALPointer statusEndPtr = CheckpointStatus.NULL_PTR.equals(status.endPtr) ? null : status.endPtr;
+            // Try to resume logging since last finished checkpoint if possible.
+            if (walTail == null) {
+                CheckpointStatus status = readCheckpointStatus();
 
-            // If binary memory recovery occurs resume from the last walTail in the other case from END checkpoint.
-            WALPointer walPtr = walTail == null ? statusEndPtr : walTail;
+                walTail = CheckpointStatus.NULL_PTR.equals(status.endPtr) ? null : status.endPtr;
+            }
 
-            cctx.wal().resumeLogging(walPtr);
+            cctx.wal().resumeLogging(walTail);
 
             walTail = null;
 
-            metaStorage = new MetaStorage(
-                cctx,
-                dataRegionMap.get(METASTORE_DATA_REGION_NAME),
-                (DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME),
-                false
-            );
-
-            // Init metastore only after WAL logging resumed. Can't do it earlier because
-            // MetaStorage first initialization also touches WAL, look at #isWalDeltaRecordNeeded.
-            metaStorage.init(this);
+            // Recreate metastorage to refresh page memory state after deactivation.
+            if (metaStorage == null)
+                metaStorage = createMetastorage(false);
 
             notifyMetastorageReadyForReadWrite();
+
+            U.log(log, "Finish recovery performed in " + (System.currentTimeMillis() - time) + " ms.");
         }
         catch (IgniteCheckedException e) {
             if (X.hasCause(e, StorageException.class, IOException.class))
@@ -905,17 +903,35 @@ private void unRegistrateMetricsMBean() {
         }
         finally {
             checkpointReadUnlock();
-
-            U.log(log, "Resume logging performed in " + (System.currentTimeMillis() - time) + " ms.");
         }
     }
 
     /**
-     * @param cacheGrps Cache groups to restore.
+     * @param readOnly Metastorage read-only mode.
+     * @return Instance of Metastorage.
+     * @throws IgniteCheckedException If failed to create metastorage.
+     */
+    private MetaStorage createMetastorage(boolean readOnly) throws IgniteCheckedException {
+        cctx.pageStore().initializeForMetastorage();
+
+        MetaStorage storage = new MetaStorage(
+            cctx,
+            dataRegion(METASTORE_DATA_REGION_NAME),
+            (DataRegionMetricsImpl) memMetricsMap.get(METASTORE_DATA_REGION_NAME),
+            readOnly
+        );
+
+        storage.init(this);
+
+        return storage;
+    }
+
+    /**
+     * @param cacheGroupsPredicate Cache groups to restore.
      * @return Last seen WAL pointer during binary memory recovery.
      * @throws IgniteCheckedException If failed.
      */
-    protected WALPointer restoreBinaryMemory(Set<Integer> cacheGrps) throws IgniteCheckedException {
+    private WALPointer restoreBinaryMemory(Predicate<Integer> cacheGroupsPredicate) throws IgniteCheckedException {
         assert !cctx.kernalContext().clientNode();
 
         long time = System.currentTimeMillis();
@@ -923,25 +939,23 @@ protected WALPointer restoreBinaryMemory(Set<Integer> cacheGrps) throws IgniteCh
         checkpointReadLock();
 
         try {
-            cctx.pageStore().initializeForMetastorage();
-
             CheckpointStatus status = readCheckpointStatus();
 
             // First, bring memory to the last consistent checkpoint state if needed.
             // This method should return a pointer to the last valid record in the WAL.
-            WALPointer tailWalPtr = restoreMemory(status,
-                false,
-                (PageMemoryEx)dataRegionMap.get(METASTORE_DATA_REGION_NAME).pageMemory(),
-                cacheGrps);
-
-            if (tailWalPtr == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) {
-                throw new StorageException("Restore wal pointer = " + tailWalPtr + ", while status.endPtr = " +
-                    status.endPtr + ". Can't restore memory - critical part of WAL archive is missing.");
+            WALPointer restored = performBinaryMemoryRestore(status, cacheGroupsPredicate, true);
+
+            if (restored == null && !status.endPtr.equals(CheckpointStatus.NULL_PTR)) {
+                throw new StorageException("The memory cannot be restored. The critical part of WAL archive is missing " +
+                    "[tailWalPtr=" + restored + ", endPtr=" + status.endPtr + ']');
             }
 
-            nodeStart(tailWalPtr);
+            nodeStart(restored);
+
+            if (log.isInfoEnabled())
+                log.info("Binary recovery performed in " + (System.currentTimeMillis() - time) + " ms.");
 
-            return tailWalPtr;
+            return restored;
         }
         catch (IgniteCheckedException e) {
             if (X.hasCause(e, StorageException.class, IOException.class))
@@ -951,9 +965,6 @@ protected WALPointer restoreBinaryMemory(Set<Integer> cacheGrps) throws IgniteCh
         }
         finally {
             checkpointReadUnlock();
-
-            if (log.isInfoEnabled())
-                log.info("Binary recovery performed in " + (System.currentTimeMillis() - time) + " ms.");
         }
     }
 
@@ -1062,22 +1073,20 @@ public List<T2<Long, WALPointer>> nodeStartedPointers() throws IgniteCheckedExce
 
         super.onKernalStop0(cancel);
 
-        unRegistrateMetricsMBean();
+        unregisterMetricsMBean(
+            cctx.gridConfig(),
+            MBEAN_GROUP,
+            MBEAN_NAME
+        );
+
+        metaStorage = null;
     }
 
     /** {@inheritDoc} */
     @Override protected void stop0(boolean cancel) {
         super.stop0(cancel);
 
-        if (!cctx.kernalContext().clientNode()) {
-            if (fileLockHolder != null) {
-                if (log.isDebugEnabled())
-                    log.debug("Release file lock [nodeId=" +
-                            cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
-
-                fileLockHolder.close();
-            }
-        }
+        releaseFileLock();
     }
 
     /** */
@@ -1369,34 +1378,30 @@ private void shutdownCheckpointer(boolean cancel) {
     }
 
     /** {@inheritDoc} */
-    @Override public boolean beforeExchange(GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException {
-        DiscoveryEvent discoEvt = fut.firstEvent();
-
-        boolean joinEvt = discoEvt.type() == EventType.EVT_NODE_JOINED;
-
-        boolean locNode = discoEvt.eventNode().isLocal();
-
-        boolean isSrvNode = !cctx.kernalContext().clientNode();
-
-        boolean clusterInTransitionStateToActive = fut.activateCluster();
-
-        boolean restored = false;
-
-        long time = System.currentTimeMillis();
+    @Override public void beforeExchange(GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException {
+        // Try to restore partition states.
+        if (fut.localJoinExchange() || fut.activateCluster()
+            || (fut.exchangeActions() != null && !F.isEmpty(fut.exchangeActions().cacheGroupsToStart()))) {
+            U.doInParallel(
+                cctx.kernalContext().getSystemExecutorService(),
+                cctx.cache().cacheGroups(),
+                cacheGroup -> {
+                    if (cacheGroup.isLocal())
+                        return;
 
-        // In case of cluster activation or local join restore, restore whole manager state.
-        if (clusterInTransitionStateToActive || (joinEvt && locNode && isSrvNode)) {
-            restoreState();
+                    cctx.database().checkpointReadLock();
 
-            restored = true;
-        }
-        // In case of starting groups, restore partition states only for these groups.
-        else if (fut.exchangeActions() != null && !F.isEmpty(fut.exchangeActions().cacheGroupsToStart())) {
-            Set<Integer> restoreGroups = fut.exchangeActions().cacheGroupsToStart().stream()
-                .map(actionData -> actionData.descriptor().groupId())
-                .collect(Collectors.toSet());
+                    try {
+                        cacheGroup.restorePartitionStates(Collections.emptyMap());
 
-            restorePartitionStates(Collections.emptyMap(), restoreGroups);
+                        if (cacheGroup.localStartVersion().equals(fut.initialVersion()))
+                            cacheGroup.topology().afterStateRestored(fut.initialVersion());
+                    }
+                    finally {
+                        cctx.database().checkpointReadUnlock();
+                    }
+                }
+            );
         }
 
         if (cctx.kernalContext().query().moduleEnabled()) {
@@ -1413,11 +1418,6 @@ else if (acts.localJoinContext() != null && !F.isEmpty(acts.localJoinContext().c
                 }
             }
         }
-
-        if (log.isInfoEnabled())
-            log.info("Logical recovery performed in " + (System.currentTimeMillis() - time) + " ms.");
-
-        return restored;
     }
 
     /**
@@ -1636,52 +1636,6 @@ private boolean safeToUpdatePageMemories() {
             CHECKPOINT_LOCK_HOLD_COUNT.set(CHECKPOINT_LOCK_HOLD_COUNT.get() - 1);
     }
 
-    /**
-     * Restores from last checkpoint and applies WAL changes since this checkpoint.
-     *
-     * @throws IgniteCheckedException If failed to restore database status from WAL.
-     */
-    private void restoreState() throws IgniteCheckedException {
-        try {
-            CheckpointStatus status = readCheckpointStatus();
-
-            checkpointReadLock();
-
-            try {
-                applyLastUpdates(status, false);
-            }
-            finally {
-                checkpointReadUnlock();
-            }
-
-            snapshotMgr.restoreState();
-        }
-        catch (StorageException e) {
-            throw new IgniteCheckedException(e);
-        }
-    }
-
-    /**
-     * Called when all partitions have been fully restored and pre-created on node start.
-     *
-     * Starts checkpointing process and initiates first checkpoint.
-     *
-     * @throws IgniteCheckedException If first checkpoint has failed.
-     */
-    @Override public void onStateRestored() throws IgniteCheckedException {
-        long time = System.currentTimeMillis();
-
-        new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
-
-        CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started");
-
-        if (chp != null)
-            chp.cpBeginFut.get();
-
-        if (log.isInfoEnabled())
-            log.info("Checkpointer initilialzation performed in " + (System.currentTimeMillis() - time) + " ms.");
-    }
-
     /** {@inheritDoc} */
     @Override public synchronized Map<Integer, Map<Integer, Long>> reserveHistoryForExchange() {
         assert reservedForExchange == null : reservedForExchange;
@@ -1973,39 +1927,146 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
         if (kctx.clientNode())
             return;
 
-        // Preform early regions startup before restoring state.
-        initAndStartRegions(kctx.config().getDataStorageConfiguration());
+        checkpointReadLock();
+
+        try {
+            // Preform early regions startup before restoring state.
+            initAndStartRegions(kctx.config().getDataStorageConfiguration());
+
+            for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx))
+                lsnr.beforeBinaryMemoryRestore(this);
+
+            log.info("Starting binary memory restore for: " + cctx.cache().cacheGroupDescriptors().keySet());
+
+            cctx.pageStore().initializeForMetastorage();
+
+            // Restore binary memory for all not WAL disabled cache groups.
+            WALPointer restored = restoreBinaryMemory(
+                    g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g)
+            );
+
+            if (restored != null)
+                U.log(log, "Binary memory state restored at node startup [restoredPtr=" + restored + ']');
+
+            for (DatabaseLifecycleListener lsnr : getDatabaseListeners(kctx))
+                lsnr.afterBinaryMemoryRestore(this);
 
-        // Only presistence caches to start.
-        for (DynamicCacheDescriptor desc : cctx.cache().cacheDescriptors().values()) {
-            if (CU.isPersistentCache(desc.cacheConfiguration(), cctx.gridConfig().getDataStorageConfiguration()))
-                storeMgr.initializeForCache(desc.groupDescriptor(), new StoredCacheData(desc.cacheConfiguration()));
+            cctx.wal().resumeLogging(restored);
+
+            // We should log this record to ensure that node start marker pointer will be found in compacted segment.
+            cctx.wal().log(new MemoryRecoveryRecord(System.currentTimeMillis()));
+
+            assert metaStorage == null;
+
+            metaStorage = createMetastorage(false);
+
+            CheckpointStatus status = readCheckpointStatus();
+
+            RestoreLogicalState logicalState = applyLogicalUpdates(
+                    status,
+                    g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g),
+                    true
+            );
+
+            // Restore state for all groups.
+            restorePartitionStates(cctx.cache().cacheGroups(), logicalState.partitionRecoveryStates);
+
+            walTail = tailPointer(logicalState.lastRead);
+
+            cctx.wal().onDeActivate(kctx);
+        }
+        catch (IgniteCheckedException e) {
+            releaseFileLock();
+
+            throw e;
+        }
+        finally {
+            checkpointReadUnlock();
         }
+    }
 
-        final WALPointer restoredPtr = restoreBinaryMemory(cctx.cache().cacheGroupDescriptors().keySet());
+    /**
+     * Calculates tail pointer for WAL at the end of logical recovery.
+     *
+     * @param from Start replay WAL from.
+     * @return Tail pointer.
+     * @throws IgniteCheckedException If failed.
+     */
+    private WALPointer tailPointer(WALPointer from) throws IgniteCheckedException {
+        WALPointer lastRead = from;
 
-        walTail = restoredPtr;
+        try (WALIterator it = cctx.wal().replay(from)) {
+            while (it.hasNextX()) {
+                IgniteBiTuple<WALPointer, WALRecord> rec = it.nextX();
 
-        if (restoredPtr != null)
-            U.log(log, "Binary memory state restored at node startup [restoredPtr=" + restoredPtr + ']');
+                if (rec == null)
+                    break;
+
+                lastRead = rec.get1();
+            }
+        }
+
+        return lastRead != null ? lastRead.next() : null;
+    }
+
+    /**
+     * @param forGroups Cache groups.
+     * @param partitionStates Partition states.
+     * @throws IgniteCheckedException If failed.
+     */
+    private void restorePartitionStates(
+        Collection<CacheGroupContext> forGroups,
+        Map<GroupPartitionId, PartitionRecoverState> partitionStates
+    ) throws IgniteCheckedException {
+        long startRestorePart = U.currentTimeMillis();
+
+        if (log.isInfoEnabled())
+            log.info("Restoring partition state for local groups.");
+
+        long totalProcessed = 0;
+
+        for (CacheGroupContext grp : forGroups)
+            totalProcessed += grp.restorePartitionStates(partitionStates);
+
+        if (log.isInfoEnabled())
+            log.info("Finished restoring partition state for local groups [" +
+                "groupsProcessed" + forGroups.size() +
+                "partitionsProcessed=" + totalProcessed +
+                ", time=" + (U.currentTimeMillis() - startRestorePart) + "ms]");
+    }
+
+    /**
+     * Called when all partitions have been fully restored and pre-created on node start.
+     *
+     * Starts checkpointing process and initiates first checkpoint.
+     *
+     * @throws IgniteCheckedException If first checkpoint has failed.
+     */
+    @Override public void onStateRestored(AffinityTopologyVersion topVer) throws IgniteCheckedException {
+        long time = System.currentTimeMillis();
+
+        new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
+
+        CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started");
+
+        if (chp != null)
+            chp.cpBeginFut.get();
+
+        if (log.isInfoEnabled())
+            log.info("Checkpointer initilialzation performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
      * @param status Checkpoint status.
-     * @param metastoreOnly If {@code True} restores Metastorage only.
-     * @param storePageMem Metastore page memory.
-     * @param cacheGrps Cache groups to restore.
+     * @param cacheGroupsPredicate Cache groups to restore.
      * @throws IgniteCheckedException If failed.
      * @throws StorageException In case I/O error occurred during operations with storage.
      */
-    @Nullable private WALPointer restoreMemory(
+    private WALPointer performBinaryMemoryRestore(
         CheckpointStatus status,
-        boolean metastoreOnly,
-        PageMemoryEx storePageMem,
-        Set<Integer> cacheGrps
+        Predicate<Integer> cacheGroupsPredicate,
+        boolean finalizeState
     ) throws IgniteCheckedException {
-        assert !metastoreOnly || storePageMem != null;
-
         if (log.isInfoEnabled())
             log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked="
                 + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
@@ -2013,8 +2074,9 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
         boolean apply = status.needRestoreMemory();
 
         if (apply) {
-            U.quietAndWarn(log, "Ignite node stopped in the middle of checkpoint. Will restore memory state and " +
-                "finish checkpoint on node start.");
+            if (finalizeState)
+                U.quietAndWarn(log, "Ignite node stopped in the middle of checkpoint. Will restore memory state and " +
+                    "finish checkpoint on node start.");
 
             cctx.pageStore().beginRecover();
         }
@@ -2025,16 +2087,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
 
         long lastArchivedSegment = cctx.wal().lastArchivedSegment();
 
-        RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, lastArchivedSegment, log);
-
-        // Always perform recovery at least meta storage cache.
-        Set<Integer> restoreGrps = new HashSet<>(Collections.singletonList(METASTORAGE_CACHE_ID));
-
-        if (!metastoreOnly && !F.isEmpty(cacheGrps)) {
-            restoreGrps.addAll(cacheGrps.stream()
-                .filter(g -> !initiallyGlobalWalDisabledGrps.contains(g) && !initiallyLocalWalDisabledGrps.contains(g))
-                .collect(Collectors.toSet()));
-        }
+        RestoreBinaryState restoreBinaryState = new RestoreBinaryState(status, lastArchivedSegment, cacheGroupsPredicate);
 
         int applied = 0;
 
@@ -2054,29 +2107,30 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
                             // several repetitive restarts and the same pages may have changed several times.
                             int grpId = pageRec.fullPageId().groupId();
 
-                            if (restoreGrps.contains(grpId)) {
-                                long pageId = pageRec.fullPageId().pageId();
+                            long pageId = pageRec.fullPageId().pageId();
 
-                                PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
+                            PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId);
 
-                                long page = pageMem.acquirePage(grpId, pageId, true);
+                            if (pageMem == null)
+                                break;
 
-                                try {
-                                    long pageAddr = pageMem.writeLock(grpId, pageId, page);
+                            long page = pageMem.acquirePage(grpId, pageId, true);
 
-                                    try {
-                                        PageUtils.putBytes(pageAddr, 0, pageRec.pageData());
-                                    }
-                                    finally {
-                                        pageMem.writeUnlock(grpId, pageId, page, null, true, true);
-                                    }
+                            try {
+                                long pageAddr = pageMem.writeLock(grpId, pageId, page);
+
+                                try {
+                                    PageUtils.putBytes(pageAddr, 0, pageRec.pageData());
                                 }
                                 finally {
-                                    pageMem.releasePage(grpId, pageId, page);
+                                    pageMem.writeUnlock(grpId, pageId, page, null, true, true);
                                 }
-
-                                applied++;
                             }
+                            finally {
+                                pageMem.releasePage(grpId, pageId, page);
+                            }
+
+                            applied++;
                         }
 
                         break;
@@ -2087,9 +2141,6 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
                         {
                             int grpId = metaStateRecord.groupId();
 
-                            if (!restoreGrps.contains(grpId))
-                                continue;
-
                             int partId = metaStateRecord.partitionId();
 
                             GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(metaStateRecord.state());
@@ -2108,10 +2159,10 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
                         {
                             int grpId = destroyRecord.groupId();
 
-                            if (!restoreGrps.contains(grpId))
-                                continue;
+                            PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId);
 
-                            PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
+                            if (pageMem == null)
+                                break;
 
                             pageMem.invalidate(grpId, destroyRecord.partitionId());
 
@@ -2126,37 +2177,38 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
 
                             int grpId = r.groupId();
 
-                            if (restoreGrps.contains(grpId)) {
-                                long pageId = r.pageId();
+                            long pageId = r.pageId();
 
-                                PageMemoryEx pageMem = grpId == METASTORAGE_CACHE_ID ? storePageMem : getPageMemoryForCacheGroup(grpId);
+                            PageMemoryEx pageMem = getPageMemoryForCacheGroup(grpId);
 
-                                // Here we do not require tag check because we may be applying memory changes after
-                                // several repetitive restarts and the same pages may have changed several times.
-                                long page = pageMem.acquirePage(grpId, pageId, true);
+                            if (pageMem == null)
+                                break;
 
-                                try {
-                                    long pageAddr = pageMem.writeLock(grpId, pageId, page);
+                            // Here we do not require tag check because we may be applying memory changes after
+                            // several repetitive restarts and the same pages may have changed several times.
+                            long page = pageMem.acquirePage(grpId, pageId, true);
 
-                                    try {
-                                        r.applyDelta(pageMem, pageAddr);
-                                    }
-                                    finally {
-                                        pageMem.writeUnlock(grpId, pageId, page, null, true, true);
-                                    }
+                            try {
+                                long pageAddr = pageMem.writeLock(grpId, pageId, page);
+
+                                try {
+                                    r.applyDelta(pageMem, pageAddr);
                                 }
                                 finally {
-                                    pageMem.releasePage(grpId, pageId, page);
+                                    pageMem.writeUnlock(grpId, pageId, page, null, true, true);
                                 }
-
-                                applied++;
                             }
+                            finally {
+                                pageMem.releasePage(grpId, pageId, page);
+                            }
+
+                            applied++;
                         }
                 }
             }
         }
 
-        if (metastoreOnly)
+        if (!finalizeState)
             return null;
 
         WALPointer lastReadPtr = restoreBinaryState.lastReadRecordPointer();
@@ -2168,7 +2220,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
                     "[cpStatus=" + status + ", lastRead=" + lastReadPtr + "]");
 
             log.info("Finished applying memory changes [changesApplied=" + applied +
-                ", time=" + (U.currentTimeMillis() - start) + "ms]");
+                ", time=" + (U.currentTimeMillis() - start) + " ms]");
 
             if (applied > 0)
                 finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr);
@@ -2176,7 +2228,7 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
 
         cpHistory.initialize(retreiveHistory());
 
-        return lastReadPtr == null ? null : lastReadPtr.next();
+        return lastReadPtr != null ? lastReadPtr.next() : null;
     }
 
     /**
@@ -2187,13 +2239,16 @@ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteC
      * @throws IgniteCheckedException if no DataRegion is configured for a name obtained from cache descriptor.
      */
     private PageMemoryEx getPageMemoryForCacheGroup(int grpId) throws IgniteCheckedException {
+        if (grpId == MetaStorage.METASTORAGE_CACHE_ID)
+            return (PageMemoryEx)dataRegion(METASTORE_DATA_REGION_NAME).pageMemory();
+
         // TODO IGNITE-5075: cache descriptor can be removed.
         GridCacheSharedContext sharedCtx = context();
 
         CacheGroupDescriptor desc = sharedCtx.cache().cacheGroupDescriptors().get(grpId);
 
         if (desc == null)
-            throw new IgniteCheckedException("Failed to find cache group descriptor [grpId=" + grpId + ']');
+            return null;
 
         String memPlcName = desc.config().getDataRegionName();
 
@@ -2206,13 +2261,13 @@ private PageMemoryEx getPageMemoryForCacheGroup(int grpId) throws IgniteCheckedE
      * @param it WalIterator.
      * @param recPredicate Wal record filter.
      * @param entryPredicate Entry filter.
-     * @param partStates Partition to restore state.
+     * @param partitionRecoveryStates Partition to restore state.
      */
     public void applyUpdatesOnRecovery(
         @Nullable WALIterator it,
         IgnitePredicate<IgniteBiTuple<WALPointer, WALRecord>> recPredicate,
         IgnitePredicate<DataEntry> entryPredicate,
-        Map<T2<Integer, Integer>, T2<Integer, Long>> partStates
+        Map<GroupPartitionId, PartitionRecoverState> partitionRecoveryStates
     ) throws IgniteCheckedException {
         cctx.walState().runWithOutWAL(() -> {
             if (it != null) {
@@ -2269,7 +2324,7 @@ else if (log != null)
             checkpointReadLock();
 
             try {
-                restorePartitionStates(partStates, null);
+                restorePartitionStates(cctx.cache().cacheGroups(), partitionRecoveryStates);
             }
             catch (IgniteCheckedException e) {
                 throw new IgniteException(e);
@@ -2282,31 +2337,30 @@ else if (log != null)
 
     /**
      * @param status Last registered checkpoint status.
-     * @param metastoreOnly If {@code True} only records related to metastorage will be processed.
      * @throws IgniteCheckedException If failed to apply updates.
      * @throws StorageException If IO exception occurred while reading write-ahead log.
      */
-    private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) throws IgniteCheckedException {
+    private RestoreLogicalState applyLogicalUpdates(
+        CheckpointStatus status,
+        Predicate<Integer> cacheGroupsPredicate,
+        boolean skipFieldLookup
+    ) throws IgniteCheckedException {
         if (log.isInfoEnabled())
             log.info("Applying lost cache updates since last checkpoint record [lastMarked="
                 + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
 
-        if (!metastoreOnly)
+        if (skipFieldLookup)
             cctx.kernalContext().query().skipFieldLookup(true);
 
         long lastArchivedSegment = cctx.wal().lastArchivedSegment();
 
-        RestoreLogicalState restoreLogicalState = new RestoreLogicalState(lastArchivedSegment, log);
+        RestoreLogicalState restoreLogicalState = new RestoreLogicalState(lastArchivedSegment, cacheGroupsPredicate);
 
         long start = U.currentTimeMillis();
-        int applied = 0;
 
-        Collection<Integer> ignoreGrps = metastoreOnly ? Collections.emptySet() :
-            F.concat(false, initiallyGlobalWalDisabledGrps, initiallyLocalWalDisabledGrps);
+        int applied = 0;
 
         try (WALIterator it = cctx.wal().replay(status.startPtr)) {
-            Map<T2<Integer, Integer>, T2<Integer, Long>> partStates = new HashMap<>();
-
             while (it.hasNextX()) {
                 WALRecord rec = restoreLogicalState.next(it);
 
@@ -2315,9 +2369,6 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
 
                 switch (rec.type()) {
                     case DATA_RECORD:
-                        if (metastoreOnly)
-                            continue;
-
                         DataRecord dataRec = (DataRecord)rec;
 
                         for (DataEntry dataEntry : dataRec.writeEntries()) {
@@ -2329,27 +2380,27 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
                             if (cacheDesc == null)
                                 continue;
 
-                            if (!ignoreGrps.contains(cacheDesc.groupId())) {
-                                GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+                            GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
 
-                                applyUpdate(cacheCtx, dataEntry);
+                            applyUpdate(cacheCtx, dataEntry);
 
-                                applied++;
-                            }
+                            applied++;
                         }
 
                         break;
 
                     case PART_META_UPDATE_STATE:
-                        if (metastoreOnly)
-                            continue;
-
                         PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord)rec;
 
-                        if (!ignoreGrps.contains(metaStateRecord.groupId())) {
-                            partStates.put(new T2<>(metaStateRecord.groupId(), metaStateRecord.partitionId()),
-                                new T2<>((int)metaStateRecord.state(), metaStateRecord.updateCounter()));
-                        }
+                        GroupPartitionId groupPartitionId = new GroupPartitionId(
+                            metaStateRecord.groupId(), metaStateRecord.partitionId()
+                        );
+
+                        PartitionRecoverState state = new PartitionRecoverState(
+                            (int)metaStateRecord.state(), metaStateRecord.updateCounter()
+                        );
+
+                        restoreLogicalState.partitionRecoveryStates.put(groupPartitionId, state);
 
                         break;
 
@@ -2363,13 +2414,13 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
                     case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
                     case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
                     case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
-                        if (metastoreOnly)
-                            continue;
-
                         PageDeltaRecord rec0 = (PageDeltaRecord) rec;
 
                         PageMemoryEx pageMem = getPageMemoryForCacheGroup(rec0.groupId());
 
+                        if (pageMem == null)
+                            break;
+
                         long page = pageMem.acquirePage(rec0.groupId(), rec0.pageId(), true);
 
                         try {
@@ -2392,147 +2443,17 @@ private void applyLastUpdates(CheckpointStatus status, boolean metastoreOnly) th
                         // Skip other records.
                 }
             }
-
-            if (!metastoreOnly)
-                restorePartitionStates(partStates, null);
         }
         finally {
-            if (!metastoreOnly)
+            if (skipFieldLookup)
                 cctx.kernalContext().query().skipFieldLookup(false);
         }
 
         if (log.isInfoEnabled())
             log.info("Finished applying WAL changes [updatesApplied=" + applied +
-                ", time=" + (U.currentTimeMillis() - start) + "ms]");
-    }
-
-    /**
-     * Initializes not empty partitions and restores their state from page memory or WAL.
-     * Partition states presented in page memory may be overriden by states restored from WAL {@code partStates}.
-     *
-     * @param partStates Partition states restored from WAL.
-     * @param onlyForGroups If not {@code null} restore states only for specified cache groups.
-     * @throws IgniteCheckedException If failed to restore partition states.
-     */
-    private void restorePartitionStates(
-        Map<T2<Integer, Integer>, T2<Integer, Long>> partStates,
-        @Nullable Set<Integer> onlyForGroups
-    ) throws IgniteCheckedException {
-        for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
-            if (grp.isLocal() || !grp.affinityNode()) {
-                // Local cache has no partitions and its states.
-                continue;
-            }
-
-            if (!grp.dataRegion().config().isPersistenceEnabled())
-                continue;
-
-            if (onlyForGroups != null && !onlyForGroups.contains(grp.groupId()))
-                continue;
-
-            int grpId = grp.groupId();
-
-            PageMemoryEx pageMem = (PageMemoryEx)grp.dataRegion().pageMemory();
-
-            for (int i = 0; i < grp.affinity().partitions(); i++) {
-                T2<Integer, Long> restore = partStates.get(new T2<>(grpId, i));
-
-                if (storeMgr.exists(grpId, i)) {
-                    storeMgr.ensure(grpId, i);
-
-                    if (storeMgr.pages(grpId, i) <= 1)
-                        continue;
-
-                    if (log.isDebugEnabled())
-                        log.debug("Creating partition on recovery (exists in page store) " +
-                            "[grp=" + grp.cacheOrGroupName() + ", p=" + i + "]");
-
-                    GridDhtLocalPartition part = grp.topology().forceCreatePartition(i);
-
-                    assert part != null;
-
-                    // TODO: https://issues.apache.org/jira/browse/IGNITE-6097
-                    grp.offheap().onPartitionInitialCounterUpdated(i, 0);
-
-                    checkpointReadLock();
-
-                    try {
-                        long partMetaId = pageMem.partitionMetaPageId(grpId, i);
-                        long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
-
-                        try {
-                            long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
-
-                            boolean changed = false;
-
-                            try {
-                                PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
-
-                                if (restore != null) {
-                                    int stateId = restore.get1();
+                ", time=" + (U.currentTimeMillis() - start) + " ms]");
 
-                                    io.setPartitionState(pageAddr, (byte)stateId);
-
-                                    changed = updateState(part, stateId);
-
-                                    if (stateId == GridDhtPartitionState.OWNING.ordinal()
-                                        || (stateId == GridDhtPartitionState.MOVING.ordinal()
-                                        && part.initialUpdateCounter() < restore.get2())) {
-                                        part.initialUpdateCounter(restore.get2());
-
-                                        changed = true;
-                                    }
-
-                                    if (log.isDebugEnabled())
-                                        log.debug("Restored partition state (from WAL) " +
-                                            "[grp=" + grp.cacheOrGroupName() + ", p=" + i + ", state=" + part.state() +
-                                            "updCntr=" + part.initialUpdateCounter() + "]");
-                                }
-                                else {
-                                    updateState(part, (int) io.getPartitionState(pageAddr));
-
-                                    if (log.isDebugEnabled())
-                                        log.debug("Restored partition state (from page memory) " +
-                                            "[grp=" + grp.cacheOrGroupName() + ", p=" + i + ", state=" + part.state() +
-                                            "updCntr=" + part.initialUpdateCounter() + "]");
-                                }
-                            }
-                            finally {
-                                pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
-                            }
-                        }
-                        finally {
-                            pageMem.releasePage(grpId, partMetaId, partMetaPage);
-                        }
-                    }
-                    finally {
-                        checkpointReadUnlock();
-                    }
-                }
-                else if (restore != null) {
-                    if (log.isDebugEnabled())
-                        log.debug("Creating partition on recovery (exists in WAL) " +
-                            "[grp=" + grp.cacheOrGroupName() + ", p=" + i + "]");
-
-                    GridDhtLocalPartition part = grp.topology().forceCreatePartition(i);
-
-                    assert part != null;
-
-                    // TODO: https://issues.apache.org/jira/browse/IGNITE-6097
-                    grp.offheap().onPartitionInitialCounterUpdated(i, 0);
-
-                    updateState(part, restore.get1());
-
-                    if (log.isDebugEnabled())
-                        log.debug("Restored partition state (from WAL) " +
-                            "[grp=" + grp.cacheOrGroupName() + ", p=" + i + ", state=" + part.state() +
-                            "updCntr=" + part.initialUpdateCounter() + "]");
-                }
-            }
-
-            // After partition states are restored, it is necessary to update internal data structures in topology.
-            grp.topology().afterStateRestored(grp.topology().lastTopologyChangeVersion());
-        }
+        return restoreLogicalState;
     }
 
     /**
@@ -2547,25 +2468,6 @@ public void onWalTruncated(WALPointer highBound) throws IgniteCheckedException {
             removeCheckpointFiles(cp);
     }
 
-    /**
-     * @param part Partition to restore state for.
-     * @param stateId State enum ordinal.
-     * @return Updated flag.
-     */
-    private boolean updateState(GridDhtLocalPartition part, int stateId) {
-        if (stateId != -1) {
-            GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(stateId);
-
-            assert state != null;
-
-            part.restoreState(state == GridDhtPartitionState.EVICTED ? GridDhtPartitionState.RENTING : state);
-
-            return true;
-        }
-
-        return false;
-    }
-
     /**
      * @param cacheCtx Cache context to apply an update.
      * @param dataEntry Data entry to apply.
@@ -2577,7 +2479,7 @@ private void applyUpdate(GridCacheContext cacheCtx, DataEntry dataEntry) throws
         if (partId == -1)
             partId = cacheCtx.affinity().partition(dataEntry.key());
 
-        GridDhtLocalPartition locPart = cacheCtx.topology().forceCreatePartition(partId);
+        GridDhtLocalPartition locPart = cacheCtx.isLocal() ? null : cacheCtx.topology().forceCreatePartition(partId);
 
         switch (dataEntry.op()) {
             case CREATE:
@@ -3069,6 +2971,13 @@ protected Checkpointer(@Nullable String gridName, String name, IgniteLogger log)
                 while (!isCancelled()) {
                     waitCheckpointEvent();
 
+                    if (skipCheckpointOnNodeStop && (isStopping() || shutdownNow)) {
+                        if (log.isInfoEnabled())
+                            log.warning("Skipping last checkpoint because node is stopping.");
+
+                        return;
+                    }
+
                     GridFutureAdapter<Void> enableChangeApplied = GridCacheDatabaseSharedManager.this.enableChangeApplied;
 
                     if (enableChangeApplied != null) {
@@ -3501,7 +3410,7 @@ private void cancelOrWaitPartitionDestroy(int grpId, int partId) throws IgniteCh
             if (req != null)
                 req.waitCompleted();
 
-            if (log.isDebugEnabled())
+            if (req != null && log.isDebugEnabled())
                 log.debug("Partition file destroy has cancelled [grpId=" + grpId + ", partId=" + partId + "]");
         }
 
@@ -4040,7 +3949,7 @@ private List<FullPageId> writePages(Collection<FullPageId> writePageIds) throws
                 if (grpId != MetaStorage.METASTORAGE_CACHE_ID) {
                     CacheGroupContext grp = context().cache().cacheGroup(grpId);
 
-                    if (grp == null)
+                    if ( grp == null )
                         continue;
 
                     if (!grp.dataRegion().config().isPersistenceEnabled())
@@ -4545,10 +4454,10 @@ public boolean isCheckpointInapplicableForWalRebalance(Long cpTs, int grpId) thr
      *
      */
     private void fillWalDisabledGroups() {
-        MetaStorage meta = cctx.database().metaStorage();
+        assert metaStorage != null;
 
         try {
-            Set<String> keys = meta.readForPredicate(WAL_KEY_PREFIX_PRED).keySet();
+            Set<String> keys = metaStorage.readForPredicate(WAL_KEY_PREFIX_PRED).keySet();
 
             if (keys.isEmpty())
                 return;
@@ -4613,23 +4522,26 @@ else if (key.startsWith(WAL_GLOBAL_KEY_PREFIX))
     /**
      * Abstract class for create restore context.
      */
-    public abstract static class RestoreStateContext {
-        /** */
-        protected final IgniteLogger log;
-
+    private abstract class RestoreStateContext {
         /** Last archived segment. */
         protected final long lastArchivedSegment;
 
         /** Last read record WAL pointer. */
         protected FileWALPointer lastRead;
 
+        /** Only {@link WalRecordCacheGroupAware} records satisfied this predicate will be applied. */
+        private final Predicate<Integer> cacheGroupPredicate;
+
+        /** Set to {@code true} if data records should be skipped. */
+        private final boolean skipDataRecords;
+
         /**
          * @param lastArchivedSegment Last archived segment index.
-         * @param log Ignite logger.
          */
-        public RestoreStateContext(long lastArchivedSegment, IgniteLogger log) {
+        public RestoreStateContext(long lastArchivedSegment, Predicate<Integer> cacheGroupPredicate, boolean skipDataRecords) {
             this.lastArchivedSegment = lastArchivedSegment;
-            this.log = log;
+            this.cacheGroupPredicate = cacheGroupPredicate;
+            this.skipDataRecords = skipDataRecords;
         }
 
         /**
@@ -4641,17 +4553,63 @@ public RestoreStateContext(long lastArchivedSegment, IgniteLogger log) {
          */
         public WALRecord next(WALIterator it) throws IgniteCheckedException {
             try {
-                IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+                for (;;) {
+                    if (!it.hasNextX())
+                        return null;
+
+                    IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
 
-                WALRecord rec = tup.get2();
+                    if (tup == null)
+                        return null;
+
+                    WALRecord rec = tup.get2();
+
+                    WALPointer ptr = tup.get1();
+
+                    lastRead = (FileWALPointer)ptr;
+
+                    rec.position(ptr);
+
+                    // Filter out records.
+                    if (rec instanceof WalRecordCacheGroupAware) {
+                        WalRecordCacheGroupAware groupAwareRecord = (WalRecordCacheGroupAware) rec;
+
+                        if (!cacheGroupPredicate.test(groupAwareRecord.groupId()))
+                            continue;
+                    }
+
+                    switch (rec.type()) {
+                        case METASTORE_DATA_RECORD:
+                        case DATA_RECORD:
+                            if (skipDataRecords)
+                                continue;
+
+                            if (rec instanceof DataRecord) {
+                                DataRecord dataRecord = (DataRecord) rec;
+
+                                // Filter data entries by group id.
+                                List<DataEntry> filteredEntries = dataRecord.writeEntries().stream()
+                                        .filter(entry -> {
+                                            if (entry == null)
+                                                return false;
 
-                WALPointer ptr = tup.get1();
+                                            int cacheId = entry.cacheId();
+
+                                            return cctx != null && cctx.cacheContext(cacheId) != null && cacheGroupPredicate.test(cctx.cacheContext(cacheId).groupId());
+                                        })
+                                        .collect(Collectors.toList());
+
+                                dataRecord.setWriteEntries(filteredEntries);
+                            }
 
-                lastRead = (FileWALPointer)ptr;
+                            break;
 
-                rec.position(ptr);
+                        default:
+                            break;
+                    }
 
-                return rec;
+                    return rec;
+                }
             }
             catch (IgniteCheckedException e) {
                 boolean throwsCRCError = throwsCRCError();
@@ -4691,7 +4649,7 @@ public boolean throwsCRCError(){
     /**
      * Restore memory context. Tracks the safety of binary recovery.
      */
-    public static class RestoreBinaryState extends RestoreStateContext {
+    private class RestoreBinaryState extends RestoreStateContext {
         /** Checkpoint status. */
         private final CheckpointStatus status;
 
@@ -4701,13 +4659,12 @@ public static class RestoreBinaryState extends RestoreStateContext {
         /**
          * @param status Checkpoint status.
          * @param lastArchivedSegment Last archived segment index.
-         * @param log Ignite logger.
          */
-        public RestoreBinaryState(CheckpointStatus status, long lastArchivedSegment, IgniteLogger log) {
-            super(lastArchivedSegment, log);
+        public RestoreBinaryState(CheckpointStatus status, long lastArchivedSegment, Predicate<Integer> cacheGroupsPredicate) {
+            super(lastArchivedSegment, cacheGroupsPredicate, true);
 
             this.status = status;
-            needApplyBinaryUpdates = status.needRestoreMemory();
+            this.needApplyBinaryUpdates = status.needRestoreMemory();
         }
 
         /**
@@ -4767,13 +4724,15 @@ public boolean needApplyBinaryUpdate() {
     /**
      * Restore logical state context. Tracks the safety of logical recovery.
      */
-    public static class RestoreLogicalState extends RestoreStateContext {
+    private class RestoreLogicalState extends RestoreStateContext {
+        /** States of partitions recovered during applying logical updates. */
+        private final Map<GroupPartitionId, PartitionRecoverState> partitionRecoveryStates = new HashMap<>();
+
         /**
          * @param lastArchivedSegment Last archived segment index.
-         * @param log Ignite logger.
          */
-        public RestoreLogicalState(long lastArchivedSegment, IgniteLogger log) {
-            super(lastArchivedSegment, log);
+        public RestoreLogicalState(long lastArchivedSegment, Predicate<Integer> cacheGroupsPredicate) {
+            super(lastArchivedSegment, cacheGroupsPredicate, false);
         }
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 096a84e5cbb31..35a14f01491b2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -25,6 +25,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import javax.management.InstanceNotFoundException;
 import org.apache.ignite.DataRegionMetrics;
 import org.apache.ignite.DataStorageMetrics;
@@ -45,6 +46,7 @@
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
@@ -58,6 +60,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.freelist.CacheFreeListImpl;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
 import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
 import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport;
 import org.apache.ignite.internal.util.typedef.F;
@@ -88,19 +91,25 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     private static final long MAX_PAGE_MEMORY_INIT_SIZE_32_BIT = 2L * 1024 * 1024 * 1024;
 
     /** {@code True} to reuse memory on deactive. */
-    private final boolean reuseMemory = IgniteSystemProperties.getBoolean(IGNITE_REUSE_MEMORY_ON_DEACTIVATE);
+    protected final boolean reuseMemory = IgniteSystemProperties.getBoolean(IGNITE_REUSE_MEMORY_ON_DEACTIVATE);
 
     /** */
-    protected volatile Map<String, DataRegion> dataRegionMap;
+    protected final Map<String, DataRegion> dataRegionMap = new ConcurrentHashMap<>();
+
+    /** Stores memory providers eligible for reuse. */
+    private final Map<String, DirectMemoryProvider> memProviderMap = new ConcurrentHashMap<>();
 
     /** */
-    private volatile boolean dataRegionsInitialized;
+    private static final String MBEAN_GROUP_NAME = "DataRegionMetrics";
 
     /** */
-    private volatile boolean dataRegionsStarted;
+    protected final Map<String, DataRegionMetrics> memMetricsMap = new ConcurrentHashMap<>();
 
     /** */
-    protected Map<String, DataRegionMetrics> memMetricsMap;
+    private volatile boolean dataRegionsInitialized;
+
+    /** */
+    private volatile boolean dataRegionsStarted;
 
     /** */
     protected DataRegion dfltDataRegion;
@@ -114,8 +123,9 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     /** Page size from memory configuration, may be set only for fake(standalone) IgniteCacheDataBaseSharedManager */
     private int pageSize;
 
-    /** Stores memory providers eligible for reuse. */
-    private Map<String, DirectMemoryProvider> memProviderMap;
+    /** First eviction was warned flag. */
+    private volatile boolean firstEvictWarn;
+
 
     /** {@inheritDoc} */
     @Override protected void start0() throws IgniteCheckedException {
@@ -134,44 +144,88 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
     }
 
     /**
-     * Registers MBeans for all DataRegionMetrics configured in this instance.
-     */
-    private void registerMetricsMBeans() {
-        if(U.IGNITE_MBEANS_DISABLED)
+     * @param cfg Ignite configuration.
+     * @param groupName Name of group.
+     * @param dataRegionName Metrics MBean name.
+     * @param impl Metrics implementation.
+     * @param clazz Metrics class type.
+     */
+    protected <T> void registerMetricsMBean(
+        IgniteConfiguration cfg,
+        String groupName,
+        String dataRegionName,
+        T impl,
+        Class<T> clazz
+    ) {
+        if (U.IGNITE_MBEANS_DISABLED)
             return;
 
-        IgniteConfiguration cfg = cctx.gridConfig();
-
-        for (DataRegionMetrics memMetrics : memMetricsMap.values()) {
-            DataRegionConfiguration memPlcCfg = dataRegionMap.get(memMetrics.getName()).config();
-
-            registerMetricsMBean((DataRegionMetricsImpl)memMetrics, memPlcCfg, cfg);
+        try {
+            U.registerMBean(
+                cfg.getMBeanServer(),
+                cfg.getIgniteInstanceName(),
+                groupName,
+                dataRegionName,
+                impl,
+                clazz);
+        }
+        catch (Throwable e) {
+            U.error(log, "Failed to register MBean with name: " + dataRegionName, e);
         }
     }
 
     /**
-     * @param memMetrics Memory metrics.
-     * @param dataRegionCfg Data region configuration.
      * @param cfg Ignite configuration.
+     * @param groupName Name of group.
+     * @param name Name of MBean.
      */
-    private void registerMetricsMBean(
-        DataRegionMetricsImpl memMetrics,
-        DataRegionConfiguration dataRegionCfg,
-        IgniteConfiguration cfg
+    protected void unregisterMetricsMBean(
+        IgniteConfiguration cfg,
+        String groupName,
+        String name
     ) {
-        assert !U.IGNITE_MBEANS_DISABLED;
+        if (U.IGNITE_MBEANS_DISABLED)
+            return;
+
+        assert cfg != null;
 
         try {
-            U.registerMBean(
-                cfg.getMBeanServer(),
-                cfg.getIgniteInstanceName(),
-                "DataRegionMetrics",
-                dataRegionCfg.getName(),
-                new DataRegionMetricsMXBeanImpl(memMetrics, dataRegionCfg),
-                DataRegionMetricsMXBean.class);
+            cfg.getMBeanServer().unregisterMBean(
+                U.makeMBeanName(
+                    cfg.getIgniteInstanceName(),
+                    groupName,
+                    name
+                ));
+        }
+        catch (InstanceNotFoundException ignored) {
+            // We tried to unregister a non-existing MBean, not a big deal.
         }
         catch (Throwable e) {
-            U.error(log, "Failed to register MBean for DataRegionMetrics with name: '" + memMetrics.getName() + "'", e);
+            U.error(log, "Failed to unregister MBean for memory metrics: " + name, e);
+        }
+    }
+
+    /**
+     * Registers MBeans for all DataRegionMetrics configured in this instance.
+     *
+     * @param cfg Ignite configuration.
+     */
+    protected void registerMetricsMBeans(IgniteConfiguration cfg) {
+        if (U.IGNITE_MBEANS_DISABLED)
+            return;
+
+        assert cfg != null;
+
+        for (DataRegionMetrics memMetrics : memMetricsMap.values()) {
+            DataRegionConfiguration memPlcCfg = dataRegionMap.get(memMetrics.getName()).config();
+
+            registerMetricsMBean(
+                cfg,
+                MBEAN_GROUP_NAME,
+                memPlcCfg.getName(),
+                new DataRegionMetricsMXBeanImpl((DataRegionMetricsImpl)memMetrics, memPlcCfg),
+                DataRegionMetricsMXBean.class
+            );
         }
     }
 
@@ -216,11 +270,11 @@ public int pageSize() {
     /**
      *
      */
-    private void startMemoryPolicies() {
-        for (DataRegion memPlc : dataRegionMap.values()) {
-            memPlc.pageMemory().start();
+    private void startDataRegions() {
+        for (DataRegion region : dataRegionMap.values()) {
+            region.pageMemory().start();
 
-            memPlc.evictionTracker().start();
+            region.evictionTracker().start();
         }
     }
 
@@ -246,12 +300,6 @@ protected void initDataRegions(DataStorageConfiguration memCfg) throws IgniteChe
     protected void initDataRegions0(DataStorageConfiguration memCfg) throws IgniteCheckedException {
         DataRegionConfiguration[] dataRegionCfgs = memCfg.getDataRegionConfigurations();
 
-        int dataRegions = dataRegionCfgs == null ? 0 : dataRegionCfgs.length;
-
-        dataRegionMap = U.newHashMap(3 + dataRegions);
-        memMetricsMap = U.newHashMap(3 + dataRegions);
-        memProviderMap = reuseMemory ? U.newHashMap(3 + dataRegions) : null;
-
         if (dataRegionCfgs != null) {
             for (DataRegionConfiguration dataRegionCfg : dataRegionCfgs)
                 addDataRegion(memCfg, dataRegionCfg, dataRegionCfg.isPersistenceEnabled());
@@ -272,6 +320,17 @@ protected void initDataRegions0(DataStorageConfiguration memCfg) throws IgniteCh
             ),
             CU.isPersistenceEnabled(memCfg)
         );
+
+        for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext()))
+            lsnr.onInitDataRegions(this);
+    }
+
+    /**
+     * @param kctx Kernal context.
+     * @return Database lifecycle listeners.
+     */
+    protected List<DatabaseLifecycleListener> getDatabaseListeners(GridKernalContext kctx) {
+        return kctx.internalSubscriptionProcessor().getDatabaseListeners();
     }
 
     /**
@@ -293,17 +352,17 @@ public void addDataRegion(
 
         DataRegionMetricsImpl memMetrics = new DataRegionMetricsImpl(dataRegionCfg, freeSpaceProvider(dataRegionCfg));
 
-        DataRegion memPlc = initMemory(dataStorageCfg, dataRegionCfg, memMetrics, trackable);
+        DataRegion region = initMemory(dataStorageCfg, dataRegionCfg, memMetrics, trackable);
 
-        dataRegionMap.put(dataRegionName, memPlc);
+        dataRegionMap.put(dataRegionName, region);
 
         memMetricsMap.put(dataRegionName, memMetrics);
 
         if (dataRegionName.equals(dfltMemPlcName))
-            dfltDataRegion = memPlc;
+            dfltDataRegion = region;
         else if (dataRegionName.equals(DFLT_DATA_REG_DEFAULT_NAME))
             U.warn(log, "Data Region with name 'default' isn't used as a default. " +
-                    "Please check Memory Policies configuration.");
+                    "Please, check Data Region configuration.");
     }
 
     /**
@@ -611,13 +670,6 @@ public DataStorageMetrics persistentStoreMetrics() {
         return null;
     }
 
-    /**
-     * @throws IgniteCheckedException If fails.
-     */
-    public void onDoneRestoreBinaryMemory() throws IgniteCheckedException {
-        // No-op.
-    }
-
     /**
      * Creates file with current timestamp and specific "node-started.bin" suffix
      * and writes into memory recovery pointer.
@@ -652,7 +704,7 @@ public DataRegion dataRegion(String memPlcName) throws IgniteCheckedException {
         if (memPlcName == null)
             return dfltDataRegion;
 
-        if (dataRegionMap == null)
+        if (dataRegionMap.isEmpty())
             return null;
 
         DataRegion plc;
@@ -784,11 +836,9 @@ public void waitForCheckpoint(String reason) throws IgniteCheckedException {
 
     /**
      * @param discoEvt Before exchange for the given discovery event.
-     *
-     * @return {@code True} if partitions have been restored from persistent storage.
      */
-    public boolean beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws IgniteCheckedException {
-        return false;
+    public void beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws IgniteCheckedException {
+
     }
 
     /**
@@ -806,7 +856,7 @@ public void startMemoryRestore(GridKernalContext kctx) throws IgniteCheckedExcep
      *
      * @throws IgniteCheckedException If failed.
      */
-    public void onStateRestored() throws IgniteCheckedException {
+    public void onStateRestored(AffinityTopologyVersion topVer) throws IgniteCheckedException {
         // No-op.
     }
 
@@ -952,7 +1002,7 @@ private DirectMemoryProvider createOrReuseMemoryProvider(DataRegionConfiguration
      *
      * @return {@code True} if policy supports memory reuse.
      */
-    private boolean supportsMemoryReuse(DataRegionConfiguration plcCfg) {
+    public boolean supportsMemoryReuse(DataRegionConfiguration plcCfg) {
         return reuseMemory && plcCfg.getSwapPath() == null;
     }
 
@@ -1129,9 +1179,9 @@ private void startDataRegions(DataStorageConfiguration cfg) throws IgniteChecked
 
         assert cfg != null;
 
-        registerMetricsMBeans();
+        registerMetricsMBeans(cctx.gridConfig());
 
-        startMemoryPolicies();
+        startDataRegions();
 
         initPageMemoryDataStructures(cfg);
 
@@ -1149,29 +1199,29 @@ private void startDataRegions(DataStorageConfiguration cfg) throws IgniteChecked
      * @param shutdown {@code True} to force memory regions shutdown.
      */
     private void onDeActivate(boolean shutdown) {
-        if (dataRegionMap != null) {
-            for (DataRegion memPlc : dataRegionMap.values()) {
-                memPlc.pageMemory().stop(shutdown);
+        for (DatabaseLifecycleListener lsnr : getDatabaseListeners(cctx.kernalContext()))
+            lsnr.beforeStop(this);
 
-                memPlc.evictionTracker().stop();
+        for (DataRegion region : dataRegionMap.values()) {
+            region.pageMemory().stop(shutdown);
 
-                unregisterMBean(memPlc.memoryMetrics().getName());
-            }
+            region.evictionTracker().stop();
 
-            dataRegionMap.clear();
-
-            dataRegionMap = null;
+            unregisterMetricsMBean(
+                cctx.gridConfig(),
+                MBEAN_GROUP_NAME,
+                region.memoryMetrics().getName()
+            );
+            }
 
-            if (shutdown && memProviderMap != null) {
-                memProviderMap.clear();
+        dataRegionMap.clear();
 
-                memProviderMap = null;
-            }
+        if (shutdown && memProviderMap != null)
+            memProviderMap.clear();
 
-            dataRegionsInitialized = false;
+        dataRegionsInitialized = false;
 
-            dataRegionsStarted = false;
-        }
+        dataRegionsStarted = false;
     }
 
     /**
@@ -1196,6 +1246,16 @@ public MetaStorage metaStorage() {
         return null;
     }
 
+    /**
+     * Notifies {@link MetastorageLifecycleListener} that {@link MetaStorage} is ready for read.
+     * This method is called when all processors and managers have already started and right before discovery manager.
+     *
+     * @throws IgniteCheckedException If failed.
+     */
+    public void notifyMetaStorageSubscribersOnReadyForRead() throws IgniteCheckedException {
+        // No-op.
+    }
+
     /**
      * @param grpId Group ID.
      * @return WAL enabled flag.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java
index 104697e810e58..e0c545b8a87c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java
@@ -22,10 +22,6 @@
 import java.nio.channels.AsynchronousFileChannel;
 import java.nio.file.OpenOption;
 
-import static java.nio.file.StandardOpenOption.CREATE;
-import static java.nio.file.StandardOpenOption.READ;
-import static java.nio.file.StandardOpenOption.WRITE;
-
 /**
  * File I/O factory which uses {@link AsynchronousFileChannel} based implementation of FileIO.
  */
@@ -36,11 +32,6 @@ public class AsyncFileIOFactory implements FileIOFactory {
     /** Thread local channel future holder. */
     private transient volatile ThreadLocal<AsyncFileIO.ChannelOpFuture> holder = initHolder();
 
-    /** {@inheritDoc} */
-    @Override public FileIO create(File file) throws IOException {
-        return create(file, CREATE, READ, WRITE);
-    }
-
     /** {@inheritDoc} */
     @Override public FileIO create(File file, OpenOption... modes) throws IOException {
         if (holder == null) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java
index 27351853269db..b236000d07c72 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java
@@ -22,6 +22,10 @@
 import java.io.Serializable;
 import java.nio.file.OpenOption;
 
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
 /**
  * {@link FileIO} factory definition.
  */
@@ -33,7 +37,9 @@ public interface FileIOFactory extends Serializable {
      * @return File I/O interface.
      * @throws IOException If I/O interface creation was failed.
      */
-    public FileIO create(File file) throws IOException;
+    default FileIO create(File file) throws IOException{
+        return create(file, CREATE, READ, WRITE);
+    };
 
     /**
      * Creates I/O interface for file with specified mode.
@@ -43,5 +49,5 @@ public interface FileIOFactory extends Serializable {
      * @return File I/O interface.
      * @throws IOException If I/O interface creation was failed.
      */
-    public FileIO create(File file, OpenOption... modes) throws IOException;
+    FileIO create(File file, OpenOption... modes) throws IOException;
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 950622f6a191b..519ae6d964a5c 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -259,6 +259,10 @@ public void stop(boolean delete) throws StorageException {
                 + ", delete=" + delete + "]", e);
         }
         finally {
+            allocatedTracker.updateTotalAllocatedPages(-1L * allocated.getAndSet(0) / pageSize);
+
+            inited = false;
+
             lock.writeLock().unlock();
         }
     }
@@ -576,7 +580,8 @@ private void reinit(FileIO fileIO) throws IOException {
                     long off = pageOffset(pageId);
 
                     assert (off >= 0 && off + headerSize() <= allocated.get()) || recover :
-                        "off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) + ", pageId=" + U.hexLong(pageId);
+                        "off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) +
+                            ", pageId=" + U.hexLong(pageId) + ", file=" + cfgFile.getPath();
 
                     assert pageBuf.capacity() == pageSize;
                     assert pageBuf.position() == 0;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java
index 856ba1c749b45..3fa3e2dded52d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java
@@ -21,10 +21,6 @@
 import java.io.IOException;
 import java.nio.file.OpenOption;
 
-import static java.nio.file.StandardOpenOption.CREATE;
-import static java.nio.file.StandardOpenOption.READ;
-import static java.nio.file.StandardOpenOption.WRITE;
-
 /**
  * File I/O factory which provides RandomAccessFileIO implementation of FileIO.
  */
@@ -32,11 +28,6 @@ public class RandomAccessFileIOFactory implements FileIOFactory {
     /** */
     private static final long serialVersionUID = 0L;
 
-    /** {@inheritDoc} */
-    @Override public FileIO create(File file) throws IOException {
-        return create(file, CREATE, READ, WRITE);
-    }
-
     /** {@inheritDoc} */
     @Override public FileIO create(File file, OpenOption... modes) throws IOException {
         return new RandomAccessFileIO(file, modes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java
index 8ab418c4f4d5e..12cc4468fc010 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageLifecycleListener.java
@@ -33,7 +33,7 @@ public interface MetastorageLifecycleListener {
      *
      * @param metastorage Read-only meta storage.
      */
-    public void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException;
+    default void onReadyForRead(ReadOnlyMetastorage metastorage) throws IgniteCheckedException { };
 
     /**
      * Fully functional metastore capable of performing reading and writing operations.
@@ -43,5 +43,5 @@ public interface MetastorageLifecycleListener {
      *
      * @param metastorage Fully functional meta storage.
      */
-    public void onReadyForReadWrite(ReadWriteMetastorage metastorage) throws IgniteCheckedException;
+    default void onReadyForReadWrite(ReadWriteMetastorage metastorage) throws IgniteCheckedException { };
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionRecoverState.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionRecoverState.java
new file mode 100644
index 0000000000000..4c7e4d7323011
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/partstate/PartitionRecoverState.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.partstate;
+
+/**
+ * Class holds state of partition during recovery process.
+ */
+public class PartitionRecoverState {
+    /** State id. */
+    private final int stateId;
+
+    /** Update counter. */
+    private final long updateCounter;
+
+    /**
+     * @param stateId State id.
+     * @param updateCounter Update counter.
+     */
+    public PartitionRecoverState(int stateId, long updateCounter) {
+        this.stateId = stateId;
+        this.updateCounter = updateCounter;
+    }
+
+    /**
+     *
+     */
+    public int stateId() {
+        return stateId;
+    }
+
+    /**
+     *
+     */
+    public long updateCounter() {
+        return updateCounter;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
index 16cc8f560c278..75aa983abe62d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteCacheSnapshotManager.java
@@ -94,10 +94,6 @@ public boolean partitionsAreFrozen(CacheGroupContext grp) {
     /**
      *
      */
-    public void restoreState() throws IgniteCheckedException {
-        // No-op.
-    }
-
     public boolean snapshotOperationInProgress(){
         return false;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 5ebb8cfa668bc..dc3e5b80ae382 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -78,8 +78,11 @@
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
 import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
 import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
@@ -496,6 +499,23 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
         }
     }
 
+    /**
+     *
+     */
+    private void startArchiverAndCompressor() {
+        if (isArchiverEnabled()) {
+            assert archiver != null;
+
+            new IgniteThread(archiver).start();
+        }
+
+        if (walSegmentSyncWorker != null)
+            new IgniteThread(walSegmentSyncWorker).start();
+
+        if (compressor != null)
+            compressor.start();
+    }
+
     /**
      * Archiver can be not created, all files will be written to WAL folder, using absolute segment index.
      *
@@ -595,11 +615,17 @@ private void checkWalConfiguration() throws IgniteCheckedException {
             if (archiver != null)
                 archiver.shutdown();
 
-            if (compressor != null)
+            if (compressor != null) {
                 compressor.shutdown();
 
-            if (decompressor != null)
+                compressor = null;
+            }
+
+            if (decompressor != null) {
                 decompressor.shutdown();
+
+                decompressor = null;
+            }
         }
         catch (Exception e) {
             U.error(log, "Failed to gracefully close WAL segment: " + this.currHnd.fileIO, e);
@@ -613,20 +639,6 @@ private void checkWalConfiguration() throws IgniteCheckedException {
                 " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
 
         start0();
-
-        if (!cctx.kernalContext().clientNode()) {
-            if (isArchiverEnabled()) {
-                assert archiver != null;
-
-                new IgniteThread(archiver).start();
-            }
-
-            if (walSegmentSyncWorker != null)
-                new IgniteThread(walSegmentSyncWorker).start();
-
-            if (compressor != null)
-                compressor.start();
-        }
     }
 
     /** {@inheritDoc} */
@@ -654,6 +666,9 @@ private void checkWalConfiguration() throws IgniteCheckedException {
     @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
         assert currHnd == null;
         assert lastPtr == null || lastPtr instanceof FileWALPointer;
+
+        startArchiverAndCompressor();
+
         assert (isArchiverEnabled() && archiver != null) || (!isArchiverEnabled() && archiver == null) :
             "Trying to restore FileWriteHandle on deactivated write ahead log manager";
 
@@ -773,6 +788,11 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
         if (serializer == null || mode == WALMode.NONE)
             return null;
 
+        // Only delta-records, page snapshots and memory recovery are allowed to write in recovery mode.
+        if (cctx.kernalContext().recoveryMode() &&
+            !(rec instanceof PageDeltaRecord || rec instanceof PageSnapshot || rec instanceof MemoryRecoveryRecord))
+            return null;
+
         FileWriteHandle currWrHandle = currentHandle();
 
         WALDisableContext isDisable = walDisableContext;
@@ -1617,7 +1637,7 @@ private void shutdown() throws IgniteInterruptedCheckedException {
 
                     segmentAware.markAsMovedToArchive(toArchive);
 
-                    if (evt.isRecordable(EVT_WAL_SEGMENT_ARCHIVED)) {
+                    if (evt.isRecordable(EVT_WAL_SEGMENT_ARCHIVED) && !cctx.kernalContext().recoveryMode()) {
                         evt.record(new WalSegmentArchivedEvent(
                             cctx.discovery().localNode(),
                             res.getAbsIdx(),
@@ -1854,6 +1874,8 @@ private void shutdown() throws IgniteInterruptedCheckedException {
                 for (FileCompressorWorker worker: workers)
                     U.join(worker);
 
+                workers.clear();
+
                 U.cancel(this);
             }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 45935c3b50760..51405243e0ba5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -73,8 +73,11 @@
 import org.apache.ignite.internal.pagemem.wal.WALIterator;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.MarshalledRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
 import org.apache.ignite.internal.pagemem.wal.record.SwitchSegmentRecord;
 import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
 import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
@@ -410,6 +413,20 @@ public void setFileIOFactory(FileIOFactory ioFactory) {
         }
     }
 
+    /**
+     *
+     */
+    private void startArchiverAndCompressor() {
+        if (isArchiverEnabled()) {
+            assert archiver != null;
+
+            new IgniteThread(archiver).start();
+        }
+
+        if (compressor != null)
+            compressor.start();
+    }
+
     /**
      * @throws IgniteCheckedException if WAL store path is configured and archive path isn't (or vice versa)
      */
@@ -449,11 +466,17 @@ private void checkWalConfiguration() throws IgniteCheckedException {
             if (archiver != null)
                 archiver.shutdown();
 
-            if (compressor != null)
+            if (compressor != null) {
                 compressor.shutdown();
 
-            if (decompressor != null)
+                compressor = null;
+            }
+
+            if (decompressor != null) {
                 decompressor.shutdown();
+
+                decompressor = null;
+            }
         }
         catch (Exception e) {
             U.error(log, "Failed to gracefully close WAL segment: " + currentHnd.fileIO, e);
@@ -467,17 +490,6 @@ private void checkWalConfiguration() throws IgniteCheckedException {
                 " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
 
         start0();
-
-        if (!cctx.kernalContext().clientNode()) {
-            if (isArchiverEnabled()) {
-                assert archiver != null;
-
-                new IgniteThread(archiver).start();
-            }
-
-            if (compressor != null)
-                compressor.start();
-        }
     }
 
     /** {@inheritDoc} */
@@ -505,6 +517,9 @@ private void checkWalConfiguration() throws IgniteCheckedException {
     @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
         assert currentHnd == null;
         assert lastPtr == null || lastPtr instanceof FileWALPointer;
+
+        startArchiverAndCompressor();
+
         assert (isArchiverEnabled() && archiver != null) || (!isArchiverEnabled() && archiver == null) :
             "Trying to restore FileWriteHandle on deactivated write ahead log manager";
 
@@ -665,6 +680,11 @@ private void checkWalRolloverRequiredDuringInactivityPeriod() {
         if (serializer == null || mode == WALMode.NONE)
             return null;
 
+        // Only delta-records, page snapshots and memory recovery are allowed to write in recovery mode.
+        if (cctx.kernalContext().recoveryMode() &&
+            !(record instanceof PageDeltaRecord || record instanceof PageSnapshot || record instanceof MemoryRecoveryRecord))
+            return null;
+
         FileWriteHandle currWrHandle = currentHandle();
 
         WALDisableContext isDisable = walDisableContext;
@@ -1490,7 +1510,7 @@ private synchronized void release(long absIdx) {
                         notifyAll();
                     }
 
-                    if (evt.isRecordable(EventType.EVT_WAL_SEGMENT_ARCHIVED)) {
+                    if (evt.isRecordable(EventType.EVT_WAL_SEGMENT_ARCHIVED) && !cctx.kernalContext().recoveryMode()) {
                         evt.record(new WalSegmentArchivedEvent(cctx.discovery().localNode(),
                             res.getAbsIdx(), res.getDstArchiveFile()));
                     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index cd4179645a3b3..956561f005126 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -640,6 +640,11 @@ protected IgniteConfiguration prepareIgniteConfiguration() {
         return null;
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean recoveryMode() {
+        return false;
+    }
+
     /** {@inheritDoc} */
     @Override public PdsFoldersResolver pdsFolderResolver() {
         return new PdsFoldersResolver() {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java
index 6c0942a1e7cca..6db7fa5ffcca0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/subscription/GridInternalSubscriptionProcessor.java
@@ -20,6 +20,7 @@
 import java.util.List;
 import org.apache.ignite.internal.GridKernalContext;
 import org.apache.ignite.internal.processors.GridProcessorAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.DatabaseLifecycleListener;
 import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetastorageLifecycleListener;
 import org.jetbrains.annotations.NotNull;
 
@@ -35,6 +36,10 @@ public class GridInternalSubscriptionProcessor extends GridProcessorAdapter {
     /** */
     private List<MetastorageLifecycleListener> metastorageListeners = new ArrayList<>();
 
+    /** */
+    private List<DatabaseLifecycleListener> databaseListeners = new ArrayList<>();
+
+
     /**
      * @param ctx Kernal context.
      */
@@ -54,4 +59,17 @@ public void registerMetastorageListener(@NotNull MetastorageLifecycleListener me
     public List<MetastorageLifecycleListener> getMetastorageSubscribers() {
         return metastorageListeners;
     }
+
+    /** */
+    public void registerDatabaseListener(@NotNull DatabaseLifecycleListener databaseListener) {
+        if (databaseListener == null)
+            throw new NullPointerException("Database subscriber should be not-null.");
+
+        databaseListeners.add(databaseListener);
+    }
+
+    /** */
+    public List<DatabaseLifecycleListener> getDatabaseListeners() {
+        return databaseListeners;
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 8415c692e5b71..706c8e9cb9ef8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -1378,7 +1378,7 @@ final void checkNoCaches(int nodes) {
             GridCacheProcessor cache = ((IgniteEx)ignite(i)).context().cache();
 
             assertTrue(cache.caches().isEmpty());
-            assertTrue(cache.internalCaches().isEmpty());
+            assertTrue(cache.internalCaches().stream().allMatch(c -> c.context().isRecoveryMode()));
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
index 989626f18ad2c..c585041f02221 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsBinaryMetadataOnClusterRestartTest.java
@@ -74,6 +74,8 @@ public class IgnitePdsBinaryMetadataOnClusterRestartTest extends GridCommonAbstr
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
+        cfg.setConsistentId(gridName);
+
         if (customWorkSubDir != null)
             cfg.setWorkDirectory(Paths.get(U.defaultWorkDirectory(), customWorkSubDir).toString());
 
@@ -358,8 +360,8 @@ private void copyIncompatibleBinaryMetadata(String fromWorkDir,
     ) throws Exception {
         String workDir = U.defaultWorkDirectory();
 
-        Path fromFile = Paths.get(workDir, fromWorkDir, "binary_meta", "node00-" + fromConsId, fileName);
-        Path toFile = Paths.get(workDir, toWorkDir, "binary_meta", "node00-" + toConsId, fileName);
+        Path fromFile = Paths.get(workDir, fromWorkDir, "binary_meta", fromConsId, fileName);
+        Path toFile = Paths.get(workDir, toWorkDir, "binary_meta", toConsId, fileName);
 
         Files.copy(fromFile, toFile, StandardCopyOption.REPLACE_EXISTING);
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
index 14d0fb6d4dfb7..da19285d6301d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedIndexTest.java
@@ -317,16 +317,6 @@ private static boolean isPartitionFile(File file) {
             return file.getName().contains("part") && file.getName().endsWith("bin");
         }
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            FileIO delegate = delegateFactory.create(file);
-
-            if (isPartitionFile(file))
-                return new HaltOnTruncateFileIO(delegate, file);
-
-            return delegate;
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... modes) throws IOException {
             FileIO delegate = delegateFactory.create(file, modes);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
index fda25e65d5da3..15a1e59d54846 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
@@ -165,6 +165,9 @@ public void testNodeInvalidatedWhenPersistenceIsCorrupted() throws Exception {
             startGrid(0);
         }
         catch (IgniteCheckedException ex) {
+            if (X.hasCause(ex, StorageException.class, IOException.class))
+                return; // Success;
+
             throw ex;
         }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
index 3605700d739fc..578692c97e840 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPartitionFilesDestroyTest.java
@@ -449,16 +449,6 @@ private static boolean isPartitionFile(File file) {
             return file.getName().contains("part") && file.getName().endsWith("bin");
         }
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            FileIO delegate = delegateFactory.create(file);
-
-            if (isPartitionFile(file))
-                return new FailingFileIO(delegate);
-
-            return delegate;
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... modes) throws IOException {
             FileIO delegate = delegateFactory.create(file, modes);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java
index d42b7885add49..5a700e7024d63 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTaskCancelingTest.java
@@ -294,11 +294,6 @@ private static class SlowIOFactory implements FileIOFactory {
         /** */
         private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return create(file, CREATE, READ, WRITE);
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... openOption) throws IOException {
             final FileIO delegate = delegateFactory.create(file, openOption);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
index 4dd5f51b28772..48b60d48c9981 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/LocalWalModeChangeDuringRebalancingSelfTest.java
@@ -545,11 +545,6 @@ private static class TestFileIOFactory implements FileIOFactory {
             this.delegate = delegate;
         }
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return new TestFileIO(delegate.create(file));
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... modes) throws IOException {
             return new TestFileIO(delegate.create(file, modes));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
index 3afafe691f462..f58d02d93e944 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/CheckpointBufferDeadlockTest.java
@@ -335,11 +335,6 @@ private static class SlowCheckpointFileIOFactory implements FileIOFactory {
         /** Delegate factory. */
         private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return create(file, CREATE, READ, WRITE);
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... openOption) throws IOException {
             final FileIO delegate = delegateFactory.create(file, openOption);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java
new file mode 100644
index 0000000000000..03a205123bcba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java
@@ -0,0 +1,560 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.OpenOption;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.collect.Lists;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.failure.FailureHandler;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+import org.junit.Assert;
+
+/**
+ * A set of tests that check correctness of logical recovery performed during node start.
+ */
+public class IgniteLogicalRecoveryTest extends GridCommonAbstractTest {
+    /** Shared group name. */
+    private static final String SHARED_GROUP_NAME = "group";
+
+    /** Dynamic cache prefix. */
+    private static final String DYNAMIC_CACHE_PREFIX = "dynamic-cache-";
+
+    /** Cache prefix. */
+    private static final String CACHE_PREFIX = "cache-";
+
+    /** Io factory. */
+    private FileIOFactory ioFactory;
+
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        cfg.setCacheConfiguration(
+            cacheConfiguration(CACHE_PREFIX + 0, CacheMode.PARTITIONED, CacheAtomicityMode.ATOMIC),
+            cacheConfiguration(CACHE_PREFIX + 1, CacheMode.PARTITIONED, CacheAtomicityMode.TRANSACTIONAL),
+            cacheConfiguration(CACHE_PREFIX + 2, CacheMode.REPLICATED, CacheAtomicityMode.ATOMIC),
+            cacheConfiguration(CACHE_PREFIX + 3, CacheMode.REPLICATED, CacheAtomicityMode.TRANSACTIONAL),
+            cacheConfiguration(CACHE_PREFIX + 4, SHARED_GROUP_NAME, CacheMode.PARTITIONED, CacheAtomicityMode.TRANSACTIONAL),
+            cacheConfiguration(CACHE_PREFIX + 5, SHARED_GROUP_NAME, CacheMode.PARTITIONED, CacheAtomicityMode.TRANSACTIONAL)
+        );
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+            .setAlwaysWriteFullPages(true)
+            .setWalMode(WALMode.LOG_ONLY)
+            .setCheckpointFrequency(1024 * 1024 * 1024) // Disable automatic checkpoints.
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setName("dflt")
+                    .setInitialSize(256 * 1024 * 1024)
+                    .setMaxSize(256 * 1024 * 1024)
+                    .setPersistenceEnabled(true)
+            );
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        if (ioFactory != null)
+            dsCfg.setFileIOFactory(ioFactory);
+
+        return cfg;
+    }
+
+    /**
+     * @param name Name.
+     * @param cacheMode Cache mode.
+     * @param atomicityMode Atomicity mode.
+     */
+    private CacheConfiguration<Object, Object> cacheConfiguration(String name, CacheMode cacheMode, CacheAtomicityMode atomicityMode) {
+        return cacheConfiguration(name, null, cacheMode, atomicityMode);
+    }
+
+    /**
+     * @param name Name.
+     * @param groupName Group name.
+     * @param cacheMode Cache mode.
+     * @param atomicityMode Atomicity mode.
+     */
+    private CacheConfiguration<Object, Object> cacheConfiguration(String name, @Nullable String groupName, CacheMode cacheMode, CacheAtomicityMode atomicityMode) {
+        CacheConfiguration<Object, Object> cfg = new CacheConfiguration<>();
+
+        cfg.setGroupName(groupName);
+        cfg.setName(name);
+        cfg.setCacheMode(cacheMode);
+        cfg.setAtomicityMode(atomicityMode);
+        cfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+        cfg.setBackups(2);
+        cfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        System.setProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP, "true");
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        System.clearProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_SKIP_CHECKPOINT_ON_NODE_STOP);
+    }
+
+    /**
+     *
+     */
+    public void testRecoveryOnJoinToActiveCluster() throws Exception {
+        IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3);
+
+        crd.cluster().active(true);
+
+        IgniteEx node = grid(2);
+
+        AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node);
+
+        aggCacheLoader.start();
+
+        U.sleep(3000);
+
+        forceCheckpoint();
+
+        U.sleep(3000);
+
+        aggCacheLoader.stop();
+
+        stopGrid(2, true);
+
+        startGrid(2);
+
+        awaitPartitionMapExchange();
+
+        aggCacheLoader.consistencyCheck(grid(2));
+    }
+
+    /**
+     *
+     */
+    public void testRecoveryOnJoinToInactiveCluster() throws Exception {
+        IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3);
+
+        crd.cluster().active(true);
+
+        IgniteEx node = grid(2);
+
+        AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node);
+
+        aggCacheLoader.start();
+
+        U.sleep(3000);
+
+        forceCheckpoint();
+
+        U.sleep(3000);
+
+        aggCacheLoader.stop();
+
+        stopGrid(2, true);
+
+        crd.cluster().active(false);
+
+        startGrid(2);
+
+        crd.cluster().active(true);
+
+        awaitPartitionMapExchange();
+
+        aggCacheLoader.consistencyCheck(grid(2));
+    }
+
+    /**
+     *
+     */
+    public void testRecoveryOnDynamicallyStartedCaches() throws Exception {
+        List<CacheConfiguration> dynamicCaches = Lists.newArrayList(
+            cacheConfiguration(DYNAMIC_CACHE_PREFIX + 0, CacheMode.PARTITIONED, CacheAtomicityMode.TRANSACTIONAL),
+            cacheConfiguration(DYNAMIC_CACHE_PREFIX + 1, CacheMode.REPLICATED, CacheAtomicityMode.TRANSACTIONAL),
+            cacheConfiguration(DYNAMIC_CACHE_PREFIX + 2, CacheMode.PARTITIONED, CacheAtomicityMode.ATOMIC),
+            cacheConfiguration(DYNAMIC_CACHE_PREFIX + 3, CacheMode.REPLICATED, CacheAtomicityMode.ATOMIC)
+        );
+
+        doTestWithDynamicCaches(dynamicCaches);
+    }
+
+    /**
+     * @param dynamicCaches Dynamic caches.
+     */
+    private void doTestWithDynamicCaches(List<CacheConfiguration> dynamicCaches) throws Exception {
+        IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3);
+
+        crd.cluster().active(true);
+
+        IgniteEx node = grid(2);
+
+        node.getOrCreateCaches(dynamicCaches);
+
+        AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node);
+
+        aggCacheLoader.start();
+
+        U.sleep(3000);
+
+        forceCheckpoint();
+
+        U.sleep(3000);
+
+        aggCacheLoader.stop();
+
+        stopGrid(2, true);
+
+        startGrid(2);
+
+        awaitPartitionMapExchange();
+
+        for (int idx = 0; idx < 3; idx++)
+            aggCacheLoader.consistencyCheck(grid(idx));
+    }
+
+    /**
+     *
+     */
+    public void testRecoveryOnJoinToDifferentBlt() throws Exception {
+        IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3);
+
+        crd.cluster().active(true);
+
+        IgniteEx node = grid(2);
+
+        AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node);
+
+        aggCacheLoader.start();
+
+        U.sleep(3000);
+
+        forceCheckpoint();
+
+        U.sleep(3000);
+
+        aggCacheLoader.stop();
+
+        stopGrid(2, true);
+
+        resetBaselineTopology();
+
+        startGrid(2);
+
+        resetBaselineTopology();
+
+        awaitPartitionMapExchange();
+
+        for (int idx = 0; idx < 3; idx++)
+            aggCacheLoader.consistencyCheck(grid(idx));
+    }
+
+    /**
+     *
+     */
+    public void testRecoveryOnCrushDuringCheckpointOnNodeStart() throws Exception {
+        // Crash recovery fails because of the bug in pages recycling.
+        // Test passes if don't perform removes in cache loader.
+        fail("https://issues.apache.org/jira/browse/IGNITE-9303");
+
+        IgniteEx crd = (IgniteEx) startGridsMultiThreaded(3, false);
+
+        crd.cluster().active(true);
+
+        IgniteEx node = grid(2);
+
+        AggregateCacheLoader aggCacheLoader = new AggregateCacheLoader(node);
+
+        aggCacheLoader.start();
+
+        U.sleep(3000);
+
+        forceCheckpoint();
+
+        U.sleep(3000);
+
+        aggCacheLoader.stop();
+
+        stopGrid(2, false);
+
+        ioFactory = new CheckpointFailIoFactory();
+
+        IgniteInternalFuture startNodeFut = GridTestUtils.runAsync(() -> startGrid(2));
+
+        try {
+            startNodeFut.get();
+        }
+        catch (Exception expected) { }
+
+        ioFactory = null;
+
+        // Start node again and check recovery.
+        startGrid(2);
+
+        awaitPartitionMapExchange();
+
+        for (int idx = 0; idx < 3; idx++)
+            aggCacheLoader.consistencyCheck(grid(idx));
+    }
+
+    /** {@inheritDoc} */
+    @Override protected FailureHandler getFailureHandler(String igniteInstanceName) {
+        return new StopNodeFailureHandler();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long getTestTimeout() {
+        return 600 * 1000;
+    }
+
+    /**
+     *
+     */
+    private class AggregateCacheLoader {
+        /** Ignite. */
+        IgniteEx ignite;
+
+        /** Stop flag. */
+        AtomicBoolean stopFlag;
+
+        /** Cache loaders. */
+        Map<CacheLoader, IgniteInternalFuture> cacheLoaders;
+
+        /**
+         * @param ignite Ignite.
+         */
+        public AggregateCacheLoader(IgniteEx ignite) {
+            this.ignite = ignite;
+        }
+
+        /**
+         *
+         */
+        public void start() {
+            if (stopFlag != null && !stopFlag.get())
+                throw new IllegalStateException("Cache loaders must be stopped before start again");
+
+            stopFlag = new AtomicBoolean();
+            cacheLoaders = new HashMap<>();
+
+            for (String cacheName : ignite.cacheNames()) {
+                CacheLoader loader = new CacheLoader(ignite, stopFlag, cacheName);
+
+                IgniteInternalFuture loadFuture = GridTestUtils.runAsync(loader);
+
+                cacheLoaders.put(loader, loadFuture);
+            }
+        }
+
+        /**
+         *
+         */
+        public void stop() throws IgniteCheckedException {
+            if (stopFlag != null)
+                stopFlag.set(true);
+
+            if (cacheLoaders != null)
+                for (IgniteInternalFuture loadFuture : cacheLoaders.values())
+                    loadFuture.get();
+        }
+
+        /**
+         * @param ignite Ignite.
+         */
+        public void consistencyCheck(IgniteEx ignite) throws IgniteCheckedException {
+            if (cacheLoaders != null)
+                for (CacheLoader cacheLoader : cacheLoaders.keySet())
+                    cacheLoader.consistencyCheck(ignite);
+        }
+    }
+
+    /**
+     *
+     */
+    static class CacheLoader implements Runnable {
+        /** Keys space. */
+        static final int KEYS_SPACE = 3096;
+
+        /** Ignite. */
+        final IgniteEx ignite;
+
+        /** Stop flag. */
+        final AtomicBoolean stopFlag;
+
+        /** Cache name. */
+        final String cacheName;
+
+        /** Local cache. */
+        final Map<Object, Object> locCache = new ConcurrentHashMap<>();
+
+        /**
+         * @param ignite Ignite.
+         * @param stopFlag Stop flag.
+         * @param cacheName Cache name.
+         */
+        public CacheLoader(IgniteEx ignite, AtomicBoolean stopFlag, String cacheName) {
+            this.ignite = ignite;
+            this.stopFlag = stopFlag;
+            this.cacheName = cacheName;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void run() {
+            while (!stopFlag.get()) {
+                ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+                int key = rnd.nextInt(KEYS_SPACE);
+
+                boolean remove = rnd.nextInt(100) <= 20;
+
+                IgniteCache<Object, Object> cache = ignite.getOrCreateCache(cacheName);
+
+                try {
+                    if (remove) {
+                        cache.remove(key);
+
+                        locCache.remove(key);
+                    }
+                    else {
+                        int[] payload = new int[KEYS_SPACE];
+                        Arrays.fill(payload, key);
+
+                        cache.put(key, payload);
+
+                        locCache.put(key, payload);
+                    }
+                }
+                catch (Exception ignored) { }
+            }
+        }
+
+        /**
+         *
+         */
+        public void consistencyCheck(IgniteEx ignite) {
+            IgniteCache<Object, Object> cache = ignite.getOrCreateCache(cacheName);
+
+            for (int key = 0; key < KEYS_SPACE; key++) {
+                int[] expectedValue = (int[]) locCache.get(key);
+                int[] actualValue = (int[]) cache.get(key);
+
+                Assert.assertEquals("Consistency check failed for: " + cache.getName() + ", key=" + key,
+                    arrayToString(expectedValue), arrayToString(actualValue));
+            }
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean equals(Object o) {
+            if (this == o)
+                return true;
+            if (o == null || getClass() != o.getClass())
+                return false;
+
+            CacheLoader loader = (CacheLoader) o;
+
+            return Objects.equals(cacheName, loader.cacheName);
+        }
+
+        /** {@inheritDoc} */
+        @Override public int hashCode() {
+            return Objects.hash(cacheName);
+        }
+    }
+
+    /**
+     *
+     */
+    static class CheckpointFailIoFactory implements FileIOFactory {
+        /** {@inheritDoc} */
+        @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+            FileIO delegate = new RandomAccessFileIOFactory().create(file, modes);
+
+            if (file.getName().contains("part-"))
+                return new FileIODecorator(delegate) {
+                    @Override public int write(ByteBuffer srcBuf) throws IOException {
+                        throw new IOException("test");
+                    }
+
+                    @Override public int write(ByteBuffer srcBuf, long position) throws IOException {
+                        throw new IOException("test");
+                    }
+
+                    @Override public int write(byte[] buf, int off, int len) throws IOException {
+                        throw new IOException("test");
+                    }
+                };
+
+            return delegate;
+        }
+    }
+
+    /**
+     * @param arr Array.
+     */
+    static String arrayToString(int[] arr) {
+        if (arr == null)
+            return "null";
+
+        StringBuilder sb = new StringBuilder();
+        sb.append('[');
+        for (int i = 0; i < Math.min(arr.length, 10); i++)
+            sb.append(i + ",");
+        sb.append(']');
+
+        return sb.toString();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
index 268d2fb53dda3..c4649528a9f7c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsDataRegionMetricsTest.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
@@ -294,7 +295,7 @@ private static DataRegionMetrics getDfltRegionMetrics(Ignite node) {
     private void checkMetricsConsistency(final IgniteEx node, String cacheName) throws Exception {
         FilePageStoreManager pageStoreManager = (FilePageStoreManager)node.context().cache().context().pageStore();
 
-        long totalPersistanceSize = 0;
+        long totalPersistenceSize = 0;
         File cacheWorkDir = pageStoreManager.cacheWorkDir(
             node.getOrCreateCache(cacheName).getConfiguration(CacheConfiguration.class)
         );
@@ -303,7 +304,7 @@ private void checkMetricsConsistency(final IgniteEx node, String cacheName) thro
             cacheWorkDir.toPath(), entry -> entry.toFile().getName().endsWith(".bin"))
         ) {
             for (Path path : files)
-                totalPersistanceSize += path.toFile().length();
+                totalPersistenceSize += path.toFile().length();
         }
 
         long totalAllocatedPagesFromMetrics = node.context().cache().context()
@@ -313,6 +314,22 @@ private void checkMetricsConsistency(final IgniteEx node, String cacheName) thro
             .memoryMetrics()
             .getTotalAllocatedPages();
 
-        assertEquals(totalPersistanceSize / pageStoreManager.pageSize(), totalAllocatedPagesFromMetrics);
+        assertEquals("Number of allocated pages is different than in metrics for [node=" + node.name() + ", cache=" + cacheName + "]",
+            totalPersistenceSize / pageStoreManager.pageSize(), totalAllocatedPagesFromMetrics);
+    }
+
+    /**
+     * @param partFile Partition file.
+     */
+    private static int partId(File partFile) {
+        String name = partFile.getName();
+
+        if (name.equals(FilePageStoreManager.INDEX_FILE_NAME))
+            return PageIdAllocator.INDEX_PARTITION;
+
+        if (name.startsWith(FilePageStoreManager.PART_FILE_PREFIX))
+            return Integer.parseInt(name.substring(FilePageStoreManager.PART_FILE_PREFIX.length(), name.indexOf('.')));
+
+        throw new IllegalStateException("Illegal partition file name: " + name);
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
index c64c17b21c24e..f44c04d254cfc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsDiskErrorsRecoveringTest.java
@@ -38,6 +38,7 @@
 import org.apache.ignite.failure.StopNodeFailureHandler;
 import org.apache.ignite.internal.GridKernalState;
 import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
 import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
@@ -50,9 +51,6 @@
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.junit.Assert;
 
-import static java.nio.file.StandardOpenOption.CREATE;
-import static java.nio.file.StandardOpenOption.READ;
-import static java.nio.file.StandardOpenOption.WRITE;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
 
 /**
@@ -95,6 +93,8 @@ public class IgnitePdsDiskErrorsRecoveringTest extends GridCommonAbstractTest {
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
+        cfg.setConsistentId(igniteInstanceName);
+
         DataStorageConfiguration dsCfg = new DataStorageConfiguration()
                 .setDefaultDataRegionConfiguration(
                         new DataRegionConfiguration().setMaxSize(100 * 1024 * 1024).setPersistenceEnabled(true))
@@ -127,27 +127,36 @@ public void testRecoveringOnCacheInitFail() throws Exception {
         // Fail to initialize page store. 2 extra pages is needed for MetaStorage.
         ioFactory = new FilteringFileIOFactory(".bin", new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), 2 * PAGE_SIZE));
 
-        final IgniteEx grid = startGrid(0);
-
         boolean failed = false;
+
+        IgniteInternalFuture startGridFut = GridTestUtils.runAsync(() -> {
+            try {
+                IgniteEx grid = startGrid(0);
+
+                grid.cluster().active(true);
+            }
+            catch (Exception e) {
+                throw new RuntimeException("Failed to start node.", e);
+            }
+        });
+
         try {
-            grid.cluster().active(true);
-        } catch (Exception expected) {
-            log.warning("Expected cache error", expected);
+            startGridFut.get();
+        }
+        catch (Exception e) {
+            Assert.assertTrue(e.getMessage().contains("Failed to start node."));
 
             failed = true;
         }
 
         Assert.assertTrue("Cache initialization must failed", failed);
 
-        // Grid should be automatically stopped after checkpoint fail.
-        awaitStop(grid);
-
         // Grid should be successfully recovered after stopping.
         ioFactory = null;
 
-        IgniteEx recoveredGrid = startGrid(0);
-        recoveredGrid.active(true);
+        IgniteEx grid = startGrid(0);
+
+        grid.cluster().active(true);
     }
 
     /**
@@ -315,7 +324,9 @@ public void testRecoveringOnCheckpointWriteFail() throws Exception {
     public void testRecoveringOnWALWritingFail1() throws Exception {
         // Allow to allocate only 1 wal segment, fail on write to second.
         ioFactory = new FilteringFileIOFactory(".wal", new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), WAL_SEGMENT_SIZE));
+
         System.setProperty(IGNITE_WAL_MMAP, "true");
+
         doTestRecoveringOnWALWritingFail();
     }
 
@@ -325,7 +336,9 @@ public void testRecoveringOnWALWritingFail1() throws Exception {
     public void testRecoveringOnWALWritingFail2() throws Exception {
         // Fail somewhere on the second wal segment.
         ioFactory = new FilteringFileIOFactory(".wal", new LimitedSizeFileIOFactory(new RandomAccessFileIOFactory(), (long) (1.5 * WAL_SEGMENT_SIZE)));
+
         System.setProperty(IGNITE_WAL_MMAP, "false");
+
         doTestRecoveringOnWALWritingFail();
     }
 
@@ -333,18 +346,23 @@ public void testRecoveringOnWALWritingFail2() throws Exception {
      * Test node stopping & recovery on WAL writing fail.
      */
     private void doTestRecoveringOnWALWritingFail() throws Exception {
-        final IgniteEx grid = startGrid(0);
+        IgniteEx grid = startGrid(0);
 
         FileWriteAheadLogManager wal = (FileWriteAheadLogManager)grid.context().cache().context().wal();
+
         wal.setFileIOFactory(ioFactory);
 
         grid.cluster().active(true);
 
         int failedPosition = -1;
 
-        for (int i = 0; i < 1000; i++) {
+        final int keysCount = 2000;
+
+        final int dataSize = 2048;
+
+        for (int i = 0; i < keysCount; i++) {
             byte payload = (byte) i;
-            byte[] data = new byte[2048];
+            byte[] data = new byte[dataSize];
             Arrays.fill(data, payload);
 
             try {
@@ -358,7 +376,7 @@ private void doTestRecoveringOnWALWritingFail() throws Exception {
         }
 
         // We must be able to put something into cache before fail.
-        Assert.assertTrue(failedPosition > 0);
+        Assert.assertTrue("One of the cache puts must be failed", failedPosition > 0);
 
         // Grid should be automatically stopped after WAL fail.
         awaitStop(grid);
@@ -366,15 +384,16 @@ private void doTestRecoveringOnWALWritingFail() throws Exception {
         ioFactory = null;
 
         // Grid should be successfully recovered after stopping.
-        IgniteEx recoveredGrid = startGrid(0);
-        recoveredGrid.cluster().active(true);
+        grid = startGrid(0);
+
+        grid.cluster().active(true);
 
         for (int i = 0; i < failedPosition; i++) {
             byte payload = (byte) i;
-            byte[] data = new byte[2048];
+            byte[] data = new byte[dataSize];
             Arrays.fill(data, payload);
 
-            byte[] actualData = (byte[]) recoveredGrid.cache(CACHE_NAME).get(i);
+            byte[] actualData = (byte[]) grid.cache(CACHE_NAME).get(i);
             Assert.assertArrayEquals(data, actualData);
         }
     }
@@ -473,11 +492,6 @@ private static class FilteringFileIOFactory implements FileIOFactory {
             this.pattern = pattern;
         }
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return create(file, CREATE, WRITE, READ);
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... modes) throws IOException {
             if (file.getName().endsWith(pattern))
@@ -508,11 +522,6 @@ private LimitedSizeFileIOFactory(FileIOFactory delegate, long fsSpaceBytes) {
             this.availableSpaceBytes = new AtomicLong(fsSpaceBytes);
         }
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return new LimitedSizeFileIO(delegate.create(file), availableSpaceBytes);
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... modes) throws IOException {
             return new LimitedSizeFileIO(delegate.create(file, modes), availableSpaceBytes);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
index bec277c29bd41..906f19199a4b7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteNodeStoppedDuringDisableWALTest.java
@@ -91,6 +91,8 @@ public class IgniteNodeStoppedDuringDisableWALTest extends GridCommonAbstractTes
     }
 
     /**
+     * Test checks that after WAL is globally disabled and node is stopped, persistent store is cleaned properly after node restart.
+     *
      * @throws Exception If failed.
      */
     public void test() throws Exception {
@@ -189,6 +191,8 @@ private void testStopNodeWithDisableWAL(NodeStopPoint nodeStopPoint) throws Exce
 
         String msg = nodeStopPoint.toString();
 
+        int pageSize = ig1.configuration().getDataStorageConfiguration().getPageSize();
+
         if (nodeStopPoint.needCleanUp) {
             PdsFoldersResolver foldersResolver = ((IgniteEx)ig1).context().pdsFolderResolver();
 
@@ -214,14 +218,14 @@ private void testStopNodeWithDisableWAL(NodeStopPoint nodeStopPoint) throws Exce
                     if (CP_FILE_NAME_PATTERN.matcher(name).matches())
                         failed = true;
 
-                    if (name.startsWith(PART_FILE_PREFIX))
+                    if (name.startsWith(PART_FILE_PREFIX) && path.toFile().length() > pageSize)
                         failed = true;
 
-                    if (name.startsWith(INDEX_FILE_NAME))
+                    if (name.startsWith(INDEX_FILE_NAME) && path.toFile().length() > pageSize)
                         failed = true;
 
                     if (failed)
-                        fail(msg + " " + filePath);
+                        fail(msg + " " + filePath + " " + path.toFile().length());
 
                     return CONTINUE;
                 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
index 042a447d032eb..2b4398eedbde2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
@@ -185,11 +185,6 @@ private static class FailingFileIOFactory implements FileIOFactory {
             this.fail = fail;
         }
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return create(file, CREATE, READ, WRITE);
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... modes) throws IOException {
             final FileIO delegate = delegateFactory.create(file, modes);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index 0ed39ce003f4e..07fdb71fca68a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -254,11 +254,6 @@ private static class FailingFileIOFactory implements FileIOFactory {
             this.fail = fail;
         }
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return create(file, CREATE, READ, WRITE);
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... modes) throws IOException {
             final FileIO delegate = delegateFactory.create(file, modes);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
index 2ea269d3f053e..d5d73b87b0d5b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFormatFileFailoverTest.java
@@ -112,6 +112,8 @@ public void testNodeStartFailedFsync() throws Exception {
      * @throws Exception If failed.
      */
     public void testFailureHandlerTriggeredFsync() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-10035");
+
         fsync = true;
 
         failFormatFileOnClusterActivate();
@@ -121,6 +123,8 @@ public void testFailureHandlerTriggeredFsync() throws Exception {
      * @throws Exception If failed.
      */
     public void testFailureHandlerTriggered() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-10035");
+
         fsync = false;
 
         failFormatFileOnClusterActivate();
@@ -224,11 +228,6 @@ private static class FailingFileIOFactory implements FileIOFactory {
             this.failMtdNameRef = failMtdNameRef;
         }
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return create(file, CREATE, READ, WRITE);
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... modes) throws IOException {
             final FileIO delegate = delegateFactory.create(file, modes);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
index 1ab8f3121bae3..bf8923ebf54c6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
@@ -112,7 +112,7 @@ public void testReservedOnExchange() throws Exception {
 
         final IgniteEx ig0 = (IgniteEx)startGrids(initGridCnt + 1);
 
-        ig0.active(true);
+        ig0.cluster().active(true);
 
         stopGrid(initGridCnt);
 
@@ -254,7 +254,7 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception {
 
         IgniteEx ig0 = (IgniteEx)startGrids(2);
 
-        ig0.active(true);
+        ig0.cluster().active(true);
 
         IgniteCache<Integer, Integer> cache = ig0.cache("cache1");
 
@@ -268,6 +268,8 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception {
 
         IgniteEx ig1 = startGrid(1);
 
+        awaitPartitionMapExchange();
+
         IgniteCache<Integer, Integer> cache1 = ig1.cache("cache1");
 
         assertEquals(entryCnt / 2, cache.size());
@@ -284,8 +286,6 @@ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception {
             }
         }
 
-        cache.rebalance().get();
-
         for (int p = 0; p < ig1.affinity("cache1").partitions(); p++) {
             GridDhtLocalPartition p0 = ig0.context().cache().cache("cache1").context().topology().localPartition(p);
             GridDhtLocalPartition p1 = ig1.context().cache().cache("cache1").context().topology().localPartition(p);
@@ -303,7 +303,7 @@ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception {
 
         IgniteEx ig0 = (IgniteEx)startGrids(2);
 
-        ig0.active(true);
+        ig0.cluster().active(true);
 
         IgniteCache<Integer, Integer> cache = ig0.cache("cache1");
 
@@ -328,6 +328,8 @@ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception {
 
         IgniteEx ig1 = startGrid(1);
 
+        awaitPartitionMapExchange();
+
         IgniteCache<Integer, Integer> cache1 = ig1.cache("cache1");
 
         assertEquals(entryCnt / 2, cache.size());
@@ -344,8 +346,6 @@ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception {
             }
         }
 
-        cache.rebalance().get();
-
         for (int p = 0; p < ig1.affinity("cache1").partitions(); p++) {
             GridDhtLocalPartition p0 = ig0.context().cache().cache("cache1").context().topology().localPartition(p);
             GridDhtLocalPartition p1 = ig1.context().cache().cache("cache1").context().topology().localPartition(p);
@@ -405,7 +405,7 @@ public void testNodeLeftDuringExchange() throws Exception {
 
         final Ignite ig0 = startGrids(initGridCnt);
 
-        ig0.active(true);
+        ig0.cluster().active(true);
 
         IgniteCache<Object, Object> cache = ig0.cache("cache1");
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
index 57565bfd58c48..0409a297be1b1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRebalanceTest.java
@@ -612,11 +612,6 @@ static class FailingIOFactory implements FileIOFactory {
             this.delegate = delegate;
         }
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return create(file, CREATE, WRITE, READ);
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... modes) throws IOException {
             FileIO delegateIO = delegate.create(file, modes);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
index 31ee8110b4453..820cd2201e442 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
@@ -630,6 +630,8 @@ public void testRecoveryNoPageLost2() throws Exception {
     }
 
     /**
+     * Test checks that the number of pages per each page store are not changing before and after node restart.
+     *
      * @throws Exception If failed.
      */
     public void testRecoveryNoPageLost3() throws Exception {
@@ -725,7 +727,7 @@ private void recoveryNoPageLost(boolean checkpoint) throws Exception {
 
                 pages = allocatedPages(ignite, CACHE2_NAME);
 
-                ignite.close();
+                stopGrid(0, true);
             }
         }
         finally {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
index 48ff980d37e46..39a2a11d9ad3d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
@@ -287,11 +287,6 @@ private static class SlowCheckpointFileIOFactory implements FileIOFactory {
         /** Delegate factory. */
         private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
 
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return create(file, CREATE, READ, WRITE);
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... openOption) throws IOException {
             final FileIO delegate = delegateFactory.create(file, openOption);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java
index ddb29582e4576..7c20e1d242368 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java
@@ -783,7 +783,9 @@ public Runnable checkCacheEmpty() {
 
                     Map<String, GridCacheAdapter> caches = caches(ig);
 
-                    Assert.assertEquals(0, caches.size());
+                    for (GridCacheAdapter cacheAdapter : caches.values())
+                        Assert.assertTrue("Cache should be in recovery mode: " + cacheAdapter.context(),
+                            cacheAdapter.context().isRecoveryMode());
                 }
             });
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
index cf660c8b86832..cca7cfcf066fe 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIteratorTest.java
@@ -160,11 +160,6 @@ private String getArchiveWalDirPath(Ignite ignite) throws IgniteCheckedException
      *
      */
     private static class CountedFileIOFactory extends RandomAccessFileIOFactory {
-        /** {@inheritDoc} */
-        @Override public FileIO create(File file) throws IOException {
-            return create(file, StandardOpenOption.CREATE, StandardOpenOption.READ, StandardOpenOption.WRITE);
-        }
-
         /** {@inheritDoc} */
         @Override public FileIO create(File file, OpenOption... modes) throws IOException {
             return new CountedFileIO(file, modes);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java
index 473eaf5ce97bc..fd31ee5e78f56 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TransactionIntegrityWithPrimaryIndexCorruptionTest.java
@@ -181,9 +181,6 @@ class IndexCorruptionFailoverScenario implements FailoverScenario {
                     return hnd;
 
                 if (treeCorruptionPred.apply(hnd, tree)) {
-                    log.info("Created corrupted tree handler [nodeOrder=" + locIgnite.localNode().order() + ", hnd=" + hnd +
-                        ", tree=" + tree + ']');
-
                     PageHandler<Object, BPlusTree.Result> delegate = (PageHandler<Object, BPlusTree.Result>)hnd;
 
                     return new PageHandler<BPlusTree.Get, BPlusTree.Result>() {
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
index 344a1cc0cdbbd..b22b6cd4d022b 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
@@ -89,6 +89,7 @@ public GridCacheTestContext(GridTestKernalContext ctx) throws Exception {
             AffinityTopologyVersion.ZERO,
             true,
             true,
+            false,
             new GridCacheEventManager(),
             new CacheOsStoreManager(null, new CacheConfiguration()),
             new GridCacheEvictionManager(),
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 05bf92457a1c4..ee3e30e6a7472 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -185,9 +185,6 @@ public abstract class GridAbstractTest extends TestCase {
     /** Timestamp for tests. */
     private static long ts = System.currentTimeMillis();
 
-    /** Starting Ignite instance name. */
-    protected static final ThreadLocal<String> startingIgniteInstanceName = new ThreadLocal<>();
-
     /** Force failure flag. */
     private boolean forceFailure;
 
@@ -892,7 +889,7 @@ protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, G
         checkConfiguration(cfg);
 
         if (!isRemoteJvm(igniteInstanceName)) {
-            startingIgniteInstanceName.set(igniteInstanceName);
+            IgniteUtils.setCurrentIgniteName(igniteInstanceName);
 
             try {
                 String cfgProcClsName = System.getProperty(IGNITE_CFG_PREPROCESSOR_CLS);
@@ -928,7 +925,7 @@ protected Ignite startGrid(String igniteInstanceName, IgniteConfiguration cfg, G
                 return node;
             }
             finally {
-                startingIgniteInstanceName.set(null);
+                IgniteUtils.setCurrentIgniteName(null);
             }
         }
         else
diff --git a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java
index 8a28e9eac1c47..ea555a0c583f8 100644
--- a/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java
+++ b/modules/direct-io/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AlignedBuffersDirectFileIOFactory.java
@@ -144,11 +144,6 @@ public AlignedBuffersDirectFileIOFactory(
         return allocate;
     }
 
-    /** {@inheritDoc} */
-    @Override public FileIO create(File file) throws IOException {
-        return create(file, CREATE, READ, WRITE);
-    }
-
     /** {@inheritDoc} */
     @Override public FileIO create(File file, OpenOption... modes) throws IOException {
         if (useBackupFactory)
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index ac7dd4033d375..d82e9051e1b89 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxHistoricalRebalancingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreCacheGroupsTest;
 import org.apache.ignite.internal.processors.cache.persistence.PersistenceDirectoryWarningLoggingTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgniteLogicalRecoveryTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCacheDestroyDuringCheckpointTest;
@@ -86,6 +87,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(IgnitePdsCorruptedIndexTest.class);
 
+        suite.addTestSuite(IgniteLogicalRecoveryTest.class);
+
         return suite;
     }
 }

From 590938f7a32baad3f7e4e193af6f989d00e3eb80 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 2 Nov 2018 17:25:27 +0700
Subject: [PATCH 473/543] IGNITE-10101 Web Console: Fixes and optimizations on
 queries page.

---
 .../backend/app/browsersHandler.js            |   5 +
 .../components/queries-notebook/controller.js | 138 +++++++-----------
 .../queries-notebook/template.tpl.pug         |   4 +-
 .../components/page-queries/notebook.data.js  |   3 -
 .../app/modules/agent/AgentManager.service.js |  52 ++++++-
 5 files changed, 110 insertions(+), 92 deletions(-)

diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js
index d0cd1128406d4..c1c9b683c4d29 100644
--- a/modules/web-console/backend/app/browsersHandler.js
+++ b/modules/web-console/backend/app/browsersHandler.js
@@ -244,6 +244,11 @@ module.exports = {
 
                 this.registerVisorTask('toggleClusterState', internalVisor('misc.VisorChangeGridActiveStateTask'), internalVisor('misc.VisorChangeGridActiveStateTaskArg'));
 
+                this.registerVisorTask('cacheNamesCollectorTask', internalVisor('cache.VisorCacheNamesCollectorTask'), 'java.lang.Void');
+
+                this.registerVisorTask('cacheNodesTask', internalVisor('cache.VisorCacheNodesTask'), 'java.lang.String');
+                this.registerVisorTask('cacheNodesTaskX2', internalVisor('cache.VisorCacheNodesTask'), internalVisor('cache.VisorCacheNodesTaskArg'));
+
                 // Return command result from grid to browser.
                 sock.on('node:visor', (arg, cb) => {
                     const {clusterId, params, credentials} = arg || {};
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
index 3ee08b31a9e56..2cb328ca50f03 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
@@ -14,6 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import _ from 'lodash';
 import {nonEmpty, nonNil} from 'app/utils/lodashMixins';
 import id8 from 'app/utils/id8';
 import 'rxjs/add/operator/mergeMap';
@@ -891,37 +893,13 @@ export class NotebookCtrl {
         /**
          * Update caches list.
          */
-        const _refreshFn = () => {
-            return agentMgr.topology(true)
-                .then((nodes) => {
-                    $scope.caches = _.sortBy(_.reduce(nodes, (cachesAcc, node) => {
-                        _.forEach(node.caches, (cache) => {
-                            let item = _.find(cachesAcc, {name: cache.name});
-
-                            if (_.isNil(item)) {
-                                cache.label = maskCacheName(cache.name, true);
-                                cache.value = cache.name;
-
-                                cache.nodes = [];
-
-                                cachesAcc.push(item = cache);
-                            }
-
-                            item.nodes.push({
-                                nid: node.nodeId.toUpperCase(),
-                                ip: _.head(node.attributes['org.apache.ignite.ips'].split(', ')),
-                                version: node.attributes['org.apache.ignite.build.ver'],
-                                gridName: node.attributes['org.apache.ignite.ignite.name'],
-                                os: `${node.attributes['os.name']} ${node.attributes['os.arch']} ${node.attributes['os.version']}`,
-                                client: node.attributes['org.apache.ignite.cache.client']
-                            });
-                        });
-
-                        return cachesAcc;
-                    }, []), (cache) => cache.label.toLowerCase());
-
-                    // Reset to first cache in case of stopped selected.
-                    const cacheNames = _.map($scope.caches, (cache) => cache.value);
+        const _refreshCaches = () => {
+            return agentMgr.publicCacheNames()
+                .then((cacheNames) => {
+                    $scope.caches = _.sortBy(_.map(cacheNames, (name) => ({
+                        label: maskCacheName(name, true),
+                        value: name
+                    })), (cache) => cache.label.toLowerCase());
 
                     _.forEach($scope.notebook.paragraphs, (paragraph) => {
                         if (!_.includes(cacheNames, paragraph.cacheName))
@@ -936,6 +914,8 @@ export class NotebookCtrl {
 
                         _.forEach($scope.notebook.paragraphs, (paragraph) => $scope.execute(paragraph));
                     }
+
+                    $scope.$applyAsync();
                 })
                 .catch((err) => Messages.showError(err));
         };
@@ -950,7 +930,7 @@ export class NotebookCtrl {
             }).take(1);
 
             const refreshCaches = (period) => {
-                return timer(0, period).exhaustMap(() => _refreshFn()).merge(finishLoading$);
+                return timer(0, period).exhaustMap(() => _refreshCaches()).merge(finishLoading$);
             };
 
             this.refresh$ = awaitClusters$
@@ -1390,29 +1370,48 @@ export class NotebookCtrl {
 
         /**
          * @param {String} name Cache name.
-         * @return {Array.<String>} Nids
+         * @param {Array.<String>} nids Cache name.
+         * @return {Promise<Array.<{nid: string, ip: string, version:string, gridName: string, os: string, client: boolean}>>}
          */
-        const cacheNodes = (name) => {
-            return _.find($scope.caches, {name}).nodes;
+        const cacheNodesModel = (name, nids) => {
+            return agentMgr.topology(true)
+                .then((nodes) =>
+                    _.reduce(nodes, (acc, node) => {
+                        if (_.includes(nids, node.nodeId)) {
+                            acc.push({
+                                nid: node.nodeId.toUpperCase(),
+                                ip: _.head(node.attributes['org.apache.ignite.ips'].split(', ')),
+                                version: node.attributes['org.apache.ignite.build.ver'],
+                                gridName: node.attributes['org.apache.ignite.ignite.name'],
+                                os: `${node.attributes['os.name']} ${node.attributes['os.arch']} ${node.attributes['os.version']}`,
+                                client: node.attributes['org.apache.ignite.cache.client']
+                            });
+                        }
+
+                        return acc;
+                    }, [])
+                );
         };
 
         /**
-         * @param {String} name Cache name.
-         * @param {Boolean} local Local query.
-         * @return {String} Nid
+         * @param {string} name Cache name.
+         * @param {boolean} local Local query.
+         * @return {Promise<string>} Nid
          */
         const _chooseNode = (name, local) => {
             if (_.isEmpty(name))
                 return Promise.resolve(null);
 
-            const nodes = _.filter(cacheNodes(name), (node) => !node.client);
-
-            if (local) {
-                return Nodes.selectNode(nodes, name)
-                    .then((selectedNids) => _.head(selectedNids));
-            }
+            return agentMgr.cacheNodes(name)
+                .then((nids) => {
+                    if (local) {
+                        return cacheNodesModel(name, nids)
+                            .then((nodes) => Nodes.selectNode(nodes, name).catch(() => {}))
+                            .then((selectedNids) => _.head(selectedNids));
+                    }
 
-            return Promise.resolve(nodes[_.random(0, nodes.length - 1)].nid);
+                    return nids[_.random(0, nids.length - 1)];
+                });
         };
 
         const _executeRefresh = (paragraph) => {
@@ -1446,53 +1445,28 @@ export class NotebookCtrl {
             ${query} 
             ) LIMIT ${limitSize}`;
 
-        $scope.nonCollocatedJoinsAvailable = (paragraph) => {
-            const cache = _.find($scope.caches, {name: paragraph.cacheName});
-
-            if (cache)
-                return !!_.find(cache.nodes, (node) => Version.since(node.version, NON_COLLOCATED_JOINS_SINCE));
-
-            return false;
+        $scope.nonCollocatedJoinsAvailable = () => {
+            return Version.since(this.agentMgr.clusterVersion, NON_COLLOCATED_JOINS_SINCE);
         };
 
-        $scope.collocatedJoinsAvailable = (paragraph) => {
-            const cache = _.find($scope.caches, {name: paragraph.cacheName});
-
-            if (cache)
-                return !!_.find(cache.nodes, (node) => Version.since(node.version, ...COLLOCATED_QUERY_SINCE));
-
-            return false;
+        $scope.collocatedJoinsAvailable = () => {
+            return Version.since(this.agentMgr.clusterVersion, ...COLLOCATED_QUERY_SINCE);
         };
 
-        $scope.enforceJoinOrderAvailable = (paragraph) => {
-            const cache = _.find($scope.caches, {name: paragraph.cacheName});
-
-            if (cache)
-                return !!_.find(cache.nodes, (node) => Version.since(node.version, ...ENFORCE_JOIN_SINCE));
-
-            return false;
+        $scope.enforceJoinOrderAvailable = () => {
+            return Version.since(this.agentMgr.clusterVersion, ...ENFORCE_JOIN_SINCE);
         };
 
-        $scope.lazyQueryAvailable = (paragraph) => {
-            const cache = _.find($scope.caches, {name: paragraph.cacheName});
-
-            if (cache)
-                return !!_.find(cache.nodes, (node) => Version.since(node.version, ...LAZY_QUERY_SINCE));
-
-            return false;
+        $scope.lazyQueryAvailable = () => {
+            return Version.since(this.agentMgr.clusterVersion, ...LAZY_QUERY_SINCE);
         };
 
-        $scope.ddlAvailable = (paragraph) => {
-            const cache = _.find($scope.caches, {name: paragraph.cacheName});
-
-            if (cache)
-                return !!_.find(cache.nodes, (node) => Version.since(node.version, ...DDL_SINCE));
-
-            return false;
+        $scope.ddlAvailable = () => {
+            return Version.since(this.agentMgr.clusterVersion, ...DDL_SINCE);
         };
 
         $scope.cacheNameForSql = (paragraph) => {
-            return $scope.ddlAvailable(paragraph) && !paragraph.useAsDefaultSchema ? null : paragraph.cacheName;
+            return $scope.ddlAvailable() && !paragraph.useAsDefaultSchema ? null : paragraph.cacheName;
         };
 
         $scope.execute = (paragraph, local = false) => {
@@ -1893,7 +1867,7 @@ export class NotebookCtrl {
             agentMgr.metadata()
                 .then((metadata) => {
                     $scope.metadata = _.sortBy(_.filter(metadata, (meta) => {
-                        const cache = _.find($scope.caches, { name: meta.cacheName });
+                        const cache = _.find($scope.caches, { value: meta.cacheName });
 
                         if (cache) {
                             meta.name = (cache.sqlSchema || '"' + meta.cacheName + '"') + '.' + meta.typeName;
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
index 781ce5190268f..af673a822d9fb 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
@@ -394,12 +394,12 @@ mixin paragraph-query
                         input.form-control(type='text' st-search='label' placeholder='Filter caches...')
 
                     .queries-notebook-displayed-caches
-                        div(ng-repeat='cache in displayedCaches track by cache.name')
+                        div(ng-repeat='cache in displayedCaches track by cache.value')
                             +form-field__radio({
                                 label: '{{ cache.label }}',
                                 model: 'paragraph.cacheName',
                                 name: '"cache_" + [paragraph.id, $index].join("_")',
-                                value: 'cache.name'
+                                value: 'cache.value'
                             })
 
                     .settings-row
diff --git a/modules/web-console/frontend/app/components/page-queries/notebook.data.js b/modules/web-console/frontend/app/components/page-queries/notebook.data.js
index 6c784bfc3ea11..d9035c2445621 100644
--- a/modules/web-console/frontend/app/components/page-queries/notebook.data.js
+++ b/modules/web-console/frontend/app/components/page-queries/notebook.data.js
@@ -21,7 +21,6 @@ const DEMO_NOTEBOOK = {
     paragraphs: [
         {
             name: 'Query with refresh rate',
-            cacheName: 'CarCache',
             qryType: 'query',
             pageSize: 100,
             limit: 0,
@@ -39,7 +38,6 @@ const DEMO_NOTEBOOK = {
         },
         {
             name: 'Simple query',
-            cacheName: 'CarCache',
             qryType: 'query',
             pageSize: 100,
             limit: 0,
@@ -54,7 +52,6 @@ const DEMO_NOTEBOOK = {
         },
         {
             name: 'Query with aggregates',
-            cacheName: 'ParkingCache',
             qryType: 'query',
             pageSize: 100,
             limit: 0,
diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
index 3a4817e6ba072..9927b10a3d250 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
@@ -45,6 +45,15 @@ const State = {
 const IGNITE_2_0 = '2.0.0';
 const LAZY_QUERY_SINCE = [['2.1.4-p1', '2.2.0'], '2.2.1'];
 const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], ['2.5.1-p13', '2.6.0'], '2.7.0'];
+const COLLECT_BY_CACHE_GROUPS_SINCE = '2.7.0';
+
+/** Reserved cache names */
+const RESERVED_CACHE_NAMES = [
+    'ignite-hadoop-mr-sys-cache',
+    'ignite-sys-cache',
+    'MetaStorage',
+    'TxLog'
+];
 
 // Error codes from o.a.i.internal.processors.restGridRestResponse.java
 
@@ -135,7 +144,7 @@ export default class AgentManager {
     ClusterLoginSrv;
 
     /** @type {String} */
-    clusterVersion = '2.4.0';
+    clusterVersion;
 
     connectionSbj = new BehaviorSubject(new ConnectionState(AgentManager.restoreActiveCluster()));
 
@@ -179,6 +188,8 @@ export default class AgentManager {
         this.Version = Version;
         this.ClusterLoginSrv = ClusterLoginSrv;
 
+        this.clusterVersion = this.Version.webConsole;
+
         let prevCluster;
 
         this.currentCluster$ = this.connectionSbj
@@ -547,12 +558,43 @@ export default class AgentManager {
     }
 
     /**
-     * @param {Boolean} [attr]
-     * @param {Boolean} [mtr]
+     * @param {boolean} [attr] Collect node attributes.
+     * @param {boolean} [mtr] Collect node metrics.
+     * @param {boolean} [caches] Collect node caches descriptors.
      * @returns {Promise}
      */
-    topology(attr = false, mtr = false) {
-        return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr, caches: false});
+    topology(attr = false, mtr = false, caches = false) {
+        return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr, caches});
+    }
+
+    collectCacheNames(nid) {
+        if (this.available(COLLECT_BY_CACHE_GROUPS_SINCE))
+            return this.visorTask('cacheNamesCollectorTask', nid);
+
+        return Promise.resolve({cacheGroupsNotAvailable: true});
+    }
+
+    publicCacheNames() {
+        return this.collectCacheNames()
+            .then((data) => {
+                if (nonEmpty(data.caches))
+                    return _.difference(_.keys(data.caches), RESERVED_CACHE_NAMES);
+
+                return this.topology(false, false, true)
+                    .then((nodes) => {
+                        return _.map(_.uniqBy(_.flatMap(nodes, 'caches'), 'name'), 'name');
+                    });
+            });
+    }
+
+    /**
+     * @param {string} cacheName Cache name.
+     */
+    cacheNodes(cacheName) {
+        if (this.available(IGNITE_2_0))
+            return this.visorTask('cacheNodesTaskX2', null, cacheName);
+
+        return this.visorTask('cacheNodesTask', null, cacheName);
     }
 
     /**

From 3138a258e7396cc740ac6cc4016998412bb72fc2 Mon Sep 17 00:00:00 2001
From: Andrey Novikov <anovikov@gridgain.com>
Date: Fri, 2 Nov 2018 17:25:27 +0700
Subject: [PATCH 474/543] IGNITE-10101 Web Console: Fixes and optimizations on
 queries page.

---
 .../backend/app/browsersHandler.js            |   5 +
 .../components/queries-notebook/controller.js | 138 +++++++-----------
 .../queries-notebook/template.tpl.pug         |   4 +-
 .../components/page-queries/notebook.data.js  |   3 -
 .../app/modules/agent/AgentManager.service.js |  52 ++++++-
 5 files changed, 110 insertions(+), 92 deletions(-)

diff --git a/modules/web-console/backend/app/browsersHandler.js b/modules/web-console/backend/app/browsersHandler.js
index d0cd1128406d4..c1c9b683c4d29 100644
--- a/modules/web-console/backend/app/browsersHandler.js
+++ b/modules/web-console/backend/app/browsersHandler.js
@@ -244,6 +244,11 @@ module.exports = {
 
                 this.registerVisorTask('toggleClusterState', internalVisor('misc.VisorChangeGridActiveStateTask'), internalVisor('misc.VisorChangeGridActiveStateTaskArg'));
 
+                this.registerVisorTask('cacheNamesCollectorTask', internalVisor('cache.VisorCacheNamesCollectorTask'), 'java.lang.Void');
+
+                this.registerVisorTask('cacheNodesTask', internalVisor('cache.VisorCacheNodesTask'), 'java.lang.String');
+                this.registerVisorTask('cacheNodesTaskX2', internalVisor('cache.VisorCacheNodesTask'), internalVisor('cache.VisorCacheNodesTaskArg'));
+
                 // Return command result from grid to browser.
                 sock.on('node:visor', (arg, cb) => {
                     const {clusterId, params, credentials} = arg || {};
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
index 3ee08b31a9e56..2cb328ca50f03 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/controller.js
@@ -14,6 +14,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+import _ from 'lodash';
 import {nonEmpty, nonNil} from 'app/utils/lodashMixins';
 import id8 from 'app/utils/id8';
 import 'rxjs/add/operator/mergeMap';
@@ -891,37 +893,13 @@ export class NotebookCtrl {
         /**
          * Update caches list.
          */
-        const _refreshFn = () => {
-            return agentMgr.topology(true)
-                .then((nodes) => {
-                    $scope.caches = _.sortBy(_.reduce(nodes, (cachesAcc, node) => {
-                        _.forEach(node.caches, (cache) => {
-                            let item = _.find(cachesAcc, {name: cache.name});
-
-                            if (_.isNil(item)) {
-                                cache.label = maskCacheName(cache.name, true);
-                                cache.value = cache.name;
-
-                                cache.nodes = [];
-
-                                cachesAcc.push(item = cache);
-                            }
-
-                            item.nodes.push({
-                                nid: node.nodeId.toUpperCase(),
-                                ip: _.head(node.attributes['org.apache.ignite.ips'].split(', ')),
-                                version: node.attributes['org.apache.ignite.build.ver'],
-                                gridName: node.attributes['org.apache.ignite.ignite.name'],
-                                os: `${node.attributes['os.name']} ${node.attributes['os.arch']} ${node.attributes['os.version']}`,
-                                client: node.attributes['org.apache.ignite.cache.client']
-                            });
-                        });
-
-                        return cachesAcc;
-                    }, []), (cache) => cache.label.toLowerCase());
-
-                    // Reset to first cache in case of stopped selected.
-                    const cacheNames = _.map($scope.caches, (cache) => cache.value);
+        const _refreshCaches = () => {
+            return agentMgr.publicCacheNames()
+                .then((cacheNames) => {
+                    $scope.caches = _.sortBy(_.map(cacheNames, (name) => ({
+                        label: maskCacheName(name, true),
+                        value: name
+                    })), (cache) => cache.label.toLowerCase());
 
                     _.forEach($scope.notebook.paragraphs, (paragraph) => {
                         if (!_.includes(cacheNames, paragraph.cacheName))
@@ -936,6 +914,8 @@ export class NotebookCtrl {
 
                         _.forEach($scope.notebook.paragraphs, (paragraph) => $scope.execute(paragraph));
                     }
+
+                    $scope.$applyAsync();
                 })
                 .catch((err) => Messages.showError(err));
         };
@@ -950,7 +930,7 @@ export class NotebookCtrl {
             }).take(1);
 
             const refreshCaches = (period) => {
-                return timer(0, period).exhaustMap(() => _refreshFn()).merge(finishLoading$);
+                return timer(0, period).exhaustMap(() => _refreshCaches()).merge(finishLoading$);
             };
 
             this.refresh$ = awaitClusters$
@@ -1390,29 +1370,48 @@ export class NotebookCtrl {
 
         /**
          * @param {String} name Cache name.
-         * @return {Array.<String>} Nids
+         * @param {Array.<String>} nids Cache name.
+         * @return {Promise<Array.<{nid: string, ip: string, version:string, gridName: string, os: string, client: boolean}>>}
          */
-        const cacheNodes = (name) => {
-            return _.find($scope.caches, {name}).nodes;
+        const cacheNodesModel = (name, nids) => {
+            return agentMgr.topology(true)
+                .then((nodes) =>
+                    _.reduce(nodes, (acc, node) => {
+                        if (_.includes(nids, node.nodeId)) {
+                            acc.push({
+                                nid: node.nodeId.toUpperCase(),
+                                ip: _.head(node.attributes['org.apache.ignite.ips'].split(', ')),
+                                version: node.attributes['org.apache.ignite.build.ver'],
+                                gridName: node.attributes['org.apache.ignite.ignite.name'],
+                                os: `${node.attributes['os.name']} ${node.attributes['os.arch']} ${node.attributes['os.version']}`,
+                                client: node.attributes['org.apache.ignite.cache.client']
+                            });
+                        }
+
+                        return acc;
+                    }, [])
+                );
         };
 
         /**
-         * @param {String} name Cache name.
-         * @param {Boolean} local Local query.
-         * @return {String} Nid
+         * @param {string} name Cache name.
+         * @param {boolean} local Local query.
+         * @return {Promise<string>} Nid
          */
         const _chooseNode = (name, local) => {
             if (_.isEmpty(name))
                 return Promise.resolve(null);
 
-            const nodes = _.filter(cacheNodes(name), (node) => !node.client);
-
-            if (local) {
-                return Nodes.selectNode(nodes, name)
-                    .then((selectedNids) => _.head(selectedNids));
-            }
+            return agentMgr.cacheNodes(name)
+                .then((nids) => {
+                    if (local) {
+                        return cacheNodesModel(name, nids)
+                            .then((nodes) => Nodes.selectNode(nodes, name).catch(() => {}))
+                            .then((selectedNids) => _.head(selectedNids));
+                    }
 
-            return Promise.resolve(nodes[_.random(0, nodes.length - 1)].nid);
+                    return nids[_.random(0, nids.length - 1)];
+                });
         };
 
         const _executeRefresh = (paragraph) => {
@@ -1446,53 +1445,28 @@ export class NotebookCtrl {
             ${query} 
             ) LIMIT ${limitSize}`;
 
-        $scope.nonCollocatedJoinsAvailable = (paragraph) => {
-            const cache = _.find($scope.caches, {name: paragraph.cacheName});
-
-            if (cache)
-                return !!_.find(cache.nodes, (node) => Version.since(node.version, NON_COLLOCATED_JOINS_SINCE));
-
-            return false;
+        $scope.nonCollocatedJoinsAvailable = () => {
+            return Version.since(this.agentMgr.clusterVersion, NON_COLLOCATED_JOINS_SINCE);
         };
 
-        $scope.collocatedJoinsAvailable = (paragraph) => {
-            const cache = _.find($scope.caches, {name: paragraph.cacheName});
-
-            if (cache)
-                return !!_.find(cache.nodes, (node) => Version.since(node.version, ...COLLOCATED_QUERY_SINCE));
-
-            return false;
+        $scope.collocatedJoinsAvailable = () => {
+            return Version.since(this.agentMgr.clusterVersion, ...COLLOCATED_QUERY_SINCE);
         };
 
-        $scope.enforceJoinOrderAvailable = (paragraph) => {
-            const cache = _.find($scope.caches, {name: paragraph.cacheName});
-
-            if (cache)
-                return !!_.find(cache.nodes, (node) => Version.since(node.version, ...ENFORCE_JOIN_SINCE));
-
-            return false;
+        $scope.enforceJoinOrderAvailable = () => {
+            return Version.since(this.agentMgr.clusterVersion, ...ENFORCE_JOIN_SINCE);
         };
 
-        $scope.lazyQueryAvailable = (paragraph) => {
-            const cache = _.find($scope.caches, {name: paragraph.cacheName});
-
-            if (cache)
-                return !!_.find(cache.nodes, (node) => Version.since(node.version, ...LAZY_QUERY_SINCE));
-
-            return false;
+        $scope.lazyQueryAvailable = () => {
+            return Version.since(this.agentMgr.clusterVersion, ...LAZY_QUERY_SINCE);
         };
 
-        $scope.ddlAvailable = (paragraph) => {
-            const cache = _.find($scope.caches, {name: paragraph.cacheName});
-
-            if (cache)
-                return !!_.find(cache.nodes, (node) => Version.since(node.version, ...DDL_SINCE));
-
-            return false;
+        $scope.ddlAvailable = () => {
+            return Version.since(this.agentMgr.clusterVersion, ...DDL_SINCE);
         };
 
         $scope.cacheNameForSql = (paragraph) => {
-            return $scope.ddlAvailable(paragraph) && !paragraph.useAsDefaultSchema ? null : paragraph.cacheName;
+            return $scope.ddlAvailable() && !paragraph.useAsDefaultSchema ? null : paragraph.cacheName;
         };
 
         $scope.execute = (paragraph, local = false) => {
@@ -1893,7 +1867,7 @@ export class NotebookCtrl {
             agentMgr.metadata()
                 .then((metadata) => {
                     $scope.metadata = _.sortBy(_.filter(metadata, (meta) => {
-                        const cache = _.find($scope.caches, { name: meta.cacheName });
+                        const cache = _.find($scope.caches, { value: meta.cacheName });
 
                         if (cache) {
                             meta.name = (cache.sqlSchema || '"' + meta.cacheName + '"') + '.' + meta.typeName;
diff --git a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
index 781ce5190268f..af673a822d9fb 100644
--- a/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
+++ b/modules/web-console/frontend/app/components/page-queries/components/queries-notebook/template.tpl.pug
@@ -394,12 +394,12 @@ mixin paragraph-query
                         input.form-control(type='text' st-search='label' placeholder='Filter caches...')
 
                     .queries-notebook-displayed-caches
-                        div(ng-repeat='cache in displayedCaches track by cache.name')
+                        div(ng-repeat='cache in displayedCaches track by cache.value')
                             +form-field__radio({
                                 label: '{{ cache.label }}',
                                 model: 'paragraph.cacheName',
                                 name: '"cache_" + [paragraph.id, $index].join("_")',
-                                value: 'cache.name'
+                                value: 'cache.value'
                             })
 
                     .settings-row
diff --git a/modules/web-console/frontend/app/components/page-queries/notebook.data.js b/modules/web-console/frontend/app/components/page-queries/notebook.data.js
index 6c784bfc3ea11..d9035c2445621 100644
--- a/modules/web-console/frontend/app/components/page-queries/notebook.data.js
+++ b/modules/web-console/frontend/app/components/page-queries/notebook.data.js
@@ -21,7 +21,6 @@ const DEMO_NOTEBOOK = {
     paragraphs: [
         {
             name: 'Query with refresh rate',
-            cacheName: 'CarCache',
             qryType: 'query',
             pageSize: 100,
             limit: 0,
@@ -39,7 +38,6 @@ const DEMO_NOTEBOOK = {
         },
         {
             name: 'Simple query',
-            cacheName: 'CarCache',
             qryType: 'query',
             pageSize: 100,
             limit: 0,
@@ -54,7 +52,6 @@ const DEMO_NOTEBOOK = {
         },
         {
             name: 'Query with aggregates',
-            cacheName: 'ParkingCache',
             qryType: 'query',
             pageSize: 100,
             limit: 0,
diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
index 3a4817e6ba072..9927b10a3d250 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
@@ -45,6 +45,15 @@ const State = {
 const IGNITE_2_0 = '2.0.0';
 const LAZY_QUERY_SINCE = [['2.1.4-p1', '2.2.0'], '2.2.1'];
 const COLLOCATED_QUERY_SINCE = [['2.3.5', '2.4.0'], ['2.4.6', '2.5.0'], ['2.5.1-p13', '2.6.0'], '2.7.0'];
+const COLLECT_BY_CACHE_GROUPS_SINCE = '2.7.0';
+
+/** Reserved cache names */
+const RESERVED_CACHE_NAMES = [
+    'ignite-hadoop-mr-sys-cache',
+    'ignite-sys-cache',
+    'MetaStorage',
+    'TxLog'
+];
 
 // Error codes from o.a.i.internal.processors.restGridRestResponse.java
 
@@ -135,7 +144,7 @@ export default class AgentManager {
     ClusterLoginSrv;
 
     /** @type {String} */
-    clusterVersion = '2.4.0';
+    clusterVersion;
 
     connectionSbj = new BehaviorSubject(new ConnectionState(AgentManager.restoreActiveCluster()));
 
@@ -179,6 +188,8 @@ export default class AgentManager {
         this.Version = Version;
         this.ClusterLoginSrv = ClusterLoginSrv;
 
+        this.clusterVersion = this.Version.webConsole;
+
         let prevCluster;
 
         this.currentCluster$ = this.connectionSbj
@@ -547,12 +558,43 @@ export default class AgentManager {
     }
 
     /**
-     * @param {Boolean} [attr]
-     * @param {Boolean} [mtr]
+     * @param {boolean} [attr] Collect node attributes.
+     * @param {boolean} [mtr] Collect node metrics.
+     * @param {boolean} [caches] Collect node caches descriptors.
      * @returns {Promise}
      */
-    topology(attr = false, mtr = false) {
-        return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr, caches: false});
+    topology(attr = false, mtr = false, caches = false) {
+        return this._executeOnCluster('node:rest', {cmd: 'top', attr, mtr, caches});
+    }
+
+    collectCacheNames(nid) {
+        if (this.available(COLLECT_BY_CACHE_GROUPS_SINCE))
+            return this.visorTask('cacheNamesCollectorTask', nid);
+
+        return Promise.resolve({cacheGroupsNotAvailable: true});
+    }
+
+    publicCacheNames() {
+        return this.collectCacheNames()
+            .then((data) => {
+                if (nonEmpty(data.caches))
+                    return _.difference(_.keys(data.caches), RESERVED_CACHE_NAMES);
+
+                return this.topology(false, false, true)
+                    .then((nodes) => {
+                        return _.map(_.uniqBy(_.flatMap(nodes, 'caches'), 'name'), 'name');
+                    });
+            });
+    }
+
+    /**
+     * @param {string} cacheName Cache name.
+     */
+    cacheNodes(cacheName) {
+        if (this.available(IGNITE_2_0))
+            return this.visorTask('cacheNodesTaskX2', null, cacheName);
+
+        return this.visorTask('cacheNodesTask', null, cacheName);
     }
 
     /**

From 7f1324e4fa26564a7ba2ac5f995a63b86f2aef29 Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Fri, 2 Nov 2018 19:39:02 +0300
Subject: [PATCH 475/543] GG-14394 Revert "IGNITE-7165 Re-balancing is
 cancelled if client node joins"

This reverts commit 733293185c8f1c645e74a02701c32f40777bde53.
---
 .../GridCachePartitionExchangeManager.java    |  67 +++-----
 .../processors/cache/GridCachePreloader.java  |  21 +--
 .../cache/GridCachePreloaderAdapter.java      |   6 -
 .../preloader/GridDhtPartitionDemander.java   |  48 +++---
 .../preloader/GridDhtPartitionSupplier.java   |  15 +-
 .../dht/preloader/GridDhtPreloader.java       |  60 +------
 .../GridDhtPreloaderAssignments.java          |   5 +-
 .../ClusterBaselineNodesMetricsSelfTest.java  |   1 +
 .../cache/CacheValidatorMetricsTest.java      |   4 +-
 .../dht/GridCacheDhtPreloadSelfTest.java      |  68 +++++++-
 .../GridCacheRebalancingAsyncSelfTest.java    |   7 +-
 .../GridCacheRebalancingCancelTest.java       | 106 -------------
 ...CacheRebalancingPartitionCountersTest.java |   3 +-
 .../GridCacheRebalancingSyncSelfTest.java     | 149 +++++++++++-------
 ...entAffinityAssignmentWithBaselineTest.java |   4 +-
 ...owHistoricalRebalanceSmallHistoryTest.java |   5 +-
 ...lushMultiNodeFailoverAbstractSelfTest.java |   2 +-
 .../GridMarshallerMappingConsistencyTest.java |   3 +-
 .../testsuites/IgniteCacheTestSuite3.java     |   2 -
 19 files changed, 236 insertions(+), 340 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 73d5b63e5b274..05ebd11251434 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionFullCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
@@ -181,14 +182,6 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     private final ConcurrentMap<AffinityTopologyVersion, AffinityReadyFuture> readyFuts =
         new ConcurrentSkipListMap<>();
 
-    /**
-     * Latest started rebalance topology version but possibly not finished yet. Value {@code NONE}
-     * means that previous rebalance is undefined and the new one should be initiated.
-     *
-     * Should not be used to determine latest rebalanced topology.
-     */
-    private volatile AffinityTopologyVersion rebTopVer = AffinityTopologyVersion.NONE;
-
     /** */
     private GridFutureAdapter<?> reconnectExchangeFut;
 
@@ -869,13 +862,6 @@ public AffinityTopologyVersion readyAffinityVersion() {
         return exchFuts.readyTopVer();
     }
 
-    /**
-     * @return Latest rebalance topology version or {@code NONE} if there is no info.
-     */
-    public AffinityTopologyVersion rebalanceTopologyVersion() {
-        return rebTopVer;
-    }
-
     /**
      * @return Last initialized topology future.
      */
@@ -2788,9 +2774,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (grp.isLocal())
                                     continue;
 
-                                if (grp.preloader().rebalanceRequired(rebTopVer, exchFut))
-                                    rebTopVer = AffinityTopologyVersion.NONE;
-
                                 changed |= grp.topology().afterExchange(exchFut);
                             }
 
@@ -2798,11 +2781,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 refreshPartitions();
                         }
 
-                        // Schedule rebalance if force rebalance or force reassign occurs.
-                        if (exchFut == null)
-                            rebTopVer = AffinityTopologyVersion.NONE;
-
-                        if (!cctx.kernalContext().clientNode() && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
+                        if (!cctx.kernalContext().clientNode()) {
                             assignsMap = new HashMap<>();
 
                             IgniteCacheSnapshotManager snp = cctx.snapshot();
@@ -2819,9 +2798,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                     assigns = grp.preloader().generateAssignments(exchId, exchFut);
 
                                 assignsMap.put(grp.groupId(), assigns);
-
-                                if (resVer == null)
-                                    resVer = grp.topology().readyTopologyVersion();
                             }
                         }
                     }
@@ -2830,7 +2806,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         busy = false;
                     }
 
-                    if (assignsMap != null && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
+                    if (assignsMap != null) {
                         int size = assignsMap.size();
 
                         NavigableMap<Integer, List<Integer>> orderMap = new TreeMap<>();
@@ -2868,6 +2844,11 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (assigns != null)
                                     assignsCancelled |= assigns.cancelled();
 
+                                // Cancels previous rebalance future (in case it's not done yet).
+                                // Sends previous rebalance stopped event (if necessary).
+                                // Creates new rebalance future.
+                                // Sends current rebalance started event (if necessary).
+                                // Finishes cache sync future (on empty assignments).
                                 Runnable cur = grp.preloader().addAssignments(assigns,
                                     forcePreload,
                                     cnt,
@@ -2885,7 +2866,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         if (forcedRebFut != null)
                             forcedRebFut.markInitialized();
 
-                        if (assignsCancelled || hasPendingExchange()) {
+                        if (assignsCancelled) { // Pending exchange.
                             U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
                                 "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
@@ -2893,31 +2874,25 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         else if (r != null) {
                             Collections.reverse(rebList);
 
-                            U.log(log, "Rebalancing scheduled [order=" + rebList +
-                                ", top=" + resVer + ", force=" + (exchFut == null) +
-                                ", evt=" + exchId.discoveryEventName() +
-                                ", node=" + exchId.nodeId() + ']');
+                            U.log(log, "Rebalancing scheduled [order=" + rebList + "]");
 
-                            rebTopVer = resVer;
+                            if (!hasPendingExchange()) {
+                                U.log(log, "Rebalancing started " +
+                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
+                                    ", node=" + exchId.nodeId() + ']');
 
-                            // Start rebalancing cache groups chain. Each group will be rebalanced
-                            // sequentially one by one e.g.:
-                            // ignite-sys-cache -> cacheGroupR1 -> cacheGroupP2 -> cacheGroupR3
-                            r.run();
+                                r.run(); // Starts rebalancing routine.
+                            }
+                            else
+                                U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
+                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
+                                    ", node=" + exchId.nodeId() + ']');
                         }
                         else
                             U.log(log, "Skipping rebalancing (nothing scheduled) " +
-                                "[top=" + resVer + ", force=" + (exchFut == null) +
-                                ", evt=" + exchId.discoveryEventName() +
+                                "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
                     }
-                    else
-                        U.log(log, "Skipping rebalancing (no affinity changes) " +
-                            "[top=" + resVer +
-                            ", rebTopVer=" + rebTopVer +
-                            ", evt=" + exchId.discoveryEventName() +
-                            ", evtNode=" + exchId.nodeId() +
-                            ", client=" + cctx.kernalContext().clientNode() + ']');
                 }
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index d629e94db7e84..5fa7a821c69ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -24,7 +24,6 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.ForceRebalanceExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
@@ -63,17 +62,10 @@ public interface GridCachePreloader {
      */
     public void onInitialExchangeComplete(@Nullable Throwable err);
 
-    /**
-     * @param rebTopVer Previous rebalance topology version or {@code NONE} if there is no info.
-     * @param exchFut Completed exchange future.
-     * @return {@code True} if rebalance should be started (previous will be interrupted).
-     */
-    public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer, GridDhtPartitionsExchangeFuture exchFut);
-
     /**
      * @param exchId Exchange ID.
-     * @param exchFut Completed exchange future. Can be {@code null} if forced or reassigned generation occurs.
-     * @return Partition assignments which will be requested from supplier nodes.
+     * @param exchFut Exchange future.
+     * @return Assignments or {@code null} if detected that there are pending exchanges.
      */
     @Nullable public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      @Nullable GridDhtPartitionsExchangeFuture exchFut);
@@ -82,10 +74,10 @@ public interface GridCachePreloader {
      * Adds assignments to preloader.
      *
      * @param assignments Assignments to add.
-     * @param forcePreload {@code True} if preload requested by {@link ForceRebalanceExchangeTask}.
-     * @param rebalanceId Rebalance id created by exchange thread.
-     * @param next Runnable responsible for cache rebalancing chain.
-     * @param forcedRebFut External future for forced rebalance.
+     * @param forcePreload Force preload flag.
+     * @param rebalanceId Rebalance id.
+     * @param next Runnable responsible for cache rebalancing start.
+     * @param forcedRebFut Rebalance future.
      * @return Rebalancing runnable.
      */
     public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
@@ -122,6 +114,7 @@ public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
      * Future result is {@code false} in case rebalancing cancelled or finished with missed partitions and will be
      * restarted at current or pending topology.
      *
+     * Note that topology change creates new futures and finishes previous.
      */
     public IgniteInternalFuture<Boolean> rebalanceFuture();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index c5e4a817d0418..af916791daa3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -151,12 +151,6 @@ public GridCachePreloaderAdapter(CacheGroupContext grp) {
         // No-op.
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
-        GridDhtPartitionsExchangeFuture exchFut) {
-        return true;
-    }
-
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      GridDhtPartitionsExchangeFuture exchFut) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 7f0d46c3d06f6..e52d8d88a3b84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -235,12 +235,12 @@ else if (log.isDebugEnabled())
 
     /**
      * @param fut Future.
-     * @return {@code True} if rebalance topology version changed by exchange thread or force
-     * reassing exchange occurs, see {@link RebalanceReassignExchangeTask} for details.
+     * @return {@code True} if topology changed.
      */
     private boolean topologyChanged(RebalanceFuture fut) {
-        return !ctx.exchange().rebalanceTopologyVersion().equals(fut.topVer) ||
-            fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
+        return
+            !grp.affinity().lastVersion().equals(fut.topologyVersion()) || // Topology already changed.
+                fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
     }
 
     /**
@@ -253,21 +253,14 @@ void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
     }
 
     /**
-     * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
-     */
-    Collection<UUID> remainingNodes() {
-        return rebalanceFut.remainingNodes();
-    }
-
-    /**
-     * This method initiates new rebalance process from given {@code assignments} by creating new rebalance
-     * future based on them. Cancels previous rebalance future and sends rebalance started event.
-     * In case of delayed rebalance method schedules the new one with configured delay based on {@code lastExchangeFut}.
+     * Initiates new rebalance process from given {@code assignments}.
+     * If previous rebalance is not finished method cancels it.
+     * In case of delayed rebalance method schedules new with configured delay.
      *
-     * @param assignments Assignments to process.
-     * @param force {@code True} if preload request by {@link ForceRebalanceExchangeTask}.
-     * @param rebalanceId Rebalance id generated from exchange thread.
-     * @param next Runnable responsible for cache rebalancing chain.
+     * @param assignments Assignments.
+     * @param force {@code True} if dummy reassign.
+     * @param rebalanceId Rebalance id.
+     * @param next Runnable responsible for cache rebalancing start.
      * @param forcedRebFut External future for forced rebalance.
      * @return Rebalancing runnable.
      */
@@ -447,7 +440,17 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
             if (fut.isDone())
                 return;
 
-            fut.remaining.forEach((key, value) -> value.set1(U.currentTimeMillis()));
+            // Must add all remaining node before send first request, for avoid race between add remaining node and
+            // processing response, see checkIsDone(boolean).
+            for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assignments.entrySet()) {
+                UUID nodeId = e.getKey().id();
+
+                IgniteDhtDemandedPartitionsMap parts = e.getValue().partitions();
+
+                assert parts != null : "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + nodeId + "]";
+
+                fut.remaining.put(nodeId, new T2<>(U.currentTimeMillis(), parts));
+            }
         }
 
         final CacheConfiguration cfg = grp.config();
@@ -1264,13 +1267,6 @@ private void checkIsDone(boolean cancelled) {
             }
         }
 
-        /**
-         * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
-         */
-        private synchronized Collection<UUID> remainingNodes() {
-            return remaining.keySet();
-        }
-
         /**
          *
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 5d9cee99630ac..3b2a03f91c22b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -17,14 +17,12 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
@@ -127,20 +125,19 @@ private static void clearContext(SupplyContext sc, IgniteLogger log) {
     }
 
     /**
-     * Handle topology change and clear supply context map of outdated contexts.
+     * Handles new topology version and clears supply context map of outdated contexts.
+     *
+     * @param topVer Topology version.
      */
-    void onTopologyChanged() {
+    @SuppressWarnings("ConstantConditions")
+    void onTopologyChanged(AffinityTopologyVersion topVer) {
         synchronized (scMap) {
             Iterator<T3<UUID, Integer, AffinityTopologyVersion>> it = scMap.keySet().iterator();
 
-            Collection<UUID> aliveNodes = grp.shared().discovery().aliveServerNodes().stream()
-                .map(ClusterNode::id)
-                .collect(Collectors.toList());
-
             while (it.hasNext()) {
                 T3<UUID, Integer, AffinityTopologyVersion> t = it.next();
 
-                if (!aliveNodes.contains(t.get1())) { // Clear all obsolete contexts.
+                if (topVer.compareTo(t.get3()) > 0) { // Clear all obsolete contexts.
                     clearContext(scMap.get(t), log);
 
                     it.remove();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 034bf72cecf4e..6e3d773abbbe3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -23,7 +23,6 @@
 import java.util.UUID;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -38,6 +37,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -157,67 +157,11 @@ private IgniteCheckedException stopError() {
 
     /** {@inheritDoc} */
     @Override public void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
-        supplier.onTopologyChanged();
+        supplier.onTopologyChanged(lastFut.initialVersion());
 
         demander.onTopologyChanged(lastFut);
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
-        GridDhtPartitionsExchangeFuture exchFut) {
-        if (ctx.kernalContext().clientNode() || rebTopVer.equals(AffinityTopologyVersion.NONE))
-            return false; // No-op.
-
-        if (exchFut.localJoinExchange())
-            return true; // Required, can have outdated updSeq partition counter if node reconnects.
-
-        if (!grp.affinity().cachedVersions().contains(rebTopVer)) {
-            assert rebTopVer.compareTo(grp.localStartVersion()) <= 0 :
-                "Empty hisroty allowed only for newly started cache group [rebTopVer=" + rebTopVer +
-                    ", localStartTopVer=" + grp.localStartVersion() + ']';
-
-            return true; // Required, since no history info available.
-        }
-
-        final IgniteInternalFuture<Boolean> rebFut = rebalanceFuture();
-
-        if (rebFut.isDone() && !rebFut.result())
-            return true; // Required, previous rebalance cancelled.
-
-        final AffinityTopologyVersion exchTopVer = exchFut.context().events().topologyVersion();
-
-        Collection<UUID> aliveNodes = ctx.discovery().aliveServerNodes().stream()
-            .map(ClusterNode::id)
-            .collect(Collectors.toList());
-
-        return assignmentsChanged(rebTopVer, exchTopVer) ||
-            !aliveNodes.containsAll(demander.remainingNodes()); // Some of nodes left before rabalance compelete.
-    }
-
-    /**
-     * @param oldTopVer Previous topology version.
-     * @param newTopVer New topology version to check result.
-     * @return {@code True} if affinity assignments changed between two versions for local node.
-     */
-    private boolean assignmentsChanged(AffinityTopologyVersion oldTopVer, AffinityTopologyVersion newTopVer) {
-        final AffinityAssignment aff = grp.affinity().readyAffinity(newTopVer);
-
-        // We should get affinity assignments based on previous rebalance to calculate difference.
-        // Whole history size described by IGNITE_AFFINITY_HISTORY_SIZE constant.
-        final AffinityAssignment prevAff = grp.affinity().cachedVersions().contains(oldTopVer) ?
-            grp.affinity().cachedAffinity(oldTopVer) : null;
-
-        if (prevAff == null)
-            return false;
-
-        boolean assignsChanged = false;
-
-        for (int p = 0; !assignsChanged && p < grp.affinity().partitions(); p++)
-            assignsChanged |= aff.get(p).contains(ctx.localNode()) != prevAff.get(p).contains(ctx.localNode());
-
-        return assignsChanged;
-    }
-
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId, GridDhtPartitionsExchangeFuture exchFut) {
         assert exchFut == null || exchFut.isDone();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
index 6f988890c6fb3..69dc79b5007c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
@@ -20,6 +20,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
@@ -73,9 +74,9 @@ GridDhtPartitionExchangeId exchangeId() {
     }
 
     /**
-     * @return Topology version based on last {@link GridDhtPartitionTopologyImpl#readyTopVer}.
+     * @return Topology version.
      */
-    public AffinityTopologyVersion topologyVersion() {
+    AffinityTopologyVersion topologyVersion() {
         return topVer;
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
index 46b09ac3b1819..565317720d2a9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
@@ -149,6 +149,7 @@ public void testBaselineNodes() throws Exception {
     private void resetBlt() throws Exception {
         resetBaselineTopology();
 
+        waitForRebalancing();
         awaitPartitionMapExchange();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
index 0b441dd94c0e8..5c601a1bd3bbf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
@@ -103,14 +103,14 @@ public void testCacheValidatorMetrics() throws Exception {
 
         startGrid(2);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         assertCacheStatus(CACHE_NAME_1, true, true);
         assertCacheStatus(CACHE_NAME_2, true, true);
 
         stopGrid(1);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         // Invalid for writing due to invalid topology.
         assertCacheStatus(CACHE_NAME_1, true, false);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
index 82ed95e52531f..7b5cc3e8101dc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
@@ -22,17 +22,20 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.CacheRebalancingEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
@@ -41,7 +44,10 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.P1;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -55,6 +61,9 @@
 import static org.apache.ignite.configuration.CacheConfiguration.DFLT_REBALANCE_BATCH_SIZE;
 import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS;
 import static org.apache.ignite.events.EventType.EVTS_CACHE_REBALANCE;
+import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
@@ -135,6 +144,15 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) {
         return cacheCfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+//        resetLog4j(Level.DEBUG, true,
+//            // Categories.
+//            GridDhtPreloader.class.getPackage().getName(),
+//            GridDhtPartitionTopologyImpl.class.getName(),
+//            GridDhtLocalPartition.class.getName());
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         backups = DFLT_BACKUPS;
@@ -212,6 +230,11 @@ public void testActivePartitionTransferAsyncRandomCoordinator() throws Exception
      */
     private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
+//        resetLog4j(Level.DEBUG, true,
+//            // Categories.
+//            GridDhtPreloader.class.getPackage().getName(),
+//            GridDhtPartitionTopologyImpl.class.getName(),
+//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -250,6 +273,8 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
             info(">>> Finished checking nodes [keyCnt=" + keyCnt + ", nodeCnt=" + nodeCnt + ", grids=" +
                 U.grids2names(ignites) + ']');
 
+            Collection<IgniteFuture<?>> futs = new LinkedList<>();
+
             Ignite last = F.last(ignites);
 
             for (Iterator<Ignite> it = ignites.iterator(); it.hasNext(); ) {
@@ -263,8 +288,21 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
 
                 checkActiveState(ignites);
 
+                final UUID nodeId = g.cluster().localNode().id();
+
                 it.remove();
 
+                futs.add(waitForLocalEvent(last.events(), new P1<Event>() {
+                    @Override public boolean apply(Event e) {
+                        CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
+
+                        ClusterNode node = evt.discoveryNode();
+
+                        return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
+                            (evt.discoveryEventType() == EVT_NODE_LEFT || evt.discoveryEventType() == EVT_NODE_FAILED);
+                    }
+                }, EVT_CACHE_REBALANCE_STOPPED));
+
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
@@ -277,6 +315,14 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
                 awaitPartitionMapExchange(); // Need wait, otherwise test logic is broken if EVT_NODE_FAILED exchanges are merged.
             }
 
+            info("Waiting for preload futures: " + F.view(futs, new IgnitePredicate<IgniteFuture<?>>() {
+                @Override public boolean apply(IgniteFuture<?> fut) {
+                    return !fut.isDone();
+                }
+            }));
+
+            X.waitAll(futs);
+
             info("Finished waiting for preload futures.");
 
             assert last != null;
@@ -456,6 +502,11 @@ private void stopGrids(Iterable<Ignite> grids) {
      */
     private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
+//        resetLog4j(Level.DEBUG, true,
+//            // Categories.
+//            GridDhtPreloader.class.getPackage().getName(),
+//            GridDhtPartitionTopologyImpl.class.getName(),
+//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -507,13 +558,28 @@ private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuf
 
                 it.remove();
 
+                Collection<IgniteFuture<?>> futs = new LinkedList<>();
+
+                for (Ignite gg : ignites)
+                    futs.add(waitForLocalEvent(gg.events(), new P1<Event>() {
+                        @Override public boolean apply(Event e) {
+                            CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
+
+                            ClusterNode node = evt.discoveryNode();
+
+                            return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
+                                (evt.discoveryEventType() == EVT_NODE_LEFT ||
+                                    evt.discoveryEventType() == EVT_NODE_FAILED);
+                        }
+                    }, EVT_CACHE_REBALANCE_STOPPED));
+
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
 
                 info(">>> Waiting for preload futures [leftNode=" + g.name() + ", remaining=" + U.grids2names(ignites) + ']');
 
-                awaitPartitionMapExchange();
+                X.waitAll(futs);
 
                 info("After grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
index 0a8698a597cd8..4ebcd5df35271 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
@@ -17,11 +17,10 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
 
-import java.util.Collections;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
@@ -44,7 +43,7 @@ public class GridCacheRebalancingAsyncSelfTest extends GridCacheRebalancingSyncS
      * @throws Exception Exception.
      */
     public void testNodeFailedAtRebalancing() throws Exception {
-        IgniteEx ignite = startGrid(0);
+        Ignite ignite = startGrid(0);
 
         generateData(ignite, 0, 0);
 
@@ -61,7 +60,7 @@ public void testNodeFailedAtRebalancing() throws Exception {
 
         ((TestTcpDiscoverySpi)grid(1).configuration().getDiscoverySpi()).simulateNodeFailure();
 
-        awaitPartitionMapExchange(false, false, Collections.singletonList(ignite.localNode()));
+        waitForRebalancing(0, 3);
 
         checkSupplyContextMapIsEmpty();
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
deleted file mode 100644
index 3965290480a3d..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
-
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.TestRecordingCommunicationSpi;
-import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
-import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Test cases for checking cancellation rebalancing process if some events occurs.
- */
-public class GridCacheRebalancingCancelTest extends GridCommonAbstractTest {
-    /** */
-    private static final String DHT_PARTITIONED_CACHE = "cacheP";
-
-    /** */
-    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
-        IgniteConfiguration dfltCfg = super.getConfiguration(igniteInstanceName);
-
-        ((TcpDiscoverySpi)dfltCfg.getDiscoverySpi()).setIpFinder(ipFinder);
-
-        dfltCfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
-
-        return dfltCfg;
-    }
-
-    /**
-     * Test rebalance not cancelled when client node join to cluster.
-     *
-     * @throws Exception Exception.
-     */
-    public void testClientNodeJoinAtRebalancing() throws Exception {
-        final IgniteEx ignite0 = startGrid(0);
-
-        IgniteCache<Integer, Integer> cache = ignite0.createCache(
-            new CacheConfiguration<Integer, Integer>(DHT_PARTITIONED_CACHE)
-                .setCacheMode(CacheMode.PARTITIONED)
-                .setRebalanceMode(CacheRebalanceMode.ASYNC)
-                .setBackups(1)
-                .setRebalanceOrder(2)
-                .setAffinity(new RendezvousAffinityFunction(false)));
-
-        for (int i = 0; i < 2048; i++)
-            cache.put(i, i);
-
-        TestRecordingCommunicationSpi.spi(ignite0)
-            .blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
-                @Override public boolean apply(ClusterNode node, Message msg) {
-                    return (msg instanceof GridDhtPartitionSupplyMessage)
-                        && ((GridCacheGroupIdMessage)msg).groupId() == groupIdForCache(ignite0, DHT_PARTITIONED_CACHE);
-                }
-            });
-
-        final IgniteEx ignite1 = startGrid(1);
-
-        TestRecordingCommunicationSpi.spi(ignite0).waitForBlocked();
-
-        GridDhtPartitionDemander.RebalanceFuture fut = (GridDhtPartitionDemander.RebalanceFuture)ignite1.context().
-            cache().internalCache(DHT_PARTITIONED_CACHE).preloader().rebalanceFuture();
-
-        String igniteClntName = getTestIgniteInstanceName(2);
-
-        startGrid(igniteClntName, optimize(getConfiguration(igniteClntName).setClientMode(true)));
-
-        // Resend delayed rebalance messages.
-        TestRecordingCommunicationSpi.spi(ignite0).stopBlock(true);
-
-        awaitPartitionMapExchange();
-
-        // Previous rebalance future should not be cancelled.
-        assertTrue(fut.result());
-    }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
index 258e36e531322..fa4b655fb41bd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
@@ -141,8 +141,7 @@ else if ("index.bin".equals(f.getName())) {
         assertTrue(primaryRemoved);
 
         ignite.cluster().active(true);
-
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         List<String> issues = new ArrayList<>();
         HashMap<Integer, Long> partMap = new HashMap<>();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
index 8b88befee5281..4d03d7dc7e281 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
@@ -21,9 +21,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cluster.ClusterNode;
@@ -41,13 +42,11 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
-import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -69,9 +68,6 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private static final int TEST_SIZE = 100_000;
 
-    /** */
-    private static final long TOPOLOGY_STILLNESS_TIME = 30_000L;
-
     /** partitioned cache name. */
     protected static final String CACHE_NAME_DHT_PARTITIONED = "cacheP";
 
@@ -93,11 +89,11 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private volatile boolean concurrentStartFinished3;
 
-    /**
-     * Time in milliseconds of last received {@link GridDhtPartitionsSingleMessage}
-     * or {@link GridDhtPartitionsFullMessage} using {@link CollectingCommunicationSpi}.
-     */
-    private static volatile long lastPartMsgTime;
+    /** */
+    private volatile boolean record;
+
+    /** */
+    private final ConcurrentHashMap<Class, AtomicInteger> map = new ConcurrentHashMap<>();
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -106,7 +102,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setIpFinder(ipFinder);
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setForceServerMode(true);
 
-        TcpCommunicationSpi commSpi = new CollectingCommunicationSpi();
+        TcpCommunicationSpi commSpi = new CountingCommunicationSpi();
 
         commSpi.setLocalPort(GridTestUtils.getNextCommPort(getClass()));
         commSpi.setTcpNoDelay(true);
@@ -236,35 +232,47 @@ public void testSimpleRebalancing() throws Exception {
 
         startGrid(1);
 
+        int waitMinorVer = ignite.configuration().isLateAffinityAssignment() ? 1 : 0;
+
+        waitForRebalancing(0, 2, waitMinorVer);
+        waitForRebalancing(1, 2, waitMinorVer);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         stopGrid(0);
 
+        waitForRebalancing(1, 3);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         startGrid(2);
 
+        waitForRebalancing(1, 4, waitMinorVer);
+        waitForRebalancing(2, 4, waitMinorVer);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         stopGrid(2);
 
+        waitForRebalancing(1, 5);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         long spend = (System.currentTimeMillis() - start) / 1000;
 
@@ -323,10 +331,13 @@ public void testLoadRebalancing() throws Exception {
 
         startGrid(4);
 
-        awaitPartitionMapExchange(true, true, null);
+        waitForRebalancing(3, 6);
+        waitForRebalancing(4, 6);
 
         concurrentStartFinished = true;
 
+        awaitPartitionMapExchange(true, true, null);
+
         checkSupplyContextMapIsEmpty();
 
         t1.join();
@@ -431,29 +442,27 @@ protected void checkPartitionMapExchangeFinished() {
     }
 
     /**
-     * Method checks for {@link GridDhtPartitionsSingleMessage} or {@link GridDhtPartitionsFullMessage}
-     * not received within {@code TOPOLOGY_STILLNESS_TIME} bound.
-     *
      * @throws Exception If failed.
      */
-    protected void awaitPartitionMessagesAbsent() throws Exception {
-        log.info("Checking GridDhtPartitions*Message absent (it will take up to " +
-            TOPOLOGY_STILLNESS_TIME + " ms) ... ");
-
-        // Start waiting new messages from current point of time.
-        lastPartMsgTime = U.currentTimeMillis();
-
-        assertTrue("Should not have partition Single or Full messages within bound " +
-                TOPOLOGY_STILLNESS_TIME + " ms.",
-            GridTestUtils.waitForCondition(
-                new GridAbsPredicateX() {
-                    @Override public boolean applyx() {
-                        return lastPartMsgTime + TOPOLOGY_STILLNESS_TIME < U.currentTimeMillis();
-                    }
-                },
-                2 * TOPOLOGY_STILLNESS_TIME // 30 sec to gain stable topology and 30 sec of silence.
-            )
-        );
+    protected void checkPartitionMapMessagesAbsent() throws Exception {
+        map.clear();
+
+        record = true;
+
+        log.info("Checking GridDhtPartitions*Message absent (it will take 30 SECONDS) ... ");
+
+        U.sleep(30_000);
+
+        record = false;
+
+        AtomicInteger iF = map.get(GridDhtPartitionsFullMessage.class);
+        AtomicInteger iS = map.get(GridDhtPartitionsSingleMessage.class);
+
+        Integer fullMap = iF != null ? iF.get() : null;
+        Integer singleMap = iS != null ? iS.get() : null;
+
+        assertTrue("Unexpected full map messages: " + fullMap, fullMap == null || fullMap.equals(1)); // 1 message can be sent right after all checks passed.
+        assertNull("Unexpected single map messages", singleMap);
     }
 
     /** {@inheritDoc} */
@@ -486,7 +495,11 @@ public void testComplexRebalancing() throws Exception {
                     while (!concurrentStartFinished2)
                         U.sleep(10);
 
-                    awaitPartitionMapExchange();
+                    waitForRebalancing(0, 5, 0);
+                    waitForRebalancing(1, 5, 0);
+                    waitForRebalancing(2, 5, 0);
+                    waitForRebalancing(3, 5, 0);
+                    waitForRebalancing(4, 5, 0);
 
                     //New cache should start rebalancing.
                     CacheConfiguration<Integer, Integer> cacheRCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
@@ -539,6 +552,12 @@ public void testComplexRebalancing() throws Exception {
         t2.join();
         t3.join();
 
+        waitForRebalancing(0, 5, 1);
+        waitForRebalancing(1, 5, 1);
+        waitForRebalancing(2, 5, 1);
+        waitForRebalancing(3, 5, 1);
+        waitForRebalancing(4, 5, 1);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
@@ -558,23 +577,35 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(1);
 
+        waitForRebalancing(0, 6);
+        waitForRebalancing(2, 6);
+        waitForRebalancing(3, 6);
+        waitForRebalancing(4, 6);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(0);
 
+        waitForRebalancing(2, 7);
+        waitForRebalancing(3, 7);
+        waitForRebalancing(4, 7);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(2);
 
+        waitForRebalancing(3, 8);
+        waitForRebalancing(4, 8);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         checkSupplyContextMapIsEmpty();
 
@@ -582,7 +613,7 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(3);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing(4, 9);
 
         checkSupplyContextMapIsEmpty();
 
@@ -603,26 +634,36 @@ public void testComplexRebalancing() throws Exception {
     /**
      *
      */
-    private static class CollectingCommunicationSpi extends TcpCommunicationSpi {
-        /** */
-        @LoggerResource
-        private IgniteLogger log;
-
+    private class CountingCommunicationSpi extends TcpCommunicationSpi {
         /** {@inheritDoc} */
         @Override public void sendMessage(final ClusterNode node, final Message msg,
             final IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
             final Object msg0 = ((GridIoMessage)msg).message();
 
-            if (msg0 instanceof GridDhtPartitionsSingleMessage ||
-                msg0 instanceof GridDhtPartitionsFullMessage) {
-                lastPartMsgTime = U.currentTimeMillis();
-
-                log.info("Last seen time of GridDhtPartitionsSingleMessage or GridDhtPartitionsFullMessage updated " +
-                    "[lastPartMsgTime=" + lastPartMsgTime +
-                    ", node=" + node.id() + ']');
-            }
+            recordMessage(msg0);
 
             super.sendMessage(node, msg, ackC);
         }
+
+        /**
+         * @param msg Message.
+         */
+        private void recordMessage(Object msg) {
+            if (record) {
+                Class id = msg.getClass();
+
+                AtomicInteger ai = map.get(id);
+
+                if (ai == null) {
+                    ai = new AtomicInteger();
+
+                    AtomicInteger oldAi = map.putIfAbsent(id, ai);
+
+                    (oldAi != null ? oldAi : ai).incrementAndGet();
+                }
+                else
+                    ai.incrementAndGet();
+            }
+        }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
index ce84156e56dca..15ec41557dfba 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
@@ -383,7 +383,7 @@ public void testRejoinWithCleanLfs() throws Exception {
         startGrid("flaky");
 
         System.out.println("### Starting rebalancing after flaky node join");
-        awaitPartitionMapExchange();
+        waitForRebalancing();
         System.out.println("### Rebalancing is finished after flaky node join");
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
@@ -689,7 +689,7 @@ private void tryChangeBaselineDown(
         ig0.cluster().setBaselineTopology(fullBlt.subList(0, newBaselineSize));
 
         System.out.println("### Starting rebalancing after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
-        awaitPartitionMapExchange();
+        waitForRebalancing();
         System.out.println("### Rebalancing is finished after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
index bd1a1a9cc3575..e6ec8a37f6538 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
@@ -162,8 +162,7 @@ public void testReservation() throws Exception {
 
         SUPPLY_MESSAGE_LATCH.get().countDown();
 
-        // Partition is OWNING on grid(0) and grid(1)
-        awaitPartitionMapExchange();
+        waitForRebalancing(); // Partition is OWNING on grid(0) and grid(1)
 
         for (int i = 0; i < 2; i++) {
             for (int j = 0; i < 500; i++)
@@ -184,7 +183,7 @@ public void testReservation() throws Exception {
 
         startGrid(0);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         assertEquals(2, grid(1).context().discovery().aliveServerNodes().size());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index 07fdb71fca68a..a24e6d5e0d738 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -192,7 +192,7 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
 
                     grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
 
-                    awaitPartitionMapExchange();
+                    waitForRebalancing();
                 }
                 catch (Throwable expected) {
                     // There can be any exception. Do nothing.
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index ba39e366e9f85..efb4fd511c4aa 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -120,8 +120,7 @@ public void testMappingsPersistedOnJoin() throws Exception {
         c1.put(k, new DummyObject(k));
 
         startGrid(2);
-
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         stopAllGrids();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index 5e94052f25013..55436d6da943b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -53,7 +53,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingAsyncSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingCancelTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
@@ -153,7 +152,6 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
         suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
         suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
-        suite.addTestSuite(GridCacheRebalancingCancelTest.class);
 
         // Test for byte array value special case.
         suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);

From a8a18a3b901dc82044141fe0980f73732fbfda88 Mon Sep 17 00:00:00 2001
From: Sergey Kosarev <skosarev@gridgain.com>
Date: Fri, 2 Nov 2018 19:39:02 +0300
Subject: [PATCH 476/543] GG-14394 Revert "IGNITE-7165 Re-balancing is
 cancelled if client node joins"

This reverts commit 733293185c8f1c645e74a02701c32f40777bde53.

(cherry picked from commit 7f1324e4fa26564a7ba2ac5f995a63b86f2aef29)
---
 .../GridCachePartitionExchangeManager.java    |  67 +++-----
 .../processors/cache/GridCachePreloader.java  |  21 +--
 .../cache/GridCachePreloaderAdapter.java      |   6 -
 .../preloader/GridDhtPartitionDemander.java   |  48 +++---
 .../preloader/GridDhtPartitionSupplier.java   |  15 +-
 .../dht/preloader/GridDhtPreloader.java       |  60 +------
 .../GridDhtPreloaderAssignments.java          |   5 +-
 .../ClusterBaselineNodesMetricsSelfTest.java  |   1 +
 .../cache/CacheValidatorMetricsTest.java      |   4 +-
 .../dht/GridCacheDhtPreloadSelfTest.java      |  68 +++++++-
 .../GridCacheRebalancingAsyncSelfTest.java    |   7 +-
 .../GridCacheRebalancingCancelTest.java       | 106 -------------
 ...CacheRebalancingPartitionCountersTest.java |   3 +-
 .../GridCacheRebalancingSyncSelfTest.java     | 149 +++++++++++-------
 ...entAffinityAssignmentWithBaselineTest.java |   4 +-
 ...owHistoricalRebalanceSmallHistoryTest.java |   5 +-
 ...lushMultiNodeFailoverAbstractSelfTest.java |   2 +-
 .../GridMarshallerMappingConsistencyTest.java |   3 +-
 .../testsuites/IgniteCacheTestSuite3.java     |   2 -
 19 files changed, 236 insertions(+), 340 deletions(-)
 delete mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 73d5b63e5b274..05ebd11251434 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -66,6 +66,7 @@
 import org.apache.ignite.internal.managers.eventstorage.DiscoveryEventListener;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionFullCountersMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap;
@@ -181,14 +182,6 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     private final ConcurrentMap<AffinityTopologyVersion, AffinityReadyFuture> readyFuts =
         new ConcurrentSkipListMap<>();
 
-    /**
-     * Latest started rebalance topology version but possibly not finished yet. Value {@code NONE}
-     * means that previous rebalance is undefined and the new one should be initiated.
-     *
-     * Should not be used to determine latest rebalanced topology.
-     */
-    private volatile AffinityTopologyVersion rebTopVer = AffinityTopologyVersion.NONE;
-
     /** */
     private GridFutureAdapter<?> reconnectExchangeFut;
 
@@ -869,13 +862,6 @@ public AffinityTopologyVersion readyAffinityVersion() {
         return exchFuts.readyTopVer();
     }
 
-    /**
-     * @return Latest rebalance topology version or {@code NONE} if there is no info.
-     */
-    public AffinityTopologyVersion rebalanceTopologyVersion() {
-        return rebTopVer;
-    }
-
     /**
      * @return Last initialized topology future.
      */
@@ -2788,9 +2774,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (grp.isLocal())
                                     continue;
 
-                                if (grp.preloader().rebalanceRequired(rebTopVer, exchFut))
-                                    rebTopVer = AffinityTopologyVersion.NONE;
-
                                 changed |= grp.topology().afterExchange(exchFut);
                             }
 
@@ -2798,11 +2781,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 refreshPartitions();
                         }
 
-                        // Schedule rebalance if force rebalance or force reassign occurs.
-                        if (exchFut == null)
-                            rebTopVer = AffinityTopologyVersion.NONE;
-
-                        if (!cctx.kernalContext().clientNode() && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
+                        if (!cctx.kernalContext().clientNode()) {
                             assignsMap = new HashMap<>();
 
                             IgniteCacheSnapshotManager snp = cctx.snapshot();
@@ -2819,9 +2798,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                     assigns = grp.preloader().generateAssignments(exchId, exchFut);
 
                                 assignsMap.put(grp.groupId(), assigns);
-
-                                if (resVer == null)
-                                    resVer = grp.topology().readyTopologyVersion();
                             }
                         }
                     }
@@ -2830,7 +2806,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         busy = false;
                     }
 
-                    if (assignsMap != null && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
+                    if (assignsMap != null) {
                         int size = assignsMap.size();
 
                         NavigableMap<Integer, List<Integer>> orderMap = new TreeMap<>();
@@ -2868,6 +2844,11 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (assigns != null)
                                     assignsCancelled |= assigns.cancelled();
 
+                                // Cancels previous rebalance future (in case it's not done yet).
+                                // Sends previous rebalance stopped event (if necessary).
+                                // Creates new rebalance future.
+                                // Sends current rebalance started event (if necessary).
+                                // Finishes cache sync future (on empty assignments).
                                 Runnable cur = grp.preloader().addAssignments(assigns,
                                     forcePreload,
                                     cnt,
@@ -2885,7 +2866,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         if (forcedRebFut != null)
                             forcedRebFut.markInitialized();
 
-                        if (assignsCancelled || hasPendingExchange()) {
+                        if (assignsCancelled) { // Pending exchange.
                             U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
                                 "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
@@ -2893,31 +2874,25 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         else if (r != null) {
                             Collections.reverse(rebList);
 
-                            U.log(log, "Rebalancing scheduled [order=" + rebList +
-                                ", top=" + resVer + ", force=" + (exchFut == null) +
-                                ", evt=" + exchId.discoveryEventName() +
-                                ", node=" + exchId.nodeId() + ']');
+                            U.log(log, "Rebalancing scheduled [order=" + rebList + "]");
 
-                            rebTopVer = resVer;
+                            if (!hasPendingExchange()) {
+                                U.log(log, "Rebalancing started " +
+                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
+                                    ", node=" + exchId.nodeId() + ']');
 
-                            // Start rebalancing cache groups chain. Each group will be rebalanced
-                            // sequentially one by one e.g.:
-                            // ignite-sys-cache -> cacheGroupR1 -> cacheGroupP2 -> cacheGroupR3
-                            r.run();
+                                r.run(); // Starts rebalancing routine.
+                            }
+                            else
+                                U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
+                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
+                                    ", node=" + exchId.nodeId() + ']');
                         }
                         else
                             U.log(log, "Skipping rebalancing (nothing scheduled) " +
-                                "[top=" + resVer + ", force=" + (exchFut == null) +
-                                ", evt=" + exchId.discoveryEventName() +
+                                "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
                     }
-                    else
-                        U.log(log, "Skipping rebalancing (no affinity changes) " +
-                            "[top=" + resVer +
-                            ", rebTopVer=" + rebTopVer +
-                            ", evt=" + exchId.discoveryEventName() +
-                            ", evtNode=" + exchId.nodeId() +
-                            ", client=" + cctx.kernalContext().clientNode() + ']');
                 }
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index d629e94db7e84..5fa7a821c69ef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -24,7 +24,6 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.ForceRebalanceExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
@@ -63,17 +62,10 @@ public interface GridCachePreloader {
      */
     public void onInitialExchangeComplete(@Nullable Throwable err);
 
-    /**
-     * @param rebTopVer Previous rebalance topology version or {@code NONE} if there is no info.
-     * @param exchFut Completed exchange future.
-     * @return {@code True} if rebalance should be started (previous will be interrupted).
-     */
-    public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer, GridDhtPartitionsExchangeFuture exchFut);
-
     /**
      * @param exchId Exchange ID.
-     * @param exchFut Completed exchange future. Can be {@code null} if forced or reassigned generation occurs.
-     * @return Partition assignments which will be requested from supplier nodes.
+     * @param exchFut Exchange future.
+     * @return Assignments or {@code null} if detected that there are pending exchanges.
      */
     @Nullable public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      @Nullable GridDhtPartitionsExchangeFuture exchFut);
@@ -82,10 +74,10 @@ public interface GridCachePreloader {
      * Adds assignments to preloader.
      *
      * @param assignments Assignments to add.
-     * @param forcePreload {@code True} if preload requested by {@link ForceRebalanceExchangeTask}.
-     * @param rebalanceId Rebalance id created by exchange thread.
-     * @param next Runnable responsible for cache rebalancing chain.
-     * @param forcedRebFut External future for forced rebalance.
+     * @param forcePreload Force preload flag.
+     * @param rebalanceId Rebalance id.
+     * @param next Runnable responsible for cache rebalancing start.
+     * @param forcedRebFut Rebalance future.
      * @return Rebalancing runnable.
      */
     public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
@@ -122,6 +114,7 @@ public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
      * Future result is {@code false} in case rebalancing cancelled or finished with missed partitions and will be
      * restarted at current or pending topology.
      *
+     * Note that topology change creates new futures and finishes previous.
      */
     public IgniteInternalFuture<Boolean> rebalanceFuture();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index c5e4a817d0418..af916791daa3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -151,12 +151,6 @@ public GridCachePreloaderAdapter(CacheGroupContext grp) {
         // No-op.
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
-        GridDhtPartitionsExchangeFuture exchFut) {
-        return true;
-    }
-
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      GridDhtPartitionsExchangeFuture exchFut) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 7f0d46c3d06f6..e52d8d88a3b84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -235,12 +235,12 @@ else if (log.isDebugEnabled())
 
     /**
      * @param fut Future.
-     * @return {@code True} if rebalance topology version changed by exchange thread or force
-     * reassing exchange occurs, see {@link RebalanceReassignExchangeTask} for details.
+     * @return {@code True} if topology changed.
      */
     private boolean topologyChanged(RebalanceFuture fut) {
-        return !ctx.exchange().rebalanceTopologyVersion().equals(fut.topVer) ||
-            fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
+        return
+            !grp.affinity().lastVersion().equals(fut.topologyVersion()) || // Topology already changed.
+                fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
     }
 
     /**
@@ -253,21 +253,14 @@ void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
     }
 
     /**
-     * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
-     */
-    Collection<UUID> remainingNodes() {
-        return rebalanceFut.remainingNodes();
-    }
-
-    /**
-     * This method initiates new rebalance process from given {@code assignments} by creating new rebalance
-     * future based on them. Cancels previous rebalance future and sends rebalance started event.
-     * In case of delayed rebalance method schedules the new one with configured delay based on {@code lastExchangeFut}.
+     * Initiates new rebalance process from given {@code assignments}.
+     * If previous rebalance is not finished method cancels it.
+     * In case of delayed rebalance method schedules new with configured delay.
      *
-     * @param assignments Assignments to process.
-     * @param force {@code True} if preload request by {@link ForceRebalanceExchangeTask}.
-     * @param rebalanceId Rebalance id generated from exchange thread.
-     * @param next Runnable responsible for cache rebalancing chain.
+     * @param assignments Assignments.
+     * @param force {@code True} if dummy reassign.
+     * @param rebalanceId Rebalance id.
+     * @param next Runnable responsible for cache rebalancing start.
      * @param forcedRebFut External future for forced rebalance.
      * @return Rebalancing runnable.
      */
@@ -447,7 +440,17 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
             if (fut.isDone())
                 return;
 
-            fut.remaining.forEach((key, value) -> value.set1(U.currentTimeMillis()));
+            // Must add all remaining node before send first request, for avoid race between add remaining node and
+            // processing response, see checkIsDone(boolean).
+            for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assignments.entrySet()) {
+                UUID nodeId = e.getKey().id();
+
+                IgniteDhtDemandedPartitionsMap parts = e.getValue().partitions();
+
+                assert parts != null : "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + nodeId + "]";
+
+                fut.remaining.put(nodeId, new T2<>(U.currentTimeMillis(), parts));
+            }
         }
 
         final CacheConfiguration cfg = grp.config();
@@ -1264,13 +1267,6 @@ private void checkIsDone(boolean cancelled) {
             }
         }
 
-        /**
-         * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
-         */
-        private synchronized Collection<UUID> remainingNodes() {
-            return remaining.keySet();
-        }
-
         /**
          *
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 5d9cee99630ac..3b2a03f91c22b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -17,14 +17,12 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
-import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
@@ -127,20 +125,19 @@ private static void clearContext(SupplyContext sc, IgniteLogger log) {
     }
 
     /**
-     * Handle topology change and clear supply context map of outdated contexts.
+     * Handles new topology version and clears supply context map of outdated contexts.
+     *
+     * @param topVer Topology version.
      */
-    void onTopologyChanged() {
+    @SuppressWarnings("ConstantConditions")
+    void onTopologyChanged(AffinityTopologyVersion topVer) {
         synchronized (scMap) {
             Iterator<T3<UUID, Integer, AffinityTopologyVersion>> it = scMap.keySet().iterator();
 
-            Collection<UUID> aliveNodes = grp.shared().discovery().aliveServerNodes().stream()
-                .map(ClusterNode::id)
-                .collect(Collectors.toList());
-
             while (it.hasNext()) {
                 T3<UUID, Integer, AffinityTopologyVersion> t = it.next();
 
-                if (!aliveNodes.contains(t.get1())) { // Clear all obsolete contexts.
+                if (topVer.compareTo(t.get3()) > 0) { // Clear all obsolete contexts.
                     clearContext(scMap.get(t), log);
 
                     it.remove();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 034bf72cecf4e..6e3d773abbbe3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -23,7 +23,6 @@
 import java.util.UUID;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -38,6 +37,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -157,67 +157,11 @@ private IgniteCheckedException stopError() {
 
     /** {@inheritDoc} */
     @Override public void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
-        supplier.onTopologyChanged();
+        supplier.onTopologyChanged(lastFut.initialVersion());
 
         demander.onTopologyChanged(lastFut);
     }
 
-    /** {@inheritDoc} */
-    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
-        GridDhtPartitionsExchangeFuture exchFut) {
-        if (ctx.kernalContext().clientNode() || rebTopVer.equals(AffinityTopologyVersion.NONE))
-            return false; // No-op.
-
-        if (exchFut.localJoinExchange())
-            return true; // Required, can have outdated updSeq partition counter if node reconnects.
-
-        if (!grp.affinity().cachedVersions().contains(rebTopVer)) {
-            assert rebTopVer.compareTo(grp.localStartVersion()) <= 0 :
-                "Empty hisroty allowed only for newly started cache group [rebTopVer=" + rebTopVer +
-                    ", localStartTopVer=" + grp.localStartVersion() + ']';
-
-            return true; // Required, since no history info available.
-        }
-
-        final IgniteInternalFuture<Boolean> rebFut = rebalanceFuture();
-
-        if (rebFut.isDone() && !rebFut.result())
-            return true; // Required, previous rebalance cancelled.
-
-        final AffinityTopologyVersion exchTopVer = exchFut.context().events().topologyVersion();
-
-        Collection<UUID> aliveNodes = ctx.discovery().aliveServerNodes().stream()
-            .map(ClusterNode::id)
-            .collect(Collectors.toList());
-
-        return assignmentsChanged(rebTopVer, exchTopVer) ||
-            !aliveNodes.containsAll(demander.remainingNodes()); // Some of nodes left before rabalance compelete.
-    }
-
-    /**
-     * @param oldTopVer Previous topology version.
-     * @param newTopVer New topology version to check result.
-     * @return {@code True} if affinity assignments changed between two versions for local node.
-     */
-    private boolean assignmentsChanged(AffinityTopologyVersion oldTopVer, AffinityTopologyVersion newTopVer) {
-        final AffinityAssignment aff = grp.affinity().readyAffinity(newTopVer);
-
-        // We should get affinity assignments based on previous rebalance to calculate difference.
-        // Whole history size described by IGNITE_AFFINITY_HISTORY_SIZE constant.
-        final AffinityAssignment prevAff = grp.affinity().cachedVersions().contains(oldTopVer) ?
-            grp.affinity().cachedAffinity(oldTopVer) : null;
-
-        if (prevAff == null)
-            return false;
-
-        boolean assignsChanged = false;
-
-        for (int p = 0; !assignsChanged && p < grp.affinity().partitions(); p++)
-            assignsChanged |= aff.get(p).contains(ctx.localNode()) != prevAff.get(p).contains(ctx.localNode());
-
-        return assignsChanged;
-    }
-
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId, GridDhtPartitionsExchangeFuture exchFut) {
         assert exchFut == null || exchFut.isDone();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
index 6f988890c6fb3..69dc79b5007c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
@@ -20,6 +20,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
@@ -73,9 +74,9 @@ GridDhtPartitionExchangeId exchangeId() {
     }
 
     /**
-     * @return Topology version based on last {@link GridDhtPartitionTopologyImpl#readyTopVer}.
+     * @return Topology version.
      */
-    public AffinityTopologyVersion topologyVersion() {
+    AffinityTopologyVersion topologyVersion() {
         return topVer;
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
index 46b09ac3b1819..565317720d2a9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
@@ -149,6 +149,7 @@ public void testBaselineNodes() throws Exception {
     private void resetBlt() throws Exception {
         resetBaselineTopology();
 
+        waitForRebalancing();
         awaitPartitionMapExchange();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
index 0b441dd94c0e8..5c601a1bd3bbf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
@@ -103,14 +103,14 @@ public void testCacheValidatorMetrics() throws Exception {
 
         startGrid(2);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         assertCacheStatus(CACHE_NAME_1, true, true);
         assertCacheStatus(CACHE_NAME_2, true, true);
 
         stopGrid(1);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         // Invalid for writing due to invalid topology.
         assertCacheStatus(CACHE_NAME_1, true, false);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
index 82ed95e52531f..7b5cc3e8101dc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
@@ -22,17 +22,20 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.events.CacheRebalancingEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
@@ -41,7 +44,10 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.P1;
+import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -55,6 +61,9 @@
 import static org.apache.ignite.configuration.CacheConfiguration.DFLT_REBALANCE_BATCH_SIZE;
 import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS;
 import static org.apache.ignite.events.EventType.EVTS_CACHE_REBALANCE;
+import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
+import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
+import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
@@ -135,6 +144,15 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) {
         return cacheCfg;
     }
 
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+//        resetLog4j(Level.DEBUG, true,
+//            // Categories.
+//            GridDhtPreloader.class.getPackage().getName(),
+//            GridDhtPartitionTopologyImpl.class.getName(),
+//            GridDhtLocalPartition.class.getName());
+    }
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         backups = DFLT_BACKUPS;
@@ -212,6 +230,11 @@ public void testActivePartitionTransferAsyncRandomCoordinator() throws Exception
      */
     private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
+//        resetLog4j(Level.DEBUG, true,
+//            // Categories.
+//            GridDhtPreloader.class.getPackage().getName(),
+//            GridDhtPartitionTopologyImpl.class.getName(),
+//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -250,6 +273,8 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
             info(">>> Finished checking nodes [keyCnt=" + keyCnt + ", nodeCnt=" + nodeCnt + ", grids=" +
                 U.grids2names(ignites) + ']');
 
+            Collection<IgniteFuture<?>> futs = new LinkedList<>();
+
             Ignite last = F.last(ignites);
 
             for (Iterator<Ignite> it = ignites.iterator(); it.hasNext(); ) {
@@ -263,8 +288,21 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
 
                 checkActiveState(ignites);
 
+                final UUID nodeId = g.cluster().localNode().id();
+
                 it.remove();
 
+                futs.add(waitForLocalEvent(last.events(), new P1<Event>() {
+                    @Override public boolean apply(Event e) {
+                        CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
+
+                        ClusterNode node = evt.discoveryNode();
+
+                        return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
+                            (evt.discoveryEventType() == EVT_NODE_LEFT || evt.discoveryEventType() == EVT_NODE_FAILED);
+                    }
+                }, EVT_CACHE_REBALANCE_STOPPED));
+
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
@@ -277,6 +315,14 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
                 awaitPartitionMapExchange(); // Need wait, otherwise test logic is broken if EVT_NODE_FAILED exchanges are merged.
             }
 
+            info("Waiting for preload futures: " + F.view(futs, new IgnitePredicate<IgniteFuture<?>>() {
+                @Override public boolean apply(IgniteFuture<?> fut) {
+                    return !fut.isDone();
+                }
+            }));
+
+            X.waitAll(futs);
+
             info("Finished waiting for preload futures.");
 
             assert last != null;
@@ -456,6 +502,11 @@ private void stopGrids(Iterable<Ignite> grids) {
      */
     private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
+//        resetLog4j(Level.DEBUG, true,
+//            // Categories.
+//            GridDhtPreloader.class.getPackage().getName(),
+//            GridDhtPartitionTopologyImpl.class.getName(),
+//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -507,13 +558,28 @@ private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuf
 
                 it.remove();
 
+                Collection<IgniteFuture<?>> futs = new LinkedList<>();
+
+                for (Ignite gg : ignites)
+                    futs.add(waitForLocalEvent(gg.events(), new P1<Event>() {
+                        @Override public boolean apply(Event e) {
+                            CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
+
+                            ClusterNode node = evt.discoveryNode();
+
+                            return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
+                                (evt.discoveryEventType() == EVT_NODE_LEFT ||
+                                    evt.discoveryEventType() == EVT_NODE_FAILED);
+                        }
+                    }, EVT_CACHE_REBALANCE_STOPPED));
+
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
 
                 info(">>> Waiting for preload futures [leftNode=" + g.name() + ", remaining=" + U.grids2names(ignites) + ']');
 
-                awaitPartitionMapExchange();
+                X.waitAll(futs);
 
                 info("After grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
index 0a8698a597cd8..4ebcd5df35271 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
@@ -17,11 +17,10 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
 
-import java.util.Collections;
+import org.apache.ignite.Ignite;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
@@ -44,7 +43,7 @@ public class GridCacheRebalancingAsyncSelfTest extends GridCacheRebalancingSyncS
      * @throws Exception Exception.
      */
     public void testNodeFailedAtRebalancing() throws Exception {
-        IgniteEx ignite = startGrid(0);
+        Ignite ignite = startGrid(0);
 
         generateData(ignite, 0, 0);
 
@@ -61,7 +60,7 @@ public void testNodeFailedAtRebalancing() throws Exception {
 
         ((TestTcpDiscoverySpi)grid(1).configuration().getDiscoverySpi()).simulateNodeFailure();
 
-        awaitPartitionMapExchange(false, false, Collections.singletonList(ignite.localNode()));
+        waitForRebalancing(0, 3);
 
         checkSupplyContextMapIsEmpty();
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
deleted file mode 100644
index 3965290480a3d..0000000000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- *  Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  See the NOTICE file distributed with
- *  this work for additional information regarding copyright ownership.
- *  The ASF licenses this file to You under the Apache License, Version 2.0
- *  (the "License"); you may not use this file except in compliance with
- *  the License.  You may obtain a copy of the License at
- *
- *       http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
-
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.TestRecordingCommunicationSpi;
-import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
-import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Test cases for checking cancellation rebalancing process if some events occurs.
- */
-public class GridCacheRebalancingCancelTest extends GridCommonAbstractTest {
-    /** */
-    private static final String DHT_PARTITIONED_CACHE = "cacheP";
-
-    /** */
-    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
-
-    /** {@inheritDoc} */
-    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
-        IgniteConfiguration dfltCfg = super.getConfiguration(igniteInstanceName);
-
-        ((TcpDiscoverySpi)dfltCfg.getDiscoverySpi()).setIpFinder(ipFinder);
-
-        dfltCfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
-
-        return dfltCfg;
-    }
-
-    /**
-     * Test rebalance not cancelled when client node join to cluster.
-     *
-     * @throws Exception Exception.
-     */
-    public void testClientNodeJoinAtRebalancing() throws Exception {
-        final IgniteEx ignite0 = startGrid(0);
-
-        IgniteCache<Integer, Integer> cache = ignite0.createCache(
-            new CacheConfiguration<Integer, Integer>(DHT_PARTITIONED_CACHE)
-                .setCacheMode(CacheMode.PARTITIONED)
-                .setRebalanceMode(CacheRebalanceMode.ASYNC)
-                .setBackups(1)
-                .setRebalanceOrder(2)
-                .setAffinity(new RendezvousAffinityFunction(false)));
-
-        for (int i = 0; i < 2048; i++)
-            cache.put(i, i);
-
-        TestRecordingCommunicationSpi.spi(ignite0)
-            .blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
-                @Override public boolean apply(ClusterNode node, Message msg) {
-                    return (msg instanceof GridDhtPartitionSupplyMessage)
-                        && ((GridCacheGroupIdMessage)msg).groupId() == groupIdForCache(ignite0, DHT_PARTITIONED_CACHE);
-                }
-            });
-
-        final IgniteEx ignite1 = startGrid(1);
-
-        TestRecordingCommunicationSpi.spi(ignite0).waitForBlocked();
-
-        GridDhtPartitionDemander.RebalanceFuture fut = (GridDhtPartitionDemander.RebalanceFuture)ignite1.context().
-            cache().internalCache(DHT_PARTITIONED_CACHE).preloader().rebalanceFuture();
-
-        String igniteClntName = getTestIgniteInstanceName(2);
-
-        startGrid(igniteClntName, optimize(getConfiguration(igniteClntName).setClientMode(true)));
-
-        // Resend delayed rebalance messages.
-        TestRecordingCommunicationSpi.spi(ignite0).stopBlock(true);
-
-        awaitPartitionMapExchange();
-
-        // Previous rebalance future should not be cancelled.
-        assertTrue(fut.result());
-    }
-}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
index 258e36e531322..fa4b655fb41bd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
@@ -141,8 +141,7 @@ else if ("index.bin".equals(f.getName())) {
         assertTrue(primaryRemoved);
 
         ignite.cluster().active(true);
-
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         List<String> issues = new ArrayList<>();
         HashMap<Integer, Long> partMap = new HashMap<>();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
index 8b88befee5281..4d03d7dc7e281 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
@@ -21,9 +21,10 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cluster.ClusterNode;
@@ -41,13 +42,11 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
-import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
-import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -69,9 +68,6 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private static final int TEST_SIZE = 100_000;
 
-    /** */
-    private static final long TOPOLOGY_STILLNESS_TIME = 30_000L;
-
     /** partitioned cache name. */
     protected static final String CACHE_NAME_DHT_PARTITIONED = "cacheP";
 
@@ -93,11 +89,11 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private volatile boolean concurrentStartFinished3;
 
-    /**
-     * Time in milliseconds of last received {@link GridDhtPartitionsSingleMessage}
-     * or {@link GridDhtPartitionsFullMessage} using {@link CollectingCommunicationSpi}.
-     */
-    private static volatile long lastPartMsgTime;
+    /** */
+    private volatile boolean record;
+
+    /** */
+    private final ConcurrentHashMap<Class, AtomicInteger> map = new ConcurrentHashMap<>();
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -106,7 +102,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setIpFinder(ipFinder);
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setForceServerMode(true);
 
-        TcpCommunicationSpi commSpi = new CollectingCommunicationSpi();
+        TcpCommunicationSpi commSpi = new CountingCommunicationSpi();
 
         commSpi.setLocalPort(GridTestUtils.getNextCommPort(getClass()));
         commSpi.setTcpNoDelay(true);
@@ -236,35 +232,47 @@ public void testSimpleRebalancing() throws Exception {
 
         startGrid(1);
 
+        int waitMinorVer = ignite.configuration().isLateAffinityAssignment() ? 1 : 0;
+
+        waitForRebalancing(0, 2, waitMinorVer);
+        waitForRebalancing(1, 2, waitMinorVer);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         stopGrid(0);
 
+        waitForRebalancing(1, 3);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         startGrid(2);
 
+        waitForRebalancing(1, 4, waitMinorVer);
+        waitForRebalancing(2, 4, waitMinorVer);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         stopGrid(2);
 
+        waitForRebalancing(1, 5);
+
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         long spend = (System.currentTimeMillis() - start) / 1000;
 
@@ -323,10 +331,13 @@ public void testLoadRebalancing() throws Exception {
 
         startGrid(4);
 
-        awaitPartitionMapExchange(true, true, null);
+        waitForRebalancing(3, 6);
+        waitForRebalancing(4, 6);
 
         concurrentStartFinished = true;
 
+        awaitPartitionMapExchange(true, true, null);
+
         checkSupplyContextMapIsEmpty();
 
         t1.join();
@@ -431,29 +442,27 @@ protected void checkPartitionMapExchangeFinished() {
     }
 
     /**
-     * Method checks for {@link GridDhtPartitionsSingleMessage} or {@link GridDhtPartitionsFullMessage}
-     * not received within {@code TOPOLOGY_STILLNESS_TIME} bound.
-     *
      * @throws Exception If failed.
      */
-    protected void awaitPartitionMessagesAbsent() throws Exception {
-        log.info("Checking GridDhtPartitions*Message absent (it will take up to " +
-            TOPOLOGY_STILLNESS_TIME + " ms) ... ");
-
-        // Start waiting new messages from current point of time.
-        lastPartMsgTime = U.currentTimeMillis();
-
-        assertTrue("Should not have partition Single or Full messages within bound " +
-                TOPOLOGY_STILLNESS_TIME + " ms.",
-            GridTestUtils.waitForCondition(
-                new GridAbsPredicateX() {
-                    @Override public boolean applyx() {
-                        return lastPartMsgTime + TOPOLOGY_STILLNESS_TIME < U.currentTimeMillis();
-                    }
-                },
-                2 * TOPOLOGY_STILLNESS_TIME // 30 sec to gain stable topology and 30 sec of silence.
-            )
-        );
+    protected void checkPartitionMapMessagesAbsent() throws Exception {
+        map.clear();
+
+        record = true;
+
+        log.info("Checking GridDhtPartitions*Message absent (it will take 30 SECONDS) ... ");
+
+        U.sleep(30_000);
+
+        record = false;
+
+        AtomicInteger iF = map.get(GridDhtPartitionsFullMessage.class);
+        AtomicInteger iS = map.get(GridDhtPartitionsSingleMessage.class);
+
+        Integer fullMap = iF != null ? iF.get() : null;
+        Integer singleMap = iS != null ? iS.get() : null;
+
+        assertTrue("Unexpected full map messages: " + fullMap, fullMap == null || fullMap.equals(1)); // 1 message can be sent right after all checks passed.
+        assertNull("Unexpected single map messages", singleMap);
     }
 
     /** {@inheritDoc} */
@@ -486,7 +495,11 @@ public void testComplexRebalancing() throws Exception {
                     while (!concurrentStartFinished2)
                         U.sleep(10);
 
-                    awaitPartitionMapExchange();
+                    waitForRebalancing(0, 5, 0);
+                    waitForRebalancing(1, 5, 0);
+                    waitForRebalancing(2, 5, 0);
+                    waitForRebalancing(3, 5, 0);
+                    waitForRebalancing(4, 5, 0);
 
                     //New cache should start rebalancing.
                     CacheConfiguration<Integer, Integer> cacheRCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
@@ -539,6 +552,12 @@ public void testComplexRebalancing() throws Exception {
         t2.join();
         t3.join();
 
+        waitForRebalancing(0, 5, 1);
+        waitForRebalancing(1, 5, 1);
+        waitForRebalancing(2, 5, 1);
+        waitForRebalancing(3, 5, 1);
+        waitForRebalancing(4, 5, 1);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
@@ -558,23 +577,35 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(1);
 
+        waitForRebalancing(0, 6);
+        waitForRebalancing(2, 6);
+        waitForRebalancing(3, 6);
+        waitForRebalancing(4, 6);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(0);
 
+        waitForRebalancing(2, 7);
+        waitForRebalancing(3, 7);
+        waitForRebalancing(4, 7);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(2);
 
+        waitForRebalancing(3, 8);
+        waitForRebalancing(4, 8);
+
         awaitPartitionMapExchange(true, true, null);
 
         checkPartitionMapExchangeFinished();
 
-        awaitPartitionMessagesAbsent();
+        checkPartitionMapMessagesAbsent();
 
         checkSupplyContextMapIsEmpty();
 
@@ -582,7 +613,7 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(3);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing(4, 9);
 
         checkSupplyContextMapIsEmpty();
 
@@ -603,26 +634,36 @@ public void testComplexRebalancing() throws Exception {
     /**
      *
      */
-    private static class CollectingCommunicationSpi extends TcpCommunicationSpi {
-        /** */
-        @LoggerResource
-        private IgniteLogger log;
-
+    private class CountingCommunicationSpi extends TcpCommunicationSpi {
         /** {@inheritDoc} */
         @Override public void sendMessage(final ClusterNode node, final Message msg,
             final IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
             final Object msg0 = ((GridIoMessage)msg).message();
 
-            if (msg0 instanceof GridDhtPartitionsSingleMessage ||
-                msg0 instanceof GridDhtPartitionsFullMessage) {
-                lastPartMsgTime = U.currentTimeMillis();
-
-                log.info("Last seen time of GridDhtPartitionsSingleMessage or GridDhtPartitionsFullMessage updated " +
-                    "[lastPartMsgTime=" + lastPartMsgTime +
-                    ", node=" + node.id() + ']');
-            }
+            recordMessage(msg0);
 
             super.sendMessage(node, msg, ackC);
         }
+
+        /**
+         * @param msg Message.
+         */
+        private void recordMessage(Object msg) {
+            if (record) {
+                Class id = msg.getClass();
+
+                AtomicInteger ai = map.get(id);
+
+                if (ai == null) {
+                    ai = new AtomicInteger();
+
+                    AtomicInteger oldAi = map.putIfAbsent(id, ai);
+
+                    (oldAi != null ? oldAi : ai).incrementAndGet();
+                }
+                else
+                    ai.incrementAndGet();
+            }
+        }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
index ce84156e56dca..15ec41557dfba 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
@@ -383,7 +383,7 @@ public void testRejoinWithCleanLfs() throws Exception {
         startGrid("flaky");
 
         System.out.println("### Starting rebalancing after flaky node join");
-        awaitPartitionMapExchange();
+        waitForRebalancing();
         System.out.println("### Rebalancing is finished after flaky node join");
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
@@ -689,7 +689,7 @@ private void tryChangeBaselineDown(
         ig0.cluster().setBaselineTopology(fullBlt.subList(0, newBaselineSize));
 
         System.out.println("### Starting rebalancing after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
-        awaitPartitionMapExchange();
+        waitForRebalancing();
         System.out.println("### Rebalancing is finished after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
index bd1a1a9cc3575..e6ec8a37f6538 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
@@ -162,8 +162,7 @@ public void testReservation() throws Exception {
 
         SUPPLY_MESSAGE_LATCH.get().countDown();
 
-        // Partition is OWNING on grid(0) and grid(1)
-        awaitPartitionMapExchange();
+        waitForRebalancing(); // Partition is OWNING on grid(0) and grid(1)
 
         for (int i = 0; i < 2; i++) {
             for (int j = 0; i < 500; i++)
@@ -184,7 +183,7 @@ public void testReservation() throws Exception {
 
         startGrid(0);
 
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         assertEquals(2, grid(1).context().discovery().aliveServerNodes().size());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index 07fdb71fca68a..a24e6d5e0d738 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -192,7 +192,7 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
 
                     grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
 
-                    awaitPartitionMapExchange();
+                    waitForRebalancing();
                 }
                 catch (Throwable expected) {
                     // There can be any exception. Do nothing.
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index ba39e366e9f85..efb4fd511c4aa 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -120,8 +120,7 @@ public void testMappingsPersistedOnJoin() throws Exception {
         c1.put(k, new DummyObject(k));
 
         startGrid(2);
-
-        awaitPartitionMapExchange();
+        waitForRebalancing();
 
         stopAllGrids();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index 5e94052f25013..55436d6da943b 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -53,7 +53,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingAsyncSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingCancelTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
@@ -153,7 +152,6 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
         suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
         suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
-        suite.addTestSuite(GridCacheRebalancingCancelTest.class);
 
         // Test for byte array value special case.
         suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);

From 6da8095f544335a1163a5faba1319ef0d80ac50f Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Fri, 2 Nov 2018 22:13:09 +0300
Subject: [PATCH 477/543] IGNITE-9420 Added throttling for logical recovery
 tests. Increased timeout for remote node join.

---
 .../cache/persistence/db/IgniteLogicalRecoveryTest.java   | 2 ++
 .../cache/persistence/db/wal/IgniteWalRecoveryTest.java   | 8 +++++++-
 .../testframework/junits/multijvm/IgniteProcessProxy.java | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java
index 03a205123bcba..12274d85d9723 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteLogicalRecoveryTest.java
@@ -477,6 +477,8 @@ public CacheLoader(IgniteEx ignite, AtomicBoolean stopFlag, String cacheName) {
 
                         locCache.put(key, payload);
                     }
+
+                    U.sleep(1);
                 }
                 catch (Exception ignored) { }
             }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
index c43396c625787..f635584bff81c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
@@ -25,6 +25,7 @@
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -881,7 +882,7 @@ public void testRandomCrash() throws Exception {
 
             rmt.run(new AsyncLoadRunnable());
 
-            Thread.sleep(20_000);
+            Thread.sleep(10_000);
 
             info(">>> Killing remote process...");
 
@@ -898,6 +899,11 @@ public void testRandomCrash() throws Exception {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override protected List<String> additionalRemoteJvmArgs() {
+        return Collections.singletonList("-D" + "IGNITE_QUIET=false");
+    }
+
     /**
      * @throws Exception if failed.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
index 1eb7ddb09deb8..fafee9d84badf 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
@@ -195,7 +195,7 @@ public IgniteProcessProxy(
         );
 
         if (locJvmGrid != null)
-            assert rmtNodeStartedLatch.await(30, TimeUnit.SECONDS): "Remote node has not joined [id=" + id + ']';
+            assert rmtNodeStartedLatch.await(60, TimeUnit.SECONDS): "Remote node has not joined [id=" + id + ']';
 
         IgniteProcessProxy prevVal = gridProxies.putIfAbsent(cfg.getIgniteInstanceName(), this);
 

From 8699ee5cf29130b36af42ef9481ca5d4ab005d69 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@apache.org>
Date: Sat, 6 Oct 2018 12:30:49 +0300
Subject: [PATCH 478/543] IGNITE-8714 Keep old semantic for REMOVED event for
 internal listeners

(cherry picked from commit 6e0ff06f8e309657a16c94da605348d9c3b804ad)

(cherry picked from commit 3e3447b)
---
 .../cache/distributed/dht/atomic/GridDhtAtomicCache.java       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 4dacde6a066eb..999785423de9f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1759,7 +1759,8 @@ private void updateAllAsyncInternal0(
 
                         // This call will convert entry processor invocation results to cache object instances.
                         // Must be done outside topology read lock to avoid deadlocks.
-                        res.returnValue().marshalResult(ctx);
+                        if (res.returnValue() != null)
+                            res.returnValue().marshalResult(ctx);
 
                         break;
                     }

From 0031bec7ef0525b95ccd91b1c9b400ef3409fe95 Mon Sep 17 00:00:00 2001
From: sboikov <sboikov@apache.org>
Date: Sat, 6 Oct 2018 12:30:49 +0300
Subject: [PATCH 479/543] IGNITE-8714 Keep old semantic for REMOVED event for
 internal listeners

(cherry picked from commit 6e0ff06f8e309657a16c94da605348d9c3b804ad)

(cherry picked from commit 3e3447b)

(cherry picked from commit 8699ee5)
---
 .../cache/distributed/dht/atomic/GridDhtAtomicCache.java       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 4dacde6a066eb..999785423de9f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1759,7 +1759,8 @@ private void updateAllAsyncInternal0(
 
                         // This call will convert entry processor invocation results to cache object instances.
                         // Must be done outside topology read lock to avoid deadlocks.
-                        res.returnValue().marshalResult(ctx);
+                        if (res.returnValue() != null)
+                            res.returnValue().marshalResult(ctx);
 
                         break;
                     }

From 9b8d7ba8431449f8efdcf37030549afef76e8889 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Wed, 7 Nov 2018 21:00:42 +0300
Subject: [PATCH 480/543] IGN-12221 project preparation for p160 release

---
 examples/pom.xml                                   | 2 +-
 modules/aop/pom.xml                                | 2 +-
 modules/apache-license-gen/pom.xml                 | 2 +-
 modules/aws/pom.xml                                | 2 +-
 modules/benchmarks/pom.xml                         | 2 +-
 modules/camel/pom.xml                              | 2 +-
 modules/cassandra/pom.xml                          | 2 +-
 modules/cassandra/serializers/pom.xml              | 4 ++--
 modules/cassandra/store/pom.xml                    | 4 ++--
 modules/clients/pom.xml                            | 2 +-
 modules/cloud/pom.xml                              | 2 +-
 modules/codegen/pom.xml                            | 2 +-
 modules/compatibility/pom.xml                      | 2 +-
 modules/core/pom.xml                               | 2 +-
 modules/core/src/main/resources/ignite.properties  | 2 +-
 modules/dev-utils/pom.xml                          | 2 +-
 modules/direct-io/pom.xml                          | 2 +-
 modules/extdata/p2p/pom.xml                        | 2 +-
 modules/extdata/platform/pom.xml                   | 2 +-
 modules/extdata/uri/modules/uri-dependency/pom.xml | 2 +-
 modules/extdata/uri/pom.xml                        | 2 +-
 modules/flink/pom.xml                              | 2 +-
 modules/flume/pom.xml                              | 2 +-
 modules/gce/pom.xml                                | 2 +-
 modules/geospatial/pom.xml                         | 2 +-
 modules/hadoop/pom.xml                             | 2 +-
 modules/hibernate-4.2/pom.xml                      | 2 +-
 modules/hibernate-5.1/pom.xml                      | 2 +-
 modules/hibernate-core/pom.xml                     | 2 +-
 modules/ignored-tests/pom.xml                      | 2 +-
 modules/indexing/pom.xml                           | 2 +-
 modules/jcl/pom.xml                                | 2 +-
 modules/jms11/pom.xml                              | 2 +-
 modules/jta/pom.xml                                | 2 +-
 modules/kafka/pom.xml                              | 2 +-
 modules/kubernetes/pom.xml                         | 2 +-
 modules/log4j/pom.xml                              | 2 +-
 modules/log4j2/pom.xml                             | 2 +-
 modules/mesos/pom.xml                              | 2 +-
 modules/ml/pom.xml                                 | 2 +-
 modules/mqtt/pom.xml                               | 2 +-
 modules/osgi-karaf/pom.xml                         | 2 +-
 modules/osgi-paxlogging/pom.xml                    | 2 +-
 modules/osgi/pom.xml                               | 2 +-
 modules/rest-http/pom.xml                          | 2 +-
 modules/rocketmq/pom.xml                           | 2 +-
 modules/scalar-2.10/pom.xml                        | 2 +-
 modules/scalar/pom.xml                             | 2 +-
 modules/schedule/pom.xml                           | 2 +-
 modules/slf4j/pom.xml                              | 2 +-
 modules/spark-2.10/pom.xml                         | 2 +-
 modules/spark/pom.xml                              | 2 +-
 modules/spring-data/pom.xml                        | 2 +-
 modules/spring/pom.xml                             | 2 +-
 modules/sqlline/pom.xml                            | 2 +-
 modules/ssh/pom.xml                                | 2 +-
 modules/storm/pom.xml                              | 2 +-
 modules/tools/pom.xml                              | 2 +-
 modules/twitter/pom.xml                            | 2 +-
 modules/urideploy/pom.xml                          | 2 +-
 modules/visor-console-2.10/pom.xml                 | 2 +-
 modules/visor-console/pom.xml                      | 2 +-
 modules/visor-plugins/pom.xml                      | 2 +-
 modules/web-console/pom.xml                        | 2 +-
 modules/web-console/web-agent/pom.xml              | 2 +-
 modules/web/ignite-appserver-test/pom.xml          | 2 +-
 modules/web/ignite-websphere-test/pom.xml          | 2 +-
 modules/web/pom.xml                                | 2 +-
 modules/yardstick/pom.xml                          | 2 +-
 modules/yarn/pom.xml                               | 2 +-
 modules/zeromq/pom.xml                             | 2 +-
 modules/zookeeper/pom.xml                          | 2 +-
 pom.xml                                            | 2 +-
 73 files changed, 75 insertions(+), 75 deletions(-)

diff --git a/examples/pom.xml b/examples/pom.xml
index 299363ac31316..3b101e2709ee3 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -27,7 +27,7 @@
     </parent>
 
     <artifactId>ignite-examples</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/aop/pom.xml b/modules/aop/pom.xml
index 6f10f6ec31630..07599a8188ab3 100644
--- a/modules/aop/pom.xml
+++ b/modules/aop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aop</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/apache-license-gen/pom.xml b/modules/apache-license-gen/pom.xml
index 68494fc00b488..dd7b0fa22b677 100644
--- a/modules/apache-license-gen/pom.xml
+++ b/modules/apache-license-gen/pom.xml
@@ -31,7 +31,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>ignite-apache-license-gen</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <build>
diff --git a/modules/aws/pom.xml b/modules/aws/pom.xml
index d2fb36cd76425..a758725727875 100644
--- a/modules/aws/pom.xml
+++ b/modules/aws/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aws</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/benchmarks/pom.xml b/modules/benchmarks/pom.xml
index 865ad53d5b688..d1e3f0dd58a41 100644
--- a/modules/benchmarks/pom.xml
+++ b/modules/benchmarks/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-benchmarks</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/camel/pom.xml b/modules/camel/pom.xml
index a295a46928ef3..0ca6e40b53fb5 100644
--- a/modules/camel/pom.xml
+++ b/modules/camel/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-camel</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/pom.xml b/modules/cassandra/pom.xml
index 4914c77d0c6ed..41283db70aaf8 100644
--- a/modules/cassandra/pom.xml
+++ b/modules/cassandra/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-cassandra</artifactId>
     <packaging>pom</packaging>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencyManagement>
diff --git a/modules/cassandra/serializers/pom.xml b/modules/cassandra/serializers/pom.xml
index 8703d77367dd3..8ba6a25f03551 100644
--- a/modules/cassandra/serializers/pom.xml
+++ b/modules/cassandra/serializers/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>2.5.1-p150-SNAPSHOT</version>
+        <version>2.5.1-p160-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-serializers</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/store/pom.xml b/modules/cassandra/store/pom.xml
index 3a0da5191ed36..4035efeaa4f4a 100644
--- a/modules/cassandra/store/pom.xml
+++ b/modules/cassandra/store/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>2.5.1-p150-SNAPSHOT</version>
+        <version>2.5.1-p160-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-store</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/clients/pom.xml b/modules/clients/pom.xml
index 8df6a8c85fc5c..9918d424dc96c 100644
--- a/modules/clients/pom.xml
+++ b/modules/clients/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-clients</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/cloud/pom.xml b/modules/cloud/pom.xml
index c750f609ecc47..d0812d9c98df1 100644
--- a/modules/cloud/pom.xml
+++ b/modules/cloud/pom.xml
@@ -29,7 +29,7 @@
     </parent>
 
     <artifactId>ignite-cloud</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/codegen/pom.xml b/modules/codegen/pom.xml
index 2320f56b42b42..f907b4c5f24a2 100644
--- a/modules/codegen/pom.xml
+++ b/modules/codegen/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-codegen</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/compatibility/pom.xml b/modules/compatibility/pom.xml
index 1794f4375865a..80a86168734c2 100644
--- a/modules/compatibility/pom.xml
+++ b/modules/compatibility/pom.xml
@@ -33,7 +33,7 @@
     </parent>
 
     <artifactId>ignite-compatibility</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 491c85c0dc3db..74df1869bcd63 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-core</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <repositories>
diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index db63ec1062b28..b903e31278425 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=2.5.1-p150-SNAPSHOT
+ignite.version=2.5.1-p160-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970
diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml
index afd6d8858f8e6..0d70055753210 100644
--- a/modules/dev-utils/pom.xml
+++ b/modules/dev-utils/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-dev-utils</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/direct-io/pom.xml b/modules/direct-io/pom.xml
index 628a23ee95290..a7cb8f6113611 100644
--- a/modules/direct-io/pom.xml
+++ b/modules/direct-io/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-direct-io</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/extdata/p2p/pom.xml b/modules/extdata/p2p/pom.xml
index 46ae0fccde7ec..a7d77aa10e2f7 100644
--- a/modules/extdata/p2p/pom.xml
+++ b/modules/extdata/p2p/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-extdata-p2p</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/extdata/platform/pom.xml b/modules/extdata/platform/pom.xml
index 523f8aa6ccd43..13811d0005527 100644
--- a/modules/extdata/platform/pom.xml
+++ b/modules/extdata/platform/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-platform</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/extdata/uri/modules/uri-dependency/pom.xml b/modules/extdata/uri/modules/uri-dependency/pom.xml
index 06d208ab07aef..8346102cfb3d0 100644
--- a/modules/extdata/uri/modules/uri-dependency/pom.xml
+++ b/modules/extdata/uri/modules/uri-dependency/pom.xml
@@ -27,7 +27,7 @@
     <artifactId>ignite-extdata-uri-dep</artifactId>
     <packaging>jar</packaging>
 
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <modelVersion>4.0.0</modelVersion>
 
     <dependencies>
diff --git a/modules/extdata/uri/pom.xml b/modules/extdata/uri/pom.xml
index f3a6a3d64fd0a..1fe98c0aab676 100644
--- a/modules/extdata/uri/pom.xml
+++ b/modules/extdata/uri/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-uri</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/flink/pom.xml b/modules/flink/pom.xml
index deaa81bf757e2..3520612038467 100644
--- a/modules/flink/pom.xml
+++ b/modules/flink/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flink</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/flume/pom.xml b/modules/flume/pom.xml
index b67d341beddab..625c134ed1326 100644
--- a/modules/flume/pom.xml
+++ b/modules/flume/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flume</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/gce/pom.xml b/modules/gce/pom.xml
index b740bb7a6d0ef..b0f28056ee087 100644
--- a/modules/gce/pom.xml
+++ b/modules/gce/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-gce</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/geospatial/pom.xml b/modules/geospatial/pom.xml
index feaea2061855b..21497b551dcf1 100644
--- a/modules/geospatial/pom.xml
+++ b/modules/geospatial/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-geospatial</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index 52599df3bb03c..f664fe0a6814c 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hadoop</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-4.2/pom.xml b/modules/hibernate-4.2/pom.xml
index 75f99322e9adc..b4c52c32ac138 100644
--- a/modules/hibernate-4.2/pom.xml
+++ b/modules/hibernate-4.2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate_4.2</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-5.1/pom.xml b/modules/hibernate-5.1/pom.xml
index fa675f339c5fd..09c5c3a9cc399 100644
--- a/modules/hibernate-5.1/pom.xml
+++ b/modules/hibernate-5.1/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate_5.1</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-core/pom.xml b/modules/hibernate-core/pom.xml
index c31b546306bdd..1ba3ab548b548 100644
--- a/modules/hibernate-core/pom.xml
+++ b/modules/hibernate-core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate-core</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ignored-tests/pom.xml b/modules/ignored-tests/pom.xml
index bacbecc968338..3ed8c9b97fc21 100644
--- a/modules/ignored-tests/pom.xml
+++ b/modules/ignored-tests/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ignored-tests</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml
index 46bfadaa7625f..70004c57ee043 100644
--- a/modules/indexing/pom.xml
+++ b/modules/indexing/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-indexing</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jcl/pom.xml b/modules/jcl/pom.xml
index 2208eda8afe63..4daa5b1754d7a 100644
--- a/modules/jcl/pom.xml
+++ b/modules/jcl/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jcl</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jms11/pom.xml b/modules/jms11/pom.xml
index 29f29d44d6360..11be42c308091 100644
--- a/modules/jms11/pom.xml
+++ b/modules/jms11/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jms11</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jta/pom.xml b/modules/jta/pom.xml
index c0f8deea21d16..c4e5d553dda28 100644
--- a/modules/jta/pom.xml
+++ b/modules/jta/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jta</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kafka/pom.xml b/modules/kafka/pom.xml
index ef9dc5ab24538..85985bf063468 100644
--- a/modules/kafka/pom.xml
+++ b/modules/kafka/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kafka</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kubernetes/pom.xml b/modules/kubernetes/pom.xml
index 39376e0f4edf3..4f1ff56d65a1a 100644
--- a/modules/kubernetes/pom.xml
+++ b/modules/kubernetes/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kubernetes</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j/pom.xml b/modules/log4j/pom.xml
index a84b8175f5ae4..5a15560dff118 100644
--- a/modules/log4j/pom.xml
+++ b/modules/log4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j2/pom.xml b/modules/log4j2/pom.xml
index c36d48ec6ac4e..ed6618a1913c7 100644
--- a/modules/log4j2/pom.xml
+++ b/modules/log4j2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j2</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml
index 8926a82a0ca10..ec0d2ee719494 100644
--- a/modules/mesos/pom.xml
+++ b/modules/mesos/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mesos</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml
index bdb87628e4c24..66ee575acfe82 100644
--- a/modules/ml/pom.xml
+++ b/modules/ml/pom.xml
@@ -30,7 +30,7 @@
     </parent>
 
     <artifactId>ignite-ml</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mqtt/pom.xml b/modules/mqtt/pom.xml
index fe7b9f3419425..5e59a161654bb 100644
--- a/modules/mqtt/pom.xml
+++ b/modules/mqtt/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mqtt</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/osgi-karaf/pom.xml b/modules/osgi-karaf/pom.xml
index 5c531760d88f6..623df3b842c0d 100644
--- a/modules/osgi-karaf/pom.xml
+++ b/modules/osgi-karaf/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-karaf</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <build>
diff --git a/modules/osgi-paxlogging/pom.xml b/modules/osgi-paxlogging/pom.xml
index 0ae571685816d..fdfce36e585da 100644
--- a/modules/osgi-paxlogging/pom.xml
+++ b/modules/osgi-paxlogging/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-paxlogging</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <dependencies>
diff --git a/modules/osgi/pom.xml b/modules/osgi/pom.xml
index b539c94f3a1db..3fff5ba889096 100644
--- a/modules/osgi/pom.xml
+++ b/modules/osgi/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/rest-http/pom.xml b/modules/rest-http/pom.xml
index ffa3385c48716..cdc67ea5a6ec3 100644
--- a/modules/rest-http/pom.xml
+++ b/modules/rest-http/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-rest-http</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/rocketmq/pom.xml b/modules/rocketmq/pom.xml
index 7949aa3b21edb..eb2582d980d84 100644
--- a/modules/rocketmq/pom.xml
+++ b/modules/rocketmq/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-rocketmq</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar-2.10/pom.xml b/modules/scalar-2.10/pom.xml
index 54d43ac75a02b..3b6bcf557f7ef 100644
--- a/modules/scalar-2.10/pom.xml
+++ b/modules/scalar-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar_2.10</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml
index 1f63009e0520f..9c6fe3bf70250 100644
--- a/modules/scalar/pom.xml
+++ b/modules/scalar/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/schedule/pom.xml b/modules/schedule/pom.xml
index 82abbf2a8d866..de69b406ab3b1 100644
--- a/modules/schedule/pom.xml
+++ b/modules/schedule/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-schedule</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/slf4j/pom.xml b/modules/slf4j/pom.xml
index 3a6754469f13e..8b684983df479 100644
--- a/modules/slf4j/pom.xml
+++ b/modules/slf4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-slf4j</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark-2.10/pom.xml b/modules/spark-2.10/pom.xml
index e74488245dbe7..d0c261654a46c 100644
--- a/modules/spark-2.10/pom.xml
+++ b/modules/spark-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark_2.10</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index 99198cd3923b1..b8047df2562bb 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring-data/pom.xml b/modules/spring-data/pom.xml
index e849f3a6ffd8a..96e2ff99d1bdb 100644
--- a/modules/spring-data/pom.xml
+++ b/modules/spring-data/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring-data</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring/pom.xml b/modules/spring/pom.xml
index bc849af8cce05..b3af4b8b5166c 100644
--- a/modules/spring/pom.xml
+++ b/modules/spring/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/sqlline/pom.xml b/modules/sqlline/pom.xml
index 34aaaffc41b3b..d643c4595f806 100644
--- a/modules/sqlline/pom.xml
+++ b/modules/sqlline/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-sqlline</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ssh/pom.xml b/modules/ssh/pom.xml
index 4504279c876fc..d5ba8e77e4c01 100644
--- a/modules/ssh/pom.xml
+++ b/modules/ssh/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ssh</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/storm/pom.xml b/modules/storm/pom.xml
index 3077b168ac639..d1101f888304a 100644
--- a/modules/storm/pom.xml
+++ b/modules/storm/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-storm</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/tools/pom.xml b/modules/tools/pom.xml
index 32656daf86e12..209a302f6ab7b 100644
--- a/modules/tools/pom.xml
+++ b/modules/tools/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-tools</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/twitter/pom.xml b/modules/twitter/pom.xml
index ef491ab283578..beeab4db8d8ec 100644
--- a/modules/twitter/pom.xml
+++ b/modules/twitter/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-twitter</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/urideploy/pom.xml b/modules/urideploy/pom.xml
index 8d194e1485f3b..06a09721a3e96 100644
--- a/modules/urideploy/pom.xml
+++ b/modules/urideploy/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-urideploy</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console-2.10/pom.xml b/modules/visor-console-2.10/pom.xml
index f858f588a0b98..2f49e72408b70 100644
--- a/modules/visor-console-2.10/pom.xml
+++ b/modules/visor-console-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console_2.10</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console/pom.xml b/modules/visor-console/pom.xml
index 39a44113cfe35..848cbef76e695 100644
--- a/modules/visor-console/pom.xml
+++ b/modules/visor-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-plugins/pom.xml b/modules/visor-plugins/pom.xml
index 3484fffe83058..523fc83e1e105 100644
--- a/modules/visor-plugins/pom.xml
+++ b/modules/visor-plugins/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-plugins</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web-console/pom.xml b/modules/web-console/pom.xml
index 57cda7ba9361a..cf4c97f2aa82e 100644
--- a/modules/web-console/pom.xml
+++ b/modules/web-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web-console</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml
index 80284f1eb6c81..06127312604fb 100644
--- a/modules/web-console/web-agent/pom.xml
+++ b/modules/web-console/web-agent/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-web-agent</artifactId>
     <packaging>jar</packaging>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web/ignite-appserver-test/pom.xml b/modules/web/ignite-appserver-test/pom.xml
index 397b4f6135f9e..de101df12fc35 100644
--- a/modules/web/ignite-appserver-test/pom.xml
+++ b/modules/web/ignite-appserver-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-appserver-test</artifactId>
     <packaging>jar</packaging>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/ignite-websphere-test/pom.xml b/modules/web/ignite-websphere-test/pom.xml
index 8fc17b83f164f..6b3b340cb1f9e 100644
--- a/modules/web/ignite-websphere-test/pom.xml
+++ b/modules/web/ignite-websphere-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-websphere-test</artifactId>
     <packaging>war</packaging>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/pom.xml b/modules/web/pom.xml
index e2e7e210a5ec7..913e9bb223601 100644
--- a/modules/web/pom.xml
+++ b/modules/web/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index f55c5c495258a..ca8f30646414e 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yardstick</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml
index a38757919eb78..34238af0aeec7 100644
--- a/modules/yarn/pom.xml
+++ b/modules/yarn/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yarn</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zeromq/pom.xml b/modules/zeromq/pom.xml
index bf8caae879433..06f5bedaf6b97 100644
--- a/modules/zeromq/pom.xml
+++ b/modules/zeromq/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zeromq</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index 9e36018c4e0eb..6abdf52b52e6a 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zookeeper</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/pom.xml b/pom.xml
index a28464d1e87d4..813f6e0f8493e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>apache-ignite</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p160-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <properties>

From a9649f27c2f8f87572a65483b2a940b3adaf3d99 Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Thu, 8 Nov 2018 15:12:11 +0300
Subject: [PATCH 481/543] IGNITE-9890 Avoid sorting partitions when possible
 during construction of CachePartitionPartialCountersMap - Fixes #5060.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit c0ac918a8380173d99b349ee851cb9be5f222f5c)
---
 .../GridCachePartitionExchangeManager.java    |  8 ++++-
 .../CachePartitionPartialCountersMap.java     |  2 +-
 .../GridDhtPartitionsSingleMessage.java       | 29 -------------------
 .../GridDhtPartitionsStateValidator.java      |  4 +--
 4 files changed, 10 insertions(+), 33 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 05ebd11251434..342859f3f64fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -134,6 +134,7 @@
 import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE;
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
+import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap.PARTIAL_COUNTERS_MAP_SINCE;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.nextDumpTimeout;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.DFLT_PRELOAD_RESEND_TIMEOUT;
 
@@ -1332,7 +1333,12 @@ private void sendLocalPartitions(
         @NotNull Collection<CacheGroupContext> grps
     ) {
         GridDhtPartitionsSingleMessage m =
-            createPartitionsSingleMessage(id, cctx.kernalContext().clientNode(), false, false, null, grps);
+            createPartitionsSingleMessage(id,
+                cctx.kernalContext().clientNode(),
+                false,
+                node.version().compareToIgnoreTimestamp(PARTIAL_COUNTERS_MAP_SINCE) >= 0,
+                null,
+                grps);
 
         if (log.isTraceEnabled())
             log.trace("Sending local partitions [nodeId=" + node.id() + ", msg=" + m + ']');
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
index c8cf3f848b1fb..971f3eaeb2715 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/CachePartitionPartialCountersMap.java
@@ -34,7 +34,7 @@ public class CachePartitionPartialCountersMap implements Serializable {
     private static final long serialVersionUID = 0L;
 
     /** */
-    static final IgniteProductVersion PARTIAL_COUNTERS_MAP_SINCE = IgniteProductVersion.fromString("2.1.4");
+    public static final IgniteProductVersion PARTIAL_COUNTERS_MAP_SINCE = IgniteProductVersion.fromString("2.1.4");
 
     /** */
     public static final CachePartitionPartialCountersMap EMPTY = new CachePartitionPartialCountersMap();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 636621da529bd..3ce3b9ffd9c23 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -227,35 +227,6 @@ public CachePartitionPartialCountersMap partitionUpdateCounters(int grpId, int p
         return CachePartitionPartialCountersMap.fromCountersMap(map, partsCnt);
     }
 
-    /**
-     * @param grpId Cache group ID.
-     * @param partsCnt Total cache partitions.
-     * @return Partition update counters.
-     */
-    @SuppressWarnings("unchecked")
-    public CachePartitionPartialCountersMap partitionUpdateCountersUnsorted(int grpId, int partsCnt) {
-        Object res = partCntrs == null ? null : partCntrs.get(grpId);
-
-        if (res == null)
-            return CachePartitionPartialCountersMap.EMPTY;
-
-        if (res instanceof CachePartitionPartialCountersMap)
-            return (CachePartitionPartialCountersMap)res;
-
-        assert res instanceof Map : res;
-
-        Map<Integer, T2<Long, Long>> map = (Map<Integer, T2<Long, Long>>)res;
-
-        CachePartitionPartialCountersMap partCounersMap = new CachePartitionPartialCountersMap(partsCnt);
-
-        for (Map.Entry<Integer, T2<Long, Long>> e : map.entrySet())
-            partCounersMap.add(e.getKey(), e.getValue().get1(), e.getValue().get2());
-
-        partCounersMap.trim();
-
-        return partCounersMap;
-    }
-
     /**
      * Adds partition sizes map for specified {@code grpId} to the current message.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
index 5e94096c0f376..5daa558ea3abd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionsStateValidator.java
@@ -186,7 +186,7 @@ public Map<Integer, Map<UUID, Long>> validatePartitionsUpdateCounters(
 
             final GridDhtPartitionsSingleMessage message = e.getValue();
 
-            CachePartitionPartialCountersMap countersMap = message.partitionUpdateCountersUnsorted(top.groupId(), partitions);
+            CachePartitionPartialCountersMap countersMap = message.partitionUpdateCounters(top.groupId(), partitions);
 
             Map<Integer, Long> sizesMap = message.partitionSizes(top.groupId());
 
@@ -246,7 +246,7 @@ public Map<Integer, Map<UUID, Long>> validatePartitionsSizes(
 
             final GridDhtPartitionsSingleMessage message = e.getValue();
 
-            CachePartitionPartialCountersMap countersMap = message.partitionUpdateCountersUnsorted(top.groupId(), partitions);
+            CachePartitionPartialCountersMap countersMap = message.partitionUpdateCounters(top.groupId(), partitions);
 
             Map<Integer, Long> sizesMap = message.partitionSizes(top.groupId());
 

From a18eaf1897fa32846d02ea5a789d63924fe0ad1e Mon Sep 17 00:00:00 2001
From: zstan <stanilovsky@gmail.com>
Date: Fri, 9 Nov 2018 11:26:14 +0300
Subject: [PATCH 482/543] IGNITE-10128 Race in read\write cache configurations.
 - Fixes #5241.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 567d7994dcd660ecb6831e5c73d02797e80e25f6)
---
 .../file/FilePageStoreManager.java            | 41 ++++++++++---------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index e0516482ea304..09ec837e00832 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -22,7 +22,6 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -186,6 +185,22 @@ public FilePageStoreManager(GridKernalContext ctx) {
                 "DataStorageConfiguration#storagePath properties). " +
                 "Current persistence store directory is: [" + tmpDir + "]");
         }
+
+        File[] files = storeWorkDir.listFiles();
+
+        for (File file : files) {
+            if (file.isDirectory()) {
+                File[] tmpFiles = file.listFiles((k, v) -> v.endsWith(CACHE_DATA_TMP_FILENAME));
+
+                if (tmpFiles != null) {
+                    for (File tmpFile : tmpFiles) {
+                        if (!tmpFile.delete())
+                            log.warning("Failed to delete temporary cache config file" +
+                                    "(make sure Ignite process has enough rights):" + file.getName());
+                    }
+                }
+            }
+        }
     }
 
     /** {@inheritDoc} */
@@ -355,17 +370,17 @@ public FilePageStoreManager(GridKernalContext ctx) {
             if (overwrite || !Files.exists(filePath) || Files.size(filePath) == 0) {
                 File tmp = new File(file.getParent(), file.getName() + TMP_SUFFIX);
 
-                tmp.createNewFile();
+                if (tmp.exists() && !tmp.delete()) {
+                    log.warning("Failed to delete temporary cache config file" +
+                            "(make sure Ignite process has enough rights):" + file.getName());
+                }
 
                 // Pre-existing file will be truncated upon stream open.
                 try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(tmp))) {
                     marshaller.marshal(cacheData, stream);
                 }
 
-                if (file.exists())
-                    file.delete();
-
-                Files.move(tmp.toPath(), file.toPath());
+                Files.move(tmp.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING);
             }
         }
         catch (IOException ex) {
@@ -736,20 +751,6 @@ else if (lockF.exists()) {
 
         for (File file : files) {
             if (file.isDirectory()) {
-                File[] tmpFiles = file.listFiles(new FilenameFilter() {
-                    @Override public boolean accept(File dir, String name) {
-                        return name.endsWith(CACHE_DATA_TMP_FILENAME);
-                    }
-                });
-
-                if (tmpFiles != null) {
-                    for (File tmpFile: tmpFiles) {
-                        if (!tmpFile.delete())
-                            log.warning("Failed to delete temporary cache config file" +
-                                    "(make sure Ignite process has enough rights):" + file.getName());
-                    }
-                }
-
                 if (file.getName().startsWith(CACHE_DIR_PREFIX)) {
                     File conf = new File(file, CACHE_DATA_FILENAME);
 

From 56e0e9a9b5da83e488335d1275e1a30b8b70e4ce Mon Sep 17 00:00:00 2001
From: zstan <stanilovsky@gmail.com>
Date: Fri, 9 Nov 2018 11:26:14 +0300
Subject: [PATCH 483/543] IGNITE-10128 Race in read\write cache configurations.
 - Fixes #5241.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 567d7994dcd660ecb6831e5c73d02797e80e25f6)
---
 .../file/FilePageStoreManager.java            | 41 ++++++++++---------
 1 file changed, 21 insertions(+), 20 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index e0516482ea304..09ec837e00832 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -22,7 +22,6 @@
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
-import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -186,6 +185,22 @@ public FilePageStoreManager(GridKernalContext ctx) {
                 "DataStorageConfiguration#storagePath properties). " +
                 "Current persistence store directory is: [" + tmpDir + "]");
         }
+
+        File[] files = storeWorkDir.listFiles();
+
+        for (File file : files) {
+            if (file.isDirectory()) {
+                File[] tmpFiles = file.listFiles((k, v) -> v.endsWith(CACHE_DATA_TMP_FILENAME));
+
+                if (tmpFiles != null) {
+                    for (File tmpFile : tmpFiles) {
+                        if (!tmpFile.delete())
+                            log.warning("Failed to delete temporary cache config file" +
+                                    "(make sure Ignite process has enough rights):" + file.getName());
+                    }
+                }
+            }
+        }
     }
 
     /** {@inheritDoc} */
@@ -355,17 +370,17 @@ public FilePageStoreManager(GridKernalContext ctx) {
             if (overwrite || !Files.exists(filePath) || Files.size(filePath) == 0) {
                 File tmp = new File(file.getParent(), file.getName() + TMP_SUFFIX);
 
-                tmp.createNewFile();
+                if (tmp.exists() && !tmp.delete()) {
+                    log.warning("Failed to delete temporary cache config file" +
+                            "(make sure Ignite process has enough rights):" + file.getName());
+                }
 
                 // Pre-existing file will be truncated upon stream open.
                 try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(tmp))) {
                     marshaller.marshal(cacheData, stream);
                 }
 
-                if (file.exists())
-                    file.delete();
-
-                Files.move(tmp.toPath(), file.toPath());
+                Files.move(tmp.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING);
             }
         }
         catch (IOException ex) {
@@ -736,20 +751,6 @@ else if (lockF.exists()) {
 
         for (File file : files) {
             if (file.isDirectory()) {
-                File[] tmpFiles = file.listFiles(new FilenameFilter() {
-                    @Override public boolean accept(File dir, String name) {
-                        return name.endsWith(CACHE_DATA_TMP_FILENAME);
-                    }
-                });
-
-                if (tmpFiles != null) {
-                    for (File tmpFile: tmpFiles) {
-                        if (!tmpFile.delete())
-                            log.warning("Failed to delete temporary cache config file" +
-                                    "(make sure Ignite process has enough rights):" + file.getName());
-                    }
-                }
-
                 if (file.getName().startsWith(CACHE_DIR_PREFIX)) {
                     File conf = new File(file, CACHE_DATA_FILENAME);
 

From c5374e18364ffad4469d7a1ef1805dbd0a8c5d06 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Fri, 9 Nov 2018 14:17:09 +0300
Subject: [PATCH 484/543] GG-14408 Backport fix for timeoutProcessor.waitAsync
 from IGNITE-10024

---
 .../internal/processors/timeout/GridTimeoutProcessor.java   | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index de9e8eb8c0a68..7175382f8106b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -144,15 +144,15 @@ public boolean removeTimeoutObject(GridTimeoutObject timeoutObj) {
     /**
      * Wait for a future (listen with timeout).
      * @param fut Future.
-     * @param timeout Timeout millis. -1 means expired timeout, 0 - no timeout.
+     * @param timeout Timeout millis. -1 means expired timeout, 0 means waiting without timeout.
      * @param clo Finish closure. First argument contains error on future or null if no errors,
-     * second is {@code true} if wait timed out.
+     * second is {@code true} if wait timed out or passed timeout argument means expired timeout.
      */
     public void waitAsync(final IgniteInternalFuture<?> fut,
         long timeout,
         IgniteBiInClosure<IgniteCheckedException, Boolean> clo) {
         if (timeout == -1) {
-            clo.apply(null, false);
+            clo.apply(null, true);
 
             return;
         }

From 164c50b5a6f2cd793e60897b968d8b34509b0bc3 Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Fri, 9 Nov 2018 15:00:03 +0300
Subject: [PATCH 485/543] IGNITE-9870
 GridDhtPartitionsFullMessage#prepareMarshal compression parallelization -
 Fixes #5330.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 70952fe92d5be7f7c9407783867d524544cd9fec)
---
 .../configuration/IgniteConfiguration.java    |  30 +++
 .../cache/CacheAffinitySharedManager.java     |  14 +-
 .../processors/cache/GridCacheProcessor.java  |  44 ++--
 .../GridDhtPartitionsExchangeFuture.java      |   6 +-
 .../GridDhtPartitionsFullMessage.java         | 214 +++++++++++-------
 .../GridCacheDatabaseSharedManager.java       |   4 +-
 .../ignite/internal/util/IgniteUtils.java     |  68 ++++--
 .../util/lang/IgniteThrowableConsumer.java    |   8 +-
 .../internal/util/IgniteUtilsSelfTest.java    | 122 ++++++++--
 .../IgniteConfigurationParityTest.cs          |   5 +-
 10 files changed, 375 insertions(+), 140 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
index cc3ea1006bef1..181881f873a3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteConfiguration.java
@@ -21,6 +21,7 @@
 import java.lang.management.ManagementFactory;
 import java.util.Map;
 import java.util.UUID;
+import java.util.zip.Deflater;
 import javax.cache.configuration.Factory;
 import javax.cache.event.CacheEntryListener;
 import javax.cache.expiry.ExpiryPolicy;
@@ -118,6 +119,9 @@ public class IgniteConfiguration {
     /** Default maximum timeout to wait for network responses in milliseconds (value is {@code 5,000ms}). */
     public static final long DFLT_NETWORK_TIMEOUT = 5000;
 
+    /** Default compression level for network messages (value is Deflater.BEST_SPEED. */
+    public static final int DFLT_NETWORK_COMPRESSION = Deflater.BEST_SPEED;
+
     /** Default interval between message send retries. */
     public static final long DFLT_SEND_RETRY_DELAY = 1000;
 
@@ -295,6 +299,9 @@ public class IgniteConfiguration {
     /** Maximum network requests timeout. */
     private long netTimeout = DFLT_NETWORK_TIMEOUT;
 
+    /** Compression level for network binary messages. */
+    private int netCompressionLevel = DFLT_NETWORK_COMPRESSION;
+
     /** Interval between message send retries. */
     private long sndRetryDelay = DFLT_SEND_RETRY_DELAY;
 
@@ -1451,6 +1458,29 @@ public IgniteConfiguration setNetworkTimeout(long netTimeout) {
         return this;
     }
 
+    /**
+     * Compression level of internal network messages.
+     * <p>
+     * If not provided, then default value
+     * Deflater.BEST_SPEED is used.
+     *
+     * @return Network messages default compression level.
+     */
+    public int getNetworkCompressionLevel() {
+        return netCompressionLevel;
+    }
+
+    /**
+     * Compression level for internal network messages.
+     * <p>
+     * If not provided, then default value
+     * Deflater.BEST_SPEED is used.
+     *
+     */
+    public void setNetworkCompressionLevel(int netCompressionLevel) {
+        this.netCompressionLevel = netCompressionLevel;
+    }
+
     /**
      * Interval in milliseconds between message send retries.
      * <p>
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 76be2b6eda23f..8eaaa5d551f95 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -981,6 +981,8 @@ private void initAffinityOnCacheGroupsStart(
                         initAffinity(cachesRegistry.group(grp.groupId()), grp.affinity(), fut);
                     }
                 }
+
+                return null;
             }
         );
     }
@@ -1239,7 +1241,11 @@ private void forAllRegisteredCacheGroups(IgniteInClosureX<CacheGroupDescriptor>
             .collect(Collectors.toList());
 
         try {
-            U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c::applyx);
+            U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, t -> {
+                c.applyx(t);
+
+                return null;
+            });
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException("Failed to execute affinity operation on cache groups", e);
@@ -1267,7 +1273,11 @@ private void forAllCacheGroups(boolean crd, IgniteInClosureX<GridAffinityAssignm
         }
 
         try {
-            U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, c::applyx);
+            U.doInParallel(cctx.kernalContext().getSystemExecutorService(), affinityCaches, t -> {
+                c.applyx(t);
+
+                return null;
+            });
         }
         catch (IgniteCheckedException e) {
             throw new IgniteException("Failed to execute affinity operation on cache groups", e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 4755f33cd2ce6..edb88886bd44b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -1965,19 +1965,23 @@ Map<StartCacheInfo, IgniteCheckedException> prepareStartCachesIfPossible(Collect
      */
     private void prepareStartCaches(
         Collection<StartCacheInfo> startCacheInfos,
-        StartCacheFailHandler<StartCacheInfo> cacheStartFailHandler
+        StartCacheFailHandler<StartCacheInfo, Void> cacheStartFailHandler
     ) throws IgniteCheckedException {
         if (!IGNITE_ALLOW_START_CACHES_IN_PARALLEL || startCacheInfos.size() <= 1) {
             for (StartCacheInfo startCacheInfo : startCacheInfos) {
                 cacheStartFailHandler.handle(
                     startCacheInfo,
-                    cacheInfo -> prepareCacheStart(
-                        cacheInfo.getCacheDescriptor().cacheConfiguration(),
-                        cacheInfo.getCacheDescriptor(),
-                        cacheInfo.getReqNearCfg(),
-                        cacheInfo.getExchangeTopVer(),
-                        cacheInfo.isDisabledAfterStart()
-                    )
+                    cacheInfo -> {
+                        prepareCacheStart(
+                            cacheInfo.getCacheDescriptor().cacheConfiguration(),
+                            cacheInfo.getCacheDescriptor(),
+                            cacheInfo.getReqNearCfg(),
+                            cacheInfo.getExchangeTopVer(),
+                            cacheInfo.isDisabledAfterStart()
+                        );
+
+                        return null;
+                    }
                 );
             }
         }
@@ -1993,7 +1997,7 @@ private void prepareStartCaches(
                 parallelismLvl,
                 sharedCtx.kernalContext().getSystemExecutorService(),
                 startCacheInfos,
-                startCacheInfo ->
+                startCacheInfo -> {
                     cacheStartFailHandler.handle(
                         startCacheInfo,
                         cacheInfo -> {
@@ -2005,8 +2009,13 @@ private void prepareStartCaches(
                                 cacheInfo.isDisabledAfterStart()
                             );
                             cacheContexts.put(cacheInfo, cacheCtx);
+
+                            return null;
                         }
-                    )
+                    );
+
+                    return null;
+                }
             );
 
             /*
@@ -2036,6 +2045,8 @@ private void prepareStartCaches(
                                     : new QuerySchema()
                             );
                         }
+
+                        return null;
                     }
                 );
             }
@@ -2044,7 +2055,7 @@ private void prepareStartCaches(
                 parallelismLvl,
                 sharedCtx.kernalContext().getSystemExecutorService(),
                 cacheContexts.entrySet(),
-                cacheCtxEntry ->
+                cacheCtxEntry -> {
                     cacheStartFailHandler.handle(
                         cacheCtxEntry.getKey(),
                         cacheInfo -> {
@@ -2054,8 +2065,13 @@ private void prepareStartCaches(
                                 finishRecovery(cacheInfo.getExchangeTopVer(), cacheContext);
                             else
                                 onCacheStarted(cacheCtxEntry.getValue());
+
+                            return null;
                         }
-                    )
+                    );
+
+                    return null;
+                }
             );
         }
     }
@@ -5204,7 +5220,7 @@ private class CacheRecoveryLifecycle implements MetastorageLifecycleListener, Da
      *
      * @param <T> Type of started data.
      */
-    private static interface StartCacheFailHandler<T> {
+    private static interface StartCacheFailHandler<T, R> {
         /**
          * Handle of fail.
          *
@@ -5212,7 +5228,7 @@ private static interface StartCacheFailHandler<T> {
          * @param startCacheOperation Operation for start cache.
          * @throws IgniteCheckedException if failed.
          */
-        void handle(T data, IgniteThrowableConsumer<T> startCacheOperation) throws IgniteCheckedException;
+        void handle(T data, IgniteThrowableConsumer<T, R> startCacheOperation) throws IgniteCheckedException;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 9cce8ee5b01d1..cbc7070afafb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -3237,7 +3237,7 @@ private void validatePartitionsState() {
                         || grpCtx.config().getRebalanceDelay() == -1
                         || grpCtx.config().getRebalanceMode() == CacheRebalanceMode.NONE
                         )
-                        return;
+                        return null;
 
                     try {
                         validator.validatePartitionCountersAndSizes(GridDhtPartitionsExchangeFuture.this, top, msgs);
@@ -3246,6 +3246,8 @@ private void validatePartitionsState() {
                         log.warning("Partition states validation has failed for group: " + grpCtx.cacheOrGroupName() + ". " + ex.getMessage());
                         // TODO: Handle such errors https://issues.apache.org/jira/browse/IGNITE-7833
                     }
+
+                    return null;
                 }
             );
         }
@@ -3278,6 +3280,8 @@ private void assignPartitionsStates() {
                         assignPartitionSizes(top);
                     else
                         assignPartitionStates(top);
+
+                    return null;
                 }
             );
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index d915426619449..536fdfd04756e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -19,8 +19,11 @@
 
 import java.io.Externalizable;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
@@ -33,6 +36,7 @@
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.T2;
@@ -410,69 +414,75 @@ void setErrorsMap(Map<UUID, Exception> errs) {
             (!F.isEmpty(errs) && errsBytes == null);
 
         if (marshal) {
-            byte[] partsBytes0 = null;
-            byte[] partCntrsBytes0 = null;
-            byte[] partCntrsBytes20 = null;
-            byte[] partHistSuppliersBytes0 = null;
-            byte[] partsToReloadBytes0 = null;
-            byte[] partsSizesBytes0 = null;
-            byte[] errsBytes0 = null;
+            // Reserve at least 2 threads for system operations.
+            int parallelismLvl = Math.max(1, ctx.kernalContext().config().getSystemThreadPoolSize() - 2);
+
+            Collection<Object> objectsToMarshall = new ArrayList<>();
+
+            if (!F.isEmpty(parts) && partsBytes == null)
+                objectsToMarshall.add(parts);
+
+            if (partCntrs != null && !partCntrs.empty() && partCntrsBytes == null)
+                objectsToMarshall.add(partCntrs);
+
+            if (partCntrs2 != null && !partCntrs2.empty() && partCntrsBytes2 == null)
+                objectsToMarshall.add(partCntrs2);
+
+            if (partHistSuppliers != null && partHistSuppliersBytes == null)
+                objectsToMarshall.add(partHistSuppliers);
+
+            if (partsToReload != null && partsToReloadBytes == null)
+                objectsToMarshall.add(partsToReload);
+
+            if (partsSizes != null && partsSizesBytes == null)
+                objectsToMarshall.add(partsSizes);
+
+            if (!F.isEmpty(errs) && errsBytes == null)
+                objectsToMarshall.add(errs);
+
+            Collection<byte[]> marshalled = U.doInParallel(
+                parallelismLvl,
+                ctx.kernalContext().getSystemExecutorService(),
+                objectsToMarshall,
+                new IgniteThrowableConsumer<Object, byte[]>() {
+                    @Override public byte[] accept(Object payload) throws IgniteCheckedException {
+                        byte[] marshalled = U.marshal(ctx, payload);
+
+                        if(compress)
+                            marshalled = U.zip(marshalled, ctx.gridConfig().getNetworkCompressionLevel());
+
+                        return marshalled;
+                    }
+            });
+
+            Iterator<byte[]> iterator = marshalled.iterator();
 
             if (!F.isEmpty(parts) && partsBytes == null)
-                partsBytes0 = U.marshal(ctx, parts);
+                partsBytes = iterator.next();
 
             if (partCntrs != null && !partCntrs.empty() && partCntrsBytes == null)
-                partCntrsBytes0 = U.marshal(ctx, partCntrs);
+                partCntrsBytes = iterator.next();
 
             if (partCntrs2 != null && !partCntrs2.empty() && partCntrsBytes2 == null)
-                partCntrsBytes20 = U.marshal(ctx, partCntrs2);
+                partCntrsBytes2 = iterator.next();
 
             if (partHistSuppliers != null && partHistSuppliersBytes == null)
-                partHistSuppliersBytes0 = U.marshal(ctx, partHistSuppliers);
+                partHistSuppliersBytes = iterator.next();
 
             if (partsToReload != null && partsToReloadBytes == null)
-                partsToReloadBytes0 = U.marshal(ctx, partsToReload);
+                partsToReloadBytes = iterator.next();
 
             if (partsSizes != null && partsSizesBytes == null)
-                partsSizesBytes0 = U.marshal(ctx, partsSizes);
+                partsSizesBytes = iterator.next();
 
             if (!F.isEmpty(errs) && errsBytes == null)
-                errsBytes0 = U.marshal(ctx, errs);
+                errsBytes = iterator.next();
 
             if (compress) {
-                assert !compressed();
-
-                try {
-                    byte[] partsBytesZip = U.zip(partsBytes0);
-                    byte[] partCntrsBytesZip = U.zip(partCntrsBytes0);
-                    byte[] partCntrsBytes2Zip = U.zip(partCntrsBytes20);
-                    byte[] partHistSuppliersBytesZip = U.zip(partHistSuppliersBytes0);
-                    byte[] partsToReloadBytesZip = U.zip(partsToReloadBytes0);
-                    byte[] partsSizesBytesZip = U.zip(partsSizesBytes0);
-                    byte[] exsBytesZip = U.zip(errsBytes0);
-
-                    partsBytes0 = partsBytesZip;
-                    partCntrsBytes0 = partCntrsBytesZip;
-                    partCntrsBytes20 = partCntrsBytes2Zip;
-                    partHistSuppliersBytes0 = partHistSuppliersBytesZip;
-                    partsToReloadBytes0 = partsToReloadBytesZip;
-                    partsSizesBytes0 = partsSizesBytesZip;
-                    errsBytes0 = exsBytesZip;
-
-                    compressed(true);
-                }
-                catch (IgniteCheckedException e) {
-                    U.error(ctx.logger(getClass()), "Failed to compress partitions data: " + e, e);
-                }
-            }
+                assert !compressed() : "Unexpected compressed state";
 
-            partsBytes = partsBytes0;
-            partCntrsBytes = partCntrsBytes0;
-            partCntrsBytes2 = partCntrsBytes20;
-            partHistSuppliersBytes = partHistSuppliersBytes0;
-            partsToReloadBytes = partsToReloadBytes0;
-            partsSizesBytes = partsSizesBytes0;
-            errsBytes = errsBytes0;
+                compressed(true);
+            }
         }
     }
 
@@ -494,11 +504,51 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
     @Override public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr) throws IgniteCheckedException {
         super.finishUnmarshal(ctx, ldr);
 
+        ClassLoader classLoader = U.resolveClassLoader(ldr, ctx.gridConfig());
+
+        Collection<byte[]> objectsToUnmarshall = new ArrayList<>();
+
+        // Reserve at least 2 threads for system operations.
+        int parallelismLvl = Math.max(1, ctx.kernalContext().config().getSystemThreadPoolSize() - 2);
+
+        if (partsBytes != null && parts == null)
+            objectsToUnmarshall.add(partsBytes);
+
+        if (partCntrsBytes != null && partCntrs == null)
+            objectsToUnmarshall.add(partCntrsBytes);
+
+        if (partCntrsBytes2 != null && partCntrs2 == null)
+            objectsToUnmarshall.add(partCntrsBytes2);
+
+        if (partHistSuppliersBytes != null && partHistSuppliers == null)
+            objectsToUnmarshall.add(partHistSuppliersBytes);
+
+        if (partsToReloadBytes != null && partsToReload == null)
+            objectsToUnmarshall.add(partsToReloadBytes);
+
+        if (partsSizesBytes != null && partsSizes == null)
+            objectsToUnmarshall.add(partsSizesBytes);
+
+        if (errsBytes != null && errs == null)
+            objectsToUnmarshall.add(errsBytes);
+
+        Collection<Object> unmarshalled = U.doInParallel(
+            parallelismLvl,
+            ctx.kernalContext().getSystemExecutorService(),
+            objectsToUnmarshall,
+            new IgniteThrowableConsumer<byte[], Object>() {
+                @Override public Object accept(byte[] binary) throws IgniteCheckedException {
+                    return compressed()
+                        ? U.unmarshalZip(ctx.marshaller(), binary, classLoader)
+                        : U.unmarshal(ctx, binary, classLoader);
+                }
+            }
+        );
+
+        Iterator<Object> iterator = unmarshalled.iterator();
+
         if (partsBytes != null && parts == null) {
-            if (compressed())
-                parts = U.unmarshalZip(ctx.marshaller(), partsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-            else
-                parts = U.unmarshal(ctx, partsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+            parts = (Map<Integer, GridDhtPartitionFullMap>)iterator.next();
 
             if (dupPartsData != null) {
                 assert parts != null;
@@ -528,53 +578,41 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
             }
         }
 
-        if (parts == null)
-            parts = new HashMap<>();
+        if (partCntrsBytes != null && partCntrs == null)
+            partCntrs = (IgniteDhtPartitionCountersMap)iterator.next();
 
-        if (partCntrsBytes != null && partCntrs == null) {
-            if (compressed())
-                partCntrs = U.unmarshalZip(ctx.marshaller(), partCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-            else
-                partCntrs = U.unmarshal(ctx, partCntrsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-        }
+        if (partCntrsBytes2 != null && partCntrs2 == null)
+            partCntrs2 = (IgniteDhtPartitionCountersMap2)iterator.next();
 
-        if (partCntrsBytes2 != null && partCntrs2 == null) {
-            if (compressed())
-                partCntrs2 = U.unmarshalZip(ctx.marshaller(), partCntrsBytes2, U.resolveClassLoader(ldr, ctx.gridConfig()));
-            else
-                partCntrs2 = U.unmarshal(ctx, partCntrsBytes2, U.resolveClassLoader(ldr, ctx.gridConfig()));
-        }
+        if (partHistSuppliersBytes != null && partHistSuppliers == null)
+            partHistSuppliers = (IgniteDhtPartitionHistorySuppliersMap)iterator.next();
 
-        if (partHistSuppliersBytes != null && partHistSuppliers == null) {
-            if (compressed())
-                partHistSuppliers = U.unmarshalZip(ctx.marshaller(), partHistSuppliersBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-            else
-                partHistSuppliers = U.unmarshal(ctx, partHistSuppliersBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-        }
+        if (partsToReloadBytes != null && partsToReload == null)
+            partsToReload = (IgniteDhtPartitionsToReloadMap)iterator.next();
 
-        if (partsToReloadBytes != null && partsToReload == null) {
-            if (compressed())
-                partsToReload = U.unmarshalZip(ctx.marshaller(), partsToReloadBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-            else
-                partsToReload = U.unmarshal(ctx, partsToReloadBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-        }
+        if (partsSizesBytes != null && partsSizes == null)
+            partsSizes = (Map<Integer, Map<Integer, Long>>)iterator.next();
 
-        if (partsSizesBytes != null && partsSizes == null) {
-            if (compressed())
-                partsSizes = U.unmarshalZip(ctx.marshaller(), partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-            else
-                partsSizes = U.unmarshal(ctx, partsSizesBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-        }
+        if (errsBytes != null && errs == null)
+            errs = (Map<UUID, Exception>)iterator.next();
+
+        if (parts == null)
+            parts = new HashMap<>();
 
         if (partCntrs == null)
             partCntrs = new IgniteDhtPartitionCountersMap();
 
-        if (errsBytes != null && errs == null) {
-            if (compressed())
-                errs = U.unmarshalZip(ctx.marshaller(), errsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-            else
-                errs = U.unmarshal(ctx, errsBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
-        }
+        if (partCntrs2 == null)
+            partCntrs2 = new IgniteDhtPartitionCountersMap2();
+
+        if(partHistSuppliers == null)
+            partHistSuppliers = new IgniteDhtPartitionHistorySuppliersMap();
+
+        if(partsToReload == null)
+            partsToReload = new IgniteDhtPartitionsToReloadMap();
+
+        if(partsSizes == null)
+            partsSizes = new HashMap<>();
 
         if (errs == null)
             errs = new HashMap<>();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index d0166e7bb7f4d..ef3c81a4def24 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1387,7 +1387,7 @@ private void shutdownCheckpointer(boolean cancel) {
                 cctx.cache().cacheGroups(),
                 cacheGroup -> {
                     if (cacheGroup.isLocal())
-                        return;
+                        return null;
 
                     cctx.database().checkpointReadLock();
 
@@ -1400,6 +1400,8 @@ private void shutdownCheckpointer(boolean cancel) {
                     finally {
                         cctx.database().checkpointReadUnlock();
                     }
+
+                    return null;
                 }
             );
         }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 69a5a4ec0b284..a82a6858dc150 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -142,6 +142,7 @@
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
+import java.util.zip.Deflater;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
 import java.util.zip.ZipInputStream;
@@ -10210,11 +10211,23 @@ public static void restoreOldIgniteName(@Nullable String oldName, @Nullable Stri
     }
 
     /**
+     * Zip binary payload using default compression.
+     *
      * @param bytes Byte array to compress.
      * @return Compressed bytes.
      * @throws IgniteCheckedException If failed.
      */
     public static byte[] zip(@Nullable byte[] bytes) throws IgniteCheckedException {
+        return zip(bytes, Deflater.DEFAULT_COMPRESSION);
+    }
+
+    /**
+     * @param bytes Byte array to compress.
+     * @param compressionLevel Level of compression to encode.
+     * @return Compressed bytes.
+     * @throws IgniteCheckedException If failed.
+     */
+    public static byte[] zip(@Nullable byte[] bytes, int compressionLevel) throws IgniteCheckedException {
         try {
             if (bytes == null)
                 return null;
@@ -10222,6 +10235,8 @@ public static byte[] zip(@Nullable byte[] bytes) throws IgniteCheckedException {
             ByteArrayOutputStream bos = new ByteArrayOutputStream();
 
             try (ZipOutputStream zos = new ZipOutputStream(bos)) {
+                zos.setLevel(compressionLevel);
+
                 ZipEntry entry = new ZipEntry("");
 
                 try {
@@ -10514,9 +10529,12 @@ public static String toHexString(ByteBuffer buf) {
      * @param <T> Type of data.
      * @throws IgniteCheckedException if parallel execution was failed.
      */
-    public static <T> void doInParallel(ExecutorService executorSvc, Collection<T> srcDatas,
-        IgniteThrowableConsumer<T> operation) throws IgniteCheckedException, IgniteInterruptedCheckedException {
-        doInParallel(srcDatas.size(), executorSvc, srcDatas, operation);
+    public static <T, R> Collection<R> doInParallel(
+        ExecutorService executorSvc,
+        Collection<T> srcDatas,
+        IgniteThrowableConsumer<T, R> operation
+    ) throws IgniteCheckedException, IgniteInterruptedCheckedException {
+        return doInParallel(srcDatas.size(), executorSvc, srcDatas, operation);
     }
 
     /**
@@ -10527,38 +10545,58 @@ public static <T> void doInParallel(ExecutorService executorSvc, Collection<T> s
      * @param srcDatas List of data for parallelization.
      * @param operation Logic for execution of on each item of data.
      * @param <T> Type of data.
+     * @param <R> Type of return value.
      * @throws IgniteCheckedException if parallel execution was failed.
      */
-    public static <T> void doInParallel(
+    public static <T, R> Collection<R> doInParallel(
         int parallelismLvl,
         ExecutorService executorSvc,
         Collection<T> srcDatas,
-        IgniteThrowableConsumer<T> operation
+        IgniteThrowableConsumer<T, R> operation
     ) throws IgniteCheckedException, IgniteInterruptedCheckedException {
+        if(srcDatas.isEmpty())
+            return Collections.emptyList();
+
+        int batchSize = srcDatas.size() / parallelismLvl;
+
+        final int finalBatchSize = batchSize == 0 ? srcDatas.size() : batchSize;
+
         List<List<T>> batches = IntStream.range(0, parallelismLvl)
-            .mapToObj(i -> new ArrayList<T>())
+            .mapToObj(i -> new ArrayList<T>(finalBatchSize))
             .collect(Collectors.toList());
 
-        int i = 0;
+        int batchIndex = 0;
+
+        final int maxBatchIndex = batches.size() -1;
+
+        List<T> currentBatch = batches.get(batchIndex);
 
-        for (T src : srcDatas)
-            batches.get(i++ % parallelismLvl).add(src);
+        for (T src : srcDatas) {
+            currentBatch.add(src);
 
-        List<Future<Object>> consumerFutures = batches.stream()
+            if(currentBatch.size() >= batchSize && batchIndex < maxBatchIndex)
+                currentBatch = batches.get(++batchIndex);
+        }
+
+        List<Future<Collection<R>>> consumerFutures = batches.stream()
             .filter(batch -> !batch.isEmpty())
             .map(batch -> executorSvc.submit(() -> {
+                Collection<R> results = new ArrayList<>(batch.size());
+
                 for (T item : batch)
-                    operation.accept(item);
+                    results.add(operation.accept(item));
 
-                return null;
+                return results;
             }))
             .collect(Collectors.toList());
 
         Throwable error =null;
 
-        for (Future<Object> future : consumerFutures) {
+        Collection<R> results = new ArrayList<>(srcDatas.size());
+
+        for (Future<Collection<R>> future : consumerFutures) {
             try {
-                future.get();
+                results.addAll(future.get());
             }
             catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
@@ -10591,6 +10629,8 @@ public static <T> void doInParallel(
 
             throw new IgniteCheckedException(error);
         }
+
+        return results;
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java
index bb746a355304b..55feed8cc2d33 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/IgniteThrowableConsumer.java
@@ -25,13 +25,15 @@
  * interfaces, {@code IgniteThrowableConsumer} is expected to operate via side-effects.
  *
  * @param <E> Type of closure parameter.
+ * @param <R> Type of result value.
  */
-public interface IgniteThrowableConsumer<E> extends Serializable {
+public interface IgniteThrowableConsumer<E, R> extends Serializable {
     /**
      * Consumer body.
      *
      * @param e Consumer parameter.
+     * @return Result of consumer operation.
      * @throws IgniteCheckedException if body execution was failed.
      */
-    public void accept(E e) throws IgniteCheckedException;
-}
\ No newline at end of file
+    public R accept(E e) throws IgniteCheckedException;
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
index c06922ea1be91..5967efcb668a9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
@@ -45,9 +45,11 @@
 import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.function.Consumer;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterGroup;
@@ -857,18 +859,26 @@ public void testCeilPow2() throws Exception {
     public void testDoInParallel() throws Throwable {
         CyclicBarrier barrier = new CyclicBarrier(3);
 
-        IgniteUtils.doInParallel(3,
-            Executors.newFixedThreadPool(3),
-            Arrays.asList(1, 2, 3),
-            i -> {
-                try {
-                    barrier.await(1, TimeUnit.SECONDS);
-                }
-                catch (Exception e) {
-                    throw new IgniteCheckedException(e);
+        ExecutorService executorService = Executors.newFixedThreadPool(3);
+
+        try {
+            IgniteUtils.doInParallel(3,
+                executorService,
+                Arrays.asList(1, 2, 3),
+                i -> {
+                    try {
+                        barrier.await(1, TimeUnit.SECONDS);
+                    }
+                    catch (Exception e) {
+                        throw new IgniteCheckedException(e);
+                    }
+
+                    return null;
                 }
-            }
-        );
+            );
+        } finally {
+            executorService.shutdownNow();
+        }
     }
 
     /**
@@ -877,9 +887,11 @@ public void testDoInParallel() throws Throwable {
     public void testDoInParallelBatch() {
         CyclicBarrier barrier = new CyclicBarrier(3);
 
+        ExecutorService executorService = Executors.newFixedThreadPool(3);
+
         try {
             IgniteUtils.doInParallel(2,
-                Executors.newFixedThreadPool(3),
+                executorService,
                 Arrays.asList(1, 2, 3),
                 i -> {
                     try {
@@ -888,6 +900,8 @@ public void testDoInParallelBatch() {
                     catch (Exception e) {
                         throw new IgniteCheckedException(e);
                     }
+
+                    return null;
                 }
             );
 
@@ -895,22 +909,98 @@ public void testDoInParallelBatch() {
         }
         catch (Exception e) {
             assertTrue(e.toString(), X.hasCause(e, TimeoutException.class));
+        } finally {
+            executorService.shutdownNow();
         }
     }
 
+    /**
+     * Test parallel execution in order.
+     */
+    public void testDoInParallelResultsOrder() throws IgniteCheckedException {
+        ExecutorService executorService = Executors.newFixedThreadPool(4);
+
+        try {
+            /*testOrder(executorService, 1, 1);
+            testOrder(executorService, 2, 1);
+            testOrder(executorService, 3, 1);
+            testOrder(executorService, 9, 1);
+            testOrder(executorService, 10, 1);
+            testOrder(executorService, 9999, 1);
+
+            testOrder(executorService, 1, 2);
+            testOrder(executorService, 2, 2);
+            testOrder(executorService, 3, 2);
+            testOrder(executorService, 9, 2);
+            testOrder(executorService, 10, 2);*/
+            testOrder(executorService, 9999, 2);
+
+            testOrder(executorService, 1, 3);
+            testOrder(executorService, 2, 3);
+            testOrder(executorService, 3, 3);
+            testOrder(executorService, 3, 3);
+            testOrder(executorService, 10, 3);
+            testOrder(executorService, 9999, 3);
+
+            testOrder(executorService, 1, 4);
+            testOrder(executorService, 2, 4);
+            testOrder(executorService, 3, 4);
+            testOrder(executorService, 9, 4);
+            testOrder(executorService, 10, 4);
+            testOrder(executorService, 9999, 4);
+        } finally {
+            executorService.shutdownNow();
+        }
+    }
+
+    /**
+     * Template method to test parallel execution
+     * @param executorService ExecutorService.
+     * @param size Size.
+     * @param parallelism Parallelism.
+     * @throws IgniteCheckedException Exception.
+     */
+    private void testOrder(ExecutorService executorService, int size, int parallelism) throws IgniteCheckedException {
+        List<Integer> list = new ArrayList<>();
+        for(int i = 0; i < size; i++)
+            list.add(i);
+
+        Collection<Integer> results = IgniteUtils.doInParallel(
+            parallelism,
+            executorService,
+            list,
+            i -> i * 2
+        );
+
+        assertEquals(list.size(), results.size());
+
+        final int[] i = {0};
+        results.forEach(new Consumer<Integer>() {
+            @Override public void accept(Integer integer) {
+                assertEquals(2 * list.get(i[0]), integer.intValue());
+                i[0]++;
+            }
+        });
+    }
+
     /**
      *
      */
     public void testDoInParallelException() {
         String expectedException = "ExpectedException";
 
+        ExecutorService executorService = Executors.newFixedThreadPool(1);
+
         try {
-            IgniteUtils.doInParallel(3,
-                Executors.newFixedThreadPool(1),
+            IgniteUtils.doInParallel(
+                1,
+                executorService,
                 Arrays.asList(1, 2, 3),
                 i -> {
-                    if (i == 1)
+                    if (Integer.valueOf(1).equals(i))
                         throw new IgniteCheckedException(expectedException);
+
+                    return  null;
                 }
             );
 
@@ -918,6 +1008,8 @@ public void testDoInParallelException() {
         }
         catch (IgniteCheckedException e) {
             assertEquals(expectedException, e.getMessage());
+        } finally {
+            executorService.shutdownNow();
         }
     }
 
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
index bf34fc0214b46..301a63e2f0396 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
@@ -81,7 +81,8 @@ public class IgniteConfigurationParityTest
             "TimeServerPortRange",
             "IncludeProperties",
             "isAutoActivationEnabled",  // IGNITE-7301
-            "failureHandler"            // IGNITE-8070
+            "failureHandler",            // IGNITE-8070
+            "NetworkCompressionLevel"
         };
 
         /// <summary>
@@ -98,4 +99,4 @@ public void TestIgniteConfiguration()
                 KnownMappings);
         }
     }
-}
\ No newline at end of file
+}

From 61f26adf0a3a509e168cd82b52a280d37c0afa21 Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Fri, 9 Nov 2018 17:57:17 +0300
Subject: [PATCH 486/543] IGNITE-9970 Added ability to set node ID for visor
 idle verify dump task - Fixes #5227.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 1fee1fd18025bb8641199ce980563fa80f2f136f)
---
 .../internal/commandline/CommandHandler.java  | 73 ++++++++++++++++---
 1 file changed, 62 insertions(+), 11 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index a5afcc69f331d..e5488bb63e318 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.internal.commandline;
 
 import java.io.Console;
+import java.io.IOException;
+import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -34,6 +36,7 @@
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeTask;
@@ -66,6 +69,7 @@
 import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2;
+import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.SB;
@@ -116,6 +120,7 @@
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTask;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskResult;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
@@ -622,6 +627,35 @@ private <R> R executeTask(GridClient client, Class<? extends ComputeTask<?, R>>
         return executeTaskByNameOnNode(client, taskCls.getName(), taskArgs, null);
     }
 
+    /**
+     * @param client Client.
+     *
+     * @return List of hosts.
+     */
+    private Stream<IgniteBiTuple<GridClientNode, String>> listHosts(GridClient client) throws GridClientException {
+        return client.compute().nodes(GridClientNode::connectable).stream()
+            .flatMap(node -> Stream.concat(
+                node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
+                node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
+            )
+            .map(addr -> new IgniteBiTuple<>(node, addr + ":" + node.tcpPort())));
+    }
+
+    /**
+     * @param client Client.
+     *
+     * @return List of hosts.
+     */
+    private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNode(GridClient client) throws GridClientException {
+        return client.compute().nodes(GridClientNode::connectable).stream()
+            .map(node -> new IgniteBiTuple<>(node,
+                Stream.concat(
+                    node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
+                    node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
+                )
+                .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
+    }
+
     /**
      * @param client Client
      * @param taskClsName Task class name.
@@ -630,7 +664,12 @@ private <R> R executeTask(GridClient client, Class<? extends ComputeTask<?, R>>
      * @return Task result.
      * @throws GridClientException If failed to execute task.
      */
-    private <R> R executeTaskByNameOnNode(GridClient client, String taskClsName, Object taskArgs, UUID nodeId
+    @SuppressWarnings("unchecked")
+    private <R> R executeTaskByNameOnNode(
+        GridClient client,
+        String taskClsName,
+        Object taskArgs,
+        UUID nodeId
     ) throws GridClientException {
         GridClientCompute compute = client.compute();
 
@@ -650,22 +689,34 @@ private <R> R executeTaskByNameOnNode(GridClient client, String taskClsName, Obj
         GridClientNode node = null;
 
         if (nodeId == null) {
-            Collection<GridClientNode> nodes = compute.nodes(GridClientNode::connectable);
-
             // Prefer node from connect string.
-            String origAddr = clientCfg.getServers().iterator().next();
+            final String cfgAddr = clientCfg.getServers().iterator().next();
 
-            for (GridClientNode clientNode : nodes) {
-                Iterator<String> it = F.concat(clientNode.tcpAddresses().iterator(), clientNode.tcpHostNames().iterator());
+            String[] parts = cfgAddr.split(":");
 
-                while (it.hasNext()) {
-                    if (origAddr.equals(it.next() + ":" + clientNode.tcpPort())) {
-                        node = clientNode;
+            if (DFLT_HOST.equals(parts[0])) {
+                InetAddress addr;
 
-                        break;
-                    }
+                try {
+                    addr = IgniteUtils.getLocalHost();
                 }
+                catch (IOException e) {
+                    throw new GridClientException("Can't get localhost name.", e);
+                }
+
+                if (addr.isLoopbackAddress())
+                    throw new GridClientException("Can't find localhost name.");
+
+                String origAddr = addr.getHostName() + ":" + parts[1];
+
+                node = listHosts(client).filter(tuple -> origAddr.equals(tuple.get2())).findFirst().map(IgniteBiTuple::get1).orElse(null);
+
+                if (node == null)
+                    node = listHostsByClientNode(client).filter(tuple -> tuple.get2().size() == 1 && cfgAddr.equals(tuple.get2().get(0))).
+                        findFirst().map(IgniteBiTuple::get1).orElse(null);
             }
+            else
+                node = listHosts(client).filter(tuple -> cfgAddr.equals(tuple.get2())).findFirst().map(IgniteBiTuple::get1).orElse(null);
 
             // Otherwise choose random node.
             if (node == null)

From 1e40b92236bd5f31627226d7a5d7cdad9425cb3a Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Mon, 12 Nov 2018 15:58:42 +0300
Subject: [PATCH 487/543] IGNITE-10085 Made compressed WAL archives more
 user-friendly - Fixes #5327.

(cherry picked from commit 24ed9a1db22a6a65eefaf9eb5a497d802aa13ca6)
---
 .../cache/persistence/wal/FileWriteAheadLogManager.java         | 2 +-
 .../persistence/wal/FsyncModeFileWriteAheadLogManager.java      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index dc3e5b80ae382..a250b471c016f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1983,7 +1983,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
 
             try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(zip)))) {
                 zos.setLevel(dsCfg.getWalCompactionLevel());
-                zos.putNextEntry(new ZipEntry(""));
+                zos.putNextEntry(new ZipEntry(nextSegment + ".wal"));
 
                 ByteBuffer buf = ByteBuffer.allocate(HEADER_RECORD_SIZE);
                 buf.order(ByteOrder.nativeOrder());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index 51405243e0ba5..57fb8c7de284b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -1912,7 +1912,7 @@ private void compressSegmentToFile(long nextSegment, File raw, File zip)
 
             try (ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(zip)))) {
                 zos.setLevel(dsCfg.getWalCompactionLevel());
-                zos.putNextEntry(new ZipEntry(""));
+                zos.putNextEntry(new ZipEntry(nextSegment + ".wal"));
 
                 zos.write(prepareSerializerVersionBuffer(nextSegment, segmentSerializerVer, true).array());
 

From cf9a7702725037880a9a26a26fea27ecc64f2b5e Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Mon, 12 Nov 2018 21:34:06 +0300
Subject: [PATCH 488/543] Prepare branch for p151 release.

---
 examples/pom.xml                                   | 2 +-
 modules/aop/pom.xml                                | 2 +-
 modules/apache-license-gen/pom.xml                 | 2 +-
 modules/aws/pom.xml                                | 2 +-
 modules/benchmarks/pom.xml                         | 2 +-
 modules/camel/pom.xml                              | 2 +-
 modules/cassandra/pom.xml                          | 2 +-
 modules/cassandra/serializers/pom.xml              | 4 ++--
 modules/cassandra/store/pom.xml                    | 4 ++--
 modules/clients/pom.xml                            | 2 +-
 modules/cloud/pom.xml                              | 2 +-
 modules/codegen/pom.xml                            | 2 +-
 modules/compatibility/pom.xml                      | 2 +-
 modules/core/pom.xml                               | 2 +-
 modules/core/src/main/resources/ignite.properties  | 2 +-
 modules/dev-utils/pom.xml                          | 2 +-
 modules/direct-io/pom.xml                          | 2 +-
 modules/extdata/p2p/pom.xml                        | 2 +-
 modules/extdata/platform/pom.xml                   | 2 +-
 modules/extdata/uri/modules/uri-dependency/pom.xml | 2 +-
 modules/extdata/uri/pom.xml                        | 2 +-
 modules/flink/pom.xml                              | 2 +-
 modules/flume/pom.xml                              | 2 +-
 modules/gce/pom.xml                                | 2 +-
 modules/geospatial/pom.xml                         | 2 +-
 modules/hadoop/pom.xml                             | 2 +-
 modules/hibernate-4.2/pom.xml                      | 2 +-
 modules/hibernate-5.1/pom.xml                      | 2 +-
 modules/hibernate-core/pom.xml                     | 2 +-
 modules/ignored-tests/pom.xml                      | 2 +-
 modules/indexing/pom.xml                           | 2 +-
 modules/jcl/pom.xml                                | 2 +-
 modules/jms11/pom.xml                              | 2 +-
 modules/jta/pom.xml                                | 2 +-
 modules/kafka/pom.xml                              | 2 +-
 modules/kubernetes/pom.xml                         | 2 +-
 modules/log4j/pom.xml                              | 2 +-
 modules/log4j2/pom.xml                             | 2 +-
 modules/mesos/pom.xml                              | 2 +-
 modules/ml/pom.xml                                 | 2 +-
 modules/mqtt/pom.xml                               | 2 +-
 modules/osgi-karaf/pom.xml                         | 2 +-
 modules/osgi-paxlogging/pom.xml                    | 2 +-
 modules/osgi/pom.xml                               | 2 +-
 modules/rest-http/pom.xml                          | 2 +-
 modules/rocketmq/pom.xml                           | 2 +-
 modules/scalar-2.10/pom.xml                        | 2 +-
 modules/scalar/pom.xml                             | 2 +-
 modules/schedule/pom.xml                           | 2 +-
 modules/slf4j/pom.xml                              | 2 +-
 modules/spark-2.10/pom.xml                         | 2 +-
 modules/spark/pom.xml                              | 2 +-
 modules/spring-data/pom.xml                        | 2 +-
 modules/spring/pom.xml                             | 2 +-
 modules/sqlline/pom.xml                            | 2 +-
 modules/ssh/pom.xml                                | 2 +-
 modules/storm/pom.xml                              | 2 +-
 modules/tools/pom.xml                              | 2 +-
 modules/twitter/pom.xml                            | 2 +-
 modules/urideploy/pom.xml                          | 2 +-
 modules/visor-console-2.10/pom.xml                 | 2 +-
 modules/visor-console/pom.xml                      | 2 +-
 modules/visor-plugins/pom.xml                      | 2 +-
 modules/web-console/pom.xml                        | 2 +-
 modules/web-console/web-agent/pom.xml              | 2 +-
 modules/web/ignite-appserver-test/pom.xml          | 2 +-
 modules/web/ignite-websphere-test/pom.xml          | 2 +-
 modules/web/pom.xml                                | 2 +-
 modules/yardstick/pom.xml                          | 2 +-
 modules/yarn/pom.xml                               | 2 +-
 modules/zeromq/pom.xml                             | 2 +-
 modules/zookeeper/pom.xml                          | 2 +-
 pom.xml                                            | 2 +-
 73 files changed, 75 insertions(+), 75 deletions(-)

diff --git a/examples/pom.xml b/examples/pom.xml
index 299363ac31316..01a570d8e2cf3 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -27,7 +27,7 @@
     </parent>
 
     <artifactId>ignite-examples</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/aop/pom.xml b/modules/aop/pom.xml
index 6f10f6ec31630..c3f250d3ff0f0 100644
--- a/modules/aop/pom.xml
+++ b/modules/aop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aop</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/apache-license-gen/pom.xml b/modules/apache-license-gen/pom.xml
index 68494fc00b488..662525a6e5143 100644
--- a/modules/apache-license-gen/pom.xml
+++ b/modules/apache-license-gen/pom.xml
@@ -31,7 +31,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>ignite-apache-license-gen</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <build>
diff --git a/modules/aws/pom.xml b/modules/aws/pom.xml
index d2fb36cd76425..39754904bf77b 100644
--- a/modules/aws/pom.xml
+++ b/modules/aws/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aws</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/benchmarks/pom.xml b/modules/benchmarks/pom.xml
index 865ad53d5b688..fe6a11207988f 100644
--- a/modules/benchmarks/pom.xml
+++ b/modules/benchmarks/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-benchmarks</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/camel/pom.xml b/modules/camel/pom.xml
index a295a46928ef3..77ecff0f65660 100644
--- a/modules/camel/pom.xml
+++ b/modules/camel/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-camel</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/pom.xml b/modules/cassandra/pom.xml
index 4914c77d0c6ed..82deb74e1ff58 100644
--- a/modules/cassandra/pom.xml
+++ b/modules/cassandra/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-cassandra</artifactId>
     <packaging>pom</packaging>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencyManagement>
diff --git a/modules/cassandra/serializers/pom.xml b/modules/cassandra/serializers/pom.xml
index 8703d77367dd3..6cf30d92a1dd0 100644
--- a/modules/cassandra/serializers/pom.xml
+++ b/modules/cassandra/serializers/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>2.5.1-p150-SNAPSHOT</version>
+        <version>2.5.1-p151-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-serializers</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/store/pom.xml b/modules/cassandra/store/pom.xml
index 3a0da5191ed36..6096b707d2600 100644
--- a/modules/cassandra/store/pom.xml
+++ b/modules/cassandra/store/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>2.5.1-p150-SNAPSHOT</version>
+        <version>2.5.1-p151-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-store</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/clients/pom.xml b/modules/clients/pom.xml
index 8df6a8c85fc5c..6a1a5b46aa4c1 100644
--- a/modules/clients/pom.xml
+++ b/modules/clients/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-clients</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/cloud/pom.xml b/modules/cloud/pom.xml
index c750f609ecc47..7c6484c204380 100644
--- a/modules/cloud/pom.xml
+++ b/modules/cloud/pom.xml
@@ -29,7 +29,7 @@
     </parent>
 
     <artifactId>ignite-cloud</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/codegen/pom.xml b/modules/codegen/pom.xml
index 2320f56b42b42..8e962ba53ef86 100644
--- a/modules/codegen/pom.xml
+++ b/modules/codegen/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-codegen</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/compatibility/pom.xml b/modules/compatibility/pom.xml
index 1794f4375865a..f1cbfde5f6cce 100644
--- a/modules/compatibility/pom.xml
+++ b/modules/compatibility/pom.xml
@@ -33,7 +33,7 @@
     </parent>
 
     <artifactId>ignite-compatibility</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 491c85c0dc3db..d79df6b01ab9c 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-core</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <repositories>
diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index db63ec1062b28..f4702c6ff409d 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=2.5.1-p150-SNAPSHOT
+ignite.version=2.5.1-p151-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970
diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml
index afd6d8858f8e6..9ebd1c18df673 100644
--- a/modules/dev-utils/pom.xml
+++ b/modules/dev-utils/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-dev-utils</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/direct-io/pom.xml b/modules/direct-io/pom.xml
index 628a23ee95290..a3ebe2f6faaae 100644
--- a/modules/direct-io/pom.xml
+++ b/modules/direct-io/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-direct-io</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/extdata/p2p/pom.xml b/modules/extdata/p2p/pom.xml
index 46ae0fccde7ec..32e2f23aa6d8f 100644
--- a/modules/extdata/p2p/pom.xml
+++ b/modules/extdata/p2p/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-extdata-p2p</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/extdata/platform/pom.xml b/modules/extdata/platform/pom.xml
index 523f8aa6ccd43..a7292b5056ad9 100644
--- a/modules/extdata/platform/pom.xml
+++ b/modules/extdata/platform/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-platform</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/extdata/uri/modules/uri-dependency/pom.xml b/modules/extdata/uri/modules/uri-dependency/pom.xml
index 06d208ab07aef..73fc854010ce4 100644
--- a/modules/extdata/uri/modules/uri-dependency/pom.xml
+++ b/modules/extdata/uri/modules/uri-dependency/pom.xml
@@ -27,7 +27,7 @@
     <artifactId>ignite-extdata-uri-dep</artifactId>
     <packaging>jar</packaging>
 
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <modelVersion>4.0.0</modelVersion>
 
     <dependencies>
diff --git a/modules/extdata/uri/pom.xml b/modules/extdata/uri/pom.xml
index f3a6a3d64fd0a..4567dd4befcbd 100644
--- a/modules/extdata/uri/pom.xml
+++ b/modules/extdata/uri/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-uri</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/flink/pom.xml b/modules/flink/pom.xml
index deaa81bf757e2..5e4c10bc4ecbc 100644
--- a/modules/flink/pom.xml
+++ b/modules/flink/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flink</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/flume/pom.xml b/modules/flume/pom.xml
index b67d341beddab..c1b60510ba258 100644
--- a/modules/flume/pom.xml
+++ b/modules/flume/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flume</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/gce/pom.xml b/modules/gce/pom.xml
index b740bb7a6d0ef..a9b55c357a4fd 100644
--- a/modules/gce/pom.xml
+++ b/modules/gce/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-gce</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/geospatial/pom.xml b/modules/geospatial/pom.xml
index feaea2061855b..3f710f7097b0f 100644
--- a/modules/geospatial/pom.xml
+++ b/modules/geospatial/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-geospatial</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index 52599df3bb03c..559f11a971095 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hadoop</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-4.2/pom.xml b/modules/hibernate-4.2/pom.xml
index 75f99322e9adc..ebc90c4252b56 100644
--- a/modules/hibernate-4.2/pom.xml
+++ b/modules/hibernate-4.2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate_4.2</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-5.1/pom.xml b/modules/hibernate-5.1/pom.xml
index fa675f339c5fd..ca88c795a424a 100644
--- a/modules/hibernate-5.1/pom.xml
+++ b/modules/hibernate-5.1/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate_5.1</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-core/pom.xml b/modules/hibernate-core/pom.xml
index c31b546306bdd..c85565e1ef02f 100644
--- a/modules/hibernate-core/pom.xml
+++ b/modules/hibernate-core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate-core</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ignored-tests/pom.xml b/modules/ignored-tests/pom.xml
index bacbecc968338..eaa7f0d790e8f 100644
--- a/modules/ignored-tests/pom.xml
+++ b/modules/ignored-tests/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ignored-tests</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml
index 46bfadaa7625f..c0181e7f9ae17 100644
--- a/modules/indexing/pom.xml
+++ b/modules/indexing/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-indexing</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jcl/pom.xml b/modules/jcl/pom.xml
index 2208eda8afe63..8ce3b13c1d6bd 100644
--- a/modules/jcl/pom.xml
+++ b/modules/jcl/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jcl</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jms11/pom.xml b/modules/jms11/pom.xml
index 29f29d44d6360..5ff3885a92f15 100644
--- a/modules/jms11/pom.xml
+++ b/modules/jms11/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jms11</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jta/pom.xml b/modules/jta/pom.xml
index c0f8deea21d16..10806ed2642fd 100644
--- a/modules/jta/pom.xml
+++ b/modules/jta/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jta</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kafka/pom.xml b/modules/kafka/pom.xml
index ef9dc5ab24538..9e65af84803ea 100644
--- a/modules/kafka/pom.xml
+++ b/modules/kafka/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kafka</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kubernetes/pom.xml b/modules/kubernetes/pom.xml
index 39376e0f4edf3..1cc9ab24b6ff8 100644
--- a/modules/kubernetes/pom.xml
+++ b/modules/kubernetes/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kubernetes</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j/pom.xml b/modules/log4j/pom.xml
index a84b8175f5ae4..b3d47c3b9099d 100644
--- a/modules/log4j/pom.xml
+++ b/modules/log4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j2/pom.xml b/modules/log4j2/pom.xml
index c36d48ec6ac4e..5f326e09eee85 100644
--- a/modules/log4j2/pom.xml
+++ b/modules/log4j2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j2</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml
index 8926a82a0ca10..1fc84c29d48d6 100644
--- a/modules/mesos/pom.xml
+++ b/modules/mesos/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mesos</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml
index bdb87628e4c24..69400b9bfa186 100644
--- a/modules/ml/pom.xml
+++ b/modules/ml/pom.xml
@@ -30,7 +30,7 @@
     </parent>
 
     <artifactId>ignite-ml</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mqtt/pom.xml b/modules/mqtt/pom.xml
index fe7b9f3419425..f106cef3577ec 100644
--- a/modules/mqtt/pom.xml
+++ b/modules/mqtt/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mqtt</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/osgi-karaf/pom.xml b/modules/osgi-karaf/pom.xml
index 5c531760d88f6..76dc7259d0a25 100644
--- a/modules/osgi-karaf/pom.xml
+++ b/modules/osgi-karaf/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-karaf</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <build>
diff --git a/modules/osgi-paxlogging/pom.xml b/modules/osgi-paxlogging/pom.xml
index 0ae571685816d..856d21d92ccb4 100644
--- a/modules/osgi-paxlogging/pom.xml
+++ b/modules/osgi-paxlogging/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-paxlogging</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <dependencies>
diff --git a/modules/osgi/pom.xml b/modules/osgi/pom.xml
index b539c94f3a1db..746109edbbc34 100644
--- a/modules/osgi/pom.xml
+++ b/modules/osgi/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/rest-http/pom.xml b/modules/rest-http/pom.xml
index ffa3385c48716..0227f2378fd2e 100644
--- a/modules/rest-http/pom.xml
+++ b/modules/rest-http/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-rest-http</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/rocketmq/pom.xml b/modules/rocketmq/pom.xml
index 7949aa3b21edb..a96b49fba36b0 100644
--- a/modules/rocketmq/pom.xml
+++ b/modules/rocketmq/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-rocketmq</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar-2.10/pom.xml b/modules/scalar-2.10/pom.xml
index 54d43ac75a02b..786ae726842b0 100644
--- a/modules/scalar-2.10/pom.xml
+++ b/modules/scalar-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar_2.10</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml
index 1f63009e0520f..0590230898620 100644
--- a/modules/scalar/pom.xml
+++ b/modules/scalar/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/schedule/pom.xml b/modules/schedule/pom.xml
index 82abbf2a8d866..641f5f1ec53a9 100644
--- a/modules/schedule/pom.xml
+++ b/modules/schedule/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-schedule</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/slf4j/pom.xml b/modules/slf4j/pom.xml
index 3a6754469f13e..7792a377f3452 100644
--- a/modules/slf4j/pom.xml
+++ b/modules/slf4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-slf4j</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark-2.10/pom.xml b/modules/spark-2.10/pom.xml
index e74488245dbe7..2e86061a863f7 100644
--- a/modules/spark-2.10/pom.xml
+++ b/modules/spark-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark_2.10</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index 99198cd3923b1..e535630444190 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring-data/pom.xml b/modules/spring-data/pom.xml
index e849f3a6ffd8a..c57bdb17ace5b 100644
--- a/modules/spring-data/pom.xml
+++ b/modules/spring-data/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring-data</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring/pom.xml b/modules/spring/pom.xml
index bc849af8cce05..ed4f876d8ec0f 100644
--- a/modules/spring/pom.xml
+++ b/modules/spring/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/sqlline/pom.xml b/modules/sqlline/pom.xml
index 34aaaffc41b3b..6c3afc63c6c5d 100644
--- a/modules/sqlline/pom.xml
+++ b/modules/sqlline/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-sqlline</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ssh/pom.xml b/modules/ssh/pom.xml
index 4504279c876fc..6d3310dd67b45 100644
--- a/modules/ssh/pom.xml
+++ b/modules/ssh/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ssh</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/storm/pom.xml b/modules/storm/pom.xml
index 3077b168ac639..58b90f6109f25 100644
--- a/modules/storm/pom.xml
+++ b/modules/storm/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-storm</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/tools/pom.xml b/modules/tools/pom.xml
index 32656daf86e12..0896f56d1a930 100644
--- a/modules/tools/pom.xml
+++ b/modules/tools/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-tools</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/twitter/pom.xml b/modules/twitter/pom.xml
index ef491ab283578..96268f20a5b3f 100644
--- a/modules/twitter/pom.xml
+++ b/modules/twitter/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-twitter</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/urideploy/pom.xml b/modules/urideploy/pom.xml
index 8d194e1485f3b..544c446b9d8b5 100644
--- a/modules/urideploy/pom.xml
+++ b/modules/urideploy/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-urideploy</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console-2.10/pom.xml b/modules/visor-console-2.10/pom.xml
index f858f588a0b98..bd515eeb92255 100644
--- a/modules/visor-console-2.10/pom.xml
+++ b/modules/visor-console-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console_2.10</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console/pom.xml b/modules/visor-console/pom.xml
index 39a44113cfe35..e018e6acb2e92 100644
--- a/modules/visor-console/pom.xml
+++ b/modules/visor-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-plugins/pom.xml b/modules/visor-plugins/pom.xml
index 3484fffe83058..bfe3a5b06feba 100644
--- a/modules/visor-plugins/pom.xml
+++ b/modules/visor-plugins/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-plugins</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web-console/pom.xml b/modules/web-console/pom.xml
index 57cda7ba9361a..c8cbeef9239a7 100644
--- a/modules/web-console/pom.xml
+++ b/modules/web-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web-console</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml
index 80284f1eb6c81..bf36b844e675f 100644
--- a/modules/web-console/web-agent/pom.xml
+++ b/modules/web-console/web-agent/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-web-agent</artifactId>
     <packaging>jar</packaging>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web/ignite-appserver-test/pom.xml b/modules/web/ignite-appserver-test/pom.xml
index 397b4f6135f9e..ebdd8a9822cc0 100644
--- a/modules/web/ignite-appserver-test/pom.xml
+++ b/modules/web/ignite-appserver-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-appserver-test</artifactId>
     <packaging>jar</packaging>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/ignite-websphere-test/pom.xml b/modules/web/ignite-websphere-test/pom.xml
index 8fc17b83f164f..1fa01875d8fa8 100644
--- a/modules/web/ignite-websphere-test/pom.xml
+++ b/modules/web/ignite-websphere-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-websphere-test</artifactId>
     <packaging>war</packaging>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/pom.xml b/modules/web/pom.xml
index e2e7e210a5ec7..6149785ea9ce6 100644
--- a/modules/web/pom.xml
+++ b/modules/web/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index f55c5c495258a..3c1170613baf8 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yardstick</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml
index a38757919eb78..6cbd28bc9fee9 100644
--- a/modules/yarn/pom.xml
+++ b/modules/yarn/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yarn</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zeromq/pom.xml b/modules/zeromq/pom.xml
index bf8caae879433..9136340e71575 100644
--- a/modules/zeromq/pom.xml
+++ b/modules/zeromq/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zeromq</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index 9e36018c4e0eb..532bb047ec351 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zookeeper</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/pom.xml b/pom.xml
index a28464d1e87d4..c266ed4434ddf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>apache-ignite</artifactId>
-    <version>2.5.1-p150-SNAPSHOT</version>
+    <version>2.5.1-p151-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <properties>

From 14649053ed4d86aeda64911b36db0b6b47c3309b Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Tue, 13 Nov 2018 11:26:36 +0300
Subject: [PATCH 489/543] IGNITE-10134 Add exception handling for job reduce
 results - Fixes #5250.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit ded41e9d126fea58e40c03d7ba86b577dbeb47ca)
---
 .../VisorCacheConfigurationCollectorTask.java | 40 +++++++++++
 .../ignite/util/GridCommandHandlerTest.java   | 72 +++++++++++++++----
 2 files changed, 99 insertions(+), 13 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java
index fd224a893eb39..418eb1b57b81b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/VisorCacheConfigurationCollectorTask.java
@@ -17,9 +17,15 @@
 
 package org.apache.ignite.internal.visor.cache;
 
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.compute.ComputeJobResult;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.visor.VisorOneNodeTask;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * Task that collect cache metrics from all nodes.
@@ -34,4 +40,38 @@ public class VisorCacheConfigurationCollectorTask
     @Override protected VisorCacheConfigurationCollectorJob job(VisorCacheConfigurationCollectorTaskArg arg) {
         return new VisorCacheConfigurationCollectorJob(arg, debug);
     }
+
+    /** {@inheritDoc} */
+    @Override protected @Nullable Map<String, VisorCacheConfiguration> reduce0(
+        List<ComputeJobResult> results
+    ) throws IgniteException {
+        if (results == null)
+            return null;
+
+        Map<String, VisorCacheConfiguration> map = new HashMap<>();
+
+        List<Exception> resultsExceptions = null;
+
+        for (ComputeJobResult res : results) {
+            if (res.getException() == null)
+                map.putAll(res.getData());
+            else {
+                if (resultsExceptions == null)
+                    resultsExceptions = new ArrayList<>(results.size());
+
+                resultsExceptions.add(new IgniteException("Job failed on node: " + res.getNode().id(), res.getException()));
+            }
+        }
+
+        if (resultsExceptions != null) {
+            IgniteException e = new IgniteException("Reduce failed because of job failed on some nodes");
+
+            for (Exception ex : resultsExceptions)
+                e.addSuppressed(ex);
+
+            throw e;
+        }
+
+        return map;
+    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index c8fb1d2b06d2d..88bd8961b1cdb 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -1264,29 +1264,71 @@ public void testCacheAffinity() throws Exception {
 
     /** */
     public void testCacheConfigNoOutputFormat() throws Exception {
-        testCacheConfig(null);
+        testCacheConfig(null, 1, 1);
     }
 
     /** */
-    public void testCacheConfigSingleLineOutputFormat() throws Exception {
-        testCacheConfig(SINGLE_LINE);
+    public void testCacheConfigSingleLineOutputFormatSingleNodeSignleCache() throws Exception {
+        testCacheConfigSingleLineOutputFormat(1, 1);
     }
 
     /** */
-    public void testCacheConfigMultiLineOutputFormat() throws Exception {
-        testCacheConfig(MULTI_LINE);
+    public void testCacheConfigSingleLineOutputFormatTwoNodeSignleCache() throws Exception {
+        testCacheConfigSingleLineOutputFormat(2, 1);
     }
 
     /** */
-    private void testCacheConfig(OutputFormat outputFormat) throws Exception {
-        Ignite ignite = startGrid();
+    public void testCacheConfigSingleLineOutputFormatTwoNodeManyCaches() throws Exception {
+        testCacheConfigSingleLineOutputFormat(2, 100);
+    }
+
+    /** */
+    public void testCacheConfigMultiLineOutputFormatSingleNodeSingleCache() throws Exception {
+        testCacheConfigMultiLineOutputFormat(1, 1);
+    }
+
+    /** */
+    public void testCacheConfigMultiLineOutputFormatTwoNodeSingleCache() throws Exception {
+        testCacheConfigMultiLineOutputFormat(2, 1);
+    }
+
+    /** */
+    public void testCacheConfigMultiLineOutputFormatTwoNodeManyCaches() throws Exception {
+        testCacheConfigMultiLineOutputFormat(2, 100);
+    }
+
+    /** */
+    private void testCacheConfigSingleLineOutputFormat(int nodesCnt, int cachesCnt) throws Exception {
+        testCacheConfig(SINGLE_LINE, nodesCnt, cachesCnt);
+    }
+
+    /** */
+    private void testCacheConfigMultiLineOutputFormat(int nodesCnt, int cachesCnt) throws Exception {
+        testCacheConfig(MULTI_LINE, nodesCnt, cachesCnt);
+    }
+
+    /** */
+    private void testCacheConfig(OutputFormat outputFormat, int nodesCnt, int cachesCnt) throws Exception {
+        assertTrue("Invalid number of nodes or caches", nodesCnt > 0 && cachesCnt > 0);
+
+        Ignite ignite = startGrid(nodesCnt);
 
         ignite.cluster().active(true);
 
-        IgniteCache<Object, Object> cache1 = ignite.createCache(new CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, 32))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
+        List<CacheConfiguration> ccfgs = new ArrayList<>(cachesCnt);
+
+        for (int i = 0; i < cachesCnt; i++) {
+            ccfgs.add(
+                new CacheConfiguration<>()
+                    .setAffinity(new RendezvousAffinityFunction(false, 32))
+                    .setBackups(1)
+                    .setName(DEFAULT_CACHE_NAME + i)
+            );
+        }
+
+        ignite.createCaches(ccfgs);
+
+        IgniteCache<Object, Object> cache1 = ignite.cache(DEFAULT_CACHE_NAME + 0);
 
         for (int i = 0; i < 100; i++)
             cache1.put(i, i);
@@ -1305,12 +1347,16 @@ private void testCacheConfig(OutputFormat outputFormat) throws Exception {
         String outStr = testOut.toString();
 
         if (outputFormat == null || outputFormat == SINGLE_LINE) {
-            assertTrue(outStr.contains("name=" + DEFAULT_CACHE_NAME));
+            for (int i = 0; i < cachesCnt; i++)
+                assertTrue(outStr.contains("name=" + DEFAULT_CACHE_NAME + i));
+
             assertTrue(outStr.contains("partitions=32"));
             assertTrue(outStr.contains("function=o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction"));
         }
         else if (outputFormat == MULTI_LINE) {
-            assertTrue(outStr.contains("[cache = '" + DEFAULT_CACHE_NAME + "']"));
+            for (int i = 0; i < cachesCnt; i++)
+                assertTrue(outStr.contains("[cache = '" + DEFAULT_CACHE_NAME + i + "']"));
+
             assertTrue(outStr.contains("Affinity Partitions: 32"));
             assertTrue(outStr.contains("Affinity Function: o.a.i.cache.affinity.rendezvous.RendezvousAffinityFunction"));
         }

From 621288ca9e765e3627c95001e0c01bee4d7a5da8 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Wed, 14 Nov 2018 15:12:56 +0300
Subject: [PATCH 490/543] IGNITE-10235 Do not register cache in QueryManager
 twice if parallel caches start is disabled - Fixes #5381.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 00b9e89e7add9c58ed9a996aaba9200267c71961)
---
 .../processors/cache/GridCacheProcessor.java  |   7 +-
 .../cache/StartCachesInParallelTest.java      | 148 ++++++++++++++++++
 ...heWithIndexingAndPersistenceTestSuite.java |   2 +
 3 files changed, 154 insertions(+), 3 deletions(-)
 create mode 100644 modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index edb88886bd44b..af6dc87162401 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2094,12 +2094,13 @@ public void prepareCacheStart(
     ) throws IgniteCheckedException {
         GridCacheContext cacheCtx = prepareCacheContext(startCfg, desc, reqNearCfg, exchTopVer, disabledAfterStart);
 
-        ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema());
-
         if (cacheCtx.isRecoveryMode())
             finishRecovery(exchTopVer, cacheCtx);
-        else
+        else {
+            ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema());
+
             onCacheStarted(cacheCtx);
+        }
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java
new file mode 100644
index 0000000000000..e48c294a5127b
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.FailureContext;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL;
+
+/**
+ * Tests, that cluster could start and activate with all possible values of IGNITE_ALLOW_START_CACHES_IN_PARALLEL.
+ */
+public class StartCachesInParallelTest extends GridCommonAbstractTest {
+    /** IGNITE_ALLOW_START_CACHES_IN_PARALLEL option value before tests. */
+    private String allowParallel;
+
+    /** Test failure handler. */
+    private TestStopNodeFailureHandler failureHnd;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDataStorageConfiguration(
+            new DataStorageConfiguration()
+                .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true)));
+
+        cfg.setCacheConfiguration(
+            new CacheConfiguration<>()
+                .setName(DEFAULT_CACHE_NAME)
+                .setIndexedTypes(Integer.class, Integer.class));
+
+        failureHnd = new TestStopNodeFailureHandler();
+
+        cfg.setFailureHandler(failureHnd);
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTestsStarted() throws Exception {
+        super.beforeTestsStarted();
+
+        allowParallel = System.getProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTestsStopped() throws Exception {
+        super.afterTestsStopped();
+
+        if (allowParallel != null)
+            System.setProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL, allowParallel);
+        else
+            System.clearProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** */
+    public void testWithEnabledOption() throws Exception {
+        doTest("true");
+    }
+
+    /** */
+    public void testWithDisabledOption() throws Exception {
+        doTest("false");
+    }
+
+    /** */
+    public void testWithoutOption() throws Exception {
+        doTest(null);
+    }
+
+    /**
+     * Test routine.
+     *
+     * @param optionVal IGNITE_ALLOW_START_CACHES_IN_PARALLEL value.
+     * @throws Exception If failed.
+     */
+    private void doTest(String optionVal) throws Exception {
+        if (optionVal == null)
+            System.clearProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL);
+        else {
+            Boolean.parseBoolean(optionVal);
+
+            System.setProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL, optionVal);
+        }
+
+        assertEquals("Property wasn't set", optionVal, System.getProperty(IGNITE_ALLOW_START_CACHES_IN_PARALLEL));
+
+        IgniteEx node = startGrid(0);
+
+        node.cluster().active(true);
+
+        assertNull("Node failed with " + failureHnd.lastFailureCtx, failureHnd.lastFailureCtx);
+
+        assertTrue(node.cluster().active());
+    }
+
+    /** */
+    private static class TestStopNodeFailureHandler extends StopNodeFailureHandler {
+        /** Last failure context. */
+        private volatile FailureContext lastFailureCtx;
+
+        /** {@inheritDoc} */
+        @Override public boolean handle(Ignite ignite, FailureContext failureCtx) {
+            lastFailureCtx = failureCtx;
+
+            return super.handle(ignite, failureCtx);
+        }
+    }
+}
\ No newline at end of file
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
index 1bc60f65658c8..2f3ddecef0d66 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheWithIndexingAndPersistenceTestSuite.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.StartCachesInParallelTest;
 import org.apache.ignite.util.GridCommandHandlerIndexingTest;
 
 /**
@@ -32,6 +33,7 @@ public static TestSuite suite() throws Exception {
         TestSuite suite = new TestSuite("Ignite Cache With Indexing And Persistence Test Suite");
 
         suite.addTestSuite(GridCommandHandlerIndexingTest.class);
+        suite.addTestSuite(StartCachesInParallelTest.class);
 
         return suite;
     }

From 74c99024e63dd01a1764298c9ef90e524349aad7 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Wed, 14 Nov 2018 16:29:48 +0300
Subject: [PATCH 491/543]  IGNITE-10235 Do not register cache in QueryManager
 twice if parallel caches start is disabled - Fixes #5381.

 Fix compilation error
---
 .../internal/processors/cache/StartCachesInParallelTest.java  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java
index e48c294a5127b..099cf7a2bfd7d 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/StartCachesInParallelTest.java
@@ -139,10 +139,10 @@ private static class TestStopNodeFailureHandler extends StopNodeFailureHandler {
         private volatile FailureContext lastFailureCtx;
 
         /** {@inheritDoc} */
-        @Override public boolean handle(Ignite ignite, FailureContext failureCtx) {
+        @Override public boolean onFailure(Ignite ignite, FailureContext failureCtx) {
             lastFailureCtx = failureCtx;
 
-            return super.handle(ignite, failureCtx);
+            return super.onFailure(ignite, failureCtx);
         }
     }
 }
\ No newline at end of file

From e0ed0c1f405872d8115770fcc903885b0bd7187e Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Wed, 14 Nov 2018 15:40:57 +0300
Subject: [PATCH 492/543] IGNITE-10228 Fixed concurrent start caches race
 leading to a missed cache context - Fixes #5377.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit d1a039d36787d1920b631d71e8228fa5c47e5d20)
---
 .../processors/cache/CacheGroupContext.java   |  97 ++++-----
 .../ignite/internal/util/IgniteUtils.java     |  39 ++--
 .../distributed/CacheParallelStartTest.java   | 202 ++++++++++++++++++
 .../internal/util/IgniteUtilsSelfTest.java    |  61 +++---
 .../testsuites/IgniteCacheTestSuite6.java     |   3 +
 5 files changed, 305 insertions(+), 97 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheParallelStartTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index e5ec83da90444..defac3200e05c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -20,7 +20,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Iterator;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -115,8 +115,8 @@ public class CacheGroupContext {
     /** */
     private final boolean storeCacheId;
 
-    /** */
-    private volatile List<GridCacheContext> caches;
+    /** We modify content under lock, by making defencive copy, field always contains unmodifiable list. */
+    private volatile List<GridCacheContext> caches = Collections.unmodifiableList(new ArrayList<>());
 
     /** */
     private volatile List<GridCacheContext> contQryCaches;
@@ -135,6 +135,7 @@ public class CacheGroupContext {
 
     /** */
     private volatile GridCachePreloader preldr;
+
     /** */
     private final DataRegion dataRegion;
 
@@ -151,13 +152,13 @@ public class CacheGroupContext {
     private final ReuseList reuseList;
 
     /** */
-    private boolean drEnabled;
+    private volatile boolean drEnabled;
 
     /** */
-    private boolean qryEnabled;
+    private volatile boolean qryEnabled;
 
     /** MXBean. */
-    private CacheGroupMetricsMXBean mxBean;
+    private final CacheGroupMetricsMXBean mxBean;
 
     /** */
     private volatile boolean localWalEnabled;
@@ -230,8 +231,6 @@ public class CacheGroupContext {
 
         log = ctx.kernalContext().log(getClass());
 
-        caches = new ArrayList<>();
-
         mxBean = new CacheGroupMetricsMXBeanImpl(this);
     }
 
@@ -294,10 +293,9 @@ void onCacheStarted(GridCacheContext cctx) throws IgniteCheckedException {
     public boolean hasCache(String cacheName) {
         List<GridCacheContext> caches = this.caches;
 
-        for (int i = 0; i < caches.size(); i++) {
-            if (caches.get(i).name().equals(cacheName))
+        for (GridCacheContext cacheContext : caches)
+            if (cacheContext.name().equals(cacheName))
                 return true;
-        }
 
         return false;
     }
@@ -309,11 +307,17 @@ private void addCacheContext(GridCacheContext cctx) {
         assert cacheType.userCache() == cctx.userCache() : cctx.name();
         assert grpId == cctx.groupId() : cctx.name();
 
-        ArrayList<GridCacheContext> caches = new ArrayList<>(this.caches);
+        final boolean add;
 
-        assert sharedGroup() || caches.isEmpty();
+        synchronized (this) {
+            List<GridCacheContext> copy = new ArrayList<>(caches);
+
+            assert sharedGroup() || copy.isEmpty();
 
-        boolean add = caches.add(cctx);
+            add = copy.add(cctx);
+
+            caches = Collections.unmodifiableList(copy);
+        }
 
         assert add : cctx.name();
 
@@ -322,39 +326,39 @@ private void addCacheContext(GridCacheContext cctx) {
 
         if (!drEnabled && cctx.isDrEnabled())
             drEnabled = true;
-
-        this.caches = caches;
     }
 
     /**
      * @param cctx Cache context.
      */
     private void removeCacheContext(GridCacheContext cctx) {
-        ArrayList<GridCacheContext> caches = new ArrayList<>(this.caches);
+        final List<GridCacheContext> copy;
 
-        // It is possible cache was not added in case of errors on cache start.
-        for (Iterator<GridCacheContext> it = caches.iterator(); it.hasNext();) {
-            GridCacheContext next = it.next();
+        synchronized (this) {
+            copy = new ArrayList<>(caches);
 
-            if (next == cctx) {
-                assert sharedGroup() || caches.size() == 1 : caches.size();
+            for (GridCacheContext next : copy) {
+                if (next == cctx) {
+                    assert sharedGroup() || copy.size() == 1 : copy.size();
 
-                it.remove();
+                    copy.remove(next);
 
-                break;
+                    break;
+                }
             }
+
+            caches = Collections.unmodifiableList(copy);
         }
 
         if (QueryUtils.isEnabled(cctx.config())) {
             boolean qryEnabled = false;
 
-            for (int i = 0; i < caches.size(); i++) {
-                if (QueryUtils.isEnabled(caches.get(i).config())) {
+            for (GridCacheContext cacheContext : copy)
+                if (QueryUtils.isEnabled(cacheContext.config())) {
                     qryEnabled = true;
 
                     break;
                 }
-            }
 
             this.qryEnabled = qryEnabled;
         }
@@ -362,18 +366,15 @@ private void removeCacheContext(GridCacheContext cctx) {
         if (cctx.isDrEnabled()) {
             boolean drEnabled = false;
 
-            for (int i = 0; i < caches.size(); i++) {
-                if (caches.get(i).isDrEnabled()) {
+            for (GridCacheContext cacheContext : copy)
+                if (QueryUtils.isEnabled(cacheContext.config())) {
                     drEnabled = true;
 
                     break;
                 }
-            }
 
             this.drEnabled = drEnabled;
         }
-
-        this.caches = caches;
     }
 
     /**
@@ -395,11 +396,8 @@ public GridCacheContext singleCacheContext() {
     public void unwindUndeploys() {
         List<GridCacheContext> caches = this.caches;
 
-        for (int i = 0; i < caches.size(); i++) {
-            GridCacheContext cctx = caches.get(i);
-
+        for (GridCacheContext cctx : caches)
             cctx.deploy().unwind(cctx);
-        }
     }
 
     /**
@@ -430,9 +428,7 @@ public void addRebalanceEvent(int part, int type, ClusterNode discoNode, int dis
 
         List<GridCacheContext> caches = this.caches;
 
-        for (int i = 0; i < caches.size(); i++) {
-            GridCacheContext cctx = caches.get(i);
-
+        for (GridCacheContext cctx : caches)
             if (!cctx.config().isEventsDisabled() && cctx.recordEvent(type)) {
                 cctx.gridEvents().record(new CacheRebalancingEvent(cctx.name(),
                     cctx.localNode(),
@@ -443,7 +439,6 @@ public void addRebalanceEvent(int part, int type, ClusterNode discoNode, int dis
                     discoType,
                     discoTs));
             }
-        }
     }
 
     /**
@@ -458,9 +453,7 @@ public void addUnloadEvent(int part) {
 
         List<GridCacheContext> caches = this.caches;
 
-        for (int i = 0; i < caches.size(); i++) {
-            GridCacheContext cctx = caches.get(i);
-
+        for (GridCacheContext cctx : caches)
             if (!cctx.config().isEventsDisabled())
                 cctx.gridEvents().record(new CacheRebalancingEvent(cctx.name(),
                     cctx.localNode(),
@@ -470,7 +463,6 @@ public void addUnloadEvent(int part) {
                     null,
                     0,
                     0));
-        }
     }
 
     /**
@@ -497,9 +489,7 @@ public void addCacheEvent(
     ) {
         List<GridCacheContext> caches = this.caches;
 
-        for (int i = 0; i < caches.size(); i++) {
-            GridCacheContext cctx = caches.get(i);
-
+        for (GridCacheContext cctx : caches)
             if (!cctx.config().isEventsDisabled())
                 cctx.events().addEvent(part,
                     key,
@@ -515,7 +505,6 @@ public void addCacheEvent(
                     null,
                     null,
                     keepBinary);
-        }
     }
 
     /**
@@ -964,17 +953,19 @@ public Set<Integer> cacheIds() {
 
         Set<Integer> ids = U.newHashSet(caches.size());
 
-        for (int i = 0; i < caches.size(); i++)
-            ids.add(caches.get(i).cacheId());
+        for (GridCacheContext cctx : caches)
+            ids.add(cctx.cacheId());
 
         return ids;
     }
 
     /**
      * @return Caches in this group.
+     *
+     * caches is already Unmodifiable list, so we don't need to explicitly wrap it here.
      */
     public List<GridCacheContext> caches() {
-        return this.caches;
+        return caches;
     }
 
     /**
@@ -992,9 +983,7 @@ public boolean hasCaches() {
     public void onPartitionEvicted(int part) {
         List<GridCacheContext> caches = this.caches;
 
-        for (int i = 0; i < caches.size(); i++) {
-            GridCacheContext cctx = caches.get(i);
-
+        for (GridCacheContext cctx : caches) {
             if (cctx.isDrEnabled())
                 cctx.dr().partitionEvicted(part);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index a82a6858dc150..f12340c6f179a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -141,7 +141,6 @@
 import java.util.logging.Logger;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 import java.util.zip.Deflater;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
@@ -10556,26 +10555,19 @@ public static <T, R> Collection<R> doInParallel(
     ) throws IgniteCheckedException, IgniteInterruptedCheckedException {
         if(srcDatas.isEmpty())
             return Collections.emptyList();
+        int[] batchSizes = calculateOptimalBatchSizes(parallelismLvl, srcDatas.size());
 
-        int batchSize = srcDatas.size() / parallelismLvl;
+        List<List<T>> batches = new ArrayList<>(batchSizes.length);
 
-        final int finalBatchSize = batchSize == 0 ? srcDatas.size() : batchSize;
+        Iterator<T> iterator = srcDatas.iterator();
 
-        List<List<T>> batches = IntStream.range(0, parallelismLvl)
-            .mapToObj(i -> new ArrayList<T>(finalBatchSize))
-            .collect(Collectors.toList());
-
-        int batchIndex = 0;
-
-        final int maxBatchIndex = batches.size() -1;
+        for (int batchSize : batchSizes) {
+            List<T> batch = new ArrayList<>(batchSize);
 
-        List<T> currentBatch = batches.get(batchIndex);
+            for (int i = 0; i < batchSize; i++)
+                batch.add(iterator.next());
 
-        for (T src : srcDatas) {
-            currentBatch.add(src);
-
-            if(currentBatch.size() >= batchSize && batchIndex < maxBatchIndex)
-                currentBatch = batches.get(++batchIndex);
+            batches.add(batch);
         }
 
         List<Future<Collection<R>>> consumerFutures = batches.stream()
@@ -10633,6 +10625,21 @@ public static <T, R> Collection<R> doInParallel(
         return results;
     }
 
+    /**
+     * Split number of tasks into optimized batches.
+     * @param parallelismLvl Level of parallelism.
+     * @param size number of tasks to split.
+     * @return array of batch sizes.
+     */
+    public static int[] calculateOptimalBatchSizes(int parallelismLvl, int size) {
+        int[] batcheSizes = new int[Math.min(parallelismLvl, size)];
+
+        for (int i = 0; i < size; i++)
+            batcheSizes[i % batcheSizes.length]++;
+
+        return batcheSizes;
+    }
+
     /**
      * @param fut Future to wait for completion.
      * @throws ExecutionException If the future
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheParallelStartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheParallelStartTest.java
new file mode 100644
index 0000000000000..2a644856ca09c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheParallelStartTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Test covers parallel start and stop of caches.
+ */
+public class CacheParallelStartTest extends GridCommonAbstractTest {
+    /** */
+    private static final int CACHES_COUNT = 500;
+
+    /** */
+    private static final String STATIC_CACHE_PREFIX = "static-cache-";
+
+    /** */
+    private static final String STATIC_CACHE_CACHE_GROUP_NAME = "static-cache-group";
+
+    /** */
+    private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setSystemThreadPoolSize(Runtime.getRuntime().availableProcessors() * 3);
+
+        TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+        discoSpi.setIpFinder(IP_FINDER);
+
+        cfg.setDiscoverySpi(discoSpi);
+
+        long sz = 100 * 1024 * 1024;
+
+        DataStorageConfiguration memCfg = new DataStorageConfiguration().setPageSize(1024)
+                .setDefaultDataRegionConfiguration(
+                        new DataRegionConfiguration().setPersistenceEnabled(false).setInitialSize(sz).setMaxSize(sz))
+                .setWalMode(WALMode.LOG_ONLY).setCheckpointFrequency(24L * 60 * 60 * 1000);
+
+        cfg.setDataStorageConfiguration(memCfg);
+
+        ArrayList<Object> staticCaches = new ArrayList<>(CACHES_COUNT);
+
+        for (int i = 0; i < CACHES_COUNT; i++)
+            staticCaches.add(cacheConfiguration(STATIC_CACHE_PREFIX + i));
+
+        cfg.setCacheConfiguration(staticCaches.toArray(new CacheConfiguration[CACHES_COUNT]));
+
+        return cfg;
+    }
+
+    /**
+     * @param cacheName Cache name.
+     * @return Cache configuration.
+     */
+    private CacheConfiguration cacheConfiguration(String cacheName) {
+        CacheConfiguration cfg = defaultCacheConfiguration();
+
+        cfg.setName(cacheName);
+        cfg.setBackups(1);
+        cfg.setGroupName(STATIC_CACHE_CACHE_GROUP_NAME);
+        cfg.setIndexedTypes(Long.class, Long.class);
+
+        return cfg;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanupTestData();
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        cleanupTestData();
+    }
+
+    /** */
+    private void cleanupTestData() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+
+        System.clearProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testParallelStartAndStop() throws Exception {
+        testParallelStartAndStop(true);
+    }
+
+    /**
+     * @throws Exception if failed.
+     */
+    public void testSequentialStartAndStop() throws Exception {
+        testParallelStartAndStop(false);
+    }
+
+    /**
+     *
+     */
+    private void testParallelStartAndStop(boolean parallel) throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_ALLOW_START_CACHES_IN_PARALLEL, String.valueOf(parallel));
+
+        IgniteEx igniteEx = startGrid(0);
+
+        IgniteEx igniteEx2 = startGrid(1);
+
+        igniteEx.cluster().active(true);
+
+        assertCaches(igniteEx);
+
+        assertCaches(igniteEx2);
+
+        igniteEx.cluster().active(false);
+
+        assertCachesAfterStop(igniteEx);
+
+        assertCachesAfterStop(igniteEx2);
+    }
+
+    /**
+     *
+     */
+    private void assertCachesAfterStop(IgniteEx igniteEx) {
+        assertNull(igniteEx
+                .context()
+                .cache()
+                .cacheGroup(CU.cacheId(STATIC_CACHE_CACHE_GROUP_NAME)));
+
+        assertTrue(igniteEx.context().cache().cacheGroups().isEmpty());
+
+        for (int i = 0; i < CACHES_COUNT; i++) {
+            assertNull(igniteEx.context().cache().cache(STATIC_CACHE_PREFIX + i));
+            assertNull(igniteEx.context().cache().internalCache(STATIC_CACHE_PREFIX + i));
+        }
+    }
+
+    /**
+     *
+     */
+    private void assertCaches(IgniteEx igniteEx) {
+        Collection<GridCacheContext> caches = igniteEx
+                .context()
+                .cache()
+                .cacheGroup(CU.cacheId(STATIC_CACHE_CACHE_GROUP_NAME))
+                .caches();
+
+        assertEquals(CACHES_COUNT, caches.size());
+
+        @Nullable CacheGroupContext cacheGroup = igniteEx
+                .context()
+                .cache()
+                .cacheGroup(CU.cacheId(STATIC_CACHE_CACHE_GROUP_NAME));
+
+        for (GridCacheContext cacheContext : caches)
+            assertEquals(cacheContext.group(), cacheGroup);
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
index 5967efcb668a9..196466d4a8c37 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
@@ -914,6 +914,37 @@ public void testDoInParallelBatch() {
         }
     }
 
+    /**
+     * Test optimal splitting on batch sizes.
+     */
+    public void testOptimalBatchSize() {
+        assertArrayEquals(new int[]{1}, IgniteUtils.calculateOptimalBatchSizes(1, 1));
+
+        assertArrayEquals(new int[]{2}, IgniteUtils.calculateOptimalBatchSizes(1, 2));
+
+        assertArrayEquals(new int[]{1, 1, 1, 1}, IgniteUtils.calculateOptimalBatchSizes(6, 4));
+
+        assertArrayEquals(new int[]{1}, IgniteUtils.calculateOptimalBatchSizes(4, 1));
+
+        assertArrayEquals(new int[]{1, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 2));
+
+        assertArrayEquals(new int[]{1, 1, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 3));
+
+        assertArrayEquals(new int[]{1, 1, 1, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 4));
+
+        assertArrayEquals(new int[]{2, 1, 1, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 5));
+
+        assertArrayEquals(new int[]{2, 2, 1, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 6));
+
+        assertArrayEquals(new int[]{2, 2, 2, 1}, IgniteUtils.calculateOptimalBatchSizes(4, 7));
+
+        assertArrayEquals(new int[]{2, 2, 2, 2}, IgniteUtils.calculateOptimalBatchSizes(4, 8));
+
+        assertArrayEquals(new int[]{3, 2, 2, 2}, IgniteUtils.calculateOptimalBatchSizes(4, 9));
+
+        assertArrayEquals(new int[]{3, 3, 2, 2}, IgniteUtils.calculateOptimalBatchSizes(4, 10));
+    }
+
     /**
      * Test parallel execution in order.
      */
@@ -921,33 +952,9 @@ public void testDoInParallelResultsOrder() throws IgniteCheckedException {
         ExecutorService executorService = Executors.newFixedThreadPool(4);
 
         try {
-            /*testOrder(executorService, 1, 1);
-            testOrder(executorService, 2, 1);
-            testOrder(executorService, 3, 1);
-            testOrder(executorService, 9, 1);
-            testOrder(executorService, 10, 1);
-            testOrder(executorService, 9999, 1);
-
-            testOrder(executorService, 1, 2);
-            testOrder(executorService, 2, 2);
-            testOrder(executorService, 3, 2);
-            testOrder(executorService, 9, 2);
-            testOrder(executorService, 10, 2);*/
-            testOrder(executorService, 9999, 2);
-
-            testOrder(executorService, 1, 3);
-            testOrder(executorService, 2, 3);
-            testOrder(executorService, 3, 3);
-            testOrder(executorService, 3, 3);
-            testOrder(executorService, 10, 3);
-            testOrder(executorService, 9999, 3);
-
-            testOrder(executorService, 1, 4);
-            testOrder(executorService, 2, 4);
-            testOrder(executorService, 3, 4);
-            testOrder(executorService, 9, 4);
-            testOrder(executorService, 10, 4);
-            testOrder(executorService, 9999, 4);
+            for(int parallelism = 1; parallelism < 16; parallelism++)
+                for(int size = 0; size < 10_000; size++)
+                    testOrder(executorService, size, parallelism);
         } finally {
             executorService.shutdownNow();
         }
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 058f6243f58ba..66064790ae450 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -27,6 +27,7 @@
 import org.apache.ignite.internal.processors.cache.ReplicatedTransactionalPessimisticCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.datastructures.IgniteExchangeLatchManagerCoordinatorFailTest;
 import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest;
+import org.apache.ignite.internal.processors.cache.distributed.CacheParallelStartTest;
 import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest;
 import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.IgniteCache150ClientsTest;
@@ -106,6 +107,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(PartitionsExchangeCoordinatorFailoverTest.class);
 
+        suite.addTestSuite(CacheParallelStartTest.class);
+
         //suite.addTestSuite(CacheClientsConcurrentStartTest.class);
         //suite.addTestSuite(GridCacheRebalancingOrderingTest.class);
         //suite.addTestSuite(IgniteCacheClientMultiNodeUpdateTopologyLockTest.class);

From 1262faa445ecf8cca84049bd7c8f77ee6e81998f Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Wed, 14 Nov 2018 22:45:13 +0700
Subject: [PATCH 493/543] IGNITE-10189 Added support for SSL Params (Cipher
 suites and protocols) for "control.sh" utility.

---
 .../client/ClientSslParametersTest.java       | 338 ++++++++++++++++++
 .../client/suite/IgniteClientTestSuite.java   |   4 +
 .../ssl/GridSslBasicContextFactory.java       |  88 ++++-
 .../internal/commandline/Arguments.java       |  41 ++-
 .../internal/commandline/CommandHandler.java  | 162 ++++++---
 .../apache/ignite/ssl/SSLContextWrapper.java  |  13 +-
 .../apache/ignite/ssl/SslContextFactory.java  |  16 +-
 .../ignite/client/SslParametersTest.java      |  16 +-
 8 files changed, 596 insertions(+), 82 deletions(-)
 create mode 100644 modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java
new file mode 100644
index 0000000000000..228ce82cd2318
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java
@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.client;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.client.ssl.GridSslBasicContextFactory;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Tests cases when node connects to cluster with different set of cipher suites.
+ */
+public class ClientSslParametersTest extends GridCommonAbstractTest {
+    /** */
+    public static final String TEST_CACHE_NAME = "TEST";
+
+    /** */
+    private volatile String[] cipherSuites;
+
+    /** */
+    private volatile String[] protocols;
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+        ConnectorConfiguration conCfg = new ConnectorConfiguration();
+        conCfg.setSslEnabled(true);
+        conCfg.setSslClientAuth(true);
+        conCfg.setSslContextFactory(createSslFactory());
+
+        cfg.setConnectorConfiguration(conCfg);
+
+        CacheConfiguration ccfg = new CacheConfiguration(TEST_CACHE_NAME);
+
+        cfg.setCacheConfiguration(ccfg);
+
+        return cfg;
+    }
+
+    /**
+     * @return Client configuration.
+     */
+    protected GridClientConfiguration getClientConfiguration() {
+        GridClientConfiguration cfg = new GridClientConfiguration();
+
+        cfg.setServers(Collections.singleton("127.0.0.1:11211"));
+
+        cfg.setSslContextFactory(createSslFactory());
+
+        return cfg;
+    }
+
+    /**
+     * @return SSL Factory.
+     */
+    @NotNull private GridSslBasicContextFactory createSslFactory() {
+        GridSslBasicContextFactory factory = (GridSslBasicContextFactory)GridTestUtils.sslContextFactory();
+
+        factory.setCipherSuites(cipherSuites);
+        factory.setProtocols(protocols);
+
+        return factory;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        protocols = null;
+        cipherSuites = null;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSameCipherSuite() throws Exception {
+        cipherSuites = new String[] {
+            "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+            "TLS_RSA_WITH_AES_128_GCM_SHA256",
+            "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+        };
+
+        startGrid();
+
+        checkSuccessfulClientStart(
+            new String[][] {
+                new String[] {
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneCommonCipherSuite() throws Exception {
+        cipherSuites = new String[] {
+            "TLS_RSA_WITH_AES_128_GCM_SHA256",
+            "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+        };
+
+        startGrid();
+        
+        checkSuccessfulClientStart(
+            new String[][] {
+                new String[] {
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoCommonCipherSuite() throws Exception {
+        cipherSuites = new String[] {
+            "TLS_RSA_WITH_AES_128_GCM_SHA256"
+        };
+
+        startGrid();
+        
+        checkClientStartFailure(
+            new String[][] {
+                new String[] {
+                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNonExistentCipherSuite() throws Exception {
+        fail("IGNITE-10245");
+
+        cipherSuites = new String[] {
+            "TLS_RSA_WITH_AES_128_GCM_SHA256"
+        };
+
+        startGrid();
+        
+        checkClientStartFailure(
+            new String[][] {
+                new String[] {
+                    "TLC_FAKE_CIPHER",
+                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
+                }
+            },
+            null,
+            GridClientException.class,
+            "Unsupported ciphersuite"
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoCommonProtocols() throws Exception {
+        protocols = new String[] {
+            "TLSv1.1",
+            "SSLv3"
+        };
+
+        startGrid();
+
+        checkClientStartFailure(
+            null,
+            new String[][] {
+                new String[] {
+                    "TLSv1",
+                    "TLSv1.2",
+                }
+            }
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNonExistentProtocol() throws Exception {
+        fail("IGNITE-10245");
+
+        protocols = new String[] {
+            "SSLv3"
+        };
+
+        startGrid();
+
+        checkClientStartFailure(
+            null,
+            new String[][] {
+                new String[] {
+                    "SSLv3",
+                    "SSLvDoesNotExist"
+                }
+            },
+            GridClientException.class,
+            "SSLvDoesNotExist"
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSameProtocols() throws Exception {
+        protocols = new String[] {
+            "TLSv1.1",
+            "TLSv1.2",
+        };
+
+        startGrid();
+
+        checkSuccessfulClientStart(null,
+            new String[][] {
+                new String[] {
+                    "TLSv1.1",
+                    "TLSv1.2",
+                }
+            }
+        );
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneCommonProtocol() throws Exception {
+        protocols = new String[] {
+            "TLSv1",
+            "TLSv1.1",
+            "TLSv1.2"
+        };
+
+        startGrid();
+
+        checkSuccessfulClientStart(null,
+            new String[][] {
+                new String[] {
+                    "TLSv1.1",
+                    "SSLv3"
+                }
+            }
+        );
+    }
+
+    /**
+     * @param cipherSuites list of cipher suites
+     * @param protocols list of protocols
+     * @throws Exception If failed.
+     */
+    private void checkSuccessfulClientStart(String[][] cipherSuites, String[][] protocols) throws Exception {
+        int n = Math.max(
+            cipherSuites != null ? cipherSuites.length : 0,
+            protocols != null ? protocols.length : 0);
+
+        for (int i = 0; i < n; i++) {
+            this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
+            this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
+
+            GridClient client = GridClientFactory.start(getClientConfiguration());
+
+            List<GridClientNode> top = client.compute().refreshTopology(false, false);
+
+            assertEquals(1, top.size());
+
+            client.close();
+        }
+    }
+
+    /**
+     * @param cipherSuites list of cipher suites
+     * @param protocols list of protocols
+     */
+    private void checkClientStartFailure(String[][] cipherSuites, String[][] protocols) {
+        checkClientStartFailure(cipherSuites, protocols, GridClientException.class, "Latest topology update failed.");
+    }
+
+    /**
+     * @param cipherSuites list of cipher suites
+     * @param protocols list of protocols
+     * @param ex expected exception class
+     * @param msg exception message
+     */
+    private void checkClientStartFailure(String[][] cipherSuites, String[][] protocols, Class<? extends Throwable> ex, String msg) {
+        int n = Math.max(
+            cipherSuites != null ? cipherSuites.length : 0,
+            protocols != null ? protocols.length : 0);
+
+        for (int i = 0; i < n; i++) {
+            this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
+            this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
+
+            GridTestUtils.assertThrows(null, new Callable<Object>() {
+                @Override public Object call() throws Exception {
+                    GridClient client = GridClientFactory.start(getClientConfiguration());
+
+                    client.compute().refreshTopology(false, false);
+
+                    return null;
+                }
+            }, ex, msg);
+        }
+    }
+
+}
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
index 163f89acb7a35..0427f5caeb548 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/suite/IgniteClientTestSuite.java
@@ -20,6 +20,7 @@
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.client.ClientDefaultCacheSelfTest;
 import org.apache.ignite.internal.client.ClientReconnectionSelfTest;
+import org.apache.ignite.internal.client.ClientSslParametersTest;
 import org.apache.ignite.internal.client.ClientTcpMultiThreadedSelfTest;
 import org.apache.ignite.internal.client.ClientTcpSslAuthenticationSelfTest;
 import org.apache.ignite.internal.client.ClientTcpSslMultiThreadedSelfTest;
@@ -155,6 +156,9 @@ public static TestSuite suite() {
 
         suite.addTestSuite(ClientTcpTaskExecutionAfterTopologyRestartSelfTest.class);
 
+        // SSL params.
+        suite.addTestSuite(ClientSslParametersTest.class);
+
         return suite;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/client/ssl/GridSslBasicContextFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/client/ssl/GridSslBasicContextFactory.java
index b651ae38041fe..c623175283630 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/client/ssl/GridSslBasicContextFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/client/ssl/GridSslBasicContextFactory.java
@@ -23,17 +23,20 @@
 import java.io.InputStream;
 import java.security.GeneralSecurityException;
 import java.security.KeyStore;
-import java.security.cert.CertificateException;
 import java.security.cert.X509Certificate;
 import java.util.Arrays;
+import java.util.Collection;
 import javax.cache.configuration.Factory;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLParameters;
 import javax.net.ssl.TrustManager;
 import javax.net.ssl.TrustManagerFactory;
 import javax.net.ssl.X509TrustManager;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.ssl.SSLContextWrapper;
 
 /**
  * Basic ssl context factory that provides ssl context configuration with specified key
@@ -87,6 +90,12 @@ public class GridSslBasicContextFactory implements GridSslContextFactory {
     /** Trust managers. */
     private TrustManager[] trustMgrs;
 
+    /** Enabled cipher suites. */
+    private String[] cipherSuites;
+
+    /** Enabled protocols. */
+    private String[] protocols;
+
     /**
      * Gets key store type used for context creation.
      *
@@ -269,6 +278,63 @@ public void setTrustManagers(TrustManager... trustMgrs) {
         this.trustMgrs = trustMgrs;
     }
 
+    /**
+     * Gets enabled cipher suites.
+     *
+     * @return Enabled cipher suites.
+     */
+    public String[] getCipherSuites() {
+        return cipherSuites;
+    }
+
+    /**
+     * Sets enabled cipher suites.
+     *
+     * @param cipherSuites Enabled cipher suites.
+     */
+    public void setCipherSuites(String... cipherSuites) {
+        this.cipherSuites = cipherSuites;
+    }
+
+
+    /**
+     * Sets enabled cipher suites.
+     *
+     * @param cipherSuites Enabled cipher suites.
+     */
+    public void setCipherSuites(Collection<String> cipherSuites) {
+        if (!F.isEmpty(cipherSuites))
+            setCipherSuites(cipherSuites.toArray(new String[0]));
+    }
+
+    /**
+     * Gets enabled protocols.
+     *
+     * @return Enabled protocols.
+     */
+    public String[] getProtocols() {
+        return protocols;
+    }
+
+    /**
+     * Sets enabled protocols.
+     *
+     * @param protocols Enabled protocols.
+     */
+    public void setProtocols(String... protocols) {
+        this.protocols = protocols;
+    }
+
+    /**
+     * Sets enabled protocols.
+     *
+     * @param protocols Enabled protocols.
+     */
+    public void setProtocols(Collection<String> protocols) {
+        if (!F.isEmpty(protocols))
+            setProtocols(protocols.toArray(new String[0]));
+    }
+
     /**
      * Returns an instance of trust manager that will always succeed regardless of certificate provided.
      *
@@ -303,6 +369,18 @@ public static TrustManager getDisabledTrustManager() {
 
             SSLContext ctx = SSLContext.getInstance(proto);
 
+            if (cipherSuites != null || protocols != null) {
+                SSLParameters sslParameters = new SSLParameters();
+
+                if (cipherSuites != null)
+                    sslParameters.setCipherSuites(cipherSuites);
+
+                if (protocols != null)
+                    sslParameters.setProtocols(protocols);
+
+                ctx = new SSLContextWrapper(ctx, sslParameters);
+            }
+
             ctx.init(keyMgrFactory.getKeyManagers(), mgrs, null);
 
             return ctx;
@@ -431,14 +509,12 @@ private static class DisabledX509TrustManager implements X509TrustManager {
         private static final X509Certificate[] CERTS = new X509Certificate[0];
 
         /** {@inheritDoc} */
-        @Override public void checkClientTrusted(X509Certificate[] x509Certificates, String s)
-            throws CertificateException {
+        @Override public void checkClientTrusted(X509Certificate[] x509Certificates, String s) {
             // No-op, all clients are trusted.
         }
 
         /** {@inheritDoc} */
-        @Override public void checkServerTrusted(X509Certificate[] x509Certificates, String s)
-            throws CertificateException {
+        @Override public void checkServerTrusted(X509Certificate[] x509Certificates, String s) {
             // No-op, all servers are trusted.
         }
 
@@ -447,4 +523,4 @@ private static class DisabledX509TrustManager implements X509TrustManager {
             return CERTS;
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index 32e1b662b3e71..d769982566320 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -86,6 +86,9 @@ public class Arguments {
     /** */
     private String sslAlgorithm;
 
+    /** */
+    private String sslCipherSuites;
+
     /** */
     private String sslKeyStorePath;
 
@@ -119,33 +122,54 @@ public class Arguments {
      * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
      * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
      * @param autoConfirmation Auto confirmation flag.
+     * @param sslEnable SSL flag.
+     * @param sslProtocol SSL protocol.
+     * @param sslAlgorithm SSL algorithm.
+     * @param sslCipherSuites SSL cipher suites.
+     * @param sslKeyStorePath Key store path.
+     * @param sslKeyStoreType Key store type.
+     * @param sslKeyStorePassword Key store password.
+     * @param sslTrustStorePath Trust store path.
+     * @param sslTrustStoreType Trust store type.
+     * @param sslTrustStorePassword Trust store password.
      */
-    public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
-                     String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
-                     Long pingTimeout, Long pingInterval, boolean autoConfirmation,
-                     boolean sslEnable, String sslProtocol, String sslAlgorithm,
-                     String sslKeyStorePath, String sslKeyStoreType, char sslKeyStorePassword[],
-                     String sslTrustStorePath, String sslTrustStoreType, char sslTrustStorePassword[]) {
+    public Arguments(Command cmd, String host, String port, String user, String pwd,
+        String baselineAct, String baselineArgs,
+        VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
+        Long pingTimeout, Long pingInterval, boolean autoConfirmation,
+        boolean sslEnable, String sslProtocol, String sslAlgorithm, String sslCipherSuites,
+        String sslKeyStorePath, String sslKeyStoreType, char sslKeyStorePassword[],
+        String sslTrustStorePath, String sslTrustStoreType, char sslTrustStorePassword[]
+    ) {
         this.cmd = cmd;
         this.host = host;
         this.port = port;
         this.user = user;
         this.pwd = pwd;
+
         this.baselineAct = baselineAct;
         this.baselineArgs = baselineArgs;
+
         this.txArg = txArg;
         this.cacheArgs = cacheArgs;
+
         this.walAct = walAct;
         this.walArgs = walArgs;
+
         this.pingTimeout = pingTimeout;
         this.pingInterval = pingInterval;
+
         this.autoConfirmation = autoConfirmation;
+
         this.sslEnable = sslEnable;
         this.sslProtocol = sslProtocol;
         this.sslAlgorithm = sslAlgorithm;
+        this.sslCipherSuites = sslCipherSuites;
+
         this.sslKeyStorePath = sslKeyStorePath;
         this.sslKeyStoreType = sslKeyStoreType;
         this.sslKeyStorePassword = sslKeyStorePassword;
+
         this.sslTrustStorePath = sslTrustStorePath;
         this.sslTrustStoreType = sslTrustStoreType;
         this.sslTrustStorePassword = sslTrustStorePassword;
@@ -282,6 +306,11 @@ public String getSslAlgorithm() {
         return sslAlgorithm;
     }
 
+    /** */
+    public String getSslCipherSuites() {
+        return sslCipherSuites;
+    }
+
     /** */
     public String getSslKeyStorePath() {
         return sslKeyStorePath;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index e5488bb63e318..58fba736ec019 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -206,6 +206,9 @@ public class CommandHandler {
     /** */
     protected static final String CMD_SSL_ALGORITHM = "--ssl_algorithm";
 
+    /** */
+    protected static final String CMD_SSL_CIPHER_SUITES = "--ssl_cipher_suites";
+
     /** */
     protected static final String CMD_SSL_KEY_STORE_PATH = "--ssl_key_store_path";
 
@@ -227,27 +230,34 @@ public class CommandHandler {
     /** */
     private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
 
-
     /** List of optional auxiliary commands. */
     private static final Set<String> AUX_COMMANDS = new HashSet<>();
 
     static {
         AUX_COMMANDS.add(CMD_HELP);
+
         AUX_COMMANDS.add(CMD_HOST);
         AUX_COMMANDS.add(CMD_PORT);
+
         AUX_COMMANDS.add(CMD_PASSWORD);
         AUX_COMMANDS.add(CMD_USER);
+
         AUX_COMMANDS.add(CMD_AUTO_CONFIRMATION);
+
         AUX_COMMANDS.add(CMD_PING_INTERVAL);
         AUX_COMMANDS.add(CMD_PING_TIMEOUT);
+
         AUX_COMMANDS.add(CMD_SSL_ENABLED);
         AUX_COMMANDS.add(CMD_SSL_PROTOCOL);
         AUX_COMMANDS.add(CMD_SSL_ALGORITHM);
-        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PATH);
+        AUX_COMMANDS.add(CMD_SSL_CIPHER_SUITES);
+
         AUX_COMMANDS.add(CMD_SSL_KEY_STORE_TYPE);
+        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PATH);
         AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PASSWORD);
-        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PATH);
+
         AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_TYPE);
+        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PATH);
         AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PASSWORD);
     }
 
@@ -357,7 +367,15 @@ public class CommandHandler {
     private static final String UTILITY_NAME = "control.sh";
 
     /** Common options. */
-    private static final String COMMON_OPTIONS = String.join(" ", op(CMD_HOST, "HOST_OR_IP"), op(CMD_PORT, "PORT"), op(CMD_USER, "USER"), op(CMD_PASSWORD, "PASSWORD"), op(CMD_PING_INTERVAL, "PING_INTERVAL"), op(CMD_PING_TIMEOUT, "PING_TIMEOUT"), op(CMD_SSL_ENABLED), op(CMD_SSL_KEY_STORE_PATH,"PATH"),op(CMD_SSL_KEY_STORE_TYPE, "jks"), op(CMD_SSL_KEY_STORE_PASSWORD, "PASSWORD"), op(CMD_SSL_TRUSTSTORE_PATH, "PATH"), op(CMD_SSL_TRUSTSTORE_TYPE, "jks"), op(CMD_SSL_TRUSTSTORE_PASSWORD, "PASSWORD"));
+    private static final String COMMON_OPTIONS = String.join(" ", op(CMD_HOST, "HOST_OR_IP"), op(CMD_PORT, "PORT"),
+        op(CMD_USER, "USER"), op(CMD_PASSWORD, "PASSWORD"),
+        op(CMD_PING_INTERVAL, "PING_INTERVAL"), op(CMD_PING_TIMEOUT, "PING_TIMEOUT"),
+        op(CMD_SSL_ENABLED),
+        op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ...]"),
+        op(CMD_SSL_ALGORITHM, "SSL_ALGORITHM"),
+        op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ...]"),
+        op(CMD_SSL_KEY_STORE_TYPE, "SSL_KEY_STORE_TYPE"), op(CMD_SSL_KEY_STORE_PATH, "PATH"), op(CMD_SSL_KEY_STORE_PASSWORD, "PASSWORD"),
+        op(CMD_SSL_TRUSTSTORE_TYPE, "SSL_TRUST_STORE_TYPE"), op(CMD_SSL_TRUSTSTORE_PATH, "PATH"), op(CMD_SSL_TRUSTSTORE_PASSWORD, "PASSWORD"));
 
     /** Utility name with common options. */
     private static final String UTILITY_NAME_WITH_COMMON_OPTIONS = String.join(" ", UTILITY_NAME, COMMON_OPTIONS);
@@ -629,7 +647,6 @@ private <R> R executeTask(GridClient client, Class<? extends ComputeTask<?, R>>
 
     /**
      * @param client Client.
-     *
      * @return List of hosts.
      */
     private Stream<IgniteBiTuple<GridClientNode, String>> listHosts(GridClient client) throws GridClientException {
@@ -643,7 +660,6 @@ private Stream<IgniteBiTuple<GridClientNode, String>> listHosts(GridClient clien
 
     /**
      * @param client Client.
-     *
      * @return List of hosts.
      */
     private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNode(GridClient client) throws GridClientException {
@@ -876,12 +892,12 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
         }
 
         for (Map.Entry<UUID, VisorValidateIndexesJobResult> nodeEntry : taskRes.results().entrySet()) {
-            boolean errors = false;
-
             log("validate_indexes result on node " + nodeEntry.getKey() + ":");
 
             Collection<IndexIntegrityCheckIssue> integrityCheckFailures = nodeEntry.getValue().integrityCheckFailures();
 
+            boolean errors = false;
+
             if (!integrityCheckFailures.isEmpty()) {
                 errors = true;
 
@@ -1030,10 +1046,9 @@ private void cachesConfig(GridClient client, CacheArguments cacheArgs,
             executeTaskByNameOnNode(client, VisorCacheConfigurationCollectorTask.class.getName(), taskArg, nodeId);
 
         Map<String, Integer> cacheToMapped =
-            viewRes.cacheInfos().stream().collect(Collectors.toMap(x -> x.getCacheName(), x -> x.getMapped()));
+            viewRes.cacheInfos().stream().collect(Collectors.toMap(CacheInfo::getCacheName, CacheInfo::getMapped));
 
         printCachesConfig(res, cacheArgs.outputFormat(), cacheToMapped);
-
     }
 
     /**
@@ -1816,22 +1831,24 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         VisorTxTaskArg txArgs = null;
 
-        boolean sslEnable= false;
+        boolean sslEnable = false;
 
         String sslProtocol = SslContextFactory.DFLT_SSL_PROTOCOL;
 
         String sslAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM;
 
-        String sslKeyStorePath = null;
+        String sslCipherSuites = "";
 
         String sslKeyStoreType = SslContextFactory.DFLT_STORE_TYPE;
 
-        char sslKeyStorePassword[] = null;
+        String sslKeyStorePath = null;
 
-        String sslTrustStorePath = null;
+        char sslKeyStorePassword[] = null;
 
         String sslTrustStoreType = SslContextFactory.DFLT_STORE_TYPE;
 
+        String sslTrustStorePath = null;
+
         char sslTrustStorePassword[] = null;
 
         while (hasNextArg()) {
@@ -1957,42 +1974,47 @@ Arguments parseAndValidate(List<String> rawArgs) {
                         break;
 
                     case CMD_SSL_PROTOCOL:
-                        sslProtocol = nextArg("Expected ssl protocol");
+                        sslProtocol = nextArg("Expected SSL protocol");
 
                         break;
 
                     case CMD_SSL_ALGORITHM:
-                        sslAlgorithm = nextArg("Expected ssl algorithm");
+                        sslAlgorithm = nextArg("Expected SSL algorithm");
+
+                        break;
+
+                    case CMD_SSL_CIPHER_SUITES:
+                        sslCipherSuites = nextArg("Expected SSL cipher suites");
 
                         break;
 
                     case CMD_SSL_KEY_STORE_PATH:
-                        sslKeyStorePath = nextArg("Expected ssl key store path");
+                        sslKeyStorePath = nextArg("Expected SSL key store path");
 
                         break;
 
                     case CMD_SSL_KEY_STORE_TYPE:
-                        sslKeyStoreType = nextArg("Expected ssl key store type");
+                        sslKeyStoreType = nextArg("Expected SSL key store type");
 
                         break;
 
                     case CMD_SSL_KEY_STORE_PASSWORD:
-                        sslKeyStorePassword = nextArg("Expected ssl key store password").toCharArray();
+                        sslKeyStorePassword = nextArg("Expected SSL key store password").toCharArray();
 
                         break;
 
                     case CMD_SSL_TRUSTSTORE_PATH:
-                        sslTrustStorePath = nextArg("Expected ssl trust store path");
+                        sslTrustStorePath = nextArg("Expected SSL trust store path");
 
                         break;
 
                     case CMD_SSL_TRUSTSTORE_TYPE:
-                        sslTrustStoreType = nextArg("Expected ssl trust store type");
+                        sslTrustStoreType = nextArg("Expected SSL trust store type");
 
                         break;
 
                     case CMD_SSL_TRUSTSTORE_PASSWORD:
-                        sslTrustStorePassword = nextArg("Expected ssl trust store password").toCharArray();
+                        sslTrustStorePassword = nextArg("Expected SSL trust store password").toCharArray();
 
                         break;
 
@@ -2012,9 +2034,12 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         Command cmd = commands.get(0);
 
-        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
+        return new Arguments(cmd, host, port, user, pwd,
+            baselineAct, baselineArgs,
+            txArgs, cacheArgs,
+            walAct, walArgs,
             pingTimeout, pingInterval, autoConfirmation,
-            sslEnable, sslProtocol, sslAlgorithm,
+            sslEnable, sslProtocol, sslAlgorithm, sslCipherSuites,
             sslKeyStorePath, sslKeyStoreType, sslKeyStorePassword,
             sslTrustStorePath, sslTrustStoreType, sslTrustStorePassword);
     }
@@ -2497,6 +2522,23 @@ private Map<String, Object> mapToPairs(VisorCacheConfiguration cfg) {
         return params;
     }
 
+    /**
+     * Split string into items.
+     *
+     * @param s String to process.
+     * @param delim Delimiter.
+     * @return List with items.
+     */
+    private List<String> split(String s, String delim) {
+        if (F.isEmpty(s))
+            return Collections.emptyList();
+
+        return Arrays.stream(s.split(delim))
+            .map(String::trim)
+            .filter(item -> !item.isEmpty())
+            .collect(Collectors.toList());
+    }
+
     /**
      * Parse and execute command.
      *
@@ -2541,6 +2583,11 @@ public int execute(List<String> rawArgs) {
                 log(i("PORT=" + DFLT_PORT, 2));
                 log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2));
                 log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2));
+                log(i("SSL_PROTOCOL=" + SslContextFactory.DFLT_SSL_PROTOCOL, 2));
+                log(i("SSL_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2));
+                log(i("SSL_KEY_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
+                log(i("SSL_TRUST_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
+
                 nl();
 
                 log("Exit codes:");
@@ -2575,28 +2622,37 @@ public int execute(List<String> rawArgs) {
 
             clientCfg.setServers(Collections.singletonList(args.host() + ":" + args.port()));
 
-            if (args.isSslEnable()){
+            if (args.isSslEnable()) {
                 GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
 
-                factory.setProtocol(args.getSslProtocol());
+                List<String> sslProtocols = split(args.getSslProtocol(), ",");
+
+                String sslProtocol = F.isEmpty(sslProtocols) ? SslContextFactory.DFLT_SSL_PROTOCOL : sslProtocols.get(0);
+
+                factory.setProtocol(sslProtocol);
                 factory.setKeyAlgorithm(args.getSslAlgorithm());
 
-                if (args.getSslKeyStorePath()==null)
+                if (sslProtocols.size() > 1)
+                    factory.setProtocols(sslProtocols);
+
+                factory.setCipherSuites(split(args.getSslCipherSuites(), ","));
+
+                if (args.getSslKeyStorePath() == null)
                     throw new IllegalArgumentException("SSL key store location is not specified.");
 
                 factory.setKeyStoreFilePath(args.getSslKeyStorePath());
 
-                if (args.getSslKeyStorePassword()!=null)
+                if (args.getSslKeyStorePassword() != null)
                     factory.setKeyStorePassword(args.getSslKeyStorePassword());
 
                 factory.setKeyStoreType(args.getSslKeyStoreType());
 
-                if (args.getSslTrustStorePath()==null)
+                if (args.getSslTrustStorePath() == null)
                     factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
                 else {
                     factory.setTrustStoreFilePath(args.getSslTrustStorePath());
 
-                    if (args.getSslTrustStorePassword()!=null)
+                    if (args.getSslTrustStorePassword() != null)
                         factory.setTrustStorePassword(args.getSslTrustStorePassword());
 
                     factory.setTrustStoreType(args.getSslTrustStoreType());
@@ -2607,7 +2663,7 @@ public int execute(List<String> rawArgs) {
 
             boolean tryConnectAgain = true;
 
-            int tryConnectMaxCount=3;
+            int tryConnectMaxCount = 3;
 
             while (tryConnectAgain) {
                 tryConnectAgain = false;
@@ -2632,42 +2688,42 @@ public int execute(List<String> rawArgs) {
                         case ACTIVATE:
                             activate(client);
 
-                        break;
+                            break;
 
-                    case DEACTIVATE:
-                        deactivate(client);
+                        case DEACTIVATE:
+                            deactivate(client);
 
-                        break;
+                            break;
 
-                    case STATE:
-                        state(client);
+                        case STATE:
+                            state(client);
 
-                        break;
+                            break;
 
-                    case BASELINE:
-                        baseline(client, args.baselineAction(), args.baselineArguments());
+                        case BASELINE:
+                            baseline(client, args.baselineAction(), args.baselineArguments());
 
-                        break;
+                            break;
 
-                    case TX:
-                        transactions(client, args.transactionArguments());
+                        case TX:
+                            transactions(client, args.transactionArguments());
 
-                        break;
+                            break;
 
-                    case CACHE:
-                        cache(client, args.cacheArgs());
+                        case CACHE:
+                            cache(client, args.cacheArgs());
 
-                        break;
+                            break;
 
-                    case WAL:
-                        wal(client, args.walAction(), args.walArguments());
+                        case WAL:
+                            wal(client, args.walAction(), args.walArguments());
 
                             break;
                     }
                 }
                 catch (Throwable e) {
                     if (tryConnectMaxCount > 0 && isAuthError(e)) {
-                        System.out.println("Authentication error, try connection again.");
+                        log("Authentication error, try connection again.");
 
                         final Console console = System.console();
 
@@ -2680,13 +2736,13 @@ public int execute(List<String> rawArgs) {
                         else {
                             Scanner scanner = new Scanner(System.in);
 
-                            if (F.isEmpty(args.getUserName())){
-                                System.out.println("user: ");
+                            if (F.isEmpty(args.getUserName())) {
+                                log("user: ");
 
                                 args.setUserName(scanner.next());
                             }
 
-                            System.out.println("password: ");
+                            log("password: ");
 
                             args.setPassword(scanner.next());
                         }
diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java b/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java
index 901d42b1f4ba9..67bc834d0886c 100644
--- a/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/ssl/SSLContextWrapper.java
@@ -20,10 +20,15 @@
 import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLParameters;
 
-/** */
-class SSLContextWrapper extends SSLContext {
-    /** */
-    SSLContextWrapper(SSLContext delegate, SSLParameters sslParameters) {
+/**
+ * Wrapper for {@link SSLContext} that extend source context with custom SSL parameters.
+ */
+public class SSLContextWrapper extends SSLContext {
+    /**
+     * @param delegate Wrapped SSL context.
+     * @param sslParameters Extended SSL parameters.
+     */
+    public SSLContextWrapper(SSLContext delegate, SSLParameters sslParameters) {
         super(new DelegatingSSLContextSpi(delegate, sslParameters),
             delegate.getProvider(),
             delegate.getProtocol());
diff --git a/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java b/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java
index c514b0fe2d2c4..aeac15af1b279 100644
--- a/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/ssl/SslContextFactory.java
@@ -93,7 +93,7 @@ public class SslContextFactory implements Factory<SSLContext> {
     /** Enabled cipher suites. */
     private String[] cipherSuites;
 
-    /** Enabled cipher suites. */
+    /** Enabled protocols. */
     private String[] protocols;
 
     /**
@@ -289,6 +289,7 @@ public static TrustManager getDisabledTrustManager() {
 
     /**
      * Sets enabled cipher suites.
+     *
      * @param cipherSuites enabled cipher suites.
      */
     public void setCipherSuites(String... cipherSuites) {
@@ -296,7 +297,8 @@ public void setCipherSuites(String... cipherSuites) {
     }
 
     /**
-     * Gets enabled cipher suites
+     * Gets enabled cipher suites.
+     *
      * @return enabled cipher suites
      */
     public String[] getCipherSuites() {
@@ -304,8 +306,9 @@ public String[] getCipherSuites() {
     }
 
     /**
-     * Gets enabled cipher suites
-     * @return enabled cipher suites
+     * Gets enabled protocols.
+     *
+     * @return Enabled protocols.
      */
     public String[] getProtocols() {
         return protocols;
@@ -313,7 +316,8 @@ public String[] getProtocols() {
 
     /**
      * Sets enabled protocols.
-     * @param protocols enabled protocols.
+     *
+     * @param protocols Enabled protocols.
      */
     public void setProtocols(String... protocols) {
         this.protocols = protocols;
@@ -515,4 +519,4 @@ private static class DisabledX509TrustManager implements X509TrustManager {
             throw new IgniteException(e);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java b/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java
index 7ac6108197a24..20eb94cfd97d2 100644
--- a/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java
@@ -18,7 +18,6 @@
 package org.apache.ignite.client;
 
 import java.util.concurrent.Callable;
-import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.Ignition;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ClientConfiguration;
@@ -33,8 +32,9 @@
  * Tests cases when node connects to cluster with different set of cipher suites.
  */
 public class SslParametersTest extends GridCommonAbstractTest {
-
+    /** */
     public static final String TEST_CACHE_NAME = "TEST";
+
     /** */
     private volatile String[] cipherSuites;
 
@@ -58,8 +58,10 @@ public class SslParametersTest extends GridCommonAbstractTest {
         return cfg;
     }
 
-    /** {@inheritDoc} */
-    protected ClientConfiguration getClientConfiguration() throws Exception {
+    /**
+     * @return Client config.
+     */
+    protected ClientConfiguration getClientConfiguration() {
         ClientConfiguration cfg = new ClientConfiguration();
 
         cfg.setAddresses("127.0.0.1:10800");
@@ -71,6 +73,9 @@ protected ClientConfiguration getClientConfiguration() throws Exception {
         return cfg;
     }
 
+    /**
+     * @return SSL factory.
+     */
     @NotNull private SslContextFactory createSslFactory() {
         SslContextFactory factory = (SslContextFactory)GridTestUtils.sslTrustedFactory(
             "node01", "trustone");
@@ -314,8 +319,6 @@ private void checkClientStartFailure(String[][] cipherSuites, String[][] protoco
             this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
             this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
 
-            int finalI = i;
-
             GridTestUtils.assertThrows(null, new Callable<Object>() {
                 @Override public Object call() throws Exception {
                     Ignition.startClient(getClientConfiguration());
@@ -325,5 +328,4 @@ private void checkClientStartFailure(String[][] cipherSuites, String[][] protoco
             }, ex, msg);
         }
     }
-
 }

From c1244cc2cbcc0f81ff91fa6635a0ca6ca76f7b67 Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Thu, 15 Nov 2018 15:28:22 +0300
Subject: [PATCH 494/543] IGNITE-8735 Metastorage creates its own index
 partition - Fixes #5383.

Signed-off-by: Ivan Rakov <irakov@apache.org>
(cherry picked from commit a74d9c4f66f407bfde56173a727b2545cc3bfb99)
---
 .../internal/pagemem/PageIdAllocator.java     |  3 +
 .../persistence/metastorage/MetaStorage.java  |  7 ++
 .../metastorage/MetastorageTree.java          |  9 +++
 .../IgniteMetaStorageBasicTest.java           | 78 +++++++++++++++++++
 4 files changed, 97 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
index c6aeabe087975..b3e4b07b9fb67 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
@@ -35,6 +35,9 @@ public interface PageIdAllocator {
     /** Special partition reserved for index space. */
     public static final int INDEX_PARTITION = 0xFFFF;
 
+    /** Special partition reserved for metastore space. */
+    public static final int METASTORE_PARTITION = 0x0;
+
     /**
      * Allocates a page from the space for the given partition ID and the given flags.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index 6ce4757c8b911..95a9df39df023 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
@@ -62,6 +63,7 @@
 import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.jetbrains.annotations.NotNull;
 
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
 import static org.apache.ignite.internal.pagemem.PageIdUtils.itemId;
 import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
 
@@ -506,6 +508,11 @@ public static class FreeListImpl extends AbstractFreeList<MetastorageDataRow> {
             return SimpleDataPageIO.VERSIONS;
         }
 
+        /** {@inheritDoc} */
+        @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
+            return pageMem.allocatePage(grpId, PageIdAllocator.METASTORE_PARTITION, FLAG_DATA);
+        }
+
         /**
          * Read row from data pages.
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
index 19a145f1019ac..0c28289dd591b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
@@ -19,6 +19,7 @@
 
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.PageUtils;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
@@ -31,6 +32,9 @@
 import org.apache.ignite.internal.processors.failure.FailureProcessor;
 import org.jetbrains.annotations.Nullable;
 
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
+
 /**
  *
  */
@@ -94,6 +98,11 @@ public MetastorageRowStore rowStore() {
         return rowStore;
     }
 
+    /** {@inheritDoc} */
+    @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
+        return pageMem.allocatePage(grpId, PageIdAllocator.METASTORE_PARTITION, FLAG_DATA);
+    }
+
     /**
      *
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
index e5a53fac1833b..b6dfc12826177 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
@@ -17,6 +17,9 @@
 package org.apache.ignite.internal.processors.cache.persistence.metastorage;
 
 import java.io.Serializable;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DataRegionConfiguration;
@@ -78,6 +81,81 @@ public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest {
         cleanPersistenceDir();
     }
 
+    /**
+     *
+     */
+    public void testMetaStorageMassivePutFixed() throws Exception {
+        IgniteEx ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database();
+
+        MetaStorage metaStorage = db.metaStorage();
+
+        assertNotNull(metaStorage);
+
+        Random rnd = new Random();
+
+        db.checkpointReadLock();
+
+        int size;
+        try {
+            for (int i = 0; i < 10_000; i++) {
+                size = rnd.nextBoolean() ? 3500 : 2 * 3500;
+                String key = "TEST_KEY_" + (i % 1000);
+
+                byte[] arr = new byte[size];
+                rnd.nextBytes(arr);
+
+                metaStorage.remove(key);
+
+                metaStorage.putData(key, arr/*b.toString().getBytes()*/);
+            }
+        }
+        finally {
+            db.checkpointReadUnlock();
+        }
+    }
+
+    /**
+     *
+     */
+    public void testMetaStorageMassivePutRandom() throws Exception {
+        IgniteEx ig = startGrid(0);
+
+        ig.cluster().active(true);
+
+        IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database();
+
+        MetaStorage metaStorage = db.metaStorage();
+
+        assertNotNull(metaStorage);
+
+        Random rnd = new Random();
+
+        db.checkpointReadLock();
+
+        int size;
+        try {
+            for (int i = 0; i < 50_000; i++) {
+                size = 100 + rnd.nextInt(9000);
+
+                String key = "TEST_KEY_" + (i % 2_000);
+
+                byte[] arr = new byte[size];
+                rnd.nextBytes(arr);
+
+                metaStorage.remove(key);
+
+                metaStorage.putData(key, arr/*b.toString().getBytes()*/);
+            }
+        }
+        finally {
+            db.checkpointReadUnlock();
+        }
+    }
+
     /**
      * Verifies that MetaStorage after massive amounts of keys stored and updated keys restores its state successfully
      * after restart.

From 0fa5231d10a344b463604c7599c104e59d800f58 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 16 Nov 2018 16:17:01 +0700
Subject: [PATCH 495/543] IGNITE-10293 Collect caches info only on active
 cluster.

(cherry picked from commit eed26bcfb26d58372459f84173c4d0969dc08c8a)
---
 .../ignite/internal/visor/node/VisorNodeDataCollectorJob.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
index 5fab8d16d5e94..9025ed084b9c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
@@ -335,7 +335,8 @@ protected VisorNodeDataCollectorJobResult run(VisorNodeDataCollectorJobResult re
         if (debug)
             start0 = log(ignite.log(), "Collected memory metrics", getClass(), start0);
 
-        caches(res, arg);
+        if (ignite.cluster().active())
+            caches(res, arg);
 
         if (debug)
             start0 = log(ignite.log(), "Collected caches", getClass(), start0);

From ba0188e3b49a3554f07b18042d62795d17edb6cd Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Fri, 16 Nov 2018 18:56:28 +0300
Subject: [PATCH 496/543] IGNITE-12364 Reworked Sending Full Message logging -
 Fixes #5413.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 34c059a15b3505c8937045b2b973874286bc148a)
---
 .../GridDhtPartitionsExchangeFuture.java          | 10 ----------
 .../communication/tcp/TcpCommunicationSpi.java    | 15 +++++++++++++--
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index cbc7070afafb7..100760ee59e86 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1668,8 +1668,6 @@ private void sendAllPartitions(
             .map(singleMessage -> fullMsg.copy().joinedNodeAffinity(affinityForJoinedNodes))
             .orElse(null);
 
-        long time = System.currentTimeMillis();
-
         // Prepare and send full messages for given nodes.
         nodes.stream()
             .map(node -> {
@@ -1722,9 +1720,6 @@ private void sendAllPartitions(
                     U.error(log, "Failed to send partitions [node=" + node + ']', e);
                 }
             });
-
-        if (log.isInfoEnabled())
-            log.info("Sending Full Message performed in " + (System.currentTimeMillis() - time) + " ms.");
     }
 
     /**
@@ -3120,16 +3115,11 @@ else if (forceAffReassignment)
                         nodes.addAll(sndResNodes);
                 }
 
-                time = System.currentTimeMillis();
-
                 if (!nodes.isEmpty())
                     sendAllPartitions(msg, nodes, mergedJoinExchMsgs0, joinedNodeAff);
 
                 partitionsSent = true;
 
-                if (log.isInfoEnabled())
-                    log.info("Sending Full Message to all nodes performed in " + (System.currentTimeMillis() - time) + " ms.");
-
                 if (!stateChangeExchange())
                     onDone(exchCtx.events().topologyVersion(), null);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
index 7f9d0666212b8..f37bdb9433977 100755
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/TcpCommunicationSpi.java
@@ -263,6 +263,9 @@
 @IgniteSpiMultipleInstancesSupport(true)
 @IgniteSpiConsistencyChecked(optional = false)
 public class TcpCommunicationSpi extends IgniteSpiAdapter implements CommunicationSpi<Message> {
+    /** Time threshold to log too long connection establish. */
+    private static final int CONNECTION_ESTABLISH_THRESHOLD_MS = 100;
+
     /** IPC error message. */
     public static final String OUT_OF_RESOURCES_TCP_MSG = "Failed to allocate shared memory segment " +
         "(switching to TCP, may be slower).";
@@ -2974,10 +2977,18 @@ else if (log.isDebugEnabled())
         connectGate.enter();
 
         try {
+            final long start = System.currentTimeMillis();
+
             GridCommunicationClient client = createTcpClient(node, connIdx);
 
-            if (log.isDebugEnabled())
-                log.debug("TCP client created: " + client);
+            final long time = System.currentTimeMillis() - start;
+
+            if (time > CONNECTION_ESTABLISH_THRESHOLD_MS) {
+                if (log.isInfoEnabled())
+                    log.info("TCP client created [client=" + client + ", duration=" + time + "ms]");
+            }
+            else if (log.isDebugEnabled())
+                log.debug("TCP client created [client=" + client + ", duration=" + time + "ms]");
 
             return client;
         }

From ff7388f77b7c29a6c9a8b5d64607d9a0f18d1659 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Sat, 17 Nov 2018 01:32:02 +0300
Subject: [PATCH 497/543] IGNITE-10257 Control.sh utility should request a SSL
 keystore password and SSL truststore password if necessary

---
 .../internal/commandline/CommandHandler.java  | 200 +++++++++++-------
 1 file changed, 125 insertions(+), 75 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 58fba736ec019..7c4c75b42fe93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -367,15 +367,7 @@ public class CommandHandler {
     private static final String UTILITY_NAME = "control.sh";
 
     /** Common options. */
-    private static final String COMMON_OPTIONS = String.join(" ", op(CMD_HOST, "HOST_OR_IP"), op(CMD_PORT, "PORT"),
-        op(CMD_USER, "USER"), op(CMD_PASSWORD, "PASSWORD"),
-        op(CMD_PING_INTERVAL, "PING_INTERVAL"), op(CMD_PING_TIMEOUT, "PING_TIMEOUT"),
-        op(CMD_SSL_ENABLED),
-        op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ...]"),
-        op(CMD_SSL_ALGORITHM, "SSL_ALGORITHM"),
-        op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ...]"),
-        op(CMD_SSL_KEY_STORE_TYPE, "SSL_KEY_STORE_TYPE"), op(CMD_SSL_KEY_STORE_PATH, "PATH"), op(CMD_SSL_KEY_STORE_PASSWORD, "PASSWORD"),
-        op(CMD_SSL_TRUSTSTORE_TYPE, "SSL_TRUST_STORE_TYPE"), op(CMD_SSL_TRUSTSTORE_PATH, "PATH"), op(CMD_SSL_TRUSTSTORE_PASSWORD, "PASSWORD"));
+    private static final String COMMON_OPTIONS = String.join(" ", getCommonOptions());
 
     /** Utility name with common options. */
     private static final String UTILITY_NAME_WITH_COMMON_OPTIONS = String.join(" ", UTILITY_NAME, COMMON_OPTIONS);
@@ -401,6 +393,34 @@ public class CommandHandler {
     /** Check if experimental commands are enabled. Default {@code false}. */
     private final boolean enableExperimental = IgniteSystemProperties.getBoolean(IGNITE_ENABLE_EXPERIMENTAL_COMMAND, false);
 
+    /**
+     * Creates list of common utility options.
+     *
+     * @return List of common utility options.
+     */
+    private static List<String> getCommonOptions() {
+        List<String> list = new ArrayList<>(32);
+
+        list.add(op(CMD_HOST, "HOST_OR_IP"));
+        list.add(op(CMD_PORT, "PORT"));
+        list.add(op(CMD_USER, "USER"));
+        list.add(op(CMD_PASSWORD, "PASSWORD"));
+        list.add(op(CMD_PING_INTERVAL, "PING_INTERVAL"));
+        list.add(op(CMD_PING_TIMEOUT, "PING_TIMEOUT"));
+        list.add(op(CMD_SSL_ENABLED));
+        list.add(op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ...]"));
+        list.add(op(CMD_SSL_ALGORITHM, "SSL_ALGORITHM"));
+        list.add(op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ...]"));
+        list.add(op(CMD_SSL_KEY_STORE_TYPE, "SSL_KEY_STORE_TYPE"));
+        list.add(op(CMD_SSL_KEY_STORE_PATH, "PATH"));
+        list.add(op(CMD_SSL_KEY_STORE_PASSWORD, "PASSWORD"));
+        list.add(op(CMD_SSL_TRUSTSTORE_TYPE, "SSL_TRUST_STORE_TYPE"));
+        list.add(op(CMD_SSL_TRUSTSTORE_PATH, "PATH"));
+        list.add(op(CMD_SSL_TRUSTSTORE_PASSWORD, "PASSWORD"));
+
+        return list;
+    }
+
     /**
      * Output specified string to console.
      *
@@ -650,7 +670,9 @@ private <R> R executeTask(GridClient client, Class<? extends ComputeTask<?, R>>
      * @return List of hosts.
      */
     private Stream<IgniteBiTuple<GridClientNode, String>> listHosts(GridClient client) throws GridClientException {
-        return client.compute().nodes(GridClientNode::connectable).stream()
+        return client.compute()
+            .nodes(GridClientNode::connectable)
+            .stream()
             .flatMap(node -> Stream.concat(
                 node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
                 node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
@@ -662,14 +684,15 @@ private Stream<IgniteBiTuple<GridClientNode, String>> listHosts(GridClient clien
      * @param client Client.
      * @return List of hosts.
      */
-    private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNode(GridClient client) throws GridClientException {
+    private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNode(
+        GridClient client) throws GridClientException {
         return client.compute().nodes(GridClientNode::connectable).stream()
             .map(node -> new IgniteBiTuple<>(node,
                 Stream.concat(
                     node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
                     node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
                 )
-                .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
+                    .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
     }
 
     /**
@@ -2412,6 +2435,41 @@ private long nextLongArg(String lb) {
         }
     }
 
+    /**
+     * Requests password from console with message.
+     *
+     * @param msg Message.
+     * @return Password.
+     */
+    private char[] requestPasswordFromConsole(String msg) {
+        Console console = System.console();
+
+        if (console == null)
+            throw new UnsupportedOperationException("Failed to securely read password (console is unavailable): " + msg);
+        else
+            return console.readPassword(msg);
+    }
+
+    /**
+     * Requests user data from console with message.
+     *
+     * @param msg Message.
+     * @return Input user data.
+     */
+    private String requestDataFromConsole(String msg) {
+        Console console = System.console();
+
+        if (console != null)
+            return console.readLine(msg);
+        else {
+            Scanner scanner = new Scanner(System.in);
+
+            log(msg);
+
+            return scanner.nextLine();
+        }
+    }
+
     /**
      * Check if raw arg is command or option.
      *
@@ -2539,6 +2597,53 @@ private List<String> split(String s, String delim) {
             .collect(Collectors.toList());
     }
 
+    /** */
+    private void printHelp() {
+        log("This utility can do the following commands:");
+
+        usage(i("Activate cluster:"), ACTIVATE);
+        usage(i("Deactivate cluster:"), DEACTIVATE, op(CMD_AUTO_CONFIRMATION));
+        usage(i("Print current cluster state:"), STATE);
+        usage(i("Print cluster baseline topology:"), BASELINE);
+        usage(i("Add nodes into baseline topology:"), BASELINE, BASELINE_ADD, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION));
+        usage(i("Remove nodes from baseline topology:"), BASELINE, BASELINE_REMOVE, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION));
+        usage(i("Set baseline topology:"), BASELINE, BASELINE_SET, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION));
+        usage(i("Set baseline topology based on version:"), BASELINE, BASELINE_SET_VERSION + " topologyVersion", op(CMD_AUTO_CONFIRMATION));
+        usage(i("List or kill transactions:"), TX, op(TX_XID, "XID"), op(TX_DURATION, "SECONDS"), op(TX_SIZE, "SIZE"), op(TX_LABEL, "PATTERN_REGEX"), op(or(TX_SERVERS, TX_CLIENTS)), op(TX_NODES, "consistentId1[,consistentId2,....,consistentIdN]"), op(TX_LIMIT, "NUMBER"), op(TX_ORDER, or("DURATION", "SIZE", CMD_TX_ORDER_START_TIME)), op(TX_KILL), op(CMD_AUTO_CONFIRMATION));
+
+        if (enableExperimental) {
+            usage(i("Print absolute paths of unused archived wal segments on each node:"), WAL, WAL_PRINT, "[consistentId1,consistentId2,....,consistentIdN]");
+            usage(i("Delete unused archived wal segments on each node:"), WAL, WAL_DELETE, "[consistentId1,consistentId2,....,consistentIdN] ", op(CMD_AUTO_CONFIRMATION));
+        }
+
+        log(i("View caches information in a cluster. For more details type:"));
+        log(i(String.join(" ", UTILITY_NAME, CACHE.text(), HELP.text()), 2));
+        nl();
+
+        log("By default commands affecting the cluster require interactive confirmation.");
+        log("Use " + CMD_AUTO_CONFIRMATION + " option to disable it.");
+        nl();
+
+        log("Default values:");
+        log(i("HOST_OR_IP=" + DFLT_HOST, 2));
+        log(i("PORT=" + DFLT_PORT, 2));
+        log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2));
+        log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2));
+        log(i("SSL_PROTOCOL=" + SslContextFactory.DFLT_SSL_PROTOCOL, 2));
+        log(i("SSL_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2));
+        log(i("SSL_KEY_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
+        log(i("SSL_TRUST_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
+
+        nl();
+
+        log("Exit codes:");
+        log(i(EXIT_CODE_OK + " - successful execution.", 2));
+        log(i(EXIT_CODE_INVALID_ARGUMENTS + " - invalid arguments.", 2));
+        log(i(EXIT_CODE_CONNECTION_FAILED + " - connection failed.", 2));
+        log(i(ERR_AUTHENTICATION_FAILED + " - authentication failed.", 2));
+        log(i(EXIT_CODE_UNEXPECTED_ERROR + " - unexpected error.", 2));
+    }
+
     /**
      * Parse and execute command.
      *
@@ -2553,49 +2658,7 @@ public int execute(List<String> rawArgs) {
 
         try {
             if (F.isEmpty(rawArgs) || (rawArgs.size() == 1 && CMD_HELP.equalsIgnoreCase(rawArgs.get(0)))) {
-                log("This utility can do the following commands:");
-
-                usage(i("Activate cluster:"), ACTIVATE);
-                usage(i("Deactivate cluster:"), DEACTIVATE, op(CMD_AUTO_CONFIRMATION));
-                usage(i("Print current cluster state:"), STATE);
-                usage(i("Print cluster baseline topology:"), BASELINE);
-                usage(i("Add nodes into baseline topology:"), BASELINE, BASELINE_ADD, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION));
-                usage(i("Remove nodes from baseline topology:"), BASELINE, BASELINE_REMOVE, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION));
-                usage(i("Set baseline topology:"), BASELINE, BASELINE_SET, "consistentId1[,consistentId2,....,consistentIdN]", op(CMD_AUTO_CONFIRMATION));
-                usage(i("Set baseline topology based on version:"), BASELINE, BASELINE_SET_VERSION + " topologyVersion", op(CMD_AUTO_CONFIRMATION));
-                usage(i("List or kill transactions:"), TX, op(TX_XID, "XID"), op(TX_DURATION, "SECONDS"), op(TX_SIZE, "SIZE"), op(TX_LABEL, "PATTERN_REGEX"), op(or(TX_SERVERS, TX_CLIENTS)), op(TX_NODES, "consistentId1[,consistentId2,....,consistentIdN]"), op(TX_LIMIT, "NUMBER"), op(TX_ORDER, or("DURATION", "SIZE", CMD_TX_ORDER_START_TIME)), op(TX_KILL), op(CMD_AUTO_CONFIRMATION));
-
-                if (enableExperimental) {
-                    usage(i("Print absolute paths of unused archived wal segments on each node:"), WAL, WAL_PRINT, "[consistentId1,consistentId2,....,consistentIdN]");
-                    usage(i("Delete unused archived wal segments on each node:"), WAL, WAL_DELETE, "[consistentId1,consistentId2,....,consistentIdN] ", op(CMD_AUTO_CONFIRMATION));
-                }
-
-                log(i("View caches information in a cluster. For more details type:"));
-                log(i(String.join(" ", UTILITY_NAME, CACHE.text(), HELP.text()), 2));
-                nl();
-
-                log("By default commands affecting the cluster require interactive confirmation.");
-                log("Use " + CMD_AUTO_CONFIRMATION + " option to disable it.");
-                nl();
-
-                log("Default values:");
-                log(i("HOST_OR_IP=" + DFLT_HOST, 2));
-                log(i("PORT=" + DFLT_PORT, 2));
-                log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2));
-                log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2));
-                log(i("SSL_PROTOCOL=" + SslContextFactory.DFLT_SSL_PROTOCOL, 2));
-                log(i("SSL_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2));
-                log(i("SSL_KEY_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
-                log(i("SSL_TRUST_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
-
-                nl();
-
-                log("Exit codes:");
-                log(i(EXIT_CODE_OK + " - successful execution.", 2));
-                log(i(EXIT_CODE_INVALID_ARGUMENTS + " - invalid arguments.", 2));
-                log(i(EXIT_CODE_CONNECTION_FAILED + " - connection failed.", 2));
-                log(i(ERR_AUTHENTICATION_FAILED + " - authentication failed.", 2));
-                log(i(EXIT_CODE_UNEXPECTED_ERROR + " - unexpected error.", 2));
+                printHelp();
 
                 return EXIT_CODE_OK;
             }
@@ -2644,6 +2707,8 @@ public int execute(List<String> rawArgs) {
 
                 if (args.getSslKeyStorePassword() != null)
                     factory.setKeyStorePassword(args.getSslKeyStorePassword());
+                else
+                    factory.setKeyStorePassword(requestPasswordFromConsole("SSL keystore password: "));
 
                 factory.setKeyStoreType(args.getSslKeyStoreType());
 
@@ -2654,6 +2719,8 @@ public int execute(List<String> rawArgs) {
 
                     if (args.getSslTrustStorePassword() != null)
                         factory.setTrustStorePassword(args.getSslTrustStorePassword());
+                    else
+                        factory.setTrustStorePassword(requestPasswordFromConsole("SSL truststore password: "));
 
                     factory.setTrustStoreType(args.getSslTrustStoreType());
                 }
@@ -2725,27 +2792,10 @@ public int execute(List<String> rawArgs) {
                     if (tryConnectMaxCount > 0 && isAuthError(e)) {
                         log("Authentication error, try connection again.");
 
-                        final Console console = System.console();
-
-                        if (console != null) {
-                            if (F.isEmpty(args.getUserName()))
-                                args.setUserName(console.readLine("user: "));
+                        if (F.isEmpty(args.getUserName()))
+                            args.setUserName(requestDataFromConsole("user: "));
 
-                            args.setPassword(new String(console.readPassword("password: ")));
-                        }
-                        else {
-                            Scanner scanner = new Scanner(System.in);
-
-                            if (F.isEmpty(args.getUserName())) {
-                                log("user: ");
-
-                                args.setUserName(scanner.next());
-                            }
-
-                            log("password: ");
-
-                            args.setPassword(scanner.next());
-                        }
+                        args.setPassword(new String(requestPasswordFromConsole("password: ")));
 
                         tryConnectAgain = true;
 

From a426ba4038644eedf7f6a228b71a0f9e8fb1d77a Mon Sep 17 00:00:00 2001
From: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
Date: Mon, 19 Nov 2018 16:41:28 +0300
Subject: [PATCH 498/543] IGNITE-10285 Fixed U.doInParallel may lead to
 deadlock - Fixes #5404.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>

(cherry picked from commit acfdcdaa0f667b1ebebc26cea90df030804056a7)
Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
---
 .../ignite/internal/util/IgniteUtils.java     | 130 +++++++++++++++--
 .../internal/util/IgniteUtilsSelfTest.java    | 131 ++++++++++++++++++
 2 files changed, 249 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index f12340c6f179a..3c4fb61922580 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -10555,40 +10555,92 @@ public static <T, R> Collection<R> doInParallel(
     ) throws IgniteCheckedException, IgniteInterruptedCheckedException {
         if(srcDatas.isEmpty())
             return Collections.emptyList();
+
         int[] batchSizes = calculateOptimalBatchSizes(parallelismLvl, srcDatas.size());
 
-        List<List<T>> batches = new ArrayList<>(batchSizes.length);
+        List<Batch<T, R>> batches = new ArrayList<>(batchSizes.length);
+
+        // Set for sharing batches between executor and current thread.
+        // If executor cannot perform immediately, we will execute task in the current thread.
+        Set<Batch<T, R>> sharedBatchesSet = new GridConcurrentHashSet<>(batchSizes.length);
 
         Iterator<T> iterator = srcDatas.iterator();
 
-        for (int batchSize : batchSizes) {
-            List<T> batch = new ArrayList<>(batchSize);
+        for (int idx = 0; idx < batchSizes.length; idx++) {
+            int batchSize = batchSizes[idx];
+
+            Batch<T, R> batch = new Batch<>(batchSize);
 
             for (int i = 0; i < batchSize; i++)
-                batch.add(iterator.next());
+                batch.addTask(iterator.next());
 
             batches.add(batch);
         }
 
-        List<Future<Collection<R>>> consumerFutures = batches.stream()
-            .filter(batch -> !batch.isEmpty())
-            .map(batch -> executorSvc.submit(() -> {
-                Collection<R> results = new ArrayList<>(batch.size());
+        batches = batches.stream()
+            .filter(batch -> !batch.tasks.isEmpty())
+            // Add to set only after check that batch is not empty.
+            .peek(sharedBatchesSet::add)
+            // Setup future in batch for waiting result.
+            .peek(batch -> batch.future = executorSvc.submit(() -> {
+                // Batch was stolen by the main stream.
+                if (!sharedBatchesSet.remove(batch)) {
+                    return null;
+                }
+
+                Collection<R> results = new ArrayList<>(batch.tasks.size());
 
-                for (T item : batch)
+                for (T item : batch.tasks)
                     results.add(operation.accept(item));
 
                 return results;
             }))
             .collect(Collectors.toList());
 
-        Throwable error =null;
+        Throwable error = null;
+
+        // Stealing jobs if executor is busy and cannot process task immediately.
+        // Perform batches in a current thread.
+        for (Batch<T, R> batch : sharedBatchesSet) {
+            // Executor steal task.
+            if (!sharedBatchesSet.remove(batch))
+                continue;
+
+            Collection<R> res = new ArrayList<>(batch.tasks.size());
+
+            try {
+                for (T item : batch.tasks)
+                    res.add(operation.accept(item));
+
+                batch.result(res);
+            }
+            catch (IgniteCheckedException e) {
+                batch.result(e);
+            }
+        }
 
+        // Final result collection.
         Collection<R> results = new ArrayList<>(srcDatas.size());
 
-        for (Future<Collection<R>> future : consumerFutures) {
+        for (Batch<T, R> batch: batches) {
             try {
-                results.addAll(future.get());
+                Throwable err = batch.error;
+
+                if (err != null) {
+                    if (error == null)
+                        error = err;
+                    else
+                        error.addSuppressed(err);
+
+                    continue;
+                }
+
+                Collection<R> res = batch.result();
+
+                if (res != null)
+                    results.addAll(res);
+                else
+                    assert error != null;
             }
             catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
@@ -10625,6 +10677,60 @@ public static <T, R> Collection<R> doInParallel(
         return results;
     }
 
+    /**
+     * The batch of tasks with a batch index in global array.
+     */
+    private static class Batch<T,R> {
+        /** List tasks. */
+        private final List<T> tasks;
+
+        /** */
+        private Collection<R> result;
+
+        /** */
+        private Throwable error;
+
+        /** */
+        private Future<Collection<R>> future;
+
+        /**
+         * @param batchSize Batch size.
+         */
+        private Batch(int batchSize) {
+            this.tasks = new ArrayList<>(batchSize);
+        }
+
+        /**
+         * @param task Add task.
+         */
+        public void addTask(T task){
+            tasks.add(task);
+        }
+
+        /**
+         * @param res Setup results for tasks.
+         */
+        public void result(Collection<R> res) {
+            this.result = res;
+        }
+
+        /**
+         * @param e Throwable if task was completed with error.
+         */
+        public void result(Throwable e) {
+            this.error = e;
+        }
+
+        /**
+         * Get tasks results.
+         */
+        public Collection<R> result() throws ExecutionException, InterruptedException {
+            assert future != null;
+
+            return result != null ? result : future.get();
+        }
+    }
+
     /**
      * Split number of tasks into optimized batches.
      * @param parallelismLvl Level of parallelism.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
index 196466d4a8c37..65b2bb324577c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
@@ -44,20 +44,27 @@
 import java.util.List;
 import java.util.Random;
 import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteInterruptedException;
 import org.apache.ignite.cluster.ClusterGroup;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobAdapter;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.igfs.IgfsUtils;
 import org.apache.ignite.internal.util.lang.GridPeerDeployAware;
+import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -68,7 +75,9 @@
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.apache.ignite.testframework.junits.common.GridCommonTest;
 import org.jetbrains.annotations.Nullable;
+import org.junit.Assert;
 
+import static java.util.Arrays.asList;
 import static org.junit.Assert.assertArrayEquals;
 
 /**
@@ -960,6 +969,128 @@ public void testDoInParallelResultsOrder() throws IgniteCheckedException {
         }
     }
 
+    /**
+     * Test parallel execution steal job.
+     */
+    public void testDoInParallelWithStealingJob() throws IgniteCheckedException {
+        // Pool size should be less that input data collection.
+        ExecutorService executorService = Executors.newFixedThreadPool(1);
+
+        CountDownLatch mainThreadLatch = new CountDownLatch(1);
+        CountDownLatch poolThreadLatch = new CountDownLatch(1);
+
+        // Busy one thread from the pool.
+        executorService.submit(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    poolThreadLatch.await();
+                }
+                catch (InterruptedException e) {
+                    throw new IgniteInterruptedException(e);
+                }
+            }
+        });
+
+        List<Integer> data = asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+        AtomicInteger taskProcessed = new AtomicInteger();
+
+        long threadId = Thread.currentThread().getId();
+
+        AtomicInteger curThreadCnt = new AtomicInteger();
+        AtomicInteger poolThreadCnt = new AtomicInteger();
+
+        Collection<Integer> res = U.doInParallel(10,
+            executorService,
+            data,
+            new IgniteThrowableConsumer<Integer, Integer>() {
+                @Override public Integer accept(Integer cnt) throws IgniteInterruptedCheckedException {
+                    // Release thread in pool in the middle of range.
+                    if (taskProcessed.getAndIncrement() == (data.size() / 2) - 1) {
+                        poolThreadLatch.countDown();
+
+                        try {
+                            // Await thread in thread pool complete task.
+                            mainThreadLatch.await();
+                        }
+                        catch (InterruptedException e) {
+                            Thread.currentThread().interrupt();
+
+                            throw new IgniteInterruptedCheckedException(e);
+                        }
+                    }
+
+                    // Increment if executed in current thread.
+                    if (Thread.currentThread().getId() == threadId)
+                        curThreadCnt.incrementAndGet();
+                    else {
+                        poolThreadCnt.incrementAndGet();
+
+                        if (taskProcessed.get() == data.size())
+                            mainThreadLatch.countDown();
+                    }
+
+                    return -cnt;
+                }
+            });
+
+        Assert.assertEquals(curThreadCnt.get() + poolThreadCnt.get(), data.size());
+        Assert.assertEquals(5, curThreadCnt.get());
+        Assert.assertEquals(5, poolThreadCnt.get());
+        Assert.assertEquals(asList(0, -1, -2, -3, -4, -5, -6, -7, -8, -9), res);
+    }
+
+    /**
+     * Test parallel execution steal job.
+     */
+    public void testDoInParallelWithStealingJobRunTaskInExecutor() throws Exception {
+        // Pool size should be less that input data collection.
+        ExecutorService executorService = Executors.newFixedThreadPool(2);
+
+        Future<?> f1 = executorService.submit(()-> runTask(executorService));
+        Future<?> f2 = executorService.submit(()-> runTask(executorService));
+        Future<?> f3 = executorService.submit(()-> runTask(executorService));
+
+        f1.get();
+        f2.get();
+        f3.get();
+    }
+
+    /**
+     *
+     * @param executorService Executor service.
+     */
+    private void runTask(ExecutorService executorService) {
+        List<Integer> data = asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
+
+        long threadId = Thread.currentThread().getId();
+
+        AtomicInteger curThreadCnt = new AtomicInteger();
+
+        Collection<Integer> res;
+
+        try {
+            res = U.doInParallel(10,
+                executorService,
+                data,
+                new IgniteThrowableConsumer<Integer, Integer>() {
+                    @Override public Integer accept(Integer cnt) {
+                        if (Thread.currentThread().getId() == threadId)
+                            curThreadCnt.incrementAndGet();
+
+                        return -cnt;
+                    }
+                });
+        }
+        catch (IgniteCheckedException e) {
+            throw new IgniteException(e);
+        }
+
+        Assert.assertTrue(curThreadCnt.get() > 0);
+        Assert.assertEquals(asList(0, -1, -2, -3, -4, -5, -6, -7, -8, -9), res);
+    }
+
     /**
      * Template method to test parallel execution
      * @param executorService ExecutorService.

From bf75c3eac183c4228e48d00340936cd96627ba02 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Mon, 19 Nov 2018 17:00:52 +0300
Subject: [PATCH 499/543] IGNITE-9558 Avoid blocking transactions on client
 connect when possible - Fixes #4933.

(cherry picked from commit a63a81a51418a8ae1dadff019d9e5eace38e1631)
---
 .../ignite/codegen/MessageCodeGenerator.java  |  10 +-
 .../ignite/internal/GridJobCancelRequest.java |   2 +-
 .../internal/GridJobExecuteRequest.java       |   4 +-
 .../internal/GridJobExecuteResponse.java      |   4 +-
 .../internal/GridJobSiblingsRequest.java      |   2 +-
 .../internal/GridJobSiblingsResponse.java     |   2 +-
 .../internal/GridTaskCancelRequest.java       |   2 +-
 .../internal/GridTaskSessionRequest.java      |   2 +-
 .../internal/direct/DirectMessageReader.java  |  30 ++
 .../internal/direct/DirectMessageWriter.java  |  26 ++
 .../direct/stream/DirectByteBufferStream.java |  11 +
 .../v1/DirectByteBufferStreamImplV1.java      |  13 +
 .../v2/DirectByteBufferStreamImplV2.java      |  19 +-
 .../v3/DirectByteBufferStreamImplV3.java      | 298 ++++++++++++++++++
 .../checkpoint/GridCheckpointRequest.java     |   2 +-
 .../managers/communication/GridIoManager.java |   2 +-
 .../communication/GridIoUserMessage.java      |   2 +-
 .../communication/IgniteIoTestMessage.java    |   2 +
 .../deployment/GridDeploymentInfoBean.java    |   2 +-
 .../deployment/GridDeploymentRequest.java     |   2 +-
 .../deployment/GridDeploymentResponse.java    |   2 +-
 .../eventstorage/GridEventStorageMessage.java |   2 +-
 .../affinity/AffinityTopologyVersion.java     |   2 +-
 .../affinity/GridAffinityAssignmentCache.java |  18 +-
 .../UserAuthenticateResponseMessage.java      |   2 +-
 .../cache/CacheEntryInfoCollection.java       |   2 +-
 .../processors/cache/CacheEntryPredicate.java |   2 +-
 .../cache/CacheEntryPredicateAdapter.java     |   2 +-
 .../CacheEntryPredicateContainsValue.java     |   2 +-
 .../cache/CacheEntryPredicateHasValue.java    |   2 +-
 .../cache/CacheEntryPredicateNoValue.java     |   2 +-
 .../CacheEntrySerializablePredicate.java      |   2 +-
 .../processors/cache/CacheEvictionEntry.java  |   2 +-
 .../cache/CacheInvokeDirectResult.java        |   2 +-
 .../processors/cache/CacheObject.java         |   2 +-
 .../processors/cache/CacheObjectAdapter.java  |   2 +-
 .../cache/CacheObjectByteArrayImpl.java       |   2 +-
 .../processors/cache/CacheObjectImpl.java     |   2 +-
 .../cache/GridCacheAffinityManager.java       |   2 +-
 .../processors/cache/GridCacheIoManager.java  |  11 +-
 .../processors/cache/GridCacheMessage.java    |  17 +-
 .../GridCachePartitionExchangeManager.java    | 120 +++++--
 .../processors/cache/KeyCacheObject.java      |   2 +-
 .../processors/cache/WalStateAckMessage.java  |   2 +
 .../GridCacheTtlUpdateRequest.java            |   4 +-
 .../GridDistributedTxFinishRequest.java       |   4 +-
 .../dht/ClientCacheDhtTopologyFuture.java     |   5 -
 .../dht/GridDhtAffinityAssignmentRequest.java |   4 +-
 .../GridDhtAffinityAssignmentResponse.java    |   4 +-
 .../distributed/dht/GridDhtCacheAdapter.java  |   7 +-
 .../distributed/dht/GridDhtLockRequest.java   |   4 +-
 .../dht/GridDhtTopologyFuture.java            |   6 -
 .../dht/GridDhtTransactionalCacheAdapter.java |   4 +-
 .../dht/GridDhtTxPrepareRequest.java          |   2 +-
 .../GridDhtAtomicAbstractUpdateRequest.java   |   4 +-
 .../dht/atomic/GridDhtAtomicCache.java        |   3 +-
 .../GridNearAtomicAbstractUpdateRequest.java  |   4 +-
 .../atomic/GridNearAtomicUpdateResponse.java  |   4 +-
 .../preloader/GridDhtForceKeysRequest.java    |   4 +-
 .../GridDhtPartitionDemandLegacyMessage.java  |   4 +-
 .../GridDhtPartitionDemandMessage.java        |  18 +-
 .../preloader/GridDhtPartitionExchangeId.java |   6 +-
 .../GridDhtPartitionSupplyMessage.java        |  12 +-
 .../GridDhtPartitionsExchangeFuture.java      |  18 +-
 .../GridDhtPartitionsFullMessage.java         |   8 +-
 .../GridDhtPartitionsSingleMessage.java       |   1 +
 .../dht/preloader/latch/LatchAckMessage.java  |   6 +-
 .../GridDhtPartitionTopologyImpl.java         |   8 +-
 .../distributed/near/CacheVersionedValue.java |   2 +-
 .../distributed/near/GridNearGetResponse.java |   4 +-
 .../distributed/near/GridNearLockRequest.java |   4 +-
 .../near/GridNearLockResponse.java            |   4 +-
 .../near/GridNearSingleGetRequest.java        |   4 +-
 .../near/GridNearSingleGetResponse.java       |   4 +-
 .../near/GridNearTxPrepareRequest.java        |   4 +-
 .../near/GridNearTxPrepareResponse.java       |   4 +-
 .../cache/query/GridCacheQueryRequest.java    |   4 +-
 .../continuous/CacheContinuousQueryEntry.java |   4 +-
 .../cache/transactions/IgniteTxHandler.java   |   7 +-
 .../cache/transactions/IgniteTxKey.java       |   2 +-
 .../version/GridCacheRawVersionedEntry.java   |   4 +-
 .../continuous/GridContinuousMessage.java     |   2 +-
 .../datastreamer/DataStreamerRequest.java     |   4 +-
 .../datastreamer/DataStreamerResponse.java    |   2 +-
 .../shuffle/HadoopDirectShuffleMessage.java   |   4 +-
 .../hadoop/shuffle/HadoopShuffleMessage.java  |   2 +
 .../processors/igfs/IgfsAckMessage.java       |   2 +-
 .../processors/igfs/IgfsBlockKey.java         |   2 +-
 .../processors/igfs/IgfsBlocksMessage.java    |   2 +-
 .../igfs/IgfsCommunicationMessage.java        |   2 +-
 .../processors/igfs/IgfsDeleteMessage.java    |   2 +-
 .../igfs/IgfsFragmentizerRequest.java         |   2 +-
 .../igfs/IgfsFragmentizerResponse.java        |   2 +-
 .../processors/igfs/IgfsSyncMessage.java      |   2 +-
 .../messages/GridQueryNextPageResponse.java   |   4 +-
 .../message/SchemaOperationStatusMessage.java |   2 +
 .../handlers/task/GridTaskResultRequest.java  |   2 +-
 .../handlers/task/GridTaskResultResponse.java |   2 +-
 .../internal/util/GridByteArrayList.java      |   2 +-
 .../ignite/internal/util/GridIntList.java     |   2 +-
 .../ignite/internal/util/GridLongList.java    |   2 +
 .../MessageCollectionItemType.java            |   5 +-
 .../communication/MessageReader.java          |   9 +
 .../communication/MessageWriter.java          |  10 +
 .../jobstealing/JobStealingRequest.java       |   2 +-
 .../tcp/messages/HandshakeMessage.java        |   2 +
 .../tcp/messages/HandshakeMessage2.java       |   2 +
 .../tcp/messages/NodeIdMessage.java           |   2 +
 .../messages/RecoveryLastReceivedMessage.java |   2 +
 .../cache/CacheNoAffinityExchangeTest.java    | 218 +++++++++++++
 .../IgniteClusterActivateDeactivateTest.java  |   3 +-
 .../distributed/CacheExchangeMergeTest.java   |   2 +
 .../testsuites/IgniteCacheTestSuite6.java     |   3 +
 .../processors/cache/query/QueryTable.java    |   2 +
 .../twostep/msg/GridH2IndexRangeRequest.java  |   2 +
 .../twostep/msg/GridH2IndexRangeResponse.java |   2 +
 .../h2/twostep/msg/GridH2QueryRequest.java    |   6 +-
 .../h2/twostep/msg/GridH2ValueMessage.java    |   2 +
 118 files changed, 952 insertions(+), 206 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v3/DirectByteBufferStreamImplV3.java
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java

diff --git a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
index 99cf84946ab92..bd0c60d0a2363 100644
--- a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
+++ b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
@@ -44,6 +44,7 @@
 import org.apache.ignite.internal.GridDirectMap;
 import org.apache.ignite.internal.GridDirectTransient;
 import org.apache.ignite.internal.IgniteCodeGeneratingFail;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.typedef.internal.SB;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteUuid;
@@ -115,6 +116,7 @@ public class MessageCodeGenerator {
         TYPES.put(BitSet.class, MessageCollectionItemType.BIT_SET);
         TYPES.put(UUID.class, MessageCollectionItemType.UUID);
         TYPES.put(IgniteUuid.class, MessageCollectionItemType.IGNITE_UUID);
+        TYPES.put(AffinityTopologyVersion.class, MessageCollectionItemType.AFFINITY_TOPOLOGY_VERSION);
     }
 
     /**
@@ -168,9 +170,7 @@ public static void main(String[] args) throws Exception {
 
 //        gen.generateAll(true);
 
-//        gen.generateAndWrite(GridChangeGlobalStateMessageResponse.class);
-
-//        gen.generateAndWrite(GridNearAtomicUpdateRequest.class);
+//        gen.generateAndWrite(GridCacheMessage.class);
 
 //        gen.generateAndWrite(GridMessageCollection.class);
 //        gen.generateAndWrite(DataStreamerEntry.class);
@@ -658,6 +658,8 @@ else if (type == UUID.class)
             returnFalseIfFailed(write, "writer.writeUuid", field, getExpr);
         else if (type == IgniteUuid.class)
             returnFalseIfFailed(write, "writer.writeIgniteUuid", field, getExpr);
+        else if (type == AffinityTopologyVersion.class)
+            returnFalseIfFailed(write, "writer.writeAffinityTopologyVersion", field, getExpr);
         else if (type.isEnum()) {
             String arg = getExpr + " != null ? (byte)" + getExpr + ".ordinal() : -1";
 
@@ -740,6 +742,8 @@ else if (type == UUID.class)
             returnFalseIfReadFailed(name, "reader.readUuid", setExpr, field);
         else if (type == IgniteUuid.class)
             returnFalseIfReadFailed(name, "reader.readIgniteUuid", setExpr, field);
+        else if (type == AffinityTopologyVersion.class)
+            returnFalseIfReadFailed(name, "reader.readAffinityTopologyVersion", setExpr, field);
         else if (type.isEnum()) {
             String loc = name + "Ord";
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobCancelRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobCancelRequest.java
index aaa69eaff7190..ac3a87336fc9f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobCancelRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobCancelRequest.java
@@ -201,4 +201,4 @@ public boolean system() {
     @Override public String toString() {
         return S.toString(GridJobCancelRequest.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java
index 4357d1da07dce..ebfeb0153ffdc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteRequest.java
@@ -664,7 +664,7 @@ public AffinityTopologyVersion getTopVer() {
                 writer.incrementState();
 
             case 24:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -885,7 +885,7 @@ public AffinityTopologyVersion getTopVer() {
                 reader.incrementState();
 
             case 24:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteResponse.java
index 312435e922750..f052edf07d446 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobExecuteResponse.java
@@ -282,7 +282,7 @@ public AffinityTopologyVersion getRetryTopologyVersion() {
                 writer.incrementState();
 
             case 6:
-                if (!writer.writeMessage("retry", retry))
+                if (!writer.writeAffinityTopologyVersion("retry", retry))
                     return false;
 
                 writer.incrementState();
@@ -355,7 +355,7 @@ public AffinityTopologyVersion getRetryTopologyVersion() {
                 reader.incrementState();
 
             case 6:
-                retry = reader.readMessage("retry");
+                retry = reader.readAffinityTopologyVersion("retry");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsRequest.java
index 8a11cef33aa21..d743a355f8d8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsRequest.java
@@ -161,4 +161,4 @@ public byte[] topicBytes() {
     @Override public String toString() {
         return S.toString(GridJobSiblingsRequest.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsResponse.java
index 3911446d2b86b..dc59ab5f3057c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridJobSiblingsResponse.java
@@ -141,4 +141,4 @@ public void unmarshalSiblings(Marshaller marsh) throws IgniteCheckedException {
     @Override public String toString() {
         return S.toString(GridJobSiblingsResponse.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskCancelRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskCancelRequest.java
index 273d0a777a468..71c318b537dd5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskCancelRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskCancelRequest.java
@@ -124,4 +124,4 @@ public IgniteUuid sessionId() {
     @Override public String toString() {
         return S.toString(GridTaskCancelRequest.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionRequest.java
index dbac893189e38..576392e097fc3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridTaskSessionRequest.java
@@ -189,4 +189,4 @@ public IgniteUuid getJobId() {
     @Override public String toString() {
         return S.toString(GridTaskSessionRequest.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageReader.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageReader.java
index 7d3644fdef18e..b4e52726d3bb2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageReader.java
@@ -27,6 +27,9 @@
 import org.apache.ignite.internal.direct.stream.DirectByteBufferStream;
 import org.apache.ignite.internal.direct.stream.v1.DirectByteBufferStreamImplV1;
 import org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2;
+import org.apache.ignite.internal.direct.stream.v3.DirectByteBufferStreamImplV3;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteOutClosure;
 import org.apache.ignite.lang.IgniteUuid;
@@ -41,8 +44,13 @@
  */
 public class DirectMessageReader implements MessageReader {
     /** State. */
+    @GridToStringInclude
     private final DirectMessageState<StateItem> state;
 
+    /** Protocol version. */
+    @GridToStringInclude
+    private final byte protoVer;
+
     /** Whether last field was fully read. */
     private boolean lastRead;
 
@@ -56,6 +64,8 @@ public DirectMessageReader(final MessageFactory msgFactory, final byte protoVer)
                 return new StateItem(msgFactory, protoVer);
             }
         });
+
+        this.protoVer = protoVer;
     }
 
     /** {@inheritDoc} */
@@ -304,6 +314,21 @@ public DirectMessageReader(final MessageFactory msgFactory, final byte protoVer)
         return val;
     }
 
+    /** {@inheritDoc} */
+    @Override public AffinityTopologyVersion readAffinityTopologyVersion(String name) {
+        if (protoVer >= 3) {
+            DirectByteBufferStream stream = state.item().stream;
+
+            AffinityTopologyVersion val = stream.readAffinityTopologyVersion();
+
+            lastRead = stream.lastFinished();
+
+            return val;
+        }
+
+        return readMessage(name);
+    }
+
     /** {@inheritDoc} */
     @Nullable @Override public <T extends Message> T readMessage(String name) {
         DirectByteBufferStream stream = state.item().stream;
@@ -409,6 +434,11 @@ public StateItem(MessageFactory msgFactory, byte protoVer) {
 
                     break;
 
+                case 3:
+                    stream = new DirectByteBufferStreamImplV3(msgFactory);
+
+                    break;
+
                 default:
                     throw new IllegalStateException("Invalid protocol version: " + protoVer);
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java
index b224d68fbd8f2..ba30538e372a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java
@@ -27,6 +27,8 @@
 import org.apache.ignite.internal.direct.stream.DirectByteBufferStream;
 import org.apache.ignite.internal.direct.stream.v1.DirectByteBufferStreamImplV1;
 import org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2;
+import org.apache.ignite.internal.direct.stream.v3.DirectByteBufferStreamImplV3;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.lang.IgniteOutClosure;
@@ -44,6 +46,10 @@ public class DirectMessageWriter implements MessageWriter {
     @GridToStringInclude
     private final DirectMessageState<StateItem> state;
 
+    /** Protocol version. */
+    @GridToStringInclude
+    private final byte protoVer;
+
     /**
      * @param protoVer Protocol version.
      */
@@ -53,6 +59,8 @@ public DirectMessageWriter(final byte protoVer) {
                 return new StateItem(protoVer);
             }
         });
+
+        this.protoVer = protoVer;
     }
 
     /** {@inheritDoc} */
@@ -272,6 +280,19 @@ public DirectMessageWriter(final byte protoVer) {
         return stream.lastFinished();
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean writeAffinityTopologyVersion(String name, AffinityTopologyVersion val) {
+        if (protoVer >= 3) {
+            DirectByteBufferStream stream = state.item().stream;
+
+            stream.writeAffinityTopologyVersion(val);
+
+            return stream.lastFinished();
+        }
+
+        return writeMessage(name, val);
+    }
+
     /** {@inheritDoc} */
     @Override public boolean writeMessage(String name, @Nullable Message msg) {
         DirectByteBufferStream stream = state.item().stream;
@@ -376,6 +397,11 @@ public StateItem(byte protoVer) {
 
                     break;
 
+                case 3:
+                    stream = new DirectByteBufferStreamImplV3(null);
+
+                    break;
+
                 default:
                     throw new IllegalStateException("Invalid protocol version: " + protoVer);
             }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java
index 204e6b034530b..ae5502eb75f8e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java
@@ -22,6 +22,7 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
@@ -160,6 +161,11 @@ public interface DirectByteBufferStream {
      */
     public void writeIgniteUuid(IgniteUuid val);
 
+    /**
+     * @param val Value.
+     */
+    public void writeAffinityTopologyVersion(AffinityTopologyVersion val);
+
     /**
      * @param msg Message.
      * @param writer Writer.
@@ -289,6 +295,11 @@ public <K, V> void writeMap(Map<K, V> map, MessageCollectionItemType keyType, Me
      */
     public IgniteUuid readIgniteUuid();
 
+    /**
+     * @return Value.
+     */
+    public AffinityTopologyVersion readAffinityTopologyVersion();
+
     /**
      * @param reader Reader.
      * @return Message.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v1/DirectByteBufferStreamImplV1.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v1/DirectByteBufferStreamImplV1.java
index 76cb76243c757..7e845e0cd6ae1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v1/DirectByteBufferStreamImplV1.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v1/DirectByteBufferStreamImplV1.java
@@ -27,6 +27,7 @@
 import java.util.NoSuchElementException;
 import java.util.UUID;
 import org.apache.ignite.internal.direct.stream.DirectByteBufferStream;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -494,6 +495,11 @@ public DirectByteBufferStreamImplV1(MessageFactory msgFactory) {
         writeByteArray(val != null ? U.igniteUuidToBytes(val) : null);
     }
 
+    /** {@inheritDoc} */
+    @Override public void writeAffinityTopologyVersion(AffinityTopologyVersion val) {
+        throw new UnsupportedOperationException("Not implemented");
+    }
+
     /** {@inheritDoc} */
     @Override public void writeMessage(Message msg, MessageWriter writer) {
         if (msg != null) {
@@ -811,6 +817,11 @@ public DirectByteBufferStreamImplV1(MessageFactory msgFactory) {
         return arr != null ? U.bytesToIgniteUuid(arr, 0) : null;
     }
 
+    /** {@inheritDoc} */
+    @Override public AffinityTopologyVersion readAffinityTopologyVersion() {
+        throw new UnsupportedOperationException("Not implemented");
+    }
+
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public <T extends Message> T readMessage(MessageReader reader) {
@@ -1212,6 +1223,7 @@ private void write(MessageCollectionItemType type, Object val, MessageWriter wri
 
                 break;
 
+            case AFFINITY_TOPOLOGY_VERSION:
             case MSG:
                 try {
                     if (val != null)
@@ -1298,6 +1310,7 @@ private Object read(MessageCollectionItemType type, MessageReader reader) {
             case IGNITE_UUID:
                 return readIgniteUuid();
 
+            case AFFINITY_TOPOLOGY_VERSION:
             case MSG:
                 return readMessage(reader);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
index e338bc0187d46..fd93cfb81f832 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v2/DirectByteBufferStreamImplV2.java
@@ -29,6 +29,7 @@
 import java.util.UUID;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.internal.direct.stream.DirectByteBufferStream;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.util.GridUnsafe;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -298,7 +299,7 @@ public class DirectByteBufferStreamImplV2 implements DirectByteBufferStream {
     private long uuidLocId;
 
     /** */
-    private boolean lastFinished;
+    protected boolean lastFinished;
 
     /**
      * @param msgFactory Message factory.
@@ -657,6 +658,11 @@ public DirectByteBufferStreamImplV2(MessageFactory msgFactory) {
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void writeAffinityTopologyVersion(AffinityTopologyVersion val) {
+        throw new UnsupportedOperationException("Not implemented");
+    }
+
     /** {@inheritDoc} */
     @Override public void writeMessage(Message msg, MessageWriter writer) {
         if (msg != null) {
@@ -1152,6 +1158,11 @@ private <T> void writeRandomAccessList(List<T> list, MessageCollectionItemType i
         return val;
     }
 
+    /** {@inheritDoc} */
+    @Override public AffinityTopologyVersion readAffinityTopologyVersion() {
+        throw new UnsupportedOperationException("Not implemented");
+    }
+
     /** {@inheritDoc} */
     @SuppressWarnings("unchecked")
     @Override public <T extends Message> T readMessage(MessageReader reader) {
@@ -1587,7 +1598,7 @@ <T> T readArrayLE(ArrayCreator<T> creator, int typeSize, int lenShift, long off)
      * @param val Value.
      * @param writer Writer.
      */
-    private void write(MessageCollectionItemType type, Object val, MessageWriter writer) {
+    protected void write(MessageCollectionItemType type, Object val, MessageWriter writer) {
         switch (type) {
             case BYTE:
                 writeByte((Byte)val);
@@ -1689,6 +1700,7 @@ private void write(MessageCollectionItemType type, Object val, MessageWriter wri
 
                 break;
 
+            case AFFINITY_TOPOLOGY_VERSION:
             case MSG:
                 try {
                     if (val != null)
@@ -1713,7 +1725,7 @@ private void write(MessageCollectionItemType type, Object val, MessageWriter wri
      * @param reader Reader.
      * @return Value.
      */
-    private Object read(MessageCollectionItemType type, MessageReader reader) {
+    protected Object read(MessageCollectionItemType type, MessageReader reader) {
         switch (type) {
             case BYTE:
                 return readByte();
@@ -1775,6 +1787,7 @@ private Object read(MessageCollectionItemType type, MessageReader reader) {
             case IGNITE_UUID:
                 return readIgniteUuid();
 
+            case AFFINITY_TOPOLOGY_VERSION:
             case MSG:
                 return readMessage(reader);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v3/DirectByteBufferStreamImplV3.java b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v3/DirectByteBufferStreamImplV3.java
new file mode 100644
index 0000000000000..89043ebcbc0c5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/v3/DirectByteBufferStreamImplV3.java
@@ -0,0 +1,298 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.direct.stream.v3;
+
+import java.util.BitSet;
+import java.util.UUID;
+import org.apache.ignite.internal.direct.stream.v2.DirectByteBufferStreamImplV2;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
+import org.apache.ignite.plugin.extensions.communication.MessageFactory;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+
+/**
+ *
+ */
+public class DirectByteBufferStreamImplV3 extends DirectByteBufferStreamImplV2 {
+    /** */
+    private byte topVerState;
+
+    /** */
+    private long topVerMajor;
+
+    /** */
+    private int topVerMinor;
+
+    /**
+     * @param msgFactory Message factory.
+     */
+    public DirectByteBufferStreamImplV3(MessageFactory msgFactory) {
+        super(msgFactory);
+    }
+
+    /** {@inheritDoc} */
+    @Override public void writeAffinityTopologyVersion(AffinityTopologyVersion val) {
+        if (val != null) {
+            switch (topVerState) {
+                case 0:
+                    writeInt(val.minorTopologyVersion());
+
+                    if (!lastFinished)
+                        return;
+
+                    topVerState++;
+
+                case 1:
+                    writeLong(val.topologyVersion());
+
+                    if (!lastFinished)
+                        return;
+
+                    topVerState = 0;
+            }
+        }
+        else
+            writeInt(-1);
+    }
+
+    /** {@inheritDoc} */
+    @Override public AffinityTopologyVersion readAffinityTopologyVersion() {
+        switch (topVerState) {
+            case 0:
+                topVerMinor = readInt();
+
+                if (!lastFinished || topVerMinor == -1)
+                    return null;
+
+                topVerState++;
+
+            case 1:
+                topVerMajor = readLong();
+
+                if (!lastFinished)
+                    return null;
+
+                topVerState = 0;
+        }
+
+        return new AffinityTopologyVersion(topVerMajor, topVerMinor);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void write(MessageCollectionItemType type, Object val, MessageWriter writer) {
+        switch (type) {
+            case BYTE:
+                writeByte((Byte)val);
+
+                break;
+
+            case SHORT:
+                writeShort((Short)val);
+
+                break;
+
+            case INT:
+                writeInt((Integer)val);
+
+                break;
+
+            case LONG:
+                writeLong((Long)val);
+
+                break;
+
+            case FLOAT:
+                writeFloat((Float)val);
+
+                break;
+
+            case DOUBLE:
+                writeDouble((Double)val);
+
+                break;
+
+            case CHAR:
+                writeChar((Character)val);
+
+                break;
+
+            case BOOLEAN:
+                writeBoolean((Boolean)val);
+
+                break;
+
+            case BYTE_ARR:
+                writeByteArray((byte[])val);
+
+                break;
+
+            case SHORT_ARR:
+                writeShortArray((short[])val);
+
+                break;
+
+            case INT_ARR:
+                writeIntArray((int[])val);
+
+                break;
+
+            case LONG_ARR:
+                writeLongArray((long[])val);
+
+                break;
+
+            case FLOAT_ARR:
+                writeFloatArray((float[])val);
+
+                break;
+
+            case DOUBLE_ARR:
+                writeDoubleArray((double[])val);
+
+                break;
+
+            case CHAR_ARR:
+                writeCharArray((char[])val);
+
+                break;
+
+            case BOOLEAN_ARR:
+                writeBooleanArray((boolean[])val);
+
+                break;
+
+            case STRING:
+                writeString((String)val);
+
+                break;
+
+            case BIT_SET:
+                writeBitSet((BitSet)val);
+
+                break;
+
+            case UUID:
+                writeUuid((UUID)val);
+
+                break;
+
+            case IGNITE_UUID:
+                writeIgniteUuid((IgniteUuid)val);
+
+                break;
+
+            case AFFINITY_TOPOLOGY_VERSION:
+                writeAffinityTopologyVersion((AffinityTopologyVersion)val);
+
+                break;
+            case MSG:
+                try {
+                    if (val != null)
+                        writer.beforeInnerMessageWrite();
+
+                    writeMessage((Message)val, writer);
+                }
+                finally {
+                    if (val != null)
+                        writer.afterInnerMessageWrite(lastFinished);
+                }
+
+                break;
+
+            default:
+                throw new IllegalArgumentException("Unknown type: " + type);
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override protected Object read(MessageCollectionItemType type, MessageReader reader) {
+        switch (type) {
+            case BYTE:
+                return readByte();
+
+            case SHORT:
+                return readShort();
+
+            case INT:
+                return readInt();
+
+            case LONG:
+                return readLong();
+
+            case FLOAT:
+                return readFloat();
+
+            case DOUBLE:
+                return readDouble();
+
+            case CHAR:
+                return readChar();
+
+            case BOOLEAN:
+                return readBoolean();
+
+            case BYTE_ARR:
+                return readByteArray();
+
+            case SHORT_ARR:
+                return readShortArray();
+
+            case INT_ARR:
+                return readIntArray();
+
+            case LONG_ARR:
+                return readLongArray();
+
+            case FLOAT_ARR:
+                return readFloatArray();
+
+            case DOUBLE_ARR:
+                return readDoubleArray();
+
+            case CHAR_ARR:
+                return readCharArray();
+
+            case BOOLEAN_ARR:
+                return readBooleanArray();
+
+            case STRING:
+                return readString();
+
+            case BIT_SET:
+                return readBitSet();
+
+            case UUID:
+                return readUuid();
+
+            case IGNITE_UUID:
+                return readIgniteUuid();
+
+            case AFFINITY_TOPOLOGY_VERSION:
+                return readAffinityTopologyVersion();
+
+            case MSG:
+                return readMessage(reader);
+
+            default:
+                throw new IllegalArgumentException("Unknown type: " + type);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointRequest.java
index 8b21ff2fe5382..4b25e0b687662 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/checkpoint/GridCheckpointRequest.java
@@ -177,4 +177,4 @@ public String getCheckpointSpi() {
     @Override public String toString() {
         return S.toString(GridCheckpointRequest.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
index 8d9a70034aeb3..267addb887169 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoManager.java
@@ -133,7 +133,7 @@ public class GridIoManager extends GridManagerAdapter<CommunicationSpi<Serializa
     public static final String DIRECT_PROTO_VER_ATTR = "comm.direct.proto.ver";
 
     /** Direct protocol version. */
-    public static final byte DIRECT_PROTO_VER = 2;
+    public static final byte DIRECT_PROTO_VER = 3;
 
     /** Current IO policy. */
     private static final ThreadLocal<Byte> CUR_PLC = new ThreadLocal<>();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoUserMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoUserMessage.java
index 332a9de511c79..408fad773f93e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoUserMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoUserMessage.java
@@ -358,4 +358,4 @@ public void deployment(GridDeployment dep) {
     @Override public String toString() {
         return S.toString(GridIoUserMessage.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
index 0a8b2b7a38e60..a6a2469736ef2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/IgniteIoTestMessage.java
@@ -20,6 +20,7 @@
 import java.nio.ByteBuffer;
 import java.util.UUID;
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
@@ -28,6 +29,7 @@
 /**
  *
  */
+@IgniteCodeGeneratingFail
 public class IgniteIoTestMessage implements Message {
     /** */
     private static byte FLAG_PROC_FROM_NIO = 1;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java
index 7f58ce36001ab..72f5ec6b30579 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java
@@ -277,4 +277,4 @@ public GridDeploymentInfoBean(GridDeploymentInfo dep) {
     @Override public String toString() {
         return S.toString(GridDeploymentInfoBean.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentRequest.java
index 729cf4c54278e..708c64860579e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentRequest.java
@@ -278,4 +278,4 @@ public void nodeIds(Collection<UUID> nodeIds) {
     @Override public String toString() {
         return S.toString(GridDeploymentRequest.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentResponse.java
index d1b0384f0fb6a..591957d37bd2f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentResponse.java
@@ -197,4 +197,4 @@ void errorMessage(String errMsg) {
     @Override public String toString() {
         return S.toString(GridDeploymentResponse.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageMessage.java
index 515500b91d0f1..fd5326cf0577c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/eventstorage/GridEventStorageMessage.java
@@ -445,4 +445,4 @@ void exceptionBytes(byte[] exBytes) {
     @Override public String toString() {
         return S.toString(GridEventStorageMessage.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java
index 44b27534dee62..333841d1b33be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java
@@ -219,4 +219,4 @@ public int minorTopologyVersion() {
     @Override public String toString() {
         return S.toString(AffinityTopologyVersion.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
index e84954b33c257..cbaff1f0152ff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/GridAffinityAssignmentCache.java
@@ -618,7 +618,10 @@ public AffinityAssignment readyAffinity(AffinityTopologyVersion topVer) {
      * @return Cached affinity.
      */
     public AffinityAssignment cachedAffinity(AffinityTopologyVersion topVer) {
-        return cachedAffinity(topVer, topVer);
+        AffinityTopologyVersion lastAffChangeTopVer =
+            ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer);
+
+        return cachedAffinity(topVer, lastAffChangeTopVer);
     }
 
     /**
@@ -641,10 +644,15 @@ public AffinityAssignment cachedAffinity(AffinityTopologyVersion topVer, Affinit
 
         AffinityAssignment cache = head.get();
 
-        if (!cache.topologyVersion().equals(topVer)) {
-            cache = affCache.get(lastAffChangeTopVer);
+        if (!(cache.topologyVersion().compareTo(lastAffChangeTopVer) >= 0 &&
+            cache.topologyVersion().compareTo(topVer) <= 0)) {
 
-            if (cache == null) {
+            Map.Entry<AffinityTopologyVersion, HistoryAffinityAssignment> e = affCache.ceilingEntry(lastAffChangeTopVer);
+
+            if (e != null)
+                cache = e.getValue();
+
+            if (cache == null || cache.topologyVersion().compareTo(topVer) > 0) {
                 throw new IllegalStateException("Getting affinity for topology version earlier than affinity is " +
                     "calculated [locNode=" + ctx.discovery().localNode() +
                     ", grp=" + cacheOrGrpName +
@@ -656,7 +664,7 @@ public AffinityAssignment cachedAffinity(AffinityTopologyVersion topVer, Affinit
         }
 
         assert cache.topologyVersion().compareTo(lastAffChangeTopVer) >= 0 &&
-            cache.topologyVersion().compareTo(topVer) <= 0 : "Invalid cached affinity: " + cache;
+            cache.topologyVersion().compareTo(topVer) <= 0 : "Invalid cached affinity: [cache=" + cache + ", topVer=" + topVer + ", lastAffChangedTopVer=" + lastAffChangeTopVer + "]";
 
         return cache;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAuthenticateResponseMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAuthenticateResponseMessage.java
index d86b1ad91a551..e3dee3ce92a4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAuthenticateResponseMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/authentication/UserAuthenticateResponseMessage.java
@@ -98,7 +98,6 @@ public IgniteUuid id() {
 
                 writer.incrementState();
 
-
         }
 
         return true;
@@ -127,6 +126,7 @@ public IgniteUuid id() {
                     return false;
 
                 reader.incrementState();
+
         }
 
         return reader.afterMessageRead(UserAuthenticateResponseMessage.class);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryInfoCollection.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryInfoCollection.java
index 49f77fa808dba..d29ac09953ac8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryInfoCollection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryInfoCollection.java
@@ -116,4 +116,4 @@ public void add(GridCacheEntryInfo info) {
     @Override public byte fieldsCount() {
         return 1;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicate.java
index 61cbb9e04fb54..36312a1591135 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicate.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicate.java
@@ -42,4 +42,4 @@ public interface CacheEntryPredicate extends IgnitePredicate<GridCacheEntryEx>,
      * @param locked Entry locked
      */
     public void entryLocked(boolean locked);
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateAdapter.java
index e41938997daa2..62325323e1d3e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateAdapter.java
@@ -97,4 +97,4 @@ public abstract class CacheEntryPredicateAdapter implements CacheEntryPredicate
     @Override public void onAckReceived() {
         // No-op.
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java
index 76806a44f5cfd..73a98b74ac0cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateContainsValue.java
@@ -140,4 +140,4 @@ public CacheEntryPredicateContainsValue(CacheObject val) {
     @Override public String toString() {
         return S.toString(CacheEntryPredicateContainsValue.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateHasValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateHasValue.java
index cac04357a5816..210cc7059b21d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateHasValue.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateHasValue.java
@@ -28,4 +28,4 @@ public class CacheEntryPredicateHasValue extends CacheEntryPredicateAdapter {
     @Override public boolean apply(GridCacheEntryEx e) {
         return peekVisibleValue(e) != null;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateNoValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateNoValue.java
index 2790170e959e8..4c8917fcc68be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateNoValue.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntryPredicateNoValue.java
@@ -28,4 +28,4 @@ public class CacheEntryPredicateNoValue extends CacheEntryPredicateAdapter {
     @Override public boolean apply(GridCacheEntryEx e) {
         return peekVisibleValue(e) == null;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntrySerializablePredicate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntrySerializablePredicate.java
index 9057e41fbfbfc..257433636e6b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntrySerializablePredicate.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEntrySerializablePredicate.java
@@ -156,4 +156,4 @@ public CacheEntryPredicate predicate() {
     @Override public byte fieldsCount() {
         return 1;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictionEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictionEntry.java
index 2717b1e03e22e..96b85df2f0721 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictionEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheEvictionEntry.java
@@ -185,4 +185,4 @@ public void finishUnmarshal(GridCacheContext ctx, ClassLoader ldr) throws Ignite
     @Override public byte fieldsCount() {
         return 3;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
index 3b463afe8a25e..3f880339eb889 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheInvokeDirectResult.java
@@ -267,4 +267,4 @@ public void finishUnmarshal(GridCacheContext ctx, ClassLoader ldr) throws Ignite
     @Override public String toString() {
         return S.toString(CacheInvokeDirectResult.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java
index 3bc2a6dcb0624..f9f384a7f9702 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java
@@ -118,4 +118,4 @@ public interface CacheObject extends Message {
      * @throws IgniteCheckedException If failed.
      */
     public void prepareMarshal(CacheObjectValueContext ctx) throws IgniteCheckedException;
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java
index e2a15ee5d0aed..5987b1317bb14 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectAdapter.java
@@ -247,4 +247,4 @@ else if (off >= headSize)
 
         return true;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectByteArrayImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectByteArrayImpl.java
index 414bebbc2d27d..efe4ab196bc00 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectByteArrayImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectByteArrayImpl.java
@@ -191,4 +191,4 @@ public CacheObjectByteArrayImpl(byte[] val) {
     public String toString() {
         return "CacheObjectByteArrayImpl [arrLen=" + (val != null ? val.length : 0) + ']';
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectImpl.java
index 2124a97940b9b..b29c19e1e254c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObjectImpl.java
@@ -152,4 +152,4 @@ else if (kernalCtx.config().isPeerClassLoadingEnabled())
     @Override public CacheObject prepareForCache(CacheObjectContext ctx) {
         return this;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
index 03f95aef80b50..e20855fef68f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAffinityManager.java
@@ -232,7 +232,7 @@ public List<ClusterNode> nodesByPartition(int part, AffinityTopologyVersion topV
      * @return Affinity assignment.
      */
     public AffinityAssignment assignment(AffinityTopologyVersion topVer) {
-        return assignment(topVer, topVer);
+        return assignment(topVer, cctx.shared().exchange().lastAffinityChangedTopologyVersion(topVer));
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 5e8b2461576c6..23cd4ba1afec9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -165,6 +165,11 @@ public void dumpPendingMessages(StringBuilder sb) {
 
             final GridCacheMessage cacheMsg = (GridCacheMessage)msg;
 
+            AffinityTopologyVersion rmtAffVer = cacheMsg.topologyVersion();
+            AffinityTopologyVersion lastAffChangedVer = cacheMsg.lastAffinityChangedTopologyVersion();
+
+            cctx.exchange().lastAffinityChangedTopologyVersion(rmtAffVer, lastAffChangedVer);
+
             IgniteInternalFuture<?> fut = null;
 
             if (cacheMsg.partitionExchangeMessage()) {
@@ -222,8 +227,6 @@ else if (desc.receivedFromStartVersion() != null)
             }
             else {
                 AffinityTopologyVersion locAffVer = cctx.exchange().readyAffinityVersion();
-                AffinityTopologyVersion rmtAffVer = cacheMsg.topologyVersion();
-                AffinityTopologyVersion lastAffChangedVer = cacheMsg.lastAffinityChangedTopologyVersion();
 
                 if (locAffVer.compareTo(lastAffChangedVer) < 0) {
                     IgniteLogger log = cacheMsg.messageLogger(cctx);
@@ -1226,6 +1229,8 @@ public void sendOrderedMessage(ClusterNode node, Object topic, GridCacheMessage
         if (!onSend(msg, node.id()))
             return;
 
+        msg.lastAffinityChangedTopologyVersion(cctx.exchange().lastAffinityChangedTopologyVersion(msg.topologyVersion()));
+
         int cnt = 0;
 
         while (cnt <= retryCnt) {
@@ -1282,6 +1287,8 @@ void sendNoRetry(ClusterNode node,
         if (!onSend(msg, null))
             return;
 
+        msg.lastAffinityChangedTopologyVersion(cctx.exchange().lastAffinityChangedTopologyVersion(msg.topologyVersion()));
+
         try {
             cctx.gridIO().sendToGridTopic(node, TOPIC_CACHE, msg, plc);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
index 3d13cf3b992ae..f6c2e2eb93831 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java
@@ -70,7 +70,7 @@ public abstract class GridCacheMessage implements Message {
 
     /** */
     @GridToStringInclude
-    @Nullable private AffinityTopologyVersion lastAffChangedTopVer;
+    private @Nullable AffinityTopologyVersion lastAffChangedTopVer;
 
     /** */
     @GridDirectTransient
@@ -189,15 +189,22 @@ public AffinityTopologyVersion topologyVersion() {
     }
 
     /**
-     * @return
+     * Returns the earliest affinity topology version for which this message is valid.
+     *
+     * @return Last affinity topology version when affinity was modified.
      */
     public AffinityTopologyVersion lastAffinityChangedTopologyVersion() {
-        if (lastAffChangedTopVer == null)
+        if (lastAffChangedTopVer == null || lastAffChangedTopVer.topologyVersion() <= 0)
             return topologyVersion();
 
         return lastAffChangedTopVer;
     }
 
+    /**
+     * Sets the earliest affinity topology version for which this message is valid.
+     *
+     * @param topVer Last affinity topology version when affinity was modified.
+     */
     public void lastAffinityChangedTopologyVersion(AffinityTopologyVersion topVer) {
         lastAffChangedTopVer = topVer;
     }
@@ -677,7 +684,7 @@ public IgniteLogger messageLogger(GridCacheSharedContext ctx) {
                 writer.incrementState();
 
             case 1:
-                if (!writer.writeMessage("lastAffChangedTopVer", lastAffChangedTopVer))
+                if (!writer.writeAffinityTopologyVersion("lastAffChangedTopVer", lastAffChangedTopVer))
                     return false;
 
                 writer.incrementState();
@@ -710,7 +717,7 @@ public IgniteLogger messageLogger(GridCacheSharedContext ctx) {
                 reader.incrementState();
 
             case 1:
-                lastAffChangedTopVer = reader.readMessage("lastAffChangedTopVer");
+                lastAffChangedTopVer = reader.readAffinityTopologyVersion("lastAffChangedTopVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 342859f3f64fe..cfdda2be83534 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -22,7 +22,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -35,6 +34,7 @@
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentNavigableMap;
 import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.LinkedBlockingDeque;
 import java.util.concurrent.TimeUnit;
@@ -98,6 +98,7 @@
 import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
 import org.apache.ignite.internal.processors.query.schema.SchemaNodeLeaveExchangeWorkerTask;
 import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
+import org.apache.ignite.internal.util.GridListSet;
 import org.apache.ignite.internal.util.GridPartitionStateMap;
 import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
@@ -135,6 +136,7 @@
 import static org.apache.ignite.internal.events.DiscoveryCustomEvent.EVT_DISCOVERY_CUSTOM_EVT;
 import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.CachePartitionPartialCountersMap.PARTIAL_COUNTERS_MAP_SINCE;
+import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture.nextDumpTimeout;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.DFLT_PRELOAD_RESEND_TIMEOUT;
 
@@ -183,6 +185,10 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     private final ConcurrentMap<AffinityTopologyVersion, AffinityReadyFuture> readyFuts =
         new ConcurrentSkipListMap<>();
 
+    /** */
+    private final ConcurrentNavigableMap<AffinityTopologyVersion, AffinityTopologyVersion> lastAffTopVers =
+        new ConcurrentSkipListMap<>();
+
     /** */
     private GridFutureAdapter<?> reconnectExchangeFut;
 
@@ -900,6 +906,17 @@ public void lastFinishedFuture(GridDhtPartitionsExchangeFuture fut) {
      * @return Future or {@code null} is future is already completed.
      */
     @Nullable public IgniteInternalFuture<AffinityTopologyVersion> affinityReadyFuture(AffinityTopologyVersion ver) {
+        GridDhtPartitionsExchangeFuture lastInitializedFut0 = lastInitializedFut;
+
+        if (lastInitializedFut0 != null && lastInitializedFut0.initialVersion().compareTo(ver) == 0
+            && lastInitializedFut0.changedAffinity()) {
+            if (log.isTraceEnabled())
+                log.trace("Return lastInitializedFut for topology ready future " +
+                    "[ver=" + ver + ", fut=" + lastInitializedFut0 + ']');
+
+            return lastInitializedFut0;
+        }
+
         AffinityTopologyVersion topVer = exchFuts.readyTopVer();
 
         if (topVer.compareTo(ver) >= 0) {
@@ -966,19 +983,44 @@ public boolean hasPendingExchange() {
 
     /**
      *
-     * @param topVer
-     * @return
+     * @param topVer Topology version.
+     * @return Last topology version before the provided one when affinity was modified.
      */
     public AffinityTopologyVersion lastAffinityChangedTopologyVersion(AffinityTopologyVersion topVer) {
         if (topVer.topologyVersion() <= 0)
             return topVer;
 
-        GridDhtPartitionsExchangeFuture exchFut = exchFuts.get(topVer);
+        AffinityTopologyVersion lastAffTopVer = lastAffTopVers.get(topVer);
 
-        if (exchFut == null || exchFut.firstEvent() == null || exchFut.changedAffinity())
-            return topVer;
+        return lastAffTopVer != null ? lastAffTopVer : topVer;
+    }
+
+    /**
+     *
+     * @param topVer Topology version.
+     * @param lastAffTopVer Last topology version before the provided one when affinity was modified.
+     * @return {@code True} if data was modified.
+     */
+    public boolean lastAffinityChangedTopologyVersion(AffinityTopologyVersion topVer, AffinityTopologyVersion lastAffTopVer) {
+        assert lastAffTopVer.compareTo(topVer) <= 0;
+
+        if (lastAffTopVer.topologyVersion() <= 0 || lastAffTopVer.equals(topVer))
+            return false;
+
+        while (true) {
+            AffinityTopologyVersion old = lastAffTopVers.putIfAbsent(topVer, lastAffTopVer);
+
+            if (old == null)
+                return true;
+
+            if (lastAffTopVer.compareTo(old) < 0) {
+                if (lastAffTopVers.replace(topVer, old, lastAffTopVer))
+                    return true;
+            }
+            else
+                return false;
+        }
 
-        return exchFut.lastAffinityChangeTopologyVersion();
     }
 
     /**
@@ -1517,7 +1559,7 @@ private GridDhtPartitionsExchangeFuture exchangeFuture(
     ) {
         GridDhtPartitionsExchangeFuture fut;
 
-        GridDhtPartitionsExchangeFuture old = exchFuts.putIfAbsent(exchId.topologyVersion(),
+        GridDhtPartitionsExchangeFuture old = exchFuts.addx(
             fut = new GridDhtPartitionsExchangeFuture(cctx, busyLock, exchId, exchActions, affChangeMsg));
 
         if (old != null) {
@@ -2328,6 +2370,9 @@ private boolean exchangeInProgress() {
 
     /** */
     public boolean affinityChanged(AffinityTopologyVersion from, AffinityTopologyVersion to) {
+        if (lastAffinityChangedTopologyVersion(to).compareTo(from) >= 0)
+            return false;
+
         Collection<GridDhtPartitionsExchangeFuture> history = exchFuts.values();
 
         boolean fromFound = false;
@@ -2701,7 +2746,21 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 crd = newCrd = !srvNodes.isEmpty() && srvNodes.get(0).isLocal();
                             }
 
-                            exchFut.init(lastFinishedFut.get(), newCrd);
+                            if (!exchFut.changedAffinity()) {
+                                GridDhtPartitionsExchangeFuture lastFut = lastFinishedFut.get();
+
+                                if (lastFut != null) {
+                                    if (!lastFut.changedAffinity()) {
+                                        AffinityTopologyVersion lastAffVer = cctx.exchange().lastAffinityChangedTopologyVersion(lastFut.initialVersion());
+
+                                        cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(), lastAffVer);
+                                    }
+                                    else
+                                        cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(), lastFut.initialVersion());
+                                }
+                            }
+
+                            exchFut.init(newCrd);
 
                             int dumpCnt = 0;
 
@@ -2985,7 +3044,7 @@ public boolean started() {
     /**
      *
      */
-    private static class ExchangeFutureSet extends ConcurrentSkipListMap<AffinityTopologyVersion, GridDhtPartitionsExchangeFuture> {
+    private static class ExchangeFutureSet extends GridListSet<GridDhtPartitionsExchangeFuture> {
         /** */
         private static final long serialVersionUID = 0L;
 
@@ -2994,7 +3053,7 @@ private static class ExchangeFutureSet extends ConcurrentSkipListMap<AffinityTop
 
         /** */
         private final AtomicReference<AffinityTopologyVersion> readyTopVer =
-            new AtomicReference<>(AffinityTopologyVersion.NONE);
+            new AtomicReference<>(NONE);
 
         /**
          * Creates ordered, not strict list set.
@@ -3002,15 +3061,16 @@ private static class ExchangeFutureSet extends ConcurrentSkipListMap<AffinityTop
          * @param histSize Max history size.
          */
         private ExchangeFutureSet(int histSize) {
-            super(new Comparator<AffinityTopologyVersion>() {
-                @Override public int compare(
-                    AffinityTopologyVersion t1,
-                    AffinityTopologyVersion t2
-                ) {
-                    // Reverse order.
-                    return t2.compareTo(t1);
-                }
-            });
+            super((f1, f2) -> {
+                AffinityTopologyVersion t1 = f1.exchangeId().topologyVersion();
+                AffinityTopologyVersion t2 = f2.exchangeId().topologyVersion();
+
+                assert t1.topologyVersion() > 0;
+                assert t2.topologyVersion() > 0;
+
+                // Reverse order.
+                return t2.compareTo(t1);
+            }, /*not strict*/false);
 
             this.histSize = histSize;
         }
@@ -3019,20 +3079,17 @@ private ExchangeFutureSet(int histSize) {
          * @param fut Future to add.
          * @return {@code True} if added.
          */
-        @Override public synchronized GridDhtPartitionsExchangeFuture putIfAbsent(
-            AffinityTopologyVersion topVer,
+        @Override public synchronized GridDhtPartitionsExchangeFuture addx(
             GridDhtPartitionsExchangeFuture fut) {
-            assert topVer.topologyVersion() > 0;
-
-            GridDhtPartitionsExchangeFuture cur = super.putIfAbsent(topVer, fut);
+            GridDhtPartitionsExchangeFuture cur = super.addx(fut);
 
             while (size() > histSize) {
-                GridDhtPartitionsExchangeFuture last = lastEntry().getValue();
+                GridDhtPartitionsExchangeFuture last = last();
 
                 if (!last.isDone() || Objects.equals(last.initialVersion(), readyTopVer()))
                     break;
 
-                remove(last.initialVersion());
+                removeLast();
             }
 
             // Return the value in the set.
@@ -3063,16 +3120,17 @@ public boolean readyTopVer(AffinityTopologyVersion readyTopVersion) {
         }
 
         /** {@inheritDoc} */
-        @Nullable @Override public synchronized GridDhtPartitionsExchangeFuture remove(
-            Object topVer) {
-            return super.remove(topVer);
+        @Nullable @Override public synchronized GridDhtPartitionsExchangeFuture removex(
+            GridDhtPartitionsExchangeFuture val) {
+
+            return super.removex(val);
         }
 
         /**
          * @return Values.
          */
         @Override public synchronized List<GridDhtPartitionsExchangeFuture> values() {
-            return new ArrayList<>(super.values());
+            return super.values();
         }
 
         /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/KeyCacheObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/KeyCacheObject.java
index 21b1f89528a7c..c747b082f13be 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/KeyCacheObject.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/KeyCacheObject.java
@@ -48,4 +48,4 @@ public interface KeyCacheObject extends CacheObject {
      * @return Copy of this object with given partition set.
      */
     public KeyCacheObject copy(int part);
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java
index 7c241068231bb..f8012a7de733e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/WalStateAckMessage.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.cache;
 
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.processors.query.schema.message.SchemaOperationStatusMessage;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.plugin.extensions.communication.Message;
@@ -31,6 +32,7 @@
 /**
  * WAL state ack message (sent from participant node to coordinator).
  */
+@IgniteCodeGeneratingFail
 public class WalStateAckMessage implements Message {
     /** */
     private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java
index a1c787a932a0e..c420aeb4f0491 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridCacheTtlUpdateRequest.java
@@ -232,7 +232,7 @@ public List<GridCacheVersion > nearVersions() {
                 writer.incrementState();
 
             case 7:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -290,7 +290,7 @@ public List<GridCacheVersion > nearVersions() {
                 reader.incrementState();
 
             case 7:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java
index 38855a0a9e827..a1af470c56deb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxFinishRequest.java
@@ -398,7 +398,7 @@ public boolean replyRequired() {
                 writer.incrementState();
 
             case 20:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -526,7 +526,7 @@ public boolean replyRequired() {
                 reader.incrementState();
 
             case 20:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java
index 56d34acdb6ed6..d525e3a70a68b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/ClientCacheDhtTopologyFuture.java
@@ -86,11 +86,6 @@ public void validate(CacheGroupContext grp, Collection<ClusterNode> topNodes) {
         return true;
     }
 
-    /** {@inheritDoc} */
-    @Override public AffinityTopologyVersion lastAffinityChangeTopologyVersion() {
-        return topVer;
-    }
-
     /** {@inheritDoc} */
     @Override public String toString() {
         return "ClientCacheDhtTopologyFuture [topVer=" + topVer + ']';
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java
index 4ae873920d231..cf7018a15c895 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentRequest.java
@@ -140,7 +140,7 @@ public long futureId() {
                 writer.incrementState();
 
             case 6:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -178,7 +178,7 @@ public long futureId() {
                 reader.incrementState();
 
             case 6:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java
index 63e8f74b033f2..e8b40e9cd8c15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtAffinityAssignmentResponse.java
@@ -297,7 +297,7 @@ private List<List<UUID>> ids(List<List<ClusterNode>> assignments) {
                 writer.incrementState();
 
             case 8:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -351,7 +351,7 @@ private List<List<UUID>> ids(List<List<ClusterNode>> assignments) {
                 reader.incrementState();
 
             case 8:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
index 9c00ecb25a6f1..17e850c73ad34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java
@@ -1230,7 +1230,12 @@ private void updateTtl(GridCacheAdapter<K, V> cache,
      * @return {@code True} if cache affinity changed and operation should be remapped.
      */
     protected final boolean needRemap(AffinityTopologyVersion expVer, AffinityTopologyVersion curVer,
-        AffinityTopologyVersion lastAffChangedTopVer, Collection<KeyCacheObject> keys) {
+        Collection<KeyCacheObject> keys) {
+        if (curVer.equals(expVer))
+            return false;
+
+        AffinityTopologyVersion lastAffChangedTopVer = ctx.shared().exchange().lastAffinityChangedTopologyVersion(expVer);
+
         if (curVer.compareTo(lastAffChangedTopVer) >= 0 && curVer.compareTo(expVer) <= 0)
             return false;
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
index 4a219f61fd470..76c3b163d577a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLockRequest.java
@@ -418,7 +418,7 @@ public long accessTtl() {
                 writer.incrementState();
 
             case 30:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -512,7 +512,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 30:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
index 14dfebbfcd5e6..09abf168d0d9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTopologyFuture.java
@@ -92,10 +92,4 @@ public interface GridDhtTopologyFuture extends IgniteInternalFuture<AffinityTopo
      * @return {@code True} if this exchange changed affinity.
      */
     public boolean changedAffinity();
-
-    /**
-     *
-     * @return Topology version of last affinity change.
-     */
-    public AffinityTopologyVersion lastAffinityChangeTopologyVersion();
 }
\ No newline at end of file
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index 64d6acb8396dc..96b45339b0148 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -939,7 +939,7 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                 }
 
             try {
-                if (top != null && needRemap(req.topologyVersion(), top.readyTopologyVersion(), req.lastAffinityChangedTopologyVersion(), req.keys())) {
+                if (top != null && needRemap(req.topologyVersion(), top.readyTopologyVersion(), req.keys())) {
                     if (log.isDebugEnabled()) {
                         log.debug("Client topology version mismatch, need remap lock request [" +
                             "reqTopVer=" + req.topologyVersion() +
@@ -1044,7 +1044,7 @@ public IgniteInternalFuture<GridNearLockResponse> lockAllAsync(
                     }
 
                     try {
-                        if (top != null && needRemap(req.topologyVersion(), top.readyTopologyVersion(), req.lastAffinityChangedTopologyVersion(), req.keys())) {
+                        if (top != null && needRemap(req.topologyVersion(), top.readyTopologyVersion(), req.keys())) {
                             if (log.isDebugEnabled()) {
                                 log.debug("Client topology version mismatch, need remap lock request [" +
                                     "reqTopVer=" + req.topologyVersion() +
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
index b48aa4432aa23..19a391697ba85 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
@@ -468,7 +468,7 @@ public boolean skipCompletedVersion() {
                 writer.incrementState();
 
             case 33:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java
index 489fe632b5e62..3e2f971e11aae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicAbstractUpdateRequest.java
@@ -533,7 +533,7 @@ final boolean isFlag(int mask) {
                 writer.incrementState();
 
             case 11:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -621,7 +621,7 @@ final boolean isFlag(int mask) {
                 reader.incrementState();
 
             case 11:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 999785423de9f..e5514b55c98d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -1738,8 +1738,7 @@ private void updateAllAsyncInternal0(
                                 // Can not wait for topology future since it will break
                                 // GridNearAtomicCheckUpdateRequest processing.
                                 remap = !top.topologyVersionFuture().exchangeDone() ||
-                                    needRemap(req.topologyVersion(), top.readyTopologyVersion(),
-                                        req.lastAffinityChangedTopologyVersion(), req.keys());
+                                    needRemap(req.topologyVersion(), top.readyTopologyVersion(), req.keys());
                             }
 
                             if (!remap) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java
index 26be658df8f15..64fe1eef8fe6c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicAbstractUpdateRequest.java
@@ -583,7 +583,7 @@ abstract void addUpdateEntry(KeyCacheObject key,
                 writer.incrementState();
 
             case 10:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -661,7 +661,7 @@ abstract void addUpdateEntry(KeyCacheObject key,
                 reader.incrementState();
 
             case 10:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
index 56eb8d7426500..6dccd8b9cdc9b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridNearAtomicUpdateResponse.java
@@ -436,7 +436,7 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
                 writer.incrementState();
 
             case 9:
-                if (!writer.writeMessage("remapTopVer", remapTopVer))
+                if (!writer.writeAffinityTopologyVersion("remapTopVer", remapTopVer))
                     return false;
 
                 writer.incrementState();
@@ -504,7 +504,7 @@ synchronized void addFailedKeys(Collection<KeyCacheObject> keys, Throwable e) {
                 reader.incrementState();
 
             case 9:
-                remapTopVer = reader.readMessage("remapTopVer");
+                remapTopVer = reader.readAffinityTopologyVersion("remapTopVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java
index ccd36a9641eab..80c45efc76805 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtForceKeysRequest.java
@@ -186,7 +186,7 @@ private int keyCount() {
                 writer.incrementState();
 
             case 7:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -232,7 +232,7 @@ private int keyCount() {
                 reader.incrementState();
 
             case 7:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java
index d75bd76739335..cd7741b554fa7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandLegacyMessage.java
@@ -310,7 +310,7 @@ Long partitionCounter(int part) {
                 writer.incrementState();
 
             case 8:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -382,7 +382,7 @@ Long partitionCounter(int part) {
                 reader.incrementState();
 
             case 8:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
index 5a8a71117c298..bae326424d0fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemandMessage.java
@@ -21,6 +21,7 @@
 import java.nio.ByteBuffer;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
 import org.apache.ignite.internal.processors.cache.GridCacheMessage;
@@ -35,6 +36,7 @@
 /**
  * Partition demand request.
  */
+@IgniteCodeGeneratingFail
 public class GridDhtPartitionDemandMessage extends GridCacheGroupIdMessage {
     /** */
     private static final long serialVersionUID = 0L;
@@ -266,25 +268,25 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) {
                 writer.incrementState();
 
             case 5:
-                if (!writer.writeLong("rebalanceId", rebalanceId))
+                if (!writer.writeLong("timeout", timeout))
                     return false;
 
                 writer.incrementState();
 
             case 6:
-                if (!writer.writeLong("timeout", timeout))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
 
             case 7:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeByteArray("topicBytes", topicBytes))
                     return false;
 
                 writer.incrementState();
 
             case 8:
-                if (!writer.writeByteArray("topicBytes", topicBytes))
+                if (!writer.writeLong("rebalanceId", rebalanceId))
                     return false;
 
                 writer.incrementState();
@@ -320,7 +322,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) {
                 reader.incrementState();
 
             case 5:
-                rebalanceId = reader.readLong("rebalanceId");
+                timeout = reader.readLong("timeout");
 
                 if (!reader.isLastRead())
                     return false;
@@ -328,7 +330,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) {
                 reader.incrementState();
 
             case 6:
-                timeout = reader.readLong("timeout");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
@@ -336,7 +338,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) {
                 reader.incrementState();
 
             case 7:
-                topVer = reader.readMessage("topVer");
+                topicBytes = reader.readByteArray("topicBytes");
 
                 if (!reader.isLastRead())
                     return false;
@@ -344,7 +346,7 @@ public GridCacheMessage convertIfNeeded(IgniteProductVersion target) {
                 reader.incrementState();
 
             case 8:
-                topicBytes = reader.readByteArray("topicBytes");
+                rebalanceId = reader.readLong("rebalanceId");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
index 741386b40707b..2206f2e372fad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionExchangeId.java
@@ -248,7 +248,7 @@ public boolean isLeft() {
                 writer.incrementState();
 
             case 2:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -283,7 +283,7 @@ public boolean isLeft() {
                 reader.incrementState();
 
             case 2:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
@@ -311,4 +311,4 @@ public boolean isLeft() {
             "nodeId", U.id8(nodeId),
             "evt", U.gridEventName(evt));
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
index ca8a8b5fed9dd..3034fb9daaf21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
@@ -28,6 +28,7 @@
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.GridDirectCollection;
 import org.apache.ignite.internal.GridDirectMap;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheEntryInfoCollection;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
@@ -45,6 +46,7 @@
 /**
  * Partition supply message.
  */
+@IgniteCodeGeneratingFail
 public class GridDhtPartitionSupplyMessage extends GridCacheGroupIdMessage implements GridCacheDeployable {
     /** */
     private static final long serialVersionUID = 0L;
@@ -325,13 +327,14 @@ public int size() {
                 writer.incrementState();
 
             case 11:
-                if (!writer.writeLong("rebalanceId", rebalanceId))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
 
             case 12:
-                if (!writer.writeMessage("topVer", topVer))
+                // Keep 'updateSeq' name for compatibility.
+                if (!writer.writeLong("updateSeq", rebalanceId))
                     return false;
 
                 writer.incrementState();
@@ -409,7 +412,7 @@ public int size() {
                 reader.incrementState();
 
             case 11:
-                rebalanceId = reader.readLong("rebalanceId");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
@@ -417,7 +420,8 @@ public int size() {
                 reader.incrementState();
 
             case 12:
-                topVer = reader.readMessage("topVer");
+                // Keep 'updateSeq' name for compatibility.
+                rebalanceId = reader.readLong("updateSeq");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 100760ee59e86..8f07b04db0b7b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -326,9 +326,6 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
     /** Future for wait all exchange listeners comepleted. */
     private final GridFutureAdapter<?> afterLsnrCompleteFut = new GridFutureAdapter<>();
 
-    /** */
-    private volatile AffinityTopologyVersion lastAffChangeTopVer;
-
     /**
      * @param cctx Cache context.
      * @param busyLock Busy lock.
@@ -569,16 +566,6 @@ public boolean changedBaseline() {
             || !firstDiscoEvt0.eventNode().isClient() || firstDiscoEvt0.eventNode().isLocal();
     }
 
-    /** {@inheritDoc} */
-    @Override public AffinityTopologyVersion lastAffinityChangeTopologyVersion() {
-        if (changedAffinity())
-            return topologyVersion();
-        else if (!exchangeDone()) // TODO: initialVersion and topologyVersion are always equal now, but it should be fixed later.
-            return initialVersion();
-        else
-            return topologyVersion();
-    }
-
     /**
      * @return {@code True} if there are caches to start.
      */
@@ -656,7 +643,7 @@ private void initCoordinatorCaches(boolean newCrd) throws IgniteCheckedException
      * @param newCrd {@code True} if node become coordinator on this exchange.
      * @throws IgniteInterruptedCheckedException If interrupted.
      */
-    public void init(@Nullable GridDhtPartitionsExchangeFuture lastFut, boolean newCrd) throws IgniteInterruptedCheckedException {
+    public void init(boolean newCrd) throws IgniteInterruptedCheckedException {
         if (isDone())
             return;
 
@@ -670,9 +657,6 @@ public void init(@Nullable GridDhtPartitionsExchangeFuture lastFut, boolean newC
         assert exchId.nodeId().equals(firstDiscoEvt.eventNode().id()) : this;
 
         try {
-            if (!changedAffinity() && lastFut != null)
-                lastAffChangeTopVer = lastFut.lastAffinityChangeTopologyVersion();
-
             AffinityTopologyVersion topVer = initialVersion();
 
             srvNodes = new ArrayList<>(firstEvtDiscoCache.serverNodes());
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index 536fdfd04756e..fbaa24193b50f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -694,13 +694,13 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
                 writer.incrementState();
 
             case 16:
-                if (!writer.writeMessage("resTopVer", resTopVer))
+                if (!writer.writeAffinityTopologyVersion("resTopVer", resTopVer))
                     return false;
 
                 writer.incrementState();
 
             case 17:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -802,7 +802,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
                 reader.incrementState();
 
             case 16:
-                resTopVer = reader.readMessage("resTopVer");
+                resTopVer = reader.readAffinityTopologyVersion("resTopVer");
 
                 if (!reader.isLastRead())
                     return false;
@@ -810,7 +810,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
                 reader.incrementState();
 
             case 17:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
index 3ce3b9ffd9c23..cb6fbb1afb863 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsSingleMessage.java
@@ -595,6 +595,7 @@ public void setError(Exception ex) {
                     return false;
 
                 reader.incrementState();
+
         }
 
         return reader.afterMessageRead(GridDhtPartitionsSingleMessage.class);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java
index bad1b6137bac5..9c69fdf581856 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/latch/LatchAckMessage.java
@@ -103,10 +103,11 @@ public boolean isFinal() {
                 writer.incrementState();
 
             case 2:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
+
         }
 
         return true;
@@ -137,12 +138,13 @@ public boolean isFinal() {
                 reader.incrementState();
 
             case 2:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
 
                 reader.incrementState();
+
         }
 
         return reader.afterMessageRead(LatchAckMessage.class);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
index ba5059ed4098a..bc77130f579e6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtPartitionTopologyImpl.java
@@ -305,8 +305,12 @@ private String mapString(GridDhtPartitionMap map) {
 
         assert topReadyFut0 != null;
 
-        if (topReadyFut0 instanceof GridDhtPartitionsExchangeFuture && !((GridDhtPartitionsExchangeFuture)topReadyFut0).changedAffinity())
-            return ctx.exchange().lastFinishedFuture();
+        if (!topReadyFut0.changedAffinity()) {
+            GridDhtTopologyFuture lastFut = ctx.exchange().lastFinishedFuture();
+
+            if (lastFut != null)
+                return lastFut;
+        }
 
         return topReadyFut0;
     }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/CacheVersionedValue.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/CacheVersionedValue.java
index 9670f8a4ea5cf..c19d486a489b4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/CacheVersionedValue.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/CacheVersionedValue.java
@@ -174,4 +174,4 @@ public void finishUnmarshal(GridCacheContext ctx, ClassLoader ldr) throws Ignite
     @Override public String toString() {
         return S.toString(CacheVersionedValue.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java
index 8e724b2af2ccc..578c46b6ac34a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearGetResponse.java
@@ -259,7 +259,7 @@ public void error(IgniteCheckedException err) {
                 writer.incrementState();
 
             case 9:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -327,7 +327,7 @@ public void error(IgniteCheckedException err) {
                 reader.incrementState();
 
             case 9:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java
index 1d18d943aceed..0712e2234f6cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockRequest.java
@@ -412,7 +412,7 @@ public long accessTtl() {
                 writer.incrementState();
 
             case 29:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -498,7 +498,7 @@ public long accessTtl() {
                 reader.incrementState();
 
             case 29:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java
index 34386205fa29e..b6c6d8c903c49 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockResponse.java
@@ -209,7 +209,7 @@ public void addValueBytes(
 
         switch (writer.state()) {
             case 11:
-                if (!writer.writeMessage("clientRemapVer", clientRemapVer))
+                if (!writer.writeAffinityTopologyVersion("clientRemapVer", clientRemapVer))
                     return false;
 
                 writer.incrementState();
@@ -261,7 +261,7 @@ public void addValueBytes(
 
         switch (reader.state()) {
             case 11:
-                clientRemapVer = reader.readMessage("clientRemapVer");
+                clientRemapVer = reader.readAffinityTopologyVersion("clientRemapVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
index 16b514968c8b1..3b88c99f37003 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetRequest.java
@@ -338,7 +338,7 @@ public boolean recovery() {
                 reader.incrementState();
 
             case 11:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
@@ -408,7 +408,7 @@ public boolean recovery() {
                 writer.incrementState();
 
             case 11:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java
index c71a1d5bdcb4f..584cec2eb1cb1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearSingleGetResponse.java
@@ -231,7 +231,7 @@ else if (res instanceof GridCacheEntryInfo)
                 writer.incrementState();
 
             case 8:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -285,7 +285,7 @@ else if (res instanceof GridCacheEntryInfo)
                 reader.incrementState();
 
             case 8:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
index 40c46ff16f49a..7b2d26f64df62 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareRequest.java
@@ -332,7 +332,7 @@ private boolean isFlag(int mask) {
                 writer.incrementState();
 
             case 26:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -394,7 +394,7 @@ private boolean isFlag(int mask) {
                 reader.incrementState();
 
             case 26:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
index 8b256d19ee9fd..5decd8613e37c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxPrepareResponse.java
@@ -359,7 +359,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
         switch (writer.state()) {
             case 11:
-                if (!writer.writeMessage("clientRemapVer", clientRemapVer))
+                if (!writer.writeAffinityTopologyVersion("clientRemapVer", clientRemapVer))
                     return false;
 
                 writer.incrementState();
@@ -435,7 +435,7 @@ public boolean hasOwnedValue(IgniteTxKey key) {
 
         switch (reader.state()) {
             case 11:
-                clientRemapVer = reader.readMessage("clientRemapVer");
+                clientRemapVer = reader.readAffinityTopologyVersion("clientRemapVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
index fae97668746fe..e9da424a8a36c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java
@@ -603,7 +603,7 @@ public int taskHash() {
                 writer.incrementState();
 
             case 21:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -773,7 +773,7 @@ public int taskHash() {
                 reader.incrementState();
 
             case 21:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
index 88005d047933f..b05c275451ffe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryEntry.java
@@ -411,7 +411,7 @@ CacheObject oldValue() {
                 writer.incrementState();
 
             case 8:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -500,7 +500,7 @@ CacheObject oldValue() {
                 reader.incrementState();
 
             case 8:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index 9075ccfce417f..bea9181df3633 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -655,7 +655,12 @@ private void sendResponseOnTimeoutOrError(@Nullable IgniteCheckedException e,
     private boolean needRemap(AffinityTopologyVersion expVer,
         AffinityTopologyVersion curVer,
         GridNearTxPrepareRequest req) {
-        if (curVer.compareTo(expVer) <= 0 && curVer.compareTo(req.lastAffinityChangedTopologyVersion()) >= 0)
+        if (curVer.equals(expVer))
+            return false;
+
+        AffinityTopologyVersion lastAffChangedTopVer = ctx.exchange().lastAffinityChangedTopologyVersion(expVer);
+
+        if (curVer.compareTo(expVer) <= 0 && curVer.compareTo(lastAffChangedTopVer) >= 0)
             return false;
 
         for (IgniteTxEntry e : F.concat(false, req.reads(), req.writes())) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxKey.java
index efcb48b4714c3..2c3892f564dde 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxKey.java
@@ -191,4 +191,4 @@ public void finishUnmarshal(GridCacheContext ctx, ClassLoader ldr) throws Ignite
     @Override public String toString() {
         return S.toString(IgniteTxKey.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheRawVersionedEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheRawVersionedEntry.java
index 586a043cc8d22..ceea0a2d9a5cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheRawVersionedEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/version/GridCacheRawVersionedEntry.java
@@ -24,6 +24,7 @@
 import java.nio.ByteBuffer;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.processors.cache.CacheObject;
 import org.apache.ignite.internal.processors.cache.CacheObjectContext;
 import org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
@@ -39,6 +40,7 @@
 /**
  * Raw versioned entry.
  */
+@IgniteCodeGeneratingFail
 public class GridCacheRawVersionedEntry<K, V> extends DataStreamerEntry implements
     GridCacheVersionedEntry<K, V>, GridCacheVersionable, Externalizable {
     /** */
@@ -381,4 +383,4 @@ public void prepareDirectMarshal(CacheObjectContext ctx) throws IgniteCheckedExc
             "valBytesLen", valBytes != null ? valBytes.length : "n/a",
             "super", super.toString());
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java
index ec04ac355e5a6..5e7cc9d3a4576 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/continuous/GridContinuousMessage.java
@@ -274,4 +274,4 @@ public void dataBytes(byte[] dataBytes) {
     @Override public String toString() {
         return S.toString(GridContinuousMessage.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java
index f70ee9c486fa0..87313de15ebd2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerRequest.java
@@ -369,7 +369,7 @@ public int partition() {
                 writer.incrementState();
 
             case 13:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -508,7 +508,7 @@ public int partition() {
                 reader.incrementState();
 
             case 13:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerResponse.java
index 4cb46e1582022..56f37c2cf0b98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastreamer/DataStreamerResponse.java
@@ -169,4 +169,4 @@ public boolean forceLocalDeployment() {
     @Override public byte fieldsCount() {
         return 3;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
index f6bddb64fa366..e990cb3cd7303 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopDirectShuffleMessage.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.hadoop.shuffle;
 
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
 import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -35,6 +36,7 @@
 /**
  * Direct shuffle message.
  */
+@IgniteCodeGeneratingFail
 public class HadoopDirectShuffleMessage implements Message, HadoopMessage {
     /** */
     private static final long serialVersionUID = 0L;
@@ -268,4 +270,4 @@ public int dataLength() {
     @Override public String toString() {
         return S.toString(HadoopDirectShuffleMessage.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
index 07b8c2fd1635a..2660e281f20de 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/shuffle/HadoopShuffleMessage.java
@@ -23,6 +23,7 @@
 import java.nio.ByteBuffer;
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.processors.hadoop.HadoopJobId;
 import org.apache.ignite.internal.processors.hadoop.message.HadoopMessage;
 import org.apache.ignite.internal.util.GridUnsafe;
@@ -36,6 +37,7 @@
 /**
  * Shuffle message.
  */
+@IgniteCodeGeneratingFail
 public class HadoopShuffleMessage implements Message, HadoopMessage {
     /** */
     private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAckMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAckMessage.java
index 5ae3fed3e47e0..76a08f9877aab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAckMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAckMessage.java
@@ -195,4 +195,4 @@ public IgniteCheckedException error() {
     @Override public byte fieldsCount() {
         return 3;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockKey.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockKey.java
index 736525de1489a..98bf306a479df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockKey.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlockKey.java
@@ -297,4 +297,4 @@ public boolean evictExclude() {
     @Override public String toString() {
         return S.toString(IgfsBlockKey.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlocksMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlocksMessage.java
index 2ec54b2ed0663..5448c50b9352d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlocksMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsBlocksMessage.java
@@ -176,4 +176,4 @@ public Map<IgfsBlockKey, byte[]> blocks() {
     @Override public byte fieldsCount() {
         return 3;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCommunicationMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCommunicationMessage.java
index 412c45baf6daf..7f5daa617bdb7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCommunicationMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsCommunicationMessage.java
@@ -77,4 +77,4 @@ public void finishUnmarshal(Marshaller marsh, @Nullable ClassLoader ldr) throws
     @Override public byte fieldsCount() {
         return 0;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDeleteMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDeleteMessage.java
index b5e96664b4634..2c2c98c82f18b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDeleteMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDeleteMessage.java
@@ -190,4 +190,4 @@ public IgniteCheckedException error() {
     @Override public String toString() {
         return S.toString(IgfsDeleteMessage.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerRequest.java
index bb605b857641c..c64b627dac641 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerRequest.java
@@ -157,4 +157,4 @@ public Collection<IgfsFileAffinityRange> fragmentRanges() {
     @Override public byte fieldsCount() {
         return 2;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerResponse.java
index 76793bfeb6d0e..f7857dcfad154 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsFragmentizerResponse.java
@@ -118,4 +118,4 @@ public IgniteUuid fileId() {
     @Override public byte fieldsCount() {
         return 1;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSyncMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSyncMessage.java
index 2b32084b7f3e6..2707de543002e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSyncMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSyncMessage.java
@@ -149,4 +149,4 @@ public boolean response() {
     @Override public byte fieldsCount() {
         return 2;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
index ea83169d37212..d838822089a19 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/messages/GridQueryNextPageResponse.java
@@ -217,7 +217,7 @@ public Collection<?> plainRows() {
                 writer.incrementState();
 
             case 6:
-                if (!writer.writeMessage("retry", retry))
+                if (!writer.writeAffinityTopologyVersion("retry", retry))
                     return false;
 
                 writer.incrementState();
@@ -301,7 +301,7 @@ public Collection<?> plainRows() {
                 reader.incrementState();
 
             case 6:
-                retry = reader.readMessage("retry");
+                retry = reader.readAffinityTopologyVersion("retry");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaOperationStatusMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaOperationStatusMessage.java
index 5f75e607d4173..9e81d5a84b432 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaOperationStatusMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/message/SchemaOperationStatusMessage.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.query.schema.message;
 
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
@@ -30,6 +31,7 @@
 /**
  * Schema operation status message.
  */
+@IgniteCodeGeneratingFail
 public class SchemaOperationStatusMessage implements Message {
     /** */
     private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultRequest.java
index b8b4edbf01bb7..f3ec0e8afacaf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultRequest.java
@@ -171,4 +171,4 @@ public void topic(String topic) {
     @Override public byte fieldsCount() {
         return 2;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultResponse.java
index b9bb27c067182..88b42f5e80a5c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/task/GridTaskResultResponse.java
@@ -217,4 +217,4 @@ public void error(String err) {
     @Override public byte fieldsCount() {
         return 4;
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridByteArrayList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridByteArrayList.java
index 0200d7747002e..e9c7d016bdb4c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridByteArrayList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridByteArrayList.java
@@ -491,4 +491,4 @@ public InputStream inputStream() {
     @Override public String toString() {
         return S.toString(GridByteArrayList.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java
index ebe2e0765c787..c54584a78c4f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridIntList.java
@@ -600,4 +600,4 @@ public GridIntIterator iterator() {
             }
         };
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java
index ded0a5d3f6181..e04b540b4627f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/GridLongList.java
@@ -26,6 +26,7 @@
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.NoSuchElementException;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.SB;
@@ -38,6 +39,7 @@
  * Minimal list API to work with primitive longs. This list exists
  * to avoid boxing/unboxing when using standard list from Java.
  */
+@IgniteCodeGeneratingFail
 public class GridLongList implements Message, Externalizable {
     /** */
     private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageCollectionItemType.java b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageCollectionItemType.java
index 2134912e5a500..2340dd9d6afe6 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageCollectionItemType.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageCollectionItemType.java
@@ -84,7 +84,10 @@ public enum MessageCollectionItemType {
     IGNITE_UUID,
 
     /** Message. */
-    MSG;
+    MSG,
+
+    /** Topology version. */
+    AFFINITY_TOPOLOGY_VERSION;
 
     /** Enum values. */
     private static final MessageCollectionItemType[] VALS = values();
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageReader.java b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageReader.java
index 050204288927d..6feee1a21eab4 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageReader.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageReader.java
@@ -23,6 +23,7 @@
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.lang.IgniteUuid;
 
 /**
@@ -229,6 +230,14 @@ public interface MessageReader {
      */
     public IgniteUuid readIgniteUuid(String name);
 
+    /**
+     * Reads {@link AffinityTopologyVersion}.
+     *
+     * @param name Field name.
+     * @return {@link AffinityTopologyVersion}.
+     */
+    public AffinityTopologyVersion readAffinityTopologyVersion(String name);
+
     /**
      * Reads nested message.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageWriter.java b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageWriter.java
index 692955f39cf15..14d4417be3ee2 100644
--- a/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageWriter.java
+++ b/modules/core/src/main/java/org/apache/ignite/plugin/extensions/communication/MessageWriter.java
@@ -22,6 +22,7 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.UUID;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.lang.IgniteUuid;
 
 /**
@@ -254,6 +255,15 @@ public interface MessageWriter {
      */
     public boolean writeIgniteUuid(String name, IgniteUuid val);
 
+    /**
+     * Writes {@link AffinityTopologyVersion}.
+     *
+     * @param name Field name.
+     * @param val {@link AffinityTopologyVersion}.
+     * @return Whether value was fully written.
+     */
+    public boolean writeAffinityTopologyVersion(String name, AffinityTopologyVersion val);
+
     /**
      * Writes nested message.
      *
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingRequest.java b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingRequest.java
index 6ecb14572ee2b..5f028047981e8 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/collision/jobstealing/JobStealingRequest.java
@@ -118,4 +118,4 @@ int delta() {
     @Override public String toString() {
         return S.toString(JobStealingRequest.class, this);
     }
-}
\ No newline at end of file
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java
index e3be9c99c7ed0..f845b0b5bbc19 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage.java
@@ -19,6 +19,7 @@
 
 import java.nio.ByteBuffer;
 import java.util.UUID;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.plugin.extensions.communication.Message;
@@ -29,6 +30,7 @@
 /**
  * Handshake message.
  */
+@IgniteCodeGeneratingFail
 public class HandshakeMessage implements Message {
     /** */
     private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java
index f27a825740cae..220781371a69f 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/HandshakeMessage2.java
@@ -19,6 +19,7 @@
 
 import java.nio.ByteBuffer;
 import java.util.UUID;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
 import org.apache.ignite.plugin.extensions.communication.MessageWriter;
@@ -26,6 +27,7 @@
 /**
  * Updated handshake message.
  */
+@IgniteCodeGeneratingFail
 public class HandshakeMessage2 extends HandshakeMessage {
     /** */
     private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java
index 2c6aa300bd8be..12e2522ce4f05 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/NodeIdMessage.java
@@ -19,6 +19,7 @@
 
 import java.nio.ByteBuffer;
 import java.util.UUID;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.plugin.extensions.communication.Message;
@@ -29,6 +30,7 @@
 /**
  * Node ID message.
  */
+@IgniteCodeGeneratingFail
 public class NodeIdMessage implements Message {
     /** */
     private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java
index eef2655a3eafa..95d43f6b868c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/communication/tcp/messages/RecoveryLastReceivedMessage.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.spi.communication.tcp.messages;
 
 import java.nio.ByteBuffer;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
@@ -27,6 +28,7 @@
 /**
  * Recovery acknowledgment message.
  */
+@IgniteCodeGeneratingFail
 public class RecoveryLastReceivedMessage implements Message {
     /** */
     private static final long serialVersionUID = 0L;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java
new file mode 100644
index 0000000000000..45c6f25bf275a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.locks.Lock;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeFailedMessage;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeLeftMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class CacheNoAffinityExchangeTest extends GridCommonAbstractTest {
+    /** */
+    private volatile boolean startClient;
+
+    /** */
+    private final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder().setShared(true);
+
+    /** */
+    private final TcpDiscoveryIpFinder CLIENT_IP_FINDER = new TcpDiscoveryVmIpFinder()
+        .setAddresses(Collections.singleton("127.0.0.1:47500"));
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setDiscoverySpi(new TestDiscoverySpi().setIpFinder(IP_FINDER));
+
+        if (startClient) {
+            cfg.setClientMode(true);
+
+            // It is necessary to ensure that client always connects to grid(0).
+            ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(CLIENT_IP_FINDER);
+        }
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        super.afterTest();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoAffinityChangeOnClientJoin() throws Exception {
+        Ignite ig = startGrids(4);
+
+        ig.cluster().active(true);
+
+        IgniteCache<Integer, Integer> atomicCache = ig.createCache(new CacheConfiguration<Integer, Integer>()
+                .setName("atomic").setAtomicityMode(CacheAtomicityMode.ATOMIC));
+
+        IgniteCache<Integer, Integer> txCache = ig.createCache(new CacheConfiguration<Integer, Integer>()
+                .setName("tx").setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL));
+
+        assertTrue(GridTestUtils.waitForCondition(() ->
+                new AffinityTopologyVersion(4, 3).equals(grid(3).context().discovery().topologyVersionEx()),
+            5_000));
+
+        TestDiscoverySpi discoSpi = (TestDiscoverySpi) grid(2).context().discovery().getInjectedDiscoverySpi();
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        discoSpi.latch = latch;
+
+        startClient = true;
+
+        startGrid(4);
+
+        assertTrue(GridTestUtils.waitForCondition(() ->
+                new AffinityTopologyVersion(5, 0).equals(grid(0).context().discovery().topologyVersionEx()) &&
+                    new AffinityTopologyVersion(5, 0).equals(grid(1).context().discovery().topologyVersionEx()) &&
+                    new AffinityTopologyVersion(4, 3).equals(grid(2).context().discovery().topologyVersionEx()) &&
+                    new AffinityTopologyVersion(4, 3).equals(grid(3).context().discovery().topologyVersionEx()),
+            10_000));
+
+        for (int k = 0; k < 100; k++) {
+            atomicCache.put(k, k);
+            txCache.put(k, k);
+
+            Lock lock = txCache.lock(k);
+            lock.lock();
+            lock.unlock();
+        }
+
+        for (int k = 0; k < 100; k++) {
+            assertEquals(Integer.valueOf(k), atomicCache.get(k));
+            assertEquals(Integer.valueOf(k), txCache.get(k));
+        }
+
+        assertEquals(new AffinityTopologyVersion(5, 0), grid(0).context().discovery().topologyVersionEx());
+        assertEquals(new AffinityTopologyVersion(5, 0), grid(1).context().discovery().topologyVersionEx());
+        assertEquals(new AffinityTopologyVersion(4, 3), grid(2).context().discovery().topologyVersionEx());
+        assertEquals(new AffinityTopologyVersion(4, 3), grid(3).context().discovery().topologyVersionEx());
+
+        latch.countDown();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoAffinityChangeOnClientLeft() throws Exception {
+        Ignite ig = startGrids(4);
+
+        ig.cluster().active(true);
+
+        IgniteCache<Integer, Integer> atomicCache = ig.createCache(new CacheConfiguration<Integer, Integer>()
+            .setName("atomic").setAtomicityMode(CacheAtomicityMode.ATOMIC));
+
+        IgniteCache<Integer, Integer> txCache = ig.createCache(new CacheConfiguration<Integer, Integer>()
+            .setName("tx").setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL));
+
+        assertTrue(GridTestUtils.waitForCondition(() ->
+            new AffinityTopologyVersion(4, 3).equals(grid(3).context().discovery().topologyVersionEx()),
+            5_000));
+
+        startClient = true;
+
+        startGrid(4);
+
+        TestDiscoverySpi discoSpi = (TestDiscoverySpi)grid(2).context().discovery().getInjectedDiscoverySpi();
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        discoSpi.latch = latch;
+
+        stopGrid(4);
+
+        assertTrue(GridTestUtils.waitForCondition(() ->
+                new AffinityTopologyVersion(6, 0).equals(grid(0).context().discovery().topologyVersionEx()) &&
+                    new AffinityTopologyVersion(6, 0).equals(grid(1).context().discovery().topologyVersionEx()) &&
+                    new AffinityTopologyVersion(5, 0).equals(grid(2).context().discovery().topologyVersionEx()) &&
+                    new AffinityTopologyVersion(5, 0).equals(grid(3).context().discovery().topologyVersionEx()),
+            10_000));
+
+        for (int k = 0; k < 100; k++) {
+            atomicCache.put(k, k);
+            txCache.put(k, k);
+
+            Lock lock = txCache.lock(k);
+            lock.lock();
+            lock.unlock();
+        }
+
+        for (int k = 0; k < 100; k++) {
+            assertEquals(Integer.valueOf(k), atomicCache.get(k));
+            assertEquals(Integer.valueOf(k), txCache.get(k));
+        }
+
+        assertEquals(new AffinityTopologyVersion(6, 0), grid(0).context().discovery().topologyVersionEx());
+        assertEquals(new AffinityTopologyVersion(6, 0), grid(1).context().discovery().topologyVersionEx());
+        assertEquals(new AffinityTopologyVersion(5, 0), grid(2).context().discovery().topologyVersionEx());
+        assertEquals(new AffinityTopologyVersion(5, 0), grid(3).context().discovery().topologyVersionEx());
+
+        latch.countDown();
+    }
+
+    /**
+     *
+     */
+    public static class TestDiscoverySpi extends TcpDiscoverySpi {
+        /** */
+        private volatile CountDownLatch latch;
+
+        /** {@inheritDoc} */
+        @Override protected void startMessageProcess(TcpDiscoveryAbstractMessage msg) {
+            if (msg instanceof TcpDiscoveryNodeAddFinishedMessage || msg instanceof TcpDiscoveryNodeLeftMessage || msg instanceof TcpDiscoveryNodeFailedMessage) {
+                CountDownLatch latch0 = latch;
+
+                if (latch0 != null)
+                    try {
+                        latch0.await();
+                    }
+                    catch (InterruptedException ex) {
+                        throw new IgniteException(ex);
+                    }
+            }
+
+            super.startMessageProcess(msg);
+        }
+    }
+
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 706c8e9cb9ef8..401a171ca0bb0 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -845,8 +845,7 @@ private void clientReconnectClusterDeactivated(final boolean transition) throws
         checkCaches1(SRVS + CLIENTS);
 
         // Wait for late affinity assignment to finish.
-        grid(0).context().cache().context().exchange().affinityReadyFuture(
-            new AffinityTopologyVersion(SRVS + CLIENTS, 1)).get();
+        awaitPartitionMapExchange();
 
         final AffinityTopologyVersion STATE_CHANGE_TOP_VER = new AffinityTopologyVersion(SRVS + CLIENTS + 1, 1);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
index a7ceb83e2dab1..25b03a686cc0b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheExchangeMergeTest.java
@@ -459,6 +459,8 @@ private void concurrentStart(final boolean withClients) throws Exception {
      * @throws Exception If failed.
      */
     public void testMergeServerAndClientJoin1() throws Exception {
+        fail("https://issues.apache.org/jira/browse/IGNITE-10186");
+
         final IgniteEx srv0 = startGrid(0);
 
         mergeExchangeWaitVersion(srv0, 3);
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index 66064790ae450..d32acd4384aff 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.testsuites;
 
 import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.CacheNoAffinityExchangeTest;
 import org.apache.ignite.internal.processors.cache.PartitionedAtomicCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.PartitionedTransactionalOptimisticCacheGetsDistributionTest;
 import org.apache.ignite.internal.processors.cache.PartitionedTransactionalPessimisticCacheGetsDistributionTest;
@@ -107,6 +108,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTestSuite(PartitionsExchangeCoordinatorFailoverTest.class);
 
+        suite.addTestSuite(CacheNoAffinityExchangeTest.class);
+
         suite.addTestSuite(CacheParallelStartTest.class);
 
         //suite.addTestSuite(CacheClientsConcurrentStartTest.class);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryTable.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryTable.java
index 54f5f03f556b1..ca6343c9c7e4d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryTable.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/cache/query/QueryTable.java
@@ -93,6 +93,7 @@ public String table() {
                     return false;
 
                 writer.incrementState();
+
         }
 
         return true;
@@ -121,6 +122,7 @@ public String table() {
                     return false;
 
                 reader.incrementState();
+
         }
 
         return reader.afterMessageRead(QueryTable.class);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java
index 702488487d9c0..186104ad44b54 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeRequest.java
@@ -21,6 +21,7 @@
 import java.util.List;
 import java.util.UUID;
 import org.apache.ignite.internal.GridDirectCollection;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
@@ -30,6 +31,7 @@
 /**
  * Range request.
  */
+@IgniteCodeGeneratingFail
 public class GridH2IndexRangeRequest implements Message {
     /** */
     private UUID originNodeId;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java
index 4fe660c0677a1..18814bb3f71a7 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2IndexRangeResponse.java
@@ -21,6 +21,7 @@
 import java.util.List;
 import java.util.UUID;
 import org.apache.ignite.internal.GridDirectCollection;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
@@ -30,6 +31,7 @@
 /**
  * Range response message.
  */
+@IgniteCodeGeneratingFail
 public class GridH2IndexRangeResponse implements Message {
     /** */
     public static final byte STATUS_OK = 0;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
index 4e1fadbfe3825..e130f335452ca 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2QueryRequest.java
@@ -29,6 +29,7 @@
 import org.apache.ignite.internal.GridDirectMap;
 import org.apache.ignite.internal.GridDirectTransient;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
@@ -48,6 +49,7 @@
 /**
  * Query request.
  */
+@IgniteCodeGeneratingFail
 public class GridH2QueryRequest implements Message, GridCacheQueryMarshallable {
     /** */
     private static final long serialVersionUID = 0L;
@@ -477,7 +479,7 @@ public GridH2QueryRequest schemaName(String schemaName) {
                 writer.incrementState();
 
             case 9:
-                if (!writer.writeMessage("topVer", topVer))
+                if (!writer.writeAffinityTopologyVersion("topVer", topVer))
                     return false;
 
                 writer.incrementState();
@@ -580,7 +582,7 @@ public GridH2QueryRequest schemaName(String schemaName) {
                 reader.incrementState();
 
             case 9:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessage.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessage.java
index 18f88803bd218..fd5b5243a7be8 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessage.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/msg/GridH2ValueMessage.java
@@ -20,6 +20,7 @@
 import java.nio.ByteBuffer;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteCodeGeneratingFail;
 import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.plugin.extensions.communication.MessageReader;
 import org.apache.ignite.plugin.extensions.communication.MessageWriter;
@@ -28,6 +29,7 @@
 /**
  * Abstract message wrapper for H2 values.
  */
+@IgniteCodeGeneratingFail
 public abstract class GridH2ValueMessage implements Message {
     /**
      * Gets H2 value.

From 1f4c58669a345af3cb3e603cdce0c4440320cfd5 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Mon, 19 Nov 2018 18:16:25 +0300
Subject: [PATCH 500/543] IGNITE-9558 Fixed backport artifact.

---
 .../cache/distributed/dht/GridDhtTxPrepareRequest.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
index 19a391697ba85..51fc189300185 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxPrepareRequest.java
@@ -586,7 +586,7 @@ public boolean skipCompletedVersion() {
                 reader.incrementState();
 
             case 33:
-                topVer = reader.readMessage("topVer");
+                topVer = reader.readAffinityTopologyVersion("topVer");
 
                 if (!reader.isLastRead())
                     return false;

From ecb6e5e52abb3bb7d3929387d848c91c64e03a51 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 5 Oct 2018 16:35:05 +0700
Subject: [PATCH 501/543] GG-14463 Backport to 2.5.1-master of
 GG-14270/IGNITE-9792 Fixed assert in case if IGNITE_MBEANS_DISABLED is true.
 Fixed tests.

(cherry picked from commit 78c2d3bbbd620bb7795d9f362785e073d2dec0a2)
---
 .../apache/ignite/internal/IgniteKernal.java   |  4 +++-
 .../junits/IgniteTestResources.java            | 14 +++++++++++---
 .../junits/common/GridCommonAbstractTest.java  |  4 ++--
 .../util/mbeans/GridMBeanDisableSelfTest.java  | 18 +++++++++---------
 4 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index d594ce8fc57ad..6ba5ce6e40f90 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1441,7 +1441,9 @@ private HadoopProcessorAdapter createHadoopComponent() throws IgniteCheckedExcep
     private void validateCommon(IgniteConfiguration cfg) {
         A.notNull(cfg.getNodeId(), "cfg.getNodeId()");
 
-        A.notNull(cfg.getMBeanServer(), "cfg.getMBeanServer()");
+        if (!U.IGNITE_MBEANS_DISABLED)
+            A.notNull(cfg.getMBeanServer(), "cfg.getMBeanServer()");
+
         A.notNull(cfg.getGridLogger(), "cfg.getGridLogger()");
         A.notNull(cfg.getMarshaller(), "cfg.getMarshaller()");
         A.notNull(cfg.getUserAttributes(), "cfg.getUserAttributes()");
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteTestResources.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteTestResources.java
index 5fef8bc52c3ee..59e29941331d7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteTestResources.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteTestResources.java
@@ -78,6 +78,13 @@ public class IgniteTestResources {
     /** */
     private GridResourceProcessor rsrcProc;
 
+    /**
+     * @return Default MBean server or {@code null} if {@code IGNITE_MBEANS_DISABLED} is configured.
+     */
+    @Nullable private static MBeanServer prepareMBeanServer() {
+        return U.IGNITE_MBEANS_DISABLED ? null : ManagementFactory.getPlatformMBeanServer();
+    }
+
     /**
      * @throws IgniteCheckedException If failed.
      */
@@ -87,7 +94,8 @@ public IgniteTestResources() throws IgniteCheckedException {
         else
             log = rootLog.getLogger(getClass());
 
-        this.jmx = ManagementFactory.getPlatformMBeanServer();
+        this.jmx = prepareMBeanServer();
+
         this.rsrcProc = new GridResourceProcessor(new GridTestKernalContext(this.log));
     }
 
@@ -97,7 +105,7 @@ public IgniteTestResources() throws IgniteCheckedException {
     public IgniteTestResources(IgniteConfiguration cfg) throws IgniteCheckedException {
         this.cfg = cfg;
         this.log = rootLog.getLogger(getClass());
-        this.jmx = ManagementFactory.getPlatformMBeanServer();
+        this.jmx = prepareMBeanServer();
         this.rsrcProc = new GridResourceProcessor(new GridTestKernalContext(this.log, this.cfg));
     }
 
@@ -119,7 +127,7 @@ public IgniteTestResources(IgniteLogger log) throws IgniteCheckedException {
         assert log != null;
 
         this.log = log.getLogger(getClass());
-        this.jmx = ManagementFactory.getPlatformMBeanServer();
+        this.jmx = prepareMBeanServer();
         this.rsrcProc = new GridResourceProcessor(new GridTestKernalContext(this.log));
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index ed9ba931ba6a0..dfacc70a946c4 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -490,7 +490,7 @@ protected <K, V> GridNearCacheAdapter<K, V> near(int idx, String cache) {
     }
 
     /** {@inheritDoc} */
-    @Override protected final void setUp() throws Exception {
+    @Override protected void setUp() throws Exception {
         // Disable SSL hostname verifier.
         HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
             @Override public boolean verify(String s, SSLSession sslSes) {
@@ -504,7 +504,7 @@ protected <K, V> GridNearCacheAdapter<K, V> near(int idx, String cache) {
     }
 
     /** {@inheritDoc} */
-    @Override protected final void tearDown() throws Exception {
+    @Override protected void tearDown() throws Exception {
         getTestCounters().incrementStopped();
 
         super.tearDown();
diff --git a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java
index 9a238325151db..b98cd46c369fe 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/mbeans/GridMBeanDisableSelfTest.java
@@ -33,17 +33,17 @@
  */
 public class GridMBeanDisableSelfTest extends GridCommonAbstractTest {
     /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
+    @Override protected void setUp() throws Exception {
         IgniteUtils.IGNITE_MBEANS_DISABLED = true;
 
-        super.beforeTestsStarted();
+        super.setUp();
     }
 
     /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        super.afterTestsStopped();
-
+    @Override protected void tearDown() throws Exception {
         IgniteUtils.IGNITE_MBEANS_DISABLED = false;
+
+        super.tearDown();
     }
 
     /**
@@ -54,12 +54,12 @@ public class GridMBeanDisableSelfTest extends GridCommonAbstractTest {
     public void testCorrectMBeanInfo() throws Exception {
         // Node should start and stopped with no errors.
         try (final Ignite ignite = startGrid(0)) {
-            final MBeanServer server = ignite.configuration().getMBeanServer();
+            final MBeanServer srv = ignite.configuration().getMBeanServer();
 
             GridTestUtils.assertThrowsWithCause(
                 new Callable<Void>() {
                     @Override public Void call() throws Exception {
-                        U.registerMBean(server, ignite.name(), "dummy", "DummyMbean1", new DummyMBeanImpl(), DummyMBean.class);
+                        U.registerMBean(srv, ignite.name(), "dummy", "DummyMbean1", new DummyMBeanImpl(), DummyMBean.class);
 
                         return null;
                     }
@@ -74,7 +74,7 @@ public void testCorrectMBeanInfo() throws Exception {
                             "DummyMbean2"
                         );
 
-                        U.registerMBean(server, objName, new DummyMBeanImpl(), DummyMBean.class);
+                        U.registerMBean(srv, objName, new DummyMBeanImpl(), DummyMBean.class);
 
                         return null;
 
@@ -111,4 +111,4 @@ static class DummyMBeanImpl implements DummyMBean {
             // No op.
         }
     }
-}
\ No newline at end of file
+}

From c8149116ae7875e73a8ae37ed8b0e28d13befd3a Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Wed, 21 Nov 2018 20:35:23 +0300
Subject: [PATCH 502/543] IGNITE-10339 Fixed cluster connection when using
 validate indexes from control.sh - Fixes #5458.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 5fc9a3fc1ef5c486e2656661e2e811b235b6543f)
---
 .../internal/commandline/CommandHandler.java  | 50 +++++++++----------
 .../verify/VisorValidateIndexesJobResult.java |  9 ++++
 .../verify/VisorValidateIndexesTaskArg.java   | 23 +++++++--
 .../verify/VisorValidateIndexesTask.java      | 29 +++++++++++
 .../util/GridCommandHandlerIndexingTest.java  |  2 +-
 5 files changed, 84 insertions(+), 29 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 7c4c75b42fe93..3f6652ab78769 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -893,39 +893,41 @@ private void cacheContention(GridClient client, CacheArguments cacheArgs) throws
     private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) throws GridClientException {
         VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(
             cacheArgs.caches(),
+            cacheArgs.nodeId() != null ? Collections.singleton(cacheArgs.nodeId()) : null,
             cacheArgs.checkFirst(),
             cacheArgs.checkThrough()
         );
 
-        UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId();
-
         VisorValidateIndexesTaskResult taskRes = executeTaskByNameOnNode(
-            client, VALIDATE_INDEXES_TASK, taskArg, nodeId);
+            client, VALIDATE_INDEXES_TASK, taskArg, null);
 
         if (!F.isEmpty(taskRes.exceptions())) {
             log("Index validation failed on nodes:");
 
             for (Map.Entry<UUID, Exception> e : taskRes.exceptions().entrySet()) {
-                log("Node ID = " + e.getKey());
+                log(i("Node ID = " + e.getKey()));
 
-                log("Exception message:");
-                log(e.getValue().getMessage());
+                log(i("Exception message:"));
+                log(i(e.getValue().getMessage(), 2));
                 nl();
             }
         }
 
+        boolean errors = false;
+
         for (Map.Entry<UUID, VisorValidateIndexesJobResult> nodeEntry : taskRes.results().entrySet()) {
-            log("validate_indexes result on node " + nodeEntry.getKey() + ":");
+            if (!nodeEntry.getValue().hasIssues())
+                continue;
 
-            Collection<IndexIntegrityCheckIssue> integrityCheckFailures = nodeEntry.getValue().integrityCheckFailures();
+            errors = true;
 
-            boolean errors = false;
+            log("Index issues found on node " + nodeEntry.getKey() + ":");
 
-            if (!integrityCheckFailures.isEmpty()) {
-                errors = true;
+            Collection<IndexIntegrityCheckIssue> integrityCheckFailures = nodeEntry.getValue().integrityCheckFailures();
 
+            if (!integrityCheckFailures.isEmpty()) {
                 for (IndexIntegrityCheckIssue is : integrityCheckFailures)
-                    log("\t" + is.toString());
+                    log(i(is.toString()));
             }
 
             Map<PartitionKey, ValidateIndexesPartitionResult> partRes = nodeEntry.getValue().partitionResult();
@@ -934,12 +936,10 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
                 ValidateIndexesPartitionResult res = e.getValue();
 
                 if (!res.issues().isEmpty()) {
-                    errors = true;
-
-                    log("\t" + e.getKey().toString() + " " + e.getValue().toString());
+                    log(i(e.getKey().toString() + " " + e.getValue().toString()));
 
                     for (IndexValidationIssue is : res.issues())
-                        log("\t\t" + is.toString());
+                        log(i(is.toString(), 2));
                 }
             }
 
@@ -949,20 +949,20 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
                 ValidateIndexesPartitionResult res = e.getValue();
 
                 if (!res.issues().isEmpty()) {
-                    errors = true;
-
-                    log("\tSQL Index " + e.getKey() + " " + e.getValue().toString());
+                    log(i("SQL Index " + e.getKey() + " " + e.getValue().toString()));
 
                     for (IndexValidationIssue is : res.issues())
-                        log("\t\t" + is.toString());
+                        log(i(is.toString(),2));
                 }
             }
-
-            if (!errors)
-                log("no issues found.\n");
-            else
-                log("issues found (listed above).\n");
         }
+
+        if (!errors)
+            log("no issues found.");
+        else
+            log("issues found (listed above).");
+
+        nl();
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
index f84fc1a259027..6d7f7652518cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
@@ -92,6 +92,15 @@ public Collection<IndexIntegrityCheckIssue> integrityCheckFailures() {
         return integrityCheckFailures == null ? Collections.emptyList() : integrityCheckFailures;
     }
 
+    /**
+     * @return {@code true} If any indexes issues found on node, otherwise returns {@code false}.
+     */
+    public boolean hasIssues() {
+        return (integrityCheckFailures != null && !integrityCheckFailures.isEmpty()) ||
+                (partRes != null && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty())) ||
+                (idxRes != null && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()));
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeMap(out, partRes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
index aa49977c7a6fa..6dbd961028e69 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
@@ -21,12 +21,14 @@
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import java.util.Set;
+import java.util.UUID;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
 
+
 /**
- * Arguments for task {@link VisorIdleVerifyTask}
+ *
  */
 public class VisorValidateIndexesTaskArg extends VisorDataTransferObject {
     /** */
@@ -41,6 +43,9 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject {
     /** Check through K element (skip K-1, check Kth). */
     private int checkThrough;
 
+    /** Nodes on which task will run. */
+    private Set<UUID> nodes;
+
     /**
      * Default constructor.
      */
@@ -51,10 +56,11 @@ public VisorValidateIndexesTaskArg() {
     /**
      * @param caches Caches.
      */
-    public VisorValidateIndexesTaskArg(Set<String> caches, int checkFirst, int checkThrough) {
+    public VisorValidateIndexesTaskArg(Set<String> caches, Set<UUID> nodes, int checkFirst, int checkThrough) {
         this.caches = caches;
         this.checkFirst = checkFirst;
         this.checkThrough = checkThrough;
+        this.nodes = nodes;
     }
 
 
@@ -65,6 +71,13 @@ public Set<String> getCaches() {
         return caches;
     }
 
+    /**
+     * @return Nodes on which task will run. If {@code null}, task will run on all server nodes.
+     */
+    public Set<UUID> getNodes() {
+        return nodes;
+    }
+
     /**
      * @return checkFirst.
      */
@@ -84,6 +97,7 @@ public int getCheckThrough() {
         U.writeCollection(out, caches);
         out.writeInt(checkFirst);
         out.writeInt(checkThrough);
+        U.writeCollection(out, nodes);
     }
 
     /** {@inheritDoc} */
@@ -98,11 +112,14 @@ public int getCheckThrough() {
             checkFirst = -1;
             checkThrough = -1;
         }
+
+        if (protoVer > V2)
+            nodes = U.readSet(in);
     }
 
     /** {@inheritDoc} */
     @Override public byte getProtocolVersion() {
-        return V2;
+        return V3;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
index abb7f7ee55ec1..922c53e15ed97 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
@@ -17,16 +17,22 @@
 
 package org.apache.ignite.internal.visor.verify;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJobResult;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -58,6 +64,29 @@ public class VisorValidateIndexesTask extends VisorMultiNodeTask<VisorValidateIn
         return new VisorValidateIndexesJob(arg, debug);
     }
 
+    /** {@inheritDoc} */
+    @Override protected Collection<UUID> jobNodes(VisorTaskArgument<VisorValidateIndexesTaskArg> arg) {
+        Collection<ClusterNode> srvNodes = ignite.cluster().forServers().nodes();
+        Collection<UUID> ret = new ArrayList<>(srvNodes.size());
+
+        VisorValidateIndexesTaskArg taskArg = arg.getArgument();
+
+        Set<UUID> nodeIds = taskArg.getNodes() != null ? new HashSet<>(taskArg.getNodes()) : null;
+
+        if (nodeIds == null) {
+            for (ClusterNode node : srvNodes)
+                ret.add(node.id());
+        }
+        else {
+            for (ClusterNode node : srvNodes) {
+                if (nodeIds.contains(node.id()))
+                    ret.add(node.id());
+            }
+        }
+
+        return ret;
+    }
+
     /**
      *
      */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
index c7693d214b0ac..dc5011ffe170b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -113,7 +113,7 @@ public void testBrokenSqlIndexShouldFailValidation() throws Exception {
     }
 
     /**
-     * Tests that missing rows in H2 indexes are detected.
+     * Tests that corrupted pages in the index partition are detected.
      */
     public void testCorruptedIndexPartitionShouldFailValidation() throws Exception {
         Ignite ignite = prepareGridForTest();

From 9291a0f45b9d96acc44e2425453f3e8f9cd8cbd4 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Wed, 21 Nov 2018 20:35:23 +0300
Subject: [PATCH 503/543] IGNITE-10339 Fixed cluster connection when using
 validate indexes from control.sh - Fixes #5458.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 5fc9a3fc1ef5c486e2656661e2e811b235b6543f)
---
 .../internal/commandline/CommandHandler.java  | 50 +++++++++----------
 .../verify/VisorValidateIndexesJobResult.java |  9 ++++
 .../verify/VisorValidateIndexesTaskArg.java   | 23 +++++++--
 .../verify/VisorValidateIndexesTask.java      | 29 +++++++++++
 .../util/GridCommandHandlerIndexingTest.java  |  2 +-
 5 files changed, 84 insertions(+), 29 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 7c4c75b42fe93..3f6652ab78769 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -893,39 +893,41 @@ private void cacheContention(GridClient client, CacheArguments cacheArgs) throws
     private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) throws GridClientException {
         VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(
             cacheArgs.caches(),
+            cacheArgs.nodeId() != null ? Collections.singleton(cacheArgs.nodeId()) : null,
             cacheArgs.checkFirst(),
             cacheArgs.checkThrough()
         );
 
-        UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId();
-
         VisorValidateIndexesTaskResult taskRes = executeTaskByNameOnNode(
-            client, VALIDATE_INDEXES_TASK, taskArg, nodeId);
+            client, VALIDATE_INDEXES_TASK, taskArg, null);
 
         if (!F.isEmpty(taskRes.exceptions())) {
             log("Index validation failed on nodes:");
 
             for (Map.Entry<UUID, Exception> e : taskRes.exceptions().entrySet()) {
-                log("Node ID = " + e.getKey());
+                log(i("Node ID = " + e.getKey()));
 
-                log("Exception message:");
-                log(e.getValue().getMessage());
+                log(i("Exception message:"));
+                log(i(e.getValue().getMessage(), 2));
                 nl();
             }
         }
 
+        boolean errors = false;
+
         for (Map.Entry<UUID, VisorValidateIndexesJobResult> nodeEntry : taskRes.results().entrySet()) {
-            log("validate_indexes result on node " + nodeEntry.getKey() + ":");
+            if (!nodeEntry.getValue().hasIssues())
+                continue;
 
-            Collection<IndexIntegrityCheckIssue> integrityCheckFailures = nodeEntry.getValue().integrityCheckFailures();
+            errors = true;
 
-            boolean errors = false;
+            log("Index issues found on node " + nodeEntry.getKey() + ":");
 
-            if (!integrityCheckFailures.isEmpty()) {
-                errors = true;
+            Collection<IndexIntegrityCheckIssue> integrityCheckFailures = nodeEntry.getValue().integrityCheckFailures();
 
+            if (!integrityCheckFailures.isEmpty()) {
                 for (IndexIntegrityCheckIssue is : integrityCheckFailures)
-                    log("\t" + is.toString());
+                    log(i(is.toString()));
             }
 
             Map<PartitionKey, ValidateIndexesPartitionResult> partRes = nodeEntry.getValue().partitionResult();
@@ -934,12 +936,10 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
                 ValidateIndexesPartitionResult res = e.getValue();
 
                 if (!res.issues().isEmpty()) {
-                    errors = true;
-
-                    log("\t" + e.getKey().toString() + " " + e.getValue().toString());
+                    log(i(e.getKey().toString() + " " + e.getValue().toString()));
 
                     for (IndexValidationIssue is : res.issues())
-                        log("\t\t" + is.toString());
+                        log(i(is.toString(), 2));
                 }
             }
 
@@ -949,20 +949,20 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
                 ValidateIndexesPartitionResult res = e.getValue();
 
                 if (!res.issues().isEmpty()) {
-                    errors = true;
-
-                    log("\tSQL Index " + e.getKey() + " " + e.getValue().toString());
+                    log(i("SQL Index " + e.getKey() + " " + e.getValue().toString()));
 
                     for (IndexValidationIssue is : res.issues())
-                        log("\t\t" + is.toString());
+                        log(i(is.toString(),2));
                 }
             }
-
-            if (!errors)
-                log("no issues found.\n");
-            else
-                log("issues found (listed above).\n");
         }
+
+        if (!errors)
+            log("no issues found.");
+        else
+            log("issues found (listed above).");
+
+        nl();
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
index f84fc1a259027..6d7f7652518cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
@@ -92,6 +92,15 @@ public Collection<IndexIntegrityCheckIssue> integrityCheckFailures() {
         return integrityCheckFailures == null ? Collections.emptyList() : integrityCheckFailures;
     }
 
+    /**
+     * @return {@code true} If any indexes issues found on node, otherwise returns {@code false}.
+     */
+    public boolean hasIssues() {
+        return (integrityCheckFailures != null && !integrityCheckFailures.isEmpty()) ||
+                (partRes != null && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty())) ||
+                (idxRes != null && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()));
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeMap(out, partRes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
index aa49977c7a6fa..6dbd961028e69 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
@@ -21,12 +21,14 @@
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
 import java.util.Set;
+import java.util.UUID;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.internal.visor.VisorDataTransferObject;
 
+
 /**
- * Arguments for task {@link VisorIdleVerifyTask}
+ *
  */
 public class VisorValidateIndexesTaskArg extends VisorDataTransferObject {
     /** */
@@ -41,6 +43,9 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject {
     /** Check through K element (skip K-1, check Kth). */
     private int checkThrough;
 
+    /** Nodes on which task will run. */
+    private Set<UUID> nodes;
+
     /**
      * Default constructor.
      */
@@ -51,10 +56,11 @@ public VisorValidateIndexesTaskArg() {
     /**
      * @param caches Caches.
      */
-    public VisorValidateIndexesTaskArg(Set<String> caches, int checkFirst, int checkThrough) {
+    public VisorValidateIndexesTaskArg(Set<String> caches, Set<UUID> nodes, int checkFirst, int checkThrough) {
         this.caches = caches;
         this.checkFirst = checkFirst;
         this.checkThrough = checkThrough;
+        this.nodes = nodes;
     }
 
 
@@ -65,6 +71,13 @@ public Set<String> getCaches() {
         return caches;
     }
 
+    /**
+     * @return Nodes on which task will run. If {@code null}, task will run on all server nodes.
+     */
+    public Set<UUID> getNodes() {
+        return nodes;
+    }
+
     /**
      * @return checkFirst.
      */
@@ -84,6 +97,7 @@ public int getCheckThrough() {
         U.writeCollection(out, caches);
         out.writeInt(checkFirst);
         out.writeInt(checkThrough);
+        U.writeCollection(out, nodes);
     }
 
     /** {@inheritDoc} */
@@ -98,11 +112,14 @@ public int getCheckThrough() {
             checkFirst = -1;
             checkThrough = -1;
         }
+
+        if (protoVer > V2)
+            nodes = U.readSet(in);
     }
 
     /** {@inheritDoc} */
     @Override public byte getProtocolVersion() {
-        return V2;
+        return V3;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
index abb7f7ee55ec1..922c53e15ed97 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
@@ -17,16 +17,22 @@
 
 package org.apache.ignite.internal.visor.verify;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.UUID;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJobResult;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.internal.S;
 import org.apache.ignite.internal.visor.VisorJob;
 import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -58,6 +64,29 @@ public class VisorValidateIndexesTask extends VisorMultiNodeTask<VisorValidateIn
         return new VisorValidateIndexesJob(arg, debug);
     }
 
+    /** {@inheritDoc} */
+    @Override protected Collection<UUID> jobNodes(VisorTaskArgument<VisorValidateIndexesTaskArg> arg) {
+        Collection<ClusterNode> srvNodes = ignite.cluster().forServers().nodes();
+        Collection<UUID> ret = new ArrayList<>(srvNodes.size());
+
+        VisorValidateIndexesTaskArg taskArg = arg.getArgument();
+
+        Set<UUID> nodeIds = taskArg.getNodes() != null ? new HashSet<>(taskArg.getNodes()) : null;
+
+        if (nodeIds == null) {
+            for (ClusterNode node : srvNodes)
+                ret.add(node.id());
+        }
+        else {
+            for (ClusterNode node : srvNodes) {
+                if (nodeIds.contains(node.id()))
+                    ret.add(node.id());
+            }
+        }
+
+        return ret;
+    }
+
     /**
      *
      */
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
index c7693d214b0ac..dc5011ffe170b 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -113,7 +113,7 @@ public void testBrokenSqlIndexShouldFailValidation() throws Exception {
     }
 
     /**
-     * Tests that missing rows in H2 indexes are detected.
+     * Tests that corrupted pages in the index partition are detected.
      */
     public void testCorruptedIndexPartitionShouldFailValidation() throws Exception {
         Ignite ignite = prepareGridForTest();

From c8841182a92b3ef36da6c7de84f1f2629d824382 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Thu, 22 Nov 2018 00:14:00 +0300
Subject: [PATCH 504/543] Project prepare for p161 release.

---
 examples/pom.xml                                   | 2 +-
 modules/aop/pom.xml                                | 2 +-
 modules/apache-license-gen/pom.xml                 | 2 +-
 modules/aws/pom.xml                                | 2 +-
 modules/benchmarks/pom.xml                         | 2 +-
 modules/camel/pom.xml                              | 2 +-
 modules/cassandra/pom.xml                          | 2 +-
 modules/cassandra/serializers/pom.xml              | 4 ++--
 modules/cassandra/store/pom.xml                    | 4 ++--
 modules/clients/pom.xml                            | 2 +-
 modules/cloud/pom.xml                              | 2 +-
 modules/codegen/pom.xml                            | 2 +-
 modules/compatibility/pom.xml                      | 2 +-
 modules/core/pom.xml                               | 2 +-
 modules/core/src/main/resources/ignite.properties  | 2 +-
 modules/dev-utils/pom.xml                          | 2 +-
 modules/direct-io/pom.xml                          | 2 +-
 modules/extdata/p2p/pom.xml                        | 2 +-
 modules/extdata/platform/pom.xml                   | 2 +-
 modules/extdata/uri/modules/uri-dependency/pom.xml | 2 +-
 modules/extdata/uri/pom.xml                        | 2 +-
 modules/flink/pom.xml                              | 2 +-
 modules/flume/pom.xml                              | 2 +-
 modules/gce/pom.xml                                | 2 +-
 modules/geospatial/pom.xml                         | 2 +-
 modules/hadoop/pom.xml                             | 2 +-
 modules/hibernate-4.2/pom.xml                      | 2 +-
 modules/hibernate-5.1/pom.xml                      | 2 +-
 modules/hibernate-core/pom.xml                     | 2 +-
 modules/ignored-tests/pom.xml                      | 2 +-
 modules/indexing/pom.xml                           | 2 +-
 modules/jcl/pom.xml                                | 2 +-
 modules/jms11/pom.xml                              | 2 +-
 modules/jta/pom.xml                                | 2 +-
 modules/kafka/pom.xml                              | 2 +-
 modules/kubernetes/pom.xml                         | 2 +-
 modules/log4j/pom.xml                              | 2 +-
 modules/log4j2/pom.xml                             | 2 +-
 modules/mesos/pom.xml                              | 2 +-
 modules/ml/pom.xml                                 | 2 +-
 modules/mqtt/pom.xml                               | 2 +-
 modules/osgi-karaf/pom.xml                         | 2 +-
 modules/osgi-paxlogging/pom.xml                    | 2 +-
 modules/osgi/pom.xml                               | 2 +-
 modules/rest-http/pom.xml                          | 2 +-
 modules/rocketmq/pom.xml                           | 2 +-
 modules/scalar-2.10/pom.xml                        | 2 +-
 modules/scalar/pom.xml                             | 2 +-
 modules/schedule/pom.xml                           | 2 +-
 modules/slf4j/pom.xml                              | 2 +-
 modules/spark-2.10/pom.xml                         | 2 +-
 modules/spark/pom.xml                              | 2 +-
 modules/spring-data/pom.xml                        | 2 +-
 modules/spring/pom.xml                             | 2 +-
 modules/sqlline/pom.xml                            | 2 +-
 modules/ssh/pom.xml                                | 2 +-
 modules/storm/pom.xml                              | 2 +-
 modules/tools/pom.xml                              | 2 +-
 modules/twitter/pom.xml                            | 2 +-
 modules/urideploy/pom.xml                          | 2 +-
 modules/visor-console-2.10/pom.xml                 | 2 +-
 modules/visor-console/pom.xml                      | 2 +-
 modules/visor-plugins/pom.xml                      | 2 +-
 modules/web-console/pom.xml                        | 2 +-
 modules/web-console/web-agent/pom.xml              | 2 +-
 modules/web/ignite-appserver-test/pom.xml          | 2 +-
 modules/web/ignite-websphere-test/pom.xml          | 2 +-
 modules/web/pom.xml                                | 2 +-
 modules/yardstick/pom.xml                          | 2 +-
 modules/yarn/pom.xml                               | 2 +-
 modules/zeromq/pom.xml                             | 2 +-
 modules/zookeeper/pom.xml                          | 2 +-
 pom.xml                                            | 2 +-
 73 files changed, 75 insertions(+), 75 deletions(-)

diff --git a/examples/pom.xml b/examples/pom.xml
index 3b101e2709ee3..01c5ba1014a06 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -27,7 +27,7 @@
     </parent>
 
     <artifactId>ignite-examples</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/aop/pom.xml b/modules/aop/pom.xml
index 07599a8188ab3..03d9f3582c27f 100644
--- a/modules/aop/pom.xml
+++ b/modules/aop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aop</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/apache-license-gen/pom.xml b/modules/apache-license-gen/pom.xml
index dd7b0fa22b677..4cf4ba3114f34 100644
--- a/modules/apache-license-gen/pom.xml
+++ b/modules/apache-license-gen/pom.xml
@@ -31,7 +31,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>ignite-apache-license-gen</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <build>
diff --git a/modules/aws/pom.xml b/modules/aws/pom.xml
index a758725727875..8378dc2c3184e 100644
--- a/modules/aws/pom.xml
+++ b/modules/aws/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aws</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/benchmarks/pom.xml b/modules/benchmarks/pom.xml
index d1e3f0dd58a41..791461df337a4 100644
--- a/modules/benchmarks/pom.xml
+++ b/modules/benchmarks/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-benchmarks</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/camel/pom.xml b/modules/camel/pom.xml
index 0ca6e40b53fb5..1cd3555ff9fee 100644
--- a/modules/camel/pom.xml
+++ b/modules/camel/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-camel</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/pom.xml b/modules/cassandra/pom.xml
index 41283db70aaf8..4a3128c7bed72 100644
--- a/modules/cassandra/pom.xml
+++ b/modules/cassandra/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-cassandra</artifactId>
     <packaging>pom</packaging>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencyManagement>
diff --git a/modules/cassandra/serializers/pom.xml b/modules/cassandra/serializers/pom.xml
index 8ba6a25f03551..2e99b14b7211d 100644
--- a/modules/cassandra/serializers/pom.xml
+++ b/modules/cassandra/serializers/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>2.5.1-p160-SNAPSHOT</version>
+        <version>2.5.1-p161-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-serializers</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/store/pom.xml b/modules/cassandra/store/pom.xml
index 4035efeaa4f4a..a09e95c649e48 100644
--- a/modules/cassandra/store/pom.xml
+++ b/modules/cassandra/store/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>2.5.1-p160-SNAPSHOT</version>
+        <version>2.5.1-p161-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-store</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/clients/pom.xml b/modules/clients/pom.xml
index 9918d424dc96c..5c9c400fcd40e 100644
--- a/modules/clients/pom.xml
+++ b/modules/clients/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-clients</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/cloud/pom.xml b/modules/cloud/pom.xml
index d0812d9c98df1..90785ea5cc63a 100644
--- a/modules/cloud/pom.xml
+++ b/modules/cloud/pom.xml
@@ -29,7 +29,7 @@
     </parent>
 
     <artifactId>ignite-cloud</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/codegen/pom.xml b/modules/codegen/pom.xml
index f907b4c5f24a2..9eaf116f838ba 100644
--- a/modules/codegen/pom.xml
+++ b/modules/codegen/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-codegen</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/compatibility/pom.xml b/modules/compatibility/pom.xml
index 80a86168734c2..fc654c6ce8af5 100644
--- a/modules/compatibility/pom.xml
+++ b/modules/compatibility/pom.xml
@@ -33,7 +33,7 @@
     </parent>
 
     <artifactId>ignite-compatibility</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 74df1869bcd63..7e70ec181b79c 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-core</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <repositories>
diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index b903e31278425..ed62fa6bd88cf 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=2.5.1-p160-SNAPSHOT
+ignite.version=2.5.1-p161-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970
diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml
index 0d70055753210..cffe6366c778e 100644
--- a/modules/dev-utils/pom.xml
+++ b/modules/dev-utils/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-dev-utils</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/direct-io/pom.xml b/modules/direct-io/pom.xml
index a7cb8f6113611..a2f93760a4b66 100644
--- a/modules/direct-io/pom.xml
+++ b/modules/direct-io/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-direct-io</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/extdata/p2p/pom.xml b/modules/extdata/p2p/pom.xml
index a7d77aa10e2f7..aa27b6f5d65d2 100644
--- a/modules/extdata/p2p/pom.xml
+++ b/modules/extdata/p2p/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-extdata-p2p</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/extdata/platform/pom.xml b/modules/extdata/platform/pom.xml
index 13811d0005527..92fb01cc35eda 100644
--- a/modules/extdata/platform/pom.xml
+++ b/modules/extdata/platform/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-platform</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/extdata/uri/modules/uri-dependency/pom.xml b/modules/extdata/uri/modules/uri-dependency/pom.xml
index 8346102cfb3d0..6e9ef0772680b 100644
--- a/modules/extdata/uri/modules/uri-dependency/pom.xml
+++ b/modules/extdata/uri/modules/uri-dependency/pom.xml
@@ -27,7 +27,7 @@
     <artifactId>ignite-extdata-uri-dep</artifactId>
     <packaging>jar</packaging>
 
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <modelVersion>4.0.0</modelVersion>
 
     <dependencies>
diff --git a/modules/extdata/uri/pom.xml b/modules/extdata/uri/pom.xml
index 1fe98c0aab676..060cbd27d20d9 100644
--- a/modules/extdata/uri/pom.xml
+++ b/modules/extdata/uri/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-uri</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/flink/pom.xml b/modules/flink/pom.xml
index 3520612038467..07ffd83b9f6b6 100644
--- a/modules/flink/pom.xml
+++ b/modules/flink/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flink</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/flume/pom.xml b/modules/flume/pom.xml
index 625c134ed1326..15f94375cb6f5 100644
--- a/modules/flume/pom.xml
+++ b/modules/flume/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flume</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/gce/pom.xml b/modules/gce/pom.xml
index b0f28056ee087..4677acd29f742 100644
--- a/modules/gce/pom.xml
+++ b/modules/gce/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-gce</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/geospatial/pom.xml b/modules/geospatial/pom.xml
index 21497b551dcf1..a66605133ca54 100644
--- a/modules/geospatial/pom.xml
+++ b/modules/geospatial/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-geospatial</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index f664fe0a6814c..9fe299e9626fc 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hadoop</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-4.2/pom.xml b/modules/hibernate-4.2/pom.xml
index b4c52c32ac138..baa4822bbcc2c 100644
--- a/modules/hibernate-4.2/pom.xml
+++ b/modules/hibernate-4.2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate_4.2</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-5.1/pom.xml b/modules/hibernate-5.1/pom.xml
index 09c5c3a9cc399..06829fcc095fe 100644
--- a/modules/hibernate-5.1/pom.xml
+++ b/modules/hibernate-5.1/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate_5.1</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-core/pom.xml b/modules/hibernate-core/pom.xml
index 1ba3ab548b548..1a6a68359b820 100644
--- a/modules/hibernate-core/pom.xml
+++ b/modules/hibernate-core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate-core</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ignored-tests/pom.xml b/modules/ignored-tests/pom.xml
index 3ed8c9b97fc21..9c01eaf188970 100644
--- a/modules/ignored-tests/pom.xml
+++ b/modules/ignored-tests/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ignored-tests</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml
index 70004c57ee043..c641abc297228 100644
--- a/modules/indexing/pom.xml
+++ b/modules/indexing/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-indexing</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jcl/pom.xml b/modules/jcl/pom.xml
index 4daa5b1754d7a..19b0523a0e50c 100644
--- a/modules/jcl/pom.xml
+++ b/modules/jcl/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jcl</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jms11/pom.xml b/modules/jms11/pom.xml
index 11be42c308091..80c2fc8f95aae 100644
--- a/modules/jms11/pom.xml
+++ b/modules/jms11/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jms11</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jta/pom.xml b/modules/jta/pom.xml
index c4e5d553dda28..fc30e7a02760e 100644
--- a/modules/jta/pom.xml
+++ b/modules/jta/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jta</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kafka/pom.xml b/modules/kafka/pom.xml
index 85985bf063468..3050778e3268f 100644
--- a/modules/kafka/pom.xml
+++ b/modules/kafka/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kafka</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kubernetes/pom.xml b/modules/kubernetes/pom.xml
index 4f1ff56d65a1a..66fa6f9520813 100644
--- a/modules/kubernetes/pom.xml
+++ b/modules/kubernetes/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kubernetes</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j/pom.xml b/modules/log4j/pom.xml
index 5a15560dff118..c467beea93721 100644
--- a/modules/log4j/pom.xml
+++ b/modules/log4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j2/pom.xml b/modules/log4j2/pom.xml
index ed6618a1913c7..dfb39057086ff 100644
--- a/modules/log4j2/pom.xml
+++ b/modules/log4j2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j2</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml
index ec0d2ee719494..283aef618c534 100644
--- a/modules/mesos/pom.xml
+++ b/modules/mesos/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mesos</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml
index 66ee575acfe82..e9d7fc83374d5 100644
--- a/modules/ml/pom.xml
+++ b/modules/ml/pom.xml
@@ -30,7 +30,7 @@
     </parent>
 
     <artifactId>ignite-ml</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mqtt/pom.xml b/modules/mqtt/pom.xml
index 5e59a161654bb..4e0cab1c073c4 100644
--- a/modules/mqtt/pom.xml
+++ b/modules/mqtt/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mqtt</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/osgi-karaf/pom.xml b/modules/osgi-karaf/pom.xml
index 623df3b842c0d..10846d9f8990f 100644
--- a/modules/osgi-karaf/pom.xml
+++ b/modules/osgi-karaf/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-karaf</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <build>
diff --git a/modules/osgi-paxlogging/pom.xml b/modules/osgi-paxlogging/pom.xml
index fdfce36e585da..76b58d35889b5 100644
--- a/modules/osgi-paxlogging/pom.xml
+++ b/modules/osgi-paxlogging/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-paxlogging</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <dependencies>
diff --git a/modules/osgi/pom.xml b/modules/osgi/pom.xml
index 3fff5ba889096..89ae5dd186d4e 100644
--- a/modules/osgi/pom.xml
+++ b/modules/osgi/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/rest-http/pom.xml b/modules/rest-http/pom.xml
index cdc67ea5a6ec3..dd18b04a665dc 100644
--- a/modules/rest-http/pom.xml
+++ b/modules/rest-http/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-rest-http</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/rocketmq/pom.xml b/modules/rocketmq/pom.xml
index eb2582d980d84..aca233ba8e2f3 100644
--- a/modules/rocketmq/pom.xml
+++ b/modules/rocketmq/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-rocketmq</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar-2.10/pom.xml b/modules/scalar-2.10/pom.xml
index 3b6bcf557f7ef..6f95df562a385 100644
--- a/modules/scalar-2.10/pom.xml
+++ b/modules/scalar-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar_2.10</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml
index 9c6fe3bf70250..24faae3e3c6e8 100644
--- a/modules/scalar/pom.xml
+++ b/modules/scalar/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/schedule/pom.xml b/modules/schedule/pom.xml
index de69b406ab3b1..432da2363c453 100644
--- a/modules/schedule/pom.xml
+++ b/modules/schedule/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-schedule</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/slf4j/pom.xml b/modules/slf4j/pom.xml
index 8b684983df479..3f335a7cb17ae 100644
--- a/modules/slf4j/pom.xml
+++ b/modules/slf4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-slf4j</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark-2.10/pom.xml b/modules/spark-2.10/pom.xml
index d0c261654a46c..7e3e109491157 100644
--- a/modules/spark-2.10/pom.xml
+++ b/modules/spark-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark_2.10</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index b8047df2562bb..817982bb2a625 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring-data/pom.xml b/modules/spring-data/pom.xml
index 96e2ff99d1bdb..5c067d03348f1 100644
--- a/modules/spring-data/pom.xml
+++ b/modules/spring-data/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring-data</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring/pom.xml b/modules/spring/pom.xml
index b3af4b8b5166c..97efcffd442be 100644
--- a/modules/spring/pom.xml
+++ b/modules/spring/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/sqlline/pom.xml b/modules/sqlline/pom.xml
index d643c4595f806..dab0a385390ff 100644
--- a/modules/sqlline/pom.xml
+++ b/modules/sqlline/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-sqlline</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ssh/pom.xml b/modules/ssh/pom.xml
index d5ba8e77e4c01..6b5269d5887e3 100644
--- a/modules/ssh/pom.xml
+++ b/modules/ssh/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ssh</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/storm/pom.xml b/modules/storm/pom.xml
index d1101f888304a..26df45210cd70 100644
--- a/modules/storm/pom.xml
+++ b/modules/storm/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-storm</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/tools/pom.xml b/modules/tools/pom.xml
index 209a302f6ab7b..050754d8eeb14 100644
--- a/modules/tools/pom.xml
+++ b/modules/tools/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-tools</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/twitter/pom.xml b/modules/twitter/pom.xml
index beeab4db8d8ec..1488915048673 100644
--- a/modules/twitter/pom.xml
+++ b/modules/twitter/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-twitter</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/urideploy/pom.xml b/modules/urideploy/pom.xml
index 06a09721a3e96..2e9fef6ca85ac 100644
--- a/modules/urideploy/pom.xml
+++ b/modules/urideploy/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-urideploy</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console-2.10/pom.xml b/modules/visor-console-2.10/pom.xml
index 2f49e72408b70..bc054d423894a 100644
--- a/modules/visor-console-2.10/pom.xml
+++ b/modules/visor-console-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console_2.10</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console/pom.xml b/modules/visor-console/pom.xml
index 848cbef76e695..99ef68ee42293 100644
--- a/modules/visor-console/pom.xml
+++ b/modules/visor-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-plugins/pom.xml b/modules/visor-plugins/pom.xml
index 523fc83e1e105..6a5203754c0f2 100644
--- a/modules/visor-plugins/pom.xml
+++ b/modules/visor-plugins/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-plugins</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web-console/pom.xml b/modules/web-console/pom.xml
index cf4c97f2aa82e..5f538e2527eb2 100644
--- a/modules/web-console/pom.xml
+++ b/modules/web-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web-console</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml
index 06127312604fb..956d0ee7e3fda 100644
--- a/modules/web-console/web-agent/pom.xml
+++ b/modules/web-console/web-agent/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-web-agent</artifactId>
     <packaging>jar</packaging>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web/ignite-appserver-test/pom.xml b/modules/web/ignite-appserver-test/pom.xml
index de101df12fc35..aa4adb3d1f8c0 100644
--- a/modules/web/ignite-appserver-test/pom.xml
+++ b/modules/web/ignite-appserver-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-appserver-test</artifactId>
     <packaging>jar</packaging>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/ignite-websphere-test/pom.xml b/modules/web/ignite-websphere-test/pom.xml
index 6b3b340cb1f9e..4b2802200b299 100644
--- a/modules/web/ignite-websphere-test/pom.xml
+++ b/modules/web/ignite-websphere-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-websphere-test</artifactId>
     <packaging>war</packaging>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/pom.xml b/modules/web/pom.xml
index 913e9bb223601..8dd45d4ac2083 100644
--- a/modules/web/pom.xml
+++ b/modules/web/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index ca8f30646414e..af9dd3002c03c 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yardstick</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml
index 34238af0aeec7..37a8c34a9240c 100644
--- a/modules/yarn/pom.xml
+++ b/modules/yarn/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yarn</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zeromq/pom.xml b/modules/zeromq/pom.xml
index 06f5bedaf6b97..7372db8d2c069 100644
--- a/modules/zeromq/pom.xml
+++ b/modules/zeromq/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zeromq</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index 6abdf52b52e6a..1bdd085473d35 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zookeeper</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/pom.xml b/pom.xml
index 813f6e0f8493e..d9baefda700c0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>apache-ignite</artifactId>
-    <version>2.5.1-p160-SNAPSHOT</version>
+    <version>2.5.1-p161-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <properties>

From be5ed4231c1f592f81f6f16325737ba4f292834c Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 22 Nov 2018 16:54:45 +0300
Subject: [PATCH 505/543] Revert

---
 .../internal/commandline/CommandHandler.java  | 72 +++----------------
 .../visor/verify/ValidateIndexesClosure.java  |  2 +-
 2 files changed, 12 insertions(+), 62 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 3f6652ab78769..f37a336de0870 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -18,8 +18,6 @@
 package org.apache.ignite.internal.commandline;
 
 import java.io.Console;
-import java.io.IOException;
-import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -36,7 +34,6 @@
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeTask;
@@ -69,7 +66,6 @@
 import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2;
-import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.SB;
@@ -120,7 +116,6 @@
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTask;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskResult;
-import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
@@ -665,35 +660,7 @@ private <R> R executeTask(GridClient client, Class<? extends ComputeTask<?, R>>
         return executeTaskByNameOnNode(client, taskCls.getName(), taskArgs, null);
     }
 
-    /**
-     * @param client Client.
-     * @return List of hosts.
-     */
-    private Stream<IgniteBiTuple<GridClientNode, String>> listHosts(GridClient client) throws GridClientException {
-        return client.compute()
-            .nodes(GridClientNode::connectable)
-            .stream()
-            .flatMap(node -> Stream.concat(
-                node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
-                node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
-            )
-            .map(addr -> new IgniteBiTuple<>(node, addr + ":" + node.tcpPort())));
-    }
 
-    /**
-     * @param client Client.
-     * @return List of hosts.
-     */
-    private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNode(
-        GridClient client) throws GridClientException {
-        return client.compute().nodes(GridClientNode::connectable).stream()
-            .map(node -> new IgniteBiTuple<>(node,
-                Stream.concat(
-                    node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
-                    node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
-                )
-                    .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
-    }
 
     /**
      * @param client Client
@@ -703,12 +670,7 @@ private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNod
      * @return Task result.
      * @throws GridClientException If failed to execute task.
      */
-    @SuppressWarnings("unchecked")
-    private <R> R executeTaskByNameOnNode(
-        GridClient client,
-        String taskClsName,
-        Object taskArgs,
-        UUID nodeId
+    private <R> R executeTaskByNameOnNode(GridClient client, String taskClsName, Object taskArgs, UUID nodeId
     ) throws GridClientException {
         GridClientCompute compute = client.compute();
 
@@ -728,34 +690,22 @@ private <R> R executeTaskByNameOnNode(
         GridClientNode node = null;
 
         if (nodeId == null) {
+            Collection<GridClientNode> nodes = compute.nodes(GridClientNode::connectable);
+
             // Prefer node from connect string.
-            final String cfgAddr = clientCfg.getServers().iterator().next();
+            String origAddr = clientCfg.getServers().iterator().next();
 
-            String[] parts = cfgAddr.split(":");
+            for (GridClientNode clientNode : nodes) {
+                Iterator<String> it = F.concat(clientNode.tcpAddresses().iterator(), clientNode.tcpHostNames().iterator());
 
-            if (DFLT_HOST.equals(parts[0])) {
-                InetAddress addr;
+                while (it.hasNext()) {
+                    if (origAddr.equals(it.next() + ":" + clientNode.tcpPort())) {
+                        node = clientNode;
 
-                try {
-                    addr = IgniteUtils.getLocalHost();
-                }
-                catch (IOException e) {
-                    throw new GridClientException("Can't get localhost name.", e);
+                        break;
+                    }
                 }
-
-                if (addr.isLoopbackAddress())
-                    throw new GridClientException("Can't find localhost name.");
-
-                String origAddr = addr.getHostName() + ":" + parts[1];
-
-                node = listHosts(client).filter(tuple -> origAddr.equals(tuple.get2())).findFirst().map(IgniteBiTuple::get1).orElse(null);
-
-                if (node == null)
-                    node = listHostsByClientNode(client).filter(tuple -> tuple.get2().size() == 1 && cfgAddr.equals(tuple.get2().get(0))).
-                        findFirst().map(IgniteBiTuple::get1).orElse(null);
             }
-            else
-                node = listHosts(client).filter(tuple -> cfgAddr.equals(tuple.get2())).findFirst().map(IgniteBiTuple::get1).orElse(null);
 
             // Otherwise choose random node.
             if (node == null)
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index ec02c25c3ca36..a5a7c9102ecd3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -296,7 +296,7 @@ private Map<Integer, IndexIntegrityCheckIssue> integrityCheckIndexesPartitions(S
         for (Integer grpId: grpIds) {
             final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
 
-            if (grpCtx == null) {
+            if (grpCtx == null || !grpCtx.persistenceEnabled()) {
                 integrityCheckedIndexes.incrementAndGet();
 
                 continue;

From 564c4dcae52e2856cdb2e026dc9866ddbb96b4fd Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 22 Nov 2018 19:48:07 +0300
Subject: [PATCH 506/543] Revert committed by mistake

This reverts commit be5ed42
---
 .../internal/commandline/CommandHandler.java  | 72 ++++++++++++++++---
 .../visor/verify/ValidateIndexesClosure.java  |  2 +-
 2 files changed, 62 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index f37a336de0870..3f6652ab78769 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -18,6 +18,8 @@
 package org.apache.ignite.internal.commandline;
 
 import java.io.Console;
+import java.io.IOException;
+import java.net.InetAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -34,6 +36,7 @@
 import java.util.regex.Pattern;
 import java.util.regex.PatternSyntaxException;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeTask;
@@ -66,6 +69,7 @@
 import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecord;
 import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
 import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2;
+import org.apache.ignite.internal.util.IgniteUtils;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.SB;
@@ -116,6 +120,7 @@
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTask;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorViewCacheTaskResult;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgniteProductVersion;
 import org.apache.ignite.plugin.security.SecurityCredentials;
 import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
@@ -660,7 +665,35 @@ private <R> R executeTask(GridClient client, Class<? extends ComputeTask<?, R>>
         return executeTaskByNameOnNode(client, taskCls.getName(), taskArgs, null);
     }
 
+    /**
+     * @param client Client.
+     * @return List of hosts.
+     */
+    private Stream<IgniteBiTuple<GridClientNode, String>> listHosts(GridClient client) throws GridClientException {
+        return client.compute()
+            .nodes(GridClientNode::connectable)
+            .stream()
+            .flatMap(node -> Stream.concat(
+                node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
+                node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
+            )
+            .map(addr -> new IgniteBiTuple<>(node, addr + ":" + node.tcpPort())));
+    }
 
+    /**
+     * @param client Client.
+     * @return List of hosts.
+     */
+    private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNode(
+        GridClient client) throws GridClientException {
+        return client.compute().nodes(GridClientNode::connectable).stream()
+            .map(node -> new IgniteBiTuple<>(node,
+                Stream.concat(
+                    node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
+                    node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
+                )
+                    .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
+    }
 
     /**
      * @param client Client
@@ -670,7 +703,12 @@ private <R> R executeTask(GridClient client, Class<? extends ComputeTask<?, R>>
      * @return Task result.
      * @throws GridClientException If failed to execute task.
      */
-    private <R> R executeTaskByNameOnNode(GridClient client, String taskClsName, Object taskArgs, UUID nodeId
+    @SuppressWarnings("unchecked")
+    private <R> R executeTaskByNameOnNode(
+        GridClient client,
+        String taskClsName,
+        Object taskArgs,
+        UUID nodeId
     ) throws GridClientException {
         GridClientCompute compute = client.compute();
 
@@ -690,22 +728,34 @@ private <R> R executeTaskByNameOnNode(GridClient client, String taskClsName, Obj
         GridClientNode node = null;
 
         if (nodeId == null) {
-            Collection<GridClientNode> nodes = compute.nodes(GridClientNode::connectable);
-
             // Prefer node from connect string.
-            String origAddr = clientCfg.getServers().iterator().next();
+            final String cfgAddr = clientCfg.getServers().iterator().next();
 
-            for (GridClientNode clientNode : nodes) {
-                Iterator<String> it = F.concat(clientNode.tcpAddresses().iterator(), clientNode.tcpHostNames().iterator());
+            String[] parts = cfgAddr.split(":");
 
-                while (it.hasNext()) {
-                    if (origAddr.equals(it.next() + ":" + clientNode.tcpPort())) {
-                        node = clientNode;
+            if (DFLT_HOST.equals(parts[0])) {
+                InetAddress addr;
 
-                        break;
-                    }
+                try {
+                    addr = IgniteUtils.getLocalHost();
+                }
+                catch (IOException e) {
+                    throw new GridClientException("Can't get localhost name.", e);
                 }
+
+                if (addr.isLoopbackAddress())
+                    throw new GridClientException("Can't find localhost name.");
+
+                String origAddr = addr.getHostName() + ":" + parts[1];
+
+                node = listHosts(client).filter(tuple -> origAddr.equals(tuple.get2())).findFirst().map(IgniteBiTuple::get1).orElse(null);
+
+                if (node == null)
+                    node = listHostsByClientNode(client).filter(tuple -> tuple.get2().size() == 1 && cfgAddr.equals(tuple.get2().get(0))).
+                        findFirst().map(IgniteBiTuple::get1).orElse(null);
             }
+            else
+                node = listHosts(client).filter(tuple -> cfgAddr.equals(tuple.get2())).findFirst().map(IgniteBiTuple::get1).orElse(null);
 
             // Otherwise choose random node.
             if (node == null)
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index a5a7c9102ecd3..ec02c25c3ca36 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -296,7 +296,7 @@ private Map<Integer, IndexIntegrityCheckIssue> integrityCheckIndexesPartitions(S
         for (Integer grpId: grpIds) {
             final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
 
-            if (grpCtx == null || !grpCtx.persistenceEnabled()) {
+            if (grpCtx == null) {
                 integrityCheckedIndexes.incrementAndGet();
 
                 continue;

From 8381160aeea217d69d6cc2bed202fdce50bdd06d Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Fri, 23 Nov 2018 19:11:31 +0300
Subject: [PATCH 507/543] IGNITE-10381 Fixed U.doInParallel not to terminate
 early - Fixes #5483.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../GridCachePartitionExchangeManager.java    |   4 +-
 .../GridDhtPartitionsExchangeFuture.java      |   3 +-
 .../ignite/internal/util/IgniteUtils.java     | 189 +++---------------
 ...amicCacheStartFailWithPersistenceTest.java |   3 +-
 .../internal/util/IgniteUtilsSelfTest.java    |  43 ----
 5 files changed, 35 insertions(+), 207 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index cfdda2be83534..53b2228894707 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -2792,8 +2792,8 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                             "topVer=" + exchFut.initialVersion() +
                                             ", node=" + cctx.localNodeId() + "]. " +
                                             (curTimeout <= 0 && !txRolledBack ? "Consider changing " +
-                                                    "TransactionConfiguration.txTimeoutOnPartitionMapSynchronization" +
-                                                    " to non default value to avoid this message. " : "") +
+                                            "TransactionConfiguration.txTimeoutOnPartitionMapSynchronization" +
+                                            " to non default value to avoid this message. " : "") +
                                             "Dumping pending objects that might be the cause: ");
 
                                         try {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 8f07b04db0b7b..cb2a424f22e62 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1057,7 +1057,8 @@ private ExchangeType onCacheChangeRequest(boolean crd) throws IgniteCheckedExcep
                 // This exception will be handled by init() method.
                 throw e;
 
-            U.error(log, "Failed to initialize cache(s) (will try to rollback). " + exchId, e);
+            U.error(log, "Failed to initialize cache(s) (will try to rollback) [exchId=" + exchId +
+                ", caches=" + exchActions.cacheGroupsToStart() + ']', e);
 
             exchangeLocE = new IgniteCheckedException(
                 "Failed to initialize exchange locally [locNodeId=" + cctx.localNodeId() + "]", e);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 3c4fb61922580..29abeefb20aa1 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -95,7 +95,6 @@
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Calendar;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
@@ -113,7 +112,6 @@
 import java.util.ServiceLoader;
 import java.util.Set;
 import java.util.StringTokenizer;
-import java.util.TimeZone;
 import java.util.TreeMap;
 import java.util.UUID;
 import java.util.concurrent.BrokenBarrierException;
@@ -7104,137 +7102,6 @@ private static boolean checkNextToken(StringTokenizer t, String str, String date
         }
     }
 
-    /**
-     *
-     * @param str ISO date.
-     * @return Calendar instance.
-     * @throws IgniteCheckedException Thrown in case of any errors.
-     */
-    public static Calendar parseIsoDate(String str) throws IgniteCheckedException {
-        StringTokenizer t = new StringTokenizer(str, "+-:.TZ", true);
-
-        Calendar cal = Calendar.getInstance();
-        cal.clear();
-
-        try {
-            if (t.hasMoreTokens())
-                cal.set(Calendar.YEAR, Integer.parseInt(t.nextToken()));
-            else
-                return cal;
-
-            if (checkNextToken(t, "-", str) && t.hasMoreTokens())
-                cal.set(Calendar.MONTH, Integer.parseInt(t.nextToken()) - 1);
-            else
-                return cal;
-
-            if (checkNextToken(t, "-", str) && t.hasMoreTokens())
-                cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(t.nextToken()));
-            else
-                return cal;
-
-            if (checkNextToken(t, "T", str) && t.hasMoreTokens())
-                cal.set(Calendar.HOUR_OF_DAY, Integer.parseInt(t.nextToken()));
-            else {
-                cal.set(Calendar.HOUR_OF_DAY, 0);
-                cal.set(Calendar.MINUTE, 0);
-                cal.set(Calendar.SECOND, 0);
-                cal.set(Calendar.MILLISECOND, 0);
-
-                return cal;
-            }
-
-            if (checkNextToken(t, ":", str) && t.hasMoreTokens())
-                cal.set(Calendar.MINUTE, Integer.parseInt(t.nextToken()));
-            else {
-                cal.set(Calendar.MINUTE, 0);
-                cal.set(Calendar.SECOND, 0);
-                cal.set(Calendar.MILLISECOND, 0);
-
-                return cal;
-            }
-
-            if (!t.hasMoreTokens())
-                return cal;
-
-            String tok = t.nextToken();
-
-            if (":".equals(tok)) { // Seconds.
-                if (t.hasMoreTokens()) {
-                    cal.set(Calendar.SECOND, Integer.parseInt(t.nextToken()));
-
-                    if (!t.hasMoreTokens())
-                        return cal;
-
-                    tok = t.nextToken();
-
-                    if (".".equals(tok)) {
-                        String nt = t.nextToken();
-
-                        while (nt.length() < 3)
-                            nt += "0";
-
-                        nt = nt.substring(0, 3); // Cut trailing chars.
-
-                        cal.set(Calendar.MILLISECOND, Integer.parseInt(nt));
-
-                        if (!t.hasMoreTokens())
-                            return cal;
-
-                        tok = t.nextToken();
-                    }
-                    else
-                        cal.set(Calendar.MILLISECOND, 0);
-                }
-                else
-                    throw new IgniteCheckedException("Invalid date format: " + str);
-            }
-            else {
-                cal.set(Calendar.SECOND, 0);
-                cal.set(Calendar.MILLISECOND, 0);
-            }
-
-            if (!"Z".equals(tok)) {
-                if (!"+".equals(tok) && !"-".equals(tok))
-                    throw new IgniteCheckedException("Invalid date format: " + str);
-
-                boolean plus = "+".equals(tok);
-
-                if (!t.hasMoreTokens())
-                    throw new IgniteCheckedException("Invalid date format: " + str);
-
-                tok = t.nextToken();
-
-                int tzHour;
-                int tzMin;
-
-                if (tok.length() == 4) {
-                    tzHour = Integer.parseInt(tok.substring(0, 2));
-                    tzMin = Integer.parseInt(tok.substring(2, 4));
-                }
-                else {
-                    tzHour = Integer.parseInt(tok);
-
-                    if (checkNextToken(t, ":", str) && t.hasMoreTokens())
-                        tzMin = Integer.parseInt(t.nextToken());
-                    else
-                        throw new IgniteCheckedException("Invalid date format: " + str);
-                }
-
-                if (plus)
-                    cal.set(Calendar.ZONE_OFFSET, (tzHour * 60 + tzMin) * 60 * 1000);
-                else
-                    cal.set(Calendar.ZONE_OFFSET, -(tzHour * 60 + tzMin) * 60 * 1000);
-            }
-            else
-                cal.setTimeZone(TimeZone.getTimeZone("GMT"));
-        }
-        catch (NumberFormatException ex) {
-            throw new IgniteCheckedException("Invalid date format: " + str, ex);
-        }
-
-        return cal;
-    }
-
     /**
      * Adds values to collection and returns the same collection to allow chaining.
      *
@@ -7340,20 +7207,6 @@ public static IgniteCheckedException cast(Throwable t) {
             : new IgniteCheckedException(t);
     }
 
-    /**
-     * Parses passed string with specified date.
-     *
-     * @param src String to parse.
-     * @param ptrn Pattern.
-     * @return Parsed date.
-     * @throws java.text.ParseException If exception occurs while parsing.
-     */
-    public static Date parse(String src, String ptrn) throws java.text.ParseException {
-        java.text.DateFormat format = new java.text.SimpleDateFormat(ptrn);
-
-        return format.parse(src);
-    }
-
     /**
      * Checks if class loader is an internal P2P class loader.
      *
@@ -10614,7 +10467,7 @@ public static <T, R> Collection<R> doInParallel(
 
                 batch.result(res);
             }
-            catch (IgniteCheckedException e) {
+            catch (Throwable e) {
                 batch.result(e);
             }
         }
@@ -10627,10 +10480,7 @@ public static <T, R> Collection<R> doInParallel(
                 Throwable err = batch.error;
 
                 if (err != null) {
-                    if (error == null)
-                        error = err;
-                    else
-                        error.addSuppressed(err);
+                    error = addSuppressed(error, err);
 
                     continue;
                 }
@@ -10648,16 +10498,10 @@ public static <T, R> Collection<R> doInParallel(
                 throw new IgniteInterruptedCheckedException(e);
             }
             catch (ExecutionException e) {
-                if(error == null)
-                    error = e.getCause();
-                else
-                    error.addSuppressed(e.getCause());
+                error = addSuppressed(error, e.getCause());
             }
             catch (CancellationException e) {
-                if(error == null)
-                    error = e;
-                else
-                    error.addSuppressed(e);
+                error = addSuppressed(error, e);
             }
         }
 
@@ -10677,6 +10521,31 @@ public static <T, R> Collection<R> doInParallel(
         return results;
     }
 
+    /**
+     * Utility method to add the given throwable error to the given throwable root error. If the given
+     * suppressed throwable is an {@code Error}, but the root error is not, will change the root to the {@code Error}.
+     *
+     * @param root Root error to add suppressed error to.
+     * @param err Error to add.
+     * @return New root error.
+     */
+    private static Throwable addSuppressed(Throwable root, Throwable err) {
+        assert err != null;
+
+        if (root == null)
+            return err;
+
+        if (err instanceof Error && !(root instanceof Error)) {
+            err.addSuppressed(root);
+
+            root = err;
+        }
+        else
+            root.addSuppressed(err);
+
+        return root;
+    }
+
     /**
      * The batch of tasks with a batch index in global array.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java
index 3b7bf52cf90dd..f7a700a304ec5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteDynamicCacheStartFailWithPersistenceTest.java
@@ -34,7 +34,8 @@ public class IgniteDynamicCacheStartFailWithPersistenceTest extends IgniteAbstra
         return 5 * 60 * 1000;
     }
 
-    protected boolean persistenceEnabled() {
+    /** {@inheritDoc} */
+    @Override protected boolean persistenceEnabled() {
         return true;
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
index 65b2bb324577c..71080300fbb11 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/util/IgniteUtilsSelfTest.java
@@ -38,7 +38,6 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Calendar;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -399,48 +398,6 @@ private static ComputeJob job(final Runnable r) {
         };
     }
 
-    /**
-     *
-     * @throws Exception If failed.
-     */
-    public void testParseIsoDate() throws Exception {
-        Calendar cal = U.parseIsoDate("2009-12-08T13:30:44.000Z");
-
-        assert cal.get(Calendar.YEAR) == 2009;
-        assert cal.get(Calendar.MONTH) == 11;
-        assert cal.get(Calendar.DAY_OF_MONTH) == 8;
-        assert cal.get(Calendar.HOUR_OF_DAY) == 13;
-        assert cal.get(Calendar.MINUTE) == 30;
-        assert cal.get(Calendar.SECOND) == 44;
-        assert cal.get(Calendar.MILLISECOND) == 0;
-        assert cal.get(Calendar.ZONE_OFFSET) == 0 :
-            "Unexpected value: " + cal.get(Calendar.ZONE_OFFSET);
-
-        cal = U.parseIsoDate("2009-12-08T13:30:44.000+03:00");
-
-        assert cal.get(Calendar.YEAR) == 2009;
-        assert cal.get(Calendar.MONTH) == 11;
-        assert cal.get(Calendar.DAY_OF_MONTH) == 8;
-        assert cal.get(Calendar.HOUR_OF_DAY) == 13;
-        assert cal.get(Calendar.MINUTE) == 30;
-        assert cal.get(Calendar.SECOND) == 44;
-        assert cal.get(Calendar.MILLISECOND) == 0;
-        assert cal.get(Calendar.ZONE_OFFSET) == 3 * 60 * 60 * 1000 :
-            "Unexpected value: " + cal.get(Calendar.ZONE_OFFSET);
-
-        cal = U.parseIsoDate("2009-12-08T13:30:44.000+0300");
-
-        assert cal.get(Calendar.YEAR) == 2009;
-        assert cal.get(Calendar.MONTH) == 11;
-        assert cal.get(Calendar.DAY_OF_MONTH) == 8;
-        assert cal.get(Calendar.HOUR_OF_DAY) == 13;
-        assert cal.get(Calendar.MINUTE) == 30;
-        assert cal.get(Calendar.SECOND) == 44;
-        assert cal.get(Calendar.MILLISECOND) == 0;
-        assert cal.get(Calendar.ZONE_OFFSET) == 3 * 60 * 60 * 1000 :
-            "Unexpected value: " + cal.get(Calendar.ZONE_OFFSET);
-    }
-
     /**
      * @throws Exception If test failed.
      */

From 657563e836d034364bccdf4014f9c9bbea197d1b Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Fri, 23 Nov 2018 19:36:00 +0300
Subject: [PATCH 508/543] IGNITE-10339 Skip index partition file integrity
 check for in-memory caches - Fixes #5475.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 5b656e2f3b2157200d650923e5aca800c4d61635)
---
 .../ignite/internal/commandline/CommandHandler.java       | 6 ++++--
 .../internal/visor/verify/ValidateIndexesClosure.java     | 8 +++++---
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 3f6652ab78769..cf980dab3d6d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -901,7 +901,11 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
         VisorValidateIndexesTaskResult taskRes = executeTaskByNameOnNode(
             client, VALIDATE_INDEXES_TASK, taskArg, null);
 
+        boolean errors = false;
+
         if (!F.isEmpty(taskRes.exceptions())) {
+            errors = true;
+
             log("Index validation failed on nodes:");
 
             for (Map.Entry<UUID, Exception> e : taskRes.exceptions().entrySet()) {
@@ -913,8 +917,6 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
             }
         }
 
-        boolean errors = false;
-
         for (Map.Entry<UUID, VisorValidateIndexesJobResult> nodeEntry : taskRes.results().entrySet()) {
             if (!nodeEntry.getValue().hasIssues())
                 continue;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index ec02c25c3ca36..b6909e3191e7f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -260,7 +260,8 @@ private VisorValidateIndexesJobResult call0() throws Exception {
 
                 Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get();
 
-                partResults.putAll(partRes);
+                if (!partRes.isEmpty() && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
+                    partResults.putAll(partRes);
             }
 
             for (; curIdx < procIdxFutures.size(); curIdx++) {
@@ -268,7 +269,8 @@ private VisorValidateIndexesJobResult call0() throws Exception {
 
                 Map<String, ValidateIndexesPartitionResult> idxRes = fut.get();
 
-                idxResults.putAll(idxRes);
+                if (!idxRes.isEmpty() && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
+                    idxResults.putAll(idxRes);
             }
 
             log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and "
@@ -296,7 +298,7 @@ private Map<Integer, IndexIntegrityCheckIssue> integrityCheckIndexesPartitions(S
         for (Integer grpId: grpIds) {
             final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
 
-            if (grpCtx == null) {
+            if (grpCtx == null || !grpCtx.persistenceEnabled()) {
                 integrityCheckedIndexes.incrementAndGet();
 
                 continue;

From e385b91f091da4a1882377c0c00d77e029d36d55 Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Fri, 23 Nov 2018 19:36:00 +0300
Subject: [PATCH 509/543] IGNITE-10339 Skip index partition file integrity
 check for in-memory caches - Fixes #5475.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 5b656e2f3b2157200d650923e5aca800c4d61635)
---
 .../ignite/internal/commandline/CommandHandler.java       | 6 ++++--
 .../internal/visor/verify/ValidateIndexesClosure.java     | 8 +++++---
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 3f6652ab78769..cf980dab3d6d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -901,7 +901,11 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
         VisorValidateIndexesTaskResult taskRes = executeTaskByNameOnNode(
             client, VALIDATE_INDEXES_TASK, taskArg, null);
 
+        boolean errors = false;
+
         if (!F.isEmpty(taskRes.exceptions())) {
+            errors = true;
+
             log("Index validation failed on nodes:");
 
             for (Map.Entry<UUID, Exception> e : taskRes.exceptions().entrySet()) {
@@ -913,8 +917,6 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
             }
         }
 
-        boolean errors = false;
-
         for (Map.Entry<UUID, VisorValidateIndexesJobResult> nodeEntry : taskRes.results().entrySet()) {
             if (!nodeEntry.getValue().hasIssues())
                 continue;
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index ec02c25c3ca36..b6909e3191e7f 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -260,7 +260,8 @@ private VisorValidateIndexesJobResult call0() throws Exception {
 
                 Map<PartitionKey, ValidateIndexesPartitionResult> partRes = fut.get();
 
-                partResults.putAll(partRes);
+                if (!partRes.isEmpty() && partRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
+                    partResults.putAll(partRes);
             }
 
             for (; curIdx < procIdxFutures.size(); curIdx++) {
@@ -268,7 +269,8 @@ private VisorValidateIndexesJobResult call0() throws Exception {
 
                 Map<String, ValidateIndexesPartitionResult> idxRes = fut.get();
 
-                idxResults.putAll(idxRes);
+                if (!idxRes.isEmpty() && idxRes.entrySet().stream().anyMatch(e -> !e.getValue().issues().isEmpty()))
+                    idxResults.putAll(idxRes);
             }
 
             log.warning("ValidateIndexesClosure finished: processed " + totalPartitions + " partitions and "
@@ -296,7 +298,7 @@ private Map<Integer, IndexIntegrityCheckIssue> integrityCheckIndexesPartitions(S
         for (Integer grpId: grpIds) {
             final CacheGroupContext grpCtx = ignite.context().cache().cacheGroup(grpId);
 
-            if (grpCtx == null) {
+            if (grpCtx == null || !grpCtx.persistenceEnabled()) {
                 integrityCheckedIndexes.incrementAndGet();
 
                 continue;

From c7702c245cd500e19cba13c2565d664cd85262f9 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Fri, 23 Nov 2018 20:27:50 +0300
Subject: [PATCH 510/543] IGNITE-10392 Use lastAffChangedTopVer if DiscoCache
 for the requested topVer isn't ready - Fixes #5488.

---
 .../internal/managers/discovery/GridDiscoveryManager.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 0d8d9a3a7f8d4..bd5b6b7328632 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2093,6 +2093,14 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer)
             snap.discoCache : discoCacheHist.get(topVer);
 
         if (cache == null) {
+            AffinityTopologyVersion lastAffChangedTopVer =
+                ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer);
+
+            DiscoCache lastAffChangedDiscoCache = discoCacheHist.get(lastAffChangedTopVer);
+
+            if (lastAffChangedDiscoCache != null)
+                return lastAffChangedDiscoCache;
+
             CacheGroupDescriptor desc = ctx.cache().cacheGroupDescriptors().get(grpId);
 
             throw new IgniteException("Failed to resolve nodes topology [" +

From 0f85f4aa843eaac0425b95ad2f858cd8f7db40c1 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Fri, 23 Nov 2018 20:34:08 +0300
Subject: [PATCH 511/543] IGNITE-10392 Use lastAffChangedTopVer if DiscoCache
 for the requested topVer isn't ready - Fixes #5488.

---
 .../internal/managers/discovery/GridDiscoveryManager.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 0d8d9a3a7f8d4..bd5b6b7328632 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2093,6 +2093,14 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer)
             snap.discoCache : discoCacheHist.get(topVer);
 
         if (cache == null) {
+            AffinityTopologyVersion lastAffChangedTopVer =
+                ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer);
+
+            DiscoCache lastAffChangedDiscoCache = discoCacheHist.get(lastAffChangedTopVer);
+
+            if (lastAffChangedDiscoCache != null)
+                return lastAffChangedDiscoCache;
+
             CacheGroupDescriptor desc = ctx.cache().cacheGroupDescriptors().get(grpId);
 
             throw new IgniteException("Failed to resolve nodes topology [" +

From c0b051f75b1a801e789f90acd7406a3889a32b3b Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Fri, 23 Nov 2018 20:49:07 +0300
Subject: [PATCH 512/543] Revert "IGNITE-10392 Use lastAffChangedTopVer if
 DiscoCache for the requested topVer isn't ready - Fixes #5488."

This reverts commit c7702c245cd500e19cba13c2565d664cd85262f9.
---
 .../internal/managers/discovery/GridDiscoveryManager.java | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index bd5b6b7328632..0d8d9a3a7f8d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2093,14 +2093,6 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer)
             snap.discoCache : discoCacheHist.get(topVer);
 
         if (cache == null) {
-            AffinityTopologyVersion lastAffChangedTopVer =
-                ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer);
-
-            DiscoCache lastAffChangedDiscoCache = discoCacheHist.get(lastAffChangedTopVer);
-
-            if (lastAffChangedDiscoCache != null)
-                return lastAffChangedDiscoCache;
-
             CacheGroupDescriptor desc = ctx.cache().cacheGroupDescriptors().get(grpId);
 
             throw new IgniteException("Failed to resolve nodes topology [" +

From 09c2b31dbc958e537cd826bc7076b2255433ee36 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Fri, 23 Nov 2018 20:23:16 +0300
Subject: [PATCH 513/543] IGNITE-10392 Use lasdAffChangedTopVer if DiscoCache
 for the requested topVer isn't ready - Fixes #5488.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 5c01c4199df3ba3a479d4566561335d57adffd0f)
---
 .../internal/managers/discovery/GridDiscoveryManager.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 0d8d9a3a7f8d4..bd5b6b7328632 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2093,6 +2093,14 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer)
             snap.discoCache : discoCacheHist.get(topVer);
 
         if (cache == null) {
+            AffinityTopologyVersion lastAffChangedTopVer =
+                ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer);
+
+            DiscoCache lastAffChangedDiscoCache = discoCacheHist.get(lastAffChangedTopVer);
+
+            if (lastAffChangedDiscoCache != null)
+                return lastAffChangedDiscoCache;
+
             CacheGroupDescriptor desc = ctx.cache().cacheGroupDescriptors().get(grpId);
 
             throw new IgniteException("Failed to resolve nodes topology [" +

From 62200b7739cabb4ed2edfc84d5f57204df7db50d Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Fri, 23 Nov 2018 20:50:21 +0300
Subject: [PATCH 514/543] Revert "IGNITE-10392 Use lastAffChangedTopVer if
 DiscoCache for the requested topVer isn't ready - Fixes #5488."

This reverts commit 0f85f4aa843eaac0425b95ad2f858cd8f7db40c1.
---
 .../internal/managers/discovery/GridDiscoveryManager.java | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index bd5b6b7328632..0d8d9a3a7f8d4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2093,14 +2093,6 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer)
             snap.discoCache : discoCacheHist.get(topVer);
 
         if (cache == null) {
-            AffinityTopologyVersion lastAffChangedTopVer =
-                ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer);
-
-            DiscoCache lastAffChangedDiscoCache = discoCacheHist.get(lastAffChangedTopVer);
-
-            if (lastAffChangedDiscoCache != null)
-                return lastAffChangedDiscoCache;
-
             CacheGroupDescriptor desc = ctx.cache().cacheGroupDescriptors().get(grpId);
 
             throw new IgniteException("Failed to resolve nodes topology [" +

From 653c245aa52bee6b8adaf4a2968f35e2a094fc0e Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Fri, 23 Nov 2018 20:23:16 +0300
Subject: [PATCH 515/543] IGNITE-10392 Use lasdAffChangedTopVer if DiscoCache
 for the requested topVer isn't ready - Fixes #5488.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
(cherry picked from commit 5c01c4199df3ba3a479d4566561335d57adffd0f)
---
 .../internal/managers/discovery/GridDiscoveryManager.java | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 0d8d9a3a7f8d4..bd5b6b7328632 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2093,6 +2093,14 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer)
             snap.discoCache : discoCacheHist.get(topVer);
 
         if (cache == null) {
+            AffinityTopologyVersion lastAffChangedTopVer =
+                ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer);
+
+            DiscoCache lastAffChangedDiscoCache = discoCacheHist.get(lastAffChangedTopVer);
+
+            if (lastAffChangedDiscoCache != null)
+                return lastAffChangedDiscoCache;
+
             CacheGroupDescriptor desc = ctx.cache().cacheGroupDescriptors().get(grpId);
 
             throw new IgniteException("Failed to resolve nodes topology [" +

From cea5de9803e1646cf0c55726665a9efe7336dad3 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Sat, 24 Nov 2018 10:35:00 +0300
Subject: [PATCH 516/543] IGNITE-10397 Run query.OnCacheStart() callback only
 when node has joined to topology.

---
 .../processors/cache/GridCacheProcessor.java  | 23 ++++++++-----------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index af6dc87162401..d2e676ed80655 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -2037,14 +2037,12 @@ private void prepareStartCaches(
                     cacheInfo -> {
                         GridCacheContext<?, ?> cctx = cacheContexts.get(cacheInfo);
 
-                        if (!cctx.isRecoveryMode()) {
-                            ctx.query().onCacheStart(
-                                cctx,
-                                cacheInfo.getCacheDescriptor().schema() != null
-                                    ? cacheInfo.getCacheDescriptor().schema()
-                                    : new QuerySchema()
-                            );
-                        }
+                        ctx.query().onCacheStart(
+                            cctx,
+                            cacheInfo.getCacheDescriptor().schema() != null
+                                ? cacheInfo.getCacheDescriptor().schema()
+                                : new QuerySchema()
+                        );
 
                         return null;
                     }
@@ -2094,13 +2092,12 @@ public void prepareCacheStart(
     ) throws IgniteCheckedException {
         GridCacheContext cacheCtx = prepareCacheContext(startCfg, desc, reqNearCfg, exchTopVer, disabledAfterStart);
 
+        ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema());
+
         if (cacheCtx.isRecoveryMode())
             finishRecovery(exchTopVer, cacheCtx);
-        else {
-            ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema());
-
+        else
             onCacheStarted(cacheCtx);
-        }
     }
 
     /**
@@ -2452,8 +2449,6 @@ private void onCacheStarted(GridCacheContext cacheCtx) throws IgniteCheckedExcep
 
         grp.onCacheStarted(cacheCtx);
 
-        ctx.query().onCacheStart(cacheCtx, desc.schema() != null ? desc.schema() : new QuerySchema());
-
         if (log.isInfoEnabled()) {
             log.info("Started cache in recovery mode [name=" + cfg.getName() +
                     ", id=" + cacheCtx.cacheId() +

From d6ed91696fea88f7fea25f90e0eebf497b7ca841 Mon Sep 17 00:00:00 2001
From: tledkov-gridgain <tledkov@gridgain.com>
Date: Tue, 5 Jun 2018 12:33:16 +0300
Subject: [PATCH 517/543] IGNITE-8037: SQL: keep system H2 JDBC connection
 until node is stopped. This closes #3962.

(cherry picked from commit 969be8101c5925a888dc8bbf982ef1438d9f1e0b)
---
 .../processors/query/h2/IgniteH2Indexing.java | 63 +++++++++++++++++--
 .../index/H2ConnectionLeaksSelfTest.java      | 51 +++++++++------
 2 files changed, 91 insertions(+), 23 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index fb02bd0abc92e..a848d8a7f6b03 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -376,6 +376,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
         }
     };
 
+    /** H2 JDBC connection for INFORMATION_SCHEMA. Holds H2 open until node is stopped. */
+    private Connection sysConn;
+
     /**
      * @return Kernal context.
      */
@@ -396,6 +399,26 @@ public Connection connectionForSchema(String schema) {
         }
     }
 
+    /**
+     * @return H2 JDBC connection to INFORMATION_SCHEMA.
+     */
+    private Connection systemConnection() {
+        assert Thread.holdsLock(schemaMux);
+
+        if (sysConn == null) {
+            try {
+                sysConn = DriverManager.getConnection(dbUrl);
+
+                sysConn.setSchema("INFORMATION_SCHEMA");
+            }
+            catch (SQLException e) {
+                throw new IgniteSQLException("Failed to initialize system DB connection: " + dbUrl, e);
+            }
+        }
+
+        return sysConn;
+    }
+
     /**
      * @param c Connection.
      * @param sql SQL.
@@ -564,10 +587,7 @@ private Connection connectionForThread(@Nullable String schema) throws IgniteChe
      * @throws IgniteCheckedException If failed to create db schema.
      */
     private void createSchema(String schema) throws IgniteCheckedException {
-        executeStatement("INFORMATION_SCHEMA", "CREATE SCHEMA IF NOT EXISTS " + H2Utils.withQuotes(schema));
-
-        // This method is typically called from internal Ignite threads on bootstrap, no need to cache this connection.
-        conns.remove(Thread.currentThread());
+        executeSystemStatement("CREATE SCHEMA IF NOT EXISTS " + H2Utils.withQuotes(schema));
 
         if (log.isDebugEnabled())
             log.debug("Created H2 schema for index database: " + schema);
@@ -580,7 +600,7 @@ private void createSchema(String schema) throws IgniteCheckedException {
      * @throws IgniteCheckedException If failed to create db schema.
      */
     private void dropSchema(String schema) throws IgniteCheckedException {
-        executeStatement("INFORMATION_SCHEMA", "DROP SCHEMA IF EXISTS " + H2Utils.withQuotes(schema));
+        executeSystemStatement("DROP SCHEMA IF EXISTS " + H2Utils.withQuotes(schema));
 
         if (log.isDebugEnabled())
             log.debug("Dropped H2 schema for index database: " + schema);
@@ -611,6 +631,30 @@ public void executeStatement(String schema, String sql) throws IgniteCheckedExce
         }
     }
 
+    /**
+     * Execute statement on H2 INFORMATION_SCHEMA.
+     * @param sql SQL statement.
+     */
+    public void executeSystemStatement(String sql) {
+        assert Thread.holdsLock(schemaMux);
+
+        Statement stmt = null;
+
+        try {
+            stmt = systemConnection().createStatement();
+
+            stmt.executeUpdate(sql);
+        }
+        catch (SQLException e) {
+            onSqlException();
+
+            throw new IgniteSQLException("Failed to execute statement: " + sql, e);
+        }
+        finally {
+            U.close(stmt, log);
+        }
+    }
+
     /**
      * Binds object to prepared statement.
      *
@@ -2789,6 +2833,15 @@ private void createSqlFunctions(String schema, Class<?>[] clss) throws IgniteChe
 
         if (log.isDebugEnabled())
             log.debug("Cache query index stopped.");
+
+        // Close system H2 connection to INFORMATION_SCHEMA
+        synchronized (schemaMux) {
+            if (sysConn != null) {
+                U.close(sysConn, log);
+
+                sysConn = null;
+            }
+        }
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2ConnectionLeaksSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2ConnectionLeaksSelfTest.java
index 417c1f320b978..cec3a90e1a2b9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2ConnectionLeaksSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2ConnectionLeaksSelfTest.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -98,20 +99,7 @@ public void testConnectionLeaks() throws Exception {
 
         latch.await();
 
-        boolean res = GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override public boolean apply() {
-                for (int i = 0; i < NODE_CNT; i++) {
-                    Map<Thread, Connection> conns = perThreadConnections(i);
-
-                    if (conns.isEmpty())
-                        return false;
-                }
-
-                return true;
-            }
-        }, 5000);
-
-        assert res;
+        checkConnectionLeaks();
     }
 
     /**
@@ -148,14 +136,41 @@ public void testConnectionLeaksOnSqlException() throws Exception {
         try {
             latch.await();
 
+            checkConnectionLeaks();
+        }
+        finally {
+            latch2.countDown();
+        }
+    }
+
+    /**
+     * @throws Exception On error.
+     */
+    private void checkConnectionLeaks() throws Exception {
+        boolean notLeak = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                for (int i = 0; i < NODE_CNT; i++) {
+                    Map<Thread, Connection> conns = perThreadConnections(i);
+
+                    for(Thread t : conns.keySet()) {
+                        if (!t.isAlive())
+                            return false;
+                    }
+                }
+
+                return true;
+            }
+        }, 5000);
+
+        if (!notLeak) {
             for (int i = 0; i < NODE_CNT; i++) {
                 Map<Thread, Connection> conns = perThreadConnections(i);
 
-                assertTrue(conns.isEmpty());
+                for(Thread t : conns.keySet())
+                    log.error("+++ Connection is not closed for thread: " + t.getName());
             }
-        }
-        finally {
-            latch2.countDown();
+
+            fail("H2 JDBC connections leak detected. See the log above.");
         }
     }
 

From 2a41d7808e426faf7971f26b091f4adcc748915d Mon Sep 17 00:00:00 2001
From: tledkov-gridgain <tledkov@gridgain.com>
Date: Tue, 5 Jun 2018 12:33:16 +0300
Subject: [PATCH 518/543] IGNITE-8037: SQL: keep system H2 JDBC connection
 until node is stopped. This closes #3962.

(cherry picked from commit 969be8101c5925a888dc8bbf982ef1438d9f1e0b)
---
 .../processors/query/h2/IgniteH2Indexing.java | 63 +++++++++++++++++--
 .../index/H2ConnectionLeaksSelfTest.java      | 51 +++++++++------
 2 files changed, 91 insertions(+), 23 deletions(-)

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index fb02bd0abc92e..a848d8a7f6b03 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -376,6 +376,9 @@ public class IgniteH2Indexing implements GridQueryIndexing {
         }
     };
 
+    /** H2 JDBC connection for INFORMATION_SCHEMA. Holds H2 open until node is stopped. */
+    private Connection sysConn;
+
     /**
      * @return Kernal context.
      */
@@ -396,6 +399,26 @@ public Connection connectionForSchema(String schema) {
         }
     }
 
+    /**
+     * @return H2 JDBC connection to INFORMATION_SCHEMA.
+     */
+    private Connection systemConnection() {
+        assert Thread.holdsLock(schemaMux);
+
+        if (sysConn == null) {
+            try {
+                sysConn = DriverManager.getConnection(dbUrl);
+
+                sysConn.setSchema("INFORMATION_SCHEMA");
+            }
+            catch (SQLException e) {
+                throw new IgniteSQLException("Failed to initialize system DB connection: " + dbUrl, e);
+            }
+        }
+
+        return sysConn;
+    }
+
     /**
      * @param c Connection.
      * @param sql SQL.
@@ -564,10 +587,7 @@ private Connection connectionForThread(@Nullable String schema) throws IgniteChe
      * @throws IgniteCheckedException If failed to create db schema.
      */
     private void createSchema(String schema) throws IgniteCheckedException {
-        executeStatement("INFORMATION_SCHEMA", "CREATE SCHEMA IF NOT EXISTS " + H2Utils.withQuotes(schema));
-
-        // This method is typically called from internal Ignite threads on bootstrap, no need to cache this connection.
-        conns.remove(Thread.currentThread());
+        executeSystemStatement("CREATE SCHEMA IF NOT EXISTS " + H2Utils.withQuotes(schema));
 
         if (log.isDebugEnabled())
             log.debug("Created H2 schema for index database: " + schema);
@@ -580,7 +600,7 @@ private void createSchema(String schema) throws IgniteCheckedException {
      * @throws IgniteCheckedException If failed to create db schema.
      */
     private void dropSchema(String schema) throws IgniteCheckedException {
-        executeStatement("INFORMATION_SCHEMA", "DROP SCHEMA IF EXISTS " + H2Utils.withQuotes(schema));
+        executeSystemStatement("DROP SCHEMA IF EXISTS " + H2Utils.withQuotes(schema));
 
         if (log.isDebugEnabled())
             log.debug("Dropped H2 schema for index database: " + schema);
@@ -611,6 +631,30 @@ public void executeStatement(String schema, String sql) throws IgniteCheckedExce
         }
     }
 
+    /**
+     * Execute statement on H2 INFORMATION_SCHEMA.
+     * @param sql SQL statement.
+     */
+    public void executeSystemStatement(String sql) {
+        assert Thread.holdsLock(schemaMux);
+
+        Statement stmt = null;
+
+        try {
+            stmt = systemConnection().createStatement();
+
+            stmt.executeUpdate(sql);
+        }
+        catch (SQLException e) {
+            onSqlException();
+
+            throw new IgniteSQLException("Failed to execute statement: " + sql, e);
+        }
+        finally {
+            U.close(stmt, log);
+        }
+    }
+
     /**
      * Binds object to prepared statement.
      *
@@ -2789,6 +2833,15 @@ private void createSqlFunctions(String schema, Class<?>[] clss) throws IgniteChe
 
         if (log.isDebugEnabled())
             log.debug("Cache query index stopped.");
+
+        // Close system H2 connection to INFORMATION_SCHEMA
+        synchronized (schemaMux) {
+            if (sysConn != null) {
+                U.close(sysConn, log);
+
+                sysConn = null;
+            }
+        }
     }
 
     /**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2ConnectionLeaksSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2ConnectionLeaksSelfTest.java
index 417c1f320b978..cec3a90e1a2b9 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2ConnectionLeaksSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2ConnectionLeaksSelfTest.java
@@ -25,6 +25,7 @@
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
 import org.apache.ignite.internal.util.lang.GridAbsPredicate;
 import org.apache.ignite.testframework.GridTestUtils;
@@ -98,20 +99,7 @@ public void testConnectionLeaks() throws Exception {
 
         latch.await();
 
-        boolean res = GridTestUtils.waitForCondition(new GridAbsPredicate() {
-            @Override public boolean apply() {
-                for (int i = 0; i < NODE_CNT; i++) {
-                    Map<Thread, Connection> conns = perThreadConnections(i);
-
-                    if (conns.isEmpty())
-                        return false;
-                }
-
-                return true;
-            }
-        }, 5000);
-
-        assert res;
+        checkConnectionLeaks();
     }
 
     /**
@@ -148,14 +136,41 @@ public void testConnectionLeaksOnSqlException() throws Exception {
         try {
             latch.await();
 
+            checkConnectionLeaks();
+        }
+        finally {
+            latch2.countDown();
+        }
+    }
+
+    /**
+     * @throws Exception On error.
+     */
+    private void checkConnectionLeaks() throws Exception {
+        boolean notLeak = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+            @Override public boolean apply() {
+                for (int i = 0; i < NODE_CNT; i++) {
+                    Map<Thread, Connection> conns = perThreadConnections(i);
+
+                    for(Thread t : conns.keySet()) {
+                        if (!t.isAlive())
+                            return false;
+                    }
+                }
+
+                return true;
+            }
+        }, 5000);
+
+        if (!notLeak) {
             for (int i = 0; i < NODE_CNT; i++) {
                 Map<Thread, Connection> conns = perThreadConnections(i);
 
-                assertTrue(conns.isEmpty());
+                for(Thread t : conns.keySet())
+                    log.error("+++ Connection is not closed for thread: " + t.getName());
             }
-        }
-        finally {
-            latch2.countDown();
+
+            fail("H2 JDBC connections leak detected. See the log above.");
         }
     }
 

From 3ddeeff52e5c12e3c096c57b277cf9f5735b10a0 Mon Sep 17 00:00:00 2001
From: Pavel Kovalenko <jokserfn@gmail.com>
Date: Wed, 28 Nov 2018 16:39:46 +0300
Subject: [PATCH 519/543] IGNITE-10298 Cover possible deadlock in case of
 caches start and frequent checkpoints. - Fixes #5517.

Signed-off-by: Pavel Kovalenko <jokserfn@gmail.com>
(cherry picked from commit fe8c8cc5816872ba58bbd2401698b9e847788ea0)
---
 .../GridCacheDatabaseSharedManager.java       |  18 +-
 ...sCacheStartStopWithFreqCheckpointTest.java | 177 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite2.java       |   3 +
 3 files changed, 197 insertions(+), 1 deletion(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheStartStopWithFreqCheckpointTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index ef3c81a4def24..6011182e6d29b 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -257,6 +257,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
     /** Checkpoint thread. Needs to be volatile because it is created in exchange worker. */
     private volatile Checkpointer checkpointer;
 
+    /** Checkpointer thread instance. */
+    private volatile IgniteThread checkpointerThread;
+
     /** For testing only. */
     private volatile boolean checkpointsEnabled = true;
 
@@ -409,6 +412,15 @@ public Checkpointer getCheckpointer() {
         return checkpointer;
     }
 
+    /**
+     * For test use only.
+     *
+     * @return Checkpointer thread instance.
+     */
+    public IgniteThread checkpointerThread() {
+        return checkpointerThread;
+    }
+
     /**
      * For test use only.
      */
@@ -2047,7 +2059,11 @@ private void restorePartitionStates(
     @Override public void onStateRestored(AffinityTopologyVersion topVer) throws IgniteCheckedException {
         long time = System.currentTimeMillis();
 
-        new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
+        IgniteThread cpThread = new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer);
+
+        cpThread.start();
+
+        checkpointerThread = cpThread;
 
         CheckpointProgressSnapshot chp = checkpointer.wakeupForCheckpoint(0, "node started");
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheStartStopWithFreqCheckpointTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheStartStopWithFreqCheckpointTest.java
new file mode 100644
index 0000000000000..8e00d8871dc36
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheStartStopWithFreqCheckpointTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class IgnitePdsCacheStartStopWithFreqCheckpointTest extends GridCommonAbstractTest {
+    /** Caches. */
+    private static final int CACHES = 10;
+
+    /** Cache name. */
+    private static final String CACHE_NAME = "test";
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        cfg.setConsistentId(igniteInstanceName);
+
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+            .setWalMode(WALMode.LOG_ONLY)
+            .setCheckpointFrequency(1000)
+            .setDefaultDataRegionConfiguration(
+                new DataRegionConfiguration()
+                    .setMaxSize(512 * 1024 * 1024)
+                    .setPersistenceEnabled(true)
+            );
+
+        cfg.setDataStorageConfiguration(dsCfg);
+
+        CacheConfiguration[] ccfgs = new CacheConfiguration[CACHES];
+
+        for (int i = 0; i < ccfgs.length; i++)
+            ccfgs[i] = cacheConfiguration(i);
+
+        cfg.setCacheConfiguration(ccfgs);
+
+        return cfg;
+    }
+
+    /** {@inheritDoc} */
+    private CacheConfiguration cacheConfiguration(int cacheIdx) {
+        return new CacheConfiguration(CACHE_NAME + cacheIdx)
+            .setCacheMode(CacheMode.REPLICATED)
+            .setBackups(0)
+            .setRebalanceMode(CacheRebalanceMode.NONE);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /**
+     * Test checkpoint deadlock during caches start/stop process and frequent checkpoints is set.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCheckpointDeadlock() throws Exception {
+        IgniteEx crd = startGrid(0);
+
+        crd.cluster().active(true);
+
+        for (int cacheId = 0; cacheId < CACHES; cacheId++) {
+            IgniteCache<Object, Object> cache = crd.getOrCreateCache(CACHE_NAME + cacheId);
+
+            for (int key = 0; key < 4096; key++)
+                cache.put(key, key);
+        }
+
+        forceCheckpoint();
+
+        final AtomicBoolean stopFlag = new AtomicBoolean();
+
+        IgniteInternalFuture<?> cacheStartStopFut = GridTestUtils.runAsync(() -> {
+            while (!stopFlag.get()) {
+                List<String> cacheNames = new ArrayList<>();
+                for (int i = 0; i < CACHES / 2; i++)
+                    cacheNames.add(CACHE_NAME + i);
+
+                try {
+                    // Stop cache without destroy.
+                    crd.context().cache().dynamicDestroyCaches(cacheNames, false, false, false).get();
+                }
+                catch (IgniteCheckedException e) {
+                    throw new IgniteException("Failed to destroy cache", e);
+                }
+
+                List<CacheConfiguration> cachesToStart = new ArrayList<>();
+                for (int i = 0; i < CACHES / 2; i++)
+                    cachesToStart.add(cacheConfiguration(i));
+
+                crd.getOrCreateCaches(cachesToStart);
+            }
+        });
+
+        U.sleep(60_000);
+
+        log.info("Stopping caches start/stop process.");
+
+        stopFlag.set(true);
+
+        try {
+            cacheStartStopFut.get(30, TimeUnit.SECONDS);
+        }
+        catch (IgniteFutureTimeoutCheckedException e) {
+            U.dumpThreads(log);
+
+            log.warning("Caches start/stop future hangs. Interrupting checkpointer...");
+
+            interruptCheckpointer(crd);
+
+            // Should succeed.
+            cacheStartStopFut.get();
+
+            Assert.assertTrue("Checkpoint and exchange is probably in deadlock (see thread dump above for details).", false);
+        }
+    }
+
+    /**
+     * Interrupts checkpoint thread for given node.
+     *
+     * @param node Node.
+     */
+    private void interruptCheckpointer(IgniteEx node) {
+        GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager) node.context().cache().context().database();
+
+        dbMgr.checkpointerThread().interrupt();
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 1f2effbd141c5..764d902060a92 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteDataStorageMetricsSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCacheStartStopWithFreqCheckpointTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsCorruptedStoreTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsExchangeDuringCheckpointTest;
@@ -95,6 +96,8 @@ public static TestSuite suite() {
         suite.addTestSuite(ClientAffinityAssignmentWithBaselineTest.class);
         suite.addTestSuite(IgniteAbsentEvictionNodeOutOfBaselineTest.class);
 
+        suite.addTestSuite(IgnitePdsCacheStartStopWithFreqCheckpointTest.class);
+
         return suite;
     }
 

From 8e491777ffee28fd38fa620227072e6054bde6c7 Mon Sep 17 00:00:00 2001
From: Denis Mekhanikov <dmekhanikov@gmail.com>
Date: Thu, 19 Jul 2018 16:11:38 +0300
Subject: [PATCH 520/543] IGNITE-8922 Fix for discovery message delivery
 guarantee can be violated - Fixes #4349.

---
 .../discovery/CustomMessageWrapper.java       |   2 +-
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  21 +-
 ...cpDiscoveryPendingMessageDeliveryTest.java | 280 ++++++++++++++++++
 .../IgniteSpiDiscoverySelfTestSuite.java      |   3 +
 4 files changed, 301 insertions(+), 5 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
index 4b6b7a2cfabf7..c7feba3cc22bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/CustomMessageWrapper.java
@@ -33,7 +33,7 @@ public class CustomMessageWrapper implements DiscoverySpiCustomMessage {
     /**
      * @param delegate Delegate.
      */
-    CustomMessageWrapper(DiscoveryCustomMessage delegate) {
+    public CustomMessageWrapper(DiscoveryCustomMessage delegate) {
         this.delegate = delegate;
     }
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index c07d0ce3af30d..a1fba58265106 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -2400,12 +2400,22 @@ void add(TcpDiscoveryAbstractMessage msg) {
             msgs.add(new PendingMessage(msg));
 
             while (msgs.size() > MAX) {
-                PendingMessage polled = msgs.poll();
+                PendingMessage queueHead = msgs.peek();
 
-                assert polled != null;
+                assert queueHead != null;
 
-                if (polled.id.equals(discardId))
+                if (queueHead.customMsg && customDiscardId != null) {
+                    if (queueHead.id.equals(customDiscardId))
+                        customDiscardId = null;
+                }
+                else if (!queueHead.customMsg && discardId != null) {
+                    if (queueHead.id.equals(discardId))
+                        discardId = null;
+                }
+                else
                     break;
+
+                msgs.poll();
             }
         }
 
@@ -5348,8 +5358,11 @@ private void processCustomMessage(TcpDiscoveryCustomEventMessage msg, boolean wa
 
                         if (sendMessageToRemotes(msg))
                             sendMessageAcrossRing(msg);
-                        else
+                        else {
+                            registerPendingMessage(msg);
+
                             processCustomMessage(msg, waitForNotification);
+                        }
                     }
 
                     msg.message(null, msg.messageBytes());
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java
new file mode 100644
index 0000000000000..9b3dfeea2e93d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.tcp;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.util.Set;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
+import org.apache.ignite.internal.managers.discovery.DiscoCache;
+import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.eclipse.jetty.util.ConcurrentHashSet;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class TcpDiscoveryPendingMessageDeliveryTest extends GridCommonAbstractTest {
+    /** */
+    private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+    /** */
+    private volatile boolean blockMsgs;
+
+    /** */
+    private Set<TcpDiscoveryAbstractMessage> receivedEnsuredMsgs;
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        blockMsgs = false;
+        receivedEnsuredMsgs = new ConcurrentHashSet<>();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        stopAllGrids();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi disco;
+
+        if (igniteInstanceName.startsWith("victim"))
+            disco = new DyingDiscoverySpi();
+        else if (igniteInstanceName.startsWith("listener"))
+            disco = new ListeningDiscoverySpi();
+        else
+            disco = new TcpDiscoverySpi();
+
+        disco.setIpFinder(IP_FINDER);
+        cfg.setDiscoverySpi(disco);
+
+        return cfg;
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testPendingMessagesOverflow() throws Exception {
+        Ignite coord = startGrid("coordinator");
+        TcpDiscoverySpi coordDisco = (TcpDiscoverySpi)coord.configuration().getDiscoverySpi();
+
+        Set<TcpDiscoveryAbstractMessage> sentEnsuredMsgs = new ConcurrentHashSet<>();
+        coordDisco.addSendMessageListener(msg -> {
+            if (coordDisco.ensured(msg))
+                sentEnsuredMsgs.add(msg);
+        });
+
+        // Victim doesn't send acknowledges, so we need an intermediate node to accept messages,
+        // so the coordinator could mark them as pending.
+        Ignite mediator = startGrid("mediator");
+
+        Ignite victim = startGrid("victim");
+
+        startGrid("listener");
+
+        sentEnsuredMsgs.clear();
+        receivedEnsuredMsgs.clear();
+
+        // Initial custom message will travel across the ring and will be discarded.
+        sendDummyCustomMessage(coordDisco, IgniteUuid.randomUuid());
+
+        assertTrue("Sent: " + sentEnsuredMsgs + "; received: " + receivedEnsuredMsgs,
+            GridTestUtils.waitForCondition(() -> {
+                log.info("Waiting for messages delivery");
+
+                return receivedEnsuredMsgs.equals(sentEnsuredMsgs);
+            }, 10000));
+
+        blockMsgs = true;
+
+        log.info("Sending dummy custom messages");
+
+        // Non-discarded messages shouldn't be dropped from the queue.
+        int msgsNum = 2000;
+
+        for (int i = 0; i < msgsNum; i++)
+            sendDummyCustomMessage(coordDisco, IgniteUuid.randomUuid());
+
+        mediator.close();
+        victim.close();
+
+        assertTrue("Sent: " + sentEnsuredMsgs + "; received: " + receivedEnsuredMsgs,
+            GridTestUtils.waitForCondition(() -> {
+                log.info("Waiting for messages delivery");
+
+                return receivedEnsuredMsgs.equals(sentEnsuredMsgs);
+            }, 10000));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testCustomMessageInSingletonCluster() throws Exception {
+        Ignite coord = startGrid("coordinator");
+        TcpDiscoverySpi coordDisco = (TcpDiscoverySpi)coord.configuration().getDiscoverySpi();
+
+        Set<TcpDiscoveryAbstractMessage> sentEnsuredMsgs = new ConcurrentHashSet<>();
+        coordDisco.addSendMessageListener(msg -> {
+            if (coordDisco.ensured(msg))
+                sentEnsuredMsgs.add(msg);
+        });
+
+        // Custom message on a singleton cluster shouldn't break consistency of PendingMessages.
+        sendDummyCustomMessage(coordDisco, IgniteUuid.randomUuid());
+
+        // Victim doesn't send acknowledges, so we need an intermediate node to accept messages,
+        // so the coordinator could mark them as pending.
+        Ignite mediator = startGrid("mediator");
+
+        Ignite victim = startGrid("victim");
+
+        startGrid("listener");
+
+        sentEnsuredMsgs.clear();
+        receivedEnsuredMsgs.clear();
+
+        blockMsgs = true;
+
+        log.info("Sending dummy custom messages");
+
+        // Non-discarded messages shouldn't be dropped from the queue.
+        int msgsNum = 100;
+
+        for (int i = 0; i < msgsNum; i++)
+            sendDummyCustomMessage(coordDisco, IgniteUuid.randomUuid());
+
+        mediator.close();
+        victim.close();
+
+        assertTrue("Sent: " + sentEnsuredMsgs + "; received: " + receivedEnsuredMsgs,
+            GridTestUtils.waitForCondition(() -> {
+                log.info("Waiting for messages delivery");
+
+                return receivedEnsuredMsgs.equals(sentEnsuredMsgs);
+            }, 10000));
+    }
+
+    /**
+     * @param disco Discovery SPI.
+     * @param id Message id.
+     */
+    private void sendDummyCustomMessage(TcpDiscoverySpi disco, IgniteUuid id) {
+        disco.sendCustomEvent(new CustomMessageWrapper(new DummyCustomDiscoveryMessage(id)));
+    }
+
+    /**
+     * Discovery SPI, that makes a node stop sending messages when {@code blockMsgs} is set to {@code true}.
+     */
+    private class DyingDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg, byte[] data,
+            long timeout) throws IOException {
+            if (!blockMsgs)
+                super.writeToSocket(sock, msg, data, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (!blockMsgs)
+                super.writeToSocket(sock, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(Socket sock, OutputStream out, TcpDiscoveryAbstractMessage msg,
+            long timeout) throws IOException, IgniteCheckedException {
+            if (!blockMsgs)
+                super.writeToSocket(sock, out, msg, timeout);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected void writeToSocket(TcpDiscoveryAbstractMessage msg, Socket sock, int res,
+            long timeout) throws IOException {
+            if (!blockMsgs)
+                super.writeToSocket(msg, sock, res, timeout);
+        }
+    }
+
+    /**
+     *
+     */
+    private class ListeningDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void startMessageProcess(TcpDiscoveryAbstractMessage msg) {
+            if (ensured(msg))
+                receivedEnsuredMsgs.add(msg);
+        }
+    }
+
+    /**
+     *
+     */
+    private static class DummyCustomDiscoveryMessage implements DiscoveryCustomMessage {
+        /** */
+        private final IgniteUuid id;
+
+        /**
+         * @param id Message id.
+         */
+        DummyCustomDiscoveryMessage(IgniteUuid id) {
+            this.id = id;
+        }
+
+        /** {@inheritDoc} */
+        @Override public IgniteUuid id() {
+            return id;
+        }
+
+        /** {@inheritDoc} */
+        @Nullable @Override public DiscoveryCustomMessage ackMessage() {
+            return null;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean isMutable() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean stopProcess() {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public DiscoCache createDiscoCache(GridDiscoveryManager mgr, AffinityTopologyVersion topVer,
+            DiscoCache discoCache) {
+            throw new UnsupportedOperationException();
+        }
+    }
+}
+
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
index 230fea5794574..ec8ab0b795592 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteSpiDiscoverySelfTestSuite.java
@@ -35,6 +35,7 @@
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeAttributesUpdateOnReconnectTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeConfigConsistentIdSelfTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryNodeConsistentIdSelfTest;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryPendingMessageDeliveryTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoveryRestartTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySegmentationPolicyTest;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySelfTest;
@@ -129,6 +130,8 @@ public static TestSuite suite() throws Exception {
 
         suite.addTest(new TestSuite(FilterDataForClientNodeDiscoveryTest.class));
 
+        suite.addTest(new TestSuite(TcpDiscoveryPendingMessageDeliveryTest.class));
+
         return suite;
     }
 }

From 1644299d463858235c584a8a0443d181af534558 Mon Sep 17 00:00:00 2001
From: Andrey Kuznetsov <stkuzma@gmail.com>
Date: Thu, 29 Nov 2018 14:54:43 +0300
Subject: [PATCH 521/543] IGNITE-10079 Fiexd WAL segments compression bug when
 FileWriteAheadLogManager return invalid lastCompactedSegment

Signed-off-by: Andrey Gura <agura@apache.org>
(cherry picked from commit f6edfd0550412a8b0bbad283adab49695849ddd1)
---
 .../wal/FileWriteAheadLogManager.java          | 18 ++++++++++--------
 .../wal/aware/SegmentArchivedStorage.java      | 16 ++++++++++++++++
 .../persistence/wal/aware/SegmentAware.java    | 13 ++++++++-----
 .../wal/aware/SegmentCompressStorage.java      |  7 ++++++-
 .../wal/aware/SegmentAwareTest.java            | 10 ++--------
 5 files changed, 42 insertions(+), 22 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index a250b471c016f..1dcaf1ec00df5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -1909,7 +1909,13 @@ private long tryReserveNextSegmentOrWait() throws IgniteInterruptedCheckedExcept
 
             boolean reserved = reserve(new FileWALPointer(segmentToCompress, 0, 0));
 
-            return reserved ? segmentToCompress : -1;
+            if (reserved)
+                return segmentToCompress;
+            else {
+                segmentAware.onSegmentCompressed(segmentToCompress);
+
+                return -1;
+            }
         }
 
         /** {@inheritDoc} */
@@ -1923,9 +1929,7 @@ private void body0() {
                 long segIdx = -1L;
 
                 try {
-                    segIdx = tryReserveNextSegmentOrWait();
-
-                    if (segIdx <= segmentAware.lastCompressedIdx())
+                    if ((segIdx = tryReserveNextSegmentOrWait()) == -1)
                         continue;
 
                     deleteObsoleteRawSegments();
@@ -1944,10 +1948,8 @@ private void body0() {
 
                     Files.move(tmpZip.toPath(), zip.toPath());
 
-                    if (mode != WALMode.NONE) {
-                        try (FileIO f0 = ioFactory.create(zip, CREATE, READ, WRITE)) {
-                            f0.force();
-                        }
+                    try (FileIO f0 = ioFactory.create(zip, CREATE, READ, WRITE)) {
+                        f0.force();
                     }
 
                     segmentAware.onSegmentCompressed(segIdx);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
index c526ae11e6feb..e31628f40abbf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentArchivedStorage.java
@@ -34,6 +34,8 @@ class SegmentArchivedStorage extends SegmentObservable {
      * no segments archived.
      */
     private volatile long lastAbsArchivedIdx = -1;
+    /** Latest truncated segment. */
+    private volatile long lastTruncatedArchiveIdx = -1;
 
     /**
      * @param segmentLockStorage Protects WAL work segments from moving.
@@ -136,4 +138,18 @@ private void checkInterrupted() throws IgniteInterruptedCheckedException {
     private synchronized void onSegmentUnlocked(long segmentId) {
         notifyAll();
     }
+
+    /**
+     * @param lastTruncatedArchiveIdx Last truncated segment.
+     */
+    void lastTruncatedArchiveIdx(long lastTruncatedArchiveIdx) {
+        this.lastTruncatedArchiveIdx = lastTruncatedArchiveIdx;
+    }
+
+    /**
+     * @return Last truncated segment.
+     */
+    long lastTruncatedArchiveIdx() {
+        return lastTruncatedArchiveIdx;
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
index e46d93f015947..a14f0cac3a325 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
@@ -27,8 +27,6 @@
  * Holder of actual information of latest manipulation on WAL segments.
  */
 public class SegmentAware {
-    /** Latest truncated segment. */
-    private volatile long lastTruncatedArchiveIdx = -1L;
     /** Segment reservations storage: Protects WAL segments from deletion during WAL log cleanup. */
     private final SegmentReservationStorage reservationStorage = new SegmentReservationStorage();
     /** Lock on segment protects from archiving segment. */
@@ -106,7 +104,12 @@ public void awaitSegmentArchived(long awaitIdx) throws IgniteInterruptedCheckedE
      * there's no segment to archive right now.
      */
     public long waitNextSegmentToCompress() throws IgniteInterruptedCheckedException {
-        return Math.max(segmentCompressStorage.nextSegmentToCompressOrWait(), lastTruncatedArchiveIdx + 1);
+        long idx;
+
+        while ((idx = segmentCompressStorage.nextSegmentToCompressOrWait()) <= lastTruncatedArchiveIdx())
+            onSegmentCompressed(idx);
+
+        return idx;
     }
 
     /**
@@ -152,14 +155,14 @@ public void curAbsWalIdx(long curAbsWalIdx) {
      * @param lastTruncatedArchiveIdx Last truncated segment;
      */
     public void lastTruncatedArchiveIdx(long lastTruncatedArchiveIdx) {
-        this.lastTruncatedArchiveIdx = lastTruncatedArchiveIdx;
+        segmentArchivedStorage.lastTruncatedArchiveIdx(lastTruncatedArchiveIdx);
     }
 
     /**
      * @return Last truncated segment.
      */
     public long lastTruncatedArchiveIdx() {
-        return lastTruncatedArchiveIdx;
+        return segmentArchivedStorage.lastTruncatedArchiveIdx();
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
index 174fb46bb6070..95d4f4a770c39 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
@@ -94,6 +94,9 @@ synchronized void onSegmentCompressed(long compressedIdx) {
             this.lastCompressedIdx = Math.min(lastMaxCompressedIdx, compressingSegments.get(0) - 1);
         else
             this.lastCompressedIdx = lastMaxCompressedIdx;
+
+        if (compressedIdx > lastEnqueuedToCompressIdx)
+            lastEnqueuedToCompressIdx = compressedIdx;
     }
 
     /**
@@ -120,9 +123,11 @@ synchronized long nextSegmentToCompressOrWait() throws IgniteInterruptedCheckedE
 
         Long idx = segmentsToCompress.poll();
 
+        assert idx != null;
+
         compressingSegments.add(idx);
 
-        return idx == null ? -1L : idx;
+        return idx;
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
index 08693564768c6..d651e01595f56 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
@@ -438,18 +438,12 @@ public void testFinishWaitSegmentToCompress_WhenMarkLastArchivedSegment() throws
      * Next segment for compress based on truncated archive idx.
      */
     public void testCorrectCalculateNextCompressSegment() throws IgniteCheckedException, InterruptedException {
-        //given: thread which awaited segment.
         SegmentAware aware = new SegmentAware(10, true);
 
-        aware.onSegmentCompressed(5);
         aware.setLastArchivedAbsoluteIndex(6);
-        aware.lastTruncatedArchiveIdx(7);
-
-        //when:
-        long segmentToCompress = aware.waitNextSegmentToCompress();
 
-        //then: segment to compress greater than truncated archive idx
-        assertEquals(8, segmentToCompress);
+        for (int exp = 0; exp <= 6; exp++)
+            assertEquals(exp, aware.waitNextSegmentToCompress());
     }
 
     /**

From baa8cc2b43d06f704346a7e917e0048d68b362eb Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Sat, 1 Dec 2018 00:53:54 +0300
Subject: [PATCH 522/543] IGNITE-10427 Fix
 GridClusterStateProcessor#changeGlobalState0() should wrap future before
 sending ChangeGlobalStateMessage - Fixes #5513.

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
(cherry picked from commit 27b88d2b4d6efa91906585cc70772bd9c640950b)
---
 .../cluster/GridClusterStateProcessor.java           | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 13694384712cb..f4f762a76f2b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -903,15 +903,19 @@ private IgniteInternalFuture<?> changeGlobalState0(final boolean activate,
             forceChangeBaselineTopology,
             System.currentTimeMillis());
 
+        IgniteInternalFuture<?> resFut = wrapStateChangeFuture(startedFut, msg);
+
         try {
             if (log.isInfoEnabled())
                 U.log(log, "Sending " + prettyStr(activate) + " request with BaselineTopology " + blt);
 
             ctx.discovery().sendCustomEvent(msg);
 
-            if (ctx.isStopping())
-                startedFut.onDone(new IgniteCheckedException("Failed to execute " + prettyStr(activate) + " request, " +
-                    "node is stopping."));
+            if (ctx.isStopping()) {
+                String errMsg = "Failed to execute " + prettyStr(activate) + " request, node is stopping.";
+
+                startedFut.onDone(new IgniteCheckedException(errMsg));
+            }
         }
         catch (IgniteCheckedException e) {
             U.error(log, "Failed to send global state change request: " + activate, e);
@@ -919,7 +923,7 @@ private IgniteInternalFuture<?> changeGlobalState0(final boolean activate,
             startedFut.onDone(e);
         }
 
-        return wrapStateChangeFuture(startedFut, msg);
+        return resFut;
     }
 
     /** {@inheritDoc} */

From eff21cb6aa24f93944037c7ec9d51316bcd31521 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Fri, 30 Nov 2018 20:16:19 +0300
Subject: [PATCH 523/543] IGNITE-10172 Added a system property to disable cache
 metrics sending across discovery - Fixes #5416.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../apache/ignite/IgniteSystemProperties.java |  10 ++
 .../discovery/GridDiscoveryManager.java       |   8 ++
 .../CacheMetricsForClusterGroupSelfTest.java  | 106 +++++++++++++++---
 3 files changed, 106 insertions(+), 18 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index b9408ef54dd8e..25882f8895016 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -965,6 +965,16 @@ public final class IgniteSystemProperties {
      */
     public static final String IGNITE_ALLOW_START_CACHES_IN_PARALLEL = "IGNITE_ALLOW_START_CACHES_IN_PARALLEL";
 
+    /**
+     * When set to {@code true}, cache metrics are not included into the discovery metrics update message (in this
+     * case message contains only cluster metrics). By default cache metrics are included into the message and
+     * calculated each time the message is sent.
+     * <p>
+     * Cache metrics sending can also be turned off by disabling statistics per each cache, but in this case some cache
+     * metrics will be unavailable via JMX too.
+     */
+    public static final String IGNITE_DISCOVERY_DISABLE_CACHE_METRICS_UPDATE = "IGNITE_DISCOVERY_DISABLE_CACHE_METRICS_UPDATE";
+
     /**
      * Enforces singleton.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index bd5b6b7328632..93c6feda8708b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -48,6 +48,7 @@
 import org.apache.ignite.IgniteClientDisconnectedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteInterruptedException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheMetrics;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cluster.BaselineNode;
@@ -1092,6 +1093,10 @@ private GridLocalMetrics createMetrics() {
      */
     public DiscoveryMetricsProvider createMetricsProvider() {
         return new DiscoveryMetricsProvider() {
+            /** Disable cache metrics update. */
+            private final boolean disableCacheMetricsUpdate = IgniteSystemProperties.getBoolean(
+                IgniteSystemProperties.IGNITE_DISCOVERY_DISABLE_CACHE_METRICS_UPDATE, false);
+
             /** */
             private final long startTime = U.currentTimeMillis();
 
@@ -1103,6 +1108,9 @@ public DiscoveryMetricsProvider createMetricsProvider() {
             /** {@inheritDoc} */
             @Override public Map<Integer, CacheMetrics> cacheMetrics() {
                 try {
+                    if (disableCacheMetricsUpdate)
+                        return Collections.emptyMap();
+
                     /** Caches should not be accessed while state transition is in progress. */
                     if (ctx.state().clusterState().transition())
                         return Collections.emptyMap();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java
index b8f9d70dc5dde..df60c42e8d4c6 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheMetricsForClusterGroupSelfTest.java
@@ -21,6 +21,7 @@
 import java.util.Map;
 import java.util.concurrent.CountDownLatch;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheMetrics;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -71,27 +72,15 @@ public class CacheMetricsForClusterGroupSelfTest extends GridCommonAbstractTest
         return cfg;
     }
 
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-        startGrids(GRID_CNT);
-
-        daemon = true;
-
-        startGrid(GRID_CNT);
-    }
-
-    /** {@inheritDoc} */
-    @Override protected void afterTestsStopped() throws Exception {
-        stopAllGrids();
-    }
-
     /**
      * Test cluster group metrics in case of statistics enabled.
      */
     public void testMetricsStatisticsEnabled() throws Exception {
-        createCaches(true);
+        startGrids();
 
         try {
+            createCaches(true);
+
             populateCacheData(cache1, ENTRY_CNT_CACHE1);
             populateCacheData(cache2, ENTRY_CNT_CACHE2);
 
@@ -112,7 +101,7 @@ public void testMetricsStatisticsEnabled() throws Exception {
             assertMetrics(cache2, true);
         }
         finally {
-            destroyCaches();
+            stopAllGrids();
         }
     }
 
@@ -122,9 +111,11 @@ public void testMetricsStatisticsEnabled() throws Exception {
      * @throws Exception If failed.
      */
     public void testMetricsStatisticsDisabled() throws Exception {
-        createCaches(false);
+        startGrids();
 
         try {
+            createCaches(false);
+
             populateCacheData(cache1, ENTRY_CNT_CACHE1);
             populateCacheData(cache2, ENTRY_CNT_CACHE2);
 
@@ -145,10 +136,61 @@ public void testMetricsStatisticsDisabled() throws Exception {
             assertMetrics(cache2, false);
         }
         finally {
-            destroyCaches();
+            stopAllGrids();
         }
     }
 
+    /**
+     * Tests that only local metrics are updating if discovery updates disabled.
+     */
+    public void testMetricsDiscoveryUpdatesDisabled() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_DISCOVERY_DISABLE_CACHE_METRICS_UPDATE, "true");
+
+        try {
+            startGrids();
+
+            try {
+                createCaches(true);
+
+                populateCacheData(cache1, ENTRY_CNT_CACHE1);
+                populateCacheData(cache2, ENTRY_CNT_CACHE2);
+
+                readCacheData(cache1, ENTRY_CNT_CACHE1);
+                readCacheData(cache2, ENTRY_CNT_CACHE2);
+
+                awaitMetricsUpdate();
+
+                Collection<ClusterNode> nodes = grid(0).cluster().forRemotes().nodes();
+
+                for (ClusterNode node : nodes) {
+                    Map<Integer, CacheMetrics> metrics = ((IgniteClusterNode)node).cacheMetrics();
+                    assertNotNull(metrics);
+                    assertTrue(metrics.isEmpty());
+                }
+
+                assertOnlyLocalMetricsUpdating(CACHE1);
+                assertOnlyLocalMetricsUpdating(CACHE2);
+            }
+            finally {
+                stopAllGrids();
+            }
+        }
+        finally {
+            System.setProperty(IgniteSystemProperties.IGNITE_DISCOVERY_DISABLE_CACHE_METRICS_UPDATE, "false");
+        }
+    }
+
+    /**
+     * Start grids.
+     */
+    private void startGrids() throws Exception {
+        startGrids(GRID_CNT);
+
+        daemon = true;
+
+        startGrid(GRID_CNT);
+    }
+
     /**
      * @param statisticsEnabled Statistics enabled.
      */
@@ -272,6 +314,34 @@ private void assertMetrics(IgniteCache<Integer, Integer> cache, boolean expectNo
         }
     }
 
+    /**
+     * Asserts that only local metrics are updating.
+     *
+     * @param cacheName Cache name.
+     */
+    private void assertOnlyLocalMetricsUpdating(String cacheName) {
+        for (int i = 0; i < GRID_CNT; i++) {
+            IgniteCache cache = grid(i).cache(cacheName);
+
+            CacheMetrics clusterMetrics = cache.metrics(grid(i).cluster().forCacheNodes(cacheName));
+            CacheMetrics locMetrics = cache.localMetrics();
+
+            assertEquals(clusterMetrics.name(), locMetrics.name());
+
+            assertEquals(0L, clusterMetrics.getCacheGets());
+            assertEquals(0L, cache.mxBean().getCacheGets());
+            assertEquals(locMetrics.getCacheGets(), cache.localMxBean().getCacheGets());
+
+            assertEquals(0L, clusterMetrics.getCachePuts());
+            assertEquals(0L, cache.mxBean().getCachePuts());
+            assertEquals(locMetrics.getCachePuts(), cache.localMxBean().getCachePuts());
+
+            assertEquals(0L, clusterMetrics.getCacheHits());
+            assertEquals(0L, cache.mxBean().getCacheHits());
+            assertEquals(locMetrics.getCacheHits(), cache.localMxBean().getCacheHits());
+        }
+    }
+
     /**
      * @param ms Milliseconds.
      * @param f Function.

From b5fdfc6185f7ddda24858383797eb97e4936ba05 Mon Sep 17 00:00:00 2001
From: Aleksey Plekhanov <plehanov.alex@gmail.com>
Date: Wed, 10 Oct 2018 11:57:11 +0300
Subject: [PATCH 524/543] IGNITE-9500: SQL: implemented system view for list of
 caches (IGNITE.CACHES). This closes #4716.

                                                                    (cherry picked from commit 8b381279017080b74e3c6fee4d8a177c50411189)
---
 .../processors/query/h2/IgniteH2Indexing.java |   2 +
 .../sys/view/SqlAbstractLocalSystemView.java  |  13 +-
 .../h2/sys/view/SqlSystemViewCaches.java      | 197 +++++++++++++++
 .../h2/sys/view/SqlSystemViewNodeMetrics.java |   2 +-
 .../query/h2/sys/view/SqlSystemViewNodes.java |   2 +-
 .../query/SqlSystemViewsSelfTest.java         | 225 ++++++++++++++++++
 6 files changed, 438 insertions(+), 3 deletions(-)
 create mode 100644 modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewCaches.java

diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index a848d8a7f6b03..57f2b884799be 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -115,6 +115,7 @@
 import org.apache.ignite.internal.processors.query.h2.sql.GridSqlStatement;
 import org.apache.ignite.internal.processors.query.h2.sys.SqlSystemTableEngine;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewBaselineNodes;
+import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewCaches;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodeAttributes;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodeMetrics;
 import org.apache.ignite.internal.processors.query.h2.sys.view.SqlSystemViewNodes;
@@ -2644,6 +2645,7 @@ public Collection<SqlSystemView> systemViews(GridKernalContext ctx) {
         views.add(new SqlSystemViewNodeAttributes(ctx));
         views.add(new SqlSystemViewBaselineNodes(ctx));
         views.add(new SqlSystemViewNodeMetrics(ctx));
+        views.add(new SqlSystemViewCaches(ctx));
 
         return views;
     }
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
index d692dbac3dafa..d028406a122c9 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlAbstractLocalSystemView.java
@@ -37,7 +37,7 @@ public abstract class SqlAbstractLocalSystemView extends SqlAbstractSystemView {
      * @param tblName Table name.
      * @param desc Description.
      * @param ctx Context.
-     * @param indexes Indexed columns.
+     * @param indexes Indexes.
      * @param cols Columns.
      */
     public SqlAbstractLocalSystemView(String tblName, String desc, GridKernalContext ctx, String[] indexes,
@@ -49,6 +49,17 @@ public SqlAbstractLocalSystemView(String tblName, String desc, GridKernalContext
         assert indexes != null;
     }
 
+    /**
+     * @param tblName Table name.
+     * @param desc Description.
+     * @param ctx Context.
+     * @param indexedCols Indexed columns.
+     * @param cols Columns.
+     */
+    public SqlAbstractLocalSystemView(String tblName, String desc, GridKernalContext ctx, String indexedCols, Column... cols) {
+        this(tblName, desc, ctx, new String[] {indexedCols}, cols);
+    }
+
     /**
      * @param tblName Table name.
      * @param desc Description.
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewCaches.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewCaches.java
new file mode 100644
index 0000000000000..ff9ef32115121
--- /dev/null
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewCaches.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2.sys.view;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.util.typedef.F;
+import org.h2.engine.Session;
+import org.h2.result.Row;
+import org.h2.result.SearchRow;
+import org.h2.value.Value;
+
+/**
+ * System view: caches.
+ */
+public class SqlSystemViewCaches extends SqlAbstractLocalSystemView {
+    /**
+     * @param ctx Grid context.
+     */
+    public SqlSystemViewCaches(GridKernalContext ctx) {
+        super("CACHES", "Ignite caches", ctx, "NAME",
+            newColumn("NAME"),
+            newColumn("CACHE_ID", Value.INT),
+            newColumn("CACHE_TYPE"),
+            newColumn("GROUP_ID", Value.INT),
+            newColumn("GROUP_NAME"),
+            newColumn("CACHE_MODE"),
+            newColumn("ATOMICITY_MODE"),
+            newColumn("IS_ONHEAP_CACHE_ENABLED", Value.BOOLEAN),
+            newColumn("IS_COPY_ON_READ", Value.BOOLEAN),
+            newColumn("IS_LOAD_PREVIOUS_VALUE", Value.BOOLEAN),
+            newColumn("IS_READ_FROM_BACKUP", Value.BOOLEAN),
+            newColumn("PARTITION_LOSS_POLICY"),
+            newColumn("NODE_FILTER"),
+            newColumn("TOPOLOGY_VALIDATOR"),
+            newColumn("IS_EAGER_TTL", Value.BOOLEAN),
+            newColumn("WRITE_SYNCHRONIZATION_MODE"),
+            newColumn("IS_INVALIDATE", Value.BOOLEAN),
+            newColumn("IS_EVENTS_DISABLED", Value.BOOLEAN),
+            newColumn("IS_STATISTICS_ENABLED", Value.BOOLEAN),
+            newColumn("IS_MANAGEMENT_ENABLED", Value.BOOLEAN),
+            newColumn("BACKUPS", Value.INT),
+            newColumn("AFFINITY"),
+            newColumn("AFFINITY_MAPPER"),
+            newColumn("REBALANCE_MODE"),
+            newColumn("REBALANCE_BATCH_SIZE", Value.INT),
+            newColumn("REBALANCE_TIMEOUT", Value.LONG),
+            newColumn("REBALANCE_DELAY", Value.LONG),
+            newColumn("REBALANCE_THROTTLE", Value.LONG),
+            newColumn("REBALANCE_BATCHES_PREFETCH_COUNT", Value.LONG),
+            newColumn("REBALANCE_ORDER", Value.INT),
+            newColumn("EVICTION_FILTER"),
+            newColumn("EVICTION_POLICY_FACTORY"),
+            newColumn("IS_NEAR_CACHE_ENABLED", Value.BOOLEAN),
+            newColumn("NEAR_CACHE_EVICTION_POLICY_FACTORY"),
+            newColumn("NEAR_CACHE_START_SIZE", Value.INT),
+            newColumn("DEFAULT_LOCK_TIMEOUT", Value.LONG),
+            newColumn("CACHE_INTERCEPTOR"),
+            newColumn("CACHE_STORE_FACTORY"),
+            newColumn("IS_STORE_KEEP_BINARY", Value.BOOLEAN),
+            newColumn("IS_READ_THROUGH", Value.BOOLEAN),
+            newColumn("IS_WRITE_THROUGH", Value.BOOLEAN),
+            newColumn("IS_WRITE_BEHIND_ENABLED", Value.BOOLEAN),
+            newColumn("WRITE_BEHIND_COALESCING", Value.BOOLEAN),
+            newColumn("WRITE_BEHIND_FLUSH_SIZE", Value.INT),
+            newColumn("WRITE_BEHIND_FLUSH_FREQUENCY", Value.LONG),
+            newColumn("WRITE_BEHIND_FLUSH_THREAD_COUNT", Value.INT),
+            newColumn("WRITE_BEHIND_FLUSH_BATCH_SIZE", Value.INT),
+            newColumn("MAX_CONCURRENT_ASYNC_OPERATIONS", Value.INT),
+            newColumn("CACHE_LOADER_FACTORY"),
+            newColumn("CACHE_WRITER_FACTORY"),
+            newColumn("EXPIRY_POLICY_FACTORY"),
+            newColumn("IS_SQL_ESCAPE_ALL", Value.BOOLEAN),
+            newColumn("SQL_SCHEMA"),
+            newColumn("SQL_INDEX_MAX_INLINE_SIZE", Value.INT),
+            newColumn("IS_SQL_ONHEAP_CACHE_ENABLED", Value.BOOLEAN),
+            newColumn("SQL_ONHEAP_CACHE_MAX_SIZE", Value.INT),
+            newColumn("QUERY_DETAILS_METRICS_SIZE", Value.INT),
+            newColumn("QUERY_PARALLELISM", Value.INT),
+            newColumn("MAX_QUERY_ITERATORS_COUNT", Value.INT),
+            newColumn("DATA_REGION_NAME")
+        );
+    }
+
+    /** {@inheritDoc} */
+    @SuppressWarnings("unchecked")
+    @Override public Iterator<Row> getRows(Session ses, SearchRow first, SearchRow last) {
+        SqlSystemViewColumnCondition nameCond = conditionForColumn("NAME", first, last);
+
+        Collection<DynamicCacheDescriptor> caches;
+
+        if (nameCond.isEquality()) {
+            DynamicCacheDescriptor cache = ctx.cache().cacheDescriptor(nameCond.valueForEquality().getString());
+
+            caches = cache == null ? Collections.emptySet() : Collections.singleton(cache);
+        }
+        else
+            caches = ctx.cache().cacheDescriptors().values();
+
+        AtomicLong rowKey = new AtomicLong();
+
+        return F.iterator(caches,
+            cache -> createRow(ses, rowKey.incrementAndGet(),
+                cache.cacheName(),
+                cache.cacheId(),
+                cache.cacheType(),
+                cache.groupId(),
+                cache.groupDescriptor().groupName(),
+                cache.cacheConfiguration().getCacheMode(),
+                cache.cacheConfiguration().getAtomicityMode(),
+                cache.cacheConfiguration().isOnheapCacheEnabled(),
+                cache.cacheConfiguration().isCopyOnRead(),
+                cache.cacheConfiguration().isLoadPreviousValue(),
+                cache.cacheConfiguration().isReadFromBackup(),
+                cache.cacheConfiguration().getPartitionLossPolicy(),
+                cache.cacheConfiguration().getNodeFilter(),
+                cache.cacheConfiguration().getTopologyValidator(),
+                cache.cacheConfiguration().isEagerTtl(),
+                cache.cacheConfiguration().getWriteSynchronizationMode(),
+                cache.cacheConfiguration().isInvalidate(),
+                cache.cacheConfiguration().isEventsDisabled(),
+                cache.cacheConfiguration().isStatisticsEnabled(),
+                cache.cacheConfiguration().isManagementEnabled(),
+                cache.cacheConfiguration().getBackups(),
+                cache.cacheConfiguration().getAffinity(),
+                cache.cacheConfiguration().getAffinityMapper(),
+                cache.cacheConfiguration().getRebalanceMode(),
+                cache.cacheConfiguration().getRebalanceBatchSize(),
+                cache.cacheConfiguration().getRebalanceTimeout(),
+                cache.cacheConfiguration().getRebalanceDelay(),
+                cache.cacheConfiguration().getRebalanceThrottle(),
+                cache.cacheConfiguration().getRebalanceBatchesPrefetchCount(),
+                cache.cacheConfiguration().getRebalanceOrder(),
+                cache.cacheConfiguration().getEvictionFilter(),
+                cache.cacheConfiguration().getEvictionPolicyFactory(),
+                cache.cacheConfiguration().getNearConfiguration() != null,
+                cache.cacheConfiguration().getNearConfiguration() != null ?
+                    cache.cacheConfiguration().getNearConfiguration().getNearEvictionPolicyFactory() : null,
+                cache.cacheConfiguration().getNearConfiguration() != null ?
+                    cache.cacheConfiguration().getNearConfiguration().getNearStartSize() : null,
+                cache.cacheConfiguration().getDefaultLockTimeout(),
+                cache.cacheConfiguration().getInterceptor(),
+                cache.cacheConfiguration().getCacheStoreFactory(),
+                cache.cacheConfiguration().isStoreKeepBinary(),
+                cache.cacheConfiguration().isReadThrough(),
+                cache.cacheConfiguration().isWriteThrough(),
+                cache.cacheConfiguration().isWriteBehindEnabled(),
+                cache.cacheConfiguration().getWriteBehindCoalescing(),
+                cache.cacheConfiguration().getWriteBehindFlushSize(),
+                cache.cacheConfiguration().getWriteBehindFlushFrequency(),
+                cache.cacheConfiguration().getWriteBehindFlushThreadCount(),
+                cache.cacheConfiguration().getWriteBehindBatchSize(),
+                cache.cacheConfiguration().getMaxConcurrentAsyncOperations(),
+                cache.cacheConfiguration().getCacheLoaderFactory(),
+                cache.cacheConfiguration().getCacheWriterFactory(),
+                cache.cacheConfiguration().getExpiryPolicyFactory(),
+                cache.cacheConfiguration().isSqlEscapeAll(),
+                cache.cacheConfiguration().getSqlSchema(),
+                cache.cacheConfiguration().getSqlIndexMaxInlineSize(),
+                cache.cacheConfiguration().isSqlOnheapCacheEnabled(),
+                cache.cacheConfiguration().getSqlOnheapCacheMaxSize(),
+                cache.cacheConfiguration().getQueryDetailMetricsSize(),
+                cache.cacheConfiguration().getQueryParallelism(),
+                cache.cacheConfiguration().getMaxQueryIteratorsCount(),
+                cache.cacheConfiguration().getDataRegionName()
+            ), true);
+    }
+
+    /** {@inheritDoc} */
+    @Override public boolean canGetRowCount() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override public long getRowCount() {
+        return ctx.cache().cacheDescriptors().size();
+    }
+}
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java
index 01b4e976f0cae..d3921aaaa0490 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodeMetrics.java
@@ -40,7 +40,7 @@ public class SqlSystemViewNodeMetrics extends SqlAbstractLocalSystemView {
      * @param ctx Grid context.
      */
     public SqlSystemViewNodeMetrics(GridKernalContext ctx) {
-        super("NODE_METRICS", "Node metrics", ctx, new String[] {"NODE_ID"},
+        super("NODE_METRICS", "Node metrics", ctx, "NODE_ID",
             newColumn("NODE_ID", Value.UUID),
             newColumn("LAST_UPDATE_TIME", Value.TIMESTAMP),
             newColumn("MAX_ACTIVE_JOBS", Value.INT),
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
index 514f92e9708ee..d8720310aef08 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/sys/view/SqlSystemViewNodes.java
@@ -39,7 +39,7 @@ public class SqlSystemViewNodes extends SqlAbstractLocalSystemView {
      * @param ctx Grid context.
      */
     public SqlSystemViewNodes(GridKernalContext ctx) {
-        super("NODES", "Topology nodes", ctx, new String[] {"ID"},
+        super("NODES", "Topology nodes", ctx, "ID",
             newColumn("ID", Value.UUID),
             newColumn("CONSISTENT_ID"),
             newColumn("VERSION"),
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
index 1a4dae794fb10..91b128d6c9276 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlSystemViewsSelfTest.java
@@ -19,24 +19,35 @@
 
 import java.sql.Time;
 import java.sql.Timestamp;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
 import java.util.TimeZone;
 import java.util.UUID;
 import java.util.concurrent.Callable;
+import javax.cache.Cache;
+import javax.cache.configuration.Factory;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.eviction.EvictableEntry;
+import org.apache.ignite.cache.eviction.EvictionFilter;
+import org.apache.ignite.cache.eviction.EvictionPolicy;
 import org.apache.ignite.cache.query.SqlFieldsQuery;
 import org.apache.ignite.cluster.ClusterMetrics;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.DataRegionConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.TopologyValidator;
 import org.apache.ignite.internal.ClusterMetricsSnapshot;
 import org.apache.ignite.internal.IgniteNodeAttributes;
 import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
+import org.apache.ignite.internal.util.lang.GridNodePredicate;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.X;
@@ -499,6 +510,154 @@ public void testBaselineViews() throws Exception {
         assertEquals("node2", res.get(0).get(0));
     }
 
+    /**
+     * Test caches system views.
+     */
+    public void testCachesViews() throws Exception {
+        DataStorageConfiguration dsCfg = new DataStorageConfiguration()
+            .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setName("def").setPersistenceEnabled(true))
+            .setDataRegionConfigurations(new DataRegionConfiguration().setName("dr1"),
+                new DataRegionConfiguration().setName("dr2"));
+
+        Ignite ignite0 = startGrid(getConfiguration().setDataStorageConfiguration(dsCfg));
+
+        Ignite ignite1 = startGrid(getConfiguration().setDataStorageConfiguration(dsCfg).setIgniteInstanceName("node1"));
+
+        ignite0.cluster().active(true);
+
+        Ignite ignite2 = startGrid(getConfiguration().setDataStorageConfiguration(dsCfg).setIgniteInstanceName("node2"));
+
+        Ignite ignite3 = startGrid(getConfiguration().setDataStorageConfiguration(dsCfg).setIgniteInstanceName("node3")
+            .setClientMode(true));
+
+        ignite0.getOrCreateCache(new CacheConfiguration<>()
+            .setName("cache_atomic_part")
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setGroupName("part_grp")
+            .setNodeFilter(new TestNodeFilter(ignite0.cluster().localNode()))
+        );
+
+        ignite0.getOrCreateCache(new CacheConfiguration<>()
+            .setName("cache_atomic_repl")
+            .setAtomicityMode(CacheAtomicityMode.ATOMIC)
+            .setCacheMode(CacheMode.REPLICATED)
+            .setDataRegionName("dr1")
+            .setTopologyValidator(new TestTopologyValidator())
+        );
+
+        ignite0.getOrCreateCache(new CacheConfiguration<>()
+            .setName("cache_tx_part")
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+            .setCacheMode(CacheMode.PARTITIONED)
+            .setGroupName("part_grp")
+            .setNodeFilter(new TestNodeFilter(ignite0.cluster().localNode()))
+        );
+
+        ignite0.getOrCreateCache(new CacheConfiguration<>()
+            .setName("cache_tx_repl")
+            .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+            .setCacheMode(CacheMode.REPLICATED)
+            .setDataRegionName("dr2")
+            .setEvictionFilter(new TestEvictionFilter())
+            .setEvictionPolicyFactory(new TestEvictionPolicyFactory())
+            .setOnheapCacheEnabled(true)
+        );
+
+        execSql("CREATE TABLE PUBLIC.cache_sql (ID INT PRIMARY KEY, VAL VARCHAR) WITH " +
+            "\"cache_name=cache_sql,template=partitioned,atomicity=atomic\"");
+
+        awaitPartitionMapExchange();
+
+        List<List<?>> resAll = execSql("SELECT NAME, CACHE_ID, CACHE_TYPE, GROUP_ID, GROUP_NAME, " +
+                "CACHE_MODE, ATOMICITY_MODE, IS_ONHEAP_CACHE_ENABLED, IS_COPY_ON_READ, IS_LOAD_PREVIOUS_VALUE, " +
+                "IS_READ_FROM_BACKUP, PARTITION_LOSS_POLICY, NODE_FILTER, TOPOLOGY_VALIDATOR, IS_EAGER_TTL, " +
+                "WRITE_SYNCHRONIZATION_MODE, IS_INVALIDATE, IS_EVENTS_DISABLED, IS_STATISTICS_ENABLED, " +
+                "IS_MANAGEMENT_ENABLED, BACKUPS, AFFINITY, AFFINITY_MAPPER, " +
+                "REBALANCE_MODE, REBALANCE_BATCH_SIZE, REBALANCE_TIMEOUT, REBALANCE_DELAY, REBALANCE_THROTTLE, " +
+                "REBALANCE_BATCHES_PREFETCH_COUNT, REBALANCE_ORDER, " +
+                "EVICTION_FILTER, EVICTION_POLICY_FACTORY, " +
+                "IS_NEAR_CACHE_ENABLED, NEAR_CACHE_EVICTION_POLICY_FACTORY, NEAR_CACHE_START_SIZE, " +
+                "DEFAULT_LOCK_TIMEOUT, CACHE_INTERCEPTOR, CACHE_STORE_FACTORY, " +
+                "IS_STORE_KEEP_BINARY, IS_READ_THROUGH, IS_WRITE_THROUGH, " +
+                "IS_WRITE_BEHIND_ENABLED, WRITE_BEHIND_COALESCING, WRITE_BEHIND_FLUSH_SIZE, " +
+                "WRITE_BEHIND_FLUSH_FREQUENCY, WRITE_BEHIND_FLUSH_THREAD_COUNT, WRITE_BEHIND_FLUSH_BATCH_SIZE, " +
+                "MAX_CONCURRENT_ASYNC_OPERATIONS, CACHE_LOADER_FACTORY, CACHE_WRITER_FACTORY, EXPIRY_POLICY_FACTORY, " +
+                "IS_SQL_ESCAPE_ALL, SQL_SCHEMA, SQL_INDEX_MAX_INLINE_SIZE, IS_SQL_ONHEAP_CACHE_ENABLED, " +
+                "SQL_ONHEAP_CACHE_MAX_SIZE, QUERY_DETAILS_METRICS_SIZE, QUERY_PARALLELISM, MAX_QUERY_ITERATORS_COUNT, " +
+                "DATA_REGION_NAME FROM IGNITE.CACHES");
+
+        assertColumnTypes(resAll.get(0),
+            String.class, Integer.class, String.class, Integer.class, String.class,
+            String.class, String.class, Boolean.class, Boolean.class, Boolean.class,
+            Boolean.class, String.class, String.class, String.class, Boolean.class,
+            String.class, Boolean.class, Boolean.class, Boolean.class,
+            Boolean.class, Integer.class, String.class, String.class,
+            String.class, Integer.class, Long.class, Long.class, Long.class, // Rebalance.
+            Long.class, Integer.class,
+            String.class, String.class, // Eviction.
+            Boolean.class, String.class, Integer.class, // Near cache.
+            Long.class, String.class, String.class,
+            Boolean.class, Boolean.class, Boolean.class,
+            Boolean.class, Boolean.class, Integer.class, // Write-behind.
+            Long.class, Integer.class, Integer.class,
+            Integer.class, String.class, String.class, String.class,
+            Boolean.class, String.class, Integer.class, Boolean.class, // SQL.
+            Integer.class, Integer.class, Integer.class, Integer.class,
+            String.class);
+
+        assertEquals("cache_tx_part", execSql("SELECT NAME FROM IGNITE.CACHES WHERE " +
+            "CACHE_MODE = 'PARTITIONED' AND ATOMICITY_MODE = 'TRANSACTIONAL' AND NAME like 'cache%'").get(0).get(0));
+
+        assertEquals("cache_atomic_repl", execSql("SELECT NAME FROM IGNITE.CACHES WHERE " +
+            "CACHE_MODE = 'REPLICATED' AND ATOMICITY_MODE = 'ATOMIC' AND NAME like 'cache%'").get(0).get(0));
+
+        assertEquals(2L, execSql("SELECT COUNT(*) FROM IGNITE.CACHES WHERE GROUP_NAME = 'part_grp'")
+            .get(0).get(0));
+
+        assertEquals("cache_atomic_repl", execSql("SELECT NAME FROM IGNITE.CACHES " +
+            "WHERE DATA_REGION_NAME = 'dr1'").get(0).get(0));
+
+        assertEquals("cache_tx_repl", execSql("SELECT NAME FROM IGNITE.CACHES " +
+            "WHERE DATA_REGION_NAME = 'dr2'").get(0).get(0));
+
+        assertEquals("PARTITIONED", execSql("SELECT CACHE_MODE FROM IGNITE.CACHES " +
+            "WHERE NAME = 'cache_atomic_part'").get(0).get(0));
+
+        assertEquals("USER", execSql("SELECT CACHE_TYPE FROM IGNITE.CACHES WHERE NAME = 'cache_sql'")
+            .get(0).get(0));
+
+        assertEquals(0L, execSql("SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME = 'no_such_cache'").get(0)
+            .get(0));
+
+        assertEquals(0L, execSql("SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME = 1").get(0).get(0));
+
+        assertEquals("TestNodeFilter", execSql("SELECT NODE_FILTER FROM IGNITE.CACHES WHERE NAME = " +
+            "'cache_atomic_part'").get(0).get(0));
+
+        assertEquals("TestEvictionFilter", execSql("SELECT EVICTION_FILTER FROM IGNITE.CACHES " +
+            "WHERE NAME = 'cache_tx_repl'").get(0).get(0));
+
+        assertEquals("TestEvictionPolicyFactory", execSql("SELECT EVICTION_POLICY_FACTORY " +
+            "FROM IGNITE.CACHES WHERE NAME = 'cache_tx_repl'").get(0).get(0));
+
+        assertEquals("TestTopologyValidator", execSql("SELECT TOPOLOGY_VALIDATOR FROM IGNITE.CACHES " +
+            "WHERE NAME = 'cache_atomic_repl'").get(0).get(0));
+
+        // Check that caches are the same on BLT, BLT filtered by node filter, non BLT and client nodes.
+        assertEquals(5L, execSql("SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME like 'cache%'").get(0)
+            .get(0));
+
+        assertEquals(5L, execSql(ignite1, "SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME like 'cache%'")
+            .get(0).get(0));
+
+        assertEquals(5L, execSql(ignite2, "SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME like 'cache%'")
+            .get(0).get(0));
+
+        assertEquals(5L, execSql(ignite3, "SELECT COUNT(*) FROM IGNITE.CACHES WHERE NAME like 'cache%'")
+            .get(0).get(0));
+    }
+
     /**
      * Gets ignite configuration with persistence enabled.
      */
@@ -528,4 +687,70 @@ private long convertToMilliseconds(Object sqlTime) {
 
         return time0.getTime() + TimeZone.getDefault().getOffset(time0.getTime());
     }
+
+    /**
+     *
+     */
+    private static class TestNodeFilter extends GridNodePredicate {
+        /**
+         * @param node Node.
+         */
+        public TestNodeFilter(ClusterNode node) {
+            super(node);
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "TestNodeFilter";
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestEvictionFilter implements EvictionFilter<Object, Object> {
+        /** {@inheritDoc} */
+        @Override public boolean evictAllowed(Cache.Entry<Object, Object> entry) {
+            return false;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "TestEvictionFilter";
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestEvictionPolicyFactory implements Factory<EvictionPolicy<Object, Object>> {
+        /** {@inheritDoc} */
+        @Override public EvictionPolicy<Object, Object> create() {
+            return new EvictionPolicy<Object, Object>() {
+                @Override public void onEntryAccessed(boolean rmv, EvictableEntry<Object, Object> entry) {
+                    // No-op.
+                }
+            };
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "TestEvictionPolicyFactory";
+        }
+    }
+
+    /**
+     *
+     */
+    private static class TestTopologyValidator implements TopologyValidator {
+        /** {@inheritDoc} */
+        @Override public boolean validate(Collection<ClusterNode> nodes) {
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return "TestTopologyValidator";
+        }
+    }
 }

From 85071000378f6076b20595e15bdff82747b6d2a9 Mon Sep 17 00:00:00 2001
From: ibessonov <bessonov.ip@gmail.com>
Date: Tue, 4 Dec 2018 12:55:45 +0300
Subject: [PATCH 525/543] IGNITE-10369 Avoid race on handling discovery
 messages if last custom message is not processed completely - Fixes #5462.

Signed-off-by: Pavel Kovalenko <jokserfn@gmail.com>

(cherry picked from commit 0cd303f1f1b4be23be8f3be6301d4defd4be6ea1)
---
 .../discovery/GridDiscoveryManager.java       |  4 +---
 .../cluster/GridClusterStateProcessor.java    |  2 +-
 .../spi/discovery/DiscoveryDataBag.java       |  6 ++++++
 .../ignite/spi/discovery/tcp/ServerImpl.java  | 21 +++++++++++++++++++
 .../IgniteClusterActivateDeactivateTest.java  |  4 ++--
 .../zk/internal/ZookeeperDiscoveryImpl.java   | 21 +++++++++++++++++++
 6 files changed, 52 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 93c6feda8708b..80a58e826387f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -768,8 +768,6 @@ else if (customMsg instanceof ChangeGlobalStateMessage) {
                     // Current version.
                     discoCache = discoCache();
 
-                final DiscoCache discoCache0 = discoCache;
-
                 // If this is a local join event, just save it and do not notify listeners.
                 if (locJoinEvt) {
                     if (gridStartTime == 0)
@@ -853,7 +851,7 @@ else if (type == EVT_CLIENT_NODE_RECONNECTED) {
                             try {
                                 fut.get();
 
-                                discoWrk.addEvent(type, nextTopVer, node, discoCache0, topSnapshot, null);
+                                discoWrk.addEvent(EVT_CLIENT_NODE_RECONNECTED, nextTopVer, node, discoCache, topSnapshot, null);
                             }
                             catch (IgniteException ignore) {
                                 // No-op.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index f4f762a76f2b1..4187fb160bfd4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -414,7 +414,7 @@ private boolean isBaselineSatisfied(BaselineTopology blt, List<ClusterNode> serv
         if (msg.requestId().equals(state.transitionRequestId())) {
             log.info("Received state change finish message: " + msg.clusterActive());
 
-            globalState = globalState.finish(msg.success());
+            globalState = state.finish(msg.success());
 
             afterStateChangeFinished(msg.id(), msg.success());
 
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoveryDataBag.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoveryDataBag.java
index b8d8908215130..2b6bbca0c5358 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoveryDataBag.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/DiscoveryDataBag.java
@@ -23,6 +23,7 @@
 import java.util.Set;
 import java.util.UUID;
 import org.apache.ignite.internal.GridComponent;
+import org.apache.ignite.internal.util.typedef.internal.S;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -312,4 +313,9 @@ public Map<Integer, Serializable> commonData() {
     @Nullable public Map<Integer, Serializable> localNodeSpecificData() {
         return nodeSpecificData.get(DEFAULT_KEY);
     }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(DiscoveryDataBag.class, this);
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index a1fba58265106..f5e7d14121d93 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -256,6 +256,9 @@ class ServerImpl extends TcpDiscoveryImpl {
     private final ConcurrentMap<InetSocketAddress, GridPingFutureAdapter<IgniteBiTuple<UUID, Boolean>>> pingMap =
         new ConcurrentHashMap<>();
 
+    /** Last listener future. */
+    private IgniteFuture<?> lastCustomEvtLsnrFut;
+
     /**
      * @param adapter Adapter.
      */
@@ -2140,6 +2143,20 @@ private <T> T unmarshalWithSecurityVersion(byte[] bytes, int ver) throws IgniteC
         }
     }
 
+    /**
+     * Wait for all the listeners from previous discovery message to be completed.
+     */
+    private void waitForLastCustomEventListenerFuture() {
+        if (lastCustomEvtLsnrFut != null) {
+            try {
+                lastCustomEvtLsnrFut.get();
+            }
+            finally {
+                lastCustomEvtLsnrFut = null;
+            }
+        }
+    }
+
     /**
      * Discovery messages history used for client reconnect.
      */
@@ -4103,6 +4120,8 @@ private void trySendMessageDirectly(TcpDiscoveryNode node, TcpDiscoveryAbstractM
         private void processNodeAddedMessage(TcpDiscoveryNodeAddedMessage msg) {
             assert msg != null;
 
+            waitForLastCustomEventListenerFuture();
+
             TcpDiscoveryNode node = msg.node();
 
             assert node != null;
@@ -5548,6 +5567,8 @@ private void notifyDiscoveryListener(TcpDiscoveryCustomEventMessage msg, boolean
 
                 if (waitForNotification || msgObj.isMutable())
                     fut.get();
+                else
+                    lastCustomEvtLsnrFut = fut;
 
                 if (msgObj.isMutable()) {
                     try {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
index 401a171ca0bb0..59d66802998b1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClusterActivateDeactivateTest.java
@@ -1202,13 +1202,13 @@ private void stateChangeFailover3(boolean activate) throws Exception {
 
         client = false;
 
-        IgniteInternalFuture startFut1 = GridTestUtils.runAsync((Callable) () -> {
+        IgniteInternalFuture<?> startFut1 = GridTestUtils.runAsync(() -> {
             startGrid(4);
 
             return null;
         }, "start-node1");
 
-        IgniteInternalFuture startFut2 = GridTestUtils.runAsync((Callable) () -> {
+        IgniteInternalFuture<?> startFut2 = GridTestUtils.runAsync(() -> {
             startGrid(5);
 
             return null;
diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
index 760d0108052ac..ffa4c90531529 100644
--- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
+++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -191,6 +191,9 @@ public class ZookeeperDiscoveryImpl {
     /** */
     private final ZookeeperDiscoveryStatistics stats;
 
+    /** Last listener future. */
+    private IgniteFuture<?> lastCustomEvtLsnrFut;
+
     /**
      * @param spi Discovery SPI.
      * @param igniteInstanceName Instance name.
@@ -2772,6 +2775,8 @@ private void processNewEvents(final ZkDiscoveryEventsData evtsData) throws Excep
     private boolean processBulkJoin(ZkDiscoveryEventsData evtsData, ZkDiscoveryNodeJoinEventData evtData)
         throws Exception
     {
+        waitForLastListenerFuture();
+
         boolean evtProcessed = false;
 
         for (int i = 0; i < evtData.joinedNodes.size(); i++) {
@@ -3452,6 +3457,8 @@ private void notifyCustomEvent(final ZkDiscoveryCustomEventData evtData, final D
 
         if (msg != null && msg.isMutable())
             fut.get();
+        else
+            lastCustomEvtLsnrFut = fut;
     }
 
     /**
@@ -4009,6 +4016,20 @@ public static byte[] unzip(byte[] zipBytes) throws DataFormatException {
         return out.toByteArray();
     }
 
+    /**
+     * Wait for all the listeners from previous discovery message to be completed.
+     */
+    private void waitForLastListenerFuture() {
+        if (lastCustomEvtLsnrFut != null) {
+            try {
+                lastCustomEvtLsnrFut.get();
+            }
+            finally {
+                lastCustomEvtLsnrFut = null;
+            }
+        }
+    }
+
     /**
      *
      */

From 285dc17765f9b0959a6785aea861352bbba91b6b Mon Sep 17 00:00:00 2001
From: Alexey Goncharuk <alexey.goncharuk@gmail.com>
Date: Fri, 30 Nov 2018 15:46:21 +0300
Subject: [PATCH 526/543] IGNITE-10484 Fixed activate/deactivate hang - Fixes
 #5534.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../org/apache/ignite/internal/IgniteKernal.java   | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 6ba5ce6e40f90..75575c173e832 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -1079,6 +1079,14 @@ public void start(
 
             IgniteInternalFuture<Boolean> transitionWaitFut = joinData.transitionWaitFuture();
 
+            // Notify discovery manager the first to make sure that topology is discovered.
+            // Active flag is not used in managers, so it is safe to pass true.
+            ctx.discovery().onKernalStart(true);
+
+            // Notify IO manager the second so further components can send and receive messages.
+            // Must notify the IO manager before transition state await to make sure IO connection can be established.
+            ctx.io().onKernalStart(true);
+
             boolean active;
 
             if (transitionWaitFut != null) {
@@ -1092,12 +1100,6 @@ public void start(
             else
                 active = joinData.active();
 
-            // Notify discovery manager the first to make sure that topology is discovered.
-            ctx.discovery().onKernalStart(active);
-
-            // Notify IO manager the second so further components can send and receive messages.
-            ctx.io().onKernalStart(active);
-
             boolean recon = false;
 
             // Callbacks.

From ba253763af9a97b1a849689681e9357beea1aa5b Mon Sep 17 00:00:00 2001
From: Ivan Daschinskiy <ivandasch@gmail.com>
Date: Thu, 6 Dec 2018 18:51:24 +0300
Subject: [PATCH 527/543] IGNITE-10242 Pause ongoing rebalance on cache group
 stopping

Signed-off-by: Pavel Kovalenko <jokserfn@gmail.com>

(cherry picked from commit 025a0360c86a5ceb8c3eef2f514c29846081d4c2)
---
 .../GridCachePartitionExchangeManager.java    |   4 +-
 .../processors/cache/GridCachePreloader.java  |  10 +
 .../cache/GridCachePreloaderAdapter.java      |  10 +
 .../processors/cache/GridCacheProcessor.java  | 106 ++++---
 .../preloader/GridDhtPartitionDemander.java   |   3 +
 .../GridDhtPartitionSupplyMessage.java        |   3 +
 .../GridDhtPartitionsFullMessage.java         |   5 +-
 .../dht/preloader/GridDhtPreloader.java       |  51 +++-
 .../ignite/internal/util/IgniteUtils.java     |  66 +++++
 ...lanceOnCachesStoppingOrDestroyingTest.java | 279 ++++++++++++++++++
 .../testsuites/IgnitePdsTestSuite4.java       |   3 +
 11 files changed, 497 insertions(+), 43 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/IgniteRebalanceOnCachesStoppingOrDestroyingTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 53b2228894707..88c87e22dab00 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -430,9 +430,11 @@ else if (m instanceof GridDhtPartitionDemandLegacyMessage) {
 
                                     return;
                                 }
+                                else
+                                    U.error(log, "Unsupported message type: " + m.getClass().getName());
                             }
 
-                            U.error(log, "Unsupported message type: " + m.getClass().getName());
+                            U.warn(log, "Cache group with id=" + m.groupId() + " is stopped or absent");
                         }
                         finally {
                             leaveBusy();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index 5fa7a821c69ef..153eb7cf5ea74 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -185,4 +185,14 @@ public GridDhtFuture<Object> request(GridCacheContext cctx,
      * Dumps debug information.
      */
     public void dumpDebugInfo();
+
+    /**
+     *  Pause preloader.
+     */
+    public void pause();
+
+    /**
+     * Resume preloader.
+     */
+    public void resume();
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index af916791daa3e..18311b9b1ca20 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -175,4 +175,14 @@ public GridCachePreloaderAdapter(CacheGroupContext grp) {
     @Override public void dumpDebugInfo() {
         // No-op.
     }
+
+    /** {@inheritDoc} */
+    @Override public void pause() {
+        // No-op
+    }
+
+    /** {@inheritDoc} */
+    @Override public void resume() {
+        // No-op
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index d2e676ed80655..60b066ad17038 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -70,6 +70,7 @@
 import org.apache.ignite.internal.binary.BinaryContext;
 import org.apache.ignite.internal.binary.BinaryMarshaller;
 import org.apache.ignite.internal.binary.GridBinaryMarshaller;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
@@ -2774,8 +2775,10 @@ void forceCloseCaches(ExchangeActions exchActions) {
      * @param exchActions Change requests.
      */
     private void processCacheStopRequestOnExchangeDone(ExchangeActions exchActions) {
-        // Force checkpoint if there is any cache stop request
-        if (exchActions.cacheStopRequests().size() > 0) {
+        // Reserve at least 2 threads for system operations.
+        int parallelismLvl = U.availableThreadCount(ctx, GridIoPolicy.SYSTEM_POOL, 2);
+
+        if (!exchActions.cacheStopRequests().isEmpty()) {
             try {
                 sharedCtx.database().waitForCheckpoint("caches stop");
             }
@@ -2784,63 +2787,88 @@ private void processCacheStopRequestOnExchangeDone(ExchangeActions exchActions)
             }
         }
 
-        for (ExchangeActions.CacheActionData action : exchActions.cacheStopRequests()) {
-            CacheGroupContext gctx = cacheGrps.get(action.descriptor().groupId());
+        List<IgniteBiTuple<CacheGroupContext, Boolean>> grpToStop = exchActions.cacheGroupsToStop().stream()
+                .filter(a -> cacheGrps.containsKey(a.descriptor().groupId()))
+                .map(a -> F.t(cacheGrps.get(a.descriptor().groupId()), a.destroy()))
+                .collect(Collectors.toList());
 
-            // Cancel all operations blocking gateway
-            if (gctx != null) {
-                final String msg = "Failed to wait for topology update, cache group is stopping.";
+        Map<Integer, List<ExchangeActions.CacheActionData>> cachesToStop = exchActions.cacheStopRequests().stream()
+                .collect(Collectors.groupingBy(action -> action.descriptor().groupId()));
 
-                // If snapshot operation in progress we must throw CacheStoppedException
-                // for correct cache proxy restart. For more details see
-                // IgniteCacheProxy.cacheException()
-                gctx.affinity().cancelFutures(new CacheStoppedException(msg));
-            }
+        try {
+            doInParallel(
+                    parallelismLvl,
+                    sharedCtx.kernalContext().getSystemExecutorService(),
+                    cachesToStop.entrySet(),
+                    cachesToStopByGrp -> {
+                        CacheGroupContext gctx = cacheGrps.get(cachesToStopByGrp.getKey());
 
-            stopGateway(action.request());
+                        if (gctx != null)
+                            gctx.preloader().pause();
 
-            sharedCtx.database().checkpointReadLock();
+                        try {
 
-            try {
-                prepareCacheStop(action.request().cacheName(), action.request().destroy());
-            }
-            finally {
-                sharedCtx.database().checkpointReadUnlock();
-            }
+                            if (gctx != null) {
+                                final String msg = "Failed to wait for topology update, cache group is stopping.";
+
+                                // If snapshot operation in progress we must throw CacheStoppedException
+                                // for correct cache proxy restart. For more details see
+                                // IgniteCacheProxy.cacheException()
+                                gctx.affinity().cancelFutures(new CacheStoppedException(msg));
+                            }
+
+                            for (ExchangeActions.CacheActionData action: cachesToStopByGrp.getValue()) {
+                                stopGateway(action.request());
+
+                                sharedCtx.database().checkpointReadLock();
+
+                                try {
+                                    prepareCacheStop(action.request().cacheName(), action.request().destroy());
+                                }
+                                finally {
+                                    sharedCtx.database().checkpointReadUnlock();
+                                }
+                            }
+                        }
+                        finally {
+                            if (gctx != null)
+                                gctx.preloader().resume();
+                        }
+
+                        return null;
+                    }
+            );
+        }
+        catch (IgniteCheckedException e) {
+            String msg = "Failed to stop caches";
+
+            log.error(msg, e);
+
+            throw new IgniteException(msg, e);
         }
 
         sharedCtx.database().checkpointReadLock();
 
         try {
             // Do not invoke checkpoint listeners for groups are going to be destroyed to prevent metadata corruption.
-            for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) {
-                Integer groupId = action.descriptor().groupId();
-                CacheGroupContext grp = cacheGrps.get(groupId);
+            grpToStop.forEach(grp -> {
+                CacheGroupContext gctx = grp.getKey();
 
-                if (grp != null && grp.persistenceEnabled() && sharedCtx.database() instanceof GridCacheDatabaseSharedManager) {
-                    GridCacheDatabaseSharedManager mngr = (GridCacheDatabaseSharedManager) sharedCtx.database();
-                    mngr.removeCheckpointListener((DbCheckpointListener) grp.offheap());
+                if (gctx != null && gctx.persistenceEnabled() && sharedCtx.database() instanceof GridCacheDatabaseSharedManager) {
+                    GridCacheDatabaseSharedManager mngr = (GridCacheDatabaseSharedManager)sharedCtx.database();
+                    mngr.removeCheckpointListener((DbCheckpointListener)gctx.offheap());
                 }
-            }
+            });
         }
         finally {
             sharedCtx.database().checkpointReadUnlock();
         }
 
-        List<IgniteBiTuple<CacheGroupContext, Boolean>> stoppedGroups = new ArrayList<>();
-
-        for (ExchangeActions.CacheGroupActionData action : exchActions.cacheGroupsToStop()) {
-            Integer groupId = action.descriptor().groupId();
-
-            if (cacheGrps.containsKey(groupId)) {
-                stoppedGroups.add(F.t(cacheGrps.get(groupId), action.destroy()));
-
-                stopCacheGroup(groupId);
-            }
-        }
+        for (IgniteBiTuple<CacheGroupContext, Boolean> grp : grpToStop)
+            stopCacheGroup(grp.get1().groupId());
 
         if (!sharedCtx.kernalContext().clientNode())
-            sharedCtx.database().onCacheGroupsStopped(stoppedGroups);
+            sharedCtx.database().onCacheGroupsStopped(grpToStop);
 
         if (exchActions.deactivate())
             sharedCtx.deactivate();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index e52d8d88a3b84..116a28a07d551 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -890,6 +890,9 @@ private boolean preloadEntry(
             try {
                 GridCacheContext cctx = grp.sharedGroup() ? ctx.cacheContext(entry.cacheId()) : grp.singleCacheContext();
 
+                if (cctx == null)
+                    return true;
+
                 cached = cctx.dhtCache().entryEx(entry.key());
 
                 if (log.isTraceEnabled())
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
index 3034fb9daaf21..7e281e59a7e1e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplyMessage.java
@@ -249,6 +249,9 @@ void addEntry0(int p, boolean historical, GridCacheEntryInfo info, GridCacheShar
 
         CacheGroupContext grp = ctx.cache().cacheGroup(grpId);
 
+        if (grp == null)
+            return;
+
         for (CacheEntryInfoCollection col : infos().values()) {
             List<GridCacheEntryInfo> entries = col.infos();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index fbaa24193b50f..a2cecb8ba527c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -31,6 +31,7 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.GridDirectMap;
 import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
@@ -415,7 +416,7 @@ void setErrorsMap(Map<UUID, Exception> errs) {
 
         if (marshal) {
             // Reserve at least 2 threads for system operations.
-            int parallelismLvl = Math.max(1, ctx.kernalContext().config().getSystemThreadPoolSize() - 2);
+            int parallelismLvl = U.availableThreadCount(ctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2);
 
             Collection<Object> objectsToMarshall = new ArrayList<>();
 
@@ -509,7 +510,7 @@ public void topologyVersion(AffinityTopologyVersion topVer) {
         Collection<byte[]> objectsToUnmarshall = new ArrayList<>();
 
         // Reserve at least 2 threads for system operations.
-        int parallelismLvl = Math.max(1, ctx.kernalContext().config().getSystemThreadPoolSize() - 2);
+        int parallelismLvl = U.availableThreadCount(ctx.kernalContext(), GridIoPolicy.SYSTEM_POOL, 2);
 
         if (partsBytes != null && parts == null)
             objectsToUnmarshall.add(partsBytes);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index 6e3d773abbbe3..e60eb1fa7140d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -20,13 +20,16 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Queue;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.NodeStoppingException;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
@@ -43,7 +46,9 @@
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
 import org.apache.ignite.internal.util.future.GridFutureAdapter;
 import org.apache.ignite.internal.util.lang.GridPlainRunnable;
+import org.apache.ignite.internal.util.lang.GridTuple3;
 import org.apache.ignite.internal.util.typedef.CI1;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.jetbrains.annotations.Nullable;
 
@@ -80,6 +85,12 @@ public class GridDhtPreloader extends GridCachePreloaderAdapter {
     /** Demand lock. */
     private final ReadWriteLock demandLock = new ReentrantReadWriteLock();
 
+    /** */
+    private boolean paused;
+
+    /** */
+    private Queue<GridTuple3<Integer, UUID, GridDhtPartitionSupplyMessage>> pausedDemanderQueue = new ConcurrentLinkedQueue<>();
+
     /** */
     private boolean stopped;
 
@@ -328,7 +339,10 @@ private List<ClusterNode> remoteOwners(int p, AffinityTopologyVersion topVer) {
             demandLock.readLock().lock();
 
             try {
-                demander.handleSupplyMessage(idx, id, s);
+                if (paused)
+                    pausedDemanderQueue.add(F.t(idx, id, s));
+                else
+                    demander.handleSupplyMessage(idx, id, s);
             }
             finally {
                 demandLock.readLock().unlock();
@@ -533,6 +547,41 @@ private GridDhtFuture<Object> request0(GridCacheContext cctx, Collection<KeyCach
         }
     }
 
+    /** {@inheritDoc} */
+    @Override public void pause() {
+        demandLock.writeLock().lock();
+
+        try {
+            paused = true;
+        }
+        finally {
+           demandLock.writeLock().unlock();
+        }
+    }
+
+    /** {@inheritDoc} */
+    @Override public void resume() {
+        demandLock.writeLock().lock();
+
+        try {
+            final List<GridTuple3<Integer, UUID, GridDhtPartitionSupplyMessage>> msgToProc =
+                    new ArrayList<>(pausedDemanderQueue);
+
+            pausedDemanderQueue.clear();
+
+            final GridDhtPreloader preloader = this;
+
+            ctx.kernalContext().closure().runLocalSafe(() -> msgToProc.forEach(
+                    m -> preloader.handleSupplyMessage(m.get1(), m.get2(), m.get3())
+            ), GridIoPolicy.SYSTEM_POOL);
+
+            paused = false;
+        }
+        finally {
+            demandLock.writeLock().unlock();
+        }
+    }
+
     /** {@inheritDoc} */
     @Override public void dumpDebugInfo() {
         // No-op
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 29abeefb20aa1..0e0b860f648e0 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -194,6 +194,7 @@
 import org.apache.ignite.internal.compute.ComputeTaskTimeoutCheckedException;
 import org.apache.ignite.internal.events.DiscoveryCustomEvent;
 import org.apache.ignite.internal.managers.communication.GridIoManager;
+import org.apache.ignite.internal.managers.communication.GridIoPolicy;
 import org.apache.ignite.internal.managers.deployment.GridDeploymentInfo;
 import org.apache.ignite.internal.mxbean.IgniteStandardMXBean;
 import org.apache.ignite.internal.processors.cache.CacheClassLoaderMarker;
@@ -10372,6 +10373,71 @@ public static String toHexString(ByteBuffer buf) {
         return sb.toString();
     }
 
+    /**
+     * @param ctx Kernal context.
+     * @param plc IO Policy.
+     * @param reserved Thread to reserve.
+     * @return Number of available threads in executor service for {@code plc}. If {@code plc}
+     *         is invalid, return {@code 1}.
+     */
+    public static int availableThreadCount(GridKernalContext ctx, byte plc, int reserved) {
+        IgniteConfiguration cfg = ctx.config();
+
+        int parallelismLvl;
+
+        switch (plc) {
+            case GridIoPolicy.P2P_POOL:
+                parallelismLvl = cfg.getPeerClassLoadingThreadPoolSize();
+
+                break;
+
+            case GridIoPolicy.SYSTEM_POOL:
+                parallelismLvl = cfg.getSystemThreadPoolSize();
+
+                break;
+
+            case GridIoPolicy.PUBLIC_POOL:
+                parallelismLvl = cfg.getPublicThreadPoolSize();
+
+                break;
+
+            case GridIoPolicy.MANAGEMENT_POOL:
+                parallelismLvl = cfg.getManagementThreadPoolSize();
+
+                break;
+
+            case GridIoPolicy.UTILITY_CACHE_POOL:
+                parallelismLvl = cfg.getUtilityCacheThreadPoolSize();
+
+                break;
+
+            case GridIoPolicy.IGFS_POOL:
+                parallelismLvl = cfg.getIgfsThreadPoolSize();
+
+                break;
+
+            case GridIoPolicy.SERVICE_POOL:
+                parallelismLvl = cfg.getServiceThreadPoolSize();
+
+                break;
+
+            case GridIoPolicy.DATA_STREAMER_POOL:
+                parallelismLvl = cfg.getDataStreamerThreadPoolSize();
+
+                break;
+
+            case GridIoPolicy.QUERY_POOL:
+                parallelismLvl = cfg.getQueryThreadPoolSize();
+
+                break;
+
+            default:
+                parallelismLvl = -1;
+        }
+
+        return Math.max(1, parallelismLvl - reserved);
+    }
+
     /**
      * Execute operation on data in parallel.
      *
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/IgniteRebalanceOnCachesStoppingOrDestroyingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/IgniteRebalanceOnCachesStoppingOrDestroyingTest.java
new file mode 100644
index 0000000000000..97f8d455e77d4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/IgniteRebalanceOnCachesStoppingOrDestroyingTest.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.TransactionConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.failure.StopNodeFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.managers.communication.GridIoMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgniteRebalanceOnCachesStoppingOrDestroyingTest extends GridCommonAbstractTest {
+    /** */
+    private static final String CACHE_1 = "cache_1";
+
+    /** */
+    private static final String CACHE_2 = "cache_2";
+
+    /** */
+    private static final String CACHE_3 = "cache_3";
+
+    /** */
+    private static final String CACHE_4 = "cache_4";
+
+    /** */
+    private static final String GROUP_1 = "group_1";
+
+    /** */
+    private static final String GROUP_2 = "group_2";
+
+    /** */
+    private static final int REBALANCE_BATCH_SIZE = 50 * 1024;
+
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void afterTest() throws Exception {
+        super.afterTest();
+
+        stopAllGrids();
+
+        cleanPersistenceDir();
+    }
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+        TcpDiscoverySpi spi = new TcpDiscoverySpi();
+
+        spi.setIpFinder(ipFinder);
+
+        cfg.setDiscoverySpi(spi);
+
+        cfg.setCommunicationSpi(new RebalanceBlockingSPI());
+
+        cfg.setFailureHandler(new StopNodeFailureHandler());
+
+        cfg.setRebalanceThreadPoolSize(4);
+
+        cfg.setTransactionConfiguration(new TransactionConfiguration()
+            .setDefaultTxTimeout(1000));
+
+        cfg.setDataStorageConfiguration(
+                new DataStorageConfiguration()
+                        .setWalMode(WALMode.LOG_ONLY)
+                        .setDefaultDataRegionConfiguration(
+                                new DataRegionConfiguration()
+                                        .setPersistenceEnabled(true)
+                                        .setMaxSize(100L * 1024 * 1024)));
+
+        return cfg;
+    }
+
+    /**
+     *
+     */
+    public void testStopCachesOnDeactivation() throws Exception {
+        performTest(ig -> {
+            ig.cluster().active(false);
+
+            // Add to escape possible long waiting in awaitPartitionMapExchange due to {@link CacheAffinityChangeMessage}.
+            ig.cluster().active(true);
+
+            return null;
+        });
+    }
+
+    /**
+     *
+     */
+    public void testDestroySpecificCachesInDifferentCacheGroups() throws Exception {
+        performTest(ig -> {
+            ig.destroyCaches(Arrays.asList(CACHE_1, CACHE_3));
+
+            return null;
+        });
+    }
+
+    /**
+     *
+     */
+    public void testDestroySpecificCacheAndCacheGroup() throws Exception {
+        performTest(ig -> {
+            ig.destroyCaches(Arrays.asList(CACHE_1, CACHE_3, CACHE_4));
+
+            return null;
+        });
+    }
+
+    /**
+     * @param testAction Action that trigger stop or destroy of caches.
+     */
+    private void performTest(IgniteThrowableConsumer<Ignite, Void> testAction) throws Exception {
+        IgniteEx ig0 = (IgniteEx)startGrids(2);
+
+        ig0.cluster().active(true);
+
+        stopGrid(1);
+
+        loadData(ig0);
+
+        startGrid(1);
+
+        runLoad(ig0);
+
+        testAction.accept(ig0);
+
+        U.sleep(1000);
+
+        awaitPartitionMapExchange(true, true, null, true);
+
+        assertNull(grid(1).context().failure().failureContext());
+    }
+
+    /**
+     * @param ig Ig.
+     */
+    private void loadData(Ignite ig) {
+        List<CacheConfiguration> configs = Stream.of(
+                F.t(CACHE_1, GROUP_1),
+                F.t(CACHE_2, GROUP_1),
+                F.t(CACHE_3, GROUP_2),
+                F.t(CACHE_4, GROUP_2)
+        ).map(names -> new CacheConfiguration<>(names.get1())
+                .setGroupName(names.get2())
+                .setRebalanceBatchSize(REBALANCE_BATCH_SIZE)
+                .setCacheMode(CacheMode.REPLICATED)
+        ).collect(Collectors.toList());
+
+        ig.getOrCreateCaches(configs);
+
+        configs.forEach(cfg -> {
+            try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(cfg.getName())) {
+                for (int i = 0; i < 3_000; i++)
+                    streamer.addData(i, new byte[1024]);
+
+                streamer.flush();
+            }
+        });
+    }
+
+    /**
+     * @param ig Ignite instance.
+     */
+    private void runLoad(Ignite ig) throws Exception{
+        GridTestUtils.runMultiThreaded(new Runnable() {
+            @Override public void run() {
+                String cacheName = F.rand(CACHE_1, CACHE_2, CACHE_3, CACHE_4);
+
+                IgniteCache cache = ig.cache(cacheName);
+
+                for (int i = 0; i < 3_000; i++) {
+                    int idx = ThreadLocalRandom.current().nextInt(3_000);
+
+                    cache.put(idx, new byte[1024]);
+                }
+            }
+        }, 4, "load-thread");
+    }
+
+    /**
+     *
+     */
+    private static class RebalanceBlockingSPI extends TcpCommunicationSpi {
+        /** */
+        public static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg) throws IgniteSpiException {
+            slowDownMessage(msg);
+
+            super.sendMessage(node, msg);
+
+        }
+
+        /** {@inheritDoc} */
+        @Override public void sendMessage(ClusterNode node, Message msg,
+                                          IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
+            slowDownMessage(msg);
+
+            super.sendMessage(node, msg, ackC);
+        }
+
+        /**
+         * @param msg Message.
+         */
+        private void slowDownMessage(Message msg) {
+            if (msg instanceof GridIoMessage && ((GridIoMessage)msg).message() instanceof GridDhtPartitionSupplyMessage) {
+                int grpId = ((GridCacheGroupIdMessage)((GridIoMessage)msg).message()).groupId();
+
+                if (grpId == CU.cacheId(GROUP_1) || grpId == CU.cacheId(GROUP_2)) {
+                    try {
+                        U.sleep(50);
+                    }
+                    catch (IgniteInterruptedCheckedException e) {
+                        e.printStackTrace();
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
index d5cc0ef233e46..782322960d3ab 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite4.java
@@ -19,6 +19,7 @@
 
 import junit.framework.TestSuite;
 import org.apache.ignite.internal.processors.cache.IgniteClusterActivateDeactivateTestWithPersistenceAndMemoryReuse;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.IgniteRebalanceOnCachesStoppingOrDestroyingTest;
 import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTaskCancelingTest;
 import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPartitionPreloadTest;
 
@@ -38,6 +39,8 @@ public static TestSuite suite() {
 
         suite.addTestSuite(IgnitePdsTaskCancelingTest.class);
 
+        suite.addTestSuite(IgniteRebalanceOnCachesStoppingOrDestroyingTest.class);
+
         return suite;
     }
 }

From c126248e935c5325c0077cc95778fcc94bca3b1a Mon Sep 17 00:00:00 2001
From: EdShangGG <eshangareev@gridgain.com>
Date: Tue, 20 Nov 2018 11:39:19 +0300
Subject: [PATCH 528/543] IGNITE-10328 Allow to destroy cache in code which
 were created via SQL - Fixes #5431.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit c8fd687)
---
 .../internal/processors/cache/ClusterCachesInfo.java  |  8 --------
 .../cache/index/H2DynamicTableSelfTest.java           | 11 ++---------
 2 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 97fae16339fd0..f3f72511fe58d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -674,14 +674,6 @@ else if (req.stop()) {
                         continue;
                     }
 
-                    if (!req.sql() && desc.sql()) {
-                        ctx.cache().completeCacheStartFuture(req, false,
-                            new IgniteCheckedException("Only cache created with cache API may be removed with " +
-                                "direct call to destroyCache [cacheName=" + req.cacheName() + ']'));
-
-                        continue;
-                    }
-
                     DynamicCacheDescriptor old = registeredCaches.remove(req.cacheName());
 
                     if (req.restart())
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
index 424205fed5935..e330cdc705554 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/H2DynamicTableSelfTest.java
@@ -808,7 +808,7 @@ public void testDropNonDynamicTable() throws Exception {
     }
 
     /**
-     * Test that attempting to destroy via cache API a cache created via SQL yields an error.
+     * Test that attempting to destroy via cache API a cache created via SQL finishes successfully.
      * @throws Exception if failed.
      */
     @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
@@ -817,14 +817,7 @@ public void testDestroyDynamicSqlCache() throws Exception {
             " \"name\" varchar, \"surname\" varchar, \"age\" int, PRIMARY KEY (\"id\", \"city\")) WITH " +
             "\"template=cache\"");
 
-        GridTestUtils.assertThrows(null, new Callable<Object>() {
-            @Override public Object call() throws Exception {
-                client().destroyCache(cacheName("Person"));
-
-                return null;
-            }
-        }, CacheException.class,
-        "Only cache created with cache API may be removed with direct call to destroyCache");
+        client().destroyCache(cacheName("Person"));
     }
 
     /**

From 816ad7c32e6e684f8e622595ba3473e03b4e0dbf Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Mon, 10 Dec 2018 10:23:20 +0300
Subject: [PATCH 529/543] IGNITE-10589 Correctly handle exchanges merge when
 calculating last affinity change topology version - Fixes #5609.

                                                                    Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

                                                                    (cherry picked from commit d9be16f23f4df6405ea7e523c255dbe44e21c853)
---
 .../discovery/GridDiscoveryManager.java       | 13 ++-
 .../affinity/AffinityTopologyVersion.java     |  9 ++
 .../GridCachePartitionExchangeManager.java    | 14 +++-
 .../cache/CacheNoAffinityExchangeTest.java    | 84 +++++++++++++++++++
 4 files changed, 113 insertions(+), 7 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index bd5b6b7328632..9d379b63ecd54 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -2096,10 +2096,17 @@ private DiscoCache resolveDiscoCache(int grpId, AffinityTopologyVersion topVer)
             AffinityTopologyVersion lastAffChangedTopVer =
                 ctx.cache().context().exchange().lastAffinityChangedTopologyVersion(topVer);
 
-            DiscoCache lastAffChangedDiscoCache = discoCacheHist.get(lastAffChangedTopVer);
+            if (!lastAffChangedTopVer.equals(topVer)) {
+                assert lastAffChangedTopVer.compareTo(topVer) < 0;
 
-            if (lastAffChangedDiscoCache != null)
-                return lastAffChangedDiscoCache;
+                for (Map.Entry<AffinityTopologyVersion, DiscoCache> e : discoCacheHist.descendingEntrySet()) {
+                    if (e.getKey().isBetween(lastAffChangedTopVer, topVer))
+                        return e.getValue();
+
+                    if (e.getKey().compareTo(lastAffChangedTopVer) < 0)
+                        break;
+                }
+            }
 
             CacheGroupDescriptor desc = ctx.cache().cacheGroupDescriptors().get(grpId);
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java
index 333841d1b33be..2c02f26be6641 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/affinity/AffinityTopologyVersion.java
@@ -112,6 +112,15 @@ public int minorTopologyVersion() {
         return cmp;
     }
 
+    /**
+     * @param lower Lower bound.
+     * @param upper Upper bound.
+     * @return {@code True} if this topology version is within provided bounds (inclusive).
+     */
+    public boolean isBetween(AffinityTopologyVersion lower, AffinityTopologyVersion upper) {
+        return compareTo(lower) >= 0 && compareTo(upper) <= 0;
+    }
+
     /** {@inheritDoc} */
     @Override public void onAckReceived() {
         // No-op.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index cfdda2be83534..2848ff71cd62a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -2751,12 +2751,18 @@ else if (task instanceof ForceRebalanceExchangeTask) {
 
                                 if (lastFut != null) {
                                     if (!lastFut.changedAffinity()) {
-                                        AffinityTopologyVersion lastAffVer = cctx.exchange().lastAffinityChangedTopologyVersion(lastFut.initialVersion());
-
-                                        cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(), lastAffVer);
+                                        // If lastFut corresponds to merged exchange, it is essential to use
+                                        // topologyVersion() instead of initialVersion() - nodes joined in this PME
+                                        // will have DiscoCache only for the last version.
+                                        AffinityTopologyVersion lastAffVer = cctx.exchange()
+                                            .lastAffinityChangedTopologyVersion(lastFut.topologyVersion());
+
+                                        cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(),
+                                            lastAffVer);
                                     }
                                     else
-                                        cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(), lastFut.initialVersion());
+                                        cctx.exchange().lastAffinityChangedTopologyVersion(exchFut.initialVersion(),
+                                            lastFut.topologyVersion());
                                 }
                             }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java
index 45c6f25bf275a..d2c04c7713527 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheNoAffinityExchangeTest.java
@@ -23,10 +23,19 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
 import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessageV2;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -55,6 +64,8 @@ public class CacheNoAffinityExchangeTest extends GridCommonAbstractTest {
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
 
+        cfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
         cfg.setDiscoverySpi(new TestDiscoverySpi().setIpFinder(IP_FINDER));
 
         if (startClient) {
@@ -190,6 +201,79 @@ public void testNoAffinityChangeOnClientLeft() throws Exception {
         latch.countDown();
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoAffinityChangeOnClientLeftWithMergedExchanges() throws Exception {
+        System.setProperty(IgniteSystemProperties.IGNITE_EXCHANGE_MERGE_DELAY, "1000");
+
+        try {
+            Ignite ig = startGridsMultiThreaded(4);
+
+            IgniteCache<Integer, Integer> atomicCache = ig.createCache(new CacheConfiguration<Integer, Integer>()
+                .setName("atomic").setAtomicityMode(CacheAtomicityMode.ATOMIC).setCacheMode(CacheMode.REPLICATED));
+
+            IgniteCache<Integer, Integer> txCache = ig.createCache(new CacheConfiguration<Integer, Integer>()
+                .setName("tx").setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL).setCacheMode(CacheMode.REPLICATED));
+
+            startClient = true;
+
+            Ignite client = startGrid("client");
+
+            startClient = false;
+
+            stopGrid(1);
+            stopGrid(2);
+            stopGrid(3);
+
+            awaitPartitionMapExchange();
+
+            atomicCache.put(-1, -1);
+            txCache.put(-1, -1);
+
+            TestRecordingCommunicationSpi.spi(ig).blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+                @Override public boolean apply(ClusterNode node, Message message) {
+                    return message instanceof GridDhtPartitionSupplyMessageV2;
+                }
+            });
+
+            startGridsMultiThreaded(1, 3);
+
+            CountDownLatch latch = new CountDownLatch(1);
+            for (Ignite ignite : G.allGrids()) {
+                if (ignite.cluster().localNode().order() == 9) {
+                    TestDiscoverySpi discoSpi =
+                        (TestDiscoverySpi)((IgniteEx)ignite).context().discovery().getInjectedDiscoverySpi();
+
+                    discoSpi.latch = latch;
+
+                    break;
+                }
+            }
+
+            client.close();
+
+            for (int k = 0; k < 100; k++) {
+                atomicCache.put(k, k);
+                txCache.put(k, k);
+
+                Lock lock = txCache.lock(k);
+                lock.lock();
+                lock.unlock();
+            }
+
+            for (int k = 0; k < 100; k++) {
+                assertEquals(Integer.valueOf(k), atomicCache.get(k));
+                assertEquals(Integer.valueOf(k), txCache.get(k));
+            }
+
+            latch.countDown();
+        }
+        finally {
+            System.clearProperty(IgniteSystemProperties.IGNITE_EXCHANGE_MERGE_DELAY);
+        }
+    }
+
     /**
      *
      */

From eaad5015d0aa4c29f1ac19e8514d5efbcf49c1b2 Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Wed, 10 Oct 2018 11:57:11 +0300
Subject: [PATCH 530/543] IGN-12696 Fixed versions in pom.xml, gg ignite
 properties.

---
 examples/pom.xml                                   | 2 +-
 modules/aop/pom.xml                                | 2 +-
 modules/apache-license-gen/pom.xml                 | 2 +-
 modules/aws/pom.xml                                | 2 +-
 modules/benchmarks/pom.xml                         | 2 +-
 modules/camel/pom.xml                              | 2 +-
 modules/cassandra/pom.xml                          | 2 +-
 modules/cassandra/serializers/pom.xml              | 4 ++--
 modules/cassandra/store/pom.xml                    | 4 ++--
 modules/clients/pom.xml                            | 2 +-
 modules/cloud/pom.xml                              | 2 +-
 modules/codegen/pom.xml                            | 2 +-
 modules/compatibility/pom.xml                      | 2 +-
 modules/core/pom.xml                               | 2 +-
 modules/core/src/main/resources/ignite.properties  | 2 +-
 modules/dev-utils/pom.xml                          | 2 +-
 modules/direct-io/pom.xml                          | 2 +-
 modules/extdata/p2p/pom.xml                        | 2 +-
 modules/extdata/platform/pom.xml                   | 2 +-
 modules/extdata/uri/modules/uri-dependency/pom.xml | 2 +-
 modules/extdata/uri/pom.xml                        | 2 +-
 modules/flink/pom.xml                              | 2 +-
 modules/flume/pom.xml                              | 2 +-
 modules/gce/pom.xml                                | 2 +-
 modules/geospatial/pom.xml                         | 2 +-
 modules/hadoop/pom.xml                             | 2 +-
 modules/hibernate-4.2/pom.xml                      | 2 +-
 modules/hibernate-5.1/pom.xml                      | 2 +-
 modules/hibernate-core/pom.xml                     | 2 +-
 modules/ignored-tests/pom.xml                      | 2 +-
 modules/indexing/pom.xml                           | 2 +-
 modules/jcl/pom.xml                                | 2 +-
 modules/jms11/pom.xml                              | 2 +-
 modules/jta/pom.xml                                | 2 +-
 modules/kafka/pom.xml                              | 2 +-
 modules/kubernetes/pom.xml                         | 2 +-
 modules/log4j/pom.xml                              | 2 +-
 modules/log4j2/pom.xml                             | 2 +-
 modules/mesos/pom.xml                              | 2 +-
 modules/ml/pom.xml                                 | 2 +-
 modules/mqtt/pom.xml                               | 2 +-
 modules/osgi-karaf/pom.xml                         | 2 +-
 modules/osgi-paxlogging/pom.xml                    | 2 +-
 modules/osgi/pom.xml                               | 2 +-
 modules/rest-http/pom.xml                          | 2 +-
 modules/rocketmq/pom.xml                           | 2 +-
 modules/scalar-2.10/pom.xml                        | 2 +-
 modules/scalar/pom.xml                             | 2 +-
 modules/schedule/pom.xml                           | 2 +-
 modules/slf4j/pom.xml                              | 2 +-
 modules/spark-2.10/pom.xml                         | 2 +-
 modules/spark/pom.xml                              | 2 +-
 modules/spring-data/pom.xml                        | 2 +-
 modules/spring/pom.xml                             | 2 +-
 modules/sqlline/pom.xml                            | 2 +-
 modules/ssh/pom.xml                                | 2 +-
 modules/storm/pom.xml                              | 2 +-
 modules/tools/pom.xml                              | 2 +-
 modules/twitter/pom.xml                            | 2 +-
 modules/urideploy/pom.xml                          | 2 +-
 modules/visor-console-2.10/pom.xml                 | 2 +-
 modules/visor-console/pom.xml                      | 2 +-
 modules/visor-plugins/pom.xml                      | 2 +-
 modules/web-console/pom.xml                        | 2 +-
 modules/web-console/web-agent/pom.xml              | 2 +-
 modules/web/ignite-appserver-test/pom.xml          | 2 +-
 modules/web/ignite-websphere-test/pom.xml          | 2 +-
 modules/web/pom.xml                                | 2 +-
 modules/yardstick/pom.xml                          | 2 +-
 modules/yarn/pom.xml                               | 2 +-
 modules/zeromq/pom.xml                             | 2 +-
 modules/zookeeper/pom.xml                          | 2 +-
 pom.xml                                            | 2 +-
 73 files changed, 75 insertions(+), 75 deletions(-)

diff --git a/examples/pom.xml b/examples/pom.xml
index 01c5ba1014a06..29fa097f24d17 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -27,7 +27,7 @@
     </parent>
 
     <artifactId>ignite-examples</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/aop/pom.xml b/modules/aop/pom.xml
index 03d9f3582c27f..885667328d89c 100644
--- a/modules/aop/pom.xml
+++ b/modules/aop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aop</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/apache-license-gen/pom.xml b/modules/apache-license-gen/pom.xml
index 4cf4ba3114f34..e2063d7129d6c 100644
--- a/modules/apache-license-gen/pom.xml
+++ b/modules/apache-license-gen/pom.xml
@@ -31,7 +31,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>ignite-apache-license-gen</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <build>
diff --git a/modules/aws/pom.xml b/modules/aws/pom.xml
index 8378dc2c3184e..951b092303b01 100644
--- a/modules/aws/pom.xml
+++ b/modules/aws/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-aws</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/benchmarks/pom.xml b/modules/benchmarks/pom.xml
index 791461df337a4..144d308b9a3d9 100644
--- a/modules/benchmarks/pom.xml
+++ b/modules/benchmarks/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-benchmarks</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/camel/pom.xml b/modules/camel/pom.xml
index 1cd3555ff9fee..0bd50eeb53c13 100644
--- a/modules/camel/pom.xml
+++ b/modules/camel/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-camel</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/pom.xml b/modules/cassandra/pom.xml
index 4a3128c7bed72..bc7dce24f127e 100644
--- a/modules/cassandra/pom.xml
+++ b/modules/cassandra/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-cassandra</artifactId>
     <packaging>pom</packaging>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencyManagement>
diff --git a/modules/cassandra/serializers/pom.xml b/modules/cassandra/serializers/pom.xml
index 2e99b14b7211d..42cd053ec0a7d 100644
--- a/modules/cassandra/serializers/pom.xml
+++ b/modules/cassandra/serializers/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>2.5.1-p161-SNAPSHOT</version>
+        <version>2.5.1-p162-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-serializers</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/cassandra/store/pom.xml b/modules/cassandra/store/pom.xml
index a09e95c649e48..05c976aff5bbb 100644
--- a/modules/cassandra/store/pom.xml
+++ b/modules/cassandra/store/pom.xml
@@ -26,12 +26,12 @@
     <parent>
         <groupId>org.apache.ignite</groupId>
         <artifactId>ignite-cassandra</artifactId>
-        <version>2.5.1-p161-SNAPSHOT</version>
+        <version>2.5.1-p162-SNAPSHOT</version>
         <relativePath>..</relativePath>
     </parent>
 
     <artifactId>ignite-cassandra-store</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/clients/pom.xml b/modules/clients/pom.xml
index 5c9c400fcd40e..dbbb1c9705658 100644
--- a/modules/clients/pom.xml
+++ b/modules/clients/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-clients</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/cloud/pom.xml b/modules/cloud/pom.xml
index 90785ea5cc63a..8b132de655687 100644
--- a/modules/cloud/pom.xml
+++ b/modules/cloud/pom.xml
@@ -29,7 +29,7 @@
     </parent>
 
     <artifactId>ignite-cloud</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/codegen/pom.xml b/modules/codegen/pom.xml
index 9eaf116f838ba..df6218144590a 100644
--- a/modules/codegen/pom.xml
+++ b/modules/codegen/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-codegen</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/compatibility/pom.xml b/modules/compatibility/pom.xml
index fc654c6ce8af5..ec266b71cbca6 100644
--- a/modules/compatibility/pom.xml
+++ b/modules/compatibility/pom.xml
@@ -33,7 +33,7 @@
     </parent>
 
     <artifactId>ignite-compatibility</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 7e70ec181b79c..8571ead1e399b 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-core</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <repositories>
diff --git a/modules/core/src/main/resources/ignite.properties b/modules/core/src/main/resources/ignite.properties
index ed62fa6bd88cf..1cbb314c3a401 100644
--- a/modules/core/src/main/resources/ignite.properties
+++ b/modules/core/src/main/resources/ignite.properties
@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-ignite.version=2.5.1-p161-SNAPSHOT
+ignite.version=2.5.1-p162-SNAPSHOT
 ignite.build=0
 ignite.revision=DEV
 ignite.rel.date=01011970
diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml
index cffe6366c778e..b10e36afd71ee 100644
--- a/modules/dev-utils/pom.xml
+++ b/modules/dev-utils/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-dev-utils</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/direct-io/pom.xml b/modules/direct-io/pom.xml
index a2f93760a4b66..8502b0d254567 100644
--- a/modules/direct-io/pom.xml
+++ b/modules/direct-io/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-direct-io</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/extdata/p2p/pom.xml b/modules/extdata/p2p/pom.xml
index aa27b6f5d65d2..cd5912d632abd 100644
--- a/modules/extdata/p2p/pom.xml
+++ b/modules/extdata/p2p/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-extdata-p2p</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/extdata/platform/pom.xml b/modules/extdata/platform/pom.xml
index 92fb01cc35eda..dafcd91aca6df 100644
--- a/modules/extdata/platform/pom.xml
+++ b/modules/extdata/platform/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-platform</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/extdata/uri/modules/uri-dependency/pom.xml b/modules/extdata/uri/modules/uri-dependency/pom.xml
index 6e9ef0772680b..25cef4fc1a2e6 100644
--- a/modules/extdata/uri/modules/uri-dependency/pom.xml
+++ b/modules/extdata/uri/modules/uri-dependency/pom.xml
@@ -27,7 +27,7 @@
     <artifactId>ignite-extdata-uri-dep</artifactId>
     <packaging>jar</packaging>
 
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <modelVersion>4.0.0</modelVersion>
 
     <dependencies>
diff --git a/modules/extdata/uri/pom.xml b/modules/extdata/uri/pom.xml
index 060cbd27d20d9..e10f8bcb6c5c0 100644
--- a/modules/extdata/uri/pom.xml
+++ b/modules/extdata/uri/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-extdata-uri</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
 
     <dependencies>
         <dependency>
diff --git a/modules/flink/pom.xml b/modules/flink/pom.xml
index 07ffd83b9f6b6..2d31ae77ad6eb 100644
--- a/modules/flink/pom.xml
+++ b/modules/flink/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flink</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/flume/pom.xml b/modules/flume/pom.xml
index 15f94375cb6f5..61f25334d98d6 100644
--- a/modules/flume/pom.xml
+++ b/modules/flume/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-flume</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/gce/pom.xml b/modules/gce/pom.xml
index 4677acd29f742..7fd284cd78da6 100644
--- a/modules/gce/pom.xml
+++ b/modules/gce/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-gce</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/geospatial/pom.xml b/modules/geospatial/pom.xml
index a66605133ca54..92a68f23dfe9e 100644
--- a/modules/geospatial/pom.xml
+++ b/modules/geospatial/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-geospatial</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index 9fe299e9626fc..22e735b2c83b7 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hadoop</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-4.2/pom.xml b/modules/hibernate-4.2/pom.xml
index baa4822bbcc2c..750a142697dc7 100644
--- a/modules/hibernate-4.2/pom.xml
+++ b/modules/hibernate-4.2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate_4.2</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-5.1/pom.xml b/modules/hibernate-5.1/pom.xml
index 06829fcc095fe..d16d6d0fad1ef 100644
--- a/modules/hibernate-5.1/pom.xml
+++ b/modules/hibernate-5.1/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate_5.1</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/hibernate-core/pom.xml b/modules/hibernate-core/pom.xml
index 1a6a68359b820..46b1a8ccb58be 100644
--- a/modules/hibernate-core/pom.xml
+++ b/modules/hibernate-core/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-hibernate-core</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ignored-tests/pom.xml b/modules/ignored-tests/pom.xml
index 9c01eaf188970..d8197ae06df47 100644
--- a/modules/ignored-tests/pom.xml
+++ b/modules/ignored-tests/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ignored-tests</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/indexing/pom.xml b/modules/indexing/pom.xml
index c641abc297228..0c464888e393d 100644
--- a/modules/indexing/pom.xml
+++ b/modules/indexing/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-indexing</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jcl/pom.xml b/modules/jcl/pom.xml
index 19b0523a0e50c..785607630e0a4 100644
--- a/modules/jcl/pom.xml
+++ b/modules/jcl/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jcl</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jms11/pom.xml b/modules/jms11/pom.xml
index 80c2fc8f95aae..8c91c2b6577fb 100644
--- a/modules/jms11/pom.xml
+++ b/modules/jms11/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jms11</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/jta/pom.xml b/modules/jta/pom.xml
index fc30e7a02760e..3246252719ca4 100644
--- a/modules/jta/pom.xml
+++ b/modules/jta/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-jta</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kafka/pom.xml b/modules/kafka/pom.xml
index 3050778e3268f..0f0108488328b 100644
--- a/modules/kafka/pom.xml
+++ b/modules/kafka/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kafka</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/kubernetes/pom.xml b/modules/kubernetes/pom.xml
index 66fa6f9520813..eee8864787f71 100644
--- a/modules/kubernetes/pom.xml
+++ b/modules/kubernetes/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-kubernetes</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j/pom.xml b/modules/log4j/pom.xml
index c467beea93721..24f889a6c1b44 100644
--- a/modules/log4j/pom.xml
+++ b/modules/log4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/log4j2/pom.xml b/modules/log4j2/pom.xml
index dfb39057086ff..5fedc02505554 100644
--- a/modules/log4j2/pom.xml
+++ b/modules/log4j2/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-log4j2</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mesos/pom.xml b/modules/mesos/pom.xml
index 283aef618c534..30314a5f0d5ab 100644
--- a/modules/mesos/pom.xml
+++ b/modules/mesos/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mesos</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/ml/pom.xml b/modules/ml/pom.xml
index e9d7fc83374d5..a6e1ee68ef4e2 100644
--- a/modules/ml/pom.xml
+++ b/modules/ml/pom.xml
@@ -30,7 +30,7 @@
     </parent>
 
     <artifactId>ignite-ml</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/mqtt/pom.xml b/modules/mqtt/pom.xml
index 4e0cab1c073c4..5de4ebe9b9d89 100644
--- a/modules/mqtt/pom.xml
+++ b/modules/mqtt/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-mqtt</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/osgi-karaf/pom.xml b/modules/osgi-karaf/pom.xml
index 10846d9f8990f..8846380322860 100644
--- a/modules/osgi-karaf/pom.xml
+++ b/modules/osgi-karaf/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-karaf</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <build>
diff --git a/modules/osgi-paxlogging/pom.xml b/modules/osgi-paxlogging/pom.xml
index 76b58d35889b5..24dcb2eac30ff 100644
--- a/modules/osgi-paxlogging/pom.xml
+++ b/modules/osgi-paxlogging/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi-paxlogging</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <dependencies>
diff --git a/modules/osgi/pom.xml b/modules/osgi/pom.xml
index 89ae5dd186d4e..f9bddb7101087 100644
--- a/modules/osgi/pom.xml
+++ b/modules/osgi/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-osgi</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/rest-http/pom.xml b/modules/rest-http/pom.xml
index dd18b04a665dc..49463dcefad16 100644
--- a/modules/rest-http/pom.xml
+++ b/modules/rest-http/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-rest-http</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/rocketmq/pom.xml b/modules/rocketmq/pom.xml
index aca233ba8e2f3..ed6339d893d34 100644
--- a/modules/rocketmq/pom.xml
+++ b/modules/rocketmq/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-rocketmq</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar-2.10/pom.xml b/modules/scalar-2.10/pom.xml
index 6f95df562a385..49cfbecffacf7 100644
--- a/modules/scalar-2.10/pom.xml
+++ b/modules/scalar-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar_2.10</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/scalar/pom.xml b/modules/scalar/pom.xml
index 24faae3e3c6e8..ed3037a540b75 100644
--- a/modules/scalar/pom.xml
+++ b/modules/scalar/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-scalar</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/schedule/pom.xml b/modules/schedule/pom.xml
index 432da2363c453..c7aed92cacbcc 100644
--- a/modules/schedule/pom.xml
+++ b/modules/schedule/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-schedule</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/slf4j/pom.xml b/modules/slf4j/pom.xml
index 3f335a7cb17ae..938eacfb81682 100644
--- a/modules/slf4j/pom.xml
+++ b/modules/slf4j/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-slf4j</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark-2.10/pom.xml b/modules/spark-2.10/pom.xml
index 7e3e109491157..cdfcb45e71eaf 100644
--- a/modules/spark-2.10/pom.xml
+++ b/modules/spark-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark_2.10</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spark/pom.xml b/modules/spark/pom.xml
index 817982bb2a625..35d4ace5f0571 100644
--- a/modules/spark/pom.xml
+++ b/modules/spark/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spark</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring-data/pom.xml b/modules/spring-data/pom.xml
index 5c067d03348f1..15db48594ecca 100644
--- a/modules/spring-data/pom.xml
+++ b/modules/spring-data/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring-data</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/spring/pom.xml b/modules/spring/pom.xml
index 97efcffd442be..bc9df5199aff1 100644
--- a/modules/spring/pom.xml
+++ b/modules/spring/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-spring</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/sqlline/pom.xml b/modules/sqlline/pom.xml
index dab0a385390ff..3d032bedaf21c 100644
--- a/modules/sqlline/pom.xml
+++ b/modules/sqlline/pom.xml
@@ -32,7 +32,7 @@
     </parent>
 
     <artifactId>ignite-sqlline</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/ssh/pom.xml b/modules/ssh/pom.xml
index 6b5269d5887e3..636399fc2f0ba 100644
--- a/modules/ssh/pom.xml
+++ b/modules/ssh/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-ssh</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/storm/pom.xml b/modules/storm/pom.xml
index 26df45210cd70..d1b4ea3d6025a 100644
--- a/modules/storm/pom.xml
+++ b/modules/storm/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-storm</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/tools/pom.xml b/modules/tools/pom.xml
index 050754d8eeb14..8e71114194912 100644
--- a/modules/tools/pom.xml
+++ b/modules/tools/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-tools</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/twitter/pom.xml b/modules/twitter/pom.xml
index 1488915048673..3c45608ce3445 100644
--- a/modules/twitter/pom.xml
+++ b/modules/twitter/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-twitter</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/urideploy/pom.xml b/modules/urideploy/pom.xml
index 2e9fef6ca85ac..c25779e974282 100644
--- a/modules/urideploy/pom.xml
+++ b/modules/urideploy/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-urideploy</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console-2.10/pom.xml b/modules/visor-console-2.10/pom.xml
index bc054d423894a..c88067cb50fc1 100644
--- a/modules/visor-console-2.10/pom.xml
+++ b/modules/visor-console-2.10/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console_2.10</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-console/pom.xml b/modules/visor-console/pom.xml
index 99ef68ee42293..2a9e69cd70fe4 100644
--- a/modules/visor-console/pom.xml
+++ b/modules/visor-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-console</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/visor-plugins/pom.xml b/modules/visor-plugins/pom.xml
index 6a5203754c0f2..cada6ad35027f 100644
--- a/modules/visor-plugins/pom.xml
+++ b/modules/visor-plugins/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-visor-plugins</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web-console/pom.xml b/modules/web-console/pom.xml
index 5f538e2527eb2..14c5d4c52d8ba 100644
--- a/modules/web-console/pom.xml
+++ b/modules/web-console/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web-console</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web-console/web-agent/pom.xml b/modules/web-console/web-agent/pom.xml
index 956d0ee7e3fda..840d5b4a46b73 100644
--- a/modules/web-console/web-agent/pom.xml
+++ b/modules/web-console/web-agent/pom.xml
@@ -32,7 +32,7 @@
 
     <artifactId>ignite-web-agent</artifactId>
     <packaging>jar</packaging>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/web/ignite-appserver-test/pom.xml b/modules/web/ignite-appserver-test/pom.xml
index aa4adb3d1f8c0..8d6cd1a07d4db 100644
--- a/modules/web/ignite-appserver-test/pom.xml
+++ b/modules/web/ignite-appserver-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-appserver-test</artifactId>
     <packaging>jar</packaging>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/ignite-websphere-test/pom.xml b/modules/web/ignite-websphere-test/pom.xml
index 4b2802200b299..4e480b7493fc2 100644
--- a/modules/web/ignite-websphere-test/pom.xml
+++ b/modules/web/ignite-websphere-test/pom.xml
@@ -30,7 +30,7 @@
 
     <artifactId>ignite-websphere-test</artifactId>
     <packaging>war</packaging>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/web/pom.xml b/modules/web/pom.xml
index 8dd45d4ac2083..c8408ed4a083f 100644
--- a/modules/web/pom.xml
+++ b/modules/web/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-web</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yardstick/pom.xml b/modules/yardstick/pom.xml
index af9dd3002c03c..8b52571003ed3 100644
--- a/modules/yardstick/pom.xml
+++ b/modules/yardstick/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yardstick</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/modules/yarn/pom.xml b/modules/yarn/pom.xml
index 37a8c34a9240c..b08965550fe5d 100644
--- a/modules/yarn/pom.xml
+++ b/modules/yarn/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-yarn</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zeromq/pom.xml b/modules/zeromq/pom.xml
index 7372db8d2c069..bfc02d4fd3fea 100644
--- a/modules/zeromq/pom.xml
+++ b/modules/zeromq/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zeromq</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <properties>
diff --git a/modules/zookeeper/pom.xml b/modules/zookeeper/pom.xml
index 1bdd085473d35..8527608755219 100644
--- a/modules/zookeeper/pom.xml
+++ b/modules/zookeeper/pom.xml
@@ -31,7 +31,7 @@
     </parent>
 
     <artifactId>ignite-zookeeper</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <url>http://ignite.apache.org</url>
 
     <dependencies>
diff --git a/pom.xml b/pom.xml
index d9baefda700c0..3e2a1853c4a52 100644
--- a/pom.xml
+++ b/pom.xml
@@ -32,7 +32,7 @@
 
     <groupId>org.apache.ignite</groupId>
     <artifactId>apache-ignite</artifactId>
-    <version>2.5.1-p161-SNAPSHOT</version>
+    <version>2.5.1-p162-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <properties>

From 2bf3f048a82292814a3a042622ab61736099fd55 Mon Sep 17 00:00:00 2001
From: Anton Kalashnikov <kaa.dev@yandex.ru>
Date: Tue, 11 Dec 2018 15:39:42 +0300
Subject: [PATCH 531/543] IGNITE-10622 Fixed missed discovery event in the case
 of the next node failure - Fixes #5628.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>
---
 .../ignite/spi/discovery/tcp/ServerImpl.java  |  2 +-
 ...cpDiscoveryPendingMessageDeliveryTest.java | 56 +++++++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index f5e7d14121d93..63a68a74437ed 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -3170,7 +3170,7 @@ else if (!spi.failureDetectionTimeoutEnabled() && (e instanceof
 
                             assert !forceSndPending || msg instanceof TcpDiscoveryNodeLeftMessage;
 
-                            if (failure || forceSndPending) {
+                            if (failure || forceSndPending || newNextNode) {
                                 if (log.isDebugEnabled())
                                     log.debug("Pending messages will be sent [failure=" + failure +
                                         ", newNextNode=" + newNextNode +
diff --git a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java
index 9b3dfeea2e93d..e95c2e8777bf8 100644
--- a/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/spi/discovery/tcp/TcpDiscoveryPendingMessageDeliveryTest.java
@@ -24,11 +24,13 @@
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.managers.discovery.CustomMessageWrapper;
 import org.apache.ignite.internal.managers.discovery.DiscoCache;
 import org.apache.ignite.internal.managers.discovery.DiscoveryCustomMessage;
 import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
 import org.apache.ignite.lang.IgniteUuid;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -72,6 +74,8 @@ public class TcpDiscoveryPendingMessageDeliveryTest extends GridCommonAbstractTe
             disco = new DyingDiscoverySpi();
         else if (igniteInstanceName.startsWith("listener"))
             disco = new ListeningDiscoverySpi();
+        else if (igniteInstanceName.startsWith("receiver"))
+            disco = new DyingThreadDiscoverySpi();
         else
             disco = new TcpDiscoverySpi();
 
@@ -184,6 +188,47 @@ public void testCustomMessageInSingletonCluster() throws Exception {
             }, 10000));
     }
 
+    /**
+     * @throws Exception If failed.
+     */
+    public void testDeliveryAllFailedMessagesInCorrectOrder() throws Exception {
+        IgniteEx coord = (IgniteEx)startGrid("coordinator");
+        TcpDiscoverySpi coordDisco = (TcpDiscoverySpi)coord.configuration().getDiscoverySpi();
+
+        Set<TcpDiscoveryAbstractMessage> sentEnsuredMsgs = new GridConcurrentHashSet<>();
+        coordDisco.addSendMessageListener(msg -> {
+            if (coordDisco.ensured(msg))
+                sentEnsuredMsgs.add(msg);
+        });
+
+        //Node which receive message but will not send it further around the ring.
+        IgniteEx receiver = (IgniteEx)startGrid("receiver");
+
+        //Node which will be failed first.
+        IgniteEx dummy = (IgniteEx)startGrid("dummy");
+
+        //Node which should received all fail message in any way.
+        startGrid("listener");
+
+        sentEnsuredMsgs.clear();
+        receivedEnsuredMsgs.clear();
+
+        blockMsgs = true;
+
+        log.info("Sending fail node messages");
+
+        coord.context().discovery().failNode(dummy.localNode().id(), "Dummy node failed");
+        coord.context().discovery().failNode(receiver.localNode().id(), "Receiver node failed");
+
+        boolean delivered = GridTestUtils.waitForCondition(() -> {
+            log.info("Waiting for messages delivery");
+
+            return receivedEnsuredMsgs.equals(sentEnsuredMsgs);
+        }, 5000);
+
+        assertTrue("Sent: " + sentEnsuredMsgs + "; received: " + receivedEnsuredMsgs, delivered);
+    }
+
     /**
      * @param disco Discovery SPI.
      * @param id Message id.
@@ -192,6 +237,17 @@ private void sendDummyCustomMessage(TcpDiscoverySpi disco, IgniteUuid id) {
         disco.sendCustomEvent(new CustomMessageWrapper(new DummyCustomDiscoveryMessage(id)));
     }
 
+    /**
+     * Discovery SPI, that makes a thread to die when {@code blockMsgs} is set to {@code true}.
+     */
+    private class DyingThreadDiscoverySpi extends TcpDiscoverySpi {
+        /** {@inheritDoc} */
+        @Override protected void startMessageProcess(TcpDiscoveryAbstractMessage msg) {
+            if (blockMsgs)
+                throw new RuntimeException("Thread is dying");
+        }
+    }
+
     /**
      * Discovery SPI, that makes a node stop sending messages when {@code blockMsgs} is set to {@code true}.
      */

From 7f94a3fe7c4398c50db818983e69eea8ef45453e Mon Sep 17 00:00:00 2001
From: deviljelly <deviljelly@gmail.com>
Date: Fri, 30 Nov 2018 16:40:40 +0300
Subject: [PATCH 532/543] IGNITE-9298 SSL support in control.sh - Fixes #5516.

Fixes #4684.
Fixes #4697.

Signed-off-by: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
Co-authored-by: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
Co-authored-by: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
(cherry picked from commit 6d7847c7f1427a337975d240b10bec8b10a8d3f2)
---
 .../internal/commandline/Arguments.java       | 113 +++++-----
 .../internal/commandline/CommandHandler.java  | 209 ++++++++----------
 .../CommandHandlerParsingTest.java            |  39 ++++
 .../IgniteBasicWithPersistenceTestSuite.java  |   2 +
 .../util/GridCommandHandlerSslTest.java       |   6 +-
 5 files changed, 193 insertions(+), 176 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index d769982566320..7fc4ac9ef5c37 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -77,35 +77,32 @@ public class Arguments {
     /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}. */
     private long pingInterval;
 
-    /** */
-    private boolean sslEnable;
-
-    /** */
+    /** SSL Protocol. */
     private String sslProtocol;
 
-    /** */
-    private String sslAlgorithm;
+    /** SSL Key Algorithm. */
+    private String sslKeyAlgorithm;
 
     /** */
     private String sslCipherSuites;
 
-    /** */
+    /** Keystore. */
     private String sslKeyStorePath;
 
-    /** */
+    /** Keystore Type. */
     private String sslKeyStoreType;
 
-    /** */
-    private char sslKeyStorePassword[];
+    /** Keystore Password. */
+    private char[] sslKeyStorePassword;
 
-    /** */
+    /** Truststore. */
     private String sslTrustStorePath;
 
-    /** */
+    /** Truststore Type. */
     private String sslTrustStoreType;
 
-    /** */
-    private char sslTrustStorePassword[];
+    /** Truststore Password. */
+    private char[] sslTrustStorePassword;
 
     /**
      * @param cmd Command.
@@ -122,25 +119,21 @@ public class Arguments {
      * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
      * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
      * @param autoConfirmation Auto confirmation flag.
-     * @param sslEnable SSL flag.
-     * @param sslProtocol SSL protocol.
-     * @param sslAlgorithm SSL algorithm.
+     * @param sslProtocol SSL Protocol.
+     * @param sslKeyAlgorithm SSL Key Algorithm.
      * @param sslCipherSuites SSL cipher suites.
-     * @param sslKeyStorePath Key store path.
-     * @param sslKeyStoreType Key store type.
-     * @param sslKeyStorePassword Key store password.
-     * @param sslTrustStorePath Trust store path.
-     * @param sslTrustStoreType Trust store type.
-     * @param sslTrustStorePassword Trust store password.
+     * @param sslKeyStorePath Keystore.
+     * @param sslKeyStorePassword Keystore Password.
+     * @param sslKeyStoreType Keystore Type.
+     * @param sslTrustStorePath Truststore.
+     * @param sslTrustStorePassword Truststore Password.
+     * @param sslTrustStoreType Truststore Type.
      */
-    public Arguments(Command cmd, String host, String port, String user, String pwd,
-        String baselineAct, String baselineArgs,
-        VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
-        Long pingTimeout, Long pingInterval, boolean autoConfirmation,
-        boolean sslEnable, String sslProtocol, String sslAlgorithm, String sslCipherSuites,
-        String sslKeyStorePath, String sslKeyStoreType, char sslKeyStorePassword[],
-        String sslTrustStorePath, String sslTrustStoreType, char sslTrustStorePassword[]
-    ) {
+    public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
+        String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
+        Long pingTimeout, Long pingInterval, boolean autoConfirmation, String sslProtocol, String sslKeyAlgorithm,
+        String sslCipherSuites, String sslKeyStorePath, char[] sslKeyStorePassword, String sslKeyStoreType,
+        String sslTrustStorePath, char[] sslTrustStorePassword, String sslTrustStoreType) {
         this.cmd = cmd;
         this.host = host;
         this.port = port;
@@ -161,9 +154,8 @@ public Arguments(Command cmd, String host, String port, String user, String pwd,
 
         this.autoConfirmation = autoConfirmation;
 
-        this.sslEnable = sslEnable;
         this.sslProtocol = sslProtocol;
-        this.sslAlgorithm = sslAlgorithm;
+        this.sslKeyAlgorithm = sslKeyAlgorithm;
         this.sslCipherSuites = sslCipherSuites;
 
         this.sslKeyStorePath = sslKeyStorePath;
@@ -291,19 +283,18 @@ public boolean autoConfirmation() {
         return autoConfirmation;
     }
 
-    /** */
-    public boolean isSslEnable() {
-        return sslEnable;
-    }
-
-    /** */
-    public String getSslProtocol() {
+    /**
+     * @return SSL protocol
+     */
+    public String sslProtocol() {
         return sslProtocol;
     }
 
-    /** */
-    public String getSslAlgorithm() {
-        return sslAlgorithm;
+    /**
+     * @return SSL Key Algorithm
+     */
+    public String sslKeyAlgorithm() {
+        return sslKeyAlgorithm;
     }
 
     /** */
@@ -311,33 +302,45 @@ public String getSslCipherSuites() {
         return sslCipherSuites;
     }
 
-    /** */
-    public String getSslKeyStorePath() {
+    /**
+     * @return Keystore
+     */
+    public String sslKeyStorePath() {
         return sslKeyStorePath;
     }
 
-    /** */
-    public String getSslKeyStoreType() {
+    /**
+     * @return Keystore type
+     */
+    public String sslKeyStoreType() {
         return sslKeyStoreType;
     }
 
-    /** */
-    public char[] getSslKeyStorePassword() {
+    /**
+     * @return Keystore password
+     */
+    public char[] sslKeyStorePassword() {
         return sslKeyStorePassword;
     }
 
-    /** */
-    public String getSslTrustStorePath() {
+    /**
+     * @return Truststore
+     */
+    public String sslTrustStorePath() {
         return sslTrustStorePath;
     }
 
-    /** */
-    public String getSslTrustStoreType() {
+    /**
+     * @return Truststore type
+     */
+    public String sslTrustStoreType() {
         return sslTrustStoreType;
     }
 
-    /** */
-    public char[] getSslTrustStorePassword() {
+    /**
+     * @return Truststore password
+     */
+    public char[] sslTrustStorePassword() {
         return sslTrustStorePassword;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index cf980dab3d6d3..a05ce9ef87d3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -195,40 +195,37 @@ public class CommandHandler {
     /** */
     private static final String CMD_SKIP_ZEROS = "--skipZeros";
 
-    // SSL configuration section
-
     /** */
-    protected static final String CMD_SSL_ENABLED = "--ssl_enabled";
+    private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
 
-    /** */
-    protected static final String CMD_SSL_PROTOCOL = "--ssl_protocol";
+    // SSL configuration section
 
     /** */
-    protected static final String CMD_SSL_ALGORITHM = "--ssl_algorithm";
+    private static final String CMD_SSL_PROTOCOL = "--ssl-protocol";
 
     /** */
-    protected static final String CMD_SSL_CIPHER_SUITES = "--ssl_cipher_suites";
+    private static final String CMD_SSL_KEY_ALGORITHM = "--ssl-key-algorithm";
 
     /** */
-    protected static final String CMD_SSL_KEY_STORE_PATH = "--ssl_key_store_path";
+    protected static final String CMD_SSL_CIPHER_SUITES = "--ssl-cipher-suites";
 
     /** */
-    protected static final String CMD_SSL_KEY_STORE_TYPE = "--ssl_key_store_type";
+    private static final String CMD_KEYSTORE = "--keystore";
 
     /** */
-    protected static final String CMD_SSL_KEY_STORE_PASSWORD = "--ssl_key_store_password";
+    private static final String CMD_KEYSTORE_PASSWORD = "--keystore-password";
 
     /** */
-    protected static final String CMD_SSL_TRUSTSTORE_PATH = "--ssl_truststore_path";
+    private static final String CMD_KEYSTORE_TYPE = "--keystore-type";
 
     /** */
-    protected static final String CMD_SSL_TRUSTSTORE_TYPE = "--ssl_truststore_type";
+    private static final String CMD_TRUSTSTORE = "--truststore";
 
     /** */
-    protected static final String CMD_SSL_TRUSTSTORE_PASSWORD = "--ssl_truststore_password";
+    private static final String CMD_TRUSTSTORE_PASSWORD = "--truststore-password";
 
     /** */
-    private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
+    private static final String CMD_TRUSTSTORE_TYPE = "--truststore-type";
 
     /** List of optional auxiliary commands. */
     private static final Set<String> AUX_COMMANDS = new HashSet<>();
@@ -247,18 +244,17 @@ public class CommandHandler {
         AUX_COMMANDS.add(CMD_PING_INTERVAL);
         AUX_COMMANDS.add(CMD_PING_TIMEOUT);
 
-        AUX_COMMANDS.add(CMD_SSL_ENABLED);
         AUX_COMMANDS.add(CMD_SSL_PROTOCOL);
-        AUX_COMMANDS.add(CMD_SSL_ALGORITHM);
+        AUX_COMMANDS.add(CMD_SSL_KEY_ALGORITHM);
         AUX_COMMANDS.add(CMD_SSL_CIPHER_SUITES);
 
-        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_TYPE);
-        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PATH);
-        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PASSWORD);
+        AUX_COMMANDS.add(CMD_KEYSTORE);
+        AUX_COMMANDS.add(CMD_KEYSTORE_PASSWORD);
+        AUX_COMMANDS.add(CMD_KEYSTORE_TYPE);
 
-        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_TYPE);
-        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PATH);
-        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PASSWORD);
+        AUX_COMMANDS.add(CMD_TRUSTSTORE);
+        AUX_COMMANDS.add(CMD_TRUSTSTORE_PASSWORD);
+        AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE);
     }
 
     /** Broadcast uuid. */
@@ -407,16 +403,15 @@ private static List<String> getCommonOptions() {
         list.add(op(CMD_PASSWORD, "PASSWORD"));
         list.add(op(CMD_PING_INTERVAL, "PING_INTERVAL"));
         list.add(op(CMD_PING_TIMEOUT, "PING_TIMEOUT"));
-        list.add(op(CMD_SSL_ENABLED));
         list.add(op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ...]"));
-        list.add(op(CMD_SSL_ALGORITHM, "SSL_ALGORITHM"));
+        list.add(op(CMD_SSL_KEY_ALGORITHM, "SSL_KEY_ALGORITHM"));
         list.add(op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ...]"));
-        list.add(op(CMD_SSL_KEY_STORE_TYPE, "SSL_KEY_STORE_TYPE"));
-        list.add(op(CMD_SSL_KEY_STORE_PATH, "PATH"));
-        list.add(op(CMD_SSL_KEY_STORE_PASSWORD, "PASSWORD"));
-        list.add(op(CMD_SSL_TRUSTSTORE_TYPE, "SSL_TRUST_STORE_TYPE"));
-        list.add(op(CMD_SSL_TRUSTSTORE_PATH, "PATH"));
-        list.add(op(CMD_SSL_TRUSTSTORE_PASSWORD, "PASSWORD"));
+        list.add(op(CMD_KEYSTORE, "KEYSTORE"));
+        list.add(op(CMD_KEYSTORE_TYPE, "KEYSTORE_TYPE"));
+        list.add(op(CMD_KEYSTORE_PASSWORD, "KEYSTORE_PASSWORD"));
+        list.add(op(CMD_TRUSTSTORE, "TRUSTSTORE"));
+        list.add(op(CMD_TRUSTSTORE_TYPE, "TRUSTSTORE_TYPE"));
+        list.add(op(CMD_TRUSTSTORE_PASSWORD, "TRUSTSTORE_PASSWORD"));
 
         return list;
     }
@@ -685,14 +680,15 @@ private Stream<IgniteBiTuple<GridClientNode, String>> listHosts(GridClient clien
      * @return List of hosts.
      */
     private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNode(
-        GridClient client) throws GridClientException {
+        GridClient client
+    ) throws GridClientException {
         return client.compute().nodes(GridClientNode::connectable).stream()
             .map(node -> new IgniteBiTuple<>(node,
                 Stream.concat(
                     node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
                     node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
                 )
-                    .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
+                .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
     }
 
     /**
@@ -703,7 +699,6 @@ private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNod
      * @return Task result.
      * @throws GridClientException If failed to execute task.
      */
-    @SuppressWarnings("unchecked")
     private <R> R executeTaskByNameOnNode(
         GridClient client,
         String taskClsName,
@@ -1856,24 +1851,22 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         VisorTxTaskArg txArgs = null;
 
-        boolean sslEnable = false;
-
         String sslProtocol = SslContextFactory.DFLT_SSL_PROTOCOL;
 
-        String sslAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM;
+        String sslKeyAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM;
 
         String sslCipherSuites = "";
 
-        String sslKeyStoreType = SslContextFactory.DFLT_STORE_TYPE;
-
         String sslKeyStorePath = null;
 
-        char sslKeyStorePassword[] = null;
+        String sslKeyStoreType = SslContextFactory.DFLT_STORE_TYPE;
 
-        String sslTrustStoreType = SslContextFactory.DFLT_STORE_TYPE;
+        char sslKeyStorePassword[] = null;
 
         String sslTrustStorePath = null;
 
+        String sslTrustStoreType = SslContextFactory.DFLT_STORE_TYPE;
+
         char sslTrustStorePassword[] = null;
 
         while (hasNextArg()) {
@@ -1988,23 +1981,13 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
-                    case CMD_AUTO_CONFIRMATION:
-                        autoConfirmation = true;
-
-                        break;
-
-                    case CMD_SSL_ENABLED:
-                        sslEnable = true;
-
-                        break;
-
                     case CMD_SSL_PROTOCOL:
                         sslProtocol = nextArg("Expected SSL protocol");
 
                         break;
 
-                    case CMD_SSL_ALGORITHM:
-                        sslAlgorithm = nextArg("Expected SSL algorithm");
+                    case CMD_SSL_KEY_ALGORITHM:
+                        sslKeyAlgorithm = nextArg("Expected SSL key algorithm");
 
                         break;
 
@@ -2013,33 +1996,38 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
-                    case CMD_SSL_KEY_STORE_PATH:
-                        sslKeyStorePath = nextArg("Expected SSL key store path");
+                    case CMD_KEYSTORE:
+                        sslKeyStorePath = nextArg("Expected keystore path");
+
+                        break;
+
+                    case CMD_KEYSTORE_PASSWORD:
+                        sslKeyStorePassword = nextArg("Expected keystore password").toCharArray();
 
                         break;
 
-                    case CMD_SSL_KEY_STORE_TYPE:
-                        sslKeyStoreType = nextArg("Expected SSL key store type");
+                    case CMD_KEYSTORE_TYPE:
+                        sslKeyStoreType = nextArg("Expected keystore type");
 
                         break;
 
-                    case CMD_SSL_KEY_STORE_PASSWORD:
-                        sslKeyStorePassword = nextArg("Expected SSL key store password").toCharArray();
+                    case CMD_TRUSTSTORE:
+                        sslTrustStorePath = nextArg("Expected truststore path");
 
                         break;
 
-                    case CMD_SSL_TRUSTSTORE_PATH:
-                        sslTrustStorePath = nextArg("Expected SSL trust store path");
+                    case CMD_TRUSTSTORE_PASSWORD:
+                        sslTrustStorePassword = nextArg("Expected truststore password").toCharArray();
 
                         break;
 
-                    case CMD_SSL_TRUSTSTORE_TYPE:
-                        sslTrustStoreType = nextArg("Expected SSL trust store type");
+                    case CMD_TRUSTSTORE_TYPE:
+                        sslTrustStoreType = nextArg("Expected truststore type");
 
                         break;
 
-                    case CMD_SSL_TRUSTSTORE_PASSWORD:
-                        sslTrustStorePassword = nextArg("Expected SSL trust store password").toCharArray();
+                    case CMD_AUTO_CONFIRMATION:
+                        autoConfirmation = true;
 
                         break;
 
@@ -2059,14 +2047,10 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         Command cmd = commands.get(0);
 
-        return new Arguments(cmd, host, port, user, pwd,
-            baselineAct, baselineArgs,
-            txArgs, cacheArgs,
-            walAct, walArgs,
+        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
             pingTimeout, pingInterval, autoConfirmation,
-            sslEnable, sslProtocol, sslAlgorithm, sslCipherSuites,
-            sslKeyStorePath, sslKeyStoreType, sslKeyStorePassword,
-            sslTrustStorePath, sslTrustStoreType, sslTrustStorePassword);
+            sslProtocol, sslKeyAlgorithm, sslCipherSuites, sslKeyStorePath, sslKeyStorePassword, sslKeyStoreType,
+            sslTrustStorePath, sslTrustStorePassword, sslTrustStoreType);
     }
 
     /**
@@ -2632,9 +2616,9 @@ private void printHelp() {
         log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2));
         log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2));
         log(i("SSL_PROTOCOL=" + SslContextFactory.DFLT_SSL_PROTOCOL, 2));
-        log(i("SSL_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2));
-        log(i("SSL_KEY_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
-        log(i("SSL_TRUST_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
+        log(i("SSL_KEY_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2));
+        log(i("KEYSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
+        log(i("TRUSTSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
 
         nl();
 
@@ -2687,49 +2671,6 @@ public int execute(List<String> rawArgs) {
 
             clientCfg.setServers(Collections.singletonList(args.host() + ":" + args.port()));
 
-            if (args.isSslEnable()) {
-                GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
-
-                List<String> sslProtocols = split(args.getSslProtocol(), ",");
-
-                String sslProtocol = F.isEmpty(sslProtocols) ? SslContextFactory.DFLT_SSL_PROTOCOL : sslProtocols.get(0);
-
-                factory.setProtocol(sslProtocol);
-                factory.setKeyAlgorithm(args.getSslAlgorithm());
-
-                if (sslProtocols.size() > 1)
-                    factory.setProtocols(sslProtocols);
-
-                factory.setCipherSuites(split(args.getSslCipherSuites(), ","));
-
-                if (args.getSslKeyStorePath() == null)
-                    throw new IllegalArgumentException("SSL key store location is not specified.");
-
-                factory.setKeyStoreFilePath(args.getSslKeyStorePath());
-
-                if (args.getSslKeyStorePassword() != null)
-                    factory.setKeyStorePassword(args.getSslKeyStorePassword());
-                else
-                    factory.setKeyStorePassword(requestPasswordFromConsole("SSL keystore password: "));
-
-                factory.setKeyStoreType(args.getSslKeyStoreType());
-
-                if (args.getSslTrustStorePath() == null)
-                    factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
-                else {
-                    factory.setTrustStoreFilePath(args.getSslTrustStorePath());
-
-                    if (args.getSslTrustStorePassword() != null)
-                        factory.setTrustStorePassword(args.getSslTrustStorePassword());
-                    else
-                        factory.setTrustStorePassword(requestPasswordFromConsole("SSL truststore password: "));
-
-                    factory.setTrustStoreType(args.getSslTrustStoreType());
-                }
-
-                clientCfg.setSslContextFactory(factory);
-            }
-
             boolean tryConnectAgain = true;
 
             int tryConnectMaxCount = 3;
@@ -2752,6 +2693,40 @@ public int execute(List<String> rawArgs) {
                     credential.setPassword(args.getPassword());
                 }
 
+                if (!F.isEmpty(args.sslKeyStorePath())) {
+                    GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
+
+                    factory.setProtocol(args.sslProtocol());
+
+                    factory.setKeyAlgorithm(args.sslKeyAlgorithm());
+
+                    factory.setCipherSuites(split(args.getSslCipherSuites(), ","));
+
+                    factory.setKeyStoreFilePath(args.sslKeyStorePath());
+
+                    if (args.sslKeyStorePassword() != null)
+                        factory.setKeyStorePassword(args.sslKeyStorePassword());
+                    else
+                        factory.setKeyStorePassword(requestPasswordFromConsole("SSL keystore password: "));
+
+                    factory.setKeyStoreType(args.sslKeyStoreType());
+
+                    if (F.isEmpty(args.sslTrustStorePath()))
+                        factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
+                    else {
+                        factory.setTrustStoreFilePath(args.sslTrustStorePath());
+
+                        if (args.sslTrustStorePassword() != null)
+                            factory.setTrustStorePassword(args.sslTrustStorePassword());
+                        else
+                            factory.setTrustStorePassword(requestPasswordFromConsole("SSL truststore password: "));
+
+                        factory.setTrustStoreType(args.sslTrustStoreType());
+                    }
+
+                    clientCfg.setSslContextFactory(factory);
+                }
+
                 try (GridClient client = GridClientFactory.start(clientCfg)) {
                     switch (args.command()) {
                         case ACTIVATE:
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 988b1d7e94a40..203d6d06d0ba3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -38,6 +38,7 @@
 import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_THROUGH;
 import static org.apache.ignite.internal.commandline.CommandHandler.WAL_DELETE;
 import static org.apache.ignite.internal.commandline.CommandHandler.WAL_PRINT;
+import static org.junit.Assert.assertArrayEquals;
 
 /**
  * Tests Command Handler parsing arguments.
@@ -166,6 +167,44 @@ public void testExperimentalCommandIsDisabled() {
         }
     }
 
+    /**
+     * Tests parsing and validation for the SSL arguments.
+     */
+    public void testParseAndValidateSSLArguments() {
+        CommandHandler hnd = new CommandHandler();
+
+        for (Command cmd : Command.values()) {
+
+            if (cmd == Command.CACHE || cmd == Command.WAL)
+                continue; // --cache subcommand requires its own specific arguments.
+
+            try {
+                hnd.parseAndValidate(asList("--truststore"));
+
+                fail("expected exception: Expected truststore");
+            }
+            catch (IllegalArgumentException e) {
+                e.printStackTrace();
+            }
+
+            Arguments args = hnd.parseAndValidate(asList("--keystore", "testKeystore", "--keystore-password", "testKeystorePassword", "--keystore-type", "testKeystoreType",
+                "--truststore", "testTruststore", "--truststore-password", "testTruststorePassword", "--truststore-type", "testTruststoreType",
+                "--ssl-key-algorithm", "testSSLKeyAlgorithm", "--ssl-protocol", "testSSLProtocol", cmd.text()));
+
+            assertEquals("testSSLProtocol", args.sslProtocol());
+            assertEquals("testSSLKeyAlgorithm", args.sslKeyAlgorithm());
+            assertEquals("testKeystore", args.sslKeyStorePath());
+            assertArrayEquals("testKeystorePassword".toCharArray(), args.sslKeyStorePassword());
+            assertEquals("testKeystoreType", args.sslKeyStoreType());
+            assertEquals("testTruststore", args.sslTrustStorePath());
+            assertArrayEquals("testTruststorePassword".toCharArray(), args.sslTrustStorePassword());
+            assertEquals("testTruststoreType", args.sslTrustStoreType());
+
+            assertEquals(cmd, args.command());
+        }
+    }
+
+
     /**
      * Tests parsing and validation for user and password arguments.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
index 7b4b4c098b85e..35ef5966e7732 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest;
 import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest;
 import org.apache.ignite.marshaller.GridMarshallerMappingConsistencyTest;
+import org.apache.ignite.util.GridCommandHandlerSslTest;
 import org.apache.ignite.util.GridCommandHandlerTest;
 import org.apache.ignite.util.GridInternalTaskUnusedWalSegmentsTest;
 import org.jetbrains.annotations.Nullable;
@@ -56,6 +57,7 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(SystemWorkersTerminationTest.class);
 
         suite.addTestSuite(GridCommandHandlerTest.class);
+        suite.addTestSuite(GridCommandHandlerSslTest.class);
         suite.addTestSuite(GridInternalTaskUnusedWalSegmentsTest.class);
 
         return suite;
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
index fb4092af9b968..59155e5b3713a 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
@@ -33,7 +33,6 @@
  * Command line handler test.
  */
 public class GridCommandHandlerSslTest extends GridCommonAbstractTest {
-
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         cleanPersistenceDir();
@@ -78,9 +77,8 @@ public void testActivate() throws Exception {
         final CommandHandler cmd = new CommandHandler();
         assertEquals(EXIT_CODE_OK, cmd.execute(Arrays.asList(
             "--activate",
-            "--ssl_enabled",
-            "--ssl_key_store_path", GridTestUtils.keyStorePath("node01"),
-            "--ssl_key_store_password", GridTestUtils.keyStorePassword())));
+            "--keystore", GridTestUtils.keyStorePath("node01"),
+            "--keystore-password", GridTestUtils.keyStorePassword())));
 
         assertTrue(ignite.cluster().active());
 

From 2e1a2c9ecefd363f2347c36d122b33a48fd144a5 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Wed, 12 Dec 2018 13:13:16 +0300
Subject: [PATCH 533/543] Revert "IGNITE-9298 SSL support in control.sh - Fixes
 #5516."

This reverts commit 7f94a3fe7c4398c50db818983e69eea8ef45453e.
---
 .../internal/commandline/Arguments.java       | 113 +++++-----
 .../internal/commandline/CommandHandler.java  | 209 ++++++++++--------
 .../CommandHandlerParsingTest.java            |  39 ----
 .../IgniteBasicWithPersistenceTestSuite.java  |   2 -
 .../util/GridCommandHandlerSslTest.java       |   6 +-
 5 files changed, 176 insertions(+), 193 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index 7fc4ac9ef5c37..d769982566320 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -77,32 +77,35 @@ public class Arguments {
     /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}. */
     private long pingInterval;
 
-    /** SSL Protocol. */
+    /** */
+    private boolean sslEnable;
+
+    /** */
     private String sslProtocol;
 
-    /** SSL Key Algorithm. */
-    private String sslKeyAlgorithm;
+    /** */
+    private String sslAlgorithm;
 
     /** */
     private String sslCipherSuites;
 
-    /** Keystore. */
+    /** */
     private String sslKeyStorePath;
 
-    /** Keystore Type. */
+    /** */
     private String sslKeyStoreType;
 
-    /** Keystore Password. */
-    private char[] sslKeyStorePassword;
+    /** */
+    private char sslKeyStorePassword[];
 
-    /** Truststore. */
+    /** */
     private String sslTrustStorePath;
 
-    /** Truststore Type. */
+    /** */
     private String sslTrustStoreType;
 
-    /** Truststore Password. */
-    private char[] sslTrustStorePassword;
+    /** */
+    private char sslTrustStorePassword[];
 
     /**
      * @param cmd Command.
@@ -119,21 +122,25 @@ public class Arguments {
      * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
      * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
      * @param autoConfirmation Auto confirmation flag.
-     * @param sslProtocol SSL Protocol.
-     * @param sslKeyAlgorithm SSL Key Algorithm.
+     * @param sslEnable SSL flag.
+     * @param sslProtocol SSL protocol.
+     * @param sslAlgorithm SSL algorithm.
      * @param sslCipherSuites SSL cipher suites.
-     * @param sslKeyStorePath Keystore.
-     * @param sslKeyStorePassword Keystore Password.
-     * @param sslKeyStoreType Keystore Type.
-     * @param sslTrustStorePath Truststore.
-     * @param sslTrustStorePassword Truststore Password.
-     * @param sslTrustStoreType Truststore Type.
+     * @param sslKeyStorePath Key store path.
+     * @param sslKeyStoreType Key store type.
+     * @param sslKeyStorePassword Key store password.
+     * @param sslTrustStorePath Trust store path.
+     * @param sslTrustStoreType Trust store type.
+     * @param sslTrustStorePassword Trust store password.
      */
-    public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
-        String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
-        Long pingTimeout, Long pingInterval, boolean autoConfirmation, String sslProtocol, String sslKeyAlgorithm,
-        String sslCipherSuites, String sslKeyStorePath, char[] sslKeyStorePassword, String sslKeyStoreType,
-        String sslTrustStorePath, char[] sslTrustStorePassword, String sslTrustStoreType) {
+    public Arguments(Command cmd, String host, String port, String user, String pwd,
+        String baselineAct, String baselineArgs,
+        VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
+        Long pingTimeout, Long pingInterval, boolean autoConfirmation,
+        boolean sslEnable, String sslProtocol, String sslAlgorithm, String sslCipherSuites,
+        String sslKeyStorePath, String sslKeyStoreType, char sslKeyStorePassword[],
+        String sslTrustStorePath, String sslTrustStoreType, char sslTrustStorePassword[]
+    ) {
         this.cmd = cmd;
         this.host = host;
         this.port = port;
@@ -154,8 +161,9 @@ public Arguments(Command cmd, String host, String port, String user, String pwd,
 
         this.autoConfirmation = autoConfirmation;
 
+        this.sslEnable = sslEnable;
         this.sslProtocol = sslProtocol;
-        this.sslKeyAlgorithm = sslKeyAlgorithm;
+        this.sslAlgorithm = sslAlgorithm;
         this.sslCipherSuites = sslCipherSuites;
 
         this.sslKeyStorePath = sslKeyStorePath;
@@ -283,18 +291,19 @@ public boolean autoConfirmation() {
         return autoConfirmation;
     }
 
-    /**
-     * @return SSL protocol
-     */
-    public String sslProtocol() {
+    /** */
+    public boolean isSslEnable() {
+        return sslEnable;
+    }
+
+    /** */
+    public String getSslProtocol() {
         return sslProtocol;
     }
 
-    /**
-     * @return SSL Key Algorithm
-     */
-    public String sslKeyAlgorithm() {
-        return sslKeyAlgorithm;
+    /** */
+    public String getSslAlgorithm() {
+        return sslAlgorithm;
     }
 
     /** */
@@ -302,45 +311,33 @@ public String getSslCipherSuites() {
         return sslCipherSuites;
     }
 
-    /**
-     * @return Keystore
-     */
-    public String sslKeyStorePath() {
+    /** */
+    public String getSslKeyStorePath() {
         return sslKeyStorePath;
     }
 
-    /**
-     * @return Keystore type
-     */
-    public String sslKeyStoreType() {
+    /** */
+    public String getSslKeyStoreType() {
         return sslKeyStoreType;
     }
 
-    /**
-     * @return Keystore password
-     */
-    public char[] sslKeyStorePassword() {
+    /** */
+    public char[] getSslKeyStorePassword() {
         return sslKeyStorePassword;
     }
 
-    /**
-     * @return Truststore
-     */
-    public String sslTrustStorePath() {
+    /** */
+    public String getSslTrustStorePath() {
         return sslTrustStorePath;
     }
 
-    /**
-     * @return Truststore type
-     */
-    public String sslTrustStoreType() {
+    /** */
+    public String getSslTrustStoreType() {
         return sslTrustStoreType;
     }
 
-    /**
-     * @return Truststore password
-     */
-    public char[] sslTrustStorePassword() {
+    /** */
+    public char[] getSslTrustStorePassword() {
         return sslTrustStorePassword;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index a05ce9ef87d3f..cf980dab3d6d3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -195,37 +195,40 @@ public class CommandHandler {
     /** */
     private static final String CMD_SKIP_ZEROS = "--skipZeros";
 
+    // SSL configuration section
+
     /** */
-    private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
+    protected static final String CMD_SSL_ENABLED = "--ssl_enabled";
 
-    // SSL configuration section
+    /** */
+    protected static final String CMD_SSL_PROTOCOL = "--ssl_protocol";
 
     /** */
-    private static final String CMD_SSL_PROTOCOL = "--ssl-protocol";
+    protected static final String CMD_SSL_ALGORITHM = "--ssl_algorithm";
 
     /** */
-    private static final String CMD_SSL_KEY_ALGORITHM = "--ssl-key-algorithm";
+    protected static final String CMD_SSL_CIPHER_SUITES = "--ssl_cipher_suites";
 
     /** */
-    protected static final String CMD_SSL_CIPHER_SUITES = "--ssl-cipher-suites";
+    protected static final String CMD_SSL_KEY_STORE_PATH = "--ssl_key_store_path";
 
     /** */
-    private static final String CMD_KEYSTORE = "--keystore";
+    protected static final String CMD_SSL_KEY_STORE_TYPE = "--ssl_key_store_type";
 
     /** */
-    private static final String CMD_KEYSTORE_PASSWORD = "--keystore-password";
+    protected static final String CMD_SSL_KEY_STORE_PASSWORD = "--ssl_key_store_password";
 
     /** */
-    private static final String CMD_KEYSTORE_TYPE = "--keystore-type";
+    protected static final String CMD_SSL_TRUSTSTORE_PATH = "--ssl_truststore_path";
 
     /** */
-    private static final String CMD_TRUSTSTORE = "--truststore";
+    protected static final String CMD_SSL_TRUSTSTORE_TYPE = "--ssl_truststore_type";
 
     /** */
-    private static final String CMD_TRUSTSTORE_PASSWORD = "--truststore-password";
+    protected static final String CMD_SSL_TRUSTSTORE_PASSWORD = "--ssl_truststore_password";
 
     /** */
-    private static final String CMD_TRUSTSTORE_TYPE = "--truststore-type";
+    private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
 
     /** List of optional auxiliary commands. */
     private static final Set<String> AUX_COMMANDS = new HashSet<>();
@@ -244,17 +247,18 @@ public class CommandHandler {
         AUX_COMMANDS.add(CMD_PING_INTERVAL);
         AUX_COMMANDS.add(CMD_PING_TIMEOUT);
 
+        AUX_COMMANDS.add(CMD_SSL_ENABLED);
         AUX_COMMANDS.add(CMD_SSL_PROTOCOL);
-        AUX_COMMANDS.add(CMD_SSL_KEY_ALGORITHM);
+        AUX_COMMANDS.add(CMD_SSL_ALGORITHM);
         AUX_COMMANDS.add(CMD_SSL_CIPHER_SUITES);
 
-        AUX_COMMANDS.add(CMD_KEYSTORE);
-        AUX_COMMANDS.add(CMD_KEYSTORE_PASSWORD);
-        AUX_COMMANDS.add(CMD_KEYSTORE_TYPE);
+        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_TYPE);
+        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PATH);
+        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PASSWORD);
 
-        AUX_COMMANDS.add(CMD_TRUSTSTORE);
-        AUX_COMMANDS.add(CMD_TRUSTSTORE_PASSWORD);
-        AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE);
+        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_TYPE);
+        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PATH);
+        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PASSWORD);
     }
 
     /** Broadcast uuid. */
@@ -403,15 +407,16 @@ private static List<String> getCommonOptions() {
         list.add(op(CMD_PASSWORD, "PASSWORD"));
         list.add(op(CMD_PING_INTERVAL, "PING_INTERVAL"));
         list.add(op(CMD_PING_TIMEOUT, "PING_TIMEOUT"));
+        list.add(op(CMD_SSL_ENABLED));
         list.add(op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ...]"));
-        list.add(op(CMD_SSL_KEY_ALGORITHM, "SSL_KEY_ALGORITHM"));
+        list.add(op(CMD_SSL_ALGORITHM, "SSL_ALGORITHM"));
         list.add(op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ...]"));
-        list.add(op(CMD_KEYSTORE, "KEYSTORE"));
-        list.add(op(CMD_KEYSTORE_TYPE, "KEYSTORE_TYPE"));
-        list.add(op(CMD_KEYSTORE_PASSWORD, "KEYSTORE_PASSWORD"));
-        list.add(op(CMD_TRUSTSTORE, "TRUSTSTORE"));
-        list.add(op(CMD_TRUSTSTORE_TYPE, "TRUSTSTORE_TYPE"));
-        list.add(op(CMD_TRUSTSTORE_PASSWORD, "TRUSTSTORE_PASSWORD"));
+        list.add(op(CMD_SSL_KEY_STORE_TYPE, "SSL_KEY_STORE_TYPE"));
+        list.add(op(CMD_SSL_KEY_STORE_PATH, "PATH"));
+        list.add(op(CMD_SSL_KEY_STORE_PASSWORD, "PASSWORD"));
+        list.add(op(CMD_SSL_TRUSTSTORE_TYPE, "SSL_TRUST_STORE_TYPE"));
+        list.add(op(CMD_SSL_TRUSTSTORE_PATH, "PATH"));
+        list.add(op(CMD_SSL_TRUSTSTORE_PASSWORD, "PASSWORD"));
 
         return list;
     }
@@ -680,15 +685,14 @@ private Stream<IgniteBiTuple<GridClientNode, String>> listHosts(GridClient clien
      * @return List of hosts.
      */
     private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNode(
-        GridClient client
-    ) throws GridClientException {
+        GridClient client) throws GridClientException {
         return client.compute().nodes(GridClientNode::connectable).stream()
             .map(node -> new IgniteBiTuple<>(node,
                 Stream.concat(
                     node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
                     node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
                 )
-                .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
+                    .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
     }
 
     /**
@@ -699,6 +703,7 @@ private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNod
      * @return Task result.
      * @throws GridClientException If failed to execute task.
      */
+    @SuppressWarnings("unchecked")
     private <R> R executeTaskByNameOnNode(
         GridClient client,
         String taskClsName,
@@ -1851,22 +1856,24 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         VisorTxTaskArg txArgs = null;
 
+        boolean sslEnable = false;
+
         String sslProtocol = SslContextFactory.DFLT_SSL_PROTOCOL;
 
-        String sslKeyAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM;
+        String sslAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM;
 
         String sslCipherSuites = "";
 
-        String sslKeyStorePath = null;
-
         String sslKeyStoreType = SslContextFactory.DFLT_STORE_TYPE;
 
-        char sslKeyStorePassword[] = null;
+        String sslKeyStorePath = null;
 
-        String sslTrustStorePath = null;
+        char sslKeyStorePassword[] = null;
 
         String sslTrustStoreType = SslContextFactory.DFLT_STORE_TYPE;
 
+        String sslTrustStorePath = null;
+
         char sslTrustStorePassword[] = null;
 
         while (hasNextArg()) {
@@ -1981,13 +1988,23 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
+                    case CMD_AUTO_CONFIRMATION:
+                        autoConfirmation = true;
+
+                        break;
+
+                    case CMD_SSL_ENABLED:
+                        sslEnable = true;
+
+                        break;
+
                     case CMD_SSL_PROTOCOL:
                         sslProtocol = nextArg("Expected SSL protocol");
 
                         break;
 
-                    case CMD_SSL_KEY_ALGORITHM:
-                        sslKeyAlgorithm = nextArg("Expected SSL key algorithm");
+                    case CMD_SSL_ALGORITHM:
+                        sslAlgorithm = nextArg("Expected SSL algorithm");
 
                         break;
 
@@ -1996,38 +2013,33 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
-                    case CMD_KEYSTORE:
-                        sslKeyStorePath = nextArg("Expected keystore path");
-
-                        break;
-
-                    case CMD_KEYSTORE_PASSWORD:
-                        sslKeyStorePassword = nextArg("Expected keystore password").toCharArray();
+                    case CMD_SSL_KEY_STORE_PATH:
+                        sslKeyStorePath = nextArg("Expected SSL key store path");
 
                         break;
 
-                    case CMD_KEYSTORE_TYPE:
-                        sslKeyStoreType = nextArg("Expected keystore type");
+                    case CMD_SSL_KEY_STORE_TYPE:
+                        sslKeyStoreType = nextArg("Expected SSL key store type");
 
                         break;
 
-                    case CMD_TRUSTSTORE:
-                        sslTrustStorePath = nextArg("Expected truststore path");
+                    case CMD_SSL_KEY_STORE_PASSWORD:
+                        sslKeyStorePassword = nextArg("Expected SSL key store password").toCharArray();
 
                         break;
 
-                    case CMD_TRUSTSTORE_PASSWORD:
-                        sslTrustStorePassword = nextArg("Expected truststore password").toCharArray();
+                    case CMD_SSL_TRUSTSTORE_PATH:
+                        sslTrustStorePath = nextArg("Expected SSL trust store path");
 
                         break;
 
-                    case CMD_TRUSTSTORE_TYPE:
-                        sslTrustStoreType = nextArg("Expected truststore type");
+                    case CMD_SSL_TRUSTSTORE_TYPE:
+                        sslTrustStoreType = nextArg("Expected SSL trust store type");
 
                         break;
 
-                    case CMD_AUTO_CONFIRMATION:
-                        autoConfirmation = true;
+                    case CMD_SSL_TRUSTSTORE_PASSWORD:
+                        sslTrustStorePassword = nextArg("Expected SSL trust store password").toCharArray();
 
                         break;
 
@@ -2047,10 +2059,14 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         Command cmd = commands.get(0);
 
-        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
+        return new Arguments(cmd, host, port, user, pwd,
+            baselineAct, baselineArgs,
+            txArgs, cacheArgs,
+            walAct, walArgs,
             pingTimeout, pingInterval, autoConfirmation,
-            sslProtocol, sslKeyAlgorithm, sslCipherSuites, sslKeyStorePath, sslKeyStorePassword, sslKeyStoreType,
-            sslTrustStorePath, sslTrustStorePassword, sslTrustStoreType);
+            sslEnable, sslProtocol, sslAlgorithm, sslCipherSuites,
+            sslKeyStorePath, sslKeyStoreType, sslKeyStorePassword,
+            sslTrustStorePath, sslTrustStoreType, sslTrustStorePassword);
     }
 
     /**
@@ -2616,9 +2632,9 @@ private void printHelp() {
         log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2));
         log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2));
         log(i("SSL_PROTOCOL=" + SslContextFactory.DFLT_SSL_PROTOCOL, 2));
-        log(i("SSL_KEY_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2));
-        log(i("KEYSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
-        log(i("TRUSTSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
+        log(i("SSL_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2));
+        log(i("SSL_KEY_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
+        log(i("SSL_TRUST_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
 
         nl();
 
@@ -2671,6 +2687,49 @@ public int execute(List<String> rawArgs) {
 
             clientCfg.setServers(Collections.singletonList(args.host() + ":" + args.port()));
 
+            if (args.isSslEnable()) {
+                GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
+
+                List<String> sslProtocols = split(args.getSslProtocol(), ",");
+
+                String sslProtocol = F.isEmpty(sslProtocols) ? SslContextFactory.DFLT_SSL_PROTOCOL : sslProtocols.get(0);
+
+                factory.setProtocol(sslProtocol);
+                factory.setKeyAlgorithm(args.getSslAlgorithm());
+
+                if (sslProtocols.size() > 1)
+                    factory.setProtocols(sslProtocols);
+
+                factory.setCipherSuites(split(args.getSslCipherSuites(), ","));
+
+                if (args.getSslKeyStorePath() == null)
+                    throw new IllegalArgumentException("SSL key store location is not specified.");
+
+                factory.setKeyStoreFilePath(args.getSslKeyStorePath());
+
+                if (args.getSslKeyStorePassword() != null)
+                    factory.setKeyStorePassword(args.getSslKeyStorePassword());
+                else
+                    factory.setKeyStorePassword(requestPasswordFromConsole("SSL keystore password: "));
+
+                factory.setKeyStoreType(args.getSslKeyStoreType());
+
+                if (args.getSslTrustStorePath() == null)
+                    factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
+                else {
+                    factory.setTrustStoreFilePath(args.getSslTrustStorePath());
+
+                    if (args.getSslTrustStorePassword() != null)
+                        factory.setTrustStorePassword(args.getSslTrustStorePassword());
+                    else
+                        factory.setTrustStorePassword(requestPasswordFromConsole("SSL truststore password: "));
+
+                    factory.setTrustStoreType(args.getSslTrustStoreType());
+                }
+
+                clientCfg.setSslContextFactory(factory);
+            }
+
             boolean tryConnectAgain = true;
 
             int tryConnectMaxCount = 3;
@@ -2693,40 +2752,6 @@ public int execute(List<String> rawArgs) {
                     credential.setPassword(args.getPassword());
                 }
 
-                if (!F.isEmpty(args.sslKeyStorePath())) {
-                    GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
-
-                    factory.setProtocol(args.sslProtocol());
-
-                    factory.setKeyAlgorithm(args.sslKeyAlgorithm());
-
-                    factory.setCipherSuites(split(args.getSslCipherSuites(), ","));
-
-                    factory.setKeyStoreFilePath(args.sslKeyStorePath());
-
-                    if (args.sslKeyStorePassword() != null)
-                        factory.setKeyStorePassword(args.sslKeyStorePassword());
-                    else
-                        factory.setKeyStorePassword(requestPasswordFromConsole("SSL keystore password: "));
-
-                    factory.setKeyStoreType(args.sslKeyStoreType());
-
-                    if (F.isEmpty(args.sslTrustStorePath()))
-                        factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
-                    else {
-                        factory.setTrustStoreFilePath(args.sslTrustStorePath());
-
-                        if (args.sslTrustStorePassword() != null)
-                            factory.setTrustStorePassword(args.sslTrustStorePassword());
-                        else
-                            factory.setTrustStorePassword(requestPasswordFromConsole("SSL truststore password: "));
-
-                        factory.setTrustStoreType(args.sslTrustStoreType());
-                    }
-
-                    clientCfg.setSslContextFactory(factory);
-                }
-
                 try (GridClient client = GridClientFactory.start(clientCfg)) {
                     switch (args.command()) {
                         case ACTIVATE:
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 203d6d06d0ba3..988b1d7e94a40 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -38,7 +38,6 @@
 import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_THROUGH;
 import static org.apache.ignite.internal.commandline.CommandHandler.WAL_DELETE;
 import static org.apache.ignite.internal.commandline.CommandHandler.WAL_PRINT;
-import static org.junit.Assert.assertArrayEquals;
 
 /**
  * Tests Command Handler parsing arguments.
@@ -167,44 +166,6 @@ public void testExperimentalCommandIsDisabled() {
         }
     }
 
-    /**
-     * Tests parsing and validation for the SSL arguments.
-     */
-    public void testParseAndValidateSSLArguments() {
-        CommandHandler hnd = new CommandHandler();
-
-        for (Command cmd : Command.values()) {
-
-            if (cmd == Command.CACHE || cmd == Command.WAL)
-                continue; // --cache subcommand requires its own specific arguments.
-
-            try {
-                hnd.parseAndValidate(asList("--truststore"));
-
-                fail("expected exception: Expected truststore");
-            }
-            catch (IllegalArgumentException e) {
-                e.printStackTrace();
-            }
-
-            Arguments args = hnd.parseAndValidate(asList("--keystore", "testKeystore", "--keystore-password", "testKeystorePassword", "--keystore-type", "testKeystoreType",
-                "--truststore", "testTruststore", "--truststore-password", "testTruststorePassword", "--truststore-type", "testTruststoreType",
-                "--ssl-key-algorithm", "testSSLKeyAlgorithm", "--ssl-protocol", "testSSLProtocol", cmd.text()));
-
-            assertEquals("testSSLProtocol", args.sslProtocol());
-            assertEquals("testSSLKeyAlgorithm", args.sslKeyAlgorithm());
-            assertEquals("testKeystore", args.sslKeyStorePath());
-            assertArrayEquals("testKeystorePassword".toCharArray(), args.sslKeyStorePassword());
-            assertEquals("testKeystoreType", args.sslKeyStoreType());
-            assertEquals("testTruststore", args.sslTrustStorePath());
-            assertArrayEquals("testTruststorePassword".toCharArray(), args.sslTrustStorePassword());
-            assertEquals("testTruststoreType", args.sslTrustStoreType());
-
-            assertEquals(cmd, args.command());
-        }
-    }
-
-
     /**
      * Tests parsing and validation for user and password arguments.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
index 35ef5966e7732..7b4b4c098b85e 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
@@ -23,7 +23,6 @@
 import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest;
 import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest;
 import org.apache.ignite.marshaller.GridMarshallerMappingConsistencyTest;
-import org.apache.ignite.util.GridCommandHandlerSslTest;
 import org.apache.ignite.util.GridCommandHandlerTest;
 import org.apache.ignite.util.GridInternalTaskUnusedWalSegmentsTest;
 import org.jetbrains.annotations.Nullable;
@@ -57,7 +56,6 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(SystemWorkersTerminationTest.class);
 
         suite.addTestSuite(GridCommandHandlerTest.class);
-        suite.addTestSuite(GridCommandHandlerSslTest.class);
         suite.addTestSuite(GridInternalTaskUnusedWalSegmentsTest.class);
 
         return suite;
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
index 59155e5b3713a..fb4092af9b968 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
@@ -33,6 +33,7 @@
  * Command line handler test.
  */
 public class GridCommandHandlerSslTest extends GridCommonAbstractTest {
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         cleanPersistenceDir();
@@ -77,8 +78,9 @@ public void testActivate() throws Exception {
         final CommandHandler cmd = new CommandHandler();
         assertEquals(EXIT_CODE_OK, cmd.execute(Arrays.asList(
             "--activate",
-            "--keystore", GridTestUtils.keyStorePath("node01"),
-            "--keystore-password", GridTestUtils.keyStorePassword())));
+            "--ssl_enabled",
+            "--ssl_key_store_path", GridTestUtils.keyStorePath("node01"),
+            "--ssl_key_store_password", GridTestUtils.keyStorePassword())));
 
         assertTrue(ignite.cluster().active());
 

From 63e241ce48dcc75df2b04a679fbcb915c3278a61 Mon Sep 17 00:00:00 2001
From: Sergey Antonov <antonovsergey93@gmail.com>
Date: Wed, 12 Dec 2018 13:18:37 +0300
Subject: [PATCH 534/543] GG-14552 Sync changes in master and
 ignite-2.5.1-master branches for [IGNITE-9298] control.sh does not support
 SSL (org.apache.ignite.internal.commandline.CommandHandler)

Author: deviljelly <deviljelly@gmail.com>
Date:   Fri Nov 30 16:40:40 2018 +0300

    IGNITE-9298 SSL support in control.sh - Fixes #5516.

    Fixes #4684.
    Fixes #4697.

    Signed-off-by: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
    Co-authored-by: a-polyakov <polyakov.alexandr.alexandrovich@gmail.com>
    Co-authored-by: Ilya Kasnacheev <ilya.kasnacheev@gmail.com>
    (cherry picked from commit 6d7847c7f1427a337975d240b10bec8b10a8d3f2)
---
 .../internal/commandline/Arguments.java       | 113 +++++-----
 .../internal/commandline/CommandHandler.java  | 209 ++++++++----------
 .../CommandHandlerParsingTest.java            |  39 ++++
 .../IgniteBasicWithPersistenceTestSuite.java  |   2 +
 .../util/GridCommandHandlerSslTest.java       |   6 +-
 5 files changed, 193 insertions(+), 176 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index d769982566320..7fc4ac9ef5c37 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -77,35 +77,32 @@ public class Arguments {
     /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}. */
     private long pingInterval;
 
-    /** */
-    private boolean sslEnable;
-
-    /** */
+    /** SSL Protocol. */
     private String sslProtocol;
 
-    /** */
-    private String sslAlgorithm;
+    /** SSL Key Algorithm. */
+    private String sslKeyAlgorithm;
 
     /** */
     private String sslCipherSuites;
 
-    /** */
+    /** Keystore. */
     private String sslKeyStorePath;
 
-    /** */
+    /** Keystore Type. */
     private String sslKeyStoreType;
 
-    /** */
-    private char sslKeyStorePassword[];
+    /** Keystore Password. */
+    private char[] sslKeyStorePassword;
 
-    /** */
+    /** Truststore. */
     private String sslTrustStorePath;
 
-    /** */
+    /** Truststore Type. */
     private String sslTrustStoreType;
 
-    /** */
-    private char sslTrustStorePassword[];
+    /** Truststore Password. */
+    private char[] sslTrustStorePassword;
 
     /**
      * @param cmd Command.
@@ -122,25 +119,21 @@ public class Arguments {
      * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
      * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
      * @param autoConfirmation Auto confirmation flag.
-     * @param sslEnable SSL flag.
-     * @param sslProtocol SSL protocol.
-     * @param sslAlgorithm SSL algorithm.
+     * @param sslProtocol SSL Protocol.
+     * @param sslKeyAlgorithm SSL Key Algorithm.
      * @param sslCipherSuites SSL cipher suites.
-     * @param sslKeyStorePath Key store path.
-     * @param sslKeyStoreType Key store type.
-     * @param sslKeyStorePassword Key store password.
-     * @param sslTrustStorePath Trust store path.
-     * @param sslTrustStoreType Trust store type.
-     * @param sslTrustStorePassword Trust store password.
+     * @param sslKeyStorePath Keystore.
+     * @param sslKeyStorePassword Keystore Password.
+     * @param sslKeyStoreType Keystore Type.
+     * @param sslTrustStorePath Truststore.
+     * @param sslTrustStorePassword Truststore Password.
+     * @param sslTrustStoreType Truststore Type.
      */
-    public Arguments(Command cmd, String host, String port, String user, String pwd,
-        String baselineAct, String baselineArgs,
-        VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
-        Long pingTimeout, Long pingInterval, boolean autoConfirmation,
-        boolean sslEnable, String sslProtocol, String sslAlgorithm, String sslCipherSuites,
-        String sslKeyStorePath, String sslKeyStoreType, char sslKeyStorePassword[],
-        String sslTrustStorePath, String sslTrustStoreType, char sslTrustStorePassword[]
-    ) {
+    public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
+        String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
+        Long pingTimeout, Long pingInterval, boolean autoConfirmation, String sslProtocol, String sslKeyAlgorithm,
+        String sslCipherSuites, String sslKeyStorePath, char[] sslKeyStorePassword, String sslKeyStoreType,
+        String sslTrustStorePath, char[] sslTrustStorePassword, String sslTrustStoreType) {
         this.cmd = cmd;
         this.host = host;
         this.port = port;
@@ -161,9 +154,8 @@ public Arguments(Command cmd, String host, String port, String user, String pwd,
 
         this.autoConfirmation = autoConfirmation;
 
-        this.sslEnable = sslEnable;
         this.sslProtocol = sslProtocol;
-        this.sslAlgorithm = sslAlgorithm;
+        this.sslKeyAlgorithm = sslKeyAlgorithm;
         this.sslCipherSuites = sslCipherSuites;
 
         this.sslKeyStorePath = sslKeyStorePath;
@@ -291,19 +283,18 @@ public boolean autoConfirmation() {
         return autoConfirmation;
     }
 
-    /** */
-    public boolean isSslEnable() {
-        return sslEnable;
-    }
-
-    /** */
-    public String getSslProtocol() {
+    /**
+     * @return SSL protocol
+     */
+    public String sslProtocol() {
         return sslProtocol;
     }
 
-    /** */
-    public String getSslAlgorithm() {
-        return sslAlgorithm;
+    /**
+     * @return SSL Key Algorithm
+     */
+    public String sslKeyAlgorithm() {
+        return sslKeyAlgorithm;
     }
 
     /** */
@@ -311,33 +302,45 @@ public String getSslCipherSuites() {
         return sslCipherSuites;
     }
 
-    /** */
-    public String getSslKeyStorePath() {
+    /**
+     * @return Keystore
+     */
+    public String sslKeyStorePath() {
         return sslKeyStorePath;
     }
 
-    /** */
-    public String getSslKeyStoreType() {
+    /**
+     * @return Keystore type
+     */
+    public String sslKeyStoreType() {
         return sslKeyStoreType;
     }
 
-    /** */
-    public char[] getSslKeyStorePassword() {
+    /**
+     * @return Keystore password
+     */
+    public char[] sslKeyStorePassword() {
         return sslKeyStorePassword;
     }
 
-    /** */
-    public String getSslTrustStorePath() {
+    /**
+     * @return Truststore
+     */
+    public String sslTrustStorePath() {
         return sslTrustStorePath;
     }
 
-    /** */
-    public String getSslTrustStoreType() {
+    /**
+     * @return Truststore type
+     */
+    public String sslTrustStoreType() {
         return sslTrustStoreType;
     }
 
-    /** */
-    public char[] getSslTrustStorePassword() {
+    /**
+     * @return Truststore password
+     */
+    public char[] sslTrustStorePassword() {
         return sslTrustStorePassword;
     }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index cf980dab3d6d3..a05ce9ef87d3f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -195,40 +195,37 @@ public class CommandHandler {
     /** */
     private static final String CMD_SKIP_ZEROS = "--skipZeros";
 
-    // SSL configuration section
-
     /** */
-    protected static final String CMD_SSL_ENABLED = "--ssl_enabled";
+    private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
 
-    /** */
-    protected static final String CMD_SSL_PROTOCOL = "--ssl_protocol";
+    // SSL configuration section
 
     /** */
-    protected static final String CMD_SSL_ALGORITHM = "--ssl_algorithm";
+    private static final String CMD_SSL_PROTOCOL = "--ssl-protocol";
 
     /** */
-    protected static final String CMD_SSL_CIPHER_SUITES = "--ssl_cipher_suites";
+    private static final String CMD_SSL_KEY_ALGORITHM = "--ssl-key-algorithm";
 
     /** */
-    protected static final String CMD_SSL_KEY_STORE_PATH = "--ssl_key_store_path";
+    protected static final String CMD_SSL_CIPHER_SUITES = "--ssl-cipher-suites";
 
     /** */
-    protected static final String CMD_SSL_KEY_STORE_TYPE = "--ssl_key_store_type";
+    private static final String CMD_KEYSTORE = "--keystore";
 
     /** */
-    protected static final String CMD_SSL_KEY_STORE_PASSWORD = "--ssl_key_store_password";
+    private static final String CMD_KEYSTORE_PASSWORD = "--keystore-password";
 
     /** */
-    protected static final String CMD_SSL_TRUSTSTORE_PATH = "--ssl_truststore_path";
+    private static final String CMD_KEYSTORE_TYPE = "--keystore-type";
 
     /** */
-    protected static final String CMD_SSL_TRUSTSTORE_TYPE = "--ssl_truststore_type";
+    private static final String CMD_TRUSTSTORE = "--truststore";
 
     /** */
-    protected static final String CMD_SSL_TRUSTSTORE_PASSWORD = "--ssl_truststore_password";
+    private static final String CMD_TRUSTSTORE_PASSWORD = "--truststore-password";
 
     /** */
-    private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
+    private static final String CMD_TRUSTSTORE_TYPE = "--truststore-type";
 
     /** List of optional auxiliary commands. */
     private static final Set<String> AUX_COMMANDS = new HashSet<>();
@@ -247,18 +244,17 @@ public class CommandHandler {
         AUX_COMMANDS.add(CMD_PING_INTERVAL);
         AUX_COMMANDS.add(CMD_PING_TIMEOUT);
 
-        AUX_COMMANDS.add(CMD_SSL_ENABLED);
         AUX_COMMANDS.add(CMD_SSL_PROTOCOL);
-        AUX_COMMANDS.add(CMD_SSL_ALGORITHM);
+        AUX_COMMANDS.add(CMD_SSL_KEY_ALGORITHM);
         AUX_COMMANDS.add(CMD_SSL_CIPHER_SUITES);
 
-        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_TYPE);
-        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PATH);
-        AUX_COMMANDS.add(CMD_SSL_KEY_STORE_PASSWORD);
+        AUX_COMMANDS.add(CMD_KEYSTORE);
+        AUX_COMMANDS.add(CMD_KEYSTORE_PASSWORD);
+        AUX_COMMANDS.add(CMD_KEYSTORE_TYPE);
 
-        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_TYPE);
-        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PATH);
-        AUX_COMMANDS.add(CMD_SSL_TRUSTSTORE_PASSWORD);
+        AUX_COMMANDS.add(CMD_TRUSTSTORE);
+        AUX_COMMANDS.add(CMD_TRUSTSTORE_PASSWORD);
+        AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE);
     }
 
     /** Broadcast uuid. */
@@ -407,16 +403,15 @@ private static List<String> getCommonOptions() {
         list.add(op(CMD_PASSWORD, "PASSWORD"));
         list.add(op(CMD_PING_INTERVAL, "PING_INTERVAL"));
         list.add(op(CMD_PING_TIMEOUT, "PING_TIMEOUT"));
-        list.add(op(CMD_SSL_ENABLED));
         list.add(op(CMD_SSL_PROTOCOL, "SSL_PROTOCOL[, SSL_PROTOCOL_2, ...]"));
-        list.add(op(CMD_SSL_ALGORITHM, "SSL_ALGORITHM"));
+        list.add(op(CMD_SSL_KEY_ALGORITHM, "SSL_KEY_ALGORITHM"));
         list.add(op(CMD_SSL_CIPHER_SUITES, "SSL_CIPHER_1[, SSL_CIPHER_2, ...]"));
-        list.add(op(CMD_SSL_KEY_STORE_TYPE, "SSL_KEY_STORE_TYPE"));
-        list.add(op(CMD_SSL_KEY_STORE_PATH, "PATH"));
-        list.add(op(CMD_SSL_KEY_STORE_PASSWORD, "PASSWORD"));
-        list.add(op(CMD_SSL_TRUSTSTORE_TYPE, "SSL_TRUST_STORE_TYPE"));
-        list.add(op(CMD_SSL_TRUSTSTORE_PATH, "PATH"));
-        list.add(op(CMD_SSL_TRUSTSTORE_PASSWORD, "PASSWORD"));
+        list.add(op(CMD_KEYSTORE, "KEYSTORE"));
+        list.add(op(CMD_KEYSTORE_TYPE, "KEYSTORE_TYPE"));
+        list.add(op(CMD_KEYSTORE_PASSWORD, "KEYSTORE_PASSWORD"));
+        list.add(op(CMD_TRUSTSTORE, "TRUSTSTORE"));
+        list.add(op(CMD_TRUSTSTORE_TYPE, "TRUSTSTORE_TYPE"));
+        list.add(op(CMD_TRUSTSTORE_PASSWORD, "TRUSTSTORE_PASSWORD"));
 
         return list;
     }
@@ -685,14 +680,15 @@ private Stream<IgniteBiTuple<GridClientNode, String>> listHosts(GridClient clien
      * @return List of hosts.
      */
     private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNode(
-        GridClient client) throws GridClientException {
+        GridClient client
+    ) throws GridClientException {
         return client.compute().nodes(GridClientNode::connectable).stream()
             .map(node -> new IgniteBiTuple<>(node,
                 Stream.concat(
                     node.tcpAddresses() == null ? Stream.empty() : node.tcpAddresses().stream(),
                     node.tcpHostNames() == null ? Stream.empty() : node.tcpHostNames().stream()
                 )
-                    .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
+                .map(addr -> addr + ":" + node.tcpPort()).collect(Collectors.toList())));
     }
 
     /**
@@ -703,7 +699,6 @@ private Stream<IgniteBiTuple<GridClientNode, List<String>>> listHostsByClientNod
      * @return Task result.
      * @throws GridClientException If failed to execute task.
      */
-    @SuppressWarnings("unchecked")
     private <R> R executeTaskByNameOnNode(
         GridClient client,
         String taskClsName,
@@ -1856,24 +1851,22 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         VisorTxTaskArg txArgs = null;
 
-        boolean sslEnable = false;
-
         String sslProtocol = SslContextFactory.DFLT_SSL_PROTOCOL;
 
-        String sslAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM;
+        String sslKeyAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM;
 
         String sslCipherSuites = "";
 
-        String sslKeyStoreType = SslContextFactory.DFLT_STORE_TYPE;
-
         String sslKeyStorePath = null;
 
-        char sslKeyStorePassword[] = null;
+        String sslKeyStoreType = SslContextFactory.DFLT_STORE_TYPE;
 
-        String sslTrustStoreType = SslContextFactory.DFLT_STORE_TYPE;
+        char sslKeyStorePassword[] = null;
 
         String sslTrustStorePath = null;
 
+        String sslTrustStoreType = SslContextFactory.DFLT_STORE_TYPE;
+
         char sslTrustStorePassword[] = null;
 
         while (hasNextArg()) {
@@ -1988,23 +1981,13 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
-                    case CMD_AUTO_CONFIRMATION:
-                        autoConfirmation = true;
-
-                        break;
-
-                    case CMD_SSL_ENABLED:
-                        sslEnable = true;
-
-                        break;
-
                     case CMD_SSL_PROTOCOL:
                         sslProtocol = nextArg("Expected SSL protocol");
 
                         break;
 
-                    case CMD_SSL_ALGORITHM:
-                        sslAlgorithm = nextArg("Expected SSL algorithm");
+                    case CMD_SSL_KEY_ALGORITHM:
+                        sslKeyAlgorithm = nextArg("Expected SSL key algorithm");
 
                         break;
 
@@ -2013,33 +1996,38 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
-                    case CMD_SSL_KEY_STORE_PATH:
-                        sslKeyStorePath = nextArg("Expected SSL key store path");
+                    case CMD_KEYSTORE:
+                        sslKeyStorePath = nextArg("Expected keystore path");
+
+                        break;
+
+                    case CMD_KEYSTORE_PASSWORD:
+                        sslKeyStorePassword = nextArg("Expected keystore password").toCharArray();
 
                         break;
 
-                    case CMD_SSL_KEY_STORE_TYPE:
-                        sslKeyStoreType = nextArg("Expected SSL key store type");
+                    case CMD_KEYSTORE_TYPE:
+                        sslKeyStoreType = nextArg("Expected keystore type");
 
                         break;
 
-                    case CMD_SSL_KEY_STORE_PASSWORD:
-                        sslKeyStorePassword = nextArg("Expected SSL key store password").toCharArray();
+                    case CMD_TRUSTSTORE:
+                        sslTrustStorePath = nextArg("Expected truststore path");
 
                         break;
 
-                    case CMD_SSL_TRUSTSTORE_PATH:
-                        sslTrustStorePath = nextArg("Expected SSL trust store path");
+                    case CMD_TRUSTSTORE_PASSWORD:
+                        sslTrustStorePassword = nextArg("Expected truststore password").toCharArray();
 
                         break;
 
-                    case CMD_SSL_TRUSTSTORE_TYPE:
-                        sslTrustStoreType = nextArg("Expected SSL trust store type");
+                    case CMD_TRUSTSTORE_TYPE:
+                        sslTrustStoreType = nextArg("Expected truststore type");
 
                         break;
 
-                    case CMD_SSL_TRUSTSTORE_PASSWORD:
-                        sslTrustStorePassword = nextArg("Expected SSL trust store password").toCharArray();
+                    case CMD_AUTO_CONFIRMATION:
+                        autoConfirmation = true;
 
                         break;
 
@@ -2059,14 +2047,10 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         Command cmd = commands.get(0);
 
-        return new Arguments(cmd, host, port, user, pwd,
-            baselineAct, baselineArgs,
-            txArgs, cacheArgs,
-            walAct, walArgs,
+        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
             pingTimeout, pingInterval, autoConfirmation,
-            sslEnable, sslProtocol, sslAlgorithm, sslCipherSuites,
-            sslKeyStorePath, sslKeyStoreType, sslKeyStorePassword,
-            sslTrustStorePath, sslTrustStoreType, sslTrustStorePassword);
+            sslProtocol, sslKeyAlgorithm, sslCipherSuites, sslKeyStorePath, sslKeyStorePassword, sslKeyStoreType,
+            sslTrustStorePath, sslTrustStorePassword, sslTrustStoreType);
     }
 
     /**
@@ -2632,9 +2616,9 @@ private void printHelp() {
         log(i("PING_INTERVAL=" + DFLT_PING_INTERVAL, 2));
         log(i("PING_TIMEOUT=" + DFLT_PING_TIMEOUT, 2));
         log(i("SSL_PROTOCOL=" + SslContextFactory.DFLT_SSL_PROTOCOL, 2));
-        log(i("SSL_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2));
-        log(i("SSL_KEY_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
-        log(i("SSL_TRUST_STORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
+        log(i("SSL_KEY_ALGORITHM=" + SslContextFactory.DFLT_KEY_ALGORITHM, 2));
+        log(i("KEYSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
+        log(i("TRUSTSTORE_TYPE=" + SslContextFactory.DFLT_STORE_TYPE, 2));
 
         nl();
 
@@ -2687,49 +2671,6 @@ public int execute(List<String> rawArgs) {
 
             clientCfg.setServers(Collections.singletonList(args.host() + ":" + args.port()));
 
-            if (args.isSslEnable()) {
-                GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
-
-                List<String> sslProtocols = split(args.getSslProtocol(), ",");
-
-                String sslProtocol = F.isEmpty(sslProtocols) ? SslContextFactory.DFLT_SSL_PROTOCOL : sslProtocols.get(0);
-
-                factory.setProtocol(sslProtocol);
-                factory.setKeyAlgorithm(args.getSslAlgorithm());
-
-                if (sslProtocols.size() > 1)
-                    factory.setProtocols(sslProtocols);
-
-                factory.setCipherSuites(split(args.getSslCipherSuites(), ","));
-
-                if (args.getSslKeyStorePath() == null)
-                    throw new IllegalArgumentException("SSL key store location is not specified.");
-
-                factory.setKeyStoreFilePath(args.getSslKeyStorePath());
-
-                if (args.getSslKeyStorePassword() != null)
-                    factory.setKeyStorePassword(args.getSslKeyStorePassword());
-                else
-                    factory.setKeyStorePassword(requestPasswordFromConsole("SSL keystore password: "));
-
-                factory.setKeyStoreType(args.getSslKeyStoreType());
-
-                if (args.getSslTrustStorePath() == null)
-                    factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
-                else {
-                    factory.setTrustStoreFilePath(args.getSslTrustStorePath());
-
-                    if (args.getSslTrustStorePassword() != null)
-                        factory.setTrustStorePassword(args.getSslTrustStorePassword());
-                    else
-                        factory.setTrustStorePassword(requestPasswordFromConsole("SSL truststore password: "));
-
-                    factory.setTrustStoreType(args.getSslTrustStoreType());
-                }
-
-                clientCfg.setSslContextFactory(factory);
-            }
-
             boolean tryConnectAgain = true;
 
             int tryConnectMaxCount = 3;
@@ -2752,6 +2693,40 @@ public int execute(List<String> rawArgs) {
                     credential.setPassword(args.getPassword());
                 }
 
+                if (!F.isEmpty(args.sslKeyStorePath())) {
+                    GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
+
+                    factory.setProtocol(args.sslProtocol());
+
+                    factory.setKeyAlgorithm(args.sslKeyAlgorithm());
+
+                    factory.setCipherSuites(split(args.getSslCipherSuites(), ","));
+
+                    factory.setKeyStoreFilePath(args.sslKeyStorePath());
+
+                    if (args.sslKeyStorePassword() != null)
+                        factory.setKeyStorePassword(args.sslKeyStorePassword());
+                    else
+                        factory.setKeyStorePassword(requestPasswordFromConsole("SSL keystore password: "));
+
+                    factory.setKeyStoreType(args.sslKeyStoreType());
+
+                    if (F.isEmpty(args.sslTrustStorePath()))
+                        factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
+                    else {
+                        factory.setTrustStoreFilePath(args.sslTrustStorePath());
+
+                        if (args.sslTrustStorePassword() != null)
+                            factory.setTrustStorePassword(args.sslTrustStorePassword());
+                        else
+                            factory.setTrustStorePassword(requestPasswordFromConsole("SSL truststore password: "));
+
+                        factory.setTrustStoreType(args.sslTrustStoreType());
+                    }
+
+                    clientCfg.setSslContextFactory(factory);
+                }
+
                 try (GridClient client = GridClientFactory.start(clientCfg)) {
                     switch (args.command()) {
                         case ACTIVATE:
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 988b1d7e94a40..203d6d06d0ba3 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -38,6 +38,7 @@
 import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_THROUGH;
 import static org.apache.ignite.internal.commandline.CommandHandler.WAL_DELETE;
 import static org.apache.ignite.internal.commandline.CommandHandler.WAL_PRINT;
+import static org.junit.Assert.assertArrayEquals;
 
 /**
  * Tests Command Handler parsing arguments.
@@ -166,6 +167,44 @@ public void testExperimentalCommandIsDisabled() {
         }
     }
 
+    /**
+     * Tests parsing and validation for the SSL arguments.
+     */
+    public void testParseAndValidateSSLArguments() {
+        CommandHandler hnd = new CommandHandler();
+
+        for (Command cmd : Command.values()) {
+
+            if (cmd == Command.CACHE || cmd == Command.WAL)
+                continue; // --cache subcommand requires its own specific arguments.
+
+            try {
+                hnd.parseAndValidate(asList("--truststore"));
+
+                fail("expected exception: Expected truststore");
+            }
+            catch (IllegalArgumentException e) {
+                e.printStackTrace();
+            }
+
+            Arguments args = hnd.parseAndValidate(asList("--keystore", "testKeystore", "--keystore-password", "testKeystorePassword", "--keystore-type", "testKeystoreType",
+                "--truststore", "testTruststore", "--truststore-password", "testTruststorePassword", "--truststore-type", "testTruststoreType",
+                "--ssl-key-algorithm", "testSSLKeyAlgorithm", "--ssl-protocol", "testSSLProtocol", cmd.text()));
+
+            assertEquals("testSSLProtocol", args.sslProtocol());
+            assertEquals("testSSLKeyAlgorithm", args.sslKeyAlgorithm());
+            assertEquals("testKeystore", args.sslKeyStorePath());
+            assertArrayEquals("testKeystorePassword".toCharArray(), args.sslKeyStorePassword());
+            assertEquals("testKeystoreType", args.sslKeyStoreType());
+            assertEquals("testTruststore", args.sslTrustStorePath());
+            assertArrayEquals("testTruststorePassword".toCharArray(), args.sslTrustStorePassword());
+            assertEquals("testTruststoreType", args.sslTrustStoreType());
+
+            assertEquals(cmd, args.command());
+        }
+    }
+
+
     /**
      * Tests parsing and validation for user and password arguments.
      */
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
index 7b4b4c098b85e..35ef5966e7732 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBasicWithPersistenceTestSuite.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.internal.ClusterBaselineNodesMetricsSelfTest;
 import org.apache.ignite.internal.processors.service.ServiceDeploymentOutsideBaselineTest;
 import org.apache.ignite.marshaller.GridMarshallerMappingConsistencyTest;
+import org.apache.ignite.util.GridCommandHandlerSslTest;
 import org.apache.ignite.util.GridCommandHandlerTest;
 import org.apache.ignite.util.GridInternalTaskUnusedWalSegmentsTest;
 import org.jetbrains.annotations.Nullable;
@@ -56,6 +57,7 @@ public static TestSuite suite(@Nullable final Set<Class> ignoredTests) throws Ex
         suite.addTestSuite(SystemWorkersTerminationTest.class);
 
         suite.addTestSuite(GridCommandHandlerTest.class);
+        suite.addTestSuite(GridCommandHandlerSslTest.class);
         suite.addTestSuite(GridInternalTaskUnusedWalSegmentsTest.class);
 
         return suite;
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
index fb4092af9b968..59155e5b3713a 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
@@ -33,7 +33,6 @@
  * Command line handler test.
  */
 public class GridCommandHandlerSslTest extends GridCommonAbstractTest {
-
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         cleanPersistenceDir();
@@ -78,9 +77,8 @@ public void testActivate() throws Exception {
         final CommandHandler cmd = new CommandHandler();
         assertEquals(EXIT_CODE_OK, cmd.execute(Arrays.asList(
             "--activate",
-            "--ssl_enabled",
-            "--ssl_key_store_path", GridTestUtils.keyStorePath("node01"),
-            "--ssl_key_store_password", GridTestUtils.keyStorePassword())));
+            "--keystore", GridTestUtils.keyStorePath("node01"),
+            "--keystore-password", GridTestUtils.keyStorePassword())));
 
         assertTrue(ignite.cluster().active());
 

From d16cca92c747909c9be9e2902675a8e354127369 Mon Sep 17 00:00:00 2001
From: Pavel Voronkin <pvoronkin@gridgain.com>
Date: Tue, 11 Dec 2018 16:40:07 +0300
Subject: [PATCH 535/543] IGNITE-10300 Fixed incorrect error message after
 authentication retries - Fixes #5425.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit ed9c1377c0d38bb1bb3352b2b943e69e776d917c)
---
 .../apache/ignite/internal/commandline/CommandHandler.java | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index a05ce9ef87d3f..849f4e58d04a2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -2778,8 +2778,13 @@ public int execute(List<String> rawArgs) {
 
                         tryConnectMaxCount--;
                     }
-                    else
+                    else {
+                        if (tryConnectMaxCount == 0)
+                            throw new GridClientAuthenticationException("Authentication error, maximum number of " +
+                                "retries exceeded");
+
                         throw e;
+                    }
                 }
             }
 

From 7e7aa39605992ef5f9d5a4eb792e37bf7e45fab0 Mon Sep 17 00:00:00 2001
From: Ilya Lantukh <ilantukh@gridgain.com>
Date: Thu, 18 Oct 2018 19:34:55 +0300
Subject: [PATCH 536/543] IGNITE-9155 ExchangeFuture will not complete with
 Exception on state change failure - Fixes #4467.

Signed-off-by: Alexey Goncharuk <alexey.goncharuk@gmail.com>

(cherry picked from commit 177d571b57f8dde2c82e2a7e4ba9b5d453f4d8df)
---
 .../GridDhtPartitionsExchangeFuture.java          | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index cb2a424f22e62..2748b4df18e93 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -3120,8 +3120,6 @@ else if (forceAffReassignment)
             }
 
             if (stateChangeExchange()) {
-                IgniteCheckedException err = null;
-
                 StateChangeRequest req = exchActions.stateChangeRequest();
 
                 assert req != null : exchActions;
@@ -3131,8 +3129,6 @@ else if (forceAffReassignment)
                 if (!F.isEmpty(exchangeGlobalExceptions)) {
                     stateChangeErr = true;
 
-                    err = new IgniteCheckedException("Cluster state change failed.");
-
                     cctx.kernalContext().state().onStateChangeError(exchangeGlobalExceptions, req);
                 }
                 else {
@@ -3164,7 +3160,7 @@ else if (forceAffReassignment)
                 cctx.discovery().sendCustomEvent(stateFinishMsg);
 
                 if (!centralizedAff)
-                    onDone(exchCtx.events().topologyVersion(), err);
+                    onDone(exchCtx.events().topologyVersion(), null);
             }
         }
         catch (IgniteCheckedException e) {
@@ -3651,15 +3647,10 @@ else if (forceAffReassignment)
 
             updatePartitionFullMap(resTopVer, msg);
 
-            IgniteCheckedException err = null;
-
-            if (stateChangeExchange() && !F.isEmpty(msg.getErrorsMap())) {
-                err = new IgniteCheckedException("Cluster state change failed");
-
+            if (stateChangeExchange() && !F.isEmpty(msg.getErrorsMap()))
                 cctx.kernalContext().state().onStateChangeError(msg.getErrorsMap(), exchActions.stateChangeRequest());
-            }
 
-            onDone(resTopVer, err);
+            onDone(resTopVer, null);
         }
         catch (IgniteCheckedException e) {
             onDone(e);

From 61ef559bf6fed0a0b3f7860f3b049332a36611ef Mon Sep 17 00:00:00 2001
From: Vasiliy Sisko <vsisko@gridgain.com>
Date: Thu, 6 Dec 2018 07:30:08 +0300
Subject: [PATCH 537/543] IGNITE-10564 Added tasks to collect rebalance
 metrics.

Co-authored-by: Alexey Kuznetsov <akuznetsov@apache.org>

(cherry picked from commit ececcbcd1883a2cda5a2eb12e0f0deffddf6b829)
---
 ...VisorCacheRebalanceCollectorJobResult.java |  91 ++++++++
 .../VisorCacheRebalanceCollectorTask.java     | 194 ++++++++++++++++++
 .../VisorCacheRebalanceCollectorTaskArg.java  |  54 +++++
 ...isorCacheRebalanceCollectorTaskResult.java |  92 +++++++++
 .../visor/node/VisorNodeBaselineStatus.java   |  45 ++++
 .../visor/node/VisorNodeDataCollectorJob.java |  74 ++++---
 .../VisorNodeDataCollectorTaskResult.java     |   3 +-
 .../internal/visor/util/VisorTaskUtils.java   |  43 ++++
 8 files changed, 564 insertions(+), 32 deletions(-)
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorJobResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskArg.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskResult.java
 create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeBaselineStatus.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorJobResult.java
new file mode 100644
index 0000000000000..5bd818d424848
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorJobResult.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.node;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.internal.dto.IgniteDataTransferObject;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Result object for cache rebalance job.
+ */
+public class VisorCacheRebalanceCollectorJobResult extends IgniteDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Rebalance percent. */
+    private double rebalance;
+
+    /** Node baseline state. */
+    private VisorNodeBaselineStatus baseline;
+
+    /**
+     * Default constructor.
+     */
+    public VisorCacheRebalanceCollectorJobResult() {
+        // No-op.
+    }
+
+    /**
+     * @return Rebalance progress.
+     */
+    public double getRebalance() {
+        return rebalance;
+    }
+
+    /**
+     * @param rebalance Rebalance progress.
+     */
+    public void setRebalance(double rebalance) {
+        this.rebalance = rebalance;
+    }
+
+    /**
+     * @return Node baseline status.
+     */
+    public VisorNodeBaselineStatus getBaseline() {
+        return baseline;
+    }
+
+    /**
+     * @param baseline Node baseline status.
+     */
+    public void setBaseline(VisorNodeBaselineStatus baseline) {
+        this.baseline = baseline;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        out.writeDouble(rebalance);
+        U.writeEnum(out, baseline);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        rebalance = in.readDouble();
+        baseline = VisorNodeBaselineStatus.fromOrdinal(in.readByte());
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorCacheRebalanceCollectorJobResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java
new file mode 100644
index 0000000000000..eda9f9419c100
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTask.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.node;
+
+import java.util.Collection;
+import java.util.List;
+import org.apache.ignite.cache.CacheMetrics;
+import org.apache.ignite.cluster.BaselineNode;
+import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
+import org.apache.ignite.internal.processors.cache.GridCacheUtils;
+import org.apache.ignite.internal.processors.task.GridInternal;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorJob;
+import org.apache.ignite.internal.visor.VisorMultiNodeTask;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.visor.node.VisorNodeBaselineStatus.BASELINE_NOT_AVAILABLE;
+import static org.apache.ignite.internal.visor.node.VisorNodeBaselineStatus.NODE_IN_BASELINE;
+import static org.apache.ignite.internal.visor.node.VisorNodeBaselineStatus.NODE_NOT_IN_BASELINE;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.MINIMAL_REBALANCE;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.NOTHING_TO_REBALANCE;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.REBALANCE_COMPLETE;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.REBALANCE_NOT_AVAILABLE;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.isProxyCache;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.isRestartingCache;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.log;
+
+/**
+ * Collects topology rebalance metrics.
+ */
+@GridInternal
+public class VisorCacheRebalanceCollectorTask extends VisorMultiNodeTask<VisorCacheRebalanceCollectorTaskArg,
+    VisorCacheRebalanceCollectorTaskResult, VisorCacheRebalanceCollectorJobResult> {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** {@inheritDoc} */
+    @Override protected VisorCacheRebalanceCollectorJob job(VisorCacheRebalanceCollectorTaskArg arg) {
+        return new VisorCacheRebalanceCollectorJob(arg, debug);
+    }
+
+    /** {@inheritDoc} */
+    @Nullable @Override protected VisorCacheRebalanceCollectorTaskResult reduce0(List<ComputeJobResult> results) {
+        return reduce(new VisorCacheRebalanceCollectorTaskResult(), results);
+    }
+
+    /**
+     * @param taskRes Task result.
+     * @param results Results.
+     * @return Topology rebalance metrics collector task result.
+     */
+    protected VisorCacheRebalanceCollectorTaskResult reduce(
+        VisorCacheRebalanceCollectorTaskResult taskRes,
+        List<ComputeJobResult> results
+    ) {
+        for (ComputeJobResult res : results) {
+            VisorCacheRebalanceCollectorJobResult jobRes = res.getData();
+
+            if (jobRes != null) {
+                if (res.getException() == null)
+                    taskRes.getRebalance().put(res.getNode().id(), jobRes.getRebalance());
+
+                taskRes.getBaseline().put(res.getNode().id(), jobRes.getBaseline());
+            }
+        }
+
+        return taskRes;
+    }
+
+    /**
+     * Job that collects rebalance metrics.
+     */
+    private static class VisorCacheRebalanceCollectorJob extends VisorJob<VisorCacheRebalanceCollectorTaskArg, VisorCacheRebalanceCollectorJobResult> {
+        /** */
+        private static final long serialVersionUID = 0L;
+
+        /**
+         * Create job with given argument.
+         *
+         * @param arg Job argument.
+         * @param debug Debug flag.
+         */
+        private VisorCacheRebalanceCollectorJob(VisorCacheRebalanceCollectorTaskArg arg, boolean debug) {
+            super(arg, debug);
+        }
+
+        /** {@inheritDoc} */
+        @Override protected VisorCacheRebalanceCollectorJobResult run(VisorCacheRebalanceCollectorTaskArg arg) {
+            VisorCacheRebalanceCollectorJobResult res = new VisorCacheRebalanceCollectorJobResult();
+
+            long start0 = U.currentTimeMillis();
+
+            try {
+                int partitions = 0;
+                double total = 0;
+                double ready = 0;
+
+                GridCacheProcessor cacheProc = ignite.context().cache();
+
+                boolean rebalanceInProgress = false;
+
+                for (CacheGroupContext grp: cacheProc.cacheGroups()) {
+                    String cacheName = grp.config().getName();
+
+                    if (isProxyCache(ignite, cacheName) || isRestartingCache(ignite, cacheName))
+                        continue;
+
+                    try {
+                        GridCacheAdapter ca = cacheProc.internalCache(cacheName);
+
+                        if (ca == null || !ca.context().started())
+                            continue;
+
+                        CacheMetrics cm = ca.localMetrics();
+
+                        partitions += cm.getTotalPartitionsCount();
+
+                        long keysTotal = cm.getEstimatedRebalancingKeys();
+                        long keysReady = cm.getRebalancedKeys();
+
+                        if (keysReady >= keysTotal)
+                            keysReady = Math.max(keysTotal - 1, 0);
+
+                        total += keysTotal;
+                        ready += keysReady;
+
+                        if (cm.getRebalancingPartitionsCount() > 0)
+                            rebalanceInProgress = true;
+                    }
+                    catch(IllegalStateException | IllegalArgumentException e) {
+                        if (debug && ignite.log() != null)
+                            ignite.log().error("Ignored cache group: " + grp.cacheOrGroupName(), e);
+                    }
+                }
+
+                if (partitions == 0)
+                    res.setRebalance(NOTHING_TO_REBALANCE);
+                else if (total == 0 && rebalanceInProgress)
+                    res.setRebalance(MINIMAL_REBALANCE);
+                else
+                    res.setRebalance(total > 0 ? Math.max(ready / total, MINIMAL_REBALANCE) : REBALANCE_COMPLETE);
+            }
+            catch (Exception e) {
+                res.setRebalance(REBALANCE_NOT_AVAILABLE);
+
+                ignite.log().error("Failed to collect rebalance metrics", e);
+            }
+
+            if (GridCacheUtils.isPersistenceEnabled(ignite.configuration())) {
+                IgniteClusterEx cluster = ignite.cluster();
+
+                Object consistentId = ignite.localNode().consistentId();
+
+                Collection<? extends BaselineNode> baseline = cluster.currentBaselineTopology();
+
+                boolean inBaseline = baseline.stream().anyMatch(n -> consistentId.equals(n.consistentId()));
+
+                res.setBaseline(inBaseline ? NODE_IN_BASELINE : NODE_NOT_IN_BASELINE);
+            }
+            else
+                res.setBaseline(BASELINE_NOT_AVAILABLE);
+
+            if (debug)
+                log(ignite.log(), "Collected rebalance metrics", getClass(), start0);
+
+            return res;
+        }
+
+        /** {@inheritDoc} */
+        @Override public String toString() {
+            return S.toString(VisorCacheRebalanceCollectorJob.class, this);
+        }
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskArg.java
new file mode 100644
index 0000000000000..d97fd50192572
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskArg.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.node;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.visor.VisorDataTransferObject;
+
+/**
+ * Argument for {@link VisorCacheRebalanceCollectorTask} task.
+ */
+public class VisorCacheRebalanceCollectorTaskArg extends VisorDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /**
+     * Default constructor.
+     */
+    public VisorCacheRebalanceCollectorTaskArg() {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        // No-op.
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorCacheRebalanceCollectorTaskArg.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskResult.java
new file mode 100644
index 0000000000000..1305cd2df87eb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorCacheRebalanceCollectorTaskResult.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.node;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.internal.dto.IgniteDataTransferObject;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Result object for {@link VisorCacheRebalanceCollectorTask} task.
+ */
+public class VisorCacheRebalanceCollectorTaskResult extends IgniteDataTransferObject {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** Rebalance state on nodes. */
+    private Map<UUID, Double> rebalance = new HashMap<>();
+
+    /** Nodes baseline status. */
+    private Map<UUID, VisorNodeBaselineStatus> baseline = new HashMap<>();
+
+    /**
+     * Default constructor.
+     */
+    public VisorCacheRebalanceCollectorTaskResult() {
+        // No-op.
+    }
+
+    /**
+     * @return Rebalance on nodes.
+     */
+    public Map<UUID, Double> getRebalance() {
+        return rebalance;
+    }
+
+    /**
+     * @return Baseline.
+     */
+    public Map<UUID, VisorNodeBaselineStatus> getBaseline() {
+        return baseline;
+    }
+
+    /**
+     * Add specified results.
+     *
+     * @param res Results to add.
+     */
+    public void add(VisorCacheRebalanceCollectorTaskResult res) {
+        assert res != null;
+
+        rebalance.putAll(res.getRebalance());
+        baseline.putAll(res.getBaseline());
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void writeExternalData(ObjectOutput out) throws IOException {
+        U.writeMap(out, rebalance);
+        U.writeMap(out, baseline);
+    }
+
+    /** {@inheritDoc} */
+    @Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
+        rebalance = U.readMap(in);
+        baseline = U.readMap(in);
+    }
+
+    /** {@inheritDoc} */
+    @Override public String toString() {
+        return S.toString(VisorCacheRebalanceCollectorTaskResult.class, this);
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeBaselineStatus.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeBaselineStatus.java
new file mode 100644
index 0000000000000..ea90be3a6ef2c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeBaselineStatus.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.visor.node;
+
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Node baseline status.
+ */
+public enum VisorNodeBaselineStatus {
+    /** */
+    NODE_IN_BASELINE,
+    /** */
+    NODE_NOT_IN_BASELINE,
+    /** */
+    BASELINE_NOT_AVAILABLE;
+
+    /** Enumerated values. */
+    private static final VisorNodeBaselineStatus[] VALS = values();
+
+    /**
+     * Efficiently gets enumerated value from its ordinal.
+     *
+     * @param ord Ordinal value.
+     * @return Enumerated value or {@code null} if ordinal out of range.
+     */
+    @Nullable public static VisorNodeBaselineStatus fromOrdinal(int ord) {
+        return ord >= 0 && ord < VALS.length ? VALS[ord] : null;
+    }
+}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
index 9025ed084b9c5..9a7d2b146618e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
@@ -28,8 +28,9 @@
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.FileSystemConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
 import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
 import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager;
 import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
 import org.apache.ignite.internal.processors.igfs.IgfsProcessorAdapter;
@@ -50,9 +51,15 @@
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isSystemCache;
 import static org.apache.ignite.internal.visor.compute.VisorComputeMonitoringHolder.COMPUTE_MONITORING_HOLDER_KEY;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.EVT_MAPPER;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.MINIMAL_REBALANCE;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.NOTHING_TO_REBALANCE;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.REBALANCE_COMPLETE;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.REBALANCE_NOT_AVAILABLE;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.VISOR_TASK_EVTS;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.checkExplicitTaskMonitoring;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.collectEvents;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.isProxyCache;
+import static org.apache.ignite.internal.visor.util.VisorTaskUtils.isRestartingCache;
 import static org.apache.ignite.internal.visor.util.VisorTaskUtils.log;
 
 /**
@@ -140,18 +147,6 @@ protected boolean compatibleWith(IgniteProductVersion ver) {
         return false;
     }
 
-    /**
-     * @param cacheName Cache name to check.
-     * @return {@code true} if cache on local node is not a data cache or near cache disabled.
-     */
-    private boolean proxyCache(String cacheName) {
-        GridDiscoveryManager discovery = ignite.context().discovery();
-
-        ClusterNode locNode = ignite.localNode();
-
-        return !(discovery.cacheAffinityNode(locNode, cacheName) || discovery.cacheNearNode(locNode, cacheName));
-    }
-
     /**
      * Collect memory metrics.
      *
@@ -194,38 +189,51 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto
 
             List<VisorCache> resCaches = res.getCaches();
 
-            for (String cacheName : cacheProc.cacheNames()) {
-                if (proxyCache(cacheName))
-                    continue;
+            boolean rebalanceInProgress = false;
 
-                boolean sysCache = isSystemCache(cacheName);
+            for (CacheGroupContext grp : cacheProc.cacheGroups()) {
+                boolean first = true;
 
-                if (arg.getSystemCaches() || !(sysCache || isIgfsCache(cfg, cacheName))) {
+                for (GridCacheContext cache : grp.caches()) {
                     long start0 = U.currentTimeMillis();
 
+                    String cacheName = cache.name();
+
                     try {
+                        if (isProxyCache(ignite, cacheName) || isRestartingCache(ignite, cacheName))
+                            continue;
+
                         GridCacheAdapter ca = cacheProc.internalCache(cacheName);
 
                         if (ca == null || !ca.context().started())
                             continue;
 
-                        CacheMetrics cm = ca.localMetrics();
+                        if (first) {
+                            CacheMetrics cm = ca.localMetrics();
+
+                            partitions += cm.getTotalPartitionsCount();
+
+                            long keysTotal = cm.getEstimatedRebalancingKeys();
+                            long keysReady = cm.getRebalancedKeys();
+
+                            if (keysReady >= keysTotal)
+                                keysReady = Math.max(keysTotal - 1, 0);
 
-                        partitions += cm.getTotalPartitionsCount();
+                            total += keysTotal;
+                            ready += keysReady;
 
-                        long partTotal = cm.getEstimatedRebalancingKeys();
-                        long partReady = cm.getRebalancedKeys();
+                            if (!rebalanceInProgress && cm.getRebalancingPartitionsCount() > 0)
+                                rebalanceInProgress = true;
 
-                        if (partReady >= partTotal)
-                            partReady = Math.max(partTotal - 1, 0);
+                            first = false;
+                        }
 
-                        total += partTotal;
-                        ready += partReady;
+                        boolean addToRes = arg.getSystemCaches() || !(isSystemCache(cacheName) || isIgfsCache(cfg, cacheName));
 
-                        if (all || cacheGrps.contains(ca.configuration().getGroupName()))
+                        if (addToRes && (all || cacheGrps.contains(ca.configuration().getGroupName())))
                             resCaches.add(new VisorCache(ignite, ca, arg.isCollectCacheMetrics()));
                     }
-                    catch(IllegalStateException | IllegalArgumentException e) {
+                    catch (IllegalStateException | IllegalArgumentException e) {
                         if (debug && ignite.log() != null)
                             ignite.log().error("Ignored cache: " + cacheName, e);
                     }
@@ -237,11 +245,14 @@ protected void caches(VisorNodeDataCollectorJobResult res, VisorNodeDataCollecto
             }
 
             if (partitions == 0)
-                res.setRebalance(-1);
+                res.setRebalance(NOTHING_TO_REBALANCE);
+            else if (total == 0 && rebalanceInProgress)
+                res.setRebalance(MINIMAL_REBALANCE);
             else
-                res.setRebalance(total > 0 ? ready / total : 1);
+                res.setRebalance(total > 0 ? Math.max(ready / total, MINIMAL_REBALANCE) : REBALANCE_COMPLETE);
         }
         catch (Exception e) {
+            res.setRebalance(REBALANCE_NOT_AVAILABLE);
             res.setCachesEx(new VisorExceptionWrapper(e));
         }
     }
@@ -260,7 +271,8 @@ protected void igfs(VisorNodeDataCollectorJobResult res) {
 
                 FileSystemConfiguration igfsCfg = igfs.configuration();
 
-                if (proxyCache(igfsCfg.getDataCacheConfiguration().getName()) || proxyCache(igfsCfg.getMetaCacheConfiguration().getName()))
+                if (isProxyCache(ignite, igfsCfg.getDataCacheConfiguration().getName()) ||
+                    isProxyCache(ignite, igfsCfg.getMetaCacheConfiguration().getName()))
                     continue;
 
                 try {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java
index eb161f82c6638..f8eb8690ce2ea 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorTaskResult.java
@@ -131,7 +131,8 @@ public boolean isEmpty() {
             readyTopVers.isEmpty() &&
             pendingExchanges.isEmpty() &&
             persistenceMetrics.isEmpty() &&
-            persistenceMetricsEx.isEmpty();
+            persistenceMetricsEx.isEmpty() &&
+            rebalance.isEmpty();
     }
 
     /**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
index fda9ba199bc0f..7ab1ffcb18b84 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
@@ -58,6 +58,10 @@
 import org.apache.ignite.cache.eviction.AbstractEvictionPolicyFactory;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.events.Event;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
+import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
+import org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl;
 import org.apache.ignite.internal.processors.igfs.IgfsEx;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.X;
@@ -109,6 +113,19 @@ public class VisorTaskUtils {
     /** Log files count limit */
     public static final int LOG_FILES_COUNT_LIMIT = 5000;
 
+    /** */
+    public static final int NOTHING_TO_REBALANCE = -1;
+
+    /** */
+    public static final int REBALANCE_NOT_AVAILABLE = -2;
+
+    /** */
+    public static final double MINIMAL_REBALANCE = 0.01;
+
+    /** */
+    public static final int REBALANCE_COMPLETE = 1;
+
+
     /** */
     private static final int DFLT_BUFFER_SIZE = 4096;
 
@@ -1248,4 +1265,30 @@ public static Collection<String> splitAddresses(String s) {
 
         return Arrays.asList(addrs);
     }
+
+    /**
+     * @param ignite Ignite.
+     * @param cacheName Cache name to check.
+     * @return {@code true} if cache on local node is not a data cache or near cache disabled.
+     */
+    public static boolean isProxyCache(IgniteEx ignite, String cacheName) {
+        GridDiscoveryManager discovery = ignite.context().discovery();
+
+        ClusterNode locNode = ignite.localNode();
+
+        return !(discovery.cacheAffinityNode(locNode, cacheName) || discovery.cacheNearNode(locNode, cacheName));
+    }
+
+    /**
+     * Check whether cache restarting in progress.
+     *
+     * @param ignite Grid.
+     * @param cacheName Cache name to check.
+     * @return {@code true} when cache restarting in progress.
+     */
+    public static boolean isRestartingCache(IgniteEx ignite, String cacheName)  {
+        IgniteCacheProxy<Object, Object> proxy = ignite.context().cache().jcache(cacheName);
+
+        return proxy instanceof IgniteCacheProxyImpl && ((IgniteCacheProxyImpl) proxy).isRestarting();
+    }
 }

From 6b49a3972c5d67a29e3f51144e20a7ba00b037f1 Mon Sep 17 00:00:00 2001
From: EdShangGG <eshangareev@gridgain.com>
Date: Wed, 12 Dec 2018 18:46:37 +0300
Subject: [PATCH 538/543] IGNITE-10492 No need to normalize QueryEntities if
 cache in restarting state.

(cherry picked from commit 0a7c288)

(cherry picked from commit 9390ea5)
---
 .../record/delta/MetaPageUpdateNextSnapshotId.java   | 10 +++++-----
 .../internal/processors/cache/ClusterCachesInfo.java |  2 +-
 .../processors/cache/GridCacheProcessor.java         | 12 ++++++++----
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java
index 2046ecd274d95..5068fe5b45ec5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java
@@ -27,22 +27,22 @@
  */
 public class MetaPageUpdateNextSnapshotId extends PageDeltaRecord {
     /** */
-    private final long nextSnapshotId;
+    private final long nextSnapshotTag;
 
     /**
      * @param pageId Meta page ID.
      */
-    public MetaPageUpdateNextSnapshotId(int grpId, long pageId, long nextSnapshotId) {
+    public MetaPageUpdateNextSnapshotId(int grpId, long pageId, long nextSnapshotTag) {
         super(grpId, pageId);
 
-        this.nextSnapshotId = nextSnapshotId;
+        this.nextSnapshotTag = nextSnapshotTag;
     }
 
     /** {@inheritDoc} */
     @Override public void applyDelta(PageMemory pageMem, long pageAddr) throws IgniteCheckedException {
         PageMetaIO io = PageMetaIO.VERSIONS.forPage(pageAddr);
 
-        io.setNextSnapshotTag(pageAddr, nextSnapshotId);
+        io.setNextSnapshotTag(pageAddr, nextSnapshotTag);
     }
 
     /** {@inheritDoc} */
@@ -54,7 +54,7 @@ public MetaPageUpdateNextSnapshotId(int grpId, long pageId, long nextSnapshotId)
      * @return Root ID.
      */
     public long nextSnapshotId() {
-        return nextSnapshotId;
+        return nextSnapshotTag;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index f3f72511fe58d..4c72e27fd5898 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -94,7 +94,7 @@ class ClusterCachesInfo {
     private final ConcurrentMap<String, DynamicCacheDescriptor> registeredTemplates = new ConcurrentHashMap<>();
 
     /** Caches currently being restarted. */
-    private final Collection<String> restartingCaches = new GridConcurrentHashSet<>();
+    private final Set<String> restartingCaches = new GridConcurrentHashSet<>();
 
     /** */
     private final IgniteLogger log;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 60b066ad17038..f48dc44798e35 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -5095,17 +5095,21 @@ else if (!isLocalAffinity(descCfg))
                 }
             }
             else {
+                CacheConfiguration cfg = new CacheConfiguration(ccfg);
+
                 req.deploymentId(IgniteUuid.randomUuid());
 
-                CacheConfiguration cfg = new CacheConfiguration(ccfg);
+                req.startCacheConfiguration(cfg);
 
                 CacheObjectContext cacheObjCtx = ctx.cacheObjects().contextForCache(cfg);
 
                 initialize(cfg, cacheObjCtx);
 
-                req.startCacheConfiguration(cfg);
-                req.schema(new QuerySchema(qryEntities != null ? QueryUtils.normalizeQueryEntities(qryEntities, cfg)
-                    : cfg.getQueryEntities()));
+                if (cachesInfo.restartingCaches().contains(req.cacheName()))
+                    req.schema(new QuerySchema(qryEntities));
+                else
+                    req.schema(new QuerySchema(qryEntities != null ? QueryUtils.normalizeQueryEntities(qryEntities, cfg)
+                            : cfg.getQueryEntities()));
             }
         }
         else {

From 820a54e8f1c01bb3a76306dd31fb1a668cdde19a Mon Sep 17 00:00:00 2001
From: Alexey Stelmak <spiderru5597@gmail.com>
Date: Fri, 7 Dec 2018 18:56:48 +0300
Subject: [PATCH 539/543] ignite-10348 Safely recreate metastore to mitigate
 IGNITE-8735

---
 .../internal/pagemem/PageIdAllocator.java     |   5 +-
 .../processors/cache/GridCacheProcessor.java  |  17 +
 .../file/FilePageStoreManager.java            |   2 +-
 .../persistence/metastorage/MetaStorage.java  | 407 +++++++++++++++++-
 .../metastorage/MetastorageDataRow.java       |   4 +-
 .../metastorage/MetastorageTree.java          |  11 +-
 .../IgnitePdsCorruptedStoreTest.java          |   3 +-
 .../IgniteMetaStorageBasicTest.java           | 273 +++++++++++-
 8 files changed, 700 insertions(+), 22 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
index b3e4b07b9fb67..d91d31da32957 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/PageIdAllocator.java
@@ -35,8 +35,11 @@ public interface PageIdAllocator {
     /** Special partition reserved for index space. */
     public static final int INDEX_PARTITION = 0xFFFF;
 
+    /** Old special partition reserved for metastore space. */
+    public static final int OLD_METASTORE_PARTITION = 0x0;
+
     /** Special partition reserved for metastore space. */
-    public static final int METASTORE_PARTITION = 0x0;
+    public static final int METASTORE_PARTITION = 0x1;
 
     /**
      * Allocates a page from the space for the given partition ID and the given flags.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index f48dc44798e35..4cba9a2ec6de7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -272,6 +272,9 @@ public class GridCacheProcessor extends GridProcessorAdapter {
     /** Cache recovery lifecycle state and actions. */
     private final CacheRecoveryLifecycle recovery = new CacheRecoveryLifecycle();
 
+    /** Tmp storage for meta migration. */
+    private MetaStorage.TmpStorage tmpStorage;
+
     /**
      * @param ctx Kernal context.
      */
@@ -5215,6 +5218,20 @@ public <T> T clone(final T obj) throws IgniteCheckedException {
         });
     }
 
+    /**
+     * Get Temporary storage
+     */
+    public MetaStorage.TmpStorage getTmpStorage() {
+        return tmpStorage;
+    }
+
+    /**
+     * Set Temporary storage
+     */
+    public void setTmpStorage(MetaStorage.TmpStorage tmpStorage) {
+        this.tmpStorage = tmpStorage;
+    }
+
     /**
      * Recovery lifecycle for caches.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 09ec837e00832..52083a6f938f4 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -341,7 +341,7 @@ public FilePageStoreManager(GridKernalContext ctx) {
             CacheStoreHolder holder = initDir(
                 new File(storeWorkDir, META_STORAGE_NAME),
                 grpId,
-                1,
+                PageIdAllocator.METASTORE_PARTITION + 1,
                 delta -> {/* No-op */} );
 
             CacheStoreHolder old = idxCacheStores.put(grpId, holder);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index 95a9df39df023..c9e8332b0b2df 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -17,13 +17,24 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.metastorage;
 
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
 import java.io.Serializable;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.Executor;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Stream;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
@@ -32,10 +43,11 @@
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.pagemem.wal.WALPointer;
 import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
 import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
 import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.IncompleteObject;
 import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
@@ -44,6 +56,7 @@
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.RootPage;
+import org.apache.ignite.internal.processors.cache.persistence.StorageException;
 import org.apache.ignite.internal.processors.cache.persistence.freelist.AbstractFreeList;
 import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
 import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
@@ -58,12 +71,14 @@
 import org.apache.ignite.internal.util.lang.GridCursor;
 import org.apache.ignite.internal.util.typedef.internal.CU;
 import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.marshaller.Marshaller;
 import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.jetbrains.annotations.NotNull;
 
 import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.OLD_METASTORE_PARTITION;
 import static org.apache.ignite.internal.pagemem.PageIdUtils.itemId;
 import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
 
@@ -77,9 +92,19 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     /** */
     public static final int METASTORAGE_CACHE_ID = CU.cacheId(METASTORAGE_CACHE_NAME);
 
+    /** This flag is used ONLY FOR TESTING the migration of a metastorage from Part 0 to Part 1. */
+    public static boolean PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = false;
+
     /** Marker for removed entry. */
     private static final byte[] TOMBSTONE = new byte[0];
 
+    /** Temporary metastorage memory size. */
+    private static final int TEMPORARY_METASTORAGE_IN_MEMORY_SIZE = 128 * 1024 * 1024;
+
+    /** Temporary metastorage buffer size (file). */
+    private static final int TEMPORARY_METASTORAGE_BUFFER_SIZE = 1024 * 1024;
+
+
     /** */
     private final IgniteWriteAheadLogManager wal;
 
@@ -99,7 +124,7 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     private DataRegionMetricsImpl regionMetrics;
 
     /** */
-    private boolean readOnly;
+    private final boolean readOnly;
 
     /** */
     private boolean empty;
@@ -122,6 +147,12 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
     /** */
     private final FailureProcessor failureProcessor;
 
+    /** Partition id. */
+    private int partId;
+
+    /** Cctx. */
+    private final GridCacheSharedContext cctx;
+
     /** */
     public MetaStorage(
         GridCacheSharedContext cctx,
@@ -129,6 +160,7 @@ public MetaStorage(
         DataRegionMetricsImpl regionMetrics,
         boolean readOnly
     ) {
+        this.cctx = cctx;
         wal = cctx.wal();
         this.dataRegion = dataRegion;
         this.regionMetrics = regionMetrics;
@@ -138,8 +170,78 @@ public MetaStorage(
     }
 
     /** */
-    public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedException {
-        getOrAllocateMetas();
+    public void init(GridCacheDatabaseSharedManager db) throws IgniteCheckedException {
+        initInternal(db);
+
+        if (!PRESERVE_LEGACY_METASTORAGE_PARTITION_ID) {
+            GridCacheProcessor gcProcessor = cctx.kernalContext().cache();
+
+            if (partId == OLD_METASTORE_PARTITION)
+                gcProcessor.setTmpStorage(copyDataToTmpStorage());
+            else if (gcProcessor.getTmpStorage() != null) {
+                restoreDataFromTmpStorage(gcProcessor.getTmpStorage());
+
+                gcProcessor.setTmpStorage(null);
+
+                // remove old partitions
+                CacheGroupContext cgc = cctx.cache().cacheGroup(METASTORAGE_CACHE_ID);
+
+                if (cgc != null) {
+                    db.schedulePartitionDestroy(METASTORAGE_CACHE_ID, OLD_METASTORE_PARTITION);
+
+                    db.schedulePartitionDestroy(METASTORAGE_CACHE_ID, PageIdAllocator.INDEX_PARTITION);
+                }
+            }
+        }
+    }
+
+    /**
+     * Copying all data from the 'meta' to temporary storage.
+     *
+     * @return Target temporary storage
+     */
+    private TmpStorage copyDataToTmpStorage() throws IgniteCheckedException {
+        TmpStorage tmpStorage = new TmpStorage(TEMPORARY_METASTORAGE_IN_MEMORY_SIZE, log);
+
+        GridCursor<MetastorageDataRow> cur = tree.find(null, null);
+
+        while (cur.next()) {
+            MetastorageDataRow row = cur.get();
+
+            tmpStorage.add(row.key(), row.value());
+        }
+
+        return tmpStorage;
+    }
+
+    /**
+     * Data recovery from temporary storage
+     *
+     * @param tmpStorage temporary storage.
+     */
+    private void restoreDataFromTmpStorage(TmpStorage tmpStorage) throws IgniteCheckedException {
+        for (Iterator<IgniteBiTuple<String, byte[]>> it = tmpStorage.stream().iterator(); it.hasNext(); ) {
+            IgniteBiTuple<String, byte[]> t = it.next();
+
+            putData(t.get1(), t.get2());
+        }
+
+        try {
+            tmpStorage.close();
+        }
+        catch (IOException e) {
+            log.error(e.getMessage(), e);
+        }
+    }
+
+    /**
+     * @param db Database.
+     */
+    private void initInternal(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedException {
+        if (PRESERVE_LEGACY_METASTORAGE_PARTITION_ID)
+            getOrAllocateMetas(partId = PageIdAllocator.OLD_METASTORE_PARTITION);
+        else if (!readOnly || getOrAllocateMetas(partId = PageIdAllocator.OLD_METASTORE_PARTITION))
+            getOrAllocateMetas(partId = PageIdAllocator.METASTORE_PARTITION);
 
         if (!empty) {
             freeList = new FreeListImpl(METASTORAGE_CACHE_ID, "metastorage",
@@ -149,7 +251,7 @@ public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedExcept
             MetastorageRowStore rowStore = new MetastorageRowStore(freeList, db);
 
             tree = new MetastorageTree(METASTORAGE_CACHE_ID, dataRegion.pageMemory(), wal, rmvId,
-                freeList, rowStore, treeRoot.pageId().pageId(), treeRoot.isAllocated(), failureProcessor);
+                freeList, rowStore, treeRoot.pageId().pageId(), treeRoot.isAllocated(), failureProcessor, partId);
 
             if (!readOnly)
                 ((GridCacheDatabaseSharedManager)db).addCheckpointListener(this);
@@ -224,6 +326,23 @@ public void init(IgniteCacheDatabaseSharedManager db) throws IgniteCheckedExcept
         return res;
     }
 
+    /**
+     * Read all items from metastore.
+     */
+    public Collection<IgniteBiTuple<String, byte[]>> readAll() throws IgniteCheckedException {
+        ArrayList<IgniteBiTuple<String, byte[]>> res = new ArrayList<>();
+
+        GridCursor<MetastorageDataRow> cur = tree.find(null, null);
+
+        while (cur.next()) {
+            MetastorageDataRow row = cur.get();
+
+            res.add(new IgniteBiTuple<>(row.key(), row.value()));
+        }
+
+        return res;
+    }
+
     /** {@inheritDoc} */
     @Override public void write(@NotNull String key, @NotNull Serializable val) throws IgniteCheckedException {
         assert val != null;
@@ -312,11 +431,16 @@ private void checkRootsPageIdFlag(long treeRoot, long reuseListRoot) throws Stor
                 + U.hexLong(reuseListRoot) + ", METASTORAGE_CACHE_ID=" + METASTORAGE_CACHE_ID);
     }
 
-    /** */
-    private void getOrAllocateMetas() throws IgniteCheckedException {
-        PageMemoryEx pageMem = (PageMemoryEx)dataRegion.pageMemory();
+    /**
+     * Initializing the selected partition for use as MetaStorage
+     *
+     * @param partId Partition id.
+     * @return true if the partion is empty
+     */
+    private boolean getOrAllocateMetas(int partId) throws IgniteCheckedException {
+        empty = false;
 
-        int partId = 0;
+        PageMemoryEx pageMem = (PageMemoryEx)dataRegion.pageMemory();
 
         long partMetaId = pageMem.partitionMetaPageId(METASTORAGE_CACHE_ID, partId);
         long partMetaPage = pageMem.acquirePage(METASTORAGE_CACHE_ID, partMetaId);
@@ -328,7 +452,7 @@ private void getOrAllocateMetas() throws IgniteCheckedException {
                     if (PageIO.getType(pageAddr) != PageIO.T_PART_META) {
                         empty = true;
 
-                        return;
+                        return true;
                     }
 
                     PagePartitionMetaIO io = PageIO.getPageIO(pageAddr);
@@ -403,6 +527,8 @@ private void getOrAllocateMetas() throws IgniteCheckedException {
         finally {
             pageMem.releasePage(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
         }
+
+        return false;
     }
 
     /**
@@ -448,8 +574,6 @@ public PageMemory pageMemory() {
     private void saveStoreMetadata() throws IgniteCheckedException {
         PageMemoryEx pageMem = (PageMemoryEx) pageMemory();
 
-        int partId = 0;
-
         long partMetaId = pageMem.partitionMetaPageId(METASTORAGE_CACHE_ID, partId);
         long partMetaPage = pageMem.acquirePage(METASTORAGE_CACHE_ID, partMetaId);
 
@@ -495,7 +619,7 @@ public void applyUpdate(String key, byte[] value) throws IgniteCheckedException
     }
 
     /** */
-    public static class FreeListImpl extends AbstractFreeList<MetastorageDataRow> {
+    public class FreeListImpl extends AbstractFreeList<MetastorageDataRow> {
         /** {@inheritDoc} */
         FreeListImpl(int cacheId, String name, DataRegionMetricsImpl regionMetrics, DataRegion dataRegion,
             ReuseList reuseList,
@@ -510,7 +634,7 @@ public static class FreeListImpl extends AbstractFreeList<MetastorageDataRow> {
 
         /** {@inheritDoc} */
         @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
-            return pageMem.allocatePage(grpId, PageIdAllocator.METASTORE_PARTITION, FLAG_DATA);
+            return pageMem.allocatePage(grpId, partId, FLAG_DATA);
         }
 
         /**
@@ -594,4 +718,259 @@ final MetastorageDataRow readRow(String key, long link)
             return new MetastorageDataRow(link, key, incomplete.data());
         }
     }
+
+    /**
+     * Temporary storage internal
+     */
+    private interface TmpStorageInternal extends Closeable {
+        /**
+         * Put data
+         *
+         * @param key Key.
+         * @param val Value.
+         */
+        boolean add(String key, byte[] val) throws IOException;
+
+        /**
+         * Read data from storage
+         */
+        Stream<IgniteBiTuple<String, byte[]>> stream() throws IOException;
+    }
+
+    /**
+     * Temporary storage (memory)
+     */
+    private static class MemoryTmpStorage implements TmpStorageInternal {
+        /** Buffer. */
+        final ByteBuffer buf;
+
+        /** Size. */
+        int size;
+
+        /**
+         * @param size Size.
+         */
+        MemoryTmpStorage(int size) {
+            buf = ByteBuffer.allocateDirect(size);
+        }
+
+        /** {@inheritDoc} */
+        @Override public boolean add(String key, byte[] val) {
+            byte[] keyData = key.getBytes(StandardCharsets.UTF_8);
+
+            if (val.length + keyData.length + 8 > buf.remaining())
+                return false;
+
+            buf.putInt(keyData.length).putInt(val.length).put(keyData).put(val);
+
+            size++;
+
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Stream<IgniteBiTuple<String, byte[]>> stream() {
+            buf.flip();
+
+            return Stream.generate(() -> {
+                int keyLen = buf.getInt();
+                int dataLen = buf.getInt();
+
+                byte[] tmpBuf = new byte[Math.max(keyLen, dataLen)];
+
+                buf.get(tmpBuf, 0, keyLen);
+
+                String key = new String(tmpBuf, 0, keyLen, StandardCharsets.UTF_8);
+
+                buf.get(tmpBuf, 0, dataLen);
+
+                return new IgniteBiTuple<>(key, tmpBuf.length > dataLen ? Arrays.copyOf(tmpBuf, dataLen) : tmpBuf);
+            }).limit(size);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void close() throws IOException {
+        }
+    }
+
+    /**
+     * Temporary storage (file)
+     */
+    private static class FileTmpStorage implements TmpStorageInternal {
+        /** Cache. */
+        final ByteBuffer cache = ByteBuffer.allocateDirect(TEMPORARY_METASTORAGE_BUFFER_SIZE);
+
+        /** File. */
+        RandomAccessFile file;
+
+        /** Size. */
+        long size;
+
+        /** {@inheritDoc} */
+        @Override public boolean add(String key, byte[] val) throws IOException {
+            if (file == null)
+                file = new RandomAccessFile(File.createTempFile("m_storage", "bin"), "rw");
+
+            byte[] keyData = key.getBytes(StandardCharsets.UTF_8);
+
+            if (val.length + keyData.length + 8 > cache.remaining())
+                flushCache(false);
+
+            cache.putInt(keyData.length).putInt(val.length).put(keyData).put(val);
+
+            size++;
+
+            return true;
+        }
+
+        /** {@inheritDoc} */
+        @Override public Stream<IgniteBiTuple<String, byte[]>> stream() throws IOException {
+            if (file == null)
+                return Stream.empty();
+
+            flushCache(true);
+
+            file.getChannel().position(0);
+
+            readToCache();
+
+            return Stream.generate(() -> {
+                if (cache.remaining() <= 8) {
+                    cache.compact();
+
+                    try {
+                        readToCache();
+                    }
+                    catch (IOException e) {
+                        throw new IgniteException(e);
+                    }
+                }
+
+                int keyLen = cache.getInt();
+                int dataLen = cache.getInt();
+
+                if (cache.remaining() < keyLen + dataLen) {
+                    cache.compact();
+
+                    try {
+                        readToCache();
+                    }
+                    catch (IOException e) {
+                        throw new IgniteException(e);
+                    }
+                }
+
+                byte[] tmpBuf = new byte[Math.max(keyLen, dataLen)];
+
+                cache.get(tmpBuf, 0, keyLen);
+
+                String key = new String(tmpBuf, 0, keyLen, StandardCharsets.UTF_8);
+
+                cache.get(tmpBuf, 0, dataLen);
+
+                return new IgniteBiTuple<>(key, tmpBuf.length > dataLen ? Arrays.copyOf(tmpBuf, dataLen) : tmpBuf);
+            }).limit(size);
+        }
+
+        /** {@inheritDoc} */
+        @Override public void close() throws IOException {
+            file.close();
+        }
+
+        /**
+         * Read data to cache
+         */
+        private void readToCache() throws IOException {
+            int len = (int)Math.min(file.length() - file.getChannel().position(), cache.remaining());
+
+            while (len > 0)
+                len -= file.getChannel().read(cache);
+
+            cache.flip();
+        }
+
+        /**
+         * Write cache to file.
+         *
+         * @param force force metadata.
+         */
+        private void flushCache(boolean force) throws IOException {
+            if (cache.position() > 0) {
+                cache.flip();
+
+                while (cache.remaining() > 0)
+                    file.getChannel().write(cache);
+
+                cache.clear();
+            }
+
+            file.getChannel().force(force);
+        }
+    }
+
+    /**
+     * Temporary storage
+     */
+    public static class TmpStorage implements Closeable {
+        /** Chain of internal storages. */
+        final List<TmpStorageInternal> chain = new ArrayList<>(2);
+
+        /** Current internal storage. */
+        TmpStorageInternal current;
+
+        /** Logger. */
+        final IgniteLogger log;
+
+        /**
+         * @param memBufSize Memory buffer size.
+         * @param log Logger.
+         */
+        TmpStorage(int memBufSize, IgniteLogger log) {
+            this.log = log;
+
+            chain.add(current = new MemoryTmpStorage(memBufSize));
+        }
+
+        /**
+         * Put data
+         *
+         * @param key Key.
+         * @param val Value.
+         */
+        public void add(String key, byte[] val) throws IgniteCheckedException {
+            try {
+                while (!current.add(key, val))
+                    chain.add(current = new FileTmpStorage());
+            }
+            catch (IOException e) {
+                throw new IgniteCheckedException(e);
+            }
+        }
+
+        /**
+         * Read data from storage
+         */
+        public Stream<IgniteBiTuple<String, byte[]>> stream() {
+            return chain.stream().flatMap(storage -> {
+                try {
+                    return storage.stream();
+                }
+                catch (IOException e) {
+                    throw new IgniteException(e);
+                }
+            });
+        }
+
+        /** {@inheritDoc} */
+        @Override public void close() throws IOException {
+            for (TmpStorageInternal storage : chain) {
+                try {
+                    storage.close();
+                }
+                catch (IOException ex) {
+                    log.error(ex.getMessage(), ex);
+                }
+            }
+        }
+    }
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java
index 271efdff8bd94..eff8ab8a718a6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageDataRow.java
@@ -17,6 +17,8 @@
 
 package org.apache.ignite.internal.processors.cache.persistence.metastorage;
 
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.processors.cache.persistence.Storable;
 
 /**
@@ -61,7 +63,7 @@ public int hash() {
     /** {@inheritDoc} */
     @Override
     public int partition() {
-        return 0;
+        return MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID ? PageIdAllocator.OLD_METASTORE_PARTITION: PageIdAllocator.METASTORE_PARTITION;
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
index 0c28289dd591b..1c0e6f3119524 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetastorageTree.java
@@ -19,7 +19,6 @@
 
 import java.util.concurrent.atomic.AtomicLong;
 import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageMemory;
 import org.apache.ignite.internal.pagemem.PageUtils;
 import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
@@ -45,6 +44,9 @@ public class MetastorageTree extends BPlusTree<MetastorageSearchRow, Metastorage
     /** */
     private MetastorageRowStore rowStore;
 
+    /** Partition id. */
+    private final int partId;
+
     /**
      * @param pageMem Page memory instance.
      * @param wal WAL manager.
@@ -64,12 +66,15 @@ public MetastorageTree(int cacheId,
         MetastorageRowStore rowStore,
         long metaPageId,
         boolean initNew,
-        @Nullable FailureProcessor failureProcessor) throws IgniteCheckedException {
+        @Nullable FailureProcessor failureProcessor,
+        int partId) throws IgniteCheckedException {
         super("Metastorage", cacheId, pageMem, wal,
             globalRmvId, metaPageId, reuseList, MetastorageInnerIO.VERSIONS, MetastoreLeafIO.VERSIONS, failureProcessor);
 
         this.rowStore = rowStore;
 
+        this.partId = partId;
+
         initTree(initNew);
     }
 
@@ -100,7 +105,7 @@ public MetastorageRowStore rowStore() {
 
     /** {@inheritDoc} */
     @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
-        return pageMem.allocatePage(grpId, PageIdAllocator.METASTORE_PARTITION, FLAG_DATA);
+        return pageMem.allocatePage(grpId, partId, FLAG_DATA);
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
index 15a1e59d54846..6beadae2b9372 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCorruptedStoreTest.java
@@ -36,6 +36,7 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
 import org.apache.ignite.internal.pagemem.PageIdUtils;
 import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
 import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
@@ -219,7 +220,7 @@ public void testMetaStorageCorruption() throws Exception {
 
         MetaStorage metaStorage = ignite.context().cache().context().database().metaStorage();
 
-        corruptTreeRoot(ignite, (PageMemoryEx)metaStorage.pageMemory(), METASTORAGE_CACHE_ID, 0);
+        corruptTreeRoot(ignite, (PageMemoryEx)metaStorage.pageMemory(), METASTORAGE_CACHE_ID, PageIdAllocator.METASTORE_PARTITION);
 
         stopGrid(0);
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
index b6dfc12826177..567ccf7ffce78 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
@@ -17,9 +17,17 @@
 package org.apache.ignite.internal.processors.cache.persistence.metastorage;
 
 import java.io.Serializable;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.configuration.DataRegionConfiguration;
@@ -29,6 +37,7 @@
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.lang.IgniteBiTuple;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
@@ -148,12 +157,274 @@ public void testMetaStorageMassivePutRandom() throws Exception {
 
                 metaStorage.remove(key);
 
-                metaStorage.putData(key, arr/*b.toString().getBytes()*/);
+                metaStorage.putData(key, arr);
             }
         }
         finally {
             db.checkpointReadUnlock();
         }
+
+        stopGrid();
+    }
+
+    /**
+     * @param metaStorage Meta storage.
+     * @param size Size.
+     */
+    private Map<String, byte[]> putDataToMetaStorage(MetaStorage metaStorage, int size, int from) throws IgniteCheckedException {
+        Map<String, byte[]> res = new HashMap<>();
+
+        for (Iterator<IgniteBiTuple<String, byte[]>> it = generateTestData(size, from).iterator(); it.hasNext(); ) {
+            IgniteBiTuple<String, byte[]> d = it.next();
+
+            metaStorage.putData(d.getKey(), d.getValue());
+
+            res.put(d.getKey(), d.getValue());
+        }
+
+        return res;
+    }
+
+    /**
+     * Testing data migration between metastorage partitions (delete partition case)
+     */
+    public void testDeletePartitionFromMetaStorageMigration() throws Exception {
+        final Map<String, byte[]> testData = new HashMap<>();
+
+        MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = true;
+
+        try {
+            IgniteEx ig = startGrid(0);
+
+            ig.cluster().active(true);
+
+            IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database();
+
+            MetaStorage metaStorage = db.metaStorage();
+
+            assertNotNull(metaStorage);
+
+            db.checkpointReadLock();
+
+            try {
+                testData.putAll(putDataToMetaStorage(metaStorage, 1_000, 0));
+            }
+            finally {
+                db.checkpointReadUnlock();
+            }
+
+            db.waitForCheckpoint("Test");
+
+            ((GridCacheDatabaseSharedManager)db).enableCheckpoints(false);
+
+            db.checkpointReadLock();
+
+            try {
+                testData.putAll(putDataToMetaStorage(metaStorage, 1_000, 1_000));
+            }
+            finally {
+                db.checkpointReadUnlock();
+            }
+
+            stopGrid(0);
+
+            MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = false;
+
+            IgniteConfiguration cfg = getConfiguration(getTestIgniteInstanceName(0));
+
+            cfg.getDataStorageConfiguration().setCheckpointFrequency(3600 * 1000L);
+
+            ig = (IgniteEx)startGrid(getTestIgniteInstanceName(0), optimize(cfg), null);
+
+            ig.cluster().active(true);
+
+            db = ig.context().cache().context().database();
+
+            metaStorage = db.metaStorage();
+
+            assertNotNull(metaStorage);
+
+            db.checkpointReadLock();
+
+            try {
+                testData.putAll(putDataToMetaStorage(metaStorage, 1_000, 2_000));
+            }
+            finally {
+                db.checkpointReadUnlock();
+            }
+
+            db.waitForCheckpoint("Test");
+
+            stopGrid(0);
+
+            ig = startGrid(0);
+
+            ig.cluster().active(true);
+
+            db = ig.context().cache().context().database();
+
+            metaStorage = db.metaStorage();
+
+            assertNotNull(metaStorage);
+
+            db.checkpointReadLock();
+            try {
+                Collection<IgniteBiTuple<String, byte[]>> read = metaStorage.readAll();
+
+                int cnt = 0;
+                for (IgniteBiTuple<String, byte[]> r : read) {
+                    byte[] test = testData.get(r.get1());
+
+                    if (test != null) {
+                        Assert.assertArrayEquals(r.get2(), test);
+
+                        cnt++;
+                    }
+                }
+
+                assertEquals(cnt, testData.size());
+            }
+            finally {
+                db.checkpointReadUnlock();
+            }
+        }
+        finally {
+            MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = false;
+        }
+
+    }
+
+    /**
+     * Testing data migration between metastorage partitions
+     */
+    public void testMetaStorageMigration() throws Exception {
+        final Map<String, byte[]> testData = new HashMap<>(5_000);
+
+        generateTestData(5_000, -1).forEach(t -> testData.put(t.get1(), t.get2()));
+
+        MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = true;
+
+        try {
+            IgniteEx ig = startGrid(0);
+
+            ig.cluster().active(true);
+
+            IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database();
+
+            MetaStorage metaStorage = db.metaStorage();
+
+            assertNotNull(metaStorage);
+
+            db.checkpointReadLock();
+
+            try {
+                for (Map.Entry<String, byte[]> v : testData.entrySet())
+                    metaStorage.putData(v.getKey(), v.getValue());
+            }
+            finally {
+                db.checkpointReadUnlock();
+            }
+
+            stopGrid(0);
+
+            MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = false;
+
+            ig = startGrid(0);
+
+            ig.cluster().active(true);
+
+            db = ig.context().cache().context().database();
+
+            metaStorage = db.metaStorage();
+
+            assertNotNull(metaStorage);
+
+            db.checkpointReadLock();
+
+            try {
+                Collection<IgniteBiTuple<String, byte[]>> read = metaStorage.readAll();
+
+                int cnt = 0;
+                for (IgniteBiTuple<String, byte[]> r : read) {
+                    byte[] test = testData.get(r.get1());
+
+                    if (test != null) {
+                        Assert.assertArrayEquals(r.get2(), test);
+
+                        cnt++;
+                    }
+                }
+
+                assertEquals(cnt, testData.size());
+            }
+            finally {
+                db.checkpointReadUnlock();
+            }
+        }
+        finally {
+            MetaStorage.PRESERVE_LEGACY_METASTORAGE_PARTITION_ID = false;
+        }
+    }
+
+    /**
+     * Testing temporary storage
+     */
+    public void testMetaStoreMigrationTmpStorage() throws Exception {
+        List<IgniteBiTuple<String, byte[]>> data = generateTestData(2_000, -1).collect(Collectors.toList());
+
+        // memory
+        try (MetaStorage.TmpStorage tmpStorage = new MetaStorage.TmpStorage(4 * 1024 * 1024, log)) {
+            for (IgniteBiTuple<String, byte[]> item : data)
+                tmpStorage.add(item.get1(), item.get2());
+
+            compare(tmpStorage.stream().iterator(), data.iterator());
+        }
+
+        // file
+        try (MetaStorage.TmpStorage tmpStorage = new MetaStorage.TmpStorage(4 * 1024, log)) {
+            for (IgniteBiTuple<String, byte[]> item : data)
+                tmpStorage.add(item.get1(), item.get2());
+
+            compare(tmpStorage.stream().iterator(), data.iterator());
+        }
+    }
+
+    /**
+     * Test data generation
+     */
+    private static Stream<IgniteBiTuple<String, byte[]>> generateTestData(int size, int fromKey) {
+        final AtomicInteger idx = new AtomicInteger(fromKey);
+        final Random rnd = new Random();
+
+        return Stream.generate(() -> {
+            byte[] val = new byte[1024];
+
+            rnd.nextBytes(val);
+
+            return new IgniteBiTuple<>("KEY_" + (fromKey < 0 ? rnd.nextInt() : idx.getAndIncrement()), val);
+        }).limit(size);
+    }
+
+    /**
+     * Compare two iterator
+     *
+     * @param it It.
+     * @param it1 It 1.
+     */
+    private static void compare(Iterator<IgniteBiTuple<String, byte[]>> it, Iterator<IgniteBiTuple<String, byte[]>> it1) {
+        while (true) {
+            Assert.assertEquals(it.hasNext(), it1.hasNext());
+
+            if (!it.hasNext())
+                break;
+
+            IgniteBiTuple<String, byte[]> i = it.next();
+            IgniteBiTuple<String, byte[]> i1 = it1.next();
+
+            Assert.assertEquals(i.get1(), i.get1());
+
+            Assert.assertArrayEquals(i.get2(), i1.get2());
+        }
     }
 
     /**

From d661730f87ec66dc78dc840a252e6e608a39c84c Mon Sep 17 00:00:00 2001
From: Vladislav Pyatkov <vldpyatkov@gmail.com>
Date: Thu, 13 Dec 2018 12:26:41 +0300
Subject: [PATCH 540/543] IGNITE-10446 Fix control.sh --cache idle_verify fail
 with NPE when node left grid

Signed-off-by: Dmitriy Govorukhin <dmitriy.govorukhin@gmail.com>
(cherry picked from commit d7af985383d9e82608fd77904415960492ed6290)
---
 .../internal/commandline/CommandHandler.java  |   4 +-
 .../cache/verify/IdleVerifyResultV2.java      |  34 +++-
 .../VerifyBackupPartitionsDumpTask.java       |  28 +++
 .../verify/VerifyBackupPartitionsTaskV2.java  |  31 ++-
 .../ignite/util/GridCommandHandlerTest.java   | 189 +++++++++++++-----
 5 files changed, 231 insertions(+), 55 deletions(-)

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 849f4e58d04a2..4cff4986223c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -869,7 +869,7 @@ private void cacheContention(GridClient client, CacheArguments cacheArgs) throws
             log("Contention check failed on nodes:");
 
             for (Map.Entry<UUID, Exception> e : res.exceptions().entrySet()) {
-                log("Node ID = " + e.getKey());
+                log("Node ID: " + e.getKey());
 
                 log("Exception message:");
                 log(e.getValue().getMessage());
@@ -904,7 +904,7 @@ private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) t
             log("Index validation failed on nodes:");
 
             for (Map.Entry<UUID, Exception> e : taskRes.exceptions().entrySet()) {
-                log(i("Node ID = " + e.getKey()));
+                log(i("Node ID: " + e.getKey()));
 
                 log(i("Exception message:"));
                 log(i(e.getValue().getMessage(), 2));
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
index c31d6627ed94a..a153063d8998a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
@@ -21,6 +21,7 @@
 import java.io.ObjectOutput;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import java.util.function.Consumer;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.S;
@@ -43,6 +44,9 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject {
     /** Moving partitions. */
     private Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions;
 
+    /** Exceptions. */
+    private Map<UUID, Exception> exceptions;
+
     /**
      * @param cntrConflicts Counter conflicts.
      * @param hashConflicts Hash conflicts.
@@ -51,11 +55,13 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject {
     public IdleVerifyResultV2(
         Map<PartitionKeyV2, List<PartitionHashRecordV2>> cntrConflicts,
         Map<PartitionKeyV2, List<PartitionHashRecordV2>> hashConflicts,
-        Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions
+        Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions,
+        Map<UUID, Exception> exceptions
     ) {
         this.cntrConflicts = cntrConflicts;
         this.hashConflicts = hashConflicts;
         this.movingPartitions = movingPartitions;
+        this.exceptions = exceptions;
     }
 
     /**
@@ -64,11 +70,17 @@ public IdleVerifyResultV2(
     public IdleVerifyResultV2() {
     }
 
+    /** {@inheritDoc} */
+    @Override public byte getProtocolVersion() {
+        return V2;
+    }
+
     /** {@inheritDoc} */
     @Override protected void writeExternalData(ObjectOutput out) throws IOException {
         U.writeMap(out, cntrConflicts);
         U.writeMap(out, hashConflicts);
         U.writeMap(out, movingPartitions);
+        U.writeMap(out, exceptions);
     }
 
     /** {@inheritDoc} */
@@ -77,6 +89,9 @@ public IdleVerifyResultV2() {
         cntrConflicts = U.readMap(in);
         hashConflicts = U.readMap(in);
         movingPartitions = U.readMap(in);
+
+        if (protoVer >= V2)
+            exceptions = U.readMap(in);
     }
 
     /**
@@ -107,6 +122,13 @@ public boolean hasConflicts() {
         return !F.isEmpty(hashConflicts()) || !F.isEmpty(counterConflicts());
     }
 
+    /**
+     * @return Exceptions on nodes.
+     */
+    public Map<UUID, Exception> exceptions() {
+        return exceptions;
+    }
+
     /**
      * Print formatted result to given printer.
      *
@@ -159,6 +181,16 @@ public void print(Consumer<String> printer) {
 
             printer.accept("\n");
         }
+
+        if (!F.isEmpty(exceptions())) {
+            printer.accept("Idle verify failed on nodes:\n");
+
+            for (Map.Entry<UUID, Exception> e : exceptions().entrySet()) {
+                printer.accept("Node ID: " + e.getKey() + "\n");
+                printer.accept("Exception message:" + "\n");
+                printer.accept(e.getValue().getMessage() + "\n");
+            }
+        }
     }
 
     /** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
index 7bbd9f19b9b15..5945f4d0990b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsDumpTask.java
@@ -30,15 +30,18 @@
 import java.util.TreeMap;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
 import org.apache.ignite.compute.ComputeTaskAdapter;
 import org.apache.ignite.internal.processors.task.GridInternal;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyDumpTaskArg;
 import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg;
 import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.resources.LoggerResource;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -68,6 +71,10 @@ public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter<VisorIdle
     @IgniteInstanceResource
     private Ignite ignite;
 
+    /** Injected logger. */
+    @LoggerResource
+    private IgniteLogger log;
+
     /** {@inheritDoc} */
     @Nullable @Override public Map<? extends ComputeJob, ClusterNode> map(
         List<ClusterNode> subgrid, VisorIdleVerifyTaskArg arg) throws IgniteException {
@@ -83,6 +90,9 @@ public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter<VisorIdle
         Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes = new TreeMap<>(buildPartitionKeyComparator());
 
         for (ComputeJobResult res : results) {
+            if (res.getException() != null)
+                continue;
+
             Map<PartitionKeyV2, PartitionHashRecordV2> nodeHashes = res.getData();
 
             for (Map.Entry<PartitionKeyV2, PartitionHashRecordV2> e : nodeHashes.entrySet()) {
@@ -111,6 +121,22 @@ public class VerifyBackupPartitionsDumpTask extends ComputeTaskAdapter<VisorIdle
         return writeHashes(partitions, delegate.reduce(results), skippedRecords);
     }
 
+    /** {@inheritDoc} */
+    @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) throws
+        IgniteException {
+        ComputeJobResultPolicy superRes = super.result(res, rcvd);
+
+        // Deny failover.
+        if (superRes == ComputeJobResultPolicy.FAILOVER) {
+            superRes = ComputeJobResultPolicy.WAIT;
+
+            log.warning("VerifyBackupPartitionsJobV2 failed on node " +
+                "[consistentId=" + res.getNode().consistentId() + "]", res.getException());
+        }
+
+        return superRes;
+    }
+
     /**
      * Checking conditions for adding given record to result.
      *
@@ -140,6 +166,8 @@ record = records.get(i);
 
     /**
      * @param partitions Dump result.
+     * @param conflictRes Conflict results.
+     * @param skippedRecords Number of empty partitions.
      * @return Path where results are written.
      * @throws IgniteException If failed to write the file.
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
index d5af22df8f947..5240fb7af5264 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
@@ -25,6 +25,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ForkJoinPool;
@@ -40,6 +41,7 @@
 import org.apache.ignite.compute.ComputeJob;
 import org.apache.ignite.compute.ComputeJobAdapter;
 import org.apache.ignite.compute.ComputeJobResult;
+import org.apache.ignite.compute.ComputeJobResultPolicy;
 import org.apache.ignite.compute.ComputeTaskAdapter;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.CacheGroupContext;
@@ -71,6 +73,10 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
     /** First version of Ignite that is capable of executing Idle Verify V2. */
     public static final IgniteProductVersion V2_SINCE_VER = IgniteProductVersion.fromString("2.5.0");
 
+    /** Injected logger. */
+    @LoggerResource
+    private IgniteLogger log;
+
     /** */
     private static final long serialVersionUID = 0L;
 
@@ -89,8 +95,15 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
     @Nullable @Override public IdleVerifyResultV2 reduce(List<ComputeJobResult> results)
         throws IgniteException {
         Map<PartitionKeyV2, List<PartitionHashRecordV2>> clusterHashes = new HashMap<>();
+        Map<UUID, Exception> exceptions = new HashMap<>();
 
         for (ComputeJobResult res : results) {
+            if (res.getException() != null) {
+                exceptions.put(res.getNode().id(), res.getException());
+
+                continue;
+            }
+
             Map<PartitionKeyV2, PartitionHashRecordV2> nodeHashes = res.getData();
 
             for (Map.Entry<PartitionKeyV2, PartitionHashRecordV2> e : nodeHashes.entrySet()) {
@@ -135,7 +148,23 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
             }
         }
 
-        return new IdleVerifyResultV2(updateCntrConflicts, hashConflicts, movingParts);
+        return new IdleVerifyResultV2(updateCntrConflicts, hashConflicts, movingParts, exceptions);
+    }
+
+    /** {@inheritDoc} */
+    @Override public ComputeJobResultPolicy result(ComputeJobResult res, List<ComputeJobResult> rcvd) throws
+        IgniteException {
+        ComputeJobResultPolicy superRes = super.result(res, rcvd);
+
+        // Deny failover.
+        if (superRes == ComputeJobResultPolicy.FAILOVER) {
+            superRes = ComputeJobResultPolicy.WAIT;
+
+            log.warning("VerifyBackupPartitionsJobV2 failed on node " +
+                "[consistentId=" + res.getNode().consistentId() + "]", res.getException());
+        }
+
+        return superRes;
     }
 
     /**
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 88bd8961b1cdb..60c2f699d1aef 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -31,6 +31,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.UUID;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
@@ -46,6 +47,7 @@
 import org.apache.ignite.IgniteAtomicSequence;
 import org.apache.ignite.IgniteCache;
 import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
@@ -54,6 +56,7 @@
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.GridJobExecuteResponse;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.TestRecordingCommunicationSpi;
@@ -885,13 +888,7 @@ public void testCacheIdleVerify() throws Exception {
 
         ignite.cluster().active(true);
 
-        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, 32))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
-
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
+        createCacheAndPreload(ignite, 100);
 
         injectTestSystemOut();
 
@@ -918,15 +915,7 @@ public void testCacheIdleVerifyTwoConflictTypes() throws Exception {
 
         ignite.cluster().active(true);
 
-        int parts = 32;
-
-        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, parts))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
-
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
+        createCacheAndPreload(ignite, 100);
 
         injectTestSystemOut();
 
@@ -938,7 +927,7 @@ public void testCacheIdleVerifyTwoConflictTypes() throws Exception {
 
         corruptDataEntry(cacheCtx, 1, true, false);
 
-        corruptDataEntry(cacheCtx, 1 + parts / 2, false, true);
+        corruptDataEntry(cacheCtx, 1 + cacheCtx.config().getAffinity().partitions() / 2, false, true);
 
         assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify"));
 
@@ -955,12 +944,11 @@ public void testCacheIdleVerifyDump() throws Exception {
 
         ignite.cluster().active(true);
 
-        int parts = 32;
+        int keysCount = 20;//less than parts number for ability to check skipZeros flag.
 
-        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, parts))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
+        createCacheAndPreload(ignite, keysCount);
+
+        int parts = ignite.affinity(DEFAULT_CACHE_NAME).partitions();
 
         ignite.createCache(new CacheConfiguration<>()
             .setAffinity(new RendezvousAffinityFunction(false, parts))
@@ -969,11 +957,6 @@ public void testCacheIdleVerifyDump() throws Exception {
 
         injectTestSystemOut();
 
-        int keysCount = 20;//less than parts number for ability to check skipZeros flag.
-
-        for (int i = 0; i < keysCount; i++)
-            cache.put(i, i);
-
         assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", DEFAULT_CACHE_NAME));
 
         assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--skipZeros", DEFAULT_CACHE_NAME));
@@ -1041,23 +1024,15 @@ public void testCacheIdleVerifyDumpForCorruptedData() throws Exception {
 
         ignite.cluster().active(true);
 
-        int parts = 32;
-
-        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, parts))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
+        createCacheAndPreload(ignite, 100);
 
         injectTestSystemOut();
 
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
-
         GridCacheContext<Object, Object> cacheCtx = ignite.cachex(DEFAULT_CACHE_NAME).context();
 
         corruptDataEntry(cacheCtx, 0, true, false);
 
-        corruptDataEntry(cacheCtx, 0 + parts / 2, false, true);
+        corruptDataEntry(cacheCtx, cacheCtx.config().getAffinity().partitions() / 2, false, true);
 
         assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump"));
 
@@ -1072,6 +1047,130 @@ public void testCacheIdleVerifyDumpForCorruptedData() throws Exception {
             fail("Should be found dump with conflicts");
     }
 
+    /**
+     * Tests that idle verify print partitions info when node failing.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCacheIdleVerifyDumpWhenNodeFailing() throws Exception {
+        Ignite ignite = startGrids(3);
+
+        Ignite unstable = startGrid("unstable");
+
+        ignite.cluster().active(true);
+
+        createCacheAndPreload(ignite, 100);
+
+        for (int i = 0; i < 3; i++) {
+            TestRecordingCommunicationSpi.spi(unstable).blockMessages(GridJobExecuteResponse.class,
+                getTestIgniteInstanceName(i));
+        }
+
+        injectTestSystemOut();
+
+        IgniteInternalFuture fut = GridTestUtils.runAsync(() -> {
+            assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump"));
+        });
+
+        TestRecordingCommunicationSpi.spi(unstable).waitForBlocked();
+
+        UUID unstableNodeId = unstable.cluster().localNode().id();
+
+        unstable.close();
+
+        fut.get();
+
+        checkExceptionMessageOnReport(unstableNodeId);
+    }
+
+    /**
+     * Tests that idle verify print partitions info when several nodes failing at same time.
+     *
+     * @throws Exception If failed.
+     */
+    public void testCacheIdleVerifyDumpWhenSeveralNodesFailing() throws Exception {
+        int nodes = 6;
+
+        Ignite ignite = startGrids(nodes);
+
+        List<Ignite> unstableNodes = new ArrayList<>(nodes / 2);
+
+        for (int i = 0; i < nodes; i++) {
+            if (i % 2 == 1)
+                unstableNodes.add(ignite(i));
+        }
+
+        ignite.cluster().active(true);
+
+        createCacheAndPreload(ignite, 100);
+
+        for (Ignite unstable : unstableNodes) {
+            for (int i = 0; i < nodes; i++) {
+                TestRecordingCommunicationSpi.spi(unstable).blockMessages(GridJobExecuteResponse.class,
+                    getTestIgniteInstanceName(i));
+            }
+        }
+
+        injectTestSystemOut();
+
+        IgniteInternalFuture fut = GridTestUtils.runAsync(() -> {
+            assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump"));
+        });
+
+        List<UUID> unstableNodeIds = new ArrayList<>(nodes / 2);
+
+        for (Ignite unstable : unstableNodes) {
+            TestRecordingCommunicationSpi.spi(unstable).waitForBlocked();
+
+            unstableNodeIds.add(unstable.cluster().localNode().id());
+
+            unstable.close();
+        }
+
+        fut.get();
+
+        for (UUID unstableId : unstableNodeIds)
+            checkExceptionMessageOnReport(unstableId);
+    }
+
+    /**
+     * Creates default cache and preload some data entries.
+     *
+     * @param ignite Ignite.
+     * @param countEntries Count of entries.
+     */
+    private void createCacheAndPreload(Ignite ignite, int countEntries) {
+        ignite.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+            .setAffinity(new RendezvousAffinityFunction(false, 32))
+            .setBackups(1));
+
+        try (IgniteDataStreamer streamer = ignite.dataStreamer(DEFAULT_CACHE_NAME)) {
+            for (int i = 0; i < countEntries; i++)
+                streamer.addData(i, i);
+        }
+    }
+
+    /**
+     * Try to finds node failed exception message on output report.
+     *
+     * @param unstableNodeId Unstable node id.
+     */
+    private void checkExceptionMessageOnReport(UUID unstableNodeId) throws IOException {
+        Matcher fileNameMatcher = dumpFileNameMatcher();
+
+        if (fileNameMatcher.find()) {
+            String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+            assertTrue(dumpWithConflicts.contains("Idle verify failed on nodes:"));
+
+            assertTrue(dumpWithConflicts.contains("Node ID: " + unstableNodeId + "\n" +
+                "Exception message:\n" +
+                "Node has left grid: " + unstableNodeId));
+        }
+        else
+            fail("Should be found dump with conflicts");
+    }
+
     /**
      * @return Build matcher for dump file name.
      */
@@ -1370,13 +1469,7 @@ public void testCacheDistribution() throws Exception {
 
         ignite.cluster().active(true);
 
-        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, 32))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
-
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
+        createCacheAndPreload(ignite, 100);
 
         injectTestSystemOut();
 
@@ -1420,13 +1513,7 @@ public void testCacheResetLostPartitions() throws Exception {
 
         ignite.cluster().active(true);
 
-        IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
-            .setAffinity(new RendezvousAffinityFunction(false, 32))
-            .setBackups(1)
-            .setName(DEFAULT_CACHE_NAME));
-
-        for (int i = 0; i < 100; i++)
-            cache.put(i, i);
+        createCacheAndPreload(ignite, 100);
 
         injectTestSystemOut();
 

From 6b395874cfa8f923e0227da0138798dcbbc9a4bc Mon Sep 17 00:00:00 2001
From: vd-pyatkov <vpyatkov@gridgain.com>
Date: Thu, 13 Dec 2018 14:30:45 +0300
Subject: [PATCH 541/543] IGNITE-10446 control.sh --cache idle_verify fail with
 NPE when node left grid (resolve)

---
 .../test/java/org/apache/ignite/util/GridCommandHandlerTest.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 60c2f699d1aef..6354e4d80618b 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -19,6 +19,7 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.IOException;
 import java.io.PrintStream;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;

From 0641ff8061bcfdbd308b07a06ce8e6264643c9d5 Mon Sep 17 00:00:00 2001
From: Alexey Kuznetsov <akuznetsov@apache.org>
Date: Fri, 14 Dec 2018 10:45:33 +0700
Subject: [PATCH 542/543] GG-14551 Backport of IGNITE-10189 Implemented support
 for multiple protocols and cipher suites for utilities that work via
 o.a.i.internal.client.GridClient.

---
 .../client/ClientSslParametersTest.java       | 159 ++++++++----------
 .../internal/commandline/Arguments.java       |  42 ++---
 .../internal/commandline/CommandHandler.java  |  56 +++---
 .../ignite/client/SslParametersTest.java      | 135 +++++++--------
 .../util/GridCommandHandlerSslTest.java       |  99 +++++++++--
 5 files changed, 275 insertions(+), 216 deletions(-)

diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java
index 228ce82cd2318..74f50ba70a77c 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/client/ClientSslParametersTest.java
@@ -20,14 +20,14 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.Callable;
-
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.ConnectorConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.client.ssl.GridSslBasicContextFactory;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.ssl.SslContextFactory;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
 import org.jetbrains.annotations.NotNull;
 
 /**
@@ -47,16 +47,13 @@ public class ClientSslParametersTest extends GridCommonAbstractTest {
     @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
         IgniteConfiguration cfg = super.getConfiguration(gridName);
 
-        ConnectorConfiguration conCfg = new ConnectorConfiguration();
-        conCfg.setSslEnabled(true);
-        conCfg.setSslClientAuth(true);
-        conCfg.setSslContextFactory(createSslFactory());
-
-        cfg.setConnectorConfiguration(conCfg);
+        cfg.setSslContextFactory(createSslFactory());
 
-        CacheConfiguration ccfg = new CacheConfiguration(TEST_CACHE_NAME);
+        cfg.setConnectorConfiguration(new ConnectorConfiguration()
+            .setSslEnabled(true)
+            .setSslClientAuth(true));
 
-        cfg.setCacheConfiguration(ccfg);
+        cfg.setCacheConfiguration(new CacheConfiguration(TEST_CACHE_NAME));
 
         return cfg;
     }
@@ -69,18 +66,32 @@ protected GridClientConfiguration getClientConfiguration() {
 
         cfg.setServers(Collections.singleton("127.0.0.1:11211"));
 
-        cfg.setSslContextFactory(createSslFactory());
+        cfg.setSslContextFactory(createOldSslFactory());
 
         return cfg;
     }
 
+    /**
+     * @return SSL factory.
+     */
+    @NotNull private SslContextFactory createSslFactory() {
+        SslContextFactory factory = (SslContextFactory)GridTestUtils.sslFactory();
+
+        factory.setCipherSuites(cipherSuites);
+
+        factory.setProtocols(protocols);
+
+        return factory;
+    }
+
     /**
      * @return SSL Factory.
      */
-    @NotNull private GridSslBasicContextFactory createSslFactory() {
+    @NotNull private GridSslBasicContextFactory createOldSslFactory() {
         GridSslBasicContextFactory factory = (GridSslBasicContextFactory)GridTestUtils.sslContextFactory();
 
         factory.setCipherSuites(cipherSuites);
+
         factory.setProtocols(protocols);
 
         return factory;
@@ -91,6 +102,7 @@ protected GridClientConfiguration getClientConfiguration() {
         stopAllGrids();
 
         protocols = null;
+
         cipherSuites = null;
     }
 
@@ -107,12 +119,10 @@ public void testSameCipherSuite() throws Exception {
         startGrid();
 
         checkSuccessfulClientStart(
-            new String[][] {
-                new String[] {
-                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
-                    "TLS_RSA_WITH_AES_128_GCM_SHA256",
-                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
-                }
+            new String[] {
+                "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                "TLS_RSA_WITH_AES_128_GCM_SHA256",
+                "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
             },
             null
         );
@@ -130,11 +140,9 @@ public void testOneCommonCipherSuite() throws Exception {
         startGrid();
         
         checkSuccessfulClientStart(
-            new String[][] {
-                new String[] {
-                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
-                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
-                }
+            new String[] {
+                "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
             },
             null
         );
@@ -151,11 +159,9 @@ public void testNoCommonCipherSuite() throws Exception {
         startGrid();
         
         checkClientStartFailure(
-            new String[][] {
-                new String[] {
-                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
-                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
-                }
+            new String[] {
+                "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
             },
             null
         );
@@ -165,7 +171,7 @@ public void testNoCommonCipherSuite() throws Exception {
      * @throws Exception If failed.
      */
     public void testNonExistentCipherSuite() throws Exception {
-        fail("IGNITE-10245");
+        fail("https://issues.apache.org/jira/browse/IGNITE-10245");
 
         cipherSuites = new String[] {
             "TLS_RSA_WITH_AES_128_GCM_SHA256"
@@ -174,14 +180,11 @@ public void testNonExistentCipherSuite() throws Exception {
         startGrid();
         
         checkClientStartFailure(
-            new String[][] {
-                new String[] {
-                    "TLC_FAKE_CIPHER",
-                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
-                }
+            new String[] {
+                "TLC_FAKE_CIPHER",
+                "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
             },
             null,
-            GridClientException.class,
             "Unsupported ciphersuite"
         );
     }
@@ -199,11 +202,9 @@ public void testNoCommonProtocols() throws Exception {
 
         checkClientStartFailure(
             null,
-            new String[][] {
-                new String[] {
-                    "TLSv1",
-                    "TLSv1.2",
-                }
+            new String[] {
+                "TLSv1",
+                "TLSv1.2"
             }
         );
     }
@@ -212,7 +213,7 @@ public void testNoCommonProtocols() throws Exception {
      * @throws Exception If failed.
      */
     public void testNonExistentProtocol() throws Exception {
-        fail("IGNITE-10245");
+        fail("https://issues.apache.org/jira/browse/IGNITE-10245");
 
         protocols = new String[] {
             "SSLv3"
@@ -222,13 +223,10 @@ public void testNonExistentProtocol() throws Exception {
 
         checkClientStartFailure(
             null,
-            new String[][] {
-                new String[] {
-                    "SSLv3",
-                    "SSLvDoesNotExist"
-                }
+            new String[] {
+                "SSLv3",
+                "SSLvDoesNotExist"
             },
-            GridClientException.class,
             "SSLvDoesNotExist"
         );
     }
@@ -239,17 +237,16 @@ public void testNonExistentProtocol() throws Exception {
     public void testSameProtocols() throws Exception {
         protocols = new String[] {
             "TLSv1.1",
-            "TLSv1.2",
+            "TLSv1.2"
         };
 
         startGrid();
 
-        checkSuccessfulClientStart(null,
-            new String[][] {
-                new String[] {
-                    "TLSv1.1",
-                    "TLSv1.2",
-                }
+        checkSuccessfulClientStart(
+            null,
+            new String[] {
+                "TLSv1.1",
+                "TLSv1.2"
             }
         );
     }
@@ -266,12 +263,11 @@ public void testOneCommonProtocol() throws Exception {
 
         startGrid();
 
-        checkSuccessfulClientStart(null,
-            new String[][] {
-                new String[] {
-                    "TLSv1.1",
-                    "SSLv3"
-                }
+        checkSuccessfulClientStart(
+            null,
+            new String[] {
+                "TLSv1.1",
+                "SSLv3"
             }
         );
     }
@@ -281,22 +277,14 @@ public void testOneCommonProtocol() throws Exception {
      * @param protocols list of protocols
      * @throws Exception If failed.
      */
-    private void checkSuccessfulClientStart(String[][] cipherSuites, String[][] protocols) throws Exception {
-        int n = Math.max(
-            cipherSuites != null ? cipherSuites.length : 0,
-            protocols != null ? protocols.length : 0);
-
-        for (int i = 0; i < n; i++) {
-            this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
-            this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
-
-            GridClient client = GridClientFactory.start(getClientConfiguration());
+    private void checkSuccessfulClientStart(String[] cipherSuites, String[] protocols) throws Exception {
+        this.cipherSuites = F.isEmpty(cipherSuites) ? null : cipherSuites;
+        this.protocols = F.isEmpty(protocols) ? null : protocols;
 
+        try (GridClient client = GridClientFactory.start(getClientConfiguration())) {
             List<GridClientNode> top = client.compute().refreshTopology(false, false);
 
             assertEquals(1, top.size());
-
-            client.close();
         }
     }
 
@@ -304,26 +292,22 @@ private void checkSuccessfulClientStart(String[][] cipherSuites, String[][] prot
      * @param cipherSuites list of cipher suites
      * @param protocols list of protocols
      */
-    private void checkClientStartFailure(String[][] cipherSuites, String[][] protocols) {
-        checkClientStartFailure(cipherSuites, protocols, GridClientException.class, "Latest topology update failed.");
+    private void checkClientStartFailure(String[] cipherSuites, String[] protocols) {
+        checkClientStartFailure(cipherSuites, protocols, "Latest topology update failed.");
     }
 
     /**
      * @param cipherSuites list of cipher suites
      * @param protocols list of protocols
-     * @param ex expected exception class
      * @param msg exception message
      */
-    private void checkClientStartFailure(String[][] cipherSuites, String[][] protocols, Class<? extends Throwable> ex, String msg) {
-        int n = Math.max(
-            cipherSuites != null ? cipherSuites.length : 0,
-            protocols != null ? protocols.length : 0);
+    private void checkClientStartFailure(String[] cipherSuites, String[] protocols, String msg) {
+        this.cipherSuites = F.isEmpty(cipherSuites) ? null : cipherSuites;
+        this.protocols = F.isEmpty(protocols) ? null : protocols;
 
-        for (int i = 0; i < n; i++) {
-            this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
-            this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
-
-            GridTestUtils.assertThrows(null, new Callable<Object>() {
+        GridTestUtils.assertThrows(
+            null,
+            new Callable<Object>() {
                 @Override public Object call() throws Exception {
                     GridClient client = GridClientFactory.start(getClientConfiguration());
 
@@ -331,8 +315,9 @@ private void checkClientStartFailure(String[][] cipherSuites, String[][] protoco
 
                     return null;
                 }
-            }, ex, msg);
-        }
+            },
+            GridClientException.class,
+            msg
+        );
     }
-
 }
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
index 7fc4ac9ef5c37..abd27bdb1b315 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/Arguments.java
@@ -71,21 +71,21 @@ public class Arguments {
      */
     private String walArgs;
 
-    /** Ping timeout for grid client. See {@link GridClientConfiguration#pingTimeout}. */
+    /** Ping timeout for grid client. See {@link GridClientConfiguration#getPingTimeout()}. */
     private long pingTimeout;
 
-    /** Ping interval for grid client. See {@link GridClientConfiguration#pingInterval}. */
+    /** Ping interval for grid client. See {@link GridClientConfiguration#getPingInterval()}. */
     private long pingInterval;
 
     /** SSL Protocol. */
     private String sslProtocol;
 
+    /** SSL Cipher suites. */
+    private String sslCipherSuites;
+
     /** SSL Key Algorithm. */
     private String sslKeyAlgorithm;
 
-    /** */
-    private String sslCipherSuites;
-
     /** Keystore. */
     private String sslKeyStorePath;
 
@@ -116,12 +116,12 @@ public class Arguments {
      * @param cacheArgs --cache subcommand arguments.
      * @param walAct WAL action.
      * @param walArgs WAL args.
-     * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#pingTimeout}.
-     * @param pingInterval Ping interval. See {@link GridClientConfiguration#pingInterval}.
+     * @param pingTimeout Ping timeout. See {@link GridClientConfiguration#getPingTimeout()}.
+     * @param pingInterval Ping interval. See {@link GridClientConfiguration#getPingInterval()}.
      * @param autoConfirmation Auto confirmation flag.
      * @param sslProtocol SSL Protocol.
-     * @param sslKeyAlgorithm SSL Key Algorithm.
      * @param sslCipherSuites SSL cipher suites.
+     * @param sslKeyAlgorithm SSL Key Algorithm.
      * @param sslKeyStorePath Keystore.
      * @param sslKeyStorePassword Keystore Password.
      * @param sslKeyStoreType Keystore Type.
@@ -131,9 +131,11 @@ public class Arguments {
      */
     public Arguments(Command cmd, String host, String port, String user, String pwd, String baselineAct,
         String baselineArgs, VisorTxTaskArg txArg, CacheArguments cacheArgs, String walAct, String walArgs,
-        Long pingTimeout, Long pingInterval, boolean autoConfirmation, String sslProtocol, String sslKeyAlgorithm,
-        String sslCipherSuites, String sslKeyStorePath, char[] sslKeyStorePassword, String sslKeyStoreType,
-        String sslTrustStorePath, char[] sslTrustStorePassword, String sslTrustStoreType) {
+        Long pingTimeout, Long pingInterval, boolean autoConfirmation,
+        String sslProtocol, String sslCipherSuites, String sslKeyAlgorithm,
+        String sslKeyStorePath, char[] sslKeyStorePassword, String sslKeyStoreType,
+        String sslTrustStorePath, char[] sslTrustStorePassword, String sslTrustStoreType
+    ) {
         this.cmd = cmd;
         this.host = host;
         this.port = port;
@@ -155,9 +157,9 @@ public Arguments(Command cmd, String host, String port, String user, String pwd,
         this.autoConfirmation = autoConfirmation;
 
         this.sslProtocol = sslProtocol;
-        this.sslKeyAlgorithm = sslKeyAlgorithm;
         this.sslCipherSuites = sslCipherSuites;
 
+        this.sslKeyAlgorithm = sslKeyAlgorithm;
         this.sslKeyStorePath = sslKeyStorePath;
         this.sslKeyStoreType = sslKeyStoreType;
         this.sslKeyStorePassword = sslKeyStorePassword;
@@ -259,7 +261,7 @@ public String walArguments() {
     }
 
     /**
-     * See {@link GridClientConfiguration#pingTimeout}.
+     * See {@link GridClientConfiguration#getPingInterval()}.
      *
      * @return Ping timeout.
      */
@@ -268,7 +270,7 @@ public long pingTimeout() {
     }
 
     /**
-     * See {@link GridClientConfiguration#pingInterval}.
+     * See {@link GridClientConfiguration#getPingInterval()}.
      *
      * @return Ping interval.
      */
@@ -290,6 +292,13 @@ public String sslProtocol() {
         return sslProtocol;
     }
 
+    /**
+     * @return SSL cipher suites.
+     */
+    public String getSslCipherSuites() {
+        return sslCipherSuites;
+    }
+
     /**
      * @return SSL Key Algorithm
      */
@@ -297,11 +306,6 @@ public String sslKeyAlgorithm() {
         return sslKeyAlgorithm;
     }
 
-    /** */
-    public String getSslCipherSuites() {
-        return sslCipherSuites;
-    }
-
     /**
      * @return Keystore
      */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 4cff4986223c0..848110148b80e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -204,10 +204,10 @@ public class CommandHandler {
     private static final String CMD_SSL_PROTOCOL = "--ssl-protocol";
 
     /** */
-    private static final String CMD_SSL_KEY_ALGORITHM = "--ssl-key-algorithm";
+    private static final String CMD_SSL_CIPHER_SUITES = "--ssl-cipher-suites";
 
     /** */
-    protected static final String CMD_SSL_CIPHER_SUITES = "--ssl-cipher-suites";
+    private static final String CMD_SSL_KEY_ALGORITHM = "--ssl-key-algorithm";
 
     /** */
     private static final String CMD_KEYSTORE = "--keystore";
@@ -245,16 +245,16 @@ public class CommandHandler {
         AUX_COMMANDS.add(CMD_PING_TIMEOUT);
 
         AUX_COMMANDS.add(CMD_SSL_PROTOCOL);
-        AUX_COMMANDS.add(CMD_SSL_KEY_ALGORITHM);
         AUX_COMMANDS.add(CMD_SSL_CIPHER_SUITES);
 
+        AUX_COMMANDS.add(CMD_SSL_KEY_ALGORITHM);
+        AUX_COMMANDS.add(CMD_KEYSTORE_TYPE);
         AUX_COMMANDS.add(CMD_KEYSTORE);
         AUX_COMMANDS.add(CMD_KEYSTORE_PASSWORD);
-        AUX_COMMANDS.add(CMD_KEYSTORE_TYPE);
 
+        AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE);
         AUX_COMMANDS.add(CMD_TRUSTSTORE);
         AUX_COMMANDS.add(CMD_TRUSTSTORE_PASSWORD);
-        AUX_COMMANDS.add(CMD_TRUSTSTORE_TYPE);
     }
 
     /** Broadcast uuid. */
@@ -1853,20 +1853,20 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         String sslProtocol = SslContextFactory.DFLT_SSL_PROTOCOL;
 
-        String sslKeyAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM;
-
         String sslCipherSuites = "";
 
-        String sslKeyStorePath = null;
+        String sslKeyAlgorithm = SslContextFactory.DFLT_KEY_ALGORITHM;
 
         String sslKeyStoreType = SslContextFactory.DFLT_STORE_TYPE;
 
-        char sslKeyStorePassword[] = null;
+        String sslKeyStorePath = null;
 
-        String sslTrustStorePath = null;
+        char sslKeyStorePassword[] = null;
 
         String sslTrustStoreType = SslContextFactory.DFLT_STORE_TYPE;
 
+        String sslTrustStorePath = null;
+
         char sslTrustStorePassword[] = null;
 
         while (hasNextArg()) {
@@ -1986,43 +1986,43 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
                         break;
 
-                    case CMD_SSL_KEY_ALGORITHM:
-                        sslKeyAlgorithm = nextArg("Expected SSL key algorithm");
+                    case CMD_SSL_CIPHER_SUITES:
+                        sslCipherSuites = nextArg("Expected SSL cipher suites");
 
                         break;
 
-                    case CMD_SSL_CIPHER_SUITES:
-                        sslCipherSuites = nextArg("Expected SSL cipher suites");
+                    case CMD_SSL_KEY_ALGORITHM:
+                        sslKeyAlgorithm = nextArg("Expected SSL key algorithm");
 
                         break;
 
                     case CMD_KEYSTORE:
-                        sslKeyStorePath = nextArg("Expected keystore path");
+                        sslKeyStorePath = nextArg("Expected SSL key store path");
 
                         break;
 
                     case CMD_KEYSTORE_PASSWORD:
-                        sslKeyStorePassword = nextArg("Expected keystore password").toCharArray();
+                        sslKeyStorePassword = nextArg("Expected SSL key store password").toCharArray();
 
                         break;
 
                     case CMD_KEYSTORE_TYPE:
-                        sslKeyStoreType = nextArg("Expected keystore type");
+                        sslKeyStoreType = nextArg("Expected SSL key store type");
 
                         break;
 
                     case CMD_TRUSTSTORE:
-                        sslTrustStorePath = nextArg("Expected truststore path");
+                        sslTrustStorePath = nextArg("Expected SSL trust store path");
 
                         break;
 
                     case CMD_TRUSTSTORE_PASSWORD:
-                        sslTrustStorePassword = nextArg("Expected truststore password").toCharArray();
+                        sslTrustStorePassword = nextArg("Expected SSL trust store password").toCharArray();
 
                         break;
 
                     case CMD_TRUSTSTORE_TYPE:
-                        sslTrustStoreType = nextArg("Expected truststore type");
+                        sslTrustStoreType = nextArg("Expected SSL trust store type");
 
                         break;
 
@@ -2047,9 +2047,13 @@ Arguments parseAndValidate(List<String> rawArgs) {
 
         Command cmd = commands.get(0);
 
-        return new Arguments(cmd, host, port, user, pwd, baselineAct, baselineArgs, txArgs, cacheArgs, walAct, walArgs,
+        return new Arguments(cmd, host, port, user, pwd,
+            baselineAct, baselineArgs,
+            txArgs, cacheArgs,
+            walAct, walArgs,
             pingTimeout, pingInterval, autoConfirmation,
-            sslProtocol, sslKeyAlgorithm, sslCipherSuites, sslKeyStorePath, sslKeyStorePassword, sslKeyStoreType,
+            sslProtocol, sslCipherSuites,
+            sslKeyAlgorithm, sslKeyStorePath, sslKeyStorePassword, sslKeyStoreType,
             sslTrustStorePath, sslTrustStorePassword, sslTrustStoreType);
     }
 
@@ -2696,10 +2700,16 @@ public int execute(List<String> rawArgs) {
                 if (!F.isEmpty(args.sslKeyStorePath())) {
                     GridSslBasicContextFactory factory = new GridSslBasicContextFactory();
 
-                    factory.setProtocol(args.sslProtocol());
+                    List<String> sslProtocols = split(args.sslProtocol(), ",");
+
+                    String sslProtocol = F.isEmpty(sslProtocols) ? SslContextFactory.DFLT_SSL_PROTOCOL : sslProtocols.get(0);
 
+                    factory.setProtocol(sslProtocol);
                     factory.setKeyAlgorithm(args.sslKeyAlgorithm());
 
+                    if (sslProtocols.size() > 1)
+                        factory.setProtocols(sslProtocols);
+
                     factory.setCipherSuites(split(args.getSslCipherSuites(), ","));
 
                     factory.setKeyStoreFilePath(args.sslKeyStorePath());
diff --git a/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java b/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java
index 20eb94cfd97d2..d61f6ac47b9df 100644
--- a/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/client/SslParametersTest.java
@@ -23,6 +23,7 @@
 import org.apache.ignite.configuration.ClientConfiguration;
 import org.apache.ignite.configuration.ClientConnectorConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.ssl.SslContextFactory;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
@@ -77,8 +78,7 @@ protected ClientConfiguration getClientConfiguration() {
      * @return SSL factory.
      */
     @NotNull private SslContextFactory createSslFactory() {
-        SslContextFactory factory = (SslContextFactory)GridTestUtils.sslTrustedFactory(
-            "node01", "trustone");
+        SslContextFactory factory = (SslContextFactory)GridTestUtils.sslTrustedFactory("node01", "trustone");
 
         factory.setCipherSuites(cipherSuites);
         factory.setProtocols(protocols);
@@ -107,12 +107,10 @@ public void testSameCipherSuite() throws Exception {
         startGrid();
 
         checkSuccessfulClientStart(
-            new String[][] {
-                new String[] {
-                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
-                    "TLS_RSA_WITH_AES_128_GCM_SHA256",
-                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
-                }
+            new String[] {
+                "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                "TLS_RSA_WITH_AES_128_GCM_SHA256",
+                "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
             },
             null
         );
@@ -128,13 +126,11 @@ public void testOneCommonCipherSuite() throws Exception {
         };
 
         startGrid();
-        
+
         checkSuccessfulClientStart(
-            new String[][] {
-                new String[] {
-                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
-                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
-                }
+            new String[] {
+                "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
             },
             null
         );
@@ -149,13 +145,11 @@ public void testNoCommonCipherSuite() throws Exception {
         };
 
         startGrid();
-        
+
         checkClientStartFailure(
-            new String[][] {
-                new String[] {
-                    "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
-                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
-                }
+            new String[] {
+                "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
+                "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
             },
             null
         );
@@ -170,13 +164,11 @@ public void testNonExistentCipherSuite() throws Exception {
         };
 
         startGrid();
-        
+
         checkClientStartFailure(
-            new String[][] {
-                new String[] {
-                    "TLC_FAKE_CIPHER",
-                    "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
-                }
+            new String[] {
+                "TLC_FAKE_CIPHER",
+                "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256"
             },
             null,
             IllegalArgumentException.class,
@@ -197,11 +189,9 @@ public void testNoCommonProtocols() throws Exception {
 
         checkClientStartFailure(
             null,
-            new String[][] {
-                new String[] {
-                    "TLSv1",
-                    "TLSv1.2",
-                }
+            new String[] {
+                "TLSv1",
+                "TLSv1.2"
             }
         );
     }
@@ -218,11 +208,9 @@ public void testNonExistentProtocol() throws Exception {
 
         checkClientStartFailure(
             null,
-            new String[][] {
-                new String[] {
-                    "SSLv3",
-                    "SSLvDoesNotExist"
-                }
+            new String[] {
+                "SSLv3",
+                "SSLvDoesNotExist"
             },
             IllegalArgumentException.class,
             "SSLvDoesNotExist"
@@ -235,17 +223,15 @@ public void testNonExistentProtocol() throws Exception {
     public void testSameProtocols() throws Exception {
         protocols = new String[] {
             "TLSv1.1",
-            "TLSv1.2",
+            "TLSv1.2"
         };
 
         startGrid();
 
         checkSuccessfulClientStart(null,
-            new String[][] {
-                new String[] {
-                    "TLSv1.1",
-                    "TLSv1.2",
-                }
+            new String[] {
+                "TLSv1.1",
+                "TLSv1.2"
             }
         );
     }
@@ -263,11 +249,9 @@ public void testOneCommonProtocol() throws Exception {
         startGrid();
 
         checkSuccessfulClientStart(null,
-            new String[][] {
-                new String[] {
-                    "TLSv1.1",
-                    "SSLv3"
-                }
+            new String[] {
+                "TLSv1.1",
+                "SSLv3"
             }
         );
     }
@@ -277,30 +261,26 @@ public void testOneCommonProtocol() throws Exception {
      * @param protocols list of protocols
      * @throws Exception If failed.
      */
-    private void checkSuccessfulClientStart(String[][] cipherSuites, String[][] protocols) throws Exception {
-        int n = Math.max(
-            cipherSuites != null ? cipherSuites.length : 0,
-            protocols != null ? protocols.length : 0);
-
-        for (int i = 0; i < n; i++) {
-            this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
-            this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
-
-            IgniteClient client = Ignition.startClient(getClientConfiguration());
+    private void checkSuccessfulClientStart(String[] cipherSuites, String[] protocols) throws Exception {
+        this.cipherSuites = F.isEmpty(cipherSuites) ? null : cipherSuites;
+        this.protocols = F.isEmpty(protocols) ? null : protocols;
 
+        try (IgniteClient client = Ignition.startClient(getClientConfiguration())) {
             client.getOrCreateCache(TEST_CACHE_NAME);
-
-            client.close();
         }
     }
 
     /**
      * @param cipherSuites list of cipher suites
      * @param protocols list of protocols
-     * @throws Exception If failed.
      */
-    private void checkClientStartFailure(String[][] cipherSuites, String[][] protocols) throws Exception {
-        checkClientStartFailure(cipherSuites, protocols, ClientConnectionException.class, "Ignite cluster is unavailable");
+    private void checkClientStartFailure(String[] cipherSuites, String[] protocols) {
+        checkClientStartFailure(
+            cipherSuites,
+            protocols,
+            ClientConnectionException.class,
+            "Ignite cluster is unavailable"
+        );
     }
 
     /**
@@ -308,24 +288,27 @@ private void checkClientStartFailure(String[][] cipherSuites, String[][] protoco
      * @param protocols list of protocols
      * @param ex expected exception class
      * @param msg exception message
-     * @throws Exception If failed.
      */
-    private void checkClientStartFailure(String[][] cipherSuites, String[][] protocols, Class<? extends Throwable> ex, String msg) throws Exception {
-        int n = Math.max(
-            cipherSuites != null ? cipherSuites.length : 0,
-            protocols != null ? protocols.length : 0);
-
-        for (int i = 0; i < n; i++) {
-            this.cipherSuites = cipherSuites != null && i < cipherSuites.length ? cipherSuites[i] : null;
-            this.protocols = protocols != null && i < protocols.length ? protocols[i] : null;
-
-            GridTestUtils.assertThrows(null, new Callable<Object>() {
-                @Override public Object call() throws Exception {
+    private void checkClientStartFailure(
+        String[] cipherSuites,
+        String[] protocols,
+        Class<? extends Throwable> ex,
+        String msg
+    ) {
+        this.cipherSuites = F.isEmpty(cipherSuites) ? null : cipherSuites;
+        this.protocols = F.isEmpty(protocols) ? null : protocols;
+
+        GridTestUtils.assertThrows(
+            null,
+            new Callable<Object>() {
+                @Override public Object call() {
                     Ignition.startClient(getClientConfiguration());
 
                     return null;
                 }
-            }, ex, msg);
-        }
+            },
+            ex,
+            msg
+        );
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
index 59155e5b3713a..9e84938bf51bd 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerSslTest.java
@@ -17,28 +17,35 @@
 
 package org.apache.ignite.util;
 
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.configuration.ConnectorConfiguration;
 import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.commandline.CommandHandler;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.ssl.SslContextFactory;
 import org.apache.ignite.testframework.GridTestUtils;
 import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
 
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_CONNECTION_FAILED;
 import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
 
 /**
- * Command line handler test.
+ * Command line handler test with SSL.
  */
 public class GridCommandHandlerSslTest extends GridCommonAbstractTest {
+    /** */
+    private volatile String[] cipherSuites;
+
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         cleanPersistenceDir();
 
         stopAllGrids();
-
     }
 
     /** {@inheritDoc} */
@@ -46,7 +53,17 @@ public class GridCommandHandlerSslTest extends GridCommonAbstractTest {
         stopAllGrids();
 
         cleanPersistenceDir();
+    }
 
+    /**
+     * @return SSL factory.
+     */
+    @NotNull private SslContextFactory createSslFactory() {
+        SslContextFactory factory = (SslContextFactory)GridTestUtils.sslFactory();
+
+        factory.setCipherSuites(cipherSuites);
+
+        return factory;
     }
 
     /** {@inheritDoc} */
@@ -59,30 +76,90 @@ public class GridCommandHandlerSslTest extends GridCommonAbstractTest {
 
         cfg.setConnectorConfiguration(new ConnectorConfiguration());
         cfg.getConnectorConfiguration().setSslEnabled(true);
-        cfg.setSslContextFactory(GridTestUtils.sslFactory());
+        cfg.setSslContextFactory(createSslFactory());
 
         return cfg;
     }
 
     /**
-     * Test activation works via control.sh
-     *
+     * @param nodeCipherSuite Ciphers suites to set on node.
+     * @param utilityCipherSuite Ciphers suites to set on utility.
+     * @param expRes Expected result.
      * @throws Exception If failed.
      */
-    public void testActivate() throws Exception {
+    private void activate(String nodeCipherSuite, String utilityCipherSuite, int expRes) throws Exception {
+        cipherSuites = F.isEmpty(nodeCipherSuite) ? null : nodeCipherSuite.split(",");
+
         Ignite ignite = startGrids(1);
 
         assertFalse(ignite.cluster().active());
 
         final CommandHandler cmd = new CommandHandler();
-        assertEquals(EXIT_CODE_OK, cmd.execute(Arrays.asList(
-            "--activate",
-            "--keystore", GridTestUtils.keyStorePath("node01"),
-            "--keystore-password", GridTestUtils.keyStorePassword())));
 
-        assertTrue(ignite.cluster().active());
+        List<String> params = new ArrayList<>();
+        params.add("--activate");
+        params.add("--keystore");
+        params.add(GridTestUtils.keyStorePath("node01"));
+        params.add("--keystore-password");
+        params.add(GridTestUtils.keyStorePassword());
+
+        if (!F.isEmpty(utilityCipherSuite)) {
+            params.add("--ssl-cipher-suites");
+            params.add(utilityCipherSuite);
+        }
+
+        assertEquals(expRes, cmd.execute(params));
+
+        if (expRes == EXIT_CODE_OK)
+            assertTrue(ignite.cluster().active());
+        else
+            assertFalse(ignite.cluster().active());
 
         assertEquals(EXIT_CODE_CONNECTION_FAILED, cmd.execute(Arrays.asList("--deactivate", "--yes")));
     }
 
+    /**
+     * @throws Exception If test failed.
+     */
+    public void testDefaultCipherSuite() throws Exception {
+        cipherSuites = null;
+
+        activate(null, null, EXIT_CODE_OK);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testSameCipherSuite() throws Exception {
+        String ciphers = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," +
+            "TLS_RSA_WITH_AES_128_GCM_SHA256," +
+            "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256";
+
+        activate(ciphers, ciphers, EXIT_CODE_OK);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testOneCommonCipherSuite() throws Exception {
+        String nodeCipherSuites = "TLS_RSA_WITH_AES_128_GCM_SHA256," +
+            "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256";
+
+        String utilityCipherSuites = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," +
+            "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256";
+
+        activate(nodeCipherSuites, utilityCipherSuites, EXIT_CODE_OK);
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testNoCommonCipherSuite() throws Exception {
+        String nodeCipherSuites = "TLS_RSA_WITH_AES_128_GCM_SHA256";
+
+        String utilityCipherSuites = "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256," +
+            "TLS_DHE_RSA_WITH_AES_128_GCM_SHA256";
+
+        activate(nodeCipherSuites, utilityCipherSuites, EXIT_CODE_CONNECTION_FAILED);
+    }
 }

From 0701198c451bdb59463291652e264deb9ed1ce7a Mon Sep 17 00:00:00 2001
From: Maxim Muzafarov <maxmuzaf@gmail.com>
Date: Wed, 1 Aug 2018 18:39:54 +0300
Subject: [PATCH 543/543] IGNITE-10374 partitions are not owned after
 rebalancing

---
 .../GridCachePartitionExchangeManager.java    |  67 +++++---
 .../processors/cache/GridCachePreloader.java  |  21 ++-
 .../cache/GridCachePreloaderAdapter.java      |   6 +
 .../preloader/GridDhtPartitionDemander.java   |  48 +++---
 .../preloader/GridDhtPartitionSupplier.java   |  15 +-
 .../dht/preloader/GridDhtPreloader.java       |  60 ++++++-
 .../GridDhtPreloaderAssignments.java          |   5 +-
 .../ClusterBaselineNodesMetricsSelfTest.java  |   1 -
 .../cache/CacheValidatorMetricsTest.java      |   4 +-
 .../dht/GridCacheDhtPreloadSelfTest.java      |  68 +-------
 .../GridCacheRebalancingAsyncSelfTest.java    |   7 +-
 .../GridCacheRebalancingCancelTest.java       | 106 +++++++++++++
 ...CacheRebalancingPartitionCountersTest.java |   3 +-
 .../GridCacheRebalancingSyncSelfTest.java     | 149 +++++++-----------
 ...entAffinityAssignmentWithBaselineTest.java |   4 +-
 ...owHistoricalRebalanceSmallHistoryTest.java |   5 +-
 ...lushMultiNodeFailoverAbstractSelfTest.java |   6 +-
 .../GridMarshallerMappingConsistencyTest.java |   3 +-
 .../testsuites/IgniteCacheTestSuite3.java     |   2 +
 19 files changed, 340 insertions(+), 240 deletions(-)
 create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java

diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
index 17715be3399c5..a0f4f915a33cf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java
@@ -86,7 +86,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteDhtPartitionsToReloadMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.RebalanceReassignExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.StopCachesOnClientReconnectExchangeTask;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.latch.ExchangeLatchManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridClientPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteCacheSnapshotManager;
@@ -189,6 +188,14 @@ public class GridCachePartitionExchangeManager<K, V> extends GridCacheSharedMana
     private final ConcurrentNavigableMap<AffinityTopologyVersion, AffinityTopologyVersion> lastAffTopVers =
         new ConcurrentSkipListMap<>();
 
+    /**
+     * Latest started rebalance topology version but possibly not finished yet. Value {@code NONE}
+     * means that previous rebalance is undefined and the new one should be initiated.
+     *
+     * Should not be used to determine latest rebalanced topology.
+     */
+    private volatile AffinityTopologyVersion rebTopVer = AffinityTopologyVersion.NONE;
+
     /** */
     private GridFutureAdapter<?> reconnectExchangeFut;
 
@@ -871,6 +878,13 @@ public AffinityTopologyVersion readyAffinityVersion() {
         return exchFuts.readyTopVer();
     }
 
+    /**
+     * @return Latest rebalance topology version or {@code NONE} if there is no info.
+     */
+    public AffinityTopologyVersion rebalanceTopologyVersion() {
+        return rebTopVer;
+    }
+
     /**
      * @return Last initialized topology future.
      */
@@ -2847,6 +2861,9 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (grp.isLocal())
                                     continue;
 
+                                if (grp.preloader().rebalanceRequired(rebTopVer, exchFut))
+                                    rebTopVer = AffinityTopologyVersion.NONE;
+
                                 changed |= grp.topology().afterExchange(exchFut);
                             }
 
@@ -2854,7 +2871,11 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 refreshPartitions();
                         }
 
-                        if (!cctx.kernalContext().clientNode()) {
+                        // Schedule rebalance if force rebalance or force reassign occurs.
+                        if (exchFut == null)
+                            rebTopVer = AffinityTopologyVersion.NONE;
+
+                        if (!cctx.kernalContext().clientNode() && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
                             assignsMap = new HashMap<>();
 
                             IgniteCacheSnapshotManager snp = cctx.snapshot();
@@ -2871,6 +2892,9 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                     assigns = grp.preloader().generateAssignments(exchId, exchFut);
 
                                 assignsMap.put(grp.groupId(), assigns);
+
+                                if (resVer == null)
+                                    resVer = grp.topology().readyTopologyVersion();
                             }
                         }
                     }
@@ -2879,7 +2903,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         busy = false;
                     }
 
-                    if (assignsMap != null) {
+                    if (assignsMap != null && rebTopVer.equals(AffinityTopologyVersion.NONE)) {
                         int size = assignsMap.size();
 
                         NavigableMap<Integer, List<Integer>> orderMap = new TreeMap<>();
@@ -2917,11 +2941,6 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                                 if (assigns != null)
                                     assignsCancelled |= assigns.cancelled();
 
-                                // Cancels previous rebalance future (in case it's not done yet).
-                                // Sends previous rebalance stopped event (if necessary).
-                                // Creates new rebalance future.
-                                // Sends current rebalance started event (if necessary).
-                                // Finishes cache sync future (on empty assignments).
                                 Runnable cur = grp.preloader().addAssignments(assigns,
                                     forcePreload,
                                     cnt,
@@ -2939,7 +2958,7 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         if (forcedRebFut != null)
                             forcedRebFut.markInitialized();
 
-                        if (assignsCancelled) { // Pending exchange.
+                        if (assignsCancelled || hasPendingExchange()) {
                             U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
                                 "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
@@ -2947,25 +2966,31 @@ else if (task instanceof ForceRebalanceExchangeTask) {
                         else if (r != null) {
                             Collections.reverse(rebList);
 
-                            U.log(log, "Rebalancing scheduled [order=" + rebList + "]");
+                            U.log(log, "Rebalancing scheduled [order=" + rebList +
+                                ", top=" + resVer + ", force=" + (exchFut == null) +
+                                ", evt=" + exchId.discoveryEventName() +
+                                ", node=" + exchId.nodeId() + ']');
 
-                            if (!hasPendingExchange()) {
-                                U.log(log, "Rebalancing started " +
-                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
-                                    ", node=" + exchId.nodeId() + ']');
+                            rebTopVer = resVer;
 
-                                r.run(); // Starts rebalancing routine.
-                            }
-                            else
-                                U.log(log, "Skipping rebalancing (obsolete exchange ID) " +
-                                    "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
-                                    ", node=" + exchId.nodeId() + ']');
+                            // Start rebalancing cache groups chain. Each group will be rebalanced
+                            // sequentially one by one e.g.:
+                            // ignite-sys-cache -> cacheGroupR1 -> cacheGroupP2 -> cacheGroupR3
+                            r.run();
                         }
                         else
                             U.log(log, "Skipping rebalancing (nothing scheduled) " +
-                                "[top=" + resVer + ", evt=" + exchId.discoveryEventName() +
+                                "[top=" + resVer + ", force=" + (exchFut == null) +
+                                ", evt=" + exchId.discoveryEventName() +
                                 ", node=" + exchId.nodeId() + ']');
                     }
+                    else
+                        U.log(log, "Skipping rebalancing (no affinity changes) " +
+                            "[top=" + resVer +
+                            ", rebTopVer=" + rebTopVer +
+                            ", evt=" + exchId.discoveryEventName() +
+                            ", evtNode=" + exchId.nodeId() +
+                            ", client=" + cctx.kernalContext().clientNode() + ']');
                 }
                 catch (IgniteInterruptedCheckedException e) {
                     throw e;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
index 153eb7cf5ea74..6ac26c958fc29 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloader.java
@@ -24,6 +24,7 @@
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.ForceRebalanceExchangeTask;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemandMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
@@ -62,10 +63,17 @@ public interface GridCachePreloader {
      */
     public void onInitialExchangeComplete(@Nullable Throwable err);
 
+    /**
+     * @param rebTopVer Previous rebalance topology version or {@code NONE} if there is no info.
+     * @param exchFut Completed exchange future.
+     * @return {@code True} if rebalance should be started (previous will be interrupted).
+     */
+    public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer, GridDhtPartitionsExchangeFuture exchFut);
+
     /**
      * @param exchId Exchange ID.
-     * @param exchFut Exchange future.
-     * @return Assignments or {@code null} if detected that there are pending exchanges.
+     * @param exchFut Completed exchange future. Can be {@code null} if forced or reassigned generation occurs.
+     * @return Partition assignments which will be requested from supplier nodes.
      */
     @Nullable public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      @Nullable GridDhtPartitionsExchangeFuture exchFut);
@@ -74,10 +82,10 @@ public interface GridCachePreloader {
      * Adds assignments to preloader.
      *
      * @param assignments Assignments to add.
-     * @param forcePreload Force preload flag.
-     * @param rebalanceId Rebalance id.
-     * @param next Runnable responsible for cache rebalancing start.
-     * @param forcedRebFut Rebalance future.
+     * @param forcePreload {@code True} if preload requested by {@link ForceRebalanceExchangeTask}.
+     * @param rebalanceId Rebalance id created by exchange thread.
+     * @param next Runnable responsible for cache rebalancing chain.
+     * @param forcedRebFut External future for forced rebalance.
      * @return Rebalancing runnable.
      */
     public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
@@ -114,7 +122,6 @@ public Runnable addAssignments(GridDhtPreloaderAssignments assignments,
      * Future result is {@code false} in case rebalancing cancelled or finished with missed partitions and will be
      * restarted at current or pending topology.
      *
-     * Note that topology change creates new futures and finishes previous.
      */
     public IgniteInternalFuture<Boolean> rebalanceFuture();
 
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
index 18311b9b1ca20..f16305cd85456 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePreloaderAdapter.java
@@ -151,6 +151,12 @@ public GridCachePreloaderAdapter(CacheGroupContext grp) {
         // No-op.
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
+        GridDhtPartitionsExchangeFuture exchFut) {
+        return true;
+    }
+
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId,
                                                                      GridDhtPartitionsExchangeFuture exchFut) {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
index 116a28a07d551..f01369f5714cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionDemander.java
@@ -235,12 +235,12 @@ else if (log.isDebugEnabled())
 
     /**
      * @param fut Future.
-     * @return {@code True} if topology changed.
+     * @return {@code True} if rebalance topology version changed by exchange thread or force
+     * reassing exchange occurs, see {@link RebalanceReassignExchangeTask} for details.
      */
     private boolean topologyChanged(RebalanceFuture fut) {
-        return
-            !grp.affinity().lastVersion().equals(fut.topologyVersion()) || // Topology already changed.
-                fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
+        return !ctx.exchange().rebalanceTopologyVersion().equals(fut.topVer) ||
+            fut != rebalanceFut; // Same topology, but dummy exchange forced because of missing partitions.
     }
 
     /**
@@ -253,14 +253,21 @@ void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
     }
 
     /**
-     * Initiates new rebalance process from given {@code assignments}.
-     * If previous rebalance is not finished method cancels it.
-     * In case of delayed rebalance method schedules new with configured delay.
+     * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
+     */
+    Collection<UUID> remainingNodes() {
+        return rebalanceFut.remainingNodes();
+    }
+
+    /**
+     * This method initiates new rebalance process from given {@code assignments} by creating new rebalance
+     * future based on them. Cancels previous rebalance future and sends rebalance started event.
+     * In case of delayed rebalance method schedules the new one with configured delay based on {@code lastExchangeFut}.
      *
-     * @param assignments Assignments.
-     * @param force {@code True} if dummy reassign.
-     * @param rebalanceId Rebalance id.
-     * @param next Runnable responsible for cache rebalancing start.
+     * @param assignments Assignments to process.
+     * @param force {@code True} if preload request by {@link ForceRebalanceExchangeTask}.
+     * @param rebalanceId Rebalance id generated from exchange thread.
+     * @param next Runnable responsible for cache rebalancing chain.
      * @param forcedRebFut External future for forced rebalance.
      * @return Rebalancing runnable.
      */
@@ -440,17 +447,7 @@ private void requestPartitions(final RebalanceFuture fut, GridDhtPreloaderAssign
             if (fut.isDone())
                 return;
 
-            // Must add all remaining node before send first request, for avoid race between add remaining node and
-            // processing response, see checkIsDone(boolean).
-            for (Map.Entry<ClusterNode, GridDhtPartitionDemandMessage> e : assignments.entrySet()) {
-                UUID nodeId = e.getKey().id();
-
-                IgniteDhtDemandedPartitionsMap parts = e.getValue().partitions();
-
-                assert parts != null : "Partitions are null [grp=" + grp.cacheOrGroupName() + ", fromNode=" + nodeId + "]";
-
-                fut.remaining.put(nodeId, new T2<>(U.currentTimeMillis(), parts));
-            }
+            fut.remaining.forEach((key, value) -> value.set1(U.currentTimeMillis()));
         }
 
         final CacheConfiguration cfg = grp.config();
@@ -1270,6 +1267,13 @@ private void checkIsDone(boolean cancelled) {
             }
         }
 
+        /**
+         * @return Collection of supplier nodes. Value {@code empty} means rebalance already finished.
+         */
+        private synchronized Collection<UUID> remainingNodes() {
+            return remaining.keySet();
+        }
+
         /**
          *
          */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 3b2a03f91c22b..5d9cee99630ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -17,12 +17,14 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.dht.preloader;
 
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.IgniteSystemProperties;
@@ -125,19 +127,20 @@ private static void clearContext(SupplyContext sc, IgniteLogger log) {
     }
 
     /**
-     * Handles new topology version and clears supply context map of outdated contexts.
-     *
-     * @param topVer Topology version.
+     * Handle topology change and clear supply context map of outdated contexts.
      */
-    @SuppressWarnings("ConstantConditions")
-    void onTopologyChanged(AffinityTopologyVersion topVer) {
+    void onTopologyChanged() {
         synchronized (scMap) {
             Iterator<T3<UUID, Integer, AffinityTopologyVersion>> it = scMap.keySet().iterator();
 
+            Collection<UUID> aliveNodes = grp.shared().discovery().aliveServerNodes().stream()
+                .map(ClusterNode::id)
+                .collect(Collectors.toList());
+
             while (it.hasNext()) {
                 T3<UUID, Integer, AffinityTopologyVersion> t = it.next();
 
-                if (topVer.compareTo(t.get3()) > 0) { // Clear all obsolete contexts.
+                if (!aliveNodes.contains(t.get1())) { // Clear all obsolete contexts.
                     clearContext(scMap.get(t), log);
 
                     it.remove();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
index e60eb1fa7140d..b1d9cd48f8163 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloader.java
@@ -25,6 +25,7 @@
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -40,7 +41,6 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest;
-import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
 import org.apache.ignite.internal.util.future.GridCompoundFuture;
 import org.apache.ignite.internal.util.future.GridFinishedFuture;
@@ -168,11 +168,67 @@ private IgniteCheckedException stopError() {
 
     /** {@inheritDoc} */
     @Override public void onTopologyChanged(GridDhtPartitionsExchangeFuture lastFut) {
-        supplier.onTopologyChanged(lastFut.initialVersion());
+        supplier.onTopologyChanged();
 
         demander.onTopologyChanged(lastFut);
     }
 
+    /** {@inheritDoc} */
+    @Override public boolean rebalanceRequired(AffinityTopologyVersion rebTopVer,
+        GridDhtPartitionsExchangeFuture exchFut) {
+        if (ctx.kernalContext().clientNode() || rebTopVer.equals(AffinityTopologyVersion.NONE))
+            return false; // No-op.
+
+        if (exchFut.localJoinExchange())
+            return true; // Required, can have outdated updSeq partition counter if node reconnects.
+
+        if (!grp.affinity().cachedVersions().contains(rebTopVer)) {
+            assert rebTopVer.compareTo(grp.localStartVersion()) <= 0 :
+                "Empty hisroty allowed only for newly started cache group [rebTopVer=" + rebTopVer +
+                    ", localStartTopVer=" + grp.localStartVersion() + ']';
+
+            return true; // Required, since no history info available.
+        }
+
+        final IgniteInternalFuture<Boolean> rebFut = rebalanceFuture();
+
+        if (rebFut.isDone() && !rebFut.result())
+            return true; // Required, previous rebalance cancelled.
+
+        final AffinityTopologyVersion exchTopVer = exchFut.context().events().topologyVersion();
+
+        Collection<UUID> aliveNodes = ctx.discovery().aliveServerNodes().stream()
+            .map(ClusterNode::id)
+            .collect(Collectors.toList());
+
+        return assignmentsChanged(rebTopVer, exchTopVer) ||
+            !aliveNodes.containsAll(demander.remainingNodes()); // Some of nodes left before rabalance compelete.
+    }
+
+    /**
+     * @param oldTopVer Previous topology version.
+     * @param newTopVer New topology version to check result.
+     * @return {@code True} if affinity assignments changed between two versions for local node.
+     */
+    private boolean assignmentsChanged(AffinityTopologyVersion oldTopVer, AffinityTopologyVersion newTopVer) {
+        final AffinityAssignment aff = grp.affinity().readyAffinity(newTopVer);
+
+        // We should get affinity assignments based on previous rebalance to calculate difference.
+        // Whole history size described by IGNITE_AFFINITY_HISTORY_SIZE constant.
+        final AffinityAssignment prevAff = grp.affinity().cachedVersions().contains(oldTopVer) ?
+            grp.affinity().cachedAffinity(oldTopVer) : null;
+
+        if (prevAff == null)
+            return false;
+
+        boolean assignsChanged = false;
+
+        for (int p = 0; !assignsChanged && p < grp.affinity().partitions(); p++)
+            assignsChanged |= aff.get(p).contains(ctx.localNode()) != prevAff.get(p).contains(ctx.localNode());
+
+        return assignsChanged;
+    }
+
     /** {@inheritDoc} */
     @Override public GridDhtPreloaderAssignments generateAssignments(GridDhtPartitionExchangeId exchId, GridDhtPartitionsExchangeFuture exchFut) {
         assert exchFut == null || exchFut.isDone();
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
index 69dc79b5007c2..6f988890c6fb3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPreloaderAssignments.java
@@ -20,7 +20,6 @@
 import java.util.concurrent.ConcurrentHashMap;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl;
 import org.apache.ignite.internal.util.typedef.internal.S;
 
@@ -74,9 +73,9 @@ GridDhtPartitionExchangeId exchangeId() {
     }
 
     /**
-     * @return Topology version.
+     * @return Topology version based on last {@link GridDhtPartitionTopologyImpl#readyTopVer}.
      */
-    AffinityTopologyVersion topologyVersion() {
+    public AffinityTopologyVersion topologyVersion() {
         return topVer;
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
index 565317720d2a9..46b09ac3b1819 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterBaselineNodesMetricsSelfTest.java
@@ -149,7 +149,6 @@ public void testBaselineNodes() throws Exception {
     private void resetBlt() throws Exception {
         resetBaselineTopology();
 
-        waitForRebalancing();
         awaitPartitionMapExchange();
     }
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
index 5c601a1bd3bbf..0b441dd94c0e8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheValidatorMetricsTest.java
@@ -103,14 +103,14 @@ public void testCacheValidatorMetrics() throws Exception {
 
         startGrid(2);
 
-        waitForRebalancing();
+        awaitPartitionMapExchange();
 
         assertCacheStatus(CACHE_NAME_1, true, true);
         assertCacheStatus(CACHE_NAME_2, true, true);
 
         stopGrid(1);
 
-        waitForRebalancing();
+        awaitPartitionMapExchange();
 
         // Invalid for writing due to invalid topology.
         assertCacheStatus(CACHE_NAME_1, true, false);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
index 7b5cc3e8101dc..82ed95e52531f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridCacheDhtPreloadSelfTest.java
@@ -22,20 +22,17 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CachePeekMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cache.affinity.Affinity;
 import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.events.CacheRebalancingEvent;
 import org.apache.ignite.events.Event;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionFullMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
@@ -44,10 +41,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
 import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology;
 import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.P1;
-import org.apache.ignite.internal.util.typedef.X;
 import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteFuture;
 import org.apache.ignite.lang.IgnitePredicate;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
@@ -61,9 +55,6 @@
 import static org.apache.ignite.configuration.CacheConfiguration.DFLT_REBALANCE_BATCH_SIZE;
 import static org.apache.ignite.configuration.DeploymentMode.CONTINUOUS;
 import static org.apache.ignite.events.EventType.EVTS_CACHE_REBALANCE;
-import static org.apache.ignite.events.EventType.EVT_CACHE_REBALANCE_STOPPED;
-import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
-import static org.apache.ignite.events.EventType.EVT_NODE_LEFT;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.MOVING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.RENTING;
@@ -144,15 +135,6 @@ protected CacheConfiguration cacheConfiguration(String igniteInstanceName) {
         return cacheCfg;
     }
 
-    /** {@inheritDoc} */
-    @Override protected void beforeTestsStarted() throws Exception {
-//        resetLog4j(Level.DEBUG, true,
-//            // Categories.
-//            GridDhtPreloader.class.getPackage().getName(),
-//            GridDhtPartitionTopologyImpl.class.getName(),
-//            GridDhtLocalPartition.class.getName());
-    }
-
     /** {@inheritDoc} */
     @Override protected void beforeTest() throws Exception {
         backups = DFLT_BACKUPS;
@@ -230,11 +212,6 @@ public void testActivePartitionTransferAsyncRandomCoordinator() throws Exception
      */
     private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
-//        resetLog4j(Level.DEBUG, true,
-//            // Categories.
-//            GridDhtPreloader.class.getPackage().getName(),
-//            GridDhtPartitionTopologyImpl.class.getName(),
-//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -273,8 +250,6 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
             info(">>> Finished checking nodes [keyCnt=" + keyCnt + ", nodeCnt=" + nodeCnt + ", grids=" +
                 U.grids2names(ignites) + ']');
 
-            Collection<IgniteFuture<?>> futs = new LinkedList<>();
-
             Ignite last = F.last(ignites);
 
             for (Iterator<Ignite> it = ignites.iterator(); it.hasNext(); ) {
@@ -288,21 +263,8 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
 
                 checkActiveState(ignites);
 
-                final UUID nodeId = g.cluster().localNode().id();
-
                 it.remove();
 
-                futs.add(waitForLocalEvent(last.events(), new P1<Event>() {
-                    @Override public boolean apply(Event e) {
-                        CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
-
-                        ClusterNode node = evt.discoveryNode();
-
-                        return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
-                            (evt.discoveryEventType() == EVT_NODE_LEFT || evt.discoveryEventType() == EVT_NODE_FAILED);
-                    }
-                }, EVT_CACHE_REBALANCE_STOPPED));
-
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
@@ -315,14 +277,6 @@ private void checkActivePartitionTransfer(int keyCnt, int nodeCnt, boolean sameC
                 awaitPartitionMapExchange(); // Need wait, otherwise test logic is broken if EVT_NODE_FAILED exchanges are merged.
             }
 
-            info("Waiting for preload futures: " + F.view(futs, new IgnitePredicate<IgniteFuture<?>>() {
-                @Override public boolean apply(IgniteFuture<?> fut) {
-                    return !fut.isDone();
-                }
-            }));
-
-            X.waitAll(futs);
-
             info("Finished waiting for preload futures.");
 
             assert last != null;
@@ -502,11 +456,6 @@ private void stopGrids(Iterable<Ignite> grids) {
      */
     private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuffle)
         throws Exception {
-//        resetLog4j(Level.DEBUG, true,
-//            // Categories.
-//            GridDhtPreloader.class.getPackage().getName(),
-//            GridDhtPartitionTopologyImpl.class.getName(),
-//            GridDhtLocalPartition.class.getName());
 
         try {
             Ignite ignite1 = startGrid(0);
@@ -558,28 +507,13 @@ private void checkNodes(int keyCnt, int nodeCnt, boolean sameCoord, boolean shuf
 
                 it.remove();
 
-                Collection<IgniteFuture<?>> futs = new LinkedList<>();
-
-                for (Ignite gg : ignites)
-                    futs.add(waitForLocalEvent(gg.events(), new P1<Event>() {
-                        @Override public boolean apply(Event e) {
-                            CacheRebalancingEvent evt = (CacheRebalancingEvent)e;
-
-                            ClusterNode node = evt.discoveryNode();
-
-                            return evt.type() == EVT_CACHE_REBALANCE_STOPPED && node.id().equals(nodeId) &&
-                                (evt.discoveryEventType() == EVT_NODE_LEFT ||
-                                    evt.discoveryEventType() == EVT_NODE_FAILED);
-                        }
-                    }, EVT_CACHE_REBALANCE_STOPPED));
-
                 info("Before grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
                 stopGrid(g.name());
 
                 info(">>> Waiting for preload futures [leftNode=" + g.name() + ", remaining=" + U.grids2names(ignites) + ']');
 
-                X.waitAll(futs);
+                awaitPartitionMapExchange();
 
                 info("After grid stop [name=" + g.name() + ", fullTop=" + top2string(ignites));
 
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
index 4ebcd5df35271..0a8698a597cd8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingAsyncSelfTest.java
@@ -17,10 +17,11 @@
 
 package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
 
-import org.apache.ignite.Ignite;
+import java.util.Collections;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.configuration.CacheConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.spi.discovery.tcp.TestTcpDiscoverySpi;
@@ -43,7 +44,7 @@ public class GridCacheRebalancingAsyncSelfTest extends GridCacheRebalancingSyncS
      * @throws Exception Exception.
      */
     public void testNodeFailedAtRebalancing() throws Exception {
-        Ignite ignite = startGrid(0);
+        IgniteEx ignite = startGrid(0);
 
         generateData(ignite, 0, 0);
 
@@ -60,7 +61,7 @@ public void testNodeFailedAtRebalancing() throws Exception {
 
         ((TestTcpDiscoverySpi)grid(1).configuration().getDiscoverySpi()).simulateNodeFailure();
 
-        waitForRebalancing(0, 3);
+        awaitPartitionMapExchange(false, false, Collections.singletonList(ignite.localNode()));
 
         checkSupplyContextMapIsEmpty();
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
new file mode 100644
index 0000000000000..3965290480a3d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingCancelTest.java
@@ -0,0 +1,106 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.rebalancing;
+
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.TestRecordingCommunicationSpi;
+import org.apache.ignite.internal.processors.cache.GridCacheGroupIdMessage;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionDemander;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplyMessage;
+import org.apache.ignite.lang.IgniteBiPredicate;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test cases for checking cancellation rebalancing process if some events occurs.
+ */
+public class GridCacheRebalancingCancelTest extends GridCommonAbstractTest {
+    /** */
+    private static final String DHT_PARTITIONED_CACHE = "cacheP";
+
+    /** */
+    private static TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+    /** {@inheritDoc} */
+    @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+        IgniteConfiguration dfltCfg = super.getConfiguration(igniteInstanceName);
+
+        ((TcpDiscoverySpi)dfltCfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+        dfltCfg.setCommunicationSpi(new TestRecordingCommunicationSpi());
+
+        return dfltCfg;
+    }
+
+    /**
+     * Test rebalance not cancelled when client node join to cluster.
+     *
+     * @throws Exception Exception.
+     */
+    public void testClientNodeJoinAtRebalancing() throws Exception {
+        final IgniteEx ignite0 = startGrid(0);
+
+        IgniteCache<Integer, Integer> cache = ignite0.createCache(
+            new CacheConfiguration<Integer, Integer>(DHT_PARTITIONED_CACHE)
+                .setCacheMode(CacheMode.PARTITIONED)
+                .setRebalanceMode(CacheRebalanceMode.ASYNC)
+                .setBackups(1)
+                .setRebalanceOrder(2)
+                .setAffinity(new RendezvousAffinityFunction(false)));
+
+        for (int i = 0; i < 2048; i++)
+            cache.put(i, i);
+
+        TestRecordingCommunicationSpi.spi(ignite0)
+            .blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+                @Override public boolean apply(ClusterNode node, Message msg) {
+                    return (msg instanceof GridDhtPartitionSupplyMessage)
+                        && ((GridCacheGroupIdMessage)msg).groupId() == groupIdForCache(ignite0, DHT_PARTITIONED_CACHE);
+                }
+            });
+
+        final IgniteEx ignite1 = startGrid(1);
+
+        TestRecordingCommunicationSpi.spi(ignite0).waitForBlocked();
+
+        GridDhtPartitionDemander.RebalanceFuture fut = (GridDhtPartitionDemander.RebalanceFuture)ignite1.context().
+            cache().internalCache(DHT_PARTITIONED_CACHE).preloader().rebalanceFuture();
+
+        String igniteClntName = getTestIgniteInstanceName(2);
+
+        startGrid(igniteClntName, optimize(getConfiguration(igniteClntName).setClientMode(true)));
+
+        // Resend delayed rebalance messages.
+        TestRecordingCommunicationSpi.spi(ignite0).stopBlock(true);
+
+        awaitPartitionMapExchange();
+
+        // Previous rebalance future should not be cancelled.
+        assertTrue(fut.result());
+    }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
index fa4b655fb41bd..258e36e531322 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingPartitionCountersTest.java
@@ -141,7 +141,8 @@ else if ("index.bin".equals(f.getName())) {
         assertTrue(primaryRemoved);
 
         ignite.cluster().active(true);
-        waitForRebalancing();
+
+        awaitPartitionMapExchange();
 
         List<String> issues = new ArrayList<>();
         HashMap<Integer, Long> partMap = new HashMap<>();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
index 4d03d7dc7e281..8b88befee5281 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/rebalancing/GridCacheRebalancingSyncSelfTest.java
@@ -21,10 +21,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.ignite.Ignite;
 import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cache.CacheMode;
 import org.apache.ignite.cache.CacheRebalanceMode;
 import org.apache.ignite.cluster.ClusterNode;
@@ -42,11 +41,13 @@
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
 import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsSingleMessage;
+import org.apache.ignite.internal.util.lang.GridAbsPredicateX;
 import org.apache.ignite.internal.util.typedef.G;
 import org.apache.ignite.internal.util.typedef.PA;
 import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteInClosure;
 import org.apache.ignite.plugin.extensions.communication.Message;
+import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
 import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -68,6 +69,9 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private static final int TEST_SIZE = 100_000;
 
+    /** */
+    private static final long TOPOLOGY_STILLNESS_TIME = 30_000L;
+
     /** partitioned cache name. */
     protected static final String CACHE_NAME_DHT_PARTITIONED = "cacheP";
 
@@ -89,11 +93,11 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
     /** */
     private volatile boolean concurrentStartFinished3;
 
-    /** */
-    private volatile boolean record;
-
-    /** */
-    private final ConcurrentHashMap<Class, AtomicInteger> map = new ConcurrentHashMap<>();
+    /**
+     * Time in milliseconds of last received {@link GridDhtPartitionsSingleMessage}
+     * or {@link GridDhtPartitionsFullMessage} using {@link CollectingCommunicationSpi}.
+     */
+    private static volatile long lastPartMsgTime;
 
     /** {@inheritDoc} */
     @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
@@ -102,7 +106,7 @@ public class GridCacheRebalancingSyncSelfTest extends GridCommonAbstractTest {
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setIpFinder(ipFinder);
         ((TcpDiscoverySpi)iCfg.getDiscoverySpi()).setForceServerMode(true);
 
-        TcpCommunicationSpi commSpi = new CountingCommunicationSpi();
+        TcpCommunicationSpi commSpi = new CollectingCommunicationSpi();
 
         commSpi.setLocalPort(GridTestUtils.getNextCommPort(getClass()));
         commSpi.setTcpNoDelay(true);
@@ -232,47 +236,35 @@ public void testSimpleRebalancing() throws Exception {
 
         startGrid(1);
 
-        int waitMinorVer = ignite.configuration().isLateAffinityAssignment() ? 1 : 0;
-
-        waitForRebalancing(0, 2, waitMinorVer);
-        waitForRebalancing(1, 2, waitMinorVer);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         stopGrid(0);
 
-        waitForRebalancing(1, 3);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         startGrid(2);
 
-        waitForRebalancing(1, 4, waitMinorVer);
-        waitForRebalancing(2, 4, waitMinorVer);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         stopGrid(2);
 
-        waitForRebalancing(1, 5);
-
         awaitPartitionMapExchange(true, true, null, true);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         long spend = (System.currentTimeMillis() - start) / 1000;
 
@@ -331,13 +323,10 @@ public void testLoadRebalancing() throws Exception {
 
         startGrid(4);
 
-        waitForRebalancing(3, 6);
-        waitForRebalancing(4, 6);
+        awaitPartitionMapExchange(true, true, null);
 
         concurrentStartFinished = true;
 
-        awaitPartitionMapExchange(true, true, null);
-
         checkSupplyContextMapIsEmpty();
 
         t1.join();
@@ -442,27 +431,29 @@ protected void checkPartitionMapExchangeFinished() {
     }
 
     /**
+     * Method checks for {@link GridDhtPartitionsSingleMessage} or {@link GridDhtPartitionsFullMessage}
+     * not received within {@code TOPOLOGY_STILLNESS_TIME} bound.
+     *
      * @throws Exception If failed.
      */
-    protected void checkPartitionMapMessagesAbsent() throws Exception {
-        map.clear();
-
-        record = true;
-
-        log.info("Checking GridDhtPartitions*Message absent (it will take 30 SECONDS) ... ");
-
-        U.sleep(30_000);
-
-        record = false;
-
-        AtomicInteger iF = map.get(GridDhtPartitionsFullMessage.class);
-        AtomicInteger iS = map.get(GridDhtPartitionsSingleMessage.class);
-
-        Integer fullMap = iF != null ? iF.get() : null;
-        Integer singleMap = iS != null ? iS.get() : null;
-
-        assertTrue("Unexpected full map messages: " + fullMap, fullMap == null || fullMap.equals(1)); // 1 message can be sent right after all checks passed.
-        assertNull("Unexpected single map messages", singleMap);
+    protected void awaitPartitionMessagesAbsent() throws Exception {
+        log.info("Checking GridDhtPartitions*Message absent (it will take up to " +
+            TOPOLOGY_STILLNESS_TIME + " ms) ... ");
+
+        // Start waiting new messages from current point of time.
+        lastPartMsgTime = U.currentTimeMillis();
+
+        assertTrue("Should not have partition Single or Full messages within bound " +
+                TOPOLOGY_STILLNESS_TIME + " ms.",
+            GridTestUtils.waitForCondition(
+                new GridAbsPredicateX() {
+                    @Override public boolean applyx() {
+                        return lastPartMsgTime + TOPOLOGY_STILLNESS_TIME < U.currentTimeMillis();
+                    }
+                },
+                2 * TOPOLOGY_STILLNESS_TIME // 30 sec to gain stable topology and 30 sec of silence.
+            )
+        );
     }
 
     /** {@inheritDoc} */
@@ -495,11 +486,7 @@ public void testComplexRebalancing() throws Exception {
                     while (!concurrentStartFinished2)
                         U.sleep(10);
 
-                    waitForRebalancing(0, 5, 0);
-                    waitForRebalancing(1, 5, 0);
-                    waitForRebalancing(2, 5, 0);
-                    waitForRebalancing(3, 5, 0);
-                    waitForRebalancing(4, 5, 0);
+                    awaitPartitionMapExchange();
 
                     //New cache should start rebalancing.
                     CacheConfiguration<Integer, Integer> cacheRCfg = new CacheConfiguration<>(DEFAULT_CACHE_NAME);
@@ -552,12 +539,6 @@ public void testComplexRebalancing() throws Exception {
         t2.join();
         t3.join();
 
-        waitForRebalancing(0, 5, 1);
-        waitForRebalancing(1, 5, 1);
-        waitForRebalancing(2, 5, 1);
-        waitForRebalancing(3, 5, 1);
-        waitForRebalancing(4, 5, 1);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
@@ -577,35 +558,23 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(1);
 
-        waitForRebalancing(0, 6);
-        waitForRebalancing(2, 6);
-        waitForRebalancing(3, 6);
-        waitForRebalancing(4, 6);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(0);
 
-        waitForRebalancing(2, 7);
-        waitForRebalancing(3, 7);
-        waitForRebalancing(4, 7);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkSupplyContextMapIsEmpty();
 
         stopGrid(2);
 
-        waitForRebalancing(3, 8);
-        waitForRebalancing(4, 8);
-
         awaitPartitionMapExchange(true, true, null);
 
         checkPartitionMapExchangeFinished();
 
-        checkPartitionMapMessagesAbsent();
+        awaitPartitionMessagesAbsent();
 
         checkSupplyContextMapIsEmpty();
 
@@ -613,7 +582,7 @@ public void testComplexRebalancing() throws Exception {
 
         stopGrid(3);
 
-        waitForRebalancing(4, 9);
+        awaitPartitionMapExchange();
 
         checkSupplyContextMapIsEmpty();
 
@@ -634,36 +603,26 @@ public void testComplexRebalancing() throws Exception {
     /**
      *
      */
-    private class CountingCommunicationSpi extends TcpCommunicationSpi {
+    private static class CollectingCommunicationSpi extends TcpCommunicationSpi {
+        /** */
+        @LoggerResource
+        private IgniteLogger log;
+
         /** {@inheritDoc} */
         @Override public void sendMessage(final ClusterNode node, final Message msg,
             final IgniteInClosure<IgniteException> ackC) throws IgniteSpiException {
             final Object msg0 = ((GridIoMessage)msg).message();
 
-            recordMessage(msg0);
+            if (msg0 instanceof GridDhtPartitionsSingleMessage ||
+                msg0 instanceof GridDhtPartitionsFullMessage) {
+                lastPartMsgTime = U.currentTimeMillis();
 
-            super.sendMessage(node, msg, ackC);
-        }
-
-        /**
-         * @param msg Message.
-         */
-        private void recordMessage(Object msg) {
-            if (record) {
-                Class id = msg.getClass();
-
-                AtomicInteger ai = map.get(id);
-
-                if (ai == null) {
-                    ai = new AtomicInteger();
-
-                    AtomicInteger oldAi = map.putIfAbsent(id, ai);
-
-                    (oldAi != null ? oldAi : ai).incrementAndGet();
-                }
-                else
-                    ai.incrementAndGet();
+                log.info("Last seen time of GridDhtPartitionsSingleMessage or GridDhtPartitionsFullMessage updated " +
+                    "[lastPartMsgTime=" + lastPartMsgTime +
+                    ", node=" + node.id() + ']');
             }
+
+            super.sendMessage(node, msg, ackC);
         }
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
index 15ec41557dfba..ce84156e56dca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/baseline/ClientAffinityAssignmentWithBaselineTest.java
@@ -383,7 +383,7 @@ public void testRejoinWithCleanLfs() throws Exception {
         startGrid("flaky");
 
         System.out.println("### Starting rebalancing after flaky node join");
-        waitForRebalancing();
+        awaitPartitionMapExchange();
         System.out.println("### Rebalancing is finished after flaky node join");
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
@@ -689,7 +689,7 @@ private void tryChangeBaselineDown(
         ig0.cluster().setBaselineTopology(fullBlt.subList(0, newBaselineSize));
 
         System.out.println("### Starting rebalancing after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
-        waitForRebalancing();
+        awaitPartitionMapExchange();
         System.out.println("### Rebalancing is finished after BLT change: " + (newBaselineSize + 1) + " -> " + newBaselineSize);
 
         awaitProgressInAllLoaders(10_000, loadError, threadProgressTracker);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
index e6ec8a37f6538..bd1a1a9cc3575 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/SlowHistoricalRebalanceSmallHistoryTest.java
@@ -162,7 +162,8 @@ public void testReservation() throws Exception {
 
         SUPPLY_MESSAGE_LATCH.get().countDown();
 
-        waitForRebalancing(); // Partition is OWNING on grid(0) and grid(1)
+        // Partition is OWNING on grid(0) and grid(1)
+        awaitPartitionMapExchange();
 
         for (int i = 0; i < 2; i++) {
             for (int j = 0; i < 500; i++)
@@ -183,7 +184,7 @@ public void testReservation() throws Exception {
 
         startGrid(0);
 
-        waitForRebalancing();
+        awaitPartitionMapExchange();
 
         assertEquals(2, grid(1).context().discovery().aliveServerNodes().size());
     }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
index a24e6d5e0d738..2a69919142ded 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushMultiNodeFailoverAbstractSelfTest.java
@@ -49,10 +49,6 @@
 import org.apache.ignite.transactions.TransactionConcurrency;
 import org.apache.ignite.transactions.TransactionIsolation;
 
-import static java.nio.file.StandardOpenOption.CREATE;
-import static java.nio.file.StandardOpenOption.READ;
-import static java.nio.file.StandardOpenOption.WRITE;
-
 /**
  * Tests error recovery while node flushing
  */
@@ -192,7 +188,7 @@ public void failWhilePut(boolean failWhileStart) throws Exception {
 
                     grid.cluster().setBaselineTopology(grid.cluster().topologyVersion());
 
-                    waitForRebalancing();
+                    awaitPartitionMapExchange();
                 }
                 catch (Throwable expected) {
                     // There can be any exception. Do nothing.
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index efb4fd511c4aa..ba39e366e9f85 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -120,7 +120,8 @@ public void testMappingsPersistedOnJoin() throws Exception {
         c1.put(k, new DummyObject(k));
 
         startGrid(2);
-        waitForRebalancing();
+
+        awaitPartitionMapExchange();
 
         stopAllGrids();
 
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
index 55436d6da943b..5e94052f25013 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite3.java
@@ -53,6 +53,7 @@
 import org.apache.ignite.internal.processors.cache.distributed.near.IgniteTxReentryNearSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRabalancingDelayedPartitionMapExchangeSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingAsyncSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingCancelTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncCheckDataTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingSyncSelfTest;
 import org.apache.ignite.internal.processors.cache.distributed.rebalancing.GridCacheRebalancingUnmarshallingFailedSelfTest;
@@ -152,6 +153,7 @@ public static TestSuite suite() throws Exception {
         suite.addTestSuite(GridCacheRebalancingUnmarshallingFailedSelfTest.class);
         suite.addTestSuite(GridCacheRebalancingAsyncSelfTest.class);
         suite.addTestSuite(GridCacheRabalancingDelayedPartitionMapExchangeSelfTest.class);
+        suite.addTestSuite(GridCacheRebalancingCancelTest.class);
 
         // Test for byte array value special case.
         suite.addTestSuite(GridCacheLocalByteArrayValuesSelfTest.class);